diff --git a/.clang-format b/.clang-format
index 2ceca764122f..c7060124a47a 100644
--- a/.clang-format
+++ b/.clang-format
@@ -748,6 +748,7 @@ ForEachMacros:
   - 'ynl_attr_for_each_nested'
   - 'ynl_attr_for_each_payload'
   - 'zorro_for_each_dev'
+  - 'zpci_bus_for_each'
 
 IncludeBlocks: Preserve
 IncludeCategories:
diff --git a/.editorconfig b/.editorconfig
index 29a30ccfc07b..b5ea32b6954b 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -1,7 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0-only
 
-root = true
-
 [{*.{awk,c,dts,dtsi,dtso,h,mk,s,S},Kconfig,Makefile,Makefile.*}]
 charset = utf-8
 end_of_line = lf
diff --git a/.mailmap b/.mailmap
index 84309a39d329..7fa74e2de40a 100644
--- a/.mailmap
+++ b/.mailmap
@@ -12,6 +12,7 @@
 #
 Aaron Durbin <adurbin@google.com>
 Abel Vesa <abelvesa@kernel.org> <abel.vesa@nxp.com>
+Abel Vesa <abelvesa@kernel.org> <abel.vesa@linaro.org>
 Abel Vesa <abelvesa@kernel.org> <abelvesa@gmail.com>
 Abhijeet Dharmapurikar <quic_adharmap@quicinc.com> <adharmap@codeaurora.org>
 Abhinav Kumar <quic_abhinavk@quicinc.com> <abhinavk@codeaurora.org>
@@ -127,7 +128,8 @@ Barry Song <baohua@kernel.org> <Baohua.Song@csr.com>
 Barry Song <baohua@kernel.org> <barry.song@analog.com>
 Bart Van Assche <bvanassche@acm.org> <bart.vanassche@sandisk.com>
 Bart Van Assche <bvanassche@acm.org> <bart.vanassche@wdc.com>
-Bartosz Golaszewski <brgl@bgdev.pl> <bgolaszewski@baylibre.com>
+Bartosz Golaszewski <brgl@kernel.org> <bartosz.golaszewski@linaro.org>
+Bartosz Golaszewski <brgl@kernel.org> <bgolaszewski@baylibre.com>
 Ben Dooks <ben-linux@fluff.org> <ben.dooks@simtec.co.uk>
 Ben Dooks <ben-linux@fluff.org> <ben.dooks@sifive.com>
 Ben Gardner <bgardner@wabtec.com>
@@ -481,6 +483,7 @@ Lorenzo Pieralisi <lpieralisi@kernel.org> <lorenzo.pieralisi@arm.com>
 Lorenzo Stoakes <lorenzo.stoakes@oracle.com> <lstoakes@gmail.com>
 Luca Ceresoli <luca.ceresoli@bootlin.com> <luca@lucaceresoli.net>
 Luca Weiss <luca@lucaweiss.eu> <luca@z3ntu.xyz>
+Lucas De Marchi <demarchi@kernel.org> <lucas.demarchi@intel.com>
 Lukasz Luba <lukasz.luba@arm.com> <l.luba@partner.samsung.com>
 Luo Jie <quic_luoj@quicinc.com> <luoj@codeaurora.org>
 Lance Yang <lance.yang@linux.dev> <ioworker0@gmail.com>
@@ -857,7 +860,6 @@ Vladimir Davydov <vdavydov.dev@gmail.com> <vdavydov@parallels.com>
 Vladimir Davydov <vdavydov.dev@gmail.com> <vdavydov@virtuozzo.com>
 WangYuli <wangyuli@aosc.io> <wangyl5933@chinaunicom.cn>
 WangYuli <wangyuli@aosc.io> <wangyuli@deepin.org>
-WangYuli <wangyuli@aosc.io> <wangyuli@uniontech.com>
 Weiwen Hu <huweiwen@linux.alibaba.com> <sehuww@mail.scut.edu.cn>
 WeiXiong Liao <gmpy.liaowx@gmail.com> <liaoweixiong@allwinnertech.com>
 Wen Gong <quic_wgong@quicinc.com> <wgong@codeaurora.org>
diff --git a/CREDITS b/CREDITS
index b735b8c58102..ca75f110edb6 100644
--- a/CREDITS
+++ b/CREDITS
@@ -1987,6 +1987,7 @@ D: netfilter: TCP window tracking code
 D: netfilter: raw table
 D: netfilter: iprange match
 D: netfilter: new logging interfaces
+D: netfilter: ipset
 D: netfilter: various other hacks
 S: Tata
 S: Hungary
diff --git a/Documentation/ABI/testing/configfs-usb-gadget-midi b/Documentation/ABI/testing/configfs-usb-gadget-midi
index 07389cddd51a..d6bd67bb91fc 100644
--- a/Documentation/ABI/testing/configfs-usb-gadget-midi
+++ b/Documentation/ABI/testing/configfs-usb-gadget-midi
@@ -4,11 +4,12 @@ KernelVersion:	3.19
 Description:
 		The attributes:
 
-		==========	====================================
-		index		index value for the USB MIDI adapter
-		id		ID string for the USB MIDI adapter
-		buflen		MIDI buffer length
-		qlen		USB read request queue length
-		in_ports	number of MIDI input ports
-		out_ports	number of MIDI output ports
-		==========	====================================
+		================	====================================
+		index			index value for the USB MIDI adapter
+		id			ID string for the USB MIDI adapter
+		buflen			MIDI buffer length
+		qlen			USB read request queue length
+		in_ports		number of MIDI input ports
+		out_ports		number of MIDI output ports
+		interface_string	USB AudioControl interface string
+		================	====================================
diff --git a/Documentation/ABI/testing/sysfs-bus-coresight-devices-dummy-source b/Documentation/ABI/testing/sysfs-bus-coresight-devices-dummy-source
index 321e3ee1fc9d..c8c58914116e 100644
--- a/Documentation/ABI/testing/sysfs-bus-coresight-devices-dummy-source
+++ b/Documentation/ABI/testing/sysfs-bus-coresight-devices-dummy-source
@@ -1,7 +1,7 @@
 What:		/sys/bus/coresight/devices/dummy_source<N>/enable_source
 Date:		Dec 2024
 KernelVersion:	6.14
-Contact:	Mao Jinlong <quic_jinlmao@quicinc.com>
+Contact:	Mao Jinlong <jinlong.mao@oss.qualcomm.com>
 Description:	(RW) Enable/disable tracing of dummy source. A sink should be activated
 		before enabling the source. The path of coresight components linking
 		the source to the sink is configured and managed automatically by the
@@ -10,7 +10,7 @@ Description:	(RW) Enable/disable tracing of dummy source. A sink should be activ
 What:		/sys/bus/coresight/devices/dummy_source<N>/traceid
 Date:		Dec 2024
 KernelVersion:	6.14
-Contact:	Mao Jinlong <quic_jinlmao@quicinc.com>
+Contact:	Mao Jinlong <jinlong.mao@oss.qualcomm.com>
 Description:	(R) Show the trace ID that will appear in the trace stream
 		coming from this trace entity.
 
diff --git a/Documentation/ABI/testing/sysfs-bus-coresight-devices-tpdm b/Documentation/ABI/testing/sysfs-bus-coresight-devices-tpdm
index 98f1c6545027..f8016df64532 100644
--- a/Documentation/ABI/testing/sysfs-bus-coresight-devices-tpdm
+++ b/Documentation/ABI/testing/sysfs-bus-coresight-devices-tpdm
@@ -1,7 +1,7 @@
 What:		/sys/bus/coresight/devices/<tpdm-name>/integration_test
 Date:		January 2023
 KernelVersion:	6.2
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(Write) Run integration test for tpdm. Integration test
 		will generate test data for tpdm. It can help to make
@@ -15,7 +15,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/reset_dataset
 Date:		March 2023
 KernelVersion:	6.7
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(Write) Reset the dataset of the tpdm.
 
@@ -25,7 +25,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_trig_type
 Date:		March 2023
 KernelVersion:	6.7
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get the trigger type of the DSB for tpdm.
 
@@ -36,7 +36,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_trig_ts
 Date:		March 2023
 KernelVersion:	6.7
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get the trigger timestamp of the DSB for tpdm.
 
@@ -47,7 +47,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_mode
 Date:		March 2023
 KernelVersion:	6.7
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get the programming mode of the DSB for tpdm.
 
@@ -61,7 +61,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_edge/ctrl_idx
 Date:		March 2023
 KernelVersion:	6.7
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get the index number of the edge detection for the DSB
 		subunit TPDM. Since there are at most 256 edge detections, this
@@ -70,7 +70,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_edge/ctrl_val
 Date:		March 2023
 KernelVersion:	6.7
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		Write a data to control the edge detection corresponding to
 		the index number. Before writing data to this sysfs file,
@@ -86,7 +86,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_edge/ctrl_mask
 Date:		March 2023
 KernelVersion:	6.7
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		Write a data to mask the edge detection corresponding to the index
 		number. Before writing data to this sysfs file, "ctrl_idx" should
@@ -98,21 +98,21 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_edge/edcr[0:15]
 Date:		March 2023
 KernelVersion:	6.7
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		Read a set of the edge control value of the DSB in TPDM.
 
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_edge/edcmr[0:7]
 Date:		March 2023
 KernelVersion:	6.7
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		Read a set of the edge control mask of the DSB in TPDM.
 
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_trig_patt/xpr[0:7]
 Date:		March 2023
 KernelVersion:	6.7
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get the value of the trigger pattern for the DSB
 		subunit TPDM.
@@ -120,7 +120,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_trig_patt/xpmr[0:7]
 Date:		March 2023
 KernelVersion:	6.7
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get the mask of the trigger pattern for the DSB
 		subunit TPDM.
@@ -128,21 +128,21 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_patt/tpr[0:7]
 Date:		March 2023
 KernelVersion:	6.7
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get the value of the pattern for the DSB subunit TPDM.
 
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_patt/tpmr[0:7]
 Date:		March 2023
 KernelVersion:	6.7
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get the mask of the pattern for the DSB subunit TPDM.
 
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_patt/enable_ts
 Date:		March 2023
 KernelVersion:	6.7
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(Write) Set the pattern timestamp of DSB tpdm. Read
 		the pattern timestamp of DSB tpdm.
@@ -154,7 +154,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_patt/set_type
 Date:		March 2023
 KernelVersion:	6.7
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(Write) Set the pattern type of DSB tpdm. Read
 		the pattern type of DSB tpdm.
@@ -166,7 +166,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_msr/msr[0:31]
 Date:		March 2023
 KernelVersion:	6.7
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get the MSR(mux select register) for the DSB subunit
 		TPDM.
@@ -174,7 +174,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/cmb_mode
 Date:		January 2024
 KernelVersion:	6.9
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:	(Write) Set the data collection mode of CMB tpdm. Continuous
 		change creates CMB data set elements on every CMBCLK edge.
 		Trace-on-change creates CMB data set elements only when a new
@@ -188,7 +188,7 @@ Description:	(Write) Set the data collection mode of CMB tpdm. Continuous
 What:		/sys/bus/coresight/devices/<tpdm-name>/cmb_trig_patt/xpr[0:1]
 Date:		January 2024
 KernelVersion:	6.9
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get the value of the trigger pattern for the CMB
 		subunit TPDM.
@@ -196,7 +196,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/cmb_trig_patt/xpmr[0:1]
 Date:		January 2024
 KernelVersion:	6.9
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get the mask of the trigger pattern for the CMB
 		subunit TPDM.
@@ -204,21 +204,21 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_patt/tpr[0:1]
 Date:		January 2024
 KernelVersion:	6.9
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get the value of the pattern for the CMB subunit TPDM.
 
 What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_patt/tpmr[0:1]
 Date:		January 2024
 KernelVersion:	6.9
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get the mask of the pattern for the CMB subunit TPDM.
 
 What:		/sys/bus/coresight/devices/<tpdm-name>/cmb_patt/enable_ts
 Date:		January 2024
 KernelVersion:	6.9
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(Write) Set the pattern timestamp of CMB tpdm. Read
 		the pattern timestamp of CMB tpdm.
@@ -230,7 +230,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/cmb_trig_ts
 Date:		January 2024
 KernelVersion:	6.9
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get the trigger timestamp of the CMB for tpdm.
 
@@ -241,7 +241,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/cmb_ts_all
 Date:		January 2024
 KernelVersion:	6.9
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Read or write the status of timestamp upon all interface.
 		Only value 0 and 1  can be written to this node. Set this node to 1 to request
@@ -253,7 +253,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/cmb_msr/msr[0:31]
 Date:		January 2024
 KernelVersion:	6.9
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get the MSR(mux select register) for the CMB subunit
 		TPDM.
@@ -261,7 +261,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/mcmb_trig_lane
 Date:		Feb 2025
 KernelVersion	6.15
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get which lane participates in the output pattern
 		match cross trigger mechanism for the MCMB subunit TPDM.
@@ -269,7 +269,7 @@ Description:
 What:		/sys/bus/coresight/devices/<tpdm-name>/mcmb_lanes_select
 Date:		Feb 2025
 KernelVersion	6.15
-Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Contact:	Jinlong Mao <jinlong.mao@oss.qualcomm.com>, Tao Zhang <tao.zhang@oss.qualcomm.com>
 Description:
 		(RW) Set/Get the enablement of the individual lane.
 
diff --git a/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov b/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov
index 2fd7e9b7bacc..7f5ef9eada53 100644
--- a/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov
+++ b/Documentation/ABI/testing/sysfs-driver-intel-xe-sriov
@@ -119,7 +119,7 @@ Description:
 			The GT preemption timeout (PT) in [us] to be applied to all functions.
 			See sriov_admin/{pf,vf<N>}/profile/preempt_timeout_us for more details.
 
-		sched_priority: (RW/RO) string
+		sched_priority: (WO) string
 			The GT scheduling priority to be applied for all functions.
 			See sriov_admin/{pf,vf<N>}/profile/sched_priority for more details.
 
diff --git a/Documentation/admin-guide/blockdev/zoned_loop.rst b/Documentation/admin-guide/blockdev/zoned_loop.rst
index 806adde664db..6aa865424ac3 100644
--- a/Documentation/admin-guide/blockdev/zoned_loop.rst
+++ b/Documentation/admin-guide/blockdev/zoned_loop.rst
@@ -134,7 +134,7 @@ MB and a zone capacity of 63 MB::
 
         $ modprobe zloop
         $ mkdir -p /var/local/zloop/0
-        $ echo "add capacity_mb=2048,zone_size_mb=64,zone_capacity=63MB" > /dev/zloop-control
+        $ echo "add capacity_mb=2048,zone_size_mb=64,zone_capacity_mb=63" > /dev/zloop-control
 
 For the device created (/dev/zloop0), the zone backing files are all created
 under the default base directory (/var/local/zloop)::
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index a8d0afde7f85..3097e4266d76 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -3412,6 +3412,11 @@ Kernel parameters
 			* [no]logdir: Enable or disable access to the general
 			  purpose log directory.
 
+			* max_sec=<sectors>: Set the transfer size limit, in
+			  number of 512-byte sectors, to the value specified in
+			  <sectors>. The value specified in <sectors> has to be
+			  a non-zero positive integer.
+
 			* max_sec_128: Set transfer size limit to 128 sectors.
 
 			* max_sec_1024: Set or clear transfer size limit to
@@ -3437,8 +3442,6 @@ Kernel parameters
 			If there are multiple matching configurations changing
 			the same attribute, the last one is used.
 
-	load_ramdisk=	[RAM] [Deprecated]
-
 	lockd.nlm_grace_period=P  [NFS] Assign grace period.
 			Format: <integer>
 
@@ -4444,8 +4447,10 @@ Kernel parameters
 			Note that this argument takes precedence over
 			the CONFIG_RCU_NOCB_CPU_DEFAULT_ALL option.
 
-	noinitrd	[RAM] Tells the kernel not to load any configured
-			initial RAM disk.
+	noinitrd	[Deprecated,RAM] Tells the kernel not to load any configured
+			initial RAM disk. Currently this parameter applies to
+			initrd only, not to initramfs. But it applies to both
+			in EFI mode.
 
 	nointremap	[X86-64,Intel-IOMMU,EARLY] Do not enable interrupt
 			remapping.
@@ -5402,8 +5407,6 @@ Kernel parameters
 			Param: <number> - step/bucket size as a power of 2 for
 				statistical time based profiling.
 
-	prompt_ramdisk=	[RAM] [Deprecated]
-
 	prot_virt=	[S390] enable hosting protected virtual machines
 			isolated from the hypervisor (if hardware supports
 			that). If enabled, the default kernel base address
@@ -5460,7 +5463,7 @@ Kernel parameters
 	ramdisk_size=	[RAM] Sizes of RAM disks in kilobytes
 			See Documentation/admin-guide/blockdev/ramdisk.rst.
 
-	ramdisk_start=	[RAM] RAM disk image start address
+	ramdisk_start=	[Deprecated,RAM] RAM disk image start address
 
 	random.trust_cpu=off
 			[KNL,EARLY] Disable trusting the use of the CPU's
@@ -6571,6 +6574,14 @@ Kernel parameters
 			replacement properties are not found. See the Kconfig
 			entry for RISCV_ISA_FALLBACK.
 
+	riscv_nousercfi=
+		all	Disable user cfi ABI to userspace even if cpu extension
+			are available.
+		bcfi	Disable user backward cfi ABI to userspace even if
+			shadow stack extension is available.
+		fcfi	Disable user forward cfi ABI to userspace even if landing
+			pad extension is available.
+
 	ro		[KNL] Mount root device read-only on boot
 
 	rodata=		[KNL,EARLY]
@@ -6934,12 +6945,12 @@ Kernel parameters
 
 	softlockup_panic=
 			[KNL] Should the soft-lockup detector generate panics.
-			Format: 0 | 1
+			Format: <int>
 
-			A value of 1 instructs the soft-lockup detector
-			to panic the machine when a soft-lockup occurs. It is
-			also controlled by the kernel.softlockup_panic sysctl
-			and CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC, which is the
+			A value of non-zero instructs the soft-lockup detector
+			to panic the machine when a soft-lockup duration exceeds
+			N thresholds. It is also controlled by the kernel.softlockup_panic
+			sysctl and CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC, which is the
 			respective build-time switch to that functionality.
 
 	softlockup_all_cpu_backtrace=
diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst
index 239da22c4e28..7bb92cc13a04 100644
--- a/Documentation/admin-guide/sysctl/kernel.rst
+++ b/Documentation/admin-guide/sysctl/kernel.rst
@@ -418,7 +418,7 @@ hung_task_detect_count
 ======================
 
 Indicates the total number of tasks that have been detected as hung since
-the system boot.
+the system boot. The counter can be reset to zero when written to.
 
 This file shows up if ``CONFIG_DETECT_HUNG_TASK`` is enabled.
 
@@ -1235,12 +1235,6 @@ that support this feature.
 ==  ===========================================================================
 
 
-real-root-dev
-=============
-
-See Documentation/admin-guide/initrd.rst.
-
-
 reboot-cmd (SPARC only)
 =======================
 
diff --git a/Documentation/arch/riscv/hwprobe.rst b/Documentation/arch/riscv/hwprobe.rst
index 06c5280b728a..03484a2546da 100644
--- a/Documentation/arch/riscv/hwprobe.rst
+++ b/Documentation/arch/riscv/hwprobe.rst
@@ -281,6 +281,14 @@ The following keys are defined:
   * :c:macro:`RISCV_HWPROBE_EXT_ZICBOP`: The Zicbop extension is supported, as
        ratified in commit 3dd606f ("Create cmobase-v1.0.pdf") of riscv-CMOs.
 
+  * :c:macro:`RISCV_HWPROBE_EXT_ZILSD`: The Zilsd extension is supported as
+       defined in the RISC-V ISA manual starting from commit f88abf1 ("Integrating
+       load/store pair for RV32 with the main manual") of the riscv-isa-manual.
+
+  * :c:macro:`RISCV_HWPROBE_EXT_ZCLSD`: The Zclsd extension is supported as
+       defined in the RISC-V ISA manual starting from commit f88abf1 ("Integrating
+       load/store pair for RV32 with the main manual") of the riscv-isa-manual.
+
 * :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: Deprecated.  Returns similar values to
      :c:macro:`RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF`, but the key was
      mistakenly classified as a bitmask rather than a value.
@@ -379,3 +387,7 @@ The following keys are defined:
 
 * :c:macro:`RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE`: An unsigned int which
   represents the size of the Zicbop block in bytes.
+
+* :c:macro:`RISCV_HWPROBE_KEY_IMA_EXT_1`: A bitmask containing additional
+  extensions that are compatible with the
+  :c:macro:`RISCV_HWPROBE_BASE_BEHAVIOR_IMA`: base system behavior.
diff --git a/Documentation/arch/riscv/index.rst b/Documentation/arch/riscv/index.rst
index eecf347ce849..e240eb0ceb70 100644
--- a/Documentation/arch/riscv/index.rst
+++ b/Documentation/arch/riscv/index.rst
@@ -14,6 +14,8 @@ RISC-V architecture
     uabi
     vector
     cmodx
+    zicfilp
+    zicfiss
 
     features
 
diff --git a/Documentation/arch/riscv/zicfilp.rst b/Documentation/arch/riscv/zicfilp.rst
new file mode 100644
index 000000000000..78a3e01ff68c
--- /dev/null
+++ b/Documentation/arch/riscv/zicfilp.rst
@@ -0,0 +1,122 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+:Author: Deepak Gupta <debug@rivosinc.com>
+:Date:   12 January 2024
+
+====================================================
+Tracking indirect control transfers on RISC-V Linux
+====================================================
+
+This document briefly describes the interface provided to userspace by Linux
+to enable indirect branch tracking for user mode applications on RISC-V.
+
+1. Feature Overview
+--------------------
+
+Memory corruption issues usually result in crashes.  However, in the
+hands of a creative adversary, these can result in a variety of
+security issues.
+
+Some of those security issues can be code re-use attacks, where an
+adversary can use corrupt function pointers, chaining them together to
+perform jump oriented programming (JOP) or call oriented programming
+(COP) and thus compromise control flow integrity (CFI) of the program.
+
+Function pointers live in read-write memory and thus are susceptible
+to corruption.  This can allow an adversary to control the program
+counter (PC) value.  On RISC-V, the zicfilp extension enforces a
+restriction on such indirect control transfers:
+
+- Indirect control transfers must land on a landing pad instruction ``lpad``.
+  There are two exceptions to this rule:
+
+  - rs1 = x1 or rs1 = x5, i.e. a return from a function and returns are
+    protected using shadow stack (see zicfiss.rst)
+
+  - rs1 = x7. On RISC-V, the compiler usually does the following to reach a
+    function which is beyond the offset of possible J-type instruction::
+
+      auipc x7, <imm>
+      jalr (x7)
+
+    This form of indirect control transfer is immutable and doesn't
+    rely on memory.  Thus rs1=x7 is exempted from tracking and
+    these are considered software guarded jumps.
+
+The ``lpad`` instruction is a pseudo-op of ``auipc rd, <imm_20bit>``
+with ``rd=x0``.  This is a HINT op.  The ``lpad`` instruction must be
+aligned on a 4 byte boundary.  It compares the 20 bit immediate with
+x7. If ``imm_20bit`` == 0, the CPU doesn't perform any comparison with
+``x7``. If ``imm_20bit`` != 0, then ``imm_20bit`` must match ``x7``
+else CPU will raise ``software check exception`` (``cause=18``) with
+``*tval = 2``.
+
+The compiler can generate a hash over function signatures and set them
+up (truncated to 20 bits) in x7 at callsites.  Function prologues can
+have ``lpad`` instructions encoded with the same function hash. This
+further reduces the number of valid program counter addresses a call
+site can reach.
+
+2. ELF and psABI
+-----------------
+
+The toolchain sets up :c:macro:`GNU_PROPERTY_RISCV_FEATURE_1_FCFI` for
+property :c:macro:`GNU_PROPERTY_RISCV_FEATURE_1_AND` in the notes
+section of the object file.
+
+3. Linux enabling
+------------------
+
+User space programs can have multiple shared objects loaded in their
+address spaces.  It's a difficult task to make sure all the
+dependencies have been compiled with indirect branch support. Thus
+it's left to the dynamic loader to enable indirect branch tracking for
+the program.
+
+4. prctl() enabling
+--------------------
+
+:c:macro:`PR_SET_INDIR_BR_LP_STATUS` / :c:macro:`PR_GET_INDIR_BR_LP_STATUS` /
+:c:macro:`PR_LOCK_INDIR_BR_LP_STATUS` are three prctls added to manage indirect
+branch tracking.  These prctls are architecture-agnostic and return -EINVAL if
+the underlying functionality is not supported.
+
+* prctl(PR_SET_INDIR_BR_LP_STATUS, unsigned long arg)
+
+If arg1 is :c:macro:`PR_INDIR_BR_LP_ENABLE` and if CPU supports
+``zicfilp`` then the kernel will enable indirect branch tracking for the
+task.  The dynamic loader can issue this :c:macro:`prctl` once it has
+determined that all the objects loaded in the address space support
+indirect branch tracking.  Additionally, if there is a `dlopen` to an
+object which wasn't compiled with ``zicfilp``, the dynamic loader can
+issue this prctl with arg1 set to 0 (i.e. :c:macro:`PR_INDIR_BR_LP_ENABLE`
+cleared).
+
+* prctl(PR_GET_INDIR_BR_LP_STATUS, unsigned long * arg)
+
+Returns the current status of indirect branch tracking. If enabled
+it'll return :c:macro:`PR_INDIR_BR_LP_ENABLE`
+
+* prctl(PR_LOCK_INDIR_BR_LP_STATUS, unsigned long arg)
+
+Locks the current status of indirect branch tracking on the task. User
+space may want to run with a strict security posture and wouldn't want
+loading of objects without ``zicfilp`` support in them, to disallow
+disabling of indirect branch tracking. In this case, user space can
+use this prctl to lock the current settings.
+
+5. violations related to indirect branch tracking
+--------------------------------------------------
+
+Pertaining to indirect branch tracking, the CPU raises a software
+check exception in the following conditions:
+
+- missing ``lpad`` after indirect call / jmp
+- ``lpad`` not on 4 byte boundary
+- ``imm_20bit`` embedded in ``lpad`` instruction doesn't match with ``x7``
+
+In all 3 cases, ``*tval = 2`` is captured and software check exception is
+raised (``cause=18``).
+
+The kernel will treat this as :c:macro:`SIGSEGV` with code =
+:c:macro:`SEGV_CPERR` and follow the normal course of signal delivery.
diff --git a/Documentation/arch/riscv/zicfiss.rst b/Documentation/arch/riscv/zicfiss.rst
new file mode 100644
index 000000000000..4d5f7addc26d
--- /dev/null
+++ b/Documentation/arch/riscv/zicfiss.rst
@@ -0,0 +1,194 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+:Author: Deepak Gupta <debug@rivosinc.com>
+:Date:   12 January 2024
+
+=========================================================
+Shadow stack to protect function returns on RISC-V Linux
+=========================================================
+
+This document briefly describes the interface provided to userspace by Linux
+to enable shadow stacks for user mode applications on RISC-V.
+
+1. Feature Overview
+--------------------
+
+Memory corruption issues usually result in crashes.  However, in the
+hands of a creative adversary, these issues can result in a variety of
+security problems.
+
+Some of those security issues can be code re-use attacks on programs
+where an adversary can use corrupt return addresses present on the
+stack. chaining them together to perform return oriented programming
+(ROP) and thus compromising the control flow integrity (CFI) of the
+program.
+
+Return addresses live on the stack in read-write memory.  Therefore
+they are susceptible to corruption, which allows an adversary to
+control the program counter. On RISC-V, the ``zicfiss`` extension
+provides an alternate stack (the "shadow stack") on which return
+addresses can be safely placed in the prologue of the function and
+retrieved in the epilogue.  The ``zicfiss`` extension makes the
+following changes:
+
+- PTE encodings for shadow stack virtual memory
+  An earlier reserved encoding in first stage translation i.e.
+  PTE.R=0, PTE.W=1, PTE.X=0  becomes the PTE encoding for shadow stack pages.
+
+- The ``sspush x1/x5`` instruction pushes (stores) ``x1/x5`` to shadow stack.
+
+- The ``sspopchk x1/x5`` instruction pops (loads) from shadow stack and compares
+  with ``x1/x5`` and if not equal, the CPU raises a ``software check exception``
+  with ``*tval = 3``
+
+The compiler toolchain ensures that function prologues have ``sspush
+x1/x5`` to save the return address on shadow stack in addition to the
+regular stack.  Similarly, function epilogues have ``ld x5,
+offset(x2)`` followed by ``sspopchk x5`` to ensure that a popped value
+from the regular stack matches with the popped value from the shadow
+stack.
+
+2. Shadow stack protections and linux memory manager
+-----------------------------------------------------
+
+As mentioned earlier, shadow stacks get new page table encodings that
+have some special properties assigned to them, along with instructions
+that operate on the shadow stacks:
+
+- Regular stores to shadow stack memory raise store access faults. This
+  protects shadow stack memory from stray writes.
+
+- Regular loads from shadow stack memory are allowed. This allows
+  stack trace utilities or backtrace functions to read the true call
+  stack and ensure that it has not been tampered with.
+
+- Only shadow stack instructions can generate shadow stack loads or
+  shadow stack stores.
+
+- Shadow stack loads and stores on read-only memory raise AMO/store
+  page faults. Thus both ``sspush x1/x5`` and ``sspopchk x1/x5`` will
+  raise AMO/store page fault. This simplies COW handling in kernel
+  during fork(). The kernel can convert shadow stack pages into
+  read-only memory (as it does for regular read-write memory).  As
+  soon as subsequent ``sspush`` or ``sspopchk`` instructions in
+  userspace are encountered, the kernel can perform COW.
+
+- Shadow stack loads and stores on read-write or read-write-execute
+  memory raise an access fault. This is a fatal condition because
+  shadow stack loads and stores should never be operating on
+  read-write or read-write-execute memory.
+
+3. ELF and psABI
+-----------------
+
+The toolchain sets up :c:macro:`GNU_PROPERTY_RISCV_FEATURE_1_BCFI` for
+property :c:macro:`GNU_PROPERTY_RISCV_FEATURE_1_AND` in the notes
+section of the object file.
+
+4. Linux enabling
+------------------
+
+User space programs can have multiple shared objects loaded in their
+address space.  It's a difficult task to make sure all the
+dependencies have been compiled with shadow stack support.  Thus
+it's left to the dynamic loader to enable shadow stacks for the
+program.
+
+5. prctl() enabling
+--------------------
+
+:c:macro:`PR_SET_SHADOW_STACK_STATUS` / :c:macro:`PR_GET_SHADOW_STACK_STATUS` /
+:c:macro:`PR_LOCK_SHADOW_STACK_STATUS` are three prctls added to manage shadow
+stack enabling for tasks.  These prctls are architecture-agnostic and return
+-EINVAL if not implemented.
+
+* prctl(PR_SET_SHADOW_STACK_STATUS, unsigned long arg)
+
+If arg = :c:macro:`PR_SHADOW_STACK_ENABLE` and if CPU supports
+``zicfiss`` then the kernel will enable shadow stacks for the task.
+The dynamic loader can issue this :c:macro:`prctl` once it has
+determined that all the objects loaded in address space have support
+for shadow stacks.  Additionally, if there is a :c:macro:`dlopen` to
+an object which wasn't compiled with ``zicfiss``, the dynamic loader
+can issue this prctl with arg set to 0 (i.e.
+:c:macro:`PR_SHADOW_STACK_ENABLE` being clear)
+
+* prctl(PR_GET_SHADOW_STACK_STATUS, unsigned long * arg)
+
+Returns the current status of indirect branch tracking. If enabled
+it'll return :c:macro:`PR_SHADOW_STACK_ENABLE`.
+
+* prctl(PR_LOCK_SHADOW_STACK_STATUS, unsigned long arg)
+
+Locks the current status of shadow stack enabling on the
+task. Userspace may want to run with a strict security posture and
+wouldn't want loading of objects without ``zicfiss`` support.  In this
+case userspace can use this prctl to disallow disabling of shadow
+stacks on the current task.
+
+5. violations related to returns with shadow stack enabled
+-----------------------------------------------------------
+
+Pertaining to shadow stacks, the CPU raises a ``software check
+exception`` upon executing ``sspopchk x1/x5`` if ``x1/x5`` doesn't
+match the top of shadow stack.  If a mismatch happens, then the CPU
+sets ``*tval = 3`` and raises the exception.
+
+The Linux kernel will treat this as a :c:macro:`SIGSEGV` with code =
+:c:macro:`SEGV_CPERR` and follow the normal course of signal delivery.
+
+6. Shadow stack tokens
+-----------------------
+
+Regular stores on shadow stacks are not allowed and thus can't be
+tampered with via arbitrary stray writes.  However, one method of
+pivoting / switching to a shadow stack is simply writing to the CSR
+``CSR_SSP``.  This will change the active shadow stack for the
+program.  Writes to ``CSR_SSP`` in the program should be mostly
+limited to context switches, stack unwinds, or longjmp or similar
+mechanisms (like context switching of Green Threads) in languages like
+Go and Rust. CSR_SSP writes can be problematic because an attacker can
+use memory corruption bugs and leverage context switching routines to
+pivot to any shadow stack. Shadow stack tokens can help mitigate this
+problem by making sure that:
+
+- When software is switching away from a shadow stack, the shadow
+  stack pointer should be saved on the shadow stack itself (this is
+  called the ``shadow stack token``).
+
+- When software is switching to a shadow stack, it should read the
+  ``shadow stack token`` from the shadow stack pointer and verify that
+  the ``shadow stack token`` itself is a pointer to the shadow stack
+  itself.
+
+- Once the token verification is done, software can perform the write
+  to ``CSR_SSP`` to switch shadow stacks.
+
+Here "software" could refer to the user mode task runtime itself,
+managing various contexts as part of a single thread.  Or "software"
+could refer to the kernel, when the kernel has to deliver a signal to
+a user task and must save the shadow stack pointer.  The kernel can
+perform similar procedure itself by saving a token on the user mode
+task's shadow stack.  This way, whenever :c:macro:`sigreturn` happens,
+the kernel can read and verify the token and then switch to the shadow
+stack. Using this mechanism, the kernel helps the user task so that
+any corruption issue in the user task is not exploited by adversaries
+arbitrarily using :c:macro:`sigreturn`. Adversaries will have to make
+sure that there is a valid ``shadow stack token`` in addition to
+invoking :c:macro:`sigreturn`.
+
+7. Signal shadow stack
+-----------------------
+The following structure has been added to sigcontext for RISC-V::
+
+    struct __sc_riscv_cfi_state {
+        unsigned long ss_ptr;
+    };
+
+As part of signal delivery, the shadow stack token is saved on the
+current shadow stack itself.  The updated pointer is saved away in the
+:c:macro:`ss_ptr` field in :c:macro:`__sc_riscv_cfi_state` under
+:c:macro:`sigcontext`. The existing shadow stack allocation is used
+for signal delivery.  During :c:macro:`sigreturn`, kernel will obtain
+:c:macro:`ss_ptr` from :c:macro:`sigcontext`, verify the saved
+token on the shadow stack, and switch the shadow stack.
diff --git a/Documentation/arch/x86/boot.rst b/Documentation/arch/x86/boot.rst
index 6d36ce86fd8e..dca3875a2435 100644
--- a/Documentation/arch/x86/boot.rst
+++ b/Documentation/arch/x86/boot.rst
@@ -95,26 +95,26 @@ Memory Layout
 The traditional memory map for the kernel loader, used for Image or
 zImage kernels, typically looks like::
 
-  		|  			 |
+		|  			 |
   0A0000	+------------------------+
-  		|  Reserved for BIOS	 |	Do not use.  Reserved for BIOS EBDA.
+		|  Reserved for BIOS	 |	Do not use.  Reserved for BIOS EBDA.
   09A000	+------------------------+
-  		|  Command line		 |
-  		|  Stack/heap		 |	For use by the kernel real-mode code.
+		|  Command line		 |
+		|  Stack/heap		 |	For use by the kernel real-mode code.
   098000	+------------------------+
-  		|  Kernel setup		 |	The kernel real-mode code.
+		|  Kernel setup		 |	The kernel real-mode code.
   090200	+------------------------+
-  		|  Kernel boot sector	 |	The kernel legacy boot sector.
+		|  Kernel boot sector	 |	The kernel legacy boot sector.
   090000	+------------------------+
-  		|  Protected-mode kernel |	The bulk of the kernel image.
+		|  Protected-mode kernel |	The bulk of the kernel image.
   010000	+------------------------+
-  		|  Boot loader		 |	<- Boot sector entry point 0000:7C00
+		|  Boot loader		 |	<- Boot sector entry point 0000:7C00
   001000	+------------------------+
-  		|  Reserved for MBR/BIOS |
+		|  Reserved for MBR/BIOS |
   000800	+------------------------+
-  		|  Typically used by MBR |
+		|  Typically used by MBR |
   000600	+------------------------+
-  		|  BIOS use only	 |
+		|  BIOS use only	 |
   000000	+------------------------+
 
 When using bzImage, the protected-mode kernel was relocated to
@@ -142,27 +142,27 @@ above the 0x9A000 point; too many BIOSes will break above that point.
 For a modern bzImage kernel with boot protocol version >= 2.02, a
 memory layout like the following is suggested::
 
-  		~  			 ~
-  		|  Protected-mode kernel |
+		~  			 ~
+		|  Protected-mode kernel |
   100000	+------------------------+
-  		|  I/O memory hole	 |
+		|  I/O memory hole	 |
   0A0000	+------------------------+
-  		|  Reserved for BIOS	 |	Leave as much as possible unused
-  		~  			 ~
-  		|  Command line		 |	(Can also be below the X+10000 mark)
+		|  Reserved for BIOS	 |	Leave as much as possible unused
+		~  			 ~
+		|  Command line		 |	(Can also be below the X+10000 mark)
   X+10000	+------------------------+
-  		|  Stack/heap		 |	For use by the kernel real-mode code.
+		|  Stack/heap		 |	For use by the kernel real-mode code.
   X+08000	+------------------------+
-  		|  Kernel setup		 |	The kernel real-mode code.
-  		|  Kernel boot sector	 |	The kernel legacy boot sector.
+		|  Kernel setup		 |	The kernel real-mode code.
+		|  Kernel boot sector	 |	The kernel legacy boot sector.
   X		+------------------------+
-  		|  Boot loader		 |	<- Boot sector entry point 0000:7C00
+		|  Boot loader		 |	<- Boot sector entry point 0000:7C00
   001000	+------------------------+
-  		|  Reserved for MBR/BIOS |
+		|  Reserved for MBR/BIOS |
   000800	+------------------------+
-  		|  Typically used by MBR |
+		|  Typically used by MBR |
   000600	+------------------------+
-  		|  BIOS use only	 |
+		|  BIOS use only	 |
   000000	+------------------------+
 
   ... where the address X is as low as the design of the boot loader permits.
@@ -433,7 +433,7 @@ Protocol:	2.00+
 
   Assigned boot loader IDs:
 
-	== =======================================
+	==== =======================================
 	0x0  LILO
 	     (0x00 reserved for pre-2.00 bootloader)
 	0x1  Loadlin
@@ -456,7 +456,7 @@ Protocol:	2.00+
 	     <http://sebastian-plotz.blogspot.de>
 	0x12 OVMF UEFI virtualization stack
 	0x13 barebox
-	== =======================================
+	==== =======================================
 
   Please contact <hpa@zytor.com> if you need a bootloader ID value assigned.
 
@@ -809,12 +809,12 @@ Protocol:	2.09+
   as follow::
 
    struct setup_data {
-   	__u64 next;
-   	__u32 type;
-   	__u32 len;
-   	__u8 data[];
+	__u64 next;
+	__u32 type;
+	__u32 len;
+	__u8 data[];
    }
-   
+
   Where, the next is a 64-bit physical pointer to the next node of
   linked list, the next field of the last node is 0; the type is used
   to identify the contents of data; the len is the length of data
@@ -835,10 +835,10 @@ Protocol:	2.09+
   protocol 2.15::
 
    struct setup_indirect {
-   	__u32 type;
-   	__u32 reserved;		/* Reserved, must be set to zero. */
-   	__u64 len;
-   	__u64 addr;
+	__u32 type;
+	__u32 reserved;		/* Reserved, must be set to zero. */
+	__u64 len;
+	__u64 addr;
    };
 
   The type member is a SETUP_INDIRECT | SETUP_* type. However, it cannot be
@@ -850,15 +850,15 @@ Protocol:	2.09+
   In this case setup_data and setup_indirect will look like this::
 
    struct setup_data {
-   	.next = 0,	/* or <addr_of_next_setup_data_struct> */
-   	.type = SETUP_INDIRECT,
-   	.len = sizeof(setup_indirect),
-   	.data[sizeof(setup_indirect)] = (struct setup_indirect) {
-   		.type = SETUP_INDIRECT | SETUP_E820_EXT,
-   		.reserved = 0,
-   		.len = <len_of_SETUP_E820_EXT_data>,
-   		.addr = <addr_of_SETUP_E820_EXT_data>,
-   	},
+	.next = 0,	/* or <addr_of_next_setup_data_struct> */
+	.type = SETUP_INDIRECT,
+	.len = sizeof(setup_indirect),
+	.data[sizeof(setup_indirect)] = (struct setup_indirect) {
+		.type = SETUP_INDIRECT | SETUP_E820_EXT,
+		.reserved = 0,
+		.len = <len_of_SETUP_E820_EXT_data>,
+		.addr = <addr_of_SETUP_E820_EXT_data>,
+	},
    }
 
 .. note::
@@ -897,11 +897,11 @@ Offset/size:	0x260/4
   The kernel runtime start address is determined by the following algorithm::
 
    if (relocatable_kernel) {
-    	if (load_address < pref_address)
-    		load_address = pref_address;
-    	runtime_start = align_up(load_address, kernel_alignment);
+	if (load_address < pref_address)
+		load_address = pref_address;
+	runtime_start = align_up(load_address, kernel_alignment);
    } else {
-    	runtime_start = pref_address;
+	runtime_start = pref_address;
    }
 
 Hence the necessary memory window location and size can be estimated by
@@ -975,22 +975,22 @@ after kernel_info_var_len_data label. Each chunk of variable size data has to
 be prefixed with header/magic and its size, e.g.::
 
   kernel_info:
-  	.ascii  "LToP"		/* Header, Linux top (structure). */
-  	.long   kernel_info_var_len_data - kernel_info
-  	.long   kernel_info_end - kernel_info
-  	.long   0x01234567	/* Some fixed size data for the bootloaders. */
+	.ascii  "LToP"		/* Header, Linux top (structure). */
+	.long   kernel_info_var_len_data - kernel_info
+	.long   kernel_info_end - kernel_info
+	.long   0x01234567	/* Some fixed size data for the bootloaders. */
   kernel_info_var_len_data:
   example_struct:		/* Some variable size data for the bootloaders. */
-  	.ascii  "0123"		/* Header/Magic. */
-  	.long   example_struct_end - example_struct
-  	.ascii  "Struct"
-  	.long   0x89012345
+	.ascii  "0123"		/* Header/Magic. */
+	.long   example_struct_end - example_struct
+	.ascii  "Struct"
+	.long   0x89012345
   example_struct_end:
   example_strings:		/* Some variable size data for the bootloaders. */
-  	.ascii  "ABCD"		/* Header/Magic. */
-  	.long   example_strings_end - example_strings
-  	.asciz  "String_0"
-  	.asciz  "String_1"
+	.ascii  "ABCD"		/* Header/Magic. */
+	.long   example_strings_end - example_strings
+	.asciz  "String_0"
+	.asciz  "String_1"
   example_strings_end:
   kernel_info_end:
 
@@ -1132,53 +1132,53 @@ Such a boot loader should enter the following fields in the header::
   unsigned long base_ptr;	/* base address for real-mode segment */
 
   if (setup_sects == 0)
-  	setup_sects = 4;
+	setup_sects = 4;
 
   if (protocol >= 0x0200) {
-  	type_of_loader = <type code>;
-  	if (loading_initrd) {
-  		ramdisk_image = <initrd_address>;
-  		ramdisk_size = <initrd_size>;
-  	}
-
-  	if (protocol >= 0x0202 && loadflags & 0x01)
-  		heap_end = 0xe000;
-  	else
-  		heap_end = 0x9800;
-
-  	if (protocol >= 0x0201) {
-  		heap_end_ptr = heap_end - 0x200;
-  		loadflags |= 0x80;		/* CAN_USE_HEAP */
-  	}
-
-  	if (protocol >= 0x0202) {
-  		cmd_line_ptr = base_ptr + heap_end;
-  		strcpy(cmd_line_ptr, cmdline);
-  	} else {
-  		cmd_line_magic	= 0xA33F;
-  		cmd_line_offset = heap_end;
-  		setup_move_size = heap_end + strlen(cmdline) + 1;
-  		strcpy(base_ptr + cmd_line_offset, cmdline);
-  	}
+	type_of_loader = <type code>;
+	if (loading_initrd) {
+		ramdisk_image = <initrd_address>;
+		ramdisk_size = <initrd_size>;
+	}
+
+	if (protocol >= 0x0202 && loadflags & 0x01)
+		heap_end = 0xe000;
+	else
+		heap_end = 0x9800;
+
+	if (protocol >= 0x0201) {
+		heap_end_ptr = heap_end - 0x200;
+		loadflags |= 0x80;		/* CAN_USE_HEAP */
+	}
+
+	if (protocol >= 0x0202) {
+		cmd_line_ptr = base_ptr + heap_end;
+		strcpy(cmd_line_ptr, cmdline);
+	} else {
+		cmd_line_magic	= 0xA33F;
+		cmd_line_offset = heap_end;
+		setup_move_size = heap_end + strlen(cmdline) + 1;
+		strcpy(base_ptr + cmd_line_offset, cmdline);
+	}
   } else {
-  	/* Very old kernel */
+	/* Very old kernel */
 
-  	heap_end = 0x9800;
+	heap_end = 0x9800;
 
-  	cmd_line_magic	= 0xA33F;
-  	cmd_line_offset = heap_end;
+	cmd_line_magic	= 0xA33F;
+	cmd_line_offset = heap_end;
 
-  	/* A very old kernel MUST have its real-mode code loaded at 0x90000 */
-  	if (base_ptr != 0x90000) {
-  		/* Copy the real-mode kernel */
-  		memcpy(0x90000, base_ptr, (setup_sects + 1) * 512);
-  		base_ptr = 0x90000;		 /* Relocated */
-  	}
+	/* A very old kernel MUST have its real-mode code loaded at 0x90000 */
+	if (base_ptr != 0x90000) {
+		/* Copy the real-mode kernel */
+		memcpy(0x90000, base_ptr, (setup_sects + 1) * 512);
+		base_ptr = 0x90000;		 /* Relocated */
+	}
 
-  	strcpy(0x90000 + cmd_line_offset, cmdline);
+	strcpy(0x90000 + cmd_line_offset, cmdline);
 
-  	/* It is recommended to clear memory up to the 32K mark */
-  	memset(0x90000 + (setup_sects + 1) * 512, 0, (64 - (setup_sects + 1)) * 512);
+	/* It is recommended to clear memory up to the 32K mark */
+	memset(0x90000 + (setup_sects + 1) * 512, 0, (64 - (setup_sects + 1)) * 512);
   }
 
 
diff --git a/Documentation/core-api/liveupdate.rst b/Documentation/core-api/liveupdate.rst
index 7960eb15a81f..03b0212aaf41 100644
--- a/Documentation/core-api/liveupdate.rst
+++ b/Documentation/core-api/liveupdate.rst
@@ -18,6 +18,11 @@ LUO Preserving File Descriptors
 .. kernel-doc:: kernel/liveupdate/luo_file.c
    :doc: LUO File Descriptors
 
+LUO File Lifecycle Bound Global Data
+====================================
+.. kernel-doc:: kernel/liveupdate/luo_flb.c
+   :doc: LUO File Lifecycle Bound Global Data
+
 Live Update Orchestrator ABI
 ============================
 .. kernel-doc:: include/linux/kho/abi/luo.h
@@ -40,6 +45,9 @@ Public API
 .. kernel-doc:: kernel/liveupdate/luo_core.c
    :export:
 
+.. kernel-doc:: kernel/liveupdate/luo_flb.c
+   :export:
+
 .. kernel-doc:: kernel/liveupdate/luo_file.c
    :export:
 
@@ -48,6 +56,9 @@ Internal API
 .. kernel-doc:: kernel/liveupdate/luo_core.c
    :internal:
 
+.. kernel-doc:: kernel/liveupdate/luo_flb.c
+   :internal:
+
 .. kernel-doc:: kernel/liveupdate/luo_session.c
    :internal:
 
diff --git a/Documentation/devicetree/bindings/Makefile b/Documentation/devicetree/bindings/Makefile
index 8d6f85f4455d..1e116425ad70 100644
--- a/Documentation/devicetree/bindings/Makefile
+++ b/Documentation/devicetree/bindings/Makefile
@@ -82,5 +82,8 @@ clean-files = $(shell find $(obj) \( -name '*.example.dts' -o \
 dt_compatible_check: $(obj)/processed-schema.json
 	$(Q)$(srctree)/scripts/dtc/dt-extract-compatibles $(srctree) | xargs dt-check-compatible -v -s $<
 
+PHONY += dt_binding_check_one
+dt_binding_check_one: $(obj)/.dt-binding.checked $(obj)/.yamllint.checked
+
 PHONY += dt_binding_check
-dt_binding_check: $(obj)/.dt-binding.checked $(obj)/.yamllint.checked $(CHK_DT_EXAMPLES)
+dt_binding_check: dt_binding_check_one $(CHK_DT_EXAMPLES)
diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-dummy-sink.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-dummy-sink.yaml
index ed091dc0c10a..206681ccaa4c 100644
--- a/Documentation/devicetree/bindings/arm/arm,coresight-dummy-sink.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,coresight-dummy-sink.yaml
@@ -31,7 +31,7 @@ maintainers:
   - Mike Leach <mike.leach@linaro.org>
   - Suzuki K Poulose <suzuki.poulose@arm.com>
   - James Clark <james.clark@linaro.org>
-  - Mao Jinlong <quic_jinlmao@quicinc.com>
+  - Mao Jinlong <jinlong.mao@oss.qualcomm.com>
   - Hao Zhang <quic_hazha@quicinc.com>
 
 properties:
diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-dummy-source.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-dummy-source.yaml
index 78337be42b55..0b1e12ae95c3 100644
--- a/Documentation/devicetree/bindings/arm/arm,coresight-dummy-source.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,coresight-dummy-source.yaml
@@ -30,7 +30,7 @@ maintainers:
   - Mike Leach <mike.leach@linaro.org>
   - Suzuki K Poulose <suzuki.poulose@arm.com>
   - James Clark <james.clark@linaro.org>
-  - Mao Jinlong <quic_jinlmao@quicinc.com>
+  - Mao Jinlong <jinlong.mao@oss.qualcomm.com>
   - Hao Zhang <quic_hazha@quicinc.com>
 
 properties:
diff --git a/Documentation/devicetree/bindings/arm/arm,integrator.yaml b/Documentation/devicetree/bindings/arm/arm,integrator.yaml
index 1bdbd1b7ee38..8fe22185a337 100644
--- a/Documentation/devicetree/bindings/arm/arm,integrator.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,integrator.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: ARM Integrator Boards
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |+
   These were the first ARM platforms officially supported by ARM Ltd.
diff --git a/Documentation/devicetree/bindings/arm/arm,realview.yaml b/Documentation/devicetree/bindings/arm/arm,realview.yaml
index 3c5f1688dbd7..0b3133ecddac 100644
--- a/Documentation/devicetree/bindings/arm/arm,realview.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,realview.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: ARM RealView Boards
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |+
   The ARM RealView series of reference designs were built to explore the Arm11,
diff --git a/Documentation/devicetree/bindings/arm/arm,scu.yaml b/Documentation/devicetree/bindings/arm/arm,scu.yaml
index dae2aa27e641..f735b7fb8e1c 100644
--- a/Documentation/devicetree/bindings/arm/arm,scu.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,scu.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: ARM Snoop Control Unit (SCU)
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   As part of the MPCore complex, Cortex-A5 and Cortex-A9 are provided
diff --git a/Documentation/devicetree/bindings/arm/arm,versatile-sysreg.yaml b/Documentation/devicetree/bindings/arm/arm,versatile-sysreg.yaml
index 3b060c36b90c..e72dc45c1afa 100644
--- a/Documentation/devicetree/bindings/arm/arm,versatile-sysreg.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,versatile-sysreg.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Arm Versatile system registers
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   This is a system control registers block, providing multiple low level
diff --git a/Documentation/devicetree/bindings/arm/arm,versatile.yaml b/Documentation/devicetree/bindings/arm/arm,versatile.yaml
index 7a3caf6af200..c777e455d038 100644
--- a/Documentation/devicetree/bindings/arm/arm,versatile.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,versatile.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: ARM Versatile Boards
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |+
   The ARM Versatile boards are two variants of ARM926EJ-S evaluation boards
diff --git a/Documentation/devicetree/bindings/arm/arm,vexpress-juno.yaml b/Documentation/devicetree/bindings/arm/arm,vexpress-juno.yaml
index 4cdca5320544..ba04576f0ad6 100644
--- a/Documentation/devicetree/bindings/arm/arm,vexpress-juno.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,vexpress-juno.yaml
@@ -8,7 +8,7 @@ title: ARM Versatile Express and Juno Boards
 
 maintainers:
   - Sudeep Holla <sudeep.holla@arm.com>
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |+
   ARM's Versatile Express platform were built as reference designs for exploring
@@ -157,6 +157,12 @@ patternProperties:
               - const: simple-bus
           - const: simple-bus
 
+      "#interrupt-cells":
+        const: 1
+
+      interrupt-map: true
+      interrupt-map-mask: true
+
     patternProperties:
       '^motherboard-bus@':
         type: object
diff --git a/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml b/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml
index 9298c1a75dd1..8a190819e066 100644
--- a/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml
+++ b/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml
@@ -107,6 +107,7 @@ properties:
               - inventec,transformer-bmc
               - jabil,rbp-bmc
               - nvidia,gb200nvl-bmc
+              - nvidia,msx4-bmc
               - qcom,dc-scm-v1-bmc
               - quanta,s6q-bmc
               - ufispace,ncplite-bmc
diff --git a/Documentation/devicetree/bindings/arm/gemini.yaml b/Documentation/devicetree/bindings/arm/gemini.yaml
index f6a0b675830f..fc092962ab56 100644
--- a/Documentation/devicetree/bindings/arm/gemini.yaml
+++ b/Documentation/devicetree/bindings/arm/gemini.yaml
@@ -20,7 +20,7 @@ description: |
   Many of the IP blocks used in the SoC comes from Faraday Technology.
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   $nodename:
diff --git a/Documentation/devicetree/bindings/arm/intel-ixp4xx.yaml b/Documentation/devicetree/bindings/arm/intel-ixp4xx.yaml
index b7b430896596..0f1bf634a98a 100644
--- a/Documentation/devicetree/bindings/arm/intel-ixp4xx.yaml
+++ b/Documentation/devicetree/bindings/arm/intel-ixp4xx.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Intel IXP4xx
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   $nodename:
diff --git a/Documentation/devicetree/bindings/arm/qcom,coresight-ctcu.yaml b/Documentation/devicetree/bindings/arm/qcom,coresight-ctcu.yaml
index c969c16c21ef..2544fc1f71f5 100644
--- a/Documentation/devicetree/bindings/arm/qcom,coresight-ctcu.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom,coresight-ctcu.yaml
@@ -7,9 +7,9 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: CoreSight TMC Control Unit
 
 maintainers:
-  - Yuanfang Zhang <quic_yuanfang@quicinc.com>
-  - Mao Jinlong <quic_jinlmao@quicinc.com>
-  - Jie Gan <quic_jiegan@quicinc.com>
+  - Yuanfang Zhang <yuanfang.zhang@oss.qualcomm.com>
+  - Mao Jinlong <jinlong.mao@oss.qualcomm.com>
+  - Jie Gan <jie.gan@oss.qualcomm.com>
 
 description: |
   The Trace Memory Controller(TMC) is used for Embedded Trace Buffer(ETB),
diff --git a/Documentation/devicetree/bindings/arm/qcom,coresight-remote-etm.yaml b/Documentation/devicetree/bindings/arm/qcom,coresight-remote-etm.yaml
index ffe613efeabe..e3a32f30551c 100644
--- a/Documentation/devicetree/bindings/arm/qcom,coresight-remote-etm.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom,coresight-remote-etm.yaml
@@ -7,8 +7,8 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Qualcomm Coresight Remote ETM(Embedded Trace Macrocell)
 
 maintainers:
-  - Jinlong Mao <quic_jinlmao@quicinc.com>
-  - Tao Zhang <quic_taozha@quicinc.com>
+  - Jinlong Mao <jinlong.mao@oss.qualcomm.com>
+  - Tao Zhang <tao.zhang@oss.qualcomm.com>
 
 description:
   Support for ETM trace collection on remote processor using coresight
diff --git a/Documentation/devicetree/bindings/arm/qcom,coresight-tnoc.yaml b/Documentation/devicetree/bindings/arm/qcom,coresight-tnoc.yaml
index 9d1c93a9ade3..ef648a15b806 100644
--- a/Documentation/devicetree/bindings/arm/qcom,coresight-tnoc.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom,coresight-tnoc.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Qualcomm Trace Network On Chip - TNOC
 
 maintainers:
-  - Yuanfang Zhang <quic_yuanfang@quicinc.com>
+  - Yuanfang Zhang <yuanfang.zhang@oss.qualcomm.com>
 
 description: >
   The Trace Network On Chip (TNOC) is an integration hierarchy hardware
diff --git a/Documentation/devicetree/bindings/arm/qcom,coresight-tpda.yaml b/Documentation/devicetree/bindings/arm/qcom,coresight-tpda.yaml
index a48c9ac3eaa9..70d297b054c3 100644
--- a/Documentation/devicetree/bindings/arm/qcom,coresight-tpda.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom,coresight-tpda.yaml
@@ -33,8 +33,8 @@ description: |
   to sink.
 
 maintainers:
-  - Mao Jinlong <quic_jinlmao@quicinc.com>
-  - Tao Zhang <quic_taozha@quicinc.com>
+  - Mao Jinlong <jinlong.mao@oss.qualcomm.com>
+  - Tao Zhang <tao.zhang@oss.qualcomm.com>
 
 # Need a custom select here or 'arm,primecell' will match on lots of nodes
 select:
diff --git a/Documentation/devicetree/bindings/arm/qcom,coresight-tpdm.yaml b/Documentation/devicetree/bindings/arm/qcom,coresight-tpdm.yaml
index c349306f0d52..152403f548c3 100644
--- a/Documentation/devicetree/bindings/arm/qcom,coresight-tpdm.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom,coresight-tpdm.yaml
@@ -19,8 +19,8 @@ description: |
   sources and send it to a TPDA for packetization, timestamping, and funneling.
 
 maintainers:
-  - Mao Jinlong <quic_jinlmao@quicinc.com>
-  - Tao Zhang <quic_taozha@quicinc.com>
+  - Mao Jinlong <jinlong.mao@oss.qualcomm.com>
+  - Tao Zhang <tao.zhang@oss.qualcomm.com>
 
 # Need a custom select here or 'arm,primecell' will match on lots of nodes
 select:
diff --git a/Documentation/devicetree/bindings/arm/qcom.yaml b/Documentation/devicetree/bindings/arm/qcom.yaml
index d84bd3bca201..4eb0a7a9ee4a 100644
--- a/Documentation/devicetree/bindings/arm/qcom.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom.yaml
@@ -327,6 +327,12 @@ properties:
               - qcom,ipq9574-ap-al02-c9
           - const: qcom,ipq9574
 
+      - items:
+          - enum:
+              - qcom,kaanapali-mtp
+              - qcom,kaanapali-qrd
+          - const: qcom,kaanapali
+
       - description: Sierra Wireless MangOH Green with WP8548 Module
         items:
           - const: swir,mangoh-green-wp8548
@@ -1067,6 +1073,13 @@ properties:
           - const: qcom,x1e78100
           - const: qcom,x1e80100
 
+      - items:
+          - enum:
+              - medion,sprchrgd14s1
+              - tuxedo,elite14gen1
+          - const: qcom,x1e78100
+          - const: qcom,x1e80100
+
       - items:
           - enum:
               - asus,vivobook-s15
diff --git a/Documentation/devicetree/bindings/arm/ux500.yaml b/Documentation/devicetree/bindings/arm/ux500.yaml
index b42d20fa4359..3a8611e5786e 100644
--- a/Documentation/devicetree/bindings/arm/ux500.yaml
+++ b/Documentation/devicetree/bindings/arm/ux500.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Ux500 platforms
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   $nodename:
diff --git a/Documentation/devicetree/bindings/arm/vexpress-config.yaml b/Documentation/devicetree/bindings/arm/vexpress-config.yaml
index b74380da3198..41c53e3acc12 100644
--- a/Documentation/devicetree/bindings/arm/vexpress-config.yaml
+++ b/Documentation/devicetree/bindings/arm/vexpress-config.yaml
@@ -103,7 +103,7 @@ required:
   - arm,vexpress,config-bridge
 
 patternProperties:
-  'clk[0-9]*$':
+  '^clock-controller.*$':
     type: object
     description:
       clocks
@@ -137,7 +137,7 @@ patternProperties:
       - arm,vexpress-sysreg,func
       - "#clock-cells"
 
-  "^volt-.+$":
+  "^regulator-.+$":
     $ref: /schemas/regulator/regulator.yaml#
     properties:
       compatible:
@@ -272,7 +272,7 @@ examples:
         compatible = "arm,vexpress,config-bus";
         arm,vexpress,config-bridge = <&v2m_sysreg>;
 
-        clk0 {
+        clock-controller {
             compatible = "arm,vexpress-osc";
             arm,vexpress-sysreg,func = <1 0>;
             #clock-cells = <0>;
diff --git a/Documentation/devicetree/bindings/ata/ata-generic.yaml b/Documentation/devicetree/bindings/ata/ata-generic.yaml
index 0697927f3d7e..9da341ea091e 100644
--- a/Documentation/devicetree/bindings/ata/ata-generic.yaml
+++ b/Documentation/devicetree/bindings/ata/ata-generic.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Generic Parallel ATA Controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   Generic Parallel ATA controllers supporting PIO modes only.
diff --git a/Documentation/devicetree/bindings/ata/cortina,gemini-sata-bridge.yaml b/Documentation/devicetree/bindings/ata/cortina,gemini-sata-bridge.yaml
index 529093666508..66de6d4769c1 100644
--- a/Documentation/devicetree/bindings/ata/cortina,gemini-sata-bridge.yaml
+++ b/Documentation/devicetree/bindings/ata/cortina,gemini-sata-bridge.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Cortina Systems Gemini SATA Bridge
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
     The Gemini SATA bridge in a SoC-internal PATA to SATA bridge that
diff --git a/Documentation/devicetree/bindings/ata/faraday,ftide010.yaml b/Documentation/devicetree/bindings/ata/faraday,ftide010.yaml
index fa16f3767c6a..32e11d8a0a3b 100644
--- a/Documentation/devicetree/bindings/ata/faraday,ftide010.yaml
+++ b/Documentation/devicetree/bindings/ata/faraday,ftide010.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Faraday Technology FTIDE010 PATA controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   This controller is the first Faraday IDE interface block, used in the
diff --git a/Documentation/devicetree/bindings/ata/intel,ixp4xx-compact-flash.yaml b/Documentation/devicetree/bindings/ata/intel,ixp4xx-compact-flash.yaml
index 378692010c56..894a8b9eb910 100644
--- a/Documentation/devicetree/bindings/ata/intel,ixp4xx-compact-flash.yaml
+++ b/Documentation/devicetree/bindings/ata/intel,ixp4xx-compact-flash.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Intel IXP4xx CompactFlash Card Controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   The IXP4xx network processors have a CompactFlash interface that presents
diff --git a/Documentation/devicetree/bindings/ata/pata-common.yaml b/Documentation/devicetree/bindings/ata/pata-common.yaml
index 4e867dd4d402..cee4bb7eb0b9 100644
--- a/Documentation/devicetree/bindings/ata/pata-common.yaml
+++ b/Documentation/devicetree/bindings/ata/pata-common.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Common Properties for Parallel AT attachment (PATA) controllers
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   This document defines device tree properties common to most Parallel
diff --git a/Documentation/devicetree/bindings/ata/sata-common.yaml b/Documentation/devicetree/bindings/ata/sata-common.yaml
index 58c9342b9925..667f48c33195 100644
--- a/Documentation/devicetree/bindings/ata/sata-common.yaml
+++ b/Documentation/devicetree/bindings/ata/sata-common.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Common Properties for Serial AT attachment (SATA) controllers
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   This document defines device tree properties common to most Serial
diff --git a/Documentation/devicetree/bindings/auxdisplay/arm,versatile-lcd.yaml b/Documentation/devicetree/bindings/auxdisplay/arm,versatile-lcd.yaml
index 439f7b811a94..51d68a778b5c 100644
--- a/Documentation/devicetree/bindings/auxdisplay/arm,versatile-lcd.yaml
+++ b/Documentation/devicetree/bindings/auxdisplay/arm,versatile-lcd.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: ARM Versatile Character LCD
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
   - Rob Herring <robh@kernel.org>
 
 description:
diff --git a/Documentation/devicetree/bindings/bus/aspeed,ast2600-ahbc.yaml b/Documentation/devicetree/bindings/bus/aspeed,ast2600-ahbc.yaml
index 2894256c976d..77e60b32d52e 100644
--- a/Documentation/devicetree/bindings/bus/aspeed,ast2600-ahbc.yaml
+++ b/Documentation/devicetree/bindings/bus/aspeed,ast2600-ahbc.yaml
@@ -17,8 +17,10 @@ description: |
 
 properties:
   compatible:
-    enum:
-      - aspeed,ast2600-ahbc
+    items:
+      - enum:
+          - aspeed,ast2600-ahbc
+      - const: syscon
 
   reg:
     maxItems: 1
@@ -32,6 +34,6 @@ additionalProperties: false
 examples:
   - |
     ahbc@1e600000 {
-        compatible = "aspeed,ast2600-ahbc";
+        compatible = "aspeed,ast2600-ahbc", "syscon";
         reg = <0x1e600000 0x100>;
     };
diff --git a/Documentation/devicetree/bindings/bus/st,stm32mp25-rifsc.yaml b/Documentation/devicetree/bindings/bus/st,stm32mp25-rifsc.yaml
index 4d19917ad2c3..c6280c8c54a3 100644
--- a/Documentation/devicetree/bindings/bus/st,stm32mp25-rifsc.yaml
+++ b/Documentation/devicetree/bindings/bus/st,stm32mp25-rifsc.yaml
@@ -54,7 +54,7 @@ properties:
     const: 1
 
   "#size-cells":
-    const: 1
+    enum: [ 1, 2 ]
 
   ranges: true
 
diff --git a/Documentation/devicetree/bindings/cache/qcom,llcc.yaml b/Documentation/devicetree/bindings/cache/qcom,llcc.yaml
index a620a2ff5c56..4e99c405aea3 100644
--- a/Documentation/devicetree/bindings/cache/qcom,llcc.yaml
+++ b/Documentation/devicetree/bindings/cache/qcom,llcc.yaml
@@ -20,6 +20,7 @@ description: |
 properties:
   compatible:
     enum:
+      - qcom,glymur-llcc
       - qcom,ipq5424-llcc
       - qcom,kaanapali-llcc
       - qcom,qcs615-llcc
@@ -46,11 +47,11 @@ properties:
 
   reg:
     minItems: 1
-    maxItems: 10
+    maxItems: 14
 
   reg-names:
     minItems: 1
-    maxItems: 10
+    maxItems: 14
 
   interrupts:
     maxItems: 1
@@ -84,6 +85,48 @@ allOf:
           items:
             - const: llcc0_base
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,glymur-llcc
+    then:
+      properties:
+        reg:
+          items:
+            - description: LLCC0 base register region
+            - description: LLCC1 base register region
+            - description: LLCC2 base register region
+            - description: LLCC3 base register region
+            - description: LLCC4 base register region
+            - description: LLCC5 base register region
+            - description: LLCC6 base register region
+            - description: LLCC7 base register region
+            - description: LLCC8 base register region
+            - description: LLCC9 base register region
+            - description: LLCC10 base register region
+            - description: LLCC11 base register region
+            - description: LLCC broadcast base register region
+            - description: LLCC broadcast AND register region
+        reg-names:
+          items:
+            - const: llcc0_base
+            - const: llcc1_base
+            - const: llcc2_base
+            - const: llcc3_base
+            - const: llcc4_base
+            - const: llcc5_base
+            - const: llcc6_base
+            - const: llcc7_base
+            - const: llcc7_base
+            - const: llcc8_base
+            - const: llcc9_base
+            - const: llcc10_base
+            - const: llcc11_base
+            - const: llcc_broadcast_base
+            - const: llcc_broadcast_and_base
+
   - if:
       properties:
         compatible:
diff --git a/Documentation/devicetree/bindings/clock/sprd,sc9860-clk.yaml b/Documentation/devicetree/bindings/clock/sprd,sc9860-clk.yaml
index 502cd723511f..b131390207d6 100644
--- a/Documentation/devicetree/bindings/clock/sprd,sc9860-clk.yaml
+++ b/Documentation/devicetree/bindings/clock/sprd,sc9860-clk.yaml
@@ -114,25 +114,6 @@ allOf:
         - reg
       properties:
         sprd,syscon: false
-  - if:
-      properties:
-        compatible:
-          contains:
-            enum:
-              - sprd,sc9860-agcp-gate
-              - sprd,sc9860-aon-gate
-              - sprd,sc9860-apahb-gate
-              - sprd,sc9860-apapb-gate
-              - sprd,sc9860-cam-gate
-              - sprd,sc9860-disp-gate
-              - sprd,sc9860-pll
-              - sprd,sc9860-pmu-gate
-              - sprd,sc9860-vsp-gate
-    then:
-      required:
-        - sprd,syscon
-      properties:
-        reg: false
 
 additionalProperties: false
 
@@ -142,13 +123,6 @@ examples:
       #address-cells = <2>;
       #size-cells = <2>;
 
-      pmu-gate {
-        compatible = "sprd,sc9860-pmu-gate";
-        clocks = <&ext_26m>;
-        #clock-cells = <1>;
-        sprd,syscon = <&pmu_regs>;
-      };
-
       clock-controller@20000000 {
         compatible = "sprd,sc9860-ap-clk";
         reg = <0 0x20000000 0 0x400>;
diff --git a/Documentation/devicetree/bindings/clock/stericsson,u8500-clks.yaml b/Documentation/devicetree/bindings/clock/stericsson,u8500-clks.yaml
index 2150307219a0..4ebfa5a8d524 100644
--- a/Documentation/devicetree/bindings/clock/stericsson,u8500-clks.yaml
+++ b/Documentation/devicetree/bindings/clock/stericsson,u8500-clks.yaml
@@ -8,7 +8,7 @@ title: ST-Ericsson DB8500 (U8500) clocks
 
 maintainers:
   - Ulf Hansson <ulf.hansson@linaro.org>
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: While named "U8500 clocks" these clocks are inside the
   DB8500 digital baseband system-on-chip and its siblings such as
diff --git a/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml b/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml
index 2d42fc3d8ef8..22eeaef14f55 100644
--- a/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml
+++ b/Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml
@@ -35,6 +35,7 @@ properties:
       - description: v2 of CPUFREQ HW (EPSS)
         items:
           - enum:
+              - qcom,milos-cpufreq-epss
               - qcom,qcs8300-cpufreq-epss
               - qcom,qdu1000-cpufreq-epss
               - qcom,sa8255p-cpufreq-epss
@@ -169,6 +170,7 @@ allOf:
         compatible:
           contains:
             enum:
+              - qcom,milos-cpufreq-epss
               - qcom,qcs8300-cpufreq-epss
               - qcom,sc7280-cpufreq-epss
               - qcom,sm8250-cpufreq-epss
diff --git a/Documentation/devicetree/bindings/crypto/aspeed,ast2600-acry.yaml b/Documentation/devicetree/bindings/crypto/aspeed,ast2600-acry.yaml
index b18f178aac06..0dac6ee5043e 100644
--- a/Documentation/devicetree/bindings/crypto/aspeed,ast2600-acry.yaml
+++ b/Documentation/devicetree/bindings/crypto/aspeed,ast2600-acry.yaml
@@ -30,11 +30,17 @@ properties:
   interrupts:
     maxItems: 1
 
+  aspeed,ahbc:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description:
+      A phandle to the AHB controller node, which must be a syscon
+
 required:
   - compatible
   - reg
   - clocks
   - interrupts
+  - aspeed,ahbc
 
 additionalProperties: false
 
@@ -46,4 +52,5 @@ examples:
         reg = <0x1e6fa000 0x400>, <0x1e710000 0x1800>;
         interrupts = <160>;
         clocks = <&syscon ASPEED_CLK_GATE_RSACLK>;
+        aspeed,ahbc = <&ahbc>;
     };
diff --git a/Documentation/devicetree/bindings/crypto/intel,ixp4xx-crypto.yaml b/Documentation/devicetree/bindings/crypto/intel,ixp4xx-crypto.yaml
index a4006237aa89..fd20b8197207 100644
--- a/Documentation/devicetree/bindings/crypto/intel,ixp4xx-crypto.yaml
+++ b/Documentation/devicetree/bindings/crypto/intel,ixp4xx-crypto.yaml
@@ -8,7 +8,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Intel IXP4xx cryptographic engine
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   The Intel IXP4xx cryptographic engine makes use of the IXP4xx NPE
diff --git a/Documentation/devicetree/bindings/display/bridge/renesas,dsi.yaml b/Documentation/devicetree/bindings/display/bridge/renesas,dsi.yaml
index 5a99d9b9635e..c20625b8425e 100644
--- a/Documentation/devicetree/bindings/display/bridge/renesas,dsi.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/renesas,dsi.yaml
@@ -14,16 +14,21 @@ description: |
   RZ/G2L alike family of SoC's. The encoder can operate in DSI mode, with
   up to four data lanes.
 
-allOf:
-  - $ref: /schemas/display/dsi-controller.yaml#
-
 properties:
   compatible:
-    items:
+    oneOf:
+      - items:
+          - enum:
+              - renesas,r9a07g044-mipi-dsi # RZ/G2{L,LC}
+              - renesas,r9a07g054-mipi-dsi # RZ/V2L
+          - const: renesas,rzg2l-mipi-dsi
+
+      - items:
+          - const: renesas,r9a09g056-mipi-dsi # RZ/V2N
+          - const: renesas,r9a09g057-mipi-dsi
+
       - enum:
-          - renesas,r9a07g044-mipi-dsi # RZ/G2{L,LC}
-          - renesas,r9a07g054-mipi-dsi # RZ/V2L
-      - const: renesas,rzg2l-mipi-dsi
+          - renesas,r9a09g057-mipi-dsi # RZ/V2H(P)
 
   reg:
     maxItems: 1
@@ -49,34 +54,56 @@ properties:
       - const: debug
 
   clocks:
-    items:
-      - description: DSI D-PHY PLL multiplied clock
-      - description: DSI D-PHY system clock
-      - description: DSI AXI bus clock
-      - description: DSI Register access clock
-      - description: DSI Video clock
-      - description: DSI D-PHY Escape mode transmit clock
+    oneOf:
+      - items:
+          - description: DSI D-PHY PLL multiplied clock
+          - description: DSI D-PHY system clock
+          - description: DSI AXI bus clock
+          - description: DSI Register access clock
+          - description: DSI Video clock
+          - description: DSI D-PHY Escape mode transmit clock
+      - items:
+          - description: DSI D-PHY PLL reference clock
+          - description: DSI AXI bus clock
+          - description: DSI Register access clock
+          - description: DSI Video clock
+          - description: DSI D-PHY Escape mode transmit clock
 
   clock-names:
-    items:
-      - const: pllclk
-      - const: sysclk
-      - const: aclk
-      - const: pclk
-      - const: vclk
-      - const: lpclk
+    oneOf:
+      - items:
+          - const: pllclk
+          - const: sysclk
+          - const: aclk
+          - const: pclk
+          - const: vclk
+          - const: lpclk
+      - items:
+          - const: pllrefclk
+          - const: aclk
+          - const: pclk
+          - const: vclk
+          - const: lpclk
 
   resets:
-    items:
-      - description: MIPI_DSI_CMN_RSTB
-      - description: MIPI_DSI_ARESET_N
-      - description: MIPI_DSI_PRESET_N
+    oneOf:
+      - items:
+          - description: MIPI_DSI_CMN_RSTB
+          - description: MIPI_DSI_ARESET_N
+          - description: MIPI_DSI_PRESET_N
+      - items:
+          - description: MIPI_DSI_ARESET_N
+          - description: MIPI_DSI_PRESET_N
 
   reset-names:
-    items:
-      - const: rst
-      - const: arst
-      - const: prst
+    oneOf:
+      - items:
+          - const: rst
+          - const: arst
+          - const: prst
+      - items:
+          - const: arst
+          - const: prst
 
   power-domains:
     maxItems: 1
@@ -130,6 +157,41 @@ required:
 
 unevaluatedProperties: false
 
+allOf:
+  - $ref: ../dsi-controller.yaml#
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: renesas,r9a09g057-mipi-dsi
+    then:
+      properties:
+        clocks:
+          maxItems: 5
+
+        clock-names:
+          maxItems: 5
+
+        resets:
+          maxItems: 2
+
+        reset-names:
+          maxItems: 2
+    else:
+      properties:
+        clocks:
+          minItems: 6
+
+        clock-names:
+          minItems: 6
+
+        resets:
+          minItems: 3
+
+        reset-names:
+          minItems: 3
+
 examples:
   - |
     #include <dt-bindings/clock/r9a07g044-cpg.h>
diff --git a/Documentation/devicetree/bindings/display/dsi-controller.yaml b/Documentation/devicetree/bindings/display/dsi-controller.yaml
index bb4d6e9e7d0c..850b86fe03cc 100644
--- a/Documentation/devicetree/bindings/display/dsi-controller.yaml
+++ b/Documentation/devicetree/bindings/display/dsi-controller.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Common Properties for DSI Display Panels
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   This document defines device tree properties common to DSI, Display
diff --git a/Documentation/devicetree/bindings/display/faraday,tve200.yaml b/Documentation/devicetree/bindings/display/faraday,tve200.yaml
index e2ee77767321..b09628b69177 100644
--- a/Documentation/devicetree/bindings/display/faraday,tve200.yaml
+++ b/Documentation/devicetree/bindings/display/faraday,tve200.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Faraday TV Encoder TVE200
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/display/panel/arm,rtsm-display.yaml b/Documentation/devicetree/bindings/display/panel/arm,rtsm-display.yaml
index 4ad484f09ba3..fc04558fcc8d 100644
--- a/Documentation/devicetree/bindings/display/panel/arm,rtsm-display.yaml
+++ b/Documentation/devicetree/bindings/display/panel/arm,rtsm-display.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Arm RTSM Virtual Platforms Display
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: panel-common.yaml#
diff --git a/Documentation/devicetree/bindings/display/panel/arm,versatile-tft-panel.yaml b/Documentation/devicetree/bindings/display/panel/arm,versatile-tft-panel.yaml
index c9958f824d9a..b6c18e7283cd 100644
--- a/Documentation/devicetree/bindings/display/panel/arm,versatile-tft-panel.yaml
+++ b/Documentation/devicetree/bindings/display/panel/arm,versatile-tft-panel.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: ARM Versatile TFT Panels
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   These panels are connected to the daughterboards found on the
diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,ili9322.yaml b/Documentation/devicetree/bindings/display/panel/ilitek,ili9322.yaml
index 44423465f6e3..4bdc33d12306 100644
--- a/Documentation/devicetree/bindings/display/panel/ilitek,ili9322.yaml
+++ b/Documentation/devicetree/bindings/display/panel/ilitek,ili9322.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Ilitek ILI9322 TFT panel driver with SPI control bus
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   This is a driver for 320x240 TFT panels, accepting a variety of input
diff --git a/Documentation/devicetree/bindings/display/panel/lg,sw43408.yaml b/Documentation/devicetree/bindings/display/panel/lg,sw43408.yaml
index 2219d3d4ac43..f641efaeb8b3 100644
--- a/Documentation/devicetree/bindings/display/panel/lg,sw43408.yaml
+++ b/Documentation/devicetree/bindings/display/panel/lg,sw43408.yaml
@@ -4,14 +4,16 @@
 $id: http://devicetree.org/schemas/display/panel/lg,sw43408.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: LG SW43408 1080x2160 DSI panel
+title: LG SW43408 AMOLED DDIC
 
 maintainers:
   - Casey Connolly <casey.connolly@linaro.org>
 
 description:
-  This panel is used on the Pixel 3, it is a 60hz OLED panel which
-  required DSC (Display Stream Compression) and has rounded corners.
+  The SW43408 is display driver IC with connected panel.
+
+  LG LH546WF1-ED01 panel is used on the Pixel 3, it is a 60hz OLED panel
+  which required DSC (Display Stream Compression) and has rounded corners.
 
 allOf:
   - $ref: panel-common.yaml#
@@ -19,6 +21,9 @@ allOf:
 properties:
   compatible:
     items:
+      - enum:
+            # LG 5.46 inch, 1080x2160 pixels, 18:9 ratio
+          - lg,sw43408-lh546wf1-ed01
       - const: lg,sw43408
 
   reg:
@@ -46,7 +51,7 @@ examples:
         #size-cells = <0>;
 
         panel@0 {
-            compatible = "lg,sw43408";
+            compatible = "lg,sw43408-lh546wf1-ed01", "lg,sw43408";
             reg = <0>;
 
             vddi-supply = <&vreg_l14a_1p88>;
diff --git a/Documentation/devicetree/bindings/display/panel/novatek,nt35510.yaml b/Documentation/devicetree/bindings/display/panel/novatek,nt35510.yaml
index bb50fd5506c3..b39fd0c5a48a 100644
--- a/Documentation/devicetree/bindings/display/panel/novatek,nt35510.yaml
+++ b/Documentation/devicetree/bindings/display/panel/novatek,nt35510.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Novatek NT35510-based display panels
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: panel-common.yaml#
diff --git a/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml b/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml
index 8d668979b62d..2f90c887b7b8 100644
--- a/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml
+++ b/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml
@@ -55,6 +55,8 @@ properties:
       - panasonic,vvx10f004b00
         # Panasonic 10" WUXGA TFT LCD panel
       - panasonic,vvx10f034n00
+        # Samsung ltl106hl02 10.6" Full HD TFT LCD panel
+      - samsung,ltl106hl02-001
         # Samsung s6e3fa7 1080x2220 based AMS559NK06 AMOLED panel
       - samsung,s6e3fa7-ams559nk06
         # Shangai Top Display Optoelectronics 7" TL070WSH30 1024x600 TFT LCD panel
diff --git a/Documentation/devicetree/bindings/display/panel/panel-simple.yaml b/Documentation/devicetree/bindings/display/panel/panel-simple.yaml
index 24e277b19094..fc244fbb5a54 100644
--- a/Documentation/devicetree/bindings/display/panel/panel-simple.yaml
+++ b/Documentation/devicetree/bindings/display/panel/panel-simple.yaml
@@ -154,6 +154,8 @@ properties:
       - hannstar,hsd070pww1
         # HannStar Display Corp. HSD100PXN1 10.1" XGA LVDS panel
       - hannstar,hsd100pxn1
+        # HannStar Display Corp. HSD156JUW2 15.6" FHD (1920x1080) TFT LCD panel
+      - hannstar,hsd156juw2
         # Hitachi Ltd. Corporation 9" WVGA (800x480) TFT LCD panel
       - hit,tx23d38vm0caa
         # Innolux AT043TN24 4.3" WQVGA TFT LCD panel
diff --git a/Documentation/devicetree/bindings/display/panel/samsung,lms380kf01.yaml b/Documentation/devicetree/bindings/display/panel/samsung,lms380kf01.yaml
index 7ce8540551f9..74c2a617c2ff 100644
--- a/Documentation/devicetree/bindings/display/panel/samsung,lms380kf01.yaml
+++ b/Documentation/devicetree/bindings/display/panel/samsung,lms380kf01.yaml
@@ -11,7 +11,7 @@ description: The LMS380KF01 is a 480x800 DPI display panel from Samsung Mobile
   used with internal or external backlight control.
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: panel-common.yaml#
diff --git a/Documentation/devicetree/bindings/display/panel/samsung,lms397kf04.yaml b/Documentation/devicetree/bindings/display/panel/samsung,lms397kf04.yaml
index 9363032883de..4cecf502a150 100644
--- a/Documentation/devicetree/bindings/display/panel/samsung,lms397kf04.yaml
+++ b/Documentation/devicetree/bindings/display/panel/samsung,lms397kf04.yaml
@@ -10,7 +10,7 @@ description: The datasheet claims this is based around a display controller
   named DB7430 with a separate backlight controller.
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: panel-common.yaml#
diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6d16d0.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6d16d0.yaml
index 2af5bc47323f..0872476a8ac9 100644
--- a/Documentation/devicetree/bindings/display/panel/samsung,s6d16d0.yaml
+++ b/Documentation/devicetree/bindings/display/panel/samsung,s6d16d0.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Samsung S6D16D0 4" 864x480 AMOLED panel
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: panel-common.yaml#
diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6e3fc2x01.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6e3fc2x01.yaml
index d48354fb52ea..fd4388f5fb11 100644
--- a/Documentation/devicetree/bindings/display/panel/samsung,s6e3fc2x01.yaml
+++ b/Documentation/devicetree/bindings/display/panel/samsung,s6e3fc2x01.yaml
@@ -6,11 +6,11 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 
 title: Samsung S6E3FC2X01 AMOLED DDIC
 
-description: The S6E3FC2X01 is display driver IC with connected panel.
-
 maintainers:
   - David Heidelberg <david@ixit.cz>
 
+description: The S6E3FC2X01 is display driver IC with connected panel.
+
 allOf:
   - $ref: panel-common.yaml#
 
@@ -25,25 +25,21 @@ properties:
   reg:
     maxItems: 1
 
-  reset-gpios: true
-
-  port: true
-
-  vddio-supply:
-    description: VDD regulator
+  poc-supply:
+    description: POC regulator
 
   vci-supply:
     description: VCI regulator
 
-  poc-supply:
-    description: POC regulator
+  vddio-supply:
+    description: VDD regulator
 
 required:
   - compatible
   - reset-gpios
-  - vddio-supply
-  - vci-supply
   - poc-supply
+  - vci-supply
+  - vddio-supply
 
 unevaluatedProperties: false
 
diff --git a/Documentation/devicetree/bindings/display/panel/sony,acx424akp.yaml b/Documentation/devicetree/bindings/display/panel/sony,acx424akp.yaml
index fd778a20f760..64fa086730b0 100644
--- a/Documentation/devicetree/bindings/display/panel/sony,acx424akp.yaml
+++ b/Documentation/devicetree/bindings/display/panel/sony,acx424akp.yaml
@@ -12,7 +12,7 @@ description: The Sony ACX424AKP and ACX424AKM are panels built around
   AKP.
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: panel-common.yaml#
diff --git a/Documentation/devicetree/bindings/display/panel/ti,nspire.yaml b/Documentation/devicetree/bindings/display/panel/ti,nspire.yaml
index 5c5a3b519e31..fc722f706ad7 100644
--- a/Documentation/devicetree/bindings/display/panel/ti,nspire.yaml
+++ b/Documentation/devicetree/bindings/display/panel/ti,nspire.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Texas Instruments NSPIRE Display Panels
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: panel-common.yaml#
diff --git a/Documentation/devicetree/bindings/display/panel/tpo,tpg110.yaml b/Documentation/devicetree/bindings/display/panel/tpo,tpg110.yaml
index 99db268eb9b3..e5f3108cde5a 100644
--- a/Documentation/devicetree/bindings/display/panel/tpo,tpg110.yaml
+++ b/Documentation/devicetree/bindings/display/panel/tpo,tpg110.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: TPO TPG110 Panel
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
   - Thierry Reding <thierry.reding@gmail.com>
 
 description: |+
diff --git a/Documentation/devicetree/bindings/display/rockchip/rockchip,dw-hdmi.yaml b/Documentation/devicetree/bindings/display/rockchip/rockchip,dw-hdmi.yaml
index 9d096856a79a..29716764413a 100644
--- a/Documentation/devicetree/bindings/display/rockchip/rockchip,dw-hdmi.yaml
+++ b/Documentation/devicetree/bindings/display/rockchip/rockchip,dw-hdmi.yaml
@@ -23,6 +23,7 @@ properties:
       - rockchip,rk3228-dw-hdmi
       - rockchip,rk3288-dw-hdmi
       - rockchip,rk3328-dw-hdmi
+      - rockchip,rk3368-dw-hdmi
       - rockchip,rk3399-dw-hdmi
       - rockchip,rk3568-dw-hdmi
 
diff --git a/Documentation/devicetree/bindings/display/sitronix,st7571.yaml b/Documentation/devicetree/bindings/display/sitronix,st7571.yaml
index b83721eb4b7f..1931a47c4217 100644
--- a/Documentation/devicetree/bindings/display/sitronix,st7571.yaml
+++ b/Documentation/devicetree/bindings/display/sitronix,st7571.yaml
@@ -76,3 +76,28 @@ examples:
         };
       };
     };
+
+    spi {
+      #address-cells = <1>;
+      #size-cells = <0>;
+
+      display@0 {
+        compatible = "sitronix,st7571";
+        reg = <0>;
+        reset-gpios = <&gpio0 3 GPIO_ACTIVE_LOW>;
+        width-mm = <37>;
+        height-mm = <27>;
+
+        panel-timing {
+          hactive = <128>;
+          vactive = <96>;
+          hback-porch = <0>;
+          vback-porch = <0>;
+          clock-frequency = <0>;
+          hfront-porch = <0>;
+          hsync-len = <0>;
+          vfront-porch = <0>;
+          vsync-len = <0>;
+        };
+      };
+    };
diff --git a/Documentation/devicetree/bindings/display/sitronix,st7920.yaml b/Documentation/devicetree/bindings/display/sitronix,st7920.yaml
new file mode 100644
index 000000000000..c4f006fc41e1
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/sitronix,st7920.yaml
@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/sitronix,st7920.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Sitronix ST7920 LCD Display Controllers
+
+maintainers:
+  - Iker Pedrosa <ikerpedrosam@gmail.com>
+
+description:
+  The Sitronix ST7920 is a controller for monochrome dot-matrix graphical LCDs,
+  most commonly used for 128x64 pixel displays.
+
+properties:
+  compatible:
+    const: sitronix,st7920
+
+  reg:
+    maxItems: 1
+
+  vdd-supply:
+    description: Regulator that provides 5V Vdd power supply
+
+  reset-gpios:
+    maxItems: 1
+
+  spi-max-frequency:
+    maximum: 600000
+
+required:
+  - compatible
+  - reg
+  - spi-max-frequency
+
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        display@0 {
+            compatible = "sitronix,st7920";
+            reg = <0>;
+            vdd-supply = <&reg_5v>;
+            reset-gpios = <&gpio 25 GPIO_ACTIVE_LOW>;
+            spi-max-frequency = <600000>;
+            spi-cs-high;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/display/ste,mcde.yaml b/Documentation/devicetree/bindings/display/ste,mcde.yaml
index 564ea845c82e..7a12d0b817e6 100644
--- a/Documentation/devicetree/bindings/display/ste,mcde.yaml
+++ b/Documentation/devicetree/bindings/display/ste,mcde.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: ST-Ericsson Multi Channel Display Engine MCDE
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml b/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml
index 361e9cae6896..38fcee91211e 100644
--- a/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml
+++ b/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml
@@ -84,8 +84,7 @@ properties:
     maxItems: 1
     description: phandle to the associated power domain
 
-  dma-coherent:
-    type: boolean
+  dma-coherent: true
 
   ports:
     $ref: /schemas/graph.yaml#/properties/ports
diff --git a/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml b/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml
index fad7cba58d39..65ae8a1c3998 100644
--- a/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml
+++ b/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml
@@ -103,8 +103,7 @@ properties:
     maxItems: 1
     description: phandle to the associated power domain
 
-  dma-coherent:
-    type: boolean
+  dma-coherent: true
 
   ports:
     $ref: /schemas/graph.yaml#/properties/ports
diff --git a/Documentation/devicetree/bindings/dma/renesas,rz-dmac.yaml b/Documentation/devicetree/bindings/dma/renesas,rz-dmac.yaml
index f891cfcc48c7..d137b9cbaee9 100644
--- a/Documentation/devicetree/bindings/dma/renesas,rz-dmac.yaml
+++ b/Documentation/devicetree/bindings/dma/renesas,rz-dmac.yaml
@@ -24,6 +24,7 @@ properties:
       - items:
           - enum:
               - renesas,r9a09g047-dmac # RZ/G3E
+              - renesas,r9a09g056-dmac # RZ/V2N
           - const: renesas,r9a09g057-dmac
 
       - const: renesas,r9a09g057-dmac # RZ/V2H(P)
diff --git a/Documentation/devicetree/bindings/dma/stericsson,dma40.yaml b/Documentation/devicetree/bindings/dma/stericsson,dma40.yaml
index 8b42d9880400..607da11e7baa 100644
--- a/Documentation/devicetree/bindings/dma/stericsson,dma40.yaml
+++ b/Documentation/devicetree/bindings/dma/stericsson,dma40.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: ST-Ericsson DMA40 DMA Engine
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: dma-controller.yaml#
diff --git a/Documentation/devicetree/bindings/eeprom/at24.yaml b/Documentation/devicetree/bindings/eeprom/at24.yaml
index c21282634780..95ac2f15f601 100644
--- a/Documentation/devicetree/bindings/eeprom/at24.yaml
+++ b/Documentation/devicetree/bindings/eeprom/at24.yaml
@@ -116,6 +116,7 @@ properties:
           - const: atmel,24c02
       - items:
           - enum:
+              - belling,bl24c04a
               - giantec,gt24c04a
               - onnn,cat24c04
               - onnn,cat24c05
@@ -124,6 +125,7 @@ properties:
       - items:
           - enum:
               - belling,bl24c16a
+              - belling,bl24c16f
               - renesas,r1ex24016
           - const: atmel,24c16
       - items:
@@ -132,6 +134,7 @@ properties:
       - items:
           - enum:
               - belling,bl24s64
+              - giantec,gt24p64a
               - onnn,n24s64b
               - puya,p24c64f
           - const: atmel,24c64
diff --git a/Documentation/devicetree/bindings/extcon/extcon-ptn5150.yaml b/Documentation/devicetree/bindings/extcon/extcon-ptn5150.yaml
index 072b3c0c5fd0..79f88b5f4e5c 100644
--- a/Documentation/devicetree/bindings/extcon/extcon-ptn5150.yaml
+++ b/Documentation/devicetree/bindings/extcon/extcon-ptn5150.yaml
@@ -42,6 +42,9 @@ properties:
     description:
       A port node to link the usb controller for the dual role switch.
 
+  connector:
+    $ref: /schemas/connector/usb-connector.yaml#
+
 required:
   - compatible
   - interrupts
diff --git a/Documentation/devicetree/bindings/extcon/fcs,fsa880.yaml b/Documentation/devicetree/bindings/extcon/fcs,fsa880.yaml
index ef6a246a1337..bff3fd5f7f4e 100644
--- a/Documentation/devicetree/bindings/extcon/fcs,fsa880.yaml
+++ b/Documentation/devicetree/bindings/extcon/fcs,fsa880.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Fairchild Semiconductor FSA880, FSA9480 and compatibles
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   The FSA880 and FSA9480 are USB port accessory detectors and switches.
diff --git a/Documentation/devicetree/bindings/firmware/intel,ixp4xx-network-processing-engine.yaml b/Documentation/devicetree/bindings/firmware/intel,ixp4xx-network-processing-engine.yaml
index 50f1f08744a1..4d66ef483522 100644
--- a/Documentation/devicetree/bindings/firmware/intel,ixp4xx-network-processing-engine.yaml
+++ b/Documentation/devicetree/bindings/firmware/intel,ixp4xx-network-processing-engine.yaml
@@ -8,7 +8,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Intel IXP4xx Network Processing Engine
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   On the IXP4xx SoCs, the Network Processing Engine (NPE) is a small
diff --git a/Documentation/devicetree/bindings/firmware/xilinx/xlnx,zynqmp-firmware.yaml b/Documentation/devicetree/bindings/firmware/xilinx/xlnx,zynqmp-firmware.yaml
index ab8f32c440df..1903c072ed6f 100644
--- a/Documentation/devicetree/bindings/firmware/xilinx/xlnx,zynqmp-firmware.yaml
+++ b/Documentation/devicetree/bindings/firmware/xilinx/xlnx,zynqmp-firmware.yaml
@@ -76,7 +76,6 @@ properties:
     type: object
 
   pinctrl:
-    $ref: /schemas/pinctrl/xlnx,zynqmp-pinctrl.yaml#
     description: The pinctrl node provides access to pinconfig and pincontrol
       functionality available in firmware.
     type: object
@@ -105,6 +104,21 @@ properties:
       vector.
     type: object
 
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: xlnx,zynqmp-firmware
+    then:
+      properties:
+        pinctrl:
+          $ref: /schemas/pinctrl/xlnx,zynqmp-pinctrl.yaml#
+    else:
+      properties:
+        pinctrl:
+          $ref: /schemas/pinctrl/xlnx,versal-pinctrl.yaml#
+
 required:
   - compatible
 
@@ -115,6 +129,7 @@ examples:
     #include <dt-bindings/power/xlnx-zynqmp-power.h>
     firmware {
       zynqmp_firmware: zynqmp-firmware {
+        compatible = "xlnx,zynqmp-firmware";
         #power-domain-cells = <1>;
         soc-nvmem {
           compatible = "xlnx,zynqmp-nvmem-fw";
@@ -162,6 +177,10 @@ examples:
         compatible = "xlnx,versal-fpga";
       };
 
+      pinctrl {
+        compatible = "xlnx,versal-pinctrl";
+      };
+
       xlnx_aes: zynqmp-aes {
         compatible = "xlnx,zynqmp-aes";
       };
diff --git a/Documentation/devicetree/bindings/gnss/brcm,bcm4751.yaml b/Documentation/devicetree/bindings/gnss/brcm,bcm4751.yaml
index 089166089498..c34b86bb7f6f 100644
--- a/Documentation/devicetree/bindings/gnss/brcm,bcm4751.yaml
+++ b/Documentation/devicetree/bindings/gnss/brcm,bcm4751.yaml
@@ -8,7 +8,7 @@ title: Broadcom BCM4751 family GNSS Receiver
 
 maintainers:
   - Johan Hovold <johan@kernel.org>
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   Broadcom GPS chips can be used over the UART or I2C bus. The UART
diff --git a/Documentation/devicetree/bindings/gpio/faraday,ftgpio010.yaml b/Documentation/devicetree/bindings/gpio/faraday,ftgpio010.yaml
index 640da5b9b0cc..3a6a47f12982 100644
--- a/Documentation/devicetree/bindings/gpio/faraday,ftgpio010.yaml
+++ b/Documentation/devicetree/bindings/gpio/faraday,ftgpio010.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Faraday Technology FTGPIO010 GPIO Controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/gpio/gpio-consumer-common.yaml b/Documentation/devicetree/bindings/gpio/gpio-consumer-common.yaml
index 40d0be31e200..fa0148758b4b 100644
--- a/Documentation/devicetree/bindings/gpio/gpio-consumer-common.yaml
+++ b/Documentation/devicetree/bindings/gpio/gpio-consumer-common.yaml
@@ -8,7 +8,7 @@ title: Common GPIO lines
 
 maintainers:
   - Bartosz Golaszewski <brgl@bgdev.pl>
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   Pay attention to using proper GPIO flag (e.g. GPIO_ACTIVE_LOW) for the GPIOs
diff --git a/Documentation/devicetree/bindings/gpio/gpio-ep9301.yaml b/Documentation/devicetree/bindings/gpio/gpio-ep9301.yaml
index 3a1079d6ee20..ebdb7ee5b790 100644
--- a/Documentation/devicetree/bindings/gpio/gpio-ep9301.yaml
+++ b/Documentation/devicetree/bindings/gpio/gpio-ep9301.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: EP93xx GPIO controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
   - Bartosz Golaszewski <brgl@bgdev.pl>
   - Nikita Shubin <nikita.shubin@maquefel.me>
 
diff --git a/Documentation/devicetree/bindings/gpio/gpio-mmio.yaml b/Documentation/devicetree/bindings/gpio/gpio-mmio.yaml
index b4d55bf6a285..7ee40b9bc562 100644
--- a/Documentation/devicetree/bindings/gpio/gpio-mmio.yaml
+++ b/Documentation/devicetree/bindings/gpio/gpio-mmio.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Generic MMIO GPIO
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
   - Bartosz Golaszewski <brgl@bgdev.pl>
 
 description:
@@ -18,11 +18,16 @@ description:
 
 properties:
   compatible:
-    enum:
-      - brcm,bcm6345-gpio
-      - ni,169445-nand-gpio
-      - wd,mbl-gpio # Western Digital MyBook Live memory-mapped GPIO controller
-      - intel,ixp4xx-expansion-bus-mmio-gpio
+    oneOf:
+      - enum:
+          - brcm,bcm6345-gpio
+          - ni,169445-nand-gpio
+          - wd,mbl-gpio # Western Digital MyBook Live memory-mapped GPIO controller
+          - intel,ixp4xx-expansion-bus-mmio-gpio
+      - items:
+          - enum:
+              - opencores,gpio
+          - const: brcm,bcm6345-gpio
 
   big-endian: true
 
diff --git a/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml b/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml
index 12134c737ad8..4f955f855e1a 100644
--- a/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml
+++ b/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml
@@ -74,6 +74,8 @@ properties:
               - ti,tca9538
               - ti,tca9539
               - ti,tca9554
+              - ti,tcal6408
+              - ti,tcal6416
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/gpio/intel,ixp4xx-gpio.yaml b/Documentation/devicetree/bindings/gpio/intel,ixp4xx-gpio.yaml
index bfcb1f364c3a..2a980c0ed86f 100644
--- a/Documentation/devicetree/bindings/gpio/intel,ixp4xx-gpio.yaml
+++ b/Documentation/devicetree/bindings/gpio/intel,ixp4xx-gpio.yaml
@@ -22,7 +22,7 @@ description: |
   and this can be enabled by a special flag.
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/gpio/mrvl-gpio.yaml b/Documentation/devicetree/bindings/gpio/mrvl-gpio.yaml
index 65155bb701a9..7f420b9c0480 100644
--- a/Documentation/devicetree/bindings/gpio/mrvl-gpio.yaml
+++ b/Documentation/devicetree/bindings/gpio/mrvl-gpio.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Marvell PXA GPIO controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
   - Bartosz Golaszewski <bgolaszewski@baylibre.com>
   - Rob Herring <robh@kernel.org>
 
diff --git a/Documentation/devicetree/bindings/gpio/pl061-gpio.yaml b/Documentation/devicetree/bindings/gpio/pl061-gpio.yaml
index c51e10680c0a..4d970e55104b 100644
--- a/Documentation/devicetree/bindings/gpio/pl061-gpio.yaml
+++ b/Documentation/devicetree/bindings/gpio/pl061-gpio.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: ARM PL061 GPIO controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
   - Rob Herring <robh@kernel.org>
 
 # We need a select here so we don't match all nodes with 'arm,primecell'
diff --git a/Documentation/devicetree/bindings/gpio/st,nomadik-gpio.yaml b/Documentation/devicetree/bindings/gpio/st,nomadik-gpio.yaml
index b3e8951959b5..40b4a7551449 100644
--- a/Documentation/devicetree/bindings/gpio/st,nomadik-gpio.yaml
+++ b/Documentation/devicetree/bindings/gpio/st,nomadik-gpio.yaml
@@ -12,7 +12,7 @@ description:
   with pinctrl-nomadik.
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   $nodename:
diff --git a/Documentation/devicetree/bindings/gpio/st,stmpe-gpio.yaml b/Documentation/devicetree/bindings/gpio/st,stmpe-gpio.yaml
index 4555f1644a4d..66dd602e797d 100644
--- a/Documentation/devicetree/bindings/gpio/st,stmpe-gpio.yaml
+++ b/Documentation/devicetree/bindings/gpio/st,stmpe-gpio.yaml
@@ -14,7 +14,7 @@ description:
   GPIO portions of these expanders.
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/gpu/img,powervr-rogue.yaml b/Documentation/devicetree/bindings/gpu/img,powervr-rogue.yaml
index 225a6e1b7fcd..86ef68985317 100644
--- a/Documentation/devicetree/bindings/gpu/img,powervr-rogue.yaml
+++ b/Documentation/devicetree/bindings/gpu/img,powervr-rogue.yaml
@@ -20,7 +20,9 @@ properties:
           - const: img,img-gx6250
           - const: img,img-rogue
       - items:
-          - const: renesas,r8a77965-gpu
+          - enum:
+              - renesas,r8a77965-gpu
+              - renesas,r8a779a0-gpu
           - const: img,img-ge7800
           - const: img,img-rogue
       - items:
diff --git a/Documentation/devicetree/bindings/hwmon/microchip,sparx5-temp.yaml b/Documentation/devicetree/bindings/hwmon/microchip,sparx5-temp.yaml
index 51e8619dbf3c..611fcadb1e77 100644
--- a/Documentation/devicetree/bindings/hwmon/microchip,sparx5-temp.yaml
+++ b/Documentation/devicetree/bindings/hwmon/microchip,sparx5-temp.yaml
@@ -14,8 +14,12 @@ description: |
 
 properties:
   compatible:
-    enum:
-      - microchip,sparx5-temp
+    oneOf:
+      - const: microchip,sparx5-temp
+      - items:
+          - enum:
+              - microchip,lan9691-temp
+          - const: microchip,sparx5-temp
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/hwmon/ntc-thermistor.yaml b/Documentation/devicetree/bindings/hwmon/ntc-thermistor.yaml
index dc8bc4c6df34..efd10bcfb082 100644
--- a/Documentation/devicetree/bindings/hwmon/ntc-thermistor.yaml
+++ b/Documentation/devicetree/bindings/hwmon/ntc-thermistor.yaml
@@ -6,7 +6,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: NTC thermistor temperature sensors
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   Thermistors with negative temperature coefficient (NTC) are resistors that
diff --git a/Documentation/devicetree/bindings/hwmon/winbond,w83781d.yaml b/Documentation/devicetree/bindings/hwmon/winbond,w83781d.yaml
index 6971ecb314eb..d97b0e698477 100644
--- a/Documentation/devicetree/bindings/hwmon/winbond,w83781d.yaml
+++ b/Documentation/devicetree/bindings/hwmon/winbond,w83781d.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Winbond W83781 and compatible hardware monitor IC
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/i2c/arm,i2c-versatile.yaml b/Documentation/devicetree/bindings/i2c/arm,i2c-versatile.yaml
index e58465d1b0c8..26026dfd788a 100644
--- a/Documentation/devicetree/bindings/i2c/arm,i2c-versatile.yaml
+++ b/Documentation/devicetree/bindings/i2c/arm,i2c-versatile.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: I2C Controller on ARM Ltd development platforms
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: /schemas/i2c/i2c-controller.yaml#
diff --git a/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml b/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml
index 33852a5ffca8..a3fe1eea6aec 100644
--- a/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml
+++ b/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml
@@ -38,6 +38,7 @@ properties:
               - qcom,sm8450-cci
               - qcom,sm8550-cci
               - qcom,sm8650-cci
+              - qcom,sm8750-cci
               - qcom,x1e80100-cci
           - const: qcom,msm8996-cci # CCI v2
 
@@ -132,6 +133,7 @@ allOf:
             enum:
               - qcom,kaanapali-cci
               - qcom,qcm2290-cci
+              - qcom,sm8750-cci
     then:
       properties:
         clocks:
diff --git a/Documentation/devicetree/bindings/i2c/snps,designware-i2c.yaml b/Documentation/devicetree/bindings/i2c/snps,designware-i2c.yaml
index d904191bb0c6..914200188809 100644
--- a/Documentation/devicetree/bindings/i2c/snps,designware-i2c.yaml
+++ b/Documentation/devicetree/bindings/i2c/snps,designware-i2c.yaml
@@ -34,8 +34,15 @@ properties:
           - const: snps,designware-i2c
       - description: Baikal-T1 SoC System I2C controller
         const: baikal,bt1-sys-i2c
+      - description: Mobileye EyeQ DesignWare I2C controller
+        items:
+          - enum:
+              - mobileye,eyeq7h-i2c
+          - const: mobileye,eyeq6lplus-i2c
+          - const: snps,designware-i2c
       - items:
           - enum:
+              - mobileye,eyeq6lplus-i2c
               - mscc,ocelot-i2c
               - sophgo,sg2044-i2c
               - thead,th1520-i2c
diff --git a/Documentation/devicetree/bindings/i2c/st,nomadik-i2c.yaml b/Documentation/devicetree/bindings/i2c/st,nomadik-i2c.yaml
index 012402debfeb..63a459c63f6a 100644
--- a/Documentation/devicetree/bindings/i2c/st,nomadik-i2c.yaml
+++ b/Documentation/devicetree/bindings/i2c/st,nomadik-i2c.yaml
@@ -12,7 +12,7 @@ description: The Nomadik I2C host controller began its life in the ST
   DB8500 after the merge of these two companies wireless divisions.
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 # Need a custom select here or 'arm,primecell' will match on lots of nodes
 select:
diff --git a/Documentation/devicetree/bindings/iio/accel/bosch,bma255.yaml b/Documentation/devicetree/bindings/iio/accel/bosch,bma255.yaml
index 85c9537f1f02..c1387e02eb82 100644
--- a/Documentation/devicetree/bindings/iio/accel/bosch,bma255.yaml
+++ b/Documentation/devicetree/bindings/iio/accel/bosch,bma255.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Bosch BMA255 and Similar Accelerometers
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
   - Stephan Gerhold <stephan@gerhold.net>
 
 description:
diff --git a/Documentation/devicetree/bindings/iio/adc/qcom,pm8018-adc.yaml b/Documentation/devicetree/bindings/iio/adc/qcom,pm8018-adc.yaml
index 58ea1ca4a5ee..c978c3a3e31a 100644
--- a/Documentation/devicetree/bindings/iio/adc/qcom,pm8018-adc.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/qcom,pm8018-adc.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Qualcomm's PM8xxx voltage XOADC
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   The Qualcomm PM8xxx PMICs contain a HK/XO ADC (Housekeeping/Crystal
diff --git a/Documentation/devicetree/bindings/iio/gyroscope/invensense,mpu3050.yaml b/Documentation/devicetree/bindings/iio/gyroscope/invensense,mpu3050.yaml
index f3242dc0e7e6..3a307ac50aa7 100644
--- a/Documentation/devicetree/bindings/iio/gyroscope/invensense,mpu3050.yaml
+++ b/Documentation/devicetree/bindings/iio/gyroscope/invensense,mpu3050.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Invensense MPU-3050 Gyroscope
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/iio/light/capella,cm3605.yaml b/Documentation/devicetree/bindings/iio/light/capella,cm3605.yaml
index c63b79c3351b..01376c386a03 100644
--- a/Documentation/devicetree/bindings/iio/light/capella,cm3605.yaml
+++ b/Documentation/devicetree/bindings/iio/light/capella,cm3605.yaml
@@ -8,7 +8,7 @@ title:
   Capella Microsystems CM3605 Ambient Light and Short Distance Proximity Sensor
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
   - Kevin Tsai <ktsai@capellamicro.com>
 
 description: |
diff --git a/Documentation/devicetree/bindings/iio/light/sharp,gp2ap002.yaml b/Documentation/devicetree/bindings/iio/light/sharp,gp2ap002.yaml
index f8a932be0d10..99bddf31cbed 100644
--- a/Documentation/devicetree/bindings/iio/light/sharp,gp2ap002.yaml
+++ b/Documentation/devicetree/bindings/iio/light/sharp,gp2ap002.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Sharp GP2AP002A00F and GP2AP002S00F proximity and ambient light sensors
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   Proximity and ambient light sensor with IR LED for the proximity
diff --git a/Documentation/devicetree/bindings/iio/magnetometer/asahi-kasei,ak8974.yaml b/Documentation/devicetree/bindings/iio/magnetometer/asahi-kasei,ak8974.yaml
index cefb70def188..f6b4d9874190 100644
--- a/Documentation/devicetree/bindings/iio/magnetometer/asahi-kasei,ak8974.yaml
+++ b/Documentation/devicetree/bindings/iio/magnetometer/asahi-kasei,ak8974.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Asahi Kasei AK8974 magnetometer sensor
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/iio/magnetometer/yamaha,yas530.yaml b/Documentation/devicetree/bindings/iio/magnetometer/yamaha,yas530.yaml
index 877226e9219b..5cbf60f3b08b 100644
--- a/Documentation/devicetree/bindings/iio/magnetometer/yamaha,yas530.yaml
+++ b/Documentation/devicetree/bindings/iio/magnetometer/yamaha,yas530.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Yamaha YAS530 family of magnetometer sensors
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   The Yamaha YAS530 magnetometers is a line of 3-axis magnetometers
diff --git a/Documentation/devicetree/bindings/iio/st,st-sensors.yaml b/Documentation/devicetree/bindings/iio/st,st-sensors.yaml
index e955eb8e8797..a1a958215cdb 100644
--- a/Documentation/devicetree/bindings/iio/st,st-sensors.yaml
+++ b/Documentation/devicetree/bindings/iio/st,st-sensors.yaml
@@ -14,7 +14,7 @@ description: The STMicroelectronics sensor devices are pretty straight-forward
 
 maintainers:
   - Denis Ciocca <denis.ciocca@st.com>
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/input/atmel,maxtouch.yaml b/Documentation/devicetree/bindings/input/atmel,maxtouch.yaml
index d79b254f1cde..9bf07acea599 100644
--- a/Documentation/devicetree/bindings/input/atmel,maxtouch.yaml
+++ b/Documentation/devicetree/bindings/input/atmel,maxtouch.yaml
@@ -8,7 +8,7 @@ title: Atmel maXTouch touchscreen/touchpad
 
 maintainers:
   - Nick Dyer <nick@shmanahar.org>
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   Atmel maXTouch touchscreen or touchpads such as the mXT244
diff --git a/Documentation/devicetree/bindings/input/touchscreen/cypress,cy8ctma140.yaml b/Documentation/devicetree/bindings/input/touchscreen/cypress,cy8ctma140.yaml
index 86a6d18f952a..afeab49a9544 100644
--- a/Documentation/devicetree/bindings/input/touchscreen/cypress,cy8ctma140.yaml
+++ b/Documentation/devicetree/bindings/input/touchscreen/cypress,cy8ctma140.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Cypress CY8CTMA140 series touchscreen controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: touchscreen.yaml#
diff --git a/Documentation/devicetree/bindings/input/touchscreen/cypress,cy8ctma340.yaml b/Documentation/devicetree/bindings/input/touchscreen/cypress,cy8ctma340.yaml
index 4dfbb93678b5..a0b8c12977a1 100644
--- a/Documentation/devicetree/bindings/input/touchscreen/cypress,cy8ctma340.yaml
+++ b/Documentation/devicetree/bindings/input/touchscreen/cypress,cy8ctma340.yaml
@@ -12,7 +12,7 @@ description: The Cypress CY8CTMA340 series (also known as "CYTTSP" after
 
 maintainers:
   - Javier Martinez Canillas <javier@dowhile0.org>
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: touchscreen.yaml#
diff --git a/Documentation/devicetree/bindings/input/touchscreen/melfas,mms114.yaml b/Documentation/devicetree/bindings/input/touchscreen/melfas,mms114.yaml
index 90ebd4f8354c..a8a93f755458 100644
--- a/Documentation/devicetree/bindings/input/touchscreen/melfas,mms114.yaml
+++ b/Documentation/devicetree/bindings/input/touchscreen/melfas,mms114.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Melfas MMS114 family touchscreen controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: touchscreen.yaml#
diff --git a/Documentation/devicetree/bindings/input/touchscreen/sitronix,st1232.yaml b/Documentation/devicetree/bindings/input/touchscreen/sitronix,st1232.yaml
index e7ee7a0d74c4..978afaa4fcef 100644
--- a/Documentation/devicetree/bindings/input/touchscreen/sitronix,st1232.yaml
+++ b/Documentation/devicetree/bindings/input/touchscreen/sitronix,st1232.yaml
@@ -14,9 +14,13 @@ allOf:
 
 properties:
   compatible:
-    enum:
-      - sitronix,st1232
-      - sitronix,st1633
+    oneOf:
+      - enum:
+          - sitronix,st1232
+          - sitronix,st1633
+      - items:
+          - const: sitronix,st1624
+          - const: sitronix,st1633
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/input/touchscreen/zinitix,bt400.yaml b/Documentation/devicetree/bindings/input/touchscreen/zinitix,bt400.yaml
index 3f663ce3e44e..f1ce837b16df 100644
--- a/Documentation/devicetree/bindings/input/touchscreen/zinitix,bt400.yaml
+++ b/Documentation/devicetree/bindings/input/touchscreen/zinitix,bt400.yaml
@@ -12,7 +12,7 @@ description: The Zinitix BT4xx and BT5xx series of touchscreen controllers
 
 maintainers:
   - Michael Srba <Michael.Srba@seznam.cz>
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: touchscreen.yaml#
diff --git a/Documentation/devicetree/bindings/interrupt-controller/arm,versatile-fpga-irq.yaml b/Documentation/devicetree/bindings/interrupt-controller/arm,versatile-fpga-irq.yaml
index 8d581b3aac3a..42ab873665e1 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/arm,versatile-fpga-irq.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/arm,versatile-fpga-irq.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: ARM Versatile FPGA IRQ Controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   One or more FPGA IRQ controllers can be synthesized in an ARM reference board
diff --git a/Documentation/devicetree/bindings/interrupt-controller/faraday,ftintc010.yaml b/Documentation/devicetree/bindings/interrupt-controller/faraday,ftintc010.yaml
index 980e5c45f25b..e6495acea038 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/faraday,ftintc010.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/faraday,ftintc010.yaml
@@ -6,7 +6,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Faraday Technology FTINTC010 interrupt controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   This interrupt controller is a stock IP block from Faraday Technology found
diff --git a/Documentation/devicetree/bindings/interrupt-controller/intel,ixp4xx-interrupt.yaml b/Documentation/devicetree/bindings/interrupt-controller/intel,ixp4xx-interrupt.yaml
index a02a6b5af205..c375e08ba410 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/intel,ixp4xx-interrupt.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/intel,ixp4xx-interrupt.yaml
@@ -8,7 +8,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Intel IXP4xx XScale Networking Processors Interrupt Controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   This interrupt controller is found in the Intel IXP4xx processors.
diff --git a/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml b/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml
index 38d0c2d57dd6..b26246de3186 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml
@@ -27,6 +27,7 @@ properties:
     items:
       - enum:
           - qcom,glymur-pdc
+          - qcom,kaanapali-pdc
           - qcom,qcs615-pdc
           - qcom,qcs8300-pdc
           - qcom,qdu1000-pdc
diff --git a/Documentation/devicetree/bindings/interrupt-controller/renesas,r9a09g077-icu.yaml b/Documentation/devicetree/bindings/interrupt-controller/renesas,r9a09g077-icu.yaml
new file mode 100644
index 000000000000..78c01d14e765
--- /dev/null
+++ b/Documentation/devicetree/bindings/interrupt-controller/renesas,r9a09g077-icu.yaml
@@ -0,0 +1,236 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interrupt-controller/renesas,r9a09g077-icu.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Renesas RZ/{T2H,N2H} Interrupt Controller
+
+maintainers:
+  - Cosmin Tanislav <cosmin-gabriel.tanislav.xa@renesas.com>
+
+allOf:
+  - $ref: /schemas/interrupt-controller.yaml#
+
+description:
+  The Interrupt Controller (ICU) handles software-triggered interrupts
+  (INTCPU), external interrupts (IRQ and SEI), error interrupts and DMAC
+  requests.
+
+properties:
+  compatible:
+    oneOf:
+      - const: renesas,r9a09g077-icu # RZ/T2H
+
+      - items:
+          - enum:
+              - renesas,r9a09g087-icu # RZ/N2H
+          - const: renesas,r9a09g077-icu
+
+  reg:
+    items:
+      - description: Non-safety registers (INTCPU0-13, IRQ0-13)
+      - description: Safety registers (INTCPU14-15, IRQ14-15, SEI)
+
+  '#interrupt-cells':
+    description: The first cell is the SPI number of the interrupt, as per user
+      manual. The second cell is used to specify the flag.
+    const: 2
+
+  '#address-cells':
+    const: 0
+
+  interrupt-controller: true
+
+  interrupts:
+    items:
+      - description: Software interrupt 0
+      - description: Software interrupt 1
+      - description: Software interrupt 2
+      - description: Software interrupt 3
+      - description: Software interrupt 4
+      - description: Software interrupt 5
+      - description: Software interrupt 6
+      - description: Software interrupt 7
+      - description: Software interrupt 8
+      - description: Software interrupt 9
+      - description: Software interrupt 10
+      - description: Software interrupt 11
+      - description: Software interrupt 12
+      - description: Software interrupt 13
+      - description: Software interrupt 14
+      - description: Software interrupt 15
+      - description: External pin interrupt 0
+      - description: External pin interrupt 1
+      - description: External pin interrupt 2
+      - description: External pin interrupt 3
+      - description: External pin interrupt 4
+      - description: External pin interrupt 5
+      - description: External pin interrupt 6
+      - description: External pin interrupt 7
+      - description: External pin interrupt 8
+      - description: External pin interrupt 9
+      - description: External pin interrupt 10
+      - description: External pin interrupt 11
+      - description: External pin interrupt 12
+      - description: External pin interrupt 13
+      - description: External pin interrupt 14
+      - description: External pin interrupt 15
+      - description: System error interrupt
+      - description: Cortex-A55 error event 0
+      - description: Cortex-A55 error event 1
+      - description: Cortex-R52 CPU 0 error event 0
+      - description: Cortex-R52 CPU 0 error event 1
+      - description: Cortex-R52 CPU 1 error event 0
+      - description: Cortex-R52 CPU 1 error event 1
+      - description: Peripherals error event 0
+      - description: Peripherals error event 1
+      - description: DSMIF error event 0
+      - description: DSMIF error event 1
+      - description: ENCIF error event 0
+      - description: ENCIF error event 1
+
+  interrupt-names:
+    items:
+      - const: intcpu0
+      - const: intcpu1
+      - const: intcpu2
+      - const: intcpu3
+      - const: intcpu4
+      - const: intcpu5
+      - const: intcpu6
+      - const: intcpu7
+      - const: intcpu8
+      - const: intcpu9
+      - const: intcpu10
+      - const: intcpu11
+      - const: intcpu12
+      - const: intcpu13
+      - const: intcpu14
+      - const: intcpu15
+      - const: irq0
+      - const: irq1
+      - const: irq2
+      - const: irq3
+      - const: irq4
+      - const: irq5
+      - const: irq6
+      - const: irq7
+      - const: irq8
+      - const: irq9
+      - const: irq10
+      - const: irq11
+      - const: irq12
+      - const: irq13
+      - const: irq14
+      - const: irq15
+      - const: sei
+      - const: ca55-err0
+      - const: ca55-err1
+      - const: cr520-err0
+      - const: cr520-err1
+      - const: cr521-err0
+      - const: cr521-err1
+      - const: peri-err0
+      - const: peri-err1
+      - const: dsmif-err0
+      - const: dsmif-err1
+      - const: encif-err0
+      - const: encif-err1
+
+  clocks:
+    maxItems: 1
+
+  power-domains:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - '#interrupt-cells'
+  - '#address-cells'
+  - interrupt-controller
+  - interrupts
+  - interrupt-names
+  - clocks
+  - power-domains
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h>
+
+    icu: interrupt-controller@802a0000 {
+      compatible = "renesas,r9a09g077-icu";
+      reg = <0x802a0000 0x10000>,
+            <0x812a0000 0x50>;
+      #interrupt-cells = <2>;
+      #address-cells = <0>;
+      interrupt-controller;
+      interrupts = <GIC_SPI 0 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 1 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 2 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 3 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 4 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 5 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 6 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 7 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 8 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 9 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 10 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 11 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 12 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 13 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 14 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 15 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 16 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 17 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 18 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 19 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 20 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 21 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 22 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 23 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 24 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 25 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 26 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 27 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 28 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 29 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 30 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 31 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 406 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 407 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 408 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 409 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 410 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 411 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 412 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 413 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 414 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 415 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 416 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 417 IRQ_TYPE_EDGE_RISING>,
+                   <GIC_SPI 418 IRQ_TYPE_EDGE_RISING>;
+      interrupt-names = "intcpu0", "intcpu1", "intcpu2",
+                        "intcpu3", "intcpu4", "intcpu5",
+                        "intcpu6", "intcpu7", "intcpu8",
+                        "intcpu9", "intcpu10", "intcpu11",
+                        "intcpu12", "intcpu13", "intcpu14",
+                        "intcpu15",
+                        "irq0", "irq1", "irq2", "irq3",
+                        "irq4", "irq5", "irq6", "irq7",
+                        "irq8", "irq9", "irq10", "irq11",
+                        "irq12", "irq13", "irq14", "irq15",
+                        "sei",
+                        "ca55-err0", "ca55-err1",
+                        "cr520-err0", "cr520-err1",
+                        "cr521-err0", "cr521-err1",
+                        "peri-err0", "peri-err1",
+                        "dsmif-err0", "dsmif-err1",
+                        "encif-err0", "encif-err1";
+      clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKM>;
+      power-domains = <&cpg>;
+    };
diff --git a/Documentation/devicetree/bindings/interrupt-controller/renesas,rzv2h-icu.yaml b/Documentation/devicetree/bindings/interrupt-controller/renesas,rzv2h-icu.yaml
index 3f99c8645767..cb244b8f5e1c 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/renesas,rzv2h-icu.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/renesas,rzv2h-icu.yaml
@@ -22,6 +22,7 @@ properties:
   compatible:
     enum:
       - renesas,r9a09g047-icu # RZ/G3E
+      - renesas,r9a09g056-icu # RZ/V2N
       - renesas,r9a09g057-icu # RZ/V2H(P)
 
   '#interrupt-cells':
diff --git a/Documentation/devicetree/bindings/leds/backlight/kinetic,ktd253.yaml b/Documentation/devicetree/bindings/leds/backlight/kinetic,ktd253.yaml
index 73fa59e62181..e7207eb26584 100644
--- a/Documentation/devicetree/bindings/leds/backlight/kinetic,ktd253.yaml
+++ b/Documentation/devicetree/bindings/leds/backlight/kinetic,ktd253.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Kinetic Technologies KTD253 and KTD259 one-wire backlight
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   The Kinetic Technologies KTD253 and KTD259 are white LED backlights
diff --git a/Documentation/devicetree/bindings/leds/register-bit-led.yaml b/Documentation/devicetree/bindings/leds/register-bit-led.yaml
index 20930d327ae9..a6bafc96bd0c 100644
--- a/Documentation/devicetree/bindings/leds/register-bit-led.yaml
+++ b/Documentation/devicetree/bindings/leds/register-bit-led.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Register Bit LEDs
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |+
   Register bit leds are used with syscon multifunctional devices where single
diff --git a/Documentation/devicetree/bindings/leds/regulator-led.yaml b/Documentation/devicetree/bindings/leds/regulator-led.yaml
index 4ef7b96e9a08..75ee87d4a786 100644
--- a/Documentation/devicetree/bindings/leds/regulator-led.yaml
+++ b/Documentation/devicetree/bindings/leds/regulator-led.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Regulator LEDs
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   Regulator LEDs are powered by a single regulator such that they can
diff --git a/Documentation/devicetree/bindings/leds/richtek,rt8515.yaml b/Documentation/devicetree/bindings/leds/richtek,rt8515.yaml
index 68c328eec03b..0356371a6b01 100644
--- a/Documentation/devicetree/bindings/leds/richtek,rt8515.yaml
+++ b/Documentation/devicetree/bindings/leds/richtek,rt8515.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Richtek RT8515 1.5A dual channel LED driver
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   The Richtek RT8515 is a dual channel (two mode) LED driver that
diff --git a/Documentation/devicetree/bindings/mailbox/mediatek,mt8196-vcp-mbox.yaml b/Documentation/devicetree/bindings/mailbox/mediatek,mt8196-vcp-mbox.yaml
new file mode 100644
index 000000000000..7b1c5165e64e
--- /dev/null
+++ b/Documentation/devicetree/bindings/mailbox/mediatek,mt8196-vcp-mbox.yaml
@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/mailbox/mediatek,mt8196-vcp-mbox.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MediaTek Video Companion Processor (VCP) mailbox
+
+maintainers:
+  - Jjian Zhou <Jjian.Zhou@mediatek.com>
+
+description:
+  The MTK VCP mailbox enables the SoC to communicate with the VCP by passing
+  messages through 64 32-bit wide registers. It has 32 interrupt vectors in
+  either direction for signalling purposes.
+
+properties:
+  compatible:
+    enum:
+      - mediatek,mt8196-vcp-mbox
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  "#mbox-cells":
+    const: 0
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - "#mbox-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    mailbox@31b80000 {
+        compatible = "mediatek,mt8196-vcp-mbox";
+        reg = <0x31b80000 0x1000>;
+        interrupts = <GIC_SPI 789 IRQ_TYPE_LEVEL_HIGH 0>;
+        #mbox-cells = <0>;
+    };
diff --git a/Documentation/devicetree/bindings/mailbox/microchip,mpfs-mailbox.yaml b/Documentation/devicetree/bindings/mailbox/microchip,mpfs-mailbox.yaml
index 1332aab9a888..5f2ec74c1b29 100644
--- a/Documentation/devicetree/bindings/mailbox/microchip,mpfs-mailbox.yaml
+++ b/Documentation/devicetree/bindings/mailbox/microchip,mpfs-mailbox.yaml
@@ -11,7 +11,11 @@ maintainers:
 
 properties:
   compatible:
-    const: microchip,mpfs-mailbox
+    oneOf:
+      - items:
+          - const: microchip,pic64gx-mailbox
+          - const: microchip,mpfs-mailbox
+      - const: microchip,mpfs-mailbox
 
   reg:
     oneOf:
diff --git a/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml b/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml
index 9122c3d2dc30..9e91247c521a 100644
--- a/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml
+++ b/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml
@@ -19,6 +19,7 @@ properties:
       - items:
           - enum:
               - qcom,glymur-cpucp-mbox
+              - qcom,kaanapali-cpucp-mbox
           - const: qcom,x1e80100-cpucp-mbox
       - enum:
           - qcom,x1e80100-cpucp-mbox
diff --git a/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml b/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml
index e5c423130db6..7c4d6170491d 100644
--- a/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml
+++ b/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml
@@ -24,6 +24,8 @@ properties:
   compatible:
     items:
       - enum:
+          - qcom,glymur-ipcc
+          - qcom,kaanapali-ipcc
           - qcom,milos-ipcc
           - qcom,qcs8300-ipcc
           - qcom,qdu1000-ipcc
diff --git a/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,ddr4.yaml b/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,ddr4.yaml
new file mode 100644
index 000000000000..928961c74026
--- /dev/null
+++ b/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,ddr4.yaml
@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/memory-controllers/ddr/jedec,ddr4.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: DDR4 SDRAM compliant to JEDEC JESD79-4D
+
+maintainers:
+  - Krzysztof Kozlowski <krzk@kernel.org>
+
+allOf:
+  - $ref: jedec,sdram-props.yaml#
+
+properties:
+  compatible:
+    items:
+      - pattern: "^ddr4-[0-9a-f]{4},[a-z]{1,20}-[0-9a-f]{2}$"
+      - const: jedec,ddr4
+
+required:
+  - compatible
+  - density
+  - io-width
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    ddr {
+        compatible = "ddr4-00ff,azaz-ff", "jedec,ddr4";
+        density = <8192>;
+        io-width = <8>;
+    };
diff --git a/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr-channel.yaml b/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr-channel.yaml
deleted file mode 100644
index 34b5bd153f63..000000000000
--- a/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr-channel.yaml
+++ /dev/null
@@ -1,146 +0,0 @@
-# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/memory-controllers/ddr/jedec,lpddr-channel.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: LPDDR channel with chip/rank topology description
-
-description:
-  An LPDDR channel is a completely independent set of LPDDR pins (DQ, CA, CS,
-  CK, etc.) that connect one or more LPDDR chips to a host system. The main
-  purpose of this node is to overall LPDDR topology of the system, including the
-  amount of individual LPDDR chips and the ranks per chip.
-
-maintainers:
-  - Julius Werner <jwerner@chromium.org>
-
-properties:
-  compatible:
-    enum:
-      - jedec,lpddr2-channel
-      - jedec,lpddr3-channel
-      - jedec,lpddr4-channel
-      - jedec,lpddr5-channel
-
-  io-width:
-    description:
-      The number of DQ pins in the channel. If this number is different
-      from (a multiple of) the io-width of the LPDDR chip, that means that
-      multiple instances of that type of chip are wired in parallel on this
-      channel (with the channel's DQ pins split up between the different
-      chips, and the CA, CS, etc. pins of the different chips all shorted
-      together).  This means that the total physical memory controlled by a
-      channel is equal to the sum of the densities of each rank on the
-      connected LPDDR chip, times the io-width of the channel divided by
-      the io-width of the LPDDR chip.
-    enum:
-      - 8
-      - 16
-      - 32
-      - 64
-      - 128
-
-  "#address-cells":
-    const: 1
-
-  "#size-cells":
-    const: 0
-
-patternProperties:
-  "^rank@[0-9]+$":
-    type: object
-    description:
-      Each physical LPDDR chip may have one or more ranks. Ranks are
-      internal but fully independent sub-units of the chip. Each LPDDR bus
-      transaction on the channel targets exactly one rank, based on the
-      state of the CS pins. Different ranks may have different densities and
-      timing requirements.
-    required:
-      - reg
-
-allOf:
-  - if:
-      properties:
-        compatible:
-          contains:
-            const: jedec,lpddr2-channel
-    then:
-      patternProperties:
-        "^rank@[0-9]+$":
-          $ref: /schemas/memory-controllers/ddr/jedec,lpddr2.yaml#
-  - if:
-      properties:
-        compatible:
-          contains:
-            const: jedec,lpddr3-channel
-    then:
-      patternProperties:
-        "^rank@[0-9]+$":
-          $ref: /schemas/memory-controllers/ddr/jedec,lpddr3.yaml#
-  - if:
-      properties:
-        compatible:
-          contains:
-            const: jedec,lpddr4-channel
-    then:
-      patternProperties:
-        "^rank@[0-9]+$":
-          $ref: /schemas/memory-controllers/ddr/jedec,lpddr4.yaml#
-  - if:
-      properties:
-        compatible:
-          contains:
-            const: jedec,lpddr5-channel
-    then:
-      patternProperties:
-        "^rank@[0-9]+$":
-          $ref: /schemas/memory-controllers/ddr/jedec,lpddr5.yaml#
-
-required:
-  - compatible
-  - io-width
-  - "#address-cells"
-  - "#size-cells"
-
-additionalProperties: false
-
-examples:
-  - |
-    lpddr-channel0 {
-      #address-cells = <1>;
-      #size-cells = <0>;
-      compatible = "jedec,lpddr3-channel";
-      io-width = <32>;
-
-      rank@0 {
-        compatible = "lpddr3-ff,0100", "jedec,lpddr3";
-        reg = <0>;
-        density = <8192>;
-        io-width = <16>;
-        revision-id = <1 0>;
-      };
-    };
-
-    lpddr-channel1 {
-      #address-cells = <1>;
-      #size-cells = <0>;
-      compatible = "jedec,lpddr4-channel";
-      io-width = <32>;
-
-      rank@0 {
-        compatible = "lpddr4-05,0301", "jedec,lpddr4";
-        reg = <0>;
-        density = <4096>;
-        io-width = <32>;
-        revision-id = <3 1>;
-      };
-
-      rank@1 {
-        compatible = "lpddr4-05,0301", "jedec,lpddr4";
-        reg = <1>;
-        density = <2048>;
-        io-width = <32>;
-        revision-id = <3 1>;
-      };
-    };
diff --git a/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr-props.yaml b/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr-props.yaml
deleted file mode 100644
index 30267ce70124..000000000000
--- a/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr-props.yaml
+++ /dev/null
@@ -1,74 +0,0 @@
-# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/memory-controllers/ddr/jedec,lpddr-props.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: Common properties for LPDDR types
-
-description:
-  Different LPDDR types generally use the same properties and only differ in the
-  range of legal values for each. This file defines the common parts that can be
-  reused for each type. Nodes using this schema should generally be nested under
-  an LPDDR channel node.
-
-maintainers:
-  - Krzysztof Kozlowski <krzk@kernel.org>
-
-properties:
-  compatible:
-    description:
-      Compatible strings can be either explicit vendor names and part numbers
-      (e.g. elpida,ECB240ABACN), or generated strings of the form
-      lpddrX-YY,ZZZZ where X is the LPDDR version, YY is the manufacturer ID
-      (from MR5) and ZZZZ is the revision ID (from MR6 and MR7). Both IDs are
-      formatted in lower case hexadecimal representation with leading zeroes.
-      The latter form can be useful when LPDDR nodes are created at runtime by
-      boot firmware that doesn't have access to static part number information.
-
-  reg:
-    description:
-      The rank number of this LPDDR rank when used as a subnode to an LPDDR
-      channel.
-    minimum: 0
-    maximum: 3
-
-  revision-id:
-    $ref: /schemas/types.yaml#/definitions/uint32-array
-    description:
-      Revision IDs read from Mode Register 6 and 7. One byte per uint32 cell (i.e. <MR6 MR7>).
-    maxItems: 2
-    items:
-      minimum: 0
-      maximum: 255
-
-  density:
-    $ref: /schemas/types.yaml#/definitions/uint32
-    description:
-      Density in megabits of SDRAM chip. Decoded from Mode Register 8.
-    enum:
-      - 64
-      - 128
-      - 256
-      - 512
-      - 1024
-      - 2048
-      - 3072
-      - 4096
-      - 6144
-      - 8192
-      - 12288
-      - 16384
-      - 24576
-      - 32768
-
-  io-width:
-    $ref: /schemas/types.yaml#/definitions/uint32
-    description:
-      IO bus width in bits of SDRAM chip. Decoded from Mode Register 8.
-    enum:
-      - 8
-      - 16
-      - 32
-
-additionalProperties: true
diff --git a/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr2.yaml b/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr2.yaml
index a237bc259273..704bbc562528 100644
--- a/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr2.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr2.yaml
@@ -10,7 +10,7 @@ maintainers:
   - Krzysztof Kozlowski <krzk@kernel.org>
 
 allOf:
-  - $ref: jedec,lpddr-props.yaml#
+  - $ref: jedec,sdram-props.yaml#
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr3.yaml b/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr3.yaml
index e328a1195ba6..0d28df3d2bfa 100644
--- a/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr3.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr3.yaml
@@ -10,7 +10,7 @@ maintainers:
   - Krzysztof Kozlowski <krzk@kernel.org>
 
 allOf:
-  - $ref: jedec,lpddr-props.yaml#
+  - $ref: jedec,sdram-props.yaml#
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr4.yaml b/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr4.yaml
index a078892fecee..65aa07861453 100644
--- a/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr4.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr4.yaml
@@ -10,7 +10,7 @@ maintainers:
   - Krzysztof Kozlowski <krzk@kernel.org>
 
 allOf:
-  - $ref: jedec,lpddr-props.yaml#
+  - $ref: jedec,sdram-props.yaml#
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr5.yaml b/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr5.yaml
index e441dac5f154..cf5d5a8e94b3 100644
--- a/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr5.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr5.yaml
@@ -10,7 +10,7 @@ maintainers:
   - Krzysztof Kozlowski <krzk@kernel.org>
 
 allOf:
-  - $ref: jedec,lpddr-props.yaml#
+  - $ref: jedec,sdram-props.yaml#
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,sdram-channel.yaml b/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,sdram-channel.yaml
new file mode 100644
index 000000000000..5cdd8ef45100
--- /dev/null
+++ b/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,sdram-channel.yaml
@@ -0,0 +1,160 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/memory-controllers/ddr/jedec,sdram-channel.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: SDRAM channel with chip/rank topology description
+
+description:
+  A memory channel of SDRAM memory like DDR SDRAM or LPDDR SDRAM is a completely
+  independent set of pins (DQ, CA, CS, CK, etc.) that connect one or more memory
+  chips to a host system. The main purpose of this node is to overall memory
+  topology of the system, including the amount of individual memory chips and
+  the ranks per chip.
+
+maintainers:
+  - Julius Werner <jwerner@chromium.org>
+
+properties:
+  $nodename:
+    pattern: "sdram-channel-[0-9]+$"
+
+  compatible:
+    enum:
+      - jedec,ddr4-channel
+      - jedec,lpddr2-channel
+      - jedec,lpddr3-channel
+      - jedec,lpddr4-channel
+      - jedec,lpddr5-channel
+
+  io-width:
+    description:
+      The number of DQ pins in the channel. If this number is different
+      from (a multiple of) the io-width of the SDRAM chip, that means that
+      multiple instances of that type of chip are wired in parallel on this
+      channel (with the channel's DQ pins split up between the different
+      chips, and the CA, CS, etc. pins of the different chips all shorted
+      together).  This means that the total physical memory controlled by a
+      channel is equal to the sum of the densities of each rank on the
+      connected SDRAM chip, times the io-width of the channel divided by
+      the io-width of the SDRAM chip.
+    enum:
+      - 8
+      - 16
+      - 32
+      - 64
+      - 128
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 0
+
+patternProperties:
+  "^rank@[0-9]+$":
+    type: object
+    description:
+      Each physical SDRAM chip may have one or more ranks. Ranks are
+      internal but fully independent sub-units of the chip. Each SDRAM bus
+      transaction on the channel targets exactly one rank, based on the
+      state of the CS pins. Different ranks may have different densities and
+      timing requirements.
+    required:
+      - reg
+
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: jedec,ddr4-channel
+    then:
+      patternProperties:
+        "^rank@[0-9]+$":
+          $ref: /schemas/memory-controllers/ddr/jedec,ddr4.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: jedec,lpddr2-channel
+    then:
+      patternProperties:
+        "^rank@[0-9]+$":
+          $ref: /schemas/memory-controllers/ddr/jedec,lpddr2.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: jedec,lpddr3-channel
+    then:
+      patternProperties:
+        "^rank@[0-9]+$":
+          $ref: /schemas/memory-controllers/ddr/jedec,lpddr3.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: jedec,lpddr4-channel
+    then:
+      patternProperties:
+        "^rank@[0-9]+$":
+          $ref: /schemas/memory-controllers/ddr/jedec,lpddr4.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: jedec,lpddr5-channel
+    then:
+      patternProperties:
+        "^rank@[0-9]+$":
+          $ref: /schemas/memory-controllers/ddr/jedec,lpddr5.yaml#
+
+required:
+  - compatible
+  - io-width
+  - "#address-cells"
+  - "#size-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    sdram-channel-0 {
+      #address-cells = <1>;
+      #size-cells = <0>;
+      compatible = "jedec,lpddr3-channel";
+      io-width = <32>;
+
+      rank@0 {
+        compatible = "lpddr3-ff,0100", "jedec,lpddr3";
+        reg = <0>;
+        density = <8192>;
+        io-width = <16>;
+        revision-id = <1 0>;
+      };
+    };
+
+    sdram-channel-1 {
+      #address-cells = <1>;
+      #size-cells = <0>;
+      compatible = "jedec,lpddr4-channel";
+      io-width = <32>;
+
+      rank@0 {
+        compatible = "lpddr4-05,0301", "jedec,lpddr4";
+        reg = <0>;
+        density = <4096>;
+        io-width = <32>;
+        revision-id = <3 1>;
+      };
+
+      rank@1 {
+        compatible = "lpddr4-05,0301", "jedec,lpddr4";
+        reg = <1>;
+        density = <2048>;
+        io-width = <32>;
+        revision-id = <3 1>;
+      };
+    };
diff --git a/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,sdram-props.yaml b/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,sdram-props.yaml
new file mode 100644
index 000000000000..fedd66eeb9d5
--- /dev/null
+++ b/Documentation/devicetree/bindings/memory-controllers/ddr/jedec,sdram-props.yaml
@@ -0,0 +1,94 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/memory-controllers/ddr/jedec,sdram-props.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Common properties for SDRAM types
+
+description:
+  Different SDRAM types generally use the same properties and only differ in the
+  range of legal values for each. This file defines the common parts that can be
+  reused for each type. Nodes using this schema should generally be nested under
+  a SDRAM channel node.
+
+maintainers:
+  - Krzysztof Kozlowski <krzk@kernel.org>
+
+properties:
+  compatible:
+    description: |
+      Compatible strings can be either explicit vendor names and part numbers
+      (e.g. elpida,ECB240ABACN), or generated strings of the form
+      lpddrX-YY,ZZZZ or ddrX-YYYY,AAAA...-ZZ where X, Y, and Z are lowercase
+      hexadecimal with leading zeroes, and A is lowercase ASCII.
+      For LPDDR and DDR SDRAM, X is the SDRAM version (2, 3, 4, etc.).
+      For LPDDR SDRAM:
+        - YY is the manufacturer ID (from MR5), 1 byte
+        - ZZZZ is the revision ID (from MR6 and MR7), 2 bytes
+      For DDR4 SDRAM with SPD, according to JEDEC SPD4.1.2.L-6:
+        - YYYY is the manufacturer ID, 2 bytes, from bytes 320 and 321
+        - AAAA... is the part number, 20 bytes (20 chars) from bytes 329 to 348
+          without trailing spaces
+        - ZZ is the revision ID, 1 byte, from byte 349
+      The former form is useful when the SDRAM vendor and part number are
+      known, for example, when memory is soldered on the board. The latter
+      form is useful when SDRAM nodes are created at runtime by boot firmware
+      that doesn't have access to static part number information.
+
+  reg:
+    description:
+      The rank number of this memory rank when used as a subnode to an memory
+      channel.
+    minimum: 0
+    maximum: 3
+
+  revision-id:
+    $ref: /schemas/types.yaml#/definitions/uint32-array
+    description: |
+      SDRAM revision ID:
+        - LPDDR SDRAM, decoded from Mode Registers 6 and 7, always 2 bytes.
+        - DDR4 SDRAM, decoded from the SPD from byte 349 according to
+          JEDEC SPD4.1.2.L-6, always 1 byte.
+      One byte per uint32 cell (e.g., <MR6 MR7>).
+    maxItems: 2
+    items:
+      minimum: 0
+      maximum: 255
+
+  density:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: |
+      Density of the SDRAM chip in megabits:
+        - LPDDR SDRAM, decoded from Mode Register 8.
+        - DDR4 SDRAM, decoded from the SPD from bits 3-0 of byte 4 according to
+          JEDEC SPD4.1.2.L-6.
+    enum:
+      - 64
+      - 128
+      - 256
+      - 512
+      - 1024
+      - 2048
+      - 3072
+      - 4096
+      - 6144
+      - 8192
+      - 12288
+      - 16384
+      - 24576
+      - 32768
+
+  io-width:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: |
+      I/O bus width in bits of the SDRAM chip:
+        - LPDDR SDRAM, decoded from Mode Register 8.
+        - DDR4 SDRAM, decoded from the SPD from bits 2-0 of byte 12 according to
+          JEDEC SPD4.1.2.L-6.
+    enum:
+      - 8
+      - 16
+      - 32
+
+additionalProperties: true
diff --git a/Documentation/devicetree/bindings/memory-controllers/intel,ixp4xx-expansion-bus-controller.yaml b/Documentation/devicetree/bindings/memory-controllers/intel,ixp4xx-expansion-bus-controller.yaml
index 3049d6bb0b1f..2a4bf905a369 100644
--- a/Documentation/devicetree/bindings/memory-controllers/intel,ixp4xx-expansion-bus-controller.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/intel,ixp4xx-expansion-bus-controller.yaml
@@ -12,7 +12,7 @@ description: |
   including IXP42x, IXP43x, IXP45x and IXP46x.
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   $nodename:
diff --git a/Documentation/devicetree/bindings/memory-controllers/intel,ixp4xx-expansion-peripheral-props.yaml b/Documentation/devicetree/bindings/memory-controllers/intel,ixp4xx-expansion-peripheral-props.yaml
index d1479a7b9c8d..020fa49c3454 100644
--- a/Documentation/devicetree/bindings/memory-controllers/intel,ixp4xx-expansion-peripheral-props.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/intel,ixp4xx-expansion-peripheral-props.yaml
@@ -12,7 +12,7 @@ description:
   including IXP42x, IXP43x, IXP45x and IXP46x.
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   intel,ixp4xx-eb-t1:
diff --git a/Documentation/devicetree/bindings/mfd/arm,dev-platforms-syscon.yaml b/Documentation/devicetree/bindings/mfd/arm,dev-platforms-syscon.yaml
index 46b164ae0831..7f3b1b77293c 100644
--- a/Documentation/devicetree/bindings/mfd/arm,dev-platforms-syscon.yaml
+++ b/Documentation/devicetree/bindings/mfd/arm,dev-platforms-syscon.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Arm Ltd Developer Platforms System Controllers
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   The Arm Ltd Integrator, Realview, and Versatile families of developer
diff --git a/Documentation/devicetree/bindings/mfd/st,stmpe.yaml b/Documentation/devicetree/bindings/mfd/st,stmpe.yaml
index b77cc3f3075d..df43878fbe18 100644
--- a/Documentation/devicetree/bindings/mfd/st,stmpe.yaml
+++ b/Documentation/devicetree/bindings/mfd/st,stmpe.yaml
@@ -12,7 +12,7 @@ description: STMicroelectronics Port Expander (STMPE) is a series of slow
   peripherals connected to SPI or I2C.
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: /schemas/spi/spi-peripheral-props.yaml#
diff --git a/Documentation/devicetree/bindings/mfd/stericsson,ab8500.yaml b/Documentation/devicetree/bindings/mfd/stericsson,ab8500.yaml
index ce5e845ab5c5..0fdfbfdfe88a 100644
--- a/Documentation/devicetree/bindings/mfd/stericsson,ab8500.yaml
+++ b/Documentation/devicetree/bindings/mfd/stericsson,ab8500.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: ST-Ericsson Analog Baseband AB8500 and AB8505
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   the AB8500 "Analog Baseband" is the mixed-signals integrated circuit
diff --git a/Documentation/devicetree/bindings/mfd/stericsson,db8500-prcmu.yaml b/Documentation/devicetree/bindings/mfd/stericsson,db8500-prcmu.yaml
index d6c13779d44e..4edd4a3bab88 100644
--- a/Documentation/devicetree/bindings/mfd/stericsson,db8500-prcmu.yaml
+++ b/Documentation/devicetree/bindings/mfd/stericsson,db8500-prcmu.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: ST-Ericsson DB8500 PRCMU - Power Reset and Control Management Unit
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   The DB8500 Power Reset and Control Management Unit is an XP70 8-bit
diff --git a/Documentation/devicetree/bindings/misc/intel,ixp4xx-ahb-queue-manager.yaml b/Documentation/devicetree/bindings/misc/intel,ixp4xx-ahb-queue-manager.yaml
index aab89946b04f..1198d87d0ab6 100644
--- a/Documentation/devicetree/bindings/misc/intel,ixp4xx-ahb-queue-manager.yaml
+++ b/Documentation/devicetree/bindings/misc/intel,ixp4xx-ahb-queue-manager.yaml
@@ -8,7 +8,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Intel IXP4xx AHB Queue Manager
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   The IXP4xx AHB Queue Manager maintains queues as circular buffers in
diff --git a/Documentation/devicetree/bindings/mmc/arasan,sdhci.yaml b/Documentation/devicetree/bindings/mmc/arasan,sdhci.yaml
index 8e79de97b242..d6b6fa6dcffb 100644
--- a/Documentation/devicetree/bindings/mmc/arasan,sdhci.yaml
+++ b/Documentation/devicetree/bindings/mmc/arasan,sdhci.yaml
@@ -121,6 +121,8 @@ properties:
       - const: clk_ahb
       - const: gate
 
+  dma-coherent: true
+
   interrupts:
     minItems: 1
     maxItems: 2
diff --git a/Documentation/devicetree/bindings/mmc/arm,pl18x.yaml b/Documentation/devicetree/bindings/mmc/arm,pl18x.yaml
index 8f62e2c7fa64..f90fd73904a2 100644
--- a/Documentation/devicetree/bindings/mmc/arm,pl18x.yaml
+++ b/Documentation/devicetree/bindings/mmc/arm,pl18x.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: ARM PrimeCell MultiMedia Card Interface (MMCI) PL180 and PL181
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
   - Ulf Hansson <ulf.hansson@linaro.org>
 
 description:
diff --git a/Documentation/devicetree/bindings/mmc/aspeed,sdhci.yaml b/Documentation/devicetree/bindings/mmc/aspeed,sdhci.yaml
index 9fce8cd7b0b6..d24950ccea95 100644
--- a/Documentation/devicetree/bindings/mmc/aspeed,sdhci.yaml
+++ b/Documentation/devicetree/bindings/mmc/aspeed,sdhci.yaml
@@ -41,7 +41,7 @@ properties:
 patternProperties:
   "^sdhci@[0-9a-f]+$":
     type: object
-    $ref: mmc-controller.yaml
+    $ref: sdhci-common.yaml
     unevaluatedProperties: false
 
     properties:
diff --git a/Documentation/devicetree/bindings/mmc/brcm,iproc-sdhci.yaml b/Documentation/devicetree/bindings/mmc/brcm,iproc-sdhci.yaml
index 2f63f2cdeb71..65bb2f66f8cf 100644
--- a/Documentation/devicetree/bindings/mmc/brcm,iproc-sdhci.yaml
+++ b/Documentation/devicetree/bindings/mmc/brcm,iproc-sdhci.yaml
@@ -26,9 +26,14 @@ properties:
   reg:
     minItems: 1
 
+  dma-coherent: true
+
   interrupts:
     maxItems: 1
 
+  iommus:
+    maxItems: 1
+
   clocks:
     maxItems: 1
     description:
diff --git a/Documentation/devicetree/bindings/mmc/cdns,sdhci.yaml b/Documentation/devicetree/bindings/mmc/cdns,sdhci.yaml
index ac75d694611a..6c7317d13aa6 100644
--- a/Documentation/devicetree/bindings/mmc/cdns,sdhci.yaml
+++ b/Documentation/devicetree/bindings/mmc/cdns,sdhci.yaml
@@ -134,8 +134,6 @@ allOf:
           items:
             - description: Host controller registers
             - description: Elba byte-lane enable register for writes
-      required:
-        - resets
     else:
       properties:
         reg:
diff --git a/Documentation/devicetree/bindings/mmc/mtk-sd.yaml b/Documentation/devicetree/bindings/mmc/mtk-sd.yaml
index 6dd26ad31491..eb3755bdfdf7 100644
--- a/Documentation/devicetree/bindings/mmc/mtk-sd.yaml
+++ b/Documentation/devicetree/bindings/mmc/mtk-sd.yaml
@@ -25,6 +25,7 @@ properties:
           - mediatek,mt8135-mmc
           - mediatek,mt8173-mmc
           - mediatek,mt8183-mmc
+          - mediatek,mt8189-mmc
           - mediatek,mt8196-mmc
           - mediatek,mt8516-mmc
       - items:
@@ -192,6 +193,7 @@ allOf:
             - mediatek,mt8183-mmc
             - mediatek,mt8186-mmc
             - mediatek,mt8188-mmc
+            - mediatek,mt8189-mmc
             - mediatek,mt8195-mmc
             - mediatek,mt8196-mmc
             - mediatek,mt8516-mmc
@@ -240,6 +242,7 @@ allOf:
               - mediatek,mt7986-mmc
               - mediatek,mt7988-mmc
               - mediatek,mt8183-mmc
+              - mediatek,mt8189-mmc
               - mediatek,mt8196-mmc
     then:
       properties:
diff --git a/Documentation/devicetree/bindings/mtd/partitions/arm,arm-firmware-suite.yaml b/Documentation/devicetree/bindings/mtd/partitions/arm,arm-firmware-suite.yaml
index 97618847ee35..e9b1a6869910 100644
--- a/Documentation/devicetree/bindings/mtd/partitions/arm,arm-firmware-suite.yaml
+++ b/Documentation/devicetree/bindings/mtd/partitions/arm,arm-firmware-suite.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: ARM Firmware Suite (AFS) Partitions
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 select: false
 
diff --git a/Documentation/devicetree/bindings/mtd/partitions/redboot-fis.yaml b/Documentation/devicetree/bindings/mtd/partitions/redboot-fis.yaml
index ba7445cd69e8..e3978d2bc056 100644
--- a/Documentation/devicetree/bindings/mtd/partitions/redboot-fis.yaml
+++ b/Documentation/devicetree/bindings/mtd/partitions/redboot-fis.yaml
@@ -14,7 +14,7 @@ description: The FLASH Image System (FIS) directory is a flash description
     32 KB in size.
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 select: false
 
diff --git a/Documentation/devicetree/bindings/mtd/partitions/seama.yaml b/Documentation/devicetree/bindings/mtd/partitions/seama.yaml
index 4c1cbf43e81a..4af185204b4b 100644
--- a/Documentation/devicetree/bindings/mtd/partitions/seama.yaml
+++ b/Documentation/devicetree/bindings/mtd/partitions/seama.yaml
@@ -18,7 +18,7 @@ allOf:
   - $ref: partition.yaml#
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/net/bluetooth/brcm,bluetooth.yaml b/Documentation/devicetree/bindings/net/bluetooth/brcm,bluetooth.yaml
index 3c410cadff23..95501e858e6f 100644
--- a/Documentation/devicetree/bindings/net/bluetooth/brcm,bluetooth.yaml
+++ b/Documentation/devicetree/bindings/net/bluetooth/brcm,bluetooth.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Broadcom Bluetooth Chips
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   This binding describes Broadcom UART-attached bluetooth chips.
diff --git a/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.yaml b/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.yaml
index a930358f6a66..f0b5bea2458d 100644
--- a/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.yaml
+++ b/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Cortina Systems Gemini Ethernet Controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   This ethernet controller is found in the Gemini SoC family:
diff --git a/Documentation/devicetree/bindings/net/dsa/micrel,ks8995.yaml b/Documentation/devicetree/bindings/net/dsa/micrel,ks8995.yaml
index 854808ff5ad5..e9ce36067033 100644
--- a/Documentation/devicetree/bindings/net/dsa/micrel,ks8995.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/micrel,ks8995.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Micrel KS8995 Family DSA Switches
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   The Micrel KS8995 DSA Switches are 100 Mbit switches that were produced in
diff --git a/Documentation/devicetree/bindings/net/dsa/realtek.yaml b/Documentation/devicetree/bindings/net/dsa/realtek.yaml
index f348e66fb515..473facd87a62 100644
--- a/Documentation/devicetree/bindings/net/dsa/realtek.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/realtek.yaml
@@ -10,7 +10,7 @@ allOf:
   - $ref: dsa.yaml#/$defs/ethernet-ports
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   Realtek advertises these chips as fast/gigabit switches or unmanaged
diff --git a/Documentation/devicetree/bindings/net/dsa/vitesse,vsc73xx.yaml b/Documentation/devicetree/bindings/net/dsa/vitesse,vsc73xx.yaml
index 51cf574249be..c41f479bdee9 100644
--- a/Documentation/devicetree/bindings/net/dsa/vitesse,vsc73xx.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/vitesse,vsc73xx.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Vitesse VSC73xx DSA Switches
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   The Vitesse DSA Switches were produced in the early-to-mid 2000s.
diff --git a/Documentation/devicetree/bindings/net/intel,ixp46x-ptp-timer.yaml b/Documentation/devicetree/bindings/net/intel,ixp46x-ptp-timer.yaml
index f92730b1d2fa..80336b7e64ec 100644
--- a/Documentation/devicetree/bindings/net/intel,ixp46x-ptp-timer.yaml
+++ b/Documentation/devicetree/bindings/net/intel,ixp46x-ptp-timer.yaml
@@ -8,7 +8,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Intel IXP46x PTP Timer (TSYNC)
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   The Intel IXP46x PTP timer is known in the manual as IEEE1588 Hardware
diff --git a/Documentation/devicetree/bindings/net/intel,ixp4xx-ethernet.yaml b/Documentation/devicetree/bindings/net/intel,ixp4xx-ethernet.yaml
index 8689de1aaea1..3b8f83b7099d 100644
--- a/Documentation/devicetree/bindings/net/intel,ixp4xx-ethernet.yaml
+++ b/Documentation/devicetree/bindings/net/intel,ixp4xx-ethernet.yaml
@@ -11,7 +11,7 @@ allOf:
   - $ref: ethernet-controller.yaml#
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   The Intel IXP4xx ethernet makes use of the IXP4xx NPE (Network
diff --git a/Documentation/devicetree/bindings/net/intel,ixp4xx-hss.yaml b/Documentation/devicetree/bindings/net/intel,ixp4xx-hss.yaml
index 7a405e9b37b2..1d952735c81b 100644
--- a/Documentation/devicetree/bindings/net/intel,ixp4xx-hss.yaml
+++ b/Documentation/devicetree/bindings/net/intel,ixp4xx-hss.yaml
@@ -8,7 +8,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Intel IXP4xx V.35 WAN High Speed Serial Link (HSS)
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   The Intel IXP4xx HSS makes use of the IXP4xx NPE (Network
diff --git a/Documentation/devicetree/bindings/pci/faraday,ftpci100.yaml b/Documentation/devicetree/bindings/pci/faraday,ftpci100.yaml
index 378dd1c8e2ee..fed393a89563 100644
--- a/Documentation/devicetree/bindings/pci/faraday,ftpci100.yaml
+++ b/Documentation/devicetree/bindings/pci/faraday,ftpci100.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Faraday Technology FTPCI100 PCI Host Bridge
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
     This PCI bridge is found inside that Cortina Systems Gemini SoC platform and
diff --git a/Documentation/devicetree/bindings/pci/intel,ixp4xx-pci.yaml b/Documentation/devicetree/bindings/pci/intel,ixp4xx-pci.yaml
index 3cae2e0f7f5e..c1806aef7bac 100644
--- a/Documentation/devicetree/bindings/pci/intel,ixp4xx-pci.yaml
+++ b/Documentation/devicetree/bindings/pci/intel,ixp4xx-pci.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Intel IXP4xx PCI controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: PCI host controller found in the Intel IXP4xx SoC series.
 
diff --git a/Documentation/devicetree/bindings/pci/v3,v360epc-pci.yaml b/Documentation/devicetree/bindings/pci/v3,v360epc-pci.yaml
index 38cac88f17bf..0e2ac2f8faed 100644
--- a/Documentation/devicetree/bindings/pci/v3,v360epc-pci.yaml
+++ b/Documentation/devicetree/bindings/pci/v3,v360epc-pci.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: V3 Semiconductor V360 EPC PCI bridge
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   This bridge is found in the ARM Integrator/AP (Application Platform)
diff --git a/Documentation/devicetree/bindings/pinctrl/pincfg-node.yaml b/Documentation/devicetree/bindings/pinctrl/pincfg-node.yaml
index d1bc389e0a6d..a916d0fc79a9 100644
--- a/Documentation/devicetree/bindings/pinctrl/pincfg-node.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/pincfg-node.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Generic Pin Configuration Node
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   Many data items that are represented in a pin configuration node are common
diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/pinctrl.yaml
index d471563119a9..290438826c50 100644
--- a/Documentation/devicetree/bindings/pinctrl/pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/pinctrl.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Pin controller device
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
   - Rafał Miłecki <rafal@milecki.pl>
 
 description: |
diff --git a/Documentation/devicetree/bindings/pinctrl/pinmux-node.yaml b/Documentation/devicetree/bindings/pinctrl/pinmux-node.yaml
index ca9d246d46fe..7ba26271c4d6 100644
--- a/Documentation/devicetree/bindings/pinctrl/pinmux-node.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/pinmux-node.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Generic Pin Multiplexing Node
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   The contents of the pin configuration child nodes are defined by the binding
diff --git a/Documentation/devicetree/bindings/power/supply/samsung,battery.yaml b/Documentation/devicetree/bindings/power/supply/samsung,battery.yaml
index 40292d581b10..fa1ccff043be 100644
--- a/Documentation/devicetree/bindings/power/supply/samsung,battery.yaml
+++ b/Documentation/devicetree/bindings/power/supply/samsung,battery.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Samsung SDI Batteries
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   Samsung SDI (Samsung Digital Interface) batteries are all different versions
diff --git a/Documentation/devicetree/bindings/regulator/adi,max77675.yaml b/Documentation/devicetree/bindings/regulator/adi,max77675.yaml
new file mode 100644
index 000000000000..c138e61380a4
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/adi,max77675.yaml
@@ -0,0 +1,184 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/adi,max77675.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Maxim MAX77675 PMIC Regulator
+
+maintainers:
+  - Joan Na <joan.na@analog.com>
+
+description:
+  The MAX77675 is a Power Management IC providing four switching buck
+  regulators (SBB0–SBB3) accessible via I2C. It supports configuration
+  of output voltages and enable controls for each regulator.
+
+allOf:
+  - $ref: /schemas/input/input.yaml
+  - $ref: /schemas/pinctrl/pincfg-node.yaml
+
+properties:
+  compatible:
+    const: adi,max77675
+
+  reg:
+    maxItems: 1
+
+  reset-time-sec:
+    description: Manual reset time in seconds
+    enum: [4, 8, 12, 16]
+    default: 4
+
+  bias-disable:
+    type: boolean
+    description: Disable internal pull-up for EN pin
+
+  input-debounce:
+    description: Debounce time for the enable pin, in microseconds
+    items:
+      - enum: [100, 30000]
+    default: 100
+
+  adi,en-mode:
+    description: |
+      Enable mode configuration.
+      The debounce time set by 'input-debounce' applies to
+      both push-button and slide-switch modes.
+      "push-button"  - A long press triggers power-on or power-down
+      "slide-switch" - Low : powers on, High : powers down
+      "logic"        - Low : powers on, High : powers down (no debounce time)
+    $ref: /schemas/types.yaml#/definitions/string
+    enum: [push-button, slide-switch, logic]
+    default: slide-switch
+
+  adi,voltage-change-latency-us:
+    description:
+      Specifies the delay (in microseconds) between an output voltage change
+      request and the start of the SBB voltage ramp.
+    enum: [10, 100]
+    default: 100
+
+  adi,drv-sbb-strength:
+    description: |
+      SIMO Buck-Boost Drive Strength Trim.
+      Controls the drive strength of the SIMO regulator's power MOSFETs.
+      This setting affects switching speed, impacting power efficiency and EMI.
+      "max"  – Maximum drive strength (~0.6 ns transition time)
+      "high" – High drive strength (~1.2 ns transition time)
+      "low"  – Low drive strength (~1.8 ns transition time)
+      "min"  – Minimum drive strength (~8 ns transition time)
+    $ref: /schemas/types.yaml#/definitions/string
+    enum: [max, high, low, min]
+    default: max
+
+  adi,dvs-slew-rate-mv-per-us:
+    description:
+      Dynamic rising slew rate for output voltage transitions, in mV/μs.
+      This setting is only used when 'adi,fixed-slew-rate' is not present.
+    enum: [5, 10]
+    default: 5
+
+  adi,bias-low-power-request:
+    type: boolean
+    description: Request low-power bias mode
+
+  adi,simo-ldo-always-on:
+    type: boolean
+    description: Set internal LDO to always supply 1.8V
+
+  regulators:
+    type: object
+    description: Regulator child nodes
+    patternProperties:
+      "^sbb[0-3]$":
+        type: object
+        $ref: regulator.yaml#
+        properties:
+          adi,fps-slot:
+            description: |
+              FPS (Flexible Power Sequencer) slot selection.
+              The Flexible Power Sequencer allows resources to power up under
+              hardware or software control. Additionally, each resource can
+              power up independently or among a group of other regulators with
+              adjustable power-up and power-down slots.
+              "slot0"   - Assign to FPS Slot 0
+              "slot1"   - Assign to FPS Slot 1
+              "slot2"   - Assign to FPS Slot 2
+              "slot3"   - Assign to FPS Slot 3
+              "default" - Use the default FPS slot value stored in register
+            $ref: /schemas/types.yaml#/definitions/string
+            enum: [slot0, slot1, slot2, slot3, default]
+            default: default
+
+          adi,fixed-slew-rate:
+            type: boolean
+            description:
+              When this property is present, the device uses a constant 2 mV/μs
+              slew rate and ignores any dynamic slew rate configuration.
+              When absent, the device uses the dynamic slew rate specified
+              by 'adi,dvs-slew-rate-mv-per-us'
+
+        unevaluatedProperties: false
+
+required:
+  - compatible
+  - reg
+  - regulators
+
+additionalProperties: false
+
+examples:
+  - |
+    i2c {
+      #address-cells = <1>;
+      #size-cells = <0>;
+
+      max77675: pmic@44 {
+        compatible = "adi,max77675";
+        reg = <0x44>;
+
+        reset-time-sec = <4>;
+        input-debounce = <100>;
+
+        adi,en-mode = "slide-switch";
+        adi,voltage-change-latency-us = <100>;
+        adi,drv-sbb-strength = "max";
+        adi,dvs-slew-rate-mv-per-us = <5>;
+
+        regulators {
+          sbb0: sbb0 {
+            regulator-name = "sbb0";
+            regulator-min-microvolt = <500000>;
+            regulator-max-microvolt = <5500000>;
+            adi,fps-slot = "default";
+            adi,fixed-slew-rate;
+          };
+
+          sbb1: sbb1 {
+            regulator-name = "sbb1";
+            regulator-min-microvolt = <500000>;
+            regulator-max-microvolt = <5500000>;
+            adi,fps-slot = "default";
+            adi,fixed-slew-rate;
+          };
+
+          sbb2: sbb2 {
+            regulator-name = "sbb2";
+            regulator-min-microvolt = <500000>;
+            regulator-max-microvolt = <5500000>;
+            adi,fps-slot = "default";
+            adi,fixed-slew-rate;
+          };
+
+          sbb3: sbb3 {
+            regulator-name = "sbb3";
+            regulator-min-microvolt = <500000>;
+            regulator-max-microvolt = <5500000>;
+            adi,fps-slot = "default";
+            adi,fixed-slew-rate;
+          };
+        };
+      };
+    };
+
diff --git a/Documentation/devicetree/bindings/remoteproc/ti,keystone-rproc.txt b/Documentation/devicetree/bindings/remoteproc/ti,keystone-rproc.txt
index 463a97c11eff..91f0a3b0c0b2 100644
--- a/Documentation/devicetree/bindings/remoteproc/ti,keystone-rproc.txt
+++ b/Documentation/devicetree/bindings/remoteproc/ti,keystone-rproc.txt
@@ -66,7 +66,7 @@ The following are the mandatory properties:
 - kick-gpios: 		Should specify the gpio device needed for the virtio IPC
 			stack. This will be used to interrupt the remote processor.
 			The gpio device to be used is as per the bindings in,
-			Documentation/devicetree/bindings/gpio/gpio-dsp-keystone.txt
+			Documentation/devicetree/bindings/gpio/ti,keystone-dsp-gpio.yaml
 
 SoC-specific Required properties:
 ---------------------------------
diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml
index 565cb2cbb49b..1aeff17c7f24 100644
--- a/Documentation/devicetree/bindings/riscv/extensions.yaml
+++ b/Documentation/devicetree/bindings/riscv/extensions.yaml
@@ -377,6 +377,20 @@ properties:
             guarantee on LR/SC sequences, as ratified in commit b1d806605f87
             ("Updated to ratified state.") of the riscv profiles specification.
 
+        - const: zilsd
+          description:
+            The standard Zilsd extension which provides support for aligned
+            register-pair load and store operations in 32-bit instruction
+            encodings, as ratified in commit f88abf1 ("Integrating
+            load/store pair for RV32 with the main manual") of riscv-isa-manual.
+
+        - const: zclsd
+          description:
+            The Zclsd extension implements the compressed (16-bit) version of the
+            Load/Store Pair for RV32. As with Zilsd, this extension was ratified
+            in commit f88abf1 ("Integrating load/store pair for RV32 with the
+            main manual") of riscv-isa-manual.
+
         - const: zk
           description:
             The standard Zk Standard Scalar cryptography extension as ratified
@@ -455,6 +469,20 @@ properties:
             The standard Zicboz extension for cache-block zeroing as ratified
             in commit 3dd606f ("Create cmobase-v1.0.pdf") of riscv-CMOs.
 
+        - const: zicfilp
+          description: |
+            The standard Zicfilp extension for enforcing forward edge
+            control-flow integrity as ratified in commit 3f8e450 ("merge
+            pull request #227 from ved-rivos/0709") of riscv-cfi
+            github repo.
+
+        - const: zicfiss
+          description: |
+            The standard Zicfiss extension for enforcing backward edge
+            control-flow integrity as ratified in commit 3f8e450 ("merge
+            pull request #227 from ved-rivos/0709") of riscv-cfi
+            github repo.
+
         - const: zicntr
           description:
             The standard Zicntr extension for base counters and timers, as
@@ -882,6 +910,16 @@ properties:
             anyOf:
               - const: v
               - const: zve32x
+      # Zclsd depends on Zilsd and Zca
+      - if:
+          contains:
+            anyOf:
+              - const: zclsd
+        then:
+          contains:
+            allOf:
+              - const: zilsd
+              - const: zca
 
 allOf:
   # Zcf extension does not exist on rv64
@@ -899,6 +937,18 @@ allOf:
           not:
             contains:
               const: zcf
+  # Zilsd extension does not exist on rv64
+  - if:
+      properties:
+        riscv,isa-base:
+          contains:
+            const: rv64i
+    then:
+      properties:
+        riscv,isa-extensions:
+          not:
+            contains:
+              const: zilsd
 
 additionalProperties: true
 ...
diff --git a/Documentation/devicetree/bindings/rng/intel,ixp46x-rng.yaml b/Documentation/devicetree/bindings/rng/intel,ixp46x-rng.yaml
index 9f7590ce6b3d..146593a669d6 100644
--- a/Documentation/devicetree/bindings/rng/intel,ixp46x-rng.yaml
+++ b/Documentation/devicetree/bindings/rng/intel,ixp46x-rng.yaml
@@ -12,7 +12,7 @@ description: |
   32 bit random number.
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/rng/samsung,exynos5250-trng.yaml b/Documentation/devicetree/bindings/rng/samsung,exynos5250-trng.yaml
index 1a71935d8a19..699831927932 100644
--- a/Documentation/devicetree/bindings/rng/samsung,exynos5250-trng.yaml
+++ b/Documentation/devicetree/bindings/rng/samsung,exynos5250-trng.yaml
@@ -12,9 +12,13 @@ maintainers:
 
 properties:
   compatible:
-    enum:
-      - samsung,exynos5250-trng
-      - samsung,exynos850-trng
+    oneOf:
+      - enum:
+          - samsung,exynos5250-trng
+          - samsung,exynos850-trng
+      - items:
+          - const: google,gs101-trng
+          - const: samsung,exynos850-trng
 
   clocks:
     minItems: 1
@@ -24,6 +28,9 @@ properties:
     minItems: 1
     maxItems: 2
 
+  power-domains:
+    maxItems: 1
+
   reg:
     maxItems: 1
 
diff --git a/Documentation/devicetree/bindings/rtc/faraday,ftrtc010.yaml b/Documentation/devicetree/bindings/rtc/faraday,ftrtc010.yaml
index b1c1a0e21318..2b1215b49580 100644
--- a/Documentation/devicetree/bindings/rtc/faraday,ftrtc010.yaml
+++ b/Documentation/devicetree/bindings/rtc/faraday,ftrtc010.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Faraday Technology FTRTC010 Real Time Clock
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: |
   This RTC appears in for example the Storlink Gemini family of SoCs.
diff --git a/Documentation/devicetree/bindings/serial/renesas,rsci.yaml b/Documentation/devicetree/bindings/serial/renesas,rsci.yaml
index 6b1f827a335b..1f8cee8171de 100644
--- a/Documentation/devicetree/bindings/serial/renesas,rsci.yaml
+++ b/Documentation/devicetree/bindings/serial/renesas,rsci.yaml
@@ -10,46 +10,72 @@ maintainers:
   - Geert Uytterhoeven <geert+renesas@glider.be>
   - Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com>
 
-allOf:
-  - $ref: serial.yaml#
-
 properties:
   compatible:
     oneOf:
-      - items:
-          - const: renesas,r9a09g087-rsci # RZ/N2H
-          - const: renesas,r9a09g077-rsci # RZ/T2H
+      - enum:
+          - renesas,r9a09g047-rsci # RZ/G3E
+          - renesas,r9a09g077-rsci # RZ/T2H
 
       - items:
+          - const: renesas,r9a09g087-rsci # RZ/N2H
           - const: renesas,r9a09g077-rsci # RZ/T2H
 
   reg:
     maxItems: 1
 
   interrupts:
+    minItems: 4
     items:
       - description: Error interrupt
       - description: Receive buffer full interrupt
       - description: Transmit buffer empty interrupt
       - description: Transmit end interrupt
+      - description: Active edge detection interrupt
+      - description: Break field detection interrupt
 
   interrupt-names:
+    minItems: 4
     items:
       - const: eri
       - const: rxi
       - const: txi
       - const: tei
+      - const: aed
+      - const: bfd
 
   clocks:
     minItems: 2
-    maxItems: 3
+    maxItems: 6
 
   clock-names:
-    minItems: 2
+    oneOf:
+      - items:
+          - const: operation
+          - const: bus
+          - const: sck # optional external clock input
+
+        minItems: 2
+
+      - items:
+          - const: pclk
+          - const: tclk
+          - const: tclk_div4
+          - const: tclk_div16
+          - const: tclk_div64
+          - const: sck # optional external clock input
+
+        minItems: 5
+
+  resets:
     items:
-      - const: operation
-      - const: bus
-      - const: sck # optional external clock input
+      - description: Input for resetting the APB clock
+      - description: Input for resetting TCLK
+
+  reset-names:
+    items:
+      - const: presetn
+      - const: tresetn
 
   power-domains:
     maxItems: 1
@@ -62,6 +88,57 @@ required:
   - clock-names
   - power-domains
 
+allOf:
+  - $ref: serial.yaml#
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: renesas,r9a09g077-rsci
+    then:
+      properties:
+        interrupts:
+          maxItems: 4
+
+        interrupt-names:
+          maxItems: 4
+
+        clocks:
+          minItems: 2
+          maxItems: 3
+
+        clock-names:
+          minItems: 2
+          maxItems: 3
+
+        resets: false
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: renesas,r9a09g047-rsci
+    then:
+      properties:
+        interrupts:
+          minItems: 6
+
+        interrupt-names:
+          minItems: 6
+
+        clocks:
+          minItems: 5
+          maxItems: 6
+
+        clock-names:
+          minItems: 5
+          maxItems: 6
+
+      required:
+        - resets
+        - reset-names
+
 unevaluatedProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/soc/samsung/exynos-pmu.yaml b/Documentation/devicetree/bindings/soc/samsung/exynos-pmu.yaml
index 6de47489ee42..2dd32db66783 100644
--- a/Documentation/devicetree/bindings/soc/samsung/exynos-pmu.yaml
+++ b/Documentation/devicetree/bindings/soc/samsung/exynos-pmu.yaml
@@ -34,9 +34,10 @@ select:
 properties:
   compatible:
     oneOf:
+      - enum:
+          - google,gs101-pmu
       - items:
           - enum:
-              - google,gs101-pmu
               - samsung,exynos3250-pmu
               - samsung,exynos4210-pmu
               - samsung,exynos4212-pmu
@@ -52,6 +53,7 @@ properties:
           - const: syscon
       - items:
           - enum:
+              - axis,artpec9-pmu
               - samsung,exynos2200-pmu
               - samsung,exynos7870-pmu
               - samsung,exynos7885-pmu
diff --git a/Documentation/devicetree/bindings/spi/andestech,ae350-spi.yaml b/Documentation/devicetree/bindings/spi/andestech,ae350-spi.yaml
new file mode 100644
index 000000000000..78093468dd5e
--- /dev/null
+++ b/Documentation/devicetree/bindings/spi/andestech,ae350-spi.yaml
@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/spi/andestech,ae350-spi.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Andes ATCSPI200 SPI controller
+
+maintainers:
+  - CL Wang <cl634@andestech.com>
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - enum:
+              - andestech,qilai-spi
+          - const: andestech,ae350-spi
+      - const: andestech,ae350-spi
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  num-cs:
+    description: Number of chip selects supported
+    maxItems: 1
+
+  dmas:
+    items:
+      - description: Transmit FIFO DMA channel
+      - description: Receive FIFO DMA channel
+
+  dma-names:
+    items:
+      - const: tx
+      - const: rx
+
+patternProperties:
+  "@[0-9a-f]+$":
+    type: object
+    additionalProperties: true
+
+    properties:
+      spi-rx-bus-width:
+        enum: [1, 4]
+
+      spi-tx-bus-width:
+        enum: [1, 4]
+
+allOf:
+  - $ref: spi-controller.yaml#
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - dmas
+  - dma-names
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    spi@f0b00000 {
+        compatible = "andestech,ae350-spi";
+        reg = <0xf0b00000 0x100>;
+        clocks = <&clk_spi>;
+        dmas = <&dma0 0>, <&dma0 1>;
+        dma-names = "tx", "rx";
+
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        flash@0 {
+            compatible = "jedec,spi-nor";
+            reg = <0>;
+            spi-tx-bus-width = <4>;
+            spi-rx-bus-width = <4>;
+            spi-cpol;
+            spi-cpha;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/spi/arm,pl022-peripheral-props.yaml b/Documentation/devicetree/bindings/spi/arm,pl022-peripheral-props.yaml
index bb8b6863b109..f976e416395b 100644
--- a/Documentation/devicetree/bindings/spi/arm,pl022-peripheral-props.yaml
+++ b/Documentation/devicetree/bindings/spi/arm,pl022-peripheral-props.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Peripheral-specific properties for Arm PL022 SPI controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 select: false
 
diff --git a/Documentation/devicetree/bindings/spi/nxp,imx94-xspi.yaml b/Documentation/devicetree/bindings/spi/nxp,imx94-xspi.yaml
new file mode 100644
index 000000000000..a0f4b162c858
--- /dev/null
+++ b/Documentation/devicetree/bindings/spi/nxp,imx94-xspi.yaml
@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/spi/nxp,imx94-xspi.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: NXP External Serial Peripheral Interface (xSPI)
+
+maintainers:
+  - Haibo Chen <haibo.chen@nxp.com>
+  - Han Xu <han.xu@nxp.com>
+
+properties:
+  compatible:
+    oneOf:
+      - enum:
+          - nxp,imx94-xspi
+
+  reg:
+    items:
+      - description: registers address space
+      - description: memory mapped address space
+
+  reg-names:
+    items:
+      - const: base
+      - const: mmap
+
+  interrupts:
+    items:
+      - description: interrupt for EENV0
+      - description: interrupt for EENV1
+      - description: interrupt for EENV2
+      - description: interrupt for EENV3
+      - description: interrupt for EENV4
+
+  clocks:
+    items:
+      - description: SPI serial clock
+
+  clock-names:
+    items:
+      - const: per
+
+required:
+  - compatible
+  - reg
+  - reg-names
+  - interrupts
+  - clocks
+  - clock-names
+
+allOf:
+  - $ref: spi-controller.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    soc {
+        #address-cells = <2>;
+        #size-cells = <2>;
+
+        spi@42b90000 {
+            compatible = "nxp,imx94-xspi";
+            reg = <0x0 0x42b90000 0x0 0x50000>, <0x0 0x28000000 0x0 0x08000000>;
+            reg-names = "base", "mmap";
+            interrupts = <GIC_SPI 390 IRQ_TYPE_LEVEL_HIGH>,
+                         <GIC_SPI 391 IRQ_TYPE_LEVEL_HIGH>,
+                         <GIC_SPI 392 IRQ_TYPE_LEVEL_HIGH>,
+                         <GIC_SPI 393 IRQ_TYPE_LEVEL_HIGH>,
+                         <GIC_SPI 394 IRQ_TYPE_LEVEL_HIGH>;
+            #address-cells = <1>;
+            #size-cells = <0>;
+            clocks = <&scmi_1>;
+            clock-names = "per";
+
+            flash@0 {
+                compatible = "jedec,spi-nor";
+                reg = <0>;
+                spi-max-frequency = <200000000>;
+                spi-rx-bus-width = <8>;
+                spi-tx-bus-width = <8>;
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/spi/renesas,rzv2h-rspi.yaml b/Documentation/devicetree/bindings/spi/renesas,rzv2h-rspi.yaml
index 069557a587b5..a588b112e11e 100644
--- a/Documentation/devicetree/bindings/spi/renesas,rzv2h-rspi.yaml
+++ b/Documentation/devicetree/bindings/spi/renesas,rzv2h-rspi.yaml
@@ -57,6 +57,14 @@ properties:
       - const: presetn
       - const: tresetn
 
+  dmas:
+    maxItems: 2
+
+  dma-names:
+    items:
+      - const: rx
+      - const: tx
+
   power-domains:
     maxItems: 1
 
diff --git a/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml b/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml
index 5c87fc8a845d..81838577cf9c 100644
--- a/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml
+++ b/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml
@@ -121,7 +121,7 @@ properties:
   num-cs:
     default: 4
     minimum: 1
-    maximum: 4
+    maximum: 16
 
   dmas:
     items:
@@ -153,14 +153,14 @@ properties:
       provides an interface to override the native DWC SSI CS control.
 
 patternProperties:
-  "@[0-9a-f]+$":
+  "@[0-9a-f]$":
     type: object
     additionalProperties: true
 
     properties:
       reg:
         minimum: 0
-        maximum: 3
+        maximum: 0xf
 
 unevaluatedProperties: false
 
diff --git a/Documentation/devicetree/bindings/spi/spi-pl022.yaml b/Documentation/devicetree/bindings/spi/spi-pl022.yaml
index 7f174b7d0a26..680fdfa184d0 100644
--- a/Documentation/devicetree/bindings/spi/spi-pl022.yaml
+++ b/Documentation/devicetree/bindings/spi/spi-pl022.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: ARM PL022 SPI controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: spi-controller.yaml#
diff --git a/Documentation/devicetree/bindings/spi/st,stm32-spi.yaml b/Documentation/devicetree/bindings/spi/st,stm32-spi.yaml
index ca880a226afa..472e92974714 100644
--- a/Documentation/devicetree/bindings/spi/st,stm32-spi.yaml
+++ b/Documentation/devicetree/bindings/spi/st,stm32-spi.yaml
@@ -96,6 +96,9 @@ properties:
       The region should be defined as child node of the AHB SRAM node
       as per the generic bindings in Documentation/devicetree/bindings/sram/sram.yaml
 
+  power-domains:
+    maxItems: 1
+
   access-controllers:
     minItems: 1
     maxItems: 2
diff --git a/Documentation/devicetree/bindings/sram/sram.yaml b/Documentation/devicetree/bindings/sram/sram.yaml
index 7c1337e159f2..c451140962c8 100644
--- a/Documentation/devicetree/bindings/sram/sram.yaml
+++ b/Documentation/devicetree/bindings/sram/sram.yaml
@@ -34,6 +34,7 @@ properties:
         - nvidia,tegra186-sysram
         - nvidia,tegra194-sysram
         - nvidia,tegra234-sysram
+        - qcom,kaanapali-imem
         - qcom,rpm-msg-ram
         - rockchip,rk3288-pmu-sram
 
@@ -89,6 +90,7 @@ patternProperties:
             - arm,juno-scp-shmem
             - arm,scmi-shmem
             - arm,scp-shmem
+            - qcom,pil-reloc-info
             - renesas,smp-sram
             - rockchip,rk3066-smp-sram
             - samsung,exynos4210-sysram
diff --git a/Documentation/devicetree/bindings/timer/faraday,fttmr010.yaml b/Documentation/devicetree/bindings/timer/faraday,fttmr010.yaml
index 39506323556c..e93c20243dba 100644
--- a/Documentation/devicetree/bindings/timer/faraday,fttmr010.yaml
+++ b/Documentation/devicetree/bindings/timer/faraday,fttmr010.yaml
@@ -8,7 +8,7 @@ title: Faraday FTTMR010 timer
 
 maintainers:
   - Joel Stanley <joel@jms.id.au>
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description:
   This timer is a generic IP block from Faraday Technology, embedded in the
diff --git a/Documentation/devicetree/bindings/timer/intel,ixp4xx-timer.yaml b/Documentation/devicetree/bindings/timer/intel,ixp4xx-timer.yaml
index 526b8db4d575..c92e6b9cd5e2 100644
--- a/Documentation/devicetree/bindings/timer/intel,ixp4xx-timer.yaml
+++ b/Documentation/devicetree/bindings/timer/intel,ixp4xx-timer.yaml
@@ -8,7 +8,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Intel IXP4xx XScale Networking Processors Timers
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: This timer is found in the Intel IXP4xx processors.
 
diff --git a/Documentation/devicetree/bindings/timer/st,nomadik-mtu.yaml b/Documentation/devicetree/bindings/timer/st,nomadik-mtu.yaml
index fa65878b3571..873a01c287f4 100644
--- a/Documentation/devicetree/bindings/timer/st,nomadik-mtu.yaml
+++ b/Documentation/devicetree/bindings/timer/st,nomadik-mtu.yaml
@@ -8,7 +8,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: ST Microelectronics Nomadik Multi-Timer Unit MTU Timer
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 description: This timer is found in the ST Microelectronics Nomadik
   SoCs STn8800, STn8810 and STn8815 as well as in ST-Ericsson DB8500.
diff --git a/Documentation/devicetree/bindings/trivial-devices.yaml b/Documentation/devicetree/bindings/trivial-devices.yaml
index d0f7dbf15d6f..1b356cb674d2 100644
--- a/Documentation/devicetree/bindings/trivial-devices.yaml
+++ b/Documentation/devicetree/bindings/trivial-devices.yaml
@@ -414,6 +414,8 @@ properties:
           - smsc,emc6d103
             # Temperature sensor with integrated fan control
           - smsc,emc6d103s
+            # Socionext Uniphier SMP control registers
+          - socionext,uniphier-smpctrl
             # SparkFun Qwiic Joystick (COM-15168) with i2c interface
           - sparkfun,qwiic-joystick
             # Sierra Wireless mangOH Green SPI IoT interface
diff --git a/Documentation/devicetree/bindings/usb/aspeed,usb-vhub.yaml b/Documentation/devicetree/bindings/usb/aspeed,usb-vhub.yaml
index 7f22f9c031b2..b8bac2cce949 100644
--- a/Documentation/devicetree/bindings/usb/aspeed,usb-vhub.yaml
+++ b/Documentation/devicetree/bindings/usb/aspeed,usb-vhub.yaml
@@ -17,8 +17,8 @@ description: |+
 
   Supported number of devices and endpoints vary depending on hardware
   revisions. AST2400 and AST2500 Virtual Hub supports 5 downstream devices
-  and 15 generic endpoints, while AST2600 Virtual Hub supports 7 downstream
-  devices and 21 generic endpoints.
+  and 15 generic endpoints, while AST2600 and AST2700 Virtual Hub supports
+  7 downstream devices and 21 generic endpoints.
 
 properties:
   compatible:
@@ -26,6 +26,7 @@ properties:
       - aspeed,ast2400-usb-vhub
       - aspeed,ast2500-usb-vhub
       - aspeed,ast2600-usb-vhub
+      - aspeed,ast2700-usb-vhub
 
   reg:
     maxItems: 1
@@ -33,6 +34,9 @@ properties:
   clocks:
     maxItems: 1
 
+  resets:
+    maxItems: 1
+
   interrupts:
     maxItems: 1
 
@@ -107,6 +111,20 @@ required:
   - aspeed,vhub-downstream-ports
   - aspeed,vhub-generic-endpoints
 
+if:
+  properties:
+    compatible:
+      contains:
+        const: aspeed,ast2700-usb-vhub
+
+then:
+  required:
+    - resets
+
+else:
+  properties:
+    resets: false
+
 additionalProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/usb/faraday,fotg210.yaml b/Documentation/devicetree/bindings/usb/faraday,fotg210.yaml
index 3fe4d1564dfe..b97ba535087c 100644
--- a/Documentation/devicetree/bindings/usb/faraday,fotg210.yaml
+++ b/Documentation/devicetree/bindings/usb/faraday,fotg210.yaml
@@ -8,7 +8,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Faraday Technology FOTG200 series HS OTG USB 2.0 controller
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: usb-drd.yaml#
diff --git a/Documentation/devicetree/bindings/usb/generic-ehci.yaml b/Documentation/devicetree/bindings/usb/generic-ehci.yaml
index 4e84bead0232..601f097c09a6 100644
--- a/Documentation/devicetree/bindings/usb/generic-ehci.yaml
+++ b/Documentation/devicetree/bindings/usb/generic-ehci.yaml
@@ -93,6 +93,8 @@ properties:
     minItems: 1
     maxItems: 2
 
+  dma-coherent: true
+
   interrupts:
     maxItems: 1
 
diff --git a/Documentation/devicetree/bindings/usb/generic-ohci.yaml b/Documentation/devicetree/bindings/usb/generic-ohci.yaml
index 3ee1586fc8b9..961cbf85eeb5 100644
--- a/Documentation/devicetree/bindings/usb/generic-ohci.yaml
+++ b/Documentation/devicetree/bindings/usb/generic-ohci.yaml
@@ -64,6 +64,8 @@ properties:
   reg:
     maxItems: 1
 
+  dma-coherent: true
+
   interrupts:
     maxItems: 1
 
diff --git a/Documentation/devicetree/bindings/usb/intel,ixp4xx-udc.yaml b/Documentation/devicetree/bindings/usb/intel,ixp4xx-udc.yaml
index 4ed602746897..91a149ad3ad6 100644
--- a/Documentation/devicetree/bindings/usb/intel,ixp4xx-udc.yaml
+++ b/Documentation/devicetree/bindings/usb/intel,ixp4xx-udc.yaml
@@ -10,7 +10,7 @@ description: The IXP4xx SoCs has a full-speed USB Device
   Controller with 16 endpoints and a built-in transceiver.
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml
index c7591b2aec2a..b7c6c528e018 100644
--- a/Documentation/devicetree/bindings/vendor-prefixes.yaml
+++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml
@@ -995,6 +995,8 @@ patternProperties:
     description: Mustek Limited
   "^mediatek,.*":
     description: MediaTek Inc.
+  "^medion,.*":
+    description: Medion AG
   "^megachips,.*":
     description: MegaChips
   "^mele,.*":
@@ -1697,6 +1699,8 @@ patternProperties:
     description: Theobroma Systems Design und Consulting GmbH
   "^turing,.*":
     description: Turing Machines, Inc.
+  "^tuxedo,.*":
+    description: TUXEDO Computers GmbH
   "^tyan,.*":
     description: Tyan Computer Corporation
   "^tyhx,.*":
diff --git a/Documentation/devicetree/bindings/watchdog/faraday,ftwdt010.yaml b/Documentation/devicetree/bindings/watchdog/faraday,ftwdt010.yaml
index 726dc872ad02..3eb35f325f4c 100644
--- a/Documentation/devicetree/bindings/watchdog/faraday,ftwdt010.yaml
+++ b/Documentation/devicetree/bindings/watchdog/faraday,ftwdt010.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Faraday Technology FTWDT010 watchdog
 
 maintainers:
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
   - Corentin Labbe <clabbe@baylibre.com>
 
 description: |
diff --git a/Documentation/devicetree/bindings/watchdog/maxim,max63xx.yaml b/Documentation/devicetree/bindings/watchdog/maxim,max63xx.yaml
index 442c21f12a3b..defe0401ded0 100644
--- a/Documentation/devicetree/bindings/watchdog/maxim,max63xx.yaml
+++ b/Documentation/devicetree/bindings/watchdog/maxim,max63xx.yaml
@@ -8,7 +8,7 @@ title: Maxim 63xx Watchdog Timers
 
 maintainers:
   - Marc Zyngier <maz@kernel.org>
-  - Linus Walleij <linus.walleij@linaro.org>
+  - Linus Walleij <linusw@kernel.org>
 
 allOf:
   - $ref: watchdog.yaml#
diff --git a/Documentation/devicetree/bindings/watchdog/xlnx,versal-wwdt.yaml b/Documentation/devicetree/bindings/watchdog/xlnx,versal-wwdt.yaml
index 14b069599740..fccfc785a077 100644
--- a/Documentation/devicetree/bindings/watchdog/xlnx,versal-wwdt.yaml
+++ b/Documentation/devicetree/bindings/watchdog/xlnx,versal-wwdt.yaml
@@ -32,6 +32,9 @@ properties:
   clocks:
     maxItems: 1
 
+  power-domains:
+    maxItems: 1
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/writing-schema.rst b/Documentation/devicetree/bindings/writing-schema.rst
index 05c34248e544..8599ea2d6771 100644
--- a/Documentation/devicetree/bindings/writing-schema.rst
+++ b/Documentation/devicetree/bindings/writing-schema.rst
@@ -214,10 +214,18 @@ binding schema. All of the DT binding documents can be validated using the
 
     make dt_binding_check
 
+Or to validate a single schema and its example::
+
+    make sram/sram.yaml
+
 In order to perform validation of DT source files, use the ``dtbs_check`` target::
 
     make dtbs_check
 
+Or to validate a single DTB target::
+
+    make CHECK_DTBS=y arm/fvp-base-revc.dtb
+
 Note that ``dtbs_check`` will skip any binding schema files with errors. It is
 necessary to use ``dt_binding_check`` to get all the validation errors in the
 binding schema files.
@@ -226,10 +234,10 @@ It is possible to run both in a single command::
 
     make dt_binding_check dtbs_check
 
-It is also possible to run checks with a subset of matching schema files by
-setting the ``DT_SCHEMA_FILES`` variable to 1 or more specific schema files or
-patterns (partial match of a fixed string). Each file or pattern should be
-separated by ':'.
+It is also possible to combine running the above commands with a subset of
+matching schema files by setting the ``DT_SCHEMA_FILES`` variable to 1 or more
+specific schema files or patterns (partial match of a fixed string). Each file
+or pattern should be separated by ':'.
 
 ::
 
diff --git a/Documentation/driver-api/dmaengine/provider.rst b/Documentation/driver-api/dmaengine/provider.rst
index 1594598b3317..f4ed98f701c9 100644
--- a/Documentation/driver-api/dmaengine/provider.rst
+++ b/Documentation/driver-api/dmaengine/provider.rst
@@ -411,7 +411,7 @@ supported.
   - This structure can be initialized using the function
     ``dma_async_tx_descriptor_init``.
 
-  - You'll also need to set two fields in this structure:
+  - You'll also need to set following fields in this structure:
 
     - flags:
       TODO: Can it be modified by the driver itself, or
@@ -421,6 +421,9 @@ supported.
       that is supposed to push the current transaction descriptor to a
       pending queue, waiting for issue_pending to be called.
 
+    - phys: Physical address of the descriptor which is used later by
+      the dma engine to read the descriptor and initiate transfer.
+
   - In this structure the function pointer callback_result can be
     initialized in order for the submitter to be notified that a
     transaction has completed. In the earlier code the function pointer
diff --git a/Documentation/driver-api/gpio/pca953x.rst b/Documentation/driver-api/gpio/pca953x.rst
index 4bd7cf1120cb..53f25fa03072 100644
--- a/Documentation/driver-api/gpio/pca953x.rst
+++ b/Documentation/driver-api/gpio/pca953x.rst
@@ -178,6 +178,8 @@ pcal9554b      8       yes    00     01     02        03
 pcal6416      16       yes    00     02     04        06
 pcal9535      16       yes    00     02     04        06
 pcal9555a     16       yes    00     02     04        06
+tcal6408       8       yes    00     01     02        03
+tcal6416      16       yes    00     02     04        06
 ========== ===== ========= ===== ====== ====== =========
 
 These chips have several additional features:
@@ -196,6 +198,8 @@ pcal9554b            40       42      43       44       45         46       4F
 pcal6416             40       44      46       48       4A         4C       4F
 pcal9535             40       44      46       48       4A         4C       4F
 pcal9555a            40       44      46       48       4A         4C       4F
+tcal6408             40       42      43       44       45         46       4F
+tcal6416             40       44      46       48       4A         4C       4F
 ========== ============ ======== ======= ======== ======== ========== ========
 
 Currently the driver has support for the input latch, pull-up/pull-down
@@ -332,6 +336,8 @@ Layouts:
         - pcal9554b
         - pcal9555a
         - pcal6524
+        - tcal6408
+        - tcal6416
 
     2. base offset 0x30, bank 5 and 6, closely packed banks
         - pcal6534
@@ -390,6 +396,8 @@ enabled.
         - pcal9554b
         - pcal9555a
         - pcal6524
+        - tcal6408
+        - tcal6416
 
     2. base offset 0x30, bank 2, closely packed banks
         - pcal6534
@@ -462,6 +470,8 @@ Layout:
         - pcal9535
         - pcal9554b
         - pcal9555a
+        - tcal6408
+        - tcal6416
 
 `PCAL chips with extended interrupt and output configuration functions`_
 can set this for each line individually. They have the same per-port out_conf
@@ -505,6 +515,8 @@ bits drive strength
         - pcal9554b
         - pcal9555a
         - pcal6524
+        - tcal6408
+        - tcal6416
 
     2. base offset 0x30, bank 0 and 1, closely packed banks
         - pcal6534
diff --git a/Documentation/driver-api/media/v4l2-dev.rst b/Documentation/driver-api/media/v4l2-dev.rst
index d5cb19b21a9f..dd239ad42051 100644
--- a/Documentation/driver-api/media/v4l2-dev.rst
+++ b/Documentation/driver-api/media/v4l2-dev.rst
@@ -157,10 +157,10 @@ changing the e.g. exposure of the webcam.
 Of course, you can always do all the locking yourself by leaving both lock
 pointers at ``NULL``.
 
-In the case of :ref:`videobuf2 <vb2_framework>` you will need to implement the
-``wait_prepare()`` and ``wait_finish()`` callbacks to unlock/lock if applicable.
-If you use the ``queue->lock`` pointer, then you can use the helper functions
-:c:func:`vb2_ops_wait_prepare` and :c:func:`vb2_ops_wait_finish`.
+In the case of :ref:`videobuf2 <vb2_framework>` you must set the ``queue->lock``
+pointer to the lock you use to serialize the queuing ioctls. This ensures that
+that lock is released while waiting in ``VIDIOC_DQBUF`` for a buffer to arrive,
+and it is retaken afterwards.
 
 The implementation of a hotplug disconnect should also take the lock from
 :c:type:`video_device` before calling v4l2_device_disconnect. If you are also
diff --git a/Documentation/driver-api/tee.rst b/Documentation/driver-api/tee.rst
index 5eaeb8103988..4d58ac0712c1 100644
--- a/Documentation/driver-api/tee.rst
+++ b/Documentation/driver-api/tee.rst
@@ -43,24 +43,12 @@ snippet would look like::
 	MODULE_DEVICE_TABLE(tee, client_id_table);
 
 	static struct tee_client_driver client_driver = {
+		.probe		= client_probe,
+		.remove		= client_remove,
 		.id_table	= client_id_table,
 		.driver		= {
 			.name		= DRIVER_NAME,
-			.bus		= &tee_bus_type,
-			.probe		= client_probe,
-			.remove		= client_remove,
 		},
 	};
 
-	static int __init client_init(void)
-	{
-		return driver_register(&client_driver.driver);
-	}
-
-	static void __exit client_exit(void)
-	{
-		driver_unregister(&client_driver.driver);
-	}
-
-	module_init(client_init);
-	module_exit(client_exit);
+	module_tee_client_driver(client_driver);
diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst
index cb90d1ae82d0..9b3b835a174e 100644
--- a/Documentation/filesystems/f2fs.rst
+++ b/Documentation/filesystems/f2fs.rst
@@ -1033,3 +1033,46 @@ the reserved space back to F2FS for its own use.
 So, the key idea is, user can do any file operations on /dev/vdc, and
 reclaim the space after the use, while the space is counted as /data.
 That doesn't require modifying partition size and filesystem format.
+
+Per-file Read-Only Large Folio Support
+--------------------------------------
+
+F2FS implements large folio support on the read path to leverage high-order
+page allocation for significant performance gains. To minimize code complexity,
+this support is currently excluded from the write path, which requires handling
+complex optimizations such as compression and block allocation modes.
+
+This optional feature is triggered only when a file's immutable bit is set.
+Consequently, F2FS will return EOPNOTSUPP if a user attempts to open a cached
+file with write permissions, even immediately after clearing the bit. Write
+access is only restored once the cached inode is dropped. The usage flow is
+demonstrated below:
+
+.. code-block::
+
+   # f2fs_io setflags immutable /data/testfile_read_seq
+
+   /* flush and reload the inode to enable the large folio */
+   # sync && echo 3 > /proc/sys/vm/drop_caches
+
+   /* mmap(MAP_POPULATE) + mlock() */
+   # f2fs_io read 128 0 1024 mmap 1 0 /data/testfile_read_seq
+
+   /* mmap() + fadvise(POSIX_FADV_WILLNEED) + mlock() */
+   # f2fs_io read 128 0 1024 fadvise 1 0 /data/testfile_read_seq
+
+   /* mmap() + mlock2(MLOCK_ONFAULT) + madvise(MADV_POPULATE_READ) */
+   # f2fs_io read 128 0 1024 madvise 1 0 /data/testfile_read_seq
+
+   # f2fs_io clearflags immutable /data/testfile_read_seq
+
+   # f2fs_io write 1 0 1 zero buffered /data/testfile_read_seq
+   Failed to open /mnt/test/test: Operation not supported
+
+   /* flush and reload the inode to disable the large folio */
+   # sync && echo 3 > /proc/sys/vm/drop_caches
+
+   # f2fs_io write 1 0 1 zero buffered /data/testfile_read_seq
+   Written 4096 bytes with pattern = zero, total_time = 29 us, max_latency = 28 us
+
+   # rm /data/testfile_read_seq
diff --git a/Documentation/filesystems/fscrypt.rst b/Documentation/filesystems/fscrypt.rst
index 70af896822e1..c0dd35f1af12 100644
--- a/Documentation/filesystems/fscrypt.rst
+++ b/Documentation/filesystems/fscrypt.rst
@@ -455,11 +455,6 @@ API, but the filenames mode still does.
 - Adiantum
     - Mandatory:
         - CONFIG_CRYPTO_ADIANTUM
-    - Recommended:
-        - arm32: CONFIG_CRYPTO_NHPOLY1305_NEON
-        - arm64: CONFIG_CRYPTO_NHPOLY1305_NEON
-        - x86: CONFIG_CRYPTO_NHPOLY1305_SSE2
-        - x86: CONFIG_CRYPTO_NHPOLY1305_AVX2
 
 - AES-128-CBC-ESSIV and AES-128-CBC-CTS:
     - Mandatory:
diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst
index 77704fde9845..35e97618868b 100644
--- a/Documentation/filesystems/locking.rst
+++ b/Documentation/filesystems/locking.rst
@@ -177,7 +177,6 @@ prototypes::
 	int (*freeze_fs) (struct super_block *);
 	int (*unfreeze_fs) (struct super_block *);
 	int (*statfs) (struct dentry *, struct kstatfs *);
-	int (*remount_fs) (struct super_block *, int *, char *);
 	void (*umount_begin) (struct super_block *);
 	int (*show_options)(struct seq_file *, struct dentry *);
 	ssize_t (*quota_read)(struct super_block *, int, char *, size_t, loff_t);
@@ -201,7 +200,6 @@ sync_fs:		read
 freeze_fs:		write
 unfreeze_fs:		write
 statfs:			maybe(read)	(see below)
-remount_fs:		write
 umount_begin:		no
 show_options:		no		(namespace_sem)
 quota_read:		no		(see below)
@@ -226,8 +224,6 @@ file_system_type
 
 prototypes::
 
-	struct dentry *(*mount) (struct file_system_type *, int,
-		       const char *, void *);
 	void (*kill_sb) (struct super_block *);
 
 locking rules:
@@ -235,13 +231,9 @@ locking rules:
 =======		=========
 ops		may block
 =======		=========
-mount		yes
 kill_sb		yes
 =======		=========
 
-->mount() returns ERR_PTR or the root dentry; its superblock should be locked
-on return.
-
 ->kill_sb() takes a write-locked superblock, does all shutdown work on it,
 unlocks and drops the reference.
 
@@ -416,6 +408,7 @@ lm_change		yes		no			no
 lm_breaker_owns_lease:	yes     	no			no
 lm_lock_expirable	yes		no			no
 lm_expire_lock		no		no			yes
+lm_open_conflict	yes		no			no
 ======================	=============	=================	=========
 
 buffer_head
diff --git a/Documentation/filesystems/mount_api.rst b/Documentation/filesystems/mount_api.rst
index c99ab1f7fea4..a064234fed5b 100644
--- a/Documentation/filesystems/mount_api.rst
+++ b/Documentation/filesystems/mount_api.rst
@@ -299,8 +299,6 @@ manage the filesystem context.  They are as follows:
      On success it should return 0.  In the case of an error, it should return
      a negative error code.
 
-     .. Note:: reconfigure is intended as a replacement for remount_fs.
-
 
 Filesystem context Security
 ===========================
diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst
index 3397937ed838..631eee9bdc33 100644
--- a/Documentation/filesystems/porting.rst
+++ b/Documentation/filesystems/porting.rst
@@ -448,11 +448,8 @@ a file off.
 
 **mandatory**
 
-->get_sb() is gone.  Switch to use of ->mount().  Typically it's just
-a matter of switching from calling ``get_sb_``... to ``mount_``... and changing
-the function type.  If you were doing it manually, just switch from setting
-->mnt_root to some pointer to returning that pointer.  On errors return
-ERR_PTR(...).
+->get_sb() and ->mount() are gone. Switch to using the new mount API. See
+Documentation/filesystems/mount_api.rst for more details.
 
 ---
 
diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst
index 670ba66b60e4..90c357b263fe 100644
--- a/Documentation/filesystems/vfs.rst
+++ b/Documentation/filesystems/vfs.rst
@@ -94,11 +94,9 @@ functions:
 
 The passed struct file_system_type describes your filesystem.  When a
 request is made to mount a filesystem onto a directory in your
-namespace, the VFS will call the appropriate mount() method for the
-specific filesystem.  New vfsmount referring to the tree returned by
-->mount() will be attached to the mountpoint, so that when pathname
-resolution reaches the mountpoint it will jump into the root of that
-vfsmount.
+namespace, the VFS will call the appropriate get_tree() method for the
+specific filesystem.  See Documentation/filesystems/mount_api.rst
+for more details.
 
 You can see all filesystems that are registered to the kernel in the
 file /proc/filesystems.
@@ -117,8 +115,6 @@ members are defined:
 		int fs_flags;
 		int (*init_fs_context)(struct fs_context *);
 		const struct fs_parameter_spec *parameters;
-		struct dentry *(*mount) (struct file_system_type *, int,
-			const char *, void *);
 		void (*kill_sb) (struct super_block *);
 		struct module *owner;
 		struct file_system_type * next;
@@ -151,10 +147,6 @@ members are defined:
 	'struct fs_parameter_spec'.
 	More info in Documentation/filesystems/mount_api.rst.
 
-``mount``
-	the method to call when a new instance of this filesystem should
-	be mounted
-
 ``kill_sb``
 	the method to call when an instance of this filesystem should be
 	shut down
@@ -173,45 +165,6 @@ members are defined:
   s_lock_key, s_umount_key, s_vfs_rename_key, s_writers_key,
   i_lock_key, i_mutex_key, invalidate_lock_key, i_mutex_dir_key: lockdep-specific
 
-The mount() method has the following arguments:
-
-``struct file_system_type *fs_type``
-	describes the filesystem, partly initialized by the specific
-	filesystem code
-
-``int flags``
-	mount flags
-
-``const char *dev_name``
-	the device name we are mounting.
-
-``void *data``
-	arbitrary mount options, usually comes as an ASCII string (see
-	"Mount Options" section)
-
-The mount() method must return the root dentry of the tree requested by
-caller.  An active reference to its superblock must be grabbed and the
-superblock must be locked.  On failure it should return ERR_PTR(error).
-
-The arguments match those of mount(2) and their interpretation depends
-on filesystem type.  E.g. for block filesystems, dev_name is interpreted
-as block device name, that device is opened and if it contains a
-suitable filesystem image the method creates and initializes struct
-super_block accordingly, returning its root dentry to caller.
-
-->mount() may choose to return a subtree of existing filesystem - it
-doesn't have to create a new one.  The main result from the caller's
-point of view is a reference to dentry at the root of (sub)tree to be
-attached; creation of new superblock is a common side effect.
-
-The most interesting member of the superblock structure that the mount()
-method fills in is the "s_op" field.  This is a pointer to a "struct
-super_operations" which describes the next level of the filesystem
-implementation.
-
-For more information on mounting (and the new mount API), see
-Documentation/filesystems/mount_api.rst.
-
 The Superblock Object
 =====================
 
@@ -244,7 +197,6 @@ filesystem.  The following members are defined:
 					enum freeze_wholder who);
 		int (*unfreeze_fs) (struct super_block *);
 		int (*statfs) (struct dentry *, struct kstatfs *);
-		int (*remount_fs) (struct super_block *, int *, char *);
 		void (*umount_begin) (struct super_block *);
 
 		int (*show_options)(struct seq_file *, struct dentry *);
@@ -351,10 +303,6 @@ or bottom half).
 ``statfs``
 	called when the VFS needs to get filesystem statistics.
 
-``remount_fs``
-	called when the filesystem is remounted.  This is called with
-	the kernel lock held
-
 ``umount_begin``
 	called when the VFS is unmounting a filesystem.
 
diff --git a/Documentation/gpu/amdgpu/amd_overview_block.svg b/Documentation/gpu/amdgpu/amd_overview_block.svg
new file mode 100644
index 000000000000..cbd705afc9e2
--- /dev/null
+++ b/Documentation/gpu/amdgpu/amd_overview_block.svg
@@ -0,0 +1,687 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   width="237.4014mm"
+   height="160.98259mm"
+   viewBox="0 0 237.4014 160.98259"
+   version="1.1"
+   id="svg1"
+   inkscape:version="1.4.2 (ebf0e940d0, 2025-05-08)"
+   sodipodi:docname="amd_overview_block.svg"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview1"
+     pagecolor="#ffffff"
+     bordercolor="#000000"
+     borderopacity="0.25"
+     inkscape:showpageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1"
+     inkscape:document-units="mm"
+     inkscape:zoom="2"
+     inkscape:cx="576.75"
+     inkscape:cy="313.25"
+     inkscape:window-width="3840"
+     inkscape:window-height="2083"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="layer1" />
+  <defs
+     id="defs1">
+    <marker
+       style="overflow:visible"
+       id="ArrowWideHeavy"
+       refX="0"
+       refY="0"
+       orient="auto-start-reverse"
+       inkscape:stockid="Wide, heavy arrow"
+       markerWidth="1"
+       markerHeight="1"
+       viewBox="0 0 1 1"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         style="fill:context-stroke;fill-rule:evenodd;stroke:none"
+         d="m 1,0 -3,3 h -2 l 3,-3 -3,-3 h 2 z"
+         id="path3" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="ArrowWideRounded"
+       refX="0"
+       refY="0"
+       orient="auto-start-reverse"
+       inkscape:stockid="Wide, rounded arrow"
+       markerWidth="1"
+       markerHeight="1"
+       viewBox="0 0 1 1"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         style="fill:none;stroke:context-stroke;stroke-width:1;stroke-linecap:round"
+         d="M 3,-3 0,0 3,3"
+         transform="rotate(180,0.125,0)"
+         sodipodi:nodetypes="ccc"
+         id="path2" />
+    </marker>
+  </defs>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(9.9255824,-64.69615)">
+    <rect
+       style="fill:#ffffff;stroke:none;stroke-width:0.999747"
+       id="rect5"
+       width="239.13895"
+       height="162.38739"
+       x="-10.311751"
+       y="63.871342" />
+    <rect
+       style="fill:#ffffff;stroke:#000000;stroke-width:0.79375"
+       id="rect1"
+       width="174.55814"
+       height="140.23256"
+       x="22.263056"
+       y="65.093025" />
+    <rect
+       style="fill:#00d400;stroke:#00d400;stroke-width:0.348444;stroke-dasharray:none"
+       id="rect8-5"
+       width="3.8659263"
+       height="4.2845292"
+       x="38.69939"
+       y="75.819946" />
+    <rect
+       style="fill:#00d400;stroke:#00d400;stroke-width:0.348444;stroke-dasharray:none"
+       id="rect9"
+       width="3.8659263"
+       height="4.2845292"
+       x="176.7458"
+       y="75.68573" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.9389px;font-family:'Linux Libertine O';-inkscape-font-specification:'Linux Libertine O';text-align:start;letter-spacing:4.91331px;writing-mode:lr-tb;direction:ltr;text-anchor:start;fill:#00d400;stroke:#00d400;stroke-width:1;stroke-dasharray:none"
+       x="45.418606"
+       y="216.62791"
+       id="text12"><tspan
+         sodipodi:role="line"
+         id="tspan12"
+         style="stroke-width:1"
+         x="45.418606"
+         y="216.62791" /></text>
+    <g
+       id="g34">
+      <rect
+         style="fill:#ffffff;stroke:#008033;stroke-width:1;stroke-dasharray:none"
+         id="rect7"
+         width="19.539951"
+         height="6.9818101"
+         x="22.811832"
+         y="210.3201" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008033;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+         x="52.663685"
+         y="216.07796"
+         id="text68"><tspan
+           sodipodi:role="line"
+           x="52.663685"
+           y="216.07796"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#008033;stroke-width:0.0690111"
+           id="tspan68">UMC</tspan></text>
+    </g>
+    <g
+       id="g29"
+       transform="translate(-0.52916667)">
+      <rect
+         style="fill:#00d400;stroke:#00d400;stroke-width:0.348444;stroke-dasharray:none"
+         id="rect8"
+         width="3.8659263"
+         height="4.2845292"
+         x="22.782616"
+         y="220.36148" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+         x="48.966854"
+         y="224.15474"
+         id="text13"><tspan
+           sodipodi:role="line"
+           x="48.966854"
+           y="224.15474"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#00ff00;stroke-width:0.0690111"
+           id="tspan13">Memory hub</tspan></text>
+    </g>
+    <g
+       id="g12"
+       transform="translate(-1.8520837,2.3812496)">
+      <g
+         id="g13">
+        <rect
+           style="fill:#ffffff;stroke:#ffd42a;stroke-width:1;stroke-dasharray:none"
+           id="rect15"
+           width="28.674419"
+           height="11.302325"
+           x="114.87544"
+           y="169.54433"
+           ry="2.6458333" />
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+           x="129.06978"
+           y="177.46243"
+           id="text15"><tspan
+             sodipodi:role="line"
+             x="129.06978"
+             y="177.46243"
+             style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+             id="tspan15">PSP</tspan></text>
+      </g>
+    </g>
+    <path
+       style="fill:none;stroke:#ffd42a;stroke-width:1;stroke-dasharray:8, 1;stroke-dashoffset:0"
+       d="m 73.88372,201.92338 h 43.74419 V 184.5631"
+       id="path16"
+       sodipodi:nodetypes="ccc" />
+    <rect
+       style="fill:#ffffff;stroke:#00ccff;stroke-width:1;stroke-dasharray:none"
+       id="rect16"
+       width="40.220226"
+       height="11.1272"
+       x="150.82011"
+       y="193.07373" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+       x="170.78101"
+       y="200.90428"
+       id="text16"><tspan
+         sodipodi:role="line"
+         x="170.78101"
+         y="200.90428"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+         id="tspan16">PCIe (NBIO)</tspan></text>
+    <g
+       id="g10"
+       transform="translate(-8.807217,1.0583333)">
+      <rect
+         style="fill:#ffffff;stroke:#ff6600;stroke-width:1;stroke-dasharray:none"
+         id="rect10"
+         width="50.551014"
+         height="10.992874"
+         x="65.641136"
+         y="122.93423" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+         x="91.040474"
+         y="130.69762"
+         id="text10"><tspan
+           sodipodi:role="line"
+           x="91.040474"
+           y="130.69762"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+           id="tspan10">SMU</tspan></text>
+    </g>
+    <rect
+       style="fill:#00d400;stroke:#00d400;stroke-width:0.348444;stroke-dasharray:none"
+       id="rect14"
+       width="3.8659263"
+       height="4.2845292"
+       x="171.61417"
+       y="125.16281" />
+    <rect
+       style="fill:#00d400;stroke:#00d400;stroke-width:0.348444;stroke-dasharray:none"
+       id="rect18"
+       width="3.8659263"
+       height="4.2845292"
+       x="67.340591"
+       y="185.6201" />
+    <g
+       id="g25"
+       transform="translate(165.76146,89.164578)">
+      <g
+         id="g17"
+         transform="translate(-127.72192,-84.269792)">
+        <rect
+           style="fill:#ffffff;stroke:#ff00ff;stroke-width:1;stroke-dasharray:none"
+           id="rect17"
+           width="50.551014"
+           height="10.992874"
+           x="83.42868"
+           y="140.39673" />
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+           x="108.55814"
+           y="148.16011"
+           id="text17"><tspan
+             sodipodi:role="line"
+             x="108.55814"
+             y="148.16011"
+             style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+             id="tspan17">SDMA</tspan></text>
+      </g>
+      <g
+         id="g24">
+        <rect
+           style="fill:#00d400;stroke:#00d400;stroke-width:0.348444;stroke-dasharray:none"
+           id="rect21"
+           width="3.8659263"
+           height="4.2845292"
+           x="9.0765572"
+           y="57.429478" />
+      </g>
+    </g>
+    <path
+       style="fill:none;stroke:#ff6600;stroke-width:1;stroke-dasharray:1, 1;stroke-dashoffset:0;marker-end:url(#ArrowWideRounded)"
+       d="M 56.046208,126.07948 H 51.151416 V 110.60136"
+       id="path21"
+       sodipodi:nodetypes="ccc" />
+    <path
+       style="fill:none;stroke:#ff6600;stroke-width:1;stroke-dasharray:1,1;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+       d="m 107.63996,129.38677 h 31.91723"
+       id="path22" />
+    <g
+       id="g26"
+       transform="translate(103.05521,-16.801041)">
+      <path
+         style="fill:none;stroke:#ff6600;stroke-width:0.684499;stroke-dasharray:0.684499, 0.684499;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+         d="M 14.374335,237.26538 H 29.87027"
+         id="path23" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+         x="41.690811"
+         y="239.50058"
+         id="text23"><tspan
+           sodipodi:role="line"
+           x="41.690811"
+           y="239.50058"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff6600;stroke-width:0.0690111"
+           id="tspan23">Reset</tspan></text>
+    </g>
+    <path
+       style="fill:none;stroke:#ff6600;stroke-width:1;stroke-dasharray:1, 1;stroke-dashoffset:0;marker-end:url(#ArrowWideRounded)"
+       d="m 56.046208,133.69217 h -4.894792 v 58.07604"
+       id="path24"
+       sodipodi:nodetypes="ccc" />
+    <path
+       style="fill:none;stroke:#ff6600;stroke-width:1;stroke-dasharray:1,1;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+       d="m 95.977781,134.7651 v 15.92063 l 23.944789,0.007"
+       id="path25"
+       sodipodi:nodetypes="ccc" />
+    <path
+       style="fill:none;stroke:#ffd42a;stroke-width:1;stroke-dasharray:8, 1;stroke-dashoffset:0"
+       d="m 116.34935,212.1162 h 16.22753"
+       id="path26"
+       sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff6600;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+       x="159.95955"
+       y="214.49745"
+       id="text26"><tspan
+         sodipodi:role="line"
+         x="159.95955"
+         y="214.49745"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ffd42a;stroke-width:0.0690111"
+         id="tspan26">PSP Interaction</tspan></text>
+    <path
+       style="fill:none;stroke:#ffd42a;stroke-width:1;stroke-dasharray:8, 1;stroke-dashoffset:0"
+       d="m 142.93997,177.44941 12.78794,1e-5 0,-20.13841"
+       id="path27"
+       sodipodi:nodetypes="ccc" />
+    <path
+       style="fill:none;stroke:#ffd42a;stroke-width:1;stroke-dasharray:8, 1;stroke-dashoffset:0"
+       d="m 154.31705,177.44941 24.03273,1e-5 v -46.34286 h -6.87917"
+       id="path28"
+       sodipodi:nodetypes="cccc" />
+    <rect
+       style="fill:#008033;stroke:#008033;stroke-width:1.12152;stroke-dasharray:none"
+       id="rect31"
+       width="28.449656"
+       height="23.148542"
+       x="-9.3648224"
+       y="152.25124" />
+    <rect
+       style="fill:#008033;stroke:#008033;stroke-width:1.12152;stroke-dasharray:none"
+       id="rect32"
+       width="28.449656"
+       height="23.148542"
+       x="-9.3648224"
+       y="71.817902" />
+    <rect
+       style="fill:#008033;stroke:#008033;stroke-width:1.12152;stroke-dasharray:none"
+       id="rect33"
+       width="28.449656"
+       height="23.148542"
+       x="198.46539"
+       y="152.25124" />
+    <rect
+       style="fill:#008033;stroke:#008033;stroke-width:1.12152;stroke-dasharray:none"
+       id="rect34"
+       width="28.449656"
+       height="23.148542"
+       x="198.46539"
+       y="71.817902" />
+    <path
+       style="fill:none;stroke:#00d455;stroke-width:1;stroke-dasharray:1, 1;stroke-dashoffset:0"
+       d="m 69.17932,192.92241 0,-26.32924 H 34.403946"
+       id="path35"
+       sodipodi:nodetypes="ccc" />
+    <path
+       style="fill:none;stroke:#00d455;stroke-width:1;stroke-dasharray:1, 1;stroke-dashoffset:0"
+       d="m 69.167488,192.92241 0,-26.32924 H 184.79581"
+       id="path36"
+       sodipodi:nodetypes="ccc" />
+    <path
+       style="fill:none;stroke:#00d455;stroke-width:1;stroke-dasharray:1, 1;stroke-dashoffset:0"
+       d="m 33.58103,118.30671 147.61609,0 v 30.03021"
+       id="path37"
+       sodipodi:nodetypes="ccc" />
+    <path
+       style="fill:none;stroke:#00d455;stroke-width:1;stroke-dasharray:1, 1;stroke-dashoffset:0"
+       d="m 169.94852,126.77338 15.61123,0"
+       id="path38"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#00d455;stroke-width:1;stroke-dasharray:1, 1;stroke-dashoffset:0"
+       d="m 172.24035,148.99838 13.3194,0"
+       id="path39"
+       sodipodi:nodetypes="cc" />
+    <g
+       id="g40"
+       transform="translate(0,1.8520834)">
+      <circle
+         style="fill:#ff8080;stroke:#ff2a2a;stroke-width:0.7;stroke-dasharray:none;stroke-dashoffset:0"
+         id="path40"
+         cx="7.2723336"
+         cy="194.07741"
+         r="10.186459" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:4.23333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+         x="7.1728497"
+         y="195.6649"
+         id="text40"><tspan
+           sodipodi:role="line"
+           x="7.1728492"
+           y="195.6649"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.23333px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+           id="tspan40">Firmware</tspan></text>
+    </g>
+    <path
+       style="fill:none;stroke:#ffd42a;stroke-width:1;stroke-dasharray:8, 1;stroke-dashoffset:0"
+       d="m 34.063928,177.71401 78.801482,0.26458"
+       id="path41"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#ffd42a;stroke-width:1;stroke-dasharray:8, 1;stroke-dashoffset:0"
+       d="M 8.1347613,184.72547 27.537285,177.97859"
+       id="path42"
+       sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+       x="4.5742564"
+       y="84.897125"
+       id="text42"><tspan
+         sodipodi:role="line"
+         x="4.5742559"
+         y="84.897125"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+         id="tspan42">Memory</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+       x="4.5742564"
+       y="165.33046"
+       id="text43"><tspan
+         sodipodi:role="line"
+         x="4.5742559"
+         y="165.33046"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+         id="tspan43">Memory</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+       x="212.40446"
+       y="165.33046"
+       id="text44"><tspan
+         sodipodi:role="line"
+         x="212.40446"
+         y="165.33046"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+         id="tspan44">Memory</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+       x="212.40446"
+       y="84.897125"
+       id="text45"><tspan
+         sodipodi:role="line"
+         x="212.40446"
+         y="84.897125"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+         id="tspan45">Memory</tspan></text>
+    <path
+       style="fill:none;stroke:#00d455;stroke-width:0.882664;stroke-dasharray:0.882664, 0.882664;stroke-dashoffset:0"
+       d="m 174.12438,77.957755 h 10.9062"
+       id="path45"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#00d455;stroke-width:0.960226;stroke-dasharray:0.960226, 0.960226;stroke-dashoffset:0"
+       d="M 34.639355,77.957755 H 47.546469"
+       id="path46"
+       sodipodi:nodetypes="cc" />
+    <rect
+       style="fill:#ffffff;stroke:#008033;stroke-width:1;stroke-dasharray:none"
+       id="rect2"
+       width="5.0221987"
+       height="111.47456"
+       x="28.691442"
+       y="72.435623" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.93889px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008033;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+       x="-127.03961"
+       y="32.94923"
+       id="text34"
+       transform="rotate(-90)"><tspan
+         sodipodi:role="line"
+         x="-127.03961"
+         y="32.94923"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#008033;stroke-width:0.0690111"
+         id="tspan34">UMC</tspan></text>
+    <path
+       style="fill:none;stroke:#ffd42a;stroke-width:1;stroke-dasharray:8, 1;stroke-dashoffset:0"
+       d="m 179.15481,177.45676 h 5.90877"
+       id="path1"
+       sodipodi:nodetypes="cc" />
+    <rect
+       style="fill:#ffffff;stroke:#008033;stroke-width:1;stroke-dasharray:none"
+       id="rect3"
+       width="5.0221987"
+       height="111.47456"
+       x="185.4259"
+       y="72.435623" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.93889px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008033;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+       x="-127.03961"
+       y="189.71486"
+       id="text35"
+       transform="rotate(-90)"><tspan
+         sodipodi:role="line"
+         x="-127.03961"
+         y="189.71486"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#008033;stroke-width:0.0690111"
+         id="tspan35">UMC</tspan></text>
+    <path
+       style="fill:none;stroke:#ffd42a;stroke-width:1;stroke-dasharray:8, 1;stroke-dashoffset:0"
+       d="m 154.31705,177.44941 24.03273,1e-5 0,-85.765777 h -4.36563"
+       id="path4"
+       sodipodi:nodetypes="cccc" />
+    <g
+       id="g5"
+       transform="translate(0,5.8208336)">
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+         x="110.18288"
+         y="73.872185"
+         id="text14-7"><tspan
+           sodipodi:role="line"
+           x="110.18288"
+           y="73.872185"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+           id="tspan14-7">Graphics &amp; Compute (GC)</tspan></text>
+      <g
+         id="g4"
+         transform="translate(10.583333)">
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+           x="58.022533"
+           y="83.518799"
+           id="text14-7-5"><tspan
+             sodipodi:role="line"
+             x="58.022533"
+             y="83.518799"
+             style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+             id="tspan14-7-6">Shader</tspan><tspan
+             sodipodi:role="line"
+             x="58.022533"
+             y="90.574348"
+             style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+             id="tspan20">Engine(SE)</tspan><tspan
+             sodipodi:role="line"
+             x="58.022533"
+             y="97.629898"
+             style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+             id="tspan21">#1</tspan></text>
+        <rect
+           style="fill:none;stroke:#aa0000;stroke-width:1.01356;stroke-dasharray:none;stroke-dashoffset:0"
+           id="rect20"
+           width="31.954243"
+           height="22.038303"
+           x="42.18652"
+           y="77.410309" />
+      </g>
+      <g
+         id="g3"
+         transform="translate(14.287499,-0.66146851)">
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+           x="97.020706"
+           y="91.105995"
+           id="text24"><tspan
+             sodipodi:role="line"
+             x="97.020706"
+             y="91.105995"
+             style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+             id="tspan24">SE #2</tspan></text>
+        <rect
+           style="fill:none;stroke:#aa0000;stroke-width:1.01356;stroke-dasharray:none;stroke-dashoffset:0"
+           id="rect24"
+           width="31.954243"
+           height="22.038303"
+           x="81.080269"
+           y="78.071777" />
+      </g>
+      <g
+         id="g2"
+         transform="translate(-3.96875)">
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:5.64444px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+           x="158.03574"
+           y="90.444527"
+           id="text30"><tspan
+             sodipodi:role="line"
+             x="158.03574"
+             y="90.444527"
+             style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+             id="tspan30">SE #N</tspan></text>
+        <rect
+           style="fill:none;stroke:#aa0000;stroke-width:1.01356;stroke-dasharray:none;stroke-dashoffset:0"
+           id="rect30"
+           width="31.954243"
+           height="22.038303"
+           x="141.93443"
+           y="77.410309" />
+      </g>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+         x="132.011"
+         y="88.873962"
+         id="text4"><tspan
+           sodipodi:role="line"
+           x="132.011"
+           y="88.873962"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+           id="tspan4">...</tspan></text>
+      <rect
+         style="fill:none;stroke:#ff8080;stroke-width:1;stroke-dasharray:none"
+         id="rect4"
+         width="126.7475"
+         height="35.863121"
+         x="46.912045"
+         y="68.129692" />
+    </g>
+    <g
+       id="g11"
+       transform="translate(-5.0270833,-8.8635417)">
+      <rect
+         style="fill:#ffffff;stroke:#5f5fd3;stroke-width:1;stroke-dasharray:none"
+         id="rect19"
+         width="28.674419"
+         height="11.302325"
+         x="146.1279"
+         y="132.70711" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+         x="160.75085"
+         y="140.62521"
+         id="text19"><tspan
+           sodipodi:role="line"
+           x="160.75085"
+           y="140.62521"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+           id="tspan19">VCN</tspan></text>
+    </g>
+    <g
+       id="g6">
+      <rect
+         style="fill:#ffffff;stroke:#0000ff;stroke-width:1;stroke-dasharray:none"
+         id="rect13"
+         width="28.674419"
+         height="11.302325"
+         x="43.988369"
+         y="192.98618" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.0690111"
+         x="58.325581"
+         y="200.90428"
+         id="text14"><tspan
+           sodipodi:role="line"
+           x="58.325581"
+           y="200.90428"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.0690111"
+           id="tspan14">DCN</tspan></text>
+    </g>
+  </g>
+</svg>
diff --git a/Documentation/gpu/amdgpu/amdgpu-glossary.rst b/Documentation/gpu/amdgpu/amdgpu-glossary.rst
index 30812d9d53c6..033167025fcc 100644
--- a/Documentation/gpu/amdgpu/amdgpu-glossary.rst
+++ b/Documentation/gpu/amdgpu/amdgpu-glossary.rst
@@ -30,6 +30,15 @@ we have a dedicated glossary for Display Core at
     CP
       Command Processor
 
+    CPC
+      Command Processor Compute
+
+    CPF
+      Command Processor Fetch
+
+    CPG
+      Command Processor Graphics
+
     CPLIB
       Content Protection Library
 
@@ -78,6 +87,9 @@ we have a dedicated glossary for Display Core at
     GMC
       Graphic Memory Controller
 
+    GPR
+      General Purpose Register
+
     GPUVM
       GPU Virtual Memory.  This is the GPU's MMU.  The GPU supports multiple
       virtual address spaces that can be in flight at any given time.  These
@@ -92,9 +104,15 @@ we have a dedicated glossary for Display Core at
       table for use by the kernel driver or into per process GPUVM page tables
       for application usage.
 
+    GWS
+      Global Wave Sync
+
     IH
       Interrupt Handler
 
+    IV
+      Interrupt Vector
+
     HQD
       Hardware Queue Descriptor
 
@@ -143,15 +161,24 @@ we have a dedicated glossary for Display Core at
     PA
       Primitive Assembler / Physical Address
 
+    PDE
+      Page Directory Entry
+
     PFP
       Pre-Fetch Parser (Graphics)
 
     PPLib
       PowerPlay Library - PowerPlay is the power management component.
 
+    PRT
+      Partially Resident Texture (also known as sparse residency)
+
     PSP
         Platform Security Processor
 
+    PTE
+      Page Table Entry
+
     RB
       Render Backends. Some people called it ROPs.
 
@@ -206,12 +233,33 @@ we have a dedicated glossary for Display Core at
     TC
       Texture Cache
 
+    TCP (AMDGPU)
+      Texture Cache per Pipe. Even though the name "Texture" is part of this
+      acronym, the TCP represents the path to memory shaders; i.e., it is not
+      related to texture. The name is a leftover from older designs where shader
+      stages had different cache designs; it refers to the L1 cache in older
+      architectures.
+
+    TMR
+      Trusted Memory Region
+
+    TMZ
+      Trusted Memory Zone
+
     TOC
       Table of Contents
 
+    UMC
+      Unified Memory Controller
+
     UMSCH
       User Mode Scheduler
 
+    UTC (AMDGPU)
+      Unified Translation Cache. UTC is equivalent to TLB. You might see a
+      variation of this acronym with L at the end, i.e., UTCL followed by a
+      number; L means the cache level (e.g., UTCL1 and UTCL2).
+
     UVD
       Unified Video Decoder
 
diff --git a/Documentation/gpu/amdgpu/display/dc-glossary.rst b/Documentation/gpu/amdgpu/display/dc-glossary.rst
index cbe737d1fcea..accb7c05308c 100644
--- a/Documentation/gpu/amdgpu/display/dc-glossary.rst
+++ b/Documentation/gpu/amdgpu/display/dc-glossary.rst
@@ -221,9 +221,6 @@ consider asking on the amd-gfx mailing list and update this page.
     TMDS
       Transition-Minimized Differential Signaling
 
-    TMZ
-      Trusted Memory Zone
-
     TTU
       Time to Underflow
 
diff --git a/Documentation/gpu/amdgpu/driver-core.rst b/Documentation/gpu/amdgpu/driver-core.rst
index 3ce276272171..2c2bbf7caf1a 100644
--- a/Documentation/gpu/amdgpu/driver-core.rst
+++ b/Documentation/gpu/amdgpu/driver-core.rst
@@ -77,6 +77,37 @@ VCN (Video Core Next)
     decode.  It's exposed to userspace for user mode drivers (VA-API,
     OpenMAX, etc.)
 
+It is important to note that these blocks can interact with each other. The
+picture below illustrates some of the components and their interconnection:
+
+.. kernel-figure:: amd_overview_block.svg
+
+In the diagram, memory-related blocks are shown in green. Notice that specific
+IPs have a green square that represents a small hardware block named 'hub',
+which is responsible for interfacing with memory. All memory hubs are connected
+in the UMCs, which in turn are connected to memory blocks. As a note,
+pre-vega devices have a dedicated block for the Graphic Memory Controller
+(GMC), which was replaced by UMC and hubs in new architectures. In the driver
+code, you can identify this component by looking for the suffix hub, for
+example: gfxhub, dchub, mmhub, vmhub, etc. Keep in mind that the component's
+interaction with the memory block may vary across architectures. For example,
+on Navi and newer, GC and SDMA are both attached to GCHUB; on pre-Navi, SDMA
+goes through MMHUB; VCN, JPEG, and VPE go through MMHUB; DCN goes through
+DCHUB.
+
+There is some protection for certain memory elements, and the PSP plays an
+essential role in this area. When a specific firmware is loaded into memory,
+the PSP takes steps to ensure it has a valid signature. It also stores firmware
+images in a protected memory area named Trusted Memory Area (TMR), so the OS or
+driver can't corrupt them at runtime. Another use of PSP is to support Trusted
+Applications (TA), which are basically small applications that run on the
+trusted processor and handles a trusted operation (e.g., HDCP). PSP is also
+used for encrypted memory for content protection via Trusted Memory Zone (TMZ).
+
+Another critical IP is the SMU. It handles reset distribution, as well as
+clock, thermal, and power management for all IPs on the SoC. SMU also helps to
+balance performance and power consumption.
+
 .. _pipes-and-queues-description:
 
 GFX, Compute, and SDMA Overall Behavior
diff --git a/Documentation/gpu/amdgpu/enforce_isolation.svg b/Documentation/gpu/amdgpu/enforce_isolation.svg
new file mode 100644
index 000000000000..2630681f1cb9
--- /dev/null
+++ b/Documentation/gpu/amdgpu/enforce_isolation.svg
@@ -0,0 +1,654 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   width="67.794067mm"
+   height="88.643349mm"
+   viewBox="0 0 67.794066 88.643348"
+   version="1.1"
+   id="svg1"
+   inkscape:version="1.4.2 (ebf0e940d0, 2025-05-08)"
+   sodipodi:docname="enforce_isolation.svg"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview1"
+     pagecolor="#ffffff"
+     bordercolor="#000000"
+     borderopacity="0.25"
+     inkscape:showpageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1"
+     inkscape:document-units="mm"
+     inkscape:zoom="3.5754724"
+     inkscape:cx="200.95247"
+     inkscape:cy="182.77305"
+     inkscape:window-width="3840"
+     inkscape:window-height="2083"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="layer1" />
+  <defs
+     id="defs1">
+    <marker
+       style="overflow:visible"
+       id="ArrowWideHeavy"
+       refX="0"
+       refY="0"
+       orient="auto-start-reverse"
+       inkscape:stockid="Wide, heavy arrow"
+       markerWidth="1"
+       markerHeight="1"
+       viewBox="0 0 1 1"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         style="fill:context-stroke;fill-rule:evenodd;stroke:none"
+         d="m 1,0 -3,3 h -2 l 3,-3 -3,-3 h 2 z"
+         id="path3" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Dot"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Dot"
+       markerWidth="1"
+       markerHeight="1"
+       viewBox="0 0 1 1"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.5)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:none"
+         d="M 5,0 C 5,2.76 2.76,5 0,5 -2.76,5 -5,2.76 -5,0 c 0,-2.76 2.3,-5 5,-5 2.76,0 5,2.24 5,5 z"
+         sodipodi:nodetypes="sssss"
+         id="path98" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker99"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Dot"
+       markerWidth="1"
+       markerHeight="1"
+       viewBox="0 0 1 1"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.5)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:none"
+         d="M 5,0 C 5,2.76 2.76,5 0,5 -2.76,5 -5,2.76 -5,0 c 0,-2.76 2.3,-5 5,-5 2.76,0 5,2.24 5,5 z"
+         sodipodi:nodetypes="sssss"
+         id="path99" />
+    </marker>
+  </defs>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(71.49059,-1.1271925)">
+    <g
+       id="g15"
+       transform="matrix(0.42247861,0,0,0.42247861,-70.575576,-25.242317)">
+      <path
+         id="path33"
+         style="fill:none;stroke:#000000;stroke-width:0.721067;stroke-dasharray:none"
+         d="M 30.278993,176.45537 A 22.905334,22.905334 0 0 0 7.3737955,199.36057 22.905334,22.905334 0 0 0 30.278993,222.26603 22.905334,22.905334 0 0 0 53.18445,199.36057 22.905334,22.905334 0 0 0 30.278993,176.45537 Z m 0,7.13274 a 15.772359,15.772359 0 0 1 15.77246,15.77246 15.772359,15.772359 0 0 1 -15.77246,15.77246 15.772359,15.772359 0 0 1 -15.772206,-15.77246 15.772359,15.772359 0 0 1 15.772206,-15.77246 z" />
+      <g
+         id="g14">
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 46.31405,200.72093 h 6.430134"
+           id="path34"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 7.6848605,200.72093 H 14.114993"
+           id="path35"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 30.103415,176.7326 v 6.43014"
+           id="path36"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 30.103415,215.50586 V 221.936"
+           id="path37"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,214.1052 3.21507,5.56866"
+           id="path38"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,184.66805 3.21507,-5.56866"
+           id="path39"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,184.66805 -3.21507,-5.56866"
+           id="path40"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,214.05288 -3.21507,5.56866"
+           id="path41"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,208.65101 4.546794,4.5468"
+           id="path42"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,189.96525 4.546794,-4.5468"
+           id="path43"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 17.202394,189.96525 12.6556,185.41845"
+           id="path44-1"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 16.940766,208.65101 -4.546794,4.5468"
+           id="path45-8"
+           sodipodi:nodetypes="cc" />
+      </g>
+    </g>
+    <rect
+       style="fill:#afe9c6;stroke:#16502d;stroke-width:0.257104;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+       id="rect80"
+       width="25.076588"
+       height="10.493422"
+       x="-71.362038"
+       y="79.148567"
+       ry="1.4529352" />
+    <rect
+       style="fill:#80e5ff;stroke:#00aad4;stroke-width:0.257104;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+       id="rect81"
+       width="32.714355"
+       height="10.436013"
+       x="-40.481403"
+       y="79.177269"
+       ry="1.4449863" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.53042px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+       x="-59.013123"
+       y="86.726654"
+       id="text14-7"><tspan
+         sodipodi:role="line"
+         x="-59.013123"
+         y="86.726654"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.53042px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.102999"
+         id="tspan14-7">GFX</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.53042px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+       x="-24.387218"
+       y="85.975647"
+       id="text95"><tspan
+         sodipodi:role="line"
+         x="-24.387218"
+         y="85.975647"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.53042px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.102999"
+         id="tspan95">Compute</tspan></text>
+    <path
+       style="fill:#de8787;stroke:#000000;stroke-width:0.385656;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#Dot);marker-end:url(#marker99)"
+       d="M -58.079264,78.410023 V 69.962845"
+       id="path127"
+       sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:2.90242px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+       x="-61.752651"
+       y="3.1995225"
+       id="text112"><tspan
+         sodipodi:role="line"
+         x="-61.752651"
+         y="3.1995225"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.90242px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.102999"
+         id="tspan112">Processes</tspan></text>
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.247306;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:1.48383, 1.48383;stroke-dashoffset:0"
+       id="rect113"
+       width="64.703476"
+       height="19.562067"
+       x="-68.52655"
+       y="4.468956"
+       ry="1.6976216" />
+    <g
+       id="g84"
+       transform="matrix(0.25710378,0,0,0.25710378,0.68647434,8.7899633)">
+      <g
+         id="g83">
+        <circle
+           style="fill:#ffffff;stroke:#ff00ff;stroke-width:1.62704;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;stroke-dashoffset:0"
+           id="circle137"
+           cx="-225.65012"
+           cy="20.747513"
+           r="30.822298" />
+      </g>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:16.9333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff00ff;fill-opacity:1;stroke:none;stroke-width:0.400612"
+         x="-225.65012"
+         y="26.8181"
+         id="text113"><tspan
+           sodipodi:role="line"
+           x="-225.65012"
+           y="26.8181"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:16.9333px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff00ff;stroke-width:0.400612"
+           id="tspan113">A</tspan></text>
+    </g>
+    <g
+       id="g85"
+       transform="matrix(0.25710378,0,0,0.25710378,20.618429,8.7899633)">
+      <circle
+         style="fill:#ffffff;stroke:#ff9955;stroke-width:1.62704;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;stroke-dashoffset:0"
+         id="circle112"
+         cx="-207.94376"
+         cy="20.747513"
+         r="30.822298" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:16.9333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.400612"
+         x="-208.25702"
+         y="26.792702"
+         id="text114"><tspan
+           sodipodi:role="line"
+           x="-208.25702"
+           y="26.792702"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:16.9333px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.400612"
+           id="tspan114">B</tspan></text>
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff00ff;fill-opacity:1;stroke:none;stroke-width:0.102999"
+       x="-56.126556"
+       y="51.72607"
+       id="text6"><tspan
+         sodipodi:role="line"
+         x="-56.126556"
+         y="51.72607"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff00ff;stroke-width:0.102999"
+         id="tspan6">A</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:2.26752px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+       x="-57.861526"
+       y="58.416431"
+       id="text136"><tspan
+         sodipodi:role="line"
+         x="-57.861526"
+         y="58.416431"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.26752px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.102999"
+         id="tspan136">Ring</tspan><tspan
+         sodipodi:role="line"
+         x="-57.861526"
+         y="61.250832"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.26752px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.102999"
+         id="tspan137">Buffer</tspan></text>
+    <g
+       id="g80"
+       transform="matrix(0.42247861,0,0,0.42247861,-37.21188,-25.242317)">
+      <path
+         id="path67"
+         style="fill:none;stroke:#000000;stroke-width:0.721067;stroke-dasharray:none"
+         d="M 30.278993,176.45537 A 22.905334,22.905334 0 0 0 7.3737955,199.36057 22.905334,22.905334 0 0 0 30.278993,222.26603 22.905334,22.905334 0 0 0 53.18445,199.36057 22.905334,22.905334 0 0 0 30.278993,176.45537 Z m 0,7.13274 a 15.772359,15.772359 0 0 1 15.77246,15.77246 15.772359,15.772359 0 0 1 -15.77246,15.77246 15.772359,15.772359 0 0 1 -15.772206,-15.77246 15.772359,15.772359 0 0 1 15.772206,-15.77246 z" />
+      <g
+         id="g79">
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 46.31405,200.72093 h 6.430134"
+           id="path68"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 7.6848605,200.72093 H 14.114993"
+           id="path69"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 30.103415,176.7326 v 6.43014"
+           id="path70"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 30.103415,215.50586 V 221.936"
+           id="path71"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,214.1052 3.21507,5.56866"
+           id="path72"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,184.66805 3.21507,-5.56866"
+           id="path73"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,184.66805 -3.21507,-5.56866"
+           id="path74"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,214.05288 -3.21507,5.56866"
+           id="path75"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,208.65101 4.546794,4.5468"
+           id="path76"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,189.96525 4.546794,-4.5468"
+           id="path77"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 17.202394,189.96525 12.6556,185.41845"
+           id="path78"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 16.940766,208.65101 -4.546794,4.5468"
+           id="path79"
+           sodipodi:nodetypes="cc" />
+      </g>
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:2.26752px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+       x="-24.497828"
+       y="58.416431"
+       id="text81"><tspan
+         sodipodi:role="line"
+         x="-24.497828"
+         y="58.416431"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.26752px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.102999"
+         id="tspan80">Ring</tspan><tspan
+         sodipodi:role="line"
+         x="-24.497828"
+         y="61.250832"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.26752px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.102999"
+         id="tspan81">Buffer</tspan></text>
+    <path
+       style="fill:#de8787;stroke:#000000;stroke-width:0.385656;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#Dot);marker-end:url(#marker99)"
+       d="M -24.338879,78.410023 V 69.962845"
+       id="path81"
+       sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff00ff;fill-opacity:1;stroke:none;stroke-width:0.102999"
+       x="-52.574932"
+       y="53.340443"
+       id="text82"><tspan
+         sodipodi:role="line"
+         x="-52.574932"
+         y="53.340443"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff00ff;stroke-width:0.102999"
+         id="tspan82">A</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff00ff;fill-opacity:1;stroke:none;stroke-width:0.102999"
+       x="-50.045757"
+       y="57.376377"
+       id="text83"><tspan
+         sodipodi:role="line"
+         x="-50.045757"
+         y="57.376377"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff00ff;stroke-width:0.102999"
+         id="tspan83">A</tspan></text>
+    <g
+       id="g1"
+       transform="translate(0,-16.057901)">
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff00ff;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-12.593401"
+         y="57.349865"
+         id="text83-59"><tspan
+           sodipodi:role="line"
+           x="-12.593401"
+           y="57.349865"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff00ff;stroke-width:0.102999"
+           id="tspan83-7">A</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff00ff;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-9.8666544"
+         y="57.349865"
+         id="text83-59-3"><tspan
+           sodipodi:role="line"
+           x="-9.8666544"
+           y="57.349865"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff00ff;stroke-width:0.102999"
+           id="tspan83-7-6">A</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff0000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-24.724035"
+         y="57.3466"
+         id="text83-5-9"><tspan
+           sodipodi:role="line"
+           x="-24.724035"
+           y="57.3466"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.102999"
+           id="tspan83-4-2">C</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff0000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-22.315107"
+         y="57.3466"
+         id="text83-5-9-9"><tspan
+           sodipodi:role="line"
+           x="-22.315107"
+           y="57.3466"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.102999"
+           id="tspan83-4-2-5">C</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-19.937738"
+         y="57.3466"
+         id="text83-5-9-9-7"><tspan
+           sodipodi:role="line"
+           x="-19.937738"
+           y="57.3466"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.102999"
+           id="tspan83-4-2-5-2">B</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-17.56146"
+         y="57.3466"
+         id="text83-5-9-9-7-0"><tspan
+           sodipodi:role="line"
+           x="-17.56146"
+           y="57.3466"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.102999"
+           id="tspan83-4-2-5-2-6">B</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-15.185183"
+         y="57.3466"
+         id="text83-5-9-9-7-0-4"><tspan
+           sodipodi:role="line"
+           x="-15.185183"
+           y="57.3466"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.102999"
+           id="tspan83-4-2-5-2-6-9">B</tspan></text>
+    </g>
+    <g
+       id="g2"
+       transform="translate(0,-16.057901)">
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff00ff;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-46.032711"
+         y="57.349865"
+         id="text83-7"><tspan
+           sodipodi:role="line"
+           x="-46.032711"
+           y="57.349865"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff00ff;stroke-width:0.102999"
+           id="tspan83-8">A</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff00ff;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-42.773308"
+         y="57.349865"
+         id="text83-7-4"><tspan
+           sodipodi:role="line"
+           x="-42.773308"
+           y="57.349865"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff00ff;stroke-width:0.102999"
+           id="tspan83-8-3">A</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff0000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-55.041409"
+         y="57.3466"
+         id="text83-5"><tspan
+           sodipodi:role="line"
+           x="-55.041409"
+           y="57.3466"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.102999"
+           id="tspan83-4">C</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff0000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-57.982994"
+         y="57.3466"
+         id="text83-5-3"><tspan
+           sodipodi:role="line"
+           x="-57.982994"
+           y="57.3466"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.102999"
+           id="tspan83-4-5">C</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff0000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-52.099823"
+         y="57.3466"
+         id="text83-5-6"><tspan
+           sodipodi:role="line"
+           x="-52.099823"
+           y="57.3466"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.102999"
+           id="tspan83-4-9">C</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff0000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-49.158237"
+         y="57.3466"
+         id="text83-5-6-5"><tspan
+           sodipodi:role="line"
+           x="-49.158237"
+           y="57.3466"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.102999"
+           id="tspan83-4-9-2">C</tspan></text>
+    </g>
+    <g
+       id="g86"
+       transform="matrix(0.25710378,0,0,0.25710378,17.422136,8.7899633)">
+      <circle
+         style="fill:#ffffff;stroke:#ff0000;stroke-width:1.62704;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;stroke-dashoffset:0"
+         id="circle84"
+         cx="-121.9205"
+         cy="20.747513"
+         r="30.822298" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:16.9333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.400612"
+         x="-122.11524"
+         y="26.792702"
+         id="text84"><tspan
+           sodipodi:role="line"
+           x="-122.11524"
+           y="26.792702"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:16.9333px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.400612"
+           id="tspan84">C</tspan></text>
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-size:2.17681px;font-family:'Linux Libertine O';-inkscape-font-specification:'Linux Libertine O';text-align:start;letter-spacing:1.26323px;writing-mode:lr-tb;direction:ltr;text-anchor:start;fill:none;stroke:#000000;stroke-width:0.257104;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+       x="-140.03215"
+       y="26.074423"
+       id="text86"><tspan
+         sodipodi:role="line"
+         id="tspan86"
+         style="stroke-width:0.257104"
+         x="-140.03215"
+         y="26.074423" /></text>
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:0.463569;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect89"
+       width="62.753353"
+       height="10.962811"
+       x="-68.911674"
+       y="32.218185" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:2.26751px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+       x="-37.552006"
+       y="35.595592"
+       id="text89"><tspan
+         sodipodi:role="line"
+         x="-37.552006"
+         y="35.595592"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.26751px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.102999"
+         id="tspan89">Enforce Isolation</tspan></text>
+    <path
+       style="fill:#de8787;stroke:#000000;stroke-width:0.385656;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#Dot);marker-end:url(#marker99)"
+       d="M -57.807162,48.273529 V 44.392491"
+       id="path90"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#de8787;stroke:#000000;stroke-width:0.385656;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#Dot);marker-end:url(#marker99)"
+       d="M -24.512426,48.273529 V 44.392491"
+       id="path91"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#ff00ff;stroke:#ff00ff;stroke-width:0.257104;stroke-linecap:square;stroke-dasharray:none;marker-end:url(#ArrowWideHeavy)"
+       d="m -57.329023,22.418572 v 9.208373"
+       id="path1" />
+    <path
+       style="fill:#ff00ff;stroke:#ff9955;stroke-width:0.257104;stroke-linecap:square;stroke-dasharray:none;marker-end:url(#ArrowWideHeavy)"
+       d="m -32.844695,22.418572 v 9.208373"
+       id="path2" />
+    <path
+       style="fill:#ff00ff;stroke:#ff0000;stroke-width:0.257104;stroke-linecap:square;stroke-dasharray:none;marker-end:url(#ArrowWideHeavy)"
+       d="m -13.924085,22.418572 v 9.208373"
+       id="path4" />
+    <path
+       style="fill:none;stroke:#ff00ff;stroke-width:0.257104;stroke-linecap:square;stroke-dasharray:none;marker-end:url(#ArrowWideHeavy)"
+       d="m -57.329023,22.418572 v 5.174952 h 20.852017 v 4.033421"
+       id="path5"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="fill:none;stroke:#ff0000;stroke-width:0.257104;stroke-linecap:square;stroke-dasharray:none;marker-end:url(#ArrowWideHeavy)"
+       d="m -13.924085,22.418572 v 3.652908 h -34.777483 v 5.555465"
+       id="path6"
+       sodipodi:nodetypes="cccc" />
+  </g>
+</svg>
diff --git a/Documentation/gpu/amdgpu/gc/index.rst b/Documentation/gpu/amdgpu/gc/index.rst
index ff6e9ef5cbee..b6b583c9dc6a 100644
--- a/Documentation/gpu/amdgpu/gc/index.rst
+++ b/Documentation/gpu/amdgpu/gc/index.rst
@@ -7,19 +7,21 @@
 The relationship between the CPU and GPU can be described as the
 producer-consumer problem, where the CPU fills out a buffer with operations
 (producer) to be executed by the GPU (consumer). The requested operations in
-the buffer are called Command Packets, which can be summarized as a compressed
-way of transmitting command information to the graphics controller.
+the buffer are called **Command Packets**, which can be summarized as a
+compressed way of transmitting command information to the graphics controller.
 
 The component that acts as the front end between the CPU and the GPU is called
-the Command Processor (CP). This component is responsible for providing greater
-flexibility to the GC since CP makes it possible to program various aspects of
-the GPU pipeline. CP also coordinates the communication between the CPU and GPU
-via a mechanism named **Ring Buffers**, where the CPU appends information to
-the buffer while the GPU removes operations. It is relevant to highlight that a
-CPU can add a pointer to the Ring Buffer that points to another region of
-memory outside the Ring Buffer, and CP can handle it; this mechanism is called
-**Indirect Buffer (IB)**. CP receives and parses the Command Streams (CS), and
-writes the operations to the correct hardware blocks.
+**Command Processor (CP)**. This component is responsible for providing greater
+flexibility to the **Graphics and Compute (GC)** since CP makes it possible to
+program various aspects of the GPU pipeline. CP also coordinates the
+communication between the CPU and GPU via a mechanism named **Ring Buffers**,
+where the CPU appends information to the buffer while the GPU removes
+operations. CP is also responsible for handling **Indirect Buffers (IB)**.
+
+For reference, internally the CP consists of several sub-blocks (CPC - CP
+compute, CPG - CP graphics, and CPF - CP fetcher). Some of these acronyms
+appear in register names, but this is more of an implementation detail and not
+something that directly impacts driver programming or debugging.
 
 Graphics (GFX) and Compute Microcontrollers
 -------------------------------------------
diff --git a/Documentation/gpu/amdgpu/gfx_pipeline_seq.svg b/Documentation/gpu/amdgpu/gfx_pipeline_seq.svg
new file mode 100644
index 000000000000..2f2c8fa98059
--- /dev/null
+++ b/Documentation/gpu/amdgpu/gfx_pipeline_seq.svg
@@ -0,0 +1,413 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   width="141.76276mm"
+   height="51.906979mm"
+   viewBox="0 0 141.76275 51.906979"
+   version="1.1"
+   id="svg1"
+   inkscape:version="1.4 (e7c3feb100, 2024-10-09)"
+   sodipodi:docname="gfx_pipeline_seq.svg"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview1"
+     pagecolor="#ffffff"
+     bordercolor="#000000"
+     borderopacity="0.25"
+     inkscape:showpageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1"
+     inkscape:document-units="mm"
+     inkscape:zoom="1.2641204"
+     inkscape:cx="470.28748"
+     inkscape:cy="63.680643"
+     inkscape:window-width="3072"
+     inkscape:window-height="1651"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="layer1" />
+  <defs
+     id="defs1">
+    <marker
+       style="overflow:visible"
+       id="ArrowWideHeavy"
+       refX="0"
+       refY="0"
+       orient="auto-start-reverse"
+       inkscape:stockid="Wide, heavy arrow"
+       markerWidth="1"
+       markerHeight="1"
+       viewBox="0 0 1 1"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         style="fill:context-stroke;fill-rule:evenodd;stroke:none"
+         d="m 1,0 -3,3 h -2 l 3,-3 -3,-3 h 2 z"
+         id="path3" />
+    </marker>
+  </defs>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(23.062206,-30.75877)">
+    <rect
+       style="fill:#ffffff;stroke:none;stroke-width:0.694678;stroke-linecap:square;stroke-dasharray:4.16805, 4.16805"
+       id="rect1"
+       width="141.76276"
+       height="51.906979"
+       x="-23.062206"
+       y="30.75877" />
+    <g
+       id="g28"
+       transform="matrix(1.0835493,0,0,1.0835493,-30.079831,-159.17628)">
+      <path
+         id="path18"
+         style="fill:none;stroke:#000000;stroke-width:0.721067;stroke-dasharray:none"
+         d="M 30.278993,176.45537 A 22.905334,22.905334 0 0 0 7.3737955,199.36057 22.905334,22.905334 0 0 0 30.278993,222.26603 22.905334,22.905334 0 0 0 53.18445,199.36057 22.905334,22.905334 0 0 0 30.278993,176.45537 Z m 0,7.13274 a 15.772359,15.772359 0 0 1 15.77246,15.77246 15.772359,15.772359 0 0 1 -15.77246,15.77246 15.772359,15.772359 0 0 1 -15.772206,-15.77246 15.772359,15.772359 0 0 1 15.772206,-15.77246 z" />
+      <g
+         id="g27">
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 46.31405,200.72093 h 6.430134"
+           id="path19"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 7.6848605,200.72093 H 14.114993"
+           id="path20"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 30.103415,176.7326 v 6.43014"
+           id="path21"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 30.103415,215.50586 V 221.936"
+           id="path22"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,214.1052 3.21507,5.56866"
+           id="path23"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,184.66805 3.21507,-5.56866"
+           id="path24"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,184.66805 -3.21507,-5.56866"
+           id="path25"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,214.05288 -3.21507,5.56866"
+           id="path26"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,208.65101 4.546794,4.5468"
+           id="path27"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,189.96525 4.546794,-4.5468"
+           id="path28"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 17.202394,189.96525 12.6556,185.41845"
+           id="path29"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 16.940766,208.65101 -4.546794,4.5468"
+           id="path30"
+           sodipodi:nodetypes="cc" />
+      </g>
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.92107px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.434083"
+       x="23.133495"
+       y="52.750404"
+       id="text24"><tspan
+         sodipodi:role="line"
+         x="23.133495"
+         y="52.750404"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.92107px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.434083"
+         id="tspan24">IB<tspan
+   style="font-size:3.92107px;baseline-shift:sub;fill:#ff9955;stroke-width:0.434083"
+   id="tspan27">b</tspan></tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.88054px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.434083"
+       x="2.4917324"
+       y="55.12072"
+       id="text139"><tspan
+         sodipodi:role="line"
+         x="2.4917324"
+         y="55.12072"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.88054px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.434083"
+         id="tspan138">Ring</tspan><tspan
+         sodipodi:role="line"
+         x="2.4917324"
+         y="63.721394"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.88054px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.434083"
+         id="tspan139">Buffer</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.92107px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.434083"
+       x="22.115709"
+       y="65.117416"
+       id="text18"><tspan
+         sodipodi:role="line"
+         x="22.115709"
+         y="65.117416"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.92107px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#87decd;stroke-width:0.434083"
+         id="tspan18">IB<tspan
+   style="font-size:3.92107px;baseline-shift:sub;fill:#87decd;stroke-width:0.434083"
+   id="tspan17">c</tspan></tspan></text>
+    <g
+       id="g62"
+       transform="matrix(0.69467788,0,0,0.69467788,-104.6214,-57.027324)">
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#37abc8;fill-opacity:1;stroke:none;stroke-width:0.400612"
+         x="221.50526"
+         y="173.96935"
+         id="text48"><tspan
+           sodipodi:role="line"
+           x="221.50526"
+           y="173.96935"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#37abc8;stroke-width:0.400612"
+           id="tspan48">SX</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#37abc8;fill-opacity:1;stroke:none;stroke-width:0.400612"
+         x="238.1783"
+         y="173.96935"
+         id="text49"><tspan
+           sodipodi:role="line"
+           x="238.1783"
+           y="173.96935"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#37abc8;stroke-width:0.400612"
+           id="tspan49">GE</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#37abc8;fill-opacity:1;stroke:none;stroke-width:0.400612"
+         x="255.66414"
+         y="173.96935"
+         id="text50"><tspan
+           sodipodi:role="line"
+           x="255.66414"
+           y="173.96935"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#37abc8;stroke-width:0.400612"
+           id="tspan50">SPI</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+         x="273.01663"
+         y="173.96935"
+         id="text51"><tspan
+           sodipodi:role="line"
+           x="273.01663"
+           y="173.96935"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+           id="tspan51">SC</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+         x="289.11816"
+         y="173.97888"
+         id="text52"><tspan
+           sodipodi:role="line"
+           x="289.11816"
+           y="173.97888"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+           id="tspan52">PA</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff2a2a;fill-opacity:1;stroke:none;stroke-width:0.400612"
+         x="311.40778"
+         y="174.08365"
+         id="text53"><tspan
+           sodipodi:role="line"
+           x="311.40778"
+           y="174.08365"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff2a2a;stroke-width:0.400612"
+           id="tspan53">Cache</tspan></text>
+      <path
+         style="fill:none;stroke:#000000;stroke-width:0.529167;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+         d="m 226.31089,171.70241 h 6.95598"
+         id="path53"
+         sodipodi:nodetypes="cc" />
+      <path
+         style="fill:none;stroke:#000000;stroke-width:0.529167;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+         d="m 242.81883,171.70241 h 6.95598"
+         id="path54"
+         sodipodi:nodetypes="cc" />
+      <path
+         style="fill:none;stroke:#000000;stroke-width:0.529167;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+         d="m 261.37781,171.70241 h 6.95598"
+         id="path55"
+         sodipodi:nodetypes="cc" />
+      <path
+         style="fill:none;stroke:#000000;stroke-width:0.529167;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+         d="m 277.50475,171.70241 h 6.95598"
+         id="path56"
+         sodipodi:nodetypes="cc" />
+      <path
+         style="fill:none;stroke:#000000;stroke-width:0.529167;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+         d="m 294.08254,171.70241 h 6.95598"
+         id="path57"
+         sodipodi:nodetypes="cc" />
+      <path
+         style="fill:none;stroke:#000000;stroke-width:0.529167;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+         d="m 189.90312,171.70241 h 26.34385"
+         id="path60"
+         sodipodi:nodetypes="cc" />
+      <path
+         style="fill:none;stroke:#000000;stroke-width:0.529167;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+         d="m 311.60514,176.14239 v 9.61994 H 184.98772"
+         id="path61"
+         sodipodi:nodetypes="ccc" />
+    </g>
+    <path
+       style="fill:#37c871;stroke:#00d455;stroke-width:0.367601;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+       d="m 40.698106,46.362333 4.844663,13.377322"
+       id="path62"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#37c871;stroke:#00d455;stroke-width:0.367601;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+       d="m 52.62426,46.362333 4.844663,13.377322"
+       id="path63"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#37c871;stroke:#00d455;stroke-width:0.367601;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+       d="m 64.55042,46.362333 4.844663,13.377322"
+       id="path64"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#37c871;stroke:#00d455;stroke-width:0.367601;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+       d="m 75.962512,46.362333 4.844663,13.377322"
+       id="path65"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#37c871;stroke:#00d455;stroke-width:0.367601;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+       d="m 87.580235,46.362333 4.844663,13.377322"
+       id="path66"
+       sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.4112px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#37abc8;fill-opacity:1;stroke:none;stroke-width:0.278297"
+       x="37.018822"
+       y="45.730473"
+       id="text2"><tspan
+         sodipodi:role="line"
+         x="37.018822"
+         y="45.730473"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.4112px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#37abc8;stroke-width:0.278297"
+         id="tspan2">SX</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.4112px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#37abc8;fill-opacity:1;stroke:none;stroke-width:0.278297"
+       x="48.601212"
+       y="45.730473"
+       id="text3"><tspan
+         sodipodi:role="line"
+         x="48.601212"
+         y="45.730473"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.4112px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#37abc8;stroke-width:0.278297"
+         id="tspan3">GE</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.4112px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#37abc8;fill-opacity:1;stroke:none;stroke-width:0.278297"
+       x="60.748234"
+       y="45.730473"
+       id="text4"><tspan
+         sodipodi:role="line"
+         x="60.748234"
+         y="45.730473"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.4112px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#37abc8;stroke-width:0.278297"
+         id="tspan4">SPI</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.4112px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.278297"
+       x="72.802635"
+       y="45.730473"
+       id="text11"><tspan
+         sodipodi:role="line"
+         x="72.802635"
+         y="45.730473"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.4112px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.278297"
+         id="tspan11">SC</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.4112px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.278297"
+       x="83.988014"
+       y="45.737099"
+       id="text13"><tspan
+         sodipodi:role="line"
+         x="83.988014"
+         y="45.737099"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.4112px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.278297"
+         id="tspan13">PA</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:4.4112px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff2a2a;fill-opacity:1;stroke:none;stroke-width:0.278297"
+       x="99.472122"
+       y="45.809875"
+       id="text14"><tspan
+         sodipodi:role="line"
+         x="99.472122"
+         y="45.809875"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.4112px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff2a2a;stroke-width:0.278297"
+         id="tspan14">Cache</tspan></text>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.367601;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+       d="m 40.357179,44.155689 h 4.832165"
+       id="path44"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.367601;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+       d="m 51.82488,44.155689 h 4.832165"
+       id="path45"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.367601;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+       d="m 64.717393,44.155689 h 4.832165"
+       id="path46"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.367601;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+       d="m 75.920422,44.155689 h 4.832165"
+       id="path47"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.367601;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+       d="m 87.436645,44.155689 h 4.832166"
+       id="path48"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.367601;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+       d="m 24.215741,44.155689 h 7.813702"
+       id="path58"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.367601;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWideHeavy)"
+       d="m 100.63732,46.931607 v 6.68276 H 29.848557"
+       id="path59"
+       sodipodi:nodetypes="ccc" />
+  </g>
+</svg>
diff --git a/Documentation/gpu/amdgpu/index.rst b/Documentation/gpu/amdgpu/index.rst
index 45523e9860fc..8732084186a4 100644
--- a/Documentation/gpu/amdgpu/index.rst
+++ b/Documentation/gpu/amdgpu/index.rst
@@ -8,6 +8,7 @@ Next (GCN), Radeon DNA (RDNA), and Compute DNA (CDNA) architectures.
 .. toctree::
 
    driver-core
+   ring-buffer
    amd-hardware-list-info
    module-parameters
    gc/index
diff --git a/Documentation/gpu/amdgpu/no_enforce_isolation.svg b/Documentation/gpu/amdgpu/no_enforce_isolation.svg
new file mode 100644
index 000000000000..b224615e1611
--- /dev/null
+++ b/Documentation/gpu/amdgpu/no_enforce_isolation.svg
@@ -0,0 +1,707 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   width="68.949203mm"
+   height="86.909332mm"
+   viewBox="0 0 68.949202 86.909332"
+   version="1.1"
+   id="svg1"
+   inkscape:version="1.4 (e7c3feb100, 2024-10-09)"
+   sodipodi:docname="enforce_isolation.svg"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview1"
+     pagecolor="#ffffff"
+     bordercolor="#000000"
+     borderopacity="0.25"
+     inkscape:showpageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1"
+     inkscape:document-units="mm"
+     inkscape:zoom="1.7877362"
+     inkscape:cx="291.15034"
+     inkscape:cy="332.54347"
+     inkscape:window-width="3072"
+     inkscape:window-height="1651"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="g61" />
+  <defs
+     id="defs1">
+    <marker
+       style="overflow:visible"
+       id="ArrowWideHeavy"
+       refX="0"
+       refY="0"
+       orient="auto-start-reverse"
+       inkscape:stockid="Wide, heavy arrow"
+       markerWidth="1"
+       markerHeight="1"
+       viewBox="0 0 1 1"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         style="fill:context-stroke;fill-rule:evenodd;stroke:none"
+         d="m 1,0 -3,3 h -2 l 3,-3 -3,-3 h 2 z"
+         id="path3" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Dot"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Dot"
+       markerWidth="1"
+       markerHeight="1"
+       viewBox="0 0 1 1"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.5)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:none"
+         d="M 5,0 C 5,2.76 2.76,5 0,5 -2.76,5 -5,2.76 -5,0 c 0,-2.76 2.3,-5 5,-5 2.76,0 5,2.24 5,5 z"
+         sodipodi:nodetypes="sssss"
+         id="path98" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker99"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Dot"
+       markerWidth="1"
+       markerHeight="1"
+       viewBox="0 0 1 1"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.5)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:none"
+         d="M 5,0 C 5,2.76 2.76,5 0,5 -2.76,5 -5,2.76 -5,0 c 0,-2.76 2.3,-5 5,-5 2.76,0 5,2.24 5,5 z"
+         sodipodi:nodetypes="sssss"
+         id="path99" />
+    </marker>
+  </defs>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(72.009598,0.94510132)">
+    <g
+       id="g61">
+      <rect
+         style="fill:#ffffff;stroke:none;stroke-width:0.25654;stroke-linecap:square;stroke-dasharray:none"
+         id="rect6"
+         width="68.949203"
+         height="86.909332"
+         x="-72.009598"
+         y="-0.94510132" />
+      <g
+         id="g15"
+         transform="matrix(0.42247861,0,0,0.42247861,-70.575576,-29.756289)">
+        <path
+           id="path33"
+           style="fill:none;stroke:#000000;stroke-width:0.721067;stroke-dasharray:none"
+           d="M 30.278993,176.45537 A 22.905334,22.905334 0 0 0 7.3737955,199.36057 22.905334,22.905334 0 0 0 30.278993,222.26603 22.905334,22.905334 0 0 0 53.18445,199.36057 22.905334,22.905334 0 0 0 30.278993,176.45537 Z m 0,7.13274 a 15.772359,15.772359 0 0 1 15.77246,15.77246 15.772359,15.772359 0 0 1 -15.77246,15.77246 15.772359,15.772359 0 0 1 -15.772206,-15.77246 15.772359,15.772359 0 0 1 15.772206,-15.77246 z" />
+        <g
+           id="g14">
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 46.31405,200.72093 h 6.430134"
+             id="path34"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="M 7.6848605,200.72093 H 14.114993"
+             id="path35"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 30.103415,176.7326 v 6.43014"
+             id="path36"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="M 30.103415,215.50586 V 221.936"
+             id="path37"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 36.501694,214.1052 3.21507,5.56866"
+             id="path38"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 36.501694,184.66805 3.21507,-5.56866"
+             id="path39"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 23.652811,184.66805 -3.21507,-5.56866"
+             id="path40"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 23.652811,214.05288 -3.21507,5.56866"
+             id="path41"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 43.31839,208.65101 4.546794,4.5468"
+             id="path42"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 43.31839,189.96525 4.546794,-4.5468"
+             id="path43"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="M 17.202394,189.96525 12.6556,185.41845"
+             id="path44-1"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 16.940766,208.65101 -4.546794,4.5468"
+             id="path45-8"
+             sodipodi:nodetypes="cc" />
+        </g>
+      </g>
+      <rect
+         style="fill:#afe9c6;stroke:#16502d;stroke-width:0.257104;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+         id="rect80"
+         width="25.076588"
+         height="10.493422"
+         x="-71.362038"
+         y="74.63459"
+         ry="1.4529352" />
+      <rect
+         style="fill:#80e5ff;stroke:#00aad4;stroke-width:0.257104;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+         id="rect81"
+         width="32.714355"
+         height="10.436013"
+         x="-40.481403"
+         y="74.663292"
+         ry="1.4449863" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:6.53042px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-59.013123"
+         y="82.212669"
+         id="text14-7"><tspan
+           sodipodi:role="line"
+           x="-59.013123"
+           y="82.212669"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.53042px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.102999"
+           id="tspan14-7">GFX</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:6.53042px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-24.387218"
+         y="81.46167"
+         id="text95"><tspan
+           sodipodi:role="line"
+           x="-24.387218"
+           y="81.46167"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.53042px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.102999"
+           id="tspan95">Compute</tspan></text>
+      <path
+         style="fill:#de8787;stroke:#000000;stroke-width:0.385656;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#Dot);marker-end:url(#marker99)"
+         d="M -58.079264,73.89605 V 65.448872"
+         id="path127"
+         sodipodi:nodetypes="cc" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.90242px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-61.752651"
+         y="3.1995225"
+         id="text112"><tspan
+           sodipodi:role="line"
+           x="-61.752651"
+           y="3.1995225"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.90242px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.102999"
+           id="tspan112">Processes</tspan></text>
+      <rect
+         style="fill:none;stroke:#000000;stroke-width:0.247306;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:1.48383, 1.48383;stroke-dashoffset:0"
+         id="rect113"
+         width="64.703476"
+         height="19.562067"
+         x="-68.52655"
+         y="4.468956"
+         ry="1.6976216" />
+      <g
+         id="g84"
+         transform="matrix(0.25710378,0,0,0.25710378,0.68647434,8.7899633)">
+        <g
+           id="g83">
+          <circle
+             style="fill:#ffffff;stroke:#ff00ff;stroke-width:1.62704;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;stroke-dashoffset:0"
+             id="circle137"
+             cx="-225.65012"
+             cy="20.747513"
+             r="30.822298" />
+        </g>
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:16.9333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff00ff;fill-opacity:1;stroke:none;stroke-width:0.400612"
+           x="-225.65012"
+           y="26.8181"
+           id="text113"><tspan
+             sodipodi:role="line"
+             x="-225.65012"
+             y="26.8181"
+             style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:16.9333px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff00ff;stroke-width:0.400612"
+             id="tspan113">A</tspan></text>
+      </g>
+      <g
+         id="g85"
+         transform="matrix(0.25710378,0,0,0.25710378,20.618429,8.7899633)">
+        <circle
+           style="fill:#ffffff;stroke:#ff9955;stroke-width:1.62704;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;stroke-dashoffset:0"
+           id="circle112"
+           cx="-207.94376"
+           cy="20.747513"
+           r="30.822298" />
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:16.9333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.400612"
+           x="-208.25702"
+           y="26.792702"
+           id="text114"><tspan
+             sodipodi:role="line"
+             x="-208.25702"
+             y="26.792702"
+             style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:16.9333px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.400612"
+             id="tspan114">B</tspan></text>
+      </g>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff00ff;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-56.126556"
+         y="47.212101"
+         id="text6"><tspan
+           sodipodi:role="line"
+           x="-56.126556"
+           y="47.212101"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff00ff;stroke-width:0.102999"
+           id="tspan6">A</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.26752px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-57.861526"
+         y="53.902462"
+         id="text136"><tspan
+           sodipodi:role="line"
+           x="-57.861526"
+           y="53.902462"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.26752px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.102999"
+           id="tspan136">Ring</tspan><tspan
+           sodipodi:role="line"
+           x="-57.861526"
+           y="56.736862"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.26752px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.102999"
+           id="tspan137">Buffer</tspan></text>
+      <g
+         id="g80"
+         transform="matrix(0.42247861,0,0,0.42247861,-37.21188,-29.756289)">
+        <path
+           id="path67"
+           style="fill:none;stroke:#000000;stroke-width:0.721067;stroke-dasharray:none"
+           d="M 30.278993,176.45537 A 22.905334,22.905334 0 0 0 7.3737955,199.36057 22.905334,22.905334 0 0 0 30.278993,222.26603 22.905334,22.905334 0 0 0 53.18445,199.36057 22.905334,22.905334 0 0 0 30.278993,176.45537 Z m 0,7.13274 a 15.772359,15.772359 0 0 1 15.77246,15.77246 15.772359,15.772359 0 0 1 -15.77246,15.77246 15.772359,15.772359 0 0 1 -15.772206,-15.77246 15.772359,15.772359 0 0 1 15.772206,-15.77246 z" />
+        <g
+           id="g79">
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 46.31405,200.72093 h 6.430134"
+             id="path68"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="M 7.6848605,200.72093 H 14.114993"
+             id="path69"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 30.103415,176.7326 v 6.43014"
+             id="path70"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="M 30.103415,215.50586 V 221.936"
+             id="path71"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 36.501694,214.1052 3.21507,5.56866"
+             id="path72"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 36.501694,184.66805 3.21507,-5.56866"
+             id="path73"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 23.652811,184.66805 -3.21507,-5.56866"
+             id="path74"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 23.652811,214.05288 -3.21507,5.56866"
+             id="path75"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 43.31839,208.65101 4.546794,4.5468"
+             id="path76"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 43.31839,189.96525 4.546794,-4.5468"
+             id="path77"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="M 17.202394,189.96525 12.6556,185.41845"
+             id="path78"
+             sodipodi:nodetypes="cc" />
+          <path
+             style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+             d="m 16.940766,208.65101 -4.546794,4.5468"
+             id="path79"
+             sodipodi:nodetypes="cc" />
+        </g>
+      </g>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.26752px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-24.497828"
+         y="53.902462"
+         id="text81"><tspan
+           sodipodi:role="line"
+           x="-24.497828"
+           y="53.902462"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.26752px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.102999"
+           id="tspan80">Ring</tspan><tspan
+           sodipodi:role="line"
+           x="-24.497828"
+           y="56.736862"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.26752px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.102999"
+           id="tspan81">Buffer</tspan></text>
+      <path
+         style="fill:#de8787;stroke:#000000;stroke-width:0.385656;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#Dot);marker-end:url(#marker99)"
+         d="M -24.338879,73.89605 V 65.448872"
+         id="path81"
+         sodipodi:nodetypes="cc" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff00ff;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-52.574932"
+         y="48.826473"
+         id="text82"><tspan
+           sodipodi:role="line"
+           x="-52.574932"
+           y="48.826473"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff00ff;stroke-width:0.102999"
+           id="tspan82">A</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff00ff;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-50.045757"
+         y="52.862404"
+         id="text83"><tspan
+           sodipodi:role="line"
+           x="-50.045757"
+           y="52.862404"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff00ff;stroke-width:0.102999"
+           id="tspan83">A</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff00ff;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-31.907158"
+         y="58.226768"
+         id="text83-59"><tspan
+           sodipodi:role="line"
+           x="-31.907158"
+           y="58.226768"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff00ff;stroke-width:0.102999"
+           id="tspan83-7">A</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff00ff;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-32.436516"
+         y="53.169308"
+         id="text83-59-3"><tspan
+           sodipodi:role="line"
+           x="-32.436516"
+           y="53.169308"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff00ff;stroke-width:0.102999"
+           id="tspan83-7-6">A</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff00ff;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-65.422112"
+         y="57.972916"
+         id="text83-7"><tspan
+           sodipodi:role="line"
+           x="-65.422112"
+           y="57.972916"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff00ff;stroke-width:0.102999"
+           id="tspan83-8">A</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff00ff;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-65.713165"
+         y="52.732723"
+         id="text83-7-4"><tspan
+           sodipodi:role="line"
+           x="-65.713165"
+           y="52.732723"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff00ff;stroke-width:0.102999"
+           id="tspan83-8-3">A</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff0000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-52.897129"
+         y="61.668709"
+         id="text83-5"><tspan
+           sodipodi:role="line"
+           x="-52.897129"
+           y="61.668709"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.102999"
+           id="tspan83-4">C</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff0000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-19.174068"
+         y="49.045818"
+         id="text83-5-9"><tspan
+           sodipodi:role="line"
+           x="-19.174068"
+           y="49.045818"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.102999"
+           id="tspan83-4-2">C</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff0000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-16.531792"
+         y="53.259804"
+         id="text83-5-9-9"><tspan
+           sodipodi:role="line"
+           x="-16.531792"
+           y="53.259804"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.102999"
+           id="tspan83-4-2-5">C</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-19.325029"
+         y="61.488995"
+         id="text83-5-9-9-7"><tspan
+           sodipodi:role="line"
+           x="-19.325029"
+           y="61.488995"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.102999"
+           id="tspan83-4-2-5-2">B</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-22.869593"
+         y="63.231686"
+         id="text83-5-9-9-7-0"><tspan
+           sodipodi:role="line"
+           x="-22.869593"
+           y="63.231686"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.102999"
+           id="tspan83-4-2-5-2-6">B</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-26.235374"
+         y="63.331181"
+         id="text83-5-9-9-7-0-4"><tspan
+           sodipodi:role="line"
+           x="-26.235374"
+           y="63.331181"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.102999"
+           id="tspan83-4-2-5-2-6-9">B</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff0000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-56.235538"
+         y="63.072704"
+         id="text83-5-6"><tspan
+           sodipodi:role="line"
+           x="-56.235538"
+           y="63.072704"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.102999"
+           id="tspan83-4-9">C</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:2.17681px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff0000;fill-opacity:1;stroke:none;stroke-width:0.102999"
+         x="-59.697765"
+         y="63.066635"
+         id="text83-5-6-5"><tspan
+           sodipodi:role="line"
+           x="-59.697765"
+           y="63.066635"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.17681px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.102999"
+           id="tspan83-4-9-2">C</tspan></text>
+      <circle
+         style="fill:#ffcc00;stroke:#00d455;stroke-width:0.213261;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+         id="path83"
+         cx="-50.145481"
+         cy="57.387428"
+         r="1.0712636" />
+      <circle
+         style="fill:#ffcc00;stroke:#00d455;stroke-width:0.213261;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+         id="path83-7"
+         cx="-16.886913"
+         cy="57.596024"
+         r="1.0712636" />
+      <circle
+         style="fill:#ffcc00;stroke:#00d455;stroke-width:0.213261;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+         id="path83-7-7"
+         cx="-29.53648"
+         cy="60.832634"
+         r="1.0712636" />
+      <circle
+         style="fill:#ffcc00;stroke:#00d455;stroke-width:0.213261;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+         id="path83-7-7-9"
+         cx="-29.751556"
+         cy="48.260994"
+         r="1.0712636" />
+      <circle
+         style="fill:#ffcc00;stroke:#00d455;stroke-width:0.213261;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+         id="path83-4"
+         cx="-63.051891"
+         cy="60.73439"
+         r="1.0712636" />
+      <circle
+         style="fill:#ffcc00;stroke:#00d455;stroke-width:0.213261;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+         id="path83-4-2"
+         cx="-63.022129"
+         cy="48.24374"
+         r="1.0712636" />
+      <g
+         id="g86"
+         transform="matrix(0.25710378,0,0,0.25710378,17.422136,8.7899633)">
+        <circle
+           style="fill:#ffffff;stroke:#ff0000;stroke-width:1.62704;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;stroke-dashoffset:0"
+           id="circle84"
+           cx="-121.9205"
+           cy="20.747513"
+           r="30.822298" />
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:16.9333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.400612"
+           x="-122.11524"
+           y="26.792702"
+           id="text84"><tspan
+             sodipodi:role="line"
+             x="-122.11524"
+             y="26.792702"
+             style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:16.9333px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.400612"
+             id="tspan84">C</tspan></text>
+      </g>
+      <text
+         xml:space="preserve"
+         style="font-size:2.17681px;font-family:'Linux Libertine O';-inkscape-font-specification:'Linux Libertine O';text-align:start;letter-spacing:1.26323px;writing-mode:lr-tb;direction:ltr;text-anchor:start;fill:none;stroke:#000000;stroke-width:0.257104;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+         x="-140.03215"
+         y="26.074423"
+         id="text86"><tspan
+           sodipodi:role="line"
+           id="tspan86"
+           style="stroke-width:0.257104"
+           x="-140.03215"
+           y="26.074423" /></text>
+      <g
+         id="g90"
+         transform="matrix(0.25710378,0,0,0.25710378,18.175509,7.82134)">
+        <rect
+           style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+           id="rect86"
+           width="92.604057"
+           height="26.883123"
+           x="-338.30258"
+           y="93.635468" />
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:8.81944px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+           x="-291.13989"
+           y="110.58046"
+           id="text88"><tspan
+             sodipodi:role="line"
+             x="-291.13989"
+             y="110.58046"
+             style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:8.81944px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+             id="tspan88">Enforce Isolation</tspan></text>
+      </g>
+      <g
+         id="g89"
+         transform="matrix(0.25710378,0,0,0.25710378,18.606009,7.82134)">
+        <rect
+           style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+           id="rect89"
+           width="119.67937"
+           height="26.749132"
+           x="-219.35185"
+           y="93.702461" />
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:8.81944px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+           x="-159.57832"
+           y="110.40636"
+           id="text89"><tspan
+             sodipodi:role="line"
+             x="-159.57832"
+             y="110.40636"
+             style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:8.81944px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+             id="tspan89">Enforce Isolation</tspan></text>
+      </g>
+      <path
+         style="fill:#de8787;stroke:#000000;stroke-width:0.385656;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#Dot);marker-end:url(#marker99)"
+         d="M -57.807162,43.759556 V 39.878518"
+         id="path90"
+         sodipodi:nodetypes="cc" />
+      <path
+         style="fill:#de8787;stroke:#000000;stroke-width:0.385656;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#Dot);marker-end:url(#marker99)"
+         d="M -24.512426,43.759556 V 39.878518"
+         id="path91"
+         sodipodi:nodetypes="cc" />
+      <path
+         style="fill:#ff00ff;stroke:#ff00ff;stroke-width:0.257104;stroke-linecap:square;stroke-dasharray:none;marker-end:url(#ArrowWideHeavy)"
+         d="m -57.329023,22.418572 v 9.208373"
+         id="path1" />
+      <path
+         style="fill:#ff00ff;stroke:#ff9955;stroke-width:0.257104;stroke-linecap:square;stroke-dasharray:none;marker-end:url(#ArrowWideHeavy)"
+         d="m -32.844695,22.418572 v 9.208373"
+         id="path2" />
+      <path
+         style="fill:#ff00ff;stroke:#ff0000;stroke-width:0.257104;stroke-linecap:square;stroke-dasharray:none;marker-end:url(#ArrowWideHeavy)"
+         d="m -13.924085,22.418572 v 9.208373"
+         id="path4" />
+      <path
+         style="fill:none;stroke:#ff00ff;stroke-width:0.257104;stroke-linecap:square;stroke-dasharray:none;marker-end:url(#ArrowWideHeavy)"
+         d="m -57.329023,22.418572 v 5.174952 h 20.852017 v 4.033421"
+         id="path5"
+         sodipodi:nodetypes="cccc" />
+      <path
+         style="fill:none;stroke:#ff0000;stroke-width:0.257104;stroke-linecap:square;stroke-dasharray:none;marker-end:url(#ArrowWideHeavy)"
+         d="m -13.924085,22.418572 v 3.652908 h -34.777483 v 5.555465"
+         id="path6"
+         sodipodi:nodetypes="cccc" />
+    </g>
+  </g>
+</svg>
diff --git a/Documentation/gpu/amdgpu/process-isolation.rst b/Documentation/gpu/amdgpu/process-isolation.rst
index 25b06ffefc33..03c4288aa8b1 100644
--- a/Documentation/gpu/amdgpu/process-isolation.rst
+++ b/Documentation/gpu/amdgpu/process-isolation.rst
@@ -1,3 +1,4 @@
+.. _amdgpu-process-isolation:
 .. SPDX-License-Identifier: GPL-2.0
 
 =========================
diff --git a/Documentation/gpu/amdgpu/ring-buffer.rst b/Documentation/gpu/amdgpu/ring-buffer.rst
new file mode 100644
index 000000000000..cc642c21316b
--- /dev/null
+++ b/Documentation/gpu/amdgpu/ring-buffer.rst
@@ -0,0 +1,95 @@
+=============
+ Ring Buffer
+=============
+
+To handle communication between user space and kernel space, AMD GPUs use a
+ring buffer design to feed the engines (GFX, Compute, SDMA, UVD, VCE, VCN, VPE,
+etc.). See the figure below that illustrates how this communication works:
+
+.. kernel-figure:: ring_buffers.svg
+
+Ring buffers in the amdgpu work as a producer-consumer model, where userspace
+acts as the producer, constantly filling the ring buffer with GPU commands to
+be executed. Meanwhile, the GPU retrieves the information from the ring, parses
+it, and distributes the specific set of instructions between the different
+amdgpu blocks.
+
+Notice from the diagram that the ring has a Read Pointer (rptr), which
+indicates where the engine is currently reading packets from the ring, and a
+Write Pointer (wptr), which indicates how many packets software has added to
+the ring. When the rptr and wptr are equal, the ring is idle. When software
+adds packets to the ring, it updates the wptr, this causes the engine to start
+fetching and processing packets. As the engine processes packets, the rptr gets
+updates until the rptr catches up to the wptr and they are equal again.
+
+Usually, ring buffers in the driver have a limited size (search for occurrences
+of `amdgpu_ring_init()`). One of the reasons for the small ring buffer size is
+that CP (Command Processor) is capable of following addresses inserted into the
+ring; this is illustrated in the image by the reference to the IB (Indirect
+Buffer). The IB gives userspace the possibility to have an area in memory that
+CP can read and feed the hardware with extra instructions.
+
+All ASICs pre-GFX11 use what is called a kernel queue, which means
+the ring is allocated in kernel space and has some restrictions, such as not
+being able to be :ref:`preempted directly by the scheduler<amdgpu-mes>`. GFX11
+and newer support kernel queues, but also provide a new mechanism named
+:ref:`user queues<amdgpu-userq>`, where the queue is moved to the user space
+and can be mapped and unmapped via the scheduler. In practice, both queues
+insert user-space-generated GPU commands from different jobs into the requested
+component ring.
+
+Enforce Isolation
+=================
+
+.. note:: After reading this section, you might want to check the
+   :ref:`Process Isolation<amdgpu-process-isolation>` page for more details.
+
+Before examining the Enforce Isolation mechanism in the ring buffer context, it
+is helpful to briefly discuss how instructions from the ring buffer are
+processed in the graphics pipeline. Let’s expand on this topic by checking the
+diagram below that illustrates the graphics pipeline:
+
+.. kernel-figure:: gfx_pipeline_seq.svg
+
+In terms of executing instructions, the GFX pipeline follows the sequence:
+Shader Export (SX), Geometry Engine (GE), Shader Process or Input (SPI), Scan
+Converter (SC), Primitive Assembler (PA), and cache manipulation (which may
+vary across ASICs). Another common way to describe the pipeline is to use Pixel
+Shader (PS), raster, and Vertex Shader (VS) to symbolize the two shader stages.
+Now, with this pipeline in mind, let's assume that Job B causes a hang issue,
+but Job C's instruction might already be executing, leading developers to
+incorrectly identify Job C as the problematic one. This problem can be
+mitigated on multiple levels; the diagram below illustrates how to minimize
+part of this problem:
+
+.. kernel-figure:: no_enforce_isolation.svg
+
+Note from the diagram that there is no guarantee of order or a clear separation
+between instructions, which is not a problem most of the time, and is also good
+for performance. Furthermore, notice some circles between jobs in the diagram
+that represent a **fence wait** used to avoid overlapping work in the ring. At
+the end of the fence, a cache flush occurs, ensuring that when the next job
+starts, it begins in a clean state and, if issues arise, the developer can
+pinpoint the problematic process more precisely.
+
+To increase the level of isolation between jobs, there is the "Enforce
+Isolation" method described in the picture below:
+
+.. kernel-figure:: enforce_isolation.svg
+
+As shown in the diagram, enforcing isolation introduces ordering between
+submissions, since the access to GFX/Compute is serialized, think about it as
+single process at a time mode for gfx/compute. Notice that this approach has a
+significant performance impact, as it allows only one job to submit commands at
+a time. However, this option can help pinpoint the job that caused the problem.
+Although enforcing isolation improves the situation, it does not fully resolve
+the issue of precisely pinpointing bad jobs, since isolation might mask the
+problem. In summary, identifying which job caused the issue may not be precise,
+but enforcing isolation might help with the debugging.
+
+Ring Operations
+===============
+
+.. kernel-doc:: drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+   :internal:
+
diff --git a/Documentation/gpu/amdgpu/ring_buffers.svg b/Documentation/gpu/amdgpu/ring_buffers.svg
new file mode 100644
index 000000000000..7a6fcb19e151
--- /dev/null
+++ b/Documentation/gpu/amdgpu/ring_buffers.svg
@@ -0,0 +1,1633 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   width="588.32483mm"
+   height="341.81656mm"
+   viewBox="0 0 588.32483 341.81656"
+   version="1.1"
+   id="svg1"
+   inkscape:version="1.4 (e7c3feb100, 2024-10-09)"
+   sodipodi:docname="RING_BUFFER.svg"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview1"
+     pagecolor="#ffffff"
+     bordercolor="#000000"
+     borderopacity="0.25"
+     inkscape:showpageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1"
+     inkscape:document-units="mm"
+     inkscape:zoom="0.89386809"
+     inkscape:cx="1106.9866"
+     inkscape:cy="729.97348"
+     inkscape:window-width="3072"
+     inkscape:window-height="1651"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="layer1" />
+  <defs
+     id="defs1">
+    <marker
+       style="overflow:visible"
+       id="marker154"
+       refX="2"
+       refY="0"
+       orient="auto-start-reverse"
+       inkscape:stockid="Empty semicircle"
+       markerWidth="1"
+       markerHeight="1"
+       viewBox="0 0 1 1"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         style="fill:none;fill-rule:evenodd;stroke:context-stroke;stroke-width:1.08"
+         d="m -0.7647042,-3.0274156 c 1.656,0 3,1.344 3,3 0,1.656 -1.344,3 -3,3"
+         id="path154" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="ArrowTriangleStylized"
+       refX="0"
+       refY="0"
+       orient="auto-start-reverse"
+       inkscape:stockid="Stylized triangle arrow"
+       markerWidth="1"
+       markerHeight="1"
+       viewBox="0 0 1 1"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.5)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="m 6,0 c -3,1 -7,3 -9,5 0,0 0,-4 2,-5 -2,-1 -2,-5 -2,-5 2,2 6,4 9,5 z"
+         id="path135" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker132"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Diamond"
+       markerWidth="1"
+       markerHeight="1"
+       viewBox="0 0 1 1"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.45)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:none"
+         d="M 0,-7.0710768 -7.0710894,0 0,7.0710589 7.0710462,0 Z"
+         id="path132" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Diamond"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Diamond"
+       markerWidth="1"
+       markerHeight="1"
+       viewBox="0 0 1 1"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.45)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:none"
+         d="M 0,-7.0710768 -7.0710894,0 0,7.0710589 7.0710462,0 Z"
+         id="path131" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="ArrowWide"
+       refX="0"
+       refY="0"
+       orient="auto-start-reverse"
+       inkscape:stockid="Wide arrow"
+       markerWidth="1"
+       markerHeight="1"
+       viewBox="0 0 1 1"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         style="fill:none;stroke:context-stroke;stroke-width:1;stroke-linecap:butt"
+         d="M 3,-3 0,0 3,3"
+         transform="rotate(180,0.125,0)"
+         sodipodi:nodetypes="ccc"
+         id="path81" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="marker99"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Dot"
+       markerWidth="1"
+       markerHeight="1"
+       viewBox="0 0 1 1"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.5)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:none"
+         d="M 5,0 C 5,2.76 2.76,5 0,5 -2.76,5 -5,2.76 -5,0 c 0,-2.76 2.3,-5 5,-5 2.76,0 5,2.24 5,5 z"
+         sodipodi:nodetypes="sssss"
+         id="path99" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="Dot"
+       refX="0"
+       refY="0"
+       orient="auto"
+       inkscape:stockid="Dot"
+       markerWidth="1"
+       markerHeight="1"
+       viewBox="0 0 1 1"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.5)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:none"
+         d="M 5,0 C 5,2.76 2.76,5 0,5 -2.76,5 -5,2.76 -5,0 c 0,-2.76 2.3,-5 5,-5 2.76,0 5,2.24 5,5 z"
+         sodipodi:nodetypes="sssss"
+         id="path98" />
+    </marker>
+  </defs>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(123.51219,3.0159921)">
+    <rect
+       style="fill:#ffffff;stroke:none;stroke-width:1;stroke-linecap:square;stroke-dasharray:1, 2;stroke-dashoffset:0"
+       id="rect167"
+       width="595.25238"
+       height="349.86984"
+       x="-128.24815"
+       y="-8.0479612" />
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:1, 2;stroke-dashoffset:0"
+       id="rect111"
+       width="500.64935"
+       height="63.365536"
+       x="-65.219322"
+       y="168.41347"
+       ry="5.4989429" />
+    <path
+       style="fill:none;stroke:#917c6f;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#marker154)"
+       d="M 24.09216,175.30492 V 125.06784 H 47.447018 V 83.98949"
+       id="path151"
+       sodipodi:nodetypes="cccc" />
+    <g
+       id="g15"
+       transform="translate(-4.735971)">
+      <path
+         id="path1"
+         style="fill:none;stroke:#000000;stroke-width:0.721067;stroke-dasharray:none"
+         d="M 30.278993,176.45537 A 22.905334,22.905334 0 0 0 7.3737955,199.36057 22.905334,22.905334 0 0 0 30.278993,222.26603 22.905334,22.905334 0 0 0 53.18445,199.36057 22.905334,22.905334 0 0 0 30.278993,176.45537 Z m 0,7.13274 a 15.772359,15.772359 0 0 1 15.77246,15.77246 15.772359,15.772359 0 0 1 -15.77246,15.77246 15.772359,15.772359 0 0 1 -15.772206,-15.77246 15.772359,15.772359 0 0 1 15.772206,-15.77246 z" />
+      <g
+         id="g14">
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 46.31405,200.72093 h 6.430134"
+           id="path3"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 7.6848605,200.72093 H 14.114993"
+           id="path4"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 30.103415,176.7326 v 6.43014"
+           id="path5"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 30.103415,215.50586 V 221.936"
+           id="path6"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,214.1052 3.21507,5.56866"
+           id="path7"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,184.66805 3.21507,-5.56866"
+           id="path8"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,184.66805 -3.21507,-5.56866"
+           id="path9"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,214.05288 -3.21507,5.56866"
+           id="path10"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,208.65101 4.546794,4.5468"
+           id="path11"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,189.96525 4.546794,-4.5468"
+           id="path12"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 17.202394,189.96525 12.6556,185.41845"
+           id="path13"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 16.940766,208.65101 -4.546794,4.5468"
+           id="path14"
+           sodipodi:nodetypes="cc" />
+      </g>
+    </g>
+    <g
+       id="g28"
+       transform="translate(46.175717)">
+      <path
+         id="path15"
+         style="fill:none;stroke:#000000;stroke-width:0.721067;stroke-dasharray:none"
+         d="M 30.278993,176.45537 A 22.905334,22.905334 0 0 0 7.3737955,199.36057 22.905334,22.905334 0 0 0 30.278993,222.26603 22.905334,22.905334 0 0 0 53.18445,199.36057 22.905334,22.905334 0 0 0 30.278993,176.45537 Z m 0,7.13274 a 15.772359,15.772359 0 0 1 15.77246,15.77246 15.772359,15.772359 0 0 1 -15.77246,15.77246 15.772359,15.772359 0 0 1 -15.772206,-15.77246 15.772359,15.772359 0 0 1 15.772206,-15.77246 z" />
+      <g
+         id="g27">
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 46.31405,200.72093 h 6.430134"
+           id="path16"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 7.6848605,200.72093 H 14.114993"
+           id="path17"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 30.103415,176.7326 v 6.43014"
+           id="path18"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 30.103415,215.50586 V 221.936"
+           id="path19"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,214.1052 3.21507,5.56866"
+           id="path20"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,184.66805 3.21507,-5.56866"
+           id="path21"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,184.66805 -3.21507,-5.56866"
+           id="path22"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,214.05288 -3.21507,5.56866"
+           id="path23"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,208.65101 4.546794,4.5468"
+           id="path24"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,189.96525 4.546794,-4.5468"
+           id="path25"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 17.202394,189.96525 12.6556,185.41845"
+           id="path26"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 16.940766,208.65101 -4.546794,4.5468"
+           id="path27"
+           sodipodi:nodetypes="cc" />
+      </g>
+    </g>
+    <g
+       id="g41"
+       transform="translate(102.62407)">
+      <path
+         id="path28"
+         style="fill:none;stroke:#000000;stroke-width:0.721067;stroke-dasharray:none"
+         d="M 30.278993,176.45537 A 22.905334,22.905334 0 0 0 7.3737955,199.36057 22.905334,22.905334 0 0 0 30.278993,222.26603 22.905334,22.905334 0 0 0 53.18445,199.36057 22.905334,22.905334 0 0 0 30.278993,176.45537 Z m 0,7.13274 a 15.772359,15.772359 0 0 1 15.77246,15.77246 15.772359,15.772359 0 0 1 -15.77246,15.77246 15.772359,15.772359 0 0 1 -15.772206,-15.77246 15.772359,15.772359 0 0 1 15.772206,-15.77246 z" />
+      <g
+         id="g40">
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 46.31405,200.72093 h 6.430134"
+           id="path29"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 7.6848605,200.72093 H 14.114993"
+           id="path30"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 30.103415,176.7326 v 6.43014"
+           id="path31"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 30.103415,215.50586 V 221.936"
+           id="path32"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,214.1052 3.21507,5.56866"
+           id="path33"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,184.66805 3.21507,-5.56866"
+           id="path34"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,184.66805 -3.21507,-5.56866"
+           id="path35"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,214.05288 -3.21507,5.56866"
+           id="path36"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,208.65101 4.546794,4.5468"
+           id="path37"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,189.96525 4.546794,-4.5468"
+           id="path38"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 17.202394,189.96525 12.6556,185.41845"
+           id="path39"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 16.940766,208.65101 -4.546794,4.5468"
+           id="path40"
+           sodipodi:nodetypes="cc" />
+      </g>
+    </g>
+    <g
+       id="g54"
+       transform="translate(154.12775)">
+      <path
+         id="path41"
+         style="fill:none;stroke:#000000;stroke-width:0.721067;stroke-dasharray:none"
+         d="M 30.278993,176.45537 A 22.905334,22.905334 0 0 0 7.3737955,199.36057 22.905334,22.905334 0 0 0 30.278993,222.26603 22.905334,22.905334 0 0 0 53.18445,199.36057 22.905334,22.905334 0 0 0 30.278993,176.45537 Z m 0,7.13274 a 15.772359,15.772359 0 0 1 15.77246,15.77246 15.772359,15.772359 0 0 1 -15.77246,15.77246 15.772359,15.772359 0 0 1 -15.772206,-15.77246 15.772359,15.772359 0 0 1 15.772206,-15.77246 z" />
+      <g
+         id="g53">
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 46.31405,200.72093 h 6.430134"
+           id="path42"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 7.6848605,200.72093 H 14.114993"
+           id="path43"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 30.103415,176.7326 v 6.43014"
+           id="path44"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 30.103415,215.50586 V 221.936"
+           id="path45"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,214.1052 3.21507,5.56866"
+           id="path46"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,184.66805 3.21507,-5.56866"
+           id="path47"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,184.66805 -3.21507,-5.56866"
+           id="path48"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,214.05288 -3.21507,5.56866"
+           id="path49"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,208.65101 4.546794,4.5468"
+           id="path50"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,189.96525 4.546794,-4.5468"
+           id="path51"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 17.202394,189.96525 12.6556,185.41845"
+           id="path52"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 16.940766,208.65101 -4.546794,4.5468"
+           id="path53"
+           sodipodi:nodetypes="cc" />
+      </g>
+    </g>
+    <g
+       id="g67"
+       transform="translate(205.37341)">
+      <path
+         id="path54"
+         style="fill:none;stroke:#000000;stroke-width:0.721067;stroke-dasharray:none"
+         d="M 30.278993,176.45537 A 22.905334,22.905334 0 0 0 7.3737955,199.36057 22.905334,22.905334 0 0 0 30.278993,222.26603 22.905334,22.905334 0 0 0 53.18445,199.36057 22.905334,22.905334 0 0 0 30.278993,176.45537 Z m 0,7.13274 a 15.772359,15.772359 0 0 1 15.77246,15.77246 15.772359,15.772359 0 0 1 -15.77246,15.77246 15.772359,15.772359 0 0 1 -15.772206,-15.77246 15.772359,15.772359 0 0 1 15.772206,-15.77246 z" />
+      <g
+         id="g66">
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 46.31405,200.72093 h 6.430134"
+           id="path55"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 7.6848605,200.72093 H 14.114993"
+           id="path56"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 30.103415,176.7326 v 6.43014"
+           id="path57"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 30.103415,215.50586 V 221.936"
+           id="path58"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,214.1052 3.21507,5.56866"
+           id="path59"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,184.66805 3.21507,-5.56866"
+           id="path60"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,184.66805 -3.21507,-5.56866"
+           id="path61"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,214.05288 -3.21507,5.56866"
+           id="path62"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,208.65101 4.546794,4.5468"
+           id="path63"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,189.96525 4.546794,-4.5468"
+           id="path64"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 17.202394,189.96525 12.6556,185.41845"
+           id="path65"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 16.940766,208.65101 -4.546794,4.5468"
+           id="path66"
+           sodipodi:nodetypes="cc" />
+      </g>
+    </g>
+    <g
+       id="g80"
+       transform="translate(256.63043)">
+      <path
+         id="path67"
+         style="fill:none;stroke:#000000;stroke-width:0.721067;stroke-dasharray:none"
+         d="M 30.278993,176.45537 A 22.905334,22.905334 0 0 0 7.3737955,199.36057 22.905334,22.905334 0 0 0 30.278993,222.26603 22.905334,22.905334 0 0 0 53.18445,199.36057 22.905334,22.905334 0 0 0 30.278993,176.45537 Z m 0,7.13274 a 15.772359,15.772359 0 0 1 15.77246,15.77246 15.772359,15.772359 0 0 1 -15.77246,15.77246 15.772359,15.772359 0 0 1 -15.772206,-15.77246 15.772359,15.772359 0 0 1 15.772206,-15.77246 z" />
+      <g
+         id="g79">
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 46.31405,200.72093 h 6.430134"
+           id="path68"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 7.6848605,200.72093 H 14.114993"
+           id="path69"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 30.103415,176.7326 v 6.43014"
+           id="path70"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 30.103415,215.50586 V 221.936"
+           id="path71"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,214.1052 3.21507,5.56866"
+           id="path72"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,184.66805 3.21507,-5.56866"
+           id="path73"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,184.66805 -3.21507,-5.56866"
+           id="path74"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,214.05288 -3.21507,5.56866"
+           id="path75"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,208.65101 4.546794,4.5468"
+           id="path76"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,189.96525 4.546794,-4.5468"
+           id="path77"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 17.202394,189.96525 12.6556,185.41845"
+           id="path78"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 16.940766,208.65101 -4.546794,4.5468"
+           id="path79"
+           sodipodi:nodetypes="cc" />
+      </g>
+    </g>
+    <rect
+       style="fill:#afe9c6;stroke:#16502d;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+       id="rect80"
+       width="97.534882"
+       height="40.813953"
+       x="2.9731095"
+       y="288.36279"
+       ry="5.6511626" />
+    <rect
+       style="fill:#80e5ff;stroke:#00aad4;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+       id="rect81"
+       width="199.46111"
+       height="40.391335"
+       x="110.41494"
+       y="288.57413"
+       ry="5.5926461" />
+    <rect
+       style="fill:#de8787;stroke:#a02c2c;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+       id="rect82"
+       width="81.028717"
+       height="40.88369"
+       x="321.87186"
+       y="288.32794"
+       ry="5.6608181" />
+    <g
+       id="g95"
+       transform="translate(332.1071)">
+      <path
+         id="path82"
+         style="fill:none;stroke:#000000;stroke-width:0.721067;stroke-dasharray:none"
+         d="M 30.278993,176.45537 A 22.905334,22.905334 0 0 0 7.3737955,199.36057 22.905334,22.905334 0 0 0 30.278993,222.26603 22.905334,22.905334 0 0 0 53.18445,199.36057 22.905334,22.905334 0 0 0 30.278993,176.45537 Z m 0,7.13274 a 15.772359,15.772359 0 0 1 15.77246,15.77246 15.772359,15.772359 0 0 1 -15.77246,15.77246 15.772359,15.772359 0 0 1 -15.772206,-15.77246 15.772359,15.772359 0 0 1 15.772206,-15.77246 z" />
+      <g
+         id="g94">
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 46.31405,200.72093 h 6.430134"
+           id="path83"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 7.6848605,200.72093 H 14.114993"
+           id="path84"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 30.103415,176.7326 v 6.43014"
+           id="path85"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 30.103415,215.50586 V 221.936"
+           id="path86"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,214.1052 3.21507,5.56866"
+           id="path87"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 36.501694,184.66805 3.21507,-5.56866"
+           id="path88"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,184.66805 -3.21507,-5.56866"
+           id="path89"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 23.652811,214.05288 -3.21507,5.56866"
+           id="path90"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,208.65101 4.546794,4.5468"
+           id="path91"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 43.31839,189.96525 4.546794,-4.5468"
+           id="path92"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="M 17.202394,189.96525 12.6556,185.41845"
+           id="path93"
+           sodipodi:nodetypes="cc" />
+        <path
+           style="fill:#ff00ff;stroke:#000000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+           d="m 16.940766,208.65101 -4.546794,4.5468"
+           id="path94"
+           sodipodi:nodetypes="cc" />
+      </g>
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:25.4px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="51.003948"
+       y="317.83759"
+       id="text14-7"><tspan
+         sodipodi:role="line"
+         x="51.003948"
+         y="317.83759"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:25.4px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan14-7">GFX</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:25.4px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="209.95499"
+       y="314.9166"
+       id="text95"><tspan
+         sodipodi:role="line"
+         x="209.95499"
+         y="314.9166"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:25.4px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan95">Compute</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:25.4px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="361.80203"
+       y="317.83759"
+       id="text96"><tspan
+         sodipodi:role="line"
+         x="361.80203"
+         y="317.83759"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:25.4px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan96">SDMA</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:36.862px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="429.28961"
+       y="313.10159"
+       id="text97"><tspan
+         sodipodi:role="line"
+         x="429.28961"
+         y="313.10159"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:36.862px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan97">...</tspan></text>
+    <path
+       style="fill:#de8787;stroke:#000000;stroke-width:1.5;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#Dot);marker-end:url(#marker99)"
+       d="M 25.543152,285.49025 V 224.58863"
+       id="path97"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#de8787;stroke:#000000;stroke-width:1.5;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#Dot);marker-end:url(#marker99)"
+       d="M 76.45484,285.49025 V 224.58863"
+       id="path100"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#de8787;stroke:#000000;stroke-width:1.5;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#Dot);marker-end:url(#marker99)"
+       d="M 132.90319,285.49025 V 224.58863"
+       id="path101"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#de8787;stroke:#000000;stroke-width:1.5;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#Dot);marker-end:url(#marker99)"
+       d="M 184.40687,285.49025 V 224.58863"
+       id="path102"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#de8787;stroke:#000000;stroke-width:1.5;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#Dot);marker-end:url(#marker99)"
+       d="M 235.65253,285.49025 V 224.58863"
+       id="path103"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#de8787;stroke:#000000;stroke-width:1.5;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#Dot);marker-end:url(#marker99)"
+       d="M 286.90955,285.49025 V 224.58863"
+       id="path104"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#de8787;stroke:#000000;stroke-width:1.5;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#Dot);marker-end:url(#marker99)"
+       d="M 362.38622,285.49025 V 224.58863"
+       id="path105"
+       sodipodi:nodetypes="cc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:36.862px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="261.28104"
+       y="259.52591"
+       id="text105"><tspan
+         sodipodi:role="line"
+         x="261.28104"
+         y="259.52591"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:36.862px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan105">...</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:36.862px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="50.998993"
+       y="259.52591"
+       id="text106"><tspan
+         sodipodi:role="line"
+         x="50.998993"
+         y="259.52591"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:36.862px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan106">...</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:36.862px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="382.22592"
+       y="259.52591"
+       id="text107"><tspan
+         sodipodi:role="line"
+         x="382.22592"
+         y="259.52591"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:36.862px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan107">...</tspan></text>
+    <path
+       style="fill:#de8787;stroke:#000000;stroke-width:2;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none"
+       d="M -65.568535,148.88709 H 459.54211"
+       id="path107" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:19.7556px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="415.21619"
+       y="166.24187"
+       id="text108"><tspan
+         sodipodi:role="line"
+         x="415.21619"
+         y="166.24187"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:19.7556px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan108">Kernel</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:19.7556px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="414.91986"
+       y="138.75868"
+       id="text109"><tspan
+         sodipodi:role="line"
+         x="414.91986"
+         y="138.75868"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:19.7556px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan109">Userspace</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:11.2889px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="-44.396496"
+       y="199.94142"
+       id="text110"><tspan
+         sodipodi:role="line"
+         x="-44.396496"
+         y="199.94142"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:11.2889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan110">Kernel</tspan><tspan
+         sodipodi:role="line"
+         x="-44.396496"
+         y="214.05255"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:11.2889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan111">Queue</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:11.2889px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="-30.39555"
+       y="53.215206"
+       id="text112"><tspan
+         sodipodi:role="line"
+         x="-30.39555"
+         y="53.215206"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:11.2889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan112">Processes</tspan></text>
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:1.5;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:9, 9;stroke-dashoffset:0"
+       id="rect113"
+       width="452.2225"
+       height="75.723717"
+       x="-60.944237"
+       y="11.868809"
+       ry="6.5714021" />
+    <circle
+       style="fill:#ffffff;stroke:#917c6f;stroke-width:1.62704;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;stroke-dashoffset:0"
+       id="path111"
+       cx="47.507706"
+       cy="49.241512"
+       r="30.822298" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:16.9333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#917c6f;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="47.50771"
+       y="55.312099"
+       id="text113"><tspan
+         sodipodi:role="line"
+         x="47.50771"
+         y="55.312099"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:16.9333px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#917c6f;stroke-width:0.400612"
+         id="tspan113">A</tspan></text>
+    <circle
+       style="fill:#ffffff;stroke:#ff9955;stroke-width:1.62704;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;stroke-dashoffset:0"
+       id="circle112"
+       cx="176.56291"
+       cy="49.241512"
+       r="30.822298" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:16.9333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="176.24965"
+       y="55.286701"
+       id="text114"><tspan
+         sodipodi:role="line"
+         x="176.24965"
+         y="55.286701"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:16.9333px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.400612"
+         id="tspan114">B</tspan></text>
+    <circle
+       style="fill:#ffffff;stroke:#55ddff;stroke-width:1.62704;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;stroke-dashoffset:0"
+       id="circle113"
+       cx="305.61813"
+       cy="49.241512"
+       r="30.822298" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:16.9333px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#55ddff;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="305.42337"
+       y="55.286701"
+       id="text115"><tspan
+         sodipodi:role="line"
+         x="305.42337"
+         y="55.286701"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:16.9333px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#55ddff;stroke-width:0.400612"
+         id="tspan115">C</tspan></text>
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:1.5;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:5.99998, 2.99999, 1.5, 2.99999;stroke-dashoffset:0"
+       id="rect117"
+       width="515.65753"
+       height="59.528286"
+       x="-65.221695"
+       y="278.51898"
+       ry="5.1659417" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:11.2889px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="-44.396496"
+       y="312.30948"
+       id="text118"><tspan
+         sodipodi:role="line"
+         x="-44.396496"
+         y="312.30948"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:11.2889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan118">GPU</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:7.05556px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="-17.792967"
+       y="194.41469"
+       id="text4"><tspan
+         sodipodi:role="line"
+         x="-17.792967"
+         y="194.41469"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.05556px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.400612"
+         id="tspan4">wptr</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:7.05556px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="36.248669"
+       y="163.10979"
+       id="text5"><tspan
+         sodipodi:role="line"
+         x="36.248669"
+         y="163.10979"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.05556px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#00ff00;stroke-width:0.400612"
+         id="tspan5">rptr</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ac9d93;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="28.952087"
+       y="182.41843"
+       id="text6"><tspan
+         sodipodi:role="line"
+         x="28.952087"
+         y="182.41843"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ac9d93;stroke-width:0.400612"
+         id="tspan6">A</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ac9d93;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="37.832031"
+       y="186.1924"
+       id="text7"><tspan
+         sodipodi:role="line"
+         x="37.832031"
+         y="186.1924"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ac9d93;stroke-width:0.400612"
+         id="tspan7">A</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ac9d93;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="37.98003"
+       y="216.31021"
+       id="text8"><tspan
+         sodipodi:role="line"
+         x="37.98003"
+         y="216.31021"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ac9d93;stroke-width:0.400612"
+         id="tspan8">A</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ac9d93;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="8.1582127"
+       y="208.54025"
+       id="text9"><tspan
+         sodipodi:role="line"
+         x="8.1582127"
+         y="208.54025"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ac9d93;stroke-width:0.400612"
+         id="tspan9">A</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ac9d93;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="114.03775"
+       y="196.40289"
+       id="text10"><tspan
+         sodipodi:role="line"
+         x="114.03775"
+         y="196.40289"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ac9d93;stroke-width:0.400612"
+         id="tspan10">A</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ac9d93;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="366.5202"
+       y="220.80096"
+       id="text11"><tspan
+         sodipodi:role="line"
+         x="366.5202"
+         y="220.80096"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ac9d93;stroke-width:0.400612"
+         id="tspan11">A</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="95.389114"
+       y="196.60231"
+       id="text12"><tspan
+         sodipodi:role="line"
+         x="95.389114"
+         y="196.60231"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.400612"
+         id="tspan12">B</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="171.62987"
+       y="186.67509"
+       id="text13"><tspan
+         sodipodi:role="line"
+         x="171.62987"
+         y="186.67509"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.400612"
+         id="tspan13">B</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="358.23288"
+       y="220.44789"
+       id="text14"><tspan
+         sodipodi:role="line"
+         x="358.23288"
+         y="220.44789"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.400612"
+         id="tspan14">B</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#55ddff;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="94.24794"
+       y="208.68309"
+       id="text15"><tspan
+         sodipodi:role="line"
+         x="94.24794"
+         y="208.68309"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#55ddff;stroke-width:0.400612"
+         id="tspan15">C</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#55ddff;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="43.869183"
+       y="196.65234"
+       id="text16"><tspan
+         sodipodi:role="line"
+         x="43.869183"
+         y="196.65234"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#55ddff;stroke-width:0.400612"
+         id="tspan16">C</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#00ccff;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="188.07849"
+       y="181.51845"
+       id="text17"><tspan
+         sodipodi:role="line"
+         x="188.07849"
+         y="181.51845"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#00ccff;stroke-width:0.400612"
+         id="tspan17"><tspan
+           style="font-size:4.93889px;fill:#00ccff"
+           id="tspan26">IB</tspan><tspan
+           style="font-size:3.52778px;baseline-shift:sub;fill:#00ccff"
+           id="tspan25">c</tspan></tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#55ddff;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="349.12769"
+       y="216.0551"
+       id="text18"><tspan
+         sodipodi:role="line"
+         x="349.12769"
+         y="216.0551"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#55ddff;stroke-width:0.400612"
+         id="tspan18">C</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="43.394695"
+       y="208.58258"
+       id="text19"><tspan
+         sodipodi:role="line"
+         x="43.394695"
+         y="208.58258"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.400612"
+         id="tspan19">B</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="29.580742"
+       y="220.72212"
+       id="text20"><tspan
+         sodipodi:role="line"
+         x="29.580742"
+         y="220.72212"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.400612"
+         id="tspan20">B</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="13.045858"
+       y="216.32677"
+       id="text21"><tspan
+         sodipodi:role="line"
+         x="13.045858"
+         y="216.32677"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.400612"
+         id="tspan21">B</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#55ddff;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="21.055231"
+       y="220.72211"
+       id="text22"><tspan
+         sodipodi:role="line"
+         x="21.055231"
+         y="220.72211"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#55ddff;stroke-width:0.400612"
+         id="tspan22">C</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#55ddff;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="179.01208"
+       y="182.54855"
+       id="text23"><tspan
+         sodipodi:role="line"
+         x="179.01208"
+         y="182.54855"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#55ddff;stroke-width:0.400612"
+         id="tspan23">C</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="88.379082"
+       y="185.52925"
+       id="text24"><tspan
+         sodipodi:role="line"
+         x="88.379082"
+         y="185.52925"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.400612"
+         id="tspan24"><tspan
+           style="font-size:4.93889px;fill:#ff9955"
+           id="tspan28">IB</tspan><tspan
+           style="font-size:3.52778px;baseline-shift:sub;fill:#ff9955"
+           id="tspan27">b</tspan></tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:7.05556px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="97.027405"
+       y="163.10979"
+       id="text29"><tspan
+         sodipodi:role="line"
+         x="97.027405"
+         y="163.10979"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.05556px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#00ff00;stroke-width:0.400612"
+         id="tspan29">rptr</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:7.05556px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="102.24193"
+       y="229.5128"
+       id="text30"><tspan
+         sodipodi:role="line"
+         x="102.24193"
+         y="229.5128"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.05556px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.400612"
+         id="tspan30">wptr</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:7.05556px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="116.89307"
+       y="163.58255"
+       id="text31"><tspan
+         sodipodi:role="line"
+         x="116.89307"
+         y="163.58255"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.05556px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.400612"
+         id="tspan31">wptr</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:7.05556px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="104.55666"
+       y="178.91209"
+       id="text32"><tspan
+         sodipodi:role="line"
+         x="104.55666"
+         y="178.91209"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.05556px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#00ff00;stroke-width:0.400612"
+         id="tspan32">rptr</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:7.05556px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="169.33571"
+       y="163.10979"
+       id="text33"><tspan
+         sodipodi:role="line"
+         x="169.33571"
+         y="163.10979"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.05556px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#00ff00;stroke-width:0.400612"
+         id="tspan33">rptr</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:7.05556px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="198.10239"
+       y="164.21048"
+       id="text34"><tspan
+         sodipodi:role="line"
+         x="198.10239"
+         y="164.21048"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.05556px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.400612"
+         id="tspan34">wptr</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:7.05556px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="321.48608"
+       y="209.17052"
+       id="text35"><tspan
+         sodipodi:role="line"
+         x="321.48608"
+         y="209.17052"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.05556px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff0000;stroke-width:0.400612"
+         id="tspan35">wptr</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:7.05556px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="381.35901"
+       y="239.92375"
+       id="text36"><tspan
+         sodipodi:role="line"
+         x="381.35901"
+         y="239.92375"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.05556px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#00ff00;stroke-width:0.400612"
+         id="tspan36">rptr</tspan></text>
+    <path
+       style="fill:#00ff00;stroke:#00ff00;stroke-width:1;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWide)"
+       d="m 35.624746,164.89008 -2.762741,10.3107"
+       id="path80" />
+    <path
+       style="fill:#ff0000;stroke:#ff0000;stroke-width:1;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWide)"
+       d="m -8.7253214,192.92597 h 10.67442"
+       id="path95" />
+    <path
+       style="fill:#00ff00;stroke:#00ff00;stroke-width:1;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWide)"
+       d="M 96.12886,165.61875 92.526458,179.0631"
+       id="path96"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#ff0000;stroke:#ff0000;stroke-width:1;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWide)"
+       d="m 99.252771,223.75691 -6.710739,-5.14099"
+       id="path106"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:none;stroke:#00ff00;stroke-width:1;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWide)"
+       d="m 104.23449,180.8622 v 13.29071 h 4.91861"
+       id="path108"
+       sodipodi:nodetypes="ccc" />
+    <path
+       style="fill:#ff0000;stroke:#ff0000;stroke-width:1;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWide)"
+       d="m 117.92183,166.63928 v 12.97675"
+       id="path109"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#ff0000;stroke:#ff0000;stroke-width:1;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWide)"
+       d="m 198.50323,166.63928 v 12.97675"
+       id="path110"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#00ff00;stroke:#00ff00;stroke-width:1;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWide)"
+       d="m 169.22285,165.37383 v 13.91861"
+       id="path112"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#00ff00;stroke:#00ff00;stroke-width:1;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWide)"
+       d="m 379.21241,233.73729 -9.84194,-9.84195"
+       id="path113"
+       sodipodi:nodetypes="cc" />
+    <path
+       style="fill:#ff0000;stroke:#ff0000;stroke-width:1;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0;marker-end:url(#ArrowWide)"
+       d="M 330.44909,207.68179 H 340.077"
+       id="path114"
+       sodipodi:nodetypes="cc" />
+    <g
+       id="g127"
+       transform="translate(-243.31051,23.087859)">
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:7.76111px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+         x="135.71123"
+         y="112.41485"
+         id="text116"><tspan
+           sodipodi:role="line"
+           x="135.71123"
+           y="112.41485"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.76111px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+           id="tspan116">PM4<tspan
+   style="font-size:65%;baseline-shift:sub"
+   id="tspan117">1</tspan></tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:7.76111px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+         x="135.39342"
+         y="122.44057"
+         id="text120"><tspan
+           sodipodi:role="line"
+           x="135.39342"
+           y="122.44057"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.76111px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+           id="tspan120">PM4<tspan
+   style="font-size:65%;baseline-shift:sub"
+   id="tspan123">2</tspan></tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:7.76111px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+         x="135.49934"
+         y="136.68669"
+         id="text122"><tspan
+           sodipodi:role="line"
+           x="135.49934"
+           y="136.68669"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.76111px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+           id="tspan122">PM4<tspan
+   style="font-size:65%;baseline-shift:sub"
+   id="tspan124">n</tspan></tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:7.76111px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+         x="135.65942"
+         y="128.1123"
+         id="text126"><tspan
+           sodipodi:role="line"
+           x="135.65942"
+           y="128.1123"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.76111px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+           id="tspan126">...</tspan></text>
+      <path
+         style="fill:none;stroke:#0000ff;stroke-width:2;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+         d="m 126.12628,103.17346 h -5.32796 v 38.77576 h 5.32796"
+         id="path126"
+         sodipodi:nodetypes="cccc" />
+      <path
+         style="fill:none;stroke:#0000ff;stroke-width:2;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+         d="m 143.29418,103.17346 h 5.32796 v 38.77576 h -5.32796"
+         id="path127"
+         sodipodi:nodetypes="cccc" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ff9955;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="-7.0803289"
+       y="244.43291"
+       id="text129"><tspan
+         sodipodi:role="line"
+         x="-7.0803289"
+         y="244.43291"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ff9955;stroke-width:0.400612"
+         id="tspan129"><tspan
+           style="font-size:7.05556px;fill:#ff9955"
+           id="tspan127">IB</tspan><tspan
+           style="font-size:3.52778px;baseline-shift:sub;fill:#ff9955"
+           id="tspan128">b</tspan></tspan></text>
+    <g
+       id="g130"
+       transform="translate(-0.5919954,-90.131448)">
+      <rect
+         style="fill:#ffff00;stroke:#ffff00;stroke-width:0.743523;stroke-linecap:square;stroke-dasharray:none;stroke-dashoffset:0"
+         id="rect130"
+         width="16.584557"
+         height="11.616416"
+         x="-21.840757"
+         y="358.62256"
+         ry="0" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:9.87778px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+         x="-13.612684"
+         y="367.95712"
+         id="text130"><tspan
+           sodipodi:role="line"
+           x="-13.612684"
+           y="367.95712"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:9.87778px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+           id="tspan130">CP</tspan></text>
+    </g>
+    <path
+       style="fill:none;stroke:#00d4aa;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#marker132);marker-end:url(#Diamond)"
+       d="M -13.431733,265.14038 V 238.02343 H 76.45484"
+       id="path130"
+       sodipodi:nodetypes="ccc" />
+    <path
+       style="fill:none;stroke:#00d4aa;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#marker132);marker-end:url(#Diamond)"
+       d="m -108.62977,169.16297 v 103.37737 h 82.669238"
+       id="path133"
+       sodipodi:nodetypes="ccc" />
+    <path
+       style="fill:none;stroke:#ff0000;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#ArrowTriangleStylized)"
+       d="M -107.14978,120.98926 V -2.5159921 H 176.92597 V 16.131894"
+       id="path134"
+       sodipodi:nodetypes="cccc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="25.324078"
+       y="197.77319"
+       id="text136"><tspan
+         sodipodi:role="line"
+         x="25.324078"
+         y="197.77319"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan136">Ring</tspan><tspan
+         sodipodi:role="line"
+         x="25.324078"
+         y="205.71069"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan137">Buffer</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="76.235764"
+       y="197.77319"
+       id="text139"><tspan
+         sodipodi:role="line"
+         x="76.235764"
+         y="197.77319"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan138">Ring</tspan><tspan
+         sodipodi:role="line"
+         x="76.235764"
+         y="205.71069"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan139">Buffer</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="132.68411"
+       y="197.77319"
+       id="text141"><tspan
+         sodipodi:role="line"
+         x="132.68411"
+         y="197.77319"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan140">Ring</tspan><tspan
+         sodipodi:role="line"
+         x="132.68411"
+         y="205.71069"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan141">Buffer</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="184.18781"
+       y="197.77319"
+       id="text143"><tspan
+         sodipodi:role="line"
+         x="184.18781"
+         y="197.77319"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan142">Ring</tspan><tspan
+         sodipodi:role="line"
+         x="184.18781"
+         y="205.71069"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan143">Buffer</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="235.43346"
+       y="197.77319"
+       id="text147"><tspan
+         sodipodi:role="line"
+         x="235.43346"
+         y="197.77319"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan146">Ring</tspan><tspan
+         sodipodi:role="line"
+         x="235.43346"
+         y="205.71069"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan147">Buffer</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="286.69049"
+       y="197.77319"
+       id="text149"><tspan
+         sodipodi:role="line"
+         x="286.69049"
+         y="197.77319"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan148">Ring</tspan><tspan
+         sodipodi:role="line"
+         x="286.69049"
+         y="205.71069"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan149">Buffer</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.35px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="362.16714"
+       y="197.77319"
+       id="text151"><tspan
+         sodipodi:role="line"
+         x="362.16714"
+         y="197.77319"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan150">Ring</tspan><tspan
+         sodipodi:role="line"
+         x="362.16714"
+         y="205.71069"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan151">Buffer</tspan></text>
+    <path
+       style="fill:none;stroke:#917c6f;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#marker154)"
+       d="M 134.065,175.23092 V 125.06784 H 47.638527 V 89.317459"
+       id="path158"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="fill:none;stroke:#917c6f;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#marker154)"
+       d="M 365.79312,175.67492 V 125.06784 H 47.447018 V 81.621506"
+       id="path159"
+       sodipodi:nodetypes="cccc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:7.76111px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.400612"
+       x="-68.488571"
+       y="250.37801"
+       id="text159"><tspan
+         sodipodi:role="line"
+         x="-68.488571"
+         y="250.37801"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.76111px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan159">CP is capable of</tspan><tspan
+         sodipodi:role="line"
+         x="-68.488571"
+         y="260.07941"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.76111px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan160">following the</tspan><tspan
+         sodipodi:role="line"
+         x="-68.488571"
+         y="269.78079"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:7.76111px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#1c241c;stroke-width:0.400612"
+         id="tspan161">IB address.</tspan></text>
+    <path
+       style="fill:none;stroke:#ff9955;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#marker154)"
+       d="M 38.977794,178.16115 V 132.60272 H 177.31847 V 80.437513"
+       id="path161"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="fill:none;stroke:#ff9955;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#marker154)"
+       d="M 76.233608,174.60301 V 132.60272 H 177.31847 V 80.733511"
+       id="path162"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="fill:none;stroke:#ff9955;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#marker154)"
+       d="m 185.48942,174.60301 v -42.00029 h -8.17095 V 81.325508"
+       id="path163"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="fill:none;stroke:#ff9955;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#marker154)"
+       d="M 353.76849,176.69603 V 132.60272 H 177.31847 V 82.657499"
+       id="path164"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="fill:none;stroke:#55ddff;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#marker154)"
+       d="M 178.37314,176.69603 V 138.67068 H 307.08591 V 81.769505"
+       id="path165"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="fill:none;stroke:#55ddff;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#marker154)"
+       d="M 84.837713,176.69603 V 138.67068 H 307.08591 V 81.769505"
+       id="path166"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="fill:none;stroke:#55ddff;stroke-width:1;stroke-linecap:square;stroke-linejoin:miter;stroke-dasharray:none;marker-start:url(#marker154)"
+       d="m 346.50011,180.10001 v -41.42933 h -39.4142 V 81.769505"
+       id="path167"
+       sodipodi:nodetypes="cccc" />
+  </g>
+</svg>
diff --git a/Documentation/gpu/amdgpu/userq.rst b/Documentation/gpu/amdgpu/userq.rst
index ca3ea71f7888..88f54393b220 100644
--- a/Documentation/gpu/amdgpu/userq.rst
+++ b/Documentation/gpu/amdgpu/userq.rst
@@ -1,3 +1,5 @@
+.. _amdgpu-userq:
+
 ==================
  User Mode Queues
 ==================
diff --git a/Documentation/gpu/drm-mm.rst b/Documentation/gpu/drm-mm.rst
index d55751cad67c..f22433470c76 100644
--- a/Documentation/gpu/drm-mm.rst
+++ b/Documentation/gpu/drm-mm.rst
@@ -155,7 +155,12 @@ drm_gem_object_init() will create an shmfs file of the
 requested size and store it into the struct :c:type:`struct
 drm_gem_object <drm_gem_object>` filp field. The memory is
 used as either main storage for the object when the graphics hardware
-uses system memory directly or as a backing store otherwise.
+uses system memory directly or as a backing store otherwise. Drivers
+can call drm_gem_huge_mnt_create() to create, mount and use a huge
+shmem mountpoint instead of the default one ('shm_mnt'). For builds
+with CONFIG_TRANSPARENT_HUGEPAGE enabled, further calls to
+drm_gem_object_init() will let shmem allocate huge pages when
+possible.
 
 Drivers are responsible for the actual physical pages allocation by
 calling shmem_read_mapping_page_gfp() for each page.
@@ -290,15 +295,27 @@ The open and close operations must update the GEM object reference
 count. Drivers can use the drm_gem_vm_open() and drm_gem_vm_close() helper
 functions directly as open and close handlers.
 
-The fault operation handler is responsible for mapping individual pages
-to userspace when a page fault occurs. Depending on the memory
-allocation scheme, drivers can allocate pages at fault time, or can
-decide to allocate memory for the GEM object at the time the object is
-created.
+The fault operation handler is responsible for mapping pages to
+userspace when a page fault occurs. Depending on the memory allocation
+scheme, drivers can allocate pages at fault time, or can decide to
+allocate memory for the GEM object at the time the object is created.
 
 Drivers that want to map the GEM object upfront instead of handling page
 faults can implement their own mmap file operation handler.
 
+In order to reduce page table overhead, if the internal shmem mountpoint
+"shm_mnt" is configured to use transparent huge pages (for builds with
+CONFIG_TRANSPARENT_HUGEPAGE enabled) and if the shmem backing store
+managed to allocate a huge page for a faulty address, the fault handler
+will first attempt to insert that huge page into the VMA before falling
+back to individual page insertion. mmap() user address alignment for GEM
+objects is handled by providing a custom get_unmapped_area file
+operation which forwards to the shmem backing store. For most drivers,
+which don't create a huge mountpoint by default or through a module
+parameter, transparent huge pages can be enabled by either setting the
+"transparent_hugepage_shmem" kernel parameter or the
+"/sys/kernel/mm/transparent_hugepage/shmem_enabled" sysfs knob.
+
 For platforms without MMU the GEM core provides a helper method
 drm_gem_dma_get_unmapped_area(). The mmap() routines will call this to get a
 proposed address for the mapping.
diff --git a/Documentation/gpu/todo.rst b/Documentation/gpu/todo.rst
index 9013ced318cb..22487ac1b011 100644
--- a/Documentation/gpu/todo.rst
+++ b/Documentation/gpu/todo.rst
@@ -878,6 +878,51 @@ Contact: Christian König
 
 Level: Starter
 
+DRM GPU Scheduler
+=================
+
+Provide a universal successor for drm_sched_resubmit_jobs()
+-----------------------------------------------------------
+
+drm_sched_resubmit_jobs() is deprecated. Main reason being that it leads to
+reinitializing dma_fences. See that function's docu for details. The better
+approach for valid resubmissions by amdgpu and Xe is (apparently) to figure out
+which job (and, through association: which entity) caused the hang. Then, the
+job's buffer data, together with all other jobs' buffer data currently in the
+same hardware ring, must be invalidated. This can for example be done by
+overwriting it. amdgpu currently determines which jobs are in the ring and need
+to be overwritten by keeping copies of the job. Xe obtains that information by
+directly accessing drm_sched's pending_list.
+
+Tasks:
+
+1. implement scheduler functionality through which the driver can obtain the
+   information which *broken* jobs are currently in the hardware ring.
+2. Such infrastructure would then typically be used in
+   drm_sched_backend_ops.timedout_job(). Document that.
+3. Port a driver as first user.
+4. Document the new alternative in the docu of deprecated
+   drm_sched_resubmit_jobs().
+
+Contact: Christian König <christian.koenig@amd.com>
+         Philipp Stanner <phasta@kernel.org>
+
+Level: Advanced
+
+Add locking for runqueues
+-------------------------
+
+There is an old FIXME by Sima in include/drm/gpu_scheduler.h. It details that
+struct drm_sched_rq is read at many places without any locks, not even with a
+READ_ONCE. At XDC 2025 no one could really tell why that is the case, whether
+locks are needed and whether they could be added. (But for real, that should
+probably be locked!). Check whether it's possible to add locks everywhere, and
+do so if yes.
+
+Contact: Philipp Stanner <phasta@kernel.org>
+
+Level: Intermediate
+
 Outside DRM
 ===========
 
diff --git a/Documentation/gpu/xe/xe_exec_queue.rst b/Documentation/gpu/xe/xe_exec_queue.rst
index 6076569e311c..8707806211c9 100644
--- a/Documentation/gpu/xe/xe_exec_queue.rst
+++ b/Documentation/gpu/xe/xe_exec_queue.rst
@@ -7,6 +7,20 @@ Execution Queue
 .. kernel-doc:: drivers/gpu/drm/xe/xe_exec_queue.c
    :doc: Execution Queue
 
+Multi Queue Group
+=================
+
+.. kernel-doc:: drivers/gpu/drm/xe/xe_exec_queue.c
+   :doc: Multi Queue Group
+
+.. _multi-queue-group-guc-interface:
+
+Multi Queue Group GuC interface
+===============================
+
+.. kernel-doc:: drivers/gpu/drm/xe/xe_guc_submit.c
+   :doc: Multi Queue Group GuC interface
+
 Internal API
 ============
 
diff --git a/Documentation/hwmon/asus_ec_sensors.rst b/Documentation/hwmon/asus_ec_sensors.rst
index 232885f24430..58986546c723 100644
--- a/Documentation/hwmon/asus_ec_sensors.rst
+++ b/Documentation/hwmon/asus_ec_sensors.rst
@@ -10,6 +10,7 @@ Supported boards:
  * PRIME X670E-PRO WIFI
  * PRIME Z270-A
  * Pro WS TRX50-SAGE WIFI
+ * Pro WS TRX50-SAGE WIFI A
  * Pro WS X570-ACE
  * Pro WS WRX90E-SAGE SE
  * ProArt X570-CREATOR WIFI
@@ -23,6 +24,7 @@ Supported boards:
  * ROG CROSSHAIR VIII IMPACT
  * ROG CROSSHAIR X670E HERO
  * ROG CROSSHAIR X670E GENE
+ * ROG MAXIMUS X HERO
  * ROG MAXIMUS XI HERO
  * ROG MAXIMUS XI HERO (WI-FI)
  * ROG MAXIMUS Z690 FORMULA
diff --git a/Documentation/hwmon/ds620.rst b/Documentation/hwmon/ds620.rst
index 2d686b17b547..e2d915a988a2 100644
--- a/Documentation/hwmon/ds620.rst
+++ b/Documentation/hwmon/ds620.rst
@@ -7,9 +7,9 @@ Supported chips:
 
     Prefix: 'ds620'
 
-    Datasheet: Publicly available at the Dallas Semiconductor website
+    Datasheet: Publicly available at the Analog Devices website
 
-	       http://www.dalsemi.com/
+	https://www.analog.com/media/en/technical-documentation/data-sheets/DS620.pdf
 
 Authors:
 	Roland Stigge <stigge@antcom.de>
diff --git a/Documentation/hwmon/sht3x.rst b/Documentation/hwmon/sht3x.rst
index 9585fa7c5a5d..ea1642920295 100644
--- a/Documentation/hwmon/sht3x.rst
+++ b/Documentation/hwmon/sht3x.rst
@@ -23,6 +23,14 @@ Supported chips:
         - https://sensirion.com/media/documents/1DA31AFD/61641F76/Sensirion_Temperature_Sensors_STS3x_Datasheet.pdf
         - https://sensirion.com/media/documents/292A335C/65537BAF/Sensirion_Datasheet_STS32_STS33.pdf
 
+  * Sensirion SHT85
+
+    Prefix: 'sht85'
+
+    Addresses scanned: none
+
+    Datasheet: https://sensirion.com/media/documents/4B40CEF3/640B2346/Sensirion_Humidity_Sensors_SHT85_Datasheet.pdf
+
 Author:
 
   - David Frey <david.frey@sensirion.com>
@@ -31,15 +39,15 @@ Author:
 Description
 -----------
 
-This driver implements support for the Sensirion SHT3x-DIS and STS3x-DIS
+This driver implements support for the Sensirion SHT3x-DIS, STS3x-DIS and SHT85
 series of humidity and temperature sensors. Temperature is measured in degrees
 celsius, relative humidity is expressed as a percentage. In the sysfs interface,
 all values are scaled by 1000, i.e. the value for 31.5 degrees celsius is 31500.
 
-The device communicates with the I2C protocol. Sensors can have the I2C
-addresses 0x44 or 0x45 (0x4a or 0x4b for sts3x), depending on the wiring. See
-Documentation/i2c/instantiating-devices.rst for methods to instantiate the
-device.
+The device communicates with the I2C protocol. SHT3x sensors can have the I2C
+addresses 0x44 or 0x45 (0x4a or 0x4b for sts3x), depending on the wiring. SHT85
+address is 0x44 and is fixed. See Documentation/i2c/instantiating-devices.rst for
+methods to instantiate the device.
 
 Even if sht3x sensor supports clock-stretch (blocking mode) and non-stretch
 (non-blocking mode) in single-shot mode, this driver only supports the latter.
diff --git a/Documentation/hwmon/submitting-patches.rst b/Documentation/hwmon/submitting-patches.rst
index 6482c4f137dc..7f7095951750 100644
--- a/Documentation/hwmon/submitting-patches.rst
+++ b/Documentation/hwmon/submitting-patches.rst
@@ -82,7 +82,10 @@ increase the chances of your change being accepted.
 * Avoid calculations in macros and macro-generated functions. While such macros
   may save a line or so in the source, it obfuscates the code and makes code
   review more difficult. It may also result in code which is more complicated
-  than necessary. Use inline functions or just regular functions instead.
+  than necessary. Such macros may also evaluate their arguments multiple times.
+  This leads to Time-of-Check to Time-of-Use (TOCTOU) race conditions when
+  accessing shared data without locking, for example when calculating values in
+  sysfs show functions. Use inline functions or just regular functions instead.
 
 * Limit the number of kernel log messages. In general, your driver should not
   generate an error message just because a runtime operation failed. Report
diff --git a/Documentation/i2c/busses/i2c-i801.rst b/Documentation/i2c/busses/i2c-i801.rst
index c939a5bfc8d0..bbbce90eb7d8 100644
--- a/Documentation/i2c/busses/i2c-i801.rst
+++ b/Documentation/i2c/busses/i2c-i801.rst
@@ -52,6 +52,7 @@ Supported adapters:
   * Intel Panther Lake (SOC)
   * Intel Wildcat Lake (SOC)
   * Intel Diamond Rapids (SOC)
+  * Intel Nova Lake (PCH)
 
    Datasheets: Publicly available at the Intel website
 
diff --git a/Documentation/power/runtime_pm.rst b/Documentation/power/runtime_pm.rst
index 455b9d135d85..a53ab09c37d5 100644
--- a/Documentation/power/runtime_pm.rst
+++ b/Documentation/power/runtime_pm.rst
@@ -712,10 +712,9 @@ out the following operations:
   * During system suspend pm_runtime_get_noresume() is called for every device
     right before executing the subsystem-level .prepare() callback for it and
     pm_runtime_barrier() is called for every device right before executing the
-    subsystem-level .suspend() callback for it.  In addition to that the PM core
-    calls __pm_runtime_disable() with 'false' as the second argument for every
-    device right before executing the subsystem-level .suspend_late() callback
-    for it.
+    subsystem-level .suspend() callback for it.  In addition to that, the PM
+    core disables runtime PM for every device right before executing the
+    subsystem-level .suspend_late() callback for it.
 
   * During system resume pm_runtime_enable() and pm_runtime_put() are called for
     every device right after executing the subsystem-level .resume_early()
diff --git a/Documentation/process/debugging/kgdb.rst b/Documentation/process/debugging/kgdb.rst
index b29b0aac2717..773b19aa1382 100644
--- a/Documentation/process/debugging/kgdb.rst
+++ b/Documentation/process/debugging/kgdb.rst
@@ -889,34 +889,6 @@ in the virtual console layer. On resuming kernel execution, the kernel
 debugger calls kgdboc_post_exp_handler() which in turn calls
 con_debug_leave().
 
-Any video driver that wants to be compatible with the kernel debugger
-and the atomic kms callbacks must implement the ``mode_set_base_atomic``,
-``fb_debug_enter`` and ``fb_debug_leave operations``. For the
-``fb_debug_enter`` and ``fb_debug_leave`` the option exists to use the
-generic drm fb helper functions or implement something custom for the
-hardware. The following example shows the initialization of the
-.mode_set_base_atomic operation in
-drivers/gpu/drm/i915/intel_display.c::
-
-
-    static const struct drm_crtc_helper_funcs intel_helper_funcs = {
-    [...]
-            .mode_set_base_atomic = intel_pipe_set_base_atomic,
-    [...]
-    };
-
-
-Here is an example of how the i915 driver initializes the
-fb_debug_enter and fb_debug_leave functions to use the generic drm
-helpers in ``drivers/gpu/drm/i915/intel_fb.c``::
-
-
-    static struct fb_ops intelfb_ops = {
-    [...]
-           .fb_debug_enter = drm_fb_helper_debug_enter,
-           .fb_debug_leave = drm_fb_helper_debug_leave,
-    [...]
-    };
 
 
 Credits
diff --git a/Documentation/usb/gadget-testing.rst b/Documentation/usb/gadget-testing.rst
index 5f90af1fb573..01a128d664cb 100644
--- a/Documentation/usb/gadget-testing.rst
+++ b/Documentation/usb/gadget-testing.rst
@@ -368,14 +368,15 @@ Function-specific configfs interface
 The function name to use when creating the function directory is "midi".
 The MIDI function provides these attributes in its function directory:
 
-	=============== ====================================
-	buflen		MIDI buffer length
-	id		ID string for the USB MIDI adapter
-	in_ports	number of MIDI input ports
-	index		index value for the USB MIDI adapter
-	out_ports	number of MIDI output ports
-	qlen		USB read request queue length
-	=============== ====================================
+	================ ====================================
+	buflen		 MIDI buffer length
+	id		 ID string for the USB MIDI adapter
+	in_ports	 number of MIDI input ports
+	index		 index value for the USB MIDI adapter
+	out_ports	 number of MIDI output ports
+	qlen		 USB read request queue length
+	interface_string USB AudioControl interface string
+	================ ====================================
 
 Testing the MIDI function
 -------------------------
diff --git a/Documentation/userspace-api/media/conf_nitpick.py b/Documentation/userspace-api/media/conf_nitpick.py
index 0a8e236d07ab..445a29c01d1b 100644
--- a/Documentation/userspace-api/media/conf_nitpick.py
+++ b/Documentation/userspace-api/media/conf_nitpick.py
@@ -42,8 +42,6 @@ nitpick_ignore = [
     ("c:func", "struct fd_set"),
     ("c:func", "struct pollfd"),
     ("c:func", "usb_make_path"),
-    ("c:func", "wait_finish"),
-    ("c:func", "wait_prepare"),
     ("c:func", "write"),
 
     ("c:type", "atomic_t"),
diff --git a/MAINTAINERS b/MAINTAINERS
index 5b11839cba9d..0dbf349fc1ed 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1817,6 +1817,12 @@ S:	Supported
 F:	drivers/clk/analogbits/*
 F:	include/linux/clk/analogbits*
 
+ANDES ATCSPI200 SPI DRIVER
+M:	CL Wang <cl634@andestech.com>
+S:	Supported
+F:	Documentation/devicetree/bindings/spi/andestech,ae350-spi.yaml
+F:	drivers/spi/spi-atcspi200.c
+
 ANDROID DRIVERS
 M:	Greg Kroah-Hartman <gregkh@linuxfoundation.org>
 M:	Arve Hjønnevåg <arve@android.com>
@@ -8067,7 +8073,7 @@ W:	https://rust-for-linux.com/nova-gpu-driver
 Q:	https://patchwork.freedesktop.org/project/nouveau/
 B:	https://gitlab.freedesktop.org/drm/nova/-/issues
 C:	irc://irc.oftc.net/nouveau
-T:	git https://gitlab.freedesktop.org/drm/nova.git nova-next
+T:	git https://gitlab.freedesktop.org/drm/rust/kernel.git drm-rust-next
 F:	Documentation/gpu/nova/
 F:	drivers/gpu/nova-core/
 
@@ -8079,7 +8085,7 @@ W:	https://rust-for-linux.com/nova-gpu-driver
 Q:	https://patchwork.freedesktop.org/project/nouveau/
 B:	https://gitlab.freedesktop.org/drm/nova/-/issues
 C:	irc://irc.oftc.net/nouveau
-T:	git https://gitlab.freedesktop.org/drm/nova.git nova-next
+T:	git https://gitlab.freedesktop.org/drm/rust/kernel.git drm-rust-next
 F:	Documentation/gpu/nova/
 F:	drivers/gpu/drm/nova/
 F:	include/uapi/drm/nova_drm.h
@@ -8201,6 +8207,9 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/display/sitronix,st7567.yaml
 F:	Documentation/devicetree/bindings/display/sitronix,st7571.yaml
 F:	drivers/gpu/drm/sitronix/st7571-i2c.c
+F:	drivers/gpu/drm/sitronix/st7571-spi.c
+F:	drivers/gpu/drm/sitronix/st7571.c
+F:	drivers/gpu/drm/sitronix/st7571.h
 
 DRM DRIVER FOR SITRONIX ST7701 PANELS
 M:	Jagan Teki <jagan@amarulasolutions.com>
@@ -8223,6 +8232,13 @@ T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
 F:	Documentation/devicetree/bindings/display/sitronix,st7735r.yaml
 F:	drivers/gpu/drm/sitronix/st7735r.c
 
+DRM DRIVER FOR SITRONIX ST7920 LCD DISPLAYS
+M:	Iker Pedrosa <ikerpedrosam@gmail.com>
+S:	Maintained
+T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
+F:	Documentation/devicetree/bindings/display/sitronix,st7920.yaml
+F:	drivers/gpu/drm/sitronix/st7920.c
+
 DRM DRIVER FOR SOLOMON SSD130X OLED DISPLAYS
 M:	Javier Martinez Canillas <javierm@redhat.com>
 S:	Maintained
@@ -11294,7 +11310,7 @@ HIMAX HX83112B TOUCHSCREEN SUPPORT
 M:	Job Noorman <job@noorman.info>
 L:	linux-input@vger.kernel.org
 S:	Maintained
-F:	Documentation/devicetree/bindings/input/touchscreen/himax,hx83112b.yaml
+F:	Documentation/devicetree/bindings/input/touchscreen/trivial-touch.yaml
 F:	drivers/input/touchscreen/himax_hx83112b.c
 
 HIMAX HX852X TOUCHSCREEN DRIVER
@@ -12630,7 +12646,7 @@ F:	include/drm/intel/
 F:	include/uapi/drm/i915_drm.h
 
 INTEL DRM XE DRIVER (Lunar Lake and newer)
-M:	Lucas De Marchi <lucas.demarchi@intel.com>
+M:	Matthew Brost <matthew.brost@intel.com>
 M:	Thomas Hellström <thomas.hellstrom@linux.intel.com>
 M:	Rodrigo Vivi <rodrigo.vivi@intel.com>
 L:	intel-xe@lists.freedesktop.org
@@ -12978,8 +12994,9 @@ S:	Orphan
 F:	drivers/ptp/ptp_dfl_tod.c
 
 INTEL QUADRATURE ENCODER PERIPHERAL DRIVER
-M:	Jarkko Nikula <jarkko.nikula@linux.intel.com>
+M:	Ilpo Järvinen <ilpo.jarvinen@linux.intel.com>
 L:	linux-iio@vger.kernel.org
+S:	Supported
 F:	drivers/counter/intel-qep.c
 
 INTEL SCU DRIVERS
@@ -13959,6 +13976,7 @@ S:	Maintained
 F:	Documentation/admin-guide/mm/kho.rst
 F:	Documentation/core-api/kho/*
 F:	include/linux/kexec_handover.h
+F:	include/linux/kho/
 F:	kernel/liveupdate/kexec_handover*
 F:	lib/test_kho.c
 F:	tools/testing/selftests/kho/
@@ -14637,10 +14655,12 @@ S:	Maintained
 F:	Documentation/core-api/liveupdate.rst
 F:	Documentation/mm/memfd_preservation.rst
 F:	Documentation/userspace-api/liveupdate.rst
+F:	include/linux/kho/abi/
 F:	include/linux/liveupdate.h
 F:	include/linux/liveupdate/
 F:	include/uapi/linux/liveupdate.h
 F:	kernel/liveupdate/
+F:	lib/tests/liveupdate.c
 F:	mm/memfd_luo.c
 F:	tools/testing/selftests/liveupdate/
 
@@ -16751,6 +16771,7 @@ F:	tools/testing/selftests/mm/transhuge-stress.c
 
 MEMORY MANAGEMENT - USERFAULTFD
 M:	Andrew Morton <akpm@linux-foundation.org>
+M:	Mike Rapoport <rppt@kernel.org>
 R:	Peter Xu <peterx@redhat.com>
 L:	linux-mm@kvack.org
 S:	Maintained
@@ -18028,7 +18049,6 @@ F:	drivers/net/ethernet/neterion/
 
 NETFILTER
 M:	Pablo Neira Ayuso <pablo@netfilter.org>
-M:	Jozsef Kadlecsik <kadlec@netfilter.org>
 M:	Florian Westphal <fw@strlen.de>
 R:	Phil Sutter <phil@nwl.cc>
 L:	netfilter-devel@vger.kernel.org
@@ -18946,6 +18966,15 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/sound/trivial-codec.yaml
 F:	sound/soc/codecs/tfa9879*
 
+NXP XSPI DRIVER
+M:	Han Xu <han.xu@nxp.com>
+M:	Haibo Chen <haibo.chen@nxp.com>
+L:	linux-spi@vger.kernel.org
+L:	imx@lists.linux.dev
+S:	Maintained
+F:	Documentation/devicetree/bindings/spi/nxp,imx94-xspi.yaml
+F:	drivers/spi/spi-nxp-xspi.c
+
 NXP-NCI NFC DRIVER
 S:	Orphan
 F:	Documentation/devicetree/bindings/net/nfc/nxp,nci.yaml
@@ -19557,6 +19586,7 @@ F:	include/linux/of*.h
 F:	rust/helpers/of.c
 F:	rust/kernel/of.rs
 F:	scripts/dtc/
+F:	scripts/Makefile.dtb*
 F:	tools/testing/selftests/dt/
 K:	of_overlay_notifier_
 K:	of_overlay_fdt_apply
@@ -21346,7 +21376,7 @@ F:	Documentation/devicetree/bindings/net/qcom,bam-dmux.yaml
 F:	drivers/net/wwan/qcom_bam_dmux.c
 
 QUALCOMM BLUETOOTH DRIVER
-M:	Bartosz Golaszewski <brgl@bgdev.pl>
+M:	Bartosz Golaszewski <brgl@kernel.org>
 L:	linux-arm-msm@vger.kernel.org
 S:	Maintained
 F:	drivers/bluetooth/btqca.[ch]
@@ -22446,6 +22476,14 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git
 F:	Documentation/devicetree/bindings/iommu/riscv,iommu.yaml
 F:	drivers/iommu/riscv/
 
+RISC-V ANLOGIC SoC SUPPORT
+M:	Conor Dooley <conor@kernel.org>
+T:	git https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git/
+L:	linux-riscv@lists.infradead.org
+S:	Odd Fixes
+F:	Documentation/devicetree/bindings/riscv/anlogic.yaml
+F:	arch/riscv/boot/dts/anlogic/
+
 RISC-V MICROCHIP SUPPORT
 M:	Conor Dooley <conor.dooley@microchip.com>
 M:	Daire McNamara <daire.mcnamara@microchip.com>
@@ -26464,6 +26502,7 @@ L:	linux-trace-kernel@vger.kernel.org
 S:	Maintained
 Q:	https://patchwork.kernel.org/project/linux-trace-kernel/list/
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git
+F:	Documentation/core-api/tracepoint.rst
 F:	Documentation/trace/*
 F:	fs/tracefs/
 F:	include/linux/trace*.h
@@ -27241,7 +27280,7 @@ R:	Andy Shevchenko <andriy.shevchenko@linux.intel.com>
 L:	linux-kernel@vger.kernel.org
 S:	Maintained
 F:	include/linux/uuid.h
-F:	lib/test_uuid.c
+F:	lib/tests/uuid_kunit.c
 F:	lib/uuid.c
 
 UV SYSFS DRIVER
diff --git a/Makefile b/Makefile
index e404e4767944..3ba42da94eb5 100644
--- a/Makefile
+++ b/Makefile
@@ -1506,6 +1506,12 @@ ifneq ($(wildcard $(srctree)/arch/$(SRCARCH)/boot/dts/),)
 dtstree := arch/$(SRCARCH)/boot/dts
 endif
 
+dtbindingtree := Documentation/devicetree/bindings
+
+%.yaml: dtbs_prepare
+	$(Q)$(MAKE) $(build)=$(dtbindingtree) \
+		    $(dtbindingtree)/$(patsubst %.yaml,%.example.dtb,$@) dt_binding_check_one
+
 ifneq ($(dtstree),)
 
 %.dtb: dtbs_prepare
@@ -1523,12 +1529,12 @@ dtbs: dtbs_prepare
 # dtbs_install depend on it as dtbs_install may run as root.
 dtbs_prepare: include/config/kernel.release scripts_dtc
 
-ifneq ($(filter dtbs_check, $(MAKECMDGOALS)),)
+ifneq ($(filter dtbs_check %.yaml, $(MAKECMDGOALS)),)
 export CHECK_DTBS=y
 endif
 
 ifneq ($(CHECK_DTBS),)
-dtbs_prepare: dt_binding_schemas
+scripts_dtc: dt_binding_schemas
 endif
 
 dtbs_check: dtbs
@@ -1556,14 +1562,14 @@ endif
 
 PHONY += dt_binding_check dt_binding_schemas
 dt_binding_check: dt_binding_schemas scripts_dtc
-	$(Q)$(MAKE) $(build)=Documentation/devicetree/bindings $@
+	$(Q)$(MAKE) $(build)=$(dtbindingtree) $@
 
 dt_binding_schemas:
-	$(Q)$(MAKE) $(build)=Documentation/devicetree/bindings
+	$(Q)$(MAKE) $(build)=$(dtbindingtree)
 
 PHONY += dt_compatible_check
 dt_compatible_check: dt_binding_schemas
-	$(Q)$(MAKE) $(build)=Documentation/devicetree/bindings $@
+	$(Q)$(MAKE) $(build)=$(dtbindingtree) $@
 
 # ---------------------------------------------------------------------------
 # Modules
diff --git a/Next/SHA1s b/Next/SHA1s
new file mode 100644
index 000000000000..58deee2d08e3
--- /dev/null
+++ b/Next/SHA1s
@@ -0,0 +1,407 @@
+Name		SHA1
+----		----
+origin		dd9b004b7ff3289fb7bae35130c0a5c0537266af
+ext4-fixes	d0b3b7b22dfa1f4b515fd3a295b3fd958f9e81af
+vfs-brauner-fixes	24835a96f21ebb295d24e265b5466d9e40f12cbd
+fscrypt-current	1e39da974ce621ed874c6d3aaf65ad14848c9f0d
+fsverity-current	211ddde0823f1442e4ad052a2f30f050145ccada
+btrfs-fixes	5c94889f8419a8141ea048e11c79986178606810
+vfs-fixes	e1b4c6a58304fd490124cc2b454d80edc786665c
+erofs-fixes	f2a12cc3b97f062186568a7b94ddb7aa2ef68140
+nfsd-fixes	913f7cf77bf14c13cfea70e89bcb6d0b22239562
+v9fs-fixes	2014c95afecee3e76ca4a56956a936e23283f05b
+overlayfs-fixes	e8bd877fb76bb9f35253e8f41ce0c772269934dd
+fscrypt		211ddde0823f1442e4ad052a2f30f050145ccada
+btrfs		58d1993910c4912d9c9198e76b3904a2b5304195
+ceph		375068b6c206cfb592208c00abfab228f0ea9372
+cifs		d8a4af8f3d9d3367b2c49b0d9dee529556bdd2f4
+configfs	f7f78098690d60a03b47942ac7d73ea17b42239e
+ecryptfs	fba133a3411847db49297c965218400c49571ebd
+dlm		8e40210788636619404871df07445fa4590138b4
+erofs		e68e6eb58221ce30497c238d37704d230321869d
+exfat		f5a5c9b4049047cdf4b5fff5e487974826116c57
+ext3		6f7c877cc397ba3c6d8ba44d4a604df3d4182eec
+ext4		6fb67ac896900e60f46ee4efba97b372a80370e0
+f2fs		4a210a5be279bfd5514dac3f5ef2c737cd984e84
+fsverity	211ddde0823f1442e4ad052a2f30f050145ccada
+fuse		8da059f2a497a2427150faae5adc3bb78e73b3e2
+gfs2		02e96fafcb3696e66694e64abd16f38f31ea00f5
+jfs		eb0cfcf265714b419cc3549895a00632e76732ae
+ksmbd		53ec4a79ff4b36d9711cfe030eeebc36afbc51dd
+nfs		bd3b04b46c7a9940989ff4b29376e899e93d3a4a
+nfs-anna	b623390045a81fc559decb9bfeb79319721d3dfb
+nfsd		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+ntfs3		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+orangefs	11f6bce77e27e82015a0d044e6c1eec8b139831a
+overlayfs	ad1423922781e6552f18d055a5742b1cff018cdc
+ubifs		0695aef23d674815c352293c49d944a2375ee9c9
+v9fs		3e281113f871d7f9c69ca55a4d806a72180b7e8a
+v9fs-ericvh	2014c95afecee3e76ca4a56956a936e23283f05b
+xfs		f401306d72f2cffa9837b7d194c8e528be5e63b2
+zonefs		a42938e80357a13f8b8592111e63f2e33a919863
+vfs-brauner	981be27a72d163610e8e1c342373930bae80ac99
+vfs		eb028c33451af08bb34f45c6be6967ef1c98cbd1
+mm-hotfixes	201037a59604abdb9f6d565ba3fb58820ede1de1
+fs-current	3bfe1da11c0dc2e284f69028b274aa07731fb952
+kbuild-current	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+arc-current	d7b8f8e20813f0179d8ef519541a3527e7661d3a
+arm-current	fd2dee1c6e2256f726ba33fd3083a7be0efc80d3
+arm64-fixes	98a97bf41528ef738b06eb07ec2b2eb1cfde6ce6
+arm-soc-fixes	ececfba255bf3616301419e47a5c824e04b60ab8
+davinci-current	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+drivers-memory-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+sophgo-fixes	19272b37aa4f83ca52bdf9c16d5d81bdd1354494
+sophgo-soc-fixes	0af2f6be1b4281385b618cb86ad946eded089ac8
+m68k-current	aaf4e92341b081bac1230f22619b0a298e285dab
+powerpc-fixes	0843ba458439f38efdc14aa359c14ad0127edb01
+s390-fixes	489e96651dfe59794195c6b2ddb78835edd9f2ed
+net		21a88f5d9ce0c328486073b75d082d85a1e98a8b
+bpf		ea1013c1539270e372fc99854bc6e4d94eaeff66
+ipsec		c196def07bbc6e8306d7a274433913444b0db20a
+netfilter	21a88f5d9ce0c328486073b75d082d85a1e98a8b
+ipvs		de0337d641bfa5b6d6b489e479792f1039274e84
+wireless	81d90d93d22ca4f61833cba921dce9a0bd82218f
+ath		d0309c054362a235077327b46f727bc48878a3bc
+iwlwifi		8f7aa3d3c7323f4ca2768a9e74ebbe359c4f8f88
+wpan		8ce4f287524c74a118b0af1eebd4b24a8efca57a
+rdma-fixes	85463eb6a46caf2f1e0e1a6d0731f2f3bab17780
+sound-current	f7cede182c963720edd1e5fb50ea4f1c7eafa30e
+sound-asoc-fixes	fa43ab13c59f4c047c479673792ed033ab567c65
+regmap-fixes	5eb484d5b2ac39c12539667230931f179636253c
+regulator-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+spi-fixes	9d651a6c62832ac8d1c445f5fa8c61a03ebfb649
+pci-current	94bf74830a977a027042f685c7231c5e07cc3372
+driver-core.current	e2f1081ca8f18c146e8f928486deac61eca2b517
+tty.current	5edb7338d6117bb0ce17ead7973c67390da42258
+usb.current	3b4961313d31e200c9e974bb1536cdea217f78b5
+usb-serial-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+phy		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+staging.current	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+iio-fixes	74f97a46835664c28bc34c0bba58c6b3eaaaa648
+counter-current	109ff654934a4752f8875ded672efd1fbfe4d31d
+char-misc.current	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+soundwire-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+thunderbolt-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+input-current	aed3716db7fff74919cc5775ca3a80c8bb246489
+crypto-current	48bc9da3c97c15f1ea24934bcb3b736acd30163d
+libcrypto-fixes	5a0b1882506858b12cc77f0e2439a5f3c5052761
+vfio-fixes	d323ad739666761646048fca587734f4ae64f2c8
+kselftest-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+dmaengine-fixes	747213b08a1ab6a76e3e3b3e7a209cc1d402b5d0
+backlight-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+mtd-fixes	2f6d2c8d9ac05a7a1c02333f6ad30868246880d8
+mfd-fixes	aa6e63373d91f82628b20f6fae7ffe9b97b568a5
+v4l-dvb-fixes	27afd6e066cfd80ddbe22a4a11b99174ac89cced
+reset-fixes	997c06330fd5c2e220b692f2a358986c6c8fd5a2
+mips-fixes	680ad315caaa2860df411cb378bf3614d96c7648
+at91-fixes	3a8660878839faadb4f1a6dd72c3179c1df56787
+omap-fixes	3a8660878839faadb4f1a6dd72c3179c1df56787
+kvm-fixes	0499add8efd72456514c6218c062911ccc922a99
+kvms390-fixes	5f9df945d4e862979b50e4ecaba3dc81fb06e8ed
+hwmon-fixes	6cde588e64934858bb1553119c6b915b2fec9011
+nvdimm-fixes	d0b3b7b22dfa1f4b515fd3a295b3fd958f9e81af
+cxl-fixes	214291cbaaceeb28debd773336642b1fca393ae0
+dma-mapping-fixes	608bf0c7be29bce3f0ccc80607bf32fab95398b1
+drivers-x86-fixes	4dcb6e4af15bf45aa1526834e8f5e296ad7ce867
+samsung-krzk-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+pinctrl-samsung-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+devicetree-fixes	54de247a0efa4c6176ba6840a58e2fb0b2130e2d
+dt-krzk-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+scsi-fixes	c9f36f04a8a2725172cdf2b5e32363e4addcb14c
+drm-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+drm-intel-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+mmc-fixes	d3ecb12e2e04ce53c95f933c462f2d8b150b965b
+rtc-fixes	162f24cbb0f6ec596e7e9f3e91610d79dc805229
+gnss-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+hyperv-fixes	173d6f64f9558ff022a777a72eb8669b6cdd2649
+risc-v-fixes	e2285bd4a46744768f2c8b0ee158495e038ed075
+riscv-dt-fixes	5ef13c3636400bec51c334ac368c8d2fbb26bc3a
+riscv-soc-fixes	0af2f6be1b4281385b618cb86ad946eded089ac8
+fpga-fixes	19272b37aa4f83ca52bdf9c16d5d81bdd1354494
+spdx		d358e5254674b70f34c847715ca509e46eb81e6f
+gpio-brgl-fixes	a05543d6b05ba998fdbb4b383319ae5121bb7407
+gpio-intel-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+pinctrl-intel-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+auxdisplay-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+kunit-fixes	c33b68801fbe9d5ee8a9178beb5747ec65873530
+memblock-fixes	c42af83c59b65d01c0f7a074e450bbbb43b22f0d
+renesas-fixes	379c590113ce46f605439d4887996c60ab8820cc
+perf-current	800ad1f0e27792cddf5928f590d3f2d4ab3c68ee
+efi-fixes	e5f0a698b34ed76002dc5cff3804a61c80233a7a
+battery-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+iommufd-fixes	e6a973af11135439de32ece3b9cbe3bfc043bea8
+rust-fixes	559e608c46553c107dbba19dae0854af7b219400
+w1-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+pmdomain-fixes	c98c99d5dbdf9fb0063650594edfd7d49b5f4e29
+i2c-host-fixes	880977fdc7f67923d1904ee23ca75fa1e375ea46
+sparc-fixes	6613476e225e090cc9aad49be7fa504e290dd33d
+clk-fixes	3c493b524ffdb3fae7f0d9dc6b887359ce13cd34
+thead-clk-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+pwrseq-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+thead-dt-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+ftrace-fixes	1650a1b6cb1ae6cb99bb4fce21b30ebdf9fc238e
+ring-buffer-fixes	b042fdf18e89a347177a49e795d8e5184778b5b6
+trace-fixes	39263f986da55c5b7bc328c757fe378a6a41799d
+tracefs-fixes	8b55572e51805184353ee7d587c720a51818fb82
+spacemit-fixes	3a8660878839faadb4f1a6dd72c3179c1df56787
+tip-fixes	a51826582e8bcc75a14b23b3c4dc802bf23b7d8b
+slab-fixes	53ca00a19d345197a37a1bf552e8d1e7b091666c
+drm-msm-fixes	f5d079564c44baaeedf5e25f4b943aa042ea0eb1
+uml-fixes	df447a3b4a4b961c9979b4b3ffb74317394b9b40
+fwctl-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+devsec-tsm-fixes	fba4ceaa242d2bdf4c04b77bda41d32d02d3925d
+drm-rust-fixes	b58c87b0fde0ac06d550659a7e3ddbcd9183b7b9
+tenstorrent-dt-fixes	3a8660878839faadb4f1a6dd72c3179c1df56787
+drm-misc-fixes	6b991ad8dc3abfe5720fc2e9ee96be63ae43e362
+rust		559e608c46553c107dbba19dae0854af7b219400
+rust-alloc	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+rust-io		86731a2a651e58953fc949573895f2fa6d456841
+rust-pin-init	42415d163e5df6db799c7de6262d707e402c2c7e
+rust-timekeeping	4521438fb076f8a6a52f45b0e508f6ef10ac0c49
+rust-xarray	fa616196fbea12462107774fb6a1908c95f71cf0
+mm-stable	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+mm-nonmm-stable	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+mm-unstable	edeb51ec1667baf92dc4b54363817435cff07d35
+mm-nonmm-unstable	bda39f9485fb20e66ec5339e493178e8f65d4968
+kbuild		a4df2071f1a2bb5b003eb179e1a51625d26b04d5
+clang-format	c147f663b6a5813b9860f3917cc473fb2c462d8d
+perf		cb015814f8b6eebcbb8e46e111d108892c5e6821
+compiler-attributes	98f7e32f20d28ec452afb208f9cffc08448a2652
+dma-mapping	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+asm-generic	edcc8a38b5ac1a3dbd05e113a38a25b937ebefe5
+alpha		d58041d2c63e09a1c9083e0e9f4151e487c4e16a
+arm		dd9143371a8619f496e29160390fcafcee1371d1
+arm64		edde060637b92607f3522252c03d64ad06369933
+arm-perf	e6a27290d8001538fec94e91ff8c7f956ee7e3e5
+arm-soc		e5b2299997d3760840111ff2e6c5df9f2224f253
+amlogic		3b2ecf0c3cc319358d7ff3f2f0f262d47b497b5d
+asahi-soc	074986379c03bca7a1de45fa81836c596c24aa1d
+at91		fde90d654751425c7b104b8fda930ed6c34cfa85
+bmc		459a5aa171c0f13fcd78faa9594dc4aa5a95770b
+broadcom	37c3a91e9730e274fe2eca9703033ae0f154cb62
+cix		4889f68b824df835bab956bab1519b132df32536
+davinci		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+drivers-memory	9805f2cfc883018f7bf84c84e3af3786c37dac7b
+fsl		c8dff80a3108a2cb85ac7ac5b1a75c247ac55fd2
+imx-mxs		89d95dbad2a3857f06cb6002566bbdcbe53bbfef
+mediatek	67ed5843a67b7ba63d79f2ba3fd21bee151d3138
+mvebu		f0e6bc0c3ef4b4afb299bd6912586cafd5d864e9
+omap		2c85858e7c6839011e423109df556ef894a71cff
+qcom		689a433e9d6bee62282a9f0e9c4e97fccb04b873
+renesas		fec0b9c25cfb056aceb90d630174f17c8b98b338
+reset		a0d57f7504b6027bc73b96711bcafa39565a9bac
+rockchip	4a4070c80fe694c684fbb67dae3294e4c9f14f55
+samsung-krzk	cb4cfbcf27641ea87c18f5a72b93fc3748f03bc3
+scmi		98675bc92a9ea56b21a362faa8b866fb8403feb7
+sophgo		c7f013d4c8c07a87d19cec90fc4d4633ad713fde
+sophgo-soc	c8754c7deab4cbfa947fa2d656cbaf83771828ef
+spacemit	5a97a38c22799a802f33001dcb022d2942fe4a41
+stm32		a6961e2ced5190c68acd78a7e372de07f565fcd6
+sunxi		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+tee		e8fd8a160bbb34925abb5e6918a7fbb0bc4b820a
+tegra		e016a50e00e68c67c0ecaa8b2069f2a694a1cf12
+tenstorrent-dt	a71e6e8eea8ae2d624f097911f43357bba06d2a5
+thead-dt	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+ti		109d58039d30b0f0cb0f5f3519c3c6a2331b6ea0
+xilinx		43da9849395e1924ad0928a7323720a269310b3e
+clk		6f172175b6f3fe35b5d519fc314f7a0b603a9af9
+clk-imx		fdc1dc7dd53b95805d3943ed36785c1ec812915e
+clk-renesas	d8921e42a1983024227c0e64253ea96b25a1ae9a
+thead-clk	30441a56b1d1e10582721b6a3b2ab3ad4aaff8ef
+csky		817d21578d51e801df58ab012654486a71073074
+loongarch	be77cf43d2fd6eca150594e997e40ca7df90f251
+m68k		aaf4e92341b081bac1230f22619b0a298e285dab
+m68knommu	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+microblaze	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+mips		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+openrisc	8c30b0018f9d93391573e091960d257fd9de120a
+parisc-hd	7d0a66e4bb9081d75c82ec4957c50034cb0ea449
+powerpc		9b36c7fc5aa5f2c6e6eeb9f312fdfe61b4291c9f
+risc-v		7e40c6791cc81e98a191d976a6f6366c77c33292
+riscv-dt	a4aad736716d345d98c0eecbb0061fc9b6e578a1
+riscv-soc	719362b8a9724f96d33fecd95b3223d3a935a186
+s390		ca4a4090a9bb3f4049a753ddf28614ca1292be7f
+sh		8f5ae30d69d7543eee0d70083daf4de8fe15d585
+sparc		fe0126702a40b2f3d315bc943ef10dc2f707e29d
+uml		a3209bb94b36351f11e0d9e72ac44e5dd777a069
+xtensa		0d4b3ca115b5436d7e68453527566db3e87ab7f0
+fs-next		e9fad83f1229ae84c676b88e5e5937d1765377fa
+printk		34793812e59224a3ba0dbe9383a5b6aea7b2fd5a
+pci		43dfc13ca972988e620a6edb72956981b75ab6b0
+pstore		b692553573683b9e4ebab40c2782a3163f984cfd
+hid		1c772200c9dcb23a304f84a9334fe2e0d9529ab0
+i2c		d202341d9b0c5b5965787061ba0d10daafb9d6c5
+i2c-host	6a28174326289834c6767bdeb1ba348aa9831e91
+i3c		136209e6bd981e60db6c0e78f2919ff2f92312d4
+dmi		4d1b28a8119c615f1e932520f9ee1f80bdda5204
+hwmon-staging	1d9d442ab179b11c629ae8f87533c9ae381cc15f
+jc_docs		464257baf99200d1be1c053f15aa617056361e81
+v4l-dvb		b70886ff5833cf499e77af77d2324ce8f68b60ce
+v4l-dvb-next	adc218676eef25575469234709c2d87185ca223a
+pm		5c062ebfa0762c42ead017e4a748aba733be8214
+cpufreq-arm	f98c414030d28a1294323e7eac4c07e158e43a6b
+cpupower	ff72619e11348ab189e232c59515dd5c33780d7c
+devfreq		d9600d57668c49308f705a660c5ad17fa3a53f73
+pmdomain	1f67707fafa598e2338dba08e3de0db3e468afd1
+opp		33ffb0aa8ce8b18aaa65e0f9346d52b4e314ad7d
+thermal		c411d8bf06992dade7abb88690dc2d467a868cc4
+rdma		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+net-next	8f7aa3d3c7323f4ca2768a9e74ebbe359c4f8f88
+bpf-next	ec439c38013550420aecc15988ae6acb670838c1
+ipsec-next	8f7aa3d3c7323f4ca2768a9e74ebbe359c4f8f88
+mlx5-next	5422318e27d7a4662701f518e2e51b9f73a331b1
+netfilter-next	8f7aa3d3c7323f4ca2768a9e74ebbe359c4f8f88
+ipvs-next	c9dfb92de0738eb7fe6a591ad1642333793e8b6e
+bluetooth	98246938a0e66e4100e95b6b7881843a9a4e4882
+wireless-next	8f7aa3d3c7323f4ca2768a9e74ebbe359c4f8f88
+ath-next	631ee338f04db713ba611883f28e94157ebb68e5
+iwlwifi-next	85d002b2ac56d74f09de7a2ef5f448ec94ceebd6
+wpan-next	1dd9291eb90378f8096669d9cb33761fd84599e0
+wpan-staging	1dd9291eb90378f8096669d9cb33761fd84599e0
+mtd		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+nand		a75a1dec037ff3de863375fa3a74569619667184
+spi-nor		ed26bd40df11ee438d623adc9c6cc2a7bf9c5dd3
+crypto		48bc9da3c97c15f1ea24934bcb3b736acd30163d
+libcrypto	325c29e7d11caaf3b4f04f2c8f7d6bc4861cce5a
+drm		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+drm-exynos	3a8660878839faadb4f1a6dd72c3179c1df56787
+drm-misc	332070795bd96193756cb4446eddc3ec9ff6a0e8
+amdgpu		0a54edc19158b3402b6c39cc682789be05e3d47d
+drm-intel	0c8afa67aaa2d6cf523361f493d9b8224448ebea
+drm-msm		7bc29d5fb6faff2f547323c9ee8d3a0790cd2530
+drm-msm-lumag	e2f085ab8636fae2ebe0adf42071e7558234cd7b
+drm-rust	97cf6bc0abd381fd84e5d8e978322a62a58fb00e
+drm-nova	93296e9d9528f0d87f2cf3fee494599060a0f14a
+drm-xe		c7b83a916d62c4d4447d7edf0bc5e06dc2afbdf8
+etnaviv		6bde14ba5f7ef59e103ac317df6cc5ac4291ff4a
+fbdev		7d0a66e4bb9081d75c82ec4957c50034cb0ea449
+regmap		82845bc8d13c63a82ab9b8d6241d305d0ef39e25
+sound		03f705b9ca58b91c6dffe64875ea3d9a38cad9b5
+ieee1394	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+sound-asoc	fa41400e9b5c6388742db7127cc61ade35ec12d6
+modules		1ddac5cd7f278345b2e8298c930e4bffe0911a45
+input		d79f302f2f9829d04b0ab6181c38716c5a68804d
+block		e68d68450f764da134f004ca61cad3e429ca7305
+device-mapper	13ea55ea20176736516b20b9ea2d8cf97dbe74f5
+libata		f474c70065e14bac928716100eebfcfb15e1a725
+pcmcia		4a81f78caa53e0633cf311ca1526377d9bff7479
+mmc		1c6c8cf3002229d120ae655c872b4b2acc26a469
+mfd		44c603f35cad3f3b0f58fece99502d81620da9b2
+backlight	d3cc7cd7bc46af587747399e956cf4508221476f
+battery		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+regulator	2bb202416d33347102b12bbd1db4837fb6685617
+security	1c0860d4415d52f3ad1c8e0a15c1272869278a06
+apparmor	8f5ae30d69d7543eee0d70083daf4de8fe15d585
+integrity	738c9738e690f5cea24a3ad6fd2d9a323cf614f6
+selinux		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+smack		29c701f90b9341f1f9c1854a9c22b71c2318457d
+tomoyo		88fe48f74ec2e76a13da7d199fa5eb54de16a875
+tpmdd		416f99c3b16f582a3fc6d64a1f77f39d94b76de5
+watchdog	5bcc5786a0cfa9249ccbe539833040a6285d0de3
+iommu		7a089c5d35aa307147e78c5cbeeb1352b92790b1
+audit		15b0c43aa621fb77b32c46eb642eaf25557e9fdb
+devicetree	5530836fb91487d915ece36b1fa8bdc3d0f4547f
+dt-krzk		b7fdd9926b22975562b1d2f76feafbdea5e5b9ad
+mailbox		1ec6e0e8c0dd6599472af944b8d703d526ff1031
+spi		81e4a2ef2782eb1c9902843ba7676e78cee2cb6f
+tip		eff801f5c2b85f1969d5436b6b86c6d8078129aa
+clockevents	d1780dce9575072303b9c574614b72b5c8c5c44c
+edac		814b332921b6c9b4b4c6c98a024f2dd9f33a8178
+ftrace		5a63f3faf6569e3aac81595b6318ac76674a282f
+rcu		9a08942f17017b708991c5089843d4a1bfac4420
+paulmck		7f8fcc6f09fb732745b3252f481def76b18fb99c
+kvm		0499add8efd72456514c6218c062911ccc922a99
+kvm-arm		3eef0c83c3f3e58933e98e678ddf4e95457d4d14
+kvms390		2bd1337a1295e012e60008ee21a64375e5234e12
+kvm-ppc		c17b750b3ad9f45f2b6f7e6f7f4679844244f0b9
+kvm-riscv	3239c52fd21257c80579875e74c9956c2f9cd1f9
+kvm-x86		58e10b63777d0aebee2cf4e6c67e1a83e7edbe0f
+xen-tip		e5aff444e3a7bdeef5ea796a2099fc3c60a070fa
+percpu		87d6aab2389e5ce0197d8257d5f8ee965a67c4cd
+workqueues	51cd2d2decf365a248ddc304b7aa6f0cadc748c3
+sched-ext	b0101ccb5b4641885f30fecc352ef891ed06e083
+drivers-x86	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+chrome-platform	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+chrome-platform-firmware	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+hsi		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+leds-lj		072cd5f458d76b9e15d89ebdaea8b5cb1312eeef
+ipmi		35bcedc1a7938da808f54510d8bc7d90cebb6278
+driver-core	e4b3118b61b6d93d4289069b9cccbffe8e714aa0
+usb		dd1fbe324a548e8057d5f3c72ce1a64a80f1753e
+thunderbolt	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+usb-serial	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+tty		284da5de616aaebf9c2c62e5fc7cb464a064eff7
+char-misc	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+accel		f03eee5fc922158654405318a02db9982c0ddf07
+coresight	7009646d937f9a1147b401362260939bba52082f
+fastrpc		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+fpga		c9243d6df0d617535e842aa4c3d1d7dfdb91d07b
+icc		60b52af082b1462e878d10a0298fad40a26a8b35
+iio		f9e05791642810a0cf6237d39fafd6fec5e0b4bb
+phy-next	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+soundwire	59946373755d71dbd7614ba235e0093159f80b69
+extcon		b4338338aa3d44bf58973b9e44e0856ba597838f
+gnss		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+vfio		d721f52e31553a848e0e9947ca15a49c5674aef3
+w1		761fcf46a1bd797bd32d23f3ea0141ffd437668a
+spmi		0ff41df1cb268fc69e703a08a57ee14ae967d0ca
+staging		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+counter-next	b462fcd08dd589d9cf9eb7f9d8fc7777b5c5521d
+mux		59b723cd2adbac2a34fc8e12c74ae26ae45bf230
+dmaengine	de4761fb57f6a71eeb5a4c1167ae3606b08d8f59
+cgroup		e02fe7ba29a2fa599885c72cb977aaaaccbfe80d
+scsi		98bcf5e5ddfd74a445ff49779c0bd8081d529d27
+scsi-mkp	8334f93075dce0a4536c096a7d471ef90506a7a4
+vhost		1bac705c2ce781c988462cfc109ac04cc2bc41f4
+rpmsg		8d201aac4d4c5fcfb90fd472a92c02f74dd7205b
+gpio-brgl	47d8cb678081d12704d52ed42b625e96f38470e0
+gpio-intel	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+pinctrl		666065caa31aeb812978740bae21871067bb14b6
+pinctrl-intel	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+pinctrl-renesas	3b0cf6ab35909d7ac8d561e18159c62d42d914c2
+pinctrl-samsung	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+pwm		9580ecfbc57d2fe4145b990de2b56362ea978f34
+ktest		d3042cbe84a060b4df764eb6c5300bbe20d125ca
+kselftest	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+kunit		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+kunit-next	2115b53400fb82ab54fc4c95a4b40e8ce0d0a7df
+livepatching	51ab33fc0a8bef9454849371ef897a1241911b37
+rtc		16bd954c93360145bc77cc601e350913fc28182d
+nvdimm		30065e73d7c018cf2e1bec68e2d6ffafc17b3c25
+at24		7a29af24b288eace769ccd9eb8044742dfbd5944
+ntb		68113d26067408736fc3be948c2fe8db935aff1c
+seccomp		7d0a66e4bb9081d75c82ec4957c50034cb0ea449
+slimbus		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+nvmem		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+xarray		863810caeb3cf04ee479f3944a678574001af7a9
+hyperv		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+auxdisplay	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+kgdb		fdbdd0ccb30af18d3b29e714ac8d5ab6163279e0
+hmm		19272b37aa4f83ca52bdf9c16d5d81bdd1354494
+cfi		3a8660878839faadb4f1a6dd72c3179c1df56787
+mhi		ac35e04f8000aaaf98635792464647e7a6f3422e
+memblock	83c8f7b5e194eaf3fb268c513e23e23e892de8ed
+cxl		ea5514e300568cbe8f19431c3e424d4791db8291
+zstd		65d1f5507ed2c78c64fce40e44e5574a9419eb09
+efi		c5a8f13f1e476c90f4bc184a58751d3e7ff88f2b
+unicode		6b56a63d286f6f57066c4b5648d8fbec9510beae
+slab		58d6fcb6ee9432e5584e8845be23a1e805c65711
+random		3f9f0252130e7dd60d41be0802bf58f6471c691d
+landlock	1a3cedbdc156e100eb1a5208a8562a3265c35d87
+sysctl		564195c1a33c8fc631cd3d306e350b0e3d3e9555
+execve		7d0a66e4bb9081d75c82ec4957c50034cb0ea449
+bitmap		abb934acc479f070bc8e78ceda5b191e17266126
+hte		9e4259716f60c96c069a38e826884ad783dc4eb4
+kspp		7454048db27d685a155aaf4ea03bb9ad0d086bb9
+nolibc		5203a78a7c60eb35557acff441d7758ce9eea476
+iommufd		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+turbostat	b65a8280c76670e70b85b39ab63df3114ac45826
+pwrseq		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+capabilities-next	9891d2f79a9fe9b77ad290f950eb8fa3e375330e
+ipe		7d0a66e4bb9081d75c82ec4957c50034cb0ea449
+kcsan		dcb6fa37fd7bc9c3d2b066329b0d27dedf8becaa
+crc		dcb6fa37fd7bc9c3d2b066329b0d27dedf8becaa
+fwctl		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+devsec-tsm	7dfbe9a6751973c17138ddc0d33deff5f5f35b94
+hisilicon	6c177775dcc5e70a64ddf4ee842c66af498f2c7c
+kthread		d8b4bf4ea04dd96fe43f6010c614149aba4c9b91
diff --git a/Next/Trees b/Next/Trees
new file mode 100644
index 000000000000..fa207ed5364a
--- /dev/null
+++ b/Next/Trees
@@ -0,0 +1,407 @@
+Trees included into this release:
+
+Name		Url
+----		---
+origin		https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git#master
+ext4-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git#fixes
+vfs-brauner-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git#vfs.fixes
+fscrypt-current	https://git.kernel.org/pub/scm/fs/fscrypt/linux.git#for-current
+fsverity-current	https://git.kernel.org/pub/scm/fs/fsverity/linux.git#for-current
+btrfs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git#next-fixes
+vfs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git#fixes
+erofs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git#fixes
+nfsd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux#nfsd-fixes
+v9fs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git#fixes/next
+overlayfs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git#ovl-fixes
+fscrypt		https://git.kernel.org/pub/scm/fs/fscrypt/linux.git#for-next
+btrfs		https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git#for-next
+ceph		https://github.com/ceph/ceph-client.git#master
+cifs		git://git.samba.org/sfrench/cifs-2.6.git#for-next
+configfs	https://git.kernel.org/pub/scm/linux/kernel/git/a.hindborg/linux.git#configfs-next
+ecryptfs	https://git.kernel.org/pub/scm/linux/kernel/git/tyhicks/ecryptfs.git#next
+dlm		https://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm.git#next
+erofs		https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git#dev
+exfat		https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/exfat.git#dev
+ext3		https://git.kernel.org/pub/scm/linux/kernel/git/jack/linux-fs.git#for_next
+ext4		https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git#dev
+f2fs		https://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git#dev
+fsverity	https://git.kernel.org/pub/scm/fs/fsverity/linux.git#for-next
+fuse		https://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git#for-next
+gfs2		https://git.kernel.org/pub/scm/linux/kernel/git/gfs2/linux-gfs2.git#for-next
+jfs		https://github.com/kleikamp/linux-shaggy.git#jfs-next
+ksmbd		https://github.com/smfrench/smb3-kernel.git#ksmbd-for-next
+nfs		git://git.linux-nfs.org/projects/trondmy/nfs-2.6.git#linux-next
+nfs-anna	git://git.linux-nfs.org/projects/anna/linux-nfs.git#linux-next
+nfsd		https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux#nfsd-next
+ntfs3		https://github.com/Paragon-Software-Group/linux-ntfs3.git#master
+orangefs	https://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux.git#for-next
+overlayfs	https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git#overlayfs-next
+ubifs		https://git.kernel.org/pub/scm/linux/kernel/git/rw/ubifs.git#next
+v9fs		https://github.com/martinetd/linux#9p-next
+v9fs-ericvh	https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git#ericvh/for-next
+xfs		https://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git#for-next
+zonefs		https://git.kernel.org/pub/scm/linux/kernel/git/dlemoal/zonefs.git#for-next
+vfs-brauner	https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git#vfs.all
+vfs		https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git#for-next
+mm-hotfixes	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-hotfixes-unstable
+kbuild-current	https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git#kbuild-fixes
+arc-current	https://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git#for-curr
+arm-current	https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git#fixes
+arm64-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux#for-next/fixes
+arm-soc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git#arm/fixes
+davinci-current	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#davinci/for-current
+drivers-memory-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git#fixes
+sophgo-fixes	https://github.com/sophgo/linux.git#fixes
+sophgo-soc-fixes	https://github.com/sophgo/linux.git#soc-fixes
+m68k-current	https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git#for-linus
+powerpc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#fixes
+s390-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git#fixes
+net		https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git#main
+bpf		https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/#master
+ipsec		https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec.git#master
+netfilter	https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf.git#main
+ipvs		https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs.git#main
+wireless	https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git#for-next
+ath		https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git#for-current
+iwlwifi		https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git#fixes
+wpan		https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan.git#master
+rdma-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#for-rc
+sound-current	https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git#for-linus
+sound-asoc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git#for-linus
+regmap-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git#for-linus
+regulator-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git#for-linus
+spi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git#for-linus
+pci-current	https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git#for-linus
+driver-core.current	https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git#driver-core-linus
+tty.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git#tty-linus
+usb.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git#usb-linus
+usb-serial-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git#usb-linus
+phy		https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git#fixes
+staging.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git#staging-linus
+iio-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git#fixes-togreg
+counter-current	https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git#counter-current
+char-misc.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git#char-misc-linus
+soundwire-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git#fixes
+thunderbolt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git#fixes
+input-current	https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git#for-linus
+crypto-current	https://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git#master
+libcrypto-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git#libcrypto-fixes
+vfio-fixes	https://github.com/awilliam/linux-vfio.git#for-linus
+kselftest-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#fixes
+dmaengine-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git#fixes
+backlight-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git#for-backlight-fixes
+mtd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#mtd/fixes
+mfd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git#for-mfd-fixes
+v4l-dvb-fixes	git://linuxtv.org/media-ci/media-pending.git#fixes
+reset-fixes	https://git.pengutronix.de/git/pza/linux#reset/fixes
+mips-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git#mips-fixes
+at91-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git#at91-fixes
+omap-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git#fixes
+kvm-fixes	git://git.kernel.org/pub/scm/virt/kvm/kvm.git#master
+kvms390-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git#master
+hwmon-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#hwmon
+nvdimm-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git#libnvdimm-fixes
+cxl-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git#fixes
+dma-mapping-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git#dma-mapping-fixes
+drivers-x86-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git#fixes
+samsung-krzk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git#fixes
+pinctrl-samsung-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git#fixes
+devicetree-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git#dt/linus
+dt-krzk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git#fixes
+scsi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git#fixes
+drm-fixes	https://gitlab.freedesktop.org/drm/kernel.git#drm-fixes
+drm-intel-fixes	https://gitlab.freedesktop.org/drm/i915/kernel.git#for-linux-next-fixes
+mmc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git#fixes
+rtc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git#rtc-fixes
+gnss-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git#gnss-linus
+hyperv-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git#hyperv-fixes
+risc-v-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git#fixes
+riscv-dt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-dt-fixes
+riscv-soc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-soc-fixes
+fpga-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git#fixes
+spdx		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/spdx.git#spdx-linus
+gpio-brgl-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#gpio/for-current
+gpio-intel-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git#fixes
+pinctrl-intel-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git#fixes
+auxdisplay-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git#fixes
+kunit-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#kunit-fixes
+memblock-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git#fixes
+renesas-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git#fixes
+perf-current	https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools.git#perf-tools
+efi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git#urgent
+battery-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git#fixes
+iommufd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git#for-rc
+rust-fixes	https://github.com/Rust-for-Linux/linux.git#rust-fixes
+w1-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git#fixes
+pmdomain-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git#fixes
+i2c-host-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git#i2c/i2c-host-fixes
+sparc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git#for-linus
+clk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git#clk-fixes
+thead-clk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-clk-fixes
+pwrseq-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#pwrseq/for-current
+thead-dt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-dt-fixes
+ftrace-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#ftrace/fixes
+ring-buffer-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#ring-buffer/fixes
+trace-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#trace/fixes
+tracefs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#tracefs/fixes
+spacemit-fixes	https://github.com/spacemit-com/linux#fixes
+tip-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git#tip/urgent
+slab-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git#slab/for-next-fixes
+drm-msm-fixes	https://gitlab.freedesktop.org/drm/msm.git#msm-fixes
+uml-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git#fixes
+fwctl-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git#for-rc
+devsec-tsm-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git#fixes
+drm-rust-fixes	https://gitlab.freedesktop.org/drm/rust/kernel.git#drm-rust-fixes
+tenstorrent-dt-fixes	https://github.com/tenstorrent/linux.git#tenstorrent-dt-fixes
+drm-misc-fixes	https://gitlab.freedesktop.org/drm/misc/kernel.git#for-linux-next-fixes
+rust		https://github.com/Rust-for-Linux/linux.git#rust-next
+rust-alloc	https://github.com/Rust-for-Linux/linux.git#alloc-next
+rust-io		https://github.com/Rust-for-Linux/linux.git#io-next
+rust-pin-init	https://github.com/Rust-for-Linux/linux.git#pin-init-next
+rust-timekeeping	https://github.com/Rust-for-Linux/linux.git#timekeeping-next
+rust-xarray	https://github.com/Rust-for-Linux/linux.git#xarray-next
+mm-stable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-stable
+mm-nonmm-stable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-nonmm-stable
+mm-unstable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-unstable
+mm-nonmm-unstable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-nonmm-unstable
+kbuild		https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git#kbuild-next
+clang-format	https://github.com/ojeda/linux.git#clang-format
+perf		https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git#perf-tools-next
+compiler-attributes	https://github.com/ojeda/linux.git#compiler-attributes
+dma-mapping	https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git#dma-mapping-for-next
+asm-generic	https://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic#master
+alpha		https://git.kernel.org/pub/scm/linux/kernel/git/mattst88/alpha.git#alpha-next
+arm		https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git#for-next
+arm64		https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux#for-next/core
+arm-perf	https://git.kernel.org/pub/scm/linux/kernel/git/will/linux.git#for-next/perf
+arm-soc		https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git#for-next
+amlogic		https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux.git#for-next
+asahi-soc	https://github.com/AsahiLinux/linux.git#asahi-soc/for-next
+at91		https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git#at91-next
+bmc		https://git.kernel.org/pub/scm/linux/kernel/git/bmc/linux.git#for-next
+broadcom	https://github.com/Broadcom/stblinux.git#next
+cix		https://git.kernel.org/pub/scm/linux/kernel/git/peter.chen/cix.git#for-next
+davinci		https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#davinci/for-next
+drivers-memory	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git#for-next
+fsl		https://git.kernel.org/pub/scm/linux/kernel/git/chleroy/linux.git#soc_fsl
+imx-mxs		https://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux.git#for-next
+mediatek	https://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux.git#for-next
+mvebu		https://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu.git#for-next
+omap		https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git#for-next
+qcom		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git#for-next
+renesas		https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git#next
+reset		https://git.pengutronix.de/git/pza/linux#reset/next
+rockchip	https://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip.git#for-next
+samsung-krzk	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git#for-next
+scmi		https://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux.git#for-linux-next
+sophgo		https://github.com/sophgo/linux.git#for-next
+sophgo-soc	https://github.com/sophgo/linux.git#soc-for-next
+spacemit	https://github.com/spacemit-com/linux#for-next
+stm32		https://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32.git#stm32-next
+sunxi		https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux.git#sunxi/for-next
+tee		https://git.kernel.org/pub/scm/linux/kernel/git/jenswi/linux-tee.git#next
+tegra		https://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git#for-next
+tenstorrent-dt	https://github.com/tenstorrent/linux.git#tenstorrent-dt-for-next
+thead-dt	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-dt-for-next
+ti		https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux.git#ti-next
+xilinx		https://github.com/Xilinx/linux-xlnx.git#for-next
+clk		https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git#clk-next
+clk-imx		https://git.kernel.org/pub/scm/linux/kernel/git/abelvesa/linux.git#for-next
+clk-renesas	https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git#renesas-clk
+thead-clk	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-clk-for-next
+csky		https://github.com/c-sky/csky-linux.git#linux-next
+loongarch	https://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson.git#loongarch-next
+m68k		https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git#for-next
+m68knommu	https://git.kernel.org/pub/scm/linux/kernel/git/gerg/m68knommu.git#for-next
+microblaze	git://git.monstr.eu/linux-2.6-microblaze.git#next
+mips		https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git#mips-next
+openrisc	https://github.com/openrisc/linux.git#for-next
+parisc-hd	https://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux.git#for-next
+powerpc		https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#next
+risc-v		https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git#for-next
+riscv-dt	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-dt-for-next
+riscv-soc	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-soc-for-next
+s390		https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git#for-next
+sh		https://git.kernel.org/pub/scm/linux/kernel/git/glaubitz/sh-linux.git#for-next
+sparc		https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git#for-next
+uml		https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git#next
+xtensa		https://github.com/jcmvbkbc/linux-xtensa.git#xtensa-for-next
+printk		https://git.kernel.org/pub/scm/linux/kernel/git/printk/linux.git#for-next
+pci		https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git#next
+pstore		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/pstore
+hid		https://git.kernel.org/pub/scm/linux/kernel/git/hid/hid.git#for-next
+i2c		https://git.kernel.org/pub/scm/linux/kernel/git/wsa/linux.git#i2c/for-next
+i2c-host	https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git#i2c/i2c-host
+i3c		https://git.kernel.org/pub/scm/linux/kernel/git/i3c/linux.git#i3c/next
+dmi		https://git.kernel.org/pub/scm/linux/kernel/git/jdelvare/staging.git#dmi-for-next
+hwmon-staging	https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#hwmon-next
+jc_docs		git://git.lwn.net/linux.git#docs-next
+v4l-dvb		git://linuxtv.org/media-ci/media-pending.git#next
+v4l-dvb-next	git://linuxtv.org/mchehab/media-next.git#master
+pm		https://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git#linux-next
+cpufreq-arm	https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git#cpufreq/arm/linux-next
+cpupower	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux.git#cpupower
+devfreq		https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/linux.git#devfreq-next
+pmdomain	https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git#next
+opp		https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git#opp/linux-next
+thermal		https://git.kernel.org/pub/scm/linux/kernel/git/thermal/linux.git#thermal/linux-next
+rdma		https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#for-next
+net-next	https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git#main
+bpf-next	https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git#for-next
+ipsec-next	https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec-next.git#master
+mlx5-next	https://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux.git#mlx5-next
+netfilter-next	https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf-next.git#main
+ipvs-next	https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs-next.git#main
+bluetooth	https://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git#master
+wireless-next	https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git#for-next
+ath-next	https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git#for-next
+iwlwifi-next	https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git#next
+wpan-next	https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git#master
+wpan-staging	https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git#staging
+mtd		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#mtd/next
+nand		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#nand/next
+spi-nor		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#spi-nor/next
+crypto		https://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git#master
+libcrypto	https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git#libcrypto-next
+drm		https://gitlab.freedesktop.org/drm/kernel.git#drm-next
+drm-exynos	https://git.kernel.org/pub/scm/linux/kernel/git/daeinki/drm-exynos.git#for-linux-next
+drm-misc	https://gitlab.freedesktop.org/drm/misc/kernel.git#for-linux-next
+amdgpu		https://gitlab.freedesktop.org/agd5f/linux.git#drm-next
+drm-intel	https://gitlab.freedesktop.org/drm/i915/kernel.git#for-linux-next
+drm-msm		https://gitlab.freedesktop.org/drm/msm.git#msm-next
+drm-msm-lumag	https://gitlab.freedesktop.org/lumag/msm.git#msm-next-lumag
+drm-rust	https://gitlab.freedesktop.org/drm/rust/kernel.git#drm-rust-next
+drm-nova	https://gitlab.freedesktop.org/drm/nova.git#nova-next
+drm-xe		https://gitlab.freedesktop.org/drm/xe/kernel.git#drm-xe-next
+etnaviv		https://git.pengutronix.de/git/lst/linux#etnaviv/next
+fbdev		https://git.kernel.org/pub/scm/linux/kernel/git/deller/linux-fbdev.git#for-next
+regmap		https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git#for-next
+sound		https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git#for-next
+ieee1394	https://git.kernel.org/pub/scm/linux/kernel/git/ieee1394/linux1394.git#for-next
+sound-asoc	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git#for-next
+modules		https://git.kernel.org/pub/scm/linux/kernel/git/modules/linux.git#modules-next
+input		https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git#next
+block		https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git#for-next
+device-mapper	https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git#for-next
+libata		https://git.kernel.org/pub/scm/linux/kernel/git/libata/linux#for-next
+pcmcia		https://git.kernel.org/pub/scm/linux/kernel/git/brodo/linux.git#pcmcia-next
+mmc		https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git#next
+mfd		https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git#for-mfd-next
+backlight	https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git#for-backlight-next
+battery		https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git#for-next
+regulator	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git#for-next
+security	https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git#next
+apparmor	https://git.kernel.org/pub/scm/linux/kernel/git/jj/linux-apparmor#apparmor-next
+integrity	https://git.kernel.org/pub/scm/linux/kernel/git/zohar/linux-integrity#next-integrity
+selinux		https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/selinux.git#next
+smack		https://github.com/cschaufler/smack-next#next
+tomoyo		git://git.code.sf.net/p/tomoyo/tomoyo.git#master
+tpmdd		https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git#next
+watchdog	git://www.linux-watchdog.org/linux-watchdog-next.git#master
+iommu		https://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git#next
+audit		https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git#next
+devicetree	https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git#for-next
+dt-krzk		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git#for-next
+mailbox		https://git.kernel.org/pub/scm/linux/kernel/git/jassibrar/mailbox.git#for-next
+spi		https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git#for-next
+tip		https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git#master
+clockevents	https://git.kernel.org/pub/scm/linux/kernel/git/daniel.lezcano/linux.git#timers/drivers/next
+edac		https://git.kernel.org/pub/scm/linux/kernel/git/ras/ras.git#edac-for-next
+ftrace		https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#for-next
+rcu		https://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux#next
+paulmck		https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git#non-rcu/next
+kvm		git://git.kernel.org/pub/scm/virt/kvm/kvm.git#next
+kvm-arm		https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git#next
+kvms390		https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git#next
+kvm-ppc		https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#topic/ppc-kvm
+kvm-riscv	https://github.com/kvm-riscv/linux.git#riscv_kvm_next
+kvm-x86		https://github.com/kvm-x86/linux.git#next
+xen-tip		https://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git#linux-next
+percpu		https://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu.git#for-next
+workqueues	https://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git#for-next
+sched-ext	https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git#for-next
+drivers-x86	https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git#for-next
+chrome-platform	https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git#for-next
+chrome-platform-firmware	https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git#for-firmware-next
+hsi		https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-hsi.git#for-next
+leds-lj		https://git.kernel.org/pub/scm/linux/kernel/git/lee/leds.git#for-leds-next
+ipmi		https://github.com/cminyard/linux-ipmi.git#for-next
+driver-core	https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git#driver-core-next
+usb		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git#usb-next
+thunderbolt	https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git#next
+usb-serial	https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git#usb-next
+tty		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git#tty-next
+char-misc	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git#char-misc-next
+accel		https://git.kernel.org/pub/scm/linux/kernel/git/ogabbay/linux.git#habanalabs-next
+coresight	https://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git#next
+fastrpc		https://git.kernel.org/pub/scm/linux/kernel/git/srini/fastrpc.git#for-next
+fpga		https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git#for-next
+icc		https://git.kernel.org/pub/scm/linux/kernel/git/djakov/icc.git#icc-next
+iio		https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git#togreg
+phy-next	https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git#next
+soundwire	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git#next
+extcon		https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/extcon.git#extcon-next
+gnss		https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git#gnss-next
+vfio		https://github.com/awilliam/linux-vfio.git#next
+w1		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git#for-next
+spmi		https://git.kernel.org/pub/scm/linux/kernel/git/sboyd/spmi.git#spmi-next
+staging		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git#staging-next
+counter-next	https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git#counter-next
+mux		https://gitlab.com/peda-linux/mux.git#for-next
+dmaengine	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git#next
+cgroup		https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git#for-next
+scsi		https://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi.git#for-next
+scsi-mkp	https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git#for-next
+vhost		https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git#linux-next
+rpmsg		https://git.kernel.org/pub/scm/linux/kernel/git/remoteproc/linux.git#for-next
+gpio-brgl	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#gpio/for-next
+gpio-intel	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git#for-next
+pinctrl		https://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl.git#for-next
+pinctrl-intel	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git#for-next
+pinctrl-renesas	https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git#renesas-pinctrl
+pinctrl-samsung	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git#for-next
+pwm		https://git.kernel.org/pub/scm/linux/kernel/git/ukleinek/linux.git#pwm/for-next
+ktest		https://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-ktest.git#for-next
+kselftest	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#next
+kunit		https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#test
+kunit-next	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#kunit
+livepatching	https://git.kernel.org/pub/scm/linux/kernel/git/livepatching/livepatching.git#for-next
+rtc		https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git#rtc-next
+nvdimm		https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git#libnvdimm-for-next
+at24		https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#at24/for-next
+ntb		https://github.com/jonmason/ntb.git#ntb-next
+seccomp		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/seccomp
+slimbus		https://git.kernel.org/pub/scm/linux/kernel/git/srini/slimbus.git#for-next
+nvmem		https://git.kernel.org/pub/scm/linux/kernel/git/srini/nvmem.git#for-next
+xarray		git://git.infradead.org/users/willy/xarray.git#main
+hyperv		https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git#hyperv-next
+auxdisplay	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git#for-next
+kgdb		https://git.kernel.org/pub/scm/linux/kernel/git/danielt/linux.git#kgdb/for-next
+hmm		https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#hmm
+cfi		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#cfi/next
+mhi		https://git.kernel.org/pub/scm/linux/kernel/git/mani/mhi.git#mhi-next
+memblock	https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git#for-next
+cxl		https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git#next
+zstd		https://github.com/terrelln/linux.git#zstd-next
+efi		https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git#next
+unicode		https://git.kernel.org/pub/scm/linux/kernel/git/krisman/unicode.git#for-next
+slab		https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git#slab/for-next
+random		https://git.kernel.org/pub/scm/linux/kernel/git/crng/random.git#master
+landlock	https://git.kernel.org/pub/scm/linux/kernel/git/mic/linux.git#next
+sysctl		https://git.kernel.org/pub/scm/linux/kernel/git/sysctl/sysctl.git#sysctl-next
+execve		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/execve
+bitmap		https://github.com/norov/linux.git#bitmap-for-next
+hte		https://git.kernel.org/pub/scm/linux/kernel/git/pateldipen1984/linux.git#for-next
+kspp		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/kspp
+nolibc		https://git.kernel.org/pub/scm/linux/kernel/git/nolibc/linux-nolibc.git#for-next
+iommufd		https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git#for-next
+turbostat	https://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux.git#next
+pwrseq		https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#pwrseq/for-next
+capabilities-next	https://git.kernel.org/pub/scm/linux/kernel/git/sergeh/linux.git#caps-next
+ipe		https://git.kernel.org/pub/scm/linux/kernel/git/wufan/ipe.git#next
+kcsan		https://git.kernel.org/pub/scm/linux/kernel/git/melver/linux.git#next
+crc		https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git#crc-next
+fwctl		https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git#for-next
+devsec-tsm	https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git#next
+hisilicon	https://github.com/hisilicon/linux-hisi.git#for-next
+kthread		https://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git#for-next
diff --git a/Next/merge.log b/Next/merge.log
new file mode 100644
index 000000000000..ee996aa0f30f
--- /dev/null
+++ b/Next/merge.log
@@ -0,0 +1,4648 @@
+$ date -R
+Fri, 19 Dec 2025 08:49:43 +1100
+$ git checkout master
+Already on 'master'
+$ git reset --hard stable
+HEAD is now at ea1013c15392 Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf
+Merging origin/master (dd9b004b7ff3 Merge tag 'trace-v6.19-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git origin/master
+Updating ea1013c15392..dd9b004b7ff3
+Fast-forward (no commit created; -m option ignored)
+ CREDITS                                            |  1 +
+ MAINTAINERS                                        |  2 +-
+ arch/arm64/include/asm/simd.h                      |  9 +-
+ drivers/acpi/acpi_pcc.c                            |  2 +-
+ drivers/acpi/cppc_acpi.c                           |  3 +-
+ drivers/base/power/runtime.c                       | 22 ++---
+ drivers/iommu/iommufd/io_pagetable.c               |  6 +-
+ drivers/iommu/iommufd/selftest.c                   | 14 +++-
+ drivers/net/can/Kconfig                            |  7 +-
+ drivers/net/can/Makefile                           |  2 +-
+ drivers/net/can/dev/Makefile                       |  5 +-
+ drivers/net/can/usb/gs_usb.c                       |  2 +-
+ drivers/net/dsa/lantiq/lantiq_gswip.c              |  3 -
+ drivers/net/dsa/lantiq/lantiq_gswip.h              |  2 -
+ drivers/net/dsa/lantiq/lantiq_gswip_common.c       | 19 +++--
+ drivers/net/dsa/lantiq/mxl-gsw1xx.c                | 46 ++++++++--
+ drivers/net/ethernet/broadcom/b44.c                |  3 +
+ drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c      |  3 +-
+ drivers/net/ethernet/freescale/enetc/enetc.c       |  3 +-
+ drivers/net/ethernet/freescale/fec_main.c          |  7 +-
+ drivers/net/ethernet/google/gve/gve_main.c         | 17 ++--
+ .../ethernet/hisilicon/hns3/hns3pf/hclge_main.c    |  3 +
+ .../net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c |  4 +-
+ .../ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c  |  4 +-
+ drivers/net/ethernet/mellanox/mlx5/core/devlink.c  |  5 ++
+ .../ethernet/mellanox/mlx5/core/diag/fw_tracer.c   | 97 +++++++++++++++++++---
+ .../ethernet/mellanox/mlx5/core/diag/fw_tracer.h   |  1 +
+ drivers/net/ethernet/mellanox/mlx5/core/en.h       |  2 +-
+ .../ethernet/mellanox/mlx5/core/en_accel/ipsec.c   |  8 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_main.c  |  1 -
+ drivers/net/ethernet/mellanox/mlx5/core/en_tx.c    |  6 +-
+ .../ethernet/mellanox/mlx5/core/eswitch_offloads.c |  6 ++
+ drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c | 48 +++++++++--
+ drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h |  1 +
+ drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c  |  1 +
+ .../net/ethernet/mellanox/mlx5/core/lag/mpesw.c    | 11 ++-
+ drivers/net/ethernet/mellanox/mlx5/core/main.c     |  1 +
+ drivers/net/ethernet/mellanox/mlxsw/spectrum_mr.c  |  2 +
+ .../net/ethernet/mellanox/mlxsw/spectrum_router.c  | 27 +++---
+ drivers/net/ethernet/realtek/r8169_main.c          |  5 +-
+ drivers/net/ethernet/ti/Kconfig                    |  3 +-
+ drivers/net/ipvlan/ipvlan_core.c                   |  3 +
+ drivers/net/phy/marvell-88q2xxx.c                  |  2 +-
+ drivers/net/phy/realtek/realtek_main.c             |  4 -
+ drivers/net/phy/sfp.c                              |  2 +
+ drivers/nfc/pn533/usb.c                            |  2 +-
+ drivers/powercap/intel_rapl_common.c               | 24 ++++--
+ drivers/powercap/intel_rapl_msr.c                  |  4 +-
+ drivers/powercap/powercap_sys.c                    | 22 +++--
+ .../int340x_thermal/processor_thermal_device_pci.c |  3 +-
+ drivers/thermal/thermal_core.c                     |  4 +-
+ fs/file_attr.c                                     |  2 +
+ fs/notify/fsnotify.c                               |  9 +-
+ fs/smb/client/cifsfs.h                             |  4 +-
+ fs/smb/client/cifspdu.h                            |  2 +-
+ fs/smb/common/smb1pdu.h                            | 56 +++++++++++++
+ fs/smb/common/smb2pdu.h                            | 41 +--------
+ fs/smb/common/smbglob.h                            |  2 -
+ fs/smb/server/smb_common.h                         |  1 +
+ include/linux/intel_rapl.h                         |  4 +
+ include/net/inet_frag.h                            | 18 +---
+ include/net/ipv6_frag.h                            |  9 +-
+ include/net/netfilter/nf_tables.h                  | 34 ++++++--
+ include/uapi/linux/energy_model.h                  |  1 +
+ include/uapi/linux/mptcp.h                         |  1 +
+ kernel/power/em_netlink_autogen.c                  |  1 +
+ kernel/power/em_netlink_autogen.h                  |  1 +
+ kernel/trace/ftrace.c                              |  7 +-
+ kernel/trace/trace.c                               |  2 +-
+ kernel/trace/trace_events.c                        |  2 +
+ lib/crypto/riscv/.gitignore                        |  2 +
+ net/caif/cffrml.c                                  |  9 +-
+ net/can/Kconfig                                    |  1 -
+ net/can/j1939/socket.c                             |  6 ++
+ net/can/j1939/transport.c                          |  2 +
+ net/ethtool/ioctl.c                                | 30 +++++--
+ net/handshake/request.c                            |  8 +-
+ net/hsr/hsr_forward.c                              |  2 +
+ net/ipv4/inet_fragment.c                           | 55 +++++++++++-
+ net/ipv4/ip_fragment.c                             | 22 ++---
+ net/mptcp/Kconfig                                  |  2 +-
+ net/mptcp/pm_netlink.c                             |  3 +-
+ net/mptcp/protocol.c                               | 22 +++--
+ net/netfilter/ipvs/ip_vs_xmit.c                    |  3 +
+ net/netfilter/nf_conncount.c                       | 25 +++---
+ net/netfilter/nf_conntrack_core.c                  |  3 +
+ net/netfilter/nf_flow_table_path.c                 |  4 +-
+ net/netfilter/nf_nat_core.c                        | 14 +---
+ net/netfilter/nf_tables_api.c                      | 84 +++++++++++++++----
+ net/netrom/nr_out.c                                |  4 +-
+ net/openvswitch/flow_netlink.c                     | 13 ++-
+ net/sched/act_mirred.c                             |  9 ++
+ net/sched/sch_ets.c                                |  6 +-
+ net/sctp/ipv6.c                                    |  2 +
+ net/sctp/socket.c                                  |  7 +-
+ net/unix/garbage.c                                 |  2 +-
+ tools/net/ynl/Makefile.deps                        |  2 +
+ tools/testing/selftests/iommu/iommufd.c            |  8 +-
+ tools/testing/selftests/net/af_unix/Makefile       |  7 +-
+ tools/testing/selftests/net/forwarding/config      |  1 +
+ .../net/forwarding/vxlan_bridge_1q_mc_ul.sh        | 76 +++++++----------
+ tools/testing/selftests/net/lib.sh                 |  3 +-
+ tools/testing/selftests/net/lib/ksft.h             |  6 +-
+ tools/testing/selftests/net/mptcp/pm_netlink.sh    |  4 +
+ tools/testing/selftests/net/mptcp/pm_nl_ctl.c      | 11 +++
+ .../selftests/net/netfilter/conntrack_clash.sh     |  9 +-
+ .../net/netfilter/conntrack_reverse_clash.c        | 13 ++-
+ .../net/netfilter/conntrack_reverse_clash.sh       |  2 +
+ .../packetdrill/conntrack_syn_challenge_ack.pkt    |  2 +-
+ tools/testing/selftests/net/tfo.c                  |  3 +-
+ tools/testing/selftests/net/tls.c                  |  2 +-
+ .../tc-testing/tc-tests/actions/mirred.json        | 46 ++++++++++
+ .../tc-testing/tc-tests/infra/qdiscs.json          | 78 +++++++++++++++++
+ 113 files changed, 935 insertions(+), 359 deletions(-)
+ create mode 100644 fs/smb/common/smb1pdu.h
+ create mode 100644 lib/crypto/riscv/.gitignore
+Merging ext4-fixes/fixes (d0b3b7b22dfa Linux 6.16-rc4)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git ext4-fixes/fixes
+Already up to date.
+Merging vfs-brauner-fixes/vfs.fixes (24835a96f21e Merge patch series "filelock: fix conflict detection with userland file delegations")
+$ git merge -m Merge branch 'vfs.fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git vfs-brauner-fixes/vfs.fixes
+Merge made by the 'ort' strategy.
+ Documentation/filesystems/locking.rst |   1 +
+ drivers/media/mc/mc-request.c         |   6 +-
+ fs/iomap/buffered-io.c                |  50 +++++++++-----
+ fs/iomap/iter.c                       |   6 +-
+ fs/locks.c                            | 119 +++++++++++++++++-----------------
+ fs/nfsd/nfs4layouts.c                 |  23 ++++++-
+ fs/nfsd/nfs4state.c                   |  19 ++++++
+ fs/xfs/xfs_iomap.c                    |  11 ++--
+ include/linux/filelock.h              |   1 +
+ include/linux/iomap.h                 |   8 ++-
+ 10 files changed, 156 insertions(+), 88 deletions(-)
+Merging fscrypt-current/for-current (1e39da974ce6 fscrypt: fix left shift underflow when inode->i_blkbits > PAGE_SHIFT)
+$ git merge -m Merge branch 'for-current' of https://git.kernel.org/pub/scm/fs/fscrypt/linux.git fscrypt-current/for-current
+Already up to date.
+Merging fsverity-current/for-current (211ddde0823f Linux 6.18-rc2)
+$ git merge -m Merge branch 'for-current' of https://git.kernel.org/pub/scm/fs/fsverity/linux.git fsverity-current/for-current
+Already up to date.
+Merging btrfs-fixes/next-fixes (5c94889f8419 Merge branch 'misc-6.19' into next-fixes)
+$ git merge -m Merge branch 'next-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git btrfs-fixes/next-fixes
+Merge made by the 'ort' strategy.
+ fs/btrfs/delayed-inode.c     | 32 +++++++++++++++++---------------
+ fs/btrfs/extent_io.c         |  8 ++++----
+ fs/btrfs/inode.c             | 22 +++++++++++++++-------
+ fs/btrfs/qgroup.c            | 21 +++++++++++++++++++--
+ fs/btrfs/transaction.c       | 11 ++++++-----
+ fs/btrfs/tree-log.c          |  6 ++----
+ include/trace/events/btrfs.h |  3 ++-
+ 7 files changed, 65 insertions(+), 38 deletions(-)
+Merging vfs-fixes/fixes (e1b4c6a58304 shmem: fix recovery on rename failures)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git vfs-fixes/fixes
+Already up to date.
+Merging erofs-fixes/fixes (f2a12cc3b97f erofs: avoid infinite loop due to incomplete zstd-compressed data)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git erofs-fixes/fixes
+Already up to date.
+Merging nfsd-fixes/nfsd-fixes (913f7cf77bf1 NFSD: NFSv4 file creation neglects setting ACL)
+$ git merge -m Merge branch 'nfsd-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux nfsd-fixes/nfsd-fixes
+Auto-merging fs/nfsd/vfs.h
+Merge made by the 'ort' strategy.
+ fs/nfsd/export.c                  | 2 +-
+ fs/nfsd/nfs4xdr.c                 | 5 +++++
+ fs/nfsd/nfsd.h                    | 8 +++++++-
+ fs/nfsd/nfssvc.c                  | 5 ++++-
+ fs/nfsd/vfs.h                     | 3 ++-
+ net/sunrpc/auth_gss/svcauth_gss.c | 3 ++-
+ net/sunrpc/xprtrdma/svc_rdma_rw.c | 7 +++++--
+ 7 files changed, 26 insertions(+), 7 deletions(-)
+Merging v9fs-fixes/fixes/next (2014c95afece Linux 6.14-rc1)
+$ git merge -m Merge branch 'fixes/next' of https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git v9fs-fixes/fixes/next
+Already up to date.
+Merging overlayfs-fixes/ovl-fixes (e8bd877fb76b ovl: fix possible double unlink)
+$ git merge -m Merge branch 'ovl-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git overlayfs-fixes/ovl-fixes
+Already up to date.
+Merging fscrypt/for-next (211ddde0823f Linux 6.18-rc2)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/fs/fscrypt/linux.git fscrypt/for-next
+Already up to date.
+Merging btrfs/for-next (58d1993910c4 Merge branch 'for-next-next-v6.19-20251218' into for-next-20251218)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git btrfs/for-next
+Merge made by the 'ort' strategy.
+ fs/btrfs/Kconfig           |   9 +-
+ fs/btrfs/bio.c             |  19 ++-
+ fs/btrfs/bio.h             |  16 ++-
+ fs/btrfs/block-group.c     |  11 +-
+ fs/btrfs/compression.c     |   1 -
+ fs/btrfs/ctree.c           |  49 +++----
+ fs/btrfs/ctree.h           |   9 ++
+ fs/btrfs/defrag.c          |   5 +-
+ fs/btrfs/delayed-inode.c   |   4 +-
+ fs/btrfs/direct-io.c       |  29 ++--
+ fs/btrfs/disk-io.c         |  87 +++---------
+ fs/btrfs/extent-io-tree.c  |   7 +-
+ fs/btrfs/extent-tree.c     |  80 +++++------
+ fs/btrfs/extent_io.c       |  70 ++++++----
+ fs/btrfs/file-item.c       |   4 -
+ fs/btrfs/file.c            |   9 +-
+ fs/btrfs/free-space-tree.c |   4 +-
+ fs/btrfs/fs.c              | 102 ++++++++++++--
+ fs/btrfs/fs.h              |  23 ++-
+ fs/btrfs/inode-item.c      |   7 +-
+ fs/btrfs/inode.c           | 338 +++++++++++++++++++++++++++------------------
+ fs/btrfs/messages.c        |  26 +---
+ fs/btrfs/messages.h        |  76 +++++-----
+ fs/btrfs/qgroup.c          | 103 ++++++++++----
+ fs/btrfs/relocation.c      |   5 +-
+ fs/btrfs/root-tree.c       |  47 +++----
+ fs/btrfs/scrub.c           |  39 ++----
+ fs/btrfs/space-info.c      |  18 +--
+ fs/btrfs/space-info.h      |  16 +++
+ fs/btrfs/super.c           |  29 ++--
+ fs/btrfs/sysfs.c           |  58 +-------
+ fs/btrfs/transaction.c     |  11 +-
+ fs/btrfs/volumes.c         |  19 ++-
+ fs/btrfs/zoned.c           |  55 ++++++++
+ fs/btrfs/zoned.h           |   8 ++
+ 35 files changed, 766 insertions(+), 627 deletions(-)
+Merging ceph/master (375068b6c206 libceph: prevent potential out-of-bounds reads in handle_auth_done())
+$ git merge -m Merge branch 'master' of https://github.com/ceph/ceph-client.git ceph/master
+Merge made by the 'ort' strategy.
+ net/ceph/messenger_v2.c | 2 ++
+ 1 file changed, 2 insertions(+)
+Merging cifs/for-next (d8a4af8f3d9d cifs: update internal module version number)
+$ git merge -m Merge branch 'for-next' of git://git.samba.org/sfrench/cifs-2.6.git cifs/for-next
+Already up to date.
+Merging configfs/configfs-next (f7f78098690d configfs: Constify ct_item_ops in struct config_item_type)
+$ git merge -m Merge branch 'configfs-next' of https://git.kernel.org/pub/scm/linux/kernel/git/a.hindborg/linux.git configfs/configfs-next
+Already up to date.
+Merging ecryptfs/next (fba133a34118 ecryptfs: Remove unused declartion ecryptfs_fill_zeros())
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/tyhicks/ecryptfs.git ecryptfs/next
+Auto-merging fs/ecryptfs/crypto.c
+Auto-merging fs/ecryptfs/ecryptfs_kernel.h
+Auto-merging fs/ecryptfs/keystore.c
+Merge made by the 'ort' strategy.
+ fs/ecryptfs/crypto.c          | 2 +-
+ fs/ecryptfs/ecryptfs_kernel.h | 1 -
+ fs/ecryptfs/keystore.c        | 4 ++--
+ 3 files changed, 3 insertions(+), 4 deletions(-)
+Merging dlm/next (8e4021078863 dlm: check for undefined release_option values)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm.git dlm/next
+Already up to date.
+Merging erofs/dev (e68e6eb58221 erofs: make z_erofs_crypto[] static)
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git erofs/dev
+Merge made by the 'ort' strategy.
+ fs/erofs/decompressor_crypto.c | 2 +-
+ fs/erofs/zdata.c               | 4 ++--
+ 2 files changed, 3 insertions(+), 3 deletions(-)
+Merging exfat/dev (f5a5c9b40490 fs: exfat: improve error code handling in exfat_find_empty_entry())
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/exfat.git exfat/dev
+Merge made by the 'ort' strategy.
+ fs/exfat/namei.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+Merging ext3/for_next (6f7c877cc397 fs: send fsnotify_xattr()/IN_ATTRIB from vfs_fileattr_set()/chattr(1))
+$ git merge -m Merge branch 'for_next' of https://git.kernel.org/pub/scm/linux/kernel/git/jack/linux-fs.git ext3/for_next
+Already up to date.
+Merging ext4/dev (6fb67ac89690 ext4: drop the TODO comment in ext4_es_insert_extent())
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git ext4/dev
+Auto-merging fs/ext4/inode.c
+Merge made by the 'ort' strategy.
+ fs/ext4/extents.c        | 137 +++++++++++++++++++++++++++++------------------
+ fs/ext4/extents_status.c | 124 ++++++++++++++++++++++++++++++++----------
+ fs/ext4/inode.c          |  18 +++----
+ 3 files changed, 188 insertions(+), 91 deletions(-)
+Merging f2fs/dev (4a210a5be279 f2fs: improve check for enough free sections)
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git f2fs/dev
+Merge made by the 'ort' strategy.
+ Documentation/filesystems/f2fs.rst |  43 +++++++
+ fs/f2fs/data.c                     | 248 +++++++++++++++++++++++++++++++++++--
+ fs/f2fs/f2fs.h                     |  16 +++
+ fs/f2fs/file.c                     |   4 +
+ fs/f2fs/gc.c                       |   2 +-
+ fs/f2fs/inode.c                    |   6 +-
+ fs/f2fs/segment.h                  |  99 +++++++--------
+ include/trace/events/f2fs.h        |  12 +-
+ 8 files changed, 365 insertions(+), 65 deletions(-)
+Merging fsverity/for-next (211ddde0823f Linux 6.18-rc2)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/fs/fsverity/linux.git fsverity/for-next
+Already up to date.
+Merging fuse/for-next (8da059f2a497 fuse: Uninitialized variable in fuse_epoch_work())
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git fuse/for-next
+Already up to date.
+Merging gfs2/for-next (02e96fafcb36 gfs2: Fix slab-use-after-free in qd_put)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gfs2/linux-gfs2.git gfs2/for-next
+Merge made by the 'ort' strategy.
+ fs/gfs2/glock.c      | 85 ++++++++++++++++++++++++++--------------------------
+ fs/gfs2/glock.h      |  6 ++--
+ fs/gfs2/glops.c      | 34 ++++++++++-----------
+ fs/gfs2/incore.h     | 16 ++++++++--
+ fs/gfs2/lock_dlm.c   | 28 ++++++++---------
+ fs/gfs2/log.c        |  7 +++--
+ fs/gfs2/lops.c       | 70 +++++++++++++++++++++++--------------------
+ fs/gfs2/lops.h       |  4 +--
+ fs/gfs2/meta_io.c    |  6 ++--
+ fs/gfs2/meta_io.h    |  2 +-
+ fs/gfs2/quota.c      |  5 ++--
+ fs/gfs2/rgrp.c       |  2 +-
+ fs/gfs2/trace_gfs2.h | 48 ++++++++++++++---------------
+ fs/gfs2/trans.c      |  4 +--
+ 14 files changed, 168 insertions(+), 149 deletions(-)
+Merging jfs/jfs-next (eb0cfcf26571 jfs: Add missing set_freezable() for freezable kthread)
+$ git merge -m Merge branch 'jfs-next' of https://github.com/kleikamp/linux-shaggy.git jfs/jfs-next
+Merge made by the 'ort' strategy.
+ fs/jfs/jfs_logmgr.c | 1 +
+ fs/jfs/namei.c      | 6 ++++--
+ 2 files changed, 5 insertions(+), 2 deletions(-)
+Merging ksmbd/ksmbd-for-next (53ec4a79ff4b Merge tag 'v6.19-rc1-ksmbd-server-fixes' of git://git.samba.org/ksmbd)
+$ git merge -m Merge branch 'ksmbd-for-next' of https://github.com/smfrench/smb3-kernel.git ksmbd/ksmbd-for-next
+Already up to date.
+Merging nfs/linux-next (bd3b04b46c7a NFSv4: Handle NFS4ERR_NOTSUPP errors for directory delegations)
+$ git merge -m Merge branch 'linux-next' of git://git.linux-nfs.org/projects/trondmy/nfs-2.6.git nfs/linux-next
+Already up to date.
+Merging nfs-anna/linux-next (b623390045a8 NFS: Fix LTP test failures when timestamps are delegated)
+$ git merge -m Merge branch 'linux-next' of git://git.linux-nfs.org/projects/anna/linux-nfs.git nfs-anna/linux-next
+Already up to date.
+Merging nfsd/nfsd-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'nfsd-next' of https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux nfsd/nfsd-next
+Already up to date.
+Merging ntfs3/master (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'master' of https://github.com/Paragon-Software-Group/linux-ntfs3.git ntfs3/master
+Already up to date.
+Merging orangefs/for-next (11f6bce77e27 fs/orangefs: Replace kzalloc + copy_from_user with memdup_user_nul)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux.git orangefs/for-next
+Already up to date.
+Merging overlayfs/overlayfs-next (ad1423922781 ovl: make sure that ovl_create_real() returns a hashed dentry)
+$ git merge -m Merge branch 'overlayfs-next' of https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git overlayfs/overlayfs-next
+Already up to date.
+Merging ubifs/next (0695aef23d67 ubifs: vmalloc(array_size()) -> vmalloc_array())
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/rw/ubifs.git ubifs/next
+Already up to date.
+Merging v9fs/9p-next (3e281113f871 9p: fix new mount API cache option handling)
+$ git merge -m Merge branch '9p-next' of https://github.com/martinetd/linux v9fs/9p-next
+Already up to date.
+Merging v9fs-ericvh/ericvh/for-next (2014c95afece Linux 6.14-rc1)
+$ git merge -m Merge branch 'ericvh/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git v9fs-ericvh/ericvh/for-next
+Already up to date.
+Merging xfs/for-next (f401306d72f2 xfs: factor out a xlog_write_space_advance helper)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git xfs/for-next
+Merge made by the 'ort' strategy.
+ fs/xfs/libxfs/xfs_log_format.h |   7 -
+ fs/xfs/libxfs/xfs_sb.c         |  15 +++
+ fs/xfs/scrub/attr_repair.c     |   2 +-
+ fs/xfs/xfs_attr_item.c         |  29 ++--
+ fs/xfs/xfs_bmap_item.c         |  10 +-
+ fs/xfs/xfs_buf_item.c          |  20 ++-
+ fs/xfs/xfs_discard.c           |  27 +---
+ fs/xfs/xfs_discard.h           |   2 +-
+ fs/xfs/xfs_dquot_item.c        |   9 +-
+ fs/xfs/xfs_exchmaps_item.c     |  11 +-
+ fs/xfs/xfs_extfree_item.c      |  10 +-
+ fs/xfs/xfs_file.c              |  58 ++++++--
+ fs/xfs/xfs_icreate_item.c      |   6 +-
+ fs/xfs/xfs_inode_item.c        |  49 ++++---
+ fs/xfs/xfs_log.c               | 292 ++++++++++++++++-------------------------
+ fs/xfs/xfs_log.h               |  65 ++-------
+ fs/xfs/xfs_log_cil.c           | 111 ++++++++++++++--
+ fs/xfs/xfs_log_priv.h          |  20 +++
+ fs/xfs/xfs_refcount_item.c     |  10 +-
+ fs/xfs/xfs_rmap_item.c         |  10 +-
+ fs/xfs/xfs_rtalloc.c           |  14 +-
+ fs/xfs/xfs_trans.h             |   4 +-
+ 22 files changed, 393 insertions(+), 388 deletions(-)
+Merging zonefs/for-next (a42938e80357 zonefs: correct some spelling mistakes)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dlemoal/zonefs.git zonefs/for-next
+Already up to date.
+Merging vfs-brauner/vfs.all (981be27a72d1 Merge branch 'vfs-7.0.atomic_open' into vfs.all)
+$ git merge -m Merge branch 'vfs.all' of https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git vfs-brauner/vfs.all
+Auto-merging include/linux/fs.h
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/kernel-parameters.txt    |  12 +-
+ Documentation/admin-guide/sysctl/kernel.rst        |   6 -
+ Documentation/filesystems/locking.rst              |   8 -
+ Documentation/filesystems/mount_api.rst            |   2 -
+ Documentation/filesystems/porting.rst              |   7 +-
+ Documentation/filesystems/vfs.rst                  |  58 +----
+ arch/arm/configs/neponset_defconfig                |   2 +-
+ fs/buffer.c                                        |   6 +-
+ fs/char_dev.c                                      |   2 +-
+ fs/exec.c                                          |   2 +-
+ fs/ext4/mballoc.c                                  |   3 +-
+ fs/ext4/resize.c                                   |   2 +-
+ fs/ext4/super.c                                    |   2 +-
+ fs/fat/dir.c                                       |   4 +-
+ fs/fat/file.c                                      |   3 +-
+ fs/fs_context.c                                    | 208 +---------------
+ fs/fsopen.c                                        |  10 -
+ fs/fuse/dev.c                                      |   2 +-
+ fs/fuse/file.c                                     |   8 +-
+ fs/init.c                                          |  14 --
+ fs/internal.h                                      |   1 -
+ fs/locks.c                                         |  14 +-
+ fs/namei.c                                         |  80 ++++++-
+ fs/namespace.c                                     | 102 +++++---
+ fs/nfsd/nfs4proc.c                                 |  11 +-
+ fs/open.c                                          |  39 ---
+ fs/splice.c                                        |   2 +-
+ include/linux/filelock.h                           |  18 +-
+ include/linux/fs.h                                 |   5 +-
+ include/linux/fs/super_types.h                     |   1 -
+ include/linux/init_syscalls.h                      |   1 -
+ include/linux/initrd.h                             |   2 -
+ include/linux/ns/ns_common_types.h                 |   4 +-
+ include/uapi/linux/mount.h                         |  10 +-
+ include/uapi/linux/sysctl.h                        |   1 -
+ init/do_mounts.c                                   |  11 +-
+ init/do_mounts.h                                   |  18 +-
+ init/do_mounts_initrd.c                            | 107 +--------
+ init/do_mounts_rd.c                                |  24 +-
+ kernel/pid.c                                       | 131 +++++++----
+ rust/helpers/fs.c                                  |   2 +-
+ rust/helpers/pid_namespace.c                       |   8 +-
+ rust/helpers/poll.c                                |   5 +-
+ .../selftests/filesystems/statmount/statmount.h    |  15 +-
+ .../filesystems/statmount/statmount_test.c         | 261 +++++++++++++++++++--
+ .../filesystems/statmount/statmount_test_ns.c      | 101 +++++++-
+ 46 files changed, 685 insertions(+), 650 deletions(-)
+Merging vfs/for-next (eb028c33451a d_make_discardable(): warn if given a non-persistent dentry)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git vfs/for-next
+Already up to date.
+Merging mm-hotfixes/mm-hotfixes-unstable (201037a59604 mm/memremap: fix spurious large folio warning for FS-DAX)
+$ git merge -m Merge branch 'mm-hotfixes-unstable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-hotfixes/mm-hotfixes-unstable
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .mailmap                                     |  4 +--
+ MAINTAINERS                                  |  5 ++-
+ arch/x86/kernel/setup.c                      | 14 ++++++++
+ include/linux/genalloc.h                     |  1 +
+ include/linux/hugetlb.h                      |  4 +--
+ include/linux/kasan.h                        | 16 +++++++++
+ include/linux/kexec.h                        |  4 +--
+ include/linux/leafops.h                      |  4 +--
+ include/linux/memory-failure.h               |  2 ++
+ include/linux/mm.h                           | 12 +++----
+ kernel/kexec_core.c                          | 16 ++++++---
+ lib/idr.c                                    |  2 ++
+ mm/damon/vaddr.c                             |  2 +-
+ mm/hugetlb.c                                 |  4 +--
+ mm/kasan/common.c                            | 32 +++++++++++++++++
+ mm/kasan/hw_tags.c                           |  2 +-
+ mm/kasan/shadow.c                            |  4 ++-
+ mm/memcontrol.c                              |  4 +--
+ mm/memory-failure.c                          | 51 +++++++++++++++++-----------
+ mm/memremap.c                                |  7 +++-
+ mm/page_alloc.c                              | 34 ++++++++++++-------
+ mm/vmalloc.c                                 |  8 ++---
+ rust/kernel/maple_tree.rs                    | 11 +++++-
+ security/integrity/ima/ima_kexec.c           |  4 +--
+ tools/mm/page_owner_sort.c                   |  6 +++-
+ tools/testing/radix-tree/idr-test.c          | 21 ++++++++++++
+ tools/testing/selftests/mm/uffd-unit-tests.c |  2 +-
+ 27 files changed, 206 insertions(+), 70 deletions(-)
+Merging fs-current (3bfe1da11c0d Merge branch 'nfsd-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux)
+$ git merge -m Merge branch 'fs-current' of linux-next fs-current
+Merge made by the 'ort' strategy.
+ Documentation/filesystems/locking.rst |   1 +
+ drivers/media/mc/mc-request.c         |   6 +-
+ fs/btrfs/delayed-inode.c              |  32 ++++-----
+ fs/btrfs/extent_io.c                  |   8 +--
+ fs/btrfs/inode.c                      |  22 +++++--
+ fs/btrfs/qgroup.c                     |  21 +++++-
+ fs/btrfs/transaction.c                |  11 ++--
+ fs/btrfs/tree-log.c                   |   6 +-
+ fs/iomap/buffered-io.c                |  50 +++++++++-----
+ fs/iomap/iter.c                       |   6 +-
+ fs/locks.c                            | 119 +++++++++++++++++-----------------
+ fs/nfsd/export.c                      |   2 +-
+ fs/nfsd/nfs4layouts.c                 |  23 ++++++-
+ fs/nfsd/nfs4state.c                   |  19 ++++++
+ fs/nfsd/nfs4xdr.c                     |   5 ++
+ fs/nfsd/nfsd.h                        |   8 ++-
+ fs/nfsd/nfssvc.c                      |   5 +-
+ fs/nfsd/vfs.h                         |   3 +-
+ fs/xfs/xfs_iomap.c                    |  11 ++--
+ include/linux/filelock.h              |   1 +
+ include/linux/iomap.h                 |   8 ++-
+ include/trace/events/btrfs.h          |   3 +-
+ net/sunrpc/auth_gss/svcauth_gss.c     |   3 +-
+ net/sunrpc/xprtrdma/svc_rdma_rw.c     |   7 +-
+ 24 files changed, 247 insertions(+), 133 deletions(-)
+Merging kbuild-current/kbuild-fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'kbuild-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git kbuild-current/kbuild-fixes
+Already up to date.
+Merging arc-current/for-curr (d7b8f8e20813 Linux 6.16-rc5)
+$ git merge -m Merge branch 'for-curr' of https://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git arc-current/for-curr
+Already up to date.
+Merging arm-current/fixes (fd2dee1c6e22 ARM: fix branch predictor hardening)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git arm-current/fixes
+Already up to date.
+Merging arm64-fixes/for-next/fixes (98a97bf41528 arm64/gcs: Flush the GCS locking state on exec)
+$ git merge -m Merge branch 'for-next/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux arm64-fixes/for-next/fixes
+Merge made by the 'ort' strategy.
+ arch/arm64/kernel/fpsimd.c              | 130 +++++---------------------------
+ arch/arm64/kernel/process.c             |   1 +
+ arch/arm64/mm/mmu.c                     |  33 ++++----
+ drivers/misc/lkdtm/bugs.c               |  53 +++++++++++++
+ tools/testing/selftests/lkdtm/tests.txt |   2 +
+ 5 files changed, 92 insertions(+), 127 deletions(-)
+Merging arm-soc-fixes/arm/fixes (ececfba255bf ARM: dts: ixp4xx: Fix up Actiontec MI424WR DTS files)
+$ git merge -m Merge branch 'arm/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git arm-soc-fixes/arm/fixes
+Merge made by the 'ort' strategy.
+ .../boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr-ac.dts  | 11 +++++++++++
+ .../boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr-d.dts   | 11 +++++++++++
+ .../boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr.dtsi    |  1 -
+ 3 files changed, 22 insertions(+), 1 deletion(-)
+Merging davinci-current/davinci/for-current (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'davinci/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git davinci-current/davinci/for-current
+Already up to date.
+Merging drivers-memory-fixes/fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git drivers-memory-fixes/fixes
+Already up to date.
+Merging sophgo-fixes/fixes (19272b37aa4f Linux 6.16-rc1)
+$ git merge -m Merge branch 'fixes' of https://github.com/sophgo/linux.git sophgo-fixes/fixes
+Already up to date.
+Merging sophgo-soc-fixes/soc-fixes (0af2f6be1b42 Linux 6.15-rc1)
+$ git merge -m Merge branch 'soc-fixes' of https://github.com/sophgo/linux.git sophgo-soc-fixes/soc-fixes
+Already up to date.
+Merging m68k-current/for-linus (aaf4e92341b0 m68k: defconfig: Update defconfigs for v6.18-rc1)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git m68k-current/for-linus
+Already up to date.
+Merging powerpc-fixes/fixes (0843ba458439 powerpc/fadump: skip parameter area allocation when fadump is disabled)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git powerpc-fixes/fixes
+Already up to date.
+Merging s390-fixes/fixes (489e96651dfe s390/stacktrace: Do not fallback to RA register)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git s390-fixes/fixes
+Already up to date.
+Merging net/main (21a88f5d9ce0 Merge tag 'linux-can-fixes-for-6.19-20251218' of git://git.kernel.org/pub/scm/linux/kernel/git/mkl/linux-can)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git net/main
+Already up to date.
+Merging bpf/master (ea1013c15392 Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/ bpf/master
+Already up to date.
+Merging ipsec/master (c196def07bbc xfrm: set ipv4 no_pmtu_disc flag only on output sa when direction is set)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec.git ipsec/master
+Merge made by the 'ort' strategy.
+ net/ipv4/esp4_offload.c | 4 ++--
+ net/ipv6/esp6_offload.c | 4 ++--
+ net/xfrm/xfrm_state.c   | 1 +
+ 3 files changed, 5 insertions(+), 4 deletions(-)
+Merging netfilter/main (21a88f5d9ce0 Merge tag 'linux-can-fixes-for-6.19-20251218' of git://git.kernel.org/pub/scm/linux/kernel/git/mkl/linux-can)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf.git netfilter/main
+Already up to date.
+Merging ipvs/main (de0337d641bf net: ionic: map SKB after pseudo-header checksum prep)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs.git ipvs/main
+Already up to date.
+Merging wireless/for-next (81d90d93d22c wifi: iwlwifi: Implement settime64 as stub for MVM/MLD PTP)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git wireless/for-next
+Merge made by the 'ort' strategy.
+ drivers/net/wireless/intel/iwlwifi/iwl-drv.c         |  4 ++--
+ drivers/net/wireless/intel/iwlwifi/mld/ptp.c         |  7 +++++++
+ drivers/net/wireless/intel/iwlwifi/mvm/ptp.c         |  7 +++++++
+ drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.c |  3 ++-
+ drivers/net/wireless/realtek/rtw88/sdio.c            |  4 +++-
+ drivers/net/wireless/realtek/rtw88/usb.c             |  3 +--
+ drivers/net/wireless/ti/wlcore/tx.c                  |  5 +++++
+ net/mac80211/cfg.c                                   | 10 ----------
+ net/mac80211/iface.c                                 |  2 +-
+ net/mac80211/mlme.c                                  |  5 ++++-
+ net/mac80211/ocb.c                                   |  3 +++
+ net/mac80211/rx.c                                    |  5 +++++
+ net/wireless/sme.c                                   |  2 +-
+ 13 files changed, 41 insertions(+), 19 deletions(-)
+Merging ath/for-current (d0309c054362 Merge tag 'net-6.18-rc6' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net)
+$ git merge -m Merge branch 'for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git ath/for-current
+Already up to date.
+Merging iwlwifi/fixes (8f7aa3d3c732 Merge tag 'net-next-6.19' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git iwlwifi/fixes
+Already up to date.
+Merging wpan/master (8ce4f287524c net: libwx: fix firmware mailbox abnormal return)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan.git wpan/master
+Already up to date.
+Merging rdma-fixes/for-rc (85463eb6a46c RDMA/efa: Remove possible negative shift)
+$ git merge -m Merge branch 'for-rc' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git rdma-fixes/for-rc
+Merge made by the 'ort' strategy.
+ drivers/infiniband/core/addr.c         | 33 ++++++++++-----------------------
+ drivers/infiniband/core/cma.c          |  3 +++
+ drivers/infiniband/hw/efa/efa_verbs.c  |  4 ----
+ drivers/infiniband/hw/irdma/utils.c    |  3 ++-
+ drivers/infiniband/hw/mana/cq.c        |  4 ++++
+ drivers/infiniband/ulp/rtrs/rtrs-pri.h | 32 +++++++++++++++++++++-----------
+ drivers/infiniband/ulp/rtrs/rtrs.h     | 24 +++++++++++++++---------
+ include/uapi/rdma/irdma-abi.h          |  2 +-
+ include/uapi/rdma/rdma_user_cm.h       |  4 +++-
+ 9 files changed, 59 insertions(+), 50 deletions(-)
+Merging sound-current/for-linus (f7cede182c96 ALSA: hda/realtek: Add Asus quirk for TAS amplifiers)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git sound-current/for-linus
+Merge made by the 'ort' strategy.
+ sound/hda/codecs/realtek/alc269.c  | 30 +++++++++++++++++++++++++-----
+ sound/hda/controllers/cix-ipbloq.c |  4 +---
+ sound/pcmcia/pdaudiocf/pdaudiocf.c |  8 +++++++-
+ sound/pcmcia/vx/vxpocket.c         |  8 +++++++-
+ sound/usb/endpoint.c               |  8 ++++----
+ sound/usb/format.c                 |  5 ++++-
+ sound/usb/mixer_us16x08.c          | 20 ++++++++++++++------
+ sound/usb/quirks.c                 | 14 ++++++++++++--
+ 8 files changed, 74 insertions(+), 23 deletions(-)
+Merging sound-asoc-fixes/for-linus (fa43ab13c59f ASoC: fsl-asoc-card: Use of_property_present() for non-boolean properties)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git sound-asoc-fixes/for-linus
+Merge made by the 'ort' strategy.
+ include/sound/soc-acpi.h                           |  5 ++-
+ sound/soc/amd/yc/acp6x-mach.c                      |  7 ++++
+ sound/soc/codecs/ak4458.c                          |  4 --
+ sound/soc/codecs/rt1320-sdw.c                      | 16 ++++---
+ sound/soc/fsl/fsl-asoc-card.c                      |  8 ++--
+ sound/soc/fsl/fsl_sai.c                            | 13 +++++-
+ sound/soc/fsl/fsl_xcvr.c                           |  3 ++
+ sound/soc/intel/common/soc-acpi-intel-nvl-match.c  | 49 ----------------------
+ sound/soc/intel/common/sof-function-topology-lib.c |  5 ++-
+ sound/soc/intel/common/sof-function-topology-lib.h |  2 +-
+ sound/soc/qcom/sdm845.c                            |  2 +
+ sound/soc/sdw_utils/soc_sdw_utils.c                |  8 +++-
+ sound/soc/sof/intel/pci-mtl.c                      |  6 +--
+ sound/soc/sof/ipc4-topology.c                      | 49 ++++++++++++++--------
+ sound/soc/sof/topology.c                           | 26 +++++++++---
+ sound/soc/tegra/tegra210_ahub.c                    |  6 +--
+ 16 files changed, 108 insertions(+), 101 deletions(-)
+Merging regmap-fixes/for-linus (5eb484d5b2ac Merge remote-tracking branch 'regmap/for-6.18' into regmap-linus)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git regmap-fixes/for-linus
+Merge made by the 'ort' strategy.
+Merging regulator-fixes/for-linus (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git regulator-fixes/for-linus
+Already up to date.
+Merging spi-fixes/for-linus (9d651a6c6283 spi: cadence-quadspi: Fix probe error path and logging)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git spi-fixes/for-linus
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml | 6 +++---
+ drivers/spi/spi-cadence-quadspi.c                          | 8 ++++++--
+ drivers/spi/spi-fsl-spi.c                                  | 2 +-
+ drivers/spi/spi-mpfs.c                                     | 1 +
+ drivers/spi/spi-mt65xx.c                                   | 2 +-
+ 5 files changed, 12 insertions(+), 7 deletions(-)
+Merging pci-current/for-linus (94bf74830a97 PCI: rzg3s-host: Initialize MSI status bitmap before use)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git pci-current/for-linus
+Already up to date.
+Merging driver-core.current/driver-core-linus (e2f1081ca8f1 samples: rust: fix endianness issue in rust_driver_pci)
+$ git merge -m Merge branch 'driver-core-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git driver-core.current/driver-core-linus
+Merge made by the 'ort' strategy.
+ rust/helpers/dma.c              | 21 +++++++++++++++++++++
+ samples/rust/rust_driver_pci.rs |  2 +-
+ 2 files changed, 22 insertions(+), 1 deletion(-)
+Merging tty.current/tty-linus (5edb7338d611 serial: 8250: longson: Fix NULL vs IS_ERR() bug in probe)
+$ git merge -m Merge branch 'tty-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git tty.current/tty-linus
+Merge made by the 'ort' strategy.
+ drivers/tty/serial/8250/8250_loongson.c |  4 ++--
+ drivers/tty/serial/serial_base_bus.c    | 10 ++++++----
+ 2 files changed, 8 insertions(+), 6 deletions(-)
+Merging usb.current/usb-linus (3b4961313d31 usb: dwc3: of-simple: fix clock resource leak in dwc3_of_simple_probe)
+$ git merge -m Merge branch 'usb-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git usb.current/usb-linus
+Merge made by the 'ort' strategy.
+ drivers/usb/dwc3/dwc3-of-simple.c        |  7 ++++---
+ drivers/usb/dwc3/gadget.c                |  2 +-
+ drivers/usb/dwc3/host.c                  |  2 +-
+ drivers/usb/gadget/udc/lpc32xx_udc.c     | 21 +++++++++++++++------
+ drivers/usb/host/xhci-dbgtty.c           |  2 +-
+ drivers/usb/phy/phy-fsl-usb.c            |  1 +
+ drivers/usb/renesas_usbhs/pipe.c         |  2 ++
+ drivers/usb/typec/altmodes/displayport.c |  8 ++++++--
+ drivers/usb/typec/ucsi/Kconfig           |  1 +
+ drivers/usb/typec/ucsi/ucsi.c            |  2 +-
+ 10 files changed, 33 insertions(+), 15 deletions(-)
+Merging usb-serial-fixes/usb-linus (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'usb-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git usb-serial-fixes/usb-linus
+Already up to date.
+Merging phy/fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git phy/fixes
+Already up to date.
+Merging staging.current/staging-linus (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'staging-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git staging.current/staging-linus
+Already up to date.
+Merging iio-fixes/fixes-togreg (74f97a468356 iio: adc: at91-sama5d2_adc: Fix potential use-after-free in sama5d2_adc driver)
+$ git merge -m Merge branch 'fixes-togreg' of https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git iio-fixes/fixes-togreg
+Merge made by the 'ort' strategy.
+ drivers/iio/adc/at91-sama5d2_adc.c | 1 +
+ 1 file changed, 1 insertion(+)
+Merging counter-current/counter-current (109ff654934a counter: microchip-tcb-capture: Allow shared IRQ for multi-channel TCBs)
+$ git merge -m Merge branch 'counter-current' of https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git counter-current/counter-current
+Already up to date.
+Merging char-misc.current/char-misc-linus (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'char-misc-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git char-misc.current/char-misc-linus
+Already up to date.
+Merging soundwire-fixes/fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git soundwire-fixes/fixes
+Already up to date.
+Merging thunderbolt-fixes/fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git thunderbolt-fixes/fixes
+Already up to date.
+Merging input-current/for-linus (aed3716db7ff Input: i8042 - add TUXEDO InfinityBook Max Gen10 AMD to i8042 quirk table)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git input-current/for-linus
+Merge made by the 'ort' strategy.
+ drivers/input/keyboard/atkbd.c        | 7 +++++++
+ drivers/input/keyboard/lkkbd.c        | 5 ++++-
+ drivers/input/serio/i8042-acpipnpio.h | 7 +++++++
+ 3 files changed, 18 insertions(+), 1 deletion(-)
+Merging crypto-current/master (48bc9da3c97c crypto: zstd - fix double-free in per-CPU stream cleanup)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git crypto-current/master
+Already up to date.
+Merging libcrypto-fixes/libcrypto-fixes (5a0b18825068 lib/crypto: riscv: Add poly1305-core.S to .gitignore)
+$ git merge -m Merge branch 'libcrypto-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git libcrypto-fixes/libcrypto-fixes
+Already up to date.
+Merging vfio-fixes/for-linus (d323ad739666 vfio: selftests: replace iova=vaddr with allocated iovas)
+$ git merge -m Merge branch 'for-linus' of https://github.com/awilliam/linux-vfio.git vfio-fixes/for-linus
+Already up to date.
+Merging kselftest-fixes/fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kselftest-fixes/fixes
+Already up to date.
+Merging dmaengine-fixes/fixes (747213b08a1a dmaengine: sh: rz-dmac: Fix rz_dmac_terminate_all())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git dmaengine-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/dma/at_hdmac.c           |  9 +++++++--
+ drivers/dma/bcm-sba-raid.c       |  6 +++++-
+ drivers/dma/cv1800b-dmamux.c     | 17 ++++++++++-------
+ drivers/dma/dw/rzn1-dmamux.c     |  4 +++-
+ drivers/dma/fsl-edma-common.c    |  1 +
+ drivers/dma/idxd/compat.c        | 23 +++++++++++++++++++----
+ drivers/dma/lpc18xx-dmamux.c     | 19 ++++++++++++++-----
+ drivers/dma/lpc32xx-dmamux.c     | 19 ++++++++++++++-----
+ drivers/dma/mmp_pdma.c           | 26 ++++++++++++++------------
+ drivers/dma/sh/rz-dmac.c         | 18 ++++++++++++++++--
+ drivers/dma/stm32/stm32-dmamux.c | 31 +++++++++++++++++++------------
+ drivers/dma/tegra210-adma.c      | 10 +++++++++-
+ drivers/dma/ti/dma-crossbar.c    | 35 ++++++++++++++++++++---------------
+ drivers/dma/ti/k3-udma-private.c |  2 +-
+ drivers/dma/xilinx/xdma-regs.h   |  1 +
+ drivers/dma/xilinx/xdma.c        |  2 +-
+ drivers/dma/xilinx/xilinx_dma.c  |  7 +++++--
+ 17 files changed, 159 insertions(+), 71 deletions(-)
+Merging backlight-fixes/for-backlight-fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-backlight-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git backlight-fixes/for-backlight-fixes
+Already up to date.
+Merging mtd-fixes/mtd/fixes (2f6d2c8d9ac0 Revert "mtd: spinand: esmt: fix id code for F50D1G41LB")
+$ git merge -m Merge branch 'mtd/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git mtd-fixes/mtd/fixes
+Merge made by the 'ort' strategy.
+ drivers/mtd/nand/spi/esmt.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging mfd-fixes/for-mfd-fixes (aa6e63373d91 mfd: ls2kbmc: Use PCI API instead of direct accesses)
+$ git merge -m Merge branch 'for-mfd-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git mfd-fixes/for-mfd-fixes
+Merge made by the 'ort' strategy.
+ drivers/mfd/ls2k-bmc-core.c | 38 ++++++++++----------------------------
+ 1 file changed, 10 insertions(+), 28 deletions(-)
+Merging v4l-dvb-fixes/fixes (27afd6e066cf media: videobuf2: forbid remove_bufs when legacy fileio is active)
+$ git merge -m Merge branch 'fixes' of git://linuxtv.org/media-ci/media-pending.git v4l-dvb-fixes/fixes
+Already up to date.
+Merging reset-fixes/reset/fixes (997c06330fd5 reset: imx8mp-audiomix: Fix bad mask values)
+$ git merge -m Merge branch 'reset/fixes' of https://git.pengutronix.de/git/pza/linux reset-fixes/reset/fixes
+Already up to date.
+Merging mips-fixes/mips-fixes (680ad315caaa MIPS: Fix a reference leak bug in ip22_check_gio())
+$ git merge -m Merge branch 'mips-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git mips-fixes/mips-fixes
+Merge made by the 'ort' strategy.
+ arch/mips/alchemy/common/setup.c | 3 +--
+ arch/mips/sgi-ip22/ip22-gio.c    | 3 ++-
+ 2 files changed, 3 insertions(+), 3 deletions(-)
+Merging at91-fixes/at91-fixes (3a8660878839 Linux 6.18-rc1)
+$ git merge -m Merge branch 'at91-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git at91-fixes/at91-fixes
+Already up to date.
+Merging omap-fixes/fixes (3a8660878839 Linux 6.18-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git omap-fixes/fixes
+Already up to date.
+Merging kvm-fixes/master (0499add8efd7 Merge tag 'kvm-x86-fixes-6.19-rc1' of https://github.com/kvm-x86/linux into HEAD)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git kvm-fixes/master
+Merge made by the 'ort' strategy.
+ arch/x86/kvm/cpuid.c                         | 11 +++++++++--
+ arch/x86/kvm/svm/nested.c                    |  4 ++--
+ arch/x86/kvm/svm/svm.c                       |  2 ++
+ arch/x86/kvm/svm/svm.h                       |  7 ++++---
+ arch/x86/kvm/vmx/nested.c                    |  3 ++-
+ arch/x86/kvm/vmx/vmx.c                       |  9 ---------
+ arch/x86/kvm/x86.c                           |  7 +++++++
+ tools/testing/selftests/kvm/rseq_test.c      |  1 +
+ tools/testing/selftests/kvm/x86/cpuid_test.c | 15 +++++++++++++++
+ virt/kvm/kvm_main.c                          | 17 ++++++++++++++++-
+ 10 files changed, 58 insertions(+), 18 deletions(-)
+Merging kvms390-fixes/master (5f9df945d4e8 KVM: s390: Fix FOLL_*/FAULT_FLAG_* confusion)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git kvms390-fixes/master
+Already up to date.
+Merging hwmon-fixes/hwmon (6cde588e6493 hwmon: (DS620) Update broken Datasheet URL in driver documentation)
+$ git merge -m Merge branch 'hwmon' of https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git hwmon-fixes/hwmon
+Merge made by the 'ort' strategy.
+ Documentation/hwmon/ds620.rst  | 4 ++--
+ drivers/hwmon/dell-smm-hwmon.c | 9 +++++++++
+ drivers/hwmon/ibmpex.c         | 9 +++++++--
+ drivers/hwmon/tmp401.c         | 2 +-
+ 4 files changed, 19 insertions(+), 5 deletions(-)
+Merging nvdimm-fixes/libnvdimm-fixes (d0b3b7b22dfa Linux 6.16-rc4)
+$ git merge -m Merge branch 'libnvdimm-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git nvdimm-fixes/libnvdimm-fixes
+Already up to date.
+Merging cxl-fixes/fixes (214291cbaace acpi/hmat: Fix lockdep warning for hmem_register_resource())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git cxl-fixes/fixes
+Already up to date.
+Merging dma-mapping-fixes/dma-mapping-fixes (608bf0c7be29 parisc: Set valid bit in high byte of 64‑bit physical address)
+$ git merge -m Merge branch 'dma-mapping-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git dma-mapping-fixes/dma-mapping-fixes
+Merge made by the 'ort' strategy.
+ arch/parisc/include/asm/special_insns.h | 15 ---------------
+ drivers/parisc/sba_iommu.c              |  4 ++--
+ 2 files changed, 2 insertions(+), 17 deletions(-)
+Merging drivers-x86-fixes/fixes (4dcb6e4af15b platform/x86/uniwill: Add TUXEDO Book BA15 Gen10)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git drivers-x86-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/platform/x86/dell/alienware-wmi-wmax.c | 32 ++++++++++++++++++++++++++
+ drivers/platform/x86/uniwill/uniwill-acpi.c    |  7 ++++++
+ 2 files changed, 39 insertions(+)
+Merging samsung-krzk-fixes/fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git samsung-krzk-fixes/fixes
+Already up to date.
+Merging pinctrl-samsung-fixes/fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git pinctrl-samsung-fixes/fixes
+Already up to date.
+Merging devicetree-fixes/dt/linus (54de247a0efa dt-bindings: Updates Linus Walleij's mail address)
+$ git merge -m Merge branch 'dt/linus' of https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git devicetree-fixes/dt/linus
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/arm/arm,integrator.yaml    |  2 +-
+ .../devicetree/bindings/arm/arm,realview.yaml      |  2 +-
+ Documentation/devicetree/bindings/arm/arm,scu.yaml |  2 +-
+ .../bindings/arm/arm,versatile-sysreg.yaml         |  2 +-
+ .../devicetree/bindings/arm/arm,versatile.yaml     |  2 +-
+ .../devicetree/bindings/arm/arm,vexpress-juno.yaml |  2 +-
+ Documentation/devicetree/bindings/arm/gemini.yaml  |  2 +-
+ .../devicetree/bindings/arm/intel-ixp4xx.yaml      |  2 +-
+ Documentation/devicetree/bindings/arm/ux500.yaml   |  2 +-
+ .../devicetree/bindings/ata/ata-generic.yaml       |  2 +-
+ .../bindings/ata/cortina,gemini-sata-bridge.yaml   |  2 +-
+ .../devicetree/bindings/ata/faraday,ftide010.yaml  |  2 +-
+ .../bindings/ata/intel,ixp4xx-compact-flash.yaml   |  2 +-
+ .../devicetree/bindings/ata/pata-common.yaml       |  2 +-
+ .../devicetree/bindings/ata/sata-common.yaml       |  2 +-
+ .../bindings/auxdisplay/arm,versatile-lcd.yaml     |  2 +-
+ .../devicetree/bindings/clock/sprd,sc9860-clk.yaml | 26 ----------------------
+ .../bindings/clock/stericsson,u8500-clks.yaml      |  2 +-
+ .../bindings/crypto/intel,ixp4xx-crypto.yaml       |  2 +-
+ .../bindings/display/dsi-controller.yaml           |  2 +-
+ .../bindings/display/faraday,tve200.yaml           |  2 +-
+ .../bindings/display/panel/arm,rtsm-display.yaml   |  2 +-
+ .../display/panel/arm,versatile-tft-panel.yaml     |  2 +-
+ .../bindings/display/panel/ilitek,ili9322.yaml     |  2 +-
+ .../bindings/display/panel/novatek,nt35510.yaml    |  2 +-
+ .../bindings/display/panel/samsung,lms380kf01.yaml |  2 +-
+ .../bindings/display/panel/samsung,lms397kf04.yaml |  2 +-
+ .../bindings/display/panel/samsung,s6d16d0.yaml    |  2 +-
+ .../bindings/display/panel/sony,acx424akp.yaml     |  2 +-
+ .../bindings/display/panel/ti,nspire.yaml          |  2 +-
+ .../bindings/display/panel/tpo,tpg110.yaml         |  2 +-
+ .../devicetree/bindings/display/ste,mcde.yaml      |  2 +-
+ .../bindings/display/ti/ti,am65x-dss.yaml          |  3 +--
+ .../bindings/display/ti/ti,j721e-dss.yaml          |  3 +--
+ .../devicetree/bindings/dma/stericsson,dma40.yaml  |  2 +-
+ .../devicetree/bindings/extcon/fcs,fsa880.yaml     |  2 +-
+ .../intel,ixp4xx-network-processing-engine.yaml    |  2 +-
+ .../devicetree/bindings/gnss/brcm,bcm4751.yaml     |  2 +-
+ .../bindings/gpio/faraday,ftgpio010.yaml           |  2 +-
+ .../bindings/gpio/gpio-consumer-common.yaml        |  2 +-
+ .../devicetree/bindings/gpio/gpio-ep9301.yaml      |  2 +-
+ .../devicetree/bindings/gpio/gpio-mmio.yaml        |  2 +-
+ .../bindings/gpio/intel,ixp4xx-gpio.yaml           |  2 +-
+ .../devicetree/bindings/gpio/mrvl-gpio.yaml        |  2 +-
+ .../devicetree/bindings/gpio/pl061-gpio.yaml       |  2 +-
+ .../devicetree/bindings/gpio/st,nomadik-gpio.yaml  |  2 +-
+ .../devicetree/bindings/gpio/st,stmpe-gpio.yaml    |  2 +-
+ .../devicetree/bindings/gpu/img,powervr-rogue.yaml |  4 +++-
+ .../devicetree/bindings/hwmon/ntc-thermistor.yaml  |  2 +-
+ .../devicetree/bindings/hwmon/winbond,w83781d.yaml |  2 +-
+ .../devicetree/bindings/i2c/arm,i2c-versatile.yaml |  2 +-
+ .../devicetree/bindings/i2c/st,nomadik-i2c.yaml    |  2 +-
+ .../bindings/iio/accel/bosch,bma255.yaml           |  2 +-
+ .../bindings/iio/adc/qcom,pm8018-adc.yaml          |  2 +-
+ .../bindings/iio/gyroscope/invensense,mpu3050.yaml |  2 +-
+ .../bindings/iio/light/capella,cm3605.yaml         |  2 +-
+ .../bindings/iio/light/sharp,gp2ap002.yaml         |  2 +-
+ .../iio/magnetometer/asahi-kasei,ak8974.yaml       |  2 +-
+ .../bindings/iio/magnetometer/yamaha,yas530.yaml   |  2 +-
+ .../devicetree/bindings/iio/st,st-sensors.yaml     |  2 +-
+ .../devicetree/bindings/input/atmel,maxtouch.yaml  |  2 +-
+ .../input/touchscreen/cypress,cy8ctma140.yaml      |  2 +-
+ .../input/touchscreen/cypress,cy8ctma340.yaml      |  2 +-
+ .../bindings/input/touchscreen/melfas,mms114.yaml  |  2 +-
+ .../bindings/input/touchscreen/zinitix,bt400.yaml  |  2 +-
+ .../arm,versatile-fpga-irq.yaml                    |  2 +-
+ .../interrupt-controller/faraday,ftintc010.yaml    |  2 +-
+ .../intel,ixp4xx-interrupt.yaml                    |  2 +-
+ .../bindings/leds/backlight/kinetic,ktd253.yaml    |  2 +-
+ .../devicetree/bindings/leds/register-bit-led.yaml |  2 +-
+ .../devicetree/bindings/leds/regulator-led.yaml    |  2 +-
+ .../devicetree/bindings/leds/richtek,rt8515.yaml   |  2 +-
+ .../intel,ixp4xx-expansion-bus-controller.yaml     |  2 +-
+ .../intel,ixp4xx-expansion-peripheral-props.yaml   |  2 +-
+ .../bindings/mfd/arm,dev-platforms-syscon.yaml     |  2 +-
+ .../devicetree/bindings/mfd/st,stmpe.yaml          |  2 +-
+ .../devicetree/bindings/mfd/stericsson,ab8500.yaml |  2 +-
+ .../bindings/mfd/stericsson,db8500-prcmu.yaml      |  2 +-
+ .../misc/intel,ixp4xx-ahb-queue-manager.yaml       |  2 +-
+ .../devicetree/bindings/mmc/arm,pl18x.yaml         |  2 +-
+ .../mtd/partitions/arm,arm-firmware-suite.yaml     |  2 +-
+ .../bindings/mtd/partitions/redboot-fis.yaml       |  2 +-
+ .../devicetree/bindings/mtd/partitions/seama.yaml  |  2 +-
+ .../bindings/net/bluetooth/brcm,bluetooth.yaml     |  2 +-
+ .../bindings/net/cortina,gemini-ethernet.yaml      |  2 +-
+ .../devicetree/bindings/net/dsa/micrel,ks8995.yaml |  2 +-
+ .../devicetree/bindings/net/dsa/realtek.yaml       |  2 +-
+ .../bindings/net/dsa/vitesse,vsc73xx.yaml          |  2 +-
+ .../bindings/net/intel,ixp46x-ptp-timer.yaml       |  2 +-
+ .../bindings/net/intel,ixp4xx-ethernet.yaml        |  2 +-
+ .../devicetree/bindings/net/intel,ixp4xx-hss.yaml  |  2 +-
+ .../devicetree/bindings/pci/faraday,ftpci100.yaml  |  2 +-
+ .../devicetree/bindings/pci/intel,ixp4xx-pci.yaml  |  2 +-
+ .../devicetree/bindings/pci/v3,v360epc-pci.yaml    |  2 +-
+ .../devicetree/bindings/pinctrl/pincfg-node.yaml   |  2 +-
+ .../devicetree/bindings/pinctrl/pinctrl.yaml       |  2 +-
+ .../devicetree/bindings/pinctrl/pinmux-node.yaml   |  2 +-
+ .../bindings/power/supply/samsung,battery.yaml     |  2 +-
+ .../devicetree/bindings/rng/intel,ixp46x-rng.yaml  |  2 +-
+ .../devicetree/bindings/rtc/faraday,ftrtc010.yaml  |  2 +-
+ .../bindings/spi/arm,pl022-peripheral-props.yaml   |  2 +-
+ .../devicetree/bindings/spi/spi-pl022.yaml         |  2 +-
+ .../bindings/timer/faraday,fttmr010.yaml           |  2 +-
+ .../bindings/timer/intel,ixp4xx-timer.yaml         |  2 +-
+ .../devicetree/bindings/timer/st,nomadik-mtu.yaml  |  2 +-
+ .../devicetree/bindings/usb/faraday,fotg210.yaml   |  2 +-
+ .../devicetree/bindings/usb/intel,ixp4xx-udc.yaml  |  2 +-
+ .../bindings/watchdog/faraday,ftwdt010.yaml        |  2 +-
+ .../bindings/watchdog/maxim,max63xx.yaml           |  2 +-
+ drivers/cpufreq/cpufreq-dt-platdev.c               |  7 +++---
+ drivers/of/fdt.c                                   |  2 +-
+ 111 files changed, 115 insertions(+), 140 deletions(-)
+Merging dt-krzk-fixes/fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git dt-krzk-fixes/fixes
+Already up to date.
+Merging scsi-fixes/fixes (c9f36f04a8a2 scsi: ufs: core: Add ufshcd_update_evt_hist() for UFS suspend error)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git scsi-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/scsi/mpi3mr/mpi/mpi30_ioc.h |  1 +
+ drivers/scsi/mpi3mr/mpi3mr_fw.c     |  2 ++
+ drivers/scsi/scsi_debug.c           |  2 +-
+ drivers/scsi/sg.c                   | 20 +++++++++++++-------
+ drivers/ufs/core/ufshcd.c           |  5 ++++-
+ 5 files changed, 21 insertions(+), 9 deletions(-)
+Merging drm-fixes/drm-fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'drm-fixes' of https://gitlab.freedesktop.org/drm/kernel.git drm-fixes/drm-fixes
+Already up to date.
+Merging drm-intel-fixes/for-linux-next-fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-linux-next-fixes' of https://gitlab.freedesktop.org/drm/i915/kernel.git drm-intel-fixes/for-linux-next-fixes
+Already up to date.
+Merging mmc-fixes/fixes (d3ecb12e2e04 mmc: sdhci-esdhc-imx: add alternate ARCH_S32 dependency to Kconfig)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git mmc-fixes/fixes
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/mmc/aspeed,sdhci.yaml | 2 +-
+ drivers/mmc/host/Kconfig                                | 4 ++--
+ drivers/mmc/host/sdhci-of-arasan.c                      | 2 +-
+ 3 files changed, 4 insertions(+), 4 deletions(-)
+Merging rtc-fixes/rtc-fixes (162f24cbb0f6 rtc: rx8025: fix incorrect register reference)
+$ git merge -m Merge branch 'rtc-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git rtc-fixes/rtc-fixes
+Already up to date.
+Merging gnss-fixes/gnss-linus (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'gnss-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git gnss-fixes/gnss-linus
+Already up to date.
+Merging hyperv-fixes/hyperv-fixes (173d6f64f955 mshv: release mutex on region invalidation failure)
+$ git merge -m Merge branch 'hyperv-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git hyperv-fixes/hyperv-fixes
+Merge made by the 'ort' strategy.
+ drivers/hv/mshv_common.c    |  2 ++
+ drivers/hv/mshv_regions.c   | 20 +++++++++++---------
+ include/hyperv/hvgdk_mini.h |  7 +++++--
+ 3 files changed, 18 insertions(+), 11 deletions(-)
+Merging risc-v-fixes/fixes (e2285bd4a467 riscv: fix KUnit test_kprobes crash when building with Clang)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git risc-v-fixes/fixes
+Merge made by the 'ort' strategy.
+ arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S | 29 ++++++++++++++++++++++
+ 1 file changed, 29 insertions(+)
+Merging riscv-dt-fixes/riscv-dt-fixes (5ef13c363640 riscv: dts: microchip: remove BeagleV Fire fabric.dtsi)
+$ git merge -m Merge branch 'riscv-dt-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-dt-fixes/riscv-dt-fixes
+Auto-merging arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts
+Merge made by the 'ort' strategy.
+ .../dts/microchip/mpfs-beaglev-fire-fabric.dtsi    | 82 ----------------------
+ .../riscv/boot/dts/microchip/mpfs-beaglev-fire.dts |  5 --
+ 2 files changed, 87 deletions(-)
+ delete mode 100644 arch/riscv/boot/dts/microchip/mpfs-beaglev-fire-fabric.dtsi
+Merging riscv-soc-fixes/riscv-soc-fixes (0af2f6be1b42 Linux 6.15-rc1)
+$ git merge -m Merge branch 'riscv-soc-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-soc-fixes/riscv-soc-fixes
+Already up to date.
+Merging fpga-fixes/fixes (19272b37aa4f Linux 6.16-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git fpga-fixes/fixes
+Already up to date.
+Merging spdx/spdx-linus (d358e5254674 Merge tag 'for-6.19/dm-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm)
+$ git merge -m Merge branch 'spdx-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/spdx.git spdx/spdx-linus
+Already up to date.
+Merging gpio-brgl-fixes/gpio/for-current (a05543d6b05b gpio: it87: balance superio enter/exit calls in error path)
+$ git merge -m Merge branch 'gpio/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git gpio-brgl-fixes/gpio/for-current
+Merge made by the 'ort' strategy.
+ drivers/gpio/gpio-it87.c | 11 +++--------
+ 1 file changed, 3 insertions(+), 8 deletions(-)
+Merging gpio-intel-fixes/fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git gpio-intel-fixes/fixes
+Already up to date.
+Merging pinctrl-intel-fixes/fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git pinctrl-intel-fixes/fixes
+Already up to date.
+Merging auxdisplay-fixes/fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git auxdisplay-fixes/fixes
+Already up to date.
+Merging kunit-fixes/kunit-fixes (c33b68801fbe kunit: make FAULT_TEST default to n when PANIC_ON_OOPS)
+$ git merge -m Merge branch 'kunit-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit-fixes/kunit-fixes
+Merge made by the 'ort' strategy.
+ lib/kunit/Kconfig  | 2 +-
+ lib/kunit/device.c | 7 +++----
+ 2 files changed, 4 insertions(+), 5 deletions(-)
+Merging memblock-fixes/fixes (c42af83c59b6 memblock: fix memblock_estimated_nr_free_pages() for soft-reserved memory)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git memblock-fixes/fixes
+Already up to date.
+Merging renesas-fixes/fixes (379c590113ce ARM: shmobile: smp: Enforce shmobile_smp_* alignment)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git renesas-fixes/fixes
+Already up to date.
+Merging perf-current/perf-tools (800ad1f0e277 perf tools: Disable BPF skeleton if no libopenssl found)
+$ git merge -m Merge branch 'perf-tools' of https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools.git perf-current/perf-tools
+Merge made by the 'ort' strategy.
+ tools/build/Makefile.feature          | 6 ++++--
+ tools/build/feature/Makefile          | 8 ++++++--
+ tools/build/feature/test-all.c        | 5 +++++
+ tools/build/feature/test-libopenssl.c | 7 +++++++
+ tools/perf/Makefile.config            | 8 ++++++++
+ 5 files changed, 30 insertions(+), 4 deletions(-)
+ create mode 100644 tools/build/feature/test-libopenssl.c
+Merging efi-fixes/urgent (e5f0a698b34e Linux 6.17)
+$ git merge -m Merge branch 'urgent' of https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git efi-fixes/urgent
+Already up to date.
+Merging battery-fixes/fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git battery-fixes/fixes
+Already up to date.
+Merging iommufd-fixes/for-rc (e6a973af1113 iommufd/selftest: Check for overflow in IOMMU_TEST_OP_ADD_RESERVED)
+$ git merge -m Merge branch 'for-rc' of https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git iommufd-fixes/for-rc
+Already up to date.
+Merging rust-fixes/rust-fixes (559e608c4655 Merge tag 'ntfs3_for_6.19' of https://github.com/Paragon-Software-Group/linux-ntfs3)
+$ git merge -m Merge branch 'rust-fixes' of https://github.com/Rust-for-Linux/linux.git rust-fixes/rust-fixes
+Already up to date.
+Merging w1-fixes/fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git w1-fixes/fixes
+Already up to date.
+Merging pmdomain-fixes/fixes (c98c99d5dbdf pmdomain: tegra: Add GENPD_FLAG_NO_STAY_ON flag)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git pmdomain-fixes/fixes
+Already up to date.
+Merging i2c-host-fixes/i2c/i2c-host-fixes (880977fdc7f6 dt-bindings: i2c: qcom-cci: Document SM8750 compatible)
+$ git merge -m Merge branch 'i2c/i2c-host-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git i2c-host-fixes/i2c/i2c-host-fixes
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml        | 2 ++
+ Documentation/devicetree/bindings/i2c/snps,designware-i2c.yaml | 7 +++++++
+ Documentation/i2c/busses/i2c-i801.rst                          | 1 +
+ drivers/i2c/busses/Kconfig                                     | 1 +
+ drivers/i2c/busses/i2c-bcm-iproc.c                             | 3 +--
+ drivers/i2c/busses/i2c-i801.c                                  | 3 +++
+ drivers/i2c/busses/i2c-pxa.c                                   | 2 +-
+ drivers/i2c/busses/i2c-rcar.c                                  | 2 +-
+ 8 files changed, 17 insertions(+), 4 deletions(-)
+Merging sparc-fixes/for-linus (6613476e225e Linux 6.8-rc1)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git sparc-fixes/for-linus
+Already up to date.
+Merging clk-fixes/clk-fixes (3c493b524ffd Merge tag 'sunxi-clk-fixes-for-6.18' of https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux into clk-fixes)
+$ git merge -m Merge branch 'clk-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git clk-fixes/clk-fixes
+Already up to date.
+Merging thead-clk-fixes/thead-clk-fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'thead-clk-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-clk-fixes/thead-clk-fixes
+Already up to date.
+Merging pwrseq-fixes/pwrseq/for-current (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'pwrseq/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git pwrseq-fixes/pwrseq/for-current
+Already up to date.
+Merging thead-dt-fixes/thead-dt-fixes (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'thead-dt-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-dt-fixes/thead-dt-fixes
+Already up to date.
+Merging ftrace-fixes/ftrace/fixes (1650a1b6cb1a fgraph: Check ftrace_pids_enabled on registration for early filtering)
+$ git merge -m Merge branch 'ftrace/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git ftrace-fixes/ftrace/fixes
+Already up to date.
+Merging ring-buffer-fixes/ring-buffer/fixes (b042fdf18e89 tracing: Fix WARN_ON in tracing_buffers_mmap_close for split VMAs)
+$ git merge -m Merge branch 'ring-buffer/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git ring-buffer-fixes/ring-buffer/fixes
+Already up to date.
+Merging trace-fixes/trace/fixes (39263f986da5 ftrace: Fix address for jmp mode in t_show())
+$ git merge -m Merge branch 'trace/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git trace-fixes/trace/fixes
+Already up to date.
+Merging tracefs-fixes/tracefs/fixes (8b55572e5180 tracing/selftests: Add tracefs mount options test)
+$ git merge -m Merge branch 'tracefs/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git tracefs-fixes/tracefs/fixes
+Already up to date.
+Merging spacemit-fixes/fixes (3a8660878839 Linux 6.18-rc1)
+$ git merge -m Merge branch 'fixes' of https://github.com/spacemit-com/linux spacemit-fixes/fixes
+Already up to date.
+Merging tip-fixes/tip/urgent (a51826582e8b Merge branch into tip/master: 'x86/urgent')
+$ git merge -m Merge branch 'tip/urgent' of https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git tip-fixes/tip/urgent
+Merge made by the 'ort' strategy.
+ Documentation/arch/x86/boot.rst | 192 ++++++++++++++++++++--------------------
+ arch/x86/include/asm/uv/bios.h  |   2 +-
+ arch/x86/kernel/cpu/sgx/ioctl.c |   2 +-
+ arch/x86/kernel/fpu/xstate.c    |   4 +-
+ include/linux/mm_types.h        |   1 -
+ include/trace/events/tlb.h      |   5 +-
+ kernel/irq/manage.c             |   2 +-
+ 7 files changed, 104 insertions(+), 104 deletions(-)
+Merging slab-fixes/slab/for-next-fixes (53ca00a19d34 mm/slub: reset KASAN tag in defer_free() before accessing freed memory)
+$ git merge -m Merge branch 'slab/for-next-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git slab-fixes/slab/for-next-fixes
+Auto-merging mm/slub.c
+Merge made by the 'ort' strategy.
+ mm/slub.c | 2 ++
+ 1 file changed, 2 insertions(+)
+Merging drm-msm-fixes/msm-fixes (f5d079564c44 drm/msm/dpu: Fix adjusted mode clock check for 3d merge)
+$ git merge -m Merge branch 'msm-fixes' of https://gitlab.freedesktop.org/drm/msm.git drm-msm-fixes/msm-fixes
+Already up to date.
+Merging uml-fixes/fixes (df447a3b4a4b um: Fix FD copy size in os_rcv_fd_msg())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git uml-fixes/fixes
+Already up to date.
+Merging fwctl-fixes/for-rc (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-rc' of https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git fwctl-fixes/for-rc
+Already up to date.
+Merging devsec-tsm-fixes/fixes (fba4ceaa242d configfs-tsm-report: Fix NULL dereference of tsm_ops)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git devsec-tsm-fixes/fixes
+Already up to date.
+Merging drm-rust-fixes/drm-rust-fixes (b58c87b0fde0 gpu: nova-core: gsp: replace firmware version with "bindings" alias)
+$ git merge -m Merge branch 'drm-rust-fixes' of https://gitlab.freedesktop.org/drm/rust/kernel.git drm-rust-fixes/drm-rust-fixes
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                                       |   4 +-
+ drivers/gpu/nova-core/Kconfig                     |   2 +-
+ drivers/gpu/nova-core/gsp/cmdq.rs                 |  14 +--
+ drivers/gpu/nova-core/gsp/fw.rs                   |  78 ++++++++--------
+ drivers/gpu/nova-core/gsp/fw/r570_144.rs          |  11 ++-
+ drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs | 105 ++++++++++++----------
+ 6 files changed, 115 insertions(+), 99 deletions(-)
+Merging tenstorrent-dt-fixes/tenstorrent-dt-fixes (3a8660878839 Linux 6.18-rc1)
+$ git merge -m Merge branch 'tenstorrent-dt-fixes' of https://github.com/tenstorrent/linux.git tenstorrent-dt-fixes/tenstorrent-dt-fixes
+Already up to date.
+Merging drm-misc-fixes/for-linux-next-fixes (6b991ad8dc3a drm/imagination: Disallow exporting of PM/FW protected objects)
+$ git merge -m Merge branch 'for-linux-next-fixes' of https://gitlab.freedesktop.org/drm/misc/kernel.git drm-misc-fixes/for-linux-next-fixes
+Merge made by the 'ort' strategy.
+ drivers/accel/amdxdna/aie2_pci.c                   |   6 +
+ drivers/gpu/drm/drm_gem.c                          |   8 +-
+ drivers/gpu/drm/imagination/pvr_gem.c              |  11 ++
+ drivers/gpu/drm/panel/Kconfig                      |   1 +
+ drivers/gpu/drm/panel/panel-sony-td4353-jdi.c      |   2 +
+ drivers/gpu/drm/tests/drm_atomic_state_test.c      |  40 +++++-
+ drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c | 143 +++++++++++++++++++++
+ 7 files changed, 204 insertions(+), 7 deletions(-)
+Merging rust/rust-next (559e608c4655 Merge tag 'ntfs3_for_6.19' of https://github.com/Paragon-Software-Group/linux-ntfs3)
+$ git merge -m Merge branch 'rust-next' of https://github.com/Rust-for-Linux/linux.git rust/rust-next
+Already up to date.
+Merging rust-alloc/alloc-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'alloc-next' of https://github.com/Rust-for-Linux/linux.git rust-alloc/alloc-next
+Already up to date.
+Merging rust-io/io-next (86731a2a651e Linux 6.16-rc3)
+$ git merge -m Merge branch 'io-next' of https://github.com/Rust-for-Linux/linux.git rust-io/io-next
+Already up to date.
+Merging rust-pin-init/pin-init-next (42415d163e5d rust: pin-init: add references to previously initialized fields)
+$ git merge -m Merge branch 'pin-init-next' of https://github.com/Rust-for-Linux/linux.git rust-pin-init/pin-init-next
+Already up to date.
+Merging rust-timekeeping/timekeeping-next (4521438fb076 rust: time: Implement basic arithmetic operations for Delta)
+$ git merge -m Merge branch 'timekeeping-next' of https://github.com/Rust-for-Linux/linux.git rust-timekeeping/timekeeping-next
+Already up to date.
+Merging rust-xarray/xarray-next (fa616196fbea MAINTAINERS: add entry for Rust XArray API)
+$ git merge -m Merge branch 'xarray-next' of https://github.com/Rust-for-Linux/linux.git rust-xarray/xarray-next
+Already up to date.
+Merging mm-stable/mm-stable (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'mm-stable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-stable/mm-stable
+Already up to date.
+Merging mm-nonmm-stable/mm-nonmm-stable (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'mm-nonmm-stable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-nonmm-stable/mm-nonmm-stable
+Already up to date.
+Merging mm-unstable/mm-unstable (edeb51ec1667 mm/hugetlb: fix excessive IPI broadcasts when unsharing PMD tables using mmu_gather)
+$ git merge -m Merge branch 'mm-unstable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-unstable/mm-unstable
+Auto-merging arch/arm64/mm/mmu.c
+Auto-merging include/linux/mm_types.h
+Auto-merging mm/shmem.c
+Merge made by the 'ort' strategy.
+ arch/arm64/Kconfig                                 |   1 +
+ arch/arm64/include/asm/pgtable.h                   |  41 +-
+ arch/arm64/include/asm/thread_info.h               |   3 +-
+ arch/arm64/mm/mmu.c                                |   8 +-
+ arch/arm64/mm/pageattr.c                           |   4 +-
+ arch/powerpc/include/asm/book3s/64/tlbflush-hash.h |  20 +-
+ arch/powerpc/include/asm/thread_info.h             |   2 -
+ arch/powerpc/kernel/process.c                      |  25 -
+ arch/powerpc/mm/book3s64/hash_tlb.c                |  14 +-
+ arch/powerpc/mm/book3s64/subpage_prot.c            |   4 +-
+ arch/powerpc/platforms/Kconfig.cputype             |   1 +
+ arch/sparc/Kconfig                                 |   1 +
+ arch/sparc/include/asm/tlbflush_64.h               |   5 +-
+ arch/sparc/mm/tlb.c                                |  14 +-
+ arch/x86/Kconfig                                   |   1 +
+ arch/x86/boot/compressed/misc.h                    |   1 +
+ arch/x86/boot/startup/sme.c                        |   1 +
+ arch/x86/include/asm/paravirt.h                    |   1 -
+ arch/x86/include/asm/pgtable.h                     |   1 +
+ arch/x86/include/asm/thread_info.h                 |   4 +-
+ arch/x86/xen/enlighten_pv.c                        |   3 +-
+ arch/x86/xen/mmu_pv.c                              |   6 +-
+ fs/proc/base.c                                     |   2 +-
+ fs/proc/task_mmu.c                                 |   4 +-
+ include/asm-generic/tlb.h                          |  74 ++-
+ include/linux/hugetlb.h                            |  21 +-
+ include/linux/mm.h                                 |  58 +-
+ include/linux/mm_types.h                           |   4 +-
+ include/linux/mm_types_task.h                      |   5 +
+ include/linux/oom.h                                |  12 +-
+ include/linux/percpu_counter_tree.h                | 242 +++++++
+ include/linux/pgtable.h                            | 158 ++++-
+ include/linux/sched.h                              |  45 ++
+ include/trace/events/huge_memory.h                 |   3 +-
+ include/trace/events/kmem.h                        |   2 +-
+ init/main.c                                        |   2 +
+ kernel/fork.c                                      |  24 +-
+ lib/Makefile                                       |   1 +
+ lib/alloc_tag.c                                    |   5 +-
+ lib/percpu_counter_tree.c                          | 705 +++++++++++++++++++++
+ mm/Kconfig                                         |  19 +
+ mm/Makefile                                        |   1 +
+ mm/damon/core.c                                    |  20 +-
+ mm/hugetlb.c                                       | 129 ++--
+ mm/kasan/shadow.c                                  |   8 +-
+ mm/khugepaged.c                                    |  48 +-
+ mm/madvise.c                                       |  18 +-
+ mm/memory.c                                        |  16 +-
+ mm/migrate_device.c                                |   8 +-
+ mm/mmu_gather.c                                    |   7 +
+ mm/mprotect.c                                      |   6 +-
+ mm/mremap.c                                        |   4 +-
+ mm/oom_kill.c                                      |  72 ++-
+ mm/rmap.c                                          |  45 +-
+ mm/shmem.c                                         |  12 +
+ mm/tests/lazy_mmu_mode_kunit.c                     |  74 +++
+ mm/userfaultfd.c                                   |   4 +-
+ mm/vmalloc.c                                       |  62 +-
+ mm/vmscan.c                                        | 110 ++--
+ 59 files changed, 1825 insertions(+), 366 deletions(-)
+ create mode 100644 include/linux/percpu_counter_tree.h
+ create mode 100644 lib/percpu_counter_tree.c
+ create mode 100644 mm/tests/lazy_mmu_mode_kunit.c
+Merging mm-nonmm-unstable/mm-nonmm-unstable (bda39f9485fb fork-comment-fix: remove ambiguous question mark in CLONE_CHILD_CLEARTID comment)
+$ git merge -m Merge branch 'mm-nonmm-unstable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-nonmm-unstable/mm-nonmm-unstable
+Auto-merging MAINTAINERS
+Auto-merging arch/arm64/net/bpf_jit_comp.c
+Auto-merging drivers/ufs/core/ufshcd.c
+Auto-merging kernel/bpf/core.c
+Auto-merging kernel/fork.c
+Auto-merging kernel/trace/ftrace.c
+Auto-merging lib/Makefile
+Merge made by the 'ort' strategy.
+ .editorconfig                                      |   2 -
+ Documentation/admin-guide/kernel-parameters.txt    |  10 +-
+ Documentation/admin-guide/sysctl/kernel.rst        |   2 +-
+ Documentation/core-api/liveupdate.rst              |  11 +
+ MAINTAINERS                                        |   3 +-
+ arch/arm/configs/aspeed_g5_defconfig               |   2 +-
+ arch/arm/configs/pxa3xx_defconfig                  |   2 +-
+ arch/arm/include/asm/syscall.h                     |   2 -
+ arch/arm64/include/asm/syscall.h                   |  18 +-
+ arch/arm64/net/bpf_jit_comp.c                      |   2 +-
+ arch/m68k/configs/amiga_defconfig                  |   1 -
+ arch/m68k/configs/apollo_defconfig                 |   1 -
+ arch/m68k/configs/atari_defconfig                  |   1 -
+ arch/m68k/configs/bvme6000_defconfig               |   1 -
+ arch/m68k/configs/hp300_defconfig                  |   1 -
+ arch/m68k/configs/mac_defconfig                    |   1 -
+ arch/m68k/configs/multi_defconfig                  |   1 -
+ arch/m68k/configs/mvme147_defconfig                |   1 -
+ arch/m68k/configs/mvme16x_defconfig                |   1 -
+ arch/m68k/configs/q40_defconfig                    |   1 -
+ arch/m68k/configs/sun3_defconfig                   |   1 -
+ arch/m68k/configs/sun3x_defconfig                  |   1 -
+ arch/mips/kernel/setup.c                           |   1 +
+ arch/mips/rb532/devices.c                          |   1 +
+ arch/openrisc/configs/or1klitex_defconfig          |   2 +-
+ arch/powerpc/configs/ppc64_defconfig               |   1 -
+ arch/powerpc/configs/skiroot_defconfig             |   2 +-
+ arch/powerpc/kernel/btext.c                        |   1 +
+ arch/powerpc/net/bpf_jit_comp.c                    |   2 +-
+ arch/s390/kernel/alternative.c                     |   1 +
+ arch/s390/kernel/stackprotector.c                  |   1 +
+ arch/um/drivers/vector_kern.c                      |   1 +
+ arch/xtensa/include/asm/syscall.h                  |   1 -
+ arch/xtensa/platforms/iss/network.c                |   1 +
+ certs/blacklist.c                                  |   1 +
+ crypto/asymmetric_keys/asymmetric_type.c           |   1 +
+ crypto/asymmetric_keys/x509_public_key.c           |   1 +
+ crypto/krb5/selftest.c                             |   1 +
+ drivers/atm/nicstar.c                              |   1 +
+ drivers/auxdisplay/hd44780_common.c                |   1 +
+ drivers/auxdisplay/lcd2s.c                         |   1 +
+ drivers/block/floppy.c                             |   2 -
+ drivers/bus/moxtet.c                               |   1 +
+ drivers/char/tpm/tpm.h                             |   1 +
+ drivers/comedi/drivers/jr3_pci.c                   |   1 +
+ drivers/firmware/broadcom/bcm47xx_sprom.c          |   1 +
+ drivers/gpio/gpio-macsmc.c                         |   1 +
+ drivers/gpu/drm/ci/arm.config                      |   2 +-
+ drivers/gpu/drm/ci/arm64.config                    |   2 +-
+ drivers/gpu/drm/ci/x86_64.config                   |   2 +-
+ drivers/hid/hid-picolcd_debugfs.c                  |   1 +
+ drivers/hwmon/pmbus/q54sj108a2.c                   |   1 +
+ drivers/hwmon/pmbus/ucd9000.c                      |   1 +
+ drivers/infiniband/ulp/srp/ib_srp.c                |   1 +
+ drivers/infiniband/ulp/srpt/ib_srpt.c              |   1 +
+ drivers/input/touchscreen/iqs5xx.c                 |   1 +
+ drivers/md/dm-crypt.c                              |   1 +
+ drivers/md/dm-integrity.c                          |   1 +
+ drivers/md/dm-verity-target.c                      |   1 +
+ .../extron-da-hd-4k-plus/extron-da-hd-4k-plus.c    |   1 +
+ drivers/media/cec/usb/rainshadow/rainshadow-cec.c  |   1 +
+ drivers/media/i2c/ccs/ccs-reg-access.c             |   1 +
+ drivers/media/usb/pvrusb2/pvrusb2-debugifc.c       |   1 +
+ drivers/misc/kgdbts.c                              |   1 +
+ drivers/misc/pch_phub.c                            |   1 +
+ drivers/net/bonding/bond_options.c                 |   1 +
+ drivers/net/can/can327.c                           |   1 +
+ drivers/net/can/slcan/slcan-core.c                 |   1 +
+ drivers/net/ethernet/chelsio/cxgb3/common.h        |   1 +
+ .../mellanox/mlx5/core/steering/sws/dr_dbg.c       |   1 +
+ drivers/net/ethernet/micrel/ksz884x.c              |   1 +
+ drivers/net/ethernet/pasemi/pasemi_mac.c           |   1 +
+ drivers/net/netconsole.c                           |   1 +
+ drivers/net/netdevsim/dev.c                        |   1 +
+ drivers/net/usb/r8152.c                            |   1 +
+ drivers/net/usb/usbnet.c                           |   1 +
+ drivers/net/wireless/ath/ath6kl/debug.c            |   1 +
+ drivers/net/wireless/intel/iwlwifi/fw/debugfs.c    |   1 +
+ drivers/net/wireless/intel/iwlwifi/mld/debugfs.c   |   1 +
+ drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c   |   1 +
+ drivers/net/wireless/intel/iwlwifi/mvm/mvm.h       |   1 +
+ drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h |   1 +
+ drivers/net/wireless/realtek/rtw89/debug.c         |   1 +
+ drivers/net/wireless/silabs/wfx/fwio.c             |   1 +
+ drivers/nvme/target/configfs.c                     |   1 +
+ drivers/nvme/target/core.c                         |   1 +
+ drivers/nvmem/brcm_nvram.c                         |   1 +
+ drivers/nvmem/layouts/u-boot-env.c                 |   1 +
+ drivers/platform/x86/intel/wmi/thunderbolt.c       |   1 +
+ drivers/pnp/support.c                              |   1 +
+ drivers/ptp/ptp_pch.c                              |   1 +
+ drivers/rapidio/rio-scan.c                         |   2 +-
+ drivers/s390/cio/blacklist.c                       |   1 +
+ drivers/s390/crypto/ap_bus.c                       |   1 +
+ drivers/s390/crypto/zcrypt_cex4.c                  |   1 +
+ drivers/s390/virtio/virtio_ccw.c                   |   1 +
+ drivers/scsi/aacraid/rx.c                          |   1 +
+ drivers/scsi/ips.c                                 |   1 +
+ drivers/scsi/libsas/sas_scsi_host.c                |   1 +
+ drivers/scsi/qla2xxx/tcm_qla2xxx.c                 |   1 +
+ drivers/scsi/scsi_transport_fc.c                   |   1 +
+ drivers/staging/rtl8723bs/core/rtw_ieee80211.c     |   1 +
+ drivers/target/iscsi/iscsi_target_auth.c           |   1 +
+ drivers/target/target_core_fabric_lib.c            |   1 +
+ drivers/target/target_core_spc.c                   |   1 +
+ drivers/target/tcm_fc/tfc_conf.c                   |   1 +
+ drivers/thunderbolt/switch.c                       |   1 +
+ drivers/tty/vt/vt.c                                |   1 +
+ drivers/ufs/core/ufshcd.c                          |   1 +
+ drivers/usb/atm/speedtch.c                         |   1 +
+ drivers/usb/atm/ueagle-atm.c                       |   1 +
+ drivers/usb/gadget/function/u_ether.c              |   1 +
+ drivers/usb/gadget/function/uvc_configfs.c         |   1 +
+ drivers/usb/typec/ucsi/debugfs.c                   |   1 +
+ drivers/usb/typec/ucsi/ucsi_ccg.c                  |   1 +
+ drivers/watchdog/hpwdt.c                           |   1 +
+ fs/adfs/dir.c                                      |   1 +
+ fs/binfmt_misc.c                                   |   1 +
+ fs/ecryptfs/ecryptfs_kernel.h                      |   1 +
+ fs/efivarfs/vars.c                                 |   1 +
+ fs/fat/cache.c                                     |   4 +-
+ fs/fat/dir.c                                       |   1 +
+ fs/fat/namei_vfat.c                                |   1 +
+ fs/gfs2/lock_dlm.c                                 |   1 +
+ fs/nfsd/nfs4recover.c                              |   1 +
+ fs/ntfs3/ntfs_fs.h                                 |   1 +
+ fs/ocfs2/cluster/heartbeat.c                       |   4 +-
+ fs/ocfs2/cluster/nodemanager.c                     |   8 +-
+ fs/ocfs2/export.c                                  |   6 +-
+ fs/ocfs2/inode.c                                   |  32 +-
+ fs/ocfs2/slot_map.c                                |  27 +-
+ fs/ocfs2/suballoc.c                                | 334 ++++++++++-
+ fs/ocfs2/xattr.c                                   |   5 +-
+ fs/overlayfs/namei.c                               |   1 +
+ fs/proc/array.c                                    |   1 +
+ fs/seq_file.c                                      |   1 +
+ fs/udf/unicode.c                                   |   1 +
+ include/asm-generic/atomic64.h                     |   2 +-
+ include/asm-generic/rqspinlock.h                   |   2 +-
+ include/linux/array_size.h                         |   6 +
+ include/linux/filter.h                             |  26 +-
+ include/linux/ftrace.h                             |   6 +-
+ include/linux/instrumented.h                       |  15 +
+ include/linux/kernel.h                             |   1 -
+ include/linux/kho/abi/luo.h                        |  81 +++
+ include/linux/liveupdate.h                         | 143 +++++
+ include/linux/module.h                             |   9 +
+ include/linux/ns/ns_common_types.h                 |   4 +-
+ include/linux/types.h                              |   2 +-
+ kernel/audit.c                                     |   1 +
+ kernel/bpf/core.c                                  |   5 +-
+ kernel/bpf/rqspinlock.c                            |   1 -
+ kernel/bpf/syscall.c                               |   1 +
+ kernel/crash_core.c                                |  17 +-
+ kernel/crash_dump_dm_crypt.c                       |   4 +-
+ kernel/debug/gdbstub.c                             |   1 +
+ kernel/fork.c                                      |   2 +-
+ kernel/hung_task.c                                 |  66 ++-
+ kernel/kallsyms.c                                  |  73 ++-
+ kernel/kcsan/kcsan_test.c                          |   4 +-
+ kernel/liveupdate/Makefile                         |   1 +
+ kernel/liveupdate/luo_core.c                       |   7 +-
+ kernel/liveupdate/luo_file.c                       |  32 +-
+ kernel/liveupdate/luo_flb.c                        | 656 +++++++++++++++++++++
+ kernel/liveupdate/luo_internal.h                   |  15 +
+ kernel/module/kallsyms.c                           |   9 +-
+ kernel/pid.c                                       | 134 +++--
+ kernel/trace/ftrace.c                              |   5 +-
+ kernel/vmcore_info.c                               |   6 +-
+ kernel/watchdog.c                                  |  24 +-
+ lib/Kconfig.debug                                  |  68 ++-
+ lib/Makefile                                       |   1 -
+ lib/build_OID_registry                             |  26 +-
+ lib/hexdump.c                                      |   1 +
+ lib/string_helpers.c                               |   1 +
+ lib/test_uuid.c                                    | 134 -----
+ lib/tests/Makefile                                 |   2 +
+ lib/tests/liveupdate.c                             | 158 +++++
+ lib/tests/uuid_kunit.c                             | 106 ++++
+ lib/uuid.c                                         |   1 +
+ lib/vsprintf.c                                     |   1 +
+ mm/kfence/kfence_test.c                            |   4 +-
+ mm/kmemleak.c                                      |   2 +-
+ mm/kmsan/kmsan_test.c                              |   2 +-
+ mm/memcontrol-v1.c                                 |   4 +-
+ net/bridge/br_sysfs_br.c                           |   1 +
+ net/core/pktgen.c                                  |   1 +
+ net/core/utils.c                                   |   1 +
+ net/ipv4/arp.c                                     |   1 +
+ net/mac80211/debugfs_netdev.c                      |   1 +
+ net/sunrpc/cache.c                                 |   1 +
+ net/tipc/core.h                                    |   1 +
+ security/integrity/evm/evm_crypto.c                |   1 +
+ security/integrity/ima/ima_api.c                   |   1 +
+ security/ipe/digest.c                              |   1 +
+ security/keys/encrypted-keys/encrypted.c           |   1 +
+ security/keys/trusted-keys/trusted_core.c          |   1 +
+ security/keys/trusted-keys/trusted_tpm1.c          |   1 +
+ security/loadpin/loadpin.c                         |   1 +
+ security/selinux/selinuxfs.c                       |   1 +
+ sound/pci/riptide/riptide.c                        |   1 +
+ sound/usb/6fire/firmware.c                         |   1 +
+ tools/testing/selftests/bpf/config                 |   2 +-
+ .../testing/selftests/wireguard/qemu/kernel.config |   2 +-
+ 204 files changed, 2109 insertions(+), 385 deletions(-)
+ create mode 100644 kernel/liveupdate/luo_flb.c
+ delete mode 100644 lib/test_uuid.c
+ create mode 100644 lib/tests/liveupdate.c
+ create mode 100644 lib/tests/uuid_kunit.c
+$ git am -3 ../patches/0001-Revert-atomic-add-option-for-weaker-alignment-check.patch
+Applying: Revert "atomic: add option for weaker alignment check"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	include/linux/instrumented.h
+M	lib/Kconfig.debug
+$ git add -A .
+$ git commit -v -a --amend
+[master c0a917988d81] Merge branch 'mm-nonmm-unstable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
+ Date: Fri Dec 19 10:21:57 2025 +1100
+$ git am -3 ../patches/0002-Revert-atomic-add-alignment-check-to-instrumented-at.patch
+Applying: Revert "atomic: add alignment check to instrumented atomic operations"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	include/linux/instrumented.h
+M	lib/Kconfig.debug
+$ git add -A .
+$ git commit -v -a --amend
+[master 851cceece6df] Merge branch 'mm-nonmm-unstable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
+ Date: Fri Dec 19 10:21:57 2025 +1100
+Merging kbuild/kbuild-next (a4df2071f1a2 modpost: drop '*_probe' from section check whitelist)
+$ git merge -m Merge branch 'kbuild-next' of https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git kbuild/kbuild-next
+Merge made by the 'ort' strategy.
+ scripts/mod/modpost.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging clang-format/clang-format (c147f663b6a5 clang-format: Update with v6.11-rc1's `for_each` macro list)
+$ git merge -m Merge branch 'clang-format' of https://github.com/ojeda/linux.git clang-format/clang-format
+Already up to date.
+Merging perf/perf-tools-next (cb015814f8b6 Merge tag 'f2fs-for-6.19-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs)
+$ git merge -m Merge branch 'perf-tools-next' of https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git perf/perf-tools-next
+Already up to date.
+Merging compiler-attributes/compiler-attributes (98f7e32f20d2 Linux 6.11)
+$ git merge -m Merge branch 'compiler-attributes' of https://github.com/ojeda/linux.git compiler-attributes/compiler-attributes
+Already up to date.
+Merging dma-mapping/dma-mapping-for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'dma-mapping-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git dma-mapping/dma-mapping-for-next
+Already up to date.
+Merging asm-generic/master (edcc8a38b5ac once: fix race by moving DO_ONCE to separate section)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic asm-generic/master
+Already up to date.
+Merging alpha/alpha-next (d58041d2c63e MAINTAINERS: Add Magnus Lindholm as maintainer for alpha port)
+$ git merge -m Merge branch 'alpha-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mattst88/alpha.git alpha/alpha-next
+Already up to date.
+Merging arm/for-next (dd9143371a86 Merge branches 'fixes' and 'misc' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git arm/for-next
+Already up to date.
+Merging arm64/for-next/core (edde060637b9 Merge branch 'for-next/set_memory' into for-next/core)
+$ git merge -m Merge branch 'for-next/core' of https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux arm64/for-next/core
+Already up to date.
+Merging arm-perf/for-next/perf (e6a27290d800 perf: arm_spe: Add support for filtering on data source)
+$ git merge -m Merge branch 'for-next/perf' of https://git.kernel.org/pub/scm/linux/kernel/git/will/linux.git arm-perf/for-next/perf
+Already up to date.
+Merging arm-soc/for-next (e5b2299997d3 soc: document merges)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git arm-soc/for-next
+Merge made by the 'ort' strategy.
+ arch/arm/arm-soc-for-next-contents.txt | 177 +++++++++++++++++++++++++++++++++
+ 1 file changed, 177 insertions(+)
+ create mode 100644 arch/arm/arm-soc-for-next-contents.txt
+Merging amlogic/for-next (3b2ecf0c3cc3 Merge branch 'v6.19/drivers' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux.git amlogic/for-next
+Merge made by the 'ort' strategy.
+Merging asahi-soc/asahi-soc/for-next (074986379c03 Merge branch 'apple-soc/dt-6.20' into asahi-soc/for-next)
+$ git merge -m Merge branch 'asahi-soc/for-next' of https://github.com/AsahiLinux/linux.git asahi-soc/asahi-soc/for-next
+Merge made by the 'ort' strategy.
+ arch/arm64/boot/dts/apple/t600x-die0.dtsi | 6 ++++++
+ arch/arm64/boot/dts/apple/t602x-die0.dtsi | 6 ++++++
+ arch/arm64/boot/dts/apple/t8103.dtsi      | 6 ++++++
+ arch/arm64/boot/dts/apple/t8112.dtsi      | 6 ++++++
+ 4 files changed, 24 insertions(+)
+Merging at91/at91-next (fde90d654751 Merge branch 'at91-dt' into at91-next)
+$ git merge -m Merge branch 'at91-next' of https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git at91/at91-next
+Merge made by the 'ort' strategy.
+Merging bmc/for-next (459a5aa171c0 ARM: dts: aspeed: bletchley: Fix ADC vref property names)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/bmc/linux.git bmc/for-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/arm/aspeed/aspeed.yaml     |   1 +
+ arch/arm/boot/dts/aspeed/Makefile                  |   1 +
+ arch/arm/boot/dts/aspeed/aspeed-ast2600-evb.dts    |   7 +-
+ .../dts/aspeed/aspeed-bmc-facebook-bletchley.dts   | 106 ++++-----
+ .../dts/aspeed/aspeed-bmc-facebook-clemente.dts    |  43 +++-
+ .../boot/dts/aspeed/aspeed-bmc-facebook-harma.dts  |   8 +-
+ .../aspeed/aspeed-bmc-facebook-santabarbara.dts    |  29 ++-
+ .../boot/dts/aspeed/aspeed-bmc-nvidia-msx4-bmc.dts | 246 +++++++++++++++++++++
+ arch/arm/boot/dts/aspeed/aspeed-g6.dtsi            |  12 +-
+ 9 files changed, 374 insertions(+), 79 deletions(-)
+ create mode 100644 arch/arm/boot/dts/aspeed/aspeed-bmc-nvidia-msx4-bmc.dts
+Merging broadcom/next (37c3a91e9730 arm64: dts: broadcom: bcm2712: Add watchdog DT node)
+$ git merge -m Merge branch 'next' of https://github.com/Broadcom/stblinux.git broadcom/next
+Auto-merging arch/arm64/boot/dts/broadcom/bcm2712.dtsi
+Merge made by the 'ort' strategy.
+ arch/arm64/boot/dts/broadcom/bcm2712.dtsi | 15 +++++++++++++++
+ 1 file changed, 15 insertions(+)
+Merging cix/for-next (4889f68b824d Merge remote-tracking branch 'cix/dt' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/peter.chen/cix.git cix/for-next
+Merge made by the 'ort' strategy.
+Merging davinci/davinci/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'davinci/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git davinci/davinci/for-next
+Already up to date.
+Merging drivers-memory/for-next (9805f2cfc883 dt-bindings: memory: SDRAM channel: standardise node name)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git drivers-memory/for-next
+Merge made by the 'ort' strategy.
+ .../memory-controllers/ddr/jedec,ddr4.yaml         | 34 ++++++++
+ .../memory-controllers/ddr/jedec,lpddr-props.yaml  | 74 -----------------
+ .../memory-controllers/ddr/jedec,lpddr2.yaml       |  2 +-
+ .../memory-controllers/ddr/jedec,lpddr3.yaml       |  2 +-
+ .../memory-controllers/ddr/jedec,lpddr4.yaml       |  2 +-
+ .../memory-controllers/ddr/jedec,lpddr5.yaml       |  2 +-
+ ...lpddr-channel.yaml => jedec,sdram-channel.yaml} | 40 ++++++---
+ .../memory-controllers/ddr/jedec,sdram-props.yaml  | 94 ++++++++++++++++++++++
+ drivers/memory/mtk-smi.c                           | 38 +++++----
+ 9 files changed, 181 insertions(+), 107 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/memory-controllers/ddr/jedec,ddr4.yaml
+ delete mode 100644 Documentation/devicetree/bindings/memory-controllers/ddr/jedec,lpddr-props.yaml
+ rename Documentation/devicetree/bindings/memory-controllers/ddr/{jedec,lpddr-channel.yaml => jedec,sdram-channel.yaml} (76%)
+ create mode 100644 Documentation/devicetree/bindings/memory-controllers/ddr/jedec,sdram-props.yaml
+Merging fsl/soc_fsl (c8dff80a3108 bus: fsl-mc: Convert to bus callbacks)
+$ git merge -m Merge branch 'soc_fsl' of https://git.kernel.org/pub/scm/linux/kernel/git/chleroy/linux.git fsl/soc_fsl
+Merge made by the 'ort' strategy.
+ drivers/bus/fsl-mc/fsl-mc-bus.c | 77 +++++++++++++++++------------------------
+ 1 file changed, 32 insertions(+), 45 deletions(-)
+Merging imx-mxs/for-next (89d95dbad2a3 Merge branch 'imx/defconfig' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux.git imx-mxs/for-next
+Merge made by the 'ort' strategy.
+Merging mediatek/for-next (67ed5843a67b Merge branch 'v6.18-next/dts64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux.git mediatek/for-next
+Merge made by the 'ort' strategy.
+Merging mvebu/for-next (f0e6bc0c3ef4 clk: mvebu: cp110 add CLK_IGNORE_UNUSED to pcie_x10, pcie_x11 & pcie_x4)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu.git mvebu/for-next
+Already up to date.
+Merging omap/for-next (2c85858e7c68 Merge branch 'omap-for-v6.19/drivers' into tmp/omap-next-20251117.085514)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git omap/for-next
+Merge made by the 'ort' strategy.
+Merging qcom/for-next (689a433e9d6b Merge branches 'arm32-fixes-for-6.19', 'arm64-defconfig-for-6.20', 'arm64-fixes-for-6.19', 'arm64-for-6.20', 'clk-fixes-for-6.19', 'clk-for-6.20' and 'drivers-for-6.20' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git qcom/for-next
+Auto-merging .mailmap
+Merge made by the 'ort' strategy.
+ .mailmap                                           |    1 +
+ Documentation/devicetree/bindings/arm/qcom.yaml    |   13 +
+ .../devicetree/bindings/cache/qcom,llcc.yaml       |   47 +-
+ .../bindings/interrupt-controller/qcom,pdc.yaml    |    1 +
+ Documentation/devicetree/bindings/sram/sram.yaml   |    2 +
+ .../devicetree/bindings/vendor-prefixes.yaml       |    4 +
+ .../arm/boot/dts/qcom/qcom-apq8074-dragonboard.dts |   13 -
+ .../qcom/qcom-msm8974-lge-nexus5-hammerhead.dts    |   12 -
+ .../boot/dts/qcom/qcom-msm8974-samsung-hlte.dts    |   12 -
+ .../dts/qcom/qcom-msm8974-sony-xperia-rhine.dtsi   |   12 -
+ arch/arm/boot/dts/qcom/qcom-msm8974.dtsi           |   50 +-
+ .../dts/qcom/qcom-msm8974pro-fairphone-fp2.dts     |    8 -
+ arch/arm/boot/dts/qcom/qcom-msm8974pro-htc-m8.dts  |   11 -
+ .../dts/qcom/qcom-msm8974pro-oneplus-bacon.dts     |    9 -
+ .../qcom/qcom-msm8974pro-samsung-klte-common.dtsi  |   11 +-
+ ...qcom-msm8974pro-sony-xperia-shinano-common.dtsi |   12 -
+ arch/arm64/boot/dts/qcom/Makefile                  |    4 +
+ arch/arm64/boot/dts/qcom/glymur-ipcc.h             |   68 +
+ arch/arm64/boot/dts/qcom/hamoa.dtsi                |   26 +
+ arch/arm64/boot/dts/qcom/kaanapali-ipcc.h          |   58 +
+ arch/arm64/boot/dts/qcom/kaanapali-mtp.dts         |  754 +++++
+ arch/arm64/boot/dts/qcom/kaanapali-qrd.dts         |  712 ++++
+ arch/arm64/boot/dts/qcom/kaanapali.dtsi            | 1606 +++++++++
+ arch/arm64/boot/dts/qcom/sc8280xp.dtsi             |   16 +-
+ .../boot/dts/qcom/sdm845-oneplus-enchilada.dts     |    4 +-
+ arch/arm64/boot/dts/qcom/sm8550.dtsi               |    2 -
+ arch/arm64/boot/dts/qcom/sm8650.dtsi               |    3 -
+ arch/arm64/boot/dts/qcom/talos.dtsi                |    4 +-
+ .../dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts    | 1515 +++++++++
+ arch/arm64/configs/defconfig                       |    4 +
+ drivers/clk/qcom/Kconfig                           |   17 +
+ drivers/clk/qcom/Makefile                          |    2 +
+ drivers/clk/qcom/clk-rpmh.c                        |   60 +-
+ drivers/clk/qcom/gcc-glymur.c                      |    4 +-
+ drivers/clk/qcom/gcc-kaanapali.c                   | 3541 ++++++++++++++++++++
+ drivers/clk/qcom/gcc-milos.c                       |    6 +-
+ drivers/clk/qcom/gcc-qdu1000.c                     |    4 +-
+ drivers/clk/qcom/gcc-sdx75.c                       |    4 +-
+ drivers/clk/qcom/gcc-sm4450.c                      |    6 +-
+ drivers/clk/qcom/gcc-sm8450.c                      |    4 +-
+ drivers/clk/qcom/gcc-sm8750.c                      |    4 +-
+ drivers/clk/qcom/gcc-x1e80100.c                    |    4 +-
+ drivers/clk/qcom/tcsrcc-kaanapali.c                |  141 +
+ drivers/firmware/qcom/qcom_scm.c                   |    1 +
+ drivers/pmdomain/qcom/rpmhpd.c                     |    4 +
+ drivers/soc/qcom/llcc-qcom.c                       |  207 ++
+ drivers/soc/qcom/smem.c                            |    4 +-
+ include/dt-bindings/power/qcom,rpmhpd.h            |    1 +
+ include/linux/soc/qcom/llcc-qcom.h                 |    4 +
+ 49 files changed, 8869 insertions(+), 143 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/qcom/glymur-ipcc.h
+ create mode 100644 arch/arm64/boot/dts/qcom/kaanapali-ipcc.h
+ create mode 100644 arch/arm64/boot/dts/qcom/kaanapali-mtp.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/kaanapali-qrd.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/kaanapali.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts
+ create mode 100644 drivers/clk/qcom/gcc-kaanapali.c
+ create mode 100644 drivers/clk/qcom/tcsrcc-kaanapali.c
+Merging renesas/next (fec0b9c25cfb Merge branches 'renesas-arm-defconfig-for-v6.20' and 'renesas-dts-for-v6.20' into renesas-next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git renesas/next
+Auto-merging arch/arm64/configs/defconfig
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/renesas/r9a06g032.dtsi           |   6 +
+ arch/arm64/boot/dts/renesas/r9a08g045.dtsi         |  65 ++++
+ arch/arm64/boot/dts/renesas/r9a09g047.dtsi         |  30 ++
+ arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts |  11 +-
+ arch/arm64/boot/dts/renesas/r9a09g056.dtsi         | 350 +++++++++++++++++++++
+ .../boot/dts/renesas/r9a09g056n48-rzv2n-evk.dts    |  28 ++
+ arch/arm64/boot/dts/renesas/r9a09g057.dtsi         |  60 ++++
+ .../boot/dts/renesas/r9a09g057h44-rzv2h-evk.dts    |  30 ++
+ arch/arm64/boot/dts/renesas/r9a09g077.dtsi         |  72 +++++
+ .../boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts    |   3 +
+ arch/arm64/boot/dts/renesas/r9a09g087.dtsi         |  72 +++++
+ .../boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts    |   3 +
+ arch/arm64/boot/dts/renesas/renesas-smarc2.dtsi    |   8 +
+ arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi   |   5 +
+ arch/arm64/boot/dts/renesas/rzg3s-smarc.dtsi       |  11 +
+ arch/arm64/configs/defconfig                       |   2 +
+ 16 files changed, 755 insertions(+), 1 deletion(-)
+Merging reset/reset/next (a0d57f7504b6 reset: gpio: add the "compatible" property)
+$ git merge -m Merge branch 'reset/next' of https://git.pengutronix.de/git/pza/linux reset/reset/next
+Merge made by the 'ort' strategy.
+ drivers/reset/core.c | 7 ++++---
+ 1 file changed, 4 insertions(+), 3 deletions(-)
+Merging rockchip/for-next (4a4070c80fe6 Merge branch 'v6.20-armsoc/dts64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip.git rockchip/for-next
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/rockchip/rk3288.dtsi             | 17 +++++++++++-
+ arch/arm64/boot/dts/rockchip/Makefile              |  5 ++++
+ .../boot/dts/rockchip/rk3399-pinephone-pro.dts     | 10 +++++--
+ .../boot/dts/rockchip/rk3576-evb1-v10-pcie1.dtso   | 31 ++++++++++++++++++++++
+ arch/arm64/boot/dts/rockchip/rk3588-rock-5-itx.dts | 23 ++++++++++++++++
+ 5 files changed, 83 insertions(+), 3 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3576-evb1-v10-pcie1.dtso
+Merging samsung-krzk/for-next (cb4cfbcf2764 Merge branch 'next/dt64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git samsung-krzk/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/rng/samsung,exynos5250-trng.yaml      | 13 ++++++--
+ .../bindings/soc/samsung/exynos-pmu.yaml           |  4 ++-
+ arch/arm/boot/dts/samsung/s3c6400.dtsi             | 38 ----------------------
+ arch/arm64/boot/dts/exynos/google/gs101.dtsi       | 11 ++++++-
+ 4 files changed, 23 insertions(+), 43 deletions(-)
+ delete mode 100644 arch/arm/boot/dts/samsung/s3c6400.dtsi
+Merging scmi/for-linux-next (98675bc92a9e Merge branches 'for-next/juno/updates', 'for-next/ffa/updates' and 'for-next/scmi/updates' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux into for-linux-next)
+$ git merge -m Merge branch 'for-linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux.git scmi/for-linux-next
+Merge made by the 'ort' strategy.
+ arch/arm64/boot/dts/arm/morello-sdp.dts |   7 +++
+ drivers/firmware/arm_ffa/driver.c       |  48 +++++++++++---
+ drivers/firmware/arm_scmi/perf.c        |   2 +-
+ drivers/firmware/arm_scmi/pinctrl.c     | 108 ++++++++++++++------------------
+ drivers/firmware/arm_scmi/reset.c       |  50 ++++++++++-----
+ drivers/firmware/arm_scmi/shmem.c       |   5 +-
+ 6 files changed, 131 insertions(+), 89 deletions(-)
+Merging sophgo/for-next (c7f013d4c8c0 Merge branch 'dt/riscv' into for-next)
+$ git merge -m Merge branch 'for-next' of https://github.com/sophgo/linux.git sophgo/for-next
+Merge made by the 'ort' strategy.
+ arch/riscv/boot/dts/sophgo/cv180x.dtsi | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+Merging sophgo-soc/soc-for-next (c8754c7deab4 soc: sophgo: cv1800: rtcsys: New driver (handling RTC only))
+$ git merge -m Merge branch 'soc-for-next' of https://github.com/sophgo/linux.git sophgo-soc/soc-for-next
+Already up to date.
+Merging spacemit/for-next (5a97a38c2279 riscv: dts: spacemit: define all missing I2C controller nodes)
+$ git merge -m Merge branch 'for-next' of https://github.com/spacemit-com/linux spacemit/for-next
+Already up to date.
+Merging stm32/stm32-next (a6961e2ced51 arm64: dts: st: set RIFSC as an access controller on stm32mp21x platforms)
+$ git merge -m Merge branch 'stm32-next' of https://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32.git stm32/stm32-next
+Already up to date.
+Merging sunxi/sunxi/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'sunxi/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux.git sunxi/sunxi/for-next
+Already up to date.
+Merging tee/next (e8fd8a160bbb Merge branch 'tee_bus_callback_for_6.20' into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/jenswi/linux-tee.git tee/next
+Merge made by the 'ort' strategy.
+ Documentation/driver-api/tee.rst | 18 ++-------
+ drivers/rtc/rtc-optee.c          | 27 ++++---------
+ drivers/tee/tee_core.c           | 84 ++++++++++++++++++++++++++++++++++++++++
+ include/linux/tee_drv.h          | 12 ++++++
+ 4 files changed, 106 insertions(+), 35 deletions(-)
+Merging tegra/for-next (e016a50e00e6 Merge branch for-6.19/arm64/defconfig into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git tegra/for-next
+Merge made by the 'ort' strategy.
+Merging tenstorrent-dt/tenstorrent-dt-for-next (a71e6e8eea8a riscv: defconfig: Enable Tenstorrent SoCs)
+$ git merge -m Merge branch 'tenstorrent-dt-for-next' of https://github.com/tenstorrent/linux.git tenstorrent-dt/tenstorrent-dt-for-next
+Already up to date.
+Merging thead-dt/thead-dt-for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'thead-dt-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-dt/thead-dt-for-next
+Already up to date.
+Merging ti/ti-next (109d58039d30 Merge branch 'ti-k3-dts-next' into ti-next)
+$ git merge -m Merge branch 'ti-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux.git ti/ti-next
+Merge made by the 'ort' strategy.
+ arch/arm64/boot/dts/ti/k3-am62-lp-sk-nand.dtso     |  2 +-
+ arch/arm64/boot/dts/ti/k3-am62-phycore-som.dtsi    |  8 ++++++
+ arch/arm64/boot/dts/ti/k3-am62a-phycore-som.dtsi   |  8 ++++++
+ arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi        |  2 +-
+ .../ti/k3-am642-phyboard-electra-peb-c-010.dtso    |  7 ++----
+ ...m642-phyboard-electra-x27-gpio1-spi1-uart3.dtso |  8 +++---
+ arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts   |  4 +--
+ arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts      |  4 +--
+ drivers/firmware/ti_sci.h                          | 29 ++++++++++++----------
+ 9 files changed, 44 insertions(+), 28 deletions(-)
+Merging xilinx/for-next (43da9849395e Merge remote-tracking branch 'git/zynqmp/dt' into for-next)
+$ git merge -m Merge branch 'for-next' of https://github.com/Xilinx/linux-xlnx.git xilinx/for-next
+Auto-merging arch/arm64/configs/defconfig
+Merge made by the 'ort' strategy.
+ .../bindings/watchdog/xlnx,versal-wwdt.yaml        |  3 ++
+ arch/arm64/boot/dts/xilinx/versal-net.dtsi         |  4 +--
+ arch/arm64/boot/dts/xilinx/zynqmp-clk-ccf.dtsi     |  2 +-
+ arch/arm64/boot/dts/xilinx/zynqmp.dtsi             | 25 ++++++-------
+ arch/arm64/configs/defconfig                       | 42 ++++++++++++++++++++++
+ drivers/soc/xilinx/zynqmp_power.c                  |  6 ++--
+ 6 files changed, 61 insertions(+), 21 deletions(-)
+Merging clk/clk-next (6f172175b6f3 Merge branches 'clk-visconti', 'clk-imx', 'clk-microchip', 'clk-rockchip' and 'clk-qcom' into clk-next)
+$ git merge -m Merge branch 'clk-next' of https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git clk/clk-next
+Already up to date.
+Merging clk-imx/for-next (fdc1dc7dd53b clk: imx: add driver for imx8ulp's sim lpav)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/abelvesa/linux.git clk-imx/for-next
+Already up to date.
+Merging clk-renesas/renesas-clk (d8921e42a198 clk: renesas: r9a09g056: Add entries for the RSPIs)
+$ git merge -m Merge branch 'renesas-clk' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git clk-renesas/renesas-clk
+Merge made by the 'ort' strategy.
+ drivers/clk/renesas/r9a09g056-cpg.c                |  46 +++++
+ drivers/clk/renesas/r9a09g077-cpg.c                | 195 ++++++++++++++++++++-
+ drivers/clk/renesas/rzg2l-cpg.c                    |   6 +-
+ .../dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h |   2 +
+ .../dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h |   2 +
+ 5 files changed, 244 insertions(+), 7 deletions(-)
+Merging thead-clk/thead-clk-for-next (30441a56b1d1 clk: thead: th1520-ap: Support CPU frequency scaling)
+$ git merge -m Merge branch 'thead-clk-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-clk/thead-clk-for-next
+Merge made by the 'ort' strategy.
+ drivers/clk/thead/clk-th1520-ap.c               | 350 +++++++++++++++++++++++-
+ include/dt-bindings/clock/thead,th1520-clk-ap.h |   1 +
+ 2 files changed, 344 insertions(+), 7 deletions(-)
+Merging csky/linux-next (817d21578d51 csky: Remove compile warning for CONFIG_SMP)
+$ git merge -m Merge branch 'linux-next' of https://github.com/c-sky/csky-linux.git csky/linux-next
+Already up to date.
+Merging loongarch/loongarch-next (be77cf43d2fd LoongArch: Adjust default config files for 32BIT/64BIT)
+$ git merge -m Merge branch 'loongarch-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson.git loongarch/loongarch-next
+Already up to date.
+Merging m68k/for-next (aaf4e92341b0 m68k: defconfig: Update defconfigs for v6.18-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git m68k/for-next
+Already up to date.
+Merging m68knommu/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gerg/m68knommu.git m68knommu/for-next
+Already up to date.
+Merging microblaze/next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'next' of git://git.monstr.eu/linux-2.6-microblaze.git microblaze/next
+Already up to date.
+Merging mips/mips-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'mips-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git mips/mips-next
+Already up to date.
+Merging openrisc/for-next (8c30b0018f9d openrisc: Add jump label support)
+$ git merge -m Merge branch 'for-next' of https://github.com/openrisc/linux.git openrisc/for-next
+Already up to date.
+Merging parisc-hd/for-next (7d0a66e4bb90 Linux 6.18)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux.git parisc-hd/for-next
+Already up to date.
+Merging powerpc/next (9b36c7fc5aa5 macintosh/via-pmu-backlight: Include <linux/fb.h> and <linux/of.h>)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git powerpc/next
+Already up to date.
+Merging risc-v/for-next (7e40c6791cc8 kselftest/riscv: add kselftest for user mode cfi)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git risc-v/for-next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging Documentation/devicetree/bindings/riscv/extensions.yaml
+Auto-merging arch/riscv/Kconfig
+Auto-merging arch/riscv/include/asm/hwcap.h
+CONFLICT (content): Merge conflict in arch/riscv/include/asm/hwcap.h
+Auto-merging arch/riscv/include/asm/pgtable.h
+Auto-merging arch/riscv/kernel/cpufeature.c
+Auto-merging include/linux/mm.h
+CONFLICT (content): Merge conflict in include/linux/mm.h
+Auto-merging tools/testing/selftests/riscv/hwprobe/which-cpus.c
+Resolved 'arch/riscv/include/asm/hwcap.h' using previous resolution.
+Resolved 'include/linux/mm.h' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master c6a42e04f0be] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/admin-guide/kernel-parameters.txt    |   8 +
+ Documentation/arch/riscv/hwprobe.rst               |  12 +
+ Documentation/arch/riscv/index.rst                 |   2 +
+ Documentation/arch/riscv/zicfilp.rst               | 122 +++++
+ Documentation/arch/riscv/zicfiss.rst               | 194 ++++++++
+ .../devicetree/bindings/riscv/extensions.yaml      |  50 ++
+ arch/riscv/Kconfig                                 |  22 +
+ arch/riscv/Makefile                                |   8 +-
+ arch/riscv/configs/hardening.config                |   4 +
+ arch/riscv/include/asm/asm-prototypes.h            |   1 +
+ arch/riscv/include/asm/assembler.h                 |  44 ++
+ arch/riscv/include/asm/atomic.h                    |   8 +-
+ arch/riscv/include/asm/cpufeature.h                |  12 +
+ arch/riscv/include/asm/csr.h                       |  14 +
+ arch/riscv/include/asm/entry-common.h              |   2 +
+ arch/riscv/include/asm/hwcap.h                     |   4 +
+ arch/riscv/include/asm/hwprobe.h                   |   3 +-
+ arch/riscv/include/asm/mman.h                      |  26 +
+ arch/riscv/include/asm/mmu_context.h               |   7 +
+ arch/riscv/include/asm/pgtable.h                   |  46 +-
+ arch/riscv/include/asm/processor.h                 |   1 +
+ arch/riscv/include/asm/sbi.h                       |  29 ++
+ arch/riscv/include/asm/thread_info.h               |   3 +
+ arch/riscv/include/asm/usercfi.h                   |  97 ++++
+ arch/riscv/include/asm/vdso.h                      |  13 +-
+ arch/riscv/include/asm/vector.h                    |   3 +
+ arch/riscv/include/uapi/asm/hwprobe.h              |   7 +
+ arch/riscv/include/uapi/asm/ptrace.h               |  34 ++
+ arch/riscv/include/uapi/asm/sigcontext.h           |   1 +
+ arch/riscv/kernel/Makefile                         |   2 +
+ arch/riscv/kernel/asm-offsets.c                    |  10 +
+ arch/riscv/kernel/cpufeature.c                     |  49 ++
+ arch/riscv/kernel/entry.S                          |  38 ++
+ arch/riscv/kernel/head.S                           |  27 +
+ arch/riscv/kernel/process.c                        |  27 +-
+ arch/riscv/kernel/ptrace.c                         |  95 ++++
+ arch/riscv/kernel/signal.c                         | 148 +++++-
+ arch/riscv/kernel/sys_hwprobe.c                    | 168 ++++---
+ arch/riscv/kernel/sys_riscv.c                      |  10 +
+ arch/riscv/kernel/traps.c                          |  54 ++
+ arch/riscv/kernel/usercfi.c                        | 544 +++++++++++++++++++++
+ arch/riscv/kernel/vdso.c                           |   7 +
+ arch/riscv/kernel/vdso/Makefile                    |  40 +-
+ arch/riscv/kernel/vdso/flush_icache.S              |   4 +
+ arch/riscv/kernel/vdso/gen_vdso_offsets.sh         |   4 +-
+ arch/riscv/kernel/vdso/getcpu.S                    |   4 +
+ arch/riscv/kernel/vdso/note.S                      |   3 +
+ arch/riscv/kernel/vdso/rt_sigreturn.S              |   4 +
+ arch/riscv/kernel/vdso/sys_hwprobe.S               |   4 +
+ arch/riscv/kernel/vdso/vgetrandom-chacha.S         |   5 +-
+ arch/riscv/kernel/vdso_cfi/Makefile                |  25 +
+ arch/riscv/kernel/vdso_cfi/vdso-cfi.S              |  11 +
+ arch/riscv/mm/init.c                               |   2 +-
+ arch/riscv/mm/pgtable.c                            |  16 +
+ include/linux/cpu.h                                |   4 +
+ include/linux/mm.h                                 |   5 +
+ include/uapi/linux/elf.h                           |   2 +
+ include/uapi/linux/prctl.h                         |  27 +
+ kernel/sys.c                                       |  30 ++
+ tools/testing/selftests/riscv/Makefile             |   2 +-
+ tools/testing/selftests/riscv/cfi/.gitignore       |   2 +
+ tools/testing/selftests/riscv/cfi/Makefile         |  23 +
+ tools/testing/selftests/riscv/cfi/cfi_rv_test.h    |  82 ++++
+ tools/testing/selftests/riscv/cfi/cfitests.c       | 173 +++++++
+ tools/testing/selftests/riscv/cfi/shadowstack.c    | 385 +++++++++++++++
+ tools/testing/selftests/riscv/cfi/shadowstack.h    |  27 +
+ tools/testing/selftests/riscv/hwprobe/which-cpus.c |  18 +-
+ 67 files changed, 2738 insertions(+), 120 deletions(-)
+ create mode 100644 Documentation/arch/riscv/zicfilp.rst
+ create mode 100644 Documentation/arch/riscv/zicfiss.rst
+ create mode 100644 arch/riscv/configs/hardening.config
+ create mode 100644 arch/riscv/include/asm/mman.h
+ create mode 100644 arch/riscv/include/asm/usercfi.h
+ create mode 100644 arch/riscv/kernel/usercfi.c
+ create mode 100644 arch/riscv/kernel/vdso_cfi/Makefile
+ create mode 100644 arch/riscv/kernel/vdso_cfi/vdso-cfi.S
+ create mode 100644 tools/testing/selftests/riscv/cfi/.gitignore
+ create mode 100644 tools/testing/selftests/riscv/cfi/Makefile
+ create mode 100644 tools/testing/selftests/riscv/cfi/cfi_rv_test.h
+ create mode 100644 tools/testing/selftests/riscv/cfi/cfitests.c
+ create mode 100644 tools/testing/selftests/riscv/cfi/shadowstack.c
+ create mode 100644 tools/testing/selftests/riscv/cfi/shadowstack.h
+Merging riscv-dt/riscv-dt-for-next (a4aad736716d riscv: dts: microchip: convert clock and reset to use syscon)
+$ git merge -m Merge branch 'riscv-dt-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-dt/riscv-dt-for-next
+Merge made by the 'ort' strategy.
+ arch/riscv/boot/dts/microchip/mpfs.dtsi | 34 +++++++++++++++++++++++----------
+ 1 file changed, 24 insertions(+), 10 deletions(-)
+Merging riscv-soc/riscv-soc-for-next (719362b8a972 Merge branch 'riscv-cache-for-next' into riscv-soc-for-next)
+$ git merge -m Merge branch 'riscv-soc-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-soc/riscv-soc-for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS | 8 ++++++++
+ 1 file changed, 8 insertions(+)
+Merging s390/for-next (ca4a4090a9bb Merge branch 'fixes' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git s390/for-next
+Auto-merging drivers/s390/crypto/ap_bus.c
+Merge made by the 'ort' strategy.
+ arch/s390/boot/Makefile          |  1 +
+ arch/s390/include/asm/ptrace.h   | 37 ++++++++++++++++++++++++++-----------
+ arch/s390/purgatory/Makefile     |  1 +
+ drivers/s390/crypto/ap_bus.c     | 12 ++++++------
+ drivers/s390/crypto/zcrypt_api.c |  2 +-
+ 5 files changed, 35 insertions(+), 18 deletions(-)
+Merging sh/for-next (8f5ae30d69d7 Linux 6.17-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/glaubitz/sh-linux.git sh/for-next
+Already up to date.
+Merging sparc/for-next (fe0126702a40 sparc: Replace deprecated strcpy() with strscpy() in handle_nextprop_quirks())
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git sparc/for-next
+Already up to date.
+Merging uml/next (a3209bb94b36 um: Disable KASAN_INLINE when STATIC_LINK is selected)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git uml/next
+Already up to date.
+Merging xtensa/xtensa-for-next (0d4b3ca115b5 xtensa: align: validate access in fast_load_store)
+$ git merge -m Merge branch 'xtensa-for-next' of https://github.com/jcmvbkbc/linux-xtensa.git xtensa/xtensa-for-next
+Merge made by the 'ort' strategy.
+ arch/xtensa/kernel/align.S | 10 ++++++++--
+ 1 file changed, 8 insertions(+), 2 deletions(-)
+Merging fs-next (e9fad83f1229 Merge branch 'vfs.all' of https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git)
+$ git merge -m Merge branch 'fs-next' of linux-next fs-next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging Documentation/admin-guide/sysctl/kernel.rst
+Auto-merging fs/ecryptfs/ecryptfs_kernel.h
+Auto-merging fs/fat/dir.c
+Auto-merging fs/gfs2/lock_dlm.c
+Auto-merging kernel/pid.c
+CONFLICT (content): Merge conflict in kernel/pid.c
+Resolved 'kernel/pid.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 4f271e6e7ce7] Merge branch 'fs-next' of linux-next
+$ git diff -M --stat --summary HEAD^..
+ Documentation/admin-guide/kernel-parameters.txt    |  12 +-
+ Documentation/admin-guide/sysctl/kernel.rst        |   6 -
+ Documentation/filesystems/f2fs.rst                 |  43 +++
+ Documentation/filesystems/locking.rst              |   8 -
+ Documentation/filesystems/mount_api.rst            |   2 -
+ Documentation/filesystems/porting.rst              |   7 +-
+ Documentation/filesystems/vfs.rst                  |  58 +---
+ arch/arm/configs/neponset_defconfig                |   2 +-
+ fs/btrfs/Kconfig                                   |   9 +-
+ fs/btrfs/bio.c                                     |  19 +-
+ fs/btrfs/bio.h                                     |  16 +-
+ fs/btrfs/block-group.c                             |  11 +-
+ fs/btrfs/compression.c                             |   1 -
+ fs/btrfs/ctree.c                                   |  49 ++-
+ fs/btrfs/ctree.h                                   |   9 +
+ fs/btrfs/defrag.c                                  |   5 +-
+ fs/btrfs/delayed-inode.c                           |   4 +-
+ fs/btrfs/direct-io.c                               |  29 +-
+ fs/btrfs/disk-io.c                                 |  87 ++----
+ fs/btrfs/extent-io-tree.c                          |   7 +-
+ fs/btrfs/extent-tree.c                             |  80 +++--
+ fs/btrfs/extent_io.c                               |  70 +++--
+ fs/btrfs/file-item.c                               |   4 -
+ fs/btrfs/file.c                                    |   9 +-
+ fs/btrfs/free-space-tree.c                         |   4 +-
+ fs/btrfs/fs.c                                      | 102 ++++++-
+ fs/btrfs/fs.h                                      |  23 +-
+ fs/btrfs/inode-item.c                              |   7 +-
+ fs/btrfs/inode.c                                   | 338 +++++++++++++--------
+ fs/btrfs/messages.c                                |  26 +-
+ fs/btrfs/messages.h                                |  76 ++---
+ fs/btrfs/qgroup.c                                  | 103 +++++--
+ fs/btrfs/relocation.c                              |   5 +-
+ fs/btrfs/root-tree.c                               |  47 ++-
+ fs/btrfs/scrub.c                                   |  39 +--
+ fs/btrfs/space-info.c                              |  18 +-
+ fs/btrfs/space-info.h                              |  16 +
+ fs/btrfs/super.c                                   |  29 +-
+ fs/btrfs/sysfs.c                                   |  58 +---
+ fs/btrfs/transaction.c                             |  11 +-
+ fs/btrfs/volumes.c                                 |  19 +-
+ fs/btrfs/zoned.c                                   |  55 ++++
+ fs/btrfs/zoned.h                                   |   8 +
+ fs/buffer.c                                        |   6 +-
+ fs/char_dev.c                                      |   2 +-
+ fs/ecryptfs/crypto.c                               |   2 +-
+ fs/ecryptfs/ecryptfs_kernel.h                      |   1 -
+ fs/ecryptfs/keystore.c                             |   4 +-
+ fs/erofs/decompressor_crypto.c                     |   2 +-
+ fs/erofs/zdata.c                                   |   4 +-
+ fs/exec.c                                          |   2 +-
+ fs/exfat/namei.c                                   |   4 +-
+ fs/ext4/extents.c                                  | 137 +++++----
+ fs/ext4/extents_status.c                           | 124 ++++++--
+ fs/ext4/inode.c                                    |  18 +-
+ fs/ext4/mballoc.c                                  |   3 +-
+ fs/ext4/resize.c                                   |   2 +-
+ fs/ext4/super.c                                    |   2 +-
+ fs/f2fs/data.c                                     | 248 ++++++++++++++-
+ fs/f2fs/f2fs.h                                     |  16 +
+ fs/f2fs/file.c                                     |   4 +
+ fs/f2fs/gc.c                                       |   2 +-
+ fs/f2fs/inode.c                                    |   6 +-
+ fs/f2fs/segment.h                                  |  99 +++---
+ fs/fat/dir.c                                       |   4 +-
+ fs/fat/file.c                                      |   3 +-
+ fs/fs_context.c                                    | 208 +------------
+ fs/fsopen.c                                        |  10 -
+ fs/fuse/dev.c                                      |   2 +-
+ fs/fuse/file.c                                     |   8 +-
+ fs/gfs2/glock.c                                    |  85 +++---
+ fs/gfs2/glock.h                                    |   6 +-
+ fs/gfs2/glops.c                                    |  34 +--
+ fs/gfs2/incore.h                                   |  16 +-
+ fs/gfs2/lock_dlm.c                                 |  28 +-
+ fs/gfs2/log.c                                      |   7 +-
+ fs/gfs2/lops.c                                     |  70 +++--
+ fs/gfs2/lops.h                                     |   4 +-
+ fs/gfs2/meta_io.c                                  |   6 +-
+ fs/gfs2/meta_io.h                                  |   2 +-
+ fs/gfs2/quota.c                                    |   5 +-
+ fs/gfs2/rgrp.c                                     |   2 +-
+ fs/gfs2/trace_gfs2.h                               |  48 +--
+ fs/gfs2/trans.c                                    |   4 +-
+ fs/init.c                                          |  14 -
+ fs/internal.h                                      |   1 -
+ fs/jfs/jfs_logmgr.c                                |   1 +
+ fs/jfs/namei.c                                     |   6 +-
+ fs/locks.c                                         |  14 +-
+ fs/namei.c                                         |  80 ++++-
+ fs/namespace.c                                     | 102 ++++---
+ fs/nfsd/nfs4proc.c                                 |  11 +-
+ fs/open.c                                          |  39 ---
+ fs/splice.c                                        |   2 +-
+ fs/xfs/libxfs/xfs_log_format.h                     |   7 -
+ fs/xfs/libxfs/xfs_sb.c                             |  15 +
+ fs/xfs/scrub/attr_repair.c                         |   2 +-
+ fs/xfs/xfs_attr_item.c                             |  29 +-
+ fs/xfs/xfs_bmap_item.c                             |  10 +-
+ fs/xfs/xfs_buf_item.c                              |  20 +-
+ fs/xfs/xfs_discard.c                               |  27 +-
+ fs/xfs/xfs_discard.h                               |   2 +-
+ fs/xfs/xfs_dquot_item.c                            |   9 +-
+ fs/xfs/xfs_exchmaps_item.c                         |  11 +-
+ fs/xfs/xfs_extfree_item.c                          |  10 +-
+ fs/xfs/xfs_file.c                                  |  58 +++-
+ fs/xfs/xfs_icreate_item.c                          |   6 +-
+ fs/xfs/xfs_inode_item.c                            |  49 ++-
+ fs/xfs/xfs_log.c                                   | 292 +++++++-----------
+ fs/xfs/xfs_log.h                                   |  65 +---
+ fs/xfs/xfs_log_cil.c                               | 111 ++++++-
+ fs/xfs/xfs_log_priv.h                              |  20 ++
+ fs/xfs/xfs_refcount_item.c                         |  10 +-
+ fs/xfs/xfs_rmap_item.c                             |  10 +-
+ fs/xfs/xfs_rtalloc.c                               |  14 +-
+ fs/xfs/xfs_trans.h                                 |   4 +-
+ include/linux/filelock.h                           |  18 +-
+ include/linux/fs.h                                 |   5 +-
+ include/linux/fs/super_types.h                     |   1 -
+ include/linux/init_syscalls.h                      |   1 -
+ include/linux/initrd.h                             |   2 -
+ include/trace/events/f2fs.h                        |  12 +-
+ include/uapi/linux/mount.h                         |  10 +-
+ include/uapi/linux/sysctl.h                        |   1 -
+ init/do_mounts.c                                   |  11 +-
+ init/do_mounts.h                                   |  18 +-
+ init/do_mounts_initrd.c                            | 107 +------
+ init/do_mounts_rd.c                                |  24 +-
+ kernel/pid.c                                       |  19 +-
+ net/ceph/messenger_v2.c                            |   2 +
+ rust/helpers/fs.c                                  |   2 +-
+ rust/helpers/pid_namespace.c                       |   8 +-
+ rust/helpers/poll.c                                |   5 +-
+ .../selftests/filesystems/statmount/statmount.h    |  15 +-
+ .../filesystems/statmount/statmount_test.c         | 261 +++++++++++++++-
+ .../filesystems/statmount/statmount_test_ns.c      | 101 +++++-
+ 136 files changed, 2502 insertions(+), 1943 deletions(-)
+Merging printk/for-next (34793812e592 Merge branch 'rework/atomic-flush-softlockup' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/printk/linux.git printk/for-next
+Merge made by the 'ort' strategy.
+ kernel/printk/nbcon.c | 45 +++++++++++++++++++++++----------------------
+ 1 file changed, 23 insertions(+), 22 deletions(-)
+Merging pci/next (43dfc13ca972 Merge tag 'pci-v6.19-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/pci/pci)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git pci/next
+Already up to date.
+Merging pstore/for-next/pstore (b69255357368 pstore/ram: Update module parameters from platform data)
+$ git merge -m Merge branch 'for-next/pstore' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git pstore/for-next/pstore
+Already up to date.
+Merging hid/for-next (1c772200c9dc Merge branch 'for-6.18/upstream-fixes' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/hid/hid.git hid/for-next
+Merge made by the 'ort' strategy.
+Merging i2c/i2c/for-next (d202341d9b0c i2c: qcom-cci: Add msm8953 compatible)
+$ git merge -m Merge branch 'i2c/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wsa/linux.git i2c/i2c/for-next
+Already up to date.
+Merging i2c-host/i2c/i2c-host (6a2817432628 i2c: designware: Add dedicated algorithm for AMD NAVI)
+$ git merge -m Merge branch 'i2c/i2c-host' of https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git i2c-host/i2c/i2c-host
+Merge made by the 'ort' strategy.
+ drivers/i2c/busses/i2c-designware-master.c  | 27 +++++++++++++++------------
+ drivers/i2c/busses/i2c-designware-platdrv.c |  4 ++--
+ 2 files changed, 17 insertions(+), 14 deletions(-)
+Merging i3c/i3c/next (136209e6bd98 i3c: adi: Fix confusing cleanup.h syntax)
+$ git merge -m Merge branch 'i3c/next' of https://git.kernel.org/pub/scm/linux/kernel/git/i3c/linux.git i3c/i3c/next
+Already up to date.
+Merging dmi/dmi-for-next (4d1b28a8119c firmware: dmi: Add info message for number of populated and total memory slots)
+$ git merge -m Merge branch 'dmi-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jdelvare/staging.git dmi/dmi-for-next
+Already up to date.
+Merging hwmon-staging/hwmon-next (1d9d442ab179 hwmon: (fam15h_power) Use generic power management)
+$ git merge -m Merge branch 'hwmon-next' of https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git hwmon-staging/hwmon-next
+Merge made by the 'ort' strategy.
+ .../bindings/hwmon/microchip,sparx5-temp.yaml          |  8 ++++++--
+ Documentation/hwmon/asus_ec_sensors.rst                |  2 ++
+ Documentation/hwmon/sht3x.rst                          | 18 +++++++++++++-----
+ Documentation/hwmon/submitting-patches.rst             |  5 ++++-
+ drivers/hwmon/Kconfig                                  |  4 ++--
+ drivers/hwmon/asus-ec-sensors.c                        | 12 ++++++++++++
+ drivers/hwmon/fam15h_power.c                           | 11 +++++------
+ drivers/hwmon/sht3x.c                                  |  1 +
+ 8 files changed, 45 insertions(+), 16 deletions(-)
+Merging jc_docs/docs-next (464257baf992 docs: makefile: move rustdoc check to the build wrapper)
+$ git merge -m Merge branch 'docs-next' of git://git.lwn.net/linux.git jc_docs/docs-next
+Already up to date.
+Merging v4l-dvb/next (b70886ff5833 media: vb2: drop wait_prepare/finish callbacks)
+$ git merge -m Merge branch 'next' of git://linuxtv.org/media-ci/media-pending.git v4l-dvb/next
+Merge made by the 'ort' strategy.
+ Documentation/driver-api/media/v4l2-dev.rst       |  8 ++--
+ Documentation/userspace-api/media/conf_nitpick.py |  2 -
+ drivers/media/common/videobuf2/videobuf2-core.c   | 49 +++++------------------
+ drivers/media/common/videobuf2/videobuf2-v4l2.c   | 14 -------
+ drivers/media/dvb-core/dmxdev.c                   | 46 +++++++++------------
+ drivers/media/dvb-core/dvb_vb2.c                  | 45 +++++++--------------
+ include/media/dvb_vb2.h                           | 17 ++++----
+ include/media/videobuf2-core.h                    | 23 +++--------
+ include/media/videobuf2-v4l2.h                    | 18 ---------
+ 9 files changed, 62 insertions(+), 160 deletions(-)
+Merging v4l-dvb-next/master (adc218676eef Linux 6.12)
+$ git merge -m Merge branch 'master' of git://linuxtv.org/mchehab/media-next.git v4l-dvb-next/master
+Already up to date.
+Merging pm/linux-next (5c062ebfa076 Merge branch 'fixes' into linux-next)
+$ git merge -m Merge branch 'linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git pm/linux-next
+Merge made by the 'ort' strategy.
+ Documentation/power/runtime_pm.rst |   7 +--
+ drivers/acpi/acpi_pnp.c            |  19 +-----
+ drivers/acpi/device_pm.c           |   3 +-
+ drivers/acpi/nfit/core.c           |  47 +++++++++------
+ drivers/acpi/pci_root.c            |   2 +-
+ drivers/acpi/scan.c                | 120 +++++++++++++++++++++++++++++++++++--
+ drivers/base/power/main.c          |   7 ++-
+ drivers/cpufreq/cpufreq.c          |  13 ++--
+ drivers/pci/pci-acpi.c             |  15 ++---
+ drivers/platform/x86/intel/hid.c   |  41 +------------
+ drivers/platform/x86/intel/vbtn.c  |  30 +---------
+ include/linux/pci-acpi.h           |   3 +-
+ kernel/power/main.c                |   2 +-
+ 13 files changed, 171 insertions(+), 138 deletions(-)
+Merging cpufreq-arm/cpufreq/arm/linux-next (f98c414030d2 dt-bindings: cpufreq: qcom-hw: document Milos CPUFREQ Hardware)
+$ git merge -m Merge branch 'cpufreq/arm/linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git cpufreq-arm/cpufreq/arm/linux-next
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/cpufreq/cpufreq-qcom-hw.yaml | 2 ++
+ rust/helpers/cpufreq.c                                         | 3 ++-
+ rust/kernel/cpufreq.rs                                         | 2 ++
+ 3 files changed, 6 insertions(+), 1 deletion(-)
+Merging cpupower/cpupower (ff72619e1134 tools/power cpupower: Show C0 in idle-info dump)
+$ git merge -m Merge branch 'cpupower' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux.git cpupower/cpupower
+Merge made by the 'ort' strategy.
+ tools/power/cpupower/lib/cpuidle.c                      | 7 +++----
+ tools/power/cpupower/utils/cpufreq-info.c               | 2 +-
+ tools/power/cpupower/utils/cpuidle-info.c               | 2 +-
+ tools/power/cpupower/utils/idle_monitor/cpuidle_sysfs.c | 2 +-
+ 4 files changed, 6 insertions(+), 7 deletions(-)
+Merging devfreq/devfreq-next (d9600d57668c PM / devfreq: Fix typo in DFSO_DOWNDIFFERENTIAL macro name)
+$ git merge -m Merge branch 'devfreq-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/linux.git devfreq/devfreq-next
+Already up to date.
+Merging pmdomain/next (1f67707fafa5 pmdomain: Merge branch fixes into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git pmdomain/next
+Already up to date.
+Merging opp/opp/linux-next (33ffb0aa8ce8 rust: opp: simplify callers of `to_c_str_array`)
+$ git merge -m Merge branch 'opp/linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git opp/opp/linux-next
+Already up to date.
+Merging thermal/thermal/linux-next (c411d8bf0699 thermal/drivers/imx91: Add support for i.MX91 thermal monitoring unit)
+$ git merge -m Merge branch 'thermal/linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/thermal/linux.git thermal/thermal/linux-next
+Already up to date.
+Merging rdma/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git rdma/for-next
+Already up to date.
+Merging net-next/main (8f7aa3d3c732 Merge tag 'net-next-6.19' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git net-next/main
+Already up to date.
+Merging bpf-next/for-next (ec439c380135 Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf after 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git bpf-next/for-next
+Auto-merging kernel/bpf/core.c
+Auto-merging kernel/bpf/syscall.c
+Merge made by the 'ort' strategy.
+ include/linux/bpf.h                                |  16 ++
+ include/uapi/linux/bpf.h                           |  17 ++
+ kernel/bpf/bpf_iter.c                              |   2 +-
+ kernel/bpf/bpf_struct_ops.c                        |  88 ++++++++++
+ kernel/bpf/core.c                                  |   3 +
+ kernel/bpf/cpumap.c                                |  21 ++-
+ kernel/bpf/syscall.c                               |  46 +++++
+ kernel/bpf/verifier.c                              |  26 +--
+ tools/include/uapi/linux/bpf.h                     |  17 ++
+ tools/lib/bpf/bpf.c                                |  19 ++
+ tools/lib/bpf/bpf.h                                |  21 +++
+ tools/lib/bpf/btf.c                                |  29 +++-
+ tools/lib/bpf/libbpf.c                             |  62 +++++--
+ tools/lib/bpf/libbpf.h                             |  16 ++
+ tools/lib/bpf/libbpf.map                           |   2 +
+ .../bpf/prog_tests/test_struct_ops_assoc.c         | 191 +++++++++++++++++++++
+ tools/testing/selftests/bpf/prog_tests/verifier.c  |   4 +
+ .../selftests/bpf/prog_tests/xdp_cpumap_attach.c   |  19 +-
+ .../testing/selftests/bpf/progs/struct_ops_assoc.c | 105 +++++++++++
+ .../bpf/progs/struct_ops_assoc_in_timer.c          |  77 +++++++++
+ .../selftests/bpf/progs/struct_ops_assoc_reuse.c   |  75 ++++++++
+ .../selftests/bpf/progs/verifier_arena_globals1.c  |  87 ++++++++++
+ .../selftests/bpf/progs/verifier_arena_globals2.c  |  49 ++++++
+ .../selftests/bpf/progs/verifier_arena_large.c     |  21 ++-
+ .../testing/selftests/bpf/progs/verifier_subreg.c  |  68 ++++++++
+ .../testing/selftests/bpf/test_kmods/bpf_testmod.c |  17 ++
+ .../selftests/bpf/test_kmods/bpf_testmod_kfunc.h   |   1 +
+ .../selftests/bpf/verifier/direct_value_access.c   |   4 +-
+ 28 files changed, 1052 insertions(+), 51 deletions(-)
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c
+ create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_assoc.c
+ create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_assoc_in_timer.c
+ create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_assoc_reuse.c
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_arena_globals1.c
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_arena_globals2.c
+Merging ipsec-next/master (8f7aa3d3c732 Merge tag 'net-next-6.19' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec-next.git ipsec-next/master
+Already up to date.
+Merging mlx5-next/mlx5-next (5422318e27d7 net/mlx5: Expose definition for 1600Gbps link mode)
+$ git merge -m Merge branch 'mlx5-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux.git mlx5-next/mlx5-next
+Already up to date.
+Merging netfilter-next/main (8f7aa3d3c732 Merge tag 'net-next-6.19' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf-next.git netfilter-next/main
+Already up to date.
+Merging ipvs-next/main (c9dfb92de073 Merge branch 'mlx5-next' of git://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs-next.git ipvs-next/main
+Already up to date.
+Merging bluetooth/master (98246938a0e6 Bluetooth: L2CAP: Add support for setting BT_PHY)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git bluetooth/master
+Merge made by the 'ort' strategy.
+ drivers/bluetooth/btmtksdio.c     |   8 +-
+ drivers/bluetooth/btusb.c         |  14 +-
+ drivers/bluetooth/hci_bcm4377.c   |  11 +-
+ drivers/bluetooth/hci_qca.c       |  33 ++++
+ include/net/bluetooth/bluetooth.h |  39 +++--
+ include/net/bluetooth/hci.h       | 332 ++++++++++++++++++++++++++++++++++++++
+ include/net/bluetooth/hci_core.h  |   9 ++
+ include/net/bluetooth/hci_sync.h  |   3 +
+ net/bluetooth/hci_conn.c          | 127 ++++++++++++++-
+ net/bluetooth/hci_event.c         |  56 ++++++-
+ net/bluetooth/hci_sync.c          |  89 +++++++++-
+ net/bluetooth/l2cap_sock.c        |  20 ++-
+ net/bluetooth/mgmt.c              |   6 +
+ net/bluetooth/mgmt_config.c       |  21 +++
+ 14 files changed, 725 insertions(+), 43 deletions(-)
+Merging wireless-next/for-next (8f7aa3d3c732 Merge tag 'net-next-6.19' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git wireless-next/for-next
+Already up to date.
+Merging ath-next/for-next (631ee338f04d Merge branch 'ath12k-ng' into ath-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git ath-next/for-next
+Merge made by the 'ort' strategy.
+ drivers/net/wireless/ath/ath12k/Makefile           |    8 +-
+ drivers/net/wireless/ath/ath12k/ahb.c              |  171 +-
+ drivers/net/wireless/ath/ath12k/ahb.h              |   31 +-
+ drivers/net/wireless/ath/ath12k/ce.c               |  326 +-
+ drivers/net/wireless/ath/ath12k/ce.h               |    7 +-
+ drivers/net/wireless/ath/ath12k/cmn_defs.h         |   20 +
+ drivers/net/wireless/ath/ath12k/core.c             |   75 +-
+ drivers/net/wireless/ath/ath12k/core.h             |  176 +-
+ drivers/net/wireless/ath/ath12k/dbring.c           |    6 +-
+ drivers/net/wireless/ath/ath12k/debug.c            |    8 +-
+ drivers/net/wireless/ath/ath12k/debugfs.c          |    8 +-
+ .../net/wireless/ath/ath12k/debugfs_htt_stats.h    |    4 +-
+ drivers/net/wireless/ath/ath12k/debugfs_sta.c      |   62 +-
+ drivers/net/wireless/ath/ath12k/dp.c               |  563 +---
+ drivers/net/wireless/ath/ath12k/dp.h               | 1752 ++--------
+ drivers/net/wireless/ath/ath12k/dp_cmn.h           |  106 +
+ drivers/net/wireless/ath/ath12k/dp_htt.c           | 1353 ++++++++
+ drivers/net/wireless/ath/ath12k/dp_htt.h           | 1546 +++++++++
+ drivers/net/wireless/ath/ath12k/dp_mon.c           | 3517 +-------------------
+ drivers/net/wireless/ath/ath12k/dp_mon.h           |   58 +-
+ drivers/net/wireless/ath/ath12k/dp_peer.c          |  690 ++++
+ drivers/net/wireless/ath/ath12k/dp_peer.h          |  182 +
+ drivers/net/wireless/ath/ath12k/dp_rx.c            | 3482 ++-----------------
+ drivers/net/wireless/ath/ath12k/dp_rx.h            |  191 +-
+ drivers/net/wireless/ath/ath12k/dp_tx.c            | 1638 +--------
+ drivers/net/wireless/ath/ath12k/dp_tx.h            |   39 +-
+ drivers/net/wireless/ath/ath12k/hal.c              | 2258 ++-----------
+ drivers/net/wireless/ath/ath12k/hal.h              | 1475 ++++----
+ drivers/net/wireless/ath/ath12k/htc.c              |    3 +-
+ drivers/net/wireless/ath/ath12k/hw.c               | 1680 ----------
+ drivers/net/wireless/ath/ath12k/hw.h               |   94 +-
+ drivers/net/wireless/ath/ath12k/mac.c              | 1076 +++---
+ drivers/net/wireless/ath/ath12k/mac.h              |  139 +-
+ drivers/net/wireless/ath/ath12k/mhi.c              |  132 +-
+ drivers/net/wireless/ath/ath12k/mhi.h              |    5 +-
+ drivers/net/wireless/ath/ath12k/pci.c              |  241 +-
+ drivers/net/wireless/ath/ath12k/pci.h              |   42 +-
+ drivers/net/wireless/ath/ath12k/peer.c             |  453 +--
+ drivers/net/wireless/ath/ath12k/peer.h             |  112 +-
+ drivers/net/wireless/ath/ath12k/testmode.c         |    3 +-
+ drivers/net/wireless/ath/ath12k/wifi7/Makefile     |   19 +
+ drivers/net/wireless/ath/ath12k/wifi7/ahb.c        |   75 +
+ drivers/net/wireless/ath/ath12k/wifi7/ahb.h        |   20 +
+ drivers/net/wireless/ath/ath12k/wifi7/ce.c         |  973 ++++++
+ drivers/net/wireless/ath/ath12k/wifi7/ce.h         |   22 +
+ drivers/net/wireless/ath/ath12k/wifi7/core.c       |   68 +
+ drivers/net/wireless/ath/ath12k/wifi7/core.h       |   11 +
+ drivers/net/wireless/ath/ath12k/wifi7/dp.c         |  181 +
+ drivers/net/wireless/ath/ath12k/wifi7/dp.h         |   20 +
+ drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c     | 3385 +++++++++++++++++++
+ drivers/net/wireless/ath/ath12k/wifi7/dp_mon.h     |   23 +
+ drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c      | 2202 ++++++++++++
+ drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h      |   59 +
+ drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c      |  978 ++++++
+ drivers/net/wireless/ath/ath12k/wifi7/dp_tx.h      |   16 +
+ drivers/net/wireless/ath/ath12k/wifi7/hal.c        |  705 ++++
+ drivers/net/wireless/ath/ath12k/wifi7/hal.h        |  564 ++++
+ .../net/wireless/ath/ath12k/{ => wifi7}/hal_desc.h |  298 +-
+ .../net/wireless/ath/ath12k/wifi7/hal_qcn9274.c    | 1032 ++++++
+ .../net/wireless/ath/ath12k/wifi7/hal_qcn9274.h    |   45 +
+ .../net/wireless/ath/ath12k/{ => wifi7}/hal_rx.c   |  221 +-
+ .../net/wireless/ath/ath12k/{ => wifi7}/hal_rx.h   |  422 +--
+ .../ath/ath12k/{rx_desc.h => wifi7/hal_rx_desc.h}  |   55 +-
+ .../net/wireless/ath/ath12k/{ => wifi7}/hal_tx.c   |   23 +-
+ .../net/wireless/ath/ath12k/{ => wifi7}/hal_tx.h   |   30 +-
+ .../net/wireless/ath/ath12k/wifi7/hal_wcn7850.c    |  805 +++++
+ .../net/wireless/ath/ath12k/wifi7/hal_wcn7850.h    |   39 +
+ drivers/net/wireless/ath/ath12k/wifi7/hw.c         |  945 ++++++
+ drivers/net/wireless/ath/ath12k/wifi7/hw.h         |   13 +
+ drivers/net/wireless/ath/ath12k/wifi7/mhi.c        |  138 +
+ drivers/net/wireless/ath/ath12k/wifi7/mhi.h        |   11 +
+ drivers/net/wireless/ath/ath12k/wifi7/pci.c        |  191 ++
+ drivers/net/wireless/ath/ath12k/wifi7/pci.h        |   12 +
+ drivers/net/wireless/ath/ath12k/wifi7/wmi.c        |  105 +
+ drivers/net/wireless/ath/ath12k/wifi7/wmi.h        |   15 +
+ drivers/net/wireless/ath/ath12k/wmi.c              |  138 +-
+ drivers/net/wireless/ath/ath12k/wmi.h              |    7 +-
+ drivers/net/wireless/ath/ath12k/wow.c              |    5 +-
+ 78 files changed, 20196 insertions(+), 17407 deletions(-)
+ create mode 100644 drivers/net/wireless/ath/ath12k/cmn_defs.h
+ create mode 100644 drivers/net/wireless/ath/ath12k/dp_cmn.h
+ create mode 100644 drivers/net/wireless/ath/ath12k/dp_htt.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/dp_htt.h
+ create mode 100644 drivers/net/wireless/ath/ath12k/dp_peer.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/dp_peer.h
+ delete mode 100644 drivers/net/wireless/ath/ath12k/hw.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/Makefile
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/ahb.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/ahb.h
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/ce.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/ce.h
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/core.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/core.h
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/dp.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/dp.h
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/dp_mon.h
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/dp_tx.h
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/hal.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/hal.h
+ rename drivers/net/wireless/ath/ath12k/{ => wifi7}/hal_desc.h (91%)
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.h
+ rename drivers/net/wireless/ath/ath12k/{ => wifi7}/hal_rx.c (84%)
+ rename drivers/net/wireless/ath/ath12k/{ => wifi7}/hal_rx.h (75%)
+ rename drivers/net/wireless/ath/ath12k/{rx_desc.h => wifi7/hal_rx_desc.h} (97%)
+ rename drivers/net/wireless/ath/ath12k/{ => wifi7}/hal_tx.c (87%)
+ rename drivers/net/wireless/ath/ath12k/{ => wifi7}/hal_tx.h (88%)
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.h
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/hw.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/hw.h
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/mhi.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/mhi.h
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/pci.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/pci.h
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/wmi.c
+ create mode 100644 drivers/net/wireless/ath/ath12k/wifi7/wmi.h
+Merging iwlwifi-next/next (85d002b2ac56 wifi: iwlwifi: mld: remove unused variable in d3.c)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git iwlwifi-next/next
+Auto-merging drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
+Merge made by the 'ort' strategy.
+ drivers/net/wireless/intel/iwlwifi/cfg/bz.c        |  12 -
+ drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c     |  12 +
+ .../net/wireless/intel/iwlwifi/fw/api/mac-cfg.h    | 158 +++++++++++-
+ drivers/net/wireless/intel/iwlwifi/fw/api/power.h  |   2 +-
+ drivers/net/wireless/intel/iwlwifi/fw/api/rs.h     |  41 ++-
+ drivers/net/wireless/intel/iwlwifi/fw/api/rx.h     |  95 ++++++-
+ drivers/net/wireless/intel/iwlwifi/fw/file.h       |   2 +
+ drivers/net/wireless/intel/iwlwifi/fw/smem.c       |   8 +-
+ drivers/net/wireless/intel/iwlwifi/mld/Makefile    |   2 +-
+ drivers/net/wireless/intel/iwlwifi/mld/d3.c        |   2 -
+ drivers/net/wireless/intel/iwlwifi/mld/iface.c     |  21 +-
+ drivers/net/wireless/intel/iwlwifi/mld/mac80211.c  |  88 ++++++-
+ drivers/net/wireless/intel/iwlwifi/mld/mld.c       |   3 +
+ drivers/net/wireless/intel/iwlwifi/mld/mld.h       |   3 +
+ drivers/net/wireless/intel/iwlwifi/mld/nan.c       | 287 +++++++++++++++++++++
+ drivers/net/wireless/intel/iwlwifi/mld/nan.h       |  28 ++
+ drivers/net/wireless/intel/iwlwifi/mld/notif.c     |  13 +
+ drivers/net/wireless/intel/iwlwifi/mld/notif.h     |   1 +
+ drivers/net/wireless/intel/iwlwifi/mld/power.c     |  39 ++-
+ drivers/net/wireless/intel/iwlwifi/mld/rx.c        | 142 +++++++++-
+ drivers/net/wireless/intel/iwlwifi/mld/scan.c      |  12 +-
+ drivers/net/wireless/intel/iwlwifi/mld/sta.c       |   3 +-
+ drivers/net/wireless/intel/iwlwifi/mld/tlc.c       |  85 +++++-
+ drivers/net/wireless/intel/iwlwifi/mld/tx.c        |  11 +
+ drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c  |  14 +
+ drivers/net/wireless/intel/iwlwifi/mvm/scan.c      |  10 +-
+ 26 files changed, 1009 insertions(+), 85 deletions(-)
+ create mode 100644 drivers/net/wireless/intel/iwlwifi/mld/nan.c
+ create mode 100644 drivers/net/wireless/intel/iwlwifi/mld/nan.h
+Merging wpan-next/master (1dd9291eb903 ieee802154: Remove WARN_ON() in cfg802154_pernet_exit())
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git wpan-next/master
+Merge made by the 'ort' strategy.
+ net/ieee802154/core.c | 49 ++++++++++++++++++++++++++++---------------------
+ 1 file changed, 28 insertions(+), 21 deletions(-)
+Merging wpan-staging/staging (1dd9291eb903 ieee802154: Remove WARN_ON() in cfg802154_pernet_exit())
+$ git merge -m Merge branch 'staging' of https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git wpan-staging/staging
+Already up to date.
+Merging mtd/mtd/next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'mtd/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git mtd/mtd/next
+Already up to date.
+Merging nand/nand/next (a75a1dec037f mtd: spinand: add support for Dosilicon DS35Q1GA/DS35M1GA)
+$ git merge -m Merge branch 'nand/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git nand/nand/next
+Merge made by the 'ort' strategy.
+ drivers/mtd/nand/spi/Makefile    |  4 +-
+ drivers/mtd/nand/spi/core.c      |  1 +
+ drivers/mtd/nand/spi/dosilicon.c | 91 ++++++++++++++++++++++++++++++++++++++++
+ include/linux/mtd/spinand.h      |  1 +
+ 4 files changed, 95 insertions(+), 2 deletions(-)
+ create mode 100644 drivers/mtd/nand/spi/dosilicon.c
+Merging spi-nor/spi-nor/next (ed26bd40df11 mailmap: update Pratyush Yadav's email address)
+$ git merge -m Merge branch 'spi-nor/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git spi-nor/spi-nor/next
+Already up to date.
+Merging crypto/master (48bc9da3c97c crypto: zstd - fix double-free in per-CPU stream cleanup)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git crypto/master
+Already up to date.
+Merging libcrypto/libcrypto-next (325c29e7d11c lib/crypto: md5: Use rol32() instead of open-coding it)
+$ git merge -m Merge branch 'libcrypto-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git libcrypto/libcrypto-next
+Merge made by the 'ort' strategy.
+ Documentation/filesystems/fscrypt.rst              |    5 -
+ arch/arm/crypto/Kconfig                            |   10 -
+ arch/arm/crypto/Makefile                           |    2 -
+ arch/arm/crypto/nhpoly1305-neon-glue.c             |   80 -
+ arch/arm64/crypto/Kconfig                          |   10 -
+ arch/arm64/crypto/Makefile                         |    3 -
+ arch/arm64/crypto/nhpoly1305-neon-glue.c           |   79 -
+ arch/x86/crypto/Kconfig                            |   20 -
+ arch/x86/crypto/Makefile                           |    5 -
+ arch/x86/crypto/nhpoly1305-avx2-glue.c             |   81 -
+ arch/x86/crypto/nhpoly1305-sse2-glue.c             |   80 -
+ crypto/Kconfig                                     |    8 +-
+ crypto/Makefile                                    |    1 -
+ crypto/adiantum.c                                  |  460 ++---
+ crypto/nhpoly1305.c                                |  255 ---
+ crypto/testmgr.c                                   |   10 +-
+ crypto/testmgr.h                                   | 1372 --------------
+ include/crypto/mldsa.h                             |   60 +
+ include/crypto/nh.h                                |   52 +
+ include/crypto/nhpoly1305.h                        |   74 -
+ lib/crypto/Kconfig                                 |   20 +
+ lib/crypto/Makefile                                |   16 +
+ {arch/arm/crypto => lib/crypto/arm}/nh-neon-core.S |    0
+ lib/crypto/arm/nh.h                                |   33 +
+ .../crypto => lib/crypto/arm64}/nh-neon-core.S     |    3 +-
+ lib/crypto/arm64/nh.h                              |   34 +
+ lib/crypto/md5.c                                   |    2 +-
+ lib/crypto/mldsa.c                                 |  652 +++++++
+ lib/crypto/nh.c                                    |   82 +
+ lib/crypto/tests/Kconfig                           |   17 +
+ lib/crypto/tests/Makefile                          |    2 +
+ lib/crypto/tests/mldsa-testvecs.h                  | 1887 ++++++++++++++++++++
+ lib/crypto/tests/mldsa_kunit.c                     |  438 +++++
+ lib/crypto/tests/nh-testvecs.h                     |  298 ++++
+ lib/crypto/tests/nh_kunit.c                        |   43 +
+ .../nh-avx2-x86_64.S => lib/crypto/x86/nh-avx2.S   |    3 +-
+ .../nh-sse2-x86_64.S => lib/crypto/x86/nh-sse2.S   |    3 +-
+ lib/crypto/x86/nh.h                                |   45 +
+ scripts/crypto/gen-hash-testvecs.py                |   40 +
+ 39 files changed, 3978 insertions(+), 2307 deletions(-)
+ delete mode 100644 arch/arm/crypto/nhpoly1305-neon-glue.c
+ delete mode 100644 arch/arm64/crypto/nhpoly1305-neon-glue.c
+ delete mode 100644 arch/x86/crypto/nhpoly1305-avx2-glue.c
+ delete mode 100644 arch/x86/crypto/nhpoly1305-sse2-glue.c
+ delete mode 100644 crypto/nhpoly1305.c
+ create mode 100644 include/crypto/mldsa.h
+ create mode 100644 include/crypto/nh.h
+ delete mode 100644 include/crypto/nhpoly1305.h
+ rename {arch/arm/crypto => lib/crypto/arm}/nh-neon-core.S (100%)
+ create mode 100644 lib/crypto/arm/nh.h
+ rename {arch/arm64/crypto => lib/crypto/arm64}/nh-neon-core.S (97%)
+ create mode 100644 lib/crypto/arm64/nh.h
+ create mode 100644 lib/crypto/mldsa.c
+ create mode 100644 lib/crypto/nh.c
+ create mode 100644 lib/crypto/tests/mldsa-testvecs.h
+ create mode 100644 lib/crypto/tests/mldsa_kunit.c
+ create mode 100644 lib/crypto/tests/nh-testvecs.h
+ create mode 100644 lib/crypto/tests/nh_kunit.c
+ rename arch/x86/crypto/nh-avx2-x86_64.S => lib/crypto/x86/nh-avx2.S (98%)
+ rename arch/x86/crypto/nh-sse2-x86_64.S => lib/crypto/x86/nh-sse2.S (97%)
+ create mode 100644 lib/crypto/x86/nh.h
+Merging drm/drm-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'drm-next' of https://gitlab.freedesktop.org/drm/kernel.git drm/drm-next
+Already up to date.
+Merging drm-exynos/for-linux-next (3a8660878839 Linux 6.18-rc1)
+$ git merge -m Merge branch 'for-linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/daeinki/drm-exynos.git drm-exynos/for-linux-next
+Already up to date.
+Merging drm-misc/for-linux-next (332070795bd9 accel/amdxdna: Enable hardware context priority)
+$ git merge -m Merge branch 'for-linux-next' of https://gitlab.freedesktop.org/drm/misc/kernel.git drm-misc/for-linux-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/accel/amdxdna/aie2_pci.c
+Auto-merging drivers/gpu/drm/drm_gem.c
+Auto-merging drivers/gpu/drm/panel/Kconfig
+Merge made by the 'ort' strategy.
+ .../bindings/display/bridge/renesas,dsi.yaml       |  120 ++-
+ .../bindings/display/panel/lg,sw43408.yaml         |   13 +-
+ .../bindings/display/panel/panel-simple-dsi.yaml   |    2 +
+ .../bindings/display/panel/panel-simple.yaml       |    2 +
+ .../bindings/display/panel/samsung,s6e3fc2x01.yaml |   20 +-
+ .../display/rockchip/rockchip,dw-hdmi.yaml         |    1 +
+ .../bindings/display/sitronix,st7571.yaml          |   25 +
+ .../bindings/display/sitronix,st7920.yaml          |   58 ++
+ Documentation/gpu/drm-mm.rst                       |   29 +-
+ Documentation/gpu/todo.rst                         |   45 +
+ Documentation/process/debugging/kgdb.rst           |   28 -
+ MAINTAINERS                                        |   10 +
+ drivers/accel/amdxdna/Makefile                     |    1 -
+ drivers/accel/amdxdna/aie2_ctx.c                   |   18 +-
+ drivers/accel/amdxdna/aie2_message.c               |   92 +-
+ drivers/accel/amdxdna/aie2_msg_priv.h              |    8 +-
+ drivers/accel/amdxdna/aie2_pci.c                   |    2 +-
+ drivers/accel/amdxdna/aie2_pci.h                   |    4 +
+ drivers/accel/amdxdna/aie2_pm.c                    |   17 +-
+ drivers/accel/amdxdna/aie2_psp.c                   |   15 +
+ drivers/accel/amdxdna/aie2_smu.c                   |   27 +-
+ drivers/accel/amdxdna/amdxdna_ctx.h                |    1 +
+ drivers/accel/amdxdna/amdxdna_mailbox.c            |   46 +-
+ drivers/accel/amdxdna/amdxdna_pci_drv.c            |   30 +-
+ drivers/accel/amdxdna/amdxdna_pci_drv.h            |    2 -
+ drivers/accel/amdxdna/amdxdna_pm.c                 |   22 +-
+ drivers/accel/amdxdna/npu1_regs.c                  |    2 +
+ drivers/accel/amdxdna/npu2_regs.c                  |  115 ---
+ drivers/accel/amdxdna/npu4_regs.c                  |    3 +
+ drivers/accel/amdxdna/npu5_regs.c                  |    2 +
+ drivers/accel/amdxdna/npu6_regs.c                  |    2 +
+ drivers/accel/ivpu/ivpu_gem.c                      |    2 +-
+ drivers/accel/ivpu/ivpu_mmu_context.c              |   20 +-
+ drivers/accel/ivpu/ivpu_mmu_context.h              |    5 +-
+ drivers/dma-buf/Kconfig                            |    2 +-
+ drivers/dma-buf/dma-buf.c                          |  128 ++-
+ drivers/dma-buf/dma-fence.c                        |  100 +-
+ drivers/dma-buf/st-dma-fence.c                     |   55 +-
+ drivers/gpu/drm/Kconfig                            |  213 ++---
+ drivers/gpu/drm/amd/amdgpu/dce_v10_0.c             |   35 +-
+ drivers/gpu/drm/amd/amdgpu/dce_v6_0.c              |   35 +-
+ drivers/gpu/drm/amd/amdgpu/dce_v8_0.c              |   35 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_process.c           |    6 +-
+ drivers/gpu/drm/ast/ast_cursor.c                   |   83 +-
+ drivers/gpu/drm/bridge/ti-sn65dsi83.c              |   86 +-
+ drivers/gpu/drm/display/drm_dp_mst_topology.c      |    3 +-
+ drivers/gpu/drm/drm_atomic.c                       |    1 +
+ drivers/gpu/drm/drm_bridge.c                       |   62 ++
+ drivers/gpu/drm/drm_fb_helper.c                    |  108 ---
+ drivers/gpu/drm/drm_gem.c                          |  203 +++-
+ drivers/gpu/drm/drm_gem_shmem_helper.c             |   86 +-
+ drivers/gpu/drm/drm_panic.c                        |   77 +-
+ drivers/gpu/drm/drm_plane.c                        |    4 +-
+ drivers/gpu/drm/drm_syncobj.c                      |   65 +-
+ drivers/gpu/drm/drm_vblank.c                       |   52 +-
+ drivers/gpu/drm/hyperv/Kconfig                     |   14 +
+ drivers/gpu/drm/i915/Makefile                      |    3 +-
+ drivers/gpu/drm/i915/gem/i915_gem_object_types.h   |   10 +-
+ drivers/gpu/drm/i915/gem/i915_gem_shmem.c          |   58 +-
+ drivers/gpu/drm/i915/gem/i915_gemfs.c              |   71 --
+ drivers/gpu/drm/i915/gem/i915_gemfs.h              |   14 -
+ drivers/gpu/drm/i915/gem/selftests/huge_pages.c    |   15 +-
+ drivers/gpu/drm/i915/i915_drv.h                    |    5 -
+ drivers/gpu/drm/nouveau/dispnv04/crtc.c            |   24 +-
+ drivers/gpu/drm/panel/Kconfig                      |   13 +
+ drivers/gpu/drm/panel/Makefile                     |    1 +
+ drivers/gpu/drm/panel/panel-edp.c                  |    3 +
+ drivers/gpu/drm/panel/panel-lg-sw43408.c           |   64 +-
+ drivers/gpu/drm/panel/panel-orisetech-otm8009a.c   |  175 ++--
+ drivers/gpu/drm/panel/panel-samsung-ltl106hl02.c   |  179 ++++
+ drivers/gpu/drm/panel/panel-simple.c               |   28 +
+ drivers/gpu/drm/panfrost/panfrost_device.c         |    3 +
+ drivers/gpu/drm/panfrost/panfrost_device.h         |    1 +
+ drivers/gpu/drm/panfrost/panfrost_drv.c            |  107 ++-
+ drivers/gpu/drm/panfrost/panfrost_drv.h            |    9 +
+ drivers/gpu/drm/panfrost/panfrost_gem.c            |  257 +++++
+ drivers/gpu/drm/panfrost/panfrost_gem.h            |   12 +
+ drivers/gpu/drm/panfrost/panfrost_gpu.c            |   26 +-
+ drivers/gpu/drm/panfrost/panfrost_regs.h           |   10 +-
+ drivers/gpu/drm/panthor/panthor_device.c           |   13 +-
+ drivers/gpu/drm/panthor/panthor_drv.c              |   89 +-
+ drivers/gpu/drm/panthor/panthor_drv.h              |    9 +
+ drivers/gpu/drm/panthor/panthor_fw.c               |    8 +-
+ drivers/gpu/drm/panthor/panthor_gem.c              |  258 ++++-
+ drivers/gpu/drm/panthor/panthor_gem.h              |    8 +
+ drivers/gpu/drm/panthor/panthor_gpu.c              |   21 +-
+ drivers/gpu/drm/panthor/panthor_mmu.c              |  314 +++---
+ drivers/gpu/drm/panthor/panthor_sched.c            |  256 ++---
+ drivers/gpu/drm/pl111/pl111_display.c              |    4 +-
+ drivers/gpu/drm/pl111/pl111_drv.c                  |   29 +-
+ drivers/gpu/drm/pl111/pl111_nomadik.c              |    4 +-
+ drivers/gpu/drm/pl111/pl111_nomadik.h              |    4 +-
+ drivers/gpu/drm/pl111/pl111_versatile.c            |   53 +-
+ drivers/gpu/drm/pl111/pl111_versatile.h            |    2 +-
+ drivers/gpu/drm/radeon/atombios_crtc.c             |   74 +-
+ drivers/gpu/drm/radeon/radeon_legacy_crtc.c        |   23 +-
+ drivers/gpu/drm/radeon/radeon_mode.h               |   10 +-
+ drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c     |  453 +++++++++
+ .../gpu/drm/renesas/rz-du/rzg2l_mipi_dsi_regs.h    |   34 +
+ drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c        |   16 +
+ drivers/gpu/drm/scheduler/sched_main.c             |   40 +-
+ drivers/gpu/drm/sitronix/Kconfig                   |   48 +-
+ drivers/gpu/drm/sitronix/Makefile                  |    3 +
+ drivers/gpu/drm/sitronix/st7571-i2c.c              | 1003 +-------------------
+ drivers/gpu/drm/sitronix/st7571-spi.c              |   76 ++
+ drivers/gpu/drm/sitronix/st7571.c                  |  918 ++++++++++++++++++
+ drivers/gpu/drm/sitronix/st7571.h                  |   91 ++
+ drivers/gpu/drm/sitronix/st7920.c                  |  867 +++++++++++++++++
+ drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c   |    2 +-
+ drivers/gpu/drm/ttm/ttm_bo.c                       |   64 +-
+ drivers/gpu/drm/ttm/ttm_bo_util.c                  |   57 +-
+ drivers/gpu/drm/ttm/ttm_bo_vm.c                    |   12 +-
+ drivers/gpu/drm/ttm/ttm_device.c                   |    5 +-
+ drivers/gpu/drm/ttm/ttm_pool.c                     |   26 +-
+ drivers/gpu/drm/ttm/ttm_resource.c                 |   14 +-
+ drivers/gpu/drm/ttm/ttm_tt.c                       |    2 +-
+ drivers/gpu/drm/v3d/Makefile                       |    3 +-
+ drivers/gpu/drm/v3d/v3d_bo.c                       |    6 +-
+ drivers/gpu/drm/v3d/v3d_drv.c                      |    2 +-
+ drivers/gpu/drm/v3d/v3d_drv.h                      |   11 +-
+ drivers/gpu/drm/v3d/v3d_gem.c                      |   21 +-
+ drivers/gpu/drm/v3d/v3d_gemfs.c                    |   62 --
+ drivers/gpu/drm/vgem/Kconfig                       |    9 +
+ drivers/gpu/drm/xe/xe_hw_fence.c                   |    4 +-
+ drivers/video/fbdev/core/fbcon.c                   |   24 -
+ drivers/video/fbdev/core/fbcon.h                   |    1 -
+ include/drm/drm_atomic.h                           |    5 +
+ include/drm/drm_bridge.h                           |   12 +
+ include/drm/drm_device.h                           |   15 +
+ include/drm/drm_fb_helper.h                        |   21 -
+ include/drm/drm_file.h                             |    7 +-
+ include/drm/drm_gem.h                              |   45 +-
+ include/drm/drm_gem_shmem_helper.h                 |    3 -
+ include/drm/drm_modeset_helper_vtables.h           |   23 -
+ include/drm/drm_vblank.h                           |    3 +-
+ include/drm/gpu_scheduler.h                        |   52 +
+ include/linux/dma-fence.h                          |   35 +-
+ include/linux/fb.h                                 |    4 -
+ include/trace/events/dma_buf.h                     |  157 +++
+ include/uapi/drm/amdxdna_accel.h                   |    8 +
+ include/uapi/drm/panfrost_drm.h                    |   76 +-
+ include/uapi/drm/panthor_drm.h                     |  157 ++-
+ 142 files changed, 6078 insertions(+), 2928 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/display/sitronix,st7920.yaml
+ delete mode 100644 drivers/accel/amdxdna/npu2_regs.c
+ create mode 100644 drivers/gpu/drm/hyperv/Kconfig
+ delete mode 100644 drivers/gpu/drm/i915/gem/i915_gemfs.c
+ delete mode 100644 drivers/gpu/drm/i915/gem/i915_gemfs.h
+ create mode 100644 drivers/gpu/drm/panel/panel-samsung-ltl106hl02.c
+ create mode 100644 drivers/gpu/drm/panfrost/panfrost_drv.h
+ create mode 100644 drivers/gpu/drm/panthor/panthor_drv.h
+ create mode 100644 drivers/gpu/drm/sitronix/st7571-spi.c
+ create mode 100644 drivers/gpu/drm/sitronix/st7571.c
+ create mode 100644 drivers/gpu/drm/sitronix/st7571.h
+ create mode 100644 drivers/gpu/drm/sitronix/st7920.c
+ delete mode 100644 drivers/gpu/drm/v3d/v3d_gemfs.c
+ create mode 100644 drivers/gpu/drm/vgem/Kconfig
+ create mode 100644 include/trace/events/dma_buf.h
+Merging amdgpu/drm-next (0a54edc19158 drm/radeon: Convert legacy DRM logging in evergreen.c to drm_* helpers)
+$ git merge -m Merge branch 'drm-next' of https://gitlab.freedesktop.org/agd5f/linux.git amdgpu/drm-next
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_process.c
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_queue.c
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+Auto-merging drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
+Auto-merging drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
+Merge made by the 'ort' strategy.
+ Documentation/gpu/amdgpu/amd_overview_block.svg    |   687 +
+ Documentation/gpu/amdgpu/amdgpu-glossary.rst       |    48 +
+ Documentation/gpu/amdgpu/display/dc-glossary.rst   |     3 -
+ Documentation/gpu/amdgpu/driver-core.rst           |    31 +
+ Documentation/gpu/amdgpu/enforce_isolation.svg     |   654 +
+ Documentation/gpu/amdgpu/gc/index.rst              |    24 +-
+ Documentation/gpu/amdgpu/gfx_pipeline_seq.svg      |   413 +
+ Documentation/gpu/amdgpu/index.rst                 |     1 +
+ Documentation/gpu/amdgpu/no_enforce_isolation.svg  |   707 +
+ Documentation/gpu/amdgpu/process-isolation.rst     |     1 +
+ Documentation/gpu/amdgpu/ring-buffer.rst           |    95 +
+ Documentation/gpu/amdgpu/ring_buffers.svg          |  1633 +
+ Documentation/gpu/amdgpu/userq.rst                 |     2 +
+ drivers/gpu/drm/amd/amdgpu/Makefile                |    19 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu.h                |   128 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c           |    17 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h         |    10 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c   |    10 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c   |     8 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c      |     6 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c           |     6 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c             |     1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c         |   347 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c      |    42 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c        |    18 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h       |    30 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c   |     2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c            |     5 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c            |    13 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c            |    48 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h            |     6 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c            |    52 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h            |    27 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h            |     3 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ip.c             |   315 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ip.h             |   125 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c            |    35 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h            |     1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c            |   112 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h            |   108 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_object.c         |    18 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c            |    31 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h            |     9 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c            |    14 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c           |    11 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h           |     2 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c            |    37 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h            |     7 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h           |    17 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_smuio.h          |     3 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c            |   281 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h            |    39 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c          |    34 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h          |    12 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c          |   131 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h          |    27 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c    |    11 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c           |    29 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h           |     4 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c             |    22 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h             |     4 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c          |     5 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c            |     1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h            |     3 +
+ drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h        |     3 +-
+ drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c         |     3 -
+ drivers/gpu/drm/amd/amdgpu/cik.c                   |     5 -
+ drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c             |    24 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c             |    45 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c             |    43 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c             |  3991 ++
+ drivers/gpu/drm/amd/amdgpu/gfx_v12_1.h             |    31 +
+ drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h         |   475 +
+ drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c              |    64 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c              |    64 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c              |    24 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c            |    33 +-
+ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c          |   879 +
+ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.h          |    31 +
+ drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c             |    23 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c             |    23 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c             |   128 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c             |   603 +
+ drivers/gpu/drm/amd/amdgpu/gmc_v12_1.h             |    30 +
+ drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c              |    43 +-
+ drivers/gpu/drm/amd/amdgpu/ih_v7_0.c               |    30 +-
+ drivers/gpu/drm/amd/amdgpu/imu_v12_1.c             |   160 +
+ drivers/gpu/drm/amd/amdgpu/imu_v12_1.h             |    30 +
+ drivers/gpu/drm/amd/amdgpu/mes_userqueue.c         |    65 +-
+ drivers/gpu/drm/amd/amdgpu/mes_v11_0.c             |    10 +-
+ drivers/gpu/drm/amd/amdgpu/mes_v12_0.c             |    14 +-
+ drivers/gpu/drm/amd/amdgpu/mes_v12_1.c             |  1890 +
+ drivers/gpu/drm/amd/amdgpu/mes_v12_1.h             |    29 +
+ drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c          |   916 +
+ drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.h          |    28 +
+ drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c              |    13 +-
+ drivers/gpu/drm/amd/amdgpu/mxgpu_nv.h              |     4 +-
+ drivers/gpu/drm/amd/amdgpu/nv.c                    |     5 -
+ drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h            |     2 +
+ drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c           |   342 +
+ drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.h           |    30 +
+ drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c             |    14 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c             |    14 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c             |  1813 +
+ drivers/gpu/drm/amd/amdgpu/sdma_v7_1.h             |    31 +
+ drivers/gpu/drm/amd/amdgpu/sdma_v7_1_0_pkt_open.h  |  5673 +++
+ drivers/gpu/drm/amd/amdgpu/si.c                    |     5 -
+ drivers/gpu/drm/amd/amdgpu/smuio_v15_0_8.c         |   213 +
+ drivers/gpu/drm/amd/amdgpu/smuio_v15_0_8.h         |    30 +
+ drivers/gpu/drm/amd/amdgpu/soc15.c                 |     1 -
+ drivers/gpu/drm/amd/amdgpu/soc21.c                 |     5 -
+ drivers/gpu/drm/amd/amdgpu/soc24.c                 |     5 -
+ drivers/gpu/drm/amd/amdgpu/soc_v1_0.c              |   786 +
+ drivers/gpu/drm/amd/amdgpu/soc_v1_0.h              |    34 +
+ drivers/gpu/drm/amd/amdgpu/ta_ras_if.h             |     1 +
+ drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c              |     1 +
+ drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c              |     1 +
+ drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c              |     1 +
+ drivers/gpu/drm/amd/amdgpu/vi.c                    |     5 -
+ drivers/gpu/drm/amd/amdkfd/Makefile                |     3 +
+ drivers/gpu/drm/amd/amdkfd/kfd_chardev.c           |    89 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_crat.c              |     1 +
+ drivers/gpu/drm/amd/amdkfd/kfd_debug.c             |     9 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_device.c            |    75 +-
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c  |    34 +-
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.h  |     2 +
+ .../amd/amdkfd/kfd_device_queue_manager_v12_1.c    |    98 +
+ drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c       |    10 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c |   391 +
+ drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c      |    61 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_migrate.c           |    18 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c |   657 +
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c    |     8 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c |     4 -
+ drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_vi.c |     5 -
+ drivers/gpu/drm/amd/amdkfd/kfd_priv.h              |    34 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_process.c           |   245 +-
+ .../gpu/drm/amd/amdkfd/kfd_process_queue_manager.c |    41 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_queue.c             |    63 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c        |     2 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_svm.c               |    33 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_topology.c          |    27 +
+ drivers/gpu/drm/amd/amdkfd/kfd_topology.h          |     6 +
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c  |    51 +-
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h  |     7 +
+ drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.h     |     8 +
+ .../amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c   |     2 -
+ drivers/gpu/drm/amd/display/dc/core/dc.c           |   575 +-
+ .../gpu/drm/amd/display/dc/core/dc_link_exports.c  |    22 +
+ drivers/gpu/drm/amd/display/dc/core/dc_stream.c    |    31 +-
+ drivers/gpu/drm/amd/display/dc/dc.h                |    89 +-
+ drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c       |    18 +-
+ drivers/gpu/drm/amd/display/dc/dc_dp_types.h       |     1 +
+ drivers/gpu/drm/amd/display/dc/dc_stream.h         |    31 +-
+ drivers/gpu/drm/amd/display/dc/dc_types.h          |    14 +
+ .../gpu/drm/amd/display/dc/dce/dce_link_encoder.c  |    47 +-
+ .../gpu/drm/amd/display/dc/dce/dce_link_encoder.h  |     8 +-
+ .../amd/display/dc/dcn201/dcn201_link_encoder.c    |     3 +
+ .../drm/amd/display/dc/dcn21/dcn21_link_encoder.c  |     3 +
+ .../amd/display/dc/dio/dcn10/dcn10_link_encoder.c  |    43 +
+ .../amd/display/dc/dio/dcn10/dcn10_link_encoder.h  |    11 +-
+ .../amd/display/dc/dio/dcn20/dcn20_link_encoder.c  |     3 +
+ .../display/dc/dio/dcn30/dcn30_dio_link_encoder.c  |     3 +
+ .../dc/dio/dcn301/dcn301_dio_link_encoder.c        |     3 +
+ .../display/dc/dio/dcn31/dcn31_dio_link_encoder.c  |     3 +
+ .../display/dc/dio/dcn32/dcn32_dio_link_encoder.c  |    18 +-
+ .../dc/dio/dcn321/dcn321_dio_link_encoder.c        |     3 +
+ .../display/dc/dio/dcn35/dcn35_dio_link_encoder.c  |     4 +-
+ .../dc/dio/dcn401/dcn401_dio_link_encoder.c        |     3 +
+ drivers/gpu/drm/amd/display/dc/dml2_0/Makefile     |     8 +-
+ .../drm/amd/display/dc/dml2_0/dml21/dml21_utils.c  |     3 +-
+ .../amd/display/dc/dml2_0/dml21/dml21_wrapper.h    |     7 +-
+ .../dml2_0/dml21/inc/dml_top_display_cfg_types.h   |     1 +
+ .../display/dc/dml2_0/dml21/inc/dml_top_types.h    |     1 +
+ .../dml21/src/dml2_core/dml2_core_dcn4_calcs.c     |     5 +-
+ .../dml21/src/dml2_core/dml2_core_shared_types.h   |     1 +
+ .../gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.c   |   599 +-
+ .../gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.h   |     9 +
+ .../drm/amd/display/dc/dml2_0/dml2_wrapper_fpu.c   |   604 +
+ .../drm/amd/display/dc/dml2_0/dml2_wrapper_fpu.h   |    25 +
+ .../gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.h   |     1 +
+ .../gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h |     9 +
+ .../amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c    |    96 +-
+ .../drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.h |     6 +-
+ .../drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c   |    14 +-
+ .../drm/amd/display/dc/hwss/dce110/dce110_hwseq.c  |    30 +-
+ .../drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c  |     2 +-
+ .../drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c    |     8 +-
+ .../drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c    |     3 +-
+ .../drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c  |    18 +-
+ drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h |    37 +
+ .../gpu/drm/amd/display/dc/inc/hw/link_encoder.h   |     4 +
+ drivers/gpu/drm/amd/display/dc/inc/link_service.h  |     7 +-
+ .../gpu/drm/amd/display/dc/link/link_detection.c   |    55 +-
+ drivers/gpu/drm/amd/display/dc/link/link_dpms.c    |    23 +-
+ drivers/gpu/drm/amd/display/dc/link/link_factory.c |   290 +-
+ .../display/dc/link/protocols/link_dp_capability.c |     9 +-
+ .../link_dp_training_fixed_vs_pe_retimer.c         |     2 +-
+ .../dc/link/protocols/link_edp_panel_control.c     |   180 +-
+ .../dc/link/protocols/link_edp_panel_control.h     |     7 +-
+ .../drm/amd/display/dc/link/protocols/link_hpd.c   |   171 +-
+ .../drm/amd/display/dc/link/protocols/link_hpd.h   |     1 -
+ drivers/gpu/drm/amd/display/dc/os_types.h          |     2 -
+ .../amd/display/dc/resource/dcn20/dcn20_resource.c |     6 +-
+ .../amd/display/dc/resource/dcn32/dcn32_resource.h |     5 +-
+ .../amd/display/dc/resource/dcn35/dcn35_resource.c |    10 +-
+ .../display/dc/resource/dcn351/dcn351_resource.c   |    10 +-
+ .../display/dc/resource/dcn401/dcn401_resource.h   |     1 +
+ drivers/gpu/drm/amd/display/dc/sspl/dc_spl.c       |   186 +-
+ drivers/gpu/drm/amd/display/dc/sspl/dc_spl.h       |     8 -
+ .../gpu/drm/amd/display/dc/sspl/dc_spl_filters.c   |     4 +-
+ .../gpu/drm/amd/display/dc/sspl/dc_spl_filters.h   |     4 +-
+ .../amd/display/dc/sspl/dc_spl_isharp_filters.c    |    31 +-
+ .../amd/display/dc/sspl/dc_spl_isharp_filters.h    |    16 +-
+ .../amd/display/dc/sspl/dc_spl_scl_easf_filters.c  |   155 +-
+ .../amd/display/dc/sspl/dc_spl_scl_easf_filters.h  |    35 +-
+ .../drm/amd/display/dc/sspl/dc_spl_scl_filters.c   |    27 +-
+ .../drm/amd/display/dc/sspl/dc_spl_scl_filters.h   |     3 +-
+ .../gpu/drm/amd/display/dc/sspl/spl_custom_float.c |    11 +-
+ .../gpu/drm/amd/display/dc/sspl/spl_custom_float.h |     4 +-
+ .../gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.c   |    78 +-
+ .../gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.h   |    56 +-
+ drivers/gpu/drm/amd/display/dc/sspl/spl_os_types.h |     9 +
+ drivers/gpu/drm/amd/display/dmub/dmub_srv.h        |    33 +
+ drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h    |    76 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c    |    57 +-
+ drivers/gpu/drm/amd/display/include/dpcd_defs.h    |     1 +
+ .../drm/amd/display/modules/inc/mod_info_packet.h  |     6 +
+ .../amd/display/modules/info_packet/info_packet.c  |   369 +-
+ .../drm/amd/include/asic_reg/gc/gc_12_1_0_offset.h | 12407 ++++++
+ .../amd/include/asic_reg/gc/gc_12_1_0_sh_mask.h    | 44620 +++++++++++++++++++
+ .../include/asic_reg/mmhub/mmhub_4_2_0_offset.h    |   802 +
+ .../include/asic_reg/mmhub/mmhub_4_2_0_sh_mask.h   |  3013 ++
+ .../drm/amd/include/asic_reg/mp/mp_15_0_8_offset.h |   868 +
+ .../amd/include/asic_reg/mp/mp_15_0_8_sh_mask.h    |   616 +
+ .../amd/include/asic_reg/oss/osssys_7_1_0_offset.h |   280 +
+ .../include/asic_reg/oss/osssys_7_1_0_sh_mask.h    |  1024 +
+ .../include/asic_reg/smuio/smuio_15_0_8_offset.h   |   512 +
+ .../include/asic_reg/smuio/smuio_15_0_8_sh_mask.h  |  1113 +
+ .../amd/include/ivsrcid/gfx/irqsrcs_gfx_12_1_0.h   |   136 +
+ .../drm/amd/include/ivsrcid/vmc/irqsrcs_vmc_1_0.h  |     2 +
+ drivers/gpu/drm/amd/include/kgd_pp_interface.h     |     4 +-
+ drivers/gpu/drm/amd/include/mes_v12_api_def.h      |    63 +-
+ drivers/gpu/drm/amd/include/soc15_hw_ip.h          |     2 +
+ drivers/gpu/drm/amd/include/soc15_ih_clientid.h    |     1 +
+ drivers/gpu/drm/amd/include/soc_v1_0_enum.h        |    38 +
+ drivers/gpu/drm/amd/include/soc_v1_0_ih_clientid.h |    52 +
+ drivers/gpu/drm/amd/include/v12_structs.h          |  2054 +
+ drivers/gpu/drm/amd/pm/amdgpu_dpm.c                |    19 -
+ drivers/gpu/drm/amd/pm/amdgpu_pm.c                 |    81 +-
+ drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c   |    16 -
+ .../gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c   |    65 +-
+ .../gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c    |    87 +-
+ .../gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.c    |    29 +-
+ .../gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c  |   141 -
+ .../gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c  |    52 +-
+ .../gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c  |   211 +-
+ drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c          |    28 -
+ drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h      |    31 +
+ .../amd/pm/swsmu/inc/pmfw_if/smu_v13_0_12_pmfw.h   |     8 +
+ drivers/gpu/drm/amd/pm/swsmu/inc/smu_v11_0.h       |    49 +-
+ drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h       |    47 +-
+ drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h       |    47 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c  |   188 +-
+ .../drm/amd/pm/swsmu/smu11/cyan_skillfish_ppt.c    |    17 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c    |   387 +-
+ .../drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c    |   212 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c     |    40 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c   |    39 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c    |    52 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c |   249 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c     |    86 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c   |   226 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c  |    34 +
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c   |    24 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_5_ppt.c   |    25 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c   |   385 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h   |     4 +
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c   |   226 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c   |    25 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c     |    86 +-
+ .../gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c   |    61 +-
+ .../gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c   |   236 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c             |   118 +
+ drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h             |    10 +
+ drivers/gpu/drm/amd/ras/ras_mgr/Makefile           |     1 +
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c   |     5 +
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c   |    45 +-
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h   |     1 +
+ .../gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c  |   469 +
+ .../gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.h  |    57 +
+ drivers/gpu/drm/amd/ras/rascore/ras_cmd.h          |    33 +
+ drivers/gpu/drm/amd/ras/rascore/ras_psp.c          |     1 +
+ drivers/gpu/drm/amd/ras/rascore/ras_psp.h          |     1 +
+ drivers/gpu/drm/amd/ras/rascore/ras_ta_if.h        |     1 +
+ drivers/gpu/drm/amd/ras/rascore/ras_umc.c          |    29 +-
+ drivers/gpu/drm/radeon/evergreen.c                 |   112 +-
+ drivers/gpu/drm/radeon/radeon_display.c            |    66 +-
+ include/uapi/drm/amdgpu_drm.h                      |     3 +-
+ include/uapi/linux/kfd_ioctl.h                     |    11 +-
+ include/uapi/linux/kfd_sysfs.h                     |     3 +-
+ 301 files changed, 98421 insertions(+), 5335 deletions(-)
+ create mode 100644 Documentation/gpu/amdgpu/amd_overview_block.svg
+ create mode 100644 Documentation/gpu/amdgpu/enforce_isolation.svg
+ create mode 100644 Documentation/gpu/amdgpu/gfx_pipeline_seq.svg
+ create mode 100644 Documentation/gpu/amdgpu/no_enforce_isolation.svg
+ create mode 100644 Documentation/gpu/amdgpu/ring-buffer.rst
+ create mode 100644 Documentation/gpu/amdgpu/ring_buffers.svg
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/gfx_v12_1.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/gmc_v12_1.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/imu_v12_1.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/imu_v12_1.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/mes_v12_1.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/sdma_v7_1.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/sdma_v7_1_0_pkt_open.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/smuio_v15_0_8.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/smuio_v15_0_8.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/soc_v1_0.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/soc_v1_0.h
+ create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12_1.c
+ create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c
+ create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper_fpu.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper_fpu.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_offset.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_sh_mask.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_4_2_0_offset.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_4_2_0_sh_mask.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/mp/mp_15_0_8_offset.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/mp/mp_15_0_8_sh_mask.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/oss/osssys_7_1_0_offset.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/oss/osssys_7_1_0_sh_mask.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_15_0_8_offset.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_15_0_8_sh_mask.h
+ create mode 100644 drivers/gpu/drm/amd/include/ivsrcid/gfx/irqsrcs_gfx_12_1_0.h
+ create mode 100644 drivers/gpu/drm/amd/include/soc_v1_0_enum.h
+ create mode 100644 drivers/gpu/drm/amd/include/soc_v1_0_ih_clientid.h
+ create mode 100644 drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c
+ create mode 100644 drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.h
+Merging drm-intel/for-linux-next (0c8afa67aaa2 drm/i915: Add intel_gvt_driver_remove() onto error cleanup path)
+$ git merge -m Merge branch 'for-linux-next' of https://gitlab.freedesktop.org/drm/i915/kernel.git drm-intel/for-linux-next
+Auto-merging drivers/gpu/drm/i915/Makefile
+Auto-merging drivers/gpu/drm/i915/i915_drv.h
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/i915/Makefile                      |   19 +-
+ drivers/gpu/drm/i915/display/g4x_dp.c              |    5 +-
+ drivers/gpu/drm/i915/display/g4x_hdmi.c            |    2 +-
+ drivers/gpu/drm/i915/display/i9xx_plane.c          |    9 +-
+ drivers/gpu/drm/i915/display/i9xx_wm.c             |    5 +-
+ drivers/gpu/drm/i915/display/icl_dsi.c             |    4 +-
+ drivers/gpu/drm/i915/display/intel_alpm.c          |    6 +-
+ drivers/gpu/drm/i915/display/intel_audio.c         |    6 +-
+ drivers/gpu/drm/i915/display/intel_bios.c          |   76 +-
+ drivers/gpu/drm/i915/display/intel_bios.h          |    2 +
+ drivers/gpu/drm/i915/display/intel_bo.c            |    6 +
+ drivers/gpu/drm/i915/display/intel_bo.h            |    1 +
+ drivers/gpu/drm/i915/display/intel_bw.c            |    5 +-
+ drivers/gpu/drm/i915/display/intel_cdclk.c         |   13 +-
+ drivers/gpu/drm/i915/display/intel_cmtg.c          |    3 +-
+ drivers/gpu/drm/i915/display/intel_connector.c     |   14 +-
+ drivers/gpu/drm/i915/display/intel_crt.c           |    6 +-
+ drivers/gpu/drm/i915/display/intel_crtc.c          |   31 +-
+ drivers/gpu/drm/i915/display/intel_crtc.h          |    2 +-
+ drivers/gpu/drm/i915/display/intel_cursor.c        |    5 +-
+ drivers/gpu/drm/i915/display/intel_cx0_phy.c       | 1074 +++++++++++---------
+ drivers/gpu/drm/i915/display/intel_cx0_phy.h       |   29 +-
+ drivers/gpu/drm/i915/display/intel_ddi.c           |  116 ++-
+ drivers/gpu/drm/i915/display/intel_display.c       |  102 +-
+ drivers/gpu/drm/i915/display/intel_display.h       |    1 +
+ drivers/gpu/drm/i915/display/intel_display_core.h  |   18 +-
+ .../gpu/drm/i915/display/intel_display_debugfs.c   |    2 +-
+ .../gpu/drm/i915/display/intel_display_device.c    |    5 +
+ .../gpu/drm/i915/display/intel_display_device.h    |    9 +-
+ .../gpu/drm/i915/display/intel_display_driver.c    |   18 +-
+ drivers/gpu/drm/i915/display/intel_display_irq.c   |  203 ++--
+ drivers/gpu/drm/i915/display/intel_display_irq.h   |    2 +
+ drivers/gpu/drm/i915/display/intel_display_power.c |   40 +-
+ drivers/gpu/drm/i915/display/intel_display_power.h |   48 +-
+ .../drm/i915/display/intel_display_power_well.c    |   64 +-
+ drivers/gpu/drm/i915/display/intel_display_regs.h  |   23 +-
+ drivers/gpu/drm/i915/display/intel_display_reset.c |    5 +-
+ drivers/gpu/drm/i915/display/intel_display_rps.c   |   31 +-
+ drivers/gpu/drm/i915/display/intel_display_rps.h   |   21 -
+ drivers/gpu/drm/i915/display/intel_display_types.h |   52 +-
+ drivers/gpu/drm/i915/display/intel_display_wa.c    |    4 +
+ drivers/gpu/drm/i915/display/intel_display_wa.h    |    2 +
+ drivers/gpu/drm/i915/display/intel_dmc.c           |    2 +-
+ drivers/gpu/drm/i915/display/intel_dp.c            |   20 +-
+ drivers/gpu/drm/i915/display/intel_dp_aux.c        |   16 +-
+ drivers/gpu/drm/i915/display/intel_dpll.c          |   24 +-
+ drivers/gpu/drm/i915/display/intel_dpll_mgr.c      |  336 +++++-
+ drivers/gpu/drm/i915/display/intel_dpll_mgr.h      |   11 +-
+ drivers/gpu/drm/i915/{soc => display}/intel_dram.c |  397 ++++----
+ drivers/gpu/drm/i915/{soc => display}/intel_dram.h |   12 +-
+ drivers/gpu/drm/i915/display/intel_dsb.c           |   42 +-
+ drivers/gpu/drm/i915/display/intel_dsb_buffer.c    |   43 +-
+ drivers/gpu/drm/i915/display/intel_dsb_buffer.h    |   13 +-
+ drivers/gpu/drm/i915/display/intel_dsi.h           |    7 +-
+ drivers/gpu/drm/i915/display/intel_fb.c            |    6 +-
+ drivers/gpu/drm/i915/display/intel_fbc.c           |  266 +++--
+ drivers/gpu/drm/i915/display/intel_fbc.h           |    3 +
+ drivers/gpu/drm/i915/display/intel_fbc_regs.h      |   12 +
+ drivers/gpu/drm/i915/display/intel_fifo_underrun.c |  109 ++
+ drivers/gpu/drm/i915/display/intel_gmbus.c         |   10 +-
+ drivers/gpu/drm/i915/display/intel_hdcp.c          |    4 +-
+ drivers/gpu/drm/i915/display/intel_hdcp_gsc.h      |   22 -
+ .../gpu/drm/i915/display/intel_hdcp_gsc_message.c  |   78 +-
+ drivers/gpu/drm/i915/display/intel_hdmi.c          |    4 +-
+ drivers/gpu/drm/i915/display/intel_hotplug.c       |    8 +-
+ drivers/gpu/drm/i915/display/intel_hotplug_irq.c   |    5 +-
+ drivers/gpu/drm/i915/display/intel_lpe_audio.c     |    1 -
+ drivers/gpu/drm/i915/display/intel_lt_phy.c        |   18 +-
+ drivers/gpu/drm/i915/display/intel_lt_phy.h        |    2 -
+ drivers/gpu/drm/i915/display/intel_lvds.c          |    2 +-
+ drivers/gpu/drm/i915/display/intel_modeset_setup.c |    2 +-
+ .../gpu/drm/i915/display/intel_modeset_verify.c    |    1 -
+ drivers/gpu/drm/i915/display/intel_panic.c         |   27 -
+ drivers/gpu/drm/i915/display/intel_panic.h         |   14 -
+ drivers/gpu/drm/i915/display/intel_parent.c        |  197 ++++
+ drivers/gpu/drm/i915/display/intel_parent.h        |   66 ++
+ drivers/gpu/drm/i915/display/intel_pipe_crc.c      |    8 +-
+ drivers/gpu/drm/i915/display/intel_plane.c         |   96 +-
+ drivers/gpu/drm/i915/display/intel_plane.h         |    4 +-
+ drivers/gpu/drm/i915/display/intel_plane_initial.c |    6 +-
+ drivers/gpu/drm/i915/display/intel_pps.c           |   63 +-
+ drivers/gpu/drm/i915/display/intel_pps.h           |   14 +-
+ drivers/gpu/drm/i915/display/intel_psr.c           |  317 +++---
+ drivers/gpu/drm/i915/display/intel_psr.h           |    5 +-
+ drivers/gpu/drm/i915/{soc => display}/intel_rom.c  |    0
+ drivers/gpu/drm/i915/{soc => display}/intel_rom.h  |    0
+ drivers/gpu/drm/i915/display/intel_sprite.c        |    9 +-
+ drivers/gpu/drm/i915/display/intel_tc.c            |   40 +-
+ drivers/gpu/drm/i915/display/intel_vbt_defs.h      |    3 +-
+ drivers/gpu/drm/i915/display/intel_vdsc.c          |    2 +-
+ drivers/gpu/drm/i915/display/intel_vga.c           |   44 +-
+ drivers/gpu/drm/i915/display/skl_universal_plane.c |   19 +-
+ drivers/gpu/drm/i915/display/skl_watermark.c       |    8 +-
+ drivers/gpu/drm/i915/display/vlv_dsi.c             |    2 +-
+ drivers/gpu/drm/i915/gem/i915_gem_stolen.c         |   52 +-
+ drivers/gpu/drm/i915/gem/i915_gem_stolen.h         |   21 +-
+ drivers/gpu/drm/i915/gt/intel_engine_cs.c          |    4 +-
+ drivers/gpu/drm/i915/gt/intel_gt.c                 |    9 +-
+ drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c     |    4 +-
+ drivers/gpu/drm/i915/gt/intel_gt_print.h           |    5 +-
+ drivers/gpu/drm/i915/gt/intel_lrc.c                |    4 -
+ drivers/gpu/drm/i915/gt/intel_rps.c                |   40 +-
+ drivers/gpu/drm/i915/gt/intel_rps.h                |    2 +
+ drivers/gpu/drm/i915/gt/intel_wopcm.c              |    3 -
+ drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c          |    8 +-
+ drivers/gpu/drm/i915/gt/uc/intel_huc.c             |    4 -
+ drivers/gpu/drm/i915/gt/uc/intel_uc.c              |   17 +-
+ drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c           |   60 +-
+ drivers/gpu/drm/i915/i915_driver.c                 |   73 +-
+ drivers/gpu/drm/i915/i915_drv.h                    |    3 -
+ drivers/gpu/drm/i915/i915_edram.c                  |   44 +
+ drivers/gpu/drm/i915/i915_edram.h                  |   11 +
+ drivers/gpu/drm/i915/i915_freq.c                   |  111 ++
+ drivers/gpu/drm/i915/i915_freq.h                   |   13 +
+ drivers/gpu/drm/i915/i915_gem.c                    |    4 +-
+ .../gpu/drm/i915/{soc/intel_gmch.c => i915_gmch.c} |   61 +-
+ drivers/gpu/drm/i915/i915_gmch.h                   |   13 +
+ .../{display/intel_hdcp_gsc.c => i915_hdcp_gsc.c}  |   22 +-
+ drivers/gpu/drm/i915/i915_hdcp_gsc.h               |    9 +
+ drivers/gpu/drm/i915/i915_irq.c                    |   16 +
+ drivers/gpu/drm/i915/i915_irq.h                    |    2 +
+ drivers/gpu/drm/i915/i915_panic.c                  |   35 +
+ drivers/gpu/drm/i915/i915_panic.h                  |    9 +
+ drivers/gpu/drm/i915/i915_params.c                 |    5 -
+ drivers/gpu/drm/i915/i915_params.h                 |    1 -
+ drivers/gpu/drm/i915/i915_pci.c                    |    6 +-
+ drivers/gpu/drm/i915/i915_utils.c                  |   30 +-
+ drivers/gpu/drm/i915/i915_utils.h                  |   22 +-
+ drivers/gpu/drm/i915/intel_clock_gating.c          |   10 +-
+ drivers/gpu/drm/i915/intel_clock_gating.h          |    6 +-
+ drivers/gpu/drm/i915/intel_gvt.c                   |    3 -
+ drivers/gpu/drm/i915/intel_mchbar_regs.h           |   68 +-
+ drivers/gpu/drm/i915/intel_uncore.c                |    4 +-
+ drivers/gpu/drm/i915/intel_wakeref.c               |    2 +-
+ drivers/gpu/drm/i915/intel_wakeref.h               |   14 +-
+ drivers/gpu/drm/i915/soc/intel_gmch.h              |   20 -
+ drivers/gpu/drm/i915/vlv_suspend.c                 |    2 +-
+ drivers/gpu/drm/xe/Makefile                        |   19 +-
+ .../xe/compat-i915-headers/gem/i915_gem_object.h   |   13 -
+ .../xe/compat-i915-headers/gem/i915_gem_stolen.h   |   40 -
+ .../drm/xe/compat-i915-headers/gt/intel_gt_types.h |   11 -
+ .../gpu/drm/xe/compat-i915-headers/i915_active.h   |   22 -
+ .../drm/xe/compat-i915-headers/i915_active_types.h |   13 -
+ drivers/gpu/drm/xe/compat-i915-headers/i915_drv.h  |   15 -
+ drivers/gpu/drm/xe/compat-i915-headers/i915_irq.h  |    6 -
+ .../gpu/drm/xe/compat-i915-headers/i915_utils.h    |    6 -
+ drivers/gpu/drm/xe/compat-i915-headers/i915_vgpu.h |   18 -
+ .../gpu/drm/xe/compat-i915-headers/intel_wakeref.h |   10 -
+ .../gpu/drm/xe/compat-i915-headers/pxp/intel_pxp.h |   29 -
+ .../drm/xe/compat-i915-headers/soc/intel_dram.h    |    6 -
+ .../drm/xe/compat-i915-headers/soc/intel_gmch.h    |    6 -
+ .../gpu/drm/xe/compat-i915-headers/soc/intel_rom.h |    6 -
+ drivers/gpu/drm/xe/display/ext/i915_irq.c          |   85 --
+ drivers/gpu/drm/xe/display/intel_bo.c              |    8 +-
+ drivers/gpu/drm/xe/display/xe_display.c            |   28 +-
+ drivers/gpu/drm/xe/display/xe_display_misc.c       |   16 -
+ drivers/gpu/drm/xe/display/xe_display_rpm.c        |    3 +
+ drivers/gpu/drm/xe/display/xe_dsb_buffer.c         |   49 +-
+ drivers/gpu/drm/xe/display/xe_hdcp_gsc.c           |   27 +-
+ drivers/gpu/drm/xe/display/xe_hdcp_gsc.h           |    9 +
+ drivers/gpu/drm/xe/display/xe_panic.c              |   16 +-
+ drivers/gpu/drm/xe/display/xe_panic.h              |    9 +
+ drivers/gpu/drm/xe/display/xe_plane_initial.c      |    6 +-
+ drivers/gpu/drm/xe/display/xe_stolen.c             |   62 +-
+ drivers/gpu/drm/xe/display/xe_stolen.h             |    9 +
+ drivers/gpu/drm/xe/xe_device_types.h               |    8 -
+ include/drm/intel/display_parent_interface.h       |   79 ++
+ 167 files changed, 3544 insertions(+), 2451 deletions(-)
+ rename drivers/gpu/drm/i915/{soc => display}/intel_dram.c (57%)
+ rename drivers/gpu/drm/i915/{soc => display}/intel_dram.h (68%)
+ delete mode 100644 drivers/gpu/drm/i915/display/intel_hdcp_gsc.h
+ delete mode 100644 drivers/gpu/drm/i915/display/intel_panic.c
+ delete mode 100644 drivers/gpu/drm/i915/display/intel_panic.h
+ create mode 100644 drivers/gpu/drm/i915/display/intel_parent.c
+ create mode 100644 drivers/gpu/drm/i915/display/intel_parent.h
+ rename drivers/gpu/drm/i915/{soc => display}/intel_rom.c (100%)
+ rename drivers/gpu/drm/i915/{soc => display}/intel_rom.h (100%)
+ create mode 100644 drivers/gpu/drm/i915/i915_edram.c
+ create mode 100644 drivers/gpu/drm/i915/i915_edram.h
+ create mode 100644 drivers/gpu/drm/i915/i915_freq.c
+ create mode 100644 drivers/gpu/drm/i915/i915_freq.h
+ rename drivers/gpu/drm/i915/{soc/intel_gmch.c => i915_gmch.c} (68%)
+ create mode 100644 drivers/gpu/drm/i915/i915_gmch.h
+ rename drivers/gpu/drm/i915/{display/intel_hdcp_gsc.c => i915_hdcp_gsc.c} (89%)
+ create mode 100644 drivers/gpu/drm/i915/i915_hdcp_gsc.h
+ create mode 100644 drivers/gpu/drm/i915/i915_panic.c
+ create mode 100644 drivers/gpu/drm/i915/i915_panic.h
+ delete mode 100644 drivers/gpu/drm/i915/soc/intel_gmch.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/gem/i915_gem_object.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/gem/i915_gem_stolen.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/gt/intel_gt_types.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/i915_active.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/i915_active_types.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/i915_irq.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/i915_vgpu.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/intel_wakeref.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/pxp/intel_pxp.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/soc/intel_dram.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/soc/intel_gmch.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/soc/intel_rom.h
+ delete mode 100644 drivers/gpu/drm/xe/display/ext/i915_irq.c
+ delete mode 100644 drivers/gpu/drm/xe/display/xe_display_misc.c
+ create mode 100644 drivers/gpu/drm/xe/display/xe_hdcp_gsc.h
+ create mode 100644 drivers/gpu/drm/xe/display/xe_panic.h
+ create mode 100644 drivers/gpu/drm/xe/display/xe_stolen.h
+Merging drm-msm/msm-next (7bc29d5fb6fa dt-bindings: display/msm/gmu: Add Adreno X2-85 GMU)
+$ git merge -m Merge branch 'msm-next' of https://gitlab.freedesktop.org/drm/msm.git drm-msm/msm-next
+Already up to date.
+Merging drm-msm-lumag/msm-next-lumag (e2f085ab8636 dt-bindings: display: msm: sm6150-mdss: Fix example indentation and OPP values)
+$ git merge -m Merge branch 'msm-next-lumag' of https://gitlab.freedesktop.org/lumag/msm.git drm-msm-lumag/msm-next-lumag
+Already up to date.
+Merging drm-rust/drm-rust-next (97cf6bc0abd3 rust: drm: use `pin_init::zeroed()` for file operations initialization)
+$ git merge -m Merge branch 'drm-rust-next' of https://gitlab.freedesktop.org/drm/rust/kernel.git drm-rust/drm-rust-next
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/tyr/driver.rs          | 2 +-
+ drivers/gpu/nova-core/gsp/sequencer.rs | 4 ++--
+ drivers/gpu/nova-core/vbios.rs         | 2 +-
+ rust/helpers/drm.c                     | 7 ++++---
+ rust/kernel/drm/gem/mod.rs             | 6 ++----
+ 5 files changed, 10 insertions(+), 11 deletions(-)
+Merging drm-nova/nova-next (93296e9d9528 gpu: nova-core: vbios: store reference to Device where relevant)
+$ git merge -m Merge branch 'nova-next' of https://gitlab.freedesktop.org/drm/nova.git drm-nova/nova-next
+Already up to date.
+Merging drm-xe/drm-xe-next (c7b83a916d62 drm/xe: Fix documentation heading levels in xe_guc_pc.c)
+$ git merge -m Merge branch 'drm-xe-next' of https://gitlab.freedesktop.org/drm/xe/kernel.git drm-xe/drm-xe-next
+Auto-merging .mailmap
+Auto-merging MAINTAINERS
+Auto-merging drivers/gpu/drm/xe/Makefile
+Auto-merging drivers/gpu/drm/xe/display/xe_hdcp_gsc.c
+Auto-merging drivers/gpu/drm/xe/tests/xe_mocs.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/xe/tests/xe_mocs.c
+Auto-merging drivers/gpu/drm/xe/xe_devcoredump.c
+Auto-merging drivers/gpu/drm/xe/xe_device_types.h
+Auto-merging drivers/gpu/drm/xe/xe_exec_queue.c
+Auto-merging drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c
+Auto-merging drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c
+Auto-merging drivers/gpu/drm/xe/xe_guc_ct.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/xe/xe_guc_ct.c
+Auto-merging drivers/gpu/drm/xe/xe_guc_submit.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/xe/xe_guc_submit.c
+Auto-merging drivers/gpu/drm/xe/xe_irq.c
+Auto-merging drivers/gpu/drm/xe/xe_pci.c
+Auto-merging drivers/gpu/drm/xe/xe_pm.c
+Auto-merging drivers/gpu/drm/xe/xe_sriov_vfio.c
+CONFLICT (add/add): Merge conflict in drivers/gpu/drm/xe/xe_sriov_vfio.c
+Auto-merging drivers/gpu/drm/xe/xe_vm.c
+Auto-merging drivers/gpu/drm/xe/xe_vram.c
+Auto-merging drivers/vfio/pci/Kconfig
+Auto-merging drivers/vfio/pci/Makefile
+Resolved 'drivers/gpu/drm/xe/tests/xe_mocs.c' using previous resolution.
+Resolved 'drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c' using previous resolution.
+Resolved 'drivers/gpu/drm/xe/xe_guc_ct.c' using previous resolution.
+Resolved 'drivers/gpu/drm/xe/xe_guc_submit.c' using previous resolution.
+Resolved 'drivers/gpu/drm/xe/xe_sriov_vfio.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 816a7181ac38] Merge branch 'drm-xe-next' of https://gitlab.freedesktop.org/drm/xe/kernel.git
+$ git diff -M --stat --summary HEAD^..
+ .mailmap                                           |   1 +
+ .../ABI/testing/sysfs-driver-intel-xe-sriov        |   2 +-
+ Documentation/gpu/xe/xe_exec_queue.rst             |  14 +
+ MAINTAINERS                                        |   2 +-
+ drivers/gpu/drm/drm_gpusvm.c                       |   3 +
+ drivers/gpu/drm/xe/Makefile                        |   2 +
+ drivers/gpu/drm/xe/abi/guc_actions_abi.h           |   6 +
+ drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h     |  67 +-
+ drivers/gpu/drm/xe/abi/guc_klvs_abi.h              |   9 +
+ drivers/gpu/drm/xe/abi/guc_lfd_abi.h               | 171 +++++
+ drivers/gpu/drm/xe/abi/guc_lic_abi.h               |  77 +++
+ drivers/gpu/drm/xe/abi/guc_log_abi.h               |  42 +-
+ drivers/gpu/drm/xe/display/xe_fb_pin.c             |  23 +-
+ drivers/gpu/drm/xe/display/xe_hdcp_gsc.c           |  31 +-
+ drivers/gpu/drm/xe/instructions/xe_gpu_commands.h  |   1 +
+ drivers/gpu/drm/xe/regs/xe_gt_regs.h               |   3 +
+ drivers/gpu/drm/xe/regs/xe_gtt_defs.h              |   1 +
+ drivers/gpu/drm/xe/regs/xe_guc_regs.h              |   3 +
+ drivers/gpu/drm/xe/regs/xe_irq_regs.h              |   1 +
+ drivers/gpu/drm/xe/regs/xe_mert_regs.h             |  21 +
+ drivers/gpu/drm/xe/regs/xe_oa_regs.h               |  17 +
+ drivers/gpu/drm/xe/tests/xe_args_test.c            |  54 ++
+ drivers/gpu/drm/xe/tests/xe_bo.c                   |  10 +-
+ drivers/gpu/drm/xe/tests/xe_dma_buf.c              |   3 +-
+ drivers/gpu/drm/xe/tests/xe_migrate.c              |  10 +-
+ drivers/gpu/drm/xe/tests/xe_mocs.c                 |  27 +-
+ drivers/gpu/drm/xe/xe_args.h                       |  27 +
+ drivers/gpu/drm/xe/xe_bo.c                         |  41 +-
+ drivers/gpu/drm/xe/xe_bo.h                         |   1 +
+ drivers/gpu/drm/xe/xe_debugfs.c                    | 143 +++-
+ drivers/gpu/drm/xe/xe_devcoredump.c                |  30 +-
+ drivers/gpu/drm/xe/xe_device.c                     | 102 ++-
+ drivers/gpu/drm/xe/xe_device.h                     |   5 +
+ drivers/gpu/drm/xe/xe_device_sysfs.c               |  33 +-
+ drivers/gpu/drm/xe/xe_device_types.h               |  27 +
+ drivers/gpu/drm/xe/xe_dma_buf.c                    |   2 +-
+ drivers/gpu/drm/xe/xe_drm_client.c                 |  67 +-
+ drivers/gpu/drm/xe/xe_eu_stall.c                   |   2 +-
+ drivers/gpu/drm/xe/xe_exec.c                       |  12 +-
+ drivers/gpu/drm/xe/xe_exec_queue.c                 | 455 ++++++++++++-
+ drivers/gpu/drm/xe/xe_exec_queue.h                 |  68 ++
+ drivers/gpu/drm/xe/xe_exec_queue_types.h           |  62 ++
+ drivers/gpu/drm/xe/xe_execlist.c                   |   2 +-
+ drivers/gpu/drm/xe/xe_force_wake.c                 |   7 +
+ drivers/gpu/drm/xe/xe_force_wake.h                 |  40 ++
+ drivers/gpu/drm/xe/xe_ggtt.c                       |   3 +-
+ drivers/gpu/drm/xe/xe_gsc.c                        |  21 +-
+ drivers/gpu/drm/xe/xe_gsc_debugfs.c                |   3 +-
+ drivers/gpu/drm/xe/xe_gsc_proxy.c                  |  17 +-
+ drivers/gpu/drm/xe/xe_gt.c                         | 181 +++---
+ drivers/gpu/drm/xe/xe_gt.h                         |   2 +
+ drivers/gpu/drm/xe/xe_gt_debugfs.c                 |  30 +-
+ drivers/gpu/drm/xe/xe_gt_freq.c                    |  31 +-
+ drivers/gpu/drm/xe/xe_gt_idle.c                    |  49 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c         |  19 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c        |  21 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c      |   2 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_vf.c                | 172 +++--
+ drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c        |  12 +
+ drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h          |  13 +
+ drivers/gpu/drm/xe/xe_gt_stats.c                   |  10 +
+ drivers/gpu/drm/xe/xe_gt_stats.h                   |  32 +
+ drivers/gpu/drm/xe/xe_gt_stats_types.h             |   5 +
+ drivers/gpu/drm/xe/xe_gt_throttle.c                |  11 +-
+ drivers/gpu/drm/xe/xe_gt_types.h                   |   5 +
+ drivers/gpu/drm/xe/xe_guc.c                        |  80 ++-
+ drivers/gpu/drm/xe/xe_guc.h                        |  23 +
+ drivers/gpu/drm/xe/xe_guc_ads.c                    |   6 +-
+ drivers/gpu/drm/xe/xe_guc_buf.c                    |   2 +-
+ drivers/gpu/drm/xe/xe_guc_capture.c                |  16 +-
+ drivers/gpu/drm/xe/xe_guc_ct.c                     | 273 +++++---
+ drivers/gpu/drm/xe/xe_guc_ct.h                     |   2 +
+ drivers/gpu/drm/xe/xe_guc_debugfs.c                |  15 +-
+ drivers/gpu/drm/xe/xe_guc_fwif.h                   |  10 +-
+ .../gpu/drm/xe/xe_guc_klv_thresholds_set_types.h   |   6 +
+ drivers/gpu/drm/xe/xe_guc_log.c                    | 507 ++++++++++++---
+ drivers/gpu/drm/xe/xe_guc_log.h                    |  30 +-
+ drivers/gpu/drm/xe/xe_guc_pc.c                     |  66 +-
+ drivers/gpu/drm/xe/xe_guc_submit.c                 | 717 +++++++++++++++++++--
+ drivers/gpu/drm/xe/xe_guc_submit.h                 |   9 +-
+ drivers/gpu/drm/xe/xe_guc_submit_types.h           |  13 +
+ drivers/gpu/drm/xe/xe_guc_tlb_inval.c              |  41 +-
+ drivers/gpu/drm/xe/xe_heci_gsc.c                   |   4 +-
+ drivers/gpu/drm/xe/xe_huc.c                        |   7 +-
+ drivers/gpu/drm/xe/xe_huc_debugfs.c                |   3 +-
+ drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c      |  16 +-
+ drivers/gpu/drm/xe/xe_hw_engine_group.c            |  79 ++-
+ drivers/gpu/drm/xe/xe_hw_engine_group.h            |   4 +-
+ drivers/gpu/drm/xe/xe_hwmon.c                      |  52 +-
+ drivers/gpu/drm/xe/xe_i2c.c                        |   2 +-
+ drivers/gpu/drm/xe/xe_irq.c                        |   2 +
+ drivers/gpu/drm/xe/xe_lmtt.c                       |  25 +-
+ drivers/gpu/drm/xe/xe_lrc.c                        |  73 ++-
+ drivers/gpu/drm/xe/xe_lrc.h                        |   7 +-
+ drivers/gpu/drm/xe/xe_lrc_types.h                  |   3 +
+ drivers/gpu/drm/xe/xe_mert.c                       |  82 +++
+ drivers/gpu/drm/xe/xe_mert.h                       |  32 +
+ drivers/gpu/drm/xe/xe_migrate.c                    |  57 +-
+ drivers/gpu/drm/xe/xe_migrate.h                    |   3 +
+ drivers/gpu/drm/xe/xe_mocs.c                       |  18 +-
+ drivers/gpu/drm/xe/xe_nvm.c                        |  34 +-
+ drivers/gpu/drm/xe/xe_oa.c                         | 106 ++-
+ drivers/gpu/drm/xe/xe_oa_types.h                   |   1 +
+ drivers/gpu/drm/xe/xe_page_reclaim.c               | 136 ++++
+ drivers/gpu/drm/xe/xe_page_reclaim.h               | 105 +++
+ drivers/gpu/drm/xe/xe_pat.c                        | 220 +++++--
+ drivers/gpu/drm/xe/xe_pat.h                        |  21 +
+ drivers/gpu/drm/xe/xe_pci.c                        |  29 +
+ drivers/gpu/drm/xe/xe_pci_sriov.c                  |  10 +-
+ drivers/gpu/drm/xe/xe_pci_types.h                  |   6 +
+ drivers/gpu/drm/xe/xe_pcode_api.h                  |   2 +
+ drivers/gpu/drm/xe/xe_pm.c                         |  10 +-
+ drivers/gpu/drm/xe/xe_pt.c                         | 135 +++-
+ drivers/gpu/drm/xe/xe_pt_types.h                   |   5 +
+ drivers/gpu/drm/xe/xe_pxp.c                        |  55 +-
+ drivers/gpu/drm/xe/xe_query.c                      |  23 +-
+ drivers/gpu/drm/xe/xe_reg_sr.c                     |  17 +-
+ drivers/gpu/drm/xe/xe_reg_whitelist.c              |  81 ++-
+ drivers/gpu/drm/xe/xe_ring_ops.c                   |  70 +-
+ drivers/gpu/drm/xe/xe_sa.c                         |  67 +-
+ drivers/gpu/drm/xe/xe_sa.h                         |  20 +-
+ drivers/gpu/drm/xe/xe_sa_types.h                   |   3 +
+ drivers/gpu/drm/xe/xe_sriov_packet.c               |   2 +-
+ drivers/gpu/drm/xe/xe_sriov_pf.c                   |   4 +
+ drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c           |   6 +-
+ drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c             |  16 +-
+ drivers/gpu/drm/xe/xe_sriov_vf.c                   |  84 ++-
+ drivers/gpu/drm/xe/xe_sriov_vf_ccs.c               |  27 +-
+ drivers/gpu/drm/xe/xe_sriov_vf_ccs.h               |   1 +
+ drivers/gpu/drm/xe/xe_sriov_vfio.c                 |   2 +-
+ drivers/gpu/drm/xe/xe_survivability_mode.c         | 275 +++++---
+ drivers/gpu/drm/xe/xe_survivability_mode_types.h   |  28 +-
+ drivers/gpu/drm/xe/xe_svm.c                        |  88 ++-
+ drivers/gpu/drm/xe/xe_svm.h                        |   2 +-
+ drivers/gpu/drm/xe/xe_sync.c                       |  33 +-
+ drivers/gpu/drm/xe/xe_sync.h                       |   2 +
+ drivers/gpu/drm/xe/xe_tile.c                       |   5 +
+ drivers/gpu/drm/xe/xe_tile_debugfs.c               |  17 +-
+ drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c      |   3 +-
+ drivers/gpu/drm/xe/xe_tlb_inval.c                  |  27 +-
+ drivers/gpu/drm/xe/xe_tlb_inval.h                  |   2 +-
+ drivers/gpu/drm/xe/xe_tlb_inval_job.c              |  36 +-
+ drivers/gpu/drm/xe/xe_tlb_inval_job.h              |   4 +
+ drivers/gpu/drm/xe/xe_tlb_inval_types.h            |   5 +-
+ drivers/gpu/drm/xe/xe_trace.h                      |  46 ++
+ drivers/gpu/drm/xe/xe_uc.c                         |  35 +-
+ drivers/gpu/drm/xe/xe_uc.h                         |   2 +
+ drivers/gpu/drm/xe/xe_uc_fw.c                      |  10 +-
+ drivers/gpu/drm/xe/xe_vm.c                         | 158 ++++-
+ drivers/gpu/drm/xe/xe_vm.h                         |   3 +
+ drivers/gpu/drm/xe/xe_vm_types.h                   |   3 +-
+ drivers/gpu/drm/xe/xe_vram.c                       |   6 +-
+ drivers/gpu/drm/xe/xe_wa.c                         |  60 +-
+ drivers/gpu/drm/xe/xe_wa_oob.rules                 |   7 +-
+ include/uapi/drm/xe_drm.h                          |  82 ++-
+ 155 files changed, 5372 insertions(+), 1516 deletions(-)
+ create mode 100644 drivers/gpu/drm/xe/abi/guc_lfd_abi.h
+ create mode 100644 drivers/gpu/drm/xe/abi/guc_lic_abi.h
+ create mode 100644 drivers/gpu/drm/xe/regs/xe_mert_regs.h
+ create mode 100644 drivers/gpu/drm/xe/xe_mert.c
+ create mode 100644 drivers/gpu/drm/xe/xe_mert.h
+ create mode 100644 drivers/gpu/drm/xe/xe_page_reclaim.c
+ create mode 100644 drivers/gpu/drm/xe/xe_page_reclaim.h
+Merging etnaviv/etnaviv/next (6bde14ba5f7e drm/etnaviv: add optional reset support)
+$ git merge -m Merge branch 'etnaviv/next' of https://git.pengutronix.de/git/lst/linux etnaviv/etnaviv/next
+Already up to date.
+Merging fbdev/for-next (7d0a66e4bb90 Linux 6.18)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/deller/linux-fbdev.git fbdev/for-next
+Already up to date.
+Merging regmap/for-next (82845bc8d13c Merge branch 'regmap-linus' into regmap-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git regmap/for-next
+Merge made by the 'ort' strategy.
+Merging sound/for-next (03f705b9ca58 ALSA: usb-audio: Relax __free() variable declarations)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git sound/for-next
+Auto-merging sound/hda/codecs/realtek/alc269.c
+Merge made by the 'ort' strategy.
+ include/sound/seq_device.h           |   2 +
+ sound/core/compress_offload.c        |  28 +++---
+ sound/core/control.c                 |  12 +--
+ sound/core/control_compat.c          |  21 +++--
+ sound/core/control_led.c             |  12 +--
+ sound/core/oss/mixer_oss.c           |  64 +++++++------
+ sound/core/oss/pcm_oss.c             |  19 ++--
+ sound/core/pcm.c                     |   4 +-
+ sound/core/pcm_compat.c              |   9 +-
+ sound/core/pcm_native.c              |  50 +++++-----
+ sound/core/seq/oss/seq_oss.c         |   4 +-
+ sound/core/seq/oss/seq_oss_init.c    |   4 +-
+ sound/core/seq/oss/seq_oss_midi.c    |  45 +++++----
+ sound/core/seq/oss/seq_oss_synth.c   |  35 ++++---
+ sound/core/seq/oss/seq_oss_synth.h   |   4 +-
+ sound/core/seq/seq_clientmgr.c       | 171 ++++++++++++++++++-----------------
+ sound/core/seq/seq_compat.c          |   4 +-
+ sound/core/seq/seq_midi.c            |  25 +++--
+ sound/core/seq/seq_ports.c           |  11 +--
+ sound/core/seq/seq_queue.c           |  32 ++++---
+ sound/core/seq/seq_ump_client.c      |  27 +++---
+ sound/core/seq/seq_virmidi.c         |   4 +-
+ sound/core/seq_device.c              |  26 +++++-
+ sound/core/timer.c                   |  12 +--
+ sound/core/vmaster.c                 |  12 +--
+ sound/drivers/opl3/opl3_seq.c        |  13 +--
+ sound/drivers/opl4/opl4_seq.c        |  13 +--
+ sound/hda/codecs/generic.c           |   4 +-
+ sound/hda/codecs/realtek/alc269.c    |   4 +-
+ sound/hda/codecs/realtek/realtek.c   |   5 +-
+ sound/hda/common/codec.c             |   4 +-
+ sound/hda/common/sysfs.c             |   5 +-
+ sound/isa/sb/emu8000_synth.c         |  13 +--
+ sound/pci/emu10k1/emu10k1_synth.c    |  13 +--
+ sound/usb/fcp.c                      |  36 ++++----
+ sound/usb/mixer_scarlett2.c          |  21 ++---
+ sound/usb/quirks.c                   |  13 ++-
+ sound/usb/usx2y/us144mkii.c          |   4 +-
+ sound/usb/usx2y/us144mkii_controls.c |   4 +-
+ sound/usb/usx2y/us144mkii_pcm.c      |   4 +-
+ 40 files changed, 408 insertions(+), 385 deletions(-)
+Merging ieee1394/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ieee1394/linux1394.git ieee1394/for-next
+Already up to date.
+Merging sound-asoc/for-next (fa41400e9b5c Merge remote-tracking branch 'asoc/for-6.20' into asoc-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git sound-asoc/for-next
+Merge made by the 'ort' strategy.
+ drivers/firmware/cirrus/cs_dsp.c                  |   31 +-
+ include/linux/pci_ids.h                           |    3 +-
+ include/sound/sof.h                               |    3 +
+ sound/hda/controllers/intel.c                     |    2 +-
+ sound/hda/core/intel-dsp-config.c                 |    4 +-
+ sound/soc/amd/acp/acp-sdw-legacy-mach.c           |    8 +-
+ sound/soc/amd/acp/acp-sdw-sof-mach.c              |    8 +-
+ sound/soc/codecs/Kconfig                          |    1 +
+ sound/soc/codecs/cs-amp-lib.c                     |   27 +-
+ sound/soc/codecs/rt1320-sdw.c                     | 1247 ++++++++++++++++++++-
+ sound/soc/codecs/rt1320-sdw.h                     |   66 ++
+ sound/soc/codecs/rt274.c                          |   12 +-
+ sound/soc/codecs/rt286.c                          |   12 +-
+ sound/soc/codecs/rt298.c                          |   12 +-
+ sound/soc/codecs/sdw-mockup.c                     |    6 -
+ sound/soc/codecs/tas2780.c                        |   31 +-
+ sound/soc/codecs/tas2783-sdw.c                    |  347 +++---
+ sound/soc/codecs/tas2783.h                        |    1 +
+ sound/soc/codecs/wm0010.c                         |    2 +-
+ sound/soc/fsl/fsl_ssi.c                           |    2 +-
+ sound/soc/intel/avs/board_selection.c             |    2 +-
+ sound/soc/intel/avs/core.c                        |    2 +-
+ sound/soc/intel/catpt/core.h                      |    7 +-
+ sound/soc/intel/catpt/device.c                    |    6 +-
+ sound/soc/intel/catpt/ipc.c                       |   12 +-
+ sound/soc/intel/catpt/loader.c                    |    8 +-
+ sound/soc/intel/catpt/messages.c                  |   89 +-
+ sound/soc/intel/catpt/pcm.c                       |   46 +-
+ sound/soc/intel/catpt/sysfs.c                     |    2 +-
+ sound/soc/intel/common/soc-acpi-intel-mtl-match.c |   14 +-
+ sound/soc/mediatek/mt8189/mt8189-nau8825.c        |    6 +-
+ sound/soc/mediatek/mt8195/mt8195-dai-etdm.c       |   44 +-
+ sound/soc/qcom/qdsp6/audioreach.c                 |  148 ++-
+ sound/soc/qcom/qdsp6/audioreach.h                 |   39 +
+ sound/soc/qcom/qdsp6/q6adm.c                      |    2 +
+ sound/soc/qcom/qdsp6/q6afe.c                      |   20 +-
+ sound/soc/qcom/qdsp6/q6apm.c                      |    9 +-
+ sound/soc/qcom/qdsp6/q6asm.c                      |   19 +-
+ sound/soc/qcom/qdsp6/q6prm.c                      |   16 +-
+ sound/soc/renesas/rz-ssi.c                        |  119 +-
+ sound/soc/sdca/sdca_asoc.c                        |    2 +-
+ sound/soc/sdca/sdca_functions.c                   |   28 +-
+ sound/soc/sdw_utils/soc_sdw_ti_amp.c              |    4 +
+ sound/soc/sdw_utils/soc_sdw_utils.c               |   22 +-
+ sound/soc/sof/compress.c                          |    8 +
+ sound/soc/sof/control.c                           |   13 +-
+ sound/soc/sof/core.c                              |    1 +
+ sound/soc/sof/debug.c                             |    7 +-
+ sound/soc/sof/intel/hda-sdw-bpt.c                 |   11 +
+ sound/soc/sof/intel/hda.c                         |   32 +-
+ sound/soc/sof/intel/pci-apl.c                     |    2 +-
+ sound/soc/sof/intel/pci-lnl.c                     |    1 +
+ sound/soc/sof/intel/pci-nvl.c                     |    1 +
+ sound/soc/sof/intel/pci-ptl.c                     |    2 +
+ sound/soc/sof/ipc3-dtrace.c                       |    7 +-
+ sound/soc/sof/ipc4-loader.c                       |   19 +-
+ sound/soc/sof/ipc4.c                              |   13 +
+ sound/soc/sof/pcm.c                               |   10 +
+ sound/soc/sof/pm.c                                |  175 +--
+ sound/soc/sof/sof-client-ipc-flood-test.c         |    7 +-
+ sound/soc/sof/sof-client-ipc-kernel-injector.c    |    4 +-
+ sound/soc/sof/sof-client-ipc-msg-injector.c       |   14 +-
+ sound/soc/sof/sof-client-probes.c                 |   26 +-
+ sound/soc/sof/sof-client.c                        |    6 +
+ sound/soc/sof/sof-client.h                        |    3 +
+ sound/soc/sof/sof-priv.h                          |    3 +
+ sound/soc/ti/davinci-mcasp.c                      |    7 +-
+ 67 files changed, 2204 insertions(+), 659 deletions(-)
+Merging modules/modules-next (1ddac5cd7f27 MAINTAINERS: Add myself as reviewer for module support)
+$ git merge -m Merge branch 'modules-next' of https://git.kernel.org/pub/scm/linux/kernel/git/modules/linux.git modules/modules-next
+Already up to date.
+Merging input/next (d79f302f2f98 dt-bindings: input: touchscreen: sitronix,st1232: Add Sitronix ST1624)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git input/next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../bindings/input/touchscreen/sitronix,st1232.yaml | 10 +++++++---
+ MAINTAINERS                                         |  2 +-
+ drivers/input/keyboard/cros_ec_keyb.c               |  3 ++-
+ drivers/input/keyboard/omap4-keypad.c               |  4 ----
+ drivers/input/misc/cs40l50-vibra.c                  |  4 ----
+ drivers/input/misc/pf1550-onkey.c                   |  2 +-
+ drivers/input/mouse/byd.c                           |  6 ++----
+ drivers/input/mouse/cyapa.c                         |  3 ---
+ drivers/input/mouse/cyapa_gen5.c                    |  1 -
+ drivers/input/touchscreen/stmfts.c                  | 21 +++++++++------------
+ 10 files changed, 22 insertions(+), 34 deletions(-)
+Merging block/for-next (e68d68450f76 Merge branch 'block-6.19' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git block/for-next
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/blockdev/zoned_loop.rst |  2 +-
+ block/bfq-iosched.c                               |  2 +-
+ block/blk-settings.c                              | 14 +++--
+ block/blk-sysfs.c                                 |  2 +-
+ block/blk-wbt.c                                   | 20 +++++--
+ block/blk-wbt.h                                   |  5 ++
+ block/elevator.c                                  |  4 --
+ block/elevator.h                                  |  1 -
+ block/ioctl.c                                     |  9 ++--
+ drivers/block/loop.c                              | 22 ++++----
+ drivers/block/rnbd/rnbd-clt.c                     | 13 +++--
+ drivers/block/ublk_drv.c                          | 32 ++++++++++--
+ drivers/block/zloop.c                             |  8 +--
+ include/linux/bio.h                               | 30 +++++------
+ include/uapi/linux/pr.h                           |  2 +
+ io_uring/rsrc.c                                   |  1 +
+ tools/testing/selftests/ublk/Makefile             |  8 +++
+ tools/testing/selftests/ublk/file_backed.c        |  7 +--
+ tools/testing/selftests/ublk/kublk.c              | 64 ++++++++++++++++++++---
+ tools/testing/selftests/ublk/kublk.h              | 23 +++++---
+ tools/testing/selftests/ublk/stripe.c             |  2 +-
+ tools/testing/selftests/ublk/test_common.sh       |  5 +-
+ tools/testing/selftests/ublk/test_generic_04.sh   |  2 +-
+ tools/testing/selftests/ublk/test_generic_05.sh   |  2 +-
+ tools/testing/selftests/ublk/test_generic_11.sh   |  2 +-
+ tools/testing/selftests/ublk/test_generic_14.sh   | 40 ++++++++++++++
+ tools/testing/selftests/ublk/test_loop_06.sh      | 25 +++++++++
+ tools/testing/selftests/ublk/test_loop_07.sh      | 21 ++++++++
+ tools/testing/selftests/ublk/test_null_03.sh      | 24 +++++++++
+ tools/testing/selftests/ublk/test_stress_04.sh    | 12 +++--
+ tools/testing/selftests/ublk/test_stress_05.sh    | 10 ++--
+ tools/testing/selftests/ublk/test_stress_06.sh    | 39 ++++++++++++++
+ tools/testing/selftests/ublk/test_stress_07.sh    | 39 ++++++++++++++
+ tools/testing/selftests/ublk/test_stripe_05.sh    | 26 +++++++++
+ tools/testing/selftests/ublk/test_stripe_06.sh    | 21 ++++++++
+ tools/testing/selftests/ublk/trace/seq_io.bt      |  3 +-
+ 36 files changed, 451 insertions(+), 91 deletions(-)
+ create mode 100755 tools/testing/selftests/ublk/test_generic_14.sh
+ create mode 100755 tools/testing/selftests/ublk/test_loop_06.sh
+ create mode 100755 tools/testing/selftests/ublk/test_loop_07.sh
+ create mode 100755 tools/testing/selftests/ublk/test_null_03.sh
+ create mode 100755 tools/testing/selftests/ublk/test_stress_06.sh
+ create mode 100755 tools/testing/selftests/ublk/test_stress_07.sh
+ create mode 100755 tools/testing/selftests/ublk/test_stripe_05.sh
+ create mode 100755 tools/testing/selftests/ublk/test_stripe_06.sh
+Merging device-mapper/for-next (13ea55ea2017 dm pcache: fix segment info indexing)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git device-mapper/for-next
+Already up to date.
+Merging libata/for-next (f474c70065e1 ata: libata: Allow more quirks)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/libata/linux libata/for-next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/kernel-parameters.txt |   5 +
+ drivers/ata/libata-core.c                       | 233 ++++++++++++++++++------
+ drivers/ata/libata-eh.c                         |   3 +-
+ drivers/ata/libata-scsi.c                       |   8 +-
+ drivers/ata/libata.h                            |   1 +
+ include/linux/ata.h                             |   3 -
+ include/linux/libata.h                          |  70 ++++---
+ 7 files changed, 225 insertions(+), 98 deletions(-)
+Merging pcmcia/pcmcia-next (4a81f78caa53 pcmcia: Add error handling for add_interval() in do_validate_mem())
+$ git merge -m Merge branch 'pcmcia-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brodo/linux.git pcmcia/pcmcia-next
+Already up to date.
+Merging mmc/next (1c6c8cf30022 mmc: Merge branch fixes into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git mmc/next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/mmc/arasan,sdhci.yaml      |   2 +
+ .../devicetree/bindings/mmc/brcm,iproc-sdhci.yaml  |   5 +
+ .../devicetree/bindings/mmc/cdns,sdhci.yaml        |   2 -
+ Documentation/devicetree/bindings/mmc/mtk-sd.yaml  |   3 +
+ drivers/mmc/core/card.h                            |   6 +
+ drivers/mmc/core/mmc.c                             |  12 +
+ drivers/mmc/core/quirks.h                          |   3 +
+ drivers/mmc/host/dw_mmc-exynos.c                   |   9 +-
+ drivers/mmc/host/dw_mmc-hi3798cv200.c              |   6 +-
+ drivers/mmc/host/dw_mmc-hi3798mv200.c              |  17 +-
+ drivers/mmc/host/dw_mmc-k3.c                       |  21 +-
+ drivers/mmc/host/dw_mmc-pltfm.c                    |   2 +-
+ drivers/mmc/host/dw_mmc-rockchip.c                 |   9 +-
+ drivers/mmc/host/dw_mmc-starfive.c                 |   5 +-
+ drivers/mmc/host/dw_mmc.c                          | 624 +++++++++------------
+ drivers/mmc/host/dw_mmc.h                          |  83 +--
+ drivers/mmc/host/mtk-sd.c                          |  32 +-
+ drivers/mmc/host/sdhci-esdhc-imx.c                 |  18 +
+ drivers/mmc/host/sdhci-msm.c                       |  77 +++
+ include/linux/mmc/card.h                           |   1 +
+ 20 files changed, 460 insertions(+), 477 deletions(-)
+Merging mfd/for-mfd-next (44c603f35cad mfd: sec: Drop a stray semicolon)
+$ git merge -m Merge branch 'for-mfd-next' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git mfd/for-mfd-next
+Already up to date.
+Merging backlight/for-backlight-next (d3cc7cd7bc46 backlight: aw99706: Fix unused function warnings from suspend/resume ops)
+$ git merge -m Merge branch 'for-backlight-next' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git backlight/for-backlight-next
+Already up to date.
+Merging battery/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git battery/for-next
+Already up to date.
+Merging regulator/for-next (2bb202416d33 Add support for MAX77675 device)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git regulator/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/regulator/adi,max77675.yaml           |  184 ++++
+ drivers/regulator/Kconfig                          |    9 +
+ drivers/regulator/Makefile                         |    1 +
+ drivers/regulator/max77675-regulator.c             | 1056 ++++++++++++++++++++
+ rust/helpers/regulator.c                           |   24 +-
+ 5 files changed, 1264 insertions(+), 10 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/regulator/adi,max77675.yaml
+ create mode 100644 drivers/regulator/max77675-regulator.c
+Merging security/next (1c0860d4415d lsm: fix kernel-doc struct member names)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git security/next
+Merge made by the 'ort' strategy.
+ include/linux/lsm_hooks.h | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+Merging apparmor/apparmor-next (8f5ae30d69d7 Linux 6.17-rc1)
+$ git merge -m Merge branch 'apparmor-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jj/linux-apparmor apparmor/apparmor-next
+Already up to date.
+Merging integrity/next-integrity (738c9738e690 ima: Handle error code returned by ima_filter_rule_match())
+$ git merge -m Merge branch 'next-integrity' of https://git.kernel.org/pub/scm/linux/kernel/git/zohar/linux-integrity integrity/next-integrity
+Already up to date.
+Merging selinux/next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/selinux.git selinux/next
+Already up to date.
+Merging smack/next (29c701f90b93 Smack: function parameter 'gfp' not described)
+$ git merge -m Merge branch 'next' of https://github.com/cschaufler/smack-next smack/next
+Already up to date.
+Merging tomoyo/master (88fe48f74ec2 RDMA/core: check device status when manipulating ib_gid_table_entry)
+$ git merge -m Merge branch 'master' of git://git.code.sf.net/p/tomoyo/tomoyo.git tomoyo/master
+Auto-merging net/can/j1939/transport.c
+Merge made by the 'ort' strategy.
+ drivers/infiniband/core/cache.c         | 111 ++++++++++++++++++-
+ drivers/infiniband/core/core_priv.h     |   7 ++
+ drivers/infiniband/core/device.c        |  33 ++++++
+ drivers/infiniband/core/roce_gid_mgmt.c |  27 ++++-
+ include/linux/netdevice.h               |  15 +++
+ kernel/softirq.c                        |   4 +
+ kernel/workqueue.c                      |   4 +
+ net/can/j1939/j1939-priv.h              |   6 ++
+ net/can/j1939/main.c                    |  93 ++++++++++++++++
+ net/can/j1939/transport.c               |   3 +
+ net/core/dev.c                          | 184 ++++++++++++++++++++++++++++++++
+ net/core/lock_debug.c                   |   1 +
+ net/socket.c                            |  32 ++++--
+ 13 files changed, 507 insertions(+), 13 deletions(-)
+Merging tpmdd/next (416f99c3b16f Merge tag 'driver-core-6.19-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git tpmdd/next
+Already up to date.
+Merging watchdog/master (5bcc5786a0cf watchdog: starfive: Fix resource leak in probe error path)
+$ git merge -m Merge branch 'master' of git://www.linux-watchdog.org/linux-watchdog-next.git watchdog/master
+Already up to date.
+Merging iommu/next (7a089c5d35aa iommupt: Return ERR_PTR from _table_alloc())
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git iommu/next
+Merge made by the 'ort' strategy.
+ drivers/iommu/generic_pt/iommu_pt.h | 3 +++
+ 1 file changed, 3 insertions(+)
+Merging audit/next (15b0c43aa621 audit: include source and destination ports to NETFILTER_PKT)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git audit/next
+Auto-merging kernel/audit.c
+Merge made by the 'ort' strategy.
+ include/asm-generic/audit_change_attr.h |   3 +
+ include/linux/audit.h                   |   8 ++
+ kernel/audit.c                          | 159 ++++++++++++++++++++++++++++++++
+ net/netfilter/nft_log.c                 |  58 +-----------
+ net/netfilter/xt_AUDIT.c                |  58 +-----------
+ 5 files changed, 172 insertions(+), 114 deletions(-)
+Merging devicetree/for-next (5530836fb914 of: replace strcmp_suffix() with strends())
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git devicetree/for-next
+Auto-merging Documentation/devicetree/bindings/arm/arm,vexpress-juno.yaml
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/Makefile         |   5 +-
+ .../devicetree/bindings/arm/arm,vexpress-juno.yaml |   6 +
+ .../devicetree/bindings/arm/vexpress-config.yaml   |   6 +-
+ .../bindings/bus/aspeed,ast2600-ahbc.yaml          |   8 +-
+ .../bindings/bus/st,stm32mp25-rifsc.yaml           |   2 +-
+ .../bindings/crypto/aspeed,ast2600-acry.yaml       |   7 +
+ .../firmware/xilinx/xlnx,zynqmp-firmware.yaml      |  21 +-
+ .../devicetree/bindings/trivial-devices.yaml       |   2 +
+ .../devicetree/bindings/writing-schema.rst         |  16 +-
+ MAINTAINERS                                        |   1 +
+ Makefile                                           |  16 +-
+ arch/arm/Kbuild                                    |   2 +
+ arch/arm64/Kbuild                                  |   2 +
+ arch/mips/Kbuild                                   |   2 +
+ arch/riscv/Kbuild                                  |   2 +
+ drivers/of/property.c                              |  15 +-
+ include/dt-bindings/clock/oxsemi,ox810se.h         |  19 --
+ include/dt-bindings/clock/oxsemi,ox820.h           |  29 ---
+ include/dt-bindings/clock/qcom,mss-sc7180.h        |  12 -
+ include/dt-bindings/clock/xlnx-versal-clk.h        | 123 ----------
+ include/dt-bindings/clock/xlnx-zynqmp-clk.h        | 133 -----------
+ include/dt-bindings/dma/jz4775-dma.h               |  44 ----
+ include/dt-bindings/dma/x2000-dma.h                |  54 -----
+ include/dt-bindings/gce/mt6779-gce.h               | 222 ------------------
+ include/dt-bindings/memory/mt6779-larb-port.h      | 206 -----------------
+ include/dt-bindings/mux/ti-serdes.h                | 190 ---------------
+ include/dt-bindings/pinctrl/mt6397-pinfunc.h       | 257 ---------------------
+ .../dt-bindings/power/allwinner,sun20i-d1-ppu.h    |  10 -
+ include/dt-bindings/reset/bcm6318-reset.h          |  20 --
+ include/dt-bindings/reset/imx8ulp-pcc-reset.h      |  59 -----
+ include/dt-bindings/reset/oxsemi,ox810se.h         |  42 ----
+ include/dt-bindings/reset/oxsemi,ox820.h           |  42 ----
+ include/dt-bindings/sound/audio-jack-events.h      |  10 -
+ scripts/Makefile.dtbs                              |   4 +
+ 34 files changed, 86 insertions(+), 1503 deletions(-)
+ delete mode 100644 include/dt-bindings/clock/oxsemi,ox810se.h
+ delete mode 100644 include/dt-bindings/clock/oxsemi,ox820.h
+ delete mode 100644 include/dt-bindings/clock/qcom,mss-sc7180.h
+ delete mode 100644 include/dt-bindings/clock/xlnx-versal-clk.h
+ delete mode 100644 include/dt-bindings/clock/xlnx-zynqmp-clk.h
+ delete mode 100644 include/dt-bindings/dma/jz4775-dma.h
+ delete mode 100644 include/dt-bindings/dma/x2000-dma.h
+ delete mode 100644 include/dt-bindings/gce/mt6779-gce.h
+ delete mode 100644 include/dt-bindings/memory/mt6779-larb-port.h
+ delete mode 100644 include/dt-bindings/mux/ti-serdes.h
+ delete mode 100644 include/dt-bindings/pinctrl/mt6397-pinfunc.h
+ delete mode 100644 include/dt-bindings/power/allwinner,sun20i-d1-ppu.h
+ delete mode 100644 include/dt-bindings/reset/bcm6318-reset.h
+ delete mode 100644 include/dt-bindings/reset/imx8ulp-pcc-reset.h
+ delete mode 100644 include/dt-bindings/reset/oxsemi,ox810se.h
+ delete mode 100644 include/dt-bindings/reset/oxsemi,ox820.h
+ delete mode 100644 include/dt-bindings/sound/audio-jack-events.h
+Merging dt-krzk/for-next (b7fdd9926b22 Merge branch 'next/dt64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git dt-krzk/for-next
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/nuvoton/nuvoton-common-npcm7xx.dtsi | 4 ++--
+ arch/arm64/boot/dts/nuvoton/nuvoton-npcm845.dtsi      | 4 ++--
+ arch/arm64/boot/dts/toshiba/tmpv7708.dtsi             | 4 ++--
+ arch/arm64/boot/dts/toshiba/tmpv7708_pins.dtsi        | 2 +-
+ 4 files changed, 7 insertions(+), 7 deletions(-)
+Merging mailbox/for-next (1ec6e0e8c0dd mailbox: mtk-cmdq: Add driver data to support for MT8196)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jassibrar/mailbox.git mailbox/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/mailbox/mediatek,mt8196-vcp-mbox.yaml |  49 ++++++
+ .../bindings/mailbox/microchip,mpfs-mailbox.yaml   |   6 +-
+ .../bindings/mailbox/qcom,cpucp-mbox.yaml          |   1 +
+ .../devicetree/bindings/mailbox/qcom-ipcc.yaml     |   2 +
+ drivers/mailbox/Kconfig                            |  22 ++-
+ drivers/mailbox/Makefile                           |   4 +
+ drivers/mailbox/mailbox-mchp-ipc-sbi.c             |  22 +--
+ drivers/mailbox/mailbox.c                          |   6 +-
+ drivers/mailbox/mtk-cmdq-mailbox.c                 |  74 ++++++++-
+ drivers/mailbox/mtk-vcp-mailbox.c                  | 170 +++++++++++++++++++++
+ drivers/mailbox/pcc.c                              | 118 +++-----------
+ drivers/mailbox/rcar-mfis-mailbox.c                | 168 ++++++++++++++++++++
+ include/acpi/pcc.h                                 |  29 ----
+ include/linux/mailbox/mtk-cmdq-mailbox.h           |  19 +++
+ include/linux/mailbox/mtk-vcp-mailbox.h            |  32 ++++
+ 15 files changed, 578 insertions(+), 144 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/mailbox/mediatek,mt8196-vcp-mbox.yaml
+ create mode 100644 drivers/mailbox/mtk-vcp-mailbox.c
+ create mode 100644 drivers/mailbox/rcar-mfis-mailbox.c
+ create mode 100644 include/linux/mailbox/mtk-vcp-mailbox.h
+Merging spi/for-next (81e4a2ef2782 Merge remote-tracking branch 'spi/for-6.20' into spi-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git spi/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../bindings/spi/andestech,ae350-spi.yaml          |   85 ++
+ .../devicetree/bindings/spi/nxp,imx94-xspi.yaml    |   88 ++
+ .../bindings/spi/renesas,rzv2h-rspi.yaml           |    8 +
+ .../devicetree/bindings/spi/st,stm32-spi.yaml      |    3 +
+ MAINTAINERS                                        |   15 +
+ drivers/spi/Kconfig                                |   19 +
+ drivers/spi/Makefile                               |    2 +
+ drivers/spi/spi-atcspi200.c                        |  680 ++++++++++
+ drivers/spi/spi-cadence-quadspi.c                  |   16 +-
+ drivers/spi/spi-cadence-xspi.c                     |   47 +-
+ drivers/spi/spi-fsl-lpspi.c                        |   10 +-
+ drivers/spi/spi-geni-qcom.c                        |  127 +-
+ drivers/spi/spi-imx.c                              |  647 +++++++--
+ drivers/spi/spi-nxp-xspi.c                         | 1385 ++++++++++++++++++++
+ drivers/spi/spi-qpic-snand.c                       |    4 -
+ drivers/spi/spi-rzv2h-rspi.c                       |  281 +++-
+ drivers/spi/spi-stm32-ospi.c                       |  107 +-
+ drivers/spi/spi-stm32-qspi.c                       |  111 +-
+ drivers/spi/spi-virtio.c                           |    4 +-
+ 19 files changed, 3227 insertions(+), 412 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/spi/andestech,ae350-spi.yaml
+ create mode 100644 Documentation/devicetree/bindings/spi/nxp,imx94-xspi.yaml
+ create mode 100644 drivers/spi/spi-atcspi200.c
+ create mode 100644 drivers/spi/spi-nxp-xspi.c
+Merging tip/master (3e0483bc1554 Merge branch into tip/master: 'x86/cpu')
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git tip/master
+Auto-merging arch/arm64/boot/dts/renesas/r9a09g077.dtsi
+Auto-merging arch/arm64/boot/dts/renesas/r9a09g087.dtsi
+Auto-merging arch/x86/kernel/setup.c
+Auto-merging include/linux/sched.h
+Auto-merging lib/Kconfig.debug
+Auto-merging virt/kvm/kvm_main.c
+Merge made by the 'ort' strategy.
+ .../renesas,r9a09g077-icu.yaml                     | 236 +++++
+ .../interrupt-controller/renesas,rzv2h-icu.yaml    |   1 +
+ arch/arm64/boot/dts/renesas/r9a09g077.dtsi         |  73 ++
+ arch/arm64/boot/dts/renesas/r9a09g087.dtsi         |  73 ++
+ arch/mips/include/asm/cevt-r4k.h                   |   1 -
+ arch/mips/kernel/cevt-r4k.c                        |  11 -
+ arch/mips/sgi-ip27/ip27-timer.c                    |  10 +-
+ arch/mips/sgi-ip30/ip30-timer.c                    |   5 +-
+ arch/x86/entry/entry_fred.c                        |   1 +
+ arch/x86/events/amd/core.c                         |   2 +
+ arch/x86/events/core.c                             |  38 +-
+ arch/x86/events/intel/core.c                       |   6 +
+ arch/x86/events/intel/cstate.c                     |  33 +-
+ arch/x86/events/msr.c                              |   1 +
+ arch/x86/include/asm/e820/api.h                    |   3 +-
+ arch/x86/include/asm/e820/types.h                  |   2 +-
+ arch/x86/include/asm/hardirq.h                     |   3 +
+ arch/x86/include/asm/idtentry.h                    |   6 +
+ arch/x86/include/asm/irq_vectors.h                 |   4 +-
+ arch/x86/include/asm/perf_event.h                  |   6 +
+ arch/x86/include/asm/unwind_user.h                 |  27 +-
+ arch/x86/kernel/acpi/boot.c                        |  12 +-
+ arch/x86/kernel/cpu/amd.c                          |  40 +-
+ arch/x86/kernel/cpu/centaur.c                      |   6 -
+ arch/x86/kernel/cpu/cyrix.c                        |   6 -
+ arch/x86/kernel/cpu/hygon.c                        |   6 -
+ arch/x86/kernel/cpu/topology.c                     |  15 -
+ arch/x86/kernel/e820.c                             | 477 +++++-----
+ arch/x86/kernel/idt.c                              |   3 +
+ arch/x86/kernel/irq.c                              |  19 +
+ arch/x86/kernel/setup.c                            |  10 +-
+ arch/x86/kvm/Kconfig                               |   1 +
+ arch/x86/platform/efi/efi.c                        |   3 +-
+ drivers/clocksource/mips-gic-timer.c               |  10 +-
+ drivers/irqchip/Kconfig                            |   8 +
+ drivers/irqchip/Makefile                           |   1 +
+ drivers/irqchip/irq-aclint-sswi.c                  |   2 +-
+ drivers/irqchip/irq-renesas-rzt2h.c                | 280 ++++++
+ drivers/irqchip/irq-renesas-rzv2h.c                |   1 +
+ drivers/irqchip/irq-riscv-aplic-direct.c           |  10 +
+ drivers/irqchip/irq-riscv-aplic-main.c             | 166 +++-
+ drivers/irqchip/irq-riscv-aplic-main.h             |  19 +
+ drivers/irqchip/irq-riscv-imsic-early.c            |  39 +-
+ drivers/pci/controller/dwc/pcie-designware-host.c  | 127 ++-
+ drivers/pci/controller/dwc/pcie-designware.h       |   7 +-
+ drivers/soc/renesas/Kconfig                        |   1 +
+ include/linux/compiler_types.h                     |  19 +
+ include/linux/interrupt.h                          |  24 +-
+ include/linux/irq.h                                |  13 +-
+ include/linux/irqchip/irq-renesas-rzt2h.h          |  23 +
+ include/linux/irqdesc.h                            |  17 +-
+ include/linux/msi.h                                |  13 +-
+ include/linux/perf_event.h                         |  35 +-
+ include/linux/sched.h                              |  13 +-
+ include/linux/timecounter.h                        |  31 +-
+ include/linux/unwind_user.h                        |  18 +-
+ init/Kconfig                                       |   4 +
+ kernel/events/core.c                               | 523 ++++++++---
+ kernel/irq/Kconfig                                 |   3 -
+ kernel/irq/Makefile                                |   4 -
+ kernel/irq/chip.c                                  |  22 +-
+ kernel/irq/handle.c                                |   2 -
+ kernel/irq/internals.h                             | 110 ---
+ kernel/irq/irqdesc.c                               |  86 +-
+ kernel/irq/manage.c                                |  63 +-
+ kernel/irq/timings.c                               | 959 ---------------------
+ kernel/sched/core.c                                |  46 +-
+ kernel/sched/deadline.c                            |  14 +-
+ kernel/sched/ext.c                                 |   9 +-
+ kernel/sched/fair.c                                | 361 ++++----
+ kernel/sched/idle.c                                |   3 -
+ kernel/sched/rt.c                                  |   9 +-
+ kernel/sched/sched.h                               |  60 +-
+ kernel/sched/stop_task.c                           |   3 -
+ kernel/time/timecounter.c                          |  35 -
+ kernel/unwind/user.c                               |  12 +-
+ lib/Kconfig.debug                                  |   8 -
+ .../beauty/arch/x86/include/asm/irq_vectors.h      |   3 +-
+ virt/kvm/kvm_main.c                                |   3 +
+ 79 files changed, 2328 insertions(+), 2031 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/interrupt-controller/renesas,r9a09g077-icu.yaml
+ create mode 100644 drivers/irqchip/irq-renesas-rzt2h.c
+ create mode 100644 include/linux/irqchip/irq-renesas-rzt2h.h
+ delete mode 100644 kernel/irq/timings.c
+$ git reset --hard HEAD^
+HEAD is now at 7157d5509361 Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git
+Merging next-20251217 version of tip
+$ git merge -m next-20251217/tip eff801f5c2b85f1969d5436b6b86c6d8078129aa
+Auto-merging arch/arm64/boot/dts/renesas/r9a09g077.dtsi
+Auto-merging arch/arm64/boot/dts/renesas/r9a09g087.dtsi
+Auto-merging arch/x86/kernel/setup.c
+Auto-merging include/linux/sched.h
+Auto-merging lib/Kconfig.debug
+Merge made by the 'ort' strategy.
+ .../renesas,r9a09g077-icu.yaml                     | 236 +++++
+ .../interrupt-controller/renesas,rzv2h-icu.yaml    |   1 +
+ arch/arm64/boot/dts/renesas/r9a09g077.dtsi         |  73 ++
+ arch/arm64/boot/dts/renesas/r9a09g087.dtsi         |  73 ++
+ arch/mips/include/asm/cevt-r4k.h                   |   1 -
+ arch/mips/kernel/cevt-r4k.c                        |  11 -
+ arch/mips/sgi-ip27/ip27-timer.c                    |  10 +-
+ arch/mips/sgi-ip30/ip30-timer.c                    |   5 +-
+ arch/x86/events/intel/cstate.c                     |  32 +-
+ arch/x86/include/asm/e820/api.h                    |   3 +-
+ arch/x86/include/asm/e820/types.h                  |   2 +-
+ arch/x86/kernel/acpi/boot.c                        |  12 +-
+ arch/x86/kernel/cpu/amd.c                          |  40 +-
+ arch/x86/kernel/cpu/centaur.c                      |   6 -
+ arch/x86/kernel/cpu/cyrix.c                        |   6 -
+ arch/x86/kernel/cpu/hygon.c                        |   6 -
+ arch/x86/kernel/cpu/topology.c                     |  15 -
+ arch/x86/kernel/e820.c                             | 477 +++++-----
+ arch/x86/kernel/setup.c                            |  10 +-
+ arch/x86/platform/efi/efi.c                        |   3 +-
+ drivers/clocksource/mips-gic-timer.c               |  10 +-
+ drivers/irqchip/Kconfig                            |   8 +
+ drivers/irqchip/Makefile                           |   1 +
+ drivers/irqchip/irq-aclint-sswi.c                  |   2 +-
+ drivers/irqchip/irq-renesas-rzt2h.c                | 280 ++++++
+ drivers/irqchip/irq-renesas-rzv2h.c                |   1 +
+ drivers/irqchip/irq-riscv-aplic-direct.c           |  10 +
+ drivers/irqchip/irq-riscv-aplic-main.c             | 166 +++-
+ drivers/irqchip/irq-riscv-aplic-main.h             |  19 +
+ drivers/irqchip/irq-riscv-imsic-early.c            |  39 +-
+ drivers/pci/controller/dwc/pcie-designware-host.c  | 127 ++-
+ drivers/pci/controller/dwc/pcie-designware.h       |   7 +-
+ drivers/soc/renesas/Kconfig                        |   1 +
+ include/linux/compiler_types.h                     |  19 +
+ include/linux/interrupt.h                          |  24 +-
+ include/linux/irq.h                                |  13 +-
+ include/linux/irqchip/irq-renesas-rzt2h.h          |  23 +
+ include/linux/irqdesc.h                            |  17 +-
+ include/linux/msi.h                                |  13 +-
+ include/linux/sched.h                              |  13 +-
+ include/linux/timecounter.h                        |  31 +-
+ kernel/irq/Kconfig                                 |   3 -
+ kernel/irq/Makefile                                |   4 -
+ kernel/irq/chip.c                                  |  22 +-
+ kernel/irq/handle.c                                |   2 -
+ kernel/irq/internals.h                             | 110 ---
+ kernel/irq/irqdesc.c                               |  86 +-
+ kernel/irq/manage.c                                |  63 +-
+ kernel/irq/timings.c                               | 959 ---------------------
+ kernel/sched/core.c                                |  45 +-
+ kernel/sched/deadline.c                            |  14 +-
+ kernel/sched/ext.c                                 |   7 +-
+ kernel/sched/fair.c                                | 361 ++++----
+ kernel/sched/idle.c                                |   3 -
+ kernel/sched/rt.c                                  |   9 +-
+ kernel/sched/sched.h                               |  59 +-
+ kernel/sched/stop_task.c                           |   3 -
+ kernel/time/timecounter.c                          |  35 -
+ lib/Kconfig.debug                                  |   8 -
+ 59 files changed, 1763 insertions(+), 1876 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/interrupt-controller/renesas,r9a09g077-icu.yaml
+ create mode 100644 drivers/irqchip/irq-renesas-rzt2h.c
+ create mode 100644 include/linux/irqchip/irq-renesas-rzt2h.h
+ delete mode 100644 kernel/irq/timings.c
+Merging clockevents/timers/drivers/next (d1780dce9575 clocksource/drivers: Add Realtek system timer driver)
+$ git merge -m Merge branch 'timers/drivers/next' of https://git.kernel.org/pub/scm/linux/kernel/git/daniel.lezcano/linux.git clockevents/timers/drivers/next
+Already up to date.
+Merging edac/edac-for-next (814b332921b6 Merge ras/edac-amd-atl into for-next)
+$ git merge -m Merge branch 'edac-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ras/ras.git edac/edac-for-next
+Merge made by the 'ort' strategy.
+Merging ftrace/for-next (5a63f3faf656 Merge unused-tracepoints/for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git ftrace/for-next
+Merge made by the 'ort' strategy.
+Merging rcu/next (9a08942f1701 Merge branch 'rcu/misc' into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux rcu/next
+Already up to date.
+Merging paulmck/non-rcu/next (7f8fcc6f09fb memory-barriers.txt: Sort wait_event* and wait_on_bit* list alphabetically)
+$ git merge -m Merge branch 'non-rcu/next' of https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git paulmck/non-rcu/next
+Already up to date.
+Merging kvm/next (0499add8efd7 Merge tag 'kvm-x86-fixes-6.19-rc1' of https://github.com/kvm-x86/linux into HEAD)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git kvm/next
+Already up to date.
+Merging kvm-arm/next (3eef0c83c3f3 Merge branch 'kvm-arm64/nv-xnx-haf' into kvmarm/next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git kvm-arm/next
+Already up to date.
+Merging kvms390/next (2bd1337a1295 KVM: s390: Use generic VIRT_XFER_TO_GUEST_WORK functions)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git kvms390/next
+Already up to date.
+Merging kvm-ppc/topic/ppc-kvm (c17b750b3ad9 Linux 6.17-rc2)
+$ git merge -m Merge branch 'topic/ppc-kvm' of https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git kvm-ppc/topic/ppc-kvm
+Already up to date.
+Merging kvm-riscv/riscv_kvm_next (3239c52fd212 RISC-V: KVM: Flush VS-stage TLB after VCPU migration for Andes cores)
+$ git merge -m Merge branch 'riscv_kvm_next' of https://github.com/kvm-riscv/linux.git kvm-riscv/riscv_kvm_next
+Already up to date.
+Merging kvm-x86/next (58e10b63777d Merge branch 'fixes')
+$ git merge -m Merge branch 'next' of https://github.com/kvm-x86/linux.git kvm-x86/next
+Merge made by the 'ort' strategy.
+Merging xen-tip/linux-next (e5aff444e3a7 x86/xen: Fix sparse warning in enlighten_pv.c)
+$ git merge -m Merge branch 'linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git xen-tip/linux-next
+Auto-merging arch/x86/xen/enlighten_pv.c
+Merge made by the 'ort' strategy.
+ arch/x86/xen/enlighten_pv.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging percpu/for-next (87d6aab2389e Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu.git percpu/for-next
+Already up to date.
+Merging workqueues/for-next (51cd2d2decf3 workqueue: Process extra works in rescuer on memory pressure)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git workqueues/for-next
+Auto-merging kernel/workqueue.c
+Merge made by the 'ort' strategy.
+ kernel/workqueue.c | 121 ++++++++++++++++++++++++++++++++++++++++-------------
+ 1 file changed, 92 insertions(+), 29 deletions(-)
+Merging sched-ext/for-next (b0101ccb5b46 sched_ext: fix uninitialized ret on alloc_percpu() failure)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git sched-ext/for-next
+Auto-merging kernel/sched/ext.c
+Merge made by the 'ort' strategy.
+ kernel/sched/ext.c | 4 +++-
+ 1 file changed, 3 insertions(+), 1 deletion(-)
+Merging drivers-x86/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git drivers-x86/for-next
+Already up to date.
+Merging chrome-platform/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git chrome-platform/for-next
+Already up to date.
+Merging chrome-platform-firmware/for-firmware-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-firmware-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git chrome-platform-firmware/for-firmware-next
+Already up to date.
+Merging hsi/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-hsi.git hsi/for-next
+Already up to date.
+Merging leds-lj/for-leds-next (072cd5f458d7 leds: rgb: leds-qcom-lpg: Don't enable TRILED when configuring PWM)
+$ git merge -m Merge branch 'for-leds-next' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/leds.git leds-lj/for-leds-next
+Already up to date.
+Merging ipmi/for-next (35bcedc1a793 MAINTAINERS: Add entry on Loongson-2K IPMI driver)
+$ git merge -m Merge branch 'for-next' of https://github.com/cminyard/linux-ipmi.git ipmi/for-next
+Already up to date.
+Merging driver-core/driver-core-next (e4b3118b61b6 rust: scatterlist: add __rust_helper to helpers)
+$ git merge -m Merge branch 'driver-core-next' of https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git driver-core/driver-core-next
+Auto-merging rust/helpers/dma.c
+Merge made by the 'ort' strategy.
+ rust/helpers/auxiliary.c   |  6 +++--
+ rust/helpers/device.c      | 16 ++++++------
+ rust/helpers/dma.c         | 15 ++++++-----
+ rust/helpers/io.c          | 64 ++++++++++++++++++++++++----------------------
+ rust/helpers/irq.c         |  6 +++--
+ rust/helpers/pci.c         | 23 +++++++++--------
+ rust/helpers/platform.c    |  2 +-
+ rust/helpers/property.c    |  2 +-
+ rust/helpers/scatterlist.c | 12 +++++----
+ 9 files changed, 81 insertions(+), 65 deletions(-)
+Merging usb/usb-next (dd1fbe324a54 usb: linux/usb.h: Correct the description of the usb_device_driver member)
+$ git merge -m Merge branch 'usb-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git usb/usb-next
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/configfs-usb-gadget-midi |  17 ++--
+ .../devicetree/bindings/usb/aspeed,usb-vhub.yaml   |  22 ++++-
+ .../devicetree/bindings/usb/generic-ehci.yaml      |   2 +
+ .../devicetree/bindings/usb/generic-ohci.yaml      |   2 +
+ Documentation/usb/gadget-testing.rst               |  17 ++--
+ drivers/usb/gadget/function/f_midi.c               | 110 +++++++++++----------
+ drivers/usb/gadget/function/u_midi.h               |   2 +-
+ drivers/usb/gadget/udc/aspeed-vhub/core.c          |  30 ++++++
+ drivers/usb/gadget/udc/aspeed-vhub/vhub.h          |   1 +
+ drivers/usb/typec/bus.c                            |  25 ++++-
+ drivers/usb/typec/bus.h                            |   6 --
+ drivers/usb/typec/class.c                          |  36 ++++---
+ drivers/usb/typec/ucsi/psy.c                       |  54 +++++++---
+ include/linux/usb.h                                |   3 +-
+ include/linux/usb/typec.h                          |   3 +
+ include/linux/usb/typec_altmode.h                  |   8 ++
+ 16 files changed, 235 insertions(+), 103 deletions(-)
+Merging thunderbolt/next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git thunderbolt/next
+Already up to date.
+Merging usb-serial/usb-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'usb-next' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git usb-serial/usb-next
+Already up to date.
+Merging tty/tty-next (284da5de616a platform/surface: Migrate to serdev specific shutdown function)
+$ git merge -m Merge branch 'tty-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git tty/tty-next
+Auto-merging drivers/bluetooth/hci_qca.c
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/serial/renesas,rsci.yaml   |  99 ++++++-
+ drivers/bluetooth/hci_aml.c                        |  16 +-
+ drivers/bluetooth/hci_qca.c                        |   5 +-
+ drivers/platform/surface/aggregator/core.c         |   6 +-
+ drivers/tty/n_hdlc.c                               |   2 +
+ drivers/tty/serdev/core.c                          |  21 ++
+ drivers/tty/serial/8250/8250_dw.c                  |  23 +-
+ drivers/tty/serial/8250/8250_keba.c                |  27 +-
+ drivers/tty/serial/8250/8250_men_mcb.c             |  12 +-
+ drivers/tty/serial/rsci.c                          | 310 ++++++++++++++++++---
+ drivers/tty/serial/rsci.h                          |   3 +-
+ drivers/tty/serial/sh-sci-common.h                 |  10 +-
+ drivers/tty/serial/sh-sci.c                        |  80 +++---
+ drivers/tty/vt/keyboard.c                          | 233 ++++++++--------
+ include/linux/serdev.h                             |   1 +
+ 15 files changed, 619 insertions(+), 229 deletions(-)
+Merging char-misc/char-misc-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'char-misc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git char-misc/char-misc-next
+Already up to date.
+Merging accel/habanalabs-next (f03eee5fc922 Merge tag 'drm-xe-next-fixes-2024-05-02' of https://gitlab.freedesktop.org/drm/xe/kernel into drm-next)
+$ git merge -m Merge branch 'habanalabs-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ogabbay/linux.git accel/habanalabs-next
+Already up to date.
+Merging coresight/next (7009646d937f dt-binding: Update oss email address for Coresight documents)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git coresight/next
+Merge made by the 'ort' strategy.
+ .../sysfs-bus-coresight-devices-dummy-source       |  4 +-
+ .../ABI/testing/sysfs-bus-coresight-devices-tpdm   | 56 +++++++++++-----------
+ .../bindings/arm/arm,coresight-dummy-sink.yaml     |  2 +-
+ .../bindings/arm/arm,coresight-dummy-source.yaml   |  2 +-
+ .../bindings/arm/qcom,coresight-ctcu.yaml          |  6 +--
+ .../bindings/arm/qcom,coresight-remote-etm.yaml    |  4 +-
+ .../bindings/arm/qcom,coresight-tnoc.yaml          |  2 +-
+ .../bindings/arm/qcom,coresight-tpda.yaml          |  4 +-
+ .../bindings/arm/qcom,coresight-tpdm.yaml          |  4 +-
+ 9 files changed, 42 insertions(+), 42 deletions(-)
+Merging fastrpc/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/srini/fastrpc.git fastrpc/for-next
+Already up to date.
+Merging fpga/for-next (c9243d6df0d6 fpga: altera-cvp: Use pci_find_vsec_capability() when probing FPGA device)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git fpga/for-next
+Already up to date.
+Merging icc/icc-next (60b52af082b1 Merge branch 'icc-sm6350' into icc-next)
+$ git merge -m Merge branch 'icc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/djakov/icc.git icc/icc-next
+Already up to date.
+Merging iio/togreg (f9e057916428 staging: iio: adt7316: replace sprintf() with sysfs_emit())
+$ git merge -m Merge branch 'togreg' of https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git iio/togreg
+Already up to date.
+Merging phy-next/next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git phy-next/next
+Already up to date.
+Merging soundwire/next (59946373755d soundwire: dmi-quirks: add mapping for Avell B.ON (OEM rebranded of NUC15))
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git soundwire/next
+Merge made by the 'ort' strategy.
+ drivers/soundwire/dmi-quirks.c      | 11 +++++++++++
+ drivers/soundwire/intel_auxdevice.c |  1 +
+ drivers/soundwire/qcom.c            | 12 +++++-------
+ 3 files changed, 17 insertions(+), 7 deletions(-)
+Merging extcon/extcon-next (b4338338aa3d extcon: usbc-tusb320: Make typec-power-opmode optional)
+$ git merge -m Merge branch 'extcon-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/extcon.git extcon/extcon-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/extcon/extcon-ptn5150.yaml |  3 ++
+ drivers/extcon/Kconfig                             |  1 +
+ drivers/extcon/extcon-intel-int3496.c              |  4 +-
+ drivers/extcon/extcon-ptn5150.c                    | 56 ++++++++++++++++++++++
+ drivers/extcon/extcon-usbc-tusb320.c               | 24 +++++-----
+ drivers/extcon/extcon.c                            |  4 +-
+ 6 files changed, 75 insertions(+), 17 deletions(-)
+Merging gnss/gnss-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'gnss-next' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git gnss/gnss-next
+Already up to date.
+Merging vfio/next (d721f52e3155 vfio: selftests: Add vfio_pci_device_init_perf_test)
+$ git merge -m Merge branch 'next' of https://github.com/awilliam/linux-vfio.git vfio/next
+Already up to date.
+Merging w1/for-next (761fcf46a1bd w1: therm: Fix off-by-one buffer overflow in alarms_store)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git w1/for-next
+Merge made by the 'ort' strategy.
+ drivers/w1/slaves/w1_therm.c | 60 ++++++++++++++------------------------------
+ 1 file changed, 19 insertions(+), 41 deletions(-)
+Merging spmi/spmi-next (0ff41df1cb26 Linux 6.15)
+$ git merge -m Merge branch 'spmi-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sboyd/spmi.git spmi/spmi-next
+Already up to date.
+Merging staging/staging-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'staging-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git staging/staging-next
+Already up to date.
+Merging counter-next/counter-next (b462fcd08dd5 MAINTAINERS: Update Intel Quadrature Encoder Peripheral maintainer)
+$ git merge -m Merge branch 'counter-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git counter-next/counter-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+Merging mux/for-next (59b723cd2adb Linux 6.12-rc6)
+$ git merge -m Merge branch 'for-next' of https://gitlab.com/peda-linux/mux.git mux/for-next
+Already up to date.
+Merging dmaengine/next (de4761fb57f6 dmaengine: shdma: correct most kernel-doc issues in shdma-base.h)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git dmaengine/next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/dma/renesas,rz-dmac.yaml   |   1 +
+ Documentation/driver-api/dmaengine/provider.rst    |   5 +-
+ drivers/dma/amba-pl08x.c                           |   2 +-
+ drivers/dma/at_xdmac.c                             |  11 --
+ drivers/dma/dw-edma/dw-edma-pcie.c                 |   4 +-
+ drivers/dma/pl330.c                                |   8 +-
+ drivers/dma/qcom/bam_dma.c                         | 152 +++++++++----------
+ drivers/dma/ste_dma40.c                            |  14 +-
+ drivers/dma/stm32/stm32-dma3.c                     | 165 +++++++++++++++++----
+ drivers/dma/ti/Kconfig                             |   5 +-
+ drivers/dma/ti/cppi41.c                            |   5 -
+ drivers/dma/xilinx/zynqmp_dma.c                    |   2 -
+ include/linux/dma/edma.h                           |  24 ++-
+ include/linux/shdma-base.h                         |  28 ++--
+ 14 files changed, 244 insertions(+), 182 deletions(-)
+Merging cgroup/for-next (e02fe7ba29a2 Merge branch 'for-6.20' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git cgroup/for-next
+Auto-merging mm/memcontrol.c
+Merge made by the 'ort' strategy.
+ fs/fs-writeback.c                                  |   2 +-
+ include/linux/cpuset.h                             |   2 +
+ include/linux/memcontrol.h                         |   2 +-
+ kernel/cgroup/cgroup.c                             |   4 +-
+ kernel/cgroup/cpuset-internal.h                    |  42 ++-
+ kernel/cgroup/cpuset-v1.c                          | 241 ++++++++++++++-
+ kernel/cgroup/cpuset.c                             | 326 +++++----------------
+ mm/memcontrol.c                                    |   2 +-
+ mm/page_owner.c                                    |   2 +-
+ tools/testing/selftests/cgroup/lib/cgroup_util.c   |  21 ++
+ .../selftests/cgroup/lib/include/cgroup_util.h     |   5 +
+ tools/testing/selftests/cgroup/test_kmem.c         |  33 +--
+ tools/testing/selftests/cgroup/test_memcontrol.c   |  20 +-
+ 13 files changed, 410 insertions(+), 292 deletions(-)
+Merging scsi/for-next (98bcf5e5ddfd Merge branch 'misc' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi.git scsi/for-next
+Merge made by the 'ort' strategy.
+Merging scsi-mkp/for-next (8334f93075dc Merge patch series "qla2xxx: Misc feature and bug fixes")
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git scsi-mkp/for-next
+Auto-merging drivers/scsi/scsi_debug.c
+Auto-merging drivers/scsi/scsi_transport_fc.c
+Auto-merging drivers/target/target_core_spc.c
+Merge made by the 'ort' strategy.
+ drivers/base/transport_class.c        |    8 +
+ drivers/scsi/lpfc/lpfc_attr.c         |   40 +
+ drivers/scsi/lpfc/lpfc_debugfs.c      |    7 +
+ drivers/scsi/lpfc/lpfc_disc.h         |    7 +
+ drivers/scsi/lpfc/lpfc_els.c          |   57 ++
+ drivers/scsi/lpfc/lpfc_hbadisc.c      |    1 +
+ drivers/scsi/lpfc/lpfc_hw4.h          |   11 +-
+ drivers/scsi/lpfc/lpfc_init.c         |    5 +
+ drivers/scsi/lpfc/lpfc_logmsg.h       |    3 +-
+ drivers/scsi/lpfc/lpfc_sli.c          |   62 +-
+ drivers/scsi/lpfc/lpfc_sli4.h         |    4 +
+ drivers/scsi/lpfc/lpfc_version.h      |    2 +-
+ drivers/scsi/mpt3sas/mpt3sas_base.c   |   14 +-
+ drivers/scsi/mpt3sas/mpt3sas_base.h   |    4 +
+ drivers/scsi/mpt3sas/mpt3sas_scsih.c  | 1332 ++++++++++++++++++++++++++++++++-
+ drivers/scsi/qla2xxx/qla_bsg.c        |  147 +++-
+ drivers/scsi/qla2xxx/qla_bsg.h        |   12 +
+ drivers/scsi/qla2xxx/qla_def.h        |   30 +-
+ drivers/scsi/qla2xxx/qla_gbl.h        |    5 +
+ drivers/scsi/qla2xxx/qla_gs.c         |   41 +-
+ drivers/scsi/qla2xxx/qla_init.c       |  232 +++++-
+ drivers/scsi/qla2xxx/qla_isr.c        |   19 +-
+ drivers/scsi/qla2xxx/qla_mbx.c        |   88 +++
+ drivers/scsi/qla2xxx/qla_nx.h         |    1 +
+ drivers/scsi/qla2xxx/qla_os.c         |    3 +-
+ drivers/scsi/qla2xxx/qla_sup.c        |   29 +
+ drivers/scsi/qla2xxx/qla_version.h    |    8 +-
+ drivers/scsi/scsi_debug.c             |  118 ++-
+ drivers/scsi/scsi_lib.c               |   11 +-
+ drivers/scsi/scsi_priv.h              |    3 +-
+ drivers/scsi/scsi_transport_fc.c      |   42 ++
+ drivers/target/target_core_configfs.c |   50 ++
+ drivers/target/target_core_spc.c      |   86 +++
+ drivers/ufs/core/ufs-mcq.c            |    3 +-
+ include/linux/transport_class.h       |    1 +
+ include/scsi/scsi.h                   |   13 +-
+ include/scsi/scsi_transport_fc.h      |   12 +
+ include/target/target_core_base.h     |    4 +
+ include/ufs/ufshci.h                  |    1 +
+ 39 files changed, 2296 insertions(+), 220 deletions(-)
+Merging vhost/linux-next (1bac705c2ce7 vhost/vsock: improve RCU read sections around vhost_vsock_get())
+$ git merge -m Merge branch 'linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git vhost/linux-next
+Auto-merging drivers/vhost/vhost.c
+Auto-merging tools/virtio/linux/compiler.h
+Merge made by the 'ort' strategy.
+ drivers/vhost/vhost.c              |   8 +--
+ drivers/vhost/vsock.c              |  15 ++++--
+ include/linux/virtio.h             |   2 +
+ include/linux/virtio_features.h    |   2 +
+ tools/virtio/Makefile              |   8 +--
+ tools/virtio/linux/compiler.h      |   6 +++
+ tools/virtio/linux/cpumask.h       |   4 ++
+ tools/virtio/linux/device.h        |   8 +++
+ tools/virtio/linux/dma-mapping.h   |   4 ++
+ tools/virtio/linux/kernel.h        |  16 ++++++
+ tools/virtio/linux/module.h        |   2 +
+ tools/virtio/linux/ucopysize.h     |  21 ++++++++
+ tools/virtio/linux/virtio.h        |  73 +-------------------------
+ tools/virtio/linux/virtio_config.h | 102 +------------------------------------
+ tools/virtio/oot-stubs.h           |  10 ++++
+ 15 files changed, 97 insertions(+), 184 deletions(-)
+ create mode 100644 tools/virtio/linux/ucopysize.h
+ create mode 100644 tools/virtio/oot-stubs.h
+Merging rpmsg/for-next (8d201aac4d4c Merge branches 'rproc-next' and 'rpmsg-next' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/remoteproc/linux.git rpmsg/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/remoteproc/ti,keystone-rproc.txt      |   2 +-
+ drivers/remoteproc/imx_dsp_rproc.c                 |  87 ++++++++++++------
+ drivers/remoteproc/imx_rproc.c                     |   2 +-
+ drivers/rpmsg/rpmsg_core.c                         |  66 ++++++--------
+ drivers/rpmsg/virtio_rpmsg_bus.c                   | 101 +++++++--------------
+ 5 files changed, 120 insertions(+), 138 deletions(-)
+Merging gpio-brgl/gpio/for-next (47d8cb678081 gpio: realtek-otto: use larger type for dev_flags)
+$ git merge -m Merge branch 'gpio/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git gpio-brgl/gpio/for-next
+Auto-merging Documentation/devicetree/bindings/gpio/gpio-mmio.yaml
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/gpio/gpio-mmio.yaml        | 15 +++++++----
+ .../devicetree/bindings/gpio/gpio-pca95xx.yaml     |  2 ++
+ Documentation/driver-api/gpio/pca953x.rst          | 12 +++++++++
+ drivers/gpio/Kconfig                               |  6 ++---
+ drivers/gpio/gpio-aggregator.c                     |  8 +++---
+ drivers/gpio/gpio-pca953x.c                        |  6 +++++
+ drivers/gpio/gpio-realtek-otto.c                   |  5 ++--
+ drivers/gpio/gpio-sim.c                            | 16 ++++++------
+ drivers/gpio/gpio-virtuser.c                       |  8 +++---
+ drivers/gpio/gpiolib-of.c                          |  4 +++
+ drivers/gpio/gpiolib-swnode.c                      | 11 +++-----
+ drivers/gpio/gpiolib.c                             | 30 +++++++++-------------
+ 12 files changed, 71 insertions(+), 52 deletions(-)
+Merging gpio-intel/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git gpio-intel/for-next
+Already up to date.
+Merging pinctrl/for-next (666065caa31a pinctrl: add CONFIG_OF dependencies for microchip drivers)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl.git pinctrl/for-next
+Already up to date.
+Merging pinctrl-intel/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git pinctrl-intel/for-next
+Already up to date.
+Merging pinctrl-renesas/renesas-pinctrl (3b0cf6ab3590 pinctrl: renesas: rzg2l: Refactor OEN register PWPR handling)
+$ git merge -m Merge branch 'renesas-pinctrl' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git pinctrl-renesas/renesas-pinctrl
+Already up to date.
+Merging pinctrl-samsung/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git pinctrl-samsung/for-next
+Already up to date.
+Merging pwm/pwm/for-next (9580ecfbc57d rust: pwm: Add UnregisteredChip wrapper around Chip)
+$ git merge -m Merge branch 'pwm/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ukleinek/linux.git pwm/pwm/for-next
+Merge made by the 'ort' strategy.
+ drivers/pwm/pwm_th1520.rs |  2 +-
+ rust/kernel/pwm.rs        | 71 ++++++++++++++++++++++++++++++-----------------
+ 2 files changed, 47 insertions(+), 26 deletions(-)
+Merging ktest/for-next (d3042cbe84a0 ktest.pl: Fix uninitialized var in config-bisect.pl)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-ktest.git ktest/for-next
+Already up to date.
+Merging kselftest/next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kselftest/next
+Already up to date.
+Merging kunit/test (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'test' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit/test
+Already up to date.
+Merging kunit-next/kunit (2115b53400fb rust: kunit: add __rust_helper to helpers)
+$ git merge -m Merge branch 'kunit' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit-next/kunit
+Merge made by the 'ort' strategy.
+ rust/helpers/kunit.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging livepatching/for-next (51ab33fc0a8b Merge tag 'livepatching-for-6.19' of git://git.kernel.org/pub/scm/linux/kernel/git/livepatching/livepatching)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/livepatching/livepatching.git livepatching/for-next
+Already up to date.
+Merging rtc/rtc-next (16bd954c9336 rtc: spacemit: MFD_SPACEMIT_P1 as dependencies)
+$ git merge -m Merge branch 'rtc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git rtc/rtc-next
+Already up to date.
+Merging nvdimm/libnvdimm-for-next (30065e73d7c0 nvdimm: Prevent integer overflow in ramdax_get_config_data())
+$ git merge -m Merge branch 'libnvdimm-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git nvdimm/libnvdimm-for-next
+Already up to date.
+Merging at24/at24/for-next (7a29af24b288 eeprom: at24: use dev_err_probe() consistently)
+$ git merge -m Merge branch 'at24/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git at24/at24/for-next
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/eeprom/at24.yaml |  3 +++
+ drivers/misc/eeprom/at24.c                         | 20 +++++++-------------
+ 2 files changed, 10 insertions(+), 13 deletions(-)
+Merging ntb/ntb-next (68113d260674 NTB/msi: Remove unused functions)
+$ git merge -m Merge branch 'ntb-next' of https://github.com/jonmason/ntb.git ntb/ntb-next
+Merge made by the 'ort' strategy.
+ drivers/ntb/hw/mscc/ntb_hw_switchtec.c | 14 ++++++--
+ drivers/ntb/msi.c                      | 64 ----------------------------------
+ include/linux/ntb.h                    | 14 --------
+ 3 files changed, 11 insertions(+), 81 deletions(-)
+Merging seccomp/for-next/seccomp (7d0a66e4bb90 Linux 6.18)
+$ git merge -m Merge branch 'for-next/seccomp' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git seccomp/for-next/seccomp
+Already up to date.
+Merging slimbus/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/srini/slimbus.git slimbus/for-next
+Already up to date.
+Merging nvmem/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/srini/nvmem.git nvmem/for-next
+Already up to date.
+Merging xarray/main (863810caeb3c XArray: Add CONFIG_DEBUG_XARRAY)
+$ git merge -m Merge branch 'main' of git://git.infradead.org/users/willy/xarray.git xarray/main
+Auto-merging lib/Kconfig.debug
+Auto-merging lib/idr.c
+Merge made by the 'ort' strategy.
+ include/linux/xarray.h               | 63 ++++++++++++++++++++----------------
+ lib/Kconfig.debug                    |  8 +++++
+ lib/idr.c                            |  1 -
+ lib/test_xarray.c                    | 10 ++++--
+ lib/xarray.c                         | 11 ++++---
+ tools/testing/radix-tree/xarray.c    |  2 +-
+ tools/testing/shared/xarray-shared.h |  2 +-
+ 7 files changed, 59 insertions(+), 38 deletions(-)
+Merging hyperv/hyperv-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'hyperv-next' of https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git hyperv/hyperv-next
+Already up to date.
+Merging auxdisplay/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git auxdisplay/for-next
+Already up to date.
+Merging kgdb/kgdb/for-next (fdbdd0ccb30a kdb: remove redundant check for scancode 0xe0)
+$ git merge -m Merge branch 'kgdb/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/danielt/linux.git kgdb/kgdb/for-next
+Already up to date.
+Merging hmm/hmm (19272b37aa4f Linux 6.16-rc1)
+$ git merge -m Merge branch 'hmm' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git hmm/hmm
+Already up to date.
+Merging cfi/cfi/next (3a8660878839 Linux 6.18-rc1)
+$ git merge -m Merge branch 'cfi/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git cfi/cfi/next
+Already up to date.
+Merging mhi/mhi-next (ac35e04f8000 bus: mhi: host: pci_generic: Add Foxconn T99W760 modem)
+$ git merge -m Merge branch 'mhi-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mani/mhi.git mhi/mhi-next
+Already up to date.
+Merging memblock/for-next (83c8f7b5e194 mm/mm_init: Introduce a boot parameter for check_pages)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git memblock/for-next
+Already up to date.
+Merging cxl/next (ea5514e30056 Merge branch 'for-6.19/cxl-misc' into cxl-for-next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git cxl/next
+Already up to date.
+Merging zstd/zstd-next (65d1f5507ed2 zstd: Import upstream v1.5.7)
+$ git merge -m Merge branch 'zstd-next' of https://github.com/terrelln/linux.git zstd/zstd-next
+Already up to date.
+Merging efi/next (c5a8f13f1e47 efi: Support EDID information)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git efi/next
+Auto-merging arch/x86/kernel/setup.c
+Merge made by the 'ort' strategy.
+ arch/arm64/kernel/image-vars.h                 |   2 +-
+ arch/loongarch/kernel/efi.c                    |  31 +++---
+ arch/loongarch/kernel/image-vars.h             |   2 +-
+ arch/riscv/kernel/image-vars.h                 |   2 +-
+ arch/x86/kernel/kexec-bzimage64.c              |   4 +-
+ arch/x86/kernel/setup.c                        |  16 ++-
+ arch/x86/video/video-common.c                  |   4 +-
+ drivers/firmware/efi/earlycon.c                |  42 ++++----
+ drivers/firmware/efi/efi-init.c                |  38 +++----
+ drivers/firmware/efi/efi.c                     |   4 +-
+ drivers/firmware/efi/libstub/Makefile          |   2 +-
+ drivers/firmware/efi/libstub/efi-stub-entry.c  |  17 +--
+ drivers/firmware/efi/libstub/efi-stub.c        |  43 +++++---
+ drivers/firmware/efi/libstub/efistub.h         |   7 +-
+ drivers/firmware/efi/libstub/primary_display.c |  56 ++++++++++
+ drivers/firmware/efi/libstub/screen_info.c     |  53 ---------
+ drivers/firmware/efi/libstub/zboot.c           |   4 +-
+ drivers/firmware/efi/sysfb_efi.c               | 144 ++++++++++++++++++-------
+ drivers/firmware/sysfb.c                       |  13 +--
+ drivers/firmware/sysfb_simplefb.c              |   2 +-
+ drivers/gpu/drm/sysfb/efidrm.c                 |  14 +--
+ drivers/gpu/drm/sysfb/vesadrm.c                |  14 +--
+ drivers/hv/vmbus_drv.c                         |   6 +-
+ drivers/pci/vgaarb.c                           |   4 +-
+ drivers/video/Kconfig                          |   8 +-
+ drivers/video/fbdev/core/fbmon.c               |   8 +-
+ drivers/video/fbdev/efifb.c                    |  10 +-
+ drivers/video/fbdev/vesafb.c                   |  10 +-
+ drivers/video/fbdev/vga16fb.c                  |   8 +-
+ drivers/video/screen_info_pci.c                |   5 +-
+ include/linux/efi.h                            |   9 +-
+ include/linux/screen_info.h                    |   2 -
+ include/linux/sysfb.h                          |  23 +++-
+ include/video/edid.h                           |   4 -
+ 34 files changed, 359 insertions(+), 252 deletions(-)
+ create mode 100644 drivers/firmware/efi/libstub/primary_display.c
+ delete mode 100644 drivers/firmware/efi/libstub/screen_info.c
+Merging unicode/for-next (6b56a63d286f MAINTAINERS: Add Unicode tree)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krisman/unicode.git unicode/for-next
+Already up to date.
+Merging slab/slab/for-next (58d6fcb6ee94 slub: keep empty main sheaf as spare in __pcs_replace_empty_main())
+$ git merge -m Merge branch 'slab/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git slab/slab/for-next
+Auto-merging mm/slub.c
+Merge made by the 'ort' strategy.
+ mm/slub.c | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+Merging random/master (3f9f0252130e Merge tag 'random-6.19-rc1-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/crng/random)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/crng/random.git random/master
+Already up to date.
+Merging landlock/next (1a3cedbdc156 landlock: Fix wrong type usage)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/mic/linux.git landlock/next
+Merge made by the 'ort' strategy.
+ security/landlock/audit.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging sysctl/sysctl-next (564195c1a33c sysctl: Wrap do_proc_douintvec with the public function proc_douintvec_conv)
+$ git merge -m Merge branch 'sysctl-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sysctl/sysctl.git sysctl/sysctl-next
+Already up to date.
+Merging execve/for-next/execve (7d0a66e4bb90 Linux 6.18)
+$ git merge -m Merge branch 'for-next/execve' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git execve/for-next/execve
+Already up to date.
+Merging bitmap/bitmap-for-next (abb934acc479 rust: cpumask: add __rust_helper to helpers)
+$ git merge -m Merge branch 'bitmap-for-next' of https://github.com/norov/linux.git bitmap/bitmap-for-next
+Merge made by the 'ort' strategy.
+ include/linux/bitfield.h |  5 +++--
+ rust/helpers/bitmap.c    |  1 +
+ rust/helpers/bitops.c    |  4 ++++
+ rust/helpers/cpumask.c   | 13 +++++++++++++
+ 4 files changed, 21 insertions(+), 2 deletions(-)
+Merging hte/for-next (9e4259716f60 hte: tegra-194: add missing MODULE_DESCRIPTION() macro)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pateldipen1984/linux.git hte/for-next
+Already up to date.
+Merging kspp/for-next/kspp (7454048db27d kbuild: Enable GCC diagnostic context for value-tracking warnings)
+$ git merge -m Merge branch 'for-next/kspp' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git kspp/for-next/kspp
+Already up to date.
+Merging nolibc/for-next (5203a78a7c60 tools/nolibc: always use 64-bit mode for s390 header checks)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/nolibc/linux-nolibc.git nolibc/for-next
+Merge made by the 'ort' strategy.
+ tools/include/nolibc/Makefile                |  6 +++-
+ tools/include/nolibc/nolibc.h                |  1 +
+ tools/include/nolibc/sys/ptrace.h            | 44 ++++++++++++++++++++++++++++
+ tools/testing/selftests/nolibc/nolibc-test.c |  2 ++
+ 4 files changed, 52 insertions(+), 1 deletion(-)
+ create mode 100644 tools/include/nolibc/sys/ptrace.h
+Merging iommufd/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git iommufd/for-next
+Already up to date.
+Merging turbostat/next (b65a8280c766 tools/power turbostat.8: Document the "--force" option)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux.git turbostat/next
+Merge made by the 'ort' strategy.
+ tools/power/x86/turbostat/turbostat.8 |   4 +
+ tools/power/x86/turbostat/turbostat.c | 143 +++++++++++++++++++++-------------
+ 2 files changed, 94 insertions(+), 53 deletions(-)
+Merging pwrseq/pwrseq/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'pwrseq/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git pwrseq/pwrseq/for-next
+Already up to date.
+Merging capabilities-next/caps-next (9891d2f79a9f Clarify the rootid_owns_currentns)
+$ git merge -m Merge branch 'caps-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sergeh/linux.git capabilities-next/caps-next
+Already up to date.
+Merging ipe/next (7d0a66e4bb90 Linux 6.18)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/wufan/ipe.git ipe/next
+Already up to date.
+Merging kcsan/next (dcb6fa37fd7b Linux 6.18-rc3)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/melver/linux.git kcsan/next
+Already up to date.
+Merging crc/crc-next (dcb6fa37fd7b Linux 6.18-rc3)
+$ git merge -m Merge branch 'crc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git crc/crc-next
+Already up to date.
+Merging fwctl/for-next (8f0b4cce4481 Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git fwctl/for-next
+Already up to date.
+Merging devsec-tsm/next (7dfbe9a67519 crypto/ccp: Fix CONFIG_PCI=n build)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git devsec-tsm/next
+Already up to date.
+Merging hisilicon/for-next (6c177775dcc5 Merge branch 'next/drivers' into for-next)
+$ git merge -m Merge branch 'for-next' of https://github.com/hisilicon/linux-hisi.git hisilicon/for-next
+Merge made by the 'ort' strategy.
+Merging kthread/for-next (d8b4bf4ea04d kthread: modify kernel-doc function name to match code)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git kthread/for-next
+Already up to date.
diff --git a/arch/arm/Kbuild b/arch/arm/Kbuild
index 69de6b6243c7..af7dd53585c3 100644
--- a/arch/arm/Kbuild
+++ b/arch/arm/Kbuild
@@ -10,5 +10,7 @@ obj-y				+= probes/
 obj-y				+= net/
 obj-y				+= crypto/
 
+subdir-y += boot/dts
+
 # for cleaning
 subdir- += boot
diff --git a/arch/arm/arm-soc-for-next-contents.txt b/arch/arm/arm-soc-for-next-contents.txt
new file mode 100644
index 000000000000..4a431608f93f
--- /dev/null
+++ b/arch/arm/arm-soc-for-next-contents.txt
@@ -0,0 +1,177 @@
+soc/arm
+	mediatek/arm
+		https://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux tags/mtk-arm32-for-v6.19
+	patch
+		ARM: versatile: Fix typo in versatile.c
+		ARM: gemini: fix typos in comments
+	omap/arm
+		git://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap tags/omap-for-v6.19/soc-signed
+
+soc/dt
+	mmp/dt
+		https://gitlab.com/pxa1908-mainline/linux tags/pxa1908-dt-for-6.19
+	aspeed/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/bmc/linux tags/aspeed-6.19-devicetree-0
+	renesas/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-dts-for-v6.19-tag1
+	thead/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux tags/thead-dt-for-v6.19
+	exynos/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-dt64-6.19
+	mediatek/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux tags/mtk-dts32-for-v6.19
+	mediatek/dt64
+		https://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux tags/mtk-dts64-for-v6.19
+	spacemit/dt
+		https://github.com/spacemit-com/linux tags/spacemit-dt-for-6.19-1
+	renesas/dt2
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-dts-for-v6.19-tag2
+	tegra/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.19-dt-bindings
+	tegra/dt-2
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.19-arm-dt
+	tegra/dt64
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.19-arm64-dt
+	sunxi/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux tags/sunxi-dt-for-6.19
+	imx/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-dt-6.19
+	imx/dt64
+		https://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-dt64-6.19
+	socfpga/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/dinguyen/linux tags/socfpga_dts_updates_for_v6.19
+	k3/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux tags/ti-k3-dt-for-v6.19
+	at91/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux tags/at91-dt-6.19
+	omap/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap tags/omap-for-v6.19/dt-signed
+	qcom/dt32
+		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux tags/qcom-arm32-for-6.19
+	qcom/dt64
+		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux tags/qcom-arm64-for-6.19
+	rockchip/dt32
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v6.19-rockchip-dts32-1
+	rockchip/dt64
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v6.19-rockchip-dts64-1
+	stm32/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32 tags/stm32-dt-for-v6.19-1
+	cix/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/peter.chen/cix tags/cix-dt-v6.19-rc1
+	sophgo/dt
+		https://github.com/sophgo/linux tags/riscv-sophgo-dt-for-v6.19
+	mvebu/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu tags/mvebu-dt64-6.19-1
+	k3/dt-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux tags/ti-k3-dt-for-v6.19-part2
+	imx/dt-bindings
+		https://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-bindings-6.19
+	patch
+		arm64: dts: sprd: sc9860: Simplify clock nodes
+	rockchip/dt-2
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v6.19-rockchip-dts64-2
+	amlogic/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux tags/amlogic-arm64-dt-for-v6.19
+	patch
+		arm64: dts: mediatek: mt8195: Fix address range for JPEG decoder core 1
+	riscv/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux tags/riscv-dt-for-v6.19
+
+soc/drivers
+	zynq/soc-driver
+		https://github.com/Xilinx/linux-xlnx tags/zynqmp-soc-for-6.18
+	renesas/drivers
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-drivers-for-v6.19-tag1
+	drivers/memory
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl tags/memory-controller-drv-6.19-2
+	exynos/drivers
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-drivers-6.19
+	renesas/drivers-2
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-drivers-for-v6.19-tag2
+	tegra/drivers
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.19-core
+	tegra/syscore-changes
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.19-syscore
+	tegra/soc-driver
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.19-soc
+	sunxi/drivers
+		https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux tags/sunxi-drivers-for-6.19
+	imx/scu-driver
+		https://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-drivers-6.19
+	ti/drivers
+		https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux tags/ti-driver-soc-for-v6.19
+	omap/drivers
+		git://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap tags/omap-for-v6.19/drivers-signed
+	qcom/drivers
+		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux tags/qcom-drivers-for-6.19
+	rockchip/drivers
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v6.19-rockchip-drivers1
+	stm32/firewall
+		git://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32 tags/stm32-bus-firewall-for-v6.19-1
+	drivers/reset
+		https://git.pengutronix.de/git/pza/linux tags/reset-for-v6.19
+	mediatek/drivers
+		https://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux tags/mtk-soc-for-v6.19
+
+soc/defconfig
+	patch
+		arm64: defconfig: Remove the redundant SCHED_MC/SCHED_SMT
+	renesas/defconfig
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-arm-defconfig-for-v6.19-tag1
+	patch
+		arm64: defconfig: enable Exynos ACPM clocks
+	mediatek/defconfig
+		https://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux tags/mtk-defconfig-for-v6.19
+	tegra/defconfig32
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.19-arm-defconfig
+	tegra/defconfig64
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.19-arm64-defconfig
+	imx/defconfig
+		https://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-defconfig-6.19
+	at91/defconfig
+		https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux tags/at91-defconfig-6.19
+	omap/defconfig
+		git://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap tags/omap-for-v6.19/defconfig-signed
+	qcom/defconfig
+		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux tags/qcom-arm64-defconfig-for-6.19
+	rockchip/defconfig
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v6.19-rockchip-defconfig64-1
+	riscv/defconfig
+		https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux tags/riscv-config-for-v6.19
+
+soc/late
+
+arm/fixes
+	imx/fixes-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-fixes-6.18-2
+	tee/qcomtee-fixes
+		git://git.kernel.org/pub/scm/linux/kernel/git/jenswi/linux-tee tags/qcomtee-fixes2-for-6.18
+	<no branch> (517d066dc080fc297f1ab5e8c6fda7c59d48ef8d)
+		git://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap tags/omap-for-v6.19/maintainers-signed
+	<no branch> (9f393d8e757f79060baf4b2e703bd6b2d0d8d323)
+		https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux tags/sunxi-fixes-for-6.18
+
+soc/drivers-late
+	drivers/reset-gpio
+		https://git.pengutronix.de/git/pza/linux tags/reset-gpio-for-v6.19
+	<no branch> (527250cd9092461f1beac3e4180a4481bffa01b5)
+		https://git.pengutronix.de/git/pza/linux tags/reset-gpio-for-v6.19-2
+	amlogic/drivers
+		https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux tags/amlogic-drivers-for-v6.19
+	drivers/fsl
+		git://git.kernel.org/pub/scm/linux/kernel/git/chleroy/linux tags/soc_fsl-6.19-1
+	apple/soc-drivers
+		https://git.kernel.org/pub/scm/linux/kernel/git/sven/linux tags/apple-soc-drivers-6.19
+	microchip/soc-drivers
+		https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux tags/soc-drivers-for-v6.19
+	drivers/cache
+		https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux tags/cache-for-v6.19
+
+soc/newsoc
+	tenstorrent/newsoc
+		https://github.com/tenstorrent/linux tags/tenstorrent-dt-for-v6.19
+	bst/newsoc
+		Merge branch 'bst/newsoc' into soc/newsoc
+	anlogic/newsoc
+		https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux tags/anlogic-initial-6.19-v2
+
diff --git a/arch/arm/boot/dts/aspeed/Makefile b/arch/arm/boot/dts/aspeed/Makefile
index 9adf9278dc94..ed25fec25fc4 100644
--- a/arch/arm/boot/dts/aspeed/Makefile
+++ b/arch/arm/boot/dts/aspeed/Makefile
@@ -58,6 +58,7 @@ dtb-$(CONFIG_ARCH_ASPEED) += \
 	aspeed-bmc-lenovo-hr855xg2.dtb \
 	aspeed-bmc-microsoft-olympus.dtb \
 	aspeed-bmc-nvidia-gb200nvl-bmc.dtb \
+	aspeed-bmc-nvidia-msx4-bmc.dtb \
 	aspeed-bmc-opp-lanyang.dtb \
 	aspeed-bmc-opp-mowgli.dtb \
 	aspeed-bmc-opp-nicole.dtb \
diff --git a/arch/arm/boot/dts/aspeed/aspeed-ast2600-evb.dts b/arch/arm/boot/dts/aspeed/aspeed-ast2600-evb.dts
index de83c0eb1d6e..3f2ca9da0be2 100644
--- a/arch/arm/boot/dts/aspeed/aspeed-ast2600-evb.dts
+++ b/arch/arm/boot/dts/aspeed/aspeed-ast2600-evb.dts
@@ -205,6 +205,7 @@ flash@0 {
 &uart5 {
 	// Workaround for A0
 	compatible = "snps,dw-apb-uart";
+	/delete-property/ no-loopback-test;
 };
 
 &i2c0 {
@@ -314,9 +315,8 @@ &sdhci0 {
 	status = "okay";
 	bus-width = <4>;
 	max-frequency = <100000000>;
-	sdhci-drive-type = /bits/ 8 <3>;
 	sdhci-caps-mask = <0x7 0x0>;
-	sdhci,wp-inverted;
+	wp-inverted;
 	vmmc-supply = <&vcc_sdhci0>;
 	vqmmc-supply = <&vccq_sdhci0>;
 	clk-phase-sd-hs = <7>, <200>;
@@ -326,9 +326,8 @@ &sdhci1 {
 	status = "okay";
 	bus-width = <4>;
 	max-frequency = <100000000>;
-	sdhci-drive-type = /bits/ 8 <3>;
 	sdhci-caps-mask = <0x7 0x0>;
-	sdhci,wp-inverted;
+	wp-inverted;
 	vmmc-supply = <&vcc_sdhci1>;
 	vqmmc-supply = <&vccq_sdhci1>;
 	clk-phase-sd-hs = <7>, <200>;
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-bletchley.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-bletchley.dts
index 24969c82d05e..7c27bf6bb51d 100644
--- a/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-bletchley.dts
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-bletchley.dts
@@ -34,14 +34,14 @@ iio-hwmon {
 			<&adc1 4>, <&adc1 5>, <&adc1 6>, <&adc1 7>;
 	};
 
-	spi1_gpio: spi1-gpio {
+	spi1_gpio: spi {
 		compatible = "spi-gpio";
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		gpio-sck = <&gpio0 ASPEED_GPIO(Z, 3) GPIO_ACTIVE_HIGH>;
-		gpio-mosi = <&gpio0 ASPEED_GPIO(Z, 4) GPIO_ACTIVE_HIGH>;
-		gpio-miso = <&gpio0 ASPEED_GPIO(Z, 5) GPIO_ACTIVE_HIGH>;
+		sck-gpios = <&gpio0 ASPEED_GPIO(Z, 3) GPIO_ACTIVE_HIGH>;
+		mosi-gpios = <&gpio0 ASPEED_GPIO(Z, 4) GPIO_ACTIVE_HIGH>;
+		miso-gpios = <&gpio0 ASPEED_GPIO(Z, 5) GPIO_ACTIVE_HIGH>;
 		num-chipselects = <1>;
 		cs-gpios = <&gpio0 ASPEED_GPIO(Z, 0) GPIO_ACTIVE_LOW>;
 
@@ -54,7 +54,8 @@ tpm@0 {
 
 	front_gpio_leds {
 		compatible = "gpio-leds";
-		sys_log_id {
+		led-0 {
+			label = "sys_log_id";
 			default-state = "off";
 			gpios = <&front_leds 0 GPIO_ACTIVE_LOW>;
 		};
@@ -62,42 +63,50 @@ sys_log_id {
 
 	fan_gpio_leds {
 		compatible = "gpio-leds";
-		fan0_blue {
+		led-0 {
+			label = "fan0_blue";
 			retain-state-shutdown;
 			default-state = "on";
 			gpios = <&fan_leds 8 GPIO_ACTIVE_HIGH>;
 		};
-		fan1_blue {
+		led-1 {
+			label = "fan1_blue";
 			retain-state-shutdown;
 			default-state = "on";
 			gpios = <&fan_leds 9 GPIO_ACTIVE_HIGH>;
 		};
-		fan2_blue {
+		led-2 {
+			label = "fan2_blue";
 			retain-state-shutdown;
 			default-state = "on";
 			gpios = <&fan_leds 10 GPIO_ACTIVE_HIGH>;
 		};
-		fan3_blue {
+		led-3 {
+			label = "fan3_blue";
 			retain-state-shutdown;
 			default-state = "on";
 			gpios = <&fan_leds 11 GPIO_ACTIVE_HIGH>;
 		};
-		fan0_amber {
+		led-4 {
+			label = "fan0_amber";
 			retain-state-shutdown;
 			default-state = "off";
 			gpios = <&fan_leds 12 GPIO_ACTIVE_HIGH>;
 		};
-		fan1_amber {
+		led-5 {
+			label = "fan1_amber";
 			retain-state-shutdown;
 			default-state = "off";
 			gpios = <&fan_leds 13 GPIO_ACTIVE_HIGH>;
 		};
-		fan2_amber {
+		led-6 {
+			label = "fan2_amber";
 			retain-state-shutdown;
 			default-state = "off";
 			gpios = <&fan_leds 14 GPIO_ACTIVE_HIGH>;
 		};
-		fan3_amber {
+		led-7 {
+			label = "fan3_amber";
 			retain-state-shutdown;
 			default-state = "off";
 			gpios = <&fan_leds 15 GPIO_ACTIVE_HIGH>;
@@ -106,12 +115,14 @@ fan3_amber {
 
 	sled1_gpio_leds {
 		compatible = "gpio-leds";
-		sled1_amber {
+		led-0 {
+			label = "sled1_amber";
 			retain-state-shutdown;
 			default-state = "keep";
 			gpios = <&sled1_leds 0 GPIO_ACTIVE_LOW>;
 		};
-		sled1_blue {
+		led-1 {
+			label = "sled1_blue";
 			retain-state-shutdown;
 			default-state = "keep";
 			gpios = <&sled1_leds 1 GPIO_ACTIVE_LOW>;
@@ -120,12 +131,14 @@ sled1_blue {
 
 	sled2_gpio_leds {
 		compatible = "gpio-leds";
-		sled2_amber {
+		led-0 {
+			label = "sled2_amber";
 			retain-state-shutdown;
 			default-state = "keep";
 			gpios = <&sled2_leds 0 GPIO_ACTIVE_LOW>;
 		};
-		sled2_blue {
+		led-1 {
+			label = "sled2_blue";
 			retain-state-shutdown;
 			default-state = "keep";
 			gpios = <&sled2_leds 1 GPIO_ACTIVE_LOW>;
@@ -134,12 +147,14 @@ sled2_blue {
 
 	sled3_gpio_leds {
 		compatible = "gpio-leds";
-		sled3_amber {
+		led-0 {
+			label = "sled3_amber";
 			retain-state-shutdown;
 			default-state = "keep";
 			gpios = <&sled3_leds 0 GPIO_ACTIVE_LOW>;
 		};
-		sled3_blue {
+		led-1 {
+			label = "sled3_blue";
 			retain-state-shutdown;
 			default-state = "keep";
 			gpios = <&sled3_leds 1 GPIO_ACTIVE_LOW>;
@@ -148,12 +163,14 @@ sled3_blue {
 
 	sled4_gpio_leds {
 		compatible = "gpio-leds";
-		sled4_amber {
+		led-0 {
+			label = "sled4_amber";
 			retain-state-shutdown;
 			default-state = "keep";
 			gpios = <&sled4_leds 0 GPIO_ACTIVE_LOW>;
 		};
-		sled4_blue {
+		led-1 {
+			label = "sled4_blue";
 			retain-state-shutdown;
 			default-state = "keep";
 			gpios = <&sled4_leds 1 GPIO_ACTIVE_LOW>;
@@ -162,12 +179,14 @@ sled4_blue {
 
 	sled5_gpio_leds {
 		compatible = "gpio-leds";
-		sled5_amber {
+		led-0 {
+			label = "sled5_amber";
 			retain-state-shutdown;
 			default-state = "keep";
 			gpios = <&sled5_leds 0 GPIO_ACTIVE_LOW>;
 		};
-		sled5_blue {
+		led-1 {
+			label = "sled5_blue";
 			retain-state-shutdown;
 			default-state = "keep";
 			gpios = <&sled5_leds 1 GPIO_ACTIVE_LOW>;
@@ -176,12 +195,14 @@ sled5_blue {
 
 	sled6_gpio_leds {
 		compatible = "gpio-leds";
-		sled6_amber {
+		led-0 {
+			label = "sled6_amber";
 			retain-state-shutdown;
 			default-state = "keep";
 			gpios = <&sled6_leds 0 GPIO_ACTIVE_LOW>;
 		};
-		sled6_blue {
+		led-1 {
+			label = "sled6_blue";
 			retain-state-shutdown;
 			default-state = "keep";
 			gpios = <&sled6_leds 1 GPIO_ACTIVE_LOW>;
@@ -191,32 +212,32 @@ sled6_blue {
 	gpio-keys {
 		compatible = "gpio-keys";
 
-		presence-sled1 {
+		presence-sled1-switch {
 			label = "presence-sled1";
 			gpios = <&gpio0 ASPEED_GPIO(H, 2) GPIO_ACTIVE_LOW>;
 			linux,code = <ASPEED_GPIO(H, 2)>;
 		};
-		presence-sled2 {
+		presence-sled2-switch {
 			label = "presence-sled2";
 			gpios = <&gpio0 ASPEED_GPIO(H, 3) GPIO_ACTIVE_LOW>;
 			linux,code = <ASPEED_GPIO(H, 3)>;
 		};
-		presence-sled3 {
+		presence-sled3-switch {
 			label = "presence-sled3";
 			gpios = <&gpio0 ASPEED_GPIO(H, 4) GPIO_ACTIVE_LOW>;
 			linux,code = <ASPEED_GPIO(H, 4)>;
 		};
-		presence-sled4 {
+		presence-sled4-switch {
 			label = "presence-sled4";
 			gpios = <&gpio0 ASPEED_GPIO(H, 5) GPIO_ACTIVE_LOW>;
 			linux,code = <ASPEED_GPIO(H, 5)>;
 		};
-		presence-sled5 {
+		presence-sled5-switch {
 			label = "presence-sled5";
 			gpios = <&gpio0 ASPEED_GPIO(H, 6) GPIO_ACTIVE_LOW>;
 			linux,code = <ASPEED_GPIO(H, 6)>;
 		};
-		presence-sled6 {
+		presence-sled6-switch {
 			label = "presence-sled6";
 			gpios = <&gpio0 ASPEED_GPIO(H, 7) GPIO_ACTIVE_LOW>;
 			linux,code = <ASPEED_GPIO(H, 7)>;
@@ -352,8 +373,6 @@ sled1_ioexp41: pca9536@41 {
 	sled1_ioexp: pca9539@76 {
 		compatible = "nxp,pca9539";
 		reg = <0x76>;
-		#address-cells = <1>;
-		#size-cells = <0>;
 		gpio-controller;
 		#gpio-cells = <2>;
 
@@ -441,8 +460,6 @@ sled2_ioexp41: pca9536@41 {
 	sled2_ioexp: pca9539@76 {
 		compatible = "nxp,pca9539";
 		reg = <0x76>;
-		#address-cells = <1>;
-		#size-cells = <0>;
 		gpio-controller;
 		#gpio-cells = <2>;
 
@@ -530,8 +547,6 @@ sled3_ioexp41: pca9536@41 {
 	sled3_ioexp: pca9539@76 {
 		compatible = "nxp,pca9539";
 		reg = <0x76>;
-		#address-cells = <1>;
-		#size-cells = <0>;
 		gpio-controller;
 		#gpio-cells = <2>;
 
@@ -619,8 +634,6 @@ sled4_ioexp41: pca9536@41 {
 	sled4_ioexp: pca9539@76 {
 		compatible = "nxp,pca9539";
 		reg = <0x76>;
-		#address-cells = <1>;
-		#size-cells = <0>;
 		gpio-controller;
 		#gpio-cells = <2>;
 
@@ -708,8 +721,6 @@ sled5_ioexp41: pca9536@41 {
 	sled5_ioexp: pca9539@76 {
 		compatible = "nxp,pca9539";
 		reg = <0x76>;
-		#address-cells = <1>;
-		#size-cells = <0>;
 		gpio-controller;
 		#gpio-cells = <2>;
 
@@ -797,8 +808,6 @@ sled6_ioexp41: pca9536@41 {
 	sled6_ioexp: pca9539@76 {
 		compatible = "nxp,pca9539";
 		reg = <0x76>;
-		#address-cells = <1>;
-		#size-cells = <0>;
 		gpio-controller;
 		#gpio-cells = <2>;
 
@@ -953,7 +962,6 @@ fan_leds: pca9552@67 {
 
 &i2c13 {
 	multi-master;
-	aspeed,hw-timeout-ms = <1000>;
 	status = "okay";
 
 	//USB Debug Connector
@@ -1024,7 +1032,7 @@ &gpio0 {
 };
 
 &adc0 {
-	vref = <1800>;
+	aspeed,int-vref-microvolt = <2500000>;
 	status = "okay";
 
 	pinctrl-names = "default";
@@ -1035,7 +1043,7 @@ &pinctrl_adc4_default &pinctrl_adc5_default
 };
 
 &adc1 {
-	vref = <2500>;
+	aspeed,int-vref-microvolt = <2500000>;
 	status = "okay";
 
 	pinctrl-names = "default";
@@ -1080,11 +1088,5 @@ pinctrl_gpiov2_unbiased_default: gpiov2 {
 
 &wdt1 {
 	status = "okay";
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_wdtrst1_default>;
 	aspeed,reset-type = "soc";
-	aspeed,external-signal;
-	aspeed,ext-push-pull;
-	aspeed,ext-active-high;
-	aspeed,ext-pulse-duration = <256>;
 };
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-clemente.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-clemente.dts
index 450446913e36..2aff21442f11 100644
--- a/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-clemente.dts
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-clemente.dts
@@ -96,7 +96,12 @@ led-3 {
 			gpios = <&gpio0 ASPEED_GPIO(P, 5) (GPIO_ACTIVE_HIGH|GPIO_TRANSITORY)>;
 		};
 
-		led-hdd {
+	};
+
+	hdd-leds {
+		compatible = "gpio-leds";
+
+		led-0 {
 			label = "hdd_led";
 			gpios = <&io_expander13 1 GPIO_ACTIVE_LOW>;
 		};
@@ -311,6 +316,12 @@ i2c0mux0ch1mux0ch0: i2c@0 {
 					#address-cells = <1>;
 					#size-cells = <0>;
 					reg = <0>;
+
+					// HDD NVMe SSD FRU 0
+					eeprom@53 {
+						compatible = "atmel,24c02";
+						reg = <0x53>;
+					};
 				};
 
 				i2c0mux0ch1mux0ch1: i2c@1 {
@@ -323,6 +334,12 @@ i2c0mux0ch1mux0ch2: i2c@2 {
 					#address-cells = <1>;
 					#size-cells = <0>;
 					reg = <2>;
+
+					// HDD NVMe SSD FRU 1
+					eeprom@53 {
+						compatible = "atmel,24c02";
+						reg = <0x53>;
+					};
 				};
 
 				i2c0mux0ch1mux0ch3: i2c@3 {
@@ -493,6 +510,12 @@ i2c0mux3ch1mux0ch0: i2c@0 {
 					#address-cells = <1>;
 					#size-cells = <0>;
 					reg = <0>;
+
+					// HDD NVMe SSD FRU 2
+					eeprom@53 {
+						compatible = "atmel,24c02";
+						reg = <0x53>;
+					};
 				};
 
 				i2c0mux3ch1mux0ch1: i2c@1 {
@@ -505,6 +528,12 @@ i2c0mux3ch1mux0ch2: i2c@2 {
 					#address-cells = <1>;
 					#size-cells = <0>;
 					reg = <2>;
+
+					// HDD NVMe SSD FRU 3
+					eeprom@53 {
+						compatible = "atmel,24c02";
+						reg = <0x53>;
+					};
 				};
 
 				i2c0mux3ch1mux0ch3: i2c@3 {
@@ -619,6 +648,12 @@ i2c0mux5ch1: i2c@1 {
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <1>;
+
+			// BOOT DRIVE FRU
+			eeprom@53 {
+				compatible = "atmel,24c02";
+				reg = <0x53>;
+			};
 		};
 
 		i2c0mux5ch2: i2c@2 {
@@ -983,7 +1018,7 @@ io_expander4: gpio@4f {
 			"",
 			"",
 			"",
-			"",
+			"shdn_force_l_cpld",
 			"",
 			"",
 			"",
@@ -1258,10 +1293,6 @@ &mac3 {
 	use-ncsi;
 };
 
-&udma {
-	status = "okay";
-};
-
 &uart1 {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-harma.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-harma.dts
index 1c50e4a367b2..5602a502d07b 100644
--- a/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-harma.dts
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-harma.dts
@@ -822,9 +822,13 @@ &sgpiom0 {
 	"irq-pvddcore1-ocp-alert","",
 	"","",
 	/*O4-O7 line 232-239*/
-	"","","","","","","","",
+	"","","","",
+	"presence-lower-fanboard1","",
+	"presence-lower-fanboard2","",
 	/*P0-P3 line 240-247*/
-	"","","","","","","","",
+	"presence-upper-fanboard1","",
+	"presence-upper-fanboard2","",
+	"","","","",
 	/*P4-P7 line 248-255*/
 	"","","","","","","","";
 };
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-santabarbara.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-santabarbara.dts
index f74f463cc878..0a3e2e241063 100644
--- a/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-santabarbara.dts
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-santabarbara.dts
@@ -845,7 +845,14 @@ temperature-sensor@49 {
 };
 
 &i2c7 {
+	multi-master;
 	status = "okay";
+
+	ipmb@10 {
+		compatible = "ipmb-dev";
+		reg = <(0x10 | I2C_OWN_SLAVE_ADDRESS)>;
+		i2c-protocol;
+	};
 };
 
 &i2c8 {
@@ -1328,6 +1335,20 @@ eeprom@50 {
 &i2c12 {
 	status = "okay";
 
+	gpio@27 {
+		compatible = "nxp,pca9555";
+		reg = <0x27>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		gpio-line-names =
+			"PEX0_MODE_SEL1_R","PEX0_MODE_SEL2_R",
+			"PEX0_MODE_SEL3_R","PEX0_MODE_SEL4_R",
+			"","","","",
+			"UART_MUX_SEL","RST_SMB_NIC_R_N",
+			"RST_SMB_N","RST_CP2102N_N",
+			"SPI_MUX_SEL","","","";
+	};
+
 	// SWB FRU
 	eeprom@52 {
 		compatible = "atmel,24c64";
@@ -1758,11 +1779,11 @@ &sgpiom0 {
 	"","BIOS_DEBUG_MODE",
 	/*H0-H3 line 112-119*/
 	"FM_IOEXP_U538_INT_N","",
-	"FM_IOEXP_U539_INT_N","",
-	"FM_IOEXP_U540_INT_N","",
-	"FM_IOEXP_U541_INT_N","",
+	"FM_IOEXP_U539_INT_N","FM_MODULE_PWR_EN_N_1B",
+	"FM_IOEXP_U540_INT_N","FM_MODULE_PWR_EN_N_2B",
+	"FM_IOEXP_U541_INT_N","FM_MODULE_PWR_EN_N_3B",
 	/*H4-H7 line 120-127*/
-	"FM_IOEXP_PDB2_U1003_INT_N","",
+	"FM_IOEXP_PDB2_U1003_INT_N","FM_MODULE_PWR_EN_N_4B",
 	"","",
 	"","",
 	"FM_MAIN_PWREN_RMC_EN_ISO_R","",
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-nvidia-msx4-bmc.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-nvidia-msx4-bmc.dts
new file mode 100644
index 000000000000..44f95a3986cb
--- /dev/null
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-nvidia-msx4-bmc.dts
@@ -0,0 +1,246 @@
+// SPDX-License-Identifier: GPL-2.0
+/dts-v1/;
+
+#include "aspeed-g6.dtsi"
+#include <dt-bindings/gpio/aspeed-gpio.h>
+#include <dt-bindings/i2c/i2c.h>
+
+/ {
+	model = "AST2600 MSX4 BMC";
+	compatible = "nvidia,msx4-bmc", "aspeed,ast2600";
+
+	aliases {
+		serial0 = &uart1;
+		serial1 = &uart2;
+		serial2 = &uart3;
+		serial3 = &uart4;
+		serial4 = &uart5;
+	};
+
+	chosen {
+		stdout-path = "uart5:115200n8";
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x80000000 0x80000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		gfx_memory: framebuffer {
+			compatible = "shared-dma-pool";
+			size = <0x01000000>;
+			alignment = <0x01000000>;
+			reusable;
+		};
+
+		video_engine_memory: jpegbuffer {
+			compatible = "shared-dma-pool";
+			size = <0x02000000>;	/* 32M */
+			alignment = <0x01000000>;
+			reusable;
+		};
+	};
+};
+
+&ehci1 {
+	status = "okay";
+};
+
+&fmc {
+	status = "okay";
+
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		label = "bmc";
+		status = "okay";
+		#include "openbmc-flash-layout-128.dtsi"
+	};
+
+	flash@1 {
+		compatible = "jedec,spi-nor";
+		label = "alt-bmc";
+		spi-tx-bus-width = <1>;
+		spi-rx-bus-width = <1>;
+		status = "okay";
+	};
+};
+
+&gfx {
+	memory-region = <&gfx_memory>;
+	status = "okay";
+};
+
+&gpio0 {
+	gpio-line-names =
+	/*A0-A7*/	"","","","","","","","",
+	/*B0-B7*/	"ASSERT_BMC_READY","","","","","","","",
+	/*C0-C7*/	"MON_PWR_GOOD","","","","","","","FP_ID_LED_N",
+	/*D0-D7*/	"","","","","","","","",
+	/*E0-E7*/	"","","","","","","","",
+	/*F0-F7*/	"","","","","","","","",
+	/*G0-G7*/	"","","FP_LED_STATUS_GREEN_N","FP_LED_STATUS_AMBER_N",
+			"","","","",
+	/*H0-H7*/	"","","","","","","","",
+	/*I0-I7*/	"","","","","","","","",
+	/*J0-J7*/	"","","","","","","","",
+	/*K0-K7*/	"","","","","","","","",
+	/*L0-L7*/	"","","","","","","","",
+	/*M0-M7*/	"","","","","","","","",
+	/*N0-N7*/	"","","","","","","","",
+	/*O0-O7*/	"","","","","","","","",
+	/*P0-P7*/	"MON_PWR_BTN_L","ASSERT_PWR_BTN_L","MON_RST_BTN_L",
+			"ASSERT_RST_BTN_L","","ASSERT_NMI_BTN_L","","",
+	/*Q0-Q7*/	"","","MEMORY_HOT_0","MEMORY_HOT_1","","","","",
+	/*R0-R7*/	"ID_BTN","","","","","VBAT_GPIO","","",
+	/*S0-S7*/	"","","RST_PCA_MUX","","","","","",
+	/*T0-T7*/	"","","","","","","","",
+	/*U0-U7*/	"","","","","","","","",
+	/*V0-V7*/	"","","","","","","","",
+	/*W0-W7*/	"","","","","","","","",
+	/*X0-X7*/	"","","","","","","","",
+	/*Y0-Y7*/	"","","","","","","","",
+	/*Z0-Z7*/	"","","","","","","","";
+};
+
+&gpio1 {
+	gpio-line-names =
+	/*18A0-18A7*/ "","","","","","","","",
+	/*18B0-18B7*/ "","","","","","","","",
+	/*18C0-18C7*/ "","","","","","","","",
+	/*18D0-18D7*/ "","","","","","","","",
+	/*18E0-18E3*/ "","","BMC_INIT_DONE","";
+};
+
+// Devices on these busses are available after POST
+// however there isn't a great way to defer probing
+// until that point today, as the BMC doesn't
+// have direct control over when the host completes
+// POST, especially from the kernel.
+&i2c0 {
+	status = "okay";
+};
+
+&i2c1 {
+	status = "okay";
+};
+
+&i2c2 {
+	status = "okay";
+};
+
+&i2c3 {
+	status = "okay";
+};
+
+&i2c4 {
+	status = "okay";
+
+	eeprom@51 {
+		compatible = "atmel,24c256";
+		reg = <0x51>;
+		pagesize = <64>;
+		label = "sku";
+	};
+};
+
+&i2c5 {
+	status = "okay";
+};
+
+&i2c6 {
+	status = "okay";
+};
+
+&i2c7 {
+	status = "okay";
+};
+
+&i2c8 {
+	status = "okay";
+};
+
+&i2c9 {
+	status = "okay";
+};
+
+&i2c10 {
+	status = "okay";
+};
+
+&i2c11 {
+	status = "okay";
+};
+
+&i2c12 {
+	status = "okay";
+};
+
+&i2c13 {
+	status = "okay";
+};
+
+&i2c15 {
+	status = "okay";
+};
+
+&kcs1 {
+	aspeed,lpc-io-reg = <0xca0>;
+	status = "okay";
+};
+
+&kcs2 {
+	aspeed,lpc-io-reg = <0xca8>;
+	status = "okay";
+};
+
+&kcs3 {
+	aspeed,lpc-io-reg = <0xca2>;
+	status = "okay";
+};
+
+&lpc_reset {
+	status = "okay";
+};
+
+&rtc {
+	status = "okay";
+};
+
+&sgpiom0 {
+	ngpios = <80>;
+	status = "okay";
+};
+
+&uart_routing {
+	status = "okay";
+};
+
+&uart1 {
+	status = "okay";
+};
+
+&uart2 {
+	status = "okay";
+};
+
+&uart3 {
+	status = "okay";
+};
+
+&uart4 {
+	status = "okay";
+};
+
+&uart5 {
+	status = "okay";
+};
+
+&video {
+	memory-region = <&video_engine_memory>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/aspeed/aspeed-g6.dtsi b/arch/arm/boot/dts/aspeed/aspeed-g6.dtsi
index f8662c8ac089..189bc3bbb47c 100644
--- a/arch/arm/boot/dts/aspeed/aspeed-g6.dtsi
+++ b/arch/arm/boot/dts/aspeed/aspeed-g6.dtsi
@@ -68,13 +68,12 @@ timer {
 			     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
 			     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
 			     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>;
-		clocks = <&syscon ASPEED_CLK_HPLL>;
 		arm,cpu-registers-not-fw-configured;
 		always-on;
 	};
 
 	edac: sdram@1e6e0000 {
-		compatible = "aspeed,ast2600-sdram-edac", "syscon";
+		compatible = "aspeed,ast2600-sdram-edac";
 		reg = <0x1e6e0000 0x174>;
 		interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>;
 	};
@@ -866,15 +865,6 @@ fsim1: fsi@1e79b100 {
 				interrupt-controller;
 				status = "disabled";
 			};
-
-			udma: dma-controller@1e79e000 {
-				compatible = "aspeed,ast2600-udma";
-				reg = <0x1e79e000 0x1000>;
-				interrupts = <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>;
-				dma-channels = <28>;
-				#dma-cells = <1>;
-				status = "disabled";
-			};
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr-ac.dts b/arch/arm/boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr-ac.dts
index 413b9255f9e3..19a8d7b07758 100644
--- a/arch/arm/boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr-ac.dts
+++ b/arch/arm/boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr-ac.dts
@@ -12,6 +12,17 @@ / {
 	model = "Actiontec MI424WR rev A/C";
 	compatible = "actiontec,mi424wr-ac", "intel,ixp42x";
 
+	/* Connect the switch to EthC */
+	spi {
+		ethernet-switch@0 {
+			ethernet-ports {
+				ethernet-port@4 {
+					ethernet = <&ethc>;
+				};
+			};
+		};
+	};
+
 	soc {
 		/* EthB used for WAN */
 		ethernet@c8009000 {
diff --git a/arch/arm/boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr-d.dts b/arch/arm/boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr-d.dts
index 3619c6411a5c..244c6ea0973f 100644
--- a/arch/arm/boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr-d.dts
+++ b/arch/arm/boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr-d.dts
@@ -12,6 +12,17 @@ / {
 	model = "Actiontec MI424WR rev D";
 	compatible = "actiontec,mi424wr-d", "intel,ixp42x";
 
+	/* Connect the switch to EthB */
+	spi {
+		ethernet-switch@0 {
+			ethernet-ports {
+				ethernet-port@4 {
+					ethernet = <&ethb>;
+				};
+			};
+		};
+	};
+
 	soc {
 		/* EthB used for LAN */
 		ethernet@c8009000 {
diff --git a/arch/arm/boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr.dtsi b/arch/arm/boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr.dtsi
index 76fd97c5beb6..9b54e3c01a34 100644
--- a/arch/arm/boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr.dtsi
+++ b/arch/arm/boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr.dtsi
@@ -152,7 +152,6 @@ ethernet-port@3 {
 				};
 				ethernet-port@4 {
 					reg = <4>;
-					ethernet = <&ethc>;
 					phy-mode = "mii";
 					fixed-link {
 						speed = <100>;
diff --git a/arch/arm/boot/dts/nuvoton/nuvoton-common-npcm7xx.dtsi b/arch/arm/boot/dts/nuvoton/nuvoton-common-npcm7xx.dtsi
index 98c35771534e..ab3c3c5713ae 100644
--- a/arch/arm/boot/dts/nuvoton/nuvoton-common-npcm7xx.dtsi
+++ b/arch/arm/boot/dts/nuvoton/nuvoton-common-npcm7xx.dtsi
@@ -154,7 +154,7 @@ sdmmc: mmc@f0842000 {
 			status = "disabled";
 			reg = <0xf0842000 0x200>;
 			interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
-			clocks =  <&clk NPCM7XX_CLK_AHB>;
+			clocks = <&clk NPCM7XX_CLK_AHB>;
 			clock-names = "clk_mmc";
 			pinctrl-names = "default";
 			pinctrl-0 = <&mmc8_pins
@@ -166,7 +166,7 @@ sdhci: mmc@f0840000 {
 			status = "disabled";
 			reg = <0xf0840000 0x200>;
 			interrupts = <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>;
-			clocks =  <&clk NPCM7XX_CLK_AHB>;
+			clocks = <&clk NPCM7XX_CLK_AHB>;
 			clock-names = "clk_sdhc";
 			pinctrl-names = "default";
 			pinctrl-0 = <&sd1_pins>;
diff --git a/arch/arm/boot/dts/qcom/qcom-apq8074-dragonboard.dts b/arch/arm/boot/dts/qcom/qcom-apq8074-dragonboard.dts
index 34b0cf35fdac..d3ae6c6a6f83 100644
--- a/arch/arm/boot/dts/qcom/qcom-apq8074-dragonboard.dts
+++ b/arch/arm/boot/dts/qcom/qcom-apq8074-dragonboard.dts
@@ -198,15 +198,12 @@ &pm8941_wled {
 };
 
 &remoteproc_adsp {
-	cx-supply = <&pm8841_s2>;
-
 	firmware-name = "qcom/apq8074/adsp.mbn";
 
 	status = "okay";
 };
 
 &remoteproc_mss {
-	cx-supply = <&pm8841_s2>;
 	mss-supply = <&pm8841_s3>;
 	mx-supply = <&pm8841_s1>;
 	pll-supply = <&pm8941_l12>;
@@ -225,20 +222,10 @@ pm8841_s1: s1 {
 			regulator-max-microvolt = <1050000>;
 		};
 
-		pm8841_s2: s2 {
-			regulator-min-microvolt = <500000>;
-			regulator-max-microvolt = <1050000>;
-		};
-
 		pm8841_s3: s3 {
 			regulator-min-microvolt = <500000>;
 			regulator-max-microvolt = <1050000>;
 		};
-
-		pm8841_s4: s4 {
-			regulator-min-microvolt = <500000>;
-			regulator-max-microvolt = <1050000>;
-		};
 	};
 
 	regulators-1 {
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8974-lge-nexus5-hammerhead.dts b/arch/arm/boot/dts/qcom/qcom-msm8974-lge-nexus5-hammerhead.dts
index b3127f0383cf..4a2958979e1c 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8974-lge-nexus5-hammerhead.dts
+++ b/arch/arm/boot/dts/qcom/qcom-msm8974-lge-nexus5-hammerhead.dts
@@ -369,12 +369,10 @@ led@5 {
 };
 
 &remoteproc_adsp {
-	cx-supply = <&pm8841_s2>;
 	status = "okay";
 };
 
 &remoteproc_mss {
-	cx-supply = <&pm8841_s2>;
 	mss-supply = <&pm8841_s3>;
 	mx-supply = <&pm8841_s1>;
 	pll-supply = <&pm8941_l12>;
@@ -390,20 +388,10 @@ pm8841_s1: s1 {
 			regulator-max-microvolt = <1050000>;
 		};
 
-		pm8841_s2: s2 {
-			regulator-min-microvolt = <500000>;
-			regulator-max-microvolt = <1050000>;
-		};
-
 		pm8841_s3: s3 {
 			regulator-min-microvolt = <1050000>;
 			regulator-max-microvolt = <1050000>;
 		};
-
-		pm8841_s4: s4 {
-			regulator-min-microvolt = <815000>;
-			regulator-max-microvolt = <900000>;
-		};
 	};
 
 	regulators-1 {
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8974-samsung-hlte.dts b/arch/arm/boot/dts/qcom/qcom-msm8974-samsung-hlte.dts
index b7a1367d3470..7f61f80761e1 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8974-samsung-hlte.dts
+++ b/arch/arm/boot/dts/qcom/qcom-msm8974-samsung-hlte.dts
@@ -188,12 +188,10 @@ touchkey_pin: touchkey-int-state {
 };
 
 &remoteproc_adsp {
-	cx-supply = <&pm8841_s2>;
 	status = "okay";
 };
 
 &remoteproc_mss {
-	cx-supply = <&pm8841_s2>;
 	mss-supply = <&pm8841_s3>;
 	mx-supply = <&pm8841_s1>;
 	pll-supply = <&pm8941_l12>;
@@ -209,20 +207,10 @@ pm8841_s1: s1 {
 			regulator-max-microvolt = <1050000>;
 		};
 
-		pm8841_s2: s2 {
-			regulator-min-microvolt = <500000>;
-			regulator-max-microvolt = <1050000>;
-		};
-
 		pm8841_s3: s3 {
 			regulator-min-microvolt = <1050000>;
 			regulator-max-microvolt = <1050000>;
 		};
-
-		pm8841_s4: s4 {
-			regulator-min-microvolt = <815000>;
-			regulator-max-microvolt = <900000>;
-		};
 	};
 
 	regulators-1 {
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8974-sony-xperia-rhine.dtsi b/arch/arm/boot/dts/qcom/qcom-msm8974-sony-xperia-rhine.dtsi
index d7322fc6a095..96682d82b1c3 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8974-sony-xperia-rhine.dtsi
+++ b/arch/arm/boot/dts/qcom/qcom-msm8974-sony-xperia-rhine.dtsi
@@ -204,12 +204,10 @@ &pm8941_wled {
 };
 
 &remoteproc_adsp {
-	cx-supply = <&pm8841_s2>;
 	status = "okay";
 };
 
 &remoteproc_mss {
-	cx-supply = <&pm8841_s2>;
 	mss-supply = <&pm8841_s3>;
 	mx-supply = <&pm8841_s1>;
 	pll-supply = <&pm8941_l12>;
@@ -225,20 +223,10 @@ pm8841_s1: s1 {
 			regulator-max-microvolt = <1050000>;
 		};
 
-		pm8841_s2: s2 {
-			regulator-min-microvolt = <500000>;
-			regulator-max-microvolt = <1050000>;
-		};
-
 		pm8841_s3: s3 {
 			regulator-min-microvolt = <500000>;
 			regulator-max-microvolt = <1050000>;
 		};
-
-		pm8841_s4: s4 {
-			regulator-min-microvolt = <500000>;
-			regulator-max-microvolt = <1050000>;
-		};
 	};
 
 	regulators-1 {
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8974.dtsi b/arch/arm/boot/dts/qcom/qcom-msm8974.dtsi
index 7e119370f337..2a82ddce94a2 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8974.dtsi
+++ b/arch/arm/boot/dts/qcom/qcom-msm8974.dtsi
@@ -1,14 +1,15 @@
 // SPDX-License-Identifier: GPL-2.0
 /dts-v1/;
 
-#include <dt-bindings/interconnect/qcom,msm8974.h>
-#include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/clock/qcom,dsi-phy-28nm.h>
 #include <dt-bindings/clock/qcom,gcc-msm8974.h>
 #include <dt-bindings/clock/qcom,mmcc-msm8974.h>
 #include <dt-bindings/clock/qcom,rpmcc.h>
-#include <dt-bindings/reset/qcom,gcc-msm8974.h>
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interconnect/qcom,msm8974.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/power/qcom-rpmpd.h>
+#include <dt-bindings/reset/qcom,gcc-msm8974.h>
 
 / {
 	#address-cells = <1>;
@@ -146,6 +147,40 @@ rpmcc: clock-controller {
 					clocks = <&xo_board>;
 					clock-names = "xo";
 				};
+
+				rpmpd: power-controller {
+					compatible = "qcom,msm8974-rpmpd";
+					#power-domain-cells = <1>;
+					operating-points-v2 = <&rpmpd_opp_table>;
+
+					rpmpd_opp_table: opp-table {
+						compatible = "operating-points-v2";
+
+						rpmpd_opp_ret: opp1 {
+							opp-level = <1>;
+						};
+
+						rpmpd_opp_svs_krait: opp2 {
+							opp-level = <2>;
+						};
+
+						rpmpd_opp_svs_soc: opp3 {
+							opp-level = <3>;
+						};
+
+						rpmpd_opp_nom: opp4 {
+							opp-level = <4>;
+						};
+
+						rpmpd_opp_turbo: opp5 {
+							opp-level = <5>;
+						};
+
+						rpmpd_opp_super_turbo: opp6 {
+							opp-level = <6>;
+						};
+					};
+				};
 			};
 		};
 	};
@@ -743,6 +778,9 @@ pronto: remoteproc@fb204000 {
 					      <&wcnss_smp2p_in 3 IRQ_TYPE_EDGE_RISING>;
 			interrupt-names = "wdog", "fatal", "ready", "handover", "stop-ack";
 
+			power-domains = <&rpmpd MSM8974_VDDCX>;
+			power-domain-names = "cx";
+
 			qcom,smem-states = <&wcnss_smp2p_out 0>;
 			qcom,smem-state-names = "stop";
 
@@ -1545,6 +1583,9 @@ remoteproc_mss: remoteproc@fc880000 {
 			resets = <&gcc GCC_MSS_RESTART>;
 			reset-names = "mss_restart";
 
+			power-domains = <&rpmpd MSM8974_VDDCX>;
+			power-domain-names = "cx";
+
 			qcom,halt-regs = <&tcsr_mutex 0x1180 0x1200 0x1280>;
 
 			qcom,smem-states = <&modem_smp2p_out 0>;
@@ -2208,6 +2249,9 @@ remoteproc_adsp: remoteproc@fe200000 {
 			clocks = <&xo_board>;
 			clock-names = "xo";
 
+			power-domains = <&rpmpd MSM8974_VDDCX>;
+			power-domain-names = "cx";
+
 			memory-region = <&adsp_region>;
 
 			qcom,smem-states = <&adsp_smp2p_out 0>;
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8974pro-fairphone-fp2.dts b/arch/arm/boot/dts/qcom/qcom-msm8974pro-fairphone-fp2.dts
index fe227fd3f908..a081aeadd1d4 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8974pro-fairphone-fp2.dts
+++ b/arch/arm/boot/dts/qcom/qcom-msm8974pro-fairphone-fp2.dts
@@ -156,7 +156,6 @@ &pronto {
 	status = "okay";
 
 	vddmx-supply = <&pm8841_s1>;
-	vddcx-supply = <&pm8841_s2>;
 	vddpx-supply = <&pm8941_s3>;
 
 	pinctrl-names = "default";
@@ -181,12 +180,10 @@ wcnss {
 
 &remoteproc_adsp {
 	status = "okay";
-	cx-supply = <&pm8841_s2>;
 };
 
 &remoteproc_mss {
 	status = "okay";
-	cx-supply = <&pm8841_s2>;
 	mss-supply = <&pm8841_s3>;
 	mx-supply = <&pm8841_s1>;
 	pll-supply = <&pm8941_l12>;
@@ -201,11 +198,6 @@ pm8841_s1: s1 {
 			regulator-max-microvolt = <1050000>;
 		};
 
-		pm8841_s2: s2 {
-			regulator-min-microvolt = <500000>;
-			regulator-max-microvolt = <1050000>;
-		};
-
 		pm8841_s3: s3 {
 			regulator-min-microvolt = <1050000>;
 			regulator-max-microvolt = <1050000>;
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8974pro-htc-m8.dts b/arch/arm/boot/dts/qcom/qcom-msm8974pro-htc-m8.dts
index b896cc1ad6f7..402372834c53 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8974pro-htc-m8.dts
+++ b/arch/arm/boot/dts/qcom/qcom-msm8974pro-htc-m8.dts
@@ -70,7 +70,6 @@ &pm8941_vib {
 
 &pronto {
 	vddmx-supply = <&pm8841_s1>;
-	vddcx-supply = <&pm8841_s2>;
 	vddpx-supply = <&pm8941_s3>;
 
 	pinctrl-0 = <&wcnss_pin_a>;
@@ -104,20 +103,10 @@ pm8841_s1: s1 {
 			regulator-max-microvolt = <1050000>;
 		};
 
-		pm8841_s2: s2 {
-			regulator-min-microvolt = <500000>;
-			regulator-max-microvolt = <1050000>;
-		};
-
 		pm8841_s3: s3 {
 			regulator-min-microvolt = <1050000>;
 			regulator-max-microvolt = <1050000>;
 		};
-
-		pm8841_s4: s4 {
-			regulator-min-microvolt = <815000>;
-			regulator-max-microvolt = <900000>;
-		};
 	};
 
 	regulators-1 {
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8974pro-oneplus-bacon.dts b/arch/arm/boot/dts/qcom/qcom-msm8974pro-oneplus-bacon.dts
index 88ff6535477b..258bbbecd927 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8974pro-oneplus-bacon.dts
+++ b/arch/arm/boot/dts/qcom/qcom-msm8974pro-oneplus-bacon.dts
@@ -214,7 +214,6 @@ &pm8941_vib {
 
 &pronto {
 	vddmx-supply = <&pm8841_s1>;
-	vddcx-supply = <&pm8841_s2>;
 	vddpx-supply = <&pm8941_s3>;
 
 	pinctrl-names = "default";
@@ -240,8 +239,6 @@ wcnss {
 };
 
 &remoteproc_adsp {
-	cx-supply = <&pm8841_s2>;
-
 	status = "okay";
 };
 
@@ -254,12 +251,6 @@ pm8841_s1: s1 {
 			regulator-max-microvolt = <1050000>;
 		};
 
-		pm8841_s2: s2 {
-			regulator-min-microvolt = <875000>;
-			regulator-max-microvolt = <1050000>;
-			regulator-always-on;
-		};
-
 		pm8841_s3: s3 {
 			regulator-min-microvolt = <1050000>;
 			regulator-max-microvolt = <1050000>;
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8974pro-samsung-klte-common.dtsi b/arch/arm/boot/dts/qcom/qcom-msm8974pro-samsung-klte-common.dtsi
index d3959741d2ea..56a1a25f3df3 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8974pro-samsung-klte-common.dtsi
+++ b/arch/arm/boot/dts/qcom/qcom-msm8974pro-samsung-klte-common.dtsi
@@ -453,12 +453,10 @@ ramoops@3e8e0000 {
 
 &remoteproc_adsp {
 	status = "okay";
-	cx-supply = <&pma8084_s2>;
 };
 
 &remoteproc_mss {
 	status = "okay";
-	cx-supply = <&pma8084_s2>;
 	mss-supply = <&pma8084_s6>;
 	mx-supply = <&pma8084_s1>;
 	pll-supply = <&pma8084_l12>;
@@ -474,11 +472,6 @@ pma8084_s1: s1 {
 			regulator-always-on;
 		};
 
-		pma8084_s2: s2 {
-			regulator-min-microvolt = <500000>;
-			regulator-max-microvolt = <1050000>;
-		};
-
 		pma8084_s3: s3 {
 			regulator-min-microvolt = <1300000>;
 			regulator-max-microvolt = <1300000>;
@@ -648,6 +641,10 @@ pma8084_l27: l27 {
 	};
 };
 
+&rpmpd {
+	compatible = "qcom,msm8974pro-pma8084-rpmpd";
+};
+
 &sdhc_1 {
 	status = "okay";
 
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8974pro-sony-xperia-shinano-common.dtsi b/arch/arm/boot/dts/qcom/qcom-msm8974pro-sony-xperia-shinano-common.dtsi
index 6af7c71c7158..3d2de30b495e 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8974pro-sony-xperia-shinano-common.dtsi
+++ b/arch/arm/boot/dts/qcom/qcom-msm8974pro-sony-xperia-shinano-common.dtsi
@@ -207,12 +207,10 @@ &pm8941_vib {
 };
 
 &remoteproc_adsp {
-	cx-supply = <&pm8841_s2>;
 	status = "okay";
 };
 
 &remoteproc_mss {
-	cx-supply = <&pm8841_s2>;
 	mss-supply = <&pm8841_s3>;
 	mx-supply = <&pm8841_s1>;
 	pll-supply = <&pm8941_l12>;
@@ -228,20 +226,10 @@ pm8841_s1: s1 {
 			regulator-max-microvolt = <1050000>;
 		};
 
-		pm8841_s2: s2 {
-			regulator-min-microvolt = <500000>;
-			regulator-max-microvolt = <1050000>;
-		};
-
 		pm8841_s3: s3 {
 			regulator-min-microvolt = <500000>;
 			regulator-max-microvolt = <1050000>;
 		};
-
-		pm8841_s4: s4 {
-			regulator-min-microvolt = <500000>;
-			regulator-max-microvolt = <1050000>;
-		};
 	};
 
 	regulators-1 {
diff --git a/arch/arm/boot/dts/renesas/r9a06g032.dtsi b/arch/arm/boot/dts/renesas/r9a06g032.dtsi
index 8debb77803bb..7c143bd5d7c9 100644
--- a/arch/arm/boot/dts/renesas/r9a06g032.dtsi
+++ b/arch/arm/boot/dts/renesas/r9a06g032.dtsi
@@ -453,6 +453,12 @@ switch: switch@44050000 {
 				 <&sysctrl R9A06G032_CLK_SWITCH>;
 			clock-names = "hclk", "clk";
 			power-domains = <&sysctrl>;
+			interrupts = <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "dlr", "switch", "prp", "hub", "ptrn";
 			status = "disabled";
 
 			ethernet-ports {
diff --git a/arch/arm/boot/dts/rockchip/rk3288.dtsi b/arch/arm/boot/dts/rockchip/rk3288.dtsi
index 7477fc5da3ec..4e5e7509de48 100644
--- a/arch/arm/boot/dts/rockchip/rk3288.dtsi
+++ b/arch/arm/boot/dts/rockchip/rk3288.dtsi
@@ -1288,6 +1288,21 @@ vpu_mmu: iommu@ff9a0800 {
 		power-domains = <&power RK3288_PD_VIDEO>;
 	};
 
+	hevc: video-codec@ff9c0000 {
+		compatible = "rockchip,rk3288-vdec";
+		reg = <0x0 0xff9c0000 0x0 0x440>;
+		interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&cru ACLK_HEVC>, <&cru HCLK_HEVC>,
+			 <&cru SCLK_HEVC_CABAC>, <&cru SCLK_HEVC_CORE>;
+		clock-names = "axi", "ahb", "cabac", "core";
+		assigned-clocks = <&cru ACLK_HEVC>, <&cru HCLK_HEVC>,
+				  <&cru SCLK_HEVC_CABAC>, <&cru SCLK_HEVC_CORE>;
+		assigned-clock-rates = <400000000>, <100000000>,
+				       <300000000>, <300000000>;
+		iommus = <&hevc_mmu>;
+		power-domains = <&power RK3288_PD_HEVC>;
+	};
+
 	hevc_mmu: iommu@ff9c0440 {
 		compatible = "rockchip,iommu";
 		reg = <0x0 0xff9c0440 0x0 0x40>, <0x0 0xff9c0480 0x0 0x40>;
@@ -1295,7 +1310,7 @@ hevc_mmu: iommu@ff9c0440 {
 		clocks = <&cru ACLK_HEVC>, <&cru HCLK_HEVC>;
 		clock-names = "aclk", "iface";
 		#iommu-cells = <0>;
-		status = "disabled";
+		power-domains = <&power RK3288_PD_HEVC>;
 	};
 
 	gpu: gpu@ffa30000 {
diff --git a/arch/arm/boot/dts/samsung/s3c6400.dtsi b/arch/arm/boot/dts/samsung/s3c6400.dtsi
deleted file mode 100644
index 7cc785a63866..000000000000
--- a/arch/arm/boot/dts/samsung/s3c6400.dtsi
+++ /dev/null
@@ -1,38 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Samsung's S3C6400 SoC device tree source
- *
- * Copyright (c) 2013 Tomasz Figa <tomasz.figa@gmail.com>
- *
- * Samsung's S3C6400 SoC device nodes are listed in this file. S3C6400
- * based board files can include this file and provide values for board specific
- * bindings.
- *
- * Note: This file does not include device nodes for all the controllers in
- * S3C6400 SoC. As device tree coverage for S3C6400 increases, additional
- * nodes can be added to this file.
- */
-
-#include "s3c64xx.dtsi"
-
-/ {
-	compatible = "samsung,s3c6400";
-};
-
-&vic0 {
-	valid-mask = <0xfffffe1f>;
-	valid-wakeup-mask = <0x00200004>;
-};
-
-&vic1 {
-	valid-mask = <0xffffffff>;
-	valid-wakeup-mask = <0x53020000>;
-};
-
-&soc {
-	clocks: clock-controller@7e00f000 {
-		compatible = "samsung,s3c6400-clock";
-		reg = <0x7e00f000 0x1000>;
-		#clock-cells = <1>;
-	};
-};
diff --git a/arch/arm/configs/aspeed_g5_defconfig b/arch/arm/configs/aspeed_g5_defconfig
index 2e6ea13c1e9b..ec558e57d081 100644
--- a/arch/arm/configs/aspeed_g5_defconfig
+++ b/arch/arm/configs/aspeed_g5_defconfig
@@ -306,7 +306,7 @@ CONFIG_SCHED_STACK_END_CHECK=y
 CONFIG_PANIC_ON_OOPS=y
 CONFIG_PANIC_TIMEOUT=-1
 CONFIG_SOFTLOCKUP_DETECTOR=y
-CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y
+CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=1
 CONFIG_BOOTPARAM_HUNG_TASK_PANIC=1
 CONFIG_WQ_WATCHDOG=y
 # CONFIG_SCHED_DEBUG is not set
diff --git a/arch/arm/configs/neponset_defconfig b/arch/arm/configs/neponset_defconfig
index 2227f86100ad..4d720001c12e 100644
--- a/arch/arm/configs/neponset_defconfig
+++ b/arch/arm/configs/neponset_defconfig
@@ -9,7 +9,7 @@ CONFIG_ASSABET_NEPONSET=y
 CONFIG_ZBOOT_ROM_TEXT=0x80000
 CONFIG_ZBOOT_ROM_BSS=0xc1000000
 CONFIG_ZBOOT_ROM=y
-CONFIG_CMDLINE="console=ttySA0,38400n8 cpufreq=221200 rw root=/dev/mtdblock2 mtdparts=sa1100:512K(boot),1M(kernel),2560K(initrd),4M(root) load_ramdisk=1 prompt_ramdisk=0 mem=32M noinitrd initrd=0xc0800000,3M"
+CONFIG_CMDLINE="console=ttySA0,38400n8 cpufreq=221200 rw root=/dev/mtdblock2 mtdparts=sa1100:512K(boot),1M(kernel),2560K(initrd),4M(root) mem=32M noinitrd initrd=0xc0800000,3M"
 CONFIG_FPE_NWFPE=y
 CONFIG_PM=y
 CONFIG_MODULES=y
diff --git a/arch/arm/configs/pxa3xx_defconfig b/arch/arm/configs/pxa3xx_defconfig
index 07d422f0ff34..fb272e3a2337 100644
--- a/arch/arm/configs/pxa3xx_defconfig
+++ b/arch/arm/configs/pxa3xx_defconfig
@@ -100,7 +100,7 @@ CONFIG_PRINTK_TIME=y
 CONFIG_DEBUG_KERNEL=y
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_DEBUG_SHIRQ=y
-CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y
+CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=1
 # CONFIG_SCHED_DEBUG is not set
 CONFIG_DEBUG_SPINLOCK=y
 CONFIG_DEBUG_SPINLOCK_SLEEP=y
diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig
index f30d743df264..3eb5071bea14 100644
--- a/arch/arm/crypto/Kconfig
+++ b/arch/arm/crypto/Kconfig
@@ -23,16 +23,6 @@ config CRYPTO_GHASH_ARM_CE
 	  that is part of the ARMv8 Crypto Extensions, or a slower variant that
 	  uses the vmull.p8 instruction that is part of the basic NEON ISA.
 
-config CRYPTO_NHPOLY1305_NEON
-	tristate "Hash functions: NHPoly1305 (NEON)"
-	depends on KERNEL_MODE_NEON
-	select CRYPTO_NHPOLY1305
-	help
-	  NHPoly1305 hash function (Adiantum)
-
-	  Architecture: arm using:
-	  - NEON (Advanced SIMD) extensions
-
 config CRYPTO_AES_ARM
 	tristate "Ciphers: AES"
 	select CRYPTO_ALGAPI
diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile
index 86dd43313dbf..d6683e9d4992 100644
--- a/arch/arm/crypto/Makefile
+++ b/arch/arm/crypto/Makefile
@@ -5,7 +5,6 @@
 
 obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o
 obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o
-obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o
 
 obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o
 obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o
@@ -14,4 +13,3 @@ aes-arm-y	:= aes-cipher-core.o aes-cipher-glue.o
 aes-arm-bs-y	:= aes-neonbs-core.o aes-neonbs-glue.o
 aes-arm-ce-y	:= aes-ce-core.o aes-ce-glue.o
 ghash-arm-ce-y	:= ghash-ce-core.o ghash-ce-glue.o
-nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o
diff --git a/arch/arm/crypto/nh-neon-core.S b/arch/arm/crypto/nh-neon-core.S
deleted file mode 100644
index 01620a0782ca..000000000000
--- a/arch/arm/crypto/nh-neon-core.S
+++ /dev/null
@@ -1,116 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * NH - ε-almost-universal hash function, NEON accelerated version
- *
- * Copyright 2018 Google LLC
- *
- * Author: Eric Biggers <ebiggers@google.com>
- */
-
-#include <linux/linkage.h>
-
-	.text
-	.fpu		neon
-
-	KEY		.req	r0
-	MESSAGE		.req	r1
-	MESSAGE_LEN	.req	r2
-	HASH		.req	r3
-
-	PASS0_SUMS	.req	q0
-	PASS0_SUM_A	.req	d0
-	PASS0_SUM_B	.req	d1
-	PASS1_SUMS	.req	q1
-	PASS1_SUM_A	.req	d2
-	PASS1_SUM_B	.req	d3
-	PASS2_SUMS	.req	q2
-	PASS2_SUM_A	.req	d4
-	PASS2_SUM_B	.req	d5
-	PASS3_SUMS	.req	q3
-	PASS3_SUM_A	.req	d6
-	PASS3_SUM_B	.req	d7
-	K0		.req	q4
-	K1		.req	q5
-	K2		.req	q6
-	K3		.req	q7
-	T0		.req	q8
-	T0_L		.req	d16
-	T0_H		.req	d17
-	T1		.req	q9
-	T1_L		.req	d18
-	T1_H		.req	d19
-	T2		.req	q10
-	T2_L		.req	d20
-	T2_H		.req	d21
-	T3		.req	q11
-	T3_L		.req	d22
-	T3_H		.req	d23
-
-.macro _nh_stride	k0, k1, k2, k3
-
-	// Load next message stride
-	vld1.8		{T3}, [MESSAGE]!
-
-	// Load next key stride
-	vld1.32		{\k3}, [KEY]!
-
-	// Add message words to key words
-	vadd.u32	T0, T3, \k0
-	vadd.u32	T1, T3, \k1
-	vadd.u32	T2, T3, \k2
-	vadd.u32	T3, T3, \k3
-
-	// Multiply 32x32 => 64 and accumulate
-	vmlal.u32	PASS0_SUMS, T0_L, T0_H
-	vmlal.u32	PASS1_SUMS, T1_L, T1_H
-	vmlal.u32	PASS2_SUMS, T2_L, T2_H
-	vmlal.u32	PASS3_SUMS, T3_L, T3_H
-.endm
-
-/*
- * void nh_neon(const u32 *key, const u8 *message, size_t message_len,
- *		__le64 hash[NH_NUM_PASSES])
- *
- * It's guaranteed that message_len % 16 == 0.
- */
-ENTRY(nh_neon)
-
-	vld1.32		{K0,K1}, [KEY]!
-	  vmov.u64	PASS0_SUMS, #0
-	  vmov.u64	PASS1_SUMS, #0
-	vld1.32		{K2}, [KEY]!
-	  vmov.u64	PASS2_SUMS, #0
-	  vmov.u64	PASS3_SUMS, #0
-
-	subs		MESSAGE_LEN, MESSAGE_LEN, #64
-	blt		.Lloop4_done
-.Lloop4:
-	_nh_stride	K0, K1, K2, K3
-	_nh_stride	K1, K2, K3, K0
-	_nh_stride	K2, K3, K0, K1
-	_nh_stride	K3, K0, K1, K2
-	subs		MESSAGE_LEN, MESSAGE_LEN, #64
-	bge		.Lloop4
-
-.Lloop4_done:
-	ands		MESSAGE_LEN, MESSAGE_LEN, #63
-	beq		.Ldone
-	_nh_stride	K0, K1, K2, K3
-
-	subs		MESSAGE_LEN, MESSAGE_LEN, #16
-	beq		.Ldone
-	_nh_stride	K1, K2, K3, K0
-
-	subs		MESSAGE_LEN, MESSAGE_LEN, #16
-	beq		.Ldone
-	_nh_stride	K2, K3, K0, K1
-
-.Ldone:
-	// Sum the accumulators for each pass, then store the sums to 'hash'
-	vadd.u64	T0_L, PASS0_SUM_A, PASS0_SUM_B
-	vadd.u64	T0_H, PASS1_SUM_A, PASS1_SUM_B
-	vadd.u64	T1_L, PASS2_SUM_A, PASS2_SUM_B
-	vadd.u64	T1_H, PASS3_SUM_A, PASS3_SUM_B
-	vst1.8		{T0-T1}, [HASH]
-	bx		lr
-ENDPROC(nh_neon)
diff --git a/arch/arm/crypto/nhpoly1305-neon-glue.c b/arch/arm/crypto/nhpoly1305-neon-glue.c
deleted file mode 100644
index 62cf7ccdde73..000000000000
--- a/arch/arm/crypto/nhpoly1305-neon-glue.c
+++ /dev/null
@@ -1,80 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
- * (NEON accelerated version)
- *
- * Copyright 2018 Google LLC
- */
-
-#include <asm/neon.h>
-#include <asm/simd.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/nhpoly1305.h>
-#include <linux/module.h>
-
-asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len,
-			__le64 hash[NH_NUM_PASSES]);
-
-static int nhpoly1305_neon_update(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen)
-{
-	if (srclen < 64 || !crypto_simd_usable())
-		return crypto_nhpoly1305_update(desc, src, srclen);
-
-	do {
-		unsigned int n = min_t(unsigned int, srclen, SZ_4K);
-
-		kernel_neon_begin();
-		crypto_nhpoly1305_update_helper(desc, src, n, nh_neon);
-		kernel_neon_end();
-		src += n;
-		srclen -= n;
-	} while (srclen);
-	return 0;
-}
-
-static int nhpoly1305_neon_digest(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen, u8 *out)
-{
-	return crypto_nhpoly1305_init(desc) ?:
-	       nhpoly1305_neon_update(desc, src, srclen) ?:
-	       crypto_nhpoly1305_final(desc, out);
-}
-
-static struct shash_alg nhpoly1305_alg = {
-	.base.cra_name		= "nhpoly1305",
-	.base.cra_driver_name	= "nhpoly1305-neon",
-	.base.cra_priority	= 200,
-	.base.cra_ctxsize	= sizeof(struct nhpoly1305_key),
-	.base.cra_module	= THIS_MODULE,
-	.digestsize		= POLY1305_DIGEST_SIZE,
-	.init			= crypto_nhpoly1305_init,
-	.update			= nhpoly1305_neon_update,
-	.final			= crypto_nhpoly1305_final,
-	.digest			= nhpoly1305_neon_digest,
-	.setkey			= crypto_nhpoly1305_setkey,
-	.descsize		= sizeof(struct nhpoly1305_state),
-};
-
-static int __init nhpoly1305_mod_init(void)
-{
-	if (!(elf_hwcap & HWCAP_NEON))
-		return -ENODEV;
-
-	return crypto_register_shash(&nhpoly1305_alg);
-}
-
-static void __exit nhpoly1305_mod_exit(void)
-{
-	crypto_unregister_shash(&nhpoly1305_alg);
-}
-
-module_init(nhpoly1305_mod_init);
-module_exit(nhpoly1305_mod_exit);
-
-MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (NEON-accelerated)");
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
-MODULE_ALIAS_CRYPTO("nhpoly1305");
-MODULE_ALIAS_CRYPTO("nhpoly1305-neon");
diff --git a/arch/arm/include/asm/syscall.h b/arch/arm/include/asm/syscall.h
index 18b102a30741..574bbcc55382 100644
--- a/arch/arm/include/asm/syscall.h
+++ b/arch/arm/include/asm/syscall.h
@@ -92,8 +92,6 @@ static inline void syscall_set_nr(struct task_struct *task,
 		(nr & __NR_SYSCALL_MASK);
 }
 
-#define SYSCALL_MAX_ARGS 7
-
 static inline void syscall_get_arguments(struct task_struct *task,
 					 struct pt_regs *regs,
 					 unsigned long *args)
diff --git a/arch/arm64/Kbuild b/arch/arm64/Kbuild
index 5bfbf7d79c99..9e9820af48c9 100644
--- a/arch/arm64/Kbuild
+++ b/arch/arm64/Kbuild
@@ -5,5 +5,7 @@ obj-$(CONFIG_XEN)	+= xen/
 obj-$(subst m,y,$(CONFIG_HYPERV))	+= hyperv/
 obj-$(CONFIG_CRYPTO)	+= crypto/
 
+subdir-y += boot/dts
+
 # for cleaning
 subdir- += boot
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 93173f0a09c7..3fb4603c0e16 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -35,6 +35,7 @@ config ARM64
 	select ARCH_HAS_KCOV
 	select ARCH_HAS_KERNEL_FPU_SUPPORT if KERNEL_MODE_NEON
 	select ARCH_HAS_KEEPINITRD
+	select ARCH_HAS_LAZY_MMU_MODE
 	select ARCH_HAS_MEMBARRIER_SYNC_CORE
 	select ARCH_HAS_MEM_ENCRYPT
 	select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS
diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi
index 3603b276a2ab..f715b19efd16 100644
--- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi
+++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi
@@ -44,6 +44,12 @@ smc_reboot: reboot {
 			nvmem-cell-names = "shutdown_flag", "boot_stage",
 				"boot_error_count", "panic_count";
 		};
+
+		rtc {
+			compatible = "apple,smc-rtc";
+			nvmem-cells = <&rtc_offset>;
+			nvmem-cell-names = "rtc_offset";
+		};
 	};
 
 	smc_mbox: mbox@290408000 {
diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi
index 2e7d2bf08ddc..8622ddea7b44 100644
--- a/arch/arm64/boot/dts/apple/t602x-die0.dtsi
+++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi
@@ -121,6 +121,12 @@ smc_reboot: reboot {
 			nvmem-cell-names = "shutdown_flag", "boot_stage",
 				"boot_error_count", "panic_count";
 		};
+
+		rtc {
+			compatible = "apple,smc-rtc";
+			nvmem-cells = <&rtc_offset>;
+			nvmem-cell-names = "rtc_offset";
+		};
 	};
 
 	pinctrl_smc: pinctrl@2a2820000 {
diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi
index 8b7b27887968..59f2678639cf 100644
--- a/arch/arm64/boot/dts/apple/t8103.dtsi
+++ b/arch/arm64/boot/dts/apple/t8103.dtsi
@@ -916,6 +916,12 @@ smc_reboot: reboot {
 				nvmem-cell-names = "shutdown_flag", "boot_stage",
 					"boot_error_count", "panic_count";
 			};
+
+			rtc {
+				compatible = "apple,smc-rtc";
+				nvmem-cells = <&rtc_offset>;
+				nvmem-cell-names = "rtc_offset";
+			};
 		};
 
 		smc_mbox: mbox@23e408000 {
diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi
index 3f79878b25af..6bc3f58b06f7 100644
--- a/arch/arm64/boot/dts/apple/t8112.dtsi
+++ b/arch/arm64/boot/dts/apple/t8112.dtsi
@@ -919,6 +919,12 @@ smc_reboot: reboot {
 				nvmem-cell-names = "shutdown_flag", "boot_stage",
 					"boot_error_count", "panic_count";
 			};
+
+			rtc {
+				compatible = "apple,smc-rtc";
+				nvmem-cells = <&rtc_offset>;
+				nvmem-cell-names = "rtc_offset";
+			};
 		};
 
 		smc_mbox: mbox@23e408000 {
diff --git a/arch/arm64/boot/dts/arm/morello-sdp.dts b/arch/arm64/boot/dts/arm/morello-sdp.dts
index cee49dee7571..42c85f450fa9 100644
--- a/arch/arm64/boot/dts/arm/morello-sdp.dts
+++ b/arch/arm64/boot/dts/arm/morello-sdp.dts
@@ -108,6 +108,13 @@ smmu_pcie: iommu@4f400000 {
 		dma-coherent;
 	};
 
+	pmu@50000000 {
+		compatible = "arm,cmn-600";
+		reg = <0x0 0x50000000 0x0 0x4000000>;
+		interrupts = <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
+		arm,root-node = <0x804000>;
+	};
+
 	pcie_ctlr: pcie@28c0000000 {
 		device_type = "pci";
 		compatible = "pci-host-ecam-generic";
diff --git a/arch/arm64/boot/dts/broadcom/bcm2712.dtsi b/arch/arm64/boot/dts/broadcom/bcm2712.dtsi
index 205b87f557d6..d57a9b1bff70 100644
--- a/arch/arm64/boot/dts/broadcom/bcm2712.dtsi
+++ b/arch/arm64/boot/dts/broadcom/bcm2712.dtsi
@@ -250,6 +250,15 @@ uart10: serial@7d001000 {
 			status = "disabled";
 		};
 
+		pm: watchdog@7d200000 {
+			compatible = "brcm,bcm2712-pm", "brcm,bcm2835-pm-wdt";
+			reg = <0x7d200000 0x604>;
+			reg-names = "pm";
+			#power-domain-cells = <1>;
+			#reset-cells = <1>;
+			system-power-controller;
+		};
+
 		pinctrl: pinctrl@7d504100 {
 			compatible = "brcm,bcm2712c0-pinctrl";
 			reg = <0x7d504100 0x30>;
@@ -283,6 +292,12 @@ pinctrl_aon: pinctrl@7d510700 {
 			reg = <0x7d510700 0x20>;
 		};
 
+		random: rng@7d208000 {
+			compatible = "brcm,bcm2711-rng200";
+			reg = <0x7d208000 0x28>;
+			status = "okay";
+		};
+
 		interrupt-controller@7d517000 {
 			compatible = "brcm,bcm7271-l2-intc";
 			reg = <0x7d517000 0x10>;
diff --git a/arch/arm64/boot/dts/exynos/google/gs101.dtsi b/arch/arm64/boot/dts/exynos/google/gs101.dtsi
index d06d1d05f364..9b38c2248016 100644
--- a/arch/arm64/boot/dts/exynos/google/gs101.dtsi
+++ b/arch/arm64/boot/dts/exynos/google/gs101.dtsi
@@ -630,6 +630,15 @@ watchdog_cl1: watchdog@10070000 {
 			status = "disabled";
 		};
 
+		trng: rng@10141400 {
+			compatible = "google,gs101-trng",
+				     "samsung,exynos850-trng";
+			reg = <0x10141400 0x100>;
+			clocks = <&cmu_misc CLK_GOUT_MISC_SSS_I_ACLK>,
+				 <&cmu_misc CLK_GOUT_MISC_SSS_I_PCLK>;
+			clock-names = "secss", "pclk";
+		};
+
 		gic: interrupt-controller@10400000 {
 			compatible = "arm,gic-v3";
 			#address-cells = <0>;
@@ -1705,7 +1714,7 @@ sysreg_apm: syscon@17420000 {
 		};
 
 		pmu_system_controller: system-controller@17460000 {
-			compatible = "google,gs101-pmu", "syscon";
+			compatible = "google,gs101-pmu";
 			reg = <0x17460000 0x10000>;
 			google,pmu-intr-gen-syscon = <&pmu_intr_gen>;
 
diff --git a/arch/arm64/boot/dts/nuvoton/nuvoton-npcm845.dtsi b/arch/arm64/boot/dts/nuvoton/nuvoton-npcm845.dtsi
index 383938dcd3ce..8239d9a9f0d2 100644
--- a/arch/arm64/boot/dts/nuvoton/nuvoton-npcm845.dtsi
+++ b/arch/arm64/boot/dts/nuvoton/nuvoton-npcm845.dtsi
@@ -64,8 +64,8 @@ arm-pmu {
 	};
 
 	psci {
-		compatible      = "arm,psci-1.0";
-		method          = "smc";
+		compatible = "arm,psci-1.0";
+		method = "smc";
 	};
 
 	timer {
diff --git a/arch/arm64/boot/dts/qcom/Makefile b/arch/arm64/boot/dts/qcom/Makefile
index 6f34d5ed331c..76cf0115a00a 100644
--- a/arch/arm64/boot/dts/qcom/Makefile
+++ b/arch/arm64/boot/dts/qcom/Makefile
@@ -30,6 +30,8 @@ dtb-$(CONFIG_ARCH_QCOM)	+= ipq9574-rdp433.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq9574-rdp449.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq9574-rdp453.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq9574-rdp454.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= kaanapali-mtp.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= kaanapali-qrd.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= lemans-evk.dtb
 
 lemans-evk-camera-csi1-imx577-dtbs	:= lemans-evk.dtb lemans-evk-camera-csi1-imx577.dtbo
@@ -346,6 +348,8 @@ x1e80100-hp-omnibook-x14-el2-dtbs	:= x1e80100-hp-omnibook-x14.dtb x1-el2.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= x1e80100-hp-omnibook-x14.dtb x1e80100-hp-omnibook-x14-el2.dtb
 x1e80100-lenovo-yoga-slim7x-el2-dtbs	:= x1e80100-lenovo-yoga-slim7x.dtb x1-el2.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= x1e80100-lenovo-yoga-slim7x.dtb x1e80100-lenovo-yoga-slim7x-el2.dtb
+x1e80100-medion-sprchrgd-14-s1-el2-dtbs	:= x1e80100-medion-sprchrgd-14-s1.dtb x1-el2.dtbo
+dtb-$(CONFIG_ARCH_QCOM)	+= x1e80100-medion-sprchrgd-14-s1.dtb x1e80100-medion-sprchrgd-14-s1-el2.dtb
 x1e80100-microsoft-romulus13-el2-dtbs	:= x1e80100-microsoft-romulus13.dtb x1-el2.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= x1e80100-microsoft-romulus13.dtb x1e80100-microsoft-romulus13-el2.dtb
 x1e80100-microsoft-romulus15-el2-dtbs	:= x1e80100-microsoft-romulus15.dtb x1-el2.dtbo
diff --git a/arch/arm64/boot/dts/qcom/glymur-ipcc.h b/arch/arm64/boot/dts/qcom/glymur-ipcc.h
new file mode 100644
index 000000000000..700cd7114909
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/glymur-ipcc.h
@@ -0,0 +1,68 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __DTS_GLYMUR_MAILBOX_IPCC_H
+#define __DTS_GLYMUR_MAILBOX_IPCC_H
+
+/* Glymur physical client IDs */
+#define IPCC_MPROC_AOP			0
+#define IPCC_MPROC_TZ			1
+#define IPCC_MPROC_MPSS			2
+#define IPCC_MPROC_LPASS		3
+#define IPCC_MPROC_SLPI			4
+#define IPCC_MPROC_SDC			5
+#define IPCC_MPROC_CDSP			6
+#define IPCC_MPROC_NPU			7
+#define IPCC_MPROC_APSS			8
+#define IPCC_MPROC_GPU			9
+#define IPCC_MPROC_ICP			11
+#define IPCC_MPROC_VPU			12
+#define IPCC_MPROC_PCIE0		13
+#define IPCC_MPROC_PCIE1		14
+#define IPCC_MPROC_PCIE2		15
+#define IPCC_MPROC_SPSS			16
+#define IPCC_MPROC_PCIE3		19
+#define IPCC_MPROC_PCIE4		20
+#define IPCC_MPROC_PCIE5		21
+#define IPCC_MPROC_PCIE6		22
+#define IPCC_MPROC_TME			23
+#define IPCC_MPROC_WPSS			24
+#define IPCC_MPROC_PCIE7		44
+#define IPCC_MPROC_SOCCP		46
+
+#define IPCC_COMPUTE_L0_LPASS		0
+#define IPCC_COMPUTE_L0_CDSP		1
+#define IPCC_COMPUTE_L0_APSS		2
+#define IPCC_COMPUTE_L0_GPU		3
+#define IPCC_COMPUTE_L0_CVP		6
+#define IPCC_COMPUTE_L0_ICP		7
+#define IPCC_COMPUTE_L0_VPU		8
+#define IPCC_COMPUTE_L0_DPU		9
+#define IPCC_COMPUTE_L0_SOCCP		11
+
+#define IPCC_COMPUTE_L1_LPASS		0
+#define IPCC_COMPUTE_L1_CDSP		1
+#define IPCC_COMPUTE_L1_APSS		2
+#define IPCC_COMPUTE_L1_GPU		3
+#define IPCC_COMPUTE_L1_CVP		6
+#define IPCC_COMPUTE_L1_ICP		7
+#define IPCC_COMPUTE_L1_VPU		8
+#define IPCC_COMPUTE_L1_DPU		9
+#define IPCC_COMPUTE_L1_SOCCP		11
+
+#define IPCC_PERIPH_LPASS		0
+#define IPCC_PERIPH_APSS		1
+#define IPCC_PERIPH_PCIE0		2
+#define IPCC_PERIPH_PCIE1		3
+#define IPCC_PERIPH_PCIE2		6
+#define IPCC_PERIPH_PCIE3		7
+#define IPCC_PERIPH_PCIE4		8
+#define IPCC_PERIPH_PCIE5		9
+#define IPCC_PERIPH_PCIE6		10
+#define IPCC_PERIPH_PCIE7		11
+#define IPCC_PERIPH_SOCCP		13
+#define IPCC_PERIPH_WPSS		16
+
+#endif
diff --git a/arch/arm64/boot/dts/qcom/hamoa.dtsi b/arch/arm64/boot/dts/qcom/hamoa.dtsi
index a17900eacb20..bb7c14d473c9 100644
--- a/arch/arm64/boot/dts/qcom/hamoa.dtsi
+++ b/arch/arm64/boot/dts/qcom/hamoa.dtsi
@@ -3845,6 +3845,32 @@ pcie4_phy: phy@1c0e000 {
 			status = "disabled";
 		};
 
+		cryptobam: dma-controller@1dc4000 {
+			compatible = "qcom,bam-v1.7.4", "qcom,bam-v1.7.0";
+			reg = <0x0 0x01dc4000 0x0 0x28000>;
+			interrupts = <GIC_SPI 272 IRQ_TYPE_LEVEL_HIGH>;
+			#dma-cells = <1>;
+			iommus = <&apps_smmu 0x480 0x0>,
+				 <&apps_smmu 0x481 0x0>;
+			qcom,ee = <0>;
+			qcom,controlled-remotely;
+			num-channels = <20>;
+			qcom,num-ees = <4>;
+		};
+
+		crypto: crypto@1dfa000 {
+			compatible = "qcom,x1e80100-qce", "qcom,sm8150-qce", "qcom,qce";
+			reg = <0x0 0x01dfa000 0x0 0x6000>;
+			dmas = <&cryptobam 4>, <&cryptobam 5>;
+			dma-names = "rx",
+				    "tx";
+			iommus = <&apps_smmu 0x480 0x0>,
+				 <&apps_smmu 0x481 0x0>;
+			interconnects = <&aggre2_noc MASTER_CRYPTO QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "memory";
+		};
+
 		tcsr_mutex: hwlock@1f40000 {
 			compatible = "qcom,tcsr-mutex";
 			reg = <0 0x01f40000 0 0x20000>;
diff --git a/arch/arm64/boot/dts/qcom/kaanapali-ipcc.h b/arch/arm64/boot/dts/qcom/kaanapali-ipcc.h
new file mode 100644
index 000000000000..125375a4aac0
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/kaanapali-ipcc.h
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __DTS_KAANAPALI_MAILBOX_IPCC_H
+#define __DTS_KAANAPALI_MAILBOX_IPCC_H
+
+/* Physical client IDs */
+#define IPCC_MPROC_AOP			0
+#define IPCC_MPROC_TZ			1
+#define IPCC_MPROC_MPSS			2
+#define IPCC_MPROC_LPASS		3
+#define IPCC_MPROC_SDC			4
+#define IPCC_MPROC_CDSP			5
+#define IPCC_MPROC_APSS			6
+#define IPCC_MPROC_SOCCP		13
+#define IPCC_MPROC_DCP			14
+#define IPCC_MPROC_SPSS			15
+#define IPCC_MPROC_TME			16
+#define IPCC_MPROC_WPSS			17
+
+#define IPCC_COMPUTE_L0_CDSP		2
+#define IPCC_COMPUTE_L0_APSS		3
+#define IPCC_COMPUTE_L0_GPU		4
+#define IPCC_COMPUTE_L0_CVP		8
+#define IPCC_COMPUTE_L0_CAM		9
+#define IPCC_COMPUTE_L0_CAM1		10
+#define IPCC_COMPUTE_L0_DCP		11
+#define IPCC_COMPUTE_L0_VPU		12
+#define IPCC_COMPUTE_L0_SOCCP		16
+
+#define IPCC_COMPUTE_L1_CDSP		2
+#define IPCC_COMPUTE_L1_APSS		3
+#define IPCC_COMPUTE_L1_GPU		4
+#define IPCC_COMPUTE_L1_CVP		8
+#define IPCC_COMPUTE_L1_CAM		9
+#define IPCC_COMPUTE_L1_CAM1		10
+#define IPCC_COMPUTE_L1_DCP		11
+#define IPCC_COMPUTE_L1_VPU		12
+#define IPCC_COMPUTE_L1_SOCCP		16
+
+#define IPCC_PERIPH_CDSP		2
+#define IPCC_PERIPH_APSS		3
+#define IPCC_PERIPH_PCIE0		4
+#define IPCC_PERIPH_PCIE1		5
+
+#define IPCC_FENCE_CDSP			2
+#define IPCC_FENCE_APSS			3
+#define IPCC_FENCE_GPU			4
+#define IPCC_FENCE_CVP			8
+#define IPCC_FENCE_CAM			8
+#define IPCC_FENCE_CAM1			10
+#define IPCC_FENCE_DCP			11
+#define IPCC_FENCE_VPU			20
+#define IPCC_FENCE_SOCCP		24
+
+#endif
diff --git a/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts b/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts
new file mode 100644
index 000000000000..32a082598434
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts
@@ -0,0 +1,754 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+#include "kaanapali.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. Kaanapali MTP";
+	compatible = "qcom,kaanapali-mtp", "qcom,kaanapali";
+	chassis-type = "handset";
+
+	aliases {
+		serial0 = &uart7;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	clocks {
+		xo_board: xo-board {
+			compatible = "fixed-clock";
+			clock-frequency = <76800000>;
+			#clock-cells = <0>;
+		};
+
+		sleep_clk: sleep-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <32764>;
+			#clock-cells = <0>;
+		};
+
+		bi_tcxo_div2: bi-tcxo-div2-clk {
+			compatible = "fixed-factor-clock";
+			#clock-cells = <0>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+			clock-mult = <1>;
+			clock-div = <2>;
+		};
+
+		bi_tcxo_ao_div2: bi-tcxo-ao-div2-clk {
+			compatible = "fixed-factor-clock";
+			#clock-cells = <0>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK_A>;
+			clock-mult = <1>;
+			clock-div = <2>;
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pmh0101-rpmh-regulators";
+		qcom,pmic-id = "B_E0";
+
+		vreg_bob1: bob1 {
+			regulator-name = "vreg_bob1";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <4000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob2: bob2 {
+			regulator-name = "vreg_bob2";
+			regulator-min-microvolt = <2704000>;
+			regulator-max-microvolt = <3552000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1b_1p8: ldo1 {
+			regulator-name = "vreg_l1b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l2b_3p0: ldo2 {
+			regulator-name = "vreg_l2b_3p0";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3048000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l4b_1p8: ldo4 {
+			regulator-name = "vreg_l4b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l5b_3p1: ldo5 {
+			regulator-name = "vreg_l5b_3p1";
+			regulator-min-microvolt = <3100000>;
+			regulator-max-microvolt = <3148000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l6b_1p8: ldo6 {
+			regulator-name = "vreg_l6b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l7b_1p8: ldo7 {
+			regulator-name = "vreg_l7b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l8b_1p8: ldo8 {
+			regulator-name = "vreg_l8b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l9b_2p9: ldo9 {
+			regulator-name = "vreg_l9b_2p9";
+			regulator-min-microvolt = <2960000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10b_1p8: ldo10 {
+			regulator-name = "vreg_l10b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l11b_1p0: ldo11 {
+			regulator-name = "vreg_l11b_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1292000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l12b_1p8: ldo12 {
+			regulator-name = "vreg_l12b_1p8";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l13b_3p0: ldo13 {
+			regulator-name = "vreg_l13b_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l14b_3p2: ldo14 {
+			regulator-name = "vreg_l14b_3p2";
+			regulator-min-microvolt = <3200000>;
+			regulator-max-microvolt = <3200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l15b_1p8: ldo15 {
+			regulator-name = "vreg_l15b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l17b_2p5: ldo17 {
+			regulator-name = "vreg_l17b_2p5";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <2504000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18b_1p2: ldo18 {
+			regulator-name = "vreg_l18b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "D_E0";
+
+		vreg_s10d_1p0: smps10 {
+			regulator-name = "vreg_s10d_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1d_1p2: ldo1 {
+			regulator-name = "vreg_l1d_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1256000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l2d_0p9: ldo2 {
+			regulator-name = "vreg_l2d_0p9";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <958000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l3d_0p8: ldo3 {
+			regulator-name = "vreg_l3d_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l4d_1p2: ldo4 {
+			regulator-name = "vreg_l4d_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "F_E0";
+
+		vreg_s6f_0p5: smps6 {
+			regulator-name = "vreg_s6f_0p5";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <570000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s7f_1p2: smps7 {
+			regulator-name = "vreg_s7f_1p2";
+			regulator-min-microvolt = <1224000>;
+			regulator-max-microvolt = <1372000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s8f_1p8: smps8 {
+			regulator-name = "vreg_s8f_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1f_1p2: ldo1 {
+			regulator-name = "vreg_l1f_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l2f_1p2: ldo2 {
+			regulator-name = "vreg_l2f_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l3f_0p8: ldo3 {
+			regulator-name = "vreg_l3f_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <936000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l4f_0p8: ldo4 {
+			regulator-name = "vreg_l4f_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+
+	regulators-3 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "G_E0";
+
+		vreg_s7g_0p9: smps7 {
+			regulator-name = "vreg_s7g_0p9";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s9g_1p0: smps9 {
+			regulator-name = "vreg_s9g_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1g_1p2: ldo1 {
+			regulator-name = "vreg_l1g_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l2g_1p8: ldo2 {
+			regulator-name = "vreg_l2g_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l3g_1p2: ldo3 {
+			regulator-name = "vreg_l3g_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l4g_0p9: ldo4 {
+			regulator-name = "vreg_l4g_0p9";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+
+	regulators-4 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "I_E0";
+
+		vreg_s7i_0p9: smps7 {
+			regulator-name = "vreg_s7i_0p9";
+			regulator-min-microvolt = <900000>;
+			regulator-max-microvolt = <972000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2i_1p2: ldo2 {
+			regulator-name = "vreg_l2i_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l3i_0p8: ldo3 {
+			regulator-name = "vreg_l3i_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+
+	regulators-5 {
+		compatible = "qcom,pmh0104-rpmh-regulators";
+		qcom,pmic-id = "J_E1";
+
+		vreg_s1j_0p8: smps1 {
+			regulator-name = "vreg_s1j_0p8";
+			regulator-min-microvolt = <400000>;
+			regulator-max-microvolt = <1000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s2j_0p8: smps2 {
+			regulator-name = "vreg_s2j_0p8";
+			regulator-min-microvolt = <500000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s3j_1p2: smps3 {
+			regulator-name = "vreg_s3j_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4j_0p7: smps4 {
+			regulator-name = "vreg_s4j_0p7";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <1000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-6 {
+		compatible = "qcom,pmr735d-rpmh-regulators";
+		qcom,pmic-id = "K_E1";
+
+		vreg_l1k_0p8: ldo1 {
+			regulator-name = "vreg_l1k_0p8";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2k_0p7: ldo2 {
+			regulator-name = "vreg_l2k_0p7";
+			regulator-min-microvolt = <700000>;
+			regulator-max-microvolt = <1000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3k_1p2: ldo3 {
+			regulator-name = "vreg_l3k_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4k_1p0: ldo4 {
+			regulator-name = "vreg_l4k_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5k_0p7: ldo5 {
+			regulator-name = "vreg_l5k_0p7";
+			regulator-min-microvolt = <700000>;
+			regulator-max-microvolt = <1000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6k_1p7: ldo6 {
+			regulator-name = "vreg_l6k_1p7";
+			regulator-min-microvolt = <1700000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7k_0p7: ldo7 {
+			regulator-name = "vreg_l7k_0p7";
+			regulator-min-microvolt = <700000>;
+			regulator-max-microvolt = <848000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-7 {
+		compatible = "qcom,pm8010-rpmh-regulators";
+		qcom,pmic-id = "M_E1";
+
+		vreg_l1m_1p0: ldo1 {
+			regulator-name = "vreg_l1m_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2m_1p0: ldo2 {
+			regulator-name = "vreg_l2m_1p0";
+			regulator-min-microvolt = <1096000>;
+			regulator-max-microvolt = <1104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3m_2p8: ldo3 {
+			regulator-name = "vreg_l3m_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2900000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4m_2p2: ldo4 {
+			regulator-name = "vreg_l4m_2p2";
+			regulator-min-microvolt = <2200000>;
+			regulator-max-microvolt = <2200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6m_2p8: ldo6 {
+			regulator-name = "vreg_l6m_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7m_2p8: ldo7 {
+			regulator-name = "vreg_l7m_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-8 {
+		compatible = "qcom,pm8010-rpmh-regulators";
+		qcom,pmic-id = "N_E1";
+
+		vreg_l1n_1p1: ldo1 {
+			regulator-name = "vreg_l1n_1p1";
+			regulator-min-microvolt = <1096000>;
+			regulator-max-microvolt = <1104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2n_1p2: ldo2 {
+			regulator-name = "vreg_l2n_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3n_1p8: ldo3 {
+			regulator-name = "vreg_l3n_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4n_1p8: ldo4 {
+			regulator-name = "vreg_l4n_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5n_2p8: ldo5 {
+			regulator-name = "vreg_l5n_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6n_2p8: ldo6 {
+			regulator-name = "vreg_l6n_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7n_3p3: ldo7 {
+			regulator-name = "vreg_l7n_3p3";
+			regulator-min-microvolt = <3304000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&pcie0 {
+	pinctrl-0 = <&pcie0_default_state>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie0_phy {
+	vdda-phy-supply = <&vreg_l3i_0p8>;
+	vdda-pll-supply = <&vreg_l1d_1p2>;
+
+	status = "okay";
+};
+
+&pcie_port0 {
+	wake-gpios = <&tlmm 104 GPIO_ACTIVE_HIGH>;
+	reset-gpios = <&tlmm 102 GPIO_ACTIVE_LOW>;
+};
+
+&sdhc_2 {
+	cd-gpios = <&tlmm 55 GPIO_ACTIVE_LOW>;
+
+	vmmc-supply = <&vreg_l9b_2p9>;
+	vqmmc-supply = <&vreg_l8b_1p8>;
+
+	bus-width = <4>;
+	no-sdio;
+	no-mmc;
+
+	pinctrl-0 = <&sdc2_default>;
+	pinctrl-1 = <&sdc2_sleep>;
+	pinctrl-names = "default", "sleep";
+
+	status = "okay";
+};
+
+&tlmm {
+	gpio-reserved-ranges = <36 4>, /* NFC eSE SPI */
+			       <74 1>, /* eSE */
+			       <119 2>, /* SoCCP */
+			       <144 4>; /* CXM UART */
+
+	pcie0_default_state: pcie0-default-state {
+		perst-n-pins {
+			pins = "gpio102";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		clkreq-n-pins {
+			pins = "gpio103";
+			function = "pcie0_clk_req_n";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		wake-n-pins {
+			pins = "gpio104";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+};
+
+&uart7 {
+	status = "okay";
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 217 GPIO_ACTIVE_LOW>;
+
+	vcc-supply = <&vreg_l17b_2p5>;
+	vcc-max-microamp = <1200000>;
+	vccq-supply = <&vreg_l4d_1p2>;
+	vccq-max-microamp = <1200000>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vreg_l4g_0p9>;
+	vdda-pll-supply = <&vreg_l1d_1p2>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts b/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts
new file mode 100644
index 000000000000..66b423a497b3
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts
@@ -0,0 +1,712 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+#include "kaanapali.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. Kaanapali QRD";
+	compatible = "qcom,kaanapali-qrd", "qcom,kaanapali";
+	chassis-type = "handset";
+
+	aliases {
+		serial0 = &uart7;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	clocks {
+		xo_board: xo-board {
+			compatible = "fixed-clock";
+			clock-frequency = <76800000>;
+			#clock-cells = <0>;
+		};
+
+		sleep_clk: sleep-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <32764>;
+			#clock-cells = <0>;
+		};
+
+		bi_tcxo_div2: bi-tcxo-div2-clk {
+			compatible = "fixed-factor-clock";
+			#clock-cells = <0>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+			clock-mult = <1>;
+			clock-div = <2>;
+		};
+
+		bi_tcxo_ao_div2: bi-tcxo-ao-div2-clk {
+			compatible = "fixed-factor-clock";
+			#clock-cells = <0>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK_A>;
+			clock-mult = <1>;
+			clock-div = <2>;
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pmh0101-rpmh-regulators";
+		qcom,pmic-id = "B_E0";
+
+		vreg_bob1: bob1 {
+			regulator-name = "vreg_bob1";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <4000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob2: bob2 {
+			regulator-name = "vreg_bob2";
+			regulator-min-microvolt = <2704000>;
+			regulator-max-microvolt = <3552000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1b_1p8: ldo1 {
+			regulator-name = "vreg_l1b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l2b_3p0: ldo2 {
+			regulator-name = "vreg_l2b_3p0";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3048000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l4b_1p8: ldo4 {
+			regulator-name = "vreg_l4b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l5b_3p1: ldo5 {
+			regulator-name = "vreg_l5b_3p1";
+			regulator-min-microvolt = <3100000>;
+			regulator-max-microvolt = <3148000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l6b_1p8: ldo6 {
+			regulator-name = "vreg_l6b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l7b_1p8: ldo7 {
+			regulator-name = "vreg_l7b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l8b_1p8: ldo8 {
+			regulator-name = "vreg_l8b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l9b_2p9: ldo9 {
+			regulator-name = "vreg_l9b_2p9";
+			regulator-min-microvolt = <2960000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10b_1p8: ldo10 {
+			regulator-name = "vreg_l10b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l11b_1p0: ldo11 {
+			regulator-name = "vreg_l11b_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1292000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l12b_1p8: ldo12 {
+			regulator-name = "vreg_l12b_1p8";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l13b_3p0: ldo13 {
+			regulator-name = "vreg_l13b_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l14b_3p2: ldo14 {
+			regulator-name = "vreg_l14b_3p2";
+			regulator-min-microvolt = <3200000>;
+			regulator-max-microvolt = <3200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l15b_1p8: ldo15 {
+			regulator-name = "vreg_l15b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l17b_2p5: ldo17 {
+			regulator-name = "vreg_l17b_2p5";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <2504000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18b_1p2: ldo18 {
+			regulator-name = "vreg_l18b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "D_E0";
+
+		vreg_s10d_1p0: smps10 {
+			regulator-name = "vreg_s10d_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1d_1p2: ldo1 {
+			regulator-name = "vreg_l1d_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1256000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l2d_0p9: ldo2 {
+			regulator-name = "vreg_l2d_0p9";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <958000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l3d_0p8: ldo3 {
+			regulator-name = "vreg_l3d_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l4d_1p2: ldo4 {
+			regulator-name = "vreg_l4d_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "F_E0";
+
+		vreg_s6f_0p5: smps6 {
+			regulator-name = "vreg_s6f_0p5";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <570000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s7f_1p2: smps7 {
+			regulator-name = "vreg_s7f_1p2";
+			regulator-min-microvolt = <1224000>;
+			regulator-max-microvolt = <1372000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s8f_1p8: smps8 {
+			regulator-name = "vreg_s8f_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1f_1p2: ldo1 {
+			regulator-name = "vreg_l1f_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l2f_1p2: ldo2 {
+			regulator-name = "vreg_l2f_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l3f_0p8: ldo3 {
+			regulator-name = "vreg_l3f_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <936000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l4f_0p8: ldo4 {
+			regulator-name = "vreg_l4f_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+
+	regulators-3 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "G_E0";
+
+		vreg_s7g_0p9: smps7 {
+			regulator-name = "vreg_s7g_0p9";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s9g_1p0: smps9 {
+			regulator-name = "vreg_s9g_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1g_1p2: ldo1 {
+			regulator-name = "vreg_l1g_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l2g_1p8: ldo2 {
+			regulator-name = "vreg_l2g_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l3g_1p2: ldo3 {
+			regulator-name = "vreg_l3g_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l4g_0p9: ldo4 {
+			regulator-name = "vreg_l4g_0p9";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+
+	regulators-4 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "I_E0";
+
+		vreg_s7i_0p9: smps7 {
+			regulator-name = "vreg_s7i_0p9";
+			regulator-min-microvolt = <900000>;
+			regulator-max-microvolt = <972000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2i_1p2: ldo2 {
+			regulator-name = "vreg_l2i_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l3i_0p8: ldo3 {
+			regulator-name = "vreg_l3i_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+
+	regulators-5 {
+		compatible = "qcom,pmh0104-rpmh-regulators";
+		qcom,pmic-id = "J_E1";
+
+		vreg_s1j_0p8: smps1 {
+			regulator-name = "vreg_s1j_0p8";
+			regulator-min-microvolt = <400000>;
+			regulator-max-microvolt = <1000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s2j_0p8: smps2 {
+			regulator-name = "vreg_s2j_0p8";
+			regulator-min-microvolt = <500000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s3j_1p2: smps3 {
+			regulator-name = "vreg_s3j_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4j_0p7: smps4 {
+			regulator-name = "vreg_s4j_0p7";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <1000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-6 {
+		compatible = "qcom,pmr735d-rpmh-regulators";
+		qcom,pmic-id = "K_E1";
+
+		vreg_l1k_0p8: ldo1 {
+			regulator-name = "vreg_l1k_0p8";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2k_0p7: ldo2 {
+			regulator-name = "vreg_l2k_0p7";
+			regulator-min-microvolt = <700000>;
+			regulator-max-microvolt = <1000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3k_1p2: ldo3 {
+			regulator-name = "vreg_l3k_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4k_1p0: ldo4 {
+			regulator-name = "vreg_l4k_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5k_0p7: ldo5 {
+			regulator-name = "vreg_l5k_0p7";
+			regulator-min-microvolt = <700000>;
+			regulator-max-microvolt = <1000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6k_1p7: ldo6 {
+			regulator-name = "vreg_l6k_1p7";
+			regulator-min-microvolt = <1700000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7k_0p7: ldo7 {
+			regulator-name = "vreg_l7k_0p7";
+			regulator-min-microvolt = <700000>;
+			regulator-max-microvolt = <848000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-7 {
+		compatible = "qcom,pm8010-rpmh-regulators";
+		qcom,pmic-id = "M_E1";
+
+		vreg_l1m_1p0: ldo1 {
+			regulator-name = "vreg_l1m_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2m_1p0: ldo2 {
+			regulator-name = "vreg_l2m_1p0";
+			regulator-min-microvolt = <1096000>;
+			regulator-max-microvolt = <1104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3m_2p8: ldo3 {
+			regulator-name = "vreg_l3m_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2900000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4m_2p2: ldo4 {
+			regulator-name = "vreg_l4m_2p2";
+			regulator-min-microvolt = <2200000>;
+			regulator-max-microvolt = <2200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6m_2p8: ldo6 {
+			regulator-name = "vreg_l6m_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7m_2p8: ldo7 {
+			regulator-name = "vreg_l7m_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-8 {
+		compatible = "qcom,pm8010-rpmh-regulators";
+		qcom,pmic-id = "N_E1";
+
+		vreg_l1n_1p1: ldo1 {
+			regulator-name = "vreg_l1n_1p1";
+			regulator-min-microvolt = <1096000>;
+			regulator-max-microvolt = <1104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2n_1p2: ldo2 {
+			regulator-name = "vreg_l2n_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3n_1p8: ldo3 {
+			regulator-name = "vreg_l3n_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4n_1p8: ldo4 {
+			regulator-name = "vreg_l4n_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5n_2p8: ldo5 {
+			regulator-name = "vreg_l5n_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6n_2p8: ldo6 {
+			regulator-name = "vreg_l6n_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7n_3p3: ldo7 {
+			regulator-name = "vreg_l7n_3p3";
+			regulator-min-microvolt = <3304000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&sdhc_2 {
+	cd-gpios = <&tlmm 55 GPIO_ACTIVE_LOW>;
+
+	vmmc-supply = <&vreg_l9b_2p9>;
+	vqmmc-supply = <&vreg_l8b_1p8>;
+
+	bus-width = <4>;
+	no-sdio;
+	no-mmc;
+
+	pinctrl-0 = <&sdc2_default>;
+	pinctrl-1 = <&sdc2_sleep>;
+	pinctrl-names = "default", "sleep";
+
+	status = "okay";
+};
+
+&tlmm {
+	gpio-reserved-ranges = <36 4>, /* NFC eSE SPI */
+			       <74 1>, /* eSE */
+			       <119 2>, /* SoCCP */
+			       <144 4>; /* CXM UART */
+};
+
+&uart7 {
+	status = "okay";
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 217 GPIO_ACTIVE_LOW>;
+
+	vcc-supply = <&vreg_l17b_2p5>;
+	vcc-max-microamp = <1200000>;
+	vccq-supply = <&vreg_l4d_1p2>;
+	vccq-max-microamp = <1200000>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vreg_l4g_0p9>;
+	vdda-pll-supply = <&vreg_l1d_1p2>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/kaanapali.dtsi b/arch/arm64/boot/dts/qcom/kaanapali.dtsi
new file mode 100644
index 000000000000..51f8b3e0749c
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/kaanapali.dtsi
@@ -0,0 +1,1606 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/clock/qcom,kaanapali-gcc.h>
+#include <dt-bindings/clock/qcom,rpmh.h>
+#include <dt-bindings/clock/qcom,sm8750-tcsr.h>
+#include <dt-bindings/firmware/qcom,scm.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interconnect/qcom,icc.h>
+#include <dt-bindings/interconnect/qcom,kaanapali-rpmh.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/mailbox/qcom-ipcc.h>
+#include <dt-bindings/phy/phy-qcom-qmp.h>
+#include <dt-bindings/power/qcom-rpmpd.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+#include <dt-bindings/soc/qcom,rpmh-rsc.h>
+
+#include "kaanapali-ipcc.h"
+
+/ {
+	interrupt-parent = <&intc>;
+
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "qcom,oryon";
+			reg = <0x0 0x0>;
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+			power-domains = <&cpu_pd0>;
+			power-domain-names = "psci";
+			clocks = <&pdp_scmi_perf 0>;
+
+			l2_0: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+			};
+		};
+
+		cpu1: cpu@100 {
+			device_type = "cpu";
+			compatible = "qcom,oryon";
+			reg = <0x0 0x100>;
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+			power-domains = <&cpu_pd1>;
+			power-domain-names = "psci";
+			clocks = <&pdp_scmi_perf 0>;
+		};
+
+		cpu2: cpu@200 {
+			device_type = "cpu";
+			compatible = "qcom,oryon";
+			reg = <0x0 0x200>;
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+			power-domains = <&cpu_pd2>;
+			power-domain-names = "psci";
+			clocks = <&pdp_scmi_perf 0>;
+		};
+
+		cpu3: cpu@300 {
+			device_type = "cpu";
+			compatible = "qcom,oryon";
+			reg = <0x0 0x300>;
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+			power-domains = <&cpu_pd3>;
+			power-domain-names = "psci";
+			clocks = <&pdp_scmi_perf 0>;
+		};
+
+		cpu4: cpu@400 {
+			device_type = "cpu";
+			compatible = "qcom,oryon";
+			reg = <0x0 0x400>;
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+			power-domains = <&cpu_pd4>;
+			power-domain-names = "psci";
+			clocks = <&pdp_scmi_perf 0>;
+		};
+
+		cpu5: cpu@500 {
+			device_type = "cpu";
+			compatible = "qcom,oryon";
+			reg = <0x0 0x500>;
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+			power-domains = <&cpu_pd5>;
+			power-domain-names = "psci";
+			clocks = <&pdp_scmi_perf 0>;
+		};
+
+		cpu6: cpu@10000 {
+			device_type = "cpu";
+			compatible = "qcom,oryon";
+			reg = <0x0 0x10000>;
+			enable-method = "psci";
+			next-level-cache = <&l2_1>;
+			power-domains = <&cpu_pd6>;
+			power-domain-names = "psci";
+			clocks = <&pdp_scmi_perf 1>;
+
+			l2_1: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+			};
+		};
+
+		cpu7: cpu@10100 {
+			device_type = "cpu";
+			compatible = "qcom,oryon";
+			reg = <0x0 0x10100>;
+			enable-method = "psci";
+			next-level-cache = <&l2_1>;
+			power-domains = <&cpu_pd7>;
+			power-domain-names = "psci";
+			clocks = <&pdp_scmi_perf 1>;
+		};
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&cpu0>;
+				};
+
+				core1 {
+					cpu = <&cpu1>;
+				};
+
+				core2 {
+					cpu = <&cpu2>;
+				};
+
+				core3 {
+					cpu = <&cpu3>;
+				};
+
+				core4 {
+					cpu = <&cpu4>;
+				};
+
+				core5 {
+					cpu = <&cpu5>;
+				};
+			};
+
+			cluster1 {
+				core0 {
+					cpu = <&cpu6>;
+				};
+
+				core1 {
+					cpu = <&cpu7>;
+				};
+			};
+		};
+
+		idle-states {
+			entry-method = "psci";
+
+			cluster0_c4: cpu-sleep-0 {
+				compatible = "arm,idle-state";
+				idle-state-name = "retention";
+				arm,psci-suspend-param = <0x00000004>;
+				entry-latency-us = <93>;
+				exit-latency-us = <129>;
+				min-residency-us = <560>;
+			};
+
+			cluster1_c4: cpu-sleep-1 {
+				compatible = "arm,idle-state";
+				idle-state-name = "retention";
+				arm,psci-suspend-param = <0x00000004>;
+				entry-latency-us = <172>;
+				exit-latency-us = <130>;
+				min-residency-us = <686>;
+			};
+		};
+
+		domain-idle-states {
+			cluster_cl5: cluster-sleep-0 {
+				compatible = "domain-idle-state";
+				arm,psci-suspend-param = <0x01000054>;
+				entry-latency-us = <2150>;
+				exit-latency-us = <1983>;
+				min-residency-us = <9144>;
+			};
+
+			domain_ss3: domain-sleep-0 {
+				compatible = "domain-idle-state";
+				arm,psci-suspend-param = <0x0200c354>;
+				entry-latency-us = <2800>;
+				exit-latency-us = <4400>;
+				min-residency-us = <10150>;
+			};
+		};
+	};
+
+	firmware {
+		scm: scm {
+			compatible = "qcom,scm-kaanapali", "qcom,scm";
+			qcom,dload-mode = <&tcsr 0x19000>;
+			interconnects = <&aggre_noc MASTER_CRYPTO QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+		};
+
+		scmi: scmi {
+			compatible = "arm,scmi";
+			mboxes = <&pdp0_mbox 0>, <&pdp0_mbox 1>;
+			mbox-names = "tx", "rx";
+			shmem = <&pdp_tx>, <&pdp_rx>;
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			pdp_scmi_perf: protocol@13 {
+				reg = <0x13>;
+				#clock-cells = <1>;
+			};
+		};
+	};
+
+	clk_virt: interconnect-0 {
+		compatible = "qcom,kaanapali-clk-virt";
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	mc_virt: interconnect-1 {
+		compatible = "qcom,kaanapali-mc-virt";
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	memory@a0000000 {
+		device_type = "memory";
+		/* We expect the bootloader to fill in the size */
+		reg = <0x0 0xa0000000 0x0 0x0>;
+	};
+
+	pmu {
+		compatible = "arm,armv8-pmuv3";
+		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	psci {
+		compatible = "arm,psci-1.0";
+		method = "smc";
+
+		cpu_pd0: power-domain-cpu0 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster0_c4>;
+		};
+
+		cpu_pd1: power-domain-cpu1 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster0_c4>;
+		};
+
+		cpu_pd2: power-domain-cpu2 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster0_c4>;
+		};
+
+		cpu_pd3: power-domain-cpu3 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster0_c4>;
+		};
+
+		cpu_pd4: power-domain-cpu4 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster0_c4>;
+		};
+
+		cpu_pd5: power-domain-cpu5 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster0_c4>;
+		};
+
+		cpu_pd6: power-domain-cpu6 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster1_c4>;
+		};
+
+		cpu_pd7: power-domain-cpu7 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster1_c4>;
+		};
+
+		cluster_pd: power-domain-cluster {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&cluster_cl5>;
+			power-domains = <&system_pd>;
+		};
+
+		system_pd: power-domain-system {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&domain_ss3>;
+		};
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		pdp_mem: pdp@81300000 {
+			reg = <0x0 0x81300000 0x0 0x100000>;
+			no-map;
+		};
+
+		aop_cmd_db_mem: aop-cmd-db@81c60000 {
+			compatible = "qcom,cmd-db";
+			reg = <0x0 0x81c60000 0x0 0x20000>;
+			no-map;
+		};
+
+		smem_mem: smem@81d00000 {
+			compatible = "qcom,smem";
+			reg = <0x0 0x81d00000 0x0 0x200000>;
+			hwlocks = <&tcsr_mutex 3>;
+			no-map;
+		};
+
+		pdp_ns_shared_mem: pdp-ns-shared@81f00000 {
+			reg = <0x0 0x81f00000 0x0 0x100000>;
+			no-map;
+		};
+
+		dsm_partition_1_mem: dsm-partition-1@84a00000 {
+			reg = <0x0 0x84a00000 0x0 0x5500000>;
+			no-map;
+		};
+
+		dsm_partition_2_mem: dsm-partition-2@89f00000 {
+			reg = <0x0 0x89f00000 0x0 0xa80000>;
+			no-map;
+		};
+
+		mpss_mem: mpss@8aa00000 {
+			reg = <0x0 0x8aa00000 0x0 0xeb00000>;
+			no-map;
+		};
+
+		q6_mpss_dtb_mem: q6-mpss-dtb@99500000 {
+			reg = <0x0 0x99500000 0x0 0x80000>;
+			no-map;
+		};
+
+		ipa_fw_mem: ipa-fw@99580000 {
+			reg = <0x0 0x99580000 0x0 0x10000>;
+			no-map;
+		};
+
+		ipa_gsi_mem: ipa-gsi@99590000 {
+			reg = <0x0 0x99590000 0x0 0xa000>;
+			no-map;
+		};
+
+		gpu_microcode_mem: gpu-microcode@9959a000 {
+			reg = <0x0 0x9959a000 0x0 0x2000>;
+			no-map;
+		};
+
+		camera_mem: camera@99600000 {
+			reg = <0x0 0x99600000 0x0 0x800000>;
+			no-map;
+		};
+
+		camera_2_mem: camera-2@99e00000 {
+			reg = <0x0 0x99e00000 0x0 0x800000>;
+			no-map;
+		};
+
+		video_mem: video@9a600000 {
+			reg = <0x0 0x9a600000 0x0 0x800000>;
+			no-map;
+		};
+
+		cvp_mem: cvp@9ae00000 {
+			reg = <0x0 0x9ae00000 0x0 0x700000>;
+			no-map;
+		};
+
+		cdsp_mem: cdsp@9b500000 {
+			reg = <0x0 0x9b500000 0x0 0x1900000>;
+			no-map;
+		};
+
+		q6_cdsp_dtb_mem: q6-cdsp-dtb@9ce00000 {
+			reg = <0x0 0x9ce00000 0x0 0x80000>;
+			no-map;
+		};
+
+		soccp_mem: soccp@a03d0000 {
+			reg = <0x0 0xa03d0000 0x0 0x500000>;
+			no-map;
+		};
+
+		soccp_dtb_mem: soccp-dtb@a08d0000 {
+			reg = <0x0 0xa08d0000 0x0 0x40000>;
+			no-map;
+		};
+
+		q6_adsp_dtb_mem: q6-adsp-dtb@a1380000 {
+			reg = <0x0 0xa1380000 0x0 0x80000>;
+			no-map;
+		};
+
+		adspslpi_mem: adspslpi@a1400000 {
+			reg = <0x0 0xa1400000 0x0 0x4c00000>;
+			no-map;
+		};
+
+		rmtfs_mem: rmtfs@d7c00000 {
+			compatible = "qcom,rmtfs-mem";
+			reg = <0x0 0xd7c00000 0x0 0x400000>;
+			no-map;
+
+			qcom,client-id = <1>;
+			qcom,vmid = <QCOM_SCM_VMID_MSS_MSA>;
+		};
+	};
+
+	soc: soc@0 {
+		compatible = "simple-bus";
+
+		#address-cells = <2>;
+		#size-cells = <2>;
+		dma-ranges = <0x0 0x0 0x0 0x0 0x10 0x0>;
+		ranges = <0x0 0x0 0x0 0x0 0x10 0x0>;
+
+		gcc: clock-controller@100000 {
+			compatible = "qcom,kaanapali-gcc";
+			reg = <0x0 0x00100000 0x0 0x1f4200>;
+
+			clocks = <&bi_tcxo_div2>,
+				 <&bi_tcxo_ao_div2>,
+				 <&sleep_clk>,
+				 <&pcie0_phy>,
+				 <&ufs_mem_phy 0>,
+				 <&ufs_mem_phy 1>,
+				 <&ufs_mem_phy 2>,
+				 <0>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
+		qupv3_1: geniqup@ac0000 {
+			compatible = "qcom,geni-se-qup";
+			reg = <0x0 0x00ac0000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_QUPV3_WRAP_1_M_AXI_CLK>,
+				 <&gcc GCC_QUPV3_WRAP_1_S_AHB_CLK>;
+			clock-names = "m-ahb",
+				      "s-ahb";
+
+			iommus = <&apps_smmu 0xa3 0x0>;
+
+			dma-coherent;
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			uart7: serial@a9c000 {
+				compatible = "qcom,geni-debug-uart";
+				reg = <0x0 0x00a9c000 0x0 0x4000>;
+
+				interrupts = <GIC_SPI 835 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S7_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart7_default>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+		};
+
+		ipcc: mailbox@1106000 {
+			compatible = "qcom,kaanapali-ipcc", "qcom,ipcc";
+			reg = <0x0 0x01106000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 246 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-controller;
+			#interrupt-cells = <3>;
+
+			#mbox-cells = <2>;
+		};
+
+		cnoc_main: interconnect@1500000 {
+			compatible = "qcom,kaanapali-cnoc-main";
+			reg = <0x0 0x01500000 0x0 0x1a080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		config_noc: interconnect@1600000 {
+			compatible = "qcom,kaanapali-cnoc-cfg";
+			reg = <0x0 0x01600000 0x0 0x6200>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		system_noc: interconnect@1680000 {
+			compatible = "qcom,kaanapali-system-noc";
+			reg = <0x0 0x01680000 0x0 0x1f080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		pcie_noc: interconnect@16c0000 {
+			compatible = "qcom,kaanapali-pcie-anoc";
+			reg = <0x0 0x016c0000 0x0 0x11400>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+			clocks = <&gcc GCC_AGGRE_NOC_PCIE_AXI_CLK>,
+				 <&gcc GCC_CFG_NOC_PCIE_ANOC_AHB_CLK>;
+		};
+
+		aggre_noc: interconnect@16e0000 {
+			compatible = "qcom,kaanapali-aggre-noc";
+			reg = <0x0 0x016e0000 0x0 0x42400>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+			clocks = <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB3_PRIM_AXI_CLK>,
+				 <&rpmhcc RPMH_IPA_CLK>;
+		};
+
+		mmss_noc: interconnect@1780000 {
+			compatible = "qcom,kaanapali-mmss-noc";
+			reg = <0x0 0x01780000 0x0 0x5b800>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		pcie0: pcie@1c00000 {
+			device_type = "pci";
+			compatible = "qcom,kaanapali-pcie", "qcom,pcie-sm8550";
+			reg = <0x0 0x01c00000 0x0 0x3000>,
+			      <0x0 0x40000000 0x0 0xf1d>,
+			      <0x0 0x40000f20 0x0 0xa8>,
+			      <0x0 0x40001000 0x0 0x1000>,
+			      <0x0 0x40100000 0x0 0x100000>,
+			      <0x0 0x01c03000 0x0 0x1000>;
+			reg-names = "parf",
+				    "dbi",
+				    "elbi",
+				    "atu",
+				    "config",
+				    "mhi";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x40200000 0x0 0x100000>,
+				 <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x23d00000>;
+
+			interrupts = <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+
+			clocks = <&gcc GCC_PCIE_0_AUX_CLK>,
+				 <&gcc GCC_PCIE_0_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_0_MSTR_AXI_CLK>,
+				 <&gcc GCC_PCIE_0_SLV_AXI_CLK>,
+				 <&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>,
+				 <&gcc GCC_DDRSS_PCIE_SF_QTB_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_AXI_CLK>,
+				 <&gcc GCC_CNOC_PCIE_SF_AXI_CLK>;
+			clock-names = "aux",
+				      "cfg",
+				      "bus_master",
+				      "bus_slave",
+				      "slave_q2a",
+				      "ddrss_sf_tbu",
+				      "noc_aggr",
+				      "cnoc_sf_axi";
+
+			resets = <&gcc GCC_PCIE_0_BCR>,
+				 <&gcc GCC_PCIE_0_LINK_DOWN_BCR>;
+			reset-names = "pci",
+				      "link_down";
+
+			interconnects = <&pcie_noc MASTER_PCIE_0 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &cnoc_main SLAVE_PCIE_0 QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "pcie-mem",
+					     "cpu-pcie";
+
+			power-domains = <&gcc GCC_PCIE_0_GDSC>;
+
+			eq-presets-8gts = /bits/ 16 <0x5555 0x5555>;
+
+			operating-points-v2 = <&pcie0_opp_table>;
+
+			iommu-map = <0 &apps_smmu 0x1400 0x1>,
+				    <0x100 &apps_smmu 0x1401 0x1>;
+
+			interrupt-map = <0 0 0 1 &intc 0 0 0 149 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 0 0 150 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 0 0 151 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 0 0 152 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			#interrupt-cells = <1>;
+
+			msi-map = <0x0 &gic_its 0x1400 0x1>,
+				  <0x100 &gic_its 0x1401 0x1>;
+			msi-map-mask = <0xff00>;
+			max-link-speed = <3>;
+			linux,pci-domain = <0>;
+			num-lanes = <2>;
+			bus-range = <0x00 0xff>;
+
+			dma-coherent;
+
+			status = "disabled";
+
+			pcie0_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				/* GEN 1 x1 */
+				opp-2500000 {
+					opp-hz = /bits/ 64 <2500000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <250000 1>;
+				};
+
+				/* GEN 1 x2 and GEN 2 x1 */
+				opp-5000000 {
+					opp-hz = /bits/ 64 <5000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <500000 1>;
+				};
+
+				/* GEN 2 x2 */
+				opp-10000000 {
+					opp-hz = /bits/ 64 <10000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1000000 1>;
+				};
+
+				/* GEN 3 x1 */
+				opp-8000000 {
+					opp-hz = /bits/ 64 <8000000>;
+					required-opps = <&rpmhpd_opp_nom>;
+					opp-peak-kBps = <984500 1>;
+				};
+
+				/* GEN 3 x2 */
+				opp-16000000 {
+					opp-hz = /bits/ 64 <16000000>;
+					required-opps = <&rpmhpd_opp_nom>;
+					opp-peak-kBps = <1969000 1>;
+				};
+			};
+
+			pcie_port0: pcie@0 {
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+				phys = <&pcie0_phy>;
+			};
+		};
+
+		pcie0_phy: phy@1c06000 {
+			compatible = "qcom,kaanapali-qmp-gen3x2-pcie-phy";
+			reg = <0x0 0x01c06000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_PCIE_0_PHY_AUX_CLK>,
+				 <&gcc GCC_PCIE_0_CFG_AHB_CLK>,
+				 <&tcsr TCSR_PCIE_0_CLKREF_EN>,
+				 <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>,
+				 <&gcc GCC_PCIE_0_PIPE_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "ref",
+				      "rchng",
+				      "pipe";
+
+			assigned-clocks = <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>;
+			assigned-clock-rates = <100000000>;
+
+			resets = <&gcc GCC_PCIE_0_PHY_BCR>,
+				 <&gcc GCC_PCIE_0_NOCSR_COM_PHY_BCR>;
+			reset-names = "phy",
+				      "phy_nocsr";
+
+			power-domains = <&gcc GCC_PCIE_0_PHY_GDSC>;
+
+			#clock-cells = <0>;
+			clock-output-names = "pcie0_pipe_clk";
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		ufs_mem_phy: phy@1d80000 {
+			compatible = "qcom,kaanapali-qmp-ufs-phy", "qcom,sm8750-qmp-ufs-phy";
+			reg = <0x0 0x01d80000 0x0 0x2000>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>,
+				 <&gcc GCC_UFS_PHY_PHY_AUX_CLK>,
+				 <&tcsr TCSR_UFS_CLKREF_EN>;
+
+			clock-names = "ref",
+				      "ref_aux",
+				      "qref";
+
+			resets = <&ufs_mem_hc 0>;
+			reset-names = "ufsphy";
+
+			power-domains = <&gcc GCC_UFS_MEM_PHY_GDSC>;
+
+			#clock-cells = <1>;
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		ufs_mem_hc: ufs@1d84000 {
+			compatible = "qcom,kaanapali-ufshc",
+				     "qcom,ufshc",
+				     "jedec,ufs-2.0";
+			reg = <0x0 0x01d84000 0x0 0x3000>;
+
+			interrupts = <GIC_SPI 265 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&gcc GCC_UFS_PHY_AXI_CLK>,
+				 <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>,
+				 <&gcc GCC_UFS_PHY_AHB_CLK>,
+				 <&gcc GCC_UFS_PHY_UNIPRO_CORE_CLK>,
+				 <&rpmhcc RPMH_LN_BB_CLK3>,
+				 <&gcc GCC_UFS_PHY_TX_SYMBOL_0_CLK>,
+				 <&gcc GCC_UFS_PHY_RX_SYMBOL_0_CLK>,
+				 <&gcc GCC_UFS_PHY_RX_SYMBOL_1_CLK>;
+			clock-names = "core_clk",
+				      "bus_aggr_clk",
+				      "iface_clk",
+				      "core_clk_unipro",
+				      "ref_clk",
+				      "tx_lane0_sync_clk",
+				      "rx_lane0_sync_clk",
+				      "rx_lane1_sync_clk";
+
+			operating-points-v2 = <&ufs_opp_table>;
+
+			resets = <&gcc GCC_UFS_PHY_BCR>;
+			reset-names = "rst";
+
+			interconnects = <&aggre_noc MASTER_UFS_MEM QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_UFS_MEM_CFG QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "ufs-ddr",
+					     "cpu-ufs";
+
+			power-domains = <&gcc GCC_UFS_PHY_GDSC>;
+			required-opps = <&rpmhpd_opp_nom>;
+
+			iommus = <&apps_smmu 0x60 0x0>;
+			dma-coherent;
+
+			lanes-per-direction = <2>;
+			qcom,ice = <&ice>;
+
+			phys = <&ufs_mem_phy>;
+			phy-names = "ufsphy";
+
+			#reset-cells = <1>;
+
+			status = "disabled";
+
+			ufs_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-75000000 {
+					opp-hz = /bits/ 64 <75000000>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <75000000>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>;
+					required-opps = <&rpmhpd_opp_low_svs_d1>;
+				};
+
+				opp-100000000 {
+					opp-hz = /bits/ 64 <100000000>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <100000000>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+				};
+
+				opp-403000000 {
+					opp-hz = /bits/ 64 <403000000>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <403000000>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>;
+					required-opps = <&rpmhpd_opp_nom>;
+				};
+			};
+		};
+
+		ice: crypto@1d88000 {
+			compatible = "qcom,kaanapali-inline-crypto-engine",
+				     "qcom,inline-crypto-engine";
+			reg = <0x0 0x01d88000 0x0 0x18000>;
+
+			clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>;
+		};
+
+		tcsr_mutex: hwlock@1f40000 {
+			compatible = "qcom,tcsr-mutex";
+			reg = <0x0 0x01f40000 0x0 0x20000>;
+			#hwlock-cells = <1>;
+		};
+
+		tcsr: clock-controller@1fc0000 {
+			compatible = "qcom,kaanapali-tcsr", "syscon";
+			reg = <0x0 0x01fc0000 0x0 0x30000>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		lpass_lpiaon_noc: interconnect@7400000 {
+			compatible = "qcom,kaanapali-lpass-lpiaon-noc";
+			reg = <0x0 0x07400000 0x0 0x19080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		lpass_lpicx_noc: interconnect@7420000 {
+			compatible = "qcom,kaanapali-lpass-lpicx-noc";
+			reg = <0x0 0x07420000 0x0 0x44080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		lpass_ag_noc: interconnect@7f40000 {
+			compatible = "qcom,kaanapali-lpass-ag-noc";
+			reg = <0x0 0x07f40000 0x0 0xe080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		sdhc_2: mmc@8804000 {
+			compatible = "qcom,kaanapali-sdhci", "qcom,sdhci-msm-v5";
+			reg = <0x0 0x08804000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 223 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "hc_irq", "pwr_irq";
+
+			clocks = <&gcc GCC_SDCC2_AHB_CLK>,
+				 <&gcc GCC_SDCC2_APPS_CLK>,
+				 <&rpmhcc RPMH_CXO_CLK>;
+			clock-names = "iface", "core", "xo";
+
+			interconnects = <&aggre_noc MASTER_SDCC_2 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_SDCC_2 QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "sdhc-ddr", "cpu-sdhc";
+
+			power-domains = <&rpmhpd RPMHPD_CX>;
+			operating-points-v2 = <&sdhc2_opp_table>;
+
+			qcom,dll-config = <0x0007442c>;
+			qcom,ddr-config = <0x80040868>;
+
+			iommus = <&apps_smmu 0x540 0x0>;
+			dma-coherent;
+
+			resets = <&gcc GCC_SDCC2_BCR>;
+			status = "disabled";
+
+			sdhc2_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-100000000 {
+					opp-hz = /bits/ 64 <100000000>;
+					opp-peak-kBps = <160000 100000>;
+					opp-avg-kBps = <50000 0>;
+					required-opps = <&rpmhpd_opp_nom>;
+				};
+
+				opp-202000000 {
+					opp-hz = /bits/ 64 <202000000>;
+					opp-peak-kBps = <200000 120000>;
+					opp-avg-kBps = <104000 0>;
+					required-opps = <&rpmhpd_opp_nom>;
+				};
+			};
+		};
+
+		pdc: interrupt-controller@b220000 {
+			compatible = "qcom,kaanapali-pdc", "qcom,pdc";
+			reg = <0x0 0x0b220000 0x0 0x10000>,
+			      <0x0 0x179600f0 0x0 0xf4>;
+
+			qcom,pdc-ranges = <0 745 38>,
+					  <40 785 11>,
+					  <51 527 4>,
+					  <58 534 2>,
+					  <61 537 20>,
+					  <84 559 14>,
+					  <98 609 32>,
+					  <130 717 12>,
+					  <142 251 5>,
+					  <147 796 16>,
+					  <163 783 2>,
+					  <165 531 2>,
+					  <167 536 1>,
+					  <168 557 2>,
+					  <170 415 1>,
+					  <171 438 1>,
+					  <172 579 1>,
+					  <173 703 1>,
+					  <174 708 1>,
+					  <175 714 1>,
+					  <176 68 1>,
+					  <177 86 1>,
+					  <178 96 1>,
+					  <179 249 1>;
+			#interrupt-cells = <2>;
+			interrupt-parent = <&intc>;
+			interrupt-controller;
+		};
+
+		aoss_qmp: power-management@c300000 {
+			compatible = "qcom,kaanapali-aoss-qmp", "qcom,aoss-qmp";
+			reg = <0x0 0x0c300000 0x0 0x400>;
+
+			interrupts-extended = <&ipcc IPCC_MPROC_AOP
+						     IPCC_MPROC_SIGNAL_GLINK_QMP
+						     IRQ_TYPE_EDGE_RISING>;
+
+			mboxes = <&ipcc IPCC_MPROC_AOP
+					IPCC_MPROC_SIGNAL_GLINK_QMP>;
+
+			#clock-cells = <0>;
+		};
+
+		tlmm: pinctrl@f100000 {
+			compatible = "qcom,kaanapali-tlmm";
+			reg = <0x0 0x0f100000 0x0 0x300000>;
+			interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&tlmm 0 0 218>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			wakeup-parent = <&pdc>;
+
+			qup_uart7_default: qup-uart7-state {
+				 /* TX, RX */
+				 pins = "gpio62", "gpio63";
+				 function = "qup1_se7";
+				 drive-strength = <2>;
+				 bias-disable;
+			};
+
+			sdc2_default: sdc2-default-state {
+				clk-pins {
+					pins = "sdc2_clk";
+					drive-strength = <16>;
+					bias-disable;
+				};
+
+				cmd-pins {
+					pins = "sdc2_cmd";
+					drive-strength = <10>;
+					bias-pull-up;
+				};
+
+				data-pins {
+					pins = "sdc2_data";
+					drive-strength = <10>;
+					bias-pull-up;
+				};
+
+				card-detect-pins {
+					pins = "gpio55";
+					function = "gpio";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+			};
+
+			sdc2_sleep: sdc2-sleep-state {
+				clk-pins {
+					pins = "sdc2_clk";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				cmd-pins {
+					pins = "sdc2_cmd";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+
+				data-pins {
+					pins = "sdc2_data";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+
+				card-detect-pins {
+					pins = "gpio55";
+					function = "gpio";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+			};
+		};
+
+		sram@14680000 {
+			compatible = "qcom,kaanapali-imem", "mmio-sram";
+			reg = <0x0 0x14680000 0x0 0x1000>;
+			ranges = <0x0 0x0 0x14680000 0x1000>;
+
+			no-memory-wc;
+
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			pil-sram@94c {
+				compatible = "qcom,pil-reloc-info";
+				reg = <0x94c 0xc8>;
+			};
+		};
+
+		apps_smmu: iommu@15000000 {
+			compatible = "qcom,kaanapali-smmu-500", "qcom,smmu-500", "arm,mmu-500";
+			reg = <0x0 0x15000000 0x0 0x100000>;
+
+			interrupts =<GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 181 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 182 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 183 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 184 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 185 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 315 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 316 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 317 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 318 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 319 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 320 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 321 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 322 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 323 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 324 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 325 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 326 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 327 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 328 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 329 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 331 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 332 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 333 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 334 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 335 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 336 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 337 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 338 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 339 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 340 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 341 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 342 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 343 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 344 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 345 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 395 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 396 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 397 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 398 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 399 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 400 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 401 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 402 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 403 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 404 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 405 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 406 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 407 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 408 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 409 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 418 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 419 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 412 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 421 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 707 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 424 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 425 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 690 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 691 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 692 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 693 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 694 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 695 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 696 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 697 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 410 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 488 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 489 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 490 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 491 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 492 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 493 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 494 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 495 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 496 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 497 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 498 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 499 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 500 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 501 IRQ_TYPE_LEVEL_HIGH>,
+				    <GIC_SPI 502 IRQ_TYPE_LEVEL_HIGH>;
+
+			#iommu-cells = <2>;
+			#global-interrupts = <1>;
+
+			dma-coherent;
+		};
+
+		intc: interrupt-controller@17000000 {
+			compatible = "arm,gic-v3";
+			reg = <0x0 0x17000000 0x0 0x10000>,
+			      <0x0 0x17080000 0x0 0x200000>;
+
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+
+			#interrupt-cells = <3>;
+			interrupt-controller;
+
+			#redistributor-regions = <1>;
+			redistributor-stride = <0x0 0x40000>;
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			gic_its: msi-controller@17040000 {
+				compatible = "arm,gic-v3-its";
+				reg = <0x0 0x17040000 0x0 0x20000>;
+
+				msi-controller;
+				#msi-cells = <1>;
+			};
+		};
+
+		watchdog@17600000 {
+			compatible = "qcom,apss-wdt-kaanapali", "qcom,kpss-wdt";
+			reg = <0x0 0x17600000 0x0 0x1000>;
+			clocks = <&sleep_clk>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_EDGE_RISING>;
+		};
+
+		pdp0_mbox: mailbox@17610000 {
+			compatible = "qcom,kaanapali-cpucp-mbox", "qcom,x1e80100-cpucp-mbox";
+			reg = <0x0 0x17610000 0x0 0x8000>, <0x0 0x19980000 0x0 0x8000>;
+			interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <1>;
+		};
+
+		timer@17810000 {
+			compatible = "arm,armv7-timer-mem";
+			reg = <0x0 0x17810000 0x0 0x1000>;
+
+			#address-cells = <2>;
+			#size-cells = <1>;
+			ranges = <0x0 0x0 0x0 0x0 0x20000000>;
+
+			frame@17811000 {
+				reg = <0x0 0x17811000 0x1000>,
+				      <0x0 0x17812000 0x1000>;
+				frame-number = <0>;
+				interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>;
+			};
+
+			frame@17813000 {
+				reg = <0x0 0x17813000 0x1000>;
+				frame-number = <1>;
+				interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>;
+				status = "disabled";
+			};
+
+			frame@17815000 {
+				reg = <0x0 0x17815000 0x1000>;
+				frame-number = <2>;
+				interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>;
+				status = "disabled";
+			};
+
+			frame@17817000 {
+				reg = <0x0 0x17817000 0x1000>;
+				frame-number = <3>;
+				interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
+				status = "disabled";
+			};
+
+			frame@17819000 {
+				reg = <0x0 0x17819000 0x1000>;
+				frame-number = <4>;
+				interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
+				status = "disabled";
+			};
+
+			frame@1781b000 {
+				reg = <0x0 0x1781b000 0x1000>;
+				frame-number = <5>;
+				interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
+				status = "disabled";
+			};
+
+			frame@1781d000 {
+				reg = <0x0 0x1781d000 0x1000>;
+				frame-number = <6>;
+				interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
+				status = "disabled";
+			};
+		};
+
+		apps_rsc: rsc@18900000 {
+			compatible = "qcom,rpmh-rsc";
+			reg = <0x0 0x18900000 0x0 0x10000>,
+			      <0x0 0x18910000 0x0 0x10000>,
+			      <0x0 0x18920000 0x0 0x10000>;
+			reg-names = "drv-0",
+				    "drv-1",
+				    "drv-2";
+			interrupts = <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
+
+			power-domains = <&system_pd>;
+			label = "apps_rsc";
+
+			qcom,tcs-offset = <0xd00>;
+			qcom,drv-id = <2>;
+			qcom,tcs-config = <ACTIVE_TCS 3>,
+					  <SLEEP_TCS 2>,
+					  <WAKE_TCS 2>,
+					  <CONTROL_TCS 0>;
+
+			apps_bcm_voter: bcm-voter {
+				compatible = "qcom,bcm-voter";
+			};
+
+			rpmhcc: clock-controller {
+				compatible = "qcom,kaanapali-rpmh-clk";
+				#clock-cells = <1>;
+				clocks = <&xo_board>;
+				clock-names = "xo";
+			};
+
+			rpmhpd: power-controller {
+				compatible = "qcom,kaanapali-rpmhpd";
+
+				operating-points-v2 = <&rpmhpd_opp_table>;
+
+				#power-domain-cells = <1>;
+
+				rpmhpd_opp_table: opp-table {
+					compatible = "operating-points-v2";
+
+					rpmhpd_opp_ret: opp-16 {
+						opp-level = <RPMH_REGULATOR_LEVEL_RETENTION>;
+					};
+
+					rpmhpd_opp_low_svs_d3: opp-50 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D3>;
+					};
+
+					rpmhpd_opp_low_svs_d2_1: opp-51 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D2_1>;
+					};
+
+					rpmhpd_opp_low_svs_d2: opp-52 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D2>;
+					};
+
+					rpmhpd_opp_low_svs_d1_1: opp-54 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D1_1>;
+					};
+
+					rpmhpd_opp_low_svs_d1: opp-56 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D1>;
+					};
+
+					rpmhpd_opp_low_svs_d0: opp-60 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D0>;
+					};
+
+					rpmhpd_opp_low_svs: opp-64 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS>;
+					};
+
+					rpmhpd_opp_low_svs_l0: opp-76 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_L0>;
+					};
+
+					rpmhpd_opp_low_svs_l1: opp-80 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_L1>;
+					};
+
+					rpmhpd_opp_low_svs_l2: opp-96 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_L2>;
+					};
+
+					rpmhpd_opp_svs: opp-128 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SVS>;
+					};
+
+					rpmhpd_opp_svs_l0: opp-144 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SVS_L0>;
+					};
+
+					rpmhpd_opp_svs_l1: opp-192 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SVS_L1>;
+					};
+
+					rpmhpd_opp_svs_l2: opp-224 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SVS_L2>;
+					};
+
+					rpmhpd_opp_nom: opp-256 {
+						opp-level = <RPMH_REGULATOR_LEVEL_NOM>;
+					};
+
+					rpmhpd_opp_nom_l1: opp-320 {
+						opp-level = <RPMH_REGULATOR_LEVEL_NOM_L1>;
+					};
+
+					rpmhpd_opp_nom_l2: opp-336 {
+						opp-level = <RPMH_REGULATOR_LEVEL_NOM_L2>;
+					};
+
+					rpmhpd_opp_turbo: opp-384 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO>;
+					};
+
+					rpmhpd_opp_turbo_l0: opp-400 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L0>;
+					};
+
+					rpmhpd_opp_turbo_l1: opp-416 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L1>;
+					};
+
+					rpmhpd_opp_turbo_l2: opp-432 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L2>;
+					};
+
+					rpmhpd_opp_turbo_l3: opp-448 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L3>;
+					};
+
+					rpmhpd_opp_turbo_l4: opp-452 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L4>;
+					};
+
+					rpmhpd_opp_turbo_l5: opp-456 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L5>;
+					};
+
+					rpmhpd_opp_super_turbo_no_cpr: opp-480 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SUPER_TURBO_NO_CPR>;
+					};
+				};
+			};
+		};
+
+		nsp_noc: interconnect@260c0000 {
+			compatible = "qcom,kaanapali-nsp-noc";
+			reg = <0x0 0x260c0000 0x0 0x21280>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		/* Cluster 0 */
+		pmu@310b3400  {
+			compatible = "qcom,kaanapali-cpu-bwmon", "qcom,sdm845-bwmon";
+			reg = <0x0 0x310b3400 0x0 0x600>;
+
+			interrupts = <GIC_SPI 581 IRQ_TYPE_LEVEL_HIGH>;
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+
+			operating-points-v2 = <&cpu_bwmon_opp_table>;
+
+			cpu_bwmon_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-0 {
+					opp-peak-kBps = <2188000>;
+				};
+
+				opp-1 {
+					opp-peak-kBps = <5412000>;
+				};
+
+				opp-2 {
+					opp-peak-kBps = <6220000>;
+				};
+
+				opp-3 {
+					opp-peak-kBps = <6832000>;
+				};
+
+				opp-4 {
+					opp-peak-kBps = <8368000>;
+				};
+
+				opp-5 {
+					opp-peak-kBps = <10944000>;
+				};
+
+				opp-6 {
+					opp-peak-kBps = <12748000>;
+				};
+
+				opp-7 {
+					opp-peak-kBps = <14744000>;
+				};
+
+				opp-8 {
+					opp-peak-kBps = <16896000>;
+				};
+
+				opp-9 {
+					opp-peak-kBps = <19120000>;
+				};
+
+				opp-10 {
+					opp-peak-kBps = <21332000>;
+				};
+			};
+		};
+
+		/* Cluster 1 */
+		pmu@310b7400  {
+			compatible = "qcom,kaanapali-cpu-bwmon", "qcom,sdm845-bwmon";
+			reg = <0x0 0x310b7400 0x0 0x600>;
+
+			interrupts = <GIC_SPI 581 IRQ_TYPE_LEVEL_HIGH>;
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+
+			operating-points-v2 = <&cpu_bwmon_opp_table>;
+		};
+
+		gem_noc: interconnect@31100000 {
+			compatible = "qcom,kaanapali-gem-noc";
+			reg = <0x0 0x31100000 0x0 0x153080>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			#interconnect-cells = <2>;
+		};
+
+		system-cache-controller@31800000 {
+			compatible = "qcom,kaanapali-llcc";
+			reg = <0x0 0x31800000 0x0 0x200000>,
+			      <0x0 0x32800000 0x0 0x200000>,
+			      <0x0 0x31c00000 0x0 0x200000>,
+			      <0x0 0x32c00000 0x0 0x200000>,
+			      <0x0 0x34800000 0x0 0x200000>,
+			      <0x0 0x34c00000 0x0 0x200000>;
+			reg-names = "llcc0_base",
+				    "llcc1_base",
+				    "llcc2_base",
+				    "llcc3_base",
+				    "llcc_broadcast_base",
+				    "llcc_broadcast_and_base";
+
+			interrupts = <GIC_SPI 266 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		sram: sram@81f08000 {
+			compatible = "mmio-sram";
+			reg = <0x0 0x81f08000 0x0 0x200>;
+
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0x0 0x0 0x81f08000 0x200>;
+
+			pdp_rx: scp-sram-section@0 {
+				compatible = "arm,scmi-shmem";
+				reg = <0x0 0x80>;
+			};
+
+			pdp_tx: scp-sram-section@100 {
+				compatible = "arm,scmi-shmem";
+				reg = <0x100 0x80>;
+			};
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>;
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
index 5334adebf278..b9e0d9c7c065 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
@@ -5788,8 +5788,12 @@ remoteproc_nsp0: remoteproc@1b300000 {
 			clocks = <&rpmhcc RPMH_CXO_CLK>;
 			clock-names = "xo";
 
-			power-domains = <&rpmhpd SC8280XP_NSP>;
-			power-domain-names = "nsp";
+			power-domains = <&rpmhpd SC8280XP_NSP>,
+					<&rpmhpd SC8280XP_CX>,
+					<&rpmhpd SC8280XP_MXC>;
+			power-domain-names = "nsp",
+					     "cx",
+					     "mxc";
 
 			memory-region = <&pil_nsp0_mem>;
 
@@ -5919,8 +5923,12 @@ remoteproc_nsp1: remoteproc@21300000 {
 			clocks = <&rpmhcc RPMH_CXO_CLK>;
 			clock-names = "xo";
 
-			power-domains = <&rpmhpd SC8280XP_NSP>;
-			power-domain-names = "nsp";
+			power-domains = <&rpmhpd SC8280XP_NSP>,
+					<&rpmhpd SC8280XP_CX>,
+					<&rpmhpd SC8280XP_MXC>;
+			power-domain-names = "nsp",
+					     "cx",
+					     "mxc";
 
 			memory-region = <&pil_nsp1_mem>;
 
diff --git a/arch/arm64/boot/dts/qcom/sdm845-oneplus-enchilada.dts b/arch/arm64/boot/dts/qcom/sdm845-oneplus-enchilada.dts
index a259eb9d45ae..8aead6dc25e0 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-oneplus-enchilada.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-oneplus-enchilada.dts
@@ -31,9 +31,9 @@ battery: battery {
 };
 
 &display_panel {
-	status = "okay";
+	compatible = "samsung,sofef00-ams628nw01", "samsung,sofef00";
 
-	compatible = "samsung,sofef00";
+	status = "okay";
 };
 
 &bq27441_fg {
diff --git a/arch/arm64/boot/dts/qcom/sm8550.dtsi b/arch/arm64/boot/dts/qcom/sm8550.dtsi
index 2ca9e50ef599..e3f93f4f412d 100644
--- a/arch/arm64/boot/dts/qcom/sm8550.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8550.dtsi
@@ -4133,8 +4133,6 @@ usb_dp_qmpphy_dp_in: endpoint {
 		usb_1: usb@a600000 {
 			compatible = "qcom,sm8550-dwc3", "qcom,snps-dwc3";
 			reg = <0x0 0x0a600000 0x0 0xfc100>;
-			#address-cells = <1>;
-			#size-cells = <0>;
 
 			clocks = <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>,
 				 <&gcc GCC_USB30_PRIM_MASTER_CLK>,
diff --git a/arch/arm64/boot/dts/qcom/sm8650.dtsi b/arch/arm64/boot/dts/qcom/sm8650.dtsi
index 07ae74851621..f8e1950a74ac 100644
--- a/arch/arm64/boot/dts/qcom/sm8650.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8650.dtsi
@@ -5150,9 +5150,6 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
 
 			dma-coherent;
 
-			#address-cells = <1>;
-			#size-cells = <0>;
-
 			status = "disabled";
 
 			ports {
diff --git a/arch/arm64/boot/dts/qcom/talos.dtsi b/arch/arm64/boot/dts/qcom/talos.dtsi
index d1dbfa3bd81c..95d26e313622 100644
--- a/arch/arm64/boot/dts/qcom/talos.dtsi
+++ b/arch/arm64/boot/dts/qcom/talos.dtsi
@@ -1399,10 +1399,10 @@ ufs_mem_hc: ufshc@1d84000 {
 				 <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>,
 				 <&gcc GCC_UFS_PHY_AHB_CLK>,
 				 <&gcc GCC_UFS_PHY_UNIPRO_CORE_CLK>,
-				 <&gcc GCC_UFS_PHY_ICE_CORE_CLK>,
 				 <&rpmhcc RPMH_CXO_CLK>,
 				 <&gcc GCC_UFS_PHY_TX_SYMBOL_0_CLK>,
-				 <&gcc GCC_UFS_PHY_RX_SYMBOL_0_CLK>;
+				 <&gcc GCC_UFS_PHY_RX_SYMBOL_0_CLK>,
+				 <&gcc GCC_UFS_PHY_ICE_CORE_CLK>;
 			clock-names = "core_clk",
 				      "bus_aggr_clk",
 				      "iface_clk",
diff --git a/arch/arm64/boot/dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts b/arch/arm64/boot/dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts
new file mode 100644
index 000000000000..9155435484fb
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts
@@ -0,0 +1,1515 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2025 TUXEDO Computers GmbH
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+
+#include "hamoa.dtsi"
+#include "hamoa-pmics.dtsi"
+
+/*
+ * This device tree also works for the TUXEDO Elite 14 Gen1 notebook
+ * prototype.
+ */
+
+/ {
+	model = "Medion SPRCHRGD 14 S1";
+	compatible = "medion,sprchrgd14s1", "qcom,x1e78100", "qcom,x1e80100";
+
+	aliases {
+		serial0 = &uart21;
+		serial1 = &uart14;
+	};
+
+	wcd938x: audio-codec {
+		compatible = "qcom,wcd9385-codec";
+
+		pinctrl-0 = <&wcd_default>;
+		pinctrl-names = "default";
+
+		qcom,micbias1-microvolt = <1800000>;
+		qcom,micbias2-microvolt = <1800000>;
+		qcom,micbias3-microvolt = <1800000>;
+		qcom,micbias4-microvolt = <1800000>;
+		qcom,mbhc-buttons-vthreshold-microvolt = <75000 150000 237000 500000 500000 500000 500000 500000>;
+		qcom,mbhc-headset-vthreshold-microvolt = <1700000>;
+		qcom,mbhc-headphone-vthreshold-microvolt = <50000>;
+		qcom,rx-device = <&wcd_rx>;
+		qcom,tx-device = <&wcd_tx>;
+
+		reset-gpios = <&tlmm 191 GPIO_ACTIVE_LOW>;
+
+		vdd-buck-supply = <&vreg_l15b_1p8>;
+		vdd-rxtx-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l15b_1p8>;
+		vdd-mic-bias-supply = <&vreg_bob1>;
+
+		#sound-dai-cells = <1>;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&hall_int_n_default>;
+		pinctrl-names = "default";
+
+		switch-lid {
+			gpios = <&tlmm 92 GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_SW>;
+			linux,code = <SW_LID>;
+			wakeup-source;
+		};
+	};
+
+	hdmi-bridge {
+		compatible = "asl-tek,cs5263";
+
+		pinctrl-0 = <&hdmi_hpd_default>;
+		pinctrl-names = "default";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				hdmi_bridge_dp_in: endpoint {
+					remote-endpoint = <&usb_1_ss2_qmpphy_out_dp>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				hdmi_bridge_tmds_out: endpoint {
+					remote-endpoint = <&hdmi_con>;
+				};
+			};
+		};
+	};
+
+	hdmi-connector {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con: endpoint {
+				remote-endpoint = <&hdmi_bridge_tmds_out>;
+			};
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,x1e80100-pmic-glink",
+			     "qcom,sm8550-pmic-glink",
+			     "qcom,pmic-glink";
+		orientation-gpios = <&tlmm 121 GPIO_ACTIVE_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_ss0_hs_in: endpoint {
+						remote-endpoint = <&usb_1_ss0_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss0_ss_in: endpoint {
+						remote-endpoint = <&retimer_ss0_ss_out>;
+					};
+				};
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_ss0_con_sbu_in: endpoint {
+						remote-endpoint = <&retimer_ss0_con_sbu_out>;
+					};
+				};
+			};
+		};
+	};
+
+	vreg_edp_3p3: regulator-edp-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_EDP_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 54 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&edp_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_nvme: regulator-nvme {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_NVME_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 18 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&nvme_reg_en>;
+		pinctrl-names = "default";
+		regulator-boot-on;
+	};
+
+	vreg_rtmr0_1p15: regulator-rtmr0-1p15 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR0_1P15";
+		regulator-min-microvolt = <1150000>;
+		regulator-max-microvolt = <1150000>;
+
+		gpio = <&pmc8380_5_gpios 8 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb0_pwr_1p15_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_rtmr0_1p8: regulator-rtmr0-1p8 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR0_1P8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+
+		gpio = <&pm8550ve_9_gpios 8 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb0_1p8_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_rtmr0_3p3: regulator-rtmr0-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_RTMR0_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pm8550_gpios 11 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&usb0_3p3_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	/*
+	 * TODO: These two regulators are actually part of the removable M.2
+	 * card and not the mainboard. Need to describe this differently.
+	 * Functionally it works correctly, because all we need to do is to
+	 * turn on the actual 3.3V supply above.
+	 */
+	vreg_wcn_0p95: regulator-wcn-0p95 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_0P95";
+		regulator-min-microvolt = <950000>;
+		regulator-max-microvolt = <950000>;
+
+		vin-supply = <&vreg_wcn_3p3>;
+	};
+
+	vreg_wcn_1p9: regulator-wcn-1p9 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_1P9";
+		regulator-min-microvolt = <1900000>;
+		regulator-max-microvolt = <1900000>;
+
+		vin-supply = <&vreg_wcn_3p3>;
+	};
+
+	vreg_wcn_3p3: regulator-wcn-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 214 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&wcn_sw_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	reserved-memory {
+		linux,cma {
+			compatible = "shared-dma-pool";
+			size = <0x0 0x8000000>;
+			reusable;
+			linux,cma-default;
+		};
+	};
+
+	sound {
+		compatible = "qcom,x1e80100-sndcard";
+		model = "X1E80100-MEDION-SPRCHRGD-14-S1";
+		audio-routing = "WooferLeft IN", "WSA WSA_SPK1 OUT",
+				"TweeterLeft IN", "WSA WSA_SPK2 OUT",
+				"WooferRight IN", "WSA2 WSA_SPK2 OUT",
+				"TweeterRight IN", "WSA2 WSA_SPK2 OUT",
+				"IN1_HPHL", "HPHL_OUT",
+				"IN2_HPHR", "HPHR_OUT",
+				"AMIC2", "MIC BIAS2",
+				"VA DMIC0", "MIC BIAS1",
+				"VA DMIC1", "MIC BIAS1",
+				"TX SWR_INPUT1", "ADC2_OUTPUT";
+
+		displayport-0-dai-link {
+			link-name = "DisplayPort0 Playback";
+
+			codec {
+				sound-dai = <&mdss_dp0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai DISPLAY_PORT_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		displayport-2-dai-link {
+			link-name = "DisplayPort2 Playback";
+
+			codec {
+				sound-dai = <&mdss_dp2>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai DISPLAY_PORT_RX_2>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		va-dai-link {
+			link-name = "VA Capture";
+
+			codec {
+				sound-dai = <&lpass_vamacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai VA_CODEC_DMA_TX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		wcd-capture-dai-link {
+			link-name = "WCD Capture";
+
+			codec {
+				sound-dai = <&wcd938x 1>, <&swr2 1>, <&lpass_txmacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai TX_CODEC_DMA_TX_3>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		wcd-playback-dai-link {
+			link-name = "WCD Playback";
+
+			codec {
+				sound-dai = <&wcd938x 0>, <&swr1 0>, <&lpass_rxmacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai RX_CODEC_DMA_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		wsa-dai-link {
+			link-name = "WSA Playback";
+
+			codec {
+				sound-dai = <&left_woofer>, <&left_tweeter>,
+					    <&swr0 0>, <&lpass_wsamacro 0>,
+					    <&right_woofer>, <&right_tweeter>,
+					    <&swr3 0>, <&lpass_wsa2macro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai WSA_CODEC_DMA_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+	};
+
+	wcn7850-pmu {
+		compatible = "qcom,wcn7850-pmu";
+
+		vdd-supply = <&vreg_wcn_0p95>;
+		vddio-supply = <&vreg_l15b_1p8>;
+		vddaon-supply = <&vreg_wcn_0p95>;
+		vdddig-supply = <&vreg_wcn_0p95>;
+		vddrfa1p2-supply = <&vreg_wcn_1p9>;
+		vddrfa1p8-supply = <&vreg_wcn_1p9>;
+
+		wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>;
+		bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&wcn_wlan_bt_en>;
+		pinctrl-names = "default";
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p8: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p8";
+			};
+
+			vreg_pmu_pcie_0p9: ldo8 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_pcie_1p8: ldo9 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm8550-rpmh-regulators";
+		qcom,pmic-id = "b";
+		vdd-bob1-supply = <&vph_pwr>;
+		vdd-bob2-supply = <&vph_pwr>;
+		vdd-l1-l4-l10-supply = <&vreg_s4c_1p8>;
+		vdd-l2-l13-l14-supply = <&vreg_bob1>;
+		vdd-l5-l16-supply = <&vreg_bob1>;
+		vdd-l6-l7-supply = <&vreg_bob2>;
+		vdd-l8-l9-supply = <&vreg_bob1>;
+		vdd-l12-supply = <&vreg_s5j_1p2>;
+		vdd-l15-supply = <&vreg_s4c_1p8>;
+		vdd-l17-supply = <&vreg_bob2>;
+
+		vreg_bob1: bob1 {
+			regulator-name = "vreg_bob1";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob2: bob2 {
+			regulator-name = "vreg_bob2";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1b_1p8: ldo1 {
+			regulator-name = "vreg_l1b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2b_3p0: ldo2 {
+			regulator-name = "vreg_l2b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4b_1p8: ldo4 {
+			regulator-name = "vreg_l4b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6b_1p8: ldo6 {
+			regulator-name = "vreg_l6b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b_2p8: ldo7 {
+			regulator-name = "vreg_l7b_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b_3p0: ldo8 {
+			regulator-name = "vreg_l8b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b_2p9: ldo9 {
+			regulator-name = "vreg_l9b_2p9";
+			regulator-min-microvolt = <2960000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10b_1p8: ldo10 {
+			regulator-name = "vreg_l10b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b_1p2: ldo12 {
+			regulator-name = "vreg_l12b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13b_3p0: ldo13 {
+			regulator-name = "vreg_l13b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14b_3p0: ldo14 {
+			regulator-name = "vreg_l14b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_1p8: ldo15 {
+			regulator-name = "vreg_l15b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l16b_2p8: ldo16 {
+			regulator-name = "vreg_l16b_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b_2p5: ldo17 {
+			regulator-name = "vreg_l17b_2p5";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <2504000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vdd-l1-supply = <&vreg_s5j_1p2>;
+		vdd-l2-supply = <&vreg_s1f_0p7>;
+		vdd-l3-supply = <&vreg_s1f_0p7>;
+		vdd-s4-supply = <&vph_pwr>;
+
+		vreg_l1c_1p2: ldo1 {
+			regulator-name = "vreg_l1c_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2c_0p8: ldo2 {
+			regulator-name = "vreg_l2c_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3c_0p9: ldo3 {
+			regulator-name = "vreg_l3c_0p9";
+			regulator-min-microvolt = <920000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4c_1p8: smps4 {
+			regulator-name = "vreg_s4c_1p8";
+			regulator-min-microvolt = <1856000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pmc8380-rpmh-regulators";
+		qcom,pmic-id = "d";
+
+		vdd-l1-supply = <&vreg_s1f_0p7>;
+		vdd-l2-supply = <&vreg_s1f_0p7>;
+		vdd-l3-supply = <&vreg_s4c_1p8>;
+		vdd-s1-supply = <&vph_pwr>;
+
+		vreg_l1d_0p8: ldo1 {
+			regulator-name = "vreg_l1d_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2d_0p9: ldo2 {
+			regulator-name = "vreg_l2d_0p9";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3d_1p8: ldo3 {
+			regulator-name = "vreg_l3d_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-3 {
+		compatible = "qcom,pmc8380-rpmh-regulators";
+		qcom,pmic-id = "e";
+
+		vdd-l2-supply = <&vreg_s1f_0p7>;
+		vdd-l3-supply = <&vreg_s5j_1p2>;
+
+		vreg_l2e_0p8: ldo2 {
+			regulator-name = "vreg_l2e_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3e_1p2: ldo3 {
+			regulator-name = "vreg_l3e_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-4 {
+		compatible = "qcom,pmc8380-rpmh-regulators";
+		qcom,pmic-id = "f";
+
+		vdd-l1-supply = <&vreg_s5j_1p2>;
+		vdd-l2-supply = <&vreg_s5j_1p2>;
+		vdd-l3-supply = <&vreg_s5j_1p2>;
+		vdd-s1-supply = <&vph_pwr>;
+
+		vreg_s1f_0p7: smps1 {
+			regulator-name = "vreg_s1f_0p7";
+			regulator-min-microvolt = <700000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-6 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "i";
+
+		vdd-l1-supply = <&vreg_s4c_1p8>;
+		vdd-l2-supply = <&vreg_s5j_1p2>;
+		vdd-l3-supply = <&vreg_s1f_0p7>;
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+
+		vreg_l1i_1p8: ldo1 {
+			regulator-name = "vreg_l1i_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2i_1p2: ldo2 {
+			regulator-name = "vreg_l2i_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3i_0p8: ldo3 {
+			regulator-name = "vreg_l3i_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s1i_0p9: smps1 {
+			regulator-name = "vreg_s1i_0p9";
+			regulator-min-microvolt = <900000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s2i_1p0: smps2 {
+			regulator-name = "vreg_s2i_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-7 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "j";
+
+		vdd-l1-supply = <&vreg_s1f_0p7>;
+		vdd-l2-supply = <&vreg_s5j_1p2>;
+		vdd-l3-supply = <&vreg_s1f_0p7>;
+		vdd-s5-supply = <&vph_pwr>;
+
+		vreg_l1j_0p9: ldo1 {
+			regulator-name = "vreg_l1j_0p9";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2j_1p2: ldo2 {
+			regulator-name = "vreg_l2j_1p2";
+			regulator-min-microvolt = <1256000>;
+			regulator-max-microvolt = <1256000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3j_0p8: ldo3 {
+			regulator-name = "vreg_l3j_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s5j_1p2: smps5 {
+			regulator-name = "vreg_s5j_1p2";
+			regulator-min-microvolt = <1256000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&gpu {
+	status = "okay";
+
+	zap-shader {
+		firmware-name = "qcom/x1e80100/medion/qcdxkmsuc8380.mbn";
+	};
+};
+
+&i2c0 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	touchpad@2c {
+		compatible = "hid-over-i2c";
+		reg = <0x2c>;
+
+		hid-descr-addr = <0x20>;
+		interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>;
+
+		pinctrl-0 = <&tpad_default>;
+		pinctrl-names = "default";
+
+		wakeup-source;
+	};
+
+	keyboard@3a {
+		compatible = "hid-over-i2c";
+		reg = <0x3a>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 67 IRQ_TYPE_LEVEL_LOW>;
+
+		pinctrl-0 = <&kybd_default>;
+		pinctrl-names = "default";
+
+		wakeup-source;
+	};
+};
+
+&i2c3 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	typec-mux@8 {
+		compatible = "parade,ps8830";
+		reg = <0x8>;
+
+		clocks = <&rpmhcc RPMH_RF_CLK3>;
+
+		vdd33-supply = <&vreg_rtmr0_3p3>;
+		vdd33-cap-supply = <&vreg_rtmr0_3p3>;
+
+		vddio-supply = <&vreg_rtmr0_1p8>;
+
+		vdd-supply = <&vreg_rtmr0_1p15>;
+		vddar-supply = <&vreg_rtmr0_1p15>;
+		vddat-supply = <&vreg_rtmr0_1p15>;
+
+		reset-gpios = <&pm8550_gpios 10 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&rtmr0_default>;
+		pinctrl-names = "default";
+
+		retimer-switch;
+		orientation-switch;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				retimer_ss0_ss_out: endpoint {
+					remote-endpoint = <&pmic_glink_ss0_ss_in>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				retimer_ss0_ss_in: endpoint {
+					remote-endpoint = <&usb_1_ss0_qmpphy_out>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+
+				retimer_ss0_con_sbu_out: endpoint {
+					remote-endpoint = <&pmic_glink_ss0_con_sbu_in>;
+				};
+			};
+		};
+	};
+};
+
+&i2c5 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	eusb5_repeater: redriver@43 {
+		compatible = "nxp,ptn3222";
+		reg = <0x43>;
+		#phy-cells = <0>;
+
+		vdd1v8-supply = <&vreg_l4b_1p8>;
+		vdd3v3-supply = <&vreg_l13b_3p0>;
+
+		reset-gpios = <&tlmm 184 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&eusb5_reset_n>;
+		pinctrl-names = "default";
+	};
+
+	eusb3_repeater: redriver@47 {
+		compatible = "nxp,ptn3222";
+		reg = <0x47>;
+		#phy-cells = <0>;
+
+		vdd1v8-supply = <&vreg_l4b_1p8>;
+		vdd3v3-supply = <&vreg_l13b_3p0>;
+
+		reset-gpios = <&tlmm 124 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&eusb3_reset_n>;
+		pinctrl-names = "default";
+	};
+
+	eusb6_repeater: redriver@4f {
+		compatible = "nxp,ptn3222";
+		reg = <0x4f>;
+		#phy-cells = <0>;
+
+		vdd1v8-supply = <&vreg_l4b_1p8>;
+		vdd3v3-supply = <&vreg_l13b_3p0>;
+
+		reset-gpios = <&tlmm 111 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&eusb6_reset_n>;
+		pinctrl-names = "default";
+	};
+};
+
+&iris {
+	firmware-name = "qcom/x1e80100/medion/qcvss8380.mbn";
+
+	status = "okay";
+};
+
+&lpass_tlmm {
+	spkr_01_sd_n_active: spkr-01-sd-n-active-state {
+		pins = "gpio12";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	spkr_23_sd_n_active: spkr-23-sd-n-active-state {
+		pins = "gpio13";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+};
+
+&lpass_vamacro {
+	pinctrl-0 = <&dmic01_default>;
+	pinctrl-names = "default";
+
+	vdd-micb-supply = <&vreg_l1b_1p8>;
+	qcom,dmic-sample-rate = <4800000>;
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dp0 {
+	sound-name-prefix = "DisplayPort0";
+
+	status = "okay";
+};
+
+&mdss_dp0_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp2 {
+	sound-name-prefix = "DisplayPort2";
+
+	status = "okay";
+};
+
+&mdss_dp2_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp3 {
+	/delete-property/ #sound-dai-cells;
+
+	pinctrl-0 = <&edp0_hpd_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+
+	aux-bus {
+		panel {
+			compatible = "edp-panel";
+			power-supply = <&vreg_edp_3p3>;
+
+			port {
+				edp_panel_in: endpoint {
+					remote-endpoint = <&mdss_dp3_out>;
+				};
+			};
+		};
+	};
+
+	ports {
+		port@1 {
+			reg = <1>;
+
+			mdss_dp3_out: endpoint {
+				data-lanes = <0 1 2 3>;
+				link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+
+				remote-endpoint = <&edp_panel_in>;
+			};
+		};
+	};
+};
+
+&mdss_dp3_phy {
+	vdda-phy-supply = <&vreg_l3j_0p8>;
+	vdda-pll-supply = <&vreg_l2j_1p2>;
+
+	status = "okay";
+};
+
+&pcie4 {
+	perst-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+
+	pinctrl-0 = <&pcie4_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie4_phy {
+	vdda-phy-supply = <&vreg_l3j_0p8>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&pcie4_port0 {
+	wifi@0 {
+		compatible = "pci17cb,1107";
+		reg = <0x10000 0x0 0x0 0x0 0x0>;
+
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+		vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>;
+		vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>;
+	};
+};
+
+&pcie6a {
+	perst-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>;
+
+	wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>;
+
+	vddpe-3v3-supply = <&vreg_nvme>;
+
+	pinctrl-0 = <&pcie6a_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie6a_phy {
+	vdda-phy-supply = <&vreg_l1d_0p8>;
+	vdda-pll-supply = <&vreg_l2j_1p2>;
+
+	status = "okay";
+};
+
+&pm8550_gpios {
+	rtmr0_default: rtmr0-reset-n-active-state {
+		pins = "gpio10";
+		function = "normal";
+		power-source = <1>; /* 1.8V */
+		bias-disable;
+		input-disable;
+		output-enable;
+	};
+
+	usb0_3p3_reg_en: usb0-3p3-reg-en-state {
+		pins = "gpio11";
+		function = "normal";
+		power-source = <1>; /* 1.8V */
+		bias-disable;
+		input-disable;
+		output-enable;
+	};
+};
+
+&pm8550ve_9_gpios {
+	usb0_1p8_reg_en: usb0-1p8-reg-en-state {
+		pins = "gpio8";
+		function = "normal";
+		power-source = <1>; /* 1.8V */
+		bias-disable;
+		input-disable;
+		output-enable;
+	};
+};
+
+&pmc8380_5_gpios {
+	usb0_pwr_1p15_reg_en: usb0-pwr-1p15-reg-en-state {
+		pins = "gpio8";
+		function = "normal";
+		power-source = <1>; /* 1.8V */
+		bias-disable;
+		input-disable;
+		output-enable;
+	};
+};
+
+&qupv3_0 {
+	status = "okay";
+};
+
+&qupv3_1 {
+	status = "okay";
+};
+
+&qupv3_2 {
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/x1e80100/medion/qcadsp8380.mbn",
+			"qcom/x1e80100/medion/adsp_dtbs.elf";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/x1e80100/medion/qccdsp8380.mbn",
+			"qcom/x1e80100/medion/cdsp_dtbs.elf";
+
+	status = "okay";
+};
+
+&smb2360_0 {
+	status = "okay";
+};
+
+&smb2360_0_eusb2_repeater {
+	vdd18-supply = <&vreg_l3d_1p8>;
+	vdd3-supply = <&vreg_l2b_3p0>;
+};
+
+&swr0 {
+	status = "okay";
+
+	pinctrl-0 = <&wsa_swr_active>, <&spkr_01_sd_n_active>;
+	pinctrl-names = "default";
+
+	/* WSA8845, Left Woofer */
+	left_woofer: speaker@0,0 {
+		compatible = "sdw20217020400";
+		reg = <0 0>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "WooferLeft";
+		vdd-1p8-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l12b_1p2>;
+		qcom,port-mapping = <1 2 3 7 10 13>;
+	};
+
+	/* WSA8845, Left Tweeter */
+	left_tweeter: speaker@0,1 {
+		compatible = "sdw20217020400";
+		reg = <0 1>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "TweeterLeft";
+		vdd-1p8-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l12b_1p2>;
+		qcom,port-mapping = <4 5 6 7 11 13>;
+	};
+};
+
+&swr1 {
+	status = "okay";
+
+	/* WCD9385 RX */
+	wcd_rx: codec@0,4 {
+		compatible = "sdw20217010d00";
+		reg = <0 4>;
+		qcom,rx-port-mapping = <1 2 3 4 5>;
+	};
+};
+
+&swr2 {
+	status = "okay";
+
+	/* WCD9385 TX */
+	wcd_tx: codec@0,3 {
+		compatible = "sdw20217010d00";
+		reg = <0 3>;
+		qcom,tx-port-mapping = <2 2 3 4>;
+	};
+};
+
+&swr3 {
+	status = "okay";
+
+	pinctrl-0 = <&wsa2_swr_active>, <&spkr_23_sd_n_active>;
+	pinctrl-names = "default";
+
+	/* WSA8845, Right Woofer */
+	right_woofer: speaker@0,0 {
+		compatible = "sdw20217020400";
+		reg = <0 0>;
+		reset-gpios = <&lpass_tlmm 13 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "WooferRight";
+		vdd-1p8-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l12b_1p2>;
+		qcom,port-mapping = <1 2 3 7 10 13>;
+	};
+
+	/* WSA8845, Right Tweeter */
+	right_tweeter: speaker@0,1 {
+		compatible = "sdw20217020400";
+		reg = <0 1>;
+		reset-gpios = <&lpass_tlmm 13 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "TweeterRight";
+		vdd-1p8-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l12b_1p2>;
+		qcom,port-mapping = <4 5 6 7 11 13>;
+	};
+};
+
+&tlmm {
+	gpio-reserved-ranges = <28 4>, /* Unused */
+			       <44 4>, /* SPI (TPM) */
+			       <238 1>; /* UFS Reset */
+
+	edp_reg_en: edp-reg-en-state {
+		pins = "gpio54";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-pull-up;
+	};
+
+	eusb3_reset_n: eusb3-reset-n-state {
+		pins = "gpio124";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	eusb6_reset_n: eusb6-reset-n-state {
+		pins = "gpio111";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	eusb5_reset_n: eusb5-reset-n-state {
+		pins = "gpio184";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	hall_int_n_default: hall-int-n-state {
+		pins = "gpio92";
+		function = "gpio";
+		bias-disable;
+	};
+
+	hdmi_hpd_default: hdmi-hpd-default-state {
+		pins = "gpio126";
+		function = "usb2_dp";
+		bias-disable;
+	};
+
+	kybd_default: kybd-default-state {
+		pins = "gpio67";
+		function = "gpio";
+		bias-disable;
+	};
+
+	nvme_reg_en: nvme-reg-en-state {
+		pins = "gpio18";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	pcie4_default: pcie4-default-state {
+		clkreq-n-pins {
+			pins = "gpio147";
+			function = "pcie4_clk";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio146";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio148";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	pcie6a_default: pcie6a-default-state {
+		clkreq-n-pins {
+			pins = "gpio153";
+			function = "pcie6a_clk";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio152";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio154";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	tpad_default: tpad-default-state {
+		pins = "gpio3";
+		function = "gpio";
+		bias-disable;
+	};
+
+	wcd_default: wcd-reset-n-active-state {
+		pins = "gpio191";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	wcn_sw_en: wcn-sw-en-state {
+		pins = "gpio214";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	wcn_wlan_bt_en: wcn-wlan-bt-en-state {
+		pins = "gpio116", "gpio117";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+};
+
+&uart14 {
+	status = "okay";
+
+	bluetooth {
+		compatible = "qcom,wcn7850-bt";
+		max-speed = <3200000>;
+
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+	};
+};
+
+&usb_1_ss0_hsphy {
+	vdd-supply = <&vreg_l3j_0p8>;
+	vdda12-supply = <&vreg_l2j_1p2>;
+
+	phys = <&smb2360_0_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_1_ss0_qmpphy {
+	vdda-phy-supply = <&vreg_l2j_1p2>;
+	vdda-pll-supply = <&vreg_l1j_0p9>;
+
+	status = "okay";
+};
+
+&usb_1_ss0 {
+	status = "okay";
+};
+
+&usb_1_ss0_dwc3 {
+	dr_mode = "host";
+};
+
+&usb_1_ss0_dwc3_hs {
+	remote-endpoint = <&pmic_glink_ss0_hs_in>;
+};
+
+&usb_1_ss0_qmpphy_out {
+	remote-endpoint = <&retimer_ss0_ss_in>;
+};
+
+&usb_1_ss2_qmpphy {
+	vdda-phy-supply = <&vreg_l2j_1p2>;
+	vdda-pll-supply = <&vreg_l2d_0p9>;
+
+	/delete-property/ mode-switch;
+	/delete-property/ orientation-switch;
+
+	status = "okay";
+
+	ports {
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			/delete-node/ endpoint;
+
+			usb_1_ss2_qmpphy_out_dp: endpoint@0 {
+				reg = <0>;
+
+				data-lanes = <3 2 1 0>;
+				remote-endpoint = <&hdmi_bridge_dp_in>;
+			};
+
+			/* No USB3 lanes connected */
+		};
+	};
+};
+
+/* Camera */
+&usb_2 {
+	status = "okay";
+};
+
+&usb_2_dwc3 {
+	dr_mode = "host";
+};
+
+&usb_2_hsphy {
+	vdd-supply = <&vreg_l2e_0p8>;
+	vdda12-supply = <&vreg_l3e_1p2>;
+
+	phys = <&eusb5_repeater>;
+
+	status = "okay";
+};
+
+&usb_mp {
+	status = "okay";
+};
+
+/* Right side USB-A (eUSB 3) */
+&usb_mp_hsphy0 {
+	vdd-supply = <&vreg_l2e_0p8>;
+	vdda12-supply = <&vreg_l3e_1p2>;
+
+	phys = <&eusb3_repeater>;
+
+	status = "okay";
+};
+
+/* Left side USB-A (eUSB 6) */
+&usb_mp_hsphy1 {
+	vdd-supply = <&vreg_l2e_0p8>;
+	vdda12-supply = <&vreg_l3e_1p2>;
+
+	phys = <&eusb6_repeater>;
+
+	status = "okay";
+};
+
+/* Right side USB-A (USB-SS 3) */
+&usb_mp_qmpphy0 {
+	vdda-phy-supply = <&vreg_l3e_1p2>;
+	vdda-pll-supply = <&vreg_l3c_0p9>;
+
+	status = "okay";
+};
+
+/* Left side USB-A (USB-SS 4) */
+&usb_mp_qmpphy1 {
+	vdda-phy-supply = <&vreg_l3e_1p2>;
+	vdda-pll-supply = <&vreg_l3c_0p9>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/r9a08g045.dtsi b/arch/arm64/boot/dts/renesas/r9a08g045.dtsi
index 876de634908e..997e6cf0bb82 100644
--- a/arch/arm64/boot/dts/renesas/r9a08g045.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a08g045.dtsi
@@ -845,6 +845,71 @@ hsusb: usb@11e20000 {
 			status = "disabled";
 		};
 
+		pcie: pcie@11e40000 {
+			compatible = "renesas,r9a08g045-pcie";
+			reg = <0 0x11e40000 0 0x10000>;
+			ranges = <0x02000000 0 0x30000000 0 0x30000000 0 0x08000000>;
+			/* Map all possible DRAM ranges (4 GB). */
+			dma-ranges = <0x42000000 0 0x40000000 0 0x40000000 1 0x00000000>;
+			bus-range = <0x0 0xff>;
+			interrupts = <GIC_SPI 395 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 396 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 397 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 398 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 399 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 400 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 401 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 402 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 403 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 404 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 405 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 406 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 407 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 408 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 409 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 410 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "serr", "serr_cor", "serr_nonfatal",
+					  "serr_fatal", "axi_err", "inta",
+					  "intb", "intc", "intd", "msi",
+					  "link_bandwidth", "pm_pme", "dma",
+					  "pcie_evt", "msg", "all";
+			#interrupt-cells = <1>;
+			interrupt-controller;
+			interrupt-map-mask = <0 0 0 7>;
+			interrupt-map = <0 0 0 1 &pcie 0 0 0 0>, /* INTA */
+					<0 0 0 2 &pcie 0 0 0 1>, /* INTB */
+					<0 0 0 3 &pcie 0 0 0 2>, /* INTC */
+					<0 0 0 4 &pcie 0 0 0 3>; /* INTD */
+			clocks = <&cpg CPG_MOD R9A08G045_PCI_ACLK>,
+				 <&cpg CPG_MOD R9A08G045_PCI_CLKL1PM>;
+			clock-names = "aclk", "pm";
+			resets = <&cpg R9A08G045_PCI_ARESETN>,
+				 <&cpg R9A08G045_PCI_RST_B>,
+				 <&cpg R9A08G045_PCI_RST_GP_B>,
+				 <&cpg R9A08G045_PCI_RST_PS_B>,
+				 <&cpg R9A08G045_PCI_RST_RSM_B>,
+				 <&cpg R9A08G045_PCI_RST_CFG_B>,
+				 <&cpg R9A08G045_PCI_RST_LOAD_B>;
+			reset-names = "aresetn", "rst_b", "rst_gp_b", "rst_ps_b",
+				      "rst_rsm_b", "rst_cfg_b", "rst_load_b";
+			power-domains = <&cpg>;
+			device_type = "pci";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			renesas,sysc = <&sysc>;
+			status = "disabled";
+
+			pcie_port0: pcie@0,0 {
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				ranges;
+				device_type = "pci";
+				vendor-id = <0x1912>;
+				device-id = <0x0033>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+			};
+		};
+
 		gic: interrupt-controller@12400000 {
 			compatible = "arm,gic-v3";
 			#interrupt-cells = <3>;
diff --git a/arch/arm64/boot/dts/renesas/r9a09g047.dtsi b/arch/arm64/boot/dts/renesas/r9a09g047.dtsi
index 7a469de3bb62..d1bf16c6ff6b 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g047.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g047.dtsi
@@ -853,6 +853,36 @@ gic: interrupt-controller@14900000 {
 			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_LOW>;
 		};
 
+		xhci: usb@15850000 {
+			compatible = "renesas,r9a09g047-xhci";
+			reg = <0 0x15850000 0 0x10000>;
+			interrupts = <GIC_SPI 759 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 758 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 757 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 756 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 755 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "all", "smi", "hse", "pme", "xhc";
+			clocks = <&cpg CPG_MOD 0xaf>;
+			power-domains = <&cpg>;
+			resets = <&cpg 0xaa>;
+			phys = <&usb3_phy>, <&usb3_phy>;
+			phy-names = "usb2-phy", "usb3-phy";
+			status = "disabled";
+		};
+
+		usb3_phy: usb-phy@15870000 {
+			compatible = "renesas,r9a09g047-usb3-phy";
+			reg = <0 0x15870000 0 0x10000>;
+			clocks = <&cpg CPG_MOD 0xb0>,
+				 <&cpg CPG_CORE R9A09G047_USB3_0_CLKCORE>,
+				 <&cpg CPG_CORE R9A09G047_USB3_0_REF_ALT_CLK_P>;
+			clock-names = "pclk", "core", "ref_alt_clk_p";
+			power-domains = <&cpg>;
+			resets = <&cpg 0xaa>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
 		sdhi0: mmc@15c00000  {
 			compatible = "renesas,sdhi-r9a09g047", "renesas,sdhi-r9a09g057";
 			reg = <0x0 0x15c00000 0 0x10000>;
diff --git a/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts b/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts
index 08e814c03fa8..805c0f67b70d 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts
@@ -8,7 +8,6 @@
 /dts-v1/;
 
 /* Switch selection settings */
-#define SW_LCD_EN		0
 #define SW_GPIO8_CAN0_STB	0
 #define SW_GPIO9_CAN1_STB	0
 #define SW_LCD_EN		0
@@ -164,6 +163,11 @@ sd1-data {
 				 <RZG3E_PORT_PINMUX(G, 5, 1)>; /* SD1DAT3 */
 		};
 	};
+
+	usb3_pins: usb3 {
+		pinmux = <RZG3E_PORT_PINMUX(4, 1, 12)>, /* USB30_VBUSEN */
+			 <RZG3E_PORT_PINMUX(4, 0, 12)>; /* USB30_OVRCURN */
+	};
 };
 
 &scif0 {
@@ -179,3 +183,8 @@ &sdhi1 {
 	vmmc-supply = <&reg_3p3v>;
 	vqmmc-supply = <&vqmmc_sd1_pvdd>;
 };
+
+&xhci {
+	pinctrl-0 = <&usb3_pins>;
+	pinctrl-names = "default";
+};
diff --git a/arch/arm64/boot/dts/renesas/r9a09g056.dtsi b/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
index 8781c2fa7313..dbc02fd364c5 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
@@ -183,12 +183,104 @@ soc: soc {
 		#size-cells = <2>;
 		ranges;
 
+		icu: interrupt-controller@10400000 {
+			compatible = "renesas,r9a09g056-icu";
+			reg = <0 0x10400000 0 0x10000>;
+			#interrupt-cells = <2>;
+			#address-cells = <0>;
+			interrupt-controller;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 419 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 420 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 421 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 422 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 424 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 425 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 426 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 427 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 428 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 429 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 430 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 431 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 432 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 433 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 434 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 435 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 436 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 437 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 438 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 439 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 440 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 441 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 442 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 443 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 444 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 445 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 446 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 447 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 448 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 450 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 262 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 263 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 264 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 265 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 266 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 451 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 452 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 453 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 454 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "nmi",
+					  "port_irq0", "port_irq1", "port_irq2",
+					  "port_irq3", "port_irq4", "port_irq5",
+					  "port_irq6", "port_irq7", "port_irq8",
+					  "port_irq9", "port_irq10", "port_irq11",
+					  "port_irq12", "port_irq13", "port_irq14",
+					  "port_irq15",
+					  "tint0", "tint1", "tint2", "tint3",
+					  "tint4", "tint5", "tint6", "tint7",
+					  "tint8", "tint9", "tint10", "tint11",
+					  "tint12", "tint13", "tint14", "tint15",
+					  "tint16", "tint17", "tint18", "tint19",
+					  "tint20", "tint21", "tint22", "tint23",
+					  "tint24", "tint25", "tint26", "tint27",
+					  "tint28", "tint29", "tint30", "tint31",
+					  "int-ca55-0", "int-ca55-1",
+					  "int-ca55-2", "int-ca55-3",
+					  "icu-error-ca55",
+					  "gpt-u0-gtciada", "gpt-u0-gtciadb",
+					  "gpt-u1-gtciada", "gpt-u1-gtciadb";
+			clocks = <&cpg CPG_MOD 0x5>;
+			power-domains = <&cpg>;
+			resets = <&cpg 0x36>;
+		};
+
 		pinctrl: pinctrl@10410000 {
 			compatible = "renesas,r9a09g056-pinctrl";
 			reg = <0 0x10410000 0 0x10000>;
 			clocks = <&cpg CPG_CORE R9A09G056_IOTOP_0_SHCLK>;
 			gpio-controller;
 			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			interrupt-parent = <&icu>;
 			gpio-ranges = <&pinctrl 0 0 96>;
 			power-domains = <&cpg>;
 			resets = <&cpg 0xa5>, <&cpg 0xa6>;
@@ -232,6 +324,171 @@ xspi: spi@11030000 {
 			status = "disabled";
 		};
 
+		dmac0: dma-controller@11400000 {
+			compatible = "renesas,r9a09g056-dmac", "renesas,r9a09g057-dmac";
+			reg = <0 0x11400000 0 0x10000>;
+			interrupts = <GIC_SPI 499 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 89  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 90  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 91  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 92  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 93  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 94  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 95  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 96  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 97  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 98  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 99  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 100 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 101 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 102 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 103 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 104 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "error",
+					  "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7",
+					  "ch8", "ch9", "ch10", "ch11",
+					  "ch12", "ch13", "ch14", "ch15";
+			clocks = <&cpg CPG_MOD 0x0>;
+			power-domains = <&cpg>;
+			resets = <&cpg 0x31>;
+			#dma-cells = <1>;
+			dma-channels = <16>;
+			renesas,icu = <&icu 4>;
+		};
+
+		dmac1: dma-controller@14830000 {
+			compatible = "renesas,r9a09g056-dmac", "renesas,r9a09g057-dmac";
+			reg = <0 0x14830000 0 0x10000>;
+			interrupts = <GIC_SPI 495 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 25  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 26  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 27  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 28  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 29  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 30  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 31  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 32  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 33  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 34  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 35  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 36  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 37  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 38  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 39  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 40  IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "error",
+					  "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7",
+					  "ch8", "ch9", "ch10", "ch11",
+					  "ch12", "ch13", "ch14", "ch15";
+			clocks = <&cpg CPG_MOD 0x1>;
+			power-domains = <&cpg>;
+			resets = <&cpg 0x32>;
+			#dma-cells = <1>;
+			dma-channels = <16>;
+			renesas,icu = <&icu 0>;
+		};
+
+		dmac2: dma-controller@14840000 {
+			compatible = "renesas,r9a09g056-dmac", "renesas,r9a09g057-dmac";
+			reg = <0 0x14840000 0 0x10000>;
+			interrupts = <GIC_SPI 496 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 41  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 42  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 43  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 44  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 45  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 46  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 47  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 48  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 49  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 50  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 51  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 52  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 53  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 54  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 55  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 56  IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "error",
+					  "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7",
+					  "ch8", "ch9", "ch10", "ch11",
+					  "ch12", "ch13", "ch14", "ch15";
+			clocks = <&cpg CPG_MOD 0x2>;
+			power-domains = <&cpg>;
+			resets = <&cpg 0x33>;
+			#dma-cells = <1>;
+			dma-channels = <16>;
+			renesas,icu = <&icu 1>;
+		};
+
+		dmac3: dma-controller@12000000 {
+			compatible = "renesas,r9a09g056-dmac", "renesas,r9a09g057-dmac";
+			reg = <0 0x12000000 0 0x10000>;
+			interrupts = <GIC_SPI 497 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 57  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 58  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 59  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 60  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 61  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 62  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 63  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 64  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 65  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 66  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 67  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 68  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 69  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 70  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 71  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 72  IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "error",
+					  "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7",
+					  "ch8", "ch9", "ch10", "ch11",
+					  "ch12", "ch13", "ch14", "ch15";
+			clocks = <&cpg CPG_MOD 0x3>;
+			power-domains = <&cpg>;
+			resets = <&cpg 0x34>;
+			#dma-cells = <1>;
+			dma-channels = <16>;
+			renesas,icu = <&icu 2>;
+		};
+
+		dmac4: dma-controller@12010000 {
+			compatible = "renesas,r9a09g056-dmac", "renesas,r9a09g057-dmac";
+			reg = <0 0x12010000 0 0x10000>;
+			interrupts = <GIC_SPI 498 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 73  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 74  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 75  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 76  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 77  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 78  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 79  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 80  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 81  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 82  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 83  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 84  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 85  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 86  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 87  IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 88  IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "error",
+					  "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7",
+					  "ch8", "ch9", "ch10", "ch11",
+					  "ch12", "ch13", "ch14", "ch15";
+			clocks = <&cpg CPG_MOD 0x4>;
+			power-domains = <&cpg>;
+			resets = <&cpg 0x35>;
+			#dma-cells = <1>;
+			dma-channels = <16>;
+			renesas,icu = <&icu 3>;
+		};
+
 		ostm0: timer@11800000 {
 			compatible = "renesas,r9a09g056-ostm", "renesas,ostm";
 			reg = <0x0 0x11800000 0x0 0x1000>;
@@ -407,6 +664,69 @@ i3c: i3c@12400000 {
 			status = "disabled";
 		};
 
+		rspi0: spi@12800000 {
+			compatible = "renesas,r9a09g056-rspi",  "renesas,r9a09g057-rspi";
+			reg = <0x0 0x12800000 0x0 0x400>;
+			interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 107 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 500 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 501 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "idle", "error", "end", "rx", "tx";
+			clocks = <&cpg CPG_MOD 0x54>,
+				 <&cpg CPG_MOD 0x55>,
+				 <&cpg CPG_MOD 0x56>;
+			clock-names = "pclk", "pclk_sfr", "tclk";
+			resets = <&cpg 0x7b>, <&cpg 0x7c>;
+			reset-names = "presetn", "tresetn";
+			power-domains = <&cpg>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		rspi1: spi@12800400 {
+			compatible = "renesas,r9a09g056-rspi",  "renesas,r9a09g057-rspi";
+			reg = <0x0 0x12800400 0x0 0x400>;
+			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 110 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 502 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 503 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "idle", "error", "end", "rx", "tx";
+			clocks = <&cpg CPG_MOD 0x57>,
+				 <&cpg CPG_MOD 0x58>,
+				 <&cpg CPG_MOD 0x59>;
+			clock-names = "pclk", "pclk_sfr", "tclk";
+			resets = <&cpg 0x7d>, <&cpg 0x7e>;
+			reset-names = "presetn", "tresetn";
+			power-domains = <&cpg>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		rspi2: spi@12800800 {
+			compatible = "renesas,r9a09g056-rspi",  "renesas,r9a09g057-rspi";
+			reg = <0x0 0x12800800 0x0 0x400>;
+			interrupts = <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 113 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 504 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 505 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "idle", "error", "end", "rx", "tx";
+			clocks = <&cpg CPG_MOD 0x5a>,
+				 <&cpg CPG_MOD 0x5b>,
+				 <&cpg CPG_MOD 0x5c>;
+			clock-names = "pclk", "pclk_sfr", "tclk";
+			resets = <&cpg 0x7f>, <&cpg 0x80>;
+			reset-names = "presetn", "tresetn";
+			power-domains = <&cpg>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		i2c0: i2c@14400400 {
 			compatible = "renesas,riic-r9a09g056", "renesas,riic-r9a09g057";
 			reg = <0 0x14400400 0 0x400>;
@@ -694,6 +1014,36 @@ usb20phyrst: usb20phy-reset@15830000 {
 			status = "disabled";
 		};
 
+		xhci: usb@15850000 {
+			compatible = "renesas,r9a09g056-xhci", "renesas,r9a09g047-xhci";
+			reg = <0 0x15850000 0 0x10000>;
+			interrupts = <GIC_SPI 759 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 758 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 757 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 756 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 755 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "all", "smi", "hse", "pme", "xhc";
+			clocks = <&cpg CPG_MOD 0xaf>;
+			power-domains = <&cpg>;
+			resets = <&cpg 0xaa>;
+			phys = <&usb3_phy>, <&usb3_phy>;
+			phy-names = "usb2-phy", "usb3-phy";
+			status = "disabled";
+		};
+
+		usb3_phy: usb-phy@15870000 {
+			compatible = "renesas,r9a09g056-usb3-phy", "renesas,r9a09g047-usb3-phy";
+			reg = <0 0x15870000 0 0x10000>;
+			clocks = <&cpg CPG_MOD 0xb0>,
+				 <&cpg CPG_CORE R9A09G056_USB3_0_CLKCORE>,
+				 <&cpg CPG_CORE R9A09G056_USB3_0_REF_ALT_CLK_P>;
+			clock-names = "pclk", "core", "ref_alt_clk_p";
+			power-domains = <&cpg>;
+			resets = <&cpg 0xaa>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
 		sdhi0: mmc@15c00000  {
 			compatible = "renesas,sdhi-r9a09g056", "renesas,sdhi-r9a09g057";
 			reg = <0x0 0x15c00000 0 0x10000>;
diff --git a/arch/arm64/boot/dts/renesas/r9a09g056n48-rzv2n-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g056n48-rzv2n-evk.dts
index 066e66b5d51a..b9cd34122744 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g056n48-rzv2n-evk.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g056n48-rzv2n-evk.dts
@@ -8,6 +8,7 @@
 /dts-v1/;
 
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
 #include "r9a09g056.dtsi"
 
 / {
@@ -33,6 +34,18 @@ chosen {
 		stdout-path = "serial0:115200n8";
 	};
 
+	keys: keys {
+		compatible = "gpio-keys";
+
+		key-wakeup {
+			interrupts-extended = <&icu 0 IRQ_TYPE_EDGE_FALLING>;
+			linux,code = <KEY_WAKEUP>;
+			label = "NMI_SW";
+			debounce-interval = <20>;
+			wakeup-source;
+		};
+	};
+
 	memory@48000000 {
 		device_type = "memory";
 		/* first 128MB is reserved for secure area. */
@@ -342,6 +355,11 @@ vbus {
 		};
 	};
 
+	usb3_pins: usb3 {
+		pinmux = <RZV2N_PORT_PINMUX(B, 0, 14)>, /* USB30_VBUSEN */
+			 <RZV2N_PORT_PINMUX(B, 1, 14)>; /* USB30_OVRCURN */
+	};
+
 	xspi_pins: xspi0 {
 		ctrl {
 			pins = "XSPI0_RESET0N", "XSPI0_CS0N", "XSPI0_CKP";
@@ -392,10 +410,20 @@ &usb2_phy0 {
 	status = "okay";
 };
 
+&usb3_phy {
+	status = "okay";
+};
+
 &wdt1 {
 	status = "okay";
 };
 
+&xhci {
+	pinctrl-0 = <&usb3_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
 &xspi {
 	pinctrl-0 = <&xspi_pins>;
 	pinctrl-names = "default";
diff --git a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
index 4df32d7e9998..5f52fedf3bcd 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
@@ -1087,6 +1087,66 @@ usb21phyrst: usb21phy-reset@15840000 {
 			status = "disabled";
 		};
 
+		xhci0: usb@15850000 {
+			compatible = "renesas,r9a09g057-xhci", "renesas,r9a09g047-xhci";
+			reg = <0 0x15850000 0 0x10000>;
+			interrupts = <GIC_SPI 759 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 758 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 757 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 756 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 755 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "all", "smi", "hse", "pme", "xhc";
+			clocks = <&cpg CPG_MOD 0xaf>;
+			power-domains = <&cpg>;
+			resets = <&cpg 0xaa>;
+			phys = <&usb3_phy0>, <&usb3_phy0>;
+			phy-names = "usb2-phy", "usb3-phy";
+			status = "disabled";
+		};
+
+		xhci1: usb@15860000 {
+			compatible = "renesas,r9a09g057-xhci", "renesas,r9a09g047-xhci";
+			reg = <0 0x15860000 0 0x10000>;
+			interrupts = <GIC_SPI 764 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 763 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 762 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 761 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 760 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "all", "smi", "hse", "pme", "xhc";
+			clocks = <&cpg CPG_MOD 0xb1>;
+			power-domains = <&cpg>;
+			resets = <&cpg 0xab>;
+			phys = <&usb3_phy1>, <&usb3_phy1>;
+			phy-names = "usb2-phy", "usb3-phy";
+			status = "disabled";
+		};
+
+		usb3_phy0: usb-phy@15870000 {
+			compatible = "renesas,r9a09g057-usb3-phy", "renesas,r9a09g047-usb3-phy";
+			reg = <0 0x15870000 0 0x10000>;
+			clocks = <&cpg CPG_MOD 0xb0>,
+				 <&cpg CPG_CORE R9A09G057_USB3_0_CLKCORE>,
+				 <&cpg CPG_CORE R9A09G057_USB3_0_REF_ALT_CLK_P>;
+			clock-names = "pclk", "core", "ref_alt_clk_p";
+			power-domains = <&cpg>;
+			resets = <&cpg 0xaa>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		usb3_phy1: usb-phy@15880000 {
+			compatible = "renesas,r9a09g057-usb3-phy", "renesas,r9a09g047-usb3-phy";
+			reg = <0 0x15880000 0 0x10000>;
+			clocks = <&cpg CPG_MOD 0xb2>,
+				 <&cpg CPG_CORE R9A09G057_USB3_1_CLKCORE>,
+				 <&cpg CPG_CORE R9A09G057_USB3_1_REF_ALT_CLK_P>;
+			clock-names = "pclk", "core", "ref_alt_clk_p";
+			power-domains = <&cpg>;
+			resets = <&cpg 0xab>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
 		sdhi0: mmc@15c00000  {
 			compatible = "renesas,sdhi-r9a09g057";
 			reg = <0x0 0x15c00000 0 0x10000>;
diff --git a/arch/arm64/boot/dts/renesas/r9a09g057h44-rzv2h-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g057h44-rzv2h-evk.dts
index 445fce156f73..a865bf404dfb 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g057h44-rzv2h-evk.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g057h44-rzv2h-evk.dts
@@ -384,6 +384,16 @@ vbus {
 		};
 	};
 
+	usb30_pins: usb30 {
+		pinmux = <RZV2H_PORT_PINMUX(B, 0, 14)>, /* USB30_VBUSEN */
+			 <RZV2H_PORT_PINMUX(B, 1, 14)>; /* USB30_OVRCURN */
+	};
+
+	usb31_pins: usb31 {
+		pinmux = <RZV2H_PORT_PINMUX(6, 2, 14)>, /* USB31_VBUSEN */
+			 <RZV2H_PORT_PINMUX(6, 3, 14)>; /* USB31_OVRCURN */
+	};
+
 	xspi_pins: xspi0 {
 		ctrl {
 			pins = "XSPI0_RESET0N", "XSPI0_CS0N", "XSPI0_CKP";
@@ -450,10 +460,30 @@ &usb2_phy1 {
 	status = "okay";
 };
 
+&usb3_phy0 {
+	status = "okay";
+};
+
+&usb3_phy1 {
+	status = "okay";
+};
+
 &wdt1 {
 	status = "okay";
 };
 
+&xhci0 {
+	pinctrl-0 = <&usb30_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&xhci1 {
+	pinctrl-0 = <&usb31_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
 &xspi {
 	pinctrl-0 = <&xspi_pins>;
 	pinctrl-names = "default";
diff --git a/arch/arm64/boot/dts/renesas/r9a09g077.dtsi b/arch/arm64/boot/dts/renesas/r9a09g077.dtsi
index f5fa6ca06409..dd03e2596c16 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g077.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g077.dtsi
@@ -165,6 +165,78 @@ sci5: serial@81005000 {
 			status = "disabled";
 		};
 
+		rspi0: spi@80007000 {
+			compatible = "renesas,r9a09g077-rspi";
+			reg = <0x0 0x80007000 0x0 0x400>;
+			interrupts = <GIC_SPI 636 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 637 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 638 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 634 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 635 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "idle", "error", "end", "rx", "tx";
+			clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKM>,
+				 <&cpg CPG_MOD 104>;
+			clock-names = "pclk", "pclkspi";
+			power-domains = <&cpg>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		rspi1: spi@80007400 {
+			compatible = "renesas,r9a09g077-rspi";
+			reg = <0x0 0x80007400 0x0 0x400>;
+			interrupts = <GIC_SPI 641 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 642 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 643 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 639 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 640 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "idle", "error", "end", "rx", "tx";
+			clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKM>,
+				 <&cpg CPG_MOD 105>;
+			clock-names = "pclk", "pclkspi";
+			power-domains = <&cpg>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		rspi2: spi@80007800 {
+			compatible = "renesas,r9a09g077-rspi";
+			reg = <0x0 0x80007800 0x0 0x400>;
+			interrupts = <GIC_SPI 646 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 647 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 648 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 644 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 645 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "idle", "error", "end", "rx", "tx";
+			clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKM>,
+				 <&cpg CPG_MOD 106>;
+			clock-names = "pclk", "pclkspi";
+			power-domains = <&cpg>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		rspi3: spi@81007000 {
+			compatible = "renesas,r9a09g077-rspi";
+			reg = <0x0 0x81007000 0x0 0x400>;
+			interrupts = <GIC_SPI 651 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 652 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 653 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 649 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 650 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "idle", "error", "end", "rx", "tx";
+			clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKM>,
+				 <&cpg CPG_MOD 602>;
+			clock-names = "pclk", "pclkspi";
+			power-domains = <&cpg>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		wdt0: watchdog@80082000 {
 			compatible = "renesas,r9a09g077-wdt";
 			reg = <0 0x80082000 0 0x400>,
@@ -756,6 +828,79 @@ cpg: clock-controller@80280000 {
 			#power-domain-cells = <0>;
 		};
 
+		icu: interrupt-controller@802a0000 {
+			compatible = "renesas,r9a09g077-icu";
+			reg = <0 0x802a0000 0 0x10000>,
+			      <0 0x812a0000 0 0x50>;
+			#interrupt-cells = <2>;
+			#address-cells = <0>;
+			interrupt-controller;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 1 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 2 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 3 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 4 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 5 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 6 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 7 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 8 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 9 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 10 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 11 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 12 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 13 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 14 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 15 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 16 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 17 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 18 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 19 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 20 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 21 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 22 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 23 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 24 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 25 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 26 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 27 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 28 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 29 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 30 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 31 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 406 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 407 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 408 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 409 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 410 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 411 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 412 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 413 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 414 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 415 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 416 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 417 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 418 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "intcpu0", "intcpu1", "intcpu2",
+					  "intcpu3", "intcpu4", "intcpu5",
+					  "intcpu6", "intcpu7", "intcpu8",
+					  "intcpu9", "intcpu10", "intcpu11",
+					  "intcpu12", "intcpu13", "intcpu14",
+					  "intcpu15",
+					  "irq0", "irq1", "irq2", "irq3",
+					  "irq4", "irq5", "irq6", "irq7",
+					  "irq8", "irq9", "irq10", "irq11",
+					  "irq12", "irq13", "irq14", "irq15",
+					  "sei",
+					  "ca55-err0", "ca55-err1",
+					  "cr520-err0", "cr520-err1",
+					  "cr521-err0", "cr521-err1",
+					  "peri-err0", "peri-err1",
+					  "dsmif-err0", "dsmif-err1",
+					  "encif-err0", "encif-err1";
+			clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKM>;
+			power-domains = <&cpg>;
+		};
+
 		pinctrl: pinctrl@802c0000 {
 			compatible = "renesas,r9a09g077-pinctrl";
 			reg = <0 0x802c0000 0 0x10000>,
diff --git a/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts
index b7706d0bc3aa..d1474f1bd523 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts
@@ -26,6 +26,9 @@
  * P17_4 = SD1_CD; SW2[3] = ON
  * P08_5 = SD1_PWEN; SW2[3] = ON
  * P08_6 = SD1_IOVS; SW2[3] = ON; SW5[3] = OFF; SW5[4] = ON
+ * To enable proper operation in 1.8V modes, CN77 must have pins 2 and 3
+ * connected by the jumper. This connects SD1 power-supply control IC output
+ * back to VCC1833_7.
  */
 #define SD1_MICRO_SD	1
 
diff --git a/arch/arm64/boot/dts/renesas/r9a09g087.dtsi b/arch/arm64/boot/dts/renesas/r9a09g087.dtsi
index 361a9235f00d..0f04ecb5c21c 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g087.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g087.dtsi
@@ -165,6 +165,78 @@ sci5: serial@81005000 {
 			status = "disabled";
 		};
 
+		rspi0: spi@80007000 {
+			compatible = "renesas,r9a09g087-rspi", "renesas,r9a09g077-rspi";
+			reg = <0x0 0x80007000 0x0 0x400>;
+			interrupts = <GIC_SPI 636 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 637 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 638 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 634 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 635 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "idle", "error", "end", "rx", "tx";
+			clocks = <&cpg CPG_CORE R9A09G087_CLK_PCLKM>,
+				 <&cpg CPG_MOD 104>;
+			clock-names = "pclk", "pclkspi";
+			power-domains = <&cpg>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		rspi1: spi@80007400 {
+			compatible = "renesas,r9a09g087-rspi", "renesas,r9a09g077-rspi";
+			reg = <0x0 0x80007400 0x0 0x400>;
+			interrupts = <GIC_SPI 641 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 642 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 643 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 639 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 640 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "idle", "error", "end", "rx", "tx";
+			clocks = <&cpg CPG_CORE R9A09G087_CLK_PCLKM>,
+				 <&cpg CPG_MOD 105>;
+			clock-names = "pclk", "pclkspi";
+			power-domains = <&cpg>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		rspi2: spi@80007800 {
+			compatible = "renesas,r9a09g087-rspi", "renesas,r9a09g077-rspi";
+			reg = <0x0 0x80007800 0x0 0x400>;
+			interrupts = <GIC_SPI 646 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 647 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 648 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 644 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 645 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "idle", "error", "end", "rx", "tx";
+			clocks = <&cpg CPG_CORE R9A09G087_CLK_PCLKM>,
+				 <&cpg CPG_MOD 106>;
+			clock-names = "pclk", "pclkspi";
+			power-domains = <&cpg>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		rspi3: spi@81007000 {
+			compatible = "renesas,r9a09g087-rspi", "renesas,r9a09g077-rspi";
+			reg = <0x0 0x81007000 0x0 0x400>;
+			interrupts = <GIC_SPI 651 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 652 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 653 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 649 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 650 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "idle", "error", "end", "rx", "tx";
+			clocks = <&cpg CPG_CORE R9A09G087_CLK_PCLKM>,
+				 <&cpg CPG_MOD 602>;
+			clock-names = "pclk", "pclkspi";
+			power-domains = <&cpg>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		wdt0: watchdog@80082000 {
 			compatible = "renesas,r9a09g087-wdt", "renesas,r9a09g077-wdt";
 			reg = <0 0x80082000 0 0x400>,
@@ -759,6 +831,79 @@ cpg: clock-controller@80280000 {
 			#power-domain-cells = <0>;
 		};
 
+		icu: interrupt-controller@802a0000 {
+			compatible = "renesas,r9a09g087-icu", "renesas,r9a09g077-icu";
+			reg = <0 0x802a0000 0 0x10000>,
+			      <0 0x812a0000 0 0x50>;
+			#interrupt-cells = <2>;
+			#address-cells = <0>;
+			interrupt-controller;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 1 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 2 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 3 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 4 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 5 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 6 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 7 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 8 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 9 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 10 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 11 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 12 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 13 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 14 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 15 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 16 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 17 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 18 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 19 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 20 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 21 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 22 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 23 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 24 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 25 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 26 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 27 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 28 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 29 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 30 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 31 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 406 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 407 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 408 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 409 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 410 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 411 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 412 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 413 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 414 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 415 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 416 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 417 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 418 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "intcpu0", "intcpu1", "intcpu2",
+					  "intcpu3", "intcpu4", "intcpu5",
+					  "intcpu6", "intcpu7", "intcpu8",
+					  "intcpu9", "intcpu10", "intcpu11",
+					  "intcpu12", "intcpu13", "intcpu14",
+					  "intcpu15",
+					  "irq0", "irq1", "irq2", "irq3",
+					  "irq4", "irq5", "irq6", "irq7",
+					  "irq8", "irq9", "irq10", "irq11",
+					  "irq12", "irq13", "irq14", "irq15",
+					  "sei",
+					  "ca55-err0", "ca55-err1",
+					  "cr520-err0", "cr520-err1",
+					  "cr521-err0", "cr521-err1",
+					  "peri-err0", "peri-err1",
+					  "dsmif-err0", "dsmif-err1",
+					  "encif-err0", "encif-err1";
+			clocks = <&cpg CPG_CORE R9A09G087_CLK_PCLKM>;
+			power-domains = <&cpg>;
+		};
+
 		pinctrl: pinctrl@802c0000 {
 			compatible = "renesas,r9a09g087-pinctrl";
 			reg = <0 0x802c0000 0 0x10000>,
diff --git a/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts
index 17c0c79fbd96..eaf73868783b 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts
@@ -30,6 +30,9 @@
 /*
  * P17_4 = SD1_CD; DSW5[3] = ON; DSW19[1] = OFF; DSW19[2] = ON
  * P08_6 = SD1_IOVS; DSW5[3] = ON
+ * To enable proper operation in 1.8V modes, JP21 must have pins 2 and 3
+ * connected by the jumper. This connects SD1 power-supply control IC output
+ * back to VCC1833_7.
  */
 #define SD1_MICRO_SD	1
 
diff --git a/arch/arm64/boot/dts/renesas/renesas-smarc2.dtsi b/arch/arm64/boot/dts/renesas/renesas-smarc2.dtsi
index 58561da3007a..8b3765b8267d 100644
--- a/arch/arm64/boot/dts/renesas/renesas-smarc2.dtsi
+++ b/arch/arm64/boot/dts/renesas/renesas-smarc2.dtsi
@@ -106,3 +106,11 @@ &sdhi1 {
 
 	status = "okay";
 };
+
+&usb3_phy {
+	status = "okay";
+};
+
+&xhci {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi
index 6f25ab617982..982f17aafbc5 100644
--- a/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi
@@ -168,6 +168,11 @@ a0 80 30 30 9c
 	};
 };
 
+&pcie_port0 {
+	clocks = <&versa3 5>;
+	clock-names = "ref";
+};
+
 #if SW_CONFIG2 == SW_ON
 /* SD0 slot */
 &sdhi0 {
diff --git a/arch/arm64/boot/dts/renesas/rzg3s-smarc.dtsi b/arch/arm64/boot/dts/renesas/rzg3s-smarc.dtsi
index 6b0bb2c441af..70af605168b0 100644
--- a/arch/arm64/boot/dts/renesas/rzg3s-smarc.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg3s-smarc.dtsi
@@ -155,6 +155,12 @@ &ohci1 {
 	status = "okay";
 };
 
+&pcie {
+	pinctrl-0 = <&pcie_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
 &phyrst {
 	status = "okay";
 };
@@ -186,6 +192,11 @@ key-3-gpio-hog {
 		line-name = "key-3-gpio-irq";
 	};
 
+	pcie_pins: pcie {
+		pinmux = <RZG2L_PORT_PINMUX(13, 2, 2)>, /* PCIE_RST_OUT_B */
+			 <RZG2L_PORT_PINMUX(13, 3, 2)>; /* PCIE_CLKREQ_B */
+	};
+
 	scif0_pins: scif0 {
 		pinmux = <RZG2L_PORT_PINMUX(6, 3, 1)>, /* RXD */
 			 <RZG2L_PORT_PINMUX(6, 4, 1)>; /* TXD */
diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile
index dbdda9783e93..3fdd1e4832e1 100644
--- a/arch/arm64/boot/dts/rockchip/Makefile
+++ b/arch/arm64/boot/dts/rockchip/Makefile
@@ -159,6 +159,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-100ask-dshanpi-a1.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-armsom-sige5.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-armsom-sige5-v1.2-wifibt.dtbo
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-evb1-v10.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-evb1-v10-pcie1.dtbo
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-luckfox-omni3576.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-nanopi-m5.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-nanopi-r76s.dtb
@@ -259,6 +260,10 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-armsom-sige5-v1.2-wifibt.dtb
 rk3576-armsom-sige5-v1.2-wifibt-dtbs := rk3576-armsom-sige5.dtb \
 	rk3576-armsom-sige5-v1.2-wifibt.dtbo
 
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-evb1-v10-pcie1.dtb
+rk3576-evb1-v10-pcie1-dtbs := rk3576-evb1-v10.dtb \
+	rk3576-evb1-v10-pcie1.dtbo
+
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-edgeble-neu6a-wifi.dtb
 rk3588-edgeble-neu6a-wifi-dtbs := rk3588-edgeble-neu6a-io.dtb \
 	rk3588-edgeble-neu6a-wifi.dtbo
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts b/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts
index 2dca1dca20b8..0c28dadfaf6b 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts
@@ -40,13 +40,13 @@ adc-keys {
 		button-up {
 			label = "Volume Up";
 			linux,code = <KEY_VOLUMEUP>;
-			press-threshold-microvolt = <100000>;
+			press-threshold-microvolt = <2000>;
 		};
 
 		button-down {
 			label = "Volume Down";
 			linux,code = <KEY_VOLUMEDOWN>;
-			press-threshold-microvolt = <600000>;
+			press-threshold-microvolt = <300000>;
 		};
 	};
 
@@ -533,7 +533,13 @@ mpu6500@68 {
 		reg = <0x68>;
 		interrupt-parent = <&gpio1>;
 		interrupts = <RK_PC6 IRQ_TYPE_LEVEL_LOW>;
+		vdd-supply = <&vcc_1v8>;
 		vddio-supply = <&vcc_1v8>;
+
+		mount-matrix =
+			"1", "0", "0",
+			"0", "-1", "0",
+			"0", "0", "-1";
 	};
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10-pcie1.dtso b/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10-pcie1.dtso
new file mode 100644
index 000000000000..dccf4a5debdb
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10-pcie1.dtso
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * DT-overlay to enable the onboard PCIe x1 slot, which shares pins and the PHY
+ * with the USB3 host port.
+ * To use the PCIe slot, apply this overlay and flip the Dial_Switch_1 right
+ * next to the PCIe slot to low state (labeled "ON - PCIe1"). USB3 host port
+ * will be unusable (not even in 2.0 mode)
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/pinctrl/rockchip.h>
+
+&pcie1 {
+	pinctrl-0 = <&pcie1m0_pins &pcie1_rst>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&pinctrl {
+	pcie1 {
+		pcie1_rst: pcie1-rst {
+			rockchip,pins = <4 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+};
+
+&usb_drd1_dwc3 {
+	status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-rock-5-itx.dts b/arch/arm64/boot/dts/rockchip/rk3588-rock-5-itx.dts
index 172aeabba72a..de154adb1497 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-rock-5-itx.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-rock-5-itx.dts
@@ -147,6 +147,24 @@ sdio_pwrseq: sdio-pwrseq {
 		reset-gpios = <&gpio0 RK_PC4 GPIO_ACTIVE_LOW>;
 	};
 
+	spdif_dit: spdif-dit {
+		compatible = "linux,spdif-dit";
+		#sound-dai-cells = <0>;
+	};
+
+	spdif_sound: spdif-sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "SPDIF";
+
+		simple-audio-card,cpu {
+			sound-dai = <&spdif_tx1>;
+		};
+
+		simple-audio-card,codec {
+			sound-dai = <&spdif_dit>;
+		};
+	};
+
 	typec_vin: regulator-typec-vin {
 		compatible = "regulator-fixed";
 		enable-active-high;
@@ -854,6 +872,11 @@ spi_flash: flash@0 {
 	};
 };
 
+&spdif_tx1 {
+	pinctrl-0 = <&spdif1m2_tx>;
+	status = "okay";
+};
+
 &spi2 {
 	status = "okay";
 	assigned-clocks = <&cru CLK_SPI2>;
diff --git a/arch/arm64/boot/dts/ti/k3-am62-lp-sk-nand.dtso b/arch/arm64/boot/dts/ti/k3-am62-lp-sk-nand.dtso
index 173ac60723b6..b4daa674eaa1 100644
--- a/arch/arm64/boot/dts/ti/k3-am62-lp-sk-nand.dtso
+++ b/arch/arm64/boot/dts/ti/k3-am62-lp-sk-nand.dtso
@@ -14,7 +14,7 @@ &mcasp1 {
 };
 
 &main_pmx0 {
-	gpmc0_pins_default: gpmc0-pins-default {
+	gpmc0_pins_default: gpmc0-default-pins {
 		pinctrl-single,pins = <
 			AM62X_IOPAD(0x003c, PIN_INPUT, 0) /* (K19) GPMC0_AD0 */
 			AM62X_IOPAD(0x0040, PIN_INPUT, 0) /* (L19) GPMC0_AD1 */
diff --git a/arch/arm64/boot/dts/ti/k3-am62-phycore-som.dtsi b/arch/arm64/boot/dts/ti/k3-am62-phycore-som.dtsi
index 878d267bc663..e15da771bc07 100644
--- a/arch/arm64/boot/dts/ti/k3-am62-phycore-som.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62-phycore-som.dtsi
@@ -220,6 +220,10 @@ &cpsw_port1 {
 	bootph-all;
 };
 
+&cpsw_mac_syscon {
+	bootph-all;
+};
+
 &cpsw3g_mdio {
 	pinctrl-names = "default";
 	pinctrl-0 = <&main_mdio1_pins_default>;
@@ -359,6 +363,10 @@ serial_flash: flash@0 {
 	};
 };
 
+&phy_gmii_sel {
+	bootph-all;
+};
+
 &sdhci0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&main_mmc0_pins_default>;
diff --git a/arch/arm64/boot/dts/ti/k3-am62a-phycore-som.dtsi b/arch/arm64/boot/dts/ti/k3-am62a-phycore-som.dtsi
index b24a63feeab8..de4048a3564b 100644
--- a/arch/arm64/boot/dts/ti/k3-am62a-phycore-som.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62a-phycore-som.dtsi
@@ -197,6 +197,10 @@ &cpsw_port1 {
 	bootph-all;
 };
 
+&cpsw_mac_syscon {
+	bootph-all;
+};
+
 &cpsw3g_mdio {
 	pinctrl-names = "default";
 	pinctrl-0 = <&main_mdio1_pins_default>;
@@ -350,6 +354,10 @@ serial_flash: flash@0 {
 	};
 };
 
+&phy_gmii_sel {
+	bootph-all;
+};
+
 &sdhci0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&main_mmc0_pins_default>;
diff --git a/arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi b/arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi
index 5e050cbb9eaf..ec9dd931fe92 100644
--- a/arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi
@@ -112,7 +112,7 @@ reg_sd1_vmmc: regulator-sdhci1-vmmc {
 		regulator-max-microvolt = <3300000>;
 		regulator-min-microvolt = <3300000>;
 		regulator-name = "+V3.3_SD";
-		startup-delay-us = <2000>;
+		startup-delay-us = <20000>;
 	};
 
 	reg_sd1_vqmmc: regulator-sdhci1-vqmmc {
diff --git a/arch/arm64/boot/dts/ti/k3-am642-phyboard-electra-peb-c-010.dtso b/arch/arm64/boot/dts/ti/k3-am642-phyboard-electra-peb-c-010.dtso
index 7fc73cfacadb..1176a52d560b 100644
--- a/arch/arm64/boot/dts/ti/k3-am642-phyboard-electra-peb-c-010.dtso
+++ b/arch/arm64/boot/dts/ti/k3-am642-phyboard-electra-peb-c-010.dtso
@@ -30,13 +30,10 @@ icssg1-ethernet {
 				<&main_pktdma 0xc206 15>, /* egress slice 1 */
 				<&main_pktdma 0xc207 15>, /* egress slice 1 */
 				<&main_pktdma 0x4200 15>, /* ingress slice 0 */
-				<&main_pktdma 0x4201 15>, /* ingress slice 1 */
-				<&main_pktdma 0x4202 0>, /* mgmnt rsp slice 0 */
-				<&main_pktdma 0x4203 0>; /* mgmnt rsp slice 1 */
+				<&main_pktdma 0x4201 15>; /* ingress slice 1 */
 		dma-names = "tx0-0", "tx0-1", "tx0-2", "tx0-3",
 					"tx1-0", "tx1-1", "tx1-2", "tx1-3",
-					"rx0", "rx1",
-					"rxmgm0", "rxmgm1";
+					"rx0", "rx1";
 
 		firmware-name = "ti-pruss/am65x-sr2-pru0-prueth-fw.elf",
 				"ti-pruss/am65x-sr2-rtu0-prueth-fw.elf",
diff --git a/arch/arm64/boot/dts/ti/k3-am642-phyboard-electra-x27-gpio1-spi1-uart3.dtso b/arch/arm64/boot/dts/ti/k3-am642-phyboard-electra-x27-gpio1-spi1-uart3.dtso
index 996c42ec4253..bea8efa3e909 100644
--- a/arch/arm64/boot/dts/ti/k3-am642-phyboard-electra-x27-gpio1-spi1-uart3.dtso
+++ b/arch/arm64/boot/dts/ti/k3-am642-phyboard-electra-x27-gpio1-spi1-uart3.dtso
@@ -20,13 +20,13 @@ aliases {
 };
 
 &main_pmx0 {
-	main_gpio1_exp_header_gpio_pins_default: main-gpio1-exp-header-gpio-pins-default {
+	main_gpio1_exp_header_gpio_pins_default: main-gpio1-exp-header-gpio-default-pins {
 		pinctrl-single,pins = <
 			AM64X_IOPAD(0x0220, PIN_INPUT, 7)	/* (D14) SPI1_CS1.GPIO1_48 */
 		>;
 	};
 
-	main_spi1_pins_default: main-spi1-pins-default {
+	main_spi1_pins_default: main-spi1-default-pins {
 		pinctrl-single,pins = <
 			AM64X_IOPAD(0x0224, PIN_INPUT, 0)	/* (C14) SPI1_CLK */
 			AM64X_IOPAD(0x021C, PIN_OUTPUT, 0)	/* (B14) SPI1_CS0 */
@@ -35,7 +35,7 @@ AM64X_IOPAD(0x022C, PIN_INPUT, 0)	/* (A15) SPI1_D1 */
 		>;
 	};
 
-	main_uart3_pins_default: main-uart3-pins-default {
+	main_uart3_pins_default: main-uart3-default-pins {
 		pinctrl-single,pins = <
 			AM64X_IOPAD(0x0048, PIN_INPUT, 2)       /* (U20) GPMC0_AD3.UART3_RXD */
 			AM64X_IOPAD(0x004c, PIN_OUTPUT, 2)      /* (U18) GPMC0_AD4.UART3_TXD */
@@ -52,7 +52,7 @@ &main_gpio1 {
 &main_spi1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&main_spi1_pins_default>;
-	ti,pindir-d0-out-d1-in = <1>;
+	ti,pindir-d0-out-d1-in;
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts b/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts
index 55fd214a82e4..c816ba3bfbdf 100644
--- a/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts
+++ b/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts
@@ -280,8 +280,8 @@ connector {
 			try-power-role = "sink";
 			self-powered;
 			source-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
-			sink-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
-			op-sink-microwatt = <1000000>;
+			sink-pdos = <PDO_FIXED(5000, 0, PDO_FIXED_USB_COMM)>;
+			op-sink-microwatt = <0>;
 
 			ports {
 				#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts b/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts
index c7ce804eac70..f48601ae38b7 100644
--- a/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts
+++ b/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts
@@ -399,8 +399,8 @@ connector {
 			try-power-role = "sink";
 			self-powered;
 			source-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
-			sink-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
-			op-sink-microwatt = <1000000>;
+			sink-pdos = <PDO_FIXED(5000, 0, PDO_FIXED_USB_COMM)>;
+			op-sink-microwatt = <0>;
 
 			ports {
 				#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/toshiba/tmpv7708.dtsi b/arch/arm64/boot/dts/toshiba/tmpv7708.dtsi
index 9aa7b1872bd6..88e38d6efcaa 100644
--- a/arch/arm64/boot/dts/toshiba/tmpv7708.dtsi
+++ b/arch/arm64/boot/dts/toshiba/tmpv7708.dtsi
@@ -161,7 +161,7 @@ gic: interrupt-controller@24001000 {
 			      <0 0x24006000 0 0x2000>;
 		};
 
-		pmux: pmux@24190000 {
+		pmux: pinctrl@24190000 {
 			compatible = "toshiba,tmpv7708-pinctrl";
 			reg = <0 0x24190000 0 0x10000>;
 		};
@@ -463,7 +463,7 @@ piether: ethernet@28000000 {
 			status = "disabled";
 		};
 
-		wdt: wdt@28330000 {
+		wdt: watchdog@28330000 {
 			compatible = "toshiba,visconti-wdt";
 			reg = <0 0x28330000 0 0x1000>;
 			clocks = <&pismu TMPV770X_CLK_WDTCLK>;
diff --git a/arch/arm64/boot/dts/toshiba/tmpv7708_pins.dtsi b/arch/arm64/boot/dts/toshiba/tmpv7708_pins.dtsi
index a480c6ba5f5d..5ea835fe08a8 100644
--- a/arch/arm64/boot/dts/toshiba/tmpv7708_pins.dtsi
+++ b/arch/arm64/boot/dts/toshiba/tmpv7708_pins.dtsi
@@ -91,7 +91,7 @@ i2c8_pins: i2c8-pins {
 		bias-pull-up;
 	};
 
-	pwm_mux: pwm_mux {
+	pwm_mux: pwm-pins {
 		function = "pwm";
 	};
 
diff --git a/arch/arm64/boot/dts/xilinx/versal-net.dtsi b/arch/arm64/boot/dts/xilinx/versal-net.dtsi
index 412af9a394aa..15f767608e67 100644
--- a/arch/arm64/boot/dts/xilinx/versal-net.dtsi
+++ b/arch/arm64/boot/dts/xilinx/versal-net.dtsi
@@ -1018,7 +1018,7 @@ smmu: iommu@ec000000 {
 		};
 
 		spi0: spi@f1960000 {
-			compatible = "cdns,spi-r1p6";
+			compatible = "xlnx,versal-net-spi-r1p6", "cdns,spi-r1p6";
 			status = "disabled";
 			interrupts = <0 23 4>;
 			reg = <0 0xf1960000 0 0x1000>;
@@ -1026,7 +1026,7 @@ spi0: spi@f1960000 {
 		};
 
 		spi1: spi@f1970000 {
-			compatible = "cdns,spi-r1p6";
+			compatible = "xlnx,versal-net-spi-r1p6", "cdns,spi-r1p6";
 			status = "disabled";
 			interrupts = <0 24 4>;
 			reg = <0 0xf1970000 0 0x1000>;
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-clk-ccf.dtsi b/arch/arm64/boot/dts/xilinx/zynqmp-clk-ccf.dtsi
index 52e122fc7c9e..482f432ba7f3 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-clk-ccf.dtsi
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-clk-ccf.dtsi
@@ -14,7 +14,7 @@ pss_ref_clk: pss-ref-clk {
 		bootph-all;
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
-		clock-frequency = <33333333>;
+		clock-frequency = <33333000>;
 		clock-output-names = "pss_ref_clk";
 	};
 
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi
index 938b014ca923..29058e633fe9 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi
+++ b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi
@@ -103,23 +103,23 @@ CPU_SLEEP_0: cpu-sleep-0 {
 	cpu_opp_table: opp-table-cpu {
 		compatible = "operating-points-v2";
 		opp-shared;
-		opp00 {
-			opp-hz = /bits/ 64 <1199999988>;
+		opp-1200000000 {
+			opp-hz = /bits/ 64 <1200000000>;
 			opp-microvolt = <1000000>;
 			clock-latency-ns = <500000>;
 		};
-		opp01 {
-			opp-hz = /bits/ 64 <599999994>;
+		opp-600000000 {
+			opp-hz = /bits/ 64 <600000000>;
 			opp-microvolt = <1000000>;
 			clock-latency-ns = <500000>;
 		};
-		opp02 {
-			opp-hz = /bits/ 64 <399999996>;
+		opp-400000000 {
+			opp-hz = /bits/ 64 <400000000>;
 			opp-microvolt = <1000000>;
 			clock-latency-ns = <500000>;
 		};
-		opp03 {
-			opp-hz = /bits/ 64 <299999997>;
+		opp-300000000 {
+			opp-hz = /bits/ 64 <300000000>;
 			opp-microvolt = <1000000>;
 			clock-latency-ns = <500000>;
 		};
@@ -192,11 +192,6 @@ psci {
 	};
 
 	firmware {
-		optee: optee  {
-			compatible = "linaro,optee-tz";
-			method = "smc";
-		};
-
 		zynqmp_firmware: zynqmp-firmware {
 			compatible = "xlnx,zynqmp-firmware";
 			#power-domain-cells = <1>;
@@ -1080,7 +1075,7 @@ smmu: iommu@fd800000 {
 		};
 
 		spi0: spi@ff040000 {
-			compatible = "cdns,spi-r1p6";
+			compatible = "xlnx,zynqmp-spi-r1p6", "cdns,spi-r1p6";
 			status = "disabled";
 			interrupt-parent = <&gic>;
 			interrupts = <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>;
@@ -1092,7 +1087,7 @@ spi0: spi@ff040000 {
 		};
 
 		spi1: spi@ff050000 {
-			compatible = "cdns,spi-r1p6";
+			compatible = "xlnx,zynqmp-spi-r1p6", "cdns,spi-r1p6";
 			status = "disabled";
 			interrupt-parent = <&gic>;
 			interrupts = <GIC_SPI 20 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index 45288ec9eaf7..2f746f779323 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -231,8 +231,13 @@ CONFIG_PCIE_MEDIATEK_GEN3=m
 CONFIG_PCI_TEGRA=y
 CONFIG_PCIE_RCAR_HOST=y
 CONFIG_PCIE_RCAR_EP=y
+CONFIG_PCIE_RENESAS_RZG3S_HOST=y
 CONFIG_PCIE_ROCKCHIP_HOST=m
 CONFIG_PCI_XGENE=y
+CONFIG_PCIE_XILINX=y
+CONFIG_PCIE_XILINX_DMA_PL=y
+CONFIG_PCIE_XILINX_NWL=y
+CONFIG_PCIE_XILINX_CPM=y
 CONFIG_PCI_IMX6_HOST=y
 CONFIG_PCI_LAYERSCAPE=y
 CONFIG_PCI_HISI=y
@@ -273,6 +278,7 @@ CONFIG_QCOM_QSEECOM=y
 CONFIG_QCOM_QSEECOM_UEFISECAPP=y
 CONFIG_EXYNOS_ACPM_PROTOCOL=m
 CONFIG_TEGRA_BPMP=y
+CONFIG_ZYNQMP_FIRMWARE_DEBUG=y
 CONFIG_GNSS=m
 CONFIG_GNSS_MTK_SERIAL=m
 CONFIG_MTD=y
@@ -305,6 +311,7 @@ CONFIG_QCOM_COINCELL=m
 CONFIG_QCOM_FASTRPC=m
 CONFIG_SRAM=y
 CONFIG_PCI_ENDPOINT_TEST=m
+CONFIG_XILINX_SDFEC=m
 CONFIG_EEPROM_AT24=m
 CONFIG_EEPROM_AT25=m
 CONFIG_UACCE=m
@@ -394,6 +401,7 @@ CONFIG_DWMAC_MEDIATEK=m
 CONFIG_DWMAC_TEGRA=m
 CONFIG_TI_K3_AM65_CPSW_NUSS=y
 CONFIG_TI_ICSSG_PRUETH=m
+CONFIG_XILINX_AXI_EMAC=m
 CONFIG_QCOM_IPA=m
 CONFIG_MESON_GXL_PHY=m
 CONFIG_AQUANTIA_PHY=y
@@ -410,7 +418,9 @@ CONFIG_DP83867_PHY=y
 CONFIG_DP83869_PHY=m
 CONFIG_DP83TD510_PHY=y
 CONFIG_VITESSE_PHY=y
+CONFIG_XILINX_GMII2RGMII=m
 CONFIG_CAN_FLEXCAN=m
+CONFIG_CAN_XILINXCAN=m
 CONFIG_CAN_M_CAN=m
 CONFIG_CAN_M_CAN_PLATFORM=m
 CONFIG_CAN_RCAR=m
@@ -562,6 +572,7 @@ CONFIG_I2C_S3C2410=y
 CONFIG_I2C_SH_MOBILE=y
 CONFIG_I2C_TEGRA=y
 CONFIG_I2C_UNIPHIER_F=y
+CONFIG_I2C_XILINX=m
 CONFIG_I2C_RCAR=y
 CONFIG_I2C_CROS_EC_TUNNEL=y
 CONFIG_SPI=y
@@ -599,6 +610,8 @@ CONFIG_SPI_STM32_OSPI=m
 CONFIG_SPI_SUN6I=y
 CONFIG_SPI_TEGRA210_QUAD=m
 CONFIG_SPI_TEGRA114=m
+CONFIG_SPI_XILINX=m
+CONFIG_SPI_ZYNQMP_GQSPI=m
 CONFIG_SPI_SPIDEV=m
 CONFIG_SPMI=y
 CONFIG_SPMI_MTK_PMIF=m
@@ -630,6 +643,7 @@ CONFIG_PINCTRL_IPQ5424=y
 CONFIG_PINCTRL_IPQ8074=y
 CONFIG_PINCTRL_IPQ6018=y
 CONFIG_PINCTRL_IPQ9574=y
+CONFIG_PINCTRL_KAANAPALI=y
 CONFIG_PINCTRL_MSM8916=y
 CONFIG_PINCTRL_MSM8953=y
 CONFIG_PINCTRL_MSM8976=y
@@ -691,6 +705,8 @@ CONFIG_GPIO_WCD934X=m
 CONFIG_GPIO_VF610=y
 CONFIG_GPIO_XGENE=y
 CONFIG_GPIO_XGENE_SB=y
+CONFIG_GPIO_XILINX=m
+CONFIG_GPIO_ZYNQ=m
 CONFIG_GPIO_MAX732X=y
 CONFIG_GPIO_PCA953X=y
 CONFIG_GPIO_PCA953X_IRQ=y
@@ -762,6 +778,8 @@ CONFIG_QCOM_LMH=m
 CONFIG_UNIPHIER_THERMAL=y
 CONFIG_KHADAS_MCU_FAN_THERMAL=m
 CONFIG_WATCHDOG=y
+CONFIG_XILINX_WATCHDOG=m
+CONFIG_XILINX_WINDOW_WATCHDOG=m
 CONFIG_SL28CPLD_WATCHDOG=m
 CONFIG_ARM_SP805_WATCHDOG=y
 CONFIG_ARM_SBSA_WATCHDOG=y
@@ -1007,6 +1025,8 @@ CONFIG_DRM_LIMA=m
 CONFIG_DRM_PANFROST=m
 CONFIG_DRM_PANTHOR=m
 CONFIG_DRM_TIDSS=m
+CONFIG_DRM_ZYNQMP_DPSUB=m
+CONFIG_DRM_ZYNQMP_DPSUB_AUDIO=y
 CONFIG_DRM_POWERVR=m
 CONFIG_FB=y
 CONFIG_FB_EFI=y
@@ -1093,6 +1113,9 @@ CONFIG_SND_SOC_TEGRA210_MIXER=m
 CONFIG_SND_SOC_TEGRA_AUDIO_GRAPH_CARD=m
 CONFIG_SND_SOC_DAVINCI_MCASP=m
 CONFIG_SND_SOC_J721E_EVM=m
+CONFIG_SND_SOC_XILINX_I2S=m
+CONFIG_SND_SOC_XILINX_AUDIO_FORMATTER=m
+CONFIG_SND_SOC_XILINX_SPDIF=m
 CONFIG_SND_SOC_AK4613=m
 CONFIG_SND_SOC_AK4619=m
 CONFIG_SND_SOC_DA7213=m
@@ -1177,6 +1200,7 @@ CONFIG_USB_SERIAL_OPTION=m
 CONFIG_USB_QCOM_EUD=m
 CONFIG_USB_HSIC_USB3503=y
 CONFIG_USB_ONBOARD_DEV=m
+CONFIG_USB_ONBOARD_DEV_USB5744=y
 CONFIG_NOP_USB_XCEIV=y
 CONFIG_USB_MXS_PHY=m
 CONFIG_USB_GADGET=y
@@ -1280,6 +1304,8 @@ CONFIG_LEDS_TRIGGER_PANIC=y
 CONFIG_EDAC=y
 CONFIG_EDAC_GHES=y
 CONFIG_EDAC_LAYERSCAPE=m
+CONFIG_EDAC_ZYNQMP=m
+CONFIG_EDAC_VERSAL=m
 CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_DS1307=m
 CONFIG_RTC_DRV_HYM8563=m
@@ -1300,6 +1326,7 @@ CONFIG_RTC_DRV_DS3232=y
 CONFIG_RTC_DRV_PCF2127=m
 CONFIG_RTC_DRV_DA9063=m
 CONFIG_RTC_DRV_EFI=y
+CONFIG_RTC_DRV_ZYNQMP=m
 CONFIG_RTC_DRV_CROS_EC=y
 CONFIG_RTC_DRV_FSL_FTM_ALARM=m
 CONFIG_RTC_DRV_S3C=y
@@ -1329,6 +1356,9 @@ CONFIG_PL330_DMA=y
 CONFIG_TEGRA186_GPC_DMA=y
 CONFIG_TEGRA20_APB_DMA=y
 CONFIG_TEGRA210_ADMA=m
+CONFIG_XILINX_DMA=m
+CONFIG_XILINX_ZYNQMP_DMA=m
+CONFIG_XILINX_ZYNQMP_DPDMA=m
 CONFIG_MTK_UART_APDMA=m
 CONFIG_QCOM_BAM_DMA=y
 CONFIG_QCOM_GPI_DMA=m
@@ -1397,6 +1427,8 @@ CONFIG_COMMON_CLK_MT8192_SCP_ADSP=y
 CONFIG_COMMON_CLK_MT8192_VDECSYS=y
 CONFIG_COMMON_CLK_MT8192_VENCSYS=y
 CONFIG_COMMON_CLK_QCOM=y
+CONFIG_CLK_KAANAPALI_GCC=y
+CONFIG_CLK_KAANAPALI_TCSRCC=m
 CONFIG_CLK_X1E80100_CAMCC=m
 CONFIG_CLK_X1E80100_DISPCC=m
 CONFIG_CLK_X1E80100_GCC=y
@@ -1501,6 +1533,8 @@ CONFIG_CLK_RCAR_USB2_CLOCK_SEL=y
 CONFIG_CLK_RENESAS_VBATTB=m
 CONFIG_EXYNOS_ACPM_CLK=m
 CONFIG_CLK_SOPHGO_CV1800=y
+CONFIG_XILINX_VCU=m
+CONFIG_COMMON_CLK_XLNX_CLKWZRD=m
 CONFIG_HWSPINLOCK=y
 CONFIG_HWSPINLOCK_OMAP=m
 CONFIG_HWSPINLOCK_QCOM=y
@@ -1613,6 +1647,8 @@ CONFIG_RZT2H_ADC=m
 CONFIG_SOPHGO_CV1800B_ADC=m
 CONFIG_TI_ADS1015=m
 CONFIG_TI_AM335X_ADC=m
+CONFIG_XILINX_XADC=m
+CONFIG_XILINX_AMS=m
 CONFIG_IIO_CROS_EC_SENSORS_CORE=m
 CONFIG_IIO_CROS_EC_SENSORS=m
 CONFIG_IIO_ST_LSM6DSX=m
@@ -1645,7 +1681,9 @@ CONFIG_PWM_TEGRA=m
 CONFIG_PWM_TIECAP=m
 CONFIG_PWM_TIEHRPWM=m
 CONFIG_PWM_VISCONTI=m
+CONFIG_PWM_XILINX=m
 CONFIG_SL28CPLD_INTC=y
+CONFIG_XILINX_INTC=y
 CONFIG_QCOM_PDC=y
 CONFIG_QCOM_MPM=y
 CONFIG_TI_SCI_INTR_IRQCHIP=y
@@ -1694,6 +1732,7 @@ CONFIG_PHY_R8A779F0_ETHERNET_SERDES=y
 CONFIG_PHY_RCAR_GEN3_PCIE=y
 CONFIG_PHY_RCAR_GEN3_USB2=y
 CONFIG_PHY_RCAR_GEN3_USB3=m
+CONFIG_PHY_RZ_G3E_USB3=m
 CONFIG_PHY_ROCKCHIP_EMMC=y
 CONFIG_PHY_ROCKCHIP_INNO_HDMI=m
 CONFIG_PHY_ROCKCHIP_INNO_USB2=y
@@ -1710,6 +1749,7 @@ CONFIG_PHY_UNIPHIER_USB3=y
 CONFIG_PHY_TEGRA_XUSB=y
 CONFIG_PHY_AM654_SERDES=m
 CONFIG_PHY_J721E_WIZ=m
+CONFIG_PHY_XILINX_ZYNQMP=m
 CONFIG_ARM_CCI_PMU=m
 CONFIG_ARM_CCN=m
 CONFIG_ARM_CMN=m
@@ -1740,14 +1780,18 @@ CONFIG_NVMEM_SNVS_LPGPR=y
 CONFIG_NVMEM_SPMI_SDAM=m
 CONFIG_NVMEM_SUNXI_SID=y
 CONFIG_NVMEM_UNIPHIER_EFUSE=y
+CONFIG_NVMEM_ZYNQMP=m
 CONFIG_FPGA=y
 CONFIG_FPGA_MGR_ALTERA_CVP=m
 CONFIG_FPGA_MGR_STRATIX10_SOC=m
 CONFIG_FPGA_BRIDGE=m
 CONFIG_ALTERA_FREEZE_BRIDGE=m
+CONFIG_XILINX_PR_DECOUPLER=m
 CONFIG_FPGA_REGION=m
 CONFIG_OF_FPGA_REGION=m
 CONFIG_OF_OVERLAY=y
+CONFIG_FPGA_MGR_ZYNQMP_FPGA=m
+CONFIG_FPGA_MGR_VERSAL_FPGA=m
 CONFIG_TEE=y
 CONFIG_OPTEE=y
 CONFIG_MUX_GPIO=m
@@ -1762,6 +1806,7 @@ CONFIG_INTERCONNECT_IMX8MN=m
 CONFIG_INTERCONNECT_IMX8MQ=m
 CONFIG_INTERCONNECT_IMX8MP=y
 CONFIG_INTERCONNECT_QCOM=y
+CONFIG_INTERCONNECT_QCOM_KAANAPALI=y
 CONFIG_INTERCONNECT_QCOM_MSM8916=m
 CONFIG_INTERCONNECT_QCOM_MSM8953=y
 CONFIG_INTERCONNECT_QCOM_MSM8996=y
@@ -1842,6 +1887,9 @@ CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM=m
 CONFIG_CRYPTO_DEV_QCE=m
 CONFIG_CRYPTO_DEV_QCOM_RNG=m
 CONFIG_CRYPTO_DEV_TEGRA=m
+CONFIG_CRYPTO_DEV_XILINX_TRNG=m
+CONFIG_CRYPTO_DEV_ZYNQMP_AES=m
+CONFIG_CRYPTO_DEV_ZYNQMP_SHA3=m
 CONFIG_CRYPTO_DEV_CCREE=m
 CONFIG_CRYPTO_DEV_HISI_SEC2=m
 CONFIG_CRYPTO_DEV_HISI_ZIP=m
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index bdd276a6e540..da1c9ea8ea83 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -15,16 +15,6 @@ config CRYPTO_GHASH_ARM64_CE
 	  Architecture: arm64 using:
 	  - ARMv8 Crypto Extensions
 
-config CRYPTO_NHPOLY1305_NEON
-	tristate "Hash functions: NHPoly1305 (NEON)"
-	depends on KERNEL_MODE_NEON
-	select CRYPTO_NHPOLY1305
-	help
-	  NHPoly1305 hash function (Adiantum)
-
-	  Architecture: arm64 using:
-	  - NEON (Advanced SIMD) extensions
-
 config CRYPTO_SM3_NEON
 	tristate "Hash functions: SM3 (NEON)"
 	depends on KERNEL_MODE_NEON
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index 1e330aa08d3f..3ab4b58e5c4c 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -41,9 +41,6 @@ aes-ce-blk-y := aes-glue-ce.o aes-ce.o
 obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o
 aes-neon-blk-y := aes-glue-neon.o aes-neon.o
 
-obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o
-nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o
-
 obj-$(CONFIG_CRYPTO_AES_ARM64) += aes-arm64.o
 aes-arm64-y := aes-cipher-core.o aes-cipher-glue.o
 
diff --git a/arch/arm64/crypto/nh-neon-core.S b/arch/arm64/crypto/nh-neon-core.S
deleted file mode 100644
index 13eda08fda1e..000000000000
--- a/arch/arm64/crypto/nh-neon-core.S
+++ /dev/null
@@ -1,104 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * NH - ε-almost-universal hash function, ARM64 NEON accelerated version
- *
- * Copyright 2018 Google LLC
- *
- * Author: Eric Biggers <ebiggers@google.com>
- */
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-	KEY		.req	x0
-	MESSAGE		.req	x1
-	MESSAGE_LEN	.req	x2
-	HASH		.req	x3
-
-	PASS0_SUMS	.req	v0
-	PASS1_SUMS	.req	v1
-	PASS2_SUMS	.req	v2
-	PASS3_SUMS	.req	v3
-	K0		.req	v4
-	K1		.req	v5
-	K2		.req	v6
-	K3		.req	v7
-	T0		.req	v8
-	T1		.req	v9
-	T2		.req	v10
-	T3		.req	v11
-	T4		.req	v12
-	T5		.req	v13
-	T6		.req	v14
-	T7		.req	v15
-
-.macro _nh_stride	k0, k1, k2, k3
-
-	// Load next message stride
-	ld1		{T3.16b}, [MESSAGE], #16
-
-	// Load next key stride
-	ld1		{\k3\().4s}, [KEY], #16
-
-	// Add message words to key words
-	add		T0.4s, T3.4s, \k0\().4s
-	add		T1.4s, T3.4s, \k1\().4s
-	add		T2.4s, T3.4s, \k2\().4s
-	add		T3.4s, T3.4s, \k3\().4s
-
-	// Multiply 32x32 => 64 and accumulate
-	mov		T4.d[0], T0.d[1]
-	mov		T5.d[0], T1.d[1]
-	mov		T6.d[0], T2.d[1]
-	mov		T7.d[0], T3.d[1]
-	umlal		PASS0_SUMS.2d, T0.2s, T4.2s
-	umlal		PASS1_SUMS.2d, T1.2s, T5.2s
-	umlal		PASS2_SUMS.2d, T2.2s, T6.2s
-	umlal		PASS3_SUMS.2d, T3.2s, T7.2s
-.endm
-
-/*
- * void nh_neon(const u32 *key, const u8 *message, size_t message_len,
- *		__le64 hash[NH_NUM_PASSES])
- *
- * It's guaranteed that message_len % 16 == 0.
- */
-SYM_TYPED_FUNC_START(nh_neon)
-
-	ld1		{K0.4s,K1.4s}, [KEY], #32
-	  movi		PASS0_SUMS.2d, #0
-	  movi		PASS1_SUMS.2d, #0
-	ld1		{K2.4s}, [KEY], #16
-	  movi		PASS2_SUMS.2d, #0
-	  movi		PASS3_SUMS.2d, #0
-
-	subs		MESSAGE_LEN, MESSAGE_LEN, #64
-	blt		.Lloop4_done
-.Lloop4:
-	_nh_stride	K0, K1, K2, K3
-	_nh_stride	K1, K2, K3, K0
-	_nh_stride	K2, K3, K0, K1
-	_nh_stride	K3, K0, K1, K2
-	subs		MESSAGE_LEN, MESSAGE_LEN, #64
-	bge		.Lloop4
-
-.Lloop4_done:
-	ands		MESSAGE_LEN, MESSAGE_LEN, #63
-	beq		.Ldone
-	_nh_stride	K0, K1, K2, K3
-
-	subs		MESSAGE_LEN, MESSAGE_LEN, #16
-	beq		.Ldone
-	_nh_stride	K1, K2, K3, K0
-
-	subs		MESSAGE_LEN, MESSAGE_LEN, #16
-	beq		.Ldone
-	_nh_stride	K2, K3, K0, K1
-
-.Ldone:
-	// Sum the accumulators for each pass, then store the sums to 'hash'
-	addp		T0.2d, PASS0_SUMS.2d, PASS1_SUMS.2d
-	addp		T1.2d, PASS2_SUMS.2d, PASS3_SUMS.2d
-	st1		{T0.16b,T1.16b}, [HASH]
-	ret
-SYM_FUNC_END(nh_neon)
diff --git a/arch/arm64/crypto/nhpoly1305-neon-glue.c b/arch/arm64/crypto/nhpoly1305-neon-glue.c
deleted file mode 100644
index 013de6ac569a..000000000000
--- a/arch/arm64/crypto/nhpoly1305-neon-glue.c
+++ /dev/null
@@ -1,79 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
- * (ARM64 NEON accelerated version)
- *
- * Copyright 2018 Google LLC
- */
-
-#include <asm/neon.h>
-#include <asm/simd.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/nhpoly1305.h>
-#include <linux/module.h>
-
-asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len,
-			__le64 hash[NH_NUM_PASSES]);
-
-static int nhpoly1305_neon_update(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen)
-{
-	if (srclen < 64 || !crypto_simd_usable())
-		return crypto_nhpoly1305_update(desc, src, srclen);
-
-	do {
-		unsigned int n = min_t(unsigned int, srclen, SZ_4K);
-
-		scoped_ksimd()
-			crypto_nhpoly1305_update_helper(desc, src, n, nh_neon);
-		src += n;
-		srclen -= n;
-	} while (srclen);
-	return 0;
-}
-
-static int nhpoly1305_neon_digest(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen, u8 *out)
-{
-	return crypto_nhpoly1305_init(desc) ?:
-	       nhpoly1305_neon_update(desc, src, srclen) ?:
-	       crypto_nhpoly1305_final(desc, out);
-}
-
-static struct shash_alg nhpoly1305_alg = {
-	.base.cra_name		= "nhpoly1305",
-	.base.cra_driver_name	= "nhpoly1305-neon",
-	.base.cra_priority	= 200,
-	.base.cra_ctxsize	= sizeof(struct nhpoly1305_key),
-	.base.cra_module	= THIS_MODULE,
-	.digestsize		= POLY1305_DIGEST_SIZE,
-	.init			= crypto_nhpoly1305_init,
-	.update			= nhpoly1305_neon_update,
-	.final			= crypto_nhpoly1305_final,
-	.digest			= nhpoly1305_neon_digest,
-	.setkey			= crypto_nhpoly1305_setkey,
-	.descsize		= sizeof(struct nhpoly1305_state),
-};
-
-static int __init nhpoly1305_mod_init(void)
-{
-	if (!cpu_have_named_feature(ASIMD))
-		return -ENODEV;
-
-	return crypto_register_shash(&nhpoly1305_alg);
-}
-
-static void __exit nhpoly1305_mod_exit(void)
-{
-	crypto_unregister_shash(&nhpoly1305_alg);
-}
-
-module_init(nhpoly1305_mod_init);
-module_exit(nhpoly1305_mod_exit);
-
-MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (NEON-accelerated)");
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
-MODULE_ALIAS_CRYPTO("nhpoly1305");
-MODULE_ALIAS_CRYPTO("nhpoly1305-neon");
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 64d5f1d9cce9..445e18e92221 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -62,61 +62,26 @@ static inline void emit_pte_barriers(void)
 
 static inline void queue_pte_barriers(void)
 {
-	unsigned long flags;
-
-	if (in_interrupt()) {
-		emit_pte_barriers();
-		return;
-	}
-
-	flags = read_thread_flags();
-
-	if (flags & BIT(TIF_LAZY_MMU)) {
+	if (is_lazy_mmu_mode_active()) {
 		/* Avoid the atomic op if already set. */
-		if (!(flags & BIT(TIF_LAZY_MMU_PENDING)))
+		if (!test_thread_flag(TIF_LAZY_MMU_PENDING))
 			set_thread_flag(TIF_LAZY_MMU_PENDING);
 	} else {
 		emit_pte_barriers();
 	}
 }
 
-#define  __HAVE_ARCH_ENTER_LAZY_MMU_MODE
-static inline void arch_enter_lazy_mmu_mode(void)
-{
-	/*
-	 * lazy_mmu_mode is not supposed to permit nesting. But in practice this
-	 * does happen with CONFIG_DEBUG_PAGEALLOC, where a page allocation
-	 * inside a lazy_mmu_mode section (such as zap_pte_range()) will change
-	 * permissions on the linear map with apply_to_page_range(), which
-	 * re-enters lazy_mmu_mode. So we tolerate nesting in our
-	 * implementation. The first call to arch_leave_lazy_mmu_mode() will
-	 * flush and clear the flag such that the remainder of the work in the
-	 * outer nest behaves as if outside of lazy mmu mode. This is safe and
-	 * keeps tracking simple.
-	 */
-
-	if (in_interrupt())
-		return;
-
-	set_thread_flag(TIF_LAZY_MMU);
-}
+static inline void arch_enter_lazy_mmu_mode(void) {}
 
 static inline void arch_flush_lazy_mmu_mode(void)
 {
-	if (in_interrupt())
-		return;
-
 	if (test_and_clear_thread_flag(TIF_LAZY_MMU_PENDING))
 		emit_pte_barriers();
 }
 
 static inline void arch_leave_lazy_mmu_mode(void)
 {
-	if (in_interrupt())
-		return;
-
 	arch_flush_lazy_mmu_mode();
-	clear_thread_flag(TIF_LAZY_MMU);
 }
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
diff --git a/arch/arm64/include/asm/simd.h b/arch/arm64/include/asm/simd.h
index 0941f6f58a14..69ecbd69ca8c 100644
--- a/arch/arm64/include/asm/simd.h
+++ b/arch/arm64/include/asm/simd.h
@@ -48,6 +48,13 @@ DEFINE_LOCK_GUARD_1(ksimd,
 		    kernel_neon_begin(_T->lock),
 		    kernel_neon_end(_T->lock))
 
-#define scoped_ksimd()	scoped_guard(ksimd, &(struct user_fpsimd_state){})
+#define __scoped_ksimd(_label)					\
+	for (struct user_fpsimd_state __uninitialized __st;	\
+	     true; ({ goto _label; }))				\
+		if (0) {					\
+_label:			break;					\
+		} else scoped_guard(ksimd, &__st)
+
+#define scoped_ksimd()	__scoped_ksimd(__UNIQUE_ID(label))
 
 #endif
diff --git a/arch/arm64/include/asm/syscall.h b/arch/arm64/include/asm/syscall.h
index 712daa90e643..5e4c7fc44f73 100644
--- a/arch/arm64/include/asm/syscall.h
+++ b/arch/arm64/include/asm/syscall.h
@@ -77,23 +77,29 @@ static inline void syscall_set_nr(struct task_struct *task,
 	}
 }
 
-#define SYSCALL_MAX_ARGS 6
-
 static inline void syscall_get_arguments(struct task_struct *task,
 					 struct pt_regs *regs,
 					 unsigned long *args)
 {
 	args[0] = regs->orig_x0;
-	args++;
-
-	memcpy(args, &regs->regs[1], 5 * sizeof(args[0]));
+	args[1] = regs->regs[1];
+	args[2] = regs->regs[2];
+	args[3] = regs->regs[3];
+	args[4] = regs->regs[4];
+	args[5] = regs->regs[5];
 }
 
 static inline void syscall_set_arguments(struct task_struct *task,
 					 struct pt_regs *regs,
 					 const unsigned long *args)
 {
-	memcpy(&regs->regs[0], args, 6 * sizeof(args[0]));
+	regs->regs[0] = args[0];
+	regs->regs[1] = args[1];
+	regs->regs[2] = args[2];
+	regs->regs[3] = args[3];
+	regs->regs[4] = args[4];
+	regs->regs[5] = args[5];
+
 	/*
 	 * Also copy the first argument into orig_x0
 	 * so that syscall_get_arguments() would return it
diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index a803b887b0b4..e7cd017b07c8 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -84,8 +84,7 @@ void arch_setup_new_exec(void);
 #define TIF_SME_VL_INHERIT	28	/* Inherit SME vl_onexec across exec */
 #define TIF_KERNEL_FPSTATE	29	/* Task is in a kernel mode FPSIMD section */
 #define TIF_TSC_SIGSEGV		30	/* SIGSEGV on counter-timer access */
-#define TIF_LAZY_MMU		31	/* Task in lazy mmu mode */
-#define TIF_LAZY_MMU_PENDING	32	/* Ops pending for lazy mmu mode exit */
+#define TIF_LAZY_MMU_PENDING	31	/* Ops pending for lazy mmu mode exit */
 
 #define _TIF_SIGPENDING		(1 << TIF_SIGPENDING)
 #define _TIF_NEED_RESCHED	(1 << TIF_NEED_RESCHED)
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index c154f72634e0..9de1d8a604cb 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -180,13 +180,6 @@ static inline void set_sve_default_vl(int val)
 	set_default_vl(ARM64_VEC_SVE, val);
 }
 
-static u8 *efi_sve_state;
-
-#else /* ! CONFIG_ARM64_SVE */
-
-/* Dummy declaration for code that will be optimised out: */
-extern u8 *efi_sve_state;
-
 #endif /* ! CONFIG_ARM64_SVE */
 
 #ifdef CONFIG_ARM64_SME
@@ -1095,36 +1088,6 @@ int vec_verify_vq_map(enum vec_type type)
 	return 0;
 }
 
-static void __init sve_efi_setup(void)
-{
-	int max_vl = 0;
-	int i;
-
-	if (!IS_ENABLED(CONFIG_EFI))
-		return;
-
-	for (i = 0; i < ARRAY_SIZE(vl_info); i++)
-		max_vl = max(vl_info[i].max_vl, max_vl);
-
-	/*
-	 * alloc_percpu() warns and prints a backtrace if this goes wrong.
-	 * This is evidence of a crippled system and we are returning void,
-	 * so no attempt is made to handle this situation here.
-	 */
-	if (!sve_vl_valid(max_vl))
-		goto fail;
-
-	efi_sve_state = kmalloc(SVE_SIG_REGS_SIZE(sve_vq_from_vl(max_vl)),
-				GFP_KERNEL);
-	if (!efi_sve_state)
-		goto fail;
-
-	return;
-
-fail:
-	panic("Cannot allocate memory for EFI SVE save/restore");
-}
-
 void cpu_enable_sve(const struct arm64_cpu_capabilities *__always_unused p)
 {
 	write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_ZEN_EL1EN, CPACR_EL1);
@@ -1185,8 +1148,6 @@ void __init sve_setup(void)
 	if (sve_max_virtualisable_vl() < sve_max_vl())
 		pr_warn("%s: unvirtualisable vector lengths present\n",
 			info->name);
-
-	sve_efi_setup();
 }
 
 /*
@@ -1947,9 +1908,6 @@ EXPORT_SYMBOL_GPL(kernel_neon_end);
 #ifdef CONFIG_EFI
 
 static struct user_fpsimd_state efi_fpsimd_state;
-static bool efi_fpsimd_state_used;
-static bool efi_sve_state_used;
-static bool efi_sm_state;
 
 /*
  * EFI runtime services support functions
@@ -1976,43 +1934,26 @@ void __efi_fpsimd_begin(void)
 	if (may_use_simd()) {
 		kernel_neon_begin(&efi_fpsimd_state);
 	} else {
-		WARN_ON(preemptible());
-
 		/*
-		 * If !efi_sve_state, SVE can't be in use yet and doesn't need
-		 * preserving:
+		 * We are running in hardirq or NMI context, and the only
+		 * legitimate case where this might happen is when EFI pstore
+		 * is attempting to record the system's dying gasps into EFI
+		 * variables. This could be due to an oops, a panic or a call
+		 * to emergency_restart(), and in none of those cases, we can
+		 * expect the current task to ever return to user space again,
+		 * or for the kernel to resume any normal execution, for that
+		 * matter (an oops in hardirq context triggers a panic too).
+		 *
+		 * Therefore, there is no point in attempting to preserve any
+		 * SVE/SME state here. On the off chance that we might have
+		 * ended up here for a different reason inadvertently, kill the
+		 * task and preserve/restore the base FP/SIMD state, which
+		 * might belong to kernel mode FP/SIMD.
 		 */
-		if (system_supports_sve() && efi_sve_state != NULL) {
-			bool ffr = true;
-			u64 svcr;
-
-			efi_sve_state_used = true;
-
-			if (system_supports_sme()) {
-				svcr = read_sysreg_s(SYS_SVCR);
-
-				efi_sm_state = svcr & SVCR_SM_MASK;
-
-				/*
-				 * Unless we have FA64 FFR does not
-				 * exist in streaming mode.
-				 */
-				if (!system_supports_fa64())
-					ffr = !(svcr & SVCR_SM_MASK);
-			}
-
-			sve_save_state(efi_sve_state + sve_ffr_offset(sve_max_vl()),
-				       &efi_fpsimd_state.fpsr, ffr);
-
-			if (system_supports_sme())
-				sysreg_clear_set_s(SYS_SVCR,
-						   SVCR_SM_MASK, 0);
-
-		} else {
-			fpsimd_save_state(&efi_fpsimd_state);
-		}
-
-		efi_fpsimd_state_used = true;
+		pr_warn_ratelimited("Calling EFI runtime from %s context\n",
+				    in_nmi() ? "NMI" : "hardirq");
+		force_signal_inject(SIGKILL, SI_KERNEL, 0, 0);
+		fpsimd_save_state(&efi_fpsimd_state);
 	}
 }
 
@@ -2024,41 +1965,10 @@ void __efi_fpsimd_end(void)
 	if (!system_supports_fpsimd())
 		return;
 
-	if (!efi_fpsimd_state_used) {
+	if (may_use_simd()) {
 		kernel_neon_end(&efi_fpsimd_state);
 	} else {
-		if (system_supports_sve() && efi_sve_state_used) {
-			bool ffr = true;
-
-			/*
-			 * Restore streaming mode; EFI calls are
-			 * normal function calls so should not return in
-			 * streaming mode.
-			 */
-			if (system_supports_sme()) {
-				if (efi_sm_state) {
-					sysreg_clear_set_s(SYS_SVCR,
-							   0,
-							   SVCR_SM_MASK);
-
-					/*
-					 * Unless we have FA64 FFR does not
-					 * exist in streaming mode.
-					 */
-					if (!system_supports_fa64())
-						ffr = false;
-				}
-			}
-
-			sve_load_state(efi_sve_state + sve_ffr_offset(sve_max_vl()),
-				       &efi_fpsimd_state.fpsr, ffr);
-
-			efi_sve_state_used = false;
-		} else {
-			fpsimd_load_state(&efi_fpsimd_state);
-		}
-
-		efi_fpsimd_state_used = false;
+		fpsimd_load_state(&efi_fpsimd_state);
 	}
 }
 
diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index 85bc629270bd..d7b0d12b1015 100644
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -38,7 +38,7 @@ PROVIDE(__efistub__end			= _end);
 PROVIDE(__efistub___inittext_end       	= __inittext_end);
 PROVIDE(__efistub__edata		= _edata);
 #if defined(CONFIG_EFI_EARLYCON) || defined(CONFIG_SYSFB)
-PROVIDE(__efistub_screen_info		= screen_info);
+PROVIDE(__efistub_sysfb_primary_display	= sysfb_primary_display);
 #endif
 PROVIDE(__efistub__ctype		= _ctype);
 
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index fba7ca102a8c..489554931231 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -292,6 +292,7 @@ static void flush_gcs(void)
 	current->thread.gcs_base = 0;
 	current->thread.gcs_size = 0;
 	current->thread.gcs_el0_mode = 0;
+	current->thread.gcs_el0_locked = 0;
 	write_sysreg_s(GCSCRE0_EL1_nTR, SYS_GCSCRE0_EL1);
 	write_sysreg_s(0, SYS_GCSPR_EL0);
 }
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 9ae7ce00a7ef..a6a00accf4f9 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -767,18 +767,6 @@ static inline bool force_pte_mapping(void)
 	return rodata_full || arm64_kfence_can_set_direct_map() || is_realm_world();
 }
 
-static inline bool split_leaf_mapping_possible(void)
-{
-	/*
-	 * !BBML2_NOABORT systems should never run into scenarios where we would
-	 * have to split. So exit early and let calling code detect it and raise
-	 * a warning.
-	 */
-	if (!system_supports_bbml2_noabort())
-		return false;
-	return !force_pte_mapping();
-}
-
 static DEFINE_MUTEX(pgtable_split_lock);
 
 int split_kernel_leaf_mapping(unsigned long start, unsigned long end)
@@ -786,11 +774,22 @@ int split_kernel_leaf_mapping(unsigned long start, unsigned long end)
 	int ret;
 
 	/*
-	 * Exit early if the region is within a pte-mapped area or if we can't
-	 * split. For the latter case, the permission change code will raise a
-	 * warning if not already pte-mapped.
+	 * !BBML2_NOABORT systems should not be trying to change permissions on
+	 * anything that is not pte-mapped in the first place. Just return early
+	 * and let the permission change code raise a warning if not already
+	 * pte-mapped.
 	 */
-	if (!split_leaf_mapping_possible() || is_kfence_address((void *)start))
+	if (!system_supports_bbml2_noabort())
+		return 0;
+
+	/*
+	 * If the region is within a pte-mapped area, there is no need to try to
+	 * split. Additionally, CONFIG_DEBUG_PAGEALLOC and CONFIG_KFENCE may
+	 * change permissions from atomic context so for those cases (which are
+	 * always pte-mapped), we must not go any further because taking the
+	 * mutex below may sleep.
+	 */
+	if (force_pte_mapping() || is_kfence_address((void *)start))
 		return 0;
 
 	/*
@@ -801,7 +800,7 @@ int split_kernel_leaf_mapping(unsigned long start, unsigned long end)
 		return -EINVAL;
 
 	mutex_lock(&pgtable_split_lock);
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 
 	/*
 	 * The split_kernel_leaf_mapping_locked() may sleep, it is not a
@@ -823,7 +822,7 @@ int split_kernel_leaf_mapping(unsigned long start, unsigned long end)
 			ret = split_kernel_leaf_mapping_locked(end);
 	}
 
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 	mutex_unlock(&pgtable_split_lock);
 	return ret;
 }
@@ -884,10 +883,10 @@ static int range_split_to_ptes(unsigned long start, unsigned long end, gfp_t gfp
 {
 	int ret;
 
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 	ret = walk_kernel_page_table_range_lockless(start, end,
 					&split_to_ptes_ops, NULL, &gfp);
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 
 	return ret;
 }
@@ -1089,7 +1088,7 @@ bool arch_kfence_init_pool(void)
 	int ret;
 
 	/* Exit early if we know the linear map is already pte-mapped. */
-	if (!split_leaf_mapping_possible())
+	if (force_pte_mapping())
 		return true;
 
 	/* Kfence pool is already pte-mapped for the early init case. */
diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c
index f0e784b963e6..508986608b49 100644
--- a/arch/arm64/mm/pageattr.c
+++ b/arch/arm64/mm/pageattr.c
@@ -110,7 +110,7 @@ static int update_range_prot(unsigned long start, unsigned long size,
 	if (WARN_ON_ONCE(ret))
 		return ret;
 
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 
 	/*
 	 * The caller must ensure that the range we are operating on does not
@@ -119,7 +119,7 @@ static int update_range_prot(unsigned long start, unsigned long size,
 	 */
 	ret = walk_kernel_page_table_range_lockless(start, start + size,
 						    &pageattr_ops, NULL, &data);
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 
 	return ret;
 }
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 74dd29816f36..1d657bd3ce65 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -1004,7 +1004,7 @@ static void __maybe_unused build_bhb_mitigation(struct jit_ctx *ctx)
 	    arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE)
 		return;
 
-	if (capable(CAP_SYS_ADMIN))
+	if (ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN))
 		return;
 
 	if (supports_clearbhb(SCOPE_SYSTEM)) {
@@ -2951,7 +2951,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
 	u64 plt_target = 0ULL;
 	bool poking_bpf_entry;
 
-	if (!__bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf))
+	if (!bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf))
 		/* Only poking bpf text is supported. Since kernel function
 		 * entry is set up by ftrace, we reply on ftrace to poke kernel
 		 * functions.
diff --git a/arch/loongarch/kernel/efi.c b/arch/loongarch/kernel/efi.c
index 52c21c895318..69dd83f8082f 100644
--- a/arch/loongarch/kernel/efi.c
+++ b/arch/loongarch/kernel/efi.c
@@ -18,7 +18,7 @@
 #include <linux/kobject.h>
 #include <linux/memblock.h>
 #include <linux/reboot.h>
-#include <linux/screen_info.h>
+#include <linux/sysfb.h>
 #include <linux/uaccess.h>
 
 #include <asm/early_ioremap.h>
@@ -72,30 +72,31 @@ bool efi_poweroff_required(void)
 		(acpi_gbl_reduced_hardware || acpi_no_s5);
 }
 
-unsigned long __initdata screen_info_table = EFI_INVALID_TABLE_ADDR;
+unsigned long __initdata primary_display_table = EFI_INVALID_TABLE_ADDR;
 
 #if defined(CONFIG_SYSFB) || defined(CONFIG_EFI_EARLYCON)
-struct screen_info screen_info __section(".data");
-EXPORT_SYMBOL_GPL(screen_info);
+struct sysfb_display_info sysfb_primary_display __section(".data");
+EXPORT_SYMBOL_GPL(sysfb_primary_display);
 #endif
 
-static void __init init_screen_info(void)
+static void __init init_primary_display(void)
 {
-	struct screen_info *si;
+	struct sysfb_display_info *dpy;
 
-	if (screen_info_table == EFI_INVALID_TABLE_ADDR)
+	if (primary_display_table == EFI_INVALID_TABLE_ADDR)
 		return;
 
-	si = early_memremap(screen_info_table, sizeof(*si));
-	if (!si) {
-		pr_err("Could not map screen_info config table\n");
+	dpy = early_memremap(primary_display_table, sizeof(*dpy));
+	if (!dpy) {
+		pr_err("Could not map primary_display config table\n");
 		return;
 	}
-	screen_info = *si;
-	memset(si, 0, sizeof(*si));
-	early_memunmap(si, sizeof(*si));
+	sysfb_primary_display = *dpy;
+	memset(dpy, 0, sizeof(*dpy));
+	early_memunmap(dpy, sizeof(*dpy));
 
-	memblock_reserve(__screen_info_lfb_base(&screen_info), screen_info.lfb_size);
+	memblock_reserve(__screen_info_lfb_base(&sysfb_primary_display.screen),
+			 sysfb_primary_display.screen.lfb_size);
 }
 
 void __init efi_init(void)
@@ -129,7 +130,7 @@ void __init efi_init(void)
 	set_bit(EFI_CONFIG_TABLES, &efi.flags);
 
 	if (IS_ENABLED(CONFIG_EFI_EARLYCON) || IS_ENABLED(CONFIG_SYSFB))
-		init_screen_info();
+		init_primary_display();
 
 	if (boot_memmap == EFI_INVALID_TABLE_ADDR)
 		return;
diff --git a/arch/loongarch/kernel/image-vars.h b/arch/loongarch/kernel/image-vars.h
index 41ddcf56d21c..e557ebd46c2b 100644
--- a/arch/loongarch/kernel/image-vars.h
+++ b/arch/loongarch/kernel/image-vars.h
@@ -12,7 +12,7 @@ __efistub_kernel_entry		= kernel_entry;
 __efistub_kernel_asize		= kernel_asize;
 __efistub_kernel_fsize		= kernel_fsize;
 #if defined(CONFIG_EFI_EARLYCON) || defined(CONFIG_SYSFB)
-__efistub_screen_info		= screen_info;
+__efistub_sysfb_primary_display	= sysfb_primary_display;
 #endif
 
 #endif
diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig
index bfc1ee7c8158..1439abb69f73 100644
--- a/arch/m68k/configs/amiga_defconfig
+++ b/arch/m68k/configs/amiga_defconfig
@@ -618,7 +618,6 @@ CONFIG_ASYNC_RAID6_TEST=m
 CONFIG_TEST_HEXDUMP=m
 CONFIG_TEST_KSTRTOX=m
 CONFIG_TEST_BITMAP=m
-CONFIG_TEST_UUID=m
 CONFIG_TEST_XARRAY=m
 CONFIG_TEST_MAPLE_TREE=m
 CONFIG_TEST_RHASHTABLE=m
diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig
index d9d1f3c4c70d..6a4e71866f60 100644
--- a/arch/m68k/configs/apollo_defconfig
+++ b/arch/m68k/configs/apollo_defconfig
@@ -575,7 +575,6 @@ CONFIG_ASYNC_RAID6_TEST=m
 CONFIG_TEST_HEXDUMP=m
 CONFIG_TEST_KSTRTOX=m
 CONFIG_TEST_BITMAP=m
-CONFIG_TEST_UUID=m
 CONFIG_TEST_XARRAY=m
 CONFIG_TEST_MAPLE_TREE=m
 CONFIG_TEST_RHASHTABLE=m
diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig
index 523205adccc8..46ad7d57b4fc 100644
--- a/arch/m68k/configs/atari_defconfig
+++ b/arch/m68k/configs/atari_defconfig
@@ -595,7 +595,6 @@ CONFIG_ASYNC_RAID6_TEST=m
 CONFIG_TEST_HEXDUMP=m
 CONFIG_TEST_KSTRTOX=m
 CONFIG_TEST_BITMAP=m
-CONFIG_TEST_UUID=m
 CONFIG_TEST_XARRAY=m
 CONFIG_TEST_MAPLE_TREE=m
 CONFIG_TEST_RHASHTABLE=m
diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig
index 7b0a4ef0b010..867bfa13a44c 100644
--- a/arch/m68k/configs/bvme6000_defconfig
+++ b/arch/m68k/configs/bvme6000_defconfig
@@ -567,7 +567,6 @@ CONFIG_ASYNC_RAID6_TEST=m
 CONFIG_TEST_HEXDUMP=m
 CONFIG_TEST_KSTRTOX=m
 CONFIG_TEST_BITMAP=m
-CONFIG_TEST_UUID=m
 CONFIG_TEST_XARRAY=m
 CONFIG_TEST_MAPLE_TREE=m
 CONFIG_TEST_RHASHTABLE=m
diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig
index 089c5c394c62..5dfe602cafd4 100644
--- a/arch/m68k/configs/hp300_defconfig
+++ b/arch/m68k/configs/hp300_defconfig
@@ -577,7 +577,6 @@ CONFIG_ASYNC_RAID6_TEST=m
 CONFIG_TEST_HEXDUMP=m
 CONFIG_TEST_KSTRTOX=m
 CONFIG_TEST_BITMAP=m
-CONFIG_TEST_UUID=m
 CONFIG_TEST_XARRAY=m
 CONFIG_TEST_MAPLE_TREE=m
 CONFIG_TEST_RHASHTABLE=m
diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig
index 5f2484c36733..f5d30310a349 100644
--- a/arch/m68k/configs/mac_defconfig
+++ b/arch/m68k/configs/mac_defconfig
@@ -594,7 +594,6 @@ CONFIG_ASYNC_RAID6_TEST=m
 CONFIG_TEST_HEXDUMP=m
 CONFIG_TEST_KSTRTOX=m
 CONFIG_TEST_BITMAP=m
-CONFIG_TEST_UUID=m
 CONFIG_TEST_XARRAY=m
 CONFIG_TEST_MAPLE_TREE=m
 CONFIG_TEST_RHASHTABLE=m
diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig
index 74f0a1f6d871..fe54e9222cc0 100644
--- a/arch/m68k/configs/multi_defconfig
+++ b/arch/m68k/configs/multi_defconfig
@@ -681,7 +681,6 @@ CONFIG_ASYNC_RAID6_TEST=m
 CONFIG_TEST_HEXDUMP=m
 CONFIG_TEST_KSTRTOX=m
 CONFIG_TEST_BITMAP=m
-CONFIG_TEST_UUID=m
 CONFIG_TEST_XARRAY=m
 CONFIG_TEST_MAPLE_TREE=m
 CONFIG_TEST_RHASHTABLE=m
diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig
index 4bee18c820e4..4ff2ff0993ad 100644
--- a/arch/m68k/configs/mvme147_defconfig
+++ b/arch/m68k/configs/mvme147_defconfig
@@ -567,7 +567,6 @@ CONFIG_ASYNC_RAID6_TEST=m
 CONFIG_TEST_HEXDUMP=m
 CONFIG_TEST_KSTRTOX=m
 CONFIG_TEST_BITMAP=m
-CONFIG_TEST_UUID=m
 CONFIG_TEST_XARRAY=m
 CONFIG_TEST_MAPLE_TREE=m
 CONFIG_TEST_RHASHTABLE=m
diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig
index 322c17e55c9a..6bb4738a65aa 100644
--- a/arch/m68k/configs/mvme16x_defconfig
+++ b/arch/m68k/configs/mvme16x_defconfig
@@ -568,7 +568,6 @@ CONFIG_ASYNC_RAID6_TEST=m
 CONFIG_TEST_HEXDUMP=m
 CONFIG_TEST_KSTRTOX=m
 CONFIG_TEST_BITMAP=m
-CONFIG_TEST_UUID=m
 CONFIG_TEST_XARRAY=m
 CONFIG_TEST_MAPLE_TREE=m
 CONFIG_TEST_RHASHTABLE=m
diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig
index 82f9baab8fea..14166c8fe234 100644
--- a/arch/m68k/configs/q40_defconfig
+++ b/arch/m68k/configs/q40_defconfig
@@ -584,7 +584,6 @@ CONFIG_ASYNC_RAID6_TEST=m
 CONFIG_TEST_HEXDUMP=m
 CONFIG_TEST_KSTRTOX=m
 CONFIG_TEST_BITMAP=m
-CONFIG_TEST_UUID=m
 CONFIG_TEST_XARRAY=m
 CONFIG_TEST_MAPLE_TREE=m
 CONFIG_TEST_RHASHTABLE=m
diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig
index f94ad226cb5b..5db924e3caf7 100644
--- a/arch/m68k/configs/sun3_defconfig
+++ b/arch/m68k/configs/sun3_defconfig
@@ -564,7 +564,6 @@ CONFIG_ASYNC_RAID6_TEST=m
 CONFIG_TEST_HEXDUMP=m
 CONFIG_TEST_KSTRTOX=m
 CONFIG_TEST_BITMAP=m
-CONFIG_TEST_UUID=m
 CONFIG_TEST_XARRAY=m
 CONFIG_TEST_MAPLE_TREE=m
 CONFIG_TEST_RHASHTABLE=m
diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig
index a5ecfc505ab2..318c9fe42f46 100644
--- a/arch/m68k/configs/sun3x_defconfig
+++ b/arch/m68k/configs/sun3x_defconfig
@@ -565,7 +565,6 @@ CONFIG_ASYNC_RAID6_TEST=m
 CONFIG_TEST_HEXDUMP=m
 CONFIG_TEST_KSTRTOX=m
 CONFIG_TEST_BITMAP=m
-CONFIG_TEST_UUID=m
 CONFIG_TEST_XARRAY=m
 CONFIG_TEST_MAPLE_TREE=m
 CONFIG_TEST_RHASHTABLE=m
diff --git a/arch/mips/Kbuild b/arch/mips/Kbuild
index e901bf554483..6125d00cdcef 100644
--- a/arch/mips/Kbuild
+++ b/arch/mips/Kbuild
@@ -21,5 +21,7 @@ ifdef CONFIG_KVM
 obj-y += kvm/
 endif
 
+subdir-y += boot/dts
+
 # for cleaning
 subdir- += boot
diff --git a/arch/mips/alchemy/common/setup.c b/arch/mips/alchemy/common/setup.c
index c35b4f809d51..992134a8c23a 100644
--- a/arch/mips/alchemy/common/setup.c
+++ b/arch/mips/alchemy/common/setup.c
@@ -94,8 +94,7 @@ phys_addr_t fixup_bigphys_addr(phys_addr_t phys_addr, phys_addr_t size)
 	return phys_addr;
 }
 
-static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn,
-		unsigned long size)
+unsigned long io_remap_pfn_range_pfn(unsigned long pfn, unsigned long size)
 {
 	phys_addr_t phys_addr = fixup_bigphys_addr(pfn << PAGE_SHIFT, size);
 
diff --git a/arch/mips/include/asm/cevt-r4k.h b/arch/mips/include/asm/cevt-r4k.h
index 2e13a038d260..5229eb34f28a 100644
--- a/arch/mips/include/asm/cevt-r4k.h
+++ b/arch/mips/include/asm/cevt-r4k.h
@@ -23,7 +23,6 @@ void mips_event_handler(struct clock_event_device *dev);
 int c0_compare_int_usable(void);
 irqreturn_t c0_compare_interrupt(int, void *);
 
-extern struct irqaction c0_compare_irqaction;
 extern int cp0_timer_irq_installed;
 
 #endif /* __ASM_CEVT_R4K_H */
diff --git a/arch/mips/kernel/cevt-r4k.c b/arch/mips/kernel/cevt-r4k.c
index 5f6e9e2ebbdb..f58325f9bd2b 100644
--- a/arch/mips/kernel/cevt-r4k.c
+++ b/arch/mips/kernel/cevt-r4k.c
@@ -159,17 +159,6 @@ irqreturn_t c0_compare_interrupt(int irq, void *dev_id)
 	return IRQ_NONE;
 }
 
-struct irqaction c0_compare_irqaction = {
-	.handler = c0_compare_interrupt,
-	/*
-	 * IRQF_SHARED: The timer interrupt may be shared with other interrupts
-	 * such as perf counter and FDC interrupts.
-	 */
-	.flags = IRQF_PERCPU | IRQF_TIMER | IRQF_SHARED,
-	.name = "timer",
-};
-
-
 void mips_event_handler(struct clock_event_device *dev)
 {
 }
diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c
index 11b9b6b63e19..c540431ed332 100644
--- a/arch/mips/kernel/setup.c
+++ b/arch/mips/kernel/setup.c
@@ -13,6 +13,7 @@
 #include <linux/init.h>
 #include <linux/cpu.h>
 #include <linux/delay.h>
+#include <linux/hex.h>
 #include <linux/ioport.h>
 #include <linux/export.h>
 #include <linux/memblock.h>
diff --git a/arch/mips/rb532/devices.c b/arch/mips/rb532/devices.c
index b7f6f782d9a1..8ecb56be81ac 100644
--- a/arch/mips/rb532/devices.c
+++ b/arch/mips/rb532/devices.c
@@ -7,6 +7,7 @@
  */
 #include <linux/kernel.h>
 #include <linux/export.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/ctype.h>
 #include <linux/string.h>
diff --git a/arch/mips/sgi-ip22/ip22-gio.c b/arch/mips/sgi-ip22/ip22-gio.c
index 5893ea4e382c..19b70928d6dc 100644
--- a/arch/mips/sgi-ip22/ip22-gio.c
+++ b/arch/mips/sgi-ip22/ip22-gio.c
@@ -372,7 +372,8 @@ static void ip22_check_gio(int slotno, unsigned long addr, int irq)
 		gio_dev->resource.flags = IORESOURCE_MEM;
 		gio_dev->irq = irq;
 		dev_set_name(&gio_dev->dev, "%d", slotno);
-		gio_device_register(gio_dev);
+		if (gio_device_register(gio_dev))
+			gio_dev_put(gio_dev);
 	} else
 		printk(KERN_INFO "GIO: slot %d : Empty\n", slotno);
 }
diff --git a/arch/mips/sgi-ip27/ip27-timer.c b/arch/mips/sgi-ip27/ip27-timer.c
index 444b5e0e935f..5f4da05cb2c9 100644
--- a/arch/mips/sgi-ip27/ip27-timer.c
+++ b/arch/mips/sgi-ip27/ip27-timer.c
@@ -58,13 +58,6 @@ static irqreturn_t hub_rt_counter_handler(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-struct irqaction hub_rt_irqaction = {
-	.handler	= hub_rt_counter_handler,
-	.percpu_dev_id	= &hub_rt_clockevent,
-	.flags		= IRQF_PERCPU | IRQF_TIMER,
-	.name		= "hub-rt",
-};
-
 /*
  * This is a hack; we really need to figure these values out dynamically
  *
@@ -103,7 +96,8 @@ static void __init hub_rt_clock_event_global_init(void)
 {
 	irq_set_handler(IP27_RT_TIMER_IRQ, handle_percpu_devid_irq);
 	irq_set_percpu_devid(IP27_RT_TIMER_IRQ);
-	setup_percpu_irq(IP27_RT_TIMER_IRQ, &hub_rt_irqaction);
+	WARN_ON(request_percpu_irq(IP27_RT_TIMER_IRQ, hub_rt_counter_handler,
+				   "hub-rt", &hub_rt_clockevent));
 }
 
 static u64 hub_rt_read(struct clocksource *cs)
diff --git a/arch/mips/sgi-ip30/ip30-timer.c b/arch/mips/sgi-ip30/ip30-timer.c
index 7652f72f0daf..294e1f7e6d8a 100644
--- a/arch/mips/sgi-ip30/ip30-timer.c
+++ b/arch/mips/sgi-ip30/ip30-timer.c
@@ -52,11 +52,10 @@ void __init plat_time_init(void)
 	int irq = get_c0_compare_int();
 
 	cp0_timer_irq_installed = 1;
-	c0_compare_irqaction.percpu_dev_id = &mips_clockevent_device;
-	c0_compare_irqaction.flags &= ~IRQF_SHARED;
 	irq_set_handler(irq, handle_percpu_devid_irq);
 	irq_set_percpu_devid(irq);
-	setup_percpu_irq(irq, &c0_compare_irqaction);
+	WARN_ON(request_percpu_irq(irq, c0_compare_interrupt,
+				   "timer", &mips_clockevent_device));
 	enable_percpu_irq(irq, IRQ_TYPE_NONE);
 
 	ip30_heart_clocksource_init();
diff --git a/arch/openrisc/configs/or1klitex_defconfig b/arch/openrisc/configs/or1klitex_defconfig
index fb1eb9a68bd6..984b0e3b2768 100644
--- a/arch/openrisc/configs/or1klitex_defconfig
+++ b/arch/openrisc/configs/or1klitex_defconfig
@@ -52,5 +52,5 @@ CONFIG_LSM="lockdown,yama,loadpin,safesetid,integrity,bpf"
 CONFIG_PRINTK_TIME=y
 CONFIG_PANIC_ON_OOPS=y
 CONFIG_SOFTLOCKUP_DETECTOR=y
-CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y
+CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=1
 CONFIG_BUG_ON_DATA_CORRUPTION=y
diff --git a/arch/parisc/include/asm/special_insns.h b/arch/parisc/include/asm/special_insns.h
index 1013eeba31e5..fbccd76180b6 100644
--- a/arch/parisc/include/asm/special_insns.h
+++ b/arch/parisc/include/asm/special_insns.h
@@ -2,21 +2,6 @@
 #ifndef __PARISC_SPECIAL_INSNS_H
 #define __PARISC_SPECIAL_INSNS_H
 
-#define lpa(va)	({					\
-	unsigned long pa;				\
-	__asm__ __volatile__(				\
-		"copy %%r0,%0\n"			\
-		"8:\tlpa %%r0(%1),%0\n"			\
-		"9:\n"					\
-		ASM_EXCEPTIONTABLE_ENTRY(8b, 9b,	\
-				"or %%r0,%%r0,%%r0")	\
-		: "=&r" (pa)				\
-		: "r" (va)				\
-		: "memory"				\
-	);						\
-	pa;						\
-})
-
 #define lpa_user(va)	({				\
 	unsigned long pa;				\
 	__asm__ __volatile__(				\
diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig
index 2d92c11eea7e..684b3ea80f39 100644
--- a/arch/powerpc/configs/ppc64_defconfig
+++ b/arch/powerpc/configs/ppc64_defconfig
@@ -443,7 +443,6 @@ CONFIG_TEST_KSTRTOX=m
 CONFIG_TEST_PRINTF=m
 CONFIG_TEST_SCANF=m
 CONFIG_TEST_BITMAP=m
-CONFIG_TEST_UUID=m
 CONFIG_TEST_XARRAY=m
 CONFIG_TEST_MAPLE_TREE=m
 CONFIG_TEST_RHASHTABLE=m
diff --git a/arch/powerpc/configs/skiroot_defconfig b/arch/powerpc/configs/skiroot_defconfig
index 2b71a6dc399e..a4114fca5a39 100644
--- a/arch/powerpc/configs/skiroot_defconfig
+++ b/arch/powerpc/configs/skiroot_defconfig
@@ -289,7 +289,7 @@ CONFIG_SCHED_STACK_END_CHECK=y
 CONFIG_DEBUG_STACKOVERFLOW=y
 CONFIG_PANIC_ON_OOPS=y
 CONFIG_SOFTLOCKUP_DETECTOR=y
-CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y
+CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=1
 CONFIG_HARDLOCKUP_DETECTOR=y
 CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y
 CONFIG_WQ_WATCHDOG=y
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h
index 146287d9580f..6cc9abcd7b3d 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h
@@ -12,7 +12,6 @@
 #define PPC64_TLB_BATCH_NR 192
 
 struct ppc64_tlb_batch {
-	int			active;
 	unsigned long		index;
 	struct mm_struct	*mm;
 	real_pte_t		pte[PPC64_TLB_BATCH_NR];
@@ -24,12 +23,8 @@ DECLARE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch);
 
 extern void __flush_tlb_pending(struct ppc64_tlb_batch *batch);
 
-#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
-
 static inline void arch_enter_lazy_mmu_mode(void)
 {
-	struct ppc64_tlb_batch *batch;
-
 	if (radix_enabled())
 		return;
 	/*
@@ -37,11 +32,9 @@ static inline void arch_enter_lazy_mmu_mode(void)
 	 * operating on kernel page tables.
 	 */
 	preempt_disable();
-	batch = this_cpu_ptr(&ppc64_tlb_batch);
-	batch->active = 1;
 }
 
-static inline void arch_leave_lazy_mmu_mode(void)
+static inline void arch_flush_lazy_mmu_mode(void)
 {
 	struct ppc64_tlb_batch *batch;
 
@@ -51,11 +44,16 @@ static inline void arch_leave_lazy_mmu_mode(void)
 
 	if (batch->index)
 		__flush_tlb_pending(batch);
-	batch->active = 0;
-	preempt_enable();
 }
 
-#define arch_flush_lazy_mmu_mode()      do {} while (0)
+static inline void arch_leave_lazy_mmu_mode(void)
+{
+	if (radix_enabled())
+		return;
+
+	arch_flush_lazy_mmu_mode();
+	preempt_enable();
+}
 
 extern void hash__tlbiel_all(unsigned int action);
 
diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h
index b0f200aba2b3..97f35f9b1a96 100644
--- a/arch/powerpc/include/asm/thread_info.h
+++ b/arch/powerpc/include/asm/thread_info.h
@@ -154,12 +154,10 @@ void arch_setup_new_exec(void);
 /* Don't move TLF_NAPPING without adjusting the code in entry_32.S */
 #define TLF_NAPPING		0	/* idle thread enabled NAP mode */
 #define TLF_SLEEPING		1	/* suspend code enabled SLEEP mode */
-#define TLF_LAZY_MMU		3	/* tlb_batch is active */
 #define TLF_RUNLATCH		4	/* Is the runlatch enabled? */
 
 #define _TLF_NAPPING		(1 << TLF_NAPPING)
 #define _TLF_SLEEPING		(1 << TLF_SLEEPING)
-#define _TLF_LAZY_MMU		(1 << TLF_LAZY_MMU)
 #define _TLF_RUNLATCH		(1 << TLF_RUNLATCH)
 
 #ifndef __ASSEMBLER__
diff --git a/arch/powerpc/kernel/btext.c b/arch/powerpc/kernel/btext.c
index 7f63f1cdc6c3..ebfd4e413b38 100644
--- a/arch/powerpc/kernel/btext.c
+++ b/arch/powerpc/kernel/btext.c
@@ -6,6 +6,7 @@
  */
 #include <linux/kernel.h>
 #include <linux/string.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/export.h>
 #include <linux/font.h>
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index a45fe147868b..a15d0b619b1f 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -1281,9 +1281,6 @@ struct task_struct *__switch_to(struct task_struct *prev,
 {
 	struct thread_struct *new_thread, *old_thread;
 	struct task_struct *last;
-#ifdef CONFIG_PPC_64S_HASH_MMU
-	struct ppc64_tlb_batch *batch;
-#endif
 
 	new_thread = &new->thread;
 	old_thread = &current->thread;
@@ -1291,14 +1288,6 @@ struct task_struct *__switch_to(struct task_struct *prev,
 	WARN_ON(!irqs_disabled());
 
 #ifdef CONFIG_PPC_64S_HASH_MMU
-	batch = this_cpu_ptr(&ppc64_tlb_batch);
-	if (batch->active) {
-		current_thread_info()->local_flags |= _TLF_LAZY_MMU;
-		if (batch->index)
-			__flush_tlb_pending(batch);
-		batch->active = 0;
-	}
-
 	/*
 	 * On POWER9 the copy-paste buffer can only paste into
 	 * foreign real addresses, so unprivileged processes can not
@@ -1369,20 +1358,6 @@ struct task_struct *__switch_to(struct task_struct *prev,
 	 */
 
 #ifdef CONFIG_PPC_BOOK3S_64
-#ifdef CONFIG_PPC_64S_HASH_MMU
-	/*
-	 * This applies to a process that was context switched while inside
-	 * arch_enter_lazy_mmu_mode(), to re-activate the batch that was
-	 * deactivated above, before _switch(). This will never be the case
-	 * for new tasks.
-	 */
-	if (current_thread_info()->local_flags & _TLF_LAZY_MMU) {
-		current_thread_info()->local_flags &= ~_TLF_LAZY_MMU;
-		batch = this_cpu_ptr(&ppc64_tlb_batch);
-		batch->active = 1;
-	}
-#endif
-
 	/*
 	 * Math facilities are masked out of the child MSR in copy_thread.
 	 * A new task does not need to restore_math because it will
diff --git a/arch/powerpc/mm/book3s64/hash_tlb.c b/arch/powerpc/mm/book3s64/hash_tlb.c
index 21fcad97ae80..ec2941cec815 100644
--- a/arch/powerpc/mm/book3s64/hash_tlb.c
+++ b/arch/powerpc/mm/book3s64/hash_tlb.c
@@ -25,11 +25,12 @@
 #include <asm/tlb.h>
 #include <asm/bug.h>
 #include <asm/pte-walk.h>
-
+#include <kunit/visibility.h>
 
 #include <trace/events/thp.h>
 
 DEFINE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch);
+EXPORT_SYMBOL_IF_KUNIT(ppc64_tlb_batch);
 
 /*
  * A linux PTE was changed and the corresponding hash table entry
@@ -100,7 +101,7 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
 	 * Check if we have an active batch on this CPU. If not, just
 	 * flush now and return.
 	 */
-	if (!batch->active) {
+	if (!is_lazy_mmu_mode_active()) {
 		flush_hash_page(vpn, rpte, psize, ssize, mm_is_thread_local(mm));
 		put_cpu_var(ppc64_tlb_batch);
 		return;
@@ -154,6 +155,7 @@ void __flush_tlb_pending(struct ppc64_tlb_batch *batch)
 		flush_hash_range(i, local);
 	batch->index = 0;
 }
+EXPORT_SYMBOL_IF_KUNIT(__flush_tlb_pending);
 
 void hash__tlb_flush(struct mmu_gather *tlb)
 {
@@ -205,7 +207,7 @@ void __flush_hash_table_range(unsigned long start, unsigned long end)
 	 * way to do things but is fine for our needs here.
 	 */
 	local_irq_save(flags);
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 	for (; start < end; start += PAGE_SIZE) {
 		pte_t *ptep = find_init_mm_pte(start, &hugepage_shift);
 		unsigned long pte;
@@ -217,7 +219,7 @@ void __flush_hash_table_range(unsigned long start, unsigned long end)
 			continue;
 		hpte_need_flush(&init_mm, start, ptep, pte, hugepage_shift);
 	}
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 	local_irq_restore(flags);
 }
 
@@ -237,7 +239,7 @@ void flush_hash_table_pmd_range(struct mm_struct *mm, pmd_t *pmd, unsigned long
 	 * way to do things but is fine for our needs here.
 	 */
 	local_irq_save(flags);
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 	start_pte = pte_offset_map(pmd, addr);
 	if (!start_pte)
 		goto out;
@@ -249,6 +251,6 @@ void flush_hash_table_pmd_range(struct mm_struct *mm, pmd_t *pmd, unsigned long
 	}
 	pte_unmap(start_pte);
 out:
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 	local_irq_restore(flags);
 }
diff --git a/arch/powerpc/mm/book3s64/subpage_prot.c b/arch/powerpc/mm/book3s64/subpage_prot.c
index ec98e526167e..07c47673bba2 100644
--- a/arch/powerpc/mm/book3s64/subpage_prot.c
+++ b/arch/powerpc/mm/book3s64/subpage_prot.c
@@ -73,13 +73,13 @@ static void hpte_flush_range(struct mm_struct *mm, unsigned long addr,
 	pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
 	if (!pte)
 		return;
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 	for (; npages > 0; --npages) {
 		pte_update(mm, addr, pte, 0, 0, 0);
 		addr += PAGE_SIZE;
 		++pte;
 	}
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 	pte_unmap_unlock(pte - 1, ptl);
 }
 
diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index 5e976730b2f5..e199976e410a 100644
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c
@@ -1122,7 +1122,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
 	bpf_func = (unsigned long)ip;
 
 	/* We currently only support poking bpf programs */
-	if (!__bpf_address_lookup(bpf_func, &size, &offset, name)) {
+	if (!bpf_address_lookup(bpf_func, &size, &offset, name)) {
 		pr_err("%s (0x%lx): kernel/modules are not supported\n", __func__, bpf_func);
 		return -EOPNOTSUPP;
 	}
diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype
index 4c321a8ea896..f399917c17bd 100644
--- a/arch/powerpc/platforms/Kconfig.cputype
+++ b/arch/powerpc/platforms/Kconfig.cputype
@@ -93,6 +93,7 @@ config PPC_BOOK3S_64
 	select IRQ_WORK
 	select PPC_64S_HASH_MMU if !PPC_RADIX_MMU
 	select KASAN_VMALLOC if KASAN
+	select ARCH_HAS_LAZY_MMU_MODE
 
 config PPC_BOOK3E_64
 	bool "Embedded processors"
diff --git a/arch/riscv/Kbuild b/arch/riscv/Kbuild
index 126fb738fc44..3cf7f84eb287 100644
--- a/arch/riscv/Kbuild
+++ b/arch/riscv/Kbuild
@@ -7,5 +7,7 @@ obj-$(CONFIG_KVM) += kvm/
 
 obj-$(CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY) += purgatory/
 
+subdir-y += boot/dts
+
 # for cleaning
 subdir- += boot
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 6b39f37f769a..7e76b6316425 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -1162,6 +1162,28 @@ config RANDOMIZE_BASE
 
           If unsure, say N.
 
+config RISCV_USER_CFI
+	def_bool y
+	bool "riscv userspace control flow integrity"
+	depends on 64BIT && MMU && \
+		$(cc-option,-mabi=lp64 -march=rv64ima_zicfiss_zicfilp -fcf-protection=full)
+	depends on RISCV_ALTERNATIVE
+	select RISCV_SBI
+	select ARCH_HAS_USER_SHADOW_STACK
+	select ARCH_USES_HIGH_VMA_FLAGS
+	select DYNAMIC_SIGFRAME
+	help
+	  Provides CPU-assisted control flow integrity to userspace tasks.
+	  Control flow integrity is provided by implementing shadow stack for
+	  backward edge and indirect branch tracking for forward edge.
+	  Shadow stack protection is a hardware feature that detects function
+	  return address corruption. This helps mitigate ROP attacks.
+	  Indirect branch tracking enforces that all indirect branches must land
+	  on a landing pad instruction else CPU will fault. This mitigates against
+	  JOP / COP attacks. Applications must be enabled to use it, and old userspace
+	  does not get protection "for free".
+	  default y.
+
 endmenu # "Kernel features"
 
 menu "Boot options"
diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile
index 4c6de57f65ef..371da75a47f9 100644
--- a/arch/riscv/Makefile
+++ b/arch/riscv/Makefile
@@ -81,9 +81,12 @@ riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZACAS) := $(riscv-march-y)_zacas
 # Check if the toolchain supports Zabha
 riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZABHA) := $(riscv-march-y)_zabha
 
+KBUILD_BASE_ISA = -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)fd([^v_]*)v?/\1\2/')
+export KBUILD_BASE_ISA
+
 # Remove F,D,V from isa string for all. Keep extensions between "fd" and "v" by
 # matching non-v and non-multi-letter extensions out with the filter ([^v_]*)
-KBUILD_CFLAGS += -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)fd([^v_]*)v?/\1\2/')
+KBUILD_CFLAGS += $(KBUILD_BASE_ISA)
 
 KBUILD_AFLAGS += -march=$(riscv-march-y)
 
@@ -158,6 +161,8 @@ ifeq ($(CONFIG_MMU),y)
 prepare: vdso_prepare
 vdso_prepare: prepare0
 	$(Q)$(MAKE) $(build)=arch/riscv/kernel/vdso include/generated/vdso-offsets.h
+	$(if $(CONFIG_RISCV_USER_CFI),$(Q)$(MAKE) \
+		$(build)=arch/riscv/kernel/vdso_cfi include/generated/vdso-cfi-offsets.h)
 	$(if $(CONFIG_COMPAT),$(Q)$(MAKE) \
 		$(build)=arch/riscv/kernel/compat_vdso include/generated/compat_vdso-offsets.h)
 
@@ -165,6 +170,7 @@ endif
 endif
 
 vdso-install-y			+= arch/riscv/kernel/vdso/vdso.so.dbg
+vdso-install-$(CONFIG_RISCV_USER_CFI)	+= arch/riscv/kernel/vdso_cfi/vdso-cfi.so.dbg
 vdso-install-$(CONFIG_COMPAT)	+= arch/riscv/kernel/compat_vdso/compat_vdso.so.dbg
 
 BOOT_TARGETS := Image Image.gz Image.bz2 Image.lz4 Image.lzma Image.lzo Image.zst Image.xz loader loader.bin xipImage vmlinuz.efi
diff --git a/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire-fabric.dtsi b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire-fabric.dtsi
deleted file mode 100644
index e153eaf9b90e..000000000000
--- a/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire-fabric.dtsi
+++ /dev/null
@@ -1,82 +0,0 @@
-// SPDX-License-Identifier: (GPL-2.0 OR MIT)
-
-/ {
-	fabric_clk3: fabric-clk3 {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <50000000>;
-	};
-
-	fabric_clk1: fabric-clk1 {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <125000000>;
-	};
-
-	fabric-bus@40000000 {
-		compatible = "simple-bus";
-		#address-cells = <2>;
-		#size-cells = <2>;
-		ranges = <0x0 0x40000000 0x0 0x40000000 0x0 0x20000000>, /* FIC3-FAB */
-			 <0x0 0x60000000 0x0 0x60000000 0x0 0x20000000>, /* FIC0, LO */
-			 <0x0 0xe0000000 0x0 0xe0000000 0x0 0x20000000>, /* FIC1, LO */
-			 <0x20 0x0 0x20 0x0 0x10 0x0>, /* FIC0,HI */
-			 <0x30 0x0 0x30 0x0 0x10 0x0>; /* FIC1,HI */
-
-		cape_gpios_p8: gpio@41100000 {
-			compatible = "microchip,coregpio-rtl-v3";
-			reg = <0x0 0x41100000 0x0 0x1000>;
-			clocks = <&fabric_clk3>;
-			gpio-controller;
-			#gpio-cells = <2>;
-			ngpios = <16>;
-			gpio-line-names = "P8_PIN31", "P8_PIN32", "P8_PIN33", "P8_PIN34",
-					  "P8_PIN35", "P8_PIN36", "P8_PIN37", "P8_PIN38",
-					  "P8_PIN39", "P8_PIN40", "P8_PIN41", "P8_PIN42",
-					  "P8_PIN43", "P8_PIN44", "P8_PIN45", "P8_PIN46";
-		};
-
-		cape_gpios_p9: gpio@41200000 {
-			compatible = "microchip,coregpio-rtl-v3";
-			reg = <0x0 0x41200000 0x0 0x1000>;
-			clocks = <&fabric_clk3>;
-			gpio-controller;
-			#gpio-cells = <2>;
-			ngpios = <20>;
-			gpio-line-names = "P9_PIN11", "P9_PIN12", "P9_PIN13", "P9_PIN14",
-					  "P9_PIN15", "P9_PIN16", "P9_PIN17", "P9_PIN18",
-					  "P9_PIN21", "P9_PIN22", "P9_PIN23", "P9_PIN24",
-					  "P9_PIN25", "P9_PIN26", "P9_PIN27", "P9_PIN28",
-					  "P9_PIN29", "P9_PIN31", "P9_PIN41", "P9_PIN42";
-		};
-
-		hsi_gpios: gpio@44000000 {
-			compatible = "microchip,coregpio-rtl-v3";
-			reg = <0x0 0x44000000 0x0 0x1000>;
-			clocks = <&fabric_clk3>;
-			gpio-controller;
-			#gpio-cells = <2>;
-			ngpios = <20>;
-			gpio-line-names = "B0_HSIO70N", "B0_HSIO71N", "B0_HSIO83N",
-					  "B0_HSIO73N_C2P_CLKN", "B0_HSIO70P", "B0_HSIO71P",
-					  "B0_HSIO83P", "B0_HSIO73N_C2P_CLKP", "XCVR1_RX_VALID",
-					  "XCVR1_LOCK", "XCVR1_ERROR", "XCVR2_RX_VALID",
-					  "XCVR2_LOCK", "XCVR2_ERROR", "XCVR3_RX_VALID",
-					  "XCVR3_LOCK", "XCVR3_ERROR", "XCVR_0B_REF_CLK_PLL_LOCK",
-					  "XCVR_0C_REF_CLK_PLL_LOCK", "B0_HSIO81N";
-		};
-	};
-
-	refclk_ccc: cccrefclk {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-	};
-};
-
-&ccc_nw {
-	clocks = <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>,
-		 <&refclk_ccc>, <&refclk_ccc>;
-	clock-names = "pll0_ref0", "pll0_ref1", "pll1_ref0", "pll1_ref1",
-		      "dll0_ref", "dll1_ref";
-	status = "okay";
-};
diff --git a/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts
index f44ad8e6f4e4..73afa2d26bb6 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts
+++ b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts
@@ -5,7 +5,6 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include "mpfs.dtsi"
-#include "mpfs-beaglev-fire-fabric.dtsi"
 
 /* Clock frequency (in Hz) of MTIMER */
 #define MTIMER_FREQ		1000000
@@ -203,10 +202,6 @@ &refclk {
 	clock-frequency = <125000000>;
 };
 
-&refclk_ccc {
-	clock-frequency = <50000000>;
-};
-
 &rtc {
 	status = "okay";
 };
diff --git a/arch/riscv/boot/dts/microchip/mpfs.dtsi b/arch/riscv/boot/dts/microchip/mpfs.dtsi
index 9883ca3554c5..5c2963e269b8 100644
--- a/arch/riscv/boot/dts/microchip/mpfs.dtsi
+++ b/arch/riscv/boot/dts/microchip/mpfs.dtsi
@@ -251,14 +251,17 @@ pdma: dma-controller@3000000 {
 			#dma-cells = <1>;
 		};
 
-		clkcfg: clkcfg@20002000 {
-			compatible = "microchip,mpfs-clkcfg";
-			reg = <0x0 0x20002000 0x0 0x1000>, <0x0 0x3E001000 0x0 0x1000>;
-			clocks = <&refclk>;
-			#clock-cells = <1>;
+		mss_top_sysreg: syscon@20002000 {
+			compatible = "microchip,mpfs-mss-top-sysreg", "syscon", "simple-mfd";
+			reg = <0x0 0x20002000 0x0 0x1000>;
 			#reset-cells = <1>;
 		};
 
+		sysreg_scb: syscon@20003000 {
+			compatible = "microchip,mpfs-sysreg-scb", "syscon";
+			reg = <0x0 0x20003000 0x0 0x1000>;
+		};
+
 		ccc_se: clock-controller@38010000 {
 			compatible = "microchip,mpfs-ccc";
 			reg = <0x0 0x38010000 0x0 0x1000>, <0x0 0x38020000 0x0 0x1000>,
@@ -447,7 +450,7 @@ mac0: ethernet@20110000 {
 			local-mac-address = [00 00 00 00 00 00];
 			clocks = <&clkcfg CLK_MAC0>, <&clkcfg CLK_AHB>;
 			clock-names = "pclk", "hclk";
-			resets = <&clkcfg CLK_MAC0>;
+			resets = <&mss_top_sysreg CLK_MAC0>;
 			status = "disabled";
 		};
 
@@ -461,7 +464,7 @@ mac1: ethernet@20112000 {
 			local-mac-address = [00 00 00 00 00 00];
 			clocks = <&clkcfg CLK_MAC1>, <&clkcfg CLK_AHB>;
 			clock-names = "pclk", "hclk";
-			resets = <&clkcfg CLK_MAC1>;
+			resets = <&mss_top_sysreg CLK_MAC1>;
 			status = "disabled";
 		};
 
@@ -521,10 +524,14 @@ usb: usb@20201000 {
 			status = "disabled";
 		};
 
-		mbox: mailbox@37020000 {
+		control_scb: syscon@37020000 {
+			compatible = "microchip,mpfs-control-scb", "syscon";
+			reg = <0x0 0x37020000 0x0 0x100>;
+		};
+
+		mbox: mailbox@37020800 {
 			compatible = "microchip,mpfs-mailbox";
-			reg = <0x0 0x37020000 0x0 0x58>, <0x0 0x2000318C 0x0 0x40>,
-			      <0x0 0x37020800 0x0 0x100>;
+			reg = <0x0 0x37020800 0x0 0x1000>;
 			interrupt-parent = <&plic>;
 			interrupts = <96>;
 			#mbox-cells = <1>;
@@ -541,5 +548,12 @@ syscontroller_qspi: spi@37020100 {
 			clocks = <&scbclk>;
 			status = "disabled";
 		};
+
+		clkcfg: clkcfg@3e001000 {
+			compatible = "microchip,mpfs-clkcfg";
+			reg = <0x0 0x3e001000 0x0 0x1000>;
+			clocks = <&refclk>;
+			#clock-cells = <1>;
+		};
 	};
 };
diff --git a/arch/riscv/boot/dts/sophgo/cv180x.dtsi b/arch/riscv/boot/dts/sophgo/cv180x.dtsi
index 1b2b1969a648..06b0ce5a2db7 100644
--- a/arch/riscv/boot/dts/sophgo/cv180x.dtsi
+++ b/arch/riscv/boot/dts/sophgo/cv180x.dtsi
@@ -438,8 +438,8 @@ usb: usb@4340000 {
 			clocks = <&clk CLK_AXI4_USB>, <&clk CLK_APB_USB>;
 			clock-names = "otg", "utmi";
 			g-np-tx-fifo-size = <32>;
-			g-rx-fifo-size = <536>;
-			g-tx-fifo-size = <768 512 512 384 128 128>;
+			g-rx-fifo-size = <1536>;
+			g-tx-fifo-size = <128 128 64 64 64 64 32 32>;
 			interrupts = <SOC_PERIPHERAL_IRQ(14) IRQ_TYPE_LEVEL_HIGH>;
 			phys = <&usbphy>;
 			phy-names = "usb2-phy";
diff --git a/arch/riscv/configs/hardening.config b/arch/riscv/configs/hardening.config
new file mode 100644
index 000000000000..089f4cee82f4
--- /dev/null
+++ b/arch/riscv/configs/hardening.config
@@ -0,0 +1,4 @@
+# RISCV specific kernel hardening options
+
+# Enable control flow integrity support for usermode.
+CONFIG_RISCV_USER_CFI=y
diff --git a/arch/riscv/include/asm/asm-prototypes.h b/arch/riscv/include/asm/asm-prototypes.h
index a9988bf21ec8..41ec5cdec367 100644
--- a/arch/riscv/include/asm/asm-prototypes.h
+++ b/arch/riscv/include/asm/asm-prototypes.h
@@ -51,6 +51,7 @@ DECLARE_DO_ERROR_INFO(do_trap_ecall_u);
 DECLARE_DO_ERROR_INFO(do_trap_ecall_s);
 DECLARE_DO_ERROR_INFO(do_trap_ecall_m);
 DECLARE_DO_ERROR_INFO(do_trap_break);
+DECLARE_DO_ERROR_INFO(do_trap_software_check);
 
 asmlinkage void ret_from_fork_kernel(void *fn_arg, int (*fn)(void *), struct pt_regs *regs);
 asmlinkage void ret_from_fork_user(struct pt_regs *regs);
diff --git a/arch/riscv/include/asm/assembler.h b/arch/riscv/include/asm/assembler.h
index 16931712beab..a8df1999118b 100644
--- a/arch/riscv/include/asm/assembler.h
+++ b/arch/riscv/include/asm/assembler.h
@@ -80,3 +80,47 @@
 	.endm
 
 #endif	/* __ASM_ASSEMBLER_H */
+
+#if defined(VDSO_CFI) && (__riscv_xlen == 64)
+.macro vdso_lpad, label = 0
+lpad \label
+.endm
+#else
+.macro vdso_lpad, label = 0
+.endm
+#endif
+
+/*
+ * This macro emits a program property note section identifying
+ * architecture features which require special handling, mainly for
+ * use in assembly files included in the VDSO.
+ */
+#define NT_GNU_PROPERTY_TYPE_0  5
+#define GNU_PROPERTY_RISCV_FEATURE_1_AND 0xc0000000
+
+#define GNU_PROPERTY_RISCV_FEATURE_1_ZICFILP		BIT(0)
+#define GNU_PROPERTY_RISCV_FEATURE_1_ZICFISS		BIT(1)
+
+#if defined(VDSO_CFI) && (__riscv_xlen == 64)
+#define GNU_PROPERTY_RISCV_FEATURE_1_DEFAULT \
+	(GNU_PROPERTY_RISCV_FEATURE_1_ZICFILP | GNU_PROPERTY_RISCV_FEATURE_1_ZICFISS)
+#endif
+
+#ifdef GNU_PROPERTY_RISCV_FEATURE_1_DEFAULT
+.macro emit_riscv_feature_1_and, feat = GNU_PROPERTY_RISCV_FEATURE_1_DEFAULT
+	.pushsection .note.gnu.property, "a"
+	.p2align        3
+	.word           4
+	.word           16
+	.word           NT_GNU_PROPERTY_TYPE_0
+	.asciz          "GNU"
+	.word           GNU_PROPERTY_RISCV_FEATURE_1_AND
+	.word           4
+	.word           \feat
+	.word           0
+	.popsection
+.endm
+#else
+.macro emit_riscv_feature_1_and, feat = 0
+.endm
+#endif
diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h
index 5b96c2f61adb..3f33dc54f94b 100644
--- a/arch/riscv/include/asm/atomic.h
+++ b/arch/riscv/include/asm/atomic.h
@@ -203,7 +203,7 @@ ATOMIC_OPS(xor, xor, i)
 		"	add            %[rc], %[p], %[a]\n"		\
 		"	sc." sfx ".rl  %[rc], %[rc], %[c]\n"		\
 		"	bnez           %[rc], 0b\n"			\
-		"	fence          rw, rw\n"			\
+		RISCV_FULL_BARRIER					\
 		"1:\n"							\
 		: [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter)	\
 		: [a]"r" (_a), [u]"r" (_u)				\
@@ -242,7 +242,7 @@ static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a,
 		"	addi            %[rc], %[p], 1\n"		\
 		"	sc." sfx ".rl   %[rc], %[rc], %[c]\n"		\
 		"	bnez            %[rc], 0b\n"			\
-		"	fence           rw, rw\n"			\
+		RISCV_FULL_BARRIER					\
 		"1:\n"							\
 		: [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter)	\
 		:							\
@@ -268,7 +268,7 @@ static __always_inline bool arch_atomic_inc_unless_negative(atomic_t *v)
 		"	addi            %[rc], %[p], -1\n"		\
 		"	sc." sfx ".rl   %[rc], %[rc], %[c]\n"		\
 		"	bnez            %[rc], 0b\n"			\
-		"	fence           rw, rw\n"			\
+		RISCV_FULL_BARRIER					\
 		"1:\n"							\
 		: [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter)	\
 		:							\
@@ -294,7 +294,7 @@ static __always_inline bool arch_atomic_dec_unless_positive(atomic_t *v)
 		"	bltz           %[rc], 1f\n"			\
 		"	sc." sfx ".rl  %[rc], %[rc], %[c]\n"		\
 		"	bnez           %[rc], 0b\n"			\
-		"	fence          rw, rw\n"			\
+		RISCV_FULL_BARRIER					\
 		"1:\n"							\
 		: [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter)	\
 		:							\
diff --git a/arch/riscv/include/asm/cpufeature.h b/arch/riscv/include/asm/cpufeature.h
index 62837fa981e8..739fcc84bf7b 100644
--- a/arch/riscv/include/asm/cpufeature.h
+++ b/arch/riscv/include/asm/cpufeature.h
@@ -152,4 +152,16 @@ static __always_inline bool riscv_cpu_has_extension_unlikely(int cpu, const unsi
 	return __riscv_isa_extension_available(hart_isa[cpu].isa, ext);
 }
 
+static inline bool cpu_supports_shadow_stack(void)
+{
+	return (IS_ENABLED(CONFIG_RISCV_USER_CFI) &&
+		riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICFISS));
+}
+
+static inline bool cpu_supports_indirect_br_lp_instr(void)
+{
+	return (IS_ENABLED(CONFIG_RISCV_USER_CFI) &&
+		riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICFILP));
+}
+
 #endif
diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h
index 4a37a98398ad..55a07e8722ff 100644
--- a/arch/riscv/include/asm/csr.h
+++ b/arch/riscv/include/asm/csr.h
@@ -18,6 +18,15 @@
 #define SR_MPP		_AC(0x00001800, UL) /* Previously Machine */
 #define SR_SUM		_AC(0x00040000, UL) /* Supervisor User Memory Access */
 
+/* zicfilp landing pad status bit */
+#define SR_SPELP	_AC(0x00800000, UL)
+#define SR_MPELP	_AC(0x020000000000, UL)
+#ifdef CONFIG_RISCV_M_MODE
+#define SR_ELP		SR_MPELP
+#else
+#define SR_ELP		SR_SPELP
+#endif
+
 #define SR_FS		_AC(0x00006000, UL) /* Floating-point Status */
 #define SR_FS_OFF	_AC(0x00000000, UL)
 #define SR_FS_INITIAL	_AC(0x00002000, UL)
@@ -212,6 +221,8 @@
 #define ENVCFG_PMM_PMLEN_16		(_AC(0x3, ULL) << 32)
 #define ENVCFG_CBZE			(_AC(1, UL) << 7)
 #define ENVCFG_CBCFE			(_AC(1, UL) << 6)
+#define ENVCFG_LPE			(_AC(1, UL) << 2)
+#define ENVCFG_SSE			(_AC(1, UL) << 3)
 #define ENVCFG_CBIE_SHIFT		4
 #define ENVCFG_CBIE			(_AC(0x3, UL) << ENVCFG_CBIE_SHIFT)
 #define ENVCFG_CBIE_ILL			_AC(0x0, UL)
@@ -321,6 +332,9 @@
 #define CSR_STIMECMP		0x14D
 #define CSR_STIMECMPH		0x15D
 
+/* zicfiss user mode csr.  CSR_SSP holds current shadow stack pointer */
+#define CSR_SSP			0x011
+
 /* xtheadvector symbolic CSR names */
 #define CSR_VXSAT		0x9
 #define CSR_VXRM		0xa
diff --git a/arch/riscv/include/asm/entry-common.h b/arch/riscv/include/asm/entry-common.h
index b28ccc6cdeea..34ed149af5d1 100644
--- a/arch/riscv/include/asm/entry-common.h
+++ b/arch/riscv/include/asm/entry-common.h
@@ -40,4 +40,6 @@ static inline int handle_misaligned_store(struct pt_regs *regs)
 }
 #endif
 
+bool handle_user_cfi_violation(struct pt_regs *regs);
+
 #endif /* _ASM_RISCV_ENTRY_COMMON_H */
diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h
index dfe57b215e6c..7ef8e5f55c8d 100644
--- a/arch/riscv/include/asm/hwcap.h
+++ b/arch/riscv/include/asm/hwcap.h
@@ -108,6 +108,10 @@
 #define RISCV_ISA_EXT_ZICBOP		99
 #define RISCV_ISA_EXT_SVRSW60T59B	100
 #define RISCV_ISA_EXT_ZALASR		101
+#define RISCV_ISA_EXT_ZILSD		102
+#define RISCV_ISA_EXT_ZCLSD		103
+#define RISCV_ISA_EXT_ZICFILP		104
+#define RISCV_ISA_EXT_ZICFISS		105
 
 #define RISCV_ISA_EXT_XLINUXENVCFG	127
 
diff --git a/arch/riscv/include/asm/hwprobe.h b/arch/riscv/include/asm/hwprobe.h
index 8c572a464719..8b9f5e1cf4cb 100644
--- a/arch/riscv/include/asm/hwprobe.h
+++ b/arch/riscv/include/asm/hwprobe.h
@@ -8,7 +8,7 @@
 
 #include <uapi/asm/hwprobe.h>
 
-#define RISCV_HWPROBE_MAX_KEY 15
+#define RISCV_HWPROBE_MAX_KEY		16
 
 static inline bool riscv_hwprobe_key_is_valid(__s64 key)
 {
@@ -20,6 +20,7 @@ static inline bool hwprobe_key_is_bitmask(__s64 key)
 	switch (key) {
 	case RISCV_HWPROBE_KEY_BASE_BEHAVIOR:
 	case RISCV_HWPROBE_KEY_IMA_EXT_0:
+	case RISCV_HWPROBE_KEY_IMA_EXT_1:
 	case RISCV_HWPROBE_KEY_CPUPERF_0:
 	case RISCV_HWPROBE_KEY_VENDOR_EXT_THEAD_0:
 	case RISCV_HWPROBE_KEY_VENDOR_EXT_MIPS_0:
diff --git a/arch/riscv/include/asm/mman.h b/arch/riscv/include/asm/mman.h
new file mode 100644
index 000000000000..0ad1d19832eb
--- /dev/null
+++ b/arch/riscv/include/asm/mman.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ASM_MMAN_H__
+#define __ASM_MMAN_H__
+
+#include <linux/compiler.h>
+#include <linux/types.h>
+#include <linux/mm.h>
+#include <uapi/asm/mman.h>
+
+static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot,
+						   unsigned long pkey __always_unused)
+{
+	unsigned long ret = 0;
+
+	/*
+	 * If PROT_WRITE was specified, force it to VM_READ | VM_WRITE.
+	 * Only VM_WRITE means shadow stack.
+	 */
+	if (prot & PROT_WRITE)
+		ret = (VM_READ | VM_WRITE);
+	return ret;
+}
+
+#define arch_calc_vm_prot_bits(prot, pkey) arch_calc_vm_prot_bits(prot, pkey)
+
+#endif /* ! __ASM_MMAN_H__ */
diff --git a/arch/riscv/include/asm/mmu_context.h b/arch/riscv/include/asm/mmu_context.h
index 8c4bc49a3a0f..dbf27a78df6c 100644
--- a/arch/riscv/include/asm/mmu_context.h
+++ b/arch/riscv/include/asm/mmu_context.h
@@ -48,6 +48,13 @@ static inline unsigned long mm_untag_mask(struct mm_struct *mm)
 }
 #endif
 
+#define deactivate_mm deactivate_mm
+static inline void deactivate_mm(struct task_struct *tsk,
+				 struct mm_struct *mm)
+{
+	shstk_release(tsk);
+}
+
 #include <asm-generic/mmu_context.h>
 
 #endif /* _ASM_RISCV_MMU_CONTEXT_H */
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 8bd36ac842eb..4b8ed4f62bb7 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -182,6 +182,7 @@ extern struct pt_alloc_ops pt_ops __meminitdata;
 #define PAGE_READ_EXEC		__pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC)
 #define PAGE_WRITE_EXEC		__pgprot(_PAGE_BASE | _PAGE_READ |	\
 					 _PAGE_EXEC | _PAGE_WRITE)
+#define PAGE_SHADOWSTACK       __pgprot(_PAGE_BASE | _PAGE_WRITE)
 
 #define PAGE_COPY		PAGE_READ
 #define PAGE_COPY_EXEC		PAGE_READ_EXEC
@@ -414,7 +415,7 @@ static inline int pte_special(pte_t pte)
 
 static inline pte_t pte_wrprotect(pte_t pte)
 {
-	return __pte(pte_val(pte) & ~(_PAGE_WRITE));
+	return __pte((pte_val(pte) & ~(_PAGE_WRITE)) | (_PAGE_READ));
 }
 
 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
@@ -454,11 +455,20 @@ static inline pte_t pte_swp_clear_uffd_wp(pte_t pte)
 
 /* static inline pte_t pte_mkread(pte_t pte) */
 
+struct vm_area_struct;
+pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma);
+#define pte_mkwrite pte_mkwrite
+
 static inline pte_t pte_mkwrite_novma(pte_t pte)
 {
 	return __pte(pte_val(pte) | _PAGE_WRITE);
 }
 
+static inline pte_t pte_mkwrite_shstk(pte_t pte)
+{
+	return __pte((pte_val(pte) & ~(_PAGE_LEAF)) | _PAGE_WRITE);
+}
+
 /* static inline pte_t pte_mkexec(pte_t pte) */
 
 static inline pte_t pte_mkdirty(pte_t pte)
@@ -660,7 +670,13 @@ extern int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long a
 static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
 				       unsigned long address, pte_t *ptep)
 {
-	pte_t pte = __pte(atomic_long_xchg((atomic_long_t *)ptep, 0));
+#ifdef CONFIG_SMP
+	pte_t pte = __pte(xchg(&ptep->pte, 0));
+#else
+	pte_t pte = *ptep;
+
+	set_pte(ptep, __pte(0));
+#endif
 
 	page_table_check_pte_clear(mm, pte);
 
@@ -671,7 +687,15 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
 static inline void ptep_set_wrprotect(struct mm_struct *mm,
 				      unsigned long address, pte_t *ptep)
 {
-	atomic_long_and(~(unsigned long)_PAGE_WRITE, (atomic_long_t *)ptep);
+	pte_t read_pte = READ_ONCE(*ptep);
+	/*
+	 * ptep_set_wrprotect can be called for shadow stack ranges too.
+	 * shadow stack memory is XWR = 010 and thus clearing _PAGE_WRITE will lead to
+	 * encoding 000b which is wrong encoding with V = 1. This should lead to page fault
+	 * but we dont want this wrong configuration to be set in page tables.
+	 */
+	atomic_long_set((atomic_long_t *)ptep,
+			((pte_val(read_pte) & ~(unsigned long)_PAGE_WRITE) | _PAGE_READ));
 }
 
 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
@@ -831,11 +855,19 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd)
 	return pte_pmd(pte_mkyoung(pmd_pte(pmd)));
 }
 
+pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
+#define pmd_mkwrite pmd_mkwrite
+
 static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
 {
 	return pte_pmd(pte_mkwrite_novma(pmd_pte(pmd)));
 }
 
+static inline pmd_t pmd_mkwrite_shstk(pmd_t pte)
+{
+	return __pmd((pmd_val(pte) & ~(_PAGE_LEAF)) | _PAGE_WRITE);
+}
+
 static inline pmd_t pmd_wrprotect(pmd_t pmd)
 {
 	return pte_pmd(pte_wrprotect(pmd_pte(pmd)));
@@ -997,7 +1029,13 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
 					unsigned long address, pmd_t *pmdp)
 {
-	pmd_t pmd = __pmd(atomic_long_xchg((atomic_long_t *)pmdp, 0));
+#ifdef CONFIG_SMP
+	pmd_t pmd = __pmd(xchg(&pmdp->pmd, 0));
+#else
+	pmd_t pmd = *pmdp;
+
+	pmd_clear(pmdp);
+#endif
 
 	page_table_check_pmd_clear(mm, pmd);
 
diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h
index da5426122d28..4c3dd94d0f63 100644
--- a/arch/riscv/include/asm/processor.h
+++ b/arch/riscv/include/asm/processor.h
@@ -16,6 +16,7 @@
 #include <asm/insn-def.h>
 #include <asm/alternative-macros.h>
 #include <asm/hwcap.h>
+#include <asm/usercfi.h>
 
 #define arch_get_mmap_end(addr, len, flags)			\
 ({								\
diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h
index ccc77a89b1e2..5725e0ca4dda 100644
--- a/arch/riscv/include/asm/sbi.h
+++ b/arch/riscv/include/asm/sbi.h
@@ -37,6 +37,7 @@ enum sbi_ext_id {
 	SBI_EXT_NACL = 0x4E41434C,
 	SBI_EXT_FWFT = 0x46574654,
 	SBI_EXT_MPXY = 0x4D505859,
+	SBI_EXT_DBTR = 0x44425452,
 
 	/* Experimentals extensions must lie within this range */
 	SBI_EXT_EXPERIMENTAL_START = 0x08000000,
@@ -505,6 +506,34 @@ enum sbi_mpxy_rpmi_attribute_id {
 #define SBI_MPXY_CHAN_CAP_SEND_WITHOUT_RESP	BIT(4)
 #define SBI_MPXY_CHAN_CAP_GET_NOTIFICATIONS	BIT(5)
 
+/* SBI debug triggers function IDs */
+enum sbi_ext_dbtr_fid {
+	SBI_EXT_DBTR_NUM_TRIGGERS = 0,
+	SBI_EXT_DBTR_SETUP_SHMEM,
+	SBI_EXT_DBTR_TRIG_READ,
+	SBI_EXT_DBTR_TRIG_INSTALL,
+	SBI_EXT_DBTR_TRIG_UPDATE,
+	SBI_EXT_DBTR_TRIG_UNINSTALL,
+	SBI_EXT_DBTR_TRIG_ENABLE,
+	SBI_EXT_DBTR_TRIG_DISABLE,
+};
+
+struct sbi_dbtr_data_msg {
+	unsigned long tstate;
+	unsigned long tdata1;
+	unsigned long tdata2;
+	unsigned long tdata3;
+};
+
+struct sbi_dbtr_id_msg {
+	unsigned long idx;
+};
+
+union sbi_dbtr_shmem_entry {
+	struct sbi_dbtr_data_msg data;
+	struct sbi_dbtr_id_msg id;
+};
+
 /* SBI spec version fields */
 #define SBI_SPEC_VERSION_DEFAULT	0x1
 #define SBI_SPEC_VERSION_MAJOR_SHIFT	24
diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h
index 836d80dd2921..36918c9200c9 100644
--- a/arch/riscv/include/asm/thread_info.h
+++ b/arch/riscv/include/asm/thread_info.h
@@ -73,6 +73,9 @@ struct thread_info {
 	 */
 	unsigned long		a0, a1, a2;
 #endif
+#ifdef CONFIG_RISCV_USER_CFI
+	struct cfi_state	user_cfi_state;
+#endif
 };
 
 #ifdef CONFIG_SHADOW_CALL_STACK
diff --git a/arch/riscv/include/asm/usercfi.h b/arch/riscv/include/asm/usercfi.h
new file mode 100644
index 000000000000..7495baae1e3c
--- /dev/null
+++ b/arch/riscv/include/asm/usercfi.h
@@ -0,0 +1,97 @@
+/* SPDX-License-Identifier: GPL-2.0
+ * Copyright (C) 2024 Rivos, Inc.
+ * Deepak Gupta <debug@rivosinc.com>
+ */
+#ifndef _ASM_RISCV_USERCFI_H
+#define _ASM_RISCV_USERCFI_H
+
+#define CMDLINE_DISABLE_RISCV_USERCFI_FCFI	1
+#define CMDLINE_DISABLE_RISCV_USERCFI_BCFI	2
+#define CMDLINE_DISABLE_RISCV_USERCFI		3
+
+#ifndef __ASSEMBLER__
+#include <linux/types.h>
+#include <linux/prctl.h>
+#include <linux/errno.h>
+
+struct task_struct;
+struct kernel_clone_args;
+
+extern unsigned long riscv_nousercfi;
+
+#ifdef CONFIG_RISCV_USER_CFI
+struct cfi_state {
+	unsigned long ubcfi_en : 1; /* Enable for backward cfi. */
+	unsigned long ubcfi_locked : 1;
+	unsigned long ufcfi_en : 1; /* Enable for forward cfi. Note that ELP goes in sstatus */
+	unsigned long ufcfi_locked : 1;
+	unsigned long user_shdw_stk; /* Current user shadow stack pointer */
+	unsigned long shdw_stk_base; /* Base address of shadow stack */
+	unsigned long shdw_stk_size; /* size of shadow stack */
+};
+
+unsigned long shstk_alloc_thread_stack(struct task_struct *tsk,
+				       const struct kernel_clone_args *args);
+void shstk_release(struct task_struct *tsk);
+void set_shstk_base(struct task_struct *task, unsigned long shstk_addr, unsigned long size);
+unsigned long get_shstk_base(struct task_struct *task, unsigned long *size);
+void set_active_shstk(struct task_struct *task, unsigned long shstk_addr);
+bool is_shstk_enabled(struct task_struct *task);
+bool is_shstk_locked(struct task_struct *task);
+bool is_shstk_allocated(struct task_struct *task);
+void set_shstk_lock(struct task_struct *task);
+void set_shstk_status(struct task_struct *task, bool enable);
+unsigned long get_active_shstk(struct task_struct *task);
+int restore_user_shstk(struct task_struct *tsk, unsigned long shstk_ptr);
+int save_user_shstk(struct task_struct *tsk, unsigned long *saved_shstk_ptr);
+bool is_indir_lp_enabled(struct task_struct *task);
+bool is_indir_lp_locked(struct task_struct *task);
+void set_indir_lp_status(struct task_struct *task, bool enable);
+void set_indir_lp_lock(struct task_struct *task);
+
+#define PR_SHADOW_STACK_SUPPORTED_STATUS_MASK (PR_SHADOW_STACK_ENABLE)
+
+#else
+
+#define shstk_alloc_thread_stack(tsk, args) 0
+
+#define shstk_release(tsk)
+
+#define get_shstk_base(task, size) 0UL
+
+#define set_shstk_base(task, shstk_addr, size) do {} while (0)
+
+#define set_active_shstk(task, shstk_addr) do {} while (0)
+
+#define is_shstk_enabled(task) false
+
+#define is_shstk_locked(task) false
+
+#define is_shstk_allocated(task) false
+
+#define set_shstk_lock(task) do {} while (0)
+
+#define set_shstk_status(task, enable) do {} while (0)
+
+#define is_indir_lp_enabled(task) false
+
+#define is_indir_lp_locked(task) false
+
+#define set_indir_lp_status(task, enable) do {} while (0)
+
+#define set_indir_lp_lock(task) do {} while (0)
+
+#define restore_user_shstk(tsk, shstk_ptr) -EINVAL
+
+#define save_user_shstk(tsk, saved_shstk_ptr) -EINVAL
+
+#define get_active_shstk(task) 0UL
+
+#endif /* CONFIG_RISCV_USER_CFI */
+
+bool is_user_shstk_enabled(void);
+bool is_user_lpad_enabled(void);
+
+#endif /* __ASSEMBLER__ */
+
+#endif /* _ASM_RISCV_USERCFI_H */
diff --git a/arch/riscv/include/asm/vdso.h b/arch/riscv/include/asm/vdso.h
index f80357fe24d1..35bf830a5576 100644
--- a/arch/riscv/include/asm/vdso.h
+++ b/arch/riscv/include/asm/vdso.h
@@ -18,9 +18,19 @@
 
 #ifndef __ASSEMBLER__
 #include <generated/vdso-offsets.h>
+#ifdef CONFIG_RISCV_USER_CFI
+#include <generated/vdso-cfi-offsets.h>
+#endif
 
+#ifdef CONFIG_RISCV_USER_CFI
 #define VDSO_SYMBOL(base, name)							\
-	(void __user *)((unsigned long)(base) + __vdso_##name##_offset)
+	  (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZIMOP) ?			\
+	  (void __user *)((unsigned long)(base) + __vdso_##name##_cfi_offset) :	\
+	  (void __user *)((unsigned long)(base) + __vdso_##name##_offset))
+#else
+#define VDSO_SYMBOL(base, name)							\
+	  ((void __user *)((unsigned long)(base) + __vdso_##name##_offset))
+#endif
 
 #ifdef CONFIG_COMPAT
 #include <generated/compat_vdso-offsets.h>
@@ -33,6 +43,7 @@ extern char compat_vdso_start[], compat_vdso_end[];
 #endif /* CONFIG_COMPAT */
 
 extern char vdso_start[], vdso_end[];
+extern char vdso_cfi_start[], vdso_cfi_end[];
 
 #endif /* !__ASSEMBLER__ */
 
diff --git a/arch/riscv/include/asm/vector.h b/arch/riscv/include/asm/vector.h
index e7aa449368ad..00cb9c0982b1 100644
--- a/arch/riscv/include/asm/vector.h
+++ b/arch/riscv/include/asm/vector.h
@@ -424,6 +424,9 @@ static inline bool riscv_v_vstate_ctrl_user_allowed(void) { return false; }
 #define riscv_v_thread_free(tsk)		do {} while (0)
 #define  riscv_v_setup_ctx_cache()		do {} while (0)
 #define riscv_v_thread_alloc(tsk)		do {} while (0)
+#define get_cpu_vector_context()		do {} while (0)
+#define put_cpu_vector_context()		do {} while (0)
+#define riscv_v_vstate_set_restore(task, regs)	do {} while (0)
 
 #endif /* CONFIG_RISCV_ISA_V */
 
diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h
index 1edea2331b8b..9139edba0aec 100644
--- a/arch/riscv/include/uapi/asm/hwprobe.h
+++ b/arch/riscv/include/uapi/asm/hwprobe.h
@@ -84,6 +84,10 @@ struct riscv_hwprobe {
 #define		RISCV_HWPROBE_EXT_ZABHA		(1ULL << 58)
 #define		RISCV_HWPROBE_EXT_ZALASR	(1ULL << 59)
 #define		RISCV_HWPROBE_EXT_ZICBOP	(1ULL << 60)
+#define		RISCV_HWPROBE_EXT_ZILSD		(1ULL << 61)
+#define		RISCV_HWPROBE_EXT_ZCLSD		(1ULL << 62)
+#define		RISCV_HWPROBE_EXT_ZICFILP	(1ULL << 63)
+
 #define RISCV_HWPROBE_KEY_CPUPERF_0	5
 #define		RISCV_HWPROBE_MISALIGNED_UNKNOWN	(0 << 0)
 #define		RISCV_HWPROBE_MISALIGNED_EMULATED	(1 << 0)
@@ -110,6 +114,9 @@ struct riscv_hwprobe {
 #define RISCV_HWPROBE_KEY_VENDOR_EXT_SIFIVE_0	13
 #define RISCV_HWPROBE_KEY_VENDOR_EXT_MIPS_0	14
 #define RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE	15
+#define RISCV_HWPROBE_KEY_IMA_EXT_1		16
+#define		RISCV_HWPROBE_EXT_ZICFISS	(1ULL << 0)
+
 /* Increase RISCV_HWPROBE_MAX_KEY when adding items. */
 
 /* Flags */
diff --git a/arch/riscv/include/uapi/asm/ptrace.h b/arch/riscv/include/uapi/asm/ptrace.h
index beff8df80ac9..18988a5f1a63 100644
--- a/arch/riscv/include/uapi/asm/ptrace.h
+++ b/arch/riscv/include/uapi/asm/ptrace.h
@@ -127,6 +127,40 @@ struct __riscv_v_regset_state {
  */
 #define RISCV_MAX_VLENB (8192)
 
+struct __sc_riscv_cfi_state {
+	unsigned long ss_ptr;   /* shadow stack pointer */
+};
+
+#define PTRACE_CFI_LP_EN_BIT	0
+#define PTRACE_CFI_LP_LOCK_BIT	1
+#define PTRACE_CFI_ELP_BIT	2
+#define PTRACE_CFI_SS_EN_BIT	3
+#define PTRACE_CFI_SS_LOCK_BIT	4
+#define PTRACE_CFI_SS_PTR_BIT	5
+
+#define PTRACE_CFI_LP_EN_STATE		BIT(PTRACE_CFI_LP_EN_BIT)
+#define PTRACE_CFI_LP_LOCK_STATE	BIT(PTRACE_CFI_LP_LOCK_BIT)
+#define PTRACE_CFI_ELP_STATE		BIT(PTRACE_CFI_ELP_BIT)
+#define PTRACE_CFI_SS_EN_STATE		BIT(PTRACE_CFI_SS_EN_BIT)
+#define PTRACE_CFI_SS_LOCK_STATE	BIT(PTRACE_CFI_SS_LOCK_BIT)
+#define PTRACE_CFI_SS_PTR_STATE		BIT(PTRACE_CFI_SS_PTR_BIT)
+
+#define PRACE_CFI_STATE_INVALID_MASK	~(PTRACE_CFI_LP_EN_STATE | \
+					  PTRACE_CFI_LP_LOCK_STATE | \
+					  PTRACE_CFI_ELP_STATE | \
+					  PTRACE_CFI_SS_EN_STATE | \
+					  PTRACE_CFI_SS_LOCK_STATE | \
+					  PTRACE_CFI_SS_PTR_STATE)
+
+struct __cfi_status {
+	__u64 cfi_state;
+};
+
+struct user_cfi_state {
+	struct __cfi_status	cfi_status;
+	__u64 shstk_ptr;
+};
+
 #endif /* __ASSEMBLER__ */
 
 #endif /* _UAPI_ASM_RISCV_PTRACE_H */
diff --git a/arch/riscv/include/uapi/asm/sigcontext.h b/arch/riscv/include/uapi/asm/sigcontext.h
index 748dffc9ae19..d22d0815d605 100644
--- a/arch/riscv/include/uapi/asm/sigcontext.h
+++ b/arch/riscv/include/uapi/asm/sigcontext.h
@@ -10,6 +10,7 @@
 
 /* The Magic number for signal context frame header. */
 #define RISCV_V_MAGIC	0x53465457
+#define RISCV_ZICFISS_MAGIC		0x9487
 #define END_MAGIC	0x0
 
 /* The size of END signal context header. */
diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index f60fce69b725..9026400cba10 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -72,6 +72,7 @@ obj-y	+= vendor_extensions/
 obj-y	+= probes/
 obj-y	+= tests/
 obj-$(CONFIG_MMU) += vdso.o vdso/
+obj-$(CONFIG_RISCV_USER_CFI) += vdso_cfi/
 
 obj-$(CONFIG_RISCV_MISALIGNED)	+= traps_misaligned.o
 obj-$(CONFIG_RISCV_MISALIGNED)	+= unaligned_access_speed.o
@@ -125,3 +126,4 @@ obj-$(CONFIG_ACPI)		+= acpi.o
 obj-$(CONFIG_ACPI_NUMA)	+= acpi_numa.o
 
 obj-$(CONFIG_GENERIC_CPU_VULNERABILITIES) += bugs.o
+obj-$(CONFIG_RISCV_USER_CFI) += usercfi.o
diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c
index 7d42d3b8a32a..af827448a609 100644
--- a/arch/riscv/kernel/asm-offsets.c
+++ b/arch/riscv/kernel/asm-offsets.c
@@ -51,6 +51,10 @@ void asm_offsets(void)
 #endif
 
 	OFFSET(TASK_TI_CPU_NUM, task_struct, thread_info.cpu);
+#ifdef CONFIG_RISCV_USER_CFI
+	OFFSET(TASK_TI_CFI_STATE, task_struct, thread_info.user_cfi_state);
+	OFFSET(TASK_TI_USER_SSP, task_struct, thread_info.user_cfi_state.user_shdw_stk);
+#endif
 	OFFSET(TASK_THREAD_F0,  task_struct, thread.fstate.f[0]);
 	OFFSET(TASK_THREAD_F1,  task_struct, thread.fstate.f[1]);
 	OFFSET(TASK_THREAD_F2,  task_struct, thread.fstate.f[2]);
@@ -529,4 +533,10 @@ void asm_offsets(void)
 	DEFINE(FREGS_A6,	    offsetof(struct __arch_ftrace_regs, a6));
 	DEFINE(FREGS_A7,	    offsetof(struct __arch_ftrace_regs, a7));
 #endif
+#ifdef CONFIG_RISCV_SBI
+	DEFINE(SBI_EXT_FWFT, SBI_EXT_FWFT);
+	DEFINE(SBI_EXT_FWFT_SET, SBI_EXT_FWFT_SET);
+	DEFINE(SBI_FWFT_SHADOW_STACK, SBI_FWFT_SHADOW_STACK);
+	DEFINE(SBI_FWFT_SET_FLAG_LOCK, SBI_FWFT_SET_FLAG_LOCK);
+#endif
 }
diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c
index b057362f8fb5..d2fa5548662e 100644
--- a/arch/riscv/kernel/cpufeature.c
+++ b/arch/riscv/kernel/cpufeature.c
@@ -28,6 +28,7 @@
 #include <asm/vector.h>
 #include <asm/vendor_extensions.h>
 #include <asm/vendor_extensions/thead.h>
+#include <asm/usercfi.h>
 
 #define NUM_ALPHA_EXTS ('z' - 'a' + 1)
 
@@ -242,6 +243,28 @@ static int riscv_ext_zcf_validate(const struct riscv_isa_ext_data *data,
 	return -EPROBE_DEFER;
 }
 
+static int riscv_ext_zilsd_validate(const struct riscv_isa_ext_data *data,
+				    const unsigned long *isa_bitmap)
+{
+	if (IS_ENABLED(CONFIG_64BIT))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int riscv_ext_zclsd_validate(const struct riscv_isa_ext_data *data,
+				    const unsigned long *isa_bitmap)
+{
+	if (IS_ENABLED(CONFIG_64BIT))
+		return -EINVAL;
+
+	if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZILSD) &&
+	    __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA))
+		return 0;
+
+	return -EPROBE_DEFER;
+}
+
 static int riscv_vector_f_validate(const struct riscv_isa_ext_data *data,
 				   const unsigned long *isa_bitmap)
 {
@@ -274,6 +297,26 @@ static int riscv_ext_svadu_validate(const struct riscv_isa_ext_data *data,
 	return 0;
 }
 
+static int riscv_cfilp_validate(const struct riscv_isa_ext_data *data,
+				const unsigned long *isa_bitmap)
+{
+	if (!IS_ENABLED(CONFIG_RISCV_USER_CFI) ||
+	    (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_FCFI))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int riscv_cfiss_validate(const struct riscv_isa_ext_data *data,
+				const unsigned long *isa_bitmap)
+{
+	if (!IS_ENABLED(CONFIG_RISCV_USER_CFI) ||
+	    (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_BCFI))
+		return -EINVAL;
+
+	return 0;
+}
+
 static const unsigned int riscv_a_exts[] = {
 	RISCV_ISA_EXT_ZAAMO,
 	RISCV_ISA_EXT_ZALRSC,
@@ -461,6 +504,10 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = {
 	__RISCV_ISA_EXT_DATA_VALIDATE(zicbop, RISCV_ISA_EXT_ZICBOP, riscv_ext_zicbop_validate),
 	__RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, riscv_ext_zicboz_validate),
 	__RISCV_ISA_EXT_DATA(ziccrse, RISCV_ISA_EXT_ZICCRSE),
+	__RISCV_ISA_EXT_SUPERSET_VALIDATE(zicfilp, RISCV_ISA_EXT_ZICFILP, riscv_xlinuxenvcfg_exts,
+					  riscv_cfilp_validate),
+	__RISCV_ISA_EXT_SUPERSET_VALIDATE(zicfiss, RISCV_ISA_EXT_ZICFISS, riscv_xlinuxenvcfg_exts,
+					  riscv_cfiss_validate),
 	__RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR),
 	__RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND),
 	__RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR),
@@ -484,6 +531,8 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = {
 	__RISCV_ISA_EXT_DATA_VALIDATE(zcd, RISCV_ISA_EXT_ZCD, riscv_ext_zcd_validate),
 	__RISCV_ISA_EXT_DATA_VALIDATE(zcf, RISCV_ISA_EXT_ZCF, riscv_ext_zcf_validate),
 	__RISCV_ISA_EXT_DATA_VALIDATE(zcmop, RISCV_ISA_EXT_ZCMOP, riscv_ext_zca_depends),
+	__RISCV_ISA_EXT_DATA_VALIDATE(zclsd, RISCV_ISA_EXT_ZCLSD, riscv_ext_zclsd_validate),
+	__RISCV_ISA_EXT_DATA_VALIDATE(zilsd, RISCV_ISA_EXT_ZILSD, riscv_ext_zilsd_validate),
 	__RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA),
 	__RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB),
 	__RISCV_ISA_EXT_DATA(zbc, RISCV_ISA_EXT_ZBC),
diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
index 9b9dec6893b8..60eb221296a6 100644
--- a/arch/riscv/kernel/entry.S
+++ b/arch/riscv/kernel/entry.S
@@ -92,6 +92,35 @@
 	REG_L	a0, TASK_TI_A0(tp)
 .endm
 
+/*
+ * If previous mode was U, capture shadow stack pointer and save it away
+ * Zero CSR_SSP at the same time for sanitization.
+ */
+.macro save_userssp tmp, status
+	ALTERNATIVE("nops(4)",
+		__stringify(				\
+		andi \tmp, \status, SR_SPP;		\
+		bnez \tmp, skip_ssp_save;		\
+		csrrw \tmp, CSR_SSP, x0;		\
+		REG_S \tmp, TASK_TI_USER_SSP(tp);	\
+		skip_ssp_save:),
+		0,
+		RISCV_ISA_EXT_ZICFISS,
+		CONFIG_RISCV_USER_CFI)
+.endm
+
+.macro restore_userssp tmp, status
+	ALTERNATIVE("nops(4)",
+		__stringify(				\
+		andi \tmp, \status, SR_SPP;		\
+		bnez \tmp, skip_ssp_restore;		\
+		REG_L \tmp, TASK_TI_USER_SSP(tp);	\
+		csrw CSR_SSP, \tmp;			\
+		skip_ssp_restore:),
+		0,
+		RISCV_ISA_EXT_ZICFISS,
+		CONFIG_RISCV_USER_CFI)
+.endm
 
 SYM_CODE_START(handle_exception)
 	/*
@@ -145,9 +174,14 @@ SYM_CODE_START(handle_exception)
 	 * or vector in kernel space.
 	 */
 	li t0, SR_SUM | SR_FS_VS
+#ifdef CONFIG_64BIT
+	li t1, SR_ELP
+	or t0, t0, t1
+#endif
 
 	REG_L s0, TASK_TI_USER_SP(tp)
 	csrrc s1, CSR_STATUS, t0
+	save_userssp s2, s1
 	csrr s2, CSR_EPC
 	csrr s3, CSR_TVAL
 	csrr s4, CSR_CAUSE
@@ -243,6 +277,7 @@ SYM_CODE_START_NOALIGN(ret_from_exception)
 	call riscv_v_context_nesting_end
 #endif
 	REG_L a0, PT_STATUS(sp)
+	restore_userssp s3, a0
 	/*
 	 * The current load reservation is effectively part of the processor's
 	 * state, in the sense that load reservations cannot be shared between
@@ -460,6 +495,9 @@ SYM_DATA_START_LOCAL(excp_vect_table)
 	RISCV_PTR do_page_fault   /* load page fault */
 	RISCV_PTR do_trap_unknown
 	RISCV_PTR do_page_fault   /* store page fault */
+	RISCV_PTR do_trap_unknown /* cause=16 */
+	RISCV_PTR do_trap_unknown /* cause=17 */
+	RISCV_PTR do_trap_software_check /* cause=18 is sw check exception */
 SYM_DATA_END_LABEL(excp_vect_table, SYM_L_LOCAL, excp_vect_table_end)
 
 #ifndef CONFIG_MMU
diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S
index bdf3352acf4c..9c99c5ad6fe8 100644
--- a/arch/riscv/kernel/head.S
+++ b/arch/riscv/kernel/head.S
@@ -15,6 +15,7 @@
 #include <asm/image.h>
 #include <asm/scs.h>
 #include <asm/xip_fixup.h>
+#include <asm/usercfi.h>
 #include "efi-header.S"
 
 __HEAD
@@ -170,6 +171,19 @@ secondary_start_sbi:
 	call relocate_enable_mmu
 #endif
 	call .Lsetup_trap_vector
+#if defined(CONFIG_RISCV_SBI) && defined(CONFIG_RISCV_USER_CFI)
+	li a7, SBI_EXT_FWFT
+	li a6, SBI_EXT_FWFT_SET
+	li a0, SBI_FWFT_SHADOW_STACK
+	li a1, 1 /* enable supervisor to access shadow stack access */
+	li a2, SBI_FWFT_SET_FLAG_LOCK
+	ecall
+	beqz a0, 1f
+	la a1, riscv_nousercfi
+	li a0, CMDLINE_DISABLE_RISCV_USERCFI_BCFI
+	REG_S a0, (a1)
+1:
+#endif
 	scs_load_current
 	call smp_callin
 #endif /* CONFIG_SMP */
@@ -330,6 +344,19 @@ SYM_CODE_START(_start_kernel)
 	la tp, init_task
 	la sp, init_thread_union + THREAD_SIZE
 	addi sp, sp, -PT_SIZE_ON_STACK
+#if defined(CONFIG_RISCV_SBI) && defined(CONFIG_RISCV_USER_CFI)
+	li a7, SBI_EXT_FWFT
+	li a6, SBI_EXT_FWFT_SET
+	li a0, SBI_FWFT_SHADOW_STACK
+	li a1, 1 /* enable supervisor to access shadow stack access */
+	li a2, SBI_FWFT_SET_FLAG_LOCK
+	ecall
+	beqz a0, 1f
+	la a1, riscv_nousercfi
+	li a0, CMDLINE_DISABLE_RISCV_USERCFI_BCFI
+	REG_S a0, (a1)
+1:
+#endif
 	scs_load_current
 
 #ifdef CONFIG_KASAN
diff --git a/arch/riscv/kernel/image-vars.h b/arch/riscv/kernel/image-vars.h
index 3df30dd1c458..3bd9d06a8b8f 100644
--- a/arch/riscv/kernel/image-vars.h
+++ b/arch/riscv/kernel/image-vars.h
@@ -29,7 +29,7 @@ __efistub__end			= _end;
 __efistub__edata		= _edata;
 __efistub___init_text_end	= __init_text_end;
 #if defined(CONFIG_EFI_EARLYCON) || defined(CONFIG_SYSFB)
-__efistub_screen_info		= screen_info;
+__efistub_sysfb_primary_display	= sysfb_primary_display;
 #endif
 
 #endif
diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
index 31a392993cb4..aacb23978f93 100644
--- a/arch/riscv/kernel/process.c
+++ b/arch/riscv/kernel/process.c
@@ -31,6 +31,7 @@
 #include <asm/vector.h>
 #include <asm/cpufeature.h>
 #include <asm/exec.h>
+#include <asm/usercfi.h>
 
 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
 #include <linux/stackprotector.h>
@@ -92,8 +93,8 @@ void __show_regs(struct pt_regs *regs)
 		regs->s8, regs->s9, regs->s10);
 	pr_cont(" s11: " REG_FMT " t3 : " REG_FMT " t4 : " REG_FMT "\n",
 		regs->s11, regs->t3, regs->t4);
-	pr_cont(" t5 : " REG_FMT " t6 : " REG_FMT "\n",
-		regs->t5, regs->t6);
+	pr_cont(" t5 : " REG_FMT " t6 : " REG_FMT " ssp : " REG_FMT "\n",
+		regs->t5, regs->t6, get_active_shstk(current));
 
 	pr_cont("status: " REG_FMT " badaddr: " REG_FMT " cause: " REG_FMT "\n",
 		regs->status, regs->badaddr, regs->cause);
@@ -155,6 +156,19 @@ void start_thread(struct pt_regs *regs, unsigned long pc,
 	regs->epc = pc;
 	regs->sp = sp;
 
+	/*
+	 * clear shadow stack state on exec.
+	 * libc will set it later via prctl.
+	 */
+	set_shstk_status(current, false);
+	set_shstk_base(current, 0, 0);
+	set_active_shstk(current, 0);
+	/*
+	 * disable indirect branch tracking on exec.
+	 * libc will enable it later via prctl.
+	 */
+	set_indir_lp_status(current, false);
+
 #ifdef CONFIG_64BIT
 	regs->status &= ~SR_UXL;
 
@@ -226,6 +240,7 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
 	u64 clone_flags = args->flags;
 	unsigned long usp = args->stack;
 	unsigned long tls = args->tls;
+	unsigned long ssp = 0;
 	struct pt_regs *childregs = task_pt_regs(p);
 
 	/* Ensure all threads in this mm have the same pointer masking mode. */
@@ -245,11 +260,19 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
 		p->thread.s[1] = (unsigned long)args->fn_arg;
 		p->thread.ra = (unsigned long)ret_from_fork_kernel_asm;
 	} else {
+		/* allocate new shadow stack if needed. In case of CLONE_VM we have to */
+		ssp = shstk_alloc_thread_stack(p, args);
+		if (IS_ERR_VALUE(ssp))
+			return PTR_ERR((void *)ssp);
+
 		*childregs = *(current_pt_regs());
 		/* Turn off status.VS */
 		riscv_v_vstate_off(childregs);
 		if (usp) /* User fork */
 			childregs->sp = usp;
+		/* if needed, set new ssp */
+		if (ssp)
+			set_active_shstk(p, ssp);
 		if (clone_flags & CLONE_SETTLS)
 			childregs->tp = tls;
 		childregs->a0 = 0; /* Return value of fork() */
diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c
index e6272d74572f..57e257d459e8 100644
--- a/arch/riscv/kernel/ptrace.c
+++ b/arch/riscv/kernel/ptrace.c
@@ -19,6 +19,7 @@
 #include <linux/regset.h>
 #include <linux/sched.h>
 #include <linux/sched/task_stack.h>
+#include <asm/usercfi.h>
 
 enum riscv_regset {
 	REGSET_X,
@@ -31,6 +32,9 @@ enum riscv_regset {
 #ifdef CONFIG_RISCV_ISA_SUPM
 	REGSET_TAGGED_ADDR_CTRL,
 #endif
+#ifdef CONFIG_RISCV_USER_CFI
+	REGSET_CFI,
+#endif
 };
 
 static int riscv_gpr_get(struct task_struct *target,
@@ -195,6 +199,87 @@ static int tagged_addr_ctrl_set(struct task_struct *target,
 }
 #endif
 
+#ifdef CONFIG_RISCV_USER_CFI
+static int riscv_cfi_get(struct task_struct *target,
+			 const struct user_regset *regset,
+			 struct membuf to)
+{
+	struct user_cfi_state user_cfi;
+	struct pt_regs *regs;
+
+	memset(&user_cfi, 0, sizeof(user_cfi));
+	regs = task_pt_regs(target);
+
+	if (is_indir_lp_enabled(target)) {
+		user_cfi.cfi_status.cfi_state |= PTRACE_CFI_LP_EN_STATE;
+		user_cfi.cfi_status.cfi_state |= is_indir_lp_locked(target) ?
+						 PTRACE_CFI_LP_LOCK_STATE : 0;
+		user_cfi.cfi_status.cfi_state |= (regs->status & SR_ELP) ?
+						PTRACE_CFI_ELP_STATE : 0;
+	}
+
+	if (is_shstk_enabled(target)) {
+		user_cfi.cfi_status.cfi_state |= (PTRACE_CFI_SS_EN_STATE |
+						  PTRACE_CFI_SS_PTR_STATE);
+		user_cfi.cfi_status.cfi_state |= is_shstk_locked(target) ?
+						 PTRACE_CFI_SS_LOCK_STATE : 0;
+		user_cfi.shstk_ptr = get_active_shstk(target);
+	}
+
+	return membuf_write(&to, &user_cfi, sizeof(user_cfi));
+}
+
+/*
+ * Does it make sense to allow enable / disable of cfi via ptrace?
+ * We don't allow enable / disable / locking control via ptrace for now.
+ * Setting the shadow stack pointer is allowed. GDB might use it to unwind or
+ * some other fixup. Similarly gdb might want to suppress elp and may want
+ * to reset elp state.
+ */
+static int riscv_cfi_set(struct task_struct *target,
+			 const struct user_regset *regset,
+			 unsigned int pos, unsigned int count,
+			 const void *kbuf, const void __user *ubuf)
+{
+	int ret;
+	struct user_cfi_state user_cfi;
+	struct pt_regs *regs;
+
+	regs = task_pt_regs(target);
+
+	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &user_cfi, 0, -1);
+	if (ret)
+		return ret;
+
+	/*
+	 * Not allowing enabling or locking shadow stack or landing pad
+	 * There is no disabling of shadow stack or landing pad via ptrace
+	 * rsvd field should be set to zero so that if those fields are needed in future
+	 */
+	if ((user_cfi.cfi_status.cfi_state &
+	     (PTRACE_CFI_LP_EN_STATE | PTRACE_CFI_LP_LOCK_STATE |
+	      PTRACE_CFI_SS_EN_STATE | PTRACE_CFI_SS_LOCK_STATE)) ||
+	     (user_cfi.cfi_status.cfi_state & PRACE_CFI_STATE_INVALID_MASK))
+		return -EINVAL;
+
+	/* If lpad is enabled on target and ptrace requests to set / clear elp, do that */
+	if (is_indir_lp_enabled(target)) {
+		if (user_cfi.cfi_status.cfi_state &
+		    PTRACE_CFI_ELP_STATE) /* set elp state */
+			regs->status |= SR_ELP;
+		else
+			regs->status &= ~SR_ELP; /* clear elp state */
+	}
+
+	/* If shadow stack enabled on target, set new shadow stack pointer */
+	if (is_shstk_enabled(target) &&
+	    (user_cfi.cfi_status.cfi_state & PTRACE_CFI_SS_PTR_STATE))
+		set_active_shstk(target, user_cfi.shstk_ptr);
+
+	return 0;
+}
+#endif
+
 static struct user_regset riscv_user_regset[] __ro_after_init = {
 	[REGSET_X] = {
 		USER_REGSET_NOTE_TYPE(PRSTATUS),
@@ -234,6 +319,16 @@ static struct user_regset riscv_user_regset[] __ro_after_init = {
 		.set = tagged_addr_ctrl_set,
 	},
 #endif
+#ifdef CONFIG_RISCV_USER_CFI
+	[REGSET_CFI] = {
+		.core_note_type = NT_RISCV_USER_CFI,
+		.align = sizeof(__u64),
+		.n = sizeof(struct user_cfi_state) / sizeof(__u64),
+		.size = sizeof(__u64),
+		.regset_get = riscv_cfi_get,
+		.set = riscv_cfi_set,
+	},
+#endif
 };
 
 static const struct user_regset_view riscv_user_native_view = {
diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c
index 08378fea3a11..c27ad8a6304e 100644
--- a/arch/riscv/kernel/signal.c
+++ b/arch/riscv/kernel/signal.c
@@ -22,11 +22,13 @@
 #include <asm/vector.h>
 #include <asm/csr.h>
 #include <asm/cacheflush.h>
+#include <asm/usercfi.h>
 
 unsigned long signal_minsigstksz __ro_after_init;
 
 extern u32 __user_rt_sigreturn[2];
 static size_t riscv_v_sc_size __ro_after_init;
+static size_t riscv_zicfiss_sc_size __ro_after_init;
 
 #define DEBUG_SIG 0
 
@@ -68,18 +70,19 @@ static long save_fp_state(struct pt_regs *regs,
 #define restore_fp_state(task, regs) (0)
 #endif
 
-#ifdef CONFIG_RISCV_ISA_V
-
-static long save_v_state(struct pt_regs *regs, void __user **sc_vec)
+static long save_v_state(struct pt_regs *regs, void __user *sc_vec)
 {
-	struct __riscv_ctx_hdr __user *hdr;
 	struct __sc_riscv_v_state __user *state;
 	void __user *datap;
 	long err;
 
-	hdr = *sc_vec;
-	/* Place state to the user's signal context space after the hdr */
-	state = (struct __sc_riscv_v_state __user *)(hdr + 1);
+	if (!IS_ENABLED(CONFIG_RISCV_ISA_V) ||
+	    !((has_vector() || has_xtheadvector()) &&
+	    riscv_v_vstate_query(regs)))
+		return 0;
+
+	/* Place state to the user's signal context space */
+	state = (struct __sc_riscv_v_state __user *)sc_vec;
 	/* Point datap right after the end of __sc_riscv_v_state */
 	datap = state + 1;
 
@@ -97,15 +100,11 @@ static long save_v_state(struct pt_regs *regs, void __user **sc_vec)
 	err |= __put_user((__force void *)datap, &state->v_state.datap);
 	/* Copy the whole vector content to user space datap. */
 	err |= __copy_to_user(datap, current->thread.vstate.datap, riscv_v_vsize);
-	/* Copy magic to the user space after saving  all vector conetext */
-	err |= __put_user(RISCV_V_MAGIC, &hdr->magic);
-	err |= __put_user(riscv_v_sc_size, &hdr->size);
 	if (unlikely(err))
-		return err;
+		return -EFAULT;
 
-	/* Only progress the sv_vec if everything has done successfully  */
-	*sc_vec += riscv_v_sc_size;
-	return 0;
+	/* Only return the size if everything has done successfully  */
+	return riscv_v_sc_size;
 }
 
 /*
@@ -142,10 +141,80 @@ static long __restore_v_state(struct pt_regs *regs, void __user *sc_vec)
 	 */
 	return copy_from_user(current->thread.vstate.datap, datap, riscv_v_vsize);
 }
-#else
-#define save_v_state(task, regs) (0)
-#define __restore_v_state(task, regs) (0)
-#endif
+
+static long save_cfiss_state(struct pt_regs *regs, void __user *sc_cfi)
+{
+	struct __sc_riscv_cfi_state __user *state = sc_cfi;
+	unsigned long ss_ptr = 0;
+	long err = 0;
+
+	if (!is_shstk_enabled(current))
+		return 0;
+
+	/*
+	 * Save a pointer to shadow stack itself on shadow stack as a form of token.
+	 * A token on shadow gives following properties
+	 * - Safe save and restore for shadow stack switching. Any save of shadow stack
+	 *   must have had saved a token on shadow stack. Similarly any restore of shadow
+	 *   stack must check the token before restore. Since writing to shadow stack with
+	 *   address of shadow stack itself is not easily allowed. A restore without a save
+	 *   is quite difficult for an attacker to perform.
+	 * - A natural break. A token in shadow stack provides a natural break in shadow stack
+	 *   So a single linear range can be bucketed into different shadow stack segments. Any
+	 *   sspopchk will detect the condition and fault to kernel as sw check exception.
+	 */
+	err |= save_user_shstk(current, &ss_ptr);
+	err |= __put_user(ss_ptr, &state->ss_ptr);
+	if (unlikely(err))
+		return -EFAULT;
+
+	return riscv_zicfiss_sc_size;
+}
+
+static long __restore_cfiss_state(struct pt_regs *regs, void __user *sc_cfi)
+{
+	struct __sc_riscv_cfi_state __user *state = sc_cfi;
+	unsigned long ss_ptr = 0;
+	long err;
+
+	/*
+	 * Restore shadow stack as a form of token stored on shadow stack itself as a safe
+	 * way to restore.
+	 * A token on shadow gives following properties
+	 * - Safe save and restore for shadow stack switching. Any save of shadow stack
+	 *   must have had saved a token on shadow stack. Similarly any restore of shadow
+	 *   stack must check the token before restore. Since writing to shadow stack with
+	 *   address of shadow stack itself is not easily allowed. A restore without a save
+	 *   is quite difficult for an attacker to perform.
+	 * - A natural break. A token in shadow stack provides a natural break in shadow stack
+	 *   So a single linear range can be bucketed into different shadow stack segments.
+	 *   sspopchk will detect the condition and fault to kernel as sw check exception.
+	 */
+	err = __copy_from_user(&ss_ptr, &state->ss_ptr, sizeof(unsigned long));
+
+	if (unlikely(err))
+		return err;
+
+	return restore_user_shstk(current, ss_ptr);
+}
+
+struct arch_ext_priv {
+	__u32 magic;
+	long (*save)(struct pt_regs *regs, void __user *sc_vec);
+};
+
+struct arch_ext_priv arch_ext_list[] = {
+	{
+		.magic = RISCV_V_MAGIC,
+		.save = &save_v_state,
+	},
+	{
+		.magic = RISCV_ZICFISS_MAGIC,
+		.save = &save_cfiss_state,
+	},
+};
+
+const size_t nr_arch_exts = ARRAY_SIZE(arch_ext_list);
 
 static long restore_sigcontext(struct pt_regs *regs,
 	struct sigcontext __user *sc)
@@ -195,6 +264,12 @@ static long restore_sigcontext(struct pt_regs *regs,
 
 			err = __restore_v_state(regs, sc_ext_ptr);
 			break;
+		case RISCV_ZICFISS_MAGIC:
+			if (!is_shstk_enabled(current) || size != riscv_zicfiss_sc_size)
+				return -EINVAL;
+
+			err = __restore_cfiss_state(regs, sc_ext_ptr);
+			break;
 		default:
 			return -EINVAL;
 		}
@@ -216,6 +291,16 @@ static size_t get_rt_frame_size(bool cal_all)
 			total_context_size += riscv_v_sc_size;
 	}
 
+	if (is_shstk_enabled(current))
+		total_context_size += riscv_zicfiss_sc_size;
+
+	/*
+	 * Preserved a __riscv_ctx_hdr for END signal context header if an
+	 * extension uses __riscv_extra_ext_header
+	 */
+	if (total_context_size)
+		total_context_size += sizeof(struct __riscv_ctx_hdr);
+
 	frame_size += total_context_size;
 
 	frame_size = round_up(frame_size, 16);
@@ -270,7 +355,8 @@ static long setup_sigcontext(struct rt_sigframe __user *frame,
 {
 	struct sigcontext __user *sc = &frame->uc.uc_mcontext;
 	struct __riscv_ctx_hdr __user *sc_ext_ptr = &sc->sc_extdesc.hdr;
-	long err;
+	struct arch_ext_priv *arch_ext;
+	long err, i, ext_size;
 
 	/* sc_regs is structured the same as the start of pt_regs */
 	err = __copy_to_user(&sc->sc_regs, regs, sizeof(sc->sc_regs));
@@ -278,8 +364,20 @@ static long setup_sigcontext(struct rt_sigframe __user *frame,
 	if (has_fpu())
 		err |= save_fp_state(regs, &sc->sc_fpregs);
 	/* Save the vector state. */
-	if ((has_vector() || has_xtheadvector()) && riscv_v_vstate_query(regs))
-		err |= save_v_state(regs, (void __user **)&sc_ext_ptr);
+	for (i = 0; i < nr_arch_exts; i++) {
+		arch_ext = &arch_ext_list[i];
+		if (!arch_ext->save)
+			continue;
+
+		ext_size = arch_ext->save(regs, sc_ext_ptr + 1);
+		if (ext_size <= 0) {
+			err |= ext_size;
+		} else {
+			err |= __put_user(arch_ext->magic, &sc_ext_ptr->magic);
+			err |= __put_user(ext_size, &sc_ext_ptr->size);
+			sc_ext_ptr = (void *)sc_ext_ptr + ext_size;
+		}
+	}
 	/* Write zero to fp-reserved space and check it on restore_sigcontext */
 	err |= __put_user(0, &sc->sc_extdesc.reserved);
 	/* And put END __riscv_ctx_hdr at the end. */
@@ -339,6 +437,11 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
 #ifdef CONFIG_MMU
 	regs->ra = (unsigned long)VDSO_SYMBOL(
 		current->mm->context.vdso, rt_sigreturn);
+
+	/* if bcfi is enabled x1 (ra) and x5 (t0) must match. not sure if we need this? */
+	if (is_shstk_enabled(current))
+		regs->t0 = regs->ra;
+
 #else
 	/*
 	 * For the nommu case we don't have a VDSO.  Instead we push two
@@ -467,6 +570,9 @@ void __init init_rt_signal_env(void)
 {
 	riscv_v_sc_size = sizeof(struct __riscv_ctx_hdr) +
 			  sizeof(struct __sc_riscv_v_state) + riscv_v_vsize;
+
+	riscv_zicfiss_sc_size = sizeof(struct __riscv_ctx_hdr) +
+			  sizeof(struct __sc_riscv_cfi_state);
 	/*
 	 * Determine the stack space required for guaranteed signal delivery.
 	 * The signal_minsigstksz will be populated into the AT_MINSIGSTKSZ entry
diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c
index 0f701ace3bb9..1659d31fd288 100644
--- a/arch/riscv/kernel/sys_hwprobe.c
+++ b/arch/riscv/kernel/sys_hwprobe.c
@@ -24,6 +24,14 @@
 #include <vdso/vsyscall.h>
 
 
+#define EXT_KEY(isa_arg, ext, pv, missing)					\
+	do {										\
+		if (__riscv_isa_extension_available(isa_arg, RISCV_ISA_EXT_##ext))	\
+			pv |= RISCV_HWPROBE_EXT_##ext;				\
+		else									\
+			missing |= RISCV_HWPROBE_EXT_##ext;				\
+	} while (false)
+
 static void hwprobe_arch_id(struct riscv_hwprobe *pair,
 			    const struct cpumask *cpus)
 {
@@ -93,88 +101,110 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair,
 	for_each_cpu(cpu, cpus) {
 		struct riscv_isainfo *isainfo = &hart_isa[cpu];
 
-#define EXT_KEY(ext)									\
-	do {										\
-		if (__riscv_isa_extension_available(isainfo->isa, RISCV_ISA_EXT_##ext))	\
-			pair->value |= RISCV_HWPROBE_EXT_##ext;				\
-		else									\
-			missing |= RISCV_HWPROBE_EXT_##ext;				\
-	} while (false)
-
 		/*
 		 * Only use EXT_KEY() for extensions which can be exposed to userspace,
 		 * regardless of the kernel's configuration, as no other checks, besides
 		 * presence in the hart_isa bitmap, are made.
 		 */
-		EXT_KEY(ZAAMO);
-		EXT_KEY(ZABHA);
-		EXT_KEY(ZACAS);
-		EXT_KEY(ZALASR);
-		EXT_KEY(ZALRSC);
-		EXT_KEY(ZAWRS);
-		EXT_KEY(ZBA);
-		EXT_KEY(ZBB);
-		EXT_KEY(ZBC);
-		EXT_KEY(ZBKB);
-		EXT_KEY(ZBKC);
-		EXT_KEY(ZBKX);
-		EXT_KEY(ZBS);
-		EXT_KEY(ZCA);
-		EXT_KEY(ZCB);
-		EXT_KEY(ZCMOP);
-		EXT_KEY(ZICBOM);
-		EXT_KEY(ZICBOP);
-		EXT_KEY(ZICBOZ);
-		EXT_KEY(ZICNTR);
-		EXT_KEY(ZICOND);
-		EXT_KEY(ZIHINTNTL);
-		EXT_KEY(ZIHINTPAUSE);
-		EXT_KEY(ZIHPM);
-		EXT_KEY(ZIMOP);
-		EXT_KEY(ZKND);
-		EXT_KEY(ZKNE);
-		EXT_KEY(ZKNH);
-		EXT_KEY(ZKSED);
-		EXT_KEY(ZKSH);
-		EXT_KEY(ZKT);
-		EXT_KEY(ZTSO);
+		EXT_KEY(isainfo->isa, ZAAMO, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZABHA, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZACAS, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZALASR, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZALRSC, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZAWRS, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZBA, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZBB, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZBC, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZBKB, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZBKC, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZBKX, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZBS, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZCA, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZCB, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZCLSD, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZCMOP, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZICBOM, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZICBOP, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZICBOZ, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZICFILP, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZICNTR, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZICOND, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZIHINTNTL, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZIHINTPAUSE, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZIHPM, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZILSD, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZIMOP, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZKND, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZKNE, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZKNH, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZKSED, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZKSH, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZKT, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZTSO, pair->value, missing);
 
 		/*
 		 * All the following extensions must depend on the kernel
 		 * support of V.
 		 */
 		if (has_vector()) {
-			EXT_KEY(ZVBB);
-			EXT_KEY(ZVBC);
-			EXT_KEY(ZVE32F);
-			EXT_KEY(ZVE32X);
-			EXT_KEY(ZVE64D);
-			EXT_KEY(ZVE64F);
-			EXT_KEY(ZVE64X);
-			EXT_KEY(ZVFBFMIN);
-			EXT_KEY(ZVFBFWMA);
-			EXT_KEY(ZVFH);
-			EXT_KEY(ZVFHMIN);
-			EXT_KEY(ZVKB);
-			EXT_KEY(ZVKG);
-			EXT_KEY(ZVKNED);
-			EXT_KEY(ZVKNHA);
-			EXT_KEY(ZVKNHB);
-			EXT_KEY(ZVKSED);
-			EXT_KEY(ZVKSH);
-			EXT_KEY(ZVKT);
+			EXT_KEY(isainfo->isa, ZVBB, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVBC, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVE32F, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVE32X, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVE64D, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVE64F, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVE64X, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVFBFMIN, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVFBFWMA, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVFH, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVFHMIN, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVKB, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVKG, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVKNED, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVKNHA, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVKNHB, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVKSED, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVKSH, pair->value, missing);
+			EXT_KEY(isainfo->isa, ZVKT, pair->value, missing);
 		}
 
-		EXT_KEY(ZCD);
-		EXT_KEY(ZCF);
-		EXT_KEY(ZFA);
-		EXT_KEY(ZFBFMIN);
-		EXT_KEY(ZFH);
-		EXT_KEY(ZFHMIN);
+		EXT_KEY(isainfo->isa, ZCD, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZCF, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZFA, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZFBFMIN, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZFH, pair->value, missing);
+		EXT_KEY(isainfo->isa, ZFHMIN, pair->value, missing);
 
 		if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM))
-			EXT_KEY(SUPM);
-#undef EXT_KEY
+			EXT_KEY(isainfo->isa, SUPM, pair->value, missing);
+	}
+
+	/* Now turn off reporting features if any CPU is missing it. */
+	pair->value &= ~missing;
+}
+
+static void hwprobe_isa_ext1(struct riscv_hwprobe *pair,
+			     const struct cpumask *cpus)
+{
+	int cpu;
+	u64 missing = 0;
+
+	pair->value = 0;
+
+	/*
+	 * Loop through and record extensions that 1) anyone has, and 2) anyone
+	 * doesn't have.
+	 */
+	for_each_cpu(cpu, cpus) {
+		struct riscv_isainfo *isainfo = &hart_isa[cpu];
+
+		/*
+		 * Only use EXT_KEY() for extensions which can be
+		 * exposed to userspace, regardless of the kernel's
+		 * configuration, as no other checks, besides presence
+		 * in the hart_isa bitmap, are made.
+		 */
+		EXT_KEY(isainfo->isa, ZICFISS, pair->value, missing);
 	}
 
 	/* Now turn off reporting features if any CPU is missing it. */
@@ -285,6 +315,10 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair,
 		hwprobe_isa_ext0(pair, cpus);
 		break;
 
+	case RISCV_HWPROBE_KEY_IMA_EXT_1:
+		hwprobe_isa_ext1(pair, cpus);
+		break;
+
 	case RISCV_HWPROBE_KEY_CPUPERF_0:
 	case RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF:
 		pair->value = hwprobe_misaligned(cpus);
diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c
index 795b2e815ac9..22fc9b3268be 100644
--- a/arch/riscv/kernel/sys_riscv.c
+++ b/arch/riscv/kernel/sys_riscv.c
@@ -7,6 +7,7 @@
 
 #include <linux/syscalls.h>
 #include <asm/cacheflush.h>
+#include <asm-generic/mman-common.h>
 
 static long riscv_sys_mmap(unsigned long addr, unsigned long len,
 			   unsigned long prot, unsigned long flags,
@@ -16,6 +17,15 @@ static long riscv_sys_mmap(unsigned long addr, unsigned long len,
 	if (unlikely(offset & (~PAGE_MASK >> page_shift_offset)))
 		return -EINVAL;
 
+	/*
+	 * If PROT_WRITE is specified then extend that to PROT_READ
+	 * protection_map[VM_WRITE] is now going to select shadow stack encodings.
+	 * So specifying PROT_WRITE actually should select protection_map [VM_WRITE | VM_READ]
+	 * If user wants to create shadow stack then they should use `map_shadow_stack` syscall.
+	 */
+	if (unlikely((prot & PROT_WRITE) && !(prot & PROT_READ)))
+		prot |= PROT_READ;
+
 	return ksys_mmap_pgoff(addr, len, prot, flags, fd,
 			       offset >> (PAGE_SHIFT - page_shift_offset));
 }
diff --git a/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S b/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S
index b951d0f12482..ac5ce305b1bd 100644
--- a/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S
+++ b/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S
@@ -7,8 +7,10 @@
 SYM_FUNC_START(test_kprobes_add)
 	li a1, KPROBE_TEST_MAGIC_UPPER
 	li a2, KPROBE_TEST_MAGIC_LOWER
+.globl test_kprobes_add_addr1
 test_kprobes_add_addr1:
 	add a1, a1, a2
+.globl test_kprobes_add_addr2
 test_kprobes_add_addr2:
 	add a0, a1, x0
 	ret
@@ -19,6 +21,7 @@ SYM_FUNC_START(test_kprobes_jal)
 	mv a1, ra
 	.option push
 	.option norvc
+.globl test_kprobes_jal_addr1
 test_kprobes_jal_addr1:
 	jal x0, 2f
 	ret
@@ -27,6 +30,7 @@ test_kprobes_jal_addr1:
 	ret
 	.option push
 	.option norvc
+.globl test_kprobes_jal_addr2
 test_kprobes_jal_addr2:
 2:	jal 1b
 	.option pop
@@ -40,6 +44,7 @@ SYM_FUNC_START(test_kprobes_jalr)
 	mv a1, ra
 	.option push
 	.option norvc
+.globl test_kprobes_jalr_addr
 test_kprobes_jalr_addr:
 	jalr a0
 	.option pop
@@ -51,6 +56,7 @@ test_kprobes_jalr_addr:
 SYM_FUNC_END(test_kprobes_jalr)
 
 SYM_FUNC_START(test_kprobes_auipc)
+.globl test_kprobes_auipc_addr
 test_kprobes_auipc_addr:
 	auipc a0, KPROBE_TEST_MAGIC_LOWER
 	la a1, test_kprobes_auipc_addr
@@ -67,20 +73,26 @@ SYM_FUNC_START(test_kprobes_branch)
 	li a0, 0
 	li a1, 1
 	li a2, 2
+.globl test_kprobes_branch_addr1
 test_kprobes_branch_addr1:
 	beqz a0, 1f
 	ret
 1:
+.globl test_kprobes_branch_addr2
 test_kprobes_branch_addr2:
 	beqz a1, 3f
+.globl test_kprobes_branch_addr3
 test_kprobes_branch_addr3:
 	bnez a0, 3f
+.globl test_kprobes_branch_addr4
 test_kprobes_branch_addr4:
 	bnez a2, 1f
 	ret
 1:
+.globl test_kprobes_branch_addr5
 test_kprobes_branch_addr5:
 	bge a1, a2, 3f
+.globl test_kprobes_branch_addr6
 test_kprobes_branch_addr6:
 	bge a2, a1, 2f
 	ret
@@ -89,9 +101,11 @@ test_kprobes_branch_addr6:
 	add a0, a0, t0
 	ret
 2:
+.globl test_kprobes_branch_addr7
 test_kprobes_branch_addr7:
 	blt a2, a1, 3f
 	li a0, KPROBE_TEST_MAGIC_LOWER
+.globl test_kprobes_branch_addr8
 test_kprobes_branch_addr8:
 	blt a1, a2, 1b
 3:
@@ -104,6 +118,7 @@ SYM_FUNC_END(test_kprobes_branch)
 
 SYM_FUNC_START(test_kprobes_c_j)
 	li a0, 0
+.globl test_kprobes_branch_c_j_addr1
 test_kprobes_branch_c_j_addr1:
 	c.j 2f
 1:
@@ -111,12 +126,14 @@ test_kprobes_branch_c_j_addr1:
 	add a0, a0, a1
 	ret
 2:	li a0, KPROBE_TEST_MAGIC_LOWER
+.globl test_kprobes_branch_c_j_addr2
 test_kprobes_branch_c_j_addr2:
 	c.j 1b
 SYM_FUNC_END(test_kprobes_c_j)
 
 SYM_FUNC_START(test_kprobes_c_jr)
 	la a0, 2f
+.globl test_kprobes_c_jr_addr1
 test_kprobes_c_jr_addr1:
 	c.jr a0
 	ret
@@ -126,6 +143,7 @@ test_kprobes_c_jr_addr1:
 2:
 	li a0, KPROBE_TEST_MAGIC_UPPER
 	la a1, 1b
+.globl test_kprobes_c_jr_addr2
 test_kprobes_c_jr_addr2:
 	c.jr a1
 SYM_FUNC_END(test_kprobes_c_jr)
@@ -133,6 +151,7 @@ SYM_FUNC_END(test_kprobes_c_jr)
 SYM_FUNC_START(test_kprobes_c_jalr)
 	mv a1, ra
 	la a0, 1f
+.globl test_kprobes_c_jalr_addr
 test_kprobes_c_jalr_addr:
 	c.jalr a0
 	li a2, KPROBE_TEST_MAGIC_UPPER
@@ -145,16 +164,19 @@ SYM_FUNC_END(test_kprobes_c_jalr)
 SYM_FUNC_START(test_kprobes_c_beqz)
 	li a0, 0
 	li a1, 1
+.globl test_kprobes_c_beqz_addr1
 test_kprobes_c_beqz_addr1:
 	c.beqz a0, 2f
 	ret
 1:	li a1, KPROBE_TEST_MAGIC_UPPER
 	add a0, a0, a1
 	ret
+.globl test_kprobes_c_beqz_addr2
 test_kprobes_c_beqz_addr2:
 2:	c.beqz a1, 3f
 	li a0, KPROBE_TEST_MAGIC_LOWER
 	mv a1, x0
+.globl test_kprobes_c_beqz_addr3
 test_kprobes_c_beqz_addr3:
 	c.beqz a1, 1b
 3:	li a0, 0
@@ -164,15 +186,18 @@ SYM_FUNC_END(test_kprobes_c_beqz)
 SYM_FUNC_START(test_kprobes_c_bnez)
 	li a0, 0
 	li a1, 1
+.globl test_kprobes_c_bnez_addr1
 test_kprobes_c_bnez_addr1:
 	c.bnez a1, 2f
 	ret
 1:	li a1, KPROBE_TEST_MAGIC_UPPER
 	add a0, a0, a1
 	ret
+.globl test_kprobes_c_bnez_addr2
 test_kprobes_c_bnez_addr2:
 2:	c.bnez a0, 3f
 	li a0, KPROBE_TEST_MAGIC_LOWER
+.globl test_kprobes_c_bnez_addr3
 test_kprobes_c_bnez_addr3:
 	c.bnez a0, 1b
 3:	li a0, 0
@@ -181,6 +206,8 @@ SYM_FUNC_END(test_kprobes_c_bnez)
 
 #endif /* CONFIG_RISCV_ISA_C */
 
+.section .rodata
+.align 3
 SYM_DATA_START(test_kprobes_addresses)
 	RISCV_PTR test_kprobes_add_addr1
 	RISCV_PTR test_kprobes_add_addr2
@@ -212,6 +239,8 @@ SYM_DATA_START(test_kprobes_addresses)
 	RISCV_PTR 0
 SYM_DATA_END(test_kprobes_addresses)
 
+.section .rodata
+.align 3
 SYM_DATA_START(test_kprobes_functions)
 	RISCV_PTR test_kprobes_add
 	RISCV_PTR test_kprobes_jal
diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c
index 80230de167de..8e4a4d5d0f47 100644
--- a/arch/riscv/kernel/traps.c
+++ b/arch/riscv/kernel/traps.c
@@ -366,6 +366,60 @@ void do_trap_ecall_u(struct pt_regs *regs)
 
 }
 
+#define CFI_TVAL_FCFI_CODE	2
+#define CFI_TVAL_BCFI_CODE	3
+/* handle cfi violations */
+bool handle_user_cfi_violation(struct pt_regs *regs)
+{
+	unsigned long tval = csr_read(CSR_TVAL);
+	bool is_fcfi = (tval == CFI_TVAL_FCFI_CODE && cpu_supports_indirect_br_lp_instr());
+	bool is_bcfi = (tval == CFI_TVAL_BCFI_CODE && cpu_supports_shadow_stack());
+
+	/*
+	 * Handle uprobe event first. The probe point can be a valid target
+	 * of indirect jumps or calls, in this case, forward cfi violation
+	 * will be triggered instead of breakpoint exception. Clear ELP flag
+	 * on sstatus image as well to avoid recurring fault.
+	 */
+	if (is_fcfi && probe_breakpoint_handler(regs)) {
+		regs->status &= ~SR_ELP;
+		return true;
+	}
+
+	if (is_fcfi || is_bcfi) {
+		do_trap_error(regs, SIGSEGV, SEGV_CPERR, regs->epc,
+			      "Oops - control flow violation");
+		return true;
+	}
+
+	return false;
+}
+
+/*
+ * software check exception is defined with risc-v cfi spec. Software check
+ * exception is raised when:
+ * a) An indirect branch doesn't land on 4 byte aligned PC or `lpad`
+ *    instruction or `label` value programmed in `lpad` instr doesn't
+ *    match with value setup in `x7`. reported code in `xtval` is 2.
+ * b) `sspopchk` instruction finds a mismatch between top of shadow stack (ssp)
+ *    and x1/x5. reported code in `xtval` is 3.
+ */
+asmlinkage __visible __trap_section void do_trap_software_check(struct pt_regs *regs)
+{
+	if (user_mode(regs)) {
+		irqentry_enter_from_user_mode(regs);
+
+		/* not a cfi violation, then merge into flow of unknown trap handler */
+		if (!handle_user_cfi_violation(regs))
+			do_trap_unknown(regs);
+
+		irqentry_exit_to_user_mode(regs);
+	} else {
+		/* sw check exception coming from kernel is a bug in kernel */
+		die(regs, "Kernel BUG");
+	}
+}
+
 #ifdef CONFIG_MMU
 asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
 {
diff --git a/arch/riscv/kernel/usercfi.c b/arch/riscv/kernel/usercfi.c
new file mode 100644
index 000000000000..afe7f4bc0034
--- /dev/null
+++ b/arch/riscv/kernel/usercfi.c
@@ -0,0 +1,544 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2024 Rivos, Inc.
+ * Deepak Gupta <debug@rivosinc.com>
+ */
+
+#include <linux/sched.h>
+#include <linux/bitops.h>
+#include <linux/types.h>
+#include <linux/mm.h>
+#include <linux/mman.h>
+#include <linux/uaccess.h>
+#include <linux/sizes.h>
+#include <linux/user.h>
+#include <linux/syscalls.h>
+#include <linux/prctl.h>
+#include <asm/csr.h>
+#include <asm/usercfi.h>
+
+unsigned long riscv_nousercfi __read_mostly;
+
+#define SHSTK_ENTRY_SIZE sizeof(void *)
+
+bool is_shstk_enabled(struct task_struct *task)
+{
+	return task->thread_info.user_cfi_state.ubcfi_en;
+}
+
+bool is_shstk_allocated(struct task_struct *task)
+{
+	return task->thread_info.user_cfi_state.shdw_stk_base;
+}
+
+bool is_shstk_locked(struct task_struct *task)
+{
+	return task->thread_info.user_cfi_state.ubcfi_locked;
+}
+
+void set_shstk_base(struct task_struct *task, unsigned long shstk_addr, unsigned long size)
+{
+	task->thread_info.user_cfi_state.shdw_stk_base = shstk_addr;
+	task->thread_info.user_cfi_state.shdw_stk_size = size;
+}
+
+unsigned long get_shstk_base(struct task_struct *task, unsigned long *size)
+{
+	if (size)
+		*size = task->thread_info.user_cfi_state.shdw_stk_size;
+	return task->thread_info.user_cfi_state.shdw_stk_base;
+}
+
+void set_active_shstk(struct task_struct *task, unsigned long shstk_addr)
+{
+	task->thread_info.user_cfi_state.user_shdw_stk = shstk_addr;
+}
+
+unsigned long get_active_shstk(struct task_struct *task)
+{
+	return task->thread_info.user_cfi_state.user_shdw_stk;
+}
+
+void set_shstk_status(struct task_struct *task, bool enable)
+{
+	if (!is_user_shstk_enabled())
+		return;
+
+	task->thread_info.user_cfi_state.ubcfi_en = enable ? 1 : 0;
+
+	if (enable)
+		task->thread.envcfg |= ENVCFG_SSE;
+	else
+		task->thread.envcfg &= ~ENVCFG_SSE;
+
+	csr_write(CSR_ENVCFG, task->thread.envcfg);
+}
+
+void set_shstk_lock(struct task_struct *task)
+{
+	task->thread_info.user_cfi_state.ubcfi_locked = 1;
+}
+
+bool is_indir_lp_enabled(struct task_struct *task)
+{
+	return task->thread_info.user_cfi_state.ufcfi_en;
+}
+
+bool is_indir_lp_locked(struct task_struct *task)
+{
+	return task->thread_info.user_cfi_state.ufcfi_locked;
+}
+
+void set_indir_lp_status(struct task_struct *task, bool enable)
+{
+	if (!is_user_lpad_enabled())
+		return;
+
+	task->thread_info.user_cfi_state.ufcfi_en = enable ? 1 : 0;
+
+	if (enable)
+		task->thread.envcfg |= ENVCFG_LPE;
+	else
+		task->thread.envcfg &= ~ENVCFG_LPE;
+
+	csr_write(CSR_ENVCFG, task->thread.envcfg);
+}
+
+void set_indir_lp_lock(struct task_struct *task)
+{
+	task->thread_info.user_cfi_state.ufcfi_locked = 1;
+}
+/*
+ * If size is 0, then to be compatible with regular stack we want it to be as big as
+ * regular stack. Else PAGE_ALIGN it and return back
+ */
+static unsigned long calc_shstk_size(unsigned long size)
+{
+	if (size)
+		return PAGE_ALIGN(size);
+
+	return PAGE_ALIGN(min_t(unsigned long long, rlimit(RLIMIT_STACK), SZ_4G));
+}
+
+/*
+ * Writes on shadow stack can either be `sspush` or `ssamoswap`. `sspush` can happen
+ * implicitly on current shadow stack pointed to by CSR_SSP. `ssamoswap` takes pointer to
+ * shadow stack. To keep it simple, we plan to use `ssamoswap` to perform writes on shadow
+ * stack.
+ */
+static noinline unsigned long amo_user_shstk(unsigned long *addr, unsigned long val)
+{
+	/*
+	 * Never expect -1 on shadow stack. Expect return addresses and zero
+	 */
+	unsigned long swap = -1;
+
+	__enable_user_access();
+	asm goto(".option push\n"
+		".option arch, +zicfiss\n"
+		"1: ssamoswap.d %[swap], %[val], %[addr]\n"
+		_ASM_EXTABLE(1b, %l[fault])
+		".option pop\n"
+		: [swap] "=r" (swap), [addr] "+A" (*addr)
+		: [val] "r" (val)
+		: "memory"
+		: fault
+		);
+	__disable_user_access();
+	return swap;
+fault:
+	__disable_user_access();
+	return -1;
+}
+
+/*
+ * Create a restore token on the shadow stack.  A token is always XLEN wide
+ * and aligned to XLEN.
+ */
+static int create_rstor_token(unsigned long ssp, unsigned long *token_addr)
+{
+	unsigned long addr;
+
+	/* Token must be aligned */
+	if (!IS_ALIGNED(ssp, SHSTK_ENTRY_SIZE))
+		return -EINVAL;
+
+	/* On RISC-V we're constructing token to be function of address itself */
+	addr = ssp - SHSTK_ENTRY_SIZE;
+
+	if (amo_user_shstk((unsigned long __user *)addr, (unsigned long)ssp) == -1)
+		return -EFAULT;
+
+	if (token_addr)
+		*token_addr = addr;
+
+	return 0;
+}
+
+/*
+ * Save user shadow stack pointer on shadow stack itself and return pointer to saved location
+ * returns -EFAULT if operation was unsuccessful
+ */
+int save_user_shstk(struct task_struct *tsk, unsigned long *saved_shstk_ptr)
+{
+	unsigned long ss_ptr = 0;
+	unsigned long token_loc = 0;
+	int ret = 0;
+
+	if (!saved_shstk_ptr)
+		return -EINVAL;
+
+	ss_ptr = get_active_shstk(tsk);
+	ret = create_rstor_token(ss_ptr, &token_loc);
+
+	if (!ret) {
+		*saved_shstk_ptr = token_loc;
+		set_active_shstk(tsk, token_loc);
+	}
+
+	return ret;
+}
+
+/*
+ * Restores user shadow stack pointer from token on shadow stack for task `tsk`
+ * returns -EFAULT if operation was unsuccessful
+ */
+int restore_user_shstk(struct task_struct *tsk, unsigned long shstk_ptr)
+{
+	unsigned long token = 0;
+
+	token = amo_user_shstk((unsigned long __user *)shstk_ptr, 0);
+
+	if (token == -1)
+		return -EFAULT;
+
+	/* invalid token, return EINVAL */
+	if ((token - shstk_ptr) != SHSTK_ENTRY_SIZE) {
+		pr_info_ratelimited("%s[%d]: bad restore token in %s: pc=%p sp=%p, token=%p, shstk_ptr=%p\n",
+				    tsk->comm, task_pid_nr(tsk), __func__,
+				    (void *)(task_pt_regs(tsk)->epc),
+				    (void *)(task_pt_regs(tsk)->sp),
+				    (void *)token, (void *)shstk_ptr);
+		return -EINVAL;
+	}
+
+	/* all checks passed, set active shstk and return success */
+	set_active_shstk(tsk, token);
+	return 0;
+}
+
+static unsigned long allocate_shadow_stack(unsigned long addr, unsigned long size,
+					   unsigned long token_offset, bool set_tok)
+{
+	int flags = MAP_ANONYMOUS | MAP_PRIVATE;
+	struct mm_struct *mm = current->mm;
+	unsigned long populate, tok_loc = 0;
+
+	if (addr)
+		flags |= MAP_FIXED_NOREPLACE;
+
+	mmap_write_lock(mm);
+	addr = do_mmap(NULL, addr, size, PROT_READ, flags,
+		       VM_SHADOW_STACK | VM_WRITE, 0, &populate, NULL);
+	mmap_write_unlock(mm);
+
+	if (!set_tok || IS_ERR_VALUE(addr))
+		goto out;
+
+	if (create_rstor_token(addr + token_offset, &tok_loc)) {
+		vm_munmap(addr, size);
+		return -EINVAL;
+	}
+
+	addr = tok_loc;
+
+out:
+	return addr;
+}
+
+SYSCALL_DEFINE3(map_shadow_stack, unsigned long, addr, unsigned long, size, unsigned int, flags)
+{
+	bool set_tok = flags & SHADOW_STACK_SET_TOKEN;
+	unsigned long aligned_size = 0;
+
+	if (!is_user_shstk_enabled())
+		return -EOPNOTSUPP;
+
+	/* Anything other than set token should result in invalid param */
+	if (flags & ~SHADOW_STACK_SET_TOKEN)
+		return -EINVAL;
+
+	/*
+	 * Unlike other architectures, on RISC-V, SSP pointer is held in CSR_SSP and is an available
+	 * CSR in all modes. CSR accesses are performed using 12bit index programmed in instruction
+	 * itself. This provides static property on register programming and writes to CSR can't
+	 * be unintentional from programmer's perspective. As long as programmer has guarded areas
+	 * which perform writes to CSR_SSP properly, shadow stack pivoting is not possible. Since
+	 * CSR_SSP is writable by user mode, it itself can setup a shadow stack token subsequent
+	 * to allocation. Although in order to provide portablity with other architectures (because
+	 * `map_shadow_stack` is arch agnostic syscall), RISC-V will follow expectation of a token
+	 * flag in flags and if provided in flags, will setup a token at the base.
+	 */
+
+	/* If there isn't space for a token */
+	if (set_tok && size < SHSTK_ENTRY_SIZE)
+		return -ENOSPC;
+
+	if (addr && (addr & (PAGE_SIZE - 1)))
+		return -EINVAL;
+
+	aligned_size = PAGE_ALIGN(size);
+	if (aligned_size < size)
+		return -EOVERFLOW;
+
+	return allocate_shadow_stack(addr, aligned_size, size, set_tok);
+}
+
+/*
+ * This gets called during clone/clone3/fork. And is needed to allocate a shadow stack for
+ * cases where CLONE_VM is specified and thus a different stack is specified by user. We
+ * thus need a separate shadow stack too. How a separate shadow stack is specified by
+ * user is still being debated. Once that's settled, remove this part of the comment.
+ * This function simply returns 0 if shadow stacks are not supported or if separate shadow
+ * stack allocation is not needed (like in case of !CLONE_VM)
+ */
+unsigned long shstk_alloc_thread_stack(struct task_struct *tsk,
+				       const struct kernel_clone_args *args)
+{
+	unsigned long addr, size;
+
+	/* If shadow stack is not supported, return 0 */
+	if (!is_user_shstk_enabled())
+		return 0;
+
+	/*
+	 * If shadow stack is not enabled on the new thread, skip any
+	 * switch to a new shadow stack.
+	 */
+	if (!is_shstk_enabled(tsk))
+		return 0;
+
+	/*
+	 * For CLONE_VFORK the child will share the parents shadow stack.
+	 * Set base = 0 and size = 0, this is special means to track this state
+	 * so the freeing logic run for child knows to leave it alone.
+	 */
+	if (args->flags & CLONE_VFORK) {
+		set_shstk_base(tsk, 0, 0);
+		return 0;
+	}
+
+	/*
+	 * For !CLONE_VM the child will use a copy of the parents shadow
+	 * stack.
+	 */
+	if (!(args->flags & CLONE_VM))
+		return 0;
+
+	/*
+	 * reaching here means, CLONE_VM was specified and thus a separate shadow
+	 * stack is needed for new cloned thread. Note: below allocation is happening
+	 * using current mm.
+	 */
+	size = calc_shstk_size(args->stack_size);
+	addr = allocate_shadow_stack(0, size, 0, false);
+	if (IS_ERR_VALUE(addr))
+		return addr;
+
+	set_shstk_base(tsk, addr, size);
+
+	return addr + size;
+}
+
+void shstk_release(struct task_struct *tsk)
+{
+	unsigned long base = 0, size = 0;
+	/* If shadow stack is not supported or not enabled, nothing to release */
+	if (!is_user_shstk_enabled() || !is_shstk_enabled(tsk))
+		return;
+
+	/*
+	 * When fork() with CLONE_VM fails, the child (tsk) already has a
+	 * shadow stack allocated, and exit_thread() calls this function to
+	 * free it.  In this case the parent (current) and the child share
+	 * the same mm struct. Move forward only when they're same.
+	 */
+	if (!tsk->mm || tsk->mm != current->mm)
+		return;
+
+	/*
+	 * We know shadow stack is enabled but if base is NULL, then
+	 * this task is not managing its own shadow stack (CLONE_VFORK). So
+	 * skip freeing it.
+	 */
+	base = get_shstk_base(tsk, &size);
+	if (!base)
+		return;
+
+	vm_munmap(base, size);
+	set_shstk_base(tsk, 0, 0);
+}
+
+int arch_get_shadow_stack_status(struct task_struct *t, unsigned long __user *status)
+{
+	unsigned long bcfi_status = 0;
+
+	if (!is_user_shstk_enabled())
+		return -EINVAL;
+
+	/* this means shadow stack is enabled on the task */
+	bcfi_status |= (is_shstk_enabled(t) ? PR_SHADOW_STACK_ENABLE : 0);
+
+	return copy_to_user(status, &bcfi_status, sizeof(bcfi_status)) ? -EFAULT : 0;
+}
+
+int arch_set_shadow_stack_status(struct task_struct *t, unsigned long status)
+{
+	unsigned long size = 0, addr = 0;
+	bool enable_shstk = false;
+
+	if (!is_user_shstk_enabled())
+		return -EINVAL;
+
+	/* Reject unknown flags */
+	if (status & ~PR_SHADOW_STACK_SUPPORTED_STATUS_MASK)
+		return -EINVAL;
+
+	/* bcfi status is locked and further can't be modified by user */
+	if (is_shstk_locked(t))
+		return -EINVAL;
+
+	enable_shstk = status & PR_SHADOW_STACK_ENABLE;
+	/* Request is to enable shadow stack and shadow stack is not enabled already */
+	if (enable_shstk && !is_shstk_enabled(t)) {
+		/* shadow stack was allocated and enable request again
+		 * no need to support such usecase and return EINVAL.
+		 */
+		if (is_shstk_allocated(t))
+			return -EINVAL;
+
+		size = calc_shstk_size(0);
+		addr = allocate_shadow_stack(0, size, 0, false);
+		if (IS_ERR_VALUE(addr))
+			return -ENOMEM;
+		set_shstk_base(t, addr, size);
+		set_active_shstk(t, addr + size);
+	}
+
+	/*
+	 * If a request to disable shadow stack happens, let's go ahead and release it
+	 * Although, if CLONE_VFORKed child did this, then in that case we will end up
+	 * not releasing the shadow stack (because it might be needed in parent). Although
+	 * we will disable it for VFORKed child. And if VFORKed child tries to enable again
+	 * then in that case, it'll get entirely new shadow stack because following condition
+	 * are true
+	 *  - shadow stack was not enabled for vforked child
+	 *  - shadow stack base was anyways pointing to 0
+	 * This shouldn't be a big issue because we want parent to have availability of shadow
+	 * stack whenever VFORKed child releases resources via exit or exec but at the same
+	 * time we want VFORKed child to break away and establish new shadow stack if it desires
+	 *
+	 */
+	if (!enable_shstk)
+		shstk_release(t);
+
+	set_shstk_status(t, enable_shstk);
+	return 0;
+}
+
+int arch_lock_shadow_stack_status(struct task_struct *task,
+				  unsigned long arg)
+{
+	/* If shtstk not supported or not enabled on task, nothing to lock here */
+	if (!is_user_shstk_enabled() ||
+	    !is_shstk_enabled(task) || arg != 0)
+		return -EINVAL;
+
+	set_shstk_lock(task);
+
+	return 0;
+}
+
+int arch_get_indir_br_lp_status(struct task_struct *t, unsigned long __user *status)
+{
+	unsigned long fcfi_status = 0;
+
+	if (!is_user_lpad_enabled())
+		return -EINVAL;
+
+	/* indirect branch tracking is enabled on the task or not */
+	fcfi_status |= (is_indir_lp_enabled(t) ? PR_INDIR_BR_LP_ENABLE : 0);
+
+	return copy_to_user(status, &fcfi_status, sizeof(fcfi_status)) ? -EFAULT : 0;
+}
+
+int arch_set_indir_br_lp_status(struct task_struct *t, unsigned long status)
+{
+	bool enable_indir_lp = false;
+
+	if (!is_user_lpad_enabled())
+		return -EINVAL;
+
+	/* indirect branch tracking is locked and further can't be modified by user */
+	if (is_indir_lp_locked(t))
+		return -EINVAL;
+
+	/* Reject unknown flags */
+	if (status & ~PR_INDIR_BR_LP_ENABLE)
+		return -EINVAL;
+
+	enable_indir_lp = (status & PR_INDIR_BR_LP_ENABLE);
+	set_indir_lp_status(t, enable_indir_lp);
+
+	return 0;
+}
+
+int arch_lock_indir_br_lp_status(struct task_struct *task,
+				 unsigned long arg)
+{
+	/*
+	 * If indirect branch tracking is not supported or not enabled on task,
+	 * nothing to lock here
+	 */
+	if (!is_user_lpad_enabled() ||
+	    !is_indir_lp_enabled(task) || arg != 0)
+		return -EINVAL;
+
+	set_indir_lp_lock(task);
+
+	return 0;
+}
+
+bool is_user_shstk_enabled(void)
+{
+	return (cpu_supports_shadow_stack() &&
+		!(riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_BCFI));
+}
+
+bool is_user_lpad_enabled(void)
+{
+	return (cpu_supports_indirect_br_lp_instr() &&
+		!(riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_FCFI));
+}
+
+static int __init setup_global_riscv_enable(char *str)
+{
+	if (strcmp(str, "all") == 0)
+		riscv_nousercfi = CMDLINE_DISABLE_RISCV_USERCFI;
+
+	if (strcmp(str, "fcfi") == 0)
+		riscv_nousercfi |= CMDLINE_DISABLE_RISCV_USERCFI_FCFI;
+
+	if (strcmp(str, "bcfi") == 0)
+		riscv_nousercfi |= CMDLINE_DISABLE_RISCV_USERCFI_BCFI;
+
+	if (riscv_nousercfi)
+		pr_info("riscv user cfi disabled via cmdline - shadow stack status : %s, landing pad status : %s\n",
+			(riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_BCFI) ? "disabled" :
+			"enabled", (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_FCFI) ?
+			"disabled" : "enabled");
+
+	return 1;
+}
+
+__setup("riscv_nousercfi=", setup_global_riscv_enable);
diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c
index 3a8e038b10a2..43f70198ac3c 100644
--- a/arch/riscv/kernel/vdso.c
+++ b/arch/riscv/kernel/vdso.c
@@ -98,6 +98,13 @@ static struct __vdso_info compat_vdso_info __ro_after_init = {
 
 static int __init vdso_init(void)
 {
+	/* Hart implements zimop, expose cfi compiled vdso */
+	if (IS_ENABLED(CONFIG_RISCV_USER_CFI) &&
+	    riscv_has_extension_unlikely(RISCV_ISA_EXT_ZIMOP)) {
+		vdso_info.vdso_code_start = vdso_cfi_start;
+		vdso_info.vdso_code_end = vdso_cfi_end;
+	}
+
 	__vdso_init(&vdso_info);
 #ifdef CONFIG_COMPAT
 	__vdso_init(&compat_vdso_info);
diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile
index 9ebb5e590f93..a842dc034571 100644
--- a/arch/riscv/kernel/vdso/Makefile
+++ b/arch/riscv/kernel/vdso/Makefile
@@ -17,6 +17,15 @@ ifdef CONFIG_VDSO_GETRANDOM
 vdso-syms += getrandom
 endif
 
+ifdef VDSO_CFI_BUILD
+CFI_MARCH = _zicfilp_zicfiss
+CFI_FULL = -fcf-protection=full
+CFI_SUFFIX = -cfi
+OFFSET_SUFFIX = _cfi
+ccflags-y += -DVDSO_CFI=1
+asflags-y += -DVDSO_CFI=1
+endif
+
 # Files to link into the vdso
 obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o
 
@@ -27,6 +36,10 @@ endif
 ccflags-y := -fno-stack-protector
 ccflags-y += -DDISABLE_BRANCH_PROFILING
 ccflags-y += -fno-builtin
+ccflags-y += $(KBUILD_BASE_ISA)$(CFI_MARCH)
+ccflags-y += $(CFI_FULL)
+asflags-y += $(KBUILD_BASE_ISA)$(CFI_MARCH)
+asflags-y += $(CFI_FULL)
 
 ifneq ($(c-gettimeofday-y),)
   CFLAGS_vgettimeofday.o += -fPIC -include $(c-gettimeofday-y)
@@ -39,13 +52,20 @@ endif
 CFLAGS_hwprobe.o += -fPIC
 
 # Build rules
-targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.lds
+vdso_offsets := vdso$(if $(VDSO_CFI_BUILD),$(CFI_SUFFIX),)-offsets.h
+vdso_o := vdso$(if $(VDSO_CFI_BUILD),$(CFI_SUFFIX),).o
+vdso_so := vdso$(if $(VDSO_CFI_BUILD),$(CFI_SUFFIX),).so
+vdso_so_dbg := vdso$(if $(VDSO_CFI_BUILD),$(CFI_SUFFIX),).so.dbg
+vdso_lds := vdso.lds
+
+targets := $(obj-vdso) $(vdso_so) $(vdso_so_dbg) $(vdso_lds)
+
 obj-vdso := $(addprefix $(obj)/, $(obj-vdso))
 
-obj-y += vdso.o
-CPPFLAGS_vdso.lds += -P -C -U$(ARCH)
+obj-y += vdso$(if $(VDSO_CFI_BUILD),$(CFI_SUFFIX),).o
+CPPFLAGS_$(vdso_lds) += -P -C -U$(ARCH)
 ifneq ($(filter vgettimeofday, $(vdso-syms)),)
-CPPFLAGS_vdso.lds += -DHAS_VGETTIMEOFDAY
+CPPFLAGS_$(vdso_lds) += -DHAS_VGETTIMEOFDAY
 endif
 
 # Disable -pg to prevent insert call site
@@ -54,12 +74,12 @@ CFLAGS_REMOVE_getrandom.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS)
 CFLAGS_REMOVE_hwprobe.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS)
 
 # Force dependency
-$(obj)/vdso.o: $(obj)/vdso.so
+$(obj)/$(vdso_o): $(obj)/$(vdso_so)
 
 # link rule for the .so file, .lds has to be first
-$(obj)/vdso.so.dbg: $(obj)/vdso.lds $(obj-vdso) FORCE
+$(obj)/$(vdso_so_dbg): $(obj)/$(vdso_lds) $(obj-vdso) FORCE
 	$(call if_changed,vdsold_and_check)
-LDFLAGS_vdso.so.dbg = -shared -soname=linux-vdso.so.1 \
+LDFLAGS_$(vdso_so_dbg) = -shared -soname=linux-vdso.so.1 \
 	--build-id=sha1 --eh-frame-hdr
 
 # strip rule for the .so file
@@ -70,16 +90,16 @@ $(obj)/%.so: $(obj)/%.so.dbg FORCE
 # Generate VDSO offsets using helper script
 gen-vdsosym := $(src)/gen_vdso_offsets.sh
 quiet_cmd_vdsosym = VDSOSYM $@
-	cmd_vdsosym = $(NM) $< | $(gen-vdsosym) | LC_ALL=C sort > $@
+	cmd_vdsosym = $(NM) $< | $(gen-vdsosym) $(OFFSET_SUFFIX) | LC_ALL=C sort > $@
 
-include/generated/vdso-offsets.h: $(obj)/vdso.so.dbg FORCE
+include/generated/$(vdso_offsets): $(obj)/$(vdso_so_dbg) FORCE
 	$(call if_changed,vdsosym)
 
 # actual build commands
 # The DSO images are built using a special linker script
 # Make sure only to export the intended __vdso_xxx symbol offsets.
 quiet_cmd_vdsold_and_check = VDSOLD  $@
-      cmd_vdsold_and_check = $(LD) $(ld_flags) -T $(filter-out FORCE,$^) -o $@.tmp && \
+      cmd_vdsold_and_check = $(LD) $(CFI_FULL) $(ld_flags) -T $(filter-out FORCE,$^) -o $@.tmp && \
                    $(OBJCOPY) $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ && \
                    rm $@.tmp && \
                    $(cmd_vdso_check)
diff --git a/arch/riscv/kernel/vdso/flush_icache.S b/arch/riscv/kernel/vdso/flush_icache.S
index 8f884227e8bc..e4c56970905e 100644
--- a/arch/riscv/kernel/vdso/flush_icache.S
+++ b/arch/riscv/kernel/vdso/flush_icache.S
@@ -5,11 +5,13 @@
 
 #include <linux/linkage.h>
 #include <asm/unistd.h>
+#include <asm/assembler.h>
 
 	.text
 /* int __vdso_flush_icache(void *start, void *end, unsigned long flags); */
 SYM_FUNC_START(__vdso_flush_icache)
 	.cfi_startproc
+	vdso_lpad
 #ifdef CONFIG_SMP
 	li a7, __NR_riscv_flush_icache
 	ecall
@@ -20,3 +22,5 @@ SYM_FUNC_START(__vdso_flush_icache)
 	ret
 	.cfi_endproc
 SYM_FUNC_END(__vdso_flush_icache)
+
+emit_riscv_feature_1_and
diff --git a/arch/riscv/kernel/vdso/gen_vdso_offsets.sh b/arch/riscv/kernel/vdso/gen_vdso_offsets.sh
index c2e5613f3495..bd5d5afaaa14 100755
--- a/arch/riscv/kernel/vdso/gen_vdso_offsets.sh
+++ b/arch/riscv/kernel/vdso/gen_vdso_offsets.sh
@@ -2,4 +2,6 @@
 # SPDX-License-Identifier: GPL-2.0
 
 LC_ALL=C
-sed -n -e 's/^[0]\+\(0[0-9a-fA-F]*\) . \(__vdso_[a-zA-Z0-9_]*\)$/\#define \2_offset\t0x\1/p'
+SUFFIX=${1:-""}
+sed -n -e \
+'s/^[0]\+\(0[0-9a-fA-F]*\) . \(__vdso_[a-zA-Z0-9_]*\)$/\#define \2'$SUFFIX'_offset\t0x\1/p'
diff --git a/arch/riscv/kernel/vdso/getcpu.S b/arch/riscv/kernel/vdso/getcpu.S
index 9c1bd531907f..5c1ecc4e1465 100644
--- a/arch/riscv/kernel/vdso/getcpu.S
+++ b/arch/riscv/kernel/vdso/getcpu.S
@@ -5,14 +5,18 @@
 
 #include <linux/linkage.h>
 #include <asm/unistd.h>
+#include <asm/assembler.h>
 
 	.text
 /* int __vdso_getcpu(unsigned *cpu, unsigned *node, void *unused); */
 SYM_FUNC_START(__vdso_getcpu)
 	.cfi_startproc
+	vdso_lpad
 	/* For now, just do the syscall. */
 	li a7, __NR_getcpu
 	ecall
 	ret
 	.cfi_endproc
 SYM_FUNC_END(__vdso_getcpu)
+
+emit_riscv_feature_1_and
diff --git a/arch/riscv/kernel/vdso/note.S b/arch/riscv/kernel/vdso/note.S
index 2a956c942211..3d92cc956b95 100644
--- a/arch/riscv/kernel/vdso/note.S
+++ b/arch/riscv/kernel/vdso/note.S
@@ -6,7 +6,10 @@
 
 #include <linux/elfnote.h>
 #include <linux/version.h>
+#include <asm/assembler.h>
 
 ELFNOTE_START(Linux, 0, "a")
 	.long LINUX_VERSION_CODE
 ELFNOTE_END
+
+emit_riscv_feature_1_and
diff --git a/arch/riscv/kernel/vdso/rt_sigreturn.S b/arch/riscv/kernel/vdso/rt_sigreturn.S
index 3dc022aa8931..e82987dc3739 100644
--- a/arch/riscv/kernel/vdso/rt_sigreturn.S
+++ b/arch/riscv/kernel/vdso/rt_sigreturn.S
@@ -5,12 +5,16 @@
 
 #include <linux/linkage.h>
 #include <asm/unistd.h>
+#include <asm/assembler.h>
 
 	.text
 SYM_FUNC_START(__vdso_rt_sigreturn)
 	.cfi_startproc
 	.cfi_signal_frame
+	vdso_lpad
 	li a7, __NR_rt_sigreturn
 	ecall
 	.cfi_endproc
 SYM_FUNC_END(__vdso_rt_sigreturn)
+
+emit_riscv_feature_1_and
diff --git a/arch/riscv/kernel/vdso/sys_hwprobe.S b/arch/riscv/kernel/vdso/sys_hwprobe.S
index 77e57f830521..f1694451a60c 100644
--- a/arch/riscv/kernel/vdso/sys_hwprobe.S
+++ b/arch/riscv/kernel/vdso/sys_hwprobe.S
@@ -3,13 +3,17 @@
 
 #include <linux/linkage.h>
 #include <asm/unistd.h>
+#include <asm/assembler.h>
 
 .text
 SYM_FUNC_START(riscv_hwprobe)
 	.cfi_startproc
+	vdso_lpad
 	li a7, __NR_riscv_hwprobe
 	ecall
 	ret
 
 	.cfi_endproc
 SYM_FUNC_END(riscv_hwprobe)
+
+emit_riscv_feature_1_and
diff --git a/arch/riscv/kernel/vdso/vgetrandom-chacha.S b/arch/riscv/kernel/vdso/vgetrandom-chacha.S
index 5f0dad8f2373..916ab30a88f7 100644
--- a/arch/riscv/kernel/vdso/vgetrandom-chacha.S
+++ b/arch/riscv/kernel/vdso/vgetrandom-chacha.S
@@ -7,6 +7,7 @@
 
 #include <asm/asm.h>
 #include <linux/linkage.h>
+#include <asm/assembler.h>
 
 .text
 
@@ -74,7 +75,7 @@ SYM_FUNC_START(__arch_chacha20_blocks_nostack)
 #define _20		20, 20, 20, 20
 #define _24		24, 24, 24, 24
 #define _25		25, 25, 25, 25
-
+	vdso_lpad
 	/*
 	 * The ABI requires s0-s9 saved.
 	 * This does not violate the stack-less requirement: no sensitive data
@@ -247,3 +248,5 @@ SYM_FUNC_START(__arch_chacha20_blocks_nostack)
 
 	ret
 SYM_FUNC_END(__arch_chacha20_blocks_nostack)
+
+emit_riscv_feature_1_and
diff --git a/arch/riscv/kernel/vdso_cfi/Makefile b/arch/riscv/kernel/vdso_cfi/Makefile
new file mode 100644
index 000000000000..8ebd190782b0
--- /dev/null
+++ b/arch/riscv/kernel/vdso_cfi/Makefile
@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# RISC-V VDSO CFI Makefile
+# This Makefile builds the VDSO with CFI support when CONFIG_RISCV_USER_CFI is enabled
+
+# setting VDSO_CFI_BUILD triggers build for vdso differently
+VDSO_CFI_BUILD := 1
+
+# Set the source directory to the main vdso directory
+src := $(srctree)/arch/riscv/kernel/vdso
+
+# Copy all .S and .c files from vdso directory to vdso_cfi object build directory
+vdso_c_sources := $(wildcard $(src)/*.c)
+vdso_S_sources := $(wildcard $(src)/*.S)
+vdso_c_objects := $(addprefix $(obj)/, $(notdir $(vdso_c_sources)))
+vdso_S_objects := $(addprefix $(obj)/, $(notdir $(vdso_S_sources)))
+
+$(vdso_S_objects): $(obj)/%.S: $(src)/%.S
+	$(Q)cp $< $@
+
+$(vdso_c_objects): $(obj)/%.c: $(src)/%.c
+	$(Q)cp $< $@
+
+# Include the main VDSO Makefile which contains all the build rules and sources
+# The VDSO_CFI_BUILD variable will be passed to it to enable CFI compilation
+include $(src)/Makefile
diff --git a/arch/riscv/kernel/vdso_cfi/vdso-cfi.S b/arch/riscv/kernel/vdso_cfi/vdso-cfi.S
new file mode 100644
index 000000000000..d426f6accb35
--- /dev/null
+++ b/arch/riscv/kernel/vdso_cfi/vdso-cfi.S
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright 2025 Rivos, Inc
+ */
+
+#define	vdso_start	vdso_cfi_start
+#define	vdso_end	vdso_cfi_end
+
+#define __VDSO_PATH "arch/riscv/kernel/vdso_cfi/vdso-cfi.so"
+
+#include "../vdso/vdso.S"
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index addb8a9305be..25a8f693a765 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -376,7 +376,7 @@ pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);
 static const pgprot_t protection_map[16] = {
 	[VM_NONE]					= PAGE_NONE,
 	[VM_READ]					= PAGE_READ,
-	[VM_WRITE]					= PAGE_COPY,
+	[VM_WRITE]					= PAGE_SHADOWSTACK,
 	[VM_WRITE | VM_READ]				= PAGE_COPY,
 	[VM_EXEC]					= PAGE_EXEC,
 	[VM_EXEC | VM_READ]				= PAGE_READ_EXEC,
diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c
index 807c0a0de182..e0ab04d721cc 100644
--- a/arch/riscv/mm/pgtable.c
+++ b/arch/riscv/mm/pgtable.c
@@ -163,3 +163,19 @@ pud_t pudp_invalidate(struct vm_area_struct *vma, unsigned long address,
 	return old;
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+
+pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma)
+{
+	if (vma->vm_flags & VM_SHADOW_STACK)
+		return pte_mkwrite_shstk(pte);
+
+	return pte_mkwrite_novma(pte);
+}
+
+pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
+{
+	if (vma->vm_flags & VM_SHADOW_STACK)
+		return pmd_mkwrite_shstk(pmd);
+
+	return pmd_mkwrite_novma(pmd);
+}
diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile
index 490167faba7a..a1e719a79d38 100644
--- a/arch/s390/boot/Makefile
+++ b/arch/s390/boot/Makefile
@@ -21,6 +21,7 @@ KBUILD_AFLAGS := $(filter-out $(CC_FLAGS_MARCH),$(KBUILD_AFLAGS_DECOMPRESSOR))
 KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_MARCH),$(KBUILD_CFLAGS_DECOMPRESSOR))
 KBUILD_AFLAGS += $(CC_FLAGS_MARCH_MINIMUM) -D__DISABLE_EXPORTS
 KBUILD_CFLAGS += $(CC_FLAGS_MARCH_MINIMUM) -D__DISABLE_EXPORTS
+KBUILD_CFLAGS += $(call cc-option, -Wno-default-const-init-unsafe)
 
 CFLAGS_sclp_early_core.o += -I$(srctree)/drivers/s390/char
 
diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h
index 962cf042c66d..e6ec0ccf3d73 100644
--- a/arch/s390/include/asm/ptrace.h
+++ b/arch/s390/include/asm/ptrace.h
@@ -214,16 +214,23 @@ void update_cr_regs(struct task_struct *task);
 #define arch_has_single_step()	(1)
 #define arch_has_block_step()	(1)
 
-#define user_mode(regs) (((regs)->psw.mask & PSW_MASK_PSTATE) != 0)
-#define instruction_pointer(regs) ((regs)->psw.addr)
-#define user_stack_pointer(regs)((regs)->gprs[15])
 #define profile_pc(regs) instruction_pointer(regs)
 
-static inline long regs_return_value(struct pt_regs *regs)
+static __always_inline bool user_mode(const struct pt_regs *regs)
+{
+	return psw_bits(regs->psw).pstate;
+}
+
+static inline long regs_return_value(const struct pt_regs *regs)
 {
 	return regs->gprs[2];
 }
 
+static __always_inline unsigned long instruction_pointer(const struct pt_regs *regs)
+{
+	return regs->psw.addr;
+}
+
 static inline void instruction_pointer_set(struct pt_regs *regs,
 					   unsigned long val)
 {
@@ -233,19 +240,26 @@ static inline void instruction_pointer_set(struct pt_regs *regs,
 int regs_query_register_offset(const char *name);
 const char *regs_query_register_name(unsigned int offset);
 
-static __always_inline unsigned long kernel_stack_pointer(struct pt_regs *regs)
+static __always_inline unsigned long kernel_stack_pointer(const struct pt_regs *regs)
+{
+	return regs->gprs[15];
+}
+
+static __always_inline unsigned long user_stack_pointer(const struct pt_regs *regs)
 {
 	return regs->gprs[15];
 }
 
-static __always_inline unsigned long regs_get_register(struct pt_regs *regs, unsigned int offset)
+static __always_inline unsigned long regs_get_register(const struct pt_regs *regs,
+						       unsigned int offset)
 {
 	if (offset >= NUM_GPRS)
 		return 0;
 	return regs->gprs[offset];
 }
 
-static __always_inline int regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr)
+static __always_inline int regs_within_kernel_stack(const struct pt_regs *regs,
+						    unsigned long addr)
 {
 	unsigned long ksp = kernel_stack_pointer(regs);
 
@@ -261,7 +275,8 @@ static __always_inline int regs_within_kernel_stack(struct pt_regs *regs, unsign
  * is specifined by @regs. If the @n th entry is NOT in the kernel stack,
  * this returns 0.
  */
-static __always_inline unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n)
+static __always_inline unsigned long regs_get_kernel_stack_nth(const struct pt_regs *regs,
+							       unsigned int n)
 {
 	unsigned long addr;
 
@@ -278,8 +293,8 @@ static __always_inline unsigned long regs_get_kernel_stack_nth(struct pt_regs *r
  *
  * regs_get_kernel_argument() returns @n th argument of the function call.
  */
-static inline unsigned long regs_get_kernel_argument(struct pt_regs *regs,
-						     unsigned int n)
+static __always_inline unsigned long regs_get_kernel_argument(const struct pt_regs *regs,
+							      unsigned int n)
 {
 	unsigned int argoffset = STACK_FRAME_OVERHEAD / sizeof(long);
 
@@ -290,7 +305,7 @@ static inline unsigned long regs_get_kernel_argument(struct pt_regs *regs,
 	return regs_get_kernel_stack_nth(regs, argoffset + n);
 }
 
-static inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc)
+static __always_inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc)
 {
 	regs->gprs[2] = rc;
 }
diff --git a/arch/s390/include/uapi/asm/ipl.h b/arch/s390/include/uapi/asm/ipl.h
index 2cd28af50dd4..3d64a2251699 100644
--- a/arch/s390/include/uapi/asm/ipl.h
+++ b/arch/s390/include/uapi/asm/ipl.h
@@ -15,6 +15,7 @@ struct ipl_pl_hdr {
 #define IPL_PL_FLAG_IPLPS	0x80
 #define IPL_PL_FLAG_SIPL	0x40
 #define IPL_PL_FLAG_IPLSR	0x20
+#define IPL_PL_FLAG_SBP		0x10
 
 /* IPL Parameter Block header */
 struct ipl_pb_hdr {
diff --git a/arch/s390/kernel/alternative.c b/arch/s390/kernel/alternative.c
index 90c0e6408992..02d04ae621ba 100644
--- a/arch/s390/kernel/alternative.c
+++ b/arch/s390/kernel/alternative.c
@@ -4,6 +4,7 @@
 #define pr_fmt(fmt)	"alt: " fmt
 #endif
 
+#include <linux/hex.h>
 #include <linux/uaccess.h>
 #include <linux/printk.h>
 #include <asm/nospec-branch.h>
diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c
index 961a3d60a4dd..dcdc7e274848 100644
--- a/arch/s390/kernel/ipl.c
+++ b/arch/s390/kernel/ipl.c
@@ -262,6 +262,24 @@ static struct kobj_attribute sys_##_prefix##_##_name##_attr =		\
 			sys_##_prefix##_##_name##_show,			\
 			sys_##_prefix##_##_name##_store)
 
+#define DEFINE_IPL_ATTR_BOOTPROG_RW(_prefix, _name, _fmt_out, _fmt_in, _hdr, _value)	\
+	IPL_ATTR_SHOW_FN(_prefix, _name, _fmt_out, (unsigned long long) _value)		\
+static ssize_t sys_##_prefix##_##_name##_store(struct kobject *kobj,			\
+		struct kobj_attribute *attr,						\
+		const char *buf, size_t len)						\
+{											\
+	unsigned long long value;							\
+	if (sscanf(buf, _fmt_in, &value) != 1)						\
+		return -EINVAL;								\
+	(_value) = value;								\
+	(_hdr).flags &= ~IPL_PL_FLAG_SBP;						\
+	return len;									\
+}											\
+static struct kobj_attribute sys_##_prefix##_##_name##_attr =				\
+	__ATTR(_name, 0644,								\
+			sys_##_prefix##_##_name##_show,					\
+			sys_##_prefix##_##_name##_store)
+
 #define DEFINE_IPL_ATTR_STR_RW(_prefix, _name, _fmt_out, _fmt_in, _value)\
 IPL_ATTR_SHOW_FN(_prefix, _name, _fmt_out, _value)			\
 static ssize_t sys_##_prefix##_##_name##_store(struct kobject *kobj,	\
@@ -818,12 +836,13 @@ DEFINE_IPL_ATTR_RW(reipl_fcp, wwpn, "0x%016llx\n", "%llx\n",
 		   reipl_block_fcp->fcp.wwpn);
 DEFINE_IPL_ATTR_RW(reipl_fcp, lun, "0x%016llx\n", "%llx\n",
 		   reipl_block_fcp->fcp.lun);
-DEFINE_IPL_ATTR_RW(reipl_fcp, bootprog, "%lld\n", "%lld\n",
-		   reipl_block_fcp->fcp.bootprog);
 DEFINE_IPL_ATTR_RW(reipl_fcp, br_lba, "%lld\n", "%lld\n",
 		   reipl_block_fcp->fcp.br_lba);
 DEFINE_IPL_ATTR_RW(reipl_fcp, device, "0.0.%04llx\n", "0.0.%llx\n",
 		   reipl_block_fcp->fcp.devno);
+DEFINE_IPL_ATTR_BOOTPROG_RW(reipl_fcp, bootprog, "%lld\n", "%lld\n",
+			    reipl_block_fcp->hdr,
+			    reipl_block_fcp->fcp.bootprog);
 
 static void reipl_get_ascii_loadparm(char *loadparm,
 				     struct ipl_parameter_block *ibp)
@@ -942,10 +961,11 @@ DEFINE_IPL_ATTR_RW(reipl_nvme, fid, "0x%08llx\n", "%llx\n",
 		   reipl_block_nvme->nvme.fid);
 DEFINE_IPL_ATTR_RW(reipl_nvme, nsid, "0x%08llx\n", "%llx\n",
 		   reipl_block_nvme->nvme.nsid);
-DEFINE_IPL_ATTR_RW(reipl_nvme, bootprog, "%lld\n", "%lld\n",
-		   reipl_block_nvme->nvme.bootprog);
 DEFINE_IPL_ATTR_RW(reipl_nvme, br_lba, "%lld\n", "%lld\n",
 		   reipl_block_nvme->nvme.br_lba);
+DEFINE_IPL_ATTR_BOOTPROG_RW(reipl_nvme, bootprog, "%lld\n", "%lld\n",
+			    reipl_block_nvme->hdr,
+			    reipl_block_nvme->nvme.bootprog);
 
 static struct attribute *reipl_nvme_attrs[] = {
 	&sys_reipl_nvme_fid_attr.attr,
@@ -1038,8 +1058,9 @@ static const struct bin_attribute *const reipl_eckd_bin_attrs[] = {
 };
 
 DEFINE_IPL_CCW_ATTR_RW(reipl_eckd, device, reipl_block_eckd->eckd);
-DEFINE_IPL_ATTR_RW(reipl_eckd, bootprog, "%lld\n", "%lld\n",
-		   reipl_block_eckd->eckd.bootprog);
+DEFINE_IPL_ATTR_BOOTPROG_RW(reipl_eckd, bootprog, "%lld\n", "%lld\n",
+			    reipl_block_eckd->hdr,
+			    reipl_block_eckd->eckd.bootprog);
 
 static struct attribute *reipl_eckd_attrs[] = {
 	&sys_reipl_eckd_device_attr.attr,
@@ -1567,12 +1588,13 @@ DEFINE_IPL_ATTR_RW(dump_fcp, wwpn, "0x%016llx\n", "%llx\n",
 		   dump_block_fcp->fcp.wwpn);
 DEFINE_IPL_ATTR_RW(dump_fcp, lun, "0x%016llx\n", "%llx\n",
 		   dump_block_fcp->fcp.lun);
-DEFINE_IPL_ATTR_RW(dump_fcp, bootprog, "%lld\n", "%lld\n",
-		   dump_block_fcp->fcp.bootprog);
 DEFINE_IPL_ATTR_RW(dump_fcp, br_lba, "%lld\n", "%lld\n",
 		   dump_block_fcp->fcp.br_lba);
 DEFINE_IPL_ATTR_RW(dump_fcp, device, "0.0.%04llx\n", "0.0.%llx\n",
 		   dump_block_fcp->fcp.devno);
+DEFINE_IPL_ATTR_BOOTPROG_RW(dump_fcp, bootprog, "%lld\n", "%lld\n",
+			    dump_block_fcp->hdr,
+			    dump_block_fcp->fcp.bootprog);
 
 DEFINE_IPL_ATTR_SCP_DATA_RW(dump_fcp, dump_block_fcp->hdr,
 			    dump_block_fcp->fcp,
@@ -1604,10 +1626,11 @@ DEFINE_IPL_ATTR_RW(dump_nvme, fid, "0x%08llx\n", "%llx\n",
 		   dump_block_nvme->nvme.fid);
 DEFINE_IPL_ATTR_RW(dump_nvme, nsid, "0x%08llx\n", "%llx\n",
 		   dump_block_nvme->nvme.nsid);
-DEFINE_IPL_ATTR_RW(dump_nvme, bootprog, "%lld\n", "%llx\n",
-		   dump_block_nvme->nvme.bootprog);
 DEFINE_IPL_ATTR_RW(dump_nvme, br_lba, "%lld\n", "%llx\n",
 		   dump_block_nvme->nvme.br_lba);
+DEFINE_IPL_ATTR_BOOTPROG_RW(dump_nvme, bootprog, "%lld\n", "%llx\n",
+			    dump_block_nvme->hdr,
+			    dump_block_nvme->nvme.bootprog);
 
 DEFINE_IPL_ATTR_SCP_DATA_RW(dump_nvme, dump_block_nvme->hdr,
 			    dump_block_nvme->nvme,
@@ -1635,8 +1658,9 @@ static const struct attribute_group dump_nvme_attr_group = {
 
 /* ECKD dump device attributes */
 DEFINE_IPL_CCW_ATTR_RW(dump_eckd, device, dump_block_eckd->eckd);
-DEFINE_IPL_ATTR_RW(dump_eckd, bootprog, "%lld\n", "%llx\n",
-		   dump_block_eckd->eckd.bootprog);
+DEFINE_IPL_ATTR_BOOTPROG_RW(dump_eckd, bootprog, "%lld\n", "%llx\n",
+			    dump_block_eckd->hdr,
+			    dump_block_eckd->eckd.bootprog);
 
 IPL_ATTR_BR_CHR_SHOW_FN(dump, dump_block_eckd->eckd);
 IPL_ATTR_BR_CHR_STORE_FN(dump, dump_block_eckd->eckd);
diff --git a/arch/s390/kernel/stackprotector.c b/arch/s390/kernel/stackprotector.c
index d4e40483f008..8bd3ecf9200a 100644
--- a/arch/s390/kernel/stackprotector.c
+++ b/arch/s390/kernel/stackprotector.c
@@ -5,6 +5,7 @@
 #endif
 
 #include <linux/export.h>
+#include <linux/hex.h>
 #include <linux/uaccess.h>
 #include <linux/printk.h>
 #include <asm/abs_lowcore.h>
diff --git a/arch/s390/kernel/stacktrace.c b/arch/s390/kernel/stacktrace.c
index 3aae7f70e6ab..18520d333058 100644
--- a/arch/s390/kernel/stacktrace.c
+++ b/arch/s390/kernel/stacktrace.c
@@ -104,7 +104,6 @@ void arch_stack_walk_user_common(stack_trace_consume_fn consume_entry, void *coo
 	struct stack_frame_vdso_wrapper __user *sf_vdso;
 	struct stack_frame_user __user *sf;
 	unsigned long ip, sp;
-	bool first = true;
 
 	if (!current->mm)
 		return;
@@ -133,24 +132,11 @@ void arch_stack_walk_user_common(stack_trace_consume_fn consume_entry, void *coo
 			if (__get_user(ip, &sf->gprs[8]))
 				break;
 		}
-		/* Sanity check: ABI requires SP to be 8 byte aligned. */
-		if (sp & 0x7)
+		/* Validate SP and RA (ABI requires SP to be 8 byte aligned). */
+		if (sp & 0x7 || ip_invalid(ip))
 			break;
-		if (ip_invalid(ip)) {
-			/*
-			 * If the instruction address is invalid, and this
-			 * is the first stack frame, assume r14 has not
-			 * been written to the stack yet. Otherwise exit.
-			 */
-			if (!first)
-				break;
-			ip = regs->gprs[14];
-			if (ip_invalid(ip))
-				break;
-		}
 		if (!store_ip(consume_entry, cookie, entry, perf, ip))
 			break;
-		first = false;
 	}
 	pagefault_enable();
 }
diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c
index 5a6ace9d875a..57f3980b98a9 100644
--- a/arch/s390/pci/pci.c
+++ b/arch/s390/pci/pci.c
@@ -961,6 +961,7 @@ void zpci_device_reserved(struct zpci_dev *zdev)
 }
 
 void zpci_release_device(struct kref *kref)
+	__releases(&zpci_list_lock)
 {
 	struct zpci_dev *zdev = container_of(kref, struct zpci_dev, kref);
 
@@ -1148,6 +1149,7 @@ static void zpci_add_devices(struct list_head *scan_list)
 
 int zpci_scan_devices(void)
 {
+	struct zpci_bus *zbus;
 	LIST_HEAD(scan_list);
 	int rc;
 
@@ -1156,7 +1158,10 @@ int zpci_scan_devices(void)
 		return rc;
 
 	zpci_add_devices(&scan_list);
-	zpci_bus_scan_busses();
+	zpci_bus_for_each(zbus) {
+		zpci_bus_scan_bus(zbus);
+		cond_resched();
+	}
 	return 0;
 }
 
diff --git a/arch/s390/pci/pci_bus.c b/arch/s390/pci/pci_bus.c
index 66c4bd888b29..42a13e451f64 100644
--- a/arch/s390/pci/pci_bus.c
+++ b/arch/s390/pci/pci_bus.c
@@ -153,23 +153,6 @@ int zpci_bus_scan_bus(struct zpci_bus *zbus)
 	return ret;
 }
 
-/* zpci_bus_scan_busses - Scan all registered busses
- *
- * Scan all available zbusses
- *
- */
-void zpci_bus_scan_busses(void)
-{
-	struct zpci_bus *zbus = NULL;
-
-	mutex_lock(&zbus_list_lock);
-	list_for_each_entry(zbus, &zbus_list, bus_next) {
-		zpci_bus_scan_bus(zbus);
-		cond_resched();
-	}
-	mutex_unlock(&zbus_list_lock);
-}
-
 static bool zpci_bus_is_multifunction_root(struct zpci_dev *zdev)
 {
 	return !s390_pci_no_rid && zdev->rid_available &&
@@ -222,10 +205,29 @@ static int zpci_bus_create_pci_bus(struct zpci_bus *zbus, struct zpci_dev *fr, s
 	return -ENOMEM;
 }
 
-static void zpci_bus_release(struct kref *kref)
+/**
+ * zpci_bus_release - Un-initialize resources associated with the zbus and
+ *		      free memory
+ * @kref:	refcount * that is part of struct zpci_bus
+ *
+ * MUST be called with `zbus_list_lock` held, but the lock is released during
+ * run of the function.
+ */
+static inline void zpci_bus_release(struct kref *kref)
+	__releases(&zbus_list_lock)
 {
 	struct zpci_bus *zbus = container_of(kref, struct zpci_bus, kref);
 
+	lockdep_assert_held(&zbus_list_lock);
+
+	list_del(&zbus->bus_next);
+	mutex_unlock(&zbus_list_lock);
+
+	/*
+	 * At this point no-one should see this object, or be able to get a new
+	 * reference to it.
+	 */
+
 	if (zbus->bus) {
 		pci_lock_rescan_remove();
 		pci_stop_root_bus(zbus->bus);
@@ -237,16 +239,19 @@ static void zpci_bus_release(struct kref *kref)
 		pci_unlock_rescan_remove();
 	}
 
-	mutex_lock(&zbus_list_lock);
-	list_del(&zbus->bus_next);
-	mutex_unlock(&zbus_list_lock);
 	zpci_remove_parent_msi_domain(zbus);
 	kfree(zbus);
 }
 
-static void zpci_bus_put(struct zpci_bus *zbus)
+static inline void __zpci_bus_get(struct zpci_bus *zbus)
+{
+	lockdep_assert_held(&zbus_list_lock);
+	kref_get(&zbus->kref);
+}
+
+static inline void zpci_bus_put(struct zpci_bus *zbus)
 {
-	kref_put(&zbus->kref, zpci_bus_release);
+	kref_put_mutex(&zbus->kref, zpci_bus_release, &zbus_list_lock);
 }
 
 static struct zpci_bus *zpci_bus_get(int topo, bool topo_is_tid)
@@ -258,7 +263,7 @@ static struct zpci_bus *zpci_bus_get(int topo, bool topo_is_tid)
 		if (!zbus->multifunction)
 			continue;
 		if (topo_is_tid == zbus->topo_is_tid && topo == zbus->topo) {
-			kref_get(&zbus->kref);
+			__zpci_bus_get(zbus);
 			goto out_unlock;
 		}
 	}
@@ -268,6 +273,44 @@ static struct zpci_bus *zpci_bus_get(int topo, bool topo_is_tid)
 	return zbus;
 }
 
+/**
+ * zpci_bus_get_next - get the next zbus object from given position in the list
+ * @pos:	current position/cursor in the global zbus list
+ *
+ * Acquires and releases references as the cursor iterates (might also free/
+ * release the cursor). Is tolerant of concurrent operations on the list.
+ *
+ * To begin the iteration, set *@pos to %NULL before calling the function.
+ *
+ * *@pos is set to %NULL in cases where either the list is empty, or *@pos is
+ * the last element in the list.
+ *
+ * Context: Process context. May sleep.
+ */
+void zpci_bus_get_next(struct zpci_bus **pos)
+{
+	struct zpci_bus *curp = *pos, *next = NULL;
+
+	mutex_lock(&zbus_list_lock);
+	if (curp)
+		next = list_next_entry(curp, bus_next);
+	else
+		next = list_first_entry(&zbus_list, typeof(*curp), bus_next);
+
+	if (list_entry_is_head(next, &zbus_list, bus_next))
+		next = NULL;
+
+	if (next)
+		__zpci_bus_get(next);
+
+	*pos = next;
+	mutex_unlock(&zbus_list_lock);
+
+	/* zpci_bus_put() might drop refcount to 0 and locks zbus_list_lock */
+	if (curp)
+		zpci_bus_put(curp);
+}
+
 static struct zpci_bus *zpci_bus_alloc(int topo, bool topo_is_tid)
 {
 	struct zpci_bus *zbus;
@@ -279,9 +322,6 @@ static struct zpci_bus *zpci_bus_alloc(int topo, bool topo_is_tid)
 	zbus->topo = topo;
 	zbus->topo_is_tid = topo_is_tid;
 	INIT_LIST_HEAD(&zbus->bus_next);
-	mutex_lock(&zbus_list_lock);
-	list_add_tail(&zbus->bus_next, &zbus_list);
-	mutex_unlock(&zbus_list_lock);
 
 	kref_init(&zbus->kref);
 	INIT_LIST_HEAD(&zbus->resources);
@@ -291,6 +331,10 @@ static struct zpci_bus *zpci_bus_alloc(int topo, bool topo_is_tid)
 	zbus->bus_resource.flags = IORESOURCE_BUS;
 	pci_add_resource(&zbus->resources, &zbus->bus_resource);
 
+	mutex_lock(&zbus_list_lock);
+	list_add_tail(&zbus->bus_next, &zbus_list);
+	mutex_unlock(&zbus_list_lock);
+
 	return zbus;
 }
 
diff --git a/arch/s390/pci/pci_bus.h b/arch/s390/pci/pci_bus.h
index ae3d7a9159bd..e440742e3145 100644
--- a/arch/s390/pci/pci_bus.h
+++ b/arch/s390/pci/pci_bus.h
@@ -15,7 +15,20 @@ int zpci_bus_device_register(struct zpci_dev *zdev, struct pci_ops *ops);
 void zpci_bus_device_unregister(struct zpci_dev *zdev);
 
 int zpci_bus_scan_bus(struct zpci_bus *zbus);
-void zpci_bus_scan_busses(void);
+void zpci_bus_get_next(struct zpci_bus **pos);
+
+/**
+ * zpci_bus_for_each - iterate over all the registered zbus objects
+ * @pos:	a struct zpci_bus * as cursor
+ *
+ * Acquires and releases references as the cursor iterates over the registered
+ * objects. Is tolerant against concurrent removals of objects.
+ *
+ * Context: Process context. May sleep.
+ */
+#define zpci_bus_for_each(pos)					     \
+	for ((pos) = NULL, zpci_bus_get_next(&(pos)); (pos) != NULL; \
+	     zpci_bus_get_next(&(pos)))
 
 int zpci_bus_scan_device(struct zpci_dev *zdev);
 void zpci_bus_remove_device(struct zpci_dev *zdev, bool set_error);
diff --git a/arch/s390/purgatory/Makefile b/arch/s390/purgatory/Makefile
index 0c196a5b194a..61d240a37633 100644
--- a/arch/s390/purgatory/Makefile
+++ b/arch/s390/purgatory/Makefile
@@ -23,6 +23,7 @@ KBUILD_CFLAGS += -D__DISABLE_EXPORTS
 KBUILD_CFLAGS += $(CLANG_FLAGS)
 KBUILD_CFLAGS += $(if $(CONFIG_CC_IS_CLANG),-Wno-microsoft-anon-tag)
 KBUILD_CFLAGS += $(call cc-option,-fno-PIE)
+KBUILD_CFLAGS += $(call cc-option, -Wno-default-const-init-unsafe)
 KBUILD_AFLAGS := $(filter-out -DCC_USING_EXPOLINE,$(KBUILD_AFLAGS))
 KBUILD_AFLAGS += -D__DISABLE_EXPORTS
 
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index a630d373e645..2bad14744ca4 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -112,6 +112,7 @@ config SPARC64
 	select NEED_PER_CPU_PAGE_FIRST_CHUNK
 	select ARCH_SUPPORTS_SCHED_SMT if SMP
 	select ARCH_SUPPORTS_SCHED_MC  if SMP
+	select ARCH_HAS_LAZY_MMU_MODE
 
 config ARCH_PROC_KCORE_TEXT
 	def_bool y
diff --git a/arch/sparc/include/asm/tlbflush_64.h b/arch/sparc/include/asm/tlbflush_64.h
index 8b8cdaa69272..6133306ba59a 100644
--- a/arch/sparc/include/asm/tlbflush_64.h
+++ b/arch/sparc/include/asm/tlbflush_64.h
@@ -12,7 +12,6 @@ struct tlb_batch {
 	unsigned int hugepage_shift;
 	struct mm_struct *mm;
 	unsigned long tlb_nr;
-	unsigned long active;
 	unsigned long vaddrs[TLB_BATCH_NR];
 };
 
@@ -39,12 +38,10 @@ static inline void flush_tlb_range(struct vm_area_struct *vma,
 
 void flush_tlb_kernel_range(unsigned long start, unsigned long end);
 
-#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
-
 void flush_tlb_pending(void);
 void arch_enter_lazy_mmu_mode(void);
+void arch_flush_lazy_mmu_mode(void);
 void arch_leave_lazy_mmu_mode(void);
-#define arch_flush_lazy_mmu_mode()      do {} while (0)
 
 /* Local cpu only.  */
 void __flush_tlb_all(void);
diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
index a35ddcca5e76..3a852071d260 100644
--- a/arch/sparc/mm/tlb.c
+++ b/arch/sparc/mm/tlb.c
@@ -52,20 +52,20 @@ void flush_tlb_pending(void)
 
 void arch_enter_lazy_mmu_mode(void)
 {
-	struct tlb_batch *tb;
-
 	preempt_disable();
-	tb = this_cpu_ptr(&tlb_batch);
-	tb->active = 1;
 }
 
-void arch_leave_lazy_mmu_mode(void)
+void arch_flush_lazy_mmu_mode(void)
 {
 	struct tlb_batch *tb = this_cpu_ptr(&tlb_batch);
 
 	if (tb->tlb_nr)
 		flush_tlb_pending();
-	tb->active = 0;
+}
+
+void arch_leave_lazy_mmu_mode(void)
+{
+	arch_flush_lazy_mmu_mode();
 	preempt_enable();
 }
 
@@ -86,7 +86,7 @@ static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
 		nr = 0;
 	}
 
-	if (!tb->active) {
+	if (!is_lazy_mmu_mode_active()) {
 		flush_tsb_user_page(mm, vaddr, hugepage_shift);
 		global_flush_tlb_page(mm, vaddr);
 		goto out;
diff --git a/arch/um/drivers/vector_kern.c b/arch/um/drivers/vector_kern.c
index 25d9258fa592..28cfe1c700f0 100644
--- a/arch/um/drivers/vector_kern.c
+++ b/arch/um/drivers/vector_kern.c
@@ -13,6 +13,7 @@
 #include <linux/memblock.h>
 #include <linux/etherdevice.h>
 #include <linux/ethtool.h>
+#include <linux/hex.h>
 #include <linux/inetdevice.h>
 #include <linux/init.h>
 #include <linux/list.h>
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 80527299f859..2427a66cb0fe 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -808,6 +808,7 @@ config PARAVIRT
 config PARAVIRT_XXL
 	bool
 	depends on X86_64
+	select ARCH_HAS_LAZY_MMU_MODE
 
 config PARAVIRT_DEBUG
 	bool "paravirt-ops debugging"
diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h
index fd855e32c9b9..4f86c5903e03 100644
--- a/arch/x86/boot/compressed/misc.h
+++ b/arch/x86/boot/compressed/misc.h
@@ -11,6 +11,7 @@
 #undef CONFIG_PARAVIRT
 #undef CONFIG_PARAVIRT_XXL
 #undef CONFIG_PARAVIRT_SPINLOCKS
+#undef CONFIG_ARCH_HAS_LAZY_MMU_MODE
 #undef CONFIG_KASAN
 #undef CONFIG_KASAN_GENERIC
 
diff --git a/arch/x86/boot/startup/sme.c b/arch/x86/boot/startup/sme.c
index e7ea65f3f1d6..b76a7c95dfe1 100644
--- a/arch/x86/boot/startup/sme.c
+++ b/arch/x86/boot/startup/sme.c
@@ -24,6 +24,7 @@
 #undef CONFIG_PARAVIRT
 #undef CONFIG_PARAVIRT_XXL
 #undef CONFIG_PARAVIRT_SPINLOCKS
+#undef CONFIG_ARCH_HAS_LAZY_MMU_MODE
 
 /*
  * This code runs before CPU feature bits are set. By default, the
diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig
index 3fd2423d3cf8..ebb0838eaf30 100644
--- a/arch/x86/crypto/Kconfig
+++ b/arch/x86/crypto/Kconfig
@@ -333,26 +333,6 @@ config CRYPTO_AEGIS128_AESNI_SSE2
 	  - AES-NI (AES New Instructions)
 	  - SSE4.1 (Streaming SIMD Extensions 4.1)
 
-config CRYPTO_NHPOLY1305_SSE2
-	tristate "Hash functions: NHPoly1305 (SSE2)"
-	depends on 64BIT
-	select CRYPTO_NHPOLY1305
-	help
-	  NHPoly1305 hash function for Adiantum
-
-	  Architecture: x86_64 using:
-	  - SSE2 (Streaming SIMD Extensions 2)
-
-config CRYPTO_NHPOLY1305_AVX2
-	tristate "Hash functions: NHPoly1305 (AVX2)"
-	depends on 64BIT
-	select CRYPTO_NHPOLY1305
-	help
-	  NHPoly1305 hash function for Adiantum
-
-	  Architecture: x86_64 using:
-	  - AVX2 (Advanced Vector Extensions 2)
-
 config CRYPTO_SM3_AVX_X86_64
 	tristate "Hash functions: SM3 (AVX)"
 	depends on 64BIT
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index 5f2fb4f148fe..b21ad0978c52 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -53,11 +53,6 @@ aesni-intel-$(CONFIG_64BIT) += aes-ctr-avx-x86_64.o \
 obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
 ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
 
-obj-$(CONFIG_CRYPTO_NHPOLY1305_SSE2) += nhpoly1305-sse2.o
-nhpoly1305-sse2-y := nh-sse2-x86_64.o nhpoly1305-sse2-glue.o
-obj-$(CONFIG_CRYPTO_NHPOLY1305_AVX2) += nhpoly1305-avx2.o
-nhpoly1305-avx2-y := nh-avx2-x86_64.o nhpoly1305-avx2-glue.o
-
 obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o
 sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o
 
diff --git a/arch/x86/crypto/nh-avx2-x86_64.S b/arch/x86/crypto/nh-avx2-x86_64.S
deleted file mode 100644
index 791386d9a83a..000000000000
--- a/arch/x86/crypto/nh-avx2-x86_64.S
+++ /dev/null
@@ -1,159 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * NH - ε-almost-universal hash function, x86_64 AVX2 accelerated
- *
- * Copyright 2018 Google LLC
- *
- * Author: Eric Biggers <ebiggers@google.com>
- */
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-#define		PASS0_SUMS	%ymm0
-#define		PASS1_SUMS	%ymm1
-#define		PASS2_SUMS	%ymm2
-#define		PASS3_SUMS	%ymm3
-#define		K0		%ymm4
-#define		K0_XMM		%xmm4
-#define		K1		%ymm5
-#define		K1_XMM		%xmm5
-#define		K2		%ymm6
-#define		K2_XMM		%xmm6
-#define		K3		%ymm7
-#define		K3_XMM		%xmm7
-#define		T0		%ymm8
-#define		T1		%ymm9
-#define		T2		%ymm10
-#define		T2_XMM		%xmm10
-#define		T3		%ymm11
-#define		T3_XMM		%xmm11
-#define		T4		%ymm12
-#define		T5		%ymm13
-#define		T6		%ymm14
-#define		T7		%ymm15
-#define		KEY		%rdi
-#define		MESSAGE		%rsi
-#define		MESSAGE_LEN	%rdx
-#define		HASH		%rcx
-
-.macro _nh_2xstride	k0, k1, k2, k3
-
-	// Add message words to key words
-	vpaddd		\k0, T3, T0
-	vpaddd		\k1, T3, T1
-	vpaddd		\k2, T3, T2
-	vpaddd		\k3, T3, T3
-
-	// Multiply 32x32 => 64 and accumulate
-	vpshufd		$0x10, T0, T4
-	vpshufd		$0x32, T0, T0
-	vpshufd		$0x10, T1, T5
-	vpshufd		$0x32, T1, T1
-	vpshufd		$0x10, T2, T6
-	vpshufd		$0x32, T2, T2
-	vpshufd		$0x10, T3, T7
-	vpshufd		$0x32, T3, T3
-	vpmuludq	T4, T0, T0
-	vpmuludq	T5, T1, T1
-	vpmuludq	T6, T2, T2
-	vpmuludq	T7, T3, T3
-	vpaddq		T0, PASS0_SUMS, PASS0_SUMS
-	vpaddq		T1, PASS1_SUMS, PASS1_SUMS
-	vpaddq		T2, PASS2_SUMS, PASS2_SUMS
-	vpaddq		T3, PASS3_SUMS, PASS3_SUMS
-.endm
-
-/*
- * void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
- *		__le64 hash[NH_NUM_PASSES])
- *
- * It's guaranteed that message_len % 16 == 0.
- */
-SYM_TYPED_FUNC_START(nh_avx2)
-
-	vmovdqu		0x00(KEY), K0
-	vmovdqu		0x10(KEY), K1
-	add		$0x20, KEY
-	vpxor		PASS0_SUMS, PASS0_SUMS, PASS0_SUMS
-	vpxor		PASS1_SUMS, PASS1_SUMS, PASS1_SUMS
-	vpxor		PASS2_SUMS, PASS2_SUMS, PASS2_SUMS
-	vpxor		PASS3_SUMS, PASS3_SUMS, PASS3_SUMS
-
-	sub		$0x40, MESSAGE_LEN
-	jl		.Lloop4_done
-.Lloop4:
-	vmovdqu		(MESSAGE), T3
-	vmovdqu		0x00(KEY), K2
-	vmovdqu		0x10(KEY), K3
-	_nh_2xstride	K0, K1, K2, K3
-
-	vmovdqu		0x20(MESSAGE), T3
-	vmovdqu		0x20(KEY), K0
-	vmovdqu		0x30(KEY), K1
-	_nh_2xstride	K2, K3, K0, K1
-
-	add		$0x40, MESSAGE
-	add		$0x40, KEY
-	sub		$0x40, MESSAGE_LEN
-	jge		.Lloop4
-
-.Lloop4_done:
-	and		$0x3f, MESSAGE_LEN
-	jz		.Ldone
-
-	cmp		$0x20, MESSAGE_LEN
-	jl		.Llast
-
-	// 2 or 3 strides remain; do 2 more.
-	vmovdqu		(MESSAGE), T3
-	vmovdqu		0x00(KEY), K2
-	vmovdqu		0x10(KEY), K3
-	_nh_2xstride	K0, K1, K2, K3
-	add		$0x20, MESSAGE
-	add		$0x20, KEY
-	sub		$0x20, MESSAGE_LEN
-	jz		.Ldone
-	vmovdqa		K2, K0
-	vmovdqa		K3, K1
-.Llast:
-	// Last stride.  Zero the high 128 bits of the message and keys so they
-	// don't affect the result when processing them like 2 strides.
-	vmovdqu		(MESSAGE), T3_XMM
-	vmovdqa		K0_XMM, K0_XMM
-	vmovdqa		K1_XMM, K1_XMM
-	vmovdqu		0x00(KEY), K2_XMM
-	vmovdqu		0x10(KEY), K3_XMM
-	_nh_2xstride	K0, K1, K2, K3
-
-.Ldone:
-	// Sum the accumulators for each pass, then store the sums to 'hash'
-
-	// PASS0_SUMS is (0A 0B 0C 0D)
-	// PASS1_SUMS is (1A 1B 1C 1D)
-	// PASS2_SUMS is (2A 2B 2C 2D)
-	// PASS3_SUMS is (3A 3B 3C 3D)
-	// We need the horizontal sums:
-	//     (0A + 0B + 0C + 0D,
-	//	1A + 1B + 1C + 1D,
-	//	2A + 2B + 2C + 2D,
-	//	3A + 3B + 3C + 3D)
-	//
-
-	vpunpcklqdq	PASS1_SUMS, PASS0_SUMS, T0	// T0 = (0A 1A 0C 1C)
-	vpunpckhqdq	PASS1_SUMS, PASS0_SUMS, T1	// T1 = (0B 1B 0D 1D)
-	vpunpcklqdq	PASS3_SUMS, PASS2_SUMS, T2	// T2 = (2A 3A 2C 3C)
-	vpunpckhqdq	PASS3_SUMS, PASS2_SUMS, T3	// T3 = (2B 3B 2D 3D)
-
-	vinserti128	$0x1, T2_XMM, T0, T4		// T4 = (0A 1A 2A 3A)
-	vinserti128	$0x1, T3_XMM, T1, T5		// T5 = (0B 1B 2B 3B)
-	vperm2i128	$0x31, T2, T0, T0		// T0 = (0C 1C 2C 3C)
-	vperm2i128	$0x31, T3, T1, T1		// T1 = (0D 1D 2D 3D)
-
-	vpaddq		T5, T4, T4
-	vpaddq		T1, T0, T0
-	vpaddq		T4, T0, T0
-	vmovdqu		T0, (HASH)
-	vzeroupper
-	RET
-SYM_FUNC_END(nh_avx2)
diff --git a/arch/x86/crypto/nh-sse2-x86_64.S b/arch/x86/crypto/nh-sse2-x86_64.S
deleted file mode 100644
index 75fb994b6d17..000000000000
--- a/arch/x86/crypto/nh-sse2-x86_64.S
+++ /dev/null
@@ -1,124 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * NH - ε-almost-universal hash function, x86_64 SSE2 accelerated
- *
- * Copyright 2018 Google LLC
- *
- * Author: Eric Biggers <ebiggers@google.com>
- */
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-#define		PASS0_SUMS	%xmm0
-#define		PASS1_SUMS	%xmm1
-#define		PASS2_SUMS	%xmm2
-#define		PASS3_SUMS	%xmm3
-#define		K0		%xmm4
-#define		K1		%xmm5
-#define		K2		%xmm6
-#define		K3		%xmm7
-#define		T0		%xmm8
-#define		T1		%xmm9
-#define		T2		%xmm10
-#define		T3		%xmm11
-#define		T4		%xmm12
-#define		T5		%xmm13
-#define		T6		%xmm14
-#define		T7		%xmm15
-#define		KEY		%rdi
-#define		MESSAGE		%rsi
-#define		MESSAGE_LEN	%rdx
-#define		HASH		%rcx
-
-.macro _nh_stride	k0, k1, k2, k3, offset
-
-	// Load next message stride
-	movdqu		\offset(MESSAGE), T1
-
-	// Load next key stride
-	movdqu		\offset(KEY), \k3
-
-	// Add message words to key words
-	movdqa		T1, T2
-	movdqa		T1, T3
-	paddd		T1, \k0    // reuse k0 to avoid a move
-	paddd		\k1, T1
-	paddd		\k2, T2
-	paddd		\k3, T3
-
-	// Multiply 32x32 => 64 and accumulate
-	pshufd		$0x10, \k0, T4
-	pshufd		$0x32, \k0, \k0
-	pshufd		$0x10, T1, T5
-	pshufd		$0x32, T1, T1
-	pshufd		$0x10, T2, T6
-	pshufd		$0x32, T2, T2
-	pshufd		$0x10, T3, T7
-	pshufd		$0x32, T3, T3
-	pmuludq		T4, \k0
-	pmuludq		T5, T1
-	pmuludq		T6, T2
-	pmuludq		T7, T3
-	paddq		\k0, PASS0_SUMS
-	paddq		T1, PASS1_SUMS
-	paddq		T2, PASS2_SUMS
-	paddq		T3, PASS3_SUMS
-.endm
-
-/*
- * void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
- *		__le64 hash[NH_NUM_PASSES])
- *
- * It's guaranteed that message_len % 16 == 0.
- */
-SYM_TYPED_FUNC_START(nh_sse2)
-
-	movdqu		0x00(KEY), K0
-	movdqu		0x10(KEY), K1
-	movdqu		0x20(KEY), K2
-	add		$0x30, KEY
-	pxor		PASS0_SUMS, PASS0_SUMS
-	pxor		PASS1_SUMS, PASS1_SUMS
-	pxor		PASS2_SUMS, PASS2_SUMS
-	pxor		PASS3_SUMS, PASS3_SUMS
-
-	sub		$0x40, MESSAGE_LEN
-	jl		.Lloop4_done
-.Lloop4:
-	_nh_stride	K0, K1, K2, K3, 0x00
-	_nh_stride	K1, K2, K3, K0, 0x10
-	_nh_stride	K2, K3, K0, K1, 0x20
-	_nh_stride	K3, K0, K1, K2, 0x30
-	add		$0x40, KEY
-	add		$0x40, MESSAGE
-	sub		$0x40, MESSAGE_LEN
-	jge		.Lloop4
-
-.Lloop4_done:
-	and		$0x3f, MESSAGE_LEN
-	jz		.Ldone
-	_nh_stride	K0, K1, K2, K3, 0x00
-
-	sub		$0x10, MESSAGE_LEN
-	jz		.Ldone
-	_nh_stride	K1, K2, K3, K0, 0x10
-
-	sub		$0x10, MESSAGE_LEN
-	jz		.Ldone
-	_nh_stride	K2, K3, K0, K1, 0x20
-
-.Ldone:
-	// Sum the accumulators for each pass, then store the sums to 'hash'
-	movdqa		PASS0_SUMS, T0
-	movdqa		PASS2_SUMS, T1
-	punpcklqdq	PASS1_SUMS, T0		// => (PASS0_SUM_A PASS1_SUM_A)
-	punpcklqdq	PASS3_SUMS, T1		// => (PASS2_SUM_A PASS3_SUM_A)
-	punpckhqdq	PASS1_SUMS, PASS0_SUMS	// => (PASS0_SUM_B PASS1_SUM_B)
-	punpckhqdq	PASS3_SUMS, PASS2_SUMS	// => (PASS2_SUM_B PASS3_SUM_B)
-	paddq		PASS0_SUMS, T0
-	paddq		PASS2_SUMS, T1
-	movdqu		T0, 0x00(HASH)
-	movdqu		T1, 0x10(HASH)
-	RET
-SYM_FUNC_END(nh_sse2)
diff --git a/arch/x86/crypto/nhpoly1305-avx2-glue.c b/arch/x86/crypto/nhpoly1305-avx2-glue.c
deleted file mode 100644
index c3a872f4d6a7..000000000000
--- a/arch/x86/crypto/nhpoly1305-avx2-glue.c
+++ /dev/null
@@ -1,81 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
- * (AVX2 accelerated version)
- *
- * Copyright 2018 Google LLC
- */
-
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/nhpoly1305.h>
-#include <linux/module.h>
-#include <linux/sizes.h>
-#include <asm/simd.h>
-
-asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
-			__le64 hash[NH_NUM_PASSES]);
-
-static int nhpoly1305_avx2_update(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen)
-{
-	if (srclen < 64 || !crypto_simd_usable())
-		return crypto_nhpoly1305_update(desc, src, srclen);
-
-	do {
-		unsigned int n = min_t(unsigned int, srclen, SZ_4K);
-
-		kernel_fpu_begin();
-		crypto_nhpoly1305_update_helper(desc, src, n, nh_avx2);
-		kernel_fpu_end();
-		src += n;
-		srclen -= n;
-	} while (srclen);
-	return 0;
-}
-
-static int nhpoly1305_avx2_digest(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen, u8 *out)
-{
-	return crypto_nhpoly1305_init(desc) ?:
-	       nhpoly1305_avx2_update(desc, src, srclen) ?:
-	       crypto_nhpoly1305_final(desc, out);
-}
-
-static struct shash_alg nhpoly1305_alg = {
-	.base.cra_name		= "nhpoly1305",
-	.base.cra_driver_name	= "nhpoly1305-avx2",
-	.base.cra_priority	= 300,
-	.base.cra_ctxsize	= sizeof(struct nhpoly1305_key),
-	.base.cra_module	= THIS_MODULE,
-	.digestsize		= POLY1305_DIGEST_SIZE,
-	.init			= crypto_nhpoly1305_init,
-	.update			= nhpoly1305_avx2_update,
-	.final			= crypto_nhpoly1305_final,
-	.digest			= nhpoly1305_avx2_digest,
-	.setkey			= crypto_nhpoly1305_setkey,
-	.descsize		= sizeof(struct nhpoly1305_state),
-};
-
-static int __init nhpoly1305_mod_init(void)
-{
-	if (!boot_cpu_has(X86_FEATURE_AVX2) ||
-	    !boot_cpu_has(X86_FEATURE_OSXSAVE))
-		return -ENODEV;
-
-	return crypto_register_shash(&nhpoly1305_alg);
-}
-
-static void __exit nhpoly1305_mod_exit(void)
-{
-	crypto_unregister_shash(&nhpoly1305_alg);
-}
-
-module_init(nhpoly1305_mod_init);
-module_exit(nhpoly1305_mod_exit);
-
-MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (AVX2-accelerated)");
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
-MODULE_ALIAS_CRYPTO("nhpoly1305");
-MODULE_ALIAS_CRYPTO("nhpoly1305-avx2");
diff --git a/arch/x86/crypto/nhpoly1305-sse2-glue.c b/arch/x86/crypto/nhpoly1305-sse2-glue.c
deleted file mode 100644
index a268a8439a5c..000000000000
--- a/arch/x86/crypto/nhpoly1305-sse2-glue.c
+++ /dev/null
@@ -1,80 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
- * (SSE2 accelerated version)
- *
- * Copyright 2018 Google LLC
- */
-
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/nhpoly1305.h>
-#include <linux/module.h>
-#include <linux/sizes.h>
-#include <asm/simd.h>
-
-asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
-			__le64 hash[NH_NUM_PASSES]);
-
-static int nhpoly1305_sse2_update(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen)
-{
-	if (srclen < 64 || !crypto_simd_usable())
-		return crypto_nhpoly1305_update(desc, src, srclen);
-
-	do {
-		unsigned int n = min_t(unsigned int, srclen, SZ_4K);
-
-		kernel_fpu_begin();
-		crypto_nhpoly1305_update_helper(desc, src, n, nh_sse2);
-		kernel_fpu_end();
-		src += n;
-		srclen -= n;
-	} while (srclen);
-	return 0;
-}
-
-static int nhpoly1305_sse2_digest(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen, u8 *out)
-{
-	return crypto_nhpoly1305_init(desc) ?:
-	       nhpoly1305_sse2_update(desc, src, srclen) ?:
-	       crypto_nhpoly1305_final(desc, out);
-}
-
-static struct shash_alg nhpoly1305_alg = {
-	.base.cra_name		= "nhpoly1305",
-	.base.cra_driver_name	= "nhpoly1305-sse2",
-	.base.cra_priority	= 200,
-	.base.cra_ctxsize	= sizeof(struct nhpoly1305_key),
-	.base.cra_module	= THIS_MODULE,
-	.digestsize		= POLY1305_DIGEST_SIZE,
-	.init			= crypto_nhpoly1305_init,
-	.update			= nhpoly1305_sse2_update,
-	.final			= crypto_nhpoly1305_final,
-	.digest			= nhpoly1305_sse2_digest,
-	.setkey			= crypto_nhpoly1305_setkey,
-	.descsize		= sizeof(struct nhpoly1305_state),
-};
-
-static int __init nhpoly1305_mod_init(void)
-{
-	if (!boot_cpu_has(X86_FEATURE_XMM2))
-		return -ENODEV;
-
-	return crypto_register_shash(&nhpoly1305_alg);
-}
-
-static void __exit nhpoly1305_mod_exit(void)
-{
-	crypto_unregister_shash(&nhpoly1305_alg);
-}
-
-module_init(nhpoly1305_mod_init);
-module_exit(nhpoly1305_mod_exit);
-
-MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (SSE2-accelerated)");
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
-MODULE_ALIAS_CRYPTO("nhpoly1305");
-MODULE_ALIAS_CRYPTO("nhpoly1305-sse2");
diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c
index fa67fda6e45b..1e2658b60d91 100644
--- a/arch/x86/events/intel/cstate.c
+++ b/arch/x86/events/intel/cstate.c
@@ -41,7 +41,7 @@
  *	MSR_CORE_C1_RES: CORE C1 Residency Counter
  *			 perf code: 0x00
  *			 Available model: SLM,AMT,GLM,CNL,ICX,TNT,ADL,RPL
- *					  MTL,SRF,GRR,ARL,LNL,PTL
+ *					  MTL,SRF,GRR,ARL,LNL,PTL,WCL,NVL
  *			 Scope: Core (each processor core has a MSR)
  *	MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter
  *			       perf code: 0x01
@@ -53,19 +53,20 @@
  *			       Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,
  *						SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX,
  *						TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF,
- *						GRR,ARL,LNL,PTL
+ *						GRR,ARL,LNL,PTL,WCL,NVL
  *			       Scope: Core
  *	MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter
  *			       perf code: 0x03
  *			       Available model: SNB,IVB,HSW,BDW,SKL,CNL,KBL,CML,
  *						ICL,TGL,RKL,ADL,RPL,MTL,ARL,LNL,
- *						PTL
+ *						PTL,WCL,NVL
  *			       Scope: Core
  *	MSR_PKG_C2_RESIDENCY:  Package C2 Residency Counter.
  *			       perf code: 0x00
  *			       Available model: SNB,IVB,HSW,BDW,SKL,KNL,GLM,CNL,
  *						KBL,CML,ICL,ICX,TGL,TNT,RKL,ADL,
- *						RPL,SPR,MTL,ARL,LNL,SRF,PTL
+ *						RPL,SPR,MTL,ARL,LNL,SRF,PTL,WCL,
+ *						NVL
  *			       Scope: Package (physical package)
  *	MSR_PKG_C3_RESIDENCY:  Package C3 Residency Counter.
  *			       perf code: 0x01
@@ -78,7 +79,7 @@
  *			       Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,
  *						SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX,
  *						TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF,
- *						ARL,LNL,PTL
+ *						ARL,LNL,PTL,WCL,NVL
  *			       Scope: Package (physical package)
  *	MSR_PKG_C7_RESIDENCY:  Package C7 Residency Counter.
  *			       perf code: 0x03
@@ -97,11 +98,12 @@
  *	MSR_PKG_C10_RESIDENCY: Package C10 Residency Counter.
  *			       perf code: 0x06
  *			       Available model: HSW ULT,KBL,GLM,CNL,CML,ICL,TGL,
- *						TNT,RKL,ADL,RPL,MTL,ARL,LNL,PTL
+ *						TNT,RKL,ADL,RPL,MTL,ARL,LNL,PTL,
+ *						WCL,NVL
  *			       Scope: Package (physical package)
  *	MSR_MODULE_C6_RES_MS:  Module C6 Residency Counter.
  *			       perf code: 0x00
- *			       Available model: SRF,GRR
+ *			       Available model: SRF,GRR,NVL
  *			       Scope: A cluster of cores shared L2 cache
  *
  */
@@ -527,6 +529,18 @@ static const struct cstate_model lnl_cstates __initconst = {
 				  BIT(PERF_CSTATE_PKG_C10_RES),
 };
 
+static const struct cstate_model nvl_cstates __initconst = {
+	.core_events		= BIT(PERF_CSTATE_CORE_C1_RES) |
+				  BIT(PERF_CSTATE_CORE_C6_RES) |
+				  BIT(PERF_CSTATE_CORE_C7_RES),
+
+	.module_events		= BIT(PERF_CSTATE_MODULE_C6_RES),
+
+	.pkg_events		= BIT(PERF_CSTATE_PKG_C2_RES) |
+				  BIT(PERF_CSTATE_PKG_C6_RES) |
+				  BIT(PERF_CSTATE_PKG_C10_RES),
+};
+
 static const struct cstate_model slm_cstates __initconst = {
 	.core_events		= BIT(PERF_CSTATE_CORE_C1_RES) |
 				  BIT(PERF_CSTATE_CORE_C6_RES),
@@ -638,6 +652,7 @@ static const struct x86_cpu_id intel_cstates_match[] __initconst = {
 	X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X,	&icx_cstates),
 	X86_MATCH_VFM(INTEL_GRANITERAPIDS_X,	&icx_cstates),
 	X86_MATCH_VFM(INTEL_GRANITERAPIDS_D,	&icx_cstates),
+	X86_MATCH_VFM(INTEL_DIAMONDRAPIDS_X,	&srf_cstates),
 
 	X86_MATCH_VFM(INTEL_TIGERLAKE_L,	&icl_cstates),
 	X86_MATCH_VFM(INTEL_TIGERLAKE,		&icl_cstates),
@@ -654,6 +669,9 @@ static const struct x86_cpu_id intel_cstates_match[] __initconst = {
 	X86_MATCH_VFM(INTEL_ARROWLAKE_U,	&adl_cstates),
 	X86_MATCH_VFM(INTEL_LUNARLAKE_M,	&lnl_cstates),
 	X86_MATCH_VFM(INTEL_PANTHERLAKE_L,	&lnl_cstates),
+	X86_MATCH_VFM(INTEL_WILDCATLAKE_L,	&lnl_cstates),
+	X86_MATCH_VFM(INTEL_NOVALAKE,		&nvl_cstates),
+	X86_MATCH_VFM(INTEL_NOVALAKE_L,		&nvl_cstates),
 	{ },
 };
 MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match);
diff --git a/arch/x86/include/asm/e820/api.h b/arch/x86/include/asm/e820/api.h
index c83645d5b2a8..bbe0c8de976c 100644
--- a/arch/x86/include/asm/e820/api.h
+++ b/arch/x86/include/asm/e820/api.h
@@ -16,10 +16,9 @@ extern bool e820__mapped_all(u64 start, u64 end, enum e820_type type);
 
 extern void e820__range_add   (u64 start, u64 size, enum e820_type type);
 extern u64  e820__range_update(u64 start, u64 size, enum e820_type old_type, enum e820_type new_type);
-extern u64  e820__range_remove(u64 start, u64 size, enum e820_type old_type, bool check_type);
+extern void e820__range_remove(u64 start, u64 size, enum e820_type filter_type);
 extern u64  e820__range_update_table(struct e820_table *t, u64 start, u64 size, enum e820_type old_type, enum e820_type new_type);
 
-extern void e820__print_table(char *who);
 extern int  e820__update_table(struct e820_table *table);
 extern void e820__update_table_print(void);
 
diff --git a/arch/x86/include/asm/e820/types.h b/arch/x86/include/asm/e820/types.h
index 80c4a7266629..df12f7ee75d3 100644
--- a/arch/x86/include/asm/e820/types.h
+++ b/arch/x86/include/asm/e820/types.h
@@ -83,7 +83,7 @@ struct e820_entry {
  * The whole array of E820 entries:
  */
 struct e820_table {
-	__u32 nr_entries;
+	u32 nr_entries;
 	struct e820_entry entries[E820_MAX_ENTRIES];
 };
 
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index b5e59a7ba0d0..13f9cd31c8f8 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -526,7 +526,6 @@ static inline void arch_end_context_switch(struct task_struct *next)
 	PVOP_VCALL1(cpu.end_context_switch, next);
 }
 
-#define  __HAVE_ARCH_ENTER_LAZY_MMU_MODE
 static inline void arch_enter_lazy_mmu_mode(void)
 {
 	PVOP_VCALL0(mmu.lazy_mode.enter);
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index e33df3da6980..2842fa1f7a2c 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -118,6 +118,7 @@ extern pmdval_t early_pmd_flags;
 #define __pte(x)	native_make_pte(x)
 
 #define arch_end_context_switch(prev)	do {} while(0)
+static inline void arch_flush_lazy_mmu_mode(void) {}
 #endif	/* CONFIG_PARAVIRT_XXL */
 
 static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index e71e0e8362ed..0067684afb5b 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -100,8 +100,7 @@ struct thread_info {
 #define TIF_FORCED_TF		24	/* true if TF in eflags artificially */
 #define TIF_SINGLESTEP		25	/* reenable singlestep on user return*/
 #define TIF_BLOCKSTEP		26	/* set when we want DEBUGCTLMSR_BTF */
-#define TIF_LAZY_MMU_UPDATES	27	/* task is updating the mmu lazily */
-#define TIF_ADDR32		28	/* 32-bit address space on 64 bits */
+#define TIF_ADDR32		27	/* 32-bit address space on 64 bits */
 
 #define _TIF_SSBD		BIT(TIF_SSBD)
 #define _TIF_SPEC_IB		BIT(TIF_SPEC_IB)
@@ -114,7 +113,6 @@ struct thread_info {
 #define _TIF_FORCED_TF		BIT(TIF_FORCED_TF)
 #define _TIF_BLOCKSTEP		BIT(TIF_BLOCKSTEP)
 #define _TIF_SINGLESTEP		BIT(TIF_SINGLESTEP)
-#define _TIF_LAZY_MMU_UPDATES	BIT(TIF_LAZY_MMU_UPDATES)
 #define _TIF_ADDR32		BIT(TIF_ADDR32)
 
 /* flags to check in __switch_to() */
diff --git a/arch/x86/include/asm/uv/bios.h b/arch/x86/include/asm/uv/bios.h
index 6989b824fd32..d0b62e255290 100644
--- a/arch/x86/include/asm/uv/bios.h
+++ b/arch/x86/include/asm/uv/bios.h
@@ -122,7 +122,7 @@ struct uv_systab {
 	struct {
 		u32 type:8;	/* type of entry */
 		u32 offset:24;	/* byte offset from struct start to entry */
-	} entry[1];		/* additional entries follow */
+	} entry[];		/* additional entries follow */
 };
 extern struct uv_systab *uv_systab;
 
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index 9fa321a95eb3..d6138b2b633a 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -35,6 +35,7 @@
 #include <asm/smp.h>
 #include <asm/i8259.h>
 #include <asm/setup.h>
+#include <asm/hypervisor.h>
 
 #include "sleep.h" /* To include x86_acpi_suspend_lowlevel */
 static int __initdata acpi_force = 0;
@@ -164,11 +165,14 @@ static bool __init acpi_is_processor_usable(u32 lapic_flags)
 	if (lapic_flags & ACPI_MADT_ENABLED)
 		return true;
 
-	if (!acpi_support_online_capable ||
-	    (lapic_flags & ACPI_MADT_ONLINE_CAPABLE))
-		return true;
+	if (acpi_support_online_capable)
+		return lapic_flags & ACPI_MADT_ONLINE_CAPABLE;
 
-	return false;
+	/*
+	 * QEMU expects legacy "Enabled=0" LAPIC entries to be counted as usable
+	 * in order to support CPU hotplug in guests.
+	 */
+	return !hypervisor_is_type(X86_HYPER_NATIVE);
 }
 
 static int __init
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index bc94ff1e250a..c792c2afd849 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -873,8 +873,8 @@ static void init_amd_bd(struct cpuinfo_x86 *c)
 }
 
 static const struct x86_cpu_id erratum_1386_microcode[] = {
-	X86_MATCH_VFM_STEPS(VFM_MAKE(X86_VENDOR_AMD, 0x17, 0x01), 0x2, 0x2, 0x0800126e),
-	X86_MATCH_VFM_STEPS(VFM_MAKE(X86_VENDOR_AMD, 0x17, 0x31), 0x0, 0x0, 0x08301052),
+	ZEN_MODEL_STEP_UCODE(0x17, 0x01, 0x2, 0x0800126e),
+	ZEN_MODEL_STEP_UCODE(0x17, 0x31, 0x0, 0x08301052),
 	{}
 };
 
@@ -951,26 +951,14 @@ static void init_amd_zen1(struct cpuinfo_x86 *c)
 	}
 }
 
-static bool cpu_has_zenbleed_microcode(void)
-{
-	u32 good_rev = 0;
-
-	switch (boot_cpu_data.x86_model) {
-	case 0x30 ... 0x3f: good_rev = 0x0830107b; break;
-	case 0x60 ... 0x67: good_rev = 0x0860010c; break;
-	case 0x68 ... 0x6f: good_rev = 0x08608107; break;
-	case 0x70 ... 0x7f: good_rev = 0x08701033; break;
-	case 0xa0 ... 0xaf: good_rev = 0x08a00009; break;
-
-	default:
-		return false;
-	}
-
-	if (boot_cpu_data.microcode < good_rev)
-		return false;
-
-	return true;
-}
+static const struct x86_cpu_id amd_zenbleed_microcode[] = {
+	ZEN_MODEL_STEP_UCODE(0x17, 0x31, 0x0, 0x0830107b),
+	ZEN_MODEL_STEP_UCODE(0x17, 0x60, 0x1, 0x0860010c),
+	ZEN_MODEL_STEP_UCODE(0x17, 0x68, 0x1, 0x08608107),
+	ZEN_MODEL_STEP_UCODE(0x17, 0x71, 0x0, 0x08701033),
+	ZEN_MODEL_STEP_UCODE(0x17, 0xa0, 0x0, 0x08a00009),
+	{}
+};
 
 static void zen2_zenbleed_check(struct cpuinfo_x86 *c)
 {
@@ -980,7 +968,7 @@ static void zen2_zenbleed_check(struct cpuinfo_x86 *c)
 	if (!cpu_has(c, X86_FEATURE_AVX))
 		return;
 
-	if (!cpu_has_zenbleed_microcode()) {
+	if (!x86_match_min_microcode_rev(amd_zenbleed_microcode)) {
 		pr_notice_once("Zenbleed: please update your microcode for the most optimal fix\n");
 		msr_set_bit(MSR_AMD64_DE_CFG, MSR_AMD64_DE_CFG_ZEN2_FP_BACKUP_FIX_BIT);
 	} else {
@@ -1063,12 +1051,6 @@ static void init_amd(struct cpuinfo_x86 *c)
 
 	early_init_amd(c);
 
-	/*
-	 * Bit 31 in normal CPUID used for nonstandard 3DNow ID;
-	 * 3DNow is IDd by bit 31 in extended CPUID (1*32+31) anyway
-	 */
-	clear_cpu_cap(c, 0*32+31);
-
 	if (c->x86 >= 0x10)
 		set_cpu_cap(c, X86_FEATURE_REP_GOOD);
 
diff --git a/arch/x86/kernel/cpu/centaur.c b/arch/x86/kernel/cpu/centaur.c
index a3b55db35c96..c8398940b975 100644
--- a/arch/x86/kernel/cpu/centaur.c
+++ b/arch/x86/kernel/cpu/centaur.c
@@ -119,12 +119,6 @@ static void init_centaur(struct cpuinfo_x86 *c)
 	u32  fcr_clr = 0;
 	u32  lo, hi, newlo;
 	u32  aa, bb, cc, dd;
-
-	/*
-	 * Bit 31 in normal CPUID used for nonstandard 3DNow ID;
-	 * 3DNow is IDd by bit 31 in extended CPUID (1*32+31) anyway
-	 */
-	clear_cpu_cap(c, 0*32+31);
 #endif
 	early_init_centaur(c);
 	init_intel_cacheinfo(c);
diff --git a/arch/x86/kernel/cpu/cyrix.c b/arch/x86/kernel/cpu/cyrix.c
index dfec2c61e354..8f22085c4dd2 100644
--- a/arch/x86/kernel/cpu/cyrix.c
+++ b/arch/x86/kernel/cpu/cyrix.c
@@ -195,12 +195,6 @@ static void init_cyrix(struct cpuinfo_x86 *c)
 	char *buf = c->x86_model_id;
 	const char *p = NULL;
 
-	/*
-	 * Bit 31 in normal CPUID used for nonstandard 3DNow ID;
-	 * 3DNow is IDd by bit 31 in extended CPUID (1*32+31) anyway
-	 */
-	clear_cpu_cap(c, 0*32+31);
-
 	/* Cyrix used bit 24 in extended (AMD) CPUID for Cyrix MMX extensions */
 	if (test_cpu_cap(c, 1*32+24)) {
 		clear_cpu_cap(c, 1*32+24);
diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c
index 1fda6c3a2b65..7f95a74e4c65 100644
--- a/arch/x86/kernel/cpu/hygon.c
+++ b/arch/x86/kernel/cpu/hygon.c
@@ -174,12 +174,6 @@ static void init_hygon(struct cpuinfo_x86 *c)
 
 	early_init_hygon(c);
 
-	/*
-	 * Bit 31 in normal CPUID used for nonstandard 3DNow ID;
-	 * 3DNow is IDd by bit 31 in extended CPUID (1*32+31) anyway
-	 */
-	clear_cpu_cap(c, 0*32+31);
-
 	set_cpu_cap(c, X86_FEATURE_REP_GOOD);
 
 	/*
diff --git a/arch/x86/kernel/cpu/sgx/ioctl.c b/arch/x86/kernel/cpu/sgx/ioctl.c
index 66f1efa16fbb..9322a9287dc7 100644
--- a/arch/x86/kernel/cpu/sgx/ioctl.c
+++ b/arch/x86/kernel/cpu/sgx/ioctl.c
@@ -242,7 +242,7 @@ static int __sgx_encl_add_page(struct sgx_encl *encl,
 /*
  * If the caller requires measurement of the page as a proof for the content,
  * use EEXTEND to add a measurement for 256 bytes of the page. Repeat this
- * operation until the entire page is measured."
+ * operation until the entire page is measured.
  */
 static int __sgx_encl_extend(struct sgx_encl *encl,
 			     struct sgx_epc_page *epc_page)
diff --git a/arch/x86/kernel/cpu/topology.c b/arch/x86/kernel/cpu/topology.c
index f55ea3cdbf88..23190a786d31 100644
--- a/arch/x86/kernel/cpu/topology.c
+++ b/arch/x86/kernel/cpu/topology.c
@@ -27,7 +27,6 @@
 #include <xen/xen.h>
 
 #include <asm/apic.h>
-#include <asm/hypervisor.h>
 #include <asm/io_apic.h>
 #include <asm/mpspec.h>
 #include <asm/msr.h>
@@ -236,20 +235,6 @@ static __init void topo_register_apic(u32 apic_id, u32 acpi_id, bool present)
 		cpuid_to_apicid[cpu] = apic_id;
 		topo_set_cpuids(cpu, apic_id, acpi_id);
 	} else {
-		u32 pkgid = topo_apicid(apic_id, TOPO_PKG_DOMAIN);
-
-		/*
-		 * Check for present APICs in the same package when running
-		 * on bare metal. Allow the bogosity in a guest.
-		 */
-		if (hypervisor_is_type(X86_HYPER_NATIVE) &&
-		    topo_unit_count(pkgid, TOPO_PKG_DOMAIN, phys_cpu_present_map)) {
-			pr_info_once("Ignoring hot-pluggable APIC ID %x in present package.\n",
-				     apic_id);
-			topo_info.nr_rejected_cpus++;
-			return;
-		}
-
 		topo_info.nr_disabled_cpus++;
 	}
 
diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c
index b15b97d3cb52..97b54bd0f482 100644
--- a/arch/x86/kernel/e820.c
+++ b/arch/x86/kernel/e820.c
@@ -9,13 +9,11 @@
  * quirks and other tweaks, and feeds that into the generic Linux memory
  * allocation code routines via a platform independent interface (memblock, etc.).
  */
-#include <linux/crash_dump.h>
 #include <linux/memblock.h>
 #include <linux/suspend.h>
 #include <linux/acpi.h>
 #include <linux/firmware-map.h>
 #include <linux/sort.h>
-#include <linux/memory_hotplug.h>
 #include <linux/kvm_types.h>
 
 #include <asm/e820/api.h>
@@ -57,13 +55,13 @@
  * re-propagated. So its main role is a temporary bootstrap storage of firmware
  * specific memory layout data during early bootup.
  */
-static struct e820_table e820_table_init		__initdata;
-static struct e820_table e820_table_kexec_init		__initdata;
-static struct e820_table e820_table_firmware_init	__initdata;
+__initdata static struct e820_table e820_table_init;
+__initdata static struct e820_table e820_table_kexec_init;
+__initdata static struct e820_table e820_table_firmware_init;
 
-struct e820_table *e820_table __refdata			= &e820_table_init;
-struct e820_table *e820_table_kexec __refdata		= &e820_table_kexec_init;
-struct e820_table *e820_table_firmware __refdata	= &e820_table_firmware_init;
+__refdata struct e820_table *e820_table			= &e820_table_init;
+__refdata struct e820_table *e820_table_kexec		= &e820_table_kexec_init;
+__refdata struct e820_table *e820_table_firmware	= &e820_table_firmware_init;
 
 /* For PCI or other memory-mapped resources */
 unsigned long pci_mem_start = 0xaeedbabe;
@@ -78,10 +76,10 @@ EXPORT_SYMBOL(pci_mem_start);
 static bool _e820__mapped_any(struct e820_table *table,
 			      u64 start, u64 end, enum e820_type type)
 {
-	int i;
+	u32 idx;
 
-	for (i = 0; i < table->nr_entries; i++) {
-		struct e820_entry *entry = &table->entries[i];
+	for (idx = 0; idx < table->nr_entries; idx++) {
+		struct e820_entry *entry = &table->entries[idx];
 
 		if (type && entry->type != type)
 			continue;
@@ -113,10 +111,10 @@ EXPORT_SYMBOL_GPL(e820__mapped_any);
 static struct e820_entry *__e820__mapped_all(u64 start, u64 end,
 					     enum e820_type type)
 {
-	int i;
+	u32 idx;
 
-	for (i = 0; i < e820_table->nr_entries; i++) {
-		struct e820_entry *entry = &e820_table->entries[i];
+	for (idx = 0; idx < e820_table->nr_entries; idx++) {
+		struct e820_entry *entry = &e820_table->entries[idx];
 
 		if (type && entry->type != type)
 			continue;
@@ -146,7 +144,7 @@ static struct e820_entry *__e820__mapped_all(u64 start, u64 end,
 /*
  * This function checks if the entire range <start,end> is mapped with type.
  */
-bool __init e820__mapped_all(u64 start, u64 end, enum e820_type type)
+__init bool e820__mapped_all(u64 start, u64 end, enum e820_type type)
 {
 	return __e820__mapped_all(start, end, type);
 }
@@ -164,54 +162,74 @@ int e820__get_entry_type(u64 start, u64 end)
 /*
  * Add a memory region to the kernel E820 map.
  */
-static void __init __e820__range_add(struct e820_table *table, u64 start, u64 size, enum e820_type type)
+__init static void __e820__range_add(struct e820_table *table, u64 start, u64 size, enum e820_type type)
 {
-	int x = table->nr_entries;
+	u32 idx = table->nr_entries;
+	struct e820_entry *entry_new;
 
-	if (x >= ARRAY_SIZE(table->entries)) {
-		pr_err("too many entries; ignoring [mem %#010llx-%#010llx]\n",
-		       start, start + size - 1);
+	if (idx >= ARRAY_SIZE(table->entries)) {
+		pr_err("E820 table full; ignoring [mem %#010llx-%#010llx]\n",
+		       start, start + size-1);
 		return;
 	}
 
-	table->entries[x].addr = start;
-	table->entries[x].size = size;
-	table->entries[x].type = type;
+	entry_new = table->entries + idx;
+
+	entry_new->addr = start;
+	entry_new->size = size;
+	entry_new->type = type;
+
 	table->nr_entries++;
 }
 
-void __init e820__range_add(u64 start, u64 size, enum e820_type type)
+__init void e820__range_add(u64 start, u64 size, enum e820_type type)
 {
 	__e820__range_add(e820_table, start, size, type);
 }
 
-static void __init e820_print_type(enum e820_type type)
+__init static void e820_print_type(enum e820_type type)
 {
 	switch (type) {
-	case E820_TYPE_RAM:		pr_cont("usable");			break;
-	case E820_TYPE_RESERVED:	pr_cont("reserved");			break;
-	case E820_TYPE_SOFT_RESERVED:	pr_cont("soft reserved");		break;
-	case E820_TYPE_ACPI:		pr_cont("ACPI data");			break;
-	case E820_TYPE_NVS:		pr_cont("ACPI NVS");			break;
-	case E820_TYPE_UNUSABLE:	pr_cont("unusable");			break;
+	case E820_TYPE_RAM:		pr_cont(" System RAM");				break;
+	case E820_TYPE_RESERVED:	pr_cont(" device reserved");			break;
+	case E820_TYPE_SOFT_RESERVED:	pr_cont(" soft reserved");			break;
+	case E820_TYPE_ACPI:		pr_cont(" ACPI data");				break;
+	case E820_TYPE_NVS:		pr_cont(" ACPI NVS");				break;
+	case E820_TYPE_UNUSABLE:	pr_cont(" unusable");				break;
 	case E820_TYPE_PMEM:		/* Fall through: */
-	case E820_TYPE_PRAM:		pr_cont("persistent (type %u)", type);	break;
-	default:			pr_cont("type %u", type);		break;
+	case E820_TYPE_PRAM:		pr_cont(" persistent RAM (type %u)", type);	break;
+	default:			pr_cont(" type %u", type);			break;
 	}
 }
 
-void __init e820__print_table(char *who)
+__init static void e820__print_table(const char *who)
 {
-	int i;
+	u64 range_end_prev = 0;
+	u32 idx;
+
+	for (idx = 0; idx < e820_table->nr_entries; idx++) {
+		struct e820_entry *entry = e820_table->entries + idx;
+		u64 range_start, range_end;
+
+		range_start = entry->addr;
+		range_end   = entry->addr + entry->size;
 
-	for (i = 0; i < e820_table->nr_entries; i++) {
-		pr_info("%s: [mem %#018Lx-%#018Lx] ",
-			who,
-			e820_table->entries[i].addr,
-			e820_table->entries[i].addr + e820_table->entries[i].size - 1);
+		/* Out of order E820 maps should not happen: */
+		if (range_start < range_end_prev)
+			pr_info(FW_BUG "out of order E820 entry!\n");
 
-		e820_print_type(e820_table->entries[i].type);
+		if (range_start > range_end_prev) {
+			pr_info("%s: [gap %#018Lx-%#018Lx]\n",
+				who,
+				range_end_prev,
+				range_start-1);
+		}
+
+		pr_info("%s: [mem %#018Lx-%#018Lx] ", who, range_start, range_end-1);
+		e820_print_type(entry->type);
 		pr_cont("\n");
+
+		range_end_prev = range_end;
 	}
 }
 
@@ -280,15 +298,15 @@ struct change_member {
 	/* Pointer to the original entry: */
 	struct e820_entry	*entry;
 	/* Address for this change point: */
-	unsigned long long	addr;
+	u64			addr;
 };
 
-static struct change_member	change_point_list[2*E820_MAX_ENTRIES]	__initdata;
-static struct change_member	*change_point[2*E820_MAX_ENTRIES]	__initdata;
-static struct e820_entry	*overlap_list[E820_MAX_ENTRIES]		__initdata;
-static struct e820_entry	new_entries[E820_MAX_ENTRIES]		__initdata;
+__initdata static struct change_member	change_point_list[2*E820_MAX_ENTRIES];
+__initdata static struct change_member	*change_point[2*E820_MAX_ENTRIES];
+__initdata static struct e820_entry	*overlap_list[E820_MAX_ENTRIES];
+__initdata static struct e820_entry	new_entries[E820_MAX_ENTRIES];
 
-static int __init cpcompare(const void *a, const void *b)
+__init static int cpcompare(const void *a, const void *b)
 {
 	struct change_member * const *app = a, * const *bpp = b;
 	const struct change_member *ap = *app, *bp = *bpp;
@@ -305,28 +323,32 @@ static int __init cpcompare(const void *a, const void *b)
 	return (ap->addr != ap->entry->addr) - (bp->addr != bp->entry->addr);
 }
 
-static bool e820_nomerge(enum e820_type type)
+/*
+ * Can two consecutive E820 entries of this same E820 type be merged?
+ */
+static bool e820_type_mergeable(enum e820_type type)
 {
 	/*
 	 * These types may indicate distinct platform ranges aligned to
-	 * numa node, protection domain, performance domain, or other
+	 * NUMA node, protection domain, performance domain, or other
 	 * boundaries. Do not merge them.
 	 */
 	if (type == E820_TYPE_PRAM)
-		return true;
+		return false;
 	if (type == E820_TYPE_SOFT_RESERVED)
-		return true;
-	return false;
+		return false;
+
+	return true;
 }
 
-int __init e820__update_table(struct e820_table *table)
+__init int e820__update_table(struct e820_table *table)
 {
 	struct e820_entry *entries = table->entries;
 	u32 max_nr_entries = ARRAY_SIZE(table->entries);
 	enum e820_type current_type, last_type;
-	unsigned long long last_addr;
+	u64 last_addr;
 	u32 new_nr_entries, overlap_entries;
-	u32 i, chg_idx, chg_nr;
+	u32 idx, chg_idx, chg_nr;
 
 	/* If there's only one memory region, don't bother: */
 	if (table->nr_entries < 2)
@@ -335,26 +357,26 @@ int __init e820__update_table(struct e820_table *table)
 	BUG_ON(table->nr_entries > max_nr_entries);
 
 	/* Bail out if we find any unreasonable addresses in the map: */
-	for (i = 0; i < table->nr_entries; i++) {
-		if (entries[i].addr + entries[i].size < entries[i].addr)
+	for (idx = 0; idx < table->nr_entries; idx++) {
+		if (entries[idx].addr + entries[idx].size < entries[idx].addr)
 			return -1;
 	}
 
 	/* Create pointers for initial change-point information (for sorting): */
-	for (i = 0; i < 2 * table->nr_entries; i++)
-		change_point[i] = &change_point_list[i];
+	for (idx = 0; idx < 2 * table->nr_entries; idx++)
+		change_point[idx] = &change_point_list[idx];
 
 	/*
 	 * Record all known change-points (starting and ending addresses),
 	 * omitting empty memory regions:
 	 */
 	chg_idx = 0;
-	for (i = 0; i < table->nr_entries; i++)	{
-		if (entries[i].size != 0) {
-			change_point[chg_idx]->addr	= entries[i].addr;
-			change_point[chg_idx++]->entry	= &entries[i];
-			change_point[chg_idx]->addr	= entries[i].addr + entries[i].size;
-			change_point[chg_idx++]->entry	= &entries[i];
+	for (idx = 0; idx < table->nr_entries; idx++)	{
+		if (entries[idx].size != 0) {
+			change_point[chg_idx]->addr	= entries[idx].addr;
+			change_point[chg_idx++]->entry	= &entries[idx];
+			change_point[chg_idx]->addr	= entries[idx].addr + entries[idx].size;
+			change_point[chg_idx++]->entry	= &entries[idx];
 		}
 	}
 	chg_nr = chg_idx;
@@ -376,9 +398,9 @@ int __init e820__update_table(struct e820_table *table)
 			overlap_list[overlap_entries++] = change_point[chg_idx]->entry;
 		} else {
 			/* Remove entry from list (order independent, so swap with last): */
-			for (i = 0; i < overlap_entries; i++) {
-				if (overlap_list[i] == change_point[chg_idx]->entry)
-					overlap_list[i] = overlap_list[overlap_entries-1];
+			for (idx = 0; idx < overlap_entries; idx++) {
+				if (overlap_list[idx] == change_point[chg_idx]->entry)
+					overlap_list[idx] = overlap_list[overlap_entries-1];
 			}
 			overlap_entries--;
 		}
@@ -388,13 +410,13 @@ int __init e820__update_table(struct e820_table *table)
 		 * 1=usable, 2,3,4,4+=unusable)
 		 */
 		current_type = 0;
-		for (i = 0; i < overlap_entries; i++) {
-			if (overlap_list[i]->type > current_type)
-				current_type = overlap_list[i]->type;
+		for (idx = 0; idx < overlap_entries; idx++) {
+			if (overlap_list[idx]->type > current_type)
+				current_type = overlap_list[idx]->type;
 		}
 
 		/* Continue building up new map based on this information: */
-		if (current_type != last_type || e820_nomerge(current_type)) {
+		if (current_type != last_type || !e820_type_mergeable(current_type)) {
 			if (last_type) {
 				new_entries[new_nr_entries].size = change_point[chg_idx]->addr - last_addr;
 				/* Move forward only if the new size was non-zero: */
@@ -419,17 +441,22 @@ int __init e820__update_table(struct e820_table *table)
 	return 0;
 }
 
-static int __init __append_e820_table(struct boot_e820_entry *entries, u32 nr_entries)
+/*
+ * Copy the BIOS E820 map into the kernel's e820_table.
+ *
+ * Sanity-check it while we're at it..
+ */
+__init static int append_e820_table(struct boot_e820_entry *entries, u32 nr_entries)
 {
 	struct boot_e820_entry *entry = entries;
 
 	while (nr_entries) {
 		u64 start = entry->addr;
-		u64 size = entry->size;
-		u64 end = start + size - 1;
-		u32 type = entry->type;
+		u64 size  = entry->size;
+		u64 end   = start + size-1;
+		u32 type  = entry->type;
 
-		/* Ignore the entry on 64-bit overflow: */
+		/* Ignore the remaining entries on 64-bit overflow: */
 		if (start > end && likely(size))
 			return -1;
 
@@ -441,29 +468,11 @@ static int __init __append_e820_table(struct boot_e820_entry *entries, u32 nr_en
 	return 0;
 }
 
-/*
- * Copy the BIOS E820 map into a safe place.
- *
- * Sanity-check it while we're at it..
- *
- * If we're lucky and live on a modern system, the setup code
- * will have given us a memory map that we can use to properly
- * set up memory.  If we aren't, we'll fake a memory map.
- */
-static int __init append_e820_table(struct boot_e820_entry *entries, u32 nr_entries)
-{
-	/* Only one memory region (or negative)? Ignore it */
-	if (nr_entries < 2)
-		return -1;
-
-	return __append_e820_table(entries, nr_entries);
-}
-
-static u64 __init
+__init static u64
 __e820__range_update(struct e820_table *table, u64 start, u64 size, enum e820_type old_type, enum e820_type new_type)
 {
 	u64 end;
-	unsigned int i;
+	u32 idx;
 	u64 real_updated_size = 0;
 
 	BUG_ON(old_type == new_type);
@@ -472,14 +481,14 @@ __e820__range_update(struct e820_table *table, u64 start, u64 size, enum e820_ty
 		size = ULLONG_MAX - start;
 
 	end = start + size;
-	printk(KERN_DEBUG "e820: update [mem %#010Lx-%#010Lx] ", start, end - 1);
+	printk(KERN_DEBUG "e820: update [mem %#010Lx-%#010Lx]", start, end - 1);
 	e820_print_type(old_type);
-	pr_cont(" ==> ");
+	pr_cont(" ==>");
 	e820_print_type(new_type);
 	pr_cont("\n");
 
-	for (i = 0; i < table->nr_entries; i++) {
-		struct e820_entry *entry = &table->entries[i];
+	for (idx = 0; idx < table->nr_entries; idx++) {
+		struct e820_entry *entry = &table->entries[idx];
 		u64 final_start, final_end;
 		u64 entry_end;
 
@@ -527,46 +536,44 @@ __e820__range_update(struct e820_table *table, u64 start, u64 size, enum e820_ty
 	return real_updated_size;
 }
 
-u64 __init e820__range_update(u64 start, u64 size, enum e820_type old_type, enum e820_type new_type)
+__init u64 e820__range_update(u64 start, u64 size, enum e820_type old_type, enum e820_type new_type)
 {
 	return __e820__range_update(e820_table, start, size, old_type, new_type);
 }
 
-u64 __init e820__range_update_table(struct e820_table *t, u64 start, u64 size,
+__init u64 e820__range_update_table(struct e820_table *t, u64 start, u64 size,
 				    enum e820_type old_type, enum e820_type new_type)
 {
 	return __e820__range_update(t, start, size, old_type, new_type);
 }
 
 /* Remove a range of memory from the E820 table: */
-u64 __init e820__range_remove(u64 start, u64 size, enum e820_type old_type, bool check_type)
+__init void e820__range_remove(u64 start, u64 size, enum e820_type filter_type)
 {
-	int i;
+	u32 idx;
 	u64 end;
-	u64 real_removed_size = 0;
 
 	if (size > (ULLONG_MAX - start))
 		size = ULLONG_MAX - start;
 
 	end = start + size;
-	printk(KERN_DEBUG "e820: remove [mem %#010Lx-%#010Lx] ", start, end - 1);
-	if (check_type)
-		e820_print_type(old_type);
+	printk(KERN_DEBUG "e820: remove [mem %#010Lx-%#010Lx]", start, end - 1);
+	if (filter_type)
+		e820_print_type(filter_type);
 	pr_cont("\n");
 
-	for (i = 0; i < e820_table->nr_entries; i++) {
-		struct e820_entry *entry = &e820_table->entries[i];
+	for (idx = 0; idx < e820_table->nr_entries; idx++) {
+		struct e820_entry *entry = &e820_table->entries[idx];
 		u64 final_start, final_end;
 		u64 entry_end;
 
-		if (check_type && entry->type != old_type)
+		if (filter_type && entry->type != filter_type)
 			continue;
 
 		entry_end = entry->addr + entry->size;
 
 		/* Completely covered? */
 		if (entry->addr >= start && entry_end <= end) {
-			real_removed_size += entry->size;
 			memset(entry, 0, sizeof(*entry));
 			continue;
 		}
@@ -575,7 +582,6 @@ u64 __init e820__range_remove(u64 start, u64 size, enum e820_type old_type, bool
 		if (entry->addr < start && entry_end > end) {
 			e820__range_add(end, entry_end - end, entry->type);
 			entry->size = start - entry->addr;
-			real_removed_size += size;
 			continue;
 		}
 
@@ -585,8 +591,6 @@ u64 __init e820__range_remove(u64 start, u64 size, enum e820_type old_type, bool
 		if (final_start >= final_end)
 			continue;
 
-		real_removed_size += final_end - final_start;
-
 		/*
 		 * Left range could be head or tail, so need to update
 		 * the size first:
@@ -597,10 +601,9 @@ u64 __init e820__range_remove(u64 start, u64 size, enum e820_type old_type, bool
 
 		entry->addr = final_end;
 	}
-	return real_removed_size;
 }
 
-void __init e820__update_table_print(void)
+__init void e820__update_table_print(void)
 {
 	if (e820__update_table(e820_table))
 		return;
@@ -609,42 +612,64 @@ void __init e820__update_table_print(void)
 	e820__print_table("modified");
 }
 
-static void __init e820__update_table_kexec(void)
+__init static void e820__update_table_kexec(void)
 {
 	e820__update_table(e820_table_kexec);
 }
 
-#define MAX_GAP_END 0x100000000ull
+#define MAX_GAP_END SZ_4G
 
 /*
  * Search for a gap in the E820 memory space from 0 to MAX_GAP_END (4GB).
  */
-static int __init e820_search_gap(unsigned long *gapstart, unsigned long *gapsize)
+__init static int e820_search_gap(unsigned long *max_gap_start, unsigned long *max_gap_size)
 {
-	unsigned long long last = MAX_GAP_END;
-	int i = e820_table->nr_entries;
+	struct e820_entry *entry;
+	u64 range_end_prev = 0;
 	int found = 0;
+	u32 idx;
+
+	for (idx = 0; idx < e820_table->nr_entries; idx++) {
+		u64 range_start, range_end;
+
+		entry = e820_table->entries + idx;
+		range_start = entry->addr;
+		range_end   = entry->addr + entry->size;
+
+		/* Process any gap before this entry: */
+		if (range_start > range_end_prev) {
+			u64 gap_start = range_end_prev;
+			u64 gap_end = range_start;
+			u64 gap_size;
+
+			if (gap_start < MAX_GAP_END) {
+				/* Make sure the entirety of the gap is below MAX_GAP_END: */
+				gap_end = min(gap_end, MAX_GAP_END);
+				gap_size = gap_end-gap_start;
+
+				if (gap_size >= *max_gap_size) {
+					*max_gap_start = gap_start;
+					*max_gap_size = gap_size;
+					found = 1;
+				}
+			}
+		}
 
-	while (--i >= 0) {
-		unsigned long long start = e820_table->entries[i].addr;
-		unsigned long long end = start + e820_table->entries[i].size;
+		range_end_prev = range_end;
+	}
 
-		/*
-		 * Since "last" is at most 4GB, we know we'll
-		 * fit in 32 bits if this condition is true:
-		 */
-		if (last > end) {
-			unsigned long gap = last - end;
+	/* Is there a usable gap beyond the last entry: */
+	if (entry->addr + entry->size < MAX_GAP_END) {
+		u64 gap_start = entry->addr + entry->size;
+		u64 gap_size = MAX_GAP_END-gap_start;
 
-			if (gap >= *gapsize) {
-				*gapsize = gap;
-				*gapstart = end;
-				found = 1;
-			}
+		if (gap_size >= *max_gap_size) {
+			*max_gap_start = gap_start;
+			*max_gap_size = gap_size;
+			found = 1;
 		}
-		if (start < last)
-			last = start;
 	}
+
 	return found;
 }
 
@@ -658,29 +683,30 @@ static int __init e820_search_gap(unsigned long *gapstart, unsigned long *gapsiz
  */
 __init void e820__setup_pci_gap(void)
 {
-	unsigned long gapstart, gapsize;
+	unsigned long max_gap_start, max_gap_size;
 	int found;
 
-	gapsize = 0x400000;
-	found  = e820_search_gap(&gapstart, &gapsize);
+	/* The minimum eligible gap size is 4MB: */
+	max_gap_size = SZ_4M;
+	found  = e820_search_gap(&max_gap_start, &max_gap_size);
 
 	if (!found) {
 #ifdef CONFIG_X86_64
-		gapstart = (max_pfn << PAGE_SHIFT) + 1024*1024;
+		max_gap_start = (max_pfn << PAGE_SHIFT) + SZ_1M;
 		pr_err("Cannot find an available gap in the 32-bit address range\n");
 		pr_err("PCI devices with unassigned 32-bit BARs may not work!\n");
 #else
-		gapstart = 0x10000000;
+		max_gap_start = SZ_256M;
 #endif
 	}
 
 	/*
 	 * e820__reserve_resources_late() protects stolen RAM already:
 	 */
-	pci_mem_start = gapstart;
+	pci_mem_start = max_gap_start;
 
-	pr_info("[mem %#010lx-%#010lx] available for PCI devices\n",
-		gapstart, gapstart + gapsize - 1);
+	pr_info("[gap %#010lx-%#010lx] available for PCI devices\n",
+		max_gap_start, max_gap_start + max_gap_size-1);
 }
 
 /*
@@ -722,7 +748,7 @@ __init void e820__reallocate_tables(void)
  * the remaining (if any) entries are passed via the SETUP_E820_EXT node of
  * struct setup_data, which is parsed here.
  */
-void __init e820__memory_setup_extended(u64 phys_addr, u32 data_len)
+__init void e820__memory_setup_extended(u64 phys_addr, u32 data_len)
 {
 	int entries;
 	struct boot_e820_entry *extmap;
@@ -732,7 +758,7 @@ void __init e820__memory_setup_extended(u64 phys_addr, u32 data_len)
 	entries = sdata->len / sizeof(*extmap);
 	extmap = (struct boot_e820_entry *)(sdata->data);
 
-	__append_e820_table(extmap, entries);
+	append_e820_table(extmap, entries);
 	e820__update_table(e820_table);
 
 	memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec));
@@ -751,13 +777,13 @@ void __init e820__memory_setup_extended(u64 phys_addr, u32 data_len)
  * This function requires the E820 map to be sorted and without any
  * overlapping entries.
  */
-void __init e820__register_nosave_regions(unsigned long limit_pfn)
+__init void e820__register_nosave_regions(unsigned long limit_pfn)
 {
-	int i;
+	u32 idx;
 	u64 last_addr = 0;
 
-	for (i = 0; i < e820_table->nr_entries; i++) {
-		struct e820_entry *entry = &e820_table->entries[i];
+	for (idx = 0; idx < e820_table->nr_entries; idx++) {
+		struct e820_entry *entry = &e820_table->entries[idx];
 
 		if (entry->type != E820_TYPE_RAM)
 			continue;
@@ -776,12 +802,12 @@ void __init e820__register_nosave_regions(unsigned long limit_pfn)
  * Register ACPI NVS memory regions, so that we can save/restore them during
  * hibernation and the subsequent resume:
  */
-static int __init e820__register_nvs_regions(void)
+__init static int e820__register_nvs_regions(void)
 {
-	int i;
+	u32 idx;
 
-	for (i = 0; i < e820_table->nr_entries; i++) {
-		struct e820_entry *entry = &e820_table->entries[i];
+	for (idx = 0; idx < e820_table->nr_entries; idx++) {
+		struct e820_entry *entry = &e820_table->entries[idx];
 
 		if (entry->type == E820_TYPE_NVS)
 			acpi_nvs_register(entry->addr, entry->size);
@@ -800,7 +826,7 @@ core_initcall(e820__register_nvs_regions);
  * This allows kexec to fake a new mptable, as if it came from the real
  * system.
  */
-u64 __init e820__memblock_alloc_reserved(u64 size, u64 align)
+__init u64 e820__memblock_alloc_reserved(u64 size, u64 align)
 {
 	u64 addr;
 
@@ -827,14 +853,14 @@ u64 __init e820__memblock_alloc_reserved(u64 size, u64 align)
 /*
  * Find the highest page frame number we have available
  */
-static unsigned long __init e820__end_ram_pfn(unsigned long limit_pfn)
+__init static unsigned long e820__end_ram_pfn(unsigned long limit_pfn)
 {
-	int i;
+	u32 idx;
 	unsigned long last_pfn = 0;
 	unsigned long max_arch_pfn = MAX_ARCH_PFN;
 
-	for (i = 0; i < e820_table->nr_entries; i++) {
-		struct e820_entry *entry = &e820_table->entries[i];
+	for (idx = 0; idx < e820_table->nr_entries; idx++) {
+		struct e820_entry *entry = &e820_table->entries[idx];
 		unsigned long start_pfn;
 		unsigned long end_pfn;
 
@@ -863,26 +889,20 @@ static unsigned long __init e820__end_ram_pfn(unsigned long limit_pfn)
 	return last_pfn;
 }
 
-unsigned long __init e820__end_of_ram_pfn(void)
+__init unsigned long e820__end_of_ram_pfn(void)
 {
 	return e820__end_ram_pfn(MAX_ARCH_PFN);
 }
 
-unsigned long __init e820__end_of_low_ram_pfn(void)
+__init unsigned long e820__end_of_low_ram_pfn(void)
 {
 	return e820__end_ram_pfn(1UL << (32 - PAGE_SHIFT));
 }
 
-static void __init early_panic(char *msg)
-{
-	early_printk(msg);
-	panic(msg);
-}
-
-static int userdef __initdata;
+__initdata static int userdef;
 
 /* The "mem=nopentium" boot option disables 4MB page tables on 32-bit kernels: */
-static int __init parse_memopt(char *p)
+__init static int parse_memopt(char *p)
 {
 	u64 mem_size;
 
@@ -906,7 +926,7 @@ static int __init parse_memopt(char *p)
 	if (mem_size == 0)
 		return -EINVAL;
 
-	e820__range_remove(mem_size, ULLONG_MAX - mem_size, E820_TYPE_RAM, 1);
+	e820__range_remove(mem_size, ULLONG_MAX - mem_size, E820_TYPE_RAM);
 
 #ifdef CONFIG_MEMORY_HOTPLUG
 	max_mem_size = mem_size;
@@ -916,7 +936,7 @@ static int __init parse_memopt(char *p)
 }
 early_param("mem", parse_memopt);
 
-static int __init parse_memmap_one(char *p)
+__init static int parse_memmap_one(char *p)
 {
 	char *oldp;
 	u64 start_at, mem_size;
@@ -962,18 +982,16 @@ static int __init parse_memmap_one(char *p)
 			e820__range_update(start_at, mem_size, from, to);
 		else if (to)
 			e820__range_add(start_at, mem_size, to);
-		else if (from)
-			e820__range_remove(start_at, mem_size, from, 1);
 		else
-			e820__range_remove(start_at, mem_size, 0, 0);
+			e820__range_remove(start_at, mem_size, from);
 	} else {
-		e820__range_remove(mem_size, ULLONG_MAX - mem_size, E820_TYPE_RAM, 1);
+		e820__range_remove(mem_size, ULLONG_MAX - mem_size, E820_TYPE_RAM);
 	}
 
 	return *p == '\0' ? 0 : -EINVAL;
 }
 
-static int __init parse_memmap_opt(char *str)
+__init static int parse_memmap_opt(char *str)
 {
 	while (str) {
 		char *k = strchr(str, ',');
@@ -994,18 +1012,18 @@ early_param("memmap", parse_memmap_opt);
  * have been processed, in which case we already have an E820 table filled in
  * via the parameter callback function(s), but it's not sorted and printed yet:
  */
-void __init e820__finish_early_params(void)
+__init void e820__finish_early_params(void)
 {
 	if (userdef) {
 		if (e820__update_table(e820_table) < 0)
-			early_panic("Invalid user supplied memory map");
+			panic("Invalid user supplied memory map");
 
 		pr_info("user-defined physical RAM map:\n");
 		e820__print_table("user");
 	}
 }
 
-static const char *__init e820_type_to_string(struct e820_entry *entry)
+__init static const char * e820_type_to_string(struct e820_entry *entry)
 {
 	switch (entry->type) {
 	case E820_TYPE_RAM:		return "System RAM";
@@ -1020,7 +1038,7 @@ static const char *__init e820_type_to_string(struct e820_entry *entry)
 	}
 }
 
-static unsigned long __init e820_type_to_iomem_type(struct e820_entry *entry)
+__init static unsigned long e820_type_to_iomem_type(struct e820_entry *entry)
 {
 	switch (entry->type) {
 	case E820_TYPE_RAM:		return IORESOURCE_SYSTEM_RAM;
@@ -1035,7 +1053,7 @@ static unsigned long __init e820_type_to_iomem_type(struct e820_entry *entry)
 	}
 }
 
-static unsigned long __init e820_type_to_iores_desc(struct e820_entry *entry)
+__init static unsigned long e820_type_to_iores_desc(struct e820_entry *entry)
 {
 	switch (entry->type) {
 	case E820_TYPE_ACPI:		return IORES_DESC_ACPI_TABLES;
@@ -1050,40 +1068,47 @@ static unsigned long __init e820_type_to_iores_desc(struct e820_entry *entry)
 	}
 }
 
-static bool __init do_mark_busy(enum e820_type type, struct resource *res)
+/*
+ * We assign one resource entry for each E820 map entry:
+ */
+__initdata static struct resource *e820_res;
+
+/*
+ * Is this a device address region that should not be marked busy?
+ * (Versus system address regions that we register & lock early.)
+ */
+__init static bool e820_device_region(enum e820_type type, struct resource *res)
 {
-	/* this is the legacy bios/dos rom-shadow + mmio region */
-	if (res->start < (1ULL<<20))
-		return true;
+	/* This is the legacy BIOS/DOS ROM-shadow + MMIO region: */
+	if (res->start < SZ_1M)
+		return false;
 
 	/*
 	 * Treat persistent memory and other special memory ranges like
-	 * device memory, i.e. reserve it for exclusive use of a driver
+	 * device memory, i.e. keep it available for exclusive use of a
+	 * driver:
 	 */
 	switch (type) {
 	case E820_TYPE_RESERVED:
 	case E820_TYPE_SOFT_RESERVED:
 	case E820_TYPE_PRAM:
 	case E820_TYPE_PMEM:
-		return false;
+		return true;
 	case E820_TYPE_RAM:
 	case E820_TYPE_ACPI:
 	case E820_TYPE_NVS:
 	case E820_TYPE_UNUSABLE:
 	default:
-		return true;
+		return false;
 	}
 }
 
 /*
- * Mark E820 reserved areas as busy for the resource manager:
+ * Mark E820 system regions as busy for the resource manager:
  */
-
-static struct resource __initdata *e820_res;
-
-void __init e820__reserve_resources(void)
+__init void e820__reserve_resources(void)
 {
-	int i;
+	u32 idx;
 	struct resource *res;
 	u64 end;
 
@@ -1091,8 +1116,8 @@ void __init e820__reserve_resources(void)
 			     SMP_CACHE_BYTES);
 	e820_res = res;
 
-	for (i = 0; i < e820_table->nr_entries; i++) {
-		struct e820_entry *entry = e820_table->entries + i;
+	for (idx = 0; idx < e820_table->nr_entries; idx++) {
+		struct e820_entry *entry = e820_table->entries + idx;
 
 		end = entry->addr + entry->size - 1;
 		if (end != (resource_size_t)end) {
@@ -1106,20 +1131,20 @@ void __init e820__reserve_resources(void)
 		res->desc  = e820_type_to_iores_desc(entry);
 
 		/*
-		 * Don't register the region that could be conflicted with
-		 * PCI device BAR resources and insert them later in
-		 * pcibios_resource_survey():
+		 * Skip and don't register device regions that could be conflicted
+		 * with PCI device BAR resources. They get inserted later in
+		 * pcibios_resource_survey() -> e820__reserve_resources_late():
 		 */
-		if (do_mark_busy(entry->type, res)) {
+		if (!e820_device_region(entry->type, res)) {
 			res->flags |= IORESOURCE_BUSY;
 			insert_resource(&iomem_resource, res);
 		}
 		res++;
 	}
 
-	/* Expose the kexec e820 table to the sysfs. */
-	for (i = 0; i < e820_table_kexec->nr_entries; i++) {
-		struct e820_entry *entry = e820_table_kexec->entries + i;
+	/* Expose the kexec e820 table to sysfs: */
+	for (idx = 0; idx < e820_table_kexec->nr_entries; idx++) {
+		struct e820_entry *entry = e820_table_kexec->entries + idx;
 
 		firmware_map_add_early(entry->addr, entry->addr + entry->size, e820_type_to_string(entry));
 	}
@@ -1128,7 +1153,7 @@ void __init e820__reserve_resources(void)
 /*
  * How much should we pad the end of RAM, depending on where it is?
  */
-static unsigned long __init ram_alignment(resource_size_t pos)
+__init static unsigned long ram_alignment(resource_size_t pos)
 {
 	unsigned long mb = pos >> 20;
 
@@ -1146,24 +1171,36 @@ static unsigned long __init ram_alignment(resource_size_t pos)
 
 #define MAX_RESOURCE_SIZE ((resource_size_t)-1)
 
-void __init e820__reserve_resources_late(void)
+__init void e820__reserve_resources_late(void)
 {
-	int i;
+	u32 idx;
 	struct resource *res;
 
+	/*
+	 * Register device address regions listed in the E820 map,
+	 * these can be claimed by device drivers later on:
+	 */
 	res = e820_res;
-	for (i = 0; i < e820_table->nr_entries; i++) {
+	for (idx = 0; idx < e820_table->nr_entries; idx++) {
 		if (!res->parent && res->end)
 			insert_resource_expand_to_fit(&iomem_resource, res);
 		res++;
 	}
 
 	/*
-	 * Try to bump up RAM regions to reasonable boundaries, to
-	 * avoid stolen RAM:
+	 * Create additional 'gaps' at the end of RAM regions,
+	 * rounding them up to 64k/1MB/64MB boundaries, should
+	 * they be weirdly sized, and register extra, locked
+	 * resource regions for them, to make sure drivers
+	 * won't claim those addresses.
+	 *
+	 * These are basically blind guesses and heuristics to
+	 * avoid resource conflicts with broken firmware that
+	 * doesn't properly list 'stolen RAM' as a system region
+	 * in the E820 map.
 	 */
-	for (i = 0; i < e820_table->nr_entries; i++) {
-		struct e820_entry *entry = &e820_table->entries[i];
+	for (idx = 0; idx < e820_table->nr_entries; idx++) {
+		struct e820_entry *entry = &e820_table->entries[idx];
 		u64 start, end;
 
 		if (entry->type != E820_TYPE_RAM)
@@ -1176,7 +1213,7 @@ void __init e820__reserve_resources_late(void)
 		if (start >= end)
 			continue;
 
-		printk(KERN_DEBUG "e820: reserve RAM buffer [mem %#010llx-%#010llx]\n", start, end);
+		pr_info("e820: register RAM buffer resource [mem %#010llx-%#010llx]\n", start, end);
 		reserve_region_with_split(&iomem_resource, start, end, "RAM buffer");
 	}
 }
@@ -1184,7 +1221,7 @@ void __init e820__reserve_resources_late(void)
 /*
  * Pass the firmware (bootloader) E820 map to the kernel and process it:
  */
-char *__init e820__memory_setup_default(void)
+__init char * e820__memory_setup_default(void)
 {
 	char *who = "BIOS-e820";
 
@@ -1222,7 +1259,7 @@ char *__init e820__memory_setup_default(void)
  * E820 map - with an optional platform quirk available for virtual platforms
  * to override this method of boot environment processing:
  */
-void __init e820__memory_setup(void)
+__init void e820__memory_setup(void)
 {
 	char *who;
 
@@ -1238,9 +1275,9 @@ void __init e820__memory_setup(void)
 	e820__print_table(who);
 }
 
-void __init e820__memblock_setup(void)
+__init void e820__memblock_setup(void)
 {
-	int i;
+	u32 idx;
 	u64 end;
 
 #ifdef CONFIG_MEMORY_HOTPLUG
@@ -1284,8 +1321,8 @@ void __init e820__memblock_setup(void)
 	 */
 	memblock_allow_resize();
 
-	for (i = 0; i < e820_table->nr_entries; i++) {
-		struct e820_entry *entry = &e820_table->entries[i];
+	for (idx = 0; idx < e820_table->nr_entries; idx++) {
+		struct e820_entry *entry = &e820_table->entries[idx];
 
 		end = entry->addr + entry->size;
 		if (end != (resource_size_t)end)
diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c
index 48113c5193aa..76153dfb58c9 100644
--- a/arch/x86/kernel/fpu/xstate.c
+++ b/arch/x86/kernel/fpu/xstate.c
@@ -1946,7 +1946,7 @@ static int dump_xsave_layout_desc(struct coredump_params *cprm)
 		};
 
 		if (!dump_emit(cprm, &xc, sizeof(xc)))
-			return 0;
+			return -1;
 
 		num_records++;
 	}
@@ -1984,7 +1984,7 @@ int elf_coredump_extra_notes_write(struct coredump_params *cprm)
 		return 1;
 
 	num_records = dump_xsave_layout_desc(cprm);
-	if (!num_records)
+	if (num_records < 0)
 		return 1;
 
 	/* Total size should be equal to the number of records */
diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c
index c3244ac680d1..7508d0ccc740 100644
--- a/arch/x86/kernel/kexec-bzimage64.c
+++ b/arch/x86/kernel/kexec-bzimage64.c
@@ -20,6 +20,7 @@
 #include <linux/of_fdt.h>
 #include <linux/efi.h>
 #include <linux/random.h>
+#include <linux/sysfb.h>
 
 #include <asm/bootparam.h>
 #include <asm/setup.h>
@@ -303,7 +304,8 @@ setup_boot_parameters(struct kimage *image, struct boot_params *params,
 	params->hdr.hardware_subarch = boot_params.hdr.hardware_subarch;
 
 	/* Copying screen_info will do? */
-	memcpy(&params->screen_info, &screen_info, sizeof(struct screen_info));
+	memcpy(&params->screen_info, &sysfb_primary_display.screen,
+	       sizeof(sysfb_primary_display.screen));
 
 	/* Fill in memsize later */
 	params->screen_info.ext_mem_k = 0;
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 1b2edd07a3e1..0780691ba7dd 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -22,6 +22,7 @@
 #include <linux/random.h>
 #include <linux/root_dev.h>
 #include <linux/static_call.h>
+#include <linux/sysfb.h>
 #include <linux/swiotlb.h>
 #include <linux/tboot.h>
 #include <linux/usb/xhci-dbgp.h>
@@ -211,12 +212,9 @@ arch_initcall(init_x86_sysctl);
 /*
  * Setup options
  */
-struct screen_info screen_info;
-EXPORT_SYMBOL(screen_info);
-#if defined(CONFIG_FIRMWARE_EDID)
-struct edid_info edid_info;
-EXPORT_SYMBOL_GPL(edid_info);
-#endif
+
+struct sysfb_display_info sysfb_primary_display;
+EXPORT_SYMBOL(sysfb_primary_display);
 
 extern int root_mountflags;
 
@@ -439,9 +437,23 @@ int __init ima_free_kexec_buffer(void)
 
 int __init ima_get_kexec_buffer(void **addr, size_t *size)
 {
+	unsigned long start_pfn, end_pfn;
+
 	if (!ima_kexec_buffer_size)
 		return -ENOENT;
 
+	/*
+	 * Calculate the PFNs for the buffer and ensure
+	 * they are within addressable memory.
+	 */
+	start_pfn = PFN_DOWN(ima_kexec_buffer_phys);
+	end_pfn = PFN_DOWN(ima_kexec_buffer_phys + ima_kexec_buffer_size - 1);
+	if (!pfn_range_is_mapped(start_pfn, end_pfn)) {
+		pr_warn("IMA buffer at 0x%llx, size = 0x%zx beyond memory\n",
+			ima_kexec_buffer_phys, ima_kexec_buffer_size);
+		return -EINVAL;
+	}
+
 	*addr = __va(ima_kexec_buffer_phys);
 	*size = ima_kexec_buffer_size;
 
@@ -526,9 +538,9 @@ static void __init parse_setup_data(void)
 static void __init parse_boot_params(void)
 {
 	ROOT_DEV = old_decode_dev(boot_params.hdr.root_dev);
-	screen_info = boot_params.screen_info;
+	sysfb_primary_display.screen = boot_params.screen_info;
 #if defined(CONFIG_FIRMWARE_EDID)
-	edid_info = boot_params.edid_info;
+	sysfb_primary_display.edid = boot_params.edid_info;
 #endif
 #ifdef CONFIG_X86_32
 	apm_info.bios = boot_params.apm_bios_info;
@@ -763,7 +775,7 @@ static void __init trim_bios_range(void)
 	 * area (640Kb -> 1Mb) as RAM even though it is not.
 	 * take them out.
 	 */
-	e820__range_remove(BIOS_BEGIN, BIOS_END - BIOS_BEGIN, E820_TYPE_RAM, 1);
+	e820__range_remove(BIOS_BEGIN, BIOS_END - BIOS_BEGIN, E820_TYPE_RAM);
 
 	e820__update_table(e820_table);
 }
@@ -785,7 +797,7 @@ static void __init e820_add_kernel_range(void)
 		return;
 
 	pr_warn(".text .data .bss are not marked as E820_TYPE_RAM!\n");
-	e820__range_remove(start, size, E820_TYPE_RAM, 0);
+	e820__range_remove(start, size, 0);
 	e820__range_add(start, size, E820_TYPE_RAM);
 }
 
@@ -1015,11 +1027,9 @@ void __init setup_arch(char **cmdline_p)
 	trim_bios_range();
 #ifdef CONFIG_X86_32
 	if (ppro_with_ram_bug()) {
-		e820__range_update(0x70000000ULL, 0x40000ULL, E820_TYPE_RAM,
-				  E820_TYPE_RESERVED);
+		pr_info("Applying PPro RAM bug workaround: punching 256 kB hole at 1.75 GB physical.\n");
+		e820__range_update(0x70000000ULL, SZ_256K, E820_TYPE_RAM, E820_TYPE_RESERVED);
 		e820__update_table(e820_table);
-		printk(KERN_INFO "fixed physical RAM map:\n");
-		e820__print_table("bad_ppro");
 	}
 #else
 	early_gart_iommu_check();
@@ -1254,7 +1264,7 @@ void __init setup_arch(char **cmdline_p)
 #ifdef CONFIG_VT
 #if defined(CONFIG_VGA_CONSOLE)
 	if (!efi_enabled(EFI_BOOT) || (efi_mem_type(0xa0000) != EFI_CONVENTIONAL_MEMORY))
-		vgacon_register_screen(&screen_info);
+		vgacon_register_screen(&sysfb_primary_display.screen);
 #endif
 #endif
 	x86_init.oem.banner();
diff --git a/arch/x86/kernel/unwind_orc.c b/arch/x86/kernel/unwind_orc.c
index 977ee75e047c..f610fde2d5c4 100644
--- a/arch/x86/kernel/unwind_orc.c
+++ b/arch/x86/kernel/unwind_orc.c
@@ -2,6 +2,7 @@
 #include <linux/objtool.h>
 #include <linux/module.h>
 #include <linux/sort.h>
+#include <linux/bpf.h>
 #include <asm/ptrace.h>
 #include <asm/stacktrace.h>
 #include <asm/unwind.h>
@@ -172,6 +173,25 @@ static struct orc_entry *orc_ftrace_find(unsigned long ip)
 }
 #endif
 
+/* Fake frame pointer entry -- used as a fallback for generated code */
+static struct orc_entry orc_fp_entry = {
+	.type		= ORC_TYPE_CALL,
+	.sp_reg		= ORC_REG_BP,
+	.sp_offset	= 16,
+	.bp_reg		= ORC_REG_PREV_SP,
+	.bp_offset	= -16,
+};
+
+static struct orc_entry *orc_bpf_find(unsigned long ip)
+{
+#ifdef CONFIG_BPF_JIT
+	if (bpf_has_frame_pointer(ip))
+		return &orc_fp_entry;
+#endif
+
+	return NULL;
+}
+
 /*
  * If we crash with IP==0, the last successfully executed instruction
  * was probably an indirect function call with a NULL function pointer,
@@ -186,15 +206,6 @@ static struct orc_entry null_orc_entry = {
 	.type = ORC_TYPE_CALL
 };
 
-/* Fake frame pointer entry -- used as a fallback for generated code */
-static struct orc_entry orc_fp_entry = {
-	.type		= ORC_TYPE_CALL,
-	.sp_reg		= ORC_REG_BP,
-	.sp_offset	= 16,
-	.bp_reg		= ORC_REG_PREV_SP,
-	.bp_offset	= -16,
-};
-
 static struct orc_entry *orc_find(unsigned long ip)
 {
 	static struct orc_entry *orc;
@@ -238,6 +249,11 @@ static struct orc_entry *orc_find(unsigned long ip)
 	if (orc)
 		return orc;
 
+	/* BPF lookup: */
+	orc = orc_bpf_find(ip);
+	if (orc)
+		return orc;
+
 	return orc_ftrace_find(ip);
 }
 
@@ -495,9 +511,8 @@ bool unwind_next_frame(struct unwind_state *state)
 	if (!orc) {
 		/*
 		 * As a fallback, try to assume this code uses a frame pointer.
-		 * This is useful for generated code, like BPF, which ORC
-		 * doesn't know about.  This is just a guess, so the rest of
-		 * the unwind is no longer considered reliable.
+		 * This is just a guess, so the rest of the unwind is no longer
+		 * considered reliable.
 		 */
 		orc = &orc_fp_entry;
 		state->error = true;
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index d563a948318b..88a5426674a1 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -509,11 +509,18 @@ static int kvm_set_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 *e2,
 	u32 vcpu_caps[NR_KVM_CPU_CAPS];
 	int r;
 
+	/*
+	 * Apply pending runtime CPUID updates to the current CPUID entries to
+	 * avoid false positives due to mismatches on KVM-owned feature flags.
+	 */
+	if (vcpu->arch.cpuid_dynamic_bits_dirty)
+		kvm_update_cpuid_runtime(vcpu);
+
 	/*
 	 * Swap the existing (old) entries with the incoming (new) entries in
 	 * order to massage the new entries, e.g. to account for dynamic bits
-	 * that KVM controls, without clobbering the current guest CPUID, which
-	 * KVM needs to preserve in order to unwind on failure.
+	 * that KVM controls, without losing the current guest CPUID, which KVM
+	 * needs to preserve in order to unwind on failure.
 	 *
 	 * Similarly, save the vCPU's current cpu_caps so that the capabilities
 	 * can be updated alongside the CPUID entries when performing runtime
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index c81005b24522..ba0f11c68372 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -985,7 +985,7 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu)
 	if (!nested_vmcb_check_save(vcpu) ||
 	    !nested_vmcb_check_controls(vcpu)) {
 		vmcb12->control.exit_code    = SVM_EXIT_ERR;
-		vmcb12->control.exit_code_hi = 0;
+		vmcb12->control.exit_code_hi = -1u;
 		vmcb12->control.exit_info_1  = 0;
 		vmcb12->control.exit_info_2  = 0;
 		goto out;
@@ -1018,7 +1018,7 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu)
 	svm->soft_int_injected = false;
 
 	svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
-	svm->vmcb->control.exit_code_hi = 0;
+	svm->vmcb->control.exit_code_hi = -1u;
 	svm->vmcb->control.exit_info_1  = 0;
 	svm->vmcb->control.exit_info_2  = 0;
 
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index f56c2d895011..24d59ccfa40d 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -2443,6 +2443,7 @@ static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu,
 
 	if (cr0 ^ val) {
 		svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
+		svm->vmcb->control.exit_code_hi = 0;
 		ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
 	}
 
@@ -4617,6 +4618,7 @@ static int svm_check_intercept(struct kvm_vcpu *vcpu,
 	if (static_cpu_has(X86_FEATURE_NRIPS))
 		vmcb->control.next_rip  = info->next_rip;
 	vmcb->control.exit_code = icpt_info.exit_code;
+	vmcb->control.exit_code_hi = 0;
 	vmexit = nested_svm_exit_handled(svm);
 
 	ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
index 9e151dbdef25..01be93a53d07 100644
--- a/arch/x86/kvm/svm/svm.h
+++ b/arch/x86/kvm/svm/svm.h
@@ -761,9 +761,10 @@ int nested_svm_vmexit(struct vcpu_svm *svm);
 
 static inline int nested_svm_simple_vmexit(struct vcpu_svm *svm, u32 exit_code)
 {
-	svm->vmcb->control.exit_code   = exit_code;
-	svm->vmcb->control.exit_info_1 = 0;
-	svm->vmcb->control.exit_info_2 = 0;
+	svm->vmcb->control.exit_code	= exit_code;
+	svm->vmcb->control.exit_code_hi	= 0;
+	svm->vmcb->control.exit_info_1	= 0;
+	svm->vmcb->control.exit_info_2	= 0;
 	return nested_svm_vmexit(svm);
 }
 
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 40777278eabb..6137e5307d0f 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -19,6 +19,7 @@
 #include "trace.h"
 #include "vmx.h"
 #include "smm.h"
+#include "x86_ops.h"
 
 static bool __read_mostly enable_shadow_vmcs = 1;
 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
@@ -5165,7 +5166,7 @@ void __nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason,
 
 	if (vmx->nested.update_vmcs01_apicv_status) {
 		vmx->nested.update_vmcs01_apicv_status = false;
-		kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
+		vmx_refresh_apicv_exec_ctrl(vcpu);
 	}
 
 	if (vmx->nested.update_vmcs01_hwapic_isr) {
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 4cbe8c84b636..6b96f7aea20b 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -6937,15 +6937,6 @@ void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
 	 * VM-Exit, otherwise L1 with run with a stale SVI.
 	 */
 	if (is_guest_mode(vcpu)) {
-		/*
-		 * KVM is supposed to forward intercepted L2 EOIs to L1 if VID
-		 * is enabled in vmcs12; as above, the EOIs affect L2's vAPIC.
-		 * Note, userspace can stuff state while L2 is active; assert
-		 * that VID is disabled if and only if the vCPU is in KVM_RUN
-		 * to avoid false positives if userspace is setting APIC state.
-		 */
-		WARN_ON_ONCE(vcpu->wants_to_run &&
-			     nested_cpu_has_vid(get_vmcs12(vcpu)));
 		to_vmx(vcpu)->nested.update_vmcs01_hwapic_isr = true;
 		return;
 	}
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 0c6d899d53dd..ff8812f3a129 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -10886,9 +10886,16 @@ void __kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)
 	 * pending. At the same time, KVM_REQ_EVENT may not be set as APICv was
 	 * still active when the interrupt got accepted. Make sure
 	 * kvm_check_and_inject_events() is called to check for that.
+	 *
+	 * Update SVI when APICv gets enabled, otherwise SVI won't reflect the
+	 * highest bit in vISR and the next accelerated EOI in the guest won't
+	 * be virtualized correctly (the CPU uses SVI to determine which vISR
+	 * vector to clear).
 	 */
 	if (!apic->apicv_active)
 		kvm_make_request(KVM_REQ_EVENT, vcpu);
+	else
+		kvm_apic_update_hwapic_isr(vcpu);
 
 out:
 	preempt_enable();
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index b69dc7194e2c..b0bac2a66eff 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1678,6 +1678,9 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image
 	emit_prologue(&prog, image, stack_depth,
 		      bpf_prog_was_classic(bpf_prog), tail_call_reachable,
 		      bpf_is_subprog(bpf_prog), bpf_prog->aux->exception_cb);
+
+	bpf_prog->aux->ksym.fp_start = prog - temp;
+
 	/* Exception callback will clobber callee regs for its own use, and
 	 * restore the original callee regs from main prog's stack frame.
 	 */
@@ -2736,6 +2739,8 @@ st:			if (is_imm8(insn->off))
 					pop_r12(&prog);
 			}
 			EMIT1(0xC9);         /* leave */
+			bpf_prog->aux->ksym.fp_end = prog - temp;
+
 			emit_return(&prog, image + addrs[i - 1] + (prog - temp));
 			break;
 
@@ -3325,6 +3330,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 	}
 	EMIT1(0x55);		 /* push rbp */
 	EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */
+	if (im)
+		im->ksym.fp_start = prog - (u8 *)rw_image;
+
 	if (!is_imm8(stack_size)) {
 		/* sub rsp, stack_size */
 		EMIT3_off32(0x48, 0x81, 0xEC, stack_size);
@@ -3462,7 +3470,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 		emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8);
 
 	emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, -rbx_off);
+
 	EMIT1(0xC9); /* leave */
+	if (im)
+		im->ksym.fp_end = prog - (u8 *)rw_image;
+
 	if (flags & BPF_TRAMP_F_SKIP_FRAME) {
 		/* skip our return address and return to parent */
 		EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */
diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c
index 463b784499a8..d00c6de7f3b7 100644
--- a/arch/x86/platform/efi/efi.c
+++ b/arch/x86/platform/efi/efi.c
@@ -333,8 +333,7 @@ static void __init efi_remove_e820_mmio(void)
 			if (size >= 256*1024) {
 				pr_info("Remove mem%02u: MMIO range=[0x%08llx-0x%08llx] (%lluMB) from e820 map\n",
 					i, start, end, size >> 20);
-				e820__range_remove(start, size,
-						   E820_TYPE_RESERVED, 1);
+				e820__range_remove(start, size, E820_TYPE_RESERVED);
 			} else {
 				pr_info("Not removing mem%02u: MMIO range=[0x%08llx-0x%08llx] (%lluKB) from e820 map\n",
 					i, start, end, size >> 10);
diff --git a/arch/x86/video/video-common.c b/arch/x86/video/video-common.c
index e0aeee99bc99..152789f00fcd 100644
--- a/arch/x86/video/video-common.c
+++ b/arch/x86/video/video-common.c
@@ -9,7 +9,7 @@
 
 #include <linux/module.h>
 #include <linux/pci.h>
-#include <linux/screen_info.h>
+#include <linux/sysfb.h>
 #include <linux/vgaarb.h>
 
 #include <asm/video.h>
@@ -29,7 +29,7 @@ EXPORT_SYMBOL(pgprot_framebuffer);
 bool video_is_primary_device(struct device *dev)
 {
 #ifdef CONFIG_SCREEN_INFO
-	struct screen_info *si = &screen_info;
+	struct screen_info *si = &sysfb_primary_display.screen;
 	struct resource res[SCREEN_INFO_MAX_RESOURCES];
 	ssize_t i, numres;
 #endif
diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
index 4806cc28d7ca..ae52436a741f 100644
--- a/arch/x86/xen/enlighten_pv.c
+++ b/arch/x86/xen/enlighten_pv.c
@@ -108,7 +108,7 @@ static int xen_cpu_dead_pv(unsigned int cpu);
  * calls.
  */
 DEFINE_PER_CPU(bool, xen_in_preemptible_hcall);
-EXPORT_SYMBOL_GPL(xen_in_preemptible_hcall);
+EXPORT_PER_CPU_SYMBOL_GPL(xen_in_preemptible_hcall);
 
 /*
  * In case of scheduling the flag must be cleared and restored after
@@ -426,7 +426,6 @@ static void xen_start_context_switch(struct task_struct *prev)
 
 	if (this_cpu_read(xen_lazy_mode) == XEN_LAZY_MMU) {
 		arch_leave_lazy_mmu_mode();
-		set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
 	}
 	enter_lazy(XEN_LAZY_CPU);
 }
@@ -437,7 +436,7 @@ static void xen_end_context_switch(struct task_struct *next)
 
 	xen_mc_flush();
 	leave_lazy(XEN_LAZY_CPU);
-	if (test_and_clear_ti_thread_flag(task_thread_info(next), TIF_LAZY_MMU_UPDATES))
+	if (__task_lazy_mmu_mode_active(next))
 		arch_enter_lazy_mmu_mode();
 }
 
diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c
index 2a4a8deaf612..7a35c3393df4 100644
--- a/arch/x86/xen/mmu_pv.c
+++ b/arch/x86/xen/mmu_pv.c
@@ -2139,10 +2139,8 @@ static void xen_flush_lazy_mmu(void)
 {
 	preempt_disable();
 
-	if (xen_get_lazy_mode() == XEN_LAZY_MMU) {
-		arch_leave_lazy_mmu_mode();
-		arch_enter_lazy_mmu_mode();
-	}
+	if (xen_get_lazy_mode() == XEN_LAZY_MMU)
+		xen_mc_flush();
 
 	preempt_enable();
 }
diff --git a/arch/xtensa/include/asm/syscall.h b/arch/xtensa/include/asm/syscall.h
index 7db3b489c8ad..bab7cdd96cbe 100644
--- a/arch/xtensa/include/asm/syscall.h
+++ b/arch/xtensa/include/asm/syscall.h
@@ -61,7 +61,6 @@ static inline void syscall_set_return_value(struct task_struct *task,
 	regs->areg[2] = (long) error ? error : val;
 }
 
-#define SYSCALL_MAX_ARGS 6
 #define XTENSA_SYSCALL_ARGUMENT_REGS {6, 3, 4, 5, 8, 9}
 
 static inline void syscall_get_arguments(struct task_struct *task,
diff --git a/arch/xtensa/kernel/align.S b/arch/xtensa/kernel/align.S
index ee97edce2300..f2d0d831ce43 100644
--- a/arch/xtensa/kernel/align.S
+++ b/arch/xtensa/kernel/align.S
@@ -21,6 +21,9 @@
 #include <asm/asm-offsets.h>
 #include <asm/asmmacro.h>
 #include <asm/processor.h>
+#ifdef CONFIG_MMU
+#include <asm/asm-uaccess.h>
+#endif
 
 #if XCHAL_UNALIGNED_LOAD_EXCEPTION || defined CONFIG_XTENSA_LOAD_STORE
 #define LOAD_EXCEPTION_HANDLER
@@ -185,8 +188,11 @@ ENTRY(fast_load_store)
 
 #ifdef CONFIG_MMU
 	/* l32e can't be used here even when it's available. */
-	/* TODO access_ok(a3) could be used here */
-	j	.Linvalid_instruction
+	rsr	a6, ps
+	bbci.l	a6, PS_UM_BIT, 1f # kernel mode
+	movi	a5, 8
+	access_ok a3, a5, a6, a2, .Linvalid_instruction
+1:
 #endif
 	l32i	a5, a3, 0
 	l32i	a6, a3, 4
diff --git a/arch/xtensa/platforms/iss/network.c b/arch/xtensa/platforms/iss/network.c
index f0a63b2f85cc..832579143891 100644
--- a/arch/xtensa/platforms/iss/network.c
+++ b/arch/xtensa/platforms/iss/network.c
@@ -13,6 +13,7 @@
 
 #define pr_fmt(fmt) "%s: " fmt, __func__
 
+#include <linux/hex.h>
 #include <linux/list.h>
 #include <linux/irq.h>
 #include <linux/spinlock.h>
diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index 4a8d3d96bfe4..6e54b1d3d8bc 100644
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -7181,7 +7181,7 @@ static void bfq_exit_queue(struct elevator_queue *e)
 
 	blk_stat_disable_accounting(bfqd->queue);
 	blk_queue_flag_clear(QUEUE_FLAG_DISABLE_WBT_DEF, bfqd->queue);
-	set_bit(ELEVATOR_FLAG_ENABLE_WBT_ON_EXIT, &e->flags);
+	wbt_enable_default(bfqd->queue->disk);
 
 	kfree(bfqd);
 }
diff --git a/block/blk-settings.c b/block/blk-settings.c
index 51401f08ce05..a9e65dc090da 100644
--- a/block/blk-settings.c
+++ b/block/blk-settings.c
@@ -161,10 +161,9 @@ static int blk_validate_integrity_limits(struct queue_limits *lim)
 		return -EINVAL;
 	}
 
-	if (bi->pi_tuple_size > bi->metadata_size) {
-		pr_warn("pi_tuple_size (%u) exceeds metadata_size (%u)\n",
-			 bi->pi_tuple_size,
-			 bi->metadata_size);
+	if (bi->pi_offset + bi->pi_tuple_size > bi->metadata_size) {
+		pr_warn("pi_offset (%u) + pi_tuple_size (%u) exceeds metadata_size (%u)\n",
+			bi->pi_offset, bi->pi_tuple_size, bi->metadata_size);
 		return -EINVAL;
 	}
 
@@ -194,8 +193,13 @@ static int blk_validate_integrity_limits(struct queue_limits *lim)
 		break;
 	}
 
-	if (!bi->interval_exp)
+	if (!bi->interval_exp) {
 		bi->interval_exp = ilog2(lim->logical_block_size);
+	} else if (bi->interval_exp < SECTOR_SHIFT ||
+		   bi->interval_exp > ilog2(lim->logical_block_size)) {
+		pr_warn("invalid interval_exp %u\n", bi->interval_exp);
+		return -EINVAL;
+	}
 
 	/*
 	 * The PI generation / validation helpers do not expect intervals to
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 8684c57498cc..e0a70d26972b 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -932,7 +932,7 @@ int blk_register_queue(struct gendisk *disk)
 		elevator_set_default(q);
 
 	blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);
-	wbt_enable_default(disk);
+	wbt_init_enable_default(disk);
 
 	/* Now everything is ready and send out KOBJ_ADD uevent */
 	kobject_uevent(&disk->queue_kobj, KOBJ_ADD);
diff --git a/block/blk-wbt.c b/block/blk-wbt.c
index eb8037bae0bd..0974875f77bd 100644
--- a/block/blk-wbt.c
+++ b/block/blk-wbt.c
@@ -699,7 +699,7 @@ static void wbt_requeue(struct rq_qos *rqos, struct request *rq)
 /*
  * Enable wbt if defaults are configured that way
  */
-void wbt_enable_default(struct gendisk *disk)
+static bool __wbt_enable_default(struct gendisk *disk)
 {
 	struct request_queue *q = disk->queue;
 	struct rq_qos *rqos;
@@ -716,19 +716,31 @@ void wbt_enable_default(struct gendisk *disk)
 		if (enable && RQWB(rqos)->enable_state == WBT_STATE_OFF_DEFAULT)
 			RQWB(rqos)->enable_state = WBT_STATE_ON_DEFAULT;
 		mutex_unlock(&disk->rqos_state_mutex);
-		return;
+		return false;
 	}
 	mutex_unlock(&disk->rqos_state_mutex);
 
 	/* Queue not registered? Maybe shutting down... */
 	if (!blk_queue_registered(q))
-		return;
+		return false;
 
 	if (queue_is_mq(q) && enable)
-		wbt_init(disk);
+		return true;
+	return false;
+}
+
+void wbt_enable_default(struct gendisk *disk)
+{
+	__wbt_enable_default(disk);
 }
 EXPORT_SYMBOL_GPL(wbt_enable_default);
 
+void wbt_init_enable_default(struct gendisk *disk)
+{
+	if (__wbt_enable_default(disk))
+		WARN_ON_ONCE(wbt_init(disk));
+}
+
 u64 wbt_default_latency_nsec(struct request_queue *q)
 {
 	/*
diff --git a/block/blk-wbt.h b/block/blk-wbt.h
index e5fc653b9b76..925f22475738 100644
--- a/block/blk-wbt.h
+++ b/block/blk-wbt.h
@@ -5,6 +5,7 @@
 #ifdef CONFIG_BLK_WBT
 
 int wbt_init(struct gendisk *disk);
+void wbt_init_enable_default(struct gendisk *disk);
 void wbt_disable_default(struct gendisk *disk);
 void wbt_enable_default(struct gendisk *disk);
 
@@ -16,6 +17,10 @@ u64 wbt_default_latency_nsec(struct request_queue *);
 
 #else
 
+static inline void wbt_init_enable_default(struct gendisk *disk)
+{
+}
+
 static inline void wbt_disable_default(struct gendisk *disk)
 {
 }
diff --git a/block/elevator.c b/block/elevator.c
index 5b37ef44f52d..a2f8b2251dc6 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -633,14 +633,10 @@ static int elevator_change_done(struct request_queue *q,
 			.et = ctx->old->et,
 			.data = ctx->old->elevator_data
 		};
-		bool enable_wbt = test_bit(ELEVATOR_FLAG_ENABLE_WBT_ON_EXIT,
-				&ctx->old->flags);
 
 		elv_unregister_queue(q, ctx->old);
 		blk_mq_free_sched_res(&res, ctx->old->type, q->tag_set);
 		kobject_put(&ctx->old->kobj);
-		if (enable_wbt)
-			wbt_enable_default(q->disk);
 	}
 	if (ctx->new) {
 		ret = elv_register_queue(q, ctx->new, !ctx->no_uevent);
diff --git a/block/elevator.h b/block/elevator.h
index a9d092c5a9e8..3eb32516be0b 100644
--- a/block/elevator.h
+++ b/block/elevator.h
@@ -156,7 +156,6 @@ struct elevator_queue
 
 #define ELEVATOR_FLAG_REGISTERED	0
 #define ELEVATOR_FLAG_DYING		1
-#define ELEVATOR_FLAG_ENABLE_WBT_ON_EXIT	2
 
 /*
  * block elevator interface
diff --git a/block/ioctl.c b/block/ioctl.c
index 61feed686418..344478348a54 100644
--- a/block/ioctl.c
+++ b/block/ioctl.c
@@ -442,11 +442,12 @@ static int blkdev_pr_read_keys(struct block_device *bdev, blk_mode_t mode,
 	if (copy_from_user(&read_keys, arg, sizeof(read_keys)))
 		return -EFAULT;
 
-	keys_info_len = struct_size(keys_info, keys, read_keys.num_keys);
-	if (keys_info_len == SIZE_MAX)
+	if (read_keys.num_keys > PR_KEYS_MAX)
 		return -EINVAL;
 
-	keys_info = kzalloc(keys_info_len, GFP_KERNEL);
+	keys_info_len = struct_size(keys_info, keys, read_keys.num_keys);
+
+	keys_info = kvzalloc(keys_info_len, GFP_KERNEL);
 	if (!keys_info)
 		return -ENOMEM;
 
@@ -473,7 +474,7 @@ static int blkdev_pr_read_keys(struct block_device *bdev, blk_mode_t mode,
 	if (copy_to_user(arg, &read_keys, sizeof(read_keys)))
 		ret = -EFAULT;
 out:
-	kfree(keys_info);
+	kvfree(keys_info);
 	return ret;
 }
 
diff --git a/certs/blacklist.c b/certs/blacklist.c
index 675dd7a8f07a..11fc858b2921 100644
--- a/certs/blacklist.c
+++ b/certs/blacklist.c
@@ -13,6 +13,7 @@
 #include <linux/sched.h>
 #include <linux/ctype.h>
 #include <linux/err.h>
+#include <linux/hex.h>
 #include <linux/seq_file.h>
 #include <linux/uidgid.h>
 #include <keys/asymmetric-type.h>
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 2e5b195b1b06..12a87f7cf150 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -601,9 +601,9 @@ menu "Length-preserving ciphers and modes"
 config CRYPTO_ADIANTUM
 	tristate "Adiantum"
 	select CRYPTO_CHACHA20
+	select CRYPTO_LIB_NH
 	select CRYPTO_LIB_POLY1305
 	select CRYPTO_LIB_POLY1305_GENERIC
-	select CRYPTO_NHPOLY1305
 	select CRYPTO_MANAGER
 	help
 	  Adiantum tweakable, length-preserving encryption mode
@@ -759,12 +759,6 @@ config CRYPTO_XTS
 	  implementation currently can't handle a sectorsize which is not a
 	  multiple of 16 bytes.
 
-config CRYPTO_NHPOLY1305
-	tristate
-	select CRYPTO_HASH
-	select CRYPTO_LIB_POLY1305
-	select CRYPTO_LIB_POLY1305_GENERIC
-
 endmenu
 
 menu "AEAD (authenticated encryption with associated data) ciphers"
diff --git a/crypto/Makefile b/crypto/Makefile
index 16a35649dd91..23d3db7be425 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -94,7 +94,6 @@ obj-$(CONFIG_CRYPTO_CTR) += ctr.o
 obj-$(CONFIG_CRYPTO_XCTR) += xctr.o
 obj-$(CONFIG_CRYPTO_HCTR2) += hctr2.o
 obj-$(CONFIG_CRYPTO_ADIANTUM) += adiantum.o
-obj-$(CONFIG_CRYPTO_NHPOLY1305) += nhpoly1305.o
 obj-$(CONFIG_CRYPTO_GCM) += gcm.o
 obj-$(CONFIG_CRYPTO_CCM) += ccm.o
 obj-$(CONFIG_CRYPTO_CHACHA20POLY1305) += chacha20poly1305.o
diff --git a/crypto/adiantum.c b/crypto/adiantum.c
index a6bca877c3c7..5ddf585abb66 100644
--- a/crypto/adiantum.c
+++ b/crypto/adiantum.c
@@ -20,23 +20,14 @@
  *
  *	- Stream cipher: XChaCha12 or XChaCha20
  *	- Block cipher: any with a 128-bit block size and 256-bit key
- *
- * This implementation doesn't currently allow other ε-∆U hash functions, i.e.
- * HPolyC is not supported.  This is because Adiantum is ~20% faster than HPolyC
- * but still provably as secure, and also the ε-∆U hash function of HBSH is
- * formally defined to take two inputs (tweak, message) which makes it difficult
- * to wrap with the crypto_shash API.  Rather, some details need to be handled
- * here.  Nevertheless, if needed in the future, support for other ε-∆U hash
- * functions could be added here.
  */
 
 #include <crypto/b128ops.h>
 #include <crypto/chacha.h>
 #include <crypto/internal/cipher.h>
-#include <crypto/internal/hash.h>
 #include <crypto/internal/poly1305.h>
 #include <crypto/internal/skcipher.h>
-#include <crypto/nhpoly1305.h>
+#include <crypto/nh.h>
 #include <crypto/scatterwalk.h>
 #include <linux/module.h>
 
@@ -50,7 +41,7 @@
 #define BLOCKCIPHER_KEY_SIZE		32
 
 /* Size of the hash key (K_H) in bytes */
-#define HASH_KEY_SIZE		(POLY1305_BLOCK_SIZE + NHPOLY1305_KEY_SIZE)
+#define HASH_KEY_SIZE		(2 * POLY1305_BLOCK_SIZE + NH_KEY_BYTES)
 
 /*
  * The specification allows variable-length tweaks, but Linux's crypto API
@@ -64,43 +55,40 @@
 struct adiantum_instance_ctx {
 	struct crypto_skcipher_spawn streamcipher_spawn;
 	struct crypto_cipher_spawn blockcipher_spawn;
-	struct crypto_shash_spawn hash_spawn;
 };
 
 struct adiantum_tfm_ctx {
 	struct crypto_skcipher *streamcipher;
 	struct crypto_cipher *blockcipher;
-	struct crypto_shash *hash;
 	struct poly1305_core_key header_hash_key;
+	struct poly1305_core_key msg_poly_key;
+	u32 nh_key[NH_KEY_WORDS];
 };
 
-struct adiantum_request_ctx {
+struct nhpoly1305_ctx {
+	/* Running total of polynomial evaluation */
+	struct poly1305_state poly_state;
+
+	/* Partial block buffer */
+	u8 buffer[NH_MESSAGE_UNIT];
+	unsigned int buflen;
 
 	/*
-	 * Buffer for right-hand part of data, i.e.
-	 *
-	 *    P_L => P_M => C_M => C_R when encrypting, or
-	 *    C_R => C_M => P_M => P_L when decrypting.
-	 *
-	 * Also used to build the IV for the stream cipher.
+	 * Number of bytes remaining until the current NH message reaches
+	 * NH_MESSAGE_BYTES.  When nonzero, 'nh_hash' holds the partial NH hash.
 	 */
-	union {
-		u8 bytes[XCHACHA_IV_SIZE];
-		__le32 words[XCHACHA_IV_SIZE / sizeof(__le32)];
-		le128 bignum;	/* interpret as element of Z/(2^{128}Z) */
-	} rbuf;
+	unsigned int nh_remaining;
 
-	bool enc; /* true if encrypting, false if decrypting */
+	__le64 nh_hash[NH_NUM_PASSES];
+};
 
+struct adiantum_request_ctx {
 	/*
-	 * The result of the Poly1305 ε-∆U hash function applied to
-	 * (bulk length, tweak)
+	 * skcipher sub-request size is unknown at compile-time, so it needs to
+	 * go after the members with known sizes.
 	 */
-	le128 header_hash;
-
-	/* Sub-requests, must be last */
 	union {
-		struct shash_desc hash_desc;
+		struct nhpoly1305_ctx hash_ctx;
 		struct skcipher_request streamcipher_req;
 	} u;
 };
@@ -170,12 +158,11 @@ static int adiantum_setkey(struct crypto_skcipher *tfm, const u8 *key,
 	/* Set the hash key (K_H) */
 	poly1305_core_setkey(&tctx->header_hash_key, keyp);
 	keyp += POLY1305_BLOCK_SIZE;
-
-	crypto_shash_clear_flags(tctx->hash, CRYPTO_TFM_REQ_MASK);
-	crypto_shash_set_flags(tctx->hash, crypto_skcipher_get_flags(tfm) &
-					   CRYPTO_TFM_REQ_MASK);
-	err = crypto_shash_setkey(tctx->hash, keyp, NHPOLY1305_KEY_SIZE);
-	keyp += NHPOLY1305_KEY_SIZE;
+	poly1305_core_setkey(&tctx->msg_poly_key, keyp);
+	keyp += POLY1305_BLOCK_SIZE;
+	for (int i = 0; i < NH_KEY_WORDS; i++)
+		tctx->nh_key[i] = get_unaligned_le32(&keyp[i * 4]);
+	keyp += NH_KEY_BYTES;
 	WARN_ON(keyp != &data->derived_keys[ARRAY_SIZE(data->derived_keys)]);
 out:
 	kfree_sensitive(data);
@@ -206,7 +193,7 @@ static inline void le128_sub(le128 *r, const le128 *v1, const le128 *v2)
 
 /*
  * Apply the Poly1305 ε-∆U hash function to (bulk length, tweak) and save the
- * result to rctx->header_hash.  This is the calculation
+ * result to @out.  This is the calculation
  *
  *	H_T ← Poly1305_{K_T}(bin_{128}(|L|) || T)
  *
@@ -216,11 +203,10 @@ static inline void le128_sub(le128 *r, const le128 *v1, const le128 *v2)
  * inputs only) taken over the left-hand part (the "bulk") of the message, to
  * give the overall Adiantum hash of the (tweak, left-hand part) pair.
  */
-static void adiantum_hash_header(struct skcipher_request *req)
+static void adiantum_hash_header(struct skcipher_request *req, le128 *out)
 {
 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 	const struct adiantum_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
-	struct adiantum_request_ctx *rctx = skcipher_request_ctx(req);
 	const unsigned int bulk_len = req->cryptlen - BLOCKCIPHER_BLOCK_SIZE;
 	struct {
 		__le64 message_bits;
@@ -240,99 +226,143 @@ static void adiantum_hash_header(struct skcipher_request *req)
 	poly1305_core_blocks(&state, &tctx->header_hash_key, req->iv,
 			     TWEAK_SIZE / POLY1305_BLOCK_SIZE, 1);
 
-	poly1305_core_emit(&state, NULL, &rctx->header_hash);
+	poly1305_core_emit(&state, NULL, out);
 }
 
-/* Hash the left-hand part (the "bulk") of the message using NHPoly1305 */
-static int adiantum_hash_message(struct skcipher_request *req,
-				 struct scatterlist *sgl, unsigned int nents,
-				 le128 *digest)
+/* Pass the next NH hash value through Poly1305 */
+static void process_nh_hash_value(struct nhpoly1305_ctx *ctx,
+				  const struct adiantum_tfm_ctx *key)
 {
-	struct adiantum_request_ctx *rctx = skcipher_request_ctx(req);
-	const unsigned int bulk_len = req->cryptlen - BLOCKCIPHER_BLOCK_SIZE;
-	struct shash_desc *hash_desc = &rctx->u.hash_desc;
-	struct sg_mapping_iter miter;
-	unsigned int i, n;
-	int err;
+	static_assert(NH_HASH_BYTES % POLY1305_BLOCK_SIZE == 0);
 
-	err = crypto_shash_init(hash_desc);
-	if (err)
-		return err;
+	poly1305_core_blocks(&ctx->poly_state, &key->msg_poly_key, ctx->nh_hash,
+			     NH_HASH_BYTES / POLY1305_BLOCK_SIZE, 1);
+}
 
-	sg_miter_start(&miter, sgl, nents, SG_MITER_FROM_SG | SG_MITER_ATOMIC);
-	for (i = 0; i < bulk_len; i += n) {
-		sg_miter_next(&miter);
-		n = min_t(unsigned int, miter.length, bulk_len - i);
-		err = crypto_shash_update(hash_desc, miter.addr, n);
-		if (err)
-			break;
-	}
-	sg_miter_stop(&miter);
-	if (err)
-		return err;
+/*
+ * Feed the next portion of the message data, as a whole number of 16-byte
+ * "NH message units", through NH and Poly1305.  Each NH hash is taken over
+ * 1024 bytes, except possibly the final one which is taken over a multiple of
+ * 16 bytes up to 1024.  Also, in the case where data is passed in misaligned
+ * chunks, we combine partial hashes; the end result is the same either way.
+ */
+static void nhpoly1305_units(struct nhpoly1305_ctx *ctx,
+			     const struct adiantum_tfm_ctx *key,
+			     const u8 *data, size_t len)
+{
+	do {
+		unsigned int bytes;
+
+		if (ctx->nh_remaining == 0) {
+			/* Starting a new NH message */
+			bytes = min(len, NH_MESSAGE_BYTES);
+			nh(key->nh_key, data, bytes, ctx->nh_hash);
+			ctx->nh_remaining = NH_MESSAGE_BYTES - bytes;
+		} else {
+			/* Continuing a previous NH message */
+			__le64 tmp_hash[NH_NUM_PASSES];
+			unsigned int pos;
+
+			pos = NH_MESSAGE_BYTES - ctx->nh_remaining;
+			bytes = min(len, ctx->nh_remaining);
+			nh(&key->nh_key[pos / 4], data, bytes, tmp_hash);
+			for (int i = 0; i < NH_NUM_PASSES; i++)
+				le64_add_cpu(&ctx->nh_hash[i],
+					     le64_to_cpu(tmp_hash[i]));
+			ctx->nh_remaining -= bytes;
+		}
+		if (ctx->nh_remaining == 0)
+			process_nh_hash_value(ctx, key);
+		data += bytes;
+		len -= bytes;
+	} while (len);
+}
 
-	return crypto_shash_final(hash_desc, (u8 *)digest);
+static void nhpoly1305_init(struct nhpoly1305_ctx *ctx)
+{
+	poly1305_core_init(&ctx->poly_state);
+	ctx->buflen = 0;
+	ctx->nh_remaining = 0;
 }
 
-/* Continue Adiantum encryption/decryption after the stream cipher step */
-static int adiantum_finish(struct skcipher_request *req)
+static void nhpoly1305_update(struct nhpoly1305_ctx *ctx,
+			      const struct adiantum_tfm_ctx *key,
+			      const u8 *data, size_t len)
 {
-	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-	const struct adiantum_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
-	struct adiantum_request_ctx *rctx = skcipher_request_ctx(req);
-	const unsigned int bulk_len = req->cryptlen - BLOCKCIPHER_BLOCK_SIZE;
-	struct scatterlist *dst = req->dst;
-	const unsigned int dst_nents = sg_nents(dst);
-	le128 digest;
-	int err;
+	unsigned int bytes;
+
+	if (ctx->buflen) {
+		bytes = min(len, (int)NH_MESSAGE_UNIT - ctx->buflen);
+		memcpy(&ctx->buffer[ctx->buflen], data, bytes);
+		ctx->buflen += bytes;
+		if (ctx->buflen < NH_MESSAGE_UNIT)
+			return;
+		nhpoly1305_units(ctx, key, ctx->buffer, NH_MESSAGE_UNIT);
+		ctx->buflen = 0;
+		data += bytes;
+		len -= bytes;
+	}
 
-	/* If decrypting, decrypt C_M with the block cipher to get P_M */
-	if (!rctx->enc)
-		crypto_cipher_decrypt_one(tctx->blockcipher, rctx->rbuf.bytes,
-					  rctx->rbuf.bytes);
+	if (len >= NH_MESSAGE_UNIT) {
+		bytes = round_down(len, NH_MESSAGE_UNIT);
+		nhpoly1305_units(ctx, key, data, bytes);
+		data += bytes;
+		len -= bytes;
+	}
 
-	/*
-	 * Second hash step
-	 *	enc: C_R = C_M - H_{K_H}(T, C_L)
-	 *	dec: P_R = P_M - H_{K_H}(T, P_L)
-	 */
-	rctx->u.hash_desc.tfm = tctx->hash;
-	le128_sub(&rctx->rbuf.bignum, &rctx->rbuf.bignum, &rctx->header_hash);
-	if (dst_nents == 1 && dst->offset + req->cryptlen <= PAGE_SIZE) {
-		/* Fast path for single-page destination */
-		struct page *page = sg_page(dst);
-		void *virt = kmap_local_page(page) + dst->offset;
+	if (len) {
+		memcpy(ctx->buffer, data, len);
+		ctx->buflen = len;
+	}
+}
 
-		err = crypto_shash_digest(&rctx->u.hash_desc, virt, bulk_len,
-					  (u8 *)&digest);
-		if (err) {
-			kunmap_local(virt);
-			return err;
-		}
-		le128_sub(&rctx->rbuf.bignum, &rctx->rbuf.bignum, &digest);
-		memcpy(virt + bulk_len, &rctx->rbuf.bignum, sizeof(le128));
-		flush_dcache_page(page);
-		kunmap_local(virt);
-	} else {
-		/* Slow path that works for any destination scatterlist */
-		err = adiantum_hash_message(req, dst, dst_nents, &digest);
-		if (err)
-			return err;
-		le128_sub(&rctx->rbuf.bignum, &rctx->rbuf.bignum, &digest);
-		scatterwalk_map_and_copy(&rctx->rbuf.bignum, dst,
-					 bulk_len, sizeof(le128), 1);
+static void nhpoly1305_final(struct nhpoly1305_ctx *ctx,
+			     const struct adiantum_tfm_ctx *key, le128 *out)
+{
+	if (ctx->buflen) {
+		memset(&ctx->buffer[ctx->buflen], 0,
+		       NH_MESSAGE_UNIT - ctx->buflen);
+		nhpoly1305_units(ctx, key, ctx->buffer, NH_MESSAGE_UNIT);
 	}
-	return 0;
+
+	if (ctx->nh_remaining)
+		process_nh_hash_value(ctx, key);
+
+	poly1305_core_emit(&ctx->poly_state, NULL, out);
 }
 
-static void adiantum_streamcipher_done(void *data, int err)
+/*
+ * Hash the left-hand part (the "bulk") of the message as follows:
+ *
+ *	H_L ← Poly1305_{K_L}(NH_{K_N}(pad_{128}(L)))
+ *
+ * See section 6.4 of the Adiantum paper.  This is an ε-almost-∆-universal
+ * (ε-∆U) hash function for equal-length inputs over Z/(2^{128}Z), where the "∆"
+ * operation is addition.  It hashes 1024-byte chunks of the input with the NH
+ * hash function, reducing the input length by 32x.  The resulting NH hashes are
+ * evaluated as a polynomial in GF(2^{130}-5), like in the Poly1305 MAC.  Note
+ * that the polynomial evaluation by itself would suffice to achieve the ε-∆U
+ * property; NH is used for performance since it's much faster than Poly1305.
+ */
+static void adiantum_hash_message(struct skcipher_request *req,
+				  struct scatterlist *sgl, le128 *out)
 {
-	struct skcipher_request *req = data;
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	const struct adiantum_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
+	struct adiantum_request_ctx *rctx = skcipher_request_ctx(req);
+	unsigned int len = req->cryptlen - BLOCKCIPHER_BLOCK_SIZE;
+	struct scatter_walk walk;
 
-	if (!err)
-		err = adiantum_finish(req);
+	nhpoly1305_init(&rctx->u.hash_ctx);
+	scatterwalk_start(&walk, sgl);
+	while (len) {
+		unsigned int n = scatterwalk_next(&walk, len);
 
-	skcipher_request_complete(req, err);
+		nhpoly1305_update(&rctx->u.hash_ctx, tctx, walk.addr, n);
+		scatterwalk_done_src(&walk, n);
+		len -= n;
+	}
+	nhpoly1305_final(&rctx->u.hash_ctx, tctx, out);
 }
 
 static int adiantum_crypt(struct skcipher_request *req, bool enc)
@@ -341,55 +371,63 @@ static int adiantum_crypt(struct skcipher_request *req, bool enc)
 	const struct adiantum_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
 	struct adiantum_request_ctx *rctx = skcipher_request_ctx(req);
 	const unsigned int bulk_len = req->cryptlen - BLOCKCIPHER_BLOCK_SIZE;
-	struct scatterlist *src = req->src;
-	const unsigned int src_nents = sg_nents(src);
+	struct scatterlist *src = req->src, *dst = req->dst;
+	/*
+	 * Buffer for right-hand part of data, i.e.
+	 *
+	 *    P_L => P_M => C_M => C_R when encrypting, or
+	 *    C_R => C_M => P_M => P_L when decrypting.
+	 *
+	 * Also used to build the IV for the stream cipher.
+	 */
+	union {
+		u8 bytes[XCHACHA_IV_SIZE];
+		__le32 words[XCHACHA_IV_SIZE / sizeof(__le32)];
+		le128 bignum; /* interpret as element of Z/(2^{128}Z) */
+	} rbuf;
+	le128 header_hash, msg_hash;
 	unsigned int stream_len;
-	le128 digest;
 	int err;
 
 	if (req->cryptlen < BLOCKCIPHER_BLOCK_SIZE)
 		return -EINVAL;
 
-	rctx->enc = enc;
-
 	/*
 	 * First hash step
 	 *	enc: P_M = P_R + H_{K_H}(T, P_L)
 	 *	dec: C_M = C_R + H_{K_H}(T, C_L)
 	 */
-	adiantum_hash_header(req);
-	rctx->u.hash_desc.tfm = tctx->hash;
-	if (src_nents == 1 && src->offset + req->cryptlen <= PAGE_SIZE) {
+	adiantum_hash_header(req, &header_hash);
+	if (src->length >= req->cryptlen &&
+	    src->offset + req->cryptlen <= PAGE_SIZE) {
 		/* Fast path for single-page source */
 		void *virt = kmap_local_page(sg_page(src)) + src->offset;
 
-		err = crypto_shash_digest(&rctx->u.hash_desc, virt, bulk_len,
-					  (u8 *)&digest);
-		memcpy(&rctx->rbuf.bignum, virt + bulk_len, sizeof(le128));
+		nhpoly1305_init(&rctx->u.hash_ctx);
+		nhpoly1305_update(&rctx->u.hash_ctx, tctx, virt, bulk_len);
+		nhpoly1305_final(&rctx->u.hash_ctx, tctx, &msg_hash);
+		memcpy(&rbuf.bignum, virt + bulk_len, sizeof(le128));
 		kunmap_local(virt);
 	} else {
 		/* Slow path that works for any source scatterlist */
-		err = adiantum_hash_message(req, src, src_nents, &digest);
-		scatterwalk_map_and_copy(&rctx->rbuf.bignum, src,
-					 bulk_len, sizeof(le128), 0);
+		adiantum_hash_message(req, src, &msg_hash);
+		memcpy_from_sglist(&rbuf.bignum, src, bulk_len, sizeof(le128));
 	}
-	if (err)
-		return err;
-	le128_add(&rctx->rbuf.bignum, &rctx->rbuf.bignum, &rctx->header_hash);
-	le128_add(&rctx->rbuf.bignum, &rctx->rbuf.bignum, &digest);
+	le128_add(&rbuf.bignum, &rbuf.bignum, &header_hash);
+	le128_add(&rbuf.bignum, &rbuf.bignum, &msg_hash);
 
 	/* If encrypting, encrypt P_M with the block cipher to get C_M */
 	if (enc)
-		crypto_cipher_encrypt_one(tctx->blockcipher, rctx->rbuf.bytes,
-					  rctx->rbuf.bytes);
+		crypto_cipher_encrypt_one(tctx->blockcipher, rbuf.bytes,
+					  rbuf.bytes);
 
 	/* Initialize the rest of the XChaCha IV (first part is C_M) */
 	BUILD_BUG_ON(BLOCKCIPHER_BLOCK_SIZE != 16);
 	BUILD_BUG_ON(XCHACHA_IV_SIZE != 32);	/* nonce || stream position */
-	rctx->rbuf.words[4] = cpu_to_le32(1);
-	rctx->rbuf.words[5] = 0;
-	rctx->rbuf.words[6] = 0;
-	rctx->rbuf.words[7] = 0;
+	rbuf.words[4] = cpu_to_le32(1);
+	rbuf.words[5] = 0;
+	rbuf.words[6] = 0;
+	rbuf.words[7] = 0;
 
 	/*
 	 * XChaCha needs to be done on all the data except the last 16 bytes;
@@ -406,12 +444,44 @@ static int adiantum_crypt(struct skcipher_request *req, bool enc)
 
 	skcipher_request_set_tfm(&rctx->u.streamcipher_req, tctx->streamcipher);
 	skcipher_request_set_crypt(&rctx->u.streamcipher_req, req->src,
-				   req->dst, stream_len, &rctx->rbuf);
+				   req->dst, stream_len, &rbuf);
 	skcipher_request_set_callback(&rctx->u.streamcipher_req,
-				      req->base.flags,
-				      adiantum_streamcipher_done, req);
-	return crypto_skcipher_encrypt(&rctx->u.streamcipher_req) ?:
-		adiantum_finish(req);
+				      req->base.flags, NULL, NULL);
+	err = crypto_skcipher_encrypt(&rctx->u.streamcipher_req);
+	if (err)
+		return err;
+
+	/* If decrypting, decrypt C_M with the block cipher to get P_M */
+	if (!enc)
+		crypto_cipher_decrypt_one(tctx->blockcipher, rbuf.bytes,
+					  rbuf.bytes);
+
+	/*
+	 * Second hash step
+	 *	enc: C_R = C_M - H_{K_H}(T, C_L)
+	 *	dec: P_R = P_M - H_{K_H}(T, P_L)
+	 */
+	le128_sub(&rbuf.bignum, &rbuf.bignum, &header_hash);
+	if (dst->length >= req->cryptlen &&
+	    dst->offset + req->cryptlen <= PAGE_SIZE) {
+		/* Fast path for single-page destination */
+		struct page *page = sg_page(dst);
+		void *virt = kmap_local_page(page) + dst->offset;
+
+		nhpoly1305_init(&rctx->u.hash_ctx);
+		nhpoly1305_update(&rctx->u.hash_ctx, tctx, virt, bulk_len);
+		nhpoly1305_final(&rctx->u.hash_ctx, tctx, &msg_hash);
+		le128_sub(&rbuf.bignum, &rbuf.bignum, &msg_hash);
+		memcpy(virt + bulk_len, &rbuf.bignum, sizeof(le128));
+		flush_dcache_page(page);
+		kunmap_local(virt);
+	} else {
+		/* Slow path that works for any destination scatterlist */
+		adiantum_hash_message(req, dst, &msg_hash);
+		le128_sub(&rbuf.bignum, &rbuf.bignum, &msg_hash);
+		memcpy_to_sglist(dst, bulk_len, &rbuf.bignum, sizeof(le128));
+	}
+	return 0;
 }
 
 static int adiantum_encrypt(struct skcipher_request *req)
@@ -431,8 +501,6 @@ static int adiantum_init_tfm(struct crypto_skcipher *tfm)
 	struct adiantum_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
 	struct crypto_skcipher *streamcipher;
 	struct crypto_cipher *blockcipher;
-	struct crypto_shash *hash;
-	unsigned int subreq_size;
 	int err;
 
 	streamcipher = crypto_spawn_skcipher(&ictx->streamcipher_spawn);
@@ -445,32 +513,18 @@ static int adiantum_init_tfm(struct crypto_skcipher *tfm)
 		goto err_free_streamcipher;
 	}
 
-	hash = crypto_spawn_shash(&ictx->hash_spawn);
-	if (IS_ERR(hash)) {
-		err = PTR_ERR(hash);
-		goto err_free_blockcipher;
-	}
-
 	tctx->streamcipher = streamcipher;
 	tctx->blockcipher = blockcipher;
-	tctx->hash = hash;
 
 	BUILD_BUG_ON(offsetofend(struct adiantum_request_ctx, u) !=
 		     sizeof(struct adiantum_request_ctx));
-	subreq_size = max(sizeof_field(struct adiantum_request_ctx,
-				       u.hash_desc) +
-			  crypto_shash_descsize(hash),
-			  sizeof_field(struct adiantum_request_ctx,
-				       u.streamcipher_req) +
-			  crypto_skcipher_reqsize(streamcipher));
-
-	crypto_skcipher_set_reqsize(tfm,
-				    offsetof(struct adiantum_request_ctx, u) +
-				    subreq_size);
+	crypto_skcipher_set_reqsize(
+		tfm, max(sizeof(struct adiantum_request_ctx),
+			 offsetofend(struct adiantum_request_ctx,
+				     u.streamcipher_req) +
+				 crypto_skcipher_reqsize(streamcipher)));
 	return 0;
 
-err_free_blockcipher:
-	crypto_free_cipher(blockcipher);
 err_free_streamcipher:
 	crypto_free_skcipher(streamcipher);
 	return err;
@@ -482,7 +536,6 @@ static void adiantum_exit_tfm(struct crypto_skcipher *tfm)
 
 	crypto_free_skcipher(tctx->streamcipher);
 	crypto_free_cipher(tctx->blockcipher);
-	crypto_free_shash(tctx->hash);
 }
 
 static void adiantum_free_instance(struct skcipher_instance *inst)
@@ -491,7 +544,6 @@ static void adiantum_free_instance(struct skcipher_instance *inst)
 
 	crypto_drop_skcipher(&ictx->streamcipher_spawn);
 	crypto_drop_cipher(&ictx->blockcipher_spawn);
-	crypto_drop_shash(&ictx->hash_spawn);
 	kfree(inst);
 }
 
@@ -499,9 +551,9 @@ static void adiantum_free_instance(struct skcipher_instance *inst)
  * Check for a supported set of inner algorithms.
  * See the comment at the beginning of this file.
  */
-static bool adiantum_supported_algorithms(struct skcipher_alg_common *streamcipher_alg,
-					  struct crypto_alg *blockcipher_alg,
-					  struct shash_alg *hash_alg)
+static bool
+adiantum_supported_algorithms(struct skcipher_alg_common *streamcipher_alg,
+			      struct crypto_alg *blockcipher_alg)
 {
 	if (strcmp(streamcipher_alg->base.cra_name, "xchacha12") != 0 &&
 	    strcmp(streamcipher_alg->base.cra_name, "xchacha20") != 0)
@@ -513,21 +565,16 @@ static bool adiantum_supported_algorithms(struct skcipher_alg_common *streamciph
 	if (blockcipher_alg->cra_blocksize != BLOCKCIPHER_BLOCK_SIZE)
 		return false;
 
-	if (strcmp(hash_alg->base.cra_name, "nhpoly1305") != 0)
-		return false;
-
 	return true;
 }
 
 static int adiantum_create(struct crypto_template *tmpl, struct rtattr **tb)
 {
 	u32 mask;
-	const char *nhpoly1305_name;
 	struct skcipher_instance *inst;
 	struct adiantum_instance_ctx *ictx;
 	struct skcipher_alg_common *streamcipher_alg;
 	struct crypto_alg *blockcipher_alg;
-	struct shash_alg *hash_alg;
 	int err;
 
 	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SKCIPHER, &mask);
@@ -542,7 +589,8 @@ static int adiantum_create(struct crypto_template *tmpl, struct rtattr **tb)
 	/* Stream cipher, e.g. "xchacha12" */
 	err = crypto_grab_skcipher(&ictx->streamcipher_spawn,
 				   skcipher_crypto_instance(inst),
-				   crypto_attr_alg_name(tb[1]), 0, mask);
+				   crypto_attr_alg_name(tb[1]), 0,
+				   mask | CRYPTO_ALG_ASYNC /* sync only */);
 	if (err)
 		goto err_free_inst;
 	streamcipher_alg = crypto_spawn_skcipher_alg_common(&ictx->streamcipher_spawn);
@@ -555,23 +603,21 @@ static int adiantum_create(struct crypto_template *tmpl, struct rtattr **tb)
 		goto err_free_inst;
 	blockcipher_alg = crypto_spawn_cipher_alg(&ictx->blockcipher_spawn);
 
-	/* NHPoly1305 ε-∆U hash function */
-	nhpoly1305_name = crypto_attr_alg_name(tb[3]);
-	if (nhpoly1305_name == ERR_PTR(-ENOENT))
-		nhpoly1305_name = "nhpoly1305";
-	err = crypto_grab_shash(&ictx->hash_spawn,
-				skcipher_crypto_instance(inst),
-				nhpoly1305_name, 0, mask);
-	if (err)
+	/*
+	 * Originally there was an optional third parameter, for requesting a
+	 * specific implementation of "nhpoly1305" for message hashing.  This is
+	 * no longer supported.  The best implementation is just always used.
+	 */
+	if (crypto_attr_alg_name(tb[3]) != ERR_PTR(-ENOENT)) {
+		err = -ENOENT;
 		goto err_free_inst;
-	hash_alg = crypto_spawn_shash_alg(&ictx->hash_spawn);
+	}
 
 	/* Check the set of algorithms */
-	if (!adiantum_supported_algorithms(streamcipher_alg, blockcipher_alg,
-					   hash_alg)) {
-		pr_warn("Unsupported Adiantum instantiation: (%s,%s,%s)\n",
+	if (!adiantum_supported_algorithms(streamcipher_alg, blockcipher_alg)) {
+		pr_warn("Unsupported Adiantum instantiation: (%s,%s)\n",
 			streamcipher_alg->base.cra_name,
-			blockcipher_alg->cra_name, hash_alg->base.cra_name);
+			blockcipher_alg->cra_name);
 		err = -EINVAL;
 		goto err_free_inst;
 	}
@@ -584,10 +630,8 @@ static int adiantum_create(struct crypto_template *tmpl, struct rtattr **tb)
 		     blockcipher_alg->cra_name) >= CRYPTO_MAX_ALG_NAME)
 		goto err_free_inst;
 	if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
-		     "adiantum(%s,%s,%s)",
-		     streamcipher_alg->base.cra_driver_name,
-		     blockcipher_alg->cra_driver_name,
-		     hash_alg->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
+		     "adiantum(%s,%s)", streamcipher_alg->base.cra_driver_name,
+		     blockcipher_alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 		goto err_free_inst;
 
 	inst->alg.base.cra_blocksize = BLOCKCIPHER_BLOCK_SIZE;
@@ -596,12 +640,12 @@ static int adiantum_create(struct crypto_template *tmpl, struct rtattr **tb)
 	/*
 	 * The block cipher is only invoked once per message, so for long
 	 * messages (e.g. sectors for disk encryption) its performance doesn't
-	 * matter as much as that of the stream cipher and hash function.  Thus,
-	 * weigh the block cipher's ->cra_priority less.
+	 * matter as much as that of the stream cipher.  Thus, weigh the block
+	 * cipher's ->cra_priority less.
 	 */
 	inst->alg.base.cra_priority = (4 * streamcipher_alg->base.cra_priority +
-				       2 * hash_alg->base.cra_priority +
-				       blockcipher_alg->cra_priority) / 7;
+				       blockcipher_alg->cra_priority) /
+				      5;
 
 	inst->alg.setkey = adiantum_setkey;
 	inst->alg.encrypt = adiantum_encrypt;
@@ -622,7 +666,7 @@ static int adiantum_create(struct crypto_template *tmpl, struct rtattr **tb)
 	return err;
 }
 
-/* adiantum(streamcipher_name, blockcipher_name [, nhpoly1305_name]) */
+/* adiantum(streamcipher_name, blockcipher_name) */
 static struct crypto_template adiantum_tmpl = {
 	.name = "adiantum",
 	.create = adiantum_create,
diff --git a/crypto/asymmetric_keys/asymmetric_type.c b/crypto/asymmetric_keys/asymmetric_type.c
index 348966ea2175..b7a08de58064 100644
--- a/crypto/asymmetric_keys/asymmetric_type.c
+++ b/crypto/asymmetric_keys/asymmetric_type.c
@@ -9,6 +9,7 @@
 #include <keys/asymmetric-subtype.h>
 #include <keys/asymmetric-parser.h>
 #include <crypto/public_key.h>
+#include <linux/hex.h>
 #include <linux/seq_file.h>
 #include <linux/module.h>
 #include <linux/overflow.h>
diff --git a/crypto/asymmetric_keys/x509_public_key.c b/crypto/asymmetric_keys/x509_public_key.c
index 12e3341e806b..0499b13ba3ef 100644
--- a/crypto/asymmetric_keys/x509_public_key.c
+++ b/crypto/asymmetric_keys/x509_public_key.c
@@ -10,6 +10,7 @@
 #include <keys/asymmetric-parser.h>
 #include <keys/asymmetric-subtype.h>
 #include <keys/system_keyring.h>
+#include <linux/hex.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
diff --git a/crypto/krb5/selftest.c b/crypto/krb5/selftest.c
index 4519c572d37e..67c4accd8cbd 100644
--- a/crypto/krb5/selftest.c
+++ b/crypto/krb5/selftest.c
@@ -7,6 +7,7 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <linux/hex.h>
 #include <linux/slab.h>
 #include <crypto/skcipher.h>
 #include <crypto/hash.h>
diff --git a/crypto/nhpoly1305.c b/crypto/nhpoly1305.c
deleted file mode 100644
index 2b648615b5ec..000000000000
--- a/crypto/nhpoly1305.c
+++ /dev/null
@@ -1,255 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
- *
- * Copyright 2018 Google LLC
- */
-
-/*
- * "NHPoly1305" is the main component of Adiantum hashing.
- * Specifically, it is the calculation
- *
- *	H_L ← Poly1305_{K_L}(NH_{K_N}(pad_{128}(L)))
- *
- * from the procedure in section 6.4 of the Adiantum paper [1].  It is an
- * ε-almost-∆-universal (ε-∆U) hash function for equal-length inputs over
- * Z/(2^{128}Z), where the "∆" operation is addition.  It hashes 1024-byte
- * chunks of the input with the NH hash function [2], reducing the input length
- * by 32x.  The resulting NH digests are evaluated as a polynomial in
- * GF(2^{130}-5), like in the Poly1305 MAC [3].  Note that the polynomial
- * evaluation by itself would suffice to achieve the ε-∆U property; NH is used
- * for performance since it's over twice as fast as Poly1305.
- *
- * This is *not* a cryptographic hash function; do not use it as such!
- *
- * [1] Adiantum: length-preserving encryption for entry-level processors
- *     (https://eprint.iacr.org/2018/720.pdf)
- * [2] UMAC: Fast and Secure Message Authentication
- *     (https://fastcrypto.org/umac/umac_proc.pdf)
- * [3] The Poly1305-AES message-authentication code
- *     (https://cr.yp.to/mac/poly1305-20050329.pdf)
- */
-
-#include <linux/unaligned.h>
-#include <crypto/algapi.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/poly1305.h>
-#include <crypto/nhpoly1305.h>
-#include <linux/crypto.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-
-static void nh_generic(const u32 *key, const u8 *message, size_t message_len,
-		       __le64 hash[NH_NUM_PASSES])
-{
-	u64 sums[4] = { 0, 0, 0, 0 };
-
-	BUILD_BUG_ON(NH_PAIR_STRIDE != 2);
-	BUILD_BUG_ON(NH_NUM_PASSES != 4);
-
-	while (message_len) {
-		u32 m0 = get_unaligned_le32(message + 0);
-		u32 m1 = get_unaligned_le32(message + 4);
-		u32 m2 = get_unaligned_le32(message + 8);
-		u32 m3 = get_unaligned_le32(message + 12);
-
-		sums[0] += (u64)(u32)(m0 + key[ 0]) * (u32)(m2 + key[ 2]);
-		sums[1] += (u64)(u32)(m0 + key[ 4]) * (u32)(m2 + key[ 6]);
-		sums[2] += (u64)(u32)(m0 + key[ 8]) * (u32)(m2 + key[10]);
-		sums[3] += (u64)(u32)(m0 + key[12]) * (u32)(m2 + key[14]);
-		sums[0] += (u64)(u32)(m1 + key[ 1]) * (u32)(m3 + key[ 3]);
-		sums[1] += (u64)(u32)(m1 + key[ 5]) * (u32)(m3 + key[ 7]);
-		sums[2] += (u64)(u32)(m1 + key[ 9]) * (u32)(m3 + key[11]);
-		sums[3] += (u64)(u32)(m1 + key[13]) * (u32)(m3 + key[15]);
-		key += NH_MESSAGE_UNIT / sizeof(key[0]);
-		message += NH_MESSAGE_UNIT;
-		message_len -= NH_MESSAGE_UNIT;
-	}
-
-	hash[0] = cpu_to_le64(sums[0]);
-	hash[1] = cpu_to_le64(sums[1]);
-	hash[2] = cpu_to_le64(sums[2]);
-	hash[3] = cpu_to_le64(sums[3]);
-}
-
-/* Pass the next NH hash value through Poly1305 */
-static void process_nh_hash_value(struct nhpoly1305_state *state,
-				  const struct nhpoly1305_key *key)
-{
-	BUILD_BUG_ON(NH_HASH_BYTES % POLY1305_BLOCK_SIZE != 0);
-
-	poly1305_core_blocks(&state->poly_state, &key->poly_key, state->nh_hash,
-			     NH_HASH_BYTES / POLY1305_BLOCK_SIZE, 1);
-}
-
-/*
- * Feed the next portion of the source data, as a whole number of 16-byte
- * "NH message units", through NH and Poly1305.  Each NH hash is taken over
- * 1024 bytes, except possibly the final one which is taken over a multiple of
- * 16 bytes up to 1024.  Also, in the case where data is passed in misaligned
- * chunks, we combine partial hashes; the end result is the same either way.
- */
-static void nhpoly1305_units(struct nhpoly1305_state *state,
-			     const struct nhpoly1305_key *key,
-			     const u8 *src, unsigned int srclen, nh_t nh_fn)
-{
-	do {
-		unsigned int bytes;
-
-		if (state->nh_remaining == 0) {
-			/* Starting a new NH message */
-			bytes = min_t(unsigned int, srclen, NH_MESSAGE_BYTES);
-			nh_fn(key->nh_key, src, bytes, state->nh_hash);
-			state->nh_remaining = NH_MESSAGE_BYTES - bytes;
-		} else {
-			/* Continuing a previous NH message */
-			__le64 tmp_hash[NH_NUM_PASSES];
-			unsigned int pos;
-			int i;
-
-			pos = NH_MESSAGE_BYTES - state->nh_remaining;
-			bytes = min(srclen, state->nh_remaining);
-			nh_fn(&key->nh_key[pos / 4], src, bytes, tmp_hash);
-			for (i = 0; i < NH_NUM_PASSES; i++)
-				le64_add_cpu(&state->nh_hash[i],
-					     le64_to_cpu(tmp_hash[i]));
-			state->nh_remaining -= bytes;
-		}
-		if (state->nh_remaining == 0)
-			process_nh_hash_value(state, key);
-		src += bytes;
-		srclen -= bytes;
-	} while (srclen);
-}
-
-int crypto_nhpoly1305_setkey(struct crypto_shash *tfm,
-			     const u8 *key, unsigned int keylen)
-{
-	struct nhpoly1305_key *ctx = crypto_shash_ctx(tfm);
-	int i;
-
-	if (keylen != NHPOLY1305_KEY_SIZE)
-		return -EINVAL;
-
-	poly1305_core_setkey(&ctx->poly_key, key);
-	key += POLY1305_BLOCK_SIZE;
-
-	for (i = 0; i < NH_KEY_WORDS; i++)
-		ctx->nh_key[i] = get_unaligned_le32(key + i * sizeof(u32));
-
-	return 0;
-}
-EXPORT_SYMBOL(crypto_nhpoly1305_setkey);
-
-int crypto_nhpoly1305_init(struct shash_desc *desc)
-{
-	struct nhpoly1305_state *state = shash_desc_ctx(desc);
-
-	poly1305_core_init(&state->poly_state);
-	state->buflen = 0;
-	state->nh_remaining = 0;
-	return 0;
-}
-EXPORT_SYMBOL(crypto_nhpoly1305_init);
-
-int crypto_nhpoly1305_update_helper(struct shash_desc *desc,
-				    const u8 *src, unsigned int srclen,
-				    nh_t nh_fn)
-{
-	struct nhpoly1305_state *state = shash_desc_ctx(desc);
-	const struct nhpoly1305_key *key = crypto_shash_ctx(desc->tfm);
-	unsigned int bytes;
-
-	if (state->buflen) {
-		bytes = min(srclen, (int)NH_MESSAGE_UNIT - state->buflen);
-		memcpy(&state->buffer[state->buflen], src, bytes);
-		state->buflen += bytes;
-		if (state->buflen < NH_MESSAGE_UNIT)
-			return 0;
-		nhpoly1305_units(state, key, state->buffer, NH_MESSAGE_UNIT,
-				 nh_fn);
-		state->buflen = 0;
-		src += bytes;
-		srclen -= bytes;
-	}
-
-	if (srclen >= NH_MESSAGE_UNIT) {
-		bytes = round_down(srclen, NH_MESSAGE_UNIT);
-		nhpoly1305_units(state, key, src, bytes, nh_fn);
-		src += bytes;
-		srclen -= bytes;
-	}
-
-	if (srclen) {
-		memcpy(state->buffer, src, srclen);
-		state->buflen = srclen;
-	}
-	return 0;
-}
-EXPORT_SYMBOL(crypto_nhpoly1305_update_helper);
-
-int crypto_nhpoly1305_update(struct shash_desc *desc,
-			     const u8 *src, unsigned int srclen)
-{
-	return crypto_nhpoly1305_update_helper(desc, src, srclen, nh_generic);
-}
-EXPORT_SYMBOL(crypto_nhpoly1305_update);
-
-int crypto_nhpoly1305_final_helper(struct shash_desc *desc, u8 *dst, nh_t nh_fn)
-{
-	struct nhpoly1305_state *state = shash_desc_ctx(desc);
-	const struct nhpoly1305_key *key = crypto_shash_ctx(desc->tfm);
-
-	if (state->buflen) {
-		memset(&state->buffer[state->buflen], 0,
-		       NH_MESSAGE_UNIT - state->buflen);
-		nhpoly1305_units(state, key, state->buffer, NH_MESSAGE_UNIT,
-				 nh_fn);
-	}
-
-	if (state->nh_remaining)
-		process_nh_hash_value(state, key);
-
-	poly1305_core_emit(&state->poly_state, NULL, dst);
-	return 0;
-}
-EXPORT_SYMBOL(crypto_nhpoly1305_final_helper);
-
-int crypto_nhpoly1305_final(struct shash_desc *desc, u8 *dst)
-{
-	return crypto_nhpoly1305_final_helper(desc, dst, nh_generic);
-}
-EXPORT_SYMBOL(crypto_nhpoly1305_final);
-
-static struct shash_alg nhpoly1305_alg = {
-	.base.cra_name		= "nhpoly1305",
-	.base.cra_driver_name	= "nhpoly1305-generic",
-	.base.cra_priority	= 100,
-	.base.cra_ctxsize	= sizeof(struct nhpoly1305_key),
-	.base.cra_module	= THIS_MODULE,
-	.digestsize		= POLY1305_DIGEST_SIZE,
-	.init			= crypto_nhpoly1305_init,
-	.update			= crypto_nhpoly1305_update,
-	.final			= crypto_nhpoly1305_final,
-	.setkey			= crypto_nhpoly1305_setkey,
-	.descsize		= sizeof(struct nhpoly1305_state),
-};
-
-static int __init nhpoly1305_mod_init(void)
-{
-	return crypto_register_shash(&nhpoly1305_alg);
-}
-
-static void __exit nhpoly1305_mod_exit(void)
-{
-	crypto_unregister_shash(&nhpoly1305_alg);
-}
-
-module_init(nhpoly1305_mod_init);
-module_exit(nhpoly1305_mod_exit);
-
-MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function");
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
-MODULE_ALIAS_CRYPTO("nhpoly1305");
-MODULE_ALIAS_CRYPTO("nhpoly1305-generic");
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index a302be53896d..5df204d9c9dd 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -4061,14 +4061,14 @@ static int alg_test_null(const struct alg_test_desc *desc,
 static const struct alg_test_desc alg_test_descs[] = {
 	{
 		.alg = "adiantum(xchacha12,aes)",
-		.generic_driver = "adiantum(xchacha12-lib,aes-generic,nhpoly1305-generic)",
+		.generic_driver = "adiantum(xchacha12-lib,aes-generic)",
 		.test = alg_test_skcipher,
 		.suite = {
 			.cipher = __VECS(adiantum_xchacha12_aes_tv_template)
 		},
 	}, {
 		.alg = "adiantum(xchacha20,aes)",
-		.generic_driver = "adiantum(xchacha20-lib,aes-generic,nhpoly1305-generic)",
+		.generic_driver = "adiantum(xchacha20-lib,aes-generic)",
 		.test = alg_test_skcipher,
 		.suite = {
 			.cipher = __VECS(adiantum_xchacha20_aes_tv_template)
@@ -5172,12 +5172,6 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.suite = {
 			.hash = __VECS(michael_mic_tv_template)
 		}
-	}, {
-		.alg = "nhpoly1305",
-		.test = alg_test_hash,
-		.suite = {
-			.hash = __VECS(nhpoly1305_tv_template)
-		}
 	}, {
 		.alg = "p1363(ecdsa-nist-p192)",
 		.test = alg_test_null,
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 80bf5f1b67a6..1a3329e1c325 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -7599,1378 +7599,6 @@ static const struct hash_testvec hmac_sha3_512_tv_template[] = {
 	},
 };
 
-/* NHPoly1305 test vectors from https://github.com/google/adiantum */
-static const struct hash_testvec nhpoly1305_tv_template[] = {
-	{
-		.key	= "\xd2\x5d\x4c\xdd\x8d\x2b\x7f\x7a"
-			  "\xd9\xbe\x71\xec\xd1\x83\x52\xe3"
-			  "\xe1\xad\xd7\x5c\x0a\x75\x9d\xec"
-			  "\x1d\x13\x7e\x5d\x71\x07\xc9\xe4"
-			  "\x57\x2d\x44\x68\xcf\xd8\xd6\xc5"
-			  "\x39\x69\x7d\x32\x75\x51\x4f\x7e"
-			  "\xb2\x4c\xc6\x90\x51\x6e\xd9\xd6"
-			  "\xa5\x8b\x2d\xf1\x94\xf9\xf7\x5e"
-			  "\x2c\x84\x7b\x41\x0f\x88\x50\x89"
-			  "\x30\xd9\xa1\x38\x46\x6c\xc0\x4f"
-			  "\xe8\xdf\xdc\x66\xab\x24\x43\x41"
-			  "\x91\x55\x29\x65\x86\x28\x5e\x45"
-			  "\xd5\x2d\xb7\x80\x08\x9a\xc3\xd4"
-			  "\x9a\x77\x0a\xd4\xef\x3e\xe6\x3f"
-			  "\x6f\x2f\x9b\x3a\x7d\x12\x1e\x80"
-			  "\x6c\x44\xa2\x25\xe1\xf6\x60\xe9"
-			  "\x0d\xaf\xc5\x3c\xa5\x79\xae\x64"
-			  "\xbc\xa0\x39\xa3\x4d\x10\xe5\x4d"
-			  "\xd5\xe7\x89\x7a\x13\xee\x06\x78"
-			  "\xdc\xa4\xdc\x14\x27\xe6\x49\x38"
-			  "\xd0\xe0\x45\x25\x36\xc5\xf4\x79"
-			  "\x2e\x9a\x98\x04\xe4\x2b\x46\x52"
-			  "\x7c\x33\xca\xe2\x56\x51\x50\xe2"
-			  "\xa5\x9a\xae\x18\x6a\x13\xf8\xd2"
-			  "\x21\x31\x66\x02\xe2\xda\x8d\x7e"
-			  "\x41\x19\xb2\x61\xee\x48\x8f\xf1"
-			  "\x65\x24\x2e\x1e\x68\xce\x05\xd9"
-			  "\x2a\xcf\xa5\x3a\x57\xdd\x35\x91"
-			  "\x93\x01\xca\x95\xfc\x2b\x36\x04"
-			  "\xe6\x96\x97\x28\xf6\x31\xfe\xa3"
-			  "\x9d\xf6\x6a\x1e\x80\x8d\xdc\xec"
-			  "\xaf\x66\x11\x13\x02\x88\xd5\x27"
-			  "\x33\xb4\x1a\xcd\xa3\xf6\xde\x31"
-			  "\x8e\xc0\x0e\x6c\xd8\x5a\x97\x5e"
-			  "\xdd\xfd\x60\x69\x38\x46\x3f\x90"
-			  "\x5e\x97\xd3\x32\x76\xc7\x82\x49"
-			  "\xfe\xba\x06\x5f\x2f\xa2\xfd\xff"
-			  "\x80\x05\x40\xe4\x33\x03\xfb\x10"
-			  "\xc0\xde\x65\x8c\xc9\x8d\x3a\x9d"
-			  "\xb5\x7b\x36\x4b\xb5\x0c\xcf\x00"
-			  "\x9c\x87\xe4\x49\xad\x90\xda\x4a"
-			  "\xdd\xbd\xff\xe2\x32\x57\xd6\x78"
-			  "\x36\x39\x6c\xd3\x5b\x9b\x88\x59"
-			  "\x2d\xf0\x46\xe4\x13\x0e\x2b\x35"
-			  "\x0d\x0f\x73\x8a\x4f\x26\x84\x75"
-			  "\x88\x3c\xc5\x58\x66\x18\x1a\xb4"
-			  "\x64\x51\x34\x27\x1b\xa4\x11\xc9"
-			  "\x6d\x91\x8a\xfa\x32\x60\x9d\xd7"
-			  "\x87\xe5\xaa\x43\x72\xf8\xda\xd1"
-			  "\x48\x44\x13\x61\xdc\x8c\x76\x17"
-			  "\x0c\x85\x4e\xf3\xdd\xa2\x42\xd2"
-			  "\x74\xc1\x30\x1b\xeb\x35\x31\x29"
-			  "\x5b\xd7\x4c\x94\x46\x35\xa1\x23"
-			  "\x50\xf2\xa2\x8e\x7e\x4f\x23\x4f"
-			  "\x51\xff\xe2\xc9\xa3\x7d\x56\x8b"
-			  "\x41\xf2\xd0\xc5\x57\x7e\x59\xac"
-			  "\xbb\x65\xf3\xfe\xf7\x17\xef\x63"
-			  "\x7c\x6f\x23\xdd\x22\x8e\xed\x84"
-			  "\x0e\x3b\x09\xb3\xf3\xf4\x8f\xcd"
-			  "\x37\xa8\xe1\xa7\x30\xdb\xb1\xa2"
-			  "\x9c\xa2\xdf\x34\x17\x3e\x68\x44"
-			  "\xd0\xde\x03\x50\xd1\x48\x6b\x20"
-			  "\xe2\x63\x45\xa5\xea\x87\xc2\x42"
-			  "\x95\x03\x49\x05\xed\xe0\x90\x29"
-			  "\x1a\xb8\xcf\x9b\x43\xcf\x29\x7a"
-			  "\x63\x17\x41\x9f\xe0\xc9\x10\xfd"
-			  "\x2c\x56\x8c\x08\x55\xb4\xa9\x27"
-			  "\x0f\x23\xb1\x05\x6a\x12\x46\xc7"
-			  "\xe1\xfe\x28\x93\x93\xd7\x2f\xdc"
-			  "\x98\x30\xdb\x75\x8a\xbe\x97\x7a"
-			  "\x02\xfb\x8c\xba\xbe\x25\x09\xbe"
-			  "\xce\xcb\xa2\xef\x79\x4d\x0e\x9d"
-			  "\x1b\x9d\xb6\x39\x34\x38\xfa\x07"
-			  "\xec\xe8\xfc\x32\x85\x1d\xf7\x85"
-			  "\x63\xc3\x3c\xc0\x02\x75\xd7\x3f"
-			  "\xb2\x68\x60\x66\x65\x81\xc6\xb1"
-			  "\x42\x65\x4b\x4b\x28\xd7\xc7\xaa"
-			  "\x9b\xd2\xdc\x1b\x01\xe0\x26\x39"
-			  "\x01\xc1\x52\x14\xd1\x3f\xb7\xe6"
-			  "\x61\x41\xc7\x93\xd2\xa2\x67\xc6"
-			  "\xf7\x11\xb5\xf5\xea\xdd\x19\xfb"
-			  "\x4d\x21\x12\xd6\x7d\xf1\x10\xb0"
-			  "\x89\x07\xc7\x5a\x52\x73\x70\x2f"
-			  "\x32\xef\x65\x2b\x12\xb2\xf0\xf5"
-			  "\x20\xe0\x90\x59\x7e\x64\xf1\x4c"
-			  "\x41\xb3\xa5\x91\x08\xe6\x5e\x5f"
-			  "\x05\x56\x76\xb4\xb0\xcd\x70\x53"
-			  "\x10\x48\x9c\xff\xc2\x69\x55\x24"
-			  "\x87\xef\x84\xea\xfb\xa7\xbf\xa0"
-			  "\x91\x04\xad\x4f\x8b\x57\x54\x4b"
-			  "\xb6\xe9\xd1\xac\x37\x2f\x1d\x2e"
-			  "\xab\xa5\xa4\xe8\xff\xfb\xd9\x39"
-			  "\x2f\xb7\xac\xd1\xfe\x0b\x9a\x80"
-			  "\x0f\xb6\xf4\x36\x39\x90\x51\xe3"
-			  "\x0a\x2f\xb6\x45\x76\x89\xcd\x61"
-			  "\xfe\x48\x5f\x75\x1d\x13\x00\x62"
-			  "\x80\x24\x47\xe7\xbc\x37\xd7\xe3"
-			  "\x15\xe8\x68\x22\xaf\x80\x6f\x4b"
-			  "\xa8\x9f\x01\x10\x48\x14\xc3\x02"
-			  "\x52\xd2\xc7\x75\x9b\x52\x6d\x30"
-			  "\xac\x13\x85\xc8\xf7\xa3\x58\x4b"
-			  "\x49\xf7\x1c\x45\x55\x8c\x39\x9a"
-			  "\x99\x6d\x97\x27\x27\xe6\xab\xdd"
-			  "\x2c\x42\x1b\x35\xdd\x9d\x73\xbb"
-			  "\x6c\xf3\x64\xf1\xfb\xb9\xf7\xe6"
-			  "\x4a\x3c\xc0\x92\xc0\x2e\xb7\x1a"
-			  "\xbe\xab\xb3\x5a\xe5\xea\xb1\x48"
-			  "\x58\x13\x53\x90\xfd\xc3\x8e\x54"
-			  "\xf9\x18\x16\x73\xe8\xcb\x6d\x39"
-			  "\x0e\xd7\xe0\xfe\xb6\x9f\x43\x97"
-			  "\xe8\xd0\x85\x56\x83\x3e\x98\x68"
-			  "\x7f\xbd\x95\xa8\x9a\x61\x21\x8f"
-			  "\x06\x98\x34\xa6\xc8\xd6\x1d\xf3"
-			  "\x3d\x43\xa4\x9a\x8c\xe5\xd3\x5a"
-			  "\x32\xa2\x04\x22\xa4\x19\x1a\x46"
-			  "\x42\x7e\x4d\xe5\xe0\xe6\x0e\xca"
-			  "\xd5\x58\x9d\x2c\xaf\xda\x33\x5c"
-			  "\xb0\x79\x9e\xc9\xfc\xca\xf0\x2f"
-			  "\xa8\xb2\x77\xeb\x7a\xa2\xdd\x37"
-			  "\x35\x83\x07\xd6\x02\x1a\xb6\x6c"
-			  "\x24\xe2\x59\x08\x0e\xfd\x3e\x46"
-			  "\xec\x40\x93\xf4\x00\x26\x4f\x2a"
-			  "\xff\x47\x2f\xeb\x02\x92\x26\x5b"
-			  "\x53\x17\xc2\x8d\x2a\xc7\xa3\x1b"
-			  "\xcd\xbc\xa7\xe8\xd1\x76\xe3\x80"
-			  "\x21\xca\x5d\x3b\xe4\x9c\x8f\xa9"
-			  "\x5b\x7f\x29\x7f\x7c\xd8\xed\x6d"
-			  "\x8c\xb2\x86\x85\xe7\x77\xf2\x85"
-			  "\xab\x38\xa9\x9d\xc1\x4e\xc5\x64"
-			  "\x33\x73\x8b\x59\x03\xad\x05\xdf"
-			  "\x25\x98\x31\xde\xef\x13\xf1\x9b"
-			  "\x3c\x91\x9d\x7b\xb1\xfa\xe6\xbf"
-			  "\x5b\xed\xa5\x55\xe6\xea\x6c\x74"
-			  "\xf4\xb9\xe4\x45\x64\x72\x81\xc2"
-			  "\x4c\x28\xd4\xcd\xac\xe2\xde\xf9"
-			  "\xeb\x5c\xeb\x61\x60\x5a\xe5\x28",
-		.ksize	= 1088,
-		.plaintext	= "",
-		.psize	= 0,
-		.digest	= "\x00\x00\x00\x00\x00\x00\x00\x00"
-			  "\x00\x00\x00\x00\x00\x00\x00\x00",
-	}, {
-		.key	= "\x29\x21\x43\xcb\xcb\x13\x07\xde"
-			  "\xbf\x48\xdf\x8a\x7f\xa2\x84\xde"
-			  "\x72\x23\x9d\xf5\xf0\x07\xf2\x4c"
-			  "\x20\x3a\x93\xb9\xcd\x5d\xfe\xcb"
-			  "\x99\x2c\x2b\x58\xc6\x50\x5f\x94"
-			  "\x56\xc3\x7c\x0d\x02\x3f\xb8\x5e"
-			  "\x7b\xc0\x6c\x51\x34\x76\xc0\x0e"
-			  "\xc6\x22\xc8\x9e\x92\xa0\x21\xc9"
-			  "\x85\x5c\x7c\xf8\xe2\x64\x47\xc9"
-			  "\xe4\xa2\x57\x93\xf8\xa2\x69\xcd"
-			  "\x62\x98\x99\xf4\xd7\x7b\x14\xb1"
-			  "\xd8\x05\xff\x04\x15\xc9\xe1\x6e"
-			  "\x9b\xe6\x50\x6b\x0b\x3f\x22\x1f"
-			  "\x08\xde\x0c\x5b\x08\x7e\xc6\x2f"
-			  "\x6c\xed\xd6\xb2\x15\xa4\xb3\xf9"
-			  "\xa7\x46\x38\x2a\xea\x69\xa5\xde"
-			  "\x02\xc3\x96\x89\x4d\x55\x3b\xed"
-			  "\x3d\x3a\x85\x77\xbf\x97\x45\x5c"
-			  "\x9e\x02\x69\xe2\x1b\x68\xbe\x96"
-			  "\xfb\x64\x6f\x0f\xf6\x06\x40\x67"
-			  "\xfa\x04\xe3\x55\xfa\xbe\xa4\x60"
-			  "\xef\x21\x66\x97\xe6\x9d\x5c\x1f"
-			  "\x62\x37\xaa\x31\xde\xe4\x9c\x28"
-			  "\x95\xe0\x22\x86\xf4\x4d\xf3\x07"
-			  "\xfd\x5f\x3a\x54\x2c\x51\x80\x71"
-			  "\xba\x78\x69\x5b\x65\xab\x1f\x81"
-			  "\xed\x3b\xff\x34\xa3\xfb\xbc\x73"
-			  "\x66\x7d\x13\x7f\xdf\x6e\xe2\xe2"
-			  "\xeb\x4f\x6c\xda\x7d\x33\x57\xd0"
-			  "\xd3\x7c\x95\x4f\x33\x58\x21\xc7"
-			  "\xc0\xe5\x6f\x42\x26\xc6\x1f\x5e"
-			  "\x85\x1b\x98\x9a\xa2\x1e\x55\x77"
-			  "\x23\xdf\x81\x5e\x79\x55\x05\xfc"
-			  "\xfb\xda\xee\xba\x5a\xba\xf7\x77"
-			  "\x7f\x0e\xd3\xe1\x37\xfe\x8d\x2b"
-			  "\xd5\x3f\xfb\xd0\xc0\x3c\x0b\x3f"
-			  "\xcf\x3c\x14\xcf\xfb\x46\x72\x4c"
-			  "\x1f\x39\xe2\xda\x03\x71\x6d\x23"
-			  "\xef\x93\xcd\x39\xd9\x37\x80\x4d"
-			  "\x65\x61\xd1\x2c\x03\xa9\x47\x72"
-			  "\x4d\x1e\x0e\x16\x33\x0f\x21\x17"
-			  "\xec\x92\xea\x6f\x37\x22\xa4\xd8"
-			  "\x03\x33\x9e\xd8\x03\x69\x9a\xe8"
-			  "\xb2\x57\xaf\x78\x99\x05\x12\xab"
-			  "\x48\x90\x80\xf0\x12\x9b\x20\x64"
-			  "\x7a\x1d\x47\x5f\xba\x3c\xf9\xc3"
-			  "\x0a\x0d\x8d\xa1\xf9\x1b\x82\x13"
-			  "\x3e\x0d\xec\x0a\x83\xc0\x65\xe1"
-			  "\xe9\x95\xff\x97\xd6\xf2\xe4\xd5"
-			  "\x86\xc0\x1f\x29\x27\x63\xd7\xde"
-			  "\xb7\x0a\x07\x99\x04\x2d\xa3\x89"
-			  "\xa2\x43\xcf\xf3\xe1\x43\xac\x4a"
-			  "\x06\x97\xd0\x05\x4f\x87\xfa\xf9"
-			  "\x9b\xbf\x52\x70\xbd\xbc\x6c\xf3"
-			  "\x03\x13\x60\x41\x28\x09\xec\xcc"
-			  "\xb1\x1a\xec\xd6\xfb\x6f\x2a\x89"
-			  "\x5d\x0b\x53\x9c\x59\xc1\x84\x21"
-			  "\x33\x51\x47\x19\x31\x9c\xd4\x0a"
-			  "\x4d\x04\xec\x50\x90\x61\xbd\xbc"
-			  "\x7e\xc8\xd9\x6c\x98\x1d\x45\x41"
-			  "\x17\x5e\x97\x1c\xc5\xa8\xe8\xea"
-			  "\x46\x58\x53\xf7\x17\xd5\xad\x11"
-			  "\xc8\x54\xf5\x7a\x33\x90\xf5\x19"
-			  "\xba\x36\xb4\xfc\x52\xa5\x72\x3d"
-			  "\x14\xbb\x55\xa7\xe9\xe3\x12\xf7"
-			  "\x1c\x30\xa2\x82\x03\xbf\x53\x91"
-			  "\x2e\x60\x41\x9f\x5b\x69\x39\xf6"
-			  "\x4d\xc8\xf8\x46\x7a\x7f\xa4\x98"
-			  "\x36\xff\x06\xcb\xca\xe7\x33\xf2"
-			  "\xc0\x4a\xf4\x3c\x14\x44\x5f\x6b"
-			  "\x75\xef\x02\x36\x75\x08\x14\xfd"
-			  "\x10\x8e\xa5\x58\xd0\x30\x46\x49"
-			  "\xaf\x3a\xf8\x40\x3d\x35\xdb\x84"
-			  "\x11\x2e\x97\x6a\xb7\x87\x7f\xad"
-			  "\xf1\xfa\xa5\x63\x60\xd8\x5e\xbf"
-			  "\x41\x78\x49\xcf\x77\xbb\x56\xbb"
-			  "\x7d\x01\x67\x05\x22\xc8\x8f\x41"
-			  "\xba\x81\xd2\xca\x2c\x38\xac\x76"
-			  "\x06\xc1\x1a\xc2\xce\xac\x90\x67"
-			  "\x57\x3e\x20\x12\x5b\xd9\x97\x58"
-			  "\x65\x05\xb7\x04\x61\x7e\xd8\x3a"
-			  "\xbf\x55\x3b\x13\xe9\x34\x5a\x37"
-			  "\x36\xcb\x94\x45\xc5\x32\xb3\xa0"
-			  "\x0c\x3e\x49\xc5\xd3\xed\xa7\xf0"
-			  "\x1c\x69\xcc\xea\xcc\x83\xc9\x16"
-			  "\x95\x72\x4b\xf4\x89\xd5\xb9\x10"
-			  "\xf6\x2d\x60\x15\xea\x3c\x06\x66"
-			  "\x9f\x82\xad\x17\xce\xd2\xa4\x48"
-			  "\x7c\x65\xd9\xf8\x02\x4d\x9b\x4c"
-			  "\x89\x06\x3a\x34\x85\x48\x89\x86"
-			  "\xf9\x24\xa9\x54\x72\xdb\x44\x95"
-			  "\xc7\x44\x1c\x19\x11\x4c\x04\xdc"
-			  "\x13\xb9\x67\xc8\xc3\x3a\x6a\x50"
-			  "\xfa\xd1\xfb\xe1\x88\xb6\xf1\xa3"
-			  "\xc5\x3b\xdc\x38\x45\x16\x26\x02"
-			  "\x3b\xb8\x8f\x8b\x58\x7d\x23\x04"
-			  "\x50\x6b\x81\x9f\xae\x66\xac\x6f"
-			  "\xcf\x2a\x9d\xf1\xfd\x1d\x57\x07"
-			  "\xbe\x58\xeb\x77\x0c\xe3\xc2\x19"
-			  "\x14\x74\x1b\x51\x1c\x4f\x41\xf3"
-			  "\x32\x89\xb3\xe7\xde\x62\xf6\x5f"
-			  "\xc7\x6a\x4a\x2a\x5b\x0f\x5f\x87"
-			  "\x9c\x08\xb9\x02\x88\xc8\x29\xb7"
-			  "\x94\x52\xfa\x52\xfe\xaa\x50\x10"
-			  "\xba\x48\x75\x5e\x11\x1b\xe6\x39"
-			  "\xd7\x82\x2c\x87\xf1\x1e\xa4\x38"
-			  "\x72\x3e\x51\xe7\xd8\x3e\x5b\x7b"
-			  "\x31\x16\x89\xba\xd6\xad\x18\x5e"
-			  "\xba\xf8\x12\xb3\xf4\x6c\x47\x30"
-			  "\xc0\x38\x58\xb3\x10\x8d\x58\x5d"
-			  "\xb4\xfb\x19\x7e\x41\xc3\x66\xb8"
-			  "\xd6\x72\x84\xe1\x1a\xc2\x71\x4c"
-			  "\x0d\x4a\x21\x7a\xab\xa2\xc0\x36"
-			  "\x15\xc5\xe9\x46\xd7\x29\x17\x76"
-			  "\x5e\x47\x36\x7f\x72\x05\xa7\xcc"
-			  "\x36\x63\xf9\x47\x7d\xe6\x07\x3c"
-			  "\x8b\x79\x1d\x96\x61\x8d\x90\x65"
-			  "\x7c\xf5\xeb\x4e\x6e\x09\x59\x6d"
-			  "\x62\x50\x1b\x0f\xe0\xdc\x78\xf2"
-			  "\x5b\x83\x1a\xa1\x11\x75\xfd\x18"
-			  "\xd7\xe2\x8d\x65\x14\x21\xce\xbe"
-			  "\xb5\x87\xe3\x0a\xda\x24\x0a\x64"
-			  "\xa9\x9f\x03\x8d\x46\x5d\x24\x1a"
-			  "\x8a\x0c\x42\x01\xca\xb1\x5f\x7c"
-			  "\xa5\xac\x32\x4a\xb8\x07\x91\x18"
-			  "\x6f\xb0\x71\x3c\xc9\xb1\xa8\xf8"
-			  "\x5f\x69\xa5\xa1\xca\x9e\x7a\xaa"
-			  "\xac\xe9\xc7\x47\x41\x75\x25\xc3"
-			  "\x73\xe2\x0b\xdd\x6d\x52\x71\xbe"
-			  "\xc5\xdc\xb4\xe7\x01\x26\x53\x77"
-			  "\x86\x90\x85\x68\x6b\x7b\x03\x53"
-			  "\xda\x52\x52\x51\x68\xc8\xf3\xec"
-			  "\x6c\xd5\x03\x7a\xa3\x0e\xb4\x02"
-			  "\x5f\x1a\xab\xee\xca\x67\x29\x7b"
-			  "\xbd\x96\x59\xb3\x8b\x32\x7a\x92"
-			  "\x9f\xd8\x25\x2b\xdf\xc0\x4c\xda",
-		.ksize	= 1088,
-		.plaintext	= "\xbc\xda\x81\xa8\x78\x79\x1c\xbf"
-			  "\x77\x53\xba\x4c\x30\x5b\xb8\x33",
-		.psize	= 16,
-		.digest	= "\x04\xbf\x7f\x6a\xce\x72\xea\x6a"
-			  "\x79\xdb\xb0\xc9\x60\xf6\x12\xcc",
-	}, {
-		.key	= "\x2e\x77\x1e\x2c\x63\x76\x34\x3f"
-			  "\x71\x08\x4f\x5a\xe3\x3d\x74\x56"
-			  "\xc7\x98\x46\x52\xe5\x8a\xba\x0d"
-			  "\x72\x41\x11\x15\x14\x72\x50\x8a"
-			  "\xd5\xec\x60\x09\xdd\x71\xcc\xb9"
-			  "\x59\x81\x65\x2d\x9e\x50\x18\xf3"
-			  "\x32\xf3\xf1\xe7\x01\x82\x1c\xad"
-			  "\x88\xa0\x21\x0c\x4b\x80\x5e\x62"
-			  "\xfc\x81\xec\x52\xaa\xe4\xa5\x86"
-			  "\xc2\xe6\x03\x11\xdc\x66\x09\x86"
-			  "\x3c\x3b\xf0\x59\x0f\xb3\xf7\x44"
-			  "\x24\xb7\x88\xc5\xfc\xc8\x77\x9f"
-			  "\x8c\x44\xc4\x11\x55\xce\x7a\xa3"
-			  "\xe0\xa2\xb8\xbf\xb5\x3d\x07\x2c"
-			  "\x32\xb6\x6c\xfc\xb4\x42\x95\x95"
-			  "\x98\x32\x81\xc4\xe7\xe2\xd9\x6a"
-			  "\x87\xf4\xf4\x1e\x74\x7c\xb5\xcd"
-			  "\x51\x45\x68\x38\x51\xdb\x30\x74"
-			  "\x11\xe0\xaa\xae\x19\x8f\x15\x55"
-			  "\xdd\x47\x4a\x35\xb9\x0c\xb4\x4e"
-			  "\xa9\xce\x2f\xfa\x8f\xc1\x8a\x5e"
-			  "\x5b\xec\xa5\x81\x3b\xb3\x43\x06"
-			  "\x24\x81\xf4\x24\xe2\x21\xfa\xcb"
-			  "\x49\xa8\xf8\xbd\x31\x4a\x5b\x2d"
-			  "\x64\x0a\x07\xf0\x80\xc9\x0d\x81"
-			  "\x14\x58\x54\x2b\xba\x22\x31\xba"
-			  "\xef\x66\xc9\x49\x69\x69\x83\x0d"
-			  "\xf2\xf9\x80\x9d\x30\x36\xfb\xe3"
-			  "\xc0\x72\x2b\xcc\x5a\x81\x2c\x5d"
-			  "\x3b\x5e\xf8\x2b\xd3\x14\x28\x73"
-			  "\xf9\x1c\x70\xe6\xd8\xbb\xac\x30"
-			  "\xf9\xd9\xa0\xe2\x33\x7c\x33\x34"
-			  "\xa5\x6a\x77\x6d\xd5\xaf\xf4\xf3"
-			  "\xc7\xb3\x0e\x83\x3d\xcb\x01\xcc"
-			  "\x81\xc0\xf9\x4a\xae\x36\x92\xf7"
-			  "\x69\x7b\x65\x01\xc3\xc8\xb8\xae"
-			  "\x16\xd8\x30\xbb\xba\x6d\x78\x6e"
-			  "\x0d\xf0\x7d\x84\xb7\x87\xda\x28"
-			  "\x7a\x18\x10\x0b\x29\xec\x29\xf3"
-			  "\xb0\x7b\xa1\x28\xbf\xbc\x2b\x2c"
-			  "\x92\x2c\x16\xfb\x02\x39\xf9\xa6"
-			  "\xa2\x15\x05\xa6\x72\x10\xbc\x62"
-			  "\x4a\x6e\xb8\xb5\x5d\x59\xae\x3c"
-			  "\x32\xd3\x68\xd7\x8e\x5a\xcd\x1b"
-			  "\xef\xf6\xa7\x5e\x10\x51\x15\x4b"
-			  "\x2c\xe3\xba\x70\x4f\x2c\xa0\x1c"
-			  "\x7b\x97\xd7\xb2\xa5\x05\x17\xcc"
-			  "\xf7\x3a\x29\x6f\xd5\x4b\xb8\x24"
-			  "\xf4\x65\x95\x12\xc0\x86\xd1\x64"
-			  "\x81\xdf\x46\x55\x0d\x22\x06\x77"
-			  "\xd8\xca\x8d\xc8\x87\xc3\xfa\xb9"
-			  "\xe1\x98\x94\xe6\x7b\xed\x65\x66"
-			  "\x0e\xc7\x25\x15\xee\x4a\xe6\x7e"
-			  "\xea\x1b\x58\xee\x96\xa0\x75\x9a"
-			  "\xa3\x00\x9e\x42\xc2\x26\x20\x8c"
-			  "\x3d\x22\x1f\x94\x3e\x74\x43\x72"
-			  "\xe9\x1d\xa6\xa1\x6c\xa7\xb8\x03"
-			  "\xdf\xb9\x7a\xaf\xe9\xe9\x3b\xfe"
-			  "\xdf\x91\xc1\x01\xa8\xba\x5d\x29"
-			  "\xa5\xe0\x98\x9b\x13\xe5\x13\x11"
-			  "\x7c\x04\x3a\xe8\x44\x7e\x78\xfc"
-			  "\xd6\x96\xa8\xbc\x7d\xc1\x89\x3d"
-			  "\x75\x64\xa9\x0e\x86\x33\xfb\x73"
-			  "\xf7\x15\xbc\x2c\x9a\x3f\x29\xce"
-			  "\x1c\x9d\x10\x4e\x85\xe1\x77\x41"
-			  "\x01\xe2\xbc\x88\xec\x81\xef\xc2"
-			  "\x6a\xed\x4f\xf7\xdf\xac\x10\x71"
-			  "\x94\xed\x71\xa4\x01\xd4\xd6\xbe"
-			  "\xfe\x3e\xc3\x92\x6a\xf2\x2b\xb5"
-			  "\xab\x15\x96\xb7\x88\x2c\xc2\xe1"
-			  "\xb0\x04\x22\xe7\x3d\xa9\xc9\x7d"
-			  "\x2c\x7c\x21\xff\x97\x86\x6b\x0c"
-			  "\x2b\x5b\xe0\xb6\x48\x74\x8f\x24"
-			  "\xef\x8e\xdd\x0f\x2a\x5f\xff\x33"
-			  "\xf4\x8e\xc5\xeb\x9c\xd7\x2a\x45"
-			  "\xf3\x50\xf1\xc0\x91\x8f\xc7\xf9"
-			  "\x97\xc1\x3c\x9c\xf4\xed\x8a\x23"
-			  "\x61\x5b\x40\x1a\x09\xee\x23\xa8"
-			  "\x7c\x7a\x96\xe1\x31\x55\x3d\x12"
-			  "\x04\x1f\x21\x78\x72\xf0\x0f\xa5"
-			  "\x80\x58\x7c\x2f\x37\xb5\x67\x24"
-			  "\x2f\xce\xf9\xf6\x86\x9f\xb3\x34"
-			  "\x0c\xfe\x0a\xaf\x27\xe6\x5e\x0a"
-			  "\x21\x44\x68\xe1\x5d\x84\x25\xae"
-			  "\x2c\x5a\x94\x66\x9a\x3f\x0e\x5a"
-			  "\xd0\x60\x2a\xd5\x3a\x4e\x2f\x40"
-			  "\x87\xe9\x27\x3e\xee\x92\xe1\x07"
-			  "\x22\x43\x52\xed\x67\x49\x13\xdd"
-			  "\x68\xd7\x54\xc2\x76\x72\x7e\x75"
-			  "\xaf\x24\x98\x5c\xe8\x22\xaa\x35"
-			  "\x0f\x9a\x1c\x4c\x0b\x43\x68\x99"
-			  "\x45\xdd\xbf\x82\xa5\x6f\x0a\xef"
-			  "\x44\x90\x85\xe7\x57\x23\x22\x41"
-			  "\x2e\xda\x24\x28\x65\x7f\x96\x85"
-			  "\x9f\x4b\x0d\x43\xb9\xa8\xbd\x84"
-			  "\xad\x0b\x09\xcc\x2c\x4a\x0c\xec"
-			  "\x71\x58\xba\xf1\xfc\x49\x4c\xca"
-			  "\x5c\x5d\xb2\x77\x0c\x99\xae\x1c"
-			  "\xce\x70\x05\x5b\x73\x6b\x7c\x28"
-			  "\x3b\xeb\x21\x3f\xa3\x71\xe1\x6a"
-			  "\xf4\x87\xd0\xbf\x73\xaa\x0b\x0b"
-			  "\xed\x70\xb3\xd4\xa3\xca\x76\x3a"
-			  "\xdb\xfa\xd8\x08\x95\xec\xac\x59"
-			  "\xd0\x79\x90\xc2\x33\x7b\xcc\x28"
-			  "\x65\xb6\x5f\x92\xc4\xac\x23\x40"
-			  "\xd1\x20\x44\x1f\xd7\x29\xab\x46"
-			  "\x79\x32\xc6\x8f\x79\xe5\xaa\x2c"
-			  "\xa6\x76\x70\x3a\x9e\x46\x3f\x8c"
-			  "\x1a\x89\x32\x28\x61\x5c\xcf\x93"
-			  "\x1e\xde\x9e\x98\xbe\x06\x30\x23"
-			  "\xc4\x8b\xda\x1c\xd1\x67\x46\x93"
-			  "\x9d\x41\xa2\x8c\x03\x22\xbd\x55"
-			  "\x7e\x91\x51\x13\xdc\xcf\x5c\x1e"
-			  "\xcb\x5d\xfb\x14\x16\x1a\x44\x56"
-			  "\x27\x77\xfd\xed\x7d\xbd\xd1\x49"
-			  "\x7f\x0d\xc3\x59\x48\x6b\x3c\x02"
-			  "\x6b\xb5\xd0\x83\xd5\x81\x29\xe7"
-			  "\xe0\xc9\x36\x23\x8d\x41\x33\x77"
-			  "\xff\x5f\x54\xde\x4d\x3f\xd2\x4e"
-			  "\xb6\x4d\xdd\x85\xf8\x9b\x20\x7d"
-			  "\x39\x27\x68\x63\xd3\x8e\x61\x39"
-			  "\xfa\xe1\xc3\x04\x74\x27\x5a\x34"
-			  "\x7f\xec\x59\x2d\xc5\x6e\x54\x23"
-			  "\xf5\x7b\x4b\xbe\x58\x2b\xf2\x81"
-			  "\x93\x63\xcc\x13\xd9\x90\xbb\x6a"
-			  "\x41\x03\x8d\x95\xeb\xbb\x5d\x06"
-			  "\x38\x4c\x0e\xd6\xa9\x5b\x84\x97"
-			  "\x3e\x64\x72\xe9\x96\x07\x0f\x73"
-			  "\x6e\xc6\x3b\x32\xbe\xac\x13\x14"
-			  "\xd0\x0a\x17\x5f\xb9\x9c\x3e\x34"
-			  "\xd9\xec\xd6\x8f\x89\xbf\x1e\xd3"
-			  "\xda\x80\xb2\x29\xff\x28\x96\xb3"
-			  "\x46\x50\x5b\x15\x80\x97\xee\x1f"
-			  "\x6c\xd8\xe8\xe0\xbd\x09\xe7\x20"
-			  "\x8c\x23\x8e\xd9\xbb\x92\xfa\x82"
-			  "\xaa\x0f\xb5\xf8\x78\x60\x11\xf0",
-		.ksize	= 1088,
-		.plaintext	= "\x0b\xb2\x31\x2d\xad\xfe\xce\xf9"
-			  "\xec\x5d\x3d\x64\x5f\x3f\x75\x43"
-			  "\x05\x5b\x97",
-		.psize	= 19,
-		.digest	= "\x5f\x02\xae\x65\x6c\x13\x21\x67"
-			  "\x77\x9e\xc4\x43\x58\x68\xde\x8f",
-	}, {
-		.key	= "\x65\x4d\xe3\xf8\xd2\x4c\xac\x28"
-			  "\x68\xf5\xb3\x81\x71\x4b\xa1\xfa"
-			  "\x04\x0e\xd3\x81\x36\xbe\x0c\x81"
-			  "\x5e\xaf\xbc\x3a\xa4\xc0\x8e\x8b"
-			  "\x55\x63\xd3\x52\x97\x88\xd6\x19"
-			  "\xbc\x96\xdf\x49\xff\x04\x63\xf5"
-			  "\x0c\x11\x13\xaa\x9e\x1f\x5a\xf7"
-			  "\xdd\xbd\x37\x80\xc3\xd0\xbe\xa7"
-			  "\x05\xc8\x3c\x98\x1e\x05\x3c\x84"
-			  "\x39\x61\xc4\xed\xed\x71\x1b\xc4"
-			  "\x74\x45\x2c\xa1\x56\x70\x97\xfd"
-			  "\x44\x18\x07\x7d\xca\x60\x1f\x73"
-			  "\x3b\x6d\x21\xcb\x61\x87\x70\x25"
-			  "\x46\x21\xf1\x1f\x21\x91\x31\x2d"
-			  "\x5d\xcc\xb7\xd1\x84\x3e\x3d\xdb"
-			  "\x03\x53\x2a\x82\xa6\x9a\x95\xbc"
-			  "\x1a\x1e\x0a\x5e\x07\x43\xab\x43"
-			  "\xaf\x92\x82\x06\x91\x04\x09\xf4"
-			  "\x17\x0a\x9a\x2c\x54\xdb\xb8\xf4"
-			  "\xd0\xf0\x10\x66\x24\x8d\xcd\xda"
-			  "\xfe\x0e\x45\x9d\x6f\xc4\x4e\xf4"
-			  "\x96\xaf\x13\xdc\xa9\xd4\x8c\xc4"
-			  "\xc8\x57\x39\x3c\xc2\xd3\x0a\x76"
-			  "\x4a\x1f\x75\x83\x44\xc7\xd1\x39"
-			  "\xd8\xb5\x41\xba\x73\x87\xfa\x96"
-			  "\xc7\x18\x53\xfb\x9b\xda\xa0\x97"
-			  "\x1d\xee\x60\x85\x9e\x14\xc3\xce"
-			  "\xc4\x05\x29\x3b\x95\x30\xa3\xd1"
-			  "\x9f\x82\x6a\x04\xf5\xa7\x75\x57"
-			  "\x82\x04\xfe\x71\x51\x71\xb1\x49"
-			  "\x50\xf8\xe0\x96\xf1\xfa\xa8\x88"
-			  "\x3f\xa0\x86\x20\xd4\x60\x79\x59"
-			  "\x17\x2d\xd1\x09\xf4\xec\x05\x57"
-			  "\xcf\x62\x7e\x0e\x7e\x60\x78\xe6"
-			  "\x08\x60\x29\xd8\xd5\x08\x1a\x24"
-			  "\xc4\x6c\x24\xe7\x92\x08\x3d\x8a"
-			  "\x98\x7a\xcf\x99\x0a\x65\x0e\xdc"
-			  "\x8c\x8a\xbe\x92\x82\x91\xcc\x62"
-			  "\x30\xb6\xf4\x3f\xc6\x8a\x7f\x12"
-			  "\x4a\x8a\x49\xfa\x3f\x5c\xd4\x5a"
-			  "\xa6\x82\xa3\xe6\xaa\x34\x76\xb2"
-			  "\xab\x0a\x30\xef\x6c\x77\x58\x3f"
-			  "\x05\x6b\xcc\x5c\xae\xdc\xd7\xb9"
-			  "\x51\x7e\x8d\x32\x5b\x24\x25\xbe"
-			  "\x2b\x24\x01\xcf\x80\xda\x16\xd8"
-			  "\x90\x72\x2c\xad\x34\x8d\x0c\x74"
-			  "\x02\xcb\xfd\xcf\x6e\xef\x97\xb5"
-			  "\x4c\xf2\x68\xca\xde\x43\x9e\x8a"
-			  "\xc5\x5f\x31\x7f\x14\x71\x38\xec"
-			  "\xbd\x98\xe5\x71\xc4\xb5\xdb\xef"
-			  "\x59\xd2\xca\xc0\xc1\x86\x75\x01"
-			  "\xd4\x15\x0d\x6f\xa4\xf7\x7b\x37"
-			  "\x47\xda\x18\x93\x63\xda\xbe\x9e"
-			  "\x07\xfb\xb2\x83\xd5\xc4\x34\x55"
-			  "\xee\x73\xa1\x42\x96\xf9\x66\x41"
-			  "\xa4\xcc\xd2\x93\x6e\xe1\x0a\xbb"
-			  "\xd2\xdd\x18\x23\xe6\x6b\x98\x0b"
-			  "\x8a\x83\x59\x2c\xc3\xa6\x59\x5b"
-			  "\x01\x22\x59\xf7\xdc\xb0\x87\x7e"
-			  "\xdb\x7d\xf4\x71\x41\xab\xbd\xee"
-			  "\x79\xbe\x3c\x01\x76\x0b\x2d\x0a"
-			  "\x42\xc9\x77\x8c\xbb\x54\x95\x60"
-			  "\x43\x2e\xe0\x17\x52\xbd\x90\xc9"
-			  "\xc2\x2c\xdd\x90\x24\x22\x76\x40"
-			  "\x5c\xb9\x41\xc9\xa1\xd5\xbd\xe3"
-			  "\x44\xe0\xa4\xab\xcc\xb8\xe2\x32"
-			  "\x02\x15\x04\x1f\x8c\xec\x5d\x14"
-			  "\xac\x18\xaa\xef\x6e\x33\x19\x6e"
-			  "\xde\xfe\x19\xdb\xeb\x61\xca\x18"
-			  "\xad\xd8\x3d\xbf\x09\x11\xc7\xa5"
-			  "\x86\x0b\x0f\xe5\x3e\xde\xe8\xd9"
-			  "\x0a\x69\x9e\x4c\x20\xff\xf9\xc5"
-			  "\xfa\xf8\xf3\x7f\xa5\x01\x4b\x5e"
-			  "\x0f\xf0\x3b\x68\xf0\x46\x8c\x2a"
-			  "\x7a\xc1\x8f\xa0\xfe\x6a\x5b\x44"
-			  "\x70\x5c\xcc\x92\x2c\x6f\x0f\xbd"
-			  "\x25\x3e\xb7\x8e\x73\x58\xda\xc9"
-			  "\xa5\xaa\x9e\xf3\x9b\xfd\x37\x3e"
-			  "\xe2\x88\xa4\x7b\xc8\x5c\xa8\x93"
-			  "\x0e\xe7\x9a\x9c\x2e\x95\x18\x9f"
-			  "\xc8\x45\x0c\x88\x9e\x53\x4f\x3a"
-			  "\x76\xc1\x35\xfa\x17\xd8\xac\xa0"
-			  "\x0c\x2d\x47\x2e\x4f\x69\x9b\xf7"
-			  "\xd0\xb6\x96\x0c\x19\xb3\x08\x01"
-			  "\x65\x7a\x1f\xc7\x31\x86\xdb\xc8"
-			  "\xc1\x99\x8f\xf8\x08\x4a\x9d\x23"
-			  "\x22\xa8\xcf\x27\x01\x01\x88\x93"
-			  "\x9c\x86\x45\xbd\xe0\x51\xca\x52"
-			  "\x84\xba\xfe\x03\xf7\xda\xc5\xce"
-			  "\x3e\x77\x75\x86\xaf\x84\xc8\x05"
-			  "\x44\x01\x0f\x02\xf3\x58\xb0\x06"
-			  "\x5a\xd7\x12\x30\x8d\xdf\x1f\x1f"
-			  "\x0a\xe6\xd2\xea\xf6\x3a\x7a\x99"
-			  "\x63\xe8\xd2\xc1\x4a\x45\x8b\x40"
-			  "\x4d\x0a\xa9\x76\x92\xb3\xda\x87"
-			  "\x36\x33\xf0\x78\xc3\x2f\x5f\x02"
-			  "\x1a\x6a\x2c\x32\xcd\x76\xbf\xbd"
-			  "\x5a\x26\x20\x28\x8c\x8c\xbc\x52"
-			  "\x3d\x0a\xc9\xcb\xab\xa4\x21\xb0"
-			  "\x54\x40\x81\x44\xc7\xd6\x1c\x11"
-			  "\x44\xc6\x02\x92\x14\x5a\xbf\x1a"
-			  "\x09\x8a\x18\xad\xcd\x64\x3d\x53"
-			  "\x4a\xb6\xa5\x1b\x57\x0e\xef\xe0"
-			  "\x8c\x44\x5f\x7d\xbd\x6c\xfd\x60"
-			  "\xae\x02\x24\xb6\x99\xdd\x8c\xaf"
-			  "\x59\x39\x75\x3c\xd1\x54\x7b\x86"
-			  "\xcc\x99\xd9\x28\x0c\xb0\x94\x62"
-			  "\xf9\x51\xd1\x19\x96\x2d\x66\xf5"
-			  "\x55\xcf\x9e\x59\xe2\x6b\x2c\x08"
-			  "\xc0\x54\x48\x24\x45\xc3\x8c\x73"
-			  "\xea\x27\x6e\x66\x7d\x1d\x0e\x6e"
-			  "\x13\xe8\x56\x65\x3a\xb0\x81\x5c"
-			  "\xf0\xe8\xd8\x00\x6b\xcd\x8f\xad"
-			  "\xdd\x53\xf3\xa4\x6c\x43\xd6\x31"
-			  "\xaf\xd2\x76\x1e\x91\x12\xdb\x3c"
-			  "\x8c\xc2\x81\xf0\x49\xdb\xe2\x6b"
-			  "\x76\x62\x0a\x04\xe4\xaa\x8a\x7c"
-			  "\x08\x0b\x5d\xd0\xee\x1d\xfb\xc4"
-			  "\x02\x75\x42\xd6\xba\xa7\x22\xa8"
-			  "\x47\x29\xb7\x85\x6d\x93\x3a\xdb"
-			  "\x00\x53\x0b\xa2\xeb\xf8\xfe\x01"
-			  "\x6f\x8a\x31\xd6\x17\x05\x6f\x67"
-			  "\x88\x95\x32\xfe\x4f\xa6\x4b\xf8"
-			  "\x03\xe4\xcd\x9a\x18\xe8\x4e\x2d"
-			  "\xf7\x97\x9a\x0c\x7d\x9f\x7e\x44"
-			  "\x69\x51\xe0\x32\x6b\x62\x86\x8f"
-			  "\xa6\x8e\x0b\x21\x96\xe5\xaf\x77"
-			  "\xc0\x83\xdf\xa5\x0e\xd0\xa1\x04"
-			  "\xaf\xc1\x10\xcb\x5a\x40\xe4\xe3"
-			  "\x38\x7e\x07\xe8\x4d\xfa\xed\xc5"
-			  "\xf0\x37\xdf\xbb\x8a\xcf\x3d\xdc"
-			  "\x61\xd2\xc6\x2b\xff\x07\xc9\x2f"
-			  "\x0c\x2d\x5c\x07\xa8\x35\x6a\xfc"
-			  "\xae\x09\x03\x45\x74\x51\x4d\xc4"
-			  "\xb8\x23\x87\x4a\x99\x27\x20\x87"
-			  "\x62\x44\x0a\x4a\xce\x78\x47\x22",
-		.ksize	= 1088,
-		.plaintext	= "\x8e\xb0\x4c\xde\x9c\x4a\x04\x5a"
-			  "\xf6\xa9\x7f\x45\x25\xa5\x7b\x3a"
-			  "\xbc\x4d\x73\x39\x81\xb5\xbd\x3d"
-			  "\x21\x6f\xd7\x37\x50\x3c\x7b\x28"
-			  "\xd1\x03\x3a\x17\xed\x7b\x7c\x2a"
-			  "\x16\xbc\xdf\x19\x89\x52\x71\x31"
-			  "\xb6\xc0\xfd\xb5\xd3\xba\x96\x99"
-			  "\xb6\x34\x0b\xd0\x99\x93\xfc\x1a"
-			  "\x01\x3c\x85\xc6\x9b\x78\x5c\x8b"
-			  "\xfe\xae\xd2\xbf\xb2\x6f\xf9\xed"
-			  "\xc8\x25\x17\xfe\x10\x3b\x7d\xda"
-			  "\xf4\x8d\x35\x4b\x7c\x7b\x82\xe7"
-			  "\xc2\xb3\xee\x60\x4a\x03\x86\xc9"
-			  "\x4e\xb5\xc4\xbe\xd2\xbd\x66\xf1"
-			  "\x13\xf1\x09\xab\x5d\xca\x63\x1f"
-			  "\xfc\xfb\x57\x2a\xfc\xca\x66\xd8"
-			  "\x77\x84\x38\x23\x1d\xac\xd3\xb3"
-			  "\x7a\xad\x4c\x70\xfa\x9c\xc9\x61"
-			  "\xa6\x1b\xba\x33\x4b\x4e\x33\xec"
-			  "\xa0\xa1\x64\x39\x40\x05\x1c\xc2"
-			  "\x3f\x49\x9d\xae\xf2\xc5\xf2\xc5"
-			  "\xfe\xe8\xf4\xc2\xf9\x96\x2d\x28"
-			  "\x92\x30\x44\xbc\xd2\x7f\xe1\x6e"
-			  "\x62\x02\x8f\x3d\x1c\x80\xda\x0e"
-			  "\x6a\x90\x7e\x75\xff\xec\x3e\xc4"
-			  "\xcd\x16\x34\x3b\x05\x6d\x4d\x20"
-			  "\x1c\x7b\xf5\x57\x4f\xfa\x3d\xac"
-			  "\xd0\x13\x55\xe8\xb3\xe1\x1b\x78"
-			  "\x30\xe6\x9f\x84\xd4\x69\xd1\x08"
-			  "\x12\x77\xa7\x4a\xbd\xc0\xf2\xd2"
-			  "\x78\xdd\xa3\x81\x12\xcb\x6c\x14"
-			  "\x90\x61\xe2\x84\xc6\x2b\x16\xcc"
-			  "\x40\x99\x50\x88\x01\x09\x64\x4f"
-			  "\x0a\x80\xbe\x61\xae\x46\xc9\x0a"
-			  "\x5d\xe0\xfb\x72\x7a\x1a\xdd\x61"
-			  "\x63\x20\x05\xa0\x4a\xf0\x60\x69"
-			  "\x7f\x92\xbc\xbf\x4e\x39\x4d\xdd"
-			  "\x74\xd1\xb7\xc0\x5a\x34\xb7\xae"
-			  "\x76\x65\x2e\xbc\x36\xb9\x04\x95"
-			  "\x42\xe9\x6f\xca\x78\xb3\x72\x07"
-			  "\xa3\xba\x02\x94\x67\x4c\xb1\xd7"
-			  "\xe9\x30\x0d\xf0\x3b\xb8\x10\x6d"
-			  "\xea\x2b\x21\xbf\x74\x59\x82\x97"
-			  "\x85\xaa\xf1\xd7\x54\x39\xeb\x05"
-			  "\xbd\xf3\x40\xa0\x97\xe6\x74\xfe"
-			  "\xb4\x82\x5b\xb1\x36\xcb\xe8\x0d"
-			  "\xce\x14\xd9\xdf\xf1\x94\x22\xcd"
-			  "\xd6\x00\xba\x04\x4c\x05\x0c\xc0"
-			  "\xd1\x5a\xeb\x52\xd5\xa8\x8e\xc8"
-			  "\x97\xa1\xaa\xc1\xea\xc1\xbe\x7c"
-			  "\x36\xb3\x36\xa0\xc6\x76\x66\xc5"
-			  "\xe2\xaf\xd6\x5c\xe2\xdb\x2c\xb3"
-			  "\x6c\xb9\x99\x7f\xff\x9f\x03\x24"
-			  "\xe1\x51\x44\x66\xd8\x0c\x5d\x7f"
-			  "\x5c\x85\x22\x2a\xcf\x6d\x79\x28"
-			  "\xab\x98\x01\x72\xfe\x80\x87\x5f"
-			  "\x46\xba\xef\x81\x24\xee\xbf\xb0"
-			  "\x24\x74\xa3\x65\x97\x12\xc4\xaf"
-			  "\x8b\xa0\x39\xda\x8a\x7e\x74\x6e"
-			  "\x1b\x42\xb4\x44\x37\xfc\x59\xfd"
-			  "\x86\xed\xfb\x8c\x66\x33\xda\x63"
-			  "\x75\xeb\xe1\xa4\x85\x4f\x50\x8f"
-			  "\x83\x66\x0d\xd3\x37\xfa\xe6\x9c"
-			  "\x4f\x30\x87\x35\x18\xe3\x0b\xb7"
-			  "\x6e\x64\x54\xcd\x70\xb3\xde\x54"
-			  "\xb7\x1d\xe6\x4c\x4d\x55\x12\x12"
-			  "\xaf\x5f\x7f\x5e\xee\x9d\xe8\x8e"
-			  "\x32\x9d\x4e\x75\xeb\xc6\xdd\xaa"
-			  "\x48\x82\xa4\x3f\x3c\xd7\xd3\xa8"
-			  "\x63\x9e\x64\xfe\xe3\x97\x00\x62"
-			  "\xe5\x40\x5d\xc3\xad\x72\xe1\x28"
-			  "\x18\x50\xb7\x75\xef\xcd\x23\xbf"
-			  "\x3f\xc0\x51\x36\xf8\x41\xc3\x08"
-			  "\xcb\xf1\x8d\x38\x34\xbd\x48\x45"
-			  "\x75\xed\xbc\x65\x7b\xb5\x0c\x9b"
-			  "\xd7\x67\x7d\x27\xb4\xc4\x80\xd7"
-			  "\xa9\xb9\xc7\x4a\x97\xaa\xda\xc8"
-			  "\x3c\x74\xcf\x36\x8f\xe4\x41\xe3"
-			  "\xd4\xd3\x26\xa7\xf3\x23\x9d\x8f"
-			  "\x6c\x20\x05\x32\x3e\xe0\xc3\xc8"
-			  "\x56\x3f\xa7\x09\xb7\xfb\xc7\xf7"
-			  "\xbe\x2a\xdd\x0f\x06\x7b\x0d\xdd"
-			  "\xb0\xb4\x86\x17\xfd\xb9\x04\xe5"
-			  "\xc0\x64\x5d\xad\x2a\x36\x38\xdb"
-			  "\x24\xaf\x5b\xff\xca\xf9\x41\xe8"
-			  "\xf9\x2f\x1e\x5e\xf9\xf5\xd5\xf2"
-			  "\xb2\x88\xca\xc9\xa1\x31\xe2\xe8"
-			  "\x10\x95\x65\xbf\xf1\x11\x61\x7a"
-			  "\x30\x1a\x54\x90\xea\xd2\x30\xf6"
-			  "\xa5\xad\x60\xf9\x4d\x84\x21\x1b"
-			  "\xe4\x42\x22\xc8\x12\x4b\xb0\x58"
-			  "\x3e\x9c\x2d\x32\x95\x0a\x8e\xb0"
-			  "\x0a\x7e\x77\x2f\xe8\x97\x31\x6a"
-			  "\xf5\x59\xb4\x26\xe6\x37\x12\xc9"
-			  "\xcb\xa0\x58\x33\x6f\xd5\x55\x55"
-			  "\x3c\xa1\x33\xb1\x0b\x7e\x2e\xb4"
-			  "\x43\x2a\x84\x39\xf0\x9c\xf4\x69"
-			  "\x4f\x1e\x79\xa6\x15\x1b\x87\xbb"
-			  "\xdb\x9b\xe0\xf1\x0b\xba\xe3\x6e"
-			  "\xcc\x2f\x49\x19\x22\x29\xfc\x71"
-			  "\xbb\x77\x38\x18\x61\xaf\x85\x76"
-			  "\xeb\xd1\x09\xcc\x86\x04\x20\x9a"
-			  "\x66\x53\x2f\x44\x8b\xc6\xa3\xd2"
-			  "\x5f\xc7\x79\x82\x66\xa8\x6e\x75"
-			  "\x7d\x94\xd1\x86\x75\x0f\xa5\x4f"
-			  "\x3c\x7a\x33\xce\xd1\x6e\x9d\x7b"
-			  "\x1f\x91\x37\xb8\x37\x80\xfb\xe0"
-			  "\x52\x26\xd0\x9a\xd4\x48\x02\x41"
-			  "\x05\xe3\x5a\x94\xf1\x65\x61\x19"
-			  "\xb8\x88\x4e\x2b\xea\xba\x8b\x58"
-			  "\x8b\x42\x01\x00\xa8\xfe\x00\x5c"
-			  "\xfe\x1c\xee\x31\x15\x69\xfa\xb3"
-			  "\x9b\x5f\x22\x8e\x0d\x2c\xe3\xa5"
-			  "\x21\xb9\x99\x8a\x8e\x94\x5a\xef"
-			  "\x13\x3e\x99\x96\x79\x6e\xd5\x42"
-			  "\x36\x03\xa9\xe2\xca\x65\x4e\x8a"
-			  "\x8a\x30\xd2\x7d\x74\xe7\xf0\xaa"
-			  "\x23\x26\xdd\xcb\x82\x39\xfc\x9d"
-			  "\x51\x76\x21\x80\xa2\xbe\x93\x03"
-			  "\x47\xb0\xc1\xb6\xdc\x63\xfd\x9f"
-			  "\xca\x9d\xa5\xca\x27\x85\xe2\xd8"
-			  "\x15\x5b\x7e\x14\x7a\xc4\x89\xcc"
-			  "\x74\x14\x4b\x46\xd2\xce\xac\x39"
-			  "\x6b\x6a\x5a\xa4\x0e\xe3\x7b\x15"
-			  "\x94\x4b\x0f\x74\xcb\x0c\x7f\xa9"
-			  "\xbe\x09\x39\xa3\xdd\x56\x5c\xc7"
-			  "\x99\x56\x65\x39\xf4\x0b\x7d\x87"
-			  "\xec\xaa\xe3\x4d\x22\x65\x39\x4e",
-		.psize	= 1024,
-		.digest	= "\x64\x3a\xbc\xc3\x3f\x74\x40\x51"
-			  "\x6e\x56\x01\x1a\x51\xec\x36\xde",
-	}, {
-		.key	= "\x1b\x82\x2e\x1b\x17\x23\xb9\x6d"
-			  "\xdc\x9c\xda\x99\x07\xe3\x5f\xd8"
-			  "\xd2\xf8\x43\x80\x8d\x86\x7d\x80"
-			  "\x1a\xd0\xcc\x13\xb9\x11\x05\x3f"
-			  "\x7e\xcf\x7e\x80\x0e\xd8\x25\x48"
-			  "\x8b\xaa\x63\x83\x92\xd0\x72\xf5"
-			  "\x4f\x67\x7e\x50\x18\x25\xa4\xd1"
-			  "\xe0\x7e\x1e\xba\xd8\xa7\x6e\xdb"
-			  "\x1a\xcc\x0d\xfe\x9f\x6d\x22\x35"
-			  "\xe1\xe6\xe0\xa8\x7b\x9c\xb1\x66"
-			  "\xa3\xf8\xff\x4d\x90\x84\x28\xbc"
-			  "\xdc\x19\xc7\x91\x49\xfc\xf6\x33"
-			  "\xc9\x6e\x65\x7f\x28\x6f\x68\x2e"
-			  "\xdf\x1a\x75\xe9\xc2\x0c\x96\xb9"
-			  "\x31\x22\xc4\x07\xc6\x0a\x2f\xfd"
-			  "\x36\x06\x5f\x5c\xc5\xb1\x3a\xf4"
-			  "\x5e\x48\xa4\x45\x2b\x88\xa7\xee"
-			  "\xa9\x8b\x52\xcc\x99\xd9\x2f\xb8"
-			  "\xa4\x58\x0a\x13\xeb\x71\x5a\xfa"
-			  "\xe5\x5e\xbe\xf2\x64\xad\x75\xbc"
-			  "\x0b\x5b\x34\x13\x3b\x23\x13\x9a"
-			  "\x69\x30\x1e\x9a\xb8\x03\xb8\x8b"
-			  "\x3e\x46\x18\x6d\x38\xd9\xb3\xd8"
-			  "\xbf\xf1\xd0\x28\xe6\x51\x57\x80"
-			  "\x5e\x99\xfb\xd0\xce\x1e\x83\xf7"
-			  "\xe9\x07\x5a\x63\xa9\xef\xce\xa5"
-			  "\xfb\x3f\x37\x17\xfc\x0b\x37\x0e"
-			  "\xbb\x4b\x21\x62\xb7\x83\x0e\xa9"
-			  "\x9e\xb0\xc4\xad\x47\xbe\x35\xe7"
-			  "\x51\xb2\xf2\xac\x2b\x65\x7b\x48"
-			  "\xe3\x3f\x5f\xb6\x09\x04\x0c\x58"
-			  "\xce\x99\xa9\x15\x2f\x4e\xc1\xf2"
-			  "\x24\x48\xc0\xd8\x6c\xd3\x76\x17"
-			  "\x83\x5d\xe6\xe3\xfd\x01\x8e\xf7"
-			  "\x42\xa5\x04\x29\x30\xdf\xf9\x00"
-			  "\x4a\xdc\x71\x22\x1a\x33\x15\xb6"
-			  "\xd7\x72\xfb\x9a\xb8\xeb\x2b\x38"
-			  "\xea\xa8\x61\xa8\x90\x11\x9d\x73"
-			  "\x2e\x6c\xce\x81\x54\x5a\x9f\xcd"
-			  "\xcf\xd5\xbd\x26\x5d\x66\xdb\xfb"
-			  "\xdc\x1e\x7c\x10\xfe\x58\x82\x10"
-			  "\x16\x24\x01\xce\x67\x55\x51\xd1"
-			  "\xdd\x6b\x44\xa3\x20\x8e\xa9\xa6"
-			  "\x06\xa8\x29\x77\x6e\x00\x38\x5b"
-			  "\xde\x4d\x58\xd8\x1f\x34\xdf\xf9"
-			  "\x2c\xac\x3e\xad\xfb\x92\x0d\x72"
-			  "\x39\xa4\xac\x44\x10\xc0\x43\xc4"
-			  "\xa4\x77\x3b\xfc\xc4\x0d\x37\xd3"
-			  "\x05\x84\xda\x53\x71\xf8\x80\xd3"
-			  "\x34\x44\xdb\x09\xb4\x2b\x8e\xe3"
-			  "\x00\x75\x50\x9e\x43\x22\x00\x0b"
-			  "\x7c\x70\xab\xd4\x41\xf1\x93\xcd"
-			  "\x25\x2d\x84\x74\xb5\xf2\x92\xcd"
-			  "\x0a\x28\xea\x9a\x49\x02\x96\xcb"
-			  "\x85\x9e\x2f\x33\x03\x86\x1d\xdc"
-			  "\x1d\x31\xd5\xfc\x9d\xaa\xc5\xe9"
-			  "\x9a\xc4\x57\xf5\x35\xed\xf4\x4b"
-			  "\x3d\x34\xc2\x29\x13\x86\x36\x42"
-			  "\x5d\xbf\x90\x86\x13\x77\xe5\xc3"
-			  "\x62\xb4\xfe\x0b\x70\x39\x35\x65"
-			  "\x02\xea\xf6\xce\x57\x0c\xbb\x74"
-			  "\x29\xe3\xfd\x60\x90\xfd\x10\x38"
-			  "\xd5\x4e\x86\xbd\x37\x70\xf0\x97"
-			  "\xa6\xab\x3b\x83\x64\x52\xca\x66"
-			  "\x2f\xf9\xa4\xca\x3a\x55\x6b\xb0"
-			  "\xe8\x3a\x34\xdb\x9e\x48\x50\x2f"
-			  "\x3b\xef\xfd\x08\x2d\x5f\xc1\x37"
-			  "\x5d\xbe\x73\xe4\xd8\xe9\xac\xca"
-			  "\x8a\xaa\x48\x7c\x5c\xf4\xa6\x96"
-			  "\x5f\xfa\x70\xa6\xb7\x8b\x50\xcb"
-			  "\xa6\xf5\xa9\xbd\x7b\x75\x4c\x22"
-			  "\x0b\x19\x40\x2e\xc9\x39\x39\x32"
-			  "\x83\x03\xa8\xa4\x98\xe6\x8e\x16"
-			  "\xb9\xde\x08\xc5\xfc\xbf\xad\x39"
-			  "\xa8\xc7\x93\x6c\x6f\x23\xaf\xc1"
-			  "\xab\xe1\xdf\xbb\x39\xae\x93\x29"
-			  "\x0e\x7d\x80\x8d\x3e\x65\xf3\xfd"
-			  "\x96\x06\x65\x90\xa1\x28\x64\x4b"
-			  "\x69\xf9\xa8\x84\x27\x50\xfc\x87"
-			  "\xf7\xbf\x55\x8e\x56\x13\x58\x7b"
-			  "\x85\xb4\x6a\x72\x0f\x40\xf1\x4f"
-			  "\x83\x81\x1f\x76\xde\x15\x64\x7a"
-			  "\x7a\x80\xe4\xc7\x5e\x63\x01\x91"
-			  "\xd7\x6b\xea\x0b\x9b\xa2\x99\x3b"
-			  "\x6c\x88\xd8\xfd\x59\x3c\x8d\x22"
-			  "\x86\x56\xbe\xab\xa1\x37\x08\x01"
-			  "\x50\x85\x69\x29\xee\x9f\xdf\x21"
-			  "\x3e\x20\x20\xf5\xb0\xbb\x6b\xd0"
-			  "\x9c\x41\x38\xec\x54\x6f\x2d\xbd"
-			  "\x0f\xe1\xbd\xf1\x2b\x6e\x60\x56"
-			  "\x29\xe5\x7a\x70\x1c\xe2\xfc\x97"
-			  "\x82\x68\x67\xd9\x3d\x1f\xfb\xd8"
-			  "\x07\x9f\xbf\x96\x74\xba\x6a\x0e"
-			  "\x10\x48\x20\xd8\x13\x1e\xb5\x44"
-			  "\xf2\xcc\xb1\x8b\xfb\xbb\xec\xd7"
-			  "\x37\x70\x1f\x7c\x55\xd2\x4b\xb9"
-			  "\xfd\x70\x5e\xa3\x91\x73\x63\x52"
-			  "\x13\x47\x5a\x06\xfb\x01\x67\xa5"
-			  "\xc0\xd0\x49\x19\x56\x66\x9a\x77"
-			  "\x64\xaf\x8c\x25\x91\x52\x87\x0e"
-			  "\x18\xf3\x5f\x97\xfd\x71\x13\xf8"
-			  "\x05\xa5\x39\xcc\x65\xd3\xcc\x63"
-			  "\x5b\xdb\x5f\x7e\x5f\x6e\xad\xc4"
-			  "\xf4\xa0\xc5\xc2\x2b\x4d\x97\x38"
-			  "\x4f\xbc\xfa\x33\x17\xb4\x47\xb9"
-			  "\x43\x24\x15\x8d\xd2\xed\x80\x68"
-			  "\x84\xdb\x04\x80\xca\x5e\x6a\x35"
-			  "\x2c\x2c\xe7\xc5\x03\x5f\x54\xb0"
-			  "\x5e\x4f\x1d\x40\x54\x3d\x78\x9a"
-			  "\xac\xda\x80\x27\x4d\x15\x4c\x1a"
-			  "\x6e\x80\xc9\xc4\x3b\x84\x0e\xd9"
-			  "\x2e\x93\x01\x8c\xc3\xc8\x91\x4b"
-			  "\xb3\xaa\x07\x04\x68\x5b\x93\xa5"
-			  "\xe7\xc4\x9d\xe7\x07\xee\xf5\x3b"
-			  "\x40\x89\xcc\x60\x34\x9d\xb4\x06"
-			  "\x1b\xef\x92\xe6\xc1\x2a\x7d\x0f"
-			  "\x81\xaa\x56\xe3\xd7\xed\xa7\xd4"
-			  "\xa7\x3a\x49\xc4\xad\x81\x5c\x83"
-			  "\x55\x8e\x91\x54\xb7\x7d\x65\xa5"
-			  "\x06\x16\xd5\x9a\x16\xc1\xb0\xa2"
-			  "\x06\xd8\x98\x47\x73\x7e\x73\xa0"
-			  "\xb8\x23\xb1\x52\xbf\x68\x74\x5d"
-			  "\x0b\xcb\xfa\x8c\x46\xe3\x24\xe6"
-			  "\xab\xd4\x69\x8d\x8c\xf2\x8a\x59"
-			  "\xbe\x48\x46\x50\x8c\x9a\xe8\xe3"
-			  "\x31\x55\x0a\x06\xed\x4f\xf8\xb7"
-			  "\x4f\xe3\x85\x17\x30\xbd\xd5\x20"
-			  "\xe7\x5b\xb2\x32\xcf\x6b\x16\x44"
-			  "\xd2\xf5\x7e\xd7\xd1\x2f\xee\x64"
-			  "\x3e\x9d\x10\xef\x27\x35\x43\x64"
-			  "\x67\xfb\x7a\x7b\xe0\x62\x31\x9a"
-			  "\x4d\xdf\xa5\xab\xc0\x20\xbb\x01"
-			  "\xe9\x7b\x54\xf1\xde\xb2\x79\x50"
-			  "\x6c\x4b\x91\xdb\x7f\xbb\x50\xc1"
-			  "\x55\x44\x38\x9a\xe0\x9f\xe8\x29"
-			  "\x6f\x15\xf8\x4e\xa6\xec\xa0\x60",
-		.ksize	= 1088,
-		.plaintext	= "\x15\x68\x9e\x2f\xad\x15\x52\xdf"
-			  "\xf0\x42\x62\x24\x2a\x2d\xea\xbf"
-			  "\xc7\xf3\xb4\x1a\xf5\xed\xb2\x08"
-			  "\x15\x60\x1c\x00\x77\xbf\x0b\x0e"
-			  "\xb7\x2c\xcf\x32\x3a\xc7\x01\x77"
-			  "\xef\xa6\x75\xd0\x29\xc7\x68\x20"
-			  "\xb2\x92\x25\xbf\x12\x34\xe9\xa4"
-			  "\xfd\x32\x7b\x3f\x7c\xbd\xa5\x02"
-			  "\x38\x41\xde\xc9\xc1\x09\xd9\xfc"
-			  "\x6e\x78\x22\x83\x18\xf7\x50\x8d"
-			  "\x8f\x9c\x2d\x02\xa5\x30\xac\xff"
-			  "\xea\x63\x2e\x80\x37\x83\xb0\x58"
-			  "\xda\x2f\xef\x21\x55\xba\x7b\xb1"
-			  "\xb6\xed\xf5\xd2\x4d\xaa\x8c\xa9"
-			  "\xdd\xdb\x0f\xb4\xce\xc1\x9a\xb1"
-			  "\xc1\xdc\xbd\xab\x86\xc2\xdf\x0b"
-			  "\xe1\x2c\xf9\xbe\xf6\xd8\xda\x62"
-			  "\x72\xdd\x98\x09\x52\xc0\xc4\xb6"
-			  "\x7b\x17\x5c\xf5\xd8\x4b\x88\xd6"
-			  "\x6b\xbf\x84\x4a\x3f\xf5\x4d\xd2"
-			  "\x94\xe2\x9c\xff\xc7\x3c\xd9\xc8"
-			  "\x37\x38\xbc\x8c\xf3\xe7\xb7\xd0"
-			  "\x1d\x78\xc4\x39\x07\xc8\x5e\x79"
-			  "\xb6\x5a\x90\x5b\x6e\x97\xc9\xd4"
-			  "\x82\x9c\xf3\x83\x7a\xe7\x97\xfc"
-			  "\x1d\xbb\xef\xdb\xce\xe0\x82\xad"
-			  "\xca\x07\x6c\x54\x62\x6f\x81\xe6"
-			  "\x7a\x5a\x96\x6e\x80\x3a\xa2\x37"
-			  "\x6f\xc6\xa4\x29\xc3\x9e\x19\x94"
-			  "\x9f\xb0\x3e\x38\xfb\x3c\x2b\x7d"
-			  "\xaa\xb8\x74\xda\x54\x23\x51\x12"
-			  "\x4b\x96\x36\x8f\x91\x4f\x19\x37"
-			  "\x83\xc9\xdd\xc7\x1a\x32\x2d\xab"
-			  "\xc7\x89\xe2\x07\x47\x6c\xe8\xa6"
-			  "\x70\x6b\x8e\x0c\xda\x5c\x6a\x59"
-			  "\x27\x33\x0e\xe1\xe1\x20\xe8\xc8"
-			  "\xae\xdc\xd0\xe3\x6d\xa8\xa6\x06"
-			  "\x41\xb4\xd4\xd4\xcf\x91\x3e\x06"
-			  "\xb0\x9a\xf7\xf1\xaa\xa6\x23\x92"
-			  "\x10\x86\xf0\x94\xd1\x7c\x2e\x07"
-			  "\x30\xfb\xc5\xd8\xf3\x12\xa9\xe8"
-			  "\x22\x1c\x97\x1a\xad\x96\xb0\xa1"
-			  "\x72\x6a\x6b\xb4\xfd\xf7\xe8\xfa"
-			  "\xe2\x74\xd8\x65\x8d\x35\x17\x4b"
-			  "\x00\x23\x5c\x8c\x70\xad\x71\xa2"
-			  "\xca\xc5\x6c\x59\xbf\xb4\xc0\x6d"
-			  "\x86\x98\x3e\x19\x5a\x90\x92\xb1"
-			  "\x66\x57\x6a\x91\x68\x7c\xbc\xf3"
-			  "\xf1\xdb\x94\xf8\x48\xf1\x36\xd8"
-			  "\x78\xac\x1c\xa9\xcc\xd6\x27\xba"
-			  "\x91\x54\x22\xf5\xe6\x05\x3f\xcc"
-			  "\xc2\x8f\x2c\x3b\x2b\xc3\x2b\x2b"
-			  "\x3b\xb8\xb6\x29\xb7\x2f\x94\xb6"
-			  "\x7b\xfc\x94\x3e\xd0\x7a\x41\x59"
-			  "\x7b\x1f\x9a\x09\xa6\xed\x4a\x82"
-			  "\x9d\x34\x1c\xbd\x4e\x1c\x3a\x66"
-			  "\x80\x74\x0e\x9a\x4f\x55\x54\x47"
-			  "\x16\xba\x2a\x0a\x03\x35\x99\xa3"
-			  "\x5c\x63\x8d\xa2\x72\x8b\x17\x15"
-			  "\x68\x39\x73\xeb\xec\xf2\xe8\xf5"
-			  "\x95\x32\x27\xd6\xc4\xfe\xb0\x51"
-			  "\xd5\x0c\x50\xc5\xcd\x6d\x16\xb3"
-			  "\xa3\x1e\x95\x69\xad\x78\x95\x06"
-			  "\xb9\x46\xf2\x6d\x24\x5a\x99\x76"
-			  "\x73\x6a\x91\xa6\xac\x12\xe1\x28"
-			  "\x79\xbc\x08\x4e\x97\x00\x98\x63"
-			  "\x07\x1c\x4e\xd1\x68\xf3\xb3\x81"
-			  "\xa8\xa6\x5f\xf1\x01\xc9\xc1\xaf"
-			  "\x3a\x96\xf9\x9d\xb5\x5a\x5f\x8f"
-			  "\x7e\xc1\x7e\x77\x0a\x40\xc8\x8e"
-			  "\xfc\x0e\xed\xe1\x0d\xb0\xe5\x5e"
-			  "\x5e\x6f\xf5\x7f\xab\x33\x7d\xcd"
-			  "\xf0\x09\x4b\xb2\x11\x37\xdc\x65"
-			  "\x97\x32\x62\x71\x3a\x29\x54\xb9"
-			  "\xc7\xa4\xbf\x75\x0f\xf9\x40\xa9"
-			  "\x8d\xd7\x8b\xa7\xe0\x9a\xbe\x15"
-			  "\xc6\xda\xd8\x00\x14\x69\x1a\xaf"
-			  "\x5f\x79\xc3\xf5\xbb\x6c\x2a\x9d"
-			  "\xdd\x3c\x5f\x97\x21\xe1\x3a\x03"
-			  "\x84\x6a\xe9\x76\x11\x1f\xd3\xd5"
-			  "\xf0\x54\x20\x4d\xc2\x91\xc3\xa4"
-			  "\x36\x25\xbe\x1b\x2a\x06\xb7\xf3"
-			  "\xd1\xd0\x55\x29\x81\x4c\x83\xa3"
-			  "\xa6\x84\x1e\x5c\xd1\xd0\x6c\x90"
-			  "\xa4\x11\xf0\xd7\x63\x6a\x48\x05"
-			  "\xbc\x48\x18\x53\xcd\xb0\x8d\xdb"
-			  "\xdc\xfe\x55\x11\x5c\x51\xb3\xab"
-			  "\xab\x63\x3e\x31\x5a\x8b\x93\x63"
-			  "\x34\xa9\xba\x2b\x69\x1a\xc0\xe3"
-			  "\xcb\x41\xbc\xd7\xf5\x7f\x82\x3e"
-			  "\x01\xa3\x3c\x72\xf4\xfe\xdf\xbe"
-			  "\xb1\x67\x17\x2b\x37\x60\x0d\xca"
-			  "\x6f\xc3\x94\x2c\xd2\x92\x6d\x9d"
-			  "\x75\x18\x77\xaa\x29\x38\x96\xed"
-			  "\x0e\x20\x70\x92\xd5\xd0\xb4\x00"
-			  "\xc0\x31\xf2\xc9\x43\x0e\x75\x1d"
-			  "\x4b\x64\xf2\x1f\xf2\x29\x6c\x7b"
-			  "\x7f\xec\x59\x7d\x8c\x0d\xd4\xd3"
-			  "\xac\x53\x4c\xa3\xde\x42\x92\x95"
-			  "\x6d\xa3\x4f\xd0\xe6\x3d\xe7\xec"
-			  "\x7a\x4d\x68\xf1\xfe\x67\x66\x09"
-			  "\x83\x22\xb1\x98\x43\x8c\xab\xb8"
-			  "\x45\xe6\x6d\xdf\x5e\x50\x71\xce"
-			  "\xf5\x4e\x40\x93\x2b\xfa\x86\x0e"
-			  "\xe8\x30\xbd\x82\xcc\x1c\x9c\x5f"
-			  "\xad\xfd\x08\x31\xbe\x52\xe7\xe6"
-			  "\xf2\x06\x01\x62\x25\x15\x99\x74"
-			  "\x33\x51\x52\x57\x3f\x57\x87\x61"
-			  "\xb9\x7f\x29\x3d\xcd\x92\x5e\xa6"
-			  "\x5c\x3b\xf1\xed\x5f\xeb\x82\xed"
-			  "\x56\x7b\x61\xe7\xfd\x02\x47\x0e"
-			  "\x2a\x15\xa4\xce\x43\x86\x9b\xe1"
-			  "\x2b\x4c\x2a\xd9\x42\x97\xf7\x9a"
-			  "\xe5\x47\x46\x48\xd3\x55\x6f\x4d"
-			  "\xd9\xeb\x4b\xdd\x7b\x21\x2f\xb3"
-			  "\xa8\x36\x28\xdf\xca\xf1\xf6\xd9"
-			  "\x10\xf6\x1c\xfd\x2e\x0c\x27\xe0"
-			  "\x01\xb3\xff\x6d\x47\x08\x4d\xd4"
-			  "\x00\x25\xee\x55\x4a\xe9\xe8\x5b"
-			  "\xd8\xf7\x56\x12\xd4\x50\xb2\xe5"
-			  "\x51\x6f\x34\x63\x69\xd2\x4e\x96"
-			  "\x4e\xbc\x79\xbf\x18\xae\xc6\x13"
-			  "\x80\x92\x77\xb0\xb4\x0f\x29\x94"
-			  "\x6f\x4c\xbb\x53\x11\x36\xc3\x9f"
-			  "\x42\x8e\x96\x8a\x91\xc8\xe9\xfc"
-			  "\xfe\xbf\x7c\x2d\x6f\xf9\xb8\x44"
-			  "\x89\x1b\x09\x53\x0a\x2a\x92\xc3"
-			  "\x54\x7a\x3a\xf9\xe2\xe4\x75\x87"
-			  "\xa0\x5e\x4b\x03\x7a\x0d\x8a\xf4"
-			  "\x55\x59\x94\x2b\x63\x96\x0e\xf5",
-		.psize	= 1040,
-		.digest	= "\xb5\xb9\x08\xb3\x24\x3e\x03\xf0"
-			  "\xd6\x0b\x57\xbc\x0a\x6d\x89\x59",
-	}, {
-		.key	= "\xf6\x34\x42\x71\x35\x52\x8b\x58"
-			  "\x02\x3a\x8e\x4a\x8d\x41\x13\xe9"
-			  "\x7f\xba\xb9\x55\x9d\x73\x4d\xf8"
-			  "\x3f\x5d\x73\x15\xff\xd3\x9e\x7f"
-			  "\x20\x2a\x6a\xa8\xd1\xf0\x8f\x12"
-			  "\x6b\x02\xd8\x6c\xde\xba\x80\x22"
-			  "\x19\x37\xc8\xd0\x4e\x89\x17\x7c"
-			  "\x7c\xdd\x88\xfd\x41\xc0\x04\xb7"
-			  "\x1d\xac\x19\xe3\x20\xc7\x16\xcf"
-			  "\x58\xee\x1d\x7a\x61\x69\xa9\x12"
-			  "\x4b\xef\x4f\xb6\x38\xdd\x78\xf8"
-			  "\x28\xee\x70\x08\xc7\x7c\xcc\xc8"
-			  "\x1e\x41\xf5\x80\x86\x70\xd0\xf0"
-			  "\xa3\x87\x6b\x0a\x00\xd2\x41\x28"
-			  "\x74\x26\xf1\x24\xf3\xd0\x28\x77"
-			  "\xd7\xcd\xf6\x2d\x61\xf4\xa2\x13"
-			  "\x77\xb4\x6f\xa0\xf4\xfb\xd6\xb5"
-			  "\x38\x9d\x5a\x0c\x51\xaf\xad\x63"
-			  "\x27\x67\x8c\x01\xea\x42\x1a\x66"
-			  "\xda\x16\x7c\x3c\x30\x0c\x66\x53"
-			  "\x1c\x88\xa4\x5c\xb2\xe3\x78\x0a"
-			  "\x13\x05\x6d\xe2\xaf\xb3\xe4\x75"
-			  "\x00\x99\x58\xee\x76\x09\x64\xaa"
-			  "\xbb\x2e\xb1\x81\xec\xd8\x0e\xd3"
-			  "\x0c\x33\x5d\xb7\x98\xef\x36\xb6"
-			  "\xd2\x65\x69\x41\x70\x12\xdc\x25"
-			  "\x41\x03\x99\x81\x41\x19\x62\x13"
-			  "\xd1\x0a\x29\xc5\x8c\xe0\x4c\xf3"
-			  "\xd6\xef\x4c\xf4\x1d\x83\x2e\x6d"
-			  "\x8e\x14\x87\xed\x80\xe0\xaa\xd3"
-			  "\x08\x04\x73\x1a\x84\x40\xf5\x64"
-			  "\xbd\x61\x32\x65\x40\x42\xfb\xb0"
-			  "\x40\xf6\x40\x8d\xc7\x7f\x14\xd0"
-			  "\x83\x99\xaa\x36\x7e\x60\xc6\xbf"
-			  "\x13\x8a\xf9\x21\xe4\x7e\x68\x87"
-			  "\xf3\x33\x86\xb4\xe0\x23\x7e\x0a"
-			  "\x21\xb1\xf5\xad\x67\x3c\x9c\x9d"
-			  "\x09\xab\xaf\x5f\xba\xe0\xd0\x82"
-			  "\x48\x22\x70\xb5\x6d\x53\xd6\x0e"
-			  "\xde\x64\x92\x41\xb0\xd3\xfb\xda"
-			  "\x21\xfe\xab\xea\x20\xc4\x03\x58"
-			  "\x18\x2e\x7d\x2f\x03\xa9\x47\x66"
-			  "\xdf\x7b\xa4\x6b\x34\x6b\x55\x9c"
-			  "\x4f\xd7\x9c\x47\xfb\xa9\x42\xec"
-			  "\x5a\x12\xfd\xfe\x76\xa0\x92\x9d"
-			  "\xfe\x1e\x16\xdd\x24\x2a\xe4\x27"
-			  "\xd5\xa9\xf2\x05\x4f\x83\xa2\xaf"
-			  "\xfe\xee\x83\x7a\xad\xde\xdf\x9a"
-			  "\x80\xd5\x81\x14\x93\x16\x7e\x46"
-			  "\x47\xc2\x14\xef\x49\x6e\xb9\xdb"
-			  "\x40\xe8\x06\x6f\x9c\x2a\xfd\x62"
-			  "\x06\x46\xfd\x15\x1d\x36\x61\x6f"
-			  "\x77\x77\x5e\x64\xce\x78\x1b\x85"
-			  "\xbf\x50\x9a\xfd\x67\xa6\x1a\x65"
-			  "\xad\x5b\x33\x30\xf1\x71\xaa\xd9"
-			  "\x23\x0d\x92\x24\x5f\xae\x57\xb0"
-			  "\x24\x37\x0a\x94\x12\xfb\xb5\xb1"
-			  "\xd3\xb8\x1d\x12\x29\xb0\x80\x24"
-			  "\x2d\x47\x9f\x96\x1f\x95\xf1\xb1"
-			  "\xda\x35\xf6\x29\xe0\xe1\x23\x96"
-			  "\xc7\xe8\x22\x9b\x7c\xac\xf9\x41"
-			  "\x39\x01\xe5\x73\x15\x5e\x99\xec"
-			  "\xb4\xc1\xf4\xe7\xa7\x97\x6a\xd5"
-			  "\x90\x9a\xa0\x1d\xf3\x5a\x8b\x5f"
-			  "\xdf\x01\x52\xa4\x93\x31\x97\xb0"
-			  "\x93\x24\xb5\xbc\xb2\x14\x24\x98"
-			  "\x4a\x8f\x19\x85\xc3\x2d\x0f\x74"
-			  "\x9d\x16\x13\x80\x5e\x59\x62\x62"
-			  "\x25\xe0\xd1\x2f\x64\xef\xba\xac"
-			  "\xcd\x09\x07\x15\x8a\xcf\x73\xb5"
-			  "\x8b\xc9\xd8\x24\xb0\x53\xd5\x6f"
-			  "\xe1\x2b\x77\xb1\xc5\xe4\xa7\x0e"
-			  "\x18\x45\xab\x36\x03\x59\xa8\xbd"
-			  "\x43\xf0\xd8\x2c\x1a\x69\x96\xbb"
-			  "\x13\xdf\x6c\x33\x77\xdf\x25\x34"
-			  "\x5b\xa5\x5b\x8c\xf9\x51\x05\xd4"
-			  "\x8b\x8b\x44\x87\x49\xfc\xa0\x8f"
-			  "\x45\x15\x5b\x40\x42\xc4\x09\x92"
-			  "\x98\x0c\x4d\xf4\x26\x37\x1b\x13"
-			  "\x76\x01\x93\x8d\x4f\xe6\xed\x18"
-			  "\xd0\x79\x7b\x3f\x44\x50\xcb\xee"
-			  "\xf7\x4a\xc9\x9e\xe0\x96\x74\xa7"
-			  "\xe6\x93\xb2\x53\xca\x55\xa8\xdc"
-			  "\x1e\x68\x07\x87\xb7\x2e\xc1\x08"
-			  "\xb2\xa4\x5b\xaf\xc6\xdb\x5c\x66"
-			  "\x41\x1c\x51\xd9\xb0\x07\x00\x0d"
-			  "\xf0\x4c\xdc\x93\xde\xa9\x1e\x8e"
-			  "\xd3\x22\x62\xd8\x8b\x88\x2c\xea"
-			  "\x5e\xf1\x6e\x14\x40\xc7\xbe\xaa"
-			  "\x42\x28\xd0\x26\x30\x78\x01\x9b"
-			  "\x83\x07\xbc\x94\xc7\x57\xa2\x9f"
-			  "\x03\x07\xff\x16\xff\x3c\x6e\x48"
-			  "\x0a\xd0\xdd\x4c\xf6\x64\x9a\xf1"
-			  "\xcd\x30\x12\x82\x2c\x38\xd3\x26"
-			  "\x83\xdb\xab\x3e\xc6\xf8\xe6\xfa"
-			  "\x77\x0a\x78\x82\x75\xf8\x63\x51"
-			  "\x59\xd0\x8d\x24\x9f\x25\xe6\xa3"
-			  "\x4c\xbc\x34\xfc\xe3\x10\xc7\x62"
-			  "\xd4\x23\xc8\x3d\xa7\xc6\xa6\x0a"
-			  "\x4f\x7e\x29\x9d\x6d\xbe\xb5\xf1"
-			  "\xdf\xa4\x53\xfa\xc0\x23\x0f\x37"
-			  "\x84\x68\xd0\xb5\xc8\xc6\xae\xf8"
-			  "\xb7\x8d\xb3\x16\xfe\x8f\x87\xad"
-			  "\xd0\xc1\x08\xee\x12\x1c\x9b\x1d"
-			  "\x90\xf8\xd1\x63\xa4\x92\x3c\xf0"
-			  "\xc7\x34\xd8\xf1\x14\xed\xa3\xbc"
-			  "\x17\x7e\xd4\x62\x42\x54\x57\x2c"
-			  "\x3e\x7a\x35\x35\x17\x0f\x0b\x7f"
-			  "\x81\xa1\x3f\xd0\xcd\xc8\x3b\x96"
-			  "\xe9\xe0\x4a\x04\xe1\xb6\x3c\xa1"
-			  "\xd6\xca\xc4\xbd\xb6\xb5\x95\x34"
-			  "\x12\x9d\xc5\x96\xf2\xdf\xba\x54"
-			  "\x76\xd1\xb2\x6b\x3b\x39\xe0\xb9"
-			  "\x18\x62\xfb\xf7\xfc\x12\xf1\x5f"
-			  "\x7e\xc7\xe3\x59\x4c\xa6\xc2\x3d"
-			  "\x40\x15\xf9\xa3\x95\x64\x4c\x74"
-			  "\x8b\x73\x77\x33\x07\xa7\x04\x1d"
-			  "\x33\x5a\x7e\x8f\xbd\x86\x01\x4f"
-			  "\x3e\xb9\x27\x6f\xe2\x41\xf7\x09"
-			  "\x67\xfd\x29\x28\xc5\xe4\xf6\x18"
-			  "\x4c\x1b\x49\xb2\x9c\x5b\xf6\x81"
-			  "\x4f\xbb\x5c\xcc\x0b\xdf\x84\x23"
-			  "\x58\xd6\x28\x34\x93\x3a\x25\x97"
-			  "\xdf\xb2\xc3\x9e\x97\x38\x0b\x7d"
-			  "\x10\xb3\x54\x35\x23\x8c\x64\xee"
-			  "\xf0\xd8\x66\xff\x8b\x22\xd2\x5b"
-			  "\x05\x16\x3c\x89\xf7\xb1\x75\xaf"
-			  "\xc0\xae\x6a\x4f\x3f\xaf\x9a\xf4"
-			  "\xf4\x9a\x24\xd9\x80\x82\xc0\x12"
-			  "\xde\x96\xd1\xbe\x15\x0b\x8d\x6a"
-			  "\xd7\x12\xe4\x85\x9f\x83\xc9\xc3"
-			  "\xff\x0b\xb5\xaf\x3b\xd8\x6d\x67"
-			  "\x81\x45\xe6\xac\xec\xc1\x7b\x16"
-			  "\x18\x0a\xce\x4b\xc0\x2e\x76\xbc"
-			  "\x1b\xfa\xb4\x34\xb8\xfc\x3e\xc8"
-			  "\x5d\x90\x71\x6d\x7a\x79\xef\x06",
-		.ksize	= 1088,
-		.plaintext	= "\xaa\x5d\x54\xcb\xea\x1e\x46\x0f"
-			  "\x45\x87\x70\x51\x8a\x66\x7a\x33"
-			  "\xb4\x18\xff\xa9\x82\xf9\x45\x4b"
-			  "\x93\xae\x2e\x7f\xab\x98\xfe\xbf"
-			  "\x01\xee\xe5\xa0\x37\x8f\x57\xa6"
-			  "\xb0\x76\x0d\xa4\xd6\x28\x2b\x5d"
-			  "\xe1\x03\xd6\x1c\x6f\x34\x0d\xe7"
-			  "\x61\x2d\x2e\xe5\xae\x5d\x47\xc7"
-			  "\x80\x4b\x18\x8f\xa8\x99\xbc\x28"
-			  "\xed\x1d\x9d\x86\x7d\xd7\x41\xd1"
-			  "\xe0\x2b\xe1\x8c\x93\x2a\xa7\x80"
-			  "\xe1\x07\xa0\xa9\x9f\x8c\x8d\x1a"
-			  "\x55\xfc\x6b\x24\x7a\xbd\x3e\x51"
-			  "\x68\x4b\x26\x59\xc8\xa7\x16\xd9"
-			  "\xb9\x61\x13\xde\x8b\x63\x1c\xf6"
-			  "\x60\x01\xfb\x08\xb3\x5b\x0a\xbf"
-			  "\x34\x73\xda\x87\x87\x3d\x6f\x97"
-			  "\x4a\x0c\xa3\x58\x20\xa2\xc0\x81"
-			  "\x5b\x8c\xef\xa9\xc2\x01\x1e\x64"
-			  "\x83\x8c\xbc\x03\xb6\xd0\x29\x9f"
-			  "\x54\xe2\xce\x8b\xc2\x07\x85\x78"
-			  "\x25\x38\x96\x4c\xb4\xbe\x17\x4a"
-			  "\x65\xa6\xfa\x52\x9d\x66\x9d\x65"
-			  "\x4a\xd1\x01\x01\xf0\xcb\x13\xcc"
-			  "\xa5\x82\xf3\xf2\x66\xcd\x3f\x9d"
-			  "\xd1\xaa\xe4\x67\xea\xf2\xad\x88"
-			  "\x56\x76\xa7\x9b\x59\x3c\xb1\x5d"
-			  "\x78\xfd\x69\x79\x74\x78\x43\x26"
-			  "\x7b\xde\x3f\xf1\xf5\x4e\x14\xd9"
-			  "\x15\xf5\x75\xb5\x2e\x19\xf3\x0c"
-			  "\x48\x72\xd6\x71\x6d\x03\x6e\xaa"
-			  "\xa7\x08\xf9\xaa\x70\xa3\x0f\x4d"
-			  "\x12\x8a\xdd\xe3\x39\x73\x7e\xa7"
-			  "\xea\x1f\x6d\x06\x26\x2a\xf2\xc5"
-			  "\x52\xb4\xbf\xfd\x52\x0c\x06\x60"
-			  "\x90\xd1\xb2\x7b\x56\xae\xac\x58"
-			  "\x5a\x6b\x50\x2a\xf5\xe0\x30\x3c"
-			  "\x2a\x98\x0f\x1b\x5b\x0a\x84\x6c"
-			  "\x31\xae\x92\xe2\xd4\xbb\x7f\x59"
-			  "\x26\x10\xb9\x89\x37\x68\x26\xbf"
-			  "\x41\xc8\x49\xc4\x70\x35\x7d\xff"
-			  "\x2d\x7f\xf6\x8a\x93\x68\x8c\x78"
-			  "\x0d\x53\xce\x7d\xff\x7d\xfb\xae"
-			  "\x13\x1b\x75\xc4\x78\xd7\x71\xd8"
-			  "\xea\xd3\xf4\x9d\x95\x64\x8e\xb4"
-			  "\xde\xb8\xe4\xa6\x68\xc8\xae\x73"
-			  "\x58\xaf\xa8\xb0\x5a\x20\xde\x87"
-			  "\x43\xb9\x0f\xe3\xad\x41\x4b\xd5"
-			  "\xb7\xad\x16\x00\xa6\xff\xf6\x74"
-			  "\xbf\x8c\x9f\xb3\x58\x1b\xb6\x55"
-			  "\xa9\x90\x56\x28\xf0\xb5\x13\x4e"
-			  "\x9e\xf7\x25\x86\xe0\x07\x7b\x98"
-			  "\xd8\x60\x5d\x38\x95\x3c\xe4\x22"
-			  "\x16\x2f\xb2\xa2\xaf\xe8\x90\x17"
-			  "\xec\x11\x83\x1a\xf4\xa9\x26\xda"
-			  "\x39\x72\xf5\x94\x61\x05\x51\xec"
-			  "\xa8\x30\x8b\x2c\x13\xd0\x72\xac"
-			  "\xb9\xd2\xa0\x4c\x4b\x78\xe8\x6e"
-			  "\x04\x85\xe9\x04\x49\x82\x91\xff"
-			  "\x89\xe5\xab\x4c\xaa\x37\x03\x12"
-			  "\xca\x8b\x74\x10\xfd\x9e\xd9\x7b"
-			  "\xcb\xdb\x82\x6e\xce\x2e\x33\x39"
-			  "\xce\xd2\x84\x6e\x34\x71\x51\x6e"
-			  "\x0d\xd6\x01\x87\xc7\xfa\x0a\xd3"
-			  "\xad\x36\xf3\x4c\x9f\x96\x5e\x62"
-			  "\x62\x54\xc3\x03\x78\xd6\xab\xdd"
-			  "\x89\x73\x55\x25\x30\xf8\xa7\xe6"
-			  "\x4f\x11\x0c\x7c\x0a\xa1\x2b\x7b"
-			  "\x3d\x0d\xde\x81\xd4\x9d\x0b\xae"
-			  "\xdf\x00\xf9\x4c\xb6\x90\x8e\x16"
-			  "\xcb\x11\xc8\xd1\x2e\x73\x13\x75"
-			  "\x75\x3e\xaa\xf5\xee\x02\xb3\x18"
-			  "\xa6\x2d\xf5\x3b\x51\xd1\x1f\x47"
-			  "\x6b\x2c\xdb\xc4\x10\xe0\xc8\xba"
-			  "\x9d\xac\xb1\x9d\x75\xd5\x41\x0e"
-			  "\x7e\xbe\x18\x5b\xa4\x1f\xf8\x22"
-			  "\x4c\xc1\x68\xda\x6d\x51\x34\x6c"
-			  "\x19\x59\xec\xb5\xb1\xec\xa7\x03"
-			  "\xca\x54\x99\x63\x05\x6c\xb1\xac"
-			  "\x9c\x31\xd6\xdb\xba\x7b\x14\x12"
-			  "\x7a\xc3\x2f\xbf\x8d\xdc\x37\x46"
-			  "\xdb\xd2\xbc\xd4\x2f\xab\x30\xd5"
-			  "\xed\x34\x99\x8e\x83\x3e\xbe\x4c"
-			  "\x86\x79\x58\xe0\x33\x8d\x9a\xb8"
-			  "\xa9\xa6\x90\x46\xa2\x02\xb8\xdd"
-			  "\xf5\xf9\x1a\x5c\x8c\x01\xaa\x6e"
-			  "\xb4\x22\x12\xf5\x0c\x1b\x9b\x7a"
-			  "\xc3\x80\xf3\x06\x00\x5f\x30\xd5"
-			  "\x06\xdb\x7d\x82\xc2\xd4\x0b\x4c"
-			  "\x5f\xe9\xc5\xf5\xdf\x97\x12\xbf"
-			  "\x56\xaf\x9b\x69\xcd\xee\x30\xb4"
-			  "\xa8\x71\xff\x3e\x7d\x73\x7a\xb4"
-			  "\x0d\xa5\x46\x7a\xf3\xf4\x15\x87"
-			  "\x5d\x93\x2b\x8c\x37\x64\xb5\xdd"
-			  "\x48\xd1\xe5\x8c\xae\xd4\xf1\x76"
-			  "\xda\xf4\xba\x9e\x25\x0e\xad\xa3"
-			  "\x0d\x08\x7c\xa8\x82\x16\x8d\x90"
-			  "\x56\x40\x16\x84\xe7\x22\x53\x3a"
-			  "\x58\xbc\xb9\x8f\x33\xc8\xc2\x84"
-			  "\x22\xe6\x0d\xe7\xb3\xdc\x5d\xdf"
-			  "\xd7\x2a\x36\xe4\x16\x06\x07\xd2"
-			  "\x97\x60\xb2\xf5\x5e\x14\xc9\xfd"
-			  "\x8b\x05\xd1\xce\xee\x9a\x65\x99"
-			  "\xb7\xae\x19\xb7\xc8\xbc\xd5\xa2"
-			  "\x7b\x95\xe1\xcc\xba\x0d\xdc\x8a"
-			  "\x1d\x59\x52\x50\xaa\x16\x02\x82"
-			  "\xdf\x61\x33\x2e\x44\xce\x49\xc7"
-			  "\xe5\xc6\x2e\x76\xcf\x80\x52\xf0"
-			  "\x3d\x17\x34\x47\x3f\xd3\x80\x48"
-			  "\xa2\xba\xd5\xc7\x7b\x02\x28\xdb"
-			  "\xac\x44\xc7\x6e\x05\x5c\xc2\x79"
-			  "\xb3\x7d\x6a\x47\x77\x66\xf1\x38"
-			  "\xf0\xf5\x4f\x27\x1a\x31\xca\x6c"
-			  "\x72\x95\x92\x8e\x3f\xb0\xec\x1d"
-			  "\xc7\x2a\xff\x73\xee\xdf\x55\x80"
-			  "\x93\xd2\xbd\x34\xd3\x9f\x00\x51"
-			  "\xfb\x2e\x41\xba\x6c\x5a\x7c\x17"
-			  "\x7f\xe6\x70\xac\x8d\x39\x3f\x77"
-			  "\xe2\x23\xac\x8f\x72\x4e\xe4\x53"
-			  "\xcc\xf1\x1b\xf1\x35\xfe\x52\xa4"
-			  "\xd6\xb8\x40\x6b\xc1\xfd\xa0\xa1"
-			  "\xf5\x46\x65\xc2\x50\xbb\x43\xe2"
-			  "\xd1\x43\x28\x34\x74\xf5\x87\xa0"
-			  "\xf2\x5e\x27\x3b\x59\x2b\x3e\x49"
-			  "\xdf\x46\xee\xaf\x71\xd7\x32\x36"
-			  "\xc7\x14\x0b\x58\x6e\x3e\x2d\x41"
-			  "\xfa\x75\x66\x3a\x54\xe0\xb2\xb9"
-			  "\xaf\xdd\x04\x80\x15\x19\x3f\x6f"
-			  "\xce\x12\xb4\xd8\xe8\x89\x3c\x05"
-			  "\x30\xeb\xf3\x3d\xcd\x27\xec\xdc"
-			  "\x56\x70\x12\xcf\x78\x2b\x77\xbf"
-			  "\x22\xf0\x1b\x17\x9c\xcc\xd6\x1b"
-			  "\x2d\x3d\xa0\x3b\xd8\xc9\x70\xa4"
-			  "\x7a\x3e\x07\xb9\x06\xc3\xfa\xb0"
-			  "\x33\xee\xc1\xd8\xf6\xe0\xf0\xb2"
-			  "\x61\x12\x69\xb0\x5f\x28\x99\xda"
-			  "\xc3\x61\x48\xfa\x07\x16\x03\xc4"
-			  "\xa8\xe1\x3c\xe8\x0e\x64\x15\x30"
-			  "\xc1\x9d\x84\x2f\x73\x98\x0e\x3a"
-			  "\xf2\x86\x21\xa4\x9e\x1d\xb5\x86"
-			  "\x16\xdb\x2b\x9a\x06\x64\x8e\x79"
-			  "\x8d\x76\x3e\xc3\xc2\x64\x44\xe3"
-			  "\xda\xbc\x1a\x52\xd7\x61\x03\x65"
-			  "\x54\x32\x77\x01\xed\x9d\x8a\x43"
-			  "\x25\x24\xe3\xc1\xbe\xb8\x2f\xcb"
-			  "\x89\x14\x64\xab\xf6\xa0\x6e\x02"
-			  "\x57\xe4\x7d\xa9\x4e\x9a\x03\x36"
-			  "\xad\xf1\xb1\xfc\x0b\xe6\x79\x51"
-			  "\x9f\x81\x77\xc4\x14\x78\x9d\xbf"
-			  "\xb6\xd6\xa3\x8c\xba\x0b\x26\xe7"
-			  "\xc8\xb9\x5c\xcc\xe1\x5f\xd5\xc6"
-			  "\xc4\xca\xc2\xa3\x45\xba\x94\x13"
-			  "\xb2\x8f\xc3\x54\x01\x09\xe7\x8b"
-			  "\xda\x2a\x0a\x11\x02\x43\xcb\x57"
-			  "\xc9\xcc\xb5\x5c\xab\xc4\xec\x54"
-			  "\x00\x06\x34\xe1\x6e\x03\x89\x7c"
-			  "\xc6\xfb\x6a\xc7\x60\x43\xd6\xc5"
-			  "\xb5\x68\x72\x89\x8f\x42\xc3\x74"
-			  "\xbd\x25\xaa\x9f\x67\xb5\xdf\x26"
-			  "\x20\xe8\xb7\x01\x3c\xe4\x77\xce"
-			  "\xc4\x65\xa7\x23\x79\xea\x33\xc7"
-			  "\x82\x14\x5c\x82\xf2\x4e\x3d\xf6"
-			  "\xc6\x4a\x0e\x29\xbb\xec\x44\xcd"
-			  "\x2f\xd1\x4f\x21\x71\xa9\xce\x0f"
-			  "\x5c\xf2\x72\x5c\x08\x2e\x21\xd2"
-			  "\xc3\x29\x13\xd8\xac\xc3\xda\x13"
-			  "\x1a\x9d\xa7\x71\x1d\x27\x1d\x27"
-			  "\x1d\xea\xab\x44\x79\xad\xe5\xeb"
-			  "\xef\x1f\x22\x0a\x44\x4f\xcb\x87"
-			  "\xa7\x58\x71\x0e\x66\xf8\x60\xbf"
-			  "\x60\x74\x4a\xb4\xec\x2e\xfe\xd3"
-			  "\xf5\xb8\xfe\x46\x08\x50\x99\x6c"
-			  "\x66\xa5\xa8\x34\x44\xb5\xe5\xf0"
-			  "\xdd\x2c\x67\x4e\x35\x96\x8e\x67"
-			  "\x48\x3f\x5f\x37\x44\x60\x51\x2e"
-			  "\x14\x91\x5e\x57\xc3\x0e\x79\x77"
-			  "\x2f\x03\xf4\xe2\x1c\x72\xbf\x85"
-			  "\x5d\xd3\x17\xdf\x6c\xc5\x70\x24"
-			  "\x42\xdf\x51\x4e\x2a\xb2\xd2\x5b"
-			  "\x9e\x69\x83\x41\x11\xfe\x73\x22"
-			  "\xde\x8a\x9e\xd8\x8a\xfb\x20\x38"
-			  "\xd8\x47\x6f\xd5\xed\x8f\x41\xfd"
-			  "\x13\x7a\x18\x03\x7d\x0f\xcd\x7d"
-			  "\xa6\x7d\x31\x9e\xf1\x8f\x30\xa3"
-			  "\x8b\x4c\x24\xb7\xf5\x48\xd7\xd9"
-			  "\x12\xe7\x84\x97\x5c\x31\x6d\xfb"
-			  "\xdf\xf3\xd3\xd1\xd5\x0c\x30\x06"
-			  "\x01\x6a\xbc\x6c\x78\x7b\xa6\x50"
-			  "\xfa\x0f\x3c\x42\x2d\xa5\xa3\x3b"
-			  "\xcf\x62\x50\xff\x71\x6d\xe7\xda"
-			  "\x27\xab\xc6\x67\x16\x65\x68\x64"
-			  "\xc7\xd5\x5f\x81\xa9\xf6\x65\xb3"
-			  "\x5e\x43\x91\x16\xcd\x3d\x55\x37"
-			  "\x55\xb3\xf0\x28\xc5\x54\x19\xc0"
-			  "\xe0\xd6\x2a\x61\xd4\xc8\x72\x51"
-			  "\xe9\xa1\x7b\x48\x21\xad\x44\x09"
-			  "\xe4\x01\x61\x3c\x8a\x5b\xf9\xa1"
-			  "\x6e\x1b\xdf\xc0\x04\xa8\x8b\xf2"
-			  "\x21\xbe\x34\x7b\xfc\xa1\xcd\xc9"
-			  "\xa9\x96\xf4\xa4\x4c\xf7\x4e\x8f"
-			  "\x84\xcc\xd3\xa8\x92\x77\x8f\x36"
-			  "\xe2\x2e\x8c\x33\xe8\x84\xa6\x0c"
-			  "\x6c\x8a\xda\x14\x32\xc2\x96\xff"
-			  "\xc6\x4a\xc2\x9b\x30\x7f\xd1\x29"
-			  "\xc0\xd5\x78\x41\x00\x80\x80\x03"
-			  "\x2a\xb1\xde\x26\x03\x48\x49\xee"
-			  "\x57\x14\x76\x51\x3c\x36\x5d\x0a"
-			  "\x5c\x9f\xe8\xd8\x53\xdb\x4f\xd4"
-			  "\x38\xbf\x66\xc9\x75\x12\x18\x75"
-			  "\x34\x2d\x93\x22\x96\x51\x24\x6e"
-			  "\x4e\xd9\x30\xea\x67\xff\x92\x1c"
-			  "\x16\x26\xe9\xb5\x33\xab\x8c\x22"
-			  "\x47\xdb\xa0\x2c\x08\xf0\x12\x69"
-			  "\x7e\x93\x52\xda\xa5\xe5\xca\xc1"
-			  "\x0f\x55\x2a\xbd\x09\x30\x88\x1b"
-			  "\x9c\xc6\x9f\xe6\xdb\xa6\x92\xeb"
-			  "\xf4\xbd\x5c\xc4\xdb\xc6\x71\x09"
-			  "\xab\x5e\x48\x0c\xed\x6f\xda\x8e"
-			  "\x8d\x0c\x98\x71\x7d\x10\xd0\x9c"
-			  "\x20\x9b\x79\x53\x26\x5d\xb9\x85"
-			  "\x8a\x31\xb8\xc5\x1c\x97\xde\x88"
-			  "\x61\x55\x7f\x7c\x21\x06\xea\xc4"
-			  "\x5f\xaf\xf2\xf0\xd5\x5e\x7d\xb4"
-			  "\x6e\xcf\xe9\xae\x1b\x0e\x11\x80"
-			  "\xc1\x9a\x74\x7e\x52\x6f\xa0\xb7"
-			  "\x24\xcd\x8d\x0a\x11\x40\x63\x72"
-			  "\xfa\xe2\xc5\xb3\x94\xef\x29\xa2"
-			  "\x1a\x23\x43\x04\x37\x55\x0d\xe9"
-			  "\x83\xb2\x29\x51\x49\x64\xa0\xbd"
-			  "\xde\x73\xfd\xa5\x7c\x95\x70\x62"
-			  "\x58\xdc\xe2\xd0\xbf\x98\xf5\x8a"
-			  "\x6a\xfd\xce\xa8\x0e\x42\x2a\xeb"
-			  "\xd2\xff\x83\x27\x53\x5c\xa0\x6e"
-			  "\x93\xef\xe2\xb9\x5d\x35\xd6\x98"
-			  "\xf6\x71\x19\x7a\x54\xa1\xa7\xe8"
-			  "\x09\xfe\xf6\x9e\xc7\xbd\x3e\x29"
-			  "\xbd\x6b\x17\xf4\xe7\x3e\x10\x5c"
-			  "\xc1\xd2\x59\x4f\x4b\x12\x1a\x5b"
-			  "\x50\x80\x59\xb9\xec\x13\x66\xa8"
-			  "\xd2\x31\x7b\x6a\x61\x22\xdd\x7d"
-			  "\x61\xee\x87\x16\x46\x9f\xf9\xc7"
-			  "\x41\xee\x74\xf8\xd0\x96\x2c\x76"
-			  "\x2a\xac\x7d\x6e\x9f\x0e\x7f\x95"
-			  "\xfe\x50\x16\xb2\x23\xca\x62\xd5"
-			  "\x68\xcf\x07\x3f\x3f\x97\x85\x2a"
-			  "\x0c\x25\x45\xba\xdb\x32\xcb\x83"
-			  "\x8c\x4f\xe0\x6d\x9a\x99\xf9\xc9"
-			  "\xda\xd4\x19\x31\xc1\x7c\x6d\xd9"
-			  "\x9c\x56\xd3\xec\xc1\x81\x4c\xed"
-			  "\x28\x9d\x87\xeb\x19\xd7\x1a\x4f"
-			  "\x04\x6a\xcb\x1f\xcf\x1f\xa2\x16"
-			  "\xfc\x2a\x0d\xa1\x14\x2d\xfa\xc5"
-			  "\x5a\xd2\xc5\xf9\x19\x7c\x20\x1f"
-			  "\x2d\x10\xc0\x66\x7c\xd9\x2d\xe5"
-			  "\x88\x70\x59\xa7\x85\xd5\x2e\x7c"
-			  "\x5c\xe3\xb7\x12\xd6\x97\x3f\x29",
-		.psize	= 2048,
-		.digest	= "\x37\x90\x92\xc2\xeb\x01\x87\xd9"
-			  "\x95\xc7\x91\xc3\x17\x8b\x38\x52",
-	}
-};
-
-
 /*
  * DES test vectors.
  */
diff --git a/drivers/accel/amdxdna/Makefile b/drivers/accel/amdxdna/Makefile
index 6344aaf523fa..3fa0e74fd8f5 100644
--- a/drivers/accel/amdxdna/Makefile
+++ b/drivers/accel/amdxdna/Makefile
@@ -18,7 +18,6 @@ amdxdna-y := \
 	amdxdna_sysfs.o \
 	amdxdna_ubuf.o \
 	npu1_regs.o \
-	npu2_regs.o \
 	npu4_regs.o \
 	npu5_regs.o \
 	npu6_regs.o
diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c
index 42d876a427c5..5511ab2ef242 100644
--- a/drivers/accel/amdxdna/aie2_ctx.c
+++ b/drivers/accel/amdxdna/aie2_ctx.c
@@ -468,6 +468,12 @@ static int aie2_alloc_resource(struct amdxdna_hwctx *hwctx)
 	struct alloc_requests *xrs_req;
 	int ret;
 
+	if (AIE2_FEATURE_ON(xdna->dev_handle, AIE2_TEMPORAL_ONLY)) {
+		hwctx->num_unused_col = xdna->dev_handle->total_col - hwctx->num_col;
+		hwctx->num_col = xdna->dev_handle->total_col;
+		return aie2_create_context(xdna->dev_handle, hwctx);
+	}
+
 	xrs_req = kzalloc(sizeof(*xrs_req), GFP_KERNEL);
 	if (!xrs_req)
 		return -ENOMEM;
@@ -499,9 +505,15 @@ static void aie2_release_resource(struct amdxdna_hwctx *hwctx)
 	struct amdxdna_dev *xdna = hwctx->client->xdna;
 	int ret;
 
-	ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx);
-	if (ret)
-		XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret);
+	if (AIE2_FEATURE_ON(xdna->dev_handle, AIE2_TEMPORAL_ONLY)) {
+		ret = aie2_destroy_context(xdna->dev_handle, hwctx);
+		if (ret)
+			XDNA_ERR(xdna, "Destroy temporal only context failed, ret %d", ret);
+	} else {
+		ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx);
+		if (ret)
+			XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret);
+	}
 }
 
 static int aie2_ctx_syncobj_create(struct amdxdna_hwctx *hwctx)
diff --git a/drivers/accel/amdxdna/aie2_message.c b/drivers/accel/amdxdna/aie2_message.c
index d493bb1c3360..051f4ceaabae 100644
--- a/drivers/accel/amdxdna/aie2_message.c
+++ b/drivers/accel/amdxdna/aie2_message.c
@@ -39,7 +39,6 @@ static int aie2_send_mgmt_msg_wait(struct amdxdna_dev_hdl *ndev,
 	if (!ndev->mgmt_chann)
 		return -ENODEV;
 
-	drm_WARN_ON(&xdna->ddev, xdna->rpm_on && !mutex_is_locked(&xdna->dev_lock));
 	ret = xdna_send_msg_wait(xdna, ndev->mgmt_chann, msg);
 	if (ret == -ETIME) {
 		xdna_mailbox_stop_channel(ndev->mgmt_chann);
@@ -59,8 +58,15 @@ static int aie2_send_mgmt_msg_wait(struct amdxdna_dev_hdl *ndev,
 int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev)
 {
 	DECLARE_AIE2_MSG(suspend, MSG_OP_SUSPEND);
+	int ret;
 
-	return aie2_send_mgmt_msg_wait(ndev, &msg);
+	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
+	if (ret) {
+		XDNA_ERR(ndev->xdna, "Failed to suspend fw, ret %d", ret);
+		return ret;
+	}
+
+	return aie2_psp_waitmode_poll(ndev->psp_hdl);
 }
 
 int aie2_resume_fw(struct amdxdna_dev_hdl *ndev)
@@ -186,6 +192,40 @@ int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev,
 	return 0;
 }
 
+static int aie2_destroy_context_req(struct amdxdna_dev_hdl *ndev, u32 id)
+{
+	DECLARE_AIE2_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT);
+	struct amdxdna_dev *xdna = ndev->xdna;
+	int ret;
+
+	req.context_id = id;
+	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
+	if (ret)
+		XDNA_WARN(xdna, "Destroy context failed, ret %d", ret);
+
+	return ret;
+}
+
+static u32 aie2_get_context_priority(struct amdxdna_dev_hdl *ndev,
+				     struct amdxdna_hwctx *hwctx)
+{
+	if (!AIE2_FEATURE_ON(ndev, AIE2_PREEMPT))
+		return PRIORITY_HIGH;
+
+	switch (hwctx->qos.priority) {
+	case AMDXDNA_QOS_REALTIME_PRIORITY:
+		return PRIORITY_REALTIME;
+	case AMDXDNA_QOS_HIGH_PRIORITY:
+		return PRIORITY_HIGH;
+	case AMDXDNA_QOS_NORMAL_PRIORITY:
+		return PRIORITY_NORMAL;
+	case AMDXDNA_QOS_LOW_PRIORITY:
+		return PRIORITY_LOW;
+	default:
+		return PRIORITY_HIGH;
+	}
+}
+
 int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx)
 {
 	DECLARE_AIE2_MSG(create_ctx, MSG_OP_CREATE_CONTEXT);
@@ -199,22 +239,24 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct
 	req.aie_type = 1;
 	req.start_col = hwctx->start_col;
 	req.num_col = hwctx->num_col;
+	req.num_unused_col = hwctx->num_unused_col;
 	req.num_cq_pairs_requested = 1;
 	req.pasid = hwctx->client->pasid;
-	req.context_priority = 2;
+	req.context_priority = aie2_get_context_priority(ndev, hwctx);
 
 	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
 	if (ret)
 		return ret;
 
 	hwctx->fw_ctx_id = resp.context_id;
-	WARN_ONCE(hwctx->fw_ctx_id == -1, "Unexpected context id");
+	if (WARN_ON_ONCE(hwctx->fw_ctx_id == -1))
+		return -EINVAL;
 
 	if (ndev->force_preempt_enabled) {
 		ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FORCE_PREEMPT, &hwctx->fw_ctx_id);
 		if (ret) {
 			XDNA_ERR(xdna, "failed to enable force preempt %d", ret);
-			return ret;
+			goto del_ctx_req;
 		}
 	}
 
@@ -231,51 +273,39 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct
 
 	ret = pci_irq_vector(to_pci_dev(xdna->ddev.dev), resp.msix_id);
 	if (ret == -EINVAL) {
-		XDNA_ERR(xdna, "not able to create channel");
-		goto out_destroy_context;
+		XDNA_ERR(xdna, "Alloc IRQ failed %d", ret);
+		goto del_ctx_req;
 	}
 
 	intr_reg = i2x.mb_head_ptr_reg + 4;
 	hwctx->priv->mbox_chann = xdna_mailbox_create_channel(ndev->mbox, &x2i, &i2x,
 							      intr_reg, ret);
 	if (!hwctx->priv->mbox_chann) {
-		XDNA_ERR(xdna, "not able to create channel");
+		XDNA_ERR(xdna, "Not able to create channel");
 		ret = -EINVAL;
-		goto out_destroy_context;
+		goto del_ctx_req;
 	}
 	ndev->hwctx_num++;
 
-	XDNA_DBG(xdna, "%s mailbox channel irq: %d, msix_id: %d",
-		 hwctx->name, ret, resp.msix_id);
-	XDNA_DBG(xdna, "%s created fw ctx %d pasid %d", hwctx->name,
-		 hwctx->fw_ctx_id, hwctx->client->pasid);
+	XDNA_DBG(xdna, "Mailbox channel irq: %d, msix_id: %d", ret, resp.msix_id);
+	XDNA_DBG(xdna, "Created fw ctx %d pasid %d", hwctx->fw_ctx_id, hwctx->client->pasid);
 
 	return 0;
 
-out_destroy_context:
-	aie2_destroy_context(ndev, hwctx);
+del_ctx_req:
+	aie2_destroy_context_req(ndev, hwctx->fw_ctx_id);
 	return ret;
 }
 
 int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx)
 {
-	DECLARE_AIE2_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT);
 	struct amdxdna_dev *xdna = ndev->xdna;
 	int ret;
 
-	if (hwctx->fw_ctx_id == -1)
-		return 0;
-
 	xdna_mailbox_stop_channel(hwctx->priv->mbox_chann);
-
-	req.context_id = hwctx->fw_ctx_id;
-	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
-	if (ret)
-		XDNA_WARN(xdna, "%s destroy context failed, ret %d", hwctx->name, ret);
-
+	ret = aie2_destroy_context_req(ndev, hwctx->fw_ctx_id);
 	xdna_mailbox_destroy_channel(hwctx->priv->mbox_chann);
-	XDNA_DBG(xdna, "%s destroyed fw ctx %d", hwctx->name,
-		 hwctx->fw_ctx_id);
+	XDNA_DBG(xdna, "Destroyed fw ctx %d", hwctx->fw_ctx_id);
 	hwctx->priv->mbox_chann = NULL;
 	hwctx->fw_ctx_id = -1;
 	ndev->hwctx_num--;
@@ -646,6 +676,7 @@ aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *siz
 	u32 cmd_len;
 	void *cmd;
 
+	memset(npu_slot, 0, sizeof(*npu_slot));
 	cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
 	if (*size < sizeof(*npu_slot) + cmd_len)
 		return -EINVAL;
@@ -654,7 +685,6 @@ aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *siz
 	if (npu_slot->cu_idx == INVALID_CU_IDX)
 		return -EINVAL;
 
-	memset(npu_slot, 0, sizeof(*npu_slot));
 	npu_slot->type = EXEC_NPU_TYPE_NON_ELF;
 	npu_slot->arg_cnt = cmd_len / sizeof(u32);
 	memcpy(npu_slot->args, cmd, cmd_len);
@@ -671,6 +701,7 @@ aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *si
 	u32 cmd_len;
 	u32 arg_sz;
 
+	memset(npu_slot, 0, sizeof(*npu_slot));
 	sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
 	arg_sz = cmd_len - sizeof(*sn);
 	if (cmd_len < sizeof(*sn) || arg_sz > MAX_NPU_ARGS_SIZE)
@@ -683,7 +714,6 @@ aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *si
 	if (npu_slot->cu_idx == INVALID_CU_IDX)
 		return -EINVAL;
 
-	memset(npu_slot, 0, sizeof(*npu_slot));
 	npu_slot->type = EXEC_NPU_TYPE_PARTIAL_ELF;
 	npu_slot->inst_buf_addr = sn->buffer;
 	npu_slot->inst_size = sn->buffer_size;
@@ -703,6 +733,7 @@ aie2_cmdlist_fill_npu_preempt(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t
 	u32 cmd_len;
 	u32 arg_sz;
 
+	memset(npu_slot, 0, sizeof(*npu_slot));
 	pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
 	arg_sz = cmd_len - sizeof(*pd);
 	if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE)
@@ -715,7 +746,6 @@ aie2_cmdlist_fill_npu_preempt(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t
 	if (npu_slot->cu_idx == INVALID_CU_IDX)
 		return -EINVAL;
 
-	memset(npu_slot, 0, sizeof(*npu_slot));
 	npu_slot->type = EXEC_NPU_TYPE_PREEMPT;
 	npu_slot->inst_buf_addr = pd->inst_buf;
 	npu_slot->save_buf_addr = pd->save_buf;
@@ -739,6 +769,7 @@ aie2_cmdlist_fill_npu_elf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *si
 	u32 cmd_len;
 	u32 arg_sz;
 
+	memset(npu_slot, 0, sizeof(*npu_slot));
 	pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
 	arg_sz = cmd_len - sizeof(*pd);
 	if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE)
@@ -747,7 +778,6 @@ aie2_cmdlist_fill_npu_elf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *si
 	if (*size < sizeof(*npu_slot) + arg_sz)
 		return -EINVAL;
 
-	memset(npu_slot, 0, sizeof(*npu_slot));
 	npu_slot->type = EXEC_NPU_TYPE_ELF;
 	npu_slot->inst_buf_addr = pd->inst_buf;
 	npu_slot->save_buf_addr = pd->save_buf;
diff --git a/drivers/accel/amdxdna/aie2_msg_priv.h b/drivers/accel/amdxdna/aie2_msg_priv.h
index 1c957a6298d3..728ef56f7f0a 100644
--- a/drivers/accel/amdxdna/aie2_msg_priv.h
+++ b/drivers/accel/amdxdna/aie2_msg_priv.h
@@ -108,11 +108,17 @@ struct cq_pair {
 	struct cq_info i2x_q;
 };
 
+#define PRIORITY_REALTIME	1
+#define PRIORITY_HIGH		2
+#define PRIORITY_NORMAL		3
+#define PRIORITY_LOW		4
+
 struct create_ctx_req {
 	__u32	aie_type;
 	__u8	start_col;
 	__u8	num_col;
-	__u16	reserved;
+	__u8    num_unused_col;
+	__u8	reserved;
 	__u8	num_cq_pairs_requested;
 	__u8	reserved1;
 	__u16	pasid;
diff --git a/drivers/accel/amdxdna/aie2_pci.c b/drivers/accel/amdxdna/aie2_pci.c
index ceef1c502e9e..ec1c3ad57d49 100644
--- a/drivers/accel/amdxdna/aie2_pci.c
+++ b/drivers/accel/amdxdna/aie2_pci.c
@@ -17,6 +17,7 @@
 #include <linux/iopoll.h>
 #include <linux/pci.h>
 #include <linux/xarray.h>
+#include <asm/hypervisor.h>
 
 #include "aie2_msg_priv.h"
 #include "aie2_pci.h"
@@ -321,7 +322,7 @@ static int aie2_xrs_set_dft_dpm_level(struct drm_device *ddev, u32 dpm_level)
 	if (ndev->pw_mode != POWER_MODE_DEFAULT || ndev->dpm_level == dpm_level)
 		return 0;
 
-	return ndev->priv->hw_ops.set_dpm(ndev, dpm_level);
+	return aie2_pm_set_dpm(ndev, dpm_level);
 }
 
 static struct xrs_action_ops aie2_xrs_actions = {
@@ -508,6 +509,11 @@ static int aie2_init(struct amdxdna_dev *xdna)
 	unsigned long bars = 0;
 	int i, nvec, ret;
 
+	if (!hypervisor_is_type(X86_HYPER_NATIVE)) {
+		XDNA_ERR(xdna, "Running under hypervisor not supported");
+		return -EINVAL;
+	}
+
 	ndev = drmm_kzalloc(&xdna->ddev, sizeof(*ndev), GFP_KERNEL);
 	if (!ndev)
 		return -ENOMEM;
diff --git a/drivers/accel/amdxdna/aie2_pci.h b/drivers/accel/amdxdna/aie2_pci.h
index a5f9c42155d1..a929fa98a121 100644
--- a/drivers/accel/amdxdna/aie2_pci.h
+++ b/drivers/accel/amdxdna/aie2_pci.h
@@ -70,6 +70,7 @@ enum psp_reg_idx {
 	PSP_INTR_REG = PSP_NUM_IN_REGS,
 	PSP_STATUS_REG,
 	PSP_RESP_REG,
+	PSP_PWAITMODE_REG,
 	PSP_MAX_REGS /* Keep this at the end */
 };
 
@@ -231,6 +232,7 @@ struct aie2_hw_ops {
 enum aie2_fw_feature {
 	AIE2_NPU_COMMAND,
 	AIE2_PREEMPT,
+	AIE2_TEMPORAL_ONLY,
 	AIE2_FEATURE_MAX
 };
 
@@ -285,11 +287,13 @@ int npu4_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level);
 /* aie2_pm.c */
 int aie2_pm_init(struct amdxdna_dev_hdl *ndev);
 int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type target);
+int aie2_pm_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level);
 
 /* aie2_psp.c */
 struct psp_device *aie2m_psp_create(struct drm_device *ddev, struct psp_config *conf);
 int aie2_psp_start(struct psp_device *psp);
 void aie2_psp_stop(struct psp_device *psp);
+int aie2_psp_waitmode_poll(struct psp_device *psp);
 
 /* aie2_error.c */
 int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev);
diff --git a/drivers/accel/amdxdna/aie2_pm.c b/drivers/accel/amdxdna/aie2_pm.c
index 426c38fce848..afcd6d4683e5 100644
--- a/drivers/accel/amdxdna/aie2_pm.c
+++ b/drivers/accel/amdxdna/aie2_pm.c
@@ -10,6 +10,7 @@
 
 #include "aie2_pci.h"
 #include "amdxdna_pci_drv.h"
+#include "amdxdna_pm.h"
 
 #define AIE2_CLK_GATING_ENABLE	1
 #define AIE2_CLK_GATING_DISABLE	0
@@ -26,6 +27,20 @@ static int aie2_pm_set_clk_gating(struct amdxdna_dev_hdl *ndev, u32 val)
 	return 0;
 }
 
+int aie2_pm_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level)
+{
+	int ret;
+
+	ret = amdxdna_pm_resume_get(ndev->xdna);
+	if (ret)
+		return ret;
+
+	ret = ndev->priv->hw_ops.set_dpm(ndev, dpm_level);
+	amdxdna_pm_suspend_put(ndev->xdna);
+
+	return ret;
+}
+
 int aie2_pm_init(struct amdxdna_dev_hdl *ndev)
 {
 	int ret;
@@ -94,7 +109,7 @@ int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type
 		return -EOPNOTSUPP;
 	}
 
-	ret = ndev->priv->hw_ops.set_dpm(ndev, dpm_level);
+	ret = aie2_pm_set_dpm(ndev, dpm_level);
 	if (ret)
 		return ret;
 
diff --git a/drivers/accel/amdxdna/aie2_psp.c b/drivers/accel/amdxdna/aie2_psp.c
index f28a060a8810..3a7130577e3e 100644
--- a/drivers/accel/amdxdna/aie2_psp.c
+++ b/drivers/accel/amdxdna/aie2_psp.c
@@ -76,6 +76,21 @@ static int psp_exec(struct psp_device *psp, u32 *reg_vals)
 	return 0;
 }
 
+int aie2_psp_waitmode_poll(struct psp_device *psp)
+{
+	struct amdxdna_dev *xdna = to_xdna_dev(psp->ddev);
+	u32 mode_reg;
+	int ret;
+
+	ret = readx_poll_timeout(readl, PSP_REG(psp, PSP_PWAITMODE_REG), mode_reg,
+				 (mode_reg & 0x1) == 1,
+				 PSP_POLL_INTERVAL, PSP_POLL_TIMEOUT);
+	if (ret)
+		XDNA_ERR(xdna, "fw waitmode reg error, ret %d", ret);
+
+	return ret;
+}
+
 void aie2_psp_stop(struct psp_device *psp)
 {
 	u32 reg_vals[PSP_NUM_IN_REGS] = { PSP_RELEASE_TMR, };
diff --git a/drivers/accel/amdxdna/aie2_smu.c b/drivers/accel/amdxdna/aie2_smu.c
index bd94ee96c2bc..2d195e41f83d 100644
--- a/drivers/accel/amdxdna/aie2_smu.c
+++ b/drivers/accel/amdxdna/aie2_smu.c
@@ -11,7 +11,6 @@
 
 #include "aie2_pci.h"
 #include "amdxdna_pci_drv.h"
-#include "amdxdna_pm.h"
 
 #define SMU_RESULT_OK		1
 
@@ -67,16 +66,12 @@ int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level)
 	u32 freq;
 	int ret;
 
-	ret = amdxdna_pm_resume_get(ndev->xdna);
-	if (ret)
-		return ret;
-
 	ret = aie2_smu_exec(ndev, AIE2_SMU_SET_MPNPUCLK_FREQ,
 			    ndev->priv->dpm_clk_tbl[dpm_level].npuclk, &freq);
 	if (ret) {
 		XDNA_ERR(ndev->xdna, "Set npu clock to %d failed, ret %d\n",
 			 ndev->priv->dpm_clk_tbl[dpm_level].npuclk, ret);
-		goto suspend_put;
+		return ret;
 	}
 	ndev->npuclk_freq = freq;
 
@@ -85,10 +80,9 @@ int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level)
 	if (ret) {
 		XDNA_ERR(ndev->xdna, "Set h clock to %d failed, ret %d\n",
 			 ndev->priv->dpm_clk_tbl[dpm_level].hclk, ret);
-		goto suspend_put;
+		return ret;
 	}
 
-	amdxdna_pm_suspend_put(ndev->xdna);
 	ndev->hclk_freq = freq;
 	ndev->dpm_level = dpm_level;
 	ndev->max_tops = 2 * ndev->total_col;
@@ -98,35 +92,26 @@ int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level)
 		 ndev->npuclk_freq, ndev->hclk_freq);
 
 	return 0;
-
-suspend_put:
-	amdxdna_pm_suspend_put(ndev->xdna);
-	return ret;
 }
 
 int npu4_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level)
 {
 	int ret;
 
-	ret = amdxdna_pm_resume_get(ndev->xdna);
-	if (ret)
-		return ret;
-
 	ret = aie2_smu_exec(ndev, AIE2_SMU_SET_HARD_DPMLEVEL, dpm_level, NULL);
 	if (ret) {
 		XDNA_ERR(ndev->xdna, "Set hard dpm level %d failed, ret %d ",
 			 dpm_level, ret);
-		goto suspend_put;
+		return ret;
 	}
 
 	ret = aie2_smu_exec(ndev, AIE2_SMU_SET_SOFT_DPMLEVEL, dpm_level, NULL);
 	if (ret) {
 		XDNA_ERR(ndev->xdna, "Set soft dpm level %d failed, ret %d",
 			 dpm_level, ret);
-		goto suspend_put;
+		return ret;
 	}
 
-	amdxdna_pm_suspend_put(ndev->xdna);
 	ndev->npuclk_freq = ndev->priv->dpm_clk_tbl[dpm_level].npuclk;
 	ndev->hclk_freq = ndev->priv->dpm_clk_tbl[dpm_level].hclk;
 	ndev->dpm_level = dpm_level;
@@ -137,10 +122,6 @@ int npu4_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level)
 		 ndev->npuclk_freq, ndev->hclk_freq);
 
 	return 0;
-
-suspend_put:
-	amdxdna_pm_suspend_put(ndev->xdna);
-	return ret;
 }
 
 int aie2_smu_init(struct amdxdna_dev_hdl *ndev)
diff --git a/drivers/accel/amdxdna/amdxdna_ctx.h b/drivers/accel/amdxdna/amdxdna_ctx.h
index b6151244d64f..b29449a92f60 100644
--- a/drivers/accel/amdxdna/amdxdna_ctx.h
+++ b/drivers/accel/amdxdna/amdxdna_ctx.h
@@ -98,6 +98,7 @@ struct amdxdna_hwctx {
 	u32				*col_list;
 	u32				start_col;
 	u32				num_col;
+	u32				num_unused_col;
 #define HWCTX_STAT_INIT  0
 #define HWCTX_STAT_READY 1
 #define HWCTX_STAT_STOP  2
diff --git a/drivers/accel/amdxdna/amdxdna_mailbox.c b/drivers/accel/amdxdna/amdxdna_mailbox.c
index 858df97cd3fb..469242ed8224 100644
--- a/drivers/accel/amdxdna/amdxdna_mailbox.c
+++ b/drivers/accel/amdxdna/amdxdna_mailbox.c
@@ -112,22 +112,6 @@ static u32 mailbox_reg_read(struct mailbox_channel *mb_chann, u32 mbox_reg)
 	return readl(ringbuf_addr);
 }
 
-static int mailbox_reg_read_non_zero(struct mailbox_channel *mb_chann, u32 mbox_reg, u32 *val)
-{
-	struct xdna_mailbox_res *mb_res = &mb_chann->mb->res;
-	void __iomem *ringbuf_addr = mb_res->mbox_base + mbox_reg;
-	int ret, value;
-
-	/* Poll till value is not zero */
-	ret = readx_poll_timeout(readl, ringbuf_addr, value,
-				 value, 1 /* us */, 100);
-	if (ret < 0)
-		return ret;
-
-	*val = value;
-	return 0;
-}
-
 static inline void
 mailbox_set_headptr(struct mailbox_channel *mb_chann, u32 headptr_val)
 {
@@ -207,26 +191,34 @@ mailbox_send_msg(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg)
 	u32 head, tail;
 	u32 start_addr;
 	u32 tmp_tail;
+	int ret;
 
 	head = mailbox_get_headptr(mb_chann, CHAN_RES_X2I);
 	tail = mb_chann->x2i_tail;
-	ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_X2I);
+	ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_X2I) - sizeof(u32);
 	start_addr = mb_chann->res[CHAN_RES_X2I].rb_start_addr;
 	tmp_tail = tail + mb_msg->pkg_size;
 
-	if (tail < head && tmp_tail >= head)
-		goto no_space;
 
-	if (tail >= head && (tmp_tail > ringbuf_size - sizeof(u32) &&
-			     mb_msg->pkg_size >= head))
-		goto no_space;
-
-	if (tail >= head && tmp_tail > ringbuf_size - sizeof(u32)) {
+check_again:
+	if (tail >= head && tmp_tail > ringbuf_size) {
 		write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail;
 		writel(TOMBSTONE, write_addr);
 
 		/* tombstone is set. Write from the start of the ringbuf */
 		tail = 0;
+		tmp_tail = tail + mb_msg->pkg_size;
+	}
+
+	if (tail < head && tmp_tail >= head) {
+		ret = read_poll_timeout(mailbox_get_headptr, head,
+					tmp_tail < head || tail >= head,
+					1, 100, false, mb_chann, CHAN_RES_X2I);
+		if (ret)
+			return ret;
+
+		if (tail >= head)
+			goto check_again;
 	}
 
 	write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail;
@@ -238,9 +230,6 @@ mailbox_send_msg(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg)
 			    mb_msg->pkg.header.id);
 
 	return 0;
-
-no_space:
-	return -ENOSPC;
 }
 
 static int
@@ -286,8 +275,7 @@ static int mailbox_get_msg(struct mailbox_channel *mb_chann)
 	u32 start_addr;
 	int ret;
 
-	if (mailbox_reg_read_non_zero(mb_chann, mb_chann->res[CHAN_RES_I2X].mb_tail_ptr_reg, &tail))
-		return -EINVAL;
+	tail = mailbox_get_tailptr(mb_chann, CHAN_RES_I2X);
 	head = mb_chann->i2x_head;
 	ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_I2X);
 	start_addr = mb_chann->res[CHAN_RES_I2X].rb_start_addr;
diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.c b/drivers/accel/amdxdna/amdxdna_pci_drv.c
index 1973ab67721b..7d59764d7869 100644
--- a/drivers/accel/amdxdna/amdxdna_pci_drv.c
+++ b/drivers/accel/amdxdna/amdxdna_pci_drv.c
@@ -51,7 +51,6 @@ MODULE_DEVICE_TABLE(pci, pci_ids);
 
 static const struct amdxdna_device_id amdxdna_ids[] = {
 	{ 0x1502, 0x0,  &dev_npu1_info },
-	{ 0x17f0, 0x0,  &dev_npu2_info },
 	{ 0x17f0, 0x10, &dev_npu4_info },
 	{ 0x17f0, 0x11, &dev_npu5_info },
 	{ 0x17f0, 0x20, &dev_npu6_info },
@@ -105,43 +104,38 @@ static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp)
 	return ret;
 }
 
-static void amdxdna_drm_close(struct drm_device *ddev, struct drm_file *filp)
+static void amdxdna_client_cleanup(struct amdxdna_client *client)
 {
-	struct amdxdna_client *client = filp->driver_priv;
-	struct amdxdna_dev *xdna = to_xdna_dev(ddev);
-
-	XDNA_DBG(xdna, "closing pid %d", client->pid);
-
+	list_del(&client->node);
+	amdxdna_hwctx_remove_all(client);
 	xa_destroy(&client->hwctx_xa);
 	cleanup_srcu_struct(&client->hwctx_srcu);
 	mutex_destroy(&client->mm_lock);
+
 	if (client->dev_heap)
 		drm_gem_object_put(to_gobj(client->dev_heap));
 
 	iommu_sva_unbind_device(client->sva);
 
-	XDNA_DBG(xdna, "pid %d closed", client->pid);
 	kfree(client);
 }
 
-static int amdxdna_flush(struct file *f, fl_owner_t id)
+static void amdxdna_drm_close(struct drm_device *ddev, struct drm_file *filp)
 {
-	struct drm_file *filp = f->private_data;
 	struct amdxdna_client *client = filp->driver_priv;
-	struct amdxdna_dev *xdna = client->xdna;
+	struct amdxdna_dev *xdna = to_xdna_dev(ddev);
 	int idx;
 
-	XDNA_DBG(xdna, "PID %d flushing...", client->pid);
+	XDNA_DBG(xdna, "closing pid %d", client->pid);
+
 	if (!drm_dev_enter(&xdna->ddev, &idx))
-		return 0;
+		return;
 
 	mutex_lock(&xdna->dev_lock);
-	list_del_init(&client->node);
-	amdxdna_hwctx_remove_all(client);
+	amdxdna_client_cleanup(client);
 	mutex_unlock(&xdna->dev_lock);
 
 	drm_dev_exit(idx);
-	return 0;
 }
 
 static int amdxdna_drm_get_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
@@ -217,7 +211,6 @@ static const struct file_operations amdxdna_fops = {
 	.owner		= THIS_MODULE,
 	.open		= accel_open,
 	.release	= drm_release,
-	.flush		= amdxdna_flush,
 	.unlocked_ioctl	= drm_ioctl,
 	.compat_ioctl	= drm_compat_ioctl,
 	.poll		= drm_poll,
@@ -333,8 +326,7 @@ static void amdxdna_remove(struct pci_dev *pdev)
 	client = list_first_entry_or_null(&xdna->client_list,
 					  struct amdxdna_client, node);
 	while (client) {
-		list_del_init(&client->node);
-		amdxdna_hwctx_remove_all(client);
+		amdxdna_client_cleanup(client);
 
 		client = list_first_entry_or_null(&xdna->client_list,
 						  struct amdxdna_client, node);
diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.h b/drivers/accel/amdxdna/amdxdna_pci_drv.h
index c99477f5e454..6580cb5ec7e2 100644
--- a/drivers/accel/amdxdna/amdxdna_pci_drv.h
+++ b/drivers/accel/amdxdna/amdxdna_pci_drv.h
@@ -101,7 +101,6 @@ struct amdxdna_dev {
 	struct amdxdna_fw_ver		fw_ver;
 	struct rw_semaphore		notifier_lock; /* for mmu notifier*/
 	struct workqueue_struct		*notifier_wq;
-	bool				rpm_on;
 };
 
 /*
@@ -138,7 +137,6 @@ struct amdxdna_client {
 
 /* Add device info below */
 extern const struct amdxdna_dev_info dev_npu1_info;
-extern const struct amdxdna_dev_info dev_npu2_info;
 extern const struct amdxdna_dev_info dev_npu4_info;
 extern const struct amdxdna_dev_info dev_npu5_info;
 extern const struct amdxdna_dev_info dev_npu6_info;
diff --git a/drivers/accel/amdxdna/amdxdna_pm.c b/drivers/accel/amdxdna/amdxdna_pm.c
index fa38e65d617c..d024d480521c 100644
--- a/drivers/accel/amdxdna/amdxdna_pm.c
+++ b/drivers/accel/amdxdna/amdxdna_pm.c
@@ -15,14 +15,9 @@ int amdxdna_pm_suspend(struct device *dev)
 {
 	struct amdxdna_dev *xdna = to_xdna_dev(dev_get_drvdata(dev));
 	int ret = -EOPNOTSUPP;
-	bool rpm;
 
-	if (xdna->dev_info->ops->suspend) {
-		rpm = xdna->rpm_on;
-		xdna->rpm_on = false;
+	if (xdna->dev_info->ops->suspend)
 		ret = xdna->dev_info->ops->suspend(xdna);
-		xdna->rpm_on = rpm;
-	}
 
 	XDNA_DBG(xdna, "Suspend done ret %d", ret);
 	return ret;
@@ -32,14 +27,9 @@ int amdxdna_pm_resume(struct device *dev)
 {
 	struct amdxdna_dev *xdna = to_xdna_dev(dev_get_drvdata(dev));
 	int ret = -EOPNOTSUPP;
-	bool rpm;
 
-	if (xdna->dev_info->ops->resume) {
-		rpm = xdna->rpm_on;
-		xdna->rpm_on = false;
+	if (xdna->dev_info->ops->resume)
 		ret = xdna->dev_info->ops->resume(xdna);
-		xdna->rpm_on = rpm;
-	}
 
 	XDNA_DBG(xdna, "Resume done ret %d", ret);
 	return ret;
@@ -50,9 +40,6 @@ int amdxdna_pm_resume_get(struct amdxdna_dev *xdna)
 	struct device *dev = xdna->ddev.dev;
 	int ret;
 
-	if (!xdna->rpm_on)
-		return 0;
-
 	ret = pm_runtime_resume_and_get(dev);
 	if (ret) {
 		XDNA_ERR(xdna, "Resume failed: %d", ret);
@@ -66,9 +53,6 @@ void amdxdna_pm_suspend_put(struct amdxdna_dev *xdna)
 {
 	struct device *dev = xdna->ddev.dev;
 
-	if (!xdna->rpm_on)
-		return;
-
 	pm_runtime_put_autosuspend(dev);
 }
 
@@ -81,14 +65,12 @@ void amdxdna_pm_init(struct amdxdna_dev *xdna)
 	pm_runtime_use_autosuspend(dev);
 	pm_runtime_allow(dev);
 	pm_runtime_put_autosuspend(dev);
-	xdna->rpm_on = true;
 }
 
 void amdxdna_pm_fini(struct amdxdna_dev *xdna)
 {
 	struct device *dev = xdna->ddev.dev;
 
-	xdna->rpm_on = false;
 	pm_runtime_get_noresume(dev);
 	pm_runtime_forbid(dev);
 }
diff --git a/drivers/accel/amdxdna/npu1_regs.c b/drivers/accel/amdxdna/npu1_regs.c
index ec407f3b48fc..ebc6e2802297 100644
--- a/drivers/accel/amdxdna/npu1_regs.c
+++ b/drivers/accel/amdxdna/npu1_regs.c
@@ -13,6 +13,7 @@
 #include "amdxdna_pci_drv.h"
 
 /* Address definition from NPU1 docs */
+#define MPNPU_PWAITMODE			0x3010034
 #define MPNPU_PUB_SEC_INTR		0x3010090
 #define MPNPU_PUB_PWRMGMT_INTR		0x3010094
 #define MPNPU_PUB_SCRATCH2		0x30100A0
@@ -92,6 +93,7 @@ static const struct amdxdna_dev_priv npu1_dev_priv = {
 		DEFINE_BAR_OFFSET(PSP_INTR_REG,   NPU1_PSP, MPNPU_PUB_SEC_INTR),
 		DEFINE_BAR_OFFSET(PSP_STATUS_REG, NPU1_PSP, MPNPU_PUB_SCRATCH2),
 		DEFINE_BAR_OFFSET(PSP_RESP_REG,   NPU1_PSP, MPNPU_PUB_SCRATCH3),
+		DEFINE_BAR_OFFSET(PSP_PWAITMODE_REG, NPU1_PSP, MPNPU_PWAITMODE),
 	},
 	.smu_regs_off   = {
 		DEFINE_BAR_OFFSET(SMU_CMD_REG,  NPU1_SMU, MPNPU_PUB_SCRATCH5),
diff --git a/drivers/accel/amdxdna/npu2_regs.c b/drivers/accel/amdxdna/npu2_regs.c
deleted file mode 100644
index 86f87d0d1354..000000000000
--- a/drivers/accel/amdxdna/npu2_regs.c
+++ /dev/null
@@ -1,115 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
- */
-
-#include <drm/amdxdna_accel.h>
-#include <drm/drm_device.h>
-#include <drm/gpu_scheduler.h>
-#include <linux/sizes.h>
-
-#include "aie2_pci.h"
-#include "amdxdna_mailbox.h"
-#include "amdxdna_pci_drv.h"
-
-/* NPU Public Registers on MpNPUAxiXbar (refer to Diag npu_registers.h) */
-#define MPNPU_PUB_SEC_INTR             0x3010060
-#define MPNPU_PUB_PWRMGMT_INTR         0x3010064
-#define MPNPU_PUB_SCRATCH0             0x301006C
-#define MPNPU_PUB_SCRATCH1             0x3010070
-#define MPNPU_PUB_SCRATCH2             0x3010074
-#define MPNPU_PUB_SCRATCH3             0x3010078
-#define MPNPU_PUB_SCRATCH4             0x301007C
-#define MPNPU_PUB_SCRATCH5             0x3010080
-#define MPNPU_PUB_SCRATCH6             0x3010084
-#define MPNPU_PUB_SCRATCH7             0x3010088
-#define MPNPU_PUB_SCRATCH8             0x301008C
-#define MPNPU_PUB_SCRATCH9             0x3010090
-#define MPNPU_PUB_SCRATCH10            0x3010094
-#define MPNPU_PUB_SCRATCH11            0x3010098
-#define MPNPU_PUB_SCRATCH12            0x301009C
-#define MPNPU_PUB_SCRATCH13            0x30100A0
-#define MPNPU_PUB_SCRATCH14            0x30100A4
-#define MPNPU_PUB_SCRATCH15            0x30100A8
-#define MP0_C2PMSG_73                  0x3810A24
-#define MP0_C2PMSG_123                 0x3810AEC
-
-#define MP1_C2PMSG_0                   0x3B10900
-#define MP1_C2PMSG_60                  0x3B109F0
-#define MP1_C2PMSG_61                  0x3B109F4
-
-#define MPNPU_SRAM_X2I_MAILBOX_0       0x3600000
-#define MPNPU_SRAM_X2I_MAILBOX_15      0x361E000
-#define MPNPU_SRAM_X2I_MAILBOX_31      0x363E000
-#define MPNPU_SRAM_I2X_MAILBOX_31      0x363F000
-
-#define MMNPU_APERTURE0_BASE           0x3000000
-#define MMNPU_APERTURE1_BASE           0x3600000
-#define MMNPU_APERTURE3_BASE           0x3810000
-#define MMNPU_APERTURE4_BASE           0x3B10000
-
-/* PCIe BAR Index for NPU2 */
-#define NPU2_REG_BAR_INDEX	0
-#define NPU2_MBOX_BAR_INDEX	0
-#define NPU2_PSP_BAR_INDEX	4
-#define NPU2_SMU_BAR_INDEX	5
-#define NPU2_SRAM_BAR_INDEX	2
-/* Associated BARs and Apertures */
-#define NPU2_REG_BAR_BASE	MMNPU_APERTURE0_BASE
-#define NPU2_MBOX_BAR_BASE	MMNPU_APERTURE0_BASE
-#define NPU2_PSP_BAR_BASE	MMNPU_APERTURE3_BASE
-#define NPU2_SMU_BAR_BASE	MMNPU_APERTURE4_BASE
-#define NPU2_SRAM_BAR_BASE	MMNPU_APERTURE1_BASE
-
-static const struct amdxdna_dev_priv npu2_dev_priv = {
-	.fw_path        = "amdnpu/17f0_00/npu.sbin",
-	.protocol_major = 0x6,
-	.protocol_minor = 0x6,
-	.rt_config	= npu4_default_rt_cfg,
-	.dpm_clk_tbl	= npu4_dpm_clk_table,
-	.fw_feature_tbl = npu4_fw_feature_table,
-	.col_align	= COL_ALIGN_NATURE,
-	.mbox_dev_addr  = NPU2_MBOX_BAR_BASE,
-	.mbox_size      = 0, /* Use BAR size */
-	.sram_dev_addr  = NPU2_SRAM_BAR_BASE,
-	.hwctx_limit    = 16,
-	.sram_offs      = {
-		DEFINE_BAR_OFFSET(MBOX_CHANN_OFF, NPU2_SRAM, MPNPU_SRAM_X2I_MAILBOX_0),
-		DEFINE_BAR_OFFSET(FW_ALIVE_OFF,   NPU2_SRAM, MPNPU_SRAM_X2I_MAILBOX_15),
-	},
-	.psp_regs_off   = {
-		DEFINE_BAR_OFFSET(PSP_CMD_REG,    NPU2_PSP, MP0_C2PMSG_123),
-		DEFINE_BAR_OFFSET(PSP_ARG0_REG,   NPU2_REG, MPNPU_PUB_SCRATCH3),
-		DEFINE_BAR_OFFSET(PSP_ARG1_REG,   NPU2_REG, MPNPU_PUB_SCRATCH4),
-		DEFINE_BAR_OFFSET(PSP_ARG2_REG,   NPU2_REG, MPNPU_PUB_SCRATCH9),
-		DEFINE_BAR_OFFSET(PSP_INTR_REG,   NPU2_PSP, MP0_C2PMSG_73),
-		DEFINE_BAR_OFFSET(PSP_STATUS_REG, NPU2_PSP, MP0_C2PMSG_123),
-		DEFINE_BAR_OFFSET(PSP_RESP_REG,   NPU2_REG, MPNPU_PUB_SCRATCH3),
-	},
-	.smu_regs_off   = {
-		DEFINE_BAR_OFFSET(SMU_CMD_REG,  NPU2_SMU, MP1_C2PMSG_0),
-		DEFINE_BAR_OFFSET(SMU_ARG_REG,  NPU2_SMU, MP1_C2PMSG_60),
-		DEFINE_BAR_OFFSET(SMU_INTR_REG, NPU2_SMU, MMNPU_APERTURE4_BASE),
-		DEFINE_BAR_OFFSET(SMU_RESP_REG, NPU2_SMU, MP1_C2PMSG_61),
-		DEFINE_BAR_OFFSET(SMU_OUT_REG,  NPU2_SMU, MP1_C2PMSG_60),
-	},
-	.hw_ops	=	 {
-		.set_dpm = npu4_set_dpm,
-	},
-};
-
-const struct amdxdna_dev_info dev_npu2_info = {
-	.reg_bar           = NPU2_REG_BAR_INDEX,
-	.mbox_bar          = NPU2_MBOX_BAR_INDEX,
-	.sram_bar          = NPU2_SRAM_BAR_INDEX,
-	.psp_bar           = NPU2_PSP_BAR_INDEX,
-	.smu_bar           = NPU2_SMU_BAR_INDEX,
-	.first_col         = 0,
-	.dev_mem_buf_shift = 15, /* 32 KiB aligned */
-	.dev_mem_base      = AIE2_DEVM_BASE,
-	.dev_mem_size      = AIE2_DEVM_SIZE,
-	.vbnv              = "RyzenAI-npu2",
-	.device_type       = AMDXDNA_DEV_TYPE_KMQ,
-	.dev_priv          = &npu2_dev_priv,
-	.ops               = &aie2_ops, /* NPU2 can share NPU1's callback */
-};
diff --git a/drivers/accel/amdxdna/npu4_regs.c b/drivers/accel/amdxdna/npu4_regs.c
index 986a5f28ba24..a62234fd266d 100644
--- a/drivers/accel/amdxdna/npu4_regs.c
+++ b/drivers/accel/amdxdna/npu4_regs.c
@@ -13,6 +13,7 @@
 #include "amdxdna_pci_drv.h"
 
 /* NPU Public Registers on MpNPUAxiXbar (refer to Diag npu_registers.h) */
+#define MPNPU_PWAITMODE                0x301003C
 #define MPNPU_PUB_SEC_INTR             0x3010060
 #define MPNPU_PUB_PWRMGMT_INTR         0x3010064
 #define MPNPU_PUB_SCRATCH0             0x301006C
@@ -89,6 +90,7 @@ const struct dpm_clk_freq npu4_dpm_clk_table[] = {
 const struct aie2_fw_feature_tbl npu4_fw_feature_table[] = {
 	{ .feature = AIE2_NPU_COMMAND, .min_minor = 15 },
 	{ .feature = AIE2_PREEMPT, .min_minor = 12 },
+	{ .feature = AIE2_TEMPORAL_ONLY, .min_minor = 12 },
 	{ 0 }
 };
 
@@ -116,6 +118,7 @@ static const struct amdxdna_dev_priv npu4_dev_priv = {
 		DEFINE_BAR_OFFSET(PSP_INTR_REG,   NPU4_PSP, MP0_C2PMSG_73),
 		DEFINE_BAR_OFFSET(PSP_STATUS_REG, NPU4_PSP, MP0_C2PMSG_123),
 		DEFINE_BAR_OFFSET(PSP_RESP_REG,   NPU4_REG, MPNPU_PUB_SCRATCH3),
+		DEFINE_BAR_OFFSET(PSP_PWAITMODE_REG, NPU4_REG, MPNPU_PWAITMODE),
 	},
 	.smu_regs_off   = {
 		DEFINE_BAR_OFFSET(SMU_CMD_REG,  NPU4_SMU, MP1_C2PMSG_0),
diff --git a/drivers/accel/amdxdna/npu5_regs.c b/drivers/accel/amdxdna/npu5_regs.c
index 75ad97f0b937..131080652ef0 100644
--- a/drivers/accel/amdxdna/npu5_regs.c
+++ b/drivers/accel/amdxdna/npu5_regs.c
@@ -13,6 +13,7 @@
 #include "amdxdna_pci_drv.h"
 
 /* NPU Public Registers on MpNPUAxiXbar (refer to Diag npu_registers.h) */
+#define MPNPU_PWAITMODE                0x301003C
 #define MPNPU_PUB_SEC_INTR             0x3010060
 #define MPNPU_PUB_PWRMGMT_INTR         0x3010064
 #define MPNPU_PUB_SCRATCH0             0x301006C
@@ -85,6 +86,7 @@ static const struct amdxdna_dev_priv npu5_dev_priv = {
 		DEFINE_BAR_OFFSET(PSP_INTR_REG,   NPU5_PSP, MP0_C2PMSG_73),
 		DEFINE_BAR_OFFSET(PSP_STATUS_REG, NPU5_PSP, MP0_C2PMSG_123),
 		DEFINE_BAR_OFFSET(PSP_RESP_REG,   NPU5_REG, MPNPU_PUB_SCRATCH3),
+		DEFINE_BAR_OFFSET(PSP_PWAITMODE_REG, NPU5_REG, MPNPU_PWAITMODE),
 	},
 	.smu_regs_off   = {
 		DEFINE_BAR_OFFSET(SMU_CMD_REG,  NPU5_SMU, MP1_C2PMSG_0),
diff --git a/drivers/accel/amdxdna/npu6_regs.c b/drivers/accel/amdxdna/npu6_regs.c
index 758dc013fe13..1f71285655b2 100644
--- a/drivers/accel/amdxdna/npu6_regs.c
+++ b/drivers/accel/amdxdna/npu6_regs.c
@@ -13,6 +13,7 @@
 #include "amdxdna_pci_drv.h"
 
 /* NPU Public Registers on MpNPUAxiXbar (refer to Diag npu_registers.h) */
+#define MPNPU_PWAITMODE                0x301003C
 #define MPNPU_PUB_SEC_INTR             0x3010060
 #define MPNPU_PUB_PWRMGMT_INTR         0x3010064
 #define MPNPU_PUB_SCRATCH0             0x301006C
@@ -85,6 +86,7 @@ static const struct amdxdna_dev_priv npu6_dev_priv = {
 		DEFINE_BAR_OFFSET(PSP_INTR_REG,   NPU6_PSP, MP0_C2PMSG_73),
 		DEFINE_BAR_OFFSET(PSP_STATUS_REG, NPU6_PSP, MP0_C2PMSG_123),
 		DEFINE_BAR_OFFSET(PSP_RESP_REG,   NPU6_REG, MPNPU_PUB_SCRATCH3),
+		DEFINE_BAR_OFFSET(PSP_PWAITMODE_REG, NPU6_REG, MPNPU_PWAITMODE),
 	},
 	.smu_regs_off   = {
 		DEFINE_BAR_OFFSET(SMU_CMD_REG,  NPU6_SMU, MP1_C2PMSG_0),
diff --git a/drivers/accel/ivpu/ivpu_gem.c b/drivers/accel/ivpu/ivpu_gem.c
index ece68f570b7e..98b9ce26962b 100644
--- a/drivers/accel/ivpu/ivpu_gem.c
+++ b/drivers/accel/ivpu/ivpu_gem.c
@@ -95,7 +95,7 @@ int __must_check ivpu_bo_bind(struct ivpu_bo *bo)
 
 	if (!bo->mmu_mapped) {
 		drm_WARN_ON(&vdev->drm, !bo->ctx);
-		ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt,
+		ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt, ivpu_bo_size(bo),
 					       ivpu_bo_is_snooped(bo), ivpu_bo_is_read_only(bo));
 		if (ret) {
 			ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
diff --git a/drivers/accel/ivpu/ivpu_mmu_context.c b/drivers/accel/ivpu/ivpu_mmu_context.c
index 87ad593ef47d..c4014c83e727 100644
--- a/drivers/accel/ivpu/ivpu_mmu_context.c
+++ b/drivers/accel/ivpu/ivpu_mmu_context.c
@@ -429,11 +429,12 @@ static void ivpu_mmu_context_unmap_pages(struct ivpu_mmu_context *ctx, u64 vpu_a
 }
 
 int
-ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
-			 u64 vpu_addr, struct sg_table *sgt, bool llc_coherent, bool read_only)
+ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr,
+			 struct sg_table *sgt, size_t bo_size, bool llc_coherent, bool read_only)
 {
 	size_t start_vpu_addr = vpu_addr;
 	struct scatterlist *sg;
+	size_t sgt_size = 0;
 	int ret;
 	u64 prot;
 	u64 i;
@@ -462,12 +463,25 @@ ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
 		ivpu_dbg(vdev, MMU_MAP, "Map ctx: %u dma_addr: 0x%llx vpu_addr: 0x%llx size: %lu\n",
 			 ctx->id, dma_addr, vpu_addr, size);
 
+		if (sgt_size + size > bo_size) {
+			ivpu_err(vdev, "Scatter-gather table size exceeds buffer object size\n");
+			ret = -EINVAL;
+			goto err_unmap_pages;
+		}
+
 		ret = ivpu_mmu_context_map_pages(vdev, ctx, vpu_addr, dma_addr, size, prot);
 		if (ret) {
 			ivpu_err(vdev, "Failed to map context pages\n");
 			goto err_unmap_pages;
 		}
 		vpu_addr += size;
+		sgt_size += size;
+	}
+
+	if (sgt_size < bo_size) {
+		ivpu_err(vdev, "Scatter-gather table size too small to cover buffer object size\n");
+		ret = -EINVAL;
+		goto err_unmap_pages;
 	}
 
 	if (!ctx->is_cd_valid) {
@@ -493,7 +507,7 @@ ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
 	return 0;
 
 err_unmap_pages:
-	ivpu_mmu_context_unmap_pages(ctx, start_vpu_addr, vpu_addr - start_vpu_addr);
+	ivpu_mmu_context_unmap_pages(ctx, start_vpu_addr, sgt_size);
 	mutex_unlock(&ctx->lock);
 	return ret;
 }
diff --git a/drivers/accel/ivpu/ivpu_mmu_context.h b/drivers/accel/ivpu/ivpu_mmu_context.h
index 663a11a9db11..cc02e7bab04e 100644
--- a/drivers/accel/ivpu/ivpu_mmu_context.h
+++ b/drivers/accel/ivpu/ivpu_mmu_context.h
@@ -41,8 +41,9 @@ int ivpu_mmu_context_insert_node(struct ivpu_mmu_context *ctx, const struct ivpu
 				 u64 size, struct drm_mm_node *node);
 void ivpu_mmu_context_remove_node(struct ivpu_mmu_context *ctx, struct drm_mm_node *node);
 
-int ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
-			     u64 vpu_addr, struct sg_table *sgt, bool llc_coherent, bool read_only);
+int
+ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr,
+			 struct sg_table *sgt, size_t bo_size, bool llc_coherent, bool read_only);
 void ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
 				u64 vpu_addr, struct sg_table *sgt);
 int ivpu_mmu_context_set_pages_ro(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
diff --git a/drivers/acpi/acpi_pcc.c b/drivers/acpi/acpi_pcc.c
index 97064e943768..e3f302b9dee5 100644
--- a/drivers/acpi/acpi_pcc.c
+++ b/drivers/acpi/acpi_pcc.c
@@ -52,7 +52,7 @@ acpi_pcc_address_space_setup(acpi_handle region_handle, u32 function,
 	struct pcc_data *data;
 	struct acpi_pcc_info *ctx = handler_context;
 	struct pcc_mbox_chan *pcc_chan;
-	static acpi_status ret;
+	acpi_status ret;
 
 	data = kzalloc(sizeof(*data), GFP_KERNEL);
 	if (!data)
diff --git a/drivers/acpi/acpi_pnp.c b/drivers/acpi/acpi_pnp.c
index 4ad88187dc7a..85d9f78619a2 100644
--- a/drivers/acpi/acpi_pnp.c
+++ b/drivers/acpi/acpi_pnp.c
@@ -125,9 +125,6 @@ static const struct acpi_device_id acpi_pnp_device_ids[] = {
 	{"PNP0401"},		/* ECP Printer Port */
 	/* apple-gmux */
 	{"APP000B"},
-	/* system */
-	{"PNP0c02"},		/* General ID for reserving resources */
-	{"PNP0c01"},		/* memory controller */
 	/* rtc_cmos */
 	{"PNP0b00"},
 	{"PNP0b01"},
@@ -346,24 +343,10 @@ static bool acpi_pnp_match(const char *idstr, const struct acpi_device_id **matc
 	return false;
 }
 
-/*
- * If one of the device IDs below is present in the list of device IDs of a
- * given ACPI device object, the PNP scan handler will not attach to that
- * object, because there is a proper non-PNP driver in the kernel for the
- * device represented by it.
- */
-static const struct acpi_device_id acpi_nonpnp_device_ids[] = {
-	{"INT3F0D"},
-	{"INTC1080"},
-	{"INTC1081"},
-	{"INTC1099"},
-	{""},
-};
-
 static int acpi_pnp_attach(struct acpi_device *adev,
 			   const struct acpi_device_id *id)
 {
-	return !!acpi_match_device_ids(adev, acpi_nonpnp_device_ids);
+	return true;
 }
 
 static struct acpi_scan_handler acpi_pnp_handler = {
diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c
index 3bdeeee3414e..e66e20d1f31b 100644
--- a/drivers/acpi/cppc_acpi.c
+++ b/drivers/acpi/cppc_acpi.c
@@ -1366,7 +1366,8 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps)
 	/* Are any of the regs PCC ?*/
 	if (CPC_IN_PCC(highest_reg) || CPC_IN_PCC(lowest_reg) ||
 		CPC_IN_PCC(lowest_non_linear_reg) || CPC_IN_PCC(nominal_reg) ||
-		CPC_IN_PCC(low_freq_reg) || CPC_IN_PCC(nom_freq_reg)) {
+		CPC_IN_PCC(low_freq_reg) || CPC_IN_PCC(nom_freq_reg) ||
+		CPC_IN_PCC(guaranteed_reg)) {
 		if (pcc_ss_id < 0) {
 			pr_debug("Invalid pcc_ss_id\n");
 			return -ENODEV;
diff --git a/drivers/acpi/device_pm.c b/drivers/acpi/device_pm.c
index 4e0583274b8f..cff3186aa945 100644
--- a/drivers/acpi/device_pm.c
+++ b/drivers/acpi/device_pm.c
@@ -586,8 +586,7 @@ acpi_status acpi_add_pm_notifier(struct acpi_device *adev, struct device *dev,
 		goto out;
 
 	mutex_lock(&acpi_pm_notifier_lock);
-	adev->wakeup.ws = wakeup_source_register(&adev->dev,
-						 dev_name(&adev->dev));
+	adev->wakeup.ws = wakeup_source_register(dev, dev_name(&adev->dev));
 	adev->wakeup.context.dev = dev;
 	adev->wakeup.context.func = func;
 	adev->wakeup.flags.notifier_present = true;
diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c
index 3eb56b77cb6d..e19aba02b800 100644
--- a/drivers/acpi/nfit/core.c
+++ b/drivers/acpi/nfit/core.c
@@ -2,6 +2,7 @@
 /*
  * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
  */
+#include <linux/platform_device.h>
 #include <linux/list_sort.h>
 #include <linux/libnvdimm.h>
 #include <linux/module.h>
@@ -89,15 +90,22 @@ static const guid_t *to_nfit_bus_uuid(int family)
 static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc)
 {
 	struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
+	struct acpi_device *adev;
 
-	/*
-	 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct
-	 * acpi_device.
-	 */
+	/* If provider == 'ACPI.NFIT', a struct acpi_device is there. */
 	if (!nd_desc->provider_name
 			|| strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0)
 		return NULL;
 
+	/*
+	 * But it can be the ACPI companion of acpi_desc->dev when it cones from
+	 * acpi_nfit_probe().
+	 */
+	adev = ACPI_COMPANION(acpi_desc->dev);
+	if (adev)
+		return adev;
+
+	/* Or it is acpi_desc->dev itself when it comes from nfit_ctl_test(). */
 	return to_acpi_device(acpi_desc->dev);
 }
 
@@ -3283,11 +3291,11 @@ static void acpi_nfit_put_table(void *table)
 
 static void acpi_nfit_notify(acpi_handle handle, u32 event, void *data)
 {
-	struct acpi_device *adev = data;
+	struct device *dev = data;
 
-	device_lock(&adev->dev);
-	__acpi_nfit_notify(&adev->dev, handle, event);
-	device_unlock(&adev->dev);
+	device_lock(dev);
+	__acpi_nfit_notify(dev, handle, event);
+	device_unlock(dev);
 }
 
 static void acpi_nfit_remove_notify_handler(void *data)
@@ -3328,18 +3336,19 @@ void acpi_nfit_shutdown(void *data)
 }
 EXPORT_SYMBOL_GPL(acpi_nfit_shutdown);
 
-static int acpi_nfit_add(struct acpi_device *adev)
+static int acpi_nfit_probe(struct platform_device *pdev)
 {
 	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
 	struct acpi_nfit_desc *acpi_desc;
-	struct device *dev = &adev->dev;
+	struct device *dev = &pdev->dev;
+	struct acpi_device *adev = ACPI_COMPANION(dev);
 	struct acpi_table_header *tbl;
 	acpi_status status = AE_OK;
 	acpi_size sz;
 	int rc = 0;
 
 	rc = acpi_dev_install_notify_handler(adev, ACPI_DEVICE_NOTIFY,
-					     acpi_nfit_notify, adev);
+					     acpi_nfit_notify, dev);
 	if (rc)
 		return rc;
 
@@ -3369,7 +3378,7 @@ static int acpi_nfit_add(struct acpi_device *adev)
 	acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
 	if (!acpi_desc)
 		return -ENOMEM;
-	acpi_nfit_desc_init(acpi_desc, &adev->dev);
+	acpi_nfit_desc_init(acpi_desc, dev);
 
 	/* Save the acpi header for exporting the revision via sysfs */
 	acpi_desc->acpi_header = *tbl;
@@ -3474,11 +3483,11 @@ static const struct acpi_device_id acpi_nfit_ids[] = {
 };
 MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids);
 
-static struct acpi_driver acpi_nfit_driver = {
-	.name = KBUILD_MODNAME,
-	.ids = acpi_nfit_ids,
-	.ops = {
-		.add = acpi_nfit_add,
+static struct platform_driver acpi_nfit_driver = {
+	.probe = acpi_nfit_probe,
+	.driver = {
+		.name = "acpi-nfit",
+		.acpi_match_table = acpi_nfit_ids,
 	},
 };
 
@@ -3516,7 +3525,7 @@ static __init int nfit_init(void)
 		return -ENOMEM;
 
 	nfit_mce_register();
-	ret = acpi_bus_register_driver(&acpi_nfit_driver);
+	ret = platform_driver_register(&acpi_nfit_driver);
 	if (ret) {
 		nfit_mce_unregister();
 		destroy_workqueue(nfit_wq);
@@ -3529,7 +3538,7 @@ static __init int nfit_init(void)
 static __exit void nfit_exit(void)
 {
 	nfit_mce_unregister();
-	acpi_bus_unregister_driver(&acpi_nfit_driver);
+	platform_driver_unregister(&acpi_nfit_driver);
 	destroy_workqueue(nfit_wq);
 	WARN_ON(!list_empty(&acpi_descs));
 }
diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c
index 74ade4160314..9d7f85dadc48 100644
--- a/drivers/acpi/pci_root.c
+++ b/drivers/acpi/pci_root.c
@@ -738,7 +738,7 @@ static int acpi_pci_root_add(struct acpi_device *device,
 	if (no_aspm)
 		pcie_no_aspm();
 
-	pci_acpi_add_bus_pm_notifier(device);
+	pci_acpi_add_root_pm_notifier(device, root);
 	device_set_wakeup_capable(root->bus->bridge, device->wakeup.flags.valid);
 
 	if (hotadd) {
diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c
index 416d87f9bd10..8a895d377e21 100644
--- a/drivers/acpi/scan.c
+++ b/drivers/acpi/scan.c
@@ -42,6 +42,7 @@ static LIST_HEAD(acpi_scan_handlers_list);
 DEFINE_MUTEX(acpi_device_lock);
 LIST_HEAD(acpi_wakeup_device_list);
 static DEFINE_MUTEX(acpi_hp_context_lock);
+static LIST_HEAD(acpi_scan_system_dev_list);
 
 /*
  * The UART device described by the SPCR table is the only object which needs
@@ -2203,19 +2204,48 @@ static acpi_status acpi_bus_check_add_2(acpi_handle handle, u32 lvl_not_used,
 	return acpi_bus_check_add(handle, false, (struct acpi_device **)ret_p);
 }
 
+struct acpi_scan_system_dev {
+	struct list_head node;
+	struct acpi_device *adev;
+};
+
+static const char * const acpi_system_dev_ids[] = {
+	"PNP0C01", /* Memory controller */
+	"PNP0C02", /* Motherboard resource */
+	NULL
+};
+
 static void acpi_default_enumeration(struct acpi_device *device)
 {
 	/*
 	 * Do not enumerate devices with enumeration_by_parent flag set as
 	 * they will be enumerated by their respective parents.
 	 */
-	if (!device->flags.enumeration_by_parent) {
-		acpi_create_platform_device(device, NULL);
-		acpi_device_set_enumerated(device);
-	} else {
+	if (device->flags.enumeration_by_parent) {
 		blocking_notifier_call_chain(&acpi_reconfig_chain,
 					     ACPI_RECONFIG_DEVICE_ADD, device);
+		return;
+	}
+	if (match_string(acpi_system_dev_ids, -1, acpi_device_hid(device)) >= 0) {
+		struct acpi_scan_system_dev *sd;
+
+		/*
+		 * This is a generic system device, so there is no need to
+		 * create a platform device for it, but its resources need to be
+		 * reserved.  However, that needs to be done after all of the
+		 * other device objects have been processed and PCI has claimed
+		 * BARs in case there are resource conflicts.
+		 */
+		sd = kmalloc(sizeof(*sd), GFP_KERNEL);
+		if (sd) {
+			sd->adev = device;
+			list_add_tail(&sd->node, &acpi_scan_system_dev_list);
+		}
+	} else {
+		/* For a regular device object, create a platform device. */
+		acpi_create_platform_device(device, NULL);
 	}
+	acpi_device_set_enumerated(device);
 }
 
 static const struct acpi_device_id generic_device_ids[] = {
@@ -2571,6 +2601,88 @@ static void acpi_scan_postponed(void)
 	mutex_unlock(&acpi_dep_list_lock);
 }
 
+static void acpi_scan_claim_resources(struct acpi_device *adev)
+{
+	struct list_head resource_list = LIST_HEAD_INIT(resource_list);
+	struct resource_entry *rentry;
+	unsigned int count = 0;
+	const char *regionid;
+
+	if (acpi_dev_get_resources(adev, &resource_list, NULL, NULL) <= 0)
+		return;
+
+	regionid = kstrdup(dev_name(&adev->dev), GFP_KERNEL);
+	if (!regionid)
+		goto exit;
+
+	list_for_each_entry(rentry, &resource_list, node) {
+		struct resource *res = rentry->res;
+		struct resource *r;
+
+		/* Skip disabled and invalid resources. */
+		if ((res->flags & IORESOURCE_DISABLED) || res->end < res->start)
+			continue;
+
+		if (res->flags & IORESOURCE_IO) {
+			/*
+			 * Follow the PNP system driver and on x86 skip I/O
+			 * resources that start below 0x100 (the "standard PC
+			 * hardware" boundary).
+			 */
+			if (IS_ENABLED(CONFIG_X86) && res->start < 0x100) {
+				dev_info(&adev->dev, "Skipped %pR\n", res);
+				continue;
+			}
+			r = request_region(res->start, resource_size(res), regionid);
+		} else if (res->flags & IORESOURCE_MEM) {
+			r = request_mem_region(res->start, resource_size(res), regionid);
+		} else {
+			continue;
+		}
+
+		if (r) {
+			r->flags &= ~IORESOURCE_BUSY;
+			dev_info(&adev->dev, "Reserved %pR\n", r);
+			count++;
+		} else {
+			/*
+			 * Failures at this point are usually harmless. PCI
+			 * quirks, for example, reserve resources they know
+			 * about too, so there may well be double reservations.
+			 */
+			dev_info(&adev->dev, "Could not reserve %pR\n", res);
+		}
+	}
+
+	if (!count)
+		kfree(regionid);
+
+exit:
+	acpi_dev_free_resource_list(&resource_list);
+}
+
+
+static int __init acpi_reserve_motherboard_resources(void)
+{
+	struct acpi_scan_system_dev *sd, *tmp;
+
+	guard(mutex)(&acpi_scan_lock);
+
+	list_for_each_entry_safe(sd, tmp, &acpi_scan_system_dev_list, node) {
+		acpi_scan_claim_resources(sd->adev);
+		list_del(&sd->node);
+		kfree(sd);
+	}
+
+	return 0;
+}
+
+/*
+ * Reserve motherboard resources after PCI claims BARs, but before PCI assigns
+ * resources for uninitialized PCI devices.
+ */
+fs_initcall(acpi_reserve_motherboard_resources);
+
 /**
  * acpi_bus_scan - Add ACPI device node objects in a given namespace scope.
  * @handle: Root of the namespace scope to scan.
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
index 0b24bd169d61..b96105481784 100644
--- a/drivers/ata/libata-core.c
+++ b/drivers/ata/libata-core.c
@@ -76,18 +76,20 @@ static unsigned int ata_dev_init_params(struct ata_device *dev,
 					u16 heads, u16 sectors);
 static unsigned int ata_dev_set_xfermode(struct ata_device *dev);
 static void ata_dev_xfermask(struct ata_device *dev);
-static unsigned int ata_dev_quirks(const struct ata_device *dev);
+static u64 ata_dev_quirks(const struct ata_device *dev);
+static u64 ata_dev_get_quirk_value(struct ata_device *dev, u64 quirk);
 
 static DEFINE_IDA(ata_ida);
 
 #ifdef CONFIG_ATA_FORCE
 struct ata_force_param {
 	const char	*name;
+	u64		value;
 	u8		cbl;
 	u8		spd_limit;
 	unsigned int	xfer_mask;
-	unsigned int	quirk_on;
-	unsigned int	quirk_off;
+	u64		quirk_on;
+	u64		quirk_off;
 	unsigned int	pflags_on;
 	u16		lflags_on;
 	u16		lflags_off;
@@ -473,19 +475,10 @@ static void ata_force_xfermask(struct ata_device *dev)
 	}
 }
 
-/**
- *	ata_force_quirks - force quirks according to libata.force
- *	@dev: ATA device of interest
- *
- *	Force quirks according to libata.force and whine about it.
- *	For consistency with link selection, device number 15 selects
- *	the first device connected to the host link.
- *
- *	LOCKING:
- *	EH context.
- */
-static void ata_force_quirks(struct ata_device *dev)
+static const struct ata_force_ent *
+ata_force_get_fe_for_dev(struct ata_device *dev)
 {
+	const struct ata_force_ent *fe;
 	int devno = dev->link->pmp + dev->devno;
 	int alt_devno = devno;
 	int i;
@@ -495,8 +488,7 @@ static void ata_force_quirks(struct ata_device *dev)
 		alt_devno += 15;
 
 	for (i = 0; i < ata_force_tbl_size; i++) {
-		const struct ata_force_ent *fe = &ata_force_tbl[i];
-
+		fe = &ata_force_tbl[i];
 		if (fe->port != -1 && fe->port != dev->link->ap->print_id)
 			continue;
 
@@ -504,16 +496,38 @@ static void ata_force_quirks(struct ata_device *dev)
 		    fe->device != alt_devno)
 			continue;
 
-		if (!(~dev->quirks & fe->param.quirk_on) &&
-		    !(dev->quirks & fe->param.quirk_off))
-			continue;
+		return fe;
+	}
 
-		dev->quirks |= fe->param.quirk_on;
-		dev->quirks &= ~fe->param.quirk_off;
+	return NULL;
+}
 
-		ata_dev_notice(dev, "FORCE: modified (%s)\n",
-			       fe->param.name);
-	}
+/**
+ *	ata_force_quirks - force quirks according to libata.force
+ *	@dev: ATA device of interest
+ *
+ *	Force quirks according to libata.force and whine about it.
+ *	For consistency with link selection, device number 15 selects
+ *	the first device connected to the host link.
+ *
+ *	LOCKING:
+ *	EH context.
+ */
+static void ata_force_quirks(struct ata_device *dev)
+{
+	const struct ata_force_ent *fe = ata_force_get_fe_for_dev(dev);
+
+	if (!fe)
+		return;
+
+	if (!(~dev->quirks & fe->param.quirk_on) &&
+	    !(dev->quirks & fe->param.quirk_off))
+		return;
+
+	dev->quirks |= fe->param.quirk_on;
+	dev->quirks &= ~fe->param.quirk_off;
+
+	ata_dev_notice(dev, "FORCE: modified (%s)\n", fe->param.name);
 }
 #else
 static inline void ata_force_pflags(struct ata_port *ap) { }
@@ -2358,6 +2372,24 @@ static bool ata_dev_check_adapter(struct ata_device *dev,
 	return false;
 }
 
+bool ata_adapter_is_online(struct ata_port *ap)
+{
+	struct device *dev;
+
+	if (!ap || !ap->host)
+		return false;
+
+	dev = ap->host->dev;
+	if (!dev)
+		return false;
+
+	if (dev_is_pci(dev) &&
+	    pci_channel_offline(to_pci_dev(dev)))
+		return false;
+
+	return true;
+}
+
 static int ata_dev_config_ncq(struct ata_device *dev,
 			       char *desc, size_t desc_sz)
 {
@@ -3138,17 +3170,10 @@ int ata_dev_configure(struct ata_device *dev)
 		dev->quirks |= ATA_QUIRK_STUCK_ERR;
 	}
 
-	if (dev->quirks & ATA_QUIRK_MAX_SEC_128)
-		dev->max_sectors = min_t(unsigned int, ATA_MAX_SECTORS_128,
-					 dev->max_sectors);
-
-	if (dev->quirks & ATA_QUIRK_MAX_SEC_1024)
-		dev->max_sectors = min_t(unsigned int, ATA_MAX_SECTORS_1024,
-					 dev->max_sectors);
-
-	if (dev->quirks & ATA_QUIRK_MAX_SEC_8191)
-		dev->max_sectors = min_t(unsigned int, ATA_MAX_SECTORS_8191,
-					 dev->max_sectors);
+	if (dev->quirks & ATA_QUIRK_MAX_SEC)
+		dev->max_sectors = min_t(unsigned int, dev->max_sectors,
+					 ata_dev_get_quirk_value(dev,
+							 ATA_QUIRK_MAX_SEC));
 
 	if (dev->quirks & ATA_QUIRK_MAX_SEC_LBA48)
 		dev->max_sectors = ATA_MAX_SECTORS_LBA48;
@@ -3980,7 +4005,6 @@ static const char * const ata_quirk_names[] = {
 	[__ATA_QUIRK_DIAGNOSTIC]	= "diagnostic",
 	[__ATA_QUIRK_NODMA]		= "nodma",
 	[__ATA_QUIRK_NONCQ]		= "noncq",
-	[__ATA_QUIRK_MAX_SEC_128]	= "maxsec128",
 	[__ATA_QUIRK_BROKEN_HPA]	= "brokenhpa",
 	[__ATA_QUIRK_DISABLE]		= "disable",
 	[__ATA_QUIRK_HPA_SIZE]		= "hpasize",
@@ -4001,8 +4025,7 @@ static const char * const ata_quirk_names[] = {
 	[__ATA_QUIRK_ZERO_AFTER_TRIM]	= "zeroaftertrim",
 	[__ATA_QUIRK_NO_DMA_LOG]	= "nodmalog",
 	[__ATA_QUIRK_NOTRIM]		= "notrim",
-	[__ATA_QUIRK_MAX_SEC_1024]	= "maxsec1024",
-	[__ATA_QUIRK_MAX_SEC_8191]	= "maxsec8191",
+	[__ATA_QUIRK_MAX_SEC]		= "maxsec",
 	[__ATA_QUIRK_MAX_TRIM_128M]	= "maxtrim128m",
 	[__ATA_QUIRK_NO_NCQ_ON_ATI]	= "noncqonati",
 	[__ATA_QUIRK_NO_LPM_ON_ATI]	= "nolpmonati",
@@ -4047,10 +4070,25 @@ static void ata_dev_print_quirks(const struct ata_device *dev,
 	kfree(str);
 }
 
+struct ata_dev_quirk_value {
+	const char	*model_num;
+	const char	*model_rev;
+	u64		val;
+};
+
+static const struct ata_dev_quirk_value __ata_dev_max_sec_quirks[] = {
+	{ "TORiSAN DVD-ROM DRD-N216",	NULL,		128 },
+	{ "ST380013AS",			"3.20",		1024 },
+	{ "LITEON CX1-JB*-HP",		NULL,		1024 },
+	{ "LITEON EP1-*",		NULL,		1024 },
+	{ "DELLBOSS VD",		"MV.R00-0",	8191 },
+	{ },
+};
+
 struct ata_dev_quirks_entry {
 	const char *model_num;
 	const char *model_rev;
-	unsigned int quirks;
+	u64 quirks;
 };
 
 static const struct ata_dev_quirks_entry __ata_dev_quirks[] = {
@@ -4091,7 +4129,7 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = {
 	{ "ASMT109x- Config",	NULL,		ATA_QUIRK_DISABLE },
 
 	/* Weird ATAPI devices */
-	{ "TORiSAN DVD-ROM DRD-N216", NULL,	ATA_QUIRK_MAX_SEC_128 },
+	{ "TORiSAN DVD-ROM DRD-N216", NULL,	ATA_QUIRK_MAX_SEC },
 	{ "QUANTUM DAT    DAT72-000", NULL,	ATA_QUIRK_ATAPI_MOD16_DMA },
 	{ "Slimtype DVD A  DS8A8SH", NULL,	ATA_QUIRK_MAX_SEC_LBA48 },
 	{ "Slimtype DVD A  DS8A9SH", NULL,	ATA_QUIRK_MAX_SEC_LBA48 },
@@ -4100,20 +4138,20 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = {
 	 * Causes silent data corruption with higher max sects.
 	 * http://lkml.kernel.org/g/x49wpy40ysk.fsf@segfault.boston.devel.redhat.com
 	 */
-	{ "ST380013AS",		"3.20",		ATA_QUIRK_MAX_SEC_1024 },
+	{ "ST380013AS",		"3.20",		ATA_QUIRK_MAX_SEC },
 
 	/*
 	 * These devices time out with higher max sects.
 	 * https://bugzilla.kernel.org/show_bug.cgi?id=121671
 	 */
-	{ "LITEON CX1-JB*-HP",	NULL,		ATA_QUIRK_MAX_SEC_1024 },
-	{ "LITEON EP1-*",	NULL,		ATA_QUIRK_MAX_SEC_1024 },
+	{ "LITEON CX1-JB*-HP",	NULL,		ATA_QUIRK_MAX_SEC },
+	{ "LITEON EP1-*",	NULL,		ATA_QUIRK_MAX_SEC },
 
 	/*
 	 * These devices time out with higher max sects.
 	 * https://bugzilla.kernel.org/show_bug.cgi?id=220693
 	 */
-	{ "DELLBOSS VD",	"MV.R00-0",	ATA_QUIRK_MAX_SEC_8191 },
+	{ "DELLBOSS VD",	"MV.R00-0",	ATA_QUIRK_MAX_SEC },
 
 	/* Devices we expect to fail diagnostics */
 
@@ -4339,14 +4377,14 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = {
 	{ }
 };
 
-static unsigned int ata_dev_quirks(const struct ata_device *dev)
+static u64 ata_dev_quirks(const struct ata_device *dev)
 {
 	unsigned char model_num[ATA_ID_PROD_LEN + 1];
 	unsigned char model_rev[ATA_ID_FW_REV_LEN + 1];
 	const struct ata_dev_quirks_entry *ad = __ata_dev_quirks;
 
-	/* dev->quirks is an unsigned int. */
-	BUILD_BUG_ON(__ATA_QUIRK_MAX > 32);
+	/* dev->quirks is an u64. */
+	BUILD_BUG_ON(__ATA_QUIRK_MAX > 64);
 
 	ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
 	ata_id_c_string(dev->id, model_rev, ATA_ID_FW_REV, sizeof(model_rev));
@@ -4363,6 +4401,48 @@ static unsigned int ata_dev_quirks(const struct ata_device *dev)
 	return 0;
 }
 
+static u64 ata_dev_get_max_sec_quirk_value(struct ata_device *dev)
+{
+	unsigned char model_num[ATA_ID_PROD_LEN + 1];
+	unsigned char model_rev[ATA_ID_FW_REV_LEN + 1];
+	const struct ata_dev_quirk_value *ad = __ata_dev_max_sec_quirks;
+	u64 val = 0;
+
+#ifdef CONFIG_ATA_FORCE
+	const struct ata_force_ent *fe = ata_force_get_fe_for_dev(dev);
+	if (fe && (fe->param.quirk_on & ATA_QUIRK_MAX_SEC) && fe->param.value)
+		val = fe->param.value;
+#endif
+	if (val)
+		goto out;
+
+	ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
+	ata_id_c_string(dev->id, model_rev, ATA_ID_FW_REV, sizeof(model_rev));
+
+	while (ad->model_num) {
+		if (glob_match(ad->model_num, model_num) &&
+		    (!ad->model_rev || glob_match(ad->model_rev, model_rev))) {
+			val = ad->val;
+			break;
+		}
+		ad++;
+	}
+
+out:
+	ata_dev_warn(dev, "%s quirk is using value: %llu\n",
+		     ata_quirk_names[__ATA_QUIRK_MAX_SEC], val);
+
+	return val;
+}
+
+static u64 ata_dev_get_quirk_value(struct ata_device *dev, u64 quirk)
+{
+	if (quirk == ATA_QUIRK_MAX_SEC)
+		return ata_dev_get_max_sec_quirk_value(dev);
+
+	return 0;
+}
+
 static bool ata_dev_nodma(const struct ata_device *dev)
 {
 	/*
@@ -5073,6 +5153,12 @@ void ata_qc_issue(struct ata_queued_cmd *qc)
 	qc->flags |= ATA_QCFLAG_ACTIVE;
 	ap->qc_active |= 1ULL << qc->tag;
 
+	/* Make sure the device is still accessible. */
+	if (!ata_adapter_is_online(ap)) {
+		qc->err_mask |= AC_ERR_HOST_BUS;
+		goto sys_err;
+	}
+
 	/*
 	 * We guarantee to LLDs that they will have at least one
 	 * non-zero sg if the command is a data command.
@@ -6396,10 +6482,21 @@ EXPORT_SYMBOL_GPL(ata_platform_remove_one);
 #define force_quirk_on(name, flag)			\
 	{ #name,	.quirk_on	= (flag) }
 
+#define force_quirk_val(name, flag, val)		\
+	{ #name,	.quirk_on	= (flag),	\
+			.value		= (val) }
+
 #define force_quirk_onoff(name, flag)			\
 	{ "no" #name,	.quirk_on	= (flag) },	\
 	{ #name,	.quirk_off	= (flag) }
 
+/*
+ * If the ata_force_param struct member 'name' ends with '=', then the value
+ * after the equal sign will be parsed as an u64, and will be saved in the
+ * ata_force_param struct member 'value'. This works because each libata.force
+ * entry (struct ata_force_ent) is separated by commas, so each entry represents
+ * a single quirk, and can thus only have a single value.
+ */
 static const struct ata_force_param force_tbl[] __initconst = {
 	force_cbl(40c,			ATA_CBL_PATA40),
 	force_cbl(80c,			ATA_CBL_PATA80),
@@ -6470,8 +6567,9 @@ static const struct ata_force_param force_tbl[] __initconst = {
 	force_quirk_onoff(iddevlog,	ATA_QUIRK_NO_ID_DEV_LOG),
 	force_quirk_onoff(logdir,	ATA_QUIRK_NO_LOG_DIR),
 
-	force_quirk_on(max_sec_128,	ATA_QUIRK_MAX_SEC_128),
-	force_quirk_on(max_sec_1024,	ATA_QUIRK_MAX_SEC_1024),
+	force_quirk_val(max_sec_128,	ATA_QUIRK_MAX_SEC,	128),
+	force_quirk_val(max_sec_1024,	ATA_QUIRK_MAX_SEC,	1024),
+	force_quirk_on(max_sec=,	ATA_QUIRK_MAX_SEC),
 	force_quirk_on(max_sec_lba48,	ATA_QUIRK_MAX_SEC_LBA48),
 
 	force_quirk_onoff(lpm,		ATA_QUIRK_NOLPM),
@@ -6487,8 +6585,9 @@ static int __init ata_parse_force_one(char **cur,
 				      const char **reason)
 {
 	char *start = *cur, *p = *cur;
-	char *id, *val, *endp;
+	char *id, *val, *endp, *equalsign, *char_after_equalsign;
 	const struct ata_force_param *match_fp = NULL;
+	u64 val_after_equalsign;
 	int nr_matches = 0, i;
 
 	/* find where this param ends and update *cur */
@@ -6531,10 +6630,36 @@ static int __init ata_parse_force_one(char **cur,
 	}
 
  parse_val:
-	/* parse val, allow shortcuts so that both 1.5 and 1.5Gbps work */
+	equalsign = strchr(val, '=');
+	if (equalsign) {
+		char_after_equalsign = equalsign + 1;
+		if (!strlen(char_after_equalsign) ||
+		    kstrtoull(char_after_equalsign, 10, &val_after_equalsign)) {
+			*reason = "invalid value after equal sign";
+			return -EINVAL;
+		}
+	}
+
+	/* Parse the parameter value. */
 	for (i = 0; i < ARRAY_SIZE(force_tbl); i++) {
 		const struct ata_force_param *fp = &force_tbl[i];
 
+		/*
+		 * If val contains equal sign, match has to be exact, i.e.
+		 * shortcuts are not supported.
+		 */
+		if (equalsign &&
+		    (strncasecmp(val, fp->name,
+				 char_after_equalsign - val) == 0)) {
+			force_ent->param = *fp;
+			force_ent->param.value = val_after_equalsign;
+			return 0;
+		}
+
+		/*
+		 * If val does not contain equal sign, allow shortcuts so that
+		 * both 1.5 and 1.5Gbps work.
+		 */
 		if (strncasecmp(val, fp->name, strlen(val)))
 			continue;
 
diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c
index 2586e77ebf45..f4c9541d1910 100644
--- a/drivers/ata/libata-eh.c
+++ b/drivers/ata/libata-eh.c
@@ -736,7 +736,8 @@ void ata_scsi_port_error_handler(struct Scsi_Host *host, struct ata_port *ap)
 	spin_unlock_irqrestore(ap->lock, flags);
 
 	/* invoke EH, skip if unloading or suspended */
-	if (!(ap->pflags & (ATA_PFLAG_UNLOADING | ATA_PFLAG_SUSPENDED)))
+	if (!(ap->pflags & (ATA_PFLAG_UNLOADING | ATA_PFLAG_SUSPENDED)) &&
+	    ata_adapter_is_online(ap))
 		ap->ops->error_handler(ap);
 	else {
 		/* if unloading, commence suicide */
diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
index 721d3f270c8e..d388a4ff9ae4 100644
--- a/drivers/ata/libata-scsi.c
+++ b/drivers/ata/libata-scsi.c
@@ -2982,6 +2982,9 @@ ata_scsi_find_dev(struct ata_port *ap, const struct scsi_device *scsidev)
 {
 	struct ata_device *dev = __ata_scsi_find_dev(ap, scsidev);
 
+	if (!ata_adapter_is_online(ap))
+		return NULL;
+
 	if (unlikely(!dev || !ata_dev_enabled(dev)))
 		return NULL;
 
@@ -3573,13 +3576,13 @@ static void ata_scsi_report_zones_complete(struct ata_queued_cmd *qc)
 {
 	struct scsi_cmnd *scmd = qc->scsicmd;
 	struct sg_mapping_iter miter;
-	unsigned long flags;
 	unsigned int bytes = 0;
 
+	lockdep_assert_held(qc->ap->lock);
+
 	sg_miter_start(&miter, scsi_sglist(scmd), scsi_sg_count(scmd),
 		       SG_MITER_TO_SG | SG_MITER_ATOMIC);
 
-	local_irq_save(flags);
 	while (sg_miter_next(&miter)) {
 		unsigned int offset = 0;
 
@@ -3627,7 +3630,6 @@ static void ata_scsi_report_zones_complete(struct ata_queued_cmd *qc)
 		}
 	}
 	sg_miter_stop(&miter);
-	local_irq_restore(flags);
 
 	ata_scsi_qc_complete(qc);
 }
diff --git a/drivers/ata/libata.h b/drivers/ata/libata.h
index 0e7ecac73680..89dd0ae2b991 100644
--- a/drivers/ata/libata.h
+++ b/drivers/ata/libata.h
@@ -94,6 +94,7 @@ extern int atapi_check_dma(struct ata_queued_cmd *qc);
 extern void swap_buf_le16(u16 *buf, unsigned int buf_words);
 extern bool ata_phys_link_online(struct ata_link *link);
 extern bool ata_phys_link_offline(struct ata_link *link);
+bool ata_adapter_is_online(struct ata_port *ap);
 extern void ata_dev_init(struct ata_device *dev);
 extern void ata_link_init(struct ata_port *ap, struct ata_link *link, int pmp);
 extern int sata_link_init_spd(struct ata_link *link);
diff --git a/drivers/atm/nicstar.c b/drivers/atm/nicstar.c
index 45952cfea06b..bc8dbba77b87 100644
--- a/drivers/atm/nicstar.c
+++ b/drivers/atm/nicstar.c
@@ -43,6 +43,7 @@
 #include <linux/types.h>
 #include <linux/string.h>
 #include <linux/delay.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/sched.h>
 #include <linux/timer.h>
diff --git a/drivers/auxdisplay/hd44780_common.c b/drivers/auxdisplay/hd44780_common.c
index 1792fe2a4460..b71db39f9249 100644
--- a/drivers/auxdisplay/hd44780_common.c
+++ b/drivers/auxdisplay/hd44780_common.c
@@ -1,4 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
+#include <linux/hex.h>
 #include <linux/module.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
diff --git a/drivers/auxdisplay/lcd2s.c b/drivers/auxdisplay/lcd2s.c
index 045dbef49dee..defb0573e43c 100644
--- a/drivers/auxdisplay/lcd2s.c
+++ b/drivers/auxdisplay/lcd2s.c
@@ -11,6 +11,7 @@
  *  Author: Lars Pöschel <poeschel@lemonage.de>
  *  All rights reserved.
  */
+#include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
index 97a8b4fcf471..189de5250f25 100644
--- a/drivers/base/power/main.c
+++ b/drivers/base/power/main.c
@@ -1647,10 +1647,11 @@ static void device_suspend_late(struct device *dev, pm_message_t state, bool asy
 		goto Complete;
 
 	/*
-	 * Disable runtime PM for the device without checking if there is a
-	 * pending resume request for it.
+	 * After this point, any runtime PM operations targeting the device
+	 * will fail until the corresponding pm_runtime_enable() call in
+	 * device_resume_early().
 	 */
-	__pm_runtime_disable(dev, false);
+	pm_runtime_disable(dev);
 
 	if (dev->power.syscore)
 		goto Skip;
diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c
index 84676cc24221..0ee8ea971aa4 100644
--- a/drivers/base/power/runtime.c
+++ b/drivers/base/power/runtime.c
@@ -1868,16 +1868,18 @@ void pm_runtime_init(struct device *dev)
  */
 void pm_runtime_reinit(struct device *dev)
 {
-	if (!pm_runtime_enabled(dev)) {
-		if (dev->power.runtime_status == RPM_ACTIVE)
-			pm_runtime_set_suspended(dev);
-		if (dev->power.irq_safe) {
-			spin_lock_irq(&dev->power.lock);
-			dev->power.irq_safe = 0;
-			spin_unlock_irq(&dev->power.lock);
-			if (dev->parent)
-				pm_runtime_put(dev->parent);
-		}
+	if (pm_runtime_enabled(dev))
+		return;
+
+	if (dev->power.runtime_status == RPM_ACTIVE)
+		pm_runtime_set_suspended(dev);
+
+	if (dev->power.irq_safe) {
+		spin_lock_irq(&dev->power.lock);
+		dev->power.irq_safe = 0;
+		spin_unlock_irq(&dev->power.lock);
+		if (dev->parent)
+			pm_runtime_put(dev->parent);
 	}
 	/*
 	 * Clear power.needs_force_resume in case it has been set by
diff --git a/drivers/base/transport_class.c b/drivers/base/transport_class.c
index 09ee2a1e35bb..69c6ac2e8263 100644
--- a/drivers/base/transport_class.c
+++ b/drivers/base/transport_class.c
@@ -169,6 +169,12 @@ static int transport_add_class_device(struct attribute_container *cont,
 			goto err_del;
 	}
 
+	if (tcont->encryption) {
+		error = sysfs_create_group(&classdev->kobj, tcont->encryption);
+		if (error)
+			goto err_del;
+	}
+
 	return 0;
 
 err_del:
@@ -244,6 +250,8 @@ static int transport_remove_classdev(struct attribute_container *cont,
 	if (tclass->remove != anon_transport_dummy_function) {
 		if (tcont->statistics)
 			sysfs_remove_group(&classdev->kobj, tcont->statistics);
+		if (tcont->encryption)
+			sysfs_remove_group(&classdev->kobj, tcont->encryption);
 		attribute_container_class_device_del(classdev);
 	}
 
diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c
index c28786e0fe1c..92e446a64371 100644
--- a/drivers/block/floppy.c
+++ b/drivers/block/floppy.c
@@ -4802,8 +4802,6 @@ static void floppy_release_allocated_regions(int fdc, const struct io_region *p)
 	}
 }
 
-#define ARRAY_END(X) (&((X)[ARRAY_SIZE(X)]))
-
 static int floppy_request_regions(int fdc)
 {
 	const struct io_region *p;
diff --git a/drivers/block/loop.c b/drivers/block/loop.c
index 272bc608e528..32a3a5b13802 100644
--- a/drivers/block/loop.c
+++ b/drivers/block/loop.c
@@ -1082,7 +1082,7 @@ static int loop_configure(struct loop_device *lo, blk_mode_t mode,
 	/* Order wrt reading lo_state in loop_validate_file(). */
 	wmb();
 
-	lo->lo_state = Lo_bound;
+	WRITE_ONCE(lo->lo_state, Lo_bound);
 	if (part_shift)
 		lo->lo_flags |= LO_FLAGS_PARTSCAN;
 	partscan = lo->lo_flags & LO_FLAGS_PARTSCAN;
@@ -1179,7 +1179,7 @@ static void __loop_clr_fd(struct loop_device *lo)
 	if (!part_shift)
 		set_bit(GD_SUPPRESS_PART_SCAN, &lo->lo_disk->state);
 	mutex_lock(&lo->lo_mutex);
-	lo->lo_state = Lo_unbound;
+	WRITE_ONCE(lo->lo_state, Lo_unbound);
 	mutex_unlock(&lo->lo_mutex);
 
 	/*
@@ -1218,7 +1218,7 @@ static int loop_clr_fd(struct loop_device *lo)
 
 	lo->lo_flags |= LO_FLAGS_AUTOCLEAR;
 	if (disk_openers(lo->lo_disk) == 1)
-		lo->lo_state = Lo_rundown;
+		WRITE_ONCE(lo->lo_state, Lo_rundown);
 	loop_global_unlock(lo, true);
 
 	return 0;
@@ -1743,7 +1743,7 @@ static void lo_release(struct gendisk *disk)
 
 	mutex_lock(&lo->lo_mutex);
 	if (lo->lo_state == Lo_bound && (lo->lo_flags & LO_FLAGS_AUTOCLEAR))
-		lo->lo_state = Lo_rundown;
+		WRITE_ONCE(lo->lo_state, Lo_rundown);
 
 	need_clear = (lo->lo_state == Lo_rundown);
 	mutex_unlock(&lo->lo_mutex);
@@ -1858,7 +1858,7 @@ static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx,
 
 	blk_mq_start_request(rq);
 
-	if (lo->lo_state != Lo_bound)
+	if (data_race(READ_ONCE(lo->lo_state)) != Lo_bound)
 		return BLK_STS_IOERR;
 
 	switch (req_op(rq)) {
@@ -2016,7 +2016,7 @@ static int loop_add(int i)
 	lo->worker_tree = RB_ROOT;
 	INIT_LIST_HEAD(&lo->idle_worker_list);
 	timer_setup(&lo->timer, loop_free_idle_workers_timer, TIMER_DEFERRABLE);
-	lo->lo_state = Lo_unbound;
+	WRITE_ONCE(lo->lo_state, Lo_unbound);
 
 	err = mutex_lock_killable(&loop_ctl_mutex);
 	if (err)
@@ -2174,7 +2174,7 @@ static int loop_control_remove(int idx)
 		goto mark_visible;
 	}
 	/* Mark this loop device as no more bound, but not quite unbound yet */
-	lo->lo_state = Lo_deleting;
+	WRITE_ONCE(lo->lo_state, Lo_deleting);
 	mutex_unlock(&lo->lo_mutex);
 
 	loop_remove(lo);
@@ -2197,8 +2197,12 @@ static int loop_control_get_free(int idx)
 	if (ret)
 		return ret;
 	idr_for_each_entry(&loop_index_idr, lo, id) {
-		/* Hitting a race results in creating a new loop device which is harmless. */
-		if (lo->idr_visible && data_race(lo->lo_state) == Lo_unbound)
+		/*
+		 * Hitting a race results in creating a new loop device
+		 * which is harmless.
+		 */
+		if (lo->idr_visible &&
+		    data_race(READ_ONCE(lo->lo_state)) == Lo_unbound)
 			goto found;
 	}
 	mutex_unlock(&loop_ctl_mutex);
diff --git a/drivers/block/rnbd/rnbd-clt.c b/drivers/block/rnbd/rnbd-clt.c
index f1409e54010a..d1c354636315 100644
--- a/drivers/block/rnbd/rnbd-clt.c
+++ b/drivers/block/rnbd/rnbd-clt.c
@@ -1423,9 +1423,11 @@ static struct rnbd_clt_dev *init_dev(struct rnbd_clt_session *sess,
 		goto out_alloc;
 	}
 
-	ret = ida_alloc_max(&index_ida, (1 << (MINORBITS - RNBD_PART_BITS)) - 1,
-			    GFP_KERNEL);
-	if (ret < 0) {
+	dev->clt_device_id = ida_alloc_max(&index_ida,
+					   (1 << (MINORBITS - RNBD_PART_BITS)) - 1,
+					   GFP_KERNEL);
+	if (dev->clt_device_id < 0) {
+		ret = dev->clt_device_id;
 		pr_err("Failed to initialize device '%s' from session %s, allocating idr failed, err: %d\n",
 		       pathname, sess->sessname, ret);
 		goto out_queues;
@@ -1434,10 +1436,9 @@ static struct rnbd_clt_dev *init_dev(struct rnbd_clt_session *sess,
 	dev->pathname = kstrdup(pathname, GFP_KERNEL);
 	if (!dev->pathname) {
 		ret = -ENOMEM;
-		goto out_queues;
+		goto out_ida;
 	}
 
-	dev->clt_device_id	= ret;
 	dev->sess		= sess;
 	dev->access_mode	= access_mode;
 	dev->nr_poll_queues	= nr_poll_queues;
@@ -1453,6 +1454,8 @@ static struct rnbd_clt_dev *init_dev(struct rnbd_clt_session *sess,
 
 	return dev;
 
+out_ida:
+	ida_free(&index_ida, dev->clt_device_id);
 out_queues:
 	kfree(dev->hw_queues);
 out_alloc:
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index df9831783a13..cfd2132410dd 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -1080,12 +1080,20 @@ static inline struct ublk_uring_cmd_pdu *ublk_get_uring_cmd_pdu(
 	return io_uring_cmd_to_pdu(ioucmd, struct ublk_uring_cmd_pdu);
 }
 
+static void ublk_end_request(struct request *req, blk_status_t error)
+{
+	local_bh_disable();
+	blk_mq_end_request(req, error);
+	local_bh_enable();
+}
+
 /* todo: handle partial completion */
 static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,
 				      bool need_map)
 {
 	unsigned int unmapped_bytes;
 	blk_status_t res = BLK_STS_OK;
+	bool requeue;
 
 	/* failed read IO if nothing is read */
 	if (!io->res && req_op(req) == REQ_OP_READ)
@@ -1117,14 +1125,30 @@ static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,
 	if (unlikely(unmapped_bytes < io->res))
 		io->res = unmapped_bytes;
 
-	if (blk_update_request(req, BLK_STS_OK, io->res))
+	/*
+	 * Run bio->bi_end_io() with softirqs disabled. If the final fput
+	 * happens off this path, then that will prevent ublk's blkdev_release()
+	 * from being called on current's task work, see fput() implementation.
+	 *
+	 * Otherwise, ublk server may not provide forward progress in case of
+	 * reading the partition table from bdev_open() with disk->open_mutex
+	 * held, and causes dead lock as we could already be holding
+	 * disk->open_mutex here.
+	 *
+	 * Preferably we would not be doing IO with a mutex held that is also
+	 * used for release, but this work-around will suffice for now.
+	 */
+	local_bh_disable();
+	requeue = blk_update_request(req, BLK_STS_OK, io->res);
+	local_bh_enable();
+	if (requeue)
 		blk_mq_requeue_request(req, true);
 	else if (likely(!blk_should_fake_timeout(req->q)))
 		__blk_mq_end_request(req, BLK_STS_OK);
 
 	return;
 exit:
-	blk_mq_end_request(req, res);
+	ublk_end_request(req, res);
 }
 
 static struct io_uring_cmd *__ublk_prep_compl_io_cmd(struct ublk_io *io,
@@ -1164,7 +1188,7 @@ static inline void __ublk_abort_rq(struct ublk_queue *ubq,
 	if (ublk_nosrv_dev_should_queue_io(ubq->dev))
 		blk_mq_requeue_request(rq, false);
 	else
-		blk_mq_end_request(rq, BLK_STS_IOERR);
+		ublk_end_request(rq, BLK_STS_IOERR);
 }
 
 static void
@@ -1209,7 +1233,7 @@ __ublk_do_auto_buf_reg(const struct ublk_queue *ubq, struct request *req,
 			ublk_auto_buf_reg_fallback(ubq, req->tag);
 			return AUTO_BUF_REG_FALLBACK;
 		}
-		blk_mq_end_request(req, BLK_STS_IOERR);
+		ublk_end_request(req, BLK_STS_IOERR);
 		return AUTO_BUF_REG_FAIL;
 	}
 
diff --git a/drivers/block/zloop.c b/drivers/block/zloop.c
index 77bd6081b244..8e334f5025fc 100644
--- a/drivers/block/zloop.c
+++ b/drivers/block/zloop.c
@@ -697,7 +697,7 @@ static blk_status_t zloop_queue_rq(struct blk_mq_hw_ctx *hctx,
 	struct zloop_cmd *cmd = blk_mq_rq_to_pdu(rq);
 	struct zloop_device *zlo = rq->q->queuedata;
 
-	if (zlo->state == Zlo_deleting)
+	if (data_race(READ_ONCE(zlo->state)) == Zlo_deleting)
 		return BLK_STS_IOERR;
 
 	/*
@@ -1002,7 +1002,7 @@ static int zloop_ctl_add(struct zloop_options *opts)
 		ret = -ENOMEM;
 		goto out;
 	}
-	zlo->state = Zlo_creating;
+	WRITE_ONCE(zlo->state, Zlo_creating);
 
 	ret = mutex_lock_killable(&zloop_ctl_mutex);
 	if (ret)
@@ -1113,7 +1113,7 @@ static int zloop_ctl_add(struct zloop_options *opts)
 	}
 
 	mutex_lock(&zloop_ctl_mutex);
-	zlo->state = Zlo_live;
+	WRITE_ONCE(zlo->state, Zlo_live);
 	mutex_unlock(&zloop_ctl_mutex);
 
 	pr_info("zloop: device %d, %u zones of %llu MiB, %u B block size\n",
@@ -1177,7 +1177,7 @@ static int zloop_ctl_remove(struct zloop_options *opts)
 		ret = -EINVAL;
 	} else {
 		idr_remove(&zloop_index_idr, zlo->id);
-		zlo->state = Zlo_deleting;
+		WRITE_ONCE(zlo->state, Zlo_deleting);
 	}
 
 	mutex_unlock(&zloop_ctl_mutex);
diff --git a/drivers/bluetooth/btmtksdio.c b/drivers/bluetooth/btmtksdio.c
index fba3ab6d30a5..e986e5af51ae 100644
--- a/drivers/bluetooth/btmtksdio.c
+++ b/drivers/bluetooth/btmtksdio.c
@@ -1472,7 +1472,6 @@ static void btmtksdio_remove(struct sdio_func *func)
 	hci_free_dev(hdev);
 }
 
-#ifdef CONFIG_PM
 static int btmtksdio_runtime_suspend(struct device *dev)
 {
 	struct sdio_func *func = dev_to_sdio_func(dev);
@@ -1542,18 +1541,13 @@ static const struct dev_pm_ops btmtksdio_pm_ops = {
 	RUNTIME_PM_OPS(btmtksdio_runtime_suspend, btmtksdio_runtime_resume, NULL)
 };
 
-#define BTMTKSDIO_PM_OPS (&btmtksdio_pm_ops)
-#else	/* CONFIG_PM */
-#define BTMTKSDIO_PM_OPS NULL
-#endif	/* CONFIG_PM */
-
 static struct sdio_driver btmtksdio_driver = {
 	.name		= "btmtksdio",
 	.probe		= btmtksdio_probe,
 	.remove		= btmtksdio_remove,
 	.id_table	= btmtksdio_table,
 	.drv = {
-		.pm = BTMTKSDIO_PM_OPS,
+		.pm = pm_ptr(&btmtksdio_pm_ops),
 	}
 };
 
diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c
index 8ed3883ab8ee..646de80c7e7b 100644
--- a/drivers/bluetooth/btusb.c
+++ b/drivers/bluetooth/btusb.c
@@ -637,6 +637,8 @@ static const struct usb_device_id quirks_table[] = {
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3622), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe158), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
 
 	/* Additional MediaTek MT7921 Bluetooth devices */
 	{ USB_DEVICE(0x0489, 0xe0c8), .driver_info = BTUSB_MEDIATEK |
@@ -4052,7 +4054,7 @@ static int btusb_probe(struct usb_interface *intf,
 			return -ENODEV;
 	}
 
-	data = devm_kzalloc(&intf->dev, sizeof(*data), GFP_KERNEL);
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 
@@ -4075,8 +4077,10 @@ static int btusb_probe(struct usb_interface *intf,
 		}
 	}
 
-	if (!data->intr_ep || !data->bulk_tx_ep || !data->bulk_rx_ep)
+	if (!data->intr_ep || !data->bulk_tx_ep || !data->bulk_rx_ep) {
+		kfree(data);
 		return -ENODEV;
+	}
 
 	if (id->driver_info & BTUSB_AMP) {
 		data->cmdreq_type = USB_TYPE_CLASS | 0x01;
@@ -4131,8 +4135,10 @@ static int btusb_probe(struct usb_interface *intf,
 	data->recv_acl = hci_recv_frame;
 
 	hdev = hci_alloc_dev_priv(priv_size);
-	if (!hdev)
+	if (!hdev) {
+		kfree(data);
 		return -ENOMEM;
+	}
 
 	hdev->bus = HCI_USB;
 	hci_set_drvdata(hdev, data);
@@ -4406,6 +4412,7 @@ static int btusb_probe(struct usb_interface *intf,
 	if (data->reset_gpio)
 		gpiod_put(data->reset_gpio);
 	hci_free_dev(hdev);
+	kfree(data);
 	return err;
 }
 
@@ -4454,6 +4461,7 @@ static void btusb_disconnect(struct usb_interface *intf)
 	}
 
 	hci_free_dev(hdev);
+	kfree(data);
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/bluetooth/hci_aml.c b/drivers/bluetooth/hci_aml.c
index b1f32c5a8a3f..4981c82d634d 100644
--- a/drivers/bluetooth/hci_aml.c
+++ b/drivers/bluetooth/hci_aml.c
@@ -677,13 +677,6 @@ static const struct hci_uart_proto aml_hci_proto = {
 	.dequeue	= aml_dequeue,
 };
 
-static void aml_device_driver_shutdown(struct device *dev)
-{
-	struct aml_serdev *amldev = dev_get_drvdata(dev);
-
-	aml_power_off(amldev);
-}
-
 static int aml_serdev_probe(struct serdev_device *serdev)
 {
 	struct aml_serdev *amldev;
@@ -714,6 +707,13 @@ static void aml_serdev_remove(struct serdev_device *serdev)
 	hci_uart_unregister_device(&amldev->serdev_hu);
 }
 
+static void aml_serdev_shutdown(struct serdev_device *serdev)
+{
+	struct aml_serdev *amldev = serdev_device_get_drvdata(serdev);
+
+	aml_power_off(amldev);
+}
+
 static const struct aml_device_data data_w155s2 = {
 	.iccm_offset = 256 * 1024,
 };
@@ -732,10 +732,10 @@ MODULE_DEVICE_TABLE(of, aml_bluetooth_of_match);
 static struct serdev_device_driver aml_serdev_driver = {
 	.probe = aml_serdev_probe,
 	.remove = aml_serdev_remove,
+	.shutdown = aml_serdev_shutdown,
 	.driver = {
 		.name = "hci_uart_aml",
 		.of_match_table = aml_bluetooth_of_match,
-		.shutdown = aml_device_driver_shutdown,
 	},
 };
 
diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c
index 45e6d84224ee..925d0a635945 100644
--- a/drivers/bluetooth/hci_bcm4377.c
+++ b/drivers/bluetooth/hci_bcm4377.c
@@ -2416,8 +2416,9 @@ static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 					hdev);
 }
 
-static int bcm4377_suspend(struct pci_dev *pdev, pm_message_t state)
+static int bcm4377_suspend(struct device *dev)
 {
+	struct pci_dev *pdev = to_pci_dev(dev);
 	struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev);
 	int ret;
 
@@ -2431,8 +2432,9 @@ static int bcm4377_suspend(struct pci_dev *pdev, pm_message_t state)
 	return 0;
 }
 
-static int bcm4377_resume(struct pci_dev *pdev)
+static int bcm4377_resume(struct device *dev)
 {
+	struct pci_dev *pdev = to_pci_dev(dev);
 	struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev);
 
 	iowrite32(BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE,
@@ -2441,6 +2443,8 @@ static int bcm4377_resume(struct pci_dev *pdev)
 	return hci_resume_dev(bcm4377->hdev);
 }
 
+static DEFINE_SIMPLE_DEV_PM_OPS(bcm4377_ops, bcm4377_suspend, bcm4377_resume);
+
 static const struct dmi_system_id bcm4377_dmi_board_table[] = {
 	{
 		.matches = {
@@ -2541,8 +2545,7 @@ static struct pci_driver bcm4377_pci_driver = {
 	.name = "hci_bcm4377",
 	.id_table = bcm4377_devid_table,
 	.probe = bcm4377_probe,
-	.suspend = bcm4377_suspend,
-	.resume = bcm4377_resume,
+	.driver.pm = &bcm4377_ops,
 };
 module_pci_driver(bcm4377_pci_driver);
 
diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c
index 888176b0faa9..b54350317a43 100644
--- a/drivers/bluetooth/hci_qca.c
+++ b/drivers/bluetooth/hci_qca.c
@@ -1653,6 +1653,39 @@ static void qca_hw_error(struct hci_dev *hdev, u8 code)
 		skb_queue_purge(&qca->rx_memdump_q);
 	}
 
+	/*
+	 * If the BT chip's bt_en pin is connected to a 3.3V power supply via
+	 * hardware and always stays high, driver cannot control the bt_en pin.
+	 * As a result, during SSR (SubSystem Restart), QCA_SSR_TRIGGERED and
+	 * QCA_IBS_DISABLED flags cannot be cleared, which leads to a reset
+	 * command timeout.
+	 * Add an msleep delay to ensure controller completes the SSR process.
+	 *
+	 * Host will not download the firmware after SSR, controller to remain
+	 * in the IBS_WAKE state, and the host needs to synchronize with it
+	 *
+	 * Since the bluetooth chip has been reset, clear the memdump state.
+	 */
+	if (!hci_test_quirk(hu->hdev, HCI_QUIRK_NON_PERSISTENT_SETUP)) {
+		/*
+		 * When the SSR (SubSystem Restart) duration exceeds 2 seconds,
+		 * it triggers host tx_idle_delay, which sets host TX state
+		 * to sleep. Reset tx_idle_timer after SSR to prevent
+		 * host enter TX IBS_Sleep mode.
+		 */
+		mod_timer(&qca->tx_idle_timer, jiffies +
+				  msecs_to_jiffies(qca->tx_idle_delay));
+
+		/* Controller reset completion time is 50ms */
+		msleep(50);
+
+		clear_bit(QCA_SSR_TRIGGERED, &qca->flags);
+		clear_bit(QCA_IBS_DISABLED, &qca->flags);
+
+		qca->tx_ibs_state = HCI_IBS_TX_AWAKE;
+		qca->memdump_state = QCA_MEMDUMP_IDLE;
+	}
+
 	clear_bit(QCA_HW_ERROR_EVENT, &qca->flags);
 }
 
@@ -2530,11 +2563,10 @@ static void qca_serdev_remove(struct serdev_device *serdev)
 	hci_uart_unregister_device(&qcadev->serdev_hu);
 }
 
-static void qca_serdev_shutdown(struct device *dev)
+static void qca_serdev_shutdown(struct serdev_device *serdev)
 {
 	int ret;
 	int timeout = msecs_to_jiffies(CMD_TRANS_TIMEOUT_MS);
-	struct serdev_device *serdev = to_serdev_device(dev);
 	struct qca_serdev *qcadev = serdev_device_get_drvdata(serdev);
 	struct hci_uart *hu = &qcadev->serdev_hu;
 	struct hci_dev *hdev = hu->hdev;
@@ -2756,11 +2788,11 @@ static void hciqca_coredump(struct device *dev)
 static struct serdev_device_driver qca_serdev_driver = {
 	.probe = qca_serdev_probe,
 	.remove = qca_serdev_remove,
+	.shutdown = qca_serdev_shutdown,
 	.driver = {
 		.name = "hci_uart_qca",
 		.of_match_table = of_match_ptr(qca_bluetooth_of_match),
 		.acpi_match_table = ACPI_PTR(qca_bluetooth_acpi_match),
-		.shutdown = qca_serdev_shutdown,
 		.pm = &qca_pm_ops,
 #ifdef CONFIG_DEV_COREDUMP
 		.coredump = hciqca_coredump,
diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c
index 25845c04e562..c08c04047ae2 100644
--- a/drivers/bus/fsl-mc/fsl-mc-bus.c
+++ b/drivers/bus/fsl-mc/fsl-mc-bus.c
@@ -137,6 +137,35 @@ static int fsl_mc_bus_uevent(const struct device *dev, struct kobj_uevent_env *e
 	return 0;
 }
 
+static int fsl_mc_probe(struct device *dev)
+{
+	struct fsl_mc_driver *mc_drv = to_fsl_mc_driver(dev->driver);
+	struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
+
+	if (mc_drv->probe)
+		return mc_drv->probe(mc_dev);
+
+	return 0;
+}
+
+static void fsl_mc_remove(struct device *dev)
+{
+	struct fsl_mc_driver *mc_drv = to_fsl_mc_driver(dev->driver);
+	struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
+
+	if (mc_drv->remove)
+		mc_drv->remove(mc_dev);
+}
+
+static void fsl_mc_shutdown(struct device *dev)
+{
+	struct fsl_mc_driver *mc_drv = to_fsl_mc_driver(dev->driver);
+	struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
+
+	if (dev->driver && mc_drv->shutdown)
+		mc_drv->shutdown(mc_dev);
+}
+
 static int fsl_mc_dma_configure(struct device *dev)
 {
 	const struct device_driver *drv = READ_ONCE(dev->driver);
@@ -314,6 +343,9 @@ const struct bus_type fsl_mc_bus_type = {
 	.name = "fsl-mc",
 	.match = fsl_mc_bus_match,
 	.uevent = fsl_mc_bus_uevent,
+	.probe = fsl_mc_probe,
+	.remove = fsl_mc_remove,
+	.shutdown = fsl_mc_shutdown,
 	.dma_configure  = fsl_mc_dma_configure,
 	.dma_cleanup = fsl_mc_dma_cleanup,
 	.dev_groups = fsl_mc_dev_groups,
@@ -434,42 +466,6 @@ static const struct device_type *fsl_mc_get_device_type(const char *type)
 	return NULL;
 }
 
-static int fsl_mc_driver_probe(struct device *dev)
-{
-	struct fsl_mc_driver *mc_drv;
-	struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
-	int error;
-
-	mc_drv = to_fsl_mc_driver(dev->driver);
-
-	error = mc_drv->probe(mc_dev);
-	if (error < 0) {
-		if (error != -EPROBE_DEFER)
-			dev_err(dev, "%s failed: %d\n", __func__, error);
-		return error;
-	}
-
-	return 0;
-}
-
-static int fsl_mc_driver_remove(struct device *dev)
-{
-	struct fsl_mc_driver *mc_drv = to_fsl_mc_driver(dev->driver);
-	struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
-
-	mc_drv->remove(mc_dev);
-
-	return 0;
-}
-
-static void fsl_mc_driver_shutdown(struct device *dev)
-{
-	struct fsl_mc_driver *mc_drv = to_fsl_mc_driver(dev->driver);
-	struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
-
-	mc_drv->shutdown(mc_dev);
-}
-
 /*
  * __fsl_mc_driver_register - registers a child device driver with the
  * MC bus
@@ -486,15 +482,6 @@ int __fsl_mc_driver_register(struct fsl_mc_driver *mc_driver,
 	mc_driver->driver.owner = owner;
 	mc_driver->driver.bus = &fsl_mc_bus_type;
 
-	if (mc_driver->probe)
-		mc_driver->driver.probe = fsl_mc_driver_probe;
-
-	if (mc_driver->remove)
-		mc_driver->driver.remove = fsl_mc_driver_remove;
-
-	if (mc_driver->shutdown)
-		mc_driver->driver.shutdown = fsl_mc_driver_shutdown;
-
 	error = driver_register(&mc_driver->driver);
 	if (error < 0) {
 		pr_err("driver_register() failed for %s: %d\n",
diff --git a/drivers/bus/moxtet.c b/drivers/bus/moxtet.c
index 7ce61d629a87..5a53bfab470a 100644
--- a/drivers/bus/moxtet.c
+++ b/drivers/bus/moxtet.c
@@ -8,6 +8,7 @@
 #include <dt-bindings/bus/moxtet.h>
 #include <linux/bitops.h>
 #include <linux/debugfs.h>
+#include <linux/hex.h>
 #include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/moxtet.h>
diff --git a/drivers/char/tpm/tpm.h b/drivers/char/tpm/tpm.h
index 02c07fef41ba..87d68ddf270a 100644
--- a/drivers/char/tpm/tpm.h
+++ b/drivers/char/tpm/tpm.h
@@ -20,6 +20,7 @@
 
 #include <linux/module.h>
 #include <linux/delay.h>
+#include <linux/hex.h>
 #include <linux/mutex.h>
 #include <linux/sched.h>
 #include <linux/platform_device.h>
diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig
index a284ba040b78..960ef7b4890e 100644
--- a/drivers/clk/qcom/Kconfig
+++ b/drivers/clk/qcom/Kconfig
@@ -46,6 +46,23 @@ config CLK_GLYMUR_TCSRCC
 	  Support for the TCSR clock controller on GLYMUR devices.
 	  Say Y if you want to use peripheral devices such as USB/PCIe/EDP.
 
+config CLK_KAANAPALI_GCC
+	tristate "Kaanapali Global Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select QCOM_GDSC
+	help
+	  Support for the global clock controller on Kaanapali devices.
+	  Say Y if you want to use peripheral devices such as UART,
+	  SPI, I2C, USB, SD/UFS, PCIe etc.
+
+config CLK_KAANAPALI_TCSRCC
+	tristate "Kaanapali TCSR Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select QCOM_GDSC
+	help
+	  Support for the TCSR clock controller on Kaanapali devices.
+	  Say Y if you want to use peripheral devices such as PCIe, USB, UFS.
+
 config CLK_X1E80100_CAMCC
 	tristate "X1E80100 Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile
index 0ac8a9055a43..159f03287edd 100644
--- a/drivers/clk/qcom/Makefile
+++ b/drivers/clk/qcom/Makefile
@@ -24,6 +24,8 @@ obj-$(CONFIG_CLK_GFM_LPASS_SM8250) += lpass-gfm-sm8250.o
 obj-$(CONFIG_CLK_GLYMUR_DISPCC) += dispcc-glymur.o
 obj-$(CONFIG_CLK_GLYMUR_GCC) += gcc-glymur.o
 obj-$(CONFIG_CLK_GLYMUR_TCSRCC) += tcsrcc-glymur.o
+obj-$(CONFIG_CLK_KAANAPALI_GCC) += gcc-kaanapali.o
+obj-$(CONFIG_CLK_KAANAPALI_TCSRCC) += tcsrcc-kaanapali.o
 obj-$(CONFIG_CLK_X1E80100_CAMCC) += camcc-x1e80100.o
 obj-$(CONFIG_CLK_X1E80100_DISPCC) += dispcc-x1e80100.o
 obj-$(CONFIG_CLK_X1E80100_GCC) += gcc-x1e80100.o
diff --git a/drivers/clk/qcom/clk-rpmh.c b/drivers/clk/qcom/clk-rpmh.c
index 1a98b3a0c528..10e84ada48ee 100644
--- a/drivers/clk/qcom/clk-rpmh.c
+++ b/drivers/clk/qcom/clk-rpmh.c
@@ -390,10 +390,21 @@ DEFINE_CLK_RPMH_VRM(clk7, _a4, "clka7", 4);
 
 DEFINE_CLK_RPMH_VRM(div_clk1, _div2, "divclka1", 2);
 
-DEFINE_CLK_RPMH_VRM(clk3, _a, "C3A_E0", 1);
-DEFINE_CLK_RPMH_VRM(clk4, _a, "C4A_E0", 1);
-DEFINE_CLK_RPMH_VRM(clk5, _a, "C5A_E0", 1);
-DEFINE_CLK_RPMH_VRM(clk8, _a, "C8A_E0", 1);
+DEFINE_CLK_RPMH_VRM(clk1, _a1_e0, "C1A_E0", 1);
+DEFINE_CLK_RPMH_VRM(clk2, _a1_e0, "C2A_E0", 1);
+DEFINE_CLK_RPMH_VRM(clk3, _a1_e0, "C3A_E0", 1);
+DEFINE_CLK_RPMH_VRM(clk4, _a1_e0, "C4A_E0", 1);
+DEFINE_CLK_RPMH_VRM(clk5, _a1_e0, "C5A_E0", 1);
+DEFINE_CLK_RPMH_VRM(clk8, _a1_e0, "C8A_E0", 1);
+
+DEFINE_CLK_RPMH_VRM(clk3, _a2_e0, "C3A_E0", 2);
+DEFINE_CLK_RPMH_VRM(clk4, _a2_e0, "C4A_E0", 2);
+DEFINE_CLK_RPMH_VRM(clk5, _a2_e0, "C5A_E0", 2);
+DEFINE_CLK_RPMH_VRM(clk6, _a2_e0, "C6A_E0", 2);
+DEFINE_CLK_RPMH_VRM(clk7, _a2_e0, "C7A_E0", 2);
+DEFINE_CLK_RPMH_VRM(clk8, _a2_e0, "C8A_E0", 2);
+
+DEFINE_CLK_RPMH_VRM(clk11, _a4_e0, "C11A_E0", 4);
 
 DEFINE_CLK_RPMH_BCM(ce, "CE0");
 DEFINE_CLK_RPMH_BCM(hwkm, "HK0");
@@ -888,12 +899,12 @@ static const struct clk_rpmh_desc clk_rpmh_sm8750 = {
 static struct clk_hw *glymur_rpmh_clocks[] = {
 	[RPMH_CXO_CLK]		= &clk_rpmh_bi_tcxo_div2.hw,
 	[RPMH_CXO_CLK_A]	= &clk_rpmh_bi_tcxo_div2_ao.hw,
-	[RPMH_RF_CLK3]		= &clk_rpmh_clk3_a.hw,
-	[RPMH_RF_CLK3_A]	= &clk_rpmh_clk3_a_ao.hw,
-	[RPMH_RF_CLK4]		= &clk_rpmh_clk4_a.hw,
-	[RPMH_RF_CLK4_A]	= &clk_rpmh_clk4_a_ao.hw,
-	[RPMH_RF_CLK5]		= &clk_rpmh_clk5_a.hw,
-	[RPMH_RF_CLK5_A]	= &clk_rpmh_clk5_a_ao.hw,
+	[RPMH_RF_CLK3]		= &clk_rpmh_clk3_a1_e0.hw,
+	[RPMH_RF_CLK3_A]	= &clk_rpmh_clk3_a1_e0_ao.hw,
+	[RPMH_RF_CLK4]		= &clk_rpmh_clk4_a1_e0.hw,
+	[RPMH_RF_CLK4_A]	= &clk_rpmh_clk4_a1_e0_ao.hw,
+	[RPMH_RF_CLK5]		= &clk_rpmh_clk5_a1_e0.hw,
+	[RPMH_RF_CLK5_A]	= &clk_rpmh_clk5_a1_e0_ao.hw,
 };
 
 static const struct clk_rpmh_desc clk_rpmh_glymur = {
@@ -901,6 +912,34 @@ static const struct clk_rpmh_desc clk_rpmh_glymur = {
 	.num_clks = ARRAY_SIZE(glymur_rpmh_clocks),
 };
 
+static struct clk_hw *kaanapali_rpmh_clocks[] = {
+	[RPMH_CXO_CLK]		= &clk_rpmh_bi_tcxo_div2.hw,
+	[RPMH_CXO_CLK_A]	= &clk_rpmh_bi_tcxo_div2_ao.hw,
+	[RPMH_DIV_CLK1]		= &clk_rpmh_clk11_a4_e0.hw,
+	[RPMH_LN_BB_CLK1]	= &clk_rpmh_clk6_a2_e0.hw,
+	[RPMH_LN_BB_CLK1_A]	= &clk_rpmh_clk6_a2_e0_ao.hw,
+	[RPMH_LN_BB_CLK2]	= &clk_rpmh_clk7_a2_e0.hw,
+	[RPMH_LN_BB_CLK2_A]	= &clk_rpmh_clk7_a2_e0_ao.hw,
+	[RPMH_LN_BB_CLK3]	= &clk_rpmh_clk8_a2_e0.hw,
+	[RPMH_LN_BB_CLK3_A]	= &clk_rpmh_clk8_a2_e0_ao.hw,
+	[RPMH_RF_CLK1]		= &clk_rpmh_clk1_a1_e0.hw,
+	[RPMH_RF_CLK1_A]	= &clk_rpmh_clk1_a1_e0_ao.hw,
+	[RPMH_RF_CLK2]		= &clk_rpmh_clk2_a1_e0.hw,
+	[RPMH_RF_CLK2_A]	= &clk_rpmh_clk2_a1_e0_ao.hw,
+	[RPMH_RF_CLK3]		= &clk_rpmh_clk3_a2_e0.hw,
+	[RPMH_RF_CLK3_A]	= &clk_rpmh_clk3_a2_e0_ao.hw,
+	[RPMH_RF_CLK4]		= &clk_rpmh_clk4_a2_e0.hw,
+	[RPMH_RF_CLK4]		= &clk_rpmh_clk4_a2_e0_ao.hw,
+	[RPMH_RF_CLK5_A]	= &clk_rpmh_clk5_a2_e0.hw,
+	[RPMH_RF_CLK5_A]	= &clk_rpmh_clk5_a2_e0_ao.hw,
+	[RPMH_IPA_CLK]		= &clk_rpmh_ipa.hw,
+};
+
+static const struct clk_rpmh_desc clk_rpmh_kaanapali = {
+	.clks = kaanapali_rpmh_clocks,
+	.num_clks = ARRAY_SIZE(kaanapali_rpmh_clocks),
+};
+
 static struct clk_hw *of_clk_rpmh_hw_get(struct of_phandle_args *clkspec,
 					 void *data)
 {
@@ -991,6 +1030,7 @@ static int clk_rpmh_probe(struct platform_device *pdev)
 
 static const struct of_device_id clk_rpmh_match_table[] = {
 	{ .compatible = "qcom,glymur-rpmh-clk", .data = &clk_rpmh_glymur},
+	{ .compatible = "qcom,kaanapali-rpmh-clk", .data = &clk_rpmh_kaanapali},
 	{ .compatible = "qcom,milos-rpmh-clk", .data = &clk_rpmh_milos},
 	{ .compatible = "qcom,qcs615-rpmh-clk", .data = &clk_rpmh_qcs615},
 	{ .compatible = "qcom,qdu1000-rpmh-clk", .data = &clk_rpmh_qdu1000},
diff --git a/drivers/clk/qcom/gcc-glymur.c b/drivers/clk/qcom/gcc-glymur.c
index deab819576d0..238e205735ed 100644
--- a/drivers/clk/qcom/gcc-glymur.c
+++ b/drivers/clk/qcom/gcc-glymur.c
@@ -2317,7 +2317,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = {
 		.parent_data = gcc_parent_data_17,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_17),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_rcg2_floor_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
@@ -2339,7 +2339,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = {
 		.parent_data = gcc_parent_data_3,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_rcg2_floor_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
diff --git a/drivers/clk/qcom/gcc-kaanapali.c b/drivers/clk/qcom/gcc-kaanapali.c
new file mode 100644
index 000000000000..182b152df14c
--- /dev/null
+++ b/drivers/clk/qcom/gcc-kaanapali.c
@@ -0,0 +1,3541 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,kaanapali-gcc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "clk-regmap-phy-mux.h"
+#include "common.h"
+#include "gdsc.h"
+#include "reset.h"
+
+enum {
+	DT_BI_TCXO,
+	DT_BI_TCXO_AO,
+	DT_SLEEP_CLK,
+	DT_PCIE_0_PIPE_CLK,
+	DT_UFS_PHY_RX_SYMBOL_0_CLK,
+	DT_UFS_PHY_RX_SYMBOL_1_CLK,
+	DT_UFS_PHY_TX_SYMBOL_0_CLK,
+	DT_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK,
+};
+
+enum {
+	P_BI_TCXO,
+	P_GCC_GPLL0_OUT_EVEN,
+	P_GCC_GPLL0_OUT_MAIN,
+	P_GCC_GPLL1_OUT_MAIN,
+	P_GCC_GPLL4_OUT_MAIN,
+	P_GCC_GPLL7_OUT_MAIN,
+	P_GCC_GPLL9_OUT_MAIN,
+	P_PCIE_0_PIPE_CLK,
+	P_SLEEP_CLK,
+	P_UFS_PHY_RX_SYMBOL_0_CLK,
+	P_UFS_PHY_RX_SYMBOL_1_CLK,
+	P_UFS_PHY_TX_SYMBOL_0_CLK,
+	P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK,
+};
+
+static struct clk_alpha_pll gcc_gpll0 = {
+	.offset = 0x0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_taycan_eko_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_gcc_gpll0_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv gcc_gpll0_out_even = {
+	.offset = 0x0,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_gcc_gpll0_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_gcc_gpll0_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gpll0_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_gpll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_taycan_eko_t_ops,
+	},
+};
+
+static struct clk_alpha_pll gcc_gpll1 = {
+	.offset = 0x1000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpll1",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_taycan_eko_t_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll gcc_gpll4 = {
+	.offset = 0x4000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(4),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpll4",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_taycan_eko_t_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll gcc_gpll7 = {
+	.offset = 0x7000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(7),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpll7",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_taycan_eko_t_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll gcc_gpll9 = {
+	.offset = 0x9000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(9),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpll9",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_taycan_eko_t_ops,
+		},
+	},
+};
+
+static const struct parent_map gcc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_0[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_GCC_GPLL1_OUT_MAIN, 4 },
+	{ P_GCC_GPLL4_OUT_MAIN, 5 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_1[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .hw = &gcc_gpll1.clkr.hw },
+	{ .hw = &gcc_gpll4.clkr.hw },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_2[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_SLEEP_CLK, 5 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_2[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .index = DT_SLEEP_CLK },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_3[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_GCC_GPLL4_OUT_MAIN, 5 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_3[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .hw = &gcc_gpll4.clkr.hw },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_4[] = {
+	{ P_BI_TCXO, 0 },
+};
+
+static const struct clk_parent_data gcc_parent_data_4[] = {
+	{ .index = DT_BI_TCXO },
+};
+
+static const struct parent_map gcc_parent_map_5[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_GCC_GPLL7_OUT_MAIN, 2 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_5[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .hw = &gcc_gpll7.clkr.hw },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_6[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_SLEEP_CLK, 5 },
+};
+
+static const struct clk_parent_data gcc_parent_data_6[] = {
+	{ .index = DT_BI_TCXO },
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct parent_map gcc_parent_map_8[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_GCC_GPLL9_OUT_MAIN, 2 },
+	{ P_GCC_GPLL4_OUT_MAIN, 5 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_8[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .hw = &gcc_gpll9.clkr.hw },
+	{ .hw = &gcc_gpll4.clkr.hw },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_12[] = {
+	{ P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, 0 },
+	{ P_BI_TCXO, 2 },
+};
+
+static const struct clk_parent_data gcc_parent_data_12[] = {
+	{ .index = DT_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK },
+	{ .index = DT_BI_TCXO },
+};
+
+static struct clk_regmap_phy_mux gcc_pcie_0_pipe_clk_src = {
+	.reg = 0x6b090,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_PCIE_0_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_ufs_phy_rx_symbol_0_clk_src = {
+	.reg = 0x77068,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_rx_symbol_0_clk_src",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_UFS_PHY_RX_SYMBOL_0_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_ufs_phy_rx_symbol_1_clk_src = {
+	.reg = 0x770ec,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_rx_symbol_1_clk_src",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_UFS_PHY_RX_SYMBOL_1_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_ufs_phy_tx_symbol_0_clk_src = {
+	.reg = 0x77058,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_tx_symbol_0_clk_src",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_UFS_PHY_TX_SYMBOL_0_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_mux gcc_usb3_prim_phy_pipe_clk_src = {
+	.reg = 0x39074,
+	.shift = 0,
+	.width = 2,
+	.parent_map = gcc_parent_map_12,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_phy_pipe_clk_src",
+			.parent_data = gcc_parent_data_12,
+			.num_parents = ARRAY_SIZE(gcc_parent_data_12),
+			.ops = &clk_regmap_mux_closest_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = {
+	F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0),
+	F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_gp1_clk_src = {
+	.cmd_rcgr = 0x64004,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gp1_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_gp2_clk_src = {
+	.cmd_rcgr = 0x65004,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gp2_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_gp3_clk_src = {
+	.cmd_rcgr = 0x66004,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gp3_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcie_0_aux_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcie_0_aux_clk_src = {
+	.cmd_rcgr = 0x6b094,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_6,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_0_aux_clk_src",
+		.parent_data = gcc_parent_data_6,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_6),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie_0_phy_aux_clk_src = {
+	.cmd_rcgr = 0x6b0ac,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_0_phy_aux_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcie_0_phy_rchng_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcie_0_phy_rchng_clk_src = {
+	.cmd_rcgr = 0x6b078,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_phy_rchng_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_0_phy_rchng_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pdm2_clk_src[] = {
+	F(60000000, P_GCC_GPLL0_OUT_MAIN, 10, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pdm2_clk_src = {
+	.cmd_rcgr = 0x33010,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pdm2_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pdm2_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_i2c_s0_clk_src = {
+	.cmd_rcgr = 0x17008,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_i2c_s0_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_i2c_s1_clk_src = {
+	.cmd_rcgr = 0x17024,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_i2c_s1_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_i2c_s2_clk_src = {
+	.cmd_rcgr = 0x17040,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_i2c_s2_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_i2c_s3_clk_src = {
+	.cmd_rcgr = 0x1705c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_i2c_s3_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_i2c_s4_clk_src = {
+	.cmd_rcgr = 0x17078,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_i2c_s4_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap1_qspi_ref_clk_src[] = {
+	F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625),
+	F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625),
+	F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75),
+	F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25),
+	F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375),
+	F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75),
+	F(66666667, P_GCC_GPLL0_OUT_MAIN, 9, 0, 0),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15),
+	F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25),
+	F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375),
+	F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75),
+	F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625),
+	F(120000000, P_GCC_GPLL0_OUT_MAIN, 5, 0, 0),
+	F(150000000, P_GCC_GPLL0_OUT_EVEN, 2, 0, 0),
+	F(250000000, P_GCC_GPLL7_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_qspi_ref_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_qspi_ref_clk_src",
+	.parent_data = gcc_parent_data_5,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_5),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_qspi_ref_clk_src = {
+	.cmd_rcgr = 0x188c0,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_5,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_qspi_ref_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap1_qspi_ref_clk_src_init,
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap1_s0_clk_src[] = {
+	F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625),
+	F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625),
+	F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75),
+	F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25),
+	F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375),
+	F(60000000, P_GCC_GPLL0_OUT_EVEN, 5, 0, 0),
+	F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75),
+	F(66666667, P_GCC_GPLL0_OUT_MAIN, 9, 0, 0),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15),
+	F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25),
+	F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0),
+	{ }
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s0_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s0_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s0_clk_src = {
+	.cmd_rcgr = 0x18014,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s0_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s1_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s1_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s1_clk_src = {
+	.cmd_rcgr = 0x18150,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s1_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s3_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s3_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s3_clk_src = {
+	.cmd_rcgr = 0x182a0,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s3_clk_src_init,
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap1_s4_clk_src[] = {
+	F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625),
+	F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625),
+	F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75),
+	F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25),
+	F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375),
+	F(60000000, P_GCC_GPLL0_OUT_EVEN, 5, 0, 0),
+	F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75),
+	F(66666667, P_GCC_GPLL0_OUT_MAIN, 9, 0, 0),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15),
+	F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25),
+	F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375),
+	F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75),
+	F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625),
+	F(120000000, P_GCC_GPLL0_OUT_MAIN, 5, 0, 0),
+	{ }
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s4_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s4_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s4_clk_src = {
+	.cmd_rcgr = 0x183dc,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s4_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s5_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s5_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s5_clk_src = {
+	.cmd_rcgr = 0x18518,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s5_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s6_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s6_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s6_clk_src = {
+	.cmd_rcgr = 0x18654,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s6_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s7_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s7_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s7_clk_src = {
+	.cmd_rcgr = 0x18790,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s7_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s0_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s0_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s0_clk_src = {
+	.cmd_rcgr = 0x1e014,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s0_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s1_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s1_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s1_clk_src = {
+	.cmd_rcgr = 0x1e150,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s1_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s2_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s2_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s2_clk_src = {
+	.cmd_rcgr = 0x1e28c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s2_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s3_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s3_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s3_clk_src = {
+	.cmd_rcgr = 0x1e3c8,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s3_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s4_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s4_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s4_clk_src = {
+	.cmd_rcgr = 0x1e504,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s4_clk_src_init,
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap3_ibi_ctrl_0_clk_src[] = {
+	F(37500000, P_GCC_GPLL0_OUT_EVEN, 8, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap3_ibi_ctrl_0_clk_src = {
+	.cmd_rcgr = 0xa877c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap3_ibi_ctrl_0_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap3_ibi_ctrl_0_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_init_data gcc_qupv3_wrap3_s0_clk_src_init = {
+	.name = "gcc_qupv3_wrap3_s0_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap3_s0_clk_src = {
+	.cmd_rcgr = 0xa8014,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap3_s0_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap3_s1_clk_src_init = {
+	.name = "gcc_qupv3_wrap3_s1_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap3_s1_clk_src = {
+	.cmd_rcgr = 0xa8150,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap3_s1_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap3_s2_clk_src_init = {
+	.name = "gcc_qupv3_wrap3_s2_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap3_s2_clk_src = {
+	.cmd_rcgr = 0xa828c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap3_s2_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap3_s3_clk_src_init = {
+	.name = "gcc_qupv3_wrap3_s3_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap3_s3_clk_src = {
+	.cmd_rcgr = 0xa83c8,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap3_s3_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap3_s4_clk_src_init = {
+	.name = "gcc_qupv3_wrap3_s4_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap3_s4_clk_src = {
+	.cmd_rcgr = 0xa8504,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap3_s4_clk_src_init,
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap3_s5_clk_src[] = {
+	F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625),
+	F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625),
+	F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75),
+	F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25),
+	F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375),
+	F(60000000, P_GCC_GPLL0_OUT_EVEN, 5, 0, 0),
+	F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75),
+	F(66666667, P_GCC_GPLL0_OUT_MAIN, 9, 0, 0),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15),
+	F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25),
+	F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375),
+	F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75),
+	F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625),
+	F(120000000, P_GCC_GPLL0_OUT_MAIN, 5, 0, 0),
+	F(128000000, P_GCC_GPLL0_OUT_MAIN, 1, 16, 75),
+	{ }
+};
+
+static struct clk_init_data gcc_qupv3_wrap3_s5_clk_src_init = {
+	.name = "gcc_qupv3_wrap3_s5_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap3_s5_clk_src = {
+	.cmd_rcgr = 0xa8640,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap3_s5_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap3_s5_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap4_s0_clk_src_init = {
+	.name = "gcc_qupv3_wrap4_s0_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap4_s0_clk_src = {
+	.cmd_rcgr = 0xa9014,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap4_s0_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap4_s1_clk_src_init = {
+	.name = "gcc_qupv3_wrap4_s1_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap4_s1_clk_src = {
+	.cmd_rcgr = 0xa9150,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap4_s1_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap4_s2_clk_src_init = {
+	.name = "gcc_qupv3_wrap4_s2_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap4_s2_clk_src = {
+	.cmd_rcgr = 0xa928c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap4_s2_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap4_s3_clk_src_init = {
+	.name = "gcc_qupv3_wrap4_s3_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap4_s3_clk_src = {
+	.cmd_rcgr = 0xa93c8,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap4_s3_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap4_s4_clk_src_init = {
+	.name = "gcc_qupv3_wrap4_s4_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap4_s4_clk_src = {
+	.cmd_rcgr = 0xa9504,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap4_s4_clk_src_init,
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk_src[] = {
+	F(400000, P_BI_TCXO, 12, 1, 4),
+	F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0),
+	F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0),
+	F(202000000, P_GCC_GPLL9_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc2_apps_clk_src = {
+	.cmd_rcgr = 0x1401c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_8,
+	.freq_tbl = ftbl_gcc_sdcc2_apps_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sdcc2_apps_clk_src",
+		.parent_data = gcc_parent_data_8,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_8),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_floor_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc4_apps_clk_src[] = {
+	F(400000, P_BI_TCXO, 12, 1, 4),
+	F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc4_apps_clk_src = {
+	.cmd_rcgr = 0x1601c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_sdcc4_apps_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sdcc4_apps_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_floor_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ufs_phy_axi_clk_src[] = {
+	F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0),
+	F(201500000, P_GCC_GPLL4_OUT_MAIN, 4, 0, 0),
+	F(403000000, P_GCC_GPLL4_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_ufs_phy_axi_clk_src = {
+	.cmd_rcgr = 0x77034,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_ufs_phy_axi_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_ufs_phy_axi_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ufs_phy_ice_core_clk_src[] = {
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0),
+	F(201500000, P_GCC_GPLL4_OUT_MAIN, 4, 0, 0),
+	F(403000000, P_GCC_GPLL4_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_ufs_phy_ice_core_clk_src = {
+	.cmd_rcgr = 0x7708c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_ufs_phy_ice_core_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_ufs_phy_ice_core_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ufs_phy_phy_aux_clk_src[] = {
+	F(9600000, P_BI_TCXO, 2, 0, 0),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_ufs_phy_phy_aux_clk_src = {
+	.cmd_rcgr = 0x770c0,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_gcc_ufs_phy_phy_aux_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_ufs_phy_phy_aux_clk_src",
+		.parent_data = gcc_parent_data_4,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_4),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_ufs_phy_unipro_core_clk_src = {
+	.cmd_rcgr = 0x770a4,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_ufs_phy_ice_core_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_ufs_phy_unipro_core_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb30_prim_master_clk_src[] = {
+	F(66666667, P_GCC_GPLL0_OUT_EVEN, 4.5, 0, 0),
+	F(133333333, P_GCC_GPLL0_OUT_MAIN, 4.5, 0, 0),
+	F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0),
+	F(240000000, P_GCC_GPLL0_OUT_MAIN, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_usb30_prim_master_clk_src = {
+	.cmd_rcgr = 0x39034,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_usb30_prim_master_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb30_prim_master_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb30_prim_mock_utmi_clk_src = {
+	.cmd_rcgr = 0x3904c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb30_prim_mock_utmi_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb3_prim_phy_aux_clk_src = {
+	.cmd_rcgr = 0x39078,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_6,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb3_prim_phy_aux_clk_src",
+		.parent_data = gcc_parent_data_6,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_6),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_qupv3_wrap1_s2_clk_src = {
+	.reg = 0x1828c,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap1_s2_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_qupv3_wrap1_qspi_ref_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_usb30_prim_mock_utmi_postdiv_clk_src = {
+	.reg = 0x39064,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb30_prim_mock_utmi_postdiv_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_usb30_prim_mock_utmi_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_branch gcc_aggre_noc_pcie_axi_clk = {
+	.halt_reg = 0x10068,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x10068,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(12),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_aggre_noc_pcie_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre_ufs_phy_axi_clk = {
+	.halt_reg = 0x770f0,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x770f0,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x770f0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_aggre_ufs_phy_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre_usb3_prim_axi_clk = {
+	.halt_reg = 0x39094,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x39094,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x39094,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_aggre_usb3_prim_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_boot_rom_ahb_clk = {
+	.halt_reg = 0x38004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x38004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(18),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_boot_rom_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camera_hf_axi_clk = {
+	.halt_reg = 0x26014,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x26014,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x26014,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camera_hf_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camera_sf_axi_clk = {
+	.halt_reg = 0x26028,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x26028,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x26028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camera_sf_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cfg_noc_pcie_anoc_ahb_clk = {
+	.halt_reg = 0x10050,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x10050,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(20),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cfg_noc_pcie_anoc_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cfg_noc_usb3_prim_axi_clk = {
+	.halt_reg = 0x39090,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x39090,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x39090,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cfg_noc_usb3_prim_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cnoc_pcie_sf_axi_clk = {
+	.halt_reg = 0x10058,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(6),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cnoc_pcie_sf_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ddrss_pcie_sf_qtb_clk = {
+	.halt_reg = 0x1007c,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x1007c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(19),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ddrss_pcie_sf_qtb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_disp_hf_axi_clk = {
+	.halt_reg = 0x2701c,
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x2701c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_disp_hf_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_disp_sf_axi_clk = {
+	.halt_reg = 0x27008,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x27008,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x27008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_disp_sf_axi_clk",
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_eva_axi0_clk = {
+	.halt_reg = 0x9f008,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x9f008,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x9f008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_eva_axi0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_eva_axi0c_clk = {
+	.halt_reg = 0x9f01c,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x9f01c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x9f01c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_eva_axi0c_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp1_clk = {
+	.halt_reg = 0x64000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x64000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gp1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gp1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp2_clk = {
+	.halt_reg = 0x65000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x65000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gp2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gp2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp3_clk = {
+	.halt_reg = 0x66000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x66000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gp3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gp3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_gemnoc_gfx_clk = {
+	.halt_reg = 0x71010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x71010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x71010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_gemnoc_gfx_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_gpll0_clk_src = {
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(15),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_gpll0_clk_src",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gpll0.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_gpll0_div_clk_src = {
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(16),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_gpll0_div_clk_src",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gpll0_out_even.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_aux_clk = {
+	.halt_reg = 0x6b044,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(3),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_0_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_cfg_ahb_clk = {
+	.halt_reg = 0x6b040,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x6b040,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(2),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_mstr_axi_clk = {
+	.halt_reg = 0x6b030,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x6b030,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_mstr_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_phy_aux_clk = {
+	.halt_reg = 0x6b054,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(31),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_phy_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_0_phy_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_phy_rchng_clk = {
+	.halt_reg = 0x6b074,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(22),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_phy_rchng_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_0_phy_rchng_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_pipe_clk = {
+	.halt_reg = 0x6b064,
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(4),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_pipe_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_0_pipe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_slv_axi_clk = {
+	.halt_reg = 0x6b020,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x6b020,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_slv_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_slv_q2a_axi_clk = {
+	.halt_reg = 0x6b01c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(5),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_slv_q2a_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm2_clk = {
+	.halt_reg = 0x3300c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3300c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pdm2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pdm2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_ahb_clk = {
+	.halt_reg = 0x33004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x33004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x33004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pdm_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_xo4_clk = {
+	.halt_reg = 0x33008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x33008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pdm_xo4_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_camera_cmd_ahb_clk = {
+	.halt_reg = 0x26010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x26010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x26010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_camera_cmd_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_camera_nrt_ahb_clk = {
+	.halt_reg = 0x26008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x26008,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x26008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_camera_nrt_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_camera_rt_ahb_clk = {
+	.halt_reg = 0x2600c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x2600c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x2600c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_camera_rt_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_disp_dcp_sf_ahb_clk = {
+	.halt_reg = 0x27030,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x27030,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x27030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_disp_dcp_sf_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_gpu_ahb_clk = {
+	.halt_reg = 0x71008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x71008,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x71008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_gpu_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_pcie_ahb_clk = {
+	.halt_reg = 0x6b018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x6b018,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(19),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_pcie_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_video_cv_cpu_ahb_clk = {
+	.halt_reg = 0x32014,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x32014,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x32014,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_video_cv_cpu_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_video_cvp_ahb_clk = {
+	.halt_reg = 0x32008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x32008,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x32008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_video_cvp_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_video_v_cpu_ahb_clk = {
+	.halt_reg = 0x32010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x32010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x32010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_video_v_cpu_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_video_vcodec_ahb_clk = {
+	.halt_reg = 0x3200c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x3200c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x3200c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_video_vcodec_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_i2c_core_clk = {
+	.halt_reg = 0x23004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(8),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_i2c_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_i2c_s0_clk = {
+	.halt_reg = 0x17004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(10),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_i2c_s0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_i2c_s0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_i2c_s1_clk = {
+	.halt_reg = 0x17020,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(11),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_i2c_s1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_i2c_s1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_i2c_s2_clk = {
+	.halt_reg = 0x1703c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(12),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_i2c_s2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_i2c_s2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_i2c_s3_clk = {
+	.halt_reg = 0x17058,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(13),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_i2c_s3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_i2c_s3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_i2c_s4_clk = {
+	.halt_reg = 0x17074,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(14),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_i2c_s4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_i2c_s4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_i2c_s_ahb_clk = {
+	.halt_reg = 0x23000,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x23000,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(7),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_i2c_s_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_core_2x_clk = {
+	.halt_reg = 0x2315c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(18),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_core_2x_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_core_clk = {
+	.halt_reg = 0x23148,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(19),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_qspi_ref_clk = {
+	.halt_reg = 0x188bc,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(29),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_qspi_ref_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_qspi_ref_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s0_clk = {
+	.halt_reg = 0x18004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(22),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s1_clk = {
+	.halt_reg = 0x18140,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(23),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s2_clk = {
+	.halt_reg = 0x1827c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(24),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s3_clk = {
+	.halt_reg = 0x18290,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(25),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s4_clk = {
+	.halt_reg = 0x183cc,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(26),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s5_clk = {
+	.halt_reg = 0x18508,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(27),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s5_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s5_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s6_clk = {
+	.halt_reg = 0x18644,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(28),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s6_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s6_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s7_clk = {
+	.halt_reg = 0x18780,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(16),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s7_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s7_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_core_2x_clk = {
+	.halt_reg = 0x232b4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(3),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_core_2x_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_core_clk = {
+	.halt_reg = 0x232a0,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s0_clk = {
+	.halt_reg = 0x1e004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(4),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s1_clk = {
+	.halt_reg = 0x1e140,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(5),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s2_clk = {
+	.halt_reg = 0x1e27c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(6),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s3_clk = {
+	.halt_reg = 0x1e3b8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(7),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s4_clk = {
+	.halt_reg = 0x1e4f4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(8),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_core_2x_clk = {
+	.halt_reg = 0x2340c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(11),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_core_2x_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_core_clk = {
+	.halt_reg = 0x233f8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(10),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_ibi_ctrl_1_clk = {
+	.halt_reg = 0xa8774,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xa8774,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(20),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_ibi_ctrl_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap3_ibi_ctrl_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_ibi_ctrl_2_clk = {
+	.halt_reg = 0xa8778,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xa8778,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(21),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_ibi_ctrl_2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap3_ibi_ctrl_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_s0_clk = {
+	.halt_reg = 0xa8004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(12),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_s0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap3_s0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_s1_clk = {
+	.halt_reg = 0xa8140,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(13),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_s1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap3_s1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_s2_clk = {
+	.halt_reg = 0xa827c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(14),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_s2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap3_s2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_s3_clk = {
+	.halt_reg = 0xa83b8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(15),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_s3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap3_s3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_s4_clk = {
+	.halt_reg = 0xa84f4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(16),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_s4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap3_s4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_s5_clk = {
+	.halt_reg = 0xa8630,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(17),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_s5_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap3_s5_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap4_core_2x_clk = {
+	.halt_reg = 0x23564,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(25),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap4_core_2x_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap4_core_clk = {
+	.halt_reg = 0x23550,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(24),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap4_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap4_s0_clk = {
+	.halt_reg = 0xa9004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(26),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap4_s0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap4_s0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap4_s1_clk = {
+	.halt_reg = 0xa9140,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(27),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap4_s1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap4_s1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap4_s2_clk = {
+	.halt_reg = 0xa927c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(28),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap4_s2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap4_s2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap4_s3_clk = {
+	.halt_reg = 0xa93b8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(29),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap4_s3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap4_s3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap4_s4_clk = {
+	.halt_reg = 0xa94f4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(30),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap4_s4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap4_s4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_1_m_axi_clk = {
+	.halt_reg = 0x23140,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x23140,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(20),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_1_m_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_1_s_ahb_clk = {
+	.halt_reg = 0x23144,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x23144,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(21),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_1_s_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_2_m_ahb_clk = {
+	.halt_reg = 0x23298,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x23298,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(2),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_2_m_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_2_s_ahb_clk = {
+	.halt_reg = 0x2329c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x2329c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_2_s_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_3_ibi_1_ahb_clk = {
+	.halt_reg = 0xa876c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xa876c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(18),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_3_ibi_1_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_3_ibi_2_ahb_clk = {
+	.halt_reg = 0xa8770,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xa8770,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(19),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_3_ibi_2_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_3_m_ahb_clk = {
+	.halt_reg = 0x233f0,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x233f0,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(8),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_3_m_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_3_s_ahb_clk = {
+	.halt_reg = 0x233f4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x233f4,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(9),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_3_s_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_4_m_ahb_clk = {
+	.halt_reg = 0x23548,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x23548,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(22),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_4_m_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_4_s_ahb_clk = {
+	.halt_reg = 0x2354c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x2354c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(23),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_4_s_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_ahb_clk = {
+	.halt_reg = 0x14014,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x14014,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc2_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_apps_clk = {
+	.halt_reg = 0x14004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x14004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc2_apps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sdcc2_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc4_ahb_clk = {
+	.halt_reg = 0x16014,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x16014,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc4_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc4_apps_clk = {
+	.halt_reg = 0x16004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x16004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc4_apps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sdcc4_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_ahb_clk = {
+	.halt_reg = 0x77028,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x77028,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x77028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_axi_clk = {
+	.halt_reg = 0x77018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x77018,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x77018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_ice_core_clk = {
+	.halt_reg = 0x7707c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x7707c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7707c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_ice_core_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_ice_core_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_phy_aux_clk = {
+	.halt_reg = 0x770bc,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x770bc,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x770bc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_phy_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_phy_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_rx_symbol_0_clk = {
+	.halt_reg = 0x77030,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x77030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_rx_symbol_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_rx_symbol_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_rx_symbol_1_clk = {
+	.halt_reg = 0x770d8,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x770d8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_rx_symbol_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_rx_symbol_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_tx_symbol_0_clk = {
+	.halt_reg = 0x7702c,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x7702c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_tx_symbol_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_tx_symbol_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_unipro_core_clk = {
+	.halt_reg = 0x7706c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x7706c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7706c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_unipro_core_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_unipro_core_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_prim_master_clk = {
+	.halt_reg = 0x39018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x39018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb30_prim_master_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_prim_mock_utmi_clk = {
+	.halt_reg = 0x3902c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3902c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb30_prim_mock_utmi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_prim_sleep_clk = {
+	.halt_reg = 0x39028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x39028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb30_prim_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_phy_aux_clk = {
+	.halt_reg = 0x39068,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x39068,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_phy_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb3_prim_phy_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_phy_com_aux_clk = {
+	.halt_reg = 0x3906c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3906c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_phy_com_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb3_prim_phy_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_phy_pipe_clk = {
+	.halt_reg = 0x39070,
+	.halt_check = BRANCH_HALT_DELAY,
+	.hwcg_reg = 0x39070,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x39070,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_phy_pipe_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb3_prim_phy_pipe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_video_axi0_clk = {
+	.halt_reg = 0x32018,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x32018,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x32018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_video_axi0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_video_axi1_clk = {
+	.halt_reg = 0x3202c,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x3202c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x3202c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_video_axi1_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc gcc_pcie_0_gdsc = {
+	.gdscr = 0x6b004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.collapse_ctrl = 0x5214c,
+	.collapse_mask = BIT(0),
+	.pd = {
+		.name = "gcc_pcie_0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE | VOTABLE,
+};
+
+static struct gdsc gcc_pcie_0_phy_gdsc = {
+	.gdscr = 0x6c000,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x2,
+	.collapse_ctrl = 0x5214c,
+	.collapse_mask = BIT(2),
+	.pd = {
+		.name = "gcc_pcie_0_phy_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE | VOTABLE,
+};
+
+static struct gdsc gcc_ufs_mem_phy_gdsc = {
+	.gdscr = 0x9e000,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x2,
+	.pd = {
+		.name = "gcc_ufs_mem_phy_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_ufs_phy_gdsc = {
+	.gdscr = 0x77004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gcc_ufs_phy_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_usb30_prim_gdsc = {
+	.gdscr = 0x39004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gcc_usb30_prim_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_usb3_phy_gdsc = {
+	.gdscr = 0x50018,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x2,
+	.pd = {
+		.name = "gcc_usb3_phy_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct clk_regmap *gcc_kaanapali_clocks[] = {
+	[GCC_AGGRE_NOC_PCIE_AXI_CLK] = &gcc_aggre_noc_pcie_axi_clk.clkr,
+	[GCC_AGGRE_UFS_PHY_AXI_CLK] = &gcc_aggre_ufs_phy_axi_clk.clkr,
+	[GCC_AGGRE_USB3_PRIM_AXI_CLK] = &gcc_aggre_usb3_prim_axi_clk.clkr,
+	[GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr,
+	[GCC_CAMERA_HF_AXI_CLK] = &gcc_camera_hf_axi_clk.clkr,
+	[GCC_CAMERA_SF_AXI_CLK] = &gcc_camera_sf_axi_clk.clkr,
+	[GCC_CFG_NOC_PCIE_ANOC_AHB_CLK] = &gcc_cfg_noc_pcie_anoc_ahb_clk.clkr,
+	[GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr,
+	[GCC_CNOC_PCIE_SF_AXI_CLK] = &gcc_cnoc_pcie_sf_axi_clk.clkr,
+	[GCC_DDRSS_PCIE_SF_QTB_CLK] = &gcc_ddrss_pcie_sf_qtb_clk.clkr,
+	[GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr,
+	[GCC_DISP_SF_AXI_CLK] = &gcc_disp_sf_axi_clk.clkr,
+	[GCC_EVA_AXI0_CLK] = &gcc_eva_axi0_clk.clkr,
+	[GCC_EVA_AXI0C_CLK] = &gcc_eva_axi0c_clk.clkr,
+	[GCC_GP1_CLK] = &gcc_gp1_clk.clkr,
+	[GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr,
+	[GCC_GP2_CLK] = &gcc_gp2_clk.clkr,
+	[GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr,
+	[GCC_GP3_CLK] = &gcc_gp3_clk.clkr,
+	[GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr,
+	[GCC_GPLL0] = &gcc_gpll0.clkr,
+	[GCC_GPLL0_OUT_EVEN] = &gcc_gpll0_out_even.clkr,
+	[GCC_GPLL1] = &gcc_gpll1.clkr,
+	[GCC_GPLL4] = &gcc_gpll4.clkr,
+	[GCC_GPLL7] = &gcc_gpll7.clkr,
+	[GCC_GPLL9] = &gcc_gpll9.clkr,
+	[GCC_GPU_GEMNOC_GFX_CLK] = &gcc_gpu_gemnoc_gfx_clk.clkr,
+	[GCC_GPU_GPLL0_CLK_SRC] = &gcc_gpu_gpll0_clk_src.clkr,
+	[GCC_GPU_GPLL0_DIV_CLK_SRC] = &gcc_gpu_gpll0_div_clk_src.clkr,
+	[GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr,
+	[GCC_PCIE_0_AUX_CLK_SRC] = &gcc_pcie_0_aux_clk_src.clkr,
+	[GCC_PCIE_0_CFG_AHB_CLK] = &gcc_pcie_0_cfg_ahb_clk.clkr,
+	[GCC_PCIE_0_MSTR_AXI_CLK] = &gcc_pcie_0_mstr_axi_clk.clkr,
+	[GCC_PCIE_0_PHY_AUX_CLK] = &gcc_pcie_0_phy_aux_clk.clkr,
+	[GCC_PCIE_0_PHY_AUX_CLK_SRC] = &gcc_pcie_0_phy_aux_clk_src.clkr,
+	[GCC_PCIE_0_PHY_RCHNG_CLK] = &gcc_pcie_0_phy_rchng_clk.clkr,
+	[GCC_PCIE_0_PHY_RCHNG_CLK_SRC] = &gcc_pcie_0_phy_rchng_clk_src.clkr,
+	[GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr,
+	[GCC_PCIE_0_PIPE_CLK_SRC] = &gcc_pcie_0_pipe_clk_src.clkr,
+	[GCC_PCIE_0_SLV_AXI_CLK] = &gcc_pcie_0_slv_axi_clk.clkr,
+	[GCC_PCIE_0_SLV_Q2A_AXI_CLK] = &gcc_pcie_0_slv_q2a_axi_clk.clkr,
+	[GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr,
+	[GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr,
+	[GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr,
+	[GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr,
+	[GCC_QUPV3_I2C_CORE_CLK] = &gcc_qupv3_i2c_core_clk.clkr,
+	[GCC_QUPV3_I2C_S0_CLK] = &gcc_qupv3_i2c_s0_clk.clkr,
+	[GCC_QUPV3_I2C_S0_CLK_SRC] = &gcc_qupv3_i2c_s0_clk_src.clkr,
+	[GCC_QUPV3_I2C_S1_CLK] = &gcc_qupv3_i2c_s1_clk.clkr,
+	[GCC_QUPV3_I2C_S1_CLK_SRC] = &gcc_qupv3_i2c_s1_clk_src.clkr,
+	[GCC_QUPV3_I2C_S2_CLK] = &gcc_qupv3_i2c_s2_clk.clkr,
+	[GCC_QUPV3_I2C_S2_CLK_SRC] = &gcc_qupv3_i2c_s2_clk_src.clkr,
+	[GCC_QUPV3_I2C_S3_CLK] = &gcc_qupv3_i2c_s3_clk.clkr,
+	[GCC_QUPV3_I2C_S3_CLK_SRC] = &gcc_qupv3_i2c_s3_clk_src.clkr,
+	[GCC_QUPV3_I2C_S4_CLK] = &gcc_qupv3_i2c_s4_clk.clkr,
+	[GCC_QUPV3_I2C_S4_CLK_SRC] = &gcc_qupv3_i2c_s4_clk_src.clkr,
+	[GCC_QUPV3_I2C_S_AHB_CLK] = &gcc_qupv3_i2c_s_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP1_CORE_2X_CLK] = &gcc_qupv3_wrap1_core_2x_clk.clkr,
+	[GCC_QUPV3_WRAP1_CORE_CLK] = &gcc_qupv3_wrap1_core_clk.clkr,
+	[GCC_QUPV3_WRAP1_QSPI_REF_CLK] = &gcc_qupv3_wrap1_qspi_ref_clk.clkr,
+	[GCC_QUPV3_WRAP1_QSPI_REF_CLK_SRC] = &gcc_qupv3_wrap1_qspi_ref_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S0_CLK] = &gcc_qupv3_wrap1_s0_clk.clkr,
+	[GCC_QUPV3_WRAP1_S0_CLK_SRC] = &gcc_qupv3_wrap1_s0_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S1_CLK] = &gcc_qupv3_wrap1_s1_clk.clkr,
+	[GCC_QUPV3_WRAP1_S1_CLK_SRC] = &gcc_qupv3_wrap1_s1_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S2_CLK] = &gcc_qupv3_wrap1_s2_clk.clkr,
+	[GCC_QUPV3_WRAP1_S2_CLK_SRC] = &gcc_qupv3_wrap1_s2_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S3_CLK] = &gcc_qupv3_wrap1_s3_clk.clkr,
+	[GCC_QUPV3_WRAP1_S3_CLK_SRC] = &gcc_qupv3_wrap1_s3_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S4_CLK] = &gcc_qupv3_wrap1_s4_clk.clkr,
+	[GCC_QUPV3_WRAP1_S4_CLK_SRC] = &gcc_qupv3_wrap1_s4_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S5_CLK] = &gcc_qupv3_wrap1_s5_clk.clkr,
+	[GCC_QUPV3_WRAP1_S5_CLK_SRC] = &gcc_qupv3_wrap1_s5_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S6_CLK] = &gcc_qupv3_wrap1_s6_clk.clkr,
+	[GCC_QUPV3_WRAP1_S6_CLK_SRC] = &gcc_qupv3_wrap1_s6_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S7_CLK] = &gcc_qupv3_wrap1_s7_clk.clkr,
+	[GCC_QUPV3_WRAP1_S7_CLK_SRC] = &gcc_qupv3_wrap1_s7_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_CORE_2X_CLK] = &gcc_qupv3_wrap2_core_2x_clk.clkr,
+	[GCC_QUPV3_WRAP2_CORE_CLK] = &gcc_qupv3_wrap2_core_clk.clkr,
+	[GCC_QUPV3_WRAP2_S0_CLK] = &gcc_qupv3_wrap2_s0_clk.clkr,
+	[GCC_QUPV3_WRAP2_S0_CLK_SRC] = &gcc_qupv3_wrap2_s0_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_S1_CLK] = &gcc_qupv3_wrap2_s1_clk.clkr,
+	[GCC_QUPV3_WRAP2_S1_CLK_SRC] = &gcc_qupv3_wrap2_s1_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_S2_CLK] = &gcc_qupv3_wrap2_s2_clk.clkr,
+	[GCC_QUPV3_WRAP2_S2_CLK_SRC] = &gcc_qupv3_wrap2_s2_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_S3_CLK] = &gcc_qupv3_wrap2_s3_clk.clkr,
+	[GCC_QUPV3_WRAP2_S3_CLK_SRC] = &gcc_qupv3_wrap2_s3_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_S4_CLK] = &gcc_qupv3_wrap2_s4_clk.clkr,
+	[GCC_QUPV3_WRAP2_S4_CLK_SRC] = &gcc_qupv3_wrap2_s4_clk_src.clkr,
+	[GCC_QUPV3_WRAP3_CORE_2X_CLK] = &gcc_qupv3_wrap3_core_2x_clk.clkr,
+	[GCC_QUPV3_WRAP3_CORE_CLK] = &gcc_qupv3_wrap3_core_clk.clkr,
+	[GCC_QUPV3_WRAP3_IBI_CTRL_0_CLK_SRC] = &gcc_qupv3_wrap3_ibi_ctrl_0_clk_src.clkr,
+	[GCC_QUPV3_WRAP3_IBI_CTRL_1_CLK] = &gcc_qupv3_wrap3_ibi_ctrl_1_clk.clkr,
+	[GCC_QUPV3_WRAP3_IBI_CTRL_2_CLK] = &gcc_qupv3_wrap3_ibi_ctrl_2_clk.clkr,
+	[GCC_QUPV3_WRAP3_S0_CLK] = &gcc_qupv3_wrap3_s0_clk.clkr,
+	[GCC_QUPV3_WRAP3_S0_CLK_SRC] = &gcc_qupv3_wrap3_s0_clk_src.clkr,
+	[GCC_QUPV3_WRAP3_S1_CLK] = &gcc_qupv3_wrap3_s1_clk.clkr,
+	[GCC_QUPV3_WRAP3_S1_CLK_SRC] = &gcc_qupv3_wrap3_s1_clk_src.clkr,
+	[GCC_QUPV3_WRAP3_S2_CLK] = &gcc_qupv3_wrap3_s2_clk.clkr,
+	[GCC_QUPV3_WRAP3_S2_CLK_SRC] = &gcc_qupv3_wrap3_s2_clk_src.clkr,
+	[GCC_QUPV3_WRAP3_S3_CLK] = &gcc_qupv3_wrap3_s3_clk.clkr,
+	[GCC_QUPV3_WRAP3_S3_CLK_SRC] = &gcc_qupv3_wrap3_s3_clk_src.clkr,
+	[GCC_QUPV3_WRAP3_S4_CLK] = &gcc_qupv3_wrap3_s4_clk.clkr,
+	[GCC_QUPV3_WRAP3_S4_CLK_SRC] = &gcc_qupv3_wrap3_s4_clk_src.clkr,
+	[GCC_QUPV3_WRAP3_S5_CLK] = &gcc_qupv3_wrap3_s5_clk.clkr,
+	[GCC_QUPV3_WRAP3_S5_CLK_SRC] = &gcc_qupv3_wrap3_s5_clk_src.clkr,
+	[GCC_QUPV3_WRAP4_CORE_2X_CLK] = &gcc_qupv3_wrap4_core_2x_clk.clkr,
+	[GCC_QUPV3_WRAP4_CORE_CLK] = &gcc_qupv3_wrap4_core_clk.clkr,
+	[GCC_QUPV3_WRAP4_S0_CLK] = &gcc_qupv3_wrap4_s0_clk.clkr,
+	[GCC_QUPV3_WRAP4_S0_CLK_SRC] = &gcc_qupv3_wrap4_s0_clk_src.clkr,
+	[GCC_QUPV3_WRAP4_S1_CLK] = &gcc_qupv3_wrap4_s1_clk.clkr,
+	[GCC_QUPV3_WRAP4_S1_CLK_SRC] = &gcc_qupv3_wrap4_s1_clk_src.clkr,
+	[GCC_QUPV3_WRAP4_S2_CLK] = &gcc_qupv3_wrap4_s2_clk.clkr,
+	[GCC_QUPV3_WRAP4_S2_CLK_SRC] = &gcc_qupv3_wrap4_s2_clk_src.clkr,
+	[GCC_QUPV3_WRAP4_S3_CLK] = &gcc_qupv3_wrap4_s3_clk.clkr,
+	[GCC_QUPV3_WRAP4_S3_CLK_SRC] = &gcc_qupv3_wrap4_s3_clk_src.clkr,
+	[GCC_QUPV3_WRAP4_S4_CLK] = &gcc_qupv3_wrap4_s4_clk.clkr,
+	[GCC_QUPV3_WRAP4_S4_CLK_SRC] = &gcc_qupv3_wrap4_s4_clk_src.clkr,
+	[GCC_QUPV3_WRAP_1_M_AXI_CLK] = &gcc_qupv3_wrap_1_m_axi_clk.clkr,
+	[GCC_QUPV3_WRAP_1_S_AHB_CLK] = &gcc_qupv3_wrap_1_s_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_2_M_AHB_CLK] = &gcc_qupv3_wrap_2_m_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_2_S_AHB_CLK] = &gcc_qupv3_wrap_2_s_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_3_IBI_1_AHB_CLK] = &gcc_qupv3_wrap_3_ibi_1_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_3_IBI_2_AHB_CLK] = &gcc_qupv3_wrap_3_ibi_2_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_3_M_AHB_CLK] = &gcc_qupv3_wrap_3_m_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_3_S_AHB_CLK] = &gcc_qupv3_wrap_3_s_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_4_M_AHB_CLK] = &gcc_qupv3_wrap_4_m_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_4_S_AHB_CLK] = &gcc_qupv3_wrap_4_s_ahb_clk.clkr,
+	[GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr,
+	[GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr,
+	[GCC_SDCC2_APPS_CLK_SRC] = &gcc_sdcc2_apps_clk_src.clkr,
+	[GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr,
+	[GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr,
+	[GCC_SDCC4_APPS_CLK_SRC] = &gcc_sdcc4_apps_clk_src.clkr,
+	[GCC_UFS_PHY_AHB_CLK] = &gcc_ufs_phy_ahb_clk.clkr,
+	[GCC_UFS_PHY_AXI_CLK] = &gcc_ufs_phy_axi_clk.clkr,
+	[GCC_UFS_PHY_AXI_CLK_SRC] = &gcc_ufs_phy_axi_clk_src.clkr,
+	[GCC_UFS_PHY_ICE_CORE_CLK] = &gcc_ufs_phy_ice_core_clk.clkr,
+	[GCC_UFS_PHY_ICE_CORE_CLK_SRC] = &gcc_ufs_phy_ice_core_clk_src.clkr,
+	[GCC_UFS_PHY_PHY_AUX_CLK] = &gcc_ufs_phy_phy_aux_clk.clkr,
+	[GCC_UFS_PHY_PHY_AUX_CLK_SRC] = &gcc_ufs_phy_phy_aux_clk_src.clkr,
+	[GCC_UFS_PHY_RX_SYMBOL_0_CLK] = &gcc_ufs_phy_rx_symbol_0_clk.clkr,
+	[GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC] = &gcc_ufs_phy_rx_symbol_0_clk_src.clkr,
+	[GCC_UFS_PHY_RX_SYMBOL_1_CLK] = &gcc_ufs_phy_rx_symbol_1_clk.clkr,
+	[GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC] = &gcc_ufs_phy_rx_symbol_1_clk_src.clkr,
+	[GCC_UFS_PHY_TX_SYMBOL_0_CLK] = &gcc_ufs_phy_tx_symbol_0_clk.clkr,
+	[GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC] = &gcc_ufs_phy_tx_symbol_0_clk_src.clkr,
+	[GCC_UFS_PHY_UNIPRO_CORE_CLK] = &gcc_ufs_phy_unipro_core_clk.clkr,
+	[GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC] = &gcc_ufs_phy_unipro_core_clk_src.clkr,
+	[GCC_USB30_PRIM_MASTER_CLK] = &gcc_usb30_prim_master_clk.clkr,
+	[GCC_USB30_PRIM_MASTER_CLK_SRC] = &gcc_usb30_prim_master_clk_src.clkr,
+	[GCC_USB30_PRIM_MOCK_UTMI_CLK] = &gcc_usb30_prim_mock_utmi_clk.clkr,
+	[GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC] = &gcc_usb30_prim_mock_utmi_clk_src.clkr,
+	[GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC] = &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr,
+	[GCC_USB30_PRIM_SLEEP_CLK] = &gcc_usb30_prim_sleep_clk.clkr,
+	[GCC_USB3_PRIM_PHY_AUX_CLK] = &gcc_usb3_prim_phy_aux_clk.clkr,
+	[GCC_USB3_PRIM_PHY_AUX_CLK_SRC] = &gcc_usb3_prim_phy_aux_clk_src.clkr,
+	[GCC_USB3_PRIM_PHY_COM_AUX_CLK] = &gcc_usb3_prim_phy_com_aux_clk.clkr,
+	[GCC_USB3_PRIM_PHY_PIPE_CLK] = &gcc_usb3_prim_phy_pipe_clk.clkr,
+	[GCC_USB3_PRIM_PHY_PIPE_CLK_SRC] = &gcc_usb3_prim_phy_pipe_clk_src.clkr,
+	[GCC_VIDEO_AXI0_CLK] = &gcc_video_axi0_clk.clkr,
+	[GCC_VIDEO_AXI1_CLK] = &gcc_video_axi1_clk.clkr,
+	[GCC_QMIP_CAMERA_CMD_AHB_CLK] = &gcc_qmip_camera_cmd_ahb_clk.clkr,
+	[GCC_QMIP_VIDEO_VCODEC_AHB_CLK] = &gcc_qmip_video_vcodec_ahb_clk.clkr,
+	[GCC_QMIP_GPU_AHB_CLK] = &gcc_qmip_gpu_ahb_clk.clkr,
+	[GCC_QMIP_CAMERA_NRT_AHB_CLK] = &gcc_qmip_camera_nrt_ahb_clk.clkr,
+	[GCC_QMIP_CAMERA_RT_AHB_CLK] = &gcc_qmip_camera_rt_ahb_clk.clkr,
+	[GCC_QMIP_DISP_DCP_SF_AHB_CLK] = &gcc_qmip_disp_dcp_sf_ahb_clk.clkr,
+	[GCC_QMIP_PCIE_AHB_CLK] = &gcc_qmip_pcie_ahb_clk.clkr,
+	[GCC_QMIP_VIDEO_CV_CPU_AHB_CLK] = &gcc_qmip_video_cv_cpu_ahb_clk.clkr,
+	[GCC_QMIP_VIDEO_CVP_AHB_CLK] = &gcc_qmip_video_cvp_ahb_clk.clkr,
+	[GCC_QMIP_VIDEO_V_CPU_AHB_CLK] = &gcc_qmip_video_v_cpu_ahb_clk.clkr,
+	[GCC_QMIP_VIDEO_VCODEC_AHB_CLK] = &gcc_qmip_video_vcodec_ahb_clk.clkr,
+};
+
+static struct gdsc *gcc_kaanapali_gdscs[] = {
+	[GCC_PCIE_0_GDSC] = &gcc_pcie_0_gdsc,
+	[GCC_PCIE_0_PHY_GDSC] = &gcc_pcie_0_phy_gdsc,
+	[GCC_UFS_MEM_PHY_GDSC] = &gcc_ufs_mem_phy_gdsc,
+	[GCC_UFS_PHY_GDSC] = &gcc_ufs_phy_gdsc,
+	[GCC_USB30_PRIM_GDSC] = &gcc_usb30_prim_gdsc,
+	[GCC_USB3_PHY_GDSC] = &gcc_usb3_phy_gdsc,
+};
+
+static const struct qcom_reset_map gcc_kaanapali_resets[] = {
+	[GCC_CAMERA_BCR] = { 0x26000 },
+	[GCC_DISPLAY_BCR] = { 0x27000 },
+	[GCC_EVA_AXI0_CLK_ARES] = { 0x9f008, 2 },
+	[GCC_EVA_AXI0C_CLK_ARES] = { 0x9f01c, 2 },
+	[GCC_EVA_BCR] = { 0x9f000 },
+	[GCC_GPU_BCR] = { 0x71000 },
+	[GCC_PCIE_0_BCR] = { 0x6b000 },
+	[GCC_PCIE_0_LINK_DOWN_BCR] = { 0x6c014 },
+	[GCC_PCIE_0_NOCSR_COM_PHY_BCR] = { 0x6c020 },
+	[GCC_PCIE_0_PHY_BCR] = { 0x6c01c },
+	[GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR] = { 0x6c028 },
+	[GCC_PCIE_PHY_BCR] = { 0x6f000 },
+	[GCC_PCIE_PHY_CFG_AHB_BCR] = { 0x6f00c },
+	[GCC_PCIE_PHY_COM_BCR] = { 0x6f010 },
+	[GCC_PCIE_RSCC_BCR] = { 0x11000 },
+	[GCC_PDM_BCR] = { 0x33000 },
+	[GCC_QUPV3_WRAPPER_1_BCR] = { 0x18000 },
+	[GCC_QUPV3_WRAPPER_2_BCR] = { 0x1e000 },
+	[GCC_QUPV3_WRAPPER_3_BCR] = { 0xa8000 },
+	[GCC_QUPV3_WRAPPER_4_BCR] = { 0xa9000 },
+	[GCC_QUPV3_WRAPPER_I2C_BCR] = { 0x17000 },
+	[GCC_QUSB2PHY_PRIM_BCR] = { 0x12000 },
+	[GCC_QUSB2PHY_SEC_BCR] = { 0x12004 },
+	[GCC_SDCC2_BCR] = { 0x14000 },
+	[GCC_SDCC4_BCR] = { 0x16000 },
+	[GCC_UFS_PHY_BCR] = { 0x77000 },
+	[GCC_USB30_PRIM_BCR] = { 0x39000 },
+	[GCC_USB3_DP_PHY_PRIM_BCR] = { 0x50008 },
+	[GCC_USB3_DP_PHY_SEC_BCR] = { 0x50014 },
+	[GCC_USB3_PHY_PRIM_BCR] = { 0x50000 },
+	[GCC_USB3_PHY_SEC_BCR] = { 0x5000c },
+	[GCC_USB3PHY_PHY_PRIM_BCR] = { 0x50004 },
+	[GCC_USB3PHY_PHY_SEC_BCR] = { 0x50010 },
+	[GCC_VIDEO_AXI0_CLK_ARES] = { 0x32018, 2 },
+	[GCC_VIDEO_AXI1_CLK_ARES] = { 0x3202c, 2 },
+	[GCC_VIDEO_BCR] = { 0x32000 },
+	[GCC_VIDEO_XO_CLK_ARES] = { 0x32040, 2 },
+};
+
+static const struct clk_rcg_dfs_data gcc_dfs_clocks[] = {
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_qspi_ref_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s0_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s1_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s3_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s4_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s5_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s6_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s7_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s0_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s1_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s2_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s3_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s4_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap3_s0_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap3_s1_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap3_s2_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap3_s3_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap3_s4_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap3_s5_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap4_s0_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap4_s1_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap4_s2_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap4_s3_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap4_s4_clk_src),
+};
+
+static u32 gcc_kaanapali_critical_cbcrs[] = {
+	0xa0004, /* GCC_CAM_BIST_MCLK_AHB_CLK */
+	0x26004, /* GCC_CAMERA_AHB_CLK */
+	0x2603c, /* GCC_CAMERA_XO_CLK */
+	0x27004, /* GCC_DISP_AHB_CLK */
+	0x9f004, /* GCC_EVA_AHB_CLK */
+	0x9f024, /* GCC_EVA_XO_CLK */
+	0x71004, /* GCC_GPU_CFG_AHB_CLK */
+	0x52010, /* GCC_PCIE_RSCC_CFG_AHB_CLK */
+	0x52010, /* GCC_PCIE_RSCC_XO_CLK */
+	0x32004, /* GCC_VIDEO_AHB_CLK */
+	0x32040, /* GCC_VIDEO_XO_CLK */
+};
+
+static const struct regmap_config gcc_kaanapali_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1f41f0,
+	.fast_io = true,
+};
+
+static void clk_kaanapali_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
+	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
+}
+
+static struct qcom_cc_driver_data gcc_kaanapali_driver_data = {
+	.clk_cbcrs = gcc_kaanapali_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_kaanapali_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+	.clk_regs_configure = clk_kaanapali_regs_configure,
+};
+
+static const struct qcom_cc_desc gcc_kaanapali_desc = {
+	.config = &gcc_kaanapali_regmap_config,
+	.clks = gcc_kaanapali_clocks,
+	.num_clks = ARRAY_SIZE(gcc_kaanapali_clocks),
+	.resets = gcc_kaanapali_resets,
+	.num_resets = ARRAY_SIZE(gcc_kaanapali_resets),
+	.gdscs = gcc_kaanapali_gdscs,
+	.num_gdscs = ARRAY_SIZE(gcc_kaanapali_gdscs),
+	.driver_data = &gcc_kaanapali_driver_data,
+};
+
+static const struct of_device_id gcc_kaanapali_match_table[] = {
+	{ .compatible = "qcom,kaanapali-gcc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, gcc_kaanapali_match_table);
+
+static int gcc_kaanapali_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &gcc_kaanapali_desc);
+}
+
+static struct platform_driver gcc_kaanapali_driver = {
+	.probe = gcc_kaanapali_probe,
+	.driver = {
+		.name = "gcc-kaanapali",
+		.of_match_table = gcc_kaanapali_match_table,
+	},
+};
+
+static int __init gcc_kaanapali_init(void)
+{
+	return platform_driver_register(&gcc_kaanapali_driver);
+}
+subsys_initcall(gcc_kaanapali_init);
+
+static void __exit gcc_kaanapali_exit(void)
+{
+	platform_driver_unregister(&gcc_kaanapali_driver);
+}
+module_exit(gcc_kaanapali_exit);
+
+MODULE_DESCRIPTION("QTI GCC Kaanapali Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-milos.c b/drivers/clk/qcom/gcc-milos.c
index c9d61b05bafa..81fa09ec55d7 100644
--- a/drivers/clk/qcom/gcc-milos.c
+++ b/drivers/clk/qcom/gcc-milos.c
@@ -917,7 +917,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = {
 		.name = "gcc_sdcc1_apps_clk_src",
 		.parent_data = gcc_parent_data_9,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_9),
-		.ops = &clk_rcg2_shared_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
@@ -938,7 +938,7 @@ static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = {
 		.name = "gcc_sdcc1_ice_core_clk_src",
 		.parent_data = gcc_parent_data_10,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_10),
-		.ops = &clk_rcg2_shared_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
@@ -962,7 +962,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = {
 		.name = "gcc_sdcc2_apps_clk_src",
 		.parent_data = gcc_parent_data_11,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_11),
-		.ops = &clk_rcg2_shared_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
diff --git a/drivers/clk/qcom/gcc-qdu1000.c b/drivers/clk/qcom/gcc-qdu1000.c
index dbe9e9437939..915bb9b4ff81 100644
--- a/drivers/clk/qcom/gcc-qdu1000.c
+++ b/drivers/clk/qcom/gcc-qdu1000.c
@@ -904,7 +904,7 @@ static struct clk_rcg2 gcc_sdcc5_apps_clk_src = {
 		.name = "gcc_sdcc5_apps_clk_src",
 		.parent_data = gcc_parent_data_8,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_8),
-		.ops = &clk_rcg2_floor_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
@@ -923,7 +923,7 @@ static struct clk_rcg2 gcc_sdcc5_ice_core_clk_src = {
 		.name = "gcc_sdcc5_ice_core_clk_src",
 		.parent_data = gcc_parent_data_2,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
-		.ops = &clk_rcg2_floor_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
diff --git a/drivers/clk/qcom/gcc-sdx75.c b/drivers/clk/qcom/gcc-sdx75.c
index 453a6bf8e878..1f3cd58483a2 100644
--- a/drivers/clk/qcom/gcc-sdx75.c
+++ b/drivers/clk/qcom/gcc-sdx75.c
@@ -1033,7 +1033,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = {
 		.name = "gcc_sdcc1_apps_clk_src",
 		.parent_data = gcc_parent_data_17,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_17),
-		.ops = &clk_rcg2_floor_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
@@ -1057,7 +1057,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = {
 		.name = "gcc_sdcc2_apps_clk_src",
 		.parent_data = gcc_parent_data_18,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_18),
-		.ops = &clk_rcg2_floor_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
diff --git a/drivers/clk/qcom/gcc-sm4450.c b/drivers/clk/qcom/gcc-sm4450.c
index e2d9e4691c5b..023d840e9f4e 100644
--- a/drivers/clk/qcom/gcc-sm4450.c
+++ b/drivers/clk/qcom/gcc-sm4450.c
@@ -769,7 +769,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = {
 		.parent_data = gcc_parent_data_4,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_4),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_rcg2_floor_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
@@ -791,7 +791,7 @@ static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = {
 		.parent_data = gcc_parent_data_4,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_4),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_rcg2_floor_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
@@ -815,7 +815,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = {
 		.parent_data = gcc_parent_data_6,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_6),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_rcg2_floor_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
diff --git a/drivers/clk/qcom/gcc-sm8450.c b/drivers/clk/qcom/gcc-sm8450.c
index 65d7d52bce03..b18bb34889ab 100644
--- a/drivers/clk/qcom/gcc-sm8450.c
+++ b/drivers/clk/qcom/gcc-sm8450.c
@@ -1034,7 +1034,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = {
 		.parent_data = gcc_parent_data_7,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_7),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_rcg2_floor_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
@@ -1057,7 +1057,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = {
 		.parent_data = gcc_parent_data_0,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_rcg2_floor_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
diff --git a/drivers/clk/qcom/gcc-sm8750.c b/drivers/clk/qcom/gcc-sm8750.c
index def86b71a3da..db81569dd4b1 100644
--- a/drivers/clk/qcom/gcc-sm8750.c
+++ b/drivers/clk/qcom/gcc-sm8750.c
@@ -1030,7 +1030,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = {
 		.parent_data = gcc_parent_data_8,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_8),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_rcg2_floor_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
@@ -1052,7 +1052,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = {
 		.parent_data = gcc_parent_data_0,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_rcg2_floor_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
diff --git a/drivers/clk/qcom/gcc-x1e80100.c b/drivers/clk/qcom/gcc-x1e80100.c
index b63c8abdd2fc..e46e65e63151 100644
--- a/drivers/clk/qcom/gcc-x1e80100.c
+++ b/drivers/clk/qcom/gcc-x1e80100.c
@@ -1516,7 +1516,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = {
 		.parent_data = gcc_parent_data_9,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_9),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_rcg2_floor_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
@@ -1538,7 +1538,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = {
 		.parent_data = gcc_parent_data_0,
 		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_rcg2_floor_ops,
+		.ops = &clk_rcg2_shared_floor_ops,
 	},
 };
 
diff --git a/drivers/clk/qcom/tcsrcc-kaanapali.c b/drivers/clk/qcom/tcsrcc-kaanapali.c
new file mode 100644
index 000000000000..4da77367c9e0
--- /dev/null
+++ b/drivers/clk/qcom/tcsrcc-kaanapali.c
@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,sm8750-tcsr.h>
+
+#include "clk-branch.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "common.h"
+
+enum {
+	DT_BI_TCXO_PAD,
+};
+
+static struct clk_branch tcsr_pcie_0_clkref_en = {
+	.halt_reg = 0x15044,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x15044,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "tcsr_pcie_0_clkref_en",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch tcsr_usb3_clkref_en = {
+	.halt_reg = 0x1504c,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x1504c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "tcsr_usb3_clkref_en",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_BI_TCXO_PAD,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch tcsr_ufs_clkref_en = {
+	.halt_reg = 0x15054,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x15054,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "tcsr_ufs_clkref_en",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_BI_TCXO_PAD,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch tcsr_usb2_clkref_en = {
+	.halt_reg = 0x1505c,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x1505c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "tcsr_usb2_clkref_en",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_BI_TCXO_PAD,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_regmap *tcsr_cc_kaanapali_clocks[] = {
+	[TCSR_PCIE_0_CLKREF_EN] = &tcsr_pcie_0_clkref_en.clkr,
+	[TCSR_UFS_CLKREF_EN] = &tcsr_ufs_clkref_en.clkr,
+	[TCSR_USB2_CLKREF_EN] = &tcsr_usb2_clkref_en.clkr,
+	[TCSR_USB3_CLKREF_EN] = &tcsr_usb3_clkref_en.clkr,
+};
+
+static const struct regmap_config tcsr_cc_kaanapali_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x3d000,
+	.fast_io = true,
+};
+
+static const struct qcom_cc_desc tcsr_cc_kaanapali_desc = {
+	.config = &tcsr_cc_kaanapali_regmap_config,
+	.clks = tcsr_cc_kaanapali_clocks,
+	.num_clks = ARRAY_SIZE(tcsr_cc_kaanapali_clocks),
+};
+
+static const struct of_device_id tcsr_cc_kaanapali_match_table[] = {
+	{ .compatible = "qcom,kaanapali-tcsr" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, tcsr_cc_kaanapali_match_table);
+
+static int tcsr_cc_kaanapali_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &tcsr_cc_kaanapali_desc);
+}
+
+static struct platform_driver tcsr_cc_kaanapali_driver = {
+	.probe = tcsr_cc_kaanapali_probe,
+	.driver = {
+		.name = "tcsr_cc-kaanapali",
+		.of_match_table = tcsr_cc_kaanapali_match_table,
+	},
+};
+
+static int __init tcsr_cc_kaanapali_init(void)
+{
+	return platform_driver_register(&tcsr_cc_kaanapali_driver);
+}
+subsys_initcall(tcsr_cc_kaanapali_init);
+
+static void __exit tcsr_cc_kaanapali_exit(void)
+{
+	platform_driver_unregister(&tcsr_cc_kaanapali_driver);
+}
+module_exit(tcsr_cc_kaanapali_exit);
+
+MODULE_DESCRIPTION("QTI TCSR_CC Kaanapali Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/renesas/r9a09g056-cpg.c b/drivers/clk/renesas/r9a09g056-cpg.c
index f48a082e65d7..4b068199edde 100644
--- a/drivers/clk/renesas/r9a09g056-cpg.c
+++ b/drivers/clk/renesas/r9a09g056-cpg.c
@@ -51,6 +51,8 @@ enum clk_ids {
 	CLK_PLLDTY_ACPU_DIV4,
 	CLK_PLLDTY_DIV8,
 	CLK_PLLDTY_DIV16,
+	CLK_PLLDTY_RCPU,
+	CLK_PLLDTY_RCPU_DIV4,
 	CLK_PLLVDO_CRU0,
 	CLK_PLLVDO_CRU1,
 	CLK_PLLVDO_ISP,
@@ -184,6 +186,8 @@ static const struct cpg_core_clk r9a09g056_core_clks[] __initconst = {
 	DEF_FIXED(".plldty_acpu_div4", CLK_PLLDTY_ACPU_DIV4, CLK_PLLDTY_ACPU, 1, 4),
 	DEF_FIXED(".plldty_div8", CLK_PLLDTY_DIV8, CLK_PLLDTY, 1, 8),
 	DEF_FIXED(".plldty_div16", CLK_PLLDTY_DIV16, CLK_PLLDTY, 1, 16),
+	DEF_DDIV(".plldty_rcpu", CLK_PLLDTY_RCPU, CLK_PLLDTY, CDDIV3_DIVCTL2, dtable_2_64),
+	DEF_FIXED(".plldty_rcpu_div4", CLK_PLLDTY_RCPU_DIV4, CLK_PLLDTY_RCPU, 1, 4),
 
 	DEF_DDIV(".pllvdo_cru0", CLK_PLLVDO_CRU0, CLK_PLLVDO, CDDIV3_DIVCTL3, dtable_2_4),
 	DEF_DDIV(".pllvdo_cru1", CLK_PLLVDO_CRU1, CLK_PLLVDO, CDDIV4_DIVCTL0, dtable_2_4),
@@ -231,6 +235,18 @@ static const struct cpg_core_clk r9a09g056_core_clks[] __initconst = {
 };
 
 static const struct rzv2h_mod_clk r9a09g056_mod_clks[] __initconst = {
+	DEF_MOD("dmac_0_aclk",			CLK_PLLCM33_GEAR, 0, 0, 0, 0,
+						BUS_MSTOP(5, BIT(9))),
+	DEF_MOD("dmac_1_aclk",			CLK_PLLDTY_ACPU_DIV2, 0, 1, 0, 1,
+						BUS_MSTOP(3, BIT(2))),
+	DEF_MOD("dmac_2_aclk",			CLK_PLLDTY_ACPU_DIV2, 0, 2, 0, 2,
+						BUS_MSTOP(3, BIT(3))),
+	DEF_MOD("dmac_3_aclk",			CLK_PLLDTY_RCPU_DIV4, 0, 3, 0, 3,
+						BUS_MSTOP(10, BIT(11))),
+	DEF_MOD("dmac_4_aclk",			CLK_PLLDTY_RCPU_DIV4, 0, 4, 0, 4,
+						BUS_MSTOP(10, BIT(12))),
+	DEF_MOD_CRITICAL("icu_0_pclk_i",	CLK_PLLCM33_DIV16, 0, 5, 0, 5,
+						BUS_MSTOP_NONE),
 	DEF_MOD_CRITICAL("gic_0_gicclk",	CLK_PLLDTY_ACPU_DIV4, 1, 3, 0, 19,
 						BUS_MSTOP(3, BIT(5))),
 	DEF_MOD("gtm_0_pclk",			CLK_PLLCM33_DIV16, 4, 3, 2, 3,
@@ -265,6 +281,24 @@ static const struct rzv2h_mod_clk r9a09g056_mod_clks[] __initconst = {
 						BUS_MSTOP(5, BIT(13))),
 	DEF_MOD("wdt_3_clk_loco",		CLK_QEXTAL, 5, 2, 2, 18,
 						BUS_MSTOP(5, BIT(13))),
+	DEF_MOD("rspi_0_pclk",			CLK_PLLCLN_DIV8, 5, 4, 2, 20,
+						BUS_MSTOP(11, BIT(0))),
+	DEF_MOD("rspi_0_pclk_sfr",		CLK_PLLCLN_DIV8, 5, 5, 2, 21,
+						BUS_MSTOP(11, BIT(0))),
+	DEF_MOD("rspi_0_tclk",			CLK_PLLCLN_DIV8, 5, 6, 2, 22,
+						BUS_MSTOP(11, BIT(0))),
+	DEF_MOD("rspi_1_pclk",			CLK_PLLCLN_DIV8, 5, 7, 2, 23,
+						BUS_MSTOP(11, BIT(1))),
+	DEF_MOD("rspi_1_pclk_sfr",		CLK_PLLCLN_DIV8, 5, 8, 2, 24,
+						BUS_MSTOP(11, BIT(1))),
+	DEF_MOD("rspi_1_tclk",			CLK_PLLCLN_DIV8, 5, 9, 2, 25,
+						BUS_MSTOP(11, BIT(1))),
+	DEF_MOD("rspi_2_pclk",			CLK_PLLCLN_DIV8, 5, 10, 2, 26,
+						BUS_MSTOP(11, BIT(2))),
+	DEF_MOD("rspi_2_pclk_sfr",		CLK_PLLCLN_DIV8, 5, 11, 2, 27,
+						BUS_MSTOP(11, BIT(2))),
+	DEF_MOD("rspi_2_tclk",			CLK_PLLCLN_DIV8, 5, 12, 2, 28,
+						BUS_MSTOP(11, BIT(2))),
 	DEF_MOD("scif_0_clk_pck",		CLK_PLLCM33_DIV16, 8, 15, 4, 15,
 						BUS_MSTOP(3, BIT(14))),
 	DEF_MOD("i3c_0_pclkrw",			CLK_PLLCLN_DIV16, 9, 0, 4, 16,
@@ -401,6 +435,12 @@ static const struct rzv2h_mod_clk r9a09g056_mod_clks[] __initconst = {
 
 static const struct rzv2h_reset r9a09g056_resets[] __initconst = {
 	DEF_RST(3, 0, 1, 1),		/* SYS_0_PRESETN */
+	DEF_RST(3, 1, 1, 2),		/* DMAC_0_ARESETN */
+	DEF_RST(3, 2, 1, 3),		/* DMAC_1_ARESETN */
+	DEF_RST(3, 3, 1, 4),		/* DMAC_2_ARESETN */
+	DEF_RST(3, 4, 1, 5),		/* DMAC_3_ARESETN */
+	DEF_RST(3, 5, 1, 6),		/* DMAC_4_ARESETN */
+	DEF_RST(3, 6, 1, 7),		/* ICU_0_PRESETN_I */
 	DEF_RST(3, 8, 1, 9),		/* GIC_0_GICRESET_N */
 	DEF_RST(3, 9, 1, 10),		/* GIC_0_DBG_GICRESET_N */
 	DEF_RST(6, 13, 2, 30),		/* GTM_0_PRESETZ */
@@ -415,6 +455,12 @@ static const struct rzv2h_reset r9a09g056_resets[] __initconst = {
 	DEF_RST(7, 6, 3, 7),		/* WDT_1_RESET */
 	DEF_RST(7, 7, 3, 8),		/* WDT_2_RESET */
 	DEF_RST(7, 8, 3, 9),		/* WDT_3_RESET */
+	DEF_RST(7, 11, 3, 12),		/* RSPI_0_PRESETN */
+	DEF_RST(7, 12, 3, 13),		/* RSPI_0_TRESETN */
+	DEF_RST(7, 13, 3, 14),		/* RSPI_1_PRESETN */
+	DEF_RST(7, 14, 3, 15),		/* RSPI_1_TRESETN */
+	DEF_RST(7, 15, 3, 16),		/* RSPI_2_PRESETN */
+	DEF_RST(8, 0, 3, 17),		/* RSPI_2_TRESETN */
 	DEF_RST(9, 5, 4, 6),		/* SCIF_0_RST_SYSTEM_N */
 	DEF_RST(9, 6, 4, 7),		/* I3C_0_PRESETN */
 	DEF_RST(9, 7, 4, 8),		/* I3C_0_TRESETN */
diff --git a/drivers/clk/renesas/r9a09g077-cpg.c b/drivers/clk/renesas/r9a09g077-cpg.c
index fb6cc94d08a1..dee25cdadf1d 100644
--- a/drivers/clk/renesas/r9a09g077-cpg.c
+++ b/drivers/clk/renesas/r9a09g077-cpg.c
@@ -11,6 +11,8 @@
 #include <linux/device.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
+#include <linux/math.h>
+#include <linux/types.h>
 
 #include <dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h>
 #include <dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h>
@@ -41,6 +43,12 @@
 #define GET_WIDTH(val)         FIELD_GET(WIDTH_MASK, val)
 #define GET_REG_OFFSET(val)    FIELD_GET(OFFSET_MASK, val)
 
+#define FSELXSPI0	CONF_PACK(SCKCR, 0, 3)
+#define FSELXSPI1	CONF_PACK(SCKCR, 8, 3)
+#define DIVSEL_XSPI0	CONF_PACK(SCKCR, 6, 1)
+#define DIVSEL_XSPI1	CONF_PACK(SCKCR, 14, 1)
+#define SEL_PLL		CONF_PACK(SCKCR, 22, 1)
+
 #define DIVCA55C0	CONF_PACK(SCKCR2, 8, 1)
 #define DIVCA55C1	CONF_PACK(SCKCR2, 9, 1)
 #define DIVCA55C2	CONF_PACK(SCKCR2, 10, 1)
@@ -58,11 +66,10 @@
 #define DIVSCI3ASYNC	CONF_PACK(SCKCR3, 12, 2)
 #define DIVSCI4ASYNC	CONF_PACK(SCKCR3, 14, 2)
 
-#define SEL_PLL		CONF_PACK(SCKCR, 22, 1)
-
 enum rzt2h_clk_types {
 	CLK_TYPE_RZT2H_DIV = CLK_TYPE_CUSTOM,	/* Clock with divider */
 	CLK_TYPE_RZT2H_MUX,			/* Clock with clock source selector */
+	CLK_TYPE_RZT2H_FSELXSPI,		/* Clock with FSELXSPIn source selector */
 };
 
 #define DEF_DIV(_name, _id, _parent, _conf, _dtable) \
@@ -71,11 +78,14 @@ enum rzt2h_clk_types {
 #define DEF_MUX(_name, _id, _conf, _parent_names, _num_parents, _mux_flags) \
 	DEF_TYPE(_name, _id, CLK_TYPE_RZT2H_MUX, .conf = _conf, \
 		 .parent_names = _parent_names, .num_parents = _num_parents, \
-		 .flag = 0, .mux_flags = _mux_flags)
+		 .flag = CLK_SET_RATE_PARENT, .mux_flags = _mux_flags)
+#define DEF_DIV_FSELXSPI(_name, _id, _parent, _conf, _dtable) \
+	DEF_TYPE(_name, _id, CLK_TYPE_RZT2H_FSELXSPI, .conf = _conf, \
+		 .parent = _parent, .dtable = _dtable, .flag = 0)
 
 enum clk_ids {
 	/* Core Clock Outputs exported to DT */
-	LAST_DT_CORE_CLK = R9A09G077_ETCLKE,
+	LAST_DT_CORE_CLK = R9A09G077_XSPI_CLK1,
 
 	/* External Input Clocks */
 	CLK_EXTAL,
@@ -91,6 +101,8 @@ enum clk_ids {
 	CLK_SEL_CLK_PLL2,
 	CLK_SEL_CLK_PLL4,
 	CLK_PLL4D1,
+	CLK_PLL4D1_DIV3,
+	CLK_PLL4D1_DIV4,
 	CLK_SCI0ASYNC,
 	CLK_SCI1ASYNC,
 	CLK_SCI2ASYNC,
@@ -101,6 +113,8 @@ enum clk_ids {
 	CLK_SPI1ASYNC,
 	CLK_SPI2ASYNC,
 	CLK_SPI3ASYNC,
+	CLK_DIVSELXSPI0_SCKCR,
+	CLK_DIVSELXSPI1_SCKCR,
 
 	/* Module Clocks */
 	MOD_CLK_BASE,
@@ -112,6 +126,15 @@ static const struct clk_div_table dtable_1_2[] = {
 	{0, 0},
 };
 
+static const struct clk_div_table dtable_6_8_16_32_64[] = {
+	{6, 64},
+	{5, 32},
+	{4, 16},
+	{3, 8},
+	{2, 6},
+	{0, 0},
+};
+
 static const struct clk_div_table dtable_24_25_30_32[] = {
 	{0, 32},
 	{1, 30},
@@ -126,6 +149,7 @@ static const char * const sel_clk_pll0[] = { ".loco", ".pll0" };
 static const char * const sel_clk_pll1[] = { ".loco", ".pll1" };
 static const char * const sel_clk_pll2[] = { ".loco", ".pll2" };
 static const char * const sel_clk_pll4[] = { ".loco", ".pll4" };
+static const char * const sel_clk_pll4d1_div3_div4[] = { ".pll4d1_div3", ".pll4d1_div4" };
 
 static const struct cpg_core_clk r9a09g077_core_clks[] __initconst = {
 	/* External Clock Inputs */
@@ -148,6 +172,9 @@ static const struct cpg_core_clk r9a09g077_core_clks[] __initconst = {
 		sel_clk_pll4, ARRAY_SIZE(sel_clk_pll4), CLK_MUX_READ_ONLY),
 
 	DEF_FIXED(".pll4d1", CLK_PLL4D1, CLK_SEL_CLK_PLL4, 1, 1),
+	DEF_FIXED(".pll4d1_div3", CLK_PLL4D1_DIV3, CLK_PLL4D1, 3, 1),
+	DEF_FIXED(".pll4d1_div4", CLK_PLL4D1_DIV4, CLK_PLL4D1, 4, 1),
+
 	DEF_DIV(".sci0async", CLK_SCI0ASYNC, CLK_PLL4D1, DIVSCI0ASYNC,
 		dtable_24_25_30_32),
 	DEF_DIV(".sci1async", CLK_SCI1ASYNC, CLK_PLL4D1, DIVSCI1ASYNC,
@@ -170,6 +197,13 @@ static const struct cpg_core_clk r9a09g077_core_clks[] __initconst = {
 	DEF_DIV(".spi3async", CLK_SPI3ASYNC, CLK_PLL4D1, DIVSPI3ASYNC,
 		dtable_24_25_30_32),
 
+	DEF_MUX(".divselxspi0", CLK_DIVSELXSPI0_SCKCR, DIVSEL_XSPI0,
+		sel_clk_pll4d1_div3_div4,
+		ARRAY_SIZE(sel_clk_pll4d1_div3_div4), 0),
+	DEF_MUX(".divselxspi1", CLK_DIVSELXSPI1_SCKCR, DIVSEL_XSPI1,
+		sel_clk_pll4d1_div3_div4,
+		ARRAY_SIZE(sel_clk_pll4d1_div3_div4), 0),
+
 	/* Core output clk */
 	DEF_DIV("CA55C0", R9A09G077_CLK_CA55C0, CLK_SEL_CLK_PLL0, DIVCA55C0,
 		dtable_1_2),
@@ -194,9 +228,15 @@ static const struct cpg_core_clk r9a09g077_core_clks[] __initconst = {
 	DEF_FIXED("ETCLKC", R9A09G077_ETCLKC, CLK_SEL_CLK_PLL1, 10, 1),
 	DEF_FIXED("ETCLKD", R9A09G077_ETCLKD, CLK_SEL_CLK_PLL1, 20, 1),
 	DEF_FIXED("ETCLKE", R9A09G077_ETCLKE, CLK_SEL_CLK_PLL1, 40, 1),
+	DEF_DIV_FSELXSPI("XSPI_CLK0", R9A09G077_XSPI_CLK0, CLK_DIVSELXSPI0_SCKCR,
+			 FSELXSPI0, dtable_6_8_16_32_64),
+	DEF_DIV_FSELXSPI("XSPI_CLK1", R9A09G077_XSPI_CLK1, CLK_DIVSELXSPI1_SCKCR,
+			 FSELXSPI1, dtable_6_8_16_32_64),
 };
 
 static const struct mssr_mod_clk r9a09g077_mod_clks[] __initconst = {
+	DEF_MOD("xspi0", 4, R9A09G077_CLK_PCLKH),
+	DEF_MOD("xspi1", 5, R9A09G077_CLK_PCLKH),
 	DEF_MOD("sci0fck", 8, CLK_SCI0ASYNC),
 	DEF_MOD("sci1fck", 9, CLK_SCI1ASYNC),
 	DEF_MOD("sci2fck", 10, CLK_SCI2ASYNC),
@@ -284,6 +324,151 @@ r9a09g077_cpg_mux_clk_register(struct device *dev,
 	return clk_hw->clk;
 }
 
+static unsigned int r9a09g077_cpg_fselxspi_get_divider(struct clk_hw *hw, unsigned long rate,
+						       unsigned int num_parents)
+{
+	struct clk_fixed_factor *ff;
+	struct clk_hw *parent_hw;
+	unsigned long best_rate;
+	unsigned int i;
+
+	for (i = 0; i < num_parents; i++) {
+		parent_hw = clk_hw_get_parent_by_index(hw, i);
+		best_rate = clk_hw_round_rate(parent_hw, rate);
+
+		if (best_rate == rate) {
+			ff = to_clk_fixed_factor(parent_hw);
+			return ff->div;
+		}
+	}
+
+	/* No parent could provide the exact rate - this should not happen */
+	return 0;
+}
+
+static int r9a09g077_cpg_fselxspi_determine_rate(struct clk_hw *hw,
+						 struct clk_rate_request *req)
+{
+	struct clk_divider *divider = to_clk_divider(hw);
+	unsigned long parent_rate, best = 0, now;
+	const struct clk_div_table *clkt;
+	unsigned long rate = req->rate;
+	unsigned int num_parents;
+	unsigned int divselxspi;
+	unsigned int div = 0;
+
+	if (!rate)
+		rate = 1;
+
+	/* Get the number of parents for FSELXSPIn */
+	num_parents = clk_hw_get_num_parents(req->best_parent_hw);
+
+	for (clkt = divider->table; clkt->div; clkt++) {
+		parent_rate = clk_hw_round_rate(req->best_parent_hw, rate * clkt->div);
+		/* Skip if parent can't provide any valid rate */
+		if (!parent_rate)
+			continue;
+
+		/* Determine which DIVSELXSPIn divider (3 or 4) provides this parent_rate */
+		divselxspi = r9a09g077_cpg_fselxspi_get_divider(req->best_parent_hw, parent_rate,
+								num_parents);
+		if (!divselxspi)
+			continue;
+
+		/*
+		 * DIVSELXSPIx supports 800MHz and 600MHz operation.
+		 * When divselxspi is 4 (600MHz operation), only FSELXSPIn dividers of 8 and 16
+		 * are supported. Otherwise, when divselxspi is 3 (800MHz operation),
+		 * dividers of 6, 8, 16, 32, and 64 are supported. This check ensures that
+		 * FSELXSPIx is set correctly based on hardware limitations.
+		 */
+		if (divselxspi == 4 && (clkt->div != 8 && clkt->div != 16))
+			continue;
+
+		now = DIV_ROUND_UP_ULL(parent_rate, clkt->div);
+		if (abs(rate - now) < abs(rate - best)) {
+			div = clkt->div;
+			best = now;
+			req->best_parent_rate = parent_rate;
+		}
+	}
+
+	if (!div) {
+		req->best_parent_rate = clk_hw_round_rate(req->best_parent_hw, 1);
+		divselxspi = r9a09g077_cpg_fselxspi_get_divider(req->best_parent_hw,
+								req->best_parent_rate,
+								num_parents);
+		/* default to divider 3 which will result DIVSELXSPIn = 800 MHz */
+		if (!divselxspi)
+			divselxspi = 3;
+
+		/*
+		 * Use the maximum divider based on the parent clock rate:
+		 *  - 64 when DIVSELXSPIx is 800 MHz (divider = 3)
+		 *  - 16 when DIVSELXSPIx is 600 MHz (divider = 4)
+		 */
+		div = divselxspi == 3 ? 64 : 16;
+	}
+
+	req->rate = DIV_ROUND_UP_ULL(req->best_parent_rate, div);
+
+	return 0;
+}
+
+static struct clk * __init
+r9a09g077_cpg_fselxspi_div_clk_register(struct device *dev,
+					const struct cpg_core_clk *core,
+					void __iomem *addr,
+					struct cpg_mssr_pub *pub)
+{
+	static struct clk_ops *xspi_div_ops;
+	struct clk_init_data init = {};
+	const struct clk *parent;
+	const char *parent_name;
+	struct clk_divider *div;
+	struct clk_hw *hw;
+	int ret;
+
+	parent = pub->clks[core->parent];
+	if (IS_ERR(parent))
+		return ERR_CAST(parent);
+
+	div = devm_kzalloc(dev, sizeof(*div), GFP_KERNEL);
+	if (!div)
+		return ERR_PTR(-ENOMEM);
+
+	if (!xspi_div_ops) {
+		xspi_div_ops = devm_kzalloc(dev, sizeof(*xspi_div_ops), GFP_KERNEL);
+		if (!xspi_div_ops)
+			return  ERR_PTR(-ENOMEM);
+		memcpy(xspi_div_ops, &clk_divider_ops,
+		       sizeof(const struct clk_ops));
+		xspi_div_ops->determine_rate = r9a09g077_cpg_fselxspi_determine_rate;
+	}
+
+	parent_name = __clk_get_name(parent);
+	init.name = core->name;
+	init.ops = xspi_div_ops;
+	init.flags = CLK_SET_RATE_PARENT;
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+
+	div->reg = addr;
+	div->shift = GET_SHIFT(core->conf);
+	div->width = GET_WIDTH(core->conf);
+	div->flags = core->flag;
+	div->lock = &pub->rmw_lock;
+	div->hw.init = &init;
+	div->table = core->dtable;
+
+	hw = &div->hw;
+	ret = devm_clk_hw_register(dev, hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return hw->clk;
+}
+
 static struct clk * __init
 r9a09g077_cpg_clk_register(struct device *dev, const struct cpg_core_clk *core,
 			   const struct cpg_mssr_info *info,
@@ -298,6 +483,8 @@ r9a09g077_cpg_clk_register(struct device *dev, const struct cpg_core_clk *core,
 		return r9a09g077_cpg_div_clk_register(dev, core, addr, pub);
 	case CLK_TYPE_RZT2H_MUX:
 		return r9a09g077_cpg_mux_clk_register(dev, core, addr, pub);
+	case CLK_TYPE_RZT2H_FSELXSPI:
+		return r9a09g077_cpg_fselxspi_div_clk_register(dev, core, addr, pub);
 	default:
 		return ERR_PTR(-EINVAL);
 	}
diff --git a/drivers/clk/renesas/rzg2l-cpg.c b/drivers/clk/renesas/rzg2l-cpg.c
index 64d1ef6e4c94..0bcf64b152e0 100644
--- a/drivers/clk/renesas/rzg2l-cpg.c
+++ b/drivers/clk/renesas/rzg2l-cpg.c
@@ -122,8 +122,8 @@ struct div_hw_data {
 
 struct rzg2l_pll5_param {
 	u32 pl5_fracin;
+	u16 pl5_intin;
 	u8 pl5_refdiv;
-	u8 pl5_intin;
 	u8 pl5_postdiv1;
 	u8 pl5_postdiv2;
 	u8 pl5_spread;
@@ -572,8 +572,8 @@ rzg2l_cpg_get_foutpostdiv_rate(struct rzg2l_pll5_param *params,
 	foutvco_rate = div_u64(mul_u32_u32(EXTAL_FREQ_IN_MEGA_HZ * MEGA,
 					   (params->pl5_intin << 24) + params->pl5_fracin),
 			       params->pl5_refdiv) >> 24;
-	foutpostdiv_rate = DIV_ROUND_CLOSEST_ULL(foutvco_rate,
-						 params->pl5_postdiv1 * params->pl5_postdiv2);
+	foutpostdiv_rate = DIV_ROUND_CLOSEST(foutvco_rate,
+					     params->pl5_postdiv1 * params->pl5_postdiv2);
 
 	return foutpostdiv_rate;
 }
diff --git a/drivers/clk/thead/clk-th1520-ap.c b/drivers/clk/thead/clk-th1520-ap.c
index 71ad03a998e8..f47041c6c4a2 100644
--- a/drivers/clk/thead/clk-th1520-ap.c
+++ b/drivers/clk/thead/clk-th1520-ap.c
@@ -7,22 +7,38 @@
 
 #include <dt-bindings/clock/thead,th1520-clk-ap.h>
 #include <linux/bitfield.h>
+#include <linux/clk.h>
 #include <linux/clk-provider.h>
+#include <linux/delay.h>
 #include <linux/device.h>
+#include <linux/minmax.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 
+#define TH1520_PLL_STS		0x80
+
 #define TH1520_PLL_POSTDIV2	GENMASK(26, 24)
 #define TH1520_PLL_POSTDIV1	GENMASK(22, 20)
 #define TH1520_PLL_FBDIV	GENMASK(19, 8)
 #define TH1520_PLL_REFDIV	GENMASK(5, 0)
 #define TH1520_PLL_BYPASS	BIT(30)
 #define TH1520_PLL_VCO_RST	BIT(29)
+#define TH1520_PLL_DACPD	BIT(25)
 #define TH1520_PLL_DSMPD	BIT(24)
 #define TH1520_PLL_FRAC		GENMASK(23, 0)
 #define TH1520_PLL_FRAC_BITS    24
 
+/*
+ * All PLLs in TH1520 take 21250ns at maximum to lock, let's take its double
+ * for safety.
+ */
+#define TH1520_PLL_LOCK_TIMEOUT_US	44
+#define TH1520_PLL_STABLE_DELAY_US	30
+
+/* c910_bus_clk must be kept below 750MHz for stability */
+#define TH1520_C910_BUS_MAX_RATE	(750 * 1000 * 1000)
+
 struct ccu_internal {
 	u8	shift;
 	u8	width;
@@ -62,8 +78,19 @@ struct ccu_div {
 	struct ccu_common	common;
 };
 
+struct ccu_pll_cfg {
+	unsigned long		freq;
+	u32			fbdiv;
+	u32			frac;
+	u32			postdiv1;
+	u32			postdiv2;
+};
+
 struct ccu_pll {
 	struct ccu_common	common;
+	u32			lock_sts_mask;
+	int			cfgnum;
+	const struct ccu_pll_cfg *cfgs;
 };
 
 #define TH_CCU_ARG(_shift, _width)					\
@@ -79,17 +106,22 @@ struct ccu_pll {
 		.flags	= _flags,					\
 	}
 
-#define TH_CCU_MUX(_name, _parents, _shift, _width)			\
+#define TH_CCU_MUX_FLAGS(_name, _parents, _shift, _width, _flags,	\
+			 _mux_flags)					\
 	{								\
 		.mask		= GENMASK(_width - 1, 0),		\
 		.shift		= _shift,				\
+		.flags		= _mux_flags,				\
 		.hw.init	= CLK_HW_INIT_PARENTS_DATA(		\
 					_name,				\
 					_parents,			\
 					&clk_mux_ops,			\
-					0),				\
+					_flags),			\
 	}
 
+#define TH_CCU_MUX(_name, _parents, _shift, _width)			\
+	TH_CCU_MUX_FLAGS(_name, _parents, _shift, _width, 0, 0)
+
 #define CCU_GATE(_clkid, _struct, _name, _parent, _reg, _bit, _flags)	\
 	struct ccu_gate _struct = {					\
 		.clkid	= _clkid,					\
@@ -299,9 +331,21 @@ static void ccu_pll_disable(struct clk_hw *hw)
 static int ccu_pll_enable(struct clk_hw *hw)
 {
 	struct ccu_pll *pll = hw_to_ccu_pll(hw);
+	u32 reg;
+	int ret;
 
-	return regmap_clear_bits(pll->common.map, pll->common.cfg1,
-				 TH1520_PLL_VCO_RST);
+	regmap_clear_bits(pll->common.map, pll->common.cfg1,
+			  TH1520_PLL_VCO_RST);
+
+	ret = regmap_read_poll_timeout_atomic(pll->common.map, TH1520_PLL_STS,
+					      reg, reg & pll->lock_sts_mask,
+					      5, TH1520_PLL_LOCK_TIMEOUT_US);
+	if (ret)
+		return ret;
+
+	udelay(TH1520_PLL_STABLE_DELAY_US);
+
+	return 0;
 }
 
 static int ccu_pll_is_enabled(struct clk_hw *hw)
@@ -368,17 +412,168 @@ static unsigned long ccu_pll_recalc_rate(struct clk_hw *hw,
 	return rate;
 }
 
+static const struct ccu_pll_cfg *ccu_pll_lookup_best_cfg(struct ccu_pll *pll,
+							 unsigned long rate)
+{
+	unsigned long best_delta = ULONG_MAX;
+	const struct ccu_pll_cfg *best_cfg;
+	int i;
+
+	for (i = 0; i < pll->cfgnum; i++) {
+		const struct ccu_pll_cfg *cfg = &pll->cfgs[i];
+		unsigned long delta;
+
+		delta = abs_diff(cfg->freq, rate);
+		if (delta < best_delta) {
+			best_delta	= delta;
+			best_cfg	= cfg;
+		}
+	}
+
+	return best_cfg;
+}
+
+static int ccu_pll_determine_rate(struct clk_hw *hw,
+				  struct clk_rate_request *req)
+{
+	struct ccu_pll *pll = hw_to_ccu_pll(hw);
+
+	req->rate = ccu_pll_lookup_best_cfg(pll, req->rate)->freq;
+
+	return 0;
+}
+
+static int ccu_pll_set_rate(struct clk_hw *hw, unsigned long rate,
+			    unsigned long parent_rate)
+{
+	struct ccu_pll *pll = hw_to_ccu_pll(hw);
+	const struct ccu_pll_cfg *cfg;
+
+	cfg = ccu_pll_lookup_best_cfg(pll, rate);
+
+	ccu_pll_disable(hw);
+
+	regmap_write(pll->common.map, pll->common.cfg0,
+		     FIELD_PREP(TH1520_PLL_REFDIV,	1)		|
+		     FIELD_PREP(TH1520_PLL_FBDIV,	cfg->fbdiv)	|
+		     FIELD_PREP(TH1520_PLL_POSTDIV1,	cfg->postdiv1)	|
+		     FIELD_PREP(TH1520_PLL_POSTDIV2,	cfg->postdiv2));
+
+	regmap_update_bits(pll->common.map, pll->common.cfg1,
+			   TH1520_PLL_DACPD | TH1520_PLL_DSMPD |
+			   TH1520_PLL_FRAC,
+			   cfg->frac ? cfg->frac :
+				TH1520_PLL_DACPD | TH1520_PLL_DSMPD);
+
+	return ccu_pll_enable(hw);
+}
+
 static const struct clk_ops clk_pll_ops = {
 	.disable	= ccu_pll_disable,
 	.enable		= ccu_pll_enable,
 	.is_enabled	= ccu_pll_is_enabled,
 	.recalc_rate	= ccu_pll_recalc_rate,
+	.determine_rate	= ccu_pll_determine_rate,
+	.set_rate	= ccu_pll_set_rate,
+};
+
+/*
+ * c910_clk could be reparented glitchlessly for DVFS. There are two parents,
+ *  - c910_i0_clk, derived from cpu_pll0_clk or osc_24m.
+ *  - cpu_pll1_clk, which provides the exact same set of rates as cpu_pll0_clk.
+ *
+ * During rate setting, always forward the request to the unused parent, and
+ * then switch c910_clk to it to avoid glitch.
+ */
+static u8 c910_clk_get_parent(struct clk_hw *hw)
+{
+	return clk_mux_ops.get_parent(hw);
+}
+
+static int c910_clk_set_parent(struct clk_hw *hw, u8 index)
+{
+	return clk_mux_ops.set_parent(hw, index);
+}
+
+static unsigned long c910_clk_recalc_rate(struct clk_hw *hw,
+					  unsigned long parent_rate)
+{
+	return parent_rate;
+}
+
+static int c910_clk_determine_rate(struct clk_hw *hw,
+				   struct clk_rate_request *req)
+{
+	u8 alt_parent_index = !c910_clk_get_parent(hw);
+	struct clk_hw *alt_parent;
+
+	alt_parent = clk_hw_get_parent_by_index(hw, alt_parent_index);
+
+	req->rate		= clk_hw_round_rate(alt_parent, req->rate);
+	req->best_parent_hw	= alt_parent;
+	req->best_parent_rate	= req->rate;
+
+	return 0;
+}
+
+static int c910_clk_set_rate(struct clk_hw *hw, unsigned long rate,
+			     unsigned long parent_rate)
+{
+	return -EOPNOTSUPP;
+}
+
+static int c910_clk_set_rate_and_parent(struct clk_hw *hw, unsigned long rate,
+					unsigned long parent_rate, u8 index)
+{
+	struct clk_hw *parent = clk_hw_get_parent_by_index(hw, index);
+
+	clk_set_rate(parent->clk, parent_rate);
+
+	c910_clk_set_parent(hw, index);
+
+	return 0;
+}
+
+static const struct clk_ops c910_clk_ops = {
+	.get_parent		= c910_clk_get_parent,
+	.set_parent		= c910_clk_set_parent,
+	.recalc_rate		= c910_clk_recalc_rate,
+	.determine_rate		= c910_clk_determine_rate,
+	.set_rate		= c910_clk_set_rate,
+	.set_rate_and_parent	= c910_clk_set_rate_and_parent,
 };
 
 static const struct clk_parent_data osc_24m_clk[] = {
 	{ .index = 0 }
 };
 
+static const struct ccu_pll_cfg cpu_pll_cfgs[] = {
+	{ 125000000,	125,	0, 6, 4 },
+	{ 200000000,	125,	0, 5, 3 },
+	{ 300000000,	125,	0, 5, 2 },
+	{ 400000000,	100,	0, 3, 2 },
+	{ 500000000,	125,	0, 6, 1 },
+	{ 600000000,	125,	0, 5, 1 },
+	{ 702000000,	117,	0, 4, 1 },
+	{ 800000000,	100,	0, 3, 1 },
+	{ 900000000,	75,	0, 2, 1 },
+	{ 1000000000,	125,	0, 3, 1 },
+	{ 1104000000,	92,	0, 2, 1 },
+	{ 1200000000,	100,	0, 2, 1 },
+	{ 1296000000,	108,	0, 2, 1 },
+	{ 1404000000,	117,	0, 2, 1 },
+	{ 1500000000,	125,	0, 2, 1 },
+	{ 1608000000,	67,	0, 1, 1 },
+	{ 1704000000,	71,	0, 1, 1 },
+	{ 1800000000,	75,	0, 1, 1 },
+	{ 1896000000,	79,	0, 1, 1 },
+	{ 1992000000,	83,	0, 1, 1 },
+	{ 2112000000,	88,	0, 1, 1 },
+	{ 2208000000,	92,	0, 1, 1 },
+	{ 2304000000,	96,	0, 1, 1 },
+	{ 2400000000,	100,	0, 1, 1 },
+};
+
 static struct ccu_pll cpu_pll0_clk = {
 	.common		= {
 		.clkid		= CLK_CPU_PLL0,
@@ -389,6 +584,9 @@ static struct ccu_pll cpu_pll0_clk = {
 					      &clk_pll_ops,
 					      CLK_IS_CRITICAL),
 	},
+	.lock_sts_mask		= BIT(1),
+	.cfgnum			= ARRAY_SIZE(cpu_pll_cfgs),
+	.cfgs			= cpu_pll_cfgs,
 };
 
 static struct ccu_pll cpu_pll1_clk = {
@@ -401,6 +599,17 @@ static struct ccu_pll cpu_pll1_clk = {
 					      &clk_pll_ops,
 					      CLK_IS_CRITICAL),
 	},
+	.lock_sts_mask		= BIT(4),
+	.cfgnum			= ARRAY_SIZE(cpu_pll_cfgs),
+	.cfgs			= cpu_pll_cfgs,
+};
+
+static const struct ccu_pll_cfg gmac_pll_cfg = {
+	.freq		= 1000000000,
+	.fbdiv		= 125,
+	.frac		= 0,
+	.postdiv1	= 3,
+	.postdiv2	= 1,
 };
 
 static struct ccu_pll gmac_pll_clk = {
@@ -413,6 +622,9 @@ static struct ccu_pll gmac_pll_clk = {
 					      &clk_pll_ops,
 					      CLK_IS_CRITICAL),
 	},
+	.lock_sts_mask		= BIT(3),
+	.cfgnum			= 1,
+	.cfgs			= &gmac_pll_cfg,
 };
 
 static const struct clk_hw *gmac_pll_clk_parent[] = {
@@ -423,6 +635,14 @@ static const struct clk_parent_data gmac_pll_clk_pd[] = {
 	{ .hw = &gmac_pll_clk.common.hw }
 };
 
+static const struct ccu_pll_cfg video_pll_cfg = {
+	.freq		= 792000000,
+	.fbdiv		= 99,
+	.frac		= 0,
+	.postdiv1	= 3,
+	.postdiv2	= 1,
+};
+
 static struct ccu_pll video_pll_clk = {
 	.common		= {
 		.clkid		= CLK_VIDEO_PLL,
@@ -433,6 +653,9 @@ static struct ccu_pll video_pll_clk = {
 					      &clk_pll_ops,
 					      CLK_IS_CRITICAL),
 	},
+	.lock_sts_mask		= BIT(7),
+	.cfgnum			= 1,
+	.cfgs			= &video_pll_cfg,
 };
 
 static const struct clk_hw *video_pll_clk_parent[] = {
@@ -443,6 +666,14 @@ static const struct clk_parent_data video_pll_clk_pd[] = {
 	{ .hw = &video_pll_clk.common.hw }
 };
 
+static const struct ccu_pll_cfg dpu_pll_cfg = {
+	.freq		= 1188000000,
+	.fbdiv		= 99,
+	.frac		= 0,
+	.postdiv1	= 2,
+	.postdiv2	= 1,
+};
+
 static struct ccu_pll dpu0_pll_clk = {
 	.common		= {
 		.clkid		= CLK_DPU0_PLL,
@@ -453,6 +684,9 @@ static struct ccu_pll dpu0_pll_clk = {
 					      &clk_pll_ops,
 					      0),
 	},
+	.lock_sts_mask		= BIT(8),
+	.cfgnum			= 1,
+	.cfgs			= &dpu_pll_cfg,
 };
 
 static const struct clk_hw *dpu0_pll_clk_parent[] = {
@@ -469,12 +703,23 @@ static struct ccu_pll dpu1_pll_clk = {
 					      &clk_pll_ops,
 					      0),
 	},
+	.lock_sts_mask		= BIT(9),
+	.cfgnum			= 1,
+	.cfgs			= &dpu_pll_cfg,
 };
 
 static const struct clk_hw *dpu1_pll_clk_parent[] = {
 	&dpu1_pll_clk.common.hw
 };
 
+static const struct ccu_pll_cfg tee_pll_cfg = {
+	.freq		= 792000000,
+	.fbdiv		= 99,
+	.frac		= 0,
+	.postdiv1	= 3,
+	.postdiv2	= 1,
+};
+
 static struct ccu_pll tee_pll_clk = {
 	.common		= {
 		.clkid		= CLK_TEE_PLL,
@@ -485,6 +730,9 @@ static struct ccu_pll tee_pll_clk = {
 					      &clk_pll_ops,
 					      CLK_IS_CRITICAL),
 	},
+	.lock_sts_mask		= BIT(10),
+	.cfgnum			= 1,
+	.cfgs			= &tee_pll_cfg,
 };
 
 static const struct clk_parent_data c910_i0_parents[] = {
@@ -495,7 +743,8 @@ static const struct clk_parent_data c910_i0_parents[] = {
 static struct ccu_mux c910_i0_clk = {
 	.clkid	= CLK_C910_I0,
 	.reg	= 0x100,
-	.mux	= TH_CCU_MUX("c910-i0", c910_i0_parents, 1, 1),
+	.mux	= TH_CCU_MUX_FLAGS("c910-i0", c910_i0_parents, 1, 1,
+				   CLK_SET_RATE_PARENT, CLK_MUX_ROUND_CLOSEST),
 };
 
 static const struct clk_parent_data c910_parents[] = {
@@ -506,7 +755,28 @@ static const struct clk_parent_data c910_parents[] = {
 static struct ccu_mux c910_clk = {
 	.clkid	= CLK_C910,
 	.reg	= 0x100,
-	.mux	= TH_CCU_MUX("c910", c910_parents, 0, 1),
+	.mux	= {
+		.mask		= BIT(0),
+		.shift		= 0,
+		.hw.init	= CLK_HW_INIT_PARENTS_DATA("c910",
+							   c910_parents,
+							   &c910_clk_ops,
+							   CLK_SET_RATE_PARENT),
+	},
+};
+
+static struct ccu_div c910_bus_clk = {
+	.enable		= BIT(7),
+	.div_en		= BIT(11),
+	.div		= TH_CCU_DIV_FLAGS(8, 3, 0),
+	.common		= {
+		.clkid		= CLK_C910_BUS,
+		.cfg0		= 0x100,
+		.hw.init	= CLK_HW_INIT_HW("c910-bus",
+						 &c910_clk.mux.hw,
+						 &ccu_div_ops,
+						 CLK_IS_CRITICAL),
+	},
 };
 
 static const struct clk_parent_data ahb2_cpusys_parents[] = {
@@ -1021,6 +1291,7 @@ static struct ccu_common *th1520_pll_clks[] = {
 };
 
 static struct ccu_common *th1520_div_clks[] = {
+	&c910_bus_clk.common,
 	&ahb2_cpusys_hclk.common,
 	&apb3_cpusys_pclk.common,
 	&axi4_cpusys2_aclk.common,
@@ -1164,7 +1435,7 @@ static const struct th1520_plat_data th1520_ap_platdata = {
 	.th1520_mux_clks = th1520_mux_clks,
 	.th1520_gate_clks = th1520_gate_clks,
 
-	.nr_clks = CLK_UART_SCLK + 1,
+	.nr_clks = CLK_C910_BUS + 1,
 
 	.nr_pll_clks = ARRAY_SIZE(th1520_pll_clks),
 	.nr_div_clks = ARRAY_SIZE(th1520_div_clks),
@@ -1180,11 +1451,69 @@ static const struct th1520_plat_data th1520_vo_platdata = {
 	.nr_gate_clks = ARRAY_SIZE(th1520_vo_gate_clks),
 };
 
+/*
+ * Maintain clock rate of c910_bus_clk below TH1520_C910_BUS_MAX_RATE (750MHz)
+ * when its parent, c910_clk, changes the rate.
+ *
+ * Additionally, TRM is unclear about c910_bus_clk behavior with divisor set to
+ * 2, thus we should ensure the new divisor stays in (2, MAXDIVISOR).
+ */
+static unsigned long c910_bus_clk_divisor(struct ccu_div *cd,
+					  unsigned long parent_rate)
+{
+	return clamp(DIV_ROUND_UP(parent_rate, TH1520_C910_BUS_MAX_RATE),
+		     2U, 1U << cd->div.width);
+}
+
+static int c910_clk_notifier_cb(struct notifier_block *nb,
+				unsigned long action, void *data)
+{
+	struct clk_notifier_data *cnd = data;
+	unsigned long new_divisor, ref_rate;
+
+	if (action != PRE_RATE_CHANGE && action != POST_RATE_CHANGE)
+		return NOTIFY_DONE;
+
+	new_divisor	= c910_bus_clk_divisor(&c910_bus_clk, cnd->new_rate);
+
+	if (cnd->new_rate > cnd->old_rate) {
+		/*
+		 * Scaling up. Adjust c910_bus_clk divisor
+		 * - before c910_clk rate change to ensure the constraints
+		 *   aren't broken after scaling to higher rates,
+		 * - after c910_clk rate change to keep c910_bus_clk as high as
+		 *   possible
+		 */
+		ref_rate = action == PRE_RATE_CHANGE ?
+				cnd->old_rate : cnd->new_rate;
+		clk_set_rate(c910_bus_clk.common.hw.clk,
+			     ref_rate / new_divisor);
+	} else if (cnd->new_rate < cnd->old_rate &&
+		    action == POST_RATE_CHANGE) {
+		/*
+		 * Scaling down. Adjust c910_bus_clk divisor only after
+		 * c910_clk rate change to keep c910_bus_clk as high as
+		 * possible, Scaling down never breaks the constraints.
+		 */
+		clk_set_rate(c910_bus_clk.common.hw.clk,
+			     cnd->new_rate / new_divisor);
+	} else {
+		return NOTIFY_DONE;
+	}
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block c910_clk_notifier = {
+	.notifier_call	= c910_clk_notifier_cb,
+};
+
 static int th1520_clk_probe(struct platform_device *pdev)
 {
 	const struct th1520_plat_data *plat_data;
 	struct device *dev = &pdev->dev;
 	struct clk_hw_onecell_data *priv;
+	struct clk *notifier_clk;
 
 	struct regmap *map;
 	void __iomem *base;
@@ -1271,6 +1600,13 @@ static int th1520_clk_probe(struct platform_device *pdev)
 		ret = devm_clk_hw_register(dev, &emmc_sdio_ref_clk.hw);
 		if (ret)
 			return ret;
+
+		notifier_clk = devm_clk_hw_get_clk(dev, &c910_clk.mux.hw,
+						   "dvfs");
+		ret = devm_clk_notifier_register(dev, notifier_clk,
+						 &c910_clk_notifier);
+		if (ret)
+			return ret;
 	}
 
 	ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, priv);
diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c
index abb685a080a5..1501c7db9a8e 100644
--- a/drivers/clocksource/mips-gic-timer.c
+++ b/drivers/clocksource/mips-gic-timer.c
@@ -77,13 +77,6 @@ static irqreturn_t gic_compare_interrupt(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static struct irqaction gic_compare_irqaction = {
-	.handler = gic_compare_interrupt,
-	.percpu_dev_id = &gic_clockevent_device,
-	.flags = IRQF_PERCPU | IRQF_TIMER,
-	.name = "timer",
-};
-
 static void gic_clockevent_cpu_init(unsigned int cpu,
 				    struct clock_event_device *cd)
 {
@@ -152,7 +145,8 @@ static int gic_clockevent_init(void)
 	if (!gic_frequency)
 		return -ENXIO;
 
-	ret = setup_percpu_irq(gic_timer_irq, &gic_compare_irqaction);
+	ret = request_percpu_irq(gic_timer_irq, gic_compare_interrupt,
+				 "timer", &gic_clockevent_device);
 	if (ret < 0) {
 		pr_err("IRQ %d setup failed (%d)\n", gic_timer_irq, ret);
 		return ret;
diff --git a/drivers/comedi/drivers/jr3_pci.c b/drivers/comedi/drivers/jr3_pci.c
index 61792d940a3d..51287cbc3e48 100644
--- a/drivers/comedi/drivers/jr3_pci.c
+++ b/drivers/comedi/drivers/jr3_pci.c
@@ -32,6 +32,7 @@
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/ctype.h>
+#include <linux/hex.h>
 #include <linux/jiffies.h>
 #include <linux/slab.h>
 #include <linux/timer.h>
diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c
index a1d11ecd1ac8..b06a43143d23 100644
--- a/drivers/cpufreq/cpufreq-dt-platdev.c
+++ b/drivers/cpufreq/cpufreq-dt-platdev.c
@@ -219,11 +219,12 @@ static bool __init cpu0_node_has_opp_v2_prop(void)
 
 static int __init cpufreq_dt_platdev_init(void)
 {
-	const void *data;
+	const void *data = NULL;
 
-	data = of_machine_get_match_data(allowlist);
-	if (data)
+	if (of_machine_device_match(allowlist)) {
+		data = of_machine_get_match_data(allowlist);
 		goto create_pdev;
+	}
 
 	if (cpu0_node_has_opp_v2_prop() && !of_machine_device_match(blocklist))
 		goto create_pdev;
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 4472bb1ec83c..50dde2980f1b 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -2803,7 +2803,7 @@ static int cpufreq_boost_trigger_state(int state)
 {
 	struct cpufreq_policy *policy;
 	unsigned long flags;
-	int ret = 0;
+	int ret = -EOPNOTSUPP;
 
 	/*
 	 * Don't compare 'cpufreq_driver->boost_enabled' with 'state' here to
@@ -2820,16 +2820,15 @@ static int cpufreq_boost_trigger_state(int state)
 			continue;
 
 		ret = policy_set_boost(policy, state);
-		if (ret)
-			goto err_reset_state;
+		if (unlikely(ret))
+			break;
 	}
-	cpus_read_unlock();
 
-	return 0;
-
-err_reset_state:
 	cpus_read_unlock();
 
+	if (likely(!ret))
+		return 0;
+
 	write_lock_irqsave(&cpufreq_driver_lock, flags);
 	cpufreq_driver->boost_enabled = !state;
 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig
index b46eb8a552d7..fdd823e446cc 100644
--- a/drivers/dma-buf/Kconfig
+++ b/drivers/dma-buf/Kconfig
@@ -55,7 +55,7 @@ config DMABUF_MOVE_NOTIFY
 config DMABUF_DEBUG
 	bool "DMA-BUF debug checks"
 	depends on DMA_SHARED_BUFFER
-	default y if DMA_API_DEBUG
+	default y if DEBUG
 	help
 	  This option enables additional checks for DMA-BUF importers and
 	  exporters. Specifically it validates that importers do not peek at the
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index edaa9e4ee4ae..a4d8f2ff94e4 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -35,6 +35,34 @@
 
 #include "dma-buf-sysfs-stats.h"
 
+#define CREATE_TRACE_POINTS
+#include <trace/events/dma_buf.h>
+
+/*
+ * dmabuf->name must be accessed with holding dmabuf->name_lock.
+ * we need to take the lock around the tracepoint call itself where
+ * it is called in the code.
+ *
+ * Note: FUNC##_enabled() is a static branch that will only
+ *       be set when the trace event is enabled.
+ */
+#define DMA_BUF_TRACE(FUNC, ...)					\
+	do {								\
+		if (FUNC##_enabled()) {					\
+			guard(spinlock)(&dmabuf->name_lock);		\
+			FUNC(__VA_ARGS__);				\
+		} else if (IS_ENABLED(CONFIG_LOCKDEP)) {		\
+			/* Expose this lock when lockdep is enabled */	\
+			guard(spinlock)(&dmabuf->name_lock);		\
+		}							\
+	} while (0)
+
+/* Wrapper to hide the sg_table page link from the importer */
+struct dma_buf_sg_table_wrapper {
+	struct sg_table *original;
+	struct sg_table wrapper;
+};
+
 static inline int is_dma_buf_file(struct file *);
 
 static DEFINE_MUTEX(dmabuf_list_mutex);
@@ -220,6 +248,8 @@ static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma)
 	    dmabuf->size >> PAGE_SHIFT)
 		return -EINVAL;
 
+	DMA_BUF_TRACE(trace_dma_buf_mmap_internal, dmabuf);
+
 	return dmabuf->ops->mmap(dmabuf, vma);
 }
 
@@ -745,6 +775,8 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
 
 	__dma_buf_list_add(dmabuf);
 
+	DMA_BUF_TRACE(trace_dma_buf_export, dmabuf);
+
 	return dmabuf;
 
 err_dmabuf:
@@ -768,10 +800,16 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_export, "DMA_BUF");
  */
 int dma_buf_fd(struct dma_buf *dmabuf, int flags)
 {
+	int fd;
+
 	if (!dmabuf || !dmabuf->file)
 		return -EINVAL;
 
-	return FD_ADD(flags, dmabuf->file);
+	fd = FD_ADD(flags, dmabuf->file);
+	if (fd >= 0)
+		DMA_BUF_TRACE(trace_dma_buf_fd, dmabuf, fd);
+
+	return fd;
 }
 EXPORT_SYMBOL_NS_GPL(dma_buf_fd, "DMA_BUF");
 
@@ -786,6 +824,7 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_fd, "DMA_BUF");
 struct dma_buf *dma_buf_get(int fd)
 {
 	struct file *file;
+	struct dma_buf *dmabuf;
 
 	file = fget(fd);
 
@@ -797,7 +836,11 @@ struct dma_buf *dma_buf_get(int fd)
 		return ERR_PTR(-EINVAL);
 	}
 
-	return file->private_data;
+	dmabuf = file->private_data;
+
+	DMA_BUF_TRACE(trace_dma_buf_get, dmabuf, fd);
+
+	return dmabuf;
 }
 EXPORT_SYMBOL_NS_GPL(dma_buf_get, "DMA_BUF");
 
@@ -817,24 +860,64 @@ void dma_buf_put(struct dma_buf *dmabuf)
 		return;
 
 	fput(dmabuf->file);
+
+	DMA_BUF_TRACE(trace_dma_buf_put, dmabuf);
 }
 EXPORT_SYMBOL_NS_GPL(dma_buf_put, "DMA_BUF");
 
-static void mangle_sg_table(struct sg_table *sg_table)
+static int dma_buf_wrap_sg_table(struct sg_table **sg_table)
 {
-#ifdef CONFIG_DMABUF_DEBUG
-	int i;
-	struct scatterlist *sg;
-
-	/* To catch abuse of the underlying struct page by importers mix
-	 * up the bits, but take care to preserve the low SG_ bits to
-	 * not corrupt the sgt. The mixing is undone on unmap
-	 * before passing the sgt back to the exporter.
+	struct scatterlist *to_sg, *from_sg;
+	struct sg_table *from = *sg_table;
+	struct dma_buf_sg_table_wrapper *to;
+	int i, ret;
+
+	if (!IS_ENABLED(CONFIG_DMABUF_DEBUG))
+		return 0;
+
+	/*
+	 * To catch abuse of the underlying struct page by importers copy the
+	 * sg_table without copying the page_link and give only the copy back to
+	 * the importer.
 	 */
-	for_each_sgtable_sg(sg_table, sg, i)
-		sg->page_link ^= ~0xffUL;
-#endif
+	to = kzalloc(sizeof(*to), GFP_KERNEL);
+	if (!to)
+		return -ENOMEM;
+
+	ret = sg_alloc_table(&to->wrapper, from->nents, GFP_KERNEL);
+	if (ret)
+		goto free_to;
+
+	to_sg = to->wrapper.sgl;
+	for_each_sgtable_dma_sg(from, from_sg, i) {
+		to_sg->offset = 0;
+		to_sg->length = 0;
+		sg_assign_page(to_sg, NULL);
+		sg_dma_address(to_sg) = sg_dma_address(from_sg);
+		sg_dma_len(to_sg) = sg_dma_len(from_sg);
+		to_sg = sg_next(to_sg);
+	}
+
+	to->original = from;
+	*sg_table = &to->wrapper;
+	return 0;
 
+free_to:
+	kfree(to);
+	return ret;
+}
+
+static void dma_buf_unwrap_sg_table(struct sg_table **sg_table)
+{
+	struct dma_buf_sg_table_wrapper *copy;
+
+	if (!IS_ENABLED(CONFIG_DMABUF_DEBUG))
+		return;
+
+	copy = container_of(*sg_table, typeof(*copy), wrapper);
+	*sg_table = copy->original;
+	sg_free_table(&copy->wrapper);
+	kfree(copy);
 }
 
 static inline bool
@@ -971,6 +1054,9 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
 	list_add(&attach->node, &dmabuf->attachments);
 	dma_resv_unlock(dmabuf->resv);
 
+	DMA_BUF_TRACE(trace_dma_buf_dynamic_attach, dmabuf, attach,
+		dma_buf_attachment_is_dynamic(attach), dev);
+
 	return attach;
 
 err_attach:
@@ -1015,6 +1101,9 @@ void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach)
 	if (dmabuf->ops->detach)
 		dmabuf->ops->detach(dmabuf, attach);
 
+	DMA_BUF_TRACE(trace_dma_buf_detach, dmabuf, attach,
+		dma_buf_attachment_is_dynamic(attach), attach->dev);
+
 	kfree(attach);
 }
 EXPORT_SYMBOL_NS_GPL(dma_buf_detach, "DMA_BUF");
@@ -1131,10 +1220,11 @@ struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach,
 		if (ret < 0)
 			goto error_unmap;
 	}
-	mangle_sg_table(sg_table);
+	ret = dma_buf_wrap_sg_table(&sg_table);
+	if (ret)
+		goto error_unmap;
 
-#ifdef CONFIG_DMA_API_DEBUG
-	{
+	if (IS_ENABLED(CONFIG_DMA_API_DEBUG)) {
 		struct scatterlist *sg;
 		u64 addr;
 		int len;
@@ -1146,10 +1236,10 @@ struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach,
 			if (!PAGE_ALIGNED(addr) || !PAGE_ALIGNED(len)) {
 				pr_debug("%s: addr %llx or len %x is not page aligned!\n",
 					 __func__, addr, len);
+				break;
 			}
 		}
 	}
-#endif /* CONFIG_DMA_API_DEBUG */
 	return sg_table;
 
 error_unmap:
@@ -1213,7 +1303,7 @@ void dma_buf_unmap_attachment(struct dma_buf_attachment *attach,
 
 	dma_resv_assert_held(attach->dmabuf->resv);
 
-	mangle_sg_table(sg_table);
+	dma_buf_unwrap_sg_table(&sg_table);
 	attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction);
 
 	if (dma_buf_pin_on_map(attach))
@@ -1480,6 +1570,8 @@ int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma,
 	vma_set_file(vma, dmabuf->file);
 	vma->vm_pgoff = pgoff;
 
+	DMA_BUF_TRACE(trace_dma_buf_mmap, dmabuf);
+
 	return dmabuf->ops->mmap(dmabuf, vma);
 }
 EXPORT_SYMBOL_NS_GPL(dma_buf_mmap, "DMA_BUF");
diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c
index b4f5c8635276..21c5c30b4f34 100644
--- a/drivers/dma-buf/dma-fence.c
+++ b/drivers/dma-buf/dma-fence.c
@@ -358,11 +358,8 @@ void __dma_fence_might_wait(void)
  *
  * Unlike dma_fence_signal_timestamp(), this function must be called with
  * &dma_fence.lock held.
- *
- * Returns 0 on success and a negative error value when @fence has been
- * signalled already.
  */
-int dma_fence_signal_timestamp_locked(struct dma_fence *fence,
+void dma_fence_signal_timestamp_locked(struct dma_fence *fence,
 				      ktime_t timestamp)
 {
 	struct dma_fence_cb *cur, *tmp;
@@ -372,7 +369,7 @@ int dma_fence_signal_timestamp_locked(struct dma_fence *fence,
 
 	if (unlikely(test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
 				      &fence->flags)))
-		return -EINVAL;
+		return;
 
 	/* Stash the cb_list before replacing it with the timestamp */
 	list_replace(&fence->cb_list, &cb_list);
@@ -385,8 +382,6 @@ int dma_fence_signal_timestamp_locked(struct dma_fence *fence,
 		INIT_LIST_HEAD(&cur->node);
 		cur->func(fence, cur);
 	}
-
-	return 0;
 }
 EXPORT_SYMBOL(dma_fence_signal_timestamp_locked);
 
@@ -401,23 +396,17 @@ EXPORT_SYMBOL(dma_fence_signal_timestamp_locked);
  * can only go from the unsignaled to the signaled state and not back, it will
  * only be effective the first time. Set the timestamp provided as the fence
  * signal timestamp.
- *
- * Returns 0 on success and a negative error value when @fence has been
- * signalled already.
  */
-int dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp)
+void dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp)
 {
 	unsigned long flags;
-	int ret;
 
 	if (WARN_ON(!fence))
-		return -EINVAL;
+		return;
 
 	spin_lock_irqsave(fence->lock, flags);
-	ret = dma_fence_signal_timestamp_locked(fence, timestamp);
+	dma_fence_signal_timestamp_locked(fence, timestamp);
 	spin_unlock_irqrestore(fence->lock, flags);
-
-	return ret;
 }
 EXPORT_SYMBOL(dma_fence_signal_timestamp);
 
@@ -433,16 +422,57 @@ EXPORT_SYMBOL(dma_fence_signal_timestamp);
  *
  * Unlike dma_fence_signal(), this function must be called with &dma_fence.lock
  * held.
- *
- * Returns 0 on success and a negative error value when @fence has been
- * signalled already.
  */
-int dma_fence_signal_locked(struct dma_fence *fence)
+void dma_fence_signal_locked(struct dma_fence *fence)
 {
-	return dma_fence_signal_timestamp_locked(fence, ktime_get());
+	dma_fence_signal_timestamp_locked(fence, ktime_get());
 }
 EXPORT_SYMBOL(dma_fence_signal_locked);
 
+/**
+ * dma_fence_check_and_signal_locked - signal the fence if it's not yet signaled
+ * @fence: the fence to check and signal
+ *
+ * Checks whether a fence was signaled and signals it if it was not yet signaled.
+ *
+ * Unlike dma_fence_check_and_signal(), this function must be called with
+ * &struct dma_fence.lock being held.
+ *
+ * Return: true if fence has been signaled already, false otherwise.
+ */
+bool dma_fence_check_and_signal_locked(struct dma_fence *fence)
+{
+	bool ret;
+
+	ret = dma_fence_test_signaled_flag(fence);
+	dma_fence_signal_locked(fence);
+
+	return ret;
+}
+EXPORT_SYMBOL(dma_fence_check_and_signal_locked);
+
+/**
+ * dma_fence_check_and_signal - signal the fence if it's not yet signaled
+ * @fence: the fence to check and signal
+ *
+ * Checks whether a fence was signaled and signals it if it was not yet signaled.
+ * All this is done in a race-free manner.
+ *
+ * Return: true if fence has been signaled already, false otherwise.
+ */
+bool dma_fence_check_and_signal(struct dma_fence *fence)
+{
+	unsigned long flags;
+	bool ret;
+
+	spin_lock_irqsave(fence->lock, flags);
+	ret = dma_fence_check_and_signal_locked(fence);
+	spin_unlock_irqrestore(fence->lock, flags);
+
+	return ret;
+}
+EXPORT_SYMBOL(dma_fence_check_and_signal);
+
 /**
  * dma_fence_signal - signal completion of a fence
  * @fence: the fence to signal
@@ -452,28 +482,22 @@ EXPORT_SYMBOL(dma_fence_signal_locked);
  * dma_fence_add_callback(). Can be called multiple times, but since a fence
  * can only go from the unsignaled to the signaled state and not back, it will
  * only be effective the first time.
- *
- * Returns 0 on success and a negative error value when @fence has been
- * signalled already.
  */
-int dma_fence_signal(struct dma_fence *fence)
+void dma_fence_signal(struct dma_fence *fence)
 {
 	unsigned long flags;
-	int ret;
 	bool tmp;
 
 	if (WARN_ON(!fence))
-		return -EINVAL;
+		return;
 
 	tmp = dma_fence_begin_signalling();
 
 	spin_lock_irqsave(fence->lock, flags);
-	ret = dma_fence_signal_timestamp_locked(fence, ktime_get());
+	dma_fence_signal_timestamp_locked(fence, ktime_get());
 	spin_unlock_irqrestore(fence->lock, flags);
 
 	dma_fence_end_signalling(tmp);
-
-	return ret;
 }
 EXPORT_SYMBOL(dma_fence_signal);
 
@@ -543,7 +567,7 @@ void dma_fence_release(struct kref *kref)
 	trace_dma_fence_destroy(fence);
 
 	if (!list_empty(&fence->cb_list) &&
-	    !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
+	    !dma_fence_test_signaled_flag(fence)) {
 		const char __rcu *timeline;
 		const char __rcu *driver;
 		unsigned long flags;
@@ -600,7 +624,7 @@ static bool __dma_fence_enable_signaling(struct dma_fence *fence)
 	was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
 				   &fence->flags);
 
-	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+	if (dma_fence_test_signaled_flag(fence))
 		return false;
 
 	if (!was_set && fence->ops->enable_signaling) {
@@ -664,7 +688,7 @@ int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,
 	if (WARN_ON(!fence || !func))
 		return -EINVAL;
 
-	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
+	if (dma_fence_test_signaled_flag(fence)) {
 		INIT_LIST_HEAD(&cb->node);
 		return -ENOENT;
 	}
@@ -781,7 +805,7 @@ dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
 
 	spin_lock_irqsave(fence->lock, flags);
 
-	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+	if (dma_fence_test_signaled_flag(fence))
 		goto out;
 
 	if (intr && signal_pending(current)) {
@@ -798,7 +822,7 @@ dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
 	cb.task = current;
 	list_add(&cb.base.node, &fence->cb_list);
 
-	while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {
+	while (!dma_fence_test_signaled_flag(fence) && ret > 0) {
 		if (intr)
 			__set_current_state(TASK_INTERRUPTIBLE);
 		else
@@ -830,7 +854,7 @@ dma_fence_test_signaled_any(struct dma_fence **fences, uint32_t count,
 
 	for (i = 0; i < count; ++i) {
 		struct dma_fence *fence = fences[i];
-		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
+		if (dma_fence_test_signaled_flag(fence)) {
 			if (idx)
 				*idx = i;
 			return true;
@@ -1108,7 +1132,7 @@ const char __rcu *dma_fence_driver_name(struct dma_fence *fence)
 	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
 			 "RCU protection is required for safe access to returned string");
 
-	if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+	if (!dma_fence_test_signaled_flag(fence))
 		return fence->ops->get_driver_name(fence);
 	else
 		return "detached-driver";
@@ -1140,7 +1164,7 @@ const char __rcu *dma_fence_timeline_name(struct dma_fence *fence)
 	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
 			 "RCU protection is required for safe access to returned string");
 
-	if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+	if (!dma_fence_test_signaled_flag(fence))
 		return fence->ops->get_timeline_name(fence);
 	else
 		return "signaled-timeline";
diff --git a/drivers/dma-buf/st-dma-fence.c b/drivers/dma-buf/st-dma-fence.c
index 27a36045410b..73ed6fd48a13 100644
--- a/drivers/dma-buf/st-dma-fence.c
+++ b/drivers/dma-buf/st-dma-fence.c
@@ -33,50 +33,9 @@ static void mock_fence_release(struct dma_fence *f)
 	kmem_cache_free(slab_fences, to_mock_fence(f));
 }
 
-struct wait_cb {
-	struct dma_fence_cb cb;
-	struct task_struct *task;
-};
-
-static void mock_wakeup(struct dma_fence *f, struct dma_fence_cb *cb)
-{
-	wake_up_process(container_of(cb, struct wait_cb, cb)->task);
-}
-
-static long mock_wait(struct dma_fence *f, bool intr, long timeout)
-{
-	const int state = intr ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
-	struct wait_cb cb = { .task = current };
-
-	if (dma_fence_add_callback(f, &cb.cb, mock_wakeup))
-		return timeout;
-
-	while (timeout) {
-		set_current_state(state);
-
-		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->flags))
-			break;
-
-		if (signal_pending_state(state, current))
-			break;
-
-		timeout = schedule_timeout(timeout);
-	}
-	__set_current_state(TASK_RUNNING);
-
-	if (!dma_fence_remove_callback(f, &cb.cb))
-		return timeout;
-
-	if (signal_pending_state(state, current))
-		return -ERESTARTSYS;
-
-	return -ETIME;
-}
-
 static const struct dma_fence_ops mock_ops = {
 	.get_driver_name = mock_name,
 	.get_timeline_name = mock_name,
-	.wait = mock_wait,
 	.release = mock_fence_release,
 };
 
@@ -126,7 +85,7 @@ static int test_signaling(void *arg)
 		goto err_free;
 	}
 
-	if (dma_fence_signal(f)) {
+	if (dma_fence_check_and_signal(f)) {
 		pr_err("Fence reported being already signaled\n");
 		goto err_free;
 	}
@@ -136,7 +95,7 @@ static int test_signaling(void *arg)
 		goto err_free;
 	}
 
-	if (!dma_fence_signal(f)) {
+	if (!dma_fence_test_signaled_flag(f)) {
 		pr_err("Fence reported not being already signaled\n");
 		goto err_free;
 	}
@@ -349,14 +308,14 @@ static int test_wait(void *arg)
 
 	dma_fence_enable_sw_signaling(f);
 
-	if (dma_fence_wait_timeout(f, false, 0) != -ETIME) {
+	if (dma_fence_wait_timeout(f, false, 0) != 0) {
 		pr_err("Wait reported complete before being signaled\n");
 		goto err_free;
 	}
 
 	dma_fence_signal(f);
 
-	if (dma_fence_wait_timeout(f, false, 0) != 0) {
+	if (dma_fence_wait_timeout(f, false, 0) != 1) {
 		pr_err("Wait reported incomplete after being signaled\n");
 		goto err_free;
 	}
@@ -393,16 +352,16 @@ static int test_wait_timeout(void *arg)
 
 	dma_fence_enable_sw_signaling(wt.f);
 
-	if (dma_fence_wait_timeout(wt.f, false, 1) != -ETIME) {
+	if (dma_fence_wait_timeout(wt.f, false, 1) != 0) {
 		pr_err("Wait reported complete before being signaled\n");
 		goto err_free;
 	}
 
 	mod_timer(&wt.timer, jiffies + 1);
 
-	if (dma_fence_wait_timeout(wt.f, false, 2) == -ETIME) {
+	if (dma_fence_wait_timeout(wt.f, false, HZ) == 0) {
 		if (timer_pending(&wt.timer)) {
-			pr_notice("Timer did not fire within the jiffy!\n");
+			pr_notice("Timer did not fire within one HZ!\n");
 			err = 0; /* not our fault! */
 		} else {
 			pr_err("Wait reported incomplete after timeout\n");
diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c
index 38cdbca59485..3bfb3b312027 100644
--- a/drivers/dma/amba-pl08x.c
+++ b/drivers/dma/amba-pl08x.c
@@ -1010,7 +1010,7 @@ static inline u32 pl08x_lli_control_bits(struct pl08x_driver_data *pl08x,
 	/*
 	 * Remove all src, dst and transfer size bits, then set the
 	 * width and size according to the parameters. The bit offsets
-	 * are different in the FTDMAC020 so we need to accound for this.
+	 * are different in the FTDMAC020 so we need to account for this.
 	 */
 	if (pl08x->vd->ftdmac020) {
 		retbits &= ~FTDMAC020_LLI_DST_WIDTH_MSK;
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c
index 7d226453961f..22bb604a3f97 100644
--- a/drivers/dma/at_hdmac.c
+++ b/drivers/dma/at_hdmac.c
@@ -1765,6 +1765,7 @@ static int atc_alloc_chan_resources(struct dma_chan *chan)
 static void atc_free_chan_resources(struct dma_chan *chan)
 {
 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
+	struct at_dma_slave	*atslave;
 
 	BUG_ON(atc_chan_is_enabled(atchan));
 
@@ -1774,8 +1775,12 @@ static void atc_free_chan_resources(struct dma_chan *chan)
 	/*
 	 * Free atslave allocated in at_dma_xlate()
 	 */
-	kfree(chan->private);
-	chan->private = NULL;
+	atslave = chan->private;
+	if (atslave) {
+		put_device(atslave->dma_dev);
+		kfree(atslave);
+		chan->private = NULL;
+	}
 
 	dev_vdbg(chan2dev(chan), "free_chan_resources: done\n");
 }
diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c
index 3fbc74710a13..ada96d490847 100644
--- a/drivers/dma/at_xdmac.c
+++ b/drivers/dma/at_xdmac.c
@@ -379,7 +379,6 @@ static void at_xdmac_runtime_suspend_descriptors(struct at_xdmac_chan *atchan)
 		if (!desc->active_xfer)
 			continue;
 
-		pm_runtime_mark_last_busy(atxdmac->dev);
 		pm_runtime_put_autosuspend(atxdmac->dev);
 	}
 }
@@ -413,7 +412,6 @@ static bool at_xdmac_chan_is_enabled(struct at_xdmac_chan *atchan)
 
 	ret = !!(at_xdmac_chan_read(atchan, AT_XDMAC_GS) & atchan->mask);
 
-	pm_runtime_mark_last_busy(atxdmac->dev);
 	pm_runtime_put_autosuspend(atxdmac->dev);
 
 	return ret;
@@ -446,7 +444,6 @@ static void at_xdmac_off(struct at_xdmac *atxdmac, bool suspend_descriptors)
 		}
 	}
 
-	pm_runtime_mark_last_busy(atxdmac->dev);
 	pm_runtime_put_autosuspend(atxdmac->dev);
 }
 
@@ -1676,7 +1673,6 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 
 spin_unlock:
 	spin_unlock_irqrestore(&atchan->lock, flags);
-	pm_runtime_mark_last_busy(atxdmac->dev);
 	pm_runtime_put_autosuspend(atxdmac->dev);
 	return ret;
 }
@@ -1758,7 +1754,6 @@ static void at_xdmac_handle_error(struct at_xdmac_chan *atchan)
 		__func__, &bad_desc->lld.mbr_sa, &bad_desc->lld.mbr_da,
 		bad_desc->lld.mbr_ubc);
 
-	pm_runtime_mark_last_busy(atxdmac->dev);
 	pm_runtime_put_autosuspend(atxdmac->dev);
 
 	/* Then continue with usual descriptor management */
@@ -1822,7 +1817,6 @@ static void at_xdmac_tasklet(struct tasklet_struct *t)
 	 * Decrement runtime PM ref counter incremented in
 	 * at_xdmac_start_xfer().
 	 */
-	pm_runtime_mark_last_busy(atxdmac->dev);
 	pm_runtime_put_autosuspend(atxdmac->dev);
 }
 
@@ -1954,7 +1948,6 @@ static int at_xdmac_device_pause(struct dma_chan *chan)
 
 	spin_unlock_irqrestore(&atchan->lock, flags);
 
-	pm_runtime_mark_last_busy(atxdmac->dev);
 	pm_runtime_put_autosuspend(atxdmac->dev);
 
 	return 0;
@@ -1998,7 +1991,6 @@ static int at_xdmac_device_resume(struct dma_chan *chan)
 
 unlock:
 	spin_unlock_irqrestore(&atchan->lock, flags);
-	pm_runtime_mark_last_busy(atxdmac->dev);
 	pm_runtime_put_autosuspend(atxdmac->dev);
 
 	return ret;
@@ -2041,7 +2033,6 @@ static int at_xdmac_device_terminate_all(struct dma_chan *chan)
 	clear_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status);
 	spin_unlock_irqrestore(&atchan->lock, flags);
 
-	pm_runtime_mark_last_busy(atxdmac->dev);
 	pm_runtime_put_autosuspend(atxdmac->dev);
 
 	return 0;
@@ -2235,7 +2226,6 @@ static int __maybe_unused atmel_xdmac_resume(struct device *dev)
 		}
 	}
 
-	pm_runtime_mark_last_busy(atxdmac->dev);
 	pm_runtime_put_autosuspend(atxdmac->dev);
 
 	return 0;
@@ -2412,7 +2402,6 @@ static int at_xdmac_probe(struct platform_device *pdev)
 
 	at_xdmac_axi_config(pdev);
 
-	pm_runtime_mark_last_busy(&pdev->dev);
 	pm_runtime_put_autosuspend(&pdev->dev);
 
 	return 0;
diff --git a/drivers/dma/bcm-sba-raid.c b/drivers/dma/bcm-sba-raid.c
index 7f0e76439ce5..ed037fa883f6 100644
--- a/drivers/dma/bcm-sba-raid.c
+++ b/drivers/dma/bcm-sba-raid.c
@@ -1699,7 +1699,7 @@ static int sba_probe(struct platform_device *pdev)
 	/* Prealloc channel resource */
 	ret = sba_prealloc_channel_resources(sba);
 	if (ret)
-		goto fail_free_mchan;
+		goto fail_put_mbox;
 
 	/* Check availability of debugfs */
 	if (!debugfs_initialized())
@@ -1729,6 +1729,8 @@ static int sba_probe(struct platform_device *pdev)
 fail_free_resources:
 	debugfs_remove_recursive(sba->root);
 	sba_freeup_channel_resources(sba);
+fail_put_mbox:
+	put_device(sba->mbox_dev);
 fail_free_mchan:
 	mbox_free_channel(sba->mchan);
 	return ret;
@@ -1744,6 +1746,8 @@ static void sba_remove(struct platform_device *pdev)
 
 	sba_freeup_channel_resources(sba);
 
+	put_device(sba->mbox_dev);
+
 	mbox_free_channel(sba->mchan);
 }
 
diff --git a/drivers/dma/cv1800b-dmamux.c b/drivers/dma/cv1800b-dmamux.c
index e900d6595617..f7a952fcbc7d 100644
--- a/drivers/dma/cv1800b-dmamux.c
+++ b/drivers/dma/cv1800b-dmamux.c
@@ -102,11 +102,11 @@ static void *cv1800_dmamux_route_allocate(struct of_phandle_args *dma_spec,
 	struct llist_node *node;
 	unsigned long flags;
 	unsigned int chid, devid, cpuid;
-	int ret;
+	int ret = -EINVAL;
 
 	if (dma_spec->args_count != DMAMUX_NCELLS) {
 		dev_err(&pdev->dev, "invalid number of dma mux args\n");
-		return ERR_PTR(-EINVAL);
+		goto err_put_pdev;
 	}
 
 	devid = dma_spec->args[0];
@@ -115,18 +115,18 @@ static void *cv1800_dmamux_route_allocate(struct of_phandle_args *dma_spec,
 
 	if (devid > MAX_DMA_MAPPING_ID) {
 		dev_err(&pdev->dev, "invalid device id: %u\n", devid);
-		return ERR_PTR(-EINVAL);
+		goto err_put_pdev;
 	}
 
 	if (cpuid > MAX_DMA_CPU_ID) {
 		dev_err(&pdev->dev, "invalid cpu id: %u\n", cpuid);
-		return ERR_PTR(-EINVAL);
+		goto err_put_pdev;
 	}
 
 	dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0);
 	if (!dma_spec->np) {
 		dev_err(&pdev->dev, "can't get dma master\n");
-		return ERR_PTR(-EINVAL);
+		goto err_put_pdev;
 	}
 
 	spin_lock_irqsave(&dmamux->lock, flags);
@@ -136,8 +136,6 @@ static void *cv1800_dmamux_route_allocate(struct of_phandle_args *dma_spec,
 			if (map->peripheral == devid && map->cpu == cpuid)
 				goto found;
 		}
-
-		ret = -EINVAL;
 		goto failed;
 	} else {
 		node = llist_del_first(&dmamux->free_maps);
@@ -171,12 +169,17 @@ static void *cv1800_dmamux_route_allocate(struct of_phandle_args *dma_spec,
 	dev_dbg(&pdev->dev, "register channel %u for req %u (cpu %u)\n",
 		chid, devid, cpuid);
 
+	put_device(&pdev->dev);
+
 	return map;
 
 failed:
 	spin_unlock_irqrestore(&dmamux->lock, flags);
 	of_node_put(dma_spec->np);
 	dev_err(&pdev->dev, "errno %d\n", ret);
+err_put_pdev:
+	put_device(&pdev->dev);
+
 	return ERR_PTR(ret);
 }
 
diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c
index 3371e0a76d3c..e9caf8adca1f 100644
--- a/drivers/dma/dw-edma/dw-edma-pcie.c
+++ b/drivers/dma/dw-edma/dw-edma-pcie.c
@@ -161,13 +161,13 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
 			      const struct pci_device_id *pid)
 {
 	struct dw_edma_pcie_data *pdata = (void *)pid->driver_data;
-	struct dw_edma_pcie_data *vsec_data __free(kfree) = NULL;
 	struct device *dev = &pdev->dev;
 	struct dw_edma_chip *chip;
 	int err, nr_irqs;
 	int i, mask;
 
-	vsec_data = kmalloc(sizeof(*vsec_data), GFP_KERNEL);
+	struct dw_edma_pcie_data *vsec_data __free(kfree) =
+		kmalloc(sizeof(*vsec_data), GFP_KERNEL);
 	if (!vsec_data)
 		return -ENOMEM;
 
diff --git a/drivers/dma/dw/rzn1-dmamux.c b/drivers/dma/dw/rzn1-dmamux.c
index deadf135681b..cbec277af4dd 100644
--- a/drivers/dma/dw/rzn1-dmamux.c
+++ b/drivers/dma/dw/rzn1-dmamux.c
@@ -90,7 +90,7 @@ static void *rzn1_dmamux_route_allocate(struct of_phandle_args *dma_spec,
 
 	if (test_and_set_bit(map->req_idx, dmamux->used_chans)) {
 		ret = -EBUSY;
-		goto free_map;
+		goto put_dma_spec_np;
 	}
 
 	mask = BIT(map->req_idx);
@@ -103,6 +103,8 @@ static void *rzn1_dmamux_route_allocate(struct of_phandle_args *dma_spec,
 
 clear_bitmap:
 	clear_bit(map->req_idx, dmamux->used_chans);
+put_dma_spec_np:
+	of_node_put(dma_spec->np);
 free_map:
 	kfree(map);
 put_device:
diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c
index a59212758029..7137f51ff6a0 100644
--- a/drivers/dma/fsl-edma-common.c
+++ b/drivers/dma/fsl-edma-common.c
@@ -873,6 +873,7 @@ int fsl_edma_alloc_chan_resources(struct dma_chan *chan)
 		free_irq(fsl_chan->txirq, fsl_chan);
 err_txirq:
 	dma_pool_destroy(fsl_chan->tcd_pool);
+	clk_disable_unprepare(fsl_chan->clk);
 
 	return ret;
 }
diff --git a/drivers/dma/idxd/compat.c b/drivers/dma/idxd/compat.c
index eff9943f1a42..95b8ef958633 100644
--- a/drivers/dma/idxd/compat.c
+++ b/drivers/dma/idxd/compat.c
@@ -20,11 +20,16 @@ static ssize_t unbind_store(struct device_driver *drv, const char *buf, size_t c
 	int rc = -ENODEV;
 
 	dev = bus_find_device_by_name(bus, NULL, buf);
-	if (dev && dev->driver) {
+	if (!dev)
+		return -ENODEV;
+
+	if (dev->driver) {
 		device_driver_detach(dev);
 		rc = count;
 	}
 
+	put_device(dev);
+
 	return rc;
 }
 static DRIVER_ATTR_IGNORE_LOCKDEP(unbind, 0200, NULL, unbind_store);
@@ -38,9 +43,12 @@ static ssize_t bind_store(struct device_driver *drv, const char *buf, size_t cou
 	struct idxd_dev *idxd_dev;
 
 	dev = bus_find_device_by_name(bus, NULL, buf);
-	if (!dev || dev->driver || drv != &dsa_drv.drv)
+	if (!dev)
 		return -ENODEV;
 
+	if (dev->driver || drv != &dsa_drv.drv)
+		goto err_put_dev;
+
 	idxd_dev = confdev_to_idxd_dev(dev);
 	if (is_idxd_dev(idxd_dev)) {
 		alt_drv = driver_find("idxd", bus);
@@ -53,13 +61,20 @@ static ssize_t bind_store(struct device_driver *drv, const char *buf, size_t cou
 			alt_drv = driver_find("user", bus);
 	}
 	if (!alt_drv)
-		return -ENODEV;
+		goto err_put_dev;
 
 	rc = device_driver_attach(alt_drv, dev);
 	if (rc < 0)
-		return rc;
+		goto err_put_dev;
+
+	put_device(dev);
 
 	return count;
+
+err_put_dev:
+	put_device(dev);
+
+	return rc;
 }
 static DRIVER_ATTR_IGNORE_LOCKDEP(bind, 0200, NULL, bind_store);
 
diff --git a/drivers/dma/lpc18xx-dmamux.c b/drivers/dma/lpc18xx-dmamux.c
index 2b6436f4b193..d3ff521951b8 100644
--- a/drivers/dma/lpc18xx-dmamux.c
+++ b/drivers/dma/lpc18xx-dmamux.c
@@ -57,30 +57,31 @@ static void *lpc18xx_dmamux_reserve(struct of_phandle_args *dma_spec,
 	struct lpc18xx_dmamux_data *dmamux = platform_get_drvdata(pdev);
 	unsigned long flags;
 	unsigned mux;
+	int ret = -EINVAL;
 
 	if (dma_spec->args_count != 3) {
 		dev_err(&pdev->dev, "invalid number of dma mux args\n");
-		return ERR_PTR(-EINVAL);
+		goto err_put_pdev;
 	}
 
 	mux = dma_spec->args[0];
 	if (mux >= dmamux->dma_master_requests) {
 		dev_err(&pdev->dev, "invalid mux number: %d\n",
 			dma_spec->args[0]);
-		return ERR_PTR(-EINVAL);
+		goto err_put_pdev;
 	}
 
 	if (dma_spec->args[1] > LPC18XX_DMAMUX_MAX_VAL) {
 		dev_err(&pdev->dev, "invalid dma mux value: %d\n",
 			dma_spec->args[1]);
-		return ERR_PTR(-EINVAL);
+		goto err_put_pdev;
 	}
 
 	/* The of_node_put() will be done in the core for the node */
 	dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0);
 	if (!dma_spec->np) {
 		dev_err(&pdev->dev, "can't get dma master\n");
-		return ERR_PTR(-EINVAL);
+		goto err_put_pdev;
 	}
 
 	spin_lock_irqsave(&dmamux->lock, flags);
@@ -89,7 +90,8 @@ static void *lpc18xx_dmamux_reserve(struct of_phandle_args *dma_spec,
 		dev_err(&pdev->dev, "dma request %u busy with %u.%u\n",
 			mux, mux, dmamux->muxes[mux].value);
 		of_node_put(dma_spec->np);
-		return ERR_PTR(-EBUSY);
+		ret = -EBUSY;
+		goto err_put_pdev;
 	}
 
 	dmamux->muxes[mux].busy = true;
@@ -106,7 +108,14 @@ static void *lpc18xx_dmamux_reserve(struct of_phandle_args *dma_spec,
 	dev_dbg(&pdev->dev, "mapping dmamux %u.%u to dma request %u\n", mux,
 		dmamux->muxes[mux].value, mux);
 
+	put_device(&pdev->dev);
+
 	return &dmamux->muxes[mux];
+
+err_put_pdev:
+	put_device(&pdev->dev);
+
+	return ERR_PTR(ret);
 }
 
 static int lpc18xx_dmamux_probe(struct platform_device *pdev)
diff --git a/drivers/dma/lpc32xx-dmamux.c b/drivers/dma/lpc32xx-dmamux.c
index 351d7e23e615..33be714740dd 100644
--- a/drivers/dma/lpc32xx-dmamux.c
+++ b/drivers/dma/lpc32xx-dmamux.c
@@ -95,11 +95,12 @@ static void *lpc32xx_dmamux_reserve(struct of_phandle_args *dma_spec,
 	struct lpc32xx_dmamux_data *dmamux = platform_get_drvdata(pdev);
 	unsigned long flags;
 	struct lpc32xx_dmamux *mux = NULL;
+	int ret = -EINVAL;
 	int i;
 
 	if (dma_spec->args_count != 3) {
 		dev_err(&pdev->dev, "invalid number of dma mux args\n");
-		return ERR_PTR(-EINVAL);
+		goto err_put_pdev;
 	}
 
 	for (i = 0; i < ARRAY_SIZE(lpc32xx_muxes); i++) {
@@ -111,20 +112,20 @@ static void *lpc32xx_dmamux_reserve(struct of_phandle_args *dma_spec,
 	if (!mux) {
 		dev_err(&pdev->dev, "invalid mux request number: %d\n",
 			dma_spec->args[0]);
-		return ERR_PTR(-EINVAL);
+		goto err_put_pdev;
 	}
 
 	if (dma_spec->args[2] > 1) {
 		dev_err(&pdev->dev, "invalid dma mux value: %d\n",
 			dma_spec->args[1]);
-		return ERR_PTR(-EINVAL);
+		goto err_put_pdev;
 	}
 
 	/* The of_node_put() will be done in the core for the node */
 	dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0);
 	if (!dma_spec->np) {
 		dev_err(&pdev->dev, "can't get dma master\n");
-		return ERR_PTR(-EINVAL);
+		goto err_put_pdev;
 	}
 
 	spin_lock_irqsave(&dmamux->lock, flags);
@@ -133,7 +134,8 @@ static void *lpc32xx_dmamux_reserve(struct of_phandle_args *dma_spec,
 		dev_err(dev, "dma request signal %d busy, routed to %s\n",
 			mux->signal, mux->muxval ? mux->name_sel1 : mux->name_sel1);
 		of_node_put(dma_spec->np);
-		return ERR_PTR(-EBUSY);
+		ret = -EBUSY;
+		goto err_put_pdev;
 	}
 
 	mux->busy = true;
@@ -148,7 +150,14 @@ static void *lpc32xx_dmamux_reserve(struct of_phandle_args *dma_spec,
 	dev_dbg(dev, "dma request signal %d routed to %s\n",
 		mux->signal, mux->muxval ? mux->name_sel1 : mux->name_sel1);
 
+	put_device(&pdev->dev);
+
 	return mux;
+
+err_put_pdev:
+	put_device(&pdev->dev);
+
+	return ERR_PTR(ret);
 }
 
 static int lpc32xx_dmamux_probe(struct platform_device *pdev)
diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c
index d07229a74886..d12e729ee12c 100644
--- a/drivers/dma/mmp_pdma.c
+++ b/drivers/dma/mmp_pdma.c
@@ -152,8 +152,8 @@ struct mmp_pdma_phy {
  *
  * Controller Configuration:
  * @run_bits:   Control bits in DCSR register for channel start/stop
- * @dma_mask:   DMA addressing capability of controller. 0 to use OF/platform
- *              settings, or explicit mask like DMA_BIT_MASK(32/64)
+ * @dma_width:  DMA addressing width in bits (32 or 64). Determines the
+ *              DMA mask capability of the controller hardware.
  */
 struct mmp_pdma_ops {
 	/* Hardware Register Operations */
@@ -173,7 +173,7 @@ struct mmp_pdma_ops {
 
 	/* Controller Configuration */
 	u32 run_bits;
-	u64 dma_mask;
+	u32 dma_width;
 };
 
 struct mmp_pdma_device {
@@ -928,6 +928,7 @@ static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan,
 {
 	struct mmp_pdma_desc_sw *sw;
 	struct mmp_pdma_device *pdev = to_mmp_pdma_dev(chan->chan.device);
+	unsigned long flags;
 	u64 curr;
 	u32 residue = 0;
 	bool passed = false;
@@ -945,6 +946,8 @@ static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan,
 	else
 		curr = pdev->ops->read_src_addr(chan->phy);
 
+	spin_lock_irqsave(&chan->desc_lock, flags);
+
 	list_for_each_entry(sw, &chan->chain_running, node) {
 		u64 start, end;
 		u32 len;
@@ -989,6 +992,7 @@ static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan,
 			continue;
 
 		if (sw->async_tx.cookie == cookie) {
+			spin_unlock_irqrestore(&chan->desc_lock, flags);
 			return residue;
 		} else {
 			residue = 0;
@@ -996,6 +1000,8 @@ static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan,
 		}
 	}
 
+	spin_unlock_irqrestore(&chan->desc_lock, flags);
+
 	/* We should only get here in case of cyclic transactions */
 	return residue;
 }
@@ -1172,7 +1178,7 @@ static const struct mmp_pdma_ops marvell_pdma_v1_ops = {
 	.get_desc_src_addr = get_desc_src_addr_32,
 	.get_desc_dst_addr = get_desc_dst_addr_32,
 	.run_bits = (DCSR_RUN),
-	.dma_mask = 0,			/* let OF/platform set DMA mask */
+	.dma_width = 32,
 };
 
 static const struct mmp_pdma_ops spacemit_k1_pdma_ops = {
@@ -1185,7 +1191,7 @@ static const struct mmp_pdma_ops spacemit_k1_pdma_ops = {
 	.get_desc_src_addr = get_desc_src_addr_64,
 	.get_desc_dst_addr = get_desc_dst_addr_64,
 	.run_bits = (DCSR_RUN | DCSR_LPAEEN),
-	.dma_mask = DMA_BIT_MASK(64),	/* force 64-bit DMA addr capability */
+	.dma_width = 64,
 };
 
 static const struct of_device_id mmp_pdma_dt_ids[] = {
@@ -1314,13 +1320,9 @@ static int mmp_pdma_probe(struct platform_device *op)
 	pdev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
 	pdev->device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
 
-	/* Set DMA mask based on ops->dma_mask, or OF/platform */
-	if (pdev->ops->dma_mask)
-		dma_set_mask(pdev->dev, pdev->ops->dma_mask);
-	else if (pdev->dev->coherent_dma_mask)
-		dma_set_mask(pdev->dev, pdev->dev->coherent_dma_mask);
-	else
-		dma_set_mask(pdev->dev, DMA_BIT_MASK(64));
+	/* Set DMA mask based on controller hardware capabilities */
+	dma_set_mask_and_coherent(pdev->dev,
+				  DMA_BIT_MASK(pdev->ops->dma_width));
 
 	ret = dma_async_device_register(&pdev->device);
 	if (ret) {
diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c
index 82a9fe88ad54..72f260328ae9 100644
--- a/drivers/dma/pl330.c
+++ b/drivers/dma/pl330.c
@@ -2133,10 +2133,8 @@ static void pl330_tasklet(struct tasklet_struct *t)
 	spin_unlock_irqrestore(&pch->lock, flags);
 
 	/* If work list empty, power down */
-	if (power_down) {
-		pm_runtime_mark_last_busy(pch->dmac->ddma.dev);
+	if (power_down)
 		pm_runtime_put_autosuspend(pch->dmac->ddma.dev);
-	}
 }
 
 static struct dma_chan *of_dma_pl330_xlate(struct of_phandle_args *dma_spec,
@@ -2313,7 +2311,6 @@ static int pl330_terminate_all(struct dma_chan *chan)
 	list_splice_tail_init(&pch->work_list, &pl330->desc_pool);
 	list_splice_tail_init(&pch->completed_list, &pl330->desc_pool);
 	spin_unlock_irqrestore(&pch->lock, flags);
-	pm_runtime_mark_last_busy(pl330->ddma.dev);
 	if (power_down)
 		pm_runtime_put_autosuspend(pl330->ddma.dev);
 	pm_runtime_put_autosuspend(pl330->ddma.dev);
@@ -2347,7 +2344,6 @@ static int pl330_pause(struct dma_chan *chan)
 			desc->status = PAUSED;
 	}
 	spin_unlock_irqrestore(&pch->lock, flags);
-	pm_runtime_mark_last_busy(pl330->ddma.dev);
 	pm_runtime_put_autosuspend(pl330->ddma.dev);
 
 	return 0;
@@ -2371,7 +2367,6 @@ static void pl330_free_chan_resources(struct dma_chan *chan)
 		list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool);
 
 	spin_unlock_irqrestore(&pl330->lock, flags);
-	pm_runtime_mark_last_busy(pch->dmac->ddma.dev);
 	pm_runtime_put_autosuspend(pch->dmac->ddma.dev);
 	pl330_unprep_slave_fifo(pch);
 }
@@ -3176,7 +3171,6 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id)
 	pm_runtime_irq_safe(&adev->dev);
 	pm_runtime_use_autosuspend(&adev->dev);
 	pm_runtime_set_autosuspend_delay(&adev->dev, PL330_AUTOSUSPEND_DELAY);
-	pm_runtime_mark_last_busy(&adev->dev);
 	pm_runtime_put_autosuspend(&adev->dev);
 
 	return 0;
diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c
index 2cf060174795..bcd8de9a9a12 100644
--- a/drivers/dma/qcom/bam_dma.c
+++ b/drivers/dma/qcom/bam_dma.c
@@ -23,24 +23,25 @@
  * indication of where the hardware is currently working.
  */
 
-#include <linux/kernel.h>
-#include <linux/io.h>
+#include <linux/circ_buf.h>
+#include <linux/cleanup.h>
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmaengine.h>
 #include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/module.h>
 #include <linux/interrupt.h>
-#include <linux/dma-mapping.h>
-#include <linux/scatterlist.h>
-#include <linux/device.h>
-#include <linux/platform_device.h>
-#include <linux/of.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
 #include <linux/of_address.h>
-#include <linux/of_irq.h>
 #include <linux/of_dma.h>
-#include <linux/circ_buf.h>
-#include <linux/clk.h>
-#include <linux/dmaengine.h>
+#include <linux/of_irq.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
 
 #include "../dmaengine.h"
 #include "../virt-dma.h"
@@ -570,7 +571,6 @@ static void bam_free_chan(struct dma_chan *chan)
 	struct bam_chan *bchan = to_bam_chan(chan);
 	struct bam_device *bdev = bchan->bdev;
 	u32 val;
-	unsigned long flags;
 	int ret;
 
 	ret = pm_runtime_get_sync(bdev->dev);
@@ -584,9 +584,8 @@ static void bam_free_chan(struct dma_chan *chan)
 		goto err;
 	}
 
-	spin_lock_irqsave(&bchan->vc.lock, flags);
-	bam_reset_channel(bchan);
-	spin_unlock_irqrestore(&bchan->vc.lock, flags);
+	scoped_guard(spinlock_irqsave, &bchan->vc.lock)
+		bam_reset_channel(bchan);
 
 	dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE, bchan->fifo_virt,
 		    bchan->fifo_phys);
@@ -624,12 +623,11 @@ static int bam_slave_config(struct dma_chan *chan,
 			    struct dma_slave_config *cfg)
 {
 	struct bam_chan *bchan = to_bam_chan(chan);
-	unsigned long flag;
 
-	spin_lock_irqsave(&bchan->vc.lock, flag);
+	guard(spinlock_irqsave)(&bchan->vc.lock);
+
 	memcpy(&bchan->slave, cfg, sizeof(*cfg));
 	bchan->reconfigure = 1;
-	spin_unlock_irqrestore(&bchan->vc.lock, flag);
 
 	return 0;
 }
@@ -726,38 +724,37 @@ static int bam_dma_terminate_all(struct dma_chan *chan)
 {
 	struct bam_chan *bchan = to_bam_chan(chan);
 	struct bam_async_desc *async_desc, *tmp;
-	unsigned long flag;
 	LIST_HEAD(head);
 
 	/* remove all transactions, including active transaction */
-	spin_lock_irqsave(&bchan->vc.lock, flag);
-	/*
-	 * If we have transactions queued, then some might be committed to the
-	 * hardware in the desc fifo.  The only way to reset the desc fifo is
-	 * to do a hardware reset (either by pipe or the entire block).
-	 * bam_chan_init_hw() will trigger a pipe reset, and also reinit the
-	 * pipe.  If the pipe is left disabled (default state after pipe reset)
-	 * and is accessed by a connected hardware engine, a fatal error in
-	 * the BAM will occur.  There is a small window where this could happen
-	 * with bam_chan_init_hw(), but it is assumed that the caller has
-	 * stopped activity on any attached hardware engine.  Make sure to do
-	 * this first so that the BAM hardware doesn't cause memory corruption
-	 * by accessing freed resources.
-	 */
-	if (!list_empty(&bchan->desc_list)) {
-		async_desc = list_first_entry(&bchan->desc_list,
-					      struct bam_async_desc, desc_node);
-		bam_chan_init_hw(bchan, async_desc->dir);
-	}
+	scoped_guard(spinlock_irqsave, &bchan->vc.lock) {
+		/*
+		 * If we have transactions queued, then some might be committed to the
+		 * hardware in the desc fifo.  The only way to reset the desc fifo is
+		 * to do a hardware reset (either by pipe or the entire block).
+		 * bam_chan_init_hw() will trigger a pipe reset, and also reinit the
+		 * pipe.  If the pipe is left disabled (default state after pipe reset)
+		 * and is accessed by a connected hardware engine, a fatal error in
+		 * the BAM will occur.  There is a small window where this could happen
+		 * with bam_chan_init_hw(), but it is assumed that the caller has
+		 * stopped activity on any attached hardware engine.  Make sure to do
+		 * this first so that the BAM hardware doesn't cause memory corruption
+		 * by accessing freed resources.
+		 */
+		if (!list_empty(&bchan->desc_list)) {
+			async_desc = list_first_entry(&bchan->desc_list,
+						      struct bam_async_desc, desc_node);
+			bam_chan_init_hw(bchan, async_desc->dir);
+		}
 
-	list_for_each_entry_safe(async_desc, tmp,
-				 &bchan->desc_list, desc_node) {
-		list_add(&async_desc->vd.node, &bchan->vc.desc_issued);
-		list_del(&async_desc->desc_node);
-	}
+		list_for_each_entry_safe(async_desc, tmp,
+					 &bchan->desc_list, desc_node) {
+			list_add(&async_desc->vd.node, &bchan->vc.desc_issued);
+			list_del(&async_desc->desc_node);
+		}
 
-	vchan_get_all_descriptors(&bchan->vc, &head);
-	spin_unlock_irqrestore(&bchan->vc.lock, flag);
+		vchan_get_all_descriptors(&bchan->vc, &head);
+	}
 
 	vchan_dma_desc_free_list(&bchan->vc, &head);
 
@@ -773,17 +770,16 @@ static int bam_pause(struct dma_chan *chan)
 {
 	struct bam_chan *bchan = to_bam_chan(chan);
 	struct bam_device *bdev = bchan->bdev;
-	unsigned long flag;
 	int ret;
 
 	ret = pm_runtime_get_sync(bdev->dev);
 	if (ret < 0)
 		return ret;
 
-	spin_lock_irqsave(&bchan->vc.lock, flag);
-	writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_HALT));
-	bchan->paused = 1;
-	spin_unlock_irqrestore(&bchan->vc.lock, flag);
+	scoped_guard(spinlock_irqsave, &bchan->vc.lock) {
+		writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_HALT));
+		bchan->paused = 1;
+	}
 	pm_runtime_mark_last_busy(bdev->dev);
 	pm_runtime_put_autosuspend(bdev->dev);
 
@@ -799,17 +795,16 @@ static int bam_resume(struct dma_chan *chan)
 {
 	struct bam_chan *bchan = to_bam_chan(chan);
 	struct bam_device *bdev = bchan->bdev;
-	unsigned long flag;
 	int ret;
 
 	ret = pm_runtime_get_sync(bdev->dev);
 	if (ret < 0)
 		return ret;
 
-	spin_lock_irqsave(&bchan->vc.lock, flag);
-	writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_HALT));
-	bchan->paused = 0;
-	spin_unlock_irqrestore(&bchan->vc.lock, flag);
+	scoped_guard(spinlock_irqsave, &bchan->vc.lock) {
+		writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_HALT));
+		bchan->paused = 0;
+	}
 	pm_runtime_mark_last_busy(bdev->dev);
 	pm_runtime_put_autosuspend(bdev->dev);
 
@@ -826,7 +821,6 @@ static int bam_resume(struct dma_chan *chan)
 static u32 process_channel_irqs(struct bam_device *bdev)
 {
 	u32 i, srcs, pipe_stts, offset, avail;
-	unsigned long flags;
 	struct bam_async_desc *async_desc, *tmp;
 
 	srcs = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_EE));
@@ -846,7 +840,7 @@ static u32 process_channel_irqs(struct bam_device *bdev)
 
 		writel_relaxed(pipe_stts, bam_addr(bdev, i, BAM_P_IRQ_CLR));
 
-		spin_lock_irqsave(&bchan->vc.lock, flags);
+		guard(spinlock_irqsave)(&bchan->vc.lock);
 
 		offset = readl_relaxed(bam_addr(bdev, i, BAM_P_SW_OFSTS)) &
 				       P_SW_OFSTS_MASK;
@@ -885,8 +879,6 @@ static u32 process_channel_irqs(struct bam_device *bdev)
 			}
 			list_del(&async_desc->desc_node);
 		}
-
-		spin_unlock_irqrestore(&bchan->vc.lock, flags);
 	}
 
 	return srcs;
@@ -950,7 +942,6 @@ static enum dma_status bam_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 	int ret;
 	size_t residue = 0;
 	unsigned int i;
-	unsigned long flags;
 
 	ret = dma_cookie_status(chan, cookie, txstate);
 	if (ret == DMA_COMPLETE)
@@ -959,23 +950,22 @@ static enum dma_status bam_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 	if (!txstate)
 		return bchan->paused ? DMA_PAUSED : ret;
 
-	spin_lock_irqsave(&bchan->vc.lock, flags);
-	vd = vchan_find_desc(&bchan->vc, cookie);
-	if (vd) {
-		residue = container_of(vd, struct bam_async_desc, vd)->length;
-	} else {
-		list_for_each_entry(async_desc, &bchan->desc_list, desc_node) {
-			if (async_desc->vd.tx.cookie != cookie)
-				continue;
-
-			for (i = 0; i < async_desc->num_desc; i++)
-				residue += le16_to_cpu(
-						async_desc->curr_desc[i].size);
+	scoped_guard(spinlock_irqsave, &bchan->vc.lock) {
+		vd = vchan_find_desc(&bchan->vc, cookie);
+		if (vd) {
+			residue = container_of(vd, struct bam_async_desc, vd)->length;
+		} else {
+			list_for_each_entry(async_desc, &bchan->desc_list, desc_node) {
+				if (async_desc->vd.tx.cookie != cookie)
+					continue;
+
+				for (i = 0; i < async_desc->num_desc; i++)
+					residue += le16_to_cpu(
+							async_desc->curr_desc[i].size);
+			}
 		}
 	}
 
-	spin_unlock_irqrestore(&bchan->vc.lock, flags);
-
 	dma_set_residue(txstate, residue);
 
 	if (ret == DMA_IN_PROGRESS && bchan->paused)
@@ -1116,17 +1106,16 @@ static void dma_tasklet(struct tasklet_struct *t)
 {
 	struct bam_device *bdev = from_tasklet(bdev, t, task);
 	struct bam_chan *bchan;
-	unsigned long flags;
 	unsigned int i;
 
 	/* go through the channels and kick off transactions */
 	for (i = 0; i < bdev->num_channels; i++) {
 		bchan = &bdev->channels[i];
-		spin_lock_irqsave(&bchan->vc.lock, flags);
+
+		guard(spinlock_irqsave)(&bchan->vc.lock);
 
 		if (!list_empty(&bchan->vc.desc_issued) && !IS_BUSY(bchan))
 			bam_start_dma(bchan);
-		spin_unlock_irqrestore(&bchan->vc.lock, flags);
 	}
 
 }
@@ -1140,15 +1129,12 @@ static void dma_tasklet(struct tasklet_struct *t)
 static void bam_issue_pending(struct dma_chan *chan)
 {
 	struct bam_chan *bchan = to_bam_chan(chan);
-	unsigned long flags;
 
-	spin_lock_irqsave(&bchan->vc.lock, flags);
+	guard(spinlock_irqsave)(&bchan->vc.lock);
 
 	/* if work pending and idle, start a transaction */
 	if (vchan_issue_pending(&bchan->vc) && !IS_BUSY(bchan))
 		bam_start_dma(bchan);
-
-	spin_unlock_irqrestore(&bchan->vc.lock, flags);
 }
 
 /**
diff --git a/drivers/dma/sh/rz-dmac.c b/drivers/dma/sh/rz-dmac.c
index 1f687b08d6b8..9e5f088355e2 100644
--- a/drivers/dma/sh/rz-dmac.c
+++ b/drivers/dma/sh/rz-dmac.c
@@ -557,11 +557,16 @@ rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 static int rz_dmac_terminate_all(struct dma_chan *chan)
 {
 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
+	struct rz_lmdesc *lmdesc = channel->lmdesc.base;
 	unsigned long flags;
+	unsigned int i;
 	LIST_HEAD(head);
 
 	rz_dmac_disable_hw(channel);
 	spin_lock_irqsave(&channel->vc.lock, flags);
+	for (i = 0; i < DMAC_NR_LMDESC; i++)
+		lmdesc[i].header = 0;
+
 	list_splice_tail_init(&channel->ld_active, &channel->ld_free);
 	list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
 	vchan_get_all_descriptors(&channel->vc, &head);
@@ -854,6 +859,13 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac,
 	return 0;
 }
 
+static void rz_dmac_put_device(void *_dev)
+{
+	struct device *dev = _dev;
+
+	put_device(dev);
+}
+
 static int rz_dmac_parse_of_icu(struct device *dev, struct rz_dmac *dmac)
 {
 	struct device_node *np = dev->of_node;
@@ -876,6 +888,10 @@ static int rz_dmac_parse_of_icu(struct device *dev, struct rz_dmac *dmac)
 		return -ENODEV;
 	}
 
+	ret = devm_add_action_or_reset(dev, rz_dmac_put_device, &dmac->icu.pdev->dev);
+	if (ret)
+		return ret;
+
 	dmac_index = args.args[0];
 	if (dmac_index > RZV2H_MAX_DMAC_INDEX) {
 		dev_err(dev, "DMAC index %u invalid.\n", dmac_index);
@@ -1055,8 +1071,6 @@ static void rz_dmac_remove(struct platform_device *pdev)
 	reset_control_assert(dmac->rstc);
 	pm_runtime_put(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-
-	platform_device_put(dmac->icu.pdev);
 }
 
 static const struct of_device_id of_rz_dmac_match[] = {
diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c
index d52e1685aed5..e67e0d66e6e8 100644
--- a/drivers/dma/ste_dma40.c
+++ b/drivers/dma/ste_dma40.c
@@ -1452,7 +1452,6 @@ static int d40_pause(struct dma_chan *chan)
 
 	res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
 
-	pm_runtime_mark_last_busy(d40c->base->dev);
 	pm_runtime_put_autosuspend(d40c->base->dev);
 	spin_unlock_irqrestore(&d40c->lock, flags);
 	return res;
@@ -1479,7 +1478,6 @@ static int d40_resume(struct dma_chan *chan)
 	if (d40_residue(d40c) || d40_tx_is_linked(d40c))
 		res = d40_channel_execute_command(d40c, D40_DMA_RUN);
 
-	pm_runtime_mark_last_busy(d40c->base->dev);
 	pm_runtime_put_autosuspend(d40c->base->dev);
 	spin_unlock_irqrestore(&d40c->lock, flags);
 	return res;
@@ -1581,7 +1579,6 @@ static void dma_tc_handle(struct d40_chan *d40c)
 		if (d40_queue_start(d40c) == NULL) {
 			d40c->busy = false;
 
-			pm_runtime_mark_last_busy(d40c->base->dev);
 			pm_runtime_put_autosuspend(d40c->base->dev);
 		}
 
@@ -2054,16 +2051,13 @@ static int d40_free_dma(struct d40_chan *d40c)
 	else
 		d40c->base->lookup_phy_chans[phy->num] = NULL;
 
-	if (d40c->busy) {
-		pm_runtime_mark_last_busy(d40c->base->dev);
+	if (d40c->busy)
 		pm_runtime_put_autosuspend(d40c->base->dev);
-	}
 
 	d40c->busy = false;
 	d40c->phy_chan = NULL;
 	d40c->configured = false;
  mark_last_busy:
-	pm_runtime_mark_last_busy(d40c->base->dev);
 	pm_runtime_put_autosuspend(d40c->base->dev);
 	return res;
 }
@@ -2466,7 +2460,6 @@ static int d40_alloc_chan_resources(struct dma_chan *chan)
 	if (is_free_phy)
 		d40_config_write(d40c);
  mark_last_busy:
-	pm_runtime_mark_last_busy(d40c->base->dev);
 	pm_runtime_put_autosuspend(d40c->base->dev);
 	spin_unlock_irqrestore(&d40c->lock, flags);
 	return err;
@@ -2618,12 +2611,9 @@ static int d40_terminate_all(struct dma_chan *chan)
 		chan_err(d40c, "Failed to stop channel\n");
 
 	d40_term_all(d40c);
-	pm_runtime_mark_last_busy(d40c->base->dev);
 	pm_runtime_put_autosuspend(d40c->base->dev);
-	if (d40c->busy) {
-		pm_runtime_mark_last_busy(d40c->base->dev);
+	if (d40c->busy)
 		pm_runtime_put_autosuspend(d40c->base->dev);
-	}
 	d40c->busy = false;
 
 	spin_unlock_irqrestore(&d40c->lock, flags);
diff --git a/drivers/dma/stm32/stm32-dma3.c b/drivers/dma/stm32/stm32-dma3.c
index 50e7106c5cb7..84b00c436134 100644
--- a/drivers/dma/stm32/stm32-dma3.c
+++ b/drivers/dma/stm32/stm32-dma3.c
@@ -288,6 +288,7 @@ struct stm32_dma3_chan {
 	u32 fifo_size;
 	u32 max_burst;
 	bool semaphore_mode;
+	bool semaphore_taken;
 	struct stm32_dma3_dt_conf dt_config;
 	struct dma_slave_config dma_config;
 	u8 config_set;
@@ -332,6 +333,11 @@ static struct device *chan2dev(struct stm32_dma3_chan *chan)
 	return &chan->vchan.chan.dev->device;
 }
 
+static struct device *ddata2dev(struct stm32_dma3_ddata *ddata)
+{
+	return ddata->dma_dev.dev;
+}
+
 static void stm32_dma3_chan_dump_reg(struct stm32_dma3_chan *chan)
 {
 	struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
@@ -1063,14 +1069,53 @@ static irqreturn_t stm32_dma3_chan_irq(int irq, void *devid)
 	return IRQ_HANDLED;
 }
 
+static int stm32_dma3_get_chan_sem(struct stm32_dma3_chan *chan)
+{
+	struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
+	u32 csemcr, ccid;
+
+	csemcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id));
+	/* Make an attempt to take the channel semaphore if not already taken */
+	if (!(csemcr & CSEMCR_SEM_MUTEX)) {
+		writel_relaxed(CSEMCR_SEM_MUTEX, ddata->base + STM32_DMA3_CSEMCR(chan->id));
+		csemcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id));
+	}
+
+	/* Check if channel is under CID1 control */
+	ccid = FIELD_GET(CSEMCR_SEM_CCID, csemcr);
+	if (!(csemcr & CSEMCR_SEM_MUTEX) || ccid != CCIDCFGR_CID1)
+		goto bad_cid;
+
+	chan->semaphore_taken = true;
+	dev_dbg(chan2dev(chan), "under CID1 control (semcr=0x%08x)\n", csemcr);
+
+	return 0;
+
+bad_cid:
+	chan->semaphore_taken = false;
+	dev_err(chan2dev(chan), "not under CID1 control (in-use by CID%d)\n", ccid);
+
+	return -EACCES;
+}
+
+static void stm32_dma3_put_chan_sem(struct stm32_dma3_chan *chan)
+{
+	struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
+
+	if (chan->semaphore_taken) {
+		writel_relaxed(0, ddata->base + STM32_DMA3_CSEMCR(chan->id));
+		chan->semaphore_taken = false;
+		dev_dbg(chan2dev(chan), "no more under CID1 control\n");
+	}
+}
+
 static int stm32_dma3_alloc_chan_resources(struct dma_chan *c)
 {
 	struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
 	struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
-	u32 id = chan->id, csemcr, ccid;
 	int ret;
 
-	ret = pm_runtime_resume_and_get(ddata->dma_dev.dev);
+	ret = pm_runtime_resume_and_get(ddata2dev(ddata));
 	if (ret < 0)
 		return ret;
 
@@ -1092,16 +1137,9 @@ static int stm32_dma3_alloc_chan_resources(struct dma_chan *c)
 
 	/* Take the channel semaphore */
 	if (chan->semaphore_mode) {
-		writel_relaxed(CSEMCR_SEM_MUTEX, ddata->base + STM32_DMA3_CSEMCR(id));
-		csemcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(id));
-		ccid = FIELD_GET(CSEMCR_SEM_CCID, csemcr);
-		/* Check that the channel is well taken */
-		if (ccid != CCIDCFGR_CID1) {
-			dev_err(chan2dev(chan), "Not under CID1 control (in-use by CID%d)\n", ccid);
-			ret = -EPERM;
+		ret = stm32_dma3_get_chan_sem(chan);
+		if (ret)
 			goto err_pool_destroy;
-		}
-		dev_dbg(chan2dev(chan), "Under CID1 control (semcr=0x%08x)\n", csemcr);
 	}
 
 	return 0;
@@ -1111,7 +1149,7 @@ static int stm32_dma3_alloc_chan_resources(struct dma_chan *c)
 	chan->lli_pool = NULL;
 
 err_put_sync:
-	pm_runtime_put_sync(ddata->dma_dev.dev);
+	pm_runtime_put_sync(ddata2dev(ddata));
 
 	return ret;
 }
@@ -1135,9 +1173,9 @@ static void stm32_dma3_free_chan_resources(struct dma_chan *c)
 
 	/* Release the channel semaphore */
 	if (chan->semaphore_mode)
-		writel_relaxed(0, ddata->base + STM32_DMA3_CSEMCR(chan->id));
+		stm32_dma3_put_chan_sem(chan);
 
-	pm_runtime_put_sync(ddata->dma_dev.dev);
+	pm_runtime_put_sync(ddata2dev(ddata));
 
 	/* Reset configuration */
 	memset(&chan->dt_config, 0, sizeof(chan->dt_config));
@@ -1204,6 +1242,10 @@ static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_memcpy(struct dma_cha
 	bool prevent_refactor = !!FIELD_GET(STM32_DMA3_DT_NOPACK, chan->dt_config.tr_conf) ||
 				!!FIELD_GET(STM32_DMA3_DT_NOREFACT, chan->dt_config.tr_conf);
 
+	/* Semaphore could be lost during suspend/resume */
+	if (chan->semaphore_mode && !chan->semaphore_taken)
+		return NULL;
+
 	count = stm32_dma3_get_ll_count(chan, len, prevent_refactor);
 
 	swdesc = stm32_dma3_chan_desc_alloc(chan, count);
@@ -1264,6 +1306,10 @@ static struct dma_async_tx_descriptor *stm32_dma3_prep_slave_sg(struct dma_chan
 				!!FIELD_GET(STM32_DMA3_DT_NOREFACT, chan->dt_config.tr_conf);
 	int ret;
 
+	/* Semaphore could be lost during suspend/resume */
+	if (chan->semaphore_mode && !chan->semaphore_taken)
+		return NULL;
+
 	count = 0;
 	for_each_sg(sgl, sg, sg_len, i)
 		count += stm32_dma3_get_ll_count(chan, sg_dma_len(sg), prevent_refactor);
@@ -1350,6 +1396,10 @@ static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_cyclic(struct dma_cha
 	u32 count, i, ctr1, ctr2;
 	int ret;
 
+	/* Semaphore could be lost during suspend/resume */
+	if (chan->semaphore_mode && !chan->semaphore_taken)
+		return NULL;
+
 	if (!buf_len || !period_len || period_len > STM32_DMA3_MAX_BLOCK_SIZE) {
 		dev_err(chan2dev(chan), "Invalid buffer/period length\n");
 		return NULL;
@@ -1565,11 +1615,11 @@ static bool stm32_dma3_filter_fn(struct dma_chan *c, void *fn_param)
 		if (!(mask & BIT(chan->id)))
 			return false;
 
-	ret = pm_runtime_resume_and_get(ddata->dma_dev.dev);
+	ret = pm_runtime_resume_and_get(ddata2dev(ddata));
 	if (ret < 0)
 		return false;
 	semcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id));
-	pm_runtime_put_sync(ddata->dma_dev.dev);
+	pm_runtime_put_sync(ddata2dev(ddata));
 
 	/* Check if chan is free */
 	if (semcr & CSEMCR_SEM_MUTEX)
@@ -1591,7 +1641,7 @@ static struct dma_chan *stm32_dma3_of_xlate(struct of_phandle_args *dma_spec, st
 	struct dma_chan *c;
 
 	if (dma_spec->args_count < 3) {
-		dev_err(ddata->dma_dev.dev, "Invalid args count\n");
+		dev_err(ddata2dev(ddata), "Invalid args count\n");
 		return NULL;
 	}
 
@@ -1600,14 +1650,14 @@ static struct dma_chan *stm32_dma3_of_xlate(struct of_phandle_args *dma_spec, st
 	conf.tr_conf = dma_spec->args[2];
 
 	if (conf.req_line >= ddata->dma_requests) {
-		dev_err(ddata->dma_dev.dev, "Invalid request line\n");
+		dev_err(ddata2dev(ddata), "Invalid request line\n");
 		return NULL;
 	}
 
 	/* Request dma channel among the generic dma controller list */
 	c = dma_request_channel(mask, stm32_dma3_filter_fn, &conf);
 	if (!c) {
-		dev_err(ddata->dma_dev.dev, "No suitable channel found\n");
+		dev_err(ddata2dev(ddata), "No suitable channel found\n");
 		return NULL;
 	}
 
@@ -1620,6 +1670,7 @@ static struct dma_chan *stm32_dma3_of_xlate(struct of_phandle_args *dma_spec, st
 
 static u32 stm32_dma3_check_rif(struct stm32_dma3_ddata *ddata)
 {
+	struct device *dev = ddata2dev(ddata);
 	u32 chan_reserved, mask = 0, i, ccidcfgr, invalid_cid = 0;
 
 	/* Reserve Secure channels */
@@ -1631,7 +1682,7 @@ static u32 stm32_dma3_check_rif(struct stm32_dma3_ddata *ddata)
 	 * In case CID filtering is not configured, dma-channel-mask property can be used to
 	 * specify available DMA channels to the kernel.
 	 */
-	of_property_read_u32(ddata->dma_dev.dev->of_node, "dma-channel-mask", &mask);
+	of_property_read_u32(dev->of_node, "dma-channel-mask", &mask);
 
 	/* Reserve !CID-filtered not in dma-channel-mask, static CID != CID1, CID1 not allowed */
 	for (i = 0; i < ddata->dma_channels; i++) {
@@ -1651,7 +1702,7 @@ static u32 stm32_dma3_check_rif(struct stm32_dma3_ddata *ddata)
 				ddata->chans[i].semaphore_mode = true;
 			}
 		}
-		dev_dbg(ddata->dma_dev.dev, "chan%d: %s mode, %s\n", i,
+		dev_dbg(dev, "chan%d: %s mode, %s\n", i,
 			!(ccidcfgr & CCIDCFGR_CFEN) ? "!CID-filtered" :
 			ddata->chans[i].semaphore_mode ? "Semaphore" : "Static CID",
 			(chan_reserved & BIT(i)) ? "denied" :
@@ -1659,7 +1710,7 @@ static u32 stm32_dma3_check_rif(struct stm32_dma3_ddata *ddata)
 	}
 
 	if (invalid_cid)
-		dev_warn(ddata->dma_dev.dev, "chan%*pbl have invalid CID configuration\n",
+		dev_warn(dev, "chan%*pbl have invalid CID configuration\n",
 			 ddata->dma_channels, &invalid_cid);
 
 	return chan_reserved;
@@ -1899,8 +1950,69 @@ static int stm32_dma3_runtime_resume(struct device *dev)
 	return ret;
 }
 
+static int stm32_dma3_pm_suspend(struct device *dev)
+{
+	struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev);
+	struct dma_device *dma_dev = &ddata->dma_dev;
+	struct dma_chan *c;
+	int ccr, ret;
+
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret < 0)
+		return ret;
+
+	list_for_each_entry(c, &dma_dev->channels, device_node) {
+		struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
+
+		ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id));
+		if (ccr & CCR_EN) {
+			dev_warn(dev, "Suspend is prevented: %s still in use by %s\n",
+				 dma_chan_name(c), dev_name(c->slave));
+			pm_runtime_put_sync(dev);
+			return -EBUSY;
+		}
+	}
+
+	pm_runtime_put_sync(dev);
+
+	pm_runtime_force_suspend(dev);
+
+	return 0;
+}
+
+static int stm32_dma3_pm_resume(struct device *dev)
+{
+	struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev);
+	struct dma_device *dma_dev = &ddata->dma_dev;
+	struct dma_chan *c;
+	int ret;
+
+	ret = pm_runtime_force_resume(dev);
+	if (ret < 0)
+		return ret;
+
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * Channel semaphores need to be restored in case of registers reset during low power.
+	 * stm32_dma3_get_chan_sem() will prior check the semaphore status.
+	 */
+	list_for_each_entry(c, &dma_dev->channels, device_node) {
+		struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
+
+		if (chan->semaphore_mode && chan->semaphore_taken)
+			stm32_dma3_get_chan_sem(chan);
+	}
+
+	pm_runtime_put_sync(dev);
+
+	return 0;
+}
+
 static const struct dev_pm_ops stm32_dma3_pm_ops = {
-	SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
+	SYSTEM_SLEEP_PM_OPS(stm32_dma3_pm_suspend, stm32_dma3_pm_resume)
 	RUNTIME_PM_OPS(stm32_dma3_runtime_suspend, stm32_dma3_runtime_resume, NULL)
 };
 
@@ -1914,12 +2026,7 @@ static struct platform_driver stm32_dma3_driver = {
 	},
 };
 
-static int __init stm32_dma3_init(void)
-{
-	return platform_driver_register(&stm32_dma3_driver);
-}
-
-subsys_initcall(stm32_dma3_init);
+module_platform_driver(stm32_dma3_driver);
 
 MODULE_DESCRIPTION("STM32 DMA3 controller driver");
 MODULE_AUTHOR("Amelie Delaunay <amelie.delaunay@foss.st.com>");
diff --git a/drivers/dma/stm32/stm32-dmamux.c b/drivers/dma/stm32/stm32-dmamux.c
index 8d77e2a7939a..db13498b9c9f 100644
--- a/drivers/dma/stm32/stm32-dmamux.c
+++ b/drivers/dma/stm32/stm32-dmamux.c
@@ -90,23 +90,25 @@ static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec,
 	struct stm32_dmamux_data *dmamux = platform_get_drvdata(pdev);
 	struct stm32_dmamux *mux;
 	u32 i, min, max;
-	int ret;
+	int ret = -EINVAL;
 	unsigned long flags;
 
 	if (dma_spec->args_count != 3) {
 		dev_err(&pdev->dev, "invalid number of dma mux args\n");
-		return ERR_PTR(-EINVAL);
+		goto err_put_pdev;
 	}
 
 	if (dma_spec->args[0] > dmamux->dmamux_requests) {
 		dev_err(&pdev->dev, "invalid mux request number: %d\n",
 			dma_spec->args[0]);
-		return ERR_PTR(-EINVAL);
+		goto err_put_pdev;
 	}
 
 	mux = kzalloc(sizeof(*mux), GFP_KERNEL);
-	if (!mux)
-		return ERR_PTR(-ENOMEM);
+	if (!mux) {
+		ret = -ENOMEM;
+		goto err_put_pdev;
+	}
 
 	spin_lock_irqsave(&dmamux->lock, flags);
 	mux->chan_id = find_first_zero_bit(dmamux->dma_inuse,
@@ -116,7 +118,7 @@ static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec,
 		spin_unlock_irqrestore(&dmamux->lock, flags);
 		dev_err(&pdev->dev, "Run out of free DMA requests\n");
 		ret = -ENOMEM;
-		goto error_chan_id;
+		goto err_free_mux;
 	}
 	set_bit(mux->chan_id, dmamux->dma_inuse);
 	spin_unlock_irqrestore(&dmamux->lock, flags);
@@ -133,8 +135,7 @@ static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec,
 	dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", i - 1);
 	if (!dma_spec->np) {
 		dev_err(&pdev->dev, "can't get dma master\n");
-		ret = -EINVAL;
-		goto error;
+		goto err_clear_inuse;
 	}
 
 	/* Set dma request */
@@ -142,7 +143,7 @@ static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec,
 	ret = pm_runtime_resume_and_get(&pdev->dev);
 	if (ret < 0) {
 		spin_unlock_irqrestore(&dmamux->lock, flags);
-		goto error;
+		goto err_put_dma_spec_np;
 	}
 	spin_unlock_irqrestore(&dmamux->lock, flags);
 
@@ -160,13 +161,19 @@ static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec,
 	dev_dbg(&pdev->dev, "Mapping DMAMUX(%u) to DMA%u(%u)\n",
 		mux->request, mux->master, mux->chan_id);
 
+	put_device(&pdev->dev);
+
 	return mux;
 
-error:
+err_put_dma_spec_np:
+	of_node_put(dma_spec->np);
+err_clear_inuse:
 	clear_bit(mux->chan_id, dmamux->dma_inuse);
-
-error_chan_id:
+err_free_mux:
 	kfree(mux);
+err_put_pdev:
+	put_device(&pdev->dev);
+
 	return ERR_PTR(ret);
 }
 
diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c
index d0e8bb27a03b..215bfef37ec6 100644
--- a/drivers/dma/tegra210-adma.c
+++ b/drivers/dma/tegra210-adma.c
@@ -429,10 +429,17 @@ static void tegra_adma_stop(struct tegra_adma_chan *tdc)
 		return;
 	}
 
-	kfree(tdc->desc);
+	vchan_terminate_vdesc(&tdc->desc->vd);
 	tdc->desc = NULL;
 }
 
+static void tegra_adma_synchronize(struct dma_chan *dc)
+{
+	struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
+
+	vchan_synchronize(&tdc->vc);
+}
+
 static void tegra_adma_start(struct tegra_adma_chan *tdc)
 {
 	struct virt_dma_desc *vd = vchan_next_desc(&tdc->vc);
@@ -1155,6 +1162,7 @@ static int tegra_adma_probe(struct platform_device *pdev)
 	tdma->dma_dev.device_config = tegra_adma_slave_config;
 	tdma->dma_dev.device_tx_status = tegra_adma_tx_status;
 	tdma->dma_dev.device_terminate_all = tegra_adma_terminate_all;
+	tdma->dma_dev.device_synchronize = tegra_adma_synchronize;
 	tdma->dma_dev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
 	tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
 	tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
diff --git a/drivers/dma/ti/Kconfig b/drivers/dma/ti/Kconfig
index dbf168146d35..cbc30ab62783 100644
--- a/drivers/dma/ti/Kconfig
+++ b/drivers/dma/ti/Kconfig
@@ -36,11 +36,12 @@ config DMA_OMAP
 
 config TI_K3_UDMA
 	tristate "Texas Instruments UDMA support"
-	depends on ARCH_K3
+	depends on ARCH_K3 || COMPILE_TEST
 	depends on TI_SCI_PROTOCOL
 	depends on TI_SCI_INTA_IRQCHIP
 	select DMA_ENGINE
 	select DMA_VIRTUAL_CHANNELS
+	select SOC_TI
 	select TI_K3_RINGACC
 	select TI_K3_PSIL
         help
@@ -49,7 +50,7 @@ config TI_K3_UDMA
 
 config TI_K3_UDMA_GLUE_LAYER
 	tristate "Texas Instruments UDMA Glue layer for non DMAengine users"
-	depends on ARCH_K3
+	depends on ARCH_K3 || COMPILE_TEST
 	depends on TI_K3_UDMA
 	help
 	  Say y here to support the K3 NAVSS DMA glue interface
diff --git a/drivers/dma/ti/cppi41.c b/drivers/dma/ti/cppi41.c
index 8d8c3d6038fc..88756dccd62c 100644
--- a/drivers/dma/ti/cppi41.c
+++ b/drivers/dma/ti/cppi41.c
@@ -390,7 +390,6 @@ static int cppi41_dma_alloc_chan_resources(struct dma_chan *chan)
 	if (!c->is_tx)
 		cppi_writel(c->q_num, c->gcr_reg + RXHPCRA0);
 
-	pm_runtime_mark_last_busy(cdd->ddev.dev);
 	pm_runtime_put_autosuspend(cdd->ddev.dev);
 
 	return 0;
@@ -411,7 +410,6 @@ static void cppi41_dma_free_chan_resources(struct dma_chan *chan)
 
 	WARN_ON(!list_empty(&cdd->pending));
 
-	pm_runtime_mark_last_busy(cdd->ddev.dev);
 	pm_runtime_put_autosuspend(cdd->ddev.dev);
 }
 
@@ -509,7 +507,6 @@ static void cppi41_dma_issue_pending(struct dma_chan *chan)
 		cppi41_run_queue(cdd);
 	spin_unlock_irqrestore(&cdd->lock, flags);
 
-	pm_runtime_mark_last_busy(cdd->ddev.dev);
 	pm_runtime_put_autosuspend(cdd->ddev.dev);
 }
 
@@ -627,7 +624,6 @@ static struct dma_async_tx_descriptor *cppi41_dma_prep_slave_sg(
 	txd = &c->txd;
 
 err_out_not_ready:
-	pm_runtime_mark_last_busy(cdd->ddev.dev);
 	pm_runtime_put_autosuspend(cdd->ddev.dev);
 
 	return txd;
@@ -1139,7 +1135,6 @@ static int cppi41_dma_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_of;
 
-	pm_runtime_mark_last_busy(dev);
 	pm_runtime_put_autosuspend(dev);
 
 	return 0;
diff --git a/drivers/dma/ti/dma-crossbar.c b/drivers/dma/ti/dma-crossbar.c
index 7f17ee87a6dc..e04077d542d2 100644
--- a/drivers/dma/ti/dma-crossbar.c
+++ b/drivers/dma/ti/dma-crossbar.c
@@ -79,34 +79,35 @@ static void *ti_am335x_xbar_route_allocate(struct of_phandle_args *dma_spec,
 {
 	struct platform_device *pdev = of_find_device_by_node(ofdma->of_node);
 	struct ti_am335x_xbar_data *xbar = platform_get_drvdata(pdev);
-	struct ti_am335x_xbar_map *map;
+	struct ti_am335x_xbar_map *map = ERR_PTR(-EINVAL);
 
 	if (dma_spec->args_count != 3)
-		return ERR_PTR(-EINVAL);
+		goto out_put_pdev;
 
 	if (dma_spec->args[2] >= xbar->xbar_events) {
 		dev_err(&pdev->dev, "Invalid XBAR event number: %d\n",
 			dma_spec->args[2]);
-		return ERR_PTR(-EINVAL);
+		goto out_put_pdev;
 	}
 
 	if (dma_spec->args[0] >= xbar->dma_requests) {
 		dev_err(&pdev->dev, "Invalid DMA request line number: %d\n",
 			dma_spec->args[0]);
-		return ERR_PTR(-EINVAL);
+		goto out_put_pdev;
 	}
 
 	/* The of_node_put() will be done in the core for the node */
 	dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0);
 	if (!dma_spec->np) {
 		dev_err(&pdev->dev, "Can't get DMA master\n");
-		return ERR_PTR(-EINVAL);
+		goto out_put_pdev;
 	}
 
 	map = kzalloc(sizeof(*map), GFP_KERNEL);
 	if (!map) {
 		of_node_put(dma_spec->np);
-		return ERR_PTR(-ENOMEM);
+		map = ERR_PTR(-ENOMEM);
+		goto out_put_pdev;
 	}
 
 	map->dma_line = (u16)dma_spec->args[0];
@@ -120,6 +121,9 @@ static void *ti_am335x_xbar_route_allocate(struct of_phandle_args *dma_spec,
 
 	ti_am335x_xbar_write(xbar->iomem, map->dma_line, map->mux_val);
 
+out_put_pdev:
+	put_device(&pdev->dev);
+
 	return map;
 }
 
@@ -241,28 +245,26 @@ static void *ti_dra7_xbar_route_allocate(struct of_phandle_args *dma_spec,
 {
 	struct platform_device *pdev = of_find_device_by_node(ofdma->of_node);
 	struct ti_dra7_xbar_data *xbar = platform_get_drvdata(pdev);
-	struct ti_dra7_xbar_map *map;
+	struct ti_dra7_xbar_map *map = ERR_PTR(-EINVAL);
 
 	if (dma_spec->args[0] >= xbar->xbar_requests) {
 		dev_err(&pdev->dev, "Invalid XBAR request number: %d\n",
 			dma_spec->args[0]);
-		put_device(&pdev->dev);
-		return ERR_PTR(-EINVAL);
+		goto out_put_pdev;
 	}
 
 	/* The of_node_put() will be done in the core for the node */
 	dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0);
 	if (!dma_spec->np) {
 		dev_err(&pdev->dev, "Can't get DMA master\n");
-		put_device(&pdev->dev);
-		return ERR_PTR(-EINVAL);
+		goto out_put_pdev;
 	}
 
 	map = kzalloc(sizeof(*map), GFP_KERNEL);
 	if (!map) {
 		of_node_put(dma_spec->np);
-		put_device(&pdev->dev);
-		return ERR_PTR(-ENOMEM);
+		map = ERR_PTR(-ENOMEM);
+		goto out_put_pdev;
 	}
 
 	mutex_lock(&xbar->mutex);
@@ -273,8 +275,8 @@ static void *ti_dra7_xbar_route_allocate(struct of_phandle_args *dma_spec,
 		dev_err(&pdev->dev, "Run out of free DMA requests\n");
 		kfree(map);
 		of_node_put(dma_spec->np);
-		put_device(&pdev->dev);
-		return ERR_PTR(-ENOMEM);
+		map = ERR_PTR(-ENOMEM);
+		goto out_put_pdev;
 	}
 	set_bit(map->xbar_out, xbar->dma_inuse);
 	mutex_unlock(&xbar->mutex);
@@ -288,6 +290,9 @@ static void *ti_dra7_xbar_route_allocate(struct of_phandle_args *dma_spec,
 
 	ti_dra7_xbar_write(xbar->iomem, map->xbar_out, map->xbar_in);
 
+out_put_pdev:
+	put_device(&pdev->dev);
+
 	return map;
 }
 
diff --git a/drivers/dma/ti/k3-udma-private.c b/drivers/dma/ti/k3-udma-private.c
index 05228bf00033..624360423ef1 100644
--- a/drivers/dma/ti/k3-udma-private.c
+++ b/drivers/dma/ti/k3-udma-private.c
@@ -42,9 +42,9 @@ struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property)
 	}
 
 	ud = platform_get_drvdata(pdev);
+	put_device(&pdev->dev);
 	if (!ud) {
 		pr_debug("UDMA has not been probed\n");
-		put_device(&pdev->dev);
 		return ERR_PTR(-EPROBE_DEFER);
 	}
 
diff --git a/drivers/dma/xilinx/xdma-regs.h b/drivers/dma/xilinx/xdma-regs.h
index 6ad08878e938..70bca92621aa 100644
--- a/drivers/dma/xilinx/xdma-regs.h
+++ b/drivers/dma/xilinx/xdma-regs.h
@@ -9,6 +9,7 @@
 
 /* The length of register space exposed to host */
 #define XDMA_REG_SPACE_LEN	65536
+#define XDMA_MAX_REG_OFFSET	(XDMA_REG_SPACE_LEN - 4)
 
 /*
  * maximum number of DMA channels for each direction:
diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c
index 0d88b1a670e1..5ecf8223c112 100644
--- a/drivers/dma/xilinx/xdma.c
+++ b/drivers/dma/xilinx/xdma.c
@@ -38,7 +38,7 @@ static const struct regmap_config xdma_regmap_config = {
 	.reg_bits = 32,
 	.val_bits = 32,
 	.reg_stride = 4,
-	.max_register = XDMA_REG_SPACE_LEN,
+	.max_register = XDMA_MAX_REG_OFFSET,
 };
 
 /**
diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c
index fabff602065f..89a8254d9cdc 100644
--- a/drivers/dma/xilinx/xilinx_dma.c
+++ b/drivers/dma/xilinx/xilinx_dma.c
@@ -131,6 +131,7 @@
 #define XILINX_MCDMA_MAX_CHANS_PER_DEVICE	0x20
 #define XILINX_DMA_MAX_CHANS_PER_DEVICE		0x2
 #define XILINX_CDMA_MAX_CHANS_PER_DEVICE	0x1
+#define XILINX_DMA_DFAULT_ADDRWIDTH		0x20
 
 #define XILINX_DMA_DMAXR_ALL_IRQ_MASK	\
 		(XILINX_DMA_DMASR_FRM_CNT_IRQ | \
@@ -3159,7 +3160,7 @@ static int xilinx_dma_probe(struct platform_device *pdev)
 	struct device_node *node = pdev->dev.of_node;
 	struct xilinx_dma_device *xdev;
 	struct device_node *child, *np = pdev->dev.of_node;
-	u32 num_frames, addr_width, len_width;
+	u32 num_frames, addr_width = XILINX_DMA_DFAULT_ADDRWIDTH, len_width;
 	int i, err;
 
 	/* Allocate and initialize the DMA engine structure */
@@ -3235,7 +3236,9 @@ static int xilinx_dma_probe(struct platform_device *pdev)
 
 	err = of_property_read_u32(node, "xlnx,addrwidth", &addr_width);
 	if (err < 0)
-		dev_warn(xdev->dev, "missing xlnx,addrwidth property\n");
+		dev_warn(xdev->dev,
+			 "missing xlnx,addrwidth property, using default value %d\n",
+			 XILINX_DMA_DFAULT_ADDRWIDTH);
 
 	if (addr_width > 32)
 		xdev->ext_addr = true;
diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c
index f7e584de4335..7bb3716e60da 100644
--- a/drivers/dma/xilinx/zynqmp_dma.c
+++ b/drivers/dma/xilinx/zynqmp_dma.c
@@ -695,7 +695,6 @@ static void zynqmp_dma_free_chan_resources(struct dma_chan *dchan)
 		(2 * ZYNQMP_DMA_DESC_SIZE(chan) * ZYNQMP_DMA_NUM_DESCS),
 		chan->desc_pool_v, chan->desc_pool_p);
 	kfree(chan->sw_desc_pool);
-	pm_runtime_mark_last_busy(chan->dev);
 	pm_runtime_put_autosuspend(chan->dev);
 }
 
@@ -1145,7 +1144,6 @@ static int zynqmp_dma_probe(struct platform_device *pdev)
 		goto free_chan_resources;
 	}
 
-	pm_runtime_mark_last_busy(zdev->dev);
 	pm_runtime_put_sync_autosuspend(zdev->dev);
 
 	return 0;
diff --git a/drivers/extcon/Kconfig b/drivers/extcon/Kconfig
index aec46bf03302..68d9df7d2dae 100644
--- a/drivers/extcon/Kconfig
+++ b/drivers/extcon/Kconfig
@@ -158,6 +158,7 @@ config EXTCON_PTN5150
 	tristate "NXP PTN5150 CC LOGIC USB EXTCON support"
 	depends on I2C && (GPIOLIB || COMPILE_TEST)
 	depends on USB_ROLE_SWITCH || !USB_ROLE_SWITCH
+	depends on TYPEC || !TYPEC
 	select REGMAP_I2C
 	help
 	  Say Y here to enable support for USB peripheral and USB host
diff --git a/drivers/extcon/extcon-intel-int3496.c b/drivers/extcon/extcon-intel-int3496.c
index ded1a85a5549..7d16d5b7d58f 100644
--- a/drivers/extcon/extcon-intel-int3496.c
+++ b/drivers/extcon/extcon-intel-int3496.c
@@ -106,7 +106,7 @@ static irqreturn_t int3496_thread_isr(int irq, void *priv)
 	struct int3496_data *data = priv;
 
 	/* Let the pin settle before processing it */
-	mod_delayed_work(system_wq, &data->work, DEBOUNCE_TIME);
+	mod_delayed_work(system_percpu_wq, &data->work, DEBOUNCE_TIME);
 
 	return IRQ_HANDLED;
 }
@@ -181,7 +181,7 @@ static int int3496_probe(struct platform_device *pdev)
 	}
 
 	/* process id-pin so that we start with the right status */
-	queue_delayed_work(system_wq, &data->work, 0);
+	queue_delayed_work(system_percpu_wq, &data->work, 0);
 	flush_delayed_work(&data->work);
 
 	platform_set_drvdata(pdev, data);
diff --git a/drivers/extcon/extcon-ptn5150.c b/drivers/extcon/extcon-ptn5150.c
index 78ad86c4a3be..eca1b140aeb0 100644
--- a/drivers/extcon/extcon-ptn5150.c
+++ b/drivers/extcon/extcon-ptn5150.c
@@ -18,6 +18,7 @@
 #include <linux/extcon-provider.h>
 #include <linux/gpio/consumer.h>
 #include <linux/usb/role.h>
+#include <linux/usb/typec_mux.h>
 
 /* PTN5150 registers */
 #define PTN5150_REG_DEVICE_ID			0x01
@@ -38,7 +39,11 @@
 #define PTN5150_REG_DEVICE_ID_VERSION		GENMASK(7, 3)
 #define PTN5150_REG_DEVICE_ID_VENDOR		GENMASK(2, 0)
 
+#define PTN5150_POLARITY_CC1			0x1
+#define PTN5150_POLARITY_CC2			0x2
+
 #define PTN5150_REG_CC_PORT_ATTACHMENT		GENMASK(4, 2)
+#define PTN5150_REG_CC_POLARITY			GENMASK(1, 0)
 #define PTN5150_REG_CC_VBUS_DETECTION		BIT(7)
 #define PTN5150_REG_INT_CABLE_ATTACH_MASK	BIT(0)
 #define PTN5150_REG_INT_CABLE_DETACH_MASK	BIT(1)
@@ -53,6 +58,7 @@ struct ptn5150_info {
 	int irq;
 	struct work_struct irq_work;
 	struct mutex mutex;
+	struct typec_switch *orient_sw;
 	struct usb_role_switch *role_sw;
 };
 
@@ -71,6 +77,7 @@ static const struct regmap_config ptn5150_regmap_config = {
 
 static void ptn5150_check_state(struct ptn5150_info *info)
 {
+	enum typec_orientation orient = TYPEC_ORIENTATION_NONE;
 	unsigned int port_status, reg_data, vbus;
 	enum usb_role usb_role = USB_ROLE_NONE;
 	int ret;
@@ -81,6 +88,23 @@ static void ptn5150_check_state(struct ptn5150_info *info)
 		return;
 	}
 
+	orient = FIELD_GET(PTN5150_REG_CC_POLARITY, reg_data);
+	switch (orient) {
+	case PTN5150_POLARITY_CC1:
+		orient = TYPEC_ORIENTATION_NORMAL;
+		break;
+	case PTN5150_POLARITY_CC2:
+		orient = TYPEC_ORIENTATION_REVERSE;
+		break;
+	default:
+		orient = TYPEC_ORIENTATION_NONE;
+		break;
+	}
+
+	ret = typec_switch_set(info->orient_sw, orient);
+	if (ret)
+		dev_err(info->dev, "failed to set orientation: %d\n", ret);
+
 	port_status = FIELD_GET(PTN5150_REG_CC_PORT_ATTACHMENT, reg_data);
 
 	switch (port_status) {
@@ -152,6 +176,12 @@ static void ptn5150_irq_work(struct work_struct *work)
 				dev_err(info->dev,
 					"failed to set none role: %d\n",
 					ret);
+
+			ret = typec_switch_set(info->orient_sw,
+					       TYPEC_ORIENTATION_NONE);
+			if (ret)
+				dev_err(info->dev,
+					"failed to set orientation: %d\n", ret);
 		}
 	}
 
@@ -219,12 +249,14 @@ static void ptn5150_work_sync_and_put(void *data)
 
 	cancel_work_sync(&info->irq_work);
 	usb_role_switch_put(info->role_sw);
+	typec_switch_put(info->orient_sw);
 }
 
 static int ptn5150_i2c_probe(struct i2c_client *i2c)
 {
 	struct device *dev = &i2c->dev;
 	struct device_node *np = i2c->dev.of_node;
+	struct fwnode_handle *connector;
 	struct ptn5150_info *info;
 	int ret;
 
@@ -311,7 +343,17 @@ static int ptn5150_i2c_probe(struct i2c_client *i2c)
 	if (ret)
 		return -EINVAL;
 
+	connector = device_get_named_child_node(dev, "connector");
+	if (connector) {
+		info->orient_sw = fwnode_typec_switch_get(connector);
+		if (IS_ERR(info->orient_sw))
+			return dev_err_probe(info->dev, PTR_ERR(info->orient_sw),
+					"failed to get orientation switch\n");
+	}
+
 	info->role_sw = usb_role_switch_get(info->dev);
+	if (!info->role_sw && connector)
+		info->role_sw = fwnode_usb_role_switch_get(connector);
 	if (IS_ERR(info->role_sw))
 		return dev_err_probe(info->dev, PTR_ERR(info->role_sw),
 				     "failed to get role switch\n");
@@ -331,6 +373,19 @@ static int ptn5150_i2c_probe(struct i2c_client *i2c)
 	return 0;
 }
 
+static int ptn5150_resume(struct device *dev)
+{
+	struct i2c_client *i2c = to_i2c_client(dev);
+	struct ptn5150_info *info = i2c_get_clientdata(i2c);
+
+	/* Need to check possible pending interrupt events */
+	schedule_work(&info->irq_work);
+
+	return 0;
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(ptn5150_pm_ops, NULL, ptn5150_resume);
+
 static const struct of_device_id ptn5150_dt_match[] = {
 	{ .compatible = "nxp,ptn5150" },
 	{ },
@@ -346,6 +401,7 @@ MODULE_DEVICE_TABLE(i2c, ptn5150_i2c_id);
 static struct i2c_driver ptn5150_i2c_driver = {
 	.driver		= {
 		.name	= "ptn5150",
+		.pm = pm_sleep_ptr(&ptn5150_pm_ops),
 		.of_match_table = ptn5150_dt_match,
 	},
 	.probe		= ptn5150_i2c_probe,
diff --git a/drivers/extcon/extcon-usbc-tusb320.c b/drivers/extcon/extcon-usbc-tusb320.c
index 2eab341de6b7..920b03421850 100644
--- a/drivers/extcon/extcon-usbc-tusb320.c
+++ b/drivers/extcon/extcon-usbc-tusb320.c
@@ -454,20 +454,18 @@ static int tusb320_typec_probe(struct i2c_client *client,
 	priv->port_type = priv->cap.type;
 
 	/* This goes into register 0x8 field CURRENT_MODE_ADVERTISE */
-	ret = fwnode_property_read_string(connector, "typec-power-opmode", &cap_str);
-	if (ret)
-		goto err_put;
-
-	ret = typec_find_pwr_opmode(cap_str);
-	if (ret < 0)
-		goto err_put;
-
-	priv->pwr_opmode = ret;
+	if (!fwnode_property_read_string(connector, "typec-power-opmode",
+					 &cap_str)) {
+		ret = typec_find_pwr_opmode(cap_str);
+		if (ret < 0)
+			goto err_put;
+		priv->pwr_opmode = ret;
 
-	/* Initialize the hardware with the devicetree settings. */
-	ret = tusb320_set_adv_pwr_mode(priv);
-	if (ret)
-		goto err_put;
+		/* Initialize the hardware with the devicetree settings. */
+		ret = tusb320_set_adv_pwr_mode(priv);
+		if (ret)
+			goto err_put;
+	}
 
 	priv->cap.revision		= USB_TYPEC_REV_1_1;
 	priv->cap.accessory[0]		= TYPEC_ACCESSORY_AUDIO;
diff --git a/drivers/extcon/extcon.c b/drivers/extcon/extcon.c
index e7f55c021e56..c08f642d7f5e 100644
--- a/drivers/extcon/extcon.c
+++ b/drivers/extcon/extcon.c
@@ -1371,10 +1371,10 @@ void extcon_dev_unregister(struct extcon_dev *edev)
 		return;
 	}
 
-	ida_free(&extcon_dev_ids, edev->id);
-
 	device_unregister(&edev->dev);
 
+	ida_free(&extcon_dev_ids, edev->id);
+
 	if (edev->mutually_exclusive && edev->max_supported) {
 		for (index = 0; edev->mutually_exclusive[index];
 				index++)
diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c
index c72ee4756585..8144f6a9f0e9 100644
--- a/drivers/firmware/arm_ffa/driver.c
+++ b/drivers/firmware/arm_ffa/driver.c
@@ -246,6 +246,11 @@ static int ffa_features(u32 func_feat_id, u32 input_props,
 }
 
 #define PARTITION_INFO_GET_RETURN_COUNT_ONLY	BIT(0)
+#define FFA_SUPPORTS_GET_COUNT_ONLY(version)	((version) > FFA_VERSION_1_0)
+#define FFA_PART_INFO_HAS_SIZE_IN_RESP(version)	((version) > FFA_VERSION_1_0)
+#define FFA_PART_INFO_HAS_UUID_IN_RESP(version)	((version) > FFA_VERSION_1_0)
+#define FFA_PART_INFO_HAS_EXEC_STATE_IN_RESP(version)	\
+	((version) > FFA_VERSION_1_0)
 
 /* buffer must be sizeof(struct ffa_partition_info) * num_partitions */
 static int
@@ -255,7 +260,7 @@ __ffa_partition_info_get(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3,
 	int idx, count, flags = 0, sz, buf_sz;
 	ffa_value_t partition_info;
 
-	if (drv_info->version > FFA_VERSION_1_0 &&
+	if (FFA_SUPPORTS_GET_COUNT_ONLY(drv_info->version) &&
 	    (!buffer || !num_partitions)) /* Just get the count for now */
 		flags = PARTITION_INFO_GET_RETURN_COUNT_ONLY;
 
@@ -273,12 +278,11 @@ __ffa_partition_info_get(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3,
 
 	count = partition_info.a2;
 
-	if (drv_info->version > FFA_VERSION_1_0) {
+	if (FFA_PART_INFO_HAS_SIZE_IN_RESP(drv_info->version)) {
 		buf_sz = sz = partition_info.a3;
 		if (sz > sizeof(*buffer))
 			buf_sz = sizeof(*buffer);
 	} else {
-		/* FFA_VERSION_1_0 lacks size in the response */
 		buf_sz = sz = 8;
 	}
 
@@ -981,10 +985,27 @@ static void __do_sched_recv_cb(u16 part_id, u16 vcpu, bool is_per_vcpu)
 	}
 }
 
+/*
+ * Map logical ID index to the u16 index within the packed ID list.
+ *
+ * For native responses (FF-A width == kernel word size), IDs are
+ * tightly packed: idx -> idx.
+ *
+ * For 32-bit responses on a 64-bit kernel, each 64-bit register
+ * contributes 4 x u16 values but only the lower 2 are defined; the
+ * upper 2 are garbage. This mapping skips those upper halves:
+ *   0,1,2,3,4,5,... -> 0,1,4,5,8,9,...
+ */
+static int list_idx_to_u16_idx(int idx, bool is_native_resp)
+{
+	return is_native_resp ? idx : idx + 2 * (idx >> 1);
+}
+
 static void ffa_notification_info_get(void)
 {
-	int idx, list, max_ids, lists_cnt, ids_processed, ids_count[MAX_IDS_64];
-	bool is_64b_resp;
+	int ids_processed, ids_count[MAX_IDS_64];
+	int idx, list, max_ids, lists_cnt;
+	bool is_64b_resp, is_native_resp;
 	ffa_value_t ret;
 	u64 id_list;
 
@@ -1001,6 +1022,7 @@ static void ffa_notification_info_get(void)
 		}
 
 		is_64b_resp = (ret.a0 == FFA_FN64_SUCCESS);
+		is_native_resp = (ret.a0 == FFA_FN_NATIVE(SUCCESS));
 
 		ids_processed = 0;
 		lists_cnt = FIELD_GET(NOTIFICATION_INFO_GET_ID_COUNT, ret.a2);
@@ -1017,12 +1039,16 @@ static void ffa_notification_info_get(void)
 
 		/* Process IDs */
 		for (list = 0; list < lists_cnt; list++) {
+			int u16_idx;
 			u16 vcpu_id, part_id, *packed_id_list = (u16 *)&ret.a3;
 
 			if (ids_processed >= max_ids - 1)
 				break;
 
-			part_id = packed_id_list[ids_processed++];
+			u16_idx = list_idx_to_u16_idx(ids_processed,
+						      is_native_resp);
+			part_id = packed_id_list[u16_idx];
+			ids_processed++;
 
 			if (ids_count[list] == 1) { /* Global Notification */
 				__do_sched_recv_cb(part_id, 0, false);
@@ -1034,7 +1060,10 @@ static void ffa_notification_info_get(void)
 				if (ids_processed >= max_ids - 1)
 					break;
 
-				vcpu_id = packed_id_list[ids_processed++];
+				u16_idx = list_idx_to_u16_idx(ids_processed,
+							      is_native_resp);
+				vcpu_id = packed_id_list[u16_idx];
+				ids_processed++;
 
 				__do_sched_recv_cb(part_id, vcpu_id, true);
 			}
@@ -1706,7 +1735,7 @@ static int ffa_setup_partitions(void)
 	struct ffa_device *ffa_dev;
 	struct ffa_partition_info *pbuf, *tpbuf;
 
-	if (drv_info->version == FFA_VERSION_1_0) {
+	if (!FFA_PART_INFO_HAS_UUID_IN_RESP(drv_info->version)) {
 		ret = bus_register_notifier(&ffa_bus_type, &ffa_bus_nb);
 		if (ret)
 			pr_err("Failed to register FF-A bus notifiers\n");
@@ -1733,7 +1762,7 @@ static int ffa_setup_partitions(void)
 			continue;
 		}
 
-		if (drv_info->version > FFA_VERSION_1_0 &&
+		if (FFA_PART_INFO_HAS_EXEC_STATE_IN_RESP(drv_info->version) &&
 		    !(tpbuf->properties & FFA_PARTITION_AARCH64_EXEC))
 			ffa_mode_32bit_set(ffa_dev);
 
@@ -2068,6 +2097,7 @@ static int __init ffa_init(void)
 
 	pr_err("failed to setup partitions\n");
 	ffa_notifications_cleanup();
+	ffa_rxtx_unmap(drv_info->vm_id);
 free_pages:
 	if (drv_info->tx_buffer)
 		free_pages_exact(drv_info->tx_buffer, rxtx_bufsz);
diff --git a/drivers/firmware/arm_scmi/perf.c b/drivers/firmware/arm_scmi/perf.c
index 683fd9b85c5c..2249ef7fe790 100644
--- a/drivers/firmware/arm_scmi/perf.c
+++ b/drivers/firmware/arm_scmi/perf.c
@@ -27,7 +27,7 @@
 /* Updated only after ALL the mandatory features for that version are merged */
 #define SCMI_PROTOCOL_SUPPORTED_VERSION		0x40000
 
-#define MAX_OPPS		32
+#define MAX_OPPS		64
 
 enum scmi_performance_protocol_cmd {
 	PERF_DOMAIN_ATTRIBUTES = 0x3,
diff --git a/drivers/firmware/arm_scmi/pinctrl.c b/drivers/firmware/arm_scmi/pinctrl.c
index 3855c98caf06..d18c2d248f04 100644
--- a/drivers/firmware/arm_scmi/pinctrl.c
+++ b/drivers/firmware/arm_scmi/pinctrl.c
@@ -596,11 +596,19 @@ static int scmi_pinctrl_pin_free(const struct scmi_protocol_handle *ph, u32 pin)
 }
 
 static int scmi_pinctrl_get_group_info(const struct scmi_protocol_handle *ph,
-				       u32 selector,
-				       struct scmi_group_info *group)
+				       u32 selector)
 {
+	struct scmi_pinctrl_info *pi = ph->get_priv(ph);
+	struct scmi_group_info *group;
 	int ret;
 
+	if (selector >= pi->nr_groups)
+		return -EINVAL;
+
+	group = &pi->groups[selector];
+	if (group->present)
+		return 0;
+
 	ret = scmi_pinctrl_attributes(ph, GROUP_TYPE, selector, group->name,
 				      &group->nr_pins);
 	if (ret)
@@ -632,21 +640,14 @@ static int scmi_pinctrl_get_group_name(const struct scmi_protocol_handle *ph,
 				       u32 selector, const char **name)
 {
 	struct scmi_pinctrl_info *pi = ph->get_priv(ph);
+	int ret;
 
 	if (!name)
 		return -EINVAL;
 
-	if (selector >= pi->nr_groups || pi->nr_groups == 0)
-		return -EINVAL;
-
-	if (!pi->groups[selector].present) {
-		int ret;
-
-		ret = scmi_pinctrl_get_group_info(ph, selector,
-						  &pi->groups[selector]);
-		if (ret)
-			return ret;
-	}
+	ret = scmi_pinctrl_get_group_info(ph, selector);
+	if (ret)
+		return ret;
 
 	*name = pi->groups[selector].name;
 
@@ -658,21 +659,14 @@ static int scmi_pinctrl_group_pins_get(const struct scmi_protocol_handle *ph,
 				       u32 *nr_pins)
 {
 	struct scmi_pinctrl_info *pi = ph->get_priv(ph);
+	int ret;
 
 	if (!pins || !nr_pins)
 		return -EINVAL;
 
-	if (selector >= pi->nr_groups || pi->nr_groups == 0)
-		return -EINVAL;
-
-	if (!pi->groups[selector].present) {
-		int ret;
-
-		ret = scmi_pinctrl_get_group_info(ph, selector,
-						  &pi->groups[selector]);
-		if (ret)
-			return ret;
-	}
+	ret = scmi_pinctrl_get_group_info(ph, selector);
+	if (ret)
+		return ret;
 
 	*pins = pi->groups[selector].group_pins;
 	*nr_pins = pi->groups[selector].nr_pins;
@@ -681,11 +675,19 @@ static int scmi_pinctrl_group_pins_get(const struct scmi_protocol_handle *ph,
 }
 
 static int scmi_pinctrl_get_function_info(const struct scmi_protocol_handle *ph,
-					  u32 selector,
-					  struct scmi_function_info *func)
+					  u32 selector)
 {
+	struct scmi_pinctrl_info *pi = ph->get_priv(ph);
+	struct scmi_function_info *func;
 	int ret;
 
+	if (selector >= pi->nr_functions)
+		return -EINVAL;
+
+	func = &pi->functions[selector];
+	if (func->present)
+		return 0;
+
 	ret = scmi_pinctrl_attributes(ph, FUNCTION_TYPE, selector, func->name,
 				      &func->nr_groups);
 	if (ret)
@@ -716,21 +718,14 @@ static int scmi_pinctrl_get_function_name(const struct scmi_protocol_handle *ph,
 					  u32 selector, const char **name)
 {
 	struct scmi_pinctrl_info *pi = ph->get_priv(ph);
+	int ret;
 
 	if (!name)
 		return -EINVAL;
 
-	if (selector >= pi->nr_functions || pi->nr_functions == 0)
-		return -EINVAL;
-
-	if (!pi->functions[selector].present) {
-		int ret;
-
-		ret = scmi_pinctrl_get_function_info(ph, selector,
-						     &pi->functions[selector]);
-		if (ret)
-			return ret;
-	}
+	ret = scmi_pinctrl_get_function_info(ph, selector);
+	if (ret)
+		return ret;
 
 	*name = pi->functions[selector].name;
 	return 0;
@@ -742,21 +737,14 @@ scmi_pinctrl_function_groups_get(const struct scmi_protocol_handle *ph,
 				 const u32 **groups)
 {
 	struct scmi_pinctrl_info *pi = ph->get_priv(ph);
+	int ret;
 
 	if (!groups || !nr_groups)
 		return -EINVAL;
 
-	if (selector >= pi->nr_functions || pi->nr_functions == 0)
-		return -EINVAL;
-
-	if (!pi->functions[selector].present) {
-		int ret;
-
-		ret = scmi_pinctrl_get_function_info(ph, selector,
-						     &pi->functions[selector]);
-		if (ret)
-			return ret;
-	}
+	ret = scmi_pinctrl_get_function_info(ph, selector);
+	if (ret)
+		return ret;
 
 	*groups = pi->functions[selector].groups;
 	*nr_groups = pi->functions[selector].nr_groups;
@@ -771,13 +759,19 @@ static int scmi_pinctrl_mux_set(const struct scmi_protocol_handle *ph,
 }
 
 static int scmi_pinctrl_get_pin_info(const struct scmi_protocol_handle *ph,
-				     u32 selector, struct scmi_pin_info *pin)
+				     u32 selector)
 {
+	struct scmi_pinctrl_info *pi = ph->get_priv(ph);
+	struct scmi_pin_info *pin;
 	int ret;
 
-	if (!pin)
+	if (selector >= pi->nr_pins)
 		return -EINVAL;
 
+	pin = &pi->pins[selector];
+	if (pin->present)
+		return 0;
+
 	ret = scmi_pinctrl_attributes(ph, PIN_TYPE, selector, pin->name, NULL);
 	if (ret)
 		return ret;
@@ -790,20 +784,14 @@ static int scmi_pinctrl_get_pin_name(const struct scmi_protocol_handle *ph,
 				     u32 selector, const char **name)
 {
 	struct scmi_pinctrl_info *pi = ph->get_priv(ph);
+	int ret;
 
 	if (!name)
 		return -EINVAL;
 
-	if (selector >= pi->nr_pins)
-		return -EINVAL;
-
-	if (!pi->pins[selector].present) {
-		int ret;
-
-		ret = scmi_pinctrl_get_pin_info(ph, selector, &pi->pins[selector]);
-		if (ret)
-			return ret;
-	}
+	ret = scmi_pinctrl_get_pin_info(ph, selector);
+	if (ret)
+		return ret;
 
 	*name = pi->pins[selector].name;
 
diff --git a/drivers/firmware/arm_scmi/reset.c b/drivers/firmware/arm_scmi/reset.c
index 0aa82b96f41b..7c4550a3e258 100644
--- a/drivers/firmware/arm_scmi/reset.c
+++ b/drivers/firmware/arm_scmi/reset.c
@@ -98,6 +98,17 @@ static int scmi_reset_attributes_get(const struct scmi_protocol_handle *ph,
 	return ret;
 }
 
+static struct reset_dom_info *
+scmi_reset_domain_lookup(const struct scmi_protocol_handle *ph, u32 domain)
+{
+	struct scmi_reset_info *pi = ph->get_priv(ph);
+
+	if (domain >= pi->num_domains)
+		return ERR_PTR(-EINVAL);
+
+	return pi->dom_info + domain;
+}
+
 static int
 scmi_reset_domain_attributes_get(const struct scmi_protocol_handle *ph,
 				 struct scmi_reset_info *pinfo,
@@ -156,20 +167,25 @@ static int scmi_reset_num_domains_get(const struct scmi_protocol_handle *ph)
 static const char *
 scmi_reset_name_get(const struct scmi_protocol_handle *ph, u32 domain)
 {
-	struct scmi_reset_info *pi = ph->get_priv(ph);
+	struct reset_dom_info *dom_info;
 
-	struct reset_dom_info *dom = pi->dom_info + domain;
+	dom_info = scmi_reset_domain_lookup(ph, domain);
+	if (IS_ERR(dom_info))
+		return "unknown";
 
-	return dom->name;
+	return dom_info->name;
 }
 
 static int scmi_reset_latency_get(const struct scmi_protocol_handle *ph,
 				  u32 domain)
 {
-	struct scmi_reset_info *pi = ph->get_priv(ph);
-	struct reset_dom_info *dom = pi->dom_info + domain;
+	struct reset_dom_info *dom_info;
+
+	dom_info = scmi_reset_domain_lookup(ph, domain);
+	if (IS_ERR(dom_info))
+		return PTR_ERR(dom_info);
 
-	return dom->latency_us;
+	return dom_info->latency_us;
 }
 
 static int scmi_domain_reset(const struct scmi_protocol_handle *ph, u32 domain,
@@ -178,14 +194,13 @@ static int scmi_domain_reset(const struct scmi_protocol_handle *ph, u32 domain,
 	int ret;
 	struct scmi_xfer *t;
 	struct scmi_msg_reset_domain_reset *dom;
-	struct scmi_reset_info *pi = ph->get_priv(ph);
-	struct reset_dom_info *rdom;
+	struct reset_dom_info *dom_info;
 
-	if (domain >= pi->num_domains)
-		return -EINVAL;
+	dom_info = scmi_reset_domain_lookup(ph, domain);
+	if (IS_ERR(dom_info))
+		return PTR_ERR(dom_info);
 
-	rdom = pi->dom_info + domain;
-	if (rdom->async_reset && flags & AUTONOMOUS_RESET)
+	if (dom_info->async_reset && flags & AUTONOMOUS_RESET)
 		flags |= ASYNCHRONOUS_RESET;
 
 	ret = ph->xops->xfer_get_init(ph, RESET, sizeof(*dom), 0, &t);
@@ -238,15 +253,16 @@ static const struct scmi_reset_proto_ops reset_proto_ops = {
 static bool scmi_reset_notify_supported(const struct scmi_protocol_handle *ph,
 					u8 evt_id, u32 src_id)
 {
-	struct reset_dom_info *dom;
-	struct scmi_reset_info *pi = ph->get_priv(ph);
+	struct reset_dom_info *dom_info;
 
-	if (evt_id != SCMI_EVENT_RESET_ISSUED || src_id >= pi->num_domains)
+	if (evt_id != SCMI_EVENT_RESET_ISSUED)
 		return false;
 
-	dom = pi->dom_info + src_id;
+	dom_info = scmi_reset_domain_lookup(ph, src_id);
+	if (IS_ERR(dom_info))
+		return false;
 
-	return dom->reset_notify;
+	return dom_info->reset_notify;
 }
 
 static int scmi_reset_notify(const struct scmi_protocol_handle *ph,
diff --git a/drivers/firmware/arm_scmi/shmem.c b/drivers/firmware/arm_scmi/shmem.c
index 11c347bff766..dadb37557f8a 100644
--- a/drivers/firmware/arm_scmi/shmem.c
+++ b/drivers/firmware/arm_scmi/shmem.c
@@ -196,7 +196,6 @@ static void __iomem *shmem_setup_iomap(struct scmi_chan_info *cinfo,
 				       struct resource *res,
 				       struct scmi_shmem_io_ops **ops)
 {
-	struct device_node *shmem __free(device_node);
 	const char *desc = tx ? "Tx" : "Rx";
 	int ret, idx = tx ? 0 : 1;
 	struct device *cdev = cinfo->dev;
@@ -205,7 +204,9 @@ static void __iomem *shmem_setup_iomap(struct scmi_chan_info *cinfo,
 	void __iomem *addr;
 	u32 reg_io_width;
 
-	shmem = of_parse_phandle(cdev->of_node, "shmem", idx);
+	struct device_node *shmem __free(device_node) = of_parse_phandle(cdev->of_node,
+		"shmem", idx);
+
 	if (!shmem)
 		return IOMEM_ERR_PTR(-ENODEV);
 
diff --git a/drivers/firmware/broadcom/bcm47xx_sprom.c b/drivers/firmware/broadcom/bcm47xx_sprom.c
index fdcd3a07abcd..bca03fd85808 100644
--- a/drivers/firmware/broadcom/bcm47xx_sprom.c
+++ b/drivers/firmware/broadcom/bcm47xx_sprom.c
@@ -30,6 +30,7 @@
 #include <linux/bcm47xx_sprom.h>
 #include <linux/bcma/bcma.h>
 #include <linux/etherdevice.h>
+#include <linux/hex.h>
 #include <linux/if_ether.h>
 #include <linux/ssb/ssb.h>
 
diff --git a/drivers/firmware/cirrus/cs_dsp.c b/drivers/firmware/cirrus/cs_dsp.c
index 525ac0f0a75d..d35d0f5ccaf7 100644
--- a/drivers/firmware/cirrus/cs_dsp.c
+++ b/drivers/firmware/cirrus/cs_dsp.c
@@ -375,18 +375,23 @@ static ssize_t cs_dsp_debugfs_string_read(struct cs_dsp *dsp,
 					  size_t count, loff_t *ppos,
 					  const char **pstr)
 {
-	const char *str __free(kfree) = NULL;
+	const char *str;
+	ssize_t ret = 0;
 
 	scoped_guard(mutex, &dsp->pwr_lock) {
-		if (!*pstr)
-			return 0;
-
-		str = kasprintf(GFP_KERNEL, "%s\n", *pstr);
-		if (!str)
-			return -ENOMEM;
-
-		return simple_read_from_buffer(user_buf, count, ppos, str, strlen(str));
+		if (*pstr) {
+			str = kasprintf(GFP_KERNEL, "%s\n", *pstr);
+			if (str) {
+				ret = simple_read_from_buffer(user_buf, count,
+							      ppos, str, strlen(str));
+				kfree(str);
+			} else {
+				ret = -ENOMEM;
+			}
+		}
 	}
+
+	return ret;
 }
 
 static ssize_t cs_dsp_debugfs_wmfw_read(struct file *file,
@@ -1446,7 +1451,7 @@ static int cs_dsp_load(struct cs_dsp *dsp, const struct firmware *firmware,
 	const struct wmfw_region *region;
 	const struct cs_dsp_region *mem;
 	const char *region_name;
-	u8 *buf __free(kfree) = NULL;
+	u8 *buf = NULL;
 	size_t buf_len = 0;
 	size_t region_len;
 	unsigned int reg;
@@ -1601,6 +1606,8 @@ static int cs_dsp_load(struct cs_dsp *dsp, const struct firmware *firmware,
 
 	ret = 0;
 out_fw:
+	kfree(buf);
+
 	if (ret == -EOVERFLOW)
 		cs_dsp_err(dsp, "%s: file content overflows file data\n", file);
 
@@ -2132,7 +2139,7 @@ static int cs_dsp_load_coeff(struct cs_dsp *dsp, const struct firmware *firmware
 	struct cs_dsp_alg_region *alg_region;
 	const char *region_name;
 	int ret, pos, blocks, type, offset, reg, version;
-	u8 *buf __free(kfree) = NULL;
+	u8 *buf = NULL;
 	size_t buf_len = 0;
 	size_t region_len;
 
@@ -2311,6 +2318,8 @@ static int cs_dsp_load_coeff(struct cs_dsp *dsp, const struct firmware *firmware
 
 	ret = 0;
 out_fw:
+	kfree(buf);
+
 	if (ret == -EOVERFLOW)
 		cs_dsp_err(dsp, "%s: file content overflows file data\n", file);
 
diff --git a/drivers/firmware/efi/earlycon.c b/drivers/firmware/efi/earlycon.c
index d18a1a5de144..3d060d59968c 100644
--- a/drivers/firmware/efi/earlycon.c
+++ b/drivers/firmware/efi/earlycon.c
@@ -9,7 +9,7 @@
 #include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/serial_core.h>
-#include <linux/screen_info.h>
+#include <linux/sysfb.h>
 #include <linux/string.h>
 
 #include <asm/early_ioremap.h>
@@ -32,12 +32,13 @@ static void *efi_fb;
  */
 static int __init efi_earlycon_remap_fb(void)
 {
+	const struct screen_info *si = &sysfb_primary_display.screen;
+
 	/* bail if there is no bootconsole or it was unregistered already */
 	if (!earlycon_console || !console_is_registered(earlycon_console))
 		return 0;
 
-	efi_fb = memremap(fb_base, screen_info.lfb_size,
-			  fb_wb ? MEMREMAP_WB : MEMREMAP_WC);
+	efi_fb = memremap(fb_base, si->lfb_size, fb_wb ? MEMREMAP_WB : MEMREMAP_WC);
 
 	return efi_fb ? 0 : -ENOMEM;
 }
@@ -71,12 +72,12 @@ static __ref void efi_earlycon_unmap(void *addr, unsigned long len)
 	early_memunmap(addr, len);
 }
 
-static void efi_earlycon_clear_scanline(unsigned int y)
+static void efi_earlycon_clear_scanline(unsigned int y, const struct screen_info *si)
 {
 	unsigned long *dst;
 	u16 len;
 
-	len = screen_info.lfb_linelength;
+	len = si->lfb_linelength;
 	dst = efi_earlycon_map(y*len, len);
 	if (!dst)
 		return;
@@ -85,7 +86,7 @@ static void efi_earlycon_clear_scanline(unsigned int y)
 	efi_earlycon_unmap(dst, len);
 }
 
-static void efi_earlycon_scroll_up(void)
+static void efi_earlycon_scroll_up(const struct screen_info *si)
 {
 	unsigned long *dst, *src;
 	u16 maxlen = 0;
@@ -99,8 +100,8 @@ static void efi_earlycon_scroll_up(void)
 	}
 	maxlen *= 4;
 
-	len = screen_info.lfb_linelength;
-	height = screen_info.lfb_height;
+	len = si->lfb_linelength;
+	height = si->lfb_height;
 
 	for (i = 0; i < height - font->height; i++) {
 		dst = efi_earlycon_map(i*len, len);
@@ -120,7 +121,8 @@ static void efi_earlycon_scroll_up(void)
 	}
 }
 
-static void efi_earlycon_write_char(u32 *dst, unsigned char c, unsigned int h)
+static void efi_earlycon_write_char(u32 *dst, unsigned char c, unsigned int h,
+				    const struct screen_info *si)
 {
 	const u32 color_black = 0x00000000;
 	const u32 color_white = 0x00ffffff;
@@ -145,13 +147,12 @@ static void efi_earlycon_write_char(u32 *dst, unsigned char c, unsigned int h)
 static void
 efi_earlycon_write(struct console *con, const char *str, unsigned int num)
 {
-	struct screen_info *si;
+	const struct screen_info *si = &sysfb_primary_display.screen;
 	u32 cur_efi_x = efi_x;
 	unsigned int len;
 	const char *s;
 	void *dst;
 
-	si = &screen_info;
 	len = si->lfb_linelength;
 
 	while (num) {
@@ -174,7 +175,7 @@ efi_earlycon_write(struct console *con, const char *str, unsigned int num)
 			x = efi_x;
 
 			while (n-- > 0) {
-				efi_earlycon_write_char(dst + x*4, *s, h);
+				efi_earlycon_write_char(dst + x * 4, *s, h, si);
 				x += font->width;
 				s++;
 			}
@@ -207,10 +208,10 @@ efi_earlycon_write(struct console *con, const char *str, unsigned int num)
 			cur_line_y = (cur_line_y + 1) % max_line_y;
 
 			efi_y -= font->height;
-			efi_earlycon_scroll_up();
+			efi_earlycon_scroll_up(si);
 
 			for (i = 0; i < font->height; i++)
-				efi_earlycon_clear_scanline(efi_y + i);
+				efi_earlycon_clear_scanline(efi_y + i, si);
 		}
 	}
 }
@@ -226,22 +227,21 @@ void __init efi_earlycon_reprobe(void)
 static int __init efi_earlycon_setup(struct earlycon_device *device,
 				     const char *opt)
 {
-	struct screen_info *si;
+	const struct screen_info *si = &sysfb_primary_display.screen;
 	u16 xres, yres;
 	u32 i;
 
 	fb_wb = opt && !strcmp(opt, "ram");
 
-	if (screen_info.orig_video_isVGA != VIDEO_TYPE_EFI) {
+	if (si->orig_video_isVGA != VIDEO_TYPE_EFI) {
 		fb_probed = true;
 		return -ENODEV;
 	}
 
-	fb_base = screen_info.lfb_base;
-	if (screen_info.capabilities & VIDEO_CAPABILITY_64BIT_BASE)
-		fb_base |= (u64)screen_info.ext_lfb_base << 32;
+	fb_base = si->lfb_base;
+	if (si->capabilities & VIDEO_CAPABILITY_64BIT_BASE)
+		fb_base |= (u64)si->ext_lfb_base << 32;
 
-	si = &screen_info;
 	xres = si->lfb_width;
 	yres = si->lfb_height;
 
@@ -266,7 +266,7 @@ static int __init efi_earlycon_setup(struct earlycon_device *device,
 
 	efi_y -= font->height;
 	for (i = 0; i < (yres - efi_y) / font->height; i++)
-		efi_earlycon_scroll_up();
+		efi_earlycon_scroll_up(si);
 
 	device->con->write = efi_earlycon_write;
 	earlycon_console = device->con;
diff --git a/drivers/firmware/efi/efi-init.c b/drivers/firmware/efi/efi-init.c
index a65c2d5b9e7b..432301dce76f 100644
--- a/drivers/firmware/efi/efi-init.c
+++ b/drivers/firmware/efi/efi-init.c
@@ -19,11 +19,11 @@
 #include <linux/of_address.h>
 #include <linux/of_fdt.h>
 #include <linux/platform_device.h>
-#include <linux/screen_info.h>
+#include <linux/sysfb.h>
 
 #include <asm/efi.h>
 
-unsigned long __initdata screen_info_table = EFI_INVALID_TABLE_ADDR;
+unsigned long __initdata primary_display_table = EFI_INVALID_TABLE_ADDR;
 
 static int __init is_memory(efi_memory_desc_t *md)
 {
@@ -57,31 +57,31 @@ static phys_addr_t __init efi_to_phys(unsigned long addr)
 extern __weak const efi_config_table_type_t efi_arch_tables[];
 
 /*
- * x86 defines its own screen_info and uses it even without EFI,
- * everything else can get it from here.
+ * x86 defines its own instance of sysfb_primary_display and uses
+ * it even without EFI, everything else can get them from here.
  */
 #if !defined(CONFIG_X86) && (defined(CONFIG_SYSFB) || defined(CONFIG_EFI_EARLYCON))
-struct screen_info screen_info __section(".data");
-EXPORT_SYMBOL_GPL(screen_info);
+struct sysfb_display_info sysfb_primary_display __section(".data");
+EXPORT_SYMBOL_GPL(sysfb_primary_display);
 #endif
 
-static void __init init_screen_info(void)
+static void __init init_primary_display(void)
 {
-	struct screen_info *si;
+	struct sysfb_display_info *dpy;
 
-	if (screen_info_table != EFI_INVALID_TABLE_ADDR) {
-		si = early_memremap(screen_info_table, sizeof(*si));
-		if (!si) {
-			pr_err("Could not map screen_info config table\n");
+	if (primary_display_table != EFI_INVALID_TABLE_ADDR) {
+		dpy = early_memremap(primary_display_table, sizeof(*dpy));
+		if (!dpy) {
+			pr_err("Could not map primary_display config table\n");
 			return;
 		}
-		screen_info = *si;
-		memset(si, 0, sizeof(*si));
-		early_memunmap(si, sizeof(*si));
+		sysfb_primary_display = *dpy;
+		memset(dpy, 0, sizeof(*dpy));
+		early_memunmap(dpy, sizeof(*dpy));
 
-		if (memblock_is_map_memory(screen_info.lfb_base))
-			memblock_mark_nomap(screen_info.lfb_base,
-					    screen_info.lfb_size);
+		if (memblock_is_map_memory(sysfb_primary_display.screen.lfb_base))
+			memblock_mark_nomap(sysfb_primary_display.screen.lfb_base,
+					    sysfb_primary_display.screen.lfb_size);
 
 		if (IS_ENABLED(CONFIG_EFI_EARLYCON))
 			efi_earlycon_reprobe();
@@ -274,5 +274,5 @@ void __init efi_init(void)
 	if (IS_ENABLED(CONFIG_X86) ||
 	    IS_ENABLED(CONFIG_SYSFB) ||
 	    IS_ENABLED(CONFIG_EFI_EARLYCON))
-		init_screen_info();
+		init_primary_display();
 }
diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c
index a9070d00b833..955193691f10 100644
--- a/drivers/firmware/efi/efi.c
+++ b/drivers/firmware/efi/efi.c
@@ -63,7 +63,7 @@ static unsigned long __initdata mem_reserve = EFI_INVALID_TABLE_ADDR;
 static unsigned long __initdata rt_prop = EFI_INVALID_TABLE_ADDR;
 static unsigned long __initdata initrd = EFI_INVALID_TABLE_ADDR;
 
-extern unsigned long screen_info_table;
+extern unsigned long primary_display_table;
 
 struct mm_struct efi_mm = {
 	.mm_mt			= MTREE_INIT_EXT(mm_mt, MM_MT_FLAGS, efi_mm.mmap_lock),
@@ -641,7 +641,7 @@ static const efi_config_table_type_t common_tables[] __initconst = {
 	{LINUX_EFI_UNACCEPTED_MEM_TABLE_GUID,	&efi.unaccepted,	"Unaccepted"	},
 #endif
 #ifdef CONFIG_EFI_GENERIC_STUB
-	{LINUX_EFI_SCREEN_INFO_TABLE_GUID,	&screen_info_table			},
+	{LINUX_EFI_PRIMARY_DISPLAY_TABLE_GUID,	&primary_display_table			},
 #endif
 	{},
 };
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
index 7d15a85d579f..e386ffd009b7 100644
--- a/drivers/firmware/efi/libstub/Makefile
+++ b/drivers/firmware/efi/libstub/Makefile
@@ -80,7 +80,7 @@ $(obj)/lib-%.o: $(srctree)/lib/%.c FORCE
 	$(call if_changed_rule,cc_o_c)
 
 lib-$(CONFIG_EFI_GENERIC_STUB)	+= efi-stub.o string.o intrinsics.o systable.o \
-				   screen_info.o efi-stub-entry.o
+				   primary_display.o efi-stub-entry.o
 
 lib-$(CONFIG_ARM)		+= arm32-stub.o
 lib-$(CONFIG_ARM64)		+= kaslr.o arm64.o arm64-stub.o smbios.o
diff --git a/drivers/firmware/efi/libstub/efi-stub-entry.c b/drivers/firmware/efi/libstub/efi-stub-entry.c
index a6c049835190..aa85e910fe59 100644
--- a/drivers/firmware/efi/libstub/efi-stub-entry.c
+++ b/drivers/firmware/efi/libstub/efi-stub-entry.c
@@ -1,23 +1,28 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
 #include <linux/efi.h>
-#include <linux/screen_info.h>
+#include <linux/sysfb.h>
 
 #include <asm/efi.h>
 
 #include "efistub.h"
 
-static unsigned long screen_info_offset;
+static unsigned long kernel_image_offset;
 
-struct screen_info *alloc_screen_info(void)
+static void *kernel_image_addr(void *addr)
+{
+	return addr + kernel_image_offset;
+}
+
+struct sysfb_display_info *alloc_primary_display(void)
 {
 	if (IS_ENABLED(CONFIG_ARM))
-		return __alloc_screen_info();
+		return __alloc_primary_display();
 
 	if (IS_ENABLED(CONFIG_X86) ||
 	    IS_ENABLED(CONFIG_EFI_EARLYCON) ||
 	    IS_ENABLED(CONFIG_SYSFB))
-		return (void *)&screen_info + screen_info_offset;
+		return kernel_image_addr(&sysfb_primary_display);
 
 	return NULL;
 }
@@ -73,7 +78,7 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle,
 		return status;
 	}
 
-	screen_info_offset = image_addr - (unsigned long)image->image_base;
+	kernel_image_offset = image_addr - (unsigned long)image->image_base;
 
 	status = efi_stub_common(handle, image, image_addr, cmdline_ptr);
 
diff --git a/drivers/firmware/efi/libstub/efi-stub.c b/drivers/firmware/efi/libstub/efi-stub.c
index 9cb814c5ba1b..42d6073bcd06 100644
--- a/drivers/firmware/efi/libstub/efi-stub.c
+++ b/drivers/firmware/efi/libstub/efi-stub.c
@@ -10,7 +10,7 @@
  */
 
 #include <linux/efi.h>
-#include <linux/screen_info.h>
+#include <linux/sysfb.h>
 #include <asm/efi.h>
 
 #include "efistub.h"
@@ -48,23 +48,33 @@
 static u64 virtmap_base = EFI_RT_VIRTUAL_BASE;
 static bool flat_va_mapping = (EFI_RT_VIRTUAL_OFFSET != 0);
 
-void __weak free_screen_info(struct screen_info *si)
-{
-}
+void __weak free_primary_display(struct sysfb_display_info *dpy)
+{ }
 
-static struct screen_info *setup_graphics(void)
+static struct sysfb_display_info *setup_primary_display(void)
 {
-	struct screen_info *si, tmp = {};
+	struct sysfb_display_info *dpy;
+	struct screen_info *screen = NULL;
+	struct edid_info *edid = NULL;
+	efi_status_t status;
 
-	if (efi_setup_graphics(&tmp, NULL) != EFI_SUCCESS)
+	dpy = alloc_primary_display();
+	if (!dpy)
 		return NULL;
+	screen = &dpy->screen;
+#if defined(CONFIG_FIRMWARE_EDID)
+	edid = &dpy->edid;
+#endif
 
-	si = alloc_screen_info();
-	if (!si)
-		return NULL;
+	status = efi_setup_graphics(screen, edid);
+	if (status != EFI_SUCCESS)
+		goto err_free_primary_display;
 
-	*si = tmp;
-	return si;
+	return dpy;
+
+err_free_primary_display:
+	free_primary_display(dpy);
+	return NULL;
 }
 
 static void install_memreserve_table(void)
@@ -145,14 +155,14 @@ efi_status_t efi_stub_common(efi_handle_t handle,
 			     unsigned long image_addr,
 			     char *cmdline_ptr)
 {
-	struct screen_info *si;
+	struct sysfb_display_info *dpy;
 	efi_status_t status;
 
 	status = check_platform_features();
 	if (status != EFI_SUCCESS)
 		return status;
 
-	si = setup_graphics();
+	dpy = setup_primary_display();
 
 	efi_retrieve_eventlog();
 
@@ -172,7 +182,8 @@ efi_status_t efi_stub_common(efi_handle_t handle,
 
 	status = efi_boot_kernel(handle, image, image_addr, cmdline_ptr);
 
-	free_screen_info(si);
+	free_primary_display(dpy);
+
 	return status;
 }
 
diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h
index b2fb0c3fa721..979a21818cc1 100644
--- a/drivers/firmware/efi/libstub/efistub.h
+++ b/drivers/firmware/efi/libstub/efistub.h
@@ -36,6 +36,7 @@
 
 struct edid_info;
 struct screen_info;
+struct sysfb_display_info;
 
 extern bool efi_no5lvl;
 extern bool efi_nochunk;
@@ -1175,9 +1176,9 @@ efi_enable_reset_attack_mitigation(void) { }
 
 void efi_retrieve_eventlog(void);
 
-struct screen_info *alloc_screen_info(void);
-struct screen_info *__alloc_screen_info(void);
-void free_screen_info(struct screen_info *si);
+struct sysfb_display_info *alloc_primary_display(void);
+struct sysfb_display_info *__alloc_primary_display(void);
+void free_primary_display(struct sysfb_display_info *dpy);
 
 void efi_cache_sync_image(unsigned long image_base,
 			  unsigned long alloc_size);
diff --git a/drivers/firmware/efi/libstub/primary_display.c b/drivers/firmware/efi/libstub/primary_display.c
new file mode 100644
index 000000000000..cdaebab26514
--- /dev/null
+++ b/drivers/firmware/efi/libstub/primary_display.c
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/efi.h>
+#include <linux/sysfb.h>
+
+#include <asm/efi.h>
+
+#include "efistub.h"
+
+/*
+ * There are two ways of populating the core kernel's sysfb_primary_display
+ * via the stub:
+ *
+ *   - using a configuration table, which relies on the EFI init code to
+ *     locate the table and copy the contents; or
+ *
+ *   - by linking directly to the core kernel's copy of the global symbol.
+ *
+ * The latter is preferred because it makes the EFIFB earlycon available very
+ * early, but it only works if the EFI stub is part of the core kernel image
+ * itself. The zboot decompressor can only use the configuration table
+ * approach.
+ */
+
+static efi_guid_t primary_display_guid = LINUX_EFI_PRIMARY_DISPLAY_TABLE_GUID;
+
+struct sysfb_display_info *__alloc_primary_display(void)
+{
+	struct sysfb_display_info *dpy;
+	efi_status_t status;
+
+	status = efi_bs_call(allocate_pool, EFI_ACPI_RECLAIM_MEMORY,
+			     sizeof(*dpy), (void **)&dpy);
+
+	if (status != EFI_SUCCESS)
+		return NULL;
+
+	memset(dpy, 0, sizeof(*dpy));
+
+	status = efi_bs_call(install_configuration_table,
+			     &primary_display_guid, dpy);
+	if (status == EFI_SUCCESS)
+		return dpy;
+
+	efi_bs_call(free_pool, dpy);
+	return NULL;
+}
+
+void free_primary_display(struct sysfb_display_info *dpy)
+{
+	if (!dpy)
+		return;
+
+	efi_bs_call(install_configuration_table, &primary_display_guid, NULL);
+	efi_bs_call(free_pool, dpy);
+}
diff --git a/drivers/firmware/efi/libstub/screen_info.c b/drivers/firmware/efi/libstub/screen_info.c
deleted file mode 100644
index 5d3a1e32d177..000000000000
--- a/drivers/firmware/efi/libstub/screen_info.c
+++ /dev/null
@@ -1,53 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-#include <linux/efi.h>
-#include <linux/screen_info.h>
-
-#include <asm/efi.h>
-
-#include "efistub.h"
-
-/*
- * There are two ways of populating the core kernel's struct screen_info via the stub:
- * - using a configuration table, like below, which relies on the EFI init code
- *   to locate the table and copy the contents;
- * - by linking directly to the core kernel's copy of the global symbol.
- *
- * The latter is preferred because it makes the EFIFB earlycon available very
- * early, but it only works if the EFI stub is part of the core kernel image
- * itself. The zboot decompressor can only use the configuration table
- * approach.
- */
-
-static efi_guid_t screen_info_guid = LINUX_EFI_SCREEN_INFO_TABLE_GUID;
-
-struct screen_info *__alloc_screen_info(void)
-{
-	struct screen_info *si;
-	efi_status_t status;
-
-	status = efi_bs_call(allocate_pool, EFI_ACPI_RECLAIM_MEMORY,
-			     sizeof(*si), (void **)&si);
-
-	if (status != EFI_SUCCESS)
-		return NULL;
-
-	memset(si, 0, sizeof(*si));
-
-	status = efi_bs_call(install_configuration_table,
-			     &screen_info_guid, si);
-	if (status == EFI_SUCCESS)
-		return si;
-
-	efi_bs_call(free_pool, si);
-	return NULL;
-}
-
-void free_screen_info(struct screen_info *si)
-{
-	if (!si)
-		return;
-
-	efi_bs_call(install_configuration_table, &screen_info_guid, NULL);
-	efi_bs_call(free_pool, si);
-}
diff --git a/drivers/firmware/efi/libstub/zboot.c b/drivers/firmware/efi/libstub/zboot.c
index c47ace06f010..4b76f74c56da 100644
--- a/drivers/firmware/efi/libstub/zboot.c
+++ b/drivers/firmware/efi/libstub/zboot.c
@@ -26,9 +26,9 @@ void __weak efi_cache_sync_image(unsigned long image_base,
 	// executable code loaded into memory to be safe for execution.
 }
 
-struct screen_info *alloc_screen_info(void)
+struct sysfb_display_info *alloc_primary_display(void)
 {
-	return __alloc_screen_info();
+	return __alloc_primary_display();
 }
 
 asmlinkage efi_status_t __efiapi
diff --git a/drivers/firmware/efi/sysfb_efi.c b/drivers/firmware/efi/sysfb_efi.c
index 1e509595ac03..4c3986ddcd54 100644
--- a/drivers/firmware/efi/sysfb_efi.c
+++ b/drivers/firmware/efi/sysfb_efi.c
@@ -92,7 +92,7 @@ void efifb_setup_from_dmi(struct screen_info *si, const char *opt)
 	})
 
 #ifdef CONFIG_EFI
-static int __init efifb_set_system(const struct dmi_system_id *id)
+static int __init efifb_set_system(struct screen_info *si, const struct dmi_system_id *id)
 {
 	struct efifb_dmi_info *info = id->driver_data;
 
@@ -101,14 +101,14 @@ static int __init efifb_set_system(const struct dmi_system_id *id)
 		return 0;
 
 	/* Trust the bootloader over the DMI tables */
-	if (screen_info.lfb_base == 0) {
+	if (si->lfb_base == 0) {
 #if defined(CONFIG_PCI)
 		struct pci_dev *dev = NULL;
 		int found_bar = 0;
 #endif
 		if (info->base) {
-			screen_info.lfb_base = choose_value(info->base,
-				screen_info.lfb_base, OVERRIDE_BASE,
+			si->lfb_base = choose_value(info->base,
+				si->lfb_base, OVERRIDE_BASE,
 				info->flags);
 
 #if defined(CONFIG_PCI)
@@ -135,49 +135,53 @@ static int __init efifb_set_system(const struct dmi_system_id *id)
 
 					start = pci_resource_start(dev, i);
 					end = pci_resource_end(dev, i);
-					if (screen_info.lfb_base >= start &&
-					    screen_info.lfb_base < end) {
+					if (si->lfb_base >= start && si->lfb_base < end) {
 						found_bar = 1;
 						break;
 					}
 				}
 			}
 			if (!found_bar)
-				screen_info.lfb_base = 0;
+				si->lfb_base = 0;
 #endif
 		}
 	}
-	if (screen_info.lfb_base) {
-		screen_info.lfb_linelength = choose_value(info->stride,
-			screen_info.lfb_linelength, OVERRIDE_STRIDE,
+	if (si->lfb_base) {
+		si->lfb_linelength = choose_value(info->stride,
+			si->lfb_linelength, OVERRIDE_STRIDE,
 			info->flags);
-		screen_info.lfb_width = choose_value(info->width,
-			screen_info.lfb_width, OVERRIDE_WIDTH,
+		si->lfb_width = choose_value(info->width,
+			si->lfb_width, OVERRIDE_WIDTH,
 			info->flags);
-		screen_info.lfb_height = choose_value(info->height,
-			screen_info.lfb_height, OVERRIDE_HEIGHT,
+		si->lfb_height = choose_value(info->height,
+			si->lfb_height, OVERRIDE_HEIGHT,
 			info->flags);
-		if (screen_info.orig_video_isVGA == 0)
-			screen_info.orig_video_isVGA = VIDEO_TYPE_EFI;
+		if (si->orig_video_isVGA == 0)
+			si->orig_video_isVGA = VIDEO_TYPE_EFI;
 	} else {
-		screen_info.lfb_linelength = 0;
-		screen_info.lfb_width = 0;
-		screen_info.lfb_height = 0;
-		screen_info.orig_video_isVGA = 0;
+		si->lfb_linelength = 0;
+		si->lfb_width = 0;
+		si->lfb_height = 0;
+		si->orig_video_isVGA = 0;
 		return 0;
 	}
 
 	printk(KERN_INFO "efifb: dmi detected %s - framebuffer at 0x%08x "
 			 "(%dx%d, stride %d)\n", id->ident,
-			 screen_info.lfb_base, screen_info.lfb_width,
-			 screen_info.lfb_height, screen_info.lfb_linelength);
+			 si->lfb_base, si->lfb_width,
+			 si->lfb_height, si->lfb_linelength);
 
 	return 1;
 }
 
+static int __init efifb_set_system_callback(const struct dmi_system_id *id)
+{
+	return efifb_set_system(&sysfb_primary_display.screen, id);
+}
+
 #define EFIFB_DMI_SYSTEM_ID(vendor, name, enumid)		\
 	{							\
-		efifb_set_system,				\
+		efifb_set_system_callback,			\
 		name,						\
 		{						\
 			DMI_MATCH(DMI_BIOS_VENDOR, vendor),	\
@@ -231,6 +235,44 @@ static const struct dmi_system_id efifb_dmi_system_table[] __initconst = {
 	{},
 };
 
+static int __init efifb_swap_width_height(const struct dmi_system_id *id)
+{
+	struct screen_info *si = &sysfb_primary_display.screen;
+	u32 bpp = __screen_info_lfb_bits_per_pixel(si);
+
+	swap(si->lfb_width, si->lfb_height);
+	si->lfb_linelength = bpp * si->lfb_width / BITS_PER_BYTE;
+
+	return 1;
+}
+
+struct efifb_mode_fixup {
+	unsigned int width;
+	unsigned int height;
+	unsigned int linelength;
+};
+
+static int __init
+efifb_check_and_swap_width_height(const struct dmi_system_id *id)
+{
+	const struct efifb_mode_fixup *data = id->driver_data;
+	struct screen_info *si = &sysfb_primary_display.screen;
+
+	if (data->width == si->lfb_width && data->height == si->lfb_height) {
+		swap(si->lfb_width, si->lfb_height);
+		si->lfb_linelength = data->linelength;
+		si->lfb_size = data->linelength * data->width;
+	}
+
+	return 1;
+}
+
+static const struct efifb_mode_fixup efifb_steamdeck_mode_fixup __initconst = {
+	.width = 1280,
+	.height = 800,
+	.linelength = 3328,
+};
+
 /*
  * Some devices have a portrait LCD but advertise a landscape resolution (and
  * pitch). We simply swap width and height for these devices so that we can
@@ -248,6 +290,7 @@ static const struct dmi_system_id efifb_dmi_swap_width_height[] __initconst = {
 			DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "MIIX 310-10ICR"),
 			DMI_EXACT_MATCH(DMI_BIOS_VERSION, "1HCN44WW"),
 		},
+		.callback = efifb_swap_width_height,
 	},
 	{
 		/* Lenovo MIIX 320-10ICR with 800x1280 portrait screen */
@@ -256,6 +299,7 @@ static const struct dmi_system_id efifb_dmi_swap_width_height[] __initconst = {
 			DMI_EXACT_MATCH(DMI_PRODUCT_VERSION,
 					"Lenovo MIIX 320-10ICR"),
 		},
+		.callback = efifb_swap_width_height,
 	},
 	{
 		/* Lenovo D330 with 800x1280 or 1200x1920 portrait screen */
@@ -264,6 +308,7 @@ static const struct dmi_system_id efifb_dmi_swap_width_height[] __initconst = {
 			DMI_EXACT_MATCH(DMI_PRODUCT_VERSION,
 					"Lenovo ideapad D330-10IGM"),
 		},
+		.callback = efifb_swap_width_height,
 	},
 	{
 		/* Lenovo IdeaPad Duet 3 10IGL5 with 1200x1920 portrait screen */
@@ -272,6 +317,7 @@ static const struct dmi_system_id efifb_dmi_swap_width_height[] __initconst = {
 			DMI_EXACT_MATCH(DMI_PRODUCT_VERSION,
 					"IdeaPad Duet 3 10IGL5"),
 		},
+		.callback = efifb_swap_width_height,
 	},
 	{
 		/* Lenovo Yoga Book X91F / X91L */
@@ -280,16 +326,38 @@ static const struct dmi_system_id efifb_dmi_swap_width_height[] __initconst = {
 			/* Non exact match to match F + L versions */
 			DMI_MATCH(DMI_PRODUCT_NAME, "Lenovo YB1-X91"),
 		},
+		.callback = efifb_swap_width_height,
+	},
+	{
+		/* Valve Steam Deck (Jupiter) */
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Valve"),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Jupiter"),
+			DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "1"),
+		},
+		.callback = efifb_check_and_swap_width_height,
+		.driver_data = (void *)&efifb_steamdeck_mode_fixup,
+	},
+	{
+		/* Valve Steam Deck (Galileo) */
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Valve"),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Galileo"),
+			DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "1"),
+		},
+		.callback = efifb_check_and_swap_width_height,
+		.driver_data = (void *)&efifb_steamdeck_mode_fixup,
 	},
 	{},
 };
 
-static bool efifb_overlaps_pci_range(const struct of_pci_range *range)
+static bool efifb_overlaps_pci_range(const struct screen_info *si,
+				     const struct of_pci_range *range)
 {
-	u64 fb_base = screen_info.lfb_base;
+	u64 fb_base = si->lfb_base;
 
-	if (screen_info.capabilities & VIDEO_CAPABILITY_64BIT_BASE)
-		fb_base |= (u64)(unsigned long)screen_info.ext_lfb_base << 32;
+	if (si->capabilities & VIDEO_CAPABILITY_64BIT_BASE)
+		fb_base |= (u64)(unsigned long)si->ext_lfb_base << 32;
 
 	return fb_base >= range->cpu_addr &&
 	       fb_base < (range->cpu_addr + range->size);
@@ -311,7 +379,7 @@ static struct device_node *find_pci_overlap_node(void)
 		}
 
 		for_each_of_pci_range(&parser, &range)
-			if (efifb_overlaps_pci_range(&range))
+			if (efifb_overlaps_pci_range(&sysfb_primary_display.screen, &range))
 				return np;
 	}
 	return NULL;
@@ -349,25 +417,19 @@ static const struct fwnode_operations efifb_fwnode_ops = {
 
 static struct fwnode_handle efifb_fwnode;
 
-__init void sysfb_apply_efi_quirks(void)
+__init void sysfb_apply_efi_quirks(struct screen_info *si)
 {
-	if (screen_info.orig_video_isVGA != VIDEO_TYPE_EFI ||
-	    !(screen_info.capabilities & VIDEO_CAPABILITY_SKIP_QUIRKS))
+	if (si->orig_video_isVGA != VIDEO_TYPE_EFI ||
+	    !(si->capabilities & VIDEO_CAPABILITY_SKIP_QUIRKS))
 		dmi_check_system(efifb_dmi_system_table);
 
-	if (screen_info.orig_video_isVGA == VIDEO_TYPE_EFI &&
-	    dmi_check_system(efifb_dmi_swap_width_height)) {
-		u16 temp = screen_info.lfb_width;
-
-		screen_info.lfb_width = screen_info.lfb_height;
-		screen_info.lfb_height = temp;
-		screen_info.lfb_linelength = 4 * screen_info.lfb_width;
-	}
+	if (si->orig_video_isVGA == VIDEO_TYPE_EFI)
+		dmi_check_system(efifb_dmi_swap_width_height);
 }
 
-__init void sysfb_set_efifb_fwnode(struct platform_device *pd)
+__init void sysfb_set_efifb_fwnode(const struct screen_info *si, struct platform_device *pd)
 {
-	if (screen_info.orig_video_isVGA == VIDEO_TYPE_EFI && IS_ENABLED(CONFIG_PCI)) {
+	if (si->orig_video_isVGA == VIDEO_TYPE_EFI && IS_ENABLED(CONFIG_PCI)) {
 		fwnode_init(&efifb_fwnode, &efifb_fwnode_ops);
 		pd->dev.fwnode = &efifb_fwnode;
 	}
diff --git a/drivers/firmware/qcom/qcom_scm.c b/drivers/firmware/qcom/qcom_scm.c
index 1a6f85e463e0..ff0f7101dfe2 100644
--- a/drivers/firmware/qcom/qcom_scm.c
+++ b/drivers/firmware/qcom/qcom_scm.c
@@ -2005,6 +2005,7 @@ static const struct of_device_id qcom_scm_qseecom_allowlist[] __maybe_unused = {
 	{ .compatible = "lenovo,thinkpad-t14s" },
 	{ .compatible = "lenovo,thinkpad-x13s", },
 	{ .compatible = "lenovo,yoga-slim7x" },
+	{ .compatible = "medion,sprchrgd14s1" },
 	{ .compatible = "microsoft,arcata", },
 	{ .compatible = "microsoft,blackrock" },
 	{ .compatible = "microsoft,romulus13", },
diff --git a/drivers/firmware/sysfb.c b/drivers/firmware/sysfb.c
index 889e5b05c739..8833582c1883 100644
--- a/drivers/firmware/sysfb.c
+++ b/drivers/firmware/sysfb.c
@@ -66,7 +66,7 @@ static bool sysfb_unregister(void)
  */
 void sysfb_disable(struct device *dev)
 {
-	struct screen_info *si = &screen_info;
+	struct screen_info *si = &sysfb_primary_display.screen;
 	struct device *parent;
 
 	mutex_lock(&disable_lock);
@@ -92,7 +92,7 @@ EXPORT_SYMBOL_GPL(sysfb_disable);
  */
 bool sysfb_handles_screen_info(void)
 {
-	const struct screen_info *si = &screen_info;
+	const struct screen_info *si = &sysfb_primary_display.screen;
 
 	return !!screen_info_video_type(si);
 }
@@ -141,7 +141,8 @@ static struct device *sysfb_parent_dev(const struct screen_info *si)
 
 static __init int sysfb_init(void)
 {
-	struct screen_info *si = &screen_info;
+	struct sysfb_display_info *dpy = &sysfb_primary_display;
+	struct screen_info *si = &dpy->screen;
 	struct device *parent;
 	unsigned int type;
 	struct simplefb_platform_data mode;
@@ -155,7 +156,7 @@ static __init int sysfb_init(void)
 	if (disabled)
 		goto unlock_mutex;
 
-	sysfb_apply_efi_quirks();
+	sysfb_apply_efi_quirks(si);
 
 	parent = sysfb_parent_dev(si);
 	if (IS_ERR(parent)) {
@@ -200,9 +201,9 @@ static __init int sysfb_init(void)
 
 	pd->dev.parent = parent;
 
-	sysfb_set_efifb_fwnode(pd);
+	sysfb_set_efifb_fwnode(si, pd);
 
-	ret = platform_device_add_data(pd, si, sizeof(*si));
+	ret = platform_device_add_data(pd, dpy, sizeof(*dpy));
 	if (ret)
 		goto err;
 
diff --git a/drivers/firmware/sysfb_simplefb.c b/drivers/firmware/sysfb_simplefb.c
index 592d8a644619..71f542e37732 100644
--- a/drivers/firmware/sysfb_simplefb.c
+++ b/drivers/firmware/sysfb_simplefb.c
@@ -117,7 +117,7 @@ __init struct platform_device *sysfb_create_simplefb(const struct screen_info *s
 
 	pd->dev.parent = parent;
 
-	sysfb_set_efifb_fwnode(pd);
+	sysfb_set_efifb_fwnode(si, pd);
 
 	ret = platform_device_add_resources(pd, &res, 1);
 	if (ret)
diff --git a/drivers/firmware/ti_sci.h b/drivers/firmware/ti_sci.h
index 91f234550c43..4616127e33ff 100644
--- a/drivers/firmware/ti_sci.h
+++ b/drivers/firmware/ti_sci.h
@@ -580,13 +580,13 @@ struct ti_sci_msg_resp_get_clock_freq {
 } __packed;
 
 /**
- * struct tisci_msg_req_prepare_sleep - Request for TISCI_MSG_PREPARE_SLEEP.
+ * struct ti_sci_msg_req_prepare_sleep - Request for TISCI_MSG_PREPARE_SLEEP.
  *
- * @hdr				TISCI header to provide ACK/NAK flags to the host.
- * @mode			Low power mode to enter.
- * @ctx_lo			Low 32-bits of physical pointer to address to use for context save.
- * @ctx_hi			High 32-bits of physical pointer to address to use for context save.
- * @debug_flags			Flags that can be set to halt the sequence during suspend or
+ * @hdr:			TISCI header to provide ACK/NAK flags to the host.
+ * @mode:			Low power mode to enter.
+ * @ctx_lo:			Low 32-bits of physical pointer to address to use for context save.
+ * @ctx_hi:			High 32-bits of physical pointer to address to use for context save.
+ * @debug_flags:		Flags that can be set to halt the sequence during suspend or
  *				resume to allow JTAG connection and debug.
  *
  * This message is used as the first step of entering a low power mode. It
@@ -610,7 +610,7 @@ struct ti_sci_msg_req_prepare_sleep {
 } __packed;
 
 /**
- * struct tisci_msg_set_io_isolation_req - Request for TI_SCI_MSG_SET_IO_ISOLATION.
+ * struct ti_sci_msg_req_set_io_isolation - Request for TI_SCI_MSG_SET_IO_ISOLATION.
  *
  * @hdr:	Generic header
  * @state:	The deseared state of the IO isolation.
@@ -676,7 +676,7 @@ struct ti_sci_msg_req_lpm_set_device_constraint {
  * TISCI_MSG_LPM_SET_LATENCY_CONSTRAINT.
  *
  * @hdr:	TISCI header to provide ACK/NAK flags to the host.
- * @wkup_latency:	The maximum acceptable latency to wake up from low power mode
+ * @latency:	The maximum acceptable latency to wake up from low power mode
  *			in milliseconds. The deeper the state, the higher the latency.
  * @state:	The desired state of wakeup latency constraint: set or clear.
  * @rsvd:	Reserved for future use.
@@ -855,7 +855,7 @@ struct ti_sci_msg_rm_ring_cfg_req {
  * UDMAP transmit channels mapped to source threads will have their
  * TCHAN_THRD_ID register programmed with the destination thread if the pairing
  * is successful.
-
+ *
  * @dst_thread: PSI-L destination thread ID within the PSI-L System thread map.
  * PSI-L destination threads start at index 0x8000.  The request is NACK'd if
  * the destination thread is not greater than or equal to 0x8000.
@@ -1000,7 +1000,8 @@ struct rm_ti_sci_msg_udmap_rx_flow_opt_cfg {
 } __packed;
 
 /**
- * Configures a Navigator Subsystem UDMAP transmit channel
+ * struct ti_sci_msg_rm_udmap_tx_ch_cfg_req - Configures a
+ *	Navigator Subsystem UDMAP transmit channel
  *
  * Configures the non-real-time registers of a Navigator Subsystem UDMAP
  * transmit channel.  The channel index must be assigned to the host defined
@@ -1128,7 +1129,8 @@ struct ti_sci_msg_rm_udmap_tx_ch_cfg_req {
 } __packed;
 
 /**
- * Configures a Navigator Subsystem UDMAP receive channel
+ * struct ti_sci_msg_rm_udmap_rx_ch_cfg_req - Configures a
+ *	Navigator Subsystem UDMAP receive channel
  *
  * Configures the non-real-time registers of a Navigator Subsystem UDMAP
  * receive channel.  The channel index must be assigned to the host defined
@@ -1247,7 +1249,8 @@ struct ti_sci_msg_rm_udmap_rx_ch_cfg_req {
 } __packed;
 
 /**
- * Configures a Navigator Subsystem UDMAP receive flow
+ * struct ti_sci_msg_rm_udmap_flow_cfg_req - Configures a
+ *	Navigator Subsystem UDMAP receive flow
  *
  * Configures a Navigator Subsystem UDMAP receive flow's registers.
  * Configuration does not include the flow registers which handle size-based
@@ -1258,7 +1261,7 @@ struct ti_sci_msg_rm_udmap_rx_ch_cfg_req {
  *
  * @hdr: Standard TISCI header
  *
- * @valid_params
+ * @valid_params:
  * Bitfield defining validity of rx flow configuration parameters.  The
  * rx flow configuration fields are not valid, and will not be used for flow
  * configuration, if their corresponding valid bit is zero.  Valid bit usage:
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index bd185482a7fd..c33f9305ab97 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -600,7 +600,7 @@ config GPIO_RDA
 
 config GPIO_REALTEK_OTTO
 	tristate "Realtek Otto GPIO support"
-	depends on MACH_REALTEK_RTL
+	depends on MACH_REALTEK_RTL || COMPILE_TEST
 	default MACH_REALTEK_RTL
 	select GPIO_GENERIC
 	select GPIOLIB_IRQCHIP
@@ -1193,11 +1193,11 @@ config GPIO_PCA953X
 
 	  8 bits:       max7310, max7315, pca6107, pca9534, pca9538, pca9554,
 	                pca9556, pca9557, pca9574, tca6408, tca9554, xra1202,
-			pcal6408, pcal9554b, tca9538
+			pcal6408, pcal9554b, tca9538, tcal6408
 
 	  16 bits:      max7312, max7313, pca9535, pca9539, pca9555, pca9575,
 	                tca6416, pca6416, pcal6416, pcal9535, pcal9555a, max7318,
-			tca9539
+			tca9539, tcal6416
 
 	  18 bits:	tca6418
 
diff --git a/drivers/gpio/gpio-aggregator.c b/drivers/gpio/gpio-aggregator.c
index 416f265d09d0..a4cd32674a96 100644
--- a/drivers/gpio/gpio-aggregator.c
+++ b/drivers/gpio/gpio-aggregator.c
@@ -1226,7 +1226,7 @@ gpio_aggregator_line_release(struct config_item *item)
 	kfree(line);
 }
 
-static struct configfs_item_operations gpio_aggregator_line_item_ops = {
+static const struct configfs_item_operations gpio_aggregator_line_item_ops = {
 	.release	= gpio_aggregator_line_release,
 };
 
@@ -1247,7 +1247,7 @@ static void gpio_aggregator_device_release(struct config_item *item)
 	gpio_aggregator_free(aggr);
 }
 
-static struct configfs_item_operations gpio_aggregator_device_item_ops = {
+static const struct configfs_item_operations gpio_aggregator_device_item_ops = {
 	.release	= gpio_aggregator_device_release,
 };
 
@@ -1292,7 +1292,7 @@ gpio_aggregator_device_make_group(struct config_group *group, const char *name)
 	return &line->group;
 }
 
-static struct configfs_group_operations gpio_aggregator_device_group_ops = {
+static const struct configfs_group_operations gpio_aggregator_device_group_ops = {
 	.make_group	= gpio_aggregator_device_make_group,
 };
 
@@ -1328,7 +1328,7 @@ gpio_aggregator_make_group(struct config_group *group, const char *name)
 	return &aggr->group;
 }
 
-static struct configfs_group_operations gpio_aggregator_group_ops = {
+static const struct configfs_group_operations gpio_aggregator_group_ops = {
 	.make_group	= gpio_aggregator_make_group,
 };
 
diff --git a/drivers/gpio/gpio-it87.c b/drivers/gpio/gpio-it87.c
index 5d677bcfccf2..2ad3c239367b 100644
--- a/drivers/gpio/gpio-it87.c
+++ b/drivers/gpio/gpio-it87.c
@@ -12,6 +12,7 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <linux/cleanup.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -241,23 +242,17 @@ static int it87_gpio_direction_out(struct gpio_chip *chip,
 	mask = 1 << (gpio_num % 8);
 	group = (gpio_num / 8);
 
-	spin_lock(&it87_gpio->lock);
+	guard(spinlock)(&it87_gpio->lock);
 
 	rc = superio_enter();
 	if (rc)
-		goto exit;
+		return rc;
 
 	/* set the output enable bit */
 	superio_set_mask(mask, group + it87_gpio->output_base);
 
 	rc = it87_gpio_set(chip, gpio_num, val);
-	if (rc)
-		goto exit;
-
 	superio_exit();
-
-exit:
-	spin_unlock(&it87_gpio->lock);
 	return rc;
 }
 
diff --git a/drivers/gpio/gpio-macsmc.c b/drivers/gpio/gpio-macsmc.c
index 30ef258e7655..b0952d066a9d 100644
--- a/drivers/gpio/gpio-macsmc.c
+++ b/drivers/gpio/gpio-macsmc.c
@@ -10,6 +10,7 @@
 #include <linux/bitmap.h>
 #include <linux/device.h>
 #include <linux/gpio/driver.h>
+#include <linux/hex.h>
 #include <linux/mfd/core.h>
 #include <linux/mfd/macsmc.h>
 
diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c
index 0a3916cc2772..dd0e09961bc9 100644
--- a/drivers/gpio/gpio-pca953x.c
+++ b/drivers/gpio/gpio-pca953x.c
@@ -126,6 +126,9 @@ static const struct i2c_device_id pca953x_id[] = {
 	{ "tca9539", 16 | PCA953X_TYPE | PCA_INT, },
 	{ "tca9554", 8  | PCA953X_TYPE | PCA_INT, },
 	{ "xra1202", 8  | PCA953X_TYPE },
+
+	{ "tcal6408", 8  | PCA953X_TYPE | PCA_LATCH_INT, },
+	{ "tcal6416", 16 | PCA953X_TYPE | PCA_LATCH_INT, },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pca953x_id);
@@ -1444,6 +1447,9 @@ static const struct of_device_id pca953x_dt_ids[] = {
 	{ .compatible = "ti,tca9538", .data = OF_953X( 8, PCA_INT), },
 	{ .compatible = "ti,tca9539", .data = OF_953X(16, PCA_INT), },
 
+	{ .compatible = "ti,tcal6408", .data = OF_953X( 8, PCA_LATCH_INT), },
+	{ .compatible = "ti,tcal6416", .data = OF_953X(16, PCA_LATCH_INT), },
+
 	{ .compatible = "onnn,cat9554", .data = OF_953X( 8, PCA_INT), },
 	{ .compatible = "onnn,pca9654", .data = OF_953X( 8, PCA_INT), },
 	{ .compatible = "onnn,pca9655", .data = OF_953X(16, PCA_INT), },
diff --git a/drivers/gpio/gpio-realtek-otto.c b/drivers/gpio/gpio-realtek-otto.c
index de527f4fc6c2..4cf91528f547 100644
--- a/drivers/gpio/gpio-realtek-otto.c
+++ b/drivers/gpio/gpio-realtek-otto.c
@@ -359,8 +359,7 @@ static int realtek_gpio_probe(struct platform_device *pdev)
 {
 	struct gpio_generic_chip_config config;
 	struct device *dev = &pdev->dev;
-	unsigned long gen_gc_flags;
-	unsigned int dev_flags;
+	unsigned long gen_gc_flags, dev_flags;
 	struct gpio_irq_chip *girq;
 	struct realtek_gpio_ctrl *ctrl;
 	struct resource *res;
@@ -372,7 +371,7 @@ static int realtek_gpio_probe(struct platform_device *pdev)
 	if (!ctrl)
 		return -ENOMEM;
 
-	dev_flags = (unsigned int) device_get_match_data(dev);
+	dev_flags = (uintptr_t)device_get_match_data(dev);
 
 	ngpios = REALTEK_GPIO_MAX;
 	device_property_read_u32(dev, "ngpios", &ngpios);
diff --git a/drivers/gpio/gpio-sim.c b/drivers/gpio/gpio-sim.c
index a83f5238427c..437b4500f56b 100644
--- a/drivers/gpio/gpio-sim.c
+++ b/drivers/gpio/gpio-sim.c
@@ -1384,7 +1384,7 @@ static void gpio_sim_hog_config_item_release(struct config_item *item)
 	kfree(hog);
 }
 
-static struct configfs_item_operations gpio_sim_hog_config_item_ops = {
+static const struct configfs_item_operations gpio_sim_hog_config_item_ops = {
 	.release	= gpio_sim_hog_config_item_release,
 };
 
@@ -1433,11 +1433,11 @@ static void gpio_sim_line_config_group_release(struct config_item *item)
 	kfree(line);
 }
 
-static struct configfs_item_operations gpio_sim_line_config_item_ops = {
+static const struct configfs_item_operations gpio_sim_line_config_item_ops = {
 	.release	= gpio_sim_line_config_group_release,
 };
 
-static struct configfs_group_operations gpio_sim_line_config_group_ops = {
+static const struct configfs_group_operations gpio_sim_line_config_group_ops = {
 	.make_item	= gpio_sim_line_config_make_hog_item,
 };
 
@@ -1494,11 +1494,11 @@ static void gpio_sim_bank_config_group_release(struct config_item *item)
 	kfree(bank);
 }
 
-static struct configfs_item_operations gpio_sim_bank_config_item_ops = {
+static const struct configfs_item_operations gpio_sim_bank_config_item_ops = {
 	.release	= gpio_sim_bank_config_group_release,
 };
 
-static struct configfs_group_operations gpio_sim_bank_config_group_ops = {
+static const struct configfs_group_operations gpio_sim_bank_config_group_ops = {
 	.make_group	= gpio_sim_bank_config_make_line_group,
 };
 
@@ -1549,11 +1549,11 @@ static void gpio_sim_device_config_group_release(struct config_item *item)
 	kfree(dev);
 }
 
-static struct configfs_item_operations gpio_sim_device_config_item_ops = {
+static const struct configfs_item_operations gpio_sim_device_config_item_ops = {
 	.release	= gpio_sim_device_config_group_release,
 };
 
-static struct configfs_group_operations gpio_sim_device_config_group_ops = {
+static const struct configfs_group_operations gpio_sim_device_config_group_ops = {
 	.make_group	= gpio_sim_device_config_make_bank_group,
 };
 
@@ -1589,7 +1589,7 @@ gpio_sim_config_make_device_group(struct config_group *group, const char *name)
 	return &no_free_ptr(dev)->group;
 }
 
-static struct configfs_group_operations gpio_sim_config_group_ops = {
+static const struct configfs_group_operations gpio_sim_config_group_ops = {
 	.make_group	= gpio_sim_config_make_device_group,
 };
 
diff --git a/drivers/gpio/gpio-virtuser.c b/drivers/gpio/gpio-virtuser.c
index 37f2ce20f1ae..090f5643afaf 100644
--- a/drivers/gpio/gpio-virtuser.c
+++ b/drivers/gpio/gpio-virtuser.c
@@ -1631,7 +1631,7 @@ static void gpio_virtuser_lookup_config_group_release(struct config_item *item)
 	kfree(lookup);
 }
 
-static struct configfs_item_operations gpio_virtuser_lookup_config_item_ops = {
+static const struct configfs_item_operations gpio_virtuser_lookup_config_item_ops = {
 	.release	= gpio_virtuser_lookup_config_group_release,
 };
 
@@ -1692,11 +1692,11 @@ static void gpio_virtuser_device_config_group_release(struct config_item *item)
 	kfree(dev);
 }
 
-static struct configfs_item_operations gpio_virtuser_device_config_item_ops = {
+static const struct configfs_item_operations gpio_virtuser_device_config_item_ops = {
 	.release	= gpio_virtuser_device_config_group_release,
 };
 
-static struct configfs_group_operations gpio_virtuser_device_config_group_ops = {
+static const struct configfs_group_operations gpio_virtuser_device_config_group_ops = {
 	.make_group	= gpio_virtuser_make_lookup_group,
 };
 
@@ -1729,7 +1729,7 @@ gpio_virtuser_config_make_device_group(struct config_group *group,
 	return &no_free_ptr(dev)->group;
 }
 
-static struct configfs_group_operations gpio_virtuser_config_group_ops = {
+static const struct configfs_group_operations gpio_virtuser_config_group_ops = {
 	.make_group	= gpio_virtuser_config_make_device_group,
 };
 
diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c
index 8657379e9165..ef1ac68b94b7 100644
--- a/drivers/gpio/gpiolib-of.c
+++ b/drivers/gpio/gpiolib-of.c
@@ -634,6 +634,7 @@ static struct gpio_desc *of_find_gpio_rename(struct device_node *np,
 	return ERR_PTR(-ENOENT);
 }
 
+#if IS_ENABLED(CONFIG_SND_SOC_MT2701_CS42448)
 static struct gpio_desc *of_find_mt2701_gpio(struct device_node *np,
 					     const char *con_id,
 					     unsigned int idx,
@@ -665,6 +666,7 @@ static struct gpio_desc *of_find_mt2701_gpio(struct device_node *np,
 
 	return desc;
 }
+#endif
 
 /*
  * Trigger sources are special, they allow us to use any GPIO as a LED trigger
@@ -699,7 +701,9 @@ typedef struct gpio_desc *(*of_find_gpio_quirk)(struct device_node *np,
 						enum of_gpio_flags *of_flags);
 static const of_find_gpio_quirk of_find_gpio_quirks[] = {
 	of_find_gpio_rename,
+#if IS_ENABLED(CONFIG_SND_SOC_MT2701_CS42448)
 	of_find_mt2701_gpio,
+#endif
 	of_find_trigger_gpio,
 	NULL
 };
diff --git a/drivers/gpio/gpiolib-swnode.c b/drivers/gpio/gpiolib-swnode.c
index b44f35d68459..54bad9e88fe5 100644
--- a/drivers/gpio/gpiolib-swnode.c
+++ b/drivers/gpio/gpiolib-swnode.c
@@ -18,19 +18,18 @@
 
 #include <linux/gpio/consumer.h>
 #include <linux/gpio/driver.h>
+#include <linux/gpio/property.h>
 
 #include "gpiolib.h"
 #include "gpiolib-swnode.h"
 
-#define GPIOLIB_SWNODE_UNDEFINED_NAME "swnode-gpio-undefined"
-
 static struct gpio_device *swnode_get_gpio_device(struct fwnode_handle *fwnode)
 {
 	const struct software_node *gdev_node;
 	struct gpio_device *gdev;
 
 	gdev_node = to_software_node(fwnode);
-	if (!gdev_node || !gdev_node->name)
+	if (!gdev_node)
 		goto fwnode_lookup;
 
 	/*
@@ -38,7 +37,7 @@ static struct gpio_device *swnode_get_gpio_device(struct fwnode_handle *fwnode)
 	 * primarily used as a key for internal chip selects in SPI bindings.
 	 */
 	if (IS_ENABLED(CONFIG_GPIO_SWNODE_UNDEFINED) &&
-	    !strcmp(gdev_node->name, GPIOLIB_SWNODE_UNDEFINED_NAME))
+	    gdev_node == &swnode_gpio_undefined)
 		return ERR_PTR(-ENOENT);
 
 fwnode_lookup:
@@ -139,9 +138,7 @@ int swnode_gpio_count(const struct fwnode_handle *fwnode, const char *con_id)
  * A special node that identifies undefined GPIOs, this is primarily used as
  * a key for internal chip selects in SPI bindings.
  */
-const struct software_node swnode_gpio_undefined = {
-	.name = GPIOLIB_SWNODE_UNDEFINED_NAME,
-};
+const struct software_node swnode_gpio_undefined = { };
 EXPORT_SYMBOL_NS_GPL(swnode_gpio_undefined, "GPIO_SWNODE");
 
 static int __init swnode_gpio_init(void)
diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
index 91e0c384f34a..409c2f415251 100644
--- a/drivers/gpio/gpiolib.c
+++ b/drivers/gpio/gpiolib.c
@@ -5242,27 +5242,21 @@ void gpiod_put_array(struct gpio_descs *descs)
 }
 EXPORT_SYMBOL_GPL(gpiod_put_array);
 
-static int gpio_stub_drv_probe(struct device *dev)
-{
-	/*
-	 * The DT node of some GPIO chips have a "compatible" property, but
-	 * never have a struct device added and probed by a driver to register
-	 * the GPIO chip with gpiolib. In such cases, fw_devlink=on will cause
-	 * the consumers of the GPIO chip to get probe deferred forever because
-	 * they will be waiting for a device associated with the GPIO chip
-	 * firmware node to get added and bound to a driver.
-	 *
-	 * To allow these consumers to probe, we associate the struct
-	 * gpio_device of the GPIO chip with the firmware node and then simply
-	 * bind it to this stub driver.
-	 */
-	return 0;
-}
-
+/*
+ * The DT node of some GPIO chips have a "compatible" property, but
+ * never have a struct device added and probed by a driver to register
+ * the GPIO chip with gpiolib. In such cases, fw_devlink=on will cause
+ * the consumers of the GPIO chip to get probe deferred forever because
+ * they will be waiting for a device associated with the GPIO chip
+ * firmware node to get added and bound to a driver.
+ *
+ * To allow these consumers to probe, we associate the struct
+ * gpio_device of the GPIO chip with the firmware node and then simply
+ * bind it to this stub driver.
+ */
 static struct device_driver gpio_stub_drv = {
 	.name = "gpio_stub_drv",
 	.bus = &gpio_bus_type,
-	.probe = gpio_stub_drv_probe,
 };
 
 static int __init gpiolib_dev_init(void)
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index 7e6bc0b3a589..a33b90251530 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -265,164 +265,83 @@ config DRM_SCHED
 	tristate
 	depends on DRM
 
-source "drivers/gpu/drm/sysfb/Kconfig"
+# Separate option as not all DRM drivers use it
+config DRM_PANEL_BACKLIGHT_QUIRKS
+	tristate
 
-source "drivers/gpu/drm/arm/Kconfig"
+config DRM_LIB_RANDOM
+	bool
+	default n
 
-source "drivers/gpu/drm/radeon/Kconfig"
+config DRM_PRIVACY_SCREEN
+	bool
+	default n
 
+# DRM driver Kconfig files, sorted
+source "drivers/gpu/drm/adp/Kconfig"
 source "drivers/gpu/drm/amd/amdgpu/Kconfig"
-
-source "drivers/gpu/drm/nouveau/Kconfig"
-
-source "drivers/gpu/drm/nova/Kconfig"
-
-source "drivers/gpu/drm/i915/Kconfig"
-
-source "drivers/gpu/drm/xe/Kconfig"
-
-source "drivers/gpu/drm/kmb/Kconfig"
-
-config DRM_VGEM
-	tristate "Virtual GEM provider"
-	depends on DRM && MMU
-	select DRM_GEM_SHMEM_HELPER
-	help
-	  Choose this option to get a virtual graphics memory manager,
-	  as used by Mesa's software renderer for enhanced performance.
-	  If M is selected the module will be called vgem.
-
-source "drivers/gpu/drm/vkms/Kconfig"
-
-source "drivers/gpu/drm/exynos/Kconfig"
-
-source "drivers/gpu/drm/rockchip/Kconfig"
-
-source "drivers/gpu/drm/vmwgfx/Kconfig"
-
-source "drivers/gpu/drm/gma500/Kconfig"
-
-source "drivers/gpu/drm/udl/Kconfig"
-
-source "drivers/gpu/drm/ast/Kconfig"
-
-source "drivers/gpu/drm/mgag200/Kconfig"
-
+source "drivers/gpu/drm/arm/Kconfig"
 source "drivers/gpu/drm/armada/Kconfig"
-
+source "drivers/gpu/drm/aspeed/Kconfig"
+source "drivers/gpu/drm/ast/Kconfig"
 source "drivers/gpu/drm/atmel-hlcdc/Kconfig"
-
-source "drivers/gpu/drm/renesas/Kconfig"
-
-source "drivers/gpu/drm/sun4i/Kconfig"
-
-source "drivers/gpu/drm/omapdrm/Kconfig"
-
-source "drivers/gpu/drm/tilcdc/Kconfig"
-
-source "drivers/gpu/drm/qxl/Kconfig"
-
-source "drivers/gpu/drm/virtio/Kconfig"
-
-source "drivers/gpu/drm/msm/Kconfig"
-
-source "drivers/gpu/drm/fsl-dcu/Kconfig"
-
-source "drivers/gpu/drm/tegra/Kconfig"
-
-source "drivers/gpu/drm/stm/Kconfig"
-
-source "drivers/gpu/drm/panel/Kconfig"
-
 source "drivers/gpu/drm/bridge/Kconfig"
-
-source "drivers/gpu/drm/sti/Kconfig"
-
-source "drivers/gpu/drm/imx/Kconfig"
-
-source "drivers/gpu/drm/ingenic/Kconfig"
-
-source "drivers/gpu/drm/v3d/Kconfig"
-
-source "drivers/gpu/drm/vc4/Kconfig"
-
-source "drivers/gpu/drm/loongson/Kconfig"
-
 source "drivers/gpu/drm/etnaviv/Kconfig"
-
+source "drivers/gpu/drm/exynos/Kconfig"
+source "drivers/gpu/drm/fsl-dcu/Kconfig"
+source "drivers/gpu/drm/gma500/Kconfig"
+source "drivers/gpu/drm/gud/Kconfig"
 source "drivers/gpu/drm/hisilicon/Kconfig"
-
+source "drivers/gpu/drm/hyperv/Kconfig"
+source "drivers/gpu/drm/i915/Kconfig"
+source "drivers/gpu/drm/imagination/Kconfig"
+source "drivers/gpu/drm/imx/Kconfig"
+source "drivers/gpu/drm/ingenic/Kconfig"
+source "drivers/gpu/drm/kmb/Kconfig"
+source "drivers/gpu/drm/lima/Kconfig"
 source "drivers/gpu/drm/logicvc/Kconfig"
-
+source "drivers/gpu/drm/loongson/Kconfig"
+source "drivers/gpu/drm/mcde/Kconfig"
 source "drivers/gpu/drm/mediatek/Kconfig"
-
-source "drivers/gpu/drm/mxsfb/Kconfig"
-
 source "drivers/gpu/drm/meson/Kconfig"
-
-source "drivers/gpu/drm/tiny/Kconfig"
-
-source "drivers/gpu/drm/pl111/Kconfig"
-
-source "drivers/gpu/drm/tve200/Kconfig"
-
-source "drivers/gpu/drm/xen/Kconfig"
-
-source "drivers/gpu/drm/vboxvideo/Kconfig"
-
-source "drivers/gpu/drm/lima/Kconfig"
-
+source "drivers/gpu/drm/mgag200/Kconfig"
+source "drivers/gpu/drm/msm/Kconfig"
+source "drivers/gpu/drm/mxsfb/Kconfig"
+source "drivers/gpu/drm/nouveau/Kconfig"
+source "drivers/gpu/drm/nova/Kconfig"
+source "drivers/gpu/drm/omapdrm/Kconfig"
+source "drivers/gpu/drm/panel/Kconfig"
 source "drivers/gpu/drm/panfrost/Kconfig"
-
 source "drivers/gpu/drm/panthor/Kconfig"
-
-source "drivers/gpu/drm/aspeed/Kconfig"
-
-source "drivers/gpu/drm/mcde/Kconfig"
-
-source "drivers/gpu/drm/tidss/Kconfig"
-
-source "drivers/gpu/drm/adp/Kconfig"
-
-source "drivers/gpu/drm/xlnx/Kconfig"
-
-source "drivers/gpu/drm/gud/Kconfig"
-
+source "drivers/gpu/drm/pl111/Kconfig"
+source "drivers/gpu/drm/qxl/Kconfig"
+source "drivers/gpu/drm/radeon/Kconfig"
+source "drivers/gpu/drm/renesas/Kconfig"
+source "drivers/gpu/drm/rockchip/Kconfig"
 source "drivers/gpu/drm/sitronix/Kconfig"
-
 source "drivers/gpu/drm/solomon/Kconfig"
-
 source "drivers/gpu/drm/sprd/Kconfig"
-
-source "drivers/gpu/drm/imagination/Kconfig"
-
+source "drivers/gpu/drm/sti/Kconfig"
+source "drivers/gpu/drm/stm/Kconfig"
+source "drivers/gpu/drm/sun4i/Kconfig"
+source "drivers/gpu/drm/sysfb/Kconfig"
+source "drivers/gpu/drm/tegra/Kconfig"
+source "drivers/gpu/drm/tidss/Kconfig"
+source "drivers/gpu/drm/tilcdc/Kconfig"
+source "drivers/gpu/drm/tiny/Kconfig"
+source "drivers/gpu/drm/tve200/Kconfig"
 source "drivers/gpu/drm/tyr/Kconfig"
-
-config DRM_HYPERV
-	tristate "DRM Support for Hyper-V synthetic video device"
-	depends on DRM && PCI && HYPERV_VMBUS
-	select DRM_CLIENT_SELECTION
-	select DRM_KMS_HELPER
-	select DRM_GEM_SHMEM_HELPER
-	help
-	 This is a KMS driver for Hyper-V synthetic video device. Choose this
-	 option if you would like to enable drm driver for Hyper-V virtual
-	 machine. Unselect Hyper-V framebuffer driver (CONFIG_FB_HYPERV) so
-	 that DRM driver is used by default.
-
-	 If M is selected the module will be called hyperv_drm.
-
-# Separate option as not all DRM drivers use it
-config DRM_PANEL_BACKLIGHT_QUIRKS
-	tristate
-
-config DRM_LIB_RANDOM
-	bool
-	default n
-
-config DRM_PRIVACY_SCREEN
-	bool
-	default n
+source "drivers/gpu/drm/udl/Kconfig"
+source "drivers/gpu/drm/v3d/Kconfig"
+source "drivers/gpu/drm/vboxvideo/Kconfig"
+source "drivers/gpu/drm/vc4/Kconfig"
+source "drivers/gpu/drm/vgem/Kconfig"
+source "drivers/gpu/drm/virtio/Kconfig"
+source "drivers/gpu/drm/vkms/Kconfig"
+source "drivers/gpu/drm/vmwgfx/Kconfig"
+source "drivers/gpu/drm/xe/Kconfig"
+source "drivers/gpu/drm/xen/Kconfig"
+source "drivers/gpu/drm/xlnx/Kconfig"
 
 endif
 
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index c88760fb52ea..d0c506a44569 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -86,7 +86,7 @@ amdgpu-y += \
 	nbio_v7_2.o hdp_v4_0.o hdp_v5_0.o aldebaran_reg_init.o aldebaran.o soc21.o soc24.o \
 	sienna_cichlid.o smu_v13_0_10.o nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o lsdma_v6_0.o \
 	nbio_v7_9.o aqua_vanjaram.o nbio_v7_11.o lsdma_v7_0.o hdp_v7_0.o nbif_v6_3_1.o \
-	cyan_skillfish_reg_init.o
+	cyan_skillfish_reg_init.o soc_v1_0.o
 
 # add DF block
 amdgpu-y += \
@@ -104,7 +104,8 @@ amdgpu-y += \
 	gfxhub_v2_0.o mmhub_v2_0.o gmc_v10_0.o gfxhub_v2_1.o mmhub_v2_3.o \
 	mmhub_v1_7.o gfxhub_v3_0.o mmhub_v3_0.o mmhub_v3_0_2.o gmc_v11_0.o \
 	mmhub_v3_0_1.o gfxhub_v3_0_3.o gfxhub_v1_2.o mmhub_v1_8.o mmhub_v3_3.o \
-	gfxhub_v11_5_0.o mmhub_v4_1_0.o gfxhub_v12_0.o gmc_v12_0.o
+	gfxhub_v11_5_0.o mmhub_v4_1_0.o gfxhub_v12_0.o gmc_v12_0.o \
+	mmhub_v4_2_0.o gfxhub_v12_1.o gmc_v12_1.o
 
 # add UMC block
 amdgpu-y += \
@@ -134,7 +135,8 @@ amdgpu-y += \
 	psp_v12_0.o \
 	psp_v13_0.o \
 	psp_v13_0_4.o \
-	psp_v14_0.o
+	psp_v14_0.o \
+	psp_v15_0_8.o
 
 # add DCE block
 amdgpu-y += \
@@ -156,7 +158,9 @@ amdgpu-y += \
 	gfx_v11_0_3.o \
 	imu_v11_0_3.o \
 	gfx_v12_0.o \
-	imu_v12_0.o
+	gfx_v12_1.o \
+	imu_v12_0.o \
+	imu_v12_1.o
 
 # add async DMA block
 amdgpu-y += \
@@ -169,13 +173,15 @@ amdgpu-y += \
 	sdma_v5_0.o \
 	sdma_v5_2.o \
 	sdma_v6_0.o \
-	sdma_v7_0.o
+	sdma_v7_0.o \
+	sdma_v7_1.o
 
 # add MES block
 amdgpu-y += \
 	amdgpu_mes.o \
 	mes_v11_0.o \
 	mes_v12_0.o \
+	mes_v12_1.o
 
 # add GFX userqueue support
 amdgpu-y += mes_userqueue.o
@@ -244,7 +250,8 @@ amdgpu-y += \
 	smuio_v13_0.o \
 	smuio_v13_0_3.o \
 	smuio_v13_0_6.o \
-	smuio_v14_0_2.o
+	smuio_v14_0_2.o \
+	smuio_v15_0_8.o
 
 # add reset block
 amdgpu-y += \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 9f9774f58ce1..de8fb746daf1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -116,6 +116,7 @@
 #include "amdgpu_reg_state.h"
 #include "amdgpu_userq.h"
 #include "amdgpu_eviction_fence.h"
+#include "amdgpu_ip.h"
 #if defined(CONFIG_DRM_AMD_ISP)
 #include "amdgpu_isp.h"
 #endif
@@ -362,59 +363,6 @@ enum amdgpu_kiq_irq {
 #define MAX_KIQ_REG_BAILOUT_INTERVAL   5 /* in msecs, 5ms */
 #define MAX_KIQ_REG_TRY 1000
 
-int amdgpu_device_ip_set_clockgating_state(void *dev,
-					   enum amd_ip_block_type block_type,
-					   enum amd_clockgating_state state);
-int amdgpu_device_ip_set_powergating_state(void *dev,
-					   enum amd_ip_block_type block_type,
-					   enum amd_powergating_state state);
-void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
-					    u64 *flags);
-int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
-				   enum amd_ip_block_type block_type);
-bool amdgpu_device_ip_is_hw(struct amdgpu_device *adev,
-			    enum amd_ip_block_type block_type);
-bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev,
-			      enum amd_ip_block_type block_type);
-int amdgpu_ip_block_suspend(struct amdgpu_ip_block *ip_block);
-
-int amdgpu_ip_block_resume(struct amdgpu_ip_block *ip_block);
-
-#define AMDGPU_MAX_IP_NUM AMD_IP_BLOCK_TYPE_NUM
-
-struct amdgpu_ip_block_status {
-	bool valid;
-	bool sw;
-	bool hw;
-	bool late_initialized;
-	bool hang;
-};
-
-struct amdgpu_ip_block_version {
-	const enum amd_ip_block_type type;
-	const u32 major;
-	const u32 minor;
-	const u32 rev;
-	const struct amd_ip_funcs *funcs;
-};
-
-struct amdgpu_ip_block {
-	struct amdgpu_ip_block_status status;
-	const struct amdgpu_ip_block_version *version;
-	struct amdgpu_device *adev;
-};
-
-int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
-				       enum amd_ip_block_type type,
-				       u32 major, u32 minor);
-
-struct amdgpu_ip_block *
-amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
-			      enum amd_ip_block_type type);
-
-int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
-			       const struct amdgpu_ip_block_version *ip_block_version);
-
 /*
  * BIOS.
  */
@@ -757,71 +705,6 @@ struct amdgpu_mmio_remap {
 	struct amdgpu_bo *bo;
 };
 
-/* Define the HW IP blocks will be used in driver , add more if necessary */
-enum amd_hw_ip_block_type {
-	GC_HWIP = 1,
-	HDP_HWIP,
-	SDMA0_HWIP,
-	SDMA1_HWIP,
-	SDMA2_HWIP,
-	SDMA3_HWIP,
-	SDMA4_HWIP,
-	SDMA5_HWIP,
-	SDMA6_HWIP,
-	SDMA7_HWIP,
-	LSDMA_HWIP,
-	MMHUB_HWIP,
-	ATHUB_HWIP,
-	NBIO_HWIP,
-	MP0_HWIP,
-	MP1_HWIP,
-	UVD_HWIP,
-	VCN_HWIP = UVD_HWIP,
-	JPEG_HWIP = VCN_HWIP,
-	VCN1_HWIP,
-	VCE_HWIP,
-	VPE_HWIP,
-	DF_HWIP,
-	DCE_HWIP,
-	OSSSYS_HWIP,
-	SMUIO_HWIP,
-	PWR_HWIP,
-	NBIF_HWIP,
-	THM_HWIP,
-	CLK_HWIP,
-	UMC_HWIP,
-	RSMU_HWIP,
-	XGMI_HWIP,
-	DCI_HWIP,
-	PCIE_HWIP,
-	ISP_HWIP,
-	MAX_HWIP
-};
-
-#define HWIP_MAX_INSTANCE	44
-
-#define HW_ID_MAX		300
-#define IP_VERSION_FULL(mj, mn, rv, var, srev) \
-	(((mj) << 24) | ((mn) << 16) | ((rv) << 8) | ((var) << 4) | (srev))
-#define IP_VERSION(mj, mn, rv)		IP_VERSION_FULL(mj, mn, rv, 0, 0)
-#define IP_VERSION_MAJ(ver)		((ver) >> 24)
-#define IP_VERSION_MIN(ver)		(((ver) >> 16) & 0xFF)
-#define IP_VERSION_REV(ver)		(((ver) >> 8) & 0xFF)
-#define IP_VERSION_VARIANT(ver)		(((ver) >> 4) & 0xF)
-#define IP_VERSION_SUBREV(ver)		((ver) & 0xF)
-#define IP_VERSION_MAJ_MIN_REV(ver)	((ver) >> 8)
-
-struct amdgpu_ip_map_info {
-	/* Map of logical to actual dev instances/mask */
-	uint32_t 		dev_inst[MAX_HWIP][HWIP_MAX_INSTANCE];
-	int8_t (*logical_to_dev_inst)(struct amdgpu_device *adev,
-				      enum amd_hw_ip_block_type block,
-				      int8_t inst);
-	uint32_t (*logical_to_dev_mask)(struct amdgpu_device *adev,
-					enum amd_hw_ip_block_type block,
-					uint32_t mask);
-};
-
 enum amdgpu_uid_type {
 	AMDGPU_UID_TYPE_XCD,
 	AMDGPU_UID_TYPE_AID,
@@ -1545,8 +1428,13 @@ int emu_soc_asic_init(struct amdgpu_device *adev);
 #define amdgpu_asic_get_pcie_usage(adev, cnt0, cnt1) ((adev)->asic_funcs->get_pcie_usage((adev), (cnt0), (cnt1)))
 #define amdgpu_asic_need_reset_on_init(adev) (adev)->asic_funcs->need_reset_on_init((adev))
 #define amdgpu_asic_get_pcie_replay_count(adev) ((adev)->asic_funcs->get_pcie_replay_count((adev)))
-#define amdgpu_asic_supports_baco(adev) (adev)->asic_funcs->supports_baco((adev))
-#define amdgpu_asic_pre_asic_init(adev) (adev)->asic_funcs->pre_asic_init((adev))
+#define amdgpu_asic_supports_baco(adev) \
+    ((adev)->asic_funcs->supports_baco ? (adev)->asic_funcs->supports_baco((adev)) : 0)
+#define amdgpu_asic_pre_asic_init(adev)                                      \
+	{                                                                    \
+		if ((adev)->asic_funcs && (adev)->asic_funcs->pre_asic_init) \
+			(adev)->asic_funcs->pre_asic_init((adev));           \
+	}
 #define amdgpu_asic_update_umd_stable_pstate(adev, enter) \
 	((adev)->asic_funcs->update_umd_stable_pstate ? (adev)->asic_funcs->update_umd_stable_pstate((adev), (enter)) : 0)
 #define amdgpu_asic_query_video_codecs(adev, e, c) (adev)->asic_funcs->query_video_codecs((adev), (e), (c))
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
index d31460a9e958..ec91c4aa84dd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
@@ -241,7 +241,8 @@ static void amdgpu_atif_parse_functions(struct amdgpu_atif_functions *f, u32 mas
  * (all asics).
  * returns 0 on success, error on failure.
  */
-static int amdgpu_atif_verify_interface(struct amdgpu_atif *atif)
+static noinline_for_stack
+int amdgpu_atif_verify_interface(struct amdgpu_atif *atif)
 {
 	union acpi_object *info;
 	struct atif_verify_interface output;
@@ -286,7 +287,8 @@ static int amdgpu_atif_verify_interface(struct amdgpu_atif *atif)
  * where n is specified in the result if a notifier is used.
  * Returns 0 on success, error on failure.
  */
-static int amdgpu_atif_get_notification_params(struct amdgpu_atif *atif)
+static noinline_for_stack
+int amdgpu_atif_get_notification_params(struct amdgpu_atif *atif)
 {
 	union acpi_object *info;
 	struct amdgpu_atif_notification_cfg *n = &atif->notification_cfg;
@@ -354,7 +356,8 @@ static int amdgpu_atif_get_notification_params(struct amdgpu_atif *atif)
  *
  * Returns 0 on success, error on failure.
  */
-static int amdgpu_atif_query_backlight_caps(struct amdgpu_atif *atif)
+static noinline_for_stack
+int amdgpu_atif_query_backlight_caps(struct amdgpu_atif *atif)
 {
 	union acpi_object *info;
 	struct atif_qbtc_output characteristics;
@@ -600,7 +603,8 @@ static void amdgpu_atcs_parse_functions(struct amdgpu_atcs_functions *f, u32 mas
  * (all asics).
  * returns 0 on success, error on failure.
  */
-static int amdgpu_atcs_verify_interface(struct amdgpu_atcs *atcs)
+static noinline_for_stack
+int amdgpu_atcs_verify_interface(struct amdgpu_atcs *atcs)
 {
 	union acpi_object *info;
 	struct atcs_verify_interface output;
@@ -1089,7 +1093,8 @@ static int amdgpu_acpi_get_xcc_info(struct amdgpu_acpi_xcc_info *xcc_info,
 	return ret;
 }
 
-static int amdgpu_acpi_enumerate_xcc(void)
+static noinline_for_stack
+int amdgpu_acpi_enumerate_xcc(void)
 {
 	struct amdgpu_acpi_dev_info *dev_info = NULL;
 	struct amdgpu_acpi_xcc_info *xcc_info;
@@ -1108,7 +1113,7 @@ static int amdgpu_acpi_enumerate_xcc(void)
 		 * one is not found, no need to check the rest.
 		 */
 		if (!acpi_dev) {
-			DRM_DEBUG_DRIVER("No matching acpi device found for %s",
+			DRM_DEBUG_DRIVER("No matching acpi device found for %s\n",
 					 hid);
 			break;
 		}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 8bdfcde2029b..da4575676335 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -37,7 +37,7 @@
 #include "amdgpu_sync.h"
 #include "amdgpu_vm.h"
 #include "amdgpu_xcp.h"
-
+#include "kfd_topology.h"
 extern uint64_t amdgpu_amdkfd_total_mem_size;
 
 enum TLB_FLUSH_TYPE {
@@ -98,6 +98,7 @@ struct amdgpu_amdkfd_fence {
 	spinlock_t lock;
 	char timeline_name[TASK_COMM_LEN];
 	struct svm_range_bo *svm_bo;
+	uint16_t context_id;
 };
 
 struct amdgpu_kfd_dev {
@@ -148,6 +149,8 @@ struct amdkfd_process_info {
 	/* MMU-notifier related fields */
 	struct mutex notifier_lock;
 	uint32_t evicted_bos;
+	/* kfd context id */
+	u16 context_id;
 	struct delayed_work restore_userptr_work;
 	struct pid *pid;
 	bool block_mmu_notifications;
@@ -188,7 +191,8 @@ int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev,
 
 struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
 				struct mm_struct *mm,
-				struct svm_range_bo *svm_bo);
+				struct svm_range_bo *svm_bo,
+				u16 context_id);
 
 int amdgpu_amdkfd_drm_client_create(struct amdgpu_device *adev);
 #if defined(CONFIG_DEBUG_FS)
@@ -407,7 +411,7 @@ int kgd2kfd_init_zone_device(struct amdgpu_device *adev)
 int kgd2kfd_quiesce_mm(struct mm_struct *mm, uint32_t trigger);
 int kgd2kfd_resume_mm(struct mm_struct *mm);
 int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
-						struct dma_fence *fence);
+					       u16 context_id, struct dma_fence *fence);
 #if IS_ENABLED(CONFIG_HSA_AMD)
 int kgd2kfd_init(void);
 void kgd2kfd_exit(void);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c
index 1ef758ac5076..193ed8becab8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c
@@ -62,7 +62,8 @@ static atomic_t fence_seq = ATOMIC_INIT(0);
 
 struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
 				struct mm_struct *mm,
-				struct svm_range_bo *svm_bo)
+				struct svm_range_bo *svm_bo,
+				u16 context_id)
 {
 	struct amdgpu_amdkfd_fence *fence;
 
@@ -76,6 +77,7 @@ struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
 	get_task_comm(fence->timeline_name, current);
 	spin_lock_init(&fence->lock);
 	fence->svm_bo = svm_bo;
+	fence->context_id = context_id;
 	dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock,
 		   context, atomic_inc_return(&fence_seq));
 
@@ -126,8 +128,12 @@ static bool amdkfd_fence_enable_signaling(struct dma_fence *f)
 	if (dma_fence_is_signaled(f))
 		return true;
 
+	/* if fence->svm_bo is NULL, means this fence is created through
+	 * init_kfd_vm() or amdgpu_amdkfd_gpuvm_restore_process_bos().
+	 * Therefore, this fence is amdgpu_amdkfd_fence->eviction_fence.
+	 */
 	if (!fence->svm_bo) {
-		if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, f))
+		if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, fence->context_id, f))
 			return true;
 	} else {
 		if (!svm_range_schedule_evict_svm_bo(fence))
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index b1c24c8fa686..6585b8b9e709 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1397,8 +1397,10 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
 		       struct dma_fence **ef)
 {
 	struct amdkfd_process_info *info = NULL;
+	struct kfd_process *process = NULL;
 	int ret;
 
+	process = container_of(process_info, struct kfd_process, kgd_process_info);
 	if (!*process_info) {
 		info = kzalloc(sizeof(*info), GFP_KERNEL);
 		if (!info)
@@ -1414,7 +1416,7 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
 		info->eviction_fence =
 			amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
 						   current->mm,
-						   NULL);
+						   NULL, process->context_id);
 		if (!info->eviction_fence) {
 			pr_err("Failed to create eviction fence\n");
 			ret = -ENOMEM;
@@ -1425,6 +1427,8 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
 		INIT_DELAYED_WORK(&info->restore_userptr_work,
 				  amdgpu_amdkfd_restore_userptr_worker);
 
+		info->context_id = process->context_id;
+
 		*process_info = info;
 	}
 
@@ -3066,7 +3070,7 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu *
 			amdgpu_amdkfd_fence_create(
 				process_info->eviction_fence->base.context,
 				process_info->eviction_fence->mm,
-				NULL);
+				NULL, process_info->context_id);
 
 		if (!new_fence) {
 			pr_err("Failed to create eviction fence\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
index 199693369c7c..a050167e76a4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
@@ -37,9 +37,9 @@ static int amdgpu_benchmark_do_move(struct amdgpu_device *adev, unsigned size,
 
 	stime = ktime_get();
 	for (i = 0; i < n; i++) {
-		struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
-		r = amdgpu_copy_buffer(ring, saddr, daddr, size, NULL, &fence,
-				       false, false, 0);
+		r = amdgpu_copy_buffer(adev, &adev->mman.default_entity,
+				       saddr, daddr, size, NULL, &fence,
+				       false, 0);
 		if (r)
 			goto exit_do_move;
 		r = dma_fence_wait(fence, false);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c
index 425a3e564360..c72c345334d0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c
@@ -23,6 +23,7 @@
  */
 #include <linux/list.h>
 #include "amdgpu.h"
+#include "amdgpu_ras_mgr.h"
 
 static const guid_t MCE			= CPER_NOTIFY_MCE;
 static const guid_t CMC			= CPER_NOTIFY_CMC;
@@ -559,7 +560,10 @@ int amdgpu_cper_init(struct amdgpu_device *adev)
 {
 	int r;
 
-	if (!amdgpu_aca_is_enabled(adev) && !amdgpu_sriov_ras_cper_en(adev))
+	if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_cper_en(adev))
+		return 0;
+	else if (!amdgpu_sriov_vf(adev) && !amdgpu_uniras_enabled(adev) &&
+		!amdgpu_aca_is_enabled(adev))
 		return 0;
 
 	r = amdgpu_cper_ring_init(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index ecdfe6cb36cc..d591dce0f3b3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -1021,6 +1021,7 @@ static int amdgpu_cs_patch_ibs(struct amdgpu_cs_parser *p,
 			       struct amdgpu_job *job)
 {
 	struct amdgpu_ring *ring = amdgpu_job_ring(job);
+	struct amdgpu_device *adev = ring->adev;
 	unsigned int i;
 	int r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 58c3ffe707d1..ae1d95e7ae90 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -313,42 +313,6 @@ void amdgpu_reg_state_sysfs_fini(struct amdgpu_device *adev)
 	sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
 }
 
-int amdgpu_ip_block_suspend(struct amdgpu_ip_block *ip_block)
-{
-	int r;
-
-	if (ip_block->version->funcs->suspend) {
-		r = ip_block->version->funcs->suspend(ip_block);
-		if (r) {
-			dev_err(ip_block->adev->dev,
-				"suspend of IP block <%s> failed %d\n",
-				ip_block->version->funcs->name, r);
-			return r;
-		}
-	}
-
-	ip_block->status.hw = false;
-	return 0;
-}
-
-int amdgpu_ip_block_resume(struct amdgpu_ip_block *ip_block)
-{
-	int r;
-
-	if (ip_block->version->funcs->resume) {
-		r = ip_block->version->funcs->resume(ip_block);
-		if (r) {
-			dev_err(ip_block->adev->dev,
-				"resume of IP block <%s> failed %d\n",
-				ip_block->version->funcs->name, r);
-			return r;
-		}
-	}
-
-	ip_block->status.hw = true;
-	return 0;
-}
-
 /**
  * DOC: board_info
  *
@@ -2263,293 +2227,6 @@ static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
 	.can_switch = amdgpu_switcheroo_can_switch,
 };
 
-/**
- * amdgpu_device_ip_set_clockgating_state - set the CG state
- *
- * @dev: amdgpu_device pointer
- * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
- * @state: clockgating state (gate or ungate)
- *
- * Sets the requested clockgating state for all instances of
- * the hardware IP specified.
- * Returns the error code from the last instance.
- */
-int amdgpu_device_ip_set_clockgating_state(void *dev,
-					   enum amd_ip_block_type block_type,
-					   enum amd_clockgating_state state)
-{
-	struct amdgpu_device *adev = dev;
-	int i, r = 0;
-
-	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_blocks[i].status.valid)
-			continue;
-		if (adev->ip_blocks[i].version->type != block_type)
-			continue;
-		if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
-			continue;
-		r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
-			&adev->ip_blocks[i], state);
-		if (r)
-			dev_err(adev->dev,
-				"set_clockgating_state of IP block <%s> failed %d\n",
-				adev->ip_blocks[i].version->funcs->name, r);
-	}
-	return r;
-}
-
-/**
- * amdgpu_device_ip_set_powergating_state - set the PG state
- *
- * @dev: amdgpu_device pointer
- * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
- * @state: powergating state (gate or ungate)
- *
- * Sets the requested powergating state for all instances of
- * the hardware IP specified.
- * Returns the error code from the last instance.
- */
-int amdgpu_device_ip_set_powergating_state(void *dev,
-					   enum amd_ip_block_type block_type,
-					   enum amd_powergating_state state)
-{
-	struct amdgpu_device *adev = dev;
-	int i, r = 0;
-
-	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_blocks[i].status.valid)
-			continue;
-		if (adev->ip_blocks[i].version->type != block_type)
-			continue;
-		if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
-			continue;
-		r = adev->ip_blocks[i].version->funcs->set_powergating_state(
-			&adev->ip_blocks[i], state);
-		if (r)
-			dev_err(adev->dev,
-				"set_powergating_state of IP block <%s> failed %d\n",
-				adev->ip_blocks[i].version->funcs->name, r);
-	}
-	return r;
-}
-
-/**
- * amdgpu_device_ip_get_clockgating_state - get the CG state
- *
- * @adev: amdgpu_device pointer
- * @flags: clockgating feature flags
- *
- * Walks the list of IPs on the device and updates the clockgating
- * flags for each IP.
- * Updates @flags with the feature flags for each hardware IP where
- * clockgating is enabled.
- */
-void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
-					    u64 *flags)
-{
-	int i;
-
-	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_blocks[i].status.valid)
-			continue;
-		if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
-			adev->ip_blocks[i].version->funcs->get_clockgating_state(
-				&adev->ip_blocks[i], flags);
-	}
-}
-
-/**
- * amdgpu_device_ip_wait_for_idle - wait for idle
- *
- * @adev: amdgpu_device pointer
- * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
- *
- * Waits for the request hardware IP to be idle.
- * Returns 0 for success or a negative error code on failure.
- */
-int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
-				   enum amd_ip_block_type block_type)
-{
-	int i, r;
-
-	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_blocks[i].status.valid)
-			continue;
-		if (adev->ip_blocks[i].version->type == block_type) {
-			if (adev->ip_blocks[i].version->funcs->wait_for_idle) {
-				r = adev->ip_blocks[i].version->funcs->wait_for_idle(
-								&adev->ip_blocks[i]);
-				if (r)
-					return r;
-			}
-			break;
-		}
-	}
-	return 0;
-
-}
-
-/**
- * amdgpu_device_ip_is_hw - is the hardware IP enabled
- *
- * @adev: amdgpu_device pointer
- * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
- *
- * Check if the hardware IP is enable or not.
- * Returns true if it the IP is enable, false if not.
- */
-bool amdgpu_device_ip_is_hw(struct amdgpu_device *adev,
-			    enum amd_ip_block_type block_type)
-{
-	int i;
-
-	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (adev->ip_blocks[i].version->type == block_type)
-			return adev->ip_blocks[i].status.hw;
-	}
-	return false;
-}
-
-/**
- * amdgpu_device_ip_is_valid - is the hardware IP valid
- *
- * @adev: amdgpu_device pointer
- * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
- *
- * Check if the hardware IP is valid or not.
- * Returns true if it the IP is valid, false if not.
- */
-bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev,
-			       enum amd_ip_block_type block_type)
-{
-	int i;
-
-	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (adev->ip_blocks[i].version->type == block_type)
-			return adev->ip_blocks[i].status.valid;
-	}
-	return false;
-
-}
-
-/**
- * amdgpu_device_ip_get_ip_block - get a hw IP pointer
- *
- * @adev: amdgpu_device pointer
- * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
- *
- * Returns a pointer to the hardware IP block structure
- * if it exists for the asic, otherwise NULL.
- */
-struct amdgpu_ip_block *
-amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
-			      enum amd_ip_block_type type)
-{
-	int i;
-
-	for (i = 0; i < adev->num_ip_blocks; i++)
-		if (adev->ip_blocks[i].version->type == type)
-			return &adev->ip_blocks[i];
-
-	return NULL;
-}
-
-/**
- * amdgpu_device_ip_block_version_cmp
- *
- * @adev: amdgpu_device pointer
- * @type: enum amd_ip_block_type
- * @major: major version
- * @minor: minor version
- *
- * return 0 if equal or greater
- * return 1 if smaller or the ip_block doesn't exist
- */
-int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
-				       enum amd_ip_block_type type,
-				       u32 major, u32 minor)
-{
-	struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
-
-	if (ip_block && ((ip_block->version->major > major) ||
-			((ip_block->version->major == major) &&
-			(ip_block->version->minor >= minor))))
-		return 0;
-
-	return 1;
-}
-
-static const char *ip_block_names[] = {
-	[AMD_IP_BLOCK_TYPE_COMMON] = "common",
-	[AMD_IP_BLOCK_TYPE_GMC] = "gmc",
-	[AMD_IP_BLOCK_TYPE_IH] = "ih",
-	[AMD_IP_BLOCK_TYPE_SMC] = "smu",
-	[AMD_IP_BLOCK_TYPE_PSP] = "psp",
-	[AMD_IP_BLOCK_TYPE_DCE] = "dce",
-	[AMD_IP_BLOCK_TYPE_GFX] = "gfx",
-	[AMD_IP_BLOCK_TYPE_SDMA] = "sdma",
-	[AMD_IP_BLOCK_TYPE_UVD] = "uvd",
-	[AMD_IP_BLOCK_TYPE_VCE] = "vce",
-	[AMD_IP_BLOCK_TYPE_ACP] = "acp",
-	[AMD_IP_BLOCK_TYPE_VCN] = "vcn",
-	[AMD_IP_BLOCK_TYPE_MES] = "mes",
-	[AMD_IP_BLOCK_TYPE_JPEG] = "jpeg",
-	[AMD_IP_BLOCK_TYPE_VPE] = "vpe",
-	[AMD_IP_BLOCK_TYPE_UMSCH_MM] = "umsch_mm",
-	[AMD_IP_BLOCK_TYPE_ISP] = "isp",
-	[AMD_IP_BLOCK_TYPE_RAS] = "ras",
-};
-
-static const char *ip_block_name(struct amdgpu_device *adev, enum amd_ip_block_type type)
-{
-	int idx = (int)type;
-
-	return idx < ARRAY_SIZE(ip_block_names) ? ip_block_names[idx] : "unknown";
-}
-
-/**
- * amdgpu_device_ip_block_add
- *
- * @adev: amdgpu_device pointer
- * @ip_block_version: pointer to the IP to add
- *
- * Adds the IP block driver information to the collection of IPs
- * on the asic.
- */
-int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
-			       const struct amdgpu_ip_block_version *ip_block_version)
-{
-	if (!ip_block_version)
-		return -EINVAL;
-
-	switch (ip_block_version->type) {
-	case AMD_IP_BLOCK_TYPE_VCN:
-		if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
-			return 0;
-		break;
-	case AMD_IP_BLOCK_TYPE_JPEG:
-		if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
-			return 0;
-		break;
-	default:
-		break;
-	}
-
-	dev_info(adev->dev, "detected ip block number %d <%s_v%d_%d_%d> (%s)\n",
-		 adev->num_ip_blocks,
-		 ip_block_name(adev, ip_block_version->type),
-		 ip_block_version->major,
-		 ip_block_version->minor,
-		 ip_block_version->rev,
-		 ip_block_version->funcs->name);
-
-	adev->ip_blocks[adev->num_ip_blocks].adev = adev;
-
-	adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
-
-	return 0;
-}
-
 /**
  * amdgpu_device_enable_virtual_display - enable virtual display feature
  *
@@ -3309,19 +2986,18 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
 	if (r)
 		goto init_failed;
 
-	if (adev->mman.buffer_funcs_ring->sched.ready)
+	if (adev->mman.buffer_funcs_ring &&
+	    adev->mman.buffer_funcs_ring->sched.ready)
 		amdgpu_ttm_set_buffer_funcs_status(adev, true);
 
 	/* Don't init kfd if whole hive need to be reset during init */
 	if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
-		kgd2kfd_init_zone_device(adev);
 		amdgpu_amdkfd_device_init(adev);
 	}
 
 	amdgpu_fru_get_product_info(adev);
 
-	if (!amdgpu_sriov_vf(adev) || amdgpu_sriov_ras_cper_en(adev))
-		r = amdgpu_cper_init(adev);
+	r = amdgpu_cper_init(adev);
 
 init_failed:
 
@@ -4931,8 +4607,15 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 		flush_delayed_work(&adev->delayed_init_work);
 	}
 
+	/* Don't init kfd if whole hive need to be reset during init */
+	if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
+		kgd2kfd_init_zone_device(adev);
+		kfd_update_svm_support_properties(adev);
+	}
+
 	if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI)
 		amdgpu_xgmi_reset_on_init(adev);
+
 	/*
 	 * Place those sysfs registering after `late_init`. As some of those
 	 * operations performed in `late_init` might affect the sysfs
@@ -6613,6 +6296,8 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 	struct amdgpu_hive_info *hive = NULL;
 	int r = 0;
 	bool need_emergency_restart = false;
+	/* save the pasid here as the job may be freed before the end of the reset */
+	int pasid = job ? job->pasid : -EINVAL;
 
 	/*
 	 * If it reaches here because of hang/timeout and a RAS error is
@@ -6713,8 +6398,12 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 	if (!r) {
 		struct amdgpu_task_info *ti = NULL;
 
-		if (job)
-			ti = amdgpu_vm_get_task_info_pasid(adev, job->pasid);
+		/*
+		 * The job may already be freed at this point via the sched tdr workqueue so
+		 * use the cached pasid.
+		 */
+		if (pasid >= 0)
+			ti = amdgpu_vm_get_task_info_pasid(adev, pasid);
 
 		drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE,
 				     ti ? &ti->task : NULL);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
index fa2a22dfa048..20d05a3e4516 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
@@ -71,6 +71,7 @@
 #include "nv.h"
 #include "soc21.h"
 #include "soc24.h"
+#include "soc_v1_0.h"
 #include "navi10_ih.h"
 #include "ih_v6_0.h"
 #include "ih_v6_1.h"
@@ -78,10 +79,12 @@
 #include "gfx_v10_0.h"
 #include "gfx_v11_0.h"
 #include "gfx_v12_0.h"
+#include "gfx_v12_1.h"
 #include "sdma_v5_0.h"
 #include "sdma_v5_2.h"
 #include "sdma_v6_0.h"
 #include "sdma_v7_0.h"
+#include "sdma_v7_1.h"
 #include "lsdma_v6_0.h"
 #include "lsdma_v7_0.h"
 #include "vcn_v2_0.h"
@@ -97,12 +100,14 @@
 #include "amdgpu_vkms.h"
 #include "mes_v11_0.h"
 #include "mes_v12_0.h"
+#include "mes_v12_1.h"
 #include "smuio_v11_0.h"
 #include "smuio_v11_0_6.h"
 #include "smuio_v13_0.h"
 #include "smuio_v13_0_3.h"
 #include "smuio_v13_0_6.h"
 #include "smuio_v14_0_2.h"
+#include "smuio_v15_0_8.h"
 #include "vcn_v5_0_0.h"
 #include "vcn_v5_0_1.h"
 #include "jpeg_v5_0_0.h"
@@ -209,6 +214,8 @@ static const char *hw_id_names[HW_ID_MAX] = {
 	[XGBE_HWID]		= "XGBE",
 	[MP0_HWID]		= "MP0",
 	[VPE_HWID]		= "VPE",
+	[ATU_HWID]		= "ATU",
+	[AIGC_HWID]		= "AIGC",
 };
 
 static int hw_id_map[MAX_HWIP] = {
@@ -240,6 +247,7 @@ static int hw_id_map[MAX_HWIP] = {
 	[PCIE_HWIP]	= PCIE_HWID,
 	[VPE_HWIP]	= VPE_HWID,
 	[ISP_HWIP]	= ISP_HWID,
+	[ATU_HWIP]	= ATU_HWID,
 };
 
 static int amdgpu_discovery_read_binary_from_sysmem(struct amdgpu_device *adev, uint8_t *binary)
@@ -1986,6 +1994,9 @@ static int amdgpu_discovery_set_common_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(12, 0, 1):
 		amdgpu_device_ip_block_add(adev, &soc24_common_ip_block);
 		break;
+	case IP_VERSION(12, 1, 0):
+		amdgpu_device_ip_block_add(adev, &soc_v1_0_common_ip_block);
+		break;
 	default:
 		dev_err(adev->dev,
 			"Failed to add common ip block(GC_HWIP:0x%x)\n",
@@ -2040,6 +2051,7 @@ static int amdgpu_discovery_set_gmc_ip_blocks(struct amdgpu_device *adev)
 		break;
 	case IP_VERSION(12, 0, 0):
 	case IP_VERSION(12, 0, 1):
+	case IP_VERSION(12, 1, 0):
 		amdgpu_device_ip_block_add(adev, &gmc_v12_0_ip_block);
 		break;
 	default:
@@ -2084,6 +2096,7 @@ static int amdgpu_discovery_set_ih_ip_blocks(struct amdgpu_device *adev)
 		amdgpu_device_ip_block_add(adev, &ih_v6_1_ip_block);
 		break;
 	case IP_VERSION(7, 0, 0):
+	case IP_VERSION(7, 1, 0):
 		amdgpu_device_ip_block_add(adev, &ih_v7_0_ip_block);
 		break;
 	default:
@@ -2150,6 +2163,9 @@ static int amdgpu_discovery_set_psp_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(14, 0, 5):
 		amdgpu_device_ip_block_add(adev, &psp_v14_0_ip_block);
 		break;
+	case IP_VERSION(15, 0, 8):
+		amdgpu_device_ip_block_add(adev, &psp_v15_0_8_ip_block);
+		break;
 	default:
 		dev_err(adev->dev,
 			"Failed to add psp ip block(MP0_HWIP:0x%x)\n",
@@ -2348,6 +2364,9 @@ static int amdgpu_discovery_set_gc_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(12, 0, 1):
 		amdgpu_device_ip_block_add(adev, &gfx_v12_0_ip_block);
 		break;
+	case IP_VERSION(12, 1, 0):
+		amdgpu_device_ip_block_add(adev, &gfx_v12_1_ip_block);
+		break;
 	default:
 		dev_err(adev->dev, "Failed to add gfx ip block(GC_HWIP:0x%x)\n",
 			amdgpu_ip_version(adev, GC_HWIP, 0));
@@ -2404,6 +2423,9 @@ static int amdgpu_discovery_set_sdma_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(7, 0, 1):
 		amdgpu_device_ip_block_add(adev, &sdma_v7_0_ip_block);
 		break;
+	case IP_VERSION(7, 1, 0):
+		amdgpu_device_ip_block_add(adev, &sdma_v7_1_ip_block);
+		break;
 	default:
 		dev_err(adev->dev,
 			"Failed to add sdma ip block(SDMA0_HWIP:0x%x)\n",
@@ -2549,6 +2571,13 @@ static int amdgpu_discovery_set_mes_ip_blocks(struct amdgpu_device *adev)
 		if (amdgpu_uni_mes)
 			adev->enable_uni_mes = true;
 		break;
+	case IP_VERSION(12, 1, 0):
+		amdgpu_device_ip_block_add(adev, &mes_v12_1_ip_block);
+		adev->enable_mes = true;
+		adev->enable_mes_kiq = true;
+		if (amdgpu_uni_mes)
+			adev->enable_uni_mes = true;
+		break;
 	default:
 		break;
 	}
@@ -2563,6 +2592,9 @@ static void amdgpu_discovery_init_soc_config(struct amdgpu_device *adev)
 	case IP_VERSION(9, 5, 0):
 		aqua_vanjaram_init_soc_config(adev);
 		break;
+	case IP_VERSION(12, 1, 0):
+		soc_v1_0_init_soc_config(adev);
+		break;
 	default:
 		break;
 	}
@@ -2933,6 +2965,7 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 		break;
 	case IP_VERSION(12, 0, 0):
 	case IP_VERSION(12, 0, 1):
+	case IP_VERSION(12, 1, 0):
 		adev->family = AMDGPU_FAMILY_GC_12_0_0;
 		break;
 	default:
@@ -3140,6 +3173,9 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(14, 0, 2):
 		adev->smuio.funcs = &smuio_v14_0_2_funcs;
 		break;
+	case IP_VERSION(15, 0, 8):
+		adev->smuio.funcs = &smuio_v15_0_8_funcs;
+		break;
 	default:
 		break;
 	}
@@ -3210,8 +3246,10 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 		return r;
 
 	if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT &&
-	     !amdgpu_sriov_vf(adev)) ||
-	    (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO && amdgpu_dpm == 1)) {
+	     !amdgpu_sriov_vf(adev) &&
+	     amdgpu_dpm == 1) ||
+	    (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO &&
+	     amdgpu_dpm == 1)) {
 		r = amdgpu_discovery_set_smu_ip_blocks(adev);
 		if (r)
 			return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index e22cfa7c6d32..5f7d19f4c221 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -37,6 +37,7 @@
 #include "amdgpu_dma_buf.h"
 #include "amdgpu_xgmi.h"
 #include "amdgpu_vm.h"
+#include "amdgpu_ttm.h"
 #include <drm/amdgpu_drm.h>
 #include <drm/ttm/ttm_tt.h>
 #include <linux/dma-buf.h>
@@ -241,6 +242,14 @@ static struct sg_table *amdgpu_dma_buf_map(struct dma_buf_attachment *attach,
 		if (r)
 			return ERR_PTR(r);
 		break;
+
+	case AMDGPU_PL_MMIO_REMAP:
+		r = amdgpu_ttm_mmio_remap_alloc_sgt(adev, bo->tbo.resource,
+						    attach->dev, dir, &sgt);
+		if (r)
+			return ERR_PTR(r);
+		break;
+
 	default:
 		return ERR_PTR(-EINVAL);
 	}
@@ -266,6 +275,15 @@ static void amdgpu_dma_buf_unmap(struct dma_buf_attachment *attach,
 				 struct sg_table *sgt,
 				 enum dma_data_direction dir)
 {
+	struct drm_gem_object *obj = attach->dmabuf->priv;
+	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
+
+	if (bo->tbo.resource &&
+	    bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP) {
+		amdgpu_ttm_mmio_remap_free_sgt(attach->dev, dir, sgt);
+		return;
+	}
+
 	if (sg_page(sgt->sgl)) {
 		dma_unmap_sgtable(attach->dev, sgt, dir, 0);
 		sg_free_table(sgt);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
index 2675689ef70f..ebd14c292978 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
@@ -348,6 +348,36 @@ enum AMDGPU_DOORBELL_ASSIGNMENT_LAYOUT1 {
 	AMDGPU_DOORBELL_LAYOUT1_INVALID                 = 0xFFFF
 };
 
+enum AMDGPU_SOC_V1_0_DOORBELL_ASSIGNMENT {
+	/* KIQ/HIQ/DIQ */
+	AMDGPU_SOC_V1_0_DOORBELL_KIQ_START		= 0x000,
+	AMDGPU_SOC_V1_0_DOORBELL_HIQ			= 0x001,
+	AMDGPU_SOC_V1_0_DOORBELL_DIQ			= 0x002,
+	/* Compute: 0x03 ~ 0x20  */
+	AMDGPU_SOC_V1_0_DOORBELL_MEC_RING_START		= 0x003,
+	AMDGPU_SOC_V1_0_DOORBELL_MEC_RING_END		= 0x00A,
+	AMDGPU_SOC_V1_0_DOORBELL_MES_RING0		= 0x00B,
+	AMDGPU_SOC_V1_0_DOORBELL_MES_RING1		= 0x00C,
+	AMDGPU_SOC_V1_0_DOORBELL_USERQUEUE_START	= 0x00D,
+	AMDGPU_SOC_V1_0_DOORBELL_USERQUEUE_END		= 0x01F,
+	AMDGPU_SOC_V1_0_DOORBELL_XCC_RANGE		= 0x020,
+
+	/* SDMA: 0x100 ~ 0x19F */
+	AMDGPU_SOC_V1_0_DOORBELL_sDMA_ENGINE_START	= 0x100,
+	AMDGPU_SOC_V1_0_DOORBELL_sDMA_ENGINE_END	= 0x19F,
+	/* IH: 0x1A0 ~ 0x1AF */
+	AMDGPU_SOC_V1_0_DOORBELL_IH			= 0x1A0,
+	/* VCN: 0x1B0 ~ 0x1EF */
+	AMDGPU_SOC_V1_0_DOORBELL_VCN_START		= 0x1B0,
+	AMDGPU_SOC_V1_0_DOORBELL_VCN_END		= 0x1EF,
+
+	AMDGPU_SOC_V1_0_DOORBELL_FIRST_NON_CP		= AMDGPU_SOC_V1_0_DOORBELL_sDMA_ENGINE_START,
+	AMDGPU_SOC_V1_0_DOORBELL_LAST_NON_CP		= AMDGPU_SOC_V1_0_DOORBELL_VCN_END,
+
+	AMDGPU_SOC_V1_0_DOORBELL_MAX_ASSIGNMENT		= 0x1EF,
+	AMDGPU_SOC_V1_0_DOORBELL_INVALID		= 0xFFFF
+};
+
 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index);
 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v);
 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
index 3040437d99c2..bc7858567321 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
@@ -129,7 +129,7 @@ uint32_t amdgpu_doorbell_index_on_bar(struct amdgpu_device *adev,
 {
 	int db_bo_offset;
 
-	db_bo_offset = amdgpu_bo_gpu_offset_no_check(db_bo);
+	db_bo_offset = amdgpu_bo_gpu_offset(db_bo);
 
 	/* doorbell index is 32 bit but doorbell's size can be 32 bit
 	 * or 64 bit, so *db_size(in byte)/4 for alignment.
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 2dfbddcef9ab..848e6b7db482 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -33,6 +33,7 @@
 #include <drm/drm_vblank.h>
 
 #include <linux/cc_platform.h>
+#include <linux/console.h>
 #include <linux/dynamic_debug.h>
 #include <linux/module.h>
 #include <linux/mmu_notifier.h>
@@ -2704,7 +2705,9 @@ static int amdgpu_pmops_thaw(struct device *dev)
 	struct drm_device *drm_dev = dev_get_drvdata(dev);
 
 	/* do not resume device if it's normal hibernation */
-	if (!pm_hibernate_is_recovering() && !pm_hibernation_mode_is_suspend())
+	if (console_suspend_enabled &&
+	    !pm_hibernate_is_recovering() &&
+	    !pm_hibernation_mode_is_suspend())
 		return 0;
 
 	return amdgpu_device_resume(drm_dev, true);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 3e38c5db2987..dda383a8cb70 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -378,7 +378,7 @@ static void amdgpu_gem_object_close(struct drm_gem_object *obj,
 		goto out_unlock;
 
 	r = amdgpu_vm_clear_freed(adev, vm, &fence);
-	if (unlikely(r < 0))
+	if (unlikely(r < 0) && !drm_dev_is_unplugged(adev_to_drm(adev)))
 		dev_err(adev->dev, "failed to clear page "
 			"tables on GEM object close (%ld)\n", r);
 	if (r || !fence)
@@ -388,7 +388,7 @@ static void amdgpu_gem_object_close(struct drm_gem_object *obj,
 	dma_fence_put(fence);
 
 out_unlock:
-	if (r)
+	if (r && !drm_dev_is_unplugged(adev_to_drm(adev)))
 		dev_err(adev->dev, "leaking bo va (%ld)\n", r);
 	drm_exec_fini(&exec);
 }
@@ -719,6 +719,15 @@ int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,
 	if (unlikely(r != 0))
 		goto out;
 
+	/* Reject MMIO_REMAP BOs at IOCTL level: metadata/tiling does not apply. */
+	if (robj->tbo.resource &&
+	    robj->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP) {
+		DRM_WARN("metadata ioctl on MMIO_REMAP BO (handle %d)\n",
+			 args->handle);
+		r = -EINVAL;
+		goto unreserve;
+	}
+
 	if (args->op == AMDGPU_GEM_METADATA_OP_GET_METADATA) {
 		amdgpu_bo_get_tiling_flags(robj, &args->data.tiling_info);
 		r = amdgpu_bo_get_metadata(robj, args->data.data,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 8b118c53f351..1683785f63cd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -511,7 +511,7 @@ int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev, int xcc_id)
 			j = i + xcc_id * adev->gfx.num_compute_rings;
 			amdgpu_mes_unmap_legacy_queue(adev,
 						   &adev->gfx.compute_ring[j],
-						   RESET_QUEUES, 0, 0);
+						   RESET_QUEUES, 0, 0, xcc_id);
 		}
 		return 0;
 	}
@@ -562,7 +562,7 @@ int amdgpu_gfx_disable_kgq(struct amdgpu_device *adev, int xcc_id)
 				j = i + xcc_id * adev->gfx.num_gfx_rings;
 				amdgpu_mes_unmap_legacy_queue(adev,
 						      &adev->gfx.gfx_ring[j],
-						      PREEMPT_QUEUES, 0, 0);
+						      PREEMPT_QUEUES, 0, 0, xcc_id);
 			}
 		}
 		return 0;
@@ -644,7 +644,8 @@ static int amdgpu_gfx_mes_enable_kcq(struct amdgpu_device *adev, int xcc_id)
 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 		j = i + xcc_id * adev->gfx.num_compute_rings;
 		r = amdgpu_mes_map_legacy_queue(adev,
-						&adev->gfx.compute_ring[j]);
+						&adev->gfx.compute_ring[j],
+						xcc_id);
 		if (r) {
 			dev_err(adev->dev, "failed to map compute queue\n");
 			return r;
@@ -733,7 +734,8 @@ int amdgpu_gfx_enable_kgq(struct amdgpu_device *adev, int xcc_id)
 		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 			j = i + xcc_id * adev->gfx.num_gfx_rings;
 			r = amdgpu_mes_map_legacy_queue(adev,
-							&adev->gfx.gfx_ring[j]);
+							&adev->gfx.gfx_ring[j],
+							xcc_id);
 			if (r) {
 				dev_err(adev->dev, "failed to map gfx queue\n");
 				return r;
@@ -1067,7 +1069,7 @@ uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t xcc_
 		return 0;
 
 	if (adev->mes.ring[0].sched.ready)
-		return amdgpu_mes_rreg(adev, reg);
+		return amdgpu_mes_rreg(adev, reg, xcc_id);
 
 	BUG_ON(!ring->funcs->emit_rreg);
 
@@ -1143,7 +1145,7 @@ void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint3
 		return;
 
 	if (adev->mes.ring[0].sched.ready) {
-		amdgpu_mes_wreg(adev, reg, v);
+		amdgpu_mes_wreg(adev, reg, v, xcc_id);
 		return;
 	}
 
@@ -1195,6 +1197,40 @@ void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint3
 	dev_err(adev->dev, "failed to write reg:%x\n", reg);
 }
 
+void amdgpu_gfx_get_hdp_flush_mask(struct amdgpu_ring *ring,
+		uint32_t *hdp_flush_mask, uint32_t *reg_mem_engine)
+{
+
+	if (!ring || !hdp_flush_mask || !reg_mem_engine) {
+		DRM_INFO("%s:invalid params\n", __func__);
+		return;
+	}
+
+	const struct nbio_hdp_flush_reg *nbio_hf_reg = ring->adev->nbio.hdp_flush_reg;
+
+	switch (ring->funcs->type) {
+	case AMDGPU_RING_TYPE_GFX:
+		*hdp_flush_mask = nbio_hf_reg->ref_and_mask_cp0 << ring->pipe;
+		*reg_mem_engine = 1; /* pfp */
+		break;
+	case AMDGPU_RING_TYPE_COMPUTE:
+		*hdp_flush_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
+		*reg_mem_engine = 0;
+		break;
+	case AMDGPU_RING_TYPE_MES:
+		*hdp_flush_mask = nbio_hf_reg->ref_and_mask_cp8;
+		*reg_mem_engine = 0;
+		break;
+	case AMDGPU_RING_TYPE_KIQ:
+		*hdp_flush_mask = nbio_hf_reg->ref_and_mask_cp9;
+		*reg_mem_engine = 0;
+		break;
+	default:
+		DRM_ERROR("%s:unsupported ring type %d\n", __func__, ring->funcs->type);
+		return;
+	}
+}
+
 int amdgpu_kiq_hdp_flush(struct amdgpu_device *adev)
 {
 	signed long r, cnt = 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index efd61a1ccc66..281e03ad3255 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -328,6 +328,8 @@ struct amdgpu_gfx_shadow_info {
 	u32 shadow_alignment;
 	u32 csa_size;
 	u32 csa_alignment;
+	u32 eop_size;
+	u32 eop_alignment;
 };
 
 struct amdgpu_gfx_funcs {
@@ -356,6 +358,8 @@ struct amdgpu_gfx_funcs {
 				     int num_xccs_per_xcp);
 	int (*ih_node_to_logical_xcc)(struct amdgpu_device *adev, int ih_node);
 	int (*get_xccs_per_xcp)(struct amdgpu_device *adev);
+	void (*get_hdp_flush_mask)(struct amdgpu_ring *ring,
+				uint32_t *ref_and_mask, uint32_t *reg_mem_engine);
 };
 
 struct sq_work {
@@ -615,6 +619,8 @@ int amdgpu_gfx_cp_ecc_error_irq(struct amdgpu_device *adev,
 				  struct amdgpu_iv_entry *entry);
 uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t xcc_id);
 void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint32_t xcc_id);
+void amdgpu_gfx_get_hdp_flush_mask(struct amdgpu_ring *ring,
+		uint32_t *ref_and_mask, uint32_t *reg_mem_engine);
 int amdgpu_kiq_hdp_flush(struct amdgpu_device *adev);
 int amdgpu_gfx_get_num_kcq(struct amdgpu_device *adev);
 void amdgpu_gfx_cp_init_microcode(struct amdgpu_device *adev, uint32_t ucode_id);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 869bceb0fe2c..cd4acc6adc9e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -524,6 +524,54 @@ void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr,
 	} while (fault->timestamp < tmp);
 }
 
+int amdgpu_gmc_handle_retry_fault(struct amdgpu_device *adev,
+				  struct amdgpu_iv_entry *entry,
+				  u64 addr,
+				  u32 cam_index,
+				  u32 node_id,
+				  bool write_fault)
+{
+	int ret;
+
+	if (adev->irq.retry_cam_enabled) {
+		/* Delegate it to a different ring if the hardware hasn't
+		 * already done it.
+		 */
+		if (entry->ih == &adev->irq.ih) {
+			amdgpu_irq_delegate(adev, entry, 8);
+			return 1;
+		}
+
+		ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id,
+					     addr, entry->timestamp, write_fault);
+		WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index);
+		if (ret)
+			return 1;
+	} else {
+		/* Process it only if it's the first fault for this address */
+		if (entry->ih != &adev->irq.ih_soft &&
+		    amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
+					     entry->timestamp))
+			return 1;
+
+		/* Delegate it to a different ring if the hardware hasn't
+		 * already done it.
+		 */
+		if (entry->ih == &adev->irq.ih) {
+			amdgpu_irq_delegate(adev, entry, 8);
+			return 1;
+		}
+
+		/* Try to handle the recoverable page faults by filling page
+		 * tables
+		 */
+		if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id,
+					   addr, entry->timestamp, write_fault))
+			return 1;
+	}
+	return 0;
+}
+
 int amdgpu_gmc_ras_sw_init(struct amdgpu_device *adev)
 {
 	int r;
@@ -690,7 +738,7 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 	 * itself at least for GART.
 	 */
 	mutex_lock(&adev->mman.gtt_window_lock);
-	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.high_pr,
+	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.default_entity.base,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
 				     &job, AMDGPU_KERNEL_JOB_ID_FLUSH_GPU_TLB);
@@ -813,7 +861,7 @@ void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev,
 
 	if (adev->mes.ring[0].sched.ready) {
 		amdgpu_mes_reg_write_reg_wait(adev, reg0, reg1,
-					      ref, mask);
+					      ref, mask, xcc_inst);
 		return;
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index 727342689d4b..e8e8bfa098c3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -32,9 +32,11 @@
 #include "amdgpu_xgmi.h"
 #include "amdgpu_ras.h"
 
-/* VA hole for 48bit addresses on Vega10 */
-#define AMDGPU_GMC_HOLE_START	0x0000800000000000ULL
-#define AMDGPU_GMC_HOLE_END	0xffff800000000000ULL
+/* VA hole for 48bit and 57bit addresses */
+#define AMDGPU_GMC_HOLE_START	(adev->vm_manager.root_level == AMDGPU_VM_PDB3 ?\
+				0x0100000000000000ULL : 0x0000800000000000ULL)
+#define AMDGPU_GMC_HOLE_END	(adev->vm_manager.root_level == AMDGPU_VM_PDB3 ?\
+				0xff00000000000000ULL : 0xffff800000000000ULL)
 
 /*
  * Hardware is programmed as if the hole doesn't exists with start and end
@@ -43,7 +45,8 @@
  * This mask is used to remove the upper 16bits of the VA and so come up with
  * the linear addr value.
  */
-#define AMDGPU_GMC_HOLE_MASK	0x0000ffffffffffffULL
+#define AMDGPU_GMC_HOLE_MASK	(adev->vm_manager.root_level == AMDGPU_VM_PDB3 ?\
+				0x00ffffffffffffffULL : 0x0000ffffffffffffULL)
 
 /*
  * Ring size as power of two for the log of recent faults.
@@ -353,6 +356,7 @@ struct amdgpu_gmc {
 	u64 MC_VM_MX_L1_TLB_CNTL;
 
 	u64 noretry_flags;
+	u64 init_pte_flags;
 
 	bool flush_tlb_needs_extra_type_0;
 	bool flush_tlb_needs_extra_type_2;
@@ -394,13 +398,8 @@ static inline bool amdgpu_gmc_vram_full_visible(struct amdgpu_gmc *gmc)
  *
  * @addr: address to extend
  */
-static inline uint64_t amdgpu_gmc_sign_extend(uint64_t addr)
-{
-	if (addr >= AMDGPU_GMC_HOLE_START)
-		addr |= AMDGPU_GMC_HOLE_END;
-
-	return addr;
-}
+#define amdgpu_gmc_sign_extend(addr)	((addr) >= AMDGPU_GMC_HOLE_START ?\
+					((addr) | AMDGPU_GMC_HOLE_END) : (addr))
 
 bool amdgpu_gmc_is_pdb0_enabled(struct amdgpu_device *adev);
 int amdgpu_gmc_pdb0_alloc(struct amdgpu_device *adev);
@@ -426,6 +425,12 @@ bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev,
 			      uint16_t pasid, uint64_t timestamp);
 void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr,
 				     uint16_t pasid);
+int amdgpu_gmc_handle_retry_fault(struct amdgpu_device *adev,
+				  struct amdgpu_iv_entry *entry,
+				  u64 addr,
+				  u32 cam_index,
+				  u32 node_id,
+				  bool write_fault);
 int amdgpu_gmc_ras_sw_init(struct amdgpu_device *adev);
 int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev);
 void amdgpu_gmc_ras_fini(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h
index 484e936812e4..eb8537558f5f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h
@@ -36,6 +36,9 @@ struct amdgpu_imu_funcs {
     int (*start_imu)(struct amdgpu_device *adev);
     void (*program_rlc_ram)(struct amdgpu_device *adev);
     int (*wait_for_reset_status)(struct amdgpu_device *adev);
+    int (*switch_compute_partition)(struct amdgpu_device *adev,
+				    int num_xccs_per_xcp,
+				    int compute_partition_mode);
 };
 
 struct imu_rlc_ram_golden {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.c
index 99e1cf4fc955..6aa54156bbc9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.c
@@ -94,3 +94,318 @@ void amdgpu_ip_map_init(struct amdgpu_device *adev)
 	adev->ip_map.logical_to_dev_inst = amdgpu_logical_to_dev_inst;
 	adev->ip_map.logical_to_dev_mask = amdgpu_logical_to_dev_mask;
 }
+
+int amdgpu_ip_block_suspend(struct amdgpu_ip_block *ip_block)
+{
+	int r;
+
+	if (ip_block->version->funcs->suspend) {
+		r = ip_block->version->funcs->suspend(ip_block);
+		if (r) {
+			dev_err(ip_block->adev->dev,
+				"suspend of IP block <%s> failed %d\n",
+				ip_block->version->funcs->name, r);
+			return r;
+		}
+	}
+
+	ip_block->status.hw = false;
+	return 0;
+}
+
+int amdgpu_ip_block_resume(struct amdgpu_ip_block *ip_block)
+{
+	int r;
+
+	if (ip_block->version->funcs->resume) {
+		r = ip_block->version->funcs->resume(ip_block);
+		if (r) {
+			dev_err(ip_block->adev->dev,
+				"resume of IP block <%s> failed %d\n",
+				ip_block->version->funcs->name, r);
+			return r;
+		}
+	}
+
+	ip_block->status.hw = true;
+	return 0;
+}
+
+/**
+ * amdgpu_device_ip_get_ip_block - get a hw IP pointer
+ *
+ * @adev: amdgpu_device pointer
+ * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
+ *
+ * Returns a pointer to the hardware IP block structure
+ * if it exists for the asic, otherwise NULL.
+ */
+struct amdgpu_ip_block *
+amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
+			      enum amd_ip_block_type type)
+{
+	int i;
+
+	for (i = 0; i < adev->num_ip_blocks; i++)
+		if (adev->ip_blocks[i].version->type == type)
+			return &adev->ip_blocks[i];
+
+	return NULL;
+}
+
+/**
+ * amdgpu_device_ip_block_version_cmp
+ *
+ * @adev: amdgpu_device pointer
+ * @type: enum amd_ip_block_type
+ * @major: major version
+ * @minor: minor version
+ *
+ * return 0 if equal or greater
+ * return 1 if smaller or the ip_block doesn't exist
+ */
+int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
+				       enum amd_ip_block_type type, u32 major,
+				       u32 minor)
+{
+	struct amdgpu_ip_block *ip_block =
+		amdgpu_device_ip_get_ip_block(adev, type);
+
+	if (ip_block && ((ip_block->version->major > major) ||
+			 ((ip_block->version->major == major) &&
+			  (ip_block->version->minor >= minor))))
+		return 0;
+
+	return 1;
+}
+
+static const char *const ip_block_names[] = {
+	[AMD_IP_BLOCK_TYPE_COMMON] = "common",
+	[AMD_IP_BLOCK_TYPE_GMC] = "gmc",
+	[AMD_IP_BLOCK_TYPE_IH] = "ih",
+	[AMD_IP_BLOCK_TYPE_SMC] = "smu",
+	[AMD_IP_BLOCK_TYPE_PSP] = "psp",
+	[AMD_IP_BLOCK_TYPE_DCE] = "dce",
+	[AMD_IP_BLOCK_TYPE_GFX] = "gfx",
+	[AMD_IP_BLOCK_TYPE_SDMA] = "sdma",
+	[AMD_IP_BLOCK_TYPE_UVD] = "uvd",
+	[AMD_IP_BLOCK_TYPE_VCE] = "vce",
+	[AMD_IP_BLOCK_TYPE_ACP] = "acp",
+	[AMD_IP_BLOCK_TYPE_VCN] = "vcn",
+	[AMD_IP_BLOCK_TYPE_MES] = "mes",
+	[AMD_IP_BLOCK_TYPE_JPEG] = "jpeg",
+	[AMD_IP_BLOCK_TYPE_VPE] = "vpe",
+	[AMD_IP_BLOCK_TYPE_UMSCH_MM] = "umsch_mm",
+	[AMD_IP_BLOCK_TYPE_ISP] = "isp",
+	[AMD_IP_BLOCK_TYPE_RAS] = "ras",
+};
+
+static const char *ip_block_name(struct amdgpu_device *adev,
+				 enum amd_ip_block_type type)
+{
+	int idx = (int)type;
+
+	return idx < ARRAY_SIZE(ip_block_names) ? ip_block_names[idx] :
+						  "unknown";
+}
+
+/**
+ * amdgpu_device_ip_block_add
+ *
+ * @adev: amdgpu_device pointer
+ * @ip_block_version: pointer to the IP to add
+ *
+ * Adds the IP block driver information to the collection of IPs
+ * on the asic.
+ */
+int amdgpu_device_ip_block_add(
+	struct amdgpu_device *adev,
+	const struct amdgpu_ip_block_version *ip_block_version)
+{
+	if (!ip_block_version)
+		return -EINVAL;
+
+	switch (ip_block_version->type) {
+	case AMD_IP_BLOCK_TYPE_VCN:
+		if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
+			return 0;
+		break;
+	case AMD_IP_BLOCK_TYPE_JPEG:
+		if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
+			return 0;
+		break;
+	default:
+		break;
+	}
+
+	dev_info(adev->dev, "detected ip block number %d <%s_v%d_%d_%d> (%s)\n",
+		 adev->num_ip_blocks,
+		 ip_block_name(adev, ip_block_version->type),
+		 ip_block_version->major, ip_block_version->minor,
+		 ip_block_version->rev, ip_block_version->funcs->name);
+
+	adev->ip_blocks[adev->num_ip_blocks].adev = adev;
+
+	adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
+
+	return 0;
+}
+
+/**
+ * amdgpu_device_ip_set_clockgating_state - set the CG state
+ *
+ * @adev: amdgpu_device pointer
+ * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
+ * @state: clockgating state (gate or ungate)
+ *
+ * Sets the requested clockgating state for all instances of
+ * the hardware IP specified.
+ * Returns the error code from the last instance.
+ */
+int amdgpu_device_ip_set_clockgating_state(struct amdgpu_device *adev,
+					   enum amd_ip_block_type block_type,
+					   enum amd_clockgating_state state)
+{
+	int i, r = 0;
+
+	for (i = 0; i < adev->num_ip_blocks; i++) {
+		if (!adev->ip_blocks[i].status.valid)
+			continue;
+		if (adev->ip_blocks[i].version->type != block_type)
+			continue;
+		if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
+			continue;
+		r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
+			&adev->ip_blocks[i], state);
+		if (r)
+			dev_err(adev->dev,
+				"set_clockgating_state of IP block <%s> failed %d\n",
+				adev->ip_blocks[i].version->funcs->name, r);
+	}
+	return r;
+}
+
+/**
+ * amdgpu_device_ip_set_powergating_state - set the PG state
+ *
+ * @adev: amdgpu_device pointer
+ * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
+ * @state: powergating state (gate or ungate)
+ *
+ * Sets the requested powergating state for all instances of
+ * the hardware IP specified.
+ * Returns the error code from the last instance.
+ */
+int amdgpu_device_ip_set_powergating_state(struct amdgpu_device *adev,
+					   enum amd_ip_block_type block_type,
+					   enum amd_powergating_state state)
+{
+	int i, r = 0;
+
+	for (i = 0; i < adev->num_ip_blocks; i++) {
+		if (!adev->ip_blocks[i].status.valid)
+			continue;
+		if (adev->ip_blocks[i].version->type != block_type)
+			continue;
+		if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
+			continue;
+		r = adev->ip_blocks[i].version->funcs->set_powergating_state(
+			&adev->ip_blocks[i], state);
+		if (r)
+			dev_err(adev->dev,
+				"set_powergating_state of IP block <%s> failed %d\n",
+				adev->ip_blocks[i].version->funcs->name, r);
+	}
+	return r;
+}
+
+/**
+ * amdgpu_device_ip_get_clockgating_state - get the CG state
+ *
+ * @adev: amdgpu_device pointer
+ * @flags: clockgating feature flags
+ *
+ * Walks the list of IPs on the device and updates the clockgating
+ * flags for each IP.
+ * Updates @flags with the feature flags for each hardware IP where
+ * clockgating is enabled.
+ */
+void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
+					    u64 *flags)
+{
+	int i;
+
+	for (i = 0; i < adev->num_ip_blocks; i++) {
+		if (!adev->ip_blocks[i].status.valid)
+			continue;
+		if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
+			adev->ip_blocks[i].version->funcs->get_clockgating_state(
+				&adev->ip_blocks[i], flags);
+	}
+}
+
+/**
+ * amdgpu_device_ip_wait_for_idle - wait for idle
+ *
+ * @adev: amdgpu_device pointer
+ * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
+ *
+ * Waits for the request hardware IP to be idle.
+ * Returns 0 for success or a negative error code on failure.
+ */
+int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
+				   enum amd_ip_block_type block_type)
+{
+	struct amdgpu_ip_block *ip_block;
+
+	ip_block = amdgpu_device_ip_get_ip_block(adev, block_type);
+	if (!ip_block || !ip_block->status.valid)
+		return 0;
+
+	if (ip_block->version->funcs->wait_for_idle)
+		return ip_block->version->funcs->wait_for_idle(ip_block);
+
+	return 0;
+}
+
+/**
+ * amdgpu_device_ip_is_hw - is the hardware IP enabled
+ *
+ * @adev: amdgpu_device pointer
+ * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
+ *
+ * Check if the hardware IP is enable or not.
+ * Returns true if it the IP is enable, false if not.
+ */
+bool amdgpu_device_ip_is_hw(struct amdgpu_device *adev,
+			    enum amd_ip_block_type block_type)
+{
+	struct amdgpu_ip_block *ip_block;
+
+	ip_block = amdgpu_device_ip_get_ip_block(adev, block_type);
+	if (ip_block)
+		return ip_block->status.hw;
+
+	return false;
+}
+
+/**
+ * amdgpu_device_ip_is_valid - is the hardware IP valid
+ *
+ * @adev: amdgpu_device pointer
+ * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
+ *
+ * Check if the hardware IP is valid or not.
+ * Returns true if it the IP is valid, false if not.
+ */
+bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev,
+			       enum amd_ip_block_type block_type)
+{
+	struct amdgpu_ip_block *ip_block;
+
+	ip_block = amdgpu_device_ip_get_ip_block(adev, block_type);
+	if (ip_block)
+		return ip_block->status.valid;
+
+	return false;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.h
index 2490fd322aec..1d0df6d93957 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.h
@@ -24,6 +24,131 @@
 #ifndef __AMDGPU_IP_H__
 #define __AMDGPU_IP_H__
 
+#include "amd_shared.h"
+
+struct amdgpu_device;
+
+/* Define the HW IP blocks will be used in driver , add more if necessary */
+enum amd_hw_ip_block_type {
+	GC_HWIP = 1,
+	HDP_HWIP,
+	SDMA0_HWIP,
+	SDMA1_HWIP,
+	SDMA2_HWIP,
+	SDMA3_HWIP,
+	SDMA4_HWIP,
+	SDMA5_HWIP,
+	SDMA6_HWIP,
+	SDMA7_HWIP,
+	LSDMA_HWIP,
+	MMHUB_HWIP,
+	ATHUB_HWIP,
+	NBIO_HWIP,
+	MP0_HWIP,
+	MP1_HWIP,
+	UVD_HWIP,
+	VCN_HWIP = UVD_HWIP,
+	JPEG_HWIP = VCN_HWIP,
+	VCN1_HWIP,
+	VCE_HWIP,
+	VPE_HWIP,
+	DF_HWIP,
+	DCE_HWIP,
+	OSSSYS_HWIP,
+	SMUIO_HWIP,
+	PWR_HWIP,
+	NBIF_HWIP,
+	THM_HWIP,
+	CLK_HWIP,
+	UMC_HWIP,
+	RSMU_HWIP,
+	XGMI_HWIP,
+	DCI_HWIP,
+	PCIE_HWIP,
+	ISP_HWIP,
+	ATU_HWIP,
+	AIGC_HWIP,
+	MAX_HWIP
+};
+
+#define HWIP_MAX_INSTANCE 48
+
+#define HW_ID_MAX 300
+#define IP_VERSION_FULL(mj, mn, rv, var, srev) \
+	(((mj) << 24) | ((mn) << 16) | ((rv) << 8) | ((var) << 4) | (srev))
+#define IP_VERSION(mj, mn, rv) IP_VERSION_FULL(mj, mn, rv, 0, 0)
+#define IP_VERSION_MAJ(ver) ((ver) >> 24)
+#define IP_VERSION_MIN(ver) (((ver) >> 16) & 0xFF)
+#define IP_VERSION_REV(ver) (((ver) >> 8) & 0xFF)
+#define IP_VERSION_VARIANT(ver) (((ver) >> 4) & 0xF)
+#define IP_VERSION_SUBREV(ver) ((ver) & 0xF)
+#define IP_VERSION_MAJ_MIN_REV(ver) ((ver) >> 8)
+
+struct amdgpu_ip_map_info {
+	/* Map of logical to actual dev instances/mask */
+	uint32_t dev_inst[MAX_HWIP][HWIP_MAX_INSTANCE];
+	int8_t (*logical_to_dev_inst)(struct amdgpu_device *adev,
+				      enum amd_hw_ip_block_type block,
+				      int8_t inst);
+	uint32_t (*logical_to_dev_mask)(struct amdgpu_device *adev,
+					enum amd_hw_ip_block_type block,
+					uint32_t mask);
+};
+
+#define AMDGPU_MAX_IP_NUM AMD_IP_BLOCK_TYPE_NUM
+
+struct amdgpu_ip_block_status {
+	bool valid;
+	bool sw;
+	bool hw;
+	bool late_initialized;
+	bool hang;
+};
+
+struct amdgpu_ip_block_version {
+	const enum amd_ip_block_type type;
+	const u32 major;
+	const u32 minor;
+	const u32 rev;
+	const struct amd_ip_funcs *funcs;
+};
+
+struct amdgpu_ip_block {
+	struct amdgpu_ip_block_status status;
+	const struct amdgpu_ip_block_version *version;
+	struct amdgpu_device *adev;
+};
+
 void amdgpu_ip_map_init(struct amdgpu_device *adev);
 
+int amdgpu_ip_block_suspend(struct amdgpu_ip_block *ip_block);
+int amdgpu_ip_block_resume(struct amdgpu_ip_block *ip_block);
+
+struct amdgpu_ip_block *
+amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
+			      enum amd_ip_block_type type);
+
+int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
+				       enum amd_ip_block_type type, u32 major,
+				       u32 minor);
+
+int amdgpu_device_ip_block_add(
+	struct amdgpu_device *adev,
+	const struct amdgpu_ip_block_version *ip_block_version);
+
+int amdgpu_device_ip_set_clockgating_state(struct amdgpu_device *adev,
+					   enum amd_ip_block_type block_type,
+					   enum amd_clockgating_state state);
+int amdgpu_device_ip_set_powergating_state(struct amdgpu_device *adev,
+					   enum amd_ip_block_type block_type,
+					   enum amd_powergating_state state);
+void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
+					    u64 *flags);
+int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
+				   enum amd_ip_block_type block_type);
+bool amdgpu_device_ip_is_hw(struct amdgpu_device *adev,
+			    enum amd_ip_block_type block_type);
+bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev,
+			       enum amd_ip_block_type block_type);
+
 #endif /* __AMDGPU_IP_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index 8112ffc85995..0a1bf61a1132 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -99,6 +99,41 @@ const char *soc15_ih_clientid_name[] = {
 	"MP1"
 };
 
+const char *soc_v1_0_ih_clientid_name[] = {
+	"IH",
+	"Reserved",
+	"ATHUB",
+	"BIF",
+	"Reserved",
+	"Reserved",
+	"Reserved",
+	"RLC",
+	"Reserved",
+	"Reserved",
+	"GFX",
+	"IMU",
+	"Reserved",
+	"Reserved",
+	"VCN1 or UVD1",
+	"THM",
+	"VCN or UVD",
+	"Reserved",
+	"VMC",
+	"Reserved",
+	"GRBM_CP",
+	"GC_AID",
+	"ROM_SMUIO",
+	"DF",
+	"Reserved",
+	"PWR",
+	"LSDMA",
+	"GC_UTCL2",
+	"nHT",
+	"Reserved",
+	"MP0",
+	"MP1",
+};
+
 const int node_id_to_phys_map[NODEID_MAX] = {
 	[AID0_NODEID] = 0,
 	[XCD0_NODEID] = 0,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
index 9f0417456abd..af72405a7226 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
@@ -26,6 +26,7 @@
 
 #include <linux/irqdomain.h>
 #include "soc15_ih_clientid.h"
+#include "soc_v1_0_ih_clientid.h"
 #include "amdgpu_ih.h"
 
 #define AMDGPU_MAX_IRQ_SRC_ID		0x100
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index 9c182ce501af..dffa0f7276b7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -31,6 +31,7 @@
 
 #define AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
 #define AMDGPU_ONE_DOORBELL_SIZE 8
+#define AMDGPU_MES_RESERVED_QUEUES	2
 
 int amdgpu_mes_doorbell_process_slice(struct amdgpu_device *adev)
 {
@@ -91,6 +92,9 @@ static void amdgpu_mes_doorbell_free(struct amdgpu_device *adev)
 int amdgpu_mes_init(struct amdgpu_device *adev)
 {
 	int i, r, num_pipes;
+	u32 total_vmid_mask, reserved_vmid_mask;
+	u32 queue_mask, reserved_queue_mask;
+	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
 
 	adev->mes.adev = adev;
 
@@ -101,12 +105,18 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 	spin_lock_init(&adev->mes.queue_id_lock);
 	mutex_init(&adev->mes.mutex_hidden);
 
-	for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++)
+	for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++)
 		spin_lock_init(&adev->mes.ring_lock[i]);
 
 	adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;
+	total_vmid_mask = (u32)((1UL << 16) - 1);
+	reserved_vmid_mask = (u32)((1UL << adev->vm_manager.first_kfd_vmid) - 1);
+
 	adev->mes.vmid_mask_mmhub = 0xFF00;
-	adev->mes.vmid_mask_gfxhub = adev->gfx.disable_kq ? 0xFFFE : 0xFF00;
+	adev->mes.vmid_mask_gfxhub = total_vmid_mask & ~reserved_vmid_mask;
+
+	queue_mask = (u32)(1UL << adev->gfx.mec.num_queue_per_pipe) - 1;
+	reserved_queue_mask = (u32)(1UL << AMDGPU_MES_RESERVED_QUEUES) - 1;
 
 	num_pipes = adev->gfx.me.num_pipe_per_me * adev->gfx.me.num_me;
 	if (num_pipes > AMDGPU_MES_MAX_GFX_PIPES)
@@ -142,7 +152,8 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 	for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) {
 		if (i >= num_pipes)
 			break;
-		adev->mes.compute_hqd_mask[i] = adev->gfx.disable_kq ? 0xF : 0xC;
+		adev->mes.compute_hqd_mask[i] =
+			adev->gfx.disable_kq ? 0xF : (queue_mask & ~reserved_queue_mask);
 	}
 
 	num_pipes = adev->sdma.num_instances;
@@ -156,7 +167,7 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 		adev->mes.sdma_hqd_mask[i] = 0xfc;
 	}
 
-	for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) {
+	for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) {
 		r = amdgpu_device_wb_get(adev, &adev->mes.sch_ctx_offs[i]);
 		if (r) {
 			dev_err(adev->dev,
@@ -192,16 +203,18 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 		goto error_doorbell;
 
 	if (adev->mes.hung_queue_db_array_size) {
-		r = amdgpu_bo_create_kernel(adev,
-					    adev->mes.hung_queue_db_array_size * sizeof(u32),
-					    PAGE_SIZE,
-					    AMDGPU_GEM_DOMAIN_GTT,
-					    &adev->mes.hung_queue_db_array_gpu_obj,
-					    &adev->mes.hung_queue_db_array_gpu_addr,
-					    &adev->mes.hung_queue_db_array_cpu_addr);
-		if (r) {
-			dev_warn(adev->dev, "failed to create MES hung db array buffer (%d)", r);
-			goto error_doorbell;
+		for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) {
+			r = amdgpu_bo_create_kernel(adev,
+						    adev->mes.hung_queue_db_array_size * sizeof(u32),
+						    PAGE_SIZE,
+						    AMDGPU_GEM_DOMAIN_GTT,
+						    &adev->mes.hung_queue_db_array_gpu_obj[i],
+						    &adev->mes.hung_queue_db_array_gpu_addr[i],
+						    &adev->mes.hung_queue_db_array_cpu_addr[i]);
+			if (r) {
+				dev_warn(adev->dev, "failed to create MES hung db array buffer (%d)", r);
+				goto error_doorbell;
+			}
 		}
 	}
 
@@ -210,12 +223,16 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 error_doorbell:
 	amdgpu_mes_doorbell_free(adev);
 error:
-	for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) {
+	for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) {
 		if (adev->mes.sch_ctx_ptr[i])
 			amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]);
 		if (adev->mes.query_status_fence_ptr[i])
 			amdgpu_device_wb_free(adev,
 				      adev->mes.query_status_fence_offs[i]);
+		if (adev->mes.hung_queue_db_array_gpu_obj[i])
+			amdgpu_bo_free_kernel(&adev->mes.hung_queue_db_array_gpu_obj[i],
+					      &adev->mes.hung_queue_db_array_gpu_addr[i],
+					      &adev->mes.hung_queue_db_array_cpu_addr[i]);
 	}
 
 	idr_destroy(&adev->mes.pasid_idr);
@@ -229,16 +246,17 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 void amdgpu_mes_fini(struct amdgpu_device *adev)
 {
 	int i;
-
-	amdgpu_bo_free_kernel(&adev->mes.hung_queue_db_array_gpu_obj,
-			      &adev->mes.hung_queue_db_array_gpu_addr,
-			      &adev->mes.hung_queue_db_array_cpu_addr);
+	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
 
 	amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj,
 			      &adev->mes.event_log_gpu_addr,
 			      &adev->mes.event_log_cpu_addr);
 
-	for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) {
+	for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) {
+		amdgpu_bo_free_kernel(&adev->mes.hung_queue_db_array_gpu_obj[i],
+				      &adev->mes.hung_queue_db_array_gpu_addr[i],
+				      &adev->mes.hung_queue_db_array_cpu_addr[i]);
+
 		if (adev->mes.sch_ctx_ptr[i])
 			amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]);
 		if (adev->mes.query_status_fence_ptr[i])
@@ -304,13 +322,14 @@ int amdgpu_mes_resume(struct amdgpu_device *adev)
 }
 
 int amdgpu_mes_map_legacy_queue(struct amdgpu_device *adev,
-				struct amdgpu_ring *ring)
+				struct amdgpu_ring *ring, uint32_t xcc_id)
 {
 	struct mes_map_legacy_queue_input queue_input;
 	int r;
 
 	memset(&queue_input, 0, sizeof(queue_input));
 
+	queue_input.xcc_id = xcc_id;
 	queue_input.queue_type = ring->funcs->type;
 	queue_input.doorbell_offset = ring->doorbell_index;
 	queue_input.pipe_id = ring->pipe;
@@ -330,11 +349,12 @@ int amdgpu_mes_map_legacy_queue(struct amdgpu_device *adev,
 int amdgpu_mes_unmap_legacy_queue(struct amdgpu_device *adev,
 				  struct amdgpu_ring *ring,
 				  enum amdgpu_unmap_queues_action action,
-				  u64 gpu_addr, u64 seq)
+				  u64 gpu_addr, u64 seq, uint32_t xcc_id)
 {
 	struct mes_unmap_legacy_queue_input queue_input;
 	int r;
 
+	queue_input.xcc_id = xcc_id;
 	queue_input.action = action;
 	queue_input.queue_type = ring->funcs->type;
 	queue_input.doorbell_offset = ring->doorbell_index;
@@ -355,13 +375,15 @@ int amdgpu_mes_unmap_legacy_queue(struct amdgpu_device *adev,
 int amdgpu_mes_reset_legacy_queue(struct amdgpu_device *adev,
 				  struct amdgpu_ring *ring,
 				  unsigned int vmid,
-				  bool use_mmio)
+				  bool use_mmio,
+				  uint32_t xcc_id)
 {
 	struct mes_reset_queue_input queue_input;
 	int r;
 
 	memset(&queue_input, 0, sizeof(queue_input));
 
+	queue_input.xcc_id = xcc_id;
 	queue_input.queue_type = ring->funcs->type;
 	queue_input.doorbell_offset = ring->doorbell_index;
 	queue_input.me_id = ring->me;
@@ -393,11 +415,11 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
 					    int queue_type,
 					    bool detect_only,
 					    unsigned int *hung_db_num,
-					    u32 *hung_db_array)
-
+					    u32 *hung_db_array,
+					    uint32_t xcc_id)
 {
 	struct mes_detect_and_reset_queue_input input;
-	u32 *db_array = adev->mes.hung_queue_db_array_cpu_addr;
+	u32 *db_array = adev->mes.hung_queue_db_array_cpu_addr[xcc_id];
 	int r, i;
 
 	if (!hung_db_num || !hung_db_array)
@@ -409,7 +431,7 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
 		return -EINVAL;
 
 	/* Clear the doorbell array before detection */
-	memset(adev->mes.hung_queue_db_array_cpu_addr, AMDGPU_MES_INVALID_DB_OFFSET,
+	memset(adev->mes.hung_queue_db_array_cpu_addr[xcc_id], AMDGPU_MES_INVALID_DB_OFFSET,
 		adev->mes.hung_queue_db_array_size * sizeof(u32));
 	input.queue_type = queue_type;
 	input.detect_only = detect_only;
@@ -436,7 +458,8 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
 	return r;
 }
 
-uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg)
+uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg,
+			 uint32_t xcc_id)
 {
 	struct mes_misc_op_input op_input;
 	int r, val = 0;
@@ -450,6 +473,7 @@ uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg)
 	}
 	read_val_gpu_addr = adev->wb.gpu_addr + (addr_offset * 4);
 	read_val_ptr = (uint32_t *)&adev->wb.wb[addr_offset];
+	op_input.xcc_id = xcc_id;
 	op_input.op = MES_MISC_OP_READ_REG;
 	op_input.read_reg.reg_offset = reg;
 	op_input.read_reg.buffer_addr = read_val_gpu_addr;
@@ -473,12 +497,13 @@ uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg)
 	return val;
 }
 
-int amdgpu_mes_wreg(struct amdgpu_device *adev,
-		    uint32_t reg, uint32_t val)
+int amdgpu_mes_wreg(struct amdgpu_device *adev, uint32_t reg,
+		    uint32_t val, uint32_t xcc_id)
 {
 	struct mes_misc_op_input op_input;
 	int r;
 
+	op_input.xcc_id = xcc_id;
 	op_input.op = MES_MISC_OP_WRITE_REG;
 	op_input.write_reg.reg_offset = reg;
 	op_input.write_reg.reg_value = val;
@@ -501,11 +526,13 @@ int amdgpu_mes_wreg(struct amdgpu_device *adev,
 
 int amdgpu_mes_reg_write_reg_wait(struct amdgpu_device *adev,
 				  uint32_t reg0, uint32_t reg1,
-				  uint32_t ref, uint32_t mask)
+				  uint32_t ref, uint32_t mask,
+				  uint32_t xcc_id)
 {
 	struct mes_misc_op_input op_input;
 	int r;
 
+	op_input.xcc_id = xcc_id;
 	op_input.op = MES_MISC_OP_WRM_REG_WR_WAIT;
 	op_input.wrm_reg.reg0 = reg0;
 	op_input.wrm_reg.reg1 = reg1;
@@ -530,14 +557,23 @@ int amdgpu_mes_reg_write_reg_wait(struct amdgpu_device *adev,
 
 int amdgpu_mes_hdp_flush(struct amdgpu_device *adev)
 {
-	uint32_t hdp_flush_req_offset, hdp_flush_done_offset, ref_and_mask;
+	uint32_t hdp_flush_req_offset, hdp_flush_done_offset;
+	struct amdgpu_ring *mes_ring;
+	uint32_t ref_and_mask = 0, reg_mem_engine = 0;
 
+	if (!adev->gfx.funcs->get_hdp_flush_mask) {
+		dev_err(adev->dev, "mes hdp flush is not supported.\n");
+		return -EINVAL;
+	}
+
+	mes_ring = &adev->mes.ring[0];
 	hdp_flush_req_offset = adev->nbio.funcs->get_hdp_flush_req_offset(adev);
 	hdp_flush_done_offset = adev->nbio.funcs->get_hdp_flush_done_offset(adev);
-	ref_and_mask = adev->nbio.hdp_flush_reg->ref_and_mask_cp0;
+
+	adev->gfx.funcs->get_hdp_flush_mask(mes_ring, &ref_and_mask, &reg_mem_engine);
 
 	return amdgpu_mes_reg_write_reg_wait(adev, hdp_flush_req_offset, hdp_flush_done_offset,
-					     ref_and_mask, ref_and_mask);
+					     ref_and_mask, ref_and_mask, 0);
 }
 
 int amdgpu_mes_set_shader_debugger(struct amdgpu_device *adev,
@@ -545,7 +581,8 @@ int amdgpu_mes_set_shader_debugger(struct amdgpu_device *adev,
 				uint32_t spi_gdbg_per_vmid_cntl,
 				const uint32_t *tcp_watch_cntl,
 				uint32_t flags,
-				bool trap_en)
+				bool trap_en,
+				uint32_t xcc_id)
 {
 	struct mes_misc_op_input op_input = {0};
 	int r;
@@ -556,6 +593,7 @@ int amdgpu_mes_set_shader_debugger(struct amdgpu_device *adev,
 		return -EINVAL;
 	}
 
+	op_input.xcc_id = xcc_id;
 	op_input.op = MES_MISC_OP_SET_SHADER_DEBUGGER;
 	op_input.set_shader_debugger.process_context_addr = process_context_addr;
 	op_input.set_shader_debugger.flags.u32all = flags;
@@ -584,7 +622,8 @@ int amdgpu_mes_set_shader_debugger(struct amdgpu_device *adev,
 }
 
 int amdgpu_mes_flush_shader_debugger(struct amdgpu_device *adev,
-				     uint64_t process_context_addr)
+				     uint64_t process_context_addr,
+				     uint32_t xcc_id)
 {
 	struct mes_misc_op_input op_input = {0};
 	int r;
@@ -595,6 +634,7 @@ int amdgpu_mes_flush_shader_debugger(struct amdgpu_device *adev,
 		return -EINVAL;
 	}
 
+	op_input.xcc_id = xcc_id;
 	op_input.op = MES_MISC_OP_SET_SHADER_DEBUGGER;
 	op_input.set_shader_debugger.process_context_addr = process_context_addr;
 	op_input.set_shader_debugger.flags.process_ctx_flush = true;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
index e989225b354b..88685c58798e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
@@ -58,11 +58,20 @@ enum amdgpu_mes_priority_level {
 struct amdgpu_mes_funcs;
 
 enum amdgpu_mes_pipe {
-	AMDGPU_MES_SCHED_PIPE = 0,
-	AMDGPU_MES_KIQ_PIPE,
+	AMDGPU_MES_PIPE_0 = 0,
+	AMDGPU_MES_PIPE_1,
 	AMDGPU_MAX_MES_PIPES = 2,
 };
 
+#define AMDGPU_MES_SCHED_PIPE AMDGPU_MES_PIPE_0
+#define AMDGPU_MES_KIQ_PIPE AMDGPU_MES_PIPE_1
+
+#define AMDGPU_MAX_MES_INST_PIPES \
+	(AMDGPU_MAX_MES_PIPES * AMDGPU_MAX_GC_INSTANCES)
+
+#define MES_PIPE_INST(xcc_id, pipe_id) \
+	(xcc_id * AMDGPU_MAX_MES_PIPES + pipe_id)
+
 struct amdgpu_mes {
 	struct amdgpu_device            *adev;
 
@@ -86,29 +95,29 @@ struct amdgpu_mes {
 	uint64_t                        default_process_quantum;
 	uint64_t                        default_gang_quantum;
 
-	struct amdgpu_ring              ring[AMDGPU_MAX_MES_PIPES];
-	spinlock_t                      ring_lock[AMDGPU_MAX_MES_PIPES];
+	struct amdgpu_ring              ring[AMDGPU_MAX_MES_INST_PIPES];
+	spinlock_t                      ring_lock[AMDGPU_MAX_MES_INST_PIPES];
 
 	const struct firmware           *fw[AMDGPU_MAX_MES_PIPES];
 
 	/* mes ucode */
-	struct amdgpu_bo		*ucode_fw_obj[AMDGPU_MAX_MES_PIPES];
-	uint64_t			ucode_fw_gpu_addr[AMDGPU_MAX_MES_PIPES];
-	uint32_t			*ucode_fw_ptr[AMDGPU_MAX_MES_PIPES];
+	struct amdgpu_bo		*ucode_fw_obj[AMDGPU_MAX_MES_INST_PIPES];
+	uint64_t			ucode_fw_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
+	uint32_t			*ucode_fw_ptr[AMDGPU_MAX_MES_INST_PIPES];
 	uint64_t                        uc_start_addr[AMDGPU_MAX_MES_PIPES];
 
 	/* mes ucode data */
-	struct amdgpu_bo		*data_fw_obj[AMDGPU_MAX_MES_PIPES];
-	uint64_t			data_fw_gpu_addr[AMDGPU_MAX_MES_PIPES];
-	uint32_t			*data_fw_ptr[AMDGPU_MAX_MES_PIPES];
+	struct amdgpu_bo		*data_fw_obj[AMDGPU_MAX_MES_INST_PIPES];
+	uint64_t			data_fw_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
+	uint32_t			*data_fw_ptr[AMDGPU_MAX_MES_INST_PIPES];
 	uint64_t                        data_start_addr[AMDGPU_MAX_MES_PIPES];
 
 	/* eop gpu obj */
-	struct amdgpu_bo		*eop_gpu_obj[AMDGPU_MAX_MES_PIPES];
-	uint64_t                        eop_gpu_addr[AMDGPU_MAX_MES_PIPES];
+	struct amdgpu_bo		*eop_gpu_obj[AMDGPU_MAX_MES_INST_PIPES];
+	uint64_t                        eop_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
 
-	void                            *mqd_backup[AMDGPU_MAX_MES_PIPES];
-	struct amdgpu_irq_src	        irq[AMDGPU_MAX_MES_PIPES];
+	void                            *mqd_backup[AMDGPU_MAX_MES_INST_PIPES];
+	struct amdgpu_irq_src	        irq[AMDGPU_MAX_MES_INST_PIPES];
 
 	uint32_t                        vmid_mask_gfxhub;
 	uint32_t                        vmid_mask_mmhub;
@@ -116,18 +125,21 @@ struct amdgpu_mes {
 	uint32_t                        compute_hqd_mask[AMDGPU_MES_MAX_COMPUTE_PIPES];
 	uint32_t                        sdma_hqd_mask[AMDGPU_MES_MAX_SDMA_PIPES];
 	uint32_t                        aggregated_doorbells[AMDGPU_MES_PRIORITY_NUM_LEVELS];
-	uint32_t                        sch_ctx_offs[AMDGPU_MAX_MES_PIPES];
-	uint64_t			sch_ctx_gpu_addr[AMDGPU_MAX_MES_PIPES];
-	uint64_t			*sch_ctx_ptr[AMDGPU_MAX_MES_PIPES];
-	uint32_t			query_status_fence_offs[AMDGPU_MAX_MES_PIPES];
-	uint64_t			query_status_fence_gpu_addr[AMDGPU_MAX_MES_PIPES];
-	uint64_t			*query_status_fence_ptr[AMDGPU_MAX_MES_PIPES];
+
+	uint32_t                        sch_ctx_offs[AMDGPU_MAX_MES_INST_PIPES];
+	uint64_t                        sch_ctx_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
+	uint64_t                        *sch_ctx_ptr[AMDGPU_MAX_MES_INST_PIPES];
+	uint32_t                        query_status_fence_offs[AMDGPU_MAX_MES_INST_PIPES];
+	uint64_t                        query_status_fence_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
+	uint64_t                        *query_status_fence_ptr[AMDGPU_MAX_MES_INST_PIPES];
 
 	uint32_t			saved_flags;
 
 	/* initialize kiq pipe */
-	int                             (*kiq_hw_init)(struct amdgpu_device *adev);
-	int                             (*kiq_hw_fini)(struct amdgpu_device *adev);
+	int                             (*kiq_hw_init)(struct amdgpu_device *adev,
+                                                   uint32_t xcc_id);
+	int                             (*kiq_hw_fini)(struct amdgpu_device *adev,
+                                                   uint32_t xcc_id);
 
 	/* MES doorbells */
 	uint32_t			db_start_dw_offset;
@@ -150,9 +162,15 @@ struct amdgpu_mes {
 
 	int				hung_queue_db_array_size;
 	int				hung_queue_hqd_info_offset;
-	struct amdgpu_bo		*hung_queue_db_array_gpu_obj;
-	uint64_t			hung_queue_db_array_gpu_addr;
-	void				*hung_queue_db_array_cpu_addr;
+	struct amdgpu_bo		*hung_queue_db_array_gpu_obj[AMDGPU_MAX_MES_PIPES];
+	uint64_t			hung_queue_db_array_gpu_addr[AMDGPU_MAX_MES_PIPES];
+	void				*hung_queue_db_array_cpu_addr[AMDGPU_MAX_MES_PIPES];
+
+	/* cooperative dispatch */
+	bool                enable_coop_mode;
+	int                 master_xcc_ids[AMDGPU_MAX_MES_INST_PIPES];
+	struct amdgpu_bo    *shared_cmd_buf_obj[AMDGPU_MAX_MES_INST_PIPES];
+	uint64_t            shared_cmd_buf_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
 };
 
 struct amdgpu_mes_gang {
@@ -208,6 +226,7 @@ struct amdgpu_mes_gang_properties {
 };
 
 struct mes_add_queue_input {
+	uint32_t        xcc_id;
 	uint32_t	process_id;
 	uint64_t	page_table_base_addr;
 	uint64_t	process_va_start;
@@ -234,15 +253,19 @@ struct mes_add_queue_input {
 	uint32_t	is_aql_queue;
 	uint32_t	queue_size;
 	uint32_t	exclusively_scheduled;
+	uint32_t	sh_mem_config_data;
+	uint32_t	vm_cntx_cntl;
 };
 
 struct mes_remove_queue_input {
+	uint32_t        xcc_id;
 	uint32_t	doorbell_offset;
 	uint64_t	gang_context_addr;
 	bool		remove_queue_after_reset;
 };
 
 struct mes_map_legacy_queue_input {
+	uint32_t			   xcc_id;
 	uint32_t                           queue_type;
 	uint32_t                           doorbell_offset;
 	uint32_t                           pipe_id;
@@ -252,6 +275,7 @@ struct mes_map_legacy_queue_input {
 };
 
 struct mes_unmap_legacy_queue_input {
+	uint32_t                           xcc_id;
 	enum amdgpu_unmap_queues_action    action;
 	uint32_t                           queue_type;
 	uint32_t                           doorbell_offset;
@@ -262,6 +286,7 @@ struct mes_unmap_legacy_queue_input {
 };
 
 struct mes_suspend_gang_input {
+	uint32_t        xcc_id;
 	bool		suspend_all_gangs;
 	uint64_t	gang_context_addr;
 	uint64_t	suspend_fence_addr;
@@ -269,11 +294,13 @@ struct mes_suspend_gang_input {
 };
 
 struct mes_resume_gang_input {
+	uint32_t	xcc_id;
 	bool		resume_all_gangs;
 	uint64_t	gang_context_addr;
 };
 
 struct mes_reset_queue_input {
+	uint32_t			   xcc_id;
 	uint32_t                           queue_type;
 	uint32_t                           doorbell_offset;
 	bool                               use_mmio;
@@ -309,7 +336,8 @@ enum mes_misc_opcode {
 };
 
 struct mes_misc_op_input {
-	enum mes_misc_opcode op;
+	uint32_t                 xcc_id;
+	enum mes_misc_opcode     op;
 
 	union {
 		struct {
@@ -395,8 +423,10 @@ struct amdgpu_mes_funcs {
 			      struct mes_inv_tlbs_pasid_input *input);
 };
 
-#define amdgpu_mes_kiq_hw_init(adev) (adev)->mes.kiq_hw_init((adev))
-#define amdgpu_mes_kiq_hw_fini(adev) (adev)->mes.kiq_hw_fini((adev))
+#define amdgpu_mes_kiq_hw_init(adev, xcc_id) \
+	(adev)->mes.kiq_hw_init((adev), (xcc_id))
+#define amdgpu_mes_kiq_hw_fini(adev, xcc_id) \
+	(adev)->mes.kiq_hw_fini((adev), (xcc_id))
 
 int amdgpu_mes_init_microcode(struct amdgpu_device *adev, int pipe);
 int amdgpu_mes_init(struct amdgpu_device *adev);
@@ -406,38 +436,42 @@ int amdgpu_mes_suspend(struct amdgpu_device *adev);
 int amdgpu_mes_resume(struct amdgpu_device *adev);
 
 int amdgpu_mes_map_legacy_queue(struct amdgpu_device *adev,
-				struct amdgpu_ring *ring);
+				struct amdgpu_ring *ring, uint32_t xcc_id);
 int amdgpu_mes_unmap_legacy_queue(struct amdgpu_device *adev,
 				  struct amdgpu_ring *ring,
 				  enum amdgpu_unmap_queues_action action,
-				  u64 gpu_addr, u64 seq);
+				  u64 gpu_addr, u64 seq, uint32_t xcc_id);
 int amdgpu_mes_reset_legacy_queue(struct amdgpu_device *adev,
 				  struct amdgpu_ring *ring,
 				  unsigned int vmid,
-				  bool use_mmio);
+				  bool use_mmio,
+				  uint32_t xcc_id);
 
 int amdgpu_mes_get_hung_queue_db_array_size(struct amdgpu_device *adev);
 int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
 					    int queue_type,
 					    bool detect_only,
 					    unsigned int *hung_db_num,
-					    u32 *hung_db_array);
+					    u32 *hung_db_array,
+					    uint32_t xcc_id);
 
-uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg);
+uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg,
+			 uint32_t xcc_id);
 int amdgpu_mes_wreg(struct amdgpu_device *adev,
-		    uint32_t reg, uint32_t val);
+		    uint32_t reg, uint32_t val, uint32_t xcc_id);
 int amdgpu_mes_reg_write_reg_wait(struct amdgpu_device *adev,
 				  uint32_t reg0, uint32_t reg1,
-				  uint32_t ref, uint32_t mask);
+				  uint32_t ref, uint32_t mask, uint32_t xcc_id);
 int amdgpu_mes_hdp_flush(struct amdgpu_device *adev);
 int amdgpu_mes_set_shader_debugger(struct amdgpu_device *adev,
 				uint64_t process_context_addr,
 				uint32_t spi_gdbg_per_vmid_cntl,
 				const uint32_t *tcp_watch_cntl,
 				uint32_t flags,
-				bool trap_en);
+				bool trap_en,
+				uint32_t xcc_id);
 int amdgpu_mes_flush_shader_debugger(struct amdgpu_device *adev,
-				uint64_t process_context_addr);
+				uint64_t process_context_addr, uint32_t xcc_id);
 
 uint32_t amdgpu_mes_get_aggregated_doorbell_index(struct amdgpu_device *adev,
 						   enum amdgpu_mes_priority_level prio);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index e08f58de4b17..415d8d88bbb6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -1050,7 +1050,8 @@ static const char * const amdgpu_vram_names[] = {
 	"DDR5",
 	"LPDDR4",
 	"LPDDR5",
-	"HBM3E"
+	"HBM3E",
+	"HBM4"
 };
 
 /**
@@ -1125,6 +1126,10 @@ int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	struct amdgpu_bo_user *ubo;
 
+	/* MMIO_REMAP is BAR I/O space; tiling should never be used here. */
+	WARN_ON_ONCE(bo->tbo.resource &&
+		     bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP);
+
 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
 	if (adev->family <= AMDGPU_FAMILY_CZ &&
 	    AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
@@ -1147,6 +1152,13 @@ void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
 {
 	struct amdgpu_bo_user *ubo;
 
+	/*
+	 * MMIO_REMAP BOs are not real VRAM/GTT memory but a fixed BAR I/O window.
+	 * They should never go through GEM tiling helpers.
+	 */
+	WARN_ON_ONCE(bo->tbo.resource &&
+		     bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP);
+
 	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
 	dma_resv_assert_held(bo->tbo.base.resv);
 	ubo = to_amdgpu_bo_user(bo);
@@ -1321,8 +1333,8 @@ void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)
 	if (r)
 		goto out;
 
-	r = amdgpu_fill_buffer(abo, 0, &bo->base._resv, &fence, true,
-			       AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE);
+	r = amdgpu_fill_buffer(&adev->mman.clear_entity, abo, 0, &bo->base._resv,
+			       &fence, AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE);
 	if (WARN_ON(r))
 		goto out;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 0b10497d487c..8b4c1775def3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -39,6 +39,7 @@
 #include "psp_v13_0.h"
 #include "psp_v13_0_4.h"
 #include "psp_v14_0.h"
+#include "psp_v15_0_8.h"
 
 #include "amdgpu_ras.h"
 #include "amdgpu_securedisplay.h"
@@ -259,6 +260,9 @@ static int psp_early_init(struct amdgpu_ip_block *ip_block)
 		psp_v14_0_set_psp_funcs(psp);
 		psp->boot_time_tmr = false;
 		break;
+	case IP_VERSION(15, 0, 8):
+		psp_v15_0_8_set_psp_funcs(psp);
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -901,6 +905,7 @@ static bool psp_skip_tmr(struct psp_context *psp)
 	case IP_VERSION(13, 0, 10):
 	case IP_VERSION(13, 0, 12):
 	case IP_VERSION(13, 0, 14):
+	case IP_VERSION(15, 0, 8):
 		return true;
 	default:
 		return false;
@@ -1995,6 +2000,7 @@ int psp_ras_initialize(struct psp_context *psp)
 		ras_cmd->ras_in_message.init_flags.nps_mode =
 			adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
 	ras_cmd->ras_in_message.init_flags.active_umc_mask = adev->umc.active_mask;
+	ras_cmd->ras_in_message.init_flags.vram_type = (uint8_t)adev->gmc.vram_type;
 
 	ret = psp_ta_load(psp, &psp->ras_context.context);
 
@@ -2631,7 +2637,7 @@ static int psp_hw_start(struct psp_context *psp)
 	return 0;
 }
 
-static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
+int amdgpu_psp_get_fw_type(struct amdgpu_firmware_info *ucode,
 			   enum psp_gfx_fw_type *type)
 {
 	switch (ucode->ucode_id) {
@@ -2719,6 +2725,12 @@ static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
 	case AMDGPU_UCODE_ID_RLC_DRAM:
 		*type = GFX_FW_TYPE_RLC_DRAM_BOOT;
 		break;
+	case AMDGPU_UCODE_ID_RLC_IRAM_1:
+		*type = GFX_FW_TYPE_RLX6_UCODE_CORE1;
+		break;
+	case AMDGPU_UCODE_ID_RLC_DRAM_1:
+		*type = GFX_FW_TYPE_RLX6_DRAM_BOOT_CORE1;
+		break;
 	case AMDGPU_UCODE_ID_GLOBAL_TAP_DELAYS:
 		*type = GFX_FW_TYPE_GLOBAL_TAP_DELAYS;
 		break;
@@ -2887,6 +2899,8 @@ static void psp_print_fw_hdr(struct psp_context *psp,
 		amdgpu_ucode_print_gfx_hdr(hdr);
 		break;
 	case AMDGPU_UCODE_ID_RLC_G:
+	case AMDGPU_UCODE_ID_RLC_DRAM_1:
+	case AMDGPU_UCODE_ID_RLC_IRAM_1:
 		hdr = (struct common_firmware_header *)adev->gfx.rlc_fw->data;
 		amdgpu_ucode_print_rlc_hdr(hdr);
 		break;
@@ -2911,10 +2925,9 @@ static int psp_prep_load_ip_fw_cmd_buf(struct psp_context *psp,
 	cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
 	cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
 
-	ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
+	ret = psp_get_fw_type(psp, ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
 	if (ret)
 		dev_err(psp->adev->dev, "Unknown firmware type\n");
-
 	return ret;
 }
 
@@ -3077,7 +3090,9 @@ static int psp_load_non_psp_fw(struct psp_context *psp)
 		     amdgpu_ip_version(adev, MP0_HWIP, 0) ==
 			     IP_VERSION(11, 0, 11) ||
 		     amdgpu_ip_version(adev, MP0_HWIP, 0) ==
-			     IP_VERSION(11, 0, 12)) &&
+			     IP_VERSION(11, 0, 12) ||
+		     amdgpu_ip_version(adev, MP0_HWIP, 0) ==
+			     IP_VERSION(15, 0, 8)) &&
 		    (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1 ||
 		     ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2 ||
 		     ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3))
@@ -4531,3 +4546,11 @@ const struct amdgpu_ip_block_version psp_v14_0_ip_block = {
 	.rev = 0,
 	.funcs = &psp_ip_funcs,
 };
+
+const struct amdgpu_ip_block_version psp_v15_0_8_ip_block = {
+	.type = AMD_IP_BLOCK_TYPE_PSP,
+	.major = 15,
+	.minor = 0,
+	.rev = 8,
+	.funcs = &psp_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
index 237b624aa51c..973998a2dc95 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
@@ -172,6 +172,8 @@ struct psp_funcs {
 	bool (*is_reload_needed)(struct psp_context *psp);
 	int (*reg_program_no_ring)(struct psp_context *psp, uint32_t val,
 				   enum psp_reg_prog_id id);
+	int (*get_fw_type)(struct amdgpu_firmware_info *ucode,
+			enum psp_gfx_fw_type *type);
 };
 
 struct ta_funcs {
@@ -524,6 +526,10 @@ struct amdgpu_psp_funcs {
 	((psp)->funcs->reg_program_no_ring ? \
 	(psp)->funcs->reg_program_no_ring((psp), val, id) : -EINVAL)
 
+#define psp_get_fw_type(psp, ucode, type) \
+	((psp)->funcs->get_fw_type ? \
+	(psp)->funcs->get_fw_type(ucode, type):amdgpu_psp_get_fw_type(ucode, type))
+
 extern const struct amd_ip_funcs psp_ip_funcs;
 
 extern const struct amdgpu_ip_block_version psp_v3_1_ip_block;
@@ -534,6 +540,7 @@ extern const struct amdgpu_ip_block_version psp_v12_0_ip_block;
 extern const struct amdgpu_ip_block_version psp_v13_0_ip_block;
 extern const struct amdgpu_ip_block_version psp_v13_0_4_ip_block;
 extern const struct amdgpu_ip_block_version psp_v14_0_ip_block;
+extern const struct amdgpu_ip_block_version psp_v15_0_8_ip_block;
 
 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 		 uint32_t field_val, uint32_t mask, uint32_t flags);
@@ -621,6 +628,8 @@ bool amdgpu_psp_tos_reload_needed(struct amdgpu_device *adev);
 int amdgpu_psp_reg_program_no_ring(struct psp_context *psp, uint32_t val,
 				   enum psp_reg_prog_id id);
 void amdgpu_psp_debugfs_init(struct amdgpu_device *adev);
+int amdgpu_psp_get_fw_type(struct amdgpu_firmware_info *ucode,
+			   enum psp_gfx_fw_type *type);
 
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
index 2a6cf7963dde..3d51a3c8852a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
@@ -3076,6 +3076,11 @@ static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev,
 	struct ras_err_handler_data *data = con->eh_data;
 
 	for (j = 0; j < count; j++) {
+		if (!data->space_left &&
+		    amdgpu_ras_realloc_eh_data_space(adev, data, 256)) {
+			return -ENOMEM;
+		}
+
 		if (amdgpu_ras_check_bad_page_unlock(con,
 			bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) {
 			data->count++;
@@ -3083,11 +3088,6 @@ static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev,
 			continue;
 		}
 
-		if (!data->space_left &&
-		    amdgpu_ras_realloc_eh_data_space(adev, data, 256)) {
-			return -ENOMEM;
-		}
-
 		amdgpu_ras_reserve_page(adev, bps[j].retired_page);
 
 		memcpy(&data->bps[data->count], &(bps[j]),
@@ -3249,8 +3249,6 @@ int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
 						/* deal with retire_unit records a time */
 						ret = __amdgpu_ras_convert_rec_array_from_rom(adev,
 										&bps[i], &err_data, nps);
-						if (ret)
-							con->bad_page_num -= adev->umc.retire_unit;
 						i += (adev->umc.retire_unit - 1);
 					} else {
 						break;
@@ -3263,8 +3261,6 @@ int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
 		for (; i < pages; i++) {
 			ret = __amdgpu_ras_convert_rec_from_rom(adev,
 				&bps[i], &err_data, nps);
-			if (ret)
-				con->bad_page_num -= adev->umc.retire_unit;
 		}
 
 		con->eh_data->count_saved = con->eh_data->count;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index c596b6df2e2d..1aee207b1352 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -76,8 +76,12 @@ unsigned int amdgpu_ring_max_ibs(enum amdgpu_ring_type type)
  * @ring: amdgpu_ring structure holding ring information
  * @ndw: number of dwords to allocate in the ring buffer
  *
- * Allocate @ndw dwords in the ring buffer (all asics).
- * Returns 0 on success, error on failure.
+ * Allocate @ndw dwords in the ring buffer. The number of dwords should be the
+ * sum of all commands written to the ring.
+ *
+ * Returns:
+ * 0 on success, otherwise -ENOMEM if it tries to allocate more than the
+ * maximum dword allowed for one submission.
  */
 int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned int ndw)
 {
@@ -123,7 +127,8 @@ static void amdgpu_ring_alloc_reemit(struct amdgpu_ring *ring, unsigned int ndw)
 		ring->funcs->begin_use(ring);
 }
 
-/** amdgpu_ring_insert_nop - insert NOP packets
+/**
+ * amdgpu_ring_insert_nop - insert NOP packets
  *
  * @ring: amdgpu_ring structure holding ring information
  * @count: the number of NOP packets to insert
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 7a27c6c4bb44..a1fb0fadb6ea 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -62,6 +62,8 @@ enum amdgpu_ring_priority_level {
 #define AMDGPU_FENCE_FLAG_64BIT         (1 << 0)
 #define AMDGPU_FENCE_FLAG_INT           (1 << 1)
 #define AMDGPU_FENCE_FLAG_TC_WB_ONLY    (1 << 2)
+
+/* Ensure the execution in case of preemption or reset */
 #define AMDGPU_FENCE_FLAG_EXEC          (1 << 3)
 
 #define to_amdgpu_ring(s) container_of((s), struct amdgpu_ring, sched)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c
index 5aa830a02d80..572a60e1b3cb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c
@@ -515,6 +515,40 @@ static void amdgpu_gfx_rlc_init_microcode_v2_4(struct amdgpu_device *adev)
 	}
 }
 
+static void amdgpu_gfx_rlc_init_microcode_v2_5(struct amdgpu_device *adev)
+{
+	const struct rlc_firmware_header_v2_5 *rlc_hdr;
+	struct amdgpu_firmware_info *info;
+
+	rlc_hdr = (const struct rlc_firmware_header_v2_5 *)adev->gfx.rlc_fw->data;
+	adev->gfx.rlc.rlc_1_iram_ucode_size_bytes =
+		le32_to_cpu(rlc_hdr->rlc_1_iram_ucode_size_bytes);
+	adev->gfx.rlc.rlc_1_iram_ucode = (u8 *)rlc_hdr +
+		le32_to_cpu(rlc_hdr->rlc_1_iram_ucode_offset_bytes);
+	adev->gfx.rlc.rlc_1_dram_ucode_size_bytes =
+		le32_to_cpu(rlc_hdr->rlc_1_dram_ucode_size_bytes);
+	adev->gfx.rlc.rlc_1_dram_ucode = (u8 *)rlc_hdr +
+		le32_to_cpu(rlc_hdr->rlc_1_dram_ucode_offset_bytes);
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		if (adev->gfx.rlc.rlc_1_iram_ucode_size_bytes) {
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_IRAM_1];
+			info->ucode_id = AMDGPU_UCODE_ID_RLC_IRAM_1;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.rlc_1_iram_ucode_size_bytes, PAGE_SIZE);
+		}
+
+		if (adev->gfx.rlc.rlc_1_dram_ucode_size_bytes) {
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_DRAM_1];
+			info->ucode_id = AMDGPU_UCODE_ID_RLC_DRAM_1;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.rlc_1_dram_ucode_size_bytes, PAGE_SIZE);
+		}
+	}
+}
+
 int amdgpu_gfx_rlc_init_microcode(struct amdgpu_device *adev,
 				  uint16_t version_major,
 				  uint16_t version_minor)
@@ -545,6 +579,7 @@ int amdgpu_gfx_rlc_init_microcode(struct amdgpu_device *adev,
 		amdgpu_gfx_rlc_init_microcode_v2_3(adev);
 	if (version_minor == 4)
 		amdgpu_gfx_rlc_init_microcode_v2_4(adev);
-
+	if (version_minor == 5)
+		amdgpu_gfx_rlc_init_microcode_v2_5(adev);
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
index 2ce310b31942..32408574548d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
@@ -257,7 +257,8 @@ struct amdgpu_rlc_funcs {
 	void (*stop)(struct amdgpu_device *adev);
 	void (*reset)(struct amdgpu_device *adev);
 	void (*start)(struct amdgpu_device *adev);
-	void (*update_spm_vmid)(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned vmid);
+	void (*update_spm_vmid)(struct amdgpu_device *adev, int xcc_id,
+			struct amdgpu_ring *ring, unsigned vmid);
 	bool (*is_rlcg_access_range)(struct amdgpu_device *adev, uint32_t reg);
 };
 
@@ -310,6 +311,8 @@ struct amdgpu_rlc {
 	u32 save_restore_list_srm_size_bytes;
 	u32 rlc_iram_ucode_size_bytes;
 	u32 rlc_dram_ucode_size_bytes;
+	u32 rlc_1_iram_ucode_size_bytes;
+	u32 rlc_1_dram_ucode_size_bytes;
 	u32 rlcp_ucode_size_bytes;
 	u32 rlcv_ucode_size_bytes;
 	u32 global_tap_delays_ucode_size_bytes;
@@ -325,6 +328,8 @@ struct amdgpu_rlc {
 	u8 *save_restore_list_srm;
 	u8 *rlc_iram_ucode;
 	u8 *rlc_dram_ucode;
+	u8 *rlc_1_iram_ucode;
+	u8 *rlc_1_dram_ucode;
 	u8 *rlcp_ucode;
 	u8 *rlcv_ucode;
 	u8 *global_tap_delays_ucode;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
index 34311f32be4c..2bf365609775 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
@@ -50,6 +50,11 @@ enum amdgpu_sdma_irq {
 
 #define NUM_SDMA(x) hweight32(x)
 
+struct amdgpu_sdma_csa_info {
+	u32 size;
+	u32 alignment;
+};
+
 struct amdgpu_sdma_funcs {
 	int (*stop_kernel_queue)(struct amdgpu_ring *ring);
 	int (*start_kernel_queue)(struct amdgpu_ring *ring);
@@ -65,7 +70,10 @@ struct amdgpu_sdma_instance {
 	struct amdgpu_ring	ring;
 	struct amdgpu_ring	page;
 	bool			burst_nop;
-	uint32_t		aid_id;
+	union {
+	    uint32_t		aid_id;
+	    uint32_t		xcc_id;
+	};
 
 	struct amdgpu_bo	*sdma_fw_obj;
 	uint64_t		sdma_fw_gpu_addr;
@@ -123,7 +131,10 @@ struct amdgpu_sdma {
 
 	int			num_instances;
 	uint32_t 		sdma_mask;
-	int			num_inst_per_aid;
+	union {
+	    int			num_inst_per_aid;
+	    int			num_inst_per_xcc;
+	};
 	uint32_t                    srbm_soft_reset;
 	bool			has_page_queue;
 	struct ras_common_if	*ras_if;
@@ -133,6 +144,8 @@ struct amdgpu_sdma {
 	struct list_head	reset_callback_list;
 	bool			no_user_submission;
 	bool			disable_uq;
+	void (*get_csa_info)(struct amdgpu_device *adev,
+			     struct amdgpu_sdma_csa_info *csa_info);
 };
 
 /*
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_smuio.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_smuio.h
index ec9d12f85f39..124b13a68f3f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_smuio.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_smuio.h
@@ -27,6 +27,7 @@ enum amdgpu_pkg_type {
 	AMDGPU_PKG_TYPE_APU = 2,
 	AMDGPU_PKG_TYPE_CEM = 3,
 	AMDGPU_PKG_TYPE_OAM = 4,
+	AMDGPU_PKG_TYPE_BB  = 5,
 	AMDGPU_PKG_TYPE_UNKNOWN,
 };
 
@@ -44,6 +45,8 @@ struct amdgpu_smuio_funcs {
 	u32 (*get_socket_id)(struct amdgpu_device *adev);
 	enum amdgpu_pkg_type (*get_pkg_type)(struct amdgpu_device *adev);
 	bool (*is_host_gpu_xgmi_supported)(struct amdgpu_device *adev);
+	bool (*is_connected_with_ethernet_switch)(struct amdgpu_device *adev);
+	bool (*is_custom_hbm_supported)(struct amdgpu_device *adev);
 	u64 (*get_gpu_clock_counter)(struct amdgpu_device *adev);
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 2b931e855abd..aff0da62cc7f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -162,13 +162,25 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 	*placement = abo->placement;
 }
 
+static struct dma_fence *
+amdgpu_ttm_job_submit(struct amdgpu_device *adev, struct amdgpu_job *job, u32 num_dw)
+{
+	struct amdgpu_ring *ring;
+
+	ring = adev->mman.buffer_funcs_ring;
+	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
+	WARN_ON(job->ibs[0].length_dw > num_dw);
+
+	return amdgpu_job_submit(job);
+}
+
 /**
  * amdgpu_ttm_map_buffer - Map memory into the GART windows
+ * @entity: entity to run the window setup job
  * @bo: buffer object to map
  * @mem: memory object to map
  * @mm_cur: range to map
  * @window: which GART window to use
- * @ring: DMA ring to use for the copy
  * @tmz: if we should setup a TMZ enabled mapping
  * @size: in number of bytes to map, out number of bytes mapped
  * @addr: resulting address inside the MC address space
@@ -176,13 +188,14 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
  * Setup one of the GART windows to access a specific piece of memory or return
  * the physical address for local memory.
  */
-static int amdgpu_ttm_map_buffer(struct ttm_buffer_object *bo,
+static int amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity *entity,
+				 struct ttm_buffer_object *bo,
 				 struct ttm_resource *mem,
 				 struct amdgpu_res_cursor *mm_cur,
-				 unsigned int window, struct amdgpu_ring *ring,
+				 unsigned int window,
 				 bool tmz, uint64_t *size, uint64_t *addr)
 {
-	struct amdgpu_device *adev = ring->adev;
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 	unsigned int offset, num_pages, num_dw, num_bytes;
 	uint64_t src_addr, dst_addr;
 	struct amdgpu_job *job;
@@ -223,7 +236,7 @@ static int amdgpu_ttm_map_buffer(struct ttm_buffer_object *bo,
 	num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
 	num_bytes = num_pages * 8 * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 
-	r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr,
+	r = amdgpu_job_alloc_with_ib(adev, &entity->base,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     num_dw * 4 + num_bytes,
 				     AMDGPU_IB_POOL_DELAYED, &job,
@@ -239,9 +252,6 @@ static int amdgpu_ttm_map_buffer(struct ttm_buffer_object *bo,
 	amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
 				dst_addr, num_bytes, 0);
 
-	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
-	WARN_ON(job->ibs[0].length_dw > num_dw);
-
 	flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, mem);
 	if (tmz)
 		flags |= AMDGPU_PTE_TMZ;
@@ -259,13 +269,14 @@ static int amdgpu_ttm_map_buffer(struct ttm_buffer_object *bo,
 		amdgpu_gart_map_vram_range(adev, pa, 0, num_pages, flags, cpu_addr);
 	}
 
-	dma_fence_put(amdgpu_job_submit(job));
+	dma_fence_put(amdgpu_ttm_job_submit(adev, job, num_dw));
 	return 0;
 }
 
 /**
  * amdgpu_ttm_copy_mem_to_mem - Helper function for copy
  * @adev: amdgpu device
+ * @entity: entity to run the jobs
  * @src: buffer/address where to read from
  * @dst: buffer/address where to write to
  * @size: number of bytes to copy
@@ -280,13 +291,13 @@ static int amdgpu_ttm_map_buffer(struct ttm_buffer_object *bo,
  */
 __attribute__((nonnull))
 static int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
+				      struct amdgpu_ttm_buffer_entity *entity,
 				      const struct amdgpu_copy_mem *src,
 				      const struct amdgpu_copy_mem *dst,
 				      uint64_t size, bool tmz,
 				      struct dma_resv *resv,
 				      struct dma_fence **f)
 {
-	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 	struct amdgpu_res_cursor src_mm, dst_mm;
 	struct dma_fence *fence = NULL;
 	int r = 0;
@@ -312,13 +323,13 @@ static int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
 		cur_size = min3(src_mm.size, dst_mm.size, 256ULL << 20);
 
 		/* Map src to window 0 and dst to window 1. */
-		r = amdgpu_ttm_map_buffer(src->bo, src->mem, &src_mm,
-					  0, ring, tmz, &cur_size, &from);
+		r = amdgpu_ttm_map_buffer(entity, src->bo, src->mem, &src_mm,
+					  0, tmz, &cur_size, &from);
 		if (r)
 			goto error;
 
-		r = amdgpu_ttm_map_buffer(dst->bo, dst->mem, &dst_mm,
-					  1, ring, tmz, &cur_size, &to);
+		r = amdgpu_ttm_map_buffer(entity, dst->bo, dst->mem, &dst_mm,
+					  1, tmz, &cur_size, &to);
 		if (r)
 			goto error;
 
@@ -345,8 +356,8 @@ static int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
 							     write_compress_disable));
 		}
 
-		r = amdgpu_copy_buffer(ring, from, to, cur_size, resv,
-				       &next, false, true, copy_flags);
+		r = amdgpu_copy_buffer(adev, entity, from, to, cur_size, resv,
+				       &next, true, copy_flags);
 		if (r)
 			goto error;
 
@@ -386,7 +397,9 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo,
 	src.offset = 0;
 	dst.offset = 0;
 
-	r = amdgpu_ttm_copy_mem_to_mem(adev, &src, &dst,
+	r = amdgpu_ttm_copy_mem_to_mem(adev,
+				       &adev->mman.move_entity,
+				       &src, &dst,
 				       new_mem->size,
 				       amdgpu_bo_encrypted(abo),
 				       bo->base.resv, &fence);
@@ -398,8 +411,9 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo,
 	    (abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) {
 		struct dma_fence *wipe_fence = NULL;
 
-		r = amdgpu_fill_buffer(abo, 0, NULL, &wipe_fence,
-				       false, AMDGPU_KERNEL_JOB_ID_MOVE_BLIT);
+		r = amdgpu_fill_buffer(&adev->mman.move_entity,
+				       abo, 0, NULL, &wipe_fence,
+				       AMDGPU_KERNEL_JOB_ID_MOVE_BLIT);
 		if (r) {
 			goto error;
 		} else if (wipe_fence) {
@@ -1062,6 +1076,86 @@ static void amdgpu_ttm_backend_destroy(struct ttm_device *bdev,
 	kfree(gtt);
 }
 
+/**
+ * amdgpu_ttm_mmio_remap_alloc_sgt - build an sg_table for MMIO_REMAP I/O aperture
+ * @adev: amdgpu device providing the remap BAR base (adev->rmmio_remap.bus_addr)
+ * @res:  TTM resource of the BO to export; expected to live in AMDGPU_PL_MMIO_REMAP
+ * @dev:  importing device to map for (typically @attach->dev in dma-buf paths)
+ * @dir:  DMA data direction for the importer (passed to dma_map_resource())
+ * @sgt:  output; on success, set to a newly allocated sg_table describing the I/O span
+ *
+ * The HDP flush page (AMDGPU_PL_MMIO_REMAP) is a fixed hardware I/O window in a PCI
+ * BAR—there are no struct pages to back it. Importers still need a DMA address list,
+ * so we synthesize a minimal sg_table and populate it from dma_map_resource(), not
+ * from pages. Using the common amdgpu_res_cursor walker keeps the offset/size math
+ * consistent with other TTM/manager users.
+ *
+ * - @res is assumed to be a small, contiguous I/O region (typically a single 4 KiB
+ *   page) in AMDGPU_PL_MMIO_REMAP. Callers should validate placement before calling.
+ * - The sg entry is created with sg_set_page(sg, NULL, …) to reflect I/O space.
+ * - The mapping uses DMA_ATTR_SKIP_CPU_SYNC because this is MMIO, not cacheable RAM.
+ * - Peer reachability / p2pdma policy checks must be done by the caller.
+ *
+ * Return:
+ * * 0 on success, with *@sgt set to a valid table that must be freed via
+ *   amdgpu_ttm_mmio_remap_free_sgt().
+ * * -ENOMEM if allocation of the sg_table fails.
+ * * -EIO if dma_map_resource() fails.
+ *
+ */
+int amdgpu_ttm_mmio_remap_alloc_sgt(struct amdgpu_device *adev,
+				    struct ttm_resource *res,
+				    struct device *dev,
+				    enum dma_data_direction dir,
+				    struct sg_table **sgt)
+{
+	struct amdgpu_res_cursor cur;
+	dma_addr_t dma;
+	resource_size_t phys;
+	struct scatterlist *sg;
+	int r;
+
+	/* Walk the resource once; MMIO_REMAP is expected to be contiguous+small. */
+	amdgpu_res_first(res, 0, res->size, &cur);
+
+	/* Translate byte offset in the remap window into a host physical BAR address. */
+	phys = adev->rmmio_remap.bus_addr + cur.start;
+
+	/* Build a single-entry sg_table mapped as I/O (no struct page backing). */
+	*sgt = kzalloc(sizeof(**sgt), GFP_KERNEL);
+	if (!*sgt)
+		return -ENOMEM;
+	r = sg_alloc_table(*sgt, 1, GFP_KERNEL);
+	if (r) {
+		kfree(*sgt);
+		return r;
+	}
+	sg = (*sgt)->sgl;
+	sg_set_page(sg, NULL, cur.size, 0);  /* WHY: I/O space → no pages */
+
+	dma = dma_map_resource(dev, phys, cur.size, dir, DMA_ATTR_SKIP_CPU_SYNC);
+	if (dma_mapping_error(dev, dma)) {
+		sg_free_table(*sgt);
+		kfree(*sgt);
+		return -EIO;
+	}
+	sg_dma_address(sg) = dma;
+	sg_dma_len(sg) = cur.size;
+	return 0;
+}
+
+void amdgpu_ttm_mmio_remap_free_sgt(struct device *dev,
+				    enum dma_data_direction dir,
+				    struct sg_table *sgt)
+{
+	struct scatterlist *sg = sgt->sgl;
+
+	dma_unmap_resource(dev, sg_dma_address(sg), sg_dma_len(sg),
+			   dir, DMA_ATTR_SKIP_CPU_SYNC);
+	sg_free_table(sgt);
+	kfree(sgt);
+}
+
 /**
  * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
  *
@@ -1478,7 +1572,7 @@ static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo,
 		memcpy(adev->mman.sdma_access_ptr, buf, len);
 
 	num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
-	r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr,
+	r = amdgpu_job_alloc_with_ib(adev, &adev->mman.default_entity.base,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     num_dw * 4, AMDGPU_IB_POOL_DELAYED,
 				     &job,
@@ -1497,10 +1591,7 @@ static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo,
 	amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, dst_addr,
 				PAGE_SIZE, 0);
 
-	amdgpu_ring_pad_ib(adev->mman.buffer_funcs_ring, &job->ibs[0]);
-	WARN_ON(job->ibs[0].length_dw > num_dw);
-
-	fence = amdgpu_job_submit(job);
+	fence = amdgpu_ttm_job_submit(adev, job, num_dw);
 	mutex_unlock(&adev->mman.gtt_window_lock);
 
 	if (!dma_fence_wait_timeout(fence, false, adev->sdma_timeout))
@@ -1820,6 +1911,10 @@ static void amdgpu_ttm_pools_fini(struct amdgpu_device *adev)
  * PAGE_SIZE is <= AMDGPU_GPU_PAGE_SIZE (4K). The BO is created as a regular
  * GEM object (amdgpu_bo_create).
  *
+ * The BO is created as a normal GEM object via amdgpu_bo_create(), then
+ * reserved and pinned at the TTM level (ttm_bo_pin()) so it can never be
+ * migrated or evicted. No CPU mapping is established here.
+ *
  * Return:
  *  * 0 on success or intentional skip (feature not present/unsupported)
  *  * negative errno on allocation failure
@@ -1848,7 +1943,26 @@ static int amdgpu_ttm_mmio_remap_bo_init(struct amdgpu_device *adev)
 	if (r)
 		return r;
 
+	r = amdgpu_bo_reserve(adev->rmmio_remap.bo, true);
+	if (r)
+		goto err_unref;
+
+	/*
+	 * MMIO_REMAP is a fixed I/O placement (AMDGPU_PL_MMIO_REMAP).
+	 * Use TTM-level pin so the BO cannot be evicted/migrated,
+	 * independent of GEM domains. This
+	 * enforces the “fixed I/O window”
+	 */
+	ttm_bo_pin(&adev->rmmio_remap.bo->tbo);
+
+	amdgpu_bo_unreserve(adev->rmmio_remap.bo);
 	return 0;
+
+err_unref:
+	if (adev->rmmio_remap.bo)
+		amdgpu_bo_unref(&adev->rmmio_remap.bo);
+	adev->rmmio_remap.bo = NULL;
+	return r;
 }
 
 /**
@@ -1860,6 +1974,15 @@ static int amdgpu_ttm_mmio_remap_bo_init(struct amdgpu_device *adev)
  */
 static void amdgpu_ttm_mmio_remap_bo_fini(struct amdgpu_device *adev)
 {
+	struct amdgpu_bo *bo = adev->rmmio_remap.bo;
+
+	if (!bo)
+		return;   /* <-- safest early exit */
+
+	if (!amdgpu_bo_reserve(adev->rmmio_remap.bo, true)) {
+		ttm_bo_unpin(&adev->rmmio_remap.bo->tbo);
+		amdgpu_bo_unreserve(adev->rmmio_remap.bo);
+	}
 	amdgpu_bo_unref(&adev->rmmio_remap.bo);
 	adev->rmmio_remap.bo = NULL;
 }
@@ -2165,7 +2288,7 @@ void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
 
 		ring = adev->mman.buffer_funcs_ring;
 		sched = &ring->sched;
-		r = drm_sched_entity_init(&adev->mman.high_pr,
+		r = drm_sched_entity_init(&adev->mman.default_entity.base,
 					  DRM_SCHED_PRIORITY_KERNEL, &sched,
 					  1, NULL);
 		if (r) {
@@ -2175,18 +2298,30 @@ void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
 			return;
 		}
 
-		r = drm_sched_entity_init(&adev->mman.low_pr,
+		r = drm_sched_entity_init(&adev->mman.clear_entity.base,
+					  DRM_SCHED_PRIORITY_NORMAL, &sched,
+					  1, NULL);
+		if (r) {
+			dev_err(adev->dev,
+				"Failed setting up TTM BO clear entity (%d)\n",
+				r);
+			goto error_free_entity;
+		}
+
+		r = drm_sched_entity_init(&adev->mman.move_entity.base,
 					  DRM_SCHED_PRIORITY_NORMAL, &sched,
 					  1, NULL);
 		if (r) {
 			dev_err(adev->dev,
 				"Failed setting up TTM BO move entity (%d)\n",
 				r);
+			drm_sched_entity_destroy(&adev->mman.clear_entity.base);
 			goto error_free_entity;
 		}
 	} else {
-		drm_sched_entity_destroy(&adev->mman.high_pr);
-		drm_sched_entity_destroy(&adev->mman.low_pr);
+		drm_sched_entity_destroy(&adev->mman.default_entity.base);
+		drm_sched_entity_destroy(&adev->mman.clear_entity.base);
+		drm_sched_entity_destroy(&adev->mman.move_entity.base);
 		/* Drop all the old fences since re-creating the scheduler entities
 		 * will allocate new contexts.
 		 */
@@ -2204,24 +2339,20 @@ void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
 	return;
 
 error_free_entity:
-	drm_sched_entity_destroy(&adev->mman.high_pr);
+	drm_sched_entity_destroy(&adev->mman.default_entity.base);
 }
 
 static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev,
-				  bool direct_submit,
+				  struct amdgpu_ttm_buffer_entity *entity,
 				  unsigned int num_dw,
 				  struct dma_resv *resv,
 				  bool vm_needs_flush,
 				  struct amdgpu_job **job,
-				  bool delayed, u64 k_job_id)
+				  u64 k_job_id)
 {
-	enum amdgpu_ib_pool_type pool = direct_submit ?
-		AMDGPU_IB_POOL_DIRECT :
-		AMDGPU_IB_POOL_DELAYED;
+	enum amdgpu_ib_pool_type pool = AMDGPU_IB_POOL_DELAYED;
 	int r;
-	struct drm_sched_entity *entity = delayed ? &adev->mman.low_pr :
-						    &adev->mman.high_pr;
-	r = amdgpu_job_alloc_with_ib(adev, entity,
+	r = amdgpu_job_alloc_with_ib(adev, &entity->base,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     num_dw * 4, pool, job, k_job_id);
 	if (r)
@@ -2240,20 +2371,24 @@ static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev,
 						   DMA_RESV_USAGE_BOOKKEEP);
 }
 
-int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
+int amdgpu_copy_buffer(struct amdgpu_device *adev,
+		       struct amdgpu_ttm_buffer_entity *entity,
+		       uint64_t src_offset,
 		       uint64_t dst_offset, uint32_t byte_count,
 		       struct dma_resv *resv,
-		       struct dma_fence **fence, bool direct_submit,
+		       struct dma_fence **fence,
 		       bool vm_needs_flush, uint32_t copy_flags)
 {
-	struct amdgpu_device *adev = ring->adev;
 	unsigned int num_loops, num_dw;
+	struct amdgpu_ring *ring;
 	struct amdgpu_job *job;
 	uint32_t max_bytes;
 	unsigned int i;
 	int r;
 
-	if (!direct_submit && !ring->sched.ready) {
+	ring = adev->mman.buffer_funcs_ring;
+
+	if (!ring->sched.ready) {
 		dev_err(adev->dev,
 			"Trying to move memory with ring turned off.\n");
 		return -EINVAL;
@@ -2262,11 +2397,11 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
 	max_bytes = adev->mman.buffer_funcs->copy_max_bytes;
 	num_loops = DIV_ROUND_UP(byte_count, max_bytes);
 	num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8);
-	r = amdgpu_ttm_prepare_job(adev, direct_submit, num_dw,
-				   resv, vm_needs_flush, &job, false,
+	r = amdgpu_ttm_prepare_job(adev, entity, num_dw,
+				   resv, vm_needs_flush, &job,
 				   AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER);
 	if (r)
-		return r;
+		goto error_free;
 
 	for (i = 0; i < num_loops; i++) {
 		uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
@@ -2278,16 +2413,9 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
 		byte_count -= cur_size_in_bytes;
 	}
 
-	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
-	WARN_ON(job->ibs[0].length_dw > num_dw);
-	if (direct_submit)
-		r = amdgpu_job_submit_direct(job, ring, fence);
-	else
-		*fence = amdgpu_job_submit(job);
-	if (r)
-		goto error_free;
+	*fence = amdgpu_ttm_job_submit(adev, job, num_dw);
 
-	return r;
+	return 0;
 
 error_free:
 	amdgpu_job_free(job);
@@ -2295,14 +2423,15 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
 	return r;
 }
 
-static int amdgpu_ttm_fill_mem(struct amdgpu_ring *ring, uint32_t src_data,
+static int amdgpu_ttm_fill_mem(struct amdgpu_device *adev,
+			       struct amdgpu_ttm_buffer_entity *entity,
+			       uint32_t src_data,
 			       uint64_t dst_addr, uint32_t byte_count,
 			       struct dma_resv *resv,
 			       struct dma_fence **fence,
-			       bool vm_needs_flush, bool delayed,
+			       bool vm_needs_flush,
 			       u64 k_job_id)
 {
-	struct amdgpu_device *adev = ring->adev;
 	unsigned int num_loops, num_dw;
 	struct amdgpu_job *job;
 	uint32_t max_bytes;
@@ -2312,8 +2441,8 @@ static int amdgpu_ttm_fill_mem(struct amdgpu_ring *ring, uint32_t src_data,
 	max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
 	num_loops = DIV_ROUND_UP_ULL(byte_count, max_bytes);
 	num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->fill_num_dw, 8);
-	r = amdgpu_ttm_prepare_job(adev, false, num_dw, resv, vm_needs_flush,
-				   &job, delayed, k_job_id);
+	r = amdgpu_ttm_prepare_job(adev, entity, num_dw, resv,
+				   vm_needs_flush, &job, k_job_id);
 	if (r)
 		return r;
 
@@ -2327,9 +2456,7 @@ static int amdgpu_ttm_fill_mem(struct amdgpu_ring *ring, uint32_t src_data,
 		byte_count -= cur_size;
 	}
 
-	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
-	WARN_ON(job->ibs[0].length_dw > num_dw);
-	*fence = amdgpu_job_submit(job);
+	*fence = amdgpu_ttm_job_submit(adev, job, num_dw);
 	return 0;
 }
 
@@ -2349,7 +2476,6 @@ int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,
 			    struct dma_fence **fence)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
-	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 	struct amdgpu_res_cursor cursor;
 	u64 addr;
 	int r = 0;
@@ -2377,13 +2503,14 @@ int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,
 		/* Never clear more than 256MiB at once to avoid timeouts */
 		size = min(cursor.size, 256ULL << 20);
 
-		r = amdgpu_ttm_map_buffer(&bo->tbo, bo->tbo.resource, &cursor,
-					  1, ring, false, &size, &addr);
+		r = amdgpu_ttm_map_buffer(&adev->mman.clear_entity,
+					  &bo->tbo, bo->tbo.resource, &cursor,
+					  1, false, &size, &addr);
 		if (r)
 			goto err;
 
-		r = amdgpu_ttm_fill_mem(ring, 0, addr, size, resv,
-					&next, true, true,
+		r = amdgpu_ttm_fill_mem(adev, &adev->mman.clear_entity, 0, addr, size, resv,
+					&next, true,
 					AMDGPU_KERNEL_JOB_ID_TTM_CLEAR_BUFFER);
 		if (r)
 			goto err;
@@ -2399,15 +2526,14 @@ int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,
 	return r;
 }
 
-int amdgpu_fill_buffer(struct amdgpu_bo *bo,
-			uint32_t src_data,
-			struct dma_resv *resv,
-			struct dma_fence **f,
-			bool delayed,
-			u64 k_job_id)
+int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity,
+		       struct amdgpu_bo *bo,
+		       uint32_t src_data,
+		       struct dma_resv *resv,
+		       struct dma_fence **f,
+		       u64 k_job_id)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
-	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 	struct dma_fence *fence = NULL;
 	struct amdgpu_res_cursor dst;
 	int r;
@@ -2428,13 +2554,14 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
 		/* Never fill more than 256MiB at once to avoid timeouts */
 		cur_size = min(dst.size, 256ULL << 20);
 
-		r = amdgpu_ttm_map_buffer(&bo->tbo, bo->tbo.resource, &dst,
-					  1, ring, false, &cur_size, &to);
+		r = amdgpu_ttm_map_buffer(entity, &bo->tbo, bo->tbo.resource, &dst,
+					  1, false, &cur_size, &to);
 		if (r)
 			goto error;
 
-		r = amdgpu_ttm_fill_mem(ring, src_data, to, cur_size, resv,
-					&next, true, delayed, k_job_id);
+		r = amdgpu_ttm_fill_mem(adev, entity,
+					src_data, to, cur_size, resv,
+					&next, true, k_job_id);
 		if (r)
 			goto error;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 577ee04ce0bf..72488124aa59 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -52,6 +52,10 @@ struct amdgpu_gtt_mgr {
 	spinlock_t lock;
 };
 
+struct amdgpu_ttm_buffer_entity {
+	struct drm_sched_entity base;
+};
+
 struct amdgpu_mman {
 	struct ttm_device		bdev;
 	struct ttm_pool			*ttm_pools;
@@ -64,10 +68,10 @@ struct amdgpu_mman {
 	bool					buffer_funcs_enabled;
 
 	struct mutex				gtt_window_lock;
-	/* High priority scheduler entity for buffer moves */
-	struct drm_sched_entity			high_pr;
-	/* Low priority scheduler entity for VRAM clearing */
-	struct drm_sched_entity			low_pr;
+
+	struct amdgpu_ttm_buffer_entity default_entity;
+	struct amdgpu_ttm_buffer_entity clear_entity;
+	struct amdgpu_ttm_buffer_entity move_entity;
 
 	struct amdgpu_vram_mgr vram_mgr;
 	struct amdgpu_gtt_mgr gtt_mgr;
@@ -163,20 +167,22 @@ int amdgpu_ttm_init(struct amdgpu_device *adev);
 void amdgpu_ttm_fini(struct amdgpu_device *adev);
 void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev,
 					bool enable);
-int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
+int amdgpu_copy_buffer(struct amdgpu_device *adev,
+		       struct amdgpu_ttm_buffer_entity *entity,
+		       uint64_t src_offset,
 		       uint64_t dst_offset, uint32_t byte_count,
 		       struct dma_resv *resv,
-		       struct dma_fence **fence, bool direct_submit,
+		       struct dma_fence **fence,
 		       bool vm_needs_flush, uint32_t copy_flags);
 int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,
 			    struct dma_resv *resv,
 			    struct dma_fence **fence);
-int amdgpu_fill_buffer(struct amdgpu_bo *bo,
-			uint32_t src_data,
-			struct dma_resv *resv,
-			struct dma_fence **fence,
-			bool delayed,
-			u64 k_job_id);
+int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity,
+		       struct amdgpu_bo *bo,
+		       uint32_t src_data,
+		       struct dma_resv *resv,
+		       struct dma_fence **f,
+		       u64 k_job_id);
 
 int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo);
 void amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo);
@@ -213,4 +219,13 @@ int amdgpu_ttm_evict_resources(struct amdgpu_device *adev, int mem_type);
 
 void amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);
 
+int amdgpu_ttm_mmio_remap_alloc_sgt(struct amdgpu_device *adev,
+				    struct ttm_resource *res,
+				    struct device *dev,
+				    enum dma_data_direction dir,
+				    struct sg_table **sgt);
+void amdgpu_ttm_mmio_remap_free_sgt(struct device *dev,
+				    enum dma_data_direction dir,
+				    struct sg_table *sgt);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
index e96f24e9ad57..1ab61e7b35db 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
@@ -166,6 +166,8 @@ void amdgpu_ucode_print_rlc_hdr(const struct common_firmware_header *hdr)
 			container_of(rlc_hdr_v2_2, struct rlc_firmware_header_v2_3, v2_2);
 		const struct rlc_firmware_header_v2_4 *rlc_hdr_v2_4 =
 			container_of(rlc_hdr_v2_3, struct rlc_firmware_header_v2_4, v2_3);
+		const struct rlc_firmware_header_v2_5 *rlc_hdr_v2_5 =
+			container_of(rlc_hdr_v2_2, struct rlc_firmware_header_v2_5, v2_2);
 
 		switch (version_minor) {
 		case 0:
@@ -287,6 +289,26 @@ void amdgpu_ucode_print_rlc_hdr(const struct common_firmware_header *hdr)
 			DRM_DEBUG("se3_tap_delays_ucode_offset_bytes: %u\n",
 				  le32_to_cpu(rlc_hdr_v2_4->se3_tap_delays_ucode_offset_bytes));
 			break;
+		case 5:
+			/* rlc_hdr v2_5 */
+			DRM_INFO("rlc_iram_ucode_size_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_5->v2_2.rlc_iram_ucode_size_bytes));
+			DRM_INFO("rlc_iram_ucode_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_5->v2_2.rlc_iram_ucode_offset_bytes));
+			DRM_INFO("rlc_dram_ucode_size_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_5->v2_2.rlc_dram_ucode_size_bytes));
+			DRM_INFO("rlc_dram_ucode_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_5->v2_2.rlc_dram_ucode_offset_bytes));
+			/* rlc_hdr v2_5 */
+			DRM_INFO("rlc_1_iram_ucode_size_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_5->rlc_1_iram_ucode_size_bytes));
+			DRM_INFO("rlc_1_iram_ucode_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_5->rlc_1_iram_ucode_offset_bytes));
+			DRM_INFO("rlc_1_dram_ucode_size_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_5->rlc_1_dram_ucode_size_bytes));
+			DRM_INFO("rlc_1_dram_ucode_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_5->rlc_1_dram_ucode_offset_bytes));
+			break;
 		default:
 			DRM_ERROR("Unknown RLC v2 ucode: v2.%u\n", version_minor);
 			break;
@@ -631,6 +653,10 @@ const char *amdgpu_ucode_name(enum AMDGPU_UCODE_ID ucode_id)
 		return "RLC_IRAM";
 	case AMDGPU_UCODE_ID_RLC_DRAM:
 		return "RLC_DRAM";
+	case AMDGPU_UCODE_ID_RLC_IRAM_1:
+		return "RLC_IRAM_1";
+	case AMDGPU_UCODE_ID_RLC_DRAM_1:
+		return "RLC_DRAM_1";
 	case AMDGPU_UCODE_ID_RLC_G:
 		return "RLC_G";
 	case AMDGPU_UCODE_ID_RLC_P:
@@ -911,6 +937,14 @@ static int amdgpu_ucode_init_single_fw(struct amdgpu_device *adev,
 			ucode->ucode_size = adev->gfx.rlc.rlc_dram_ucode_size_bytes;
 			ucode_addr = adev->gfx.rlc.rlc_dram_ucode;
 			break;
+		case AMDGPU_UCODE_ID_RLC_IRAM_1:
+			ucode->ucode_size = adev->gfx.rlc.rlc_1_iram_ucode_size_bytes;
+			ucode_addr = adev->gfx.rlc.rlc_1_iram_ucode;
+			break;
+		case AMDGPU_UCODE_ID_RLC_DRAM_1:
+			ucode->ucode_size = adev->gfx.rlc.rlc_1_dram_ucode_size_bytes;
+			ucode_addr = adev->gfx.rlc.rlc_1_dram_ucode;
+			break;
 		case AMDGPU_UCODE_ID_RLC_P:
 			ucode->ucode_size = adev->gfx.rlc.rlcp_ucode_size_bytes;
 			ucode_addr = adev->gfx.rlc.rlcp_ucode;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
index 6349aad6da35..f316776fe950 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
@@ -300,6 +300,15 @@ struct rlc_firmware_header_v2_4 {
     uint32_t se3_tap_delays_ucode_offset_bytes;
 };
 
+/* version_major=2, version_minor=5 */
+struct rlc_firmware_header_v2_5 {
+	struct rlc_firmware_header_v2_2 v2_2;
+	uint32_t rlc_1_iram_ucode_size_bytes;
+	uint32_t rlc_1_iram_ucode_offset_bytes;
+	uint32_t rlc_1_dram_ucode_size_bytes;
+	uint32_t rlc_1_dram_ucode_offset_bytes;
+};
+
 /* version_major=1, version_minor=0 */
 struct sdma_firmware_header_v1_0 {
 	struct common_firmware_header header;
@@ -449,6 +458,7 @@ union amdgpu_firmware_header {
 	struct rlc_firmware_header_v2_2 rlc_v2_2;
 	struct rlc_firmware_header_v2_3 rlc_v2_3;
 	struct rlc_firmware_header_v2_4 rlc_v2_4;
+	struct rlc_firmware_header_v2_5 rlc_v2_5;
 	struct sdma_firmware_header_v1_0 sdma;
 	struct sdma_firmware_header_v1_1 sdma_v1_1;
 	struct sdma_firmware_header_v2_0 sdma_v2_0;
@@ -512,6 +522,8 @@ enum AMDGPU_UCODE_ID {
 	AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM,
 	AMDGPU_UCODE_ID_RLC_IRAM,
 	AMDGPU_UCODE_ID_RLC_DRAM,
+	AMDGPU_UCODE_ID_RLC_IRAM_1,
+	AMDGPU_UCODE_ID_RLC_DRAM_1,
 	AMDGPU_UCODE_ID_RLC_P,
 	AMDGPU_UCODE_ID_RLC_V,
 	AMDGPU_UCODE_ID_RLC_G,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 9a969175900e..c5dd5815056c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -166,7 +166,8 @@ static int amdgpu_userq_buffer_va_list_add(struct amdgpu_usermode_queue *queue,
 	return 0;
 }
 
-int amdgpu_userq_input_va_validate(struct amdgpu_usermode_queue *queue,
+int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
+				   struct amdgpu_usermode_queue *queue,
 				   u64 addr, u64 expected_size)
 {
 	struct amdgpu_bo_va_mapping *va_map;
@@ -271,10 +272,9 @@ static int amdgpu_userq_buffer_vas_list_cleanup(struct amdgpu_device *adev,
 	return r;
 }
 
-static int
-amdgpu_userq_preempt_helper(struct amdgpu_userq_mgr *uq_mgr,
-			  struct amdgpu_usermode_queue *queue)
+static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue)
 {
+	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
 	struct amdgpu_device *adev = uq_mgr->adev;
 	const struct amdgpu_userq_funcs *userq_funcs =
 		adev->userq_funcs[queue->queue_type];
@@ -282,7 +282,7 @@ amdgpu_userq_preempt_helper(struct amdgpu_userq_mgr *uq_mgr,
 	int r = 0;
 
 	if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
-		r = userq_funcs->preempt(uq_mgr, queue);
+		r = userq_funcs->preempt(queue);
 		if (r) {
 			queue->state = AMDGPU_USERQ_STATE_HUNG;
 			found_hung_queue = true;
@@ -297,17 +297,16 @@ amdgpu_userq_preempt_helper(struct amdgpu_userq_mgr *uq_mgr,
 	return r;
 }
 
-static int
-amdgpu_userq_restore_helper(struct amdgpu_userq_mgr *uq_mgr,
-			struct amdgpu_usermode_queue *queue)
+static int amdgpu_userq_restore_helper(struct amdgpu_usermode_queue *queue)
 {
+	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
 	struct amdgpu_device *adev = uq_mgr->adev;
 	const struct amdgpu_userq_funcs *userq_funcs =
 		adev->userq_funcs[queue->queue_type];
 	int r = 0;
 
 	if (queue->state == AMDGPU_USERQ_STATE_PREEMPTED) {
-		r = userq_funcs->restore(uq_mgr, queue);
+		r = userq_funcs->restore(queue);
 		if (r) {
 			queue->state = AMDGPU_USERQ_STATE_HUNG;
 		} else {
@@ -318,10 +317,9 @@ amdgpu_userq_restore_helper(struct amdgpu_userq_mgr *uq_mgr,
 	return r;
 }
 
-static int
-amdgpu_userq_unmap_helper(struct amdgpu_userq_mgr *uq_mgr,
-			  struct amdgpu_usermode_queue *queue)
+static int amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue *queue)
 {
+	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
 	struct amdgpu_device *adev = uq_mgr->adev;
 	const struct amdgpu_userq_funcs *userq_funcs =
 		adev->userq_funcs[queue->queue_type];
@@ -330,7 +328,7 @@ amdgpu_userq_unmap_helper(struct amdgpu_userq_mgr *uq_mgr,
 
 	if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) ||
 		(queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
-		r = userq_funcs->unmap(uq_mgr, queue);
+		r = userq_funcs->unmap(queue);
 		if (r) {
 			queue->state = AMDGPU_USERQ_STATE_HUNG;
 			found_hung_queue = true;
@@ -345,17 +343,16 @@ amdgpu_userq_unmap_helper(struct amdgpu_userq_mgr *uq_mgr,
 	return r;
 }
 
-static int
-amdgpu_userq_map_helper(struct amdgpu_userq_mgr *uq_mgr,
-			struct amdgpu_usermode_queue *queue)
+static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue)
 {
+	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
 	struct amdgpu_device *adev = uq_mgr->adev;
 	const struct amdgpu_userq_funcs *userq_funcs =
 		adev->userq_funcs[queue->queue_type];
 	int r = 0;
 
 	if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) {
-		r = userq_funcs->map(uq_mgr, queue);
+		r = userq_funcs->map(queue);
 		if (r) {
 			queue->state = AMDGPU_USERQ_STATE_HUNG;
 			amdgpu_userq_detect_and_reset_queues(uq_mgr);
@@ -367,10 +364,9 @@ amdgpu_userq_map_helper(struct amdgpu_userq_mgr *uq_mgr,
 	return r;
 }
 
-static int
-amdgpu_userq_wait_for_last_fence(struct amdgpu_userq_mgr *uq_mgr,
-				 struct amdgpu_usermode_queue *queue)
+static int amdgpu_userq_wait_for_last_fence(struct amdgpu_usermode_queue *queue)
 {
+	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
 	struct dma_fence *f = queue->last_fence;
 	int ret = 0;
 
@@ -387,11 +383,10 @@ amdgpu_userq_wait_for_last_fence(struct amdgpu_userq_mgr *uq_mgr,
 	return ret;
 }
 
-static void
-amdgpu_userq_cleanup(struct amdgpu_userq_mgr *uq_mgr,
-		     struct amdgpu_usermode_queue *queue,
-		     int queue_id)
+static void amdgpu_userq_cleanup(struct amdgpu_usermode_queue *queue,
+				 int queue_id)
 {
+	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
 	struct amdgpu_device *adev = uq_mgr->adev;
 	const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type];
 
@@ -400,10 +395,10 @@ amdgpu_userq_cleanup(struct amdgpu_userq_mgr *uq_mgr,
 
 	/* Drop the userq reference. */
 	amdgpu_userq_buffer_vas_list_cleanup(adev, queue);
-	uq_funcs->mqd_destroy(uq_mgr, queue);
+	uq_funcs->mqd_destroy(queue);
 	amdgpu_userq_fence_driver_free(queue);
 	/* Use interrupt-safe locking since IRQ handlers may access these XArrays */
-	xa_erase_irq(&uq_mgr->userq_mgr_xa, (unsigned long)queue_id);
+	xa_erase_irq(&uq_mgr->userq_xa, (unsigned long)queue_id);
 	xa_erase_irq(&adev->userq_doorbell_xa, queue->doorbell_index);
 	queue->userq_mgr = NULL;
 	list_del(&queue->userq_va_list);
@@ -415,7 +410,7 @@ amdgpu_userq_cleanup(struct amdgpu_userq_mgr *uq_mgr,
 static struct amdgpu_usermode_queue *
 amdgpu_userq_find(struct amdgpu_userq_mgr *uq_mgr, int qid)
 {
-	return xa_load(&uq_mgr->userq_mgr_xa, qid);
+	return xa_load(&uq_mgr->userq_xa, qid);
 }
 
 void
@@ -584,25 +579,33 @@ amdgpu_userq_destroy(struct drm_file *filp, int queue_id)
 		mutex_unlock(&uq_mgr->userq_mutex);
 		return -EINVAL;
 	}
-	amdgpu_userq_wait_for_last_fence(uq_mgr, queue);
+	amdgpu_userq_wait_for_last_fence(queue);
 	r = amdgpu_bo_reserve(queue->db_obj.obj, true);
 	if (!r) {
 		amdgpu_bo_unpin(queue->db_obj.obj);
 		amdgpu_bo_unreserve(queue->db_obj.obj);
 	}
 	amdgpu_bo_unref(&queue->db_obj.obj);
+
+	r = amdgpu_bo_reserve(queue->wptr_obj.obj, true);
+	if (!r) {
+		amdgpu_bo_unpin(queue->wptr_obj.obj);
+		amdgpu_bo_unreserve(queue->wptr_obj.obj);
+	}
+	amdgpu_bo_unref(&queue->wptr_obj.obj);
+
 	atomic_dec(&uq_mgr->userq_count[queue->queue_type]);
 #if defined(CONFIG_DEBUG_FS)
 	debugfs_remove_recursive(queue->debugfs_queue);
 #endif
 	amdgpu_userq_detect_and_reset_queues(uq_mgr);
-	r = amdgpu_userq_unmap_helper(uq_mgr, queue);
+	r = amdgpu_userq_unmap_helper(queue);
 	/*TODO: It requires a reset for userq hw unmap error*/
 	if (unlikely(r != AMDGPU_USERQ_STATE_UNMAPPED)) {
 		drm_warn(adev_to_drm(uq_mgr->adev), "trying to destroy a HW mapping userq\n");
 		queue->state = AMDGPU_USERQ_STATE_HUNG;
 	}
-	amdgpu_userq_cleanup(uq_mgr, queue, queue_id);
+	amdgpu_userq_cleanup(queue, queue_id);
 	mutex_unlock(&uq_mgr->userq_mutex);
 
 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
@@ -729,10 +732,11 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 	db_info.db_obj = &queue->db_obj;
 	db_info.doorbell_offset = args->in.doorbell_offset;
 
+	queue->userq_mgr = uq_mgr;
 	/* Validate the userq virtual address.*/
-	if (amdgpu_userq_input_va_validate(queue, args->in.queue_va, args->in.queue_size) ||
-	    amdgpu_userq_input_va_validate(queue, args->in.rptr_va, AMDGPU_GPU_PAGE_SIZE) ||
-	    amdgpu_userq_input_va_validate(queue, args->in.wptr_va, AMDGPU_GPU_PAGE_SIZE)) {
+	if (amdgpu_userq_input_va_validate(adev, queue, args->in.queue_va, args->in.queue_size) ||
+	    amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va, AMDGPU_GPU_PAGE_SIZE) ||
+	    amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va, AMDGPU_GPU_PAGE_SIZE)) {
 		r = -EINVAL;
 		kfree(queue);
 		goto unlock;
@@ -755,7 +759,7 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 		goto unlock;
 	}
 
-	r = uq_funcs->mqd_create(uq_mgr, &args->in, queue);
+	r = uq_funcs->mqd_create(queue, &args->in);
 	if (r) {
 		drm_file_err(uq_mgr->file, "Failed to create Queue\n");
 		amdgpu_userq_fence_driver_free(queue);
@@ -772,18 +776,18 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 		goto unlock;
 	}
 
-	r = xa_alloc(&uq_mgr->userq_mgr_xa, &qid, queue, XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT), GFP_KERNEL);
+	r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
+		     XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT), GFP_KERNEL);
 	if (r) {
 		drm_file_err(uq_mgr->file, "Failed to allocate a queue id\n");
 		amdgpu_userq_fence_driver_free(queue);
-		uq_funcs->mqd_destroy(uq_mgr, queue);
+		uq_funcs->mqd_destroy(queue);
 		kfree(queue);
 		r = -ENOMEM;
 		up_read(&adev->reset_domain->sem);
 		goto unlock;
 	}
 	up_read(&adev->reset_domain->sem);
-	queue->userq_mgr = uq_mgr;
 
 	/* don't map the queue if scheduling is halted */
 	if (adev->userq_halt_for_enforce_isolation &&
@@ -793,12 +797,12 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 	else
 		skip_map_queue = false;
 	if (!skip_map_queue) {
-		r = amdgpu_userq_map_helper(uq_mgr, queue);
+		r = amdgpu_userq_map_helper(queue);
 		if (r) {
 			drm_file_err(uq_mgr->file, "Failed to map Queue\n");
-			xa_erase(&uq_mgr->userq_mgr_xa, qid);
+			xa_erase(&uq_mgr->userq_xa, qid);
 			amdgpu_userq_fence_driver_free(queue);
-			uq_funcs->mqd_destroy(uq_mgr, queue);
+			uq_funcs->mqd_destroy(queue);
 			kfree(queue);
 			goto unlock;
 		}
@@ -923,8 +927,7 @@ amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr)
 	int ret = 0, r;
 
 	/* Resume all the queues for this process */
-	xa_for_each(&uq_mgr->userq_mgr_xa, queue_id, queue) {
-
+	xa_for_each(&uq_mgr->userq_xa, queue_id, queue) {
 		if (!amdgpu_userq_buffer_vas_mapped(queue)) {
 			drm_file_err(uq_mgr->file,
 				     "trying restore queue without va mapping\n");
@@ -932,7 +935,7 @@ amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr)
 			continue;
 		}
 
-		r = amdgpu_userq_restore_helper(uq_mgr, queue);
+		r = amdgpu_userq_restore_helper(queue);
 		if (r)
 			ret = r;
 	}
@@ -1167,8 +1170,8 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
 
 	amdgpu_userq_detect_and_reset_queues(uq_mgr);
 	/* Try to unmap all the queues in this process ctx */
-	xa_for_each(&uq_mgr->userq_mgr_xa, queue_id, queue) {
-		r = amdgpu_userq_preempt_helper(uq_mgr, queue);
+	xa_for_each(&uq_mgr->userq_xa, queue_id, queue) {
+		r = amdgpu_userq_preempt_helper(queue);
 		if (r)
 			ret = r;
 	}
@@ -1202,7 +1205,7 @@ amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr)
 	unsigned long queue_id;
 	int ret;
 
-	xa_for_each(&uq_mgr->userq_mgr_xa, queue_id, queue) {
+	xa_for_each(&uq_mgr->userq_xa, queue_id, queue) {
 		struct dma_fence *f = queue->last_fence;
 
 		if (!f || dma_fence_is_signaled(f))
@@ -1252,7 +1255,7 @@ int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *f
 			  struct amdgpu_device *adev)
 {
 	mutex_init(&userq_mgr->userq_mutex);
-	xa_init_flags(&userq_mgr->userq_mgr_xa, XA_FLAGS_ALLOC);
+	xa_init_flags(&userq_mgr->userq_xa, XA_FLAGS_ALLOC);
 	userq_mgr->adev = adev;
 	userq_mgr->file = file_priv;
 
@@ -1269,13 +1272,13 @@ void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr)
 
 	mutex_lock(&userq_mgr->userq_mutex);
 	amdgpu_userq_detect_and_reset_queues(userq_mgr);
-	xa_for_each(&userq_mgr->userq_mgr_xa, queue_id, queue) {
-		amdgpu_userq_wait_for_last_fence(userq_mgr, queue);
-		amdgpu_userq_unmap_helper(userq_mgr, queue);
-		amdgpu_userq_cleanup(userq_mgr, queue, queue_id);
+	xa_for_each(&userq_mgr->userq_xa, queue_id, queue) {
+		amdgpu_userq_wait_for_last_fence(queue);
+		amdgpu_userq_unmap_helper(queue);
+		amdgpu_userq_cleanup(queue, queue_id);
 	}
 
-	xa_destroy(&userq_mgr->userq_mgr_xa);
+	xa_destroy(&userq_mgr->userq_xa);
 	mutex_unlock(&userq_mgr->userq_mutex);
 	mutex_destroy(&userq_mgr->userq_mutex);
 }
@@ -1297,9 +1300,9 @@ int amdgpu_userq_suspend(struct amdgpu_device *adev)
 		guard(mutex)(&uqm->userq_mutex);
 		amdgpu_userq_detect_and_reset_queues(uqm);
 		if (adev->in_s0ix)
-			r = amdgpu_userq_preempt_helper(uqm, queue);
+			r = amdgpu_userq_preempt_helper(queue);
 		else
-			r = amdgpu_userq_unmap_helper(uqm, queue);
+			r = amdgpu_userq_unmap_helper(queue);
 		if (r)
 			return r;
 	}
@@ -1321,9 +1324,9 @@ int amdgpu_userq_resume(struct amdgpu_device *adev)
 		uqm = queue->userq_mgr;
 		guard(mutex)(&uqm->userq_mutex);
 		if (adev->in_s0ix)
-			r = amdgpu_userq_restore_helper(uqm, queue);
+			r = amdgpu_userq_restore_helper(queue);
 		else
-			r = amdgpu_userq_map_helper(uqm, queue);
+			r = amdgpu_userq_map_helper(queue);
 		if (r)
 			return r;
 	}
@@ -1355,7 +1358,7 @@ int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev,
 		     (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) &&
 		    (queue->xcp_id == idx)) {
 			amdgpu_userq_detect_and_reset_queues(uqm);
-			r = amdgpu_userq_preempt_helper(uqm, queue);
+			r = amdgpu_userq_preempt_helper(queue);
 			if (r)
 				ret = r;
 		}
@@ -1387,9 +1390,9 @@ int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev,
 			if (((queue->queue_type == AMDGPU_HW_IP_GFX) ||
 			     (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) &&
 			    (queue->xcp_id == idx)) {
-				r = amdgpu_userq_restore_helper(uqm, queue);
-				if (r)
-					ret = r;
+			r = amdgpu_userq_restore_helper(queue);
+			if (r)
+				ret = r;
 			}
 		mutex_unlock(&uqm->userq_mutex);
 	}
@@ -1439,9 +1442,9 @@ void amdgpu_userq_pre_reset(struct amdgpu_device *adev)
 		uqm = queue->userq_mgr;
 		cancel_delayed_work_sync(&uqm->resume_work);
 		if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
-			amdgpu_userq_wait_for_last_fence(uqm, queue);
+			amdgpu_userq_wait_for_last_fence(queue);
 			userq_funcs = adev->userq_funcs[queue->queue_type];
-			userq_funcs->unmap(uqm, queue);
+			userq_funcs->unmap(queue);
 			/* just mark all queues as hung at this point.
 			 * if unmap succeeds, we could map again
 			 * in amdgpu_userq_post_reset() if vram is not lost
@@ -1458,18 +1461,16 @@ int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost)
 	 * at this point, we should be able to map it again
 	 * and continue if vram is not lost.
 	 */
-	struct amdgpu_userq_mgr *uqm;
 	struct amdgpu_usermode_queue *queue;
 	const struct amdgpu_userq_funcs *userq_funcs;
 	unsigned long queue_id;
 	int r = 0;
 
 	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
-		uqm = queue->userq_mgr;
 		if (queue->state == AMDGPU_USERQ_STATE_HUNG && !vram_lost) {
 			userq_funcs = adev->userq_funcs[queue->queue_type];
 			/* Re-map queue */
-			r = userq_funcs->map(uqm, queue);
+			r = userq_funcs->map(queue);
 			if (r) {
 				dev_err(adev->dev, "Failed to remap queue %ld\n", queue_id);
 				continue;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index c37444427a14..1eaa94f8a291 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -77,19 +77,13 @@ struct amdgpu_usermode_queue {
 };
 
 struct amdgpu_userq_funcs {
-	int (*mqd_create)(struct amdgpu_userq_mgr *uq_mgr,
-			  struct drm_amdgpu_userq_in *args,
-			  struct amdgpu_usermode_queue *queue);
-	void (*mqd_destroy)(struct amdgpu_userq_mgr *uq_mgr,
-			    struct amdgpu_usermode_queue *uq);
-	int (*unmap)(struct amdgpu_userq_mgr *uq_mgr,
-		     struct amdgpu_usermode_queue *queue);
-	int (*map)(struct amdgpu_userq_mgr *uq_mgr,
-		   struct amdgpu_usermode_queue *queue);
-	int (*preempt)(struct amdgpu_userq_mgr *uq_mgr,
-		   struct amdgpu_usermode_queue *queue);
-	int (*restore)(struct amdgpu_userq_mgr *uq_mgr,
-		   struct amdgpu_usermode_queue *queue);
+	int (*mqd_create)(struct amdgpu_usermode_queue *queue,
+			  struct drm_amdgpu_userq_in *args);
+	void (*mqd_destroy)(struct amdgpu_usermode_queue *uq);
+	int (*unmap)(struct amdgpu_usermode_queue *queue);
+	int (*map)(struct amdgpu_usermode_queue *queue);
+	int (*preempt)(struct amdgpu_usermode_queue *queue);
+	int (*restore)(struct amdgpu_usermode_queue *queue);
 	int (*detect_and_reset)(struct amdgpu_device *adev,
 		  int queue_type);
 };
@@ -97,11 +91,11 @@ struct amdgpu_userq_funcs {
 /* Usermode queues for gfx */
 struct amdgpu_userq_mgr {
 	/**
-	 * @userq_mgr_xa: Per-process user queue map (queue ID → queue)
+	 * @userq_xa: Per-process user queue map (queue ID → queue)
 	 * Key: queue_id (unique ID within the process's userq manager)
 	 * Value: struct amdgpu_usermode_queue
 	 */
-	struct xarray			userq_mgr_xa;
+	struct xarray			userq_xa;
 	struct mutex			userq_mutex;
 	struct amdgpu_device		*adev;
 	struct delayed_work		resume_work;
@@ -153,7 +147,8 @@ void amdgpu_userq_reset_work(struct work_struct *work);
 void amdgpu_userq_pre_reset(struct amdgpu_device *adev);
 int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost);
 
-int amdgpu_userq_input_va_validate(struct amdgpu_usermode_queue *queue,
+int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
+				   struct amdgpu_usermode_queue *queue,
 				   u64 addr, u64 expected_size);
 int amdgpu_userq_gem_va_unmap_validate(struct amdgpu_device *adev,
 				       struct amdgpu_bo_va_mapping *mapping,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
index eba9fb359047..25f178536469 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
@@ -352,6 +352,7 @@ static const struct dma_fence_ops amdgpu_userq_fence_ops = {
 /**
  * amdgpu_userq_fence_read_wptr - Read the userq wptr value
  *
+ * @adev: amdgpu_device pointer
  * @queue: user mode queue structure pointer
  * @wptr: write pointer value
  *
@@ -361,7 +362,8 @@ static const struct dma_fence_ops amdgpu_userq_fence_ops = {
  *
  * Returns wptr value on success, error on failure.
  */
-static int amdgpu_userq_fence_read_wptr(struct amdgpu_usermode_queue *queue,
+static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev,
+					struct amdgpu_usermode_queue *queue,
 					u64 *wptr)
 {
 	struct amdgpu_bo_va_mapping *mapping;
@@ -455,6 +457,7 @@ amdgpu_userq_fence_driver_force_completion(struct amdgpu_usermode_queue *userq)
 int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 			      struct drm_file *filp)
 {
+	struct amdgpu_device *adev = drm_to_adev(dev);
 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
 	struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
 	struct drm_amdgpu_userq_signal *args = data;
@@ -539,13 +542,13 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 	}
 
 	/* Retrieve the user queue */
-	queue = xa_load(&userq_mgr->userq_mgr_xa, args->queue_id);
+	queue = xa_load(&userq_mgr->userq_xa, args->queue_id);
 	if (!queue) {
 		r = -ENOENT;
 		goto put_gobj_write;
 	}
 
-	r = amdgpu_userq_fence_read_wptr(queue, &wptr);
+	r = amdgpu_userq_fence_read_wptr(adev, queue, &wptr);
 	if (r)
 		goto put_gobj_write;
 
@@ -901,7 +904,7 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
 		 */
 		num_fences = dma_fence_dedup_array(fences, num_fences);
 
-		waitq = xa_load(&userq_mgr->userq_mgr_xa, wait_info->waitq_id);
+		waitq = xa_load(&userq_mgr->userq_xa, wait_info->waitq_id);
 		if (!waitq) {
 			r = -EINVAL;
 			goto free_fences;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 47a6ce4fdc74..759f3c642331 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -37,6 +37,7 @@
 #include "vi.h"
 #include "soc15.h"
 #include "nv.h"
+#include "amdgpu_virt_ras_cmd.h"
 
 #define POPULATE_UCODE_INFO(vf2pf_info, ucode, ver) \
 	do { \
@@ -1533,6 +1534,9 @@ bool amdgpu_virt_get_ras_capability(struct amdgpu_device *adev)
 	if (adev->virt.ras_en_caps.bits.poison_propogation_mode)
 		con->poison_supported = true; /* Poison is handled by host */
 
+	if (adev->virt.ras_en_caps.bits.uniras_supported)
+		amdgpu_virt_ras_set_remote_uniras(adev, true);
+
 	return true;
 }
 
@@ -1845,3 +1849,28 @@ int amdgpu_virt_check_vf_critical_region(struct amdgpu_device *adev, u64 addr, b
 
 	return r;
 }
+
+static int req_remote_ras_cmd(struct amdgpu_device *adev,
+			u32 param1, u32 param2, u32 param3)
+{
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (virt->ops && virt->ops->req_remote_ras_cmd)
+		return virt->ops->req_remote_ras_cmd(adev, param1, param2, param3);
+	return -ENOENT;
+}
+
+int amdgpu_virt_send_remote_ras_cmd(struct amdgpu_device *adev,
+		uint64_t buf, uint32_t buf_len)
+{
+	uint64_t gpa = buf;
+	int ret = -EIO;
+
+	if (down_read_trylock(&adev->reset_domain->sem)) {
+		ret = req_remote_ras_cmd(adev,
+			lower_32_bits(gpa), upper_32_bits(gpa), buf_len);
+		up_read(&adev->reset_domain->sem);
+	}
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 01d5bca2dee1..608d966701a4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -105,6 +105,8 @@ struct amdgpu_virt_ops {
 	int (*req_ras_cper_dump)(struct amdgpu_device *adev, u64 vf_rptr);
 	int (*req_bad_pages)(struct amdgpu_device *adev);
 	int (*req_ras_chk_criti)(struct amdgpu_device *adev, u64 addr);
+	int (*req_remote_ras_cmd)(struct amdgpu_device *adev,
+			u32 param1, u32 param2, u32 param3);
 };
 
 /*
@@ -483,4 +485,6 @@ bool amdgpu_virt_ras_telemetry_block_en(struct amdgpu_device *adev,
 					enum amdgpu_ras_block block);
 void amdgpu_virt_request_bad_pages(struct amdgpu_device *adev);
 int amdgpu_virt_check_vf_critical_region(struct amdgpu_device *adev, u64 addr, bool *hit);
+int amdgpu_virt_send_remote_ras_cmd(struct amdgpu_device *adev,
+		uint64_t buf, uint32_t buf_len);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index c362d4dfb5bb..0eccb31793ca 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -834,7 +834,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
 		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
 
 	if (spm_update_needed && adev->gfx.rlc.funcs->update_spm_vmid)
-		adev->gfx.rlc.funcs->update_spm_vmid(adev, ring, job->vmid);
+		adev->gfx.rlc.funcs->update_spm_vmid(adev, ring->xcc_id, ring, job->vmid);
 
 	if (ring->funcs->emit_gds_switch &&
 	    gds_switch_needed) {
@@ -2362,9 +2362,26 @@ void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
 			   unsigned max_bits)
 {
 	unsigned int max_size = 1 << (max_bits - 30);
+	bool sys_5level_pgtable = false;
 	unsigned int vm_size;
 	uint64_t tmp;
 
+#ifdef CONFIG_X86_64
+	/*
+	 * Refer to function configure_5level_paging() for details.
+	 */
+	sys_5level_pgtable = (native_read_cr4() & X86_CR4_LA57);
+#endif
+
+	/*
+	 * If GPU supports 5-level page table, but system uses 4-level page table,
+	 * then use 4-level page table on GPU
+	 */
+	if (max_level == 4 && !sys_5level_pgtable) {
+		min_vm_size = 256 * 1024;
+		max_level = 3;
+	}
+
 	/* adjust vm size first */
 	if (amdgpu_vm_size != -1) {
 		vm_size = amdgpu_vm_size;
@@ -2407,6 +2424,9 @@ void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
 	tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
 	adev->vm_manager.num_level = min_t(unsigned int, max_level, tmp);
 	switch (adev->vm_manager.num_level) {
+	case 4:
+		adev->vm_manager.root_level = AMDGPU_VM_PDB3;
+		break;
 	case 3:
 		adev->vm_manager.root_level = AMDGPU_VM_PDB2;
 		break;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 15d757c016cb..139642eacdd0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -129,6 +129,7 @@ struct amdgpu_bo_vm;
 	  AMDGPU_PTE_MTYPE_GFX12_SHIFT(mtype))
 
 #define AMDGPU_PTE_DCC			(1ULL << 58)
+#define AMDGPU_PTE_BUS_ATOMICS		(1ULL << 59)
 #define AMDGPU_PTE_IS_PTE		(1ULL << 63)
 
 /* PDE Block Fragment Size for gfx v12 */
@@ -185,9 +186,10 @@ struct amdgpu_bo_vm;
 #define AMDGPU_VM_USE_CPU_FOR_COMPUTE (1 << 1)
 
 /* VMPT level enumerate, and the hiberachy is:
- * PDB2->PDB1->PDB0->PTB
+ * PDB3->PDB2->PDB1->PDB0->PTB
  */
 enum amdgpu_vm_level {
+	AMDGPU_VM_PDB3,
 	AMDGPU_VM_PDB2,
 	AMDGPU_VM_PDB1,
 	AMDGPU_VM_PDB0,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index f794fb1cc06e..31a437ce9570 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -51,6 +51,7 @@ static unsigned int amdgpu_vm_pt_level_shift(struct amdgpu_device *adev,
 					     unsigned int level)
 {
 	switch (level) {
+	case AMDGPU_VM_PDB3:
 	case AMDGPU_VM_PDB2:
 	case AMDGPU_VM_PDB1:
 	case AMDGPU_VM_PDB0:
@@ -366,6 +367,7 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	struct amdgpu_bo *ancestor = &vmbo->bo;
 	unsigned int entries;
 	struct amdgpu_bo *bo = &vmbo->bo;
+	uint64_t value = 0, flags = 0;
 	uint64_t addr;
 	int r, idx;
 
@@ -403,7 +405,6 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
 	addr = 0;
 
-	uint64_t value = 0, flags = 0;
 	if (adev->asic_type >= CHIP_VEGA10) {
 		if (level != AMDGPU_VM_PTB) {
 			/* Handle leaf PDEs as PTEs */
@@ -412,7 +413,7 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 					      &value, &flags);
 		} else {
 			/* Workaround for fault priority problem on GMC9 */
-			flags = AMDGPU_PTE_EXECUTABLE;
+			flags = AMDGPU_PTE_EXECUTABLE | adev->gmc.init_pte_flags;
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
index 1083db8cea2e..73250ab45f20 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
@@ -540,6 +540,7 @@ static void amdgpu_set_xcp_id(struct amdgpu_device *adev,
 	case AMDGPU_HW_IP_GFX:
 	case AMDGPU_RING_TYPE_COMPUTE:
 	case AMDGPU_RING_TYPE_KIQ:
+	case AMDGPU_RING_TYPE_MES:
 		ip_blk = AMDGPU_XCP_GFX;
 		break;
 	case AMDGPU_RING_TYPE_SDMA:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
index 1928d9e224fc..8058e8f35d41 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
@@ -41,6 +41,9 @@
 
 #define AMDGPU_XCP_OPS_KFD	(1 << 0)
 
+#define XCP_INST_MASK(num_inst, xcp_id)		\
+	(num_inst ? GENMASK(num_inst - 1, 0) << (xcp_id * num_inst) : 0)
+
 struct amdgpu_fpriv;
 
 enum AMDGPU_XCP_IP_BLOCK {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h
index 3cdb1e0eca37..cffb2f805de2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h
@@ -201,7 +201,8 @@ union amd_sriov_ras_caps {
 		uint64_t block_mpio			: 1;
 		uint64_t block_mmsch			: 1;
 		uint64_t poison_propogation_mode	: 1;
-		uint64_t reserved			: 43;
+		uint64_t uniras_supported		: 1;
+		uint64_t reserved			: 42;
 	} bits;
 	uint64_t all;
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c
index f9e2edf5260b..d9842aa25283 100644
--- a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c
+++ b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c
@@ -31,9 +31,6 @@
 #include "sdma_v4_4_2.h"
 #include "amdgpu_ip.h"
 
-#define XCP_INST_MASK(num_inst, xcp_id)                                        \
-	(num_inst ? GENMASK(num_inst - 1, 0) << (xcp_id * num_inst) : 0)
-
 void aqua_vanjaram_doorbell_index_init(struct amdgpu_device *adev)
 {
 	int i;
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index 9cd63b4177bf..fc0d2524b16f 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -1957,10 +1957,6 @@ static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
 	return (nak_r + nak_g);
 }
 
-static void cik_pre_asic_init(struct amdgpu_device *adev)
-{
-}
-
 static const struct amdgpu_asic_funcs cik_asic_funcs =
 {
 	.read_disabled_bios = &cik_read_disabled_bios,
@@ -1981,7 +1977,6 @@ static const struct amdgpu_asic_funcs cik_asic_funcs =
 	.need_reset_on_init = &cik_need_reset_on_init,
 	.get_pcie_replay_count = &cik_get_pcie_replay_count,
 	.supports_baco = &cik_asic_supports_baco,
-	.pre_asic_init = &cik_pre_asic_init,
 	.query_video_codecs = &cik_query_video_codecs,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index 72ca6538b2e4..61302204e9b4 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -1838,7 +1838,7 @@ static void dce_v10_0_grph_enable(struct drm_crtc *crtc, bool enable)
 
 static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
 				     struct drm_framebuffer *fb,
-				     int x, int y, int atomic)
+				     int x, int y)
 {
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
@@ -1855,15 +1855,12 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
 	bool bypass_lut = false;
 
 	/* no fb bound */
-	if (!atomic && !crtc->primary->fb) {
+	if (!crtc->primary->fb) {
 		DRM_DEBUG_KMS("No FB bound\n");
 		return 0;
 	}
 
-	if (atomic)
-		target_fb = fb;
-	else
-		target_fb = crtc->primary->fb;
+	target_fb = crtc->primary->fb;
 
 	/* If atomic, assume fb object is pinned & idle & fenced and
 	 * just update base pointers
@@ -1874,13 +1871,11 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
 	if (unlikely(r != 0))
 		return r;
 
-	if (!atomic) {
-		abo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
-		r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM);
-		if (unlikely(r != 0)) {
-			amdgpu_bo_unreserve(abo);
-			return -EINVAL;
-		}
+	abo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+	r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM);
+	if (unlikely(r != 0)) {
+		amdgpu_bo_unreserve(abo);
+		return -EINVAL;
 	}
 	fb_location = amdgpu_bo_gpu_offset(abo);
 
@@ -2068,7 +2063,7 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
 	/* set pageflip to happen anywhere in vblank interval */
 	WREG32(mmMASTER_UPDATE_MODE + amdgpu_crtc->crtc_offset, 0);
 
-	if (!atomic && fb && fb != crtc->primary->fb) {
+	if (fb && fb != crtc->primary->fb) {
 		abo = gem_to_amdgpu_bo(fb->obj[0]);
 		r = amdgpu_bo_reserve(abo, true);
 		if (unlikely(r != 0))
@@ -2611,7 +2606,7 @@ static int dce_v10_0_crtc_mode_set(struct drm_crtc *crtc,
 
 	amdgpu_atombios_crtc_set_pll(crtc, adjusted_mode);
 	amdgpu_atombios_crtc_set_dtd_timing(crtc, adjusted_mode);
-	dce_v10_0_crtc_do_set_base(crtc, old_fb, x, y, 0);
+	dce_v10_0_crtc_do_set_base(crtc, old_fb, x, y);
 	amdgpu_atombios_crtc_overscan_setup(crtc, mode, adjusted_mode);
 	amdgpu_atombios_crtc_scaler_setup(crtc);
 	dce_v10_0_cursor_reset(crtc);
@@ -2659,14 +2654,7 @@ static bool dce_v10_0_crtc_mode_fixup(struct drm_crtc *crtc,
 static int dce_v10_0_crtc_set_base(struct drm_crtc *crtc, int x, int y,
 				  struct drm_framebuffer *old_fb)
 {
-	return dce_v10_0_crtc_do_set_base(crtc, old_fb, x, y, 0);
-}
-
-static int dce_v10_0_crtc_set_base_atomic(struct drm_crtc *crtc,
-					 struct drm_framebuffer *fb,
-					 int x, int y, enum mode_set_atomic state)
-{
-	return dce_v10_0_crtc_do_set_base(crtc, fb, x, y, 1);
+	return dce_v10_0_crtc_do_set_base(crtc, old_fb, x, y);
 }
 
 static const struct drm_crtc_helper_funcs dce_v10_0_crtc_helper_funcs = {
@@ -2674,7 +2662,6 @@ static const struct drm_crtc_helper_funcs dce_v10_0_crtc_helper_funcs = {
 	.mode_fixup = dce_v10_0_crtc_mode_fixup,
 	.mode_set = dce_v10_0_crtc_mode_set,
 	.mode_set_base = dce_v10_0_crtc_set_base,
-	.mode_set_base_atomic = dce_v10_0_crtc_set_base_atomic,
 	.prepare = dce_v10_0_crtc_prepare,
 	.commit = dce_v10_0_crtc_commit,
 	.disable = dce_v10_0_crtc_disable,
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
index acc887a58518..8f4b4c2e36b9 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
@@ -1876,7 +1876,7 @@ static void dce_v6_0_grph_enable(struct drm_crtc *crtc, bool enable)
 
 static int dce_v6_0_crtc_do_set_base(struct drm_crtc *crtc,
 				     struct drm_framebuffer *fb,
-				     int x, int y, int atomic)
+				     int x, int y)
 {
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
@@ -1892,15 +1892,12 @@ static int dce_v6_0_crtc_do_set_base(struct drm_crtc *crtc,
 	bool bypass_lut = false;
 
 	/* no fb bound */
-	if (!atomic && !crtc->primary->fb) {
+	if (!crtc->primary->fb) {
 		DRM_DEBUG_KMS("No FB bound\n");
 		return 0;
 	}
 
-	if (atomic)
-		target_fb = fb;
-	else
-		target_fb = crtc->primary->fb;
+	target_fb = crtc->primary->fb;
 
 	/* If atomic, assume fb object is pinned & idle & fenced and
 	 * just update base pointers
@@ -1911,13 +1908,11 @@ static int dce_v6_0_crtc_do_set_base(struct drm_crtc *crtc,
 	if (unlikely(r != 0))
 		return r;
 
-	if (!atomic) {
-		abo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
-		r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM);
-		if (unlikely(r != 0)) {
-			amdgpu_bo_unreserve(abo);
-			return -EINVAL;
-		}
+	abo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+	r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM);
+	if (unlikely(r != 0)) {
+		amdgpu_bo_unreserve(abo);
+		return -EINVAL;
 	}
 	fb_location = amdgpu_bo_gpu_offset(abo);
 
@@ -2083,7 +2078,7 @@ static int dce_v6_0_crtc_do_set_base(struct drm_crtc *crtc,
 	/* set pageflip to happen anywhere in vblank interval */
 	WREG32(mmMASTER_UPDATE_MODE + amdgpu_crtc->crtc_offset, 0);
 
-	if (!atomic && fb && fb != crtc->primary->fb) {
+	if (fb && fb != crtc->primary->fb) {
 		abo = gem_to_amdgpu_bo(fb->obj[0]);
 		r = amdgpu_bo_reserve(abo, true);
 		if (unlikely(r != 0))
@@ -2578,7 +2573,7 @@ static int dce_v6_0_crtc_mode_set(struct drm_crtc *crtc,
 
 	amdgpu_atombios_crtc_set_pll(crtc, adjusted_mode);
 	amdgpu_atombios_crtc_set_dtd_timing(crtc, adjusted_mode);
-	dce_v6_0_crtc_do_set_base(crtc, old_fb, x, y, 0);
+	dce_v6_0_crtc_do_set_base(crtc, old_fb, x, y);
 	amdgpu_atombios_crtc_overscan_setup(crtc, mode, adjusted_mode);
 	amdgpu_atombios_crtc_scaler_setup(crtc);
 	dce_v6_0_cursor_reset(crtc);
@@ -2626,14 +2621,7 @@ static bool dce_v6_0_crtc_mode_fixup(struct drm_crtc *crtc,
 static int dce_v6_0_crtc_set_base(struct drm_crtc *crtc, int x, int y,
 				  struct drm_framebuffer *old_fb)
 {
-	return dce_v6_0_crtc_do_set_base(crtc, old_fb, x, y, 0);
-}
-
-static int dce_v6_0_crtc_set_base_atomic(struct drm_crtc *crtc,
-					 struct drm_framebuffer *fb,
-					 int x, int y, enum mode_set_atomic state)
-{
-	return dce_v6_0_crtc_do_set_base(crtc, fb, x, y, 1);
+	return dce_v6_0_crtc_do_set_base(crtc, old_fb, x, y);
 }
 
 static const struct drm_crtc_helper_funcs dce_v6_0_crtc_helper_funcs = {
@@ -2641,7 +2629,6 @@ static const struct drm_crtc_helper_funcs dce_v6_0_crtc_helper_funcs = {
 	.mode_fixup = dce_v6_0_crtc_mode_fixup,
 	.mode_set = dce_v6_0_crtc_mode_set,
 	.mode_set_base = dce_v6_0_crtc_set_base,
-	.mode_set_base_atomic = dce_v6_0_crtc_set_base_atomic,
 	.prepare = dce_v6_0_crtc_prepare,
 	.commit = dce_v6_0_crtc_commit,
 	.disable = dce_v6_0_crtc_disable,
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index 2ccd6aad8dd6..9d1853c41fcd 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -1785,7 +1785,7 @@ static void dce_v8_0_grph_enable(struct drm_crtc *crtc, bool enable)
 
 static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
 				     struct drm_framebuffer *fb,
-				     int x, int y, int atomic)
+				     int x, int y)
 {
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
@@ -1802,15 +1802,12 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
 	bool bypass_lut = false;
 
 	/* no fb bound */
-	if (!atomic && !crtc->primary->fb) {
+	if (!crtc->primary->fb) {
 		DRM_DEBUG_KMS("No FB bound\n");
 		return 0;
 	}
 
-	if (atomic)
-		target_fb = fb;
-	else
-		target_fb = crtc->primary->fb;
+	target_fb = crtc->primary->fb;
 
 	/* If atomic, assume fb object is pinned & idle & fenced and
 	 * just update base pointers
@@ -1821,13 +1818,11 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
 	if (unlikely(r != 0))
 		return r;
 
-	if (!atomic) {
-		abo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
-		r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM);
-		if (unlikely(r != 0)) {
-			amdgpu_bo_unreserve(abo);
-			return -EINVAL;
-		}
+	abo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+	r = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM);
+	if (unlikely(r != 0)) {
+		amdgpu_bo_unreserve(abo);
+		return -EINVAL;
 	}
 	fb_location = amdgpu_bo_gpu_offset(abo);
 
@@ -1995,7 +1990,7 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
 	/* set pageflip to happen anywhere in vblank interval */
 	WREG32(mmMASTER_UPDATE_MODE + amdgpu_crtc->crtc_offset, 0);
 
-	if (!atomic && fb && fb != crtc->primary->fb) {
+	if (fb && fb != crtc->primary->fb) {
 		abo = gem_to_amdgpu_bo(fb->obj[0]);
 		r = amdgpu_bo_reserve(abo, true);
 		if (unlikely(r != 0))
@@ -2537,7 +2532,7 @@ static int dce_v8_0_crtc_mode_set(struct drm_crtc *crtc,
 
 	amdgpu_atombios_crtc_set_pll(crtc, adjusted_mode);
 	amdgpu_atombios_crtc_set_dtd_timing(crtc, adjusted_mode);
-	dce_v8_0_crtc_do_set_base(crtc, old_fb, x, y, 0);
+	dce_v8_0_crtc_do_set_base(crtc, old_fb, x, y);
 	amdgpu_atombios_crtc_overscan_setup(crtc, mode, adjusted_mode);
 	amdgpu_atombios_crtc_scaler_setup(crtc);
 	dce_v8_0_cursor_reset(crtc);
@@ -2585,14 +2580,7 @@ static bool dce_v8_0_crtc_mode_fixup(struct drm_crtc *crtc,
 static int dce_v8_0_crtc_set_base(struct drm_crtc *crtc, int x, int y,
 				  struct drm_framebuffer *old_fb)
 {
-	return dce_v8_0_crtc_do_set_base(crtc, old_fb, x, y, 0);
-}
-
-static int dce_v8_0_crtc_set_base_atomic(struct drm_crtc *crtc,
-					 struct drm_framebuffer *fb,
-					 int x, int y, enum mode_set_atomic state)
-{
-	return dce_v8_0_crtc_do_set_base(crtc, fb, x, y, 1);
+	return dce_v8_0_crtc_do_set_base(crtc, old_fb, x, y);
 }
 
 static const struct drm_crtc_helper_funcs dce_v8_0_crtc_helper_funcs = {
@@ -2600,7 +2588,6 @@ static const struct drm_crtc_helper_funcs dce_v8_0_crtc_helper_funcs = {
 	.mode_fixup = dce_v8_0_crtc_mode_fixup,
 	.mode_set = dce_v8_0_crtc_mode_set,
 	.mode_set_base = dce_v8_0_crtc_set_base,
-	.mode_set_base_atomic = dce_v8_0_crtc_set_base_atomic,
 	.prepare = dce_v8_0_crtc_prepare,
 	.commit = dce_v8_0_crtc_commit,
 	.disable = dce_v8_0_crtc_disable,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
index d75b9940f248..31c2d33f02d5 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
@@ -4575,6 +4575,7 @@ static const struct amdgpu_gfx_funcs gfx_v10_0_gfx_funcs = {
 	.select_me_pipe_q = &gfx_v10_0_select_me_pipe_q,
 	.init_spm_golden = &gfx_v10_0_init_spm_golden_registers,
 	.update_perfmon_mgcg = &gfx_v10_0_update_perfmon_mgcg,
+	.get_hdp_flush_mask = &amdgpu_gfx_get_hdp_flush_mask,
 };
 
 static void gfx_v10_0_gpu_early_init(struct amdgpu_device *adev)
@@ -8318,7 +8319,8 @@ static void gfx_v10_0_update_spm_vmid_internal(struct amdgpu_device *adev,
 	}
 }
 
-static void gfx_v10_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned int vmid)
+static void gfx_v10_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id,
+		struct amdgpu_ring *ring, unsigned int vmid)
 {
 	amdgpu_gfx_off_ctrl(adev, false);
 
@@ -8613,25 +8615,13 @@ static void gfx_v10_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 	u32 ref_and_mask, reg_mem_engine;
-	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
 
-	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
-		switch (ring->me) {
-		case 1:
-			ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
-			break;
-		case 2:
-			ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
-			break;
-		default:
-			return;
-		}
-		reg_mem_engine = 0;
-	} else {
-		ref_and_mask = nbio_hf_reg->ref_and_mask_cp0 << ring->pipe;
-		reg_mem_engine = 1; /* pfp */
+	if (!adev->gfx.funcs->get_hdp_flush_mask) {
+		dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__);
+		return;
 	}
 
+	adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, &reg_mem_engine);
 	gfx_v10_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
 			       adev->nbio.funcs->get_hdp_flush_req_offset(adev),
 			       adev->nbio.funcs->get_hdp_flush_done_offset(adev),
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index 8a2ee2de390f..79a6977d56b0 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -416,7 +416,8 @@ static void gfx11_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 
 	if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) {
-		amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr, seq);
+		amdgpu_mes_unmap_legacy_queue(adev, ring, action,
+					      gpu_addr, seq, 0);
 		return;
 	}
 
@@ -917,7 +918,7 @@ static int gfx_v11_0_rlc_init(struct amdgpu_device *adev)
 
 	/* init spm vmid with 0xf */
 	if (adev->gfx.rlc.funcs->update_spm_vmid)
-		adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf);
+		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf);
 
 	return 0;
 }
@@ -1052,10 +1053,14 @@ static void gfx_v11_0_select_me_pipe_q(struct amdgpu_device *adev,
 static void gfx_v11_0_get_gfx_shadow_info_nocheck(struct amdgpu_device *adev,
 					 struct amdgpu_gfx_shadow_info *shadow_info)
 {
+	/* for gfx */
 	shadow_info->shadow_size = MQD_SHADOW_BASE_SIZE;
 	shadow_info->shadow_alignment = MQD_SHADOW_BASE_ALIGNMENT;
 	shadow_info->csa_size = MQD_FWWORKAREA_SIZE;
 	shadow_info->csa_alignment = MQD_FWWORKAREA_ALIGNMENT;
+	/* for compute */
+	shadow_info->eop_size = GFX11_MEC_HPD_SIZE;
+	shadow_info->eop_alignment = 256;
 }
 
 static int gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device *adev,
@@ -1080,6 +1085,7 @@ static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = {
 	.select_me_pipe_q = &gfx_v11_0_select_me_pipe_q,
 	.update_perfmon_mgcg = &gfx_v11_0_update_perf_clk,
 	.get_gfx_shadow_info = &gfx_v11_0_get_gfx_shadow_info,
+	.get_hdp_flush_mask = &amdgpu_gfx_get_hdp_flush_mask,
 };
 
 static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev)
@@ -4593,7 +4599,7 @@ static int gfx_v11_0_cp_resume(struct amdgpu_device *adev)
 	}
 
 	if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)
-		r = amdgpu_mes_kiq_hw_init(adev);
+		r = amdgpu_mes_kiq_hw_init(adev, 0);
 	else
 		r = gfx_v11_0_kiq_resume(adev);
 	if (r)
@@ -4901,7 +4907,7 @@ static int gfx_v11_0_hw_fini(struct amdgpu_ip_block *ip_block)
 		if (amdgpu_gfx_disable_kcq(adev, 0))
 			DRM_ERROR("KCQ disable failed\n");
 
-		amdgpu_mes_kiq_hw_fini(adev);
+		amdgpu_mes_kiq_hw_fini(adev, 0);
 	}
 
 	if (amdgpu_sriov_vf(adev))
@@ -5568,7 +5574,8 @@ static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev,
 	return 0;
 }
 
-static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned vmid)
+static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id,
+		struct amdgpu_ring *ring, unsigned vmid)
 {
 	u32 reg, pre_data, data;
 
@@ -5831,25 +5838,13 @@ static void gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 	u32 ref_and_mask, reg_mem_engine;
-	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
 
-	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
-		switch (ring->me) {
-		case 1:
-			ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
-			break;
-		case 2:
-			ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
-			break;
-		default:
-			return;
-		}
-		reg_mem_engine = 0;
-	} else {
-		ref_and_mask = nbio_hf_reg->ref_and_mask_cp0 << ring->pipe;
-		reg_mem_engine = 1; /* pfp */
+	if (!adev->gfx.funcs->get_hdp_flush_mask) {
+		dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__);
+		return;
 	}
 
+	adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, &reg_mem_engine);
 	gfx_v11_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
 			       adev->nbio.funcs->get_hdp_flush_req_offset(adev),
 			       adev->nbio.funcs->get_hdp_flush_done_offset(adev),
@@ -6827,7 +6822,7 @@ static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring,
 
 	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
 
-	r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, false);
+	r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, false, 0);
 	if (r) {
 
 		dev_warn(adev->dev, "reset via MES failed and try pipe reset %d\n", r);
@@ -6842,7 +6837,7 @@ static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring,
 		return r;
 	}
 
-	r = amdgpu_mes_map_legacy_queue(adev, ring);
+	r = amdgpu_mes_map_legacy_queue(adev, ring, 0);
 	if (r) {
 		dev_err(adev->dev, "failed to remap kgq\n");
 		return r;
@@ -6990,7 +6985,7 @@ static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring,
 
 	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
 
-	r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, true);
+	r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, true, 0);
 	if (r) {
 		dev_warn(adev->dev, "fail(%d) to reset kcq and try pipe reset\n", r);
 		r = gfx_v11_0_reset_compute_pipe(ring);
@@ -7003,7 +6998,7 @@ static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring,
 		dev_err(adev->dev, "fail to init kcq\n");
 		return r;
 	}
-	r = amdgpu_mes_map_legacy_queue(adev, ring);
+	r = amdgpu_mes_map_legacy_queue(adev, ring, 0);
 	if (r) {
 		dev_err(adev->dev, "failed to remap kcq\n");
 		return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
index d01d2712cf57..b4dd954363ee 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
@@ -356,7 +356,8 @@ static void gfx_v12_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
 	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
 
 	if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) {
-		amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr, seq);
+		amdgpu_mes_unmap_legacy_queue(adev, ring, action,
+					      gpu_addr, seq, 0);
 		return;
 	}
 
@@ -761,7 +762,7 @@ static int gfx_v12_0_rlc_init(struct amdgpu_device *adev)
 
 	/* init spm vmid with 0xf */
 	if (adev->gfx.rlc.funcs->update_spm_vmid)
-		adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf);
+		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf);
 
 	return 0;
 }
@@ -909,10 +910,14 @@ static void gfx_v12_0_select_me_pipe_q(struct amdgpu_device *adev,
 static void gfx_v12_0_get_gfx_shadow_info_nocheck(struct amdgpu_device *adev,
 						  struct amdgpu_gfx_shadow_info *shadow_info)
 {
+	/* for gfx */
 	shadow_info->shadow_size = MQD_SHADOW_BASE_SIZE;
 	shadow_info->shadow_alignment = MQD_SHADOW_BASE_ALIGNMENT;
 	shadow_info->csa_size = MQD_FWWORKAREA_SIZE;
 	shadow_info->csa_alignment = MQD_FWWORKAREA_ALIGNMENT;
+	/* for compute */
+	shadow_info->eop_size = GFX12_MEC_HPD_SIZE;
+	shadow_info->eop_alignment = 256;
 }
 
 static int gfx_v12_0_get_gfx_shadow_info(struct amdgpu_device *adev,
@@ -937,6 +942,7 @@ static const struct amdgpu_gfx_funcs gfx_v12_0_gfx_funcs = {
 	.select_me_pipe_q = &gfx_v12_0_select_me_pipe_q,
 	.update_perfmon_mgcg = &gfx_v12_0_update_perf_clk,
 	.get_gfx_shadow_info = &gfx_v12_0_get_gfx_shadow_info,
+	.get_hdp_flush_mask = &amdgpu_gfx_get_hdp_flush_mask,
 };
 
 static int gfx_v12_0_gpu_early_init(struct amdgpu_device *adev)
@@ -3470,7 +3476,7 @@ static int gfx_v12_0_cp_resume(struct amdgpu_device *adev)
 	}
 
 	if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)
-		r = amdgpu_mes_kiq_hw_init(adev);
+		r = amdgpu_mes_kiq_hw_init(adev, 0);
 	else
 		r = gfx_v12_0_kiq_resume(adev);
 	if (r)
@@ -3759,7 +3765,7 @@ static int gfx_v12_0_hw_fini(struct amdgpu_ip_block *ip_block)
 		if (amdgpu_gfx_disable_kcq(adev, 0))
 			DRM_ERROR("KCQ disable failed\n");
 
-		amdgpu_mes_kiq_hw_fini(adev);
+		amdgpu_mes_kiq_hw_fini(adev, 0);
 	}
 
 	if (amdgpu_sriov_vf(adev)) {
@@ -3956,6 +3962,7 @@ static void gfx_v12_0_update_perf_clk(struct amdgpu_device *adev,
 }
 
 static void gfx_v12_0_update_spm_vmid(struct amdgpu_device *adev,
+				      int xcc_id,
 				      struct amdgpu_ring *ring,
 				      unsigned vmid)
 {
@@ -4387,25 +4394,13 @@ static void gfx_v12_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 	u32 ref_and_mask, reg_mem_engine;
-	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
 
-	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
-		switch (ring->me) {
-		case 1:
-			ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
-			break;
-		case 2:
-			ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
-			break;
-		default:
-			return;
-		}
-		reg_mem_engine = 0;
-	} else {
-		ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
-		reg_mem_engine = 1; /* pfp */
+	if (!adev->gfx.funcs->get_hdp_flush_mask) {
+		dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__);
+		return;
 	}
 
+	adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, &reg_mem_engine);
 	gfx_v12_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
 			       adev->nbio.funcs->get_hdp_flush_req_offset(adev),
 			       adev->nbio.funcs->get_hdp_flush_done_offset(adev),
@@ -5312,7 +5307,7 @@ static int gfx_v12_0_reset_kgq(struct amdgpu_ring *ring,
 
 	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
 
-	r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, false);
+	r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, false, 0);
 	if (r) {
 		dev_warn(adev->dev, "reset via MES failed and try pipe reset %d\n", r);
 		r = gfx_v12_reset_gfx_pipe(ring);
@@ -5326,7 +5321,7 @@ static int gfx_v12_0_reset_kgq(struct amdgpu_ring *ring,
 		return r;
 	}
 
-	r = amdgpu_mes_map_legacy_queue(adev, ring);
+	r = amdgpu_mes_map_legacy_queue(adev, ring, 0);
 	if (r) {
 		dev_err(adev->dev, "failed to remap kgq\n");
 		return r;
@@ -5427,7 +5422,7 @@ static int gfx_v12_0_reset_kcq(struct amdgpu_ring *ring,
 
 	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
 
-	r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, true);
+	r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, true, 0);
 	if (r) {
 		dev_warn(adev->dev, "fail(%d) to reset kcq  and try pipe reset\n", r);
 		r = gfx_v12_0_reset_compute_pipe(ring);
@@ -5440,7 +5435,7 @@ static int gfx_v12_0_reset_kcq(struct amdgpu_ring *ring,
 		dev_err(adev->dev, "failed to init kcq\n");
 		return r;
 	}
-	r = amdgpu_mes_map_legacy_queue(adev, ring);
+	r = amdgpu_mes_map_legacy_queue(adev, ring, 0);
 	if (r) {
 		dev_err(adev->dev, "failed to remap kcq\n");
 		return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
new file mode 100644
index 000000000000..6a4ecded103a
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
@@ -0,0 +1,3991 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <linux/delay.h>
+#include <linux/kernel.h>
+#include <linux/firmware.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include "amdgpu.h"
+#include "amdgpu_gfx.h"
+#include "amdgpu_psp.h"
+#include "amdgpu_smu.h"
+#include "amdgpu_atomfirmware.h"
+#include "imu_v12_1.h"
+#include "soc_v1_0.h"
+#include "gfx_v12_1_pkt.h"
+
+#include "gc/gc_12_1_0_offset.h"
+#include "gc/gc_12_1_0_sh_mask.h"
+#include "soc24_enum.h"
+#include "ivsrcid/gfx/irqsrcs_gfx_12_1_0.h"
+
+#include "soc15.h"
+#include "clearstate_gfx12.h"
+#include "v12_structs.h"
+#include "gfx_v12_1.h"
+#include "mes_v12_1.h"
+
+#define GFX12_MEC_HPD_SIZE	2048
+#define NUM_SIMD_PER_CU_GFX12_1	4
+
+#define RLCG_UCODE_LOADING_START_ADDRESS	0x00002000L
+
+MODULE_FIRMWARE("amdgpu/gc_12_1_0_mec.bin");
+MODULE_FIRMWARE("amdgpu/gc_12_1_0_rlc.bin");
+
+#define SH_MEM_ALIGNMENT_MODE_UNALIGNED_GFX12_1_0	0x00000001
+#define DEFAULT_SH_MEM_CONFIG \
+	((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \
+	 (SH_MEM_ALIGNMENT_MODE_UNALIGNED_GFX12_1_0 << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
+	 (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
+
+#define XCC_REG_RANGE_0_LOW  0x1260     /* XCC gfxdec0 lower Bound */
+#define XCC_REG_RANGE_0_HIGH 0x3C00     /* XCC gfxdec0 upper Bound */
+#define XCC_REG_RANGE_1_LOW  0xA000     /* XCC gfxdec1 lower Bound */
+#define XCC_REG_RANGE_1_HIGH 0x10000    /* XCC gfxdec1 upper Bound */
+#define NORMALIZE_XCC_REG_OFFSET(offset) \
+	(offset & 0xFFFF)
+
+static void gfx_v12_1_xcc_disable_gpa_mode(struct amdgpu_device *adev, int xcc_id);
+static void gfx_v12_1_set_ring_funcs(struct amdgpu_device *adev);
+static void gfx_v12_1_set_irq_funcs(struct amdgpu_device *adev);
+static void gfx_v12_1_set_rlc_funcs(struct amdgpu_device *adev);
+static void gfx_v12_1_set_mqd_funcs(struct amdgpu_device *adev);
+static void gfx_v12_1_set_imu_funcs(struct amdgpu_device *adev);
+static int gfx_v12_1_get_cu_info(struct amdgpu_device *adev,
+				 struct amdgpu_cu_info *cu_info);
+static uint64_t gfx_v12_1_get_gpu_clock_counter(struct amdgpu_device *adev);
+static void gfx_v12_1_xcc_select_se_sh(struct amdgpu_device *adev, u32 se_num,
+				       u32 sh_num, u32 instance, int xcc_id);
+static void gfx_v12_1_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
+				     uint32_t val);
+static int gfx_v12_1_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
+static void gfx_v12_1_ring_invalidate_tlbs(struct amdgpu_ring *ring,
+					   uint16_t pasid, uint32_t flush_type,
+					   bool all_hub, uint8_t dst_sel);
+static void gfx_v12_1_xcc_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
+static void gfx_v12_1_xcc_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
+static void gfx_v12_1_update_perf_clk(struct amdgpu_device *adev,
+				      bool enable);
+static void gfx_v12_1_xcc_update_perf_clk(struct amdgpu_device *adev,
+					 bool enable, int xcc_id);
+
+static void gfx_v12_1_kiq_set_resources(struct amdgpu_ring *kiq_ring,
+					uint64_t queue_mask)
+{
+	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
+	amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
+			  PACKET3_SET_RESOURCES_QUEUE_TYPE(0));	/* vmid_mask:0 queue_type:0 (KIQ) */
+	amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask));	/* queue mask lo */
+	amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask));	/* queue mask hi */
+	amdgpu_ring_write(kiq_ring, 0);	/* gws mask lo */
+	amdgpu_ring_write(kiq_ring, 0);	/* gws mask hi */
+	amdgpu_ring_write(kiq_ring, 0);	/* oac mask */
+	amdgpu_ring_write(kiq_ring, 0);
+}
+
+static void gfx_v12_1_kiq_map_queues(struct amdgpu_ring *kiq_ring,
+				     struct amdgpu_ring *ring)
+{
+	uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
+	uint64_t wptr_addr = ring->wptr_gpu_addr;
+	uint32_t me = 0, eng_sel = 0;
+
+	switch (ring->funcs->type) {
+	case AMDGPU_RING_TYPE_COMPUTE:
+		me = 1;
+		eng_sel = 0;
+		break;
+	case AMDGPU_RING_TYPE_MES:
+		me = 2;
+		eng_sel = 5;
+		break;
+	default:
+		WARN_ON(1);
+	}
+
+	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
+	/* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
+	amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
+			  PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
+			  PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
+			  PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
+			  PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
+			  PACKET3_MAP_QUEUES_ME((me)) |
+			  PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
+			  PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
+			  PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
+			  PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
+	amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
+	amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
+	amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
+	amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
+	amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
+}
+
+static void gfx_v12_1_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
+				       struct amdgpu_ring *ring,
+				       enum amdgpu_unmap_queues_action action,
+				       u64 gpu_addr, u64 seq)
+{
+	struct amdgpu_device *adev = kiq_ring->adev;
+	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
+
+	if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) {
+		amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr,
+					      seq, kiq_ring->xcc_id);
+		return;
+	}
+
+	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
+	amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
+			  PACKET3_UNMAP_QUEUES_ACTION(action) |
+			  PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
+			  PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
+			  PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
+	amdgpu_ring_write(kiq_ring,
+		  PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
+
+	if (action == PREEMPT_QUEUES_NO_UNMAP) {
+		amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
+		amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
+		amdgpu_ring_write(kiq_ring, seq);
+	} else {
+		amdgpu_ring_write(kiq_ring, 0);
+		amdgpu_ring_write(kiq_ring, 0);
+		amdgpu_ring_write(kiq_ring, 0);
+	}
+}
+
+static void gfx_v12_1_kiq_query_status(struct amdgpu_ring *kiq_ring,
+				       struct amdgpu_ring *ring,
+				       u64 addr, u64 seq)
+{
+	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
+
+	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
+	amdgpu_ring_write(kiq_ring,
+			  PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
+			  PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
+			  PACKET3_QUERY_STATUS_COMMAND(2));
+	amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
+			  PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
+			  PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
+	amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
+	amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
+	amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
+	amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
+}
+
+static void gfx_v12_1_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
+					  uint16_t pasid,
+					  uint32_t flush_type,
+					  bool all_hub)
+{
+	gfx_v12_1_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1);
+}
+
+static const struct kiq_pm4_funcs gfx_v12_1_kiq_pm4_funcs = {
+	.kiq_set_resources = gfx_v12_1_kiq_set_resources,
+	.kiq_map_queues = gfx_v12_1_kiq_map_queues,
+	.kiq_unmap_queues = gfx_v12_1_kiq_unmap_queues,
+	.kiq_query_status = gfx_v12_1_kiq_query_status,
+	.kiq_invalidate_tlbs = gfx_v12_1_kiq_invalidate_tlbs,
+	.set_resources_size = 8,
+	.map_queues_size = 7,
+	.unmap_queues_size = 6,
+	.query_status_size = 7,
+	.invalidate_tlbs_size = 2,
+};
+
+static void gfx_v12_1_set_kiq_pm4_funcs(struct amdgpu_device *adev)
+{
+	int i, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i =0; i < num_xcc; i++)
+		adev->gfx.kiq[i].pmf = &gfx_v12_1_kiq_pm4_funcs;
+}
+
+static uint32_t gfx_v12_1_normalize_xcc_reg_offset(uint32_t reg)
+{
+	uint32_t normalized_reg = NORMALIZE_XCC_REG_OFFSET(reg);
+
+	/* If it is an XCC reg, normalize the reg to keep
+	   lower 16 bits in local xcc */
+
+	if (((normalized_reg >= XCC_REG_RANGE_0_LOW) && (normalized_reg < XCC_REG_RANGE_0_HIGH)) ||
+		((normalized_reg >= XCC_REG_RANGE_1_LOW) && (normalized_reg < XCC_REG_RANGE_1_HIGH)))
+		return normalized_reg;
+	else
+		return reg;
+}
+
+static void gfx_v12_1_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
+				   int mem_space, int opt, uint32_t addr0,
+				   uint32_t addr1, uint32_t ref,
+				   uint32_t mask, uint32_t inv)
+{
+	if (mem_space == 0) {
+		addr0 = gfx_v12_1_normalize_xcc_reg_offset(addr0);
+		addr1 = gfx_v12_1_normalize_xcc_reg_offset(addr1);
+	}
+
+	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
+	amdgpu_ring_write(ring,
+			  /* memory (1) or register (0) */
+			  (WAIT_REG_MEM_MEM_SPACE(mem_space) |
+			   WAIT_REG_MEM_OPERATION(opt) | /* wait */
+			   WAIT_REG_MEM_FUNCTION(3) |  /* equal */
+			   WAIT_REG_MEM_ENGINE(eng_sel)));
+
+	if (mem_space)
+		BUG_ON(addr0 & 0x3); /* Dword align */
+	amdgpu_ring_write(ring, addr0);
+	amdgpu_ring_write(ring, addr1);
+	amdgpu_ring_write(ring, ref);
+	amdgpu_ring_write(ring, mask);
+	amdgpu_ring_write(ring, inv); /* poll interval */
+}
+
+static int gfx_v12_1_ring_test_ring(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+	uint32_t scratch_reg0_offset, xcc_offset;
+	uint32_t tmp = 0;
+	unsigned i;
+	int r;
+
+	/* Use register offset which is local to XCC in the packet */
+	xcc_offset = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
+	scratch_reg0_offset = SOC15_REG_OFFSET(GC, GET_INST(GC, ring->xcc_id), regSCRATCH_REG0);
+	WREG32(scratch_reg0_offset, 0xCAFEDEAD);
+	tmp = RREG32(scratch_reg0_offset);
+
+	r = amdgpu_ring_alloc(ring, 5);
+	if (r) {
+		dev_err(adev->dev,
+			"amdgpu: cp failed to lock ring %d (%d).\n",
+			ring->idx, r);
+		return r;
+	}
+
+	if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {
+		gfx_v12_1_ring_emit_wreg(ring, xcc_offset, 0xDEADBEEF);
+	} else {
+		amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
+		amdgpu_ring_write(ring, xcc_offset -
+				  PACKET3_SET_UCONFIG_REG_START);
+		amdgpu_ring_write(ring, 0xDEADBEEF);
+	}
+	amdgpu_ring_commit(ring);
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		tmp = RREG32(scratch_reg0_offset);
+		if (tmp == 0xDEADBEEF)
+			break;
+		if (amdgpu_emu_mode == 1)
+			msleep(1);
+		else
+			udelay(1);
+	}
+
+	if (i >= adev->usec_timeout)
+		r = -ETIMEDOUT;
+	return r;
+}
+
+static int gfx_v12_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
+{
+	struct amdgpu_device *adev = ring->adev;
+	struct amdgpu_ib ib;
+	struct dma_fence *f = NULL;
+	unsigned index;
+	uint64_t gpu_addr;
+	volatile uint32_t *cpu_ptr;
+	long r;
+
+	/* MES KIQ fw hasn't indirect buffer support for now */
+	if (adev->enable_mes_kiq &&
+	    ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
+		return 0;
+
+	memset(&ib, 0, sizeof(ib));
+
+	r = amdgpu_device_wb_get(adev, &index);
+	if (r)
+		return r;
+
+	gpu_addr = adev->wb.gpu_addr + (index * 4);
+	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
+	cpu_ptr = &adev->wb.wb[index];
+
+	r = amdgpu_ib_get(adev, NULL, 16, AMDGPU_IB_POOL_DIRECT, &ib);
+	if (r) {
+		dev_err(adev->dev, "amdgpu: failed to get ib (%ld).\n", r);
+		goto err1;
+	}
+
+	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
+	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
+	ib.ptr[2] = lower_32_bits(gpu_addr);
+	ib.ptr[3] = upper_32_bits(gpu_addr);
+	ib.ptr[4] = 0xDEADBEEF;
+	ib.length_dw = 5;
+
+	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
+	if (r)
+		goto err2;
+
+	r = dma_fence_wait_timeout(f, false, timeout);
+	if (r == 0) {
+		r = -ETIMEDOUT;
+		goto err2;
+	} else if (r < 0) {
+		goto err2;
+	}
+
+	if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF)
+		r = 0;
+	else
+		r = -EINVAL;
+err2:
+	amdgpu_ib_free(&ib, NULL);
+	dma_fence_put(f);
+err1:
+	amdgpu_device_wb_free(adev, index);
+	return r;
+}
+
+static void gfx_v12_1_free_microcode(struct amdgpu_device *adev)
+{
+	amdgpu_ucode_release(&adev->gfx.rlc_fw);
+	amdgpu_ucode_release(&adev->gfx.mec_fw);
+
+	kfree(adev->gfx.rlc.register_list_format);
+}
+
+static int gfx_v12_1_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix)
+{
+	const struct psp_firmware_header_v1_0 *toc_hdr;
+	int err = 0;
+
+	err = amdgpu_ucode_request(adev, &adev->psp.toc_fw,
+				   AMDGPU_UCODE_REQUIRED,
+				   "amdgpu/%s_toc.bin", ucode_prefix);
+	if (err)
+		goto out;
+
+	toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
+	adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
+	adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
+	adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
+	adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
+			le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
+	return 0;
+out:
+	amdgpu_ucode_release(&adev->psp.toc_fw);
+	return err;
+}
+
+static int gfx_v12_1_init_microcode(struct amdgpu_device *adev)
+{
+	char ucode_prefix[15];
+	int err;
+	const struct rlc_firmware_header_v2_0 *rlc_hdr;
+	uint16_t version_major;
+	uint16_t version_minor;
+
+	DRM_DEBUG("\n");
+
+	amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
+
+	if (!amdgpu_sriov_vf(adev)) {
+		err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
+					   AMDGPU_UCODE_REQUIRED,
+					   "amdgpu/%s_rlc.bin", ucode_prefix);
+		if (err)
+			goto out;
+		rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
+		version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
+		version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
+		err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
+		if (err)
+			goto out;
+	}
+
+	err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
+				   AMDGPU_UCODE_REQUIRED,
+				   "amdgpu/%s_mec.bin", ucode_prefix);
+	if (err)
+		goto out;
+	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC);
+	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK);
+	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK);
+	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK);
+	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK);
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
+		err = gfx_v12_1_init_toc_microcode(adev, ucode_prefix);
+
+	/* only one MEC for gfx 12 */
+	adev->gfx.mec2_fw = NULL;
+
+	if (adev->gfx.imu.funcs) {
+		if (adev->gfx.imu.funcs->init_microcode) {
+			err = adev->gfx.imu.funcs->init_microcode(adev);
+			if (err)
+				dev_err(adev->dev, "Failed to load imu firmware!\n");
+		}
+	}
+
+out:
+	if (err) {
+		amdgpu_ucode_release(&adev->gfx.rlc_fw);
+		amdgpu_ucode_release(&adev->gfx.mec_fw);
+	}
+
+	return err;
+}
+
+static u32 gfx_v12_1_get_csb_size(struct amdgpu_device *adev)
+{
+	u32 count = 0;
+	const struct cs_section_def *sect = NULL;
+	const struct cs_extent_def *ext = NULL;
+
+	count += 1;
+
+	for (sect = gfx12_cs_data; sect->section != NULL; ++sect) {
+		if (sect->id == SECT_CONTEXT) {
+			for (ext = sect->section; ext->extent != NULL; ++ext)
+				count += 2 + ext->reg_count;
+		} else
+			return 0;
+	}
+
+	return count;
+}
+
+static void gfx_v12_1_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer)
+{
+	u32 count = 0, clustercount = 0, i;
+	const struct cs_section_def *sect = NULL;
+	const struct cs_extent_def *ext = NULL;
+
+	if (adev->gfx.rlc.cs_data == NULL)
+		return;
+	if (buffer == NULL)
+		return;
+
+	count += 1;
+
+	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
+		if (sect->id == SECT_CONTEXT) {
+			for (ext = sect->section; ext->extent != NULL; ++ext) {
+				clustercount++;
+				buffer[count++] = ext->reg_count;
+				buffer[count++] = ext->reg_index;
+
+				for (i = 0; i < ext->reg_count; i++)
+					buffer[count++] = cpu_to_le32(ext->extent[i]);
+			}
+		} else
+			return;
+	}
+
+	buffer[0] = clustercount;
+}
+
+static void gfx_v12_1_rlc_fini(struct amdgpu_device *adev)
+{
+	/* clear state block */
+	amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
+			&adev->gfx.rlc.clear_state_gpu_addr,
+			(void **)&adev->gfx.rlc.cs_ptr);
+
+	/* jump table block */
+	amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
+			&adev->gfx.rlc.cp_table_gpu_addr,
+			(void **)&adev->gfx.rlc.cp_table_ptr);
+}
+
+static void gfx_v12_1_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
+{
+	int xcc_id, num_xcc;
+	struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
+		reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[GET_INST(GC, xcc_id)];
+		reg_access_ctrl->scratch_reg0 =
+			SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regSCRATCH_REG0);
+		reg_access_ctrl->scratch_reg1 =
+			SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regSCRATCH_REG1);
+		reg_access_ctrl->scratch_reg2 =
+			SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regSCRATCH_REG2);
+		reg_access_ctrl->scratch_reg3 =
+			SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regSCRATCH_REG3);
+		reg_access_ctrl->grbm_cntl =
+			SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regGRBM_GFX_CNTL);
+		reg_access_ctrl->grbm_idx =
+			SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regGRBM_GFX_INDEX);
+		reg_access_ctrl->spare_int =
+			SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SPARE_INT_0);
+	}
+	adev->gfx.rlc.rlcg_reg_access_supported = true;
+}
+
+static int gfx_v12_1_rlc_init(struct amdgpu_device *adev)
+{
+	const struct cs_section_def *cs_data;
+	int r, i, num_xcc;
+
+	adev->gfx.rlc.cs_data = gfx12_cs_data;
+
+	cs_data = adev->gfx.rlc.cs_data;
+
+	if (cs_data) {
+		/* init clear state block */
+		r = amdgpu_gfx_rlc_init_csb(adev);
+		if (r)
+			return r;
+	}
+
+	/* init spm vmid with 0xf */
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < num_xcc; i++) {
+		if (adev->gfx.rlc.funcs->update_spm_vmid)
+			adev->gfx.rlc.funcs->update_spm_vmid(adev, i, NULL, 0xf);
+	}
+
+	return 0;
+}
+
+static void gfx_v12_1_mec_fini(struct amdgpu_device *adev)
+{
+	amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
+	amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
+	amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL);
+}
+
+static int gfx_v12_1_mec_init(struct amdgpu_device *adev)
+{
+	int r, i, num_xcc;
+	u32 *hpd;
+	size_t mec_hpd_size;
+
+	bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < num_xcc; i++)
+		bitmap_zero(adev->gfx.mec_bitmap[i].queue_bitmap,
+			    AMDGPU_MAX_COMPUTE_QUEUES);
+
+	/* take ownership of the relevant compute queues */
+	amdgpu_gfx_compute_queue_acquire(adev);
+	mec_hpd_size = adev->gfx.num_compute_rings *
+		       GFX12_MEC_HPD_SIZE * num_xcc;
+
+	if (mec_hpd_size) {
+		r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
+					      AMDGPU_GEM_DOMAIN_GTT,
+					      &adev->gfx.mec.hpd_eop_obj,
+					      &adev->gfx.mec.hpd_eop_gpu_addr,
+					      (void **)&hpd);
+		if (r) {
+			dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
+			gfx_v12_1_mec_fini(adev);
+			return r;
+		}
+
+		memset(hpd, 0, mec_hpd_size);
+
+		amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
+		amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
+	}
+
+	return 0;
+}
+
+static uint32_t wave_read_ind(struct amdgpu_device *adev,
+			      uint32_t xcc_id, uint32_t wave,
+			      uint32_t address)
+{
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_INDEX,
+		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
+		(address << SQ_IND_INDEX__INDEX__SHIFT));
+	return RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_DATA);
+}
+
+static void wave_read_regs(struct amdgpu_device *adev,
+			   uint32_t xcc_id, uint32_t wave,
+			   uint32_t thread, uint32_t regno,
+			   uint32_t num, uint32_t *out)
+{
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_INDEX,
+		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
+		(regno << SQ_IND_INDEX__INDEX__SHIFT) |
+		(thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) |
+		(SQ_IND_INDEX__AUTO_INCR_MASK));
+	while (num--)
+		*(out++) = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_DATA);
+}
+
+static void gfx_v12_1_read_wave_data(struct amdgpu_device *adev,
+				     uint32_t xcc_id,
+				     uint32_t simd, uint32_t wave,
+				     uint32_t *dst, int *no_fields)
+{
+	/* in gfx12 the SIMD_ID is specified as part of the INSTANCE
+	 * field when performing a select_se_sh so it should be
+	 * zero here */
+	WARN_ON(simd != 0);
+
+	/* type 4 wave data */
+	dst[(*no_fields)++] = 4;
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_STATUS);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_PC_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_PC_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_EXEC_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_EXEC_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_HW_ID1);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_HW_ID2);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_GPR_ALLOC);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_LDS_ALLOC);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_IB_STS);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_IB_STS2);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_IB_DBG1);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_M0);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_MODE);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_STATE_PRIV);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_EXCP_FLAG_PRIV);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_EXCP_FLAG_USER);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_TRAP_CTRL);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_ACTIVE);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_VALID_AND_IDLE);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_DVGPR_ALLOC_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_DVGPR_ALLOC_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_SCHED_MODE);
+}
+
+static void gfx_v12_1_read_wave_sgprs(struct amdgpu_device *adev,
+				      uint32_t xcc_id, uint32_t simd,
+				      uint32_t wave, uint32_t start,
+				      uint32_t size, uint32_t *dst)
+{
+	WARN_ON(simd != 0);
+
+	wave_read_regs(adev, xcc_id, wave, 0,
+		       start + SQIND_WAVE_SGPRS_OFFSET,
+		       size, dst);
+}
+
+static void gfx_v12_1_read_wave_vgprs(struct amdgpu_device *adev,
+				      uint32_t xcc_id, uint32_t simd,
+				      uint32_t wave, uint32_t thread,
+				      uint32_t start, uint32_t size,
+				      uint32_t *dst)
+{
+	wave_read_regs(adev, xcc_id, wave, thread,
+		       start + SQIND_WAVE_VGPRS_OFFSET,
+		       size, dst);
+}
+
+static void gfx_v12_1_select_me_pipe_q(struct amdgpu_device *adev,
+				       u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
+{
+	soc_v1_0_grbm_select(adev, me, pipe, q, vm, GET_INST(GC, xcc_id));
+}
+
+static int gfx_v12_1_get_xccs_per_xcp(struct amdgpu_device *adev)
+{
+	/* Fill this in when the interface is ready */
+	return 1;
+}
+
+static int gfx_v12_1_ih_to_xcc_inst(struct amdgpu_device *adev, int ih_node)
+{
+	int logic_xcc;
+	int xcc = (ih_node & 0x7) - 2 + (ih_node >> 3) * 4;
+
+	for (logic_xcc = 0; logic_xcc < NUM_XCC(adev->gfx.xcc_mask); logic_xcc++) {
+		if (xcc == GET_INST(GC, logic_xcc))
+			return logic_xcc;
+	}
+
+	dev_err(adev->dev, "Couldn't find xcc mapping from IH node");
+	return -EINVAL;
+}
+
+static const struct amdgpu_gfx_funcs gfx_v12_1_gfx_funcs = {
+	.get_gpu_clock_counter = &gfx_v12_1_get_gpu_clock_counter,
+	.select_se_sh = &gfx_v12_1_xcc_select_se_sh,
+	.read_wave_data = &gfx_v12_1_read_wave_data,
+	.read_wave_sgprs = &gfx_v12_1_read_wave_sgprs,
+	.read_wave_vgprs = &gfx_v12_1_read_wave_vgprs,
+	.select_me_pipe_q = &gfx_v12_1_select_me_pipe_q,
+	.update_perfmon_mgcg = &gfx_v12_1_update_perf_clk,
+	.get_xccs_per_xcp = &gfx_v12_1_get_xccs_per_xcp,
+	.ih_node_to_logical_xcc = &gfx_v12_1_ih_to_xcc_inst,
+};
+
+static int gfx_v12_1_gpu_early_init(struct amdgpu_device *adev)
+{
+	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
+	case IP_VERSION(12, 1, 0):
+		adev->gfx.config.max_hw_contexts = 8;
+		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
+		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
+		adev->gfx.config.sc_hiz_tile_fifo_size = 0;
+		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
+		break;
+	default:
+		BUG();
+		break;
+	}
+
+	return 0;
+}
+
+static int gfx_v12_1_compute_ring_init(struct amdgpu_device *adev, int ring_id,
+				       int xcc_id, int mec, int pipe, int queue)
+{
+	int r;
+	unsigned irq_type;
+	struct amdgpu_ring *ring;
+	unsigned int hw_prio;
+	uint32_t xcc_doorbell_start;
+
+	ring = &adev->gfx.compute_ring[xcc_id * adev->gfx.num_compute_rings +
+				       ring_id];
+
+	/* mec0 is me1 */
+	ring->xcc_id = xcc_id;
+	ring->me = mec + 1;
+	ring->pipe = pipe;
+	ring->queue = queue;
+
+	ring->ring_obj = NULL;
+	ring->use_doorbell = true;
+	xcc_doorbell_start = adev->doorbell_index.mec_ring0 +
+			     xcc_id * adev->doorbell_index.xcc_doorbell_range;
+	ring->doorbell_index = (xcc_doorbell_start + ring_id) << 1;
+	ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr +
+			     (ring_id + xcc_id * adev->gfx.num_compute_rings) *
+			     GFX12_MEC_HPD_SIZE;
+	ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
+	sprintf(ring->name, "comp_%d.%d.%d.%d",
+			ring->xcc_id, ring->me, ring->pipe, ring->queue);
+
+	irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
+		+ ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
+		+ ring->pipe;
+	hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
+			AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
+	/* type-2 packets are deprecated on MEC, use type-3 instead */
+	r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
+			     hw_prio, NULL);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static struct {
+	SOC24_FIRMWARE_ID	id;
+	unsigned int		offset;
+	unsigned int		size;
+	unsigned int		size_x16;
+	unsigned int		num_inst;
+} rlc_autoload_info[SOC24_FIRMWARE_ID_MAX];
+
+#define RLC_TOC_OFFSET_DWUNIT   8
+#define RLC_SIZE_MULTIPLE       1024
+#define RLC_TOC_UMF_SIZE_inM	23ULL
+#define RLC_TOC_FORMAT_API	165ULL
+
+#define RLC_NUM_INS_CODE0   1
+#define RLC_NUM_INS_CODE1   8
+#define RLC_NUM_INS_CODE2   2
+#define RLC_NUM_INS_CODE3   16
+
+static void gfx_v12_1_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc)
+{
+	RLC_TABLE_OF_CONTENT_V2 *ucode = rlc_toc;
+
+	while (ucode && (ucode->id > SOC24_FIRMWARE_ID_INVALID)) {
+		rlc_autoload_info[ucode->id].id = ucode->id;
+		rlc_autoload_info[ucode->id].offset =
+			ucode->offset * RLC_TOC_OFFSET_DWUNIT * 4;
+		rlc_autoload_info[ucode->id].size =
+			ucode->size_x16 ? ucode->size * RLC_SIZE_MULTIPLE * 4 :
+					  ucode->size * 4;
+		switch (ucode->vfflr_image_code) {
+		case 0:
+			rlc_autoload_info[ucode->id].num_inst =
+				RLC_NUM_INS_CODE0;
+			break;
+		case 1:
+			rlc_autoload_info[ucode->id].num_inst =
+				RLC_NUM_INS_CODE1;
+			break;
+		case 2:
+			rlc_autoload_info[ucode->id].num_inst =
+				RLC_NUM_INS_CODE2;
+			break;
+		case 3:
+			rlc_autoload_info[ucode->id].num_inst =
+				RLC_NUM_INS_CODE3;
+			break;
+		default:
+			dev_err(adev->dev,
+				"Invalid Instance number detected\n");
+			break;
+		}
+		ucode++;
+	}
+}
+
+static uint32_t gfx_v12_1_calc_toc_total_size(struct amdgpu_device *adev)
+{
+	uint32_t total_size = 0;
+	SOC24_FIRMWARE_ID id;
+
+	gfx_v12_1_parse_rlc_toc(adev, adev->psp.toc.start_addr);
+
+	for (id = SOC24_FIRMWARE_ID_RLC_G_UCODE; id < SOC24_FIRMWARE_ID_MAX; id++)
+		total_size += rlc_autoload_info[id].size;
+
+	/* In case the offset in rlc toc ucode is aligned */
+	if (total_size < rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].offset)
+		total_size = rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].offset +
+			rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].size;
+	if (total_size < (RLC_TOC_UMF_SIZE_inM << 20))
+		total_size = RLC_TOC_UMF_SIZE_inM << 20;
+
+	return total_size;
+}
+
+static int gfx_v12_1_rlc_autoload_buffer_init(struct amdgpu_device *adev)
+{
+	int r;
+	uint32_t total_size;
+
+	total_size = gfx_v12_1_calc_toc_total_size(adev);
+
+	r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024,
+				      AMDGPU_GEM_DOMAIN_VRAM,
+				      &adev->gfx.rlc.rlc_autoload_bo,
+				      &adev->gfx.rlc.rlc_autoload_gpu_addr,
+				      (void **)&adev->gfx.rlc.rlc_autoload_ptr);
+
+	if (r) {
+		dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
+		return r;
+	}
+
+	return 0;
+}
+
+static void gfx_v12_1_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
+						       SOC24_FIRMWARE_ID id,
+						       const void *fw_data,
+						       uint32_t fw_size)
+{
+	uint32_t toc_offset;
+	uint32_t toc_fw_size, toc_fw_inst_size;
+	char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
+	int i, num_inst;
+
+	if (id <= SOC24_FIRMWARE_ID_INVALID || id >= SOC24_FIRMWARE_ID_MAX)
+		return;
+
+	toc_offset = rlc_autoload_info[id].offset;
+	toc_fw_size = rlc_autoload_info[id].size;
+	num_inst = rlc_autoload_info[id].num_inst;
+	toc_fw_inst_size = toc_fw_size / num_inst;
+
+	if (fw_size == 0)
+		fw_size = toc_fw_inst_size;
+
+	if (fw_size > toc_fw_inst_size)
+		fw_size = toc_fw_inst_size;
+
+	for (i = 0; i < num_inst; i++) {
+		if ((num_inst == RLC_NUM_INS_CODE0) ||
+		    ((1 << (i / 2)) & adev->gfx.xcc_mask)) {
+			memcpy(ptr + toc_offset + i * toc_fw_inst_size, fw_data, fw_size);
+
+			if (fw_size < toc_fw_inst_size)
+				memset(ptr + toc_offset + fw_size + i * toc_fw_inst_size,
+				       0, toc_fw_inst_size - fw_size);
+		}
+	}
+}
+
+static void
+gfx_v12_1_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev)
+{
+	void *data;
+	uint32_t size;
+	uint32_t *toc_ptr;
+
+	data = adev->psp.toc.start_addr;
+	size = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_TOC].size;
+
+	toc_ptr = (uint32_t *)data + size / 4 - 2;
+	*toc_ptr = (RLC_TOC_FORMAT_API << 24) | 0x1;
+
+	gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_TOC,
+						   data, size);
+}
+
+static void
+gfx_v12_1_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev)
+{
+	const __le32 *fw_data;
+	uint32_t fw_size;
+	const struct gfx_firmware_header_v2_0 *cpv2_hdr;
+	const struct rlc_firmware_header_v2_0 *rlc_hdr;
+	const struct rlc_firmware_header_v2_1 *rlcv21_hdr;
+	const struct rlc_firmware_header_v2_2 *rlcv22_hdr;
+	uint16_t version_major, version_minor;
+
+	/* mec ucode */
+	cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
+		adev->gfx.mec_fw->data;
+	/* instruction */
+	fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
+		le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
+	fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
+	gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC,
+						   fw_data, fw_size);
+	/* data */
+	fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
+		le32_to_cpu(cpv2_hdr->data_offset_bytes));
+	fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
+	gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P0_STACK,
+						   fw_data, fw_size);
+	gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P1_STACK,
+						   fw_data, fw_size);
+	gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P2_STACK,
+						   fw_data, fw_size);
+	gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P3_STACK,
+						   fw_data, fw_size);
+
+	/* rlc ucode */
+	rlc_hdr = (const struct rlc_firmware_header_v2_0 *)
+		adev->gfx.rlc_fw->data;
+	fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
+			le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes));
+	fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes);
+	gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_G_UCODE,
+						   fw_data, fw_size);
+
+	version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
+	version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
+	if (version_major == 2) {
+		if (version_minor >= 1) {
+			rlcv21_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
+
+			fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
+					le32_to_cpu(rlcv21_hdr->save_restore_list_gpm_offset_bytes));
+			fw_size = le32_to_cpu(rlcv21_hdr->save_restore_list_gpm_size_bytes);
+			gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLCG_SCRATCH,
+						   fw_data, fw_size);
+
+			fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
+					le32_to_cpu(rlcv21_hdr->save_restore_list_srm_offset_bytes));
+			fw_size = le32_to_cpu(rlcv21_hdr->save_restore_list_srm_size_bytes);
+			gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_SRM_ARAM,
+						   fw_data, fw_size);
+		}
+		if (version_minor >= 2) {
+			rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
+
+			fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
+					le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes));
+			fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes);
+			gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLX6_UCODE,
+						   fw_data, fw_size);
+
+			fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
+					le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes));
+			fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes);
+			gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLX6_DRAM_BOOT,
+						   fw_data, fw_size);
+		}
+	}
+}
+
+static void
+gfx_v12_1_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev)
+{
+	const __le32 *fw_data;
+	uint32_t fw_size;
+	const struct sdma_firmware_header_v3_0 *sdma_hdr;
+
+	if (adev->sdma.instance[0].fw) {
+		sdma_hdr = (const struct sdma_firmware_header_v3_0 *)
+			adev->sdma.instance[0].fw->data;
+		fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
+				le32_to_cpu(sdma_hdr->ucode_offset_bytes));
+		fw_size = le32_to_cpu(sdma_hdr->ucode_size_bytes);
+
+		gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_SDMA_UCODE_TH0,
+							   fw_data, fw_size);
+	}
+}
+
+static void
+gfx_v12_1_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev)
+{
+	const __le32 *fw_data;
+	unsigned fw_size;
+	const struct mes_firmware_header_v1_0 *mes_hdr;
+	int pipe, ucode_id, data_id;
+
+	for (pipe = 0; pipe < 2; pipe++) {
+		if (pipe == 0) {
+			ucode_id = SOC24_FIRMWARE_ID_RS64_MES_P0;
+			data_id  = SOC24_FIRMWARE_ID_RS64_MES_P0_STACK;
+		} else {
+			ucode_id = SOC24_FIRMWARE_ID_RS64_MES_P1;
+			data_id  = SOC24_FIRMWARE_ID_RS64_MES_P1_STACK;
+		}
+
+		mes_hdr = (const struct mes_firmware_header_v1_0 *)
+			adev->mes.fw[pipe]->data;
+
+		fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
+				le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
+		fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
+
+		gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, ucode_id, fw_data, fw_size);
+
+		fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
+				le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
+		fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
+
+		gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, data_id, fw_data, fw_size);
+	}
+}
+
+static int gfx_v12_1_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
+{
+	uint32_t rlc_g_offset, rlc_g_size;
+	uint64_t gpu_addr;
+	uint32_t data;
+	int i, num_xcc;
+
+	/* RLC autoload sequence 2: copy ucode */
+	gfx_v12_1_rlc_backdoor_autoload_copy_sdma_ucode(adev);
+	gfx_v12_1_rlc_backdoor_autoload_copy_gfx_ucode(adev);
+	gfx_v12_1_rlc_backdoor_autoload_copy_mes_ucode(adev);
+	gfx_v12_1_rlc_backdoor_autoload_copy_toc_ucode(adev);
+
+	rlc_g_offset = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_G_UCODE].offset;
+	rlc_g_size = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_G_UCODE].size;
+	gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset - adev->gmc.vram_start;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < num_xcc; i++) {
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGFX_IMU_RLC_BOOTLOADER_ADDR_HI,
+			     upper_32_bits(gpu_addr));
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGFX_IMU_RLC_BOOTLOADER_ADDR_LO,
+			     lower_32_bits(gpu_addr));
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGFX_IMU_RLC_BOOTLOADER_SIZE,
+			     rlc_g_size);
+	}
+
+	if (adev->gfx.imu.funcs) {
+		/* RLC autoload sequence 3: load IMU fw */
+		if (adev->gfx.imu.funcs->load_microcode)
+			adev->gfx.imu.funcs->load_microcode(adev);
+	}
+
+	/* unhalt rlc to start autoload */
+	for (i = 0; i < num_xcc; i++) {
+		data = RREG32_SOC15(GC, GET_INST(GC, i), regRLC_GPM_THREAD_ENABLE);
+		data = REG_SET_FIELD(data, RLC_GPM_THREAD_ENABLE, THREAD0_ENABLE, 1);
+		data = REG_SET_FIELD(data, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1);
+		WREG32_SOC15(GC, GET_INST(GC, i), regRLC_GPM_THREAD_ENABLE, data);
+		WREG32_SOC15(GC, GET_INST(GC, i), regRLC_CNTL, RLC_CNTL__RLC_ENABLE_F32_MASK);
+	}
+
+	return 0;
+}
+
+static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
+{
+	int i, j, k, r, ring_id = 0;
+	unsigned num_compute_rings;
+	int xcc_id, num_xcc;
+	struct amdgpu_device *adev = ip_block->adev;
+
+	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
+	case IP_VERSION(12, 1, 0):
+		adev->gfx.mec.num_mec = 1;
+		adev->gfx.mec.num_pipe_per_mec = 4;
+		adev->gfx.mec.num_queue_per_pipe = 8;
+		break;
+	default:
+		adev->gfx.mec.num_mec = 2;
+		adev->gfx.mec.num_pipe_per_mec = 2;
+		adev->gfx.mec.num_queue_per_pipe = 4;
+		break;
+	}
+
+	/* recalculate compute rings to use based on hardware configuration */
+	num_compute_rings = (adev->gfx.mec.num_pipe_per_mec *
+			     adev->gfx.mec.num_queue_per_pipe) / 2;
+	adev->gfx.num_compute_rings = min(adev->gfx.num_compute_rings,
+					  num_compute_rings);
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+
+	/* EOP Event */
+	r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_GRBM_CP,
+			      GFX_12_1_0__SRCID__CP_EOP_INTERRUPT,
+			      &adev->gfx.eop_irq);
+	if (r)
+		return r;
+
+	/* Privileged reg */
+	r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_GRBM_CP,
+			      GFX_12_1_0__SRCID__CP_PRIV_REG_FAULT,
+			      &adev->gfx.priv_reg_irq);
+	if (r)
+		return r;
+
+	/* Privileged inst */
+	r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_GRBM_CP,
+			      GFX_12_1_0__SRCID__CP_PRIV_INSTR_FAULT,
+			      &adev->gfx.priv_inst_irq);
+	if (r)
+		return r;
+
+	adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
+
+	r = gfx_v12_1_rlc_init(adev);
+	if (r) {
+		dev_err(adev->dev, "Failed to init rlc BOs!\n");
+		return r;
+	}
+
+	r = gfx_v12_1_mec_init(adev);
+	if (r) {
+		dev_err(adev->dev, "Failed to init MEC BOs!\n");
+		return r;
+	}
+
+	/* set up the compute queues - allocate horizontally across pipes */
+	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
+		ring_id = 0;
+		for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
+			for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
+				for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
+					if (!amdgpu_gfx_is_mec_queue_enabled(adev,
+								xcc_id, i, k, j))
+						continue;
+
+					r = gfx_v12_1_compute_ring_init(adev, ring_id,
+								xcc_id, i, k, j);
+					if (r)
+						return r;
+
+					ring_id++;
+				}
+			}
+		}
+
+		if (!adev->enable_mes_kiq) {
+			r = amdgpu_gfx_kiq_init(adev, GFX12_MEC_HPD_SIZE, xcc_id);
+			if (r) {
+				dev_err(adev->dev, "Failed to init KIQ BOs!\n");
+				return r;
+			}
+
+			r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
+			if (r)
+				return r;
+		}
+
+		r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v12_1_compute_mqd), xcc_id);
+		if (r)
+			return r;
+	}
+
+	/* allocate visible FB for rlc auto-loading fw */
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
+		r = gfx_v12_1_rlc_autoload_buffer_init(adev);
+		if (r)
+			return r;
+	}
+
+	r = gfx_v12_1_gpu_early_init(adev);
+	if (r)
+		return r;
+
+	r = amdgpu_gfx_sysfs_init(adev);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static void gfx_v12_1_rlc_autoload_buffer_fini(struct amdgpu_device *adev)
+{
+	amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
+			&adev->gfx.rlc.rlc_autoload_gpu_addr,
+			(void **)&adev->gfx.rlc.rlc_autoload_ptr);
+}
+
+static int gfx_v12_1_sw_fini(struct amdgpu_ip_block *ip_block)
+{
+	int i, num_xcc;
+	struct amdgpu_device *adev = ip_block->adev;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < adev->gfx.num_compute_rings * num_xcc; i++)
+		amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
+
+	for (i = 0; i < num_xcc; i++) {
+		amdgpu_gfx_mqd_sw_fini(adev, i);
+
+		if (!adev->enable_mes_kiq) {
+			amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[i].ring);
+			amdgpu_gfx_kiq_fini(adev, i);
+		}
+	}
+
+	gfx_v12_1_rlc_fini(adev);
+	gfx_v12_1_mec_fini(adev);
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
+		gfx_v12_1_rlc_autoload_buffer_fini(adev);
+
+	gfx_v12_1_free_microcode(adev);
+
+	return 0;
+}
+
+static void gfx_v12_1_xcc_select_se_sh(struct amdgpu_device *adev, u32 se_num,
+				       u32 sh_num, u32 instance, int xcc_id)
+{
+	u32 data;
+
+	if (instance == 0xffffffff)
+		data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
+				     INSTANCE_BROADCAST_WRITES, 1);
+	else
+		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX,
+				     instance);
+
+	if (se_num == 0xffffffff)
+		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES,
+				     1);
+	else
+		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
+
+	if (sh_num == 0xffffffff)
+		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES,
+				     1);
+	else
+		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num);
+
+	WREG32_SOC15_RLC_SHADOW_EX(reg, GC, GET_INST(GC, xcc_id), regGRBM_GFX_INDEX, data);
+}
+
+static u32 gfx_v12_1_get_sa_active_bitmap(struct amdgpu_device *adev,
+					  int xcc_id)
+{
+	u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask;
+
+	gc_disabled_sa_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCC_GC_SA_UNIT_DISABLE);
+	gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask,
+					    CC_GC_SA_UNIT_DISABLE,
+					    SA_DISABLE);
+	gc_user_disabled_sa_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regGC_USER_SA_UNIT_DISABLE);
+	gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask,
+						 GC_USER_SA_UNIT_DISABLE,
+						 SA_DISABLE);
+	sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se *
+					    adev->gfx.config.max_shader_engines);
+
+	return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask));
+}
+
+static u32 gfx_v12_1_get_rb_active_bitmap(struct amdgpu_device *adev,
+					  int xcc_id)
+{
+	u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask;
+	u32 rb_mask;
+
+	gc_disabled_rb_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
+					   regCC_RB_BACKEND_DISABLE);
+	gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask,
+					    CC_RB_BACKEND_DISABLE,
+					    BACKEND_DISABLE);
+	gc_user_disabled_rb_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
+						regGC_USER_RB_BACKEND_DISABLE);
+	gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask,
+						 GC_USER_RB_BACKEND_DISABLE,
+						 BACKEND_DISABLE);
+	rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se *
+					    adev->gfx.config.max_shader_engines);
+
+	return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask));
+}
+
+static void gfx_v12_1_setup_rb(struct amdgpu_device *adev)
+{
+	u32 rb_bitmap_width_per_sa;
+	u32 max_sa;
+	u32 active_sa_bitmap;
+	u32 global_active_rb_bitmap;
+	u32 active_rb_bitmap = 0;
+	u32 i;
+	int xcc_id;
+
+	for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++) {
+		/* query sa bitmap from SA_UNIT_DISABLE registers */
+		active_sa_bitmap = gfx_v12_1_get_sa_active_bitmap(adev, xcc_id);
+		/* query rb bitmap from RB_BACKEND_DISABLE registers */
+		global_active_rb_bitmap = gfx_v12_1_get_rb_active_bitmap(adev, xcc_id);
+
+		/* generate active rb bitmap according to active sa bitmap */
+		max_sa = adev->gfx.config.max_shader_engines *
+			 adev->gfx.config.max_sh_per_se;
+		rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se /
+					 adev->gfx.config.max_sh_per_se;
+		for (i = 0; i < max_sa; i++) {
+			if (active_sa_bitmap & (1 << i))
+				active_rb_bitmap |= (0x3 << (i * rb_bitmap_width_per_sa));
+		}
+
+		active_rb_bitmap |= global_active_rb_bitmap;
+	}
+
+	adev->gfx.config.backend_enable_mask = active_rb_bitmap;
+	adev->gfx.config.num_rbs = hweight32(active_rb_bitmap);
+}
+
+static void gfx_v12_1_xcc_init_compute_vmid(struct amdgpu_device *adev,
+					    int xcc_id)
+{
+	int i;
+	uint32_t sh_mem_bases;
+	uint32_t data;
+
+	/*
+	 * Configure apertures:
+	 * LDS:         0x20000000'00000000 - 0x20000001'00000000 (4GB)
+	 * Scratch:     0x10000000'00000000 - 0x10000001'00000000 (4GB)
+	 */
+	sh_mem_bases = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
+				     (adev->gmc.private_aperture_start >> 58));
+	sh_mem_bases = REG_SET_FIELD(sh_mem_bases, SH_MEM_BASES, SHARED_BASE,
+				     (adev->gmc.shared_aperture_start >> 48));
+
+	mutex_lock(&adev->srbm_mutex);
+	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
+		soc_v1_0_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id));
+		/* CP and shaders */
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSH_MEM_BASES, sh_mem_bases);
+
+		/* Enable trap for each kfd vmid. */
+		data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSPI_GDBG_PER_VMID_CNTL);
+		data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSPI_GDBG_PER_VMID_CNTL, data);
+
+		/* Disable VGPR deallocation instruction for each KFD vmid. */
+		data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_DEBUG);
+		data = REG_SET_FIELD(data, SQ_DEBUG, DISABLE_VGPR_DEALLOC, 1);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_DEBUG, data);
+	}
+	soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
+	mutex_unlock(&adev->srbm_mutex);
+}
+
+static void gfx_v12_1_tcp_harvest(struct amdgpu_device *adev)
+{
+	/* TODO: harvest feature to be added later. */
+}
+
+static void gfx_v12_1_get_tcc_info(struct amdgpu_device *adev)
+{
+}
+
+static void gfx_v12_1_xcc_constants_init(struct amdgpu_device *adev,
+					 int xcc_id)
+{
+	u32 tmp;
+	int i;
+
+	/* XXX SH_MEM regs */
+	/* where to put LDS, scratch, GPUVM in FSA64 space */
+	mutex_lock(&adev->srbm_mutex);
+	for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
+		soc_v1_0_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id));
+		/* CP and shaders */
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+			     regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
+		if (i != 0) {
+			tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
+				(adev->gmc.private_aperture_start >> 58));
+			tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
+				(adev->gmc.shared_aperture_start >> 48));
+			WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSH_MEM_BASES, tmp);
+		}
+	}
+	soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, 0));
+
+	mutex_unlock(&adev->srbm_mutex);
+
+	gfx_v12_1_xcc_init_compute_vmid(adev, xcc_id);
+}
+
+static void gfx_v12_1_constants_init(struct amdgpu_device *adev)
+{
+	int i, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+
+	gfx_v12_1_setup_rb(adev);
+	gfx_v12_1_get_cu_info(adev, &adev->gfx.cu_info);
+	gfx_v12_1_get_tcc_info(adev);
+	adev->gfx.config.pa_sc_tile_steering_override = 0;
+
+	for (i = 0; i < num_xcc; i++)
+		gfx_v12_1_xcc_constants_init(adev, i);
+}
+
+static void gfx_v12_1_xcc_enable_gui_idle_interrupt(struct amdgpu_device *adev,
+						    bool enable, int xcc_id)
+{
+	u32 tmp;
+
+	if (amdgpu_sriov_vf(adev))
+		return;
+
+	tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL_RING0);
+
+	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE,
+			    enable ? 1 : 0);
+	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE,
+			    enable ? 1 : 0);
+	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE,
+			    enable ? 1 : 0);
+	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE,
+			    enable ? 1 : 0);
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL_RING0, tmp);
+}
+
+static int gfx_v12_1_xcc_init_csb(struct amdgpu_device *adev,
+				  int xcc_id)
+{
+	adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CSIB_ADDR_HI,
+			adev->gfx.rlc.clear_state_gpu_addr >> 32);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CSIB_ADDR_LO,
+			adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+		     regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
+
+	return 0;
+}
+
+static void gfx_v12_1_xcc_rlc_stop(struct amdgpu_device *adev,
+				   int xcc_id)
+{
+	u32 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CNTL, tmp);
+}
+
+static void gfx_v12_1_rlc_stop(struct amdgpu_device *adev)
+{
+	int i, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < num_xcc; i++)
+		gfx_v12_1_xcc_rlc_stop(adev, i);
+}
+
+static void gfx_v12_1_xcc_rlc_reset(struct amdgpu_device *adev,
+				    int xcc_id)
+{
+	WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id),
+			      GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
+	udelay(50);
+	WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id),
+			      GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
+	udelay(50);
+}
+
+static void gfx_v12_1_rlc_reset(struct amdgpu_device *adev)
+{
+	int i, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < num_xcc; i++)
+		gfx_v12_1_xcc_rlc_reset(adev, i);
+}
+
+static void gfx_v12_1_xcc_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
+						 bool enable, int xcc_id)
+{
+	uint32_t rlc_pg_cntl;
+
+	rlc_pg_cntl = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_PG_CNTL);
+
+	if (!enable) {
+		/* RLC_PG_CNTL[23] = 0 (default)
+		 * RLC will wait for handshake acks with SMU
+		 * GFXOFF will be enabled
+		 * RLC_PG_CNTL[23] = 1
+		 * RLC will not issue any message to SMU
+		 * hence no handshake between SMU & RLC
+		 * GFXOFF will be disabled
+		 */
+		rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
+	} else
+		rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_PG_CNTL, rlc_pg_cntl);
+}
+
+static void gfx_v12_1_xcc_rlc_start(struct amdgpu_device *adev,
+				    int xcc_id)
+{
+	/* TODO: enable rlc & smu handshake until smu
+	 * and gfxoff feature works as expected */
+	if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))
+		gfx_v12_1_xcc_rlc_smu_handshake_cntl(adev, false, xcc_id);
+
+	WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), RLC_CNTL, RLC_ENABLE_F32, 1);
+	udelay(50);
+}
+
+static void gfx_v12_1_rlc_start(struct amdgpu_device *adev)
+{
+	int i, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < num_xcc; i++) {
+		gfx_v12_1_xcc_rlc_start(adev, i);
+	}
+}
+
+static void gfx_v12_1_xcc_rlc_enable_srm(struct amdgpu_device *adev,
+					 int xcc_id)
+{
+	uint32_t tmp;
+
+	/* enable Save Restore Machine */
+	tmp = RREG32(SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SRM_CNTL));
+	tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
+	tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
+	WREG32(SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SRM_CNTL), tmp);
+}
+
+static void gfx_v12_1_xcc_load_rlcg_microcode(struct amdgpu_device *adev,
+					      int xcc_id)
+{
+	const struct rlc_firmware_header_v2_0 *hdr;
+	const __le32 *fw_data;
+	unsigned i, fw_size;
+
+	hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
+	fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
+			   le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+	fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_GPM_UCODE_ADDR,
+		     RLCG_UCODE_LOADING_START_ADDRESS);
+
+	for (i = 0; i < fw_size; i++)
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+			     regRLC_GPM_UCODE_DATA,
+			     le32_to_cpup(fw_data++));
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+		     regRLC_GPM_UCODE_ADDR,
+		     adev->gfx.rlc_fw_version);
+}
+
+static void gfx_v12_1_xcc_load_rlc_iram_dram_microcode(struct amdgpu_device *adev,
+						       int xcc_id)
+{
+	const struct rlc_firmware_header_v2_2 *hdr;
+	const __le32 *fw_data;
+	unsigned i, fw_size;
+	u32 tmp;
+
+	hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
+
+	fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
+			le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes));
+	fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4;
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_LX6_IRAM_ADDR, 0);
+
+	for (i = 0; i < fw_size; i++) {
+		if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
+			msleep(1);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+			     regRLC_LX6_IRAM_DATA,
+			     le32_to_cpup(fw_data++));
+	}
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+		     regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
+
+	fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
+			le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes));
+	fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4;
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+		     regRLC_LX6_DRAM_ADDR, 0);
+	for (i = 0; i < fw_size; i++) {
+		if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
+			msleep(1);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+			     regRLC_LX6_DRAM_DATA,
+			     le32_to_cpup(fw_data++));
+	}
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+		     regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
+
+	tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_LX6_CNTL);
+	tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1);
+	tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_LX6_CNTL, tmp);
+}
+
+static int gfx_v12_1_xcc_rlc_load_microcode(struct amdgpu_device *adev,
+					    int xcc_id)
+{
+	const struct rlc_firmware_header_v2_0 *hdr;
+	uint16_t version_major;
+	uint16_t version_minor;
+
+	if (!adev->gfx.rlc_fw)
+		return -EINVAL;
+
+	hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
+	amdgpu_ucode_print_rlc_hdr(&hdr->header);
+
+	version_major = le16_to_cpu(hdr->header.header_version_major);
+	version_minor = le16_to_cpu(hdr->header.header_version_minor);
+
+	if (version_major == 2) {
+		gfx_v12_1_xcc_load_rlcg_microcode(adev, xcc_id);
+		if (amdgpu_dpm == 1) {
+			if (version_minor >= 2)
+				gfx_v12_1_xcc_load_rlc_iram_dram_microcode(adev, xcc_id);
+		}
+
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static int gfx_v12_1_xcc_rlc_resume(struct amdgpu_device *adev,
+				    int xcc_id)
+{
+	int r;
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		gfx_v12_1_xcc_init_csb(adev, xcc_id);
+
+		if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
+			gfx_v12_1_xcc_rlc_enable_srm(adev, xcc_id);
+	} else {
+		if (amdgpu_sriov_vf(adev)) {
+			gfx_v12_1_xcc_init_csb(adev, xcc_id);
+			return 0;
+		}
+
+		gfx_v12_1_xcc_rlc_stop(adev, xcc_id);
+
+		/* disable CG */
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, 0);
+
+		/* disable PG */
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_PG_CNTL, 0);
+
+		if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+			/* legacy rlc firmware loading */
+			r = gfx_v12_1_xcc_rlc_load_microcode(adev, xcc_id);
+			if (r)
+				return r;
+		}
+
+		gfx_v12_1_xcc_init_csb(adev, xcc_id);
+
+		gfx_v12_1_xcc_rlc_start(adev, xcc_id);
+	}
+
+	return 0;
+}
+
+static int gfx_v12_1_rlc_resume(struct amdgpu_device *adev)
+{
+	int r, i, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < num_xcc; i++) {
+		r = gfx_v12_1_xcc_rlc_resume(adev, i);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+
+static void gfx_v12_1_xcc_config_gfx_rs64(struct amdgpu_device *adev,
+					  int xcc_id)
+{
+	const struct gfx_firmware_header_v2_0 *mec_hdr;
+	uint32_t pipe_id, tmp;
+
+	mec_hdr = (const struct gfx_firmware_header_v2_0 *)
+		adev->gfx.mec_fw->data;
+
+	/* config mec program start addr */
+	for (pipe_id = 0; pipe_id < 4; pipe_id++) {
+		soc_v1_0_grbm_select(adev, 1, pipe_id, 0, 0, GET_INST(GC, xcc_id));
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_PRGRM_CNTR_START,
+					mec_hdr->ucode_start_addr_lo >> 2 |
+					mec_hdr->ucode_start_addr_hi << 30);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_PRGRM_CNTR_START_HI,
+					mec_hdr->ucode_start_addr_hi >> 2);
+	}
+	soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, 0));
+
+	/* reset mec pipe */
+	tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL);
+	tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1);
+	tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1);
+	tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1);
+	tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL, tmp);
+
+	/* clear mec pipe reset */
+	tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0);
+	tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0);
+	tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0);
+	tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL, tmp);
+}
+
+static void gfx_v12_1_config_gfx_rs64(struct amdgpu_device *adev)
+{
+	int i, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+
+	for (i = 0; i < num_xcc; i++)
+		gfx_v12_1_xcc_config_gfx_rs64(adev, i);
+}
+
+static void gfx_v12_1_xcc_set_mec_ucode_start_addr(struct amdgpu_device *adev,
+						   int xcc_id)
+{
+	const struct gfx_firmware_header_v2_0 *cp_hdr;
+	unsigned pipe_id;
+
+	cp_hdr = (const struct gfx_firmware_header_v2_0 *)
+		adev->gfx.mec_fw->data;
+	mutex_lock(&adev->srbm_mutex);
+	for (pipe_id = 0; pipe_id < adev->gfx.mec.num_pipe_per_mec; pipe_id++) {
+		soc_v1_0_grbm_select(adev, 1, pipe_id, 0, 0, GET_INST(GC, xcc_id));
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_PRGRM_CNTR_START,
+			     cp_hdr->ucode_start_addr_lo >> 2 |
+			     cp_hdr->ucode_start_addr_hi << 30);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_PRGRM_CNTR_START_HI,
+			     cp_hdr->ucode_start_addr_hi >> 2);
+	}
+	soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, 0));
+	mutex_unlock(&adev->srbm_mutex);
+}
+
+static int gfx_v12_1_xcc_wait_for_rlc_autoload_complete(struct amdgpu_device *adev,
+							int xcc_id)
+{
+	uint32_t cp_status;
+	uint32_t bootload_status;
+	int i;
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		cp_status = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_STAT);
+		bootload_status = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
+					       regRLC_RLCS_BOOTLOAD_STATUS);
+
+		if ((cp_status == 0) &&
+		    (REG_GET_FIELD(bootload_status,
+			RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) {
+			break;
+		}
+		udelay(1);
+		if (amdgpu_emu_mode)
+			msleep(10);
+	}
+
+	if (i >= adev->usec_timeout) {
+		dev_err(adev->dev,
+			"rlc autoload: xcc%d gc ucode autoload timeout\n", xcc_id);
+		return -ETIMEDOUT;
+	}
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
+		gfx_v12_1_xcc_set_mec_ucode_start_addr(adev, xcc_id);
+	}
+
+	return 0;
+}
+
+static int gfx_v12_1_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
+{
+	int xcc_id;
+
+	for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++)
+		gfx_v12_1_xcc_wait_for_rlc_autoload_complete(adev, xcc_id);
+
+	return 0;
+}
+
+static void gfx_v12_1_xcc_cp_compute_enable(struct amdgpu_device *adev,
+					    bool enable, int xcc_id)
+{
+	u32 data;
+
+	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL);
+	data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE,
+						 enable ? 0 : 1);
+	data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET,
+						 enable ? 0 : 1);
+	data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET,
+						 enable ? 0 : 1);
+	data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET,
+						 enable ? 0 : 1);
+	data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET,
+						 enable ? 0 : 1);
+	data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE,
+						 enable ? 1 : 0);
+	data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE,
+			                         enable ? 1 : 0);
+	data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE,
+						 enable ? 1 : 0);
+	data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE,
+						 enable ? 1 : 0);
+	data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT,
+						 enable ? 0 : 1);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL, data);
+
+	adev->gfx.kiq[xcc_id].ring.sched.ready = enable;
+
+	udelay(50);
+}
+
+static int gfx_v12_1_xcc_cp_compute_load_microcode_rs64(struct amdgpu_device *adev,
+							uint16_t xcc_mask)
+{
+	const struct gfx_firmware_header_v2_0 *mec_hdr;
+	const __le32 *fw_ucode, *fw_data;
+	u32 tmp, fw_ucode_size, fw_data_size;
+	u32 i, usec_timeout = 50000; /* Wait for 50 ms */
+	u32 *fw_ucode_ptr, *fw_data_ptr;
+	int r, xcc_id;
+
+	if (!adev->gfx.mec_fw)
+		return -EINVAL;
+
+	mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
+	amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
+
+	fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data +
+				le32_to_cpu(mec_hdr->ucode_offset_bytes));
+	fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes);
+
+	fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
+				le32_to_cpu(mec_hdr->data_offset_bytes));
+	fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes);
+
+	r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
+				      64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
+				      &adev->gfx.mec.mec_fw_obj,
+				      &adev->gfx.mec.mec_fw_gpu_addr,
+				      (void **)&fw_ucode_ptr);
+	if (r) {
+		dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
+		gfx_v12_1_mec_fini(adev);
+		return r;
+	}
+
+	r = amdgpu_bo_create_reserved(adev,
+				      ALIGN(fw_data_size, 64 * 1024) *
+				      adev->gfx.mec.num_pipe_per_mec * NUM_XCC(xcc_mask),
+				      64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
+				      &adev->gfx.mec.mec_fw_data_obj,
+				      &adev->gfx.mec.mec_fw_data_gpu_addr,
+				      (void **)&fw_data_ptr);
+	if (r) {
+		dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
+		gfx_v12_1_mec_fini(adev);
+		return r;
+	}
+
+	memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size);
+	for (xcc_id = 0; xcc_id < NUM_XCC(xcc_mask); xcc_id++) {
+		for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
+			u32 offset = (xcc_id * adev->gfx.mec.num_pipe_per_mec + i) *
+				     ALIGN(fw_data_size, 64 * 1024) / 4;
+			memcpy(fw_data_ptr + offset, fw_data, fw_data_size);
+		}
+	}
+
+	amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
+	amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj);
+	amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
+	amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj);
+
+	for (xcc_id = 0; xcc_id < NUM_XCC(xcc_mask); xcc_id++) {
+		gfx_v12_1_xcc_cp_compute_enable(adev, false, xcc_id);
+
+		tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_CNTL);
+		tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
+		tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
+		tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_CNTL, tmp);
+
+		tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_BASE_CNTL);
+		tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
+		tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_BASE_CNTL, tmp);
+
+		mutex_lock(&adev->srbm_mutex);
+		for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
+			soc_v1_0_grbm_select(adev, 1, i, 0, 0, GET_INST(GC, xcc_id));
+
+			WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_MDBASE_LO,
+					lower_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr +
+									(xcc_id * adev->gfx.mec.num_pipe_per_mec + i) *
+									ALIGN(fw_data_size, 64 * 1024)));
+			WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_MDBASE_HI,
+					upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr +
+									(xcc_id * adev->gfx.mec.num_pipe_per_mec + i) *
+									ALIGN(fw_data_size, 64 * 1024)));
+
+			WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_LO,
+					lower_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
+			WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_HI,
+					upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
+		}
+		mutex_unlock(&adev->srbm_mutex);
+		soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, 0));
+
+		/* Trigger an invalidation of the L1 instruction caches */
+		tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_OP_CNTL);
+		tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_OP_CNTL, tmp);
+
+		/* Wait for invalidation complete */
+		for (i = 0; i < usec_timeout; i++) {
+			tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_OP_CNTL);
+			if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
+						INVALIDATE_DCACHE_COMPLETE))
+				break;
+			udelay(1);
+		}
+
+		if (i >= usec_timeout) {
+			dev_err(adev->dev, "failed to invalidate instruction cache\n");
+			return -EINVAL;
+		}
+
+		/* Trigger an invalidation of the L1 instruction caches */
+		tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_OP_CNTL);
+		tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_OP_CNTL, tmp);
+
+		/* Wait for invalidation complete */
+		for (i = 0; i < usec_timeout; i++) {
+			tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_OP_CNTL);
+			if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
+						INVALIDATE_CACHE_COMPLETE))
+				break;
+			udelay(1);
+		}
+
+		if (i >= usec_timeout) {
+			dev_err(adev->dev, "failed to invalidate instruction cache\n");
+			return -EINVAL;
+		}
+
+		gfx_v12_1_xcc_set_mec_ucode_start_addr(adev, xcc_id);
+	}
+
+	return 0;
+}
+
+static void gfx_v12_1_xcc_kiq_setting(struct amdgpu_ring *ring,
+				      int xcc_id)
+{
+	uint32_t tmp;
+	struct amdgpu_device *adev = ring->adev;
+
+	/* tell RLC which is KIQ queue */
+	tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS);
+	tmp &= 0xffffff00;
+	tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);
+	tmp |= 0x80;
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);
+}
+
+static void gfx_v12_1_xcc_cp_set_doorbell_range(struct amdgpu_device *adev,
+						int xcc_id)
+{
+	/* disable gfx engine doorbell range */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_DOORBELL_RANGE_LOWER, 0);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_DOORBELL_RANGE_UPPER, 0);
+
+	/* set compute engine doorbell range */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DOORBELL_RANGE_LOWER,
+		     ((adev->doorbell_index.kiq +
+		       xcc_id * adev->doorbell_index.xcc_doorbell_range) *
+		      2) << 2);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DOORBELL_RANGE_UPPER,
+		     ((adev->doorbell_index.userqueue_end +
+		       xcc_id * adev->doorbell_index.xcc_doorbell_range) *
+		      2) << 2);
+}
+
+static int gfx_v12_1_compute_mqd_init(struct amdgpu_device *adev, void *m,
+				      struct amdgpu_mqd_prop *prop)
+{
+	struct v12_1_compute_mqd *mqd = m;
+	uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
+	uint32_t tmp;
+
+	mqd->header = 0xC0310800;
+	mqd->compute_pipelinestat_enable = 0x00000001;
+	mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
+	mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
+	mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
+	mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
+	mqd->compute_misc_reserved = 0x00000007;
+
+	eop_base_addr = prop->eop_gpu_addr >> 8;
+	mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
+	mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
+
+	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
+	tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HQD_EOP_CONTROL);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
+			(order_base_2(GFX12_MEC_HPD_SIZE / 4) - 1));
+
+	mqd->cp_hqd_eop_control = tmp;
+
+	/* enable doorbell? */
+	tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HQD_PQ_DOORBELL_CONTROL);
+
+	if (prop->use_doorbell) {
+		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+				    DOORBELL_OFFSET, prop->doorbell_index);
+		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+				    DOORBELL_EN, 1);
+		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+				    DOORBELL_SOURCE, 0);
+		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+				    DOORBELL_HIT, 0);
+	} else {
+		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+				    DOORBELL_EN, 0);
+	}
+
+	mqd->cp_hqd_pq_doorbell_control = tmp;
+
+	/* disable the queue if it's active */
+	mqd->cp_hqd_dequeue_request = 0;
+	mqd->cp_hqd_pq_rptr = 0;
+	mqd->cp_hqd_pq_wptr_lo = 0;
+	mqd->cp_hqd_pq_wptr_hi = 0;
+
+	/* set the pointer to the MQD */
+	mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc;
+	mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
+
+	/* set MQD vmid to 0 */
+	tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_MQD_CONTROL);
+	tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
+	mqd->cp_mqd_control = tmp;
+
+	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
+	hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
+	mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
+	mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
+
+	/* set up the HQD, this is similar to CP_RB0_CNTL */
+	tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HQD_PQ_CONTROL);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
+			    (order_base_2(prop->queue_size / 4) - 1));
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
+			    (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
+	mqd->cp_hqd_pq_control = tmp;
+
+	/* set the wb address whether it's enabled or not */
+	wb_gpu_addr = prop->rptr_gpu_addr;
+	mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
+	mqd->cp_hqd_pq_rptr_report_addr_hi =
+		upper_32_bits(wb_gpu_addr) & 0xffff;
+
+	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
+	wb_gpu_addr = prop->wptr_gpu_addr;
+	mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
+	mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
+
+	tmp = 0;
+	/* enable the doorbell if requested */
+	if (prop->use_doorbell) {
+		tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HQD_PQ_DOORBELL_CONTROL);
+		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+				DOORBELL_OFFSET, prop->doorbell_index);
+
+		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+				    DOORBELL_EN, 1);
+		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+				    DOORBELL_SOURCE, 0);
+		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+				    DOORBELL_HIT, 0);
+	}
+
+	mqd->cp_hqd_pq_doorbell_control = tmp;
+
+	/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
+	mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HQD_PQ_RPTR);
+
+	/* set the vmid for the queue */
+	mqd->cp_hqd_vmid = 0;
+
+	tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HQD_PERSISTENT_STATE);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x63);
+	mqd->cp_hqd_persistent_state = tmp;
+
+	/* set MIN_IB_AVAIL_SIZE */
+	tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HQD_IB_CONTROL);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 1);
+	mqd->cp_hqd_ib_control = tmp;
+
+	/* set static priority for a compute queue/ring */
+	mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
+	mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
+
+	mqd->cp_hqd_active = prop->hqd_active;
+
+	return 0;
+}
+
+static int gfx_v12_1_xcc_kiq_init_register(struct amdgpu_ring *ring,
+					   int xcc_id)
+{
+	struct amdgpu_device *adev = ring->adev;
+	struct v12_1_compute_mqd *mqd = ring->mqd_ptr;
+	int j;
+
+	/* inactivate the queue */
+	if (amdgpu_sriov_vf(adev))
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE, 0);
+
+	/* disable wptr polling */
+	WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), CP_PQ_WPTR_POLL_CNTL, EN, 0);
+
+	/* write the EOP addr */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_BASE_ADDR,
+	       mqd->cp_hqd_eop_base_addr_lo);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_BASE_ADDR_HI,
+	       mqd->cp_hqd_eop_base_addr_hi);
+
+	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_CONTROL,
+	       mqd->cp_hqd_eop_control);
+
+	/* enable doorbell? */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL,
+	       mqd->cp_hqd_pq_doorbell_control);
+
+	/* disable the queue if it's active */
+	if (RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1) {
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST, 1);
+		for (j = 0; j < adev->usec_timeout; j++) {
+			if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1))
+				break;
+			udelay(1);
+		}
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST,
+		       mqd->cp_hqd_dequeue_request);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR,
+		       mqd->cp_hqd_pq_rptr);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO,
+		       mqd->cp_hqd_pq_wptr_lo);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI,
+		       mqd->cp_hqd_pq_wptr_hi);
+	}
+
+	/* set the pointer to the MQD */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR,
+	       mqd->cp_mqd_base_addr_lo);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR_HI,
+	       mqd->cp_mqd_base_addr_hi);
+
+	/* set MQD vmid to 0 */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_CONTROL,
+	       mqd->cp_mqd_control);
+
+	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE,
+	       mqd->cp_hqd_pq_base_lo);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE_HI,
+	       mqd->cp_hqd_pq_base_hi);
+
+	/* set up the HQD, this is similar to CP_RB0_CNTL */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_CONTROL,
+	       mqd->cp_hqd_pq_control);
+
+	/* set the wb address whether it's enabled or not */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR,
+		mqd->cp_hqd_pq_rptr_report_addr_lo);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
+		mqd->cp_hqd_pq_rptr_report_addr_hi);
+
+	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR,
+	       mqd->cp_hqd_pq_wptr_poll_addr_lo);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
+	       mqd->cp_hqd_pq_wptr_poll_addr_hi);
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL,
+	       mqd->cp_hqd_pq_doorbell_control);
+
+	/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO,
+	       mqd->cp_hqd_pq_wptr_lo);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI,
+	       mqd->cp_hqd_pq_wptr_hi);
+
+	/* set the vmid for the queue */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_VMID, mqd->cp_hqd_vmid);
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PERSISTENT_STATE,
+	       mqd->cp_hqd_persistent_state);
+
+	/* activate the queue */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE,
+	       mqd->cp_hqd_active);
+
+	if (ring->use_doorbell)
+		WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), CP_PQ_STATUS, DOORBELL_ENABLE, 1);
+
+	return 0;
+}
+
+static int gfx_v12_1_xcc_kiq_init_queue(struct amdgpu_ring *ring,
+					int xcc_id)
+{
+	struct amdgpu_device *adev = ring->adev;
+	struct v12_1_compute_mqd *mqd = ring->mqd_ptr;
+
+	gfx_v12_1_xcc_kiq_setting(ring, xcc_id);
+
+	if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
+		/* reset MQD to a clean status */
+		if (adev->gfx.kiq[xcc_id].mqd_backup)
+			memcpy(mqd, adev->gfx.kiq[xcc_id].mqd_backup, sizeof(*mqd));
+
+		/* reset ring buffer */
+		ring->wptr = 0;
+		amdgpu_ring_clear_ring(ring);
+
+		mutex_lock(&adev->srbm_mutex);
+		soc_v1_0_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id));
+		gfx_v12_1_xcc_kiq_init_register(ring, xcc_id);
+		soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
+		mutex_unlock(&adev->srbm_mutex);
+	} else {
+		memset((void *)mqd, 0, sizeof(*mqd));
+		if (amdgpu_sriov_vf(adev) && adev->in_suspend)
+			amdgpu_ring_clear_ring(ring);
+		mutex_lock(&adev->srbm_mutex);
+		soc_v1_0_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id));
+		amdgpu_ring_init_mqd(ring);
+		gfx_v12_1_xcc_kiq_init_register(ring, xcc_id);
+		soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
+		mutex_unlock(&adev->srbm_mutex);
+
+		if (adev->gfx.kiq[xcc_id].mqd_backup)
+			memcpy(adev->gfx.kiq[xcc_id].mqd_backup, mqd, sizeof(*mqd));
+	}
+
+	return 0;
+}
+
+static int gfx_v12_1_xcc_kcq_init_queue(struct amdgpu_ring *ring,
+					int xcc_id)
+{
+	struct amdgpu_device *adev = ring->adev;
+	struct v12_1_compute_mqd *mqd = ring->mqd_ptr;
+	int mqd_idx = ring - &adev->gfx.compute_ring[0];
+
+	if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
+		memset((void *)mqd, 0, sizeof(*mqd));
+		mutex_lock(&adev->srbm_mutex);
+		soc_v1_0_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id));
+		amdgpu_ring_init_mqd(ring);
+		soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
+		mutex_unlock(&adev->srbm_mutex);
+
+		if (adev->gfx.mec.mqd_backup[mqd_idx])
+			memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
+	} else {
+		/* restore MQD to a clean status */
+		if (adev->gfx.mec.mqd_backup[mqd_idx])
+			memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
+		/* reset ring buffer */
+		ring->wptr = 0;
+		atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
+		amdgpu_ring_clear_ring(ring);
+	}
+
+	return 0;
+}
+
+static int gfx_v12_1_xcc_kiq_resume(struct amdgpu_device *adev,
+				    int xcc_id)
+{
+	struct amdgpu_ring *ring;
+	int r;
+
+	ring = &adev->gfx.kiq[xcc_id].ring;
+
+	r = amdgpu_bo_reserve(ring->mqd_obj, false);
+	if (unlikely(r != 0))
+		return r;
+
+	r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
+	if (unlikely(r != 0)) {
+		amdgpu_bo_unreserve(ring->mqd_obj);
+		return r;
+	}
+
+	gfx_v12_1_xcc_kiq_init_queue(ring, xcc_id);
+	amdgpu_bo_kunmap(ring->mqd_obj);
+	ring->mqd_ptr = NULL;
+	amdgpu_bo_unreserve(ring->mqd_obj);
+	ring->sched.ready = true;
+	return 0;
+}
+
+static int gfx_v12_1_xcc_kcq_resume(struct amdgpu_device *adev,
+				    int xcc_id)
+{
+	struct amdgpu_ring *ring = NULL;
+	int r = 0, i;
+
+	if (!amdgpu_async_gfx_ring)
+		gfx_v12_1_xcc_cp_compute_enable(adev, true, xcc_id);
+
+	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+		ring = &adev->gfx.compute_ring[i + xcc_id * adev->gfx.num_compute_rings];
+
+		r = amdgpu_bo_reserve(ring->mqd_obj, false);
+		if (unlikely(r != 0))
+			goto done;
+		r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
+		if (!r) {
+			r = gfx_v12_1_xcc_kcq_init_queue(ring, xcc_id);
+			amdgpu_bo_kunmap(ring->mqd_obj);
+			ring->mqd_ptr = NULL;
+		}
+		amdgpu_bo_unreserve(ring->mqd_obj);
+		if (r)
+			goto done;
+	}
+
+	r = amdgpu_gfx_enable_kcq(adev, xcc_id);
+done:
+	return r;
+}
+
+static int gfx_v12_1_xcc_cp_resume(struct amdgpu_device *adev, uint16_t xcc_mask)
+{
+	int r, i, xcc_id;
+	struct amdgpu_ring *ring;
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+		/* legacy firmware loading */
+		r = gfx_v12_1_xcc_cp_compute_load_microcode_rs64(adev, xcc_mask);
+		if (r)
+			return r;
+	}
+
+	for (xcc_id = 0; xcc_id < NUM_XCC(xcc_mask); xcc_id++) {
+		if (!(adev->flags & AMD_IS_APU))
+			gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, false, xcc_id);
+
+		gfx_v12_1_xcc_cp_set_doorbell_range(adev, xcc_id);
+
+		gfx_v12_1_xcc_cp_compute_enable(adev, true, xcc_id);
+
+		if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)
+			r = amdgpu_mes_kiq_hw_init(adev, xcc_id);
+		else
+			r = gfx_v12_1_xcc_kiq_resume(adev, xcc_id);
+		if (r)
+			return r;
+
+		r = gfx_v12_1_xcc_kcq_resume(adev, xcc_id);
+		if (r)
+			return r;
+
+		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+			ring = &adev->gfx.compute_ring[i + xcc_id * adev->gfx.num_compute_rings];
+			r = amdgpu_ring_test_helper(ring);
+			if (r)
+				return r;
+		}
+	}
+
+	return 0;
+}
+
+static int gfx_v12_1_cp_resume(struct amdgpu_device *adev)
+{
+	int num_xcc, num_xcp, num_xcc_per_xcp;
+	int r = 0;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	if (amdgpu_sriov_vf(adev)) {
+		enum amdgpu_gfx_partition mode;
+
+		mode = amdgpu_xcp_query_partition_mode(adev->xcp_mgr,
+						       AMDGPU_XCP_FL_NONE);
+		if (mode == AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE)
+			return -EINVAL;
+		if (adev->gfx.funcs &&
+		    adev->gfx.funcs->get_xccs_per_xcp) {
+			num_xcc_per_xcp = adev->gfx.funcs->get_xccs_per_xcp(adev);
+			adev->gfx.num_xcc_per_xcp = num_xcc_per_xcp;
+			num_xcp = num_xcc / num_xcc_per_xcp;
+		} else {
+			return -EINVAL;
+		}
+		r = amdgpu_xcp_init(adev->xcp_mgr, num_xcp, mode);
+
+	} else {
+		if (amdgpu_xcp_query_partition_mode(adev->xcp_mgr,
+						    AMDGPU_XCP_FL_NONE) ==
+		    AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE)
+			r = amdgpu_xcp_switch_partition_mode(adev->xcp_mgr,
+							     amdgpu_user_partt_mode);
+	}
+
+	if (r)
+		return r;
+
+	return gfx_v12_1_xcc_cp_resume(adev, adev->gfx.xcc_mask);
+}
+
+static int gfx_v12_1_gfxhub_enable(struct amdgpu_device *adev)
+{
+	int r, i;
+	bool value;
+
+	r = adev->gfxhub.funcs->gart_enable(adev);
+	if (r)
+		return r;
+
+	value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ?
+		false : true;
+
+	adev->gfxhub.funcs->set_fault_enable_default(adev, value);
+	/* TODO investigate why TLB flush is needed,
+	 * are we missing a flush somewhere else? */
+	for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
+		if (AMDGPU_IS_GFXHUB(i))
+			adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(i), 0);
+	}
+
+	return 0;
+}
+
+static int get_gb_addr_config(struct amdgpu_device *adev)
+{
+	u32 gb_addr_config;
+
+	gb_addr_config = RREG32_SOC15(GC, GET_INST(GC, 0), regGB_ADDR_CONFIG_READ);
+	if (gb_addr_config == 0)
+		return -EINVAL;
+
+	adev->gfx.config.gb_addr_config_fields.num_pkrs =
+		1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG_READ, NUM_PKRS);
+
+	adev->gfx.config.gb_addr_config = gb_addr_config;
+
+	adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
+			REG_GET_FIELD(adev->gfx.config.gb_addr_config,
+				      GB_ADDR_CONFIG_READ, NUM_PIPES);
+
+	adev->gfx.config.max_tile_pipes =
+		adev->gfx.config.gb_addr_config_fields.num_pipes;
+
+	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
+			REG_GET_FIELD(adev->gfx.config.gb_addr_config,
+				      GB_ADDR_CONFIG_READ, MAX_COMPRESSED_FRAGS);
+	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
+			REG_GET_FIELD(adev->gfx.config.gb_addr_config,
+				      GB_ADDR_CONFIG_READ, NUM_RB_PER_SE);
+	adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
+			REG_GET_FIELD(adev->gfx.config.gb_addr_config,
+				      GB_ADDR_CONFIG_READ, NUM_SHADER_ENGINES);
+	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
+			REG_GET_FIELD(adev->gfx.config.gb_addr_config,
+				      GB_ADDR_CONFIG_READ, PIPE_INTERLEAVE_SIZE));
+
+	return 0;
+}
+
+static void gfx_v12_1_xcc_disable_gpa_mode(struct amdgpu_device *adev,
+					   int xcc_id)
+{
+	uint32_t data;
+
+	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPC_PSP_DEBUG);
+	data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK;
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPC_PSP_DEBUG, data);
+
+	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPG_PSP_DEBUG);
+	data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK;
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPG_PSP_DEBUG, data);
+}
+
+static void gfx_v12_1_xcc_enable_atomics(struct amdgpu_device *adev,
+					 int xcc_id)
+{
+	uint32_t data;
+
+	/* Set the TCP UTCL0 register to enable atomics */
+	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regTCP_UTCL0_CNTL1);
+	data = REG_SET_FIELD(data, TCP_UTCL0_CNTL1, ATOMIC_REQUESTER_EN, 0x1);
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regTCP_UTCL0_CNTL1, data);
+}
+
+static void gfx_v12_1_init_golden_registers(struct amdgpu_device *adev)
+{
+	int i;
+
+	for (i = 0; i < NUM_XCC(adev->gfx.xcc_mask); i++)
+		gfx_v12_1_xcc_enable_atomics(adev, i);
+}
+
+static int gfx_v12_1_hw_init(struct amdgpu_ip_block *ip_block)
+{
+	int r, i, num_xcc;
+	struct amdgpu_device *adev = ip_block->adev;
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
+		/* rlc autoload firmware */
+		r = gfx_v12_1_rlc_backdoor_autoload_enable(adev);
+		if (r)
+			return r;
+	} else {
+		if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+			num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+
+			if (adev->gfx.imu.funcs) {
+				if (adev->gfx.imu.funcs->load_microcode)
+					adev->gfx.imu.funcs->load_microcode(adev);
+			}
+
+			for (i = 0; i < num_xcc; i++) {
+				/* disable gpa mode in backdoor loading */
+				gfx_v12_1_xcc_disable_gpa_mode(adev, i);
+			}
+		}
+	}
+
+	if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) ||
+	    (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
+		r = gfx_v12_1_wait_for_rlc_autoload_complete(adev);
+		if (r) {
+			dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r);
+			return r;
+		}
+	}
+
+	adev->gfx.is_poweron = true;
+
+	if (get_gb_addr_config(adev))
+		DRM_WARN("Invalid gb_addr_config !\n");
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)
+		gfx_v12_1_config_gfx_rs64(adev);
+
+	r = gfx_v12_1_gfxhub_enable(adev);
+	if (r)
+		return r;
+
+	gfx_v12_1_init_golden_registers(adev);
+
+	gfx_v12_1_constants_init(adev);
+
+	if (adev->nbio.funcs->gc_doorbell_init)
+		adev->nbio.funcs->gc_doorbell_init(adev);
+
+	r = gfx_v12_1_rlc_resume(adev);
+	if (r)
+		return r;
+
+	/*
+	 * init golden registers and rlc resume may override some registers,
+	 * reconfig them here
+	 */
+	gfx_v12_1_tcp_harvest(adev);
+
+	r = gfx_v12_1_cp_resume(adev);
+	if (r)
+		return r;
+
+	return r;
+}
+
+static void gfx_v12_1_xcc_fini(struct amdgpu_device *adev,
+			      int xcc_id)
+{
+	uint32_t tmp;
+
+	if (!adev->no_hw_access) {
+		if (amdgpu_gfx_disable_kcq(adev, xcc_id))
+			DRM_ERROR("KCQ disable failed\n");
+
+		amdgpu_mes_kiq_hw_fini(adev, xcc_id);
+	}
+
+	if (amdgpu_sriov_vf(adev)) {
+		/* Program KIQ position of RLC_CP_SCHEDULERS during destroy */
+		tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS);
+		tmp &= 0xffffff00;
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);
+	}
+	gfx_v12_1_xcc_cp_compute_enable(adev, false, xcc_id);
+	gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, false, xcc_id);
+}
+
+static int gfx_v12_1_hw_fini(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int i, num_xcc;
+
+	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
+	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < num_xcc; i++) {
+		gfx_v12_1_xcc_fini(adev, i);
+	}
+
+	adev->gfxhub.funcs->gart_disable(adev);
+
+	adev->gfx.is_poweron = false;
+
+	return 0;
+}
+
+static int gfx_v12_1_suspend(struct amdgpu_ip_block *ip_block)
+{
+	return gfx_v12_1_hw_fini(ip_block);
+}
+
+static int gfx_v12_1_resume(struct amdgpu_ip_block *ip_block)
+{
+	return gfx_v12_1_hw_init(ip_block);
+}
+
+static bool gfx_v12_1_is_idle(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int i, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < num_xcc; i++) {
+		if (REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, i),
+				regGRBM_STATUS), GRBM_STATUS, GUI_ACTIVE))
+			return false;
+	}
+	return true;
+}
+
+static int gfx_v12_1_wait_for_idle(struct amdgpu_ip_block *ip_block)
+{
+	unsigned i;
+	struct amdgpu_device *adev = ip_block->adev;
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		if (gfx_v12_1_is_idle(ip_block))
+			return 0;
+		udelay(1);
+	}
+	return -ETIMEDOUT;
+}
+
+static uint64_t gfx_v12_1_get_gpu_clock_counter(struct amdgpu_device *adev)
+{
+	uint64_t clock = 0;
+
+	if (adev->smuio.funcs &&
+	    adev->smuio.funcs->get_gpu_clock_counter)
+		clock = adev->smuio.funcs->get_gpu_clock_counter(adev);
+	else
+		dev_warn(adev->dev, "query gpu clock counter is not supported\n");
+
+	return clock;
+}
+
+static int gfx_v12_1_early_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+
+	adev->gfx.funcs = &gfx_v12_1_gfx_funcs;
+
+	adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
+					  AMDGPU_MAX_COMPUTE_RINGS);
+
+	gfx_v12_1_set_kiq_pm4_funcs(adev);
+	gfx_v12_1_set_ring_funcs(adev);
+	gfx_v12_1_set_irq_funcs(adev);
+	gfx_v12_1_set_rlc_funcs(adev);
+	gfx_v12_1_set_mqd_funcs(adev);
+	gfx_v12_1_set_imu_funcs(adev);
+
+	gfx_v12_1_init_rlcg_reg_access_ctrl(adev);
+
+	return gfx_v12_1_init_microcode(adev);
+}
+
+static int gfx_v12_1_late_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int r;
+
+	r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
+	if (r)
+		return r;
+
+	r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static bool gfx_v12_1_is_rlc_enabled(struct amdgpu_device *adev)
+{
+	uint32_t rlc_cntl;
+
+	/* if RLC is not enabled, do nothing */
+	rlc_cntl = RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CNTL);
+	return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false;
+}
+
+static void gfx_v12_1_xcc_set_safe_mode(struct amdgpu_device *adev,
+					int xcc_id)
+{
+	uint32_t data;
+	unsigned i;
+
+	data = RLC_SAFE_MODE__CMD_MASK;
+	data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SAFE_MODE, data);
+
+	/* wait for RLC_SAFE_MODE */
+	for (i = 0; i < adev->usec_timeout; i++) {
+		if (!REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, xcc_id),
+						regRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
+			break;
+		udelay(1);
+	}
+}
+
+static void gfx_v12_1_xcc_unset_safe_mode(struct amdgpu_device *adev,
+					  int xcc_id)
+{
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+		     regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK);
+}
+
+static void gfx_v12_1_update_perf_clk(struct amdgpu_device *adev,
+				      bool enable)
+{
+	int i, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < num_xcc; i++)
+		gfx_v12_1_xcc_update_perf_clk(adev, enable, i);
+}
+
+static void gfx_v12_1_update_spm_vmid(struct amdgpu_device *adev,
+				      int xcc_id,
+				      struct amdgpu_ring *ring,
+				      unsigned vmid)
+{
+	u32 reg, data;
+
+	reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL);
+	if (amdgpu_sriov_is_pp_one_vf(adev))
+		data = RREG32_NO_KIQ(reg);
+	else
+		data = RREG32(reg);
+
+	data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
+	data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
+
+	if (amdgpu_sriov_is_pp_one_vf(adev))
+		WREG32_SOC15_NO_KIQ(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL, data);
+	else
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL, data);
+
+	if (ring
+	    && amdgpu_sriov_is_pp_one_vf(adev)
+	    && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX)
+		|| (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) {
+		uint32_t reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL);
+		amdgpu_ring_emit_wreg(ring, reg, data);
+	}
+}
+
+static const struct amdgpu_rlc_funcs gfx_v12_1_rlc_funcs = {
+	.is_rlc_enabled = gfx_v12_1_is_rlc_enabled,
+	.set_safe_mode = gfx_v12_1_xcc_set_safe_mode,
+	.unset_safe_mode = gfx_v12_1_xcc_unset_safe_mode,
+	.init = gfx_v12_1_rlc_init,
+	.get_csb_size = gfx_v12_1_get_csb_size,
+	.get_csb_buffer = gfx_v12_1_get_csb_buffer,
+	.resume = gfx_v12_1_rlc_resume,
+	.stop = gfx_v12_1_rlc_stop,
+	.reset = gfx_v12_1_rlc_reset,
+	.start = gfx_v12_1_rlc_start,
+	.update_spm_vmid = gfx_v12_1_update_spm_vmid,
+};
+
+#if 0
+static void gfx_v12_cntl_power_gating(struct amdgpu_device *adev, bool enable)
+{
+	/* TODO */
+}
+
+static void gfx_v12_cntl_pg(struct amdgpu_device *adev, bool enable)
+{
+	/* TODO */
+}
+#endif
+
+static int gfx_v12_1_set_powergating_state(struct amdgpu_ip_block *ip_block,
+					   enum amd_powergating_state state)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	bool enable = (state == AMD_PG_STATE_GATE);
+
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
+	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
+	case IP_VERSION(12, 1, 0):
+		amdgpu_gfx_off_ctrl(adev, enable);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static void gfx_v12_1_xcc_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
+							   bool enable, int xcc_id)
+{
+	uint32_t def, data;
+
+	if (!(adev->cg_flags &
+	      (AMD_CG_SUPPORT_GFX_CGCG |
+	      AMD_CG_SUPPORT_GFX_CGLS |
+	      AMD_CG_SUPPORT_GFX_3D_CGCG |
+	      AMD_CG_SUPPORT_GFX_3D_CGLS)))
+		return;
+
+	if (enable) {
+		def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
+					  regRLC_CGTT_MGCG_OVERRIDE);
+
+		/* unset CGCG override */
+		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
+			data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
+		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
+			data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
+		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG ||
+		    adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
+			data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
+
+		/* update CGCG override bits */
+		if (def != data)
+			WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+				     regRLC_CGTT_MGCG_OVERRIDE, data);
+
+		/* enable cgcg FSM(0x0000363F) */
+		def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL);
+
+		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
+			data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK;
+			data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
+				 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
+		}
+
+		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
+			data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK;
+			data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
+				 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
+		}
+
+		if (def != data)
+			WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+				     regRLC_CGCG_CGLS_CTRL, data);
+
+		/* set IDLE_POLL_COUNT(0x00900100) */
+		def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL);
+
+		data &= ~CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK;
+		data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
+		data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
+			(0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
+
+		if (def != data)
+			WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL, data);
+
+		data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL);
+		data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
+		data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
+		data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
+		data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL, data);
+	} else {
+		/* Program RLC_CGCG_CGLS_CTRL */
+		def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL);
+
+		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
+			data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
+
+		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
+			data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
+
+		if (def != data)
+			WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, data);
+	}
+}
+
+static void gfx_v12_1_xcc_update_medium_grain_clock_gating(struct amdgpu_device *adev,
+							   bool enable, int xcc_id)
+{
+	uint32_t data, def;
+	if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))
+		return;
+
+	/* It is disabled by HW by default */
+	if (enable) {
+		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
+			/* 1 - RLC_CGTT_MGCG_OVERRIDE */
+			def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
+
+			data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
+				  RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
+				  RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
+
+			if (def != data)
+				WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
+		}
+	} else {
+		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
+			def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
+
+			data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
+				 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
+				 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
+
+			if (def != data)
+				WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
+		}
+	}
+}
+
+static void gfx_v12_1_xcc_update_repeater_fgcg(struct amdgpu_device *adev,
+					       bool enable, int xcc_id)
+{
+	uint32_t def, data;
+
+	if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
+		return;
+
+	def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
+
+	if (enable)
+		data &= ~(RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK |
+				  RLC_CGTT_MGCG_OVERRIDE__RLC_REPEATER_FGCG_OVERRIDE_MASK);
+	else
+		data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK |
+				RLC_CGTT_MGCG_OVERRIDE__RLC_REPEATER_FGCG_OVERRIDE_MASK;
+
+	if (def != data)
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
+}
+
+static void gfx_v12_1_xcc_update_sram_fgcg(struct amdgpu_device *adev,
+					   bool enable, int xcc_id)
+{
+	uint32_t def, data;
+
+	if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
+		return;
+
+	def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
+
+	if (enable)
+		data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
+	else
+		data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
+
+	if (def != data)
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
+}
+
+static void gfx_v12_1_xcc_update_perf_clk(struct amdgpu_device *adev,
+					  bool enable, int xcc_id)
+{
+	uint32_t def, data;
+
+	if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK))
+		return;
+
+	def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
+
+	if (enable)
+		data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
+	else
+		data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
+
+	if (def != data)
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
+}
+
+static int gfx_v12_1_xcc_update_gfx_clock_gating(struct amdgpu_device *adev,
+					     bool enable, int xcc_id)
+{
+	amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);
+
+	gfx_v12_1_xcc_update_coarse_grain_clock_gating(adev, enable, xcc_id);
+
+	gfx_v12_1_xcc_update_medium_grain_clock_gating(adev, enable, xcc_id);
+
+	gfx_v12_1_xcc_update_repeater_fgcg(adev, enable, xcc_id);
+
+	gfx_v12_1_xcc_update_sram_fgcg(adev, enable, xcc_id);
+
+	gfx_v12_1_xcc_update_perf_clk(adev, enable, xcc_id);
+
+	if (adev->cg_flags &
+	    (AMD_CG_SUPPORT_GFX_MGCG |
+	     AMD_CG_SUPPORT_GFX_CGLS |
+	     AMD_CG_SUPPORT_GFX_CGCG |
+	     AMD_CG_SUPPORT_GFX_3D_CGCG |
+	     AMD_CG_SUPPORT_GFX_3D_CGLS))
+		gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, enable, xcc_id);
+
+	amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);
+
+	return 0;
+}
+
+static int gfx_v12_1_set_clockgating_state(struct amdgpu_ip_block *ip_block,
+					   enum amd_clockgating_state state)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int i, num_xcc;
+
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	switch (adev->ip_versions[GC_HWIP][0]) {
+	case IP_VERSION(12, 1, 0):
+		for (i = 0; i < num_xcc; i++)
+			gfx_v12_1_xcc_update_gfx_clock_gating(adev,
+				  state == AMD_CG_STATE_GATE, i);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static void gfx_v12_1_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int data;
+
+	/* AMD_CG_SUPPORT_GFX_MGCG */
+	data = RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CGTT_MGCG_OVERRIDE);
+	if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
+		*flags |= AMD_CG_SUPPORT_GFX_MGCG;
+
+	/* AMD_CG_SUPPORT_REPEATER_FGCG */
+	if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK))
+		*flags |= AMD_CG_SUPPORT_REPEATER_FGCG;
+
+	/* AMD_CG_SUPPORT_GFX_FGCG */
+	if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK))
+		*flags |= AMD_CG_SUPPORT_GFX_FGCG;
+
+	/* AMD_CG_SUPPORT_GFX_PERF_CLK */
+	if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK))
+		*flags |= AMD_CG_SUPPORT_GFX_PERF_CLK;
+
+	/* AMD_CG_SUPPORT_GFX_CGCG */
+	data = RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CGCG_CGLS_CTRL);
+	if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
+		*flags |= AMD_CG_SUPPORT_GFX_CGCG;
+
+	/* AMD_CG_SUPPORT_GFX_CGLS */
+	if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
+		*flags |= AMD_CG_SUPPORT_GFX_CGLS;
+}
+
+static u64 gfx_v12_1_ring_get_rptr_compute(struct amdgpu_ring *ring)
+{
+	/* gfx12 hardware is 32bit rptr */
+	return *(uint32_t *)ring->rptr_cpu_addr;
+}
+
+static u64 gfx_v12_1_ring_get_wptr_compute(struct amdgpu_ring *ring)
+{
+	u64 wptr;
+
+	/* XXX check if swapping is necessary on BE */
+	if (ring->use_doorbell)
+		wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
+	else
+		BUG();
+	return wptr;
+}
+
+static void gfx_v12_1_ring_set_wptr_compute(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	/* XXX check if swapping is necessary on BE */
+	if (ring->use_doorbell) {
+		atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
+			     ring->wptr);
+		WDOORBELL64(ring->doorbell_index, ring->wptr);
+	} else {
+		BUG(); /* only DOORBELL method supported on gfx12 now */
+	}
+}
+
+static void gfx_v12_1_ring_emit_ib_compute(struct amdgpu_ring *ring,
+					   struct amdgpu_job *job,
+					   struct amdgpu_ib *ib,
+					   uint32_t flags)
+{
+	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
+	u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
+
+	/* Currently, there is a high possibility to get wave ID mismatch
+	 * between ME and GDS, leading to a hw deadlock, because ME generates
+	 * different wave IDs than the GDS expects. This situation happens
+	 * randomly when at least 5 compute pipes use GDS ordered append.
+	 * The wave IDs generated by ME are also wrong after suspend/resume.
+	 * Those are probably bugs somewhere else in the kernel driver.
+	 *
+	 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
+	 * GDS to 0 for this ring (me/pipe).
+	 */
+	if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
+		amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
+		amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID);
+	}
+
+	amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
+	BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
+	amdgpu_ring_write(ring,
+#ifdef __BIG_ENDIAN
+				(2 << 0) |
+#endif
+				lower_32_bits(ib->gpu_addr));
+	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
+	amdgpu_ring_write(ring, control);
+}
+
+static void gfx_v12_1_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
+				     u64 seq, unsigned flags)
+{
+	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
+	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
+
+	/* RELEASE_MEM - flush caches, send int */
+	amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
+	amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ(1) |
+				 PACKET3_RELEASE_MEM_GCR_GLV_WB |
+				 PACKET3_RELEASE_MEM_GCR_GL2_WB |
+				 PACKET3_RELEASE_MEM_GCR_GL2_SCOPE(2) |
+				 PACKET3_RELEASE_MEM_TEMPORAL(3) |
+				 PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
+				 PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
+	amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
+				 PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
+
+	/*
+	 * the address should be Qword aligned if 64bit write, Dword
+	 * aligned if only send 32bit data low (discard data high)
+	 */
+	if (write64bit)
+		BUG_ON(addr & 0x7);
+	else
+		BUG_ON(addr & 0x3);
+	amdgpu_ring_write(ring, lower_32_bits(addr));
+	amdgpu_ring_write(ring, upper_32_bits(addr));
+	amdgpu_ring_write(ring, lower_32_bits(seq));
+	amdgpu_ring_write(ring, upper_32_bits(seq));
+	amdgpu_ring_write(ring, 0);
+}
+
+static void gfx_v12_1_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
+{
+	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
+	uint32_t seq = ring->fence_drv.sync_seq;
+	uint64_t addr = ring->fence_drv.gpu_addr;
+
+	gfx_v12_1_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr),
+			       upper_32_bits(addr), seq, 0xffffffff, 4);
+}
+
+static void gfx_v12_1_ring_invalidate_tlbs(struct amdgpu_ring *ring,
+				   uint16_t pasid, uint32_t flush_type,
+				   bool all_hub, uint8_t dst_sel)
+{
+	amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
+	amdgpu_ring_write(ring,
+			  PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) |
+			  PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
+			  PACKET3_INVALIDATE_TLBS_PASID(pasid) |
+			  PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
+}
+
+static void gfx_v12_1_ring_emit_vm_flush(struct amdgpu_ring *ring,
+					 unsigned vmid, uint64_t pd_addr)
+{
+	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
+
+	/* compute doesn't have PFP */
+	if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
+		/* sync PFP to ME, otherwise we might get invalid PFP reads */
+		amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
+		amdgpu_ring_write(ring, 0x0);
+	}
+}
+
+static void gfx_v12_1_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
+					  u64 seq, unsigned int flags)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	/* we only allocate 32bit for each seq wb address */
+	BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
+
+	/* write fence seq to the "addr" */
+	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+				 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
+	amdgpu_ring_write(ring, lower_32_bits(addr));
+	amdgpu_ring_write(ring, upper_32_bits(addr));
+	amdgpu_ring_write(ring, lower_32_bits(seq));
+
+	if (flags & AMDGPU_FENCE_FLAG_INT) {
+		/* set register to trigger INT */
+		amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+		amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+					 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
+		amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regCPC_INT_STATUS));
+		amdgpu_ring_write(ring, 0);
+		amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
+	}
+}
+
+static void gfx_v12_1_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
+				     uint32_t reg_val_offs)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	reg = gfx_v12_1_normalize_xcc_reg_offset(reg);
+
+	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
+	amdgpu_ring_write(ring, 0 |	/* src: register*/
+				(5 << 8) |	/* dst: memory */
+				(1 << 20));	/* write confirm */
+	amdgpu_ring_write(ring, reg);
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
+				reg_val_offs * 4));
+	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
+				reg_val_offs * 4));
+}
+
+static void gfx_v12_1_ring_emit_wreg(struct amdgpu_ring *ring,
+				     uint32_t reg,
+				     uint32_t val)
+{
+	uint32_t cmd = 0;
+
+	reg = gfx_v12_1_normalize_xcc_reg_offset(reg);
+
+	switch (ring->funcs->type) {
+	case AMDGPU_RING_TYPE_KIQ:
+		cmd = (1 << 16); /* no inc addr */
+		break;
+	default:
+		cmd = WR_CONFIRM;
+		break;
+	}
+	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+	amdgpu_ring_write(ring, cmd);
+	amdgpu_ring_write(ring, reg);
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, val);
+}
+
+static void gfx_v12_1_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
+					uint32_t val, uint32_t mask)
+{
+	gfx_v12_1_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
+}
+
+static void gfx_v12_1_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
+						   uint32_t reg0, uint32_t reg1,
+						   uint32_t ref, uint32_t mask)
+{
+	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
+
+	gfx_v12_1_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
+			       ref, mask, 0x20);
+}
+
+static void gfx_v12_1_xcc_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
+							int me, int pipe,
+							enum amdgpu_interrupt_state state,
+							int xcc_id)
+{
+	u32 mec_int_cntl, mec_int_cntl_reg;
+
+	/*
+	 * amdgpu controls only the first MEC. That's why this function only
+	 * handles the setting of interrupts for this specific MEC. All other
+	 * pipes' interrupts are set by amdkfd.
+	 */
+
+	if (me == 1) {
+		switch (pipe) {
+		case 0:
+			mec_int_cntl_reg = SOC15_REG_OFFSET(
+					GC, GET_INST(GC, xcc_id),
+					regCP_ME1_PIPE0_INT_CNTL);
+			break;
+		case 1:
+			mec_int_cntl_reg = SOC15_REG_OFFSET(
+					GC, GET_INST(GC, xcc_id),
+					regCP_ME1_PIPE1_INT_CNTL);
+			break;
+		case 2:
+			mec_int_cntl_reg = SOC15_REG_OFFSET(
+					GC, GET_INST(GC, xcc_id),
+					regCP_ME1_PIPE2_INT_CNTL);
+			break;
+		case 3:
+			mec_int_cntl_reg = SOC15_REG_OFFSET(
+					GC, GET_INST(GC, xcc_id),
+					regCP_ME1_PIPE3_INT_CNTL);
+			break;
+		default:
+			DRM_DEBUG("invalid pipe %d\n", pipe);
+			return;
+		}
+	} else {
+		DRM_DEBUG("invalid me %d\n", me);
+		return;
+	}
+
+	switch (state) {
+	case AMDGPU_IRQ_STATE_DISABLE:
+		mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, xcc_id);
+		mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
+					     TIME_STAMP_INT_ENABLE, 0);
+		mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
+					     GENERIC0_INT_ENABLE, 0);
+		WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, xcc_id);
+		break;
+	case AMDGPU_IRQ_STATE_ENABLE:
+		mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, xcc_id);
+		mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
+					     TIME_STAMP_INT_ENABLE, 1);
+		mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
+					     GENERIC0_INT_ENABLE, 1);
+		WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, xcc_id);
+		break;
+	default:
+		break;
+	}
+}
+
+static int gfx_v12_1_set_eop_interrupt_state(struct amdgpu_device *adev,
+					    struct amdgpu_irq_src *src,
+					    unsigned type,
+					    enum amdgpu_interrupt_state state)
+{
+	int i, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < num_xcc; i++) {
+		switch (type) {
+		case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
+			gfx_v12_1_xcc_set_compute_eop_interrupt_state(
+					adev, 1, 0, state, i);
+			break;
+		case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
+			gfx_v12_1_xcc_set_compute_eop_interrupt_state(
+					adev, 1, 1, state, i);
+			break;
+		case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
+			gfx_v12_1_xcc_set_compute_eop_interrupt_state(
+					adev, 1, 2, state, i);
+			break;
+		case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
+			gfx_v12_1_xcc_set_compute_eop_interrupt_state(
+					adev, 1, 3, state, i);
+			break;
+		default:
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static int gfx_v12_1_eop_irq(struct amdgpu_device *adev,
+			     struct amdgpu_irq_src *source,
+			     struct amdgpu_iv_entry *entry)
+{
+	int i, xcc_id;
+	u8 me_id, pipe_id, queue_id;
+	struct amdgpu_ring *ring;
+	uint32_t mes_queue_id = entry->src_data[0];
+
+	DRM_DEBUG("IH: CP EOP\n");
+
+	if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
+		struct amdgpu_mes_queue *queue;
+
+		mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
+
+		spin_lock(&adev->mes.queue_id_lock);
+		queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
+		if (queue) {
+			DRM_DEBUG("process mes queue id = %d\n", mes_queue_id);
+			amdgpu_fence_process(queue->ring);
+		}
+		spin_unlock(&adev->mes.queue_id_lock);
+	} else {
+		me_id = (entry->ring_id & 0x0c) >> 2;
+		pipe_id = (entry->ring_id & 0x03) >> 0;
+		queue_id = (entry->ring_id & 0x70) >> 4;
+		xcc_id = gfx_v12_1_ih_to_xcc_inst(adev, entry->node_id);
+
+		if (xcc_id == -EINVAL)
+			return -EINVAL;
+
+		switch (me_id) {
+		case 0:
+			if (pipe_id == 0)
+				amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
+			else
+				amdgpu_fence_process(&adev->gfx.gfx_ring[1]);
+			break;
+		case 1:
+		case 2:
+			for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+				ring = &adev->gfx.compute_ring
+						[i +
+						 xcc_id * adev->gfx.num_compute_rings];
+				/* Per-queue interrupt is supported for MEC starting from VI.
+				 * The interrupt can only be enabled/disabled per pipe instead
+				 * of per queue.
+				 */
+				if ((ring->me == me_id) &&
+				    (ring->pipe == pipe_id) &&
+				    (ring->queue == queue_id))
+					amdgpu_fence_process(ring);
+			}
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static int gfx_v12_1_set_priv_reg_fault_state(struct amdgpu_device *adev,
+					      struct amdgpu_irq_src *source,
+					      unsigned type,
+					      enum amdgpu_interrupt_state state)
+{
+	int i, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	switch (state) {
+	case AMDGPU_IRQ_STATE_DISABLE:
+	case AMDGPU_IRQ_STATE_ENABLE:
+		for (i = 0; i < num_xcc; i++)
+			WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0,
+					      PRIV_REG_INT_ENABLE,
+					      state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int gfx_v12_1_set_priv_inst_fault_state(struct amdgpu_device *adev,
+					       struct amdgpu_irq_src *source,
+					       unsigned type,
+					       enum amdgpu_interrupt_state state)
+{
+	int i, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	switch (state) {
+	case AMDGPU_IRQ_STATE_DISABLE:
+	case AMDGPU_IRQ_STATE_ENABLE:
+		for (i = 0; i < num_xcc; i++)
+			WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0,
+				       PRIV_INSTR_INT_ENABLE,
+				       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static void gfx_v12_1_handle_priv_fault(struct amdgpu_device *adev,
+					struct amdgpu_iv_entry *entry)
+{
+	u8 me_id, pipe_id, queue_id;
+	struct amdgpu_ring *ring;
+	int i, xcc_id;
+
+	me_id = (entry->ring_id & 0x0c) >> 2;
+	pipe_id = (entry->ring_id & 0x03) >> 0;
+	queue_id = (entry->ring_id & 0x70) >> 4;
+	xcc_id = gfx_v12_1_ih_to_xcc_inst(adev, entry->node_id);
+
+	if (xcc_id == -EINVAL)
+		return;
+
+	switch (me_id) {
+	case 0:
+		for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
+			ring = &adev->gfx.gfx_ring[i];
+			/* we only enabled 1 gfx queue per pipe for now */
+			if (ring->me == me_id && ring->pipe == pipe_id)
+				drm_sched_fault(&ring->sched);
+		}
+		break;
+	case 1:
+	case 2:
+		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+			ring = &adev->gfx.compute_ring
+					[i +
+					 xcc_id * adev->gfx.num_compute_rings];
+			if (ring->me == me_id && ring->pipe == pipe_id &&
+			    ring->queue == queue_id)
+				drm_sched_fault(&ring->sched);
+		}
+		break;
+	default:
+		BUG();
+		break;
+	}
+}
+
+static int gfx_v12_1_priv_reg_irq(struct amdgpu_device *adev,
+				  struct amdgpu_irq_src *source,
+				  struct amdgpu_iv_entry *entry)
+{
+	DRM_ERROR("Illegal register access in command stream\n");
+	gfx_v12_1_handle_priv_fault(adev, entry);
+	return 0;
+}
+
+static int gfx_v12_1_priv_inst_irq(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry)
+{
+	DRM_ERROR("Illegal instruction in command stream\n");
+	gfx_v12_1_handle_priv_fault(adev, entry);
+	return 0;
+}
+
+static void gfx_v12_1_emit_mem_sync(struct amdgpu_ring *ring)
+{
+	const unsigned int gcr_cntl =
+			PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) |
+			PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) |
+			PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) |
+			PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) |
+			PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1) |
+			PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_SCOPE(2);
+
+	/* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */
+	amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6));
+	amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */
+	amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
+	amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
+	amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
+	amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
+	amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
+	amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */
+}
+
+static const struct amd_ip_funcs gfx_v12_1_ip_funcs = {
+	.name = "gfx_v12_1",
+	.early_init = gfx_v12_1_early_init,
+	.late_init = gfx_v12_1_late_init,
+	.sw_init = gfx_v12_1_sw_init,
+	.sw_fini = gfx_v12_1_sw_fini,
+	.hw_init = gfx_v12_1_hw_init,
+	.hw_fini = gfx_v12_1_hw_fini,
+	.suspend = gfx_v12_1_suspend,
+	.resume = gfx_v12_1_resume,
+	.is_idle = gfx_v12_1_is_idle,
+	.wait_for_idle = gfx_v12_1_wait_for_idle,
+	.set_clockgating_state = gfx_v12_1_set_clockgating_state,
+	.set_powergating_state = gfx_v12_1_set_powergating_state,
+	.get_clockgating_state = gfx_v12_1_get_clockgating_state,
+};
+
+static const struct amdgpu_ring_funcs gfx_v12_1_ring_funcs_compute = {
+	.type = AMDGPU_RING_TYPE_COMPUTE,
+	.align_mask = 0xff,
+	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
+	.support_64bit_ptrs = true,
+	.get_rptr = gfx_v12_1_ring_get_rptr_compute,
+	.get_wptr = gfx_v12_1_ring_get_wptr_compute,
+	.set_wptr = gfx_v12_1_ring_set_wptr_compute,
+	.emit_frame_size =
+		7 + /* gfx_v12_1_ring_emit_pipeline_sync */
+		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
+		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
+		2 + /* gfx_v12_1_ring_emit_vm_flush */
+		8 + 8 + 8 + /* gfx_v12_1_ring_emit_fence x3 for user fence, vm fence */
+		8, /* gfx_v12_1_emit_mem_sync */
+	.emit_ib_size =	7, /* gfx_v12_1_ring_emit_ib_compute */
+	.emit_ib = gfx_v12_1_ring_emit_ib_compute,
+	.emit_fence = gfx_v12_1_ring_emit_fence,
+	.emit_pipeline_sync = gfx_v12_1_ring_emit_pipeline_sync,
+	.emit_vm_flush = gfx_v12_1_ring_emit_vm_flush,
+	.test_ring = gfx_v12_1_ring_test_ring,
+	.test_ib = gfx_v12_1_ring_test_ib,
+	.insert_nop = amdgpu_ring_insert_nop,
+	.pad_ib = amdgpu_ring_generic_pad_ib,
+	.emit_wreg = gfx_v12_1_ring_emit_wreg,
+	.emit_reg_wait = gfx_v12_1_ring_emit_reg_wait,
+	.emit_reg_write_reg_wait = gfx_v12_1_ring_emit_reg_write_reg_wait,
+	.emit_mem_sync = gfx_v12_1_emit_mem_sync,
+};
+
+static const struct amdgpu_ring_funcs gfx_v12_1_ring_funcs_kiq = {
+	.type = AMDGPU_RING_TYPE_KIQ,
+	.align_mask = 0xff,
+	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
+	.support_64bit_ptrs = true,
+	.get_rptr = gfx_v12_1_ring_get_rptr_compute,
+	.get_wptr = gfx_v12_1_ring_get_wptr_compute,
+	.set_wptr = gfx_v12_1_ring_set_wptr_compute,
+	.emit_frame_size =
+		7 + /* gfx_v12_1_ring_emit_pipeline_sync */
+		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
+		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
+		2 + /* gfx_v12_1_ring_emit_vm_flush */
+		8 + 8 + 8, /* gfx_v12_1_ring_emit_fence_kiq x3 for user fence, vm fence */
+	.emit_ib_size =	7, /* gfx_v12_1_ring_emit_ib_compute */
+	.emit_ib = gfx_v12_1_ring_emit_ib_compute,
+	.emit_fence = gfx_v12_1_ring_emit_fence_kiq,
+	.test_ring = gfx_v12_1_ring_test_ring,
+	.test_ib = gfx_v12_1_ring_test_ib,
+	.insert_nop = amdgpu_ring_insert_nop,
+	.pad_ib = amdgpu_ring_generic_pad_ib,
+	.emit_rreg = gfx_v12_1_ring_emit_rreg,
+	.emit_wreg = gfx_v12_1_ring_emit_wreg,
+	.emit_reg_wait = gfx_v12_1_ring_emit_reg_wait,
+	.emit_reg_write_reg_wait = gfx_v12_1_ring_emit_reg_write_reg_wait,
+};
+
+static void gfx_v12_1_set_ring_funcs(struct amdgpu_device *adev)
+{
+	int i, j, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < num_xcc; i++) {
+		adev->gfx.kiq[i].ring.funcs = &gfx_v12_1_ring_funcs_kiq;
+
+		for (j = 0; j < adev->gfx.num_compute_rings; j++)
+			adev->gfx.compute_ring[j + i * adev->gfx.num_compute_rings].funcs =
+						&gfx_v12_1_ring_funcs_compute;
+	}
+}
+
+static const struct amdgpu_irq_src_funcs gfx_v12_1_eop_irq_funcs = {
+	.set = gfx_v12_1_set_eop_interrupt_state,
+	.process = gfx_v12_1_eop_irq,
+};
+
+static const struct amdgpu_irq_src_funcs gfx_v12_1_priv_reg_irq_funcs = {
+	.set = gfx_v12_1_set_priv_reg_fault_state,
+	.process = gfx_v12_1_priv_reg_irq,
+};
+
+static const struct amdgpu_irq_src_funcs gfx_v12_1_priv_inst_irq_funcs = {
+	.set = gfx_v12_1_set_priv_inst_fault_state,
+	.process = gfx_v12_1_priv_inst_irq,
+};
+
+static void gfx_v12_1_set_irq_funcs(struct amdgpu_device *adev)
+{
+	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
+	adev->gfx.eop_irq.funcs = &gfx_v12_1_eop_irq_funcs;
+
+	adev->gfx.priv_reg_irq.num_types = 1;
+	adev->gfx.priv_reg_irq.funcs = &gfx_v12_1_priv_reg_irq_funcs;
+
+	adev->gfx.priv_inst_irq.num_types = 1;
+	adev->gfx.priv_inst_irq.funcs = &gfx_v12_1_priv_inst_irq_funcs;
+}
+
+static void gfx_v12_1_set_imu_funcs(struct amdgpu_device *adev)
+{
+	if (adev->flags & AMD_IS_APU)
+		adev->gfx.imu.mode = MISSION_MODE;
+	else
+		adev->gfx.imu.mode = DEBUG_MODE;
+
+	adev->gfx.imu.funcs = &gfx_v12_1_imu_funcs;
+}
+
+static void gfx_v12_1_set_rlc_funcs(struct amdgpu_device *adev)
+{
+	adev->gfx.rlc.funcs = &gfx_v12_1_rlc_funcs;
+}
+
+static void gfx_v12_1_set_mqd_funcs(struct amdgpu_device *adev)
+{
+	/* set compute eng mqd */
+	adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size =
+		sizeof(struct v12_1_compute_mqd);
+	adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd =
+		gfx_v12_1_compute_mqd_init;
+}
+
+static void gfx_v12_1_set_user_cu_inactive_bitmap_per_sh(struct amdgpu_device *adev,
+							  u32 bitmap, int xcc_id)
+{
+	u32 data;
+
+	if (!bitmap)
+		return;
+
+	data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
+	data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regGC_USER_SHADER_ARRAY_CONFIG, data);
+}
+
+static u32 gfx_v12_1_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev,
+						 int xcc_id)
+{
+	u32 data, mask;
+
+	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCC_GC_SHADER_ARRAY_CONFIG);
+	data |= RREG32_SOC15(GC, GET_INST(GC, xcc_id), regGC_USER_SHADER_ARRAY_CONFIG);
+
+	data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
+	data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
+
+	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
+
+	return (~data) & mask;
+}
+
+static int gfx_v12_1_get_cu_info(struct amdgpu_device *adev,
+				 struct amdgpu_cu_info *cu_info)
+{
+	int i, j, k, counter, xcc_id, active_cu_number = 0;
+	u32 mask, bitmap;
+	unsigned int disable_masks[2 * 2];
+
+	if (!adev || !cu_info)
+		return -EINVAL;
+
+	if (adev->gfx.config.max_shader_engines > 2 ||
+	    adev->gfx.config.max_sh_per_se > 2) {
+		dev_err(adev->dev,
+			"Max SE (%d) and Max SA per SE (%d) is greater than expected\n",
+			adev->gfx.config.max_shader_engines,
+			adev->gfx.config.max_sh_per_se);
+		return -EINVAL;
+	}
+
+	amdgpu_gfx_parse_disable_cu(disable_masks,
+				    adev->gfx.config.max_shader_engines,
+				    adev->gfx.config.max_sh_per_se);
+
+	mutex_lock(&adev->grbm_idx_mutex);
+	for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++) {
+		for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
+			for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
+				bitmap = i * adev->gfx.config.max_sh_per_se + j;
+				if (!((gfx_v12_1_get_sa_active_bitmap(adev, xcc_id) >> bitmap) & 1))
+					continue;
+				mask = 1;
+				counter = 0;
+				gfx_v12_1_xcc_select_se_sh(adev, i, j, 0xffffffff, xcc_id);
+				gfx_v12_1_set_user_cu_inactive_bitmap_per_sh(
+					adev,
+					disable_masks[i * adev->gfx.config.max_sh_per_se + j],
+					xcc_id);
+				bitmap = gfx_v12_1_get_cu_active_bitmap_per_sh(adev, xcc_id);
+
+				cu_info->bitmap[xcc_id][i][j] = bitmap;
+
+				for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
+					if (bitmap & mask)
+						counter++;
+
+					mask <<= 1;
+				}
+				active_cu_number += counter;
+			}
+		}
+		gfx_v12_1_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, xcc_id);
+	}
+	mutex_unlock(&adev->grbm_idx_mutex);
+
+	cu_info->number = active_cu_number;
+	cu_info->simd_per_cu = NUM_SIMD_PER_CU_GFX12_1;
+	cu_info->lds_size = 320;
+
+	return 0;
+}
+
+const struct amdgpu_ip_block_version gfx_v12_1_ip_block = {
+	.type = AMD_IP_BLOCK_TYPE_GFX,
+	.major = 12,
+	.minor = 1,
+	.rev = 0,
+	.funcs = &gfx_v12_1_ip_funcs,
+};
+
+static int gfx_v12_1_xcp_resume(void *handle, uint32_t inst_mask)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	uint32_t tmp_mask;
+	int i, r;
+
+	/* TODO : Initialize golden regs */
+	/* gfx_v12_1_init_golden_registers(adev); */
+
+	tmp_mask = inst_mask;
+	for_each_inst(i, tmp_mask)
+		gfx_v12_1_xcc_constants_init(adev, i);
+
+	if (!amdgpu_sriov_vf(adev)) {
+		tmp_mask = inst_mask;
+		for_each_inst(i, tmp_mask) {
+			r = gfx_v12_1_xcc_rlc_resume(adev, i);
+			if (r)
+				return r;
+		}
+	}
+
+	r = gfx_v12_1_xcc_cp_resume(adev, inst_mask);
+
+	return r;
+}
+
+static int gfx_v12_1_xcp_suspend(void *handle, uint32_t inst_mask)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int i;
+
+	for_each_inst(i, inst_mask)
+		gfx_v12_1_xcc_fini(adev, i);
+
+	return 0;
+}
+
+struct amdgpu_xcp_ip_funcs gfx_v12_1_xcp_funcs = {
+	.suspend = &gfx_v12_1_xcp_suspend,
+	.resume = &gfx_v12_1_xcp_resume
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.h b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.h
new file mode 100644
index 000000000000..684d93a1e2f8
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __GFX_V12_1_H__
+#define __GFX_V12_1_H__
+
+extern const struct amdgpu_ip_block_version gfx_v12_1_ip_block;
+
+extern struct amdgpu_xcp_ip_funcs gfx_v12_1_xcp_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h
new file mode 100644
index 000000000000..9a5c82c8db53
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h
@@ -0,0 +1,475 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef GFX_V12_1_PKT_H
+#define GFX_V12_1_PKT_H
+
+/**
+ * PM4 definitions
+ */
+#define	PACKET_TYPE0	0
+#define	PACKET_TYPE1	1
+#define	PACKET_TYPE2	2
+#define	PACKET_TYPE3	3
+
+#define CP_PACKET_GET_TYPE(h) (((h) >> 30) & 3)
+#define CP_PACKET_GET_COUNT(h) (((h) >> 16) & 0x3FFF)
+#define CP_PACKET0_GET_REG(h) ((h) & 0xFFFF)
+#define CP_PACKET3_GET_OPCODE(h) (((h) >> 8) & 0xFF)
+#define PACKET0(reg, n)	((PACKET_TYPE0 << 30) |				\
+			 ((reg) & 0xFFFF) |			\
+			 ((n) & 0x3FFF) << 16)
+#define CP_PACKET2			0x80000000
+#define		PACKET2_PAD_SHIFT		0
+#define		PACKET2_PAD_MASK		(0x3fffffff << 0)
+
+#define PACKET2(v)	(CP_PACKET2 | REG_SET(PACKET2_PAD, (v)))
+
+#define PACKET3(op, n)	((PACKET_TYPE3 << 30) |				\
+			 (((op) & 0xFF) << 8) |				\
+			 ((n) & 0x3FFF) << 16)
+
+#define PACKET3_COMPUTE(op, n) (PACKET3(op, n) | 1 << 1)
+
+/* Packet 3 types */
+#define	PACKET3_NOP					0x10
+#define	PACKET3_SET_BASE				0x11
+#define		PACKET3_BASE_INDEX(x)                  ((x) << 0)
+#define			CE_PARTITION_BASE		3
+#define	PACKET3_CLEAR_STATE				0x12
+#define	PACKET3_INDEX_BUFFER_SIZE			0x13
+#define	PACKET3_DISPATCH_DIRECT				0x15
+#define	PACKET3_DISPATCH_INDIRECT			0x16
+#define	PACKET3_INDIRECT_BUFFER_END			0x17
+#define	PACKET3_INDIRECT_BUFFER_CNST_END		0x19
+#define	PACKET3_ATOMIC_GDS				0x1D
+#define	PACKET3_ATOMIC_MEM				0x1E
+#define	PACKET3_OCCLUSION_QUERY				0x1F
+#define	PACKET3_SET_PREDICATION				0x20
+#define	PACKET3_REG_RMW					0x21
+#define	PACKET3_COND_EXEC				0x22
+#define	PACKET3_PRED_EXEC				0x23
+#define	PACKET3_DRAW_INDIRECT				0x24
+#define	PACKET3_DRAW_INDEX_INDIRECT			0x25
+#define	PACKET3_INDEX_BASE				0x26
+#define	PACKET3_DRAW_INDEX_2				0x27
+#define	PACKET3_CONTEXT_CONTROL				0x28
+#define	PACKET3_INDEX_TYPE				0x2A
+#define	PACKET3_DRAW_INDIRECT_MULTI			0x2C
+#define	PACKET3_DRAW_INDEX_AUTO				0x2D
+#define	PACKET3_NUM_INSTANCES				0x2F
+#define	PACKET3_DRAW_INDEX_MULTI_AUTO			0x30
+#define	PACKET3_INDIRECT_BUFFER_PRIV			0x32
+#define	PACKET3_INDIRECT_BUFFER_CNST			0x33
+#define	PACKET3_COND_INDIRECT_BUFFER_CNST		0x33
+#define	PACKET3_STRMOUT_BUFFER_UPDATE			0x34
+#define	PACKET3_DRAW_INDEX_OFFSET_2			0x35
+#define	PACKET3_DRAW_PREAMBLE				0x36
+#define	PACKET3_WRITE_DATA				0x37
+#define		WRITE_DATA_DST_SEL(x)                   ((x) << 8)
+		/* 0 - register
+		 * 1 - memory (sync - via GRBM)
+		 * 2 - gl2
+		 * 3 - gds
+		 * 4 - reserved
+		 * 5 - memory (async - direct)
+		 */
+#define		WR_ONE_ADDR                             (1 << 16)
+#define		WR_CONFIRM                              (1 << 20)
+#define		WRITE_DATA_CACHE_POLICY(x)              ((x) << 25)
+		/* 0 - LRU
+		 * 1 - Stream
+		 */
+#define		WRITE_DATA_ENGINE_SEL(x)                ((x) << 30)
+		/* 0 - me
+		 * 1 - pfp
+		 * 2 - ce
+		 */
+#define	PACKET3_DRAW_INDEX_INDIRECT_MULTI		0x38
+#define	PACKET3_MEM_SEMAPHORE				0x39
+#              define PACKET3_SEM_USE_MAILBOX       (0x1 << 16)
+#              define PACKET3_SEM_SEL_SIGNAL_TYPE   (0x1 << 20) /* 0 = increment, 1 = write 1 */
+#              define PACKET3_SEM_SEL_SIGNAL	    (0x6 << 29)
+#              define PACKET3_SEM_SEL_WAIT	    (0x7 << 29)
+#define	PACKET3_DRAW_INDEX_MULTI_INST			0x3A
+#define	PACKET3_COPY_DW					0x3B
+#define	PACKET3_WAIT_REG_MEM				0x3C
+#define		WAIT_REG_MEM_FUNCTION(x)                ((x) << 0)
+		/* 0 - always
+		 * 1 - <
+		 * 2 - <=
+		 * 3 - ==
+		 * 4 - !=
+		 * 5 - >=
+		 * 6 - >
+		 */
+#define		WAIT_REG_MEM_MEM_SPACE(x)               ((x) << 4)
+		/* 0 - reg
+		 * 1 - mem
+		 */
+#define		WAIT_REG_MEM_OPERATION(x)               ((x) << 6)
+		/* 0 - wait_reg_mem
+		 * 1 - wr_wait_wr_reg
+		 */
+#define		WAIT_REG_MEM_ENGINE(x)                  ((x) << 8)
+		/* 0 - me
+		 * 1 - pfp
+		 */
+#define	PACKET3_INDIRECT_BUFFER				0x3F
+#define		INDIRECT_BUFFER_VALID                   (1 << 23)
+#define		INDIRECT_BUFFER_CACHE_POLICY(x)         ((x) << 28)
+		/* 0 - LRU
+		 * 1 - Stream
+		 * 2 - Bypass
+		 */
+#define		INDIRECT_BUFFER_PRE_ENB(x)		((x) << 21)
+#define		INDIRECT_BUFFER_PRE_RESUME(x)           ((x) << 30)
+#define	PACKET3_COND_INDIRECT_BUFFER			0x3F
+#define	PACKET3_COPY_DATA				0x40
+#define	PACKET3_CP_DMA					0x41
+#define	PACKET3_PFP_SYNC_ME				0x42
+#define	PACKET3_SURFACE_SYNC				0x43
+#define	PACKET3_ME_INITIALIZE				0x44
+#define	PACKET3_COND_WRITE				0x45
+#define	PACKET3_EVENT_WRITE				0x46
+#define		EVENT_TYPE(x)                           ((x) << 0)
+#define		EVENT_INDEX(x)                          ((x) << 8)
+		/* 0 - any non-TS event
+		 * 1 - ZPASS_DONE, PIXEL_PIPE_STAT_*
+		 * 2 - SAMPLE_PIPELINESTAT
+		 * 3 - SAMPLE_STREAMOUTSTAT*
+		 * 4 - *S_PARTIAL_FLUSH
+		 */
+#define	PACKET3_EVENT_WRITE_EOP				0x47
+#define	PACKET3_EVENT_WRITE_EOS				0x48
+#define	PACKET3_RELEASE_MEM				0x49
+#define		PACKET3_RELEASE_MEM_EVENT_TYPE(x)	    ((x) << 0)
+#define		PACKET3_RELEASE_MEM_EVENT_INDEX(x)	    ((x) << 8)
+#define		PACKET3_RELEASE_MEM_GCR_GL2_SCOPE(x)	((x) << 12)
+#define		PACKET3_RELEASE_MEM_GCR_GLV_INV		    (1 << 14)
+#define		PACKET3_RELEASE_MEM_GCR_GL2_US		    (1 << 16)
+#define		PACKET3_RELEASE_MEM_GCR_GL2_RANGE(x)	((x) << 17)
+#define		PACKET3_RELEASE_MEM_GCR_GL2_DISCARD	    (1 << 19)
+#define		PACKET3_RELEASE_MEM_GCR_GL2_INV		    (1 << 20)
+#define		PACKET3_RELEASE_MEM_GCR_GL2_WB		    (1 << 21)
+#define		PACKET3_RELEASE_MEM_GCR_SEQ(x)		    ((x) << 22)
+#define		PACKET3_RELEASE_MEM_GCR_GLV_WB		    (1 << 24)
+#define		PACKET3_RELEASE_MEM_TEMPORAL(x)	        ((x) << 25)
+		/* 0 - temporal__release_mem__rt
+		 * 1 - temporal__release_mem__nt
+		 * 2 - temporal__release_mem__ht
+		 * 3 - temporal__release_mem__lu
+		 */
+#define		PACKET3_RELEASE_MEM_EXECUTE		(1 << 28)
+
+#define		PACKET3_RELEASE_MEM_DATA_SEL(x)		((x) << 29)
+		/* 0 - discard
+		 * 1 - send low 32bit data
+		 * 2 - send 64bit data
+		 * 3 - send 64bit GPU counter value
+		 * 4 - send 64bit sys counter value
+		 */
+#define		PACKET3_RELEASE_MEM_INT_SEL(x)		((x) << 24)
+		/* 0 - none
+		 * 1 - interrupt only (DATA_SEL = 0)
+		 * 2 - interrupt when data write is confirmed
+		 */
+#define		PACKET3_RELEASE_MEM_DST_SEL(x)		((x) << 16)
+		/* 0 - MC
+		 * 1 - TC/L2
+		 */
+
+
+
+#define	PACKET3_PREAMBLE_CNTL				0x4A
+#              define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE     (2 << 28)
+#              define PACKET3_PREAMBLE_END_CLEAR_STATE       (3 << 28)
+#define	PACKET3_DMA_DATA				0x50
+/* 1. header
+ * 2. CONTROL
+ * 3. SRC_ADDR_LO or DATA [31:0]
+ * 4. SRC_ADDR_HI [31:0]
+ * 5. DST_ADDR_LO [31:0]
+ * 6. DST_ADDR_HI [7:0]
+ * 7. COMMAND [31:26] | BYTE_COUNT [25:0]
+ */
+/* CONTROL */
+#              define PACKET3_DMA_DATA_ENGINE(x)     ((x) << 0)
+		/* 0 - ME
+		 * 1 - PFP
+		 */
+#              define PACKET3_DMA_DATA_SRC_CACHE_POLICY(x) ((x) << 13)
+		/* 0 - LRU
+		 * 1 - Stream
+		 */
+#              define PACKET3_DMA_DATA_DST_SEL(x)  ((x) << 20)
+		/* 0 - DST_ADDR using DAS
+		 * 1 - GDS
+		 * 3 - DST_ADDR using L2
+		 */
+#              define PACKET3_DMA_DATA_DST_CACHE_POLICY(x) ((x) << 25)
+		/* 0 - LRU
+		 * 1 - Stream
+		 */
+#              define PACKET3_DMA_DATA_SRC_SEL(x)  ((x) << 29)
+		/* 0 - SRC_ADDR using SAS
+		 * 1 - GDS
+		 * 2 - DATA
+		 * 3 - SRC_ADDR using L2
+		 */
+#              define PACKET3_DMA_DATA_CP_SYNC     (1 << 31)
+/* COMMAND */
+#              define PACKET3_DMA_DATA_CMD_SAS     (1 << 26)
+		/* 0 - memory
+		 * 1 - register
+		 */
+#              define PACKET3_DMA_DATA_CMD_DAS     (1 << 27)
+		/* 0 - memory
+		 * 1 - register
+		 */
+#              define PACKET3_DMA_DATA_CMD_SAIC    (1 << 28)
+#              define PACKET3_DMA_DATA_CMD_DAIC    (1 << 29)
+#              define PACKET3_DMA_DATA_CMD_RAW_WAIT  (1 << 30)
+#define	PACKET3_CONTEXT_REG_RMW				0x51
+#define	PACKET3_GFX_CNTX_UPDATE				0x52
+#define	PACKET3_BLK_CNTX_UPDATE				0x53
+#define	PACKET3_INCR_UPDT_STATE				0x55
+#define	PACKET3_ACQUIRE_MEM				0x58
+/* 1.  HEADER
+ * 2.  COHER_CNTL [30:0]
+ * 2.1 ENGINE_SEL [31:31]
+ * 2.  COHER_SIZE [31:0]
+ * 3.  COHER_SIZE_HI [7:0]
+ * 4.  COHER_BASE_LO [31:0]
+ * 5.  COHER_BASE_HI [23:0]
+ * 7.  POLL_INTERVAL [15:0]
+ * 8.  GCR_CNTL [18:0]
+ */
+#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(x) ((x) << 0)
+		/*
+		 * 0:NOP
+		 * 1:ALL
+		 * 2:RANGE
+		 * 3:FIRST_LAST
+		 */
+#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_RANGE(x) ((x) << 2)
+		/*
+		 * 0:ALL
+		 * 1:reserved
+		 * 2:RANGE
+		 * 3:FIRST_LAST
+		 */
+#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_SCOPE(x) ((x) << 4)
+        /*
+         * 0:Device scope
+         * 1:System scope
+         * 2:Force INV/WB all
+         * 3:Reserved
+         */
+#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_WB(x) ((x) << 6)
+#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(x) ((x) << 7)
+#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(x) ((x) << 8)
+#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_US(x) ((x) << 10)
+#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_RANGE(x) ((x) << 11)
+		/*
+		 * 0:ALL
+		 * 1:VOL
+		 * 2:RANGE
+		 * 3:FIRST_LAST
+		 */
+#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_DISCARD(x)  ((x) << 13)
+#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(x) ((x) << 14)
+#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(x) ((x) << 15)
+#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_SEQ(x) ((x) << 16)
+		/*
+		 * 0: PARALLEL
+		 * 1: FORWARD
+		 * 2: REVERSE
+		 */
+#define 	PACKET3_ACQUIRE_MEM_GCR_RANGE_IS_PA  (1 << 18)
+#define	PACKET3_REWIND					0x59
+#define	PACKET3_INTERRUPT				0x5A
+#define	PACKET3_GEN_PDEPTE				0x5B
+#define	PACKET3_INDIRECT_BUFFER_PASID			0x5C
+#define	PACKET3_PRIME_UTCL2				0x5D
+#define	PACKET3_LOAD_UCONFIG_REG			0x5E
+#define	PACKET3_LOAD_SH_REG				0x5F
+#define	PACKET3_LOAD_CONFIG_REG				0x60
+#define	PACKET3_LOAD_CONTEXT_REG			0x61
+#define	PACKET3_LOAD_COMPUTE_STATE			0x62
+#define	PACKET3_LOAD_SH_REG_INDEX			0x63
+#define	PACKET3_SET_CONFIG_REG				0x68
+#define		PACKET3_SET_CONFIG_REG_START			0x00002000
+#define		PACKET3_SET_CONFIG_REG_END			0x00002c00
+#define	PACKET3_SET_CONTEXT_REG				0x69
+#define		PACKET3_SET_CONTEXT_REG_START			0x0000a000
+#define		PACKET3_SET_CONTEXT_REG_END			0x0000a400
+#define	PACKET3_SET_CONTEXT_REG_INDEX			0x6A
+#define	PACKET3_SET_VGPR_REG_DI_MULTI			0x71
+#define	PACKET3_SET_SH_REG_DI				0x72
+#define	PACKET3_SET_CONTEXT_REG_INDIRECT		0x73
+#define	PACKET3_SET_SH_REG_DI_MULTI			0x74
+#define	PACKET3_GFX_PIPE_LOCK				0x75
+#define	PACKET3_SET_SH_REG				0x76
+#define		PACKET3_SET_SH_REG_START			0x00002c00
+#define		PACKET3_SET_SH_REG_END				0x00003000
+#define	PACKET3_SET_SH_REG_OFFSET			0x77
+#define	PACKET3_SET_QUEUE_REG				0x78
+#define	PACKET3_SET_UCONFIG_REG				0x79
+#define		PACKET3_SET_UCONFIG_REG_START			0x0000c000
+#define		PACKET3_SET_UCONFIG_REG_END			0x0000c400
+#define	PACKET3_SET_UCONFIG_REG_INDEX			0x7A
+#define	PACKET3_FORWARD_HEADER				0x7C
+#define	PACKET3_SCRATCH_RAM_WRITE			0x7D
+#define	PACKET3_SCRATCH_RAM_READ			0x7E
+#define	PACKET3_LOAD_CONST_RAM				0x80
+#define	PACKET3_WRITE_CONST_RAM				0x81
+#define	PACKET3_DUMP_CONST_RAM				0x83
+#define	PACKET3_INCREMENT_CE_COUNTER			0x84
+#define	PACKET3_INCREMENT_DE_COUNTER			0x85
+#define	PACKET3_WAIT_ON_CE_COUNTER			0x86
+#define	PACKET3_WAIT_ON_DE_COUNTER_DIFF			0x88
+#define	PACKET3_SWITCH_BUFFER				0x8B
+#define	PACKET3_DISPATCH_DRAW_PREAMBLE			0x8C
+#define	PACKET3_DISPATCH_DRAW_PREAMBLE_ACE		0x8C
+#define	PACKET3_DISPATCH_DRAW				0x8D
+#define	PACKET3_DISPATCH_DRAW_ACE			0x8D
+#define	PACKET3_GET_LOD_STATS				0x8E
+#define	PACKET3_DRAW_MULTI_PREAMBLE			0x8F
+#define	PACKET3_FRAME_CONTROL				0x90
+#			define FRAME_TMZ	(1 << 0)
+#			define FRAME_CMD(x) ((x) << 28)
+			/*
+			 * x=0: tmz_begin
+			 * x=1: tmz_end
+			 */
+#define	PACKET3_INDEX_ATTRIBUTES_INDIRECT		0x91
+#define	PACKET3_WAIT_REG_MEM64				0x93
+#define	PACKET3_COND_PREEMPT				0x94
+#define	PACKET3_HDP_FLUSH				0x95
+#define	PACKET3_COPY_DATA_RB				0x96
+#define	PACKET3_INVALIDATE_TLBS				0x98
+#define PACKET3_INVALIDATE_TLBS_DST_SEL(x)     ((x) << 0)
+#define PACKET3_INVALIDATE_TLBS_ALL_HUB(x)     ((x) << 4)
+#define PACKET3_INVALIDATE_TLBS_PASID(x)       ((x) << 5)
+#define PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(x)  ((x) << 29)
+
+#define	PACKET3_AQL_PACKET				0x99
+#define	PACKET3_DMA_DATA_FILL_MULTI			0x9A
+#define	PACKET3_SET_SH_REG_INDEX			0x9B
+#define	PACKET3_DRAW_INDIRECT_COUNT_MULTI		0x9C
+#define	PACKET3_DRAW_INDEX_INDIRECT_COUNT_MULTI		0x9D
+#define	PACKET3_DUMP_CONST_RAM_OFFSET			0x9E
+#define	PACKET3_LOAD_CONTEXT_REG_INDEX			0x9F
+#define	PACKET3_SET_RESOURCES				0xA0
+/* 1. header
+ * 2. CONTROL
+ * 3. QUEUE_MASK_LO [31:0]
+ * 4. QUEUE_MASK_HI [31:0]
+ * 5. GWS_MASK_LO [31:0]
+ * 6. GWS_MASK_HI [31:0]
+ * 7. OAC_MASK [15:0]
+ * 8. GDS_HEAP_SIZE [16:11] | GDS_HEAP_BASE [5:0]
+ */
+#              define PACKET3_SET_RESOURCES_VMID_MASK(x)     ((x) << 0)
+#              define PACKET3_SET_RESOURCES_UNMAP_LATENTY(x) ((x) << 16)
+#              define PACKET3_SET_RESOURCES_QUEUE_TYPE(x)    ((x) << 29)
+#define PACKET3_MAP_PROCESS				0xA1
+#define PACKET3_MAP_QUEUES				0xA2
+/* 1. header
+ * 2. CONTROL
+ * 3. CONTROL2
+ * 4. MQD_ADDR_LO [31:0]
+ * 5. MQD_ADDR_HI [31:0]
+ * 6. WPTR_ADDR_LO [31:0]
+ * 7. WPTR_ADDR_HI [31:0]
+ */
+/* CONTROL */
+#              define PACKET3_MAP_QUEUES_QUEUE_SEL(x)       ((x) << 4)
+#              define PACKET3_MAP_QUEUES_VMID(x)            ((x) << 8)
+#              define PACKET3_MAP_QUEUES_QUEUE(x)           ((x) << 13)
+#              define PACKET3_MAP_QUEUES_PIPE(x)            ((x) << 16)
+#              define PACKET3_MAP_QUEUES_ME(x)              ((x) << 18)
+#              define PACKET3_MAP_QUEUES_QUEUE_TYPE(x)      ((x) << 21)
+#              define PACKET3_MAP_QUEUES_ALLOC_FORMAT(x)    ((x) << 24)
+#              define PACKET3_MAP_QUEUES_ENGINE_SEL(x)      ((x) << 26)
+#              define PACKET3_MAP_QUEUES_NUM_QUEUES(x)      ((x) << 29)
+/* CONTROL2 */
+#              define PACKET3_MAP_QUEUES_CHECK_DISABLE(x)   ((x) << 1)
+#              define PACKET3_MAP_QUEUES_DOORBELL_OFFSET(x) ((x) << 2)
+#define	PACKET3_UNMAP_QUEUES				0xA3
+/* 1. header
+ * 2. CONTROL
+ * 3. CONTROL2
+ * 4. CONTROL3
+ * 5. CONTROL4
+ * 6. CONTROL5
+ */
+/* CONTROL */
+#              define PACKET3_UNMAP_QUEUES_ACTION(x)           ((x) << 0)
+		/* 0 - PREEMPT_QUEUES
+		 * 1 - RESET_QUEUES
+		 * 2 - DISABLE_PROCESS_QUEUES
+		 * 3 - PREEMPT_QUEUES_NO_UNMAP
+		 */
+#              define PACKET3_UNMAP_QUEUES_QUEUE_SEL(x)        ((x) << 4)
+#              define PACKET3_UNMAP_QUEUES_ENGINE_SEL(x)       ((x) << 26)
+#              define PACKET3_UNMAP_QUEUES_NUM_QUEUES(x)       ((x) << 29)
+/* CONTROL2a */
+#              define PACKET3_UNMAP_QUEUES_PASID(x)            ((x) << 0)
+/* CONTROL2b */
+#              define PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(x) ((x) << 2)
+/* CONTROL3a */
+#              define PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET1(x) ((x) << 2)
+/* CONTROL3b */
+#              define PACKET3_UNMAP_QUEUES_RB_WPTR(x)          ((x) << 0)
+/* CONTROL4 */
+#              define PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET2(x) ((x) << 2)
+/* CONTROL5 */
+#              define PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET3(x) ((x) << 2)
+#define	PACKET3_QUERY_STATUS				0xA4
+/* 1. header
+ * 2. CONTROL
+ * 3. CONTROL2
+ * 4. ADDR_LO [31:0]
+ * 5. ADDR_HI [31:0]
+ * 6. DATA_LO [31:0]
+ * 7. DATA_HI [31:0]
+ */
+/* CONTROL */
+#              define PACKET3_QUERY_STATUS_CONTEXT_ID(x)       ((x) << 0)
+#              define PACKET3_QUERY_STATUS_INTERRUPT_SEL(x)    ((x) << 28)
+#              define PACKET3_QUERY_STATUS_COMMAND(x)          ((x) << 30)
+/* CONTROL2a */
+#              define PACKET3_QUERY_STATUS_PASID(x)            ((x) << 0)
+/* CONTROL2b */
+#              define PACKET3_QUERY_STATUS_DOORBELL_OFFSET(x)  ((x) << 2)
+#              define PACKET3_QUERY_STATUS_ENG_SEL(x)          ((x) << 25)
+#define	PACKET3_RUN_LIST				0xA5
+#define	PACKET3_MAP_PROCESS_VM				0xA6
+/* GFX11 */
+#define	PACKET3_SET_Q_PREEMPTION_MODE			0xF0
+#              define PACKET3_SET_Q_PREEMPTION_MODE_IB_VMID(x)  ((x) << 0)
+#              define PACKET3_SET_Q_PREEMPTION_MODE_INIT_SHADOW_MEM    (1 << 0)
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 2b7aba22ecc1..9c0bcf836b2e 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -2068,23 +2068,15 @@ static int gfx_v7_0_ring_test_ring(struct amdgpu_ring *ring)
 static void gfx_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
 	u32 ref_and_mask;
-	int usepfp = ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE ? 0 : 1;
+	int usepfp;
+	struct amdgpu_device *adev = ring->adev;
 
-	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
-		switch (ring->me) {
-		case 1:
-			ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;
-			break;
-		case 2:
-			ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe;
-			break;
-		default:
-			return;
-		}
-	} else {
-		ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK;
+	if (!adev->gfx.funcs->get_hdp_flush_mask) {
+		dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__);
+		return;
 	}
 
+	adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, &usepfp);
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
 	amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
 				 WAIT_REG_MEM_FUNCTION(3) |  /* == */
@@ -3245,7 +3237,7 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
 
 	/* init spm vmid with 0xf */
 	if (adev->gfx.rlc.funcs->update_spm_vmid)
-		adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf);
+		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf);
 
 	return 0;
 }
@@ -3471,7 +3463,8 @@ static int gfx_v7_0_rlc_resume(struct amdgpu_device *adev)
 	return 0;
 }
 
-static void gfx_v7_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned vmid)
+static void gfx_v7_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id,
+		struct amdgpu_ring *ring, unsigned vmid)
 {
 	u32 data;
 
@@ -4074,12 +4067,49 @@ static void gfx_v7_0_select_me_pipe_q(struct amdgpu_device *adev,
 	cik_srbm_select(adev, me, pipe, q, vm);
 }
 
+/**
+ * gfx_v7_0_get_hdp_flush_mask - get the reference and mask for HDP flush
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ * @ref_and_mask: pointer to store the reference and mask
+ * @reg_mem_engine: pointer to store the register memory engine
+ *
+ * Calculates the reference and mask for HDP flush based on the ring type and me.
+ */
+static void gfx_v7_0_get_hdp_flush_mask(struct amdgpu_ring *ring,
+					uint32_t *ref_and_mask, uint32_t *reg_mem_engine)
+{
+	if (!ring || !ref_and_mask || !reg_mem_engine) {
+		DRM_INFO("%s:invalid params\n", __func__);
+		return;
+	}
+
+	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE ||
+		ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {
+		switch (ring->me) {
+		case 1:
+			*ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;
+			break;
+		case 2:
+			*ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe;
+			break;
+		default:
+			return;
+		}
+		*reg_mem_engine = 0;
+	} else {
+		*ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK;
+		*reg_mem_engine = 1;
+	}
+}
+
 static const struct amdgpu_gfx_funcs gfx_v7_0_gfx_funcs = {
 	.get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
 	.select_se_sh = &gfx_v7_0_select_se_sh,
 	.read_wave_data = &gfx_v7_0_read_wave_data,
 	.read_wave_sgprs = &gfx_v7_0_read_wave_sgprs,
-	.select_me_pipe_q = &gfx_v7_0_select_me_pipe_q
+	.select_me_pipe_q = &gfx_v7_0_select_me_pipe_q,
+	.get_hdp_flush_mask = &gfx_v7_0_get_hdp_flush_mask,
 };
 
 static const struct amdgpu_rlc_funcs gfx_v7_0_rlc_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index 1c87375e1dd5..5c1f8230be8c 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -1274,7 +1274,7 @@ static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
 
 	/* init spm vmid with 0xf */
 	if (adev->gfx.rlc.funcs->update_spm_vmid)
-		adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf);
+		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf);
 
 	return 0;
 }
@@ -5211,13 +5211,49 @@ static void gfx_v8_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id
 		start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
 }
 
+/**
+ * gfx_v8_0_get_hdp_flush_mask - get the reference and mask for HDP flush
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ * @ref_and_mask: pointer to store the reference and mask
+ * @reg_mem_engine: pointer to store the register memory engine
+ *
+ * Calculates the reference and mask for HDP flush based on the ring type and me.
+ */
+static void gfx_v8_0_get_hdp_flush_mask(struct amdgpu_ring *ring,
+					uint32_t *ref_and_mask, uint32_t *reg_mem_engine)
+{
+	if (!ring || !ref_and_mask || !reg_mem_engine) {
+		DRM_INFO("%s:invalid params\n", __func__);
+		return;
+	}
+
+	if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) ||
+	    (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)) {
+		switch (ring->me) {
+		case 1:
+			*ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;
+			break;
+		case 2:
+			*ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe;
+			break;
+		default:
+			return;
+		}
+		*reg_mem_engine = 0;
+	} else {
+		*ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK;
+		*reg_mem_engine = WAIT_REG_MEM_ENGINE(1); /* pfp */
+	}
+}
 
 static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = {
 	.get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter,
 	.select_se_sh = &gfx_v8_0_select_se_sh,
 	.read_wave_data = &gfx_v8_0_read_wave_data,
 	.read_wave_sgprs = &gfx_v8_0_read_wave_sgprs,
-	.select_me_pipe_q = &gfx_v8_0_select_me_pipe_q
+	.select_me_pipe_q = &gfx_v8_0_select_me_pipe_q,
+	.get_hdp_flush_mask = &gfx_v8_0_get_hdp_flush_mask,
 };
 
 static int gfx_v8_0_early_init(struct amdgpu_ip_block *ip_block)
@@ -5541,7 +5577,8 @@ static void gfx_v8_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
 	}
 }
 
-static void gfx_v8_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned vmid)
+static void gfx_v8_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id,
+		struct amdgpu_ring *ring, unsigned vmid)
 {
 	u32 data;
 
@@ -5999,25 +6036,14 @@ static void gfx_v8_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
 static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
 	u32 ref_and_mask, reg_mem_engine;
+	struct amdgpu_device *adev = ring->adev;
 
-	if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) ||
-	    (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)) {
-		switch (ring->me) {
-		case 1:
-			ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;
-			break;
-		case 2:
-			ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe;
-			break;
-		default:
-			return;
-		}
-		reg_mem_engine = 0;
-	} else {
-		ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK;
-		reg_mem_engine = WAIT_REG_MEM_ENGINE(1); /* pfp */
+	if (!adev->gfx.funcs->get_hdp_flush_mask) {
+		dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__);
+		return;
 	}
 
+	adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, &reg_mem_engine);
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
 	amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
 				 WAIT_REG_MEM_FUNCTION(3) |  /* == */
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 0148d7ff34d9..bb1465a98c7c 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -2004,6 +2004,7 @@ static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
         .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
         .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
+	.get_hdp_flush_mask = &amdgpu_gfx_get_hdp_flush_mask,
 };
 
 const struct amdgpu_ras_block_hw_ops  gfx_v9_0_ras_ops = {
@@ -5171,7 +5172,8 @@ static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
 		WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
 }
 
-static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned int vmid)
+static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id,
+		struct amdgpu_ring *ring, unsigned int vmid)
 {
 	amdgpu_gfx_off_ctrl(adev, false);
 
@@ -5379,25 +5381,13 @@ static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 	u32 ref_and_mask, reg_mem_engine;
-	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
 
-	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
-		switch (ring->me) {
-		case 1:
-			ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
-			break;
-		case 2:
-			ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
-			break;
-		default:
-			return;
-		}
-		reg_mem_engine = 0;
-	} else {
-		ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
-		reg_mem_engine = 1; /* pfp */
+	if (!adev->gfx.funcs->get_hdp_flush_mask) {
+		dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__);
+		return;
 	}
 
+	adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, &reg_mem_engine);
 	gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
 			      adev->nbio.funcs->get_hdp_flush_req_offset(adev),
 			      adev->nbio.funcs->get_hdp_flush_done_offset(adev),
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
index cbb74ffc4792..4e4d88d6357f 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
@@ -848,6 +848,7 @@ static const struct amdgpu_gfx_funcs gfx_v9_4_3_gfx_funcs = {
 	.switch_partition_mode = &gfx_v9_4_3_switch_compute_partition,
 	.ih_node_to_logical_xcc = &gfx_v9_4_3_ih_to_xcc_inst,
 	.get_xccs_per_xcp = &gfx_v9_4_3_get_xccs_per_xcp,
+	.get_hdp_flush_mask = &amdgpu_gfx_get_hdp_flush_mask,
 };
 
 static int gfx_v9_4_3_aca_bank_parser(struct aca_handle *handle,
@@ -1455,7 +1456,7 @@ static int gfx_v9_4_3_rlc_init(struct amdgpu_device *adev)
 {
 	/* init spm vmid with 0xf */
 	if (adev->gfx.rlc.funcs->update_spm_vmid)
-		adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf);
+		adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf);
 
 	return 0;
 }
@@ -1666,12 +1667,12 @@ static int gfx_v9_4_3_rlc_resume(struct amdgpu_device *adev)
 	return 0;
 }
 
-static void gfx_v9_4_3_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring,
-				       unsigned vmid)
+static void gfx_v9_4_3_update_spm_vmid(struct amdgpu_device *adev,
+					      int inst, struct amdgpu_ring *ring, unsigned int vmid)
 {
 	u32 reg, pre_data, data;
 
-	reg = SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regRLC_SPM_MC_CNTL);
+	reg = SOC15_REG_OFFSET(GC, GET_INST(GC, inst), regRLC_SPM_MC_CNTL);
 	if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev))
 		pre_data = RREG32_NO_KIQ(reg);
 	else
@@ -1682,9 +1683,9 @@ static void gfx_v9_4_3_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu
 
 	if (pre_data != data) {
 		if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) {
-			WREG32_SOC15_NO_KIQ(GC, GET_INST(GC, 0), regRLC_SPM_MC_CNTL, data);
+			WREG32_SOC15_NO_KIQ(GC, GET_INST(GC, inst), regRLC_SPM_MC_CNTL, data);
 		} else
-			WREG32_SOC15(GC, GET_INST(GC, 0), regRLC_SPM_MC_CNTL, data);
+			WREG32_SOC15(GC, GET_INST(GC, inst), regRLC_SPM_MC_CNTL, data);
 	}
 }
 
@@ -2818,25 +2819,13 @@ static void gfx_v9_4_3_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 	u32 ref_and_mask, reg_mem_engine;
-	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
 
-	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
-		switch (ring->me) {
-		case 1:
-			ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
-			break;
-		case 2:
-			ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
-			break;
-		default:
-			return;
-		}
-		reg_mem_engine = 0;
-	} else {
-		ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
-		reg_mem_engine = 1; /* pfp */
+	if (!adev->gfx.funcs->get_hdp_flush_mask) {
+		dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__);
+		return;
 	}
 
+	adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, &reg_mem_engine);
 	gfx_v9_4_3_wait_reg_mem(ring, reg_mem_engine, 0, 1,
 			      adev->nbio.funcs->get_hdp_flush_req_offset(adev),
 			      adev->nbio.funcs->get_hdp_flush_done_offset(adev),
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
new file mode 100644
index 000000000000..662542b9cc36
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
@@ -0,0 +1,879 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "amdgpu_xcp.h"
+#include "gfxhub_v12_1.h"
+
+#include "gc/gc_12_1_0_offset.h"
+#include "gc/gc_12_1_0_sh_mask.h"
+#include "soc_v1_0_enum.h"
+
+#include "soc15_common.h"
+
+#define regGCVM_L2_CNTL3_DEFAULT		0x80120007
+#define regGCVM_L2_CNTL4_DEFAULT		0x000000c1
+#define regGCVM_L2_CNTL5_DEFAULT		0x00003fe0
+#define regGRBM_GFX_INDEX_DEFAULT			0xe0000000
+
+
+static u64 gfxhub_v12_1_get_fb_location(struct amdgpu_device *adev)
+{
+	u64 base;
+
+	base = RREG32_SOC15(GC, GET_INST(GC, 0),
+			    regGCMC_VM_FB_LOCATION_BASE_LO32);
+	base &= GCMC_VM_FB_LOCATION_BASE_LO32__FB_BASE_LO32_MASK;
+	base <<= 24;
+
+	base |= ((u64)(GCMC_VM_FB_LOCATION_BASE_HI32__FB_BASE_HI1_MASK &
+		       RREG32_SOC15(GC, GET_INST(GC, 0),
+				    regGCMC_VM_FB_LOCATION_BASE_HI32)) << 56);
+	return base;
+}
+
+static u64 gfxhub_v12_1_get_mc_fb_offset(struct amdgpu_device *adev)
+{
+	return (u64)(RREG32_SOC15(GC, GET_INST(GC, 0),
+				  regGCMC_VM_FB_OFFSET) << 24);
+}
+
+static void gfxhub_v12_1_xcc_setup_vm_pt_regs(struct amdgpu_device *adev,
+					      uint32_t vmid,
+					      uint64_t page_table_base,
+					      uint32_t xcc_mask)
+{
+	struct amdgpu_vmhub *hub;
+	int i;
+
+	for (i = 0; i < NUM_XCC(xcc_mask); i++) {
+		hub = &adev->vmhub[AMDGPU_GFXHUB(i)];
+		WREG32_SOC15_OFFSET(GC, GET_INST(GC, i),
+				    regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
+				    hub->ctx_addr_distance * vmid,
+				    lower_32_bits(page_table_base));
+
+		WREG32_SOC15_OFFSET(GC, GET_INST(GC, i),
+				    regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
+				    hub->ctx_addr_distance * vmid,
+				    upper_32_bits(page_table_base));
+	}
+}
+
+static void gfxhub_v12_1_setup_vm_pt_regs(struct amdgpu_device *adev,
+					  uint32_t vmid,
+					  uint64_t page_table_base)
+{
+	gfxhub_v12_1_xcc_setup_vm_pt_regs(adev, vmid, page_table_base,
+					  adev->gfx.xcc_mask);
+}
+
+static void gfxhub_v12_1_xcc_init_gart_aperture_regs(struct amdgpu_device *adev,
+						     uint32_t xcc_mask)
+{
+	uint64_t pt_base;
+	int i;
+
+	if (adev->gmc.pdb0_bo)
+		pt_base = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo);
+	else
+		pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
+
+	gfxhub_v12_1_xcc_setup_vm_pt_regs(adev, 0, pt_base, xcc_mask);
+
+	/* If use GART for FB translation, vmid0 page table covers both
+	 * vram and system memory (gart)
+	 */
+	for (i = 0; i < NUM_XCC(xcc_mask); i++) {
+		if (adev->gmc.pdb0_bo) {
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
+				     (u32)(adev->gmc.fb_start >> 12));
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
+				     (u32)(adev->gmc.fb_start >> 44));
+
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
+				     (u32)(adev->gmc.gart_end >> 12));
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
+				     (u32)(adev->gmc.gart_end >> 44));
+		} else {
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
+				     (u32)(adev->gmc.gart_start >> 12));
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
+				     (u32)(adev->gmc.gart_start >> 44));
+
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
+				     (u32)(adev->gmc.gart_end >> 12));
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
+				     (u32)(adev->gmc.gart_end >> 44));
+		}
+	}
+}
+
+static void gfxhub_v12_1_xcc_init_system_aperture_regs(struct amdgpu_device *adev,
+						       uint32_t xcc_mask)
+{
+	uint64_t value;
+	uint32_t tmp;
+	int i;
+
+	for (i = 0; i < NUM_XCC(xcc_mask); i++) {
+		/* Program the AGP BAR */
+		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
+				 regGCMC_VM_AGP_BASE_LO32, 0);
+		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
+				 regGCMC_VM_AGP_BASE_HI32, 0);
+		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
+				 regGCMC_VM_AGP_BOT_LO32,
+				 lower_32_bits(adev->gmc.agp_start >> 24));
+		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
+				 regGCMC_VM_AGP_BOT_HI32,
+				 upper_32_bits(adev->gmc.agp_start >> 24));
+		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
+				 regGCMC_VM_AGP_TOP_LO32,
+				 lower_32_bits(adev->gmc.agp_end >> 24));
+		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
+				 regGCMC_VM_AGP_TOP_HI32,
+				 upper_32_bits(adev->gmc.agp_end >> 24));
+
+		if (!amdgpu_sriov_vf(adev)) {
+			/* Program the system aperture low logical page number. */
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32,
+				     lower_32_bits(min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18));
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32,
+				     upper_32_bits(min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18));
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32,
+				     lower_32_bits(max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18));
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32,
+				     upper_32_bits(max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18));
+
+			/* Set default page address. */
+			value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr);
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
+				     (u32)(value >> 12));
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
+				     (u32)(value >> 44));
+
+			/* Program "protection fault". */
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
+				     (u32)(adev->dummy_page_addr >> 12));
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
+				     (u32)((u64)adev->dummy_page_addr >> 44));
+
+			tmp = RREG32_SOC15(GC, GET_INST(GC, i),
+					   regGCVM_L2_PROTECTION_FAULT_CNTL2);
+			tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
+					    ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+			tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
+					    ENABLE_RETRY_FAULT_INTERRUPT, 0x1);
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCVM_L2_PROTECTION_FAULT_CNTL2, tmp);
+		}
+
+		/* In the case squeezing vram into GART aperture, we don't use
+		 * FB aperture and AGP aperture. Disable them.
+		 */
+		if (adev->gmc.pdb0_bo) {
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_FB_LOCATION_TOP_LO32, 0);
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_FB_LOCATION_TOP_HI32, 0);
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_FB_LOCATION_BASE_LO32,
+				     0xFFFFFFFF);
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_FB_LOCATION_BASE_HI32, 1);
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_AGP_TOP_LO32, 0);
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_AGP_TOP_HI32, 0);
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_AGP_BOT_LO32, 0xFFFFFFFF);
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_AGP_BOT_HI32, 1);
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32,
+				     0xFFFFFFFF);
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32,
+				     0x7F);
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32, 0);
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32, 0);
+		}
+	}
+}
+
+static void gfxhub_v12_1_xcc_init_tlb_regs(struct amdgpu_device *adev,
+					   uint32_t xcc_mask)
+{
+	uint32_t tmp;
+	int i;
+
+	for (i = 0; i < NUM_XCC(xcc_mask); i++) {
+		/* Setup TLB control */
+		tmp = RREG32_SOC15(GC, GET_INST(GC, i),
+				   regGCMC_VM_MX_L1_TLB_CNTL);
+
+		tmp = REG_SET_FIELD(tmp,
+				    GCMC_VM_MX_L1_TLB_CNTL,
+				    ENABLE_L1_TLB, 1);
+		tmp = REG_SET_FIELD(tmp,
+				    GCMC_VM_MX_L1_TLB_CNTL,
+				    SYSTEM_ACCESS_MODE, 3);
+		tmp = REG_SET_FIELD(tmp,
+				    GCMC_VM_MX_L1_TLB_CNTL,
+				    ENABLE_ADVANCED_DRIVER_MODEL, 1);
+		tmp = REG_SET_FIELD(tmp,
+				    GCMC_VM_MX_L1_TLB_CNTL,
+				    SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
+		tmp = REG_SET_FIELD(tmp,
+				    GCMC_VM_MX_L1_TLB_CNTL,
+				    ECO_BITS, 0);
+		tmp = REG_SET_FIELD(tmp,
+				    GCMC_VM_MX_L1_TLB_CNTL,
+				    MTYPE, MTYPE_UC);
+
+		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
+				 regGCMC_VM_MX_L1_TLB_CNTL, tmp);
+	}
+}
+
+static void gfxhub_v12_1_xcc_init_cache_regs(struct amdgpu_device *adev,
+					     uint32_t xcc_mask)
+{
+	uint32_t tmp;
+	int i;
+
+	for (i = 0; i < NUM_XCC(xcc_mask); i++) {
+		/* Setup L2 cache */
+		tmp = RREG32_SOC15(GC, GET_INST(GC, i), regGCVM_L2_CNTL);
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
+				    ENABLE_L2_CACHE, 1);
+		/*TODO: set ENABLE_L2_FRAGMENT_PROCESSING to 1? */
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
+				    ENABLE_L2_FRAGMENT_PROCESSING, 0);
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
+				    ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
+				    L2_PDE0_CACHE_TAG_GENERATION_MODE, 0);
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
+				    PDE_FAULT_CLASSIFICATION, 0);
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
+				    CONTEXT1_IDENTITY_ACCESS_MODE, 1);
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
+				    IDENTITY_MODE_FRAGMENT_SIZE, 0);
+		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
+				 regGCVM_L2_CNTL, tmp);
+
+		tmp = RREG32_SOC15(GC, GET_INST(GC, i), regGCVM_L2_CNTL2);
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2,
+				    INVALIDATE_ALL_L1_TLBS, 1);
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2,
+				    INVALIDATE_L2_CACHE, 1);
+		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
+				 regGCVM_L2_CNTL2, tmp);
+
+		tmp = regGCVM_L2_CNTL3_DEFAULT;
+		if (adev->gmc.translate_further) {
+			tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 12);
+			tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3,
+					    L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
+		} else {
+			tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 9);
+			tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3,
+					    L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
+		}
+		WREG32_SOC15_RLC(GC, GET_INST(GC, i), regGCVM_L2_CNTL3, tmp);
+
+		tmp = regGCVM_L2_CNTL4_DEFAULT;
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4,
+				    VMC_TAP_PDE_REQUEST_PHYSICAL, 1);
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4,
+				    VMC_TAP_PTE_REQUEST_PHYSICAL, 1);
+		WREG32_SOC15_RLC(GC, GET_INST(GC, i), regGCVM_L2_CNTL4, tmp);
+
+		tmp = regGCVM_L2_CNTL5_DEFAULT;
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL5,
+				    L2_CACHE_SMALLK_FRAGMENT_SIZE, 0);
+		WREG32_SOC15_RLC(GC, GET_INST(GC, i), regGCVM_L2_CNTL5, tmp);
+	}
+}
+
+static void gfxhub_v12_1_xcc_enable_system_domain(struct amdgpu_device *adev,
+						  uint32_t xcc_mask)
+{
+	uint32_t tmp;
+	int i;
+
+	for (i = 0; i < NUM_XCC(xcc_mask); i++) {
+		tmp = RREG32_SOC15(GC, GET_INST(GC, i),
+				   regGCVM_CONTEXT0_CNTL);
+		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL,
+				    ENABLE_CONTEXT, 1);
+		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL,
+				    PAGE_TABLE_DEPTH,
+				    adev->gmc.vmid0_page_table_depth);
+		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL,
+				    PAGE_TABLE_BLOCK_SIZE,
+				    adev->gmc.vmid0_page_table_block_size);
+		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL,
+				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGCVM_CONTEXT0_CNTL, tmp);
+	}
+}
+
+static void gfxhub_v12_1_xcc_disable_identity_aperture(struct amdgpu_device *adev,
+						       uint32_t xcc_mask)
+{
+	int i;
+
+	for (i = 0; i < NUM_XCC(xcc_mask); i++) {
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
+			     0XFFFFFFFF);
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
+			     0x00001FFF);
+
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32,
+			     0);
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32,
+			     0);
+
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32,
+			     0);
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32,
+			     0);
+	}
+}
+
+static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev,
+					       uint32_t xcc_mask)
+{
+	struct amdgpu_vmhub *hub;
+	unsigned int num_level, block_size;
+	uint32_t tmp;
+	int i, j;
+
+	num_level = adev->vm_manager.num_level;
+	block_size = adev->vm_manager.block_size;
+	block_size -= 9;
+
+	for (j = 0; j < NUM_XCC(xcc_mask); j++) {
+		hub = &adev->vmhub[AMDGPU_GFXHUB(j)];
+		for (i = 0; i <= 14; i++) {
+			tmp = RREG32_SOC15_OFFSET(GC, GET_INST(GC, j),
+					          regGCVM_CONTEXT1_CNTL,
+						  i * hub->ctx_distance);
+			tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+					    ENABLE_CONTEXT, 1);
+			tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+					    PAGE_TABLE_DEPTH, num_level);
+			tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+					    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+			tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+					    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+			tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+					    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+			tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+					    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+			tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+					    READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+			tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+					    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+			tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+					    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+			tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+					    PAGE_TABLE_BLOCK_SIZE, block_size);
+			/* Send no-retry XNACK on fault to suppress VM fault storm */
+			tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+					    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
+					    1);
+			WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), regGCVM_CONTEXT1_CNTL,
+					    i * hub->ctx_distance, tmp);
+			WREG32_SOC15_OFFSET(GC, GET_INST(GC, j),
+					    regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
+					    i * hub->ctx_addr_distance, 0);
+			WREG32_SOC15_OFFSET(GC, GET_INST(GC, j),
+					    regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
+					    i * hub->ctx_addr_distance, 0);
+			WREG32_SOC15_OFFSET(GC, GET_INST(GC, j),
+					    regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
+					    i * hub->ctx_addr_distance,
+					    lower_32_bits(adev->vm_manager.max_pfn - 1));
+			WREG32_SOC15_OFFSET(GC, GET_INST(GC, j),
+					    regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
+					    i * hub->ctx_addr_distance,
+					    upper_32_bits(adev->vm_manager.max_pfn - 1));
+		}
+
+		hub->vm_cntx_cntl = tmp;
+	}
+}
+
+static void gfxhub_v12_1_xcc_program_invalidation(struct amdgpu_device *adev,
+						  uint32_t xcc_mask)
+{
+	struct amdgpu_vmhub *hub;
+	unsigned int i, j;
+
+	for (j = 0; j < NUM_XCC(xcc_mask); j++) {
+		hub = &adev->vmhub[AMDGPU_GFXHUB(j)];
+
+		for (i = 0 ; i < 18; ++i) {
+			WREG32_SOC15_OFFSET(GC, GET_INST(GC, j),
+					    regGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
+					    i * hub->eng_addr_distance, 0xFFFFFFFF);
+			WREG32_SOC15_OFFSET(GC, GET_INST(GC, j),
+					    regGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
+					    i * hub->eng_addr_distance, 0x3FFF);
+		}
+	}
+}
+
+static int gfxhub_v12_1_xcc_gart_enable(struct amdgpu_device *adev,
+					uint32_t xcc_mask)
+{
+	uint32_t i;
+
+	if (amdgpu_sriov_vf(adev)) {
+		/* GCMC_VM_FB_LOCATION_BASE/TOP are VF copy registers
+		 * VBIO post does not program them at boot up phase
+		 * Need driver to program them from guest side */
+		for (i = 0; i < NUM_XCC(xcc_mask); i++) {
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_FB_LOCATION_BASE_LO32,
+				     lower_32_bits(adev->gmc.vram_start >> 24));
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_FB_LOCATION_BASE_HI32,
+				     upper_32_bits(adev->gmc.vram_start >> 24));
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_FB_LOCATION_TOP_LO32,
+				     lower_32_bits(adev->gmc.vram_end >> 24));
+			WREG32_SOC15(GC, GET_INST(GC, i),
+				     regGCMC_VM_FB_LOCATION_TOP_HI32,
+				     upper_32_bits(adev->gmc.vram_end >> 24));
+		}
+	}
+	/* GART Enable. */
+	gfxhub_v12_1_xcc_init_gart_aperture_regs(adev, xcc_mask);
+	gfxhub_v12_1_xcc_init_system_aperture_regs(adev, xcc_mask);
+	gfxhub_v12_1_xcc_init_tlb_regs(adev, xcc_mask);
+	if (!amdgpu_sriov_vf(adev))
+		gfxhub_v12_1_xcc_init_cache_regs(adev, xcc_mask);
+
+	gfxhub_v12_1_xcc_enable_system_domain(adev, xcc_mask);
+	if (!amdgpu_sriov_vf(adev))
+		gfxhub_v12_1_xcc_disable_identity_aperture(adev, xcc_mask);
+	gfxhub_v12_1_xcc_setup_vmid_config(adev, xcc_mask);
+	gfxhub_v12_1_xcc_program_invalidation(adev, xcc_mask);
+
+	return 0;
+}
+
+static int gfxhub_v12_1_gart_enable(struct amdgpu_device *adev)
+{
+	return gfxhub_v12_1_xcc_gart_enable(adev,
+					    adev->gfx.xcc_mask);
+}
+
+static void gfxhub_v12_1_xcc_gart_disable(struct amdgpu_device *adev,
+					  uint32_t xcc_mask)
+{
+	struct amdgpu_vmhub *hub;
+	u32 tmp;
+	u32 i, j;
+
+	for (j = 0; j < NUM_XCC(xcc_mask); j++) {
+		hub = &adev->vmhub[AMDGPU_GFXHUB(j)];
+		/* Disable all tables */
+		for (i = 0; i < 16; i++)
+			WREG32_SOC15_OFFSET(GC, GET_INST(GC, j),
+					    regGCVM_CONTEXT0_CNTL,
+					    i * hub->ctx_distance, 0);
+
+		/* Setup TLB control */
+		tmp = RREG32_SOC15(GC, GET_INST(GC, j),
+				   regGCMC_VM_MX_L1_TLB_CNTL);
+		tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
+				    ENABLE_L1_TLB, 0);
+		tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
+				    ENABLE_ADVANCED_DRIVER_MODEL, 0);
+		WREG32_SOC15_RLC(GC, GET_INST(GC, j),
+				 regGCMC_VM_MX_L1_TLB_CNTL, tmp);
+
+		/* Setup L2 cache */
+		if (!amdgpu_sriov_vf(adev)) {
+			tmp = RREG32_SOC15(GC, GET_INST(GC, j), regGCVM_L2_CNTL);
+			tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_CACHE, 0);
+			WREG32_SOC15(GC, GET_INST(GC, j), regGCVM_L2_CNTL, tmp);
+			WREG32_SOC15(GC, GET_INST(GC, j), regGCVM_L2_CNTL3, 0);
+		}
+	}
+}
+
+static void gfxhub_v12_1_gart_disable(struct amdgpu_device *adev)
+{
+	gfxhub_v12_1_xcc_gart_disable(adev, adev->gfx.xcc_mask);
+}
+
+static void gfxhub_v12_1_xcc_set_fault_enable_default(struct amdgpu_device *adev,
+						      bool value, uint32_t xcc_mask)
+{
+	u32 tmp;
+	int i;
+
+	for (i = 0; i < NUM_XCC(xcc_mask); i++) {
+		tmp = RREG32_SOC15(GC, GET_INST(GC, i),
+				   regGCVM_L2_PROTECTION_FAULT_CNTL_LO32);
+		tmp = REG_SET_FIELD(tmp,
+				    GCVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp,
+				    GCVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp,
+				    GCVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp,
+				    GCVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp,
+				    GCVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    PDE3_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp,
+				    GCVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
+				    value);
+		tmp = REG_SET_FIELD(tmp,
+				    GCVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp,
+				    GCVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp,
+				    GCVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp,
+				    GCVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp,
+				    GCVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp,
+				    GCVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp,
+				    GCVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    CLIENT_ID_NO_RETRY_FAULT_INTERRUPT, value ? 0xFFFF:0);
+		tmp = REG_SET_FIELD(tmp,
+				    GCVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    OTHER_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT, value);
+		if (!value)
+			tmp = REG_SET_FIELD(tmp,
+					    GCVM_L2_PROTECTION_FAULT_CNTL_LO32,
+					    CRASH_ON_NO_RETRY_FAULT, 1);
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGCVM_L2_PROTECTION_FAULT_CNTL_LO32, tmp);
+
+		tmp = RREG32_SOC15(GC, GET_INST(GC, i),
+				   regGCVM_L2_PROTECTION_FAULT_CNTL_HI32);
+		if (!value)
+			tmp = REG_SET_FIELD(tmp,
+					    GCVM_L2_PROTECTION_FAULT_CNTL_HI32,
+					    CRASH_ON_RETRY_FAULT, 1);
+		WREG32_SOC15(GC, GET_INST(GC, i),
+			     regGCVM_L2_PROTECTION_FAULT_CNTL_HI32, tmp);
+	}
+}
+
+/**
+ * gfxhub_v12_1_set_fault_enable_default - update GART/VM fault handling
+ *
+ * @adev: amdgpu_device pointer
+ * @value: true redirects VM faults to the default page
+ */
+static void gfxhub_v12_1_set_fault_enable_default(struct amdgpu_device *adev,
+						  bool value)
+{
+	gfxhub_v12_1_xcc_set_fault_enable_default(adev, value, adev->gfx.xcc_mask);
+}
+
+static uint32_t gfxhub_v12_1_get_invalidate_req(unsigned int vmid,
+						uint32_t flush_type)
+{
+	u32 req = 0;
+
+	/* invalidate using legacy mode on vmid*/
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
+			    PER_VMID_INVALIDATE_REQ, 1 << vmid);
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
+			    FLUSH_TYPE, flush_type);
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
+			    INVALIDATE_L2_PTES, 1);
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
+			    INVALIDATE_L2_PDE0, 1);
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
+			    INVALIDATE_L2_PDE1, 1);
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
+			    INVALIDATE_L2_PDE2, 1);
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
+			    INVALIDATE_L2_PDE3, 1);
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
+			    INVALIDATE_L1_PTES, 1);
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
+			    CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0);
+
+	return req;
+}
+
+static const char *gfxhub_v12_1_client_ids[] = {
+	"CB",
+	"DB",
+	"GE1",
+	"GE2",
+	"CPF",
+	"CPC",
+	"CPG",
+	"RLC",
+	"TCP",
+	"SQC (inst)",
+	"SQC (data)",
+	"SQG/PC/SC",
+	"Reserved",
+	"SDMA0",
+	"SDMA1",
+	"GCR",
+	"Reserved",
+	"Reserved",
+	"WGS",
+	"DSM",
+	"PA"
+};
+
+/*TODO: l2 protection fault status is increased to 64bits.
+ * some critical fields like FED are moved to STATUS_HI32 */
+static void gfxhub_v12_1_print_l2_protection_fault_status(struct amdgpu_device *adev,
+							  uint32_t status)
+{
+	u32 cid = REG_GET_FIELD(status,
+				GCVM_L2_PROTECTION_FAULT_STATUS_LO32,
+				CID);
+
+	dev_err(adev->dev,
+		"GCVM_L2_PROTECTION_FAULT_STATUS_LO32:0x%08X\n",
+		status);
+	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
+		cid >= ARRAY_SIZE(gfxhub_v12_1_client_ids) ?
+		"unknown" : gfxhub_v12_1_client_ids[cid], cid);
+	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
+		REG_GET_FIELD(status,
+		GCVM_L2_PROTECTION_FAULT_STATUS_LO32, MORE_FAULTS));
+	dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
+		REG_GET_FIELD(status,
+		GCVM_L2_PROTECTION_FAULT_STATUS_LO32, WALKER_ERROR));
+	dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
+		REG_GET_FIELD(status,
+		GCVM_L2_PROTECTION_FAULT_STATUS_LO32, PERMISSION_FAULTS));
+	dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
+		REG_GET_FIELD(status,
+		GCVM_L2_PROTECTION_FAULT_STATUS_LO32, MAPPING_ERROR));
+	dev_err(adev->dev, "\t RW: 0x%lx\n",
+		REG_GET_FIELD(status,
+		GCVM_L2_PROTECTION_FAULT_STATUS_LO32, RW));
+}
+
+static const struct amdgpu_vmhub_funcs gfxhub_v12_1_vmhub_funcs = {
+	.print_l2_protection_fault_status = gfxhub_v12_1_print_l2_protection_fault_status,
+	.get_invalidate_req = gfxhub_v12_1_get_invalidate_req,
+};
+
+static void gfxhub_v12_1_xcc_init(struct amdgpu_device *adev, uint32_t xcc_mask)
+{
+	struct amdgpu_vmhub *hub;
+	int i;
+
+	for (i = 0; i < NUM_XCC(xcc_mask); i++) {
+		hub = &adev->vmhub[AMDGPU_GFXHUB(i)];
+
+		hub->ctx0_ptb_addr_lo32 =
+			SOC15_REG_OFFSET(GC, GET_INST(GC, i),
+				regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
+		hub->ctx0_ptb_addr_hi32 =
+			SOC15_REG_OFFSET(GC, GET_INST(GC, i),
+				regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
+		hub->vm_inv_eng0_sem =
+			SOC15_REG_OFFSET(GC, GET_INST(GC, i),
+				regGCVM_INVALIDATE_ENG0_SEM);
+		hub->vm_inv_eng0_req =
+			SOC15_REG_OFFSET(GC, GET_INST(GC, i),
+				regGCVM_INVALIDATE_ENG0_REQ);
+		hub->vm_inv_eng0_ack =
+			SOC15_REG_OFFSET(GC, GET_INST(GC, i),
+				regGCVM_INVALIDATE_ENG0_ACK);
+		hub->vm_context0_cntl =
+			SOC15_REG_OFFSET(GC, GET_INST(GC, i),
+				regGCVM_CONTEXT0_CNTL);
+		/* TODO: add a new member to accomandate additional fault status/cntl reg */
+		hub->vm_l2_pro_fault_status =
+			SOC15_REG_OFFSET(GC, GET_INST(GC, i),
+				regGCVM_L2_PROTECTION_FAULT_STATUS_LO32);
+		hub->vm_l2_pro_fault_cntl =
+			SOC15_REG_OFFSET(GC, GET_INST(GC, i),
+				regGCVM_L2_PROTECTION_FAULT_CNTL_LO32);
+		hub->ctx_distance =
+				regGCVM_CONTEXT1_CNTL -
+				regGCVM_CONTEXT0_CNTL;
+		hub->ctx_addr_distance =
+				regGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 -
+				regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
+		hub->eng_distance =
+				regGCVM_INVALIDATE_ENG1_REQ -
+				regGCVM_INVALIDATE_ENG0_REQ;
+		hub->eng_addr_distance =
+				regGCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 -
+				regGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32;
+
+		hub->vm_cntx_cntl_vm_fault =
+			GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+			GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+			GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+			GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+			GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+			GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+			GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
+
+		hub->vmhub_funcs = &gfxhub_v12_1_vmhub_funcs;
+	}
+}
+
+static void gfxhub_v12_1_init(struct amdgpu_device *adev)
+{
+	gfxhub_v12_1_xcc_init(adev, adev->gfx.xcc_mask);
+}
+
+static int gfxhub_v12_1_get_xgmi_info(struct amdgpu_device *adev)
+{
+	u32 max_num_physical_nodes;
+	u32 max_physical_node_id;
+	u32 xgmi_lfb_cntl;
+	u32 max_region;
+	u64 seg_size;
+
+	xgmi_lfb_cntl = RREG32_SOC15(GC, GET_INST(GC, 0),
+				     regGCMC_VM_XGMI_LFB_CNTL);
+	seg_size = REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, 0),
+				 regGCMC_VM_XGMI_LFB_SIZE),
+				 GCMC_VM_XGMI_LFB_SIZE, PF_LFB_SIZE) << 24;
+	max_region = REG_GET_FIELD(xgmi_lfb_cntl,
+				   GCMC_VM_XGMI_LFB_CNTL,
+				   PF_MAX_REGION);
+
+	max_num_physical_nodes   = 8;
+	max_physical_node_id     = 7;
+
+	/* PF_MAX_REGION=0 means xgmi is disabled */
+	if (max_region || adev->gmc.xgmi.connected_to_cpu) {
+		adev->gmc.xgmi.num_physical_nodes = max_region + 1;
+
+		if (adev->gmc.xgmi.num_physical_nodes > max_num_physical_nodes)
+			return -EINVAL;
+
+		adev->gmc.xgmi.physical_node_id =
+			REG_GET_FIELD(xgmi_lfb_cntl,
+				      GCMC_VM_XGMI_LFB_CNTL,
+				      PF_LFB_REGION);
+
+		if (adev->gmc.xgmi.physical_node_id > max_physical_node_id)
+			return -EINVAL;
+
+		adev->gmc.xgmi.node_segment_size = seg_size;
+	}
+
+	return 0;
+}
+
+const struct amdgpu_gfxhub_funcs gfxhub_v12_1_funcs = {
+	.get_fb_location = gfxhub_v12_1_get_fb_location,
+	.get_mc_fb_offset = gfxhub_v12_1_get_mc_fb_offset,
+	.setup_vm_pt_regs = gfxhub_v12_1_setup_vm_pt_regs,
+	.gart_enable = gfxhub_v12_1_gart_enable,
+	.gart_disable = gfxhub_v12_1_gart_disable,
+	.set_fault_enable_default = gfxhub_v12_1_set_fault_enable_default,
+	.init = gfxhub_v12_1_init,
+	.get_xgmi_info = gfxhub_v12_1_get_xgmi_info,
+};
+
+static int gfxhub_v12_1_xcp_resume(void *handle, uint32_t inst_mask)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	bool value;
+
+	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
+		value = false;
+	else
+		value = true;
+
+	gfxhub_v12_1_xcc_set_fault_enable_default(adev, value, inst_mask);
+
+	if (!amdgpu_sriov_vf(adev))
+		return gfxhub_v12_1_xcc_gart_enable(adev, inst_mask);
+
+	return 0;
+}
+
+static int gfxhub_v12_1_xcp_suspend(void *handle, uint32_t inst_mask)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (!amdgpu_sriov_vf(adev))
+		gfxhub_v12_1_xcc_gart_disable(adev, inst_mask);
+
+	return 0;
+}
+
+struct amdgpu_xcp_ip_funcs gfxhub_v12_1_xcp_funcs = {
+	.suspend = &gfxhub_v12_1_xcp_suspend,
+	.resume = &gfxhub_v12_1_xcp_resume
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.h b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.h
new file mode 100644
index 000000000000..b3f8f0aa0024
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __GFXHUB_V12_1_H__
+#define __GFXHUB_V12_1_H__
+
+extern const struct amdgpu_gfxhub_funcs gfxhub_v12_1_funcs;
+
+extern struct amdgpu_xcp_ip_funcs gfxhub_v12_1_xcp_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
index ce6e04242c52..0b385a15194d 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
@@ -115,27 +115,10 @@ static int gmc_v10_0_process_interrupt(struct amdgpu_device *adev,
 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
 
 	if (retry_fault) {
+		int ret = amdgpu_gmc_handle_retry_fault(adev, entry, addr, 0, 0,
+							write_fault);
 		/* Returning 1 here also prevents sending the IV to the KFD */
-
-		/* Process it onyl if it's the first fault for this address */
-		if (entry->ih != &adev->irq.ih_soft &&
-		    amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
-					     entry->timestamp))
-			return 1;
-
-		/* Delegate it to a different ring if the hardware hasn't
-		 * already done it.
-		 */
-		if (entry->ih == &adev->irq.ih) {
-			amdgpu_irq_delegate(adev, entry, 8);
-			return 1;
-		}
-
-		/* Try to handle the recoverable page faults by filling page
-		 * tables
-		 */
-		if (amdgpu_vm_handle_fault(adev, entry->pasid, 0, 0, addr,
-					   entry->timestamp, write_fault))
+		if (ret == 1)
 			return 1;
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
index ba59ee8e398a..7a1f0742754a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
@@ -114,27 +114,10 @@ static int gmc_v11_0_process_interrupt(struct amdgpu_device *adev,
 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
 
 	if (retry_fault) {
+		int ret = amdgpu_gmc_handle_retry_fault(adev, entry, addr, 0, 0,
+							write_fault);
 		/* Returning 1 here also prevents sending the IV to the KFD */
-
-		/* Process it only if it's the first fault for this address */
-		if (entry->ih != &adev->irq.ih_soft &&
-		    amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
-					     entry->timestamp))
-			return 1;
-
-		/* Delegate it to a different ring if the hardware hasn't
-		 * already done it.
-		 */
-		if (entry->ih == &adev->irq.ih) {
-			amdgpu_irq_delegate(adev, entry, 8);
-			return 1;
-		}
-
-		/* Try to handle the recoverable page faults by filling page
-		 * tables
-		 */
-		if (amdgpu_vm_handle_fault(adev, entry->pasid, 0, 0, addr,
-					   entry->timestamp, write_fault))
+		if (ret == 1)
 			return 1;
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
index 7a9d6894e321..57aabe0dcfdf 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
@@ -28,6 +28,7 @@
 #include "amdgpu.h"
 #include "amdgpu_atomfirmware.h"
 #include "gmc_v12_0.h"
+#include "gmc_v12_1.h"
 #include "athub/athub_4_1_0_sh_mask.h"
 #include "athub/athub_4_1_0_offset.h"
 #include "oss/osssys_7_0_0_offset.h"
@@ -38,7 +39,9 @@
 #include "soc15_common.h"
 #include "nbif_v6_3_1.h"
 #include "gfxhub_v12_0.h"
+#include "gfxhub_v12_1.h"
 #include "mmhub_v4_1_0.h"
+#include "mmhub_v4_2_0.h"
 #include "athub_v4_1_0.h"
 #include "umc_v8_14.h"
 
@@ -107,27 +110,10 @@ static int gmc_v12_0_process_interrupt(struct amdgpu_device *adev,
 		hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
 
 	if (retry_fault) {
+		int ret = amdgpu_gmc_handle_retry_fault(adev, entry, addr, 0, 0,
+							write_fault);
 		/* Returning 1 here also prevents sending the IV to the KFD */
-
-		/* Process it only if it's the first fault for this address */
-		if (entry->ih != &adev->irq.ih_soft &&
-		    amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
-					     entry->timestamp))
-			return 1;
-
-		/* Delegate it to a different ring if the hardware hasn't
-		 * already done it.
-		 */
-		if (entry->ih == &adev->irq.ih) {
-			amdgpu_irq_delegate(adev, entry, 8);
-			return 1;
-		}
-
-		/* Try to handle the recoverable page faults by filling page
-		 * tables
-		 */
-		if (amdgpu_vm_handle_fault(adev, entry->pasid, 0, 0, addr,
-					   entry->timestamp, write_fault))
+		if (ret == 1)
 			return 1;
 	}
 
@@ -623,6 +609,9 @@ static void gmc_v12_0_set_mmhub_funcs(struct amdgpu_device *adev)
 	case IP_VERSION(4, 1, 0):
 		adev->mmhub.funcs = &mmhub_v4_1_0_funcs;
 		break;
+	case IP_VERSION(4, 2, 0):
+		adev->mmhub.funcs = &mmhub_v4_2_0_funcs;
+		break;
 	default:
 		break;
 	}
@@ -635,6 +624,9 @@ static void gmc_v12_0_set_gfxhub_funcs(struct amdgpu_device *adev)
 	case IP_VERSION(12, 0, 1):
 		adev->gfxhub.funcs = &gfxhub_v12_0_funcs;
 		break;
+	case IP_VERSION(12, 1, 0):
+		adev->gfxhub.funcs = &gfxhub_v12_1_funcs;
+		break;
 	default:
 		break;
 	}
@@ -644,10 +636,19 @@ static int gmc_v12_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
+	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
+	case IP_VERSION(12, 1, 0):
+		gmc_v12_1_set_gmc_funcs(adev);
+		gmc_v12_1_set_irq_funcs(adev);
+		adev->gmc.init_pte_flags = AMDGPU_PTE_IS_PTE;
+		break;
+	default:
+		gmc_v12_0_set_gmc_funcs(adev);
+		gmc_v12_0_set_irq_funcs(adev);
+		break;
+	}
 	gmc_v12_0_set_gfxhub_funcs(adev);
 	gmc_v12_0_set_mmhub_funcs(adev);
-	gmc_v12_0_set_gmc_funcs(adev);
-	gmc_v12_0_set_irq_funcs(adev);
 	gmc_v12_0_set_umc_funcs(adev);
 
 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
@@ -656,6 +657,7 @@ static int gmc_v12_0_early_init(struct amdgpu_ip_block *ip_block)
 	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
 	adev->gmc.private_aperture_end =
 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
+	adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
 
 	return 0;
 }
@@ -771,6 +773,7 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 {
 	int r, vram_width = 0, vram_type = 0, vram_vendor = 0;
 	struct amdgpu_device *adev = ip_block->adev;
+	int i;
 
 	adev->mmhub.funcs->init(adev);
 
@@ -778,12 +781,15 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 
 	spin_lock_init(&adev->gmc.invalidate_lock);
 
-	r = amdgpu_atomfirmware_get_vram_info(adev,
-					      &vram_width, &vram_type, &vram_vendor);
-	adev->gmc.vram_width = vram_width;
-
-	adev->gmc.vram_type = vram_type;
-	adev->gmc.vram_vendor = vram_vendor;
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) {
+		gmc_v12_1_init_vram_info(adev);
+	} else {
+		r = amdgpu_atomfirmware_get_vram_info(adev,
+			      &vram_width, &vram_type, &vram_vendor);
+		adev->gmc.vram_width = vram_width;
+		adev->gmc.vram_type = vram_type;
+		adev->gmc.vram_vendor = vram_vendor;
+	}
 
 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 	case IP_VERSION(12, 0, 0):
@@ -797,6 +803,18 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 		 */
 		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
 		break;
+	case IP_VERSION(12, 1, 0):
+		bitmap_set(adev->vmhubs_mask, AMDGPU_GFXHUB(0),
+				NUM_XCC(adev->gfx.xcc_mask));
+		for (i = 0; i < hweight32(adev->aid_mask); i++)
+			set_bit(AMDGPU_MMHUB0(i), adev->vmhubs_mask);
+		/*
+		 * To fulfill 5-level page support,
+		 * vm size is 128PetaByte (57bit), maximum size,
+		 * block size 512 (9bit)
+		 */
+		amdgpu_vm_adjust_size(adev, 128 * 1024 * 1024, 9, 4, 57);
+		break;
 	default:
 		break;
 	}
@@ -809,13 +827,33 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		return r;
 
-	r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
-			      UTCL2_1_0__SRCID__FAULT,
-			      &adev->gmc.vm_fault);
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) {
+		r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_UTCL2,
+				      UTCL2_1_0__SRCID__FAULT,
+				      &adev->gmc.vm_fault);
+		if (r)
+			return r;
+		/* Add GCVM UTCL2 Retry fault */
+		r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_UTCL2,
+				      UTCL2_1_0__SRCID__RETRY,
+				      &adev->gmc.vm_fault);
+		if (r)
+			return r;
+
+		/* Add MMVM UTCL2 Retry fault */
+		r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_VMC,
+				      VMC_1_0__SRCID__VM_RETRY,
+				      &adev->gmc.vm_fault);
+	} else {
+		r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
+				      UTCL2_1_0__SRCID__FAULT,
+				      &adev->gmc.vm_fault);
+	}
 	if (r)
 		return r;
 
-	if (!amdgpu_sriov_vf(adev)) {
+	if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(12, 1, 0)) &&
+	    !amdgpu_sriov_vf(adev)) {
 		/* interrupt sent to DF. */
 		r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_DF, 0,
 				      &adev->gmc.ecc_irq);
@@ -827,7 +865,7 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 	 * Set the internal MC address mask This is the max address of the GPU's
 	 * internal address space.
 	 */
-	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
+	adev->gmc.mc_mask = AMDGPU_GMC_HOLE_MASK;
 
 	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44));
 	if (r) {
@@ -843,6 +881,14 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 
 	amdgpu_gmc_get_vbios_allocations(adev);
 
+#ifdef HAVE_ACPI_DEV_GET_FIRST_MATCH_DEV
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) {
+		r = amdgpu_gmc_init_mem_ranges(adev);
+		if (r)
+			return r;
+	}
+#endif
+
 	/* Memory manager */
 	r = amdgpu_bo_init(adev);
 	if (r)
@@ -858,7 +904,11 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 	 * amdgpu graphics/compute will use VMIDs 1-7
 	 * amdkfd will use VMIDs 8-15
 	 */
-	adev->vm_manager.first_kfd_vmid = adev->gfx.disable_kq ? 1 : 8;
+	adev->vm_manager.first_kfd_vmid =
+		 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) ?
+		3 : 8;
+	adev->vm_manager.first_kfd_vmid =
+		adev->gfx.disable_kq ? 1 : (adev->vm_manager.first_kfd_vmid);
 
 	amdgpu_vm_manager_init(adev);
 
@@ -924,7 +974,7 @@ static int gmc_v12_0_gart_enable(struct amdgpu_device *adev)
 	value = amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS;
 
 	adev->mmhub.funcs->set_fault_enable_default(adev, value);
-	gmc_v12_0_flush_gpu_tlb(adev, 0, AMDGPU_MMHUB0(0), 0);
+	adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_MMHUB0(0), 0);
 
 	dev_info(adev->dev, "PCIE GART of %uM enabled (table at 0x%016llX).\n",
 		 (unsigned)(adev->gmc.gart_size >> 20),
@@ -1026,7 +1076,10 @@ static int gmc_v12_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
 	if (r)
 		return r;
 
-	return athub_v4_1_0_set_clockgating(adev, state);
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(12, 1, 0))
+		return athub_v4_1_0_set_clockgating(adev, state);
+	else
+		return 0;
 }
 
 static void gmc_v12_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
@@ -1035,7 +1088,8 @@ static void gmc_v12_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u6
 
 	adev->mmhub.funcs->get_clockgating(adev, flags);
 
-	athub_v4_1_0_get_clockgating(adev, flags);
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(12, 1, 0))
+		athub_v4_1_0_get_clockgating(adev, flags);
 }
 
 static int gmc_v12_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
new file mode 100644
index 000000000000..061d1be72340
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
@@ -0,0 +1,603 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "gmc_v12_1.h"
+#include "soc15_common.h"
+#include "soc_v1_0_enum.h"
+#include "oss/osssys_7_1_0_offset.h"
+#include "oss/osssys_7_1_0_sh_mask.h"
+#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
+
+static int gmc_v12_1_vm_fault_interrupt_state(struct amdgpu_device *adev,
+					      struct amdgpu_irq_src *src,
+					      unsigned int type,
+					      enum amdgpu_interrupt_state state)
+{
+	struct amdgpu_vmhub *hub;
+	u32 tmp, reg, i, j;
+
+	switch (state) {
+	case AMDGPU_IRQ_STATE_DISABLE:
+		for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
+			hub = &adev->vmhub[j];
+			for (i = 0; i < 16; i++) {
+				reg = hub->vm_context0_cntl + i;
+
+				/* This works because this interrupt is only
+				 * enabled at init/resume and disabled in
+				 * fini/suspend, so the overall state doesn't
+				 * change over the course of suspend/resume.
+				 */
+				if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
+					continue;
+
+				if (j >= AMDGPU_MMHUB0(0))
+					tmp = RREG32_SOC15_IP(MMHUB, reg);
+				else
+					tmp = RREG32_XCC(reg, j);
+
+				tmp &= ~hub->vm_cntx_cntl_vm_fault;
+
+				if (j >= AMDGPU_MMHUB0(0))
+					WREG32_SOC15_IP(MMHUB, reg, tmp);
+				else
+					WREG32_XCC(reg, tmp, j);
+			}
+		}
+		break;
+	case AMDGPU_IRQ_STATE_ENABLE:
+		for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
+			hub = &adev->vmhub[j];
+			for (i = 0; i < 16; i++) {
+				reg = hub->vm_context0_cntl + i;
+
+				/* This works because this interrupt is only
+				 * enabled at init/resume and disabled in
+				 * fini/suspend, so the overall state doesn't
+				 * change over the course of suspend/resume.
+				 */
+				if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
+					continue;
+
+				if (j >= AMDGPU_MMHUB0(0))
+					tmp = RREG32_SOC15_IP(MMHUB, reg);
+				else
+					tmp = RREG32_XCC(reg, j);
+
+				tmp |= hub->vm_cntx_cntl_vm_fault;
+
+				if (j >= AMDGPU_MMHUB0(0))
+					WREG32_SOC15_IP(MMHUB, reg, tmp);
+				else
+					WREG32_XCC(reg, tmp, j);
+			}
+		}
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *source,
+				       struct amdgpu_iv_entry *entry)
+{
+	struct amdgpu_task_info *task_info;
+	bool retry_fault = false, write_fault = false;
+	unsigned int vmhub, node_id;
+	struct amdgpu_vmhub *hub;
+	uint32_t cam_index = 0;
+	const char *hub_name;
+	int ret, xcc_id = 0;
+	uint32_t status = 0;
+	u64 addr;
+
+	node_id = entry->node_id;
+
+	addr = (u64)entry->src_data[0] << 12;
+	addr |= ((u64)entry->src_data[1] & 0x1fff) << 44;
+
+	if (entry->src_id == UTCL2_1_0__SRCID__RETRY) {
+		retry_fault = true;
+		write_fault = !!(entry->src_data[1] & 0x200000);
+	}
+
+	if (entry->client_id == SOC_V1_0_IH_CLIENTID_VMC) {
+		hub_name = "mmhub0";
+		vmhub = AMDGPU_MMHUB0(node_id / 4);
+	} else {
+		hub_name = "gfxhub0";
+		if (adev->gfx.funcs->ih_node_to_logical_xcc) {
+			xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev,
+								node_id);
+			if (xcc_id < 0)
+				xcc_id = 0;
+		}
+		vmhub = xcc_id;
+	}
+
+	hub = &adev->vmhub[vmhub];
+
+	if (retry_fault) {
+		if (adev->irq.retry_cam_enabled) {
+			/* Delegate it to a different ring if the hardware hasn't
+			 * already done it.
+			 */
+			if (entry->ih == &adev->irq.ih) {
+				amdgpu_irq_delegate(adev, entry, 8);
+				return 1;
+			}
+
+			cam_index = entry->src_data[3] & 0x3ff;
+
+			ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id,
+							addr, entry->timestamp, write_fault);
+			WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index);
+			if (ret)
+				return 1;
+		} else {
+			/* Process it onyl if it's the first fault for this address */
+			if (entry->ih != &adev->irq.ih_soft &&
+				amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
+							 entry->timestamp))
+				return 1;
+
+			/* Delegate it to a different ring if the hardware hasn't
+			 * already done it.
+			 */
+			if (entry->ih == &adev->irq.ih) {
+				amdgpu_irq_delegate(adev, entry, 8);
+				return 1;
+			}
+
+			/* Try to handle the recoverable page faults by filling page
+			 * tables
+			 */
+			if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id,
+						   addr, entry->timestamp, write_fault))
+				return 1;
+		}
+	}
+
+	if (kgd2kfd_vmfault_fast_path(adev, entry, retry_fault))
+		return 1;
+
+	if (!printk_ratelimit())
+		return 0;
+
+	dev_err(adev->dev,
+		"[%s] %s page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n", hub_name,
+		retry_fault ? "retry" : "no-retry",
+		entry->src_id, entry->ring_id, entry->vmid, entry->pasid);
+
+	task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid);
+	if (task_info) {
+		amdgpu_vm_print_task_info(adev, task_info);
+		amdgpu_vm_put_task_info(task_info);
+	}
+
+	dev_err(adev->dev, "  in page starting at address 0x%016llx from IH client %d (%s)\n",
+		addr, entry->client_id, soc_v1_0_ih_clientid_name[entry->client_id]);
+
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
+	/*
+	 * Issue a dummy read to wait for the status register to
+	 * be updated to avoid reading an incorrect value due to
+	 * the new fast GRBM interface.
+	 */
+	if (entry->vmid_src == AMDGPU_GFXHUB(0))
+		RREG32(hub->vm_l2_pro_fault_status);
+
+	status = RREG32(hub->vm_l2_pro_fault_status);
+
+	/* Only print L2 fault status if the status register could be read and
+	 * contains useful information
+	 */
+	if (!status)
+		return 0;
+
+	WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
+
+	amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status, vmhub);
+
+	hub->vmhub_funcs->print_l2_protection_fault_status(adev, status);
+
+	return 0;
+}
+
+static bool gmc_v12_1_get_vmid_pasid_mapping_info(struct amdgpu_device *adev,
+						  uint8_t vmid, uint16_t *p_pasid)
+{
+	*p_pasid = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid) & 0xffff;
+
+	return !!(*p_pasid);
+}
+
+/*
+ * GART
+ * VMID 0 is the physical GPU addresses as used by the kernel.
+ * VMIDs 1-15 are used for userspace clients and are handled
+ * by the amdgpu vm/hsa code.
+ */
+
+static void gmc_v12_1_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid,
+				   unsigned int vmhub, uint32_t flush_type)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
+	u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
+	u32 tmp;
+	/* Use register 17 for GART */
+	const unsigned eng = 17;
+	unsigned int i;
+	unsigned char hub_ip = 0;
+
+	hub_ip = (AMDGPU_IS_GFXHUB(vmhub)) ?
+		   GC_HWIP : MMHUB_HWIP;
+
+	spin_lock(&adev->gmc.invalidate_lock);
+
+	WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req, hub_ip);
+
+	/* Wait for ACK with a delay.*/
+	for (i = 0; i < adev->usec_timeout; i++) {
+		tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_ack +
+				    hub->eng_distance * eng, hub_ip);
+		tmp &= 1 << vmid;
+		if (tmp)
+			break;
+
+		udelay(1);
+	}
+
+	/* Issue additional private vm invalidation to MMHUB */
+	if (!AMDGPU_IS_GFXHUB(vmhub) &&
+	    (hub->vm_l2_bank_select_reserved_cid2) &&
+		!amdgpu_sriov_vf(adev)) {
+		inv_req = RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
+		/* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */
+		inv_req |= (1 << 25);
+		/* Issue private invalidation */
+		WREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2, inv_req);
+		/* Read back to ensure invalidation is done*/
+		RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
+	}
+
+	spin_unlock(&adev->gmc.invalidate_lock);
+
+	if (i < adev->usec_timeout)
+		return;
+
+	dev_err(adev->dev, "Timeout waiting for VM flush ACK!\n");
+}
+
+/**
+ * gmc_v12_1_flush_gpu_tlb - gart tlb flush callback
+ *
+ * @adev: amdgpu_device pointer
+ * @vmid: vm instance to flush
+ * @vmhub: which hub to flush
+ * @flush_type: the flush type
+ *
+ * Flush the TLB for the requested page table.
+ */
+static void gmc_v12_1_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
+				    uint32_t vmhub, uint32_t flush_type)
+{
+	/* This is necessary for SRIOV as well as for GFXOFF to function
+	 * properly under bare metal
+	 */
+	if (((adev->gfx.kiq[0].ring.sched.ready || adev->mes.ring[0].sched.ready) &&
+	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)))) {
+		struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
+		const unsigned eng = 17;
+		u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
+		u32 req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
+		u32 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
+
+		amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req,
+				1 << vmid, 0);
+		return;
+	}
+
+	mutex_lock(&adev->mman.gtt_window_lock);
+	gmc_v12_1_flush_vm_hub(adev, vmid, vmhub, 0);
+	mutex_unlock(&adev->mman.gtt_window_lock);
+	return;
+}
+
+/**
+ * gmc_v12_1_flush_gpu_tlb_pasid - tlb flush via pasid
+ *
+ * @adev: amdgpu_device pointer
+ * @pasid: pasid to be flush
+ * @flush_type: the flush type
+ * @all_hub: flush all hubs
+ * @inst: is used to select which instance of KIQ to use for the invalidation
+ *
+ * Flush the TLB for the requested pasid.
+ */
+static void gmc_v12_1_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
+					  uint16_t pasid, uint32_t flush_type,
+					  bool all_hub, uint32_t inst)
+{
+	uint16_t queried;
+	int vmid, i;
+
+	for (vmid = 1; vmid < 16; vmid++) {
+		bool valid;
+
+		valid = gmc_v12_1_get_vmid_pasid_mapping_info(adev, vmid,
+							      &queried);
+		if (!valid || queried != pasid)
+			continue;
+
+		if (all_hub) {
+			for_each_set_bit(i, adev->vmhubs_mask,
+					 AMDGPU_MAX_VMHUBS)
+				gmc_v12_1_flush_gpu_tlb(adev, vmid, i,
+							flush_type);
+		} else {
+			gmc_v12_1_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(inst),
+						flush_type);
+		}
+	}
+}
+
+static uint64_t gmc_v12_1_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
+					     unsigned vmid, uint64_t pd_addr)
+{
+	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
+	uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
+	unsigned eng = ring->vm_inv_eng;
+
+	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
+			      (hub->ctx_addr_distance * vmid),
+			      lower_32_bits(pd_addr));
+
+	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
+			      (hub->ctx_addr_distance * vmid),
+			      upper_32_bits(pd_addr));
+
+	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
+					    hub->eng_distance * eng,
+					    hub->vm_inv_eng0_ack +
+					    hub->eng_distance * eng,
+					    req, 1 << vmid);
+
+	return pd_addr;
+}
+
+static void gmc_v12_1_emit_pasid_mapping(struct amdgpu_ring *ring,
+					 unsigned vmid, unsigned pasid)
+{
+	struct amdgpu_device *adev = ring->adev;
+	uint32_t reg;
+
+	if (ring->vm_hub == AMDGPU_GFXHUB(0))
+		reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid;
+	else
+		reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT_MM) + vmid;
+
+	amdgpu_ring_emit_wreg(ring, reg, pasid);
+}
+
+/*
+ * PTE format:
+ * 63 P
+ * 62:59 reserved
+ * 58 D
+ * 57 G
+ * 56 T
+ * 55:54 M
+ * 53:52 SW
+ * 51:48 reserved for future
+ * 47:12 4k physical page base address
+ * 11:7 fragment
+ * 6 write
+ * 5 read
+ * 4 exe
+ * 3 Z
+ * 2 snooped
+ * 1 system
+ * 0 valid
+ *
+ * PDE format:
+ * 63 P
+ * 62:58 block fragment size
+ * 57 reserved
+ * 56 A
+ * 55:54 M
+ * 53:52 reserved
+ * 51:48 reserved for future
+ * 47:6 physical base address of PD or PTE
+ * 5:3 reserved
+ * 2 C
+ * 1 system
+ * 0 valid
+ */
+
+static void gmc_v12_1_get_vm_pde(struct amdgpu_device *adev, int level,
+				 uint64_t *addr, uint64_t *flags)
+{
+	if (!(*flags & AMDGPU_PDE_PTE_GFX12) && !(*flags & AMDGPU_PTE_SYSTEM))
+		*addr = adev->vm_manager.vram_base_offset + *addr -
+			adev->gmc.vram_start;
+	BUG_ON(*addr & 0xFFFF00000000003FULL);
+
+	*flags |= AMDGPU_PTE_SNOOPED;
+
+	if (!adev->gmc.translate_further)
+		return;
+
+	if (level == AMDGPU_VM_PDB1) {
+		/* Set the block fragment size */
+		if (!(*flags & AMDGPU_PDE_PTE_GFX12))
+			*flags |= AMDGPU_PDE_BFS_GFX12(0x9);
+
+	} else if (level == AMDGPU_VM_PDB0) {
+		if (*flags & AMDGPU_PDE_PTE_GFX12)
+			*flags &= ~AMDGPU_PDE_PTE_GFX12;
+	}
+}
+
+#if 0
+static void gmc_v12_1_get_coherence_flags(struct amdgpu_device *adev,
+					  struct amdgpu_bo *bo,
+					  uint64_t *flags)
+{
+	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
+	bool is_vram = bo->tbo.resource &&
+		       bo->tbo.resource->mem_type == TTM_PL_VRAM;
+	bool coherent = bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
+				     AMDGPU_GEM_CREATE_EXT_COHERENT);
+	bool ext_coherent = bo->flags & AMDGPU_GEM_CREATE_EXT_COHERENT;
+	uint32_t gc_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0);
+	bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED;
+	unsigned int mtype, mtype_local;
+	bool snoop = false;
+	bool is_local;
+
+	switch (gc_ip_version) {
+	case IP_VERSION(12, 1, 0):
+		mtype_local = MTYPE_RW;
+		if (amdgpu_mtype_local == 1) {
+			DRM_INFO_ONCE("Using MTYPE_NC for local memory\n");
+			mtype_local = MTYPE_NC;
+		} else if (amdgpu_mtype_local == 2) {
+			DRM_INFO_ONCE("MTYPE_CC not supported, using MTYPE_RW instead for local memory\n");
+		} else {
+			DRM_INFO_ONCE("Using MTYPE_RW for local memory\n");
+		}
+
+		is_local = (is_vram && adev == bo_adev);
+		snoop = true;
+		if (uncached) {
+			mtype = MTYPE_UC;
+		} else if (ext_coherent) {
+			mtype = is_local ? mtype_local : MTYPE_UC;
+		} else {
+			if (is_local)
+				mtype = mtype_local;
+			else
+				mtype = MTYPE_NC;
+		}
+		break;
+	default:
+		if (uncached || coherent)
+			mtype = MTYPE_UC;
+		else
+			mtype = MTYPE_NC;
+	}
+
+	if (mtype != MTYPE_NC)
+		*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, mtype);
+
+	*flags |= snoop ? AMDGPU_PTE_SNOOPED : 0;
+}
+#endif
+
+static void gmc_v12_1_get_vm_pte(struct amdgpu_device *adev,
+				 struct amdgpu_vm *vm,
+				 struct amdgpu_bo *bo,
+				 uint32_t vm_flags,
+				 uint64_t *flags)
+{
+	if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE)
+		*flags |= AMDGPU_PTE_EXECUTABLE;
+	else
+		*flags &= ~AMDGPU_PTE_EXECUTABLE;
+
+	switch (vm_flags & AMDGPU_VM_MTYPE_MASK) {
+	case AMDGPU_VM_MTYPE_DEFAULT:
+		*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC);
+		break;
+	case AMDGPU_VM_MTYPE_NC:
+	default:
+		*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC);
+		break;
+	case AMDGPU_VM_MTYPE_UC:
+		*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_UC);
+		break;
+	}
+
+	if (vm_flags & AMDGPU_VM_PAGE_NOALLOC)
+		*flags |= AMDGPU_PTE_NOALLOC;
+	else
+		*flags &= ~AMDGPU_PTE_NOALLOC;
+
+	if (vm_flags & AMDGPU_VM_PAGE_PRT) {
+		*flags |= AMDGPU_PTE_SNOOPED;
+		*flags |= AMDGPU_PTE_SYSTEM;
+		*flags |= AMDGPU_PTE_IS_PTE;
+		*flags &= ~AMDGPU_PTE_VALID;
+	}
+
+	if (bo && bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
+			       AMDGPU_GEM_CREATE_EXT_COHERENT |
+			       AMDGPU_GEM_CREATE_UNCACHED))
+		*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_UC);
+
+	if (adev->have_atomics_support)
+		*flags |= AMDGPU_PTE_BUS_ATOMICS;
+
+	if (bo && bo->flags & AMDGPU_GEM_CREATE_UNCACHED)
+		*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_UC);
+}
+
+static const struct amdgpu_gmc_funcs gmc_v12_1_gmc_funcs = {
+	.flush_gpu_tlb = gmc_v12_1_flush_gpu_tlb,
+	.flush_gpu_tlb_pasid = gmc_v12_1_flush_gpu_tlb_pasid,
+	.emit_flush_gpu_tlb = gmc_v12_1_emit_flush_gpu_tlb,
+	.emit_pasid_mapping = gmc_v12_1_emit_pasid_mapping,
+	.get_vm_pde = gmc_v12_1_get_vm_pde,
+	.get_vm_pte = gmc_v12_1_get_vm_pte,
+	.query_mem_partition_mode = &amdgpu_gmc_query_memory_partition,
+	.request_mem_partition_mode = &amdgpu_gmc_request_memory_partition,
+};
+
+void gmc_v12_1_set_gmc_funcs(struct amdgpu_device *adev)
+{
+	adev->gmc.gmc_funcs = &gmc_v12_1_gmc_funcs;
+}
+
+static const struct amdgpu_irq_src_funcs gmc_v12_1_irq_funcs = {
+	.set = gmc_v12_1_vm_fault_interrupt_state,
+	.process = gmc_v12_1_process_interrupt,
+};
+
+void gmc_v12_1_set_irq_funcs(struct amdgpu_device *adev)
+{
+	adev->gmc.vm_fault.num_types = 1;
+	adev->gmc.vm_fault.funcs = &gmc_v12_1_irq_funcs;
+}
+
+void gmc_v12_1_init_vram_info(struct amdgpu_device *adev)
+{
+	/* TODO: query vram_info from ip discovery binary */
+	adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM4;
+	adev->gmc.vram_width = 384 * 64;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.h b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.h
new file mode 100644
index 000000000000..2f08f4b8cd99
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __GMC_V12_1_H__
+#define __GMC_V12_1_H__
+
+void gmc_v12_1_set_gmc_funcs(struct amdgpu_device *adev);
+void gmc_v12_1_set_irq_funcs(struct amdgpu_device *adev);
+void gmc_v12_1_init_vram_info(struct amdgpu_device *adev);
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 8ad7519f7b58..97a04e3171f2 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -583,44 +583,13 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
 	hub = &adev->vmhub[vmhub];
 
 	if (retry_fault) {
-		if (adev->irq.retry_cam_enabled) {
-			/* Delegate it to a different ring if the hardware hasn't
-			 * already done it.
-			 */
-			if (entry->ih == &adev->irq.ih) {
-				amdgpu_irq_delegate(adev, entry, 8);
-				return 1;
-			}
-
-			cam_index = entry->src_data[2] & 0x3ff;
+		cam_index = entry->src_data[2] & 0x3ff;
 
-			ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id,
-						     addr, entry->timestamp, write_fault);
-			WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index);
-			if (ret)
-				return 1;
-		} else {
-			/* Process it onyl if it's the first fault for this address */
-			if (entry->ih != &adev->irq.ih_soft &&
-			    amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
-					     entry->timestamp))
-				return 1;
-
-			/* Delegate it to a different ring if the hardware hasn't
-			 * already done it.
-			 */
-			if (entry->ih == &adev->irq.ih) {
-				amdgpu_irq_delegate(adev, entry, 8);
-				return 1;
-			}
-
-			/* Try to handle the recoverable page faults by filling page
-			 * tables
-			 */
-			if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id,
-						   addr, entry->timestamp, write_fault))
-				return 1;
-		}
+		ret = amdgpu_gmc_handle_retry_fault(adev, entry, addr, cam_index, node_id,
+						    write_fault);
+		/* Returning 1 here also prevents sending the IV to the KFD */
+		if (ret == 1)
+			return 1;
 	}
 
 	if (kgd2kfd_vmfault_fast_path(adev, entry, retry_fault))
diff --git a/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c b/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c
index b32ea4129c61..451828bf583e 100644
--- a/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c
@@ -279,6 +279,16 @@ static int ih_v7_0_enable_ring(struct amdgpu_device *adev,
 	return 0;
 }
 
+static uint32_t ih_v7_0_setup_retry_doorbell(u32 doorbell_index)
+{
+	u32 val = 0;
+
+	val = REG_SET_FIELD(val, IH_DOORBELL_RPTR, OFFSET, doorbell_index);
+	val = REG_SET_FIELD(val, IH_DOORBELL_RPTR, ENABLE, 1);
+
+	return val;
+}
+
 /**
  * ih_v7_0_irq_init - init and enable the interrupt ring
  *
@@ -363,6 +373,21 @@ static int ih_v7_0_irq_init(struct amdgpu_device *adev)
 
 	pci_set_master(adev->pdev);
 
+	if (amdgpu_ip_version(adev, OSSSYS_HWIP, 0) == IP_VERSION(7, 1, 0)) {
+		/* Allocate the doorbell for IH Retry CAM */
+		adev->irq.retry_cam_doorbell_index = (adev->doorbell_index.ih + 2) << 1;
+		WREG32_SOC15(OSSSYS, 0, regIH_DOORBELL_RETRY_CAM,
+				ih_v7_0_setup_retry_doorbell(adev->irq.retry_cam_doorbell_index));
+
+		/* Enable IH Retry CAM */
+		tmp = RREG32_SOC15(OSSSYS, 0, regIH_RETRY_INT_CAM_CNTL);
+		tmp = REG_SET_FIELD(tmp, IH_RETRY_INT_CAM_CNTL, ENABLE, 1);
+		tmp = REG_SET_FIELD(tmp, IH_RETRY_INT_CAM_CNTL, CAM_SIZE, 0xF);
+		WREG32_SOC15(OSSSYS, 0, regIH_RETRY_INT_CAM_CNTL, tmp);
+
+		adev->irq.retry_cam_enabled = true;
+	}
+
 	/* enable interrupts */
 	ret = ih_v7_0_toggle_interrupts(adev, true);
 	if (ret)
@@ -542,6 +567,7 @@ static int ih_v7_0_sw_init(struct amdgpu_ip_block *ip_block)
 	int r;
 	struct amdgpu_device *adev = ip_block->adev;
 	bool use_bus_addr;
+	unsigned int sw_ring_size;
 
 	r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_IH, 0,
 			      &adev->irq.self_irq);
@@ -573,7 +599,9 @@ static int ih_v7_0_sw_init(struct amdgpu_ip_block *ip_block)
 	/* initialize ih control register offset */
 	ih_v7_0_init_register_offset(adev);
 
-	r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, PAGE_SIZE, true);
+	sw_ring_size = (amdgpu_ip_version(adev, OSSSYS_HWIP, 0) == IP_VERSION(7, 1, 0)) ?
+			IH_SW_RING_SIZE : PAGE_SIZE;
+	r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, sw_ring_size, true);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/imu_v12_1.c b/drivers/gpu/drm/amd/amdgpu/imu_v12_1.c
new file mode 100644
index 000000000000..8e33ffcdfa4c
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/imu_v12_1.c
@@ -0,0 +1,160 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/firmware.h>
+#include "amdgpu.h"
+#include "amdgpu_imu.h"
+#include "amdgpu_dpm.h"
+
+#include "imu_v12_1.h"
+
+#include "gc/gc_12_1_0_offset.h"
+#include "gc/gc_12_1_0_sh_mask.h"
+#include "mmhub/mmhub_4_2_0_offset.h"
+
+MODULE_FIRMWARE("amdgpu/gc_12_1_0_imu.bin");
+
+#define TRANSFER_RAM_MASK	0x001c0000
+
+static int imu_v12_1_init_microcode(struct amdgpu_device *adev)
+{
+	char ucode_prefix[15];
+	int err;
+	const struct imu_firmware_header_v1_0 *imu_hdr;
+	struct amdgpu_firmware_info *info = NULL;
+
+	DRM_DEBUG("\n");
+
+	amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
+	err = amdgpu_ucode_request(adev, &adev->gfx.imu_fw, AMDGPU_UCODE_REQUIRED,
+				   "amdgpu/%s_imu.bin", ucode_prefix);
+	if (err)
+		goto out;
+
+	imu_hdr = (const struct imu_firmware_header_v1_0 *)adev->gfx.imu_fw->data;
+	adev->gfx.imu_fw_version = le32_to_cpu(imu_hdr->header.ucode_version);
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_IMU_I];
+		info->ucode_id = AMDGPU_UCODE_ID_IMU_I;
+		info->fw = adev->gfx.imu_fw;
+		adev->firmware.fw_size +=
+			ALIGN(le32_to_cpu(imu_hdr->imu_iram_ucode_size_bytes), PAGE_SIZE);
+		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_IMU_D];
+		info->ucode_id = AMDGPU_UCODE_ID_IMU_D;
+		info->fw = adev->gfx.imu_fw;
+		adev->firmware.fw_size +=
+			ALIGN(le32_to_cpu(imu_hdr->imu_dram_ucode_size_bytes), PAGE_SIZE);
+	}
+
+out:
+	if (err) {
+		dev_err(adev->dev,
+			"gfx12: Failed to load firmware \"%s_imu.bin\"\n",
+			ucode_prefix);
+		amdgpu_ucode_release(&adev->gfx.imu_fw);
+	}
+
+	return err;
+}
+
+static void imu_v12_1_xcc_load_microcode(struct amdgpu_device *adev,
+					 int xcc_id)
+{
+	const struct imu_firmware_header_v1_0 *hdr;
+	const __le32 *fw_data;
+	unsigned i, fw_size;
+
+	hdr = (const struct imu_firmware_header_v1_0 *)adev->gfx.imu_fw->data;
+	fw_data = (const __le32 *)(adev->gfx.imu_fw->data +
+			le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+	fw_size = le32_to_cpu(hdr->imu_iram_ucode_size_bytes) / 4;
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regGFX_IMU_I_RAM_ADDR, 0);
+
+	for (i = 0; i < fw_size; i++)
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+			     regGFX_IMU_I_RAM_DATA,
+			     le32_to_cpup(fw_data++));
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+		     regGFX_IMU_I_RAM_ADDR,
+		     adev->gfx.imu_fw_version);
+
+	fw_data = (const __le32 *)(adev->gfx.imu_fw->data +
+			le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
+			le32_to_cpu(hdr->imu_iram_ucode_size_bytes));
+	fw_size = le32_to_cpu(hdr->imu_dram_ucode_size_bytes) / 4;
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regGFX_IMU_D_RAM_ADDR, 0);
+
+	for (i = 0; i < fw_size; i++)
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+			     regGFX_IMU_D_RAM_DATA,
+			     le32_to_cpup(fw_data++));
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+		     regGFX_IMU_D_RAM_ADDR,
+		     adev->gfx.imu_fw_version);
+}
+
+static int imu_v12_1_load_microcode(struct amdgpu_device *adev)
+{
+	int i, num_xcc;
+
+	if (!adev->gfx.imu_fw)
+		return -EINVAL;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < num_xcc; i++) {
+		imu_v12_1_xcc_load_microcode(adev, i);
+	}
+
+	return 0;
+}
+
+static int imu_v12_1_switch_compute_partition(struct amdgpu_device *adev,
+					      int num_xccs_per_xcp,
+					      int compute_partition_mode)
+{
+	int ret;
+
+	if (adev->psp.funcs) {
+		/*TODO: revisit asp interface once it's avaialble */
+		ret = psp_spatial_partition(&adev->psp,
+					    NUM_XCC(adev->gfx.xcc_mask) /
+						    num_xccs_per_xcp);
+		if (ret)
+			return ret;
+	}
+
+	adev->gfx.num_xcc_per_xcp = num_xccs_per_xcp;
+
+	return 0;
+}
+
+const struct amdgpu_imu_funcs gfx_v12_1_imu_funcs = {
+	.init_microcode = imu_v12_1_init_microcode,
+	.load_microcode = imu_v12_1_load_microcode,
+	.switch_compute_partition = imu_v12_1_switch_compute_partition,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/imu_v12_1.h b/drivers/gpu/drm/amd/amdgpu/imu_v12_1.h
new file mode 100644
index 000000000000..4a7c12bf7b0f
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/imu_v12_1.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __IMU_V12_1_H__
+#define __IMU_V12_1_H__
+
+extern const struct amdgpu_imu_funcs gfx_v12_1_imu_funcs;
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
index 64cae89357b6..f2309d72bbe6 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
@@ -59,7 +59,8 @@ mes_userq_map_gtt_bo_to_gart(struct amdgpu_bo *bo)
 }
 
 static int
-mes_userq_create_wptr_mapping(struct amdgpu_userq_mgr *uq_mgr,
+mes_userq_create_wptr_mapping(struct amdgpu_device *adev,
+			      struct amdgpu_userq_mgr *uq_mgr,
 			      struct amdgpu_usermode_queue *queue,
 			      uint64_t wptr)
 {
@@ -93,8 +94,28 @@ mes_userq_create_wptr_mapping(struct amdgpu_userq_mgr *uq_mgr,
 		return ret;
 	}
 
-	queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset_no_check(wptr_obj->obj);
+	ret = amdgpu_bo_reserve(wptr_obj->obj, true);
+	if (ret) {
+		DRM_ERROR("Failed to reserve wptr bo\n");
+		return ret;
+	}
+
+	/* TODO use eviction fence instead of pinning. */
+	ret = amdgpu_bo_pin(wptr_obj->obj, AMDGPU_GEM_DOMAIN_GTT);
+	if (ret) {
+		drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin wptr bo\n");
+		goto unresv_bo;
+	}
+
+	queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(wptr_obj->obj);
+	amdgpu_bo_unreserve(wptr_obj->obj);
+
 	return 0;
+
+unresv_bo:
+	amdgpu_bo_unreserve(wptr_obj->obj);
+	return ret;
+
 }
 
 static int convert_to_mes_priority(int priority)
@@ -112,9 +133,9 @@ static int convert_to_mes_priority(int priority)
 	}
 }
 
-static int mes_userq_map(struct amdgpu_userq_mgr *uq_mgr,
-			 struct amdgpu_usermode_queue *queue)
+static int mes_userq_map(struct amdgpu_usermode_queue *queue)
 {
+	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
 	struct amdgpu_device *adev = uq_mgr->adev;
 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
 	struct amdgpu_mqd_prop *userq_props = queue->userq_prop;
@@ -157,9 +178,9 @@ static int mes_userq_map(struct amdgpu_userq_mgr *uq_mgr,
 	return 0;
 }
 
-static int mes_userq_unmap(struct amdgpu_userq_mgr *uq_mgr,
-			   struct amdgpu_usermode_queue *queue)
+static int mes_userq_unmap(struct amdgpu_usermode_queue *queue)
 {
+	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
 	struct amdgpu_device *adev = uq_mgr->adev;
 	struct mes_remove_queue_input queue_input;
 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
@@ -223,7 +244,7 @@ static int mes_userq_detect_and_reset(struct amdgpu_device *adev,
 
 	amdgpu_mes_lock(&adev->mes);
 	r = amdgpu_mes_detect_and_reset_hung_queues(adev, queue_type, false,
-						    &hung_db_num, db_array);
+						    &hung_db_num, db_array, 0);
 	amdgpu_mes_unlock(&adev->mes);
 	if (r) {
 		dev_err(adev->dev, "Failed to detect and reset queues, err (%d)\n", r);
@@ -251,10 +272,10 @@ static int mes_userq_detect_and_reset(struct amdgpu_device *adev,
 	return r;
 }
 
-static int mes_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr,
-				struct drm_amdgpu_userq_in *args_in,
-				struct amdgpu_usermode_queue *queue)
+static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
+				struct drm_amdgpu_userq_in *args_in)
 {
+	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
 	struct amdgpu_device *adev = uq_mgr->adev;
 	struct amdgpu_mqd *mqd_hw_default = &adev->mqds[queue->queue_type];
 	struct drm_amdgpu_userq_in *mqd_user = args_in;
@@ -300,7 +321,7 @@ static int mes_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr,
 			goto free_mqd;
 		}
 
-		r = amdgpu_userq_input_va_validate(queue, compute_mqd->eop_va,
+		r = amdgpu_userq_input_va_validate(adev, queue, compute_mqd->eop_va,
 						   2048);
 		if (r)
 			goto free_mqd;
@@ -341,11 +362,11 @@ static int mes_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr,
 		userq_props->tmz_queue =
 			mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE;
 
-		r = amdgpu_userq_input_va_validate(queue, mqd_gfx_v11->shadow_va,
+		r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->shadow_va,
 						   shadow_info.shadow_size);
 		if (r)
 			goto free_mqd;
-		r = amdgpu_userq_input_va_validate(queue, mqd_gfx_v11->csa_va,
+		r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->csa_va,
 						   shadow_info.csa_size);
 		if (r)
 			goto free_mqd;
@@ -366,7 +387,7 @@ static int mes_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr,
 			r = -ENOMEM;
 			goto free_mqd;
 		}
-		r = amdgpu_userq_input_va_validate(queue, mqd_sdma_v11->csa_va,
+		r = amdgpu_userq_input_va_validate(adev, queue, mqd_sdma_v11->csa_va,
 						   32);
 		if (r)
 			goto free_mqd;
@@ -391,7 +412,7 @@ static int mes_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr,
 	}
 
 	/* FW expects WPTR BOs to be mapped into GART */
-	r = mes_userq_create_wptr_mapping(uq_mgr, queue, userq_props->wptr_gpu_addr);
+	r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue, userq_props->wptr_gpu_addr);
 	if (r) {
 		DRM_ERROR("Failed to create WPTR mapping\n");
 		goto free_ctx;
@@ -411,18 +432,18 @@ static int mes_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr,
 	return r;
 }
 
-static void
-mes_userq_mqd_destroy(struct amdgpu_userq_mgr *uq_mgr,
-		      struct amdgpu_usermode_queue *queue)
+static void mes_userq_mqd_destroy(struct amdgpu_usermode_queue *queue)
 {
+	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
+
 	amdgpu_userq_destroy_object(uq_mgr, &queue->fw_obj);
 	kfree(queue->userq_prop);
 	amdgpu_userq_destroy_object(uq_mgr, &queue->mqd);
 }
 
-static int mes_userq_preempt(struct amdgpu_userq_mgr *uq_mgr,
-				struct amdgpu_usermode_queue *queue)
+static int mes_userq_preempt(struct amdgpu_usermode_queue *queue)
 {
+	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
 	struct amdgpu_device *adev = uq_mgr->adev;
 	struct mes_suspend_gang_input queue_input;
 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
@@ -466,9 +487,9 @@ static int mes_userq_preempt(struct amdgpu_userq_mgr *uq_mgr,
 	return r;
 }
 
-static int mes_userq_restore(struct amdgpu_userq_mgr *uq_mgr,
-				struct amdgpu_usermode_queue *queue)
+static int mes_userq_restore(struct amdgpu_usermode_queue *queue)
 {
+	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
 	struct amdgpu_device *adev = uq_mgr->adev;
 	struct mes_resume_gang_input queue_input;
 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
index 3a52754b5cad..5159f4a9787c 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
@@ -59,8 +59,8 @@ MODULE_FIRMWARE("amdgpu/gc_11_5_3_mes1.bin");
 
 static int mes_v11_0_hw_init(struct amdgpu_ip_block *ip_block);
 static int mes_v11_0_hw_fini(struct amdgpu_ip_block *ip_block);
-static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev);
-static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev);
+static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id);
+static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev, uint32_t xcc_id);
 
 #define MES_EOP_SIZE   2048
 #define GFX_MES_DRAM_SIZE	0x80000
@@ -811,7 +811,7 @@ static int mes_v11_0_detect_and_reset_hung_queues(struct amdgpu_mes *mes,
 	mes_reset_queue_pkt.queue_type =
 		convert_to_mes_queue_type(input->queue_type);
 	mes_reset_queue_pkt.doorbell_offset_addr =
-		mes->hung_queue_db_array_gpu_addr;
+		mes->hung_queue_db_array_gpu_addr[0];
 
 	if (input->detect_only)
 		mes_reset_queue_pkt.hang_detect_only = 1;
@@ -1570,7 +1570,7 @@ static void mes_v11_0_kiq_clear(struct amdgpu_device *adev)
 	WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
 }
 
-static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev)
+static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id)
 {
 	int r = 0;
 	struct amdgpu_ip_block *ip_block;
@@ -1625,7 +1625,7 @@ static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev)
 	return r;
 }
 
-static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev)
+static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev, uint32_t xcc_id)
 {
 	if (adev->mes.ring[0].sched.ready) {
 		mes_v11_0_kiq_dequeue(&adev->mes.ring[0]);
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
index 744e95d3984a..5cbd3a4ff597 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
@@ -42,8 +42,8 @@ MODULE_FIRMWARE("amdgpu/gc_12_0_1_uni_mes.bin");
 
 static int mes_v12_0_hw_init(struct amdgpu_ip_block *ip_block);
 static int mes_v12_0_hw_fini(struct amdgpu_ip_block *ip_block);
-static int mes_v12_0_kiq_hw_init(struct amdgpu_device *adev);
-static int mes_v12_0_kiq_hw_fini(struct amdgpu_device *adev);
+static int mes_v12_0_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id);
+static int mes_v12_0_kiq_hw_fini(struct amdgpu_device *adev, uint32_t xcc_id);
 
 #define MES_EOP_SIZE   2048
 
@@ -939,7 +939,7 @@ static int mes_v12_0_detect_and_reset_hung_queues(struct amdgpu_mes *mes,
 	mes_reset_queue_pkt.queue_type =
 		convert_to_mes_queue_type(input->queue_type);
 	mes_reset_queue_pkt.doorbell_offset_addr =
-		mes->hung_queue_db_array_gpu_addr;
+		mes->hung_queue_db_array_gpu_addr[0];
 
 	if (input->detect_only)
 		mes_reset_queue_pkt.hang_detect_only = 1;
@@ -1489,7 +1489,7 @@ static int mes_v12_0_queue_init(struct amdgpu_device *adev,
 
 	if (pipe == AMDGPU_MES_SCHED_PIPE) {
 		if (adev->enable_uni_mes)
-			r = amdgpu_mes_map_legacy_queue(adev, ring);
+			r = amdgpu_mes_map_legacy_queue(adev, ring, 0);
 		else
 			r = mes_v12_0_kiq_enable_queue(adev);
 		if (r)
@@ -1739,7 +1739,7 @@ static void mes_v12_0_kiq_setting(struct amdgpu_ring *ring)
 	WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp | 0x80);
 }
 
-static int mes_v12_0_kiq_hw_init(struct amdgpu_device *adev)
+static int mes_v12_0_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id)
 {
 	int r = 0;
 	struct amdgpu_ip_block *ip_block;
@@ -1801,13 +1801,13 @@ static int mes_v12_0_kiq_hw_init(struct amdgpu_device *adev)
 	return r;
 }
 
-static int mes_v12_0_kiq_hw_fini(struct amdgpu_device *adev)
+static int mes_v12_0_kiq_hw_fini(struct amdgpu_device *adev, uint32_t xcc_id)
 {
 	if (adev->mes.ring[0].sched.ready) {
 		if (adev->enable_uni_mes)
 			amdgpu_mes_unmap_legacy_queue(adev,
 				      &adev->mes.ring[AMDGPU_MES_SCHED_PIPE],
-				      RESET_QUEUES, 0, 0);
+				      RESET_QUEUES, 0, 0, 0);
 		else
 			mes_v12_0_kiq_dequeue_sched(adev);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
new file mode 100644
index 000000000000..b0b1df5f1d4b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
@@ -0,0 +1,1890 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/firmware.h>
+#include <linux/module.h>
+#include "amdgpu.h"
+#include "soc15_common.h"
+#include "soc_v1_0.h"
+#include "gc/gc_12_1_0_offset.h"
+#include "gc/gc_12_1_0_sh_mask.h"
+#include "gc/gc_11_0_0_default.h"
+#include "v12_structs.h"
+#include "mes_v12_api_def.h"
+
+MODULE_FIRMWARE("amdgpu/gc_12_1_0_mes.bin");
+MODULE_FIRMWARE("amdgpu/gc_12_1_0_mes1.bin");
+MODULE_FIRMWARE("amdgpu/gc_12_1_0_uni_mes.bin");
+
+static int mes_v12_1_hw_init(struct amdgpu_ip_block *ip_block);
+static int mes_v12_1_xcc_hw_init(struct amdgpu_ip_block *ip_block, int xcc_id);
+static int mes_v12_1_hw_fini(struct amdgpu_ip_block *ip_block);
+static int mes_v12_1_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id);
+static int mes_v12_1_kiq_hw_fini(struct amdgpu_device *adev, uint32_t xcc_id);
+
+#define MES_EOP_SIZE   2048
+
+#define regCP_HQD_IB_CONTROL_MES_12_1_DEFAULT 0x100000
+#define XCC_REG_RANGE_0_LOW  0x1260     /* XCC gfxdec0 lower Bound */
+#define XCC_REG_RANGE_0_HIGH 0x3C00     /* XCC gfxdec0 upper Bound */
+#define XCC_REG_RANGE_1_LOW  0xA000     /* XCC gfxdec1 lower Bound */
+#define XCC_REG_RANGE_1_HIGH 0x10000    /* XCC gfxdec1 upper Bound */
+#define XCC_MID_MASK 0x41000000
+
+#define NORMALIZE_XCC_REG_OFFSET(offset) \
+	(offset & 0x3FFFF)
+
+static void mes_v12_1_ring_set_wptr(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	if (ring->use_doorbell) {
+		atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
+			     ring->wptr);
+		WDOORBELL64(ring->doorbell_index, ring->wptr);
+	} else {
+		BUG();
+	}
+}
+
+static u64 mes_v12_1_ring_get_rptr(struct amdgpu_ring *ring)
+{
+	return *ring->rptr_cpu_addr;
+}
+
+static u64 mes_v12_1_ring_get_wptr(struct amdgpu_ring *ring)
+{
+	u64 wptr;
+
+	if (ring->use_doorbell)
+		wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
+	else
+		BUG();
+	return wptr;
+}
+
+static const struct amdgpu_ring_funcs mes_v12_1_ring_funcs = {
+	.type = AMDGPU_RING_TYPE_MES,
+	.align_mask = 1,
+	.nop = 0,
+	.support_64bit_ptrs = true,
+	.get_rptr = mes_v12_1_ring_get_rptr,
+	.get_wptr = mes_v12_1_ring_get_wptr,
+	.set_wptr = mes_v12_1_ring_set_wptr,
+	.insert_nop = amdgpu_ring_insert_nop,
+};
+
+static const char *mes_v12_1_opcodes[] = {
+	"SET_HW_RSRC",
+	"SET_SCHEDULING_CONFIG",
+	"ADD_QUEUE",
+	"REMOVE_QUEUE",
+	"PERFORM_YIELD",
+	"SET_GANG_PRIORITY_LEVEL",
+	"SUSPEND",
+	"RESUME",
+	"RESET",
+	"SET_LOG_BUFFER",
+	"CHANGE_GANG_PRORITY",
+	"QUERY_SCHEDULER_STATUS",
+	"unused",
+	"SET_DEBUG_VMID",
+	"MISC",
+	"UPDATE_ROOT_PAGE_TABLE",
+	"AMD_LOG",
+	"SET_SE_MODE",
+	"SET_GANG_SUBMIT",
+	"SET_HW_RSRC_1",
+};
+
+static const char *mes_v12_1_misc_opcodes[] = {
+	"WRITE_REG",
+	"INV_GART",
+	"QUERY_STATUS",
+	"READ_REG",
+	"WAIT_REG_MEM",
+	"SET_SHADER_DEBUGGER",
+	"NOTIFY_WORK_ON_UNMAPPED_QUEUE",
+	"NOTIFY_TO_UNMAP_PROCESSES",
+};
+
+static const char *mes_v12_1_get_op_string(union MESAPI__MISC *x_pkt)
+{
+	const char *op_str = NULL;
+
+	if (x_pkt->header.opcode < ARRAY_SIZE(mes_v12_1_opcodes))
+		op_str = mes_v12_1_opcodes[x_pkt->header.opcode];
+
+	return op_str;
+}
+
+static const char *mes_v12_1_get_misc_op_string(union MESAPI__MISC *x_pkt)
+{
+	const char *op_str = NULL;
+
+	if ((x_pkt->header.opcode == MES_SCH_API_MISC) &&
+	    (x_pkt->opcode < ARRAY_SIZE(mes_v12_1_misc_opcodes)))
+		op_str = mes_v12_1_misc_opcodes[x_pkt->opcode];
+
+	return op_str;
+}
+
+static int mes_v12_1_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
+					    int xcc_id, int pipe, void *pkt,
+					    int size, int api_status_off)
+{
+	union MESAPI__QUERY_MES_STATUS mes_status_pkt;
+	signed long timeout = 2100000; /* 2100 ms */
+	struct amdgpu_device *adev = mes->adev;
+	struct amdgpu_ring *ring = &mes->ring[MES_PIPE_INST(xcc_id, pipe)];
+	spinlock_t *ring_lock = &mes->ring_lock[MES_PIPE_INST(xcc_id, pipe)];
+	struct MES_API_STATUS *api_status;
+	union MESAPI__MISC *x_pkt = pkt;
+	const char *op_str, *misc_op_str;
+	unsigned long flags;
+	u64 status_gpu_addr;
+	u32 seq, status_offset;
+	u64 *status_ptr;
+	signed long r;
+	int ret;
+
+	if (x_pkt->header.opcode >= MES_SCH_API_MAX)
+		return -EINVAL;
+
+	if (amdgpu_emu_mode) {
+		timeout *= 1000;
+	} else if (amdgpu_sriov_vf(adev)) {
+		/* Worst case in sriov where all other 15 VF timeout, each VF needs about 600ms */
+		timeout = 15 * 600 * 1000;
+	}
+
+	ret = amdgpu_device_wb_get(adev, &status_offset);
+	if (ret)
+		return ret;
+
+	status_gpu_addr = adev->wb.gpu_addr + (status_offset * 4);
+	status_ptr = (u64 *)&adev->wb.wb[status_offset];
+	*status_ptr = 0;
+
+	spin_lock_irqsave(ring_lock, flags);
+	r = amdgpu_ring_alloc(ring, (size + sizeof(mes_status_pkt)) / 4);
+	if (r)
+		goto error_unlock_free;
+
+	seq = ++ring->fence_drv.sync_seq;
+	r = amdgpu_fence_wait_polling(ring,
+				      seq - ring->fence_drv.num_fences_mask,
+				      timeout);
+	if (r < 1)
+		goto error_undo;
+
+	api_status = (struct MES_API_STATUS *)((char *)pkt + api_status_off);
+	api_status->api_completion_fence_addr = status_gpu_addr;
+	api_status->api_completion_fence_value = 1;
+
+	amdgpu_ring_write_multiple(ring, pkt, size / 4);
+
+	memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
+	mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
+	mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+	mes_status_pkt.api_status.api_completion_fence_addr =
+		ring->fence_drv.gpu_addr;
+	mes_status_pkt.api_status.api_completion_fence_value = seq;
+
+	amdgpu_ring_write_multiple(ring, &mes_status_pkt,
+				   sizeof(mes_status_pkt) / 4);
+
+	amdgpu_ring_commit(ring);
+	spin_unlock_irqrestore(ring_lock, flags);
+
+	op_str = mes_v12_1_get_op_string(x_pkt);
+	misc_op_str = mes_v12_1_get_misc_op_string(x_pkt);
+
+	if (misc_op_str)
+		dev_dbg(adev->dev, "MES(%d, %d) msg=%s (%s) was emitted\n",
+			xcc_id, pipe, op_str, misc_op_str);
+	else if (op_str)
+		dev_dbg(adev->dev, "MES(%d, %d) msg=%s was emitted\n",
+			xcc_id, pipe, op_str);
+	else
+		dev_dbg(adev->dev, "MES(%d, %d) msg=%d was emitted\n",
+			xcc_id, pipe, x_pkt->header.opcode);
+
+	r = amdgpu_fence_wait_polling(ring, seq, timeout);
+	if (r < 1 || !*status_ptr) {
+		if (misc_op_str)
+			dev_err(adev->dev,
+				"MES(%d, %d) failed to respond to msg=%s (%s)\n",
+				xcc_id, pipe, op_str, misc_op_str);
+		else if (op_str)
+			dev_err(adev->dev,
+				"MES(%d, %d) failed to respond to msg=%s\n",
+				xcc_id, pipe, op_str);
+		else
+			dev_err(adev->dev,
+				"MES(%d, %d) failed to respond to msg=%d\n",
+				xcc_id, pipe, x_pkt->header.opcode);
+
+		while (halt_if_hws_hang)
+			schedule();
+
+		r = -ETIMEDOUT;
+		goto error_wb_free;
+	}
+
+	amdgpu_device_wb_free(adev, status_offset);
+	return 0;
+
+error_undo:
+	dev_err(adev->dev, "MES(%d, %d) ring buffer is full.\n", xcc_id, pipe);
+	amdgpu_ring_undo(ring);
+
+error_unlock_free:
+	spin_unlock_irqrestore(ring_lock, flags);
+
+error_wb_free:
+	amdgpu_device_wb_free(adev, status_offset);
+	return r;
+}
+
+static int convert_to_mes_queue_type(int queue_type)
+{
+	if (queue_type == AMDGPU_RING_TYPE_GFX)
+		return MES_QUEUE_TYPE_GFX;
+	else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
+		return MES_QUEUE_TYPE_COMPUTE;
+	else if (queue_type == AMDGPU_RING_TYPE_SDMA)
+		return MES_QUEUE_TYPE_SDMA;
+	else if (queue_type == AMDGPU_RING_TYPE_MES)
+		return MES_QUEUE_TYPE_SCHQ;
+	else
+		BUG();
+	return -1;
+}
+
+static int mes_v12_1_add_hw_queue(struct amdgpu_mes *mes,
+				  struct mes_add_queue_input *input)
+{
+	union MESAPI__ADD_QUEUE mes_add_queue_pkt;
+	int xcc_id = input->xcc_id;
+	int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_SCHED_PIPE);
+
+	if (mes->enable_coop_mode)
+		xcc_id = mes->master_xcc_ids[inst];
+
+	memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
+
+	mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
+	mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+	mes_add_queue_pkt.process_id = input->process_id;
+	mes_add_queue_pkt.page_table_base_addr = input->page_table_base_addr;
+	mes_add_queue_pkt.process_va_start = input->process_va_start;
+	mes_add_queue_pkt.process_va_end = input->process_va_end;
+	mes_add_queue_pkt.process_quantum = input->process_quantum;
+	mes_add_queue_pkt.process_context_addr = input->process_context_addr;
+	mes_add_queue_pkt.gang_quantum = input->gang_quantum;
+	mes_add_queue_pkt.gang_context_addr = input->gang_context_addr;
+	mes_add_queue_pkt.inprocess_gang_priority =
+		input->inprocess_gang_priority;
+	mes_add_queue_pkt.gang_global_priority_level =
+		input->gang_global_priority_level;
+	mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
+	mes_add_queue_pkt.mqd_addr = input->mqd_addr;
+
+	mes_add_queue_pkt.wptr_addr = input->wptr_mc_addr;
+
+	mes_add_queue_pkt.queue_type =
+		convert_to_mes_queue_type(input->queue_type);
+	mes_add_queue_pkt.paging = input->paging;
+	mes_add_queue_pkt.vm_context_cntl = input->vm_cntx_cntl;
+	mes_add_queue_pkt.gws_base = input->gws_base;
+	mes_add_queue_pkt.gws_size = input->gws_size;
+	mes_add_queue_pkt.trap_handler_addr = input->tba_addr;
+	mes_add_queue_pkt.tma_addr = input->tma_addr;
+	mes_add_queue_pkt.trap_en = input->trap_en;
+	mes_add_queue_pkt.skip_process_ctx_clear = input->skip_process_ctx_clear;
+	mes_add_queue_pkt.is_kfd_process = input->is_kfd_process;
+
+	/* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
+	mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
+	mes_add_queue_pkt.gds_size = input->queue_size;
+
+	/* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
+	mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
+	mes_add_queue_pkt.gds_size = input->queue_size;
+
+	mes_add_queue_pkt.full_sh_mem_config_data = input->sh_mem_config_data;
+
+	return mes_v12_1_submit_pkt_and_poll_completion(mes,
+			xcc_id, AMDGPU_MES_SCHED_PIPE,
+			&mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
+			offsetof(union MESAPI__ADD_QUEUE, api_status));
+}
+
+static int mes_v12_1_remove_hw_queue(struct amdgpu_mes *mes,
+				     struct mes_remove_queue_input *input)
+{
+	union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
+	int xcc_id = input->xcc_id;
+	int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_SCHED_PIPE);
+
+	if (mes->enable_coop_mode)
+		xcc_id = mes->master_xcc_ids[inst];
+
+	memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
+
+	mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
+	mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+	mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
+	mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr;
+
+	return mes_v12_1_submit_pkt_and_poll_completion(mes,
+			xcc_id, AMDGPU_MES_SCHED_PIPE,
+			&mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
+			offsetof(union MESAPI__REMOVE_QUEUE, api_status));
+}
+
+static int mes_v12_1_reset_hw_queue(struct amdgpu_mes *mes,
+				    struct mes_reset_queue_input *input)
+{
+	union MESAPI__RESET mes_reset_queue_pkt;
+	int pipe;
+
+	memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
+
+	mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
+	mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+	mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;
+	/* mes_reset_queue_pkt.gang_context_addr = input->gang_context_addr; */
+	/*mes_reset_queue_pkt.reset_queue_only = 1;*/
+
+	if (mes->adev->enable_uni_mes)
+		pipe = AMDGPU_MES_KIQ_PIPE;
+	else
+		pipe = AMDGPU_MES_SCHED_PIPE;
+
+	return mes_v12_1_submit_pkt_and_poll_completion(mes,
+			input->xcc_id, pipe,
+			&mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
+			offsetof(union MESAPI__REMOVE_QUEUE, api_status));
+}
+
+static int mes_v12_1_map_legacy_queue(struct amdgpu_mes *mes,
+				      struct mes_map_legacy_queue_input *input)
+{
+	union MESAPI__ADD_QUEUE mes_add_queue_pkt;
+	int pipe;
+
+	memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
+
+	mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
+	mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+	mes_add_queue_pkt.pipe_id = input->pipe_id;
+	mes_add_queue_pkt.queue_id = input->queue_id;
+	mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
+	mes_add_queue_pkt.mqd_addr = input->mqd_addr;
+	mes_add_queue_pkt.wptr_addr = input->wptr_addr;
+	mes_add_queue_pkt.queue_type =
+		convert_to_mes_queue_type(input->queue_type);
+	mes_add_queue_pkt.map_legacy_kq = 1;
+
+	if (mes->adev->enable_uni_mes)
+		pipe = AMDGPU_MES_KIQ_PIPE;
+	else
+		pipe = AMDGPU_MES_SCHED_PIPE;
+
+	return mes_v12_1_submit_pkt_and_poll_completion(mes,
+			input->xcc_id, pipe,
+			&mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
+			offsetof(union MESAPI__ADD_QUEUE, api_status));
+}
+
+static int mes_v12_1_unmap_legacy_queue(struct amdgpu_mes *mes,
+			struct mes_unmap_legacy_queue_input *input)
+{
+	union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
+	int pipe;
+
+	memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
+
+	mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
+	mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+	mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
+	mes_remove_queue_pkt.gang_context_addr = 0;
+
+	mes_remove_queue_pkt.pipe_id = input->pipe_id;
+	mes_remove_queue_pkt.queue_id = input->queue_id;
+
+	if (input->action == PREEMPT_QUEUES_NO_UNMAP) {
+		mes_remove_queue_pkt.preempt_legacy_gfx_queue = 1;
+		mes_remove_queue_pkt.tf_addr = input->trail_fence_addr;
+		mes_remove_queue_pkt.tf_data =
+			lower_32_bits(input->trail_fence_data);
+	} else {
+		mes_remove_queue_pkt.unmap_legacy_queue = 1;
+		mes_remove_queue_pkt.queue_type =
+			convert_to_mes_queue_type(input->queue_type);
+	}
+
+	if (mes->adev->enable_uni_mes)
+		pipe = AMDGPU_MES_KIQ_PIPE;
+	else
+		pipe = AMDGPU_MES_SCHED_PIPE;
+
+	return mes_v12_1_submit_pkt_and_poll_completion(mes,
+			input->xcc_id, pipe,
+			&mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
+			offsetof(union MESAPI__REMOVE_QUEUE, api_status));
+}
+
+static int mes_v12_1_suspend_gang(struct amdgpu_mes *mes,
+				  struct mes_suspend_gang_input *input)
+{
+	return 0;
+}
+
+static int mes_v12_1_resume_gang(struct amdgpu_mes *mes,
+				 struct mes_resume_gang_input *input)
+{
+	return 0;
+}
+
+static int mes_v12_1_query_sched_status(struct amdgpu_mes *mes,
+					  int pipe, int xcc_id)
+{
+	union MESAPI__QUERY_MES_STATUS mes_status_pkt;
+
+	memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
+
+	mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
+	mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+	return mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, pipe,
+			&mes_status_pkt, sizeof(mes_status_pkt),
+			offsetof(union MESAPI__QUERY_MES_STATUS, api_status));
+}
+static uint32_t mes_v12_1_get_xcc_from_reg(uint32_t reg_offset)
+{
+	/* Check xcc reg offset range */
+	uint32_t xcc = (reg_offset & XCC_MID_MASK) ? 4 : 0;
+	/* Each XCC has two register ranges.
+	 * These are represented in reg_offset[17:16]
+	 */
+	return ((reg_offset >> 16) & 0x3) + xcc;
+}
+
+static void mes_v12_1_get_rrmt(uint32_t reg, uint32_t xcc_id,
+				 struct RRMT_OPTION *rrmt_opt)
+{
+	uint32_t normalized_reg = NORMALIZE_XCC_REG_OFFSET(reg);
+
+	if (((normalized_reg >= XCC_REG_RANGE_0_LOW) && (normalized_reg < XCC_REG_RANGE_0_HIGH)) ||
+		((normalized_reg >= XCC_REG_RANGE_1_LOW) && (normalized_reg < XCC_REG_RANGE_1_HIGH))) {
+		rrmt_opt->xcd_die_id = mes_v12_1_get_xcc_from_reg(reg);
+		rrmt_opt->mode = (xcc_id == rrmt_opt->xcd_die_id) ?
+			 MES_RRMT_MODE_LOCAL_XCD : MES_RRMT_MODE_REMOTE_XCD;
+	} else {
+		rrmt_opt->mode = MES_RRMT_MODE_LOCAL_REMOTE_AID;
+	}
+}
+
+static int mes_v12_1_misc_op(struct amdgpu_mes *mes,
+			     struct mes_misc_op_input *input)
+{
+	struct amdgpu_device *adev = mes->adev;
+	union MESAPI__MISC misc_pkt;
+	int pipe;
+
+	if (mes->adev->enable_uni_mes)
+		pipe = AMDGPU_MES_KIQ_PIPE;
+	else
+		pipe = AMDGPU_MES_SCHED_PIPE;
+
+	memset(&misc_pkt, 0, sizeof(misc_pkt));
+
+	misc_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	misc_pkt.header.opcode = MES_SCH_API_MISC;
+	misc_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+	switch (input->op) {
+	case MES_MISC_OP_READ_REG:
+		misc_pkt.opcode = MESAPI_MISC__READ_REG;
+		misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset;
+		misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr;
+		mes_v12_1_get_rrmt(input->read_reg.reg_offset,
+				   GET_INST(GC, input->xcc_id),
+				   &misc_pkt.read_reg.rrmt_opt);
+		if (misc_pkt.read_reg.rrmt_opt.mode != MES_RRMT_MODE_REMOTE_MID) {
+			misc_pkt.read_reg.reg_offset =
+				NORMALIZE_XCC_REG_OFFSET(misc_pkt.read_reg.reg_offset);
+		}
+		break;
+	case MES_MISC_OP_WRITE_REG:
+		misc_pkt.opcode = MESAPI_MISC__WRITE_REG;
+		misc_pkt.write_reg.reg_offset = input->write_reg.reg_offset;
+		misc_pkt.write_reg.reg_value = input->write_reg.reg_value;
+		mes_v12_1_get_rrmt(input->write_reg.reg_offset,
+				   GET_INST(GC, input->xcc_id),
+				   &misc_pkt.write_reg.rrmt_opt);
+		if (misc_pkt.write_reg.rrmt_opt.mode != MES_RRMT_MODE_REMOTE_MID) {
+			misc_pkt.write_reg.reg_offset =
+				NORMALIZE_XCC_REG_OFFSET(misc_pkt.write_reg.reg_offset);
+		}
+		break;
+	case MES_MISC_OP_WRM_REG_WAIT:
+		misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
+		misc_pkt.wait_reg_mem.op = WRM_OPERATION__WAIT_REG_MEM;
+		misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
+		misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
+		misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
+		misc_pkt.wait_reg_mem.reg_offset2 = 0;
+		mes_v12_1_get_rrmt(input->wrm_reg.reg0,
+				   GET_INST(GC, input->xcc_id),
+				   &misc_pkt.wait_reg_mem.rrmt_opt1);
+		if (misc_pkt.wait_reg_mem.rrmt_opt1.mode != MES_RRMT_MODE_REMOTE_MID) {
+			misc_pkt.wait_reg_mem.reg_offset1 =
+				NORMALIZE_XCC_REG_OFFSET(misc_pkt.wait_reg_mem.reg_offset1);
+		}
+		break;
+	case MES_MISC_OP_WRM_REG_WR_WAIT:
+		misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
+		misc_pkt.wait_reg_mem.op = WRM_OPERATION__WR_WAIT_WR_REG;
+		misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
+		misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
+		misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
+		misc_pkt.wait_reg_mem.reg_offset2 = input->wrm_reg.reg1;
+		mes_v12_1_get_rrmt(input->wrm_reg.reg0,
+				   GET_INST(GC, input->xcc_id),
+				   &misc_pkt.wait_reg_mem.rrmt_opt1);
+		mes_v12_1_get_rrmt(input->wrm_reg.reg1,
+				   GET_INST(GC, input->xcc_id),
+				   &misc_pkt.wait_reg_mem.rrmt_opt2);
+		if (misc_pkt.wait_reg_mem.rrmt_opt1.mode != MES_RRMT_MODE_REMOTE_MID) {
+			misc_pkt.wait_reg_mem.reg_offset1 =
+				NORMALIZE_XCC_REG_OFFSET(misc_pkt.wait_reg_mem.reg_offset1);
+		}
+		if (misc_pkt.wait_reg_mem.rrmt_opt2.mode != MES_RRMT_MODE_REMOTE_MID) {
+			misc_pkt.wait_reg_mem.reg_offset2 =
+				NORMALIZE_XCC_REG_OFFSET(misc_pkt.wait_reg_mem.reg_offset2);
+		}
+		break;
+	case MES_MISC_OP_SET_SHADER_DEBUGGER:
+		pipe = AMDGPU_MES_SCHED_PIPE;
+		misc_pkt.opcode = MESAPI_MISC__SET_SHADER_DEBUGGER;
+		misc_pkt.set_shader_debugger.process_context_addr =
+				input->set_shader_debugger.process_context_addr;
+		misc_pkt.set_shader_debugger.flags.u32all =
+				input->set_shader_debugger.flags.u32all;
+		misc_pkt.set_shader_debugger.spi_gdbg_per_vmid_cntl =
+				input->set_shader_debugger.spi_gdbg_per_vmid_cntl;
+		memcpy(misc_pkt.set_shader_debugger.tcp_watch_cntl,
+				input->set_shader_debugger.tcp_watch_cntl,
+				sizeof(misc_pkt.set_shader_debugger.tcp_watch_cntl));
+		misc_pkt.set_shader_debugger.trap_en = input->set_shader_debugger.trap_en;
+		break;
+	case MES_MISC_OP_CHANGE_CONFIG:
+		misc_pkt.opcode = MESAPI_MISC__CHANGE_CONFIG;
+		misc_pkt.change_config.opcode =
+			MESAPI_MISC__CHANGE_CONFIG_OPTION_LIMIT_SINGLE_PROCESS;
+		misc_pkt.change_config.option.bits.limit_single_process =
+			input->change_config.option.limit_single_process;
+		break;
+	default:
+		DRM_ERROR("unsupported misc op (%d) \n", input->op);
+		return -EINVAL;
+	}
+
+	return mes_v12_1_submit_pkt_and_poll_completion(mes,
+			input->xcc_id, pipe,
+			&misc_pkt, sizeof(misc_pkt),
+			offsetof(union MESAPI__MISC, api_status));
+}
+
+static int mes_v12_1_set_hw_resources_1(struct amdgpu_mes *mes,
+					  int pipe, int xcc_id)
+{
+	union MESAPI_SET_HW_RESOURCES_1 mes_set_hw_res_1_pkt;
+	int master_xcc_id, inst = MES_PIPE_INST(xcc_id, pipe);
+
+	memset(&mes_set_hw_res_1_pkt, 0, sizeof(mes_set_hw_res_1_pkt));
+
+	mes_set_hw_res_1_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_set_hw_res_1_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC_1;
+	mes_set_hw_res_1_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+	mes_set_hw_res_1_pkt.mes_kiq_unmap_timeout = 100;
+
+	if (mes->enable_coop_mode && pipe == AMDGPU_MES_SCHED_PIPE) {
+		master_xcc_id = mes->master_xcc_ids[inst];
+		mes_set_hw_res_1_pkt.mes_coop_mode = 1;
+		mes_set_hw_res_1_pkt.coop_sch_shared_mc_addr =
+			mes->shared_cmd_buf_gpu_addr[master_xcc_id];
+	}
+
+	return mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, pipe,
+			&mes_set_hw_res_1_pkt, sizeof(mes_set_hw_res_1_pkt),
+			offsetof(union MESAPI_SET_HW_RESOURCES_1, api_status));
+}
+
+static void mes_v12_1_set_gfx_hqd_mask(union MESAPI_SET_HW_RESOURCES *pkt)
+{
+	/*
+	 * GFX V12 has only one GFX pipe, but 8 queues in it.
+	 * GFX pipe 0 queue 0 is being used by Kernel queue.
+	 * Set GFX pipe 0 queue 1-7 for MES scheduling
+	 * mask = 1111 1110b
+	 */
+	pkt->gfx_hqd_mask[0] = 0xFE;
+}
+
+static int mes_v12_1_set_hw_resources(struct amdgpu_mes *mes,
+					int pipe, int xcc_id)
+{
+	int i;
+	struct amdgpu_device *adev = mes->adev;
+	union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt;
+
+	memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt));
+
+	mes_set_hw_res_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_set_hw_res_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC;
+	mes_set_hw_res_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+	if (pipe == AMDGPU_MES_SCHED_PIPE) {
+		mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub;
+		mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub;
+		mes_set_hw_res_pkt.gds_size = adev->gds.gds_size;
+		mes_set_hw_res_pkt.paging_vmid = 0;
+
+		for (i = 0; i < MAX_COMPUTE_PIPES; i++)
+			mes_set_hw_res_pkt.compute_hqd_mask[i] =
+				mes->compute_hqd_mask[i];
+
+		mes_v12_1_set_gfx_hqd_mask(&mes_set_hw_res_pkt);
+
+		for (i = 0; i < MAX_SDMA_PIPES; i++)
+			mes_set_hw_res_pkt.sdma_hqd_mask[i] =
+				mes->sdma_hqd_mask[i];
+
+		for (i = 0; i < AMD_PRIORITY_NUM_LEVELS; i++)
+			mes_set_hw_res_pkt.aggregated_doorbells[i] =
+				mes->aggregated_doorbells[i];
+	}
+
+	mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr =
+		mes->sch_ctx_gpu_addr[pipe];
+	mes_set_hw_res_pkt.query_status_fence_gpu_mc_ptr =
+		mes->query_status_fence_gpu_addr[pipe];
+
+	for (i = 0; i < 5; i++) {
+		mes_set_hw_res_pkt.gc_base[i] =
+			adev->reg_offset[GC_HWIP][0][i];
+		mes_set_hw_res_pkt.mmhub_base[i] =
+				adev->reg_offset[MMHUB_HWIP][0][i];
+		mes_set_hw_res_pkt.osssys_base[i] =
+		adev->reg_offset[OSSSYS_HWIP][0][i];
+	}
+
+	mes_set_hw_res_pkt.disable_reset = 1;
+	mes_set_hw_res_pkt.disable_mes_log = 1;
+	mes_set_hw_res_pkt.use_different_vmid_compute = 1;
+	mes_set_hw_res_pkt.enable_reg_active_poll = 1;
+	mes_set_hw_res_pkt.enable_level_process_quantum_check = 1;
+
+	/*
+	 * Keep oversubscribe timer for sdma . When we have unmapped doorbell
+	 * handling support, other queue will not use the oversubscribe timer.
+	 * handling  mode - 0: disabled; 1: basic version; 2: basic+ version
+	 */
+	mes_set_hw_res_pkt.oversubscription_timer = 50;
+	mes_set_hw_res_pkt.unmapped_doorbell_handling = 1;
+
+	if (amdgpu_mes_log_enable) {
+		mes_set_hw_res_pkt.enable_mes_event_int_logging = 1;
+		mes_set_hw_res_pkt.event_intr_history_gpu_mc_ptr =
+			mes->event_log_gpu_addr + MES_PIPE_INST(xcc_id, pipe) * AMDGPU_MES_LOG_BUFFER_SIZE;
+	}
+
+	if (adev->enforce_isolation[0] == AMDGPU_ENFORCE_ISOLATION_ENABLE)
+		mes_set_hw_res_pkt.limit_single_process = 1;
+
+	return mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, pipe,
+			&mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),
+			offsetof(union MESAPI_SET_HW_RESOURCES, api_status));
+}
+
+static void mes_v12_1_init_aggregated_doorbell(struct amdgpu_mes *mes,
+						 int xcc_id)
+{
+	struct amdgpu_device *adev = mes->adev;
+	uint32_t data;
+
+	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL1);
+	data &= ~(CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET_MASK |
+		  CP_MES_DOORBELL_CONTROL1__DOORBELL_EN_MASK |
+		  CP_MES_DOORBELL_CONTROL1__DOORBELL_HIT_MASK);
+	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_LOW] <<
+		CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET__SHIFT;
+	data |= 1 << CP_MES_DOORBELL_CONTROL1__DOORBELL_EN__SHIFT;
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL1, data);
+
+	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL2);
+	data &= ~(CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET_MASK |
+		  CP_MES_DOORBELL_CONTROL2__DOORBELL_EN_MASK |
+		  CP_MES_DOORBELL_CONTROL2__DOORBELL_HIT_MASK);
+	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_NORMAL] <<
+		CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET__SHIFT;
+	data |= 1 << CP_MES_DOORBELL_CONTROL2__DOORBELL_EN__SHIFT;
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL2, data);
+
+	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL3);
+	data &= ~(CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET_MASK |
+		  CP_MES_DOORBELL_CONTROL3__DOORBELL_EN_MASK |
+		  CP_MES_DOORBELL_CONTROL3__DOORBELL_HIT_MASK);
+	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_MEDIUM] <<
+		CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET__SHIFT;
+	data |= 1 << CP_MES_DOORBELL_CONTROL3__DOORBELL_EN__SHIFT;
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL3, data);
+
+	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL4);
+	data &= ~(CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET_MASK |
+		  CP_MES_DOORBELL_CONTROL4__DOORBELL_EN_MASK |
+		  CP_MES_DOORBELL_CONTROL4__DOORBELL_HIT_MASK);
+	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_HIGH] <<
+		CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET__SHIFT;
+	data |= 1 << CP_MES_DOORBELL_CONTROL4__DOORBELL_EN__SHIFT;
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL4, data);
+
+	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL5);
+	data &= ~(CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET_MASK |
+		  CP_MES_DOORBELL_CONTROL5__DOORBELL_EN_MASK |
+		  CP_MES_DOORBELL_CONTROL5__DOORBELL_HIT_MASK);
+	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_REALTIME] <<
+		CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET__SHIFT;
+	data |= 1 << CP_MES_DOORBELL_CONTROL5__DOORBELL_EN__SHIFT;
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_DOORBELL_CONTROL5, data);
+
+	data = 1 << CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN__SHIFT;
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_GFX_CONTROL, data);
+}
+
+
+static void mes_v12_1_enable_unmapped_doorbell_handling(
+	struct amdgpu_mes *mes, bool enable, int xcc_id)
+{
+	struct amdgpu_device *adev = mes->adev;
+	uint32_t data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_UNMAPPED_DOORBELL);
+
+	/*
+	 * The default PROC_LSB settng is 0xc which means doorbell
+	 * addr[16:12] gives the doorbell page number. For kfd, each
+	 * process will use 2 pages of doorbell, we need to change the
+	 * setting to 0xd
+	 */
+	data &= ~CP_UNMAPPED_DOORBELL__PROC_LSB_MASK;
+	data |= 0xd <<  CP_UNMAPPED_DOORBELL__PROC_LSB__SHIFT;
+
+	data |= (enable ? 1 : 0) << CP_UNMAPPED_DOORBELL__ENABLE__SHIFT;
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_UNMAPPED_DOORBELL, data);
+}
+
+#if 0
+static int mes_v12_1_reset_legacy_queue(struct amdgpu_mes *mes,
+					struct mes_reset_legacy_queue_input *input)
+{
+	union MESAPI__RESET mes_reset_queue_pkt;
+	int pipe;
+
+	memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
+
+	mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
+	mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+	mes_reset_queue_pkt.queue_type =
+		convert_to_mes_queue_type(input->queue_type);
+
+	if (mes_reset_queue_pkt.queue_type == MES_QUEUE_TYPE_GFX) {
+		mes_reset_queue_pkt.reset_legacy_gfx = 1;
+		mes_reset_queue_pkt.pipe_id_lp = input->pipe_id;
+		mes_reset_queue_pkt.queue_id_lp = input->queue_id;
+		mes_reset_queue_pkt.mqd_mc_addr_lp = input->mqd_addr;
+		mes_reset_queue_pkt.doorbell_offset_lp = input->doorbell_offset;
+		mes_reset_queue_pkt.wptr_addr_lp = input->wptr_addr;
+		mes_reset_queue_pkt.vmid_id_lp = input->vmid;
+	} else {
+		mes_reset_queue_pkt.reset_queue_only = 1;
+		mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;
+	}
+
+	if (mes->adev->enable_uni_mes)
+		pipe = AMDGPU_MES_KIQ_PIPE;
+	else
+		pipe = AMDGPU_MES_SCHED_PIPE;
+
+	return mes_v12_1_submit_pkt_and_poll_completion(mes,
+			input->xcc_id, pipe,
+			&mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
+			offsetof(union MESAPI__RESET, api_status));
+}
+#endif
+
+static const struct amdgpu_mes_funcs mes_v12_1_funcs = {
+	.add_hw_queue = mes_v12_1_add_hw_queue,
+	.remove_hw_queue = mes_v12_1_remove_hw_queue,
+	.map_legacy_queue = mes_v12_1_map_legacy_queue,
+	.unmap_legacy_queue = mes_v12_1_unmap_legacy_queue,
+	.suspend_gang = mes_v12_1_suspend_gang,
+	.resume_gang = mes_v12_1_resume_gang,
+	.misc_op = mes_v12_1_misc_op,
+	.reset_hw_queue = mes_v12_1_reset_hw_queue,
+};
+
+static int mes_v12_1_allocate_ucode_buffer(struct amdgpu_device *adev,
+					     enum amdgpu_mes_pipe pipe,
+					     int xcc_id)
+{
+	int r, inst = MES_PIPE_INST(xcc_id, pipe);
+	const struct mes_firmware_header_v1_0 *mes_hdr;
+	const __le32 *fw_data;
+	unsigned fw_size;
+
+	mes_hdr = (const struct mes_firmware_header_v1_0 *)
+		adev->mes.fw[pipe]->data;
+
+	fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
+		   le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
+	fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
+
+	r = amdgpu_bo_create_reserved(adev, fw_size,
+				      PAGE_SIZE,
+				      AMDGPU_GEM_DOMAIN_VRAM,
+				      &adev->mes.ucode_fw_obj[inst],
+				      &adev->mes.ucode_fw_gpu_addr[inst],
+				      (void **)&adev->mes.ucode_fw_ptr[inst]);
+	if (r) {
+		dev_err(adev->dev, "(%d) failed to create mes fw bo\n", r);
+		return r;
+	}
+
+	memcpy(adev->mes.ucode_fw_ptr[inst], fw_data, fw_size);
+
+	amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[inst]);
+	amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[inst]);
+
+	return 0;
+}
+
+static int mes_v12_1_allocate_ucode_data_buffer(struct amdgpu_device *adev,
+						  enum amdgpu_mes_pipe pipe,
+						  int xcc_id)
+{
+	int r, inst = MES_PIPE_INST(xcc_id, pipe);
+	const struct mes_firmware_header_v1_0 *mes_hdr;
+	const __le32 *fw_data;
+	unsigned fw_size;
+
+	mes_hdr = (const struct mes_firmware_header_v1_0 *)
+		adev->mes.fw[pipe]->data;
+
+	fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
+		   le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
+	fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
+
+	r = amdgpu_bo_create_reserved(adev, fw_size,
+				      64 * 1024,
+				      AMDGPU_GEM_DOMAIN_VRAM,
+				      &adev->mes.data_fw_obj[inst],
+				      &adev->mes.data_fw_gpu_addr[inst],
+				      (void **)&adev->mes.data_fw_ptr[inst]);
+	if (r) {
+		dev_err(adev->dev, "(%d) failed to create mes data fw bo\n", r);
+		return r;
+	}
+
+	memcpy(adev->mes.data_fw_ptr[inst], fw_data, fw_size);
+
+	amdgpu_bo_kunmap(adev->mes.data_fw_obj[inst]);
+	amdgpu_bo_unreserve(adev->mes.data_fw_obj[inst]);
+
+	return 0;
+}
+
+static void mes_v12_1_free_ucode_buffers(struct amdgpu_device *adev,
+					   enum amdgpu_mes_pipe pipe,
+					   int xcc_id)
+{
+	int inst = MES_PIPE_INST(xcc_id, pipe);
+
+	amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[inst],
+			      &adev->mes.data_fw_gpu_addr[inst],
+			      (void **)&adev->mes.data_fw_ptr[inst]);
+
+	amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[inst],
+			      &adev->mes.ucode_fw_gpu_addr[inst],
+			      (void **)&adev->mes.ucode_fw_ptr[inst]);
+}
+
+static void mes_v12_1_enable(struct amdgpu_device *adev,
+			       bool enable, int xcc_id)
+{
+	uint64_t ucode_addr;
+	uint32_t pipe, data = 0;
+
+	if (enable) {
+		data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_CNTL);
+		data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
+		data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET, 1);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_CNTL, data);
+
+		mutex_lock(&adev->srbm_mutex);
+		for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
+			soc_v1_0_grbm_select(adev, 3, pipe, 0, 0,
+					     GET_INST(GC, xcc_id));
+
+			ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
+			WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+				     regCP_MES_PRGRM_CNTR_START,
+				     lower_32_bits(ucode_addr));
+			WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+				     regCP_MES_PRGRM_CNTR_START_HI,
+				     upper_32_bits(ucode_addr));
+		}
+		soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
+		mutex_unlock(&adev->srbm_mutex);
+
+		/* unhalt MES and activate pipe0 */
+		data = REG_SET_FIELD(0, CP_MES_CNTL, MES_PIPE0_ACTIVE, 1);
+		data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 1);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_CNTL, data);
+
+		if (amdgpu_emu_mode)
+			msleep(500);
+		else if (adev->enable_uni_mes)
+			udelay(500);
+		else
+			udelay(50);
+	} else {
+		data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_CNTL);
+		data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_ACTIVE, 0);
+		data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 0);
+		data = REG_SET_FIELD(data, CP_MES_CNTL,
+				     MES_INVALIDATE_ICACHE, 1);
+		data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
+		data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET, 1);
+		data = REG_SET_FIELD(data, CP_MES_CNTL, MES_HALT, 1);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_CNTL, data);
+	}
+}
+
+static void mes_v12_1_set_ucode_start_addr(struct amdgpu_device *adev,
+					     int xcc_id)
+{
+	uint64_t ucode_addr;
+	int pipe;
+
+	mes_v12_1_enable(adev, false, xcc_id);
+
+	mutex_lock(&adev->srbm_mutex);
+	for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
+		/* me=3, queue=0 */
+		soc_v1_0_grbm_select(adev, 3, pipe, 0, 0, GET_INST(GC, xcc_id));
+
+		/* set ucode start address */
+		ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_PRGRM_CNTR_START,
+				lower_32_bits(ucode_addr));
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_PRGRM_CNTR_START_HI,
+				upper_32_bits(ucode_addr));
+
+		soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
+	}
+	mutex_unlock(&adev->srbm_mutex);
+}
+
+/* This function is for backdoor MES firmware */
+static int mes_v12_1_load_microcode(struct amdgpu_device *adev,
+				      enum amdgpu_mes_pipe pipe,
+				      bool prime_icache, int xcc_id)
+{
+	int r, inst = MES_PIPE_INST(xcc_id, pipe);
+	uint32_t data;
+
+	mes_v12_1_enable(adev, false, xcc_id);
+
+	if (!adev->mes.fw[pipe])
+		return -EINVAL;
+
+	r = mes_v12_1_allocate_ucode_buffer(adev, pipe, xcc_id);
+	if (r)
+		return r;
+
+	r = mes_v12_1_allocate_ucode_data_buffer(adev, pipe, xcc_id);
+	if (r) {
+		mes_v12_1_free_ucode_buffers(adev, pipe, xcc_id);
+		return r;
+	}
+
+	mutex_lock(&adev->srbm_mutex);
+	/* me=3, pipe=0, queue=0 */
+	soc_v1_0_grbm_select(adev, 3, pipe, 0, 0, GET_INST(GC, xcc_id));
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_IC_BASE_CNTL, 0);
+
+	/* set ucode fimrware address */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_IC_BASE_LO,
+		     lower_32_bits(adev->mes.ucode_fw_gpu_addr[inst]));
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_IC_BASE_HI,
+		     upper_32_bits(adev->mes.ucode_fw_gpu_addr[inst]));
+
+	/* set ucode instruction cache boundary to 2M-1 */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_MIBOUND_LO, 0x1FFFFF);
+
+	/* set ucode data firmware address */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_MDBASE_LO,
+		     lower_32_bits(adev->mes.data_fw_gpu_addr[inst]));
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_MDBASE_HI,
+		     upper_32_bits(adev->mes.data_fw_gpu_addr[inst]));
+
+	/* Set data cache boundary CP_MES_MDBOUND_LO */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_MDBOUND_LO, 0x7FFFF);
+
+	if (prime_icache) {
+		/* invalidate ICACHE */
+		data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_IC_OP_CNTL);
+		data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 0);
+		data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, INVALIDATE_CACHE, 1);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_IC_OP_CNTL, data);
+
+		/* prime the ICACHE. */
+		data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_IC_OP_CNTL);
+		data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 1);
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_IC_OP_CNTL, data);
+	}
+
+	soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
+	mutex_unlock(&adev->srbm_mutex);
+
+	return 0;
+}
+
+static int mes_v12_1_allocate_eop_buf(struct amdgpu_device *adev,
+					enum amdgpu_mes_pipe pipe,
+					int xcc_id)
+{
+	int r, inst = MES_PIPE_INST(xcc_id, pipe);
+	u32 *eop;
+
+	r = amdgpu_bo_create_reserved(adev, MES_EOP_SIZE, PAGE_SIZE,
+			      AMDGPU_GEM_DOMAIN_GTT,
+			      &adev->mes.eop_gpu_obj[inst],
+			      &adev->mes.eop_gpu_addr[inst],
+			      (void **)&eop);
+	if (r) {
+		dev_warn(adev->dev, "(%d) create EOP bo failed\n", r);
+		return r;
+	}
+
+	memset(eop, 0,
+	       adev->mes.eop_gpu_obj[inst]->tbo.base.size);
+
+	amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[inst]);
+	amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[inst]);
+
+	return 0;
+}
+
+static int mes_v12_1_allocate_shared_cmd_buf(struct amdgpu_device *adev,
+					     enum amdgpu_mes_pipe pipe,
+					     int xcc_id)
+{
+	int r, inst = MES_PIPE_INST(xcc_id, pipe);
+
+	if (pipe == AMDGPU_MES_KIQ_PIPE)
+		return 0;
+
+	r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
+				    AMDGPU_GEM_DOMAIN_VRAM,
+				    &adev->mes.shared_cmd_buf_obj[inst],
+				    &adev->mes.shared_cmd_buf_gpu_addr[inst],
+				    NULL);
+	if (r) {
+		dev_err(adev->dev,
+			"(%d) failed to create shared cmd buf bo\n", r);
+		return r;
+	}
+
+	return 0;
+}
+
+static int mes_v12_1_mqd_init(struct amdgpu_ring *ring)
+{
+	struct v12_1_mes_mqd *mqd = ring->mqd_ptr;
+	uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
+	uint32_t tmp;
+
+	mqd->header = 0xC0310800;
+	mqd->compute_pipelinestat_enable = 0x00000001;
+	mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
+	mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
+	mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
+	mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
+	mqd->compute_misc_reserved = 0x00000007;
+
+	eop_base_addr = ring->eop_gpu_addr >> 8;
+
+	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
+	tmp = regCP_HQD_EOP_CONTROL_DEFAULT;
+	tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
+			(order_base_2(MES_EOP_SIZE / 4) - 1));
+
+	mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_base_addr);
+	mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
+	mqd->cp_hqd_eop_control = tmp;
+
+	/* disable the queue if it's active */
+	ring->wptr = 0;
+	mqd->cp_hqd_pq_rptr = 0;
+	mqd->cp_hqd_pq_wptr_lo = 0;
+	mqd->cp_hqd_pq_wptr_hi = 0;
+
+	/* set the pointer to the MQD */
+	mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
+	mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
+
+	/* set MQD vmid to 0 */
+	tmp = regCP_MQD_CONTROL_DEFAULT;
+	tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
+	mqd->cp_mqd_control = tmp;
+
+	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
+	hqd_gpu_addr = ring->gpu_addr >> 8;
+	mqd->cp_hqd_pq_base_lo = lower_32_bits(hqd_gpu_addr);
+	mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
+
+	/* set the wb address whether it's enabled or not */
+	wb_gpu_addr = ring->rptr_gpu_addr;
+	mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
+	mqd->cp_hqd_pq_rptr_report_addr_hi =
+		upper_32_bits(wb_gpu_addr) & 0xffff;
+
+	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
+	wb_gpu_addr = ring->wptr_gpu_addr;
+	mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffff8;
+	mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
+
+	/* set up the HQD, this is similar to CP_RB0_CNTL */
+	tmp = regCP_HQD_PQ_CONTROL_DEFAULT;
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
+			    (order_base_2(ring->ring_size / 4) - 1));
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
+			    ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, NO_UPDATE_RPTR, 1);
+	mqd->cp_hqd_pq_control = tmp;
+
+	/* enable doorbell */
+	tmp = 0;
+	if (ring->use_doorbell) {
+		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+				    DOORBELL_OFFSET, ring->doorbell_index);
+		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+				    DOORBELL_EN, 1);
+		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+				    DOORBELL_SOURCE, 0);
+		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+				    DOORBELL_HIT, 0);
+	} else {
+		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
+				    DOORBELL_EN, 0);
+	}
+	mqd->cp_hqd_pq_doorbell_control = tmp;
+
+	mqd->cp_hqd_vmid = 0;
+	/* activate the queue */
+	mqd->cp_hqd_active = 1;
+
+	tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT;
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE,
+			    PRELOAD_SIZE, 0x63);
+	mqd->cp_hqd_persistent_state = tmp;
+
+	mqd->cp_hqd_ib_control = regCP_HQD_IB_CONTROL_MES_12_1_DEFAULT;
+	mqd->cp_hqd_iq_timer = regCP_HQD_IQ_TIMER_DEFAULT;
+	mqd->cp_hqd_quantum = regCP_HQD_QUANTUM_DEFAULT;
+
+	/*
+	 * Set CP_HQD_GFX_CONTROL.DB_UPDATED_MSG_EN[15] to enable unmapped
+	 * doorbell handling. This is a reserved CP internal register can
+	 * not be accesss by others
+	 */
+	mqd->cp_hqd_gfx_control = BIT(15);
+
+	return 0;
+}
+
+static void mes_v12_1_queue_init_register(struct amdgpu_ring *ring,
+					    int xcc_id)
+{
+	struct v12_1_mes_mqd *mqd = ring->mqd_ptr;
+	struct amdgpu_device *adev = ring->adev;
+	uint32_t data = 0;
+
+	mutex_lock(&adev->srbm_mutex);
+	soc_v1_0_grbm_select(adev, 3, ring->pipe, 0, 0, GET_INST(GC, xcc_id));
+
+	/* set CP_HQD_VMID.VMID = 0. */
+	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_VMID);
+	data = REG_SET_FIELD(data, CP_HQD_VMID, VMID, 0);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_VMID, data);
+
+	/* set CP_HQD_PQ_DOORBELL_CONTROL.DOORBELL_EN=0 */
+	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL);
+	data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
+			     DOORBELL_EN, 0);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL, data);
+
+	/* set CP_MQD_BASE_ADDR/HI with the MQD base address */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
+
+	/* set CP_MQD_CONTROL.VMID=0 */
+	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_CONTROL);
+	data = REG_SET_FIELD(data, CP_MQD_CONTROL, VMID, 0);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_CONTROL, 0);
+
+	/* set CP_HQD_PQ_BASE/HI with the ring buffer base address */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi);
+
+	/* set CP_HQD_PQ_RPTR_REPORT_ADDR/HI */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR,
+		     mqd->cp_hqd_pq_rptr_report_addr_lo);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
+		     mqd->cp_hqd_pq_rptr_report_addr_hi);
+
+	/* set CP_HQD_PQ_CONTROL */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control);
+
+	/* set CP_HQD_PQ_WPTR_POLL_ADDR/HI */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR,
+		     mqd->cp_hqd_pq_wptr_poll_addr_lo);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
+		     mqd->cp_hqd_pq_wptr_poll_addr_hi);
+
+	/* set CP_HQD_PQ_DOORBELL_CONTROL */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL,
+		     mqd->cp_hqd_pq_doorbell_control);
+
+	/* set CP_HQD_PERSISTENT_STATE.PRELOAD_SIZE=0x53 */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state);
+
+	/* set CP_HQD_ACTIVE.ACTIVE=1 */
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE, mqd->cp_hqd_active);
+
+	soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
+	mutex_unlock(&adev->srbm_mutex);
+}
+
+static int mes_v12_1_kiq_enable_queue(struct amdgpu_device *adev, int xcc_id)
+{
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
+	struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[xcc_id].ring;
+	int r, inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_SCHED_PIPE);
+
+	if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
+		return -EINVAL;
+
+	r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
+	if (r) {
+		DRM_ERROR("Failed to lock KIQ (%d).\n", r);
+		return r;
+	}
+
+	kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring[inst]);
+
+	r = amdgpu_ring_test_ring(kiq_ring);
+	if (r) {
+		DRM_ERROR("kfq enable failed\n");
+		kiq_ring->sched.ready = false;
+	}
+	return r;
+}
+
+static int mes_v12_1_queue_init(struct amdgpu_device *adev,
+				  enum amdgpu_mes_pipe pipe,
+				  int xcc_id)
+{
+	struct amdgpu_ring *ring;
+	int r;
+
+	if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE)
+		ring = &adev->gfx.kiq[xcc_id].ring;
+	else
+		ring = &adev->mes.ring[MES_PIPE_INST(xcc_id, pipe)];
+
+	if ((adev->enable_uni_mes || pipe == AMDGPU_MES_SCHED_PIPE) &&
+	    (amdgpu_in_reset(adev) || adev->in_suspend)) {
+		*(ring->wptr_cpu_addr) = 0;
+		*(ring->rptr_cpu_addr) = 0;
+		amdgpu_ring_clear_ring(ring);
+	}
+
+	r = mes_v12_1_mqd_init(ring);
+	if (r)
+		return r;
+
+	if (pipe == AMDGPU_MES_SCHED_PIPE) {
+		if (adev->enable_uni_mes)
+			r = amdgpu_mes_map_legacy_queue(adev, ring, xcc_id);
+		else
+			r = mes_v12_1_kiq_enable_queue(adev, xcc_id);
+		if (r)
+			return r;
+	} else {
+		mes_v12_1_queue_init_register(ring, xcc_id);
+	}
+
+	/* get MES scheduler/KIQ versions */
+	mutex_lock(&adev->srbm_mutex);
+	soc_v1_0_grbm_select(adev, 3, pipe, 0, 0, GET_INST(GC, xcc_id));
+
+	if (pipe == AMDGPU_MES_SCHED_PIPE)
+		adev->mes.sched_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO);
+	else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)
+		adev->mes.kiq_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO);
+
+	soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
+	mutex_unlock(&adev->srbm_mutex);
+
+	return 0;
+}
+
+static int mes_v12_1_ring_init(struct amdgpu_device *adev,
+				 int xcc_id, int pipe)
+{
+	struct amdgpu_ring *ring;
+	int inst = MES_PIPE_INST(xcc_id, pipe);
+
+	ring = &adev->mes.ring[inst];
+
+	ring->funcs = &mes_v12_1_ring_funcs;
+
+	ring->me = 3;
+	ring->pipe = pipe;
+	ring->queue = 0;
+	ring->xcc_id = xcc_id;
+	ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
+
+	ring->ring_obj = NULL;
+	ring->use_doorbell = true;
+	ring->eop_gpu_addr = adev->mes.eop_gpu_addr[inst];
+	ring->no_scheduler = true;
+	snprintf(ring->name, sizeof(ring->name), "mes_%hhu.%hhu.%hhu.%hhu",
+		 (unsigned char)xcc_id, (unsigned char)ring->me,
+		 (unsigned char)ring->pipe, (unsigned char)ring->queue);
+
+	if (pipe == AMDGPU_MES_SCHED_PIPE)
+		ring->doorbell_index =
+			(adev->doorbell_index.mes_ring0 +
+			 xcc_id * adev->doorbell_index.xcc_doorbell_range)
+			<< 1;
+	else
+		ring->doorbell_index =
+			(adev->doorbell_index.mes_ring1 +
+			 xcc_id * adev->doorbell_index.xcc_doorbell_range)
+			<< 1;
+
+	return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
+				AMDGPU_RING_PRIO_DEFAULT, NULL);
+}
+
+static int mes_v12_1_kiq_ring_init(struct amdgpu_device *adev, int xcc_id)
+{
+	struct amdgpu_ring *ring;
+	int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_KIQ_PIPE);
+
+	spin_lock_init(&adev->gfx.kiq[xcc_id].ring_lock);
+
+	ring = &adev->gfx.kiq[xcc_id].ring;
+
+	ring->me = 3;
+	ring->pipe = 1;
+	ring->queue = 0;
+	ring->xcc_id = xcc_id;
+	ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
+
+	ring->adev = NULL;
+	ring->ring_obj = NULL;
+	ring->use_doorbell = true;
+	ring->eop_gpu_addr = adev->mes.eop_gpu_addr[inst];
+	ring->no_scheduler = true;
+	ring->doorbell_index =
+		(adev->doorbell_index.mes_ring1 +
+		 xcc_id * adev->doorbell_index.xcc_doorbell_range)
+		<< 1;
+
+	snprintf(ring->name, sizeof(ring->name), "mes_kiq_%hhu.%hhu.%hhu.%hhu",
+		 (unsigned char)xcc_id, (unsigned char)ring->me,
+		 (unsigned char)ring->pipe, (unsigned char)ring->queue);
+
+	return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
+				AMDGPU_RING_PRIO_DEFAULT, NULL);
+}
+
+static int mes_v12_1_mqd_sw_init(struct amdgpu_device *adev,
+				   enum amdgpu_mes_pipe pipe,
+				   int xcc_id)
+{
+	int r, mqd_size = sizeof(struct v12_1_mes_mqd);
+	struct amdgpu_ring *ring;
+	int inst = MES_PIPE_INST(xcc_id, pipe);
+
+	if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE)
+		ring = &adev->gfx.kiq[xcc_id].ring;
+	else
+		ring = &adev->mes.ring[inst];
+
+	if (ring->mqd_obj)
+		return 0;
+
+	r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
+				    AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
+				    &ring->mqd_gpu_addr, &ring->mqd_ptr);
+	if (r) {
+		dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
+		return r;
+	}
+
+	memset(ring->mqd_ptr, 0, mqd_size);
+
+	/* prepare MQD backup */
+	adev->mes.mqd_backup[inst] = kmalloc(mqd_size, GFP_KERNEL);
+	if (!adev->mes.mqd_backup[inst])
+		dev_warn(adev->dev,
+			 "no memory to create MQD backup for ring %s\n",
+			 ring->name);
+
+	return 0;
+}
+
+static int mes_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int pipe, r, xcc_id, num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+
+	if (adev->enable_uni_mes && num_xcc > 1)
+		adev->mes.enable_coop_mode = true;
+
+	adev->mes.funcs = &mes_v12_1_funcs;
+	adev->mes.kiq_hw_init = &mes_v12_1_kiq_hw_init;
+	adev->mes.kiq_hw_fini = &mes_v12_1_kiq_hw_fini;
+	adev->mes.enable_legacy_queue_map = true;
+
+	adev->mes.event_log_size =
+		adev->enable_uni_mes ? (AMDGPU_MAX_MES_PIPES * AMDGPU_MES_LOG_BUFFER_SIZE * num_xcc) : AMDGPU_MES_LOG_BUFFER_SIZE;
+
+	r = amdgpu_mes_init(adev);
+	if (r)
+		return r;
+
+	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
+		for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
+			r = mes_v12_1_allocate_eop_buf(adev, pipe, xcc_id);
+			if (r)
+				return r;
+
+			r = mes_v12_1_mqd_sw_init(adev, pipe, xcc_id);
+			if (r)
+				return r;
+
+			if (!adev->enable_uni_mes && pipe ==
+			    AMDGPU_MES_KIQ_PIPE)
+				r = mes_v12_1_kiq_ring_init(adev, xcc_id);
+			else
+				r = mes_v12_1_ring_init(adev, xcc_id, pipe);
+			if (r)
+				return r;
+
+			if (adev->mes.enable_coop_mode) {
+				r = mes_v12_1_allocate_shared_cmd_buf(adev,
+							      pipe, xcc_id);
+				if (r)
+					return r;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int mes_v12_1_sw_fini(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int pipe, inst, xcc_id, num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+
+	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
+		for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
+			inst = MES_PIPE_INST(xcc_id, pipe);
+
+			amdgpu_bo_free_kernel(&adev->mes.shared_cmd_buf_obj[inst],
+					      &adev->mes.shared_cmd_buf_gpu_addr[inst],
+					      NULL);
+
+			kfree(adev->mes.mqd_backup[inst]);
+
+			amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[inst],
+					      &adev->mes.eop_gpu_addr[inst],
+					      NULL);
+			amdgpu_ucode_release(&adev->mes.fw[inst]);
+
+			if (adev->enable_uni_mes || pipe == AMDGPU_MES_SCHED_PIPE) {
+				amdgpu_bo_free_kernel(&adev->mes.ring[inst].mqd_obj,
+						      &adev->mes.ring[inst].mqd_gpu_addr,
+						      &adev->mes.ring[inst].mqd_ptr);
+				amdgpu_ring_fini(&adev->mes.ring[inst]);
+			}
+		}
+	}
+
+	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
+		if (!adev->enable_uni_mes) {
+			amdgpu_bo_free_kernel(&adev->gfx.kiq[xcc_id].ring.mqd_obj,
+				      &adev->gfx.kiq[xcc_id].ring.mqd_gpu_addr,
+				      &adev->gfx.kiq[xcc_id].ring.mqd_ptr);
+			amdgpu_ring_fini(&adev->gfx.kiq[xcc_id].ring);
+		}
+
+		if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+			mes_v12_1_free_ucode_buffers(adev,
+				       AMDGPU_MES_KIQ_PIPE, xcc_id);
+			mes_v12_1_free_ucode_buffers(adev,
+				       AMDGPU_MES_SCHED_PIPE, xcc_id);
+		}
+	}
+
+	amdgpu_mes_fini(adev);
+	return 0;
+}
+
+static void mes_v12_1_kiq_dequeue_sched(struct amdgpu_device *adev,
+					  int xcc_id)
+{
+	uint32_t data;
+	int i;
+
+	mutex_lock(&adev->srbm_mutex);
+	soc_v1_0_grbm_select(adev, 3, AMDGPU_MES_SCHED_PIPE, 0, 0,
+			     GET_INST(GC, xcc_id));
+
+	/* disable the queue if it's active */
+	if (RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1) {
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST, 1);
+		for (i = 0; i < adev->usec_timeout; i++) {
+			if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1))
+				break;
+			udelay(1);
+		}
+	}
+	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL);
+	data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
+				DOORBELL_EN, 0);
+	data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
+				DOORBELL_HIT, 1);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL, data);
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL, 0);
+
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO, 0);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI, 0);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR, 0);
+
+	soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
+	mutex_unlock(&adev->srbm_mutex);
+
+	adev->mes.ring[MES_PIPE_INST(xcc_id, 0)].sched.ready = false;
+}
+
+static void mes_v12_1_kiq_setting(struct amdgpu_ring *ring, int xcc_id)
+{
+	uint32_t tmp;
+	struct amdgpu_device *adev = ring->adev;
+
+	/* tell RLC which is KIQ queue */
+	tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS);
+	tmp &= 0xffffff00;
+	tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);
+	tmp |= 0x80;
+	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);
+}
+
+static int mes_v12_1_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id)
+{
+	int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_KIQ_PIPE);
+	int r = 0;
+	struct amdgpu_ip_block *ip_block;
+
+	if (adev->enable_uni_mes)
+		mes_v12_1_kiq_setting(&adev->mes.ring[inst], xcc_id);
+	else
+		mes_v12_1_kiq_setting(&adev->gfx.kiq[xcc_id].ring, xcc_id);
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+
+		r = mes_v12_1_load_microcode(adev, AMDGPU_MES_SCHED_PIPE,
+					       false, xcc_id);
+		if (r) {
+			DRM_ERROR("failed to load MES fw, r=%d\n", r);
+			return r;
+		}
+
+		r = mes_v12_1_load_microcode(adev, AMDGPU_MES_KIQ_PIPE,
+					       true, xcc_id);
+		if (r) {
+			DRM_ERROR("failed to load MES kiq fw, r=%d\n", r);
+			return r;
+		}
+
+		mes_v12_1_set_ucode_start_addr(adev, xcc_id);
+
+	} else if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
+		mes_v12_1_set_ucode_start_addr(adev, xcc_id);
+
+	mes_v12_1_enable(adev, true, xcc_id);
+
+	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_MES);
+	if (unlikely(!ip_block)) {
+		dev_err(adev->dev, "Failed to get MES handle\n");
+		return -EINVAL;
+	}
+
+	r = mes_v12_1_queue_init(adev, AMDGPU_MES_KIQ_PIPE, xcc_id);
+	if (r)
+		goto failure;
+
+	if (adev->enable_uni_mes) {
+		r = mes_v12_1_set_hw_resources(&adev->mes,
+						 AMDGPU_MES_KIQ_PIPE, xcc_id);
+		if (r)
+			goto failure;
+
+		mes_v12_1_set_hw_resources_1(&adev->mes,
+					       AMDGPU_MES_KIQ_PIPE, xcc_id);
+	}
+
+	if (adev->mes.enable_legacy_queue_map) {
+		r = mes_v12_1_xcc_hw_init(ip_block, xcc_id);
+		if (r)
+			goto failure;
+	}
+
+	return r;
+
+failure:
+	mes_v12_1_hw_fini(ip_block);
+	return r;
+}
+
+static int mes_v12_1_kiq_hw_fini(struct amdgpu_device *adev, uint32_t xcc_id)
+{
+	int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_SCHED_PIPE);
+
+	if (adev->mes.ring[inst].sched.ready) {
+		if (adev->enable_uni_mes)
+			amdgpu_mes_unmap_legacy_queue(adev,
+				      &adev->mes.ring[inst],
+				      RESET_QUEUES, 0, 0, xcc_id);
+		else
+			mes_v12_1_kiq_dequeue_sched(adev, xcc_id);
+
+		adev->mes.ring[inst].sched.ready = false;
+	}
+
+	mes_v12_1_enable(adev, false, xcc_id);
+
+	return 0;
+}
+
+static int mes_v12_1_xcc_hw_init(struct amdgpu_ip_block *ip_block, int xcc_id)
+{
+	int r;
+	struct amdgpu_device *adev = ip_block->adev;
+
+	if (adev->mes.ring[MES_PIPE_INST(xcc_id, 0)].sched.ready)
+		goto out;
+
+	if (!adev->enable_mes_kiq) {
+		if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+			r = mes_v12_1_load_microcode(adev,
+				       AMDGPU_MES_SCHED_PIPE, true, xcc_id);
+			if (r) {
+				DRM_ERROR("failed to MES fw, r=%d\n", r);
+				return r;
+			}
+
+			mes_v12_1_set_ucode_start_addr(adev, xcc_id);
+
+		} else if (adev->firmware.load_type ==
+			   AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
+
+			mes_v12_1_set_ucode_start_addr(adev, xcc_id);
+		}
+
+		mes_v12_1_enable(adev, true, xcc_id);
+	}
+
+	/* Enable the MES to handle doorbell ring on unmapped queue */
+	mes_v12_1_enable_unmapped_doorbell_handling(&adev->mes, true, xcc_id);
+
+	r = mes_v12_1_queue_init(adev, AMDGPU_MES_SCHED_PIPE, xcc_id);
+	if (r)
+		goto failure;
+
+	r = mes_v12_1_set_hw_resources(&adev->mes,
+					 AMDGPU_MES_SCHED_PIPE, xcc_id);
+	if (r)
+		goto failure;
+
+	if (adev->enable_uni_mes)
+		mes_v12_1_set_hw_resources_1(&adev->mes,
+					       AMDGPU_MES_SCHED_PIPE, xcc_id);
+
+	mes_v12_1_init_aggregated_doorbell(&adev->mes, xcc_id);
+
+	r = mes_v12_1_query_sched_status(&adev->mes,
+					   AMDGPU_MES_SCHED_PIPE, xcc_id);
+	if (r) {
+		DRM_ERROR("MES is busy\n");
+		goto failure;
+	}
+
+out:
+	/*
+	 * Disable KIQ ring usage from the driver once MES is enabled.
+	 * MES uses KIQ ring exclusively so driver cannot access KIQ ring
+	 * with MES enabled.
+	 */
+	adev->gfx.kiq[xcc_id].ring.sched.ready = false;
+	adev->mes.ring[MES_PIPE_INST(xcc_id, 0)].sched.ready = true;
+
+	return 0;
+
+failure:
+	mes_v12_1_hw_fini(ip_block);
+	return r;
+}
+
+static int mes_v12_1_hw_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int r, xcc_id, num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+
+	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
+		/* for SPX mode, all master xcc ids are set to 0 */
+		if (adev->mes.enable_coop_mode)
+			adev->mes.master_xcc_ids[xcc_id] = 0;
+
+		r = mes_v12_1_xcc_hw_init(ip_block, xcc_id);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+
+static int mes_v12_1_hw_fini(struct amdgpu_ip_block *ip_block)
+{
+	return 0;
+}
+
+static int mes_v12_1_suspend(struct amdgpu_ip_block *ip_block)
+{
+	int r;
+
+	r = amdgpu_mes_suspend(ip_block->adev);
+	if (r)
+		return r;
+
+	return mes_v12_1_hw_fini(ip_block);
+}
+
+static int mes_v12_1_resume(struct amdgpu_ip_block *ip_block)
+{
+	int r;
+
+	r = mes_v12_1_hw_init(ip_block);
+	if (r)
+		return r;
+
+	return amdgpu_mes_resume(ip_block->adev);
+}
+
+static int mes_v12_1_early_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int pipe, r;
+
+	for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
+		r = amdgpu_mes_init_microcode(adev, pipe);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+
+static const struct amd_ip_funcs mes_v12_1_ip_funcs = {
+	.name = "mes_v12_1",
+	.early_init = mes_v12_1_early_init,
+	.late_init = NULL,
+	.sw_init = mes_v12_1_sw_init,
+	.sw_fini = mes_v12_1_sw_fini,
+	.hw_init = mes_v12_1_hw_init,
+	.hw_fini = mes_v12_1_hw_fini,
+	.suspend = mes_v12_1_suspend,
+	.resume = mes_v12_1_resume,
+};
+
+const struct amdgpu_ip_block_version mes_v12_1_ip_block = {
+	.type = AMD_IP_BLOCK_TYPE_MES,
+	.major = 12,
+	.minor = 1,
+	.rev = 0,
+	.funcs = &mes_v12_1_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.h b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.h
new file mode 100644
index 000000000000..cfc048f652b0
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __MES_V12_1_H__
+#define __MES_V12_1_H__
+
+extern const struct amdgpu_ip_block_version mes_v12_1_ip_block;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c
new file mode 100644
index 000000000000..7e917eb47a8c
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c
@@ -0,0 +1,916 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "amdgpu.h"
+#include "mmhub_v4_2_0.h"
+
+#include "mmhub/mmhub_4_2_0_offset.h"
+#include "mmhub/mmhub_4_2_0_sh_mask.h"
+
+#include "soc15_common.h"
+#include "soc24_enum.h"
+
+#define regMMVM_L2_CNTL3_DEFAULT				0x80100007
+#define regMMVM_L2_CNTL4_DEFAULT				0x000000c1
+#define regMMVM_L2_CNTL5_DEFAULT				0x00003fe0
+
+static const char *mmhub_client_ids_v4_2_0[][2] = {
+	[0][0] = "VMC",
+	[4][0] = "DCEDMC",
+	[5][0] = "DCEVGA",
+	[6][0] = "MP0",
+	[7][0] = "MP1",
+	[8][0] = "MPIO",
+	[16][0] = "HDP",
+	[17][0] = "LSDMA",
+	[18][0] = "JPEG",
+	[19][0] = "VCNU0",
+	[21][0] = "VSCH",
+	[22][0] = "VCNU1",
+	[23][0] = "VCN1",
+	[32+20][0] = "VCN0",
+	[2][1] = "DBGUNBIO",
+	[3][1] = "DCEDWB",
+	[4][1] = "DCEDMC",
+	[5][1] = "DCEVGA",
+	[6][1] = "MP0",
+	[7][1] = "MP1",
+	[8][1] = "MPIO",
+	[10][1] = "DBGU0",
+	[11][1] = "DBGU1",
+	[12][1] = "DBGU2",
+	[13][1] = "DBGU3",
+	[14][1] = "XDP",
+	[15][1] = "OSSSYS",
+	[16][1] = "HDP",
+	[17][1] = "LSDMA",
+	[18][1] = "JPEG",
+	[19][1] = "VCNU0",
+	[20][1] = "VCN0",
+	[21][1] = "VSCH",
+	[22][1] = "VCNU1",
+	[23][1] = "VCN1",
+};
+
+static u64 mmhub_v4_2_0_get_fb_location(struct amdgpu_device *adev)
+{
+	u64 base;
+
+	base = RREG32_SOC15(MMHUB, GET_INST(MMHUB, 0),
+			    regMMMC_VM_FB_LOCATION_BASE_LO32);
+	base &= MMMC_VM_FB_LOCATION_BASE_LO32__FB_BASE_LO32_MASK;
+	base <<= 24;
+
+	base |= ((u64)(MMMC_VM_FB_LOCATION_BASE_HI32__FB_BASE_HI1_MASK &
+		       RREG32_SOC15(MMHUB, GET_INST(MMHUB, 0),
+				    regMMMC_VM_FB_LOCATION_BASE_HI32)) << 56);
+
+	return base;
+}
+
+static u64 mmhub_v4_2_0_get_mc_fb_offset(struct amdgpu_device *adev)
+{
+	return (u64)RREG32_SOC15(MMHUB, GET_INST(MMHUB, 0),
+			         regMMMC_VM_FB_OFFSET) << 24;
+}
+
+static void mmhub_v4_2_0_mid_setup_vm_pt_regs(struct amdgpu_device *adev,
+					      uint32_t vmid,
+					      uint64_t page_table_base,
+					      uint32_t mid_mask)
+{
+	struct amdgpu_vmhub *hub;
+	int i;
+
+	for_each_inst(i, mid_mask) {
+		hub = &adev->vmhub[AMDGPU_MMHUB0(i)];
+		WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, i),
+				    regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
+				    hub->ctx_addr_distance * vmid,
+				    lower_32_bits(page_table_base));
+
+		WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, i),
+				    regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
+				    hub->ctx_addr_distance * vmid,
+				    upper_32_bits(page_table_base));
+	}
+}
+
+static void mmhub_v4_2_0_setup_vm_pt_regs(struct amdgpu_device *adev,
+					  uint32_t vmid,
+					  uint64_t page_table_base)
+{
+	uint32_t mid_mask;
+
+	mid_mask = adev->aid_mask;
+	mmhub_v4_2_0_mid_setup_vm_pt_regs(adev, vmid,
+					  page_table_base,
+					  mid_mask);
+}
+
+static void mmhub_v4_2_0_mid_init_gart_aperture_regs(struct amdgpu_device *adev,
+						     uint32_t mid_mask)
+{
+	uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
+	int i;
+
+	if (adev->gmc.pdb0_bo)
+		pt_base = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo);
+	else
+		pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
+
+	mmhub_v4_2_0_mid_setup_vm_pt_regs(adev, 0, pt_base, mid_mask);
+
+	for_each_inst(i, mid_mask) {
+		if (adev->gmc.pdb0_bo) {
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
+				     (u32)(adev->gmc.fb_start >> 12));
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
+				     (u32)(adev->gmc.fb_start >> 44));
+
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
+				     (u32)(adev->gmc.fb_end >> 12));
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
+				     (u32)(adev->gmc.fb_end >> 44));
+		} else {
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
+				     (u32)(adev->gmc.gart_start >> 12));
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
+				     (u32)(adev->gmc.gart_start >> 44));
+
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
+				     (u32)(adev->gmc.gart_end >> 12));
+			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				     regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
+				     (u32)(adev->gmc.gart_end >> 44));
+		}
+	}
+}
+
+static void mmhub_v4_2_0_mid_init_system_aperture_regs(struct amdgpu_device *adev,
+						       uint32_t mid_mask)
+{
+	uint64_t value;
+	uint32_t tmp;
+	int i;
+
+	/*
+	 * the new L1 policy will block SRIOV guest from writing
+	 * these regs, and they will be programed at host.
+	 * so skip programing these regs.
+	 */
+	if (amdgpu_sriov_vf(adev))
+		return;
+
+	for_each_inst(i, mid_mask) {
+		/* Program the AGP BAR */
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_AGP_BASE_LO32, 0);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_AGP_BASE_HI32, 0);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_AGP_BOT_LO32,
+			     lower_32_bits(adev->gmc.agp_start >> 24));
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_AGP_BOT_HI32,
+			     upper_32_bits(adev->gmc.agp_start >> 24));
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_AGP_TOP_LO32,
+			     lower_32_bits(adev->gmc.agp_end >> 24));
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_AGP_TOP_HI32,
+			     upper_32_bits(adev->gmc.agp_end >> 24));
+
+		/* Program the system aperture low logical page number. */
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32,
+			     lower_32_bits(min(adev->gmc.fb_start,
+					       adev->gmc.agp_start) >> 18));
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32,
+			     upper_32_bits(min(adev->gmc.fb_start,
+					       adev->gmc.agp_start) >> 18));
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32,
+			     lower_32_bits(max(adev->gmc.fb_end,
+					       adev->gmc.agp_end) >> 18));
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32,
+			     upper_32_bits(max(adev->gmc.fb_end,
+					       adev->gmc.agp_end) >> 18));
+
+		/* Set default page address. */
+		value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
+			     (u32)(value >> 12));
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
+			     (u32)(value >> 44));
+
+		/* Program "protection fault". */
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
+			     (u32)(adev->dummy_page_addr >> 12));
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
+			     (u32)((u64)adev->dummy_page_addr >> 44));
+
+		tmp = RREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				   regMMVM_L2_PROTECTION_FAULT_CNTL2);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL2,
+				    ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL2,
+				    ENABLE_RETRY_FAULT_INTERRUPT, 0x1);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMVM_L2_PROTECTION_FAULT_CNTL2, tmp);
+	}
+
+	/* In the case squeezing vram into GART aperture, we don't use
+	 * FB aperture and AGP aperture. Disable them.
+	 */
+	if (adev->gmc.pdb0_bo) {
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_FB_LOCATION_TOP_LO32, 0);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_FB_LOCATION_TOP_HI32, 0);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_FB_LOCATION_BASE_LO32, 0xFFFFFFFF);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_FB_LOCATION_BASE_HI32, 1);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_AGP_TOP_LO32, 0);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_AGP_TOP_HI32, 0);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_AGP_BOT_LO32, 0xFFFFFFFF);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_AGP_BOT_HI32, 1);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32,
+			     0xFFFFFFFF);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32,
+			     0x7F);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32, 0);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32, 0);
+	}
+}
+
+static void mmhub_v4_2_0_mid_init_tlb_regs(struct amdgpu_device *adev,
+					   uint32_t mid_mask)
+{
+	uint32_t tmp;
+	int i;
+
+	for_each_inst(i, mid_mask) {
+		/* Setup TLB control */
+		tmp = RREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				   regMMMC_VM_MX_L1_TLB_CNTL);
+
+		tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
+		tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
+		tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
+				    ENABLE_ADVANCED_DRIVER_MODEL, 1);
+		tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
+				    SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
+		tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
+		tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
+				    MTYPE, MTYPE_UC); /* UC, uncached */
+
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMMC_VM_MX_L1_TLB_CNTL, tmp);
+	}
+}
+
+static void mmhub_v4_2_0_mid_init_cache_regs(struct amdgpu_device *adev,
+					     uint32_t mid_mask)
+{
+	uint32_t tmp;
+	int i;
+
+	/* These registers are not accessible to VF-SRIOV.
+	 * The PF will program them instead.
+	 */
+	if (amdgpu_sriov_vf(adev))
+		return;
+
+	for_each_inst(i, mid_mask) {
+		/* Setup L2 cache */
+		tmp = RREG32_SOC15(MMHUB, GET_INST(MMHUB, i), regMMVM_L2_CNTL);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_CACHE, 1);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL,
+				    ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
+		/* XXX for emulation, Refer to closed source code.*/
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL,
+				    L2_PDE0_CACHE_TAG_GENERATION_MODE, 0);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL,
+				    PDE_FAULT_CLASSIFICATION, 0);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL,
+				    CONTEXT1_IDENTITY_ACCESS_MODE, 1);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL,
+				    IDENTITY_MODE_FRAGMENT_SIZE, 0);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i), regMMVM_L2_CNTL, tmp);
+
+		tmp = RREG32_SOC15(MMHUB, GET_INST(MMHUB, i), regMMVM_L2_CNTL2);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL2,
+				    INVALIDATE_ALL_L1_TLBS, 1);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL2,
+				    INVALIDATE_L2_CACHE, 1);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i), regMMVM_L2_CNTL2, tmp);
+
+		tmp = regMMVM_L2_CNTL3_DEFAULT;
+		if (adev->gmc.translate_further) {
+			tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3, BANK_SELECT, 12);
+			tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3,
+					    L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
+		} else {
+			tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3, BANK_SELECT, 9);
+			tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3,
+					    L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
+		}
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i), regMMVM_L2_CNTL3, tmp);
+
+		tmp = regMMVM_L2_CNTL4_DEFAULT;
+		/* For AMD APP APUs setup WC memory */
+		if (adev->gmc.xgmi.connected_to_cpu || adev->gmc.is_app_apu) {
+			tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL4,
+					    VMC_TAP_PDE_REQUEST_PHYSICAL, 1);
+			tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL4,
+					    VMC_TAP_PTE_REQUEST_PHYSICAL, 1);
+		} else {
+			tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL4,
+					    VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
+			tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL4,
+					    VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
+		}
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i), regMMVM_L2_CNTL4, tmp);
+
+		tmp = regMMVM_L2_CNTL5_DEFAULT;
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL5,
+				    L2_CACHE_SMALLK_FRAGMENT_SIZE, 0);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i), regMMVM_L2_CNTL5, tmp);
+	}
+}
+
+static void mmhub_v4_2_0_mid_enable_system_domain(struct amdgpu_device *adev,
+						  uint32_t mid_mask)
+{
+	uint32_t tmp;
+	int i;
+
+	for_each_inst(i, mid_mask) {
+		tmp = RREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				   regMMVM_CONTEXT0_CNTL);
+		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL,
+				    ENABLE_CONTEXT, 1);
+		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL,
+				    PAGE_TABLE_DEPTH, 0);
+		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL,
+				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMVM_CONTEXT0_CNTL, tmp);
+	}
+}
+
+static void mmhub_v4_2_0_mid_disable_identity_aperture(struct amdgpu_device *adev,
+						       uint32_t mid_mask)
+{
+	int i;
+
+	/* These registers are not accessible to VF-SRIOV.
+	 * The PF will program them instead.
+	 */
+	if (amdgpu_sriov_vf(adev))
+		return;
+
+	for_each_inst(i, mid_mask) {
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
+			     0xFFFFFFFF);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
+			     0x00001FFF);
+
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32,
+			     0);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32,
+			     0);
+
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32,
+			     0);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32,
+			     0);
+	}
+}
+
+static void mmhub_v4_2_0_mid_setup_vmid_config(struct amdgpu_device *adev,
+					       uint32_t mid_mask)
+{
+	struct amdgpu_vmhub *hub;
+	uint32_t tmp;
+	int i, j;
+
+	for_each_inst(j, mid_mask) {
+		hub = &adev->vmhub[AMDGPU_MMHUB0(j)];
+		for (i = 0; i <= 14; i++) {
+			tmp = RREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j),
+					          regMMVM_CONTEXT1_CNTL,
+						  i * hub->ctx_distance);
+			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
+			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
+					    adev->vm_manager.num_level);
+			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+					    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+					    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
+					    1);
+			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+					    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+					    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+					    READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+					    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+					    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+					    PAGE_TABLE_BLOCK_SIZE,
+					    adev->vm_manager.block_size - 9);
+			/* Send no-retry XNACK on fault to suppress VM fault storm. */
+			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
+					    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
+					    !amdgpu_noretry);
+			WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j), regMMVM_CONTEXT1_CNTL,
+					    i * hub->ctx_distance, tmp);
+			WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j), regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
+					    i * hub->ctx_addr_distance, 0);
+			WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j), regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
+					    i * hub->ctx_addr_distance, 0);
+			WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j), regMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
+					    i * hub->ctx_addr_distance,
+					    lower_32_bits(adev->vm_manager.max_pfn - 1));
+			WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j), regMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
+					    i * hub->ctx_addr_distance,
+					    upper_32_bits(adev->vm_manager.max_pfn - 1));
+		}
+	}
+
+	hub->vm_cntx_cntl = tmp;
+}
+
+static void mmhub_v4_2_0_mid_program_invalidation(struct amdgpu_device *adev,
+						  uint32_t mid_mask)
+{
+	struct amdgpu_vmhub *hub;
+	unsigned int i, j;
+
+	for_each_inst(j, mid_mask) {
+		hub = &adev->vmhub[AMDGPU_MMHUB0(j)];
+
+		for (i = 0; i < 18; ++i) {
+			WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j),
+					    regMMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
+					    i * hub->eng_addr_distance, 0xffffffff);
+			WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j),
+					    regMMVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
+					    i * hub->eng_addr_distance, 0x3fff);
+		}
+	}
+}
+
+static int mmhub_v4_2_0_mid_gart_enable(struct amdgpu_device *adev,
+					uint32_t mid_mask)
+{
+	/* GART Enable. */
+	mmhub_v4_2_0_mid_init_gart_aperture_regs(adev, mid_mask);
+	mmhub_v4_2_0_mid_init_system_aperture_regs(adev, mid_mask);
+	mmhub_v4_2_0_mid_init_tlb_regs(adev, mid_mask);
+	mmhub_v4_2_0_mid_init_cache_regs(adev, mid_mask);
+
+	mmhub_v4_2_0_mid_enable_system_domain(adev, mid_mask);
+	mmhub_v4_2_0_mid_disable_identity_aperture(adev, mid_mask);
+	mmhub_v4_2_0_mid_setup_vmid_config(adev, mid_mask);
+	mmhub_v4_2_0_mid_program_invalidation(adev, mid_mask);
+
+	return 0;
+}
+static int mmhub_v4_2_0_gart_enable(struct amdgpu_device *adev)
+{
+	uint32_t mid_mask;
+
+	mid_mask = adev->aid_mask;
+	return mmhub_v4_2_0_mid_gart_enable(adev, mid_mask);
+}
+
+static void mmhub_v4_2_0_mid_gart_disable(struct amdgpu_device *adev,
+					  uint32_t mid_mask)
+{
+	struct amdgpu_vmhub *hub;
+	u32 tmp;
+	u32 i, j;
+
+	for_each_inst(j, mid_mask) {
+		hub = &adev->vmhub[AMDGPU_MMHUB0(j)];
+		/* Disable all tables */
+		for (i = 0; i < 16; i++)
+			WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j),
+					    regMMVM_CONTEXT0_CNTL,
+					    i * hub->ctx_distance, 0);
+
+		/* Setup TLB control */
+		tmp = RREG32_SOC15(MMHUB, GET_INST(MMHUB, j),
+				   regMMMC_VM_MX_L1_TLB_CNTL);
+		tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
+				    ENABLE_L1_TLB, 0);
+		tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
+				    ENABLE_ADVANCED_DRIVER_MODEL, 0);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, j),
+			     regMMMC_VM_MX_L1_TLB_CNTL, tmp);
+
+		/* Setup L2 cache */
+		tmp = RREG32_SOC15(MMHUB, GET_INST(MMHUB, j), regMMVM_L2_CNTL);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_CACHE, 0);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, j), regMMVM_L2_CNTL, tmp);
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, j), regMMVM_L2_CNTL3, 0);
+	}
+}
+
+static void mmhub_v4_2_0_gart_disable(struct amdgpu_device *adev)
+{
+	uint32_t mid_mask;
+
+	mid_mask = adev->aid_mask;
+	mmhub_v4_2_0_mid_gart_disable(adev, mid_mask);
+}
+
+static void
+mmhub_v4_2_0_mid_set_fault_enable_default(struct amdgpu_device *adev,
+					  bool value, uint32_t mid_mask)
+{
+	u32 tmp;
+	int i;
+
+	/* These registers are not accessible to VF-SRIOV.
+	 * The PF will program them instead.
+	 */
+	if (amdgpu_sriov_vf(adev))
+		return;
+
+	for_each_inst(i, mid_mask) {
+		tmp = RREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+				   regMMVM_L2_PROTECTION_FAULT_CNTL_LO32);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
+				    value);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL_LO32,
+				    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+		if (!value) {
+			tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL_LO32,
+					    CRASH_ON_NO_RETRY_FAULT, 1);
+		}
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			     regMMVM_L2_PROTECTION_FAULT_CNTL_LO32, tmp);
+	}
+}
+
+
+/**
+ * mmhub_v4_2_0_set_fault_enable_default - update GART/VM fault handling
+ *
+ * @adev: amdgpu_device pointer
+ * @value: true redirects VM faults to the default page
+ */
+static void
+mmhub_v4_2_0_set_fault_enable_default(struct amdgpu_device *adev,
+				      bool value)
+{
+	uint32_t mid_mask;
+
+	mid_mask = adev->aid_mask;
+	mmhub_v4_2_0_mid_set_fault_enable_default(adev, value, mid_mask);
+}
+
+static uint32_t mmhub_v4_2_0_get_invalidate_req(unsigned int vmid,
+						uint32_t flush_type)
+{
+	u32 req = 0;
+
+	/* invalidate using legacy mode on vmid*/
+	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ,
+			    PER_VMID_INVALIDATE_REQ, 1 << vmid);
+	/* Only use legacy inv on mmhub side */
+	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, 0);
+	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
+	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
+	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
+	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
+	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE3, 1);
+	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
+	req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ,
+			    CLEAR_PROTECTION_FAULT_STATUS_ADDR,	0);
+
+	return req;
+}
+
+/*TODO: l2 protection fault status is increased to 64bits.
+ * some critical fields like FED are moved to STATUS_HI32 */
+static void
+mmhub_v4_2_0_print_l2_protection_fault_status(struct amdgpu_device *adev,
+					      uint32_t status)
+{
+	uint32_t cid, rw;
+	const char *mmhub_cid = NULL;
+
+	cid = REG_GET_FIELD(status,
+			    MMVM_L2_PROTECTION_FAULT_STATUS_LO32, CID);
+	rw = REG_GET_FIELD(status,
+			   MMVM_L2_PROTECTION_FAULT_STATUS_LO32, RW);
+
+	dev_err(adev->dev,
+		"MMVM_L2_PROTECTION_FAULT_STATUS_LO32:0x%08X\n",
+		status);
+	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
+	case IP_VERSION(4, 2, 0):
+		mmhub_cid = mmhub_client_ids_v4_2_0[cid][rw];
+		break;
+	default:
+		mmhub_cid = NULL;
+		break;
+	}
+	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
+		mmhub_cid ? mmhub_cid : "unknown", cid);
+	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
+		REG_GET_FIELD(status,
+		MMVM_L2_PROTECTION_FAULT_STATUS_LO32, MORE_FAULTS));
+	dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
+		REG_GET_FIELD(status,
+		MMVM_L2_PROTECTION_FAULT_STATUS_LO32, WALKER_ERROR));
+	dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
+		REG_GET_FIELD(status,
+		MMVM_L2_PROTECTION_FAULT_STATUS_LO32, PERMISSION_FAULTS));
+	dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
+		REG_GET_FIELD(status,
+		MMVM_L2_PROTECTION_FAULT_STATUS_LO32, MAPPING_ERROR));
+	dev_err(adev->dev, "\t RW: 0x%x\n", rw);
+}
+
+
+static const struct amdgpu_vmhub_funcs mmhub_v4_2_0_vmhub_funcs = {
+	.print_l2_protection_fault_status = mmhub_v4_2_0_print_l2_protection_fault_status,
+	.get_invalidate_req = mmhub_v4_2_0_get_invalidate_req,
+};
+
+static void mmhub_v4_2_0_mid_init(struct amdgpu_device *adev,
+				  uint32_t mid_mask)
+{
+	struct amdgpu_vmhub *hub;
+	int i;
+
+	for_each_inst(i, mid_mask) {
+		hub = &adev->vmhub[AMDGPU_MMHUB0(i)];
+
+		hub->ctx0_ptb_addr_lo32 =
+			SOC15_REG_OFFSET(MMHUB, GET_INST(MMHUB, i),
+					 regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
+		hub->ctx0_ptb_addr_hi32 =
+			SOC15_REG_OFFSET(MMHUB, GET_INST(MMHUB, i),
+					 regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
+		hub->vm_inv_eng0_sem =
+			SOC15_REG_OFFSET(MMHUB, GET_INST(MMHUB, i),
+					 regMMVM_INVALIDATE_ENG0_SEM);
+		hub->vm_inv_eng0_req =
+			SOC15_REG_OFFSET(MMHUB, GET_INST(MMHUB, i),
+					 regMMVM_INVALIDATE_ENG0_REQ);
+		hub->vm_inv_eng0_ack =
+			SOC15_REG_OFFSET(MMHUB, GET_INST(MMHUB, i),
+					 regMMVM_INVALIDATE_ENG0_ACK);
+		hub->vm_context0_cntl =
+			SOC15_REG_OFFSET(MMHUB, GET_INST(MMHUB, i),
+					 regMMVM_CONTEXT0_CNTL);
+		/* TODO: add a new member to accomandate additional fault status/cntl reg */
+		hub->vm_l2_pro_fault_status =
+			SOC15_REG_OFFSET(MMHUB, GET_INST(MMHUB, i),
+					 regMMVM_L2_PROTECTION_FAULT_STATUS_LO32);
+		hub->vm_l2_pro_fault_cntl =
+			SOC15_REG_OFFSET(MMHUB, GET_INST(MMHUB, i),
+					 regMMVM_L2_PROTECTION_FAULT_CNTL_LO32);
+
+		hub->ctx_distance = regMMVM_CONTEXT1_CNTL - regMMVM_CONTEXT0_CNTL;
+		hub->ctx_addr_distance = regMMVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 -
+					 regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
+		hub->eng_distance = regMMVM_INVALIDATE_ENG1_REQ -
+				    regMMVM_INVALIDATE_ENG0_REQ;
+		hub->eng_addr_distance = regMMVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 -
+					 regMMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32;
+
+		hub->vm_cntx_cntl_vm_fault = MMVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+			MMVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+			MMVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+			MMVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+			MMVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+			MMVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+			MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
+
+		hub->vm_l2_bank_select_reserved_cid2 =
+			SOC15_REG_OFFSET(MMHUB, GET_INST(MMHUB, i), regMMVM_L2_BANK_SELECT_RESERVED_CID2);
+
+		hub->vm_contexts_disable =
+			SOC15_REG_OFFSET(MMHUB, GET_INST(MMHUB, i), regMMVM_CONTEXTS_DISABLE);
+
+		hub->vmhub_funcs = &mmhub_v4_2_0_vmhub_funcs;
+	}
+}
+
+static void mmhub_v4_2_0_init(struct amdgpu_device *adev)
+{
+	uint32_t mid_mask;
+
+	mid_mask = adev->aid_mask;
+	mmhub_v4_2_0_mid_init(adev, mid_mask);
+}
+
+static void
+mmhub_v4_2_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
+					      bool enable)
+{
+	uint32_t def, data;
+	uint32_t def1, data1, def2 = 0, data2 = 0;
+	def  = data  = RREG32_SOC15(MMHUB, GET_INST(MMHUB, 0), regMM_ATC_L2_MISC_CG);
+	def1 = data1 = RREG32_SOC15(MMHUB, GET_INST(MMHUB, 0), regDAGB0_CNTL_MISC2);
+	def2 = data2 = RREG32_SOC15(MMHUB, GET_INST(MMHUB, 0), regDAGB1_CNTL_MISC2);
+
+	if (enable) {
+		data |= MM_ATC_L2_MISC_CG__ENABLE_MASK;
+		data1 &= ~(DAGB0_CNTL_MISC2__DISABLE_RDRET_TAP_CHAIN_FGCG_MASK |
+			   DAGB0_CNTL_MISC2__DISABLE_WRRET_TAP_CHAIN_FGCG_MASK);
+
+		data2 &= ~(DAGB1_CNTL_MISC2__DISABLE_RDRET_TAP_CHAIN_FGCG_MASK |
+			   DAGB1_CNTL_MISC2__DISABLE_WRRET_TAP_CHAIN_FGCG_MASK);
+	} else {
+		data &= ~MM_ATC_L2_MISC_CG__ENABLE_MASK;
+		data1 |= (DAGB0_CNTL_MISC2__DISABLE_RDRET_TAP_CHAIN_FGCG_MASK |
+			  DAGB0_CNTL_MISC2__DISABLE_WRRET_TAP_CHAIN_FGCG_MASK);
+
+		data2 |= (DAGB1_CNTL_MISC2__DISABLE_RDRET_TAP_CHAIN_FGCG_MASK |
+			  DAGB1_CNTL_MISC2__DISABLE_WRRET_TAP_CHAIN_FGCG_MASK);
+	}
+
+	if (def != data)
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, 0), regMM_ATC_L2_MISC_CG, data);
+	if (def1 != data1)
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, 0), regDAGB0_CNTL_MISC2, data1);
+
+	if (def2 != data2)
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, 0), regDAGB1_CNTL_MISC2, data2);
+}
+
+static void
+mmhub_v4_2_0_update_medium_grain_light_sleep(struct amdgpu_device *adev,
+					     bool enable)
+{
+	uint32_t def, data;
+
+	def = data = RREG32_SOC15(MMHUB, GET_INST(MMHUB, 0), regMM_ATC_L2_MISC_CG);
+
+	if (enable)
+		data |= MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
+	else
+		data &= ~MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
+
+	if (def != data)
+		WREG32_SOC15(MMHUB, GET_INST(MMHUB, 0), regMM_ATC_L2_MISC_CG, data);
+}
+
+static int mmhub_v4_2_0_set_clockgating(struct amdgpu_device *adev,
+					enum amd_clockgating_state state)
+{
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
+	if (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)
+		mmhub_v4_2_0_update_medium_grain_clock_gating(adev,
+				state == AMD_CG_STATE_GATE);
+
+	if (adev->cg_flags & AMD_CG_SUPPORT_MC_LS)
+		mmhub_v4_2_0_update_medium_grain_light_sleep(adev,
+				state == AMD_CG_STATE_GATE);
+
+	return 0;
+}
+
+static void mmhub_v4_2_0_get_clockgating(struct amdgpu_device *adev, u64 *flags)
+{
+	int data;
+
+	if (amdgpu_sriov_vf(adev))
+		*flags = 0;
+
+	data = RREG32_SOC15(MMHUB, GET_INST(MMHUB, 0), regMM_ATC_L2_MISC_CG);
+
+	/* AMD_CG_SUPPORT_MC_MGCG */
+	if (data & MM_ATC_L2_MISC_CG__ENABLE_MASK)
+		*flags |= AMD_CG_SUPPORT_MC_MGCG;
+
+	/* AMD_CG_SUPPORT_MC_LS */
+	if (data & MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK)
+		*flags |= AMD_CG_SUPPORT_MC_LS;
+}
+
+const struct amdgpu_mmhub_funcs mmhub_v4_2_0_funcs = {
+	.init = mmhub_v4_2_0_init,
+	.get_fb_location = mmhub_v4_2_0_get_fb_location,
+	.get_mc_fb_offset = mmhub_v4_2_0_get_mc_fb_offset,
+	.setup_vm_pt_regs = mmhub_v4_2_0_setup_vm_pt_regs,
+	.gart_enable = mmhub_v4_2_0_gart_enable,
+	.gart_disable = mmhub_v4_2_0_gart_disable,
+	.set_fault_enable_default = mmhub_v4_2_0_set_fault_enable_default,
+	.set_clockgating = mmhub_v4_2_0_set_clockgating,
+	.get_clockgating = mmhub_v4_2_0_get_clockgating,
+};
+
+static int mmhub_v4_2_0_xcp_resume(void *handle, uint32_t inst_mask)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	bool value;
+
+	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
+		value = false;
+	else
+		value = true;
+
+	mmhub_v4_2_0_mid_set_fault_enable_default(adev, value, inst_mask);
+
+	if (!amdgpu_sriov_vf(adev))
+		return mmhub_v4_2_0_mid_gart_enable(adev, inst_mask);
+
+	return 0;
+}
+
+static int mmhub_v4_2_0_xcp_suspend(void *handle, uint32_t inst_mask)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	if (!amdgpu_sriov_vf(adev))
+		mmhub_v4_2_0_mid_gart_disable(adev, inst_mask);
+
+	return 0;
+}
+
+struct amdgpu_xcp_ip_funcs mmhub_v4_2_0_xcp_funcs = {
+	.suspend = &mmhub_v4_2_0_xcp_suspend,
+	.resume = &mmhub_v4_2_0_xcp_resume
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.h b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.h
new file mode 100644
index 000000000000..4ea6de7ac7ae
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __MMHUB_V4_2_0_H__
+#define __MMHUB_V4_2_0_H__
+
+extern const struct amdgpu_mmhub_funcs mmhub_v4_2_0_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c
index e7cd07383d56..78408b257afc 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c
@@ -209,6 +209,9 @@ static int xgpu_nv_send_access_requests_with_param(struct amdgpu_device *adev,
 	case IDH_REQ_RAS_CHK_CRITI:
 		event = IDH_REQ_RAS_CHK_CRITI_READY;
 		break;
+	case IDH_REQ_RAS_REMOTE_CMD:
+		event = IDH_REQ_RAS_REMOTE_CMD_READY;
+		break;
 	default:
 		break;
 	}
@@ -585,6 +588,13 @@ static int xgpu_nv_check_vf_critical_region(struct amdgpu_device *adev, u64 addr
 		adev, IDH_REQ_RAS_CHK_CRITI, addr_hi, addr_lo, 0);
 }
 
+static int xgpu_nv_req_remote_ras_cmd(struct amdgpu_device *adev,
+		u32 param1, u32 param2, u32 param3)
+{
+	return xgpu_nv_send_access_requests_with_param(
+		adev, IDH_REQ_RAS_REMOTE_CMD, param1, param2, param3);
+}
+
 const struct amdgpu_virt_ops xgpu_nv_virt_ops = {
 	.req_full_gpu	= xgpu_nv_request_full_gpu_access,
 	.rel_full_gpu	= xgpu_nv_release_full_gpu_access,
@@ -598,5 +608,6 @@ const struct amdgpu_virt_ops xgpu_nv_virt_ops = {
 	.req_ras_err_count = xgpu_nv_req_ras_err_count,
 	.req_ras_cper_dump = xgpu_nv_req_ras_cper_dump,
 	.req_bad_pages = xgpu_nv_req_ras_bad_pages,
-	.req_ras_chk_criti = xgpu_nv_check_vf_critical_region
+	.req_ras_chk_criti = xgpu_nv_check_vf_critical_region,
+	.req_remote_ras_cmd = xgpu_nv_req_remote_ras_cmd,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.h b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.h
index c1083e5e41e0..dc57a4f697ee 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.h
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.h
@@ -43,7 +43,8 @@ enum idh_request {
 	IDH_REQ_RAS_ERROR_COUNT = 203,
 	IDH_REQ_RAS_CPER_DUMP = 204,
 	IDH_REQ_RAS_BAD_PAGES = 205,
-	IDH_REQ_RAS_CHK_CRITI = 206
+	IDH_REQ_RAS_CHK_CRITI = 206,
+	IDH_REQ_RAS_REMOTE_CMD = 207,
 };
 
 enum idh_event {
@@ -64,6 +65,7 @@ enum idh_event {
 	IDH_RAS_BAD_PAGES_NOTIFICATION = 16,
 	IDH_UNRECOV_ERR_NOTIFICATION = 17,
 	IDH_REQ_RAS_CHK_CRITI_READY = 18,
+	IDH_REQ_RAS_REMOTE_CMD_READY = 19,
 
 	IDH_TEXT_MESSAGE = 255,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/nv.c b/drivers/gpu/drm/amd/amdgpu/nv.c
index 50e77d9b30af..f17c3839aea1 100644
--- a/drivers/gpu/drm/amd/amdgpu/nv.c
+++ b/drivers/gpu/drm/amd/amdgpu/nv.c
@@ -593,10 +593,6 @@ static void nv_init_doorbell_index(struct amdgpu_device *adev)
 	adev->doorbell_index.sdma_doorbell_range = 20;
 }
 
-static void nv_pre_asic_init(struct amdgpu_device *adev)
-{
-}
-
 static int nv_update_umd_stable_pstate(struct amdgpu_device *adev,
 				       bool enter)
 {
@@ -630,7 +626,6 @@ static const struct amdgpu_asic_funcs nv_asic_funcs = {
 	.need_reset_on_init = &nv_need_reset_on_init,
 	.get_pcie_replay_count = &amdgpu_nbio_get_pcie_replay_count,
 	.supports_baco = &amdgpu_dpm_is_baco_supported,
-	.pre_asic_init = &nv_pre_asic_init,
 	.update_umd_stable_pstate = &nv_update_umd_stable_pstate,
 	.query_video_codecs = &nv_query_video_codecs,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
index 73f87131a7e9..e8f768638fd5 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
+++ b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
@@ -299,6 +299,8 @@ enum psp_gfx_fw_type {
 	GFX_FW_TYPE_RS64_MEC_P1_STACK               = 95,   /* RS64 MEC stack P1        SOC21   */
 	GFX_FW_TYPE_RS64_MEC_P2_STACK               = 96,   /* RS64 MEC stack P2        SOC21   */
 	GFX_FW_TYPE_RS64_MEC_P3_STACK               = 97,   /* RS64 MEC stack P3        SOC21   */
+	GFX_FW_TYPE_RLX6_UCODE_CORE1                = 98,   /* RLCV_IRAM                MI      */
+	GFX_FW_TYPE_RLX6_DRAM_BOOT_CORE1            = 99,   /* RLCV DRAM BOOT           MI      */
 	GFX_FW_TYPE_VPEC_FW1                        = 100,  /* VPEC FW1 To Save         VPE     */
 	GFX_FW_TYPE_VPEC_FW2                        = 101,  /* VPEC FW2 To Save         VPE     */
 	GFX_FW_TYPE_VPE                             = 102,
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c b/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c
new file mode 100644
index 000000000000..5249f5bd2a10
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c
@@ -0,0 +1,342 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include <drm/drm_drv.h>
+#include <linux/vmalloc.h>
+#include "amdgpu.h"
+#include "amdgpu_psp.h"
+#include "amdgpu_ucode.h"
+#include "soc15_common.h"
+#include "psp_v15_0_8.h"
+
+#include "mp/mp_15_0_8_offset.h"
+#include "mp/mp_15_0_8_sh_mask.h"
+
+MODULE_FIRMWARE("amdgpu/psp_15_0_8_toc.bin");
+
+static int psp_v15_0_8_init_microcode(struct psp_context *psp)
+{
+	struct amdgpu_device *adev = psp->adev;
+	char ucode_prefix[30];
+	int err = 0;
+
+	amdgpu_ucode_ip_version_decode(adev, MP0_HWIP, ucode_prefix, sizeof(ucode_prefix));
+
+	err = psp_init_toc_microcode(psp, ucode_prefix);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static int psp_v15_0_8_ring_stop(struct psp_context *psp,
+			       enum psp_ring_type ring_type)
+{
+	int ret = 0;
+	struct amdgpu_device *adev = psp->adev;
+
+	if (amdgpu_sriov_vf(adev)) {
+		/* Write the ring destroy command*/
+		WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_101,
+			     GFX_CTRL_CMD_ID_DESTROY_GPCOM_RING);
+		/* there might be handshake issue with hardware which needs delay */
+		mdelay(20);
+		/* Wait for response flag (bit 31) */
+		ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, regMPASP_SMN_C2PMSG_101),
+				   0x80000000, 0x80000000, false);
+	} else {
+		/* Write the ring destroy command*/
+		WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_64,
+			     GFX_CTRL_CMD_ID_DESTROY_RINGS);
+		/* there might be handshake issue with hardware which needs delay */
+		mdelay(20);
+		/* Wait for response flag (bit 31) */
+		ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, regMPASP_SMN_C2PMSG_64),
+				   0x80000000, 0x80000000, false);
+	}
+
+	return ret;
+}
+
+static int psp_v15_0_8_ring_create(struct psp_context *psp,
+				 enum psp_ring_type ring_type)
+{
+	int ret = 0;
+	unsigned int psp_ring_reg = 0;
+	struct psp_ring *ring = &psp->km_ring;
+	struct amdgpu_device *adev = psp->adev;
+
+	if (amdgpu_sriov_vf(adev)) {
+		ret = psp_v15_0_8_ring_stop(psp, ring_type);
+		if (ret) {
+			DRM_ERROR("psp_v14_0_ring_stop_sriov failed!\n");
+			return ret;
+		}
+
+		/* Write low address of the ring to C2PMSG_102 */
+		psp_ring_reg = lower_32_bits(ring->ring_mem_mc_addr);
+		WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_102, psp_ring_reg);
+		/* Write high address of the ring to C2PMSG_103 */
+		psp_ring_reg = upper_32_bits(ring->ring_mem_mc_addr);
+		WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_103, psp_ring_reg);
+
+		/* Write the ring initialization command to C2PMSG_101 */
+		WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_101,
+			     GFX_CTRL_CMD_ID_INIT_GPCOM_RING);
+
+		/* there might be handshake issue with hardware which needs delay */
+		mdelay(20);
+
+		/* Wait for response flag (bit 31) in C2PMSG_101 */
+		ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, regMPASP_SMN_C2PMSG_101),
+				   0x80000000, 0x8000FFFF, false);
+
+	} else {
+		/* Wait for sOS ready for ring creation */
+		ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, regMPASP_SMN_C2PMSG_64),
+				   0x80000000, 0x80000000, false);
+		if (ret) {
+			DRM_ERROR("Failed to wait for trust OS ready for ring creation\n");
+			return ret;
+		}
+
+		/* Write low address of the ring to C2PMSG_69 */
+		psp_ring_reg = lower_32_bits(ring->ring_mem_mc_addr);
+		WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_69, psp_ring_reg);
+		/* Write high address of the ring to C2PMSG_70 */
+		psp_ring_reg = upper_32_bits(ring->ring_mem_mc_addr);
+		WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_70, psp_ring_reg);
+		/* Write size of ring to C2PMSG_71 */
+		psp_ring_reg = ring->ring_size;
+		WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_71, psp_ring_reg);
+		/* Write the ring initialization command to C2PMSG_64 */
+		psp_ring_reg = ring_type;
+		psp_ring_reg = psp_ring_reg << 16;
+		WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_64, psp_ring_reg);
+
+		/* there might be handshake issue with hardware which needs delay */
+		mdelay(20);
+
+		/* Wait for response flag (bit 31) in C2PMSG_64 */
+		ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, regMPASP_SMN_C2PMSG_64),
+				   0x80000000, 0x8000FFFF, false);
+	}
+
+	return ret;
+}
+
+static int psp_v15_0_8_ring_destroy(struct psp_context *psp,
+				  enum psp_ring_type ring_type)
+{
+	int ret = 0;
+	struct psp_ring *ring = &psp->km_ring;
+	struct amdgpu_device *adev = psp->adev;
+
+	ret = psp_v15_0_8_ring_stop(psp, ring_type);
+	if (ret)
+		DRM_ERROR("Fail to stop psp ring\n");
+
+	amdgpu_bo_free_kernel(&adev->firmware.rbuf,
+			      &ring->ring_mem_mc_addr,
+			      (void **)&ring->ring_mem);
+
+	return ret;
+}
+
+static uint32_t psp_v15_0_8_ring_get_wptr(struct psp_context *psp)
+{
+	uint32_t data;
+	struct amdgpu_device *adev = psp->adev;
+
+	if (amdgpu_sriov_vf(adev))
+		data = RREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_102);
+	else
+		data = RREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_67);
+
+	return data;
+}
+
+static void psp_v15_0_8_ring_set_wptr(struct psp_context *psp, uint32_t value)
+{
+	struct amdgpu_device *adev = psp->adev;
+
+	if (amdgpu_sriov_vf(adev)) {
+		WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_102, value);
+		WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_101,
+			     GFX_CTRL_CMD_ID_CONSUME_CMD);
+	} else
+		WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_67, value);
+}
+
+static int psp_v15_0_8_get_fw_type(struct amdgpu_firmware_info *ucode,
+				   enum psp_gfx_fw_type *type)
+{
+	switch (ucode->ucode_id) {
+	case AMDGPU_UCODE_ID_CAP:
+		*type = GFX_FW_TYPE_CAP;
+		break;
+	case AMDGPU_UCODE_ID_SDMA0:
+		*type = GFX_FW_TYPE_SDMA0;
+		break;
+	case AMDGPU_UCODE_ID_SDMA1:
+		*type = GFX_FW_TYPE_SDMA1;
+		break;
+	case AMDGPU_UCODE_ID_SDMA2:
+		*type = GFX_FW_TYPE_SDMA2;
+		break;
+	case AMDGPU_UCODE_ID_SDMA3:
+		*type = GFX_FW_TYPE_SDMA3;
+		break;
+	case AMDGPU_UCODE_ID_SDMA4:
+		*type = GFX_FW_TYPE_SDMA4;
+		break;
+	case AMDGPU_UCODE_ID_SDMA5:
+		*type = GFX_FW_TYPE_SDMA5;
+		break;
+	case AMDGPU_UCODE_ID_SDMA6:
+		*type = GFX_FW_TYPE_SDMA6;
+		break;
+	case AMDGPU_UCODE_ID_SDMA7:
+		*type = GFX_FW_TYPE_SDMA7;
+		break;
+	case AMDGPU_UCODE_ID_CP_MES:
+		*type = GFX_FW_TYPE_RS64_MES;
+		break;
+	case AMDGPU_UCODE_ID_CP_MES_DATA:
+		*type = GFX_FW_TYPE_RS64_MES_STACK;
+		break;
+	case AMDGPU_UCODE_ID_CP_MES1:
+		*type = GFX_FW_TYPE_RS64_KIQ;
+		break;
+	case AMDGPU_UCODE_ID_CP_MES1_DATA:
+		*type = GFX_FW_TYPE_RS64_KIQ_STACK;
+		break;
+	case AMDGPU_UCODE_ID_RLC_P:
+		*type = GFX_FW_TYPE_RLC_P;
+		break;
+	case AMDGPU_UCODE_ID_RLC_V:
+		*type = GFX_FW_TYPE_RLC_V;
+		break;
+	case AMDGPU_UCODE_ID_RLC_G:
+		*type = GFX_FW_TYPE_RLC_G;
+		break;
+	case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
+		*type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
+		break;
+	case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
+		*type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
+		break;
+	case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
+		*type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
+		break;
+	case AMDGPU_UCODE_ID_RLC_IRAM:
+		*type = GFX_FW_TYPE_RLC_IRAM;
+		break;
+	case AMDGPU_UCODE_ID_RLC_DRAM:
+		*type = GFX_FW_TYPE_RLC_DRAM_BOOT;
+		break;
+	case AMDGPU_UCODE_ID_RLC_IRAM_1:
+		*type = GFX_FW_TYPE_RLX6_UCODE_CORE1;
+		break;
+	case AMDGPU_UCODE_ID_RLC_DRAM_1:
+		*type = GFX_FW_TYPE_RLX6_DRAM_BOOT_CORE1;
+		break;
+	case AMDGPU_UCODE_ID_SMC:
+		*type = GFX_FW_TYPE_SMU;
+		break;
+	case AMDGPU_UCODE_ID_PPTABLE:
+		*type = GFX_FW_TYPE_PPTABLE;
+		break;
+	case AMDGPU_UCODE_ID_VCN:
+		*type = GFX_FW_TYPE_VCN;
+		break;
+	case AMDGPU_UCODE_ID_VCN1:
+		*type = GFX_FW_TYPE_VCN1;
+		break;
+	case AMDGPU_UCODE_ID_VCN0_RAM:
+		*type = GFX_FW_TYPE_VCN0_RAM;
+		break;
+	case AMDGPU_UCODE_ID_VCN1_RAM:
+		*type = GFX_FW_TYPE_VCN1_RAM;
+		break;
+	case AMDGPU_UCODE_ID_SDMA_UCODE_TH0:
+	case AMDGPU_UCODE_ID_SDMA_RS64:
+		*type = GFX_FW_TYPE_SDMA0;
+		break;
+	case AMDGPU_UCODE_ID_SDMA_UCODE_TH1:
+		*type = GFX_FW_TYPE_SDMA_UCODE_TH1;
+		break;
+	case AMDGPU_UCODE_ID_IMU_I:
+		*type = GFX_FW_TYPE_IMU_I;
+		break;
+	case AMDGPU_UCODE_ID_IMU_D:
+		*type = GFX_FW_TYPE_IMU_D;
+		break;
+	case AMDGPU_UCODE_ID_CP_RS64_MEC:
+		*type = GFX_FW_TYPE_RS64_MEC;
+		break;
+	case AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK:
+		*type = GFX_FW_TYPE_RS64_MEC_P0_STACK;
+		break;
+	case AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK:
+		*type = GFX_FW_TYPE_RS64_MEC_P1_STACK;
+		break;
+	case AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK:
+		*type = GFX_FW_TYPE_RS64_MEC_P2_STACK;
+		break;
+	case AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK:
+		*type = GFX_FW_TYPE_RS64_MEC_P3_STACK;
+		break;
+	case AMDGPU_UCODE_ID_UMSCH_MM_UCODE:
+		*type = GFX_FW_TYPE_UMSCH_UCODE;
+		break;
+	case AMDGPU_UCODE_ID_UMSCH_MM_DATA:
+		*type = GFX_FW_TYPE_UMSCH_DATA;
+		break;
+	case AMDGPU_UCODE_ID_UMSCH_MM_CMD_BUFFER:
+		*type = GFX_FW_TYPE_UMSCH_CMD_BUFFER;
+		break;
+	case AMDGPU_UCODE_ID_P2S_TABLE:
+		*type = GFX_FW_TYPE_P2S_TABLE;
+		break;
+	case AMDGPU_UCODE_ID_MAXIMUM:
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct psp_funcs psp_v15_0_8_funcs = {
+	.init_microcode = psp_v15_0_8_init_microcode,
+	.ring_create = psp_v15_0_8_ring_create,
+	.ring_stop = psp_v15_0_8_ring_stop,
+	.ring_destroy = psp_v15_0_8_ring_destroy,
+	.ring_get_wptr = psp_v15_0_8_ring_get_wptr,
+	.ring_set_wptr = psp_v15_0_8_ring_set_wptr,
+	.get_fw_type = psp_v15_0_8_get_fw_type,
+};
+
+void psp_v15_0_8_set_psp_funcs(struct psp_context *psp)
+{
+	psp->funcs = &psp_v15_0_8_funcs;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.h b/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.h
new file mode 100644
index 000000000000..6bb1bb517007
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __PSP_V15_0_8_H__
+#define __PSP_V15_0_8_H__
+
+#include "amdgpu_psp.h"
+
+void psp_v15_0_8_set_psp_funcs(struct psp_context *psp);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
index 217040044987..f38c2bdd01e3 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
@@ -1268,6 +1268,17 @@ static void sdma_v6_0_set_ras_funcs(struct amdgpu_device *adev)
 	}
 }
 
+/* all sizes are in bytes */
+#define SDMA6_CSA_SIZE       32
+#define SDMA6_CSA_ALIGNMENT  4
+
+static void sdma_v6_0_get_csa_info(struct amdgpu_device *adev,
+				   struct amdgpu_sdma_csa_info *csa_info)
+{
+	csa_info->size = SDMA6_CSA_SIZE;
+	csa_info->alignment = SDMA6_CSA_ALIGNMENT;
+}
+
 static int sdma_v6_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -1300,6 +1311,7 @@ static int sdma_v6_0_early_init(struct amdgpu_ip_block *ip_block)
 	sdma_v6_0_set_irq_funcs(adev);
 	sdma_v6_0_set_mqd_funcs(adev);
 	sdma_v6_0_set_ras_funcs(adev);
+	adev->sdma.get_csa_info = &sdma_v6_0_get_csa_info;
 
 	return 0;
 }
@@ -1586,7 +1598,7 @@ static int sdma_v6_0_reset_queue(struct amdgpu_ring *ring,
 
 	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
 
-	r = amdgpu_mes_reset_legacy_queue(adev, ring, vmid, true);
+	r = amdgpu_mes_reset_legacy_queue(adev, ring, vmid, true, 0);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
index 2b81344dcd66..d2a09e7a1bc2 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
@@ -816,7 +816,7 @@ static int sdma_v7_0_reset_queue(struct amdgpu_ring *ring,
 
 	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
 
-	r = amdgpu_mes_reset_legacy_queue(adev, ring, vmid, true);
+	r = amdgpu_mes_reset_legacy_queue(adev, ring, vmid, true, 0);
 	if (r)
 		return r;
 
@@ -1253,6 +1253,17 @@ static void sdma_v7_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
 	amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);
 }
 
+/* all sizes are in bytes */
+#define SDMA7_CSA_SIZE       32
+#define SDMA7_CSA_ALIGNMENT  4
+
+static void sdma_v7_0_get_csa_info(struct amdgpu_device *adev,
+				   struct amdgpu_sdma_csa_info *csa_info)
+{
+	csa_info->size = SDMA7_CSA_SIZE;
+	csa_info->alignment = SDMA7_CSA_ALIGNMENT;
+}
+
 static int sdma_v7_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -1286,6 +1297,7 @@ static int sdma_v7_0_early_init(struct amdgpu_ip_block *ip_block)
 	sdma_v7_0_set_vm_pte_funcs(adev);
 	sdma_v7_0_set_irq_funcs(adev);
 	sdma_v7_0_set_mqd_funcs(adev);
+	adev->sdma.get_csa_info = &sdma_v7_0_get_csa_info;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
new file mode 100644
index 000000000000..fe0e84b45cf4
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
@@ -0,0 +1,1813 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/delay.h>
+#include <linux/firmware.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
+#include "amdgpu.h"
+#include "amdgpu_ucode.h"
+#include "amdgpu_trace.h"
+
+#include "gc/gc_12_1_0_offset.h"
+#include "gc/gc_12_1_0_sh_mask.h"
+#include "ivsrcid/gfx/irqsrcs_gfx_12_1_0.h"
+
+#include "soc15_common.h"
+#include "soc15.h"
+#include "sdma_v7_1_0_pkt_open.h"
+#include "nbio_v4_3.h"
+#include "sdma_common.h"
+#include "sdma_v7_1.h"
+#include "v12_structs.h"
+#include "mes_userqueue.h"
+
+MODULE_FIRMWARE("amdgpu/sdma_7_1_0.bin");
+
+#define SDMA1_REG_OFFSET 0x600
+#define SDMA0_SDMA_IDX_0_END 0x450
+#define SDMA1_HYP_DEC_REG_OFFSET 0x30
+
+static const struct amdgpu_hwip_reg_entry sdma_reg_list_7_1[] = {
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_STATUS_REG),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_STATUS1_REG),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_STATUS2_REG),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_STATUS3_REG),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_STATUS4_REG),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_STATUS5_REG),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_STATUS6_REG),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_UCODE_REV),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_RB_RPTR_FETCH_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_RB_RPTR_FETCH),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_UTCL1_RD_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_UTCL1_WR_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_UTCL1_RD_XNACK0),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_UTCL1_RD_XNACK1),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_UTCL1_WR_XNACK0),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_UTCL1_WR_XNACK1),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_RB_CNTL),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_RB_RPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_RB_RPTR_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_RB_WPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_RB_WPTR_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_IB_OFFSET),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_IB_BASE_LO),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_IB_BASE_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_IB_CNTL),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_IB_RPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_IB_SUB_REMAIN),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_DUMMY_REG),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE_STATUS0),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_RB_CNTL),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_RB_RPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_RB_RPTR_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_RB_WPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_RB_WPTR_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_IB_OFFSET),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_IB_BASE_LO),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_IB_BASE_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_IB_RPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_IB_SUB_REMAIN),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_DUMMY_REG),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_RB_CNTL),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_RB_RPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_RB_RPTR_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_RB_WPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_RB_WPTR_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_IB_OFFSET),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_IB_BASE_LO),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_IB_BASE_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_IB_RPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_IB_SUB_REMAIN),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_DUMMY_REG),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_INT_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_VM_CNTL),
+	SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2),
+	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_CHICKEN_BITS),
+};
+
+static void sdma_v7_1_set_ring_funcs(struct amdgpu_device *adev);
+static void sdma_v7_1_set_buffer_funcs(struct amdgpu_device *adev);
+static void sdma_v7_1_set_vm_pte_funcs(struct amdgpu_device *adev);
+static void sdma_v7_1_set_irq_funcs(struct amdgpu_device *adev);
+static int sdma_v7_1_inst_start(struct amdgpu_device *adev,
+				uint32_t inst_mask);
+
+static u32 sdma_v7_1_get_reg_offset(struct amdgpu_device *adev, u32 instance, u32 internal_offset)
+{
+	u32 base;
+	u32 dev_inst = GET_INST(SDMA0, instance);
+	int xcc_id = adev->sdma.instance[instance].xcc_id;
+	int xcc_inst = dev_inst % adev->sdma.num_inst_per_xcc;
+
+	if (internal_offset >= SDMA0_SDMA_IDX_0_END) {
+		base = adev->reg_offset[GC_HWIP][xcc_id][1];
+		if (xcc_inst != 0)
+			internal_offset += SDMA1_HYP_DEC_REG_OFFSET * xcc_inst;
+	} else {
+		base = adev->reg_offset[GC_HWIP][xcc_id][0];
+		if (xcc_inst != 0)
+			internal_offset += SDMA1_REG_OFFSET * xcc_inst;
+	}
+
+	return base + internal_offset;
+}
+
+static unsigned sdma_v7_1_ring_init_cond_exec(struct amdgpu_ring *ring,
+					      uint64_t addr)
+{
+	unsigned ret;
+
+	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COND_EXE));
+	amdgpu_ring_write(ring, lower_32_bits(addr));
+	amdgpu_ring_write(ring, upper_32_bits(addr));
+	amdgpu_ring_write(ring, 1);
+	/* this is the offset we need patch later */
+	ret = ring->wptr & ring->buf_mask;
+	/* insert dummy here and patch it later */
+	amdgpu_ring_write(ring, 0);
+
+	return ret;
+}
+
+/**
+ * sdma_v7_1_ring_get_rptr - get the current read pointer
+ *
+ * @ring: amdgpu ring pointer
+ *
+ * Get the current rptr from the hardware.
+ */
+static uint64_t sdma_v7_1_ring_get_rptr(struct amdgpu_ring *ring)
+{
+	u64 *rptr;
+
+	/* XXX check if swapping is necessary on BE */
+	rptr = (u64 *)ring->rptr_cpu_addr;
+
+	DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr);
+	return ((*rptr) >> 2);
+}
+
+/**
+ * sdma_v7_1_ring_get_wptr - get the current write pointer
+ *
+ * @ring: amdgpu ring pointer
+ *
+ * Get the current wptr from the hardware.
+ */
+static uint64_t sdma_v7_1_ring_get_wptr(struct amdgpu_ring *ring)
+{
+	u64 wptr = 0;
+
+	if (ring->use_doorbell) {
+		/* XXX check if swapping is necessary on BE */
+		wptr = READ_ONCE(*((u64 *)ring->wptr_cpu_addr));
+		DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
+	}
+
+	return wptr >> 2;
+}
+
+/**
+ * sdma_v7_1_ring_set_wptr - commit the write pointer
+ *
+ * @ring: amdgpu ring pointer
+ *
+ * Write the wptr back to the hardware.
+ */
+static void sdma_v7_1_ring_set_wptr(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+
+	DRM_DEBUG("Setting write pointer\n");
+
+	if (ring->use_doorbell) {
+		DRM_DEBUG("Using doorbell -- "
+			  "wptr_offs == 0x%08x "
+			  "lower_32_bits(ring->wptr) << 2 == 0x%08x "
+			  "upper_32_bits(ring->wptr) << 2 == 0x%08x\n",
+			  ring->wptr_offs,
+			  lower_32_bits(ring->wptr << 2),
+			  upper_32_bits(ring->wptr << 2));
+		/* XXX check if swapping is necessary on BE */
+		atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
+			     ring->wptr << 2);
+		DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
+			  ring->doorbell_index, ring->wptr << 2);
+		WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
+	} else {
+		DRM_DEBUG("Not using doorbell -- "
+			  "regSDMA%i_GFX_RB_WPTR == 0x%08x "
+			  "regSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
+			  ring->me,
+			  lower_32_bits(ring->wptr << 2),
+			  ring->me,
+			  upper_32_bits(ring->wptr << 2));
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev,
+							     ring->me,
+							     regSDMA0_SDMA_QUEUE0_RB_WPTR),
+				lower_32_bits(ring->wptr << 2));
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev,
+							     ring->me,
+							     regSDMA0_SDMA_QUEUE0_RB_WPTR_HI),
+				upper_32_bits(ring->wptr << 2));
+	}
+}
+
+static void sdma_v7_1_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
+{
+	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
+	int i;
+
+	for (i = 0; i < count; i++)
+		if (sdma && sdma->burst_nop && (i == 0))
+			amdgpu_ring_write(ring, ring->funcs->nop |
+				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
+		else
+			amdgpu_ring_write(ring, ring->funcs->nop);
+}
+
+/**
+ * sdma_v7_1_ring_emit_ib - Schedule an IB on the DMA engine
+ *
+ * @ring: amdgpu ring pointer
+ * @job: job to retrieve vmid from
+ * @ib: IB object to schedule
+ * @flags: unused
+ *
+ * Schedule an IB in the DMA ring.
+ */
+static void sdma_v7_1_ring_emit_ib(struct amdgpu_ring *ring,
+				   struct amdgpu_job *job,
+				   struct amdgpu_ib *ib,
+				   uint32_t flags)
+{
+	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
+	uint64_t csa_mc_addr = amdgpu_sdma_get_csa_mc_addr(ring, vmid);
+
+	/* An IB packet must end on a 8 DW boundary--the next dword
+	 * must be on a 8-dword boundary. Our IB packet below is 6
+	 * dwords long, thus add x number of NOPs, such that, in
+	 * modular arithmetic,
+	 * wptr + 6 + x = 8k, k >= 0, which in C is,
+	 * (wptr + 6 + x) % 8 = 0.
+	 * The expression below, is a solution of x.
+	 */
+	sdma_v7_1_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);
+
+	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_INDIRECT) |
+			  SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
+	/* base must be 32 byte aligned */
+	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
+	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
+	amdgpu_ring_write(ring, ib->length_dw);
+	amdgpu_ring_write(ring, lower_32_bits(csa_mc_addr));
+	amdgpu_ring_write(ring, upper_32_bits(csa_mc_addr));
+}
+
+/**
+ * sdma_v7_1_ring_emit_mem_sync - flush the IB by graphics cache rinse
+ *
+ * @ring: amdgpu ring pointer
+ *
+ * flush the IB by graphics cache rinse.
+ */
+static void sdma_v7_1_ring_emit_mem_sync(struct amdgpu_ring *ring)
+{
+	uint32_t gcr_cntl = SDMA_GCR_GL2_INV | SDMA_GCR_GL2_WB | SDMA_GCR_GLM_INV |
+		SDMA_GCR_GL1_INV | SDMA_GCR_GLV_INV | SDMA_GCR_GLK_INV |
+		SDMA_GCR_GLI_INV(1);
+
+	/* flush entire cache L0/L1/L2, this can be optimized by performance requirement */
+	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_GCR_REQ));
+	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD1_BASE_VA_31_7(0));
+	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD2_BASE_VA_56_32(0));
+	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD3_GCR_CONTROL_18_0(gcr_cntl) |
+			  SDMA_PKT_GCR_REQ_PAYLOAD3_LIMIT_VA_15_7(0));
+	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD4_LIMIT_VA_47_16(0));
+	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD5_LIMIT_VA_56_48(0) |
+			  SDMA_PKT_GCR_REQ_PAYLOAD5_VMID(0));
+}
+
+
+/**
+ * sdma_v7_1_ring_emit_fence - emit a fence on the DMA ring
+ *
+ * @ring: amdgpu ring pointer
+ * @addr: address
+ * @seq: fence seq number
+ * @flags: fence flags
+ *
+ * Add a DMA fence packet to the ring to write
+ * the fence seq number and DMA trap packet to generate
+ * an interrupt if needed.
+ */
+static void sdma_v7_1_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
+				      unsigned flags)
+{
+	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
+	/* write the fence */
+	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) |
+			  SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */
+	/* zero in first two bits */
+	BUG_ON(addr & 0x3);
+	amdgpu_ring_write(ring, lower_32_bits(addr));
+	amdgpu_ring_write(ring, upper_32_bits(addr));
+	amdgpu_ring_write(ring, lower_32_bits(seq));
+
+	/* optionally write high bits as well */
+	if (write64bit) {
+		addr += 4;
+		amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) |
+				  SDMA_PKT_FENCE_HEADER_MTYPE(0x3));
+		/* zero in first two bits */
+		BUG_ON(addr & 0x3);
+		amdgpu_ring_write(ring, lower_32_bits(addr));
+		amdgpu_ring_write(ring, upper_32_bits(addr));
+		amdgpu_ring_write(ring, upper_32_bits(seq));
+	}
+
+	if (flags & AMDGPU_FENCE_FLAG_INT) {
+		/* generate an interrupt */
+		amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_TRAP));
+		amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0));
+	}
+}
+
+/**
+ * sdma_v7_1_inst_gfx_stop - stop the gfx async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ * @inst_mask: mask of dma engine instances to be disabled
+ *
+ * Stop the gfx async dma ring buffers.
+ */
+static void sdma_v7_1_inst_gfx_stop(struct amdgpu_device *adev,
+				    uint32_t inst_mask)
+{
+	u32 rb_cntl, ib_cntl;
+	int i;
+
+	for (i = 0; i < NUM_XCC(inst_mask); i++) {
+		rb_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_CNTL));
+		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, RB_ENABLE, 0);
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_CNTL), rb_cntl);
+		ib_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_IB_CNTL));
+		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_SDMA_QUEUE0_IB_CNTL, IB_ENABLE, 0);
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_IB_CNTL), ib_cntl);
+	}
+}
+
+/**
+ * sdma_v7_1_inst_rlc_stop - stop the compute async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ * @inst_mask: mask of dma engine instances to be disabled
+ *
+ * Stop the compute async dma queues.
+ */
+static void sdma_v7_1_inst_rlc_stop(struct amdgpu_device *adev,
+				    uint32_t inst_mask)
+{
+	/* XXX todo */
+}
+
+/**
+ * sdma_v7_1_inst_ctx_switch_enable - stop the async dma engines context switch
+ *
+ * @adev: amdgpu_device pointer
+ * @enable: enable/disable the DMA MEs context switch.
+ * @inst_mask: mask of dma engine instances to be enabled
+ *
+ * Halt or unhalt the async dma engines context switch.
+ */
+static void sdma_v7_1_inst_ctx_switch_enable(struct amdgpu_device *adev,
+					     bool enable, uint32_t inst_mask)
+{
+	int i;
+
+	for_each_inst(i, inst_mask) {
+		WREG32_SOC15_IP(GC,
+			sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_UTCL1_TIMEOUT), 0x80);
+	}
+}
+
+/**
+ * sdma_v7_1_inst_enable - stop the async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ * @enable: enable/disable the DMA MEs.
+ * @inst_mask: mask of dma engine instances to be enabled
+ *
+ * Halt or unhalt the async dma engines.
+ */
+static void sdma_v7_1_inst_enable(struct amdgpu_device *adev,
+				  bool enable, uint32_t inst_mask)
+{
+	u32 mcu_cntl;
+	int i;
+
+	if (!enable) {
+		sdma_v7_1_inst_gfx_stop(adev, inst_mask);
+		sdma_v7_1_inst_rlc_stop(adev, inst_mask);
+	}
+
+	if (amdgpu_sriov_vf(adev))
+		return;
+
+	for (i = 0; i < NUM_XCC(inst_mask); i++) {
+		mcu_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_MCU_CNTL));
+		mcu_cntl = REG_SET_FIELD(mcu_cntl, SDMA0_SDMA_MCU_CNTL, HALT, enable ? 0 : 1);
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_MCU_CNTL), mcu_cntl);
+	}
+}
+
+/**
+ * sdma_v7_1_gfx_resume_instance - start/restart a certain sdma engine
+ *
+ * @adev: amdgpu_device pointer
+ * @i: instance
+ * @restore: used to restore wptr when restart
+ *
+ * Set up the gfx DMA ring buffers and enable them. On restart, we will restore wptr and rptr.
+ * Return 0 for success.
+ */
+static int sdma_v7_1_gfx_resume_instance(struct amdgpu_device *adev, int i, bool restore)
+{
+	struct amdgpu_ring *ring;
+	u32 rb_cntl, ib_cntl;
+	u32 rb_bufsz;
+	u32 doorbell;
+	u32 doorbell_offset;
+	u32 temp;
+	u64 wptr_gpu_addr;
+	int r;
+
+	ring = &adev->sdma.instance[i].ring;
+
+	/* Set ring buffer size in dwords */
+	rb_bufsz = order_base_2(ring->ring_size / 4);
+	rb_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_CNTL));
+	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, RB_SIZE, rb_bufsz);
+#ifdef __BIG_ENDIAN
+	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, RB_SWAP_ENABLE, 1);
+	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL,
+				RPTR_WRITEBACK_SWAP_ENABLE, 1);
+#endif
+	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, RB_PRIV, 1);
+	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_CNTL), rb_cntl);
+
+	/* Initialize the ring buffer's read and write pointers */
+	if (restore) {
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_RPTR), lower_32_bits(ring->wptr << 2));
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_RPTR_HI), upper_32_bits(ring->wptr << 2));
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_WPTR), lower_32_bits(ring->wptr << 2));
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_WPTR_HI), upper_32_bits(ring->wptr << 2));
+	} else {
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_RPTR), 0);
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_RPTR_HI), 0);
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_WPTR), 0);
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_WPTR_HI), 0);
+	}
+	/* setup the wptr shadow polling */
+	wptr_gpu_addr = ring->wptr_gpu_addr;
+	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_LO),
+	       lower_32_bits(wptr_gpu_addr));
+	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_HI),
+	       upper_32_bits(wptr_gpu_addr));
+
+	/* set the wb address whether it's enabled or not */
+	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_RPTR_ADDR_HI),
+	       upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);
+	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_RPTR_ADDR_LO),
+	       lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC);
+
+	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1);
+	if (amdgpu_sriov_vf(adev))
+		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, WPTR_POLL_ENABLE, 1);
+	else
+		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, WPTR_POLL_ENABLE, 0);
+
+	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, MCU_WPTR_POLL_ENABLE, 1);
+
+	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_BASE), ring->gpu_addr >> 8);
+	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_BASE_HI), ring->gpu_addr >> 40);
+
+	if (!restore)
+		ring->wptr = 0;
+
+	/* before programing wptr to a less value, need set minor_ptr_update first */
+	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_MINOR_PTR_UPDATE), 1);
+
+	if (!amdgpu_sriov_vf(adev)) { /* only bare-metal use register write for wptr */
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_WPTR), lower_32_bits(ring->wptr) << 2);
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_WPTR_HI), upper_32_bits(ring->wptr) << 2);
+	}
+
+	doorbell = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_DOORBELL));
+	doorbell_offset = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_DOORBELL_OFFSET));
+
+	if (ring->use_doorbell) {
+		doorbell = REG_SET_FIELD(doorbell, SDMA0_SDMA_QUEUE0_DOORBELL, ENABLE, 1);
+		doorbell_offset = REG_SET_FIELD(doorbell_offset, SDMA0_SDMA_QUEUE0_DOORBELL_OFFSET,
+				OFFSET, ring->doorbell_index);
+	} else {
+		doorbell = REG_SET_FIELD(doorbell, SDMA0_SDMA_QUEUE0_DOORBELL, ENABLE, 0);
+	}
+	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_DOORBELL), doorbell);
+	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_DOORBELL_OFFSET), doorbell_offset);
+
+	if (i == 0)
+		adev->nbio.funcs->sdma_doorbell_range(adev, i, ring->use_doorbell,
+					      ring->doorbell_index,
+					      adev->doorbell_index.sdma_doorbell_range * adev->sdma.num_instances);
+
+	if (amdgpu_sriov_vf(adev))
+		sdma_v7_1_ring_set_wptr(ring);
+
+	/* set minor_ptr_update to 0 after wptr programed */
+	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_MINOR_PTR_UPDATE), 0);
+
+	/* Set up sdma hang watchdog */
+	temp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_WATCHDOG_CNTL));
+	/* 100ms per unit */
+	temp = REG_SET_FIELD(temp, SDMA0_SDMA_WATCHDOG_CNTL, QUEUE_HANG_COUNT,
+			     max(adev->usec_timeout/100000, 1));
+	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_WATCHDOG_CNTL), temp);
+
+	/* Set up RESP_MODE to non-copy addresses */
+	temp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_UTCL1_CNTL));
+	temp = REG_SET_FIELD(temp, SDMA0_SDMA_UTCL1_CNTL, RESP_MODE, 3);
+	temp = REG_SET_FIELD(temp, SDMA0_SDMA_UTCL1_CNTL, REDO_DELAY, 9);
+	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_UTCL1_CNTL), temp);
+
+	/* program default cache read and write policy */
+	temp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_UTCL1_PAGE));
+	/* clean read policy and write policy bits */
+	temp &= 0xFF0FFF;
+	temp |= ((CACHE_READ_POLICY_L2__DEFAULT << 12) |
+		 (CACHE_WRITE_POLICY_L2__DEFAULT << 14));
+	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_UTCL1_PAGE), temp);
+
+	if (!amdgpu_sriov_vf(adev)) {
+		/* unhalt engine */
+		temp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_MCU_CNTL));
+		temp = REG_SET_FIELD(temp, SDMA0_SDMA_MCU_CNTL, HALT, 0);
+		temp = REG_SET_FIELD(temp, SDMA0_SDMA_MCU_CNTL, RESET, 0);
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_MCU_CNTL), temp);
+	}
+
+	/* enable DMA RB */
+	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, RB_ENABLE, 1);
+	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_CNTL), rb_cntl);
+
+	ib_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_IB_CNTL));
+	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_SDMA_QUEUE0_IB_CNTL, IB_ENABLE, 1);
+#ifdef __BIG_ENDIAN
+	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_SDMA_QUEUE0_IB_CNTL, IB_SWAP_ENABLE, 1);
+#endif
+	/* enable DMA IBs */
+	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_IB_CNTL), ib_cntl);
+	ring->sched.ready = true;
+
+	if (amdgpu_sriov_vf(adev)) { /* bare-metal sequence doesn't need below to lines */
+		sdma_v7_1_inst_ctx_switch_enable(adev, true, i);
+		sdma_v7_1_inst_enable(adev, true, i);
+	}
+
+	r = amdgpu_ring_test_helper(ring);
+	if (r)
+		ring->sched.ready = false;
+
+	return r;
+}
+
+/**
+ * sdma_v7_1_inst_gfx_resume - setup and start the async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ * @inst_mask: mask of dma engine instances to be enabled
+ *
+ * Set up the gfx DMA ring buffers and enable them.
+ * Returns 0 for success, error for failure.
+ */
+static int sdma_v7_1_inst_gfx_resume(struct amdgpu_device *adev,
+				     uint32_t inst_mask)
+{
+	int i, r;
+
+	for (i = 0; i < NUM_XCC(inst_mask); i++) {
+		r = sdma_v7_1_gfx_resume_instance(adev, i, false);
+		if (r)
+			return r;
+	}
+
+	return 0;
+
+}
+
+/**
+ * sdma_v7_1_inst_rlc_resume - setup and start the async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ * @inst_mask: mask of dma engine instances to be enabled
+ *
+ * Set up the compute DMA queues and enable them.
+ * Returns 0 for success, error for failure.
+ */
+static int sdma_v7_1_inst_rlc_resume(struct amdgpu_device *adev,
+				     uint32_t inst_mask)
+{
+	return 0;
+}
+
+static void sdma_v7_1_inst_free_ucode_buffer(struct amdgpu_device *adev,
+					     uint32_t inst_mask)
+{
+	int i;
+
+	for (i = 0; i < NUM_XCC(inst_mask); i++) {
+		amdgpu_bo_free_kernel(&adev->sdma.instance[i].sdma_fw_obj,
+				      &adev->sdma.instance[i].sdma_fw_gpu_addr,
+				      (void **)&adev->sdma.instance[i].sdma_fw_ptr);
+	}
+}
+
+/**
+ * sdma_v7_1_inst_load_microcode - load the sDMA ME ucode
+ *
+ * @adev: amdgpu_device pointer
+ * @inst_mask: mask of dma engine instances to be enabled
+ *
+ * Loads the sDMA0/1 ucode.
+ * Returns 0 for success, -EINVAL if the ucode is not available.
+ */
+static int sdma_v7_1_inst_load_microcode(struct amdgpu_device *adev,
+					 uint32_t inst_mask)
+{
+	const struct sdma_firmware_header_v3_0 *hdr;
+	const __le32 *fw_data;
+	u32 fw_size;
+	uint32_t tmp, sdma_status, ic_op_cntl;
+	int i, r, j;
+
+	/* halt the MEs */
+	sdma_v7_1_inst_enable(adev, false, inst_mask);
+
+	if (!adev->sdma.instance[0].fw)
+		return -EINVAL;
+
+	hdr = (const struct sdma_firmware_header_v3_0 *)
+		adev->sdma.instance[0].fw->data;
+	amdgpu_ucode_print_sdma_hdr(&hdr->header);
+
+	fw_data = (const __le32 *)(adev->sdma.instance[0].fw->data +
+			le32_to_cpu(hdr->ucode_offset_bytes));
+	fw_size = le32_to_cpu(hdr->ucode_size_bytes);
+
+	for (i = 0; i < NUM_XCC(inst_mask); i++) {
+		r = amdgpu_bo_create_reserved(adev, fw_size,
+					      PAGE_SIZE,
+					      AMDGPU_GEM_DOMAIN_VRAM,
+					      &adev->sdma.instance[i].sdma_fw_obj,
+					      &adev->sdma.instance[i].sdma_fw_gpu_addr,
+					      (void **)&adev->sdma.instance[i].sdma_fw_ptr);
+		if (r) {
+			dev_err(adev->dev, "(%d) failed to create sdma ucode bo\n", r);
+			return r;
+		}
+
+		memcpy(adev->sdma.instance[i].sdma_fw_ptr, fw_data, fw_size);
+
+		amdgpu_bo_kunmap(adev->sdma.instance[i].sdma_fw_obj);
+		amdgpu_bo_unreserve(adev->sdma.instance[i].sdma_fw_obj);
+
+		tmp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_IC_CNTL));
+		tmp = REG_SET_FIELD(tmp, SDMA0_SDMA_IC_CNTL, GPA, 0);
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_IC_CNTL), tmp);
+
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_IC_BASE_LO),
+			lower_32_bits(adev->sdma.instance[i].sdma_fw_gpu_addr));
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_IC_BASE_HI),
+			upper_32_bits(adev->sdma.instance[i].sdma_fw_gpu_addr));
+
+		tmp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_IC_OP_CNTL));
+		tmp = REG_SET_FIELD(tmp, SDMA0_SDMA_IC_OP_CNTL, PRIME_ICACHE, 1);
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_IC_OP_CNTL), tmp);
+
+		/* Wait for sdma ucode init complete */
+		for (j = 0; j < adev->usec_timeout; j++) {
+			ic_op_cntl = RREG32_SOC15_IP(GC,
+					sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_IC_OP_CNTL));
+			sdma_status = RREG32_SOC15_IP(GC,
+					sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_STATUS_REG));
+			if ((REG_GET_FIELD(ic_op_cntl, SDMA0_SDMA_IC_OP_CNTL, ICACHE_PRIMED) == 1) &&
+			    (REG_GET_FIELD(sdma_status, SDMA0_SDMA_STATUS_REG, UCODE_INIT_DONE) == 1))
+				break;
+			udelay(1);
+		}
+
+		if (j >= adev->usec_timeout) {
+			dev_err(adev->dev, "failed to init sdma ucode\n");
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int sdma_v7_1_soft_reset(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	uint32_t inst_mask;
+	u32 tmp;
+	int i;
+
+	inst_mask = adev->sdma.sdma_mask;
+	sdma_v7_1_inst_gfx_stop(adev, inst_mask);
+
+	for (i = 0; i < NUM_XCC(inst_mask); i++) {
+		//tmp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_FREEZE));
+		//tmp |= SDMA0_SDMA_FREEZE__FREEZE_MASK;
+		//WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_FREEZE), tmp);
+		tmp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_MCU_CNTL));
+		tmp |= SDMA0_SDMA_MCU_CNTL__HALT_MASK;
+		tmp |= SDMA0_SDMA_MCU_CNTL__RESET_MASK;
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_MCU_CNTL), tmp);
+
+		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_PREEMPT), 0);
+
+		udelay(100);
+
+		tmp = GRBM_SOFT_RESET__SOFT_RESET_SDMA0_MASK << i;
+		WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, tmp);
+		tmp = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
+
+		udelay(100);
+
+		WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, 0);
+		tmp = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
+
+		udelay(100);
+	}
+
+	return sdma_v7_1_inst_start(adev, inst_mask);
+}
+
+static bool sdma_v7_1_check_soft_reset(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	struct amdgpu_ring *ring;
+	int i, r;
+	long tmo = msecs_to_jiffies(1000);
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		ring = &adev->sdma.instance[i].ring;
+		r = amdgpu_ring_test_ib(ring, tmo);
+		if (r)
+			return true;
+	}
+
+	return false;
+}
+
+static int sdma_v7_1_reset_queue(struct amdgpu_ring *ring,
+				 unsigned int vmid,
+				 struct amdgpu_fence *timedout_fence)
+{
+	struct amdgpu_device *adev = ring->adev;
+	int r;
+
+	if (ring->me >= adev->sdma.num_instances) {
+		dev_err(adev->dev, "sdma instance not found\n");
+		return -EINVAL;
+	}
+
+	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
+
+	r = amdgpu_mes_reset_legacy_queue(adev, ring, vmid, true, 0);
+	if (r)
+		return r;
+
+	r = sdma_v7_1_gfx_resume_instance(adev, ring->me, true);
+	if (r)
+		return r;
+
+	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
+}
+
+/**
+ * sdma_v7_1_inst_start - setup and start the async dma engines
+ *
+ * @adev: amdgpu_device pointer
+ * @inst_mask: mask of dma engine instances to be enabled
+ *
+ * Set up the DMA engines and enable them.
+ * Returns 0 for success, error for failure.
+ */
+static int sdma_v7_1_inst_start(struct amdgpu_device *adev,
+				uint32_t inst_mask)
+{
+	int r = 0;
+
+	if (amdgpu_sriov_vf(adev)) {
+		sdma_v7_1_inst_ctx_switch_enable(adev, false, inst_mask);
+		sdma_v7_1_inst_enable(adev, false, inst_mask);
+
+		/* set RB registers */
+		r = sdma_v7_1_inst_gfx_resume(adev, inst_mask);
+		return r;
+	}
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
+		r = sdma_v7_1_inst_load_microcode(adev, inst_mask);
+		if (r) {
+			sdma_v7_1_inst_free_ucode_buffer(adev, inst_mask);
+			return r;
+		}
+
+		if (amdgpu_emu_mode == 1)
+			msleep(1000);
+	}
+
+	/* unhalt the MEs */
+	sdma_v7_1_inst_enable(adev, true, inst_mask);
+	/* enable sdma ring preemption */
+	sdma_v7_1_inst_ctx_switch_enable(adev, true, inst_mask);
+
+	/* start the gfx rings and rlc compute queues */
+	r = sdma_v7_1_inst_gfx_resume(adev, inst_mask);
+	if (r)
+		return r;
+	r = sdma_v7_1_inst_rlc_resume(adev, inst_mask);
+
+	return r;
+}
+
+static int sdma_v7_1_mqd_init(struct amdgpu_device *adev, void *mqd,
+			      struct amdgpu_mqd_prop *prop)
+{
+	struct v12_sdma_mqd *m = mqd;
+	uint64_t wb_gpu_addr;
+
+	m->sdmax_rlcx_rb_cntl =
+		order_base_2(prop->queue_size / 4) << SDMA0_SDMA_QUEUE0_RB_CNTL__RB_SIZE__SHIFT |
+		1 << SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
+		4 << SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT |
+		1 << SDMA0_SDMA_QUEUE0_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT;
+
+	m->sdmax_rlcx_rb_base = lower_32_bits(prop->hqd_base_gpu_addr >> 8);
+	m->sdmax_rlcx_rb_base_hi = upper_32_bits(prop->hqd_base_gpu_addr >> 8);
+
+	wb_gpu_addr = prop->wptr_gpu_addr;
+	m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits(wb_gpu_addr);
+	m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr);
+
+	wb_gpu_addr = prop->rptr_gpu_addr;
+	m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits(wb_gpu_addr);
+	m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits(wb_gpu_addr);
+
+	m->sdmax_rlcx_ib_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, 0,
+							regSDMA0_SDMA_QUEUE0_IB_CNTL));
+
+	m->sdmax_rlcx_doorbell_offset =
+		prop->doorbell_index << SDMA0_SDMA_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
+
+	m->sdmax_rlcx_doorbell = REG_SET_FIELD(0, SDMA0_SDMA_QUEUE0_DOORBELL, ENABLE, 1);
+
+	m->sdmax_rlcx_doorbell_log = 0;
+	m->sdmax_rlcx_rb_aql_cntl = 0x4000;	//regSDMA0_SDMA_QUEUE0_RB_AQL_CNTL_DEFAULT;
+	m->sdmax_rlcx_dummy_reg = 0xf;	//regSDMA0_SDMA_QUEUE0_DUMMY_REG_DEFAULT;
+
+	m->sdmax_rlcx_csa_addr_lo = lower_32_bits(prop->csa_addr);
+	m->sdmax_rlcx_csa_addr_hi = upper_32_bits(prop->csa_addr);
+
+	return 0;
+}
+
+static void sdma_v7_1_set_mqd_funcs(struct amdgpu_device *adev)
+{
+	adev->mqds[AMDGPU_HW_IP_DMA].mqd_size = sizeof(struct v12_sdma_mqd);
+	adev->mqds[AMDGPU_HW_IP_DMA].init_mqd = sdma_v7_1_mqd_init;
+}
+
+/**
+ * sdma_v7_1_ring_test_ring - simple async dma engine test
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ *
+ * Test the DMA engine by writing using it to write an
+ * value to memory.
+ * Returns 0 for success, error for failure.
+ */
+static int sdma_v7_1_ring_test_ring(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+	unsigned i;
+	unsigned index;
+	int r;
+	u32 tmp;
+	u64 gpu_addr;
+
+	tmp = 0xCAFEDEAD;
+
+	r = amdgpu_device_wb_get(adev, &index);
+	if (r) {
+		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
+		return r;
+	}
+
+	gpu_addr = adev->wb.gpu_addr + (index * 4);
+	adev->wb.wb[index] = cpu_to_le32(tmp);
+
+	r = amdgpu_ring_alloc(ring, 5);
+	if (r) {
+		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
+		amdgpu_device_wb_free(adev, index);
+		return r;
+	}
+
+	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
+			  SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
+	amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
+	amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
+	amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
+	amdgpu_ring_write(ring, 0xDEADBEEF);
+	amdgpu_ring_commit(ring);
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		tmp = le32_to_cpu(adev->wb.wb[index]);
+		if (tmp == 0xDEADBEEF)
+			break;
+		if (amdgpu_emu_mode == 1)
+			msleep(1);
+		else
+			udelay(1);
+	}
+
+	if (i >= adev->usec_timeout)
+		r = -ETIMEDOUT;
+
+	amdgpu_device_wb_free(adev, index);
+
+	return r;
+}
+
+/**
+ * sdma_v7_1_ring_test_ib - test an IB on the DMA engine
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
+ *
+ * Test a simple IB in the DMA ring.
+ * Returns 0 on success, error on failure.
+ */
+static int sdma_v7_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
+{
+	struct amdgpu_device *adev = ring->adev;
+	struct amdgpu_ib ib;
+	struct dma_fence *f = NULL;
+	unsigned index;
+	long r;
+	u32 tmp = 0;
+	u64 gpu_addr;
+
+	tmp = 0xCAFEDEAD;
+	memset(&ib, 0, sizeof(ib));
+
+	r = amdgpu_device_wb_get(adev, &index);
+	if (r) {
+		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
+		return r;
+	}
+
+	gpu_addr = adev->wb.gpu_addr + (index * 4);
+	adev->wb.wb[index] = cpu_to_le32(tmp);
+
+	r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
+	if (r) {
+		DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
+		goto err0;
+	}
+
+	ib.ptr[0] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
+		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
+	ib.ptr[1] = lower_32_bits(gpu_addr);
+	ib.ptr[2] = upper_32_bits(gpu_addr);
+	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
+	ib.ptr[4] = 0xDEADBEEF;
+	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib.length_dw = 8;
+
+	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
+	if (r)
+		goto err1;
+
+	r = dma_fence_wait_timeout(f, false, timeout);
+	if (r == 0) {
+		DRM_ERROR("amdgpu: IB test timed out\n");
+		r = -ETIMEDOUT;
+		goto err1;
+	} else if (r < 0) {
+		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
+		goto err1;
+	}
+
+	tmp = le32_to_cpu(adev->wb.wb[index]);
+
+	if (tmp == 0xDEADBEEF)
+		r = 0;
+	else
+		r = -EINVAL;
+
+err1:
+	amdgpu_ib_free(&ib, NULL);
+	dma_fence_put(f);
+err0:
+	amdgpu_device_wb_free(adev, index);
+	return r;
+}
+
+
+/**
+ * sdma_v7_1_vm_copy_pte - update PTEs by copying them from the GART
+ *
+ * @ib: indirect buffer to fill with commands
+ * @pe: addr of the page entry
+ * @src: src addr to copy from
+ * @count: number of page entries to update
+ *
+ * Update PTEs by copying them from the GART using sDMA.
+ */
+static void sdma_v7_1_vm_copy_pte(struct amdgpu_ib *ib,
+				  uint64_t pe, uint64_t src,
+				  unsigned count)
+{
+	unsigned bytes = count * 8;
+
+	ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COPY) |
+		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
+
+	ib->ptr[ib->length_dw++] = bytes - 1;
+	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
+	ib->ptr[ib->length_dw++] = lower_32_bits(src);
+	ib->ptr[ib->length_dw++] = upper_32_bits(src);
+	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+
+}
+
+/**
+ * sdma_v7_1_vm_write_pte - update PTEs by writing them manually
+ *
+ * @ib: indirect buffer to fill with commands
+ * @pe: addr of the page entry
+ * @value: dst addr to write into pe
+ * @count: number of page entries to update
+ * @incr: increase next addr by incr bytes
+ *
+ * Update PTEs by writing them manually using sDMA.
+ */
+static void sdma_v7_1_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
+				   uint64_t value, unsigned count,
+				   uint32_t incr)
+{
+	unsigned ndw = count * 2;
+
+	ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
+		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
+	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+	ib->ptr[ib->length_dw++] = ndw - 1;
+	for (; ndw > 0; ndw -= 2) {
+		ib->ptr[ib->length_dw++] = lower_32_bits(value);
+		ib->ptr[ib->length_dw++] = upper_32_bits(value);
+		value += incr;
+	}
+}
+
+/**
+ * sdma_v7_1_vm_set_pte_pde - update the page tables using sDMA
+ *
+ * @ib: indirect buffer to fill with commands
+ * @pe: addr of the page entry
+ * @addr: dst addr to write into pe
+ * @count: number of page entries to update
+ * @incr: increase next addr by incr bytes
+ * @flags: access flags
+ *
+ * Update the page tables using sDMA.
+ */
+static void sdma_v7_1_vm_set_pte_pde(struct amdgpu_ib *ib,
+				     uint64_t pe,
+				     uint64_t addr, unsigned count,
+				     uint32_t incr, uint64_t flags)
+{
+	/* for physically contiguous pages (vram) */
+	u32 header = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_PTEPDE);
+
+	if (amdgpu_mtype_local)
+		header |= SDMA_PKT_PTEPDE_COPY_HEADER_MTYPE(0x3);
+	else
+		header |= (SDMA_PKT_PTEPDE_COPY_HEADER_MTYPE(0x2) |
+			   SDMA_PKT_PTEPDE_COPY_HEADER_SNOOP(0x1) |
+			   SDMA_PKT_PTEPDE_COPY_HEADER_SCOPE(0x3));
+
+	ib->ptr[ib->length_dw++] = header;
+	ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
+	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
+	ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
+	ib->ptr[ib->length_dw++] = upper_32_bits(flags);
+	ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
+	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
+	ib->ptr[ib->length_dw++] = incr; /* increment size */
+	ib->ptr[ib->length_dw++] = 0;
+	ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
+}
+
+/**
+ * sdma_v7_1_ring_pad_ib - pad the IB
+ *
+ * @ring: amdgpu ring pointer
+ * @ib: indirect buffer to fill with padding
+ *
+ * Pad the IB with NOPs to a boundary multiple of 8.
+ */
+static void sdma_v7_1_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
+{
+	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
+	u32 pad_count;
+	int i;
+
+	pad_count = (-ib->length_dw) & 0x7;
+	for (i = 0; i < pad_count; i++)
+		if (sdma && sdma->burst_nop && (i == 0))
+			ib->ptr[ib->length_dw++] =
+				SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_NOP) |
+				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
+		else
+			ib->ptr[ib->length_dw++] =
+				SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_NOP);
+}
+
+/**
+ * sdma_v7_1_ring_emit_pipeline_sync - sync the pipeline
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Make sure all previous operations are completed (CIK).
+ */
+static void sdma_v7_1_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
+{
+	uint32_t seq = ring->fence_drv.sync_seq;
+	uint64_t addr = ring->fence_drv.gpu_addr;
+
+	/* wait for idle */
+	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_POLL_REGMEM) |
+			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */
+			  SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1));
+	amdgpu_ring_write(ring, addr & 0xfffffffc);
+	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
+	amdgpu_ring_write(ring, seq); /* reference */
+	amdgpu_ring_write(ring, 0xffffffff); /* mask */
+	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
+			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */
+}
+
+/**
+ * sdma_v7_1_ring_emit_vm_flush - vm flush using sDMA
+ *
+ * @ring: amdgpu_ring pointer
+ * @vmid: vmid number to use
+ * @pd_addr: address
+ *
+ * Update the page table base and flush the VM TLB
+ * using sDMA.
+ */
+static void sdma_v7_1_ring_emit_vm_flush(struct amdgpu_ring *ring,
+					 unsigned vmid, uint64_t pd_addr)
+{
+	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
+}
+
+static void sdma_v7_1_ring_emit_wreg(struct amdgpu_ring *ring,
+				     uint32_t reg, uint32_t val)
+{
+	/* SRBM WRITE command will not support on sdma v7.
+	 * Use Register WRITE command instead, which OPCODE is same as SRBM WRITE
+	 */
+	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_SRBM_WRITE));
+	amdgpu_ring_write(ring, reg << 2);
+	amdgpu_ring_write(ring, val);
+}
+
+static void sdma_v7_1_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
+					 uint32_t val, uint32_t mask)
+{
+	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_POLL_REGMEM) |
+			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* equal */
+	amdgpu_ring_write(ring, reg << 2);
+	amdgpu_ring_write(ring, 0);
+	amdgpu_ring_write(ring, val); /* reference */
+	amdgpu_ring_write(ring, mask); /* mask */
+	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
+			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10));
+}
+
+static void sdma_v7_1_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
+						   uint32_t reg0, uint32_t reg1,
+						   uint32_t ref, uint32_t mask)
+{
+	amdgpu_ring_emit_wreg(ring, reg0, ref);
+	/* wait for a cycle to reset vm_inv_eng*_ack */
+	amdgpu_ring_emit_reg_wait(ring, reg0, 0, 0);
+	amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);
+}
+
+static int sdma_v7_1_early_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int r;
+
+	r = amdgpu_sdma_init_microcode(adev, 0, true);
+	if (r) {
+		DRM_ERROR("Failed to init sdma firmware!\n");
+		return r;
+	}
+
+	sdma_v7_1_set_ring_funcs(adev);
+	sdma_v7_1_set_buffer_funcs(adev);
+	sdma_v7_1_set_vm_pte_funcs(adev);
+	sdma_v7_1_set_irq_funcs(adev);
+	sdma_v7_1_set_mqd_funcs(adev);
+
+	return 0;
+}
+
+static int sdma_v7_1_sw_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_ring *ring;
+	int r, i;
+	struct amdgpu_device *adev = ip_block->adev;
+	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_7_1);
+	uint32_t *ptr;
+	u32 xcc_id;
+
+	/* SDMA trap event */
+	r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_GFX,
+			      GFX_12_1_0__SRCID__SDMA_TRAP,
+			      &adev->sdma.trap_irq);
+	if (r)
+		return r;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		ring = &adev->sdma.instance[i].ring;
+		ring->ring_obj = NULL;
+		ring->use_doorbell = true;
+		ring->me = i;
+
+		for (xcc_id = 0; xcc_id < fls(adev->gfx.xcc_mask); xcc_id++) {
+			if (adev->sdma.instance[i].xcc_id == GET_INST(GC, xcc_id))
+				break;
+		}
+
+		DRM_DEBUG("SDMA%d.%d use_doorbell being set to: [%s]\n",
+				xcc_id, GET_INST(SDMA0, i) % adev->sdma.num_inst_per_xcc,
+				ring->use_doorbell?"true":"false");
+
+		ring->doorbell_index =
+			(adev->doorbell_index.sdma_engine[i] << 1); // get DWORD offset
+
+		ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
+		sprintf(ring->name, "sdma%d.%d", xcc_id,
+				GET_INST(SDMA0, i) % adev->sdma.num_inst_per_xcc);
+		r = amdgpu_ring_init(adev, ring, 1024,
+				     &adev->sdma.trap_irq,
+				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
+				     AMDGPU_RING_PRIO_DEFAULT, NULL);
+		if (r)
+			return r;
+	}
+
+	adev->sdma.supported_reset =
+		amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring);
+	if (!amdgpu_sriov_vf(adev) &&
+	    !adev->debug_disable_gpu_ring_reset)
+		adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
+
+	r = amdgpu_sdma_sysfs_reset_mask_init(adev);
+	if (r)
+		return r;
+	/* Allocate memory for SDMA IP Dump buffer */
+	ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL);
+	if (ptr)
+		adev->sdma.ip_dump = ptr;
+	else
+		DRM_ERROR("Failed to allocated memory for SDMA IP Dump\n");
+
+#ifdef CONFIG_DRM_AMDGPU_NAVI3X_USERQ
+	adev->userq_funcs[AMDGPU_HW_IP_DMA] = &userq_mes_funcs;
+#endif
+
+	return r;
+}
+
+static int sdma_v7_1_sw_fini(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int i;
+
+	for (i = 0; i < adev->sdma.num_instances; i++)
+		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
+
+	amdgpu_sdma_sysfs_reset_mask_fini(adev);
+	amdgpu_sdma_destroy_inst_ctx(adev, true);
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT)
+		sdma_v7_1_inst_free_ucode_buffer(adev, adev->sdma.sdma_mask);
+
+	kfree(adev->sdma.ip_dump);
+
+	return 0;
+}
+
+static int sdma_v7_1_hw_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+
+	return sdma_v7_1_inst_start(adev, adev->sdma.sdma_mask);
+}
+
+static int sdma_v7_1_hw_fini(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
+	sdma_v7_1_inst_ctx_switch_enable(adev, false, adev->sdma.sdma_mask);
+	sdma_v7_1_inst_enable(adev, false, adev->sdma.sdma_mask);
+
+	return 0;
+}
+
+static int sdma_v7_1_suspend(struct amdgpu_ip_block *ip_block)
+{
+	return sdma_v7_1_hw_fini(ip_block);
+}
+
+static int sdma_v7_1_resume(struct amdgpu_ip_block *ip_block)
+{
+	return sdma_v7_1_hw_init(ip_block);
+}
+
+static bool sdma_v7_1_is_idle(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	u32 i;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		u32 tmp = RREG32(sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_STATUS_REG));
+
+		if (!(tmp & SDMA0_SDMA_STATUS_REG__IDLE_MASK))
+			return false;
+	}
+
+	return true;
+}
+
+static int sdma_v7_1_wait_for_idle(struct amdgpu_ip_block *ip_block)
+{
+	unsigned i, j;
+	u32 sdma[AMDGPU_MAX_SDMA_INSTANCES];
+	struct amdgpu_device *adev = ip_block->adev;
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		for (j = 0; j < adev->sdma.num_instances; j++) {
+			sdma[j] = RREG32(sdma_v7_1_get_reg_offset(adev,
+						j, regSDMA0_SDMA_STATUS_REG));
+			if (!(sdma[j] & SDMA0_SDMA_STATUS_REG__IDLE_MASK))
+				break;
+		}
+		if (j == adev->sdma.num_instances)
+			return 0;
+		udelay(1);
+	}
+	return -ETIMEDOUT;
+}
+
+static int sdma_v7_1_ring_preempt_ib(struct amdgpu_ring *ring)
+{
+	int i, r = 0;
+	struct amdgpu_device *adev = ring->adev;
+	u32 index = 0;
+	u64 sdma_gfx_preempt;
+
+	amdgpu_sdma_get_index_from_ring(ring, &index);
+	sdma_gfx_preempt =
+		sdma_v7_1_get_reg_offset(adev, index, regSDMA0_SDMA_QUEUE0_PREEMPT);
+
+	/* assert preemption condition */
+	amdgpu_ring_set_preempt_cond_exec(ring, false);
+
+	/* emit the trailing fence */
+	ring->trail_seq += 1;
+	r = amdgpu_ring_alloc(ring, 10);
+	if (r) {
+		DRM_ERROR("ring %d failed to be allocated \n", ring->idx);
+		return r;
+	}
+	sdma_v7_1_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
+				  ring->trail_seq, 0);
+	amdgpu_ring_commit(ring);
+
+	/* assert IB preemption */
+	WREG32(sdma_gfx_preempt, 1);
+
+	/* poll the trailing fence */
+	for (i = 0; i < adev->usec_timeout; i++) {
+		if (ring->trail_seq ==
+		    le32_to_cpu(*(ring->trail_fence_cpu_addr)))
+			break;
+		udelay(1);
+	}
+
+	if (i >= adev->usec_timeout) {
+		r = -EINVAL;
+		DRM_ERROR("ring %d failed to be preempted\n", ring->idx);
+	}
+
+	/* deassert IB preemption */
+	WREG32(sdma_gfx_preempt, 0);
+
+	/* deassert the preemption condition */
+	amdgpu_ring_set_preempt_cond_exec(ring, true);
+	return r;
+}
+
+static int sdma_v7_1_set_trap_irq_state(struct amdgpu_device *adev,
+					struct amdgpu_irq_src *source,
+					unsigned type,
+					enum amdgpu_interrupt_state state)
+{
+	u32 sdma_cntl;
+
+	u32 reg_offset = sdma_v7_1_get_reg_offset(adev, type, regSDMA0_SDMA_CNTL);
+
+	sdma_cntl = RREG32(reg_offset);
+	sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_SDMA_CNTL, TRAP_ENABLE,
+		       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
+	WREG32(reg_offset, sdma_cntl);
+
+	return 0;
+}
+
+static int sdma_v7_1_process_trap_irq(struct amdgpu_device *adev,
+				      struct amdgpu_irq_src *source,
+				      struct amdgpu_iv_entry *entry)
+{
+	int inst, instances, queue, xcc_id = 0;
+	uint32_t mes_queue_id = entry->src_data[0];
+
+	DRM_DEBUG("IH: SDMA trap\n");
+
+	if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
+		struct amdgpu_mes_queue *queue;
+
+		mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
+
+		spin_lock(&adev->mes.queue_id_lock);
+		queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
+		if (queue) {
+			DRM_DEBUG("process smda queue id = %d\n", mes_queue_id);
+			amdgpu_fence_process(queue->ring);
+		}
+		spin_unlock(&adev->mes.queue_id_lock);
+		return 0;
+	}
+
+	queue = entry->ring_id & 0xf;
+	if (adev->gfx.funcs && adev->gfx.funcs->ih_node_to_logical_xcc)
+		xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev, entry->node_id);
+	else
+		dev_warn(adev->dev, "IH: SDMA may get wrong xcc id as gfx function not available\n");
+	inst = ((entry->ring_id & 0xf0) >> 4) +
+		GET_INST(GC, xcc_id) * adev->sdma.num_inst_per_xcc;
+	for (instances = 0; instances < adev->sdma.num_instances; instances++) {
+		if (inst == GET_INST(SDMA0, instances))
+			break;
+	}
+	if (instances > adev->sdma.num_instances - 1) {
+		DRM_ERROR("IH: wrong ring_ID detected, as wrong sdma instance\n");
+		return -EINVAL;
+	}
+
+	switch (entry->client_id) {
+	case SOC_V1_0_IH_CLIENTID_GFX:
+		switch (queue) {
+		case 0:
+			amdgpu_fence_process(&adev->sdma.instance[instances].ring);
+			break;
+		default:
+			break;
+		}
+		break;
+	}
+	return 0;
+}
+
+static int sdma_v7_1_process_illegal_inst_irq(struct amdgpu_device *adev,
+					      struct amdgpu_irq_src *source,
+					      struct amdgpu_iv_entry *entry)
+{
+	return 0;
+}
+
+static int sdma_v7_1_set_clockgating_state(struct amdgpu_ip_block *ip_block,
+					   enum amd_clockgating_state state)
+{
+	return 0;
+}
+
+static int sdma_v7_1_set_powergating_state(struct amdgpu_ip_block *ip_block,
+					  enum amd_powergating_state state)
+{
+	return 0;
+}
+
+static void sdma_v7_1_get_clockgating_state(struct amdgpu_ip_block *ip_block,
+					    u64 *flags)
+{
+}
+
+static void sdma_v7_1_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int i, j;
+	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_7_1);
+	uint32_t instance_offset;
+
+	if (!adev->sdma.ip_dump)
+		return;
+
+	drm_printf(p, "num_instances:%d\n", adev->sdma.num_instances);
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		instance_offset = i * reg_count;
+		drm_printf(p, "\nInstance:%d\n", i);
+
+		for (j = 0; j < reg_count; j++)
+			drm_printf(p, "%-50s \t 0x%08x\n", sdma_reg_list_7_1[j].reg_name,
+				   adev->sdma.ip_dump[instance_offset + j]);
+	}
+}
+
+static void sdma_v7_1_dump_ip_state(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int i, j;
+	uint32_t instance_offset;
+	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_7_1);
+
+	if (!adev->sdma.ip_dump)
+		return;
+
+	amdgpu_gfx_off_ctrl(adev, false);
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		instance_offset = i * reg_count;
+		for (j = 0; j < reg_count; j++)
+			adev->sdma.ip_dump[instance_offset + j] =
+				RREG32(sdma_v7_1_get_reg_offset(adev, i,
+				       sdma_reg_list_7_1[j].reg_offset));
+	}
+	amdgpu_gfx_off_ctrl(adev, true);
+}
+
+const struct amd_ip_funcs sdma_v7_1_ip_funcs = {
+	.name = "sdma_v7_1",
+	.early_init = sdma_v7_1_early_init,
+	.late_init = NULL,
+	.sw_init = sdma_v7_1_sw_init,
+	.sw_fini = sdma_v7_1_sw_fini,
+	.hw_init = sdma_v7_1_hw_init,
+	.hw_fini = sdma_v7_1_hw_fini,
+	.suspend = sdma_v7_1_suspend,
+	.resume = sdma_v7_1_resume,
+	.is_idle = sdma_v7_1_is_idle,
+	.wait_for_idle = sdma_v7_1_wait_for_idle,
+	.soft_reset = sdma_v7_1_soft_reset,
+	.check_soft_reset = sdma_v7_1_check_soft_reset,
+	.set_clockgating_state = sdma_v7_1_set_clockgating_state,
+	.set_powergating_state = sdma_v7_1_set_powergating_state,
+	.get_clockgating_state = sdma_v7_1_get_clockgating_state,
+	.dump_ip_state = sdma_v7_1_dump_ip_state,
+	.print_ip_state = sdma_v7_1_print_ip_state,
+};
+
+static const struct amdgpu_ring_funcs sdma_v7_1_ring_funcs = {
+	.type = AMDGPU_RING_TYPE_SDMA,
+	.align_mask = 0xf,
+	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
+	.support_64bit_ptrs = true,
+	.secure_submission_supported = true,
+	.get_rptr = sdma_v7_1_ring_get_rptr,
+	.get_wptr = sdma_v7_1_ring_get_wptr,
+	.set_wptr = sdma_v7_1_ring_set_wptr,
+	.emit_frame_size =
+		5 + /* sdma_v7_1_ring_init_cond_exec */
+		6 + /* sdma_v7_1_ring_emit_pipeline_sync */
+		/* sdma_v7_1_ring_emit_vm_flush */
+		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
+		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
+		10 + 10 + 10, /* sdma_v7_1_ring_emit_fence x3 for user fence, vm fence */
+	.emit_ib_size = 5 + 7 + 6, /* sdma_v7_1_ring_emit_ib */
+	.emit_ib = sdma_v7_1_ring_emit_ib,
+	.emit_mem_sync = sdma_v7_1_ring_emit_mem_sync,
+	.emit_fence = sdma_v7_1_ring_emit_fence,
+	.emit_pipeline_sync = sdma_v7_1_ring_emit_pipeline_sync,
+	.emit_vm_flush = sdma_v7_1_ring_emit_vm_flush,
+	.test_ring = sdma_v7_1_ring_test_ring,
+	.test_ib = sdma_v7_1_ring_test_ib,
+	.insert_nop = sdma_v7_1_ring_insert_nop,
+	.pad_ib = sdma_v7_1_ring_pad_ib,
+	.emit_wreg = sdma_v7_1_ring_emit_wreg,
+	.emit_reg_wait = sdma_v7_1_ring_emit_reg_wait,
+	.emit_reg_write_reg_wait = sdma_v7_1_ring_emit_reg_write_reg_wait,
+	.init_cond_exec = sdma_v7_1_ring_init_cond_exec,
+	.preempt_ib = sdma_v7_1_ring_preempt_ib,
+	.reset = sdma_v7_1_reset_queue,
+};
+
+static void sdma_v7_1_set_ring_funcs(struct amdgpu_device *adev)
+{
+	int i, dev_inst;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		adev->sdma.instance[i].ring.funcs = &sdma_v7_1_ring_funcs;
+		adev->sdma.instance[i].ring.me = i;
+
+		dev_inst = GET_INST(SDMA0, i);
+		/* XCC to which SDMA belongs depends on physical instance */
+		adev->sdma.instance[i].xcc_id =
+			dev_inst / adev->sdma.num_inst_per_xcc;
+	}
+}
+
+static const struct amdgpu_irq_src_funcs sdma_v7_1_trap_irq_funcs = {
+	.set = sdma_v7_1_set_trap_irq_state,
+	.process = sdma_v7_1_process_trap_irq,
+};
+
+static const struct amdgpu_irq_src_funcs sdma_v7_1_illegal_inst_irq_funcs = {
+	.process = sdma_v7_1_process_illegal_inst_irq,
+};
+
+static void sdma_v7_1_set_irq_funcs(struct amdgpu_device *adev)
+{
+	adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE0 +
+					adev->sdma.num_instances;
+	adev->sdma.trap_irq.funcs = &sdma_v7_1_trap_irq_funcs;
+	adev->sdma.illegal_inst_irq.funcs = &sdma_v7_1_illegal_inst_irq_funcs;
+}
+
+/**
+ * sdma_v7_1_emit_copy_buffer - copy buffer using the sDMA engine
+ *
+ * @ib: indirect buffer to fill with commands
+ * @src_offset: src GPU address
+ * @dst_offset: dst GPU address
+ * @byte_count: number of bytes to xfer
+ * @copy_flags: copy flags for the buffers
+ *
+ * Copy GPU buffers using the DMA engine.
+ * Used by the amdgpu ttm implementation to move pages if
+ * registered as the asic copy callback.
+ */
+static void sdma_v7_1_emit_copy_buffer(struct amdgpu_ib *ib,
+				       uint64_t src_offset,
+				       uint64_t dst_offset,
+				       uint32_t byte_count,
+				       uint32_t copy_flags)
+{
+	ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COPY) |
+		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
+		SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0);
+
+	ib->ptr[ib->length_dw++] = byte_count - 1;
+	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
+	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
+	ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
+	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
+	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
+}
+
+/**
+ * sdma_v7_1_emit_fill_buffer - fill buffer using the sDMA engine
+ *
+ * @ib: indirect buffer to fill
+ * @src_data: value to write to buffer
+ * @dst_offset: dst GPU address
+ * @byte_count: number of bytes to xfer
+ *
+ * Fill GPU buffers using the DMA engine.
+ */
+static void sdma_v7_1_emit_fill_buffer(struct amdgpu_ib *ib,
+				       uint32_t src_data,
+				       uint64_t dst_offset,
+				       uint32_t byte_count)
+{
+	ib->ptr[ib->length_dw++] = SDMA_PKT_CONSTANT_FILL_HEADER_OP(SDMA_OP_CONST_FILL);
+	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
+	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
+	ib->ptr[ib->length_dw++] = src_data;
+	ib->ptr[ib->length_dw++] = byte_count - 1;
+}
+
+static const struct amdgpu_buffer_funcs sdma_v7_1_buffer_funcs = {
+	.copy_max_bytes = 0x400000,
+	.copy_num_dw = 8,
+	.emit_copy_buffer = sdma_v7_1_emit_copy_buffer,
+	.fill_max_bytes = 0x400000,
+	.fill_num_dw = 5,
+	.emit_fill_buffer = sdma_v7_1_emit_fill_buffer,
+};
+
+static void sdma_v7_1_set_buffer_funcs(struct amdgpu_device *adev)
+{
+	adev->mman.buffer_funcs = &sdma_v7_1_buffer_funcs;
+	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+}
+
+static const struct amdgpu_vm_pte_funcs sdma_v7_1_vm_pte_funcs = {
+	.copy_pte_num_dw = 8,
+	.copy_pte = sdma_v7_1_vm_copy_pte,
+	.write_pte = sdma_v7_1_vm_write_pte,
+	.set_pte_pde = sdma_v7_1_vm_set_pte_pde,
+};
+
+static void sdma_v7_1_set_vm_pte_funcs(struct amdgpu_device *adev)
+{
+	unsigned i;
+
+	adev->vm_manager.vm_pte_funcs = &sdma_v7_1_vm_pte_funcs;
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		adev->vm_manager.vm_pte_scheds[i] =
+			&adev->sdma.instance[i].ring.sched;
+	}
+	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
+}
+
+const struct amdgpu_ip_block_version sdma_v7_1_ip_block = {
+	.type = AMD_IP_BLOCK_TYPE_SDMA,
+	.major = 7,
+	.minor = 1,
+	.rev = 0,
+	.funcs = &sdma_v7_1_ip_funcs,
+};
+
+static int sdma_v7_1_xcp_resume(void *handle, uint32_t inst_mask)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int r;
+
+	r = sdma_v7_1_inst_start(adev, inst_mask);
+
+	return r;
+}
+
+static int sdma_v7_1_xcp_suspend(void *handle, uint32_t inst_mask)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	sdma_v7_1_inst_ctx_switch_enable(adev, false, inst_mask);
+	sdma_v7_1_inst_enable(adev, false, inst_mask);
+
+	return 0;
+}
+
+struct amdgpu_xcp_ip_funcs sdma_v7_1_xcp_funcs = {
+	.suspend = &sdma_v7_1_xcp_suspend,
+	.resume = &sdma_v7_1_xcp_resume
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.h b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.h
new file mode 100644
index 000000000000..1a07ef09a103
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __SDMA_V7_1_H__
+#define __SDMA_V7_1_H__
+
+extern const struct amd_ip_funcs sdma_v7_1_ip_funcs;
+extern const struct amdgpu_ip_block_version sdma_v7_1_ip_block;
+extern struct amdgpu_xcp_ip_funcs sdma_v7_1_xcp_funcs;
+
+#endif /* __SDMA_V7_1_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1_0_pkt_open.h b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1_0_pkt_open.h
new file mode 100644
index 000000000000..b051e4f92088
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1_0_pkt_open.h
@@ -0,0 +1,5673 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __SDMA_V7_1_0_PKT_OPEN_H_
+#define __SDMA_V7_1_0_PKT_OPEN_H_
+
+#define SDMA_OP_NOP  0
+#define SDMA_OP_COPY  1
+#define SDMA_OP_WRITE  2
+#define SDMA_OP_INDIRECT  4
+#define SDMA_OP_FENCE  5
+#define SDMA_OP_TRAP  6
+#define SDMA_OP_SEM  7
+#define SDMA_OP_POLL_REGMEM  8
+#define SDMA_OP_COND_EXE  9
+#define SDMA_OP_ATOMIC  10
+#define SDMA_OP_CONST_FILL  11
+#define SDMA_OP_PTEPDE  12
+#define SDMA_OP_TIMESTAMP  13
+#define SDMA_OP_SRBM_WRITE  14
+#define SDMA_OP_PRE_EXE  15
+#define SDMA_OP_GPUVM_INV  16
+#define SDMA_OP_GCR_REQ  17
+#define SDMA_OP_DUMMY_TRAP  32
+#define SDMA_SUBOP_TIMESTAMP_SET  0
+#define SDMA_SUBOP_TIMESTAMP_GET  1
+#define SDMA_SUBOP_TIMESTAMP_GET_GLOBAL  2
+#define SDMA_SUBOP_COPY_LINEAR  0
+#define SDMA_SUBOP_COPY_LINEAR_SUB_WIND  4
+#define SDMA_SUBOP_COPY_TILED  1
+#define SDMA_SUBOP_COPY_TILED_SUB_WIND  5
+#define SDMA_SUBOP_COPY_T2T_SUB_WIND  6
+#define SDMA_SUBOP_COPY_SOA  3
+#define SDMA_SUBOP_COPY_DIRTY_PAGE  7
+#define SDMA_SUBOP_COPY_LINEAR_PHY  8
+#define SDMA_SUBOP_COPY_LINEAR_SUB_WIND_LARGE  36
+#define SDMA_SUBOP_COPY_LINEAR_BC  16
+#define SDMA_SUBOP_COPY_TILED_BC  17
+#define SDMA_SUBOP_COPY_LINEAR_SUB_WIND_BC  20
+#define SDMA_SUBOP_COPY_TILED_SUB_WIND_BC  21
+#define SDMA_SUBOP_COPY_T2T_SUB_WIND_BC  22
+#define SDMA_SUBOP_WRITE_LINEAR  0
+#define SDMA_SUBOP_WRITE_TILED  1
+#define SDMA_SUBOP_WRITE_TILED_BC  17
+#define SDMA_SUBOP_PTEPDE_GEN  0
+#define SDMA_SUBOP_PTEPDE_COPY  1
+#define SDMA_SUBOP_PTEPDE_RMW  2
+#define SDMA_SUBOP_PTEPDE_COPY_BACKWARDS  3
+#define SDMA_SUBOP_MEM_INCR  1
+#define SDMA_SUBOP_DATA_FILL_MULTI  1
+#define SDMA_SUBOP_POLL_REG_WRITE_MEM  1
+#define SDMA_SUBOP_POLL_DBIT_WRITE_MEM  2
+#define SDMA_SUBOP_POLL_MEM_VERIFY  3
+#define SDMA_SUBOP_VM_INVALIDATION  4
+#define HEADER_AGENT_DISPATCH  4
+#define HEADER_BARRIER  5
+#define SDMA_OP_AQL_COPY  0
+#define SDMA_OP_AQL_BARRIER_OR  0
+
+#define SDMA_GCR_RANGE_IS_PA		(1 << 18)
+#define SDMA_GCR_SEQ(x)			(((x) & 0x3) << 16)
+#define SDMA_GCR_GL2_WB			(1 << 15)
+#define SDMA_GCR_GL2_INV		(1 << 14)
+#define SDMA_GCR_GL2_DISCARD		(1 << 13)
+#define SDMA_GCR_GL2_RANGE(x)		(((x) & 0x3) << 11)
+#define SDMA_GCR_GL2_US			(1 << 10)
+#define SDMA_GCR_GL1_INV		(1 << 9)
+#define SDMA_GCR_GLV_INV		(1 << 8)
+#define SDMA_GCR_GLK_INV		(1 << 7)
+#define SDMA_GCR_GLK_WB			(1 << 6)
+#define SDMA_GCR_GLM_INV		(1 << 5)
+#define SDMA_GCR_GLM_WB			(1 << 4)
+#define SDMA_GCR_GL1_RANGE(x)		(((x) & 0x3) << 2)
+#define SDMA_GCR_GLI_INV(x)		(((x) & 0x3) << 0)
+
+#define SDMA_DCC_DATA_FORMAT(x) ((x) & 0x3f)
+#define SDMA_DCC_NUM_TYPE(x) (((x) & 0x7) << 9)
+#define SDMA_DCC_READ_CM(x) (((x) & 0x3) << 16)
+#define SDMA_DCC_WRITE_CM(x) (((x) & 0x3) << 18)
+#define SDMA_DCC_MAX_COM(x) (((x) & 0x3) << 24)
+#define SDMA_DCC_MAX_UCOM(x) (((x) & 0x1) << 26)
+
+/*
+** Definitions for SDMA_PKT_COPY_LINEAR packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_LINEAR_HEADER_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COPY_LINEAR_HEADER_op_shift  0
+#define SDMA_PKT_COPY_LINEAR_HEADER_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_HEADER_op_mask) << SDMA_PKT_COPY_LINEAR_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_LINEAR_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COPY_LINEAR_HEADER_sub_op_shift  8
+#define SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_HEADER_sub_op_mask) << SDMA_PKT_COPY_LINEAR_HEADER_sub_op_shift)
+
+/*define for encrypt field*/
+#define SDMA_PKT_COPY_LINEAR_HEADER_encrypt_offset 0
+#define SDMA_PKT_COPY_LINEAR_HEADER_encrypt_mask   0x00000001
+#define SDMA_PKT_COPY_LINEAR_HEADER_encrypt_shift  16
+#define SDMA_PKT_COPY_LINEAR_HEADER_ENCRYPT(x) (((x) & SDMA_PKT_COPY_LINEAR_HEADER_encrypt_mask) << SDMA_PKT_COPY_LINEAR_HEADER_encrypt_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_LINEAR_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_LINEAR_HEADER_tmz_mask   0x00000001
+#define SDMA_PKT_COPY_LINEAR_HEADER_tmz_shift  18
+#define SDMA_PKT_COPY_LINEAR_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_LINEAR_HEADER_tmz_mask) << SDMA_PKT_COPY_LINEAR_HEADER_tmz_shift)
+
+/*define for npd field*/
+#define SDMA_PKT_COPY_LINEAR_HEADER_npd_offset 0
+#define SDMA_PKT_COPY_LINEAR_HEADER_npd_mask   0x00000001
+#define SDMA_PKT_COPY_LINEAR_HEADER_npd_shift  28
+#define SDMA_PKT_COPY_LINEAR_HEADER_NPD(x) (((x) & SDMA_PKT_COPY_LINEAR_HEADER_npd_mask) << SDMA_PKT_COPY_LINEAR_HEADER_npd_shift)
+
+/*define for backwards field*/
+#define SDMA_PKT_COPY_LINEAR_HEADER_backwards_offset 0
+#define SDMA_PKT_COPY_LINEAR_HEADER_backwards_mask   0x00000001
+#define SDMA_PKT_COPY_LINEAR_HEADER_backwards_shift  25
+#define SDMA_PKT_COPY_LINEAR_HEADER_BACKWARDS(x) (((x) & SDMA_PKT_COPY_LINEAR_HEADER_backwards_mask) << SDMA_PKT_COPY_LINEAR_HEADER_backwards_shift)
+
+/*define for broadcast field*/
+#define SDMA_PKT_COPY_LINEAR_HEADER_broadcast_offset 0
+#define SDMA_PKT_COPY_LINEAR_HEADER_broadcast_mask   0x00000001
+#define SDMA_PKT_COPY_LINEAR_HEADER_broadcast_shift  27
+#define SDMA_PKT_COPY_LINEAR_HEADER_BROADCAST(x) (((x) & SDMA_PKT_COPY_LINEAR_HEADER_broadcast_mask) << SDMA_PKT_COPY_LINEAR_HEADER_broadcast_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_LINEAR_COUNT_count_offset 1
+#define SDMA_PKT_COPY_LINEAR_COUNT_count_mask   0x3FFFFFFF
+#define SDMA_PKT_COPY_LINEAR_COUNT_count_shift  0
+#define SDMA_PKT_COPY_LINEAR_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_LINEAR_COUNT_count_mask) << SDMA_PKT_COPY_LINEAR_COUNT_count_shift)
+
+/*define for PARAMETER word*/
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_dst_sw_offset 2
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_dst_sw_mask   0x00000003
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_dst_sw_shift  16
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_DST_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_PARAMETER_dst_sw_mask) << SDMA_PKT_COPY_LINEAR_PARAMETER_dst_sw_shift)
+
+/*define for dst_cache_policy field*/
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_dst_cache_policy_offset 2
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_dst_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_dst_cache_policy_shift  18
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_DST_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_LINEAR_PARAMETER_dst_cache_policy_mask) << SDMA_PKT_COPY_LINEAR_PARAMETER_dst_cache_policy_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_src_sw_offset 2
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_src_sw_mask   0x00000003
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_src_sw_shift  24
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_SRC_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_PARAMETER_src_sw_mask) << SDMA_PKT_COPY_LINEAR_PARAMETER_src_sw_shift)
+
+/*define for src_cache_policy field*/
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_src_cache_policy_offset 2
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_src_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_src_cache_policy_shift  26
+#define SDMA_PKT_COPY_LINEAR_PARAMETER_SRC_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_LINEAR_PARAMETER_src_cache_policy_mask) << SDMA_PKT_COPY_LINEAR_PARAMETER_src_cache_policy_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_offset 3
+#define SDMA_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_shift  0
+#define SDMA_PKT_COPY_LINEAR_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_offset 4
+#define SDMA_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_shift  0
+#define SDMA_PKT_COPY_LINEAR_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_offset 5
+#define SDMA_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_COPY_LINEAR_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_offset 6
+#define SDMA_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_COPY_LINEAR_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_LINEAR_BC packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_LINEAR_BC_HEADER_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_BC_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COPY_LINEAR_BC_HEADER_op_shift  0
+#define SDMA_PKT_COPY_LINEAR_BC_HEADER_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_HEADER_op_mask) << SDMA_PKT_COPY_LINEAR_BC_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_LINEAR_BC_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_BC_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COPY_LINEAR_BC_HEADER_sub_op_shift  8
+#define SDMA_PKT_COPY_LINEAR_BC_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_HEADER_sub_op_mask) << SDMA_PKT_COPY_LINEAR_BC_HEADER_sub_op_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_LINEAR_BC_COUNT_count_offset 1
+#define SDMA_PKT_COPY_LINEAR_BC_COUNT_count_mask   0x003FFFFF
+#define SDMA_PKT_COPY_LINEAR_BC_COUNT_count_shift  0
+#define SDMA_PKT_COPY_LINEAR_BC_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_COUNT_count_mask) << SDMA_PKT_COPY_LINEAR_BC_COUNT_count_shift)
+
+/*define for PARAMETER word*/
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_sw_offset 2
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_sw_mask   0x00000003
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_sw_shift  16
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_DST_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_sw_mask) << SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_sw_shift)
+
+/*define for dst_ha field*/
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_ha_offset 2
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_ha_mask   0x00000001
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_ha_shift  19
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_DST_HA(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_ha_mask) << SDMA_PKT_COPY_LINEAR_BC_PARAMETER_dst_ha_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_sw_offset 2
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_sw_mask   0x00000003
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_sw_shift  24
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_SRC_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_sw_mask) << SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_sw_shift)
+
+/*define for src_ha field*/
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_ha_offset 2
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_ha_mask   0x00000001
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_ha_shift  27
+#define SDMA_PKT_COPY_LINEAR_BC_PARAMETER_SRC_HA(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_ha_mask) << SDMA_PKT_COPY_LINEAR_BC_PARAMETER_src_ha_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_LO_src_addr_31_0_offset 3
+#define SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_LO_src_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_LO_src_addr_31_0_shift  0
+#define SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_HI_src_addr_63_32_offset 4
+#define SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_HI_src_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_HI_src_addr_63_32_shift  0
+#define SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_BC_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_LO_dst_addr_31_0_offset 5
+#define SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_HI_dst_addr_63_32_offset 6
+#define SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_BC_DST_ADDR_HI_dst_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_DIRTY_PAGE packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_op_offset 0
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_op_shift  0
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_OP(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_HEADER_op_mask) << SDMA_PKT_COPY_DIRTY_PAGE_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_sub_op_shift  8
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_HEADER_sub_op_mask) << SDMA_PKT_COPY_DIRTY_PAGE_HEADER_sub_op_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_tmz_mask   0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_tmz_shift  18
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_HEADER_tmz_mask) << SDMA_PKT_COPY_DIRTY_PAGE_HEADER_tmz_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_cpv_offset 0
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_cpv_shift  19
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_CPV(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_HEADER_cpv_mask) << SDMA_PKT_COPY_DIRTY_PAGE_HEADER_cpv_shift)
+
+/*define for all field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_all_offset 0
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_all_mask   0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_all_shift  31
+#define SDMA_PKT_COPY_DIRTY_PAGE_HEADER_ALL(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_HEADER_all_mask) << SDMA_PKT_COPY_DIRTY_PAGE_HEADER_all_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_COUNT_count_offset 1
+#define SDMA_PKT_COPY_DIRTY_PAGE_COUNT_count_mask   0x003FFFFF
+#define SDMA_PKT_COPY_DIRTY_PAGE_COUNT_count_shift  0
+#define SDMA_PKT_COPY_DIRTY_PAGE_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_COUNT_count_mask) << SDMA_PKT_COPY_DIRTY_PAGE_COUNT_count_shift)
+
+/*define for PARAMETER word*/
+/*define for dst_mtype field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_mtype_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_mtype_mask   0x00000007
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_mtype_shift  3
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_DST_MTYPE(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_mtype_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_mtype_shift)
+
+/*define for dst_l2_policy field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_l2_policy_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_l2_policy_mask   0x00000003
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_l2_policy_shift  6
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_DST_L2_POLICY(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_l2_policy_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_l2_policy_shift)
+
+/*define for dst_llc field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_llc_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_llc_mask   0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_llc_shift  8
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_DST_LLC(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_llc_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_llc_shift)
+
+/*define for src_mtype field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_mtype_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_mtype_mask   0x00000007
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_mtype_shift  11
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_SRC_MTYPE(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_mtype_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_mtype_shift)
+
+/*define for src_l2_policy field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_l2_policy_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_l2_policy_mask   0x00000003
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_l2_policy_shift  14
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_SRC_L2_POLICY(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_l2_policy_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_l2_policy_shift)
+
+/*define for src_llc field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_llc_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_llc_mask   0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_llc_shift  16
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_SRC_LLC(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_llc_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_llc_shift)
+
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sw_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sw_mask   0x00000003
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sw_shift  17
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_DST_SW(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sw_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sw_shift)
+
+/*define for dst_gcc field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gcc_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gcc_mask   0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gcc_shift  19
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_DST_GCC(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gcc_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gcc_shift)
+
+/*define for dst_sys field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sys_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sys_mask   0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sys_shift  20
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_DST_SYS(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sys_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_sys_shift)
+
+/*define for dst_snoop field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_snoop_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_snoop_mask   0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_snoop_shift  22
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_DST_SNOOP(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_snoop_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_snoop_shift)
+
+/*define for dst_gpa field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gpa_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gpa_mask   0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gpa_shift  23
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_DST_GPA(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gpa_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_dst_gpa_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sw_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sw_mask   0x00000003
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sw_shift  24
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_SRC_SW(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sw_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sw_shift)
+
+/*define for src_sys field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sys_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sys_mask   0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sys_shift  28
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_SRC_SYS(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sys_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_sys_shift)
+
+/*define for src_snoop field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_snoop_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_snoop_mask   0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_snoop_shift  30
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_SRC_SNOOP(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_snoop_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_snoop_shift)
+
+/*define for src_gpa field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_gpa_offset 2
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_gpa_mask   0x00000001
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_gpa_shift  31
+#define SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_SRC_GPA(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_gpa_mask) << SDMA_PKT_COPY_DIRTY_PAGE_PARAMETER_src_gpa_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_LO_src_addr_31_0_offset 3
+#define SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_LO_src_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_LO_src_addr_31_0_shift  0
+#define SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_HI_src_addr_63_32_offset 4
+#define SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_HI_src_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_HI_src_addr_63_32_shift  0
+#define SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_DIRTY_PAGE_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_LO_dst_addr_31_0_offset 5
+#define SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_HI_dst_addr_63_32_offset 6
+#define SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_DIRTY_PAGE_DST_ADDR_HI_dst_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_PHYSICAL_LINEAR packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_op_offset 0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_op_shift  0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_OP(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_op_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_sub_op_shift  8
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_sub_op_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_sub_op_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_tmz_mask   0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_tmz_shift  18
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_tmz_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_tmz_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_cpv_offset 0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_cpv_shift  19
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_CPV(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_cpv_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_HEADER_cpv_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_count_offset 1
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_count_mask   0x003FFFFF
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_count_shift  0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_count_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_count_shift)
+
+/*define for addr_pair_num field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_addr_pair_num_offset 1
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_addr_pair_num_mask   0x000000FF
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_addr_pair_num_shift  24
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_ADDR_PAIR_NUM(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_addr_pair_num_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_COUNT_addr_pair_num_shift)
+
+/*define for PARAMETER word*/
+/*define for dst_mtype field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_mtype_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_mtype_mask   0x00000007
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_mtype_shift  3
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_MTYPE(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_mtype_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_mtype_shift)
+
+/*define for dst_l2_policy field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_l2_policy_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_l2_policy_mask   0x00000003
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_l2_policy_shift  6
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_L2_POLICY(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_l2_policy_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_l2_policy_shift)
+
+/*define for dst_llc field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_llc_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_llc_mask   0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_llc_shift  8
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_LLC(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_llc_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_llc_shift)
+
+/*define for src_mtype field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_mtype_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_mtype_mask   0x00000007
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_mtype_shift  11
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_SRC_MTYPE(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_mtype_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_mtype_shift)
+
+/*define for src_l2_policy field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_l2_policy_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_l2_policy_mask   0x00000003
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_l2_policy_shift  14
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_SRC_L2_POLICY(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_l2_policy_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_l2_policy_shift)
+
+/*define for src_llc field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_llc_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_llc_mask   0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_llc_shift  16
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_SRC_LLC(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_llc_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_llc_shift)
+
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sw_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sw_mask   0x00000003
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sw_shift  17
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_SW(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sw_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sw_shift)
+
+/*define for dst_gcc field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gcc_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gcc_mask   0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gcc_shift  19
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_GCC(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gcc_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gcc_shift)
+
+/*define for dst_sys field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sys_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sys_mask   0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sys_shift  20
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_SYS(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sys_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_sys_shift)
+
+/*define for dst_log field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_log_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_log_mask   0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_log_shift  21
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_LOG(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_log_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_log_shift)
+
+/*define for dst_snoop field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_snoop_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_snoop_mask   0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_snoop_shift  22
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_SNOOP(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_snoop_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_snoop_shift)
+
+/*define for dst_gpa field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gpa_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gpa_mask   0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gpa_shift  23
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_DST_GPA(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gpa_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_dst_gpa_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sw_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sw_mask   0x00000003
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sw_shift  24
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_SRC_SW(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sw_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sw_shift)
+
+/*define for src_gcc field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gcc_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gcc_mask   0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gcc_shift  27
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_SRC_GCC(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gcc_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gcc_shift)
+
+/*define for src_sys field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sys_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sys_mask   0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sys_shift  28
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_SRC_SYS(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sys_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_sys_shift)
+
+/*define for src_snoop field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_snoop_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_snoop_mask   0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_snoop_shift  30
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_SRC_SNOOP(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_snoop_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_snoop_shift)
+
+/*define for src_gpa field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gpa_offset 2
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gpa_mask   0x00000001
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gpa_shift  31
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_SRC_GPA(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gpa_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_PARAMETER_src_gpa_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_LO_src_addr_31_0_offset 3
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_LO_src_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_LO_src_addr_31_0_shift  0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_HI_src_addr_63_32_offset 4
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_HI_src_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_HI_src_addr_63_32_shift  0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_LO_dst_addr_31_0_offset 5
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_HI_dst_addr_63_32_offset 6
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_PHYSICAL_LINEAR_DST_ADDR_HI_dst_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_BROADCAST_LINEAR packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_op_offset 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_op_shift  0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_OP(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_op_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_sub_op_shift  8
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_sub_op_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_sub_op_shift)
+
+/*define for encrypt field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_encrypt_offset 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_encrypt_mask   0x00000001
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_encrypt_shift  16
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_ENCRYPT(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_encrypt_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_encrypt_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_tmz_mask   0x00000001
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_tmz_shift  18
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_tmz_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_tmz_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_cpv_offset 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_cpv_shift  19
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_CPV(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_cpv_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_cpv_shift)
+
+/*define for broadcast field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_broadcast_offset 0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_broadcast_mask   0x00000001
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_broadcast_shift  27
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_BROADCAST(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_broadcast_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_HEADER_broadcast_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_COUNT_count_offset 1
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_COUNT_count_mask   0x3FFFFFFF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_COUNT_count_shift  0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_COUNT_count_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_COUNT_count_shift)
+
+/*define for PARAMETER word*/
+/*define for dst2_sw field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst2_sw_offset 2
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst2_sw_mask   0x00000003
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst2_sw_shift  8
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_DST2_SW(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst2_sw_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst2_sw_shift)
+
+/*define for dst2_cache_policy field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst2_cache_policy_offset 2
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst2_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst2_cache_policy_shift  10
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_DST2_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst2_cache_policy_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst2_cache_policy_shift)
+
+/*define for dst1_sw field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst1_sw_offset 2
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst1_sw_mask   0x00000003
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst1_sw_shift  16
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_DST1_SW(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst1_sw_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst1_sw_shift)
+
+/*define for dst1_cache_policy field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst1_cache_policy_offset 2
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst1_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst1_cache_policy_shift  18
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_DST1_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst1_cache_policy_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_dst1_cache_policy_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_src_sw_offset 2
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_src_sw_mask   0x00000003
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_src_sw_shift  24
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_SRC_SW(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_src_sw_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_src_sw_shift)
+
+/*define for src_cache_policy field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_src_cache_policy_offset 2
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_src_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_src_cache_policy_shift  26
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_SRC_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_src_cache_policy_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_PARAMETER_src_cache_policy_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_LO_src_addr_31_0_offset 3
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_LO_src_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_LO_src_addr_31_0_shift  0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_HI_src_addr_63_32_offset 4
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_HI_src_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_HI_src_addr_63_32_shift  0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DST1_ADDR_LO word*/
+/*define for dst1_addr_31_0 field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_LO_dst1_addr_31_0_offset 5
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_LO_dst1_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_LO_dst1_addr_31_0_shift  0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_LO_DST1_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_LO_dst1_addr_31_0_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_LO_dst1_addr_31_0_shift)
+
+/*define for DST1_ADDR_HI word*/
+/*define for dst1_addr_63_32 field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_HI_dst1_addr_63_32_offset 6
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_HI_dst1_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_HI_dst1_addr_63_32_shift  0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_HI_DST1_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_HI_dst1_addr_63_32_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_DST1_ADDR_HI_dst1_addr_63_32_shift)
+
+/*define for DST2_ADDR_LO word*/
+/*define for dst2_addr_31_0 field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_LO_dst2_addr_31_0_offset 7
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_LO_dst2_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_LO_dst2_addr_31_0_shift  0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_LO_DST2_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_LO_dst2_addr_31_0_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_LO_dst2_addr_31_0_shift)
+
+/*define for DST2_ADDR_HI word*/
+/*define for dst2_addr_63_32 field*/
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_HI_dst2_addr_63_32_offset 8
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_HI_dst2_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_HI_dst2_addr_63_32_shift  0
+#define SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_HI_DST2_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_HI_dst2_addr_63_32_mask) << SDMA_PKT_COPY_BROADCAST_LINEAR_DST2_ADDR_HI_dst2_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_LINEAR_SUBWIN packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_op_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_op_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_sub_op_shift  8
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_sub_op_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_sub_op_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_tmz_mask   0x00000001
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_tmz_shift  18
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_tmz_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_tmz_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_cpv_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_cpv_shift  19
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_CPV(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_cpv_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_cpv_shift)
+
+/*define for elementsize field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_elementsize_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_elementsize_mask   0x00000007
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_elementsize_shift  29
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_ELEMENTSIZE(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_elementsize_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_HEADER_elementsize_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_LO_src_addr_31_0_offset 1
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_LO_src_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_LO_src_addr_31_0_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_HI_src_addr_63_32_offset 2
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_HI_src_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_HI_src_addr_63_32_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for src_x field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_x_offset 3
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_x_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_SRC_X(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_x_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_x_shift)
+
+/*define for src_y field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_y_offset 3
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_y_shift  16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_SRC_Y(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_y_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_3_src_y_shift)
+
+/*define for DW_4 word*/
+/*define for src_z field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_z_offset 4
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_z_mask   0x00001FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_z_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_SRC_Z(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_z_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_z_shift)
+
+/*define for src_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_pitch_offset 4
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_pitch_mask   0x0007FFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_pitch_shift  13
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_SRC_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_4_src_pitch_shift)
+
+/*define for DW_5 word*/
+/*define for src_slice_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_5_src_slice_pitch_offset 5
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_5_src_slice_pitch_mask   0x0FFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_5_src_slice_pitch_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_5_SRC_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_5_src_slice_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_5_src_slice_pitch_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_LO_dst_addr_31_0_offset 6
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_HI_dst_addr_63_32_offset 7
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DW_8 word*/
+/*define for dst_x field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_x_offset 8
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_x_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_DST_X(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_x_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_x_shift)
+
+/*define for dst_y field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_y_offset 8
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_y_shift  16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_DST_Y(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_y_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_8_dst_y_shift)
+
+/*define for DW_9 word*/
+/*define for dst_z field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_z_offset 9
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_z_mask   0x00001FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_z_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_DST_Z(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_z_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_z_shift)
+
+/*define for dst_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_pitch_offset 9
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_pitch_mask   0x0007FFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_pitch_shift  13
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_DST_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_9_dst_pitch_shift)
+
+/*define for DW_10 word*/
+/*define for dst_slice_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_10_dst_slice_pitch_offset 10
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_10_dst_slice_pitch_mask   0x0FFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_10_dst_slice_pitch_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_10_DST_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_10_dst_slice_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_10_dst_slice_pitch_shift)
+
+/*define for DW_11 word*/
+/*define for rect_x field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_x_offset 11
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_x_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_RECT_X(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_x_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_x_shift)
+
+/*define for rect_y field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_y_offset 11
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_y_shift  16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_RECT_Y(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_y_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_11_rect_y_shift)
+
+/*define for DW_12 word*/
+/*define for rect_z field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_rect_z_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_rect_z_mask   0x00001FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_rect_z_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_RECT_Z(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_rect_z_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_rect_z_shift)
+
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_dst_sw_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_dst_sw_mask   0x00000003
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_dst_sw_shift  16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_DST_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_dst_sw_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_dst_sw_shift)
+
+/*define for dst_cache_policy field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_dst_cache_policy_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_dst_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_dst_cache_policy_shift  18
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_DST_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_dst_cache_policy_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_dst_cache_policy_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_src_sw_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_src_sw_mask   0x00000003
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_src_sw_shift  24
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_SRC_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_src_sw_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_src_sw_shift)
+
+/*define for src_cache_policy field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_src_cache_policy_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_src_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_src_cache_policy_shift  26
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_SRC_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_src_cache_policy_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_DW_12_src_cache_policy_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_op_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_op_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_sub_op_shift  8
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_sub_op_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_sub_op_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_tmz_mask   0x00000001
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_tmz_shift  18
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_tmz_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_tmz_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_cpv_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_cpv_shift  19
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_CPV(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_cpv_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_HEADER_cpv_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_SRC_ADDR_LO_src_addr_31_0_offset 1
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_SRC_ADDR_LO_src_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_SRC_ADDR_LO_src_addr_31_0_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_SRC_ADDR_HI_src_addr_63_32_offset 2
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_SRC_ADDR_HI_src_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_SRC_ADDR_HI_src_addr_63_32_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for src_x field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_3_src_x_offset 3
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_3_src_x_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_3_src_x_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_3_SRC_X(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_3_src_x_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_3_src_x_shift)
+
+/*define for DW_4 word*/
+/*define for src_y field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_4_src_y_offset 4
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_4_src_y_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_4_src_y_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_4_SRC_Y(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_4_src_y_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_4_src_y_shift)
+
+/*define for DW_5 word*/
+/*define for src_z field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_5_src_z_offset 5
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_5_src_z_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_5_src_z_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_5_SRC_Z(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_5_src_z_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_5_src_z_shift)
+
+/*define for DW_6 word*/
+/*define for src_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_6_src_pitch_offset 6
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_6_src_pitch_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_6_src_pitch_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_6_SRC_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_6_src_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_6_src_pitch_shift)
+
+/*define for DW_7 word*/
+/*define for src_slice_pitch_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_7_src_slice_pitch_31_0_offset 7
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_7_src_slice_pitch_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_7_src_slice_pitch_31_0_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_7_SRC_SLICE_PITCH_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_7_src_slice_pitch_31_0_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_7_src_slice_pitch_31_0_shift)
+
+/*define for DW_8 word*/
+/*define for src_slice_pitch_47_32 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_8_src_slice_pitch_47_32_offset 8
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_8_src_slice_pitch_47_32_mask   0x0000FFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_8_src_slice_pitch_47_32_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_8_SRC_SLICE_PITCH_47_32(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_8_src_slice_pitch_47_32_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_8_src_slice_pitch_47_32_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DST_ADDR_LO_dst_addr_31_0_offset 9
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DST_ADDR_HI_dst_addr_63_32_offset 10
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DW_11 word*/
+/*define for dst_x field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_11_dst_x_offset 11
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_11_dst_x_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_11_dst_x_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_11_DST_X(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_11_dst_x_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_11_dst_x_shift)
+
+/*define for DW_12 word*/
+/*define for dst_y field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_12_dst_y_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_12_dst_y_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_12_dst_y_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_12_DST_Y(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_12_dst_y_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_12_dst_y_shift)
+
+/*define for DW_13 word*/
+/*define for dst_z field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_13_dst_z_offset 13
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_13_dst_z_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_13_dst_z_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_13_DST_Z(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_13_dst_z_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_13_dst_z_shift)
+
+/*define for DW_14 word*/
+/*define for dst_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_14_dst_pitch_offset 14
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_14_dst_pitch_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_14_dst_pitch_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_14_DST_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_14_dst_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_14_dst_pitch_shift)
+
+/*define for DW_15 word*/
+/*define for dst_slice_pitch_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_15_dst_slice_pitch_31_0_offset 15
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_15_dst_slice_pitch_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_15_dst_slice_pitch_31_0_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_15_DST_SLICE_PITCH_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_15_dst_slice_pitch_31_0_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_15_dst_slice_pitch_31_0_shift)
+
+/*define for DW_16 word*/
+/*define for dst_slice_pitch_47_32 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_dst_slice_pitch_47_32_offset 16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_dst_slice_pitch_47_32_mask   0x0000FFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_dst_slice_pitch_47_32_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_DST_SLICE_PITCH_47_32(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_dst_slice_pitch_47_32_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_dst_slice_pitch_47_32_shift)
+
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_dst_sw_offset 16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_dst_sw_mask   0x00000003
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_dst_sw_shift  16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_DST_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_dst_sw_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_dst_sw_shift)
+
+/*define for dst_policy field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_dst_policy_offset 16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_dst_policy_mask   0x00000007
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_dst_policy_shift  18
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_DST_POLICY(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_dst_policy_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_dst_policy_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_src_sw_offset 16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_src_sw_mask   0x00000003
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_src_sw_shift  24
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_SRC_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_src_sw_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_src_sw_shift)
+
+/*define for src_policy field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_src_policy_offset 16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_src_policy_mask   0x00000007
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_src_policy_shift  26
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_SRC_POLICY(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_src_policy_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_16_src_policy_shift)
+
+/*define for DW_17 word*/
+/*define for rect_x field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_17_rect_x_offset 17
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_17_rect_x_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_17_rect_x_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_17_RECT_X(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_17_rect_x_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_17_rect_x_shift)
+
+/*define for DW_18 word*/
+/*define for rect_y field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_18_rect_y_offset 18
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_18_rect_y_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_18_rect_y_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_18_RECT_Y(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_18_rect_y_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_18_rect_y_shift)
+
+/*define for DW_19 word*/
+/*define for rect_z field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_19_rect_z_offset 19
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_19_rect_z_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_19_rect_z_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_19_RECT_Z(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_19_rect_z_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_LARGE_DW_19_rect_z_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_LINEAR_SUBWIN_BC packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_op_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_op_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_sub_op_shift  8
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_sub_op_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_sub_op_shift)
+
+/*define for elementsize field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_elementsize_offset 0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_elementsize_mask   0x00000007
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_elementsize_shift  29
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_ELEMENTSIZE(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_elementsize_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_HEADER_elementsize_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_LO_src_addr_31_0_offset 1
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_LO_src_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_LO_src_addr_31_0_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_HI_src_addr_63_32_offset 2
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_HI_src_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_HI_src_addr_63_32_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for src_x field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_x_offset 3
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_x_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_SRC_X(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_x_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_x_shift)
+
+/*define for src_y field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_y_offset 3
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_y_shift  16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_SRC_Y(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_y_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_3_src_y_shift)
+
+/*define for DW_4 word*/
+/*define for src_z field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_z_offset 4
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_z_mask   0x000007FF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_z_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_SRC_Z(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_z_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_z_shift)
+
+/*define for src_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_pitch_offset 4
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_pitch_mask   0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_pitch_shift  13
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_SRC_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_4_src_pitch_shift)
+
+/*define for DW_5 word*/
+/*define for src_slice_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_5_src_slice_pitch_offset 5
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_5_src_slice_pitch_mask   0x0FFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_5_src_slice_pitch_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_5_SRC_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_5_src_slice_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_5_src_slice_pitch_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_LO_dst_addr_31_0_offset 6
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_HI_dst_addr_63_32_offset 7
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DW_8 word*/
+/*define for dst_x field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_x_offset 8
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_x_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_DST_X(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_x_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_x_shift)
+
+/*define for dst_y field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_y_offset 8
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_y_shift  16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_DST_Y(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_y_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_8_dst_y_shift)
+
+/*define for DW_9 word*/
+/*define for dst_z field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_z_offset 9
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_z_mask   0x000007FF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_z_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_DST_Z(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_z_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_z_shift)
+
+/*define for dst_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_pitch_offset 9
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_pitch_mask   0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_pitch_shift  13
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_DST_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_9_dst_pitch_shift)
+
+/*define for DW_10 word*/
+/*define for dst_slice_pitch field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_10_dst_slice_pitch_offset 10
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_10_dst_slice_pitch_mask   0x0FFFFFFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_10_dst_slice_pitch_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_10_DST_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_10_dst_slice_pitch_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_10_dst_slice_pitch_shift)
+
+/*define for DW_11 word*/
+/*define for rect_x field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_x_offset 11
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_x_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_RECT_X(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_x_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_x_shift)
+
+/*define for rect_y field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_y_offset 11
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_y_shift  16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_RECT_Y(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_y_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_11_rect_y_shift)
+
+/*define for DW_12 word*/
+/*define for rect_z field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_rect_z_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_rect_z_mask   0x000007FF
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_rect_z_shift  0
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_RECT_Z(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_rect_z_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_rect_z_shift)
+
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_sw_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_sw_mask   0x00000003
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_sw_shift  16
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_DST_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_sw_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_sw_shift)
+
+/*define for dst_ha field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_ha_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_ha_mask   0x00000001
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_ha_shift  19
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_DST_HA(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_ha_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_dst_ha_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_sw_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_sw_mask   0x00000003
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_sw_shift  24
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_SRC_SW(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_sw_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_sw_shift)
+
+/*define for src_ha field*/
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_ha_offset 12
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_ha_mask   0x00000001
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_ha_shift  27
+#define SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_SRC_HA(x) (((x) & SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_ha_mask) << SDMA_PKT_COPY_LINEAR_SUBWIN_BC_DW_12_src_ha_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_TILED packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_TILED_HEADER_op_offset 0
+#define SDMA_PKT_COPY_TILED_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COPY_TILED_HEADER_op_shift  0
+#define SDMA_PKT_COPY_TILED_HEADER_OP(x) (((x) & SDMA_PKT_COPY_TILED_HEADER_op_mask) << SDMA_PKT_COPY_TILED_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_TILED_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_TILED_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COPY_TILED_HEADER_sub_op_shift  8
+#define SDMA_PKT_COPY_TILED_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_TILED_HEADER_sub_op_mask) << SDMA_PKT_COPY_TILED_HEADER_sub_op_shift)
+
+/*define for encrypt field*/
+#define SDMA_PKT_COPY_TILED_HEADER_encrypt_offset 0
+#define SDMA_PKT_COPY_TILED_HEADER_encrypt_mask   0x00000001
+#define SDMA_PKT_COPY_TILED_HEADER_encrypt_shift  16
+#define SDMA_PKT_COPY_TILED_HEADER_ENCRYPT(x) (((x) & SDMA_PKT_COPY_TILED_HEADER_encrypt_mask) << SDMA_PKT_COPY_TILED_HEADER_encrypt_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_TILED_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_TILED_HEADER_tmz_mask   0x00000001
+#define SDMA_PKT_COPY_TILED_HEADER_tmz_shift  18
+#define SDMA_PKT_COPY_TILED_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_TILED_HEADER_tmz_mask) << SDMA_PKT_COPY_TILED_HEADER_tmz_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_COPY_TILED_HEADER_cpv_offset 0
+#define SDMA_PKT_COPY_TILED_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_COPY_TILED_HEADER_cpv_shift  19
+#define SDMA_PKT_COPY_TILED_HEADER_CPV(x) (((x) & SDMA_PKT_COPY_TILED_HEADER_cpv_mask) << SDMA_PKT_COPY_TILED_HEADER_cpv_shift)
+
+/*define for detile field*/
+#define SDMA_PKT_COPY_TILED_HEADER_detile_offset 0
+#define SDMA_PKT_COPY_TILED_HEADER_detile_mask   0x00000001
+#define SDMA_PKT_COPY_TILED_HEADER_detile_shift  31
+#define SDMA_PKT_COPY_TILED_HEADER_DETILE(x) (((x) & SDMA_PKT_COPY_TILED_HEADER_detile_mask) << SDMA_PKT_COPY_TILED_HEADER_detile_shift)
+
+/*define for TILED_ADDR_LO word*/
+/*define for tiled_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_TILED_ADDR_LO_tiled_addr_31_0_offset 1
+#define SDMA_PKT_COPY_TILED_TILED_ADDR_LO_tiled_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_TILED_ADDR_LO_tiled_addr_31_0_shift  0
+#define SDMA_PKT_COPY_TILED_TILED_ADDR_LO_TILED_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_TILED_ADDR_LO_tiled_addr_31_0_mask) << SDMA_PKT_COPY_TILED_TILED_ADDR_LO_tiled_addr_31_0_shift)
+
+/*define for TILED_ADDR_HI word*/
+/*define for tiled_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_TILED_ADDR_HI_tiled_addr_63_32_offset 2
+#define SDMA_PKT_COPY_TILED_TILED_ADDR_HI_tiled_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_TILED_ADDR_HI_tiled_addr_63_32_shift  0
+#define SDMA_PKT_COPY_TILED_TILED_ADDR_HI_TILED_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_TILED_ADDR_HI_tiled_addr_63_32_mask) << SDMA_PKT_COPY_TILED_TILED_ADDR_HI_tiled_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for width field*/
+#define SDMA_PKT_COPY_TILED_DW_3_width_offset 3
+#define SDMA_PKT_COPY_TILED_DW_3_width_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_DW_3_width_shift  0
+#define SDMA_PKT_COPY_TILED_DW_3_WIDTH(x) (((x) & SDMA_PKT_COPY_TILED_DW_3_width_mask) << SDMA_PKT_COPY_TILED_DW_3_width_shift)
+
+/*define for DW_4 word*/
+/*define for height field*/
+#define SDMA_PKT_COPY_TILED_DW_4_height_offset 4
+#define SDMA_PKT_COPY_TILED_DW_4_height_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_DW_4_height_shift  0
+#define SDMA_PKT_COPY_TILED_DW_4_HEIGHT(x) (((x) & SDMA_PKT_COPY_TILED_DW_4_height_mask) << SDMA_PKT_COPY_TILED_DW_4_height_shift)
+
+/*define for depth field*/
+#define SDMA_PKT_COPY_TILED_DW_4_depth_offset 4
+#define SDMA_PKT_COPY_TILED_DW_4_depth_mask   0x00001FFF
+#define SDMA_PKT_COPY_TILED_DW_4_depth_shift  16
+#define SDMA_PKT_COPY_TILED_DW_4_DEPTH(x) (((x) & SDMA_PKT_COPY_TILED_DW_4_depth_mask) << SDMA_PKT_COPY_TILED_DW_4_depth_shift)
+
+/*define for DW_5 word*/
+/*define for element_size field*/
+#define SDMA_PKT_COPY_TILED_DW_5_element_size_offset 5
+#define SDMA_PKT_COPY_TILED_DW_5_element_size_mask   0x00000007
+#define SDMA_PKT_COPY_TILED_DW_5_element_size_shift  0
+#define SDMA_PKT_COPY_TILED_DW_5_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_TILED_DW_5_element_size_mask) << SDMA_PKT_COPY_TILED_DW_5_element_size_shift)
+
+/*define for swizzle_mode field*/
+#define SDMA_PKT_COPY_TILED_DW_5_swizzle_mode_offset 5
+#define SDMA_PKT_COPY_TILED_DW_5_swizzle_mode_mask   0x0000001F
+#define SDMA_PKT_COPY_TILED_DW_5_swizzle_mode_shift  3
+#define SDMA_PKT_COPY_TILED_DW_5_SWIZZLE_MODE(x) (((x) & SDMA_PKT_COPY_TILED_DW_5_swizzle_mode_mask) << SDMA_PKT_COPY_TILED_DW_5_swizzle_mode_shift)
+
+/*define for dimension field*/
+#define SDMA_PKT_COPY_TILED_DW_5_dimension_offset 5
+#define SDMA_PKT_COPY_TILED_DW_5_dimension_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_DW_5_dimension_shift  9
+#define SDMA_PKT_COPY_TILED_DW_5_DIMENSION(x) (((x) & SDMA_PKT_COPY_TILED_DW_5_dimension_mask) << SDMA_PKT_COPY_TILED_DW_5_dimension_shift)
+
+/*define for mip_max field*/
+#define SDMA_PKT_COPY_TILED_DW_5_mip_max_offset 5
+#define SDMA_PKT_COPY_TILED_DW_5_mip_max_mask   0x0000000F
+#define SDMA_PKT_COPY_TILED_DW_5_mip_max_shift  16
+#define SDMA_PKT_COPY_TILED_DW_5_MIP_MAX(x) (((x) & SDMA_PKT_COPY_TILED_DW_5_mip_max_mask) << SDMA_PKT_COPY_TILED_DW_5_mip_max_shift)
+
+/*define for DW_6 word*/
+/*define for x field*/
+#define SDMA_PKT_COPY_TILED_DW_6_x_offset 6
+#define SDMA_PKT_COPY_TILED_DW_6_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_DW_6_x_shift  0
+#define SDMA_PKT_COPY_TILED_DW_6_X(x) (((x) & SDMA_PKT_COPY_TILED_DW_6_x_mask) << SDMA_PKT_COPY_TILED_DW_6_x_shift)
+
+/*define for y field*/
+#define SDMA_PKT_COPY_TILED_DW_6_y_offset 6
+#define SDMA_PKT_COPY_TILED_DW_6_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_DW_6_y_shift  16
+#define SDMA_PKT_COPY_TILED_DW_6_Y(x) (((x) & SDMA_PKT_COPY_TILED_DW_6_y_mask) << SDMA_PKT_COPY_TILED_DW_6_y_shift)
+
+/*define for DW_7 word*/
+/*define for z field*/
+#define SDMA_PKT_COPY_TILED_DW_7_z_offset 7
+#define SDMA_PKT_COPY_TILED_DW_7_z_mask   0x00001FFF
+#define SDMA_PKT_COPY_TILED_DW_7_z_shift  0
+#define SDMA_PKT_COPY_TILED_DW_7_Z(x) (((x) & SDMA_PKT_COPY_TILED_DW_7_z_mask) << SDMA_PKT_COPY_TILED_DW_7_z_shift)
+
+/*define for linear_sw field*/
+#define SDMA_PKT_COPY_TILED_DW_7_linear_sw_offset 7
+#define SDMA_PKT_COPY_TILED_DW_7_linear_sw_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_DW_7_linear_sw_shift  16
+#define SDMA_PKT_COPY_TILED_DW_7_LINEAR_SW(x) (((x) & SDMA_PKT_COPY_TILED_DW_7_linear_sw_mask) << SDMA_PKT_COPY_TILED_DW_7_linear_sw_shift)
+
+/*define for linear_cache_policy field*/
+#define SDMA_PKT_COPY_TILED_DW_7_linear_cache_policy_offset 7
+#define SDMA_PKT_COPY_TILED_DW_7_linear_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_TILED_DW_7_linear_cache_policy_shift  18
+#define SDMA_PKT_COPY_TILED_DW_7_LINEAR_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_TILED_DW_7_linear_cache_policy_mask) << SDMA_PKT_COPY_TILED_DW_7_linear_cache_policy_shift)
+
+/*define for tile_sw field*/
+#define SDMA_PKT_COPY_TILED_DW_7_tile_sw_offset 7
+#define SDMA_PKT_COPY_TILED_DW_7_tile_sw_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_DW_7_tile_sw_shift  24
+#define SDMA_PKT_COPY_TILED_DW_7_TILE_SW(x) (((x) & SDMA_PKT_COPY_TILED_DW_7_tile_sw_mask) << SDMA_PKT_COPY_TILED_DW_7_tile_sw_shift)
+
+/*define for tile_cache_policy field*/
+#define SDMA_PKT_COPY_TILED_DW_7_tile_cache_policy_offset 7
+#define SDMA_PKT_COPY_TILED_DW_7_tile_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_TILED_DW_7_tile_cache_policy_shift  26
+#define SDMA_PKT_COPY_TILED_DW_7_TILE_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_TILED_DW_7_tile_cache_policy_mask) << SDMA_PKT_COPY_TILED_DW_7_tile_cache_policy_shift)
+
+/*define for LINEAR_ADDR_LO word*/
+/*define for linear_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_LINEAR_ADDR_LO_linear_addr_31_0_offset 8
+#define SDMA_PKT_COPY_TILED_LINEAR_ADDR_LO_linear_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_LINEAR_ADDR_LO_linear_addr_31_0_shift  0
+#define SDMA_PKT_COPY_TILED_LINEAR_ADDR_LO_LINEAR_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_LINEAR_ADDR_LO_linear_addr_31_0_mask) << SDMA_PKT_COPY_TILED_LINEAR_ADDR_LO_linear_addr_31_0_shift)
+
+/*define for LINEAR_ADDR_HI word*/
+/*define for linear_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_LINEAR_ADDR_HI_linear_addr_63_32_offset 9
+#define SDMA_PKT_COPY_TILED_LINEAR_ADDR_HI_linear_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_LINEAR_ADDR_HI_linear_addr_63_32_shift  0
+#define SDMA_PKT_COPY_TILED_LINEAR_ADDR_HI_LINEAR_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_LINEAR_ADDR_HI_linear_addr_63_32_mask) << SDMA_PKT_COPY_TILED_LINEAR_ADDR_HI_linear_addr_63_32_shift)
+
+/*define for LINEAR_PITCH word*/
+/*define for linear_pitch field*/
+#define SDMA_PKT_COPY_TILED_LINEAR_PITCH_linear_pitch_offset 10
+#define SDMA_PKT_COPY_TILED_LINEAR_PITCH_linear_pitch_mask   0x0007FFFF
+#define SDMA_PKT_COPY_TILED_LINEAR_PITCH_linear_pitch_shift  0
+#define SDMA_PKT_COPY_TILED_LINEAR_PITCH_LINEAR_PITCH(x) (((x) & SDMA_PKT_COPY_TILED_LINEAR_PITCH_linear_pitch_mask) << SDMA_PKT_COPY_TILED_LINEAR_PITCH_linear_pitch_shift)
+
+/*define for LINEAR_SLICE_PITCH word*/
+/*define for linear_slice_pitch field*/
+#define SDMA_PKT_COPY_TILED_LINEAR_SLICE_PITCH_linear_slice_pitch_offset 11
+#define SDMA_PKT_COPY_TILED_LINEAR_SLICE_PITCH_linear_slice_pitch_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_LINEAR_SLICE_PITCH_linear_slice_pitch_shift  0
+#define SDMA_PKT_COPY_TILED_LINEAR_SLICE_PITCH_LINEAR_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_TILED_LINEAR_SLICE_PITCH_linear_slice_pitch_mask) << SDMA_PKT_COPY_TILED_LINEAR_SLICE_PITCH_linear_slice_pitch_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_TILED_COUNT_count_offset 12
+#define SDMA_PKT_COPY_TILED_COUNT_count_mask   0x3FFFFFFF
+#define SDMA_PKT_COPY_TILED_COUNT_count_shift  0
+#define SDMA_PKT_COPY_TILED_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_TILED_COUNT_count_mask) << SDMA_PKT_COPY_TILED_COUNT_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_TILED_BC packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_TILED_BC_HEADER_op_offset 0
+#define SDMA_PKT_COPY_TILED_BC_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COPY_TILED_BC_HEADER_op_shift  0
+#define SDMA_PKT_COPY_TILED_BC_HEADER_OP(x) (((x) & SDMA_PKT_COPY_TILED_BC_HEADER_op_mask) << SDMA_PKT_COPY_TILED_BC_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_TILED_BC_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_TILED_BC_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COPY_TILED_BC_HEADER_sub_op_shift  8
+#define SDMA_PKT_COPY_TILED_BC_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_TILED_BC_HEADER_sub_op_mask) << SDMA_PKT_COPY_TILED_BC_HEADER_sub_op_shift)
+
+/*define for detile field*/
+#define SDMA_PKT_COPY_TILED_BC_HEADER_detile_offset 0
+#define SDMA_PKT_COPY_TILED_BC_HEADER_detile_mask   0x00000001
+#define SDMA_PKT_COPY_TILED_BC_HEADER_detile_shift  31
+#define SDMA_PKT_COPY_TILED_BC_HEADER_DETILE(x) (((x) & SDMA_PKT_COPY_TILED_BC_HEADER_detile_mask) << SDMA_PKT_COPY_TILED_BC_HEADER_detile_shift)
+
+/*define for TILED_ADDR_LO word*/
+/*define for tiled_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_BC_TILED_ADDR_LO_tiled_addr_31_0_offset 1
+#define SDMA_PKT_COPY_TILED_BC_TILED_ADDR_LO_tiled_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_BC_TILED_ADDR_LO_tiled_addr_31_0_shift  0
+#define SDMA_PKT_COPY_TILED_BC_TILED_ADDR_LO_TILED_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_BC_TILED_ADDR_LO_tiled_addr_31_0_mask) << SDMA_PKT_COPY_TILED_BC_TILED_ADDR_LO_tiled_addr_31_0_shift)
+
+/*define for TILED_ADDR_HI word*/
+/*define for tiled_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_BC_TILED_ADDR_HI_tiled_addr_63_32_offset 2
+#define SDMA_PKT_COPY_TILED_BC_TILED_ADDR_HI_tiled_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_BC_TILED_ADDR_HI_tiled_addr_63_32_shift  0
+#define SDMA_PKT_COPY_TILED_BC_TILED_ADDR_HI_TILED_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_BC_TILED_ADDR_HI_tiled_addr_63_32_mask) << SDMA_PKT_COPY_TILED_BC_TILED_ADDR_HI_tiled_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for width field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_3_width_offset 3
+#define SDMA_PKT_COPY_TILED_BC_DW_3_width_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_BC_DW_3_width_shift  0
+#define SDMA_PKT_COPY_TILED_BC_DW_3_WIDTH(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_3_width_mask) << SDMA_PKT_COPY_TILED_BC_DW_3_width_shift)
+
+/*define for DW_4 word*/
+/*define for height field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_4_height_offset 4
+#define SDMA_PKT_COPY_TILED_BC_DW_4_height_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_BC_DW_4_height_shift  0
+#define SDMA_PKT_COPY_TILED_BC_DW_4_HEIGHT(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_4_height_mask) << SDMA_PKT_COPY_TILED_BC_DW_4_height_shift)
+
+/*define for depth field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_4_depth_offset 4
+#define SDMA_PKT_COPY_TILED_BC_DW_4_depth_mask   0x000007FF
+#define SDMA_PKT_COPY_TILED_BC_DW_4_depth_shift  16
+#define SDMA_PKT_COPY_TILED_BC_DW_4_DEPTH(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_4_depth_mask) << SDMA_PKT_COPY_TILED_BC_DW_4_depth_shift)
+
+/*define for DW_5 word*/
+/*define for element_size field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_element_size_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_element_size_mask   0x00000007
+#define SDMA_PKT_COPY_TILED_BC_DW_5_element_size_shift  0
+#define SDMA_PKT_COPY_TILED_BC_DW_5_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_element_size_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_element_size_shift)
+
+/*define for array_mode field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_array_mode_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_array_mode_mask   0x0000000F
+#define SDMA_PKT_COPY_TILED_BC_DW_5_array_mode_shift  3
+#define SDMA_PKT_COPY_TILED_BC_DW_5_ARRAY_MODE(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_array_mode_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_array_mode_shift)
+
+/*define for mit_mode field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_mit_mode_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_mit_mode_mask   0x00000007
+#define SDMA_PKT_COPY_TILED_BC_DW_5_mit_mode_shift  8
+#define SDMA_PKT_COPY_TILED_BC_DW_5_MIT_MODE(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_mit_mode_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_mit_mode_shift)
+
+/*define for tilesplit_size field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_tilesplit_size_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_tilesplit_size_mask   0x00000007
+#define SDMA_PKT_COPY_TILED_BC_DW_5_tilesplit_size_shift  11
+#define SDMA_PKT_COPY_TILED_BC_DW_5_TILESPLIT_SIZE(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_tilesplit_size_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_tilesplit_size_shift)
+
+/*define for bank_w field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_bank_w_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_bank_w_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_BC_DW_5_bank_w_shift  15
+#define SDMA_PKT_COPY_TILED_BC_DW_5_BANK_W(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_bank_w_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_bank_w_shift)
+
+/*define for bank_h field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_bank_h_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_bank_h_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_BC_DW_5_bank_h_shift  18
+#define SDMA_PKT_COPY_TILED_BC_DW_5_BANK_H(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_bank_h_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_bank_h_shift)
+
+/*define for num_bank field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_num_bank_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_num_bank_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_BC_DW_5_num_bank_shift  21
+#define SDMA_PKT_COPY_TILED_BC_DW_5_NUM_BANK(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_num_bank_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_num_bank_shift)
+
+/*define for mat_aspt field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_mat_aspt_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_mat_aspt_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_BC_DW_5_mat_aspt_shift  24
+#define SDMA_PKT_COPY_TILED_BC_DW_5_MAT_ASPT(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_mat_aspt_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_mat_aspt_shift)
+
+/*define for pipe_config field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_5_pipe_config_offset 5
+#define SDMA_PKT_COPY_TILED_BC_DW_5_pipe_config_mask   0x0000001F
+#define SDMA_PKT_COPY_TILED_BC_DW_5_pipe_config_shift  26
+#define SDMA_PKT_COPY_TILED_BC_DW_5_PIPE_CONFIG(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_5_pipe_config_mask) << SDMA_PKT_COPY_TILED_BC_DW_5_pipe_config_shift)
+
+/*define for DW_6 word*/
+/*define for x field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_6_x_offset 6
+#define SDMA_PKT_COPY_TILED_BC_DW_6_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_BC_DW_6_x_shift  0
+#define SDMA_PKT_COPY_TILED_BC_DW_6_X(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_6_x_mask) << SDMA_PKT_COPY_TILED_BC_DW_6_x_shift)
+
+/*define for y field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_6_y_offset 6
+#define SDMA_PKT_COPY_TILED_BC_DW_6_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_BC_DW_6_y_shift  16
+#define SDMA_PKT_COPY_TILED_BC_DW_6_Y(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_6_y_mask) << SDMA_PKT_COPY_TILED_BC_DW_6_y_shift)
+
+/*define for DW_7 word*/
+/*define for z field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_7_z_offset 7
+#define SDMA_PKT_COPY_TILED_BC_DW_7_z_mask   0x000007FF
+#define SDMA_PKT_COPY_TILED_BC_DW_7_z_shift  0
+#define SDMA_PKT_COPY_TILED_BC_DW_7_Z(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_7_z_mask) << SDMA_PKT_COPY_TILED_BC_DW_7_z_shift)
+
+/*define for linear_sw field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_7_linear_sw_offset 7
+#define SDMA_PKT_COPY_TILED_BC_DW_7_linear_sw_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_BC_DW_7_linear_sw_shift  16
+#define SDMA_PKT_COPY_TILED_BC_DW_7_LINEAR_SW(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_7_linear_sw_mask) << SDMA_PKT_COPY_TILED_BC_DW_7_linear_sw_shift)
+
+/*define for tile_sw field*/
+#define SDMA_PKT_COPY_TILED_BC_DW_7_tile_sw_offset 7
+#define SDMA_PKT_COPY_TILED_BC_DW_7_tile_sw_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_BC_DW_7_tile_sw_shift  24
+#define SDMA_PKT_COPY_TILED_BC_DW_7_TILE_SW(x) (((x) & SDMA_PKT_COPY_TILED_BC_DW_7_tile_sw_mask) << SDMA_PKT_COPY_TILED_BC_DW_7_tile_sw_shift)
+
+/*define for LINEAR_ADDR_LO word*/
+/*define for linear_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_LO_linear_addr_31_0_offset 8
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_LO_linear_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_LO_linear_addr_31_0_shift  0
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_LO_LINEAR_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_LO_linear_addr_31_0_mask) << SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_LO_linear_addr_31_0_shift)
+
+/*define for LINEAR_ADDR_HI word*/
+/*define for linear_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_HI_linear_addr_63_32_offset 9
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_HI_linear_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_HI_linear_addr_63_32_shift  0
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_HI_LINEAR_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_HI_linear_addr_63_32_mask) << SDMA_PKT_COPY_TILED_BC_LINEAR_ADDR_HI_linear_addr_63_32_shift)
+
+/*define for LINEAR_PITCH word*/
+/*define for linear_pitch field*/
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_PITCH_linear_pitch_offset 10
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_PITCH_linear_pitch_mask   0x0007FFFF
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_PITCH_linear_pitch_shift  0
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_PITCH_LINEAR_PITCH(x) (((x) & SDMA_PKT_COPY_TILED_BC_LINEAR_PITCH_linear_pitch_mask) << SDMA_PKT_COPY_TILED_BC_LINEAR_PITCH_linear_pitch_shift)
+
+/*define for LINEAR_SLICE_PITCH word*/
+/*define for linear_slice_pitch field*/
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_SLICE_PITCH_linear_slice_pitch_offset 11
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_SLICE_PITCH_linear_slice_pitch_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_SLICE_PITCH_linear_slice_pitch_shift  0
+#define SDMA_PKT_COPY_TILED_BC_LINEAR_SLICE_PITCH_LINEAR_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_TILED_BC_LINEAR_SLICE_PITCH_linear_slice_pitch_mask) << SDMA_PKT_COPY_TILED_BC_LINEAR_SLICE_PITCH_linear_slice_pitch_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_TILED_BC_COUNT_count_offset 12
+#define SDMA_PKT_COPY_TILED_BC_COUNT_count_mask   0x000FFFFF
+#define SDMA_PKT_COPY_TILED_BC_COUNT_count_shift  2
+#define SDMA_PKT_COPY_TILED_BC_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_TILED_BC_COUNT_count_mask) << SDMA_PKT_COPY_TILED_BC_COUNT_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_L2T_BROADCAST packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_op_offset 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_op_shift  0
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_OP(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_HEADER_op_mask) << SDMA_PKT_COPY_L2T_BROADCAST_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_sub_op_shift  8
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_HEADER_sub_op_mask) << SDMA_PKT_COPY_L2T_BROADCAST_HEADER_sub_op_shift)
+
+/*define for encrypt field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_encrypt_offset 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_encrypt_mask   0x00000001
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_encrypt_shift  16
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_ENCRYPT(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_HEADER_encrypt_mask) << SDMA_PKT_COPY_L2T_BROADCAST_HEADER_encrypt_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_tmz_mask   0x00000001
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_tmz_shift  18
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_HEADER_tmz_mask) << SDMA_PKT_COPY_L2T_BROADCAST_HEADER_tmz_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_cpv_offset 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_cpv_shift  19
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_CPV(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_HEADER_cpv_mask) << SDMA_PKT_COPY_L2T_BROADCAST_HEADER_cpv_shift)
+
+/*define for videocopy field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_videocopy_offset 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_videocopy_mask   0x00000001
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_videocopy_shift  26
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_VIDEOCOPY(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_HEADER_videocopy_mask) << SDMA_PKT_COPY_L2T_BROADCAST_HEADER_videocopy_shift)
+
+/*define for broadcast field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_broadcast_offset 0
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_broadcast_mask   0x00000001
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_broadcast_shift  27
+#define SDMA_PKT_COPY_L2T_BROADCAST_HEADER_BROADCAST(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_HEADER_broadcast_mask) << SDMA_PKT_COPY_L2T_BROADCAST_HEADER_broadcast_shift)
+
+/*define for TILED_ADDR_LO_0 word*/
+/*define for tiled_addr0_31_0 field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_0_tiled_addr0_31_0_offset 1
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_0_tiled_addr0_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_0_tiled_addr0_31_0_shift  0
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_0_TILED_ADDR0_31_0(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_0_tiled_addr0_31_0_mask) << SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_0_tiled_addr0_31_0_shift)
+
+/*define for TILED_ADDR_HI_0 word*/
+/*define for tiled_addr0_63_32 field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_0_tiled_addr0_63_32_offset 2
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_0_tiled_addr0_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_0_tiled_addr0_63_32_shift  0
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_0_TILED_ADDR0_63_32(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_0_tiled_addr0_63_32_mask) << SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_0_tiled_addr0_63_32_shift)
+
+/*define for TILED_ADDR_LO_1 word*/
+/*define for tiled_addr1_31_0 field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_1_tiled_addr1_31_0_offset 3
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_1_tiled_addr1_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_1_tiled_addr1_31_0_shift  0
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_1_TILED_ADDR1_31_0(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_1_tiled_addr1_31_0_mask) << SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_LO_1_tiled_addr1_31_0_shift)
+
+/*define for TILED_ADDR_HI_1 word*/
+/*define for tiled_addr1_63_32 field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_1_tiled_addr1_63_32_offset 4
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_1_tiled_addr1_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_1_tiled_addr1_63_32_shift  0
+#define SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_1_TILED_ADDR1_63_32(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_1_tiled_addr1_63_32_mask) << SDMA_PKT_COPY_L2T_BROADCAST_TILED_ADDR_HI_1_tiled_addr1_63_32_shift)
+
+/*define for DW_5 word*/
+/*define for width field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_5_width_offset 5
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_5_width_mask   0x00003FFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_5_width_shift  0
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_5_WIDTH(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_5_width_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_5_width_shift)
+
+/*define for DW_6 word*/
+/*define for height field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_6_height_offset 6
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_6_height_mask   0x00003FFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_6_height_shift  0
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_6_HEIGHT(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_6_height_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_6_height_shift)
+
+/*define for depth field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_6_depth_offset 6
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_6_depth_mask   0x00001FFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_6_depth_shift  16
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_6_DEPTH(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_6_depth_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_6_depth_shift)
+
+/*define for DW_7 word*/
+/*define for element_size field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_element_size_offset 7
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_element_size_mask   0x00000007
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_element_size_shift  0
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_7_element_size_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_7_element_size_shift)
+
+/*define for swizzle_mode field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_swizzle_mode_offset 7
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_swizzle_mode_mask   0x0000001F
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_swizzle_mode_shift  3
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_SWIZZLE_MODE(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_7_swizzle_mode_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_7_swizzle_mode_shift)
+
+/*define for dimension field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_dimension_offset 7
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_dimension_mask   0x00000003
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_dimension_shift  9
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_DIMENSION(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_7_dimension_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_7_dimension_shift)
+
+/*define for mip_max field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_mip_max_offset 7
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_mip_max_mask   0x0000000F
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_mip_max_shift  16
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_7_MIP_MAX(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_7_mip_max_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_7_mip_max_shift)
+
+/*define for DW_8 word*/
+/*define for x field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_8_x_offset 8
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_8_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_8_x_shift  0
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_8_X(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_8_x_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_8_x_shift)
+
+/*define for y field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_8_y_offset 8
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_8_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_8_y_shift  16
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_8_Y(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_8_y_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_8_y_shift)
+
+/*define for DW_9 word*/
+/*define for z field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_9_z_offset 9
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_9_z_mask   0x00001FFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_9_z_shift  0
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_9_Z(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_9_z_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_9_z_shift)
+
+/*define for DW_10 word*/
+/*define for dst2_sw field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_dst2_sw_offset 10
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_dst2_sw_mask   0x00000003
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_dst2_sw_shift  8
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_DST2_SW(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_10_dst2_sw_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_10_dst2_sw_shift)
+
+/*define for dst2_cache_policy field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_dst2_cache_policy_offset 10
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_dst2_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_dst2_cache_policy_shift  10
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_DST2_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_10_dst2_cache_policy_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_10_dst2_cache_policy_shift)
+
+/*define for linear_sw field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_linear_sw_offset 10
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_linear_sw_mask   0x00000003
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_linear_sw_shift  16
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_LINEAR_SW(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_10_linear_sw_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_10_linear_sw_shift)
+
+/*define for linear_cache_policy field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_linear_cache_policy_offset 10
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_linear_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_linear_cache_policy_shift  18
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_LINEAR_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_10_linear_cache_policy_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_10_linear_cache_policy_shift)
+
+/*define for tile_sw field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_tile_sw_offset 10
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_tile_sw_mask   0x00000003
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_tile_sw_shift  24
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_TILE_SW(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_10_tile_sw_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_10_tile_sw_shift)
+
+/*define for tile_cache_policy field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_tile_cache_policy_offset 10
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_tile_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_tile_cache_policy_shift  26
+#define SDMA_PKT_COPY_L2T_BROADCAST_DW_10_TILE_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_DW_10_tile_cache_policy_mask) << SDMA_PKT_COPY_L2T_BROADCAST_DW_10_tile_cache_policy_shift)
+
+/*define for LINEAR_ADDR_LO word*/
+/*define for linear_addr_31_0 field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_LO_linear_addr_31_0_offset 11
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_LO_linear_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_LO_linear_addr_31_0_shift  0
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_LO_LINEAR_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_LO_linear_addr_31_0_mask) << SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_LO_linear_addr_31_0_shift)
+
+/*define for LINEAR_ADDR_HI word*/
+/*define for linear_addr_63_32 field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_HI_linear_addr_63_32_offset 12
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_HI_linear_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_HI_linear_addr_63_32_shift  0
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_HI_LINEAR_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_HI_linear_addr_63_32_mask) << SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_ADDR_HI_linear_addr_63_32_shift)
+
+/*define for LINEAR_PITCH word*/
+/*define for linear_pitch field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_PITCH_linear_pitch_offset 13
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_PITCH_linear_pitch_mask   0x0007FFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_PITCH_linear_pitch_shift  0
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_PITCH_LINEAR_PITCH(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_PITCH_linear_pitch_mask) << SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_PITCH_linear_pitch_shift)
+
+/*define for LINEAR_SLICE_PITCH word*/
+/*define for linear_slice_pitch field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_SLICE_PITCH_linear_slice_pitch_offset 14
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_SLICE_PITCH_linear_slice_pitch_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_SLICE_PITCH_linear_slice_pitch_shift  0
+#define SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_SLICE_PITCH_LINEAR_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_SLICE_PITCH_linear_slice_pitch_mask) << SDMA_PKT_COPY_L2T_BROADCAST_LINEAR_SLICE_PITCH_linear_slice_pitch_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_L2T_BROADCAST_COUNT_count_offset 15
+#define SDMA_PKT_COPY_L2T_BROADCAST_COUNT_count_mask   0x3FFFFFFF
+#define SDMA_PKT_COPY_L2T_BROADCAST_COUNT_count_shift  0
+#define SDMA_PKT_COPY_L2T_BROADCAST_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_L2T_BROADCAST_COUNT_count_mask) << SDMA_PKT_COPY_L2T_BROADCAST_COUNT_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_T2T packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_T2T_HEADER_op_offset 0
+#define SDMA_PKT_COPY_T2T_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COPY_T2T_HEADER_op_shift  0
+#define SDMA_PKT_COPY_T2T_HEADER_OP(x) (((x) & SDMA_PKT_COPY_T2T_HEADER_op_mask) << SDMA_PKT_COPY_T2T_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_T2T_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_T2T_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COPY_T2T_HEADER_sub_op_shift  8
+#define SDMA_PKT_COPY_T2T_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_T2T_HEADER_sub_op_mask) << SDMA_PKT_COPY_T2T_HEADER_sub_op_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_T2T_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_T2T_HEADER_tmz_mask   0x00000001
+#define SDMA_PKT_COPY_T2T_HEADER_tmz_shift  18
+#define SDMA_PKT_COPY_T2T_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_T2T_HEADER_tmz_mask) << SDMA_PKT_COPY_T2T_HEADER_tmz_shift)
+
+/*define for dcc field*/
+#define SDMA_PKT_COPY_T2T_HEADER_dcc_offset 0
+#define SDMA_PKT_COPY_T2T_HEADER_dcc_mask   0x00000001
+#define SDMA_PKT_COPY_T2T_HEADER_dcc_shift  19
+#define SDMA_PKT_COPY_T2T_HEADER_DCC(x) (((x) & SDMA_PKT_COPY_T2T_HEADER_dcc_mask) << SDMA_PKT_COPY_T2T_HEADER_dcc_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_COPY_T2T_HEADER_cpv_offset 0
+#define SDMA_PKT_COPY_T2T_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_COPY_T2T_HEADER_cpv_shift  28
+#define SDMA_PKT_COPY_T2T_HEADER_CPV(x) (((x) & SDMA_PKT_COPY_T2T_HEADER_cpv_mask) << SDMA_PKT_COPY_T2T_HEADER_cpv_shift)
+
+/*define for dcc_dir field*/
+#define SDMA_PKT_COPY_T2T_HEADER_dcc_dir_offset 0
+#define SDMA_PKT_COPY_T2T_HEADER_dcc_dir_mask   0x00000001
+#define SDMA_PKT_COPY_T2T_HEADER_dcc_dir_shift  31
+#define SDMA_PKT_COPY_T2T_HEADER_DCC_DIR(x) (((x) & SDMA_PKT_COPY_T2T_HEADER_dcc_dir_mask) << SDMA_PKT_COPY_T2T_HEADER_dcc_dir_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_T2T_SRC_ADDR_LO_src_addr_31_0_offset 1
+#define SDMA_PKT_COPY_T2T_SRC_ADDR_LO_src_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_SRC_ADDR_LO_src_addr_31_0_shift  0
+#define SDMA_PKT_COPY_T2T_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_T2T_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_T2T_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_T2T_SRC_ADDR_HI_src_addr_63_32_offset 2
+#define SDMA_PKT_COPY_T2T_SRC_ADDR_HI_src_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_SRC_ADDR_HI_src_addr_63_32_shift  0
+#define SDMA_PKT_COPY_T2T_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_T2T_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_T2T_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for src_x field*/
+#define SDMA_PKT_COPY_T2T_DW_3_src_x_offset 3
+#define SDMA_PKT_COPY_T2T_DW_3_src_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_3_src_x_shift  0
+#define SDMA_PKT_COPY_T2T_DW_3_SRC_X(x) (((x) & SDMA_PKT_COPY_T2T_DW_3_src_x_mask) << SDMA_PKT_COPY_T2T_DW_3_src_x_shift)
+
+/*define for src_y field*/
+#define SDMA_PKT_COPY_T2T_DW_3_src_y_offset 3
+#define SDMA_PKT_COPY_T2T_DW_3_src_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_3_src_y_shift  16
+#define SDMA_PKT_COPY_T2T_DW_3_SRC_Y(x) (((x) & SDMA_PKT_COPY_T2T_DW_3_src_y_mask) << SDMA_PKT_COPY_T2T_DW_3_src_y_shift)
+
+/*define for DW_4 word*/
+/*define for src_z field*/
+#define SDMA_PKT_COPY_T2T_DW_4_src_z_offset 4
+#define SDMA_PKT_COPY_T2T_DW_4_src_z_mask   0x00001FFF
+#define SDMA_PKT_COPY_T2T_DW_4_src_z_shift  0
+#define SDMA_PKT_COPY_T2T_DW_4_SRC_Z(x) (((x) & SDMA_PKT_COPY_T2T_DW_4_src_z_mask) << SDMA_PKT_COPY_T2T_DW_4_src_z_shift)
+
+/*define for src_width field*/
+#define SDMA_PKT_COPY_T2T_DW_4_src_width_offset 4
+#define SDMA_PKT_COPY_T2T_DW_4_src_width_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_4_src_width_shift  16
+#define SDMA_PKT_COPY_T2T_DW_4_SRC_WIDTH(x) (((x) & SDMA_PKT_COPY_T2T_DW_4_src_width_mask) << SDMA_PKT_COPY_T2T_DW_4_src_width_shift)
+
+/*define for DW_5 word*/
+/*define for src_height field*/
+#define SDMA_PKT_COPY_T2T_DW_5_src_height_offset 5
+#define SDMA_PKT_COPY_T2T_DW_5_src_height_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_5_src_height_shift  0
+#define SDMA_PKT_COPY_T2T_DW_5_SRC_HEIGHT(x) (((x) & SDMA_PKT_COPY_T2T_DW_5_src_height_mask) << SDMA_PKT_COPY_T2T_DW_5_src_height_shift)
+
+/*define for src_depth field*/
+#define SDMA_PKT_COPY_T2T_DW_5_src_depth_offset 5
+#define SDMA_PKT_COPY_T2T_DW_5_src_depth_mask   0x00001FFF
+#define SDMA_PKT_COPY_T2T_DW_5_src_depth_shift  16
+#define SDMA_PKT_COPY_T2T_DW_5_SRC_DEPTH(x) (((x) & SDMA_PKT_COPY_T2T_DW_5_src_depth_mask) << SDMA_PKT_COPY_T2T_DW_5_src_depth_shift)
+
+/*define for DW_6 word*/
+/*define for src_element_size field*/
+#define SDMA_PKT_COPY_T2T_DW_6_src_element_size_offset 6
+#define SDMA_PKT_COPY_T2T_DW_6_src_element_size_mask   0x00000007
+#define SDMA_PKT_COPY_T2T_DW_6_src_element_size_shift  0
+#define SDMA_PKT_COPY_T2T_DW_6_SRC_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_T2T_DW_6_src_element_size_mask) << SDMA_PKT_COPY_T2T_DW_6_src_element_size_shift)
+
+/*define for src_swizzle_mode field*/
+#define SDMA_PKT_COPY_T2T_DW_6_src_swizzle_mode_offset 6
+#define SDMA_PKT_COPY_T2T_DW_6_src_swizzle_mode_mask   0x0000001F
+#define SDMA_PKT_COPY_T2T_DW_6_src_swizzle_mode_shift  3
+#define SDMA_PKT_COPY_T2T_DW_6_SRC_SWIZZLE_MODE(x) (((x) & SDMA_PKT_COPY_T2T_DW_6_src_swizzle_mode_mask) << SDMA_PKT_COPY_T2T_DW_6_src_swizzle_mode_shift)
+
+/*define for src_dimension field*/
+#define SDMA_PKT_COPY_T2T_DW_6_src_dimension_offset 6
+#define SDMA_PKT_COPY_T2T_DW_6_src_dimension_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_DW_6_src_dimension_shift  9
+#define SDMA_PKT_COPY_T2T_DW_6_SRC_DIMENSION(x) (((x) & SDMA_PKT_COPY_T2T_DW_6_src_dimension_mask) << SDMA_PKT_COPY_T2T_DW_6_src_dimension_shift)
+
+/*define for src_mip_max field*/
+#define SDMA_PKT_COPY_T2T_DW_6_src_mip_max_offset 6
+#define SDMA_PKT_COPY_T2T_DW_6_src_mip_max_mask   0x0000000F
+#define SDMA_PKT_COPY_T2T_DW_6_src_mip_max_shift  16
+#define SDMA_PKT_COPY_T2T_DW_6_SRC_MIP_MAX(x) (((x) & SDMA_PKT_COPY_T2T_DW_6_src_mip_max_mask) << SDMA_PKT_COPY_T2T_DW_6_src_mip_max_shift)
+
+/*define for src_mip_id field*/
+#define SDMA_PKT_COPY_T2T_DW_6_src_mip_id_offset 6
+#define SDMA_PKT_COPY_T2T_DW_6_src_mip_id_mask   0x0000000F
+#define SDMA_PKT_COPY_T2T_DW_6_src_mip_id_shift  20
+#define SDMA_PKT_COPY_T2T_DW_6_SRC_MIP_ID(x) (((x) & SDMA_PKT_COPY_T2T_DW_6_src_mip_id_mask) << SDMA_PKT_COPY_T2T_DW_6_src_mip_id_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_T2T_DST_ADDR_LO_dst_addr_31_0_offset 7
+#define SDMA_PKT_COPY_T2T_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_COPY_T2T_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_T2T_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_T2T_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_T2T_DST_ADDR_HI_dst_addr_63_32_offset 8
+#define SDMA_PKT_COPY_T2T_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_COPY_T2T_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_T2T_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_T2T_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DW_9 word*/
+/*define for dst_x field*/
+#define SDMA_PKT_COPY_T2T_DW_9_dst_x_offset 9
+#define SDMA_PKT_COPY_T2T_DW_9_dst_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_9_dst_x_shift  0
+#define SDMA_PKT_COPY_T2T_DW_9_DST_X(x) (((x) & SDMA_PKT_COPY_T2T_DW_9_dst_x_mask) << SDMA_PKT_COPY_T2T_DW_9_dst_x_shift)
+
+/*define for dst_y field*/
+#define SDMA_PKT_COPY_T2T_DW_9_dst_y_offset 9
+#define SDMA_PKT_COPY_T2T_DW_9_dst_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_9_dst_y_shift  16
+#define SDMA_PKT_COPY_T2T_DW_9_DST_Y(x) (((x) & SDMA_PKT_COPY_T2T_DW_9_dst_y_mask) << SDMA_PKT_COPY_T2T_DW_9_dst_y_shift)
+
+/*define for DW_10 word*/
+/*define for dst_z field*/
+#define SDMA_PKT_COPY_T2T_DW_10_dst_z_offset 10
+#define SDMA_PKT_COPY_T2T_DW_10_dst_z_mask   0x00001FFF
+#define SDMA_PKT_COPY_T2T_DW_10_dst_z_shift  0
+#define SDMA_PKT_COPY_T2T_DW_10_DST_Z(x) (((x) & SDMA_PKT_COPY_T2T_DW_10_dst_z_mask) << SDMA_PKT_COPY_T2T_DW_10_dst_z_shift)
+
+/*define for dst_width field*/
+#define SDMA_PKT_COPY_T2T_DW_10_dst_width_offset 10
+#define SDMA_PKT_COPY_T2T_DW_10_dst_width_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_10_dst_width_shift  16
+#define SDMA_PKT_COPY_T2T_DW_10_DST_WIDTH(x) (((x) & SDMA_PKT_COPY_T2T_DW_10_dst_width_mask) << SDMA_PKT_COPY_T2T_DW_10_dst_width_shift)
+
+/*define for DW_11 word*/
+/*define for dst_height field*/
+#define SDMA_PKT_COPY_T2T_DW_11_dst_height_offset 11
+#define SDMA_PKT_COPY_T2T_DW_11_dst_height_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_11_dst_height_shift  0
+#define SDMA_PKT_COPY_T2T_DW_11_DST_HEIGHT(x) (((x) & SDMA_PKT_COPY_T2T_DW_11_dst_height_mask) << SDMA_PKT_COPY_T2T_DW_11_dst_height_shift)
+
+/*define for dst_depth field*/
+#define SDMA_PKT_COPY_T2T_DW_11_dst_depth_offset 11
+#define SDMA_PKT_COPY_T2T_DW_11_dst_depth_mask   0x00001FFF
+#define SDMA_PKT_COPY_T2T_DW_11_dst_depth_shift  16
+#define SDMA_PKT_COPY_T2T_DW_11_DST_DEPTH(x) (((x) & SDMA_PKT_COPY_T2T_DW_11_dst_depth_mask) << SDMA_PKT_COPY_T2T_DW_11_dst_depth_shift)
+
+/*define for DW_12 word*/
+/*define for dst_element_size field*/
+#define SDMA_PKT_COPY_T2T_DW_12_dst_element_size_offset 12
+#define SDMA_PKT_COPY_T2T_DW_12_dst_element_size_mask   0x00000007
+#define SDMA_PKT_COPY_T2T_DW_12_dst_element_size_shift  0
+#define SDMA_PKT_COPY_T2T_DW_12_DST_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_T2T_DW_12_dst_element_size_mask) << SDMA_PKT_COPY_T2T_DW_12_dst_element_size_shift)
+
+/*define for dst_swizzle_mode field*/
+#define SDMA_PKT_COPY_T2T_DW_12_dst_swizzle_mode_offset 12
+#define SDMA_PKT_COPY_T2T_DW_12_dst_swizzle_mode_mask   0x0000001F
+#define SDMA_PKT_COPY_T2T_DW_12_dst_swizzle_mode_shift  3
+#define SDMA_PKT_COPY_T2T_DW_12_DST_SWIZZLE_MODE(x) (((x) & SDMA_PKT_COPY_T2T_DW_12_dst_swizzle_mode_mask) << SDMA_PKT_COPY_T2T_DW_12_dst_swizzle_mode_shift)
+
+/*define for dst_dimension field*/
+#define SDMA_PKT_COPY_T2T_DW_12_dst_dimension_offset 12
+#define SDMA_PKT_COPY_T2T_DW_12_dst_dimension_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_DW_12_dst_dimension_shift  9
+#define SDMA_PKT_COPY_T2T_DW_12_DST_DIMENSION(x) (((x) & SDMA_PKT_COPY_T2T_DW_12_dst_dimension_mask) << SDMA_PKT_COPY_T2T_DW_12_dst_dimension_shift)
+
+/*define for dst_mip_max field*/
+#define SDMA_PKT_COPY_T2T_DW_12_dst_mip_max_offset 12
+#define SDMA_PKT_COPY_T2T_DW_12_dst_mip_max_mask   0x0000000F
+#define SDMA_PKT_COPY_T2T_DW_12_dst_mip_max_shift  16
+#define SDMA_PKT_COPY_T2T_DW_12_DST_MIP_MAX(x) (((x) & SDMA_PKT_COPY_T2T_DW_12_dst_mip_max_mask) << SDMA_PKT_COPY_T2T_DW_12_dst_mip_max_shift)
+
+/*define for dst_mip_id field*/
+#define SDMA_PKT_COPY_T2T_DW_12_dst_mip_id_offset 12
+#define SDMA_PKT_COPY_T2T_DW_12_dst_mip_id_mask   0x0000000F
+#define SDMA_PKT_COPY_T2T_DW_12_dst_mip_id_shift  20
+#define SDMA_PKT_COPY_T2T_DW_12_DST_MIP_ID(x) (((x) & SDMA_PKT_COPY_T2T_DW_12_dst_mip_id_mask) << SDMA_PKT_COPY_T2T_DW_12_dst_mip_id_shift)
+
+/*define for DW_13 word*/
+/*define for rect_x field*/
+#define SDMA_PKT_COPY_T2T_DW_13_rect_x_offset 13
+#define SDMA_PKT_COPY_T2T_DW_13_rect_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_13_rect_x_shift  0
+#define SDMA_PKT_COPY_T2T_DW_13_RECT_X(x) (((x) & SDMA_PKT_COPY_T2T_DW_13_rect_x_mask) << SDMA_PKT_COPY_T2T_DW_13_rect_x_shift)
+
+/*define for rect_y field*/
+#define SDMA_PKT_COPY_T2T_DW_13_rect_y_offset 13
+#define SDMA_PKT_COPY_T2T_DW_13_rect_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_DW_13_rect_y_shift  16
+#define SDMA_PKT_COPY_T2T_DW_13_RECT_Y(x) (((x) & SDMA_PKT_COPY_T2T_DW_13_rect_y_mask) << SDMA_PKT_COPY_T2T_DW_13_rect_y_shift)
+
+/*define for DW_14 word*/
+/*define for rect_z field*/
+#define SDMA_PKT_COPY_T2T_DW_14_rect_z_offset 14
+#define SDMA_PKT_COPY_T2T_DW_14_rect_z_mask   0x00001FFF
+#define SDMA_PKT_COPY_T2T_DW_14_rect_z_shift  0
+#define SDMA_PKT_COPY_T2T_DW_14_RECT_Z(x) (((x) & SDMA_PKT_COPY_T2T_DW_14_rect_z_mask) << SDMA_PKT_COPY_T2T_DW_14_rect_z_shift)
+
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_T2T_DW_14_dst_sw_offset 14
+#define SDMA_PKT_COPY_T2T_DW_14_dst_sw_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_DW_14_dst_sw_shift  16
+#define SDMA_PKT_COPY_T2T_DW_14_DST_SW(x) (((x) & SDMA_PKT_COPY_T2T_DW_14_dst_sw_mask) << SDMA_PKT_COPY_T2T_DW_14_dst_sw_shift)
+
+/*define for dst_cache_policy field*/
+#define SDMA_PKT_COPY_T2T_DW_14_dst_cache_policy_offset 14
+#define SDMA_PKT_COPY_T2T_DW_14_dst_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_T2T_DW_14_dst_cache_policy_shift  18
+#define SDMA_PKT_COPY_T2T_DW_14_DST_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_T2T_DW_14_dst_cache_policy_mask) << SDMA_PKT_COPY_T2T_DW_14_dst_cache_policy_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_T2T_DW_14_src_sw_offset 14
+#define SDMA_PKT_COPY_T2T_DW_14_src_sw_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_DW_14_src_sw_shift  24
+#define SDMA_PKT_COPY_T2T_DW_14_SRC_SW(x) (((x) & SDMA_PKT_COPY_T2T_DW_14_src_sw_mask) << SDMA_PKT_COPY_T2T_DW_14_src_sw_shift)
+
+/*define for src_cache_policy field*/
+#define SDMA_PKT_COPY_T2T_DW_14_src_cache_policy_offset 14
+#define SDMA_PKT_COPY_T2T_DW_14_src_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_T2T_DW_14_src_cache_policy_shift  26
+#define SDMA_PKT_COPY_T2T_DW_14_SRC_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_T2T_DW_14_src_cache_policy_mask) << SDMA_PKT_COPY_T2T_DW_14_src_cache_policy_shift)
+
+/*define for META_ADDR_LO word*/
+/*define for meta_addr_31_0 field*/
+#define SDMA_PKT_COPY_T2T_META_ADDR_LO_meta_addr_31_0_offset 15
+#define SDMA_PKT_COPY_T2T_META_ADDR_LO_meta_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_META_ADDR_LO_meta_addr_31_0_shift  0
+#define SDMA_PKT_COPY_T2T_META_ADDR_LO_META_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_T2T_META_ADDR_LO_meta_addr_31_0_mask) << SDMA_PKT_COPY_T2T_META_ADDR_LO_meta_addr_31_0_shift)
+
+/*define for META_ADDR_HI word*/
+/*define for meta_addr_63_32 field*/
+#define SDMA_PKT_COPY_T2T_META_ADDR_HI_meta_addr_63_32_offset 16
+#define SDMA_PKT_COPY_T2T_META_ADDR_HI_meta_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_META_ADDR_HI_meta_addr_63_32_shift  0
+#define SDMA_PKT_COPY_T2T_META_ADDR_HI_META_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_T2T_META_ADDR_HI_meta_addr_63_32_mask) << SDMA_PKT_COPY_T2T_META_ADDR_HI_meta_addr_63_32_shift)
+
+/*define for META_CONFIG word*/
+/*define for data_format field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_data_format_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_data_format_mask   0x0000007F
+#define SDMA_PKT_COPY_T2T_META_CONFIG_data_format_shift  0
+#define SDMA_PKT_COPY_T2T_META_CONFIG_DATA_FORMAT(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_data_format_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_data_format_shift)
+
+/*define for color_transform_disable field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_color_transform_disable_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_color_transform_disable_mask   0x00000001
+#define SDMA_PKT_COPY_T2T_META_CONFIG_color_transform_disable_shift  7
+#define SDMA_PKT_COPY_T2T_META_CONFIG_COLOR_TRANSFORM_DISABLE(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_color_transform_disable_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_color_transform_disable_shift)
+
+/*define for alpha_is_on_msb field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_alpha_is_on_msb_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_alpha_is_on_msb_mask   0x00000001
+#define SDMA_PKT_COPY_T2T_META_CONFIG_alpha_is_on_msb_shift  8
+#define SDMA_PKT_COPY_T2T_META_CONFIG_ALPHA_IS_ON_MSB(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_alpha_is_on_msb_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_alpha_is_on_msb_shift)
+
+/*define for number_type field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_number_type_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_number_type_mask   0x00000007
+#define SDMA_PKT_COPY_T2T_META_CONFIG_number_type_shift  9
+#define SDMA_PKT_COPY_T2T_META_CONFIG_NUMBER_TYPE(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_number_type_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_number_type_shift)
+
+/*define for surface_type field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_surface_type_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_surface_type_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_META_CONFIG_surface_type_shift  12
+#define SDMA_PKT_COPY_T2T_META_CONFIG_SURFACE_TYPE(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_surface_type_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_surface_type_shift)
+
+/*define for meta_llc field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_meta_llc_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_meta_llc_mask   0x00000001
+#define SDMA_PKT_COPY_T2T_META_CONFIG_meta_llc_shift  14
+#define SDMA_PKT_COPY_T2T_META_CONFIG_META_LLC(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_meta_llc_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_meta_llc_shift)
+
+/*define for max_comp_block_size field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_max_comp_block_size_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_max_comp_block_size_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_META_CONFIG_max_comp_block_size_shift  24
+#define SDMA_PKT_COPY_T2T_META_CONFIG_MAX_COMP_BLOCK_SIZE(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_max_comp_block_size_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_max_comp_block_size_shift)
+
+/*define for max_uncomp_block_size field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_max_uncomp_block_size_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_max_uncomp_block_size_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_META_CONFIG_max_uncomp_block_size_shift  26
+#define SDMA_PKT_COPY_T2T_META_CONFIG_MAX_UNCOMP_BLOCK_SIZE(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_max_uncomp_block_size_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_max_uncomp_block_size_shift)
+
+/*define for write_compress_enable field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_write_compress_enable_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_write_compress_enable_mask   0x00000001
+#define SDMA_PKT_COPY_T2T_META_CONFIG_write_compress_enable_shift  28
+#define SDMA_PKT_COPY_T2T_META_CONFIG_WRITE_COMPRESS_ENABLE(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_write_compress_enable_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_write_compress_enable_shift)
+
+/*define for meta_tmz field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_meta_tmz_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_meta_tmz_mask   0x00000001
+#define SDMA_PKT_COPY_T2T_META_CONFIG_meta_tmz_shift  29
+#define SDMA_PKT_COPY_T2T_META_CONFIG_META_TMZ(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_meta_tmz_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_meta_tmz_shift)
+
+/*define for pipe_aligned field*/
+#define SDMA_PKT_COPY_T2T_META_CONFIG_pipe_aligned_offset 17
+#define SDMA_PKT_COPY_T2T_META_CONFIG_pipe_aligned_mask   0x00000001
+#define SDMA_PKT_COPY_T2T_META_CONFIG_pipe_aligned_shift  31
+#define SDMA_PKT_COPY_T2T_META_CONFIG_PIPE_ALIGNED(x) (((x) & SDMA_PKT_COPY_T2T_META_CONFIG_pipe_aligned_mask) << SDMA_PKT_COPY_T2T_META_CONFIG_pipe_aligned_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_T2T_BC packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_T2T_BC_HEADER_op_offset 0
+#define SDMA_PKT_COPY_T2T_BC_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COPY_T2T_BC_HEADER_op_shift  0
+#define SDMA_PKT_COPY_T2T_BC_HEADER_OP(x) (((x) & SDMA_PKT_COPY_T2T_BC_HEADER_op_mask) << SDMA_PKT_COPY_T2T_BC_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_T2T_BC_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_T2T_BC_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COPY_T2T_BC_HEADER_sub_op_shift  8
+#define SDMA_PKT_COPY_T2T_BC_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_T2T_BC_HEADER_sub_op_mask) << SDMA_PKT_COPY_T2T_BC_HEADER_sub_op_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_COPY_T2T_BC_SRC_ADDR_LO_src_addr_31_0_offset 1
+#define SDMA_PKT_COPY_T2T_BC_SRC_ADDR_LO_src_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_BC_SRC_ADDR_LO_src_addr_31_0_shift  0
+#define SDMA_PKT_COPY_T2T_BC_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_T2T_BC_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_COPY_T2T_BC_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_COPY_T2T_BC_SRC_ADDR_HI_src_addr_63_32_offset 2
+#define SDMA_PKT_COPY_T2T_BC_SRC_ADDR_HI_src_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_BC_SRC_ADDR_HI_src_addr_63_32_shift  0
+#define SDMA_PKT_COPY_T2T_BC_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_T2T_BC_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_COPY_T2T_BC_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for src_x field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_3_src_x_offset 3
+#define SDMA_PKT_COPY_T2T_BC_DW_3_src_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_3_src_x_shift  0
+#define SDMA_PKT_COPY_T2T_BC_DW_3_SRC_X(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_3_src_x_mask) << SDMA_PKT_COPY_T2T_BC_DW_3_src_x_shift)
+
+/*define for src_y field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_3_src_y_offset 3
+#define SDMA_PKT_COPY_T2T_BC_DW_3_src_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_3_src_y_shift  16
+#define SDMA_PKT_COPY_T2T_BC_DW_3_SRC_Y(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_3_src_y_mask) << SDMA_PKT_COPY_T2T_BC_DW_3_src_y_shift)
+
+/*define for DW_4 word*/
+/*define for src_z field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_4_src_z_offset 4
+#define SDMA_PKT_COPY_T2T_BC_DW_4_src_z_mask   0x000007FF
+#define SDMA_PKT_COPY_T2T_BC_DW_4_src_z_shift  0
+#define SDMA_PKT_COPY_T2T_BC_DW_4_SRC_Z(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_4_src_z_mask) << SDMA_PKT_COPY_T2T_BC_DW_4_src_z_shift)
+
+/*define for src_width field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_4_src_width_offset 4
+#define SDMA_PKT_COPY_T2T_BC_DW_4_src_width_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_4_src_width_shift  16
+#define SDMA_PKT_COPY_T2T_BC_DW_4_SRC_WIDTH(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_4_src_width_mask) << SDMA_PKT_COPY_T2T_BC_DW_4_src_width_shift)
+
+/*define for DW_5 word*/
+/*define for src_height field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_5_src_height_offset 5
+#define SDMA_PKT_COPY_T2T_BC_DW_5_src_height_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_5_src_height_shift  0
+#define SDMA_PKT_COPY_T2T_BC_DW_5_SRC_HEIGHT(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_5_src_height_mask) << SDMA_PKT_COPY_T2T_BC_DW_5_src_height_shift)
+
+/*define for src_depth field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_5_src_depth_offset 5
+#define SDMA_PKT_COPY_T2T_BC_DW_5_src_depth_mask   0x000007FF
+#define SDMA_PKT_COPY_T2T_BC_DW_5_src_depth_shift  16
+#define SDMA_PKT_COPY_T2T_BC_DW_5_SRC_DEPTH(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_5_src_depth_mask) << SDMA_PKT_COPY_T2T_BC_DW_5_src_depth_shift)
+
+/*define for DW_6 word*/
+/*define for src_element_size field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_element_size_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_element_size_mask   0x00000007
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_element_size_shift  0
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_element_size_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_element_size_shift)
+
+/*define for src_array_mode field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_array_mode_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_array_mode_mask   0x0000000F
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_array_mode_shift  3
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_ARRAY_MODE(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_array_mode_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_array_mode_shift)
+
+/*define for src_mit_mode field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_mit_mode_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_mit_mode_mask   0x00000007
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_mit_mode_shift  8
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_MIT_MODE(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_mit_mode_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_mit_mode_shift)
+
+/*define for src_tilesplit_size field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_tilesplit_size_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_tilesplit_size_mask   0x00000007
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_tilesplit_size_shift  11
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_TILESPLIT_SIZE(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_tilesplit_size_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_tilesplit_size_shift)
+
+/*define for src_bank_w field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_w_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_w_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_w_shift  15
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_BANK_W(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_w_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_w_shift)
+
+/*define for src_bank_h field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_h_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_h_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_h_shift  18
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_BANK_H(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_h_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_bank_h_shift)
+
+/*define for src_num_bank field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_num_bank_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_num_bank_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_num_bank_shift  21
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_NUM_BANK(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_num_bank_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_num_bank_shift)
+
+/*define for src_mat_aspt field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_mat_aspt_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_mat_aspt_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_mat_aspt_shift  24
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_MAT_ASPT(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_mat_aspt_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_mat_aspt_shift)
+
+/*define for src_pipe_config field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_pipe_config_offset 6
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_pipe_config_mask   0x0000001F
+#define SDMA_PKT_COPY_T2T_BC_DW_6_src_pipe_config_shift  26
+#define SDMA_PKT_COPY_T2T_BC_DW_6_SRC_PIPE_CONFIG(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_6_src_pipe_config_mask) << SDMA_PKT_COPY_T2T_BC_DW_6_src_pipe_config_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_COPY_T2T_BC_DST_ADDR_LO_dst_addr_31_0_offset 7
+#define SDMA_PKT_COPY_T2T_BC_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_BC_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_COPY_T2T_BC_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_T2T_BC_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_COPY_T2T_BC_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_COPY_T2T_BC_DST_ADDR_HI_dst_addr_63_32_offset 8
+#define SDMA_PKT_COPY_T2T_BC_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_T2T_BC_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_COPY_T2T_BC_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_T2T_BC_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_COPY_T2T_BC_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DW_9 word*/
+/*define for dst_x field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_9_dst_x_offset 9
+#define SDMA_PKT_COPY_T2T_BC_DW_9_dst_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_9_dst_x_shift  0
+#define SDMA_PKT_COPY_T2T_BC_DW_9_DST_X(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_9_dst_x_mask) << SDMA_PKT_COPY_T2T_BC_DW_9_dst_x_shift)
+
+/*define for dst_y field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_9_dst_y_offset 9
+#define SDMA_PKT_COPY_T2T_BC_DW_9_dst_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_9_dst_y_shift  16
+#define SDMA_PKT_COPY_T2T_BC_DW_9_DST_Y(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_9_dst_y_mask) << SDMA_PKT_COPY_T2T_BC_DW_9_dst_y_shift)
+
+/*define for DW_10 word*/
+/*define for dst_z field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_10_dst_z_offset 10
+#define SDMA_PKT_COPY_T2T_BC_DW_10_dst_z_mask   0x000007FF
+#define SDMA_PKT_COPY_T2T_BC_DW_10_dst_z_shift  0
+#define SDMA_PKT_COPY_T2T_BC_DW_10_DST_Z(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_10_dst_z_mask) << SDMA_PKT_COPY_T2T_BC_DW_10_dst_z_shift)
+
+/*define for dst_width field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_10_dst_width_offset 10
+#define SDMA_PKT_COPY_T2T_BC_DW_10_dst_width_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_10_dst_width_shift  16
+#define SDMA_PKT_COPY_T2T_BC_DW_10_DST_WIDTH(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_10_dst_width_mask) << SDMA_PKT_COPY_T2T_BC_DW_10_dst_width_shift)
+
+/*define for DW_11 word*/
+/*define for dst_height field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_11_dst_height_offset 11
+#define SDMA_PKT_COPY_T2T_BC_DW_11_dst_height_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_11_dst_height_shift  0
+#define SDMA_PKT_COPY_T2T_BC_DW_11_DST_HEIGHT(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_11_dst_height_mask) << SDMA_PKT_COPY_T2T_BC_DW_11_dst_height_shift)
+
+/*define for dst_depth field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_11_dst_depth_offset 11
+#define SDMA_PKT_COPY_T2T_BC_DW_11_dst_depth_mask   0x00000FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_11_dst_depth_shift  16
+#define SDMA_PKT_COPY_T2T_BC_DW_11_DST_DEPTH(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_11_dst_depth_mask) << SDMA_PKT_COPY_T2T_BC_DW_11_dst_depth_shift)
+
+/*define for DW_12 word*/
+/*define for dst_element_size field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_element_size_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_element_size_mask   0x00000007
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_element_size_shift  0
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_element_size_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_element_size_shift)
+
+/*define for dst_array_mode field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_array_mode_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_array_mode_mask   0x0000000F
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_array_mode_shift  3
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_ARRAY_MODE(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_array_mode_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_array_mode_shift)
+
+/*define for dst_mit_mode field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_mit_mode_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_mit_mode_mask   0x00000007
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_mit_mode_shift  8
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_MIT_MODE(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_mit_mode_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_mit_mode_shift)
+
+/*define for dst_tilesplit_size field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_tilesplit_size_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_tilesplit_size_mask   0x00000007
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_tilesplit_size_shift  11
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_TILESPLIT_SIZE(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_tilesplit_size_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_tilesplit_size_shift)
+
+/*define for dst_bank_w field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_w_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_w_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_w_shift  15
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_BANK_W(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_w_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_w_shift)
+
+/*define for dst_bank_h field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_h_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_h_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_h_shift  18
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_BANK_H(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_h_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_bank_h_shift)
+
+/*define for dst_num_bank field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_num_bank_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_num_bank_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_num_bank_shift  21
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_NUM_BANK(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_num_bank_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_num_bank_shift)
+
+/*define for dst_mat_aspt field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_mat_aspt_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_mat_aspt_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_mat_aspt_shift  24
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_MAT_ASPT(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_mat_aspt_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_mat_aspt_shift)
+
+/*define for dst_pipe_config field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_pipe_config_offset 12
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_pipe_config_mask   0x0000001F
+#define SDMA_PKT_COPY_T2T_BC_DW_12_dst_pipe_config_shift  26
+#define SDMA_PKT_COPY_T2T_BC_DW_12_DST_PIPE_CONFIG(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_12_dst_pipe_config_mask) << SDMA_PKT_COPY_T2T_BC_DW_12_dst_pipe_config_shift)
+
+/*define for DW_13 word*/
+/*define for rect_x field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_13_rect_x_offset 13
+#define SDMA_PKT_COPY_T2T_BC_DW_13_rect_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_13_rect_x_shift  0
+#define SDMA_PKT_COPY_T2T_BC_DW_13_RECT_X(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_13_rect_x_mask) << SDMA_PKT_COPY_T2T_BC_DW_13_rect_x_shift)
+
+/*define for rect_y field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_13_rect_y_offset 13
+#define SDMA_PKT_COPY_T2T_BC_DW_13_rect_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_T2T_BC_DW_13_rect_y_shift  16
+#define SDMA_PKT_COPY_T2T_BC_DW_13_RECT_Y(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_13_rect_y_mask) << SDMA_PKT_COPY_T2T_BC_DW_13_rect_y_shift)
+
+/*define for DW_14 word*/
+/*define for rect_z field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_14_rect_z_offset 14
+#define SDMA_PKT_COPY_T2T_BC_DW_14_rect_z_mask   0x000007FF
+#define SDMA_PKT_COPY_T2T_BC_DW_14_rect_z_shift  0
+#define SDMA_PKT_COPY_T2T_BC_DW_14_RECT_Z(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_14_rect_z_mask) << SDMA_PKT_COPY_T2T_BC_DW_14_rect_z_shift)
+
+/*define for dst_sw field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_14_dst_sw_offset 14
+#define SDMA_PKT_COPY_T2T_BC_DW_14_dst_sw_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_14_dst_sw_shift  16
+#define SDMA_PKT_COPY_T2T_BC_DW_14_DST_SW(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_14_dst_sw_mask) << SDMA_PKT_COPY_T2T_BC_DW_14_dst_sw_shift)
+
+/*define for src_sw field*/
+#define SDMA_PKT_COPY_T2T_BC_DW_14_src_sw_offset 14
+#define SDMA_PKT_COPY_T2T_BC_DW_14_src_sw_mask   0x00000003
+#define SDMA_PKT_COPY_T2T_BC_DW_14_src_sw_shift  24
+#define SDMA_PKT_COPY_T2T_BC_DW_14_SRC_SW(x) (((x) & SDMA_PKT_COPY_T2T_BC_DW_14_src_sw_mask) << SDMA_PKT_COPY_T2T_BC_DW_14_src_sw_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_TILED_SUBWIN packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_op_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_op_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_OP(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_HEADER_op_mask) << SDMA_PKT_COPY_TILED_SUBWIN_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_sub_op_shift  8
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_HEADER_sub_op_mask) << SDMA_PKT_COPY_TILED_SUBWIN_HEADER_sub_op_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_tmz_mask   0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_tmz_shift  18
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_HEADER_tmz_mask) << SDMA_PKT_COPY_TILED_SUBWIN_HEADER_tmz_shift)
+
+/*define for dcc field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_dcc_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_dcc_mask   0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_dcc_shift  19
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_DCC(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_HEADER_dcc_mask) << SDMA_PKT_COPY_TILED_SUBWIN_HEADER_dcc_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_cpv_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_cpv_shift  28
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_CPV(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_HEADER_cpv_mask) << SDMA_PKT_COPY_TILED_SUBWIN_HEADER_cpv_shift)
+
+/*define for detile field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_detile_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_detile_mask   0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_detile_shift  31
+#define SDMA_PKT_COPY_TILED_SUBWIN_HEADER_DETILE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_HEADER_detile_mask) << SDMA_PKT_COPY_TILED_SUBWIN_HEADER_detile_shift)
+
+/*define for TILED_ADDR_LO word*/
+/*define for tiled_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_LO_tiled_addr_31_0_offset 1
+#define SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_LO_tiled_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_LO_tiled_addr_31_0_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_LO_TILED_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_LO_tiled_addr_31_0_mask) << SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_LO_tiled_addr_31_0_shift)
+
+/*define for TILED_ADDR_HI word*/
+/*define for tiled_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_HI_tiled_addr_63_32_offset 2
+#define SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_HI_tiled_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_HI_tiled_addr_63_32_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_HI_TILED_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_HI_tiled_addr_63_32_mask) << SDMA_PKT_COPY_TILED_SUBWIN_TILED_ADDR_HI_tiled_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for tiled_x field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_x_offset 3
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_x_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_3_TILED_X(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_x_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_x_shift)
+
+/*define for tiled_y field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_y_offset 3
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_y_shift  16
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_3_TILED_Y(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_y_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_3_tiled_y_shift)
+
+/*define for DW_4 word*/
+/*define for tiled_z field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_4_tiled_z_offset 4
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_4_tiled_z_mask   0x00001FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_4_tiled_z_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_4_TILED_Z(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_4_tiled_z_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_4_tiled_z_shift)
+
+/*define for width field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_4_width_offset 4
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_4_width_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_4_width_shift  16
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_4_WIDTH(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_4_width_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_4_width_shift)
+
+/*define for DW_5 word*/
+/*define for height field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_5_height_offset 5
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_5_height_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_5_height_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_5_HEIGHT(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_5_height_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_5_height_shift)
+
+/*define for depth field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_5_depth_offset 5
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_5_depth_mask   0x00001FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_5_depth_shift  16
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_5_DEPTH(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_5_depth_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_5_depth_shift)
+
+/*define for DW_6 word*/
+/*define for element_size field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_element_size_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_element_size_mask   0x00000007
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_element_size_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_6_element_size_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_6_element_size_shift)
+
+/*define for swizzle_mode field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_swizzle_mode_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_swizzle_mode_mask   0x0000001F
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_swizzle_mode_shift  3
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_SWIZZLE_MODE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_6_swizzle_mode_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_6_swizzle_mode_shift)
+
+/*define for dimension field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_dimension_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_dimension_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_dimension_shift  9
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_DIMENSION(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_6_dimension_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_6_dimension_shift)
+
+/*define for mip_max field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_max_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_max_mask   0x0000000F
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_max_shift  16
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_MIP_MAX(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_max_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_max_shift)
+
+/*define for mip_id field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_id_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_id_mask   0x0000000F
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_id_shift  20
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_6_MIP_ID(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_id_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_6_mip_id_shift)
+
+/*define for LINEAR_ADDR_LO word*/
+/*define for linear_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_LO_linear_addr_31_0_offset 7
+#define SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_LO_linear_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_LO_linear_addr_31_0_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_LO_LINEAR_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_LO_linear_addr_31_0_mask) << SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_LO_linear_addr_31_0_shift)
+
+/*define for LINEAR_ADDR_HI word*/
+/*define for linear_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_HI_linear_addr_63_32_offset 8
+#define SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_HI_linear_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_HI_linear_addr_63_32_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_HI_LINEAR_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_HI_linear_addr_63_32_mask) << SDMA_PKT_COPY_TILED_SUBWIN_LINEAR_ADDR_HI_linear_addr_63_32_shift)
+
+/*define for DW_9 word*/
+/*define for linear_x field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_x_offset 9
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_x_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_9_LINEAR_X(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_x_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_x_shift)
+
+/*define for linear_y field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_y_offset 9
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_y_shift  16
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_9_LINEAR_Y(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_y_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_9_linear_y_shift)
+
+/*define for DW_10 word*/
+/*define for linear_z field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_z_offset 10
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_z_mask   0x00001FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_z_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_10_LINEAR_Z(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_z_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_z_shift)
+
+/*define for linear_pitch field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_pitch_offset 10
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_pitch_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_pitch_shift  16
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_10_LINEAR_PITCH(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_pitch_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_10_linear_pitch_shift)
+
+/*define for DW_11 word*/
+/*define for linear_slice_pitch field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_11_linear_slice_pitch_offset 11
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_11_linear_slice_pitch_mask   0x0FFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_11_linear_slice_pitch_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_11_LINEAR_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_11_linear_slice_pitch_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_11_linear_slice_pitch_shift)
+
+/*define for DW_12 word*/
+/*define for rect_x field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_x_offset 12
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_x_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_12_RECT_X(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_x_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_x_shift)
+
+/*define for rect_y field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_y_offset 12
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_y_shift  16
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_12_RECT_Y(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_y_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_12_rect_y_shift)
+
+/*define for DW_13 word*/
+/*define for rect_z field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_rect_z_offset 13
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_rect_z_mask   0x00001FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_rect_z_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_RECT_Z(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_13_rect_z_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_13_rect_z_shift)
+
+/*define for linear_sw field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_linear_sw_offset 13
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_linear_sw_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_linear_sw_shift  16
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_LINEAR_SW(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_13_linear_sw_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_13_linear_sw_shift)
+
+/*define for linear_cache_policy field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_linear_cache_policy_offset 13
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_linear_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_linear_cache_policy_shift  18
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_LINEAR_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_13_linear_cache_policy_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_13_linear_cache_policy_shift)
+
+/*define for tile_sw field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_tile_sw_offset 13
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_tile_sw_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_tile_sw_shift  24
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_TILE_SW(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_13_tile_sw_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_13_tile_sw_shift)
+
+/*define for tile_cache_policy field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_tile_cache_policy_offset 13
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_tile_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_tile_cache_policy_shift  26
+#define SDMA_PKT_COPY_TILED_SUBWIN_DW_13_TILE_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_DW_13_tile_cache_policy_mask) << SDMA_PKT_COPY_TILED_SUBWIN_DW_13_tile_cache_policy_shift)
+
+/*define for META_ADDR_LO word*/
+/*define for meta_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_LO_meta_addr_31_0_offset 14
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_LO_meta_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_LO_meta_addr_31_0_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_LO_META_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_LO_meta_addr_31_0_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_LO_meta_addr_31_0_shift)
+
+/*define for META_ADDR_HI word*/
+/*define for meta_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_HI_meta_addr_63_32_offset 15
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_HI_meta_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_HI_meta_addr_63_32_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_HI_META_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_HI_meta_addr_63_32_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_ADDR_HI_meta_addr_63_32_shift)
+
+/*define for META_CONFIG word*/
+/*define for data_format field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_data_format_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_data_format_mask   0x0000007F
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_data_format_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_DATA_FORMAT(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_data_format_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_data_format_shift)
+
+/*define for color_transform_disable field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_color_transform_disable_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_color_transform_disable_mask   0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_color_transform_disable_shift  7
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_COLOR_TRANSFORM_DISABLE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_color_transform_disable_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_color_transform_disable_shift)
+
+/*define for alpha_is_on_msb field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_alpha_is_on_msb_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_alpha_is_on_msb_mask   0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_alpha_is_on_msb_shift  8
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_ALPHA_IS_ON_MSB(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_alpha_is_on_msb_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_alpha_is_on_msb_shift)
+
+/*define for number_type field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_number_type_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_number_type_mask   0x00000007
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_number_type_shift  9
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_NUMBER_TYPE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_number_type_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_number_type_shift)
+
+/*define for surface_type field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_surface_type_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_surface_type_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_surface_type_shift  12
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_SURFACE_TYPE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_surface_type_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_surface_type_shift)
+
+/*define for meta_llc field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_meta_llc_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_meta_llc_mask   0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_meta_llc_shift  14
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_META_LLC(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_meta_llc_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_meta_llc_shift)
+
+/*define for max_comp_block_size field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_comp_block_size_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_comp_block_size_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_comp_block_size_shift  24
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_MAX_COMP_BLOCK_SIZE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_comp_block_size_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_comp_block_size_shift)
+
+/*define for max_uncomp_block_size field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_uncomp_block_size_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_uncomp_block_size_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_uncomp_block_size_shift  26
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_MAX_UNCOMP_BLOCK_SIZE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_uncomp_block_size_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_max_uncomp_block_size_shift)
+
+/*define for write_compress_enable field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_write_compress_enable_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_write_compress_enable_mask   0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_write_compress_enable_shift  28
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_WRITE_COMPRESS_ENABLE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_write_compress_enable_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_write_compress_enable_shift)
+
+/*define for meta_tmz field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_meta_tmz_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_meta_tmz_mask   0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_meta_tmz_shift  29
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_META_TMZ(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_meta_tmz_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_meta_tmz_shift)
+
+/*define for pipe_aligned field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_pipe_aligned_offset 16
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_pipe_aligned_mask   0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_pipe_aligned_shift  31
+#define SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_PIPE_ALIGNED(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_pipe_aligned_mask) << SDMA_PKT_COPY_TILED_SUBWIN_META_CONFIG_pipe_aligned_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_TILED_SUBWIN_BC packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_op_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_op_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_OP(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_op_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_sub_op_shift  8
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_sub_op_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_sub_op_shift)
+
+/*define for detile field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_detile_offset 0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_detile_mask   0x00000001
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_detile_shift  31
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_DETILE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_detile_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_HEADER_detile_shift)
+
+/*define for TILED_ADDR_LO word*/
+/*define for tiled_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_LO_tiled_addr_31_0_offset 1
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_LO_tiled_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_LO_tiled_addr_31_0_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_LO_TILED_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_LO_tiled_addr_31_0_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_LO_tiled_addr_31_0_shift)
+
+/*define for TILED_ADDR_HI word*/
+/*define for tiled_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_HI_tiled_addr_63_32_offset 2
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_HI_tiled_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_HI_tiled_addr_63_32_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_HI_TILED_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_HI_tiled_addr_63_32_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_TILED_ADDR_HI_tiled_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for tiled_x field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_x_offset 3
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_x_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_TILED_X(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_x_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_x_shift)
+
+/*define for tiled_y field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_y_offset 3
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_y_shift  16
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_TILED_Y(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_y_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_3_tiled_y_shift)
+
+/*define for DW_4 word*/
+/*define for tiled_z field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_tiled_z_offset 4
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_tiled_z_mask   0x000007FF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_tiled_z_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_TILED_Z(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_tiled_z_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_tiled_z_shift)
+
+/*define for width field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_width_offset 4
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_width_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_width_shift  16
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_WIDTH(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_width_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_4_width_shift)
+
+/*define for DW_5 word*/
+/*define for height field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_height_offset 5
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_height_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_height_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_HEIGHT(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_height_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_height_shift)
+
+/*define for depth field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_depth_offset 5
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_depth_mask   0x000007FF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_depth_shift  16
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_DEPTH(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_depth_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_5_depth_shift)
+
+/*define for DW_6 word*/
+/*define for element_size field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_element_size_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_element_size_mask   0x00000007
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_element_size_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_ELEMENT_SIZE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_element_size_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_element_size_shift)
+
+/*define for array_mode field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_array_mode_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_array_mode_mask   0x0000000F
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_array_mode_shift  3
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_ARRAY_MODE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_array_mode_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_array_mode_shift)
+
+/*define for mit_mode field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mit_mode_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mit_mode_mask   0x00000007
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mit_mode_shift  8
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_MIT_MODE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mit_mode_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mit_mode_shift)
+
+/*define for tilesplit_size field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_tilesplit_size_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_tilesplit_size_mask   0x00000007
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_tilesplit_size_shift  11
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_TILESPLIT_SIZE(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_tilesplit_size_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_tilesplit_size_shift)
+
+/*define for bank_w field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_w_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_w_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_w_shift  15
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_BANK_W(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_w_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_w_shift)
+
+/*define for bank_h field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_h_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_h_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_h_shift  18
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_BANK_H(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_h_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_bank_h_shift)
+
+/*define for num_bank field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_num_bank_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_num_bank_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_num_bank_shift  21
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_NUM_BANK(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_num_bank_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_num_bank_shift)
+
+/*define for mat_aspt field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mat_aspt_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mat_aspt_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mat_aspt_shift  24
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_MAT_ASPT(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mat_aspt_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_mat_aspt_shift)
+
+/*define for pipe_config field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_pipe_config_offset 6
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_pipe_config_mask   0x0000001F
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_pipe_config_shift  26
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_PIPE_CONFIG(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_pipe_config_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_6_pipe_config_shift)
+
+/*define for LINEAR_ADDR_LO word*/
+/*define for linear_addr_31_0 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_LO_linear_addr_31_0_offset 7
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_LO_linear_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_LO_linear_addr_31_0_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_LO_LINEAR_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_LO_linear_addr_31_0_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_LO_linear_addr_31_0_shift)
+
+/*define for LINEAR_ADDR_HI word*/
+/*define for linear_addr_63_32 field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_HI_linear_addr_63_32_offset 8
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_HI_linear_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_HI_linear_addr_63_32_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_HI_LINEAR_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_HI_linear_addr_63_32_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_LINEAR_ADDR_HI_linear_addr_63_32_shift)
+
+/*define for DW_9 word*/
+/*define for linear_x field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_x_offset 9
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_x_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_LINEAR_X(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_x_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_x_shift)
+
+/*define for linear_y field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_y_offset 9
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_y_shift  16
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_LINEAR_Y(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_y_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_9_linear_y_shift)
+
+/*define for DW_10 word*/
+/*define for linear_z field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_z_offset 10
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_z_mask   0x000007FF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_z_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_LINEAR_Z(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_z_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_z_shift)
+
+/*define for linear_pitch field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_pitch_offset 10
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_pitch_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_pitch_shift  16
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_LINEAR_PITCH(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_pitch_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_10_linear_pitch_shift)
+
+/*define for DW_11 word*/
+/*define for linear_slice_pitch field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_11_linear_slice_pitch_offset 11
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_11_linear_slice_pitch_mask   0x0FFFFFFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_11_linear_slice_pitch_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_11_LINEAR_SLICE_PITCH(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_11_linear_slice_pitch_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_11_linear_slice_pitch_shift)
+
+/*define for DW_12 word*/
+/*define for rect_x field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_x_offset 12
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_x_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_x_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_RECT_X(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_x_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_x_shift)
+
+/*define for rect_y field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_y_offset 12
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_y_mask   0x00003FFF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_y_shift  16
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_RECT_Y(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_y_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_12_rect_y_shift)
+
+/*define for DW_13 word*/
+/*define for rect_z field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_rect_z_offset 13
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_rect_z_mask   0x000007FF
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_rect_z_shift  0
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_RECT_Z(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_rect_z_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_rect_z_shift)
+
+/*define for linear_sw field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_linear_sw_offset 13
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_linear_sw_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_linear_sw_shift  16
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_LINEAR_SW(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_linear_sw_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_linear_sw_shift)
+
+/*define for tile_sw field*/
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_tile_sw_offset 13
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_tile_sw_mask   0x00000003
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_tile_sw_shift  24
+#define SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_TILE_SW(x) (((x) & SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_tile_sw_mask) << SDMA_PKT_COPY_TILED_SUBWIN_BC_DW_13_tile_sw_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COPY_STRUCT packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COPY_STRUCT_HEADER_op_offset 0
+#define SDMA_PKT_COPY_STRUCT_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COPY_STRUCT_HEADER_op_shift  0
+#define SDMA_PKT_COPY_STRUCT_HEADER_OP(x) (((x) & SDMA_PKT_COPY_STRUCT_HEADER_op_mask) << SDMA_PKT_COPY_STRUCT_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COPY_STRUCT_HEADER_sub_op_offset 0
+#define SDMA_PKT_COPY_STRUCT_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COPY_STRUCT_HEADER_sub_op_shift  8
+#define SDMA_PKT_COPY_STRUCT_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COPY_STRUCT_HEADER_sub_op_mask) << SDMA_PKT_COPY_STRUCT_HEADER_sub_op_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_COPY_STRUCT_HEADER_tmz_offset 0
+#define SDMA_PKT_COPY_STRUCT_HEADER_tmz_mask   0x00000001
+#define SDMA_PKT_COPY_STRUCT_HEADER_tmz_shift  18
+#define SDMA_PKT_COPY_STRUCT_HEADER_TMZ(x) (((x) & SDMA_PKT_COPY_STRUCT_HEADER_tmz_mask) << SDMA_PKT_COPY_STRUCT_HEADER_tmz_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_COPY_STRUCT_HEADER_cpv_offset 0
+#define SDMA_PKT_COPY_STRUCT_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_COPY_STRUCT_HEADER_cpv_shift  28
+#define SDMA_PKT_COPY_STRUCT_HEADER_CPV(x) (((x) & SDMA_PKT_COPY_STRUCT_HEADER_cpv_mask) << SDMA_PKT_COPY_STRUCT_HEADER_cpv_shift)
+
+/*define for detile field*/
+#define SDMA_PKT_COPY_STRUCT_HEADER_detile_offset 0
+#define SDMA_PKT_COPY_STRUCT_HEADER_detile_mask   0x00000001
+#define SDMA_PKT_COPY_STRUCT_HEADER_detile_shift  31
+#define SDMA_PKT_COPY_STRUCT_HEADER_DETILE(x) (((x) & SDMA_PKT_COPY_STRUCT_HEADER_detile_mask) << SDMA_PKT_COPY_STRUCT_HEADER_detile_shift)
+
+/*define for SB_ADDR_LO word*/
+/*define for sb_addr_31_0 field*/
+#define SDMA_PKT_COPY_STRUCT_SB_ADDR_LO_sb_addr_31_0_offset 1
+#define SDMA_PKT_COPY_STRUCT_SB_ADDR_LO_sb_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_STRUCT_SB_ADDR_LO_sb_addr_31_0_shift  0
+#define SDMA_PKT_COPY_STRUCT_SB_ADDR_LO_SB_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_STRUCT_SB_ADDR_LO_sb_addr_31_0_mask) << SDMA_PKT_COPY_STRUCT_SB_ADDR_LO_sb_addr_31_0_shift)
+
+/*define for SB_ADDR_HI word*/
+/*define for sb_addr_63_32 field*/
+#define SDMA_PKT_COPY_STRUCT_SB_ADDR_HI_sb_addr_63_32_offset 2
+#define SDMA_PKT_COPY_STRUCT_SB_ADDR_HI_sb_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_STRUCT_SB_ADDR_HI_sb_addr_63_32_shift  0
+#define SDMA_PKT_COPY_STRUCT_SB_ADDR_HI_SB_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_STRUCT_SB_ADDR_HI_sb_addr_63_32_mask) << SDMA_PKT_COPY_STRUCT_SB_ADDR_HI_sb_addr_63_32_shift)
+
+/*define for START_INDEX word*/
+/*define for start_index field*/
+#define SDMA_PKT_COPY_STRUCT_START_INDEX_start_index_offset 3
+#define SDMA_PKT_COPY_STRUCT_START_INDEX_start_index_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_STRUCT_START_INDEX_start_index_shift  0
+#define SDMA_PKT_COPY_STRUCT_START_INDEX_START_INDEX(x) (((x) & SDMA_PKT_COPY_STRUCT_START_INDEX_start_index_mask) << SDMA_PKT_COPY_STRUCT_START_INDEX_start_index_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_COPY_STRUCT_COUNT_count_offset 4
+#define SDMA_PKT_COPY_STRUCT_COUNT_count_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_STRUCT_COUNT_count_shift  0
+#define SDMA_PKT_COPY_STRUCT_COUNT_COUNT(x) (((x) & SDMA_PKT_COPY_STRUCT_COUNT_count_mask) << SDMA_PKT_COPY_STRUCT_COUNT_count_shift)
+
+/*define for DW_5 word*/
+/*define for stride field*/
+#define SDMA_PKT_COPY_STRUCT_DW_5_stride_offset 5
+#define SDMA_PKT_COPY_STRUCT_DW_5_stride_mask   0x000007FF
+#define SDMA_PKT_COPY_STRUCT_DW_5_stride_shift  0
+#define SDMA_PKT_COPY_STRUCT_DW_5_STRIDE(x) (((x) & SDMA_PKT_COPY_STRUCT_DW_5_stride_mask) << SDMA_PKT_COPY_STRUCT_DW_5_stride_shift)
+
+/*define for linear_sw field*/
+#define SDMA_PKT_COPY_STRUCT_DW_5_linear_sw_offset 5
+#define SDMA_PKT_COPY_STRUCT_DW_5_linear_sw_mask   0x00000003
+#define SDMA_PKT_COPY_STRUCT_DW_5_linear_sw_shift  16
+#define SDMA_PKT_COPY_STRUCT_DW_5_LINEAR_SW(x) (((x) & SDMA_PKT_COPY_STRUCT_DW_5_linear_sw_mask) << SDMA_PKT_COPY_STRUCT_DW_5_linear_sw_shift)
+
+/*define for linear_cache_policy field*/
+#define SDMA_PKT_COPY_STRUCT_DW_5_linear_cache_policy_offset 5
+#define SDMA_PKT_COPY_STRUCT_DW_5_linear_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_STRUCT_DW_5_linear_cache_policy_shift  18
+#define SDMA_PKT_COPY_STRUCT_DW_5_LINEAR_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_STRUCT_DW_5_linear_cache_policy_mask) << SDMA_PKT_COPY_STRUCT_DW_5_linear_cache_policy_shift)
+
+/*define for struct_sw field*/
+#define SDMA_PKT_COPY_STRUCT_DW_5_struct_sw_offset 5
+#define SDMA_PKT_COPY_STRUCT_DW_5_struct_sw_mask   0x00000003
+#define SDMA_PKT_COPY_STRUCT_DW_5_struct_sw_shift  24
+#define SDMA_PKT_COPY_STRUCT_DW_5_STRUCT_SW(x) (((x) & SDMA_PKT_COPY_STRUCT_DW_5_struct_sw_mask) << SDMA_PKT_COPY_STRUCT_DW_5_struct_sw_shift)
+
+/*define for struct_cache_policy field*/
+#define SDMA_PKT_COPY_STRUCT_DW_5_struct_cache_policy_offset 5
+#define SDMA_PKT_COPY_STRUCT_DW_5_struct_cache_policy_mask   0x00000007
+#define SDMA_PKT_COPY_STRUCT_DW_5_struct_cache_policy_shift  26
+#define SDMA_PKT_COPY_STRUCT_DW_5_STRUCT_CACHE_POLICY(x) (((x) & SDMA_PKT_COPY_STRUCT_DW_5_struct_cache_policy_mask) << SDMA_PKT_COPY_STRUCT_DW_5_struct_cache_policy_shift)
+
+/*define for LINEAR_ADDR_LO word*/
+/*define for linear_addr_31_0 field*/
+#define SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_LO_linear_addr_31_0_offset 6
+#define SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_LO_linear_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_LO_linear_addr_31_0_shift  0
+#define SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_LO_LINEAR_ADDR_31_0(x) (((x) & SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_LO_linear_addr_31_0_mask) << SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_LO_linear_addr_31_0_shift)
+
+/*define for LINEAR_ADDR_HI word*/
+/*define for linear_addr_63_32 field*/
+#define SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_HI_linear_addr_63_32_offset 7
+#define SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_HI_linear_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_HI_linear_addr_63_32_shift  0
+#define SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_HI_LINEAR_ADDR_63_32(x) (((x) & SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_HI_linear_addr_63_32_mask) << SDMA_PKT_COPY_STRUCT_LINEAR_ADDR_HI_linear_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_WRITE_UNTILED packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_WRITE_UNTILED_HEADER_op_offset 0
+#define SDMA_PKT_WRITE_UNTILED_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_WRITE_UNTILED_HEADER_op_shift  0
+#define SDMA_PKT_WRITE_UNTILED_HEADER_OP(x) (((x) & SDMA_PKT_WRITE_UNTILED_HEADER_op_mask) << SDMA_PKT_WRITE_UNTILED_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_WRITE_UNTILED_HEADER_sub_op_offset 0
+#define SDMA_PKT_WRITE_UNTILED_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_WRITE_UNTILED_HEADER_sub_op_shift  8
+#define SDMA_PKT_WRITE_UNTILED_HEADER_SUB_OP(x) (((x) & SDMA_PKT_WRITE_UNTILED_HEADER_sub_op_mask) << SDMA_PKT_WRITE_UNTILED_HEADER_sub_op_shift)
+
+/*define for encrypt field*/
+#define SDMA_PKT_WRITE_UNTILED_HEADER_encrypt_offset 0
+#define SDMA_PKT_WRITE_UNTILED_HEADER_encrypt_mask   0x00000001
+#define SDMA_PKT_WRITE_UNTILED_HEADER_encrypt_shift  16
+#define SDMA_PKT_WRITE_UNTILED_HEADER_ENCRYPT(x) (((x) & SDMA_PKT_WRITE_UNTILED_HEADER_encrypt_mask) << SDMA_PKT_WRITE_UNTILED_HEADER_encrypt_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_WRITE_UNTILED_HEADER_tmz_offset 0
+#define SDMA_PKT_WRITE_UNTILED_HEADER_tmz_mask   0x00000001
+#define SDMA_PKT_WRITE_UNTILED_HEADER_tmz_shift  18
+#define SDMA_PKT_WRITE_UNTILED_HEADER_TMZ(x) (((x) & SDMA_PKT_WRITE_UNTILED_HEADER_tmz_mask) << SDMA_PKT_WRITE_UNTILED_HEADER_tmz_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_WRITE_UNTILED_HEADER_cpv_offset 0
+#define SDMA_PKT_WRITE_UNTILED_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_WRITE_UNTILED_HEADER_cpv_shift  28
+#define SDMA_PKT_WRITE_UNTILED_HEADER_CPV(x) (((x) & SDMA_PKT_WRITE_UNTILED_HEADER_cpv_mask) << SDMA_PKT_WRITE_UNTILED_HEADER_cpv_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_WRITE_UNTILED_DST_ADDR_LO_dst_addr_31_0_offset 1
+#define SDMA_PKT_WRITE_UNTILED_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_UNTILED_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_WRITE_UNTILED_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_WRITE_UNTILED_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_WRITE_UNTILED_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_WRITE_UNTILED_DST_ADDR_HI_dst_addr_63_32_offset 2
+#define SDMA_PKT_WRITE_UNTILED_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_UNTILED_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_WRITE_UNTILED_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_WRITE_UNTILED_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_WRITE_UNTILED_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for count field*/
+#define SDMA_PKT_WRITE_UNTILED_DW_3_count_offset 3
+#define SDMA_PKT_WRITE_UNTILED_DW_3_count_mask   0x000FFFFF
+#define SDMA_PKT_WRITE_UNTILED_DW_3_count_shift  0
+#define SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(x) (((x) & SDMA_PKT_WRITE_UNTILED_DW_3_count_mask) << SDMA_PKT_WRITE_UNTILED_DW_3_count_shift)
+
+/*define for sw field*/
+#define SDMA_PKT_WRITE_UNTILED_DW_3_sw_offset 3
+#define SDMA_PKT_WRITE_UNTILED_DW_3_sw_mask   0x00000003
+#define SDMA_PKT_WRITE_UNTILED_DW_3_sw_shift  24
+#define SDMA_PKT_WRITE_UNTILED_DW_3_SW(x) (((x) & SDMA_PKT_WRITE_UNTILED_DW_3_sw_mask) << SDMA_PKT_WRITE_UNTILED_DW_3_sw_shift)
+
+/*define for cache_policy field*/
+#define SDMA_PKT_WRITE_UNTILED_DW_3_cache_policy_offset 3
+#define SDMA_PKT_WRITE_UNTILED_DW_3_cache_policy_mask   0x00000007
+#define SDMA_PKT_WRITE_UNTILED_DW_3_cache_policy_shift  26
+#define SDMA_PKT_WRITE_UNTILED_DW_3_CACHE_POLICY(x) (((x) & SDMA_PKT_WRITE_UNTILED_DW_3_cache_policy_mask) << SDMA_PKT_WRITE_UNTILED_DW_3_cache_policy_shift)
+
+/*define for DATA0 word*/
+/*define for data0 field*/
+#define SDMA_PKT_WRITE_UNTILED_DATA0_data0_offset 4
+#define SDMA_PKT_WRITE_UNTILED_DATA0_data0_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_UNTILED_DATA0_data0_shift  0
+#define SDMA_PKT_WRITE_UNTILED_DATA0_DATA0(x) (((x) & SDMA_PKT_WRITE_UNTILED_DATA0_data0_mask) << SDMA_PKT_WRITE_UNTILED_DATA0_data0_shift)
+
+
+/*
+** Definitions for SDMA_PKT_WRITE_TILED packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_WRITE_TILED_HEADER_op_offset 0
+#define SDMA_PKT_WRITE_TILED_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_WRITE_TILED_HEADER_op_shift  0
+#define SDMA_PKT_WRITE_TILED_HEADER_OP(x) (((x) & SDMA_PKT_WRITE_TILED_HEADER_op_mask) << SDMA_PKT_WRITE_TILED_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_WRITE_TILED_HEADER_sub_op_offset 0
+#define SDMA_PKT_WRITE_TILED_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_WRITE_TILED_HEADER_sub_op_shift  8
+#define SDMA_PKT_WRITE_TILED_HEADER_SUB_OP(x) (((x) & SDMA_PKT_WRITE_TILED_HEADER_sub_op_mask) << SDMA_PKT_WRITE_TILED_HEADER_sub_op_shift)
+
+/*define for encrypt field*/
+#define SDMA_PKT_WRITE_TILED_HEADER_encrypt_offset 0
+#define SDMA_PKT_WRITE_TILED_HEADER_encrypt_mask   0x00000001
+#define SDMA_PKT_WRITE_TILED_HEADER_encrypt_shift  16
+#define SDMA_PKT_WRITE_TILED_HEADER_ENCRYPT(x) (((x) & SDMA_PKT_WRITE_TILED_HEADER_encrypt_mask) << SDMA_PKT_WRITE_TILED_HEADER_encrypt_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_WRITE_TILED_HEADER_tmz_offset 0
+#define SDMA_PKT_WRITE_TILED_HEADER_tmz_mask   0x00000001
+#define SDMA_PKT_WRITE_TILED_HEADER_tmz_shift  18
+#define SDMA_PKT_WRITE_TILED_HEADER_TMZ(x) (((x) & SDMA_PKT_WRITE_TILED_HEADER_tmz_mask) << SDMA_PKT_WRITE_TILED_HEADER_tmz_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_WRITE_TILED_HEADER_cpv_offset 0
+#define SDMA_PKT_WRITE_TILED_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_WRITE_TILED_HEADER_cpv_shift  28
+#define SDMA_PKT_WRITE_TILED_HEADER_CPV(x) (((x) & SDMA_PKT_WRITE_TILED_HEADER_cpv_mask) << SDMA_PKT_WRITE_TILED_HEADER_cpv_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_WRITE_TILED_DST_ADDR_LO_dst_addr_31_0_offset 1
+#define SDMA_PKT_WRITE_TILED_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_TILED_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_WRITE_TILED_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_WRITE_TILED_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_WRITE_TILED_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_WRITE_TILED_DST_ADDR_HI_dst_addr_63_32_offset 2
+#define SDMA_PKT_WRITE_TILED_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_TILED_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_WRITE_TILED_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_WRITE_TILED_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_WRITE_TILED_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for width field*/
+#define SDMA_PKT_WRITE_TILED_DW_3_width_offset 3
+#define SDMA_PKT_WRITE_TILED_DW_3_width_mask   0x00003FFF
+#define SDMA_PKT_WRITE_TILED_DW_3_width_shift  0
+#define SDMA_PKT_WRITE_TILED_DW_3_WIDTH(x) (((x) & SDMA_PKT_WRITE_TILED_DW_3_width_mask) << SDMA_PKT_WRITE_TILED_DW_3_width_shift)
+
+/*define for DW_4 word*/
+/*define for height field*/
+#define SDMA_PKT_WRITE_TILED_DW_4_height_offset 4
+#define SDMA_PKT_WRITE_TILED_DW_4_height_mask   0x00003FFF
+#define SDMA_PKT_WRITE_TILED_DW_4_height_shift  0
+#define SDMA_PKT_WRITE_TILED_DW_4_HEIGHT(x) (((x) & SDMA_PKT_WRITE_TILED_DW_4_height_mask) << SDMA_PKT_WRITE_TILED_DW_4_height_shift)
+
+/*define for depth field*/
+#define SDMA_PKT_WRITE_TILED_DW_4_depth_offset 4
+#define SDMA_PKT_WRITE_TILED_DW_4_depth_mask   0x00001FFF
+#define SDMA_PKT_WRITE_TILED_DW_4_depth_shift  16
+#define SDMA_PKT_WRITE_TILED_DW_4_DEPTH(x) (((x) & SDMA_PKT_WRITE_TILED_DW_4_depth_mask) << SDMA_PKT_WRITE_TILED_DW_4_depth_shift)
+
+/*define for DW_5 word*/
+/*define for element_size field*/
+#define SDMA_PKT_WRITE_TILED_DW_5_element_size_offset 5
+#define SDMA_PKT_WRITE_TILED_DW_5_element_size_mask   0x00000007
+#define SDMA_PKT_WRITE_TILED_DW_5_element_size_shift  0
+#define SDMA_PKT_WRITE_TILED_DW_5_ELEMENT_SIZE(x) (((x) & SDMA_PKT_WRITE_TILED_DW_5_element_size_mask) << SDMA_PKT_WRITE_TILED_DW_5_element_size_shift)
+
+/*define for swizzle_mode field*/
+#define SDMA_PKT_WRITE_TILED_DW_5_swizzle_mode_offset 5
+#define SDMA_PKT_WRITE_TILED_DW_5_swizzle_mode_mask   0x0000001F
+#define SDMA_PKT_WRITE_TILED_DW_5_swizzle_mode_shift  3
+#define SDMA_PKT_WRITE_TILED_DW_5_SWIZZLE_MODE(x) (((x) & SDMA_PKT_WRITE_TILED_DW_5_swizzle_mode_mask) << SDMA_PKT_WRITE_TILED_DW_5_swizzle_mode_shift)
+
+/*define for dimension field*/
+#define SDMA_PKT_WRITE_TILED_DW_5_dimension_offset 5
+#define SDMA_PKT_WRITE_TILED_DW_5_dimension_mask   0x00000003
+#define SDMA_PKT_WRITE_TILED_DW_5_dimension_shift  9
+#define SDMA_PKT_WRITE_TILED_DW_5_DIMENSION(x) (((x) & SDMA_PKT_WRITE_TILED_DW_5_dimension_mask) << SDMA_PKT_WRITE_TILED_DW_5_dimension_shift)
+
+/*define for mip_max field*/
+#define SDMA_PKT_WRITE_TILED_DW_5_mip_max_offset 5
+#define SDMA_PKT_WRITE_TILED_DW_5_mip_max_mask   0x0000000F
+#define SDMA_PKT_WRITE_TILED_DW_5_mip_max_shift  16
+#define SDMA_PKT_WRITE_TILED_DW_5_MIP_MAX(x) (((x) & SDMA_PKT_WRITE_TILED_DW_5_mip_max_mask) << SDMA_PKT_WRITE_TILED_DW_5_mip_max_shift)
+
+/*define for DW_6 word*/
+/*define for x field*/
+#define SDMA_PKT_WRITE_TILED_DW_6_x_offset 6
+#define SDMA_PKT_WRITE_TILED_DW_6_x_mask   0x00003FFF
+#define SDMA_PKT_WRITE_TILED_DW_6_x_shift  0
+#define SDMA_PKT_WRITE_TILED_DW_6_X(x) (((x) & SDMA_PKT_WRITE_TILED_DW_6_x_mask) << SDMA_PKT_WRITE_TILED_DW_6_x_shift)
+
+/*define for y field*/
+#define SDMA_PKT_WRITE_TILED_DW_6_y_offset 6
+#define SDMA_PKT_WRITE_TILED_DW_6_y_mask   0x00003FFF
+#define SDMA_PKT_WRITE_TILED_DW_6_y_shift  16
+#define SDMA_PKT_WRITE_TILED_DW_6_Y(x) (((x) & SDMA_PKT_WRITE_TILED_DW_6_y_mask) << SDMA_PKT_WRITE_TILED_DW_6_y_shift)
+
+/*define for DW_7 word*/
+/*define for z field*/
+#define SDMA_PKT_WRITE_TILED_DW_7_z_offset 7
+#define SDMA_PKT_WRITE_TILED_DW_7_z_mask   0x00001FFF
+#define SDMA_PKT_WRITE_TILED_DW_7_z_shift  0
+#define SDMA_PKT_WRITE_TILED_DW_7_Z(x) (((x) & SDMA_PKT_WRITE_TILED_DW_7_z_mask) << SDMA_PKT_WRITE_TILED_DW_7_z_shift)
+
+/*define for sw field*/
+#define SDMA_PKT_WRITE_TILED_DW_7_sw_offset 7
+#define SDMA_PKT_WRITE_TILED_DW_7_sw_mask   0x00000003
+#define SDMA_PKT_WRITE_TILED_DW_7_sw_shift  24
+#define SDMA_PKT_WRITE_TILED_DW_7_SW(x) (((x) & SDMA_PKT_WRITE_TILED_DW_7_sw_mask) << SDMA_PKT_WRITE_TILED_DW_7_sw_shift)
+
+/*define for cache_policy field*/
+#define SDMA_PKT_WRITE_TILED_DW_7_cache_policy_offset 7
+#define SDMA_PKT_WRITE_TILED_DW_7_cache_policy_mask   0x00000007
+#define SDMA_PKT_WRITE_TILED_DW_7_cache_policy_shift  26
+#define SDMA_PKT_WRITE_TILED_DW_7_CACHE_POLICY(x) (((x) & SDMA_PKT_WRITE_TILED_DW_7_cache_policy_mask) << SDMA_PKT_WRITE_TILED_DW_7_cache_policy_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_WRITE_TILED_COUNT_count_offset 8
+#define SDMA_PKT_WRITE_TILED_COUNT_count_mask   0x000FFFFF
+#define SDMA_PKT_WRITE_TILED_COUNT_count_shift  0
+#define SDMA_PKT_WRITE_TILED_COUNT_COUNT(x) (((x) & SDMA_PKT_WRITE_TILED_COUNT_count_mask) << SDMA_PKT_WRITE_TILED_COUNT_count_shift)
+
+/*define for DATA0 word*/
+/*define for data0 field*/
+#define SDMA_PKT_WRITE_TILED_DATA0_data0_offset 9
+#define SDMA_PKT_WRITE_TILED_DATA0_data0_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_TILED_DATA0_data0_shift  0
+#define SDMA_PKT_WRITE_TILED_DATA0_DATA0(x) (((x) & SDMA_PKT_WRITE_TILED_DATA0_data0_mask) << SDMA_PKT_WRITE_TILED_DATA0_data0_shift)
+
+
+/*
+** Definitions for SDMA_PKT_WRITE_TILED_BC packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_WRITE_TILED_BC_HEADER_op_offset 0
+#define SDMA_PKT_WRITE_TILED_BC_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_WRITE_TILED_BC_HEADER_op_shift  0
+#define SDMA_PKT_WRITE_TILED_BC_HEADER_OP(x) (((x) & SDMA_PKT_WRITE_TILED_BC_HEADER_op_mask) << SDMA_PKT_WRITE_TILED_BC_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_WRITE_TILED_BC_HEADER_sub_op_offset 0
+#define SDMA_PKT_WRITE_TILED_BC_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_WRITE_TILED_BC_HEADER_sub_op_shift  8
+#define SDMA_PKT_WRITE_TILED_BC_HEADER_SUB_OP(x) (((x) & SDMA_PKT_WRITE_TILED_BC_HEADER_sub_op_mask) << SDMA_PKT_WRITE_TILED_BC_HEADER_sub_op_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_WRITE_TILED_BC_DST_ADDR_LO_dst_addr_31_0_offset 1
+#define SDMA_PKT_WRITE_TILED_BC_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_TILED_BC_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_WRITE_TILED_BC_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_WRITE_TILED_BC_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_WRITE_TILED_BC_DST_ADDR_HI_dst_addr_63_32_offset 2
+#define SDMA_PKT_WRITE_TILED_BC_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_TILED_BC_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_WRITE_TILED_BC_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_WRITE_TILED_BC_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DW_3 word*/
+/*define for width field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_3_width_offset 3
+#define SDMA_PKT_WRITE_TILED_BC_DW_3_width_mask   0x00003FFF
+#define SDMA_PKT_WRITE_TILED_BC_DW_3_width_shift  0
+#define SDMA_PKT_WRITE_TILED_BC_DW_3_WIDTH(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_3_width_mask) << SDMA_PKT_WRITE_TILED_BC_DW_3_width_shift)
+
+/*define for DW_4 word*/
+/*define for height field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_4_height_offset 4
+#define SDMA_PKT_WRITE_TILED_BC_DW_4_height_mask   0x00003FFF
+#define SDMA_PKT_WRITE_TILED_BC_DW_4_height_shift  0
+#define SDMA_PKT_WRITE_TILED_BC_DW_4_HEIGHT(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_4_height_mask) << SDMA_PKT_WRITE_TILED_BC_DW_4_height_shift)
+
+/*define for depth field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_4_depth_offset 4
+#define SDMA_PKT_WRITE_TILED_BC_DW_4_depth_mask   0x000007FF
+#define SDMA_PKT_WRITE_TILED_BC_DW_4_depth_shift  16
+#define SDMA_PKT_WRITE_TILED_BC_DW_4_DEPTH(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_4_depth_mask) << SDMA_PKT_WRITE_TILED_BC_DW_4_depth_shift)
+
+/*define for DW_5 word*/
+/*define for element_size field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_element_size_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_element_size_mask   0x00000007
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_element_size_shift  0
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_ELEMENT_SIZE(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_element_size_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_element_size_shift)
+
+/*define for array_mode field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_array_mode_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_array_mode_mask   0x0000000F
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_array_mode_shift  3
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_ARRAY_MODE(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_array_mode_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_array_mode_shift)
+
+/*define for mit_mode field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_mit_mode_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_mit_mode_mask   0x00000007
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_mit_mode_shift  8
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_MIT_MODE(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_mit_mode_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_mit_mode_shift)
+
+/*define for tilesplit_size field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_tilesplit_size_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_tilesplit_size_mask   0x00000007
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_tilesplit_size_shift  11
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_TILESPLIT_SIZE(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_tilesplit_size_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_tilesplit_size_shift)
+
+/*define for bank_w field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_bank_w_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_bank_w_mask   0x00000003
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_bank_w_shift  15
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_BANK_W(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_bank_w_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_bank_w_shift)
+
+/*define for bank_h field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_bank_h_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_bank_h_mask   0x00000003
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_bank_h_shift  18
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_BANK_H(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_bank_h_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_bank_h_shift)
+
+/*define for num_bank field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_num_bank_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_num_bank_mask   0x00000003
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_num_bank_shift  21
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_NUM_BANK(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_num_bank_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_num_bank_shift)
+
+/*define for mat_aspt field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_mat_aspt_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_mat_aspt_mask   0x00000003
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_mat_aspt_shift  24
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_MAT_ASPT(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_mat_aspt_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_mat_aspt_shift)
+
+/*define for pipe_config field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_pipe_config_offset 5
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_pipe_config_mask   0x0000001F
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_pipe_config_shift  26
+#define SDMA_PKT_WRITE_TILED_BC_DW_5_PIPE_CONFIG(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_5_pipe_config_mask) << SDMA_PKT_WRITE_TILED_BC_DW_5_pipe_config_shift)
+
+/*define for DW_6 word*/
+/*define for x field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_6_x_offset 6
+#define SDMA_PKT_WRITE_TILED_BC_DW_6_x_mask   0x00003FFF
+#define SDMA_PKT_WRITE_TILED_BC_DW_6_x_shift  0
+#define SDMA_PKT_WRITE_TILED_BC_DW_6_X(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_6_x_mask) << SDMA_PKT_WRITE_TILED_BC_DW_6_x_shift)
+
+/*define for y field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_6_y_offset 6
+#define SDMA_PKT_WRITE_TILED_BC_DW_6_y_mask   0x00003FFF
+#define SDMA_PKT_WRITE_TILED_BC_DW_6_y_shift  16
+#define SDMA_PKT_WRITE_TILED_BC_DW_6_Y(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_6_y_mask) << SDMA_PKT_WRITE_TILED_BC_DW_6_y_shift)
+
+/*define for DW_7 word*/
+/*define for z field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_7_z_offset 7
+#define SDMA_PKT_WRITE_TILED_BC_DW_7_z_mask   0x000007FF
+#define SDMA_PKT_WRITE_TILED_BC_DW_7_z_shift  0
+#define SDMA_PKT_WRITE_TILED_BC_DW_7_Z(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_7_z_mask) << SDMA_PKT_WRITE_TILED_BC_DW_7_z_shift)
+
+/*define for sw field*/
+#define SDMA_PKT_WRITE_TILED_BC_DW_7_sw_offset 7
+#define SDMA_PKT_WRITE_TILED_BC_DW_7_sw_mask   0x00000003
+#define SDMA_PKT_WRITE_TILED_BC_DW_7_sw_shift  24
+#define SDMA_PKT_WRITE_TILED_BC_DW_7_SW(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DW_7_sw_mask) << SDMA_PKT_WRITE_TILED_BC_DW_7_sw_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_WRITE_TILED_BC_COUNT_count_offset 8
+#define SDMA_PKT_WRITE_TILED_BC_COUNT_count_mask   0x000FFFFF
+#define SDMA_PKT_WRITE_TILED_BC_COUNT_count_shift  2
+#define SDMA_PKT_WRITE_TILED_BC_COUNT_COUNT(x) (((x) & SDMA_PKT_WRITE_TILED_BC_COUNT_count_mask) << SDMA_PKT_WRITE_TILED_BC_COUNT_count_shift)
+
+/*define for DATA0 word*/
+/*define for data0 field*/
+#define SDMA_PKT_WRITE_TILED_BC_DATA0_data0_offset 9
+#define SDMA_PKT_WRITE_TILED_BC_DATA0_data0_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_TILED_BC_DATA0_data0_shift  0
+#define SDMA_PKT_WRITE_TILED_BC_DATA0_DATA0(x) (((x) & SDMA_PKT_WRITE_TILED_BC_DATA0_data0_mask) << SDMA_PKT_WRITE_TILED_BC_DATA0_data0_shift)
+
+
+/*
+** Definitions for SDMA_PKT_PTEPDE_COPY packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_PTEPDE_COPY_HEADER_op_offset 0
+#define SDMA_PKT_PTEPDE_COPY_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_PTEPDE_COPY_HEADER_op_shift  0
+#define SDMA_PKT_PTEPDE_COPY_HEADER_OP(x) (((x) & SDMA_PKT_PTEPDE_COPY_HEADER_op_mask) << SDMA_PKT_PTEPDE_COPY_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_PTEPDE_COPY_HEADER_sub_op_offset 0
+#define SDMA_PKT_PTEPDE_COPY_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_PTEPDE_COPY_HEADER_sub_op_shift  8
+#define SDMA_PKT_PTEPDE_COPY_HEADER_SUB_OP(x) (((x) & SDMA_PKT_PTEPDE_COPY_HEADER_sub_op_mask) << SDMA_PKT_PTEPDE_COPY_HEADER_sub_op_shift)
+
+/*define for mtype field*/
+#define SDMA_PKT_PTEPDE_COPY_HEADER_mtype_offset 0
+#define SDMA_PKT_PTEPDE_COPY_HEADER_mtype_mask   0x00000003
+#define SDMA_PKT_PTEPDE_COPY_HEADER_mtype_shift  16
+#define SDMA_PKT_PTEPDE_COPY_HEADER_MTYPE(x) (((x) & SDMA_PKT_PTEPDE_COPY_HEADER_mtype_mask) << SDMA_PKT_PTEPDE_COPY_HEADER_mtype_shift)
+
+/*define for snoop field*/
+#define SDMA_PKT_PTEPDE_COPY_HEADER_snoop_offset 0
+#define SDMA_PKT_PTEPDE_COPY_HEADER_snoop_mask   0x00000001
+#define SDMA_PKT_PTEPDE_COPY_HEADER_snoop_shift  22
+#define SDMA_PKT_PTEPDE_COPY_HEADER_SNOOP(x) (((x) & SDMA_PKT_PTEPDE_COPY_HEADER_snoop_mask) << SDMA_PKT_PTEPDE_COPY_HEADER_snoop_shift)
+
+/*define for scope field*/
+#define SDMA_PKT_PTEPDE_COPY_HEADER_scope_offset 0
+#define SDMA_PKT_PTEPDE_COPY_HEADER_scope_mask   0x00000003
+#define SDMA_PKT_PTEPDE_COPY_HEADER_scope_shift  24
+#define SDMA_PKT_PTEPDE_COPY_HEADER_SCOPE(x) (((x) & SDMA_PKT_PTEPDE_COPY_HEADER_scope_mask) << SDMA_PKT_PTEPDE_COPY_HEADER_scope_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_PTEPDE_COPY_SRC_ADDR_LO_src_addr_31_0_offset 1
+#define SDMA_PKT_PTEPDE_COPY_SRC_ADDR_LO_src_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_SRC_ADDR_LO_src_addr_31_0_shift  0
+#define SDMA_PKT_PTEPDE_COPY_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_PTEPDE_COPY_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_PTEPDE_COPY_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_PTEPDE_COPY_SRC_ADDR_HI_src_addr_63_32_offset 2
+#define SDMA_PKT_PTEPDE_COPY_SRC_ADDR_HI_src_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_SRC_ADDR_HI_src_addr_63_32_shift  0
+#define SDMA_PKT_PTEPDE_COPY_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_PTEPDE_COPY_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_PTEPDE_COPY_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_PTEPDE_COPY_DST_ADDR_LO_dst_addr_31_0_offset 3
+#define SDMA_PKT_PTEPDE_COPY_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_PTEPDE_COPY_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_PTEPDE_COPY_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_PTEPDE_COPY_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_PTEPDE_COPY_DST_ADDR_HI_dst_addr_63_32_offset 4
+#define SDMA_PKT_PTEPDE_COPY_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_PTEPDE_COPY_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_PTEPDE_COPY_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_PTEPDE_COPY_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for MASK_DW0 word*/
+/*define for mask_dw0 field*/
+#define SDMA_PKT_PTEPDE_COPY_MASK_DW0_mask_dw0_offset 5
+#define SDMA_PKT_PTEPDE_COPY_MASK_DW0_mask_dw0_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_MASK_DW0_mask_dw0_shift  0
+#define SDMA_PKT_PTEPDE_COPY_MASK_DW0_MASK_DW0(x) (((x) & SDMA_PKT_PTEPDE_COPY_MASK_DW0_mask_dw0_mask) << SDMA_PKT_PTEPDE_COPY_MASK_DW0_mask_dw0_shift)
+
+/*define for MASK_DW1 word*/
+/*define for mask_dw1 field*/
+#define SDMA_PKT_PTEPDE_COPY_MASK_DW1_mask_dw1_offset 6
+#define SDMA_PKT_PTEPDE_COPY_MASK_DW1_mask_dw1_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_MASK_DW1_mask_dw1_shift  0
+#define SDMA_PKT_PTEPDE_COPY_MASK_DW1_MASK_DW1(x) (((x) & SDMA_PKT_PTEPDE_COPY_MASK_DW1_mask_dw1_mask) << SDMA_PKT_PTEPDE_COPY_MASK_DW1_mask_dw1_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_PTEPDE_COPY_COUNT_count_offset 7
+#define SDMA_PKT_PTEPDE_COPY_COUNT_count_mask   0x0007FFFF
+#define SDMA_PKT_PTEPDE_COPY_COUNT_count_shift  0
+#define SDMA_PKT_PTEPDE_COPY_COUNT_COUNT(x) (((x) & SDMA_PKT_PTEPDE_COPY_COUNT_count_mask) << SDMA_PKT_PTEPDE_COPY_COUNT_count_shift)
+
+/*define for dst_cache_policy field*/
+#define SDMA_PKT_PTEPDE_COPY_COUNT_dst_cache_policy_offset 7
+#define SDMA_PKT_PTEPDE_COPY_COUNT_dst_cache_policy_mask   0x00000007
+#define SDMA_PKT_PTEPDE_COPY_COUNT_dst_cache_policy_shift  22
+#define SDMA_PKT_PTEPDE_COPY_COUNT_DST_CACHE_POLICY(x) (((x) & SDMA_PKT_PTEPDE_COPY_COUNT_dst_cache_policy_mask) << SDMA_PKT_PTEPDE_COPY_COUNT_dst_cache_policy_shift)
+
+/*define for src_cache_policy field*/
+#define SDMA_PKT_PTEPDE_COPY_COUNT_src_cache_policy_offset 7
+#define SDMA_PKT_PTEPDE_COPY_COUNT_src_cache_policy_mask   0x00000007
+#define SDMA_PKT_PTEPDE_COPY_COUNT_src_cache_policy_shift  29
+#define SDMA_PKT_PTEPDE_COPY_COUNT_SRC_CACHE_POLICY(x) (((x) & SDMA_PKT_PTEPDE_COPY_COUNT_src_cache_policy_mask) << SDMA_PKT_PTEPDE_COPY_COUNT_src_cache_policy_shift)
+
+
+/*
+** Definitions for SDMA_PKT_PTEPDE_COPY_BACKWARDS packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_op_offset 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_op_shift  0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_OP(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_op_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_sub_op_offset 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_sub_op_shift  8
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_SUB_OP(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_sub_op_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_sub_op_shift)
+
+/*define for pte_size field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_pte_size_offset 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_pte_size_mask   0x00000003
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_pte_size_shift  28
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_PTE_SIZE(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_pte_size_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_pte_size_shift)
+
+/*define for direction field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_direction_offset 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_direction_mask   0x00000001
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_direction_shift  30
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_DIRECTION(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_direction_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_direction_shift)
+
+/*define for ptepde_op field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_ptepde_op_offset 0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_ptepde_op_mask   0x00000001
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_ptepde_op_shift  31
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_PTEPDE_OP(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_ptepde_op_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_HEADER_ptepde_op_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_LO_src_addr_31_0_offset 1
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_LO_src_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_LO_src_addr_31_0_shift  0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_HI_src_addr_63_32_offset 2
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_HI_src_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_HI_src_addr_63_32_shift  0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_LO_dst_addr_31_0_offset 3
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_HI_dst_addr_63_32_offset 4
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for MASK_BIT_FOR_DW word*/
+/*define for mask_first_xfer field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_first_xfer_offset 5
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_first_xfer_mask   0x000000FF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_first_xfer_shift  0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_MASK_FIRST_XFER(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_first_xfer_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_first_xfer_shift)
+
+/*define for mask_last_xfer field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_last_xfer_offset 5
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_last_xfer_mask   0x000000FF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_last_xfer_shift  8
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_MASK_LAST_XFER(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_last_xfer_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_MASK_BIT_FOR_DW_mask_last_xfer_shift)
+
+/*define for COUNT_IN_32B_XFER word*/
+/*define for count field*/
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_COUNT_IN_32B_XFER_count_offset 6
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_COUNT_IN_32B_XFER_count_mask   0x0001FFFF
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_COUNT_IN_32B_XFER_count_shift  0
+#define SDMA_PKT_PTEPDE_COPY_BACKWARDS_COUNT_IN_32B_XFER_COUNT(x) (((x) & SDMA_PKT_PTEPDE_COPY_BACKWARDS_COUNT_IN_32B_XFER_count_mask) << SDMA_PKT_PTEPDE_COPY_BACKWARDS_COUNT_IN_32B_XFER_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_PTEPDE_RMW packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_op_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_PTEPDE_RMW_HEADER_op_shift  0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_OP(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_op_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_sub_op_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_PTEPDE_RMW_HEADER_sub_op_shift  8
+#define SDMA_PKT_PTEPDE_RMW_HEADER_SUB_OP(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_sub_op_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_sub_op_shift)
+
+/*define for mtype field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_mtype_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_mtype_mask   0x00000007
+#define SDMA_PKT_PTEPDE_RMW_HEADER_mtype_shift  16
+#define SDMA_PKT_PTEPDE_RMW_HEADER_MTYPE(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_mtype_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_mtype_shift)
+
+/*define for gcc field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_gcc_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_gcc_mask   0x00000001
+#define SDMA_PKT_PTEPDE_RMW_HEADER_gcc_shift  19
+#define SDMA_PKT_PTEPDE_RMW_HEADER_GCC(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_gcc_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_gcc_shift)
+
+/*define for sys field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_sys_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_sys_mask   0x00000001
+#define SDMA_PKT_PTEPDE_RMW_HEADER_sys_shift  20
+#define SDMA_PKT_PTEPDE_RMW_HEADER_SYS(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_sys_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_sys_shift)
+
+/*define for snp field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_snp_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_snp_mask   0x00000001
+#define SDMA_PKT_PTEPDE_RMW_HEADER_snp_shift  22
+#define SDMA_PKT_PTEPDE_RMW_HEADER_SNP(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_snp_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_snp_shift)
+
+/*define for gpa field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_gpa_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_gpa_mask   0x00000001
+#define SDMA_PKT_PTEPDE_RMW_HEADER_gpa_shift  23
+#define SDMA_PKT_PTEPDE_RMW_HEADER_GPA(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_gpa_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_gpa_shift)
+
+/*define for l2_policy field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_l2_policy_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_l2_policy_mask   0x00000003
+#define SDMA_PKT_PTEPDE_RMW_HEADER_l2_policy_shift  24
+#define SDMA_PKT_PTEPDE_RMW_HEADER_L2_POLICY(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_l2_policy_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_l2_policy_shift)
+
+/*define for llc_policy field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_llc_policy_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_llc_policy_mask   0x00000001
+#define SDMA_PKT_PTEPDE_RMW_HEADER_llc_policy_shift  26
+#define SDMA_PKT_PTEPDE_RMW_HEADER_LLC_POLICY(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_llc_policy_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_llc_policy_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_PTEPDE_RMW_HEADER_cpv_offset 0
+#define SDMA_PKT_PTEPDE_RMW_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_PTEPDE_RMW_HEADER_cpv_shift  28
+#define SDMA_PKT_PTEPDE_RMW_HEADER_CPV(x) (((x) & SDMA_PKT_PTEPDE_RMW_HEADER_cpv_mask) << SDMA_PKT_PTEPDE_RMW_HEADER_cpv_shift)
+
+/*define for ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_PTEPDE_RMW_ADDR_LO_addr_31_0_offset 1
+#define SDMA_PKT_PTEPDE_RMW_ADDR_LO_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_RMW_ADDR_LO_addr_31_0_shift  0
+#define SDMA_PKT_PTEPDE_RMW_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_PTEPDE_RMW_ADDR_LO_addr_31_0_mask) << SDMA_PKT_PTEPDE_RMW_ADDR_LO_addr_31_0_shift)
+
+/*define for ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_PTEPDE_RMW_ADDR_HI_addr_63_32_offset 2
+#define SDMA_PKT_PTEPDE_RMW_ADDR_HI_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_RMW_ADDR_HI_addr_63_32_shift  0
+#define SDMA_PKT_PTEPDE_RMW_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_PTEPDE_RMW_ADDR_HI_addr_63_32_mask) << SDMA_PKT_PTEPDE_RMW_ADDR_HI_addr_63_32_shift)
+
+/*define for MASK_LO word*/
+/*define for mask_31_0 field*/
+#define SDMA_PKT_PTEPDE_RMW_MASK_LO_mask_31_0_offset 3
+#define SDMA_PKT_PTEPDE_RMW_MASK_LO_mask_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_RMW_MASK_LO_mask_31_0_shift  0
+#define SDMA_PKT_PTEPDE_RMW_MASK_LO_MASK_31_0(x) (((x) & SDMA_PKT_PTEPDE_RMW_MASK_LO_mask_31_0_mask) << SDMA_PKT_PTEPDE_RMW_MASK_LO_mask_31_0_shift)
+
+/*define for MASK_HI word*/
+/*define for mask_63_32 field*/
+#define SDMA_PKT_PTEPDE_RMW_MASK_HI_mask_63_32_offset 4
+#define SDMA_PKT_PTEPDE_RMW_MASK_HI_mask_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_RMW_MASK_HI_mask_63_32_shift  0
+#define SDMA_PKT_PTEPDE_RMW_MASK_HI_MASK_63_32(x) (((x) & SDMA_PKT_PTEPDE_RMW_MASK_HI_mask_63_32_mask) << SDMA_PKT_PTEPDE_RMW_MASK_HI_mask_63_32_shift)
+
+/*define for VALUE_LO word*/
+/*define for value_31_0 field*/
+#define SDMA_PKT_PTEPDE_RMW_VALUE_LO_value_31_0_offset 5
+#define SDMA_PKT_PTEPDE_RMW_VALUE_LO_value_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_RMW_VALUE_LO_value_31_0_shift  0
+#define SDMA_PKT_PTEPDE_RMW_VALUE_LO_VALUE_31_0(x) (((x) & SDMA_PKT_PTEPDE_RMW_VALUE_LO_value_31_0_mask) << SDMA_PKT_PTEPDE_RMW_VALUE_LO_value_31_0_shift)
+
+/*define for VALUE_HI word*/
+/*define for value_63_32 field*/
+#define SDMA_PKT_PTEPDE_RMW_VALUE_HI_value_63_32_offset 6
+#define SDMA_PKT_PTEPDE_RMW_VALUE_HI_value_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_RMW_VALUE_HI_value_63_32_shift  0
+#define SDMA_PKT_PTEPDE_RMW_VALUE_HI_VALUE_63_32(x) (((x) & SDMA_PKT_PTEPDE_RMW_VALUE_HI_value_63_32_mask) << SDMA_PKT_PTEPDE_RMW_VALUE_HI_value_63_32_shift)
+
+/*define for COUNT word*/
+/*define for num_of_pte field*/
+#define SDMA_PKT_PTEPDE_RMW_COUNT_num_of_pte_offset 7
+#define SDMA_PKT_PTEPDE_RMW_COUNT_num_of_pte_mask   0xFFFFFFFF
+#define SDMA_PKT_PTEPDE_RMW_COUNT_num_of_pte_shift  0
+#define SDMA_PKT_PTEPDE_RMW_COUNT_NUM_OF_PTE(x) (((x) & SDMA_PKT_PTEPDE_RMW_COUNT_num_of_pte_mask) << SDMA_PKT_PTEPDE_RMW_COUNT_num_of_pte_shift)
+
+
+/*
+** Definitions for SDMA_PKT_REGISTER_RMW packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_REGISTER_RMW_HEADER_op_offset 0
+#define SDMA_PKT_REGISTER_RMW_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_REGISTER_RMW_HEADER_op_shift  0
+#define SDMA_PKT_REGISTER_RMW_HEADER_OP(x) (((x) & SDMA_PKT_REGISTER_RMW_HEADER_op_mask) << SDMA_PKT_REGISTER_RMW_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_REGISTER_RMW_HEADER_sub_op_offset 0
+#define SDMA_PKT_REGISTER_RMW_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_REGISTER_RMW_HEADER_sub_op_shift  8
+#define SDMA_PKT_REGISTER_RMW_HEADER_SUB_OP(x) (((x) & SDMA_PKT_REGISTER_RMW_HEADER_sub_op_mask) << SDMA_PKT_REGISTER_RMW_HEADER_sub_op_shift)
+
+/*define for ADDR word*/
+/*define for addr field*/
+#define SDMA_PKT_REGISTER_RMW_ADDR_addr_offset 1
+#define SDMA_PKT_REGISTER_RMW_ADDR_addr_mask   0x000FFFFF
+#define SDMA_PKT_REGISTER_RMW_ADDR_addr_shift  0
+#define SDMA_PKT_REGISTER_RMW_ADDR_ADDR(x) (((x) & SDMA_PKT_REGISTER_RMW_ADDR_addr_mask) << SDMA_PKT_REGISTER_RMW_ADDR_addr_shift)
+
+/*define for aperture_id field*/
+#define SDMA_PKT_REGISTER_RMW_ADDR_aperture_id_offset 1
+#define SDMA_PKT_REGISTER_RMW_ADDR_aperture_id_mask   0x00000FFF
+#define SDMA_PKT_REGISTER_RMW_ADDR_aperture_id_shift  20
+#define SDMA_PKT_REGISTER_RMW_ADDR_APERTURE_ID(x) (((x) & SDMA_PKT_REGISTER_RMW_ADDR_aperture_id_mask) << SDMA_PKT_REGISTER_RMW_ADDR_aperture_id_shift)
+
+/*define for MASK word*/
+/*define for mask field*/
+#define SDMA_PKT_REGISTER_RMW_MASK_mask_offset 2
+#define SDMA_PKT_REGISTER_RMW_MASK_mask_mask   0xFFFFFFFF
+#define SDMA_PKT_REGISTER_RMW_MASK_mask_shift  0
+#define SDMA_PKT_REGISTER_RMW_MASK_MASK(x) (((x) & SDMA_PKT_REGISTER_RMW_MASK_mask_mask) << SDMA_PKT_REGISTER_RMW_MASK_mask_shift)
+
+/*define for VALUE word*/
+/*define for value field*/
+#define SDMA_PKT_REGISTER_RMW_VALUE_value_offset 3
+#define SDMA_PKT_REGISTER_RMW_VALUE_value_mask   0xFFFFFFFF
+#define SDMA_PKT_REGISTER_RMW_VALUE_value_shift  0
+#define SDMA_PKT_REGISTER_RMW_VALUE_VALUE(x) (((x) & SDMA_PKT_REGISTER_RMW_VALUE_value_mask) << SDMA_PKT_REGISTER_RMW_VALUE_value_shift)
+
+/*define for MISC word*/
+/*define for stride field*/
+#define SDMA_PKT_REGISTER_RMW_MISC_stride_offset 4
+#define SDMA_PKT_REGISTER_RMW_MISC_stride_mask   0x000FFFFF
+#define SDMA_PKT_REGISTER_RMW_MISC_stride_shift  0
+#define SDMA_PKT_REGISTER_RMW_MISC_STRIDE(x) (((x) & SDMA_PKT_REGISTER_RMW_MISC_stride_mask) << SDMA_PKT_REGISTER_RMW_MISC_stride_shift)
+
+/*define for num_of_reg field*/
+#define SDMA_PKT_REGISTER_RMW_MISC_num_of_reg_offset 4
+#define SDMA_PKT_REGISTER_RMW_MISC_num_of_reg_mask   0x00000FFF
+#define SDMA_PKT_REGISTER_RMW_MISC_num_of_reg_shift  20
+#define SDMA_PKT_REGISTER_RMW_MISC_NUM_OF_REG(x) (((x) & SDMA_PKT_REGISTER_RMW_MISC_num_of_reg_mask) << SDMA_PKT_REGISTER_RMW_MISC_num_of_reg_shift)
+
+
+/*
+** Definitions for SDMA_PKT_WRITE_INCR packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_WRITE_INCR_HEADER_op_offset 0
+#define SDMA_PKT_WRITE_INCR_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_WRITE_INCR_HEADER_op_shift  0
+#define SDMA_PKT_WRITE_INCR_HEADER_OP(x) (((x) & SDMA_PKT_WRITE_INCR_HEADER_op_mask) << SDMA_PKT_WRITE_INCR_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_WRITE_INCR_HEADER_sub_op_offset 0
+#define SDMA_PKT_WRITE_INCR_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_WRITE_INCR_HEADER_sub_op_shift  8
+#define SDMA_PKT_WRITE_INCR_HEADER_SUB_OP(x) (((x) & SDMA_PKT_WRITE_INCR_HEADER_sub_op_mask) << SDMA_PKT_WRITE_INCR_HEADER_sub_op_shift)
+
+/*define for cache_policy field*/
+#define SDMA_PKT_WRITE_INCR_HEADER_cache_policy_offset 0
+#define SDMA_PKT_WRITE_INCR_HEADER_cache_policy_mask   0x00000007
+#define SDMA_PKT_WRITE_INCR_HEADER_cache_policy_shift  24
+#define SDMA_PKT_WRITE_INCR_HEADER_CACHE_POLICY(x) (((x) & SDMA_PKT_WRITE_INCR_HEADER_cache_policy_mask) << SDMA_PKT_WRITE_INCR_HEADER_cache_policy_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_WRITE_INCR_HEADER_cpv_offset 0
+#define SDMA_PKT_WRITE_INCR_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_WRITE_INCR_HEADER_cpv_shift  28
+#define SDMA_PKT_WRITE_INCR_HEADER_CPV(x) (((x) & SDMA_PKT_WRITE_INCR_HEADER_cpv_mask) << SDMA_PKT_WRITE_INCR_HEADER_cpv_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_WRITE_INCR_DST_ADDR_LO_dst_addr_31_0_offset 1
+#define SDMA_PKT_WRITE_INCR_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_INCR_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_WRITE_INCR_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_WRITE_INCR_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_WRITE_INCR_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_WRITE_INCR_DST_ADDR_HI_dst_addr_63_32_offset 2
+#define SDMA_PKT_WRITE_INCR_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_INCR_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_WRITE_INCR_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_WRITE_INCR_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_WRITE_INCR_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for MASK_DW0 word*/
+/*define for mask_dw0 field*/
+#define SDMA_PKT_WRITE_INCR_MASK_DW0_mask_dw0_offset 3
+#define SDMA_PKT_WRITE_INCR_MASK_DW0_mask_dw0_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_INCR_MASK_DW0_mask_dw0_shift  0
+#define SDMA_PKT_WRITE_INCR_MASK_DW0_MASK_DW0(x) (((x) & SDMA_PKT_WRITE_INCR_MASK_DW0_mask_dw0_mask) << SDMA_PKT_WRITE_INCR_MASK_DW0_mask_dw0_shift)
+
+/*define for MASK_DW1 word*/
+/*define for mask_dw1 field*/
+#define SDMA_PKT_WRITE_INCR_MASK_DW1_mask_dw1_offset 4
+#define SDMA_PKT_WRITE_INCR_MASK_DW1_mask_dw1_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_INCR_MASK_DW1_mask_dw1_shift  0
+#define SDMA_PKT_WRITE_INCR_MASK_DW1_MASK_DW1(x) (((x) & SDMA_PKT_WRITE_INCR_MASK_DW1_mask_dw1_mask) << SDMA_PKT_WRITE_INCR_MASK_DW1_mask_dw1_shift)
+
+/*define for INIT_DW0 word*/
+/*define for init_dw0 field*/
+#define SDMA_PKT_WRITE_INCR_INIT_DW0_init_dw0_offset 5
+#define SDMA_PKT_WRITE_INCR_INIT_DW0_init_dw0_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_INCR_INIT_DW0_init_dw0_shift  0
+#define SDMA_PKT_WRITE_INCR_INIT_DW0_INIT_DW0(x) (((x) & SDMA_PKT_WRITE_INCR_INIT_DW0_init_dw0_mask) << SDMA_PKT_WRITE_INCR_INIT_DW0_init_dw0_shift)
+
+/*define for INIT_DW1 word*/
+/*define for init_dw1 field*/
+#define SDMA_PKT_WRITE_INCR_INIT_DW1_init_dw1_offset 6
+#define SDMA_PKT_WRITE_INCR_INIT_DW1_init_dw1_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_INCR_INIT_DW1_init_dw1_shift  0
+#define SDMA_PKT_WRITE_INCR_INIT_DW1_INIT_DW1(x) (((x) & SDMA_PKT_WRITE_INCR_INIT_DW1_init_dw1_mask) << SDMA_PKT_WRITE_INCR_INIT_DW1_init_dw1_shift)
+
+/*define for INCR_DW0 word*/
+/*define for incr_dw0 field*/
+#define SDMA_PKT_WRITE_INCR_INCR_DW0_incr_dw0_offset 7
+#define SDMA_PKT_WRITE_INCR_INCR_DW0_incr_dw0_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_INCR_INCR_DW0_incr_dw0_shift  0
+#define SDMA_PKT_WRITE_INCR_INCR_DW0_INCR_DW0(x) (((x) & SDMA_PKT_WRITE_INCR_INCR_DW0_incr_dw0_mask) << SDMA_PKT_WRITE_INCR_INCR_DW0_incr_dw0_shift)
+
+/*define for INCR_DW1 word*/
+/*define for incr_dw1 field*/
+#define SDMA_PKT_WRITE_INCR_INCR_DW1_incr_dw1_offset 8
+#define SDMA_PKT_WRITE_INCR_INCR_DW1_incr_dw1_mask   0xFFFFFFFF
+#define SDMA_PKT_WRITE_INCR_INCR_DW1_incr_dw1_shift  0
+#define SDMA_PKT_WRITE_INCR_INCR_DW1_INCR_DW1(x) (((x) & SDMA_PKT_WRITE_INCR_INCR_DW1_incr_dw1_mask) << SDMA_PKT_WRITE_INCR_INCR_DW1_incr_dw1_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_WRITE_INCR_COUNT_count_offset 9
+#define SDMA_PKT_WRITE_INCR_COUNT_count_mask   0x0007FFFF
+#define SDMA_PKT_WRITE_INCR_COUNT_count_shift  0
+#define SDMA_PKT_WRITE_INCR_COUNT_COUNT(x) (((x) & SDMA_PKT_WRITE_INCR_COUNT_count_mask) << SDMA_PKT_WRITE_INCR_COUNT_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_INDIRECT packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_INDIRECT_HEADER_op_offset 0
+#define SDMA_PKT_INDIRECT_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_INDIRECT_HEADER_op_shift  0
+#define SDMA_PKT_INDIRECT_HEADER_OP(x) (((x) & SDMA_PKT_INDIRECT_HEADER_op_mask) << SDMA_PKT_INDIRECT_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_INDIRECT_HEADER_sub_op_offset 0
+#define SDMA_PKT_INDIRECT_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_INDIRECT_HEADER_sub_op_shift  8
+#define SDMA_PKT_INDIRECT_HEADER_SUB_OP(x) (((x) & SDMA_PKT_INDIRECT_HEADER_sub_op_mask) << SDMA_PKT_INDIRECT_HEADER_sub_op_shift)
+
+/*define for vmid field*/
+#define SDMA_PKT_INDIRECT_HEADER_vmid_offset 0
+#define SDMA_PKT_INDIRECT_HEADER_vmid_mask   0x0000000F
+#define SDMA_PKT_INDIRECT_HEADER_vmid_shift  16
+#define SDMA_PKT_INDIRECT_HEADER_VMID(x) (((x) & SDMA_PKT_INDIRECT_HEADER_vmid_mask) << SDMA_PKT_INDIRECT_HEADER_vmid_shift)
+
+/*define for priv field*/
+#define SDMA_PKT_INDIRECT_HEADER_priv_offset 0
+#define SDMA_PKT_INDIRECT_HEADER_priv_mask   0x00000001
+#define SDMA_PKT_INDIRECT_HEADER_priv_shift  31
+#define SDMA_PKT_INDIRECT_HEADER_PRIV(x) (((x) & SDMA_PKT_INDIRECT_HEADER_priv_mask) << SDMA_PKT_INDIRECT_HEADER_priv_shift)
+
+/*define for BASE_LO word*/
+/*define for ib_base_31_0 field*/
+#define SDMA_PKT_INDIRECT_BASE_LO_ib_base_31_0_offset 1
+#define SDMA_PKT_INDIRECT_BASE_LO_ib_base_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_INDIRECT_BASE_LO_ib_base_31_0_shift  0
+#define SDMA_PKT_INDIRECT_BASE_LO_IB_BASE_31_0(x) (((x) & SDMA_PKT_INDIRECT_BASE_LO_ib_base_31_0_mask) << SDMA_PKT_INDIRECT_BASE_LO_ib_base_31_0_shift)
+
+/*define for BASE_HI word*/
+/*define for ib_base_63_32 field*/
+#define SDMA_PKT_INDIRECT_BASE_HI_ib_base_63_32_offset 2
+#define SDMA_PKT_INDIRECT_BASE_HI_ib_base_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_INDIRECT_BASE_HI_ib_base_63_32_shift  0
+#define SDMA_PKT_INDIRECT_BASE_HI_IB_BASE_63_32(x) (((x) & SDMA_PKT_INDIRECT_BASE_HI_ib_base_63_32_mask) << SDMA_PKT_INDIRECT_BASE_HI_ib_base_63_32_shift)
+
+/*define for IB_SIZE word*/
+/*define for ib_size field*/
+#define SDMA_PKT_INDIRECT_IB_SIZE_ib_size_offset 3
+#define SDMA_PKT_INDIRECT_IB_SIZE_ib_size_mask   0x000FFFFF
+#define SDMA_PKT_INDIRECT_IB_SIZE_ib_size_shift  0
+#define SDMA_PKT_INDIRECT_IB_SIZE_IB_SIZE(x) (((x) & SDMA_PKT_INDIRECT_IB_SIZE_ib_size_mask) << SDMA_PKT_INDIRECT_IB_SIZE_ib_size_shift)
+
+/*define for CSA_ADDR_LO word*/
+/*define for csa_addr_31_0 field*/
+#define SDMA_PKT_INDIRECT_CSA_ADDR_LO_csa_addr_31_0_offset 4
+#define SDMA_PKT_INDIRECT_CSA_ADDR_LO_csa_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_INDIRECT_CSA_ADDR_LO_csa_addr_31_0_shift  0
+#define SDMA_PKT_INDIRECT_CSA_ADDR_LO_CSA_ADDR_31_0(x) (((x) & SDMA_PKT_INDIRECT_CSA_ADDR_LO_csa_addr_31_0_mask) << SDMA_PKT_INDIRECT_CSA_ADDR_LO_csa_addr_31_0_shift)
+
+/*define for CSA_ADDR_HI word*/
+/*define for csa_addr_63_32 field*/
+#define SDMA_PKT_INDIRECT_CSA_ADDR_HI_csa_addr_63_32_offset 5
+#define SDMA_PKT_INDIRECT_CSA_ADDR_HI_csa_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_INDIRECT_CSA_ADDR_HI_csa_addr_63_32_shift  0
+#define SDMA_PKT_INDIRECT_CSA_ADDR_HI_CSA_ADDR_63_32(x) (((x) & SDMA_PKT_INDIRECT_CSA_ADDR_HI_csa_addr_63_32_mask) << SDMA_PKT_INDIRECT_CSA_ADDR_HI_csa_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_SEMAPHORE packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_SEMAPHORE_HEADER_op_offset 0
+#define SDMA_PKT_SEMAPHORE_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_SEMAPHORE_HEADER_op_shift  0
+#define SDMA_PKT_SEMAPHORE_HEADER_OP(x) (((x) & SDMA_PKT_SEMAPHORE_HEADER_op_mask) << SDMA_PKT_SEMAPHORE_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_SEMAPHORE_HEADER_sub_op_offset 0
+#define SDMA_PKT_SEMAPHORE_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_SEMAPHORE_HEADER_sub_op_shift  8
+#define SDMA_PKT_SEMAPHORE_HEADER_SUB_OP(x) (((x) & SDMA_PKT_SEMAPHORE_HEADER_sub_op_mask) << SDMA_PKT_SEMAPHORE_HEADER_sub_op_shift)
+
+/*define for write_one field*/
+#define SDMA_PKT_SEMAPHORE_HEADER_write_one_offset 0
+#define SDMA_PKT_SEMAPHORE_HEADER_write_one_mask   0x00000001
+#define SDMA_PKT_SEMAPHORE_HEADER_write_one_shift  29
+#define SDMA_PKT_SEMAPHORE_HEADER_WRITE_ONE(x) (((x) & SDMA_PKT_SEMAPHORE_HEADER_write_one_mask) << SDMA_PKT_SEMAPHORE_HEADER_write_one_shift)
+
+/*define for signal field*/
+#define SDMA_PKT_SEMAPHORE_HEADER_signal_offset 0
+#define SDMA_PKT_SEMAPHORE_HEADER_signal_mask   0x00000001
+#define SDMA_PKT_SEMAPHORE_HEADER_signal_shift  30
+#define SDMA_PKT_SEMAPHORE_HEADER_SIGNAL(x) (((x) & SDMA_PKT_SEMAPHORE_HEADER_signal_mask) << SDMA_PKT_SEMAPHORE_HEADER_signal_shift)
+
+/*define for mailbox field*/
+#define SDMA_PKT_SEMAPHORE_HEADER_mailbox_offset 0
+#define SDMA_PKT_SEMAPHORE_HEADER_mailbox_mask   0x00000001
+#define SDMA_PKT_SEMAPHORE_HEADER_mailbox_shift  31
+#define SDMA_PKT_SEMAPHORE_HEADER_MAILBOX(x) (((x) & SDMA_PKT_SEMAPHORE_HEADER_mailbox_mask) << SDMA_PKT_SEMAPHORE_HEADER_mailbox_shift)
+
+/*define for ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_SEMAPHORE_ADDR_LO_addr_31_0_offset 1
+#define SDMA_PKT_SEMAPHORE_ADDR_LO_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_SEMAPHORE_ADDR_LO_addr_31_0_shift  0
+#define SDMA_PKT_SEMAPHORE_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_SEMAPHORE_ADDR_LO_addr_31_0_mask) << SDMA_PKT_SEMAPHORE_ADDR_LO_addr_31_0_shift)
+
+/*define for ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_SEMAPHORE_ADDR_HI_addr_63_32_offset 2
+#define SDMA_PKT_SEMAPHORE_ADDR_HI_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_SEMAPHORE_ADDR_HI_addr_63_32_shift  0
+#define SDMA_PKT_SEMAPHORE_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_SEMAPHORE_ADDR_HI_addr_63_32_mask) << SDMA_PKT_SEMAPHORE_ADDR_HI_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_MEM_INCR packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_MEM_INCR_HEADER_op_offset 0
+#define SDMA_PKT_MEM_INCR_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_MEM_INCR_HEADER_op_shift  0
+#define SDMA_PKT_MEM_INCR_HEADER_OP(x) (((x) & SDMA_PKT_MEM_INCR_HEADER_op_mask) << SDMA_PKT_MEM_INCR_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_MEM_INCR_HEADER_sub_op_offset 0
+#define SDMA_PKT_MEM_INCR_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_MEM_INCR_HEADER_sub_op_shift  8
+#define SDMA_PKT_MEM_INCR_HEADER_SUB_OP(x) (((x) & SDMA_PKT_MEM_INCR_HEADER_sub_op_mask) << SDMA_PKT_MEM_INCR_HEADER_sub_op_shift)
+
+/*define for l2_policy field*/
+#define SDMA_PKT_MEM_INCR_HEADER_l2_policy_offset 0
+#define SDMA_PKT_MEM_INCR_HEADER_l2_policy_mask   0x00000003
+#define SDMA_PKT_MEM_INCR_HEADER_l2_policy_shift  24
+#define SDMA_PKT_MEM_INCR_HEADER_L2_POLICY(x) (((x) & SDMA_PKT_MEM_INCR_HEADER_l2_policy_mask) << SDMA_PKT_MEM_INCR_HEADER_l2_policy_shift)
+
+/*define for llc_policy field*/
+#define SDMA_PKT_MEM_INCR_HEADER_llc_policy_offset 0
+#define SDMA_PKT_MEM_INCR_HEADER_llc_policy_mask   0x00000001
+#define SDMA_PKT_MEM_INCR_HEADER_llc_policy_shift  26
+#define SDMA_PKT_MEM_INCR_HEADER_LLC_POLICY(x) (((x) & SDMA_PKT_MEM_INCR_HEADER_llc_policy_mask) << SDMA_PKT_MEM_INCR_HEADER_llc_policy_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_MEM_INCR_HEADER_cpv_offset 0
+#define SDMA_PKT_MEM_INCR_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_MEM_INCR_HEADER_cpv_shift  28
+#define SDMA_PKT_MEM_INCR_HEADER_CPV(x) (((x) & SDMA_PKT_MEM_INCR_HEADER_cpv_mask) << SDMA_PKT_MEM_INCR_HEADER_cpv_shift)
+
+/*define for ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_MEM_INCR_ADDR_LO_addr_31_0_offset 1
+#define SDMA_PKT_MEM_INCR_ADDR_LO_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_MEM_INCR_ADDR_LO_addr_31_0_shift  0
+#define SDMA_PKT_MEM_INCR_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_MEM_INCR_ADDR_LO_addr_31_0_mask) << SDMA_PKT_MEM_INCR_ADDR_LO_addr_31_0_shift)
+
+/*define for ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_MEM_INCR_ADDR_HI_addr_63_32_offset 2
+#define SDMA_PKT_MEM_INCR_ADDR_HI_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_MEM_INCR_ADDR_HI_addr_63_32_shift  0
+#define SDMA_PKT_MEM_INCR_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_MEM_INCR_ADDR_HI_addr_63_32_mask) << SDMA_PKT_MEM_INCR_ADDR_HI_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_VM_INVALIDATION packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_VM_INVALIDATION_HEADER_op_offset 0
+#define SDMA_PKT_VM_INVALIDATION_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_VM_INVALIDATION_HEADER_op_shift  0
+#define SDMA_PKT_VM_INVALIDATION_HEADER_OP(x) (((x) & SDMA_PKT_VM_INVALIDATION_HEADER_op_mask) << SDMA_PKT_VM_INVALIDATION_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_VM_INVALIDATION_HEADER_sub_op_offset 0
+#define SDMA_PKT_VM_INVALIDATION_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_VM_INVALIDATION_HEADER_sub_op_shift  8
+#define SDMA_PKT_VM_INVALIDATION_HEADER_SUB_OP(x) (((x) & SDMA_PKT_VM_INVALIDATION_HEADER_sub_op_mask) << SDMA_PKT_VM_INVALIDATION_HEADER_sub_op_shift)
+
+/*define for gfx_eng_id field*/
+#define SDMA_PKT_VM_INVALIDATION_HEADER_gfx_eng_id_offset 0
+#define SDMA_PKT_VM_INVALIDATION_HEADER_gfx_eng_id_mask   0x0000001F
+#define SDMA_PKT_VM_INVALIDATION_HEADER_gfx_eng_id_shift  16
+#define SDMA_PKT_VM_INVALIDATION_HEADER_GFX_ENG_ID(x) (((x) & SDMA_PKT_VM_INVALIDATION_HEADER_gfx_eng_id_mask) << SDMA_PKT_VM_INVALIDATION_HEADER_gfx_eng_id_shift)
+
+/*define for mm_eng_id field*/
+#define SDMA_PKT_VM_INVALIDATION_HEADER_mm_eng_id_offset 0
+#define SDMA_PKT_VM_INVALIDATION_HEADER_mm_eng_id_mask   0x0000001F
+#define SDMA_PKT_VM_INVALIDATION_HEADER_mm_eng_id_shift  24
+#define SDMA_PKT_VM_INVALIDATION_HEADER_MM_ENG_ID(x) (((x) & SDMA_PKT_VM_INVALIDATION_HEADER_mm_eng_id_mask) << SDMA_PKT_VM_INVALIDATION_HEADER_mm_eng_id_shift)
+
+/*define for INVALIDATEREQ word*/
+/*define for invalidatereq field*/
+#define SDMA_PKT_VM_INVALIDATION_INVALIDATEREQ_invalidatereq_offset 1
+#define SDMA_PKT_VM_INVALIDATION_INVALIDATEREQ_invalidatereq_mask   0xFFFFFFFF
+#define SDMA_PKT_VM_INVALIDATION_INVALIDATEREQ_invalidatereq_shift  0
+#define SDMA_PKT_VM_INVALIDATION_INVALIDATEREQ_INVALIDATEREQ(x) (((x) & SDMA_PKT_VM_INVALIDATION_INVALIDATEREQ_invalidatereq_mask) << SDMA_PKT_VM_INVALIDATION_INVALIDATEREQ_invalidatereq_shift)
+
+/*define for ADDRESSRANGELO word*/
+/*define for addressrangelo field*/
+#define SDMA_PKT_VM_INVALIDATION_ADDRESSRANGELO_addressrangelo_offset 2
+#define SDMA_PKT_VM_INVALIDATION_ADDRESSRANGELO_addressrangelo_mask   0xFFFFFFFF
+#define SDMA_PKT_VM_INVALIDATION_ADDRESSRANGELO_addressrangelo_shift  0
+#define SDMA_PKT_VM_INVALIDATION_ADDRESSRANGELO_ADDRESSRANGELO(x) (((x) & SDMA_PKT_VM_INVALIDATION_ADDRESSRANGELO_addressrangelo_mask) << SDMA_PKT_VM_INVALIDATION_ADDRESSRANGELO_addressrangelo_shift)
+
+/*define for ADDRESSRANGEHI word*/
+/*define for invalidateack field*/
+#define SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_invalidateack_offset 3
+#define SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_invalidateack_mask   0x0000FFFF
+#define SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_invalidateack_shift  0
+#define SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_INVALIDATEACK(x) (((x) & SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_invalidateack_mask) << SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_invalidateack_shift)
+
+/*define for addressrangehi field*/
+#define SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_addressrangehi_offset 3
+#define SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_addressrangehi_mask   0x0000001F
+#define SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_addressrangehi_shift  16
+#define SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_ADDRESSRANGEHI(x) (((x) & SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_addressrangehi_mask) << SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_addressrangehi_shift)
+
+/*define for reserved field*/
+#define SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_reserved_offset 3
+#define SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_reserved_mask   0x000001FF
+#define SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_reserved_shift  23
+#define SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_RESERVED(x) (((x) & SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_reserved_mask) << SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_reserved_shift)
+
+
+/*
+** Definitions for SDMA_PKT_FENCE packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_FENCE_HEADER_op_offset 0
+#define SDMA_PKT_FENCE_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_FENCE_HEADER_op_shift  0
+#define SDMA_PKT_FENCE_HEADER_OP(x) (((x) & SDMA_PKT_FENCE_HEADER_op_mask) << SDMA_PKT_FENCE_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_FENCE_HEADER_sub_op_offset 0
+#define SDMA_PKT_FENCE_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_FENCE_HEADER_sub_op_shift  8
+#define SDMA_PKT_FENCE_HEADER_SUB_OP(x) (((x) & SDMA_PKT_FENCE_HEADER_sub_op_mask) << SDMA_PKT_FENCE_HEADER_sub_op_shift)
+
+/*define for mtype field*/
+#define SDMA_PKT_FENCE_HEADER_mtype_offset 0
+#define SDMA_PKT_FENCE_HEADER_mtype_mask   0x00000007
+#define SDMA_PKT_FENCE_HEADER_mtype_shift  16
+#define SDMA_PKT_FENCE_HEADER_MTYPE(x) (((x) & SDMA_PKT_FENCE_HEADER_mtype_mask) << SDMA_PKT_FENCE_HEADER_mtype_shift)
+
+/*define for gcc field*/
+#define SDMA_PKT_FENCE_HEADER_gcc_offset 0
+#define SDMA_PKT_FENCE_HEADER_gcc_mask   0x00000001
+#define SDMA_PKT_FENCE_HEADER_gcc_shift  19
+#define SDMA_PKT_FENCE_HEADER_GCC(x) (((x) & SDMA_PKT_FENCE_HEADER_gcc_mask) << SDMA_PKT_FENCE_HEADER_gcc_shift)
+
+/*define for sys field*/
+#define SDMA_PKT_FENCE_HEADER_sys_offset 0
+#define SDMA_PKT_FENCE_HEADER_sys_mask   0x00000001
+#define SDMA_PKT_FENCE_HEADER_sys_shift  20
+#define SDMA_PKT_FENCE_HEADER_SYS(x) (((x) & SDMA_PKT_FENCE_HEADER_sys_mask) << SDMA_PKT_FENCE_HEADER_sys_shift)
+
+/*define for snp field*/
+#define SDMA_PKT_FENCE_HEADER_snp_offset 0
+#define SDMA_PKT_FENCE_HEADER_snp_mask   0x00000001
+#define SDMA_PKT_FENCE_HEADER_snp_shift  22
+#define SDMA_PKT_FENCE_HEADER_SNP(x) (((x) & SDMA_PKT_FENCE_HEADER_snp_mask) << SDMA_PKT_FENCE_HEADER_snp_shift)
+
+/*define for gpa field*/
+#define SDMA_PKT_FENCE_HEADER_gpa_offset 0
+#define SDMA_PKT_FENCE_HEADER_gpa_mask   0x00000001
+#define SDMA_PKT_FENCE_HEADER_gpa_shift  23
+#define SDMA_PKT_FENCE_HEADER_GPA(x) (((x) & SDMA_PKT_FENCE_HEADER_gpa_mask) << SDMA_PKT_FENCE_HEADER_gpa_shift)
+
+/*define for l2_policy field*/
+#define SDMA_PKT_FENCE_HEADER_l2_policy_offset 0
+#define SDMA_PKT_FENCE_HEADER_l2_policy_mask   0x00000003
+#define SDMA_PKT_FENCE_HEADER_l2_policy_shift  24
+#define SDMA_PKT_FENCE_HEADER_L2_POLICY(x) (((x) & SDMA_PKT_FENCE_HEADER_l2_policy_mask) << SDMA_PKT_FENCE_HEADER_l2_policy_shift)
+
+/*define for llc_policy field*/
+#define SDMA_PKT_FENCE_HEADER_llc_policy_offset 0
+#define SDMA_PKT_FENCE_HEADER_llc_policy_mask   0x00000001
+#define SDMA_PKT_FENCE_HEADER_llc_policy_shift  26
+#define SDMA_PKT_FENCE_HEADER_LLC_POLICY(x) (((x) & SDMA_PKT_FENCE_HEADER_llc_policy_mask) << SDMA_PKT_FENCE_HEADER_llc_policy_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_FENCE_HEADER_cpv_offset 0
+#define SDMA_PKT_FENCE_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_FENCE_HEADER_cpv_shift  28
+#define SDMA_PKT_FENCE_HEADER_CPV(x) (((x) & SDMA_PKT_FENCE_HEADER_cpv_mask) << SDMA_PKT_FENCE_HEADER_cpv_shift)
+
+/*define for ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_FENCE_ADDR_LO_addr_31_0_offset 1
+#define SDMA_PKT_FENCE_ADDR_LO_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_FENCE_ADDR_LO_addr_31_0_shift  0
+#define SDMA_PKT_FENCE_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_FENCE_ADDR_LO_addr_31_0_mask) << SDMA_PKT_FENCE_ADDR_LO_addr_31_0_shift)
+
+/*define for ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_FENCE_ADDR_HI_addr_63_32_offset 2
+#define SDMA_PKT_FENCE_ADDR_HI_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_FENCE_ADDR_HI_addr_63_32_shift  0
+#define SDMA_PKT_FENCE_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_FENCE_ADDR_HI_addr_63_32_mask) << SDMA_PKT_FENCE_ADDR_HI_addr_63_32_shift)
+
+/*define for DATA word*/
+/*define for data field*/
+#define SDMA_PKT_FENCE_DATA_data_offset 3
+#define SDMA_PKT_FENCE_DATA_data_mask   0xFFFFFFFF
+#define SDMA_PKT_FENCE_DATA_data_shift  0
+#define SDMA_PKT_FENCE_DATA_DATA(x) (((x) & SDMA_PKT_FENCE_DATA_data_mask) << SDMA_PKT_FENCE_DATA_data_shift)
+
+
+/*
+** Definitions for SDMA_PKT_SRBM_WRITE packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_SRBM_WRITE_HEADER_op_offset 0
+#define SDMA_PKT_SRBM_WRITE_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_SRBM_WRITE_HEADER_op_shift  0
+#define SDMA_PKT_SRBM_WRITE_HEADER_OP(x) (((x) & SDMA_PKT_SRBM_WRITE_HEADER_op_mask) << SDMA_PKT_SRBM_WRITE_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_SRBM_WRITE_HEADER_sub_op_offset 0
+#define SDMA_PKT_SRBM_WRITE_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_SRBM_WRITE_HEADER_sub_op_shift  8
+#define SDMA_PKT_SRBM_WRITE_HEADER_SUB_OP(x) (((x) & SDMA_PKT_SRBM_WRITE_HEADER_sub_op_mask) << SDMA_PKT_SRBM_WRITE_HEADER_sub_op_shift)
+
+/*define for byte_en field*/
+#define SDMA_PKT_SRBM_WRITE_HEADER_byte_en_offset 0
+#define SDMA_PKT_SRBM_WRITE_HEADER_byte_en_mask   0x0000000F
+#define SDMA_PKT_SRBM_WRITE_HEADER_byte_en_shift  28
+#define SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(x) (((x) & SDMA_PKT_SRBM_WRITE_HEADER_byte_en_mask) << SDMA_PKT_SRBM_WRITE_HEADER_byte_en_shift)
+
+/*define for ADDR word*/
+/*define for addr field*/
+#define SDMA_PKT_SRBM_WRITE_ADDR_addr_offset 1
+#define SDMA_PKT_SRBM_WRITE_ADDR_addr_mask   0x0003FFFF
+#define SDMA_PKT_SRBM_WRITE_ADDR_addr_shift  0
+#define SDMA_PKT_SRBM_WRITE_ADDR_ADDR(x) (((x) & SDMA_PKT_SRBM_WRITE_ADDR_addr_mask) << SDMA_PKT_SRBM_WRITE_ADDR_addr_shift)
+
+/*define for apertureid field*/
+#define SDMA_PKT_SRBM_WRITE_ADDR_apertureid_offset 1
+#define SDMA_PKT_SRBM_WRITE_ADDR_apertureid_mask   0x00000FFF
+#define SDMA_PKT_SRBM_WRITE_ADDR_apertureid_shift  20
+#define SDMA_PKT_SRBM_WRITE_ADDR_APERTUREID(x) (((x) & SDMA_PKT_SRBM_WRITE_ADDR_apertureid_mask) << SDMA_PKT_SRBM_WRITE_ADDR_apertureid_shift)
+
+/*define for DATA word*/
+/*define for data field*/
+#define SDMA_PKT_SRBM_WRITE_DATA_data_offset 2
+#define SDMA_PKT_SRBM_WRITE_DATA_data_mask   0xFFFFFFFF
+#define SDMA_PKT_SRBM_WRITE_DATA_data_shift  0
+#define SDMA_PKT_SRBM_WRITE_DATA_DATA(x) (((x) & SDMA_PKT_SRBM_WRITE_DATA_data_mask) << SDMA_PKT_SRBM_WRITE_DATA_data_shift)
+
+
+/*
+** Definitions for SDMA_PKT_PRE_EXE packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_PRE_EXE_HEADER_op_offset 0
+#define SDMA_PKT_PRE_EXE_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_PRE_EXE_HEADER_op_shift  0
+#define SDMA_PKT_PRE_EXE_HEADER_OP(x) (((x) & SDMA_PKT_PRE_EXE_HEADER_op_mask) << SDMA_PKT_PRE_EXE_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_PRE_EXE_HEADER_sub_op_offset 0
+#define SDMA_PKT_PRE_EXE_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_PRE_EXE_HEADER_sub_op_shift  8
+#define SDMA_PKT_PRE_EXE_HEADER_SUB_OP(x) (((x) & SDMA_PKT_PRE_EXE_HEADER_sub_op_mask) << SDMA_PKT_PRE_EXE_HEADER_sub_op_shift)
+
+/*define for dev_sel field*/
+#define SDMA_PKT_PRE_EXE_HEADER_dev_sel_offset 0
+#define SDMA_PKT_PRE_EXE_HEADER_dev_sel_mask   0x000000FF
+#define SDMA_PKT_PRE_EXE_HEADER_dev_sel_shift  16
+#define SDMA_PKT_PRE_EXE_HEADER_DEV_SEL(x) (((x) & SDMA_PKT_PRE_EXE_HEADER_dev_sel_mask) << SDMA_PKT_PRE_EXE_HEADER_dev_sel_shift)
+
+/*define for EXEC_COUNT word*/
+/*define for exec_count field*/
+#define SDMA_PKT_PRE_EXE_EXEC_COUNT_exec_count_offset 1
+#define SDMA_PKT_PRE_EXE_EXEC_COUNT_exec_count_mask   0x00003FFF
+#define SDMA_PKT_PRE_EXE_EXEC_COUNT_exec_count_shift  0
+#define SDMA_PKT_PRE_EXE_EXEC_COUNT_EXEC_COUNT(x) (((x) & SDMA_PKT_PRE_EXE_EXEC_COUNT_exec_count_mask) << SDMA_PKT_PRE_EXE_EXEC_COUNT_exec_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_COND_EXE packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_COND_EXE_HEADER_op_offset 0
+#define SDMA_PKT_COND_EXE_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_COND_EXE_HEADER_op_shift  0
+#define SDMA_PKT_COND_EXE_HEADER_OP(x) (((x) & SDMA_PKT_COND_EXE_HEADER_op_mask) << SDMA_PKT_COND_EXE_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_COND_EXE_HEADER_sub_op_offset 0
+#define SDMA_PKT_COND_EXE_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_COND_EXE_HEADER_sub_op_shift  8
+#define SDMA_PKT_COND_EXE_HEADER_SUB_OP(x) (((x) & SDMA_PKT_COND_EXE_HEADER_sub_op_mask) << SDMA_PKT_COND_EXE_HEADER_sub_op_shift)
+
+/*define for cache_policy field*/
+#define SDMA_PKT_COND_EXE_HEADER_cache_policy_offset 0
+#define SDMA_PKT_COND_EXE_HEADER_cache_policy_mask   0x00000007
+#define SDMA_PKT_COND_EXE_HEADER_cache_policy_shift  24
+#define SDMA_PKT_COND_EXE_HEADER_CACHE_POLICY(x) (((x) & SDMA_PKT_COND_EXE_HEADER_cache_policy_mask) << SDMA_PKT_COND_EXE_HEADER_cache_policy_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_COND_EXE_HEADER_cpv_offset 0
+#define SDMA_PKT_COND_EXE_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_COND_EXE_HEADER_cpv_shift  28
+#define SDMA_PKT_COND_EXE_HEADER_CPV(x) (((x) & SDMA_PKT_COND_EXE_HEADER_cpv_mask) << SDMA_PKT_COND_EXE_HEADER_cpv_shift)
+
+/*define for ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_COND_EXE_ADDR_LO_addr_31_0_offset 1
+#define SDMA_PKT_COND_EXE_ADDR_LO_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_COND_EXE_ADDR_LO_addr_31_0_shift  0
+#define SDMA_PKT_COND_EXE_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_COND_EXE_ADDR_LO_addr_31_0_mask) << SDMA_PKT_COND_EXE_ADDR_LO_addr_31_0_shift)
+
+/*define for ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_COND_EXE_ADDR_HI_addr_63_32_offset 2
+#define SDMA_PKT_COND_EXE_ADDR_HI_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_COND_EXE_ADDR_HI_addr_63_32_shift  0
+#define SDMA_PKT_COND_EXE_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_COND_EXE_ADDR_HI_addr_63_32_mask) << SDMA_PKT_COND_EXE_ADDR_HI_addr_63_32_shift)
+
+/*define for REFERENCE word*/
+/*define for reference field*/
+#define SDMA_PKT_COND_EXE_REFERENCE_reference_offset 3
+#define SDMA_PKT_COND_EXE_REFERENCE_reference_mask   0xFFFFFFFF
+#define SDMA_PKT_COND_EXE_REFERENCE_reference_shift  0
+#define SDMA_PKT_COND_EXE_REFERENCE_REFERENCE(x) (((x) & SDMA_PKT_COND_EXE_REFERENCE_reference_mask) << SDMA_PKT_COND_EXE_REFERENCE_reference_shift)
+
+/*define for EXEC_COUNT word*/
+/*define for exec_count field*/
+#define SDMA_PKT_COND_EXE_EXEC_COUNT_exec_count_offset 4
+#define SDMA_PKT_COND_EXE_EXEC_COUNT_exec_count_mask   0x00003FFF
+#define SDMA_PKT_COND_EXE_EXEC_COUNT_exec_count_shift  0
+#define SDMA_PKT_COND_EXE_EXEC_COUNT_EXEC_COUNT(x) (((x) & SDMA_PKT_COND_EXE_EXEC_COUNT_exec_count_mask) << SDMA_PKT_COND_EXE_EXEC_COUNT_exec_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_CONSTANT_FILL packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_CONSTANT_FILL_HEADER_op_offset 0
+#define SDMA_PKT_CONSTANT_FILL_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_CONSTANT_FILL_HEADER_op_shift  0
+#define SDMA_PKT_CONSTANT_FILL_HEADER_OP(x) (((x) & SDMA_PKT_CONSTANT_FILL_HEADER_op_mask) << SDMA_PKT_CONSTANT_FILL_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_CONSTANT_FILL_HEADER_sub_op_offset 0
+#define SDMA_PKT_CONSTANT_FILL_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_CONSTANT_FILL_HEADER_sub_op_shift  8
+#define SDMA_PKT_CONSTANT_FILL_HEADER_SUB_OP(x) (((x) & SDMA_PKT_CONSTANT_FILL_HEADER_sub_op_mask) << SDMA_PKT_CONSTANT_FILL_HEADER_sub_op_shift)
+
+/*define for sw field*/
+#define SDMA_PKT_CONSTANT_FILL_HEADER_sw_offset 0
+#define SDMA_PKT_CONSTANT_FILL_HEADER_sw_mask   0x00000003
+#define SDMA_PKT_CONSTANT_FILL_HEADER_sw_shift  16
+#define SDMA_PKT_CONSTANT_FILL_HEADER_SW(x) (((x) & SDMA_PKT_CONSTANT_FILL_HEADER_sw_mask) << SDMA_PKT_CONSTANT_FILL_HEADER_sw_shift)
+
+/*define for cache_policy field*/
+#define SDMA_PKT_CONSTANT_FILL_HEADER_cache_policy_offset 0
+#define SDMA_PKT_CONSTANT_FILL_HEADER_cache_policy_mask   0x00000007
+#define SDMA_PKT_CONSTANT_FILL_HEADER_cache_policy_shift  24
+#define SDMA_PKT_CONSTANT_FILL_HEADER_CACHE_POLICY(x) (((x) & SDMA_PKT_CONSTANT_FILL_HEADER_cache_policy_mask) << SDMA_PKT_CONSTANT_FILL_HEADER_cache_policy_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_CONSTANT_FILL_HEADER_cpv_offset 0
+#define SDMA_PKT_CONSTANT_FILL_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_CONSTANT_FILL_HEADER_cpv_shift  28
+#define SDMA_PKT_CONSTANT_FILL_HEADER_CPV(x) (((x) & SDMA_PKT_CONSTANT_FILL_HEADER_cpv_mask) << SDMA_PKT_CONSTANT_FILL_HEADER_cpv_shift)
+
+/*define for fillsize field*/
+#define SDMA_PKT_CONSTANT_FILL_HEADER_fillsize_offset 0
+#define SDMA_PKT_CONSTANT_FILL_HEADER_fillsize_mask   0x00000003
+#define SDMA_PKT_CONSTANT_FILL_HEADER_fillsize_shift  30
+#define SDMA_PKT_CONSTANT_FILL_HEADER_FILLSIZE(x) (((x) & SDMA_PKT_CONSTANT_FILL_HEADER_fillsize_mask) << SDMA_PKT_CONSTANT_FILL_HEADER_fillsize_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_CONSTANT_FILL_DST_ADDR_LO_dst_addr_31_0_offset 1
+#define SDMA_PKT_CONSTANT_FILL_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_CONSTANT_FILL_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_CONSTANT_FILL_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_CONSTANT_FILL_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_CONSTANT_FILL_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_CONSTANT_FILL_DST_ADDR_HI_dst_addr_63_32_offset 2
+#define SDMA_PKT_CONSTANT_FILL_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_CONSTANT_FILL_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_CONSTANT_FILL_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_CONSTANT_FILL_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_CONSTANT_FILL_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for DATA word*/
+/*define for src_data_31_0 field*/
+#define SDMA_PKT_CONSTANT_FILL_DATA_src_data_31_0_offset 3
+#define SDMA_PKT_CONSTANT_FILL_DATA_src_data_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_CONSTANT_FILL_DATA_src_data_31_0_shift  0
+#define SDMA_PKT_CONSTANT_FILL_DATA_SRC_DATA_31_0(x) (((x) & SDMA_PKT_CONSTANT_FILL_DATA_src_data_31_0_mask) << SDMA_PKT_CONSTANT_FILL_DATA_src_data_31_0_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_CONSTANT_FILL_COUNT_count_offset 4
+#define SDMA_PKT_CONSTANT_FILL_COUNT_count_mask   0x3FFFFFFF
+#define SDMA_PKT_CONSTANT_FILL_COUNT_count_shift  0
+#define SDMA_PKT_CONSTANT_FILL_COUNT_COUNT(x) (((x) & SDMA_PKT_CONSTANT_FILL_COUNT_count_mask) << SDMA_PKT_CONSTANT_FILL_COUNT_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_DATA_FILL_MULTI packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_op_offset 0
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_op_shift  0
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_OP(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_HEADER_op_mask) << SDMA_PKT_DATA_FILL_MULTI_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_sub_op_offset 0
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_sub_op_shift  8
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_SUB_OP(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_HEADER_sub_op_mask) << SDMA_PKT_DATA_FILL_MULTI_HEADER_sub_op_shift)
+
+/*define for cache_policy field*/
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_cache_policy_offset 0
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_cache_policy_mask   0x00000007
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_cache_policy_shift  24
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_CACHE_POLICY(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_HEADER_cache_policy_mask) << SDMA_PKT_DATA_FILL_MULTI_HEADER_cache_policy_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_cpv_offset 0
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_cpv_shift  28
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_CPV(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_HEADER_cpv_mask) << SDMA_PKT_DATA_FILL_MULTI_HEADER_cpv_shift)
+
+/*define for memlog_clr field*/
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_memlog_clr_offset 0
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_memlog_clr_mask   0x00000001
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_memlog_clr_shift  31
+#define SDMA_PKT_DATA_FILL_MULTI_HEADER_MEMLOG_CLR(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_HEADER_memlog_clr_mask) << SDMA_PKT_DATA_FILL_MULTI_HEADER_memlog_clr_shift)
+
+/*define for BYTE_STRIDE word*/
+/*define for byte_stride field*/
+#define SDMA_PKT_DATA_FILL_MULTI_BYTE_STRIDE_byte_stride_offset 1
+#define SDMA_PKT_DATA_FILL_MULTI_BYTE_STRIDE_byte_stride_mask   0xFFFFFFFF
+#define SDMA_PKT_DATA_FILL_MULTI_BYTE_STRIDE_byte_stride_shift  0
+#define SDMA_PKT_DATA_FILL_MULTI_BYTE_STRIDE_BYTE_STRIDE(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_BYTE_STRIDE_byte_stride_mask) << SDMA_PKT_DATA_FILL_MULTI_BYTE_STRIDE_byte_stride_shift)
+
+/*define for DMA_COUNT word*/
+/*define for dma_count field*/
+#define SDMA_PKT_DATA_FILL_MULTI_DMA_COUNT_dma_count_offset 2
+#define SDMA_PKT_DATA_FILL_MULTI_DMA_COUNT_dma_count_mask   0xFFFFFFFF
+#define SDMA_PKT_DATA_FILL_MULTI_DMA_COUNT_dma_count_shift  0
+#define SDMA_PKT_DATA_FILL_MULTI_DMA_COUNT_DMA_COUNT(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_DMA_COUNT_dma_count_mask) << SDMA_PKT_DATA_FILL_MULTI_DMA_COUNT_dma_count_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_LO_dst_addr_31_0_offset 3
+#define SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_HI_dst_addr_63_32_offset 4
+#define SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_PKT_DATA_FILL_MULTI_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for BYTE_COUNT word*/
+/*define for count field*/
+#define SDMA_PKT_DATA_FILL_MULTI_BYTE_COUNT_count_offset 5
+#define SDMA_PKT_DATA_FILL_MULTI_BYTE_COUNT_count_mask   0x03FFFFFF
+#define SDMA_PKT_DATA_FILL_MULTI_BYTE_COUNT_count_shift  0
+#define SDMA_PKT_DATA_FILL_MULTI_BYTE_COUNT_COUNT(x) (((x) & SDMA_PKT_DATA_FILL_MULTI_BYTE_COUNT_count_mask) << SDMA_PKT_DATA_FILL_MULTI_BYTE_COUNT_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_POLL_REGMEM packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_POLL_REGMEM_HEADER_op_offset 0
+#define SDMA_PKT_POLL_REGMEM_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_POLL_REGMEM_HEADER_op_shift  0
+#define SDMA_PKT_POLL_REGMEM_HEADER_OP(x) (((x) & SDMA_PKT_POLL_REGMEM_HEADER_op_mask) << SDMA_PKT_POLL_REGMEM_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_POLL_REGMEM_HEADER_sub_op_offset 0
+#define SDMA_PKT_POLL_REGMEM_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_POLL_REGMEM_HEADER_sub_op_shift  8
+#define SDMA_PKT_POLL_REGMEM_HEADER_SUB_OP(x) (((x) & SDMA_PKT_POLL_REGMEM_HEADER_sub_op_mask) << SDMA_PKT_POLL_REGMEM_HEADER_sub_op_shift)
+
+/*define for cache_policy field*/
+#define SDMA_PKT_POLL_REGMEM_HEADER_cache_policy_offset 0
+#define SDMA_PKT_POLL_REGMEM_HEADER_cache_policy_mask   0x00000007
+#define SDMA_PKT_POLL_REGMEM_HEADER_cache_policy_shift  20
+#define SDMA_PKT_POLL_REGMEM_HEADER_CACHE_POLICY(x) (((x) & SDMA_PKT_POLL_REGMEM_HEADER_cache_policy_mask) << SDMA_PKT_POLL_REGMEM_HEADER_cache_policy_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_POLL_REGMEM_HEADER_cpv_offset 0
+#define SDMA_PKT_POLL_REGMEM_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_POLL_REGMEM_HEADER_cpv_shift  24
+#define SDMA_PKT_POLL_REGMEM_HEADER_CPV(x) (((x) & SDMA_PKT_POLL_REGMEM_HEADER_cpv_mask) << SDMA_PKT_POLL_REGMEM_HEADER_cpv_shift)
+
+/*define for hdp_flush field*/
+#define SDMA_PKT_POLL_REGMEM_HEADER_hdp_flush_offset 0
+#define SDMA_PKT_POLL_REGMEM_HEADER_hdp_flush_mask   0x00000001
+#define SDMA_PKT_POLL_REGMEM_HEADER_hdp_flush_shift  26
+#define SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(x) (((x) & SDMA_PKT_POLL_REGMEM_HEADER_hdp_flush_mask) << SDMA_PKT_POLL_REGMEM_HEADER_hdp_flush_shift)
+
+/*define for func field*/
+#define SDMA_PKT_POLL_REGMEM_HEADER_func_offset 0
+#define SDMA_PKT_POLL_REGMEM_HEADER_func_mask   0x00000007
+#define SDMA_PKT_POLL_REGMEM_HEADER_func_shift  28
+#define SDMA_PKT_POLL_REGMEM_HEADER_FUNC(x) (((x) & SDMA_PKT_POLL_REGMEM_HEADER_func_mask) << SDMA_PKT_POLL_REGMEM_HEADER_func_shift)
+
+/*define for mem_poll field*/
+#define SDMA_PKT_POLL_REGMEM_HEADER_mem_poll_offset 0
+#define SDMA_PKT_POLL_REGMEM_HEADER_mem_poll_mask   0x00000001
+#define SDMA_PKT_POLL_REGMEM_HEADER_mem_poll_shift  31
+#define SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(x) (((x) & SDMA_PKT_POLL_REGMEM_HEADER_mem_poll_mask) << SDMA_PKT_POLL_REGMEM_HEADER_mem_poll_shift)
+
+/*define for ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_POLL_REGMEM_ADDR_LO_addr_31_0_offset 1
+#define SDMA_PKT_POLL_REGMEM_ADDR_LO_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_REGMEM_ADDR_LO_addr_31_0_shift  0
+#define SDMA_PKT_POLL_REGMEM_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_POLL_REGMEM_ADDR_LO_addr_31_0_mask) << SDMA_PKT_POLL_REGMEM_ADDR_LO_addr_31_0_shift)
+
+/*define for ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_POLL_REGMEM_ADDR_HI_addr_63_32_offset 2
+#define SDMA_PKT_POLL_REGMEM_ADDR_HI_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_REGMEM_ADDR_HI_addr_63_32_shift  0
+#define SDMA_PKT_POLL_REGMEM_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_POLL_REGMEM_ADDR_HI_addr_63_32_mask) << SDMA_PKT_POLL_REGMEM_ADDR_HI_addr_63_32_shift)
+
+/*define for VALUE word*/
+/*define for value field*/
+#define SDMA_PKT_POLL_REGMEM_VALUE_value_offset 3
+#define SDMA_PKT_POLL_REGMEM_VALUE_value_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_REGMEM_VALUE_value_shift  0
+#define SDMA_PKT_POLL_REGMEM_VALUE_VALUE(x) (((x) & SDMA_PKT_POLL_REGMEM_VALUE_value_mask) << SDMA_PKT_POLL_REGMEM_VALUE_value_shift)
+
+/*define for MASK word*/
+/*define for mask field*/
+#define SDMA_PKT_POLL_REGMEM_MASK_mask_offset 4
+#define SDMA_PKT_POLL_REGMEM_MASK_mask_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_REGMEM_MASK_mask_shift  0
+#define SDMA_PKT_POLL_REGMEM_MASK_MASK(x) (((x) & SDMA_PKT_POLL_REGMEM_MASK_mask_mask) << SDMA_PKT_POLL_REGMEM_MASK_mask_shift)
+
+/*define for DW5 word*/
+/*define for interval field*/
+#define SDMA_PKT_POLL_REGMEM_DW5_interval_offset 5
+#define SDMA_PKT_POLL_REGMEM_DW5_interval_mask   0x0000FFFF
+#define SDMA_PKT_POLL_REGMEM_DW5_interval_shift  0
+#define SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(x) (((x) & SDMA_PKT_POLL_REGMEM_DW5_interval_mask) << SDMA_PKT_POLL_REGMEM_DW5_interval_shift)
+
+/*define for retry_count field*/
+#define SDMA_PKT_POLL_REGMEM_DW5_retry_count_offset 5
+#define SDMA_PKT_POLL_REGMEM_DW5_retry_count_mask   0x00000FFF
+#define SDMA_PKT_POLL_REGMEM_DW5_retry_count_shift  16
+#define SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(x) (((x) & SDMA_PKT_POLL_REGMEM_DW5_retry_count_mask) << SDMA_PKT_POLL_REGMEM_DW5_retry_count_shift)
+
+
+/*
+** Definitions for SDMA_PKT_POLL_REG_WRITE_MEM packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_op_offset 0
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_op_shift  0
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_OP(x) (((x) & SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_op_mask) << SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_sub_op_offset 0
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_sub_op_shift  8
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_SUB_OP(x) (((x) & SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_sub_op_mask) << SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_sub_op_shift)
+
+/*define for cache_policy field*/
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_cache_policy_offset 0
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_cache_policy_mask   0x00000007
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_cache_policy_shift  24
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_CACHE_POLICY(x) (((x) & SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_cache_policy_mask) << SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_cache_policy_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_cpv_offset 0
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_cpv_shift  28
+#define SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_CPV(x) (((x) & SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_cpv_mask) << SDMA_PKT_POLL_REG_WRITE_MEM_HEADER_cpv_shift)
+
+/*define for SRC_ADDR word*/
+/*define for addr_31_2 field*/
+#define SDMA_PKT_POLL_REG_WRITE_MEM_SRC_ADDR_addr_31_2_offset 1
+#define SDMA_PKT_POLL_REG_WRITE_MEM_SRC_ADDR_addr_31_2_mask   0x3FFFFFFF
+#define SDMA_PKT_POLL_REG_WRITE_MEM_SRC_ADDR_addr_31_2_shift  2
+#define SDMA_PKT_POLL_REG_WRITE_MEM_SRC_ADDR_ADDR_31_2(x) (((x) & SDMA_PKT_POLL_REG_WRITE_MEM_SRC_ADDR_addr_31_2_mask) << SDMA_PKT_POLL_REG_WRITE_MEM_SRC_ADDR_addr_31_2_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_LO_addr_31_0_offset 2
+#define SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_LO_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_LO_addr_31_0_shift  0
+#define SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_LO_addr_31_0_mask) << SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_LO_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_HI_addr_63_32_offset 3
+#define SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_HI_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_HI_addr_63_32_shift  0
+#define SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_HI_addr_63_32_mask) << SDMA_PKT_POLL_REG_WRITE_MEM_DST_ADDR_HI_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_POLL_DBIT_WRITE_MEM packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_op_offset 0
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_op_shift  0
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_OP(x) (((x) & SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_op_mask) << SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_sub_op_offset 0
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_sub_op_shift  8
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_SUB_OP(x) (((x) & SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_sub_op_mask) << SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_sub_op_shift)
+
+/*define for ea field*/
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_ea_offset 0
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_ea_mask   0x00000003
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_ea_shift  16
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_EA(x) (((x) & SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_ea_mask) << SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_ea_shift)
+
+/*define for cache_policy field*/
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_cache_policy_offset 0
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_cache_policy_mask   0x00000007
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_cache_policy_shift  24
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_CACHE_POLICY(x) (((x) & SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_cache_policy_mask) << SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_cache_policy_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_cpv_offset 0
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_cpv_shift  28
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_CPV(x) (((x) & SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_cpv_mask) << SDMA_PKT_POLL_DBIT_WRITE_MEM_HEADER_cpv_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_LO_addr_31_0_offset 1
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_LO_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_LO_addr_31_0_shift  0
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_LO_addr_31_0_mask) << SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_LO_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_HI_addr_63_32_offset 2
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_HI_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_HI_addr_63_32_shift  0
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_HI_addr_63_32_mask) << SDMA_PKT_POLL_DBIT_WRITE_MEM_DST_ADDR_HI_addr_63_32_shift)
+
+/*define for START_PAGE word*/
+/*define for addr_31_4 field*/
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_START_PAGE_addr_31_4_offset 3
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_START_PAGE_addr_31_4_mask   0x0FFFFFFF
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_START_PAGE_addr_31_4_shift  4
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_START_PAGE_ADDR_31_4(x) (((x) & SDMA_PKT_POLL_DBIT_WRITE_MEM_START_PAGE_addr_31_4_mask) << SDMA_PKT_POLL_DBIT_WRITE_MEM_START_PAGE_addr_31_4_shift)
+
+/*define for PAGE_NUM word*/
+/*define for page_num_31_0 field*/
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_PAGE_NUM_page_num_31_0_offset 4
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_PAGE_NUM_page_num_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_PAGE_NUM_page_num_31_0_shift  0
+#define SDMA_PKT_POLL_DBIT_WRITE_MEM_PAGE_NUM_PAGE_NUM_31_0(x) (((x) & SDMA_PKT_POLL_DBIT_WRITE_MEM_PAGE_NUM_page_num_31_0_mask) << SDMA_PKT_POLL_DBIT_WRITE_MEM_PAGE_NUM_page_num_31_0_shift)
+
+
+/*
+** Definitions for SDMA_PKT_POLL_MEM_VERIFY packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_op_offset 0
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_op_shift  0
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_OP(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_HEADER_op_mask) << SDMA_PKT_POLL_MEM_VERIFY_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_sub_op_offset 0
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_sub_op_shift  8
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_SUB_OP(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_HEADER_sub_op_mask) << SDMA_PKT_POLL_MEM_VERIFY_HEADER_sub_op_shift)
+
+/*define for cache_policy field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_cache_policy_offset 0
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_cache_policy_mask   0x00000007
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_cache_policy_shift  24
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_CACHE_POLICY(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_HEADER_cache_policy_mask) << SDMA_PKT_POLL_MEM_VERIFY_HEADER_cache_policy_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_cpv_offset 0
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_cpv_shift  28
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_CPV(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_HEADER_cpv_mask) << SDMA_PKT_POLL_MEM_VERIFY_HEADER_cpv_shift)
+
+/*define for mode field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_mode_offset 0
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_mode_mask   0x00000001
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_mode_shift  31
+#define SDMA_PKT_POLL_MEM_VERIFY_HEADER_MODE(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_HEADER_mode_mask) << SDMA_PKT_POLL_MEM_VERIFY_HEADER_mode_shift)
+
+/*define for PATTERN word*/
+/*define for pattern field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_PATTERN_pattern_offset 1
+#define SDMA_PKT_POLL_MEM_VERIFY_PATTERN_pattern_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_PATTERN_pattern_shift  0
+#define SDMA_PKT_POLL_MEM_VERIFY_PATTERN_PATTERN(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_PATTERN_pattern_mask) << SDMA_PKT_POLL_MEM_VERIFY_PATTERN_pattern_shift)
+
+/*define for CMP0_ADDR_START_LO word*/
+/*define for cmp0_start_31_0 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_LO_cmp0_start_31_0_offset 2
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_LO_cmp0_start_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_LO_cmp0_start_31_0_shift  0
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_LO_CMP0_START_31_0(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_LO_cmp0_start_31_0_mask) << SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_LO_cmp0_start_31_0_shift)
+
+/*define for CMP0_ADDR_START_HI word*/
+/*define for cmp0_start_63_32 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_HI_cmp0_start_63_32_offset 3
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_HI_cmp0_start_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_HI_cmp0_start_63_32_shift  0
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_HI_CMP0_START_63_32(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_HI_cmp0_start_63_32_mask) << SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_START_HI_cmp0_start_63_32_shift)
+
+/*define for CMP0_ADDR_END_LO word*/
+/*define for cmp0_end_31_0 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_LO_cmp0_end_31_0_offset 4
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_LO_cmp0_end_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_LO_cmp0_end_31_0_shift  0
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_LO_CMP0_END_31_0(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_LO_cmp0_end_31_0_mask) << SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_LO_cmp0_end_31_0_shift)
+
+/*define for CMP0_ADDR_END_HI word*/
+/*define for cmp0_end_63_32 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_HI_cmp0_end_63_32_offset 5
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_HI_cmp0_end_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_HI_cmp0_end_63_32_shift  0
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_HI_CMP0_END_63_32(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_HI_cmp0_end_63_32_mask) << SDMA_PKT_POLL_MEM_VERIFY_CMP0_ADDR_END_HI_cmp0_end_63_32_shift)
+
+/*define for CMP1_ADDR_START_LO word*/
+/*define for cmp1_start_31_0 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_LO_cmp1_start_31_0_offset 6
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_LO_cmp1_start_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_LO_cmp1_start_31_0_shift  0
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_LO_CMP1_START_31_0(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_LO_cmp1_start_31_0_mask) << SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_LO_cmp1_start_31_0_shift)
+
+/*define for CMP1_ADDR_START_HI word*/
+/*define for cmp1_start_63_32 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_HI_cmp1_start_63_32_offset 7
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_HI_cmp1_start_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_HI_cmp1_start_63_32_shift  0
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_HI_CMP1_START_63_32(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_HI_cmp1_start_63_32_mask) << SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_START_HI_cmp1_start_63_32_shift)
+
+/*define for CMP1_ADDR_END_LO word*/
+/*define for cmp1_end_31_0 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_LO_cmp1_end_31_0_offset 8
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_LO_cmp1_end_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_LO_cmp1_end_31_0_shift  0
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_LO_CMP1_END_31_0(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_LO_cmp1_end_31_0_mask) << SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_LO_cmp1_end_31_0_shift)
+
+/*define for CMP1_ADDR_END_HI word*/
+/*define for cmp1_end_63_32 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_HI_cmp1_end_63_32_offset 9
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_HI_cmp1_end_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_HI_cmp1_end_63_32_shift  0
+#define SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_HI_CMP1_END_63_32(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_HI_cmp1_end_63_32_mask) << SDMA_PKT_POLL_MEM_VERIFY_CMP1_ADDR_END_HI_cmp1_end_63_32_shift)
+
+/*define for REC_ADDR_LO word*/
+/*define for rec_31_0 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_LO_rec_31_0_offset 10
+#define SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_LO_rec_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_LO_rec_31_0_shift  0
+#define SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_LO_REC_31_0(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_LO_rec_31_0_mask) << SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_LO_rec_31_0_shift)
+
+/*define for REC_ADDR_HI word*/
+/*define for rec_63_32 field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_HI_rec_63_32_offset 11
+#define SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_HI_rec_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_HI_rec_63_32_shift  0
+#define SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_HI_REC_63_32(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_HI_rec_63_32_mask) << SDMA_PKT_POLL_MEM_VERIFY_REC_ADDR_HI_rec_63_32_shift)
+
+/*define for RESERVED word*/
+/*define for reserved field*/
+#define SDMA_PKT_POLL_MEM_VERIFY_RESERVED_reserved_offset 12
+#define SDMA_PKT_POLL_MEM_VERIFY_RESERVED_reserved_mask   0xFFFFFFFF
+#define SDMA_PKT_POLL_MEM_VERIFY_RESERVED_reserved_shift  0
+#define SDMA_PKT_POLL_MEM_VERIFY_RESERVED_RESERVED(x) (((x) & SDMA_PKT_POLL_MEM_VERIFY_RESERVED_reserved_mask) << SDMA_PKT_POLL_MEM_VERIFY_RESERVED_reserved_shift)
+
+
+/*
+** Definitions for SDMA_PKT_ATOMIC packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_ATOMIC_HEADER_op_offset 0
+#define SDMA_PKT_ATOMIC_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_ATOMIC_HEADER_op_shift  0
+#define SDMA_PKT_ATOMIC_HEADER_OP(x) (((x) & SDMA_PKT_ATOMIC_HEADER_op_mask) << SDMA_PKT_ATOMIC_HEADER_op_shift)
+
+/*define for loop field*/
+#define SDMA_PKT_ATOMIC_HEADER_loop_offset 0
+#define SDMA_PKT_ATOMIC_HEADER_loop_mask   0x00000001
+#define SDMA_PKT_ATOMIC_HEADER_loop_shift  16
+#define SDMA_PKT_ATOMIC_HEADER_LOOP(x) (((x) & SDMA_PKT_ATOMIC_HEADER_loop_mask) << SDMA_PKT_ATOMIC_HEADER_loop_shift)
+
+/*define for tmz field*/
+#define SDMA_PKT_ATOMIC_HEADER_tmz_offset 0
+#define SDMA_PKT_ATOMIC_HEADER_tmz_mask   0x00000001
+#define SDMA_PKT_ATOMIC_HEADER_tmz_shift  18
+#define SDMA_PKT_ATOMIC_HEADER_TMZ(x) (((x) & SDMA_PKT_ATOMIC_HEADER_tmz_mask) << SDMA_PKT_ATOMIC_HEADER_tmz_shift)
+
+/*define for cache_policy field*/
+#define SDMA_PKT_ATOMIC_HEADER_cache_policy_offset 0
+#define SDMA_PKT_ATOMIC_HEADER_cache_policy_mask   0x00000007
+#define SDMA_PKT_ATOMIC_HEADER_cache_policy_shift  20
+#define SDMA_PKT_ATOMIC_HEADER_CACHE_POLICY(x) (((x) & SDMA_PKT_ATOMIC_HEADER_cache_policy_mask) << SDMA_PKT_ATOMIC_HEADER_cache_policy_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_ATOMIC_HEADER_cpv_offset 0
+#define SDMA_PKT_ATOMIC_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_ATOMIC_HEADER_cpv_shift  24
+#define SDMA_PKT_ATOMIC_HEADER_CPV(x) (((x) & SDMA_PKT_ATOMIC_HEADER_cpv_mask) << SDMA_PKT_ATOMIC_HEADER_cpv_shift)
+
+/*define for atomic_op field*/
+#define SDMA_PKT_ATOMIC_HEADER_atomic_op_offset 0
+#define SDMA_PKT_ATOMIC_HEADER_atomic_op_mask   0x0000007F
+#define SDMA_PKT_ATOMIC_HEADER_atomic_op_shift  25
+#define SDMA_PKT_ATOMIC_HEADER_ATOMIC_OP(x) (((x) & SDMA_PKT_ATOMIC_HEADER_atomic_op_mask) << SDMA_PKT_ATOMIC_HEADER_atomic_op_shift)
+
+/*define for ADDR_LO word*/
+/*define for addr_31_0 field*/
+#define SDMA_PKT_ATOMIC_ADDR_LO_addr_31_0_offset 1
+#define SDMA_PKT_ATOMIC_ADDR_LO_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_ATOMIC_ADDR_LO_addr_31_0_shift  0
+#define SDMA_PKT_ATOMIC_ADDR_LO_ADDR_31_0(x) (((x) & SDMA_PKT_ATOMIC_ADDR_LO_addr_31_0_mask) << SDMA_PKT_ATOMIC_ADDR_LO_addr_31_0_shift)
+
+/*define for ADDR_HI word*/
+/*define for addr_63_32 field*/
+#define SDMA_PKT_ATOMIC_ADDR_HI_addr_63_32_offset 2
+#define SDMA_PKT_ATOMIC_ADDR_HI_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_ATOMIC_ADDR_HI_addr_63_32_shift  0
+#define SDMA_PKT_ATOMIC_ADDR_HI_ADDR_63_32(x) (((x) & SDMA_PKT_ATOMIC_ADDR_HI_addr_63_32_mask) << SDMA_PKT_ATOMIC_ADDR_HI_addr_63_32_shift)
+
+/*define for SRC_DATA_LO word*/
+/*define for src_data_31_0 field*/
+#define SDMA_PKT_ATOMIC_SRC_DATA_LO_src_data_31_0_offset 3
+#define SDMA_PKT_ATOMIC_SRC_DATA_LO_src_data_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_ATOMIC_SRC_DATA_LO_src_data_31_0_shift  0
+#define SDMA_PKT_ATOMIC_SRC_DATA_LO_SRC_DATA_31_0(x) (((x) & SDMA_PKT_ATOMIC_SRC_DATA_LO_src_data_31_0_mask) << SDMA_PKT_ATOMIC_SRC_DATA_LO_src_data_31_0_shift)
+
+/*define for SRC_DATA_HI word*/
+/*define for src_data_63_32 field*/
+#define SDMA_PKT_ATOMIC_SRC_DATA_HI_src_data_63_32_offset 4
+#define SDMA_PKT_ATOMIC_SRC_DATA_HI_src_data_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_ATOMIC_SRC_DATA_HI_src_data_63_32_shift  0
+#define SDMA_PKT_ATOMIC_SRC_DATA_HI_SRC_DATA_63_32(x) (((x) & SDMA_PKT_ATOMIC_SRC_DATA_HI_src_data_63_32_mask) << SDMA_PKT_ATOMIC_SRC_DATA_HI_src_data_63_32_shift)
+
+/*define for CMP_DATA_LO word*/
+/*define for cmp_data_31_0 field*/
+#define SDMA_PKT_ATOMIC_CMP_DATA_LO_cmp_data_31_0_offset 5
+#define SDMA_PKT_ATOMIC_CMP_DATA_LO_cmp_data_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_ATOMIC_CMP_DATA_LO_cmp_data_31_0_shift  0
+#define SDMA_PKT_ATOMIC_CMP_DATA_LO_CMP_DATA_31_0(x) (((x) & SDMA_PKT_ATOMIC_CMP_DATA_LO_cmp_data_31_0_mask) << SDMA_PKT_ATOMIC_CMP_DATA_LO_cmp_data_31_0_shift)
+
+/*define for CMP_DATA_HI word*/
+/*define for cmp_data_63_32 field*/
+#define SDMA_PKT_ATOMIC_CMP_DATA_HI_cmp_data_63_32_offset 6
+#define SDMA_PKT_ATOMIC_CMP_DATA_HI_cmp_data_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_ATOMIC_CMP_DATA_HI_cmp_data_63_32_shift  0
+#define SDMA_PKT_ATOMIC_CMP_DATA_HI_CMP_DATA_63_32(x) (((x) & SDMA_PKT_ATOMIC_CMP_DATA_HI_cmp_data_63_32_mask) << SDMA_PKT_ATOMIC_CMP_DATA_HI_cmp_data_63_32_shift)
+
+/*define for LOOP_INTERVAL word*/
+/*define for loop_interval field*/
+#define SDMA_PKT_ATOMIC_LOOP_INTERVAL_loop_interval_offset 7
+#define SDMA_PKT_ATOMIC_LOOP_INTERVAL_loop_interval_mask   0x00001FFF
+#define SDMA_PKT_ATOMIC_LOOP_INTERVAL_loop_interval_shift  0
+#define SDMA_PKT_ATOMIC_LOOP_INTERVAL_LOOP_INTERVAL(x) (((x) & SDMA_PKT_ATOMIC_LOOP_INTERVAL_loop_interval_mask) << SDMA_PKT_ATOMIC_LOOP_INTERVAL_loop_interval_shift)
+
+
+/*
+** Definitions for SDMA_PKT_TIMESTAMP_SET packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_TIMESTAMP_SET_HEADER_op_offset 0
+#define SDMA_PKT_TIMESTAMP_SET_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_TIMESTAMP_SET_HEADER_op_shift  0
+#define SDMA_PKT_TIMESTAMP_SET_HEADER_OP(x) (((x) & SDMA_PKT_TIMESTAMP_SET_HEADER_op_mask) << SDMA_PKT_TIMESTAMP_SET_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_TIMESTAMP_SET_HEADER_sub_op_offset 0
+#define SDMA_PKT_TIMESTAMP_SET_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_TIMESTAMP_SET_HEADER_sub_op_shift  8
+#define SDMA_PKT_TIMESTAMP_SET_HEADER_SUB_OP(x) (((x) & SDMA_PKT_TIMESTAMP_SET_HEADER_sub_op_mask) << SDMA_PKT_TIMESTAMP_SET_HEADER_sub_op_shift)
+
+/*define for INIT_DATA_LO word*/
+/*define for init_data_31_0 field*/
+#define SDMA_PKT_TIMESTAMP_SET_INIT_DATA_LO_init_data_31_0_offset 1
+#define SDMA_PKT_TIMESTAMP_SET_INIT_DATA_LO_init_data_31_0_mask   0xFFFFFFFF
+#define SDMA_PKT_TIMESTAMP_SET_INIT_DATA_LO_init_data_31_0_shift  0
+#define SDMA_PKT_TIMESTAMP_SET_INIT_DATA_LO_INIT_DATA_31_0(x) (((x) & SDMA_PKT_TIMESTAMP_SET_INIT_DATA_LO_init_data_31_0_mask) << SDMA_PKT_TIMESTAMP_SET_INIT_DATA_LO_init_data_31_0_shift)
+
+/*define for INIT_DATA_HI word*/
+/*define for init_data_63_32 field*/
+#define SDMA_PKT_TIMESTAMP_SET_INIT_DATA_HI_init_data_63_32_offset 2
+#define SDMA_PKT_TIMESTAMP_SET_INIT_DATA_HI_init_data_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_TIMESTAMP_SET_INIT_DATA_HI_init_data_63_32_shift  0
+#define SDMA_PKT_TIMESTAMP_SET_INIT_DATA_HI_INIT_DATA_63_32(x) (((x) & SDMA_PKT_TIMESTAMP_SET_INIT_DATA_HI_init_data_63_32_mask) << SDMA_PKT_TIMESTAMP_SET_INIT_DATA_HI_init_data_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_TIMESTAMP_GET packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_op_offset 0
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_op_shift  0
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_OP(x) (((x) & SDMA_PKT_TIMESTAMP_GET_HEADER_op_mask) << SDMA_PKT_TIMESTAMP_GET_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_sub_op_offset 0
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_sub_op_shift  8
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_SUB_OP(x) (((x) & SDMA_PKT_TIMESTAMP_GET_HEADER_sub_op_mask) << SDMA_PKT_TIMESTAMP_GET_HEADER_sub_op_shift)
+
+/*define for l2_policy field*/
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_l2_policy_offset 0
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_l2_policy_mask   0x00000003
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_l2_policy_shift  24
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_L2_POLICY(x) (((x) & SDMA_PKT_TIMESTAMP_GET_HEADER_l2_policy_mask) << SDMA_PKT_TIMESTAMP_GET_HEADER_l2_policy_shift)
+
+/*define for llc_policy field*/
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_llc_policy_offset 0
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_llc_policy_mask   0x00000001
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_llc_policy_shift  26
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_LLC_POLICY(x) (((x) & SDMA_PKT_TIMESTAMP_GET_HEADER_llc_policy_mask) << SDMA_PKT_TIMESTAMP_GET_HEADER_llc_policy_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_cpv_offset 0
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_cpv_shift  28
+#define SDMA_PKT_TIMESTAMP_GET_HEADER_CPV(x) (((x) & SDMA_PKT_TIMESTAMP_GET_HEADER_cpv_mask) << SDMA_PKT_TIMESTAMP_GET_HEADER_cpv_shift)
+
+/*define for WRITE_ADDR_LO word*/
+/*define for write_addr_31_3 field*/
+#define SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_LO_write_addr_31_3_offset 1
+#define SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_LO_write_addr_31_3_mask   0x1FFFFFFF
+#define SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_LO_write_addr_31_3_shift  3
+#define SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_LO_WRITE_ADDR_31_3(x) (((x) & SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_LO_write_addr_31_3_mask) << SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_LO_write_addr_31_3_shift)
+
+/*define for WRITE_ADDR_HI word*/
+/*define for write_addr_63_32 field*/
+#define SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_HI_write_addr_63_32_offset 2
+#define SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_HI_write_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_HI_write_addr_63_32_shift  0
+#define SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_HI_WRITE_ADDR_63_32(x) (((x) & SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_HI_write_addr_63_32_mask) << SDMA_PKT_TIMESTAMP_GET_WRITE_ADDR_HI_write_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_TIMESTAMP_GET_GLOBAL packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_op_offset 0
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_op_shift  0
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_OP(x) (((x) & SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_op_mask) << SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_sub_op_offset 0
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_sub_op_shift  8
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_SUB_OP(x) (((x) & SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_sub_op_mask) << SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_sub_op_shift)
+
+/*define for l2_policy field*/
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_l2_policy_offset 0
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_l2_policy_mask   0x00000003
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_l2_policy_shift  24
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_L2_POLICY(x) (((x) & SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_l2_policy_mask) << SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_l2_policy_shift)
+
+/*define for llc_policy field*/
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_llc_policy_offset 0
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_llc_policy_mask   0x00000001
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_llc_policy_shift  26
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_LLC_POLICY(x) (((x) & SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_llc_policy_mask) << SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_llc_policy_shift)
+
+/*define for cpv field*/
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_cpv_offset 0
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_cpv_mask   0x00000001
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_cpv_shift  28
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_CPV(x) (((x) & SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_cpv_mask) << SDMA_PKT_TIMESTAMP_GET_GLOBAL_HEADER_cpv_shift)
+
+/*define for WRITE_ADDR_LO word*/
+/*define for write_addr_31_3 field*/
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_LO_write_addr_31_3_offset 1
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_LO_write_addr_31_3_mask   0x1FFFFFFF
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_LO_write_addr_31_3_shift  3
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_LO_WRITE_ADDR_31_3(x) (((x) & SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_LO_write_addr_31_3_mask) << SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_LO_write_addr_31_3_shift)
+
+/*define for WRITE_ADDR_HI word*/
+/*define for write_addr_63_32 field*/
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_HI_write_addr_63_32_offset 2
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_HI_write_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_HI_write_addr_63_32_shift  0
+#define SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_HI_WRITE_ADDR_63_32(x) (((x) & SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_HI_write_addr_63_32_mask) << SDMA_PKT_TIMESTAMP_GET_GLOBAL_WRITE_ADDR_HI_write_addr_63_32_shift)
+
+
+/*
+** Definitions for SDMA_PKT_TRAP packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_TRAP_HEADER_op_offset 0
+#define SDMA_PKT_TRAP_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_TRAP_HEADER_op_shift  0
+#define SDMA_PKT_TRAP_HEADER_OP(x) (((x) & SDMA_PKT_TRAP_HEADER_op_mask) << SDMA_PKT_TRAP_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_TRAP_HEADER_sub_op_offset 0
+#define SDMA_PKT_TRAP_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_TRAP_HEADER_sub_op_shift  8
+#define SDMA_PKT_TRAP_HEADER_SUB_OP(x) (((x) & SDMA_PKT_TRAP_HEADER_sub_op_mask) << SDMA_PKT_TRAP_HEADER_sub_op_shift)
+
+/*define for INT_CONTEXT word*/
+/*define for int_context field*/
+#define SDMA_PKT_TRAP_INT_CONTEXT_int_context_offset 1
+#define SDMA_PKT_TRAP_INT_CONTEXT_int_context_mask   0x0FFFFFFF
+#define SDMA_PKT_TRAP_INT_CONTEXT_int_context_shift  0
+#define SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(x) (((x) & SDMA_PKT_TRAP_INT_CONTEXT_int_context_mask) << SDMA_PKT_TRAP_INT_CONTEXT_int_context_shift)
+
+
+/*
+** Definitions for SDMA_PKT_DUMMY_TRAP packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_DUMMY_TRAP_HEADER_op_offset 0
+#define SDMA_PKT_DUMMY_TRAP_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_DUMMY_TRAP_HEADER_op_shift  0
+#define SDMA_PKT_DUMMY_TRAP_HEADER_OP(x) (((x) & SDMA_PKT_DUMMY_TRAP_HEADER_op_mask) << SDMA_PKT_DUMMY_TRAP_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_DUMMY_TRAP_HEADER_sub_op_offset 0
+#define SDMA_PKT_DUMMY_TRAP_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_DUMMY_TRAP_HEADER_sub_op_shift  8
+#define SDMA_PKT_DUMMY_TRAP_HEADER_SUB_OP(x) (((x) & SDMA_PKT_DUMMY_TRAP_HEADER_sub_op_mask) << SDMA_PKT_DUMMY_TRAP_HEADER_sub_op_shift)
+
+/*define for INT_CONTEXT word*/
+/*define for int_context field*/
+#define SDMA_PKT_DUMMY_TRAP_INT_CONTEXT_int_context_offset 1
+#define SDMA_PKT_DUMMY_TRAP_INT_CONTEXT_int_context_mask   0x0FFFFFFF
+#define SDMA_PKT_DUMMY_TRAP_INT_CONTEXT_int_context_shift  0
+#define SDMA_PKT_DUMMY_TRAP_INT_CONTEXT_INT_CONTEXT(x) (((x) & SDMA_PKT_DUMMY_TRAP_INT_CONTEXT_int_context_mask) << SDMA_PKT_DUMMY_TRAP_INT_CONTEXT_int_context_shift)
+
+
+/*
+** Definitions for SDMA_PKT_GPUVM_INV packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_GPUVM_INV_HEADER_op_offset 0
+#define SDMA_PKT_GPUVM_INV_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_GPUVM_INV_HEADER_op_shift  0
+#define SDMA_PKT_GPUVM_INV_HEADER_OP(x) (((x) & SDMA_PKT_GPUVM_INV_HEADER_op_mask) << SDMA_PKT_GPUVM_INV_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_GPUVM_INV_HEADER_sub_op_offset 0
+#define SDMA_PKT_GPUVM_INV_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_GPUVM_INV_HEADER_sub_op_shift  8
+#define SDMA_PKT_GPUVM_INV_HEADER_SUB_OP(x) (((x) & SDMA_PKT_GPUVM_INV_HEADER_sub_op_mask) << SDMA_PKT_GPUVM_INV_HEADER_sub_op_shift)
+
+/*define for PAYLOAD1 word*/
+/*define for per_vmid_inv_req field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_per_vmid_inv_req_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_per_vmid_inv_req_mask   0x0000FFFF
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_per_vmid_inv_req_shift  0
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_PER_VMID_INV_REQ(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_per_vmid_inv_req_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_per_vmid_inv_req_shift)
+
+/*define for flush_type field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_flush_type_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_flush_type_mask   0x00000007
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_flush_type_shift  16
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_FLUSH_TYPE(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_flush_type_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_flush_type_shift)
+
+/*define for l2_ptes field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_ptes_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_ptes_mask   0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_ptes_shift  19
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_L2_PTES(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_ptes_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_ptes_shift)
+
+/*define for l2_pde0 field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde0_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde0_mask   0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde0_shift  20
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_L2_PDE0(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde0_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde0_shift)
+
+/*define for l2_pde1 field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde1_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde1_mask   0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde1_shift  21
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_L2_PDE1(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde1_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde1_shift)
+
+/*define for l2_pde2 field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde2_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde2_mask   0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde2_shift  22
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_L2_PDE2(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde2_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_l2_pde2_shift)
+
+/*define for l1_ptes field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l1_ptes_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l1_ptes_mask   0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_l1_ptes_shift  23
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_L1_PTES(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_l1_ptes_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_l1_ptes_shift)
+
+/*define for clr_protection_fault_status_addr field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_clr_protection_fault_status_addr_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_clr_protection_fault_status_addr_mask   0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_clr_protection_fault_status_addr_shift  24
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_CLR_PROTECTION_FAULT_STATUS_ADDR(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_clr_protection_fault_status_addr_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_clr_protection_fault_status_addr_shift)
+
+/*define for log_request field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_log_request_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_log_request_mask   0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_log_request_shift  25
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_LOG_REQUEST(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_log_request_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_log_request_shift)
+
+/*define for four_kilobytes field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_four_kilobytes_offset 1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_four_kilobytes_mask   0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_four_kilobytes_shift  26
+#define SDMA_PKT_GPUVM_INV_PAYLOAD1_FOUR_KILOBYTES(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD1_four_kilobytes_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD1_four_kilobytes_shift)
+
+/*define for PAYLOAD2 word*/
+/*define for s field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD2_s_offset 2
+#define SDMA_PKT_GPUVM_INV_PAYLOAD2_s_mask   0x00000001
+#define SDMA_PKT_GPUVM_INV_PAYLOAD2_s_shift  0
+#define SDMA_PKT_GPUVM_INV_PAYLOAD2_S(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD2_s_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD2_s_shift)
+
+/*define for page_va_42_12 field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD2_page_va_42_12_offset 2
+#define SDMA_PKT_GPUVM_INV_PAYLOAD2_page_va_42_12_mask   0x7FFFFFFF
+#define SDMA_PKT_GPUVM_INV_PAYLOAD2_page_va_42_12_shift  1
+#define SDMA_PKT_GPUVM_INV_PAYLOAD2_PAGE_VA_42_12(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD2_page_va_42_12_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD2_page_va_42_12_shift)
+
+/*define for PAYLOAD3 word*/
+/*define for page_va_47_43 field*/
+#define SDMA_PKT_GPUVM_INV_PAYLOAD3_page_va_47_43_offset 3
+#define SDMA_PKT_GPUVM_INV_PAYLOAD3_page_va_47_43_mask   0x0000003F
+#define SDMA_PKT_GPUVM_INV_PAYLOAD3_page_va_47_43_shift  0
+#define SDMA_PKT_GPUVM_INV_PAYLOAD3_PAGE_VA_47_43(x) (((x) & SDMA_PKT_GPUVM_INV_PAYLOAD3_page_va_47_43_mask) << SDMA_PKT_GPUVM_INV_PAYLOAD3_page_va_47_43_shift)
+
+
+/*
+** Definitions for SDMA_PKT_GCR_REQ packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_GCR_REQ_HEADER_op_offset 0
+#define SDMA_PKT_GCR_REQ_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_GCR_REQ_HEADER_op_shift  0
+#define SDMA_PKT_GCR_REQ_HEADER_OP(x) (((x) & SDMA_PKT_GCR_REQ_HEADER_op_mask) << SDMA_PKT_GCR_REQ_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_GCR_REQ_HEADER_sub_op_offset 0
+#define SDMA_PKT_GCR_REQ_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_GCR_REQ_HEADER_sub_op_shift  8
+#define SDMA_PKT_GCR_REQ_HEADER_SUB_OP(x) (((x) & SDMA_PKT_GCR_REQ_HEADER_sub_op_mask) << SDMA_PKT_GCR_REQ_HEADER_sub_op_shift)
+
+/*define for PAYLOAD1 word*/
+/*define for base_va_31_7 field*/
+#define SDMA_PKT_GCR_REQ_PAYLOAD1_base_va_31_7_offset 1
+#define SDMA_PKT_GCR_REQ_PAYLOAD1_base_va_31_7_mask   0x01FFFFFF
+#define SDMA_PKT_GCR_REQ_PAYLOAD1_base_va_31_7_shift  7
+#define SDMA_PKT_GCR_REQ_PAYLOAD1_BASE_VA_31_7(x) (((x) & SDMA_PKT_GCR_REQ_PAYLOAD1_base_va_31_7_mask) << SDMA_PKT_GCR_REQ_PAYLOAD1_base_va_31_7_shift)
+
+/*define for PAYLOAD2 word*/
+/*define for base_va_56_32 field*/
+#define SDMA_PKT_GCR_REQ_PAYLOAD2_base_va_56_32_offset 2
+#define SDMA_PKT_GCR_REQ_PAYLOAD2_base_va_56_32_mask   0x00FFFFFF
+#define SDMA_PKT_GCR_REQ_PAYLOAD2_base_va_56_32_shift  0
+#define SDMA_PKT_GCR_REQ_PAYLOAD2_BASE_VA_56_32(x) (((x) & SDMA_PKT_GCR_REQ_PAYLOAD2_base_va_56_32_mask) << SDMA_PKT_GCR_REQ_PAYLOAD2_base_va_56_32_shift)
+
+/*define for PAYLOAD3 word*/
+/*define for gcr_control_18_0 field*/
+#define SDMA_PKT_GCR_REQ_PAYLOAD3_gcr_control_18_0_offset 3
+#define SDMA_PKT_GCR_REQ_PAYLOAD3_gcr_control_18_0_mask   0x0007FFFF
+#define SDMA_PKT_GCR_REQ_PAYLOAD3_gcr_control_18_0_shift  0
+#define SDMA_PKT_GCR_REQ_PAYLOAD3_GCR_CONTROL_18_0(x) (((x) & SDMA_PKT_GCR_REQ_PAYLOAD3_gcr_control_18_0_mask) << SDMA_PKT_GCR_REQ_PAYLOAD3_gcr_control_18_0_shift)
+
+/*define for limit_va_15_7 field*/
+#define SDMA_PKT_GCR_REQ_PAYLOAD3_limit_va_15_7_offset 3
+#define SDMA_PKT_GCR_REQ_PAYLOAD3_limit_va_15_7_mask   0x000001FF
+#define SDMA_PKT_GCR_REQ_PAYLOAD3_limit_va_15_7_shift  23
+#define SDMA_PKT_GCR_REQ_PAYLOAD3_LIMIT_VA_15_7(x) (((x) & SDMA_PKT_GCR_REQ_PAYLOAD3_limit_va_15_7_mask) << SDMA_PKT_GCR_REQ_PAYLOAD3_limit_va_15_7_shift)
+
+/*define for PAYLOAD4 word*/
+/*define for limit_va_47_16 field*/
+#define SDMA_PKT_GCR_REQ_PAYLOAD4_limit_va_47_16_offset 4
+#define SDMA_PKT_GCR_REQ_PAYLOAD4_limit_va_47_16_mask   0xFFFFFFFF
+#define SDMA_PKT_GCR_REQ_PAYLOAD4_limit_va_47_16_shift  0
+#define SDMA_PKT_GCR_REQ_PAYLOAD4_LIMIT_VA_47_16(x) (((x) & SDMA_PKT_GCR_REQ_PAYLOAD4_limit_va_47_16_mask) << SDMA_PKT_GCR_REQ_PAYLOAD4_limit_va_47_16_shift)
+
+/*define for PAYLOAD5 word*/
+/*define for limit_va_56_48 field*/
+#define SDMA_PKT_GCR_REQ_PAYLOAD5_limit_va_56_48_offset 5
+#define SDMA_PKT_GCR_REQ_PAYLOAD5_limit_va_56_48_mask   0x000001FF
+#define SDMA_PKT_GCR_REQ_PAYLOAD5_limit_va_56_48_shift  0
+#define SDMA_PKT_GCR_REQ_PAYLOAD5_LIMIT_VA_56_48(x) (((x) & SDMA_PKT_GCR_REQ_PAYLOAD5_limit_va_56_48_mask) << SDMA_PKT_GCR_REQ_PAYLOAD5_limit_va_56_48_shift)
+
+/*define for vmid field*/
+#define SDMA_PKT_GCR_REQ_PAYLOAD5_vmid_offset 5
+#define SDMA_PKT_GCR_REQ_PAYLOAD5_vmid_mask   0x0000000F
+#define SDMA_PKT_GCR_REQ_PAYLOAD5_vmid_shift  26
+#define SDMA_PKT_GCR_REQ_PAYLOAD5_VMID(x) (((x) & SDMA_PKT_GCR_REQ_PAYLOAD5_vmid_mask) << SDMA_PKT_GCR_REQ_PAYLOAD5_vmid_shift)
+
+
+/*
+** Definitions for SDMA_PKT_NOP packet
+*/
+
+/*define for HEADER word*/
+/*define for op field*/
+#define SDMA_PKT_NOP_HEADER_op_offset 0
+#define SDMA_PKT_NOP_HEADER_op_mask   0x000000FF
+#define SDMA_PKT_NOP_HEADER_op_shift  0
+#define SDMA_PKT_NOP_HEADER_OP(x) (((x) & SDMA_PKT_NOP_HEADER_op_mask) << SDMA_PKT_NOP_HEADER_op_shift)
+
+/*define for sub_op field*/
+#define SDMA_PKT_NOP_HEADER_sub_op_offset 0
+#define SDMA_PKT_NOP_HEADER_sub_op_mask   0x000000FF
+#define SDMA_PKT_NOP_HEADER_sub_op_shift  8
+#define SDMA_PKT_NOP_HEADER_SUB_OP(x) (((x) & SDMA_PKT_NOP_HEADER_sub_op_mask) << SDMA_PKT_NOP_HEADER_sub_op_shift)
+
+/*define for count field*/
+#define SDMA_PKT_NOP_HEADER_count_offset 0
+#define SDMA_PKT_NOP_HEADER_count_mask   0x00003FFF
+#define SDMA_PKT_NOP_HEADER_count_shift  16
+#define SDMA_PKT_NOP_HEADER_COUNT(x) (((x) & SDMA_PKT_NOP_HEADER_count_mask) << SDMA_PKT_NOP_HEADER_count_shift)
+
+/*define for DATA0 word*/
+/*define for data0 field*/
+#define SDMA_PKT_NOP_DATA0_data0_offset 1
+#define SDMA_PKT_NOP_DATA0_data0_mask   0xFFFFFFFF
+#define SDMA_PKT_NOP_DATA0_data0_shift  0
+#define SDMA_PKT_NOP_DATA0_DATA0(x) (((x) & SDMA_PKT_NOP_DATA0_data0_mask) << SDMA_PKT_NOP_DATA0_data0_shift)
+
+
+/*
+** Definitions for SDMA_AQL_PKT_HEADER packet
+*/
+
+/*define for HEADER word*/
+/*define for format field*/
+#define SDMA_AQL_PKT_HEADER_HEADER_format_offset 0
+#define SDMA_AQL_PKT_HEADER_HEADER_format_mask   0x000000FF
+#define SDMA_AQL_PKT_HEADER_HEADER_format_shift  0
+#define SDMA_AQL_PKT_HEADER_HEADER_FORMAT(x) (((x) & SDMA_AQL_PKT_HEADER_HEADER_format_mask) << SDMA_AQL_PKT_HEADER_HEADER_format_shift)
+
+/*define for barrier field*/
+#define SDMA_AQL_PKT_HEADER_HEADER_barrier_offset 0
+#define SDMA_AQL_PKT_HEADER_HEADER_barrier_mask   0x00000001
+#define SDMA_AQL_PKT_HEADER_HEADER_barrier_shift  8
+#define SDMA_AQL_PKT_HEADER_HEADER_BARRIER(x) (((x) & SDMA_AQL_PKT_HEADER_HEADER_barrier_mask) << SDMA_AQL_PKT_HEADER_HEADER_barrier_shift)
+
+/*define for acquire_fence_scope field*/
+#define SDMA_AQL_PKT_HEADER_HEADER_acquire_fence_scope_offset 0
+#define SDMA_AQL_PKT_HEADER_HEADER_acquire_fence_scope_mask   0x00000003
+#define SDMA_AQL_PKT_HEADER_HEADER_acquire_fence_scope_shift  9
+#define SDMA_AQL_PKT_HEADER_HEADER_ACQUIRE_FENCE_SCOPE(x) (((x) & SDMA_AQL_PKT_HEADER_HEADER_acquire_fence_scope_mask) << SDMA_AQL_PKT_HEADER_HEADER_acquire_fence_scope_shift)
+
+/*define for release_fence_scope field*/
+#define SDMA_AQL_PKT_HEADER_HEADER_release_fence_scope_offset 0
+#define SDMA_AQL_PKT_HEADER_HEADER_release_fence_scope_mask   0x00000003
+#define SDMA_AQL_PKT_HEADER_HEADER_release_fence_scope_shift  11
+#define SDMA_AQL_PKT_HEADER_HEADER_RELEASE_FENCE_SCOPE(x) (((x) & SDMA_AQL_PKT_HEADER_HEADER_release_fence_scope_mask) << SDMA_AQL_PKT_HEADER_HEADER_release_fence_scope_shift)
+
+/*define for reserved field*/
+#define SDMA_AQL_PKT_HEADER_HEADER_reserved_offset 0
+#define SDMA_AQL_PKT_HEADER_HEADER_reserved_mask   0x00000007
+#define SDMA_AQL_PKT_HEADER_HEADER_reserved_shift  13
+#define SDMA_AQL_PKT_HEADER_HEADER_RESERVED(x) (((x) & SDMA_AQL_PKT_HEADER_HEADER_reserved_mask) << SDMA_AQL_PKT_HEADER_HEADER_reserved_shift)
+
+/*define for op field*/
+#define SDMA_AQL_PKT_HEADER_HEADER_op_offset 0
+#define SDMA_AQL_PKT_HEADER_HEADER_op_mask   0x0000000F
+#define SDMA_AQL_PKT_HEADER_HEADER_op_shift  16
+#define SDMA_AQL_PKT_HEADER_HEADER_OP(x) (((x) & SDMA_AQL_PKT_HEADER_HEADER_op_mask) << SDMA_AQL_PKT_HEADER_HEADER_op_shift)
+
+/*define for subop field*/
+#define SDMA_AQL_PKT_HEADER_HEADER_subop_offset 0
+#define SDMA_AQL_PKT_HEADER_HEADER_subop_mask   0x00000007
+#define SDMA_AQL_PKT_HEADER_HEADER_subop_shift  20
+#define SDMA_AQL_PKT_HEADER_HEADER_SUBOP(x) (((x) & SDMA_AQL_PKT_HEADER_HEADER_subop_mask) << SDMA_AQL_PKT_HEADER_HEADER_subop_shift)
+
+/*define for cpv field*/
+#define SDMA_AQL_PKT_HEADER_HEADER_cpv_offset 0
+#define SDMA_AQL_PKT_HEADER_HEADER_cpv_mask   0x00000001
+#define SDMA_AQL_PKT_HEADER_HEADER_cpv_shift  28
+#define SDMA_AQL_PKT_HEADER_HEADER_CPV(x) (((x) & SDMA_AQL_PKT_HEADER_HEADER_cpv_mask) << SDMA_AQL_PKT_HEADER_HEADER_cpv_shift)
+
+
+/*
+** Definitions for SDMA_AQL_PKT_COPY_LINEAR packet
+*/
+
+/*define for HEADER word*/
+/*define for format field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_format_offset 0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_format_mask   0x000000FF
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_format_shift  0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_FORMAT(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_HEADER_format_mask) << SDMA_AQL_PKT_COPY_LINEAR_HEADER_format_shift)
+
+/*define for barrier field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_barrier_offset 0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_barrier_mask   0x00000001
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_barrier_shift  8
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_BARRIER(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_HEADER_barrier_mask) << SDMA_AQL_PKT_COPY_LINEAR_HEADER_barrier_shift)
+
+/*define for acquire_fence_scope field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_acquire_fence_scope_offset 0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_acquire_fence_scope_mask   0x00000003
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_acquire_fence_scope_shift  9
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_ACQUIRE_FENCE_SCOPE(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_HEADER_acquire_fence_scope_mask) << SDMA_AQL_PKT_COPY_LINEAR_HEADER_acquire_fence_scope_shift)
+
+/*define for release_fence_scope field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_release_fence_scope_offset 0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_release_fence_scope_mask   0x00000003
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_release_fence_scope_shift  11
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_RELEASE_FENCE_SCOPE(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_HEADER_release_fence_scope_mask) << SDMA_AQL_PKT_COPY_LINEAR_HEADER_release_fence_scope_shift)
+
+/*define for reserved field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_reserved_offset 0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_reserved_mask   0x00000007
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_reserved_shift  13
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_RESERVED(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_HEADER_reserved_mask) << SDMA_AQL_PKT_COPY_LINEAR_HEADER_reserved_shift)
+
+/*define for op field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_op_offset 0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_op_mask   0x0000000F
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_op_shift  16
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_OP(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_HEADER_op_mask) << SDMA_AQL_PKT_COPY_LINEAR_HEADER_op_shift)
+
+/*define for subop field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_subop_offset 0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_subop_mask   0x00000007
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_subop_shift  20
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_SUBOP(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_HEADER_subop_mask) << SDMA_AQL_PKT_COPY_LINEAR_HEADER_subop_shift)
+
+/*define for cpv field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_cpv_offset 0
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_cpv_mask   0x00000001
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_cpv_shift  28
+#define SDMA_AQL_PKT_COPY_LINEAR_HEADER_CPV(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_HEADER_cpv_mask) << SDMA_AQL_PKT_COPY_LINEAR_HEADER_cpv_shift)
+
+/*define for RESERVED_DW1 word*/
+/*define for reserved_dw1 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW1_reserved_dw1_offset 1
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW1_reserved_dw1_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW1_reserved_dw1_shift  0
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW1_RESERVED_DW1(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW1_reserved_dw1_mask) << SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW1_reserved_dw1_shift)
+
+/*define for RETURN_ADDR_LO word*/
+/*define for return_addr_31_0 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_LO_return_addr_31_0_offset 2
+#define SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_LO_return_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_LO_return_addr_31_0_shift  0
+#define SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_LO_RETURN_ADDR_31_0(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_LO_return_addr_31_0_mask) << SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_LO_return_addr_31_0_shift)
+
+/*define for RETURN_ADDR_HI word*/
+/*define for return_addr_63_32 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_HI_return_addr_63_32_offset 3
+#define SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_HI_return_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_HI_return_addr_63_32_shift  0
+#define SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_HI_RETURN_ADDR_63_32(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_HI_return_addr_63_32_mask) << SDMA_AQL_PKT_COPY_LINEAR_RETURN_ADDR_HI_return_addr_63_32_shift)
+
+/*define for COUNT word*/
+/*define for count field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_COUNT_count_offset 4
+#define SDMA_AQL_PKT_COPY_LINEAR_COUNT_count_mask   0x003FFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_COUNT_count_shift  0
+#define SDMA_AQL_PKT_COPY_LINEAR_COUNT_COUNT(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_COUNT_count_mask) << SDMA_AQL_PKT_COPY_LINEAR_COUNT_count_shift)
+
+/*define for PARAMETER word*/
+/*define for dst_sw field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_dst_sw_offset 5
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_dst_sw_mask   0x00000003
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_dst_sw_shift  16
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_DST_SW(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_dst_sw_mask) << SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_dst_sw_shift)
+
+/*define for dst_cache_policy field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_dst_cache_policy_offset 5
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_dst_cache_policy_mask   0x00000007
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_dst_cache_policy_shift  18
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_DST_CACHE_POLICY(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_dst_cache_policy_mask) << SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_dst_cache_policy_shift)
+
+/*define for src_sw field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_src_sw_offset 5
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_src_sw_mask   0x00000003
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_src_sw_shift  24
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_SRC_SW(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_src_sw_mask) << SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_src_sw_shift)
+
+/*define for src_cache_policy field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_src_cache_policy_offset 5
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_src_cache_policy_mask   0x00000007
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_src_cache_policy_shift  26
+#define SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_SRC_CACHE_POLICY(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_src_cache_policy_mask) << SDMA_AQL_PKT_COPY_LINEAR_PARAMETER_src_cache_policy_shift)
+
+/*define for SRC_ADDR_LO word*/
+/*define for src_addr_31_0 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_offset 6
+#define SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_shift  0
+#define SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_LO_SRC_ADDR_31_0(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_mask) << SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_LO_src_addr_31_0_shift)
+
+/*define for SRC_ADDR_HI word*/
+/*define for src_addr_63_32 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_offset 7
+#define SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_shift  0
+#define SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_HI_SRC_ADDR_63_32(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_mask) << SDMA_AQL_PKT_COPY_LINEAR_SRC_ADDR_HI_src_addr_63_32_shift)
+
+/*define for DST_ADDR_LO word*/
+/*define for dst_addr_31_0 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_offset 8
+#define SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_shift  0
+#define SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_LO_DST_ADDR_31_0(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_mask) << SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_LO_dst_addr_31_0_shift)
+
+/*define for DST_ADDR_HI word*/
+/*define for dst_addr_63_32 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_offset 9
+#define SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_shift  0
+#define SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_HI_DST_ADDR_63_32(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_mask) << SDMA_AQL_PKT_COPY_LINEAR_DST_ADDR_HI_dst_addr_63_32_shift)
+
+/*define for RESERVED_DW10 word*/
+/*define for reserved_dw10 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW10_reserved_dw10_offset 10
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW10_reserved_dw10_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW10_reserved_dw10_shift  0
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW10_RESERVED_DW10(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW10_reserved_dw10_mask) << SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW10_reserved_dw10_shift)
+
+/*define for RESERVED_DW11 word*/
+/*define for reserved_dw11 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW11_reserved_dw11_offset 11
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW11_reserved_dw11_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW11_reserved_dw11_shift  0
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW11_RESERVED_DW11(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW11_reserved_dw11_mask) << SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW11_reserved_dw11_shift)
+
+/*define for RESERVED_DW12 word*/
+/*define for reserved_dw12 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW12_reserved_dw12_offset 12
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW12_reserved_dw12_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW12_reserved_dw12_shift  0
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW12_RESERVED_DW12(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW12_reserved_dw12_mask) << SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW12_reserved_dw12_shift)
+
+/*define for RESERVED_DW13 word*/
+/*define for reserved_dw13 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW13_reserved_dw13_offset 13
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW13_reserved_dw13_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW13_reserved_dw13_shift  0
+#define SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW13_RESERVED_DW13(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW13_reserved_dw13_mask) << SDMA_AQL_PKT_COPY_LINEAR_RESERVED_DW13_reserved_dw13_shift)
+
+/*define for COMPLETION_SIGNAL_LO word*/
+/*define for completion_signal_31_0 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_LO_completion_signal_31_0_offset 14
+#define SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_LO_completion_signal_31_0_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_LO_completion_signal_31_0_shift  0
+#define SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_LO_COMPLETION_SIGNAL_31_0(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_LO_completion_signal_31_0_mask) << SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_LO_completion_signal_31_0_shift)
+
+/*define for COMPLETION_SIGNAL_HI word*/
+/*define for completion_signal_63_32 field*/
+#define SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_HI_completion_signal_63_32_offset 15
+#define SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_HI_completion_signal_63_32_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_HI_completion_signal_63_32_shift  0
+#define SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_HI_COMPLETION_SIGNAL_63_32(x) (((x) & SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_HI_completion_signal_63_32_mask) << SDMA_AQL_PKT_COPY_LINEAR_COMPLETION_SIGNAL_HI_completion_signal_63_32_shift)
+
+
+/*
+** Definitions for SDMA_AQL_PKT_BARRIER_OR packet
+*/
+
+/*define for HEADER word*/
+/*define for format field*/
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_format_offset 0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_format_mask   0x000000FF
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_format_shift  0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_FORMAT(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_HEADER_format_mask) << SDMA_AQL_PKT_BARRIER_OR_HEADER_format_shift)
+
+/*define for barrier field*/
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_barrier_offset 0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_barrier_mask   0x00000001
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_barrier_shift  8
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_BARRIER(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_HEADER_barrier_mask) << SDMA_AQL_PKT_BARRIER_OR_HEADER_barrier_shift)
+
+/*define for acquire_fence_scope field*/
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_acquire_fence_scope_offset 0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_acquire_fence_scope_mask   0x00000003
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_acquire_fence_scope_shift  9
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_ACQUIRE_FENCE_SCOPE(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_HEADER_acquire_fence_scope_mask) << SDMA_AQL_PKT_BARRIER_OR_HEADER_acquire_fence_scope_shift)
+
+/*define for release_fence_scope field*/
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_release_fence_scope_offset 0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_release_fence_scope_mask   0x00000003
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_release_fence_scope_shift  11
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_RELEASE_FENCE_SCOPE(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_HEADER_release_fence_scope_mask) << SDMA_AQL_PKT_BARRIER_OR_HEADER_release_fence_scope_shift)
+
+/*define for reserved field*/
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_reserved_offset 0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_reserved_mask   0x00000007
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_reserved_shift  13
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_RESERVED(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_HEADER_reserved_mask) << SDMA_AQL_PKT_BARRIER_OR_HEADER_reserved_shift)
+
+/*define for op field*/
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_op_offset 0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_op_mask   0x0000000F
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_op_shift  16
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_OP(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_HEADER_op_mask) << SDMA_AQL_PKT_BARRIER_OR_HEADER_op_shift)
+
+/*define for subop field*/
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_subop_offset 0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_subop_mask   0x00000007
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_subop_shift  20
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_SUBOP(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_HEADER_subop_mask) << SDMA_AQL_PKT_BARRIER_OR_HEADER_subop_shift)
+
+/*define for cpv field*/
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_cpv_offset 0
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_cpv_mask   0x00000001
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_cpv_shift  28
+#define SDMA_AQL_PKT_BARRIER_OR_HEADER_CPV(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_HEADER_cpv_mask) << SDMA_AQL_PKT_BARRIER_OR_HEADER_cpv_shift)
+
+/*define for RESERVED_DW1 word*/
+/*define for reserved_dw1 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW1_reserved_dw1_offset 1
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW1_reserved_dw1_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW1_reserved_dw1_shift  0
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW1_RESERVED_DW1(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW1_reserved_dw1_mask) << SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW1_reserved_dw1_shift)
+
+/*define for DEPENDENT_ADDR_0_LO word*/
+/*define for dependent_addr_0_31_0 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_LO_dependent_addr_0_31_0_offset 2
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_LO_dependent_addr_0_31_0_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_LO_dependent_addr_0_31_0_shift  0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_LO_DEPENDENT_ADDR_0_31_0(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_LO_dependent_addr_0_31_0_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_LO_dependent_addr_0_31_0_shift)
+
+/*define for DEPENDENT_ADDR_0_HI word*/
+/*define for dependent_addr_0_63_32 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_HI_dependent_addr_0_63_32_offset 3
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_HI_dependent_addr_0_63_32_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_HI_dependent_addr_0_63_32_shift  0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_HI_DEPENDENT_ADDR_0_63_32(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_HI_dependent_addr_0_63_32_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_0_HI_dependent_addr_0_63_32_shift)
+
+/*define for DEPENDENT_ADDR_1_LO word*/
+/*define for dependent_addr_1_31_0 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_LO_dependent_addr_1_31_0_offset 4
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_LO_dependent_addr_1_31_0_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_LO_dependent_addr_1_31_0_shift  0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_LO_DEPENDENT_ADDR_1_31_0(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_LO_dependent_addr_1_31_0_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_LO_dependent_addr_1_31_0_shift)
+
+/*define for DEPENDENT_ADDR_1_HI word*/
+/*define for dependent_addr_1_63_32 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_HI_dependent_addr_1_63_32_offset 5
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_HI_dependent_addr_1_63_32_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_HI_dependent_addr_1_63_32_shift  0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_HI_DEPENDENT_ADDR_1_63_32(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_HI_dependent_addr_1_63_32_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_1_HI_dependent_addr_1_63_32_shift)
+
+/*define for DEPENDENT_ADDR_2_LO word*/
+/*define for dependent_addr_2_31_0 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_LO_dependent_addr_2_31_0_offset 6
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_LO_dependent_addr_2_31_0_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_LO_dependent_addr_2_31_0_shift  0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_LO_DEPENDENT_ADDR_2_31_0(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_LO_dependent_addr_2_31_0_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_LO_dependent_addr_2_31_0_shift)
+
+/*define for DEPENDENT_ADDR_2_HI word*/
+/*define for dependent_addr_2_63_32 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_HI_dependent_addr_2_63_32_offset 7
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_HI_dependent_addr_2_63_32_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_HI_dependent_addr_2_63_32_shift  0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_HI_DEPENDENT_ADDR_2_63_32(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_HI_dependent_addr_2_63_32_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_2_HI_dependent_addr_2_63_32_shift)
+
+/*define for DEPENDENT_ADDR_3_LO word*/
+/*define for dependent_addr_3_31_0 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_LO_dependent_addr_3_31_0_offset 8
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_LO_dependent_addr_3_31_0_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_LO_dependent_addr_3_31_0_shift  0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_LO_DEPENDENT_ADDR_3_31_0(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_LO_dependent_addr_3_31_0_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_LO_dependent_addr_3_31_0_shift)
+
+/*define for DEPENDENT_ADDR_3_HI word*/
+/*define for dependent_addr_3_63_32 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_HI_dependent_addr_3_63_32_offset 9
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_HI_dependent_addr_3_63_32_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_HI_dependent_addr_3_63_32_shift  0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_HI_DEPENDENT_ADDR_3_63_32(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_HI_dependent_addr_3_63_32_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_3_HI_dependent_addr_3_63_32_shift)
+
+/*define for DEPENDENT_ADDR_4_LO word*/
+/*define for dependent_addr_4_31_0 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_LO_dependent_addr_4_31_0_offset 10
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_LO_dependent_addr_4_31_0_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_LO_dependent_addr_4_31_0_shift  0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_LO_DEPENDENT_ADDR_4_31_0(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_LO_dependent_addr_4_31_0_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_LO_dependent_addr_4_31_0_shift)
+
+/*define for DEPENDENT_ADDR_4_HI word*/
+/*define for dependent_addr_4_63_32 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_HI_dependent_addr_4_63_32_offset 11
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_HI_dependent_addr_4_63_32_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_HI_dependent_addr_4_63_32_shift  0
+#define SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_HI_DEPENDENT_ADDR_4_63_32(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_HI_dependent_addr_4_63_32_mask) << SDMA_AQL_PKT_BARRIER_OR_DEPENDENT_ADDR_4_HI_dependent_addr_4_63_32_shift)
+
+/*define for CACHE_POLICY word*/
+/*define for cache_policy0 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy0_offset 12
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy0_mask   0x00000007
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy0_shift  0
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_CACHE_POLICY0(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy0_mask) << SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy0_shift)
+
+/*define for cache_policy1 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy1_offset 12
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy1_mask   0x00000007
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy1_shift  5
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_CACHE_POLICY1(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy1_mask) << SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy1_shift)
+
+/*define for cache_policy2 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy2_offset 12
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy2_mask   0x00000007
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy2_shift  10
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_CACHE_POLICY2(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy2_mask) << SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy2_shift)
+
+/*define for cache_policy3 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy3_offset 12
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy3_mask   0x00000007
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy3_shift  15
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_CACHE_POLICY3(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy3_mask) << SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy3_shift)
+
+/*define for cache_policy4 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy4_offset 12
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy4_mask   0x00000007
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy4_shift  20
+#define SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_CACHE_POLICY4(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy4_mask) << SDMA_AQL_PKT_BARRIER_OR_CACHE_POLICY_cache_policy4_shift)
+
+/*define for RESERVED_DW13 word*/
+/*define for reserved_dw13 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW13_reserved_dw13_offset 13
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW13_reserved_dw13_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW13_reserved_dw13_shift  0
+#define SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW13_RESERVED_DW13(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW13_reserved_dw13_mask) << SDMA_AQL_PKT_BARRIER_OR_RESERVED_DW13_reserved_dw13_shift)
+
+/*define for COMPLETION_SIGNAL_LO word*/
+/*define for completion_signal_31_0 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_LO_completion_signal_31_0_offset 14
+#define SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_LO_completion_signal_31_0_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_LO_completion_signal_31_0_shift  0
+#define SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_LO_COMPLETION_SIGNAL_31_0(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_LO_completion_signal_31_0_mask) << SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_LO_completion_signal_31_0_shift)
+
+/*define for COMPLETION_SIGNAL_HI word*/
+/*define for completion_signal_63_32 field*/
+#define SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_HI_completion_signal_63_32_offset 15
+#define SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_HI_completion_signal_63_32_mask   0xFFFFFFFF
+#define SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_HI_completion_signal_63_32_shift  0
+#define SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_HI_COMPLETION_SIGNAL_63_32(x) (((x) & SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_HI_completion_signal_63_32_mask) << SDMA_AQL_PKT_BARRIER_OR_COMPLETION_SIGNAL_HI_completion_signal_63_32_shift)
+
+
+#endif /* __SDMA_V7_1_0_PKT_OPEN_H_ */
diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c
index f7288372ee61..70e9ba9d927c 100644
--- a/drivers/gpu/drm/amd/amdgpu/si.c
+++ b/drivers/gpu/drm/amd/amdgpu/si.c
@@ -2003,10 +2003,6 @@ static int si_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
 	return 0;
 }
 
-static void si_pre_asic_init(struct amdgpu_device *adev)
-{
-}
-
 static const struct amdgpu_asic_funcs si_asic_funcs =
 {
 	.read_disabled_bios = &si_read_disabled_bios,
@@ -2028,7 +2024,6 @@ static const struct amdgpu_asic_funcs si_asic_funcs =
 	.need_reset_on_init = &si_need_reset_on_init,
 	.get_pcie_replay_count = &si_get_pcie_replay_count,
 	.supports_baco = &si_asic_supports_baco,
-	.pre_asic_init = &si_pre_asic_init,
 	.query_video_codecs = &si_query_video_codecs,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/smuio_v15_0_8.c b/drivers/gpu/drm/amd/amdgpu/smuio_v15_0_8.c
new file mode 100644
index 000000000000..ef29424c26cc
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/smuio_v15_0_8.c
@@ -0,0 +1,213 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "smuio_v15_0_8.h"
+#include "smuio/smuio_15_0_8_offset.h"
+#include "smuio/smuio_15_0_8_sh_mask.h"
+
+#define SMUIO_MCM_CONFIG__HOST_GPU_XGMI_MASK	0x00000001L
+#define SMUIO_MCM_CONFIG__ETHERNET_SWITCH_MASK	0x00000008L
+#define SMUIO_MCM_CONFIG__CUSTOM_HBM_MASK	0x00000001L
+
+static u32 smuio_v15_0_8_get_rom_index_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(SMUIO, 0, regROM_INDEX);
+}
+
+static u32 smuio_v15_0_8_get_rom_data_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(SMUIO, 0, regROM_DATA);
+}
+
+static void smuio_v15_0_8_update_rom_clock_gating(struct amdgpu_device *adev, bool enable)
+{
+	return;
+}
+
+static u64 smuio_v15_0_8_get_gpu_clock_counter(struct amdgpu_device *adev)
+{
+	u64 clock;
+	u64 clock_counter_lo, clock_counter_hi_pre, clock_counter_hi_after;
+
+	preempt_disable();
+	clock_counter_hi_pre = (u64)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
+	clock_counter_lo = (u64)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
+	/* the clock counter may be udpated during polling the counters */
+	clock_counter_hi_after = (u64)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
+	if (clock_counter_hi_pre != clock_counter_hi_after)
+		clock_counter_lo = (u64)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
+	preempt_enable();
+
+	clock = clock_counter_lo | (clock_counter_hi_after << 32ULL);
+
+	return clock;
+}
+
+static void smuio_v15_0_8_get_clock_gating_state(struct amdgpu_device *adev, u64 *flags)
+{
+	u32 data;
+
+	/* CGTT_ROM_CLK_CTRL0 is not available for APU */
+	if (adev->flags & AMD_IS_APU)
+		return;
+
+	data = RREG32_SOC15(SMUIO, 0, regCGTT_ROM_CLK_CTRL0);
+	if (!(data & CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK))
+		*flags |= AMD_CG_SUPPORT_ROM_MGCG;
+}
+
+/**
+ * smuio_v15_0_8_get_die_id - query die id from FCH.
+ *
+ * @adev: amdgpu device pointer
+ *
+ * Returns die id
+ */
+static u32 smuio_v15_0_8_get_die_id(struct amdgpu_device *adev)
+{
+	u32 data, die_id;
+
+	data = RREG32_SOC15(SMUIO, 0, regSMUIO_MCM_CONFIG);
+	die_id = REG_GET_FIELD(data, SMUIO_MCM_CONFIG, DIE_ID);
+
+	return die_id;
+}
+
+/**
+ * smuio_v15_0_8_get_socket_id - query socket id from FCH
+ *
+ * @adev: amdgpu device pointer
+ *
+ * Returns socket id
+ */
+static u32 smuio_v15_0_8_get_socket_id(struct amdgpu_device *adev)
+{
+	u32 data, socket_id;
+
+	data = RREG32_SOC15(SMUIO, 0, regSMUIO_MCM_CONFIG);
+	socket_id = REG_GET_FIELD(data, SMUIO_MCM_CONFIG, SOCKET_ID);
+
+	return socket_id;
+}
+
+/**
+ * smuio_v15_0_8_is_host_gpu_xgmi_supported - detect xgmi interface between cpu and gpu/s.
+ *
+ * @adev: amdgpu device pointer
+ *
+ * Returns true on success or false otherwise.
+ */
+static bool smuio_v15_0_8_is_host_gpu_xgmi_supported(struct amdgpu_device *adev)
+{
+	u32 data;
+
+	data = RREG32_SOC15(SMUIO, 0, regSMUIO_MCM_CONFIG);
+	data = REG_GET_FIELD(data, SMUIO_MCM_CONFIG, TOPOLOGY_ID);
+	/* data[4:0]
+	 * bit 0 == 0 host-gpu interface is PCIE
+	 * bit 0 == 1 host-gpu interface is Alternate Protocal
+	 * for AMD, this is XGMI
+	 */
+	data &= SMUIO_MCM_CONFIG__HOST_GPU_XGMI_MASK;
+
+	return data ? true : false;
+}
+
+#if 0
+/*
+ * smuio_v15_0_8_is_connected_with_ethernet_switch - detect systems connected with ethernet switch
+ *
+ * @adev: amdgpu device pointer
+ *
+ * Returns true on success or false otherwise.
+ */
+static bool smuio_v15_0_8_is_connected_with_ethernet_switch(struct amdgpu_device *adev)
+{
+	u32 data;
+
+	if (!(adev->flags & AMD_IS_APU))
+		return false;
+
+	data = RREG32_SOC15(SMUIO, 0, regSMUIO_MCM_CONFIG);
+	data = REG_GET_FIELD(data, SMUIO_MCM_CONFIG, TOPOLOGY_ID);
+	/* data[4:0]
+	 * bit 3 == 0 systems connected with ethernet switch
+	 */
+	data &= SMUIO_MCM_CONFIG__ETHERNET_SWITCH_MASK;
+
+	return data ? false : true;
+}
+#endif
+
+static enum amdgpu_pkg_type smuio_v15_0_8_get_pkg_type(struct amdgpu_device *adev)
+{
+	enum amdgpu_pkg_type pkg_type;
+	u32 data;
+
+	data = RREG32_SOC15(SMUIO, 0, regSMUIO_MCM_CONFIG);
+	data = REG_GET_FIELD(data, SMUIO_MCM_CONFIG, PKG_TYPE);
+
+	/* data [3:0]
+	 bit 2 and bit 3 identifies the pkg type */
+	switch (data & 0xC) {
+	case 0x0:
+		pkg_type = AMDGPU_PKG_TYPE_BB;
+		break;
+	case 0x8:
+		pkg_type = AMDGPU_PKG_TYPE_CEM;
+		break;
+	default:
+		pkg_type = AMDGPU_PKG_TYPE_UNKNOWN;
+		break;
+	}
+
+	return pkg_type;
+}
+
+#if 0
+static bool smuio_v15_0_8_is_custom_hbm_supported(struct amdgpu_device *adev)
+{
+	u32 data;
+
+	data = RREG32_SOC15(SMUIO, 0, regSMUIO_MCM_CONFIG);
+	data = REG_GET_FIELD(data, SMUIO_MCM_CONFIG, PKG_TYPE);
+
+	/* data [3:0]
+	 * bit 0 identifies custom HBM module */
+	data &= SMUIO_MCM_CONFIG__CUSTOM_HBM_MASK;
+
+	return data ? true : false;
+}
+#endif
+
+const struct amdgpu_smuio_funcs smuio_v15_0_8_funcs = {
+	.get_rom_index_offset = smuio_v15_0_8_get_rom_index_offset,
+	.get_rom_data_offset = smuio_v15_0_8_get_rom_data_offset,
+	.get_gpu_clock_counter = smuio_v15_0_8_get_gpu_clock_counter,
+	.get_die_id = smuio_v15_0_8_get_die_id,
+	.get_socket_id = smuio_v15_0_8_get_socket_id,
+	.is_host_gpu_xgmi_supported = smuio_v15_0_8_is_host_gpu_xgmi_supported,
+	.update_rom_clock_gating = smuio_v15_0_8_update_rom_clock_gating,
+	.get_clock_gating_state = smuio_v15_0_8_get_clock_gating_state,
+	.get_pkg_type = smuio_v15_0_8_get_pkg_type,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/smuio_v15_0_8.h b/drivers/gpu/drm/amd/amdgpu/smuio_v15_0_8.h
new file mode 100644
index 000000000000..508547fcce03
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/smuio_v15_0_8.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __SMUIO_V15_0_8_H__
+#define __SMUIO_V15_0_8_H__
+
+#include "soc15_common.h"
+
+extern const struct amdgpu_smuio_funcs smuio_v15_0_8_funcs;
+
+#endif /* __SMUIO_V15_0_8_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 42f5d9c0e3af..54b14751fd7a 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -951,7 +951,6 @@ static const struct amdgpu_asic_funcs aqua_vanjaram_asic_funcs =
 	.need_reset_on_init = &soc15_need_reset_on_init,
 	.get_pcie_replay_count = &amdgpu_nbio_get_pcie_replay_count,
 	.supports_baco = &soc15_supports_baco,
-	.pre_asic_init = &soc15_pre_asic_init,
 	.query_video_codecs = &soc15_query_video_codecs,
 	.encode_ext_smn_addressing = &aqua_vanjaram_encode_ext_smn_addressing,
 	.get_reg_state = &aqua_vanjaram_get_reg_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c
index ad36c96478a8..55c3781fc730 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc21.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc21.c
@@ -513,10 +513,6 @@ static void soc21_init_doorbell_index(struct amdgpu_device *adev)
 	adev->doorbell_index.sdma_doorbell_range = 20;
 }
 
-static void soc21_pre_asic_init(struct amdgpu_device *adev)
-{
-}
-
 static int soc21_update_umd_stable_pstate(struct amdgpu_device *adev,
 					  bool enter)
 {
@@ -546,7 +542,6 @@ static const struct amdgpu_asic_funcs soc21_asic_funcs = {
 	.need_reset_on_init = &soc21_need_reset_on_init,
 	.get_pcie_replay_count = &amdgpu_nbio_get_pcie_replay_count,
 	.supports_baco = &amdgpu_dpm_is_baco_supported,
-	.pre_asic_init = &soc21_pre_asic_init,
 	.query_video_codecs = &soc21_query_video_codecs,
 	.update_umd_stable_pstate = &soc21_update_umd_stable_pstate,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/soc24.c b/drivers/gpu/drm/amd/amdgpu/soc24.c
index 972b449ab89f..ecb6c3fcfbd1 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc24.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc24.c
@@ -327,10 +327,6 @@ static void soc24_init_doorbell_index(struct amdgpu_device *adev)
 	adev->doorbell_index.sdma_doorbell_range = 20;
 }
 
-static void soc24_pre_asic_init(struct amdgpu_device *adev)
-{
-}
-
 static int soc24_update_umd_stable_pstate(struct amdgpu_device *adev,
 					  bool enter)
 {
@@ -357,7 +353,6 @@ static const struct amdgpu_asic_funcs soc24_asic_funcs = {
 	.need_reset_on_init = &soc24_need_reset_on_init,
 	.get_pcie_replay_count = &soc24_get_pcie_replay_count,
 	.supports_baco = &amdgpu_dpm_is_baco_supported,
-	.pre_asic_init = &soc24_pre_asic_init,
 	.query_video_codecs = &soc24_query_video_codecs,
 	.update_umd_stable_pstate = &soc24_update_umd_stable_pstate,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c
new file mode 100644
index 000000000000..ec9754f687d0
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c
@@ -0,0 +1,786 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "soc15.h"
+#include "soc15_common.h"
+#include "soc_v1_0.h"
+#include "amdgpu_ip.h"
+#include "amdgpu_imu.h"
+#include "gfxhub_v12_1.h"
+#include "sdma_v7_1.h"
+#include "gfx_v12_1.h"
+
+#include "gc/gc_12_1_0_offset.h"
+#include "gc/gc_12_1_0_sh_mask.h"
+#include "mp/mp_15_0_8_offset.h"
+
+/* Initialized doorbells for amdgpu including multimedia
+ * KFD can use all the rest in 2M doorbell bar */
+static void soc_v1_0_doorbell_index_init(struct amdgpu_device *adev)
+{
+	int i;
+
+	adev->doorbell_index.kiq = AMDGPU_SOC_V1_0_DOORBELL_KIQ_START;
+
+	adev->doorbell_index.mec_ring0 = AMDGPU_SOC_V1_0_DOORBELL_MEC_RING_START;
+	adev->doorbell_index.mes_ring0 = AMDGPU_SOC_V1_0_DOORBELL_MES_RING0;
+	adev->doorbell_index.mes_ring1 = AMDGPU_SOC_V1_0_DOORBELL_MES_RING1;
+
+	adev->doorbell_index.userqueue_start = AMDGPU_SOC_V1_0_DOORBELL_USERQUEUE_START;
+	adev->doorbell_index.userqueue_end = AMDGPU_SOC_V1_0_DOORBELL_USERQUEUE_END;
+	adev->doorbell_index.xcc_doorbell_range = AMDGPU_SOC_V1_0_DOORBELL_XCC_RANGE;
+
+	adev->doorbell_index.sdma_doorbell_range = 20;
+	for (i = 0; i < adev->sdma.num_instances; i++)
+		adev->doorbell_index.sdma_engine[i] =
+			AMDGPU_SOC_V1_0_DOORBELL_sDMA_ENGINE_START +
+			i * (adev->doorbell_index.sdma_doorbell_range >> 1);
+
+	adev->doorbell_index.ih = AMDGPU_SOC_V1_0_DOORBELL_IH;
+	adev->doorbell_index.vcn.vcn_ring0_1 = AMDGPU_SOC_V1_0_DOORBELL_VCN_START;
+
+	adev->doorbell_index.first_non_cp = AMDGPU_SOC_V1_0_DOORBELL_FIRST_NON_CP;
+	adev->doorbell_index.last_non_cp = AMDGPU_SOC_V1_0_DOORBELL_LAST_NON_CP;
+
+	adev->doorbell_index.max_assignment = AMDGPU_SOC_V1_0_DOORBELL_MAX_ASSIGNMENT << 1;
+}
+
+static u32 soc_v1_0_get_config_memsize(struct amdgpu_device *adev)
+{
+	return adev->nbio.funcs->get_memsize(adev);
+}
+
+static u32 soc_v1_0_get_xclk(struct amdgpu_device *adev)
+{
+	return adev->clock.spll.reference_freq;
+}
+
+void soc_v1_0_grbm_select(struct amdgpu_device *adev,
+			  u32 me, u32 pipe,
+			  u32 queue, u32 vmid,
+			  int xcc_id)
+{
+	u32 grbm_gfx_cntl = 0;
+	grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, PIPEID, pipe);
+	grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, MEID, me);
+	grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, VMID, vmid);
+	grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, QUEUEID, queue);
+
+	WREG32_SOC15_RLC_SHADOW(GC, xcc_id, regGRBM_GFX_CNTL, grbm_gfx_cntl);
+}
+
+static struct soc15_allowed_register_entry soc_v1_0_allowed_read_registers[] = {
+	{ SOC15_REG_ENTRY(GC, 0, regGRBM_STATUS) },
+	{ SOC15_REG_ENTRY(GC, 0, regGRBM_STATUS2) },
+	{ SOC15_REG_ENTRY(GC, 0, regGRBM_STATUS3) },
+	{ SOC15_REG_ENTRY(GC, 0, regGRBM_STATUS_SE0) },
+	{ SOC15_REG_ENTRY(GC, 0, regGRBM_STATUS_SE1) },
+	{ SOC15_REG_ENTRY(GC, 0, regCP_STAT) },
+	{ SOC15_REG_ENTRY(GC, 0, regCP_STALLED_STAT1) },
+	{ SOC15_REG_ENTRY(GC, 0, regCP_STALLED_STAT2) },
+	{ SOC15_REG_ENTRY(GC, 0, regCP_STALLED_STAT3) },
+	{ SOC15_REG_ENTRY(GC, 0, regCP_CPF_BUSY_STAT) },
+	{ SOC15_REG_ENTRY(GC, 0, regCP_CPF_STALLED_STAT1) },
+	{ SOC15_REG_ENTRY(GC, 0, regCP_CPF_STATUS) },
+	{ SOC15_REG_ENTRY(GC, 0, regCP_CPC_BUSY_STAT) },
+	{ SOC15_REG_ENTRY(GC, 0, regCP_CPC_STALLED_STAT1) },
+	{ SOC15_REG_ENTRY(GC, 0, regCP_CPC_STATUS) },
+	{ SOC15_REG_ENTRY(GC, 0, regGB_ADDR_CONFIG_1) },
+};
+
+static uint32_t soc_v1_0_read_indexed_register(struct amdgpu_device *adev,
+					       u32 se_num,
+					       u32 sh_num,
+					       u32 reg_offset)
+{
+	uint32_t val;
+
+	mutex_lock(&adev->grbm_idx_mutex);
+	if (se_num != 0xffffffff || sh_num != 0xffffffff)
+		amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
+
+	val = RREG32(reg_offset);
+
+	if (se_num != 0xffffffff || sh_num != 0xffffffff)
+		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
+	mutex_unlock(&adev->grbm_idx_mutex);
+	return val;
+}
+
+static uint32_t soc_v1_0_get_register_value(struct amdgpu_device *adev,
+					    bool indexed, u32 se_num,
+					    u32 sh_num, u32 reg_offset)
+{
+	if (indexed) {
+		return soc_v1_0_read_indexed_register(adev, se_num, sh_num, reg_offset);
+	} else {
+		if (reg_offset == SOC15_REG_OFFSET(GC, 0, regGB_ADDR_CONFIG_1) &&
+		    adev->gfx.config.gb_addr_config)
+			return adev->gfx.config.gb_addr_config;
+		return RREG32(reg_offset);
+	}
+}
+
+static int soc_v1_0_read_register(struct amdgpu_device *adev,
+				  u32 se_num, u32 sh_num,
+				  u32 reg_offset, u32 *value)
+{
+	uint32_t i;
+	struct soc15_allowed_register_entry  *en;
+
+	*value = 0;
+	for (i = 0; i < ARRAY_SIZE(soc_v1_0_allowed_read_registers); i++) {
+		en = &soc_v1_0_allowed_read_registers[i];
+		if (!adev->reg_offset[en->hwip][en->inst])
+			continue;
+		else if (reg_offset != (adev->reg_offset[en->hwip][en->inst][en->seg]
+					+ en->reg_offset))
+			continue;
+
+		*value = soc_v1_0_get_register_value(adev,
+				soc_v1_0_allowed_read_registers[i].grbm_indexed,
+				se_num, sh_num, reg_offset);
+		return 0;
+	}
+	return -EINVAL;
+}
+
+static bool soc_v1_0_need_full_reset(struct amdgpu_device *adev)
+{
+	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
+	case IP_VERSION(12, 1, 0):
+	default:
+		return true;
+	}
+}
+
+static bool soc_v1_0_need_reset_on_init(struct amdgpu_device *adev)
+{
+	u32 sol_reg;
+
+	if (adev->flags & AMD_IS_APU)
+		return false;
+
+	/* Check sOS sign of life register to confirm sys driver and sOS
+	 * are already been loaded.
+	 */
+	sol_reg = RREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_81);
+	if (sol_reg)
+		return true;
+
+	return false;
+}
+
+static int soc_v1_0_asic_reset(struct amdgpu_device *adev)
+{
+	return 0;
+}
+
+static const struct amdgpu_asic_funcs soc_v1_0_asic_funcs = {
+	.read_bios_from_rom = &amdgpu_soc15_read_bios_from_rom,
+	.read_register = &soc_v1_0_read_register,
+	.get_config_memsize = &soc_v1_0_get_config_memsize,
+	.get_xclk = &soc_v1_0_get_xclk,
+	.need_full_reset = &soc_v1_0_need_full_reset,
+	.init_doorbell_index = &soc_v1_0_doorbell_index_init,
+	.need_reset_on_init = &soc_v1_0_need_reset_on_init,
+	.reset = soc_v1_0_asic_reset,
+};
+
+static int soc_v1_0_common_early_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+
+	adev->smc_rreg = NULL;
+	adev->smc_wreg = NULL;
+	adev->pcie_rreg = &amdgpu_device_indirect_rreg;
+	adev->pcie_wreg = &amdgpu_device_indirect_wreg;
+	adev->pcie_rreg_ext = &amdgpu_device_indirect_rreg_ext;
+	adev->pcie_wreg_ext = &amdgpu_device_indirect_wreg_ext;
+	adev->pcie_rreg64 = &amdgpu_device_indirect_rreg64;
+	adev->pcie_wreg64 = &amdgpu_device_indirect_wreg64;
+	adev->pciep_rreg = amdgpu_device_pcie_port_rreg;
+	adev->pciep_wreg = amdgpu_device_pcie_port_wreg;
+	adev->pcie_rreg64_ext = &amdgpu_device_indirect_rreg64_ext;
+	adev->pcie_wreg64_ext = &amdgpu_device_indirect_wreg64_ext;
+	adev->uvd_ctx_rreg = NULL;
+	adev->uvd_ctx_wreg = NULL;
+	adev->didt_rreg = NULL;
+	adev->didt_wreg = NULL;
+
+	adev->asic_funcs = &soc_v1_0_asic_funcs;
+
+	adev->rev_id = amdgpu_device_get_rev_id(adev);
+	adev->external_rev_id = 0xff;
+
+	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
+	case IP_VERSION(12, 1, 0):
+		adev->cg_flags = 0;
+		adev->pg_flags = 0;
+		adev->external_rev_id = adev->rev_id + 0x50;
+		break;
+	default:
+		/* FIXME: not supported yet */
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int soc_v1_0_common_late_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+
+	/* Enable selfring doorbell aperture late because doorbell BAR
+	 * aperture will change if resize BAR successfully in gmc sw_init.
+	 */
+	adev->nbio.funcs->enable_doorbell_selfring_aperture(adev, true);
+
+	return 0;
+}
+
+static int soc_v1_0_common_sw_init(struct amdgpu_ip_block *ip_block)
+{
+	return 0;
+}
+
+static int soc_v1_0_common_hw_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+
+	/* enable the doorbell aperture */
+	adev->nbio.funcs->enable_doorbell_aperture(adev, true);
+
+	return 0;
+}
+
+static int soc_v1_0_common_hw_fini(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+
+	adev->nbio.funcs->enable_doorbell_aperture(adev, false);
+	adev->nbio.funcs->enable_doorbell_selfring_aperture(adev, false);
+
+	return 0;
+}
+
+static int soc_v1_0_common_suspend(struct amdgpu_ip_block *ip_block)
+{
+	return soc_v1_0_common_hw_fini(ip_block);
+}
+
+static int soc_v1_0_common_resume(struct amdgpu_ip_block *ip_block)
+{
+	return soc_v1_0_common_hw_init(ip_block);
+}
+
+static bool soc_v1_0_common_is_idle(struct amdgpu_ip_block *ip_block)
+{
+	return true;
+}
+
+static int soc_v1_0_common_set_clockgating_state(struct amdgpu_ip_block *ip_block,
+						 enum amd_clockgating_state state)
+{
+	return 0;
+}
+
+static int soc_v1_0_common_set_powergating_state(struct amdgpu_ip_block *ip_block,
+						 enum amd_powergating_state state)
+{
+	return 0;
+}
+
+static void soc_v1_0_common_get_clockgating_state(struct amdgpu_ip_block *ip_block,
+						  u64 *flags)
+{
+	return;
+}
+
+static const struct amd_ip_funcs soc_v1_0_common_ip_funcs = {
+	.name = "soc_v1_0_common",
+	.early_init = soc_v1_0_common_early_init,
+	.late_init = soc_v1_0_common_late_init,
+	.sw_init = soc_v1_0_common_sw_init,
+	.hw_init = soc_v1_0_common_hw_init,
+	.hw_fini = soc_v1_0_common_hw_fini,
+	.suspend = soc_v1_0_common_suspend,
+	.resume = soc_v1_0_common_resume,
+	.is_idle = soc_v1_0_common_is_idle,
+	.set_clockgating_state = soc_v1_0_common_set_clockgating_state,
+	.set_powergating_state = soc_v1_0_common_set_powergating_state,
+	.get_clockgating_state = soc_v1_0_common_get_clockgating_state,
+};
+
+const struct amdgpu_ip_block_version soc_v1_0_common_ip_block = {
+	.type = AMD_IP_BLOCK_TYPE_COMMON,
+	.major = 1,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &soc_v1_0_common_ip_funcs,
+};
+
+static enum amdgpu_gfx_partition __soc_v1_0_calc_xcp_mode(struct amdgpu_xcp_mgr *xcp_mgr)
+{
+	struct amdgpu_device *adev = xcp_mgr->adev;
+	int num_xcc, num_xcc_per_xcp = 0, mode = 0;
+
+	num_xcc = NUM_XCC(xcp_mgr->adev->gfx.xcc_mask);
+	if (adev->gfx.funcs &&
+	    adev->gfx.funcs->get_xccs_per_xcp)
+		num_xcc_per_xcp = adev->gfx.funcs->get_xccs_per_xcp(adev);
+	if ((num_xcc_per_xcp) && (num_xcc % num_xcc_per_xcp == 0))
+		mode = num_xcc / num_xcc_per_xcp;
+
+	if (num_xcc_per_xcp == 1)
+		return AMDGPU_CPX_PARTITION_MODE;
+
+	switch (mode) {
+	case 1:
+		return AMDGPU_SPX_PARTITION_MODE;
+	case 2:
+		return AMDGPU_DPX_PARTITION_MODE;
+	case 3:
+		return AMDGPU_TPX_PARTITION_MODE;
+	case 4:
+		return AMDGPU_QPX_PARTITION_MODE;
+	default:
+		return AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE;
+	}
+
+	return AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE;
+}
+
+static int soc_v1_0_query_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr)
+{
+	enum amdgpu_gfx_partition derv_mode, mode;
+	struct amdgpu_device *adev = xcp_mgr->adev;
+
+	mode = AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE;
+	derv_mode = __soc_v1_0_calc_xcp_mode(xcp_mgr);
+
+	if (amdgpu_sriov_vf(adev))
+		return derv_mode;
+
+	if (adev->nbio.funcs &&
+	    adev->nbio.funcs->get_compute_partition_mode) {
+		mode = adev->nbio.funcs->get_compute_partition_mode(adev);
+		if (mode != derv_mode)
+			dev_warn(adev->dev,
+				 "Mismatch in compute partition mode - reported : %d derived : %d",
+				 mode, derv_mode);
+	}
+
+	return mode;
+}
+
+static int __soc_v1_0_get_xcc_per_xcp(struct amdgpu_xcp_mgr *xcp_mgr, int mode)
+{
+	int num_xcc, num_xcc_per_xcp = 0;
+
+	num_xcc = NUM_XCC(xcp_mgr->adev->gfx.xcc_mask);
+
+	switch (mode) {
+	case AMDGPU_SPX_PARTITION_MODE:
+		num_xcc_per_xcp = num_xcc;
+		break;
+	case AMDGPU_DPX_PARTITION_MODE:
+		num_xcc_per_xcp = num_xcc / 2;
+		break;
+	case AMDGPU_TPX_PARTITION_MODE:
+		num_xcc_per_xcp = num_xcc / 3;
+		break;
+	case AMDGPU_QPX_PARTITION_MODE:
+		num_xcc_per_xcp = num_xcc / 4;
+		break;
+	case AMDGPU_CPX_PARTITION_MODE:
+		num_xcc_per_xcp = 1;
+		break;
+	}
+
+	return num_xcc_per_xcp;
+}
+
+static int __soc_v1_0_get_xcp_ip_info(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id,
+				      enum AMDGPU_XCP_IP_BLOCK ip_id,
+				      struct amdgpu_xcp_ip *ip)
+{
+	struct amdgpu_device *adev = xcp_mgr->adev;
+	int num_sdma, num_vcn, num_shared_vcn, num_xcp;
+	int num_xcc_xcp, num_sdma_xcp, num_vcn_xcp;
+
+	num_sdma = adev->sdma.num_instances;
+	num_vcn = adev->vcn.num_vcn_inst;
+	num_shared_vcn = 1;
+
+	num_xcc_xcp = adev->gfx.num_xcc_per_xcp;
+	num_xcp = NUM_XCC(adev->gfx.xcc_mask) / num_xcc_xcp;
+
+	switch (xcp_mgr->mode) {
+	case AMDGPU_SPX_PARTITION_MODE:
+	case AMDGPU_DPX_PARTITION_MODE:
+	case AMDGPU_TPX_PARTITION_MODE:
+	case AMDGPU_QPX_PARTITION_MODE:
+	case AMDGPU_CPX_PARTITION_MODE:
+		num_sdma_xcp = DIV_ROUND_UP(num_sdma, num_xcp);
+		num_vcn_xcp = DIV_ROUND_UP(num_vcn, num_xcp);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (num_vcn && num_xcp > num_vcn)
+		num_shared_vcn = num_xcp / num_vcn;
+
+	switch (ip_id) {
+	case AMDGPU_XCP_GFXHUB:
+		ip->inst_mask = XCP_INST_MASK(num_xcc_xcp, xcp_id);
+		ip->ip_funcs = &gfxhub_v12_1_xcp_funcs;
+		break;
+	case AMDGPU_XCP_GFX:
+		ip->inst_mask = XCP_INST_MASK(num_xcc_xcp, xcp_id);
+		ip->ip_funcs = &gfx_v12_1_xcp_funcs;
+		break;
+	case AMDGPU_XCP_SDMA:
+		ip->inst_mask = XCP_INST_MASK(num_sdma_xcp, xcp_id);
+		ip->ip_funcs = &sdma_v7_1_xcp_funcs;
+		break;
+	case AMDGPU_XCP_VCN:
+		ip->inst_mask =
+			XCP_INST_MASK(num_vcn_xcp, xcp_id / num_shared_vcn);
+		/* TODO : Assign IP funcs */
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ip->ip_id = ip_id;
+
+	return 0;
+}
+
+static int soc_v1_0_get_xcp_res_info(struct amdgpu_xcp_mgr *xcp_mgr,
+				     int mode,
+				     struct amdgpu_xcp_cfg *xcp_cfg)
+{
+	struct amdgpu_device *adev = xcp_mgr->adev;
+	int max_res[AMDGPU_XCP_RES_MAX] = {};
+	bool res_lt_xcp;
+	int num_xcp, i;
+	u16 nps_modes;
+
+	if (!(xcp_mgr->supp_xcp_modes & BIT(mode)))
+		return -EINVAL;
+
+	max_res[AMDGPU_XCP_RES_XCC] = NUM_XCC(adev->gfx.xcc_mask);
+	max_res[AMDGPU_XCP_RES_DMA] = adev->sdma.num_instances;
+	max_res[AMDGPU_XCP_RES_DEC] = adev->vcn.num_vcn_inst;
+	max_res[AMDGPU_XCP_RES_JPEG] = adev->jpeg.num_jpeg_inst;
+
+	switch (mode) {
+	case AMDGPU_SPX_PARTITION_MODE:
+		num_xcp = 1;
+		nps_modes = BIT(AMDGPU_NPS1_PARTITION_MODE);
+		break;
+	case AMDGPU_DPX_PARTITION_MODE:
+		num_xcp = 2;
+		nps_modes = BIT(AMDGPU_NPS1_PARTITION_MODE);
+		break;
+	case AMDGPU_TPX_PARTITION_MODE:
+		num_xcp = 3;
+		nps_modes = BIT(AMDGPU_NPS1_PARTITION_MODE) |
+			    BIT(AMDGPU_NPS4_PARTITION_MODE);
+		break;
+	case AMDGPU_QPX_PARTITION_MODE:
+		num_xcp = 4;
+		nps_modes = BIT(AMDGPU_NPS1_PARTITION_MODE) |
+			    BIT(AMDGPU_NPS4_PARTITION_MODE);
+		break;
+	case AMDGPU_CPX_PARTITION_MODE:
+		num_xcp = NUM_XCC(adev->gfx.xcc_mask);
+		nps_modes = BIT(AMDGPU_NPS1_PARTITION_MODE) |
+			    BIT(AMDGPU_NPS4_PARTITION_MODE);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	xcp_cfg->compatible_nps_modes =
+		(adev->gmc.supported_nps_modes & nps_modes);
+	xcp_cfg->num_res = ARRAY_SIZE(max_res);
+
+	for (i = 0; i < xcp_cfg->num_res; i++) {
+		res_lt_xcp = max_res[i] < num_xcp;
+		xcp_cfg->xcp_res[i].id = i;
+		xcp_cfg->xcp_res[i].num_inst =
+			res_lt_xcp ? 1 : max_res[i] / num_xcp;
+		xcp_cfg->xcp_res[i].num_inst =
+			i == AMDGPU_XCP_RES_JPEG ?
+			xcp_cfg->xcp_res[i].num_inst *
+			adev->jpeg.num_jpeg_rings : xcp_cfg->xcp_res[i].num_inst;
+		xcp_cfg->xcp_res[i].num_shared =
+			res_lt_xcp ? num_xcp / max_res[i] : 1;
+	}
+
+	return 0;
+}
+
+static enum amdgpu_gfx_partition __soc_v1_0_get_auto_mode(struct amdgpu_xcp_mgr *xcp_mgr)
+{
+	struct amdgpu_device *adev = xcp_mgr->adev;
+	int num_xcc;
+
+	num_xcc = NUM_XCC(xcp_mgr->adev->gfx.xcc_mask);
+
+	if (adev->gmc.num_mem_partitions == 1)
+		return AMDGPU_SPX_PARTITION_MODE;
+
+	if (adev->gmc.num_mem_partitions == num_xcc)
+		return AMDGPU_CPX_PARTITION_MODE;
+
+	if (adev->gmc.num_mem_partitions == 2)
+		return AMDGPU_DPX_PARTITION_MODE;
+
+	return AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE;
+}
+
+static bool __soc_v1_0_is_valid_mode(struct amdgpu_xcp_mgr *xcp_mgr,
+				     enum amdgpu_gfx_partition mode)
+{
+	struct amdgpu_device *adev = xcp_mgr->adev;
+	int num_xcc, num_xccs_per_xcp;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	switch (mode) {
+	case AMDGPU_SPX_PARTITION_MODE:
+		return adev->gmc.num_mem_partitions == 1 && num_xcc > 0;
+	case AMDGPU_DPX_PARTITION_MODE:
+		return adev->gmc.num_mem_partitions <= 2 && (num_xcc % 4) == 0;
+	case AMDGPU_TPX_PARTITION_MODE:
+		return (adev->gmc.num_mem_partitions == 1 ||
+			adev->gmc.num_mem_partitions == 3) &&
+		       ((num_xcc % 3) == 0);
+	case AMDGPU_QPX_PARTITION_MODE:
+		num_xccs_per_xcp = num_xcc / 4;
+		return (adev->gmc.num_mem_partitions == 1 ||
+			adev->gmc.num_mem_partitions == 4) &&
+		       (num_xccs_per_xcp >= 2);
+	case AMDGPU_CPX_PARTITION_MODE:
+		/* (num_xcc > 1) because 1 XCC is considered SPX, not CPX.
+		 * (num_xcc % adev->gmc.num_mem_partitions) == 0 because
+		 * num_compute_partitions can't be less than num_mem_partitions
+		 */
+		return ((num_xcc > 1) &&
+		       (num_xcc % adev->gmc.num_mem_partitions) == 0);
+	default:
+		return false;
+	}
+
+	return false;
+}
+
+static void __soc_v1_0_update_available_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr)
+{
+	int mode;
+
+	xcp_mgr->avail_xcp_modes = 0;
+
+	for_each_inst(mode, xcp_mgr->supp_xcp_modes) {
+		if (__soc_v1_0_is_valid_mode(xcp_mgr, mode))
+			xcp_mgr->avail_xcp_modes |= BIT(mode);
+	}
+}
+
+static int soc_v1_0_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr,
+					  int mode, int *num_xcps)
+{
+	int num_xcc_per_xcp, num_xcc, ret;
+	struct amdgpu_device *adev;
+	u32 flags = 0;
+
+	adev = xcp_mgr->adev;
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+
+	if (mode == AMDGPU_AUTO_COMPUTE_PARTITION_MODE) {
+		mode = __soc_v1_0_get_auto_mode(xcp_mgr);
+		if (mode == AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE) {
+			dev_err(adev->dev,
+				"Invalid config, no compatible compute partition mode found, available memory partitions: %d",
+				adev->gmc.num_mem_partitions);
+			return -EINVAL;
+		}
+	} else if (!__soc_v1_0_is_valid_mode(xcp_mgr, mode)) {
+		dev_err(adev->dev,
+			"Invalid compute partition mode requested, requested: %s, available memory partitions: %d",
+			amdgpu_gfx_compute_mode_desc(mode), adev->gmc.num_mem_partitions);
+		return -EINVAL;
+	}
+
+	if (adev->kfd.init_complete && !amdgpu_in_reset(adev))
+		flags |= AMDGPU_XCP_OPS_KFD;
+
+	if (flags & AMDGPU_XCP_OPS_KFD) {
+		ret = amdgpu_amdkfd_check_and_lock_kfd(adev);
+		if (ret)
+			goto out;
+	}
+
+	ret = amdgpu_xcp_pre_partition_switch(xcp_mgr, flags);
+	if (ret)
+		goto unlock;
+
+	num_xcc_per_xcp = __soc_v1_0_get_xcc_per_xcp(xcp_mgr, mode);
+	if (adev->gfx.imu.funcs &&
+	    adev->gfx.imu.funcs->switch_compute_partition)
+		adev->gfx.imu.funcs->switch_compute_partition(xcp_mgr->adev, num_xcc_per_xcp, mode);
+
+	/* Init info about new xcps */
+	*num_xcps = num_xcc / num_xcc_per_xcp;
+	amdgpu_xcp_init(xcp_mgr, *num_xcps, mode);
+
+	ret = amdgpu_xcp_post_partition_switch(xcp_mgr, flags);
+	if (!ret)
+		__soc_v1_0_update_available_partition_mode(xcp_mgr);
+unlock:
+	if (flags & AMDGPU_XCP_OPS_KFD)
+		amdgpu_amdkfd_unlock_kfd(adev);
+out:
+	return ret;
+}
+
+#ifdef HAVE_ACPI_DEV_GET_FIRST_MATCH_DEV
+static int __soc_v1_0_get_xcp_mem_id(struct amdgpu_device *adev,
+				     int xcc_id, uint8_t *mem_id)
+{
+	/* memory/spatial modes validation check is already done */
+	*mem_id = xcc_id / adev->gfx.num_xcc_per_xcp;
+	*mem_id /= adev->xcp_mgr->num_xcp_per_mem_partition;
+
+	return 0;
+}
+
+static int soc_v1_0_get_xcp_mem_id(struct amdgpu_xcp_mgr *xcp_mgr,
+				   struct amdgpu_xcp *xcp, uint8_t *mem_id)
+{
+	struct amdgpu_numa_info numa_info;
+	struct amdgpu_device *adev;
+	uint32_t xcc_mask;
+	int r, i, xcc_id;
+
+	adev = xcp_mgr->adev;
+	/* TODO: BIOS is not returning the right info now
+	 * Check on this later
+	 */
+	/*
+	if (adev->gmc.gmc_funcs->query_mem_partition_mode)
+		mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
+	*/
+	if (adev->gmc.num_mem_partitions == 1) {
+		/* Only one range */
+		*mem_id = 0;
+		return 0;
+	}
+
+	r = amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_GFX, &xcc_mask);
+	if (r || !xcc_mask)
+		return -EINVAL;
+
+	xcc_id = ffs(xcc_mask) - 1;
+	if (!adev->gmc.is_app_apu)
+		return __soc_v1_0_get_xcp_mem_id(adev, xcc_id, mem_id);
+
+	r = amdgpu_acpi_get_mem_info(adev, xcc_id, &numa_info);
+
+	if (r)
+		return r;
+
+	r = -EINVAL;
+	for (i = 0; i < adev->gmc.num_mem_partitions; ++i) {
+		if (adev->gmc.mem_partitions[i].numa.node == numa_info.nid) {
+			*mem_id = i;
+			r = 0;
+			break;
+		}
+	}
+
+	return r;
+}
+#endif
+
+static int soc_v1_0_get_xcp_ip_details(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id,
+				       enum AMDGPU_XCP_IP_BLOCK ip_id,
+				       struct amdgpu_xcp_ip *ip)
+{
+	if (!ip)
+		return -EINVAL;
+
+	return __soc_v1_0_get_xcp_ip_info(xcp_mgr, xcp_id, ip_id, ip);
+}
+
+struct amdgpu_xcp_mgr_funcs soc_v1_0_xcp_funcs = {
+	.switch_partition_mode = &soc_v1_0_switch_partition_mode,
+	.query_partition_mode = &soc_v1_0_query_partition_mode,
+	.get_ip_details = &soc_v1_0_get_xcp_ip_details,
+	.get_xcp_res_info = &soc_v1_0_get_xcp_res_info,
+#ifdef HAVE_ACPI_DEV_GET_FIRST_MATCH_DEV
+	.get_xcp_mem_id = &soc_v1_0_get_xcp_mem_id,
+#endif
+};
+
+static int soc_v1_0_xcp_mgr_init(struct amdgpu_device *adev)
+{
+	int ret;
+
+	if (amdgpu_sriov_vf(adev))
+		soc_v1_0_xcp_funcs.switch_partition_mode = NULL;
+
+	ret = amdgpu_xcp_mgr_init(adev, AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE,
+				  1, &soc_v1_0_xcp_funcs);
+	if (ret)
+		return ret;
+
+	amdgpu_xcp_update_supported_modes(adev->xcp_mgr);
+	/* TODO: Default memory node affinity init */
+
+	return ret;
+}
+
+int soc_v1_0_init_soc_config(struct amdgpu_device *adev)
+{
+	int ret;
+
+	/*TODO: init soc config */
+	adev->sdma.num_inst_per_xcc = 2;
+	adev->sdma.num_instances =
+		NUM_XCC(adev->gfx.xcc_mask) * adev->sdma.num_inst_per_xcc;
+	adev->sdma.sdma_mask =
+		GENMASK(adev->sdma.num_instances - 1, 0);
+
+	ret = soc_v1_0_xcp_mgr_init(adev);
+	if (ret)
+		return ret;
+
+	amdgpu_ip_map_init(adev);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/soc_v1_0.h b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.h
new file mode 100644
index 000000000000..23517c3a3d1b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __SOC_V1_0_H__
+#define __SOC_V1_0_H__
+
+extern const struct amdgpu_ip_block_version soc_v1_0_common_ip_block;
+
+void soc_v1_0_grbm_select(struct amdgpu_device *adev,
+			  u32 me, u32 pipe,
+			  u32 queue, u32 vmid,
+			  int xcc_id);
+int soc_v1_0_init_soc_config(struct amdgpu_device *adev);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h b/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
index 8a3f326474e5..f6178da910d3 100644
--- a/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
+++ b/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
@@ -155,6 +155,7 @@ struct ta_ras_init_flags {
 	uint8_t channel_dis_num;
 	uint8_t nps_mode;
 	uint32_t active_umc_mask;
+	uint8_t vram_type;
 };
 
 struct ta_ras_mca_addr {
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
index a316797875a8..e9d790914761 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
@@ -2053,6 +2053,7 @@ static int vcn_v1_0_validate_bo(struct amdgpu_cs_parser *parser,
 {
 	struct ttm_operation_ctx ctx = { false, false };
 	struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
+	struct amdgpu_device *adev = parser->adev;
 	struct amdgpu_vm *vm = &fpriv->vm;
 	struct amdgpu_bo_va_mapping *mapping;
 	struct amdgpu_bo *bo;
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
index d9cf8f0feeb3..02d5c5af65f2 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
@@ -1907,6 +1907,7 @@ static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
 			    uint64_t addr)
 {
 	struct ttm_operation_ctx ctx = { false, false };
+	struct amdgpu_device *adev = p->adev;
 	struct amdgpu_bo_va_mapping *map;
 	uint32_t *msg, num_buffers;
 	struct amdgpu_bo *bo;
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
index 3ae666522d57..d17219be50f3 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
@@ -1824,6 +1824,7 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
 			    uint64_t addr)
 {
 	struct ttm_operation_ctx ctx = { false, false };
+	struct amdgpu_device *adev = p->adev;
 	struct amdgpu_bo_va_mapping *map;
 	uint32_t *msg, num_buffers;
 	struct amdgpu_bo *bo;
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index a611a7345125..6a574b6c8e63 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -1424,10 +1424,6 @@ static bool vi_need_reset_on_init(struct amdgpu_device *adev)
 	return false;
 }
 
-static void vi_pre_asic_init(struct amdgpu_device *adev)
-{
-}
-
 static const struct amdgpu_asic_funcs vi_asic_funcs =
 {
 	.read_disabled_bios = &vi_read_disabled_bios,
@@ -1447,7 +1443,6 @@ static const struct amdgpu_asic_funcs vi_asic_funcs =
 	.need_reset_on_init = &vi_need_reset_on_init,
 	.get_pcie_replay_count = &vi_get_pcie_replay_count,
 	.supports_baco = &vi_asic_supports_baco,
-	.pre_asic_init = &vi_pre_asic_init,
 	.query_video_codecs = &vi_query_video_codecs,
 };
 
diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile
index 0ce08113c9f0..85fc67d521e5 100644
--- a/drivers/gpu/drm/amd/amdkfd/Makefile
+++ b/drivers/gpu/drm/amd/amdkfd/Makefile
@@ -38,6 +38,7 @@ AMDKFD_FILES	:= $(AMDKFD_PATH)/kfd_module.o \
 		$(AMDKFD_PATH)/kfd_mqd_manager_v10.o \
 		$(AMDKFD_PATH)/kfd_mqd_manager_v11.o \
 		$(AMDKFD_PATH)/kfd_mqd_manager_v12.o \
+		$(AMDKFD_PATH)/kfd_mqd_manager_v12_1.o \
 		$(AMDKFD_PATH)/kfd_kernel_queue.o \
 		$(AMDKFD_PATH)/kfd_packet_manager.o \
 		$(AMDKFD_PATH)/kfd_packet_manager_vi.o \
@@ -50,12 +51,14 @@ AMDKFD_FILES	:= $(AMDKFD_PATH)/kfd_module.o \
 		$(AMDKFD_PATH)/kfd_device_queue_manager_v10.o \
 		$(AMDKFD_PATH)/kfd_device_queue_manager_v11.o \
 		$(AMDKFD_PATH)/kfd_device_queue_manager_v12.o \
+		$(AMDKFD_PATH)/kfd_device_queue_manager_v12_1.o \
 		$(AMDKFD_PATH)/kfd_interrupt.o \
 		$(AMDKFD_PATH)/kfd_events.o \
 		$(AMDKFD_PATH)/cik_event_interrupt.o \
 		$(AMDKFD_PATH)/kfd_int_process_v9.o \
 		$(AMDKFD_PATH)/kfd_int_process_v10.o \
 		$(AMDKFD_PATH)/kfd_int_process_v11.o \
+		$(AMDKFD_PATH)/kfd_int_process_v12_1.o \
 		$(AMDKFD_PATH)/kfd_smi_events.o \
 		$(AMDKFD_PATH)/kfd_crat.o \
 		$(AMDKFD_PATH)/kfd_debug.o
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 22925df6a791..041237861107 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -164,8 +164,13 @@ static int kfd_release(struct inode *inode, struct file *filep)
 {
 	struct kfd_process *process = filep->private_data;
 
-	if (process)
-		kfd_unref_process(process);
+	if (!process)
+		return 0;
+
+	if (process->context_id != KFD_CONTEXT_ID_PRIMARY)
+		kfd_process_notifier_release_internal(process);
+
+	kfd_unref_process(process);
 
 	return 0;
 }
@@ -1057,6 +1062,12 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
 	if (args->size == 0)
 		return -EINVAL;
 
+	if (p->context_id != KFD_CONTEXT_ID_PRIMARY && (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)) {
+		pr_debug("USERPTR is not supported on non-primary kfd_process\n");
+
+		return -EOPNOTSUPP;
+	}
+
 #if IS_ENABLED(CONFIG_HSA_AMD_SVM)
 	/* Flush pending deferred work to avoid racing with deferred actions
 	 * from previous memory map changes (e.g. munmap).
@@ -1713,6 +1724,12 @@ static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
 	struct kfd_ioctl_svm_args *args = data;
 	int r = 0;
 
+	if (p->context_id != KFD_CONTEXT_ID_PRIMARY) {
+		pr_debug("SVM ioctl not supported on non-primary kfd process\n");
+
+		return -EOPNOTSUPP;
+	}
+
 	pr_debug("start 0x%llx size 0x%llx op 0x%x nattr 0x%x\n",
 		 args->start_addr, args->size, args->op, args->nattr);
 
@@ -2914,6 +2931,12 @@ static int kfd_ioctl_set_debug_trap(struct file *filep, struct kfd_process *p, v
 	struct kfd_process_device *pdd = NULL;
 	int r = 0;
 
+	if (p->context_id != KFD_CONTEXT_ID_PRIMARY) {
+		pr_debug("Set debug trap ioctl can not be invoked on non-primary kfd process\n");
+
+		return -EOPNOTSUPP;
+	}
+
 	if (sched_policy == KFD_SCHED_POLICY_NO_HWS) {
 		pr_err("Debugging does not support sched_policy %i", sched_policy);
 		return -EINVAL;
@@ -2958,6 +2981,12 @@ static int kfd_ioctl_set_debug_trap(struct file *filep, struct kfd_process *p, v
 		goto out;
 	}
 
+	if (target->context_id != KFD_CONTEXT_ID_PRIMARY) {
+		pr_debug("Set debug trap ioctl not supported on non-primary kfd process\n");
+		r = -EOPNOTSUPP;
+		goto out;
+	}
+
 	/* Check if target is still PTRACED. */
 	rcu_read_lock();
 	if (target != p && args->op != KFD_IOC_DBG_TRAP_DISABLE
@@ -3121,6 +3150,48 @@ static int kfd_ioctl_set_debug_trap(struct file *filep, struct kfd_process *p, v
 	return r;
 }
 
+/* userspace programs need to invoke this ioctl explicitly on a FD to
+ * create a secondary kfd_process which replacing its primary kfd_process
+ */
+static int kfd_ioctl_create_process(struct file *filep, struct kfd_process *p, void *data)
+{
+	struct kfd_process *process;
+	int ret;
+
+	/* Each FD owns only one kfd_process */
+	if (p->context_id != KFD_CONTEXT_ID_PRIMARY)
+		return -EINVAL;
+
+	if (!filep->private_data || !p)
+		return -EINVAL;
+
+	mutex_lock(&kfd_processes_mutex);
+	if (p != filep->private_data) {
+		mutex_unlock(&kfd_processes_mutex);
+		return -EINVAL;
+	}
+
+	process = create_process(current, false);
+	if (IS_ERR(process)) {
+		mutex_unlock(&kfd_processes_mutex);
+		return PTR_ERR(process);
+	}
+
+	filep->private_data = process;
+	mutex_unlock(&kfd_processes_mutex);
+
+	ret = kfd_create_process_sysfs(process);
+	if (ret)
+		pr_warn("Failed to create sysfs entry for the kfd_process");
+
+	/* Each open() increases kref of the primary kfd_process,
+	 * so we need to reduce it here when we create a new secondary process replacing it
+	 */
+	kfd_unref_process(p);
+
+	return 0;
+}
+
 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
 	[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
 			    .cmd_drv = 0, .name = #ioctl}
@@ -3239,6 +3310,9 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
 
 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_TRAP,
 			kfd_ioctl_set_debug_trap, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_PROCESS,
+			kfd_ioctl_create_process, 0),
 };
 
 #define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)
@@ -3397,16 +3471,19 @@ static int kfd_mmio_mmap(struct kfd_node *dev, struct kfd_process *process,
 }
 
 
-static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
+static int kfd_mmap(struct file *filep, struct vm_area_struct *vma)
 {
 	struct kfd_process *process;
 	struct kfd_node *dev = NULL;
 	unsigned long mmap_offset;
 	unsigned int gpu_id;
 
-	process = kfd_get_process(current);
-	if (IS_ERR(process))
-		return PTR_ERR(process);
+	process = filep->private_data;
+	if (!process)
+		return -ESRCH;
+
+	if (process->lead_thread != current->group_leader)
+		return -EBADF;
 
 	mmap_offset = vma->vm_pgoff << PAGE_SHIFT;
 	gpu_id = KFD_MMAP_GET_GPU_ID(mmap_offset);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
index 4a7180b46b71..5f2dd378936e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
@@ -1712,6 +1712,7 @@ int kfd_get_gpu_cache_info(struct kfd_node *kdev, struct kfd_gpu_cache_info **pc
 			fallthrough;
 		case IP_VERSION(12, 0, 0):
 		case IP_VERSION(12, 0, 1):
+		case IP_VERSION(12, 1, 0):
 			num_of_cache_types =
 				kfd_fill_gpu_cache_info_from_gfx_config(kdev->kfd,
 									cache_line_size_missing,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c
index ba99e0f258ae..f83e1238c1b3 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c
@@ -372,7 +372,7 @@ int kfd_dbg_set_mes_debug_mode(struct kfd_process_device *pdd, bool sq_trap_en)
 	}
 
 	return amdgpu_mes_set_shader_debugger(pdd->dev->adev, pdd->proc_ctx_gpu_addr, spi_dbg_cntl,
-						pdd->watch_points, flags, sq_trap_en);
+						pdd->watch_points, flags, sq_trap_en, 0);
 }
 
 #define KFD_DEBUGGER_INVALID_WATCH_POINT_ID -1
@@ -519,6 +519,7 @@ int kfd_dbg_trap_set_flags(struct kfd_process *target, uint32_t *flags)
 		struct kfd_topology_device *topo_dev =
 				kfd_topology_device_by_id(target->pdds[i]->dev->id);
 		uint32_t caps = topo_dev->node_props.capability;
+		uint32_t caps2 = topo_dev->node_props.capability2;
 
 		if (!(caps & HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED) &&
 			(*flags & KFD_DBG_TRAP_FLAG_SINGLE_MEM_OP)) {
@@ -531,6 +532,12 @@ int kfd_dbg_trap_set_flags(struct kfd_process *target, uint32_t *flags)
 			*flags = prev_flags;
 			return -EACCES;
 		}
+
+		if (!(caps2 & HSA_CAP2_TRAP_DEBUG_LDS_OUT_OF_ADDR_RANGE_SUPPORTED) &&
+		    (*flags & KFD_DBG_TRAP_FLAG_LDS_OUT_OF_ADDR_RANGE)) {
+			*flags = prev_flags;
+			return -EACCES;
+		}
 	}
 
 	target->dbg_flags = *flags;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index e9cfb80bd436..b91843b2af6a 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -94,6 +94,8 @@ static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd)
 	case IP_VERSION(5, 2, 2):/* NAVY_FLOUNDER */
 	case IP_VERSION(5, 2, 4):/* DIMGREY_CAVEFISH */
 	case IP_VERSION(5, 2, 5):/* BEIGE_GOBY */
+		kfd->device_info.num_sdma_queues_per_engine = 8;
+		break;
 	case IP_VERSION(6, 0, 0):
 	case IP_VERSION(6, 0, 1):
 	case IP_VERSION(6, 0, 2):
@@ -104,7 +106,10 @@ static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd)
 	case IP_VERSION(6, 1, 3):
 	case IP_VERSION(7, 0, 0):
 	case IP_VERSION(7, 0, 1):
+	case IP_VERSION(7, 1, 0):
 		kfd->device_info.num_sdma_queues_per_engine = 8;
+		/* Reserve 1 for paging and 1 for gfx */
+		kfd->device_info.num_reserved_sdma_queues_per_engine = 2;
 		break;
 	default:
 		dev_warn(kfd_device,
@@ -112,30 +117,6 @@ static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd)
 			sdma_version);
 		kfd->device_info.num_sdma_queues_per_engine = 8;
 	}
-
-	bitmap_zero(kfd->device_info.reserved_sdma_queues_bitmap, KFD_MAX_SDMA_QUEUES);
-
-	switch (sdma_version) {
-	case IP_VERSION(6, 0, 0):
-	case IP_VERSION(6, 0, 1):
-	case IP_VERSION(6, 0, 2):
-	case IP_VERSION(6, 0, 3):
-	case IP_VERSION(6, 1, 0):
-	case IP_VERSION(6, 1, 1):
-	case IP_VERSION(6, 1, 2):
-	case IP_VERSION(6, 1, 3):
-	case IP_VERSION(7, 0, 0):
-	case IP_VERSION(7, 0, 1):
-		/* Reserve 1 for paging and 1 for gfx */
-		kfd->device_info.num_reserved_sdma_queues_per_engine = 2;
-		/* BIT(0)=engine-0 queue-0; BIT(1)=engine-1 queue-0; BIT(2)=engine-0 queue-1; ... */
-		bitmap_set(kfd->device_info.reserved_sdma_queues_bitmap, 0,
-			   kfd->adev->sdma.num_instances *
-			   kfd->device_info.num_reserved_sdma_queues_per_engine);
-		break;
-	default:
-		break;
-	}
 }
 
 static void kfd_device_info_set_event_interrupt_class(struct kfd_dev *kfd)
@@ -190,6 +171,10 @@ static void kfd_device_info_set_event_interrupt_class(struct kfd_dev *kfd)
 		/* GFX12_TODO: Change to v12 version. */
 		kfd->device_info.event_interrupt_class = &event_interrupt_class_v11;
 		break;
+	case IP_VERSION(12, 1, 0):
+		kfd->device_info.event_interrupt_class =
+						&event_interrupt_class_v12_1;
+		break;
 	default:
 		dev_warn(kfd_device, "v9 event interrupt handler is set due to "
 			"mismatch of gc ip block(GC_HWIP:0x%x).\n", gc_version);
@@ -464,6 +449,10 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf)
 			gfx_target_version = 120001;
 			f2g = &gfx_v12_kfd2kgd;
 			break;
+		case IP_VERSION(12, 1, 0):
+			gfx_target_version = 120500;
+			f2g = &gfx_v12_kfd2kgd;
+			break;
 		default:
 			break;
 		}
@@ -682,6 +671,7 @@ static void kfd_setup_interrupt_bitmap(struct kfd_node *node,
 	struct amdgpu_device *adev = node->adev;
 	uint32_t xcc_mask = node->xcc_mask;
 	uint32_t xcc, mapped_xcc;
+	uint32_t bitmap;
 	/*
 	 * Interrupt bitmap is setup for processing interrupts from
 	 * different XCDs and AIDs.
@@ -703,9 +693,22 @@ static void kfd_setup_interrupt_bitmap(struct kfd_node *node,
 	 * - AND VMID reported in the interrupt lies within the
 	 *   VMID range of the node.
 	 */
-	for_each_inst(xcc, xcc_mask) {
-		mapped_xcc = GET_INST(GC, xcc);
-		node->interrupt_bitmap |= (mapped_xcc % 2 ? 5 : 3) << (4 * (mapped_xcc / 2));
+	switch (KFD_GC_VERSION(node)) {
+	case IP_VERSION(12, 1, 0):
+		for_each_inst(xcc, xcc_mask) {
+			mapped_xcc = GET_INST(GC, xcc);
+			bitmap = 0x2 | (0x4 << (mapped_xcc % 4));
+			if (mapped_xcc/4)
+				bitmap = bitmap << 8;
+			node->interrupt_bitmap |= bitmap;
+		}
+		break;
+	default:
+		for_each_inst(xcc, xcc_mask) {
+			mapped_xcc = GET_INST(GC, xcc);
+			node->interrupt_bitmap |= (mapped_xcc % 2 ? 5 : 3) << (4 * (mapped_xcc / 2));
+		}
+		break;
 	}
 	dev_info(kfd_device, "Node: %d, interrupt_bitmap: %x\n", kfd_node_idx,
 							node->interrupt_bitmap);
@@ -768,7 +771,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 	 * If the VMID range changes for multi-partition capable GPUs, then
 	 * this code MUST be revisited.
 	 */
-	if (kfd->adev->xcp_mgr) {
+	if (kfd->adev->xcp_mgr && (KFD_GC_VERSION(kfd) != IP_VERSION(12, 1, 0))) {
 		partition_mode = amdgpu_xcp_query_partition_mode(kfd->adev->xcp_mgr,
 								 AMDGPU_XCP_FL_LOCKED);
 		if (partition_mode == AMDGPU_CPX_PARTITION_MODE &&
@@ -1059,7 +1062,7 @@ void kgd2kfd_suspend(struct kfd_dev *kfd, bool suspend_proc)
 
 int kgd2kfd_resume(struct kfd_dev *kfd, bool resume_proc)
 {
-	int ret, i;
+	int ret = 0, i;
 
 	if (!kfd->init_complete)
 		return 0;
@@ -1202,12 +1205,13 @@ int kgd2kfd_resume_mm(struct mm_struct *mm)
  *   prepare for safe eviction of KFD BOs that belong to the specified
  *   process.
  *
- * @mm: mm_struct that identifies the specified KFD process
+ * @mm: mm_struct that identifies a group of KFD processes
+ * @context_id: an id that identifies a specific KFD context in the above kfd process group
  * @fence: eviction fence attached to KFD process BOs
  *
  */
 int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
-					       struct dma_fence *fence)
+					       u16 context_id, struct dma_fence *fence)
 {
 	struct kfd_process *p;
 	unsigned long active_time;
@@ -1219,7 +1223,7 @@ int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
 	if (dma_fence_is_signaled(fence))
 		return 0;
 
-	p = kfd_lookup_process_by_mm(mm);
+	p = kfd_lookup_process_by_id(mm, context_id);
 	if (!p)
 		return -ENODEV;
 
@@ -1681,6 +1685,10 @@ bool kgd2kfd_vmfault_fast_path(struct amdgpu_device *adev, struct amdgpu_iv_entr
 {
 	struct kfd_process *p;
 	u32 cam_index;
+	u32 src_data_idx;
+
+	src_data_idx = (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) ?
+			3 : 2;
 
 	if (entry->ih == &adev->irq.ih_soft || entry->ih == &adev->irq.ih1) {
 		p = kfd_lookup_process_by_pasid(entry->pasid, NULL);
@@ -1689,7 +1697,8 @@ bool kgd2kfd_vmfault_fast_path(struct amdgpu_device *adev, struct amdgpu_iv_entr
 
 		if (p->gpu_page_fault && !p->debug_trap_enabled) {
 			if (retry_fault && adev->irq.retry_cam_enabled) {
-				cam_index = entry->src_data[2] & 0x3ff;
+				cam_index = entry->src_data[src_data_idx] & 0x3ff;
+
 				WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index);
 			}
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index d7a2e7178ea9..1dbdf9465e59 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -137,9 +137,10 @@ static void init_sdma_bitmaps(struct device_queue_manager *dqm)
 	bitmap_set(dqm->xgmi_sdma_bitmap, 0, get_num_xgmi_sdma_queues(dqm));
 
 	/* Mask out the reserved queues */
-	bitmap_andnot(dqm->sdma_bitmap, dqm->sdma_bitmap,
-		      dqm->dev->kfd->device_info.reserved_sdma_queues_bitmap,
-		      KFD_MAX_SDMA_QUEUES);
+	bitmap_clear(dqm->sdma_bitmap, 0, kfd_get_num_sdma_engines(dqm->dev) *
+			dqm->dev->kfd->device_info.num_reserved_sdma_queues_per_engine);
+	bitmap_clear(dqm->xgmi_sdma_bitmap, 0, kfd_get_num_xgmi_sdma_engines(dqm->dev) *
+			dqm->dev->kfd->device_info.num_reserved_sdma_queues_per_engine);
 }
 
 void program_sh_mem_settings(struct device_queue_manager *dqm,
@@ -254,6 +255,8 @@ static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
 	queue_input.queue_type = (uint32_t)queue_type;
 
 	queue_input.exclusively_scheduled = q->properties.is_gws;
+	queue_input.sh_mem_config_data = qpd->sh_mem_config;
+	queue_input.vm_cntx_cntl = qpd->vm_cntx_cntl;
 
 	amdgpu_mes_lock(&adev->mes);
 	r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
@@ -399,8 +402,7 @@ static void increment_queue_count(struct device_queue_manager *dqm,
 				  struct queue *q)
 {
 	dqm->active_queue_count++;
-	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
-	    q->properties.type == KFD_QUEUE_TYPE_DIQ)
+	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
 		dqm->active_cp_queue_count++;
 
 	if (q->properties.is_gws) {
@@ -414,8 +416,7 @@ static void decrement_queue_count(struct device_queue_manager *dqm,
 				  struct queue *q)
 {
 	dqm->active_queue_count--;
-	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
-	    q->properties.type == KFD_QUEUE_TYPE_DIQ)
+	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
 		dqm->active_cp_queue_count--;
 
 	if (q->properties.is_gws) {
@@ -856,8 +857,7 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
 	int retval;
 	struct mqd_manager *mqd_mgr;
 
-	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
-			q->properties.type)];
+	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)];
 
 	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
 		deallocate_hqd(dqm, q);
@@ -1443,13 +1443,12 @@ static int register_process(struct device_queue_manager *dqm,
 static int unregister_process(struct device_queue_manager *dqm,
 					struct qcm_process_device *qpd)
 {
-	int retval;
+	int retval = 0;
 	struct device_process_node *cur, *next;
 
 	pr_debug("qpd->queues_list is %s\n",
 			list_empty(&qpd->queues_list) ? "empty" : "not empty");
 
-	retval = 0;
 	dqm_lock(dqm);
 
 	list_for_each_entry_safe(cur, next, &dqm->queues, list) {
@@ -1479,7 +1478,7 @@ set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid,
 			unsigned int vmid)
 {
 	uint32_t xcc_mask = dqm->dev->xcc_mask;
-	int xcc_id, ret;
+	int xcc_id, ret = 0;
 
 	for_each_inst(xcc_id, xcc_mask) {
 		ret = dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
@@ -1831,8 +1830,6 @@ static int start_cpsch(struct device_queue_manager *dqm)
 	struct device *dev = dqm->dev->adev->dev;
 	int retval, num_hw_queue_slots;
 
-	retval = 0;
-
 	dqm_lock(dqm);
 
 	if (!dqm->dev->kfd->shared_resources.enable_mes) {
@@ -2784,7 +2781,7 @@ static int checkpoint_mqd(struct device_queue_manager *dqm,
 static int process_termination_cpsch(struct device_queue_manager *dqm,
 		struct qcm_process_device *qpd)
 {
-	int retval;
+	int retval = 0;
 	struct queue *q;
 	struct device *dev = dqm->dev->adev->dev;
 	struct kernel_queue *kq, *kq_next;
@@ -2794,8 +2791,6 @@ static int process_termination_cpsch(struct device_queue_manager *dqm,
 		KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
 	bool found = false;
 
-	retval = 0;
-
 	dqm_lock(dqm);
 
 	/* Clean all kernel queues */
@@ -3014,7 +3009,9 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
 		break;
 
 	default:
-		if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 0, 0))
+		if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 1, 0))
+			device_queue_manager_init_v12_1(&dqm->asic_ops);
+		else if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 0, 0))
 			device_queue_manager_init_v12(&dqm->asic_ops);
 		else if (KFD_GC_VERSION(dev) >= IP_VERSION(11, 0, 0))
 			device_queue_manager_init_v11(&dqm->asic_ops);
@@ -3464,7 +3461,6 @@ int suspend_queues(struct kfd_process *p,
 					else
 						per_device_suspended++;
 				} else if (err != -EBUSY) {
-					r = err;
 					queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK;
 					break;
 				}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index 74a61b5b2f0b..ef07e44916f8 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -299,6 +299,8 @@ void device_queue_manager_init_v11(
 		struct device_queue_manager_asic_ops *asic_ops);
 void device_queue_manager_init_v12(
 		struct device_queue_manager_asic_ops *asic_ops);
+void device_queue_manager_init_v12_1(
+		struct device_queue_manager_asic_ops *asic_ops);
 void program_sh_mem_settings(struct device_queue_manager *dqm,
 					struct qcm_process_device *qpd);
 unsigned int get_cp_queues_num(struct device_queue_manager *dqm);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12_1.c
new file mode 100644
index 000000000000..9e70a5f8a50b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12_1.c
@@ -0,0 +1,98 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "kfd_device_queue_manager.h"
+#include "gc/gc_12_1_0_sh_mask.h"
+#include "soc_v1_0_enum.h"
+
+static int update_qpd_v12_1(struct device_queue_manager *dqm,
+			 struct qcm_process_device *qpd);
+static void init_sdma_vm_v12_1(struct device_queue_manager *dqm, struct queue *q,
+			    struct qcm_process_device *qpd);
+
+void device_queue_manager_init_v12_1(
+	struct device_queue_manager_asic_ops *asic_ops)
+{
+	asic_ops->update_qpd = update_qpd_v12_1;
+	asic_ops->init_sdma_vm = init_sdma_vm_v12_1;
+	asic_ops->mqd_manager_init = mqd_manager_init_v12_1;
+}
+
+static uint32_t compute_sh_mem_bases_64bit(struct kfd_process_device *pdd)
+{
+	uint32_t shared_base = pdd->lds_base >> 48;
+	uint32_t private_base = pdd->scratch_base >> 58;
+
+	return (shared_base << SH_MEM_BASES__SHARED_BASE__SHIFT) |
+		(private_base << SH_MEM_BASES__PRIVATE_BASE__SHIFT);
+}
+
+static int update_qpd_v12_1(struct device_queue_manager *dqm,
+			 struct qcm_process_device *qpd)
+{
+	struct kfd_process_device *pdd;
+	struct amdgpu_device *adev = dqm->dev->adev;
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
+
+	pdd = qpd_to_pdd(qpd);
+	qpd->vm_cntx_cntl = hub->vm_cntx_cntl;
+
+	/* check if sh_mem_config register already configured */
+	if (qpd->sh_mem_config == 0) {
+		qpd->sh_mem_config =
+			(SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
+				SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) |
+			(3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT);
+
+		qpd->sh_mem_config |=
+			(1 << SH_MEM_CONFIG__F8_MODE__SHIFT);
+		qpd->sh_mem_ape1_limit = 0;
+		qpd->sh_mem_ape1_base = 0;
+	}
+
+	if (KFD_SUPPORT_XNACK_PER_PROCESS(dqm->dev)) {
+		if (!pdd->process->xnack_enabled) {
+			qpd->sh_mem_config |= 1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
+			qpd->vm_cntx_cntl &=
+			~(1 << GCVM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT);
+		} else {
+			qpd->sh_mem_config &= ~(1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT);
+			qpd->vm_cntx_cntl |=
+			(1 << GCVM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT);
+		}
+	}
+
+	qpd->sh_mem_bases = compute_sh_mem_bases_64bit(pdd);
+
+	pr_debug("sh_mem_bases 0x%X\n", qpd->sh_mem_bases);
+
+	return 0;
+}
+
+static void init_sdma_vm_v12_1(struct device_queue_manager *dqm, struct queue *q,
+			    struct qcm_process_device *qpd)
+{
+	/* Not needed on SDMAv4 onwards any more */
+	q->properties.sdma_vm_addr = 0;
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
index 1d170dc50df3..557a5ade329a 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
@@ -342,14 +342,20 @@ static void kfd_init_apertures_vi(struct kfd_process_device *pdd, uint8_t id)
 
 static void kfd_init_apertures_v9(struct kfd_process_device *pdd, uint8_t id)
 {
-	pdd->lds_base = MAKE_LDS_APP_BASE_V9();
+	if (pdd->dev->adev->vm_manager.root_level == AMDGPU_VM_PDB3)
+		pdd->lds_base = pdd->dev->adev->gmc.shared_aperture_start;
+	else
+		pdd->lds_base = MAKE_LDS_APP_BASE_V9();
 	pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base);
 
 	pdd->gpuvm_base = AMDGPU_VA_RESERVED_BOTTOM;
 	pdd->gpuvm_limit =
 		pdd->dev->kfd->shared_resources.gpuvm_size - 1;
 
-	pdd->scratch_base = MAKE_SCRATCH_APP_BASE_V9();
+	if (pdd->dev->adev->vm_manager.root_level == AMDGPU_VM_PDB3)
+		pdd->scratch_base = pdd->dev->adev->gmc.private_aperture_start;
+	else
+		pdd->scratch_base = MAKE_SCRATCH_APP_BASE_V9();
 	pdd->scratch_limit = MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base);
 
 	/*
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c
new file mode 100644
index 000000000000..230e57dbcec5
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v12_1.c
@@ -0,0 +1,391 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "kfd_priv.h"
+#include "kfd_events.h"
+#include "soc15_int.h"
+#include "kfd_device_queue_manager.h"
+#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
+#include "kfd_smi_events.h"
+#include "kfd_debug.h"
+
+/*
+ * GFX12.1 SQ Interrupts
+ *
+ * There are 3 encoding types of interrupts sourced from SQ sent as a 44-bit
+ * packet to the Interrupt Handler:
+ * Auto - Generated by the SQG (various cmd overflows, timestamps etc)
+ * Wave - Generated by S_SENDMSG through a shader program
+ * Error - HW generated errors (Illegal instructions, Memviols, EDC etc)
+ *
+ * The 44-bit packet is mapped as {context_id1[7:0],context_id0[31:0]} plus
+ * 4-bits for VMID (SOC15_VMID_FROM_IH_ENTRY) as such:
+ *
+ * - context_id1[7:6]
+ * Encoding type (0 = Auto, 1 = Wave, 2 = Error)
+ *
+ * - context_id0[26]
+ * PRIV bit indicates that Wave S_SEND or error occurred within trap
+ *
+ * - context_id0[24:0]
+ * 25-bit data with the following layout per encoding type:
+ * Auto - only context_id0[8:0] is used, which reports various interrupts
+ * generated by SQG.  The rest is 0.
+ * Wave - user data sent from m0 via S_SENDMSG (context_id0[23:0])
+ * Error - Error Type (context_id0[24:21]), Error Details (context_id0[20:0])
+ *
+ * The other context_id bits show coordinates (SE/SH/CU/SIMD/WGP) for wave
+ * S_SENDMSG and Errors.  These are 0 for Auto.
+ */
+
+enum SQ_INTERRUPT_WORD_ENCODING {
+	SQ_INTERRUPT_WORD_ENCODING_AUTO = 0x0,
+	SQ_INTERRUPT_WORD_ENCODING_INST,
+	SQ_INTERRUPT_WORD_ENCODING_ERROR,
+};
+
+enum SQ_INTERRUPT_ERROR_TYPE {
+	SQ_INTERRUPT_ERROR_TYPE_EDC_FUE = 0x0,
+	SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST,
+	SQ_INTERRUPT_ERROR_TYPE_MEMVIOL,
+	SQ_INTERRUPT_ERROR_TYPE_EDC_FED,
+};
+
+/* SQ_INTERRUPT_WORD_AUTO_CTXID */
+#define SQ_INTERRUPT_WORD_AUTO_CTXID0__THREAD_TRACE__SHIFT		0
+#define SQ_INTERRUPT_WORD_AUTO_CTXID0__WLT__SHIFT			1
+#define SQ_INTERRUPT_WORD_AUTO_CTXID0__THREAD_TRACE_BUF0_FULL__SHIFT	2
+#define SQ_INTERRUPT_WORD_AUTO_CTXID0__THREAD_TRACE_BUF1_FULL__SHIFT	3
+#define SQ_INTERRUPT_WORD_AUTO_CTXID0__THREAD_TRACE_UTC_ERROR__SHIFT	8
+#define SQ_INTERRUPT_WORD_AUTO_CTXID1__ENCODING__SHIFT			6
+
+#define SQ_INTERRUPT_WORD_AUTO_CTXID0__THREAD_TRACE_MASK		0x00000001
+#define SQ_INTERRUPT_WORD_AUTO_CTXID0__WLT_MASK				0x00000002
+#define SQ_INTERRUPT_WORD_AUTO_CTXID0__THREAD_TRACE_BUF0_FULL_MASK	0x00000004
+#define SQ_INTERRUPT_WORD_AUTO_CTXID0__THREAD_TRACE_BUF1_FULL_MASK	0x00000008
+#define SQ_INTERRUPT_WORD_AUTO_CTXID0__THREAD_TRACE_UTC_ERROR_MASK	0x00000100
+#define SQ_INTERRUPT_WORD_AUTO_CTXID1__ENCODING_MASK			0x000000c0
+
+/* SQ_INTERRUPT_WORD_WAVE_CTXID */
+#define SQ_INTERRUPT_WORD_WAVE_CTXID0__DATA__SHIFT	0
+#define SQ_INTERRUPT_WORD_WAVE_CTXID0__SA_ID__SHIFT	25
+#define SQ_INTERRUPT_WORD_WAVE_CTXID0__PRIV__SHIFT	26
+#define SQ_INTERRUPT_WORD_WAVE_CTXID0__WAVE_ID__SHIFT	27
+#define SQ_INTERRUPT_WORD_WAVE_CTXID1__SIMD_ID__SHIFT	0
+#define SQ_INTERRUPT_WORD_WAVE_CTXID1__WGP_ID__SHIFT	2
+#define SQ_INTERRUPT_WORD_WAVE_CTXID1__ENCODING__SHIFT	6
+
+#define SQ_INTERRUPT_WORD_WAVE_CTXID0__DATA_MASK	0x00ffffff /* [23:0] */
+#define SQ_INTERRUPT_WORD_WAVE_CTXID0__SA_ID_MASK	0x02000000 /* [25] */
+#define SQ_INTERRUPT_WORD_WAVE_CTXID0__PRIV_MASK	0x04000000 /* [26] */
+#define SQ_INTERRUPT_WORD_WAVE_CTXID0__WAVE_ID_MASK	0xf8000000 /* [31:27] */
+#define SQ_INTERRUPT_WORD_WAVE_CTXID1__SIMD_ID_MASK	0x00000003 /* [33:32] */
+#define SQ_INTERRUPT_WORD_WAVE_CTXID1__WGP_ID_MASK	0x0000003c /* [37:34] */
+#define SQ_INTERRUPT_WORD_WAVE_CTXID1__ENCODING_MASK	0x000000c0 /* [39:38] */
+
+/* SQ_INTERRUPT_WORD_ERROR_CTXID */
+#define SQ_INTERRUPT_WORD_ERROR_CTXID0__DETAIL__SHIFT	0
+#define SQ_INTERRUPT_WORD_ERROR_CTXID0__MEM_VIOL__SHIFT	19
+#define SQ_INTERRUPT_WORD_ERROR_CTXID0__TYPE__SHIFT	21
+#define SQ_INTERRUPT_WORD_ERROR_CTXID0__SA_ID__SHIFT	25
+#define SQ_INTERRUPT_WORD_ERROR_CTXID0__PRIV__SHIFT	26
+#define SQ_INTERRUPT_WORD_ERROR_CTXID0__WAVE_ID__SHIFT	27
+#define SQ_INTERRUPT_WORD_ERROR_CTXID1__SIMD_ID__SHIFT	0
+#define SQ_INTERRUPT_WORD_ERROR_CTXID1__WGP_ID__SHIFT	2
+#define SQ_INTERRUPT_WORD_ERROR_CTXID1__ENCODING__SHIFT	6
+
+#define SQ_INTERRUPT_WORD_ERROR_CTXID0__DETAIL_MASK	0x0007ffff /* [18:0] */
+#define SQ_INTERRUPT_WORD_ERROR_CTXID0__MEM_VIOL_MASK	0x00180000 /* [20:19] */
+#define SQ_INTERRUPT_WORD_ERROR_CTXID0__TYPE_MASK	0x01e00000 /* [24:21] */
+#define SQ_INTERRUPT_WORD_ERROR_CTXID0__SA_ID_MASK	0x02000000 /* [25] */
+#define SQ_INTERRUPT_WORD_ERROR_CTXID0__PRIV_MASK	0x04000000 /* [26] */
+#define SQ_INTERRUPT_WORD_ERROR_CTXID0__WAVE_ID_MASK	0xf8000000 /* [31:27] */
+#define SQ_INTERRUPT_WORD_ERROR_CTXID1__SIMD_ID_MASK	0x00000003 /* [33:32] */
+#define SQ_INTERRUPT_WORD_ERROR_CTXID1__WGP_ID_MASK	0x0000003c /* [37:34] */
+#define SQ_INTERRUPT_WORD_ERROR_CTXID1__ENCODING_MASK	0x000000c0 /* [39:38] */
+
+/*
+ * The debugger will send user data(m0) with PRIV=1 to indicate it requires
+ * notification from the KFD with the following queue id (DOORBELL_ID) and
+ * trap code (TRAP_CODE).
+ */
+#define KFD_CTXID0_TRAP_CODE_SHIFT	10
+#define KFD_CTXID0_TRAP_CODE_MASK	0xfffc00
+#define KFD_CTXID0_CP_BAD_OP_ECODE_MASK	0x3ffffff
+#define KFD_CTXID0_DOORBELL_ID_MASK	0x0003ff
+
+#define KFD_CTXID0_TRAP_CODE(ctxid0)		(((ctxid0) &  \
+				KFD_CTXID0_TRAP_CODE_MASK) >> \
+				KFD_CTXID0_TRAP_CODE_SHIFT)
+#define KFD_CTXID0_CP_BAD_OP_ECODE(ctxid0)	(((ctxid0) &        \
+				KFD_CTXID0_CP_BAD_OP_ECODE_MASK) >> \
+				KFD_CTXID0_TRAP_CODE_SHIFT)
+#define KFD_CTXID0_DOORBELL_ID(ctxid0)		((ctxid0) & \
+				KFD_CTXID0_DOORBELL_ID_MASK)
+
+static void print_sq_intr_info_auto(uint32_t context_id0, uint32_t context_id1)
+{
+	pr_debug_ratelimited(
+		"sq_intr: auto, ttrace %d, wlt %d, ttrace_buf0_full %d, ttrace_buf1_full %d ttrace_utc_err %d\n",
+		REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID0, THREAD_TRACE),
+		REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID0, WLT),
+		REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID0, THREAD_TRACE_BUF0_FULL),
+		REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID0, THREAD_TRACE_BUF1_FULL),
+		REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID0, THREAD_TRACE_UTC_ERROR));
+}
+
+static void print_sq_intr_info_inst(uint32_t context_id0, uint32_t context_id1)
+{
+	pr_debug_ratelimited(
+		"sq_intr: inst, data 0x%08x, sh %d, priv %d, wave_id %d, simd_id %d, wgp_id %d\n",
+		REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID0, DATA),
+		REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID0, SA_ID),
+		REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID0, PRIV),
+		REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID0, WAVE_ID),
+		REG_GET_FIELD(context_id1, SQ_INTERRUPT_WORD_WAVE_CTXID1, SIMD_ID),
+		REG_GET_FIELD(context_id1, SQ_INTERRUPT_WORD_WAVE_CTXID1, WGP_ID));
+}
+
+static void print_sq_intr_info_error(uint32_t context_id0, uint32_t context_id1)
+{
+	pr_debug_ratelimited(
+		"sq_intr: error, detail 0x%08x, type %d, sh %d, priv %d, wave_id %d, simd_id %d, wgp_id %d\n",
+		REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_ERROR_CTXID0, DETAIL),
+		REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_ERROR_CTXID0, TYPE),
+		REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_ERROR_CTXID0, SA_ID),
+		REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_ERROR_CTXID0, PRIV),
+		REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_ERROR_CTXID0, WAVE_ID),
+		REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_ERROR_CTXID1, SIMD_ID),
+		REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_ERROR_CTXID1, WGP_ID));
+}
+
+static void event_interrupt_poison_consumption_v12_1(struct kfd_node *node,
+				uint16_t pasid, uint16_t source_id)
+{
+	enum amdgpu_ras_block block = 0;
+	int ret = -EINVAL;
+	uint32_t reset = 0;
+	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, NULL);
+
+	if (!p)
+		return;
+
+	/* all queues of a process will be unmapped in one time */
+	if (atomic_read(&p->poison)) {
+		kfd_unref_process(p);
+		return;
+	}
+
+	atomic_set(&p->poison, 1);
+	kfd_unref_process(p);
+
+	switch (source_id) {
+	case SOC15_INTSRC_SQ_INTERRUPT_MSG:
+		if (node->dqm->ops.reset_queues)
+			ret = node->dqm->ops.reset_queues(node->dqm, pasid);
+		block = AMDGPU_RAS_BLOCK__GFX;
+		if (ret)
+			reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
+		break;
+	case SOC21_INTSRC_SDMA_ECC:
+	default:
+		block = AMDGPU_RAS_BLOCK__GFX;
+		reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
+		break;
+	}
+
+	kfd_signal_poison_consumed_event(node, pasid);
+
+	/*
+	 * resetting queue passes, do page retirement without gpu reset
+	 * resetting queue fails, fallback to gpu reset solution
+	 */
+	amdgpu_amdkfd_ras_poison_consumption_handler(node->adev, block, reset);
+}
+
+static bool event_interrupt_isr_v12_1(struct kfd_node *node,
+					const uint32_t *ih_ring_entry,
+					uint32_t *patched_ihre,
+					bool *patched_flag)
+{
+	uint16_t source_id, client_id, pasid, vmid, node_id;
+	const uint32_t *data = ih_ring_entry;
+	uint32_t context_id0;
+
+	node_id = SOC15_NODEID_FROM_IH_ENTRY(ih_ring_entry);
+	vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
+
+	if (!kfd_irq_is_from_node(node, node_id, vmid)) {
+		pr_debug("Interrupt not for Node, node_id: %d, vmid: %d\n", node_id, vmid);
+		return false;
+	}
+
+	source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry);
+	client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
+
+	/* Only handle interrupts from KFD VMIDs */
+	if (!KFD_IRQ_IS_FENCE(client_id, source_id) &&
+	    (vmid < node->vm_info.first_vmid_kfd ||
+	    vmid > node->vm_info.last_vmid_kfd))
+		return false;
+
+	pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry);
+	context_id0 = SOC15_CONTEXT_ID0_FROM_IH_ENTRY(ih_ring_entry);
+
+	if ((source_id == SOC15_INTSRC_CP_END_OF_PIPE) &&
+	    (context_id0 & AMDGPU_FENCE_MES_QUEUE_FLAG))
+		return false;
+
+	pr_debug("client id 0x%x, source id %d, vmid %d, pasid 0x%x. raw data:\n",
+		 client_id, source_id, vmid, pasid);
+	pr_debug("%8X, %8X, %8X, %8X, %8X, %8X, %8X, %8X.\n",
+		 data[0], data[1], data[2], data[3],
+		 data[4], data[5], data[6], data[7]);
+
+	/* If there is no valid PASID, it's likely a bug */
+	if (WARN_ONCE(pasid == 0, "Bug: No PASID in KFD interrupt"))
+		return false;
+
+	/* Interrupt types we care about: various signals and faults.
+	 * They will be forwarded to a work queue (see below).
+	 */
+	return source_id == SOC15_INTSRC_CP_END_OF_PIPE ||
+		source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG ||
+		source_id == SOC15_INTSRC_CP_BAD_OPCODE ||
+		source_id == SOC21_INTSRC_SDMA_TRAP ||
+		KFD_IRQ_IS_FENCE(client_id, source_id) ||
+		((client_id == SOC21_IH_CLIENTID_VMC ||
+		 client_id == SOC21_IH_CLIENTID_UTCL2) &&
+		  !amdgpu_no_queue_eviction_on_vm_fault);
+}
+
+static void event_interrupt_wq_v12_1(struct kfd_node *node,
+					const uint32_t *ih_ring_entry)
+{
+	uint16_t source_id, client_id, ring_id, pasid, vmid;
+	uint32_t context_id0, context_id1;
+	uint8_t sq_int_enc, sq_int_priv, sq_int_errtype;
+	struct kfd_vm_fault_info info = {0};
+	struct kfd_hsa_memory_exception_data exception_data;
+
+	source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry);
+	client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
+	ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry);
+	pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry);
+	vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
+	context_id0 = SOC15_CONTEXT_ID0_FROM_IH_ENTRY(ih_ring_entry);
+	context_id1 = SOC15_CONTEXT_ID1_FROM_IH_ENTRY(ih_ring_entry);
+
+	/* VMC, UTCL2 */
+	if (client_id == SOC21_IH_CLIENTID_VMC ||
+	    client_id == SOC21_IH_CLIENTID_UTCL2) {
+		info.vmid = vmid;
+		info.mc_id = client_id;
+		info.page_addr = ih_ring_entry[4] |
+			(uint64_t)(ih_ring_entry[5] & 0xf) << 32;
+		info.prot_valid = ring_id & 0x08;
+		info.prot_read  = ring_id & 0x10;
+		info.prot_write = ring_id & 0x20;
+
+		memset(&exception_data, 0, sizeof(exception_data));
+		exception_data.gpu_id = node->id;
+		exception_data.va = (info.page_addr) << PAGE_SHIFT;
+		exception_data.failure.NotPresent = info.prot_valid ? 1 : 0;
+		exception_data.failure.NoExecute = info.prot_exec ? 1 : 0;
+		exception_data.failure.ReadOnly = info.prot_write ? 1 : 0;
+		exception_data.failure.imprecise = 0;
+
+		kfd_set_dbg_ev_from_interrupt(node, pasid, -1,
+					      KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION),
+					      &exception_data, sizeof(exception_data));
+		kfd_smi_event_update_vmfault(node, pasid);
+
+	/* GRBM, SDMA, SE, PMM */
+	} else if (client_id == SOC21_IH_CLIENTID_GRBM_CP ||
+		   client_id == SOC21_IH_CLIENTID_GFX) {
+
+		/* CP */
+		if (source_id == SOC15_INTSRC_CP_END_OF_PIPE)
+			kfd_signal_event_interrupt(pasid, context_id0, 32);
+		else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE &&
+			 KFD_DBG_EC_TYPE_IS_PACKET(KFD_CTXID0_CP_BAD_OP_ECODE(context_id0))) {
+			u32 doorbell_id = KFD_CTXID0_DOORBELL_ID(context_id0);
+
+			kfd_set_dbg_ev_from_interrupt(node, pasid, doorbell_id,
+				KFD_EC_MASK(KFD_CTXID0_CP_BAD_OP_ECODE(context_id0)),
+				NULL, 0);
+			kfd_dqm_suspend_bad_queue_mes(node, pasid, doorbell_id);
+		}
+
+		/* SDMA */
+		else if (source_id == SOC21_INTSRC_SDMA_TRAP)
+			kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28);
+		else if (source_id == SOC21_INTSRC_SDMA_ECC) {
+			event_interrupt_poison_consumption_v12_1(node, pasid, source_id);
+			return;
+		}
+
+		/* SQ */
+		else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG) {
+			sq_int_enc = REG_GET_FIELD(context_id1,
+					SQ_INTERRUPT_WORD_WAVE_CTXID1, ENCODING);
+			switch (sq_int_enc) {
+			case SQ_INTERRUPT_WORD_ENCODING_AUTO:
+				print_sq_intr_info_auto(context_id0, context_id1);
+				break;
+			case SQ_INTERRUPT_WORD_ENCODING_INST:
+				print_sq_intr_info_inst(context_id0, context_id1);
+				sq_int_priv = REG_GET_FIELD(context_id0,
+						SQ_INTERRUPT_WORD_WAVE_CTXID0, PRIV);
+				if (sq_int_priv && (kfd_set_dbg_ev_from_interrupt(node, pasid,
+						KFD_CTXID0_DOORBELL_ID(context_id0),
+						KFD_CTXID0_TRAP_CODE(context_id0),
+						NULL, 0)))
+					return;
+				break;
+			case SQ_INTERRUPT_WORD_ENCODING_ERROR:
+				print_sq_intr_info_error(context_id0, context_id1);
+				sq_int_errtype = REG_GET_FIELD(context_id0,
+						SQ_INTERRUPT_WORD_ERROR_CTXID0, TYPE);
+				if (sq_int_errtype != SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST &&
+				    sq_int_errtype != SQ_INTERRUPT_ERROR_TYPE_MEMVIOL) {
+					event_interrupt_poison_consumption_v12_1(
+							node, pasid, source_id);
+					return;
+				}
+				break;
+			default:
+				break;
+			}
+			kfd_signal_event_interrupt(pasid, context_id0 & 0xffffff, 24);
+		}
+
+	} else if (KFD_IRQ_IS_FENCE(client_id, source_id)) {
+		kfd_process_close_interrupt_drain(pasid);
+	}
+}
+
+const struct kfd_event_interrupt_class event_interrupt_class_v12_1 = {
+	.interrupt_isr = event_interrupt_isr_v12_1,
+	.interrupt_wq = event_interrupt_wq_v12_1,
+};
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
index fb3129883a4c..baeabe3feef8 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
@@ -57,19 +57,7 @@ static bool kq_initialize(struct kernel_queue *kq, struct kfd_node *dev,
 
 	kq->dev = dev;
 	kq->nop_packet = nop.u32all;
-	switch (type) {
-	case KFD_QUEUE_TYPE_DIQ:
-		kq->mqd_mgr = dev->dqm->mqd_mgrs[KFD_MQD_TYPE_DIQ];
-		break;
-	case KFD_QUEUE_TYPE_HIQ:
-		kq->mqd_mgr = dev->dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ];
-		break;
-	default:
-		WARN(1, "Invalid queue type %d\n", type);
-		dev_err(dev->adev->dev, "Invalid queue type %d\n", type);
-		return false;
-	}
-
+	kq->mqd_mgr = dev->dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ];
 	if (!kq->mqd_mgr)
 		return false;
 
@@ -141,7 +129,6 @@ static bool kq_initialize(struct kernel_queue *kq, struct kfd_node *dev,
 		goto err_init_queue;
 
 	kq->queue->device = dev;
-	kq->queue->process = kfd_get_process(current);
 
 	kq->queue->mqd_mem_obj = kq->mqd_mgr->allocate_mqd(kq->mqd_mgr->dev,
 					&kq->queue->properties);
@@ -159,24 +146,11 @@ static bool kq_initialize(struct kernel_queue *kq, struct kfd_node *dev,
 		kq->mqd_mgr->load_mqd(kq->mqd_mgr, kq->queue->mqd,
 				kq->queue->pipe, kq->queue->queue,
 				&kq->queue->properties, NULL);
-	} else {
-		/* allocate fence for DIQ */
-
-		retval = kfd_gtt_sa_allocate(dev, sizeof(uint32_t),
-						&kq->fence_mem_obj);
-
-		if (retval != 0)
-			goto err_alloc_fence;
-
-		kq->fence_kernel_address = kq->fence_mem_obj->cpu_ptr;
-		kq->fence_gpu_addr = kq->fence_mem_obj->gpu_addr;
 	}
 
 	print_queue(kq->queue);
 
 	return true;
-err_alloc_fence:
-	kq->mqd_mgr->free_mqd(kq->mqd_mgr, kq->queue->mqd, kq->queue->mqd_mem_obj);
 err_allocate_mqd:
 	uninit_queue(kq->queue);
 err_init_queue:
@@ -206,8 +180,6 @@ static void kq_uninitialize(struct kernel_queue *kq)
 					kq->queue->queue);
 		up_read(&kq->dev->adev->reset_domain->sem);
 	}
-	else if (kq->queue->properties.type == KFD_QUEUE_TYPE_DIQ)
-		kfd_gtt_sa_free(kq->dev, kq->fence_mem_obj);
 
 	kq->mqd_mgr->free_mqd(kq->mqd_mgr, kq->queue->mqd,
 				kq->queue->mqd_mem_obj);
@@ -355,34 +327,3 @@ void kernel_queue_uninit(struct kernel_queue *kq)
 	kq_uninitialize(kq);
 	kfree(kq);
 }
-
-/* FIXME: Can this test be removed? */
-static __attribute__((unused)) void test_kq(struct kfd_node *dev)
-{
-	struct kernel_queue *kq;
-	uint32_t *buffer, i;
-	int retval;
-
-	dev_err(dev->adev->dev, "Starting kernel queue test\n");
-
-	kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_HIQ);
-	if (unlikely(!kq)) {
-		dev_err(dev->adev->dev, "  Failed to initialize HIQ\n");
-		dev_err(dev->adev->dev, "Kernel queue test failed\n");
-		return;
-	}
-
-	retval = kq_acquire_packet_buffer(kq, 5, &buffer);
-	if (unlikely(retval != 0)) {
-		dev_err(dev->adev->dev, "  Failed to acquire packet buffer\n");
-		dev_err(dev->adev->dev, "Kernel queue test failed\n");
-		return;
-	}
-	for (i = 0; i < 5; i++)
-		buffer[i] = kq->nop_packet;
-	kq_submit_packet(kq);
-
-	dev_err(dev->adev->dev, "Ending kernel queue test\n");
-}
-
-
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index af53e796ea1b..a3b1722108b7 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -45,7 +45,9 @@ svm_migrate_direct_mapping_addr(struct amdgpu_device *adev, u64 addr)
 }
 
 static int
-svm_migrate_gart_map(struct amdgpu_ring *ring, u64 npages,
+svm_migrate_gart_map(struct amdgpu_ring *ring,
+		     struct amdgpu_ttm_buffer_entity *entity,
+		     u64 npages,
 		     dma_addr_t *addr, u64 *gart_addr, u64 flags)
 {
 	struct amdgpu_device *adev = ring->adev;
@@ -63,7 +65,7 @@ svm_migrate_gart_map(struct amdgpu_ring *ring, u64 npages,
 	num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
 	num_bytes = npages * 8;
 
-	r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr,
+	r = amdgpu_job_alloc_with_ib(adev, &entity->base,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     num_dw * 4 + num_bytes,
 				     AMDGPU_IB_POOL_DELAYED,
@@ -128,11 +130,14 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
 {
 	const u64 GTT_MAX_PAGES = AMDGPU_GTT_MAX_TRANSFER_SIZE;
 	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+	struct amdgpu_ttm_buffer_entity *entity;
 	u64 gart_s, gart_d;
 	struct dma_fence *next;
 	u64 size;
 	int r;
 
+	entity = &adev->mman.default_entity;
+
 	mutex_lock(&adev->mman.gtt_window_lock);
 
 	while (npages) {
@@ -140,10 +145,10 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
 
 		if (direction == FROM_VRAM_TO_RAM) {
 			gart_s = svm_migrate_direct_mapping_addr(adev, *vram);
-			r = svm_migrate_gart_map(ring, size, sys, &gart_d, 0);
+			r = svm_migrate_gart_map(ring, entity, size, sys, &gart_d, 0);
 
 		} else if (direction == FROM_RAM_TO_VRAM) {
-			r = svm_migrate_gart_map(ring, size, sys, &gart_s,
+			r = svm_migrate_gart_map(ring, entity, size, sys, &gart_s,
 						 KFD_IOCTL_SVM_FLAG_GPU_RO);
 			gart_d = svm_migrate_direct_mapping_addr(adev, *vram);
 		}
@@ -152,8 +157,9 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
 			goto out_unlock;
 		}
 
-		r = amdgpu_copy_buffer(ring, gart_s, gart_d, size * PAGE_SIZE,
-				       NULL, &next, false, true, 0);
+		r = amdgpu_copy_buffer(adev, entity,
+				       gart_s, gart_d, size * PAGE_SIZE,
+				       NULL, &next, true, 0);
 		if (r) {
 			dev_err(adev->dev, "fail %d to copy memory\n", r);
 			goto out_unlock;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
new file mode 100644
index 000000000000..06ecc86fcb4c
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
@@ -0,0 +1,657 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/printk.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include "kfd_priv.h"
+#include "kfd_mqd_manager.h"
+#include "v12_structs.h"
+#include "gc/gc_12_1_0_sh_mask.h"
+#include "amdgpu_amdkfd.h"
+#include "kfd_device_queue_manager.h"
+
+#define MQD_SIZE	(2 * PAGE_SIZE)
+
+static uint64_t mqd_stride_v12_1(struct mqd_manager *mm,
+				struct queue_properties *q)
+{
+	if (q->type == KFD_QUEUE_TYPE_COMPUTE)
+		return MQD_SIZE;
+	else
+		return PAGE_SIZE;
+}
+
+static inline struct v12_1_compute_mqd *get_mqd(void *mqd)
+{
+	return (struct v12_1_compute_mqd *)mqd;
+}
+
+static inline struct v12_sdma_mqd *get_sdma_mqd(void *mqd)
+{
+	return (struct v12_sdma_mqd *)mqd;
+}
+
+static void update_cu_mask(struct mqd_manager *mm, void *mqd,
+			   struct mqd_update_info *minfo, uint32_t inst)
+{
+	struct v12_1_compute_mqd *m;
+	uint32_t se_mask[KFD_MAX_NUM_SE] = {0};
+
+	if (!minfo || !minfo->cu_mask.ptr)
+		return;
+
+	mqd_symmetrically_map_cu_mask(mm,
+		minfo->cu_mask.ptr, minfo->cu_mask.count, se_mask, inst);
+
+	m = get_mqd(mqd);
+	m->compute_static_thread_mgmt_se0 = se_mask[0];
+	m->compute_static_thread_mgmt_se1 = se_mask[1];
+	m->compute_static_thread_mgmt_se2 = se_mask[2];
+	m->compute_static_thread_mgmt_se3 = se_mask[3];
+	m->compute_static_thread_mgmt_se4 = se_mask[4];
+	m->compute_static_thread_mgmt_se5 = se_mask[5];
+	m->compute_static_thread_mgmt_se6 = se_mask[6];
+	m->compute_static_thread_mgmt_se7 = se_mask[7];
+
+	pr_debug("update cu mask to %#x %#x %#x %#x %#x %#x %#x %#x\n",
+		m->compute_static_thread_mgmt_se0,
+		m->compute_static_thread_mgmt_se1,
+		m->compute_static_thread_mgmt_se2,
+		m->compute_static_thread_mgmt_se3,
+		m->compute_static_thread_mgmt_se4,
+		m->compute_static_thread_mgmt_se5,
+		m->compute_static_thread_mgmt_se6,
+		m->compute_static_thread_mgmt_se7);
+}
+
+static void set_priority(struct v12_1_compute_mqd *m, struct queue_properties *q)
+{
+	m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
+	m->cp_hqd_queue_priority = q->priority;
+}
+
+static struct kfd_mem_obj *allocate_mqd(struct kfd_node *node,
+		struct queue_properties *q)
+{
+	struct kfd_mem_obj *mqd_mem_obj;
+	unsigned int size;
+
+	/*
+	 * Allocate two PAGE_SIZE memory for Compute MQD as MES writes to areas beyond
+	 * struct MQD size. Size of the Compute MQD is 1 PAGE_SIZE.
+	 * For SDMA MQD, we allocate 1 Page_size.
+	 */
+	if (q->type == KFD_QUEUE_TYPE_COMPUTE)
+		size = MQD_SIZE * NUM_XCC(node->xcc_mask);
+	else
+		size = PAGE_SIZE;
+
+	if (kfd_gtt_sa_allocate(node, size, &mqd_mem_obj))
+		return NULL;
+
+	return mqd_mem_obj;
+}
+
+static void init_mqd(struct mqd_manager *mm, void **mqd,
+			struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+			struct queue_properties *q)
+{
+	uint64_t addr;
+	struct v12_1_compute_mqd *m;
+
+	m = (struct v12_1_compute_mqd *) mqd_mem_obj->cpu_ptr;
+	addr = mqd_mem_obj->gpu_addr;
+
+	memset(m, 0, MQD_SIZE);
+
+	m->header = 0xC0310800;
+	m->compute_pipelinestat_enable = 1;
+	m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
+	m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
+	m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
+	m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
+	m->compute_static_thread_mgmt_se4 = 0xFFFFFFFF;
+	m->compute_static_thread_mgmt_se5 = 0xFFFFFFFF;
+	m->compute_static_thread_mgmt_se6 = 0xFFFFFFFF;
+	m->compute_static_thread_mgmt_se7 = 0xFFFFFFFF;
+	m->compute_static_thread_mgmt_se8 = 0xFFFFFFFF;
+
+	m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK |
+			0x63 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT;
+
+	m->cp_mqd_control = 1 << CP_MQD_CONTROL__PRIV_STATE__SHIFT;
+
+	m->cp_mqd_base_addr_lo	= lower_32_bits(addr);
+	m->cp_mqd_base_addr_hi	= upper_32_bits(addr);
+
+	m->cp_hqd_quantum = 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT |
+			1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT |
+			1 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT;
+
+	/* Set cp_hqd_hq_status0.c_queue_debug_en to 1 to have the CP set up the
+	 * DISPATCH_PTR.  This is required for the kfd debugger
+	 */
+	m->cp_hqd_hq_status0 = 1 << 14;
+
+	if (amdgpu_amdkfd_have_atomics_support(mm->dev->adev))
+		m->cp_hqd_hq_status0 |= 1 << 29;
+
+	if (q->format == KFD_QUEUE_FORMAT_AQL) {
+		m->cp_hqd_aql_control =
+			1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
+	}
+
+	if (mm->dev->kfd->cwsr_enabled) {
+		m->cp_hqd_persistent_state |=
+			(1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);
+		m->cp_hqd_ctx_save_base_addr_lo =
+			lower_32_bits(q->ctx_save_restore_area_address);
+		m->cp_hqd_ctx_save_base_addr_hi =
+			upper_32_bits(q->ctx_save_restore_area_address);
+		m->cp_hqd_ctx_save_size = q->ctx_save_restore_area_size;
+		m->cp_hqd_cntl_stack_size = q->ctl_stack_size;
+		m->cp_hqd_cntl_stack_offset = q->ctl_stack_size;
+		m->cp_hqd_wg_state_offset = q->ctl_stack_size;
+	}
+
+	*mqd = m;
+	if (gart_addr)
+		*gart_addr = addr;
+	mm->update_mqd(mm, m, q, NULL);
+}
+
+static int load_mqd(struct mqd_manager *mm, void *mqd,
+			uint32_t pipe_id, uint32_t queue_id,
+			struct queue_properties *p, struct mm_struct *mms)
+{
+	int r = 0;
+	/* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
+	uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
+
+	r = mm->dev->kfd2kgd->hqd_load(mm->dev->adev, mqd, pipe_id, queue_id,
+					  (uint32_t __user *)p->write_ptr,
+					  wptr_shift, 0, mms, 0);
+	return r;
+}
+
+static void update_mqd(struct mqd_manager *mm, void *mqd,
+		       struct queue_properties *q,
+		       struct mqd_update_info *minfo)
+{
+	struct v12_1_compute_mqd *m;
+
+	m = get_mqd(mqd);
+
+	m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT;
+	m->cp_hqd_pq_control |=
+			ffs(q->queue_size / sizeof(unsigned int)) - 1 - 1;
+	m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__UNORD_DISPATCH_MASK;
+	pr_debug("cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control);
+
+	m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
+	m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
+
+	m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+	m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+	m->cp_hqd_pq_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr);
+	m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr);
+
+	m->cp_hqd_pq_doorbell_control =
+		q->doorbell_off <<
+			CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+	pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
+			m->cp_hqd_pq_doorbell_control);
+
+	m->cp_hqd_ib_control = 1 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT;
+
+	/*
+	 * HW does not clamp this field correctly. Maximum EOP queue size
+	 * is constrained by per-SE EOP done signal count, which is 8-bit.
+	 * Limit is 0xFF EOP entries (= 0x7F8 dwords). CP will not submit
+	 * more than (EOP entry count - 1) so a queue size of 0x800 dwords
+	 * is safe, giving a maximum field value of 0xA.
+	 */
+	m->cp_hqd_eop_control = min(0xA,
+		ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1);
+	m->cp_hqd_eop_base_addr_lo =
+			lower_32_bits(q->eop_ring_buffer_address >> 8);
+	m->cp_hqd_eop_base_addr_hi =
+			upper_32_bits(q->eop_ring_buffer_address >> 8);
+
+	m->cp_hqd_iq_timer = 0;
+
+	m->cp_hqd_vmid = q->vmid;
+
+	if (q->format == KFD_QUEUE_FORMAT_AQL) {
+		/* GC 10 removed WPP_CLAMP from PQ Control */
+		m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK |
+				2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT |
+				1 << CP_HQD_PQ_CONTROL__QUEUE_FULL_EN__SHIFT;
+		m->cp_hqd_pq_doorbell_control |=
+			1 << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT;
+	}
+	if (mm->dev->kfd->cwsr_enabled)
+		m->cp_hqd_ctx_save_control = 0;
+
+	set_priority(m, q);
+
+	q->is_active = QUEUE_IS_ACTIVE(*q);
+}
+
+static bool check_preemption_failed(struct mqd_manager *mm, void *mqd)
+{
+	return false;
+}
+
+static int get_wave_state(struct mqd_manager *mm, void *mqd,
+			  struct queue_properties *q,
+			  void __user *ctl_stack,
+			  u32 *ctl_stack_used_size,
+			  u32 *save_area_used_size)
+{
+	struct v12_1_compute_mqd *m;
+	struct mqd_user_context_save_area_header header;
+
+	m = get_mqd(mqd);
+
+	/* Control stack is written backwards, while workgroup context data
+	 * is written forwards. Both starts from m->cp_hqd_cntl_stack_size.
+	 * Current position is at m->cp_hqd_cntl_stack_offset and
+	 * m->cp_hqd_wg_state_offset, respectively.
+	 */
+	*ctl_stack_used_size = m->cp_hqd_cntl_stack_size -
+		m->cp_hqd_cntl_stack_offset;
+	*save_area_used_size = m->cp_hqd_wg_state_offset -
+		m->cp_hqd_cntl_stack_size;
+
+	/* Control stack is not copied to user mode for GFXv12 because
+	 * it's part of the context save area that is already
+	 * accessible to user mode
+	 */
+	header.control_stack_size = *ctl_stack_used_size;
+	header.wave_state_size = *save_area_used_size;
+
+	header.wave_state_offset = m->cp_hqd_wg_state_offset;
+	header.control_stack_offset = m->cp_hqd_cntl_stack_offset;
+
+	if (copy_to_user(ctl_stack, &header, sizeof(header)))
+		return -EFAULT;
+
+	return 0;
+}
+
+static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
+			struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+			struct queue_properties *q)
+{
+	struct v12_1_compute_mqd *m;
+
+	init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
+
+	m = get_mqd(*mqd);
+
+	m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT |
+			1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
+}
+
+static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,
+		struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+		struct queue_properties *q)
+{
+	struct v12_sdma_mqd *m;
+
+	m = (struct v12_sdma_mqd *) mqd_mem_obj->cpu_ptr;
+
+	memset(m, 0, PAGE_SIZE);
+
+	*mqd = m;
+	if (gart_addr)
+		*gart_addr = mqd_mem_obj->gpu_addr;
+
+	mm->update_mqd(mm, m, q, NULL);
+}
+
+#define SDMA_RLC_DUMMY_DEFAULT 0xf
+
+static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,
+		struct queue_properties *q,
+		struct mqd_update_info *minfo)
+{
+	struct v12_sdma_mqd *m;
+
+	m = get_sdma_mqd(mqd);
+	m->sdmax_rlcx_rb_cntl = (ffs(q->queue_size / sizeof(unsigned int)) - 1)
+		<< SDMA0_SDMA_QUEUE0_RB_CNTL__RB_SIZE__SHIFT |
+		q->vmid << SDMA0_SDMA_QUEUE0_RB_CNTL__RB_VMID__SHIFT |
+		1 << SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
+		6 << SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT |
+		1 << SDMA0_SDMA_QUEUE0_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT;
+
+	m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8);
+	m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8);
+	m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+	m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+	m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr);
+	m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr);
+	m->sdmax_rlcx_doorbell_offset =
+		q->doorbell_off << SDMA0_SDMA_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
+
+	m->sdma_engine_id = q->sdma_engine_id;
+	m->sdma_queue_id = q->sdma_queue_id;
+
+	m->sdmax_rlcx_dummy_reg = SDMA_RLC_DUMMY_DEFAULT;
+
+	q->is_active = QUEUE_IS_ACTIVE(*q);
+}
+
+static void get_xcc_mqd(struct kfd_mem_obj *mqd_mem_obj,
+			       struct kfd_mem_obj *xcc_mqd_mem_obj,
+			       uint64_t offset)
+{
+	xcc_mqd_mem_obj->gtt_mem = (offset == 0) ?
+					mqd_mem_obj->gtt_mem : NULL;
+	xcc_mqd_mem_obj->gpu_addr = mqd_mem_obj->gpu_addr + offset;
+	xcc_mqd_mem_obj->cpu_ptr = (uint32_t *)((uintptr_t)mqd_mem_obj->cpu_ptr
+						+ offset);
+}
+
+static void init_mqd_v12_1(struct mqd_manager *mm, void **mqd,
+			struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+			struct queue_properties *q)
+{
+	struct v12_1_compute_mqd *m;
+	int xcc = 0;
+	struct kfd_mem_obj xcc_mqd_mem_obj;
+	uint64_t xcc_gart_addr = 0;
+	uint64_t xcc_ctx_save_restore_area_address;
+	uint64_t offset = mm->mqd_stride(mm, q);
+	uint32_t local_xcc_start = mm->dev->dqm->current_logical_xcc_start++;
+
+	memset(&xcc_mqd_mem_obj, 0x0, sizeof(struct kfd_mem_obj));
+	for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) {
+		get_xcc_mqd(mqd_mem_obj, &xcc_mqd_mem_obj, offset*xcc);
+
+		init_mqd(mm, (void **)&m, &xcc_mqd_mem_obj, &xcc_gart_addr, q);
+
+		m->cp_mqd_stride_size = offset;
+
+		/*
+		 * Update the CWSR address for each XCC if CWSR is enabled
+		 * and CWSR area is allocated in thunk
+		 */
+		if (mm->dev->kfd->cwsr_enabled &&
+		    q->ctx_save_restore_area_address) {
+			xcc_ctx_save_restore_area_address =
+				q->ctx_save_restore_area_address +
+				(xcc * q->ctx_save_restore_area_size);
+
+			m->cp_hqd_ctx_save_base_addr_lo =
+				lower_32_bits(xcc_ctx_save_restore_area_address);
+			m->cp_hqd_ctx_save_base_addr_hi =
+				upper_32_bits(xcc_ctx_save_restore_area_address);
+		}
+
+		if (q->format == KFD_QUEUE_FORMAT_AQL) {
+			m->compute_tg_chunk_size = 1;
+			m->compute_current_logical_xcc_id =
+					(local_xcc_start + xcc) %
+					NUM_XCC(mm->dev->xcc_mask);
+		} else {
+			/* PM4 Queue */
+			m->compute_current_logical_xcc_id = 0;
+			m->compute_tg_chunk_size = 0;
+			m->pm4_target_xcc_in_xcp = q->pm4_target_xcc;
+		}
+
+		if (xcc == 0) {
+			/* Set the MQD pointer and gart address to XCC0 MQD */
+			*mqd = m;
+			*gart_addr = xcc_gart_addr;
+		}
+	}
+}
+
+static void update_mqd_v12_1(struct mqd_manager *mm, void *mqd,
+		      struct queue_properties *q, struct mqd_update_info *minfo)
+{
+	struct v12_1_compute_mqd *m;
+	int xcc = 0;
+	uint64_t size = mm->mqd_stride(mm, q);
+
+	for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) {
+		m = get_mqd(mqd + size * xcc);
+		update_mqd(mm, m, q, minfo);
+
+		update_cu_mask(mm, m, minfo, xcc);
+
+		if (q->format == KFD_QUEUE_FORMAT_AQL) {
+			m->compute_tg_chunk_size = 1;
+		} else {
+			/* PM4 Queue */
+			m->compute_current_logical_xcc_id = 0;
+			m->compute_tg_chunk_size = 0;
+			m->pm4_target_xcc_in_xcp = q->pm4_target_xcc;
+		}
+	}
+}
+
+static int destroy_mqd_v12_1(struct mqd_manager *mm, void *mqd,
+		   enum kfd_preempt_type type, unsigned int timeout,
+		   uint32_t pipe_id, uint32_t queue_id)
+{
+	uint32_t xcc_mask = mm->dev->xcc_mask;
+	int xcc_id, err, inst = 0;
+	void *xcc_mqd;
+	struct v12_1_compute_mqd *m;
+	uint64_t mqd_offset;
+
+	m = get_mqd(mqd);
+	mqd_offset = m->cp_mqd_stride_size;
+
+	for_each_inst(xcc_id, xcc_mask) {
+		xcc_mqd = mqd + mqd_offset * inst;
+		err = mm->dev->kfd2kgd->hqd_destroy(mm->dev->adev, xcc_mqd,
+						    type, timeout, pipe_id,
+						    queue_id, xcc_id);
+		if (err) {
+			pr_debug("Destroy MQD failed for xcc: %d\n", inst);
+			break;
+		}
+		++inst;
+	}
+
+	return err;
+}
+
+static int load_mqd_v12_1(struct mqd_manager *mm, void *mqd,
+			uint32_t pipe_id, uint32_t queue_id,
+			struct queue_properties *p, struct mm_struct *mms)
+{
+	/* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
+	uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
+	uint32_t xcc_mask = mm->dev->xcc_mask;
+	int xcc_id, err, inst = 0;
+	void *xcc_mqd;
+	uint64_t mqd_stride_size = mm->mqd_stride(mm, p);
+
+	for_each_inst(xcc_id, xcc_mask) {
+		xcc_mqd = mqd + mqd_stride_size * inst;
+		err = mm->dev->kfd2kgd->hqd_load(
+			mm->dev->adev, xcc_mqd, pipe_id, queue_id,
+			(uint32_t __user *)p->write_ptr, wptr_shift, 0, mms,
+			xcc_id);
+		if (err) {
+			pr_debug("Load MQD failed for xcc: %d\n", inst);
+			break;
+		}
+		++inst;
+	}
+
+	return err;
+}
+
+static int get_wave_state_v12_1(struct mqd_manager *mm, void *mqd,
+				 struct queue_properties *q,
+				 void __user *ctl_stack,
+				 u32 *ctl_stack_used_size,
+				 u32 *save_area_used_size)
+{
+	int xcc, err = 0;
+	void *xcc_mqd;
+	void __user *xcc_ctl_stack;
+	uint64_t mqd_stride_size = mm->mqd_stride(mm, q);
+	u32 tmp_ctl_stack_used_size = 0, tmp_save_area_used_size = 0;
+
+	for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) {
+		xcc_mqd = mqd + mqd_stride_size * xcc;
+		xcc_ctl_stack = (void __user *)((uintptr_t)ctl_stack +
+					q->ctx_save_restore_area_size * xcc);
+
+		err = get_wave_state(mm, xcc_mqd, q, xcc_ctl_stack,
+				     &tmp_ctl_stack_used_size,
+				     &tmp_save_area_used_size);
+		if (err)
+			break;
+
+		/*
+		 * Set the ctl_stack_used_size and save_area_used_size to
+		 * ctl_stack_used_size and save_area_used_size of XCC 0 when
+		 * passing the info to user-space.
+		 * For multi XCC, user-space would have to look at the header
+		 * info of each Control stack area to determine the control
+		 * stack size and save area used.
+		 */
+		if (xcc == 0) {
+			*ctl_stack_used_size = tmp_ctl_stack_used_size;
+			*save_area_used_size = tmp_save_area_used_size;
+		}
+	}
+
+	return err;
+}
+
+#if defined(CONFIG_DEBUG_FS)
+
+static int debugfs_show_mqd(struct seq_file *m, void *data)
+{
+	seq_hex_dump(m, "    ", DUMP_PREFIX_OFFSET, 32, 4,
+		     data, sizeof(struct v12_1_compute_mqd), false);
+	return 0;
+}
+
+static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
+{
+	seq_hex_dump(m, "    ", DUMP_PREFIX_OFFSET, 32, 4,
+		     data, sizeof(struct v12_sdma_mqd), false);
+	return 0;
+}
+
+#endif
+
+struct mqd_manager *mqd_manager_init_v12_1(enum KFD_MQD_TYPE type,
+		struct kfd_node *dev)
+{
+	struct mqd_manager *mqd;
+
+	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
+		return NULL;
+
+	mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
+	if (!mqd)
+		return NULL;
+
+	mqd->dev = dev;
+
+	switch (type) {
+	case KFD_MQD_TYPE_CP:
+		pr_debug("%s@%i\n", __func__, __LINE__);
+		mqd->allocate_mqd = allocate_mqd;
+		mqd->init_mqd = init_mqd_v12_1;
+		mqd->free_mqd = kfd_free_mqd_cp;
+		mqd->load_mqd = load_mqd_v12_1;
+		mqd->update_mqd = update_mqd_v12_1;
+		mqd->destroy_mqd = destroy_mqd_v12_1;
+		mqd->is_occupied = kfd_is_occupied_cp;
+		mqd->mqd_size = sizeof(struct v12_1_compute_mqd);
+		mqd->get_wave_state = get_wave_state_v12_1;
+		mqd->mqd_stride = mqd_stride_v12_1;
+#if defined(CONFIG_DEBUG_FS)
+		mqd->debugfs_show_mqd = debugfs_show_mqd;
+#endif
+		pr_debug("%s@%i\n", __func__, __LINE__);
+		break;
+	case KFD_MQD_TYPE_HIQ:
+		pr_debug("%s@%i\n", __func__, __LINE__);
+		mqd->allocate_mqd = allocate_hiq_mqd;
+		mqd->init_mqd = init_mqd_hiq;
+		mqd->free_mqd = free_mqd_hiq_sdma;
+		mqd->load_mqd = kfd_hiq_load_mqd_kiq;
+		mqd->update_mqd = update_mqd;
+		mqd->destroy_mqd = kfd_destroy_mqd_cp;
+		mqd->is_occupied = kfd_is_occupied_cp;
+		mqd->mqd_size = sizeof(struct v12_1_compute_mqd);
+		mqd->mqd_stride = kfd_mqd_stride;
+#if defined(CONFIG_DEBUG_FS)
+		mqd->debugfs_show_mqd = debugfs_show_mqd;
+#endif
+		mqd->check_preemption_failed = check_preemption_failed;
+		pr_debug("%s@%i\n", __func__, __LINE__);
+		break;
+	case KFD_MQD_TYPE_DIQ:
+		mqd->allocate_mqd = allocate_mqd;
+		mqd->init_mqd = init_mqd_hiq;
+		mqd->free_mqd = kfd_free_mqd_cp;
+		mqd->load_mqd = load_mqd;
+		mqd->update_mqd = update_mqd;
+		mqd->destroy_mqd = kfd_destroy_mqd_cp;
+		mqd->is_occupied = kfd_is_occupied_cp;
+		mqd->mqd_size = sizeof(struct v12_1_compute_mqd);
+#if defined(CONFIG_DEBUG_FS)
+		mqd->debugfs_show_mqd = debugfs_show_mqd;
+#endif
+		break;
+	case KFD_MQD_TYPE_SDMA:
+		pr_debug("%s@%i\n", __func__, __LINE__);
+		mqd->allocate_mqd = allocate_mqd;
+		mqd->init_mqd = init_mqd_sdma;
+		mqd->free_mqd = kfd_free_mqd_cp;
+		mqd->load_mqd = kfd_load_mqd_sdma;
+		mqd->update_mqd = update_mqd_sdma;
+		mqd->destroy_mqd = kfd_destroy_mqd_sdma;
+		mqd->is_occupied = kfd_is_occupied_sdma;
+		mqd->mqd_size = sizeof(struct v12_sdma_mqd);
+		mqd->mqd_stride = kfd_mqd_stride;
+#if defined(CONFIG_DEBUG_FS)
+		mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
+#endif
+		pr_debug("%s@%i\n", __func__, __LINE__);
+		break;
+	default:
+		kfree(mqd);
+		return NULL;
+	}
+
+	return mqd;
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index f2dee320fada..2e9b6bcf2704 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -596,7 +596,7 @@ static int hiq_load_mqd_kiq_v9_4_3(struct mqd_manager *mm, void *mqd,
 			struct queue_properties *p, struct mm_struct *mms)
 {
 	uint32_t xcc_mask = mm->dev->xcc_mask;
-	int xcc_id, err, inst = 0;
+	int xcc_id, err = 0, inst = 0;
 	void *xcc_mqd;
 	uint64_t hiq_mqd_size = kfd_hiq_mqd_stride(mm->dev);
 
@@ -620,7 +620,7 @@ static int destroy_hiq_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
 			uint32_t pipe_id, uint32_t queue_id)
 {
 	uint32_t xcc_mask = mm->dev->xcc_mask;
-	int xcc_id, err, inst = 0;
+	int xcc_id, err = 0, inst = 0;
 	uint64_t hiq_mqd_size = kfd_hiq_mqd_stride(mm->dev);
 	struct v9_mqd *m;
 	u32 doorbell_off;
@@ -818,7 +818,7 @@ static int destroy_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
 		   uint32_t pipe_id, uint32_t queue_id)
 {
 	uint32_t xcc_mask = mm->dev->xcc_mask;
-	int xcc_id, err, inst = 0;
+	int xcc_id, err = 0, inst = 0;
 	void *xcc_mqd;
 	struct v9_mqd *m;
 	uint64_t mqd_offset;
@@ -848,7 +848,7 @@ static int load_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
 	/* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
 	uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
 	uint32_t xcc_mask = mm->dev->xcc_mask;
-	int xcc_id, err, inst = 0;
+	int xcc_id, err = 0, inst = 0;
 	void *xcc_mqd;
 	uint64_t mqd_stride_size = mm->mqd_stride(mm, p);
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c
index 505036968a77..3d2375817c3e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c
@@ -252,10 +252,6 @@ static int pm_map_queues_v9(struct packet_manager *pm, uint32_t *buffer,
 			packet->bitfields2.queue_type =
 		queue_type__mes_map_queues__normal_latency_static_queue_vi;
 		break;
-	case KFD_QUEUE_TYPE_DIQ:
-		packet->bitfields2.queue_type =
-			queue_type__mes_map_queues__debug_interface_queue_vi;
-		break;
 	case KFD_QUEUE_TYPE_SDMA:
 	case KFD_QUEUE_TYPE_SDMA_XGMI:
 		if (q->properties.sdma_engine_id < 2 &&
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_vi.c
index a1de5d7e173a..1f0b8ef7c966 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_vi.c
@@ -166,15 +166,10 @@ static int pm_map_queues_vi(struct packet_manager *pm, uint32_t *buffer,
 			packet->bitfields2.queue_type =
 		queue_type__mes_map_queues__normal_latency_static_queue_vi;
 		break;
-	case KFD_QUEUE_TYPE_DIQ:
-		packet->bitfields2.queue_type =
-			queue_type__mes_map_queues__debug_interface_queue_vi;
-		break;
 	case KFD_QUEUE_TYPE_SDMA:
 	case KFD_QUEUE_TYPE_SDMA_XGMI:
 		packet->bitfields2.engine_sel = q->properties.sdma_engine_id +
 				engine_sel__mes_map_queues__sdma0_vi;
-		use_static = false; /* no static queues under SDMA */
 		break;
 	default:
 		WARN(1, "queue type %d", q->properties.type);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 70ef051511bb..ebc637c38c04 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -215,7 +215,8 @@ enum cache_policy {
 	((KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2)) ||	\
 	 (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3)) ||	\
 	 (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 4)) ||	\
-	 (KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0)))
+	 (KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0)) ||	\
+	 (KFD_GC_VERSION(dev) == IP_VERSION(12, 1, 0)))
 
 struct kfd_node;
 
@@ -241,7 +242,6 @@ struct kfd_device_info {
 	uint32_t no_atomic_fw_version;
 	unsigned int num_sdma_queues_per_engine;
 	unsigned int num_reserved_sdma_queues_per_engine;
-	DECLARE_BITMAP(reserved_sdma_queues_bitmap, KFD_MAX_SDMA_QUEUES);
 };
 
 unsigned int kfd_get_num_sdma_engines(struct kfd_node *kdev);
@@ -434,7 +434,6 @@ enum kfd_queue_type  {
 	KFD_QUEUE_TYPE_COMPUTE,
 	KFD_QUEUE_TYPE_SDMA,
 	KFD_QUEUE_TYPE_HIQ,
-	KFD_QUEUE_TYPE_DIQ,
 	KFD_QUEUE_TYPE_SDMA_XGMI,
 	KFD_QUEUE_TYPE_SDMA_BY_ENG_ID
 };
@@ -698,6 +697,7 @@ struct qcm_process_device {
 	uint32_t num_gws;
 	uint32_t num_oac;
 	uint32_t sh_hidden_private_base;
+	uint32_t vm_cntx_cntl;
 
 	/* CWSR memory */
 	struct kgd_mem *cwsr_mem;
@@ -1018,9 +1018,19 @@ struct kfd_process {
 
 	/* if gpu page fault sent to KFD */
 	bool gpu_page_fault;
+
+	/*kfd context id */
+	u16 context_id;
+
+	/* The primary kfd_process allocating IDs for its secondary kfd_process, 0 for primary kfd_process */
+	struct ida id_table;
+
 };
 
-#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
+#define KFD_PROCESS_TABLE_SIZE 8 /* bits: 256 entries */
+#define KFD_CONTEXT_ID_PRIMARY	0xFFFF
+#define KFD_CONTEXT_ID_MIN 0
+
 extern DECLARE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
 extern struct srcu_struct kfd_processes_srcu;
 
@@ -1045,14 +1055,16 @@ struct amdkfd_ioctl_desc {
 };
 bool kfd_dev_is_large_bar(struct kfd_node *dev);
 
+struct kfd_process *create_process(const struct task_struct *thread, bool primary);
 int kfd_process_create_wq(void);
 void kfd_process_destroy_wq(void);
 void kfd_cleanup_processes(void);
 struct kfd_process *kfd_create_process(struct task_struct *thread);
-struct kfd_process *kfd_get_process(const struct task_struct *task);
+int kfd_create_process_sysfs(struct kfd_process *process);
 struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid,
 						 struct kfd_process_device **pdd);
 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
+struct kfd_process *kfd_lookup_process_by_id(const struct mm_struct *mm, u16 id);
 
 int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id);
 int kfd_process_gpuid_from_node(struct kfd_process *p, struct kfd_node *node,
@@ -1090,6 +1102,7 @@ bool kfd_process_xnack_mode(struct kfd_process *p, bool supported);
 
 int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
 			  struct vm_area_struct *vma);
+void kfd_process_notifier_release_internal(struct kfd_process *p);
 
 /* KFD process API for creating and translating handles */
 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
@@ -1163,9 +1176,11 @@ static inline struct kfd_node *kfd_node_by_irq_ids(struct amdgpu_device *adev,
 	struct kfd_dev *dev = adev->kfd.dev;
 	uint32_t i;
 
-	if (KFD_GC_VERSION(dev) != IP_VERSION(9, 4, 3) &&
-	    KFD_GC_VERSION(dev) != IP_VERSION(9, 4, 4) &&
-	    KFD_GC_VERSION(dev) != IP_VERSION(9, 5, 0))
+	/*
+	 * On multi-aid system, attempt per-node matching. Otherwise,
+	 * fall back to the first node.
+	 */
+	if (!amdgpu_is_multi_aid(adev))
 		return dev->nodes[0];
 
 	for (i = 0; i < dev->num_nodes; i++)
@@ -1341,6 +1356,8 @@ struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type,
 		struct kfd_node *dev);
 struct mqd_manager *mqd_manager_init_v12(enum KFD_MQD_TYPE type,
 		struct kfd_node *dev);
+struct mqd_manager *mqd_manager_init_v12_1(enum KFD_MQD_TYPE type,
+		struct kfd_node *dev);
 struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev);
 void device_queue_manager_uninit(struct device_queue_manager *dqm);
 struct kernel_queue *kernel_queue_init(struct kfd_node *dev,
@@ -1494,6 +1511,7 @@ extern const struct kfd_event_interrupt_class event_interrupt_class_v9;
 extern const struct kfd_event_interrupt_class event_interrupt_class_v9_4_3;
 extern const struct kfd_event_interrupt_class event_interrupt_class_v10;
 extern const struct kfd_event_interrupt_class event_interrupt_class_v11;
+extern const struct kfd_event_interrupt_class event_interrupt_class_v12_1;
 
 extern const struct kfd_device_global_init_class device_global_init_class_cik;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index a085faac9fe1..9d39f9451ff7 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -68,7 +68,6 @@ static struct workqueue_struct *kfd_restore_wq;
 static struct kfd_process *find_process(const struct task_struct *thread,
 					bool ref);
 static void kfd_process_ref_release(struct kref *ref);
-static struct kfd_process *create_process(const struct task_struct *thread);
 
 static void evict_process_worker(struct work_struct *work);
 static void restore_process_worker(struct work_struct *work);
@@ -825,6 +824,102 @@ static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd)
 	kfd_process_free_gpuvm(qpd->ib_mem, pdd, &qpd->ib_kaddr);
 }
 
+int kfd_create_process_sysfs(struct kfd_process *process)
+{
+	struct kfd_process *primary_process;
+	int ret;
+
+	if (process->kobj) {
+		pr_warn("kobject already exsists for the kfd_process\n");
+		return -EINVAL;
+	}
+
+	process->kobj = kfd_alloc_struct(process->kobj);
+	if (!process->kobj) {
+		pr_warn("Creating procfs kobject failed");
+		return -ENOMEM;
+	}
+
+	if (process->context_id == KFD_CONTEXT_ID_PRIMARY)
+		ret = kobject_init_and_add(process->kobj, &procfs_type,
+					   procfs.kobj, "%d",
+					   (int)process->lead_thread->pid);
+	else {
+		primary_process = kfd_lookup_process_by_mm(process->lead_thread->mm);
+		if (!primary_process)
+			return -ESRCH;
+
+		ret = kobject_init_and_add(process->kobj, &procfs_type,
+					   primary_process->kobj, "context_%u",
+					   process->context_id);
+		kfd_unref_process(primary_process);
+	}
+
+	if (ret) {
+		pr_warn("Creating procfs pid directory failed");
+		kobject_put(process->kobj);
+		return ret;
+	}
+
+	kfd_sysfs_create_file(process->kobj, &process->attr_pasid,
+			      "pasid");
+
+	process->kobj_queues = kobject_create_and_add("queues",
+						process->kobj);
+	if (!process->kobj_queues)
+		pr_warn("Creating KFD proc/queues folder failed");
+
+	kfd_procfs_add_sysfs_stats(process);
+	kfd_procfs_add_sysfs_files(process);
+	kfd_procfs_add_sysfs_counters(process);
+
+	return 0;
+}
+
+static int kfd_process_alloc_id(struct kfd_process *process)
+{
+	int ret;
+	struct kfd_process *primary_process;
+
+	/* already assign 0xFFFF when create */
+	if (process->context_id == KFD_CONTEXT_ID_PRIMARY)
+		return 0;
+
+	primary_process = kfd_lookup_process_by_mm(process->lead_thread->mm);
+	if (!primary_process)
+		return -ESRCH;
+
+	/* id range: KFD_CONTEXT_ID_MIN to 0xFFFE */
+	ret = ida_alloc_range(&primary_process->id_table, KFD_CONTEXT_ID_MIN,
+	      KFD_CONTEXT_ID_PRIMARY - 1, GFP_KERNEL);
+	if (ret < 0)
+		goto out;
+
+	process->context_id = ret;
+	ret = 0;
+
+out:
+	kfd_unref_process(primary_process);
+
+	return ret;
+}
+
+static void kfd_process_free_id(struct kfd_process *process)
+{
+	struct kfd_process *primary_process;
+
+	if (process->context_id != KFD_CONTEXT_ID_PRIMARY)
+		return;
+
+	primary_process = kfd_lookup_process_by_mm(process->lead_thread->mm);
+	if (!primary_process)
+		return;
+
+	ida_free(&primary_process->id_table, process->context_id);
+
+	kfd_unref_process(primary_process);
+}
+
 struct kfd_process *kfd_create_process(struct task_struct *thread)
 {
 	struct kfd_process *process;
@@ -867,38 +962,16 @@ struct kfd_process *kfd_create_process(struct task_struct *thread)
 	if (process) {
 		pr_debug("Process already found\n");
 	} else {
-		process = create_process(thread);
+		process = create_process(thread, true);
 		if (IS_ERR(process))
 			goto out;
 
 		if (!procfs.kobj)
 			goto out;
 
-		process->kobj = kfd_alloc_struct(process->kobj);
-		if (!process->kobj) {
-			pr_warn("Creating procfs kobject failed");
-			goto out;
-		}
-		ret = kobject_init_and_add(process->kobj, &procfs_type,
-					   procfs.kobj, "%d",
-					   (int)process->lead_thread->pid);
-		if (ret) {
-			pr_warn("Creating procfs pid directory failed");
-			kobject_put(process->kobj);
-			goto out;
-		}
-
-		kfd_sysfs_create_file(process->kobj, &process->attr_pasid,
-				      "pasid");
-
-		process->kobj_queues = kobject_create_and_add("queues",
-							process->kobj);
-		if (!process->kobj_queues)
-			pr_warn("Creating KFD proc/queues folder failed");
-
-		kfd_procfs_add_sysfs_stats(process);
-		kfd_procfs_add_sysfs_files(process);
-		kfd_procfs_add_sysfs_counters(process);
+		ret = kfd_create_process_sysfs(process);
+		if (ret)
+			pr_warn("Failed to create sysfs entry for the kfd_process");
 
 		kfd_debugfs_add_process(process);
 
@@ -911,31 +984,13 @@ struct kfd_process *kfd_create_process(struct task_struct *thread)
 	return process;
 }
 
-struct kfd_process *kfd_get_process(const struct task_struct *thread)
-{
-	struct kfd_process *process;
-
-	if (!thread->mm)
-		return ERR_PTR(-EINVAL);
-
-	/* Only the pthreads threading model is supported. */
-	if (thread->group_leader->mm != thread->mm)
-		return ERR_PTR(-EINVAL);
-
-	process = find_process(thread, false);
-	if (!process)
-		return ERR_PTR(-EINVAL);
-
-	return process;
-}
-
 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
 {
 	struct kfd_process *process;
 
 	hash_for_each_possible_rcu(kfd_processes_table, process,
 					kfd_processes, (uintptr_t)mm)
-		if (process->mm == mm)
+		if (process->mm == mm && process->context_id == KFD_CONTEXT_ID_PRIMARY)
 			return process;
 
 	return NULL;
@@ -1175,6 +1230,11 @@ static void kfd_process_wq_release(struct work_struct *work)
 	if (ef)
 		dma_fence_signal(ef);
 
+	if (p->context_id != KFD_CONTEXT_ID_PRIMARY)
+		kfd_process_free_id(p);
+	else
+		ida_destroy(&p->id_table);
+
 	kfd_process_remove_sysfs(p);
 	kfd_debugfs_remove_process(p);
 
@@ -1215,10 +1275,30 @@ static void kfd_process_free_notifier(struct mmu_notifier *mn)
 	kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
 }
 
-static void kfd_process_notifier_release_internal(struct kfd_process *p)
+static void kfd_process_table_remove(struct kfd_process *p)
+{
+	mutex_lock(&kfd_processes_mutex);
+	/*
+	 * Do early return if table is empty.
+	 *
+	 * This could potentially happen if this function is called concurrently
+	 * by mmu_notifier and by kfd_cleanup_pocesses.
+	 *
+	 */
+	if (hash_empty(kfd_processes_table)) {
+		mutex_unlock(&kfd_processes_mutex);
+		return;
+	}
+	hash_del_rcu(&p->kfd_processes);
+	mutex_unlock(&kfd_processes_mutex);
+	synchronize_srcu(&kfd_processes_srcu);
+}
+
+void kfd_process_notifier_release_internal(struct kfd_process *p)
 {
 	int i;
 
+	kfd_process_table_remove(p);
 	cancel_delayed_work_sync(&p->eviction_work);
 	cancel_delayed_work_sync(&p->restore_work);
 
@@ -1260,7 +1340,8 @@ static void kfd_process_notifier_release_internal(struct kfd_process *p)
 		srcu_read_unlock(&kfd_processes_srcu, idx);
 	}
 
-	mmu_notifier_put(&p->mmu_notifier);
+	if (p->context_id == KFD_CONTEXT_ID_PRIMARY)
+		mmu_notifier_put(&p->mmu_notifier);
 }
 
 static void kfd_process_notifier_release(struct mmu_notifier *mn,
@@ -1276,22 +1357,6 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn,
 	if (WARN_ON(p->mm != mm))
 		return;
 
-	mutex_lock(&kfd_processes_mutex);
-	/*
-	 * Do early return if table is empty.
-	 *
-	 * This could potentially happen if this function is called concurrently
-	 * by mmu_notifier and by kfd_cleanup_pocesses.
-	 *
-	 */
-	if (hash_empty(kfd_processes_table)) {
-		mutex_unlock(&kfd_processes_mutex);
-		return;
-	}
-	hash_del_rcu(&p->kfd_processes);
-	mutex_unlock(&kfd_processes_mutex);
-	synchronize_srcu(&kfd_processes_srcu);
-
 	kfd_process_notifier_release_internal(p);
 }
 
@@ -1492,7 +1557,8 @@ bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
 		 * management and memory-manager-related preemptions or
 		 * even deadlocks.
 		 */
-		if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
+		if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1) &&
+		    KFD_GC_VERSION(dev) < IP_VERSION(12, 1, 0))
 			return false;
 
 		if (dev->kfd->noretry)
@@ -1516,7 +1582,7 @@ void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
  * On return the kfd_process is fully operational and will be freed when the
  * mm is released
  */
-static struct kfd_process *create_process(const struct task_struct *thread)
+struct kfd_process *create_process(const struct task_struct *thread, bool primary)
 {
 	struct kfd_process *process;
 	struct mmu_notifier *mn;
@@ -1532,6 +1598,7 @@ static struct kfd_process *create_process(const struct task_struct *thread)
 	process->lead_thread = thread->group_leader;
 	process->n_pdds = 0;
 	process->queues_paused = false;
+
 	INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
 	INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
 	process->last_restore_timestamp = get_jiffies_64();
@@ -1575,12 +1642,22 @@ static struct kfd_process *create_process(const struct task_struct *thread)
 	 * After this point, mmu_notifier_put will trigger the cleanup by
 	 * dropping the last process reference in the free_notifier.
 	 */
-	mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm);
-	if (IS_ERR(mn)) {
-		err = PTR_ERR(mn);
-		goto err_register_notifier;
+	if (primary) {
+		process->context_id = KFD_CONTEXT_ID_PRIMARY;
+		mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm);
+		if (IS_ERR(mn)) {
+			err = PTR_ERR(mn);
+			goto err_register_notifier;
+		}
+		BUG_ON(mn != &process->mmu_notifier);
+		ida_init(&process->id_table);
+	}
+
+	err = kfd_process_alloc_id(process);
+	if (err) {
+		pr_err("Creating kfd process: failed to alloc an id\n");
+		goto err_alloc_id;
 	}
-	BUG_ON(mn != &process->mmu_notifier);
 
 	kfd_unref_process(process);
 	get_task_struct(process->lead_thread);
@@ -1589,6 +1666,8 @@ static struct kfd_process *create_process(const struct task_struct *thread)
 
 	return process;
 
+err_alloc_id:
+	kfd_process_free_id(process);
 err_register_notifier:
 	hash_del_rcu(&process->kfd_processes);
 	svm_range_list_fini(process);
@@ -1880,6 +1959,27 @@ struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
 	return p;
 }
 
+/* This increments the process->ref counter. */
+struct kfd_process *kfd_lookup_process_by_id(const struct mm_struct *mm, u16 id)
+{
+	struct kfd_process *p, *ret_p = NULL;
+	unsigned int temp;
+
+	int idx = srcu_read_lock(&kfd_processes_srcu);
+
+	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
+		if (p->mm == mm && p->context_id == id) {
+			kref_get(&p->ref);
+			ret_p = p;
+			break;
+		}
+	}
+
+	srcu_read_unlock(&kfd_processes_srcu, idx);
+
+	return ret_p;
+}
+
 /* kfd_process_evict_queues - Evict all user queues of a process
  *
  * Eviction is reference-counted per process-device. This means multiple
@@ -1987,10 +2087,10 @@ kfd_process_gpuid_from_node(struct kfd_process *p, struct kfd_node *node,
 	return -EINVAL;
 }
 
-static int signal_eviction_fence(struct kfd_process *p)
+static bool signal_eviction_fence(struct kfd_process *p)
 {
 	struct dma_fence *ef;
-	int ret;
+	bool ret;
 
 	rcu_read_lock();
 	ef = dma_fence_get_rcu_safe(&p->ef);
@@ -1998,7 +2098,7 @@ static int signal_eviction_fence(struct kfd_process *p)
 	if (!ef)
 		return -EINVAL;
 
-	ret = dma_fence_signal(ef);
+	ret = dma_fence_check_and_signal(ef);
 	dma_fence_put(ef);
 
 	return ret;
@@ -2179,7 +2279,8 @@ int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
 	 */
 	if (KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 3) ||
 	    KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 4) ||
-	    KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 5, 0)) {
+	    KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 5, 0) ||
+	    KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(12, 1, 0)) {
 		node_id = ffs(pdd->dev->interrupt_bitmap) - 1;
 		irq_drain_fence[3] |= node_id << 16;
 	}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index 7fbb5c274ccc..5f8cda4733f9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -94,7 +94,7 @@ void kfd_process_dequeue_from_device(struct kfd_process_device *pdd)
 	if (dev->kfd->shared_resources.enable_mes && !!pdd->proc_ctx_gpu_addr &&
 	    down_read_trylock(&dev->adev->reset_domain->sem)) {
 		amdgpu_mes_flush_shader_debugger(dev->adev,
-						 pdd->proc_ctx_gpu_addr);
+						 pdd->proc_ctx_gpu_addr, 0);
 		up_read(&dev->adev->reset_domain->sem);
 	}
 	pdd->already_dequeued = true;
@@ -345,7 +345,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 	 * If we are just about to create DIQ, the is_debug flag is not set yet
 	 * Hence we also check the type as well
 	 */
-	if ((pdd->qpd.is_debug) || (type == KFD_QUEUE_TYPE_DIQ))
+	if (pdd->qpd.is_debug)
 		max_queues = dev->kfd->device_info.max_no_of_hqd/2;
 
 	if (pdd->qpd.queue_count >= max_queues)
@@ -426,22 +426,6 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 						    restore_mqd, restore_ctl_stack);
 		print_queue(q);
 		break;
-	case KFD_QUEUE_TYPE_DIQ:
-		kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_DIQ);
-		if (!kq) {
-			retval = -ENOMEM;
-			goto err_create_queue;
-		}
-		kq->queue->properties.queue_id = *qid;
-		pqn->kq = kq;
-		pqn->q = NULL;
-		retval = kfd_process_drain_interrupts(pdd);
-		if (retval)
-			break;
-
-		retval = dev->dqm->ops.create_kernel_queue(dev->dqm,
-							kq, &pdd->qpd);
-		break;
 	default:
 		WARN(1, "Invalid queue type %d", type);
 		retval = -EINVAL;
@@ -1131,32 +1115,13 @@ int pqm_debugfs_mqds(struct seq_file *m, void *data)
 				break;
 			default:
 				seq_printf(m,
-				"  Bad user queue type %d on device %x\n",
+				"  Qeueu node with bad user queue type %d on device %x\n",
 					   q->properties.type, q->device->id);
 				continue;
 			}
 			mqd_mgr = q->device->dqm->mqd_mgrs[mqd_type];
 			size = mqd_mgr->mqd_stride(mqd_mgr,
 							&q->properties);
-		} else if (pqn->kq) {
-			q = pqn->kq->queue;
-			mqd_mgr = pqn->kq->mqd_mgr;
-			switch (q->properties.type) {
-			case KFD_QUEUE_TYPE_DIQ:
-				seq_printf(m, "  DIQ on device %x\n",
-					   pqn->kq->dev->id);
-				break;
-			default:
-				seq_printf(m,
-				"  Bad kernel queue type %d on device %x\n",
-					   q->properties.type,
-					   pqn->kq->dev->id);
-				continue;
-			}
-		} else {
-			seq_printf(m,
-		"  Weird: Queue node with neither kernel nor user queue\n");
-			continue;
 		}
 
 		for (xcc = 0; xcc < num_xccs; xcc++) {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
index 80c4fa2b0975..56c97189e7f1 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
@@ -392,12 +392,20 @@ int kfd_queue_unref_bo_vas(struct kfd_process_device *pdd,
 	return 0;
 }
 
-#define SGPR_SIZE_PER_CU	0x4000
-#define LDS_SIZE_PER_CU		0x10000
-#define HWREG_SIZE_PER_CU	0x1000
 #define DEBUGGER_BYTES_ALIGN	64
 #define DEBUGGER_BYTES_PER_WAVE	32
 
+static u32 kfd_get_sgpr_size_per_cu(u32 gfxv)
+{
+	u32 sgpr_size = 0x4000;
+
+	if (gfxv == 120500 ||
+	    gfxv == 120501)
+		sgpr_size = 0x8000;
+
+	return sgpr_size;
+}
+
 static u32 kfd_get_vgpr_size_per_cu(u32 gfxv)
 {
 	u32 vgpr_size = 0x40000;
@@ -413,14 +421,53 @@ static u32 kfd_get_vgpr_size_per_cu(u32 gfxv)
 		 gfxv == 120000 ||		/* GFX_VERSION_GFX1200 */
 		 gfxv == 120001)		/* GFX_VERSION_GFX1201 */
 		vgpr_size = 0x60000;
+	else if (gfxv == 120500 ||		/* GFX_VERSION_GFX1250 */
+		 gfxv == 120501)		/* GFX_VERSION_GFX1251 */
+		vgpr_size = 0x80000;
 
 	return vgpr_size;
 }
 
+static u32 kfd_get_hwreg_size_per_cu(u32 gfxv)
+{
+	u32 hwreg_size = 0x1000;
+
+	if (gfxv == 120500 || gfxv == 120501)
+		hwreg_size = 0x8000;
+
+	return hwreg_size;
+}
+
+static u32 kfd_get_lds_size_per_cu(u32 gfxv, struct kfd_node_properties *props)
+{
+	u32 lds_size = 0x10000;
+
+	if (gfxv == 90500 || gfxv == 120500 || gfxv == 120501)
+		lds_size = props->lds_size_in_kb << 10;
+
+	return lds_size;
+}
+
+static u32 get_num_waves(struct kfd_node_properties *props, u32 gfxv, u32 cu_num)
+{
+	u32 wave_num = 0;
+
+	if (gfxv < 100100)
+		wave_num = min(cu_num * 40,
+				props->array_count / props->simd_arrays_per_engine * 512);
+	else if (gfxv < 120500)
+		wave_num = cu_num * 32;
+	else if (gfxv <= 120501)
+		wave_num = cu_num * 64;
+
+	WARN_ON(wave_num == 0);
+
+	return wave_num;
+}
+
 #define WG_CONTEXT_DATA_SIZE_PER_CU(gfxv, props)	\
-	(kfd_get_vgpr_size_per_cu(gfxv) + SGPR_SIZE_PER_CU +\
-	 (((gfxv) == 90500) ? (props->lds_size_in_kb << 10) : LDS_SIZE_PER_CU) +\
-	 HWREG_SIZE_PER_CU)
+	(kfd_get_vgpr_size_per_cu(gfxv) + kfd_get_sgpr_size_per_cu(gfxv) +\
+	 kfd_get_lds_size_per_cu(gfxv, props) + kfd_get_hwreg_size_per_cu(gfxv))
 
 #define CNTL_STACK_BYTES_PER_WAVE(gfxv)	\
 	((gfxv) >= 100100 ? 12 : 8)	/* GFX_VERSION_NAVI10*/
@@ -440,9 +487,7 @@ void kfd_queue_ctx_save_restore_size(struct kfd_topology_device *dev)
 		return;
 
 	cu_num = props->simd_count / props->simd_per_cu / NUM_XCC(dev->gpu->xcc_mask);
-	wave_num = (gfxv < 100100) ?	/* GFX_VERSION_NAVI10 */
-		    min(cu_num * 40, props->array_count / props->simd_arrays_per_engine * 512)
-		    : cu_num * 32;
+	wave_num = get_num_waves(props, gfxv, cu_num);
 
 	wg_data_size = ALIGN(cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(gfxv, props), PAGE_SIZE);
 	ctl_stack_size = wave_num * CNTL_STACK_BYTES_PER_WAVE(gfxv) + 8;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c
index a499449fcb06..d2bc169e84b0 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c
@@ -312,7 +312,7 @@ void kfd_smi_event_queue_restore(struct kfd_node *node, pid_t pid)
 {
 	kfd_smi_event_add(pid, node, KFD_SMI_EVENT_QUEUE_RESTORE,
 			  KFD_EVENT_FMT_QUEUE_RESTORE(ktime_get_boottime_ns(), pid,
-			  node->id, 0));
+			  node->id, '0'));
 }
 
 void kfd_smi_event_queue_restore_rescheduled(struct mm_struct *mm)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 79ea138897fc..1ed08388d364 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -33,6 +33,7 @@
 #include "amdgpu_hmm.h"
 #include "amdgpu.h"
 #include "amdgpu_xgmi.h"
+#include "amdgpu_reset.h"
 #include "kfd_priv.h"
 #include "kfd_svm.h"
 #include "kfd_migrate.h"
@@ -585,7 +586,7 @@ svm_range_vram_node_new(struct kfd_node *node, struct svm_range *prange,
 	svm_bo->eviction_fence =
 		amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
 					   mm,
-					   svm_bo);
+					   svm_bo, p->context_id);
 	mmput(mm);
 	INIT_WORK(&svm_bo->eviction_work, svm_range_evict_svm_bo_worker);
 	svm_bo->evicting = 0;
@@ -1305,6 +1306,23 @@ svm_range_get_pte_flags(struct kfd_node *node, struct amdgpu_vm *vm,
 	case IP_VERSION(12, 0, 1):
 		mapping_flags |= AMDGPU_VM_MTYPE_NC;
 		break;
+	case IP_VERSION(12, 1, 0):
+		snoop = true;
+		if (domain == SVM_RANGE_VRAM_DOMAIN) {
+			mtype_local = amdgpu_mtype_local == 1 ? AMDGPU_VM_MTYPE_NC :
+								AMDGPU_VM_MTYPE_RW;
+			/* local HBM  */
+			if (bo_node->adev == node->adev)
+				mapping_flags |= mtype_local;
+			/* Remote GPU memory */
+			else
+				mapping_flags |= ext_coherent ? AMDGPU_VM_MTYPE_UC :
+								AMDGPU_VM_MTYPE_NC;
+		/* system memory accessed by the dGPU */
+		} else {
+			mapping_flags |= ext_coherent ? AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
+		}
+		break;
 	default:
 		mapping_flags |= coherent ?
 			AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
@@ -1323,6 +1341,11 @@ svm_range_get_pte_flags(struct kfd_node *node, struct amdgpu_vm *vm,
 	pte_flags |= AMDGPU_PTE_READABLE;
 	if (!(flags & KFD_IOCTL_SVM_FLAG_GPU_RO))
 		pte_flags |= AMDGPU_PTE_WRITEABLE;
+
+	if ((gc_ip_version == IP_VERSION(12, 1, 0)) &&
+	    node->adev->have_atomics_support)
+		pte_flags |= AMDGPU_PTE_BUS_ATOMICS;
+
 	return pte_flags;
 }
 
@@ -1331,7 +1354,7 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			 uint64_t start, uint64_t last,
 			 struct dma_fence **fence)
 {
-	uint64_t init_pte_value = 0;
+	uint64_t init_pte_value = adev->gmc.init_pte_flags;
 
 	pr_debug("[0x%llx 0x%llx]\n", start, last);
 
@@ -2349,6 +2372,9 @@ static void svm_range_drain_retry_fault(struct svm_range_list *svms)
 
 		pr_debug("drain retry fault gpu %d svms %p\n", i, svms);
 
+		if (!down_read_trylock(&pdd->dev->adev->reset_domain->sem))
+			continue;
+
 		amdgpu_ih_wait_on_checkpoint_process_ts(pdd->dev->adev,
 				pdd->dev->adev->irq.retry_cam_enabled ?
 				&pdd->dev->adev->irq.ih :
@@ -2358,6 +2384,7 @@ static void svm_range_drain_retry_fault(struct svm_range_list *svms)
 			amdgpu_ih_wait_on_checkpoint_process_ts(pdd->dev->adev,
 				&pdd->dev->adev->irq.ih_soft);
 
+		up_read(&pdd->dev->adev->reset_domain->sem);
 
 		pr_debug("drain retry fault gpu %d svms 0x%p done\n", i, svms);
 	}
@@ -2541,7 +2568,7 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange,
 		adev = pdd->dev->adev;
 
 		/* Check and drain ih1 ring if cam not available */
-		if (adev->irq.ih1.ring_size) {
+		if (!adev->irq.retry_cam_enabled && adev->irq.ih1.ring_size) {
 			ih = &adev->irq.ih1;
 			checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih);
 			if (ih->rptr != checkpoint_wptr) {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 3eb32d58a120..7a402c9c1b6e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -2028,6 +2028,10 @@ static void kfd_topology_set_capabilities(struct kfd_topology_device *dev)
 		if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(12, 0, 0))
 			dev->node_props.capability |=
 				HSA_CAP_TRAP_DEBUG_PRECISE_ALU_OPERATIONS_SUPPORTED;
+
+		if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(12, 1, 0))
+			dev->node_props.capability2 |=
+				HSA_CAP2_TRAP_DEBUG_LDS_OUT_OF_ADDR_RANGE_SUPPORTED;
 	}
 
 	kfd_topology_set_dbg_firmware_support(dev);
@@ -2408,3 +2412,26 @@ int kfd_debugfs_rls_by_device(struct seq_file *m, void *data)
 }
 
 #endif
+
+void kfd_update_svm_support_properties(struct amdgpu_device *adev)
+{
+	struct kfd_topology_device *dev;
+	int ret;
+
+	down_write(&topology_lock);
+	list_for_each_entry(dev, &topology_device_list, list) {
+		if (!dev->gpu || dev->gpu->adev != adev)
+			continue;
+
+		if (KFD_IS_SVM_API_SUPPORTED(adev)) {
+			dev->node_props.capability |= HSA_CAP_SVMAPI_SUPPORTED;
+			ret = kfd_topology_update_sysfs();
+			if (!ret)
+				sys_props.generation_count++;
+			else
+				dev_err(adev->dev, "Failed to update SVM support properties. ret=%d\n", ret);
+		} else
+			dev->node_props.capability &= ~HSA_CAP_SVMAPI_SUPPORTED;
+	}
+	up_write(&topology_lock);
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
index 3de8ec0043bb..ad63ba67b577 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
@@ -201,4 +201,10 @@ struct kfd_topology_device *kfd_create_topology_device(
 		struct list_head *device_list);
 void kfd_release_topology_device_list(struct list_head *device_list);
 
+#if IS_ENABLED(CONFIG_HSA_AMD)
+void kfd_update_svm_support_properties(struct amdgpu_device *adev);
+#else
+static inline void kfd_update_svm_support_properties(struct amdgpu_device *adev) {}
+#endif
+
 #endif /* __KFD_TOPOLOGY_H__ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index 740711ac1037..219c53797752 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -152,12 +152,6 @@ MODULE_FIRMWARE(FIRMWARE_DCN_36_DMUB);
 #define FIRMWARE_DCN_401_DMUB "amdgpu/dcn_4_0_1_dmcub.bin"
 MODULE_FIRMWARE(FIRMWARE_DCN_401_DMUB);
 
-/* Number of bytes in PSP header for firmware. */
-#define PSP_HEADER_BYTES 0x100
-
-/* Number of bytes in PSP footer for firmware. */
-#define PSP_FOOTER_BYTES 0x100
-
 /**
  * DOC: overview
  *
@@ -1298,15 +1292,14 @@ static int dm_dmub_hw_init(struct amdgpu_device *adev)
 
 	fw_inst_const = dmub_fw->data +
 			le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
-			PSP_HEADER_BYTES;
+			PSP_HEADER_BYTES_256;
 
 	fw_bss_data = dmub_fw->data +
 		      le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
 		      le32_to_cpu(hdr->inst_const_bytes);
 
 	/* Copy firmware and bios info into FB memory. */
-	fw_inst_const_size = le32_to_cpu(hdr->inst_const_bytes) -
-			     PSP_HEADER_BYTES - PSP_FOOTER_BYTES;
+	fw_inst_const_size = adev->dm.fw_inst_size;
 
 	fw_bss_data_size = le32_to_cpu(hdr->bss_data_bytes);
 
@@ -2437,9 +2430,11 @@ static void amdgpu_dm_dmub_reg_write(void *ctx, uint32_t address,
 static int dm_dmub_sw_init(struct amdgpu_device *adev)
 {
 	struct dmub_srv_create_params create_params;
+	struct dmub_srv_fw_meta_info_params fw_meta_info_params;
 	struct dmub_srv_region_params region_params;
 	struct dmub_srv_region_info region_info;
 	struct dmub_srv_memory_params memory_params;
+	struct dmub_fw_meta_info fw_info;
 	struct dmub_srv_fb_info *fb_info;
 	struct dmub_srv *dmub_srv;
 	const struct dmcub_firmware_header_v1_0 *hdr;
@@ -2547,22 +2542,37 @@ static int dm_dmub_sw_init(struct amdgpu_device *adev)
 		return -EINVAL;
 	}
 
+	/* Extract the FW meta info. */
+	memset(&fw_meta_info_params, 0, sizeof(fw_meta_info_params));
+
+	fw_meta_info_params.inst_const_size = le32_to_cpu(hdr->inst_const_bytes) -
+					      PSP_HEADER_BYTES_256;
+	fw_meta_info_params.bss_data_size = le32_to_cpu(hdr->bss_data_bytes);
+	fw_meta_info_params.fw_inst_const = adev->dm.dmub_fw->data +
+					    le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
+					    PSP_HEADER_BYTES_256;
+	fw_meta_info_params.fw_bss_data = region_params.bss_data_size ? adev->dm.dmub_fw->data +
+					  le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
+					  le32_to_cpu(hdr->inst_const_bytes) : NULL;
+	fw_meta_info_params.custom_psp_footer_size = 0;
+
+	status = dmub_srv_get_fw_meta_info_from_raw_fw(&fw_meta_info_params, &fw_info);
+	if (status != DMUB_STATUS_OK) {
+		/* Skip returning early, just log the error. */
+		drm_err(adev_to_drm(adev), "Error getting DMUB FW meta info: %d\n", status);
+		// return -EINVAL;
+	}
+
 	/* Calculate the size of all the regions for the DMUB service. */
 	memset(&region_params, 0, sizeof(region_params));
 
-	region_params.inst_const_size = le32_to_cpu(hdr->inst_const_bytes) -
-					PSP_HEADER_BYTES - PSP_FOOTER_BYTES;
-	region_params.bss_data_size = le32_to_cpu(hdr->bss_data_bytes);
+	region_params.inst_const_size = fw_meta_info_params.inst_const_size;
+	region_params.bss_data_size = fw_meta_info_params.bss_data_size;
 	region_params.vbios_size = adev->bios_size;
-	region_params.fw_bss_data = region_params.bss_data_size ?
-		adev->dm.dmub_fw->data +
-		le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
-		le32_to_cpu(hdr->inst_const_bytes) : NULL;
-	region_params.fw_inst_const =
-		adev->dm.dmub_fw->data +
-		le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
-		PSP_HEADER_BYTES;
+	region_params.fw_bss_data = fw_meta_info_params.fw_bss_data;
+	region_params.fw_inst_const = fw_meta_info_params.fw_inst_const;
 	region_params.window_memory_type = window_memory_type;
+	region_params.fw_info = (status == DMUB_STATUS_OK) ? &fw_info : NULL;
 
 	status = dmub_srv_calc_region_info(dmub_srv, &region_params,
 					   &region_info);
@@ -2609,6 +2619,7 @@ static int dm_dmub_sw_init(struct amdgpu_device *adev)
 	}
 
 	adev->dm.bb_from_dmub = dm_dmub_get_vbios_bounding_box(adev);
+	adev->dm.fw_inst_size = fw_meta_info_params.inst_const_size;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
index bd0403005f37..ab363f2f6d47 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
@@ -413,6 +413,13 @@ struct amdgpu_display_manager {
 	 */
 	uint32_t dmcub_fw_version;
 
+	/**
+	 * @fw_inst_size:
+	 *
+	 * Size of the firmware instruction buffer.
+	 */
+	uint32_t fw_inst_size;
+
 	/**
 	 * @cgs_device:
 	 *
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.h b/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.h
index b8275b397920..4e921632bc4e 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.h
@@ -31,4 +31,12 @@ void dc_assert_fp_enabled(void);
 void dc_fpu_begin(const char *function_name, const int line);
 void dc_fpu_end(const char *function_name, const int line);
 
+#ifndef _LINUX_FPU_COMPILATION_UNIT
+#define DC_FP_START()	dc_fpu_begin(__func__, __LINE__)
+#define DC_FP_END()	dc_fpu_end(__func__, __LINE__)
+#else
+#define DC_FP_START()	BUILD_BUG()
+#define DC_FP_END()	BUILD_BUG()
+#endif
+
 #endif /* __DC_FPU_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c
index dfd0c9505af0..72558cc55a9a 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c
@@ -405,7 +405,6 @@ void dcn35_update_clocks(struct clk_mgr *clk_mgr_base,
 		if (new_clocks->zstate_support != DCN_ZSTATE_SUPPORT_DISALLOW &&
 				new_clocks->zstate_support != clk_mgr_base->clks.zstate_support) {
 			dcn35_smu_set_zstate_support(clk_mgr, new_clocks->zstate_support);
-			dm_helpers_enable_periodic_detection(clk_mgr_base->ctx, true);
 			clk_mgr_base->clks.zstate_support = new_clocks->zstate_support;
 		}
 
@@ -425,7 +424,6 @@ void dcn35_update_clocks(struct clk_mgr *clk_mgr_base,
 		if (new_clocks->zstate_support == DCN_ZSTATE_SUPPORT_DISALLOW &&
 				new_clocks->zstate_support != clk_mgr_base->clks.zstate_support) {
 			dcn35_smu_set_zstate_support(clk_mgr, DCN_ZSTATE_SUPPORT_DISALLOW);
-			dm_helpers_enable_periodic_detection(clk_mgr_base->ctx, false);
 			clk_mgr_base->clks.zstate_support = new_clocks->zstate_support;
 		}
 
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
index 8be9cbd43e18..1be5c1c15798 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
@@ -2963,6 +2963,11 @@ static struct surface_update_descriptor check_update_surfaces_for_stream(
 {
 	struct surface_update_descriptor overall_type = { UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_NONE };
 
+	/* When countdown finishes, promote this flip to full to trigger deferred final transition */
+	if (check_config->deferred_transition_state && !check_config->transition_countdown_to_steady_state) {
+		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+	}
+
 	if (stream_update && stream_update->pending_test_pattern) {
 		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 	}
@@ -3441,6 +3446,49 @@ static bool full_update_required_weak(
 		const struct dc_stream_update *stream_update,
 		const struct dc_stream_state *stream);
 
+struct pipe_split_policy_backup {
+	bool dynamic_odm_policy;
+	bool subvp_policy;
+	enum pipe_split_policy mpc_policy;
+	char force_odm[MAX_PIPES];
+};
+
+static void backup_and_set_minimal_pipe_split_policy(struct dc *dc,
+		struct dc_state *context,
+		struct pipe_split_policy_backup *policy)
+{
+	int i;
+
+	if (!dc->config.is_vmin_only_asic) {
+		policy->mpc_policy = dc->debug.pipe_split_policy;
+		dc->debug.pipe_split_policy = MPC_SPLIT_AVOID;
+	}
+	policy->dynamic_odm_policy = dc->debug.enable_single_display_2to1_odm_policy;
+	dc->debug.enable_single_display_2to1_odm_policy = false;
+	policy->subvp_policy = dc->debug.force_disable_subvp;
+	dc->debug.force_disable_subvp = true;
+	for (i = 0; i < context->stream_count; i++) {
+		policy->force_odm[i] = context->streams[i]->debug.force_odm_combine_segments;
+		if (context->streams[i]->debug.allow_transition_for_forced_odm)
+			context->streams[i]->debug.force_odm_combine_segments = 0;
+	}
+}
+
+static void restore_minimal_pipe_split_policy(struct dc *dc,
+		struct dc_state *context,
+		struct pipe_split_policy_backup *policy)
+{
+	uint8_t i;
+
+	if (!dc->config.is_vmin_only_asic)
+		dc->debug.pipe_split_policy = policy->mpc_policy;
+	dc->debug.enable_single_display_2to1_odm_policy =
+			policy->dynamic_odm_policy;
+	dc->debug.force_disable_subvp = policy->subvp_policy;
+	for (i = 0; i < context->stream_count; i++)
+		context->streams[i]->debug.force_odm_combine_segments = policy->force_odm[i];
+}
+
 /**
  * update_planes_and_stream_state() - The function takes planes and stream
  * updates as inputs and determines the appropriate update type. If update type
@@ -3591,10 +3639,30 @@ static bool update_planes_and_stream_state(struct dc *dc,
 	}
 
 	if (update_type == UPDATE_TYPE_FULL) {
+		struct pipe_split_policy_backup policy;
+		bool minimize = false;
+
+		if (dc->check_config.deferred_transition_state) {
+			if (dc->check_config.transition_countdown_to_steady_state) {
+				/* During countdown, all new contexts created as minimal transition states */
+				minimize = true;
+			} else {
+				dc->check_config.deferred_transition_state = false;
+			}
+		}
+
+		if (minimize)
+			backup_and_set_minimal_pipe_split_policy(dc, context, &policy);
+
 		if (dc->res_pool->funcs->validate_bandwidth(dc, context, DC_VALIDATE_MODE_AND_PROGRAMMING) != DC_OK) {
+			if (minimize)
+				restore_minimal_pipe_split_policy(dc, context, &policy);
 			BREAK_TO_DEBUGGER();
 			goto fail;
 		}
+
+		if (minimize)
+			restore_minimal_pipe_split_policy(dc, context, &policy);
 	}
 	update_seamless_boot_flags(dc, context, surface_count, stream);
 
@@ -3781,7 +3849,7 @@ static bool dc_dmub_should_send_dirty_rect_cmd(struct dc *dc, struct dc_stream_s
 void dc_dmub_update_dirty_rect(struct dc *dc,
 			       int surface_count,
 			       struct dc_stream_state *stream,
-			       struct dc_surface_update *srf_updates,
+			       const struct dc_surface_update *srf_updates,
 			       struct dc_state *context)
 {
 	union dmub_rb_cmd cmd;
@@ -4086,7 +4154,7 @@ static void commit_planes_for_stream_fast(struct dc *dc,
 }
 
 static void commit_planes_for_stream(struct dc *dc,
-		struct dc_surface_update *srf_updates,
+		const struct dc_surface_update *srf_updates,
 		int surface_count,
 		struct dc_stream_state *stream,
 		struct dc_stream_update *stream_update,
@@ -4622,48 +4690,6 @@ static bool could_mpcc_tree_change_for_active_pipes(struct dc *dc,
 	return force_minimal_pipe_splitting;
 }
 
-struct pipe_split_policy_backup {
-	bool dynamic_odm_policy;
-	bool subvp_policy;
-	enum pipe_split_policy mpc_policy;
-	char force_odm[MAX_PIPES];
-};
-
-static void backup_and_set_minimal_pipe_split_policy(struct dc *dc,
-		struct dc_state *context,
-		struct pipe_split_policy_backup *policy)
-{
-	int i;
-
-	if (!dc->config.is_vmin_only_asic) {
-		policy->mpc_policy = dc->debug.pipe_split_policy;
-		dc->debug.pipe_split_policy = MPC_SPLIT_AVOID;
-	}
-	policy->dynamic_odm_policy = dc->debug.enable_single_display_2to1_odm_policy;
-	dc->debug.enable_single_display_2to1_odm_policy = false;
-	policy->subvp_policy = dc->debug.force_disable_subvp;
-	dc->debug.force_disable_subvp = true;
-	for (i = 0; i < context->stream_count; i++) {
-		policy->force_odm[i] = context->streams[i]->debug.force_odm_combine_segments;
-		if (context->streams[i]->debug.allow_transition_for_forced_odm)
-			context->streams[i]->debug.force_odm_combine_segments = 0;
-	}
-}
-
-static void restore_minimal_pipe_split_policy(struct dc *dc,
-		struct dc_state *context,
-		struct pipe_split_policy_backup *policy)
-{
-	uint8_t i;
-
-	if (!dc->config.is_vmin_only_asic)
-		dc->debug.pipe_split_policy = policy->mpc_policy;
-	dc->debug.enable_single_display_2to1_odm_policy =
-			policy->dynamic_odm_policy;
-	dc->debug.force_disable_subvp = policy->subvp_policy;
-	for (i = 0; i < context->stream_count; i++)
-		context->streams[i]->debug.force_odm_combine_segments = policy->force_odm[i];
-}
 
 static void release_minimal_transition_state(struct dc *dc,
 		struct dc_state *minimal_transition_context,
@@ -4773,6 +4799,7 @@ static int initialize_empty_surface_updates(
 static bool commit_minimal_transition_based_on_new_context(struct dc *dc,
 		struct dc_state *new_context,
 		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update,
 		struct dc_surface_update *srf_updates,
 		int surface_count)
 {
@@ -4790,7 +4817,7 @@ static bool commit_minimal_transition_based_on_new_context(struct dc *dc,
 				new_context)) {
 			DC_LOG_DC("commit minimal transition state: base = new state\n");
 			commit_planes_for_stream(dc, srf_updates,
-					surface_count, stream, NULL,
+					surface_count, stream, stream_update,
 					UPDATE_TYPE_FULL, intermediate_context);
 			swap_and_release_current_context(
 					dc, intermediate_context, stream);
@@ -4884,8 +4911,8 @@ static bool commit_minimal_transition_state_in_dc_update(struct dc *dc,
 		int surface_count)
 {
 	bool success = commit_minimal_transition_based_on_new_context(
-				dc, new_context, stream, srf_updates,
-				surface_count);
+				dc, new_context, stream, NULL,
+				srf_updates, surface_count);
 	if (!success)
 		success = commit_minimal_transition_based_on_current_context(dc,
 				new_context, stream);
@@ -5294,32 +5321,63 @@ static void commit_planes_and_stream_update_with_new_context(struct dc *dc,
 		enum surface_update_type update_type,
 		struct dc_state *new_context)
 {
+	bool skip_new_context = false;
 	ASSERT(update_type >= UPDATE_TYPE_FULL);
-	if (!dc->hwss.is_pipe_topology_transition_seamless(dc,
-			dc->current_state, new_context))
-		/*
-		 * It is required by the feature design that all pipe topologies
-		 * using extra free pipes for power saving purposes such as
-		 * dynamic ODM or SubVp shall only be enabled when it can be
-		 * transitioned seamlessly to AND from its minimal transition
-		 * state. A minimal transition state is defined as the same dc
-		 * state but with all power saving features disabled. So it uses
-		 * the minimum pipe topology. When we can't seamlessly
-		 * transition from state A to state B, we will insert the
-		 * minimal transition state A' or B' in between so seamless
-		 * transition between A and B can be made possible.
-		 */
-		commit_minimal_transition_state_in_dc_update(dc, new_context,
-				stream, srf_updates, surface_count);
+	/*
+	 * It is required by the feature design that all pipe topologies
+	 * using extra free pipes for power saving purposes such as
+	 * dynamic ODM or SubVp shall only be enabled when it can be
+	 * transitioned seamlessly to AND from its minimal transition
+	 * state. A minimal transition state is defined as the same dc
+	 * state but with all power saving features disabled. So it uses
+	 * the minimum pipe topology. When we can't seamlessly
+	 * transition from state A to state B, we will insert the
+	 * minimal transition state A' or B' in between so seamless
+	 * transition between A and B can be made possible.
+	 *
+	 * To optimize for the time it takes to execute flips,
+	 * the transition from the minimal state to the final state is
+	 * deferred until a steady state (no more transitions) is reached.
+	 */
+	if (!dc->hwss.is_pipe_topology_transition_seamless(dc, dc->current_state, new_context)) {
+		if (!dc->debug.disable_deferred_minimal_transitions) {
+			dc->check_config.deferred_transition_state = true;
+			dc->check_config.transition_countdown_to_steady_state =
+					dc->debug.num_fast_flips_to_steady_state_override ?
+					dc->debug.num_fast_flips_to_steady_state_override :
+					NUM_FAST_FLIPS_TO_STEADY_STATE;
+
+			if (commit_minimal_transition_based_on_new_context(dc, new_context, stream, stream_update,
+					srf_updates, surface_count)) {
+				skip_new_context = true;
+				dc_state_release(new_context);
+				new_context = dc->current_state;
+			} else {
+				/*
+				 * In this case a new mpo plane is being enabled on pipes that were
+				 * previously in use, and the surface update to the existing plane
+				 * includes an alpha box where the new plane will be, so the update
+				 * from minimal to final cannot be deferred as the alpha box would
+				 * be visible to the user
+				 */
+				commit_minimal_transition_based_on_current_context(dc, new_context, stream);
+			}
+		} else {
+			commit_minimal_transition_state_in_dc_update(dc, new_context, stream,
+					srf_updates, surface_count);
+		}
+	} else if (dc->check_config.deferred_transition_state) {
+		/* reset countdown as steady state not reached */
+		dc->check_config.transition_countdown_to_steady_state =
+				dc->debug.num_fast_flips_to_steady_state_override ?
+				dc->debug.num_fast_flips_to_steady_state_override :
+				NUM_FAST_FLIPS_TO_STEADY_STATE;
+	}
 
-	commit_planes_for_stream(
-			dc,
-			srf_updates,
-			surface_count,
-			stream,
-			stream_update,
-			update_type,
-			new_context);
+	if (!skip_new_context) {
+		commit_planes_for_stream(dc, srf_updates, surface_count, stream, stream_update, update_type, new_context);
+		swap_and_release_current_context(dc, new_context, stream);
+	}
 }
 
 static bool update_planes_and_stream_v3(struct dc *dc,
@@ -5349,11 +5407,13 @@ static bool update_planes_and_stream_v3(struct dc *dc,
 		commit_planes_and_stream_update_on_current_context(dc,
 				srf_updates, surface_count, stream,
 				stream_update, update_type);
+
+		if (dc->check_config.transition_countdown_to_steady_state)
+			dc->check_config.transition_countdown_to_steady_state--;
 	} else {
 		commit_planes_and_stream_update_with_new_context(dc,
 				srf_updates, surface_count, stream,
 				stream_update, update_type, new_context);
-		swap_and_release_current_context(dc, new_context, stream);
 	}
 
 	return true;
@@ -7091,3 +7151,374 @@ void dc_log_preos_dmcub_info(const struct dc *dc)
 {
 	dc_dmub_srv_log_preos_dmcub_info(dc->ctx->dmub_srv);
 }
+
+bool dc_get_qos_info(struct dc *dc, struct dc_qos_info *info)
+{
+	const struct dc_clocks *clk = &dc->current_state->bw_ctx.bw.dcn.clk;
+
+	memset(info, 0, sizeof(*info));
+
+	// Check if all measurement functions are available
+	if (!dc->hwss.measure_peak_bw_mbps ||
+	    !dc->hwss.measure_avg_bw_mbps ||
+	    !dc->hwss.measure_max_latency_ns ||
+	    !dc->hwss.measure_avg_latency_ns) {
+		return false;
+	}
+
+	// Call measurement functions to get actual values
+	info->actual_peak_bw_in_mbps = dc->hwss.measure_peak_bw_mbps(dc);
+	info->actual_avg_bw_in_mbps = dc->hwss.measure_avg_bw_mbps(dc);
+	info->actual_max_latency_in_ns = dc->hwss.measure_max_latency_ns(dc);
+	info->actual_avg_latency_in_ns = dc->hwss.measure_avg_latency_ns(dc);
+	info->dcn_bandwidth_ub_in_mbps = (uint32_t)(clk->fclk_khz / 1000 * 64);
+
+	return true;
+}
+
+enum update_v3_flow {
+	UPDATE_V3_FLOW_INVALID,
+	UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FAST,
+	UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FULL,
+	UPDATE_V3_FLOW_NEW_CONTEXT_SEAMLESS,
+	UPDATE_V3_FLOW_NEW_CONTEXT_MINIMAL_NEW,
+	UPDATE_V3_FLOW_NEW_CONTEXT_MINIMAL_CURRENT,
+};
+
+struct dc_update_scratch_space {
+	struct dc *dc;
+	struct dc_surface_update *surface_updates;
+	int surface_count;
+	struct dc_stream_state *stream;
+	struct dc_stream_update *stream_update;
+	bool update_v3;
+	bool do_clear_update_flags;
+	enum surface_update_type update_type;
+	struct dc_state *new_context;
+	enum update_v3_flow flow;
+	struct dc_state *backup_context;
+	struct dc_state *intermediate_context;
+	struct pipe_split_policy_backup intermediate_policy;
+	struct dc_surface_update intermediate_updates[MAX_SURFACES];
+	int intermediate_count;
+};
+
+size_t dc_update_scratch_space_size(void)
+{
+	return sizeof(struct dc_update_scratch_space);
+}
+
+static bool update_planes_and_stream_prepare_v2(
+		struct dc_update_scratch_space *scratch
+)
+{
+	// v2 is too tangled to break into stages, so just execute everything under lock
+	dc_exit_ips_for_hw_access(scratch->dc);
+	return update_planes_and_stream_v2(
+		scratch->dc,
+		scratch->surface_updates,
+		scratch->surface_count,
+		scratch->stream,
+		scratch->stream_update
+	);
+}
+
+static void update_planes_and_stream_execute_v2(
+		const struct dc_update_scratch_space *scratch
+)
+{
+	// Nothing to do, see `update_planes_and_stream_prepare_v2`
+	(void) scratch;
+}
+
+static bool update_planes_and_stream_cleanup_v2(
+		const struct dc_update_scratch_space *scratch
+)
+{
+	if (scratch->do_clear_update_flags)
+		clear_update_flags(scratch->surface_updates, scratch->surface_count, scratch->stream);
+
+	return false;
+}
+
+static void update_planes_and_stream_cleanup_v3_intermediate(
+		struct dc_update_scratch_space *scratch,
+		bool backup
+);
+
+static bool update_planes_and_stream_prepare_v3_intermediate_seamless(
+		struct dc_update_scratch_space *scratch
+)
+{
+	return is_pipe_topology_transition_seamless_with_intermediate_step(
+			scratch->dc,
+			scratch->dc->current_state,
+			scratch->intermediate_context,
+			scratch->new_context
+	);
+}
+
+static bool update_planes_and_stream_prepare_v3(
+		struct dc_update_scratch_space *scratch
+)
+{
+	dc_exit_ips_for_hw_access(scratch->dc);
+
+	if (!update_planes_and_stream_state(
+			scratch->dc,
+			scratch->surface_updates,
+			scratch->surface_count,
+			scratch->stream,
+			scratch->stream_update,
+			&scratch->update_type,
+			&scratch->new_context
+	)) {
+		return false;
+	}
+
+	if (scratch->new_context == scratch->dc->current_state) {
+		ASSERT(scratch->update_type < UPDATE_TYPE_FULL);
+
+		// TODO: Do we need this to be alive in execute?
+		struct dc_fast_update fast_update[MAX_SURFACES] = { 0 };
+
+		populate_fast_updates(
+				fast_update,
+				scratch->surface_updates,
+				scratch->surface_count,
+				scratch->stream_update
+		);
+		const bool fast = fast_update_only(
+				scratch->dc,
+				fast_update,
+				scratch->surface_updates,
+				scratch->surface_count,
+				scratch->stream_update,
+				scratch->stream
+		)
+		// TODO: Can this be used to skip `populate_fast_updates`?
+				&& !scratch->dc->check_config.enable_legacy_fast_update;
+		scratch->flow = fast
+				? UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FAST
+				: UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FULL;
+		return true;
+	}
+
+	ASSERT(scratch->update_type >= UPDATE_TYPE_FULL);
+
+	const bool seamless = scratch->dc->hwss.is_pipe_topology_transition_seamless(
+			scratch->dc,
+			scratch->dc->current_state,
+			scratch->new_context
+	);
+	if (seamless) {
+		scratch->flow = UPDATE_V3_FLOW_NEW_CONTEXT_SEAMLESS;
+		return true;
+	}
+
+	scratch->intermediate_context = create_minimal_transition_state(
+		scratch->dc,
+		scratch->new_context,
+		&scratch->intermediate_policy
+	);
+	if (scratch->intermediate_context) {
+		if (update_planes_and_stream_prepare_v3_intermediate_seamless(scratch)) {
+			scratch->flow = UPDATE_V3_FLOW_NEW_CONTEXT_MINIMAL_NEW;
+			return true;
+		}
+
+		update_planes_and_stream_cleanup_v3_intermediate(scratch, false);
+	}
+
+	restore_planes_and_stream_state(&scratch->dc->scratch.current_state, scratch->stream);
+	scratch->backup_context = scratch->dc->current_state;
+	dc_state_retain(scratch->backup_context);
+	scratch->intermediate_context = create_minimal_transition_state(
+			scratch->dc,
+			scratch->backup_context,
+			&scratch->intermediate_policy
+	);
+	if (scratch->intermediate_context) {
+		if (update_planes_and_stream_prepare_v3_intermediate_seamless(scratch)) {
+			scratch->flow = UPDATE_V3_FLOW_NEW_CONTEXT_MINIMAL_CURRENT;
+			scratch->intermediate_count = initialize_empty_surface_updates(
+					scratch->stream, scratch->intermediate_updates
+			);
+			return true;
+		}
+
+		update_planes_and_stream_cleanup_v3_intermediate(scratch, true);
+	}
+
+	scratch->flow = UPDATE_V3_FLOW_INVALID;
+	dc_state_release(scratch->backup_context);
+	restore_planes_and_stream_state(&scratch->dc->scratch.new_state, scratch->stream);
+	return false;
+}
+
+static void update_planes_and_stream_execute_v3_commit(
+		const struct dc_update_scratch_space *scratch,
+		bool intermediate_update,
+		bool intermediate_context
+)
+{
+	commit_planes_for_stream(
+			scratch->dc,
+			intermediate_update ? scratch->intermediate_updates : scratch->surface_updates,
+			intermediate_update ? scratch->intermediate_count : scratch->surface_count,
+			scratch->stream,
+			intermediate_context ? NULL : scratch->stream_update,
+			intermediate_context ? UPDATE_TYPE_FULL : scratch->update_type,
+			// `dc->current_state` only used in `NO_NEW_CONTEXT`, where it is equal to `new_context`
+			intermediate_context ? scratch->intermediate_context : scratch->new_context
+	);
+}
+
+static void update_planes_and_stream_execute_v3(
+		const struct dc_update_scratch_space *scratch
+)
+{
+	switch (scratch->flow) {
+	case UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FAST:
+		commit_planes_for_stream_fast(
+				scratch->dc,
+				scratch->surface_updates,
+				scratch->surface_count,
+				scratch->stream,
+				scratch->stream_update,
+				scratch->update_type,
+				scratch->new_context
+		);
+		break;
+
+	case UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FULL:
+	case UPDATE_V3_FLOW_NEW_CONTEXT_SEAMLESS:
+		update_planes_and_stream_execute_v3_commit(scratch, false, false);
+		break;
+
+	case UPDATE_V3_FLOW_NEW_CONTEXT_MINIMAL_NEW:
+		update_planes_and_stream_execute_v3_commit(scratch, false, true);
+		update_planes_and_stream_execute_v3_commit(scratch, false, false);
+		break;
+
+	case UPDATE_V3_FLOW_NEW_CONTEXT_MINIMAL_CURRENT:
+		update_planes_and_stream_execute_v3_commit(scratch, true, true);
+		update_planes_and_stream_execute_v3_commit(scratch, false, false);
+		break;
+
+	case UPDATE_V3_FLOW_INVALID:
+	default:
+		ASSERT(false);
+	}
+}
+
+static void update_planes_and_stream_cleanup_v3_new_context(
+		struct dc_update_scratch_space *scratch
+)
+{
+	swap_and_release_current_context(scratch->dc, scratch->new_context, scratch->stream);
+}
+
+static void update_planes_and_stream_cleanup_v3_intermediate(
+		struct dc_update_scratch_space *scratch,
+		bool backup
+)
+{
+	release_minimal_transition_state(
+			scratch->dc,
+			scratch->intermediate_context,
+			backup ? scratch->backup_context : scratch->new_context,
+			&scratch->intermediate_policy
+	);
+}
+
+static bool update_planes_and_stream_cleanup_v3(
+		struct dc_update_scratch_space *scratch
+)
+{
+	switch (scratch->flow) {
+	case UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FAST:
+	case UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FULL:
+		// No cleanup required
+		break;
+
+	case UPDATE_V3_FLOW_NEW_CONTEXT_SEAMLESS:
+		update_planes_and_stream_cleanup_v3_new_context(scratch);
+		break;
+
+	case UPDATE_V3_FLOW_NEW_CONTEXT_MINIMAL_NEW:
+		update_planes_and_stream_cleanup_v3_intermediate(scratch, false);
+		update_planes_and_stream_cleanup_v3_new_context(scratch);
+		break;
+
+	case UPDATE_V3_FLOW_NEW_CONTEXT_MINIMAL_CURRENT:
+		swap_and_release_current_context(scratch->dc, scratch->intermediate_context, scratch->stream);
+		dc_state_retain(scratch->dc->current_state);
+		update_planes_and_stream_cleanup_v3_intermediate(scratch, true);
+		dc_state_release(scratch->backup_context);
+		restore_planes_and_stream_state(&scratch->dc->scratch.new_state, scratch->stream);
+		update_planes_and_stream_cleanup_v3_new_context(scratch);
+		break;
+
+	case UPDATE_V3_FLOW_INVALID:
+	default:
+		ASSERT(false);
+	}
+
+	if (scratch->do_clear_update_flags)
+		clear_update_flags(scratch->surface_updates, scratch->surface_count, scratch->stream);
+
+	return false;
+}
+
+struct dc_update_scratch_space *dc_update_planes_and_stream_init(
+		struct dc *dc,
+		struct dc_surface_update *surface_updates,
+		int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update
+)
+{
+	const enum dce_version version = dc->ctx->dce_version;
+	struct dc_update_scratch_space *scratch = stream->update_scratch;
+
+	*scratch = (struct dc_update_scratch_space){
+		.dc = dc,
+		.surface_updates = surface_updates,
+		.surface_count = surface_count,
+		.stream = stream,
+		.stream_update = stream_update,
+		.update_v3 = version >= DCN_VERSION_4_01 || version == DCN_VERSION_3_2 || version == DCN_VERSION_3_21,
+		.do_clear_update_flags = version >= DCN_VERSION_3_2 || version == DCN_VERSION_3_01,
+	};
+
+	return scratch;
+}
+
+bool dc_update_planes_and_stream_prepare(
+		struct dc_update_scratch_space *scratch
+)
+{
+	return scratch->update_v3
+			? update_planes_and_stream_prepare_v3(scratch)
+			: update_planes_and_stream_prepare_v2(scratch);
+}
+
+void dc_update_planes_and_stream_execute(
+		const struct dc_update_scratch_space *scratch
+)
+{
+	scratch->update_v3
+			? update_planes_and_stream_execute_v3(scratch)
+			: update_planes_and_stream_execute_v2(scratch);
+}
+
+bool dc_update_planes_and_stream_cleanup(
+		struct dc_update_scratch_space *scratch
+)
+{
+	return scratch->update_v3
+			? update_planes_and_stream_cleanup_v3(scratch)
+			: update_planes_and_stream_cleanup_v2(scratch);
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c
index 9acd30019717..a8d7228907c2 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c
@@ -491,6 +491,28 @@ bool dc_link_get_replay_state(const struct dc_link *link, uint64_t *state)
 	return link->dc->link_srv->edp_get_replay_state(link, state);
 }
 
+bool dc_link_set_pr_enable(struct dc_link *link, bool enable)
+{
+	return link->dc->link_srv->edp_pr_enable(link, enable);
+}
+
+bool dc_link_update_pr_state(struct dc_link *link,
+		struct dmub_cmd_pr_update_state_data *update_state_data)
+{
+	return link->dc->link_srv->edp_pr_update_state(link, update_state_data);
+}
+
+bool dc_link_set_pr_general_cmd(struct dc_link *link,
+		struct dmub_cmd_pr_general_cmd_data *general_cmd_data)
+{
+	return link->dc->link_srv->edp_pr_set_general_cmd(link, general_cmd_data);
+}
+
+bool dc_link_get_pr_state(const struct dc_link *link, uint64_t *state)
+{
+	return link->dc->link_srv->edp_pr_get_state(link, state);
+}
+
 bool dc_link_wait_for_t12(struct dc_link *link)
 {
 	return link->dc->link_srv->edp_wait_for_t12(link);
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
index 129cd5f84983..9349cccc8438 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
@@ -151,6 +151,7 @@ static void dc_stream_free(struct kref *kref)
 	struct dc_stream_state *stream = container_of(kref, struct dc_stream_state, refcount);
 
 	dc_stream_destruct(stream);
+	kfree(stream->update_scratch);
 	kfree(stream);
 }
 
@@ -164,26 +165,32 @@ void dc_stream_release(struct dc_stream_state *stream)
 struct dc_stream_state *dc_create_stream_for_sink(
 		struct dc_sink *sink)
 {
-	struct dc_stream_state *stream;
+	struct dc_stream_state *stream = NULL;
 
 	if (sink == NULL)
-		return NULL;
+		goto fail;
 
 	stream = kzalloc(sizeof(struct dc_stream_state), GFP_KERNEL);
 	if (stream == NULL)
-		goto alloc_fail;
+		goto fail;
+
+	stream->update_scratch = kzalloc((int32_t) dc_update_scratch_space_size(), GFP_KERNEL);
+	if (stream->update_scratch == NULL)
+		goto fail;
 
 	if (dc_stream_construct(stream, sink) == false)
-		goto construct_fail;
+		goto fail;
 
 	kref_init(&stream->refcount);
 
 	return stream;
 
-construct_fail:
-	kfree(stream);
+fail:
+	if (stream) {
+		kfree(stream->update_scratch);
+		kfree(stream);
+	}
 
-alloc_fail:
 	return NULL;
 }
 
@@ -195,6 +202,16 @@ struct dc_stream_state *dc_copy_stream(const struct dc_stream_state *stream)
 	if (!new_stream)
 		return NULL;
 
+	// Scratch is not meant to be reused across copies, as might have self-referential pointers
+	new_stream->update_scratch = kzalloc(
+			(int32_t) dc_update_scratch_space_size(),
+			GFP_KERNEL
+	);
+	if (!new_stream->update_scratch) {
+		kfree(new_stream);
+		return NULL;
+	}
+
 	if (new_stream->sink)
 		dc_sink_retain(new_stream->sink);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
index 29edfa51ea2c..f1036a06b778 100644
--- a/drivers/gpu/drm/amd/display/dc/dc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc.h
@@ -63,7 +63,7 @@ struct dcn_dsc_reg_state;
 struct dcn_optc_reg_state;
 struct dcn_dccg_reg_state;
 
-#define DC_VER "3.2.359"
+#define DC_VER "3.2.363"
 
 /**
  * MAX_SURFACES - representative of the upper bound of surfaces that can be piped to a single CRTC
@@ -82,6 +82,8 @@ struct dcn_dccg_reg_state;
 #define MAX_DPIA_PER_HOST_ROUTER 3
 #define MAX_DPIA_NUM  (MAX_HOST_ROUTERS_NUM * MAX_DPIA_PER_HOST_ROUTER)
 
+#define NUM_FAST_FLIPS_TO_STEADY_STATE 20
+
 /* Display Core Interfaces */
 struct dc_versions {
 	const char *dc_ver;
@@ -293,6 +295,9 @@ struct dc_check_config {
 	 */
 	unsigned int max_optimizable_video_width;
 	bool enable_legacy_fast_update;
+
+	bool deferred_transition_state;
+	unsigned int transition_countdown_to_steady_state;
 };
 
 struct dc_caps {
@@ -951,6 +956,18 @@ struct dc_bounding_box_overrides {
 	int min_dcfclk_mhz;
 };
 
+struct dc_qos_info {
+	uint32_t actual_peak_bw_in_mbps;
+	uint32_t qos_bandwidth_lb_in_mbps;
+	uint32_t actual_avg_bw_in_mbps;
+	uint32_t calculated_avg_bw_in_mbps;
+	uint32_t actual_max_latency_in_ns;
+	uint32_t qos_max_latency_ub_in_ns;
+	uint32_t actual_avg_latency_in_ns;
+	uint32_t qos_avg_latency_ub_in_ns;
+	uint32_t dcn_bandwidth_ub_in_mbps;
+};
+
 struct dc_state;
 struct resource_pool;
 struct dce_hwseq;
@@ -1188,6 +1205,9 @@ struct dc_debug_options {
 	short auxless_alpm_lfps_t1t2_offset_us;
 	bool disable_stutter_for_wm_program;
 	bool enable_block_sequence_programming;
+	uint32_t custom_psp_footer_size;
+	bool disable_deferred_minimal_transitions;
+	unsigned int num_fast_flips_to_steady_state_override;
 };
 
 
@@ -1707,7 +1727,6 @@ struct dc_scratch_space {
 	struct dc_link_status link_status;
 	struct dprx_states dprx_states;
 
-	struct gpio *hpd_gpio;
 	enum dc_link_fec_state fec_state;
 	bool is_dds;
 	bool is_display_mux_present;
@@ -2455,6 +2474,48 @@ bool dc_link_set_replay_allow_active(struct dc_link *dc_link, const bool *enable
 
 bool dc_link_get_replay_state(const struct dc_link *dc_link, uint64_t *state);
 
+/*
+ * Enable or disable Panel Replay on the specified link:
+ *
+ * @link: pointer to the dc_link struct instance
+ * @enable: enable or disable Panel Replay
+ *
+ * return: true if successful, false otherwise
+ */
+bool dc_link_set_pr_enable(struct dc_link *link, bool enable);
+
+/*
+ * Update Panel Replay state parameters:
+ *
+ * @link: pointer to the dc_link struct instance
+ * @update_state_data: pointer to state update data structure
+ *
+ * return: true if successful, false otherwise
+ */
+bool dc_link_update_pr_state(struct dc_link *link,
+		struct dmub_cmd_pr_update_state_data *update_state_data);
+
+/*
+ * Send general command to Panel Replay firmware:
+ *
+ * @link: pointer to the dc_link struct instance
+ * @general_cmd_data: pointer to general command data structure
+ *
+ * return: true if successful, false otherwise
+ */
+bool dc_link_set_pr_general_cmd(struct dc_link *link,
+		struct dmub_cmd_pr_general_cmd_data *general_cmd_data);
+
+/*
+ * Get Panel Replay state:
+ *
+ * @link: pointer to the dc_link struct instance
+ * @state: pointer to store the Panel Replay state
+ *
+ * return: true if successful, false otherwise
+ */
+bool dc_link_get_pr_state(const struct dc_link *link, uint64_t *state);
+
 /* On eDP links this function call will stall until T12 has elapsed.
  * If the panel is not in power off state, this function will return
  * immediately.
@@ -3280,4 +3341,28 @@ struct dc_register_software_state {
  */
 bool dc_capture_register_software_state(struct dc *dc, struct dc_register_software_state *state);
 
+/**
+ * dc_get_qos_info() - Retrieve Quality of Service (QoS) information from display core
+ * @dc: DC context containing current display configuration
+ * @info: Pointer to dc_qos_info structure to populate with QoS metrics
+ *
+ * This function retrieves QoS metrics from the display core that can be used by
+ * benchmark tools to analyze display system performance. The function may take
+ * several milliseconds to execute due to hardware measurement requirements.
+ *
+ * QoS information includes:
+ * - Bandwidth bounds (lower limits in Mbps)
+ * - Latency bounds (upper limits in nanoseconds)
+ * - Hardware-measured bandwidth metrics (peak/average in Mbps)
+ * - Hardware-measured latency metrics (maximum/average in nanoseconds)
+ *
+ * The function will populate the provided dc_qos_info structure with current
+ * QoS measurements. If hardware measurement functions are not available for
+ * the current DCN version, the function returns false with zero'd info structure.
+ *
+ * Return: true if QoS information was successfully retrieved, false if measurement
+ *         functions are unavailable or hardware measurements cannot be performed
+ */
+bool dc_get_qos_info(struct dc *dc, struct dc_qos_info *info);
+
 #endif /* DC_INTERFACE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
index 7b09af1cb306..602655dd1323 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
@@ -1833,9 +1833,10 @@ static void dc_dmub_srv_rb_based_fams2_update_config(struct dc *dc,
 
 	/* apply feature configuration based on current driver state */
 	global_cmd->config.global.features.bits.enable_visual_confirm = dc->debug.visual_confirm == VISUAL_CONFIRM_FAMS2;
-	global_cmd->config.global.features.bits.enable = enable;
+	global_cmd->config.global.features.bits.enable = enable && context->bw_ctx.bw.dcn.fams2_global_config.features.bits.enable;
+	global_cmd->config.global.features.bits.enable_ppt_check = dc->debug.fams2_config.bits.enable_ppt_check;
 
-	if (enable && context->bw_ctx.bw.dcn.fams2_global_config.features.bits.enable) {
+	if (enable) {
 		/* set multi pending for global, and unset for last stream cmd */
 		global_cmd->header.multi_cmd_pending = 1;
 		cmd[2 * context->bw_ctx.bw.dcn.fams2_global_config.num_streams].fams2_config.header.multi_cmd_pending = 0;
@@ -1862,16 +1863,16 @@ static void dc_dmub_srv_ib_based_fams2_update_config(struct dc *dc,
 	cmd.ib_fams2_config.ib_data.src.quad_part = dc->ctx->dmub_srv->dmub->ib_mem_gart.gpu_addr;
 	cmd.ib_fams2_config.ib_data.size = sizeof(*config);
 
-	if (enable && context->bw_ctx.bw.dcn.fams2_global_config.features.bits.enable) {
+	if (enable) {
+		/* send global configuration parameters */
+		memcpy(&config->global, &context->bw_ctx.bw.dcn.fams2_global_config,
+			sizeof(struct dmub_cmd_fams2_global_config));
+
 		/* copy static feature configuration overrides */
 		config->global.features.bits.enable_stall_recovery = dc->debug.fams2_config.bits.enable_stall_recovery;
 		config->global.features.bits.enable_offload_flip = dc->debug.fams2_config.bits.enable_offload_flip;
 		config->global.features.bits.enable_debug = dc->debug.fams2_config.bits.enable_debug;
 
-		/* send global configuration parameters */
-		memcpy(&config->global, &context->bw_ctx.bw.dcn.fams2_global_config,
-			sizeof(struct dmub_cmd_fams2_global_config));
-
 		/* construct per-stream configs */
 		for (i = 0; i < context->bw_ctx.bw.dcn.fams2_global_config.num_streams; i++) {
 			/* copy stream static base state */
@@ -1887,7 +1888,8 @@ static void dc_dmub_srv_ib_based_fams2_update_config(struct dc *dc,
 	}
 
 	config->global.features.bits.enable_visual_confirm = dc->debug.visual_confirm == VISUAL_CONFIRM_FAMS2;
-	config->global.features.bits.enable = enable;
+	config->global.features.bits.enable = enable && context->bw_ctx.bw.dcn.fams2_global_config.features.bits.enable;
+	config->global.features.bits.enable_ppt_check = dc->debug.fams2_config.bits.enable_ppt_check;
 
 	dm_execute_dmub_cmd_list(dc->ctx, 1, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
index 79e1696def63..273610d85438 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
@@ -1296,6 +1296,7 @@ struct dpcd_caps {
 	/* Indicates the number of SST links supported by MSO (Multi-Stream Output) */
 	uint8_t mso_cap_sst_links_supported;
 	uint8_t dp_edp_general_cap_2;
+	uint16_t drr_granularity;
 };
 
 union dpcd_sink_ext_caps {
diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h
index 321cfe92d799..719b98d8e8ca 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_stream.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h
@@ -315,6 +315,8 @@ struct dc_stream_state {
 	struct luminance_data lumin_data;
 	bool scaler_sharpener_update;
 	bool sharpening_required;
+
+	struct dc_update_scratch_space *update_scratch;
 };
 
 #define ABM_LEVEL_IMMEDIATE_DISABLE 255
@@ -390,6 +392,33 @@ bool dc_update_planes_and_stream(struct dc *dc,
 		struct dc_stream_state *dc_stream,
 		struct dc_stream_update *stream_update);
 
+struct dc_update_scratch_space;
+
+size_t dc_update_scratch_space_size(void);
+
+struct dc_update_scratch_space *dc_update_planes_and_stream_init(
+		struct dc *dc,
+		struct dc_surface_update *surface_updates,
+		int surface_count,
+		struct dc_stream_state *dc_stream,
+		struct dc_stream_update *stream_update
+);
+
+// Locked, false is failed
+bool dc_update_planes_and_stream_prepare(
+		struct dc_update_scratch_space *scratch
+);
+
+// Unlocked
+void dc_update_planes_and_stream_execute(
+		const struct dc_update_scratch_space *scratch
+);
+
+// Locked, true if call again
+bool dc_update_planes_and_stream_cleanup(
+		struct dc_update_scratch_space *scratch
+);
+
 /*
  * Set up surface attributes and associate to a stream
  * The surfaces parameter is an absolute set of all surface active for the stream.
@@ -597,7 +626,7 @@ struct pipe_ctx *dc_stream_get_pipe_ctx(struct dc_stream_state *stream);
 void dc_dmub_update_dirty_rect(struct dc *dc,
 			       int surface_count,
 			       struct dc_stream_state *stream,
-			       struct dc_surface_update *srf_updates,
+			       const struct dc_surface_update *srf_updates,
 			       struct dc_state *context);
 
 bool dc_stream_is_cursor_limit_pending(struct dc *dc, struct dc_stream_state *stream);
diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h
index f46039f64203..cf4bf0faf1cd 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_types.h
@@ -1078,6 +1078,7 @@ enum replay_coasting_vtotal_type {
 	PR_COASTING_TYPE_STATIC,
 	PR_COASTING_TYPE_FULL_SCREEN_VIDEO,
 	PR_COASTING_TYPE_TEST_HARNESS,
+	PR_COASTING_TYPE_VIDEO_CONFERENCING_V2,
 	PR_COASTING_TYPE_NUM,
 };
 
@@ -1134,6 +1135,17 @@ union replay_low_refresh_rate_enable_options {
 	unsigned int raw;
 };
 
+union replay_optimization {
+	struct {
+		//BIT[0-3]: Replay Teams Optimization
+		unsigned int TEAMS_OPTIMIZATION_VER_1           :1;
+		unsigned int TEAMS_OPTIMIZATION_VER_2           :1;
+		unsigned int RESERVED_2_3                       :2;
+	} bits;
+
+	unsigned int raw;
+};
+
 struct replay_config {
 	/* Replay version */
 	enum dc_replay_version replay_version;
@@ -1171,6 +1183,8 @@ struct replay_config {
 	enum dc_alpm_mode alpm_mode;
 	/* Replay full screen only */
 	bool os_request_force_ffu;
+	/* Replay optimization */
+	union replay_optimization replay_optimization;
 };
 
 /* Replay feature flags*/
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
index 87dbb8d7ed27..b44c364519dc 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
@@ -117,6 +117,8 @@ static const struct link_encoder_funcs dce110_lnk_enc_funcs = {
 	.destroy = dce110_link_encoder_destroy,
 	.get_max_link_cap = dce110_link_encoder_get_max_link_cap,
 	.get_dig_frontend = dce110_get_dig_frontend,
+	.get_hpd_state = dce110_get_hpd_state,
+	.program_hpd_filter = dce110_program_hpd_filter,
 };
 
 static enum bp_result link_transmitter_control(
@@ -851,6 +853,7 @@ void dce110_link_encoder_construct(
 	enc110->base.ctx = init_data->ctx;
 	enc110->base.id = init_data->encoder;
 
+	enc110->base.hpd_gpio = init_data->hpd_gpio;
 	enc110->base.hpd_source = init_data->hpd_source;
 	enc110->base.connector = init_data->connector;
 
@@ -1053,6 +1056,11 @@ void dce110_link_encoder_hw_init(
 
 void dce110_link_encoder_destroy(struct link_encoder **enc)
 {
+	if ((*enc)->hpd_gpio) {
+		dal_gpio_destroy_irq(&(*enc)->hpd_gpio);
+		(*enc)->hpd_gpio = NULL;
+	}
+
 	kfree(TO_DCE110_LINK_ENC(*enc));
 	*enc = NULL;
 }
@@ -1751,6 +1759,40 @@ void dce110_link_encoder_get_max_link_cap(struct link_encoder *enc,
 	*link_settings = max_link_cap;
 }
 
+bool dce110_get_hpd_state(struct link_encoder *enc)
+{
+	uint32_t state = 0;
+
+	if (!enc->hpd_gpio)
+		return false;
+
+	dal_gpio_lock_pin(enc->hpd_gpio);
+	dal_gpio_get_value(enc->hpd_gpio, &state);
+	dal_gpio_unlock_pin(enc->hpd_gpio);
+
+	return state;
+}
+
+bool dce110_program_hpd_filter(struct link_encoder *enc, int delay_on_connect_in_ms, int delay_on_disconnect_in_ms)
+{
+	/* Setup HPD filtering */
+	if (enc->hpd_gpio && dal_gpio_lock_pin(enc->hpd_gpio) == GPIO_RESULT_OK) {
+		struct gpio_hpd_config config;
+
+		config.delay_on_connect = delay_on_connect_in_ms;
+		config.delay_on_disconnect = delay_on_disconnect_in_ms;
+
+		dal_irq_setup_hpd_filter(enc->hpd_gpio, &config);
+
+		dal_gpio_unlock_pin(enc->hpd_gpio);
+
+		return true;
+	} else {
+		ASSERT(0);
+		return false;
+	}
+}
+
 #if defined(CONFIG_DRM_AMD_DC_SI)
 static const struct link_encoder_funcs dce60_lnk_enc_funcs = {
 	.validate_output_with_stream =
@@ -1775,7 +1817,9 @@ static const struct link_encoder_funcs dce60_lnk_enc_funcs = {
 	.is_dig_enabled = dce110_is_dig_enabled,
 	.destroy = dce110_link_encoder_destroy,
 	.get_max_link_cap = dce110_link_encoder_get_max_link_cap,
-	.get_dig_frontend = dce110_get_dig_frontend
+	.get_dig_frontend = dce110_get_dig_frontend,
+	.get_hpd_state = dce110_get_hpd_state,
+	.program_hpd_filter = dce110_program_hpd_filter,
 };
 
 void dce60_link_encoder_construct(
@@ -1794,6 +1838,7 @@ void dce60_link_encoder_construct(
 	enc110->base.ctx = init_data->ctx;
 	enc110->base.id = init_data->encoder;
 
+	enc110->base.hpd_gpio = init_data->hpd_gpio;
 	enc110->base.hpd_source = init_data->hpd_source;
 	enc110->base.connector = init_data->connector;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h
index c58b69bc319b..9ba533aa6f88 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h
@@ -130,11 +130,6 @@
 	SRI(DP_DPHY_HBR2_PATTERN_CONTROL, DP, id), \
 	SR(DCI_MEM_PWR_STATUS)
 
-#define LE_DCN10_REG_LIST(id)\
-	LE_COMMON_REG_LIST_BASE(id), \
-	SRI(DP_DPHY_BS_SR_SWAP_CNTL, DP, id), \
-	SRI(DP_DPHY_INTERNAL_CTRL, DP, id), \
-	SRI(DP_DPHY_HBR2_PATTERN_CONTROL, DP, id)
 
 struct dce110_link_enc_aux_registers {
 	uint32_t AUX_CONTROL;
@@ -319,4 +314,7 @@ bool dce110_is_dig_enabled(struct link_encoder *enc);
 void dce110_link_encoder_get_max_link_cap(struct link_encoder *enc,
 	struct dc_link_settings *link_settings);
 
+bool dce110_get_hpd_state(struct link_encoder *enc);
+bool dce110_program_hpd_filter(struct link_encoder *enc, int delay_on_connect_in_ms, int delay_on_disconnect_in_ms);
+
 #endif /* __DC_LINK_ENCODER__DCE110_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_link_encoder.c
index 8d31fa131cd6..9459e8f28338 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_link_encoder.c
@@ -104,6 +104,8 @@ static const struct link_encoder_funcs dcn201_link_enc_funcs = {
 	.fec_is_active = enc2_fec_is_active,
 	.is_in_alt_mode = dcn201_link_encoder_is_in_alt_mode,
 	.get_max_link_cap = dcn201_link_encoder_get_max_link_cap,
+	.get_hpd_state = dcn10_get_hpd_state,
+	.program_hpd_filter = dcn10_program_hpd_filter,
 };
 
 void dcn201_link_encoder_construct(
@@ -125,6 +127,7 @@ void dcn201_link_encoder_construct(
 	enc10->base.ctx = init_data->ctx;
 	enc10->base.id = init_data->encoder;
 
+	enc10->base.hpd_gpio = init_data->hpd_gpio;
 	enc10->base.hpd_source = init_data->hpd_source;
 	enc10->base.connector = init_data->connector;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_link_encoder.c
index eb9abb9f9698..36456c9971c8 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_link_encoder.c
@@ -325,6 +325,8 @@ static const struct link_encoder_funcs dcn21_link_enc_funcs = {
 	.get_dig_frontend = dcn10_get_dig_frontend,
 	.is_in_alt_mode = dcn20_link_encoder_is_in_alt_mode,
 	.get_max_link_cap = dcn20_link_encoder_get_max_link_cap,
+	.get_hpd_state = dcn10_get_hpd_state,
+	.program_hpd_filter = dcn10_program_hpd_filter,
 };
 
 void dcn21_link_encoder_construct(
@@ -346,6 +348,7 @@ void dcn21_link_encoder_construct(
 	enc10->base.ctx = init_data->ctx;
 	enc10->base.id = init_data->encoder;
 
+	enc10->base.hpd_gpio = init_data->hpd_gpio;
 	enc10->base.hpd_source = init_data->hpd_source;
 	enc10->base.connector = init_data->connector;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.c
index 1c1228116487..13e14aad3daa 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.c
@@ -88,8 +88,11 @@ static const struct link_encoder_funcs dcn10_lnk_enc_funcs = {
 	.get_dig_mode = dcn10_get_dig_mode,
 	.destroy = dcn10_link_encoder_destroy,
 	.get_max_link_cap = dcn10_link_encoder_get_max_link_cap,
+	.get_hpd_state = dcn10_get_hpd_state,
+	.program_hpd_filter = dcn10_program_hpd_filter,
 };
 
+
 static enum bp_result link_transmitter_control(
 	struct dcn10_link_encoder *enc10,
 	struct bp_transmitter_control *cntl)
@@ -682,6 +685,7 @@ void dcn10_link_encoder_construct(
 	enc10->base.ctx = init_data->ctx;
 	enc10->base.id = init_data->encoder;
 
+	enc10->base.hpd_gpio = init_data->hpd_gpio;
 	enc10->base.hpd_source = init_data->hpd_source;
 	enc10->base.connector = init_data->connector;
 
@@ -873,6 +877,11 @@ void dcn10_link_encoder_hw_init(
 
 void dcn10_link_encoder_destroy(struct link_encoder **enc)
 {
+	if ((*enc)->hpd_gpio) {
+		dal_gpio_destroy_irq(&(*enc)->hpd_gpio);
+		(*enc)->hpd_gpio = NULL;
+	}
+
 	kfree(TO_DCN10_LINK_ENC(*enc));
 	*enc = NULL;
 }
@@ -1472,3 +1481,37 @@ void dcn10_link_encoder_get_max_link_cap(struct link_encoder *enc,
 
 	*link_settings = max_link_cap;
 }
+
+bool dcn10_get_hpd_state(struct link_encoder *enc)
+{
+	uint32_t state = 0;
+
+	if (!enc->hpd_gpio)
+		return false;
+
+	dal_gpio_lock_pin(enc->hpd_gpio);
+	dal_gpio_get_value(enc->hpd_gpio, &state);
+	dal_gpio_unlock_pin(enc->hpd_gpio);
+
+	return state;
+}
+
+bool dcn10_program_hpd_filter(struct link_encoder *enc, int delay_on_connect_in_ms, int delay_on_disconnect_in_ms)
+{
+	/* Setup HPD filtering */
+	if (enc->hpd_gpio && dal_gpio_lock_pin(enc->hpd_gpio) == GPIO_RESULT_OK) {
+		struct gpio_hpd_config config;
+
+		config.delay_on_connect = delay_on_connect_in_ms;
+		config.delay_on_disconnect = delay_on_disconnect_in_ms;
+
+		dal_irq_setup_hpd_filter(enc->hpd_gpio, &config);
+
+		dal_gpio_unlock_pin(enc->hpd_gpio);
+
+		return true;
+	} else {
+		ASSERT(0);
+		return false;
+	}
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.h
index b7a89c39f445..eedbd5d2756e 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.h
@@ -79,6 +79,8 @@ struct dcn10_link_enc_aux_registers {
 
 struct dcn10_link_enc_hpd_registers {
 	uint32_t DC_HPD_CONTROL;
+	uint32_t DC_HPD_INT_STATUS;
+	uint32_t DC_HPD_TOGGLE_FILT_CNTL;
 };
 
 struct dcn10_link_enc_registers {
@@ -274,7 +276,10 @@ struct dcn10_link_enc_registers {
 	type TMDS_CTL0;\
 	type AUX_HPD_SEL;\
 	type AUX_LS_READ_EN;\
-	type AUX_RX_RECEIVE_WINDOW
+	type AUX_RX_RECEIVE_WINDOW;\
+	type DC_HPD_SENSE;\
+	type DC_HPD_CONNECT_INT_DELAY;\
+	type DC_HPD_DISCONNECT_INT_DELAY
 
 
 #define DCN20_LINK_ENCODER_DPCS_REG_FIELD_LIST(type) \
@@ -656,4 +661,8 @@ enum signal_type dcn10_get_dig_mode(
 
 void dcn10_link_encoder_get_max_link_cap(struct link_encoder *enc,
 	struct dc_link_settings *link_settings);
+
+bool dcn10_get_hpd_state(struct link_encoder *enc);
+bool dcn10_program_hpd_filter(struct link_encoder *enc, int delay_on_connect_in_ms, int delay_on_disconnect_in_ms);
+
 #endif /* __DC_LINK_ENCODER__DCN10_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn20/dcn20_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn20/dcn20_link_encoder.c
index 51a57dae1811..3bd35f3392dc 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn20/dcn20_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn20/dcn20_link_encoder.c
@@ -384,6 +384,8 @@ static const struct link_encoder_funcs dcn20_link_enc_funcs = {
 	.get_dig_frontend = dcn10_get_dig_frontend,
 	.is_in_alt_mode = dcn20_link_encoder_is_in_alt_mode,
 	.get_max_link_cap = dcn20_link_encoder_get_max_link_cap,
+	.get_hpd_state = dcn10_get_hpd_state,
+	.program_hpd_filter = dcn10_program_hpd_filter,
 };
 
 void dcn20_link_encoder_construct(
@@ -405,6 +407,7 @@ void dcn20_link_encoder_construct(
 	enc10->base.ctx = init_data->ctx;
 	enc10->base.id = init_data->encoder;
 
+	enc10->base.hpd_gpio = init_data->hpd_gpio;
 	enc10->base.hpd_source = init_data->hpd_source;
 	enc10->base.connector = init_data->connector;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn30/dcn30_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn30/dcn30_dio_link_encoder.c
index b8e31b5ea114..57b9ae5fca1d 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn30/dcn30_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn30/dcn30_dio_link_encoder.c
@@ -84,6 +84,8 @@ static const struct link_encoder_funcs dcn30_link_enc_funcs = {
 	.get_dig_mode = dcn10_get_dig_mode,
 	.is_in_alt_mode = dcn20_link_encoder_is_in_alt_mode,
 	.get_max_link_cap = dcn20_link_encoder_get_max_link_cap,
+	.get_hpd_state = dcn10_get_hpd_state,
+	.program_hpd_filter = dcn10_program_hpd_filter,
 };
 
 void dcn30_link_encoder_construct(
@@ -105,6 +107,7 @@ void dcn30_link_encoder_construct(
 	enc10->base.ctx = init_data->ctx;
 	enc10->base.id = init_data->encoder;
 
+	enc10->base.hpd_gpio = init_data->hpd_gpio;
 	enc10->base.hpd_source = init_data->hpd_source;
 	enc10->base.connector = init_data->connector;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn301/dcn301_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn301/dcn301_dio_link_encoder.c
index 1b39a6e8a1ac..47d84a2a48ce 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn301/dcn301_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn301/dcn301_dio_link_encoder.c
@@ -73,6 +73,8 @@ static const struct link_encoder_funcs dcn301_link_enc_funcs = {
 	.get_dig_mode = dcn10_get_dig_mode,
 	.is_in_alt_mode = dcn20_link_encoder_is_in_alt_mode,
 	.get_max_link_cap = dcn20_link_encoder_get_max_link_cap,
+	.get_hpd_state = dcn10_get_hpd_state,
+	.program_hpd_filter = dcn10_program_hpd_filter,
 };
 
 void dcn301_link_encoder_construct(
@@ -94,6 +96,7 @@ void dcn301_link_encoder_construct(
 	enc10->base.ctx = init_data->ctx;
 	enc10->base.id = init_data->encoder;
 
+	enc10->base.hpd_gpio = init_data->hpd_gpio;
 	enc10->base.hpd_source = init_data->hpd_source;
 	enc10->base.connector = init_data->connector;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn31/dcn31_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn31/dcn31_dio_link_encoder.c
index 84cc2ddc52fe..07d362ef0daf 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn31/dcn31_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn31/dcn31_dio_link_encoder.c
@@ -276,6 +276,8 @@ static const struct link_encoder_funcs dcn31_link_enc_funcs = {
 	.is_in_alt_mode = dcn31_link_encoder_is_in_alt_mode,
 	.get_max_link_cap = dcn31_link_encoder_get_max_link_cap,
 	.set_dio_phy_mux = dcn31_link_encoder_set_dio_phy_mux,
+	.get_hpd_state = dcn10_get_hpd_state,
+	.program_hpd_filter = dcn10_program_hpd_filter,
 };
 
 void dcn31_link_encoder_construct(
@@ -297,6 +299,7 @@ void dcn31_link_encoder_construct(
 	enc10->base.ctx = init_data->ctx;
 	enc10->base.id = init_data->encoder;
 
+	enc10->base.hpd_gpio = init_data->hpd_gpio;
 	enc10->base.hpd_source = init_data->hpd_source;
 	enc10->base.connector = init_data->connector;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c
index 06907e8a4eda..65d28cb07b04 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c
@@ -188,9 +188,18 @@ void dcn32_link_encoder_get_max_link_cap(struct link_encoder *enc,
 	if (!query_dp_alt_from_dmub(enc, &cmd))
 		return;
 
-	if (cmd.query_dp_alt.data.is_usb &&
-			cmd.query_dp_alt.data.is_dp4 == 0)
-		link_settings->lane_count = MIN(LANE_COUNT_TWO, link_settings->lane_count);
+	/*
+	 * USB-C DisplayPort Alt Mode lane count limitation logic:
+	 * When USB and DP share the same USB-C connector, hardware must allocate
+	 * some lanes for USB data, limiting DP to maximum 2 lanes instead of 4.
+	 * This ensures USB functionality remains available while DP is active.
+	 */
+	if (cmd.query_dp_alt.data.is_dp_alt_disable == 0 &&
+		cmd.query_dp_alt.data.is_usb &&
+		cmd.query_dp_alt.data.is_dp4 == 0) {
+		link_settings->lane_count =
+			MIN(LANE_COUNT_TWO, link_settings->lane_count);
+	}
 }
 
 
@@ -224,6 +233,8 @@ static const struct link_encoder_funcs dcn32_link_enc_funcs = {
 	.is_in_alt_mode = dcn32_link_encoder_is_in_alt_mode,
 	.get_max_link_cap = dcn32_link_encoder_get_max_link_cap,
 	.set_dio_phy_mux = dcn31_link_encoder_set_dio_phy_mux,
+	.get_hpd_state = dcn10_get_hpd_state,
+	.program_hpd_filter = dcn10_program_hpd_filter,
 };
 
 void dcn32_link_encoder_construct(
@@ -245,6 +256,7 @@ void dcn32_link_encoder_construct(
 	enc10->base.ctx = init_data->ctx;
 	enc10->base.id = init_data->encoder;
 
+	enc10->base.hpd_gpio = init_data->hpd_gpio;
 	enc10->base.hpd_source = init_data->hpd_source;
 	enc10->base.connector = init_data->connector;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn321/dcn321_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn321/dcn321_dio_link_encoder.c
index 2ed382a8e79c..968f89295b64 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn321/dcn321_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn321/dcn321_dio_link_encoder.c
@@ -89,6 +89,8 @@ static const struct link_encoder_funcs dcn321_link_enc_funcs = {
 	.is_in_alt_mode = dcn20_link_encoder_is_in_alt_mode,
 	.get_max_link_cap = dcn20_link_encoder_get_max_link_cap,
 	.set_dio_phy_mux = dcn31_link_encoder_set_dio_phy_mux,
+	.get_hpd_state = dcn10_get_hpd_state,
+	.program_hpd_filter = dcn10_program_hpd_filter,
 };
 
 void dcn321_link_encoder_construct(
@@ -110,6 +112,7 @@ void dcn321_link_encoder_construct(
 	enc10->base.ctx = init_data->ctx;
 	enc10->base.id = init_data->encoder;
 
+	enc10->base.hpd_gpio = init_data->hpd_gpio;
 	enc10->base.hpd_source = init_data->hpd_source;
 	enc10->base.connector = init_data->connector;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn35/dcn35_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn35/dcn35_dio_link_encoder.c
index 9972911330b6..20bf04dac609 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn35/dcn35_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn35/dcn35_dio_link_encoder.c
@@ -120,7 +120,6 @@ void dcn35_link_encoder_setup(
 void dcn35_link_encoder_init(struct link_encoder *enc)
 {
 	enc31_hw_init(enc);
-	dcn35_link_encoder_set_fgcg(enc, enc->ctx->dc->debug.enable_fine_grain_clock_gating.bits.dio);
 }
 
 void dcn35_link_encoder_set_fgcg(struct link_encoder *enc, bool enable)
@@ -162,6 +161,8 @@ static const struct link_encoder_funcs dcn35_link_enc_funcs = {
 	.set_dio_phy_mux = dcn31_link_encoder_set_dio_phy_mux,
 	.enable_dpia_output = dcn35_link_encoder_enable_dpia_output,
 	.disable_dpia_output = dcn35_link_encoder_disable_dpia_output,
+	.get_hpd_state = dcn10_get_hpd_state,
+	.program_hpd_filter = dcn10_program_hpd_filter,
 };
 
 void dcn35_link_encoder_construct(
@@ -183,6 +184,7 @@ void dcn35_link_encoder_construct(
 	enc10->base.ctx = init_data->ctx;
 	enc10->base.id = init_data->encoder;
 
+	enc10->base.hpd_gpio = init_data->hpd_gpio;
 	enc10->base.hpd_source = init_data->hpd_source;
 	enc10->base.connector = init_data->connector;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn401/dcn401_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn401/dcn401_dio_link_encoder.c
index 7e558ca195ef..e1f0a1bf1075 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn401/dcn401_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn401/dcn401_dio_link_encoder.c
@@ -215,6 +215,8 @@ static const struct link_encoder_funcs dcn401_link_enc_funcs = {
 	.is_in_alt_mode = dcn32_link_encoder_is_in_alt_mode,
 	.get_max_link_cap = dcn32_link_encoder_get_max_link_cap,
 	.set_dio_phy_mux = dcn31_link_encoder_set_dio_phy_mux,
+	.get_hpd_state = dcn10_get_hpd_state,
+	.program_hpd_filter = dcn10_program_hpd_filter,
 };
 
 void dcn401_link_encoder_construct(
@@ -236,6 +238,7 @@ void dcn401_link_encoder_construct(
 	enc10->base.ctx = init_data->ctx;
 	enc10->base.id = init_data->encoder;
 
+	enc10->base.hpd_gpio = init_data->hpd_gpio;
 	enc10->base.hpd_source = init_data->hpd_source;
 	enc10->base.connector = init_data->connector;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile b/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile
index 97e068b6bf6b..30cfc0848792 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile
@@ -55,7 +55,7 @@ subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/dml2_0/dml21/
 
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/display_mode_core.o := $(dml2_ccflags) $(frame_warn_flag)
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/display_mode_util.o := $(dml2_ccflags)
-CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml2_wrapper.o := $(dml2_ccflags)
+CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml2_wrapper_fpu.o := $(dml2_ccflags)
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml2_utils.o := $(dml2_ccflags)
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml2_policy.o := $(dml2_ccflags)
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml2_translation_helper.o := $(dml2_ccflags)
@@ -65,7 +65,7 @@ CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml2_dc_resource_mgmt.o := $(dml2_ccflags)
 
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/display_mode_core.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/display_mode_util.o := $(dml2_rcflags)
-CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml2_wrapper.o := $(dml2_rcflags)
+CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml2_wrapper_fpu.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml2_utils.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml2_policy.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml2_translation_helper.o := $(dml2_rcflags)
@@ -73,7 +73,7 @@ CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml2_mall_phantom.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml_display_rq_dlg_calc.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml2_dc_resource_mgmt.o := $(dml2_rcflags)
 
-DML2 = display_mode_core.o display_mode_util.o dml2_wrapper.o \
+DML2 = display_mode_core.o display_mode_util.o dml2_wrapper_fpu.o dml2_wrapper.o \
 		dml2_utils.o dml2_policy.o dml2_translation_helper.o dml2_dc_resource_mgmt.o dml2_mall_phantom.o \
 		dml_display_rq_dlg_calc.o
 
@@ -95,7 +95,6 @@ CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn3.o := $(dml2_ccfl
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn4_fams2.o := $(dml2_ccflags)
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.o := $(dml2_ccflags)
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_standalone_libraries/lib_float_math.o := $(dml2_ccflags)
-CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml21_wrapper.o := $(dml2_ccflags)
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/dml21_translation_helper.o := $(dml2_ccflags)
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/dml21_utils.o := $(dml2_ccflags)
 
@@ -113,7 +112,6 @@ CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn3.o := $(dm
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn4_fams2.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_standalone_libraries/lib_float_math.o := $(dml2_rcflags)
-CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml21_wrapper.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/dml21_translation_helper.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/dml21_utils.o := $(dml2_rcflags)
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.c
index ee721606b883..f667026cb43e 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.c
@@ -387,7 +387,8 @@ void dml21_build_fams2_programming(const struct dc *dc,
 	memset(&context->bw_ctx.bw.dcn.fams2_stream_sub_params_v2, 0, sizeof(union dmub_fams2_stream_static_sub_state_v2) * DML2_MAX_PLANES);
 	memset(&context->bw_ctx.bw.dcn.fams2_global_config, 0, sizeof(struct dmub_cmd_fams2_global_config));
 
-	if (dml_ctx->v21.mode_programming.programming->fams2_required) {
+	if ((dml_ctx->v21.mode_programming.programming->fams2_required) ||
+		(dml_ctx->v21.mode_programming.programming->legacy_pstate_info_for_dmu)) {
 		for (i = 0; i < context->stream_count; i++) {
 			int dml_stream_idx;
 			struct dc_stream_state *phantom_stream;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper.h
index 15f92029d2e5..b508bbcc0e16 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper.h
@@ -17,9 +17,9 @@ struct dml2_context;
 enum dc_validate_mode;
 
 /**
- * dml2_create - Creates dml21_context.
+ * dml21_create - Creates dml21_context.
  * @in_dc: dc.
- * @dml2: Created dml21 context.
+ * @dml_ctx: Created dml21 context.
  * @config: dml21 configuration options.
  *
  * Create of DML21 is done as part of dc_state creation.
@@ -40,6 +40,7 @@ void dml21_reinit(const struct dc *in_dc, struct dml2_context *dml_ctx, const st
  * dml21_validate - Determines if a display configuration is supported or not.
  * @in_dc: dc.
  * @context: dc_state to be validated.
+ * @dml_ctx: dml21 context.
  * @validate_mode: DC_VALIDATE_MODE_ONLY and DC_VALIDATE_MODE_AND_STATE_INDEX
  *           will not populate context.res_ctx.
  *
@@ -53,7 +54,7 @@ void dml21_reinit(const struct dc *in_dc, struct dml2_context *dml_ctx, const st
  * -dml21_check_mode_support - for DC_VALIDATE_MODE_ONLY and DC_VALIDATE_MODE_AND_STATE_INDEX option
  * Calculates if dc_state can be supported for the input display
  * config.
-
+ *
  * Context: Two threads may not invoke this function concurrently unless they reference
  *          separate dc_states for validation.
  * Return: True if mode is supported, false otherwise.
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_display_cfg_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_display_cfg_types.h
index 35aa954248cd..b44762e21550 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_display_cfg_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_display_cfg_types.h
@@ -407,6 +407,7 @@ struct dml2_plane_parameters {
 		unsigned int hostvm_min_page_size_kbytes;
 
 		enum dml2_svp_mode_override legacy_svp_config; //TODO remove in favor of svp_config
+		bool use_max_lsw;
 
 		struct {
 			// HW specific overrides, there's almost no reason to mess with these
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_types.h
index 452e4a2e72c0..943fd3f040c3 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_types.h
@@ -418,6 +418,7 @@ struct dml2_display_cfg_programming {
 
 	/* indicates this configuration requires FW to support */
 	bool fams2_required;
+	bool legacy_pstate_info_for_dmu;
 	struct dmub_cmd_fams2_global_config fams2_global_config;
 
 	struct {
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c
index a02e9fd6b5ca..01b87be24ce3 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c
@@ -326,6 +326,7 @@ dml_get_var_func(stutter_efficiency, double, mode_lib->mp.StutterEfficiency);
 dml_get_var_func(stutter_efficiency_no_vblank, double, mode_lib->mp.StutterEfficiencyNotIncludingVBlank);
 dml_get_var_func(stutter_num_bursts, double, mode_lib->mp.NumberOfStutterBurstsPerFrame);
 dml_get_var_func(stutter_efficiency_z8, double, mode_lib->mp.Z8StutterEfficiency);
+dml_get_var_func(stutter_efficiency_no_vblank_z8, double, mode_lib->mp.Z8StutterEfficiencyNotIncludingVBlank);
 dml_get_var_func(stutter_num_bursts_z8, double, mode_lib->mp.Z8NumberOfStutterBurstsPerFrame);
 dml_get_var_func(stutter_period, double, mode_lib->mp.StutterPeriod);
 dml_get_var_func(stutter_efficiency_z8_bestcase, double, mode_lib->mp.Z8StutterEfficiencyBestCase);
@@ -13198,8 +13199,8 @@ void dml2_core_calcs_get_informative(const struct dml2_core_internal_display_mod
 	out->informative.power_management.stutter_efficiency_with_vblank = dml_get_stutter_efficiency(mode_lib);
 	out->informative.power_management.stutter_num_bursts = dml_get_stutter_num_bursts(mode_lib);
 
-	out->informative.power_management.z8.stutter_efficiency = dml_get_stutter_efficiency_z8(mode_lib);
-	out->informative.power_management.z8.stutter_efficiency_with_vblank = dml_get_stutter_efficiency(mode_lib);
+	out->informative.power_management.z8.stutter_efficiency = dml_get_stutter_efficiency_no_vblank_z8(mode_lib);
+	out->informative.power_management.z8.stutter_efficiency_with_vblank = dml_get_stutter_efficiency_z8(mode_lib);
 	out->informative.power_management.z8.stutter_num_bursts = dml_get_stutter_num_bursts_z8(mode_lib);
 	out->informative.power_management.z8.stutter_period = dml_get_stutter_period(mode_lib);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_shared_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_shared_types.h
index 1087a8c926ff..953f40fde1e1 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_shared_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_shared_types.h
@@ -1950,6 +1950,7 @@ struct dml2_core_calcs_CalculatePrefetchSchedule_params {
 	double Ttrip;
 	double Turg;
 	bool setup_for_tdlut;
+	bool use_max_lsw;
 	unsigned int tdlut_pte_bytes_per_frame;
 	unsigned int tdlut_bytes_per_frame;
 	double tdlut_opt_time;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.c
index 9deb03a18ccc..307186eb6af0 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.c
@@ -1,549 +1,12 @@
-/* SPDX-License-Identifier: MIT */
+// SPDX-License-Identifier: MIT
 /*
- * Copyright 2023 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
+ * Copyright 2025 Advanced Micro Devices, Inc.
  *
  * Authors: AMD
- *
  */
 
-#include "display_mode_core.h"
 #include "dml2_internal_types.h"
-#include "dml2_utils.h"
-#include "dml2_policy.h"
-#include "dml2_translation_helper.h"
-#include "dml2_mall_phantom.h"
-#include "dml2_dc_resource_mgmt.h"
-#include "dml21_wrapper.h"
-
-static void initialize_dml2_ip_params(struct dml2_context *dml2, const struct dc *in_dc, struct ip_params_st *out)
-{
-	if (dml2->config.use_native_soc_bb_construction)
-		dml2_init_ip_params(dml2, in_dc, out);
-	else
-		dml2_translate_ip_params(in_dc, out);
-}
-
-static void initialize_dml2_soc_bbox(struct dml2_context *dml2, const struct dc *in_dc, struct soc_bounding_box_st *out)
-{
-	if (dml2->config.use_native_soc_bb_construction)
-		dml2_init_socbb_params(dml2, in_dc, out);
-	else
-		dml2_translate_socbb_params(in_dc, out);
-}
-
-static void initialize_dml2_soc_states(struct dml2_context *dml2,
-	const struct dc *in_dc, const struct soc_bounding_box_st *in_bbox, struct soc_states_st *out)
-{
-	if (dml2->config.use_native_soc_bb_construction)
-		dml2_init_soc_states(dml2, in_dc, in_bbox, out);
-	else
-		dml2_translate_soc_states(in_dc, out, in_dc->dml.soc.num_states);
-}
-
-static void map_hw_resources(struct dml2_context *dml2,
-		struct dml_display_cfg_st *in_out_display_cfg, struct dml_mode_support_info_st *mode_support_info)
-{
-	unsigned int num_pipes = 0;
-	int i, j;
-
-	for (i = 0; i < __DML_NUM_PLANES__; i++) {
-		in_out_display_cfg->hw.ODMMode[i] = mode_support_info->ODMMode[i];
-		in_out_display_cfg->hw.DPPPerSurface[i] = mode_support_info->DPPPerSurface[i];
-		in_out_display_cfg->hw.DSCEnabled[i] = mode_support_info->DSCEnabled[i];
-		in_out_display_cfg->hw.NumberOfDSCSlices[i] = mode_support_info->NumberOfDSCSlices[i];
-		in_out_display_cfg->hw.DLGRefClkFreqMHz = 24;
-		if (dml2->v20.dml_core_ctx.project != dml_project_dcn35 &&
-			dml2->v20.dml_core_ctx.project != dml_project_dcn36 &&
-			dml2->v20.dml_core_ctx.project != dml_project_dcn351) {
-			/*dGPU default as 50Mhz*/
-			in_out_display_cfg->hw.DLGRefClkFreqMHz = 50;
-		}
-		for (j = 0; j < mode_support_info->DPPPerSurface[i]; j++) {
-			if (i >= __DML2_WRAPPER_MAX_STREAMS_PLANES__) {
-				dml_print("DML::%s: Index out of bounds: i=%d, __DML2_WRAPPER_MAX_STREAMS_PLANES__=%d\n",
-					  __func__, i, __DML2_WRAPPER_MAX_STREAMS_PLANES__);
-				break;
-			}
-			dml2->v20.scratch.dml_to_dc_pipe_mapping.dml_pipe_idx_to_stream_id[num_pipes] = dml2->v20.scratch.dml_to_dc_pipe_mapping.disp_cfg_to_stream_id[i];
-			dml2->v20.scratch.dml_to_dc_pipe_mapping.dml_pipe_idx_to_stream_id_valid[num_pipes] = true;
-			dml2->v20.scratch.dml_to_dc_pipe_mapping.dml_pipe_idx_to_plane_id[num_pipes] = dml2->v20.scratch.dml_to_dc_pipe_mapping.disp_cfg_to_plane_id[i];
-			dml2->v20.scratch.dml_to_dc_pipe_mapping.dml_pipe_idx_to_plane_id_valid[num_pipes] = true;
-			num_pipes++;
-		}
-	}
-}
-
-static unsigned int pack_and_call_dml_mode_support_ex(struct dml2_context *dml2,
-	const struct dml_display_cfg_st *display_cfg,
-	struct dml_mode_support_info_st *evaluation_info,
-	enum dc_validate_mode validate_mode)
-{
-	struct dml2_wrapper_scratch *s = &dml2->v20.scratch;
-
-	s->mode_support_params.mode_lib = &dml2->v20.dml_core_ctx;
-	s->mode_support_params.in_display_cfg = display_cfg;
-	if (validate_mode == DC_VALIDATE_MODE_ONLY)
-		s->mode_support_params.in_start_state_idx = dml2->v20.dml_core_ctx.states.num_states - 1;
-	else
-		s->mode_support_params.in_start_state_idx = 0;
-	s->mode_support_params.out_evaluation_info = evaluation_info;
-
-	memset(evaluation_info, 0, sizeof(struct dml_mode_support_info_st));
-	s->mode_support_params.out_lowest_state_idx = 0;
-
-	return dml_mode_support_ex(&s->mode_support_params);
-}
-
-static bool optimize_configuration(struct dml2_context *dml2, struct dml2_wrapper_optimize_configuration_params *p)
-{
-	int unused_dpps = p->ip_params->max_num_dpp;
-	int i;
-	int odms_needed;
-	int largest_blend_and_timing = 0;
-	bool optimization_done = false;
-
-	for (i = 0; i < (int) p->cur_display_config->num_timings; i++) {
-		if (p->cur_display_config->plane.BlendingAndTiming[i] > largest_blend_and_timing)
-			largest_blend_and_timing = p->cur_display_config->plane.BlendingAndTiming[i];
-	}
-
-	if (p->new_policy != p->cur_policy)
-		*p->new_policy = *p->cur_policy;
-
-	if (p->new_display_config != p->cur_display_config)
-		*p->new_display_config = *p->cur_display_config;
-
-
-	// Optimize Clocks
-	if (!optimization_done) {
-		if (largest_blend_and_timing == 0 && p->cur_policy->ODMUse[0] == dml_odm_use_policy_combine_as_needed && dml2->config.minimize_dispclk_using_odm) {
-			odms_needed = dml2_util_get_maximum_odm_combine_for_output(dml2->config.optimize_odm_4to1,
-				p->cur_display_config->output.OutputEncoder[0], p->cur_mode_support_info->DSCEnabled[0]) - 1;
-
-			if (odms_needed <= unused_dpps) {
-				if (odms_needed == 1) {
-					p->new_policy->ODMUse[0] = dml_odm_use_policy_combine_2to1;
-					optimization_done = true;
-				} else if (odms_needed == 3) {
-					p->new_policy->ODMUse[0] = dml_odm_use_policy_combine_4to1;
-					optimization_done = true;
-				} else
-					optimization_done = false;
-			}
-		}
-	}
-
-	return optimization_done;
-}
-
-static int calculate_lowest_supported_state_for_temp_read(struct dml2_context *dml2, struct dc_state *display_state,
-		enum dc_validate_mode validate_mode)
-{
-	struct dml2_calculate_lowest_supported_state_for_temp_read_scratch *s = &dml2->v20.scratch.dml2_calculate_lowest_supported_state_for_temp_read_scratch;
-	struct dml2_wrapper_scratch *s_global = &dml2->v20.scratch;
-
-	unsigned int dml_result = 0;
-	int result = -1, i, j;
-
-	build_unoptimized_policy_settings(dml2->v20.dml_core_ctx.project, &dml2->v20.dml_core_ctx.policy);
-
-	/* Zero out before each call before proceeding */
-	memset(s, 0, sizeof(struct dml2_calculate_lowest_supported_state_for_temp_read_scratch));
-	memset(&s_global->mode_support_params, 0, sizeof(struct dml_mode_support_ex_params_st));
-	memset(&s_global->dml_to_dc_pipe_mapping, 0, sizeof(struct dml2_dml_to_dc_pipe_mapping));
-
-	for (i = 0; i < dml2->config.dcn_pipe_count; i++) {
-		/* Calling resource_build_scaling_params will populate the pipe params
-		 * with the necessary information needed for correct DML calculations
-		 * This is also done in DML1 driver code path and hence display_state
-		 * cannot be const.
-		 */
-		struct pipe_ctx *pipe = &display_state->res_ctx.pipe_ctx[i];
-
-		if (pipe->plane_state) {
-			if (!dml2->config.callbacks.build_scaling_params(pipe)) {
-				ASSERT(false);
-				return false;
-			}
-		}
-	}
-
-	map_dc_state_into_dml_display_cfg(dml2, display_state, &s->cur_display_config);
-
-	for (i = 0; i < dml2->v20.dml_core_ctx.states.num_states; i++) {
-		s->uclk_change_latencies[i] = dml2->v20.dml_core_ctx.states.state_array[i].dram_clock_change_latency_us;
-	}
-
-	for (i = 0; i < 4; i++) {
-		for (j = 0; j < dml2->v20.dml_core_ctx.states.num_states; j++) {
-			dml2->v20.dml_core_ctx.states.state_array[j].dram_clock_change_latency_us = s_global->dummy_pstate_table[i].dummy_pstate_latency_us;
-		}
-
-		dml_result = pack_and_call_dml_mode_support_ex(dml2, &s->cur_display_config, &s->evaluation_info,
-						validate_mode);
-
-		if (dml_result && s->evaluation_info.DRAMClockChangeSupport[0] == dml_dram_clock_change_vactive) {
-			map_hw_resources(dml2, &s->cur_display_config, &s->evaluation_info);
-			dml_result = dml_mode_programming(&dml2->v20.dml_core_ctx, s_global->mode_support_params.out_lowest_state_idx, &s->cur_display_config, true);
-
-			ASSERT(dml_result);
-
-			dml2_extract_watermark_set(&dml2->v20.g6_temp_read_watermark_set, &dml2->v20.dml_core_ctx);
-			dml2->v20.g6_temp_read_watermark_set.cstate_pstate.fclk_pstate_change_ns = dml2->v20.g6_temp_read_watermark_set.cstate_pstate.pstate_change_ns;
-
-			result = s_global->mode_support_params.out_lowest_state_idx;
-
-			while (dml2->v20.dml_core_ctx.states.state_array[result].dram_speed_mts < s_global->dummy_pstate_table[i].dram_speed_mts)
-				result++;
-
-			break;
-		}
-	}
-
-	for (i = 0; i < dml2->v20.dml_core_ctx.states.num_states; i++) {
-		dml2->v20.dml_core_ctx.states.state_array[i].dram_clock_change_latency_us = s->uclk_change_latencies[i];
-	}
-
-	return result;
-}
-
-static void copy_dummy_pstate_table(struct dummy_pstate_entry *dest, struct dummy_pstate_entry *src, unsigned int num_entries)
-{
-	for (int i = 0; i < num_entries; i++) {
-		dest[i] = src[i];
-	}
-}
-
-static bool are_timings_requiring_odm_doing_blending(const struct dml_display_cfg_st *display_cfg,
-		const struct dml_mode_support_info_st *evaluation_info)
-{
-	unsigned int planes_per_timing[__DML_NUM_PLANES__] = {0};
-	int i;
-
-	for (i = 0; i < display_cfg->num_surfaces; i++)
-		planes_per_timing[display_cfg->plane.BlendingAndTiming[i]]++;
-
-	for (i = 0; i < __DML_NUM_PLANES__; i++) {
-		if (planes_per_timing[i] > 1 && evaluation_info->ODMMode[i] != dml_odm_mode_bypass)
-			return true;
-	}
-
-	return false;
-}
-
-static bool does_configuration_meet_sw_policies(struct dml2_context *ctx, const struct dml_display_cfg_st *display_cfg,
-	const struct dml_mode_support_info_st *evaluation_info)
-{
-	bool pass = true;
-
-	if (!ctx->config.enable_windowed_mpo_odm) {
-		if (are_timings_requiring_odm_doing_blending(display_cfg, evaluation_info))
-			pass = false;
-	}
-
-	return pass;
-}
-
-static bool dml_mode_support_wrapper(struct dml2_context *dml2,
-		struct dc_state *display_state,
-		enum dc_validate_mode validate_mode)
-{
-	struct dml2_wrapper_scratch *s = &dml2->v20.scratch;
-	unsigned int result = 0, i;
-	unsigned int optimized_result = true;
-
-	build_unoptimized_policy_settings(dml2->v20.dml_core_ctx.project, &dml2->v20.dml_core_ctx.policy);
-
-	/* Zero out before each call before proceeding */
-	memset(&s->cur_display_config, 0, sizeof(struct dml_display_cfg_st));
-	memset(&s->mode_support_params, 0, sizeof(struct dml_mode_support_ex_params_st));
-	memset(&s->dml_to_dc_pipe_mapping, 0, sizeof(struct dml2_dml_to_dc_pipe_mapping));
-	memset(&s->optimize_configuration_params, 0, sizeof(struct dml2_wrapper_optimize_configuration_params));
-
-	for (i = 0; i < dml2->config.dcn_pipe_count; i++) {
-		/* Calling resource_build_scaling_params will populate the pipe params
-		 * with the necessary information needed for correct DML calculations
-		 * This is also done in DML1 driver code path and hence display_state
-		 * cannot be const.
-		 */
-		struct pipe_ctx *pipe = &display_state->res_ctx.pipe_ctx[i];
-
-		if (pipe->plane_state) {
-			if (!dml2->config.callbacks.build_scaling_params(pipe)) {
-				ASSERT(false);
-				return false;
-			}
-		}
-	}
-
-	map_dc_state_into_dml_display_cfg(dml2, display_state, &s->cur_display_config);
-	if (!dml2->config.skip_hw_state_mapping)
-		dml2_apply_det_buffer_allocation_policy(dml2, &s->cur_display_config);
-
-	result = pack_and_call_dml_mode_support_ex(dml2,
-		&s->cur_display_config,
-		&s->mode_support_info,
-		validate_mode);
-
-	if (result)
-		result = does_configuration_meet_sw_policies(dml2, &s->cur_display_config, &s->mode_support_info);
-
-	// Try to optimize
-	if (result) {
-		s->cur_policy = dml2->v20.dml_core_ctx.policy;
-		s->optimize_configuration_params.dml_core_ctx = &dml2->v20.dml_core_ctx;
-		s->optimize_configuration_params.config = &dml2->config;
-		s->optimize_configuration_params.ip_params = &dml2->v20.dml_core_ctx.ip;
-		s->optimize_configuration_params.cur_display_config = &s->cur_display_config;
-		s->optimize_configuration_params.cur_mode_support_info = &s->mode_support_info;
-		s->optimize_configuration_params.cur_policy = &s->cur_policy;
-		s->optimize_configuration_params.new_display_config = &s->new_display_config;
-		s->optimize_configuration_params.new_policy = &s->new_policy;
-
-		while (optimized_result && optimize_configuration(dml2, &s->optimize_configuration_params)) {
-			dml2->v20.dml_core_ctx.policy = s->new_policy;
-			optimized_result = pack_and_call_dml_mode_support_ex(dml2,
-				&s->new_display_config,
-				&s->mode_support_info,
-				validate_mode);
-
-			if (optimized_result)
-				optimized_result = does_configuration_meet_sw_policies(dml2, &s->new_display_config, &s->mode_support_info);
-
-			// If the new optimized state is supposed, then set current = new
-			if (optimized_result) {
-				s->cur_display_config = s->new_display_config;
-				s->cur_policy = s->new_policy;
-			} else {
-				// Else, restore policy to current
-				dml2->v20.dml_core_ctx.policy = s->cur_policy;
-			}
-		}
-
-		// Optimize ended with a failed config, so we need to restore DML state to last passing
-		if (!optimized_result) {
-			result = pack_and_call_dml_mode_support_ex(dml2,
-				&s->cur_display_config,
-				&s->mode_support_info,
-				validate_mode);
-		}
-	}
-
-	if (result)
-		map_hw_resources(dml2, &s->cur_display_config, &s->mode_support_info);
-
-	return result;
-}
-
-static bool call_dml_mode_support_and_programming(struct dc_state *context, enum dc_validate_mode validate_mode)
-{
-	unsigned int result = 0;
-	unsigned int min_state = 0;
-	int min_state_for_g6_temp_read = 0;
-
-
-	if (!context)
-		return false;
-
-	struct dml2_context *dml2 = context->bw_ctx.dml2;
-	struct dml2_wrapper_scratch *s = &dml2->v20.scratch;
-
-	if (!context->streams[0]->sink->link->dc->caps.is_apu) {
-		min_state_for_g6_temp_read = calculate_lowest_supported_state_for_temp_read(dml2, context,
-										validate_mode);
-
-		ASSERT(min_state_for_g6_temp_read >= 0);
-	}
-
-	result = dml_mode_support_wrapper(dml2, context, validate_mode);
-
-	/* Upon trying to sett certain frequencies in FRL, min_state_for_g6_temp_read is reported as -1. This leads to an invalid value of min_state causing crashes later on.
-	 * Use the default logic for min_state only when min_state_for_g6_temp_read is a valid value. In other cases, use the value calculated by the DML directly.
-	 */
-	if (!context->streams[0]->sink->link->dc->caps.is_apu) {
-		if (min_state_for_g6_temp_read >= 0)
-			min_state = min_state_for_g6_temp_read > s->mode_support_params.out_lowest_state_idx ? min_state_for_g6_temp_read : s->mode_support_params.out_lowest_state_idx;
-		else
-			min_state = s->mode_support_params.out_lowest_state_idx;
-	}
-
-	if (result) {
-		if (!context->streams[0]->sink->link->dc->caps.is_apu) {
-			result = dml_mode_programming(&dml2->v20.dml_core_ctx, min_state, &s->cur_display_config, true);
-		} else {
-			result = dml_mode_programming(&dml2->v20.dml_core_ctx, s->mode_support_params.out_lowest_state_idx, &s->cur_display_config, true);
-		}
-	}
-	return result;
-}
-
-static bool dml2_validate_and_build_resource(const struct dc *in_dc, struct dc_state *context,
-		enum dc_validate_mode validate_mode)
-{
-	struct dml2_context *dml2 = context->bw_ctx.dml2;
-	struct dml2_wrapper_scratch *s = &dml2->v20.scratch;
-	struct dml2_dcn_clocks out_clks;
-	unsigned int result = 0;
-	bool need_recalculation = false;
-	uint32_t cstate_enter_plus_exit_z8_ns;
-
-	if (context->stream_count == 0) {
-		unsigned int lowest_state_idx = 0;
-
-		out_clks.p_state_supported = true;
-		out_clks.dispclk_khz = 0; /* No requirement, and lowest index will generally be maximum dispclk. */
-		out_clks.dcfclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].dcfclk_mhz * 1000;
-		out_clks.fclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].fabricclk_mhz * 1000;
-		out_clks.uclk_mts = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].dram_speed_mts;
-		out_clks.phyclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].phyclk_mhz * 1000;
-		out_clks.socclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].socclk_mhz * 1000;
-		out_clks.ref_dtbclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].dtbclk_mhz * 1000;
-		context->bw_ctx.bw.dcn.clk.dtbclk_en = false;
-		dml2_copy_clocks_to_dc_state(&out_clks, context);
-		return true;
-	}
-
-	/* Zero out before each call before proceeding */
-	memset(&dml2->v20.scratch, 0, sizeof(struct dml2_wrapper_scratch));
-	memset(&dml2->v20.dml_core_ctx.policy, 0, sizeof(struct dml_mode_eval_policy_st));
-	memset(&dml2->v20.dml_core_ctx.ms, 0, sizeof(struct mode_support_st));
-	memset(&dml2->v20.dml_core_ctx.mp, 0, sizeof(struct mode_program_st));
-
-	/* Initialize DET scratch */
-	dml2_initialize_det_scratch(dml2);
-
-	copy_dummy_pstate_table(s->dummy_pstate_table, in_dc->clk_mgr->bw_params->dummy_pstate_table, 4);
-
-	result = call_dml_mode_support_and_programming(context, validate_mode);
-	/* Call map dc pipes to map the pipes based on the DML output. For correctly determining if recalculation
-	 * is required or not, the resource context needs to correctly reflect the number of active pipes. We would
-	 * only know the correct number if active pipes after dml2_map_dc_pipes is called.
-	 */
-	if (result && !dml2->config.skip_hw_state_mapping)
-		dml2_map_dc_pipes(dml2, context, &s->cur_display_config, &s->dml_to_dc_pipe_mapping, in_dc->current_state);
-
-	/* Verify and update DET Buffer configuration if needed. dml2_verify_det_buffer_configuration will check if DET Buffer
-	 * size needs to be updated. If yes it will update the DETOverride variable and set need_recalculation flag to true.
-	 * Based on that flag, run mode support again. Verification needs to be run after dml_mode_programming because the getters
-	 * return correct det buffer values only after dml_mode_programming is called.
-	 */
-	if (result && !dml2->config.skip_hw_state_mapping) {
-		need_recalculation = dml2_verify_det_buffer_configuration(dml2, context, &dml2->det_helper_scratch);
-		if (need_recalculation) {
-			/* Engage the DML again if recalculation is required. */
-			call_dml_mode_support_and_programming(context, validate_mode);
-			if (!dml2->config.skip_hw_state_mapping) {
-				dml2_map_dc_pipes(dml2, context, &s->cur_display_config, &s->dml_to_dc_pipe_mapping, in_dc->current_state);
-			}
-			need_recalculation = dml2_verify_det_buffer_configuration(dml2, context, &dml2->det_helper_scratch);
-			ASSERT(need_recalculation == false);
-		}
-	}
-
-	if (result) {
-		unsigned int lowest_state_idx = s->mode_support_params.out_lowest_state_idx;
-		out_clks.dispclk_khz = (unsigned int)dml2->v20.dml_core_ctx.mp.Dispclk_calculated * 1000;
-		out_clks.p_state_supported = s->mode_support_info.DRAMClockChangeSupport[0] != dml_dram_clock_change_unsupported;
-		if (in_dc->config.use_default_clock_table &&
-			(lowest_state_idx < dml2->v20.dml_core_ctx.states.num_states - 1)) {
-			lowest_state_idx = dml2->v20.dml_core_ctx.states.num_states - 1;
-			out_clks.dispclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].dispclk_mhz * 1000;
-		}
-
-		out_clks.dcfclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].dcfclk_mhz * 1000;
-		out_clks.fclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].fabricclk_mhz * 1000;
-		out_clks.uclk_mts = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].dram_speed_mts;
-		out_clks.phyclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].phyclk_mhz * 1000;
-		out_clks.socclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].socclk_mhz * 1000;
-		out_clks.ref_dtbclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].dtbclk_mhz * 1000;
-		context->bw_ctx.bw.dcn.clk.dtbclk_en = is_dtbclk_required(in_dc, context);
-
-		if (!dml2->config.skip_hw_state_mapping) {
-			/* Call dml2_calculate_rq_and_dlg_params */
-			dml2_calculate_rq_and_dlg_params(in_dc, context, &context->res_ctx, dml2, in_dc->res_pool->pipe_count);
-		}
-
-		dml2_copy_clocks_to_dc_state(&out_clks, context);
-		dml2_extract_watermark_set(&context->bw_ctx.bw.dcn.watermarks.a, &dml2->v20.dml_core_ctx);
-		dml2_extract_watermark_set(&context->bw_ctx.bw.dcn.watermarks.b, &dml2->v20.dml_core_ctx);
-		if (context->streams[0]->sink->link->dc->caps.is_apu)
-			dml2_extract_watermark_set(&context->bw_ctx.bw.dcn.watermarks.c, &dml2->v20.dml_core_ctx);
-		else
-			memcpy(&context->bw_ctx.bw.dcn.watermarks.c, &dml2->v20.g6_temp_read_watermark_set, sizeof(context->bw_ctx.bw.dcn.watermarks.c));
-		dml2_extract_watermark_set(&context->bw_ctx.bw.dcn.watermarks.d, &dml2->v20.dml_core_ctx);
-		dml2_extract_writeback_wm(context, &dml2->v20.dml_core_ctx);
-		//copy for deciding zstate use
-		context->bw_ctx.dml.vba.StutterPeriod = context->bw_ctx.dml2->v20.dml_core_ctx.mp.StutterPeriod;
-
-		cstate_enter_plus_exit_z8_ns = context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_z8_ns;
-
-		if (context->bw_ctx.dml.vba.StutterPeriod < in_dc->debug.minimum_z8_residency_time &&
-				cstate_enter_plus_exit_z8_ns < in_dc->debug.minimum_z8_residency_time * 1000)
-			cstate_enter_plus_exit_z8_ns = in_dc->debug.minimum_z8_residency_time * 1000;
-
-		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_z8_ns = cstate_enter_plus_exit_z8_ns;
-	}
-
-	return result;
-}
-
-static bool dml2_validate_only(struct dc_state *context, enum dc_validate_mode validate_mode)
-{
-	struct dml2_context *dml2;
-	unsigned int result = 0;
-
-	if (!context || context->stream_count == 0)
-		return true;
-
-	dml2 = context->bw_ctx.dml2;
-
-	/* Zero out before each call before proceeding */
-	memset(&dml2->v20.scratch, 0, sizeof(struct dml2_wrapper_scratch));
-	memset(&dml2->v20.dml_core_ctx.policy, 0, sizeof(struct dml_mode_eval_policy_st));
-	memset(&dml2->v20.dml_core_ctx.ms, 0, sizeof(struct mode_support_st));
-	memset(&dml2->v20.dml_core_ctx.mp, 0, sizeof(struct mode_program_st));
-
-	build_unoptimized_policy_settings(dml2->v20.dml_core_ctx.project, &dml2->v20.dml_core_ctx.policy);
-
-	map_dc_state_into_dml_display_cfg(dml2, context, &dml2->v20.scratch.cur_display_config);
-	 if (!dml2->config.skip_hw_state_mapping)
-		 dml2_apply_det_buffer_allocation_policy(dml2, &dml2->v20.scratch.cur_display_config);
-
-	result = pack_and_call_dml_mode_support_ex(dml2,
-		&dml2->v20.scratch.cur_display_config,
-		&dml2->v20.scratch.mode_support_info,
-		validate_mode);
-
-	if (result)
-		result = does_configuration_meet_sw_policies(dml2, &dml2->v20.scratch.cur_display_config, &dml2->v20.scratch.mode_support_info);
-
-	return result == 1;
-}
-
-static void dml2_apply_debug_options(const struct dc *dc, struct dml2_context *dml2)
-{
-	if (dc->debug.override_odm_optimization) {
-		dml2->config.minimize_dispclk_using_odm = dc->debug.minimize_dispclk_using_odm;
-	}
-}
+#include "dml2_wrapper_fpu.h"
 
 bool dml2_validate(const struct dc *in_dc, struct dc_state *context, struct dml2_context *dml2,
 	enum dc_validate_mode validate_mode)
@@ -573,11 +36,6 @@ bool dml2_validate(const struct dc *in_dc, struct dc_state *context, struct dml2
 	return out;
 }
 
-static inline struct dml2_context *dml2_allocate_memory(void)
-{
-	return (struct dml2_context *) vzalloc(sizeof(struct dml2_context));
-}
-
 static void dml2_init(const struct dc *in_dc, const struct dml2_configuration_options *config, struct dml2_context **dml2)
 {
 	if ((in_dc->debug.using_dml21) && (in_dc->ctx->dce_version >= DCN_VERSION_4_01)) {
@@ -640,57 +98,6 @@ bool dml2_create(const struct dc *in_dc, const struct dml2_configuration_options
 	return true;
 }
 
-void dml2_destroy(struct dml2_context *dml2)
-{
-	if (!dml2)
-		return;
-
-	if (dml2->architecture == dml2_architecture_21)
-		dml21_destroy(dml2);
-	vfree(dml2);
-}
-
-void dml2_extract_dram_and_fclk_change_support(struct dml2_context *dml2,
-	unsigned int *fclk_change_support, unsigned int *dram_clk_change_support)
-{
-	*fclk_change_support = (unsigned int) dml2->v20.dml_core_ctx.ms.support.FCLKChangeSupport[0];
-	*dram_clk_change_support = (unsigned int) dml2->v20.dml_core_ctx.ms.support.DRAMClockChangeSupport[0];
-}
-
-void dml2_prepare_mcache_programming(struct dc *in_dc, struct dc_state *context, struct dml2_context *dml2)
-{
-	if (dml2->architecture == dml2_architecture_21)
-		dml21_prepare_mcache_programming(in_dc, context, dml2);
-}
-
-void dml2_copy(struct dml2_context *dst_dml2,
-	struct dml2_context *src_dml2)
-{
-	if (src_dml2->architecture == dml2_architecture_21) {
-		dml21_copy(dst_dml2, src_dml2);
-		return;
-	}
-	/* copy Mode Lib Ctx */
-	memcpy(dst_dml2, src_dml2, sizeof(struct dml2_context));
-}
-
-bool dml2_create_copy(struct dml2_context **dst_dml2,
-	struct dml2_context *src_dml2)
-{
-	if (src_dml2->architecture == dml2_architecture_21)
-		return dml21_create_copy(dst_dml2, src_dml2);
-	/* Allocate Mode Lib Ctx */
-	*dst_dml2 = dml2_allocate_memory();
-
-	if (!(*dst_dml2))
-		return false;
-
-	/* copy Mode Lib Ctx */
-	dml2_copy(*dst_dml2, src_dml2);
-
-	return true;
-}
-
 void dml2_reinit(const struct dc *in_dc,
 				 const struct dml2_configuration_options *config,
 				 struct dml2_context **dml2)
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.h
index c384e141cebc..9a9c27962f68 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.h
@@ -306,4 +306,13 @@ bool dml2_validate(const struct dc *in_dc,
 void dml2_extract_dram_and_fclk_change_support(struct dml2_context *dml2,
 	unsigned int *fclk_change_support, unsigned int *dram_clk_change_support);
 void dml2_prepare_mcache_programming(struct dc *in_dc, struct dc_state *context, struct dml2_context *dml2);
+
+void dml2_apply_debug_options(const struct dc *dc, struct dml2_context *dml2);
+bool dml2_validate_only(struct dc_state *context, enum dc_validate_mode validate_mode);
+bool dml2_validate_and_build_resource(const struct dc *in_dc, struct dc_state *context,
+		enum dc_validate_mode validate_mode);
+
+struct dml2_context *dml2_allocate_memory(void);
+
 #endif //_DML2_WRAPPER_H_
+
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper_fpu.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper_fpu.c
new file mode 100644
index 000000000000..203eef747262
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper_fpu.c
@@ -0,0 +1,604 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "display_mode_core.h"
+#include "dml2_internal_types.h"
+#include "dml2_utils.h"
+#include "dml2_policy.h"
+#include "dml2_translation_helper.h"
+#include "dml2_mall_phantom.h"
+#include "dml2_dc_resource_mgmt.h"
+#include "dml21_wrapper.h"
+#include "dml2_wrapper_fpu.h"
+
+void initialize_dml2_ip_params(struct dml2_context *dml2, const struct dc *in_dc, struct ip_params_st *out)
+{
+	if (dml2->config.use_native_soc_bb_construction)
+		dml2_init_ip_params(dml2, in_dc, out);
+	else
+		dml2_translate_ip_params(in_dc, out);
+}
+
+void initialize_dml2_soc_bbox(struct dml2_context *dml2, const struct dc *in_dc, struct soc_bounding_box_st *out)
+{
+	if (dml2->config.use_native_soc_bb_construction)
+		dml2_init_socbb_params(dml2, in_dc, out);
+	else
+		dml2_translate_socbb_params(in_dc, out);
+}
+
+void initialize_dml2_soc_states(struct dml2_context *dml2,
+	const struct dc *in_dc, const struct soc_bounding_box_st *in_bbox, struct soc_states_st *out)
+{
+	if (dml2->config.use_native_soc_bb_construction)
+		dml2_init_soc_states(dml2, in_dc, in_bbox, out);
+	else
+		dml2_translate_soc_states(in_dc, out, in_dc->dml.soc.num_states);
+}
+
+static void map_hw_resources(struct dml2_context *dml2,
+		struct dml_display_cfg_st *in_out_display_cfg, struct dml_mode_support_info_st *mode_support_info)
+{
+	unsigned int num_pipes = 0;
+	int i, j;
+
+	for (i = 0; i < __DML_NUM_PLANES__; i++) {
+		in_out_display_cfg->hw.ODMMode[i] = mode_support_info->ODMMode[i];
+		in_out_display_cfg->hw.DPPPerSurface[i] = mode_support_info->DPPPerSurface[i];
+		in_out_display_cfg->hw.DSCEnabled[i] = mode_support_info->DSCEnabled[i];
+		in_out_display_cfg->hw.NumberOfDSCSlices[i] = mode_support_info->NumberOfDSCSlices[i];
+		in_out_display_cfg->hw.DLGRefClkFreqMHz = 24;
+		if (dml2->v20.dml_core_ctx.project != dml_project_dcn35 &&
+			dml2->v20.dml_core_ctx.project != dml_project_dcn36 &&
+			dml2->v20.dml_core_ctx.project != dml_project_dcn351) {
+			/*dGPU default as 50Mhz*/
+			in_out_display_cfg->hw.DLGRefClkFreqMHz = 50;
+		}
+		for (j = 0; j < mode_support_info->DPPPerSurface[i]; j++) {
+			if (i >= __DML2_WRAPPER_MAX_STREAMS_PLANES__) {
+				dml_print("DML::%s: Index out of bounds: i=%d, __DML2_WRAPPER_MAX_STREAMS_PLANES__=%d\n",
+					  __func__, i, __DML2_WRAPPER_MAX_STREAMS_PLANES__);
+				break;
+			}
+			dml2->v20.scratch.dml_to_dc_pipe_mapping.dml_pipe_idx_to_stream_id[num_pipes] = dml2->v20.scratch.dml_to_dc_pipe_mapping.disp_cfg_to_stream_id[i];
+			dml2->v20.scratch.dml_to_dc_pipe_mapping.dml_pipe_idx_to_stream_id_valid[num_pipes] = true;
+			dml2->v20.scratch.dml_to_dc_pipe_mapping.dml_pipe_idx_to_plane_id[num_pipes] = dml2->v20.scratch.dml_to_dc_pipe_mapping.disp_cfg_to_plane_id[i];
+			dml2->v20.scratch.dml_to_dc_pipe_mapping.dml_pipe_idx_to_plane_id_valid[num_pipes] = true;
+			num_pipes++;
+		}
+	}
+}
+
+static unsigned int pack_and_call_dml_mode_support_ex(struct dml2_context *dml2,
+	const struct dml_display_cfg_st *display_cfg,
+	struct dml_mode_support_info_st *evaluation_info,
+	enum dc_validate_mode validate_mode)
+{
+	struct dml2_wrapper_scratch *s = &dml2->v20.scratch;
+
+	s->mode_support_params.mode_lib = &dml2->v20.dml_core_ctx;
+	s->mode_support_params.in_display_cfg = display_cfg;
+	if (validate_mode == DC_VALIDATE_MODE_ONLY)
+		s->mode_support_params.in_start_state_idx = dml2->v20.dml_core_ctx.states.num_states - 1;
+	else
+		s->mode_support_params.in_start_state_idx = 0;
+	s->mode_support_params.out_evaluation_info = evaluation_info;
+
+	memset(evaluation_info, 0, sizeof(struct dml_mode_support_info_st));
+	s->mode_support_params.out_lowest_state_idx = 0;
+
+	return dml_mode_support_ex(&s->mode_support_params);
+}
+
+static bool optimize_configuration(struct dml2_context *dml2, struct dml2_wrapper_optimize_configuration_params *p)
+{
+	int unused_dpps = p->ip_params->max_num_dpp;
+	int i;
+	int odms_needed;
+	int largest_blend_and_timing = 0;
+	bool optimization_done = false;
+
+	for (i = 0; i < (int) p->cur_display_config->num_timings; i++) {
+		if (p->cur_display_config->plane.BlendingAndTiming[i] > largest_blend_and_timing)
+			largest_blend_and_timing = p->cur_display_config->plane.BlendingAndTiming[i];
+	}
+
+	if (p->new_policy != p->cur_policy)
+		*p->new_policy = *p->cur_policy;
+
+	if (p->new_display_config != p->cur_display_config)
+		*p->new_display_config = *p->cur_display_config;
+
+
+	// Optimize Clocks
+	if (!optimization_done) {
+		if (largest_blend_and_timing == 0 && p->cur_policy->ODMUse[0] == dml_odm_use_policy_combine_as_needed && dml2->config.minimize_dispclk_using_odm) {
+			odms_needed = dml2_util_get_maximum_odm_combine_for_output(dml2->config.optimize_odm_4to1,
+				p->cur_display_config->output.OutputEncoder[0], p->cur_mode_support_info->DSCEnabled[0]) - 1;
+
+			if (odms_needed <= unused_dpps) {
+				if (odms_needed == 1) {
+					p->new_policy->ODMUse[0] = dml_odm_use_policy_combine_2to1;
+					optimization_done = true;
+				} else if (odms_needed == 3) {
+					p->new_policy->ODMUse[0] = dml_odm_use_policy_combine_4to1;
+					optimization_done = true;
+				} else
+					optimization_done = false;
+			}
+		}
+	}
+
+	return optimization_done;
+}
+
+static int calculate_lowest_supported_state_for_temp_read(struct dml2_context *dml2, struct dc_state *display_state,
+		enum dc_validate_mode validate_mode)
+{
+	struct dml2_calculate_lowest_supported_state_for_temp_read_scratch *s = &dml2->v20.scratch.dml2_calculate_lowest_supported_state_for_temp_read_scratch;
+	struct dml2_wrapper_scratch *s_global = &dml2->v20.scratch;
+
+	unsigned int dml_result = 0;
+	int result = -1, i, j;
+
+	build_unoptimized_policy_settings(dml2->v20.dml_core_ctx.project, &dml2->v20.dml_core_ctx.policy);
+
+	/* Zero out before each call before proceeding */
+	memset(s, 0, sizeof(struct dml2_calculate_lowest_supported_state_for_temp_read_scratch));
+	memset(&s_global->mode_support_params, 0, sizeof(struct dml_mode_support_ex_params_st));
+	memset(&s_global->dml_to_dc_pipe_mapping, 0, sizeof(struct dml2_dml_to_dc_pipe_mapping));
+
+	for (i = 0; i < dml2->config.dcn_pipe_count; i++) {
+		/* Calling resource_build_scaling_params will populate the pipe params
+		 * with the necessary information needed for correct DML calculations
+		 * This is also done in DML1 driver code path and hence display_state
+		 * cannot be const.
+		 */
+		struct pipe_ctx *pipe = &display_state->res_ctx.pipe_ctx[i];
+
+		if (pipe->plane_state) {
+			if (!dml2->config.callbacks.build_scaling_params(pipe)) {
+				ASSERT(false);
+				return false;
+			}
+		}
+	}
+
+	map_dc_state_into_dml_display_cfg(dml2, display_state, &s->cur_display_config);
+
+	for (i = 0; i < dml2->v20.dml_core_ctx.states.num_states; i++) {
+		s->uclk_change_latencies[i] = dml2->v20.dml_core_ctx.states.state_array[i].dram_clock_change_latency_us;
+	}
+
+	for (i = 0; i < 4; i++) {
+		for (j = 0; j < dml2->v20.dml_core_ctx.states.num_states; j++) {
+			dml2->v20.dml_core_ctx.states.state_array[j].dram_clock_change_latency_us = s_global->dummy_pstate_table[i].dummy_pstate_latency_us;
+		}
+
+		dml_result = pack_and_call_dml_mode_support_ex(dml2, &s->cur_display_config, &s->evaluation_info,
+						validate_mode);
+
+		if (dml_result && s->evaluation_info.DRAMClockChangeSupport[0] == dml_dram_clock_change_vactive) {
+			map_hw_resources(dml2, &s->cur_display_config, &s->evaluation_info);
+			dml_result = dml_mode_programming(&dml2->v20.dml_core_ctx, s_global->mode_support_params.out_lowest_state_idx, &s->cur_display_config, true);
+
+			ASSERT(dml_result);
+
+			dml2_extract_watermark_set(&dml2->v20.g6_temp_read_watermark_set, &dml2->v20.dml_core_ctx);
+			dml2->v20.g6_temp_read_watermark_set.cstate_pstate.fclk_pstate_change_ns = dml2->v20.g6_temp_read_watermark_set.cstate_pstate.pstate_change_ns;
+
+			result = s_global->mode_support_params.out_lowest_state_idx;
+
+			while (dml2->v20.dml_core_ctx.states.state_array[result].dram_speed_mts < s_global->dummy_pstate_table[i].dram_speed_mts)
+				result++;
+
+			break;
+		}
+	}
+
+	for (i = 0; i < dml2->v20.dml_core_ctx.states.num_states; i++) {
+		dml2->v20.dml_core_ctx.states.state_array[i].dram_clock_change_latency_us = s->uclk_change_latencies[i];
+	}
+
+	return result;
+}
+
+static void copy_dummy_pstate_table(struct dummy_pstate_entry *dest, struct dummy_pstate_entry *src, unsigned int num_entries)
+{
+	for (int i = 0; i < num_entries; i++) {
+		dest[i] = src[i];
+	}
+}
+
+static bool are_timings_requiring_odm_doing_blending(const struct dml_display_cfg_st *display_cfg,
+		const struct dml_mode_support_info_st *evaluation_info)
+{
+	unsigned int planes_per_timing[__DML_NUM_PLANES__] = {0};
+	int i;
+
+	for (i = 0; i < display_cfg->num_surfaces; i++)
+		planes_per_timing[display_cfg->plane.BlendingAndTiming[i]]++;
+
+	for (i = 0; i < __DML_NUM_PLANES__; i++) {
+		if (planes_per_timing[i] > 1 && evaluation_info->ODMMode[i] != dml_odm_mode_bypass)
+			return true;
+	}
+
+	return false;
+}
+
+static bool does_configuration_meet_sw_policies(struct dml2_context *ctx, const struct dml_display_cfg_st *display_cfg,
+	const struct dml_mode_support_info_st *evaluation_info)
+{
+	bool pass = true;
+
+	if (!ctx->config.enable_windowed_mpo_odm) {
+		if (are_timings_requiring_odm_doing_blending(display_cfg, evaluation_info))
+			pass = false;
+	}
+
+	return pass;
+}
+
+static bool dml_mode_support_wrapper(struct dml2_context *dml2,
+		struct dc_state *display_state,
+		enum dc_validate_mode validate_mode)
+{
+	struct dml2_wrapper_scratch *s = &dml2->v20.scratch;
+	unsigned int result = 0, i;
+	unsigned int optimized_result = true;
+
+	build_unoptimized_policy_settings(dml2->v20.dml_core_ctx.project, &dml2->v20.dml_core_ctx.policy);
+
+	/* Zero out before each call before proceeding */
+	memset(&s->cur_display_config, 0, sizeof(struct dml_display_cfg_st));
+	memset(&s->mode_support_params, 0, sizeof(struct dml_mode_support_ex_params_st));
+	memset(&s->dml_to_dc_pipe_mapping, 0, sizeof(struct dml2_dml_to_dc_pipe_mapping));
+	memset(&s->optimize_configuration_params, 0, sizeof(struct dml2_wrapper_optimize_configuration_params));
+
+	for (i = 0; i < dml2->config.dcn_pipe_count; i++) {
+		/* Calling resource_build_scaling_params will populate the pipe params
+		 * with the necessary information needed for correct DML calculations
+		 * This is also done in DML1 driver code path and hence display_state
+		 * cannot be const.
+		 */
+		struct pipe_ctx *pipe = &display_state->res_ctx.pipe_ctx[i];
+
+		if (pipe->plane_state) {
+			if (!dml2->config.callbacks.build_scaling_params(pipe)) {
+				ASSERT(false);
+				return false;
+			}
+		}
+	}
+
+	map_dc_state_into_dml_display_cfg(dml2, display_state, &s->cur_display_config);
+	if (!dml2->config.skip_hw_state_mapping)
+		dml2_apply_det_buffer_allocation_policy(dml2, &s->cur_display_config);
+
+	result = pack_and_call_dml_mode_support_ex(dml2,
+		&s->cur_display_config,
+		&s->mode_support_info,
+		validate_mode);
+
+	if (result)
+		result = does_configuration_meet_sw_policies(dml2, &s->cur_display_config, &s->mode_support_info);
+
+	// Try to optimize
+	if (result) {
+		s->cur_policy = dml2->v20.dml_core_ctx.policy;
+		s->optimize_configuration_params.dml_core_ctx = &dml2->v20.dml_core_ctx;
+		s->optimize_configuration_params.config = &dml2->config;
+		s->optimize_configuration_params.ip_params = &dml2->v20.dml_core_ctx.ip;
+		s->optimize_configuration_params.cur_display_config = &s->cur_display_config;
+		s->optimize_configuration_params.cur_mode_support_info = &s->mode_support_info;
+		s->optimize_configuration_params.cur_policy = &s->cur_policy;
+		s->optimize_configuration_params.new_display_config = &s->new_display_config;
+		s->optimize_configuration_params.new_policy = &s->new_policy;
+
+		while (optimized_result && optimize_configuration(dml2, &s->optimize_configuration_params)) {
+			dml2->v20.dml_core_ctx.policy = s->new_policy;
+			optimized_result = pack_and_call_dml_mode_support_ex(dml2,
+				&s->new_display_config,
+				&s->mode_support_info,
+				validate_mode);
+
+			if (optimized_result)
+				optimized_result = does_configuration_meet_sw_policies(dml2, &s->new_display_config, &s->mode_support_info);
+
+			// If the new optimized state is supposed, then set current = new
+			if (optimized_result) {
+				s->cur_display_config = s->new_display_config;
+				s->cur_policy = s->new_policy;
+			} else {
+				// Else, restore policy to current
+				dml2->v20.dml_core_ctx.policy = s->cur_policy;
+			}
+		}
+
+		// Optimize ended with a failed config, so we need to restore DML state to last passing
+		if (!optimized_result) {
+			result = pack_and_call_dml_mode_support_ex(dml2,
+				&s->cur_display_config,
+				&s->mode_support_info,
+				validate_mode);
+		}
+	}
+
+	if (result)
+		map_hw_resources(dml2, &s->cur_display_config, &s->mode_support_info);
+
+	return result;
+}
+
+static bool call_dml_mode_support_and_programming(struct dc_state *context, enum dc_validate_mode validate_mode)
+{
+	unsigned int result = 0;
+	unsigned int min_state = 0;
+	int min_state_for_g6_temp_read = 0;
+
+
+	if (!context)
+		return false;
+
+	struct dml2_context *dml2 = context->bw_ctx.dml2;
+	struct dml2_wrapper_scratch *s = &dml2->v20.scratch;
+
+	if (!context->streams[0]->sink->link->dc->caps.is_apu) {
+		min_state_for_g6_temp_read = calculate_lowest_supported_state_for_temp_read(dml2, context,
+										validate_mode);
+
+		ASSERT(min_state_for_g6_temp_read >= 0);
+	}
+
+	result = dml_mode_support_wrapper(dml2, context, validate_mode);
+
+	/* Upon trying to sett certain frequencies in FRL, min_state_for_g6_temp_read is reported as -1. This leads to an invalid value of min_state causing crashes later on.
+	 * Use the default logic for min_state only when min_state_for_g6_temp_read is a valid value. In other cases, use the value calculated by the DML directly.
+	 */
+	if (!context->streams[0]->sink->link->dc->caps.is_apu) {
+		if (min_state_for_g6_temp_read >= 0)
+			min_state = min_state_for_g6_temp_read > s->mode_support_params.out_lowest_state_idx ? min_state_for_g6_temp_read : s->mode_support_params.out_lowest_state_idx;
+		else
+			min_state = s->mode_support_params.out_lowest_state_idx;
+	}
+
+	if (result) {
+		if (!context->streams[0]->sink->link->dc->caps.is_apu) {
+			result = dml_mode_programming(&dml2->v20.dml_core_ctx, min_state, &s->cur_display_config, true);
+		} else {
+			result = dml_mode_programming(&dml2->v20.dml_core_ctx, s->mode_support_params.out_lowest_state_idx, &s->cur_display_config, true);
+		}
+	}
+	return result;
+}
+
+bool dml2_validate_and_build_resource(const struct dc *in_dc, struct dc_state *context,
+		enum dc_validate_mode validate_mode)
+{
+	struct dml2_context *dml2 = context->bw_ctx.dml2;
+	struct dml2_wrapper_scratch *s = &dml2->v20.scratch;
+	struct dml2_dcn_clocks out_clks;
+	unsigned int result = 0;
+	bool need_recalculation = false;
+	uint32_t cstate_enter_plus_exit_z8_ns;
+
+	if (context->stream_count == 0) {
+		unsigned int lowest_state_idx = 0;
+
+		out_clks.p_state_supported = true;
+		out_clks.dispclk_khz = 0; /* No requirement, and lowest index will generally be maximum dispclk. */
+		out_clks.dcfclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].dcfclk_mhz * 1000;
+		out_clks.fclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].fabricclk_mhz * 1000;
+		out_clks.uclk_mts = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].dram_speed_mts;
+		out_clks.phyclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].phyclk_mhz * 1000;
+		out_clks.socclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].socclk_mhz * 1000;
+		out_clks.ref_dtbclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].dtbclk_mhz * 1000;
+		context->bw_ctx.bw.dcn.clk.dtbclk_en = false;
+		dml2_copy_clocks_to_dc_state(&out_clks, context);
+		return true;
+	}
+
+	/* Zero out before each call before proceeding */
+	memset(&dml2->v20.scratch, 0, sizeof(struct dml2_wrapper_scratch));
+	memset(&dml2->v20.dml_core_ctx.policy, 0, sizeof(struct dml_mode_eval_policy_st));
+	memset(&dml2->v20.dml_core_ctx.ms, 0, sizeof(struct mode_support_st));
+	memset(&dml2->v20.dml_core_ctx.mp, 0, sizeof(struct mode_program_st));
+
+	/* Initialize DET scratch */
+	dml2_initialize_det_scratch(dml2);
+
+	copy_dummy_pstate_table(s->dummy_pstate_table, in_dc->clk_mgr->bw_params->dummy_pstate_table, 4);
+
+	result = call_dml_mode_support_and_programming(context, validate_mode);
+	/* Call map dc pipes to map the pipes based on the DML output. For correctly determining if recalculation
+	 * is required or not, the resource context needs to correctly reflect the number of active pipes. We would
+	 * only know the correct number if active pipes after dml2_map_dc_pipes is called.
+	 */
+	if (result && !dml2->config.skip_hw_state_mapping)
+		dml2_map_dc_pipes(dml2, context, &s->cur_display_config, &s->dml_to_dc_pipe_mapping, in_dc->current_state);
+
+	/* Verify and update DET Buffer configuration if needed. dml2_verify_det_buffer_configuration will check if DET Buffer
+	 * size needs to be updated. If yes it will update the DETOverride variable and set need_recalculation flag to true.
+	 * Based on that flag, run mode support again. Verification needs to be run after dml_mode_programming because the getters
+	 * return correct det buffer values only after dml_mode_programming is called.
+	 */
+	if (result && !dml2->config.skip_hw_state_mapping) {
+		need_recalculation = dml2_verify_det_buffer_configuration(dml2, context, &dml2->det_helper_scratch);
+		if (need_recalculation) {
+			/* Engage the DML again if recalculation is required. */
+			call_dml_mode_support_and_programming(context, validate_mode);
+			if (!dml2->config.skip_hw_state_mapping) {
+				dml2_map_dc_pipes(dml2, context, &s->cur_display_config, &s->dml_to_dc_pipe_mapping, in_dc->current_state);
+			}
+			need_recalculation = dml2_verify_det_buffer_configuration(dml2, context, &dml2->det_helper_scratch);
+			ASSERT(need_recalculation == false);
+		}
+	}
+
+	if (result) {
+		unsigned int lowest_state_idx = s->mode_support_params.out_lowest_state_idx;
+		out_clks.dispclk_khz = (unsigned int)dml2->v20.dml_core_ctx.mp.Dispclk_calculated * 1000;
+		out_clks.p_state_supported = s->mode_support_info.DRAMClockChangeSupport[0] != dml_dram_clock_change_unsupported;
+		if (in_dc->config.use_default_clock_table &&
+			(lowest_state_idx < dml2->v20.dml_core_ctx.states.num_states - 1)) {
+			lowest_state_idx = dml2->v20.dml_core_ctx.states.num_states - 1;
+			out_clks.dispclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].dispclk_mhz * 1000;
+		}
+
+		out_clks.dcfclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].dcfclk_mhz * 1000;
+		out_clks.fclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].fabricclk_mhz * 1000;
+		out_clks.uclk_mts = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].dram_speed_mts;
+		out_clks.phyclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].phyclk_mhz * 1000;
+		out_clks.socclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].socclk_mhz * 1000;
+		out_clks.ref_dtbclk_khz = (unsigned int)dml2->v20.dml_core_ctx.states.state_array[lowest_state_idx].dtbclk_mhz * 1000;
+		context->bw_ctx.bw.dcn.clk.dtbclk_en = is_dtbclk_required(in_dc, context);
+
+		if (!dml2->config.skip_hw_state_mapping) {
+			/* Call dml2_calculate_rq_and_dlg_params */
+			dml2_calculate_rq_and_dlg_params(in_dc, context, &context->res_ctx, dml2, in_dc->res_pool->pipe_count);
+		}
+
+		dml2_copy_clocks_to_dc_state(&out_clks, context);
+		dml2_extract_watermark_set(&context->bw_ctx.bw.dcn.watermarks.a, &dml2->v20.dml_core_ctx);
+		dml2_extract_watermark_set(&context->bw_ctx.bw.dcn.watermarks.b, &dml2->v20.dml_core_ctx);
+		if (context->streams[0]->sink->link->dc->caps.is_apu)
+			dml2_extract_watermark_set(&context->bw_ctx.bw.dcn.watermarks.c, &dml2->v20.dml_core_ctx);
+		else
+			memcpy(&context->bw_ctx.bw.dcn.watermarks.c, &dml2->v20.g6_temp_read_watermark_set, sizeof(context->bw_ctx.bw.dcn.watermarks.c));
+		dml2_extract_watermark_set(&context->bw_ctx.bw.dcn.watermarks.d, &dml2->v20.dml_core_ctx);
+		dml2_extract_writeback_wm(context, &dml2->v20.dml_core_ctx);
+		//copy for deciding zstate use
+		context->bw_ctx.dml.vba.StutterPeriod = context->bw_ctx.dml2->v20.dml_core_ctx.mp.StutterPeriod;
+
+		cstate_enter_plus_exit_z8_ns = context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_z8_ns;
+
+		if (context->bw_ctx.dml.vba.StutterPeriod < in_dc->debug.minimum_z8_residency_time &&
+				cstate_enter_plus_exit_z8_ns < in_dc->debug.minimum_z8_residency_time * 1000)
+			cstate_enter_plus_exit_z8_ns = in_dc->debug.minimum_z8_residency_time * 1000;
+
+		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_z8_ns = cstate_enter_plus_exit_z8_ns;
+	}
+
+	return result;
+}
+
+bool dml2_validate_only(struct dc_state *context, enum dc_validate_mode validate_mode)
+{
+	struct dml2_context *dml2;
+	unsigned int result = 0;
+
+	if (!context || context->stream_count == 0)
+		return true;
+
+	dml2 = context->bw_ctx.dml2;
+
+	/* Zero out before each call before proceeding */
+	memset(&dml2->v20.scratch, 0, sizeof(struct dml2_wrapper_scratch));
+	memset(&dml2->v20.dml_core_ctx.policy, 0, sizeof(struct dml_mode_eval_policy_st));
+	memset(&dml2->v20.dml_core_ctx.ms, 0, sizeof(struct mode_support_st));
+	memset(&dml2->v20.dml_core_ctx.mp, 0, sizeof(struct mode_program_st));
+
+	build_unoptimized_policy_settings(dml2->v20.dml_core_ctx.project, &dml2->v20.dml_core_ctx.policy);
+
+	map_dc_state_into_dml_display_cfg(dml2, context, &dml2->v20.scratch.cur_display_config);
+	 if (!dml2->config.skip_hw_state_mapping)
+		 dml2_apply_det_buffer_allocation_policy(dml2, &dml2->v20.scratch.cur_display_config);
+
+	result = pack_and_call_dml_mode_support_ex(dml2,
+		&dml2->v20.scratch.cur_display_config,
+		&dml2->v20.scratch.mode_support_info,
+		validate_mode);
+
+	if (result)
+		result = does_configuration_meet_sw_policies(dml2, &dml2->v20.scratch.cur_display_config, &dml2->v20.scratch.mode_support_info);
+
+	return result == 1;
+}
+
+void dml2_apply_debug_options(const struct dc *dc, struct dml2_context *dml2)
+{
+	if (dc->debug.override_odm_optimization) {
+		dml2->config.minimize_dispclk_using_odm = dc->debug.minimize_dispclk_using_odm;
+	}
+}
+
+inline struct dml2_context *dml2_allocate_memory(void)
+{
+	return (struct dml2_context *) vzalloc(sizeof(struct dml2_context));
+}
+
+void dml2_destroy(struct dml2_context *dml2)
+{
+	if (!dml2)
+		return;
+
+	if (dml2->architecture == dml2_architecture_21)
+		dml21_destroy(dml2);
+	vfree(dml2);
+}
+
+void dml2_extract_dram_and_fclk_change_support(struct dml2_context *dml2,
+	unsigned int *fclk_change_support, unsigned int *dram_clk_change_support)
+{
+	*fclk_change_support = (unsigned int) dml2->v20.dml_core_ctx.ms.support.FCLKChangeSupport[0];
+	*dram_clk_change_support = (unsigned int) dml2->v20.dml_core_ctx.ms.support.DRAMClockChangeSupport[0];
+}
+
+void dml2_prepare_mcache_programming(struct dc *in_dc, struct dc_state *context, struct dml2_context *dml2)
+{
+	if (dml2->architecture == dml2_architecture_21)
+		dml21_prepare_mcache_programming(in_dc, context, dml2);
+}
+
+void dml2_copy(struct dml2_context *dst_dml2,
+	struct dml2_context *src_dml2)
+{
+	if (src_dml2->architecture == dml2_architecture_21) {
+		dml21_copy(dst_dml2, src_dml2);
+		return;
+	}
+	/* copy Mode Lib Ctx */
+	memcpy(dst_dml2, src_dml2, sizeof(struct dml2_context));
+}
+
+bool dml2_create_copy(struct dml2_context **dst_dml2,
+	struct dml2_context *src_dml2)
+{
+	if (src_dml2->architecture == dml2_architecture_21)
+		return dml21_create_copy(dst_dml2, src_dml2);
+	/* Allocate Mode Lib Ctx */
+	*dst_dml2 = dml2_allocate_memory();
+
+	if (!(*dst_dml2))
+		return false;
+
+	/* copy Mode Lib Ctx */
+	dml2_copy(*dst_dml2, src_dml2);
+
+	return true;
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper_fpu.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper_fpu.h
new file mode 100644
index 000000000000..573df874b901
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper_fpu.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Authors: AMD
+ */
+
+#ifndef _DML2_WRAPPER_FPU_H_
+#define _DML2_WRAPPER_FPU_H_
+
+#include "os_types.h"
+
+struct dml2_context;
+struct dc;
+struct ip_params_st;
+struct soc_bounding_box_st;
+struct soc_states_st;
+
+void initialize_dml2_ip_params(struct dml2_context *dml2, const struct dc *in_dc, struct ip_params_st *out);
+void initialize_dml2_soc_bbox(struct dml2_context *dml2, const struct dc *in_dc, struct soc_bounding_box_st *out);
+void initialize_dml2_soc_states(struct dml2_context *dml2,
+	const struct dc *in_dc, const struct soc_bounding_box_st *in_bbox, struct soc_states_st *out);
+
+#endif //_DML2_WRAPPER_FPU_H_
+
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.h
index b12f34345a58..890d04b1ddaf 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp.h
@@ -545,6 +545,7 @@
 	type SCL_COEF_RAM_SELECT_CURRENT; \
 	type LUT_MEM_PWR_FORCE; \
 	type LUT_MEM_PWR_STATE; \
+	type LUT_MEM_PWR_DIS; \
 	type CM_GAMUT_REMAP_MODE; \
 	type CM_GAMUT_REMAP_C11; \
 	type CM_GAMUT_REMAP_C12; \
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h
index 5f6b431ec398..58b9f1cd0825 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h
@@ -41,6 +41,7 @@
 	TF_SF(CM0_CM_BIAS_Y_G_CB_B, CM_BIAS_CB_B, mask_sh),\
 	TF_SF(CM0_CM_MEM_PWR_CTRL, GAMCOR_MEM_PWR_DIS, mask_sh),\
 	TF_SF(CM0_CM_MEM_PWR_CTRL, GAMCOR_MEM_PWR_FORCE, mask_sh),\
+	TF_SF(CM0_CM_MEM_PWR_CTRL, GAMCOR_MEM_PWR_DIS, mask_sh),\
 	TF_SF(CNVC_CFG0_PRE_DEGAM, PRE_DEGAM_MODE, mask_sh),\
 	TF_SF(CNVC_CFG0_PRE_DEGAM, PRE_DEGAM_SELECT, mask_sh),\
 	TF_SF(CM0_CM_GAMCOR_CONTROL, CM_GAMCOR_MODE, mask_sh),\
@@ -208,6 +209,7 @@
 	TF_SF(CM_CUR0_CURSOR0_CONTROL, CUR0_ROM_EN, mask_sh),\
 	TF_SF(DSCL0_OBUF_MEM_PWR_CTRL, OBUF_MEM_PWR_FORCE, mask_sh),\
 	TF_SF(DSCL0_DSCL_MEM_PWR_CTRL, LUT_MEM_PWR_FORCE, mask_sh),\
+	TF_SF(DSCL0_DSCL_MEM_PWR_CTRL, LUT_MEM_PWR_DIS, mask_sh),\
 	TF_SF(DSCL0_DSCL_MEM_PWR_STATUS, LUT_MEM_PWR_STATE, mask_sh),\
 	TF_SF(DSCL0_DSCL_SC_MODE, SCL_SC_MATRIX_MODE, mask_sh),\
 	TF_SF(DSCL0_DSCL_SC_MODE, SCL_SC_LTONL_EN, mask_sh),\
@@ -336,6 +338,9 @@
 	TF_SF(DSCL0_DSCL_SC_MATRIX_C2C3, SCL_SC_MATRIX_C2, mask_sh),\
 	TF_SF(DSCL0_DSCL_SC_MATRIX_C2C3, SCL_SC_MATRIX_C3, mask_sh),\
 	TF_SF(DSCL0_ISHARP_DELTA_CTRL, ISHARP_DELTA_LUT_HOST_SELECT, mask_sh),\
+	TF_SF(DSCL0_ISHARP_DELTA_LUT_MEM_PWR_CTRL, ISHARP_DELTA_LUT_MEM_PWR_DIS, mask_sh),\
+	TF_SF(DSCL0_ISHARP_DELTA_LUT_MEM_PWR_CTRL, ISHARP_DELTA_LUT_MEM_PWR_FORCE, mask_sh),\
+	TF_SF(DSCL0_ISHARP_DELTA_LUT_MEM_PWR_CTRL, ISHARP_DELTA_LUT_MEM_PWR_STATE, mask_sh),\
 	TF_SF(DSCL0_ISHARP_DELTA_DATA, ISHARP_DELTA_DATA, mask_sh),\
 	TF_SF(DSCL0_ISHARP_DELTA_INDEX, ISHARP_DELTA_INDEX, mask_sh),\
 	TF_SF(DSCL0_ISHARP_MODE, ISHARP_EN, mask_sh),\
@@ -558,6 +563,9 @@
 	type ISHARP_DELTA_LUT_SELECT;	\
 	type ISHARP_DELTA_LUT_SELECT_CURRENT;	\
 	type ISHARP_DELTA_LUT_HOST_SELECT;	\
+	type ISHARP_DELTA_LUT_MEM_PWR_DIS;	\
+	type ISHARP_DELTA_LUT_MEM_PWR_FORCE;\
+	type ISHARP_DELTA_LUT_MEM_PWR_STATE;\
 	type ISHARP_DELTA_DATA;	\
 	type ISHARP_DELTA_INDEX; \
 	type ISHARP_NLDELTA_SCLIP_EN_P;	\
@@ -629,6 +637,7 @@
 	uint32_t DSCL_SC_MATRIX_C0C1; \
 	uint32_t DSCL_SC_MATRIX_C2C3; \
 	uint32_t ISHARP_MODE; \
+	uint32_t ISHARP_DELTA_LUT_MEM_PWR_CTRL; \
 	uint32_t ISHARP_NOISEDET_THRESHOLD; \
 	uint32_t ISHARP_NOISE_GAIN_PWL; \
 	uint32_t ISHARP_LBA_PWL_SEG0; \
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c
index 6df3419f825f..a62c4733ed3b 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c
@@ -966,62 +966,57 @@ static void dpp401_dscl_program_isharp(struct dpp *dpp_base,
 		ISHARP_FMT_NORM, scl_data->dscl_prog_data.isharp_fmt.norm);
 
 	/* Skip remaining register programming if ISHARP is disabled */
-	if (!scl_data->dscl_prog_data.isharp_en) {
-		PERF_TRACE();
-		return;
-	}
-
-	/* ISHARP_NOISEDET_THRESHOLD */
-	REG_SET_2(ISHARP_NOISEDET_THRESHOLD, 0,
-		ISHARP_NOISEDET_UTHRE, scl_data->dscl_prog_data.isharp_noise_det.uthreshold,
-		ISHARP_NOISEDET_DTHRE, scl_data->dscl_prog_data.isharp_noise_det.dthreshold);
-
-	/* ISHARP_NOISE_GAIN_PWL */
-	REG_SET_3(ISHARP_NOISE_GAIN_PWL, 0,
-		ISHARP_NOISEDET_PWL_START_IN, scl_data->dscl_prog_data.isharp_noise_det.pwl_start_in,
-		ISHARP_NOISEDET_PWL_END_IN, scl_data->dscl_prog_data.isharp_noise_det.pwl_end_in,
-		ISHARP_NOISEDET_PWL_SLOPE, scl_data->dscl_prog_data.isharp_noise_det.pwl_slope);
-
-	/* ISHARP_LBA: IN_SEG, BASE_SEG, SLOPE_SEG */
-	REG_SET_3(ISHARP_LBA_PWL_SEG0, 0,
-		ISHARP_LBA_PWL_IN_SEG0, scl_data->dscl_prog_data.isharp_lba.in_seg[0],
-		ISHARP_LBA_PWL_BASE_SEG0, scl_data->dscl_prog_data.isharp_lba.base_seg[0],
-		ISHARP_LBA_PWL_SLOPE_SEG0, scl_data->dscl_prog_data.isharp_lba.slope_seg[0]);
-	REG_SET_3(ISHARP_LBA_PWL_SEG1, 0,
-		ISHARP_LBA_PWL_IN_SEG1, scl_data->dscl_prog_data.isharp_lba.in_seg[1],
-		ISHARP_LBA_PWL_BASE_SEG1, scl_data->dscl_prog_data.isharp_lba.base_seg[1],
-		ISHARP_LBA_PWL_SLOPE_SEG1, scl_data->dscl_prog_data.isharp_lba.slope_seg[1]);
-	REG_SET_3(ISHARP_LBA_PWL_SEG2, 0,
-		ISHARP_LBA_PWL_IN_SEG2, scl_data->dscl_prog_data.isharp_lba.in_seg[2],
-		ISHARP_LBA_PWL_BASE_SEG2, scl_data->dscl_prog_data.isharp_lba.base_seg[2],
-		ISHARP_LBA_PWL_SLOPE_SEG2, scl_data->dscl_prog_data.isharp_lba.slope_seg[2]);
-	REG_SET_3(ISHARP_LBA_PWL_SEG3, 0,
-		ISHARP_LBA_PWL_IN_SEG3, scl_data->dscl_prog_data.isharp_lba.in_seg[3],
-		ISHARP_LBA_PWL_BASE_SEG3, scl_data->dscl_prog_data.isharp_lba.base_seg[3],
-		ISHARP_LBA_PWL_SLOPE_SEG3, scl_data->dscl_prog_data.isharp_lba.slope_seg[3]);
-	REG_SET_3(ISHARP_LBA_PWL_SEG4, 0,
-		ISHARP_LBA_PWL_IN_SEG4, scl_data->dscl_prog_data.isharp_lba.in_seg[4],
-		ISHARP_LBA_PWL_BASE_SEG4, scl_data->dscl_prog_data.isharp_lba.base_seg[4],
-		ISHARP_LBA_PWL_SLOPE_SEG4, scl_data->dscl_prog_data.isharp_lba.slope_seg[4]);
-	REG_SET_2(ISHARP_LBA_PWL_SEG5, 0,
-		ISHARP_LBA_PWL_IN_SEG5, scl_data->dscl_prog_data.isharp_lba.in_seg[5],
-		ISHARP_LBA_PWL_BASE_SEG5, scl_data->dscl_prog_data.isharp_lba.base_seg[5]);
-
-	/* ISHARP_DELTA_LUT */
-	if (!program_isharp_1dlut)
-		dpp401_dscl_set_isharp_filter(dpp, scl_data->dscl_prog_data.isharp_delta);
+	if (scl_data->dscl_prog_data.isharp_en) {
+		/* ISHARP_NOISEDET_THRESHOLD */
+		REG_SET_2(ISHARP_NOISEDET_THRESHOLD, 0,
+			ISHARP_NOISEDET_UTHRE, scl_data->dscl_prog_data.isharp_noise_det.uthreshold,
+			ISHARP_NOISEDET_DTHRE, scl_data->dscl_prog_data.isharp_noise_det.dthreshold);
+
+		/* ISHARP_NOISE_GAIN_PWL */
+		REG_SET_3(ISHARP_NOISE_GAIN_PWL, 0,
+			ISHARP_NOISEDET_PWL_START_IN, scl_data->dscl_prog_data.isharp_noise_det.pwl_start_in,
+			ISHARP_NOISEDET_PWL_END_IN, scl_data->dscl_prog_data.isharp_noise_det.pwl_end_in,
+			ISHARP_NOISEDET_PWL_SLOPE, scl_data->dscl_prog_data.isharp_noise_det.pwl_slope);
+
+		/* ISHARP_LBA: IN_SEG, BASE_SEG, SLOPE_SEG */
+		REG_SET_3(ISHARP_LBA_PWL_SEG0, 0,
+			ISHARP_LBA_PWL_IN_SEG0, scl_data->dscl_prog_data.isharp_lba.in_seg[0],
+			ISHARP_LBA_PWL_BASE_SEG0, scl_data->dscl_prog_data.isharp_lba.base_seg[0],
+			ISHARP_LBA_PWL_SLOPE_SEG0, scl_data->dscl_prog_data.isharp_lba.slope_seg[0]);
+		REG_SET_3(ISHARP_LBA_PWL_SEG1, 0,
+			ISHARP_LBA_PWL_IN_SEG1, scl_data->dscl_prog_data.isharp_lba.in_seg[1],
+			ISHARP_LBA_PWL_BASE_SEG1, scl_data->dscl_prog_data.isharp_lba.base_seg[1],
+			ISHARP_LBA_PWL_SLOPE_SEG1, scl_data->dscl_prog_data.isharp_lba.slope_seg[1]);
+		REG_SET_3(ISHARP_LBA_PWL_SEG2, 0,
+			ISHARP_LBA_PWL_IN_SEG2, scl_data->dscl_prog_data.isharp_lba.in_seg[2],
+			ISHARP_LBA_PWL_BASE_SEG2, scl_data->dscl_prog_data.isharp_lba.base_seg[2],
+			ISHARP_LBA_PWL_SLOPE_SEG2, scl_data->dscl_prog_data.isharp_lba.slope_seg[2]);
+		REG_SET_3(ISHARP_LBA_PWL_SEG3, 0,
+			ISHARP_LBA_PWL_IN_SEG3, scl_data->dscl_prog_data.isharp_lba.in_seg[3],
+			ISHARP_LBA_PWL_BASE_SEG3, scl_data->dscl_prog_data.isharp_lba.base_seg[3],
+			ISHARP_LBA_PWL_SLOPE_SEG3, scl_data->dscl_prog_data.isharp_lba.slope_seg[3]);
+		REG_SET_3(ISHARP_LBA_PWL_SEG4, 0,
+			ISHARP_LBA_PWL_IN_SEG4, scl_data->dscl_prog_data.isharp_lba.in_seg[4],
+			ISHARP_LBA_PWL_BASE_SEG4, scl_data->dscl_prog_data.isharp_lba.base_seg[4],
+			ISHARP_LBA_PWL_SLOPE_SEG4, scl_data->dscl_prog_data.isharp_lba.slope_seg[4]);
+		REG_SET_2(ISHARP_LBA_PWL_SEG5, 0,
+			ISHARP_LBA_PWL_IN_SEG5, scl_data->dscl_prog_data.isharp_lba.in_seg[5],
+			ISHARP_LBA_PWL_BASE_SEG5, scl_data->dscl_prog_data.isharp_lba.base_seg[5]);
 
-	/* ISHARP_NLDELTA_SOFT_CLIP */
-	REG_SET_6(ISHARP_NLDELTA_SOFT_CLIP, 0,
-		ISHARP_NLDELTA_SCLIP_EN_P, scl_data->dscl_prog_data.isharp_nldelta_sclip.enable_p,
-		ISHARP_NLDELTA_SCLIP_PIVOT_P, scl_data->dscl_prog_data.isharp_nldelta_sclip.pivot_p,
-		ISHARP_NLDELTA_SCLIP_SLOPE_P, scl_data->dscl_prog_data.isharp_nldelta_sclip.slope_p,
-		ISHARP_NLDELTA_SCLIP_EN_N, scl_data->dscl_prog_data.isharp_nldelta_sclip.enable_n,
-		ISHARP_NLDELTA_SCLIP_PIVOT_N, scl_data->dscl_prog_data.isharp_nldelta_sclip.pivot_n,
-		ISHARP_NLDELTA_SCLIP_SLOPE_N, scl_data->dscl_prog_data.isharp_nldelta_sclip.slope_n);
+		/* ISHARP_DELTA_LUT */
+		if (!program_isharp_1dlut)
+			dpp401_dscl_set_isharp_filter(dpp, scl_data->dscl_prog_data.isharp_delta);
+
+		/* ISHARP_NLDELTA_SOFT_CLIP */
+		REG_SET_6(ISHARP_NLDELTA_SOFT_CLIP, 0,
+			ISHARP_NLDELTA_SCLIP_EN_P, scl_data->dscl_prog_data.isharp_nldelta_sclip.enable_p,
+			ISHARP_NLDELTA_SCLIP_PIVOT_P, scl_data->dscl_prog_data.isharp_nldelta_sclip.pivot_p,
+			ISHARP_NLDELTA_SCLIP_SLOPE_P, scl_data->dscl_prog_data.isharp_nldelta_sclip.slope_p,
+			ISHARP_NLDELTA_SCLIP_EN_N, scl_data->dscl_prog_data.isharp_nldelta_sclip.enable_n,
+			ISHARP_NLDELTA_SCLIP_PIVOT_N, scl_data->dscl_prog_data.isharp_nldelta_sclip.pivot_n,
+			ISHARP_NLDELTA_SCLIP_SLOPE_N, scl_data->dscl_prog_data.isharp_nldelta_sclip.slope_n);
 
 	/* Blur and Scale Coefficients - SCL_COEF_RAM_TAP_SELECT */
-	if (scl_data->dscl_prog_data.isharp_en) {
 		if (scl_data->dscl_prog_data.filter_blur_scale_v) {
 			dpp401_dscl_set_scaler_filter(
 				dpp, scl_data->taps.v_taps,
@@ -1037,6 +1032,7 @@ static void dpp401_dscl_program_isharp(struct dpp *dpp_base,
 			*bs_coeffs_updated = true;
 		}
 	}
+
 	PERF_TRACE();
 } // dpp401_dscl_program_isharp
 /**
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.h b/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.h
index fa5c4c18ed59..0a29a758d013 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.h
@@ -437,7 +437,9 @@ struct dcn_hubbub_registers {
 		type DCHUBBUB_ARB_REFCYC_PER_META_TRIP_A;\
 		type DCHUBBUB_ARB_REFCYC_PER_META_TRIP_B;\
 		type DCHUBBUB_ARB_FRAC_URG_BW_MALL_A;\
-		type DCHUBBUB_ARB_FRAC_URG_BW_MALL_B
+		type DCHUBBUB_ARB_FRAC_URG_BW_MALL_B;\
+		type DCHUBBUB_ARB_ALLOW_DCFCLK_DEEP_SLEEP_FORCE_VALUE;\
+		type DCHUBBUB_ARB_ALLOW_DCFCLK_DEEP_SLEEP_FORCE_ENABLE
 
 struct dcn_hubbub_shift {
 	DCN_HUBBUB_REG_FIELD_LIST(uint8_t);
@@ -447,6 +449,7 @@ struct dcn_hubbub_shift {
 	HUBBUB_REG_FIELD_LIST_DCN32(uint8_t);
 	HUBBUB_REG_FIELD_LIST_DCN35(uint8_t);
 	HUBBUB_REG_FIELD_LIST_DCN4_01(uint8_t);
+
 };
 
 struct dcn_hubbub_mask {
@@ -457,6 +460,7 @@ struct dcn_hubbub_mask {
 	HUBBUB_REG_FIELD_LIST_DCN32(uint32_t);
 	HUBBUB_REG_FIELD_LIST_DCN35(uint32_t);
 	HUBBUB_REG_FIELD_LIST_DCN4_01(uint32_t);
+
 };
 
 struct dc;
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c
index f01eae50d02f..c205500290ec 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c
@@ -733,10 +733,8 @@ void hubp401_cursor_set_position(
 	const struct dc_cursor_mi_param *param)
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
-	int x_pos = pos->x - param->recout.x;
-	int y_pos = pos->y - param->recout.y;
-	int rec_x_offset = x_pos - pos->x_hotspot;
-	int rec_y_offset = y_pos - pos->y_hotspot;
+	int rec_x_offset = pos->x - pos->x_hotspot;
+	int rec_y_offset = pos->y - pos->y_hotspot;
 	int dst_x_offset;
 	int x_pos_viewport = 0;
 	int x_hot_viewport = 0;
@@ -748,10 +746,10 @@ void hubp401_cursor_set_position(
 	 * within preceeding ODM slices.
 	 */
 	if (param->recout.width) {
-		x_pos_viewport = x_pos * param->viewport.width / param->recout.width;
+		x_pos_viewport = pos->x * param->viewport.width / param->recout.width;
 		x_hot_viewport = pos->x_hotspot * param->viewport.width / param->recout.width;
 	} else {
-		ASSERT(!cur_en || x_pos == 0);
+		ASSERT(!cur_en || pos->x == 0);
 		ASSERT(!cur_en || pos->x_hotspot == 0);
 	}
 
@@ -790,8 +788,8 @@ void hubp401_cursor_set_position(
 
 	if (!hubp->cursor_offload) {
 		REG_SET_2(CURSOR_POSITION, 0,
-			CURSOR_X_POSITION, x_pos,
-			CURSOR_Y_POSITION, y_pos);
+			CURSOR_X_POSITION, pos->x,
+			CURSOR_Y_POSITION, pos->y);
 
 		REG_SET_2(CURSOR_HOT_SPOT, 0,
 			CURSOR_HOT_SPOT_X, pos->x_hotspot,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
index 4986f12dc9df..ebdb885826aa 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
@@ -728,7 +728,6 @@ void dce110_edp_wait_for_hpd_ready(
 {
 	struct dc_context *ctx = link->ctx;
 	struct graphics_object_id connector = link->link_enc->connector;
-	struct gpio *hpd;
 	bool edp_hpd_high = false;
 	uint32_t time_elapsed = 0;
 	uint32_t timeout = power_up ?
@@ -752,31 +751,16 @@ void dce110_edp_wait_for_hpd_ready(
 	 * we need to wait until SENSE bit is high/low.
 	 */
 
-	/* obtain HPD */
-	/* TODO what to do with this? */
-	hpd = ctx->dc->link_srv->get_hpd_gpio(ctx->dc_bios, connector, ctx->gpio_service);
-
-	if (!hpd) {
-		BREAK_TO_DEBUGGER();
-		return;
-	}
-
 	if (link->panel_config.pps.extra_t3_ms > 0) {
 		int extra_t3_in_ms = link->panel_config.pps.extra_t3_ms;
 
 		msleep(extra_t3_in_ms);
 	}
 
-	dal_gpio_open(hpd, GPIO_MODE_INTERRUPT);
-
 	/* wait until timeout or panel detected */
 
 	do {
-		uint32_t detected = 0;
-
-		dal_gpio_get_value(hpd, &detected);
-
-		if (!(detected ^ power_up)) {
+		if (!(link->dc->link_srv->get_hpd_state(link) ^ power_up)) {
 			edp_hpd_high = true;
 			break;
 		}
@@ -786,10 +770,6 @@ void dce110_edp_wait_for_hpd_ready(
 		time_elapsed += HPD_CHECK_INTERVAL;
 	} while (time_elapsed < timeout);
 
-	dal_gpio_close(hpd);
-
-	dal_gpio_destroy_irq(&hpd);
-
 	/* ensure that the panel is detected */
 	if (!edp_hpd_high)
 		DC_LOG_DC("%s: wait timed out!\n", __func__);
@@ -1118,13 +1098,13 @@ void dce110_enable_audio_stream(struct pipe_ctx *pipe_ctx)
 			if (dc->current_state->res_ctx.pipe_ctx[i].stream_res.audio != NULL)
 				num_audio++;
 		}
+		if (num_audio >= 1 && clk_mgr->funcs->enable_pme_wa) {
+			/*wake AZ from D3 first before access az endpoint*/
+			clk_mgr->funcs->enable_pme_wa(clk_mgr);
+		}
 
 		pipe_ctx->stream_res.audio->funcs->az_enable(pipe_ctx->stream_res.audio);
 
-		if (num_audio >= 1 && clk_mgr->funcs->enable_pme_wa)
-			/*this is the first audio. apply the PME w/a in order to wake AZ from D3*/
-			clk_mgr->funcs->enable_pme_wa(clk_mgr);
-
 		link_hwss->enable_audio_packet(pipe_ctx);
 
 		if (pipe_ctx->stream_res.audio)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c
index 4ee6ed610de0..3e239124c17d 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c
@@ -108,7 +108,7 @@ static void update_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 		dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg;
 		ASSERT(dsc_cfg.dc_dsc_cfg.num_slices_h % opp_cnt == 0);
 		dsc_cfg.dc_dsc_cfg.num_slices_h /= opp_cnt;
-		dsc_cfg.dsc_padding = pipe_ctx->dsc_padding_params.dsc_hactive_padding;
+		dsc_cfg.dsc_padding = 0;
 
 		dsc->funcs->dsc_set_config(dsc, &dsc_cfg, &dsc_optc_cfg);
 		dsc->funcs->dsc_enable(dsc, pipe_ctx->stream_res.opp->inst);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
index bf19ba65d09a..3cd44c6602b3 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
@@ -82,6 +82,9 @@ void dcn32_dsc_pg_control(
 	if (!dc->debug.enable_double_buffered_dsc_pg_support)
 		return;
 
+	if (dc->debug.ignore_pg)
+		return;
+
 	REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
 	if (org_ip_request_cntl == 0)
 		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
@@ -168,6 +171,9 @@ void dcn32_hubp_pg_control(struct dce_hwseq *hws, unsigned int hubp_inst, bool p
 	if (hws->ctx->dc->debug.disable_hubp_power_gate)
 		return;
 
+	if (hws->ctx->dc->debug.ignore_pg)
+		return;
+
 	if (REG(DOMAIN0_PG_CONFIG) == 0)
 		return;
 
@@ -1061,7 +1067,7 @@ void dcn32_update_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 		dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg;
 		ASSERT(dsc_cfg.dc_dsc_cfg.num_slices_h % opp_cnt == 0);
 		dsc_cfg.dc_dsc_cfg.num_slices_h /= opp_cnt;
-		dsc_cfg.dsc_padding = pipe_ctx->dsc_padding_params.dsc_hactive_padding;
+		dsc_cfg.dsc_padding = 0;
 
 		if (should_use_dto_dscclk)
 			dccg->funcs->set_dto_dscclk(dccg, dsc->inst, dsc_cfg.dc_dsc_cfg.num_slices_h);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
index 7aa0f452e8f7..1271bf55dac3 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
@@ -364,7 +364,7 @@ static void update_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 		dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg;
 		ASSERT(dsc_cfg.dc_dsc_cfg.num_slices_h % opp_cnt == 0);
 		dsc_cfg.dc_dsc_cfg.num_slices_h /= opp_cnt;
-		dsc_cfg.dsc_padding = pipe_ctx->dsc_padding_params.dsc_hactive_padding;
+		dsc_cfg.dsc_padding = 0;
 
 		dsc->funcs->dsc_set_config(dsc, &dsc_cfg, &dsc_optc_cfg);
 		dsc->funcs->dsc_enable(dsc, pipe_ctx->stream_res.opp->inst);
@@ -1631,6 +1631,7 @@ void dcn35_begin_cursor_offload_update(struct dc *dc, const struct pipe_ctx *pip
 	payload_idx = write_idx % ARRAY_SIZE(cs->offload_streams[stream_idx].payloads);
 
 	cs->offload_streams[stream_idx].payloads[payload_idx].write_idx_start = write_idx;
+	cs->offload_streams[stream_idx].payloads[payload_idx].pipe_mask = 0;
 
 	if (pipe->plane_res.hubp)
 		pipe->plane_res.hubp->cursor_offload = true;
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
index 2fbc22afb89c..63cd13d4617a 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
@@ -297,7 +297,6 @@ void dcn401_init_hw(struct dc *dc)
 			}
 		}
 	}
-
 	for (i = 0; i < res_pool->audio_count; i++) {
 		struct audio *audio = res_pool->audios[i];
 
@@ -1213,6 +1212,9 @@ void dcn401_set_cursor_position(struct pipe_ctx *pipe_ctx)
 	if (recout_y_pos + (int)hubp->curs_attr.height <= 0)
 		pos_cpy.enable = false;  /* not visible beyond top edge*/
 
+	pos_cpy.x = x_pos;
+	pos_cpy.y = y_pos;
+
 	hubp->funcs->set_cursor_position(hubp, &pos_cpy, &param);
 	dpp->funcs->set_cursor_position(dpp, &pos_cpy, &param, hubp->curs_attr.width, hubp->curs_attr.height);
 }
@@ -1508,14 +1510,15 @@ void dcn401_dmub_hw_control_lock_fast(union block_sequence_params *params)
 
 void dcn401_fams2_update_config(struct dc *dc, struct dc_state *context, bool enable)
 {
-	bool fams2_required;
+	bool fams2_info_required;
 
 	if (!dc->ctx || !dc->ctx->dmub_srv || !dc->debug.fams2_config.bits.enable)
 		return;
 
-	fams2_required = context->bw_ctx.bw.dcn.fams2_global_config.features.bits.enable;
+	fams2_info_required = context->bw_ctx.bw.dcn.fams2_global_config.features.bits.enable;
+	fams2_info_required |= context->bw_ctx.bw.dcn.fams2_global_config.features.bits.legacy_method_no_fams2;
 
-	dc_dmub_srv_fams2_update_config(dc, context, enable && fams2_required);
+	dc_dmub_srv_fams2_update_config(dc, context, enable && fams2_info_required);
 }
 
 static void update_dsc_for_odm_change(struct dc *dc, struct dc_state *context,
@@ -1769,7 +1772,8 @@ void dcn401_unblank_stream(struct pipe_ctx *pipe_ctx,
 void dcn401_hardware_release(struct dc *dc)
 {
 	if (!dc->debug.disable_force_pstate_allow_on_hw_release) {
-		dc_dmub_srv_fams2_update_config(dc, dc->current_state, false);
+		if (dc->ctx->dmub_srv && dc->debug.fams2_config.bits.enable)
+			dc_dmub_srv_fams2_update_config(dc, dc->current_state, false);
 
 		/* If pstate unsupported, or still supported
 		* by firmware, force it supported by dcn
@@ -1789,7 +1793,9 @@ void dcn401_hardware_release(struct dc *dc)
 			dc->clk_mgr->clks.p_state_change_support = false;
 			dc->clk_mgr->funcs->update_clocks(dc->clk_mgr, dc->current_state, true);
 		}
-		dc_dmub_srv_fams2_update_config(dc, dc->current_state, false);
+
+		if (dc->ctx->dmub_srv && dc->debug.fams2_config.bits.enable)
+			dc_dmub_srv_fams2_update_config(dc, dc->current_state, false);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
index 8ed9eea40c56..490a6fccebff 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
@@ -1287,6 +1287,43 @@ struct hw_sequencer_funcs {
 	void (*get_underflow_debug_data)(const struct dc *dc,
 			struct timing_generator *tg,
 			struct dc_underflow_debug_data *out_data);
+
+	/**
+	 * measure_peak_bw_mbps - Measure actual peak bandwidth in Mbps
+	 * @dc: DC structure
+	 *
+	 * Returns the measured peak bandwidth value in Mbps from hardware
+	 * performance counters or registers.
+	 */
+	uint32_t (*measure_peak_bw_mbps)(struct dc *dc);
+
+	/**
+	 * measure_avg_bw_mbps - Measure actual average bandwidth in Mbps
+	 * @dc: DC structure
+	 *
+	 * Returns the measured average bandwidth value in Mbps from hardware
+	 * performance counters or registers.
+	 */
+	uint32_t (*measure_avg_bw_mbps)(struct dc *dc);
+
+	/**
+	 * measure_max_latency_ns - Measure actual maximum latency in nanoseconds
+	 * @dc: DC structure
+	 *
+	 * Returns the measured maximum latency value in nanoseconds from hardware
+	 * performance counters or registers.
+	 */
+	uint32_t (*measure_max_latency_ns)(struct dc *dc);
+
+	/**
+	 * measure_avg_latency_ns - Measure actual average latency in nanoseconds
+	 * @dc: DC structure
+	 *
+	 * Returns the measured average latency value in nanoseconds from hardware
+	 * performance counters or registers.
+	 */
+	uint32_t (*measure_avg_latency_ns)(struct dc *dc);
+
 };
 
 void color_space_to_black_color(
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
index df512920a9fa..cff705c49564 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
@@ -44,6 +44,7 @@ struct pipe_ctx;
 struct encoder_init_data {
 	enum channel_id channel;
 	struct graphics_object_id connector;
+	struct gpio *hpd_gpio;
 	enum hpd_source_id hpd_source;
 	/* TODO: in DAL2, here was pointer to EventManagerInterface */
 	struct graphics_object_id encoder;
@@ -87,6 +88,7 @@ struct link_encoder {
 	enum engine_id analog_engine;
 	struct encoder_feature_support features;
 	enum transmitter transmitter;
+	struct gpio *hpd_gpio;
 	enum hpd_source_id hpd_source;
 	bool usbc_combo_phy;
 };
@@ -178,6 +180,8 @@ struct link_encoder_funcs {
 	void (*disable_dpia_output)(struct link_encoder *link_enc,
 		uint8_t dpia_id,
 		uint8_t digmode);
+	bool (*get_hpd_state)(struct link_encoder *enc);
+	bool (*program_hpd_filter)(struct link_encoder *enc, int delay_on_connect_in_ms, int delay_on_disconnect_in_ms);
 };
 
 /*
diff --git a/drivers/gpu/drm/amd/display/dc/inc/link_service.h b/drivers/gpu/drm/amd/display/dc/inc/link_service.h
index 6f94e48a24d1..4b092a9ee4c6 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/link_service.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/link_service.h
@@ -114,9 +114,6 @@ struct link_service {
 			struct dc_sink_init_data *init_data);
 	void (*remove_remote_sink)(struct dc_link *link, struct dc_sink *sink);
 	bool (*get_hpd_state)(struct dc_link *link);
-	struct gpio *(*get_hpd_gpio)(struct dc_bios *dcb,
-			struct graphics_object_id link_id,
-			struct gpio_service *gpio_service);
 	void (*enable_hpd)(const struct dc_link *link);
 	void (*disable_hpd)(const struct dc_link *link);
 	void (*enable_hpd_filter)(struct dc_link *link, bool enable);
@@ -307,6 +304,10 @@ struct link_service {
 	bool (*edp_receiver_ready_T9)(struct dc_link *link);
 	bool (*edp_receiver_ready_T7)(struct dc_link *link);
 	bool (*edp_power_alpm_dpcd_enable)(struct dc_link *link, bool enable);
+	bool (*edp_pr_enable)(struct dc_link *link, bool enable);
+	bool (*edp_pr_update_state)(struct dc_link *link, struct dmub_cmd_pr_update_state_data *update_state_data);
+	bool (*edp_pr_set_general_cmd)(struct dc_link *link, struct dmub_cmd_pr_general_cmd_data *general_cmd_data);
+	bool (*edp_pr_get_state)(const struct dc_link *link, uint64_t *state);
 	void (*edp_set_panel_power)(struct dc_link *link, bool powerOn);
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_detection.c b/drivers/gpu/drm/amd/display/dc/link/link_detection.c
index 6d31f4967f1a..373e68cf2bde 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_detection.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_detection.c
@@ -171,6 +171,7 @@ static enum signal_type link_detect_sink_signal_type(struct dc_link *link,
 					 enum dc_detect_reason reason)
 {
 	enum signal_type result;
+	struct audio_support *aud_support;
 	struct graphics_object_id enc_id;
 
 	if (link->is_dig_mapping_flexible)
@@ -183,53 +184,51 @@ static enum signal_type link_detect_sink_signal_type(struct dc_link *link,
 	if (link->ep_type != DISPLAY_ENDPOINT_PHY)
 		return result;
 
-	/* Internal digital encoder will detect only dongles
+	/*
+	 * Internal digital encoder will detect only dongles
 	 * that require digital signal
 	 */
 
-	/* Detection mechanism is different
+	/*
+	 * Detection mechanism is different
 	 * for different native connectors.
 	 * LVDS connector supports only LVDS signal;
 	 * PCIE is a bus slot, the actual connector needs to be detected first;
 	 * eDP connector supports only eDP signal;
 	 * HDMI should check straps for audio
 	 */
-
-	/* PCIE detects the actual connector on add-on board */
-	if (link->link_id.id == CONNECTOR_ID_PCIE) {
-		/* ZAZTODO implement PCIE add-on card detection */
-	}
-
 	switch (link->link_id.id) {
-	case CONNECTOR_ID_HDMI_TYPE_A: {
-		/* check audio support:
+	case CONNECTOR_ID_HDMI_TYPE_A:
+		/*
+		 * check audio support:
 		 * if native HDMI is not supported, switch to DVI
 		 */
-		struct audio_support *aud_support =
-					&link->dc->res_pool->audio_support;
+		aud_support = &link->dc->res_pool->audio_support;
 
 		if (!aud_support->hdmi_audio_native)
-			if (link->link_id.id == CONNECTOR_ID_HDMI_TYPE_A)
-				result = SIGNAL_TYPE_DVI_SINGLE_LINK;
-	}
-	break;
+			result = SIGNAL_TYPE_DVI_SINGLE_LINK;
+		break;
 	case CONNECTOR_ID_DISPLAY_PORT:
-	case CONNECTOR_ID_USBC: {
-		/* DP HPD short pulse. Passive DP dongle will not
+	case CONNECTOR_ID_USBC:
+		/*
+		 * DP HPD short pulse. Passive DP dongle will not
 		 * have short pulse
 		 */
 		if (reason != DETECT_REASON_HPDRX) {
-			/* Check whether DP signal detected: if not -
+			/*
+			 * Check whether DP signal detected: if not -
 			 * we assume signal is DVI; it could be corrected
 			 * to HDMI after dongle detection
 			 */
 			if (!dm_helpers_is_dp_sink_present(link))
 				result = SIGNAL_TYPE_DVI_SINGLE_LINK;
 		}
-	}
-	break;
+		break;
+	case CONNECTOR_ID_PCIE:
+		/* ZAZTODO implement PCIE add-on card detection */
+		break;
 	default:
-	break;
+		break;
 	}
 
 	return result;
@@ -989,7 +988,7 @@ static bool detect_link_and_local_sink(struct dc_link *link,
 			(link->dpcd_sink_ext_caps.bits.oled == 1)) {
 			dpcd_set_source_specific_data(link);
 			msleep(post_oui_delay);
-			set_default_brightness_aux(link);
+			set_default_brightness(link);
 		}
 
 		return true;
@@ -1401,8 +1400,6 @@ static bool link_detect_analog(struct dc_link *link, enum dc_connection_type *ty
  */
 bool link_detect_connection_type(struct dc_link *link, enum dc_connection_type *type)
 {
-	uint32_t is_hpd_high = 0;
-
 	if (link->connector_signal == SIGNAL_TYPE_LVDS) {
 		*type = dc_connection_single;
 		return true;
@@ -1437,10 +1434,7 @@ bool link_detect_connection_type(struct dc_link *link, enum dc_connection_type *
 	}
 
 
-	if (!query_hpd_status(link, &is_hpd_high))
-		goto hpd_gpio_failure;
-
-	if (is_hpd_high) {
+	if (link_get_hpd_state(link)) {
 		*type = dc_connection_single;
 		/* TODO: need to do the actual detection */
 	} else {
@@ -1453,9 +1447,6 @@ bool link_detect_connection_type(struct dc_link *link, enum dc_connection_type *
 	}
 
 	return true;
-
-hpd_gpio_failure:
-	return false;
 }
 
 bool link_detect(struct dc_link *link, enum dc_detect_reason reason)
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
index 6ae134147617..7384676e646d 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
@@ -841,7 +841,7 @@ void link_set_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 		dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg;
 		ASSERT(dsc_cfg.dc_dsc_cfg.num_slices_h % opp_cnt == 0);
 		dsc_cfg.dc_dsc_cfg.num_slices_h /= opp_cnt;
-		dsc_cfg.dsc_padding = pipe_ctx->dsc_padding_params.dsc_hactive_padding;
+		dsc_cfg.dsc_padding = 0;
 
 		if (should_use_dto_dscclk)
 			dccg->funcs->set_dto_dscclk(dccg, dsc->inst, dsc_cfg.dc_dsc_cfg.num_slices_h);
@@ -857,6 +857,7 @@ void link_set_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 		}
 		dsc_cfg.dc_dsc_cfg.num_slices_h *= opp_cnt;
 		dsc_cfg.pic_width *= opp_cnt;
+		dsc_cfg.dsc_padding = pipe_ctx->dsc_padding_params.dsc_hactive_padding;
 
 		optc_dsc_mode = dsc_optc_cfg.is_pixel_format_444 ? OPTC_DSC_ENABLED_444 : OPTC_DSC_ENABLED_NATIVE_SUBSAMPLED;
 
@@ -1930,7 +1931,7 @@ static void disable_link_dp(struct dc_link *link,
 			link->dc->hwss.edp_power_control(link, false);
 	}
 
-	if (signal == SIGNAL_TYPE_DISPLAY_PORT_MST)
+	if (signal == SIGNAL_TYPE_DISPLAY_PORT_MST && link->sink_count == 0)
 		/* set the sink to SST mode after disabling the link */
 		enable_mst_on_sink(link, false);
 
@@ -2081,7 +2082,12 @@ static enum dc_status enable_link_dp(struct dc_state *state,
 			pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT &&
 			link->dc->debug.set_mst_en_for_sst) {
 		enable_mst_on_sink(link, true);
+	} else if (link->dpcd_caps.is_mst_capable &&
+		pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT) {
+		/* disable mst on sink */
+		enable_mst_on_sink(link, false);
 	}
+
 	if (pipe_ctx->stream->signal == SIGNAL_TYPE_EDP) {
 		/*in case it is not on*/
 		if (!link->dc->config.edp_no_power_sequencing)
@@ -2115,7 +2121,7 @@ static enum dc_status enable_link_dp(struct dc_state *state,
 		skip_video_pattern = false;
 
 	if (stream->sink_patches.oled_optimize_display_on)
-		set_default_brightness_aux(link);
+		set_default_brightness(link);
 
 	if (perform_link_training_with_retries(link_settings,
 					       skip_video_pattern,
@@ -2141,7 +2147,7 @@ static enum dc_status enable_link_dp(struct dc_state *state,
 		link->dpcd_sink_ext_caps.bits.sdr_aux_backlight_control == 1 ||
 		link->dpcd_sink_ext_caps.bits.hdr_aux_backlight_control == 1) {
 		if (!stream->sink_patches.oled_optimize_display_on) {
-			set_default_brightness_aux(link);
+			set_default_brightness(link);
 			if (link->dpcd_sink_ext_caps.bits.oled == 1)
 				msleep(bl_oled_enable_delay);
 			edp_backlight_enable_aux(link, true);
@@ -2367,9 +2373,9 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx)
 	if (pipe_ctx->stream->sink) {
 		if (pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_VIRTUAL &&
 			pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_NONE) {
-			DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d\n", __func__,
+			DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d sink_count=%d\n", __func__,
 			pipe_ctx->stream->sink->edid_caps.display_name,
-			pipe_ctx->stream->signal, link->link_index);
+			pipe_ctx->stream->signal, link->link_index, link->sink_count);
 		}
 	}
 
@@ -2483,10 +2489,11 @@ void link_set_dpms_on(
 	if (pipe_ctx->stream->sink) {
 		if (pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_VIRTUAL &&
 			pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_NONE) {
-			DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d\n", __func__,
+			DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d sink_count=%d\n", __func__,
 			pipe_ctx->stream->sink->edid_caps.display_name,
 			pipe_ctx->stream->signal,
-			link->link_index);
+			link->link_index,
+			link->sink_count);
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_factory.c b/drivers/gpu/drm/amd/display/dc/link/link_factory.c
index a6e2b0821969..56fdd1446a59 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_factory.c
@@ -73,7 +73,6 @@ static void construct_link_service_detection(struct link_service *link_srv)
 	link_srv->add_remote_sink = link_add_remote_sink;
 	link_srv->remove_remote_sink = link_remove_remote_sink;
 	link_srv->get_hpd_state = link_get_hpd_state;
-	link_srv->get_hpd_gpio = link_get_hpd_gpio;
 	link_srv->enable_hpd = link_enable_hpd;
 	link_srv->disable_hpd = link_disable_hpd;
 	link_srv->enable_hpd_filter = link_enable_hpd_filter;
@@ -229,6 +228,10 @@ static void construct_link_service_edp_panel_control(struct link_service *link_s
 	link_srv->edp_receiver_ready_T9 = edp_receiver_ready_T9;
 	link_srv->edp_receiver_ready_T7 = edp_receiver_ready_T7;
 	link_srv->edp_power_alpm_dpcd_enable = edp_power_alpm_dpcd_enable;
+	link_srv->edp_pr_enable = edp_pr_enable;
+	link_srv->edp_pr_update_state = edp_pr_update_state;
+	link_srv->edp_pr_set_general_cmd = edp_pr_set_general_cmd;
+	link_srv->edp_pr_get_state = edp_pr_get_state;
 	link_srv->edp_set_panel_power = edp_set_panel_power;
 }
 
@@ -350,24 +353,6 @@ static enum transmitter translate_encoder_to_transmitter(
 			return TRANSMITTER_UNKNOWN;
 		}
 	break;
-	case ENCODER_ID_EXTERNAL_NUTMEG:
-		switch (encoder.enum_id) {
-		case ENUM_ID_1:
-			return TRANSMITTER_NUTMEG_CRT;
-		default:
-			return TRANSMITTER_UNKNOWN;
-		}
-	break;
-	case ENCODER_ID_EXTERNAL_TRAVIS:
-		switch (encoder.enum_id) {
-		case ENUM_ID_1:
-			return TRANSMITTER_TRAVIS_CRT;
-		case ENUM_ID_2:
-			return TRANSMITTER_TRAVIS_LCD;
-		default:
-			return TRANSMITTER_UNKNOWN;
-		}
-	break;
 	default:
 		return TRANSMITTER_UNKNOWN;
 	}
@@ -377,11 +362,6 @@ static void link_destruct(struct dc_link *link)
 {
 	int i;
 
-	if (link->hpd_gpio) {
-		dal_gpio_destroy_irq(&link->hpd_gpio);
-		link->hpd_gpio = NULL;
-	}
-
 	if (link->ddc)
 		link_destroy_ddc_service(&link->ddc);
 
@@ -477,14 +457,6 @@ static enum engine_id find_analog_engine(struct dc_link *link)
 	return ENGINE_ID_UNKNOWN;
 }
 
-static bool transmitter_supported(const enum transmitter transmitter)
-{
-	return transmitter != TRANSMITTER_UNKNOWN &&
-		transmitter != TRANSMITTER_NUTMEG_CRT &&
-		transmitter != TRANSMITTER_TRAVIS_CRT &&
-		transmitter != TRANSMITTER_TRAVIS_LCD;
-}
-
 static bool analog_engine_supported(const enum engine_id engine_id)
 {
 	return engine_id == ENGINE_ID_DACA ||
@@ -502,6 +474,9 @@ static bool construct_phy(struct dc_link *link,
 	struct dc_bios *bios = init_params->dc->ctx->dc_bios;
 	const struct dc_vbios_funcs *bp_funcs = bios->funcs;
 	struct bp_disp_connector_caps_info disp_connect_caps_info = { 0 };
+	struct graphics_object_id link_encoder = { 0 };
+	enum transmitter transmitter_from_encoder;
+	enum engine_id link_analog_engine;
 
 	DC_LOGGER_INIT(dc_ctx->logger);
 
@@ -522,21 +497,21 @@ static bool construct_phy(struct dc_link *link,
 	link->link_id =
 		bios->funcs->get_connector_id(bios, init_params->connector_index);
 
-	/* Determine early if the link has any supported encoders,
-	 * so that we avoid initializing DDC and HPD, etc.
-	 */
-	bp_funcs->get_src_obj(bios, link->link_id, 0, &enc_init_data.encoder);
-	enc_init_data.transmitter = translate_encoder_to_transmitter(enc_init_data.encoder);
-	enc_init_data.analog_engine = find_analog_engine(link);
-
 	link->ep_type = DISPLAY_ENDPOINT_PHY;
 
 	DC_LOG_DC("BIOS object table - link_id: %d", link->link_id.id);
 
-	if (!transmitter_supported(enc_init_data.transmitter) &&
-	    !analog_engine_supported(enc_init_data.analog_engine)) {
+	/* Determine early if the link has any supported encoders,
+	 * so that we avoid initializing DDC and HPD, etc.
+	 */
+	bp_funcs->get_src_obj(bios, link->link_id, 0, &link_encoder);
+	transmitter_from_encoder = translate_encoder_to_transmitter(link_encoder);
+	link_analog_engine = find_analog_engine(link);
+
+	if (transmitter_from_encoder == TRANSMITTER_UNKNOWN &&
+	    !analog_engine_supported(link_analog_engine)) {
 		DC_LOG_WARNING("link_id %d has unsupported encoder\n", link->link_id.id);
-		goto unsupported_fail;
+		goto create_fail;
 	}
 
 	if (bios->funcs->get_disp_connector_caps_info) {
@@ -555,25 +530,76 @@ static bool construct_phy(struct dc_link *link,
 	if (link->dc->res_pool->funcs->link_init)
 		link->dc->res_pool->funcs->link_init(link);
 
-	link->hpd_gpio = link_get_hpd_gpio(link->ctx->dc_bios, link->link_id,
+	ddc_service_init_data.ctx = link->ctx;
+	ddc_service_init_data.id = link->link_id;
+	ddc_service_init_data.link = link;
+	link->ddc = link_create_ddc_service(&ddc_service_init_data);
+
+	if (!link->ddc) {
+		DC_ERROR("Failed to create ddc_service!\n");
+		goto ddc_create_fail;
+	}
+
+	if (!link->ddc->ddc_pin) {
+		DC_ERROR("Failed to get I2C info for connector!\n");
+		goto ddc_create_fail;
+	}
+
+	link->ddc_hw_inst =
+		dal_ddc_get_line(get_ddc_pin(link->ddc));
+
+	enc_init_data.ctx = dc_ctx;
+	enc_init_data.connector = link->link_id;
+	enc_init_data.channel = get_ddc_line(link);
+	enc_init_data.transmitter = transmitter_from_encoder;
+	enc_init_data.encoder = link_encoder;
+	enc_init_data.analog_engine = link_analog_engine;
+	enc_init_data.hpd_gpio = link_get_hpd_gpio(link->ctx->dc_bios, link->link_id,
 				      link->ctx->gpio_service);
 
-	if (link->hpd_gpio) {
-		dal_gpio_open(link->hpd_gpio, GPIO_MODE_INTERRUPT);
-		dal_gpio_unlock_pin(link->hpd_gpio);
-		link->irq_source_hpd = dal_irq_get_source(link->hpd_gpio);
+	if (enc_init_data.hpd_gpio) {
+		dal_gpio_open(enc_init_data.hpd_gpio, GPIO_MODE_INTERRUPT);
+		dal_gpio_unlock_pin(enc_init_data.hpd_gpio);
+		link->irq_source_hpd = dal_irq_get_source(enc_init_data.hpd_gpio);
+		enc_init_data.hpd_source = get_hpd_line(link);
+		link->hpd_src = enc_init_data.hpd_source;
+
+		DC_LOG_DC("BIOS object table - hpd_gpio id: %d", enc_init_data.hpd_gpio->id);
+		DC_LOG_DC("BIOS object table - hpd_gpio en: %d", enc_init_data.hpd_gpio->en);
+	} else {
+		struct graphics_object_hpd_info hpd_info;
+
+		if (link->ctx->dc_bios->funcs->get_hpd_info(link->ctx->dc_bios, link->link_id, &hpd_info) == BP_RESULT_OK) {
+			link->hpd_src = hpd_info.hpd_int_gpio_uid - 1;
+			link->irq_source_hpd =  DC_IRQ_SOURCE_HPD1 + link->hpd_src;
+			enc_init_data.hpd_source = link->hpd_src;
+			DC_LOG_DC("BIOS object table - hpd_int_gpio_uid id: %d", hpd_info.hpd_int_gpio_uid);
+		} else {
+			ASSERT(0);
+			enc_init_data.hpd_source = HPD_SOURCEID_UNKNOWN;
+		}
+	}
+
+	link->link_enc =
+		link->dc->res_pool->funcs->link_enc_create(dc_ctx, &enc_init_data);
 
-		DC_LOG_DC("BIOS object table - hpd_gpio id: %d", link->hpd_gpio->id);
-		DC_LOG_DC("BIOS object table - hpd_gpio en: %d", link->hpd_gpio->en);
+	if (!link->link_enc) {
+		DC_ERROR("Failed to create link encoder!\n");
+		goto link_enc_create_fail;
 	}
 
+	DC_LOG_DC("BIOS object table - DP_IS_USB_C: %d", link->link_enc->features.flags.bits.DP_IS_USB_C);
+	DC_LOG_DC("BIOS object table - IS_DP2_CAPABLE: %d", link->link_enc->features.flags.bits.IS_DP2_CAPABLE);
+
 	switch (link->link_id.id) {
 	case CONNECTOR_ID_HDMI_TYPE_A:
 		link->connector_signal = SIGNAL_TYPE_HDMI_TYPE_A;
 
-		if (link->hpd_gpio)
+		if (link->link_enc->hpd_gpio)
 			link->irq_source_read_request =
-					dal_irq_get_read_request(link->hpd_gpio);
+					dal_irq_get_read_request(link->link_enc->hpd_gpio);
+		else if (link->hpd_src != HPD_SOURCEID_UNKNOWN)
+			link->irq_source_read_request = DC_IRQ_SOURCE_DCI2C_RR_DDC1 + link->hpd_src;
 		break;
 	case CONNECTOR_ID_SINGLE_LINK_DVID:
 	case CONNECTOR_ID_SINGLE_LINK_DVII:
@@ -591,9 +617,11 @@ static bool construct_phy(struct dc_link *link,
 	case CONNECTOR_ID_USBC:
 		link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
 
-		if (link->hpd_gpio)
+		if (link->link_enc->hpd_gpio)
 			link->irq_source_hpd_rx =
-					dal_irq_get_rx_source(link->hpd_gpio);
+					dal_irq_get_rx_source(link->link_enc->hpd_gpio);
+		else if (link->hpd_src != HPD_SOURCEID_UNKNOWN)
+			link->irq_source_hpd_rx = DC_IRQ_SOURCE_HPD1RX + link->hpd_src;
 
 		break;
 	case CONNECTOR_ID_EDP:
@@ -603,37 +631,45 @@ static bool construct_phy(struct dc_link *link,
 			goto create_fail;
 
 		link->connector_signal = SIGNAL_TYPE_EDP;
+		if (!link->dc->config.allow_edp_hotplug_detection
+			&& !is_smartmux_suported(link))
+			link->irq_source_hpd = DC_IRQ_SOURCE_INVALID;
 
-		if (link->hpd_gpio) {
-			if (!link->dc->config.allow_edp_hotplug_detection
-				&& !is_smartmux_suported(link))
-				link->irq_source_hpd = DC_IRQ_SOURCE_INVALID;
-
-			switch (link->dc->config.allow_edp_hotplug_detection) {
-			case HPD_EN_FOR_ALL_EDP:
+		switch (link->dc->config.allow_edp_hotplug_detection) {
+		case HPD_EN_FOR_ALL_EDP:
+			if (link->link_enc->hpd_gpio) {
 				link->irq_source_hpd_rx =
-						dal_irq_get_rx_source(link->hpd_gpio);
-				break;
-			case HPD_EN_FOR_PRIMARY_EDP_ONLY:
-				if (link->link_index == 0)
+						dal_irq_get_rx_source(link->link_enc->hpd_gpio);
+				} else if (link->hpd_src != HPD_SOURCEID_UNKNOWN) {
+					link->irq_source_hpd_rx = DC_IRQ_SOURCE_HPD1RX + link->hpd_src;
+				}
+			break;
+		case HPD_EN_FOR_PRIMARY_EDP_ONLY:
+			if (link->link_index == 0) {
+				if (link->link_enc->hpd_gpio) {
 					link->irq_source_hpd_rx =
-						dal_irq_get_rx_source(link->hpd_gpio);
-				else
-					link->irq_source_hpd = DC_IRQ_SOURCE_INVALID;
-				break;
-			case HPD_EN_FOR_SECONDARY_EDP_ONLY:
-				if (link->link_index == 1)
+						dal_irq_get_rx_source(link->link_enc->hpd_gpio);
+				} else if (link->hpd_src != HPD_SOURCEID_UNKNOWN) {
+					link->irq_source_hpd_rx = DC_IRQ_SOURCE_HPD1RX + link->hpd_src;
+				}
+			} else
+				link->irq_source_hpd = DC_IRQ_SOURCE_INVALID;
+			break;
+		case HPD_EN_FOR_SECONDARY_EDP_ONLY:
+			if (link->link_index == 1) {
+				if (link->link_enc->hpd_gpio) {
 					link->irq_source_hpd_rx =
-						dal_irq_get_rx_source(link->hpd_gpio);
-				else
-					link->irq_source_hpd = DC_IRQ_SOURCE_INVALID;
-				break;
-			default:
+						dal_irq_get_rx_source(link->link_enc->hpd_gpio);
+				} else if (link->hpd_src != HPD_SOURCEID_UNKNOWN) {
+					link->irq_source_hpd_rx = DC_IRQ_SOURCE_HPD1RX + link->hpd_src;
+				}
+			} else
 				link->irq_source_hpd = DC_IRQ_SOURCE_INVALID;
-				break;
-			}
+			break;
+		default:
+			link->irq_source_hpd = DC_IRQ_SOURCE_INVALID;
+			break;
 		}
-
 		break;
 	case CONNECTOR_ID_LVDS:
 		link->connector_signal = SIGNAL_TYPE_LVDS;
@@ -648,42 +684,6 @@ static bool construct_phy(struct dc_link *link,
 		  init_params->connector_index,
 		  signal_type_to_string(link->connector_signal));
 
-	ddc_service_init_data.ctx = link->ctx;
-	ddc_service_init_data.id = link->link_id;
-	ddc_service_init_data.link = link;
-	link->ddc = link_create_ddc_service(&ddc_service_init_data);
-
-	if (!link->ddc) {
-		DC_ERROR("Failed to create ddc_service!\n");
-		goto ddc_create_fail;
-	}
-
-	if (!link->ddc->ddc_pin) {
-		DC_ERROR("Failed to get I2C info for connector!\n");
-		goto ddc_create_fail;
-	}
-
-	link->ddc_hw_inst =
-		dal_ddc_get_line(get_ddc_pin(link->ddc));
-
-	enc_init_data.ctx = dc_ctx;
-	enc_init_data.connector = link->link_id;
-	enc_init_data.channel = get_ddc_line(link);
-	enc_init_data.hpd_source = get_hpd_line(link);
-
-	link->hpd_src = enc_init_data.hpd_source;
-
-	link->link_enc =
-		link->dc->res_pool->funcs->link_enc_create(dc_ctx, &enc_init_data);
-
-	if (!link->link_enc) {
-		DC_ERROR("Failed to create link encoder!\n");
-		goto link_enc_create_fail;
-	}
-
-	DC_LOG_DC("BIOS object table - DP_IS_USB_C: %d", link->link_enc->features.flags.bits.DP_IS_USB_C);
-	DC_LOG_DC("BIOS object table - IS_DP2_CAPABLE: %d", link->link_enc->features.flags.bits.IS_DP2_CAPABLE);
-
 	/* Update link encoder tracking variables. These are used for the dynamic
 	 * assignment of link encoders to streams.
 	 */
@@ -792,21 +792,17 @@ static bool construct_phy(struct dc_link *link,
 	DC_LOG_DC("BIOS object table - %s finished successfully.\n", __func__);
 	return true;
 device_tag_fail:
-	link->link_enc->funcs->destroy(&link->link_enc);
 link_enc_create_fail:
-	if (link->panel_cntl != NULL)
-		link->panel_cntl->funcs->destroy(&link->panel_cntl);
 panel_cntl_create_fail:
-	link_destroy_ddc_service(&link->ddc);
 ddc_create_fail:
 create_fail:
+	if (link->ddc)
+		link_destroy_ddc_service(&link->ddc);
+	if (link->panel_cntl)
+		link->panel_cntl->funcs->destroy(&link->panel_cntl);
+	if (link->link_enc)
+		link->link_enc->funcs->destroy(&link->link_enc);
 
-	if (link->hpd_gpio) {
-		dal_gpio_destroy_irq(&link->hpd_gpio);
-		link->hpd_gpio = NULL;
-	}
-
-unsupported_fail:
 	DC_LOG_DC("BIOS object table - %s failed.\n", __func__);
 	return false;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
index ad90a0106938..54c417928b61 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
@@ -2167,7 +2167,7 @@ void detect_edp_sink_caps(struct dc_link *link)
 	link->dpcd_caps.set_power_state_capable_edp =
 				(general_edp_cap & DP_EDP_SET_POWER_CAP) ? true : false;
 
-	set_default_brightness_aux(link);
+	set_default_brightness(link);
 
 	core_link_read_dpcd(link, DP_EDP_DPCD_REV,
 		&link->dpcd_caps.edp_rev,
@@ -2219,6 +2219,13 @@ void detect_edp_sink_caps(struct dc_link *link)
 				(uint8_t *)&link->dpcd_caps.edp_oled_emission_rate,
 				sizeof(link->dpcd_caps.edp_oled_emission_rate));
 
+	/*
+	 * Read DRR granularity
+	 */
+	core_link_read_dpcd(link, DP_SINK_DRR_GRANULARITY,
+			(uint8_t *)&link->dpcd_caps.drr_granularity,
+			sizeof(link->dpcd_caps.drr_granularity));
+
 	/*
 	 * Read Multi-SST (Single Stream Transport) capability
 	 * for eDP version 1.4 or higher.
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c
index ce174ce5579c..6a7c4a59ff4c 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c
@@ -271,7 +271,7 @@ enum link_training_result dp_perform_fixed_vs_pe_training_sequence(
 	rate = get_dpcd_link_rate(&lt_settings->link_settings);
 
 	// Only perform toggle if FIXED_VS LTTPR reports no IEEE OUI
-	if (memcmp("\x0,\x0,\x0", &link->dpcd_caps.lttpr_caps.lttpr_ieee_oui[0], 3) == 0) {
+	if (memcmp("\x00\x00\x00", &link->dpcd_caps.lttpr_caps.lttpr_ieee_oui[0], 3) == 0) {
 		/* Vendor specific: Toggle link rate */
 		toggle_rate = (rate == 0x6) ? 0xA : 0x6;
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
index c56e69eb27ef..cf06b9b62e1d 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
@@ -50,6 +50,11 @@ static const uint8_t DP_VGA_LVDS_CONVERTER_ID_2[] = "sivarT";
 /* Nutmeg */
 static const uint8_t DP_VGA_LVDS_CONVERTER_ID_3[] = "dnomlA";
 
+static const unsigned int pwr_default_min_brightness_millinits = 1000;
+static const unsigned int pwr_default_sdr_brightness_millinits = 270000;
+static const unsigned int pwr_default_min_backlight_pwm = 0xC0C;
+static const unsigned int pwr_default_max_backlight_pwm = 0xFFFF;
+
 void dp_set_panel_mode(struct dc_link *link, enum dp_panel_mode panel_mode)
 {
 	union dpcd_edp_config edp_config_set;
@@ -309,7 +314,7 @@ static bool read_default_bl_aux(struct dc_link *link, uint32_t *backlight_millin
 	return true;
 }
 
-bool set_default_brightness_aux(struct dc_link *link)
+bool set_default_brightness(struct dc_link *link)
 {
 	uint32_t default_backlight;
 
@@ -320,8 +325,23 @@ bool set_default_brightness_aux(struct dc_link *link)
 		if (default_backlight < 1000 || default_backlight > 5000000)
 			default_backlight = 150000;
 
-		return edp_set_backlight_level_nits(link, true,
-				default_backlight, 0);
+		if (link->backlight_control_type == BACKLIGHT_CONTROL_VESA_AUX &&
+			link->dc->caps.dmub_caps.aux_backlight_support) {
+			struct set_backlight_level_params backlight_level_params = { 0 };
+
+			backlight_level_params.aux_inst =  link->ddc->ddc_pin->hw_info.ddc_channel;
+			backlight_level_params.control_type = BACKLIGHT_CONTROL_VESA_AUX;
+			backlight_level_params.backlight_pwm_u16_16 = default_backlight;
+			backlight_level_params.transition_time_in_ms = 0;
+			// filled in the driver BL default values
+			backlight_level_params.min_luminance = pwr_default_min_brightness_millinits;
+			backlight_level_params.max_luminance = pwr_default_sdr_brightness_millinits;
+			backlight_level_params.min_backlight_pwm = pwr_default_min_backlight_pwm;
+			backlight_level_params.max_backlight_pwm = pwr_default_max_backlight_pwm;
+			return edp_set_backlight_level(link, &backlight_level_params);
+		} else
+			return edp_set_backlight_level_nits(link, true,
+					default_backlight, 0);
 	}
 	return false;
 }
@@ -1049,8 +1069,7 @@ static bool edp_setup_panel_replay(struct dc_link *link, const struct dc_stream_
 
 	replay_context.line_time_in_ns = lineTimeInNs;
 
-	link->replay_settings.replay_feature_enabled =
-			replay->funcs->replay_copy_settings(replay, link, &replay_context, panel_inst);
+	link->replay_settings.replay_feature_enabled = edp_pr_copy_settings(link, &replay_context);
 
 	if (link->replay_settings.replay_feature_enabled) {
 		pr_config_1.bits.PANEL_REPLAY_ENABLE = 1;
@@ -1305,6 +1324,157 @@ bool edp_set_replay_power_opt_and_coasting_vtotal(struct dc_link *link,
 	return true;
 }
 
+bool edp_pr_enable(struct dc_link *link, bool enable)
+{
+	struct dc  *dc = link->ctx->dc;
+	unsigned int panel_inst = 0;
+	union dmub_rb_cmd cmd;
+
+	if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst))
+		return false;
+
+	if (link->replay_settings.replay_allow_active != enable) {
+		//for sending PR enable commands to DMUB
+		memset(&cmd, 0, sizeof(cmd));
+
+		cmd.pr_enable.header.type = DMUB_CMD__PR;
+		cmd.pr_enable.header.sub_type = DMUB_CMD__PR_ENABLE;
+		cmd.pr_enable.header.payload_bytes = sizeof(struct dmub_cmd_pr_enable_data);
+		cmd.pr_enable.data.panel_inst = panel_inst;
+		cmd.pr_enable.data.enable = enable ? 1 : 0;
+
+		dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+
+		link->replay_settings.replay_allow_active = enable;
+	}
+	return true;
+}
+
+bool edp_pr_copy_settings(struct dc_link *link, struct replay_context *replay_context)
+{
+	struct dc  *dc = link->ctx->dc;
+	unsigned int panel_inst = 0;
+	union dmub_rb_cmd cmd;
+	struct pipe_ctx *pipe_ctx = NULL;
+
+	if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst))
+		return false;
+
+	for (unsigned int i = 0; i < MAX_PIPES; i++) {
+		if (dc->current_state->res_ctx.pipe_ctx[i].stream &&
+			dc->current_state->res_ctx.pipe_ctx[i].stream->link &&
+			dc->current_state->res_ctx.pipe_ctx[i].stream->link == link &&
+			dc->current_state->res_ctx.pipe_ctx[i].stream->link->connector_signal == SIGNAL_TYPE_EDP) {
+			pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
+			//TODO: refactor for multi edp support
+			break;
+		}
+	}
+
+	if (!pipe_ctx)
+		return false;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.pr_copy_settings.header.type = DMUB_CMD__PR;
+	cmd.pr_copy_settings.header.sub_type = DMUB_CMD__PR_COPY_SETTINGS;
+	cmd.pr_copy_settings.header.payload_bytes = sizeof(struct dmub_cmd_pr_copy_settings_data);
+	cmd.pr_copy_settings.data.panel_inst = panel_inst;
+	// HW inst
+	cmd.pr_copy_settings.data.aux_inst = replay_context->aux_inst;
+	cmd.pr_copy_settings.data.digbe_inst = replay_context->digbe_inst;
+	cmd.pr_copy_settings.data.digfe_inst = replay_context->digfe_inst;
+	if (pipe_ctx->plane_res.dpp)
+		cmd.pr_copy_settings.data.dpp_inst = pipe_ctx->plane_res.dpp->inst;
+	else
+		cmd.pr_copy_settings.data.dpp_inst = 0;
+	if (pipe_ctx->stream_res.tg)
+		cmd.pr_copy_settings.data.otg_inst = pipe_ctx->stream_res.tg->inst;
+	else
+		cmd.pr_copy_settings.data.otg_inst = 0;
+
+	cmd.pr_copy_settings.data.dpphy_inst = link->link_enc->transmitter;
+
+	cmd.pr_copy_settings.data.line_time_in_ns = replay_context->line_time_in_ns;
+	cmd.pr_copy_settings.data.flags.bitfields.fec_enable_status = (link->fec_state == dc_link_fec_enabled);
+	cmd.pr_copy_settings.data.flags.bitfields.dsc_enable_status = (pipe_ctx->stream->timing.flags.DSC == 1);
+	cmd.pr_copy_settings.data.debug.u32All = link->replay_settings.config.debug_flags;
+
+	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+	return true;
+}
+
+bool edp_pr_update_state(struct dc_link *link, struct dmub_cmd_pr_update_state_data *update_state_data)
+{
+	struct dc  *dc = link->ctx->dc;
+	unsigned int panel_inst = 0;
+	union dmub_rb_cmd cmd;
+
+	if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst))
+		return false;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.pr_update_state.header.type = DMUB_CMD__PR;
+	cmd.pr_update_state.header.sub_type = DMUB_CMD__PR_UPDATE_STATE;
+	cmd.pr_update_state.header.payload_bytes = sizeof(struct dmub_cmd_pr_update_state_data);
+	cmd.pr_update_state.data.panel_inst = panel_inst;
+
+	memcpy(&cmd.pr_update_state.data, update_state_data, sizeof(struct dmub_cmd_pr_update_state_data));
+
+	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+	return true;
+}
+
+bool edp_pr_set_general_cmd(struct dc_link *link, struct dmub_cmd_pr_general_cmd_data *general_cmd_data)
+{
+	struct dc  *dc = link->ctx->dc;
+	unsigned int panel_inst = 0;
+	union dmub_rb_cmd cmd;
+
+	if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst))
+		return false;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.pr_general_cmd.header.type = DMUB_CMD__PR;
+	cmd.pr_general_cmd.header.sub_type = DMUB_CMD__PR_GENERAL_CMD;
+	cmd.pr_general_cmd.header.payload_bytes = sizeof(struct dmub_cmd_pr_general_cmd_data);
+	cmd.pr_general_cmd.data.panel_inst = panel_inst;
+
+	memcpy(&cmd.pr_general_cmd.data, general_cmd_data, sizeof(struct dmub_cmd_pr_general_cmd_data));
+
+	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+	return true;
+}
+
+bool edp_pr_get_state(const struct dc_link *link, uint64_t *state)
+{
+	const struct dc  *dc = link->ctx->dc;
+	unsigned int panel_inst = 0;
+	uint32_t retry_count = 0;
+	uint32_t replay_state = 0;
+
+	if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst))
+		return false;
+
+	do {
+		// Send gpint command and wait for ack
+		if (!dc_wake_and_execute_gpint(dc->ctx, DMUB_GPINT__GET_REPLAY_STATE, panel_inst,
+					       &replay_state, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY)) {
+			// Return invalid state when GPINT times out
+			replay_state = PR_STATE_INVALID;
+		}
+		/* Copy 32-bit result into 64-bit output */
+		*state = replay_state;
+	} while (++retry_count <= 1000 && *state == PR_STATE_INVALID);
+
+	// Assert if max retry hit
+	if (retry_count >= 1000 && *state == PR_STATE_INVALID) {
+		ASSERT(0);
+		/* To-do: Add retry fail log */
+	}
+
+	return true;
+}
+
 static struct abm *get_abm_from_stream_res(const struct dc_link *link)
 {
 	int i;
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.h
index dd79c7cd2828..fd63b5d0f948 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.h
@@ -29,7 +29,7 @@
 
 enum dp_panel_mode dp_get_panel_mode(struct dc_link *link);
 void dp_set_panel_mode(struct dc_link *link, enum dp_panel_mode panel_mode);
-bool set_default_brightness_aux(struct dc_link *link);
+bool set_default_brightness(struct dc_link *link);
 bool is_smartmux_suported(struct dc_link *link);
 void edp_panel_backlight_power_on(struct dc_link *link, bool wait_for_hpd);
 int edp_get_backlight_level(const struct dc_link *link);
@@ -75,6 +75,11 @@ void edp_add_delay_for_T9(struct dc_link *link);
 bool edp_receiver_ready_T9(struct dc_link *link);
 bool edp_receiver_ready_T7(struct dc_link *link);
 bool edp_power_alpm_dpcd_enable(struct dc_link *link, bool enable);
+bool edp_pr_enable(struct dc_link *link, bool enable);
+bool edp_pr_copy_settings(struct dc_link *link, struct replay_context *replay_context);
+bool edp_pr_update_state(struct dc_link *link, struct dmub_cmd_pr_update_state_data *update_state_data);
+bool edp_pr_set_general_cmd(struct dc_link *link, struct dmub_cmd_pr_general_cmd_data *general_cmd_data);
+bool edp_pr_get_state(const struct dc_link *link, uint64_t *state);
 void edp_set_panel_power(struct dc_link *link, bool powerOn);
 void edp_set_panel_assr(struct dc_link *link, struct pipe_ctx *pipe_ctx,
 		enum dp_panel_mode *panel_mode, bool enable);
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.c
index caa617883f62..29f3a03687b2 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.c
@@ -35,62 +35,75 @@
 
 bool link_get_hpd_state(struct dc_link *link)
 {
-	uint32_t state = 0;
-
-	dal_gpio_lock_pin(link->hpd_gpio);
-	dal_gpio_get_value(link->hpd_gpio, &state);
-	dal_gpio_unlock_pin(link->hpd_gpio);
-
-	return state;
+	if (link->link_enc)
+		return link->link_enc->funcs->get_hpd_state(link->link_enc);
+	else
+		return false;
 }
 
 void link_enable_hpd(const struct dc_link *link)
 {
-	struct link_encoder *encoder = link->link_enc;
-
-	if (encoder != NULL && encoder->funcs->enable_hpd != NULL)
-		encoder->funcs->enable_hpd(encoder);
+	if (link->link_enc)
+		link->link_enc->funcs->enable_hpd(link->link_enc);
 }
 
 void link_disable_hpd(const struct dc_link *link)
 {
-	struct link_encoder *encoder = link->link_enc;
-
-	if (encoder != NULL && encoder->funcs->enable_hpd != NULL)
-		encoder->funcs->disable_hpd(encoder);
+	if (link->link_enc)
+		link->link_enc->funcs->disable_hpd(link->link_enc);
 }
 
 void link_enable_hpd_filter(struct dc_link *link, bool enable)
 {
-	struct gpio *hpd;
-
 	if (enable) {
 		link->is_hpd_filter_disabled = false;
 		program_hpd_filter(link);
 	} else {
 		link->is_hpd_filter_disabled = true;
-		/* Obtain HPD handle */
-		hpd = link_get_hpd_gpio(link->ctx->dc_bios, link->link_id, link->ctx->gpio_service);
-
-		if (!hpd)
-			return;
-
-		/* Setup HPD filtering */
-		if (dal_gpio_open(hpd, GPIO_MODE_INTERRUPT) == GPIO_RESULT_OK) {
-			struct gpio_hpd_config config;
+		if (link->link_enc)
+			link->link_enc->funcs->program_hpd_filter(link->link_enc, 0, 0);
+	}
+}
 
-			config.delay_on_connect = 0;
-			config.delay_on_disconnect = 0;
+bool program_hpd_filter(const struct dc_link *link)
+{
+	int delay_on_connect_in_ms = 0;
+	int delay_on_disconnect_in_ms = 0;
 
-			dal_irq_setup_hpd_filter(hpd, &config);
+	if (link->is_hpd_filter_disabled || !link->link_enc) {
+		ASSERT(link->link_enc);
+		return false;
+	}
 
-			dal_gpio_close(hpd);
-		} else {
-			ASSERT_CRITICAL(false);
-		}
-		/* Release HPD handle */
-		dal_gpio_destroy_irq(&hpd);
+	/* Verify feature is supported */
+	switch (link->connector_signal) {
+	case SIGNAL_TYPE_DVI_SINGLE_LINK:
+	case SIGNAL_TYPE_DVI_DUAL_LINK:
+	case SIGNAL_TYPE_HDMI_TYPE_A:
+		/* Program hpd filter */
+		delay_on_connect_in_ms = 500;
+		delay_on_disconnect_in_ms = 100;
+		break;
+	case SIGNAL_TYPE_DISPLAY_PORT:
+	case SIGNAL_TYPE_DISPLAY_PORT_MST:
+		/* Program hpd filter to allow DP signal to settle */
+		/* 500:	not able to detect MST <-> SST switch as HPD is low for
+		 * only 100ms on DELL U2413
+		 * 0: some passive dongle still show aux mode instead of i2c
+		 * 20-50: not enough to hide bouncing HPD with passive dongle.
+		 * also see intermittent i2c read issues.
+		 */
+		delay_on_connect_in_ms = 80;
+		delay_on_disconnect_in_ms = 0;
+		break;
+	case SIGNAL_TYPE_LVDS:
+	case SIGNAL_TYPE_EDP:
+	default:
+		/* Don't program hpd filter */
+		return false;
 	}
+
+	return link->link_enc->funcs->program_hpd_filter(link->link_enc, delay_on_connect_in_ms, delay_on_disconnect_in_ms);
 }
 
 struct gpio *link_get_hpd_gpio(struct dc_bios *dcb,
@@ -108,7 +121,6 @@ struct gpio *link_get_hpd_gpio(struct dc_bios *dcb,
 		hpd_info.hpd_int_gpio_uid, &pin_info);
 
 	if (bp_result != BP_RESULT_OK) {
-		ASSERT(bp_result == BP_RESULT_NORECORD);
 		return NULL;
 	}
 
@@ -117,21 +129,6 @@ struct gpio *link_get_hpd_gpio(struct dc_bios *dcb,
 					   pin_info.mask);
 }
 
-bool query_hpd_status(struct dc_link *link, uint32_t *is_hpd_high)
-{
-	struct gpio *hpd_pin = link_get_hpd_gpio(
-			link->ctx->dc_bios, link->link_id,
-			link->ctx->gpio_service);
-	if (!hpd_pin)
-		return false;
-
-	dal_gpio_open(hpd_pin, GPIO_MODE_INTERRUPT);
-	dal_gpio_get_value(hpd_pin, is_hpd_high);
-	dal_gpio_close(hpd_pin);
-	dal_gpio_destroy_irq(&hpd_pin);
-	return true;
-}
-
 enum hpd_source_id get_hpd_line(struct dc_link *link)
 {
 	struct gpio *hpd;
@@ -172,69 +169,3 @@ enum hpd_source_id get_hpd_line(struct dc_link *link)
 
 	return hpd_id;
 }
-
-bool program_hpd_filter(const struct dc_link *link)
-{
-	bool result = false;
-	struct gpio *hpd;
-	int delay_on_connect_in_ms = 0;
-	int delay_on_disconnect_in_ms = 0;
-
-	if (link->is_hpd_filter_disabled)
-		return false;
-	/* Verify feature is supported */
-	switch (link->connector_signal) {
-	case SIGNAL_TYPE_DVI_SINGLE_LINK:
-	case SIGNAL_TYPE_DVI_DUAL_LINK:
-	case SIGNAL_TYPE_HDMI_TYPE_A:
-		/* Program hpd filter */
-		delay_on_connect_in_ms = 500;
-		delay_on_disconnect_in_ms = 100;
-		break;
-	case SIGNAL_TYPE_DISPLAY_PORT:
-	case SIGNAL_TYPE_DISPLAY_PORT_MST:
-		/* Program hpd filter to allow DP signal to settle */
-		/* 500:	not able to detect MST <-> SST switch as HPD is low for
-		 * only 100ms on DELL U2413
-		 * 0: some passive dongle still show aux mode instead of i2c
-		 * 20-50: not enough to hide bouncing HPD with passive dongle.
-		 * also see intermittent i2c read issues.
-		 */
-		delay_on_connect_in_ms = 80;
-		delay_on_disconnect_in_ms = 0;
-		break;
-	case SIGNAL_TYPE_LVDS:
-	case SIGNAL_TYPE_EDP:
-	default:
-		/* Don't program hpd filter */
-		return false;
-	}
-
-	/* Obtain HPD handle */
-	hpd = link_get_hpd_gpio(link->ctx->dc_bios, link->link_id,
-			   link->ctx->gpio_service);
-
-	if (!hpd)
-		return result;
-
-	/* Setup HPD filtering */
-	if (dal_gpio_open(hpd, GPIO_MODE_INTERRUPT) == GPIO_RESULT_OK) {
-		struct gpio_hpd_config config;
-
-		config.delay_on_connect = delay_on_connect_in_ms;
-		config.delay_on_disconnect = delay_on_disconnect_in_ms;
-
-		dal_irq_setup_hpd_filter(hpd, &config);
-
-		dal_gpio_close(hpd);
-
-		result = true;
-	} else {
-		ASSERT_CRITICAL(false);
-	}
-
-	/* Release HPD handle */
-	dal_gpio_destroy_irq(&hpd);
-
-	return result;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.h
index af529328ba17..b4e449de960e 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_hpd.h
@@ -43,7 +43,6 @@ bool program_hpd_filter(const struct dc_link *link);
  * Returns true if HPD high.
  */
 bool dpia_query_hpd_status(struct dc_link *link);
-bool query_hpd_status(struct dc_link *link, uint32_t *is_hpd_high);
 bool link_get_hpd_state(struct dc_link *link);
 struct gpio *link_get_hpd_gpio(struct dc_bios *dcb,
 		struct graphics_object_id link_id,
diff --git a/drivers/gpu/drm/amd/display/dc/os_types.h b/drivers/gpu/drm/amd/display/dc/os_types.h
index 782316348941..6af831710489 100644
--- a/drivers/gpu/drm/amd/display/dc/os_types.h
+++ b/drivers/gpu/drm/amd/display/dc/os_types.h
@@ -55,8 +55,6 @@
 
 #if defined(CONFIG_DRM_AMD_DC_FP)
 #include "amdgpu_dm/dc_fpu.h"
-#define DC_FP_START() dc_fpu_begin(__func__, __LINE__)
-#define DC_FP_END() dc_fpu_end(__func__, __LINE__)
 #endif /* CONFIG_DRM_AMD_DC_FP */
 
 /*
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c
index 6679c1a14f2f..8d10aac9c510 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c
@@ -1660,8 +1660,8 @@ bool dcn20_validate_dsc(struct dc *dc, struct dc_state *new_ctx)
 		if (pipe_ctx->top_pipe || pipe_ctx->prev_odm_pipe || !stream || !stream->timing.flags.DSC)
 			continue;
 
-		dsc_cfg.pic_width = (stream->timing.h_addressable + stream->timing.h_border_left
-				+ stream->timing.h_border_right) / opp_cnt;
+		dsc_cfg.pic_width = (stream->timing.h_addressable + pipe_ctx->dsc_padding_params.dsc_hactive_padding
+				+ stream->timing.h_border_left + stream->timing.h_border_right) / opp_cnt;
 		dsc_cfg.pic_height = stream->timing.v_addressable + stream->timing.v_border_top
 				+ stream->timing.v_border_bottom;
 		dsc_cfg.pixel_encoding = stream->timing.pixel_encoding;
@@ -1669,7 +1669,7 @@ bool dcn20_validate_dsc(struct dc *dc, struct dc_state *new_ctx)
 		dsc_cfg.is_odm = pipe_ctx->next_odm_pipe ? true : false;
 		dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg;
 		dsc_cfg.dc_dsc_cfg.num_slices_h /= opp_cnt;
-		dsc_cfg.dsc_padding = pipe_ctx->dsc_padding_params.dsc_hactive_padding;
+		dsc_cfg.dsc_padding = 0;
 
 		if (!pipe_ctx->stream_res.dsc->funcs->dsc_validate_stream(pipe_ctx->stream_res.dsc, &dsc_cfg))
 			return false;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.h
index 99f0432288b4..91be493e0bb6 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.h
@@ -317,7 +317,10 @@ unsigned int dcn32_get_max_hw_cursor_size(const struct dc *dc,
   AUX_REG_LIST_RI(id), SRI_ARR(AUX_DPHY_TX_CONTROL, DP_AUX, id)
 
 /* HDP */
-#define HPD_REG_LIST_RI(id) SRI_ARR(DC_HPD_CONTROL, HPD, id)
+#define HPD_REG_LIST_RI(id)                                                  \
+	(SRI_ARR(DC_HPD_CONTROL, HPD, id),                                   \
+		SRI_ARR(DC_HPD_INT_STATUS, HPD, id),                         \
+		SRI_ARR(DC_HPD_TOGGLE_FILT_CNTL, HPD, id))
 
 /* Link encoder */
 #define LE_DCN3_REG_LIST_RI(id)                                                \
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
index ef69898d2cc5..45454a097264 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
@@ -203,12 +203,12 @@ enum dcn35_clk_src_array_id {
 	NBIO_BASE_INNER(seg)
 
 #define NBIO_SR(reg_name)\
-	REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX2_ ## reg_name ## _BASE_IDX) + \
-				regBIF_BX2_ ## reg_name
+	REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX1_ ## reg_name ## _BASE_IDX) + \
+				regBIF_BX1_ ## reg_name
 
 #define NBIO_SR_ARR(reg_name, id)\
-	REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX2_ ## reg_name ## _BASE_IDX) + \
-		regBIF_BX2_ ## reg_name
+	REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX1_ ## reg_name ## _BASE_IDX) + \
+		regBIF_BX1_ ## reg_name
 
 #define bios_regs_init() \
 		( \
@@ -788,7 +788,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 };
 
 static const struct dc_check_config config_defaults = {
-	.enable_legacy_fast_update = true,
+	.enable_legacy_fast_update = false,
 };
 
 static const struct dc_panel_config panel_config_defaults = {
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
index f3c614c4490c..e3c587165807 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
@@ -183,12 +183,12 @@ enum dcn351_clk_src_array_id {
 	NBIO_BASE_INNER(seg)
 
 #define NBIO_SR(reg_name)\
-	REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX2_ ## reg_name ## _BASE_IDX) + \
-				regBIF_BX2_ ## reg_name
+	REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX1_ ## reg_name ## _BASE_IDX) + \
+				regBIF_BX1_ ## reg_name
 
 #define NBIO_SR_ARR(reg_name, id)\
-	REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX2_ ## reg_name ## _BASE_IDX) + \
-		regBIF_BX2_ ## reg_name
+	REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX1_ ## reg_name ## _BASE_IDX) + \
+		regBIF_BX1_ ## reg_name
 
 #define bios_regs_init() \
 		( \
@@ -768,7 +768,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 };
 
 static const struct dc_check_config config_defaults = {
-	.enable_legacy_fast_update = true,
+	.enable_legacy_fast_update = false,
 };
 
 static const struct dc_panel_config panel_config_defaults = {
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.h
index e1fa2e80a15a..406246a9867e 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.h
@@ -394,6 +394,7 @@ void dcn401_prepare_mcache_programming(struct dc *dc, struct dc_state *context);
 	SRI_ARR(DSCL_SC_MATRIX_C0C1, DSCL, id),                                  \
 	SRI_ARR(DSCL_SC_MATRIX_C2C3, DSCL, id),                                  \
 	SRI_ARR(ISHARP_MODE, DSCL, id),                                          \
+	SRI_ARR(ISHARP_DELTA_LUT_MEM_PWR_CTRL, DSCL, id),                                          \
 	SRI_ARR(ISHARP_NOISEDET_THRESHOLD, DSCL, id),                            \
 	SRI_ARR(ISHARP_NOISE_GAIN_PWL, DSCL, id),                                \
 	SRI_ARR(ISHARP_LBA_PWL_SEG0, DSCL, id),                                  \
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl.c b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl.c
index 7a839984dbc0..f506ab70a307 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl.c
+++ b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl.c
@@ -7,7 +7,7 @@
 #include "dc_spl_isharp_filters.h"
 #include "spl_debug.h"
 
-#define IDENTITY_RATIO(ratio) (spl_fixpt_u3d19(ratio) == (1 << 19))
+#define IDENTITY_RATIO(ratio) (SPL_NAMESPACE(spl_fixpt_u3d19(ratio)) == (1 << 19))
 #define MIN_VIEWPORT_SIZE 12
 
 static bool spl_is_yuv420(enum spl_pixel_format format)
@@ -161,22 +161,24 @@ static struct spl_rect calculate_plane_rec_in_timing_active(
 	struct spl_fixed31_32 temp;
 
 
-	temp = spl_fixpt_from_fraction(rec_in->x * (long long)stream_dst->width,
-			stream_src->width);
+	temp = SPL_NAMESPACE(spl_fixpt_from_fraction(
+			rec_in->x * (long long)stream_dst->width,
+			stream_src->width));
 	rec_out.x = stream_dst->x + spl_fixpt_round(temp);
 
-	temp = spl_fixpt_from_fraction(
+	temp = SPL_NAMESPACE(spl_fixpt_from_fraction(
 			(rec_in->x + rec_in->width) * (long long)stream_dst->width,
-			stream_src->width);
+			stream_src->width));
 	rec_out.width = stream_dst->x + spl_fixpt_round(temp) - rec_out.x;
 
-	temp = spl_fixpt_from_fraction(rec_in->y * (long long)stream_dst->height,
-			stream_src->height);
+	temp = SPL_NAMESPACE(spl_fixpt_from_fraction(
+			rec_in->y * (long long)stream_dst->height,
+			stream_src->height));
 	rec_out.y = stream_dst->y + spl_fixpt_round(temp);
 
-	temp = spl_fixpt_from_fraction(
+	temp = SPL_NAMESPACE(spl_fixpt_from_fraction(
 			(rec_in->y + rec_in->height) * (long long)stream_dst->height,
-			stream_src->height);
+			stream_src->height));
 	rec_out.height = stream_dst->y + spl_fixpt_round(temp) - rec_out.y;
 
 	return rec_out;
@@ -442,12 +444,12 @@ static void spl_calculate_scaling_ratios(struct spl_in *spl_in,
 		spl_in->basic_in.rotation == SPL_ROTATION_ANGLE_270)
 		spl_swap(surf_src.height, surf_src.width);
 
-	spl_scratch->scl_data.ratios.horz = spl_fixpt_from_fraction(
+	spl_scratch->scl_data.ratios.horz = SPL_NAMESPACE(spl_fixpt_from_fraction(
 					surf_src.width,
-					spl_in->basic_in.dst_rect.width);
-	spl_scratch->scl_data.ratios.vert = spl_fixpt_from_fraction(
+					spl_in->basic_in.dst_rect.width));
+	spl_scratch->scl_data.ratios.vert = SPL_NAMESPACE(spl_fixpt_from_fraction(
 					surf_src.height,
-					spl_in->basic_in.dst_rect.height);
+					spl_in->basic_in.dst_rect.height));
 
 	if (spl_in->basic_out.view_format == SPL_VIEW_3D_SIDE_BY_SIDE)
 		spl_scratch->scl_data.ratios.horz.value *= 2;
@@ -480,14 +482,14 @@ static void spl_calculate_scaling_ratios(struct spl_in *spl_in,
 	 * that is output/input.  Currently we calculate input/output
 	 * Store 1/ratio in recip_ratio for those lookups
 	 */
-	spl_scratch->scl_data.recip_ratios.horz = spl_fixpt_recip(
-			spl_scratch->scl_data.ratios.horz);
-	spl_scratch->scl_data.recip_ratios.vert = spl_fixpt_recip(
-			spl_scratch->scl_data.ratios.vert);
-	spl_scratch->scl_data.recip_ratios.horz_c = spl_fixpt_recip(
-			spl_scratch->scl_data.ratios.horz_c);
-	spl_scratch->scl_data.recip_ratios.vert_c = spl_fixpt_recip(
-			spl_scratch->scl_data.ratios.vert_c);
+	spl_scratch->scl_data.recip_ratios.horz = SPL_NAMESPACE(spl_fixpt_recip(
+			spl_scratch->scl_data.ratios.horz));
+	spl_scratch->scl_data.recip_ratios.vert = SPL_NAMESPACE(spl_fixpt_recip(
+			spl_scratch->scl_data.ratios.vert));
+	spl_scratch->scl_data.recip_ratios.horz_c = SPL_NAMESPACE(spl_fixpt_recip(
+			spl_scratch->scl_data.ratios.horz_c));
+	spl_scratch->scl_data.recip_ratios.vert_c = SPL_NAMESPACE(spl_fixpt_recip(
+			spl_scratch->scl_data.ratios.vert_c));
 }
 
 /* Calculate Viewport size */
@@ -646,11 +648,11 @@ static void spl_calculate_inits_and_viewports(struct spl_in *spl_in,
 
 		switch (spl_in->basic_in.cositing) {
 		case CHROMA_COSITING_TOPLEFT:
-			init_adj_h = spl_fixpt_from_fraction(h_sign, 4);
-			init_adj_v = spl_fixpt_from_fraction(v_sign, 4);
+			init_adj_h = SPL_NAMESPACE(spl_fixpt_from_fraction(h_sign, 4));
+			init_adj_v = SPL_NAMESPACE(spl_fixpt_from_fraction(v_sign, 4));
 			break;
 		case CHROMA_COSITING_LEFT:
-			init_adj_h = spl_fixpt_from_fraction(h_sign, 4);
+			init_adj_h = SPL_NAMESPACE(spl_fixpt_from_fraction(h_sign, 4));
 			init_adj_v = spl_fixpt_zero;
 			break;
 		case CHROMA_COSITING_NONE:
@@ -939,16 +941,16 @@ static void spl_get_taps_non_adaptive_scaler(
 	 * Max downscale supported is 6.0x.  Add ASSERT to catch if go beyond that
 	 */
 	check_max_downscale = spl_fixpt_le(spl_scratch->scl_data.ratios.horz,
-		spl_fixpt_from_fraction(6, 1));
+		SPL_NAMESPACE(spl_fixpt_from_fraction(6, 1)));
 	SPL_ASSERT(check_max_downscale);
 	check_max_downscale = spl_fixpt_le(spl_scratch->scl_data.ratios.vert,
-		spl_fixpt_from_fraction(6, 1));
+		SPL_NAMESPACE(spl_fixpt_from_fraction(6, 1)));
 	SPL_ASSERT(check_max_downscale);
 	check_max_downscale = spl_fixpt_le(spl_scratch->scl_data.ratios.horz_c,
-		spl_fixpt_from_fraction(6, 1));
+		SPL_NAMESPACE(spl_fixpt_from_fraction(6, 1)));
 	SPL_ASSERT(check_max_downscale);
 	check_max_downscale = spl_fixpt_le(spl_scratch->scl_data.ratios.vert_c,
-		spl_fixpt_from_fraction(6, 1));
+		SPL_NAMESPACE(spl_fixpt_from_fraction(6, 1)));
 	SPL_ASSERT(check_max_downscale);
 
 
@@ -1194,35 +1196,39 @@ static void spl_set_manual_ratio_init_data(struct dscl_prog_data *dscl_prog_data
 {
 	struct spl_fixed31_32 bot;
 
-	dscl_prog_data->ratios.h_scale_ratio = spl_fixpt_u3d19(scl_data->ratios.horz) << 5;
-	dscl_prog_data->ratios.v_scale_ratio = spl_fixpt_u3d19(scl_data->ratios.vert) << 5;
-	dscl_prog_data->ratios.h_scale_ratio_c = spl_fixpt_u3d19(scl_data->ratios.horz_c) << 5;
-	dscl_prog_data->ratios.v_scale_ratio_c = spl_fixpt_u3d19(scl_data->ratios.vert_c) << 5;
+	dscl_prog_data->ratios.h_scale_ratio = SPL_NAMESPACE(spl_fixpt_u3d19(
+			scl_data->ratios.horz)) << 5;
+	dscl_prog_data->ratios.v_scale_ratio = SPL_NAMESPACE(spl_fixpt_u3d19(
+			scl_data->ratios.vert)) << 5;
+	dscl_prog_data->ratios.h_scale_ratio_c = SPL_NAMESPACE(spl_fixpt_u3d19(
+			scl_data->ratios.horz_c)) << 5;
+	dscl_prog_data->ratios.v_scale_ratio_c = SPL_NAMESPACE(spl_fixpt_u3d19(
+			scl_data->ratios.vert_c)) << 5;
 	/*
 	 * 0.24 format for fraction, first five bits zeroed
 	 */
 	dscl_prog_data->init.h_filter_init_frac =
-			spl_fixpt_u0d19(scl_data->inits.h) << 5;
+			SPL_NAMESPACE(spl_fixpt_u0d19(scl_data->inits.h)) << 5;
 	dscl_prog_data->init.h_filter_init_int =
 			spl_fixpt_floor(scl_data->inits.h);
 	dscl_prog_data->init.h_filter_init_frac_c =
-			spl_fixpt_u0d19(scl_data->inits.h_c) << 5;
+			SPL_NAMESPACE(spl_fixpt_u0d19(scl_data->inits.h_c)) << 5;
 	dscl_prog_data->init.h_filter_init_int_c =
 			spl_fixpt_floor(scl_data->inits.h_c);
 	dscl_prog_data->init.v_filter_init_frac =
-			spl_fixpt_u0d19(scl_data->inits.v) << 5;
+			SPL_NAMESPACE(spl_fixpt_u0d19(scl_data->inits.v)) << 5;
 	dscl_prog_data->init.v_filter_init_int =
 			spl_fixpt_floor(scl_data->inits.v);
 	dscl_prog_data->init.v_filter_init_frac_c =
-			spl_fixpt_u0d19(scl_data->inits.v_c) << 5;
+			SPL_NAMESPACE(spl_fixpt_u0d19(scl_data->inits.v_c)) << 5;
 	dscl_prog_data->init.v_filter_init_int_c =
 			spl_fixpt_floor(scl_data->inits.v_c);
 
 	bot = spl_fixpt_add(scl_data->inits.v, scl_data->ratios.vert);
-	dscl_prog_data->init.v_filter_init_bot_frac = spl_fixpt_u0d19(bot) << 5;
+	dscl_prog_data->init.v_filter_init_bot_frac = SPL_NAMESPACE(spl_fixpt_u0d19(bot)) << 5;
 	dscl_prog_data->init.v_filter_init_bot_int = spl_fixpt_floor(bot);
 	bot = spl_fixpt_add(scl_data->inits.v_c, scl_data->ratios.vert_c);
-	dscl_prog_data->init.v_filter_init_bot_frac_c = spl_fixpt_u0d19(bot) << 5;
+	dscl_prog_data->init.v_filter_init_bot_frac_c = SPL_NAMESPACE(spl_fixpt_u0d19(bot)) << 5;
 	dscl_prog_data->init.v_filter_init_bot_int_c = spl_fixpt_floor(bot);
 }
 
@@ -1270,7 +1276,7 @@ static void spl_set_dscl_prog_data(struct spl_in *spl_in, struct spl_scratch *sp
 	// Set viewport_c
 	dscl_prog_data->viewport_c = spl_scratch->scl_data.viewport_c;
 	// Set filters data
-	spl_set_filters_data(dscl_prog_data, data, enable_easf_v, enable_easf_h);
+	SPL_NAMESPACE(spl_set_filters_data(dscl_prog_data, data, enable_easf_v, enable_easf_h));
 }
 
 /* Calculate C0-C3 coefficients based on HDR_mult */
@@ -1286,28 +1292,31 @@ static void spl_calculate_c0_c3_hdr(struct dscl_prog_data *dscl_prog_data, uint3
 	else
 		hdr_multx100_int = 100; /* default for 80 nits otherwise */
 
-	hdr_mult = spl_fixpt_from_fraction((long long)hdr_multx100_int, 100LL);
-	c0_mult = spl_fixpt_from_fraction(2126LL, 10000LL);
-	c1_mult = spl_fixpt_from_fraction(7152LL, 10000LL);
-	c2_mult = spl_fixpt_from_fraction(722LL, 10000LL);
+	hdr_mult = SPL_NAMESPACE(spl_fixpt_from_fraction((long long)hdr_multx100_int, 100LL));
+	c0_mult = SPL_NAMESPACE(spl_fixpt_from_fraction(2126LL, 10000LL));
+	c1_mult = SPL_NAMESPACE(spl_fixpt_from_fraction(7152LL, 10000LL));
+	c2_mult = SPL_NAMESPACE(spl_fixpt_from_fraction(722LL, 10000LL));
 
-	c0_calc = spl_fixpt_mul(hdr_mult, spl_fixpt_mul(c0_mult, spl_fixpt_from_fraction(
-		16384LL, 125LL)));
-	c1_calc = spl_fixpt_mul(hdr_mult, spl_fixpt_mul(c1_mult, spl_fixpt_from_fraction(
-		16384LL, 125LL)));
-	c2_calc = spl_fixpt_mul(hdr_mult, spl_fixpt_mul(c2_mult, spl_fixpt_from_fraction(
-		16384LL, 125LL)));
+	c0_calc = SPL_NAMESPACE(spl_fixpt_mul(hdr_mult, SPL_NAMESPACE(spl_fixpt_mul(c0_mult,
+		SPL_NAMESPACE(spl_fixpt_from_fraction(16384LL, 125LL))))));
+	c1_calc = SPL_NAMESPACE(spl_fixpt_mul(hdr_mult, SPL_NAMESPACE(spl_fixpt_mul(c1_mult,
+		SPL_NAMESPACE(spl_fixpt_from_fraction(16384LL, 125LL))))));
+	c2_calc = SPL_NAMESPACE(spl_fixpt_mul(hdr_mult, SPL_NAMESPACE(spl_fixpt_mul(c2_mult,
+		SPL_NAMESPACE(spl_fixpt_from_fraction(16384LL, 125LL))))));
 
 	fmt.exponenta_bits = 5;
 	fmt.mantissa_bits = 10;
 	fmt.sign = true;
 
 	// fp1.5.10, C0 coefficient (LN_rec709:  HDR_MULT * 0.212600 * 2^14/125)
-	spl_convert_to_custom_float_format(c0_calc, &fmt, &dscl_prog_data->easf_matrix_c0);
+	SPL_NAMESPACE(spl_convert_to_custom_float_format(c0_calc, &fmt,
+		&dscl_prog_data->easf_matrix_c0));
 	// fp1.5.10, C1 coefficient (LN_rec709:  HDR_MULT * 0.715200 * 2^14/125)
-	spl_convert_to_custom_float_format(c1_calc, &fmt, &dscl_prog_data->easf_matrix_c1);
+	SPL_NAMESPACE(spl_convert_to_custom_float_format(c1_calc, &fmt,
+		&dscl_prog_data->easf_matrix_c1));
 	// fp1.5.10, C2 coefficient (LN_rec709:  HDR_MULT * 0.072200 * 2^14/125)
-	spl_convert_to_custom_float_format(c2_calc, &fmt, &dscl_prog_data->easf_matrix_c2);
+	SPL_NAMESPACE(spl_convert_to_custom_float_format(c2_calc, &fmt,
+		&dscl_prog_data->easf_matrix_c2));
 	dscl_prog_data->easf_matrix_c3 = 0x0; // fp1.5.10, C3 coefficient
 }
 
@@ -1325,48 +1334,48 @@ static void spl_set_easf_data(struct spl_scratch *spl_scratch, struct spl_out *s
 		dscl_prog_data->easf_v_bf1_en = 1;	// 1-bit, BF1 calculation enable, 0=disable, 1=enable
 		dscl_prog_data->easf_v_bf2_mode = 0xF;	// 4-bit, BF2 calculation mode
 		/* 2-bit, BF3 chroma mode correction calculation mode */
-		dscl_prog_data->easf_v_bf3_mode = spl_get_v_bf3_mode(
-			spl_scratch->scl_data.recip_ratios.vert);
+		dscl_prog_data->easf_v_bf3_mode = SPL_NAMESPACE(spl_get_v_bf3_mode(
+			spl_scratch->scl_data.recip_ratios.vert));
 		/* FP1.5.10 [ minCoef ]*/
 		dscl_prog_data->easf_v_ringest_3tap_dntilt_uptilt =
-			spl_get_3tap_dntilt_uptilt_offset(spl_scratch->scl_data.taps.v_taps,
-				spl_scratch->scl_data.recip_ratios.vert);
+			SPL_NAMESPACE(spl_get_3tap_dntilt_uptilt_offset(spl_scratch->scl_data.taps.v_taps,
+				spl_scratch->scl_data.recip_ratios.vert));
 		/* FP1.5.10 [ upTiltMaxVal ]*/
 		dscl_prog_data->easf_v_ringest_3tap_uptilt_max =
-			spl_get_3tap_uptilt_maxval(spl_scratch->scl_data.taps.v_taps,
-				spl_scratch->scl_data.recip_ratios.vert);
+			SPL_NAMESPACE(spl_get_3tap_uptilt_maxval(spl_scratch->scl_data.taps.v_taps,
+				spl_scratch->scl_data.recip_ratios.vert));
 		/* FP1.5.10 [ dnTiltSlope ]*/
 		dscl_prog_data->easf_v_ringest_3tap_dntilt_slope =
-			spl_get_3tap_dntilt_slope(spl_scratch->scl_data.taps.v_taps,
-				spl_scratch->scl_data.recip_ratios.vert);
+			SPL_NAMESPACE(spl_get_3tap_dntilt_slope(spl_scratch->scl_data.taps.v_taps,
+				spl_scratch->scl_data.recip_ratios.vert));
 		/* FP1.5.10 [ upTilt1Slope ]*/
 		dscl_prog_data->easf_v_ringest_3tap_uptilt1_slope =
-			spl_get_3tap_uptilt1_slope(spl_scratch->scl_data.taps.v_taps,
-				spl_scratch->scl_data.recip_ratios.vert);
+			SPL_NAMESPACE(spl_get_3tap_uptilt1_slope(spl_scratch->scl_data.taps.v_taps,
+				spl_scratch->scl_data.recip_ratios.vert));
 		/* FP1.5.10 [ upTilt2Slope ]*/
 		dscl_prog_data->easf_v_ringest_3tap_uptilt2_slope =
-			spl_get_3tap_uptilt2_slope(spl_scratch->scl_data.taps.v_taps,
-				spl_scratch->scl_data.recip_ratios.vert);
+			SPL_NAMESPACE(spl_get_3tap_uptilt2_slope(spl_scratch->scl_data.taps.v_taps,
+				spl_scratch->scl_data.recip_ratios.vert));
 		/* FP1.5.10 [ upTilt2Offset ]*/
 		dscl_prog_data->easf_v_ringest_3tap_uptilt2_offset =
-			spl_get_3tap_uptilt2_offset(spl_scratch->scl_data.taps.v_taps,
-				spl_scratch->scl_data.recip_ratios.vert);
+			SPL_NAMESPACE(spl_get_3tap_uptilt2_offset(spl_scratch->scl_data.taps.v_taps,
+				spl_scratch->scl_data.recip_ratios.vert));
 		/* FP1.5.10; (2.0) Ring reducer gain for 4 or 6-tap mode [H_REDUCER_GAIN4] */
 		dscl_prog_data->easf_v_ringest_eventap_reduceg1 =
-			spl_get_reducer_gain4(spl_scratch->scl_data.taps.v_taps,
-				spl_scratch->scl_data.recip_ratios.vert);
+			SPL_NAMESPACE(spl_get_reducer_gain4(spl_scratch->scl_data.taps.v_taps,
+				spl_scratch->scl_data.recip_ratios.vert));
 		/* FP1.5.10; (2.5) Ring reducer gain for 6-tap mode [V_REDUCER_GAIN6] */
 		dscl_prog_data->easf_v_ringest_eventap_reduceg2 =
-			spl_get_reducer_gain6(spl_scratch->scl_data.taps.v_taps,
-				spl_scratch->scl_data.recip_ratios.vert);
+			SPL_NAMESPACE(spl_get_reducer_gain6(spl_scratch->scl_data.taps.v_taps,
+				spl_scratch->scl_data.recip_ratios.vert));
 		/* FP1.5.10; (-0.135742) Ring gain for 6-tap set to -139/1024 */
 		dscl_prog_data->easf_v_ringest_eventap_gain1 =
-			spl_get_gainRing4(spl_scratch->scl_data.taps.v_taps,
-				spl_scratch->scl_data.recip_ratios.vert);
+			SPL_NAMESPACE(spl_get_gainRing4(spl_scratch->scl_data.taps.v_taps,
+				spl_scratch->scl_data.recip_ratios.vert));
 		/* FP1.5.10; (-0.024414) Ring gain for 6-tap set to -25/1024 */
 		dscl_prog_data->easf_v_ringest_eventap_gain2 =
-			spl_get_gainRing6(spl_scratch->scl_data.taps.v_taps,
-				spl_scratch->scl_data.recip_ratios.vert);
+			SPL_NAMESPACE(spl_get_gainRing6(spl_scratch->scl_data.taps.v_taps,
+				spl_scratch->scl_data.recip_ratios.vert));
 		dscl_prog_data->easf_v_bf_maxa = 63; //Vertical Max BF value A in U0.6 format.Selected if V_FCNTL == 0
 		dscl_prog_data->easf_v_bf_maxb = 63; //Vertical Max BF value A in U0.6 format.Selected if V_FCNTL == 1
 		dscl_prog_data->easf_v_bf_mina = 0;	//Vertical Min BF value A in U0.6 format.Selected if V_FCNTL == 0
@@ -1491,24 +1500,24 @@ static void spl_set_easf_data(struct spl_scratch *spl_scratch, struct spl_out *s
 		dscl_prog_data->easf_h_bf2_mode =
 			0xF;	// 4-bit, BF2 calculation mode
 		/* 2-bit, BF3 chroma mode correction calculation mode */
-		dscl_prog_data->easf_h_bf3_mode = spl_get_h_bf3_mode(
-			spl_scratch->scl_data.recip_ratios.horz);
+		dscl_prog_data->easf_h_bf3_mode = SPL_NAMESPACE(spl_get_h_bf3_mode(
+			spl_scratch->scl_data.recip_ratios.horz));
 		/* FP1.5.10; (2.0) Ring reducer gain for 4 or 6-tap mode [H_REDUCER_GAIN4] */
 		dscl_prog_data->easf_h_ringest_eventap_reduceg1 =
-			spl_get_reducer_gain4(spl_scratch->scl_data.taps.h_taps,
-				spl_scratch->scl_data.recip_ratios.horz);
+			SPL_NAMESPACE(spl_get_reducer_gain4(spl_scratch->scl_data.taps.h_taps,
+				spl_scratch->scl_data.recip_ratios.horz));
 		/* FP1.5.10; (2.5) Ring reducer gain for 6-tap mode [V_REDUCER_GAIN6] */
 		dscl_prog_data->easf_h_ringest_eventap_reduceg2 =
-			spl_get_reducer_gain6(spl_scratch->scl_data.taps.h_taps,
-				spl_scratch->scl_data.recip_ratios.horz);
+			SPL_NAMESPACE(spl_get_reducer_gain6(spl_scratch->scl_data.taps.h_taps,
+				spl_scratch->scl_data.recip_ratios.horz));
 		/* FP1.5.10; (-0.135742) Ring gain for 6-tap set to -139/1024 */
 		dscl_prog_data->easf_h_ringest_eventap_gain1 =
-			spl_get_gainRing4(spl_scratch->scl_data.taps.h_taps,
-				spl_scratch->scl_data.recip_ratios.horz);
+			SPL_NAMESPACE(spl_get_gainRing4(spl_scratch->scl_data.taps.h_taps,
+				spl_scratch->scl_data.recip_ratios.horz));
 		/* FP1.5.10; (-0.024414) Ring gain for 6-tap set to -25/1024 */
 		dscl_prog_data->easf_h_ringest_eventap_gain2 =
-			spl_get_gainRing6(spl_scratch->scl_data.taps.h_taps,
-				spl_scratch->scl_data.recip_ratios.horz);
+			SPL_NAMESPACE(spl_get_gainRing6(spl_scratch->scl_data.taps.h_taps,
+				spl_scratch->scl_data.recip_ratios.horz));
 		dscl_prog_data->easf_h_bf_maxa = 63; //Horz Max BF value A in U0.6 format.Selected if H_FCNTL==0
 		dscl_prog_data->easf_h_bf_maxb = 63; //Horz Max BF value B in U0.6 format.Selected if H_FCNTL==1
 		dscl_prog_data->easf_h_bf_mina = 0;	//Horz Min BF value B in U0.6 format.Selected if H_FCNTL==0
@@ -1689,9 +1698,9 @@ static void spl_set_isharp_data(struct dscl_prog_data *dscl_prog_data,
 		return;
 	}
 
-	spl_build_isharp_1dlut_from_reference_curve(ratio, setup, adp_sharpness,
-		scale_to_sharpness_policy);
-	memcpy(dscl_prog_data->isharp_delta, spl_get_pregen_filter_isharp_1D_lut(setup),
+	SPL_NAMESPACE(spl_build_isharp_1dlut_from_reference_curve(ratio, setup, adp_sharpness,
+		scale_to_sharpness_policy));
+	memcpy(dscl_prog_data->isharp_delta, SPL_NAMESPACE(spl_get_pregen_filter_isharp_1D_lut(setup)),
 		sizeof(uint32_t) * ISHARP_LUT_TABLE_SIZE);
 	dscl_prog_data->sharpness_level = adp_sharpness.sharpness_level;
 
@@ -1810,7 +1819,7 @@ static void spl_set_isharp_data(struct dscl_prog_data *dscl_prog_data,
 	}
 
 	// Set the values as per lookup table
-	spl_set_blur_scale_data(dscl_prog_data, data);
+	SPL_NAMESPACE(spl_set_blur_scale_data(dscl_prog_data, data));
 }
 
 /* Calculate recout, scaling ratio, and viewport, then get optimal number of taps */
@@ -1922,4 +1931,3 @@ bool SPL_NAMESPACE(spl_get_number_of_taps(struct spl_in *spl_in, struct spl_out
 	spl_set_taps_data(dscl_prog_data, data);
 	return res;
 }
-
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl.h b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl.h
index d621c42a237e..f9503c368db5 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl.h
+++ b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl.h
@@ -9,14 +9,6 @@
 #define BLACK_OFFSET_RGB_Y 0x0
 #define BLACK_OFFSET_CBCR  0x8000
 
-#ifndef SPL_PFX_
-#define SPL_PFX_
-#endif
-
-#define SPL_EXPAND2(a, b)         a##b
-#define SPL_EXPAND(a, b)          SPL_EXPAND2(a, b)
-#define SPL_NAMESPACE(symbol)     SPL_EXPAND(SPL_PFX_, symbol)
-
 
 /* SPL interfaces */
 
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_filters.c b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_filters.c
index 99238644e0a1..088aba3c00a1 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_filters.c
+++ b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_filters.c
@@ -4,8 +4,8 @@
 
 #include "dc_spl_filters.h"
 
-void convert_filter_s1_10_to_s1_12(const uint16_t *s1_10_filter,
-	uint16_t *s1_12_filter, int num_taps)
+void SPL_NAMESPACE(convert_filter_s1_10_to_s1_12(const uint16_t *s1_10_filter,
+	uint16_t *s1_12_filter, int num_taps))
 {
 	int num_entries = NUM_PHASES_COEFF * num_taps;
 	int i;
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_filters.h b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_filters.h
index 20439cdbdb10..f3ee51c42bf2 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_filters.h
+++ b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_filters.h
@@ -9,7 +9,7 @@
 
 #define NUM_PHASES_COEFF 33
 
-void convert_filter_s1_10_to_s1_12(const uint16_t *s1_10_filter,
-	uint16_t *s1_12_filter, int num_taps);
+void SPL_NAMESPACE(convert_filter_s1_10_to_s1_12(const uint16_t *s1_10_filter,
+	uint16_t *s1_12_filter, int num_taps));
 
 #endif /* __DC_SPL_FILTERS_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_isharp_filters.c b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_isharp_filters.c
index 12acdd34e6a6..1d9edb89e47a 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_isharp_filters.c
+++ b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_isharp_filters.c
@@ -367,8 +367,8 @@ static unsigned int spl_calculate_sharpness_level_adj(struct spl_fixed31_32 rati
 	sharpness_level_down_adj = 0;
 	lookup_ptr = sharpness_level_adj;
 	while (j < NUM_SHARPNESS_ADJ_LEVELS) {
-		ratio_level = spl_fixpt_from_fraction(lookup_ptr->ratio_numer,
-			lookup_ptr->ratio_denom);
+		ratio_level = SPL_NAMESPACE(spl_fixpt_from_fraction(lookup_ptr->ratio_numer,
+			lookup_ptr->ratio_denom));
 		if (ratio.value >= ratio_level.value) {
 			sharpness_level_down_adj = lookup_ptr->level_down_adj;
 			break;
@@ -447,8 +447,9 @@ static unsigned int spl_calculate_sharpness_level(struct spl_fixed31_32 ratio,
 	return sharpness_level;
 }
 
-void spl_build_isharp_1dlut_from_reference_curve(struct spl_fixed31_32 ratio, enum system_setup setup,
-	struct adaptive_sharpness sharpness, enum scale_to_sharpness_policy scale_to_sharpness_policy)
+void SPL_NAMESPACE(spl_build_isharp_1dlut_from_reference_curve(
+	struct spl_fixed31_32 ratio, enum system_setup setup,
+	struct adaptive_sharpness sharpness, enum scale_to_sharpness_policy scale_to_sharpness_policy))
 {
 	uint8_t *byte_ptr_1dlut_src, *byte_ptr_1dlut_dst;
 	struct spl_fixed31_32 sharp_base, sharp_calc, sharp_level;
@@ -461,7 +462,7 @@ void spl_build_isharp_1dlut_from_reference_curve(struct spl_fixed31_32 ratio, en
 	unsigned int sharpnessX1000 = spl_calculate_sharpness_level(ratio,
 			sharpness.sharpness_level, setup,
 			sharpness.sharpness_range, scale_to_sharpness_policy);
-	sharp_level = spl_fixpt_from_fraction(sharpnessX1000, 1000);
+	sharp_level = SPL_NAMESPACE(spl_fixpt_from_fraction(sharpnessX1000, 1000));
 
 	/*
 	 * Check if pregen 1dlut table is already precalculated
@@ -486,10 +487,11 @@ void spl_build_isharp_1dlut_from_reference_curve(struct spl_fixed31_32 ratio, en
 	memset(byte_ptr_1dlut_dst, 0, size_1dlut);
 	for (j = 0; j < size_1dlut; j++) {
 		sharp_base = spl_fixpt_from_int((int)*byte_ptr_1dlut_src);
-		sharp_calc = spl_fixpt_mul(sharp_base, sharp_level);
+		sharp_calc = SPL_NAMESPACE(spl_fixpt_mul(sharp_base, sharp_level));
 		sharp_calc = spl_fixpt_div(sharp_calc, spl_fixpt_from_int(3));
 		sharp_calc = spl_fixpt_min(spl_fixpt_from_int(255), sharp_calc);
-		sharp_calc = spl_fixpt_add(sharp_calc, spl_fixpt_from_fraction(1, 2));
+		sharp_calc = spl_fixpt_add(sharp_calc,
+			SPL_NAMESPACE(spl_fixpt_from_fraction(1, 2)));
 		sharp_calc_int = spl_fixpt_floor(sharp_calc);
 		/* Clamp it at 0x7F so it doesn't wrap */
 		if (sharp_calc_int > 127)
@@ -506,12 +508,12 @@ void spl_build_isharp_1dlut_from_reference_curve(struct spl_fixed31_32 ratio, en
 	filter_isharp_1D_lut_pregen[setup].sharpness_denom = 1000;
 }
 
-uint32_t *spl_get_pregen_filter_isharp_1D_lut(enum system_setup setup)
+uint32_t *SPL_NAMESPACE(spl_get_pregen_filter_isharp_1D_lut(enum system_setup setup))
 {
 	return filter_isharp_1D_lut_pregen[setup].value;
 }
 
-const uint16_t *spl_dscl_get_blur_scale_coeffs_64p(int taps)
+const uint16_t *SPL_NAMESPACE(spl_dscl_get_blur_scale_coeffs_64p(int taps))
 {
 	if (taps == 3)
 		return filter_isharp_bs_3tap_64p_s1_12;
@@ -526,7 +528,7 @@ const uint16_t *spl_dscl_get_blur_scale_coeffs_64p(int taps)
 	}
 }
 
-const uint16_t *spl_dscl_get_blur_scale_coeffs_64p_s1_10(int taps)
+const uint16_t *SPL_NAMESPACE(spl_dscl_get_blur_scale_coeffs_64p_s1_10(int taps))
 {
 	if (taps == 3)
 		return filter_isharp_bs_3tap_64p;
@@ -541,13 +543,12 @@ const uint16_t *spl_dscl_get_blur_scale_coeffs_64p_s1_10(int taps)
 	}
 }
 
-void spl_set_blur_scale_data(struct dscl_prog_data *dscl_prog_data,
-		const struct spl_scaler_data *data)
+void SPL_NAMESPACE(spl_set_blur_scale_data(struct dscl_prog_data *dscl_prog_data,
+		const struct spl_scaler_data *data))
 {
 	dscl_prog_data->filter_blur_scale_h =
-		spl_dscl_get_blur_scale_coeffs_64p(data->taps.h_taps);
+		SPL_NAMESPACE(spl_dscl_get_blur_scale_coeffs_64p(data->taps.h_taps));
 
 	dscl_prog_data->filter_blur_scale_v =
-		spl_dscl_get_blur_scale_coeffs_64p(data->taps.v_taps);
+		SPL_NAMESPACE(spl_dscl_get_blur_scale_coeffs_64p(data->taps.v_taps));
 }
-
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_isharp_filters.h b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_isharp_filters.h
index f5e3d3ecc913..d4082d4969e4 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_isharp_filters.h
+++ b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_isharp_filters.h
@@ -28,15 +28,17 @@ enum system_setup {
 	NUM_SHARPNESS_SETUPS
 };
 
-void spl_set_blur_scale_data(struct dscl_prog_data *dscl_prog_data,
-	const struct spl_scaler_data *data);
+void SPL_NAMESPACE(spl_set_blur_scale_data(struct dscl_prog_data *dscl_prog_data,
+	const struct spl_scaler_data *data));
 
-void spl_build_isharp_1dlut_from_reference_curve(struct spl_fixed31_32 ratio, enum system_setup setup,
-	struct adaptive_sharpness sharpness, enum scale_to_sharpness_policy scale_to_sharpness_policy);
-uint32_t *spl_get_pregen_filter_isharp_1D_lut(enum system_setup setup);
+void SPL_NAMESPACE(spl_build_isharp_1dlut_from_reference_curve(
+	struct spl_fixed31_32 ratio, enum system_setup setup,
+	struct adaptive_sharpness sharpness,
+	enum scale_to_sharpness_policy scale_to_sharpness_policy));
+uint32_t *SPL_NAMESPACE(spl_get_pregen_filter_isharp_1D_lut(enum system_setup setup));
 
 // public API
-const uint16_t *spl_dscl_get_blur_scale_coeffs_64p(int taps);
-const uint16_t *spl_dscl_get_blur_scale_coeffs_64p_s1_10(int taps);
+const uint16_t *SPL_NAMESPACE(spl_dscl_get_blur_scale_coeffs_64p(int taps));
+const uint16_t *SPL_NAMESPACE(spl_dscl_get_blur_scale_coeffs_64p_s1_10(int taps));
 
 #endif /* __DC_SPL_ISHARP_FILTERS_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_easf_filters.c b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_easf_filters.c
index 0d1bd81ff04a..de16ee586073 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_easf_filters.c
+++ b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_easf_filters.c
@@ -2194,19 +2194,19 @@ static struct scale_ratio_to_reg_value_lookup easf_3tap_uptilt2_offset_lookup[]
 
 static const uint16_t *spl_get_easf_filter_3tap_64p(struct spl_fixed31_32 ratio)
 {
-	if (ratio.value < spl_fixpt_from_fraction(3, 10).value)
+	if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(3, 10)).value)
 		return easf_filter_3tap_64p_ratio_0_30_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(4, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 10)).value)
 		return easf_filter_3tap_64p_ratio_0_40_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(5, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 10)).value)
 		return easf_filter_3tap_64p_ratio_0_50_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(6, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(6, 10)).value)
 		return easf_filter_3tap_64p_ratio_0_60_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(7, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(7, 10)).value)
 		return easf_filter_3tap_64p_ratio_0_70_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(8, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(8, 10)).value)
 		return easf_filter_3tap_64p_ratio_0_80_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(9, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(9, 10)).value)
 		return easf_filter_3tap_64p_ratio_0_90_s1_12;
 	else
 		return easf_filter_3tap_64p_ratio_1_00_s1_12;
@@ -2214,19 +2214,19 @@ static const uint16_t *spl_get_easf_filter_3tap_64p(struct spl_fixed31_32 ratio)
 
 static const uint16_t *spl_get_easf_filter_4tap_64p(struct spl_fixed31_32 ratio)
 {
-	if (ratio.value < spl_fixpt_from_fraction(3, 10).value)
+	if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(3, 10)).value)
 		return easf_filter_4tap_64p_ratio_0_30_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(4, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 10)).value)
 		return easf_filter_4tap_64p_ratio_0_40_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(5, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 10)).value)
 		return easf_filter_4tap_64p_ratio_0_50_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(6, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(6, 10)).value)
 		return easf_filter_4tap_64p_ratio_0_60_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(7, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(7, 10)).value)
 		return easf_filter_4tap_64p_ratio_0_70_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(8, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(8, 10)).value)
 		return easf_filter_4tap_64p_ratio_0_80_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(9, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(9, 10)).value)
 		return easf_filter_4tap_64p_ratio_0_90_s1_12;
 	else
 		return easf_filter_4tap_64p_ratio_1_00_s1_12;
@@ -2234,25 +2234,26 @@ static const uint16_t *spl_get_easf_filter_4tap_64p(struct spl_fixed31_32 ratio)
 
 static const uint16_t *spl_get_easf_filter_6tap_64p(struct spl_fixed31_32 ratio)
 {
-	if (ratio.value < spl_fixpt_from_fraction(3, 10).value)
+	if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(3, 10)).value)
 		return easf_filter_6tap_64p_ratio_0_30_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(4, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 10)).value)
 		return easf_filter_6tap_64p_ratio_0_40_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(5, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 10)).value)
 		return easf_filter_6tap_64p_ratio_0_50_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(6, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(6, 10)).value)
 		return easf_filter_6tap_64p_ratio_0_60_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(7, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(7, 10)).value)
 		return easf_filter_6tap_64p_ratio_0_70_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(8, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(8, 10)).value)
 		return easf_filter_6tap_64p_ratio_0_80_s1_12;
-	else if (ratio.value < spl_fixpt_from_fraction(9, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(9, 10)).value)
 		return easf_filter_6tap_64p_ratio_0_90_s1_12;
 	else
 		return easf_filter_6tap_64p_ratio_1_00_s1_12;
 }
 
-const uint16_t *spl_dscl_get_easf_filter_coeffs_64p(int taps, struct spl_fixed31_32 ratio)
+const uint16_t *SPL_NAMESPACE(spl_dscl_get_easf_filter_coeffs_64p(
+	int taps, struct spl_fixed31_32 ratio))
 {
 	if (taps == 6)
 		return spl_get_easf_filter_6tap_64p(ratio);
@@ -2269,19 +2270,19 @@ const uint16_t *spl_dscl_get_easf_filter_coeffs_64p(int taps, struct spl_fixed31
 
 static const uint16_t *spl_get_easf_filter_3tap_64p_s1_10(struct spl_fixed31_32 ratio)
 {
-	if (ratio.value < spl_fixpt_from_fraction(3, 10).value)
+	if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(3, 10)).value)
 		return easf_filter_3tap_64p_ratio_0_30;
-	else if (ratio.value < spl_fixpt_from_fraction(4, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 10)).value)
 		return easf_filter_3tap_64p_ratio_0_40;
-	else if (ratio.value < spl_fixpt_from_fraction(5, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 10)).value)
 		return easf_filter_3tap_64p_ratio_0_50;
-	else if (ratio.value < spl_fixpt_from_fraction(6, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(6, 10)).value)
 		return easf_filter_3tap_64p_ratio_0_60;
-	else if (ratio.value < spl_fixpt_from_fraction(7, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(7, 10)).value)
 		return easf_filter_3tap_64p_ratio_0_70;
-	else if (ratio.value < spl_fixpt_from_fraction(8, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(8, 10)).value)
 		return easf_filter_3tap_64p_ratio_0_80;
-	else if (ratio.value < spl_fixpt_from_fraction(9, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(9, 10)).value)
 		return easf_filter_3tap_64p_ratio_0_90;
 	else
 		return easf_filter_3tap_64p_ratio_1_00;
@@ -2289,19 +2290,19 @@ static const uint16_t *spl_get_easf_filter_3tap_64p_s1_10(struct spl_fixed31_32
 
 static const uint16_t *spl_get_easf_filter_4tap_64p_s1_10(struct spl_fixed31_32 ratio)
 {
-	if (ratio.value < spl_fixpt_from_fraction(3, 10).value)
+	if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(3, 10)).value)
 		return easf_filter_4tap_64p_ratio_0_30;
-	else if (ratio.value < spl_fixpt_from_fraction(4, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 10)).value)
 		return easf_filter_4tap_64p_ratio_0_40;
-	else if (ratio.value < spl_fixpt_from_fraction(5, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 10)).value)
 		return easf_filter_4tap_64p_ratio_0_50;
-	else if (ratio.value < spl_fixpt_from_fraction(6, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(6, 10)).value)
 		return easf_filter_4tap_64p_ratio_0_60;
-	else if (ratio.value < spl_fixpt_from_fraction(7, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(7, 10)).value)
 		return easf_filter_4tap_64p_ratio_0_70;
-	else if (ratio.value < spl_fixpt_from_fraction(8, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(8, 10)).value)
 		return easf_filter_4tap_64p_ratio_0_80;
-	else if (ratio.value < spl_fixpt_from_fraction(9, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(9, 10)).value)
 		return easf_filter_4tap_64p_ratio_0_90;
 	else
 		return easf_filter_4tap_64p_ratio_1_00;
@@ -2309,25 +2310,26 @@ static const uint16_t *spl_get_easf_filter_4tap_64p_s1_10(struct spl_fixed31_32
 
 static const uint16_t *spl_get_easf_filter_6tap_64p_s1_10(struct spl_fixed31_32 ratio)
 {
-	if (ratio.value < spl_fixpt_from_fraction(3, 10).value)
+	if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(3, 10)).value)
 		return easf_filter_6tap_64p_ratio_0_30;
-	else if (ratio.value < spl_fixpt_from_fraction(4, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 10)).value)
 		return easf_filter_6tap_64p_ratio_0_40;
-	else if (ratio.value < spl_fixpt_from_fraction(5, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 10)).value)
 		return easf_filter_6tap_64p_ratio_0_50;
-	else if (ratio.value < spl_fixpt_from_fraction(6, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(6, 10)).value)
 		return easf_filter_6tap_64p_ratio_0_60;
-	else if (ratio.value < spl_fixpt_from_fraction(7, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(7, 10)).value)
 		return easf_filter_6tap_64p_ratio_0_70;
-	else if (ratio.value < spl_fixpt_from_fraction(8, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(8, 10)).value)
 		return easf_filter_6tap_64p_ratio_0_80;
-	else if (ratio.value < spl_fixpt_from_fraction(9, 10).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(9, 10)).value)
 		return easf_filter_6tap_64p_ratio_0_90;
 	else
 		return easf_filter_6tap_64p_ratio_1_00;
 }
 
-const uint16_t *spl_dscl_get_easf_filter_coeffs_64p_s1_10(int taps, struct spl_fixed31_32 ratio)
+const uint16_t *SPL_NAMESPACE(spl_dscl_get_easf_filter_coeffs_64p_s1_10(
+	int taps, struct spl_fixed31_32 ratio))
 {
 	if (taps == 6)
 		return spl_get_easf_filter_6tap_64p_s1_10(ratio);
@@ -2342,39 +2344,39 @@ const uint16_t *spl_dscl_get_easf_filter_coeffs_64p_s1_10(int taps, struct spl_f
 	}
 }
 
-void spl_set_filters_data(struct dscl_prog_data *dscl_prog_data,
+void SPL_NAMESPACE(spl_set_filters_data(struct dscl_prog_data *dscl_prog_data,
 		const struct spl_scaler_data *data, bool enable_easf_v,
-		bool enable_easf_h)
+		bool enable_easf_h))
 {
 	/*
 	 * Old coefficients calculated scaling ratio = input / output
 	 * New coefficients are calculated based on = output / input
 	 */
 	if (enable_easf_h) {
-		dscl_prog_data->filter_h = spl_dscl_get_easf_filter_coeffs_64p(
-			data->taps.h_taps, data->recip_ratios.horz);
+		dscl_prog_data->filter_h = SPL_NAMESPACE(spl_dscl_get_easf_filter_coeffs_64p(
+			data->taps.h_taps, data->recip_ratios.horz));
 
-		dscl_prog_data->filter_h_c = spl_dscl_get_easf_filter_coeffs_64p(
-			data->taps.h_taps_c, data->recip_ratios.horz_c);
+		dscl_prog_data->filter_h_c = SPL_NAMESPACE(spl_dscl_get_easf_filter_coeffs_64p(
+			data->taps.h_taps_c, data->recip_ratios.horz_c));
 	} else {
-		dscl_prog_data->filter_h = spl_dscl_get_filter_coeffs_64p(
-			data->taps.h_taps, data->ratios.horz);
+		dscl_prog_data->filter_h = SPL_NAMESPACE(spl_dscl_get_filter_coeffs_64p(
+			data->taps.h_taps, data->ratios.horz));
 
-		dscl_prog_data->filter_h_c = spl_dscl_get_filter_coeffs_64p(
-			data->taps.h_taps_c, data->ratios.horz_c);
+		dscl_prog_data->filter_h_c = SPL_NAMESPACE(spl_dscl_get_filter_coeffs_64p(
+			data->taps.h_taps_c, data->ratios.horz_c));
 	}
 	if (enable_easf_v) {
-		dscl_prog_data->filter_v = spl_dscl_get_easf_filter_coeffs_64p(
-			data->taps.v_taps, data->recip_ratios.vert);
+		dscl_prog_data->filter_v = SPL_NAMESPACE(spl_dscl_get_easf_filter_coeffs_64p(
+			data->taps.v_taps, data->recip_ratios.vert));
 
-		dscl_prog_data->filter_v_c = spl_dscl_get_easf_filter_coeffs_64p(
-			data->taps.v_taps_c, data->recip_ratios.vert_c);
+		dscl_prog_data->filter_v_c = SPL_NAMESPACE(spl_dscl_get_easf_filter_coeffs_64p(
+			data->taps.v_taps_c, data->recip_ratios.vert_c));
 	} else {
-		dscl_prog_data->filter_v = spl_dscl_get_filter_coeffs_64p(
-			data->taps.v_taps, data->ratios.vert);
+		dscl_prog_data->filter_v = SPL_NAMESPACE(spl_dscl_get_filter_coeffs_64p(
+			data->taps.v_taps, data->ratios.vert));
 
-		dscl_prog_data->filter_v_c = spl_dscl_get_filter_coeffs_64p(
-			data->taps.v_taps_c, data->ratios.vert_c);
+		dscl_prog_data->filter_v_c = SPL_NAMESPACE(spl_dscl_get_filter_coeffs_64p(
+			data->taps.v_taps_c, data->ratios.vert_c));
 	}
 }
 
@@ -2395,9 +2397,9 @@ static uint32_t spl_easf_get_scale_ratio_to_reg_value(struct spl_fixed31_32 rati
 		if (lookup_table_index_ptr->numer < 0)
 			break;
 
-		if (ratio.value < spl_fixpt_from_fraction(
+		if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(
 			lookup_table_index_ptr->numer,
-			lookup_table_index_ptr->denom).value) {
+			lookup_table_index_ptr->denom)).value) {
 			value = lookup_table_index_ptr->reg_value;
 			break;
 		}
@@ -2406,7 +2408,7 @@ static uint32_t spl_easf_get_scale_ratio_to_reg_value(struct spl_fixed31_32 rati
 	}
 	return value;
 }
-uint32_t spl_get_v_bf3_mode(struct spl_fixed31_32 ratio)
+uint32_t SPL_NAMESPACE(spl_get_v_bf3_mode(struct spl_fixed31_32 ratio))
 {
 	uint32_t value;
 	unsigned int num_entries = sizeof(easf_v_bf3_mode_lookup) /
@@ -2415,7 +2417,7 @@ uint32_t spl_get_v_bf3_mode(struct spl_fixed31_32 ratio)
 		easf_v_bf3_mode_lookup, num_entries);
 	return value;
 }
-uint32_t spl_get_h_bf3_mode(struct spl_fixed31_32 ratio)
+uint32_t SPL_NAMESPACE(spl_get_h_bf3_mode(struct spl_fixed31_32 ratio))
 {
 	uint32_t value;
 	unsigned int num_entries = sizeof(easf_h_bf3_mode_lookup) /
@@ -2424,7 +2426,7 @@ uint32_t spl_get_h_bf3_mode(struct spl_fixed31_32 ratio)
 		easf_h_bf3_mode_lookup, num_entries);
 	return value;
 }
-uint32_t spl_get_reducer_gain6(int taps, struct spl_fixed31_32 ratio)
+uint32_t SPL_NAMESPACE(spl_get_reducer_gain6(int taps, struct spl_fixed31_32 ratio))
 {
 	uint32_t value;
 	unsigned int num_entries;
@@ -2443,7 +2445,7 @@ uint32_t spl_get_reducer_gain6(int taps, struct spl_fixed31_32 ratio)
 		value = 0;
 	return value;
 }
-uint32_t spl_get_reducer_gain4(int taps, struct spl_fixed31_32 ratio)
+uint32_t SPL_NAMESPACE(spl_get_reducer_gain4(int taps, struct spl_fixed31_32 ratio))
 {
 	uint32_t value;
 	unsigned int num_entries;
@@ -2462,7 +2464,7 @@ uint32_t spl_get_reducer_gain4(int taps, struct spl_fixed31_32 ratio)
 		value = 0;
 	return value;
 }
-uint32_t spl_get_gainRing6(int taps, struct spl_fixed31_32 ratio)
+uint32_t SPL_NAMESPACE(spl_get_gainRing6(int taps, struct spl_fixed31_32 ratio))
 {
 	uint32_t value;
 	unsigned int num_entries;
@@ -2481,7 +2483,7 @@ uint32_t spl_get_gainRing6(int taps, struct spl_fixed31_32 ratio)
 		value = 0;
 	return value;
 }
-uint32_t spl_get_gainRing4(int taps, struct spl_fixed31_32 ratio)
+uint32_t SPL_NAMESPACE(spl_get_gainRing4(int taps, struct spl_fixed31_32 ratio))
 {
 	uint32_t value;
 	unsigned int num_entries;
@@ -2500,7 +2502,8 @@ uint32_t spl_get_gainRing4(int taps, struct spl_fixed31_32 ratio)
 		value = 0;
 	return value;
 }
-uint32_t spl_get_3tap_dntilt_uptilt_offset(int taps, struct spl_fixed31_32 ratio)
+uint32_t SPL_NAMESPACE(spl_get_3tap_dntilt_uptilt_offset(
+	int taps, struct spl_fixed31_32 ratio))
 {
 	uint32_t value;
 	unsigned int num_entries;
@@ -2514,7 +2517,7 @@ uint32_t spl_get_3tap_dntilt_uptilt_offset(int taps, struct spl_fixed31_32 ratio
 		value = 0;
 	return value;
 }
-uint32_t spl_get_3tap_uptilt_maxval(int taps, struct spl_fixed31_32 ratio)
+uint32_t SPL_NAMESPACE(spl_get_3tap_uptilt_maxval(int taps, struct spl_fixed31_32 ratio))
 {
 	uint32_t value;
 	unsigned int num_entries;
@@ -2528,7 +2531,7 @@ uint32_t spl_get_3tap_uptilt_maxval(int taps, struct spl_fixed31_32 ratio)
 		value = 0;
 	return value;
 }
-uint32_t spl_get_3tap_dntilt_slope(int taps, struct spl_fixed31_32 ratio)
+uint32_t SPL_NAMESPACE(spl_get_3tap_dntilt_slope(int taps, struct spl_fixed31_32 ratio))
 {
 	uint32_t value;
 	unsigned int num_entries;
@@ -2542,7 +2545,7 @@ uint32_t spl_get_3tap_dntilt_slope(int taps, struct spl_fixed31_32 ratio)
 		value = 0;
 	return value;
 }
-uint32_t spl_get_3tap_uptilt1_slope(int taps, struct spl_fixed31_32 ratio)
+uint32_t SPL_NAMESPACE(spl_get_3tap_uptilt1_slope(int taps, struct spl_fixed31_32 ratio))
 {
 	uint32_t value;
 	unsigned int num_entries;
@@ -2556,7 +2559,7 @@ uint32_t spl_get_3tap_uptilt1_slope(int taps, struct spl_fixed31_32 ratio)
 		value = 0;
 	return value;
 }
-uint32_t spl_get_3tap_uptilt2_slope(int taps, struct spl_fixed31_32 ratio)
+uint32_t SPL_NAMESPACE(spl_get_3tap_uptilt2_slope(int taps, struct spl_fixed31_32 ratio))
 {
 	uint32_t value;
 	unsigned int num_entries;
@@ -2570,7 +2573,7 @@ uint32_t spl_get_3tap_uptilt2_slope(int taps, struct spl_fixed31_32 ratio)
 		value = 0;
 	return value;
 }
-uint32_t spl_get_3tap_uptilt2_offset(int taps, struct spl_fixed31_32 ratio)
+uint32_t SPL_NAMESPACE(spl_get_3tap_uptilt2_offset(int taps, struct spl_fixed31_32 ratio))
 {
 	uint32_t value;
 	unsigned int num_entries;
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_easf_filters.h b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_easf_filters.h
index 321ae22a04d4..ba1cdb8be417 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_easf_filters.h
+++ b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_easf_filters.h
@@ -13,25 +13,28 @@ struct scale_ratio_to_reg_value_lookup {
 	const uint32_t reg_value;
 };
 
-void spl_set_filters_data(struct dscl_prog_data *dscl_prog_data,
+void SPL_NAMESPACE(spl_set_filters_data(struct dscl_prog_data *dscl_prog_data,
 	const struct spl_scaler_data *data, bool enable_easf_v,
-	bool enable_easf_h);
-
-uint32_t spl_get_v_bf3_mode(struct spl_fixed31_32 ratio);
-uint32_t spl_get_h_bf3_mode(struct spl_fixed31_32 ratio);
-uint32_t spl_get_reducer_gain6(int taps, struct spl_fixed31_32 ratio);
-uint32_t spl_get_reducer_gain4(int taps, struct spl_fixed31_32 ratio);
-uint32_t spl_get_gainRing6(int taps, struct spl_fixed31_32 ratio);
-uint32_t spl_get_gainRing4(int taps, struct spl_fixed31_32 ratio);
-uint32_t spl_get_3tap_dntilt_uptilt_offset(int taps, struct spl_fixed31_32 ratio);
-uint32_t spl_get_3tap_uptilt_maxval(int taps, struct spl_fixed31_32 ratio);
-uint32_t spl_get_3tap_dntilt_slope(int taps, struct spl_fixed31_32 ratio);
-uint32_t spl_get_3tap_uptilt1_slope(int taps, struct spl_fixed31_32 ratio);
-uint32_t spl_get_3tap_uptilt2_slope(int taps, struct spl_fixed31_32 ratio);
-uint32_t spl_get_3tap_uptilt2_offset(int taps, struct spl_fixed31_32 ratio);
+	bool enable_easf_h));
+
+uint32_t SPL_NAMESPACE(spl_get_v_bf3_mode(struct spl_fixed31_32 ratio));
+uint32_t SPL_NAMESPACE(spl_get_h_bf3_mode(struct spl_fixed31_32 ratio));
+uint32_t SPL_NAMESPACE(spl_get_reducer_gain6(int taps, struct spl_fixed31_32 ratio));
+uint32_t SPL_NAMESPACE(spl_get_reducer_gain4(int taps, struct spl_fixed31_32 ratio));
+uint32_t SPL_NAMESPACE(spl_get_gainRing6(int taps, struct spl_fixed31_32 ratio));
+uint32_t SPL_NAMESPACE(spl_get_gainRing4(int taps, struct spl_fixed31_32 ratio));
+uint32_t SPL_NAMESPACE(spl_get_3tap_dntilt_uptilt_offset(
+	int taps, struct spl_fixed31_32 ratio));
+uint32_t SPL_NAMESPACE(spl_get_3tap_uptilt_maxval(int taps, struct spl_fixed31_32 ratio));
+uint32_t SPL_NAMESPACE(spl_get_3tap_dntilt_slope(int taps, struct spl_fixed31_32 ratio));
+uint32_t SPL_NAMESPACE(spl_get_3tap_uptilt1_slope(int taps, struct spl_fixed31_32 ratio));
+uint32_t SPL_NAMESPACE(spl_get_3tap_uptilt2_slope(int taps, struct spl_fixed31_32 ratio));
+uint32_t SPL_NAMESPACE(spl_get_3tap_uptilt2_offset(int taps, struct spl_fixed31_32 ratio));
 
 /* public API */
-const uint16_t *spl_dscl_get_easf_filter_coeffs_64p(int taps, struct spl_fixed31_32 ratio);
-const uint16_t *spl_dscl_get_easf_filter_coeffs_64p_s1_10(int taps, struct spl_fixed31_32 ratio);
+const uint16_t *SPL_NAMESPACE(spl_dscl_get_easf_filter_coeffs_64p(
+	int taps, struct spl_fixed31_32 ratio));
+const uint16_t *SPL_NAMESPACE(spl_dscl_get_easf_filter_coeffs_64p_s1_10(
+	int taps, struct spl_fixed31_32 ratio));
 
 #endif /* __DC_SPL_SCL_EASF_FILTERS_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_filters.c b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_filters.c
index 5e52bdf1ad44..2d73d0dce5ff 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_filters.c
+++ b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_filters.c
@@ -1134,9 +1134,9 @@ static const uint16_t *spl_get_filter_3tap_64p(struct spl_fixed31_32 ratio)
 {
 	if (ratio.value < spl_fixpt_one.value)
 		return filter_3tap_64p_upscale;
-	else if (ratio.value < spl_fixpt_from_fraction(4, 3).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 3)).value)
 		return filter_3tap_64p_116;
-	else if (ratio.value < spl_fixpt_from_fraction(5, 3).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 3)).value)
 		return filter_3tap_64p_149;
 	else
 		return filter_3tap_64p_183;
@@ -1146,9 +1146,9 @@ static const uint16_t *spl_get_filter_4tap_64p(struct spl_fixed31_32 ratio)
 {
 	if (ratio.value < spl_fixpt_one.value)
 		return filter_4tap_64p_upscale;
-	else if (ratio.value < spl_fixpt_from_fraction(4, 3).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 3)).value)
 		return filter_4tap_64p_116;
-	else if (ratio.value < spl_fixpt_from_fraction(5, 3).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 3)).value)
 		return filter_4tap_64p_149;
 	else
 		return filter_4tap_64p_183;
@@ -1158,9 +1158,9 @@ static const uint16_t *spl_get_filter_5tap_64p(struct spl_fixed31_32 ratio)
 {
 	if (ratio.value < spl_fixpt_one.value)
 		return filter_5tap_64p_upscale;
-	else if (ratio.value < spl_fixpt_from_fraction(4, 3).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 3)).value)
 		return filter_5tap_64p_116;
-	else if (ratio.value < spl_fixpt_from_fraction(5, 3).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 3)).value)
 		return filter_5tap_64p_149;
 	else
 		return filter_5tap_64p_183;
@@ -1170,9 +1170,9 @@ static const uint16_t *spl_get_filter_6tap_64p(struct spl_fixed31_32 ratio)
 {
 	if (ratio.value < spl_fixpt_one.value)
 		return filter_6tap_64p_upscale;
-	else if (ratio.value < spl_fixpt_from_fraction(4, 3).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 3)).value)
 		return filter_6tap_64p_116;
-	else if (ratio.value < spl_fixpt_from_fraction(5, 3).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 3)).value)
 		return filter_6tap_64p_149;
 	else
 		return filter_6tap_64p_183;
@@ -1182,9 +1182,9 @@ static const uint16_t *spl_get_filter_7tap_64p(struct spl_fixed31_32 ratio)
 {
 	if (ratio.value < spl_fixpt_one.value)
 		return filter_7tap_64p_upscale;
-	else if (ratio.value < spl_fixpt_from_fraction(4, 3).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 3)).value)
 		return filter_7tap_64p_116;
-	else if (ratio.value < spl_fixpt_from_fraction(5, 3).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 3)).value)
 		return filter_7tap_64p_149;
 	else
 		return filter_7tap_64p_183;
@@ -1194,9 +1194,9 @@ static const uint16_t *spl_get_filter_8tap_64p(struct spl_fixed31_32 ratio)
 {
 	if (ratio.value < spl_fixpt_one.value)
 		return filter_8tap_64p_upscale;
-	else if (ratio.value < spl_fixpt_from_fraction(4, 3).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 3)).value)
 		return filter_8tap_64p_116;
-	else if (ratio.value < spl_fixpt_from_fraction(5, 3).value)
+	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 3)).value)
 		return filter_8tap_64p_149;
 	else
 		return filter_8tap_64p_183;
@@ -1207,7 +1207,8 @@ static const uint16_t *spl_get_filter_2tap_64p(void)
 	return filter_2tap_64p;
 }
 
-const uint16_t *spl_dscl_get_filter_coeffs_64p(int taps, struct spl_fixed31_32 ratio)
+const uint16_t *SPL_NAMESPACE(spl_dscl_get_filter_coeffs_64p(
+	int taps, struct spl_fixed31_32 ratio))
 {
 	if (taps == 8)
 		return spl_get_filter_8tap_64p(ratio);
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_filters.h b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_filters.h
index c315a438d064..445d626863c2 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_filters.h
+++ b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_filters.h
@@ -8,6 +8,7 @@
 #include "dc_spl_types.h"
 
 /* public API */
-const uint16_t *spl_dscl_get_filter_coeffs_64p(int taps, struct spl_fixed31_32 ratio);
+const uint16_t *SPL_NAMESPACE(spl_dscl_get_filter_coeffs_64p(
+	int taps, struct spl_fixed31_32 ratio));
 
 #endif /* __DC_SPL_SCL_FILTERS_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/spl_custom_float.c b/drivers/gpu/drm/amd/display/dc/sspl/spl_custom_float.c
index be2f34d034c5..0700b3dbbda7 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/spl_custom_float.c
+++ b/drivers/gpu/drm/amd/display/dc/sspl/spl_custom_float.c
@@ -14,8 +14,8 @@ static bool spl_build_custom_float(struct spl_fixed31_32 value,
 	uint32_t exp_offset = (1 << (format->exponenta_bits - 1)) - 1;
 
 	const struct spl_fixed31_32 mantissa_constant_plus_max_fraction =
-		spl_fixpt_from_fraction((1LL << (format->mantissa_bits + 1)) - 1,
-				       1LL << format->mantissa_bits);
+		SPL_NAMESPACE(spl_fixpt_from_fraction((1LL << (format->mantissa_bits + 1)) - 1,
+				       1LL << format->mantissa_bits));
 
 	struct spl_fixed31_32 mantiss;
 
@@ -134,9 +134,10 @@ static bool spl_setup_custom_float(const struct spl_custom_float_format *format,
 	return true;
 }
 
-bool spl_convert_to_custom_float_format(struct spl_fixed31_32 value,
-				    const struct spl_custom_float_format *format,
-				    uint32_t *result)
+bool SPL_NAMESPACE(spl_convert_to_custom_float_format(
+	struct spl_fixed31_32 value,
+	const struct spl_custom_float_format *format,
+	uint32_t *result))
 {
 	uint32_t mantissa;
 	uint32_t exponenta;
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/spl_custom_float.h b/drivers/gpu/drm/amd/display/dc/sspl/spl_custom_float.h
index cdc4e107b9de..f3fd8d30e638 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/spl_custom_float.h
+++ b/drivers/gpu/drm/amd/display/dc/sspl/spl_custom_float.h
@@ -21,9 +21,9 @@ struct spl_custom_float_value {
 	bool negative;
 };
 
-bool spl_convert_to_custom_float_format(
+bool SPL_NAMESPACE(spl_convert_to_custom_float_format(
 	struct spl_fixed31_32 value,
 	const struct spl_custom_float_format *format,
-	uint32_t *result);
+	uint32_t *result));
 
 #endif //SPL_CUSTOM_FLOAT_H_
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.c b/drivers/gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.c
index ebf0287417e0..ff0bdc3c33c8 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.c
+++ b/drivers/gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.c
@@ -44,7 +44,8 @@ static inline unsigned long long spl_complete_integer_division_u64(
 #define GET_FRACTIONAL_PART(x) \
 	(FRACTIONAL_PART_MASK & (x))
 
-struct spl_fixed31_32 spl_fixpt_from_fraction(long long numerator, long long denominator)
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_from_fraction(
+	long long numerator, long long denominator))
 {
 	struct spl_fixed31_32 res;
 
@@ -96,7 +97,8 @@ struct spl_fixed31_32 spl_fixpt_from_fraction(long long numerator, long long den
 	return res;
 }
 
-struct spl_fixed31_32 spl_fixpt_mul(struct spl_fixed31_32 arg1, struct spl_fixed31_32 arg2)
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_mul(
+	struct spl_fixed31_32 arg1, struct spl_fixed31_32 arg2))
 {
 	struct spl_fixed31_32 res;
 
@@ -147,7 +149,7 @@ struct spl_fixed31_32 spl_fixpt_mul(struct spl_fixed31_32 arg1, struct spl_fixed
 	return res;
 }
 
-struct spl_fixed31_32 spl_fixpt_sqr(struct spl_fixed31_32 arg)
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_sqr(struct spl_fixed31_32 arg))
 {
 	struct spl_fixed31_32 res;
 
@@ -187,19 +189,19 @@ struct spl_fixed31_32 spl_fixpt_sqr(struct spl_fixed31_32 arg)
 	return res;
 }
 
-struct spl_fixed31_32 spl_fixpt_recip(struct spl_fixed31_32 arg)
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_recip(struct spl_fixed31_32 arg))
 {
 	/*
 	 * @note
 	 * Good idea to use Newton's method
 	 */
 
-	return spl_fixpt_from_fraction(
+	return SPL_NAMESPACE(spl_fixpt_from_fraction(
 		spl_fixpt_one.value,
-		arg.value);
+		arg.value));
 }
 
-struct spl_fixed31_32 spl_fixpt_sinc(struct spl_fixed31_32 arg)
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_sinc(struct spl_fixed31_32 arg))
 {
 	struct spl_fixed31_32 square;
 
@@ -221,15 +223,15 @@ struct spl_fixed31_32 spl_fixpt_sinc(struct spl_fixed31_32 arg)
 					spl_fixpt_two_pi.value)));
 	}
 
-	square = spl_fixpt_sqr(arg_norm);
+	square = SPL_NAMESPACE(spl_fixpt_sqr(arg_norm));
 
 	do {
 		res = spl_fixpt_sub(
 			spl_fixpt_one,
 			spl_fixpt_div_int(
-				spl_fixpt_mul(
+				SPL_NAMESPACE(spl_fixpt_mul(
 					square,
-					res),
+					res)),
 				n * (n - 1)));
 
 		n -= 2;
@@ -237,24 +239,24 @@ struct spl_fixed31_32 spl_fixpt_sinc(struct spl_fixed31_32 arg)
 
 	if (arg.value != arg_norm.value)
 		res = spl_fixpt_div(
-			spl_fixpt_mul(res, arg_norm),
+			SPL_NAMESPACE(spl_fixpt_mul(res, arg_norm)),
 			arg);
 
 	return res;
 }
 
-struct spl_fixed31_32 spl_fixpt_sin(struct spl_fixed31_32 arg)
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_sin(struct spl_fixed31_32 arg))
 {
-	return spl_fixpt_mul(
+	return SPL_NAMESPACE(spl_fixpt_mul(
 		arg,
-		spl_fixpt_sinc(arg));
+		SPL_NAMESPACE(spl_fixpt_sinc(arg))));
 }
 
-struct spl_fixed31_32 spl_fixpt_cos(struct spl_fixed31_32 arg)
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_cos(struct spl_fixed31_32 arg))
 {
 	/* TODO implement argument normalization */
 
-	const struct spl_fixed31_32 square = spl_fixpt_sqr(arg);
+	const struct spl_fixed31_32 square = SPL_NAMESPACE(spl_fixpt_sqr(arg));
 
 	struct spl_fixed31_32 res = spl_fixpt_one;
 
@@ -264,9 +266,9 @@ struct spl_fixed31_32 spl_fixpt_cos(struct spl_fixed31_32 arg)
 		res = spl_fixpt_sub(
 			spl_fixpt_one,
 			spl_fixpt_div_int(
-				spl_fixpt_mul(
+				SPL_NAMESPACE(spl_fixpt_mul(
 					square,
-					res),
+					res)),
 				n * (n - 1)));
 
 		n -= 2;
@@ -286,9 +288,9 @@ static struct spl_fixed31_32 spl_fixed31_32_exp_from_taylor_series(struct spl_fi
 {
 	unsigned int n = 9;
 
-	struct spl_fixed31_32 res = spl_fixpt_from_fraction(
+	struct spl_fixed31_32 res = SPL_NAMESPACE(spl_fixpt_from_fraction(
 		n + 2,
-		n + 1);
+		n + 1));
 	/* TODO find correct res */
 
 	SPL_ASSERT(spl_fixpt_lt(arg, spl_fixpt_one));
@@ -297,20 +299,20 @@ static struct spl_fixed31_32 spl_fixed31_32_exp_from_taylor_series(struct spl_fi
 		res = spl_fixpt_add(
 			spl_fixpt_one,
 			spl_fixpt_div_int(
-				spl_fixpt_mul(
+				SPL_NAMESPACE(spl_fixpt_mul(
 					arg,
-					res),
+					res)),
 				n));
 	while (--n != 1);
 
 	return spl_fixpt_add(
 		spl_fixpt_one,
-		spl_fixpt_mul(
+		SPL_NAMESPACE(spl_fixpt_mul(
 			arg,
-			res));
+			res)));
 }
 
-struct spl_fixed31_32 spl_fixpt_exp(struct spl_fixed31_32 arg)
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_exp(struct spl_fixed31_32 arg))
 {
 	/*
 	 * @brief
@@ -353,7 +355,7 @@ struct spl_fixed31_32 spl_fixpt_exp(struct spl_fixed31_32 arg)
 		return spl_fixpt_one;
 }
 
-struct spl_fixed31_32 spl_fixpt_log(struct spl_fixed31_32 arg)
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_log(struct spl_fixed31_32 arg))
 {
 	struct spl_fixed31_32 res = spl_fixpt_neg(spl_fixpt_one);
 	/* TODO improve 1st estimation */
@@ -371,7 +373,7 @@ struct spl_fixed31_32 spl_fixpt_log(struct spl_fixed31_32 arg)
 				spl_fixpt_one),
 			spl_fixpt_div(
 				arg,
-				spl_fixpt_exp(res)));
+				SPL_NAMESPACE(spl_fixpt_exp(res))));
 
 		error = spl_fixpt_sub(
 			res,
@@ -427,37 +429,37 @@ static inline unsigned int spl_clamp_ux_dy(
 		return min_clamp;
 }
 
-unsigned int spl_fixpt_u4d19(struct spl_fixed31_32 arg)
+unsigned int SPL_NAMESPACE(spl_fixpt_u4d19(struct spl_fixed31_32 arg))
 {
 	return spl_ux_dy(arg.value, 4, 19);
 }
 
-unsigned int spl_fixpt_u3d19(struct spl_fixed31_32 arg)
+unsigned int SPL_NAMESPACE(spl_fixpt_u3d19(struct spl_fixed31_32 arg))
 {
 	return spl_ux_dy(arg.value, 3, 19);
 }
 
-unsigned int spl_fixpt_u2d19(struct spl_fixed31_32 arg)
+unsigned int SPL_NAMESPACE(spl_fixpt_u2d19(struct spl_fixed31_32 arg))
 {
 	return spl_ux_dy(arg.value, 2, 19);
 }
 
-unsigned int spl_fixpt_u0d19(struct spl_fixed31_32 arg)
+unsigned int SPL_NAMESPACE(spl_fixpt_u0d19(struct spl_fixed31_32 arg))
 {
 	return spl_ux_dy(arg.value, 0, 19);
 }
 
-unsigned int spl_fixpt_clamp_u0d14(struct spl_fixed31_32 arg)
+unsigned int SPL_NAMESPACE(spl_fixpt_clamp_u0d14(struct spl_fixed31_32 arg))
 {
 	return spl_clamp_ux_dy(arg.value, 0, 14, 1);
 }
 
-unsigned int spl_fixpt_clamp_u0d10(struct spl_fixed31_32 arg)
+unsigned int SPL_NAMESPACE(spl_fixpt_clamp_u0d10(struct spl_fixed31_32 arg))
 {
 	return spl_clamp_ux_dy(arg.value, 0, 10, 1);
 }
 
-int spl_fixpt_s4d19(struct spl_fixed31_32 arg)
+int SPL_NAMESPACE(spl_fixpt_s4d19(struct spl_fixed31_32 arg))
 {
 	if (arg.value < 0)
 		return -(int)spl_ux_dy(spl_fixpt_abs(arg).value, 4, 19);
@@ -465,9 +467,9 @@ int spl_fixpt_s4d19(struct spl_fixed31_32 arg)
 		return spl_ux_dy(arg.value, 4, 19);
 }
 
-struct spl_fixed31_32 spl_fixpt_from_ux_dy(unsigned int value,
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_from_ux_dy(unsigned int value,
 	unsigned int integer_bits,
-	unsigned int fractional_bits)
+	unsigned int fractional_bits))
 {
 	struct spl_fixed31_32 fixpt_value = spl_fixpt_zero;
 	struct spl_fixed31_32 fixpt_int_value = spl_fixpt_zero;
@@ -481,10 +483,10 @@ struct spl_fixed31_32 spl_fixpt_from_ux_dy(unsigned int value,
 	return fixpt_value;
 }
 
-struct spl_fixed31_32 spl_fixpt_from_int_dy(unsigned int int_value,
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_from_int_dy(unsigned int int_value,
 	unsigned int frac_value,
 	unsigned int integer_bits,
-	unsigned int fractional_bits)
+	unsigned int fractional_bits))
 {
 	struct spl_fixed31_32 fixpt_value = spl_fixpt_from_int(int_value);
 
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.h b/drivers/gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.h
index 9f349ffe9148..b0e639d6e97d 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.h
+++ b/drivers/gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.h
@@ -60,7 +60,8 @@ static const struct spl_fixed31_32 spl_fixpt_one = { 0x100000000LL };
  * @brief
  * result = numerator / denominator
  */
-struct spl_fixed31_32 spl_fixpt_from_fraction(long long numerator, long long denominator);
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_from_fraction(
+	long long numerator, long long denominator));
 
 /*
  * @brief
@@ -280,7 +281,8 @@ static inline struct spl_fixed31_32 spl_fixpt_sub_int(struct spl_fixed31_32 arg1
  * @brief
  * result = arg1 * arg2
  */
-struct spl_fixed31_32 spl_fixpt_mul(struct spl_fixed31_32 arg1, struct spl_fixed31_32 arg2);
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_mul(
+	struct spl_fixed31_32 arg1, struct spl_fixed31_32 arg2));
 
 
 /*
@@ -289,14 +291,14 @@ struct spl_fixed31_32 spl_fixpt_mul(struct spl_fixed31_32 arg1, struct spl_fixed
  */
 static inline struct spl_fixed31_32 spl_fixpt_mul_int(struct spl_fixed31_32 arg1, int arg2)
 {
-	return spl_fixpt_mul(arg1, spl_fixpt_from_int(arg2));
+	return SPL_NAMESPACE(spl_fixpt_mul(arg1, spl_fixpt_from_int(arg2)));
 }
 
 /*
  * @brief
  * result = square(arg) := arg * arg
  */
-struct spl_fixed31_32 spl_fixpt_sqr(struct spl_fixed31_32 arg);
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_sqr(struct spl_fixed31_32 arg));
 
 /*
  * @brief
@@ -304,7 +306,8 @@ struct spl_fixed31_32 spl_fixpt_sqr(struct spl_fixed31_32 arg);
  */
 static inline struct spl_fixed31_32 spl_fixpt_div_int(struct spl_fixed31_32 arg1, long long arg2)
 {
-	return spl_fixpt_from_fraction(arg1.value, spl_fixpt_from_int((int)arg2).value);
+	return SPL_NAMESPACE(spl_fixpt_from_fraction(arg1.value,
+		spl_fixpt_from_int((int)arg2).value));
 }
 
 /*
@@ -313,7 +316,7 @@ static inline struct spl_fixed31_32 spl_fixpt_div_int(struct spl_fixed31_32 arg1
  */
 static inline struct spl_fixed31_32 spl_fixpt_div(struct spl_fixed31_32 arg1, struct spl_fixed31_32 arg2)
 {
-	return spl_fixpt_from_fraction(arg1.value, arg2.value);
+	return SPL_NAMESPACE(spl_fixpt_from_fraction(arg1.value, arg2.value));
 }
 
 /*
@@ -328,7 +331,7 @@ static inline struct spl_fixed31_32 spl_fixpt_div(struct spl_fixed31_32 arg1, st
  * @note
  * No special actions taken in case argument is zero.
  */
-struct spl_fixed31_32 spl_fixpt_recip(struct spl_fixed31_32 arg);
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_recip(struct spl_fixed31_32 arg));
 
 /*
  * @brief
@@ -343,7 +346,7 @@ struct spl_fixed31_32 spl_fixpt_recip(struct spl_fixed31_32 arg);
  * Argument specified in radians,
  * internally it's normalized to [-2pi...2pi] range.
  */
-struct spl_fixed31_32 spl_fixpt_sinc(struct spl_fixed31_32 arg);
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_sinc(struct spl_fixed31_32 arg));
 
 /*
  * @brief
@@ -353,7 +356,7 @@ struct spl_fixed31_32 spl_fixpt_sinc(struct spl_fixed31_32 arg);
  * Argument specified in radians,
  * internally it's normalized to [-2pi...2pi] range.
  */
-struct spl_fixed31_32 spl_fixpt_sin(struct spl_fixed31_32 arg);
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_sin(struct spl_fixed31_32 arg));
 
 /*
  * @brief
@@ -365,7 +368,7 @@ struct spl_fixed31_32 spl_fixpt_sin(struct spl_fixed31_32 arg);
  * passing arguments outside that range
  * will cause incorrect result!
  */
-struct spl_fixed31_32 spl_fixpt_cos(struct spl_fixed31_32 arg);
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_cos(struct spl_fixed31_32 arg));
 
 /*
  * @brief
@@ -379,7 +382,7 @@ struct spl_fixed31_32 spl_fixpt_cos(struct spl_fixed31_32 arg);
  * @note
  * Currently, function is verified for abs(arg) <= 1.
  */
-struct spl_fixed31_32 spl_fixpt_exp(struct spl_fixed31_32 arg);
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_exp(struct spl_fixed31_32 arg));
 
 /*
  * @brief
@@ -391,7 +394,7 @@ struct spl_fixed31_32 spl_fixpt_exp(struct spl_fixed31_32 arg);
  * Currently, no special actions taken
  * in case of invalid argument(s). Take care!
  */
-struct spl_fixed31_32 spl_fixpt_log(struct spl_fixed31_32 arg);
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_log(struct spl_fixed31_32 arg));
 
 /*
  * @brief
@@ -410,10 +413,10 @@ static inline struct spl_fixed31_32 spl_fixpt_pow(struct spl_fixed31_32 arg1, st
 	if (arg1.value == 0)
 		return arg2.value == 0 ? spl_fixpt_one : spl_fixpt_zero;
 
-	return spl_fixpt_exp(
-		spl_fixpt_mul(
-			spl_fixpt_log(arg1),
-			arg2));
+	return SPL_NAMESPACE(spl_fixpt_exp(
+		SPL_NAMESPACE(spl_fixpt_mul(
+			SPL_NAMESPACE(spl_fixpt_log(arg1)),
+			arg2))));
 }
 
 /*
@@ -482,19 +485,19 @@ static inline int spl_fixpt_ceil(struct spl_fixed31_32 arg)
  * fractional
  */
 
-unsigned int spl_fixpt_u4d19(struct spl_fixed31_32 arg);
+unsigned int SPL_NAMESPACE(spl_fixpt_u4d19(struct spl_fixed31_32 arg));
 
-unsigned int spl_fixpt_u3d19(struct spl_fixed31_32 arg);
+unsigned int SPL_NAMESPACE(spl_fixpt_u3d19(struct spl_fixed31_32 arg));
 
-unsigned int spl_fixpt_u2d19(struct spl_fixed31_32 arg);
+unsigned int SPL_NAMESPACE(spl_fixpt_u2d19(struct spl_fixed31_32 arg));
 
-unsigned int spl_fixpt_u0d19(struct spl_fixed31_32 arg);
+unsigned int SPL_NAMESPACE(spl_fixpt_u0d19(struct spl_fixed31_32 arg));
 
-unsigned int spl_fixpt_clamp_u0d14(struct spl_fixed31_32 arg);
+unsigned int SPL_NAMESPACE(spl_fixpt_clamp_u0d14(struct spl_fixed31_32 arg));
 
-unsigned int spl_fixpt_clamp_u0d10(struct spl_fixed31_32 arg);
+unsigned int SPL_NAMESPACE(spl_fixpt_clamp_u0d10(struct spl_fixed31_32 arg));
 
-int spl_fixpt_s4d19(struct spl_fixed31_32 arg);
+int SPL_NAMESPACE(spl_fixpt_s4d19(struct spl_fixed31_32 arg));
 
 static inline struct spl_fixed31_32 spl_fixpt_truncate(struct spl_fixed31_32 arg, unsigned int frac_bits)
 {
@@ -513,10 +516,11 @@ static inline struct spl_fixed31_32 spl_fixpt_truncate(struct spl_fixed31_32 arg
 	return arg;
 }
 
-struct spl_fixed31_32 spl_fixpt_from_ux_dy(unsigned int value, unsigned int integer_bits, unsigned int fractional_bits);
-struct spl_fixed31_32 spl_fixpt_from_int_dy(unsigned int int_value,
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_from_ux_dy(unsigned int value,
+		unsigned int integer_bits, unsigned int fractional_bits));
+struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_from_int_dy(unsigned int int_value,
 		unsigned int frac_value,
 		unsigned int integer_bits,
-		unsigned int fractional_bits);
+		unsigned int fractional_bits));
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/spl_os_types.h b/drivers/gpu/drm/amd/display/dc/sspl/spl_os_types.h
index 2e6ba71960ac..ae2d24c856cf 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/spl_os_types.h
+++ b/drivers/gpu/drm/amd/display/dc/sspl/spl_os_types.h
@@ -53,4 +53,13 @@ static inline int64_t spl_div64_s64(int64_t dividend, int64_t divisor)
 #define spl_min(a, b)    (((a) < (b)) ? (a):(b))
 #endif
 
+/* SPL namespace macros */
+#ifndef SPL_PFX_
+#define SPL_PFX_
+#endif
+
+#define SPL_EXPAND2(a, b)         a##b
+#define SPL_EXPAND(a, b)          SPL_EXPAND2(a, b)
+#define SPL_NAMESPACE(symbol)     SPL_EXPAND(SPL_PFX_, symbol)
+
 #endif /* _SPL_OS_TYPES_H_ */
diff --git a/drivers/gpu/drm/amd/display/dmub/dmub_srv.h b/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
index 9d0168986fe7..12c1f9f7115a 100644
--- a/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
+++ b/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
@@ -72,6 +72,9 @@
 /* Default tracebuffer size if meta is absent. */
 #define DMUB_TRACE_BUFFER_SIZE (64 * 1024)
 
+#define PSP_HEADER_BYTES_256 0x100 // 256 bytes
+#define PSP_FOOTER_BYTES_256 0x100 // 256 bytes
+
 /* Forward declarations */
 struct dmub_srv;
 struct dmub_srv_common_regs;
@@ -227,6 +230,23 @@ struct dmub_srv_region_params {
 	const uint8_t *fw_inst_const;
 	const uint8_t *fw_bss_data;
 	const enum dmub_window_memory_type *window_memory_type;
+	const struct dmub_fw_meta_info *fw_info;
+};
+
+/**
+ * struct dmub_srv_fw_meta_info_params - params used for fetching fw meta info from fw_image
+ * @inst_const_size: size of the fw inst const section
+ * @bss_data_size: size of the fw bss data section
+ * @fw_inst_const: raw firmware inst const section
+ * @fw_bss_data: raw firmware bss data section
+ * @custom_psp_footer_size: custom psp footer size to use when indexing for fw meta info
+ */
+struct dmub_srv_fw_meta_info_params {
+	uint32_t inst_const_size;
+	uint32_t bss_data_size;
+	const uint8_t *fw_inst_const;
+	const uint8_t *fw_bss_data;
+	uint32_t custom_psp_footer_size;
 };
 
 /**
@@ -249,6 +269,7 @@ struct dmub_srv_region_info {
 	uint32_t gart_size;
 	uint8_t num_regions;
 	struct dmub_region regions[DMUB_WINDOW_TOTAL];
+	uint32_t verified_psp_footer_size;
 };
 
 /**
@@ -1098,4 +1119,16 @@ enum dmub_status dmub_srv_update_inbox_status(struct dmub_srv *dmub);
  */
 bool dmub_srv_get_preos_info(struct dmub_srv *dmub);
 
+/**
+ * dmub_srv_get_fw_meta_info_from_raw_fw() - Fetch firmware metadata info from raw firmware image
+ * @params: parameters for fetching firmware metadata info
+ * @fw_info_out: output buffer for firmware metadata info
+ *
+ * Return:
+ *   DMUB_STATUS_OK - success
+ *   DMUB_STATUS_INVALID - no FW meta info found
+ */
+enum dmub_status dmub_srv_get_fw_meta_info_from_raw_fw(struct dmub_srv_fw_meta_info_params *params,
+						       struct dmub_fw_meta_info *fw_info_out);
+
 #endif /* _DMUB_SRV_H_ */
diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
index 3f2a0ed02c59..6c599559c5da 100644
--- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
+++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
@@ -636,17 +636,29 @@ union pr_debug_flags {
 
 		/**
 		 * 0x10 (bit 4)
-		 * @skip_crtc_disabled: CRTC disable skipped
+		 * @visual_confirm_rate_control: Enable Visual Confirm rate control detection
 		 */
-		uint32_t skip_crtc_disabled : 1;
+		uint32_t visual_confirm_rate_control : 1;
 
-		/*
+		/**
 		 * 0x20 (bit 5)
-		 * @visual_confirm_rate_control: Enable Visual Confirm rate control detection
+		 * @force_full_frame_update: Force all selective updates to be full frame updates
 		 */
-		uint32_t visual_confirm_rate_control : 1;
+		uint32_t force_full_frame_update : 1;
 
-		uint32_t reserved : 26;
+		/**
+		 * 0x40 (bit 6)
+		 * @force_dpg_on: Force DPG on
+		 */
+		uint32_t force_dpg_on : 1;
+
+		/**
+		 * 0x80 (bit 7)
+		 * @force_hubp_on: Force Hubp on
+		 */
+		uint32_t force_hubp_on : 1;
+
+		uint32_t reserved : 24;
 	} bitfields;
 
 	uint32_t u32All;
@@ -669,19 +681,12 @@ union pr_hw_flags {
 		 * @fec_enable_status: receive fec enable/disable status from driver
 		 */
 		uint32_t fec_enable_status : 1;
-
 		/*
 		 * @smu_optimizations_en: SMU power optimization.
 		 * Only when active display is Replay capable and display enters Replay.
 		 * Trigger interrupt to SMU to powerup/down.
 		 */
 		uint32_t smu_optimizations_en : 1;
-
-		/**
-		 * @phy_power_state: Indicates current phy power state
-		 */
-		uint32_t phy_power_state : 1;
-
 		/**
 		 * @link_power_state: Indicates current link power state
 		 */
@@ -698,6 +703,7 @@ union pr_hw_flags {
 		 * @alpm_mode: Indicates ALPM mode selected
 		 */
 		uint32_t alpm_mode : 2;
+		uint32_t reserved : 23;
 	} bitfields;
 
 	uint32_t u32All;
@@ -2496,6 +2502,7 @@ struct dmub_fams2_drr_stream_static_state {
 struct dmub_fams2_cmd_legacy_stream_static_state {
 	uint16_t vactive_det_fill_delay_otg_vlines;
 	uint16_t programming_delay_otg_vlines;
+	uint32_t disallow_time_us;
 }; //v1
 
 struct dmub_fams2_cmd_subvp_stream_static_state {
@@ -2630,7 +2637,8 @@ union dmub_fams2_global_feature_config {
 		uint32_t enable_offload_flip: 1;
 		uint32_t enable_visual_confirm: 1;
 		uint32_t allow_delay_check_mode: 2;
-		uint32_t reserved: 24;
+		uint32_t legacy_method_no_fams2 : 1;
+		uint32_t reserved: 23;
 	} bits;
 	uint32_t all;
 };
@@ -4277,7 +4285,7 @@ enum pr_state {
 	// Active and Pending Power Up
 	PR_STATE_2_PENDING_POWER_UP					= 0x33,
 	// Active and Powered Up, Pending DPG latch
-	PR_STATE_2_PENDING_LOCK_FOR_DPG_POWER_ON	= 0x34,
+	PR_STATE_2_PENDING_LOCK	= 0x34,
 	// Active and Powered Up, Pending SDP and Unlock
 	PR_STATE_2_PENDING_SDP_AND_UNLOCK			= 0x35,
 	// Pending transmission of AS SDP for timing sync, but no rfb update
@@ -6603,10 +6611,6 @@ struct dmub_cmd_pr_copy_settings_data {
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
-	/**
-	 * Length of each horizontal line in ns.
-	 */
-	uint32_t line_time_in_ns;
 	/**
 	 * PHY instance.
 	 */
@@ -6615,10 +6619,10 @@ struct dmub_cmd_pr_copy_settings_data {
 	 * Determines if SMU optimzations are enabled/disabled.
 	 */
 	uint8_t smu_optimizations_en;
-	/*
-	 * Use FSM state for Replay power up/down
+	/**
+	 * Length of each horizontal line in ns.
 	 */
-	uint8_t use_phy_fsm;
+	uint32_t line_time_in_ns;
 	/*
 	 * Use FSFT afftet pixel clk
 	 */
@@ -6631,6 +6635,14 @@ struct dmub_cmd_pr_copy_settings_data {
 	 * Use for AUX-less ALPM LFPS wake operation
 	 */
 	struct dmub_alpm_auxless_data auxless_alpm_data;
+	/**
+	 * DSC Slice height.
+	 */
+	uint16_t dsc_slice_height;
+	/*
+	 * Use FSM state for Replay power up/down
+	 */
+	uint8_t use_phy_fsm;
 	/**
 	 * @hpo_stream_enc_inst: HPO stream encoder instance
 	 */
@@ -6639,10 +6651,26 @@ struct dmub_cmd_pr_copy_settings_data {
 	 * @hpo_link_enc_inst: HPO link encoder instance
 	 */
 	uint8_t hpo_link_enc_inst;
+	/*
+	 * Selective Update granularity needed.
+	 */
+	uint8_t su_granularity_needed;
+	/*
+	 * Horizontal granularity for Selective Update.
+	 */
+	uint16_t su_x_granularity;
+	/*
+	 * Extended caps of vertical granularity for Selective Update.
+	 */
+	uint16_t su_y_granularity_extended_caps;
+	/*
+	 * Vertical granularity for Selective Update.
+	 */
+	uint8_t su_y_granularity;
 	/**
 	 * @pad: Align structure to 4 byte boundary.
 	 */
-	uint8_t pad[2];
+	uint8_t pad;
 };
 
 /**
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c
index a6ae1d2e9685..be893531ae7d 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c
@@ -134,7 +134,7 @@ dmub_get_fw_meta_info_from_blob(const uint8_t *blob, uint32_t blob_size, uint32_
 }
 
 static const struct dmub_fw_meta_info *
-dmub_get_fw_meta_info(const struct dmub_srv_region_params *params)
+dmub_get_fw_meta_info(const struct dmub_srv_fw_meta_info_params *params)
 {
 	const struct dmub_fw_meta_info *info = NULL;
 
@@ -159,6 +159,46 @@ dmub_get_fw_meta_info(const struct dmub_srv_region_params *params)
 	return info;
 }
 
+enum dmub_status
+dmub_srv_get_fw_meta_info_from_raw_fw(struct dmub_srv_fw_meta_info_params *params,
+				      struct dmub_fw_meta_info *fw_info_out)
+{
+	const struct dmub_fw_meta_info *fw_info = NULL;
+	uint32_t inst_const_size_temp = params->inst_const_size;
+
+	/* First try custom psp footer size, if present */
+	if (params->custom_psp_footer_size) {
+		params->inst_const_size -= params->custom_psp_footer_size;
+		fw_info = dmub_get_fw_meta_info(params);
+		if (fw_info) {
+			memcpy(fw_info_out, fw_info, sizeof(*fw_info));
+			return DMUB_STATUS_OK;
+		}
+		params->inst_const_size = inst_const_size_temp;
+	}
+
+	/* Try 256-byte psp footer size */
+	params->inst_const_size -= PSP_FOOTER_BYTES_256;
+	fw_info = dmub_get_fw_meta_info(params);
+	if (fw_info) {
+		memcpy(fw_info_out, fw_info, sizeof(*fw_info));
+		return DMUB_STATUS_OK;
+	}
+
+	/* Try 512-byte psp footer size - final attempt */
+	params->inst_const_size -= PSP_FOOTER_BYTES_256; // 256 bytes already subtracted, subtract 256 again
+	fw_info = dmub_get_fw_meta_info(params);
+	if (fw_info) {
+		memcpy(fw_info_out, fw_info, sizeof(*fw_info));
+		return DMUB_STATUS_OK;
+	}
+
+	/* Restore original inst_const_size and subtract default PSP footer size - default behaviour */
+	params->inst_const_size = inst_const_size_temp - PSP_FOOTER_BYTES_256;
+
+	return DMUB_STATUS_INVALID;
+}
+
 static bool dmub_srv_hw_setup(struct dmub_srv *dmub, enum dmub_asic asic)
 {
 	struct dmub_srv_hw_funcs *funcs = &dmub->hw_funcs;
@@ -524,7 +564,6 @@ enum dmub_status
 		const struct dmub_srv_region_params *params,
 		struct dmub_srv_region_info *out)
 {
-	const struct dmub_fw_meta_info *fw_info;
 	uint32_t fw_state_size = DMUB_FW_STATE_SIZE;
 	uint32_t trace_buffer_size = DMUB_TRACE_BUFFER_SIZE;
 	uint32_t shared_state_size = DMUB_FW_HEADER_SHARED_STATE_SIZE;
@@ -538,14 +577,12 @@ enum dmub_status
 
 	out->num_regions = DMUB_NUM_WINDOWS;
 
-	fw_info = dmub_get_fw_meta_info(params);
-
-	if (fw_info) {
-		memcpy(&dmub->meta_info, fw_info, sizeof(*fw_info));
+	if (params->fw_info) {
+		memcpy(&dmub->meta_info, params->fw_info, sizeof(*params->fw_info));
 
-		fw_state_size = fw_info->fw_region_size;
-		trace_buffer_size = fw_info->trace_buffer_size;
-		shared_state_size = fw_info->shared_state_size;
+		fw_state_size = params->fw_info->fw_region_size;
+		trace_buffer_size = params->fw_info->trace_buffer_size;
+		shared_state_size = params->fw_info->shared_state_size;
 
 		/**
 		 * If DM didn't fill in a version, then fill it in based on
@@ -555,7 +592,7 @@ enum dmub_status
 		 * pass during creation.
 		 */
 		if (dmub->fw_version == 0)
-			dmub->fw_version = fw_info->fw_version;
+			dmub->fw_version = params->fw_info->fw_version;
 	}
 
 	window_sizes[DMUB_WINDOW_0_INST_CONST] = params->inst_const_size;
diff --git a/drivers/gpu/drm/amd/display/include/dpcd_defs.h b/drivers/gpu/drm/amd/display/include/dpcd_defs.h
index 07b937b92efc..8445c540f042 100644
--- a/drivers/gpu/drm/amd/display/include/dpcd_defs.h
+++ b/drivers/gpu/drm/amd/display/include/dpcd_defs.h
@@ -188,6 +188,7 @@ enum dpcd_psr_sink_states {
 #define DP_SOURCE_BACKLIGHT_CURRENT_PEAK        0x326
 #define DP_SOURCE_BACKLIGHT_CONTROL             0x32E
 #define DP_SOURCE_BACKLIGHT_ENABLE              0x32F
+#define DP_SINK_DRR_GRANULARITY                 0x33B
 #define DP_SOURCE_MINIMUM_HBLANK_SUPPORTED      0x340
 #define DP_SINK_PR_REPLAY_STATUS                0x378
 #define DP_SINK_PR_PIXEL_DEVIATION_PER_LINE     0x379
diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h
index 66dc9a19aebe..ddd64b7e4c04 100644
--- a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h
+++ b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h
@@ -33,6 +33,12 @@ struct dc_stream_state;
 struct dc_info_packet;
 struct mod_vrr_params;
 
+void set_vsc_packet_colorimetry_data(
+		const struct dc_stream_state *stream,
+		struct dc_info_packet *info_packet,
+		enum dc_color_space cs,
+		enum color_transfer_func tf);
+
 void mod_build_vsc_infopacket(const struct dc_stream_state *stream,
 		struct dc_info_packet *info_packet,
 		enum dc_color_space cs,
diff --git a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
index b3d55cac3569..00473c6284d5 100644
--- a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
+++ b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
@@ -42,6 +42,10 @@ enum vsc_packet_revision {
 	vsc_packet_rev4 = 4,
 	//05h = 3D stereo + PSR/PSR2 + Y-coordinate + Pixel Encoding/Colorimetry Format
 	vsc_packet_rev5 = 5,
+	//06h = 3D stereo + PR + Y-coordinate
+	vsc_packet_rev6 = 6,
+	//07h = 3D stereo + PR + Y-coordinate + Pixel Encoding/Colorimetry Format
+	vsc_packet_rev7 = 7,
 };
 
 #define HDMI_INFOFRAME_TYPE_VENDOR 0x81
@@ -130,6 +134,163 @@ enum ColorimetryYCCDP {
 	ColorimetryYCC_DP_ITU2020YCbCr  = 7,
 };
 
+/* Helper function to set VSC packet colorimetry data */
+void set_vsc_packet_colorimetry_data(
+		const struct dc_stream_state *stream,
+		struct dc_info_packet *info_packet,
+		enum dc_color_space cs,
+		enum color_transfer_func tf)
+{
+	/* Set VSC SDP fields for pixel encoding and colorimetry format from DP 1.3 specs
+	 * Data Bytes DB 18~16
+	 * Bits 3:0 (Colorimetry Format)        |  Bits 7:4 (Pixel Encoding)
+	 * ----------------------------------------------------------------------------------------------------
+	 * 0x0 = sRGB                           |  0 = RGB
+	 * 0x1 = RGB Wide Gamut Fixed Point
+	 * 0x2 = RGB Wide Gamut Floating Point
+	 * 0x3 = AdobeRGB
+	 * 0x4 = DCI-P3
+	 * 0x5 = CustomColorProfile
+	 * (others reserved)
+	 * ----------------------------------------------------------------------------------------------------
+	 * 0x0 = ITU-R BT.601                   |  1 = YCbCr444
+	 * 0x1 = ITU-R BT.709
+	 * 0x2 = xvYCC601
+	 * 0x3 = xvYCC709
+	 * 0x4 = sYCC601
+	 * 0x5 = AdobeYCC601
+	 * 0x6 = ITU-R BT.2020 Y'cC'bcC'rc
+	 * 0x7 = ITU-R BT.2020 Y'C'bC'r
+	 * (others reserved)
+	 * ----------------------------------------------------------------------------------------------------
+	 * 0x0 = ITU-R BT.601                   |  2 = YCbCr422
+	 * 0x1 = ITU-R BT.709
+	 * 0x2 = xvYCC601
+	 * 0x3 = xvYCC709
+	 * 0x4 = sYCC601
+	 * 0x5 = AdobeYCC601
+	 * 0x6 = ITU-R BT.2020 Y'cC'bcC'rc
+	 * 0x7 = ITU-R BT.2020 Y'C'bC'r
+	 * (others reserved)
+	 * ----------------------------------------------------------------------------------------------------
+	 * 0x0 = ITU-R BT.601                   |  3 = YCbCr420
+	 * 0x1 = ITU-R BT.709
+	 * 0x2 = xvYCC601
+	 * 0x3 = xvYCC709
+	 * 0x4 = sYCC601
+	 * 0x5 = AdobeYCC601
+	 * 0x6 = ITU-R BT.2020 Y'cC'bcC'rc
+	 * 0x7 = ITU-R BT.2020 Y'C'bC'r
+	 * (others reserved)
+	 * ----------------------------------------------------------------------------------------------------
+	 * 0x0 =DICOM Part14 Grayscale          |  4 = Yonly
+	 * Display Function
+	 * (others reserved)
+	 */
+	unsigned int pixelEncoding = 0;
+	unsigned int colorimetryFormat = 0;
+
+	/* Set Pixel Encoding */
+	switch (stream->timing.pixel_encoding) {
+	case PIXEL_ENCODING_RGB:
+		pixelEncoding = 0x0;  /* RGB = 0h */
+		break;
+	case PIXEL_ENCODING_YCBCR444:
+		pixelEncoding = 0x1;  /* YCbCr444 = 1h */
+		break;
+	case PIXEL_ENCODING_YCBCR422:
+		pixelEncoding = 0x2;  /* YCbCr422 = 2h */
+		break;
+	case PIXEL_ENCODING_YCBCR420:
+		pixelEncoding = 0x3;  /* YCbCr420 = 3h */
+		break;
+	default:
+		pixelEncoding = 0x0;  /* default RGB = 0h */
+		break;
+	}
+
+	/* Set Colorimetry format based on pixel encoding */
+	switch (stream->timing.pixel_encoding) {
+	case PIXEL_ENCODING_RGB:
+		if ((cs == COLOR_SPACE_SRGB) ||
+				(cs == COLOR_SPACE_SRGB_LIMITED))
+			colorimetryFormat = ColorimetryRGB_DP_sRGB;
+		else if (cs == COLOR_SPACE_ADOBERGB)
+			colorimetryFormat = ColorimetryRGB_DP_AdobeRGB;
+		else if ((cs == COLOR_SPACE_2020_RGB_FULLRANGE) ||
+				(cs == COLOR_SPACE_2020_RGB_LIMITEDRANGE))
+			colorimetryFormat = ColorimetryRGB_DP_ITU_R_BT2020RGB;
+		break;
+
+	case PIXEL_ENCODING_YCBCR444:
+	case PIXEL_ENCODING_YCBCR422:
+	case PIXEL_ENCODING_YCBCR420:
+		/* Note: xvYCC probably not supported correctly here on DP since colorspace translation
+		 * loses distinction between BT601 vs xvYCC601 in translation
+		 */
+		if (cs == COLOR_SPACE_YCBCR601)
+			colorimetryFormat = ColorimetryYCC_DP_ITU601;
+		else if (cs == COLOR_SPACE_YCBCR709)
+			colorimetryFormat = ColorimetryYCC_DP_ITU709;
+		else if (cs == COLOR_SPACE_ADOBERGB)
+			colorimetryFormat = ColorimetryYCC_DP_AdobeYCC;
+		else if (cs == COLOR_SPACE_2020_YCBCR_LIMITED)
+			colorimetryFormat = ColorimetryYCC_DP_ITU2020YCbCr;
+
+		if (cs == COLOR_SPACE_2020_YCBCR_LIMITED && tf == TRANSFER_FUNC_GAMMA_22)
+			colorimetryFormat = ColorimetryYCC_DP_ITU709;
+		break;
+
+	default:
+		colorimetryFormat = ColorimetryRGB_DP_sRGB;
+		break;
+	}
+
+	info_packet->sb[16] = (pixelEncoding << 4) | colorimetryFormat;
+
+	/* Set color depth */
+	switch (stream->timing.display_color_depth) {
+	case COLOR_DEPTH_666:
+		/* NOTE: This is actually not valid for YCbCr pixel encoding to have 6 bpc
+		 *       as of DP1.4 spec, but value of 0 probably reserved here for potential future use.
+		 */
+		info_packet->sb[17] = 0;
+		break;
+	case COLOR_DEPTH_888:
+		info_packet->sb[17] = 1;
+		break;
+	case COLOR_DEPTH_101010:
+		info_packet->sb[17] = 2;
+		break;
+	case COLOR_DEPTH_121212:
+		info_packet->sb[17] = 3;
+		break;
+	/*case COLOR_DEPTH_141414: -- NO SUCH FORMAT IN DP SPEC */
+	case COLOR_DEPTH_161616:
+		info_packet->sb[17] = 4;
+		break;
+	default:
+		info_packet->sb[17] = 0;
+		break;
+	}
+
+	/* all YCbCr are always limited range */
+	if ((cs == COLOR_SPACE_SRGB_LIMITED) ||
+			(cs == COLOR_SPACE_2020_RGB_LIMITEDRANGE) ||
+			(pixelEncoding != 0x0)) {
+		info_packet->sb[17] |= 0x80; /* DB17 bit 7 set to 1 for CEA timing. */
+	}
+
+	/* Content Type (Bits 2:0)
+	 *  0 = Not defined.
+	 *  1 = Graphics.
+	 *  2 = Photo.
+	 *  3 = Video.
+	 *  4 = Game.
+	 */
+	info_packet->sb[18] = 0;
+}
+
 void mod_build_vsc_infopacket(const struct dc_stream_state *stream,
 		struct dc_info_packet *info_packet,
 		enum dc_color_space cs,
@@ -137,8 +298,6 @@ void mod_build_vsc_infopacket(const struct dc_stream_state *stream,
 {
 	unsigned int vsc_packet_revision = vsc_packet_undefined;
 	unsigned int i;
-	unsigned int pixelEncoding = 0;
-	unsigned int colorimetryFormat = 0;
 	bool stereo3dSupport = false;
 
 	if (stream->timing.timing_3d_format != TIMING_3D_FORMAT_NONE && stream->view_format != VIEW_3D_FORMAT_NONE) {
@@ -158,12 +317,38 @@ void mod_build_vsc_infopacket(const struct dc_stream_state *stream,
 	if (stream->use_vsc_sdp_for_colorimetry)
 		vsc_packet_revision = vsc_packet_rev5;
 
+	/* Check for Panel Replay (highest priority) */
+	if (stream->link->replay_settings.config.replay_version == DC_VESA_PANEL_REPLAY)
+		vsc_packet_revision = stream->use_vsc_sdp_for_colorimetry ? vsc_packet_rev7 : vsc_packet_rev6;
+
 	/* VSC packet not needed based on the features
 	 * supported by this DP display
 	 */
 	if (vsc_packet_revision == vsc_packet_undefined)
 		return;
 
+	if (vsc_packet_revision == vsc_packet_rev6) {
+		/* Secondary-data Packet ID = 0*/
+		info_packet->hb0 = 0x00;
+		/* 07h - Packet Type Value indicating Video
+		 * Stream Configuration packet
+		 */
+		info_packet->hb1 = 0x07;
+		/* 06h = VSC SDP supporting 3D stereo + PR
+		 */
+		info_packet->hb2 = 0x06;
+		/* 0Eh = VSC SDP supporting 3D stereo + PR
+		 * (HB2 = 06h), with Y-coordinate of first scan
+		 * line of the SU region
+		 */
+		info_packet->hb3 = 0x10;
+
+		for (i = 0; i < 28; i++)
+			info_packet->sb[i] = 0;
+
+		info_packet->valid = true;
+	}
+
 	if (vsc_packet_revision == vsc_packet_rev4) {
 		/* Secondary-data Packet ID = 0*/
 		info_packet->hb0 = 0x00;
@@ -292,152 +477,22 @@ void mod_build_vsc_infopacket(const struct dc_stream_state *stream,
 
 		info_packet->valid = true;
 
-		/* Set VSC SDP fields for pixel encoding and colorimetry format from DP 1.3 specs
-		 * Data Bytes DB 18~16
-		 * Bits 3:0 (Colorimetry Format)        |  Bits 7:4 (Pixel Encoding)
-		 * ----------------------------------------------------------------------------------------------------
-		 * 0x0 = sRGB                           |  0 = RGB
-		 * 0x1 = RGB Wide Gamut Fixed Point
-		 * 0x2 = RGB Wide Gamut Floating Point
-		 * 0x3 = AdobeRGB
-		 * 0x4 = DCI-P3
-		 * 0x5 = CustomColorProfile
-		 * (others reserved)
-		 * ----------------------------------------------------------------------------------------------------
-		 * 0x0 = ITU-R BT.601                   |  1 = YCbCr444
-		 * 0x1 = ITU-R BT.709
-		 * 0x2 = xvYCC601
-		 * 0x3 = xvYCC709
-		 * 0x4 = sYCC601
-		 * 0x5 = AdobeYCC601
-		 * 0x6 = ITU-R BT.2020 Y'cC'bcC'rc
-		 * 0x7 = ITU-R BT.2020 Y'C'bC'r
-		 * (others reserved)
-		 * ----------------------------------------------------------------------------------------------------
-		 * 0x0 = ITU-R BT.601                   |  2 = YCbCr422
-		 * 0x1 = ITU-R BT.709
-		 * 0x2 = xvYCC601
-		 * 0x3 = xvYCC709
-		 * 0x4 = sYCC601
-		 * 0x5 = AdobeYCC601
-		 * 0x6 = ITU-R BT.2020 Y'cC'bcC'rc
-		 * 0x7 = ITU-R BT.2020 Y'C'bC'r
-		 * (others reserved)
-		 * ----------------------------------------------------------------------------------------------------
-		 * 0x0 = ITU-R BT.601                   |  3 = YCbCr420
-		 * 0x1 = ITU-R BT.709
-		 * 0x2 = xvYCC601
-		 * 0x3 = xvYCC709
-		 * 0x4 = sYCC601
-		 * 0x5 = AdobeYCC601
-		 * 0x6 = ITU-R BT.2020 Y'cC'bcC'rc
-		 * 0x7 = ITU-R BT.2020 Y'C'bC'r
-		 * (others reserved)
-		 * ----------------------------------------------------------------------------------------------------
-		 * 0x0 =DICOM Part14 Grayscale          |  4 = Yonly
-		 * Display Function
-		 * (others reserved)
-		 */
-
-		/* Set Pixel Encoding */
-		switch (stream->timing.pixel_encoding) {
-		case PIXEL_ENCODING_RGB:
-			pixelEncoding = 0x0;  /* RGB = 0h */
-			break;
-		case PIXEL_ENCODING_YCBCR444:
-			pixelEncoding = 0x1;  /* YCbCr444 = 1h */
-			break;
-		case PIXEL_ENCODING_YCBCR422:
-			pixelEncoding = 0x2;  /* YCbCr422 = 2h */
-			break;
-		case PIXEL_ENCODING_YCBCR420:
-			pixelEncoding = 0x3;  /* YCbCr420 = 3h */
-			break;
-		default:
-			pixelEncoding = 0x0;  /* default RGB = 0h */
-			break;
-		}
-
-		/* Set Colorimetry format based on pixel encoding */
-		switch (stream->timing.pixel_encoding) {
-		case PIXEL_ENCODING_RGB:
-			if ((cs == COLOR_SPACE_SRGB) ||
-					(cs == COLOR_SPACE_SRGB_LIMITED))
-				colorimetryFormat = ColorimetryRGB_DP_sRGB;
-			else if (cs == COLOR_SPACE_ADOBERGB)
-				colorimetryFormat = ColorimetryRGB_DP_AdobeRGB;
-			else if ((cs == COLOR_SPACE_2020_RGB_FULLRANGE) ||
-					(cs == COLOR_SPACE_2020_RGB_LIMITEDRANGE))
-				colorimetryFormat = ColorimetryRGB_DP_ITU_R_BT2020RGB;
-			break;
-
-		case PIXEL_ENCODING_YCBCR444:
-		case PIXEL_ENCODING_YCBCR422:
-		case PIXEL_ENCODING_YCBCR420:
-			/* Note: xvYCC probably not supported correctly here on DP since colorspace translation
-			 * loses distinction between BT601 vs xvYCC601 in translation
-			 */
-			if (cs == COLOR_SPACE_YCBCR601)
-				colorimetryFormat = ColorimetryYCC_DP_ITU601;
-			else if (cs == COLOR_SPACE_YCBCR709)
-				colorimetryFormat = ColorimetryYCC_DP_ITU709;
-			else if (cs == COLOR_SPACE_ADOBERGB)
-				colorimetryFormat = ColorimetryYCC_DP_AdobeYCC;
-			else if (cs == COLOR_SPACE_2020_YCBCR_LIMITED)
-				colorimetryFormat = ColorimetryYCC_DP_ITU2020YCbCr;
-
-			if (cs == COLOR_SPACE_2020_YCBCR_LIMITED && tf == TRANSFER_FUNC_GAMMA_22)
-				colorimetryFormat = ColorimetryYCC_DP_ITU709;
-			break;
-
-		default:
-			colorimetryFormat = ColorimetryRGB_DP_sRGB;
-			break;
-		}
-
-		info_packet->sb[16] = (pixelEncoding << 4) | colorimetryFormat;
+		set_vsc_packet_colorimetry_data(stream, info_packet, cs, tf);
+	}
 
-		/* Set color depth */
-		switch (stream->timing.display_color_depth) {
-		case COLOR_DEPTH_666:
-			/* NOTE: This is actually not valid for YCbCr pixel encoding to have 6 bpc
-			 *       as of DP1.4 spec, but value of 0 probably reserved here for potential future use.
-			 */
-			info_packet->sb[17] = 0;
-			break;
-		case COLOR_DEPTH_888:
-			info_packet->sb[17] = 1;
-			break;
-		case COLOR_DEPTH_101010:
-			info_packet->sb[17] = 2;
-			break;
-		case COLOR_DEPTH_121212:
-			info_packet->sb[17] = 3;
-			break;
-		/*case COLOR_DEPTH_141414: -- NO SUCH FORMAT IN DP SPEC */
-		case COLOR_DEPTH_161616:
-			info_packet->sb[17] = 4;
-			break;
-		default:
-			info_packet->sb[17] = 0;
-			break;
-		}
+	if (vsc_packet_revision == vsc_packet_rev7) {
+		/* Secondary-data Packet ID = 0 */
+		info_packet->hb0 = 0x00;
+		/* 07h - Packet Type Value indicating Video Stream Configuration packet */
+		info_packet->hb1 = 0x07;
+		/* 07h = VSC SDP supporting 3D stereo, PR, and Pixel Encoding/Colorimetry Format indication. */
+		info_packet->hb2 = 0x07;
+		/* 13h = VSC SDP supporting 3D stereo, + PR, + Pixel Encoding/Colorimetry Format indication (HB2 = 07h). */
+		info_packet->hb3 = 0x13;
 
-		/* all YCbCr are always limited range */
-		if ((cs == COLOR_SPACE_SRGB_LIMITED) ||
-				(cs == COLOR_SPACE_2020_RGB_LIMITEDRANGE) ||
-				(pixelEncoding != 0x0)) {
-			info_packet->sb[17] |= 0x80; /* DB17 bit 7 set to 1 for CEA timing. */
-		}
+		info_packet->valid = true;
 
-		/* Content Type (Bits 2:0)
-		 *  0 = Not defined.
-		 *  1 = Graphics.
-		 *  2 = Photo.
-		 *  3 = Video.
-		 *  4 = Game.
-		 */
-		info_packet->sb[18] = 0;
+		set_vsc_packet_colorimetry_data(stream, info_packet, cs, tf);
 	}
 }
 
@@ -537,7 +592,11 @@ void mod_build_adaptive_sync_infopacket(const struct dc_stream_state *stream,
 		break;
 	case FREESYNC_TYPE_PCON_IN_WHITELIST:
 	case ADAPTIVE_SYNC_TYPE_EDP:
-		mod_build_adaptive_sync_infopacket_v1(info_packet);
+		if (stream && stream->link->replay_settings.config.replay_supported &&
+			stream->link->replay_settings.config.replay_version == DC_VESA_PANEL_REPLAY)
+			mod_build_adaptive_sync_infopacket_v2(stream, param, info_packet);
+		else
+			mod_build_adaptive_sync_infopacket_v1(info_packet);
 		break;
 	case ADAPTIVE_SYNC_TYPE_NONE:
 	case FREESYNC_TYPE_PCON_NOT_IN_WHITELIST:
@@ -567,13 +626,15 @@ void mod_build_adaptive_sync_infopacket_v2(const struct dc_stream_state *stream,
 	info_packet->hb2 = AS_SDP_VER_2;
 	info_packet->hb3 = AS_DP_SDP_LENGTH;
 
-	//Payload
-	info_packet->sb[0] = param->supportMode; //1: AVT; 0: FAVT
-	info_packet->sb[1] = (stream->timing.v_total & 0x00FF);
-	info_packet->sb[2] = (stream->timing.v_total & 0xFF00) >> 8;
-	//info_packet->sb[3] = 0x00; Target RR, not use fot AVT
-	info_packet->sb[4] = (param->increase.support << 6 | param->decrease.support << 7);
-	info_packet->sb[5] = param->increase.frame_duration_hex;
-	info_packet->sb[6] = param->decrease.frame_duration_hex;
+	if (param) {
+		//Payload
+		info_packet->sb[0] = param->supportMode; //1: AVT; 0: FAVT
+		info_packet->sb[1] = (stream->timing.v_total & 0x00FF);
+		info_packet->sb[2] = (stream->timing.v_total & 0xFF00) >> 8;
+		//info_packet->sb[3] = 0x00; Target RR, not use fot AVT
+		info_packet->sb[4] = (param->increase.support << 6 | param->decrease.support << 7);
+		info_packet->sb[5] = param->increase.frame_duration_hex;
+		info_packet->sb[6] = param->decrease.frame_duration_hex;
+	}
 }
 
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_offset.h
new file mode 100644
index 000000000000..69731dcc324a
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_offset.h
@@ -0,0 +1,12407 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _gc_12_1_0_OFFSET_HEADER
+#define _gc_12_1_0_OFFSET_HEADER
+
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmadec
+// base address: 0x4980
+#define regSDMA0_SDMA_DEC_START                                                                         0x0000
+#define regSDMA0_SDMA_DEC_START_BASE_IDX                                                                0
+#define regSDMA0_SDMA_MCU_MISC_CNTL                                                                     0x0001
+#define regSDMA0_SDMA_MCU_MISC_CNTL_BASE_IDX                                                            0
+#define regSDMA0_SDMA_UCODE_REV                                                                         0x0003
+#define regSDMA0_SDMA_UCODE_REV_BASE_IDX                                                                0
+#define regSDMA0_SDMA_GLOBAL_TIMESTAMP_LO                                                               0x0005
+#define regSDMA0_SDMA_GLOBAL_TIMESTAMP_LO_BASE_IDX                                                      0
+#define regSDMA0_SDMA_GLOBAL_TIMESTAMP_HI                                                               0x0006
+#define regSDMA0_SDMA_GLOBAL_TIMESTAMP_HI_BASE_IDX                                                      0
+#define regSDMA0_SDMA_POWER_CNTL                                                                        0x000c
+#define regSDMA0_SDMA_POWER_CNTL_BASE_IDX                                                               0
+#define regSDMA0_SDMA_CNTL                                                                              0x000d
+#define regSDMA0_SDMA_CNTL_BASE_IDX                                                                     0
+#define regSDMA0_SDMA_CHICKEN_BITS                                                                      0x000e
+#define regSDMA0_SDMA_CHICKEN_BITS_BASE_IDX                                                             0
+#define regSDMA0_SDMA_CACHE_CNTL                                                                        0x000f
+#define regSDMA0_SDMA_CACHE_CNTL_BASE_IDX                                                               0
+#define regSDMA0_SDMA_RB_RPTR_FETCH                                                                     0x0020
+#define regSDMA0_SDMA_RB_RPTR_FETCH_BASE_IDX                                                            0
+#define regSDMA0_SDMA_RB_RPTR_FETCH_HI                                                                  0x0021
+#define regSDMA0_SDMA_RB_RPTR_FETCH_HI_BASE_IDX                                                         0
+#define regSDMA0_SDMA_IB_OFFSET_FETCH                                                                   0x0022
+#define regSDMA0_SDMA_IB_OFFSET_FETCH_BASE_IDX                                                          0
+#define regSDMA0_SDMA_PROGRAM                                                                           0x0023
+#define regSDMA0_SDMA_PROGRAM_BASE_IDX                                                                  0
+#define regSDMA0_SDMA_STATUS_REG                                                                        0x0024
+#define regSDMA0_SDMA_STATUS_REG_BASE_IDX                                                               0
+#define regSDMA0_SDMA_STATUS1_REG                                                                       0x0025
+#define regSDMA0_SDMA_STATUS1_REG_BASE_IDX                                                              0
+#define regSDMA0_SDMA_CNTL1                                                                             0x0026
+#define regSDMA0_SDMA_CNTL1_BASE_IDX                                                                    0
+#define regSDMA0_SDMA_HBM_PAGE_CONFIG                                                                   0x0027
+#define regSDMA0_SDMA_HBM_PAGE_CONFIG_BASE_IDX                                                          0
+#define regSDMA0_SDMA_FREEZE                                                                            0x0028
+#define regSDMA0_SDMA_FREEZE_BASE_IDX                                                                   0
+#define regSDMA0_SDMA_PROCESS_QUANTUM0                                                                  0x0029
+#define regSDMA0_SDMA_PROCESS_QUANTUM0_BASE_IDX                                                         0
+#define regSDMA0_SDMA_PROCESS_QUANTUM1                                                                  0x002a
+#define regSDMA0_SDMA_PROCESS_QUANTUM1_BASE_IDX                                                         0
+#define regSDMA0_SDMA_WATCHDOG_CNTL                                                                     0x002b
+#define regSDMA0_SDMA_WATCHDOG_CNTL_BASE_IDX                                                            0
+#define regSDMA0_SDMA_QUEUE_STATUS0                                                                     0x002c
+#define regSDMA0_SDMA_QUEUE_STATUS0_BASE_IDX                                                            0
+#define regSDMA0_SDMA_QUEUE_STATUS1                                                                     0x002d
+#define regSDMA0_SDMA_QUEUE_STATUS1_BASE_IDX                                                            0
+#define regSDMA0_SDMA_ID                                                                                0x002e
+#define regSDMA0_SDMA_ID_BASE_IDX                                                                       0
+#define regSDMA0_SDMA_VERSION                                                                           0x002f
+#define regSDMA0_SDMA_VERSION_BASE_IDX                                                                  0
+#define regSDMA0_SDMA_STATUS2_REG                                                                       0x0030
+#define regSDMA0_SDMA_STATUS2_REG_BASE_IDX                                                              0
+#define regSDMA0_SDMA_ATOMIC_CNTL                                                                       0x0031
+#define regSDMA0_SDMA_ATOMIC_CNTL_BASE_IDX                                                              0
+#define regSDMA0_SDMA_ATOMIC_PERMS_CNTL                                                                 0x0032
+#define regSDMA0_SDMA_ATOMIC_PERMS_CNTL_BASE_IDX                                                        0
+#define regSDMA0_SDMA_ATOMIC_PREOP_LO                                                                   0x0033
+#define regSDMA0_SDMA_ATOMIC_PREOP_LO_BASE_IDX                                                          0
+#define regSDMA0_SDMA_ATOMIC_PREOP_HI                                                                   0x0034
+#define regSDMA0_SDMA_ATOMIC_PREOP_HI_BASE_IDX                                                          0
+#define regSDMA0_SDMA_DCC_CNTL                                                                          0x0035
+#define regSDMA0_SDMA_DCC_CNTL_BASE_IDX                                                                 0
+#define regSDMA0_SDMA_UTCL1_WARMUPL2_CNTL                                                               0x0036
+#define regSDMA0_SDMA_UTCL1_WARMUPL2_CNTL_BASE_IDX                                                      0
+#define regSDMA0_SDMA_UTCL1_CNTL                                                                        0x0037
+#define regSDMA0_SDMA_UTCL1_CNTL_BASE_IDX                                                               0
+#define regSDMA0_SDMA_UTCL1_WATERMK                                                                     0x0038
+#define regSDMA0_SDMA_UTCL1_WATERMK_BASE_IDX                                                            0
+#define regSDMA0_SDMA_UTCL1_TIMEOUT                                                                     0x0039
+#define regSDMA0_SDMA_UTCL1_TIMEOUT_BASE_IDX                                                            0
+#define regSDMA0_SDMA_UTCL1_PAGE                                                                        0x003a
+#define regSDMA0_SDMA_UTCL1_PAGE_BASE_IDX                                                               0
+#define regSDMA0_SDMA_EXTERNAL_FROZEN                                                                   0x003b
+#define regSDMA0_SDMA_EXTERNAL_FROZEN_BASE_IDX                                                          0
+#define regSDMA0_SDMA_FREEZE_TRIGGER                                                                    0x003c
+#define regSDMA0_SDMA_FREEZE_TRIGGER_BASE_IDX                                                           0
+#define regSDMA0_SDMA_UTCL1_CACHE_CNTL                                                                  0x0040
+#define regSDMA0_SDMA_UTCL1_CACHE_CNTL_BASE_IDX                                                         0
+#define regSDMA0_SDMA_UTCL1_RD_STATUS                                                                   0x0041
+#define regSDMA0_SDMA_UTCL1_RD_STATUS_BASE_IDX                                                          0
+#define regSDMA0_SDMA_UTCL1_WR_STATUS                                                                   0x0042
+#define regSDMA0_SDMA_UTCL1_WR_STATUS_BASE_IDX                                                          0
+#define regSDMA0_SDMA_UTCL1_INV0                                                                        0x0043
+#define regSDMA0_SDMA_UTCL1_INV0_BASE_IDX                                                               0
+#define regSDMA0_SDMA_UTCL1_INV1                                                                        0x0044
+#define regSDMA0_SDMA_UTCL1_INV1_BASE_IDX                                                               0
+#define regSDMA0_SDMA_UTCL1_INV2                                                                        0x0045
+#define regSDMA0_SDMA_UTCL1_INV2_BASE_IDX                                                               0
+#define regSDMA0_SDMA_UTCL1_RD_XNACK0                                                                   0x0046
+#define regSDMA0_SDMA_UTCL1_RD_XNACK0_BASE_IDX                                                          0
+#define regSDMA0_SDMA_UTCL1_RD_XNACK1                                                                   0x0047
+#define regSDMA0_SDMA_UTCL1_RD_XNACK1_BASE_IDX                                                          0
+#define regSDMA0_SDMA_UTCL1_WR_XNACK0                                                                   0x0048
+#define regSDMA0_SDMA_UTCL1_WR_XNACK0_BASE_IDX                                                          0
+#define regSDMA0_SDMA_UTCL1_WR_XNACK1                                                                   0x0049
+#define regSDMA0_SDMA_UTCL1_WR_XNACK1_BASE_IDX                                                          0
+#define regSDMA0_SDMA_CHICKEN_BITS_2                                                                    0x004a
+#define regSDMA0_SDMA_CHICKEN_BITS_2_BASE_IDX                                                           0
+#define regSDMA0_SDMA_STATUS3_REG                                                                       0x004b
+#define regSDMA0_SDMA_STATUS3_REG_BASE_IDX                                                              0
+#define regSDMA0_SDMA_GLOBAL_QUANTUM                                                                    0x004c
+#define regSDMA0_SDMA_GLOBAL_QUANTUM_BASE_IDX                                                           0
+#define regSDMA0_SDMA_ERROR_LOG                                                                         0x004d
+#define regSDMA0_SDMA_ERROR_LOG_BASE_IDX                                                                0
+#define regSDMA0_SDMA_PUB_DUMMY_REG0                                                                    0x004e
+#define regSDMA0_SDMA_PUB_DUMMY_REG0_BASE_IDX                                                           0
+#define regSDMA0_SDMA_PUB_DUMMY_REG1                                                                    0x004f
+#define regSDMA0_SDMA_PUB_DUMMY_REG1_BASE_IDX                                                           0
+#define regSDMA0_SDMA_PUB_DUMMY_REG2                                                                    0x0050
+#define regSDMA0_SDMA_PUB_DUMMY_REG2_BASE_IDX                                                           0
+#define regSDMA0_SDMA_PUB_DUMMY_REG3                                                                    0x0051
+#define regSDMA0_SDMA_PUB_DUMMY_REG3_BASE_IDX                                                           0
+#define regSDMA0_SDMA_MCU_COUNTER                                                                       0x0052
+#define regSDMA0_SDMA_MCU_COUNTER_BASE_IDX                                                              0
+#define regSDMA0_SDMA_CRD_CNTL                                                                          0x0053
+#define regSDMA0_SDMA_CRD_CNTL_BASE_IDX                                                                 0
+#define regSDMA0_SDMA_RLC_CGCG_CTRL                                                                     0x0054
+#define regSDMA0_SDMA_RLC_CGCG_CTRL_BASE_IDX                                                            0
+#define regSDMA0_SDMA_GPU_IOV_VIOLATION_LOG                                                             0x0055
+#define regSDMA0_SDMA_GPU_IOV_VIOLATION_LOG_BASE_IDX                                                    0
+#define regSDMA0_SDMA_AQL_STATUS                                                                        0x0057
+#define regSDMA0_SDMA_AQL_STATUS_BASE_IDX                                                               0
+#define regSDMA0_SDMA_TLBI_GCR_CNTL                                                                     0x0060
+#define regSDMA0_SDMA_TLBI_GCR_CNTL_BASE_IDX                                                            0
+#define regSDMA0_SDMA_INT_STATUS                                                                        0x0061
+#define regSDMA0_SDMA_INT_STATUS_BASE_IDX                                                               0
+#define regSDMA0_SDMA_GPU_IOV_VIOLATION_LOG2                                                            0x0062
+#define regSDMA0_SDMA_GPU_IOV_VIOLATION_LOG2_BASE_IDX                                                   0
+#define regSDMA0_SDMA_INVALID_ADDR_LO                                                                   0x0063
+#define regSDMA0_SDMA_INVALID_ADDR_LO_BASE_IDX                                                          0
+#define regSDMA0_SDMA_INVALID_ADDR_HI                                                                   0x0064
+#define regSDMA0_SDMA_INVALID_ADDR_HI_BASE_IDX                                                          0
+#define regSDMA0_SDMA_INVALID_ADDR_SOURCE                                                               0x0065
+#define regSDMA0_SDMA_INVALID_ADDR_SOURCE_BASE_IDX                                                      0
+#define regSDMA0_SDMA_CLOCK_GATING_STATUS                                                               0x0066
+#define regSDMA0_SDMA_CLOCK_GATING_STATUS_BASE_IDX                                                      0
+#define regSDMA0_SDMA_STATUS4_REG                                                                       0x0067
+#define regSDMA0_SDMA_STATUS4_REG_BASE_IDX                                                              0
+#define regSDMA0_SDMA_SCRATCH_RAM_DATA                                                                  0x0068
+#define regSDMA0_SDMA_SCRATCH_RAM_DATA_BASE_IDX                                                         0
+#define regSDMA0_SDMA_SCRATCH_RAM_ADDR                                                                  0x0069
+#define regSDMA0_SDMA_SCRATCH_RAM_ADDR_BASE_IDX                                                         0
+#define regSDMA0_SDMA_TIMESTAMP_CNTL                                                                    0x006a
+#define regSDMA0_SDMA_TIMESTAMP_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_STATUS5_REG                                                                       0x006b
+#define regSDMA0_SDMA_STATUS5_REG_BASE_IDX                                                              0
+#define regSDMA0_SDMA_QUEUE_RESET_REQ                                                                   0x006c
+#define regSDMA0_SDMA_QUEUE_RESET_REQ_BASE_IDX                                                          0
+#define regSDMA0_SDMA_QUEUE_DEQUEUE_REQUEST                                                             0x006d
+#define regSDMA0_SDMA_QUEUE_DEQUEUE_REQUEST_BASE_IDX                                                    0
+#define regSDMA0_SDMA_STATUS6_REG                                                                       0x006e
+#define regSDMA0_SDMA_STATUS6_REG_BASE_IDX                                                              0
+#define regSDMA0_SDMA_STATUS7_REG                                                                       0x006f
+#define regSDMA0_SDMA_STATUS7_REG_BASE_IDX                                                              0
+#define regSDMA0_SDMA_STATUS8_REG                                                                       0x0070
+#define regSDMA0_SDMA_STATUS8_REG_BASE_IDX                                                              0
+#define regSDMA0_SDMA_CE_CTRL                                                                           0x0071
+#define regSDMA0_SDMA_CE_CTRL_BASE_IDX                                                                  0
+#define regSDMA0_SDMA_RAS_STATUS                                                                        0x0072
+#define regSDMA0_SDMA_RAS_STATUS_BASE_IDX                                                               0
+#define regSDMA0_SDMA_POISON_INFO                                                                       0x0073
+#define regSDMA0_SDMA_POISON_INFO_BASE_IDX                                                              0
+#define regSDMA0_SDMA_MEMHUB_CNTL                                                                       0x0074
+#define regSDMA0_SDMA_MEMHUB_CNTL_BASE_IDX                                                              0
+#define regSDMA0_SDMA_QUEUE0_RB_CNTL                                                                    0x0200
+#define regSDMA0_SDMA_QUEUE0_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE0_RB_BASE                                                                    0x0201
+#define regSDMA0_SDMA_QUEUE0_RB_BASE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE0_RB_BASE_HI                                                                 0x0202
+#define regSDMA0_SDMA_QUEUE0_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE0_RB_RPTR                                                                    0x0203
+#define regSDMA0_SDMA_QUEUE0_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE0_RB_RPTR_HI                                                                 0x0204
+#define regSDMA0_SDMA_QUEUE0_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE0_RB_WPTR                                                                    0x0205
+#define regSDMA0_SDMA_QUEUE0_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE0_RB_WPTR_HI                                                                 0x0206
+#define regSDMA0_SDMA_QUEUE0_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE0_RB_RPTR_ADDR_LO                                                            0x0207
+#define regSDMA0_SDMA_QUEUE0_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE0_RB_RPTR_ADDR_HI                                                            0x0208
+#define regSDMA0_SDMA_QUEUE0_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE0_IB_CNTL                                                                    0x0209
+#define regSDMA0_SDMA_QUEUE0_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE0_IB_RPTR                                                                    0x020a
+#define regSDMA0_SDMA_QUEUE0_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE0_IB_OFFSET                                                                  0x020b
+#define regSDMA0_SDMA_QUEUE0_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE0_IB_BASE_LO                                                                 0x020c
+#define regSDMA0_SDMA_QUEUE0_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE0_IB_BASE_HI                                                                 0x020d
+#define regSDMA0_SDMA_QUEUE0_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE0_IB_SIZE                                                                    0x020e
+#define regSDMA0_SDMA_QUEUE0_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE0_DOORBELL                                                                   0x020f
+#define regSDMA0_SDMA_QUEUE0_DOORBELL_BASE_IDX                                                          0
+#define regSDMA0_SDMA_QUEUE0_DOORBELL_LOG                                                               0x0210
+#define regSDMA0_SDMA_QUEUE0_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE0_DOORBELL_OFFSET                                                            0x0211
+#define regSDMA0_SDMA_QUEUE0_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE0_CSA_ADDR_LO                                                                0x0212
+#define regSDMA0_SDMA_QUEUE0_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE0_CSA_ADDR_HI                                                                0x0213
+#define regSDMA0_SDMA_QUEUE0_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE0_SCHEDULE_CNTL                                                              0x0214
+#define regSDMA0_SDMA_QUEUE0_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE0_IB_SUB_REMAIN                                                              0x0215
+#define regSDMA0_SDMA_QUEUE0_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE0_PREEMPT                                                                    0x0216
+#define regSDMA0_SDMA_QUEUE0_PREEMPT_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE0_DUMMY_REG                                                                  0x0217
+#define regSDMA0_SDMA_QUEUE0_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_LO                                                       0x0218
+#define regSDMA0_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_HI                                                       0x0219
+#define regSDMA0_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE0_RB_AQL_CNTL                                                                0x021a
+#define regSDMA0_SDMA_QUEUE0_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE0_MINOR_PTR_UPDATE                                                           0x021b
+#define regSDMA0_SDMA_QUEUE0_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS                                                      0x021e
+#define regSDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_CNTL                                                                0x021f
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA0                                                               0x0220
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA1                                                               0x0221
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA2                                                               0x0222
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA3                                                               0x0223
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA4                                                               0x0224
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA5                                                               0x0225
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA6                                                               0x0226
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA7                                                               0x0227
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA8                                                               0x0228
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA9                                                               0x0229
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA10                                                              0x022a
+#define regSDMA0_SDMA_QUEUE0_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE0_UTILIZATION_LO                                                             0x022b
+#define regSDMA0_SDMA_QUEUE0_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE0_UTILIZATION_HI                                                             0x022c
+#define regSDMA0_SDMA_QUEUE0_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE0_WAIT_UNSATISFIED_THD                                                       0x022d
+#define regSDMA0_SDMA_QUEUE0_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE0_MQD_BASE_ADDR_LO                                                           0x022e
+#define regSDMA0_SDMA_QUEUE0_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE0_MQD_BASE_ADDR_HI                                                           0x022f
+#define regSDMA0_SDMA_QUEUE0_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE0_MQD_CONTROL                                                                0x0230
+#define regSDMA0_SDMA_QUEUE0_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE0_CONTEXT_STATUS                                                             0x0231
+#define regSDMA0_SDMA_QUEUE0_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE0_DUMMY_REG1                                                                 0x0232
+#define regSDMA0_SDMA_QUEUE0_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE0_DUMMY_REG2                                                                 0x0233
+#define regSDMA0_SDMA_QUEUE0_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE1_RB_CNTL                                                                    0x023c
+#define regSDMA0_SDMA_QUEUE1_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE1_RB_BASE                                                                    0x023d
+#define regSDMA0_SDMA_QUEUE1_RB_BASE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE1_RB_BASE_HI                                                                 0x023e
+#define regSDMA0_SDMA_QUEUE1_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE1_RB_RPTR                                                                    0x023f
+#define regSDMA0_SDMA_QUEUE1_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE1_RB_RPTR_HI                                                                 0x0240
+#define regSDMA0_SDMA_QUEUE1_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE1_RB_WPTR                                                                    0x0241
+#define regSDMA0_SDMA_QUEUE1_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE1_RB_WPTR_HI                                                                 0x0242
+#define regSDMA0_SDMA_QUEUE1_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE1_RB_RPTR_ADDR_LO                                                            0x0243
+#define regSDMA0_SDMA_QUEUE1_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE1_RB_RPTR_ADDR_HI                                                            0x0244
+#define regSDMA0_SDMA_QUEUE1_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE1_IB_CNTL                                                                    0x0245
+#define regSDMA0_SDMA_QUEUE1_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE1_IB_RPTR                                                                    0x0246
+#define regSDMA0_SDMA_QUEUE1_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE1_IB_OFFSET                                                                  0x0247
+#define regSDMA0_SDMA_QUEUE1_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE1_IB_BASE_LO                                                                 0x0248
+#define regSDMA0_SDMA_QUEUE1_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE1_IB_BASE_HI                                                                 0x0249
+#define regSDMA0_SDMA_QUEUE1_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE1_IB_SIZE                                                                    0x024a
+#define regSDMA0_SDMA_QUEUE1_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE1_DOORBELL                                                                   0x024b
+#define regSDMA0_SDMA_QUEUE1_DOORBELL_BASE_IDX                                                          0
+#define regSDMA0_SDMA_QUEUE1_DOORBELL_LOG                                                               0x024c
+#define regSDMA0_SDMA_QUEUE1_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE1_DOORBELL_OFFSET                                                            0x024d
+#define regSDMA0_SDMA_QUEUE1_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE1_CSA_ADDR_LO                                                                0x024e
+#define regSDMA0_SDMA_QUEUE1_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE1_CSA_ADDR_HI                                                                0x024f
+#define regSDMA0_SDMA_QUEUE1_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE1_SCHEDULE_CNTL                                                              0x0250
+#define regSDMA0_SDMA_QUEUE1_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE1_IB_SUB_REMAIN                                                              0x0251
+#define regSDMA0_SDMA_QUEUE1_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE1_PREEMPT                                                                    0x0252
+#define regSDMA0_SDMA_QUEUE1_PREEMPT_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE1_DUMMY_REG                                                                  0x0253
+#define regSDMA0_SDMA_QUEUE1_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_LO                                                       0x0254
+#define regSDMA0_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_HI                                                       0x0255
+#define regSDMA0_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE1_RB_AQL_CNTL                                                                0x0256
+#define regSDMA0_SDMA_QUEUE1_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE1_MINOR_PTR_UPDATE                                                           0x0257
+#define regSDMA0_SDMA_QUEUE1_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS                                                      0x025a
+#define regSDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_CNTL                                                                0x025b
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA0                                                               0x025c
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA1                                                               0x025d
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA2                                                               0x025e
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA3                                                               0x025f
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA4                                                               0x0260
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA5                                                               0x0261
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA6                                                               0x0262
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA7                                                               0x0263
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA8                                                               0x0264
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA9                                                               0x0265
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA10                                                              0x0266
+#define regSDMA0_SDMA_QUEUE1_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE1_UTILIZATION_LO                                                             0x0267
+#define regSDMA0_SDMA_QUEUE1_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE1_UTILIZATION_HI                                                             0x0268
+#define regSDMA0_SDMA_QUEUE1_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE1_WAIT_UNSATISFIED_THD                                                       0x0269
+#define regSDMA0_SDMA_QUEUE1_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE1_MQD_BASE_ADDR_LO                                                           0x026a
+#define regSDMA0_SDMA_QUEUE1_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE1_MQD_BASE_ADDR_HI                                                           0x026b
+#define regSDMA0_SDMA_QUEUE1_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE1_MQD_CONTROL                                                                0x026c
+#define regSDMA0_SDMA_QUEUE1_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE1_CONTEXT_STATUS                                                             0x026d
+#define regSDMA0_SDMA_QUEUE1_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE1_DUMMY_REG1                                                                 0x026e
+#define regSDMA0_SDMA_QUEUE1_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE1_DUMMY_REG2                                                                 0x026f
+#define regSDMA0_SDMA_QUEUE1_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE2_RB_CNTL                                                                    0x0278
+#define regSDMA0_SDMA_QUEUE2_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE2_RB_BASE                                                                    0x0279
+#define regSDMA0_SDMA_QUEUE2_RB_BASE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE2_RB_BASE_HI                                                                 0x027a
+#define regSDMA0_SDMA_QUEUE2_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE2_RB_RPTR                                                                    0x027b
+#define regSDMA0_SDMA_QUEUE2_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE2_RB_RPTR_HI                                                                 0x027c
+#define regSDMA0_SDMA_QUEUE2_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE2_RB_WPTR                                                                    0x027d
+#define regSDMA0_SDMA_QUEUE2_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE2_RB_WPTR_HI                                                                 0x027e
+#define regSDMA0_SDMA_QUEUE2_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE2_RB_RPTR_ADDR_LO                                                            0x027f
+#define regSDMA0_SDMA_QUEUE2_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE2_RB_RPTR_ADDR_HI                                                            0x0280
+#define regSDMA0_SDMA_QUEUE2_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE2_IB_CNTL                                                                    0x0281
+#define regSDMA0_SDMA_QUEUE2_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE2_IB_RPTR                                                                    0x0282
+#define regSDMA0_SDMA_QUEUE2_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE2_IB_OFFSET                                                                  0x0283
+#define regSDMA0_SDMA_QUEUE2_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE2_IB_BASE_LO                                                                 0x0284
+#define regSDMA0_SDMA_QUEUE2_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE2_IB_BASE_HI                                                                 0x0285
+#define regSDMA0_SDMA_QUEUE2_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE2_IB_SIZE                                                                    0x0286
+#define regSDMA0_SDMA_QUEUE2_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE2_DOORBELL                                                                   0x0287
+#define regSDMA0_SDMA_QUEUE2_DOORBELL_BASE_IDX                                                          0
+#define regSDMA0_SDMA_QUEUE2_DOORBELL_LOG                                                               0x0288
+#define regSDMA0_SDMA_QUEUE2_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE2_DOORBELL_OFFSET                                                            0x0289
+#define regSDMA0_SDMA_QUEUE2_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE2_CSA_ADDR_LO                                                                0x028a
+#define regSDMA0_SDMA_QUEUE2_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE2_CSA_ADDR_HI                                                                0x028b
+#define regSDMA0_SDMA_QUEUE2_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE2_SCHEDULE_CNTL                                                              0x028c
+#define regSDMA0_SDMA_QUEUE2_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE2_IB_SUB_REMAIN                                                              0x028d
+#define regSDMA0_SDMA_QUEUE2_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE2_PREEMPT                                                                    0x028e
+#define regSDMA0_SDMA_QUEUE2_PREEMPT_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE2_DUMMY_REG                                                                  0x028f
+#define regSDMA0_SDMA_QUEUE2_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_LO                                                       0x0290
+#define regSDMA0_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_HI                                                       0x0291
+#define regSDMA0_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE2_RB_AQL_CNTL                                                                0x0292
+#define regSDMA0_SDMA_QUEUE2_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE2_MINOR_PTR_UPDATE                                                           0x0293
+#define regSDMA0_SDMA_QUEUE2_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS                                                      0x0296
+#define regSDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_CNTL                                                                0x0297
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA0                                                               0x0298
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA1                                                               0x0299
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA2                                                               0x029a
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA3                                                               0x029b
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA4                                                               0x029c
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA5                                                               0x029d
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA6                                                               0x029e
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA7                                                               0x029f
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA8                                                               0x02a0
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA9                                                               0x02a1
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA10                                                              0x02a2
+#define regSDMA0_SDMA_QUEUE2_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE2_UTILIZATION_LO                                                             0x02a3
+#define regSDMA0_SDMA_QUEUE2_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE2_UTILIZATION_HI                                                             0x02a4
+#define regSDMA0_SDMA_QUEUE2_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE2_WAIT_UNSATISFIED_THD                                                       0x02a5
+#define regSDMA0_SDMA_QUEUE2_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE2_MQD_BASE_ADDR_LO                                                           0x02a6
+#define regSDMA0_SDMA_QUEUE2_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE2_MQD_BASE_ADDR_HI                                                           0x02a7
+#define regSDMA0_SDMA_QUEUE2_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE2_MQD_CONTROL                                                                0x02a8
+#define regSDMA0_SDMA_QUEUE2_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE2_CONTEXT_STATUS                                                             0x02a9
+#define regSDMA0_SDMA_QUEUE2_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE2_DUMMY_REG1                                                                 0x02aa
+#define regSDMA0_SDMA_QUEUE2_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE2_DUMMY_REG2                                                                 0x02ab
+#define regSDMA0_SDMA_QUEUE2_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE3_RB_CNTL                                                                    0x02b4
+#define regSDMA0_SDMA_QUEUE3_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE3_RB_BASE                                                                    0x02b5
+#define regSDMA0_SDMA_QUEUE3_RB_BASE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE3_RB_BASE_HI                                                                 0x02b6
+#define regSDMA0_SDMA_QUEUE3_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE3_RB_RPTR                                                                    0x02b7
+#define regSDMA0_SDMA_QUEUE3_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE3_RB_RPTR_HI                                                                 0x02b8
+#define regSDMA0_SDMA_QUEUE3_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE3_RB_WPTR                                                                    0x02b9
+#define regSDMA0_SDMA_QUEUE3_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE3_RB_WPTR_HI                                                                 0x02ba
+#define regSDMA0_SDMA_QUEUE3_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE3_RB_RPTR_ADDR_LO                                                            0x02bb
+#define regSDMA0_SDMA_QUEUE3_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE3_RB_RPTR_ADDR_HI                                                            0x02bc
+#define regSDMA0_SDMA_QUEUE3_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE3_IB_CNTL                                                                    0x02bd
+#define regSDMA0_SDMA_QUEUE3_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE3_IB_RPTR                                                                    0x02be
+#define regSDMA0_SDMA_QUEUE3_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE3_IB_OFFSET                                                                  0x02bf
+#define regSDMA0_SDMA_QUEUE3_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE3_IB_BASE_LO                                                                 0x02c0
+#define regSDMA0_SDMA_QUEUE3_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE3_IB_BASE_HI                                                                 0x02c1
+#define regSDMA0_SDMA_QUEUE3_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE3_IB_SIZE                                                                    0x02c2
+#define regSDMA0_SDMA_QUEUE3_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE3_DOORBELL                                                                   0x02c3
+#define regSDMA0_SDMA_QUEUE3_DOORBELL_BASE_IDX                                                          0
+#define regSDMA0_SDMA_QUEUE3_DOORBELL_LOG                                                               0x02c4
+#define regSDMA0_SDMA_QUEUE3_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE3_DOORBELL_OFFSET                                                            0x02c5
+#define regSDMA0_SDMA_QUEUE3_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE3_CSA_ADDR_LO                                                                0x02c6
+#define regSDMA0_SDMA_QUEUE3_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE3_CSA_ADDR_HI                                                                0x02c7
+#define regSDMA0_SDMA_QUEUE3_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE3_SCHEDULE_CNTL                                                              0x02c8
+#define regSDMA0_SDMA_QUEUE3_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE3_IB_SUB_REMAIN                                                              0x02c9
+#define regSDMA0_SDMA_QUEUE3_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE3_PREEMPT                                                                    0x02ca
+#define regSDMA0_SDMA_QUEUE3_PREEMPT_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE3_DUMMY_REG                                                                  0x02cb
+#define regSDMA0_SDMA_QUEUE3_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_LO                                                       0x02cc
+#define regSDMA0_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_HI                                                       0x02cd
+#define regSDMA0_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE3_RB_AQL_CNTL                                                                0x02ce
+#define regSDMA0_SDMA_QUEUE3_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE3_MINOR_PTR_UPDATE                                                           0x02cf
+#define regSDMA0_SDMA_QUEUE3_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS                                                      0x02d2
+#define regSDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_CNTL                                                                0x02d3
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA0                                                               0x02d4
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA1                                                               0x02d5
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA2                                                               0x02d6
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA3                                                               0x02d7
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA4                                                               0x02d8
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA5                                                               0x02d9
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA6                                                               0x02da
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA7                                                               0x02db
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA8                                                               0x02dc
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA9                                                               0x02dd
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA10                                                              0x02de
+#define regSDMA0_SDMA_QUEUE3_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE3_UTILIZATION_LO                                                             0x02df
+#define regSDMA0_SDMA_QUEUE3_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE3_UTILIZATION_HI                                                             0x02e0
+#define regSDMA0_SDMA_QUEUE3_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE3_WAIT_UNSATISFIED_THD                                                       0x02e1
+#define regSDMA0_SDMA_QUEUE3_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE3_MQD_BASE_ADDR_LO                                                           0x02e2
+#define regSDMA0_SDMA_QUEUE3_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE3_MQD_BASE_ADDR_HI                                                           0x02e3
+#define regSDMA0_SDMA_QUEUE3_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE3_MQD_CONTROL                                                                0x02e4
+#define regSDMA0_SDMA_QUEUE3_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE3_CONTEXT_STATUS                                                             0x02e5
+#define regSDMA0_SDMA_QUEUE3_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE3_DUMMY_REG1                                                                 0x02e6
+#define regSDMA0_SDMA_QUEUE3_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE3_DUMMY_REG2                                                                 0x02e7
+#define regSDMA0_SDMA_QUEUE3_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE4_RB_CNTL                                                                    0x02f0
+#define regSDMA0_SDMA_QUEUE4_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE4_RB_BASE                                                                    0x02f1
+#define regSDMA0_SDMA_QUEUE4_RB_BASE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE4_RB_BASE_HI                                                                 0x02f2
+#define regSDMA0_SDMA_QUEUE4_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE4_RB_RPTR                                                                    0x02f3
+#define regSDMA0_SDMA_QUEUE4_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE4_RB_RPTR_HI                                                                 0x02f4
+#define regSDMA0_SDMA_QUEUE4_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE4_RB_WPTR                                                                    0x02f5
+#define regSDMA0_SDMA_QUEUE4_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE4_RB_WPTR_HI                                                                 0x02f6
+#define regSDMA0_SDMA_QUEUE4_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE4_RB_RPTR_ADDR_LO                                                            0x02f7
+#define regSDMA0_SDMA_QUEUE4_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE4_RB_RPTR_ADDR_HI                                                            0x02f8
+#define regSDMA0_SDMA_QUEUE4_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE4_IB_CNTL                                                                    0x02f9
+#define regSDMA0_SDMA_QUEUE4_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE4_IB_RPTR                                                                    0x02fa
+#define regSDMA0_SDMA_QUEUE4_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE4_IB_OFFSET                                                                  0x02fb
+#define regSDMA0_SDMA_QUEUE4_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE4_IB_BASE_LO                                                                 0x02fc
+#define regSDMA0_SDMA_QUEUE4_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE4_IB_BASE_HI                                                                 0x02fd
+#define regSDMA0_SDMA_QUEUE4_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE4_IB_SIZE                                                                    0x02fe
+#define regSDMA0_SDMA_QUEUE4_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE4_DOORBELL                                                                   0x02ff
+#define regSDMA0_SDMA_QUEUE4_DOORBELL_BASE_IDX                                                          0
+#define regSDMA0_SDMA_QUEUE4_DOORBELL_LOG                                                               0x0300
+#define regSDMA0_SDMA_QUEUE4_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE4_DOORBELL_OFFSET                                                            0x0301
+#define regSDMA0_SDMA_QUEUE4_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE4_CSA_ADDR_LO                                                                0x0302
+#define regSDMA0_SDMA_QUEUE4_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE4_CSA_ADDR_HI                                                                0x0303
+#define regSDMA0_SDMA_QUEUE4_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE4_SCHEDULE_CNTL                                                              0x0304
+#define regSDMA0_SDMA_QUEUE4_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE4_IB_SUB_REMAIN                                                              0x0305
+#define regSDMA0_SDMA_QUEUE4_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE4_PREEMPT                                                                    0x0306
+#define regSDMA0_SDMA_QUEUE4_PREEMPT_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE4_DUMMY_REG                                                                  0x0307
+#define regSDMA0_SDMA_QUEUE4_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_LO                                                       0x0308
+#define regSDMA0_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_HI                                                       0x0309
+#define regSDMA0_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE4_RB_AQL_CNTL                                                                0x030a
+#define regSDMA0_SDMA_QUEUE4_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE4_MINOR_PTR_UPDATE                                                           0x030b
+#define regSDMA0_SDMA_QUEUE4_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS                                                      0x030e
+#define regSDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_CNTL                                                                0x030f
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA0                                                               0x0310
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA1                                                               0x0311
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA2                                                               0x0312
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA3                                                               0x0313
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA4                                                               0x0314
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA5                                                               0x0315
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA6                                                               0x0316
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA7                                                               0x0317
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA8                                                               0x0318
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA9                                                               0x0319
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA10                                                              0x031a
+#define regSDMA0_SDMA_QUEUE4_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE4_UTILIZATION_LO                                                             0x031b
+#define regSDMA0_SDMA_QUEUE4_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE4_UTILIZATION_HI                                                             0x031c
+#define regSDMA0_SDMA_QUEUE4_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE4_WAIT_UNSATISFIED_THD                                                       0x031d
+#define regSDMA0_SDMA_QUEUE4_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE4_MQD_BASE_ADDR_LO                                                           0x031e
+#define regSDMA0_SDMA_QUEUE4_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE4_MQD_BASE_ADDR_HI                                                           0x031f
+#define regSDMA0_SDMA_QUEUE4_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE4_MQD_CONTROL                                                                0x0320
+#define regSDMA0_SDMA_QUEUE4_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE4_CONTEXT_STATUS                                                             0x0321
+#define regSDMA0_SDMA_QUEUE4_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE4_DUMMY_REG1                                                                 0x0322
+#define regSDMA0_SDMA_QUEUE4_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE4_DUMMY_REG2                                                                 0x0323
+#define regSDMA0_SDMA_QUEUE4_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE5_RB_CNTL                                                                    0x032c
+#define regSDMA0_SDMA_QUEUE5_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE5_RB_BASE                                                                    0x032d
+#define regSDMA0_SDMA_QUEUE5_RB_BASE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE5_RB_BASE_HI                                                                 0x032e
+#define regSDMA0_SDMA_QUEUE5_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE5_RB_RPTR                                                                    0x032f
+#define regSDMA0_SDMA_QUEUE5_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE5_RB_RPTR_HI                                                                 0x0330
+#define regSDMA0_SDMA_QUEUE5_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE5_RB_WPTR                                                                    0x0331
+#define regSDMA0_SDMA_QUEUE5_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE5_RB_WPTR_HI                                                                 0x0332
+#define regSDMA0_SDMA_QUEUE5_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE5_RB_RPTR_ADDR_LO                                                            0x0333
+#define regSDMA0_SDMA_QUEUE5_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE5_RB_RPTR_ADDR_HI                                                            0x0334
+#define regSDMA0_SDMA_QUEUE5_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE5_IB_CNTL                                                                    0x0335
+#define regSDMA0_SDMA_QUEUE5_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE5_IB_RPTR                                                                    0x0336
+#define regSDMA0_SDMA_QUEUE5_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE5_IB_OFFSET                                                                  0x0337
+#define regSDMA0_SDMA_QUEUE5_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE5_IB_BASE_LO                                                                 0x0338
+#define regSDMA0_SDMA_QUEUE5_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE5_IB_BASE_HI                                                                 0x0339
+#define regSDMA0_SDMA_QUEUE5_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE5_IB_SIZE                                                                    0x033a
+#define regSDMA0_SDMA_QUEUE5_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE5_DOORBELL                                                                   0x033b
+#define regSDMA0_SDMA_QUEUE5_DOORBELL_BASE_IDX                                                          0
+#define regSDMA0_SDMA_QUEUE5_DOORBELL_LOG                                                               0x033c
+#define regSDMA0_SDMA_QUEUE5_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE5_DOORBELL_OFFSET                                                            0x033d
+#define regSDMA0_SDMA_QUEUE5_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE5_CSA_ADDR_LO                                                                0x033e
+#define regSDMA0_SDMA_QUEUE5_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE5_CSA_ADDR_HI                                                                0x033f
+#define regSDMA0_SDMA_QUEUE5_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE5_SCHEDULE_CNTL                                                              0x0340
+#define regSDMA0_SDMA_QUEUE5_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE5_IB_SUB_REMAIN                                                              0x0341
+#define regSDMA0_SDMA_QUEUE5_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE5_PREEMPT                                                                    0x0342
+#define regSDMA0_SDMA_QUEUE5_PREEMPT_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE5_DUMMY_REG                                                                  0x0343
+#define regSDMA0_SDMA_QUEUE5_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_LO                                                       0x0344
+#define regSDMA0_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_HI                                                       0x0345
+#define regSDMA0_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE5_RB_AQL_CNTL                                                                0x0346
+#define regSDMA0_SDMA_QUEUE5_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE5_MINOR_PTR_UPDATE                                                           0x0347
+#define regSDMA0_SDMA_QUEUE5_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS                                                      0x034a
+#define regSDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_CNTL                                                                0x034b
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA0                                                               0x034c
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA1                                                               0x034d
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA2                                                               0x034e
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA3                                                               0x034f
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA4                                                               0x0350
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA5                                                               0x0351
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA6                                                               0x0352
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA7                                                               0x0353
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA8                                                               0x0354
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA9                                                               0x0355
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA10                                                              0x0356
+#define regSDMA0_SDMA_QUEUE5_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE5_UTILIZATION_LO                                                             0x0357
+#define regSDMA0_SDMA_QUEUE5_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE5_UTILIZATION_HI                                                             0x0358
+#define regSDMA0_SDMA_QUEUE5_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE5_WAIT_UNSATISFIED_THD                                                       0x0359
+#define regSDMA0_SDMA_QUEUE5_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE5_MQD_BASE_ADDR_LO                                                           0x035a
+#define regSDMA0_SDMA_QUEUE5_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE5_MQD_BASE_ADDR_HI                                                           0x035b
+#define regSDMA0_SDMA_QUEUE5_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE5_MQD_CONTROL                                                                0x035c
+#define regSDMA0_SDMA_QUEUE5_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE5_CONTEXT_STATUS                                                             0x035d
+#define regSDMA0_SDMA_QUEUE5_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE5_DUMMY_REG1                                                                 0x035e
+#define regSDMA0_SDMA_QUEUE5_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE5_DUMMY_REG2                                                                 0x035f
+#define regSDMA0_SDMA_QUEUE5_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE6_RB_CNTL                                                                    0x0368
+#define regSDMA0_SDMA_QUEUE6_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE6_RB_BASE                                                                    0x0369
+#define regSDMA0_SDMA_QUEUE6_RB_BASE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE6_RB_BASE_HI                                                                 0x036a
+#define regSDMA0_SDMA_QUEUE6_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE6_RB_RPTR                                                                    0x036b
+#define regSDMA0_SDMA_QUEUE6_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE6_RB_RPTR_HI                                                                 0x036c
+#define regSDMA0_SDMA_QUEUE6_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE6_RB_WPTR                                                                    0x036d
+#define regSDMA0_SDMA_QUEUE6_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE6_RB_WPTR_HI                                                                 0x036e
+#define regSDMA0_SDMA_QUEUE6_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE6_RB_RPTR_ADDR_LO                                                            0x036f
+#define regSDMA0_SDMA_QUEUE6_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE6_RB_RPTR_ADDR_HI                                                            0x0370
+#define regSDMA0_SDMA_QUEUE6_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE6_IB_CNTL                                                                    0x0371
+#define regSDMA0_SDMA_QUEUE6_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE6_IB_RPTR                                                                    0x0372
+#define regSDMA0_SDMA_QUEUE6_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE6_IB_OFFSET                                                                  0x0373
+#define regSDMA0_SDMA_QUEUE6_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE6_IB_BASE_LO                                                                 0x0374
+#define regSDMA0_SDMA_QUEUE6_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE6_IB_BASE_HI                                                                 0x0375
+#define regSDMA0_SDMA_QUEUE6_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE6_IB_SIZE                                                                    0x0376
+#define regSDMA0_SDMA_QUEUE6_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE6_DOORBELL                                                                   0x0377
+#define regSDMA0_SDMA_QUEUE6_DOORBELL_BASE_IDX                                                          0
+#define regSDMA0_SDMA_QUEUE6_DOORBELL_LOG                                                               0x0378
+#define regSDMA0_SDMA_QUEUE6_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE6_DOORBELL_OFFSET                                                            0x0379
+#define regSDMA0_SDMA_QUEUE6_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE6_CSA_ADDR_LO                                                                0x037a
+#define regSDMA0_SDMA_QUEUE6_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE6_CSA_ADDR_HI                                                                0x037b
+#define regSDMA0_SDMA_QUEUE6_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE6_SCHEDULE_CNTL                                                              0x037c
+#define regSDMA0_SDMA_QUEUE6_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE6_IB_SUB_REMAIN                                                              0x037d
+#define regSDMA0_SDMA_QUEUE6_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE6_PREEMPT                                                                    0x037e
+#define regSDMA0_SDMA_QUEUE6_PREEMPT_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE6_DUMMY_REG                                                                  0x037f
+#define regSDMA0_SDMA_QUEUE6_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_LO                                                       0x0380
+#define regSDMA0_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_HI                                                       0x0381
+#define regSDMA0_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE6_RB_AQL_CNTL                                                                0x0382
+#define regSDMA0_SDMA_QUEUE6_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE6_MINOR_PTR_UPDATE                                                           0x0383
+#define regSDMA0_SDMA_QUEUE6_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS                                                      0x0386
+#define regSDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_CNTL                                                                0x0387
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA0                                                               0x0388
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA1                                                               0x0389
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA2                                                               0x038a
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA3                                                               0x038b
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA4                                                               0x038c
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA5                                                               0x038d
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA6                                                               0x038e
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA7                                                               0x038f
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA8                                                               0x0390
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA9                                                               0x0391
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA10                                                              0x0392
+#define regSDMA0_SDMA_QUEUE6_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE6_UTILIZATION_LO                                                             0x0393
+#define regSDMA0_SDMA_QUEUE6_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE6_UTILIZATION_HI                                                             0x0394
+#define regSDMA0_SDMA_QUEUE6_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE6_WAIT_UNSATISFIED_THD                                                       0x0395
+#define regSDMA0_SDMA_QUEUE6_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE6_MQD_BASE_ADDR_LO                                                           0x0396
+#define regSDMA0_SDMA_QUEUE6_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE6_MQD_BASE_ADDR_HI                                                           0x0397
+#define regSDMA0_SDMA_QUEUE6_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE6_MQD_CONTROL                                                                0x0398
+#define regSDMA0_SDMA_QUEUE6_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE6_CONTEXT_STATUS                                                             0x0399
+#define regSDMA0_SDMA_QUEUE6_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE6_DUMMY_REG1                                                                 0x039a
+#define regSDMA0_SDMA_QUEUE6_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE6_DUMMY_REG2                                                                 0x039b
+#define regSDMA0_SDMA_QUEUE6_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE7_RB_CNTL                                                                    0x03a4
+#define regSDMA0_SDMA_QUEUE7_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE7_RB_BASE                                                                    0x03a5
+#define regSDMA0_SDMA_QUEUE7_RB_BASE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE7_RB_BASE_HI                                                                 0x03a6
+#define regSDMA0_SDMA_QUEUE7_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE7_RB_RPTR                                                                    0x03a7
+#define regSDMA0_SDMA_QUEUE7_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE7_RB_RPTR_HI                                                                 0x03a8
+#define regSDMA0_SDMA_QUEUE7_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE7_RB_WPTR                                                                    0x03a9
+#define regSDMA0_SDMA_QUEUE7_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE7_RB_WPTR_HI                                                                 0x03aa
+#define regSDMA0_SDMA_QUEUE7_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE7_RB_RPTR_ADDR_LO                                                            0x03ab
+#define regSDMA0_SDMA_QUEUE7_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE7_RB_RPTR_ADDR_HI                                                            0x03ac
+#define regSDMA0_SDMA_QUEUE7_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE7_IB_CNTL                                                                    0x03ad
+#define regSDMA0_SDMA_QUEUE7_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE7_IB_RPTR                                                                    0x03ae
+#define regSDMA0_SDMA_QUEUE7_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE7_IB_OFFSET                                                                  0x03af
+#define regSDMA0_SDMA_QUEUE7_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE7_IB_BASE_LO                                                                 0x03b0
+#define regSDMA0_SDMA_QUEUE7_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE7_IB_BASE_HI                                                                 0x03b1
+#define regSDMA0_SDMA_QUEUE7_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE7_IB_SIZE                                                                    0x03b2
+#define regSDMA0_SDMA_QUEUE7_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE7_DOORBELL                                                                   0x03b3
+#define regSDMA0_SDMA_QUEUE7_DOORBELL_BASE_IDX                                                          0
+#define regSDMA0_SDMA_QUEUE7_DOORBELL_LOG                                                               0x03b4
+#define regSDMA0_SDMA_QUEUE7_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE7_DOORBELL_OFFSET                                                            0x03b5
+#define regSDMA0_SDMA_QUEUE7_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE7_CSA_ADDR_LO                                                                0x03b6
+#define regSDMA0_SDMA_QUEUE7_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE7_CSA_ADDR_HI                                                                0x03b7
+#define regSDMA0_SDMA_QUEUE7_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE7_SCHEDULE_CNTL                                                              0x03b8
+#define regSDMA0_SDMA_QUEUE7_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE7_IB_SUB_REMAIN                                                              0x03b9
+#define regSDMA0_SDMA_QUEUE7_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE7_PREEMPT                                                                    0x03ba
+#define regSDMA0_SDMA_QUEUE7_PREEMPT_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE7_DUMMY_REG                                                                  0x03bb
+#define regSDMA0_SDMA_QUEUE7_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_LO                                                       0x03bc
+#define regSDMA0_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_HI                                                       0x03bd
+#define regSDMA0_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE7_RB_AQL_CNTL                                                                0x03be
+#define regSDMA0_SDMA_QUEUE7_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE7_MINOR_PTR_UPDATE                                                           0x03bf
+#define regSDMA0_SDMA_QUEUE7_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS                                                      0x03c2
+#define regSDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_CNTL                                                                0x03c3
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA0                                                               0x03c4
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA1                                                               0x03c5
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA2                                                               0x03c6
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA3                                                               0x03c7
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA4                                                               0x03c8
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA5                                                               0x03c9
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA6                                                               0x03ca
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA7                                                               0x03cb
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA8                                                               0x03cc
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA9                                                               0x03cd
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA10                                                              0x03ce
+#define regSDMA0_SDMA_QUEUE7_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE7_UTILIZATION_LO                                                             0x03cf
+#define regSDMA0_SDMA_QUEUE7_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE7_UTILIZATION_HI                                                             0x03d0
+#define regSDMA0_SDMA_QUEUE7_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE7_WAIT_UNSATISFIED_THD                                                       0x03d1
+#define regSDMA0_SDMA_QUEUE7_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE7_MQD_BASE_ADDR_LO                                                           0x03d2
+#define regSDMA0_SDMA_QUEUE7_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE7_MQD_BASE_ADDR_HI                                                           0x03d3
+#define regSDMA0_SDMA_QUEUE7_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE7_MQD_CONTROL                                                                0x03d4
+#define regSDMA0_SDMA_QUEUE7_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE7_CONTEXT_STATUS                                                             0x03d5
+#define regSDMA0_SDMA_QUEUE7_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE7_DUMMY_REG1                                                                 0x03d6
+#define regSDMA0_SDMA_QUEUE7_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE7_DUMMY_REG2                                                                 0x03d7
+#define regSDMA0_SDMA_QUEUE7_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE8_RB_CNTL                                                                    0x03e0
+#define regSDMA0_SDMA_QUEUE8_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE8_RB_BASE                                                                    0x03e1
+#define regSDMA0_SDMA_QUEUE8_RB_BASE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE8_RB_BASE_HI                                                                 0x03e2
+#define regSDMA0_SDMA_QUEUE8_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE8_RB_RPTR                                                                    0x03e3
+#define regSDMA0_SDMA_QUEUE8_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE8_RB_RPTR_HI                                                                 0x03e4
+#define regSDMA0_SDMA_QUEUE8_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE8_RB_WPTR                                                                    0x03e5
+#define regSDMA0_SDMA_QUEUE8_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE8_RB_WPTR_HI                                                                 0x03e6
+#define regSDMA0_SDMA_QUEUE8_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE8_RB_RPTR_ADDR_LO                                                            0x03e7
+#define regSDMA0_SDMA_QUEUE8_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE8_RB_RPTR_ADDR_HI                                                            0x03e8
+#define regSDMA0_SDMA_QUEUE8_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE8_IB_CNTL                                                                    0x03e9
+#define regSDMA0_SDMA_QUEUE8_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE8_IB_RPTR                                                                    0x03ea
+#define regSDMA0_SDMA_QUEUE8_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE8_IB_OFFSET                                                                  0x03eb
+#define regSDMA0_SDMA_QUEUE8_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE8_IB_BASE_LO                                                                 0x03ec
+#define regSDMA0_SDMA_QUEUE8_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE8_IB_BASE_HI                                                                 0x03ed
+#define regSDMA0_SDMA_QUEUE8_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE8_IB_SIZE                                                                    0x03ee
+#define regSDMA0_SDMA_QUEUE8_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE8_DOORBELL                                                                   0x03ef
+#define regSDMA0_SDMA_QUEUE8_DOORBELL_BASE_IDX                                                          0
+#define regSDMA0_SDMA_QUEUE8_DOORBELL_LOG                                                               0x03f0
+#define regSDMA0_SDMA_QUEUE8_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE8_DOORBELL_OFFSET                                                            0x03f1
+#define regSDMA0_SDMA_QUEUE8_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE8_CSA_ADDR_LO                                                                0x03f2
+#define regSDMA0_SDMA_QUEUE8_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE8_CSA_ADDR_HI                                                                0x03f3
+#define regSDMA0_SDMA_QUEUE8_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE8_SCHEDULE_CNTL                                                              0x03f4
+#define regSDMA0_SDMA_QUEUE8_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE8_IB_SUB_REMAIN                                                              0x03f5
+#define regSDMA0_SDMA_QUEUE8_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE8_PREEMPT                                                                    0x03f6
+#define regSDMA0_SDMA_QUEUE8_PREEMPT_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE8_DUMMY_REG                                                                  0x03f7
+#define regSDMA0_SDMA_QUEUE8_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_LO                                                       0x03f8
+#define regSDMA0_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_HI                                                       0x03f9
+#define regSDMA0_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE8_RB_AQL_CNTL                                                                0x03fa
+#define regSDMA0_SDMA_QUEUE8_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE8_MINOR_PTR_UPDATE                                                           0x03fb
+#define regSDMA0_SDMA_QUEUE8_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS                                                      0x03fe
+#define regSDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_CNTL                                                                0x03ff
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA0                                                               0x0400
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA1                                                               0x0401
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA2                                                               0x0402
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA3                                                               0x0403
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA4                                                               0x0404
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA5                                                               0x0405
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA6                                                               0x0406
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA7                                                               0x0407
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA8                                                               0x0408
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA9                                                               0x0409
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA10                                                              0x040a
+#define regSDMA0_SDMA_QUEUE8_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE8_UTILIZATION_LO                                                             0x040b
+#define regSDMA0_SDMA_QUEUE8_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE8_UTILIZATION_HI                                                             0x040c
+#define regSDMA0_SDMA_QUEUE8_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE8_WAIT_UNSATISFIED_THD                                                       0x040d
+#define regSDMA0_SDMA_QUEUE8_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE8_MQD_BASE_ADDR_LO                                                           0x040e
+#define regSDMA0_SDMA_QUEUE8_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE8_MQD_BASE_ADDR_HI                                                           0x040f
+#define regSDMA0_SDMA_QUEUE8_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE8_MQD_CONTROL                                                                0x0410
+#define regSDMA0_SDMA_QUEUE8_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE8_CONTEXT_STATUS                                                             0x0411
+#define regSDMA0_SDMA_QUEUE8_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE8_DUMMY_REG1                                                                 0x0412
+#define regSDMA0_SDMA_QUEUE8_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE8_DUMMY_REG2                                                                 0x0413
+#define regSDMA0_SDMA_QUEUE8_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE9_RB_CNTL                                                                    0x041c
+#define regSDMA0_SDMA_QUEUE9_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE9_RB_BASE                                                                    0x041d
+#define regSDMA0_SDMA_QUEUE9_RB_BASE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE9_RB_BASE_HI                                                                 0x041e
+#define regSDMA0_SDMA_QUEUE9_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE9_RB_RPTR                                                                    0x041f
+#define regSDMA0_SDMA_QUEUE9_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE9_RB_RPTR_HI                                                                 0x0420
+#define regSDMA0_SDMA_QUEUE9_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE9_RB_WPTR                                                                    0x0421
+#define regSDMA0_SDMA_QUEUE9_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE9_RB_WPTR_HI                                                                 0x0422
+#define regSDMA0_SDMA_QUEUE9_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE9_RB_RPTR_ADDR_LO                                                            0x0423
+#define regSDMA0_SDMA_QUEUE9_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE9_RB_RPTR_ADDR_HI                                                            0x0424
+#define regSDMA0_SDMA_QUEUE9_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE9_IB_CNTL                                                                    0x0425
+#define regSDMA0_SDMA_QUEUE9_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE9_IB_RPTR                                                                    0x0426
+#define regSDMA0_SDMA_QUEUE9_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE9_IB_OFFSET                                                                  0x0427
+#define regSDMA0_SDMA_QUEUE9_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE9_IB_BASE_LO                                                                 0x0428
+#define regSDMA0_SDMA_QUEUE9_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE9_IB_BASE_HI                                                                 0x0429
+#define regSDMA0_SDMA_QUEUE9_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE9_IB_SIZE                                                                    0x042a
+#define regSDMA0_SDMA_QUEUE9_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE9_DOORBELL                                                                   0x042b
+#define regSDMA0_SDMA_QUEUE9_DOORBELL_BASE_IDX                                                          0
+#define regSDMA0_SDMA_QUEUE9_DOORBELL_LOG                                                               0x042c
+#define regSDMA0_SDMA_QUEUE9_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE9_DOORBELL_OFFSET                                                            0x042d
+#define regSDMA0_SDMA_QUEUE9_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA0_SDMA_QUEUE9_CSA_ADDR_LO                                                                0x042e
+#define regSDMA0_SDMA_QUEUE9_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE9_CSA_ADDR_HI                                                                0x042f
+#define regSDMA0_SDMA_QUEUE9_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE9_SCHEDULE_CNTL                                                              0x0430
+#define regSDMA0_SDMA_QUEUE9_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE9_IB_SUB_REMAIN                                                              0x0431
+#define regSDMA0_SDMA_QUEUE9_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE9_PREEMPT                                                                    0x0432
+#define regSDMA0_SDMA_QUEUE9_PREEMPT_BASE_IDX                                                           0
+#define regSDMA0_SDMA_QUEUE9_DUMMY_REG                                                                  0x0433
+#define regSDMA0_SDMA_QUEUE9_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA0_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_LO                                                       0x0434
+#define regSDMA0_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_HI                                                       0x0435
+#define regSDMA0_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE9_RB_AQL_CNTL                                                                0x0436
+#define regSDMA0_SDMA_QUEUE9_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE9_MINOR_PTR_UPDATE                                                           0x0437
+#define regSDMA0_SDMA_QUEUE9_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS                                                      0x043a
+#define regSDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_CNTL                                                                0x043b
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA0                                                               0x043c
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA1                                                               0x043d
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA2                                                               0x043e
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA3                                                               0x043f
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA4                                                               0x0440
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA5                                                               0x0441
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA6                                                               0x0442
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA7                                                               0x0443
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA8                                                               0x0444
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA9                                                               0x0445
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA10                                                              0x0446
+#define regSDMA0_SDMA_QUEUE9_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA0_SDMA_QUEUE9_UTILIZATION_LO                                                             0x0447
+#define regSDMA0_SDMA_QUEUE9_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE9_UTILIZATION_HI                                                             0x0448
+#define regSDMA0_SDMA_QUEUE9_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE9_WAIT_UNSATISFIED_THD                                                       0x0449
+#define regSDMA0_SDMA_QUEUE9_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA0_SDMA_QUEUE9_MQD_BASE_ADDR_LO                                                           0x044a
+#define regSDMA0_SDMA_QUEUE9_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE9_MQD_BASE_ADDR_HI                                                           0x044b
+#define regSDMA0_SDMA_QUEUE9_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA0_SDMA_QUEUE9_MQD_CONTROL                                                                0x044c
+#define regSDMA0_SDMA_QUEUE9_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA0_SDMA_QUEUE9_CONTEXT_STATUS                                                             0x044d
+#define regSDMA0_SDMA_QUEUE9_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA0_SDMA_QUEUE9_DUMMY_REG1                                                                 0x044e
+#define regSDMA0_SDMA_QUEUE9_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA0_SDMA_QUEUE9_DUMMY_REG2                                                                 0x044f
+#define regSDMA0_SDMA_QUEUE9_DUMMY_REG2_BASE_IDX                                                        0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmahypdec
+// base address: 0x3e200
+#define regSDMA0_SDMA_VM_CTX_LO                                                                         0x5880
+#define regSDMA0_SDMA_VM_CTX_LO_BASE_IDX                                                                1
+#define regSDMA0_SDMA_VM_CTX_HI                                                                         0x5881
+#define regSDMA0_SDMA_VM_CTX_HI_BASE_IDX                                                                1
+#define regSDMA0_SDMA_ACTIVE_FCN_ID                                                                     0x5882
+#define regSDMA0_SDMA_ACTIVE_FCN_ID_BASE_IDX                                                            1
+#define regSDMA0_SDMA_VM_CTX_CNTL                                                                       0x5883
+#define regSDMA0_SDMA_VM_CTX_CNTL_BASE_IDX                                                              1
+#define regSDMA0_SDMA_VIRT_RESET_REQ                                                                    0x5884
+#define regSDMA0_SDMA_VIRT_RESET_REQ_BASE_IDX                                                           1
+#define regSDMA0_SDMA_CONTEXT_REG_TYPE0                                                                 0x5885
+#define regSDMA0_SDMA_CONTEXT_REG_TYPE0_BASE_IDX                                                        1
+#define regSDMA0_SDMA_CONTEXT_REG_TYPE1                                                                 0x5886
+#define regSDMA0_SDMA_CONTEXT_REG_TYPE1_BASE_IDX                                                        1
+#define regSDMA0_SDMA_CONTEXT_REG_TYPE2                                                                 0x5887
+#define regSDMA0_SDMA_CONTEXT_REG_TYPE2_BASE_IDX                                                        1
+#define regSDMA0_SDMA_PUB_REG_TYPE0                                                                     0x5888
+#define regSDMA0_SDMA_PUB_REG_TYPE0_BASE_IDX                                                            1
+#define regSDMA0_SDMA_PUB_REG_TYPE1                                                                     0x5889
+#define regSDMA0_SDMA_PUB_REG_TYPE1_BASE_IDX                                                            1
+#define regSDMA0_SDMA_PUB_REG_TYPE2                                                                     0x588a
+#define regSDMA0_SDMA_PUB_REG_TYPE2_BASE_IDX                                                            1
+#define regSDMA0_SDMA_PUB_REG_TYPE3                                                                     0x588b
+#define regSDMA0_SDMA_PUB_REG_TYPE3_BASE_IDX                                                            1
+#define regSDMA0_SDMA_VM_CNTL                                                                           0x588d
+#define regSDMA0_SDMA_VM_CNTL_BASE_IDX                                                                  1
+#define regSDMA0_SDMA_MCU_CNTL                                                                          0x588e
+#define regSDMA0_SDMA_MCU_CNTL_BASE_IDX                                                                 1
+#define regSDMA0_SDMA_IC_BASE_LO                                                                        0x589a
+#define regSDMA0_SDMA_IC_BASE_LO_BASE_IDX                                                               1
+#define regSDMA0_SDMA_IC_BASE_HI                                                                        0x589b
+#define regSDMA0_SDMA_IC_BASE_HI_BASE_IDX                                                               1
+#define regSDMA0_SDMA_IC_BASE_CNTL                                                                      0x589c
+#define regSDMA0_SDMA_IC_BASE_CNTL_BASE_IDX                                                             1
+#define regSDMA0_SDMA_IC_OP_CNTL                                                                        0x589d
+#define regSDMA0_SDMA_IC_OP_CNTL_BASE_IDX                                                               1
+#define regSDMA0_SDMA_IC_CNTL                                                                           0x589f
+#define regSDMA0_SDMA_IC_CNTL_BASE_IDX                                                                  1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmapspdec
+// base address: 0x3f200
+#define regSDMA0_SDMA_MCU_DM_FROM_RST_ADDR_OFFSET                                                       0x5cc4
+#define regSDMA0_SDMA_MCU_DM_FROM_RST_ADDR_OFFSET_BASE_IDX                                              1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmaperfsdec
+// base address: 0x37880
+#define regSDMA0_SDMA_PERFCNT_PERFCOUNTER0_CFG                                                          0x3e20
+#define regSDMA0_SDMA_PERFCNT_PERFCOUNTER0_CFG_BASE_IDX                                                 1
+#define regSDMA0_SDMA_PERFCNT_PERFCOUNTER1_CFG                                                          0x3e21
+#define regSDMA0_SDMA_PERFCNT_PERFCOUNTER1_CFG_BASE_IDX                                                 1
+#define regSDMA0_SDMA_PERFCNT_PERFCOUNTER2_CFG                                                          0x3e22
+#define regSDMA0_SDMA_PERFCNT_PERFCOUNTER2_CFG_BASE_IDX                                                 1
+#define regSDMA0_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL                                                     0x3e23
+#define regSDMA0_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                            1
+#define regSDMA0_SDMA_PERFCNT_MISC_CNTL                                                                 0x3e24
+#define regSDMA0_SDMA_PERFCNT_MISC_CNTL_BASE_IDX                                                        1
+#define regSDMA0_SDMA_PERFCOUNTER0_SELECT                                                               0x3e25
+#define regSDMA0_SDMA_PERFCOUNTER0_SELECT_BASE_IDX                                                      1
+#define regSDMA0_SDMA_PERFCOUNTER0_SELECT1                                                              0x3e26
+#define regSDMA0_SDMA_PERFCOUNTER0_SELECT1_BASE_IDX                                                     1
+#define regSDMA0_SDMA_PERFCOUNTER1_SELECT                                                               0x3e27
+#define regSDMA0_SDMA_PERFCOUNTER1_SELECT_BASE_IDX                                                      1
+#define regSDMA0_SDMA_PERFCOUNTER1_SELECT1                                                              0x3e28
+#define regSDMA0_SDMA_PERFCOUNTER1_SELECT1_BASE_IDX                                                     1
+#define regSDMA0_SDMA_PERFCOUNTER2_SELECT                                                               0x3e29
+#define regSDMA0_SDMA_PERFCOUNTER2_SELECT_BASE_IDX                                                      1
+#define regSDMA0_SDMA_PERFCOUNTER2_SELECT1                                                              0x3e2a
+#define regSDMA0_SDMA_PERFCOUNTER2_SELECT1_BASE_IDX                                                     1
+#define regSDMA0_SDMA_PERFCOUNTER3_SELECT                                                               0x3e2b
+#define regSDMA0_SDMA_PERFCOUNTER3_SELECT_BASE_IDX                                                      1
+#define regSDMA0_SDMA_PERFCOUNTER3_SELECT1                                                              0x3e2c
+#define regSDMA0_SDMA_PERFCOUNTER3_SELECT1_BASE_IDX                                                     1
+#define regSDMA0_SDMA_PERFCOUNTER4_SELECT                                                               0x3e2d
+#define regSDMA0_SDMA_PERFCOUNTER4_SELECT_BASE_IDX                                                      1
+#define regSDMA0_SDMA_PERFCOUNTER4_SELECT1                                                              0x3e2e
+#define regSDMA0_SDMA_PERFCOUNTER4_SELECT1_BASE_IDX                                                     1
+#define regSDMA0_SDMA_PERFCOUNTER5_SELECT                                                               0x3e2f
+#define regSDMA0_SDMA_PERFCOUNTER5_SELECT_BASE_IDX                                                      1
+#define regSDMA0_SDMA_PERFCOUNTER5_SELECT1                                                              0x3e30
+#define regSDMA0_SDMA_PERFCOUNTER5_SELECT1_BASE_IDX                                                     1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmaperfddec
+// base address: 0x35980
+#define regSDMA0_SDMA_PERFCNT_PERFCOUNTER_LO                                                            0x3660
+#define regSDMA0_SDMA_PERFCNT_PERFCOUNTER_LO_BASE_IDX                                                   1
+#define regSDMA0_SDMA_PERFCNT_PERFCOUNTER_HI                                                            0x3661
+#define regSDMA0_SDMA_PERFCNT_PERFCOUNTER_HI_BASE_IDX                                                   1
+#define regSDMA0_SDMA_PERFCOUNTER0_LO                                                                   0x3662
+#define regSDMA0_SDMA_PERFCOUNTER0_LO_BASE_IDX                                                          1
+#define regSDMA0_SDMA_PERFCOUNTER0_HI                                                                   0x3663
+#define regSDMA0_SDMA_PERFCOUNTER0_HI_BASE_IDX                                                          1
+#define regSDMA0_SDMA_PERFCOUNTER1_LO                                                                   0x3664
+#define regSDMA0_SDMA_PERFCOUNTER1_LO_BASE_IDX                                                          1
+#define regSDMA0_SDMA_PERFCOUNTER1_HI                                                                   0x3665
+#define regSDMA0_SDMA_PERFCOUNTER1_HI_BASE_IDX                                                          1
+#define regSDMA0_SDMA_PERFCOUNTER2_LO                                                                   0x3666
+#define regSDMA0_SDMA_PERFCOUNTER2_LO_BASE_IDX                                                          1
+#define regSDMA0_SDMA_PERFCOUNTER2_HI                                                                   0x3667
+#define regSDMA0_SDMA_PERFCOUNTER2_HI_BASE_IDX                                                          1
+#define regSDMA0_SDMA_PERFCOUNTER3_LO                                                                   0x3668
+#define regSDMA0_SDMA_PERFCOUNTER3_LO_BASE_IDX                                                          1
+#define regSDMA0_SDMA_PERFCOUNTER3_HI                                                                   0x3669
+#define regSDMA0_SDMA_PERFCOUNTER3_HI_BASE_IDX                                                          1
+#define regSDMA0_SDMA_PERFCOUNTER4_LO                                                                   0x366a
+#define regSDMA0_SDMA_PERFCOUNTER4_LO_BASE_IDX                                                          1
+#define regSDMA0_SDMA_PERFCOUNTER4_HI                                                                   0x366b
+#define regSDMA0_SDMA_PERFCOUNTER4_HI_BASE_IDX                                                          1
+#define regSDMA0_SDMA_PERFCOUNTER5_LO                                                                   0x366c
+#define regSDMA0_SDMA_PERFCOUNTER5_LO_BASE_IDX                                                          1
+#define regSDMA0_SDMA_PERFCOUNTER5_HI                                                                   0x366d
+#define regSDMA0_SDMA_PERFCOUNTER5_HI_BASE_IDX                                                          1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmapwrdec
+// base address: 0x3c430
+#define regSDMA0_GFX_ICG_SDMA_CTRL                                                                      0x510c
+#define regSDMA0_GFX_ICG_SDMA_CTRL_BASE_IDX                                                             1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmadec:1
+// base address: 0x6180
+#define regSDMA1_SDMA_DEC_START                                                                         0x0600
+#define regSDMA1_SDMA_DEC_START_BASE_IDX                                                                0
+#define regSDMA1_SDMA_MCU_MISC_CNTL                                                                     0x0601
+#define regSDMA1_SDMA_MCU_MISC_CNTL_BASE_IDX                                                            0
+#define regSDMA1_SDMA_UCODE_REV                                                                         0x0603
+#define regSDMA1_SDMA_UCODE_REV_BASE_IDX                                                                0
+#define regSDMA1_SDMA_GLOBAL_TIMESTAMP_LO                                                               0x0605
+#define regSDMA1_SDMA_GLOBAL_TIMESTAMP_LO_BASE_IDX                                                      0
+#define regSDMA1_SDMA_GLOBAL_TIMESTAMP_HI                                                               0x0606
+#define regSDMA1_SDMA_GLOBAL_TIMESTAMP_HI_BASE_IDX                                                      0
+#define regSDMA1_SDMA_POWER_CNTL                                                                        0x060c
+#define regSDMA1_SDMA_POWER_CNTL_BASE_IDX                                                               0
+#define regSDMA1_SDMA_CNTL                                                                              0x060d
+#define regSDMA1_SDMA_CNTL_BASE_IDX                                                                     0
+#define regSDMA1_SDMA_CHICKEN_BITS                                                                      0x060e
+#define regSDMA1_SDMA_CHICKEN_BITS_BASE_IDX                                                             0
+#define regSDMA1_SDMA_CACHE_CNTL                                                                        0x060f
+#define regSDMA1_SDMA_CACHE_CNTL_BASE_IDX                                                               0
+#define regSDMA1_SDMA_RB_RPTR_FETCH                                                                     0x0620
+#define regSDMA1_SDMA_RB_RPTR_FETCH_BASE_IDX                                                            0
+#define regSDMA1_SDMA_RB_RPTR_FETCH_HI                                                                  0x0621
+#define regSDMA1_SDMA_RB_RPTR_FETCH_HI_BASE_IDX                                                         0
+#define regSDMA1_SDMA_IB_OFFSET_FETCH                                                                   0x0622
+#define regSDMA1_SDMA_IB_OFFSET_FETCH_BASE_IDX                                                          0
+#define regSDMA1_SDMA_PROGRAM                                                                           0x0623
+#define regSDMA1_SDMA_PROGRAM_BASE_IDX                                                                  0
+#define regSDMA1_SDMA_STATUS_REG                                                                        0x0624
+#define regSDMA1_SDMA_STATUS_REG_BASE_IDX                                                               0
+#define regSDMA1_SDMA_STATUS1_REG                                                                       0x0625
+#define regSDMA1_SDMA_STATUS1_REG_BASE_IDX                                                              0
+#define regSDMA1_SDMA_CNTL1                                                                             0x0626
+#define regSDMA1_SDMA_CNTL1_BASE_IDX                                                                    0
+#define regSDMA1_SDMA_HBM_PAGE_CONFIG                                                                   0x0627
+#define regSDMA1_SDMA_HBM_PAGE_CONFIG_BASE_IDX                                                          0
+#define regSDMA1_SDMA_FREEZE                                                                            0x0628
+#define regSDMA1_SDMA_FREEZE_BASE_IDX                                                                   0
+#define regSDMA1_SDMA_PROCESS_QUANTUM0                                                                  0x0629
+#define regSDMA1_SDMA_PROCESS_QUANTUM0_BASE_IDX                                                         0
+#define regSDMA1_SDMA_PROCESS_QUANTUM1                                                                  0x062a
+#define regSDMA1_SDMA_PROCESS_QUANTUM1_BASE_IDX                                                         0
+#define regSDMA1_SDMA_WATCHDOG_CNTL                                                                     0x062b
+#define regSDMA1_SDMA_WATCHDOG_CNTL_BASE_IDX                                                            0
+#define regSDMA1_SDMA_QUEUE_STATUS0                                                                     0x062c
+#define regSDMA1_SDMA_QUEUE_STATUS0_BASE_IDX                                                            0
+#define regSDMA1_SDMA_QUEUE_STATUS1                                                                     0x062d
+#define regSDMA1_SDMA_QUEUE_STATUS1_BASE_IDX                                                            0
+#define regSDMA1_SDMA_ID                                                                                0x062e
+#define regSDMA1_SDMA_ID_BASE_IDX                                                                       0
+#define regSDMA1_SDMA_VERSION                                                                           0x062f
+#define regSDMA1_SDMA_VERSION_BASE_IDX                                                                  0
+#define regSDMA1_SDMA_STATUS2_REG                                                                       0x0630
+#define regSDMA1_SDMA_STATUS2_REG_BASE_IDX                                                              0
+#define regSDMA1_SDMA_ATOMIC_CNTL                                                                       0x0631
+#define regSDMA1_SDMA_ATOMIC_CNTL_BASE_IDX                                                              0
+#define regSDMA1_SDMA_ATOMIC_PERMS_CNTL                                                                 0x0632
+#define regSDMA1_SDMA_ATOMIC_PERMS_CNTL_BASE_IDX                                                        0
+#define regSDMA1_SDMA_ATOMIC_PREOP_LO                                                                   0x0633
+#define regSDMA1_SDMA_ATOMIC_PREOP_LO_BASE_IDX                                                          0
+#define regSDMA1_SDMA_ATOMIC_PREOP_HI                                                                   0x0634
+#define regSDMA1_SDMA_ATOMIC_PREOP_HI_BASE_IDX                                                          0
+#define regSDMA1_SDMA_DCC_CNTL                                                                          0x0635
+#define regSDMA1_SDMA_DCC_CNTL_BASE_IDX                                                                 0
+#define regSDMA1_SDMA_UTCL1_WARMUPL2_CNTL                                                               0x0636
+#define regSDMA1_SDMA_UTCL1_WARMUPL2_CNTL_BASE_IDX                                                      0
+#define regSDMA1_SDMA_UTCL1_CNTL                                                                        0x0637
+#define regSDMA1_SDMA_UTCL1_CNTL_BASE_IDX                                                               0
+#define regSDMA1_SDMA_UTCL1_WATERMK                                                                     0x0638
+#define regSDMA1_SDMA_UTCL1_WATERMK_BASE_IDX                                                            0
+#define regSDMA1_SDMA_UTCL1_TIMEOUT                                                                     0x0639
+#define regSDMA1_SDMA_UTCL1_TIMEOUT_BASE_IDX                                                            0
+#define regSDMA1_SDMA_UTCL1_PAGE                                                                        0x063a
+#define regSDMA1_SDMA_UTCL1_PAGE_BASE_IDX                                                               0
+#define regSDMA1_SDMA_EXTERNAL_FROZEN                                                                   0x063b
+#define regSDMA1_SDMA_EXTERNAL_FROZEN_BASE_IDX                                                          0
+#define regSDMA1_SDMA_FREEZE_TRIGGER                                                                    0x063c
+#define regSDMA1_SDMA_FREEZE_TRIGGER_BASE_IDX                                                           0
+#define regSDMA1_SDMA_UTCL1_CACHE_CNTL                                                                  0x0640
+#define regSDMA1_SDMA_UTCL1_CACHE_CNTL_BASE_IDX                                                         0
+#define regSDMA1_SDMA_UTCL1_RD_STATUS                                                                   0x0641
+#define regSDMA1_SDMA_UTCL1_RD_STATUS_BASE_IDX                                                          0
+#define regSDMA1_SDMA_UTCL1_WR_STATUS                                                                   0x0642
+#define regSDMA1_SDMA_UTCL1_WR_STATUS_BASE_IDX                                                          0
+#define regSDMA1_SDMA_UTCL1_INV0                                                                        0x0643
+#define regSDMA1_SDMA_UTCL1_INV0_BASE_IDX                                                               0
+#define regSDMA1_SDMA_UTCL1_INV1                                                                        0x0644
+#define regSDMA1_SDMA_UTCL1_INV1_BASE_IDX                                                               0
+#define regSDMA1_SDMA_UTCL1_INV2                                                                        0x0645
+#define regSDMA1_SDMA_UTCL1_INV2_BASE_IDX                                                               0
+#define regSDMA1_SDMA_UTCL1_RD_XNACK0                                                                   0x0646
+#define regSDMA1_SDMA_UTCL1_RD_XNACK0_BASE_IDX                                                          0
+#define regSDMA1_SDMA_UTCL1_RD_XNACK1                                                                   0x0647
+#define regSDMA1_SDMA_UTCL1_RD_XNACK1_BASE_IDX                                                          0
+#define regSDMA1_SDMA_UTCL1_WR_XNACK0                                                                   0x0648
+#define regSDMA1_SDMA_UTCL1_WR_XNACK0_BASE_IDX                                                          0
+#define regSDMA1_SDMA_UTCL1_WR_XNACK1                                                                   0x0649
+#define regSDMA1_SDMA_UTCL1_WR_XNACK1_BASE_IDX                                                          0
+#define regSDMA1_SDMA_CHICKEN_BITS_2                                                                    0x064a
+#define regSDMA1_SDMA_CHICKEN_BITS_2_BASE_IDX                                                           0
+#define regSDMA1_SDMA_STATUS3_REG                                                                       0x064b
+#define regSDMA1_SDMA_STATUS3_REG_BASE_IDX                                                              0
+#define regSDMA1_SDMA_GLOBAL_QUANTUM                                                                    0x064c
+#define regSDMA1_SDMA_GLOBAL_QUANTUM_BASE_IDX                                                           0
+#define regSDMA1_SDMA_ERROR_LOG                                                                         0x064d
+#define regSDMA1_SDMA_ERROR_LOG_BASE_IDX                                                                0
+#define regSDMA1_SDMA_PUB_DUMMY_REG0                                                                    0x064e
+#define regSDMA1_SDMA_PUB_DUMMY_REG0_BASE_IDX                                                           0
+#define regSDMA1_SDMA_PUB_DUMMY_REG1                                                                    0x064f
+#define regSDMA1_SDMA_PUB_DUMMY_REG1_BASE_IDX                                                           0
+#define regSDMA1_SDMA_PUB_DUMMY_REG2                                                                    0x0650
+#define regSDMA1_SDMA_PUB_DUMMY_REG2_BASE_IDX                                                           0
+#define regSDMA1_SDMA_PUB_DUMMY_REG3                                                                    0x0651
+#define regSDMA1_SDMA_PUB_DUMMY_REG3_BASE_IDX                                                           0
+#define regSDMA1_SDMA_MCU_COUNTER                                                                       0x0652
+#define regSDMA1_SDMA_MCU_COUNTER_BASE_IDX                                                              0
+#define regSDMA1_SDMA_CRD_CNTL                                                                          0x0653
+#define regSDMA1_SDMA_CRD_CNTL_BASE_IDX                                                                 0
+#define regSDMA1_SDMA_RLC_CGCG_CTRL                                                                     0x0654
+#define regSDMA1_SDMA_RLC_CGCG_CTRL_BASE_IDX                                                            0
+#define regSDMA1_SDMA_GPU_IOV_VIOLATION_LOG                                                             0x0655
+#define regSDMA1_SDMA_GPU_IOV_VIOLATION_LOG_BASE_IDX                                                    0
+#define regSDMA1_SDMA_AQL_STATUS                                                                        0x0657
+#define regSDMA1_SDMA_AQL_STATUS_BASE_IDX                                                               0
+#define regSDMA1_SDMA_TLBI_GCR_CNTL                                                                     0x0660
+#define regSDMA1_SDMA_TLBI_GCR_CNTL_BASE_IDX                                                            0
+#define regSDMA1_SDMA_INT_STATUS                                                                        0x0661
+#define regSDMA1_SDMA_INT_STATUS_BASE_IDX                                                               0
+#define regSDMA1_SDMA_GPU_IOV_VIOLATION_LOG2                                                            0x0662
+#define regSDMA1_SDMA_GPU_IOV_VIOLATION_LOG2_BASE_IDX                                                   0
+#define regSDMA1_SDMA_INVALID_ADDR_LO                                                                   0x0663
+#define regSDMA1_SDMA_INVALID_ADDR_LO_BASE_IDX                                                          0
+#define regSDMA1_SDMA_INVALID_ADDR_HI                                                                   0x0664
+#define regSDMA1_SDMA_INVALID_ADDR_HI_BASE_IDX                                                          0
+#define regSDMA1_SDMA_INVALID_ADDR_SOURCE                                                               0x0665
+#define regSDMA1_SDMA_INVALID_ADDR_SOURCE_BASE_IDX                                                      0
+#define regSDMA1_SDMA_CLOCK_GATING_STATUS                                                               0x0666
+#define regSDMA1_SDMA_CLOCK_GATING_STATUS_BASE_IDX                                                      0
+#define regSDMA1_SDMA_STATUS4_REG                                                                       0x0667
+#define regSDMA1_SDMA_STATUS4_REG_BASE_IDX                                                              0
+#define regSDMA1_SDMA_SCRATCH_RAM_DATA                                                                  0x0668
+#define regSDMA1_SDMA_SCRATCH_RAM_DATA_BASE_IDX                                                         0
+#define regSDMA1_SDMA_SCRATCH_RAM_ADDR                                                                  0x0669
+#define regSDMA1_SDMA_SCRATCH_RAM_ADDR_BASE_IDX                                                         0
+#define regSDMA1_SDMA_TIMESTAMP_CNTL                                                                    0x066a
+#define regSDMA1_SDMA_TIMESTAMP_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_STATUS5_REG                                                                       0x066b
+#define regSDMA1_SDMA_STATUS5_REG_BASE_IDX                                                              0
+#define regSDMA1_SDMA_QUEUE_RESET_REQ                                                                   0x066c
+#define regSDMA1_SDMA_QUEUE_RESET_REQ_BASE_IDX                                                          0
+#define regSDMA1_SDMA_QUEUE_DEQUEUE_REQUEST                                                             0x066d
+#define regSDMA1_SDMA_QUEUE_DEQUEUE_REQUEST_BASE_IDX                                                    0
+#define regSDMA1_SDMA_STATUS6_REG                                                                       0x066e
+#define regSDMA1_SDMA_STATUS6_REG_BASE_IDX                                                              0
+#define regSDMA1_SDMA_STATUS7_REG                                                                       0x066f
+#define regSDMA1_SDMA_STATUS7_REG_BASE_IDX                                                              0
+#define regSDMA1_SDMA_STATUS8_REG                                                                       0x0670
+#define regSDMA1_SDMA_STATUS8_REG_BASE_IDX                                                              0
+#define regSDMA1_SDMA_CE_CTRL                                                                           0x0671
+#define regSDMA1_SDMA_CE_CTRL_BASE_IDX                                                                  0
+#define regSDMA1_SDMA_RAS_STATUS                                                                        0x0672
+#define regSDMA1_SDMA_RAS_STATUS_BASE_IDX                                                               0
+#define regSDMA1_SDMA_POISON_INFO                                                                       0x0673
+#define regSDMA1_SDMA_POISON_INFO_BASE_IDX                                                              0
+#define regSDMA1_SDMA_MEMHUB_CNTL                                                                       0x0674
+#define regSDMA1_SDMA_MEMHUB_CNTL_BASE_IDX                                                              0
+#define regSDMA1_SDMA_QUEUE0_RB_CNTL                                                                    0x0800
+#define regSDMA1_SDMA_QUEUE0_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE0_RB_BASE                                                                    0x0801
+#define regSDMA1_SDMA_QUEUE0_RB_BASE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE0_RB_BASE_HI                                                                 0x0802
+#define regSDMA1_SDMA_QUEUE0_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE0_RB_RPTR                                                                    0x0803
+#define regSDMA1_SDMA_QUEUE0_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE0_RB_RPTR_HI                                                                 0x0804
+#define regSDMA1_SDMA_QUEUE0_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE0_RB_WPTR                                                                    0x0805
+#define regSDMA1_SDMA_QUEUE0_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE0_RB_WPTR_HI                                                                 0x0806
+#define regSDMA1_SDMA_QUEUE0_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE0_RB_RPTR_ADDR_LO                                                            0x0807
+#define regSDMA1_SDMA_QUEUE0_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE0_RB_RPTR_ADDR_HI                                                            0x0808
+#define regSDMA1_SDMA_QUEUE0_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE0_IB_CNTL                                                                    0x0809
+#define regSDMA1_SDMA_QUEUE0_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE0_IB_RPTR                                                                    0x080a
+#define regSDMA1_SDMA_QUEUE0_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE0_IB_OFFSET                                                                  0x080b
+#define regSDMA1_SDMA_QUEUE0_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE0_IB_BASE_LO                                                                 0x080c
+#define regSDMA1_SDMA_QUEUE0_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE0_IB_BASE_HI                                                                 0x080d
+#define regSDMA1_SDMA_QUEUE0_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE0_IB_SIZE                                                                    0x080e
+#define regSDMA1_SDMA_QUEUE0_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE0_DOORBELL                                                                   0x080f
+#define regSDMA1_SDMA_QUEUE0_DOORBELL_BASE_IDX                                                          0
+#define regSDMA1_SDMA_QUEUE0_DOORBELL_LOG                                                               0x0810
+#define regSDMA1_SDMA_QUEUE0_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE0_DOORBELL_OFFSET                                                            0x0811
+#define regSDMA1_SDMA_QUEUE0_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE0_CSA_ADDR_LO                                                                0x0812
+#define regSDMA1_SDMA_QUEUE0_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE0_CSA_ADDR_HI                                                                0x0813
+#define regSDMA1_SDMA_QUEUE0_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE0_SCHEDULE_CNTL                                                              0x0814
+#define regSDMA1_SDMA_QUEUE0_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE0_IB_SUB_REMAIN                                                              0x0815
+#define regSDMA1_SDMA_QUEUE0_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE0_PREEMPT                                                                    0x0816
+#define regSDMA1_SDMA_QUEUE0_PREEMPT_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE0_DUMMY_REG                                                                  0x0817
+#define regSDMA1_SDMA_QUEUE0_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_LO                                                       0x0818
+#define regSDMA1_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_HI                                                       0x0819
+#define regSDMA1_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE0_RB_AQL_CNTL                                                                0x081a
+#define regSDMA1_SDMA_QUEUE0_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE0_MINOR_PTR_UPDATE                                                           0x081b
+#define regSDMA1_SDMA_QUEUE0_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS                                                      0x081e
+#define regSDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_CNTL                                                                0x081f
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA0                                                               0x0820
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA1                                                               0x0821
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA2                                                               0x0822
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA3                                                               0x0823
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA4                                                               0x0824
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA5                                                               0x0825
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA6                                                               0x0826
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA7                                                               0x0827
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA8                                                               0x0828
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA9                                                               0x0829
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA10                                                              0x082a
+#define regSDMA1_SDMA_QUEUE0_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE0_UTILIZATION_LO                                                             0x082b
+#define regSDMA1_SDMA_QUEUE0_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE0_UTILIZATION_HI                                                             0x082c
+#define regSDMA1_SDMA_QUEUE0_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE0_WAIT_UNSATISFIED_THD                                                       0x082d
+#define regSDMA1_SDMA_QUEUE0_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE0_MQD_BASE_ADDR_LO                                                           0x082e
+#define regSDMA1_SDMA_QUEUE0_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE0_MQD_BASE_ADDR_HI                                                           0x082f
+#define regSDMA1_SDMA_QUEUE0_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE0_MQD_CONTROL                                                                0x0830
+#define regSDMA1_SDMA_QUEUE0_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE0_CONTEXT_STATUS                                                             0x0831
+#define regSDMA1_SDMA_QUEUE0_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE0_DUMMY_REG1                                                                 0x0832
+#define regSDMA1_SDMA_QUEUE0_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE0_DUMMY_REG2                                                                 0x0833
+#define regSDMA1_SDMA_QUEUE0_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE1_RB_CNTL                                                                    0x083c
+#define regSDMA1_SDMA_QUEUE1_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE1_RB_BASE                                                                    0x083d
+#define regSDMA1_SDMA_QUEUE1_RB_BASE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE1_RB_BASE_HI                                                                 0x083e
+#define regSDMA1_SDMA_QUEUE1_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE1_RB_RPTR                                                                    0x083f
+#define regSDMA1_SDMA_QUEUE1_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE1_RB_RPTR_HI                                                                 0x0840
+#define regSDMA1_SDMA_QUEUE1_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE1_RB_WPTR                                                                    0x0841
+#define regSDMA1_SDMA_QUEUE1_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE1_RB_WPTR_HI                                                                 0x0842
+#define regSDMA1_SDMA_QUEUE1_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE1_RB_RPTR_ADDR_LO                                                            0x0843
+#define regSDMA1_SDMA_QUEUE1_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE1_RB_RPTR_ADDR_HI                                                            0x0844
+#define regSDMA1_SDMA_QUEUE1_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE1_IB_CNTL                                                                    0x0845
+#define regSDMA1_SDMA_QUEUE1_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE1_IB_RPTR                                                                    0x0846
+#define regSDMA1_SDMA_QUEUE1_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE1_IB_OFFSET                                                                  0x0847
+#define regSDMA1_SDMA_QUEUE1_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE1_IB_BASE_LO                                                                 0x0848
+#define regSDMA1_SDMA_QUEUE1_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE1_IB_BASE_HI                                                                 0x0849
+#define regSDMA1_SDMA_QUEUE1_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE1_IB_SIZE                                                                    0x084a
+#define regSDMA1_SDMA_QUEUE1_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE1_DOORBELL                                                                   0x084b
+#define regSDMA1_SDMA_QUEUE1_DOORBELL_BASE_IDX                                                          0
+#define regSDMA1_SDMA_QUEUE1_DOORBELL_LOG                                                               0x084c
+#define regSDMA1_SDMA_QUEUE1_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE1_DOORBELL_OFFSET                                                            0x084d
+#define regSDMA1_SDMA_QUEUE1_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE1_CSA_ADDR_LO                                                                0x084e
+#define regSDMA1_SDMA_QUEUE1_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE1_CSA_ADDR_HI                                                                0x084f
+#define regSDMA1_SDMA_QUEUE1_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE1_SCHEDULE_CNTL                                                              0x0850
+#define regSDMA1_SDMA_QUEUE1_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE1_IB_SUB_REMAIN                                                              0x0851
+#define regSDMA1_SDMA_QUEUE1_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE1_PREEMPT                                                                    0x0852
+#define regSDMA1_SDMA_QUEUE1_PREEMPT_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE1_DUMMY_REG                                                                  0x0853
+#define regSDMA1_SDMA_QUEUE1_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_LO                                                       0x0854
+#define regSDMA1_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_HI                                                       0x0855
+#define regSDMA1_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE1_RB_AQL_CNTL                                                                0x0856
+#define regSDMA1_SDMA_QUEUE1_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE1_MINOR_PTR_UPDATE                                                           0x0857
+#define regSDMA1_SDMA_QUEUE1_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS                                                      0x085a
+#define regSDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_CNTL                                                                0x085b
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA0                                                               0x085c
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA1                                                               0x085d
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA2                                                               0x085e
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA3                                                               0x085f
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA4                                                               0x0860
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA5                                                               0x0861
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA6                                                               0x0862
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA7                                                               0x0863
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA8                                                               0x0864
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA9                                                               0x0865
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA10                                                              0x0866
+#define regSDMA1_SDMA_QUEUE1_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE1_UTILIZATION_LO                                                             0x0867
+#define regSDMA1_SDMA_QUEUE1_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE1_UTILIZATION_HI                                                             0x0868
+#define regSDMA1_SDMA_QUEUE1_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE1_WAIT_UNSATISFIED_THD                                                       0x0869
+#define regSDMA1_SDMA_QUEUE1_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE1_MQD_BASE_ADDR_LO                                                           0x086a
+#define regSDMA1_SDMA_QUEUE1_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE1_MQD_BASE_ADDR_HI                                                           0x086b
+#define regSDMA1_SDMA_QUEUE1_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE1_MQD_CONTROL                                                                0x086c
+#define regSDMA1_SDMA_QUEUE1_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE1_CONTEXT_STATUS                                                             0x086d
+#define regSDMA1_SDMA_QUEUE1_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE1_DUMMY_REG1                                                                 0x086e
+#define regSDMA1_SDMA_QUEUE1_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE1_DUMMY_REG2                                                                 0x086f
+#define regSDMA1_SDMA_QUEUE1_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE2_RB_CNTL                                                                    0x0878
+#define regSDMA1_SDMA_QUEUE2_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE2_RB_BASE                                                                    0x0879
+#define regSDMA1_SDMA_QUEUE2_RB_BASE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE2_RB_BASE_HI                                                                 0x087a
+#define regSDMA1_SDMA_QUEUE2_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE2_RB_RPTR                                                                    0x087b
+#define regSDMA1_SDMA_QUEUE2_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE2_RB_RPTR_HI                                                                 0x087c
+#define regSDMA1_SDMA_QUEUE2_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE2_RB_WPTR                                                                    0x087d
+#define regSDMA1_SDMA_QUEUE2_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE2_RB_WPTR_HI                                                                 0x087e
+#define regSDMA1_SDMA_QUEUE2_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE2_RB_RPTR_ADDR_LO                                                            0x087f
+#define regSDMA1_SDMA_QUEUE2_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE2_RB_RPTR_ADDR_HI                                                            0x0880
+#define regSDMA1_SDMA_QUEUE2_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE2_IB_CNTL                                                                    0x0881
+#define regSDMA1_SDMA_QUEUE2_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE2_IB_RPTR                                                                    0x0882
+#define regSDMA1_SDMA_QUEUE2_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE2_IB_OFFSET                                                                  0x0883
+#define regSDMA1_SDMA_QUEUE2_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE2_IB_BASE_LO                                                                 0x0884
+#define regSDMA1_SDMA_QUEUE2_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE2_IB_BASE_HI                                                                 0x0885
+#define regSDMA1_SDMA_QUEUE2_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE2_IB_SIZE                                                                    0x0886
+#define regSDMA1_SDMA_QUEUE2_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE2_DOORBELL                                                                   0x0887
+#define regSDMA1_SDMA_QUEUE2_DOORBELL_BASE_IDX                                                          0
+#define regSDMA1_SDMA_QUEUE2_DOORBELL_LOG                                                               0x0888
+#define regSDMA1_SDMA_QUEUE2_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE2_DOORBELL_OFFSET                                                            0x0889
+#define regSDMA1_SDMA_QUEUE2_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE2_CSA_ADDR_LO                                                                0x088a
+#define regSDMA1_SDMA_QUEUE2_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE2_CSA_ADDR_HI                                                                0x088b
+#define regSDMA1_SDMA_QUEUE2_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE2_SCHEDULE_CNTL                                                              0x088c
+#define regSDMA1_SDMA_QUEUE2_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE2_IB_SUB_REMAIN                                                              0x088d
+#define regSDMA1_SDMA_QUEUE2_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE2_PREEMPT                                                                    0x088e
+#define regSDMA1_SDMA_QUEUE2_PREEMPT_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE2_DUMMY_REG                                                                  0x088f
+#define regSDMA1_SDMA_QUEUE2_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_LO                                                       0x0890
+#define regSDMA1_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_HI                                                       0x0891
+#define regSDMA1_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE2_RB_AQL_CNTL                                                                0x0892
+#define regSDMA1_SDMA_QUEUE2_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE2_MINOR_PTR_UPDATE                                                           0x0893
+#define regSDMA1_SDMA_QUEUE2_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS                                                      0x0896
+#define regSDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_CNTL                                                                0x0897
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA0                                                               0x0898
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA1                                                               0x0899
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA2                                                               0x089a
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA3                                                               0x089b
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA4                                                               0x089c
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA5                                                               0x089d
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA6                                                               0x089e
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA7                                                               0x089f
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA8                                                               0x08a0
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA9                                                               0x08a1
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA10                                                              0x08a2
+#define regSDMA1_SDMA_QUEUE2_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE2_UTILIZATION_LO                                                             0x08a3
+#define regSDMA1_SDMA_QUEUE2_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE2_UTILIZATION_HI                                                             0x08a4
+#define regSDMA1_SDMA_QUEUE2_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE2_WAIT_UNSATISFIED_THD                                                       0x08a5
+#define regSDMA1_SDMA_QUEUE2_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE2_MQD_BASE_ADDR_LO                                                           0x08a6
+#define regSDMA1_SDMA_QUEUE2_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE2_MQD_BASE_ADDR_HI                                                           0x08a7
+#define regSDMA1_SDMA_QUEUE2_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE2_MQD_CONTROL                                                                0x08a8
+#define regSDMA1_SDMA_QUEUE2_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE2_CONTEXT_STATUS                                                             0x08a9
+#define regSDMA1_SDMA_QUEUE2_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE2_DUMMY_REG1                                                                 0x08aa
+#define regSDMA1_SDMA_QUEUE2_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE2_DUMMY_REG2                                                                 0x08ab
+#define regSDMA1_SDMA_QUEUE2_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE3_RB_CNTL                                                                    0x08b4
+#define regSDMA1_SDMA_QUEUE3_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE3_RB_BASE                                                                    0x08b5
+#define regSDMA1_SDMA_QUEUE3_RB_BASE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE3_RB_BASE_HI                                                                 0x08b6
+#define regSDMA1_SDMA_QUEUE3_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE3_RB_RPTR                                                                    0x08b7
+#define regSDMA1_SDMA_QUEUE3_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE3_RB_RPTR_HI                                                                 0x08b8
+#define regSDMA1_SDMA_QUEUE3_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE3_RB_WPTR                                                                    0x08b9
+#define regSDMA1_SDMA_QUEUE3_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE3_RB_WPTR_HI                                                                 0x08ba
+#define regSDMA1_SDMA_QUEUE3_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE3_RB_RPTR_ADDR_LO                                                            0x08bb
+#define regSDMA1_SDMA_QUEUE3_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE3_RB_RPTR_ADDR_HI                                                            0x08bc
+#define regSDMA1_SDMA_QUEUE3_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE3_IB_CNTL                                                                    0x08bd
+#define regSDMA1_SDMA_QUEUE3_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE3_IB_RPTR                                                                    0x08be
+#define regSDMA1_SDMA_QUEUE3_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE3_IB_OFFSET                                                                  0x08bf
+#define regSDMA1_SDMA_QUEUE3_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE3_IB_BASE_LO                                                                 0x08c0
+#define regSDMA1_SDMA_QUEUE3_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE3_IB_BASE_HI                                                                 0x08c1
+#define regSDMA1_SDMA_QUEUE3_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE3_IB_SIZE                                                                    0x08c2
+#define regSDMA1_SDMA_QUEUE3_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE3_DOORBELL                                                                   0x08c3
+#define regSDMA1_SDMA_QUEUE3_DOORBELL_BASE_IDX                                                          0
+#define regSDMA1_SDMA_QUEUE3_DOORBELL_LOG                                                               0x08c4
+#define regSDMA1_SDMA_QUEUE3_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE3_DOORBELL_OFFSET                                                            0x08c5
+#define regSDMA1_SDMA_QUEUE3_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE3_CSA_ADDR_LO                                                                0x08c6
+#define regSDMA1_SDMA_QUEUE3_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE3_CSA_ADDR_HI                                                                0x08c7
+#define regSDMA1_SDMA_QUEUE3_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE3_SCHEDULE_CNTL                                                              0x08c8
+#define regSDMA1_SDMA_QUEUE3_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE3_IB_SUB_REMAIN                                                              0x08c9
+#define regSDMA1_SDMA_QUEUE3_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE3_PREEMPT                                                                    0x08ca
+#define regSDMA1_SDMA_QUEUE3_PREEMPT_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE3_DUMMY_REG                                                                  0x08cb
+#define regSDMA1_SDMA_QUEUE3_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_LO                                                       0x08cc
+#define regSDMA1_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_HI                                                       0x08cd
+#define regSDMA1_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE3_RB_AQL_CNTL                                                                0x08ce
+#define regSDMA1_SDMA_QUEUE3_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE3_MINOR_PTR_UPDATE                                                           0x08cf
+#define regSDMA1_SDMA_QUEUE3_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS                                                      0x08d2
+#define regSDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_CNTL                                                                0x08d3
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA0                                                               0x08d4
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA1                                                               0x08d5
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA2                                                               0x08d6
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA3                                                               0x08d7
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA4                                                               0x08d8
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA5                                                               0x08d9
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA6                                                               0x08da
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA7                                                               0x08db
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA8                                                               0x08dc
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA9                                                               0x08dd
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA10                                                              0x08de
+#define regSDMA1_SDMA_QUEUE3_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE3_UTILIZATION_LO                                                             0x08df
+#define regSDMA1_SDMA_QUEUE3_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE3_UTILIZATION_HI                                                             0x08e0
+#define regSDMA1_SDMA_QUEUE3_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE3_WAIT_UNSATISFIED_THD                                                       0x08e1
+#define regSDMA1_SDMA_QUEUE3_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE3_MQD_BASE_ADDR_LO                                                           0x08e2
+#define regSDMA1_SDMA_QUEUE3_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE3_MQD_BASE_ADDR_HI                                                           0x08e3
+#define regSDMA1_SDMA_QUEUE3_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE3_MQD_CONTROL                                                                0x08e4
+#define regSDMA1_SDMA_QUEUE3_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE3_CONTEXT_STATUS                                                             0x08e5
+#define regSDMA1_SDMA_QUEUE3_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE3_DUMMY_REG1                                                                 0x08e6
+#define regSDMA1_SDMA_QUEUE3_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE3_DUMMY_REG2                                                                 0x08e7
+#define regSDMA1_SDMA_QUEUE3_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE4_RB_CNTL                                                                    0x08f0
+#define regSDMA1_SDMA_QUEUE4_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE4_RB_BASE                                                                    0x08f1
+#define regSDMA1_SDMA_QUEUE4_RB_BASE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE4_RB_BASE_HI                                                                 0x08f2
+#define regSDMA1_SDMA_QUEUE4_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE4_RB_RPTR                                                                    0x08f3
+#define regSDMA1_SDMA_QUEUE4_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE4_RB_RPTR_HI                                                                 0x08f4
+#define regSDMA1_SDMA_QUEUE4_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE4_RB_WPTR                                                                    0x08f5
+#define regSDMA1_SDMA_QUEUE4_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE4_RB_WPTR_HI                                                                 0x08f6
+#define regSDMA1_SDMA_QUEUE4_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE4_RB_RPTR_ADDR_LO                                                            0x08f7
+#define regSDMA1_SDMA_QUEUE4_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE4_RB_RPTR_ADDR_HI                                                            0x08f8
+#define regSDMA1_SDMA_QUEUE4_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE4_IB_CNTL                                                                    0x08f9
+#define regSDMA1_SDMA_QUEUE4_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE4_IB_RPTR                                                                    0x08fa
+#define regSDMA1_SDMA_QUEUE4_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE4_IB_OFFSET                                                                  0x08fb
+#define regSDMA1_SDMA_QUEUE4_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE4_IB_BASE_LO                                                                 0x08fc
+#define regSDMA1_SDMA_QUEUE4_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE4_IB_BASE_HI                                                                 0x08fd
+#define regSDMA1_SDMA_QUEUE4_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE4_IB_SIZE                                                                    0x08fe
+#define regSDMA1_SDMA_QUEUE4_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE4_DOORBELL                                                                   0x08ff
+#define regSDMA1_SDMA_QUEUE4_DOORBELL_BASE_IDX                                                          0
+#define regSDMA1_SDMA_QUEUE4_DOORBELL_LOG                                                               0x0900
+#define regSDMA1_SDMA_QUEUE4_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE4_DOORBELL_OFFSET                                                            0x0901
+#define regSDMA1_SDMA_QUEUE4_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE4_CSA_ADDR_LO                                                                0x0902
+#define regSDMA1_SDMA_QUEUE4_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE4_CSA_ADDR_HI                                                                0x0903
+#define regSDMA1_SDMA_QUEUE4_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE4_SCHEDULE_CNTL                                                              0x0904
+#define regSDMA1_SDMA_QUEUE4_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE4_IB_SUB_REMAIN                                                              0x0905
+#define regSDMA1_SDMA_QUEUE4_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE4_PREEMPT                                                                    0x0906
+#define regSDMA1_SDMA_QUEUE4_PREEMPT_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE4_DUMMY_REG                                                                  0x0907
+#define regSDMA1_SDMA_QUEUE4_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_LO                                                       0x0908
+#define regSDMA1_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_HI                                                       0x0909
+#define regSDMA1_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE4_RB_AQL_CNTL                                                                0x090a
+#define regSDMA1_SDMA_QUEUE4_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE4_MINOR_PTR_UPDATE                                                           0x090b
+#define regSDMA1_SDMA_QUEUE4_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS                                                      0x090e
+#define regSDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_CNTL                                                                0x090f
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA0                                                               0x0910
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA1                                                               0x0911
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA2                                                               0x0912
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA3                                                               0x0913
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA4                                                               0x0914
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA5                                                               0x0915
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA6                                                               0x0916
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA7                                                               0x0917
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA8                                                               0x0918
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA9                                                               0x0919
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA10                                                              0x091a
+#define regSDMA1_SDMA_QUEUE4_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE4_UTILIZATION_LO                                                             0x091b
+#define regSDMA1_SDMA_QUEUE4_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE4_UTILIZATION_HI                                                             0x091c
+#define regSDMA1_SDMA_QUEUE4_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE4_WAIT_UNSATISFIED_THD                                                       0x091d
+#define regSDMA1_SDMA_QUEUE4_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE4_MQD_BASE_ADDR_LO                                                           0x091e
+#define regSDMA1_SDMA_QUEUE4_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE4_MQD_BASE_ADDR_HI                                                           0x091f
+#define regSDMA1_SDMA_QUEUE4_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE4_MQD_CONTROL                                                                0x0920
+#define regSDMA1_SDMA_QUEUE4_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE4_CONTEXT_STATUS                                                             0x0921
+#define regSDMA1_SDMA_QUEUE4_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE4_DUMMY_REG1                                                                 0x0922
+#define regSDMA1_SDMA_QUEUE4_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE4_DUMMY_REG2                                                                 0x0923
+#define regSDMA1_SDMA_QUEUE4_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE5_RB_CNTL                                                                    0x092c
+#define regSDMA1_SDMA_QUEUE5_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE5_RB_BASE                                                                    0x092d
+#define regSDMA1_SDMA_QUEUE5_RB_BASE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE5_RB_BASE_HI                                                                 0x092e
+#define regSDMA1_SDMA_QUEUE5_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE5_RB_RPTR                                                                    0x092f
+#define regSDMA1_SDMA_QUEUE5_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE5_RB_RPTR_HI                                                                 0x0930
+#define regSDMA1_SDMA_QUEUE5_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE5_RB_WPTR                                                                    0x0931
+#define regSDMA1_SDMA_QUEUE5_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE5_RB_WPTR_HI                                                                 0x0932
+#define regSDMA1_SDMA_QUEUE5_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE5_RB_RPTR_ADDR_LO                                                            0x0933
+#define regSDMA1_SDMA_QUEUE5_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE5_RB_RPTR_ADDR_HI                                                            0x0934
+#define regSDMA1_SDMA_QUEUE5_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE5_IB_CNTL                                                                    0x0935
+#define regSDMA1_SDMA_QUEUE5_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE5_IB_RPTR                                                                    0x0936
+#define regSDMA1_SDMA_QUEUE5_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE5_IB_OFFSET                                                                  0x0937
+#define regSDMA1_SDMA_QUEUE5_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE5_IB_BASE_LO                                                                 0x0938
+#define regSDMA1_SDMA_QUEUE5_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE5_IB_BASE_HI                                                                 0x0939
+#define regSDMA1_SDMA_QUEUE5_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE5_IB_SIZE                                                                    0x093a
+#define regSDMA1_SDMA_QUEUE5_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE5_DOORBELL                                                                   0x093b
+#define regSDMA1_SDMA_QUEUE5_DOORBELL_BASE_IDX                                                          0
+#define regSDMA1_SDMA_QUEUE5_DOORBELL_LOG                                                               0x093c
+#define regSDMA1_SDMA_QUEUE5_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE5_DOORBELL_OFFSET                                                            0x093d
+#define regSDMA1_SDMA_QUEUE5_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE5_CSA_ADDR_LO                                                                0x093e
+#define regSDMA1_SDMA_QUEUE5_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE5_CSA_ADDR_HI                                                                0x093f
+#define regSDMA1_SDMA_QUEUE5_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE5_SCHEDULE_CNTL                                                              0x0940
+#define regSDMA1_SDMA_QUEUE5_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE5_IB_SUB_REMAIN                                                              0x0941
+#define regSDMA1_SDMA_QUEUE5_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE5_PREEMPT                                                                    0x0942
+#define regSDMA1_SDMA_QUEUE5_PREEMPT_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE5_DUMMY_REG                                                                  0x0943
+#define regSDMA1_SDMA_QUEUE5_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_LO                                                       0x0944
+#define regSDMA1_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_HI                                                       0x0945
+#define regSDMA1_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE5_RB_AQL_CNTL                                                                0x0946
+#define regSDMA1_SDMA_QUEUE5_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE5_MINOR_PTR_UPDATE                                                           0x0947
+#define regSDMA1_SDMA_QUEUE5_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS                                                      0x094a
+#define regSDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_CNTL                                                                0x094b
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA0                                                               0x094c
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA1                                                               0x094d
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA2                                                               0x094e
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA3                                                               0x094f
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA4                                                               0x0950
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA5                                                               0x0951
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA6                                                               0x0952
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA7                                                               0x0953
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA8                                                               0x0954
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA9                                                               0x0955
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA10                                                              0x0956
+#define regSDMA1_SDMA_QUEUE5_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE5_UTILIZATION_LO                                                             0x0957
+#define regSDMA1_SDMA_QUEUE5_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE5_UTILIZATION_HI                                                             0x0958
+#define regSDMA1_SDMA_QUEUE5_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE5_WAIT_UNSATISFIED_THD                                                       0x0959
+#define regSDMA1_SDMA_QUEUE5_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE5_MQD_BASE_ADDR_LO                                                           0x095a
+#define regSDMA1_SDMA_QUEUE5_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE5_MQD_BASE_ADDR_HI                                                           0x095b
+#define regSDMA1_SDMA_QUEUE5_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE5_MQD_CONTROL                                                                0x095c
+#define regSDMA1_SDMA_QUEUE5_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE5_CONTEXT_STATUS                                                             0x095d
+#define regSDMA1_SDMA_QUEUE5_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE5_DUMMY_REG1                                                                 0x095e
+#define regSDMA1_SDMA_QUEUE5_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE5_DUMMY_REG2                                                                 0x095f
+#define regSDMA1_SDMA_QUEUE5_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE6_RB_CNTL                                                                    0x0968
+#define regSDMA1_SDMA_QUEUE6_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE6_RB_BASE                                                                    0x0969
+#define regSDMA1_SDMA_QUEUE6_RB_BASE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE6_RB_BASE_HI                                                                 0x096a
+#define regSDMA1_SDMA_QUEUE6_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE6_RB_RPTR                                                                    0x096b
+#define regSDMA1_SDMA_QUEUE6_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE6_RB_RPTR_HI                                                                 0x096c
+#define regSDMA1_SDMA_QUEUE6_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE6_RB_WPTR                                                                    0x096d
+#define regSDMA1_SDMA_QUEUE6_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE6_RB_WPTR_HI                                                                 0x096e
+#define regSDMA1_SDMA_QUEUE6_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE6_RB_RPTR_ADDR_LO                                                            0x096f
+#define regSDMA1_SDMA_QUEUE6_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE6_RB_RPTR_ADDR_HI                                                            0x0970
+#define regSDMA1_SDMA_QUEUE6_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE6_IB_CNTL                                                                    0x0971
+#define regSDMA1_SDMA_QUEUE6_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE6_IB_RPTR                                                                    0x0972
+#define regSDMA1_SDMA_QUEUE6_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE6_IB_OFFSET                                                                  0x0973
+#define regSDMA1_SDMA_QUEUE6_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE6_IB_BASE_LO                                                                 0x0974
+#define regSDMA1_SDMA_QUEUE6_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE6_IB_BASE_HI                                                                 0x0975
+#define regSDMA1_SDMA_QUEUE6_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE6_IB_SIZE                                                                    0x0976
+#define regSDMA1_SDMA_QUEUE6_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE6_DOORBELL                                                                   0x0977
+#define regSDMA1_SDMA_QUEUE6_DOORBELL_BASE_IDX                                                          0
+#define regSDMA1_SDMA_QUEUE6_DOORBELL_LOG                                                               0x0978
+#define regSDMA1_SDMA_QUEUE6_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE6_DOORBELL_OFFSET                                                            0x0979
+#define regSDMA1_SDMA_QUEUE6_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE6_CSA_ADDR_LO                                                                0x097a
+#define regSDMA1_SDMA_QUEUE6_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE6_CSA_ADDR_HI                                                                0x097b
+#define regSDMA1_SDMA_QUEUE6_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE6_SCHEDULE_CNTL                                                              0x097c
+#define regSDMA1_SDMA_QUEUE6_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE6_IB_SUB_REMAIN                                                              0x097d
+#define regSDMA1_SDMA_QUEUE6_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE6_PREEMPT                                                                    0x097e
+#define regSDMA1_SDMA_QUEUE6_PREEMPT_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE6_DUMMY_REG                                                                  0x097f
+#define regSDMA1_SDMA_QUEUE6_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_LO                                                       0x0980
+#define regSDMA1_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_HI                                                       0x0981
+#define regSDMA1_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE6_RB_AQL_CNTL                                                                0x0982
+#define regSDMA1_SDMA_QUEUE6_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE6_MINOR_PTR_UPDATE                                                           0x0983
+#define regSDMA1_SDMA_QUEUE6_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS                                                      0x0986
+#define regSDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_CNTL                                                                0x0987
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA0                                                               0x0988
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA1                                                               0x0989
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA2                                                               0x098a
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA3                                                               0x098b
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA4                                                               0x098c
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA5                                                               0x098d
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA6                                                               0x098e
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA7                                                               0x098f
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA8                                                               0x0990
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA9                                                               0x0991
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA10                                                              0x0992
+#define regSDMA1_SDMA_QUEUE6_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE6_UTILIZATION_LO                                                             0x0993
+#define regSDMA1_SDMA_QUEUE6_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE6_UTILIZATION_HI                                                             0x0994
+#define regSDMA1_SDMA_QUEUE6_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE6_WAIT_UNSATISFIED_THD                                                       0x0995
+#define regSDMA1_SDMA_QUEUE6_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE6_MQD_BASE_ADDR_LO                                                           0x0996
+#define regSDMA1_SDMA_QUEUE6_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE6_MQD_BASE_ADDR_HI                                                           0x0997
+#define regSDMA1_SDMA_QUEUE6_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE6_MQD_CONTROL                                                                0x0998
+#define regSDMA1_SDMA_QUEUE6_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE6_CONTEXT_STATUS                                                             0x0999
+#define regSDMA1_SDMA_QUEUE6_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE6_DUMMY_REG1                                                                 0x099a
+#define regSDMA1_SDMA_QUEUE6_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE6_DUMMY_REG2                                                                 0x099b
+#define regSDMA1_SDMA_QUEUE6_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE7_RB_CNTL                                                                    0x09a4
+#define regSDMA1_SDMA_QUEUE7_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE7_RB_BASE                                                                    0x09a5
+#define regSDMA1_SDMA_QUEUE7_RB_BASE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE7_RB_BASE_HI                                                                 0x09a6
+#define regSDMA1_SDMA_QUEUE7_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE7_RB_RPTR                                                                    0x09a7
+#define regSDMA1_SDMA_QUEUE7_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE7_RB_RPTR_HI                                                                 0x09a8
+#define regSDMA1_SDMA_QUEUE7_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE7_RB_WPTR                                                                    0x09a9
+#define regSDMA1_SDMA_QUEUE7_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE7_RB_WPTR_HI                                                                 0x09aa
+#define regSDMA1_SDMA_QUEUE7_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE7_RB_RPTR_ADDR_LO                                                            0x09ab
+#define regSDMA1_SDMA_QUEUE7_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE7_RB_RPTR_ADDR_HI                                                            0x09ac
+#define regSDMA1_SDMA_QUEUE7_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE7_IB_CNTL                                                                    0x09ad
+#define regSDMA1_SDMA_QUEUE7_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE7_IB_RPTR                                                                    0x09ae
+#define regSDMA1_SDMA_QUEUE7_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE7_IB_OFFSET                                                                  0x09af
+#define regSDMA1_SDMA_QUEUE7_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE7_IB_BASE_LO                                                                 0x09b0
+#define regSDMA1_SDMA_QUEUE7_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE7_IB_BASE_HI                                                                 0x09b1
+#define regSDMA1_SDMA_QUEUE7_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE7_IB_SIZE                                                                    0x09b2
+#define regSDMA1_SDMA_QUEUE7_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE7_DOORBELL                                                                   0x09b3
+#define regSDMA1_SDMA_QUEUE7_DOORBELL_BASE_IDX                                                          0
+#define regSDMA1_SDMA_QUEUE7_DOORBELL_LOG                                                               0x09b4
+#define regSDMA1_SDMA_QUEUE7_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE7_DOORBELL_OFFSET                                                            0x09b5
+#define regSDMA1_SDMA_QUEUE7_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE7_CSA_ADDR_LO                                                                0x09b6
+#define regSDMA1_SDMA_QUEUE7_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE7_CSA_ADDR_HI                                                                0x09b7
+#define regSDMA1_SDMA_QUEUE7_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE7_SCHEDULE_CNTL                                                              0x09b8
+#define regSDMA1_SDMA_QUEUE7_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE7_IB_SUB_REMAIN                                                              0x09b9
+#define regSDMA1_SDMA_QUEUE7_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE7_PREEMPT                                                                    0x09ba
+#define regSDMA1_SDMA_QUEUE7_PREEMPT_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE7_DUMMY_REG                                                                  0x09bb
+#define regSDMA1_SDMA_QUEUE7_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_LO                                                       0x09bc
+#define regSDMA1_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_HI                                                       0x09bd
+#define regSDMA1_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE7_RB_AQL_CNTL                                                                0x09be
+#define regSDMA1_SDMA_QUEUE7_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE7_MINOR_PTR_UPDATE                                                           0x09bf
+#define regSDMA1_SDMA_QUEUE7_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS                                                      0x09c2
+#define regSDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_CNTL                                                                0x09c3
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA0                                                               0x09c4
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA1                                                               0x09c5
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA2                                                               0x09c6
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA3                                                               0x09c7
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA4                                                               0x09c8
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA5                                                               0x09c9
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA6                                                               0x09ca
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA7                                                               0x09cb
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA8                                                               0x09cc
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA9                                                               0x09cd
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA10                                                              0x09ce
+#define regSDMA1_SDMA_QUEUE7_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE7_UTILIZATION_LO                                                             0x09cf
+#define regSDMA1_SDMA_QUEUE7_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE7_UTILIZATION_HI                                                             0x09d0
+#define regSDMA1_SDMA_QUEUE7_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE7_WAIT_UNSATISFIED_THD                                                       0x09d1
+#define regSDMA1_SDMA_QUEUE7_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE7_MQD_BASE_ADDR_LO                                                           0x09d2
+#define regSDMA1_SDMA_QUEUE7_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE7_MQD_BASE_ADDR_HI                                                           0x09d3
+#define regSDMA1_SDMA_QUEUE7_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE7_MQD_CONTROL                                                                0x09d4
+#define regSDMA1_SDMA_QUEUE7_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE7_CONTEXT_STATUS                                                             0x09d5
+#define regSDMA1_SDMA_QUEUE7_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE7_DUMMY_REG1                                                                 0x09d6
+#define regSDMA1_SDMA_QUEUE7_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE7_DUMMY_REG2                                                                 0x09d7
+#define regSDMA1_SDMA_QUEUE7_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE8_RB_CNTL                                                                    0x09e0
+#define regSDMA1_SDMA_QUEUE8_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE8_RB_BASE                                                                    0x09e1
+#define regSDMA1_SDMA_QUEUE8_RB_BASE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE8_RB_BASE_HI                                                                 0x09e2
+#define regSDMA1_SDMA_QUEUE8_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE8_RB_RPTR                                                                    0x09e3
+#define regSDMA1_SDMA_QUEUE8_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE8_RB_RPTR_HI                                                                 0x09e4
+#define regSDMA1_SDMA_QUEUE8_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE8_RB_WPTR                                                                    0x09e5
+#define regSDMA1_SDMA_QUEUE8_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE8_RB_WPTR_HI                                                                 0x09e6
+#define regSDMA1_SDMA_QUEUE8_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE8_RB_RPTR_ADDR_LO                                                            0x09e7
+#define regSDMA1_SDMA_QUEUE8_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE8_RB_RPTR_ADDR_HI                                                            0x09e8
+#define regSDMA1_SDMA_QUEUE8_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE8_IB_CNTL                                                                    0x09e9
+#define regSDMA1_SDMA_QUEUE8_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE8_IB_RPTR                                                                    0x09ea
+#define regSDMA1_SDMA_QUEUE8_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE8_IB_OFFSET                                                                  0x09eb
+#define regSDMA1_SDMA_QUEUE8_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE8_IB_BASE_LO                                                                 0x09ec
+#define regSDMA1_SDMA_QUEUE8_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE8_IB_BASE_HI                                                                 0x09ed
+#define regSDMA1_SDMA_QUEUE8_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE8_IB_SIZE                                                                    0x09ee
+#define regSDMA1_SDMA_QUEUE8_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE8_DOORBELL                                                                   0x09ef
+#define regSDMA1_SDMA_QUEUE8_DOORBELL_BASE_IDX                                                          0
+#define regSDMA1_SDMA_QUEUE8_DOORBELL_LOG                                                               0x09f0
+#define regSDMA1_SDMA_QUEUE8_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE8_DOORBELL_OFFSET                                                            0x09f1
+#define regSDMA1_SDMA_QUEUE8_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE8_CSA_ADDR_LO                                                                0x09f2
+#define regSDMA1_SDMA_QUEUE8_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE8_CSA_ADDR_HI                                                                0x09f3
+#define regSDMA1_SDMA_QUEUE8_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE8_SCHEDULE_CNTL                                                              0x09f4
+#define regSDMA1_SDMA_QUEUE8_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE8_IB_SUB_REMAIN                                                              0x09f5
+#define regSDMA1_SDMA_QUEUE8_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE8_PREEMPT                                                                    0x09f6
+#define regSDMA1_SDMA_QUEUE8_PREEMPT_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE8_DUMMY_REG                                                                  0x09f7
+#define regSDMA1_SDMA_QUEUE8_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_LO                                                       0x09f8
+#define regSDMA1_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_HI                                                       0x09f9
+#define regSDMA1_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE8_RB_AQL_CNTL                                                                0x09fa
+#define regSDMA1_SDMA_QUEUE8_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE8_MINOR_PTR_UPDATE                                                           0x09fb
+#define regSDMA1_SDMA_QUEUE8_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS                                                      0x09fe
+#define regSDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_CNTL                                                                0x09ff
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA0                                                               0x0a00
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA1                                                               0x0a01
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA2                                                               0x0a02
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA3                                                               0x0a03
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA4                                                               0x0a04
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA5                                                               0x0a05
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA6                                                               0x0a06
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA7                                                               0x0a07
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA8                                                               0x0a08
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA9                                                               0x0a09
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA10                                                              0x0a0a
+#define regSDMA1_SDMA_QUEUE8_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE8_UTILIZATION_LO                                                             0x0a0b
+#define regSDMA1_SDMA_QUEUE8_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE8_UTILIZATION_HI                                                             0x0a0c
+#define regSDMA1_SDMA_QUEUE8_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE8_WAIT_UNSATISFIED_THD                                                       0x0a0d
+#define regSDMA1_SDMA_QUEUE8_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE8_MQD_BASE_ADDR_LO                                                           0x0a0e
+#define regSDMA1_SDMA_QUEUE8_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE8_MQD_BASE_ADDR_HI                                                           0x0a0f
+#define regSDMA1_SDMA_QUEUE8_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE8_MQD_CONTROL                                                                0x0a10
+#define regSDMA1_SDMA_QUEUE8_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE8_CONTEXT_STATUS                                                             0x0a11
+#define regSDMA1_SDMA_QUEUE8_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE8_DUMMY_REG1                                                                 0x0a12
+#define regSDMA1_SDMA_QUEUE8_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE8_DUMMY_REG2                                                                 0x0a13
+#define regSDMA1_SDMA_QUEUE8_DUMMY_REG2_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE9_RB_CNTL                                                                    0x0a1c
+#define regSDMA1_SDMA_QUEUE9_RB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE9_RB_BASE                                                                    0x0a1d
+#define regSDMA1_SDMA_QUEUE9_RB_BASE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE9_RB_BASE_HI                                                                 0x0a1e
+#define regSDMA1_SDMA_QUEUE9_RB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE9_RB_RPTR                                                                    0x0a1f
+#define regSDMA1_SDMA_QUEUE9_RB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE9_RB_RPTR_HI                                                                 0x0a20
+#define regSDMA1_SDMA_QUEUE9_RB_RPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE9_RB_WPTR                                                                    0x0a21
+#define regSDMA1_SDMA_QUEUE9_RB_WPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE9_RB_WPTR_HI                                                                 0x0a22
+#define regSDMA1_SDMA_QUEUE9_RB_WPTR_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE9_RB_RPTR_ADDR_LO                                                            0x0a23
+#define regSDMA1_SDMA_QUEUE9_RB_RPTR_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE9_RB_RPTR_ADDR_HI                                                            0x0a24
+#define regSDMA1_SDMA_QUEUE9_RB_RPTR_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE9_IB_CNTL                                                                    0x0a25
+#define regSDMA1_SDMA_QUEUE9_IB_CNTL_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE9_IB_RPTR                                                                    0x0a26
+#define regSDMA1_SDMA_QUEUE9_IB_RPTR_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE9_IB_OFFSET                                                                  0x0a27
+#define regSDMA1_SDMA_QUEUE9_IB_OFFSET_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE9_IB_BASE_LO                                                                 0x0a28
+#define regSDMA1_SDMA_QUEUE9_IB_BASE_LO_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE9_IB_BASE_HI                                                                 0x0a29
+#define regSDMA1_SDMA_QUEUE9_IB_BASE_HI_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE9_IB_SIZE                                                                    0x0a2a
+#define regSDMA1_SDMA_QUEUE9_IB_SIZE_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE9_DOORBELL                                                                   0x0a2b
+#define regSDMA1_SDMA_QUEUE9_DOORBELL_BASE_IDX                                                          0
+#define regSDMA1_SDMA_QUEUE9_DOORBELL_LOG                                                               0x0a2c
+#define regSDMA1_SDMA_QUEUE9_DOORBELL_LOG_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE9_DOORBELL_OFFSET                                                            0x0a2d
+#define regSDMA1_SDMA_QUEUE9_DOORBELL_OFFSET_BASE_IDX                                                   0
+#define regSDMA1_SDMA_QUEUE9_CSA_ADDR_LO                                                                0x0a2e
+#define regSDMA1_SDMA_QUEUE9_CSA_ADDR_LO_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE9_CSA_ADDR_HI                                                                0x0a2f
+#define regSDMA1_SDMA_QUEUE9_CSA_ADDR_HI_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE9_SCHEDULE_CNTL                                                              0x0a30
+#define regSDMA1_SDMA_QUEUE9_SCHEDULE_CNTL_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE9_IB_SUB_REMAIN                                                              0x0a31
+#define regSDMA1_SDMA_QUEUE9_IB_SUB_REMAIN_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE9_PREEMPT                                                                    0x0a32
+#define regSDMA1_SDMA_QUEUE9_PREEMPT_BASE_IDX                                                           0
+#define regSDMA1_SDMA_QUEUE9_DUMMY_REG                                                                  0x0a33
+#define regSDMA1_SDMA_QUEUE9_DUMMY_REG_BASE_IDX                                                         0
+#define regSDMA1_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_LO                                                       0x0a34
+#define regSDMA1_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_HI                                                       0x0a35
+#define regSDMA1_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE9_RB_AQL_CNTL                                                                0x0a36
+#define regSDMA1_SDMA_QUEUE9_RB_AQL_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE9_MINOR_PTR_UPDATE                                                           0x0a37
+#define regSDMA1_SDMA_QUEUE9_MINOR_PTR_UPDATE_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS                                                      0x0a3a
+#define regSDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS_BASE_IDX                                             0
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_CNTL                                                                0x0a3b
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_CNTL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA0                                                               0x0a3c
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA0_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA1                                                               0x0a3d
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA1_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA2                                                               0x0a3e
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA2_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA3                                                               0x0a3f
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA3_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA4                                                               0x0a40
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA4_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA5                                                               0x0a41
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA5_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA6                                                               0x0a42
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA6_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA7                                                               0x0a43
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA7_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA8                                                               0x0a44
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA8_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA9                                                               0x0a45
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA9_BASE_IDX                                                      0
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA10                                                              0x0a46
+#define regSDMA1_SDMA_QUEUE9_MIDCMD_DATA10_BASE_IDX                                                     0
+#define regSDMA1_SDMA_QUEUE9_UTILIZATION_LO                                                             0x0a47
+#define regSDMA1_SDMA_QUEUE9_UTILIZATION_LO_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE9_UTILIZATION_HI                                                             0x0a48
+#define regSDMA1_SDMA_QUEUE9_UTILIZATION_HI_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE9_WAIT_UNSATISFIED_THD                                                       0x0a49
+#define regSDMA1_SDMA_QUEUE9_WAIT_UNSATISFIED_THD_BASE_IDX                                              0
+#define regSDMA1_SDMA_QUEUE9_MQD_BASE_ADDR_LO                                                           0x0a4a
+#define regSDMA1_SDMA_QUEUE9_MQD_BASE_ADDR_LO_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE9_MQD_BASE_ADDR_HI                                                           0x0a4b
+#define regSDMA1_SDMA_QUEUE9_MQD_BASE_ADDR_HI_BASE_IDX                                                  0
+#define regSDMA1_SDMA_QUEUE9_MQD_CONTROL                                                                0x0a4c
+#define regSDMA1_SDMA_QUEUE9_MQD_CONTROL_BASE_IDX                                                       0
+#define regSDMA1_SDMA_QUEUE9_CONTEXT_STATUS                                                             0x0a4d
+#define regSDMA1_SDMA_QUEUE9_CONTEXT_STATUS_BASE_IDX                                                    0
+#define regSDMA1_SDMA_QUEUE9_DUMMY_REG1                                                                 0x0a4e
+#define regSDMA1_SDMA_QUEUE9_DUMMY_REG1_BASE_IDX                                                        0
+#define regSDMA1_SDMA_QUEUE9_DUMMY_REG2                                                                 0x0a4f
+#define regSDMA1_SDMA_QUEUE9_DUMMY_REG2_BASE_IDX                                                        0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmahypdec:1
+// base address: 0x3e2c0
+#define regSDMA1_SDMA_VM_CTX_LO                                                                         0x58b0
+#define regSDMA1_SDMA_VM_CTX_LO_BASE_IDX                                                                1
+#define regSDMA1_SDMA_VM_CTX_HI                                                                         0x58b1
+#define regSDMA1_SDMA_VM_CTX_HI_BASE_IDX                                                                1
+#define regSDMA1_SDMA_ACTIVE_FCN_ID                                                                     0x58b2
+#define regSDMA1_SDMA_ACTIVE_FCN_ID_BASE_IDX                                                            1
+#define regSDMA1_SDMA_VM_CTX_CNTL                                                                       0x58b3
+#define regSDMA1_SDMA_VM_CTX_CNTL_BASE_IDX                                                              1
+#define regSDMA1_SDMA_VIRT_RESET_REQ                                                                    0x58b4
+#define regSDMA1_SDMA_VIRT_RESET_REQ_BASE_IDX                                                           1
+#define regSDMA1_SDMA_CONTEXT_REG_TYPE0                                                                 0x58b5
+#define regSDMA1_SDMA_CONTEXT_REG_TYPE0_BASE_IDX                                                        1
+#define regSDMA1_SDMA_CONTEXT_REG_TYPE1                                                                 0x58b6
+#define regSDMA1_SDMA_CONTEXT_REG_TYPE1_BASE_IDX                                                        1
+#define regSDMA1_SDMA_CONTEXT_REG_TYPE2                                                                 0x58b7
+#define regSDMA1_SDMA_CONTEXT_REG_TYPE2_BASE_IDX                                                        1
+#define regSDMA1_SDMA_PUB_REG_TYPE0                                                                     0x58b8
+#define regSDMA1_SDMA_PUB_REG_TYPE0_BASE_IDX                                                            1
+#define regSDMA1_SDMA_PUB_REG_TYPE1                                                                     0x58b9
+#define regSDMA1_SDMA_PUB_REG_TYPE1_BASE_IDX                                                            1
+#define regSDMA1_SDMA_PUB_REG_TYPE2                                                                     0x58ba
+#define regSDMA1_SDMA_PUB_REG_TYPE2_BASE_IDX                                                            1
+#define regSDMA1_SDMA_PUB_REG_TYPE3                                                                     0x58bb
+#define regSDMA1_SDMA_PUB_REG_TYPE3_BASE_IDX                                                            1
+#define regSDMA1_SDMA_VM_CNTL                                                                           0x58bd
+#define regSDMA1_SDMA_VM_CNTL_BASE_IDX                                                                  1
+#define regSDMA1_SDMA_MCU_CNTL                                                                          0x58be
+#define regSDMA1_SDMA_MCU_CNTL_BASE_IDX                                                                 1
+#define regSDMA1_SDMA_IC_BASE_LO                                                                        0x58ca
+#define regSDMA1_SDMA_IC_BASE_LO_BASE_IDX                                                               1
+#define regSDMA1_SDMA_IC_BASE_HI                                                                        0x58cb
+#define regSDMA1_SDMA_IC_BASE_HI_BASE_IDX                                                               1
+#define regSDMA1_SDMA_IC_BASE_CNTL                                                                      0x58cc
+#define regSDMA1_SDMA_IC_BASE_CNTL_BASE_IDX                                                             1
+#define regSDMA1_SDMA_IC_OP_CNTL                                                                        0x58cd
+#define regSDMA1_SDMA_IC_OP_CNTL_BASE_IDX                                                               1
+#define regSDMA1_SDMA_IC_CNTL                                                                           0x58cf
+#define regSDMA1_SDMA_IC_CNTL_BASE_IDX                                                                  1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmapspdec:1
+// base address: 0x3f340
+#define regSDMA1_SDMA_MCU_DM_FROM_RST_ADDR_OFFSET                                                       0x5d14
+#define regSDMA1_SDMA_MCU_DM_FROM_RST_ADDR_OFFSET_BASE_IDX                                              1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmaperfsdec:1
+// base address: 0x378e0
+#define regSDMA1_SDMA_PERFCNT_PERFCOUNTER0_CFG                                                          0x3e38
+#define regSDMA1_SDMA_PERFCNT_PERFCOUNTER0_CFG_BASE_IDX                                                 1
+#define regSDMA1_SDMA_PERFCNT_PERFCOUNTER1_CFG                                                          0x3e39
+#define regSDMA1_SDMA_PERFCNT_PERFCOUNTER1_CFG_BASE_IDX                                                 1
+#define regSDMA1_SDMA_PERFCNT_PERFCOUNTER2_CFG                                                          0x3e3a
+#define regSDMA1_SDMA_PERFCNT_PERFCOUNTER2_CFG_BASE_IDX                                                 1
+#define regSDMA1_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL                                                     0x3e3b
+#define regSDMA1_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                            1
+#define regSDMA1_SDMA_PERFCNT_MISC_CNTL                                                                 0x3e3c
+#define regSDMA1_SDMA_PERFCNT_MISC_CNTL_BASE_IDX                                                        1
+#define regSDMA1_SDMA_PERFCOUNTER0_SELECT                                                               0x3e3d
+#define regSDMA1_SDMA_PERFCOUNTER0_SELECT_BASE_IDX                                                      1
+#define regSDMA1_SDMA_PERFCOUNTER0_SELECT1                                                              0x3e3e
+#define regSDMA1_SDMA_PERFCOUNTER0_SELECT1_BASE_IDX                                                     1
+#define regSDMA1_SDMA_PERFCOUNTER1_SELECT                                                               0x3e3f
+#define regSDMA1_SDMA_PERFCOUNTER1_SELECT_BASE_IDX                                                      1
+#define regSDMA1_SDMA_PERFCOUNTER1_SELECT1                                                              0x3e40
+#define regSDMA1_SDMA_PERFCOUNTER1_SELECT1_BASE_IDX                                                     1
+#define regSDMA1_SDMA_PERFCOUNTER2_SELECT                                                               0x3e41
+#define regSDMA1_SDMA_PERFCOUNTER2_SELECT_BASE_IDX                                                      1
+#define regSDMA1_SDMA_PERFCOUNTER2_SELECT1                                                              0x3e42
+#define regSDMA1_SDMA_PERFCOUNTER2_SELECT1_BASE_IDX                                                     1
+#define regSDMA1_SDMA_PERFCOUNTER3_SELECT                                                               0x3e43
+#define regSDMA1_SDMA_PERFCOUNTER3_SELECT_BASE_IDX                                                      1
+#define regSDMA1_SDMA_PERFCOUNTER3_SELECT1                                                              0x3e44
+#define regSDMA1_SDMA_PERFCOUNTER3_SELECT1_BASE_IDX                                                     1
+#define regSDMA1_SDMA_PERFCOUNTER4_SELECT                                                               0x3e45
+#define regSDMA1_SDMA_PERFCOUNTER4_SELECT_BASE_IDX                                                      1
+#define regSDMA1_SDMA_PERFCOUNTER4_SELECT1                                                              0x3e46
+#define regSDMA1_SDMA_PERFCOUNTER4_SELECT1_BASE_IDX                                                     1
+#define regSDMA1_SDMA_PERFCOUNTER5_SELECT                                                               0x3e47
+#define regSDMA1_SDMA_PERFCOUNTER5_SELECT_BASE_IDX                                                      1
+#define regSDMA1_SDMA_PERFCOUNTER5_SELECT1                                                              0x3e48
+#define regSDMA1_SDMA_PERFCOUNTER5_SELECT1_BASE_IDX                                                     1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmaperfddec:1
+// base address: 0x359e0
+#define regSDMA1_SDMA_PERFCNT_PERFCOUNTER_LO                                                            0x3678
+#define regSDMA1_SDMA_PERFCNT_PERFCOUNTER_LO_BASE_IDX                                                   1
+#define regSDMA1_SDMA_PERFCNT_PERFCOUNTER_HI                                                            0x3679
+#define regSDMA1_SDMA_PERFCNT_PERFCOUNTER_HI_BASE_IDX                                                   1
+#define regSDMA1_SDMA_PERFCOUNTER0_LO                                                                   0x367a
+#define regSDMA1_SDMA_PERFCOUNTER0_LO_BASE_IDX                                                          1
+#define regSDMA1_SDMA_PERFCOUNTER0_HI                                                                   0x367b
+#define regSDMA1_SDMA_PERFCOUNTER0_HI_BASE_IDX                                                          1
+#define regSDMA1_SDMA_PERFCOUNTER1_LO                                                                   0x367c
+#define regSDMA1_SDMA_PERFCOUNTER1_LO_BASE_IDX                                                          1
+#define regSDMA1_SDMA_PERFCOUNTER1_HI                                                                   0x367d
+#define regSDMA1_SDMA_PERFCOUNTER1_HI_BASE_IDX                                                          1
+#define regSDMA1_SDMA_PERFCOUNTER2_LO                                                                   0x367e
+#define regSDMA1_SDMA_PERFCOUNTER2_LO_BASE_IDX                                                          1
+#define regSDMA1_SDMA_PERFCOUNTER2_HI                                                                   0x367f
+#define regSDMA1_SDMA_PERFCOUNTER2_HI_BASE_IDX                                                          1
+#define regSDMA1_SDMA_PERFCOUNTER3_LO                                                                   0x3680
+#define regSDMA1_SDMA_PERFCOUNTER3_LO_BASE_IDX                                                          1
+#define regSDMA1_SDMA_PERFCOUNTER3_HI                                                                   0x3681
+#define regSDMA1_SDMA_PERFCOUNTER3_HI_BASE_IDX                                                          1
+#define regSDMA1_SDMA_PERFCOUNTER4_LO                                                                   0x3682
+#define regSDMA1_SDMA_PERFCOUNTER4_LO_BASE_IDX                                                          1
+#define regSDMA1_SDMA_PERFCOUNTER4_HI                                                                   0x3683
+#define regSDMA1_SDMA_PERFCOUNTER4_HI_BASE_IDX                                                          1
+#define regSDMA1_SDMA_PERFCOUNTER5_LO                                                                   0x3684
+#define regSDMA1_SDMA_PERFCOUNTER5_LO_BASE_IDX                                                          1
+#define regSDMA1_SDMA_PERFCOUNTER5_HI                                                                   0x3685
+#define regSDMA1_SDMA_PERFCOUNTER5_HI_BASE_IDX                                                          1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmapwrdec:1
+// base address: 0x3c434
+#define regSDMA1_GFX_ICG_SDMA_CTRL                                                                      0x510d
+#define regSDMA1_GFX_ICG_SDMA_CTRL_BASE_IDX                                                             1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_grbmdec
+// base address: 0x8000
+#define regGRBM_CNTL                                                                                    0x0da0
+#define regGRBM_CNTL_BASE_IDX                                                                           0
+#define regGRBM_SKEW_CNTL                                                                               0x0da1
+#define regGRBM_SKEW_CNTL_BASE_IDX                                                                      0
+#define regGRBM_STATUS2                                                                                 0x0da2
+#define regGRBM_STATUS2_BASE_IDX                                                                        0
+#define regGRBM_PWR_CNTL                                                                                0x0da3
+#define regGRBM_PWR_CNTL_BASE_IDX                                                                       0
+#define regGRBM_STATUS                                                                                  0x0da4
+#define regGRBM_STATUS_BASE_IDX                                                                         0
+#define regGRBM_STATUS_SE0                                                                              0x0da5
+#define regGRBM_STATUS_SE0_BASE_IDX                                                                     0
+#define regGRBM_STATUS_SE1                                                                              0x0da6
+#define regGRBM_STATUS_SE1_BASE_IDX                                                                     0
+#define regGRBM_STATUS3                                                                                 0x0da7
+#define regGRBM_STATUS3_BASE_IDX                                                                        0
+#define regGRBM_SOFT_RESET                                                                              0x0da8
+#define regGRBM_SOFT_RESET_BASE_IDX                                                                     0
+#define regGRBM_FAO_CNT                                                                                 0x0dab
+#define regGRBM_FAO_CNT_BASE_IDX                                                                        0
+#define regGRBM_GFX_CLKEN_CNTL                                                                          0x0dac
+#define regGRBM_GFX_CLKEN_CNTL_BASE_IDX                                                                 0
+#define regGRBM_WAIT_IDLE_CLOCKS                                                                        0x0dad
+#define regGRBM_WAIT_IDLE_CLOCKS_BASE_IDX                                                               0
+#define regGRBM_READ_ERROR                                                                              0x0db6
+#define regGRBM_READ_ERROR_BASE_IDX                                                                     0
+#define regGRBM_READ_ERROR2                                                                             0x0db7
+#define regGRBM_READ_ERROR2_BASE_IDX                                                                    0
+#define regGRBM_INT_CNTL                                                                                0x0db8
+#define regGRBM_INT_CNTL_BASE_IDX                                                                       0
+#define regGRBM_TRAP_OP                                                                                 0x0db9
+#define regGRBM_TRAP_OP_BASE_IDX                                                                        0
+#define regGRBM_TRAP_ADDR                                                                               0x0dba
+#define regGRBM_TRAP_ADDR_BASE_IDX                                                                      0
+#define regGRBM_TRAP_ADDR_MSK                                                                           0x0dbb
+#define regGRBM_TRAP_ADDR_MSK_BASE_IDX                                                                  0
+#define regGRBM_TRAP_WD                                                                                 0x0dbc
+#define regGRBM_TRAP_WD_BASE_IDX                                                                        0
+#define regGRBM_TRAP_WD_MSK                                                                             0x0dbd
+#define regGRBM_TRAP_WD_MSK_BASE_IDX                                                                    0
+#define regGRBM_DSM_BYPASS                                                                              0x0dbe
+#define regGRBM_DSM_BYPASS_BASE_IDX                                                                     0
+#define regGRBM_WRITE_ERROR                                                                             0x0dbf
+#define regGRBM_WRITE_ERROR_BASE_IDX                                                                    0
+#define regGRBM_IOV_ERROR_FIFO                                                                          0x0dc0
+#define regGRBM_IOV_ERROR_FIFO_BASE_IDX                                                                 0
+#define regGRBM_CHIP_REVISION                                                                           0x0dc1
+#define regGRBM_CHIP_REVISION_BASE_IDX                                                                  0
+#define regGRBM_RSMU_CFG                                                                                0x0dc3
+#define regGRBM_RSMU_CFG_BASE_IDX                                                                       0
+#define regGRBM_IH_CREDIT                                                                               0x0dc4
+#define regGRBM_IH_CREDIT_BASE_IDX                                                                      0
+#define regGRBM_PWR_CNTL2                                                                               0x0dc5
+#define regGRBM_PWR_CNTL2_BASE_IDX                                                                      0
+#define regGRBM_UTCL2_INVAL_RANGE_START                                                                 0x0dc6
+#define regGRBM_UTCL2_INVAL_RANGE_START_BASE_IDX                                                        0
+#define regGRBM_UTCL2_INVAL_RANGE_END                                                                   0x0dc7
+#define regGRBM_UTCL2_INVAL_RANGE_END_BASE_IDX                                                          0
+#define regGRBM_RSMU_READ_ERROR                                                                         0x0dc8
+#define regGRBM_RSMU_READ_ERROR_BASE_IDX                                                                0
+#define regGRBM_INVALID_PIPE                                                                            0x0dc9
+#define regGRBM_INVALID_PIPE_BASE_IDX                                                                   0
+#define regGRBM_FENCE_RANGE0                                                                            0x0dca
+#define regGRBM_FENCE_RANGE0_BASE_IDX                                                                   0
+#define regGRBM_FENCE_RANGE1                                                                            0x0dcb
+#define regGRBM_FENCE_RANGE1_BASE_IDX                                                                   0
+#define regGRBM_CHICKEN_BITS0                                                                           0x0dcc
+#define regGRBM_CHICKEN_BITS0_BASE_IDX                                                                  0
+#define regGRBM_CHICKEN_BITS1                                                                           0x0dcd
+#define regGRBM_CHICKEN_BITS1_BASE_IDX                                                                  0
+#define regCC_GC_FULL_SA_UNIT_DISABLE                                                                   0x0dd9
+#define regCC_GC_FULL_SA_UNIT_DISABLE_BASE_IDX                                                          0
+#define regGRBM_SCRATCH_REG0                                                                            0x0de0
+#define regGRBM_SCRATCH_REG0_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG1                                                                            0x0de1
+#define regGRBM_SCRATCH_REG1_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG2                                                                            0x0de2
+#define regGRBM_SCRATCH_REG2_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG3                                                                            0x0de3
+#define regGRBM_SCRATCH_REG3_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG4                                                                            0x0de4
+#define regGRBM_SCRATCH_REG4_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG5                                                                            0x0de5
+#define regGRBM_SCRATCH_REG5_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG6                                                                            0x0de6
+#define regGRBM_SCRATCH_REG6_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG7                                                                            0x0de7
+#define regGRBM_SCRATCH_REG7_BASE_IDX                                                                   0
+#define regVIOLATION_DATA_ASYNC_VF_PROG                                                                 0x0df1
+#define regVIOLATION_DATA_ASYNC_VF_PROG_BASE_IDX                                                        0
+#define regGRBM_INTF_CNTL                                                                               0x0df6
+#define regGRBM_INTF_CNTL_BASE_IDX                                                                      0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_cpdec
+// base address: 0x8200
+#define regCP_CPC_DEBUG_CNTL                                                                            0x0e20
+#define regCP_CPC_DEBUG_CNTL_BASE_IDX                                                                   0
+#define regCP_CPC_DEBUG_DATA                                                                            0x0e21
+#define regCP_CPC_DEBUG_DATA_BASE_IDX                                                                   0
+#define regCP_CPF_DEBUG_CNTL                                                                            0x0e22
+#define regCP_CPF_DEBUG_CNTL_BASE_IDX                                                                   0
+#define regCP_CPF_DEBUG_DATA                                                                            0x0e23
+#define regCP_CPF_DEBUG_DATA_BASE_IDX                                                                   0
+#define regCP_CPC_STATUS                                                                                0x0e24
+#define regCP_CPC_STATUS_BASE_IDX                                                                       0
+#define regCP_CPC_BUSY_STAT                                                                             0x0e25
+#define regCP_CPC_BUSY_STAT_BASE_IDX                                                                    0
+#define regCP_CPC_STALLED_STAT1                                                                         0x0e26
+#define regCP_CPC_STALLED_STAT1_BASE_IDX                                                                0
+#define regCP_CPF_STATUS                                                                                0x0e27
+#define regCP_CPF_STATUS_BASE_IDX                                                                       0
+#define regCP_CPF_BUSY_STAT                                                                             0x0e28
+#define regCP_CPF_BUSY_STAT_BASE_IDX                                                                    0
+#define regCP_CPF_STALLED_STAT1                                                                         0x0e29
+#define regCP_CPF_STALLED_STAT1_BASE_IDX                                                                0
+#define regCP_CPC_BUSY_STAT2                                                                            0x0e2a
+#define regCP_CPC_BUSY_STAT2_BASE_IDX                                                                   0
+#define regCP_CPC_GRBM_FREE_COUNT                                                                       0x0e2b
+#define regCP_CPC_GRBM_FREE_COUNT_BASE_IDX                                                              0
+#define regCP_CPC_PRIV_VIOLATION_ADDR                                                                   0x0e2c
+#define regCP_CPC_PRIV_VIOLATION_ADDR_BASE_IDX                                                          0
+#define regCP_CPC_PRIV_VIOLATION_ADDR_HI                                                                0x0e2d
+#define regCP_CPC_PRIV_VIOLATION_ADDR_HI_BASE_IDX                                                       0
+#define regCP_MEC_ME1_HEADER_DUMP                                                                       0x0e2e
+#define regCP_MEC_ME1_HEADER_DUMP_BASE_IDX                                                              0
+#define regCP_CPC_SCRATCH_INDEX                                                                         0x0e30
+#define regCP_CPC_SCRATCH_INDEX_BASE_IDX                                                                0
+#define regCP_CPC_SCRATCH_DATA                                                                          0x0e31
+#define regCP_CPC_SCRATCH_DATA_BASE_IDX                                                                 0
+#define regCP_CPF_GRBM_FREE_COUNT                                                                       0x0e32
+#define regCP_CPF_GRBM_FREE_COUNT_BASE_IDX                                                              0
+#define regCP_CPF_BUSY_STAT2                                                                            0x0e33
+#define regCP_CPF_BUSY_STAT2_BASE_IDX                                                                   0
+#define regCP_CPC_HALT_HYST_COUNT                                                                       0x0e47
+#define regCP_CPC_HALT_HYST_COUNT_BASE_IDX                                                              0
+#define regCP_STALLED_STAT3                                                                             0x0f3c
+#define regCP_STALLED_STAT3_BASE_IDX                                                                    0
+#define regCP_STALLED_STAT1                                                                             0x0f3d
+#define regCP_STALLED_STAT1_BASE_IDX                                                                    0
+#define regCP_STALLED_STAT2                                                                             0x0f3e
+#define regCP_STALLED_STAT2_BASE_IDX                                                                    0
+#define regCP_BUSY_STAT                                                                                 0x0f3f
+#define regCP_BUSY_STAT_BASE_IDX                                                                        0
+#define regCP_STAT                                                                                      0x0f40
+#define regCP_STAT_BASE_IDX                                                                             0
+#define regCP_ME_HEADER_DUMP                                                                            0x0f41
+#define regCP_ME_HEADER_DUMP_BASE_IDX                                                                   0
+#define regCP_PFP_HEADER_DUMP                                                                           0x0f42
+#define regCP_PFP_HEADER_DUMP_BASE_IDX                                                                  0
+#define regCP_GRBM_FREE_COUNT                                                                           0x0f43
+#define regCP_GRBM_FREE_COUNT_BASE_IDX                                                                  0
+#define regCP_PFP_INSTR_PNTR                                                                            0x0f45
+#define regCP_PFP_INSTR_PNTR_BASE_IDX                                                                   0
+#define regCP_ME_INSTR_PNTR                                                                             0x0f46
+#define regCP_ME_INSTR_PNTR_BASE_IDX                                                                    0
+#define regCP_CSF_STAT                                                                                  0x0f54
+#define regCP_CSF_STAT_BASE_IDX                                                                         0
+#define regCP_CNTX_STAT                                                                                 0x0f58
+#define regCP_CNTX_STAT_BASE_IDX                                                                        0
+#define regCP_ME_PREEMPTION                                                                             0x0f59
+#define regCP_ME_PREEMPTION_BASE_IDX                                                                    0
+#define regCP_RB0_RPTR                                                                                  0x0f60
+#define regCP_RB0_RPTR_BASE_IDX                                                                         0
+#define regCP_RB_RPTR                                                                                   0x0f60
+#define regCP_RB_RPTR_BASE_IDX                                                                          0
+#define regCP_RB_WPTR_DELAY                                                                             0x0f61
+#define regCP_RB_WPTR_DELAY_BASE_IDX                                                                    0
+#define regCP_RB_WPTR_POLL_CNTL                                                                         0x0f62
+#define regCP_RB_WPTR_POLL_CNTL_BASE_IDX                                                                0
+#define regCP_ROQ1_THRESHOLDS                                                                           0x0f75
+#define regCP_ROQ1_THRESHOLDS_BASE_IDX                                                                  0
+#define regCP_ROQ2_THRESHOLDS                                                                           0x0f76
+#define regCP_ROQ2_THRESHOLDS_BASE_IDX                                                                  0
+#define regCP_STQ_THRESHOLDS                                                                            0x0f77
+#define regCP_STQ_THRESHOLDS_BASE_IDX                                                                   0
+#define regCP_MEQ_THRESHOLDS                                                                            0x0f79
+#define regCP_MEQ_THRESHOLDS_BASE_IDX                                                                   0
+#define regCP_ROQ_AVAIL                                                                                 0x0f7a
+#define regCP_ROQ_AVAIL_BASE_IDX                                                                        0
+#define regCP_STQ_AVAIL                                                                                 0x0f7b
+#define regCP_STQ_AVAIL_BASE_IDX                                                                        0
+#define regCP_ROQ2_AVAIL                                                                                0x0f7c
+#define regCP_ROQ2_AVAIL_BASE_IDX                                                                       0
+#define regCP_MEQ_AVAIL                                                                                 0x0f7d
+#define regCP_MEQ_AVAIL_BASE_IDX                                                                        0
+#define regCP_CMD_INDEX                                                                                 0x0f7e
+#define regCP_CMD_INDEX_BASE_IDX                                                                        0
+#define regCP_CMD_DATA                                                                                  0x0f7f
+#define regCP_CMD_DATA_BASE_IDX                                                                         0
+#define regCP_ROQ_RB_STAT                                                                               0x0f80
+#define regCP_ROQ_RB_STAT_BASE_IDX                                                                      0
+#define regCP_ROQ_IB1_STAT                                                                              0x0f81
+#define regCP_ROQ_IB1_STAT_BASE_IDX                                                                     0
+#define regCP_ROQ_IB2_STAT                                                                              0x0f82
+#define regCP_ROQ_IB2_STAT_BASE_IDX                                                                     0
+#define regCP_STQ_STAT                                                                                  0x0f83
+#define regCP_STQ_STAT_BASE_IDX                                                                         0
+#define regCP_STQ_WR_STAT                                                                               0x0f84
+#define regCP_STQ_WR_STAT_BASE_IDX                                                                      0
+#define regCP_MEQ_STAT                                                                                  0x0f85
+#define regCP_MEQ_STAT_BASE_IDX                                                                         0
+#define regCP_ROQ3_THRESHOLDS                                                                           0x0f8c
+#define regCP_ROQ3_THRESHOLDS_BASE_IDX                                                                  0
+#define regCP_ROQ_DB_STAT                                                                               0x0f8d
+#define regCP_ROQ_DB_STAT_BASE_IDX                                                                      0
+#define regCP_INT_STAT_DEBUG                                                                            0x0f97
+#define regCP_INT_STAT_DEBUG_BASE_IDX                                                                   0
+#define regCP_DEBUG_CNTL                                                                                0x0f98
+#define regCP_DEBUG_CNTL_BASE_IDX                                                                       0
+#define regCP_DEBUG_DATA                                                                                0x0f99
+#define regCP_DEBUG_DATA_BASE_IDX                                                                       0
+#define regCP_PRIV_VIOLATION_ADDR                                                                       0x0f9a
+#define regCP_PRIV_VIOLATION_ADDR_BASE_IDX                                                              0
+#define regCP_PRIV_VIOLATION_ADDR_HI                                                                    0x0f9b
+#define regCP_PRIV_VIOLATION_ADDR_HI_BASE_IDX                                                           0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_padec
+// base address: 0x8800
+#define regVGT_DMA_DATA_FIFO_DEPTH                                                                      0x0fcd
+#define regVGT_DMA_DATA_FIFO_DEPTH_BASE_IDX                                                             0
+#define regVGT_DMA_REQ_FIFO_DEPTH                                                                       0x0fce
+#define regVGT_DMA_REQ_FIFO_DEPTH_BASE_IDX                                                              0
+#define regVGT_DRAW_INIT_FIFO_DEPTH                                                                     0x0fcf
+#define regVGT_DRAW_INIT_FIFO_DEPTH_BASE_IDX                                                            0
+#define regVGT_MC_LAT_CNTL                                                                              0x0fd6
+#define regVGT_MC_LAT_CNTL_BASE_IDX                                                                     0
+#define regIA_UTCL1_STATUS_2                                                                            0x0fd7
+#define regIA_UTCL1_STATUS_2_BASE_IDX                                                                   0
+#define regGE_WD_CNTL_STATUS                                                                            0x0fdf
+#define regGE_WD_CNTL_STATUS_BASE_IDX                                                                   0
+#define regWD_UTCL1_CNTL                                                                                0x0fe3
+#define regWD_UTCL1_CNTL_BASE_IDX                                                                       0
+#define regWD_UTCL1_STATUS                                                                              0x0fe4
+#define regWD_UTCL1_STATUS_BASE_IDX                                                                     0
+#define regIA_UTCL1_CNTL                                                                                0x0fe6
+#define regIA_UTCL1_CNTL_BASE_IDX                                                                       0
+#define regIA_UTCL1_STATUS                                                                              0x0fe7
+#define regIA_UTCL1_STATUS_BASE_IDX                                                                     0
+#define regCC_GC_SA_UNIT_DISABLE                                                                        0x0fe9
+#define regCC_GC_SA_UNIT_DISABLE_BASE_IDX                                                               0
+#define regGE_FED_STATUS                                                                                0x0fea
+#define regGE_FED_STATUS_BASE_IDX                                                                       0
+#define regGE_PRIV_CONTROL                                                                              0x1004
+#define regGE_PRIV_CONTROL_BASE_IDX                                                                     0
+#define regGE_STATUS                                                                                    0x1005
+#define regGE_STATUS_BASE_IDX                                                                           0
+#define regVGT_GS_MAX_WAVE_ID                                                                           0x1009
+#define regVGT_GS_MAX_WAVE_ID_BASE_IDX                                                                  0
+#define regGFX_PIPE_CONTROL                                                                             0x100d
+#define regGFX_PIPE_CONTROL_BASE_IDX                                                                    0
+#define regVGT_RESET_DEBUG                                                                              0x1014
+#define regVGT_RESET_DEBUG_BASE_IDX                                                                     0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_shdec
+// base address: 0xb000
+#define regCOMPUTE_DISPATCH_INITIATOR                                                                   0x1ba0
+#define regCOMPUTE_DISPATCH_INITIATOR_BASE_IDX                                                          0
+#define regCOMPUTE_DIM_X                                                                                0x1ba1
+#define regCOMPUTE_DIM_X_BASE_IDX                                                                       0
+#define regCOMPUTE_DIM_Y                                                                                0x1ba2
+#define regCOMPUTE_DIM_Y_BASE_IDX                                                                       0
+#define regCOMPUTE_DIM_Z                                                                                0x1ba3
+#define regCOMPUTE_DIM_Z_BASE_IDX                                                                       0
+#define regCOMPUTE_START_X                                                                              0x1ba4
+#define regCOMPUTE_START_X_BASE_IDX                                                                     0
+#define regCOMPUTE_START_Y                                                                              0x1ba5
+#define regCOMPUTE_START_Y_BASE_IDX                                                                     0
+#define regCOMPUTE_START_Z                                                                              0x1ba6
+#define regCOMPUTE_START_Z_BASE_IDX                                                                     0
+#define regCOMPUTE_NUM_THREAD_X                                                                         0x1ba7
+#define regCOMPUTE_NUM_THREAD_X_BASE_IDX                                                                0
+#define regCOMPUTE_NUM_THREAD_Y                                                                         0x1ba8
+#define regCOMPUTE_NUM_THREAD_Y_BASE_IDX                                                                0
+#define regCOMPUTE_NUM_THREAD_Z                                                                         0x1ba9
+#define regCOMPUTE_NUM_THREAD_Z_BASE_IDX                                                                0
+#define regCOMPUTE_PIPELINESTAT_ENABLE                                                                  0x1baa
+#define regCOMPUTE_PIPELINESTAT_ENABLE_BASE_IDX                                                         0
+#define regCOMPUTE_PERFCOUNT_ENABLE                                                                     0x1bab
+#define regCOMPUTE_PERFCOUNT_ENABLE_BASE_IDX                                                            0
+#define regCOMPUTE_PGM_LO                                                                               0x1bac
+#define regCOMPUTE_PGM_LO_BASE_IDX                                                                      0
+#define regCOMPUTE_PGM_HI                                                                               0x1bad
+#define regCOMPUTE_PGM_HI_BASE_IDX                                                                      0
+#define regCOMPUTE_DISPATCH_PKT_ADDR_LO                                                                 0x1bae
+#define regCOMPUTE_DISPATCH_PKT_ADDR_LO_BASE_IDX                                                        0
+#define regCOMPUTE_DISPATCH_PKT_ADDR_HI                                                                 0x1baf
+#define regCOMPUTE_DISPATCH_PKT_ADDR_HI_BASE_IDX                                                        0
+#define regCOMPUTE_DISPATCH_SCRATCH_BASE_LO                                                             0x1bb0
+#define regCOMPUTE_DISPATCH_SCRATCH_BASE_LO_BASE_IDX                                                    0
+#define regCOMPUTE_DISPATCH_SCRATCH_BASE_HI                                                             0x1bb1
+#define regCOMPUTE_DISPATCH_SCRATCH_BASE_HI_BASE_IDX                                                    0
+#define regCOMPUTE_PGM_RSRC1                                                                            0x1bb2
+#define regCOMPUTE_PGM_RSRC1_BASE_IDX                                                                   0
+#define regCOMPUTE_PGM_RSRC2                                                                            0x1bb3
+#define regCOMPUTE_PGM_RSRC2_BASE_IDX                                                                   0
+#define regCOMPUTE_VMID                                                                                 0x1bb4
+#define regCOMPUTE_VMID_BASE_IDX                                                                        0
+#define regCOMPUTE_RESOURCE_LIMITS                                                                      0x1bb5
+#define regCOMPUTE_RESOURCE_LIMITS_BASE_IDX                                                             0
+#define regCOMPUTE_TMPRING_SIZE                                                                         0x1bb6
+#define regCOMPUTE_TMPRING_SIZE_BASE_IDX                                                                0
+#define regCOMPUTE_RESTART_X                                                                            0x1bb7
+#define regCOMPUTE_RESTART_X_BASE_IDX                                                                   0
+#define regCOMPUTE_RESTART_Y                                                                            0x1bb8
+#define regCOMPUTE_RESTART_Y_BASE_IDX                                                                   0
+#define regCOMPUTE_RESTART_Z                                                                            0x1bb9
+#define regCOMPUTE_RESTART_Z_BASE_IDX                                                                   0
+#define regCOMPUTE_THREAD_TRACE_ENABLE                                                                  0x1bba
+#define regCOMPUTE_THREAD_TRACE_ENABLE_BASE_IDX                                                         0
+#define regCOMPUTE_MISC_RESERVED                                                                        0x1bbb
+#define regCOMPUTE_MISC_RESERVED_BASE_IDX                                                               0
+#define regCOMPUTE_DISPATCH_ID                                                                          0x1bbc
+#define regCOMPUTE_DISPATCH_ID_BASE_IDX                                                                 0
+#define regCOMPUTE_THREADGROUP_ID                                                                       0x1bbd
+#define regCOMPUTE_THREADGROUP_ID_BASE_IDX                                                              0
+#define regCOMPUTE_REQ_CTRL                                                                             0x1bbe
+#define regCOMPUTE_REQ_CTRL_BASE_IDX                                                                    0
+#define regCOMPUTE_USER_ACCUM_0                                                                         0x1bbf
+#define regCOMPUTE_USER_ACCUM_0_BASE_IDX                                                                0
+#define regCOMPUTE_USER_ACCUM_1                                                                         0x1bc0
+#define regCOMPUTE_USER_ACCUM_1_BASE_IDX                                                                0
+#define regCOMPUTE_USER_ACCUM_2                                                                         0x1bc1
+#define regCOMPUTE_USER_ACCUM_2_BASE_IDX                                                                0
+#define regCOMPUTE_USER_ACCUM_3                                                                         0x1bc2
+#define regCOMPUTE_USER_ACCUM_3_BASE_IDX                                                                0
+#define regCOMPUTE_PGM_RSRC3                                                                            0x1bc3
+#define regCOMPUTE_PGM_RSRC3_BASE_IDX                                                                   0
+#define regCOMPUTE_DDID_INDEX                                                                           0x1bc4
+#define regCOMPUTE_DDID_INDEX_BASE_IDX                                                                  0
+#define regCOMPUTE_SHADER_CHKSUM                                                                        0x1bc5
+#define regCOMPUTE_SHADER_CHKSUM_BASE_IDX                                                               0
+#define regCOMPUTE_DISPATCH_INTERLEAVE                                                                  0x1bc6
+#define regCOMPUTE_DISPATCH_INTERLEAVE_BASE_IDX                                                         0
+#define regCOMPUTE_CURRENT_LOGICAL_XCC_ID                                                               0x1bc7
+#define regCOMPUTE_CURRENT_LOGICAL_XCC_ID_BASE_IDX                                                      0
+#define regCOMPUTE_RESTART_CG_TG_ID                                                                     0x1bc8
+#define regCOMPUTE_RESTART_CG_TG_ID_BASE_IDX                                                            0
+#define regCOMPUTE_TG_CHUNK_SIZE                                                                        0x1bc9
+#define regCOMPUTE_TG_CHUNK_SIZE_BASE_IDX                                                               0
+#define regCOMPUTE_RESTORE_TG_CHUNK_SIZE                                                                0x1bca
+#define regCOMPUTE_RESTORE_TG_CHUNK_SIZE_BASE_IDX                                                       0
+#define regCOMPUTE_SPARE_1                                                                              0x1bcb
+#define regCOMPUTE_SPARE_1_BASE_IDX                                                                     0
+#define regCOMPUTE_SPARE_2                                                                              0x1bcc
+#define regCOMPUTE_SPARE_2_BASE_IDX                                                                     0
+#define regCOMPUTE_PRESCALED_DIM_X                                                                      0x1bcd
+#define regCOMPUTE_PRESCALED_DIM_X_BASE_IDX                                                             0
+#define regCOMPUTE_PRESCALED_DIM_Y                                                                      0x1bce
+#define regCOMPUTE_PRESCALED_DIM_Y_BASE_IDX                                                             0
+#define regCOMPUTE_PRESCALED_DIM_Z                                                                      0x1bcf
+#define regCOMPUTE_PRESCALED_DIM_Z_BASE_IDX                                                             0
+#define regCOMPUTE_DESTINATION_EN_SE0                                                                   0x1bd0
+#define regCOMPUTE_DESTINATION_EN_SE0_BASE_IDX                                                          0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE0                                                               0x1bd0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE0_BASE_IDX                                                      0
+#define regCOMPUTE_DESTINATION_EN_SE1                                                                   0x1bd1
+#define regCOMPUTE_DESTINATION_EN_SE1_BASE_IDX                                                          0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE1                                                               0x1bd1
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE1_BASE_IDX                                                      0
+#define regCOMPUTE_DESTINATION_EN_SE2                                                                   0x1bd2
+#define regCOMPUTE_DESTINATION_EN_SE2_BASE_IDX                                                          0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE2                                                               0x1bd2
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE2_BASE_IDX                                                      0
+#define regCOMPUTE_DESTINATION_EN_SE3                                                                   0x1bd3
+#define regCOMPUTE_DESTINATION_EN_SE3_BASE_IDX                                                          0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE3                                                               0x1bd3
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE3_BASE_IDX                                                      0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE4                                                               0x1bd4
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE4_BASE_IDX                                                      0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE5                                                               0x1bd5
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE5_BASE_IDX                                                      0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE6                                                               0x1bd6
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE6_BASE_IDX                                                      0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE7                                                               0x1bd7
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE7_BASE_IDX                                                      0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE8                                                               0x1bd8
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE8_BASE_IDX                                                      0
+#define regCOMPUTE_USER_DATA_0                                                                          0x1be0
+#define regCOMPUTE_USER_DATA_0_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_1                                                                          0x1be1
+#define regCOMPUTE_USER_DATA_1_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_2                                                                          0x1be2
+#define regCOMPUTE_USER_DATA_2_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_3                                                                          0x1be3
+#define regCOMPUTE_USER_DATA_3_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_4                                                                          0x1be4
+#define regCOMPUTE_USER_DATA_4_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_5                                                                          0x1be5
+#define regCOMPUTE_USER_DATA_5_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_6                                                                          0x1be6
+#define regCOMPUTE_USER_DATA_6_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_7                                                                          0x1be7
+#define regCOMPUTE_USER_DATA_7_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_8                                                                          0x1be8
+#define regCOMPUTE_USER_DATA_8_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_9                                                                          0x1be9
+#define regCOMPUTE_USER_DATA_9_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_10                                                                         0x1bea
+#define regCOMPUTE_USER_DATA_10_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_11                                                                         0x1beb
+#define regCOMPUTE_USER_DATA_11_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_12                                                                         0x1bec
+#define regCOMPUTE_USER_DATA_12_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_13                                                                         0x1bed
+#define regCOMPUTE_USER_DATA_13_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_14                                                                         0x1bee
+#define regCOMPUTE_USER_DATA_14_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_15                                                                         0x1bef
+#define regCOMPUTE_USER_DATA_15_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_16                                                                         0x1bf0
+#define regCOMPUTE_USER_DATA_16_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_17                                                                         0x1bf1
+#define regCOMPUTE_USER_DATA_17_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_18                                                                         0x1bf2
+#define regCOMPUTE_USER_DATA_18_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_19                                                                         0x1bf3
+#define regCOMPUTE_USER_DATA_19_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_20                                                                         0x1bf4
+#define regCOMPUTE_USER_DATA_20_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_21                                                                         0x1bf5
+#define regCOMPUTE_USER_DATA_21_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_22                                                                         0x1bf6
+#define regCOMPUTE_USER_DATA_22_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_23                                                                         0x1bf7
+#define regCOMPUTE_USER_DATA_23_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_24                                                                         0x1bf8
+#define regCOMPUTE_USER_DATA_24_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_25                                                                         0x1bf9
+#define regCOMPUTE_USER_DATA_25_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_26                                                                         0x1bfa
+#define regCOMPUTE_USER_DATA_26_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_27                                                                         0x1bfb
+#define regCOMPUTE_USER_DATA_27_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_28                                                                         0x1bfc
+#define regCOMPUTE_USER_DATA_28_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_29                                                                         0x1bfd
+#define regCOMPUTE_USER_DATA_29_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_30                                                                         0x1bfe
+#define regCOMPUTE_USER_DATA_30_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_31                                                                         0x1bff
+#define regCOMPUTE_USER_DATA_31_BASE_IDX                                                                0
+#define regCOMPUTE_RELAUNCH                                                                             0x1c0d
+#define regCOMPUTE_RELAUNCH_BASE_IDX                                                                    0
+#define regCOMPUTE_RELAUNCH_STATE_PAYLOAD                                                               0x1c0d
+#define regCOMPUTE_RELAUNCH_STATE_PAYLOAD_BASE_IDX                                                      0
+#define regCOMPUTE_WAVE_RESTORE_ADDR_LO                                                                 0x1c0e
+#define regCOMPUTE_WAVE_RESTORE_ADDR_LO_BASE_IDX                                                        0
+#define regCOMPUTE_WAVE_RESTORE_ADDR_HI                                                                 0x1c0f
+#define regCOMPUTE_WAVE_RESTORE_ADDR_HI_BASE_IDX                                                        0
+#define regCOMPUTE_RELAUNCH2                                                                            0x1c10
+#define regCOMPUTE_RELAUNCH2_BASE_IDX                                                                   0
+#define regCOMPUTE_RELAUNCH2_STATE_PAYLOAD                                                              0x1c10
+#define regCOMPUTE_RELAUNCH2_STATE_PAYLOAD_BASE_IDX                                                     0
+#define regCOMPUTE_DISPATCH_TUNNEL                                                                      0x1c1d
+#define regCOMPUTE_DISPATCH_TUNNEL_BASE_IDX                                                             0
+#define regCOMPUTE_DISPATCH_END                                                                         0x1c1e
+#define regCOMPUTE_DISPATCH_END_BASE_IDX                                                                0
+#define regCOMPUTE_NOWHERE                                                                              0x1c1f
+#define regCOMPUTE_NOWHERE_BASE_IDX                                                                     0
+#define regSH_RESERVED_REG0                                                                             0x1c20
+#define regSH_RESERVED_REG0_BASE_IDX                                                                    0
+#define regSH_RESERVED_REG1                                                                             0x1c21
+#define regSH_RESERVED_REG1_BASE_IDX                                                                    0
+#define regCOMPUTE_PREALLOC_CR_DB_BUFFER_SIZE                                                           0x1c22
+#define regCOMPUTE_PREALLOC_CR_DB_BUFFER_SIZE_BASE_IDX                                                  0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pfonly_gccacdec
+// base address: 0x2eb40
+#define regGC_CAC_CTRL_1                                                                                0x1ad0
+#define regGC_CAC_CTRL_1_BASE_IDX                                                                       1
+#define regGC_CAC_CTRL_2                                                                                0x1ad1
+#define regGC_CAC_CTRL_2_BASE_IDX                                                                       1
+#define regGC_CAC_AGGR_LOWER                                                                            0x1ad2
+#define regGC_CAC_AGGR_LOWER_BASE_IDX                                                                   1
+#define regGC_CAC_AGGR_UPPER                                                                            0x1ad3
+#define regGC_CAC_AGGR_UPPER_BASE_IDX                                                                   1
+#define regSE0_CAC_AGGR_LOWER                                                                           0x1ad4
+#define regSE0_CAC_AGGR_LOWER_BASE_IDX                                                                  1
+#define regSE0_CAC_AGGR_UPPER                                                                           0x1ad5
+#define regSE0_CAC_AGGR_UPPER_BASE_IDX                                                                  1
+#define regSE1_CAC_AGGR_LOWER                                                                           0x1ad6
+#define regSE1_CAC_AGGR_LOWER_BASE_IDX                                                                  1
+#define regSE1_CAC_AGGR_UPPER                                                                           0x1ad7
+#define regSE1_CAC_AGGR_UPPER_BASE_IDX                                                                  1
+#define regGC_CAC_AGGR_GFXCLK_CYCLE                                                                     0x1ae6
+#define regGC_CAC_AGGR_GFXCLK_CYCLE_BASE_IDX                                                            1
+#define regSE0_CAC_AGGR_GFXCLK_CYCLE                                                                    0x1ae7
+#define regSE0_CAC_AGGR_GFXCLK_CYCLE_BASE_IDX                                                           1
+#define regSE1_CAC_AGGR_GFXCLK_CYCLE                                                                    0x1ae8
+#define regSE1_CAC_AGGR_GFXCLK_CYCLE_BASE_IDX                                                           1
+#define regGC_CAC_OVR_VAL_LOWER                                                                         0x1af0
+#define regGC_CAC_OVR_VAL_LOWER_BASE_IDX                                                                1
+#define regGC_CAC_OVR_VAL_UPPER                                                                         0x1af1
+#define regGC_CAC_OVR_VAL_UPPER_BASE_IDX                                                                1
+#define regGC_EDC_CTRL                                                                                  0x1af4
+#define regGC_EDC_CTRL_BASE_IDX                                                                         1
+#define regGC_EDC_STRETCH_CTRL                                                                          0x1af5
+#define regGC_EDC_STRETCH_CTRL_BASE_IDX                                                                 1
+#define regGC_EDC_THRESHOLD_LO                                                                          0x1af6
+#define regGC_EDC_THRESHOLD_LO_BASE_IDX                                                                 1
+#define regGC_EDC_THRESHOLD_HI                                                                          0x1af7
+#define regGC_EDC_THRESHOLD_HI_BASE_IDX                                                                 1
+#define regGC_EDC_STRETCH_THRESHOLD_LO                                                                  0x1af8
+#define regGC_EDC_STRETCH_THRESHOLD_LO_BASE_IDX                                                         1
+#define regGC_EDC_STRETCH_THRESHOLD_HI                                                                  0x1af9
+#define regGC_EDC_STRETCH_THRESHOLD_HI_BASE_IDX                                                         1
+#define regEDC_HYSTERESIS_CNTL                                                                          0x1afa
+#define regEDC_HYSTERESIS_CNTL_BASE_IDX                                                                 1
+#define regGC_THROTTLE_CTRL                                                                             0x1afb
+#define regGC_THROTTLE_CTRL_BASE_IDX                                                                    1
+#define regGC_THROTTLE_CTRL1                                                                            0x1afc
+#define regGC_THROTTLE_CTRL1_BASE_IDX                                                                   1
+#define regGC_THROTTLE_CTRL2                                                                            0x1afd
+#define regGC_THROTTLE_CTRL2_BASE_IDX                                                                   1
+#define regEDC_STALL_PATTERN_CTRL                                                                       0x1afe
+#define regEDC_STALL_PATTERN_CTRL_BASE_IDX                                                              1
+#define regPCC_STALL_PATTERN_CTRL                                                                       0x1aff
+#define regPCC_STALL_PATTERN_CTRL_BASE_IDX                                                              1
+#define regPWRBRK_STALL_PATTERN_CTRL                                                                    0x1b00
+#define regPWRBRK_STALL_PATTERN_CTRL_BASE_IDX                                                           1
+#define regEDC_STALL_PATTERN_1_2                                                                        0x1b01
+#define regEDC_STALL_PATTERN_1_2_BASE_IDX                                                               1
+#define regEDC_STALL_PATTERN_3_4                                                                        0x1b02
+#define regEDC_STALL_PATTERN_3_4_BASE_IDX                                                               1
+#define regEDC_STALL_PATTERN_5_6                                                                        0x1b03
+#define regEDC_STALL_PATTERN_5_6_BASE_IDX                                                               1
+#define regEDC_STALL_PATTERN_7                                                                          0x1b04
+#define regEDC_STALL_PATTERN_7_BASE_IDX                                                                 1
+#define regPCC_STALL_PATTERN_1_2                                                                        0x1b05
+#define regPCC_STALL_PATTERN_1_2_BASE_IDX                                                               1
+#define regPCC_STALL_PATTERN_3_4                                                                        0x1b06
+#define regPCC_STALL_PATTERN_3_4_BASE_IDX                                                               1
+#define regPCC_STALL_PATTERN_5_6                                                                        0x1b07
+#define regPCC_STALL_PATTERN_5_6_BASE_IDX                                                               1
+#define regPCC_STALL_PATTERN_7                                                                          0x1b08
+#define regPCC_STALL_PATTERN_7_BASE_IDX                                                                 1
+#define regPWRBRK_STALL_PATTERN_1_2                                                                     0x1b09
+#define regPWRBRK_STALL_PATTERN_1_2_BASE_IDX                                                            1
+#define regPWRBRK_STALL_PATTERN_3_4                                                                     0x1b0a
+#define regPWRBRK_STALL_PATTERN_3_4_BASE_IDX                                                            1
+#define regPWRBRK_STALL_PATTERN_5_6                                                                     0x1b0b
+#define regPWRBRK_STALL_PATTERN_5_6_BASE_IDX                                                            1
+#define regPWRBRK_STALL_PATTERN_7                                                                       0x1b0c
+#define regPWRBRK_STALL_PATTERN_7_BASE_IDX                                                              1
+#define regDIDT_STALL_PATTERN_CTRL                                                                      0x1b0d
+#define regDIDT_STALL_PATTERN_CTRL_BASE_IDX                                                             1
+#define regDIDT_STALL_PATTERN_1_2                                                                       0x1b0e
+#define regDIDT_STALL_PATTERN_1_2_BASE_IDX                                                              1
+#define regDIDT_STALL_PATTERN_3_4                                                                       0x1b0f
+#define regDIDT_STALL_PATTERN_3_4_BASE_IDX                                                              1
+#define regDIDT_STALL_PATTERN_5_6                                                                       0x1b10
+#define regDIDT_STALL_PATTERN_5_6_BASE_IDX                                                              1
+#define regDIDT_STALL_PATTERN_7                                                                         0x1b11
+#define regDIDT_STALL_PATTERN_7_BASE_IDX                                                                1
+#define regPCC_PWRBRK_HYSTERESIS_CTRL                                                                   0x1b12
+#define regPCC_PWRBRK_HYSTERESIS_CTRL_BASE_IDX                                                          1
+#define regEDC_STRETCH_PERF_COUNTER                                                                     0x1b13
+#define regEDC_STRETCH_PERF_COUNTER_BASE_IDX                                                            1
+#define regEDC_UNSTRETCH_PERF_COUNTER                                                                   0x1b14
+#define regEDC_UNSTRETCH_PERF_COUNTER_BASE_IDX                                                          1
+#define regEDC_STRETCH_NUM_PERF_COUNTER                                                                 0x1b15
+#define regEDC_STRETCH_NUM_PERF_COUNTER_BASE_IDX                                                        1
+#define regGC_EDC_STATUS                                                                                0x1b16
+#define regGC_EDC_STATUS_BASE_IDX                                                                       1
+#define regGC_EDC_OVERFLOW                                                                              0x1b17
+#define regGC_EDC_OVERFLOW_BASE_IDX                                                                     1
+#define regGC_EDC_ROLLING_POWER_DELTA_LO                                                                0x1b18
+#define regGC_EDC_ROLLING_POWER_DELTA_LO_BASE_IDX                                                       1
+#define regGC_EDC_ROLLING_POWER_DELTA_HI                                                                0x1b19
+#define regGC_EDC_ROLLING_POWER_DELTA_HI_BASE_IDX                                                       1
+#define regGC_EDC_AVG_POWER_DELTA_LO                                                                    0x1b1a
+#define regGC_EDC_AVG_POWER_DELTA_LO_BASE_IDX                                                           1
+#define regGC_EDC_AVG_POWER_DELTA_HI                                                                    0x1b1b
+#define regGC_EDC_AVG_POWER_DELTA_HI_BASE_IDX                                                           1
+#define regGC_THROTTLE_STATUS                                                                           0x1b1c
+#define regGC_THROTTLE_STATUS_BASE_IDX                                                                  1
+#define regEDC_PERF_COUNTER                                                                             0x1b1d
+#define regEDC_PERF_COUNTER_BASE_IDX                                                                    1
+#define regPCC_PERF_COUNTER                                                                             0x1b1e
+#define regPCC_PERF_COUNTER_BASE_IDX                                                                    1
+#define regPWRBRK_PERF_COUNTER                                                                          0x1b1f
+#define regPWRBRK_PERF_COUNTER_BASE_IDX                                                                 1
+#define regEDC_HYSTERESIS_STAT                                                                          0x1b20
+#define regEDC_HYSTERESIS_STAT_BASE_IDX                                                                 1
+#define regDIDT_HYSTERESIS_STAT                                                                         0x1b21
+#define regDIDT_HYSTERESIS_STAT_BASE_IDX                                                                1
+#define regDIDT_PERF_COUNTER                                                                            0x1b22
+#define regDIDT_PERF_COUNTER_BASE_IDX                                                                   1
+#define regGC_EDC_CLK_MONITOR_CTRL                                                                      0x1b23
+#define regGC_EDC_CLK_MONITOR_CTRL_BASE_IDX                                                             1
+#define regGC_CAC_SOFT_CTRL                                                                             0x1b24
+#define regGC_CAC_SOFT_CTRL_BASE_IDX                                                                    1
+#define regGC_CAC_WEIGHT_CP_0                                                                           0x1b30
+#define regGC_CAC_WEIGHT_CP_0_BASE_IDX                                                                  1
+#define regGC_CAC_WEIGHT_CP_1                                                                           0x1b31
+#define regGC_CAC_WEIGHT_CP_1_BASE_IDX                                                                  1
+#define regGC_CAC_WEIGHT_PMM_0                                                                          0x1b4b
+#define regGC_CAC_WEIGHT_PMM_0_BASE_IDX                                                                 1
+#define regGC_CAC_WEIGHT_SDMA_0                                                                         0x1b50
+#define regGC_CAC_WEIGHT_SDMA_0_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_SDMA_1                                                                         0x1b51
+#define regGC_CAC_WEIGHT_SDMA_1_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_SDMA_2                                                                         0x1b52
+#define regGC_CAC_WEIGHT_SDMA_2_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_SDMA_3                                                                         0x1b53
+#define regGC_CAC_WEIGHT_SDMA_3_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_SDMA_4                                                                         0x1b54
+#define regGC_CAC_WEIGHT_SDMA_4_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_SDMA_5                                                                         0x1b55
+#define regGC_CAC_WEIGHT_SDMA_5_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_CHC_0                                                                          0x1b56
+#define regGC_CAC_WEIGHT_CHC_0_BASE_IDX                                                                 1
+#define regGC_CAC_WEIGHT_CHC_1                                                                          0x1b57
+#define regGC_CAC_WEIGHT_CHC_1_BASE_IDX                                                                 1
+#define regGC_CAC_WEIGHT_RLC_0                                                                          0x1b5a
+#define regGC_CAC_WEIGHT_RLC_0_BASE_IDX                                                                 1
+#define regGC_CAC_WEIGHT_GRBM_0                                                                         0x1b5e
+#define regGC_CAC_WEIGHT_GRBM_0_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_SDMA_6                                                                         0x1b76
+#define regGC_CAC_WEIGHT_SDMA_6_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_SDMA_7                                                                         0x1b77
+#define regGC_CAC_WEIGHT_SDMA_7_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_GLARB_0                                                                        0x1b78
+#define regGC_CAC_WEIGHT_GLARB_0_BASE_IDX                                                               1
+#define regGC_CAC_WEIGHT_GLARB_1                                                                        0x1b79
+#define regGC_CAC_WEIGHT_GLARB_1_BASE_IDX                                                               1
+#define regXCD_TOTAL_CAC_AGGR_LOWER                                                                     0x1b7c
+#define regXCD_TOTAL_CAC_AGGR_LOWER_BASE_IDX                                                            1
+#define regXCD_TOTAL_CAC_AGGR_UPPER                                                                     0x1b7d
+#define regXCD_TOTAL_CAC_AGGR_UPPER_BASE_IDX                                                            1
+#define regGC_EDC_WEIGHTED_DATA_MULTIPLIER                                                              0x1b7e
+#define regGC_EDC_WEIGHTED_DATA_MULTIPLIER_BASE_IDX                                                     1
+#define regGC_CAC_IND_INDEX                                                                             0x1b8e
+#define regGC_CAC_IND_INDEX_BASE_IDX                                                                    1
+#define regGC_CAC_IND_DATA                                                                              0x1b8f
+#define regGC_CAC_IND_DATA_BASE_IDX                                                                     1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_gc_ea_cpwd_gceadec
+// base address: 0xa800
+#define regGC_EA_CPWD_VC_MAP                                                                            0x17a0
+#define regGC_EA_CPWD_VC_MAP_BASE_IDX                                                                   0
+#define regGC_EA_CPWD_SDP_ARB_FINAL                                                                     0x17a1
+#define regGC_EA_CPWD_SDP_ARB_FINAL_BASE_IDX                                                            0
+#define regGC_EA_CPWD_SDP_PRIORITY                                                                      0x17a2
+#define regGC_EA_CPWD_SDP_PRIORITY_BASE_IDX                                                             0
+#define regGC_EA_CPWD_SDP_CREDITS                                                                       0x17a3
+#define regGC_EA_CPWD_SDP_CREDITS_BASE_IDX                                                              0
+#define regGC_EA_CPWD_SDP_TAG_RESERVE0                                                                  0x17a4
+#define regGC_EA_CPWD_SDP_TAG_RESERVE0_BASE_IDX                                                         0
+#define regGC_EA_CPWD_SDP_TAG_RESERVE1                                                                  0x17a5
+#define regGC_EA_CPWD_SDP_TAG_RESERVE1_BASE_IDX                                                         0
+#define regGC_EA_CPWD_SDP_TAG_RESERVE2                                                                  0x17a6
+#define regGC_EA_CPWD_SDP_TAG_RESERVE2_BASE_IDX                                                         0
+#define regGC_EA_CPWD_SDP_VCC_RESERVE0                                                                  0x17a7
+#define regGC_EA_CPWD_SDP_VCC_RESERVE0_BASE_IDX                                                         0
+#define regGC_EA_CPWD_SDP_VCC_RESERVE1                                                                  0x17a8
+#define regGC_EA_CPWD_SDP_VCC_RESERVE1_BASE_IDX                                                         0
+#define regGC_EA_CPWD_SDP_VCD_RESERVE0                                                                  0x17a9
+#define regGC_EA_CPWD_SDP_VCD_RESERVE0_BASE_IDX                                                         0
+#define regGC_EA_CPWD_SDP_VCD_RESERVE1                                                                  0x17aa
+#define regGC_EA_CPWD_SDP_VCD_RESERVE1_BASE_IDX                                                         0
+#define regGC_EA_CPWD_SDP_REQ_CNTL                                                                      0x17ab
+#define regGC_EA_CPWD_SDP_REQ_CNTL_BASE_IDX                                                             0
+#define regGC_EA_CPWD_MISC                                                                              0x17ac
+#define regGC_EA_CPWD_MISC_BASE_IDX                                                                     0
+#define regGC_EA_CPWD_ERR_STATUS                                                                        0x17ad
+#define regGC_EA_CPWD_ERR_STATUS_BASE_IDX                                                               0
+#define regGC_EA_CPWD_MISC2                                                                             0x17ae
+#define regGC_EA_CPWD_MISC2_BASE_IDX                                                                    0
+#define regGC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0                                                          0x17af
+#define regGC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0_BASE_IDX                                                 0
+#define regGC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0_WRITE                                                    0x17b0
+#define regGC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0_WRITE_BASE_IDX                                           0
+#define regGC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1                                                          0x17b1
+#define regGC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1_BASE_IDX                                                 0
+#define regGC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1_WRITE                                                    0x17b2
+#define regGC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1_WRITE_BASE_IDX                                           0
+#define regGC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0                                                         0x17b3
+#define regGC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0_BASE_IDX                                                0
+#define regGC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0_WRITE                                                   0x17b4
+#define regGC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0_WRITE_BASE_IDX                                          0
+#define regGC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1                                                         0x17b5
+#define regGC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1_BASE_IDX                                                0
+#define regGC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1_WRITE                                                   0x17b6
+#define regGC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1_WRITE_BASE_IDX                                          0
+#define regGC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2                                                         0x17b7
+#define regGC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2_BASE_IDX                                                0
+#define regGC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2_WRITE                                                   0x17b8
+#define regGC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2_WRITE_BASE_IDX                                          0
+#define regGC_EA_CPWD_SDP_BACKDOOR_MISCCTL                                                              0x17b9
+#define regGC_EA_CPWD_SDP_BACKDOOR_MISCCTL_BASE_IDX                                                     0
+#define regGC_EA_CPWD_SDP_BACKDOOR_MISCCTL_WRITE                                                        0x17ba
+#define regGC_EA_CPWD_SDP_BACKDOOR_MISCCTL_WRITE_BASE_IDX                                               0
+#define regGC_EA_CPWD_INVALID_OPCODE                                                                    0x17bb
+#define regGC_EA_CPWD_INVALID_OPCODE_BASE_IDX                                                           0
+#define regGC_EA_CPWD_SDP_POISON_ERR_INJ_CTRL                                                           0x17bc
+#define regGC_EA_CPWD_SDP_POISON_ERR_INJ_CTRL_BASE_IDX                                                  0
+#define regGC_EA_CPWD_SDP_POISON_ERR_LOG                                                                0x17bd
+#define regGC_EA_CPWD_SDP_POISON_ERR_LOG_BASE_IDX                                                       0
+#define regGC_EA_CPWD_SDP_PARITY_ERR_INJ_CTRL                                                           0x17be
+#define regGC_EA_CPWD_SDP_PARITY_ERR_INJ_CTRL_BASE_IDX                                                  0
+#define regGC_EA_CPWD_SDP_PARITY_ERR_LOG                                                                0x17bf
+#define regGC_EA_CPWD_SDP_PARITY_ERR_LOG_BASE_IDX                                                       0
+#define regGC_EA_CPWD_SDP_ENABLE                                                                        0x181f
+#define regGC_EA_CPWD_SDP_ENABLE_BASE_IDX                                                               0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_gcrdec
+// base address: 0x9f80
+#define regGCR_PIO_CNTL                                                                                 0x1580
+#define regGCR_PIO_CNTL_BASE_IDX                                                                        0
+#define regGCR_PIO_DATA                                                                                 0x1581
+#define regGCR_PIO_DATA_BASE_IDX                                                                        0
+#define regGCR_NHTOE_CNTL                                                                               0x1584
+#define regGCR_NHTOE_CNTL_BASE_IDX                                                                      0
+#define regGCR_NHTOE_DATA                                                                               0x1585
+#define regGCR_NHTOE_DATA_BASE_IDX                                                                      0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_cppdec
+// base address: 0xc080
+#define regCPC_INT_ADDR1                                                                                0x1dd1
+#define regCPC_INT_ADDR1_BASE_IDX                                                                       0
+#define regCP_CU_MASK_ADDR_LO                                                                           0x1dd2
+#define regCP_CU_MASK_ADDR_LO_BASE_IDX                                                                  0
+#define regCP_CU_MASK_ADDR_HI                                                                           0x1dd3
+#define regCP_CU_MASK_ADDR_HI_BASE_IDX                                                                  0
+#define regCP_CU_MASK_CNTL                                                                              0x1dd4
+#define regCP_CU_MASK_CNTL_BASE_IDX                                                                     0
+#define regCP_EOPQ_WAIT_TIME                                                                            0x1dd5
+#define regCP_EOPQ_WAIT_TIME_BASE_IDX                                                                   0
+#define regCP_CPC_MGCG_SYNC_CNTL                                                                        0x1dd6
+#define regCP_CPC_MGCG_SYNC_CNTL_BASE_IDX                                                               0
+#define regCPC_INT_INFO                                                                                 0x1dd7
+#define regCPC_INT_INFO_BASE_IDX                                                                        0
+#define regCP_VIRT_STATUS                                                                               0x1dd8
+#define regCP_VIRT_STATUS_BASE_IDX                                                                      0
+#define regCPC_INT_ADDR                                                                                 0x1dd9
+#define regCPC_INT_ADDR_BASE_IDX                                                                        0
+#define regCPC_INT_PASID                                                                                0x1dda
+#define regCPC_INT_PASID_BASE_IDX                                                                       0
+#define regCP_GFX_ERROR                                                                                 0x1ddb
+#define regCP_GFX_ERROR_BASE_IDX                                                                        0
+#define regCPG_UTCL1_CNTL                                                                               0x1ddc
+#define regCPG_UTCL1_CNTL_BASE_IDX                                                                      0
+#define regCPC_UTCL1_CNTL                                                                               0x1ddd
+#define regCPC_UTCL1_CNTL_BASE_IDX                                                                      0
+#define regCPF_UTCL1_CNTL                                                                               0x1dde
+#define regCPF_UTCL1_CNTL_BASE_IDX                                                                      0
+#define regCP_AQL_SMM_STATUS                                                                            0x1ddf
+#define regCP_AQL_SMM_STATUS_BASE_IDX                                                                   0
+#define regCP_RB0_BASE                                                                                  0x1de0
+#define regCP_RB0_BASE_BASE_IDX                                                                         0
+#define regCP_RB_BASE                                                                                   0x1de0
+#define regCP_RB_BASE_BASE_IDX                                                                          0
+#define regCP_RB0_CNTL                                                                                  0x1de1
+#define regCP_RB0_CNTL_BASE_IDX                                                                         0
+#define regCP_RB_CNTL                                                                                   0x1de1
+#define regCP_RB_CNTL_BASE_IDX                                                                          0
+#define regCP_RB_RPTR_WR                                                                                0x1de2
+#define regCP_RB_RPTR_WR_BASE_IDX                                                                       0
+#define regCP_RB0_RPTR_ADDR                                                                             0x1de3
+#define regCP_RB0_RPTR_ADDR_BASE_IDX                                                                    0
+#define regCP_RB_RPTR_ADDR                                                                              0x1de3
+#define regCP_RB_RPTR_ADDR_BASE_IDX                                                                     0
+#define regCP_RB0_RPTR_ADDR_HI                                                                          0x1de4
+#define regCP_RB0_RPTR_ADDR_HI_BASE_IDX                                                                 0
+#define regCP_RB_RPTR_ADDR_HI                                                                           0x1de4
+#define regCP_RB_RPTR_ADDR_HI_BASE_IDX                                                                  0
+#define regCP_RB0_BUFSZ_MASK                                                                            0x1de5
+#define regCP_RB0_BUFSZ_MASK_BASE_IDX                                                                   0
+#define regCP_RB_BUFSZ_MASK                                                                             0x1de5
+#define regCP_RB_BUFSZ_MASK_BASE_IDX                                                                    0
+#define regCP_ME3_INT_STAT_DEBUG                                                                        0x1de6
+#define regCP_ME3_INT_STAT_DEBUG_BASE_IDX                                                               0
+#define regGC_PRIV_MODE                                                                                 0x1de8
+#define regGC_PRIV_MODE_BASE_IDX                                                                        0
+#define regCP_INT_CNTL                                                                                  0x1de9
+#define regCP_INT_CNTL_BASE_IDX                                                                         0
+#define regCP_INT_STATUS                                                                                0x1dea
+#define regCP_INT_STATUS_BASE_IDX                                                                       0
+#define regCP_DEVICE_ID                                                                                 0x1deb
+#define regCP_DEVICE_ID_BASE_IDX                                                                        0
+#define regCP_ME0_PIPE_PRIORITY_CNTS                                                                    0x1dec
+#define regCP_ME0_PIPE_PRIORITY_CNTS_BASE_IDX                                                           0
+#define regCP_RING_PRIORITY_CNTS                                                                        0x1dec
+#define regCP_RING_PRIORITY_CNTS_BASE_IDX                                                               0
+#define regCP_ME0_PIPE0_PRIORITY                                                                        0x1ded
+#define regCP_ME0_PIPE0_PRIORITY_BASE_IDX                                                               0
+#define regCP_RING0_PRIORITY                                                                            0x1ded
+#define regCP_RING0_PRIORITY_BASE_IDX                                                                   0
+#define regCP_FATAL_ERROR                                                                               0x1df0
+#define regCP_FATAL_ERROR_BASE_IDX                                                                      0
+#define regCP_RB_VMID                                                                                   0x1df1
+#define regCP_RB_VMID_BASE_IDX                                                                          0
+#define regCP_ME0_PIPE0_VMID                                                                            0x1df2
+#define regCP_ME0_PIPE0_VMID_BASE_IDX                                                                   0
+#define regCP_RB0_WPTR                                                                                  0x1df4
+#define regCP_RB0_WPTR_BASE_IDX                                                                         0
+#define regCP_RB_WPTR                                                                                   0x1df4
+#define regCP_RB_WPTR_BASE_IDX                                                                          0
+#define regCP_RB0_WPTR_HI                                                                               0x1df5
+#define regCP_RB0_WPTR_HI_BASE_IDX                                                                      0
+#define regCP_RB_WPTR_HI                                                                                0x1df5
+#define regCP_RB_WPTR_HI_BASE_IDX                                                                       0
+#define regCP_PROCESS_QUANTUM                                                                           0x1df9
+#define regCP_PROCESS_QUANTUM_BASE_IDX                                                                  0
+#define regCP_RB_DOORBELL_RANGE_LOWER                                                                   0x1dfa
+#define regCP_RB_DOORBELL_RANGE_LOWER_BASE_IDX                                                          0
+#define regCP_RB_DOORBELL_RANGE_UPPER                                                                   0x1dfb
+#define regCP_RB_DOORBELL_RANGE_UPPER_BASE_IDX                                                          0
+#define regCP_MEC_DOORBELL_RANGE_LOWER                                                                  0x1dfc
+#define regCP_MEC_DOORBELL_RANGE_LOWER_BASE_IDX                                                         0
+#define regCP_MEC_DOORBELL_RANGE_UPPER                                                                  0x1dfd
+#define regCP_MEC_DOORBELL_RANGE_UPPER_BASE_IDX                                                         0
+#define regCPG_UTCL1_ERROR                                                                              0x1dfe
+#define regCPG_UTCL1_ERROR_BASE_IDX                                                                     0
+#define regCPC_UTCL1_ERROR                                                                              0x1dff
+#define regCPC_UTCL1_ERROR_BASE_IDX                                                                     0
+#define regCPF_FED_STATUS                                                                               0x1e05
+#define regCPF_FED_STATUS_BASE_IDX                                                                      0
+#define regCPC_FED_STATUS                                                                               0x1e06
+#define regCPC_FED_STATUS_BASE_IDX                                                                      0
+#define regCPG_FED_STATUS                                                                               0x1e07
+#define regCPG_FED_STATUS_BASE_IDX                                                                      0
+#define regCP_IB1_BUFFER_COUNT                                                                          0x1e08
+#define regCP_IB1_BUFFER_COUNT_BASE_IDX                                                                 0
+#define regCP_IB2_BUFFER_COUNT                                                                          0x1e09
+#define regCP_IB2_BUFFER_COUNT_BASE_IDX                                                                 0
+#define regCP_INT_CNTL_RING0                                                                            0x1e0a
+#define regCP_INT_CNTL_RING0_BASE_IDX                                                                   0
+#define regCP_DEBUG_2                                                                                   0x1e0c
+#define regCP_DEBUG_2_BASE_IDX                                                                          0
+#define regCP_INT_STATUS_RING0                                                                          0x1e0d
+#define regCP_INT_STATUS_RING0_BASE_IDX                                                                 0
+#define regCP_ME_F32_INTERRUPT                                                                          0x1e13
+#define regCP_ME_F32_INTERRUPT_BASE_IDX                                                                 0
+#define regCP_PFP_F32_INTERRUPT                                                                         0x1e14
+#define regCP_PFP_F32_INTERRUPT_BASE_IDX                                                                0
+#define regCP_MEC1_F32_INTERRUPT                                                                        0x1e16
+#define regCP_MEC1_F32_INTERRUPT_BASE_IDX                                                               0
+#define regCP_PWR_CNTL                                                                                  0x1e18
+#define regCP_PWR_CNTL_BASE_IDX                                                                         0
+#define regCP_ECC_FIRSTOCCURRENCE                                                                       0x1e1a
+#define regCP_ECC_FIRSTOCCURRENCE_BASE_IDX                                                              0
+#define regCP_ECC_FIRSTOCCURRENCE_RING0                                                                 0x1e1b
+#define regCP_ECC_FIRSTOCCURRENCE_RING0_BASE_IDX                                                        0
+#define regGB_EDC_MODE                                                                                  0x1e1e
+#define regGB_EDC_MODE_BASE_IDX                                                                         0
+#define regCP_DEBUG                                                                                     0x1e1f
+#define regCP_DEBUG_BASE_IDX                                                                            0
+#define regCP_CPF_DEBUG                                                                                 0x1e20
+#define regCP_CPF_DEBUG_BASE_IDX                                                                        0
+#define regCP_CPC_DEBUG                                                                                 0x1e21
+#define regCP_CPC_DEBUG_BASE_IDX                                                                        0
+#define regCP_PQ_WPTR_POLL_CNTL                                                                         0x1e23
+#define regCP_PQ_WPTR_POLL_CNTL_BASE_IDX                                                                0
+#define regCP_PQ_WPTR_POLL_CNTL1                                                                        0x1e24
+#define regCP_PQ_WPTR_POLL_CNTL1_BASE_IDX                                                               0
+#define regCP_ME1_PIPE0_INT_CNTL                                                                        0x1e25
+#define regCP_ME1_PIPE0_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME1_PIPE1_INT_CNTL                                                                        0x1e26
+#define regCP_ME1_PIPE1_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME1_PIPE2_INT_CNTL                                                                        0x1e27
+#define regCP_ME1_PIPE2_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME1_PIPE3_INT_CNTL                                                                        0x1e28
+#define regCP_ME1_PIPE3_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME1_PIPE0_INT_STATUS                                                                      0x1e2d
+#define regCP_ME1_PIPE0_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME1_PIPE1_INT_STATUS                                                                      0x1e2e
+#define regCP_ME1_PIPE1_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME1_PIPE2_INT_STATUS                                                                      0x1e2f
+#define regCP_ME1_PIPE2_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME1_PIPE3_INT_STATUS                                                                      0x1e30
+#define regCP_ME1_PIPE3_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME1_INT_STAT_DEBUG                                                                        0x1e35
+#define regCP_ME1_INT_STAT_DEBUG_BASE_IDX                                                               0
+#define regCP_GFX_QUEUE_INDEX                                                                           0x1e37
+#define regCP_GFX_QUEUE_INDEX_BASE_IDX                                                                  0
+#define regCC_GC_EDC_CONFIG                                                                             0x1e38
+#define regCC_GC_EDC_CONFIG_BASE_IDX                                                                    0
+#define regCP_ME1_PIPE_PRIORITY_CNTS                                                                    0x1e39
+#define regCP_ME1_PIPE_PRIORITY_CNTS_BASE_IDX                                                           0
+#define regCP_ME1_PIPE0_PRIORITY                                                                        0x1e3a
+#define regCP_ME1_PIPE0_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME1_PIPE1_PRIORITY                                                                        0x1e3b
+#define regCP_ME1_PIPE1_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME1_PIPE2_PRIORITY                                                                        0x1e3c
+#define regCP_ME1_PIPE2_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME1_PIPE3_PRIORITY                                                                        0x1e3d
+#define regCP_ME1_PIPE3_PRIORITY_BASE_IDX                                                               0
+#define regCP_PFP_PRGRM_CNTR_START                                                                      0x1e44
+#define regCP_PFP_PRGRM_CNTR_START_BASE_IDX                                                             0
+#define regCP_ME_PRGRM_CNTR_START                                                                       0x1e45
+#define regCP_ME_PRGRM_CNTR_START_BASE_IDX                                                              0
+#define regCP_MEC1_PRGRM_CNTR_START                                                                     0x1e46
+#define regCP_MEC1_PRGRM_CNTR_START_BASE_IDX                                                            0
+#define regCP_PFP_INTR_ROUTINE_START                                                                    0x1e49
+#define regCP_PFP_INTR_ROUTINE_START_BASE_IDX                                                           0
+#define regCP_ME_INTR_ROUTINE_START                                                                     0x1e4a
+#define regCP_ME_INTR_ROUTINE_START_BASE_IDX                                                            0
+#define regCP_MEC1_INTR_ROUTINE_START                                                                   0x1e4b
+#define regCP_MEC1_INTR_ROUTINE_START_BASE_IDX                                                          0
+#define regCP_CONTEXT_CNTL                                                                              0x1e4d
+#define regCP_CONTEXT_CNTL_BASE_IDX                                                                     0
+#define regCP_MAX_CONTEXT                                                                               0x1e4e
+#define regCP_MAX_CONTEXT_BASE_IDX                                                                      0
+#define regCP_IQ_WAIT_TIME1                                                                             0x1e4f
+#define regCP_IQ_WAIT_TIME1_BASE_IDX                                                                    0
+#define regCP_IQ_WAIT_TIME2                                                                             0x1e50
+#define regCP_IQ_WAIT_TIME2_BASE_IDX                                                                    0
+#define regCP_RB0_BASE_HI                                                                               0x1e51
+#define regCP_RB0_BASE_HI_BASE_IDX                                                                      0
+#define regCP_VMID_RESET                                                                                0x1e53
+#define regCP_VMID_RESET_BASE_IDX                                                                       0
+#define regCPC_INT_CNTL                                                                                 0x1e54
+#define regCPC_INT_CNTL_BASE_IDX                                                                        0
+#define regCPC_INT_STATUS                                                                               0x1e55
+#define regCPC_INT_STATUS_BASE_IDX                                                                      0
+#define regCP_VMID_PREEMPT                                                                              0x1e56
+#define regCP_VMID_PREEMPT_BASE_IDX                                                                     0
+#define regCPC_INT_CNTX_ID                                                                              0x1e57
+#define regCPC_INT_CNTX_ID_BASE_IDX                                                                     0
+#define regCP_PQ_STATUS                                                                                 0x1e58
+#define regCP_PQ_STATUS_BASE_IDX                                                                        0
+#define regCP_PFP_PRGRM_CNTR_START_HI                                                                   0x1e59
+#define regCP_PFP_PRGRM_CNTR_START_HI_BASE_IDX                                                          0
+#define regCP_MAX_DRAW_COUNT                                                                            0x1e5c
+#define regCP_MAX_DRAW_COUNT_BASE_IDX                                                                   0
+#define regCP_VMID_STATUS                                                                               0x1e5f
+#define regCP_VMID_STATUS_BASE_IDX                                                                      0
+#define regCPC_SUSPEND_CTX_SAVE_BASE_ADDR_LO                                                            0x1e60
+#define regCPC_SUSPEND_CTX_SAVE_BASE_ADDR_LO_BASE_IDX                                                   0
+#define regCPC_SUSPEND_CTX_SAVE_BASE_ADDR_HI                                                            0x1e61
+#define regCPC_SUSPEND_CTX_SAVE_BASE_ADDR_HI_BASE_IDX                                                   0
+#define regCPC_SUSPEND_CTX_SAVE_CONTROL                                                                 0x1e62
+#define regCPC_SUSPEND_CTX_SAVE_CONTROL_BASE_IDX                                                        0
+#define regCPC_SUSPEND_CNTL_STACK_OFFSET                                                                0x1e63
+#define regCPC_SUSPEND_CNTL_STACK_OFFSET_BASE_IDX                                                       0
+#define regCPC_SUSPEND_CNTL_STACK_SIZE                                                                  0x1e64
+#define regCPC_SUSPEND_CNTL_STACK_SIZE_BASE_IDX                                                         0
+#define regCPC_SUSPEND_WG_STATE_OFFSET                                                                  0x1e65
+#define regCPC_SUSPEND_WG_STATE_OFFSET_BASE_IDX                                                         0
+#define regCPC_SUSPEND_CTX_SAVE_SIZE                                                                    0x1e66
+#define regCPC_SUSPEND_CTX_SAVE_SIZE_BASE_IDX                                                           0
+#define regCPC_OS_PIPES                                                                                 0x1e67
+#define regCPC_OS_PIPES_BASE_IDX                                                                        0
+#define regCP_SUSPEND_RESUME_REQ                                                                        0x1e68
+#define regCP_SUSPEND_RESUME_REQ_BASE_IDX                                                               0
+#define regCP_SUSPEND_CNTL                                                                              0x1e69
+#define regCP_SUSPEND_CNTL_BASE_IDX                                                                     0
+#define regCP_IQ_WAIT_TIME3                                                                             0x1e6a
+#define regCP_IQ_WAIT_TIME3_BASE_IDX                                                                    0
+#define regCPC_DDID_BASE_ADDR_LO                                                                        0x1e6b
+#define regCPC_DDID_BASE_ADDR_LO_BASE_IDX                                                               0
+#define regCP_DDID_BASE_ADDR_LO                                                                         0x1e6b
+#define regCP_DDID_BASE_ADDR_LO_BASE_IDX                                                                0
+#define regCPC_DDID_BASE_ADDR_HI                                                                        0x1e6c
+#define regCPC_DDID_BASE_ADDR_HI_BASE_IDX                                                               0
+#define regCP_DDID_BASE_ADDR_HI                                                                         0x1e6c
+#define regCP_DDID_BASE_ADDR_HI_BASE_IDX                                                                0
+#define regCPC_DDID_CNTL                                                                                0x1e6d
+#define regCPC_DDID_CNTL_BASE_IDX                                                                       0
+#define regCP_DDID_CNTL                                                                                 0x1e6d
+#define regCP_DDID_CNTL_BASE_IDX                                                                        0
+#define regCP_GFX_DDID_INFLIGHT_COUNT                                                                   0x1e6e
+#define regCP_GFX_DDID_INFLIGHT_COUNT_BASE_IDX                                                          0
+#define regCP_GFX_DDID_WPTR                                                                             0x1e6f
+#define regCP_GFX_DDID_WPTR_BASE_IDX                                                                    0
+#define regCP_GFX_DDID_RPTR                                                                             0x1e70
+#define regCP_GFX_DDID_RPTR_BASE_IDX                                                                    0
+#define regCP_GFX_DDID_DELTA_RPT_COUNT                                                                  0x1e71
+#define regCP_GFX_DDID_DELTA_RPT_COUNT_BASE_IDX                                                         0
+#define regCP_GFX_HPD_STATUS0                                                                           0x1e72
+#define regCP_GFX_HPD_STATUS0_BASE_IDX                                                                  0
+#define regCP_GFX_HPD_CONTROL0                                                                          0x1e73
+#define regCP_GFX_HPD_CONTROL0_BASE_IDX                                                                 0
+#define regCP_GFX_HPD_OSPRE_FENCE_ADDR_LO                                                               0x1e74
+#define regCP_GFX_HPD_OSPRE_FENCE_ADDR_LO_BASE_IDX                                                      0
+#define regCP_GFX_HPD_OSPRE_FENCE_ADDR_HI                                                               0x1e75
+#define regCP_GFX_HPD_OSPRE_FENCE_ADDR_HI_BASE_IDX                                                      0
+#define regCP_GFX_HPD_OSPRE_FENCE_DATA_LO                                                               0x1e76
+#define regCP_GFX_HPD_OSPRE_FENCE_DATA_LO_BASE_IDX                                                      0
+#define regCP_GFX_HPD_OSPRE_FENCE_DATA_HI                                                               0x1e77
+#define regCP_GFX_HPD_OSPRE_FENCE_DATA_HI_BASE_IDX                                                      0
+#define regCP_GFX_INDEX_MUTEX                                                                           0x1e78
+#define regCP_GFX_INDEX_MUTEX_BASE_IDX                                                                  0
+#define regCP_ME_PRGRM_CNTR_START_HI                                                                    0x1e79
+#define regCP_ME_PRGRM_CNTR_START_HI_BASE_IDX                                                           0
+#define regCP_PFP_INTR_ROUTINE_START_HI                                                                 0x1e7a
+#define regCP_PFP_INTR_ROUTINE_START_HI_BASE_IDX                                                        0
+#define regCP_ME_INTR_ROUTINE_START_HI                                                                  0x1e7b
+#define regCP_ME_INTR_ROUTINE_START_HI_BASE_IDX                                                         0
+#define regCP_GFX_MQD_BASE_ADDR                                                                         0x1e7e
+#define regCP_GFX_MQD_BASE_ADDR_BASE_IDX                                                                0
+#define regCP_GFX_MQD_BASE_ADDR_HI                                                                      0x1e7f
+#define regCP_GFX_MQD_BASE_ADDR_HI_BASE_IDX                                                             0
+#define regCP_GFX_HQD_ACTIVE                                                                            0x1e80
+#define regCP_GFX_HQD_ACTIVE_BASE_IDX                                                                   0
+#define regCP_GFX_HQD_VMID                                                                              0x1e81
+#define regCP_GFX_HQD_VMID_BASE_IDX                                                                     0
+#define regCP_GFX_HQD_QUEUE_PRIORITY                                                                    0x1e84
+#define regCP_GFX_HQD_QUEUE_PRIORITY_BASE_IDX                                                           0
+#define regCP_GFX_HQD_QUANTUM                                                                           0x1e85
+#define regCP_GFX_HQD_QUANTUM_BASE_IDX                                                                  0
+#define regCP_GFX_HQD_BASE                                                                              0x1e86
+#define regCP_GFX_HQD_BASE_BASE_IDX                                                                     0
+#define regCP_GFX_HQD_BASE_HI                                                                           0x1e87
+#define regCP_GFX_HQD_BASE_HI_BASE_IDX                                                                  0
+#define regCP_GFX_HQD_RPTR                                                                              0x1e88
+#define regCP_GFX_HQD_RPTR_BASE_IDX                                                                     0
+#define regCP_GFX_HQD_RPTR_ADDR                                                                         0x1e89
+#define regCP_GFX_HQD_RPTR_ADDR_BASE_IDX                                                                0
+#define regCP_GFX_HQD_RPTR_ADDR_HI                                                                      0x1e8a
+#define regCP_GFX_HQD_RPTR_ADDR_HI_BASE_IDX                                                             0
+#define regCP_RB_WPTR_POLL_ADDR_LO                                                                      0x1e8b
+#define regCP_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                             0
+#define regCP_RB_WPTR_POLL_ADDR_HI                                                                      0x1e8c
+#define regCP_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                             0
+#define regCP_RB_DOORBELL_CONTROL                                                                       0x1e8d
+#define regCP_RB_DOORBELL_CONTROL_BASE_IDX                                                              0
+#define regCP_GFX_HQD_OFFSET                                                                            0x1e8e
+#define regCP_GFX_HQD_OFFSET_BASE_IDX                                                                   0
+#define regCP_GFX_HQD_CNTL                                                                              0x1e8f
+#define regCP_GFX_HQD_CNTL_BASE_IDX                                                                     0
+#define regCP_GFX_HQD_CSMD_RPTR                                                                         0x1e90
+#define regCP_GFX_HQD_CSMD_RPTR_BASE_IDX                                                                0
+#define regCP_GFX_HQD_WPTR                                                                              0x1e91
+#define regCP_GFX_HQD_WPTR_BASE_IDX                                                                     0
+#define regCP_GFX_HQD_WPTR_HI                                                                           0x1e92
+#define regCP_GFX_HQD_WPTR_HI_BASE_IDX                                                                  0
+#define regCP_GFX_HQD_DEQUEUE_REQUEST                                                                   0x1e93
+#define regCP_GFX_HQD_DEQUEUE_REQUEST_BASE_IDX                                                          0
+#define regCP_GFX_HQD_MAPPED                                                                            0x1e94
+#define regCP_GFX_HQD_MAPPED_BASE_IDX                                                                   0
+#define regCP_GFX_HQD_QUE_MGR_CONTROL                                                                   0x1e95
+#define regCP_GFX_HQD_QUE_MGR_CONTROL_BASE_IDX                                                          0
+#define regCP_GFX_HQD_IQ_TIMER                                                                          0x1e96
+#define regCP_GFX_HQD_IQ_TIMER_BASE_IDX                                                                 0
+#define regCP_GFX_HQD_HQ_STATUS0                                                                        0x1e98
+#define regCP_GFX_HQD_HQ_STATUS0_BASE_IDX                                                               0
+#define regCP_GFX_HQD_HQ_CONTROL0                                                                       0x1e99
+#define regCP_GFX_HQD_HQ_CONTROL0_BASE_IDX                                                              0
+#define regCP_GFX_MQD_CONTROL                                                                           0x1e9a
+#define regCP_GFX_MQD_CONTROL_BASE_IDX                                                                  0
+#define regCP_HQD_GFX_CONTROL                                                                           0x1e9f
+#define regCP_HQD_GFX_CONTROL_BASE_IDX                                                                  0
+#define regCP_HQD_GFX_STATUS                                                                            0x1ea0
+#define regCP_HQD_GFX_STATUS_BASE_IDX                                                                   0
+#define regCP_DMA_WATCH0_ADDR_LO                                                                        0x1ec0
+#define regCP_DMA_WATCH0_ADDR_LO_BASE_IDX                                                               0
+#define regCP_DMA_WATCH0_ADDR_HI                                                                        0x1ec1
+#define regCP_DMA_WATCH0_ADDR_HI_BASE_IDX                                                               0
+#define regCP_DMA_WATCH0_MASK                                                                           0x1ec2
+#define regCP_DMA_WATCH0_MASK_BASE_IDX                                                                  0
+#define regCP_DMA_WATCH0_CNTL                                                                           0x1ec3
+#define regCP_DMA_WATCH0_CNTL_BASE_IDX                                                                  0
+#define regCP_DMA_WATCH1_ADDR_LO                                                                        0x1ec4
+#define regCP_DMA_WATCH1_ADDR_LO_BASE_IDX                                                               0
+#define regCP_DMA_WATCH1_ADDR_HI                                                                        0x1ec5
+#define regCP_DMA_WATCH1_ADDR_HI_BASE_IDX                                                               0
+#define regCP_DMA_WATCH1_MASK                                                                           0x1ec6
+#define regCP_DMA_WATCH1_MASK_BASE_IDX                                                                  0
+#define regCP_DMA_WATCH1_CNTL                                                                           0x1ec7
+#define regCP_DMA_WATCH1_CNTL_BASE_IDX                                                                  0
+#define regCP_DMA_WATCH2_ADDR_LO                                                                        0x1ec8
+#define regCP_DMA_WATCH2_ADDR_LO_BASE_IDX                                                               0
+#define regCP_DMA_WATCH2_ADDR_HI                                                                        0x1ec9
+#define regCP_DMA_WATCH2_ADDR_HI_BASE_IDX                                                               0
+#define regCP_DMA_WATCH2_MASK                                                                           0x1eca
+#define regCP_DMA_WATCH2_MASK_BASE_IDX                                                                  0
+#define regCP_DMA_WATCH2_CNTL                                                                           0x1ecb
+#define regCP_DMA_WATCH2_CNTL_BASE_IDX                                                                  0
+#define regCP_DMA_WATCH3_ADDR_LO                                                                        0x1ecc
+#define regCP_DMA_WATCH3_ADDR_LO_BASE_IDX                                                               0
+#define regCP_DMA_WATCH3_ADDR_HI                                                                        0x1ecd
+#define regCP_DMA_WATCH3_ADDR_HI_BASE_IDX                                                               0
+#define regCP_DMA_WATCH3_MASK                                                                           0x1ece
+#define regCP_DMA_WATCH3_MASK_BASE_IDX                                                                  0
+#define regCP_DMA_WATCH3_CNTL                                                                           0x1ecf
+#define regCP_DMA_WATCH3_CNTL_BASE_IDX                                                                  0
+#define regCP_DMA_WATCH_STAT_ADDR_LO                                                                    0x1ed0
+#define regCP_DMA_WATCH_STAT_ADDR_LO_BASE_IDX                                                           0
+#define regCP_DMA_WATCH_STAT_ADDR_HI                                                                    0x1ed1
+#define regCP_DMA_WATCH_STAT_ADDR_HI_BASE_IDX                                                           0
+#define regCP_DMA_WATCH_STAT                                                                            0x1ed2
+#define regCP_DMA_WATCH_STAT_BASE_IDX                                                                   0
+#define regCP_PFP_JT_STAT                                                                               0x1ed3
+#define regCP_PFP_JT_STAT_BASE_IDX                                                                      0
+#define regCP_MEC_JT_STAT                                                                               0x1ed5
+#define regCP_MEC_JT_STAT_BASE_IDX                                                                      0
+#define regCP_CPC_BUSY_HYSTERESIS                                                                       0x1edb
+#define regCP_CPC_BUSY_HYSTERESIS_BASE_IDX                                                              0
+#define regCP_CPF_BUSY_HYSTERESIS1                                                                      0x1edc
+#define regCP_CPF_BUSY_HYSTERESIS1_BASE_IDX                                                             0
+#define regCP_CPF_BUSY_HYSTERESIS2                                                                      0x1edd
+#define regCP_CPF_BUSY_HYSTERESIS2_BASE_IDX                                                             0
+#define regCP_CPG_BUSY_HYSTERESIS1                                                                      0x1ede
+#define regCP_CPG_BUSY_HYSTERESIS1_BASE_IDX                                                             0
+#define regCP_CPG_BUSY_HYSTERESIS2                                                                      0x1edf
+#define regCP_CPG_BUSY_HYSTERESIS2_BASE_IDX                                                             0
+#define regCPF_FED_ERROR_ADDR_LO                                                                        0x1ee0
+#define regCPF_FED_ERROR_ADDR_LO_BASE_IDX                                                               0
+#define regCPF_FED_ERROR_ADDR_HI                                                                        0x1ee1
+#define regCPF_FED_ERROR_ADDR_HI_BASE_IDX                                                               0
+#define regCPC_FED_ERROR_ADDR_LO                                                                        0x1ee2
+#define regCPC_FED_ERROR_ADDR_LO_BASE_IDX                                                               0
+#define regCPC_FED_ERROR_ADDR_HI                                                                        0x1ee3
+#define regCPC_FED_ERROR_ADDR_HI_BASE_IDX                                                               0
+#define regCPG_FED_ERROR_ADDR_LO                                                                        0x1ee4
+#define regCPG_FED_ERROR_ADDR_LO_BASE_IDX                                                               0
+#define regCPG_FED_ERROR_ADDR_HI                                                                        0x1ee5
+#define regCPG_FED_ERROR_ADDR_HI_BASE_IDX                                                               0
+#define regCP_RB_DOORBELL_CLEAR                                                                         0x1f28
+#define regCP_RB_DOORBELL_CLEAR_BASE_IDX                                                                0
+#define regCP_RB0_ACTIVE                                                                                0x1f40
+#define regCP_RB0_ACTIVE_BASE_IDX                                                                       0
+#define regCP_RB_ACTIVE                                                                                 0x1f40
+#define regCP_RB_ACTIVE_BASE_IDX                                                                        0
+#define regCP_RB_STATUS                                                                                 0x1f43
+#define regCP_RB_STATUS_BASE_IDX                                                                        0
+#define regCPG_RCIU_CAM_INDEX                                                                           0x1f44
+#define regCPG_RCIU_CAM_INDEX_BASE_IDX                                                                  0
+#define regCPG_RCIU_CAM_DATA                                                                            0x1f45
+#define regCPG_RCIU_CAM_DATA_BASE_IDX                                                                   0
+#define regCPG_RCIU_CAM_DATA_PHASE0                                                                     0x1f45
+#define regCPG_RCIU_CAM_DATA_PHASE0_BASE_IDX                                                            0
+#define regCPG_RCIU_CAM_DATA_PHASE1                                                                     0x1f45
+#define regCPG_RCIU_CAM_DATA_PHASE1_BASE_IDX                                                            0
+#define regCPG_RCIU_CAM_DATA_PHASE2                                                                     0x1f45
+#define regCPG_RCIU_CAM_DATA_PHASE2_BASE_IDX                                                            0
+#define regCPG_RCIU_CAM_DATA_PHASE3                                                                     0x1f45
+#define regCPG_RCIU_CAM_DATA_PHASE3_BASE_IDX                                                            0
+#define regCP_GPU_TIMESTAMP_OFFSET_LO                                                                   0x1f4c
+#define regCP_GPU_TIMESTAMP_OFFSET_LO_BASE_IDX                                                          0
+#define regCP_GPU_TIMESTAMP_OFFSET_HI                                                                   0x1f4d
+#define regCP_GPU_TIMESTAMP_OFFSET_HI_BASE_IDX                                                          0
+#define regCP_SDMA_DMA_DONE                                                                             0x1f4e
+#define regCP_SDMA_DMA_DONE_BASE_IDX                                                                    0
+#define regCP_PFP_SDMA_CS                                                                               0x1f4f
+#define regCP_PFP_SDMA_CS_BASE_IDX                                                                      0
+#define regCP_ME_SDMA_CS                                                                                0x1f50
+#define regCP_ME_SDMA_CS_BASE_IDX                                                                       0
+#define regCPF_GCR_CNTL                                                                                 0x1f53
+#define regCPF_GCR_CNTL_BASE_IDX                                                                        0
+#define regCPG_UTCL1_STATUS                                                                             0x1f54
+#define regCPG_UTCL1_STATUS_BASE_IDX                                                                    0
+#define regCPC_UTCL1_STATUS                                                                             0x1f55
+#define regCPC_UTCL1_STATUS_BASE_IDX                                                                    0
+#define regCPF_UTCL1_STATUS                                                                             0x1f56
+#define regCPF_UTCL1_STATUS_BASE_IDX                                                                    0
+#define regCP_SD_CNTL                                                                                   0x1f57
+#define regCP_SD_CNTL_BASE_IDX                                                                          0
+#define regCP_SOFT_RESET_CNTL                                                                           0x1f59
+#define regCP_SOFT_RESET_CNTL_BASE_IDX                                                                  0
+#define regCP_CPC_GFX_CNTL                                                                              0x1f5a
+#define regCP_CPC_GFX_CNTL_BASE_IDX                                                                     0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_cpphqddec
+// base address: 0xc800
+#define regCP_HPD_UTCL1_CNTL                                                                            0x1fa3
+#define regCP_HPD_UTCL1_CNTL_BASE_IDX                                                                   0
+#define regCP_HPD_UTCL1_ERROR                                                                           0x1fa7
+#define regCP_HPD_UTCL1_ERROR_BASE_IDX                                                                  0
+#define regCP_HPD_UTCL1_ERROR_ADDR                                                                      0x1fa8
+#define regCP_HPD_UTCL1_ERROR_ADDR_BASE_IDX                                                             0
+#define regCP_MQD_BASE_ADDR                                                                             0x1fa9
+#define regCP_MQD_BASE_ADDR_BASE_IDX                                                                    0
+#define regCP_MQD_BASE_ADDR_HI                                                                          0x1faa
+#define regCP_MQD_BASE_ADDR_HI_BASE_IDX                                                                 0
+#define regCP_HQD_ACTIVE                                                                                0x1fab
+#define regCP_HQD_ACTIVE_BASE_IDX                                                                       0
+#define regCP_HQD_VMID                                                                                  0x1fac
+#define regCP_HQD_VMID_BASE_IDX                                                                         0
+#define regCP_HQD_PERSISTENT_STATE                                                                      0x1fad
+#define regCP_HQD_PERSISTENT_STATE_BASE_IDX                                                             0
+#define regCP_HQD_PIPE_PRIORITY                                                                         0x1fae
+#define regCP_HQD_PIPE_PRIORITY_BASE_IDX                                                                0
+#define regCP_HQD_QUEUE_PRIORITY                                                                        0x1faf
+#define regCP_HQD_QUEUE_PRIORITY_BASE_IDX                                                               0
+#define regCP_HQD_QUANTUM                                                                               0x1fb0
+#define regCP_HQD_QUANTUM_BASE_IDX                                                                      0
+#define regCP_HQD_PQ_BASE                                                                               0x1fb1
+#define regCP_HQD_PQ_BASE_BASE_IDX                                                                      0
+#define regCP_HQD_PQ_BASE_HI                                                                            0x1fb2
+#define regCP_HQD_PQ_BASE_HI_BASE_IDX                                                                   0
+#define regCP_HQD_PQ_RPTR                                                                               0x1fb3
+#define regCP_HQD_PQ_RPTR_BASE_IDX                                                                      0
+#define regCP_HQD_PQ_RPTR_REPORT_ADDR                                                                   0x1fb4
+#define regCP_HQD_PQ_RPTR_REPORT_ADDR_BASE_IDX                                                          0
+#define regCP_HQD_PQ_RPTR_REPORT_ADDR_HI                                                                0x1fb5
+#define regCP_HQD_PQ_RPTR_REPORT_ADDR_HI_BASE_IDX                                                       0
+#define regCP_HQD_PQ_WPTR_POLL_ADDR                                                                     0x1fb6
+#define regCP_HQD_PQ_WPTR_POLL_ADDR_BASE_IDX                                                            0
+#define regCP_HQD_PQ_WPTR_POLL_ADDR_HI                                                                  0x1fb7
+#define regCP_HQD_PQ_WPTR_POLL_ADDR_HI_BASE_IDX                                                         0
+#define regCP_HQD_PQ_DOORBELL_CONTROL                                                                   0x1fb8
+#define regCP_HQD_PQ_DOORBELL_CONTROL_BASE_IDX                                                          0
+#define regCP_HQD_PQ_CONTROL                                                                            0x1fba
+#define regCP_HQD_PQ_CONTROL_BASE_IDX                                                                   0
+#define regCP_HQD_IB_BASE_ADDR                                                                          0x1fbb
+#define regCP_HQD_IB_BASE_ADDR_BASE_IDX                                                                 0
+#define regCP_HQD_IB_BASE_ADDR_HI                                                                       0x1fbc
+#define regCP_HQD_IB_BASE_ADDR_HI_BASE_IDX                                                              0
+#define regCP_HQD_IB_RPTR                                                                               0x1fbd
+#define regCP_HQD_IB_RPTR_BASE_IDX                                                                      0
+#define regCP_HQD_IB_CONTROL                                                                            0x1fbe
+#define regCP_HQD_IB_CONTROL_BASE_IDX                                                                   0
+#define regCP_HQD_IQ_TIMER                                                                              0x1fbf
+#define regCP_HQD_IQ_TIMER_BASE_IDX                                                                     0
+#define regCP_HQD_IQ_RPTR                                                                               0x1fc0
+#define regCP_HQD_IQ_RPTR_BASE_IDX                                                                      0
+#define regCP_HQD_DEQUEUE_REQUEST                                                                       0x1fc1
+#define regCP_HQD_DEQUEUE_REQUEST_BASE_IDX                                                              0
+#define regCP_HQD_DMA_OFFLOAD                                                                           0x1fc2
+#define regCP_HQD_DMA_OFFLOAD_BASE_IDX                                                                  0
+#define regCP_HQD_OFFLOAD                                                                               0x1fc2
+#define regCP_HQD_OFFLOAD_BASE_IDX                                                                      0
+#define regCP_HQD_MSG_TYPE                                                                              0x1fc4
+#define regCP_HQD_MSG_TYPE_BASE_IDX                                                                     0
+#define regCP_HQD_ATOMIC0_PREOP_LO                                                                      0x1fc5
+#define regCP_HQD_ATOMIC0_PREOP_LO_BASE_IDX                                                             0
+#define regCP_HQD_ATOMIC0_PREOP_HI                                                                      0x1fc6
+#define regCP_HQD_ATOMIC0_PREOP_HI_BASE_IDX                                                             0
+#define regCP_HQD_ATOMIC1_PREOP_LO                                                                      0x1fc7
+#define regCP_HQD_ATOMIC1_PREOP_LO_BASE_IDX                                                             0
+#define regCP_HQD_ATOMIC1_PREOP_HI                                                                      0x1fc8
+#define regCP_HQD_ATOMIC1_PREOP_HI_BASE_IDX                                                             0
+#define regCP_HQD_HQ_SCHEDULER0                                                                         0x1fc9
+#define regCP_HQD_HQ_SCHEDULER0_BASE_IDX                                                                0
+#define regCP_HQD_HQ_STATUS0                                                                            0x1fc9
+#define regCP_HQD_HQ_STATUS0_BASE_IDX                                                                   0
+#define regCP_HQD_HQ_CONTROL0                                                                           0x1fca
+#define regCP_HQD_HQ_CONTROL0_BASE_IDX                                                                  0
+#define regCP_HQD_HQ_SCHEDULER1                                                                         0x1fca
+#define regCP_HQD_HQ_SCHEDULER1_BASE_IDX                                                                0
+#define regCP_MQD_CONTROL                                                                               0x1fcb
+#define regCP_MQD_CONTROL_BASE_IDX                                                                      0
+#define regCP_HQD_HQ_STATUS1                                                                            0x1fcc
+#define regCP_HQD_HQ_STATUS1_BASE_IDX                                                                   0
+#define regCP_HQD_HQ_CONTROL1                                                                           0x1fcd
+#define regCP_HQD_HQ_CONTROL1_BASE_IDX                                                                  0
+#define regCP_HQD_EOP_BASE_ADDR                                                                         0x1fce
+#define regCP_HQD_EOP_BASE_ADDR_BASE_IDX                                                                0
+#define regCP_HQD_EOP_BASE_ADDR_HI                                                                      0x1fcf
+#define regCP_HQD_EOP_BASE_ADDR_HI_BASE_IDX                                                             0
+#define regCP_HQD_EOP_CONTROL                                                                           0x1fd0
+#define regCP_HQD_EOP_CONTROL_BASE_IDX                                                                  0
+#define regCP_HQD_EOP_RPTR                                                                              0x1fd1
+#define regCP_HQD_EOP_RPTR_BASE_IDX                                                                     0
+#define regCP_HQD_EOP_WPTR                                                                              0x1fd2
+#define regCP_HQD_EOP_WPTR_BASE_IDX                                                                     0
+#define regCP_HQD_EOP_EVENTS                                                                            0x1fd3
+#define regCP_HQD_EOP_EVENTS_BASE_IDX                                                                   0
+#define regCP_HQD_CTX_SAVE_BASE_ADDR_LO                                                                 0x1fd4
+#define regCP_HQD_CTX_SAVE_BASE_ADDR_LO_BASE_IDX                                                        0
+#define regCP_HQD_CTX_SAVE_BASE_ADDR_HI                                                                 0x1fd5
+#define regCP_HQD_CTX_SAVE_BASE_ADDR_HI_BASE_IDX                                                        0
+#define regCP_HQD_CTX_SAVE_CONTROL                                                                      0x1fd6
+#define regCP_HQD_CTX_SAVE_CONTROL_BASE_IDX                                                             0
+#define regCP_HQD_CNTL_STACK_OFFSET                                                                     0x1fd7
+#define regCP_HQD_CNTL_STACK_OFFSET_BASE_IDX                                                            0
+#define regCP_HQD_CNTL_STACK_SIZE                                                                       0x1fd8
+#define regCP_HQD_CNTL_STACK_SIZE_BASE_IDX                                                              0
+#define regCP_HQD_WG_STATE_OFFSET                                                                       0x1fd9
+#define regCP_HQD_WG_STATE_OFFSET_BASE_IDX                                                              0
+#define regCP_HQD_CTX_SAVE_SIZE                                                                         0x1fda
+#define regCP_HQD_CTX_SAVE_SIZE_BASE_IDX                                                                0
+#define regCP_HQD_GDS_RESOURCE_STATE                                                                    0x1fdb
+#define regCP_HQD_GDS_RESOURCE_STATE_BASE_IDX                                                           0
+#define regCP_HQD_ERROR                                                                                 0x1fdc
+#define regCP_HQD_ERROR_BASE_IDX                                                                        0
+#define regCP_HQD_EOP_WPTR_MEM                                                                          0x1fdd
+#define regCP_HQD_EOP_WPTR_MEM_BASE_IDX                                                                 0
+#define regCP_HQD_AQL_CONTROL                                                                           0x1fde
+#define regCP_HQD_AQL_CONTROL_BASE_IDX                                                                  0
+#define regCP_HQD_PQ_WPTR_LO                                                                            0x1fdf
+#define regCP_HQD_PQ_WPTR_LO_BASE_IDX                                                                   0
+#define regCP_HQD_PQ_WPTR_HI                                                                            0x1fe0
+#define regCP_HQD_PQ_WPTR_HI_BASE_IDX                                                                   0
+#define regCP_HQD_SUSPEND_CNTL_STACK_OFFSET                                                             0x1fe1
+#define regCP_HQD_SUSPEND_CNTL_STACK_OFFSET_BASE_IDX                                                    0
+#define regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT                                                             0x1fe2
+#define regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT_BASE_IDX                                                    0
+#define regCP_HQD_SUSPEND_WG_STATE_OFFSET                                                               0x1fe3
+#define regCP_HQD_SUSPEND_WG_STATE_OFFSET_BASE_IDX                                                      0
+#define regCP_HQD_DDID_RPTR                                                                             0x1fe4
+#define regCP_HQD_DDID_RPTR_BASE_IDX                                                                    0
+#define regCP_HQD_DDID_WPTR                                                                             0x1fe5
+#define regCP_HQD_DDID_WPTR_BASE_IDX                                                                    0
+#define regCP_HQD_DDID_INFLIGHT_COUNT                                                                   0x1fe6
+#define regCP_HQD_DDID_INFLIGHT_COUNT_BASE_IDX                                                          0
+#define regCP_HQD_DDID_DELTA_RPT_COUNT                                                                  0x1fe7
+#define regCP_HQD_DDID_DELTA_RPT_COUNT_BASE_IDX                                                         0
+#define regCP_HQD_DEQUEUE_STATUS                                                                        0x1fe8
+#define regCP_HQD_DEQUEUE_STATUS_BASE_IDX                                                               0
+#define regCP_HQD_AQL_CONTROL_1                                                                         0x1fe9
+#define regCP_HQD_AQL_CONTROL_1_BASE_IDX                                                                0
+#define regCP_HQD_AQL_DISPATCH_ID                                                                       0x1fea
+#define regCP_HQD_AQL_DISPATCH_ID_BASE_IDX                                                              0
+#define regCP_HQD_AQL_DISPATCH_ID_HI                                                                    0x1feb
+#define regCP_HQD_AQL_DISPATCH_ID_HI_BASE_IDX                                                           0
+#define regCP_HQD_KD_BASE                                                                               0x1fec
+#define regCP_HQD_KD_BASE_BASE_IDX                                                                      0
+#define regCP_HQD_KD_BASE_HI                                                                            0x1fed
+#define regCP_HQD_KD_BASE_HI_BASE_IDX                                                                   0
+#define regCP_HQD_KD_CNTL                                                                               0x1fee
+#define regCP_HQD_KD_CNTL_BASE_IDX                                                                      0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_gfxdec0
+// base address: 0x28000
+#define regCOHER_DEST_BASE_HI_0                                                                         0x007a
+#define regCOHER_DEST_BASE_HI_0_BASE_IDX                                                                1
+#define regCOHER_DEST_BASE_HI_1                                                                         0x007b
+#define regCOHER_DEST_BASE_HI_1_BASE_IDX                                                                1
+#define regCOHER_DEST_BASE_HI_2                                                                         0x007c
+#define regCOHER_DEST_BASE_HI_2_BASE_IDX                                                                1
+#define regCOHER_DEST_BASE_HI_3                                                                         0x007d
+#define regCOHER_DEST_BASE_HI_3_BASE_IDX                                                                1
+#define regCOHER_DEST_BASE_2                                                                            0x007e
+#define regCOHER_DEST_BASE_2_BASE_IDX                                                                   1
+#define regCOHER_DEST_BASE_3                                                                            0x007f
+#define regCOHER_DEST_BASE_3_BASE_IDX                                                                   1
+#define regCOHER_DEST_BASE_0                                                                            0x0092
+#define regCOHER_DEST_BASE_0_BASE_IDX                                                                   1
+#define regCOHER_DEST_BASE_1                                                                            0x0093
+#define regCOHER_DEST_BASE_1_BASE_IDX                                                                   1
+#define regCP_PERFMON_CNTX_CNTL                                                                         0x00d8
+#define regCP_PERFMON_CNTX_CNTL_BASE_IDX                                                                1
+#define regCP_PIPEID                                                                                    0x00d9
+#define regCP_PIPEID_BASE_IDX                                                                           1
+#define regCP_RINGID                                                                                    0x00d9
+#define regCP_RINGID_BASE_IDX                                                                           1
+#define regCP_VMID                                                                                      0x00da
+#define regCP_VMID_BASE_IDX                                                                             1
+#define regCONTEXT_RESERVED_REG0                                                                        0x00db
+#define regCONTEXT_RESERVED_REG0_BASE_IDX                                                               1
+#define regCONTEXT_RESERVED_REG1                                                                        0x00dc
+#define regCONTEXT_RESERVED_REG1_BASE_IDX                                                               1
+#define regVGT_MULTI_PRIM_IB_RESET_INDX                                                                 0x0103
+#define regVGT_MULTI_PRIM_IB_RESET_INDX_BASE_IDX                                                        1
+#define regGFX_COPY_STATE                                                                               0x01f4
+#define regGFX_COPY_STATE_BASE_IDX                                                                      1
+#define regVGT_DMA_BASE_HI                                                                              0x01f9
+#define regVGT_DMA_BASE_HI_BASE_IDX                                                                     1
+#define regVGT_DMA_BASE                                                                                 0x01fa
+#define regVGT_DMA_BASE_BASE_IDX                                                                        1
+#define regVGT_DRAW_INITIATOR                                                                           0x01fc
+#define regVGT_DRAW_INITIATOR_BASE_IDX                                                                  1
+#define regVGT_EVENT_ADDRESS_REG                                                                        0x01fe
+#define regVGT_EVENT_ADDRESS_REG_BASE_IDX                                                               1
+#define regVGT_HOS_MAX_TESS_LEVEL                                                                       0x0286
+#define regVGT_HOS_MAX_TESS_LEVEL_BASE_IDX                                                              1
+#define regVGT_HOS_MIN_TESS_LEVEL                                                                       0x0287
+#define regVGT_HOS_MIN_TESS_LEVEL_BASE_IDX                                                              1
+#define regGE_IA_ENHANCE                                                                                0x029c
+#define regGE_IA_ENHANCE_BASE_IDX                                                                       1
+#define regVGT_DMA_SIZE                                                                                 0x029d
+#define regVGT_DMA_SIZE_BASE_IDX                                                                        1
+#define regVGT_DMA_MAX_SIZE                                                                             0x029e
+#define regVGT_DMA_MAX_SIZE_BASE_IDX                                                                    1
+#define regVGT_DMA_INDEX_TYPE                                                                           0x029f
+#define regVGT_DMA_INDEX_TYPE_BASE_IDX                                                                  1
+#define regGE_WD_ENHANCE                                                                                0x02a0
+#define regGE_WD_ENHANCE_BASE_IDX                                                                       1
+#define regVGT_DMA_NUM_INSTANCES                                                                        0x02a2
+#define regVGT_DMA_NUM_INSTANCES_BASE_IDX                                                               1
+#define regVGT_EVENT_INITIATOR                                                                          0x02a4
+#define regVGT_EVENT_INITIATOR_BASE_IDX                                                                 1
+#define regVGT_SHADER_STAGES_EN                                                                         0x02a6
+#define regVGT_SHADER_STAGES_EN_BASE_IDX                                                                1
+#define regVGT_TF_PARAM                                                                                 0x02a9
+#define regVGT_TF_PARAM_BASE_IDX                                                                        1
+#define regVGT_STRMOUT_DRAW_OPAQUE_OFFSET                                                               0x02ca
+#define regVGT_STRMOUT_DRAW_OPAQUE_OFFSET_BASE_IDX                                                      1
+#define regVGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE                                                   0x02cb
+#define regVGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE_BASE_IDX                                          1
+#define regVGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE                                                        0x02cc
+#define regVGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE_BASE_IDX                                               1
+#define regVGT_TESS_DISTRIBUTION                                                                        0x02d4
+#define regVGT_TESS_DISTRIBUTION_BASE_IDX                                                               1
+#define regVGT_LS_HS_CONFIG                                                                             0x02d6
+#define regVGT_LS_HS_CONFIG_BASE_IDX                                                                    1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pfvf_cpdec
+// base address: 0x2a000
+#define regCONFIG_RESERVED_REG0                                                                         0x0800
+#define regCONFIG_RESERVED_REG0_BASE_IDX                                                                1
+#define regCONFIG_RESERVED_REG1                                                                         0x0801
+#define regCONFIG_RESERVED_REG1_BASE_IDX                                                                1
+#define regCP_MEC_CNTL                                                                                  0x0802
+#define regCP_MEC_CNTL_BASE_IDX                                                                         1
+#define regCP_ME_CNTL                                                                                   0x0803
+#define regCP_ME_CNTL_BASE_IDX                                                                          1
+#define regCP_UNMAPPED_QUEUE0                                                                           0x0840
+#define regCP_UNMAPPED_QUEUE0_BASE_IDX                                                                  1
+#define regCP_UNMAPPED_QUEUE1                                                                           0x0841
+#define regCP_UNMAPPED_QUEUE1_BASE_IDX                                                                  1
+#define regCP_UNMAPPED_QUEUE2                                                                           0x0842
+#define regCP_UNMAPPED_QUEUE2_BASE_IDX                                                                  1
+#define regCP_UNMAPPED_QUEUE3                                                                           0x0843
+#define regCP_UNMAPPED_QUEUE3_BASE_IDX                                                                  1
+#define regCP_UNMAPPED_QUEUE4                                                                           0x0844
+#define regCP_UNMAPPED_QUEUE4_BASE_IDX                                                                  1
+#define regCP_UNMAPPED_QUEUE5                                                                           0x0845
+#define regCP_UNMAPPED_QUEUE5_BASE_IDX                                                                  1
+#define regCP_UNMAPPED_QUEUE6                                                                           0x0846
+#define regCP_UNMAPPED_QUEUE6_BASE_IDX                                                                  1
+#define regCP_UNMAPPED_QUEUE7                                                                           0x0847
+#define regCP_UNMAPPED_QUEUE7_BASE_IDX                                                                  1
+#define regCP_UNMAPPED_QUEUE8                                                                           0x0848
+#define regCP_UNMAPPED_QUEUE8_BASE_IDX                                                                  1
+#define regCP_UNMAPPED_QUEUE9                                                                           0x0849
+#define regCP_UNMAPPED_QUEUE9_BASE_IDX                                                                  1
+#define regCP_UNMAPPED_QUEUE10                                                                          0x084a
+#define regCP_UNMAPPED_QUEUE10_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE11                                                                          0x084b
+#define regCP_UNMAPPED_QUEUE11_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE12                                                                          0x084c
+#define regCP_UNMAPPED_QUEUE12_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE13                                                                          0x084d
+#define regCP_UNMAPPED_QUEUE13_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE14                                                                          0x084e
+#define regCP_UNMAPPED_QUEUE14_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE15                                                                          0x084f
+#define regCP_UNMAPPED_QUEUE15_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE16                                                                          0x0850
+#define regCP_UNMAPPED_QUEUE16_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE17                                                                          0x0851
+#define regCP_UNMAPPED_QUEUE17_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE18                                                                          0x0852
+#define regCP_UNMAPPED_QUEUE18_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE19                                                                          0x0853
+#define regCP_UNMAPPED_QUEUE19_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE20                                                                          0x0854
+#define regCP_UNMAPPED_QUEUE20_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE21                                                                          0x0855
+#define regCP_UNMAPPED_QUEUE21_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE22                                                                          0x0856
+#define regCP_UNMAPPED_QUEUE22_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE23                                                                          0x0857
+#define regCP_UNMAPPED_QUEUE23_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE24                                                                          0x0858
+#define regCP_UNMAPPED_QUEUE24_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE25                                                                          0x0859
+#define regCP_UNMAPPED_QUEUE25_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE26                                                                          0x085a
+#define regCP_UNMAPPED_QUEUE26_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE27                                                                          0x085b
+#define regCP_UNMAPPED_QUEUE27_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE28                                                                          0x085c
+#define regCP_UNMAPPED_QUEUE28_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE29                                                                          0x085d
+#define regCP_UNMAPPED_QUEUE29_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE30                                                                          0x085e
+#define regCP_UNMAPPED_QUEUE30_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE31                                                                          0x085f
+#define regCP_UNMAPPED_QUEUE31_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE32                                                                          0x0860
+#define regCP_UNMAPPED_QUEUE32_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE33                                                                          0x0861
+#define regCP_UNMAPPED_QUEUE33_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE34                                                                          0x0862
+#define regCP_UNMAPPED_QUEUE34_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE35                                                                          0x0863
+#define regCP_UNMAPPED_QUEUE35_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE36                                                                          0x0864
+#define regCP_UNMAPPED_QUEUE36_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE37                                                                          0x0865
+#define regCP_UNMAPPED_QUEUE37_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE38                                                                          0x0866
+#define regCP_UNMAPPED_QUEUE38_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE39                                                                          0x0867
+#define regCP_UNMAPPED_QUEUE39_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE40                                                                          0x0868
+#define regCP_UNMAPPED_QUEUE40_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE41                                                                          0x0869
+#define regCP_UNMAPPED_QUEUE41_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE42                                                                          0x086a
+#define regCP_UNMAPPED_QUEUE42_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE43                                                                          0x086b
+#define regCP_UNMAPPED_QUEUE43_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE44                                                                          0x086c
+#define regCP_UNMAPPED_QUEUE44_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE45                                                                          0x086d
+#define regCP_UNMAPPED_QUEUE45_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE46                                                                          0x086e
+#define regCP_UNMAPPED_QUEUE46_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE47                                                                          0x086f
+#define regCP_UNMAPPED_QUEUE47_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE48                                                                          0x0870
+#define regCP_UNMAPPED_QUEUE48_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE49                                                                          0x0871
+#define regCP_UNMAPPED_QUEUE49_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE50                                                                          0x0872
+#define regCP_UNMAPPED_QUEUE50_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE51                                                                          0x0873
+#define regCP_UNMAPPED_QUEUE51_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE52                                                                          0x0874
+#define regCP_UNMAPPED_QUEUE52_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE53                                                                          0x0875
+#define regCP_UNMAPPED_QUEUE53_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE54                                                                          0x0876
+#define regCP_UNMAPPED_QUEUE54_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE55                                                                          0x0877
+#define regCP_UNMAPPED_QUEUE55_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE56                                                                          0x0878
+#define regCP_UNMAPPED_QUEUE56_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE57                                                                          0x0879
+#define regCP_UNMAPPED_QUEUE57_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE58                                                                          0x087a
+#define regCP_UNMAPPED_QUEUE58_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE59                                                                          0x087b
+#define regCP_UNMAPPED_QUEUE59_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE60                                                                          0x087c
+#define regCP_UNMAPPED_QUEUE60_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE61                                                                          0x087d
+#define regCP_UNMAPPED_QUEUE61_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE62                                                                          0x087e
+#define regCP_UNMAPPED_QUEUE62_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_QUEUE63                                                                          0x087f
+#define regCP_UNMAPPED_QUEUE63_BASE_IDX                                                                 1
+#define regCP_UNMAPPED_DOORBELL                                                                         0x0880
+#define regCP_UNMAPPED_DOORBELL_BASE_IDX                                                                1
+#define regCP_UNMAPPED_QUEUE_BANK0                                                                      0x0881
+#define regCP_UNMAPPED_QUEUE_BANK0_BASE_IDX                                                             1
+#define regCP_UNMAPPED_QUEUE_BANK1                                                                      0x0882
+#define regCP_UNMAPPED_QUEUE_BANK1_BASE_IDX                                                             1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pfvf_grbmdec
+// base address: 0x2a400
+#define regGRBM_GFX_CNTL                                                                                0x0900
+#define regGRBM_GFX_CNTL_BASE_IDX                                                                       1
+#define regGRBM_NOWHERE                                                                                 0x0901
+#define regGRBM_NOWHERE_BASE_IDX                                                                        1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pfonly_cpdec
+// base address: 0x2e000
+#define regCP_FETCHER_SOURCE                                                                            0x1801
+#define regCP_FETCHER_SOURCE_BASE_IDX                                                                   1
+#define regCP_DFY_CNTL                                                                                  0x1804
+#define regCP_DFY_CNTL_BASE_IDX                                                                         1
+#define regCP_DFY_STAT                                                                                  0x1805
+#define regCP_DFY_STAT_BASE_IDX                                                                         1
+#define regCP_DFY_ADDR_HI                                                                               0x1806
+#define regCP_DFY_ADDR_HI_BASE_IDX                                                                      1
+#define regCP_DFY_ADDR_LO                                                                               0x1807
+#define regCP_DFY_ADDR_LO_BASE_IDX                                                                      1
+#define regCP_DFY_DATA_0                                                                                0x1808
+#define regCP_DFY_DATA_0_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_1                                                                                0x1809
+#define regCP_DFY_DATA_1_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_2                                                                                0x180a
+#define regCP_DFY_DATA_2_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_3                                                                                0x180b
+#define regCP_DFY_DATA_3_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_4                                                                                0x180c
+#define regCP_DFY_DATA_4_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_5                                                                                0x180d
+#define regCP_DFY_DATA_5_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_6                                                                                0x180e
+#define regCP_DFY_DATA_6_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_7                                                                                0x180f
+#define regCP_DFY_DATA_7_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_8                                                                                0x1810
+#define regCP_DFY_DATA_8_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_9                                                                                0x1811
+#define regCP_DFY_DATA_9_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_10                                                                               0x1812
+#define regCP_DFY_DATA_10_BASE_IDX                                                                      1
+#define regCP_DFY_DATA_11                                                                               0x1813
+#define regCP_DFY_DATA_11_BASE_IDX                                                                      1
+#define regCP_DFY_DATA_12                                                                               0x1814
+#define regCP_DFY_DATA_12_BASE_IDX                                                                      1
+#define regCP_DFY_DATA_13                                                                               0x1815
+#define regCP_DFY_DATA_13_BASE_IDX                                                                      1
+#define regCP_DFY_DATA_14                                                                               0x1816
+#define regCP_DFY_DATA_14_BASE_IDX                                                                      1
+#define regCP_DFY_DATA_15                                                                               0x1817
+#define regCP_DFY_DATA_15_BASE_IDX                                                                      1
+#define regCP_DFY_CMD                                                                                   0x1818
+#define regCP_DFY_CMD_BASE_IDX                                                                          1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pfonly_cpphqddec
+// base address: 0x2e080
+#define regCP_HPD_MES_ROQ_OFFSETS                                                                       0x1821
+#define regCP_HPD_MES_ROQ_OFFSETS_BASE_IDX                                                              1
+#define regCP_HPD_ROQ_OFFSETS                                                                           0x1821
+#define regCP_HPD_ROQ_OFFSETS_BASE_IDX                                                                  1
+#define regCP_HPD_STATUS0                                                                               0x1822
+#define regCP_HPD_STATUS0_BASE_IDX                                                                      1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pfonly_gcrdec
+// base address: 0x2e640
+#define regGCR_GENERAL_CNTL                                                                             0x1990
+#define regGCR_GENERAL_CNTL_BASE_IDX                                                                    1
+#define regGCR_TARGET_DISABLE                                                                           0x1991
+#define regGCR_TARGET_DISABLE_BASE_IDX                                                                  1
+#define regGCR_CMD_STATUS                                                                               0x1992
+#define regGCR_CMD_STATUS_BASE_IDX                                                                      1
+#define regGCR_SPARE                                                                                    0x1993
+#define regGCR_SPARE_BASE_IDX                                                                           1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_gfxudec
+// base address: 0x30000
+#define regCP_EOP_DONE_ADDR_LO                                                                          0x2000
+#define regCP_EOP_DONE_ADDR_LO_BASE_IDX                                                                 1
+#define regCP_EOP_DONE_ADDR_HI                                                                          0x2001
+#define regCP_EOP_DONE_ADDR_HI_BASE_IDX                                                                 1
+#define regCP_EOP_DONE_DATA_LO                                                                          0x2002
+#define regCP_EOP_DONE_DATA_LO_BASE_IDX                                                                 1
+#define regCP_EOP_DONE_DATA_HI                                                                          0x2003
+#define regCP_EOP_DONE_DATA_HI_BASE_IDX                                                                 1
+#define regCP_EOP_LAST_FENCE_LO                                                                         0x2004
+#define regCP_EOP_LAST_FENCE_LO_BASE_IDX                                                                1
+#define regCP_EOP_LAST_FENCE_HI                                                                         0x2005
+#define regCP_EOP_LAST_FENCE_HI_BASE_IDX                                                                1
+#define regCP_PIPE_STATS_ADDR_LO                                                                        0x2018
+#define regCP_PIPE_STATS_ADDR_LO_BASE_IDX                                                               1
+#define regCP_PIPE_STATS_ADDR_HI                                                                        0x2019
+#define regCP_PIPE_STATS_ADDR_HI_BASE_IDX                                                               1
+#define regCP_VGT_IAVERT_COUNT_LO                                                                       0x201a
+#define regCP_VGT_IAVERT_COUNT_LO_BASE_IDX                                                              1
+#define regCP_VGT_IAVERT_COUNT_HI                                                                       0x201b
+#define regCP_VGT_IAVERT_COUNT_HI_BASE_IDX                                                              1
+#define regCP_VGT_IAPRIM_COUNT_LO                                                                       0x201c
+#define regCP_VGT_IAPRIM_COUNT_LO_BASE_IDX                                                              1
+#define regCP_VGT_IAPRIM_COUNT_HI                                                                       0x201d
+#define regCP_VGT_IAPRIM_COUNT_HI_BASE_IDX                                                              1
+#define regCP_VGT_GSPRIM_COUNT_LO                                                                       0x201e
+#define regCP_VGT_GSPRIM_COUNT_LO_BASE_IDX                                                              1
+#define regCP_VGT_GSPRIM_COUNT_HI                                                                       0x201f
+#define regCP_VGT_GSPRIM_COUNT_HI_BASE_IDX                                                              1
+#define regCP_VGT_VSINVOC_COUNT_LO                                                                      0x2020
+#define regCP_VGT_VSINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_VSINVOC_COUNT_HI                                                                      0x2021
+#define regCP_VGT_VSINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_VGT_GSINVOC_COUNT_LO                                                                      0x2022
+#define regCP_VGT_GSINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_GSINVOC_COUNT_HI                                                                      0x2023
+#define regCP_VGT_GSINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_VGT_HSINVOC_COUNT_LO                                                                      0x2024
+#define regCP_VGT_HSINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_HSINVOC_COUNT_HI                                                                      0x2025
+#define regCP_VGT_HSINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_VGT_DSINVOC_COUNT_LO                                                                      0x2026
+#define regCP_VGT_DSINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_DSINVOC_COUNT_HI                                                                      0x2027
+#define regCP_VGT_DSINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_PA_CINVOC_COUNT_LO                                                                        0x2028
+#define regCP_PA_CINVOC_COUNT_LO_BASE_IDX                                                               1
+#define regCP_PA_CINVOC_COUNT_HI                                                                        0x2029
+#define regCP_PA_CINVOC_COUNT_HI_BASE_IDX                                                               1
+#define regCP_PA_CPRIM_COUNT_LO                                                                         0x202a
+#define regCP_PA_CPRIM_COUNT_LO_BASE_IDX                                                                1
+#define regCP_PA_CPRIM_COUNT_HI                                                                         0x202b
+#define regCP_PA_CPRIM_COUNT_HI_BASE_IDX                                                                1
+#define regCP_SC_PSINVOC_COUNT0_LO                                                                      0x202c
+#define regCP_SC_PSINVOC_COUNT0_LO_BASE_IDX                                                             1
+#define regCP_SC_PSINVOC_COUNT0_HI                                                                      0x202d
+#define regCP_SC_PSINVOC_COUNT0_HI_BASE_IDX                                                             1
+#define regCP_SC_PSINVOC_COUNT1_LO                                                                      0x202e
+#define regCP_SC_PSINVOC_COUNT1_LO_BASE_IDX                                                             1
+#define regCP_SC_PSINVOC_COUNT1_HI                                                                      0x202f
+#define regCP_SC_PSINVOC_COUNT1_HI_BASE_IDX                                                             1
+#define regCP_VGT_CSINVOC_COUNT_LO                                                                      0x2030
+#define regCP_VGT_CSINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_CSINVOC_COUNT_HI                                                                      0x2031
+#define regCP_VGT_CSINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_VGT_ASINVOC_COUNT_LO                                                                      0x2032
+#define regCP_VGT_ASINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_ASINVOC_COUNT_HI                                                                      0x2033
+#define regCP_VGT_ASINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_PIPE_STATS_CONTROL                                                                        0x203d
+#define regCP_PIPE_STATS_CONTROL_BASE_IDX                                                               1
+#define regSCRATCH_REG0                                                                                 0x2040
+#define regSCRATCH_REG0_BASE_IDX                                                                        1
+#define regSCRATCH_REG1                                                                                 0x2041
+#define regSCRATCH_REG1_BASE_IDX                                                                        1
+#define regSCRATCH_REG2                                                                                 0x2042
+#define regSCRATCH_REG2_BASE_IDX                                                                        1
+#define regSCRATCH_REG3                                                                                 0x2043
+#define regSCRATCH_REG3_BASE_IDX                                                                        1
+#define regSCRATCH_REG4                                                                                 0x2044
+#define regSCRATCH_REG4_BASE_IDX                                                                        1
+#define regSCRATCH_REG5                                                                                 0x2045
+#define regSCRATCH_REG5_BASE_IDX                                                                        1
+#define regSCRATCH_REG6                                                                                 0x2046
+#define regSCRATCH_REG6_BASE_IDX                                                                        1
+#define regSCRATCH_REG7                                                                                 0x2047
+#define regSCRATCH_REG7_BASE_IDX                                                                        1
+#define regSCRATCH_REG_ATOMIC                                                                           0x2048
+#define regSCRATCH_REG_ATOMIC_BASE_IDX                                                                  1
+#define regSCRATCH_REG_CMPSWAP_ATOMIC                                                                   0x2048
+#define regSCRATCH_REG_CMPSWAP_ATOMIC_BASE_IDX                                                          1
+#define regCP_APPEND_DDID_CNT                                                                           0x204b
+#define regCP_APPEND_DDID_CNT_BASE_IDX                                                                  1
+#define regCP_APPEND_DATA_HI                                                                            0x204c
+#define regCP_APPEND_DATA_HI_BASE_IDX                                                                   1
+#define regCP_APPEND_LAST_CS_FENCE_HI                                                                   0x204d
+#define regCP_APPEND_LAST_CS_FENCE_HI_BASE_IDX                                                          1
+#define regCP_APPEND_LAST_PS_FENCE_HI                                                                   0x204e
+#define regCP_APPEND_LAST_PS_FENCE_HI_BASE_IDX                                                          1
+#define regCP_PFP_ATOMIC_PREOP_LO                                                                       0x2052
+#define regCP_PFP_ATOMIC_PREOP_LO_BASE_IDX                                                              1
+#define regCP_PFP_ATOMIC_PREOP_HI                                                                       0x2053
+#define regCP_PFP_ATOMIC_PREOP_HI_BASE_IDX                                                              1
+#define regCP_APPEND_ADDR_LO                                                                            0x2058
+#define regCP_APPEND_ADDR_LO_BASE_IDX                                                                   1
+#define regCP_APPEND_ADDR_HI                                                                            0x2059
+#define regCP_APPEND_ADDR_HI_BASE_IDX                                                                   1
+#define regCP_APPEND_DATA                                                                               0x205a
+#define regCP_APPEND_DATA_BASE_IDX                                                                      1
+#define regCP_APPEND_DATA_LO                                                                            0x205a
+#define regCP_APPEND_DATA_LO_BASE_IDX                                                                   1
+#define regCP_APPEND_LAST_CS_FENCE                                                                      0x205b
+#define regCP_APPEND_LAST_CS_FENCE_BASE_IDX                                                             1
+#define regCP_APPEND_LAST_CS_FENCE_LO                                                                   0x205b
+#define regCP_APPEND_LAST_CS_FENCE_LO_BASE_IDX                                                          1
+#define regCP_APPEND_LAST_PS_FENCE                                                                      0x205c
+#define regCP_APPEND_LAST_PS_FENCE_BASE_IDX                                                             1
+#define regCP_APPEND_LAST_PS_FENCE_LO                                                                   0x205c
+#define regCP_APPEND_LAST_PS_FENCE_LO_BASE_IDX                                                          1
+#define regCP_ATOMIC_PREOP_LO                                                                           0x205d
+#define regCP_ATOMIC_PREOP_LO_BASE_IDX                                                                  1
+#define regCP_ME_ATOMIC_PREOP_LO                                                                        0x205d
+#define regCP_ME_ATOMIC_PREOP_LO_BASE_IDX                                                               1
+#define regCP_ATOMIC_PREOP_HI                                                                           0x205e
+#define regCP_ATOMIC_PREOP_HI_BASE_IDX                                                                  1
+#define regCP_ME_ATOMIC_PREOP_HI                                                                        0x205e
+#define regCP_ME_ATOMIC_PREOP_HI_BASE_IDX                                                               1
+#define regCP_ME_MC_WADDR_LO                                                                            0x2069
+#define regCP_ME_MC_WADDR_LO_BASE_IDX                                                                   1
+#define regCP_ME_MC_WADDR_HI                                                                            0x206a
+#define regCP_ME_MC_WADDR_HI_BASE_IDX                                                                   1
+#define regCP_ME_MC_WDATA_LO                                                                            0x206b
+#define regCP_ME_MC_WDATA_LO_BASE_IDX                                                                   1
+#define regCP_ME_MC_WDATA_HI                                                                            0x206c
+#define regCP_ME_MC_WDATA_HI_BASE_IDX                                                                   1
+#define regCP_ME_MC_RADDR_LO                                                                            0x206d
+#define regCP_ME_MC_RADDR_LO_BASE_IDX                                                                   1
+#define regCP_ME_MC_RADDR_HI                                                                            0x206e
+#define regCP_ME_MC_RADDR_HI_BASE_IDX                                                                   1
+#define regCP_WAIT_REG_MEM_TIMEOUT                                                                      0x2074
+#define regCP_WAIT_REG_MEM_TIMEOUT_BASE_IDX                                                             1
+#define regCP_DMA_PFP_CONTROL                                                                           0x2077
+#define regCP_DMA_PFP_CONTROL_BASE_IDX                                                                  1
+#define regCP_DMA_ME_CONTROL                                                                            0x2078
+#define regCP_DMA_ME_CONTROL_BASE_IDX                                                                   1
+#define regCP_DMA_ME_SRC_ADDR                                                                           0x2080
+#define regCP_DMA_ME_SRC_ADDR_BASE_IDX                                                                  1
+#define regCP_DMA_ME_SRC_ADDR_HI                                                                        0x2081
+#define regCP_DMA_ME_SRC_ADDR_HI_BASE_IDX                                                               1
+#define regCP_DMA_ME_DST_ADDR                                                                           0x2082
+#define regCP_DMA_ME_DST_ADDR_BASE_IDX                                                                  1
+#define regCP_DMA_ME_DST_ADDR_HI                                                                        0x2083
+#define regCP_DMA_ME_DST_ADDR_HI_BASE_IDX                                                               1
+#define regCP_DMA_ME_COMMAND                                                                            0x2084
+#define regCP_DMA_ME_COMMAND_BASE_IDX                                                                   1
+#define regCP_DMA_PFP_SRC_ADDR                                                                          0x2085
+#define regCP_DMA_PFP_SRC_ADDR_BASE_IDX                                                                 1
+#define regCP_DMA_PFP_SRC_ADDR_HI                                                                       0x2086
+#define regCP_DMA_PFP_SRC_ADDR_HI_BASE_IDX                                                              1
+#define regCP_DMA_PFP_DST_ADDR                                                                          0x2087
+#define regCP_DMA_PFP_DST_ADDR_BASE_IDX                                                                 1
+#define regCP_DMA_PFP_DST_ADDR_HI                                                                       0x2088
+#define regCP_DMA_PFP_DST_ADDR_HI_BASE_IDX                                                              1
+#define regCP_DMA_PFP_COMMAND                                                                           0x2089
+#define regCP_DMA_PFP_COMMAND_BASE_IDX                                                                  1
+#define regCP_DMA_CNTL                                                                                  0x208a
+#define regCP_DMA_CNTL_BASE_IDX                                                                         1
+#define regCP_DMA_READ_TAGS                                                                             0x208b
+#define regCP_DMA_READ_TAGS_BASE_IDX                                                                    1
+#define regCP_PFP_IB_CONTROL                                                                            0x208d
+#define regCP_PFP_IB_CONTROL_BASE_IDX                                                                   1
+#define regCP_PFP_LOAD_CONTROL                                                                          0x208e
+#define regCP_PFP_LOAD_CONTROL_BASE_IDX                                                                 1
+#define regCP_SCRATCH_INDEX                                                                             0x208f
+#define regCP_SCRATCH_INDEX_BASE_IDX                                                                    1
+#define regCP_SCRATCH_DATA                                                                              0x2090
+#define regCP_SCRATCH_DATA_BASE_IDX                                                                     1
+#define regCP_RB_OFFSET                                                                                 0x2091
+#define regCP_RB_OFFSET_BASE_IDX                                                                        1
+#define regCP_IB1_OFFSET                                                                                0x2092
+#define regCP_IB1_OFFSET_BASE_IDX                                                                       1
+#define regCP_IB2_OFFSET                                                                                0x2093
+#define regCP_IB2_OFFSET_BASE_IDX                                                                       1
+#define regCP_IB1_PREAMBLE_BEGIN                                                                        0x2094
+#define regCP_IB1_PREAMBLE_BEGIN_BASE_IDX                                                               1
+#define regCP_IB1_PREAMBLE_END                                                                          0x2095
+#define regCP_IB1_PREAMBLE_END_BASE_IDX                                                                 1
+#define regCP_IB2_PREAMBLE_BEGIN                                                                        0x2096
+#define regCP_IB2_PREAMBLE_BEGIN_BASE_IDX                                                               1
+#define regCP_IB2_PREAMBLE_END                                                                          0x2097
+#define regCP_IB2_PREAMBLE_END_BASE_IDX                                                                 1
+#define regCP_DMA_ME_CMD_ADDR_LO                                                                        0x209c
+#define regCP_DMA_ME_CMD_ADDR_LO_BASE_IDX                                                               1
+#define regCP_DMA_ME_CMD_ADDR_HI                                                                        0x209d
+#define regCP_DMA_ME_CMD_ADDR_HI_BASE_IDX                                                               1
+#define regCP_DMA_PFP_CMD_ADDR_LO                                                                       0x209e
+#define regCP_DMA_PFP_CMD_ADDR_LO_BASE_IDX                                                              1
+#define regCP_DMA_PFP_CMD_ADDR_HI                                                                       0x209f
+#define regCP_DMA_PFP_CMD_ADDR_HI_BASE_IDX                                                              1
+#define regUCONFIG_RESERVED_REG0                                                                        0x20a2
+#define regUCONFIG_RESERVED_REG0_BASE_IDX                                                               1
+#define regUCONFIG_RESERVED_REG1                                                                        0x20a3
+#define regUCONFIG_RESERVED_REG1_BASE_IDX                                                               1
+#define regCP_PA_MSPRIM_COUNT_LO                                                                        0x20a4
+#define regCP_PA_MSPRIM_COUNT_LO_BASE_IDX                                                               1
+#define regCP_PA_MSPRIM_COUNT_HI                                                                        0x20a5
+#define regCP_PA_MSPRIM_COUNT_HI_BASE_IDX                                                               1
+#define regCP_GE_MSINVOC_COUNT_LO                                                                       0x20a6
+#define regCP_GE_MSINVOC_COUNT_LO_BASE_IDX                                                              1
+#define regCP_GE_MSINVOC_COUNT_HI                                                                       0x20a7
+#define regCP_GE_MSINVOC_COUNT_HI_BASE_IDX                                                              1
+#define regCP_IB1_CMD_BUFSZ                                                                             0x20c0
+#define regCP_IB1_CMD_BUFSZ_BASE_IDX                                                                    1
+#define regCP_IB2_CMD_BUFSZ                                                                             0x20c1
+#define regCP_IB2_CMD_BUFSZ_BASE_IDX                                                                    1
+#define regCP_ST_CMD_BUFSZ                                                                              0x20c2
+#define regCP_ST_CMD_BUFSZ_BASE_IDX                                                                     1
+#define regCP_IB1_BASE_LO                                                                               0x20cc
+#define regCP_IB1_BASE_LO_BASE_IDX                                                                      1
+#define regCP_IB1_BASE_HI                                                                               0x20cd
+#define regCP_IB1_BASE_HI_BASE_IDX                                                                      1
+#define regCP_IB1_BUFSZ                                                                                 0x20ce
+#define regCP_IB1_BUFSZ_BASE_IDX                                                                        1
+#define regCP_IB2_BASE_LO                                                                               0x20cf
+#define regCP_IB2_BASE_LO_BASE_IDX                                                                      1
+#define regCP_IB2_BASE_HI                                                                               0x20d0
+#define regCP_IB2_BASE_HI_BASE_IDX                                                                      1
+#define regCP_IB2_BUFSZ                                                                                 0x20d1
+#define regCP_IB2_BUFSZ_BASE_IDX                                                                        1
+#define regCP_ST_BASE_LO                                                                                0x20d2
+#define regCP_ST_BASE_LO_BASE_IDX                                                                       1
+#define regCP_ST_BASE_HI                                                                                0x20d3
+#define regCP_ST_BASE_HI_BASE_IDX                                                                       1
+#define regCP_ST_BUFSZ                                                                                  0x20d4
+#define regCP_ST_BUFSZ_BASE_IDX                                                                         1
+#define regCP_EOP_DONE_EVENT_CNTL                                                                       0x20d5
+#define regCP_EOP_DONE_EVENT_CNTL_BASE_IDX                                                              1
+#define regCP_EOP_DONE_DATA_CNTL                                                                        0x20d6
+#define regCP_EOP_DONE_DATA_CNTL_BASE_IDX                                                               1
+#define regCP_EOP_DONE_CNTX_ID                                                                          0x20d7
+#define regCP_EOP_DONE_CNTX_ID_BASE_IDX                                                                 1
+#define regCP_DB_BASE_LO                                                                                0x20d8
+#define regCP_DB_BASE_LO_BASE_IDX                                                                       1
+#define regCP_DB_BASE_HI                                                                                0x20d9
+#define regCP_DB_BASE_HI_BASE_IDX                                                                       1
+#define regCP_DB_BUFSZ                                                                                  0x20da
+#define regCP_DB_BUFSZ_BASE_IDX                                                                         1
+#define regCP_DB_CMD_BUFSZ                                                                              0x20db
+#define regCP_DB_CMD_BUFSZ_BASE_IDX                                                                     1
+#define regCP_PFP_COMPLETION_STATUS                                                                     0x20ec
+#define regCP_PFP_COMPLETION_STATUS_BASE_IDX                                                            1
+#define regCP_PRED_NOT_VISIBLE                                                                          0x20ee
+#define regCP_PRED_NOT_VISIBLE_BASE_IDX                                                                 1
+#define regCP_PFP_METADATA_BASE_ADDR                                                                    0x20f0
+#define regCP_PFP_METADATA_BASE_ADDR_BASE_IDX                                                           1
+#define regCP_PFP_METADATA_BASE_ADDR_HI                                                                 0x20f1
+#define regCP_PFP_METADATA_BASE_ADDR_HI_BASE_IDX                                                        1
+#define regCP_DRAW_INDX_INDR_ADDR                                                                       0x20f4
+#define regCP_DRAW_INDX_INDR_ADDR_BASE_IDX                                                              1
+#define regCP_DRAW_INDX_INDR_ADDR_HI                                                                    0x20f5
+#define regCP_DRAW_INDX_INDR_ADDR_HI_BASE_IDX                                                           1
+#define regCP_DISPATCH_INDR_ADDR                                                                        0x20f6
+#define regCP_DISPATCH_INDR_ADDR_BASE_IDX                                                               1
+#define regCP_DISPATCH_INDR_ADDR_HI                                                                     0x20f7
+#define regCP_DISPATCH_INDR_ADDR_HI_BASE_IDX                                                            1
+#define regCP_INDEX_BASE_ADDR                                                                           0x20f8
+#define regCP_INDEX_BASE_ADDR_BASE_IDX                                                                  1
+#define regCP_INDEX_BASE_ADDR_HI                                                                        0x20f9
+#define regCP_INDEX_BASE_ADDR_HI_BASE_IDX                                                               1
+#define regCP_INDEX_TYPE                                                                                0x20fa
+#define regCP_INDEX_TYPE_BASE_IDX                                                                       1
+#define regCP_SAMPLE_STATUS                                                                             0x20fd
+#define regCP_SAMPLE_STATUS_BASE_IDX                                                                    1
+#define regCP_ME_COHER_CNTL                                                                             0x20fe
+#define regCP_ME_COHER_CNTL_BASE_IDX                                                                    1
+#define regCP_ME_COHER_SIZE                                                                             0x20ff
+#define regCP_ME_COHER_SIZE_BASE_IDX                                                                    1
+#define regCP_ME_COHER_SIZE_HI                                                                          0x2100
+#define regCP_ME_COHER_SIZE_HI_BASE_IDX                                                                 1
+#define regCP_ME_COHER_BASE                                                                             0x2101
+#define regCP_ME_COHER_BASE_BASE_IDX                                                                    1
+#define regCP_ME_COHER_BASE_HI                                                                          0x2102
+#define regCP_ME_COHER_BASE_HI_BASE_IDX                                                                 1
+#define regCP_ME_COHER_STATUS                                                                           0x2103
+#define regCP_ME_COHER_STATUS_BASE_IDX                                                                  1
+#define regRLC_GPM_PERF_COUNT_0                                                                         0x2140
+#define regRLC_GPM_PERF_COUNT_0_BASE_IDX                                                                1
+#define regRLC_GPM_PERF_COUNT_1                                                                         0x2141
+#define regRLC_GPM_PERF_COUNT_1_BASE_IDX                                                                1
+#define regGRBM_GFX_INDEX                                                                               0x2200
+#define regGRBM_GFX_INDEX_BASE_IDX                                                                      1
+#define regGRBM_NOWHERE_2                                                                               0x2201
+#define regGRBM_NOWHERE_2_BASE_IDX                                                                      1
+#define regVGT_PRIMITIVE_TYPE                                                                           0x2242
+#define regVGT_PRIMITIVE_TYPE_BASE_IDX                                                                  1
+#define regVGT_INDEX_TYPE                                                                               0x2243
+#define regVGT_INDEX_TYPE_BASE_IDX                                                                      1
+#define regGE_MIN_VTX_INDX                                                                              0x2249
+#define regGE_MIN_VTX_INDX_BASE_IDX                                                                     1
+#define regGE_INDX_OFFSET                                                                               0x224a
+#define regGE_INDX_OFFSET_BASE_IDX                                                                      1
+#define regGE_MULTI_PRIM_IB_RESET_EN                                                                    0x224b
+#define regGE_MULTI_PRIM_IB_RESET_EN_BASE_IDX                                                           1
+#define regVGT_NUM_INDICES                                                                              0x224c
+#define regVGT_NUM_INDICES_BASE_IDX                                                                     1
+#define regVGT_NUM_INSTANCES                                                                            0x224d
+#define regVGT_NUM_INSTANCES_BASE_IDX                                                                   1
+#define regVGT_TF_MEMORY_BASE                                                                           0x2250
+#define regVGT_TF_MEMORY_BASE_BASE_IDX                                                                  1
+#define regGE_GS_THROTTLE                                                                               0x2254
+#define regGE_GS_THROTTLE_BASE_IDX                                                                      1
+#define regGE_MAX_VTX_INDX                                                                              0x2259
+#define regGE_MAX_VTX_INDX_BASE_IDX                                                                     1
+#define regVGT_INSTANCE_BASE_ID                                                                         0x225a
+#define regVGT_INSTANCE_BASE_ID_BASE_IDX                                                                1
+#define regGE_CNTL                                                                                      0x225b
+#define regGE_CNTL_BASE_IDX                                                                             1
+#define regGE_USER_VGPR1                                                                                0x225c
+#define regGE_USER_VGPR1_BASE_IDX                                                                       1
+#define regGE_USER_VGPR2                                                                                0x225d
+#define regGE_USER_VGPR2_BASE_IDX                                                                       1
+#define regGE_USER_VGPR3                                                                                0x225e
+#define regGE_USER_VGPR3_BASE_IDX                                                                       1
+#define regGE_STEREO_CNTL                                                                               0x225f
+#define regGE_STEREO_CNTL_BASE_IDX                                                                      1
+#define regGE_USER_VGPR_EN                                                                              0x2260
+#define regGE_USER_VGPR_EN_BASE_IDX                                                                     1
+#define regVGT_PRIMITIVEID_EN                                                                           0x2262
+#define regVGT_PRIMITIVEID_EN_BASE_IDX                                                                  1
+#define regGE_VRS_RATE                                                                                  0x2263
+#define regGE_VRS_RATE_BASE_IDX                                                                         1
+#define regGE_GS_FAST_LAUNCH_WG_DIM                                                                     0x2264
+#define regGE_GS_FAST_LAUNCH_WG_DIM_BASE_IDX                                                            1
+#define regGE_GS_FAST_LAUNCH_WG_DIM_1                                                                   0x2265
+#define regGE_GS_FAST_LAUNCH_WG_DIM_1_BASE_IDX                                                          1
+#define regVGT_GS_OUT_PRIM_TYPE                                                                         0x2266
+#define regVGT_GS_OUT_PRIM_TYPE_BASE_IDX                                                                1
+#define regVGT_TF_MEMORY_BASE_HI                                                                        0x2267
+#define regVGT_TF_MEMORY_BASE_HI_BASE_IDX                                                               1
+#define regGE_GS_ORDERED_ID_BASE                                                                        0x226c
+#define regGE_GS_ORDERED_ID_BASE_BASE_IDX                                                               1
+#define regVGT_PRIMITIVEID_RESET                                                                        0x226d
+#define regVGT_PRIMITIVEID_RESET_BASE_IDX                                                               1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_cprs64dec
+// base address: 0x32000
+#define regCP_MES_PRGRM_CNTR_START                                                                      0x2800
+#define regCP_MES_PRGRM_CNTR_START_BASE_IDX                                                             1
+#define regCP_MES_INTR_ROUTINE_START                                                                    0x2801
+#define regCP_MES_INTR_ROUTINE_START_BASE_IDX                                                           1
+#define regCP_MES_MTVEC_LO                                                                              0x2801
+#define regCP_MES_MTVEC_LO_BASE_IDX                                                                     1
+#define regCP_MES_INTR_ROUTINE_START_HI                                                                 0x2802
+#define regCP_MES_INTR_ROUTINE_START_HI_BASE_IDX                                                        1
+#define regCP_MES_MTVEC_HI                                                                              0x2802
+#define regCP_MES_MTVEC_HI_BASE_IDX                                                                     1
+#define regCP_MES_CNTL                                                                                  0x2807
+#define regCP_MES_CNTL_BASE_IDX                                                                         1
+#define regCP_MES_PIPE_PRIORITY_CNTS                                                                    0x2808
+#define regCP_MES_PIPE_PRIORITY_CNTS_BASE_IDX                                                           1
+#define regCP_MES_PIPE0_PRIORITY                                                                        0x2809
+#define regCP_MES_PIPE0_PRIORITY_BASE_IDX                                                               1
+#define regCP_MES_PIPE1_PRIORITY                                                                        0x280a
+#define regCP_MES_PIPE1_PRIORITY_BASE_IDX                                                               1
+#define regCP_MES_PIPE2_PRIORITY                                                                        0x280b
+#define regCP_MES_PIPE2_PRIORITY_BASE_IDX                                                               1
+#define regCP_MES_PIPE3_PRIORITY                                                                        0x280c
+#define regCP_MES_PIPE3_PRIORITY_BASE_IDX                                                               1
+#define regCP_MES_HEADER_DUMP                                                                           0x280d
+#define regCP_MES_HEADER_DUMP_BASE_IDX                                                                  1
+#define regCP_MES_MIE_LO                                                                                0x280e
+#define regCP_MES_MIE_LO_BASE_IDX                                                                       1
+#define regCP_MES_MIE_HI                                                                                0x280f
+#define regCP_MES_MIE_HI_BASE_IDX                                                                       1
+#define regCP_MES_INTERRUPT                                                                             0x2810
+#define regCP_MES_INTERRUPT_BASE_IDX                                                                    1
+#define regCP_MES_SCRATCH_INDEX                                                                         0x2811
+#define regCP_MES_SCRATCH_INDEX_BASE_IDX                                                                1
+#define regCP_MES_SCRATCH_DATA                                                                          0x2812
+#define regCP_MES_SCRATCH_DATA_BASE_IDX                                                                 1
+#define regCP_MES_INSTR_PNTR                                                                            0x2813
+#define regCP_MES_INSTR_PNTR_BASE_IDX                                                                   1
+#define regCP_MES_MSCRATCH_HI                                                                           0x2814
+#define regCP_MES_MSCRATCH_HI_BASE_IDX                                                                  1
+#define regCP_MES_MSCRATCH_LO                                                                           0x2815
+#define regCP_MES_MSCRATCH_LO_BASE_IDX                                                                  1
+#define regCP_MES_MSTATUS_LO                                                                            0x2816
+#define regCP_MES_MSTATUS_LO_BASE_IDX                                                                   1
+#define regCP_MES_MSTATUS_HI                                                                            0x2817
+#define regCP_MES_MSTATUS_HI_BASE_IDX                                                                   1
+#define regCP_MES_MEPC_LO                                                                               0x2818
+#define regCP_MES_MEPC_LO_BASE_IDX                                                                      1
+#define regCP_MES_MEPC_HI                                                                               0x2819
+#define regCP_MES_MEPC_HI_BASE_IDX                                                                      1
+#define regCP_MES_MCAUSE_LO                                                                             0x281a
+#define regCP_MES_MCAUSE_LO_BASE_IDX                                                                    1
+#define regCP_MES_MCAUSE_HI                                                                             0x281b
+#define regCP_MES_MCAUSE_HI_BASE_IDX                                                                    1
+#define regCP_MES_MBADADDR_LO                                                                           0x281c
+#define regCP_MES_MBADADDR_LO_BASE_IDX                                                                  1
+#define regCP_MES_MBADADDR_HI                                                                           0x281d
+#define regCP_MES_MBADADDR_HI_BASE_IDX                                                                  1
+#define regCP_MES_MIP_LO                                                                                0x281e
+#define regCP_MES_MIP_LO_BASE_IDX                                                                       1
+#define regCP_MES_MIP_HI                                                                                0x281f
+#define regCP_MES_MIP_HI_BASE_IDX                                                                       1
+#define regCP_MES_IC_OP_CNTL                                                                            0x2820
+#define regCP_MES_IC_OP_CNTL_BASE_IDX                                                                   1
+#define regCP_MES_MCYCLE_LO                                                                             0x2826
+#define regCP_MES_MCYCLE_LO_BASE_IDX                                                                    1
+#define regCP_MES_MCYCLE_HI                                                                             0x2827
+#define regCP_MES_MCYCLE_HI_BASE_IDX                                                                    1
+#define regCP_MES_MTIME_LO                                                                              0x2828
+#define regCP_MES_MTIME_LO_BASE_IDX                                                                     1
+#define regCP_MES_MTIME_HI                                                                              0x2829
+#define regCP_MES_MTIME_HI_BASE_IDX                                                                     1
+#define regCP_MES_MINSTRET_LO                                                                           0x282a
+#define regCP_MES_MINSTRET_LO_BASE_IDX                                                                  1
+#define regCP_MES_MINSTRET_HI                                                                           0x282b
+#define regCP_MES_MINSTRET_HI_BASE_IDX                                                                  1
+#define regCP_MES_MISA_LO                                                                               0x282c
+#define regCP_MES_MISA_LO_BASE_IDX                                                                      1
+#define regCP_MES_MISA_HI                                                                               0x282d
+#define regCP_MES_MISA_HI_BASE_IDX                                                                      1
+#define regCP_MES_MVENDORID_LO                                                                          0x282e
+#define regCP_MES_MVENDORID_LO_BASE_IDX                                                                 1
+#define regCP_MES_MVENDORID_HI                                                                          0x282f
+#define regCP_MES_MVENDORID_HI_BASE_IDX                                                                 1
+#define regCP_MES_MARCHID_LO                                                                            0x2830
+#define regCP_MES_MARCHID_LO_BASE_IDX                                                                   1
+#define regCP_MES_MARCHID_HI                                                                            0x2831
+#define regCP_MES_MARCHID_HI_BASE_IDX                                                                   1
+#define regCP_MES_MIMPID_LO                                                                             0x2832
+#define regCP_MES_MIMPID_LO_BASE_IDX                                                                    1
+#define regCP_MES_MIMPID_HI                                                                             0x2833
+#define regCP_MES_MIMPID_HI_BASE_IDX                                                                    1
+#define regCP_MES_MHARTID_LO                                                                            0x2834
+#define regCP_MES_MHARTID_LO_BASE_IDX                                                                   1
+#define regCP_MES_MHARTID_HI                                                                            0x2835
+#define regCP_MES_MHARTID_HI_BASE_IDX                                                                   1
+#define regCP_MES_DC_BASE_CNTL                                                                          0x2836
+#define regCP_MES_DC_BASE_CNTL_BASE_IDX                                                                 1
+#define regCP_MES_DC_OP_CNTL                                                                            0x2837
+#define regCP_MES_DC_OP_CNTL_BASE_IDX                                                                   1
+#define regCP_MES_MTIMECMP_LO                                                                           0x2838
+#define regCP_MES_MTIMECMP_LO_BASE_IDX                                                                  1
+#define regCP_MES_MTIMECMP_HI                                                                           0x2839
+#define regCP_MES_MTIMECMP_HI_BASE_IDX                                                                  1
+#define regCP_MES_PROCESS_QUANTUM_PIPE0                                                                 0x283a
+#define regCP_MES_PROCESS_QUANTUM_PIPE0_BASE_IDX                                                        1
+#define regCP_MES_PROCESS_QUANTUM_PIPE1                                                                 0x283b
+#define regCP_MES_PROCESS_QUANTUM_PIPE1_BASE_IDX                                                        1
+#define regCP_MES_DOORBELL_CONTROL1                                                                     0x283c
+#define regCP_MES_DOORBELL_CONTROL1_BASE_IDX                                                            1
+#define regCP_MES_DOORBELL_CONTROL2                                                                     0x283d
+#define regCP_MES_DOORBELL_CONTROL2_BASE_IDX                                                            1
+#define regCP_MES_DOORBELL_CONTROL3                                                                     0x283e
+#define regCP_MES_DOORBELL_CONTROL3_BASE_IDX                                                            1
+#define regCP_MES_DOORBELL_CONTROL4                                                                     0x283f
+#define regCP_MES_DOORBELL_CONTROL4_BASE_IDX                                                            1
+#define regCP_MES_DOORBELL_CONTROL5                                                                     0x2840
+#define regCP_MES_DOORBELL_CONTROL5_BASE_IDX                                                            1
+#define regCP_MES_DOORBELL_CONTROL6                                                                     0x2841
+#define regCP_MES_DOORBELL_CONTROL6_BASE_IDX                                                            1
+#define regCP_MES_GP0_LO                                                                                0x2843
+#define regCP_MES_GP0_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP0_HI                                                                                0x2844
+#define regCP_MES_GP0_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP1_LO                                                                                0x2845
+#define regCP_MES_GP1_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP1_HI                                                                                0x2846
+#define regCP_MES_GP1_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP2_LO                                                                                0x2847
+#define regCP_MES_GP2_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP2_HI                                                                                0x2848
+#define regCP_MES_GP2_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP3_LO                                                                                0x2849
+#define regCP_MES_GP3_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP3_HI                                                                                0x284a
+#define regCP_MES_GP3_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP4_LO                                                                                0x284b
+#define regCP_MES_GP4_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP4_HI                                                                                0x284c
+#define regCP_MES_GP4_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP5_LO                                                                                0x284d
+#define regCP_MES_GP5_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP5_HI                                                                                0x284e
+#define regCP_MES_GP5_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP6_LO                                                                                0x284f
+#define regCP_MES_GP6_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP6_HI                                                                                0x2850
+#define regCP_MES_GP6_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP7_LO                                                                                0x2851
+#define regCP_MES_GP7_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP7_HI                                                                                0x2852
+#define regCP_MES_GP7_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP8_LO                                                                                0x2853
+#define regCP_MES_GP8_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP8_HI                                                                                0x2854
+#define regCP_MES_GP8_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP9_LO                                                                                0x2855
+#define regCP_MES_GP9_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP9_HI                                                                                0x2856
+#define regCP_MES_GP9_HI_BASE_IDX                                                                       1
+#define regCP_MES_LOCAL_BASE0_LO                                                                        0x2883
+#define regCP_MES_LOCAL_BASE0_LO_BASE_IDX                                                               1
+#define regCP_MES_LOCAL_BASE0_HI                                                                        0x2884
+#define regCP_MES_LOCAL_BASE0_HI_BASE_IDX                                                               1
+#define regCP_MES_LOCAL_MASK0_LO                                                                        0x2885
+#define regCP_MES_LOCAL_MASK0_LO_BASE_IDX                                                               1
+#define regCP_MES_LOCAL_MASK0_HI                                                                        0x2886
+#define regCP_MES_LOCAL_MASK0_HI_BASE_IDX                                                               1
+#define regCP_MES_LOCAL_APERTURE                                                                        0x2887
+#define regCP_MES_LOCAL_APERTURE_BASE_IDX                                                               1
+#define regCP_MES_LOCAL_INSTR_BASE_LO                                                                   0x2888
+#define regCP_MES_LOCAL_INSTR_BASE_LO_BASE_IDX                                                          1
+#define regCP_MES_LOCAL_INSTR_BASE_HI                                                                   0x2889
+#define regCP_MES_LOCAL_INSTR_BASE_HI_BASE_IDX                                                          1
+#define regCP_MES_LOCAL_INSTR_MASK_LO                                                                   0x288a
+#define regCP_MES_LOCAL_INSTR_MASK_LO_BASE_IDX                                                          1
+#define regCP_MES_LOCAL_INSTR_MASK_HI                                                                   0x288b
+#define regCP_MES_LOCAL_INSTR_MASK_HI_BASE_IDX                                                          1
+#define regCP_MES_LOCAL_INSTR_APERTURE                                                                  0x288c
+#define regCP_MES_LOCAL_INSTR_APERTURE_BASE_IDX                                                         1
+#define regCP_MES_LOCAL_SCRATCH_APERTURE                                                                0x288d
+#define regCP_MES_LOCAL_SCRATCH_APERTURE_BASE_IDX                                                       1
+#define regCP_MES_LOCAL_SCRATCH_BASE_LO                                                                 0x288e
+#define regCP_MES_LOCAL_SCRATCH_BASE_LO_BASE_IDX                                                        1
+#define regCP_MES_LOCAL_SCRATCH_BASE_HI                                                                 0x288f
+#define regCP_MES_LOCAL_SCRATCH_BASE_HI_BASE_IDX                                                        1
+#define regCP_MES_PERFCOUNT_CNTL                                                                        0x2899
+#define regCP_MES_PERFCOUNT_CNTL_BASE_IDX                                                               1
+#define regCP_MES_PENDING_INTERRUPT                                                                     0x289a
+#define regCP_MES_PENDING_INTERRUPT_BASE_IDX                                                            1
+#define regCP_MES_RS64_EXCEPTION_STATUS                                                                 0x289c
+#define regCP_MES_RS64_EXCEPTION_STATUS_BASE_IDX                                                        1
+#define regCP_MES_PRGRM_CNTR_START_HI                                                                   0x289d
+#define regCP_MES_PRGRM_CNTR_START_HI_BASE_IDX                                                          1
+#define regCP_MES_INTERRUPT_DATA_16                                                                     0x289f
+#define regCP_MES_INTERRUPT_DATA_16_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_17                                                                     0x28a0
+#define regCP_MES_INTERRUPT_DATA_17_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_18                                                                     0x28a1
+#define regCP_MES_INTERRUPT_DATA_18_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_19                                                                     0x28a2
+#define regCP_MES_INTERRUPT_DATA_19_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_20                                                                     0x28a3
+#define regCP_MES_INTERRUPT_DATA_20_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_21                                                                     0x28a4
+#define regCP_MES_INTERRUPT_DATA_21_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_22                                                                     0x28a5
+#define regCP_MES_INTERRUPT_DATA_22_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_23                                                                     0x28a6
+#define regCP_MES_INTERRUPT_DATA_23_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_24                                                                     0x28a7
+#define regCP_MES_INTERRUPT_DATA_24_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_25                                                                     0x28a8
+#define regCP_MES_INTERRUPT_DATA_25_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_26                                                                     0x28a9
+#define regCP_MES_INTERRUPT_DATA_26_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_27                                                                     0x28aa
+#define regCP_MES_INTERRUPT_DATA_27_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_28                                                                     0x28ab
+#define regCP_MES_INTERRUPT_DATA_28_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_29                                                                     0x28ac
+#define regCP_MES_INTERRUPT_DATA_29_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_30                                                                     0x28ad
+#define regCP_MES_INTERRUPT_DATA_30_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_31                                                                     0x28ae
+#define regCP_MES_INTERRUPT_DATA_31_BASE_IDX                                                            1
+#define regCP_MES_METADATA_CNTL                                                                         0x28df
+#define regCP_MES_METADATA_CNTL_BASE_IDX                                                                1
+#define regCP_MEC_RS64_PRGRM_CNTR_START                                                                 0x2900
+#define regCP_MEC_RS64_PRGRM_CNTR_START_BASE_IDX                                                        1
+#define regCP_MEC_MTVEC_LO                                                                              0x2901
+#define regCP_MEC_MTVEC_LO_BASE_IDX                                                                     1
+#define regCP_MEC_MTVEC_HI                                                                              0x2902
+#define regCP_MEC_MTVEC_HI_BASE_IDX                                                                     1
+#define regCP_MEC_RS64_CNTL                                                                             0x2904
+#define regCP_MEC_RS64_CNTL_BASE_IDX                                                                    1
+#define regCP_MEC_MIE_LO                                                                                0x2905
+#define regCP_MEC_MIE_LO_BASE_IDX                                                                       1
+#define regCP_MEC_MIE_HI                                                                                0x2906
+#define regCP_MEC_MIE_HI_BASE_IDX                                                                       1
+#define regCP_MEC_RS64_INTERRUPT                                                                        0x2907
+#define regCP_MEC_RS64_INTERRUPT_BASE_IDX                                                               1
+#define regCP_MEC_RS64_INSTR_PNTR                                                                       0x2908
+#define regCP_MEC_RS64_INSTR_PNTR_BASE_IDX                                                              1
+#define regCP_MEC_MIP_LO                                                                                0x2909
+#define regCP_MEC_MIP_LO_BASE_IDX                                                                       1
+#define regCP_MEC_MIP_HI                                                                                0x290a
+#define regCP_MEC_MIP_HI_BASE_IDX                                                                       1
+#define regCP_MEC_DC_BASE_CNTL                                                                          0x290b
+#define regCP_MEC_DC_BASE_CNTL_BASE_IDX                                                                 1
+#define regCP_MEC_DC_OP_CNTL                                                                            0x290c
+#define regCP_MEC_DC_OP_CNTL_BASE_IDX                                                                   1
+#define regCP_MEC_MTIMECMP_LO                                                                           0x290d
+#define regCP_MEC_MTIMECMP_LO_BASE_IDX                                                                  1
+#define regCP_MEC_MTIMECMP_HI                                                                           0x290e
+#define regCP_MEC_MTIMECMP_HI_BASE_IDX                                                                  1
+#define regCP_MEC_GP0_LO                                                                                0x2910
+#define regCP_MEC_GP0_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP0_HI                                                                                0x2911
+#define regCP_MEC_GP0_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP1_LO                                                                                0x2912
+#define regCP_MEC_GP1_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP1_HI                                                                                0x2913
+#define regCP_MEC_GP1_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP2_LO                                                                                0x2914
+#define regCP_MEC_GP2_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP2_HI                                                                                0x2915
+#define regCP_MEC_GP2_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP3_LO                                                                                0x2916
+#define regCP_MEC_GP3_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP3_HI                                                                                0x2917
+#define regCP_MEC_GP3_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP4_LO                                                                                0x2918
+#define regCP_MEC_GP4_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP4_HI                                                                                0x2919
+#define regCP_MEC_GP4_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP5_LO                                                                                0x291a
+#define regCP_MEC_GP5_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP5_HI                                                                                0x291b
+#define regCP_MEC_GP5_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP6_LO                                                                                0x291c
+#define regCP_MEC_GP6_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP6_HI                                                                                0x291d
+#define regCP_MEC_GP6_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP7_LO                                                                                0x291e
+#define regCP_MEC_GP7_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP7_HI                                                                                0x291f
+#define regCP_MEC_GP7_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP8_LO                                                                                0x2920
+#define regCP_MEC_GP8_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP8_HI                                                                                0x2921
+#define regCP_MEC_GP8_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP9_LO                                                                                0x2922
+#define regCP_MEC_GP9_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP9_HI                                                                                0x2923
+#define regCP_MEC_GP9_HI_BASE_IDX                                                                       1
+#define regCP_MEC_LOCAL_BASE0_LO                                                                        0x2927
+#define regCP_MEC_LOCAL_BASE0_LO_BASE_IDX                                                               1
+#define regCP_MEC_LOCAL_BASE0_HI                                                                        0x2928
+#define regCP_MEC_LOCAL_BASE0_HI_BASE_IDX                                                               1
+#define regCP_MEC_LOCAL_MASK0_LO                                                                        0x2929
+#define regCP_MEC_LOCAL_MASK0_LO_BASE_IDX                                                               1
+#define regCP_MEC_LOCAL_MASK0_HI                                                                        0x292a
+#define regCP_MEC_LOCAL_MASK0_HI_BASE_IDX                                                               1
+#define regCP_MEC_LOCAL_APERTURE                                                                        0x292b
+#define regCP_MEC_LOCAL_APERTURE_BASE_IDX                                                               1
+#define regCP_MEC_LOCAL_INSTR_BASE_LO                                                                   0x292c
+#define regCP_MEC_LOCAL_INSTR_BASE_LO_BASE_IDX                                                          1
+#define regCP_MEC_LOCAL_INSTR_BASE_HI                                                                   0x292d
+#define regCP_MEC_LOCAL_INSTR_BASE_HI_BASE_IDX                                                          1
+#define regCP_MEC_LOCAL_INSTR_MASK_LO                                                                   0x292e
+#define regCP_MEC_LOCAL_INSTR_MASK_LO_BASE_IDX                                                          1
+#define regCP_MEC_LOCAL_INSTR_MASK_HI                                                                   0x292f
+#define regCP_MEC_LOCAL_INSTR_MASK_HI_BASE_IDX                                                          1
+#define regCP_MEC_LOCAL_INSTR_APERTURE                                                                  0x2930
+#define regCP_MEC_LOCAL_INSTR_APERTURE_BASE_IDX                                                         1
+#define regCP_MEC_LOCAL_SCRATCH_APERTURE                                                                0x2931
+#define regCP_MEC_LOCAL_SCRATCH_APERTURE_BASE_IDX                                                       1
+#define regCP_MEC_LOCAL_SCRATCH_BASE_LO                                                                 0x2932
+#define regCP_MEC_LOCAL_SCRATCH_BASE_LO_BASE_IDX                                                        1
+#define regCP_MEC_LOCAL_SCRATCH_BASE_HI                                                                 0x2933
+#define regCP_MEC_LOCAL_SCRATCH_BASE_HI_BASE_IDX                                                        1
+#define regCP_MEC_RS64_PERFCOUNT_CNTL                                                                   0x2934
+#define regCP_MEC_RS64_PERFCOUNT_CNTL_BASE_IDX                                                          1
+#define regCP_MEC_RS64_PENDING_INTERRUPT                                                                0x2935
+#define regCP_MEC_RS64_PENDING_INTERRUPT_BASE_IDX                                                       1
+#define regCP_MEC_RS64_EXCEPTION_STATUS                                                                 0x2937
+#define regCP_MEC_RS64_EXCEPTION_STATUS_BASE_IDX                                                        1
+#define regCP_MEC_RS64_PRGRM_CNTR_START_HI                                                              0x2938
+#define regCP_MEC_RS64_PRGRM_CNTR_START_HI_BASE_IDX                                                     1
+#define regCP_MEC_RS64_INTERRUPT_DATA_16                                                                0x293a
+#define regCP_MEC_RS64_INTERRUPT_DATA_16_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_17                                                                0x293b
+#define regCP_MEC_RS64_INTERRUPT_DATA_17_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_18                                                                0x293c
+#define regCP_MEC_RS64_INTERRUPT_DATA_18_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_19                                                                0x293d
+#define regCP_MEC_RS64_INTERRUPT_DATA_19_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_20                                                                0x293e
+#define regCP_MEC_RS64_INTERRUPT_DATA_20_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_21                                                                0x293f
+#define regCP_MEC_RS64_INTERRUPT_DATA_21_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_22                                                                0x2940
+#define regCP_MEC_RS64_INTERRUPT_DATA_22_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_23                                                                0x2941
+#define regCP_MEC_RS64_INTERRUPT_DATA_23_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_24                                                                0x2942
+#define regCP_MEC_RS64_INTERRUPT_DATA_24_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_25                                                                0x2943
+#define regCP_MEC_RS64_INTERRUPT_DATA_25_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_26                                                                0x2944
+#define regCP_MEC_RS64_INTERRUPT_DATA_26_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_27                                                                0x2945
+#define regCP_MEC_RS64_INTERRUPT_DATA_27_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_28                                                                0x2946
+#define regCP_MEC_RS64_INTERRUPT_DATA_28_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_29                                                                0x2947
+#define regCP_MEC_RS64_INTERRUPT_DATA_29_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_30                                                                0x2948
+#define regCP_MEC_RS64_INTERRUPT_DATA_30_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_31                                                                0x2949
+#define regCP_MEC_RS64_INTERRUPT_DATA_31_BASE_IDX                                                       1
+#define regCP_CPC_IC_OP_CNTL                                                                            0x297a
+#define regCP_CPC_IC_OP_CNTL_BASE_IDX                                                                   1
+#define regCP_GFX_RS64_INTERRUPT0                                                                       0x2a01
+#define regCP_GFX_RS64_INTERRUPT0_BASE_IDX                                                              1
+#define regCP_GFX_RS64_INTR_EN0                                                                         0x2a02
+#define regCP_GFX_RS64_INTR_EN0_BASE_IDX                                                                1
+#define regCP_GFX_RS64_INTR_EN1                                                                         0x2a03
+#define regCP_GFX_RS64_INTR_EN1_BASE_IDX                                                                1
+#define regCP_GFX_RS64_DC_BASE_CNTL                                                                     0x2a08
+#define regCP_GFX_RS64_DC_BASE_CNTL_BASE_IDX                                                            1
+#define regCP_GFX_RS64_DC_OP_CNTL                                                                       0x2a09
+#define regCP_GFX_RS64_DC_OP_CNTL_BASE_IDX                                                              1
+#define regCP_GFX_RS64_LOCAL_BASE0_LO                                                                   0x2a0a
+#define regCP_GFX_RS64_LOCAL_BASE0_LO_BASE_IDX                                                          1
+#define regCP_GFX_RS64_LOCAL_BASE0_HI                                                                   0x2a0b
+#define regCP_GFX_RS64_LOCAL_BASE0_HI_BASE_IDX                                                          1
+#define regCP_GFX_RS64_LOCAL_MASK0_LO                                                                   0x2a0c
+#define regCP_GFX_RS64_LOCAL_MASK0_LO_BASE_IDX                                                          1
+#define regCP_GFX_RS64_LOCAL_MASK0_HI                                                                   0x2a0d
+#define regCP_GFX_RS64_LOCAL_MASK0_HI_BASE_IDX                                                          1
+#define regCP_GFX_RS64_LOCAL_APERTURE                                                                   0x2a0e
+#define regCP_GFX_RS64_LOCAL_APERTURE_BASE_IDX                                                          1
+#define regCP_GFX_RS64_LOCAL_INSTR_BASE_LO                                                              0x2a0f
+#define regCP_GFX_RS64_LOCAL_INSTR_BASE_LO_BASE_IDX                                                     1
+#define regCP_GFX_RS64_LOCAL_INSTR_BASE_HI                                                              0x2a10
+#define regCP_GFX_RS64_LOCAL_INSTR_BASE_HI_BASE_IDX                                                     1
+#define regCP_GFX_RS64_LOCAL_INSTR_MASK_LO                                                              0x2a11
+#define regCP_GFX_RS64_LOCAL_INSTR_MASK_LO_BASE_IDX                                                     1
+#define regCP_GFX_RS64_LOCAL_INSTR_MASK_HI                                                              0x2a12
+#define regCP_GFX_RS64_LOCAL_INSTR_MASK_HI_BASE_IDX                                                     1
+#define regCP_GFX_RS64_LOCAL_INSTR_APERTURE                                                             0x2a13
+#define regCP_GFX_RS64_LOCAL_INSTR_APERTURE_BASE_IDX                                                    1
+#define regCP_GFX_RS64_LOCAL_SCRATCH_APERTURE                                                           0x2a14
+#define regCP_GFX_RS64_LOCAL_SCRATCH_APERTURE_BASE_IDX                                                  1
+#define regCP_GFX_RS64_LOCAL_SCRATCH_BASE_LO                                                            0x2a15
+#define regCP_GFX_RS64_LOCAL_SCRATCH_BASE_LO_BASE_IDX                                                   1
+#define regCP_GFX_RS64_LOCAL_SCRATCH_BASE_HI                                                            0x2a16
+#define regCP_GFX_RS64_LOCAL_SCRATCH_BASE_HI_BASE_IDX                                                   1
+#define regCP_PFP_RS64_EXCEPTION_STATUS                                                                 0x2a19
+#define regCP_PFP_RS64_EXCEPTION_STATUS_BASE_IDX                                                        1
+#define regCP_GFX_RS64_PERFCOUNT_CNTL0                                                                  0x2a1a
+#define regCP_GFX_RS64_PERFCOUNT_CNTL0_BASE_IDX                                                         1
+#define regCP_GFX_RS64_PERFCOUNT_CNTL1                                                                  0x2a1b
+#define regCP_GFX_RS64_PERFCOUNT_CNTL1_BASE_IDX                                                         1
+#define regCP_GFX_RS64_MIP_LO0                                                                          0x2a1c
+#define regCP_GFX_RS64_MIP_LO0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_MIP_LO1                                                                          0x2a1d
+#define regCP_GFX_RS64_MIP_LO1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_MIP_HI0                                                                          0x2a1e
+#define regCP_GFX_RS64_MIP_HI0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_MIP_HI1                                                                          0x2a1f
+#define regCP_GFX_RS64_MIP_HI1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_MTIMECMP_LO0                                                                     0x2a20
+#define regCP_GFX_RS64_MTIMECMP_LO0_BASE_IDX                                                            1
+#define regCP_GFX_RS64_MTIMECMP_LO1                                                                     0x2a21
+#define regCP_GFX_RS64_MTIMECMP_LO1_BASE_IDX                                                            1
+#define regCP_GFX_RS64_MTIMECMP_HI0                                                                     0x2a22
+#define regCP_GFX_RS64_MTIMECMP_HI0_BASE_IDX                                                            1
+#define regCP_GFX_RS64_MTIMECMP_HI1                                                                     0x2a23
+#define regCP_GFX_RS64_MTIMECMP_HI1_BASE_IDX                                                            1
+#define regCP_GFX_RS64_GP0_LO0                                                                          0x2a24
+#define regCP_GFX_RS64_GP0_LO0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP0_LO1                                                                          0x2a25
+#define regCP_GFX_RS64_GP0_LO1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP0_HI0                                                                          0x2a26
+#define regCP_GFX_RS64_GP0_HI0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP0_HI1                                                                          0x2a27
+#define regCP_GFX_RS64_GP0_HI1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP1_LO0                                                                          0x2a28
+#define regCP_GFX_RS64_GP1_LO0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP1_LO1                                                                          0x2a29
+#define regCP_GFX_RS64_GP1_LO1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP1_HI0                                                                          0x2a2a
+#define regCP_GFX_RS64_GP1_HI0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP1_HI1                                                                          0x2a2b
+#define regCP_GFX_RS64_GP1_HI1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP2_LO0                                                                          0x2a2c
+#define regCP_GFX_RS64_GP2_LO0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP2_LO1                                                                          0x2a2d
+#define regCP_GFX_RS64_GP2_LO1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP2_HI0                                                                          0x2a2e
+#define regCP_GFX_RS64_GP2_HI0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP2_HI1                                                                          0x2a2f
+#define regCP_GFX_RS64_GP2_HI1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP3_LO0                                                                          0x2a30
+#define regCP_GFX_RS64_GP3_LO0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP3_LO1                                                                          0x2a31
+#define regCP_GFX_RS64_GP3_LO1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP3_HI0                                                                          0x2a32
+#define regCP_GFX_RS64_GP3_HI0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP3_HI1                                                                          0x2a33
+#define regCP_GFX_RS64_GP3_HI1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP4_LO0                                                                          0x2a34
+#define regCP_GFX_RS64_GP4_LO0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP4_LO1                                                                          0x2a35
+#define regCP_GFX_RS64_GP4_LO1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP4_HI0                                                                          0x2a36
+#define regCP_GFX_RS64_GP4_HI0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP4_HI1                                                                          0x2a37
+#define regCP_GFX_RS64_GP4_HI1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP5_LO0                                                                          0x2a38
+#define regCP_GFX_RS64_GP5_LO0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP5_LO1                                                                          0x2a39
+#define regCP_GFX_RS64_GP5_LO1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP5_HI0                                                                          0x2a3a
+#define regCP_GFX_RS64_GP5_HI0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP5_HI1                                                                          0x2a3b
+#define regCP_GFX_RS64_GP5_HI1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP6_LO                                                                           0x2a3c
+#define regCP_GFX_RS64_GP6_LO_BASE_IDX                                                                  1
+#define regCP_GFX_RS64_GP6_HI                                                                           0x2a3d
+#define regCP_GFX_RS64_GP6_HI_BASE_IDX                                                                  1
+#define regCP_GFX_RS64_GP7_LO                                                                           0x2a3e
+#define regCP_GFX_RS64_GP7_LO_BASE_IDX                                                                  1
+#define regCP_GFX_RS64_GP7_HI                                                                           0x2a3f
+#define regCP_GFX_RS64_GP7_HI_BASE_IDX                                                                  1
+#define regCP_GFX_RS64_GP8_LO                                                                           0x2a40
+#define regCP_GFX_RS64_GP8_LO_BASE_IDX                                                                  1
+#define regCP_GFX_RS64_GP8_HI                                                                           0x2a41
+#define regCP_GFX_RS64_GP8_HI_BASE_IDX                                                                  1
+#define regCP_GFX_RS64_GP9_LO                                                                           0x2a42
+#define regCP_GFX_RS64_GP9_LO_BASE_IDX                                                                  1
+#define regCP_GFX_RS64_GP9_HI                                                                           0x2a43
+#define regCP_GFX_RS64_GP9_HI_BASE_IDX                                                                  1
+#define regCP_GFX_RS64_INSTR_PNTR0                                                                      0x2a44
+#define regCP_GFX_RS64_INSTR_PNTR0_BASE_IDX                                                             1
+#define regCP_GFX_RS64_INSTR_PNTR1                                                                      0x2a45
+#define regCP_GFX_RS64_INSTR_PNTR1_BASE_IDX                                                             1
+#define regCP_GFX_RS64_PENDING_INTERRUPT0                                                               0x2a46
+#define regCP_GFX_RS64_PENDING_INTERRUPT0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_PENDING_INTERRUPT1                                                               0x2a47
+#define regCP_GFX_RS64_PENDING_INTERRUPT1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE0_BASE0                                                               0x2a49
+#define regCP_GFX_RS64_DC_APERTURE0_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE0_MASK0                                                               0x2a4a
+#define regCP_GFX_RS64_DC_APERTURE0_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE0_CNTL0                                                               0x2a4b
+#define regCP_GFX_RS64_DC_APERTURE0_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE1_BASE0                                                               0x2a4c
+#define regCP_GFX_RS64_DC_APERTURE1_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE1_MASK0                                                               0x2a4d
+#define regCP_GFX_RS64_DC_APERTURE1_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE1_CNTL0                                                               0x2a4e
+#define regCP_GFX_RS64_DC_APERTURE1_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE2_BASE0                                                               0x2a4f
+#define regCP_GFX_RS64_DC_APERTURE2_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE2_MASK0                                                               0x2a50
+#define regCP_GFX_RS64_DC_APERTURE2_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE2_CNTL0                                                               0x2a51
+#define regCP_GFX_RS64_DC_APERTURE2_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE3_BASE0                                                               0x2a52
+#define regCP_GFX_RS64_DC_APERTURE3_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE3_MASK0                                                               0x2a53
+#define regCP_GFX_RS64_DC_APERTURE3_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE3_CNTL0                                                               0x2a54
+#define regCP_GFX_RS64_DC_APERTURE3_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE4_BASE0                                                               0x2a55
+#define regCP_GFX_RS64_DC_APERTURE4_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE4_MASK0                                                               0x2a56
+#define regCP_GFX_RS64_DC_APERTURE4_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE4_CNTL0                                                               0x2a57
+#define regCP_GFX_RS64_DC_APERTURE4_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE5_BASE0                                                               0x2a58
+#define regCP_GFX_RS64_DC_APERTURE5_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE5_MASK0                                                               0x2a59
+#define regCP_GFX_RS64_DC_APERTURE5_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE5_CNTL0                                                               0x2a5a
+#define regCP_GFX_RS64_DC_APERTURE5_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE6_BASE0                                                               0x2a5b
+#define regCP_GFX_RS64_DC_APERTURE6_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE6_MASK0                                                               0x2a5c
+#define regCP_GFX_RS64_DC_APERTURE6_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE6_CNTL0                                                               0x2a5d
+#define regCP_GFX_RS64_DC_APERTURE6_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE7_BASE0                                                               0x2a5e
+#define regCP_GFX_RS64_DC_APERTURE7_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE7_MASK0                                                               0x2a5f
+#define regCP_GFX_RS64_DC_APERTURE7_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE7_CNTL0                                                               0x2a60
+#define regCP_GFX_RS64_DC_APERTURE7_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE8_BASE0                                                               0x2a61
+#define regCP_GFX_RS64_DC_APERTURE8_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE8_MASK0                                                               0x2a62
+#define regCP_GFX_RS64_DC_APERTURE8_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE8_CNTL0                                                               0x2a63
+#define regCP_GFX_RS64_DC_APERTURE8_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE9_BASE0                                                               0x2a64
+#define regCP_GFX_RS64_DC_APERTURE9_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE9_MASK0                                                               0x2a65
+#define regCP_GFX_RS64_DC_APERTURE9_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE9_CNTL0                                                               0x2a66
+#define regCP_GFX_RS64_DC_APERTURE9_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE10_BASE0                                                              0x2a67
+#define regCP_GFX_RS64_DC_APERTURE10_BASE0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE10_MASK0                                                              0x2a68
+#define regCP_GFX_RS64_DC_APERTURE10_MASK0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE10_CNTL0                                                              0x2a69
+#define regCP_GFX_RS64_DC_APERTURE10_CNTL0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE11_BASE0                                                              0x2a6a
+#define regCP_GFX_RS64_DC_APERTURE11_BASE0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE11_MASK0                                                              0x2a6b
+#define regCP_GFX_RS64_DC_APERTURE11_MASK0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE11_CNTL0                                                              0x2a6c
+#define regCP_GFX_RS64_DC_APERTURE11_CNTL0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE12_BASE0                                                              0x2a6d
+#define regCP_GFX_RS64_DC_APERTURE12_BASE0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE12_MASK0                                                              0x2a6e
+#define regCP_GFX_RS64_DC_APERTURE12_MASK0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE12_CNTL0                                                              0x2a6f
+#define regCP_GFX_RS64_DC_APERTURE12_CNTL0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE13_BASE0                                                              0x2a70
+#define regCP_GFX_RS64_DC_APERTURE13_BASE0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE13_MASK0                                                              0x2a71
+#define regCP_GFX_RS64_DC_APERTURE13_MASK0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE13_CNTL0                                                              0x2a72
+#define regCP_GFX_RS64_DC_APERTURE13_CNTL0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE14_BASE0                                                              0x2a73
+#define regCP_GFX_RS64_DC_APERTURE14_BASE0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE14_MASK0                                                              0x2a74
+#define regCP_GFX_RS64_DC_APERTURE14_MASK0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE14_CNTL0                                                              0x2a75
+#define regCP_GFX_RS64_DC_APERTURE14_CNTL0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE15_BASE0                                                              0x2a76
+#define regCP_GFX_RS64_DC_APERTURE15_BASE0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE15_MASK0                                                              0x2a77
+#define regCP_GFX_RS64_DC_APERTURE15_MASK0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE15_CNTL0                                                              0x2a78
+#define regCP_GFX_RS64_DC_APERTURE15_CNTL0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE0_BASE1                                                               0x2a79
+#define regCP_GFX_RS64_DC_APERTURE0_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE0_MASK1                                                               0x2a7a
+#define regCP_GFX_RS64_DC_APERTURE0_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE0_CNTL1                                                               0x2a7b
+#define regCP_GFX_RS64_DC_APERTURE0_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE1_BASE1                                                               0x2a7c
+#define regCP_GFX_RS64_DC_APERTURE1_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE1_MASK1                                                               0x2a7d
+#define regCP_GFX_RS64_DC_APERTURE1_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE1_CNTL1                                                               0x2a7e
+#define regCP_GFX_RS64_DC_APERTURE1_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE2_BASE1                                                               0x2a7f
+#define regCP_GFX_RS64_DC_APERTURE2_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE2_MASK1                                                               0x2a80
+#define regCP_GFX_RS64_DC_APERTURE2_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE2_CNTL1                                                               0x2a81
+#define regCP_GFX_RS64_DC_APERTURE2_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE3_BASE1                                                               0x2a82
+#define regCP_GFX_RS64_DC_APERTURE3_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE3_MASK1                                                               0x2a83
+#define regCP_GFX_RS64_DC_APERTURE3_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE3_CNTL1                                                               0x2a84
+#define regCP_GFX_RS64_DC_APERTURE3_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE4_BASE1                                                               0x2a85
+#define regCP_GFX_RS64_DC_APERTURE4_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE4_MASK1                                                               0x2a86
+#define regCP_GFX_RS64_DC_APERTURE4_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE4_CNTL1                                                               0x2a87
+#define regCP_GFX_RS64_DC_APERTURE4_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE5_BASE1                                                               0x2a88
+#define regCP_GFX_RS64_DC_APERTURE5_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE5_MASK1                                                               0x2a89
+#define regCP_GFX_RS64_DC_APERTURE5_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE5_CNTL1                                                               0x2a8a
+#define regCP_GFX_RS64_DC_APERTURE5_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE6_BASE1                                                               0x2a8b
+#define regCP_GFX_RS64_DC_APERTURE6_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE6_MASK1                                                               0x2a8c
+#define regCP_GFX_RS64_DC_APERTURE6_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE6_CNTL1                                                               0x2a8d
+#define regCP_GFX_RS64_DC_APERTURE6_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE7_BASE1                                                               0x2a8e
+#define regCP_GFX_RS64_DC_APERTURE7_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE7_MASK1                                                               0x2a8f
+#define regCP_GFX_RS64_DC_APERTURE7_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE7_CNTL1                                                               0x2a90
+#define regCP_GFX_RS64_DC_APERTURE7_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE8_BASE1                                                               0x2a91
+#define regCP_GFX_RS64_DC_APERTURE8_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE8_MASK1                                                               0x2a92
+#define regCP_GFX_RS64_DC_APERTURE8_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE8_CNTL1                                                               0x2a93
+#define regCP_GFX_RS64_DC_APERTURE8_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE9_BASE1                                                               0x2a94
+#define regCP_GFX_RS64_DC_APERTURE9_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE9_MASK1                                                               0x2a95
+#define regCP_GFX_RS64_DC_APERTURE9_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE9_CNTL1                                                               0x2a96
+#define regCP_GFX_RS64_DC_APERTURE9_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE10_BASE1                                                              0x2a97
+#define regCP_GFX_RS64_DC_APERTURE10_BASE1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE10_MASK1                                                              0x2a98
+#define regCP_GFX_RS64_DC_APERTURE10_MASK1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE10_CNTL1                                                              0x2a99
+#define regCP_GFX_RS64_DC_APERTURE10_CNTL1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE11_BASE1                                                              0x2a9a
+#define regCP_GFX_RS64_DC_APERTURE11_BASE1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE11_MASK1                                                              0x2a9b
+#define regCP_GFX_RS64_DC_APERTURE11_MASK1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE11_CNTL1                                                              0x2a9c
+#define regCP_GFX_RS64_DC_APERTURE11_CNTL1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE12_BASE1                                                              0x2a9d
+#define regCP_GFX_RS64_DC_APERTURE12_BASE1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE12_MASK1                                                              0x2a9e
+#define regCP_GFX_RS64_DC_APERTURE12_MASK1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE12_CNTL1                                                              0x2a9f
+#define regCP_GFX_RS64_DC_APERTURE12_CNTL1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE13_BASE1                                                              0x2aa0
+#define regCP_GFX_RS64_DC_APERTURE13_BASE1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE13_MASK1                                                              0x2aa1
+#define regCP_GFX_RS64_DC_APERTURE13_MASK1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE13_CNTL1                                                              0x2aa2
+#define regCP_GFX_RS64_DC_APERTURE13_CNTL1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE14_BASE1                                                              0x2aa3
+#define regCP_GFX_RS64_DC_APERTURE14_BASE1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE14_MASK1                                                              0x2aa4
+#define regCP_GFX_RS64_DC_APERTURE14_MASK1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE14_CNTL1                                                              0x2aa5
+#define regCP_GFX_RS64_DC_APERTURE14_CNTL1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE15_BASE1                                                              0x2aa6
+#define regCP_GFX_RS64_DC_APERTURE15_BASE1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE15_MASK1                                                              0x2aa7
+#define regCP_GFX_RS64_DC_APERTURE15_MASK1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE15_CNTL1                                                              0x2aa8
+#define regCP_GFX_RS64_DC_APERTURE15_CNTL1_BASE_IDX                                                     1
+#define regCP_ME_RS64_EXCEPTION_STATUS                                                                  0x2aaa
+#define regCP_ME_RS64_EXCEPTION_STATUS_BASE_IDX                                                         1
+#define regCP_GFX_RS64_INTERRUPT1                                                                       0x2aac
+#define regCP_GFX_RS64_INTERRUPT1_BASE_IDX                                                              1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_chdec
+// base address: 0x33600
+#define regCH_ARB_CTRL                                                                                  0x2d80
+#define regCH_ARB_CTRL_BASE_IDX                                                                         1
+#define regCH_DRAM_BURST_MASK                                                                           0x2d82
+#define regCH_DRAM_BURST_MASK_BASE_IDX                                                                  1
+#define regCH_ARB_STATUS                                                                                0x2d83
+#define regCH_ARB_STATUS_BASE_IDX                                                                       1
+#define regCH_DRAM_BURST_CTRL                                                                           0x2d84
+#define regCH_DRAM_BURST_CTRL_BASE_IDX                                                                  1
+#define regCHA_CHC_CREDITS                                                                              0x2d88
+#define regCHA_CHC_CREDITS_BASE_IDX                                                                     1
+#define regCHA_CLIENT_FREE_DELAY                                                                        0x2d89
+#define regCHA_CLIENT_FREE_DELAY_BASE_IDX                                                               1
+#define regCHI_CHR_REP_FGCG_OVERRIDE                                                                    0x2d8c
+#define regCHI_CHR_REP_FGCG_OVERRIDE_BASE_IDX                                                           1
+#define regCHC_CTRL                                                                                     0x2dc0
+#define regCHC_CTRL_BASE_IDX                                                                            1
+#define regCHC_STATUS                                                                                   0x2dc1
+#define regCHC_STATUS_BASE_IDX                                                                          1
+#define regCHC_STATUS2                                                                                  0x2dc3
+#define regCHC_STATUS2_BASE_IDX                                                                         1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_glarbdec
+// base address: 0x33c00
+#define regGLARB_ARB_CTRL                                                                               0x2f00
+#define regGLARB_ARB_CTRL_BASE_IDX                                                                      1
+#define regGLARB_DRAM_BURST_MASK                                                                        0x2f02
+#define regGLARB_DRAM_BURST_MASK_BASE_IDX                                                               1
+#define regGLARB_ARB_STATUS                                                                             0x2f03
+#define regGLARB_ARB_STATUS_BASE_IDX                                                                    1
+#define regGLARB_DRAM_BURST_CTRL                                                                        0x2f04
+#define regGLARB_DRAM_BURST_CTRL_BASE_IDX                                                               1
+#define regGLARBA_GLARBC_CREDITS                                                                        0x2f08
+#define regGLARBA_GLARBC_CREDITS_BASE_IDX                                                               1
+#define regGLARBA_CLIENT_FREE_DELAY                                                                     0x2f09
+#define regGLARBA_CLIENT_FREE_DELAY_BASE_IDX                                                            1
+#define regGLARBI_GLARBR_REP_FGCG_OVERRIDE                                                              0x2f0c
+#define regGLARBI_GLARBR_REP_FGCG_OVERRIDE_BASE_IDX                                                     1
+#define regGLARBC_CTRL                                                                                  0x2f40
+#define regGLARBC_CTRL_BASE_IDX                                                                         1
+#define regGLARBC_STATUS                                                                                0x2f41
+#define regGLARBC_STATUS_BASE_IDX                                                                       1
+#define regGLARBC_CTRL2                                                                                 0x2f42
+#define regGLARBC_CTRL2_BASE_IDX                                                                        1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_perfddec
+// base address: 0x34000
+#define regCPG_PERFCOUNTER1_LO                                                                          0x3000
+#define regCPG_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regCPG_PERFCOUNTER1_HI                                                                          0x3001
+#define regCPG_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regCPG_PERFCOUNTER0_LO                                                                          0x3002
+#define regCPG_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regCPG_PERFCOUNTER0_HI                                                                          0x3003
+#define regCPG_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regCPC_PERFCOUNTER1_LO                                                                          0x3004
+#define regCPC_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regCPC_PERFCOUNTER1_HI                                                                          0x3005
+#define regCPC_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regCPC_PERFCOUNTER0_LO                                                                          0x3006
+#define regCPC_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regCPC_PERFCOUNTER0_HI                                                                          0x3007
+#define regCPC_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regCPF_PERFCOUNTER1_LO                                                                          0x3008
+#define regCPF_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regCPF_PERFCOUNTER1_HI                                                                          0x3009
+#define regCPF_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regCPF_PERFCOUNTER0_LO                                                                          0x300a
+#define regCPF_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regCPF_PERFCOUNTER0_HI                                                                          0x300b
+#define regCPF_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regCPF_LATENCY_STATS_DATA                                                                       0x300c
+#define regCPF_LATENCY_STATS_DATA_BASE_IDX                                                              1
+#define regCPG_LATENCY_STATS_DATA                                                                       0x300d
+#define regCPG_LATENCY_STATS_DATA_BASE_IDX                                                              1
+#define regCPC_LATENCY_STATS_DATA                                                                       0x300e
+#define regCPC_LATENCY_STATS_DATA_BASE_IDX                                                              1
+#define regGRBM_PERFCOUNTER0_LO                                                                         0x3040
+#define regGRBM_PERFCOUNTER0_LO_BASE_IDX                                                                1
+#define regGRBM_PERFCOUNTER0_HI                                                                         0x3041
+#define regGRBM_PERFCOUNTER0_HI_BASE_IDX                                                                1
+#define regGRBM_PERFCOUNTER1_LO                                                                         0x3043
+#define regGRBM_PERFCOUNTER1_LO_BASE_IDX                                                                1
+#define regGRBM_PERFCOUNTER1_HI                                                                         0x3044
+#define regGRBM_PERFCOUNTER1_HI_BASE_IDX                                                                1
+#define regGE1_PERFCOUNTER0_LO                                                                          0x30a4
+#define regGE1_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regGE1_PERFCOUNTER0_HI                                                                          0x30a5
+#define regGE1_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regGE1_PERFCOUNTER1_LO                                                                          0x30a6
+#define regGE1_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regGE1_PERFCOUNTER1_HI                                                                          0x30a7
+#define regGE1_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regGE1_PERFCOUNTER2_LO                                                                          0x30a8
+#define regGE1_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regGE1_PERFCOUNTER2_HI                                                                          0x30a9
+#define regGE1_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regGE1_PERFCOUNTER3_LO                                                                          0x30aa
+#define regGE1_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regGE1_PERFCOUNTER3_HI                                                                          0x30ab
+#define regGE1_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regGE2_DIST_PERFCOUNTER0_LO                                                                     0x30ac
+#define regGE2_DIST_PERFCOUNTER0_LO_BASE_IDX                                                            1
+#define regGE2_DIST_PERFCOUNTER0_HI                                                                     0x30ad
+#define regGE2_DIST_PERFCOUNTER0_HI_BASE_IDX                                                            1
+#define regGE2_DIST_PERFCOUNTER1_LO                                                                     0x30ae
+#define regGE2_DIST_PERFCOUNTER1_LO_BASE_IDX                                                            1
+#define regGE2_DIST_PERFCOUNTER1_HI                                                                     0x30af
+#define regGE2_DIST_PERFCOUNTER1_HI_BASE_IDX                                                            1
+#define regGE2_DIST_PERFCOUNTER2_LO                                                                     0x30b0
+#define regGE2_DIST_PERFCOUNTER2_LO_BASE_IDX                                                            1
+#define regGE2_DIST_PERFCOUNTER2_HI                                                                     0x30b1
+#define regGE2_DIST_PERFCOUNTER2_HI_BASE_IDX                                                            1
+#define regGE2_DIST_PERFCOUNTER3_LO                                                                     0x30b2
+#define regGE2_DIST_PERFCOUNTER3_LO_BASE_IDX                                                            1
+#define regGE2_DIST_PERFCOUNTER3_HI                                                                     0x30b3
+#define regGE2_DIST_PERFCOUNTER3_HI_BASE_IDX                                                            1
+#define regGC_EA_CPWD_PERFCOUNTER0_LO                                                                   0x3270
+#define regGC_EA_CPWD_PERFCOUNTER0_LO_BASE_IDX                                                          1
+#define regGC_EA_CPWD_PERFCOUNTER0_HI                                                                   0x3271
+#define regGC_EA_CPWD_PERFCOUNTER0_HI_BASE_IDX                                                          1
+#define regGC_EA_CPWD_PERFCOUNTER1_LO                                                                   0x3272
+#define regGC_EA_CPWD_PERFCOUNTER1_LO_BASE_IDX                                                          1
+#define regGC_EA_CPWD_PERFCOUNTER1_HI                                                                   0x3273
+#define regGC_EA_CPWD_PERFCOUNTER1_HI_BASE_IDX                                                          1
+#define regCHC_PERFCOUNTER0_LO                                                                          0x33c0
+#define regCHC_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regCHC_PERFCOUNTER0_HI                                                                          0x33c1
+#define regCHC_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regCHC_PERFCOUNTER1_LO                                                                          0x33c2
+#define regCHC_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regCHC_PERFCOUNTER1_HI                                                                          0x33c3
+#define regCHC_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regCHC_PERFCOUNTER2_LO                                                                          0x33c4
+#define regCHC_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regCHC_PERFCOUNTER2_HI                                                                          0x33c5
+#define regCHC_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regCHC_PERFCOUNTER3_LO                                                                          0x33c6
+#define regCHC_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regCHC_PERFCOUNTER3_HI                                                                          0x33c7
+#define regCHC_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regGLARBA_PERFCOUNTER0_LO                                                                       0x33e0
+#define regGLARBA_PERFCOUNTER0_LO_BASE_IDX                                                              1
+#define regGLARBA_PERFCOUNTER0_HI                                                                       0x33e1
+#define regGLARBA_PERFCOUNTER0_HI_BASE_IDX                                                              1
+#define regGLARBA_PERFCOUNTER1_LO                                                                       0x33e2
+#define regGLARBA_PERFCOUNTER1_LO_BASE_IDX                                                              1
+#define regGLARBA_PERFCOUNTER1_HI                                                                       0x33e3
+#define regGLARBA_PERFCOUNTER1_HI_BASE_IDX                                                              1
+#define regGLARBA_PERFCOUNTER2_LO                                                                       0x33e4
+#define regGLARBA_PERFCOUNTER2_LO_BASE_IDX                                                              1
+#define regGLARBA_PERFCOUNTER2_HI                                                                       0x33e5
+#define regGLARBA_PERFCOUNTER2_HI_BASE_IDX                                                              1
+#define regGLARBA_PERFCOUNTER3_LO                                                                       0x33e6
+#define regGLARBA_PERFCOUNTER3_LO_BASE_IDX                                                              1
+#define regGLARBA_PERFCOUNTER3_HI                                                                       0x33e7
+#define regGLARBA_PERFCOUNTER3_HI_BASE_IDX                                                              1
+#define regGLARBC_PERFCOUNTER0_LO                                                                       0x33f0
+#define regGLARBC_PERFCOUNTER0_LO_BASE_IDX                                                              1
+#define regGLARBC_PERFCOUNTER0_HI                                                                       0x33f1
+#define regGLARBC_PERFCOUNTER0_HI_BASE_IDX                                                              1
+#define regGLARBC_PERFCOUNTER1_LO                                                                       0x33f2
+#define regGLARBC_PERFCOUNTER1_LO_BASE_IDX                                                              1
+#define regGLARBC_PERFCOUNTER1_HI                                                                       0x33f3
+#define regGLARBC_PERFCOUNTER1_HI_BASE_IDX                                                              1
+#define regGLARBC_PERFCOUNTER2_LO                                                                       0x33f4
+#define regGLARBC_PERFCOUNTER2_LO_BASE_IDX                                                              1
+#define regGLARBC_PERFCOUNTER2_HI                                                                       0x33f5
+#define regGLARBC_PERFCOUNTER2_HI_BASE_IDX                                                              1
+#define regGLARBC_PERFCOUNTER3_LO                                                                       0x33f6
+#define regGLARBC_PERFCOUNTER3_LO_BASE_IDX                                                              1
+#define regGLARBC_PERFCOUNTER3_HI                                                                       0x33f7
+#define regGLARBC_PERFCOUNTER3_HI_BASE_IDX                                                              1
+#define regRLC_PERFCOUNTER0_LO                                                                          0x3480
+#define regRLC_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regRLC_PERFCOUNTER0_HI                                                                          0x3481
+#define regRLC_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regRLC_PERFCOUNTER1_LO                                                                          0x3482
+#define regRLC_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regRLC_PERFCOUNTER1_HI                                                                          0x3483
+#define regRLC_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regGCR_PERFCOUNTER0_LO                                                                          0x3520
+#define regGCR_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regGCR_PERFCOUNTER0_HI                                                                          0x3521
+#define regGCR_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regGCR_PERFCOUNTER1_LO                                                                          0x3522
+#define regGCR_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regGCR_PERFCOUNTER1_HI                                                                          0x3523
+#define regGCR_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regCHA_PERFCOUNTER0_LO                                                                          0x3600
+#define regCHA_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regCHA_PERFCOUNTER0_HI                                                                          0x3601
+#define regCHA_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regCHA_PERFCOUNTER1_LO                                                                          0x3602
+#define regCHA_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regCHA_PERFCOUNTER1_HI                                                                          0x3603
+#define regCHA_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regCHA_PERFCOUNTER2_LO                                                                          0x3604
+#define regCHA_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regCHA_PERFCOUNTER2_HI                                                                          0x3605
+#define regCHA_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regCHA_PERFCOUNTER3_LO                                                                          0x3606
+#define regCHA_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regCHA_PERFCOUNTER3_HI                                                                          0x3607
+#define regCHA_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_perfsdec
+// base address: 0x36000
+#define regCPG_PERFCOUNTER1_SELECT                                                                      0x3800
+#define regCPG_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regCPG_PERFCOUNTER0_SELECT1                                                                     0x3801
+#define regCPG_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regCPG_PERFCOUNTER0_SELECT                                                                      0x3802
+#define regCPG_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regCPC_PERFCOUNTER1_SELECT                                                                      0x3803
+#define regCPC_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regCPC_PERFCOUNTER0_SELECT1                                                                     0x3804
+#define regCPC_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regCPF_PERFCOUNTER1_SELECT                                                                      0x3805
+#define regCPF_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regCPF_PERFCOUNTER0_SELECT1                                                                     0x3806
+#define regCPF_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regCPF_PERFCOUNTER0_SELECT                                                                      0x3807
+#define regCPF_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regCP_PERFMON_CNTL                                                                              0x3808
+#define regCP_PERFMON_CNTL_BASE_IDX                                                                     1
+#define regCPC_PERFCOUNTER0_SELECT                                                                      0x3809
+#define regCPC_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regCPF_TC_PERF_COUNTER_WINDOW_SELECT                                                            0x380a
+#define regCPF_TC_PERF_COUNTER_WINDOW_SELECT_BASE_IDX                                                   1
+#define regCPG_TC_PERF_COUNTER_WINDOW_SELECT                                                            0x380b
+#define regCPG_TC_PERF_COUNTER_WINDOW_SELECT_BASE_IDX                                                   1
+#define regCPF_LATENCY_STATS_SELECT                                                                     0x380c
+#define regCPF_LATENCY_STATS_SELECT_BASE_IDX                                                            1
+#define regCPG_LATENCY_STATS_SELECT                                                                     0x380d
+#define regCPG_LATENCY_STATS_SELECT_BASE_IDX                                                            1
+#define regCPC_LATENCY_STATS_SELECT                                                                     0x380e
+#define regCPC_LATENCY_STATS_SELECT_BASE_IDX                                                            1
+#define regCPC_TC_PERF_COUNTER_WINDOW_SELECT                                                            0x380f
+#define regCPC_TC_PERF_COUNTER_WINDOW_SELECT_BASE_IDX                                                   1
+#define regCP_DRAW_OBJECT                                                                               0x3810
+#define regCP_DRAW_OBJECT_BASE_IDX                                                                      1
+#define regCP_DRAW_OBJECT_COUNTER                                                                       0x3811
+#define regCP_DRAW_OBJECT_COUNTER_BASE_IDX                                                              1
+#define regCP_DRAW_WINDOW_MASK_HI                                                                       0x3812
+#define regCP_DRAW_WINDOW_MASK_HI_BASE_IDX                                                              1
+#define regCP_DRAW_WINDOW_HI                                                                            0x3813
+#define regCP_DRAW_WINDOW_HI_BASE_IDX                                                                   1
+#define regCP_DRAW_WINDOW_LO                                                                            0x3814
+#define regCP_DRAW_WINDOW_LO_BASE_IDX                                                                   1
+#define regCP_DRAW_WINDOW_CNTL                                                                          0x3815
+#define regCP_DRAW_WINDOW_CNTL_BASE_IDX                                                                 1
+#define regGRBM_PERFCOUNTER0_SELECT                                                                     0x3840
+#define regGRBM_PERFCOUNTER0_SELECT_BASE_IDX                                                            1
+#define regGRBM_PERFCOUNTER1_SELECT                                                                     0x3841
+#define regGRBM_PERFCOUNTER1_SELECT_BASE_IDX                                                            1
+#define regGRBM_PERFCOUNTER0_SELECT_HI                                                                  0x384d
+#define regGRBM_PERFCOUNTER0_SELECT_HI_BASE_IDX                                                         1
+#define regGRBM_PERFCOUNTER1_SELECT_HI                                                                  0x384e
+#define regGRBM_PERFCOUNTER1_SELECT_HI_BASE_IDX                                                         1
+#define regGE1_PERFCOUNTER0_SELECT                                                                      0x38a4
+#define regGE1_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regGE1_PERFCOUNTER0_SELECT1                                                                     0x38a5
+#define regGE1_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regGE1_PERFCOUNTER1_SELECT                                                                      0x38a6
+#define regGE1_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regGE1_PERFCOUNTER1_SELECT1                                                                     0x38a7
+#define regGE1_PERFCOUNTER1_SELECT1_BASE_IDX                                                            1
+#define regGE1_PERFCOUNTER2_SELECT                                                                      0x38a8
+#define regGE1_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regGE1_PERFCOUNTER2_SELECT1                                                                     0x38a9
+#define regGE1_PERFCOUNTER2_SELECT1_BASE_IDX                                                            1
+#define regGE1_PERFCOUNTER3_SELECT                                                                      0x38aa
+#define regGE1_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regGE1_PERFCOUNTER3_SELECT1                                                                     0x38ab
+#define regGE1_PERFCOUNTER3_SELECT1_BASE_IDX                                                            1
+#define regGE2_DIST_PERFCOUNTER0_SELECT                                                                 0x38ac
+#define regGE2_DIST_PERFCOUNTER0_SELECT_BASE_IDX                                                        1
+#define regGE2_DIST_PERFCOUNTER0_SELECT1                                                                0x38ad
+#define regGE2_DIST_PERFCOUNTER0_SELECT1_BASE_IDX                                                       1
+#define regGE2_DIST_PERFCOUNTER1_SELECT                                                                 0x38ae
+#define regGE2_DIST_PERFCOUNTER1_SELECT_BASE_IDX                                                        1
+#define regGE2_DIST_PERFCOUNTER1_SELECT1                                                                0x38af
+#define regGE2_DIST_PERFCOUNTER1_SELECT1_BASE_IDX                                                       1
+#define regGE2_DIST_PERFCOUNTER2_SELECT                                                                 0x38b0
+#define regGE2_DIST_PERFCOUNTER2_SELECT_BASE_IDX                                                        1
+#define regGE2_DIST_PERFCOUNTER2_SELECT1                                                                0x38b1
+#define regGE2_DIST_PERFCOUNTER2_SELECT1_BASE_IDX                                                       1
+#define regGE2_DIST_PERFCOUNTER3_SELECT                                                                 0x38b2
+#define regGE2_DIST_PERFCOUNTER3_SELECT_BASE_IDX                                                        1
+#define regGE2_DIST_PERFCOUNTER3_SELECT1                                                                0x38b3
+#define regGE2_DIST_PERFCOUNTER3_SELECT1_BASE_IDX                                                       1
+#define regGC_EA_CPWD_PERFCOUNTER0_SELECT                                                               0x3a20
+#define regGC_EA_CPWD_PERFCOUNTER0_SELECT_BASE_IDX                                                      1
+#define regGC_EA_CPWD_PERFCOUNTER0_SELECT1                                                              0x3a21
+#define regGC_EA_CPWD_PERFCOUNTER0_SELECT1_BASE_IDX                                                     1
+#define regGC_EA_CPWD_PERFCOUNTER1_SELECT                                                               0x3a22
+#define regGC_EA_CPWD_PERFCOUNTER1_SELECT_BASE_IDX                                                      1
+#define regCHC_PERFCOUNTER0_SELECT                                                                      0x3bc0
+#define regCHC_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regCHC_PERFCOUNTER0_SELECT1                                                                     0x3bc1
+#define regCHC_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regCHC_PERFCOUNTER1_SELECT                                                                      0x3bc2
+#define regCHC_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regCHC_PERFCOUNTER1_SELECT1                                                                     0x3bc3
+#define regCHC_PERFCOUNTER1_SELECT1_BASE_IDX                                                            1
+#define regCHC_PERFCOUNTER2_SELECT                                                                      0x3bc4
+#define regCHC_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regCHC_PERFCOUNTER2_SELECT1                                                                     0x3bc5
+#define regCHC_PERFCOUNTER2_SELECT1_BASE_IDX                                                            1
+#define regCHC_PERFCOUNTER3_SELECT                                                                      0x3bc6
+#define regCHC_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regCHC_PERFCOUNTER3_SELECT1                                                                     0x3bc7
+#define regCHC_PERFCOUNTER3_SELECT1_BASE_IDX                                                            1
+#define regGLARBA_PERFCOUNTER0_SELECT                                                                   0x3be0
+#define regGLARBA_PERFCOUNTER0_SELECT_BASE_IDX                                                          1
+#define regGLARBA_PERFCOUNTER0_SELECT1                                                                  0x3be1
+#define regGLARBA_PERFCOUNTER0_SELECT1_BASE_IDX                                                         1
+#define regGLARBA_PERFCOUNTER1_SELECT                                                                   0x3be2
+#define regGLARBA_PERFCOUNTER1_SELECT_BASE_IDX                                                          1
+#define regGLARBA_PERFCOUNTER1_SELECT1                                                                  0x3be3
+#define regGLARBA_PERFCOUNTER1_SELECT1_BASE_IDX                                                         1
+#define regGLARBA_PERFCOUNTER2_SELECT                                                                   0x3be4
+#define regGLARBA_PERFCOUNTER2_SELECT_BASE_IDX                                                          1
+#define regGLARBA_PERFCOUNTER2_SELECT1                                                                  0x3be5
+#define regGLARBA_PERFCOUNTER2_SELECT1_BASE_IDX                                                         1
+#define regGLARBA_PERFCOUNTER3_SELECT                                                                   0x3be6
+#define regGLARBA_PERFCOUNTER3_SELECT_BASE_IDX                                                          1
+#define regGLARBA_PERFCOUNTER3_SELECT1                                                                  0x3be7
+#define regGLARBA_PERFCOUNTER3_SELECT1_BASE_IDX                                                         1
+#define regGLARBC_PERFCOUNTER0_SELECT                                                                   0x3bf0
+#define regGLARBC_PERFCOUNTER0_SELECT_BASE_IDX                                                          1
+#define regGLARBC_PERFCOUNTER0_SELECT1                                                                  0x3bf1
+#define regGLARBC_PERFCOUNTER0_SELECT1_BASE_IDX                                                         1
+#define regGLARBC_PERFCOUNTER1_SELECT                                                                   0x3bf2
+#define regGLARBC_PERFCOUNTER1_SELECT_BASE_IDX                                                          1
+#define regGLARBC_PERFCOUNTER1_SELECT1                                                                  0x3bf3
+#define regGLARBC_PERFCOUNTER1_SELECT1_BASE_IDX                                                         1
+#define regGLARBC_PERFCOUNTER2_SELECT                                                                   0x3bf4
+#define regGLARBC_PERFCOUNTER2_SELECT_BASE_IDX                                                          1
+#define regGLARBC_PERFCOUNTER2_SELECT1                                                                  0x3bf5
+#define regGLARBC_PERFCOUNTER2_SELECT1_BASE_IDX                                                         1
+#define regGLARBC_PERFCOUNTER3_SELECT                                                                   0x3bf6
+#define regGLARBC_PERFCOUNTER3_SELECT_BASE_IDX                                                          1
+#define regGLARBC_PERFCOUNTER3_SELECT1                                                                  0x3bf7
+#define regGLARBC_PERFCOUNTER3_SELECT1_BASE_IDX                                                         1
+#define regRLC_SPM_PERFMON_CNTL                                                                         0x3c80
+#define regRLC_SPM_PERFMON_CNTL_BASE_IDX                                                                1
+#define regRLC_SPM_PERFMON_RING_BASE_LO                                                                 0x3c81
+#define regRLC_SPM_PERFMON_RING_BASE_LO_BASE_IDX                                                        1
+#define regRLC_SPM_PERFMON_RING_BASE_HI                                                                 0x3c82
+#define regRLC_SPM_PERFMON_RING_BASE_HI_BASE_IDX                                                        1
+#define regRLC_SPM_PERFMON_RING_SIZE                                                                    0x3c83
+#define regRLC_SPM_PERFMON_RING_SIZE_BASE_IDX                                                           1
+#define regRLC_SPM_RING_WRPTR                                                                           0x3c84
+#define regRLC_SPM_RING_WRPTR_BASE_IDX                                                                  1
+#define regRLC_SPM_RING_RDPTR                                                                           0x3c85
+#define regRLC_SPM_RING_RDPTR_BASE_IDX                                                                  1
+#define regRLC_SPM_SEGMENT_THRESHOLD                                                                    0x3c86
+#define regRLC_SPM_SEGMENT_THRESHOLD_BASE_IDX                                                           1
+#define regRLC_SPM_PERFMON_SEGMENT_SIZE                                                                 0x3c87
+#define regRLC_SPM_PERFMON_SEGMENT_SIZE_BASE_IDX                                                        1
+#define regRLC_SPM_GLOBAL_MUXSEL_ADDR                                                                   0x3c88
+#define regRLC_SPM_GLOBAL_MUXSEL_ADDR_BASE_IDX                                                          1
+#define regRLC_SPM_GLOBAL_MUXSEL_DATA                                                                   0x3c89
+#define regRLC_SPM_GLOBAL_MUXSEL_DATA_BASE_IDX                                                          1
+#define regRLC_SPM_SE_MUXSEL_ADDR                                                                       0x3c8a
+#define regRLC_SPM_SE_MUXSEL_ADDR_BASE_IDX                                                              1
+#define regRLC_SPM_SE_MUXSEL_DATA                                                                       0x3c8b
+#define regRLC_SPM_SE_MUXSEL_DATA_BASE_IDX                                                              1
+#define regRLC_SPM_SE_USER_DATA_0                                                                       0x3c8c
+#define regRLC_SPM_SE_USER_DATA_0_BASE_IDX                                                              1
+#define regRLC_SPM_SE_USER_DATA_1                                                                       0x3c8d
+#define regRLC_SPM_SE_USER_DATA_1_BASE_IDX                                                              1
+#define regRLC_SPM_SE_USER_DATA_2                                                                       0x3c8e
+#define regRLC_SPM_SE_USER_DATA_2_BASE_IDX                                                              1
+#define regRLC_SPM_SE_USER_DATA_3                                                                       0x3c8f
+#define regRLC_SPM_SE_USER_DATA_3_BASE_IDX                                                              1
+#define regRLC_SPM_ACCUM_DATARAM_ADDR                                                                   0x3c92
+#define regRLC_SPM_ACCUM_DATARAM_ADDR_BASE_IDX                                                          1
+#define regRLC_SPM_ACCUM_DATARAM_DATA                                                                   0x3c93
+#define regRLC_SPM_ACCUM_DATARAM_DATA_BASE_IDX                                                          1
+#define regRLC_SPM_ACCUM_SWA_DATARAM_ADDR                                                               0x3c94
+#define regRLC_SPM_ACCUM_SWA_DATARAM_ADDR_BASE_IDX                                                      1
+#define regRLC_SPM_ACCUM_SWA_DATARAM_DATA                                                               0x3c95
+#define regRLC_SPM_ACCUM_SWA_DATARAM_DATA_BASE_IDX                                                      1
+#define regRLC_SPM_ACCUM_CTRLRAM_ADDR                                                                   0x3c96
+#define regRLC_SPM_ACCUM_CTRLRAM_ADDR_BASE_IDX                                                          1
+#define regRLC_SPM_ACCUM_CTRLRAM_DATA                                                                   0x3c97
+#define regRLC_SPM_ACCUM_CTRLRAM_DATA_BASE_IDX                                                          1
+#define regRLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET                                                            0x3c98
+#define regRLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET_BASE_IDX                                                   1
+#define regRLC_SPM_ACCUM_STATUS                                                                         0x3c99
+#define regRLC_SPM_ACCUM_STATUS_BASE_IDX                                                                1
+#define regRLC_SPM_ACCUM_CTRL                                                                           0x3c9a
+#define regRLC_SPM_ACCUM_CTRL_BASE_IDX                                                                  1
+#define regRLC_SPM_ACCUM_MODE                                                                           0x3c9b
+#define regRLC_SPM_ACCUM_MODE_BASE_IDX                                                                  1
+#define regRLC_SPM_ACCUM_THRESHOLD                                                                      0x3c9c
+#define regRLC_SPM_ACCUM_THRESHOLD_BASE_IDX                                                             1
+#define regRLC_SPM_ACCUM_SAMPLES_REQUESTED                                                              0x3c9d
+#define regRLC_SPM_ACCUM_SAMPLES_REQUESTED_BASE_IDX                                                     1
+#define regRLC_SPM_ACCUM_DATARAM_WRCOUNT                                                                0x3c9e
+#define regRLC_SPM_ACCUM_DATARAM_WRCOUNT_BASE_IDX                                                       1
+#define regRLC_SPM_ACCUM_DATARAM_32BITCNTRS_REGIONS                                                     0x3c9f
+#define regRLC_SPM_ACCUM_DATARAM_32BITCNTRS_REGIONS_BASE_IDX                                            1
+#define regRLC_SPM_PAUSE                                                                                0x3ca2
+#define regRLC_SPM_PAUSE_BASE_IDX                                                                       1
+#define regRLC_SPM_STATUS                                                                               0x3ca3
+#define regRLC_SPM_STATUS_BASE_IDX                                                                      1
+#define regRLC_SPM_GFXCLOCK_LOWCOUNT                                                                    0x3ca4
+#define regRLC_SPM_GFXCLOCK_LOWCOUNT_BASE_IDX                                                           1
+#define regRLC_SPM_GFXCLOCK_HIGHCOUNT                                                                   0x3ca5
+#define regRLC_SPM_GFXCLOCK_HIGHCOUNT_BASE_IDX                                                          1
+#define regRLC_SPM_GTS_TRIGGER_VALUE_LO                                                                 0x3ca6
+#define regRLC_SPM_GTS_TRIGGER_VALUE_LO_BASE_IDX                                                        1
+#define regRLC_SPM_GTS_TRIGGER_VALUE_HI                                                                 0x3ca7
+#define regRLC_SPM_GTS_TRIGGER_VALUE_HI_BASE_IDX                                                        1
+#define regRLC_SPM_GLOBAL_USER_DATA_0                                                                   0x3ca8
+#define regRLC_SPM_GLOBAL_USER_DATA_0_BASE_IDX                                                          1
+#define regRLC_SPM_GLOBAL_USER_DATA_1                                                                   0x3ca9
+#define regRLC_SPM_GLOBAL_USER_DATA_1_BASE_IDX                                                          1
+#define regRLC_SPM_GLOBAL_USER_DATA_2                                                                   0x3caa
+#define regRLC_SPM_GLOBAL_USER_DATA_2_BASE_IDX                                                          1
+#define regRLC_SPM_GLOBAL_USER_DATA_3                                                                   0x3cab
+#define regRLC_SPM_GLOBAL_USER_DATA_3_BASE_IDX                                                          1
+#define regRLC_SPM_MODE                                                                                 0x3cad
+#define regRLC_SPM_MODE_BASE_IDX                                                                        1
+#define regRLC_SPM_RSPM_REQ_DATA                                                                        0x3cae
+#define regRLC_SPM_RSPM_REQ_DATA_BASE_IDX                                                               1
+#define regRLC_SPM_RSPM_REQ_OP                                                                          0x3cb0
+#define regRLC_SPM_RSPM_REQ_OP_BASE_IDX                                                                 1
+#define regRLC_SPM_RSPM_RET_DATA                                                                        0x3cb1
+#define regRLC_SPM_RSPM_RET_DATA_BASE_IDX                                                               1
+#define regRLC_SPM_RSPM_RET_OP                                                                          0x3cb2
+#define regRLC_SPM_RSPM_RET_OP_BASE_IDX                                                                 1
+#define regRLC_SPM_SE_RSPM_REQ_DATA                                                                     0x3cb3
+#define regRLC_SPM_SE_RSPM_REQ_DATA_BASE_IDX                                                            1
+#define regRLC_SPM_SE_RSPM_REQ_OP                                                                       0x3cb5
+#define regRLC_SPM_SE_RSPM_REQ_OP_BASE_IDX                                                              1
+#define regRLC_SPM_SE_RSPM_RET_DATA                                                                     0x3cb6
+#define regRLC_SPM_SE_RSPM_RET_DATA_BASE_IDX                                                            1
+#define regRLC_SPM_SE_RSPM_RET_OP                                                                       0x3cb7
+#define regRLC_SPM_SE_RSPM_RET_OP_BASE_IDX                                                              1
+#define regRLC_SPM_RSPM_CMD                                                                             0x3cb8
+#define regRLC_SPM_RSPM_CMD_BASE_IDX                                                                    1
+#define regRLC_SPM_RSPM_CMD_ACK                                                                         0x3cb9
+#define regRLC_SPM_RSPM_CMD_ACK_BASE_IDX                                                                1
+#define regRLC_SPM_SPARE                                                                                0x3cbf
+#define regRLC_SPM_SPARE_BASE_IDX                                                                       1
+#define regRLC_PERFMON_CNTL                                                                             0x3cc0
+#define regRLC_PERFMON_CNTL_BASE_IDX                                                                    1
+#define regRLC_PERFCOUNTER0_SELECT                                                                      0x3cc1
+#define regRLC_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regRLC_PERFCOUNTER1_SELECT                                                                      0x3cc2
+#define regRLC_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regRLC_GPU_IOV_PERF_CNT_CNTL                                                                    0x3cc3
+#define regRLC_GPU_IOV_PERF_CNT_CNTL_BASE_IDX                                                           1
+#define regRLC_GPU_IOV_PERF_CNT_WR_ADDR                                                                 0x3cc4
+#define regRLC_GPU_IOV_PERF_CNT_WR_ADDR_BASE_IDX                                                        1
+#define regRLC_GPU_IOV_PERF_CNT_WR_DATA                                                                 0x3cc5
+#define regRLC_GPU_IOV_PERF_CNT_WR_DATA_BASE_IDX                                                        1
+#define regRLC_GPU_IOV_PERF_CNT_RD_ADDR                                                                 0x3cc6
+#define regRLC_GPU_IOV_PERF_CNT_RD_ADDR_BASE_IDX                                                        1
+#define regRLC_GPU_IOV_PERF_CNT_RD_DATA                                                                 0x3cc7
+#define regRLC_GPU_IOV_PERF_CNT_RD_DATA_BASE_IDX                                                        1
+#define regGCR_PERFCOUNTER0_SELECT                                                                      0x3d60
+#define regGCR_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regGCR_PERFCOUNTER0_SELECT1                                                                     0x3d61
+#define regGCR_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regGCR_PERFCOUNTER1_SELECT                                                                      0x3d62
+#define regGCR_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regGCR_PERFCOUNTER1_SELECT1                                                                     0x3d63
+#define regGCR_PERFCOUNTER1_SELECT1_BASE_IDX                                                            1
+#define regCHA_PERFCOUNTER0_SELECT                                                                      0x3de0
+#define regCHA_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regCHA_PERFCOUNTER0_SELECT1                                                                     0x3de1
+#define regCHA_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regCHA_PERFCOUNTER1_SELECT                                                                      0x3de2
+#define regCHA_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regCHA_PERFCOUNTER1_SELECT1                                                                     0x3de3
+#define regCHA_PERFCOUNTER1_SELECT1_BASE_IDX                                                            1
+#define regCHA_PERFCOUNTER2_SELECT                                                                      0x3de4
+#define regCHA_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regCHA_PERFCOUNTER2_SELECT1                                                                     0x3de5
+#define regCHA_PERFCOUNTER2_SELECT1_BASE_IDX                                                            1
+#define regCHA_PERFCOUNTER3_SELECT                                                                      0x3de6
+#define regCHA_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regCHA_PERFCOUNTER3_SELECT1                                                                     0x3de7
+#define regCHA_PERFCOUNTER3_SELECT1_BASE_IDX                                                            1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_gdfll_xvmin_xvmin_xvmin_reg_blk
+// base address: 0x3a014
+#define regXVMIN_XVMIN_WR_DATA                                                                          0x4806
+#define regXVMIN_XVMIN_WR_DATA_BASE_IDX                                                                 1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_hypdec
+// base address: 0x3e000
+#define regRLC_GPU_IOV_VF_ENABLE                                                                        0x5b00
+#define regRLC_GPU_IOV_VF_ENABLE_BASE_IDX                                                               1
+#define regRLC_FED_DRVR_STATUS                                                                          0x5b04
+#define regRLC_FED_DRVR_STATUS_BASE_IDX                                                                 1
+#define regRLC_GPU_IOV_CFG_REG6                                                                         0x5b06
+#define regRLC_GPU_IOV_CFG_REG6_BASE_IDX                                                                1
+#define regRLC_SDMA0_STATUS                                                                             0x5b18
+#define regRLC_SDMA0_STATUS_BASE_IDX                                                                    1
+#define regRLC_SDMA1_STATUS                                                                             0x5b19
+#define regRLC_SDMA1_STATUS_BASE_IDX                                                                    1
+#define regRLC_SDMA2_STATUS                                                                             0x5b1a
+#define regRLC_SDMA2_STATUS_BASE_IDX                                                                    1
+#define regRLC_SDMA3_STATUS                                                                             0x5b1b
+#define regRLC_SDMA3_STATUS_BASE_IDX                                                                    1
+#define regRLC_SDMA0_BUSY_STATUS                                                                        0x5b1c
+#define regRLC_SDMA0_BUSY_STATUS_BASE_IDX                                                               1
+#define regRLC_SDMA1_BUSY_STATUS                                                                        0x5b1d
+#define regRLC_SDMA1_BUSY_STATUS_BASE_IDX                                                               1
+#define regRLC_SDMA2_BUSY_STATUS                                                                        0x5b1e
+#define regRLC_SDMA2_BUSY_STATUS_BASE_IDX                                                               1
+#define regRLC_SDMA3_BUSY_STATUS                                                                        0x5b1f
+#define regRLC_SDMA3_BUSY_STATUS_BASE_IDX                                                               1
+#define regRLC_GPU_IOV_CFG_REG8                                                                         0x5b20
+#define regRLC_GPU_IOV_CFG_REG8_BASE_IDX                                                                1
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS                                                               0x5b2a
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS_BASE_IDX                                                      1
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS_SET                                                           0x5b2b
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS_SET_BASE_IDX                                                  1
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS_CLR                                                           0x5b2c
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS_CLR_BASE_IDX                                                  1
+#define regRLC_GPU_IOV_VF_MASK                                                                          0x5b2d
+#define regRLC_GPU_IOV_VF_MASK_BASE_IDX                                                                 1
+#define regRLC_HYP_SEMAPHORE_0                                                                          0x5b2e
+#define regRLC_HYP_SEMAPHORE_0_BASE_IDX                                                                 1
+#define regRLC_HYP_SEMAPHORE_1                                                                          0x5b2f
+#define regRLC_HYP_SEMAPHORE_1_BASE_IDX                                                                 1
+#define regRLC_BUSY_CLK_CNTL                                                                            0x5b30
+#define regRLC_BUSY_CLK_CNTL_BASE_IDX                                                                   1
+#define regRLC_CLK_CNTL                                                                                 0x5b31
+#define regRLC_CLK_CNTL_BASE_IDX                                                                        1
+#define regRLC_GPU_IOV_SCH_BLOCK                                                                        0x5b34
+#define regRLC_GPU_IOV_SCH_BLOCK_BASE_IDX                                                               1
+#define regRLC_GPU_IOV_CFG_REG1                                                                         0x5b35
+#define regRLC_GPU_IOV_CFG_REG1_BASE_IDX                                                                1
+#define regRLC_GPU_IOV_CFG_REG2                                                                         0x5b36
+#define regRLC_GPU_IOV_CFG_REG2_BASE_IDX                                                                1
+#define regRLC_GPU_IOV_VM_BUSY_STATUS                                                                   0x5b37
+#define regRLC_GPU_IOV_VM_BUSY_STATUS_BASE_IDX                                                          1
+#define regRLC_GPU_IOV_SCH_0                                                                            0x5b38
+#define regRLC_GPU_IOV_SCH_0_BASE_IDX                                                                   1
+#define regRLC_GPU_IOV_ACTIVE_FCN_ID                                                                    0x5b39
+#define regRLC_GPU_IOV_ACTIVE_FCN_ID_BASE_IDX                                                           1
+#define regRLC_GPU_IOV_SCH_3                                                                            0x5b3a
+#define regRLC_GPU_IOV_SCH_3_BASE_IDX                                                                   1
+#define regRLC_GPU_IOV_SCH_1                                                                            0x5b3b
+#define regRLC_GPU_IOV_SCH_1_BASE_IDX                                                                   1
+#define regRLC_GPU_IOV_SCH_2                                                                            0x5b3c
+#define regRLC_GPU_IOV_SCH_2_BASE_IDX                                                                   1
+#define regRLC_IH_COOKIE                                                                                0x5b41
+#define regRLC_IH_COOKIE_BASE_IDX                                                                       1
+#define regRLC_IH_COOKIE_CNTL                                                                           0x5b42
+#define regRLC_IH_COOKIE_CNTL_BASE_IDX                                                                  1
+#define regRLC_HYP_RLCG_UCODE_CHKSUM                                                                    0x5b43
+#define regRLC_HYP_RLCG_UCODE_CHKSUM_BASE_IDX                                                           1
+#define regRLC_GPU_IOV_SMU_RESPONSE                                                                     0x5b4a
+#define regRLC_GPU_IOV_SMU_RESPONSE_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_VIRT_RESET_REQ                                                                   0x5b4c
+#define regRLC_GPU_IOV_VIRT_RESET_REQ_BASE_IDX                                                          1
+#define regRLC_GPU_IOV_RLC_RESPONSE                                                                     0x5b4d
+#define regRLC_GPU_IOV_RLC_RESPONSE_BASE_IDX                                                            1
+#define regRLC_HYP_SEMAPHORE_2                                                                          0x5b52
+#define regRLC_HYP_SEMAPHORE_2_BASE_IDX                                                                 1
+#define regRLC_HYP_SEMAPHORE_3                                                                          0x5b53
+#define regRLC_HYP_SEMAPHORE_3_BASE_IDX                                                                 1
+#define regRLC_GPM_UCODE_ADDR                                                                           0x5b60
+#define regRLC_GPM_UCODE_ADDR_BASE_IDX                                                                  1
+#define regRLC_GPM_UCODE_DATA                                                                           0x5b61
+#define regRLC_GPM_UCODE_DATA_BASE_IDX                                                                  1
+#define regRLC_GPM_IRAM_ADDR                                                                            0x5b62
+#define regRLC_GPM_IRAM_ADDR_BASE_IDX                                                                   1
+#define regRLC_GPM_IRAM_DATA                                                                            0x5b63
+#define regRLC_GPM_IRAM_DATA_BASE_IDX                                                                   1
+#define regRLC_LX6_DRAM_ADDR                                                                            0x5b68
+#define regRLC_LX6_DRAM_ADDR_BASE_IDX                                                                   1
+#define regRLC_LX6_DRAM_DATA                                                                            0x5b69
+#define regRLC_LX6_DRAM_DATA_BASE_IDX                                                                   1
+#define regRLC_LX6_IRAM_ADDR                                                                            0x5b6a
+#define regRLC_LX6_IRAM_ADDR_BASE_IDX                                                                   1
+#define regRLC_LX6_IRAM_DATA                                                                            0x5b6b
+#define regRLC_LX6_IRAM_DATA_BASE_IDX                                                                   1
+#define regRLC_GPM_SCRATCH_ADDR                                                                         0x5b6e
+#define regRLC_GPM_SCRATCH_ADDR_BASE_IDX                                                                1
+#define regRLC_GPM_SCRATCH_DATA                                                                         0x5b6f
+#define regRLC_GPM_SCRATCH_DATA_BASE_IDX                                                                1
+#define regRLC_SRM_DRAM_ADDR                                                                            0x5b71
+#define regRLC_SRM_DRAM_ADDR_BASE_IDX                                                                   1
+#define regRLC_SRM_DRAM_DATA                                                                            0x5b72
+#define regRLC_SRM_DRAM_DATA_BASE_IDX                                                                   1
+#define regRLC_SRM_ARAM_ADDR                                                                            0x5b73
+#define regRLC_SRM_ARAM_ADDR_BASE_IDX                                                                   1
+#define regRLC_SRM_ARAM_DATA                                                                            0x5b74
+#define regRLC_SRM_ARAM_DATA_BASE_IDX                                                                   1
+#define regRLC_GTS_OFFSET_LSB                                                                           0x5b79
+#define regRLC_GTS_OFFSET_LSB_BASE_IDX                                                                  1
+#define regRLC_GTS_OFFSET_MSB                                                                           0x5b7a
+#define regRLC_GTS_OFFSET_MSB_BASE_IDX                                                                  1
+#define regRLC_GTS_OFFSET_SNAP_LSB                                                                      0x5b7b
+#define regRLC_GTS_OFFSET_SNAP_LSB_BASE_IDX                                                             1
+#define regRLC_GTS_OFFSET_SNAP_MSB                                                                      0x5b7c
+#define regRLC_GTS_OFFSET_SNAP_MSB_BASE_IDX                                                             1
+#define regRLC_GAP_PWROK                                                                                0x5b7d
+#define regRLC_GAP_PWROK_BASE_IDX                                                                       1
+#define regRLC_GAP_PWROK_COUNT                                                                          0x5b7e
+#define regRLC_GAP_PWROK_COUNT_BASE_IDX                                                                 1
+#define regGLARB_HASH_CFG                                                                               0x5b87
+#define regGLARB_HASH_CFG_BASE_IDX                                                                      1
+#define regCH_PIPE_STEER                                                                                0x5b88
+#define regCH_PIPE_STEER_BASE_IDX                                                                       1
+#define regGLARB_PIPE_STEER_LSB                                                                         0x5b89
+#define regGLARB_PIPE_STEER_LSB_BASE_IDX                                                                1
+#define regGLARB_PIPE_STEER_MSB                                                                         0x5b8a
+#define regGLARB_PIPE_STEER_MSB_BASE_IDX                                                                1
+#define regCH_HASH_CFG                                                                                  0x5b8b
+#define regCH_HASH_CFG_BASE_IDX                                                                         1
+#define regGLARB_LPDDR_PIPE_STEER_LSB                                                                   0x5b8c
+#define regGLARB_LPDDR_PIPE_STEER_LSB_BASE_IDX                                                          1
+#define regGLARB_LPDDR_PIPE_STEER_MSB                                                                   0x5b8d
+#define regGLARB_LPDDR_PIPE_STEER_MSB_BASE_IDX                                                          1
+#define regGLARB_LOCAL_CAC                                                                              0x5b8e
+#define regGLARB_LOCAL_CAC_BASE_IDX                                                                     1
+#define regGLARB_AID_SEL_CFG                                                                            0x5b8f
+#define regGLARB_AID_SEL_CFG_BASE_IDX                                                                   1
+#define regGC_USER_FULL_SA_UNIT_DISABLE                                                                 0x5b91
+#define regGC_USER_FULL_SA_UNIT_DISABLE_BASE_IDX                                                        1
+#define regGC_USER_SA_UNIT_DISABLE                                                                      0x5b92
+#define regGC_USER_SA_UNIT_DISABLE_BASE_IDX                                                             1
+#define regGC_USER_HBM_DISABLE                                                                          0x5b93
+#define regGC_USER_HBM_DISABLE_BASE_IDX                                                                 1
+#define regRLC_GPU_IOV_SDMA0_STATUS                                                                     0x5bc0
+#define regRLC_GPU_IOV_SDMA0_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA1_STATUS                                                                     0x5bc1
+#define regRLC_GPU_IOV_SDMA1_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA2_STATUS                                                                     0x5bc2
+#define regRLC_GPU_IOV_SDMA2_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA3_STATUS                                                                     0x5bc3
+#define regRLC_GPU_IOV_SDMA3_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA4_STATUS                                                                     0x5bc4
+#define regRLC_GPU_IOV_SDMA4_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA5_STATUS                                                                     0x5bc5
+#define regRLC_GPU_IOV_SDMA5_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA6_STATUS                                                                     0x5bc6
+#define regRLC_GPU_IOV_SDMA6_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA7_STATUS                                                                     0x5bc7
+#define regRLC_GPU_IOV_SDMA7_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA0_BUSY_STATUS                                                                0x5bc8
+#define regRLC_GPU_IOV_SDMA0_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA1_BUSY_STATUS                                                                0x5bc9
+#define regRLC_GPU_IOV_SDMA1_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA2_BUSY_STATUS                                                                0x5bca
+#define regRLC_GPU_IOV_SDMA2_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA3_BUSY_STATUS                                                                0x5bcb
+#define regRLC_GPU_IOV_SDMA3_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA4_BUSY_STATUS                                                                0x5bcc
+#define regRLC_GPU_IOV_SDMA4_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA5_BUSY_STATUS                                                                0x5bcd
+#define regRLC_GPU_IOV_SDMA5_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA6_BUSY_STATUS                                                                0x5bce
+#define regRLC_GPU_IOV_SDMA6_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA7_BUSY_STATUS                                                                0x5bcf
+#define regRLC_GPU_IOV_SDMA7_BUSY_STATUS_BASE_IDX                                                       1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_cphypdec
+// base address: 0x3e000
+#define regCP_HYP_CONTEXT_RANGE_BASE                                                                    0x580a
+#define regCP_HYP_CONTEXT_RANGE_BASE_BASE_IDX                                                           1
+#define regCP_HYP_CONTEXT_RANGE_END                                                                     0x580b
+#define regCP_HYP_CONTEXT_RANGE_END_BASE_IDX                                                            1
+#define regCP_HYP_PFP_UCODE_ADDR                                                                        0x5814
+#define regCP_HYP_PFP_UCODE_ADDR_BASE_IDX                                                               1
+#define regCP_PFP_UCODE_ADDR                                                                            0x5814
+#define regCP_PFP_UCODE_ADDR_BASE_IDX                                                                   1
+#define regCP_HYP_PFP_UCODE_DATA                                                                        0x5815
+#define regCP_HYP_PFP_UCODE_DATA_BASE_IDX                                                               1
+#define regCP_PFP_UCODE_DATA                                                                            0x5815
+#define regCP_PFP_UCODE_DATA_BASE_IDX                                                                   1
+#define regCP_HYP_ME_UCODE_ADDR                                                                         0x5816
+#define regCP_HYP_ME_UCODE_ADDR_BASE_IDX                                                                1
+#define regCP_ME_RAM_RADDR                                                                              0x5816
+#define regCP_ME_RAM_RADDR_BASE_IDX                                                                     1
+#define regCP_ME_RAM_WADDR                                                                              0x5816
+#define regCP_ME_RAM_WADDR_BASE_IDX                                                                     1
+#define regCP_HYP_ME_UCODE_DATA                                                                         0x5817
+#define regCP_HYP_ME_UCODE_DATA_BASE_IDX                                                                1
+#define regCP_ME_RAM_DATA                                                                               0x5817
+#define regCP_ME_RAM_DATA_BASE_IDX                                                                      1
+#define regCP_HYP_MEC1_UCODE_ADDR                                                                       0x581a
+#define regCP_HYP_MEC1_UCODE_ADDR_BASE_IDX                                                              1
+#define regCP_MEC_ME1_UCODE_ADDR                                                                        0x581a
+#define regCP_MEC_ME1_UCODE_ADDR_BASE_IDX                                                               1
+#define regCP_HYP_MEC1_UCODE_DATA                                                                       0x581b
+#define regCP_HYP_MEC1_UCODE_DATA_BASE_IDX                                                              1
+#define regCP_MEC_ME1_UCODE_DATA                                                                        0x581b
+#define regCP_MEC_ME1_UCODE_DATA_BASE_IDX                                                               1
+#define regCP_HYP_PFP_UCODE_CHKSUM                                                                      0x581e
+#define regCP_HYP_PFP_UCODE_CHKSUM_BASE_IDX                                                             1
+#define regCP_HYP_ME_UCODE_CHKSUM                                                                       0x5820
+#define regCP_HYP_ME_UCODE_CHKSUM_BASE_IDX                                                              1
+#define regCP_HYP_MEC_ME1_UCODE_CHKSUM                                                                  0x5821
+#define regCP_HYP_MEC_ME1_UCODE_CHKSUM_BASE_IDX                                                         1
+#define regCP_PFP_IC_BASE_LO                                                                            0x5840
+#define regCP_PFP_IC_BASE_LO_BASE_IDX                                                                   1
+#define regCP_PFP_IC_BASE_HI                                                                            0x5841
+#define regCP_PFP_IC_BASE_HI_BASE_IDX                                                                   1
+#define regCP_PFP_IC_BASE_CNTL                                                                          0x5842
+#define regCP_PFP_IC_BASE_CNTL_BASE_IDX                                                                 1
+#define regCP_PFP_IC_OP_CNTL                                                                            0x5843
+#define regCP_PFP_IC_OP_CNTL_BASE_IDX                                                                   1
+#define regCP_ME_IC_BASE_LO                                                                             0x5844
+#define regCP_ME_IC_BASE_LO_BASE_IDX                                                                    1
+#define regCP_ME_IC_BASE_HI                                                                             0x5845
+#define regCP_ME_IC_BASE_HI_BASE_IDX                                                                    1
+#define regCP_ME_IC_BASE_CNTL                                                                           0x5846
+#define regCP_ME_IC_BASE_CNTL_BASE_IDX                                                                  1
+#define regCP_ME_IC_OP_CNTL                                                                             0x5847
+#define regCP_ME_IC_OP_CNTL_BASE_IDX                                                                    1
+#define regCP_CPC_IC_BASE_LO                                                                            0x584c
+#define regCP_CPC_IC_BASE_LO_BASE_IDX                                                                   1
+#define regCP_CPC_IC_BASE_HI                                                                            0x584d
+#define regCP_CPC_IC_BASE_HI_BASE_IDX                                                                   1
+#define regCP_CPC_IC_BASE_CNTL                                                                          0x584e
+#define regCP_CPC_IC_BASE_CNTL_BASE_IDX                                                                 1
+#define regCP_MES_IC_BASE_LO                                                                            0x5850
+#define regCP_MES_IC_BASE_LO_BASE_IDX                                                                   1
+#define regCP_MES_MIBASE_LO                                                                             0x5850
+#define regCP_MES_MIBASE_LO_BASE_IDX                                                                    1
+#define regCP_MES_IC_BASE_HI                                                                            0x5851
+#define regCP_MES_IC_BASE_HI_BASE_IDX                                                                   1
+#define regCP_MES_MIBASE_HI                                                                             0x5851
+#define regCP_MES_MIBASE_HI_BASE_IDX                                                                    1
+#define regCP_MES_IC_BASE_CNTL                                                                          0x5852
+#define regCP_MES_IC_BASE_CNTL_BASE_IDX                                                                 1
+#define regCP_MES_DC_BASE_LO                                                                            0x5854
+#define regCP_MES_DC_BASE_LO_BASE_IDX                                                                   1
+#define regCP_MES_MDBASE_LO                                                                             0x5854
+#define regCP_MES_MDBASE_LO_BASE_IDX                                                                    1
+#define regCP_MES_DC_BASE_HI                                                                            0x5855
+#define regCP_MES_DC_BASE_HI_BASE_IDX                                                                   1
+#define regCP_MES_MDBASE_HI                                                                             0x5855
+#define regCP_MES_MDBASE_HI_BASE_IDX                                                                    1
+#define regCP_MES_MIBOUND_LO                                                                            0x585b
+#define regCP_MES_MIBOUND_LO_BASE_IDX                                                                   1
+#define regCP_MES_MIBOUND_HI                                                                            0x585c
+#define regCP_MES_MIBOUND_HI_BASE_IDX                                                                   1
+#define regCP_MES_MDBOUND_LO                                                                            0x585d
+#define regCP_MES_MDBOUND_LO_BASE_IDX                                                                   1
+#define regCP_MES_MDBOUND_HI                                                                            0x585e
+#define regCP_MES_MDBOUND_HI_BASE_IDX                                                                   1
+#define regCP_HYP_PFP_UCODE_VERS                                                                        0x5861
+#define regCP_HYP_PFP_UCODE_VERS_BASE_IDX                                                               1
+#define regCP_HYP_ME_UCODE_VERS                                                                         0x5862
+#define regCP_HYP_ME_UCODE_VERS_BASE_IDX                                                                1
+#define regCP_GFX_RS64_DC_BASE0_LO                                                                      0x5863
+#define regCP_GFX_RS64_DC_BASE0_LO_BASE_IDX                                                             1
+#define regCP_GFX_RS64_DC_BASE1_LO                                                                      0x5864
+#define regCP_GFX_RS64_DC_BASE1_LO_BASE_IDX                                                             1
+#define regCP_GFX_RS64_DC_BASE0_HI                                                                      0x5865
+#define regCP_GFX_RS64_DC_BASE0_HI_BASE_IDX                                                             1
+#define regCP_GFX_RS64_DC_BASE1_HI                                                                      0x5866
+#define regCP_GFX_RS64_DC_BASE1_HI_BASE_IDX                                                             1
+#define regCP_GFX_RS64_MIBOUND_LO                                                                       0x586c
+#define regCP_GFX_RS64_MIBOUND_LO_BASE_IDX                                                              1
+#define regCP_GFX_RS64_MIBOUND_HI                                                                       0x586d
+#define regCP_GFX_RS64_MIBOUND_HI_BASE_IDX                                                              1
+#define regCP_MEC_DC_BASE_LO                                                                            0x5870
+#define regCP_MEC_DC_BASE_LO_BASE_IDX                                                                   1
+#define regCP_MEC_MDBASE_LO                                                                             0x5870
+#define regCP_MEC_MDBASE_LO_BASE_IDX                                                                    1
+#define regCP_MEC_DC_BASE_HI                                                                            0x5871
+#define regCP_MEC_DC_BASE_HI_BASE_IDX                                                                   1
+#define regCP_MEC_MDBASE_HI                                                                             0x5871
+#define regCP_MEC_MDBASE_HI_BASE_IDX                                                                    1
+#define regCP_MEC_MIBOUND_LO                                                                            0x5872
+#define regCP_MEC_MIBOUND_LO_BASE_IDX                                                                   1
+#define regCP_MEC_MIBOUND_HI                                                                            0x5873
+#define regCP_MEC_MIBOUND_HI_BASE_IDX                                                                   1
+#define regCP_MEC_MDBOUND_LO                                                                            0x5874
+#define regCP_MEC_MDBOUND_LO_BASE_IDX                                                                   1
+#define regCP_MEC_MDBOUND_HI                                                                            0x5875
+#define regCP_MEC_MDBOUND_HI_BASE_IDX                                                                   1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_grbm_hypdec
+// base address: 0x3eea0
+#define regGRBM_GFX_INDEX_SR_SELECT                                                                     0x5ba8
+#define regGRBM_GFX_INDEX_SR_SELECT_BASE_IDX                                                            1
+#define regGRBM_GFX_INDEX_SR_DATA                                                                       0x5ba9
+#define regGRBM_GFX_INDEX_SR_DATA_BASE_IDX                                                              1
+#define regGRBM_GFX_CNTL_SR_SELECT                                                                      0x5baa
+#define regGRBM_GFX_CNTL_SR_SELECT_BASE_IDX                                                             1
+#define regGRBM_GFX_CNTL_SR_DATA                                                                        0x5bab
+#define regGRBM_GFX_CNTL_SR_DATA_BASE_IDX                                                               1
+#define regGC_IH_COOKIE_0_PTR_HIGH                                                                      0x5bac
+#define regGC_IH_COOKIE_0_PTR_HIGH_BASE_IDX                                                             1
+#define regGC_IH_COOKIE_0_PTR                                                                           0x5baf
+#define regGC_IH_COOKIE_0_PTR_BASE_IDX                                                                  1
+#define regGRBM_SE_REMAP_CNTL                                                                           0x5bb0
+#define regGRBM_SE_REMAP_CNTL_BASE_IDX                                                                  1
+#define regGRBM_SA_REMAP_CNTL                                                                           0x5bb1
+#define regGRBM_SA_REMAP_CNTL_BASE_IDX                                                                  1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_rlcdec
+// base address: 0x3b000
+#define regRLC_CNTL                                                                                     0x4c00
+#define regRLC_CNTL_BASE_IDX                                                                            1
+#define regRLC_F32_UCODE_VERSION                                                                        0x4c03
+#define regRLC_F32_UCODE_VERSION_BASE_IDX                                                               1
+#define regRLC_STAT                                                                                     0x4c04
+#define regRLC_STAT_BASE_IDX                                                                            1
+#define regRLC_ACTIVE_MASK                                                                              0x4c05
+#define regRLC_ACTIVE_MASK_BASE_IDX                                                                     1
+#define regRLC_GFX_SE_STATUS                                                                            0x4c06
+#define regRLC_GFX_SE_STATUS_BASE_IDX                                                                   1
+#define regRLC_REFCLOCK_TIMESTAMP_LSB                                                                   0x4c0c
+#define regRLC_REFCLOCK_TIMESTAMP_LSB_BASE_IDX                                                          1
+#define regRLC_REFCLOCK_TIMESTAMP_MSB                                                                   0x4c0d
+#define regRLC_REFCLOCK_TIMESTAMP_MSB_BASE_IDX                                                          1
+#define regRLC_GPM_TIMER_INT_0                                                                          0x4c0e
+#define regRLC_GPM_TIMER_INT_0_BASE_IDX                                                                 1
+#define regRLC_GPM_TIMER_INT_1                                                                          0x4c0f
+#define regRLC_GPM_TIMER_INT_1_BASE_IDX                                                                 1
+#define regRLC_GPM_TIMER_INT_2                                                                          0x4c10
+#define regRLC_GPM_TIMER_INT_2_BASE_IDX                                                                 1
+#define regRLC_GPM_TIMER_INT_3                                                                          0x4c11
+#define regRLC_GPM_TIMER_INT_3_BASE_IDX                                                                 1
+#define regRLC_GPM_TIMER_INT_4                                                                          0x4c12
+#define regRLC_GPM_TIMER_INT_4_BASE_IDX                                                                 1
+#define regRLC_GPM_TIMER_CTRL                                                                           0x4c13
+#define regRLC_GPM_TIMER_CTRL_BASE_IDX                                                                  1
+#define regRLC_GPM_TIMER_STAT                                                                           0x4c14
+#define regRLC_GPM_TIMER_STAT_BASE_IDX                                                                  1
+#define regRLC_GPM_LEGACY_INT_STAT                                                                      0x4c16
+#define regRLC_GPM_LEGACY_INT_STAT_BASE_IDX                                                             1
+#define regRLC_GPM_LEGACY_INT_CLEAR                                                                     0x4c17
+#define regRLC_GPM_LEGACY_INT_CLEAR_BASE_IDX                                                            1
+#define regRLC_INT_STAT                                                                                 0x4c18
+#define regRLC_INT_STAT_BASE_IDX                                                                        1
+#define regRLC_MGCG_CTRL                                                                                0x4c1a
+#define regRLC_MGCG_CTRL_BASE_IDX                                                                       1
+#define regRLC_MCA_DF_RASINTRCTRL_REG_CTRL                                                              0x4c1c
+#define regRLC_MCA_DF_RASINTRCTRL_REG_CTRL_BASE_IDX                                                     1
+#define regRLC_JUMP_TABLE_RESTORE                                                                       0x4c1e
+#define regRLC_JUMP_TABLE_RESTORE_BASE_IDX                                                              1
+#define regRLC_PG_DELAY_2                                                                               0x4c1f
+#define regRLC_PG_DELAY_2_BASE_IDX                                                                      1
+#define regRLC_GPU_CLOCK_COUNT_LSB                                                                      0x4c24
+#define regRLC_GPU_CLOCK_COUNT_LSB_BASE_IDX                                                             1
+#define regRLC_GPU_CLOCK_COUNT_MSB                                                                      0x4c25
+#define regRLC_GPU_CLOCK_COUNT_MSB_BASE_IDX                                                             1
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT                                                                  0x4c26
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT_BASE_IDX                                                         1
+#define regRLC_UCODE_CNTL                                                                               0x4c27
+#define regRLC_UCODE_CNTL_BASE_IDX                                                                      1
+#define regRLC_GPM_THREAD_RESET                                                                         0x4c28
+#define regRLC_GPM_THREAD_RESET_BASE_IDX                                                                1
+#define regRLC_GPM_CP_DMA_COMPLETE_T0                                                                   0x4c29
+#define regRLC_GPM_CP_DMA_COMPLETE_T0_BASE_IDX                                                          1
+#define regRLC_GPM_CP_DMA_COMPLETE_T1                                                                   0x4c2a
+#define regRLC_GPM_CP_DMA_COMPLETE_T1_BASE_IDX                                                          1
+#define regRLC_GPM_THREAD_INVALIDATE_CACHE                                                              0x4c2b
+#define regRLC_GPM_THREAD_INVALIDATE_CACHE_BASE_IDX                                                     1
+#define regRLC_MCA_DF_RASINTRCTRL_REG_LO                                                                0x4c2c
+#define regRLC_MCA_DF_RASINTRCTRL_REG_LO_BASE_IDX                                                       1
+#define regRLC_MCA_DF_RASINTRCTRL_REG_HI                                                                0x4c2d
+#define regRLC_MCA_DF_RASINTRCTRL_REG_HI_BASE_IDX                                                       1
+#define regRLC_MCA_INTR_CTRL                                                                            0x4c2e
+#define regRLC_MCA_INTR_CTRL_BASE_IDX                                                                   1
+#define regRLC_MCA_INTR_INFO                                                                            0x4c2f
+#define regRLC_MCA_INTR_INFO_BASE_IDX                                                                   1
+#define regRLC_CLK_COUNT_GFXCLK_LSB                                                                     0x4c30
+#define regRLC_CLK_COUNT_GFXCLK_LSB_BASE_IDX                                                            1
+#define regRLC_CLK_COUNT_GFXCLK_MSB                                                                     0x4c31
+#define regRLC_CLK_COUNT_GFXCLK_MSB_BASE_IDX                                                            1
+#define regRLC_CLK_COUNT_REFCLK_LSB                                                                     0x4c32
+#define regRLC_CLK_COUNT_REFCLK_LSB_BASE_IDX                                                            1
+#define regRLC_CLK_COUNT_REFCLK_MSB                                                                     0x4c33
+#define regRLC_CLK_COUNT_REFCLK_MSB_BASE_IDX                                                            1
+#define regRLC_CLK_COUNT_CTRL                                                                           0x4c34
+#define regRLC_CLK_COUNT_CTRL_BASE_IDX                                                                  1
+#define regRLC_CLK_COUNT_STAT                                                                           0x4c35
+#define regRLC_CLK_COUNT_STAT_BASE_IDX                                                                  1
+#define regRLC_RLCG_DOORBELL_CNTL                                                                       0x4c36
+#define regRLC_RLCG_DOORBELL_CNTL_BASE_IDX                                                              1
+#define regRLC_RLCG_DOORBELL_STAT                                                                       0x4c37
+#define regRLC_RLCG_DOORBELL_STAT_BASE_IDX                                                              1
+#define regRLC_RLCG_DOORBELL_0_DATA_LO                                                                  0x4c38
+#define regRLC_RLCG_DOORBELL_0_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCG_DOORBELL_0_DATA_HI                                                                  0x4c39
+#define regRLC_RLCG_DOORBELL_0_DATA_HI_BASE_IDX                                                         1
+#define regRLC_RLCG_DOORBELL_1_DATA_LO                                                                  0x4c3a
+#define regRLC_RLCG_DOORBELL_1_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCG_DOORBELL_1_DATA_HI                                                                  0x4c3b
+#define regRLC_RLCG_DOORBELL_1_DATA_HI_BASE_IDX                                                         1
+#define regRLC_RLCG_DOORBELL_2_DATA_LO                                                                  0x4c3c
+#define regRLC_RLCG_DOORBELL_2_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCG_DOORBELL_2_DATA_HI                                                                  0x4c3d
+#define regRLC_RLCG_DOORBELL_2_DATA_HI_BASE_IDX                                                         1
+#define regRLC_RLCG_DOORBELL_3_DATA_LO                                                                  0x4c3e
+#define regRLC_RLCG_DOORBELL_3_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCG_DOORBELL_3_DATA_HI                                                                  0x4c3f
+#define regRLC_RLCG_DOORBELL_3_DATA_HI_BASE_IDX                                                         1
+#define regRLC_GPU_CLOCK_32_RES_SEL                                                                     0x4c41
+#define regRLC_GPU_CLOCK_32_RES_SEL_BASE_IDX                                                            1
+#define regRLC_GPU_CLOCK_32                                                                             0x4c42
+#define regRLC_GPU_CLOCK_32_BASE_IDX                                                                    1
+#define regRLC_PG_CNTL                                                                                  0x4c43
+#define regRLC_PG_CNTL_BASE_IDX                                                                         1
+#define regRLC_GPM_THREAD_PRIORITY                                                                      0x4c44
+#define regRLC_GPM_THREAD_PRIORITY_BASE_IDX                                                             1
+#define regRLC_GPM_THREAD_ENABLE                                                                        0x4c45
+#define regRLC_GPM_THREAD_ENABLE_BASE_IDX                                                               1
+#define regRLC_RLCG_DOORBELL_RANGE                                                                      0x4c47
+#define regRLC_RLCG_DOORBELL_RANGE_BASE_IDX                                                             1
+#define regRLC_CGTT_MGCG_OVERRIDE                                                                       0x4c48
+#define regRLC_CGTT_MGCG_OVERRIDE_BASE_IDX                                                              1
+#define regRLC_CGCG_CGLS_CTRL                                                                           0x4c49
+#define regRLC_CGCG_CGLS_CTRL_BASE_IDX                                                                  1
+#define regRLC_CGCG_RAMP_CTRL                                                                           0x4c4a
+#define regRLC_CGCG_RAMP_CTRL_BASE_IDX                                                                  1
+#define regRLC_DYN_PG_STATUS                                                                            0x4c4b
+#define regRLC_DYN_PG_STATUS_BASE_IDX                                                                   1
+#define regRLC_DYN_PG_REQUEST                                                                           0x4c4c
+#define regRLC_DYN_PG_REQUEST_BASE_IDX                                                                  1
+#define regRLC_PG_DELAY                                                                                 0x4c4d
+#define regRLC_PG_DELAY_BASE_IDX                                                                        1
+#define regRLC_CGCG_CGLS_CTRL_2                                                                         0x4c4e
+#define regRLC_CGCG_CGLS_CTRL_2_BASE_IDX                                                                1
+#define regRLC_PG_ALWAYS_ON_WGP_MASK                                                                    0x4c53
+#define regRLC_PG_ALWAYS_ON_WGP_MASK_BASE_IDX                                                           1
+#define regRLC_MAX_PG_WGP                                                                               0x4c54
+#define regRLC_MAX_PG_WGP_BASE_IDX                                                                      1
+#define regRLC_AUTO_PG_CTRL                                                                             0x4c55
+#define regRLC_AUTO_PG_CTRL_BASE_IDX                                                                    1
+#define regRLC_SERDES_RD_INDEX                                                                          0x4c59
+#define regRLC_SERDES_RD_INDEX_BASE_IDX                                                                 1
+#define regRLC_SERDES_RD_DATA_0                                                                         0x4c5a
+#define regRLC_SERDES_RD_DATA_0_BASE_IDX                                                                1
+#define regRLC_SERDES_RD_DATA_1                                                                         0x4c5b
+#define regRLC_SERDES_RD_DATA_1_BASE_IDX                                                                1
+#define regRLC_SERDES_RD_DATA_2                                                                         0x4c5c
+#define regRLC_SERDES_RD_DATA_2_BASE_IDX                                                                1
+#define regRLC_SERDES_RD_DATA_3                                                                         0x4c5d
+#define regRLC_SERDES_RD_DATA_3_BASE_IDX                                                                1
+#define regRLC_SERDES_MASK                                                                              0x4c5e
+#define regRLC_SERDES_MASK_BASE_IDX                                                                     1
+#define regRLC_SERDES_CTRL                                                                              0x4c5f
+#define regRLC_SERDES_CTRL_BASE_IDX                                                                     1
+#define regRLC_SERDES_DATA                                                                              0x4c60
+#define regRLC_SERDES_DATA_BASE_IDX                                                                     1
+#define regRLC_SERDES_BUSY                                                                              0x4c61
+#define regRLC_SERDES_BUSY_BASE_IDX                                                                     1
+#define regRLC_GPM_GENERAL_0                                                                            0x4c63
+#define regRLC_GPM_GENERAL_0_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_1                                                                            0x4c64
+#define regRLC_GPM_GENERAL_1_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_2                                                                            0x4c65
+#define regRLC_GPM_GENERAL_2_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_3                                                                            0x4c66
+#define regRLC_GPM_GENERAL_3_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_4                                                                            0x4c67
+#define regRLC_GPM_GENERAL_4_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_5                                                                            0x4c68
+#define regRLC_GPM_GENERAL_5_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_6                                                                            0x4c69
+#define regRLC_GPM_GENERAL_6_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_7                                                                            0x4c6a
+#define regRLC_GPM_GENERAL_7_BASE_IDX                                                                   1
+#define regRLC_STATIC_PG_STATUS                                                                         0x4c6e
+#define regRLC_STATIC_PG_STATUS_BASE_IDX                                                                1
+#define regRLC_GPM_GENERAL_16                                                                           0x4c76
+#define regRLC_GPM_GENERAL_16_BASE_IDX                                                                  1
+#define regRLC_PG_DELAY_3                                                                               0x4c78
+#define regRLC_PG_DELAY_3_BASE_IDX                                                                      1
+#define regRLC_GPR_REG1                                                                                 0x4c79
+#define regRLC_GPR_REG1_BASE_IDX                                                                        1
+#define regRLC_GPR_REG2                                                                                 0x4c7a
+#define regRLC_GPR_REG2_BASE_IDX                                                                        1
+#define regRLC_GPM_INT_DISABLE_TH0                                                                      0x4c7c
+#define regRLC_GPM_INT_DISABLE_TH0_BASE_IDX                                                             1
+#define regRLC_GPM_LEGACY_INT_DISABLE                                                                   0x4c7d
+#define regRLC_GPM_LEGACY_INT_DISABLE_BASE_IDX                                                          1
+#define regRLC_GPM_INT_FORCE_TH0                                                                        0x4c7e
+#define regRLC_GPM_INT_FORCE_TH0_BASE_IDX                                                               1
+#define regRLC_SRM_CNTL                                                                                 0x4c80
+#define regRLC_SRM_CNTL_BASE_IDX                                                                        1
+#define regRLC_SRM_GPM_COMMAND_STATUS                                                                   0x4c88
+#define regRLC_SRM_GPM_COMMAND_STATUS_BASE_IDX                                                          1
+#define regRLC_SRM_INDEX_CNTL_ADDR_0                                                                    0x4c8b
+#define regRLC_SRM_INDEX_CNTL_ADDR_0_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_1                                                                    0x4c8c
+#define regRLC_SRM_INDEX_CNTL_ADDR_1_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_2                                                                    0x4c8d
+#define regRLC_SRM_INDEX_CNTL_ADDR_2_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_3                                                                    0x4c8e
+#define regRLC_SRM_INDEX_CNTL_ADDR_3_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_4                                                                    0x4c8f
+#define regRLC_SRM_INDEX_CNTL_ADDR_4_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_5                                                                    0x4c90
+#define regRLC_SRM_INDEX_CNTL_ADDR_5_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_6                                                                    0x4c91
+#define regRLC_SRM_INDEX_CNTL_ADDR_6_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_7                                                                    0x4c92
+#define regRLC_SRM_INDEX_CNTL_ADDR_7_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_0                                                                    0x4c93
+#define regRLC_SRM_INDEX_CNTL_DATA_0_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_1                                                                    0x4c94
+#define regRLC_SRM_INDEX_CNTL_DATA_1_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_2                                                                    0x4c95
+#define regRLC_SRM_INDEX_CNTL_DATA_2_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_3                                                                    0x4c96
+#define regRLC_SRM_INDEX_CNTL_DATA_3_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_4                                                                    0x4c97
+#define regRLC_SRM_INDEX_CNTL_DATA_4_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_5                                                                    0x4c98
+#define regRLC_SRM_INDEX_CNTL_DATA_5_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_6                                                                    0x4c99
+#define regRLC_SRM_INDEX_CNTL_DATA_6_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_7                                                                    0x4c9a
+#define regRLC_SRM_INDEX_CNTL_DATA_7_BASE_IDX                                                           1
+#define regRLC_SRM_STAT                                                                                 0x4c9b
+#define regRLC_SRM_STAT_BASE_IDX                                                                        1
+#define regRLC_LX6_UTCL1_ERROR_2                                                                        0x4ca8
+#define regRLC_LX6_UTCL1_ERROR_2_BASE_IDX                                                               1
+#define regRLC_GPM_GENERAL_8                                                                            0x4cad
+#define regRLC_GPM_GENERAL_8_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_9                                                                            0x4cae
+#define regRLC_GPM_GENERAL_9_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_10                                                                           0x4caf
+#define regRLC_GPM_GENERAL_10_BASE_IDX                                                                  1
+#define regRLC_GPM_GENERAL_11                                                                           0x4cb0
+#define regRLC_GPM_GENERAL_11_BASE_IDX                                                                  1
+#define regRLC_GPM_GENERAL_12                                                                           0x4cb1
+#define regRLC_GPM_GENERAL_12_BASE_IDX                                                                  1
+#define regRLC_GPM_UTCL1_CNTL_0                                                                         0x4cb2
+#define regRLC_GPM_UTCL1_CNTL_0_BASE_IDX                                                                1
+#define regRLC_SPM_UTCL1_CNTL                                                                           0x4cb5
+#define regRLC_SPM_UTCL1_CNTL_BASE_IDX                                                                  1
+#define regRLC_UTCL1_STATUS_2                                                                           0x4cb6
+#define regRLC_UTCL1_STATUS_2_BASE_IDX                                                                  1
+#define regRLC_SPM_UTCL1_ERROR_1                                                                        0x4cbc
+#define regRLC_SPM_UTCL1_ERROR_1_BASE_IDX                                                               1
+#define regRLC_SPM_UTCL1_ERROR_2                                                                        0x4cbd
+#define regRLC_SPM_UTCL1_ERROR_2_BASE_IDX                                                               1
+#define regRLC_GPM_UTCL1_TH0_ERROR_1                                                                    0x4cbe
+#define regRLC_GPM_UTCL1_TH0_ERROR_1_BASE_IDX                                                           1
+#define regRLC_GPM_UTCL1_TH0_ERROR_2                                                                    0x4cc0
+#define regRLC_GPM_UTCL1_TH0_ERROR_2_BASE_IDX                                                           1
+#define regRLC_SEMAPHORE_0                                                                              0x4cc7
+#define regRLC_SEMAPHORE_0_BASE_IDX                                                                     1
+#define regRLC_SEMAPHORE_1                                                                              0x4cc8
+#define regRLC_SEMAPHORE_1_BASE_IDX                                                                     1
+#define regRLC_SEMAPHORE_2                                                                              0x4cc9
+#define regRLC_SEMAPHORE_2_BASE_IDX                                                                     1
+#define regRLC_SEMAPHORE_3                                                                              0x4cca
+#define regRLC_SEMAPHORE_3_BASE_IDX                                                                     1
+#define regRLC_SRM_UTCL1_CNTL                                                                           0x4ccc
+#define regRLC_SRM_UTCL1_CNTL_BASE_IDX                                                                  1
+#define regRLC_SRM_UTCL1_ERROR_1                                                                        0x4ccd
+#define regRLC_SRM_UTCL1_ERROR_1_BASE_IDX                                                               1
+#define regRLC_SRM_UTCL1_ERROR_2                                                                        0x4cce
+#define regRLC_SRM_UTCL1_ERROR_2_BASE_IDX                                                               1
+#define regRLC_DLG_UTCL1_ERROR_2                                                                        0x4cd1
+#define regRLC_DLG_UTCL1_ERROR_2_BASE_IDX                                                               1
+#define regRLC_UTCL1_STATUS                                                                             0x4cd4
+#define regRLC_UTCL1_STATUS_BASE_IDX                                                                    1
+#define regRLC_R2I_CNTL_0                                                                               0x4cd5
+#define regRLC_R2I_CNTL_0_BASE_IDX                                                                      1
+#define regRLC_R2I_CNTL_1                                                                               0x4cd6
+#define regRLC_R2I_CNTL_1_BASE_IDX                                                                      1
+#define regRLC_R2I_CNTL_2                                                                               0x4cd7
+#define regRLC_R2I_CNTL_2_BASE_IDX                                                                      1
+#define regRLC_R2I_CNTL_3                                                                               0x4cd8
+#define regRLC_R2I_CNTL_3_BASE_IDX                                                                      1
+#define regRLC_GPM_INT_STAT_TH0                                                                         0x4cdc
+#define regRLC_GPM_INT_STAT_TH0_BASE_IDX                                                                1
+#define regRLC_GPM_GENERAL_13                                                                           0x4cdd
+#define regRLC_GPM_GENERAL_13_BASE_IDX                                                                  1
+#define regRLC_GPM_GENERAL_14                                                                           0x4cde
+#define regRLC_GPM_GENERAL_14_BASE_IDX                                                                  1
+#define regRLC_GPM_GENERAL_15                                                                           0x4cdf
+#define regRLC_GPM_GENERAL_15_BASE_IDX                                                                  1
+#define regRLC_LX6_UTCL1_ERROR_1                                                                        0x4ce3
+#define regRLC_LX6_UTCL1_ERROR_1_BASE_IDX                                                               1
+#define regRLC_LX6_UTCL1_CNTL                                                                           0x4ce4
+#define regRLC_LX6_UTCL1_CNTL_BASE_IDX                                                                  1
+#define regRLC_DMA_UTCL1_ERROR_1                                                                        0x4ce6
+#define regRLC_DMA_UTCL1_ERROR_1_BASE_IDX                                                               1
+#define regRLC_DMA_UTCL1_ERROR_2                                                                        0x4ce7
+#define regRLC_DMA_UTCL1_ERROR_2_BASE_IDX                                                               1
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT_1                                                                0x4cea
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT_1_BASE_IDX                                                       1
+#define regRLC_GPU_CLOCK_COUNT_LSB_2                                                                    0x4ceb
+#define regRLC_GPU_CLOCK_COUNT_LSB_2_BASE_IDX                                                           1
+#define regRLC_GPU_CLOCK_COUNT_MSB_2                                                                    0x4cec
+#define regRLC_GPU_CLOCK_COUNT_MSB_2_BASE_IDX                                                           1
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT_2                                                                0x4cef
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT_2_BASE_IDX                                                       1
+#define regRLC_RLCV_DOORBELL_RANGE                                                                      0x4cf0
+#define regRLC_RLCV_DOORBELL_RANGE_BASE_IDX                                                             1
+#define regRLC_RLCV_DOORBELL_CNTL                                                                       0x4cf1
+#define regRLC_RLCV_DOORBELL_CNTL_BASE_IDX                                                              1
+#define regRLC_RLCV_DOORBELL_STAT                                                                       0x4cf2
+#define regRLC_RLCV_DOORBELL_STAT_BASE_IDX                                                              1
+#define regRLC_RLCV_DOORBELL_0_DATA_LO                                                                  0x4cf3
+#define regRLC_RLCV_DOORBELL_0_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCV_DOORBELL_0_DATA_HI                                                                  0x4cf4
+#define regRLC_RLCV_DOORBELL_0_DATA_HI_BASE_IDX                                                         1
+#define regRLC_RLCV_DOORBELL_1_DATA_LO                                                                  0x4cf5
+#define regRLC_RLCV_DOORBELL_1_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCV_DOORBELL_1_DATA_HI                                                                  0x4cf6
+#define regRLC_RLCV_DOORBELL_1_DATA_HI_BASE_IDX                                                         1
+#define regRLC_RLCV_DOORBELL_2_DATA_LO                                                                  0x4cf7
+#define regRLC_RLCV_DOORBELL_2_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCV_DOORBELL_2_DATA_HI                                                                  0x4cf8
+#define regRLC_RLCV_DOORBELL_2_DATA_HI_BASE_IDX                                                         1
+#define regRLC_RLCV_DOORBELL_3_DATA_LO                                                                  0x4cf9
+#define regRLC_RLCV_DOORBELL_3_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCV_DOORBELL_3_DATA_HI                                                                  0x4cfa
+#define regRLC_RLCV_DOORBELL_3_DATA_HI_BASE_IDX                                                         1
+#define regRLC_GPU_CLOCK_COUNT_LSB_1                                                                    0x4cfb
+#define regRLC_GPU_CLOCK_COUNT_LSB_1_BASE_IDX                                                           1
+#define regRLC_GPU_CLOCK_COUNT_MSB_1                                                                    0x4cfc
+#define regRLC_GPU_CLOCK_COUNT_MSB_1_BASE_IDX                                                           1
+#define regRLC_RLCV_SPARE_INT                                                                           0x4d00
+#define regRLC_RLCV_SPARE_INT_BASE_IDX                                                                  1
+#define regRLC_SMU_CLK_REQ                                                                              0x4d08
+#define regRLC_SMU_CLK_REQ_BASE_IDX                                                                     1
+#define regRLC_SPARE                                                                                    0x4d0b
+#define regRLC_SPARE_BASE_IDX                                                                           1
+#define regRLC_SPP_CTRL                                                                                 0x4d0c
+#define regRLC_SPP_CTRL_BASE_IDX                                                                        1
+#define regRLC_SPP_SHADER_PROFILE_EN                                                                    0x4d0d
+#define regRLC_SPP_SHADER_PROFILE_EN_BASE_IDX                                                           1
+#define regRLC_SPP_SSF_CAPTURE_EN                                                                       0x4d0e
+#define regRLC_SPP_SSF_CAPTURE_EN_BASE_IDX                                                              1
+#define regRLC_SPP_SSF_THRESHOLD_0                                                                      0x4d0f
+#define regRLC_SPP_SSF_THRESHOLD_0_BASE_IDX                                                             1
+#define regRLC_SPP_SSF_THRESHOLD_1                                                                      0x4d10
+#define regRLC_SPP_SSF_THRESHOLD_1_BASE_IDX                                                             1
+#define regRLC_SPP_SSF_THRESHOLD_2                                                                      0x4d11
+#define regRLC_SPP_SSF_THRESHOLD_2_BASE_IDX                                                             1
+#define regRLC_SPP_INFLIGHT_RD_ADDR                                                                     0x4d12
+#define regRLC_SPP_INFLIGHT_RD_ADDR_BASE_IDX                                                            1
+#define regRLC_SPP_INFLIGHT_RD_DATA                                                                     0x4d13
+#define regRLC_SPP_INFLIGHT_RD_DATA_BASE_IDX                                                            1
+#define regRLC_SPP_PROF_INFO_1                                                                          0x4d18
+#define regRLC_SPP_PROF_INFO_1_BASE_IDX                                                                 1
+#define regRLC_SPP_PROF_INFO_2                                                                          0x4d19
+#define regRLC_SPP_PROF_INFO_2_BASE_IDX                                                                 1
+#define regRLC_SPP_GLOBAL_SH_ID                                                                         0x4d1a
+#define regRLC_SPP_GLOBAL_SH_ID_BASE_IDX                                                                1
+#define regRLC_SPP_GLOBAL_SH_ID_VALID                                                                   0x4d1b
+#define regRLC_SPP_GLOBAL_SH_ID_VALID_BASE_IDX                                                          1
+#define regRLC_SPP_STATUS                                                                               0x4d1c
+#define regRLC_SPP_STATUS_BASE_IDX                                                                      1
+#define regRLC_SPP_PVT_STAT_0                                                                           0x4d1d
+#define regRLC_SPP_PVT_STAT_0_BASE_IDX                                                                  1
+#define regRLC_SPP_PVT_STAT_1                                                                           0x4d1e
+#define regRLC_SPP_PVT_STAT_1_BASE_IDX                                                                  1
+#define regRLC_SPP_PVT_STAT_2                                                                           0x4d1f
+#define regRLC_SPP_PVT_STAT_2_BASE_IDX                                                                  1
+#define regRLC_SPP_PVT_STAT_3                                                                           0x4d20
+#define regRLC_SPP_PVT_STAT_3_BASE_IDX                                                                  1
+#define regRLC_SPP_PVT_LEVEL_MAX                                                                        0x4d21
+#define regRLC_SPP_PVT_LEVEL_MAX_BASE_IDX                                                               1
+#define regRLC_SPP_STALL_STATE_UPDATE                                                                   0x4d22
+#define regRLC_SPP_STALL_STATE_UPDATE_BASE_IDX                                                          1
+#define regRLC_SPP_PBB_INFO                                                                             0x4d23
+#define regRLC_SPP_PBB_INFO_BASE_IDX                                                                    1
+#define regRLC_SPP_RESET                                                                                0x4d24
+#define regRLC_SPP_RESET_BASE_IDX                                                                       1
+#define regRLC_CAC_MASK_CNTL                                                                            0x4d45
+#define regRLC_CAC_MASK_CNTL_BASE_IDX                                                                   1
+#define regRLC_POWER_RESIDENCY_CNTR_CTRL                                                                0x4d48
+#define regRLC_POWER_RESIDENCY_CNTR_CTRL_BASE_IDX                                                       1
+#define regRLC_CLK_RESIDENCY_CNTR_CTRL                                                                  0x4d49
+#define regRLC_CLK_RESIDENCY_CNTR_CTRL_BASE_IDX                                                         1
+#define regRLC_DS_RESIDENCY_CNTR_CTRL                                                                   0x4d4a
+#define regRLC_DS_RESIDENCY_CNTR_CTRL_BASE_IDX                                                          1
+#define regRLC_ULV_RESIDENCY_CNTR_CTRL                                                                  0x4d4b
+#define regRLC_ULV_RESIDENCY_CNTR_CTRL_BASE_IDX                                                         1
+#define regRLC_PCC_RESIDENCY_CNTR_CTRL                                                                  0x4d4c
+#define regRLC_PCC_RESIDENCY_CNTR_CTRL_BASE_IDX                                                         1
+#define regRLC_GENERAL_RESIDENCY_CNTR_CTRL                                                              0x4d4d
+#define regRLC_GENERAL_RESIDENCY_CNTR_CTRL_BASE_IDX                                                     1
+#define regRLC_POWER_RESIDENCY_EVENT_CNTR                                                               0x4d50
+#define regRLC_POWER_RESIDENCY_EVENT_CNTR_BASE_IDX                                                      1
+#define regRLC_CLK_RESIDENCY_EVENT_CNTR                                                                 0x4d51
+#define regRLC_CLK_RESIDENCY_EVENT_CNTR_BASE_IDX                                                        1
+#define regRLC_DS_RESIDENCY_EVENT_CNTR                                                                  0x4d52
+#define regRLC_DS_RESIDENCY_EVENT_CNTR_BASE_IDX                                                         1
+#define regRLC_ULV_RESIDENCY_EVENT_CNTR                                                                 0x4d53
+#define regRLC_ULV_RESIDENCY_EVENT_CNTR_BASE_IDX                                                        1
+#define regRLC_PCC_RESIDENCY_EVENT_CNTR                                                                 0x4d54
+#define regRLC_PCC_RESIDENCY_EVENT_CNTR_BASE_IDX                                                        1
+#define regRLC_GENERAL_RESIDENCY_EVENT_CNTR                                                             0x4d55
+#define regRLC_GENERAL_RESIDENCY_EVENT_CNTR_BASE_IDX                                                    1
+#define regRLC_POWER_RESIDENCY_REF_CNTR                                                                 0x4d58
+#define regRLC_POWER_RESIDENCY_REF_CNTR_BASE_IDX                                                        1
+#define regRLC_CLK_RESIDENCY_REF_CNTR                                                                   0x4d59
+#define regRLC_CLK_RESIDENCY_REF_CNTR_BASE_IDX                                                          1
+#define regRLC_DS_RESIDENCY_REF_CNTR                                                                    0x4d5a
+#define regRLC_DS_RESIDENCY_REF_CNTR_BASE_IDX                                                           1
+#define regRLC_ULV_RESIDENCY_REF_CNTR                                                                   0x4d5b
+#define regRLC_ULV_RESIDENCY_REF_CNTR_BASE_IDX                                                          1
+#define regRLC_PCC_RESIDENCY_REF_CNTR                                                                   0x4d5c
+#define regRLC_PCC_RESIDENCY_REF_CNTR_BASE_IDX                                                          1
+#define regRLC_GENERAL_RESIDENCY_REF_CNTR                                                               0x4d5d
+#define regRLC_GENERAL_RESIDENCY_REF_CNTR_BASE_IDX                                                      1
+#define regRLC_GFX_IH_CLIENT_CTRL                                                                       0x4d5e
+#define regRLC_GFX_IH_CLIENT_CTRL_BASE_IDX                                                              1
+#define regRLC_GFX_IH_ARBITER_STAT                                                                      0x4d5f
+#define regRLC_GFX_IH_ARBITER_STAT_BASE_IDX                                                             1
+#define regRLC_GFX_IH_CLIENT_SE_STAT_L                                                                  0x4d60
+#define regRLC_GFX_IH_CLIENT_SE_STAT_L_BASE_IDX                                                         1
+#define regRLC_GFX_IH_CLIENT_SE_STAT_H                                                                  0x4d61
+#define regRLC_GFX_IH_CLIENT_SE_STAT_H_BASE_IDX                                                         1
+#define regRLC_GFX_IH_CLIENT_SDMA_STAT                                                                  0x4d62
+#define regRLC_GFX_IH_CLIENT_SDMA_STAT_BASE_IDX                                                         1
+#define regRLC_GFX_IH_CLIENT_OTHER_STAT                                                                 0x4d63
+#define regRLC_GFX_IH_CLIENT_OTHER_STAT_BASE_IDX                                                        1
+#define regRLC_SPM_GLOBAL_DELAY_IND_ADDR                                                                0x4d64
+#define regRLC_SPM_GLOBAL_DELAY_IND_ADDR_BASE_IDX                                                       1
+#define regRLC_SPM_GLOBAL_DELAY_IND_DATA                                                                0x4d65
+#define regRLC_SPM_GLOBAL_DELAY_IND_DATA_BASE_IDX                                                       1
+#define regRLC_SPM_SE_DELAY_IND_ADDR                                                                    0x4d66
+#define regRLC_SPM_SE_DELAY_IND_ADDR_BASE_IDX                                                           1
+#define regRLC_SPM_SE_DELAY_IND_DATA                                                                    0x4d67
+#define regRLC_SPM_SE_DELAY_IND_DATA_BASE_IDX                                                           1
+#define regRLC_SPM_GLOBAL_BLK_EN_MASK_IND_ADDR                                                          0x4d68
+#define regRLC_SPM_GLOBAL_BLK_EN_MASK_IND_ADDR_BASE_IDX                                                 1
+#define regRLC_SPM_GLOBAL_BLK_EN_MASK_IND_DATA                                                          0x4d69
+#define regRLC_SPM_GLOBAL_BLK_EN_MASK_IND_DATA_BASE_IDX                                                 1
+#define regRLC_SPM_SE_BLK_EN_MASK_IND_ADDR                                                              0x4d6a
+#define regRLC_SPM_SE_BLK_EN_MASK_IND_ADDR_BASE_IDX                                                     1
+#define regRLC_SPM_SE_BLK_EN_MASK_IND_DATA                                                              0x4d6b
+#define regRLC_SPM_SE_BLK_EN_MASK_IND_DATA_BASE_IDX                                                     1
+#define regRLC_GFX_IH_22BEAT_STAT                                                                       0x4d6c
+#define regRLC_GFX_IH_22BEAT_STAT_BASE_IDX                                                              1
+#define regRLC_GFX_IH_ARBITER_STAT_22BEAT                                                               0x4d6d
+#define regRLC_GFX_IH_ARBITER_STAT_22BEAT_BASE_IDX                                                      1
+#define regRLC_LX6_CNTL                                                                                 0x4d80
+#define regRLC_LX6_CNTL_BASE_IDX                                                                        1
+#define regRLC_LX6_STATUS                                                                               0x4d81
+#define regRLC_LX6_STATUS_BASE_IDX                                                                      1
+#define regRLC_LX6_FW_STATUS                                                                            0x4dcb
+#define regRLC_LX6_FW_STATUS_BASE_IDX                                                                   1
+#define regRLC_LX6_FW_VERSION                                                                           0x4dcc
+#define regRLC_LX6_FW_VERSION_BASE_IDX                                                                  1
+#define regRLC_XT_CORE_STATUS                                                                           0x4dd4
+#define regRLC_XT_CORE_STATUS_BASE_IDX                                                                  1
+#define regRLC_XT_CORE_INTERRUPT                                                                        0x4dd5
+#define regRLC_XT_CORE_INTERRUPT_BASE_IDX                                                               1
+#define regRLC_XT_CORE_FAULT_INFO                                                                       0x4dd6
+#define regRLC_XT_CORE_FAULT_INFO_BASE_IDX                                                              1
+#define regRLC_XT_CORE_ALT_RESET_VEC                                                                    0x4dd7
+#define regRLC_XT_CORE_ALT_RESET_VEC_BASE_IDX                                                           1
+#define regRLC_XT_CORE_RESERVED                                                                         0x4dd8
+#define regRLC_XT_CORE_RESERVED_BASE_IDX                                                                1
+#define regRLC_XT_INT_VEC_FORCE                                                                         0x4dd9
+#define regRLC_XT_INT_VEC_FORCE_BASE_IDX                                                                1
+#define regRLC_XT_INT_VEC_CLEAR                                                                         0x4dda
+#define regRLC_XT_INT_VEC_CLEAR_BASE_IDX                                                                1
+#define regRLC_XT_INT_VEC_MUX_SEL                                                                       0x4ddb
+#define regRLC_XT_INT_VEC_MUX_SEL_BASE_IDX                                                              1
+#define regRLC_XT_INT_VEC_MUX_INT_SEL                                                                   0x4ddc
+#define regRLC_XT_INT_VEC_MUX_INT_SEL_BASE_IDX                                                          1
+#define regRLC_GPU_CLOCK_COUNT_SPM_LSB                                                                  0x4de4
+#define regRLC_GPU_CLOCK_COUNT_SPM_LSB_BASE_IDX                                                         1
+#define regRLC_GPU_CLOCK_COUNT_SPM_MSB                                                                  0x4de5
+#define regRLC_GPU_CLOCK_COUNT_SPM_MSB_BASE_IDX                                                         1
+#define regRLC_SPM_THREAD_TRACE_CTRL                                                                    0x4de6
+#define regRLC_SPM_THREAD_TRACE_CTRL_BASE_IDX                                                           1
+#define regRLC_SPP_CAM_ADDR                                                                             0x4de8
+#define regRLC_SPP_CAM_ADDR_BASE_IDX                                                                    1
+#define regRLC_SPP_CAM_DATA                                                                             0x4de9
+#define regRLC_SPP_CAM_DATA_BASE_IDX                                                                    1
+#define regRLC_SPP_CAM_EXT_ADDR                                                                         0x4dea
+#define regRLC_SPP_CAM_EXT_ADDR_BASE_IDX                                                                1
+#define regRLC_SPP_CAM_EXT_DATA                                                                         0x4deb
+#define regRLC_SPP_CAM_EXT_DATA_BASE_IDX                                                                1
+#define regRLC_CPAXI_DOORBELL_MON_CTRL                                                                  0x4df1
+#define regRLC_CPAXI_DOORBELL_MON_CTRL_BASE_IDX                                                         1
+#define regRLC_CPAXI_DOORBELL_MON_STAT                                                                  0x4df2
+#define regRLC_CPAXI_DOORBELL_MON_STAT_BASE_IDX                                                         1
+#define regRLC_CPAXI_DOORBELL_MON_DATA_LSB                                                              0x4df3
+#define regRLC_CPAXI_DOORBELL_MON_DATA_LSB_BASE_IDX                                                     1
+#define regRLC_CPAXI_DOORBELL_MON_DATA_MSB                                                              0x4df4
+#define regRLC_CPAXI_DOORBELL_MON_DATA_MSB_BASE_IDX                                                     1
+#define regRLC_XT_DOORBELL_RANGE                                                                        0x4df5
+#define regRLC_XT_DOORBELL_RANGE_BASE_IDX                                                               1
+#define regRLC_XT_DOORBELL_CNTL                                                                         0x4df6
+#define regRLC_XT_DOORBELL_CNTL_BASE_IDX                                                                1
+#define regRLC_XT_DOORBELL_STAT                                                                         0x4df7
+#define regRLC_XT_DOORBELL_STAT_BASE_IDX                                                                1
+#define regRLC_XT_DOORBELL_0_DATA_LO                                                                    0x4df8
+#define regRLC_XT_DOORBELL_0_DATA_LO_BASE_IDX                                                           1
+#define regRLC_XT_DOORBELL_0_DATA_HI                                                                    0x4df9
+#define regRLC_XT_DOORBELL_0_DATA_HI_BASE_IDX                                                           1
+#define regRLC_XT_DOORBELL_1_DATA_LO                                                                    0x4dfa
+#define regRLC_XT_DOORBELL_1_DATA_LO_BASE_IDX                                                           1
+#define regRLC_XT_DOORBELL_1_DATA_HI                                                                    0x4dfb
+#define regRLC_XT_DOORBELL_1_DATA_HI_BASE_IDX                                                           1
+#define regRLC_XT_DOORBELL_2_DATA_LO                                                                    0x4dfc
+#define regRLC_XT_DOORBELL_2_DATA_LO_BASE_IDX                                                           1
+#define regRLC_XT_DOORBELL_2_DATA_HI                                                                    0x4dfd
+#define regRLC_XT_DOORBELL_2_DATA_HI_BASE_IDX                                                           1
+#define regRLC_XT_DOORBELL_3_DATA_LO                                                                    0x4dfe
+#define regRLC_XT_DOORBELL_3_DATA_LO_BASE_IDX                                                           1
+#define regRLC_XT_DOORBELL_3_DATA_HI                                                                    0x4dff
+#define regRLC_XT_DOORBELL_3_DATA_HI_BASE_IDX                                                           1
+#define regRLC_MEM_SLP_CNTL                                                                             0x4e00
+#define regRLC_MEM_SLP_CNTL_BASE_IDX                                                                    1
+#define regRLC_RLCV_SAFE_MODE                                                                           0x4e02
+#define regRLC_RLCV_SAFE_MODE_BASE_IDX                                                                  1
+#define regRLC_SMU_SAFE_MODE                                                                            0x4e03
+#define regRLC_SMU_SAFE_MODE_BASE_IDX                                                                   1
+#define regRLC_RLCV_COMMAND                                                                             0x4e04
+#define regRLC_RLCV_COMMAND_BASE_IDX                                                                    1
+#define regRLC_SMU_MESSAGE                                                                              0x4e05
+#define regRLC_SMU_MESSAGE_BASE_IDX                                                                     1
+#define regRLC_SMU_MESSAGE_1                                                                            0x4e06
+#define regRLC_SMU_MESSAGE_1_BASE_IDX                                                                   1
+#define regRLC_SMU_MESSAGE_2                                                                            0x4e07
+#define regRLC_SMU_MESSAGE_2_BASE_IDX                                                                   1
+#define regRLC_SRM_GPM_COMMAND                                                                          0x4e08
+#define regRLC_SRM_GPM_COMMAND_BASE_IDX                                                                 1
+#define regRLC_SRM_GPM_ABORT                                                                            0x4e09
+#define regRLC_SRM_GPM_ABORT_BASE_IDX                                                                   1
+#define regRLC_SMU_COMMAND                                                                              0x4e0a
+#define regRLC_SMU_COMMAND_BASE_IDX                                                                     1
+#define regRLC_SMU_ARGUMENT_1                                                                           0x4e0b
+#define regRLC_SMU_ARGUMENT_1_BASE_IDX                                                                  1
+#define regRLC_SMU_ARGUMENT_2                                                                           0x4e0c
+#define regRLC_SMU_ARGUMENT_2_BASE_IDX                                                                  1
+#define regRLC_SMU_ARGUMENT_3                                                                           0x4e0d
+#define regRLC_SMU_ARGUMENT_3_BASE_IDX                                                                  1
+#define regRLC_SMU_ARGUMENT_4                                                                           0x4e0e
+#define regRLC_SMU_ARGUMENT_4_BASE_IDX                                                                  1
+#define regRLC_SMU_ARGUMENT_5                                                                           0x4e0f
+#define regRLC_SMU_ARGUMENT_5_BASE_IDX                                                                  1
+#define regRLC_IMU_BOOTLOAD_ADDR_HI                                                                     0x4e10
+#define regRLC_IMU_BOOTLOAD_ADDR_HI_BASE_IDX                                                            1
+#define regRLC_IMU_BOOTLOAD_ADDR_LO                                                                     0x4e11
+#define regRLC_IMU_BOOTLOAD_ADDR_LO_BASE_IDX                                                            1
+#define regRLC_IMU_BOOTLOAD_SIZE                                                                        0x4e12
+#define regRLC_IMU_BOOTLOAD_SIZE_BASE_IDX                                                               1
+#define regRLC_IMU_MISC                                                                                 0x4e16
+#define regRLC_IMU_MISC_BASE_IDX                                                                        1
+#define regRLC_IMU_RESET_VECTOR                                                                         0x4e17
+#define regRLC_IMU_RESET_VECTOR_BASE_IDX                                                                1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_rlcsdec
+// base address: 0x3b980
+#define regRLC_RLCS_DEC_START                                                                           0x4e60
+#define regRLC_RLCS_DEC_START_BASE_IDX                                                                  1
+#define regRLC_RLCS_DEC_DUMP_ADDR                                                                       0x4e61
+#define regRLC_RLCS_DEC_DUMP_ADDR_BASE_IDX                                                              1
+#define regRLC_RLCS_EXCEPTION_REG_1                                                                     0x4e62
+#define regRLC_RLCS_EXCEPTION_REG_1_BASE_IDX                                                            1
+#define regRLC_RLCS_EXCEPTION_REG_2                                                                     0x4e63
+#define regRLC_RLCS_EXCEPTION_REG_2_BASE_IDX                                                            1
+#define regRLC_RLCS_EXCEPTION_REG_3                                                                     0x4e64
+#define regRLC_RLCS_EXCEPTION_REG_3_BASE_IDX                                                            1
+#define regRLC_RLCS_EXCEPTION_REG_4                                                                     0x4e65
+#define regRLC_RLCS_EXCEPTION_REG_4_BASE_IDX                                                            1
+#define regRLC_RLCS_FENCE_CNTL                                                                          0x4e66
+#define regRLC_RLCS_FENCE_CNTL_BASE_IDX                                                                 1
+#define regRLC_RLCS_CGCG_REQUEST                                                                        0x4e67
+#define regRLC_RLCS_CGCG_REQUEST_BASE_IDX                                                               1
+#define regRLC_RLCS_CGCG_STATUS                                                                         0x4e68
+#define regRLC_RLCS_CGCG_STATUS_BASE_IDX                                                                1
+#define regRLC_RLCS_SOC_DS_CNTL                                                                         0x4e69
+#define regRLC_RLCS_SOC_DS_CNTL_BASE_IDX                                                                1
+#define regRLC_RLCS_GFX_DS_CNTL                                                                         0x4e6a
+#define regRLC_RLCS_GFX_DS_CNTL_BASE_IDX                                                                1
+#define regRLC_RLCS_GFX_DS_ALLOW_MASK_CNTL                                                              0x4e6b
+#define regRLC_RLCS_GFX_DS_ALLOW_MASK_CNTL_BASE_IDX                                                     1
+#define regRLC_GPM_STAT                                                                                 0x4e6c
+#define regRLC_GPM_STAT_BASE_IDX                                                                        1
+#define regRLC_RLCS_GPM_STAT                                                                            0x4e6c
+#define regRLC_RLCS_GPM_STAT_BASE_IDX                                                                   1
+#define regRLC_RLCS_ABORTED_PD_SEQUENCE                                                                 0x4e6d
+#define regRLC_RLCS_ABORTED_PD_SEQUENCE_BASE_IDX                                                        1
+#define regRLC_RLCS_IOV_CMD_STATUS                                                                      0x4e6e
+#define regRLC_RLCS_IOV_CMD_STATUS_BASE_IDX                                                             1
+#define regRLC_RLCS_IOV_CNTX_LOC_SIZE                                                                   0x4e6f
+#define regRLC_RLCS_IOV_CNTX_LOC_SIZE_BASE_IDX                                                          1
+#define regRLC_RLCS_IOV_SCH_BLOCK                                                                       0x4e70
+#define regRLC_RLCS_IOV_SCH_BLOCK_BASE_IDX                                                              1
+#define regRLC_RLCS_IOV_VM_BUSY_STATUS                                                                  0x4e71
+#define regRLC_RLCS_IOV_VM_BUSY_STATUS_BASE_IDX                                                         1
+#define regRLC_RLCS_GPM_STAT_2                                                                          0x4e72
+#define regRLC_RLCS_GPM_STAT_2_BASE_IDX                                                                 1
+#define regRLC_RLCS_GRBM_SOFT_RESET                                                                     0x4e73
+#define regRLC_RLCS_GRBM_SOFT_RESET_BASE_IDX                                                            1
+#define regRLC_RLCS_PG_CHANGE_STATUS                                                                    0x4e74
+#define regRLC_RLCS_PG_CHANGE_STATUS_BASE_IDX                                                           1
+#define regRLC_RLCS_PG_CHANGE_READ                                                                      0x4e75
+#define regRLC_RLCS_PG_CHANGE_READ_BASE_IDX                                                             1
+#define regRLC_RLCS_IH_SEMAPHORE                                                                        0x4e76
+#define regRLC_RLCS_IH_SEMAPHORE_BASE_IDX                                                               1
+#define regRLC_RLCS_IH_COOKIE_SEMAPHORE                                                                 0x4e77
+#define regRLC_RLCS_IH_COOKIE_SEMAPHORE_BASE_IDX                                                        1
+#define regRLC_RLCS_CP_INT_CTRL_1                                                                       0x4e78
+#define regRLC_RLCS_CP_INT_CTRL_1_BASE_IDX                                                              1
+#define regRLC_RLCS_CP_INT_CTRL_2                                                                       0x4e79
+#define regRLC_RLCS_CP_INT_CTRL_2_BASE_IDX                                                              1
+#define regRLC_RLCS_CP_INT_INFO_1                                                                       0x4e7a
+#define regRLC_RLCS_CP_INT_INFO_1_BASE_IDX                                                              1
+#define regRLC_RLCS_CP_INT_INFO_2                                                                       0x4e7b
+#define regRLC_RLCS_CP_INT_INFO_2_BASE_IDX                                                              1
+#define regRLC_RLCS_SPM_INT_CTRL                                                                        0x4e7c
+#define regRLC_RLCS_SPM_INT_CTRL_BASE_IDX                                                               1
+#define regRLC_RLCS_SPM_INT_INFO_1                                                                      0x4e7d
+#define regRLC_RLCS_SPM_INT_INFO_1_BASE_IDX                                                             1
+#define regRLC_RLCS_SPM_INT_INFO_2                                                                      0x4e7e
+#define regRLC_RLCS_SPM_INT_INFO_2_BASE_IDX                                                             1
+#define regRLC_RLCS_DSM_TRIG                                                                            0x4e7f
+#define regRLC_RLCS_DSM_TRIG_BASE_IDX                                                                   1
+#define regRLC_RLCS_BOOTLOAD_STATUS                                                                     0x4e80
+#define regRLC_RLCS_BOOTLOAD_STATUS_BASE_IDX                                                            1
+#define regRLC_RLCS_GRBM_IDLE_BUSY_STAT                                                                 0x4e81
+#define regRLC_RLCS_GRBM_IDLE_BUSY_STAT_BASE_IDX                                                        1
+#define regRLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL                                                             0x4e82
+#define regRLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL_BASE_IDX                                                    1
+#define regRLC_RLCS_CMP_IDLE_CNTL                                                                       0x4e83
+#define regRLC_RLCS_CMP_IDLE_CNTL_BASE_IDX                                                              1
+#define regRLC_RLCS_GENERAL_0                                                                           0x4e84
+#define regRLC_RLCS_GENERAL_0_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_1                                                                           0x4e85
+#define regRLC_RLCS_GENERAL_1_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_2                                                                           0x4e86
+#define regRLC_RLCS_GENERAL_2_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_3                                                                           0x4e87
+#define regRLC_RLCS_GENERAL_3_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_4                                                                           0x4e88
+#define regRLC_RLCS_GENERAL_4_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_5                                                                           0x4e89
+#define regRLC_RLCS_GENERAL_5_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_6                                                                           0x4e8a
+#define regRLC_RLCS_GENERAL_6_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_7                                                                           0x4e8b
+#define regRLC_RLCS_GENERAL_7_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_8                                                                           0x4e8c
+#define regRLC_RLCS_GENERAL_8_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_9                                                                           0x4e8d
+#define regRLC_RLCS_GENERAL_9_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_10                                                                          0x4e8e
+#define regRLC_RLCS_GENERAL_10_BASE_IDX                                                                 1
+#define regRLC_RLCS_GENERAL_11                                                                          0x4e8f
+#define regRLC_RLCS_GENERAL_11_BASE_IDX                                                                 1
+#define regRLC_RLCS_GENERAL_12                                                                          0x4e90
+#define regRLC_RLCS_GENERAL_12_BASE_IDX                                                                 1
+#define regRLC_RLCS_GENERAL_13                                                                          0x4e91
+#define regRLC_RLCS_GENERAL_13_BASE_IDX                                                                 1
+#define regRLC_RLCS_GENERAL_14                                                                          0x4e92
+#define regRLC_RLCS_GENERAL_14_BASE_IDX                                                                 1
+#define regRLC_RLCS_GENERAL_15                                                                          0x4e93
+#define regRLC_RLCS_GENERAL_15_BASE_IDX                                                                 1
+#define regRLC_RLCS_GENERAL_16                                                                          0x4e94
+#define regRLC_RLCS_GENERAL_16_BASE_IDX                                                                 1
+#define regRLC_RLCS_AUXILIARY_REG_1                                                                     0x4ec1
+#define regRLC_RLCS_AUXILIARY_REG_1_BASE_IDX                                                            1
+#define regRLC_RLCS_AUXILIARY_REG_2                                                                     0x4ec2
+#define regRLC_RLCS_AUXILIARY_REG_2_BASE_IDX                                                            1
+#define regRLC_RLCS_AUXILIARY_REG_3                                                                     0x4ec3
+#define regRLC_RLCS_AUXILIARY_REG_3_BASE_IDX                                                            1
+#define regRLC_RLCS_AUXILIARY_REG_4                                                                     0x4ec4
+#define regRLC_RLCS_AUXILIARY_REG_4_BASE_IDX                                                            1
+#define regRLC_RLCS_SPM_SQTT_MODE                                                                       0x4ec5
+#define regRLC_RLCS_SPM_SQTT_MODE_BASE_IDX                                                              1
+#define regRLC_RLCS_CP_DMA_SRCID_OVER                                                                   0x4ec6
+#define regRLC_RLCS_CP_DMA_SRCID_OVER_BASE_IDX                                                          1
+#define regRLC_RLCS_BOOTLOAD_ID_STATUS1                                                                 0x4ec7
+#define regRLC_RLCS_BOOTLOAD_ID_STATUS1_BASE_IDX                                                        1
+#define regRLC_RLCS_BOOTLOAD_ID_STATUS2                                                                 0x4ec8
+#define regRLC_RLCS_BOOTLOAD_ID_STATUS2_BASE_IDX                                                        1
+#define regRLC_RLCS_KMD_LOG_CNTL1                                                                       0x4ec9
+#define regRLC_RLCS_KMD_LOG_CNTL1_BASE_IDX                                                              1
+#define regRLC_RLCS_KMD_LOG_CNTL2                                                                       0x4eca
+#define regRLC_RLCS_KMD_LOG_CNTL2_BASE_IDX                                                              1
+#define regRLC_RLCS_GPM_LEGACY_INT_STAT                                                                 0x4ecb
+#define regRLC_RLCS_GPM_LEGACY_INT_STAT_BASE_IDX                                                        1
+#define regRLC_RLCS_GPM_LEGACY_INT_DISABLE                                                              0x4ecc
+#define regRLC_RLCS_GPM_LEGACY_INT_DISABLE_BASE_IDX                                                     1
+#define regRLC_RLCS_GCR_DATA_0                                                                          0x4ed3
+#define regRLC_RLCS_GCR_DATA_0_BASE_IDX                                                                 1
+#define regRLC_RLCS_GCR_DATA_1                                                                          0x4ed4
+#define regRLC_RLCS_GCR_DATA_1_BASE_IDX                                                                 1
+#define regRLC_RLCS_GCR_DATA_2                                                                          0x4ed5
+#define regRLC_RLCS_GCR_DATA_2_BASE_IDX                                                                 1
+#define regRLC_RLCS_GCR_DATA_3                                                                          0x4ed6
+#define regRLC_RLCS_GCR_DATA_3_BASE_IDX                                                                 1
+#define regRLC_RLCS_GCR_DATA_4                                                                          0x4ed7
+#define regRLC_RLCS_GCR_DATA_4_BASE_IDX                                                                 1
+#define regRLC_RLCS_GCR_STATUS                                                                          0x4ed8
+#define regRLC_RLCS_GCR_STATUS_BASE_IDX                                                                 1
+#define regRLC_RLCS_PERFMON_CLK_CNTL_UCODE                                                              0x4ed9
+#define regRLC_RLCS_PERFMON_CLK_CNTL_UCODE_BASE_IDX                                                     1
+#define regRLC_RLCS_UTCL2_CNTL                                                                          0x4eda
+#define regRLC_RLCS_UTCL2_CNTL_BASE_IDX                                                                 1
+#define regRLC_RLCS_IMU_RLC_MSG_DATA0                                                                   0x4edb
+#define regRLC_RLCS_IMU_RLC_MSG_DATA0_BASE_IDX                                                          1
+#define regRLC_RLCS_IMU_RLC_MSG_DATA1                                                                   0x4edc
+#define regRLC_RLCS_IMU_RLC_MSG_DATA1_BASE_IDX                                                          1
+#define regRLC_RLCS_IMU_RLC_MSG_DATA2                                                                   0x4edd
+#define regRLC_RLCS_IMU_RLC_MSG_DATA2_BASE_IDX                                                          1
+#define regRLC_RLCS_IMU_RLC_MSG_DATA3                                                                   0x4ede
+#define regRLC_RLCS_IMU_RLC_MSG_DATA3_BASE_IDX                                                          1
+#define regRLC_RLCS_IMU_RLC_MSG_DATA4                                                                   0x4edf
+#define regRLC_RLCS_IMU_RLC_MSG_DATA4_BASE_IDX                                                          1
+#define regRLC_RLCS_IMU_RLC_MSG_CONTROL                                                                 0x4ee0
+#define regRLC_RLCS_IMU_RLC_MSG_CONTROL_BASE_IDX                                                        1
+#define regRLC_RLCS_IMU_RLC_MSG_CNTL                                                                    0x4ee1
+#define regRLC_RLCS_IMU_RLC_MSG_CNTL_BASE_IDX                                                           1
+#define regRLC_RLCS_RLC_IMU_MSG_DATA0                                                                   0x4ee2
+#define regRLC_RLCS_RLC_IMU_MSG_DATA0_BASE_IDX                                                          1
+#define regRLC_RLCS_RLC_IMU_MSG_CONTROL                                                                 0x4ee3
+#define regRLC_RLCS_RLC_IMU_MSG_CONTROL_BASE_IDX                                                        1
+#define regRLC_RLCS_RLC_IMU_MSG_CNTL                                                                    0x4ee4
+#define regRLC_RLCS_RLC_IMU_MSG_CNTL_BASE_IDX                                                           1
+#define regRLC_RLCS_IMU_RLC_MUTEX_CNTL                                                                  0x4ee5
+#define regRLC_RLCS_IMU_RLC_MUTEX_CNTL_BASE_IDX                                                         1
+#define regRLC_RLCS_IMU_RLC_STATUS                                                                      0x4ee6
+#define regRLC_RLCS_IMU_RLC_STATUS_BASE_IDX                                                             1
+#define regRLC_RLCS_RLC_IMU_STATUS                                                                      0x4ee7
+#define regRLC_RLCS_RLC_IMU_STATUS_BASE_IDX                                                             1
+#define regRLC_RLCS_IMU_RAM_DATA_1                                                                      0x4ee8
+#define regRLC_RLCS_IMU_RAM_DATA_1_BASE_IDX                                                             1
+#define regRLC_RLCS_IMU_RAM_ADDR_1_LSB                                                                  0x4ee9
+#define regRLC_RLCS_IMU_RAM_ADDR_1_LSB_BASE_IDX                                                         1
+#define regRLC_RLCS_IMU_RAM_ADDR_1_MSB                                                                  0x4eea
+#define regRLC_RLCS_IMU_RAM_ADDR_1_MSB_BASE_IDX                                                         1
+#define regRLC_RLCS_IMU_RAM_DATA_0                                                                      0x4eeb
+#define regRLC_RLCS_IMU_RAM_DATA_0_BASE_IDX                                                             1
+#define regRLC_RLCS_IMU_RAM_ADDR_0_LSB                                                                  0x4eec
+#define regRLC_RLCS_IMU_RAM_ADDR_0_LSB_BASE_IDX                                                         1
+#define regRLC_RLCS_IMU_RAM_ADDR_0_MSB                                                                  0x4eed
+#define regRLC_RLCS_IMU_RAM_ADDR_0_MSB_BASE_IDX                                                         1
+#define regRLC_RLCS_IMU_RAM_CNTL                                                                        0x4eee
+#define regRLC_RLCS_IMU_RAM_CNTL_BASE_IDX                                                               1
+#define regRLC_RLCS_IMU_GFX_DOORBELL_FENCE                                                              0x4eef
+#define regRLC_RLCS_IMU_GFX_DOORBELL_FENCE_BASE_IDX                                                     1
+#define regRLC_RLCS_SDMA_INT_CNTL_1                                                                     0x4ef1
+#define regRLC_RLCS_SDMA_INT_CNTL_1_BASE_IDX                                                            1
+#define regRLC_RLCS_SDMA_INT_CNTL_2                                                                     0x4ef2
+#define regRLC_RLCS_SDMA_INT_CNTL_2_BASE_IDX                                                            1
+#define regRLC_RLCS_SDMA_INT_STAT                                                                       0x4ef3
+#define regRLC_RLCS_SDMA_INT_STAT_BASE_IDX                                                              1
+#define regRLC_RLCS_SDMA_INT_INFO                                                                       0x4ef4
+#define regRLC_RLCS_SDMA_INT_INFO_BASE_IDX                                                              1
+#define regRLC_RLCS_GFX_MEM_POWER_CTRL_0                                                                0x4ef5
+#define regRLC_RLCS_GFX_MEM_POWER_CTRL_0_BASE_IDX                                                       1
+#define regRLC_RLCS_GFX_MEM_POWER_CTRL_1                                                                0x4ef6
+#define regRLC_RLCS_GFX_MEM_POWER_CTRL_1_BASE_IDX                                                       1
+#define regRLC_RLCS_GFX_MEM_POWER_CTRL_2                                                                0x4ef7
+#define regRLC_RLCS_GFX_MEM_POWER_CTRL_2_BASE_IDX                                                       1
+#define regRLC_RLCS_IH_CTRL_1                                                                           0x4f01
+#define regRLC_RLCS_IH_CTRL_1_BASE_IDX                                                                  1
+#define regRLC_RLCS_IH_CTRL_2                                                                           0x4f02
+#define regRLC_RLCS_IH_CTRL_2_BASE_IDX                                                                  1
+#define regRLC_RLCS_IH_CTRL_3                                                                           0x4f03
+#define regRLC_RLCS_IH_CTRL_3_BASE_IDX                                                                  1
+#define regRLC_RLCS_IH_STATUS                                                                           0x4f04
+#define regRLC_RLCS_IH_STATUS_BASE_IDX                                                                  1
+#define regRLC_RLCS_FED_STATUS                                                                          0x4f05
+#define regRLC_RLCS_FED_STATUS_BASE_IDX                                                                 1
+#define regRLC_RLCS_FED_RESP                                                                            0x4f06
+#define regRLC_RLCS_FED_RESP_BASE_IDX                                                                   1
+#define regRLC_RLCS_FED_INT_MASK                                                                        0x4f07
+#define regRLC_RLCS_FED_INT_MASK_BASE_IDX                                                               1
+#define regRLC_RLCS_MEM_FED                                                                             0x4f08
+#define regRLC_RLCS_MEM_FED_BASE_IDX                                                                    1
+#define regRLC_RLCS_SE_PWR_CTRL                                                                         0x4f09
+#define regRLC_RLCS_SE_PWR_CTRL_BASE_IDX                                                                1
+#define regRLC_UTCL2_FED_STATUS_SNAP                                                                    0x4f0a
+#define regRLC_UTCL2_FED_STATUS_SNAP_BASE_IDX                                                           1
+#define regRLC_SE0_FED_STATUS_SNAP                                                                      0x4f0b
+#define regRLC_SE0_FED_STATUS_SNAP_BASE_IDX                                                             1
+#define regRLC_SE1_FED_STATUS_SNAP                                                                      0x4f0c
+#define regRLC_SE1_FED_STATUS_SNAP_BASE_IDX                                                             1
+#define regRLC_CANE_FED_STATUS_SNAP                                                                     0x4f0d
+#define regRLC_CANE_FED_STATUS_SNAP_BASE_IDX                                                            1
+#define regRLC_CONSOLIDATED_FED_STS                                                                     0x4f0e
+#define regRLC_CONSOLIDATED_FED_STS_BASE_IDX                                                            1
+#define regRLC_HOST_FED_ACK                                                                             0x4f0f
+#define regRLC_HOST_FED_ACK_BASE_IDX                                                                    1
+#define regRLC_RLCS_SB_RLC_ROM_STATUS                                                                   0x4f71
+#define regRLC_RLCS_SB_RLC_ROM_STATUS_BASE_IDX                                                          1
+#define regRLC_RLCS_SB_ROM_STATUS                                                                       0x4f72
+#define regRLC_RLCS_SB_ROM_STATUS_BASE_IDX                                                              1
+#define regRLC_RLCS_UTCL2_BUSY_CNTL                                                                     0x4f73
+#define regRLC_RLCS_UTCL2_BUSY_CNTL_BASE_IDX                                                            1
+#define regRLC_RLCS_UTCL2_BUSY_STAT                                                                     0x4f74
+#define regRLC_RLCS_UTCL2_BUSY_STAT_BASE_IDX                                                            1
+#define regRLC_RLCS_DEC_END                                                                             0x4fff
+#define regRLC_RLCS_DEC_END_BASE_IDX                                                                    1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pfvfdec_rlc
+// base address: 0x2a600
+#define regRLC_SAFE_MODE                                                                                0x0980
+#define regRLC_SAFE_MODE_BASE_IDX                                                                       1
+#define regRLC_SPM_SAMPLE_CNT                                                                           0x0981
+#define regRLC_SPM_SAMPLE_CNT_BASE_IDX                                                                  1
+#define regRLC_SPM_MC_CNTL                                                                              0x0982
+#define regRLC_SPM_MC_CNTL_BASE_IDX                                                                     1
+#define regRLC_SPM_INT_CNTL                                                                             0x0983
+#define regRLC_SPM_INT_CNTL_BASE_IDX                                                                    1
+#define regRLC_SPM_INT_STATUS                                                                           0x0984
+#define regRLC_SPM_INT_STATUS_BASE_IDX                                                                  1
+#define regRLC_SPM_INT_INFO_1                                                                           0x0985
+#define regRLC_SPM_INT_INFO_1_BASE_IDX                                                                  1
+#define regRLC_SPM_INT_INFO_2                                                                           0x0986
+#define regRLC_SPM_INT_INFO_2_BASE_IDX                                                                  1
+#define regRLC_CSIB_ADDR_LO                                                                             0x0987
+#define regRLC_CSIB_ADDR_LO_BASE_IDX                                                                    1
+#define regRLC_CSIB_ADDR_HI                                                                             0x0988
+#define regRLC_CSIB_ADDR_HI_BASE_IDX                                                                    1
+#define regRLC_CSIB_LENGTH                                                                              0x0989
+#define regRLC_CSIB_LENGTH_BASE_IDX                                                                     1
+#define regRLC_CP_SCHEDULERS                                                                            0x098a
+#define regRLC_CP_SCHEDULERS_BASE_IDX                                                                   1
+#define regRLC_CP_EOF_INT                                                                               0x098b
+#define regRLC_CP_EOF_INT_BASE_IDX                                                                      1
+#define regRLC_CP_EOF_INT_CNTL                                                                          0x098c
+#define regRLC_CP_EOF_INT_CNTL_BASE_IDX                                                                 1
+#define regRLC_SPARE_INT_0                                                                              0x098d
+#define regRLC_SPARE_INT_0_BASE_IDX                                                                     1
+#define regRLC_SPARE_INT_1                                                                              0x098e
+#define regRLC_SPARE_INT_1_BASE_IDX                                                                     1
+#define regRLC_SPARE_INT_2                                                                              0x098f
+#define regRLC_SPARE_INT_2_BASE_IDX                                                                     1
+#define regRLC_RLCV_SPARE_INT_1                                                                         0x0992
+#define regRLC_RLCV_SPARE_INT_1_BASE_IDX                                                                1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pwrdec
+// base address: 0x3c000
+#define regGRBMX_GLARB_BUSY_MASK                                                                        0x504d
+#define regGRBMX_GLARB_BUSY_MASK_BASE_IDX                                                               1
+#define regCGTT_IA_CLK_CTRL                                                                             0x5085
+#define regCGTT_IA_CLK_CTRL_BASE_IDX                                                                    1
+#define regCGTT_WD_CLK_CTRL                                                                             0x5086
+#define regCGTT_WD_CLK_CTRL_BASE_IDX                                                                    1
+#define regCGTT_CP_CLK_CTRL                                                                             0x50b0
+#define regCGTT_CP_CLK_CTRL_BASE_IDX                                                                    1
+#define regCGTT_CPF_CLK_CTRL                                                                            0x50b1
+#define regCGTT_CPF_CLK_CTRL_BASE_IDX                                                                   1
+#define regCGTT_CPC_CLK_CTRL                                                                            0x50b2
+#define regCGTT_CPC_CLK_CTRL_BASE_IDX                                                                   1
+#define regCGTT_RLC_CLK_CTRL                                                                            0x50b5
+#define regCGTT_RLC_CLK_CTRL_BASE_IDX                                                                   1
+#define regGFX_ICG_GCR_CTRL                                                                             0x50c2
+#define regGFX_ICG_GCR_CTRL_BASE_IDX                                                                    1
+#define regGC_EA_CPWD_ICG_CTRL                                                                          0x50c6
+#define regGC_EA_CPWD_ICG_CTRL_BASE_IDX                                                                 1
+#define regGFX_ICG_GC_CAC_CLK_CTRL                                                                      0x50d8
+#define regGFX_ICG_GC_CAC_CLK_CTRL_BASE_IDX                                                             1
+#define regGFX_ICG_GRBM_CTRL                                                                            0x50e0
+#define regGFX_ICG_GRBM_CTRL_BASE_IDX                                                                   1
+#define regCC_GC_HBM_DISABLE                                                                            0x50e1
+#define regCC_GC_HBM_DISABLE_BASE_IDX                                                                   1
+#define regGRBMX_LPDDR_DISABLE_0                                                                        0x50e2
+#define regGRBMX_LPDDR_DISABLE_0_BASE_IDX                                                               1
+#define regGRBMX_LPDDR_DISABLE_1                                                                        0x50e3
+#define regGRBMX_LPDDR_DISABLE_1_BASE_IDX                                                               1
+#define regGLARBI_GLARBR_MGCG_OVERRIDE                                                                  0x50f4
+#define regGLARBI_GLARBR_MGCG_OVERRIDE_BASE_IDX                                                         1
+#define regICG_GLARBA_CTRL                                                                              0x50f5
+#define regICG_GLARBA_CTRL_BASE_IDX                                                                     1
+#define regICG_GLARBC_CLK_CTRL                                                                          0x50f6
+#define regICG_GLARBC_CLK_CTRL_BASE_IDX                                                                 1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pspdec
+// base address: 0x3f000
+#define regGC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0                                                           0x5c61
+#define regGC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0_BASE_IDX                                                  1
+#define regGC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1                                                           0x5c62
+#define regGC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1_BASE_IDX                                                  1
+#define regGC_EA_CPWD_SDP_ERR_CTRL                                                                      0x5c64
+#define regGC_EA_CPWD_SDP_ERR_CTRL_BASE_IDX                                                             1
+#define regGC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2                                                           0x5c66
+#define regGC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2_BASE_IDX                                                  1
+#define regGC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3                                                           0x5c67
+#define regGC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3_BASE_IDX                                                  1
+#define regGRBM_SRCID_CAM_INDEX                                                                         0x5e00
+#define regGRBM_SRCID_CAM_INDEX_BASE_IDX                                                                1
+#define regGRBM_SRCID_CAM_DATA                                                                          0x5e01
+#define regGRBM_SRCID_CAM_DATA_BASE_IDX                                                                 1
+#define regGRBM_IOV_RANGE_ENABLE                                                                        0x5e06
+#define regGRBM_IOV_RANGE_ENABLE_BASE_IDX                                                               1
+#define regGRBM_IOV_RANGE_READ_BLOCK_ENABLE                                                             0x5e08
+#define regGRBM_IOV_RANGE_READ_BLOCK_ENABLE_BASE_IDX                                                    1
+#define regGRBM_SEC_CNTL                                                                                0x5e0d
+#define regGRBM_SEC_CNTL_BASE_IDX                                                                       1
+#define regGRBM_IOV_SPARE_RANGE_START                                                                   0x5e0e
+#define regGRBM_IOV_SPARE_RANGE_START_BASE_IDX                                                          1
+#define regGRBM_IOV_SPARE_RANGE_END                                                                     0x5e0f
+#define regGRBM_IOV_SPARE_RANGE_END_BASE_IDX                                                            1
+#define regGRBM_CAM_INDEX                                                                               0x5e10
+#define regGRBM_CAM_INDEX_BASE_IDX                                                                      1
+#define regGRBM_CAM_DATA                                                                                0x5e11
+#define regGRBM_CAM_DATA_BASE_IDX                                                                       1
+#define regGRBM_CAM_DATA_UPPER                                                                          0x5e12
+#define regGRBM_CAM_DATA_UPPER_BASE_IDX                                                                 1
+#define regRLC_REG_SEC_INT_STATUS                                                                       0x5f3d
+#define regRLC_REG_SEC_INT_STATUS_BASE_IDX                                                              1
+#define regRLC_FWL_FIRST_VIOL_ADDR                                                                      0x5f3e
+#define regRLC_FWL_FIRST_VIOL_ADDR_BASE_IDX                                                             1
+#define regRLC_FWL_FIRST_VIOL_ADDR_HI                                                                   0x5f3f
+#define regRLC_FWL_FIRST_VIOL_ADDR_HI_BASE_IDX                                                          1
+#define regRLC_UTC_BYPASS_CNTL                                                                          0x5f42
+#define regRLC_UTC_BYPASS_CNTL_BASE_IDX                                                                 1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_cp_pspdec
+// base address: 0x3f000
+#define regCP_MES_DM_INDEX_ADDR                                                                         0x5c00
+#define regCP_MES_DM_INDEX_ADDR_BASE_IDX                                                                1
+#define regCP_MES_DM_INDEX_DATA                                                                         0x5c01
+#define regCP_MES_DM_INDEX_DATA_BASE_IDX                                                                1
+#define regCP_MEC_DM_INDEX_ADDR                                                                         0x5c02
+#define regCP_MEC_DM_INDEX_ADDR_BASE_IDX                                                                1
+#define regCP_MEC_DM_INDEX_DATA                                                                         0x5c03
+#define regCP_MEC_DM_INDEX_DATA_BASE_IDX                                                                1
+#define regCP_GFX_RS64_DM_INDEX_ADDR                                                                    0x5c04
+#define regCP_GFX_RS64_DM_INDEX_ADDR_BASE_IDX                                                           1
+#define regCP_GFX_RS64_DM_INDEX_DATA                                                                    0x5c05
+#define regCP_GFX_RS64_DM_INDEX_DATA_BASE_IDX                                                           1
+#define regCPG_PSP_DEBUG                                                                                0x5c10
+#define regCPG_PSP_DEBUG_BASE_IDX                                                                       1
+#define regCPC_PSP_DEBUG                                                                                0x5c11
+#define regCPC_PSP_DEBUG_BASE_IDX                                                                       1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_ch_pwrdec
+// base address: 0x3c3b0
+#define regCHI_CHR_MGCG_OVERRIDE                                                                        0x50ec
+#define regCHI_CHR_MGCG_OVERRIDE_BASE_IDX                                                               1
+#define regICG_CHA_CTRL                                                                                 0x50ed
+#define regICG_CHA_CTRL_BASE_IDX                                                                        1
+#define regICG_CHC_CLK_CTRL                                                                             0x50ee
+#define regICG_CHC_CLK_CTRL_BASE_IDX                                                                    1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_gfx_imu_cpwd_gfx_imudec
+// base address: 0x38000
+#define regGFX_IMU_C2PMSG_0                                                                             0x4000
+#define regGFX_IMU_C2PMSG_0_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_1                                                                             0x4001
+#define regGFX_IMU_C2PMSG_1_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_2                                                                             0x4002
+#define regGFX_IMU_C2PMSG_2_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_3                                                                             0x4003
+#define regGFX_IMU_C2PMSG_3_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_4                                                                             0x4004
+#define regGFX_IMU_C2PMSG_4_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_5                                                                             0x4005
+#define regGFX_IMU_C2PMSG_5_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_6                                                                             0x4006
+#define regGFX_IMU_C2PMSG_6_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_7                                                                             0x4007
+#define regGFX_IMU_C2PMSG_7_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_8                                                                             0x4008
+#define regGFX_IMU_C2PMSG_8_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_9                                                                             0x4009
+#define regGFX_IMU_C2PMSG_9_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_10                                                                            0x400a
+#define regGFX_IMU_C2PMSG_10_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_11                                                                            0x400b
+#define regGFX_IMU_C2PMSG_11_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_12                                                                            0x400c
+#define regGFX_IMU_C2PMSG_12_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_13                                                                            0x400d
+#define regGFX_IMU_C2PMSG_13_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_14                                                                            0x400e
+#define regGFX_IMU_C2PMSG_14_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_15                                                                            0x400f
+#define regGFX_IMU_C2PMSG_15_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_16                                                                            0x4010
+#define regGFX_IMU_C2PMSG_16_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_17                                                                            0x4011
+#define regGFX_IMU_C2PMSG_17_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_18                                                                            0x4012
+#define regGFX_IMU_C2PMSG_18_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_19                                                                            0x4013
+#define regGFX_IMU_C2PMSG_19_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_20                                                                            0x4014
+#define regGFX_IMU_C2PMSG_20_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_21                                                                            0x4015
+#define regGFX_IMU_C2PMSG_21_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_22                                                                            0x4016
+#define regGFX_IMU_C2PMSG_22_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_23                                                                            0x4017
+#define regGFX_IMU_C2PMSG_23_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_24                                                                            0x4018
+#define regGFX_IMU_C2PMSG_24_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_25                                                                            0x4019
+#define regGFX_IMU_C2PMSG_25_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_26                                                                            0x401a
+#define regGFX_IMU_C2PMSG_26_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_27                                                                            0x401b
+#define regGFX_IMU_C2PMSG_27_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_28                                                                            0x401c
+#define regGFX_IMU_C2PMSG_28_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_29                                                                            0x401d
+#define regGFX_IMU_C2PMSG_29_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_30                                                                            0x401e
+#define regGFX_IMU_C2PMSG_30_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_31                                                                            0x401f
+#define regGFX_IMU_C2PMSG_31_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_32                                                                            0x4020
+#define regGFX_IMU_C2PMSG_32_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_33                                                                            0x4021
+#define regGFX_IMU_C2PMSG_33_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_34                                                                            0x4022
+#define regGFX_IMU_C2PMSG_34_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_35                                                                            0x4023
+#define regGFX_IMU_C2PMSG_35_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_36                                                                            0x4024
+#define regGFX_IMU_C2PMSG_36_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_37                                                                            0x4025
+#define regGFX_IMU_C2PMSG_37_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_38                                                                            0x4026
+#define regGFX_IMU_C2PMSG_38_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_39                                                                            0x4027
+#define regGFX_IMU_C2PMSG_39_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_40                                                                            0x4028
+#define regGFX_IMU_C2PMSG_40_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_41                                                                            0x4029
+#define regGFX_IMU_C2PMSG_41_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_42                                                                            0x402a
+#define regGFX_IMU_C2PMSG_42_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_43                                                                            0x402b
+#define regGFX_IMU_C2PMSG_43_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_44                                                                            0x402c
+#define regGFX_IMU_C2PMSG_44_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_45                                                                            0x402d
+#define regGFX_IMU_C2PMSG_45_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_46                                                                            0x402e
+#define regGFX_IMU_C2PMSG_46_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_47                                                                            0x402f
+#define regGFX_IMU_C2PMSG_47_BASE_IDX                                                                   1
+#define regGFX_IMU_MSG_FLAGS                                                                            0x403f
+#define regGFX_IMU_MSG_FLAGS_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_ACCESS_CTRL0                                                                  0x4040
+#define regGFX_IMU_C2PMSG_ACCESS_CTRL0_BASE_IDX                                                         1
+#define regGFX_IMU_C2PMSG_ACCESS_CTRL1                                                                  0x4041
+#define regGFX_IMU_C2PMSG_ACCESS_CTRL1_BASE_IDX                                                         1
+#define regGFX_IMU_PWRMGT_IRQ_CTRL                                                                      0x4042
+#define regGFX_IMU_PWRMGT_IRQ_CTRL_BASE_IDX                                                             1
+#define regGFX_IMU_MP1_MUTEX                                                                            0x4043
+#define regGFX_IMU_MP1_MUTEX_BASE_IDX                                                                   1
+#define regGFX_IMU_RLC_DATA_4                                                                           0x4046
+#define regGFX_IMU_RLC_DATA_4_BASE_IDX                                                                  1
+#define regGFX_IMU_RLC_DATA_3                                                                           0x4047
+#define regGFX_IMU_RLC_DATA_3_BASE_IDX                                                                  1
+#define regGFX_IMU_RLC_DATA_2                                                                           0x4048
+#define regGFX_IMU_RLC_DATA_2_BASE_IDX                                                                  1
+#define regGFX_IMU_RLC_DATA_1                                                                           0x4049
+#define regGFX_IMU_RLC_DATA_1_BASE_IDX                                                                  1
+#define regGFX_IMU_RLC_DATA_0                                                                           0x404a
+#define regGFX_IMU_RLC_DATA_0_BASE_IDX                                                                  1
+#define regGFX_IMU_RLC_CMD                                                                              0x404b
+#define regGFX_IMU_RLC_CMD_BASE_IDX                                                                     1
+#define regGFX_IMU_RLC_MUTEX                                                                            0x404c
+#define regGFX_IMU_RLC_MUTEX_BASE_IDX                                                                   1
+#define regGFX_IMU_RLC_MSG_STATUS                                                                       0x404f
+#define regGFX_IMU_RLC_MSG_STATUS_BASE_IDX                                                              1
+#define regRLC_GFX_IMU_DATA_0                                                                           0x4052
+#define regRLC_GFX_IMU_DATA_0_BASE_IDX                                                                  1
+#define regRLC_GFX_IMU_CMD                                                                              0x4053
+#define regRLC_GFX_IMU_CMD_BASE_IDX                                                                     1
+#define regGFX_IMU_RLC_STATUS                                                                           0x4054
+#define regGFX_IMU_RLC_STATUS_BASE_IDX                                                                  1
+#define regGFX_IMU_STATUS                                                                               0x4055
+#define regGFX_IMU_STATUS_BASE_IDX                                                                      1
+#define regGFX_IMU_SOC_DATA                                                                             0x4059
+#define regGFX_IMU_SOC_DATA_BASE_IDX                                                                    1
+#define regGFX_IMU_SOC_ADDR                                                                             0x405a
+#define regGFX_IMU_SOC_ADDR_BASE_IDX                                                                    1
+#define regGFX_IMU_SOC_REQ                                                                              0x405b
+#define regGFX_IMU_SOC_REQ_BASE_IDX                                                                     1
+#define regGFX_IMU_VF_CTRL                                                                              0x405c
+#define regGFX_IMU_VF_CTRL_BASE_IDX                                                                     1
+#define regGFX_IMU_SCRATCH_0                                                                            0x4068
+#define regGFX_IMU_SCRATCH_0_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_1                                                                            0x4069
+#define regGFX_IMU_SCRATCH_1_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_2                                                                            0x406a
+#define regGFX_IMU_SCRATCH_2_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_3                                                                            0x406b
+#define regGFX_IMU_SCRATCH_3_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_4                                                                            0x406c
+#define regGFX_IMU_SCRATCH_4_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_5                                                                            0x406d
+#define regGFX_IMU_SCRATCH_5_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_6                                                                            0x406e
+#define regGFX_IMU_SCRATCH_6_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_7                                                                            0x406f
+#define regGFX_IMU_SCRATCH_7_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_8                                                                            0x4070
+#define regGFX_IMU_SCRATCH_8_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_9                                                                            0x4071
+#define regGFX_IMU_SCRATCH_9_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_10                                                                           0x4072
+#define regGFX_IMU_SCRATCH_10_BASE_IDX                                                                  1
+#define regGFX_IMU_SCRATCH_11                                                                           0x4073
+#define regGFX_IMU_SCRATCH_11_BASE_IDX                                                                  1
+#define regGFX_IMU_SCRATCH_12                                                                           0x4074
+#define regGFX_IMU_SCRATCH_12_BASE_IDX                                                                  1
+#define regGFX_IMU_SCRATCH_13                                                                           0x4075
+#define regGFX_IMU_SCRATCH_13_BASE_IDX                                                                  1
+#define regGFX_IMU_SCRATCH_14                                                                           0x4076
+#define regGFX_IMU_SCRATCH_14_BASE_IDX                                                                  1
+#define regGFX_IMU_SCRATCH_15                                                                           0x4077
+#define regGFX_IMU_SCRATCH_15_BASE_IDX                                                                  1
+#define regGFX_IMU_FW_GTS_LO                                                                            0x4078
+#define regGFX_IMU_FW_GTS_LO_BASE_IDX                                                                   1
+#define regGFX_IMU_FW_GTS_HI                                                                            0x4079
+#define regGFX_IMU_FW_GTS_HI_BASE_IDX                                                                   1
+#define regGFX_IMU_GTS_OFFSET_LO                                                                        0x407a
+#define regGFX_IMU_GTS_OFFSET_LO_BASE_IDX                                                               1
+#define regGFX_IMU_GTS_OFFSET_HI                                                                        0x407b
+#define regGFX_IMU_GTS_OFFSET_HI_BASE_IDX                                                               1
+#define regGFX_IMU_RLC_GTS_OFFSET_LO                                                                    0x407c
+#define regGFX_IMU_RLC_GTS_OFFSET_LO_BASE_IDX                                                           1
+#define regGFX_IMU_RLC_GTS_OFFSET_HI                                                                    0x407d
+#define regGFX_IMU_RLC_GTS_OFFSET_HI_BASE_IDX                                                           1
+#define regGFX_IMU_CORE_INT_STATUS                                                                      0x407f
+#define regGFX_IMU_CORE_INT_STATUS_BASE_IDX                                                             1
+#define regGFX_IMU_PIC_INT_MASK                                                                         0x4080
+#define regGFX_IMU_PIC_INT_MASK_BASE_IDX                                                                1
+#define regGFX_IMU_PIC_INT_LVL                                                                          0x4081
+#define regGFX_IMU_PIC_INT_LVL_BASE_IDX                                                                 1
+#define regGFX_IMU_PIC_INT_EDGE                                                                         0x4082
+#define regGFX_IMU_PIC_INT_EDGE_BASE_IDX                                                                1
+#define regGFX_IMU_PIC_INT_PRI_0                                                                        0x4083
+#define regGFX_IMU_PIC_INT_PRI_0_BASE_IDX                                                               1
+#define regGFX_IMU_PIC_INT_PRI_1                                                                        0x4084
+#define regGFX_IMU_PIC_INT_PRI_1_BASE_IDX                                                               1
+#define regGFX_IMU_PIC_INT_PRI_2                                                                        0x4085
+#define regGFX_IMU_PIC_INT_PRI_2_BASE_IDX                                                               1
+#define regGFX_IMU_PIC_INT_PRI_3                                                                        0x4086
+#define regGFX_IMU_PIC_INT_PRI_3_BASE_IDX                                                               1
+#define regGFX_IMU_PIC_INT_PRI_4                                                                        0x4087
+#define regGFX_IMU_PIC_INT_PRI_4_BASE_IDX                                                               1
+#define regGFX_IMU_PIC_INT_PRI_5                                                                        0x4088
+#define regGFX_IMU_PIC_INT_PRI_5_BASE_IDX                                                               1
+#define regGFX_IMU_PIC_INT_PRI_6                                                                        0x4089
+#define regGFX_IMU_PIC_INT_PRI_6_BASE_IDX                                                               1
+#define regGFX_IMU_PIC_INT_PRI_7                                                                        0x408a
+#define regGFX_IMU_PIC_INT_PRI_7_BASE_IDX                                                               1
+#define regGFX_IMU_PIC_INT_STATUS                                                                       0x408b
+#define regGFX_IMU_PIC_INT_STATUS_BASE_IDX                                                              1
+#define regGFX_IMU_PIC_INTR                                                                             0x408c
+#define regGFX_IMU_PIC_INTR_BASE_IDX                                                                    1
+#define regGFX_IMU_PIC_INTR_ID                                                                          0x408d
+#define regGFX_IMU_PIC_INTR_ID_BASE_IDX                                                                 1
+#define regGFX_IMU_IH_CTRL_1                                                                            0x4090
+#define regGFX_IMU_IH_CTRL_1_BASE_IDX                                                                   1
+#define regGFX_IMU_IH_CTRL_2                                                                            0x4091
+#define regGFX_IMU_IH_CTRL_2_BASE_IDX                                                                   1
+#define regGFX_IMU_IH_CTRL_3                                                                            0x4092
+#define regGFX_IMU_IH_CTRL_3_BASE_IDX                                                                   1
+#define regGFX_IMU_IH_STATUS                                                                            0x4093
+#define regGFX_IMU_IH_STATUS_BASE_IDX                                                                   1
+#define regGFX_IMU_FUSESTRAP                                                                            0x4094
+#define regGFX_IMU_FUSESTRAP_BASE_IDX                                                                   1
+#define regGFX_IMU_GFXCLK_BYPASS_CTRL                                                                   0x409c
+#define regGFX_IMU_GFXCLK_BYPASS_CTRL_BASE_IDX                                                          1
+#define regGFX_IMU_CLK_CTRL                                                                             0x409d
+#define regGFX_IMU_CLK_CTRL_BASE_IDX                                                                    1
+#define regGFX_IMU_DOORBELL_CONTROL                                                                     0x409f
+#define regGFX_IMU_DOORBELL_CONTROL_BASE_IDX                                                            1
+#define regGFX_IMU_RLC_CG_CTRL                                                                          0x40a0
+#define regGFX_IMU_RLC_CG_CTRL_BASE_IDX                                                                 1
+#define regGFX_IMU_RLC_THROTTLE_GFX                                                                     0x40a1
+#define regGFX_IMU_RLC_THROTTLE_GFX_BASE_IDX                                                            1
+#define regGFX_IMU_RLC_RESET_VECTOR                                                                     0x40a2
+#define regGFX_IMU_RLC_RESET_VECTOR_BASE_IDX                                                            1
+#define regGFX_IMU_RLC_OVERRIDE                                                                         0x40a3
+#define regGFX_IMU_RLC_OVERRIDE_BASE_IDX                                                                1
+#define regGFX_IMU_DPM_CONTROL                                                                          0x40a8
+#define regGFX_IMU_DPM_CONTROL_BASE_IDX                                                                 1
+#define regGFX_IMU_DPM_ACC                                                                              0x40a9
+#define regGFX_IMU_DPM_ACC_BASE_IDX                                                                     1
+#define regGFX_IMU_DPM_REF_COUNTER                                                                      0x40aa
+#define regGFX_IMU_DPM_REF_COUNTER_BASE_IDX                                                             1
+#define regGFX_IMU_RLC_RAM_INDEX                                                                        0x40ac
+#define regGFX_IMU_RLC_RAM_INDEX_BASE_IDX                                                               1
+#define regGFX_IMU_RLC_RAM_ADDR_HIGH                                                                    0x40ad
+#define regGFX_IMU_RLC_RAM_ADDR_HIGH_BASE_IDX                                                           1
+#define regGFX_IMU_RLC_RAM_ADDR_LOW                                                                     0x40ae
+#define regGFX_IMU_RLC_RAM_ADDR_LOW_BASE_IDX                                                            1
+#define regGFX_IMU_RLC_RAM_DATA                                                                         0x40af
+#define regGFX_IMU_RLC_RAM_DATA_BASE_IDX                                                                1
+#define regGFX_IMU_FENCE_CTRL                                                                           0x40b0
+#define regGFX_IMU_FENCE_CTRL_BASE_IDX                                                                  1
+#define regGFX_IMU_FENCE_LOG_INIT                                                                       0x40b1
+#define regGFX_IMU_FENCE_LOG_INIT_BASE_IDX                                                              1
+#define regGFX_IMU_FENCE_LOG_ADDR                                                                       0x40b2
+#define regGFX_IMU_FENCE_LOG_ADDR_BASE_IDX                                                              1
+#define regGFX_IMU_PROGRAM_CTR                                                                          0x40b5
+#define regGFX_IMU_PROGRAM_CTR_BASE_IDX                                                                 1
+#define regGFX_IMU_CORE_CTRL                                                                            0x40b6
+#define regGFX_IMU_CORE_CTRL_BASE_IDX                                                                   1
+#define regGFX_IMU_CORE_STATUS                                                                          0x40b7
+#define regGFX_IMU_CORE_STATUS_BASE_IDX                                                                 1
+#define regGFX_IMU_PWROKRAW                                                                             0x40b8
+#define regGFX_IMU_PWROKRAW_BASE_IDX                                                                    1
+#define regGFX_IMU_PWROK                                                                                0x40b9
+#define regGFX_IMU_PWROK_BASE_IDX                                                                       1
+#define regGFX_IMU_GAP_PWROK                                                                            0x40ba
+#define regGFX_IMU_GAP_PWROK_BASE_IDX                                                                   1
+#define regGFX_IMU_RESETn                                                                               0x40bb
+#define regGFX_IMU_RESETn_BASE_IDX                                                                      1
+#define regGFX_IMU_GFX_RESET_CTRL                                                                       0x40bc
+#define regGFX_IMU_GFX_RESET_CTRL_BASE_IDX                                                              1
+#define regGFX_IMU_VDCI_RESET_CTRL                                                                      0x40be
+#define regGFX_IMU_VDCI_RESET_CTRL_BASE_IDX                                                             1
+#define regGFX_IMU_GFX_ISO_CTRL                                                                         0x40bf
+#define regGFX_IMU_GFX_ISO_CTRL_BASE_IDX                                                                1
+#define regGFX_IMU_TIMER0_CTRL0                                                                         0x40c0
+#define regGFX_IMU_TIMER0_CTRL0_BASE_IDX                                                                1
+#define regGFX_IMU_TIMER0_CTRL1                                                                         0x40c1
+#define regGFX_IMU_TIMER0_CTRL1_BASE_IDX                                                                1
+#define regGFX_IMU_TIMER0_CMP_AUTOINC                                                                   0x40c2
+#define regGFX_IMU_TIMER0_CMP_AUTOINC_BASE_IDX                                                          1
+#define regGFX_IMU_TIMER0_CMP_INTEN                                                                     0x40c3
+#define regGFX_IMU_TIMER0_CMP_INTEN_BASE_IDX                                                            1
+#define regGFX_IMU_TIMER0_CMP0                                                                          0x40c4
+#define regGFX_IMU_TIMER0_CMP0_BASE_IDX                                                                 1
+#define regGFX_IMU_TIMER0_CMP1                                                                          0x40c5
+#define regGFX_IMU_TIMER0_CMP1_BASE_IDX                                                                 1
+#define regGFX_IMU_TIMER0_CMP3                                                                          0x40c7
+#define regGFX_IMU_TIMER0_CMP3_BASE_IDX                                                                 1
+#define regGFX_IMU_TIMER0_VALUE                                                                         0x40c8
+#define regGFX_IMU_TIMER0_VALUE_BASE_IDX                                                                1
+#define regGFX_IMU_TIMER1_CTRL0                                                                         0x40c9
+#define regGFX_IMU_TIMER1_CTRL0_BASE_IDX                                                                1
+#define regGFX_IMU_TIMER1_CTRL1                                                                         0x40ca
+#define regGFX_IMU_TIMER1_CTRL1_BASE_IDX                                                                1
+#define regGFX_IMU_TIMER1_CMP_AUTOINC                                                                   0x40cb
+#define regGFX_IMU_TIMER1_CMP_AUTOINC_BASE_IDX                                                          1
+#define regGFX_IMU_TIMER1_CMP_INTEN                                                                     0x40cc
+#define regGFX_IMU_TIMER1_CMP_INTEN_BASE_IDX                                                            1
+#define regGFX_IMU_TIMER1_CMP0                                                                          0x40cd
+#define regGFX_IMU_TIMER1_CMP0_BASE_IDX                                                                 1
+#define regGFX_IMU_TIMER1_CMP1                                                                          0x40ce
+#define regGFX_IMU_TIMER1_CMP1_BASE_IDX                                                                 1
+#define regGFX_IMU_TIMER1_CMP3                                                                          0x40d0
+#define regGFX_IMU_TIMER1_CMP3_BASE_IDX                                                                 1
+#define regGFX_IMU_TIMER1_VALUE                                                                         0x40d1
+#define regGFX_IMU_TIMER1_VALUE_BASE_IDX                                                                1
+#define regGFX_IMU_FUSE_CTRL                                                                            0x40e0
+#define regGFX_IMU_FUSE_CTRL_BASE_IDX                                                                   1
+#define regGFX_IMU_D_RAM_ADDR                                                                           0x40fc
+#define regGFX_IMU_D_RAM_ADDR_BASE_IDX                                                                  1
+#define regGFX_IMU_D_RAM_DATA                                                                           0x40fd
+#define regGFX_IMU_D_RAM_DATA_BASE_IDX                                                                  1
+#define regGFX_IMU_GFX_IH_GASKET_CTRL                                                                   0x40ff
+#define regGFX_IMU_GFX_IH_GASKET_CTRL_BASE_IDX                                                          1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_gfx_imu_cpwd_gfx_imu_pspdec
+// base address: 0x3fe00
+#define regGFX_IMU_RLC_BOOTLOADER_ADDR_HI                                                               0x5f81
+#define regGFX_IMU_RLC_BOOTLOADER_ADDR_HI_BASE_IDX                                                      1
+#define regGFX_IMU_RLC_BOOTLOADER_ADDR_LO                                                               0x5f82
+#define regGFX_IMU_RLC_BOOTLOADER_ADDR_LO_BASE_IDX                                                      1
+#define regGFX_IMU_RLC_BOOTLOADER_SIZE                                                                  0x5f83
+#define regGFX_IMU_RLC_BOOTLOADER_SIZE_BASE_IDX                                                         1
+#define regGFX_IMU_I_RAM_ADDR                                                                           0x5f90
+#define regGFX_IMU_I_RAM_ADDR_BASE_IDX                                                                  1
+#define regGFX_IMU_I_RAM_DATA                                                                           0x5f91
+#define regGFX_IMU_I_RAM_DATA_BASE_IDX                                                                  1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_grbmhdec
+// base address: 0x8180
+#define regGRBMH_CNTL                                                                                   0x0e00
+#define regGRBMH_CNTL_BASE_IDX                                                                          0
+#define regGRBMH_INTF_CNTL                                                                              0x0e01
+#define regGRBMH_INTF_CNTL_BASE_IDX                                                                     0
+#define regGRBMH_STATUS                                                                                 0x0e02
+#define regGRBMH_STATUS_BASE_IDX                                                                        0
+#define regGRBMH_FGCG0_TARG                                                                             0x0e04
+#define regGRBMH_FGCG0_TARG_BASE_IDX                                                                    0
+#define regGRBMH_SOFT_RESET                                                                             0x0e05
+#define regGRBMH_SOFT_RESET_BASE_IDX                                                                    0
+#define regGRBMH_READ_ERROR                                                                             0x0e06
+#define regGRBMH_READ_ERROR_BASE_IDX                                                                    0
+#define regGRBMH_GFX_CLKEN_CNTL                                                                         0x0e0d
+#define regGRBMH_GFX_CLKEN_CNTL_BASE_IDX                                                                0
+#define regGRBMH_FGCG2_MISC                                                                             0x0e0e
+#define regGRBMH_FGCG2_MISC_BASE_IDX                                                                    0
+#define regGRBMH_FGCG1_TARGVF                                                                           0x0e0f
+#define regGRBMH_FGCG1_TARGVF_BASE_IDX                                                                  0
+#define regGRBMH_NOWHERE                                                                                0x0e10
+#define regGRBMH_NOWHERE_BASE_IDX                                                                       0
+#define regGRBMH_INVALID_PIPE                                                                           0x0e12
+#define regGRBMH_INVALID_PIPE_BASE_IDX                                                                  0
+#define regGRBMH_SYNC                                                                                   0x0e13
+#define regGRBMH_SYNC_BASE_IDX                                                                          0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_padec
+// base address: 0x8800
+#define regGE_RATE_CNTL_1                                                                               0x0ff4
+#define regGE_RATE_CNTL_1_BASE_IDX                                                                      0
+#define regGE_RATE_CNTL_2                                                                               0x0ff5
+#define regGE_RATE_CNTL_2_BASE_IDX                                                                      0
+#define regCC_GC_SHADER_ARRAY_CONFIG                                                                    0x100f
+#define regCC_GC_SHADER_ARRAY_CONFIG_BASE_IDX                                                           0
+#define regGE_SE_CNTL_STATUS                                                                            0x1011
+#define regGE_SE_CNTL_STATUS_BASE_IDX                                                                   0
+#define regGE_SPI_IF_SAFE_REG                                                                           0x1018
+#define regGE_SPI_IF_SAFE_REG_BASE_IDX                                                                  0
+#define regGE_PA_IF_SAFE_REG                                                                            0x1019
+#define regGE_PA_IF_SAFE_REG_BASE_IDX                                                                   0
+#define regPA_SU_DEBUG_CNTL                                                                             0x1020
+#define regPA_SU_DEBUG_CNTL_BASE_IDX                                                                    0
+#define regPA_CL_CNTL_STATUS                                                                            0x1024
+#define regPA_CL_CNTL_STATUS_BASE_IDX                                                                   0
+#define regPA_CL_ENHANCE                                                                                0x1025
+#define regPA_CL_ENHANCE_BASE_IDX                                                                       0
+#define regPA_CL_RESET_DEBUG                                                                            0x1026
+#define regPA_CL_RESET_DEBUG_BASE_IDX                                                                   0
+#define regPA_SU_CNTL_STATUS                                                                            0x1034
+#define regPA_SU_CNTL_STATUS_BASE_IDX                                                                   0
+#define regPA_SC_FIFO_DEPTH_CNTL                                                                        0x1035
+#define regPA_SC_FIFO_DEPTH_CNTL_BASE_IDX                                                               0
+#define regPA_PH_DEBUG_CNTL                                                                             0x1082
+#define regPA_PH_DEBUG_CNTL_BASE_IDX                                                                    0
+#define regPA_SC_DEBUG_CNTL                                                                             0x1096
+#define regPA_SC_DEBUG_CNTL_BASE_IDX                                                                    0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_sqdec
+// base address: 0x8c00
+#define regSQ_CONFIG                                                                                    0x10a0
+#define regSQ_CONFIG_BASE_IDX                                                                           0
+#define regSQC_CONFIG                                                                                   0x10a1
+#define regSQC_CONFIG_BASE_IDX                                                                          0
+#define regSQ_RANDOM_WAVE_PRI                                                                           0x10a3
+#define regSQ_RANDOM_WAVE_PRI_BASE_IDX                                                                  0
+#define regSQG_STATUS                                                                                   0x10a4
+#define regSQG_STATUS_BASE_IDX                                                                          0
+#define regSQ_FIFO_SIZES                                                                                0x10a5
+#define regSQ_FIFO_SIZES_BASE_IDX                                                                       0
+#define regSQ_DSM_CNTL                                                                                  0x10a6
+#define regSQ_DSM_CNTL_BASE_IDX                                                                         0
+#define regSQ_DSM_CNTL2                                                                                 0x10a7
+#define regSQ_DSM_CNTL2_BASE_IDX                                                                        0
+#define regSQ_PIT_WEIGHT                                                                                0x10a8
+#define regSQ_PIT_WEIGHT_BASE_IDX                                                                       0
+#define regSQG_THREAD_TRACE_CONFIG                                                                      0x10aa
+#define regSQG_THREAD_TRACE_CONFIG_BASE_IDX                                                             0
+#define regSP_CONFIG                                                                                    0x10ab
+#define regSP_CONFIG_BASE_IDX                                                                           0
+#define regSQ_ARB_CONFIG                                                                                0x10ac
+#define regSQ_ARB_CONFIG_BASE_IDX                                                                       0
+#define regSQ_DYN_VGPR                                                                                  0x10ad
+#define regSQ_DYN_VGPR_BASE_IDX                                                                         0
+#define regSQ_SHADER_CYCLES_LO                                                                          0x10ae
+#define regSQ_SHADER_CYCLES_LO_BASE_IDX                                                                 0
+#define regSQ_SHADER_CYCLES_HI                                                                          0x10af
+#define regSQ_SHADER_CYCLES_HI_BASE_IDX                                                                 0
+#define regSQ_DEBUG_HOST_TRAP_STATUS                                                                    0x10b6
+#define regSQ_DEBUG_HOST_TRAP_STATUS_BASE_IDX                                                           0
+#define regSQG_GL1X_CTRL                                                                                0x10b8
+#define regSQG_GL1X_CTRL_BASE_IDX                                                                       0
+#define regSQG_GL1X_STATUS                                                                              0x10b9
+#define regSQG_GL1X_STATUS_BASE_IDX                                                                     0
+#define regSQG_CONFIG                                                                                   0x10ba
+#define regSQG_CONFIG_BASE_IDX                                                                          0
+#define regSQ_PERF_SNAPSHOT_CTRL                                                                        0x10bb
+#define regSQ_PERF_SNAPSHOT_CTRL_BASE_IDX                                                               0
+#define regCC_GC_SHADER_RATE_CONFIG_1                                                                   0x10bc
+#define regCC_GC_SHADER_RATE_CONFIG_1_BASE_IDX                                                          0
+#define regSQ_INTERRUPT_AUTO_MASK                                                                       0x10be
+#define regSQ_INTERRUPT_AUTO_MASK_BASE_IDX                                                              0
+#define regSQ_INTERRUPT_MSG_CTRL                                                                        0x10bf
+#define regSQ_INTERRUPT_MSG_CTRL_BASE_IDX                                                               0
+#define regSQ_WATCH0_ADDR_H                                                                             0x10d0
+#define regSQ_WATCH0_ADDR_H_BASE_IDX                                                                    0
+#define regSQ_WATCH0_ADDR_L                                                                             0x10d1
+#define regSQ_WATCH0_ADDR_L_BASE_IDX                                                                    0
+#define regSQ_WATCH0_CNTL                                                                               0x10d2
+#define regSQ_WATCH0_CNTL_BASE_IDX                                                                      0
+#define regSQ_WATCH1_ADDR_H                                                                             0x10d3
+#define regSQ_WATCH1_ADDR_H_BASE_IDX                                                                    0
+#define regSQ_WATCH1_ADDR_L                                                                             0x10d4
+#define regSQ_WATCH1_ADDR_L_BASE_IDX                                                                    0
+#define regSQ_WATCH1_CNTL                                                                               0x10d5
+#define regSQ_WATCH1_CNTL_BASE_IDX                                                                      0
+#define regSQ_WATCH2_ADDR_H                                                                             0x10d6
+#define regSQ_WATCH2_ADDR_H_BASE_IDX                                                                    0
+#define regSQ_WATCH2_ADDR_L                                                                             0x10d7
+#define regSQ_WATCH2_ADDR_L_BASE_IDX                                                                    0
+#define regSQ_WATCH2_CNTL                                                                               0x10d8
+#define regSQ_WATCH2_CNTL_BASE_IDX                                                                      0
+#define regSQ_WATCH3_ADDR_H                                                                             0x10d9
+#define regSQ_WATCH3_ADDR_H_BASE_IDX                                                                    0
+#define regSQ_WATCH3_ADDR_L                                                                             0x10da
+#define regSQ_WATCH3_ADDR_L_BASE_IDX                                                                    0
+#define regSQ_WATCH3_CNTL                                                                               0x10db
+#define regSQ_WATCH3_CNTL_BASE_IDX                                                                      0
+#define regSQG_UTCL0_STATUS                                                                             0x10dc
+#define regSQG_UTCL0_STATUS_BASE_IDX                                                                    0
+#define regSQG_UTCL0_CNTL1                                                                              0x10dd
+#define regSQG_UTCL0_CNTL1_BASE_IDX                                                                     0
+#define regSQG_UTCL0_CNTL2                                                                              0x10de
+#define regSQG_UTCL0_CNTL2_BASE_IDX                                                                     0
+#define regSQG_UTCL0_RETRY                                                                              0x10df
+#define regSQG_UTCL0_RETRY_BASE_IDX                                                                     0
+#define regSQG_VMID_FED_STATUS                                                                          0x10e0
+#define regSQG_VMID_FED_STATUS_BASE_IDX                                                                 0
+#define regSQG_VMID_FED_CLEAR                                                                           0x10e1
+#define regSQG_VMID_FED_CLEAR_BASE_IDX                                                                  0
+#define regSQG_SE_PIT_CONTROL                                                                           0x10e2
+#define regSQG_SE_PIT_CONTROL_BASE_IDX                                                                  0
+#define regSQG_UTCL0_RETRY_TIMER_THRESHOLD_HI                                                           0x10e3
+#define regSQG_UTCL0_RETRY_TIMER_THRESHOLD_HI_BASE_IDX                                                  0
+#define regSQG_UTCL0_RETRY_TIMER_THRESHOLD_LO                                                           0x10e4
+#define regSQG_UTCL0_RETRY_TIMER_THRESHOLD_LO_BASE_IDX                                                  0
+#define regSQG_WS_INSERTION_COUNT                                                                       0x10e5
+#define regSQG_WS_INSERTION_COUNT_BASE_IDX                                                              0
+#define regSQG_WS_SPI_STATUS                                                                            0x10e6
+#define regSQG_WS_SPI_STATUS_BASE_IDX                                                                   0
+#define regSQ_IND_INDEX                                                                                 0x1118
+#define regSQ_IND_INDEX_BASE_IDX                                                                        0
+#define regSQ_IND_DATA                                                                                  0x1119
+#define regSQ_IND_DATA_BASE_IDX                                                                         0
+#define regSQ_CMD                                                                                       0x111b
+#define regSQ_CMD_BASE_IDX                                                                              0
+#define regSQC_WSM_CONTROL                                                                              0x1174
+#define regSQC_WSM_CONTROL_BASE_IDX                                                                     0
+#define regSQC_PIT_CONTROL0                                                                             0x1175
+#define regSQC_PIT_CONTROL0_BASE_IDX                                                                    0
+#define regSQC_PIT_CONTROL1                                                                             0x1176
+#define regSQC_PIT_CONTROL1_BASE_IDX                                                                    0
+#define regSQC_PIT_CONTROL2                                                                             0x1177
+#define regSQC_PIT_CONTROL2_BASE_IDX                                                                    0
+#define regSQC_UTCL0_RETRY                                                                              0x1178
+#define regSQC_UTCL0_RETRY_BASE_IDX                                                                     0
+#define regSQC_MISC_CONFIG                                                                              0x1179
+#define regSQC_MISC_CONFIG_BASE_IDX                                                                     0
+#define regSQ_TIMEOUT_CONFIG                                                                            0x117a
+#define regSQ_TIMEOUT_CONFIG_BASE_IDX                                                                   0
+#define regSQ_TIMEOUT_STATUS_SIMD01                                                                     0x117b
+#define regSQ_TIMEOUT_STATUS_SIMD01_BASE_IDX                                                            0
+#define regSQ_TIMEOUT_STATUS_SIMD23                                                                     0x117c
+#define regSQ_TIMEOUT_STATUS_SIMD23_BASE_IDX                                                            0
+#define regSQC_UTCL0_STATUS                                                                             0x117d
+#define regSQC_UTCL0_STATUS_BASE_IDX                                                                    0
+#define regSQC_UTCL0_CNTL1                                                                              0x117e
+#define regSQC_UTCL0_CNTL1_BASE_IDX                                                                     0
+#define regSQC_UTCL0_CNTL2                                                                              0x117f
+#define regSQC_UTCL0_CNTL2_BASE_IDX                                                                     0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_shsdec
+// base address: 0x9000
+#define regSX_DEBUG_BUSY                                                                                0x11b4
+#define regSX_DEBUG_BUSY_BASE_IDX                                                                       0
+#define regSX_DEBUG_BUSY_2                                                                              0x11b5
+#define regSX_DEBUG_BUSY_2_BASE_IDX                                                                     0
+#define regSX_DEBUG_BUSY_3                                                                              0x11b6
+#define regSX_DEBUG_BUSY_3_BASE_IDX                                                                     0
+#define regSX_DEBUG_BUSY_4                                                                              0x11b7
+#define regSX_DEBUG_BUSY_4_BASE_IDX                                                                     0
+#define regSX_DEBUG_1                                                                                   0x11b8
+#define regSX_DEBUG_1_BASE_IDX                                                                          0
+#define regSX_DEBUG_BUSY_5                                                                              0x11b9
+#define regSX_DEBUG_BUSY_5_BASE_IDX                                                                     0
+#define regSX_DEBUG_BUSY_6                                                                              0x11ba
+#define regSX_DEBUG_BUSY_6_BASE_IDX                                                                     0
+#define regSX_DEBUG_BUSY_7                                                                              0x11bb
+#define regSX_DEBUG_BUSY_7_BASE_IDX                                                                     0
+#define regSX_DEBUG_BUSY_8                                                                              0x11bc
+#define regSX_DEBUG_BUSY_8_BASE_IDX                                                                     0
+#define regSX_DEBUG_BUSY_9                                                                              0x11bd
+#define regSX_DEBUG_BUSY_9_BASE_IDX                                                                     0
+#define regSX_DEBUG_BUSY_10                                                                             0x11be
+#define regSX_DEBUG_BUSY_10_BASE_IDX                                                                    0
+#define regSPI_PS_MAX_WAVE_ID                                                                           0x11da
+#define regSPI_PS_MAX_WAVE_ID_BASE_IDX                                                                  0
+#define regSPI_SCRATCH_ADDR_STATUS                                                                      0x11db
+#define regSPI_SCRATCH_ADDR_STATUS_BASE_IDX                                                             0
+#define regSPI_GFX_CNTL                                                                                 0x11dc
+#define regSPI_GFX_CNTL_BASE_IDX                                                                        0
+#define regSPI_DEBUG_CNTL_2                                                                             0x11de
+#define regSPI_DEBUG_CNTL_2_BASE_IDX                                                                    0
+#define regSPI_DEBUG_CNTL_3                                                                             0x11df
+#define regSPI_DEBUG_CNTL_3_BASE_IDX                                                                    0
+#define regSPI_DEBUG_CNTL                                                                               0x11e1
+#define regSPI_DEBUG_CNTL_BASE_IDX                                                                      0
+#define regSPI_DEBUG_READ                                                                               0x11e2
+#define regSPI_DEBUG_READ_BASE_IDX                                                                      0
+#define regSPI_DSM_CNTL                                                                                 0x11e3
+#define regSPI_DSM_CNTL_BASE_IDX                                                                        0
+#define regSPI_DSM_CNTL2                                                                                0x11e4
+#define regSPI_DSM_CNTL2_BASE_IDX                                                                       0
+#define regSPI_EDC_CNT                                                                                  0x11e5
+#define regSPI_EDC_CNT_BASE_IDX                                                                         0
+#define regSPIRA_DEBUG_READ                                                                             0x11e6
+#define regSPIRA_DEBUG_READ_BASE_IDX                                                                    0
+#define regSPI_DEBUG_BUSY                                                                               0x11f0
+#define regSPI_DEBUG_BUSY_BASE_IDX                                                                      0
+#define regSPI_CONFIG_PS_CU_EN                                                                          0x11f2
+#define regSPI_CONFIG_PS_CU_EN_BASE_IDX                                                                 0
+#define regSPI_CONFIG_CU_MASK_GFX0                                                                      0x11f3
+#define regSPI_CONFIG_CU_MASK_GFX0_BASE_IDX                                                             0
+#define regSPI_CONFIG_CU_MASK_HP3D0                                                                     0x11f4
+#define regSPI_CONFIG_CU_MASK_HP3D0_BASE_IDX                                                            0
+#define regSPI_CONFIG_CU_MASK_GFX1                                                                      0x11f5
+#define regSPI_CONFIG_CU_MASK_GFX1_BASE_IDX                                                             0
+#define regSPI_CONFIG_CU_MASK_HP3D1                                                                     0x11f6
+#define regSPI_CONFIG_CU_MASK_HP3D1_BASE_IDX                                                            0
+#define regSPI_CONFIG_CU_MASK_CS0                                                                       0x11f7
+#define regSPI_CONFIG_CU_MASK_CS0_BASE_IDX                                                              0
+#define regSPI_CONFIG_CU_MASK_CS1                                                                       0x11f8
+#define regSPI_CONFIG_CU_MASK_CS1_BASE_IDX                                                              0
+#define regSPI_CONFIG_CU_MASK_CS2                                                                       0x11f9
+#define regSPI_CONFIG_CU_MASK_CS2_BASE_IDX                                                              0
+#define regSPI_CONFIG_CU_MASK_CS3                                                                       0x11fa
+#define regSPI_CONFIG_CU_MASK_CS3_BASE_IDX                                                              0
+#define regSPI_CONFIG_CU_MASK_CS4                                                                       0x11fb
+#define regSPI_CONFIG_CU_MASK_CS4_BASE_IDX                                                              0
+#define regSPI_CONFIG_CU_MASK_CS5                                                                       0x11fc
+#define regSPI_CONFIG_CU_MASK_CS5_BASE_IDX                                                              0
+#define regSPI_CONFIG_CU_MASK_CS6                                                                       0x11fd
+#define regSPI_CONFIG_CU_MASK_CS6_BASE_IDX                                                              0
+#define regSPI_CONFIG_CU_MASK_CS7                                                                       0x11fe
+#define regSPI_CONFIG_CU_MASK_CS7_BASE_IDX                                                              0
+#define regSPI_WF_LIFETIME_CNTL                                                                         0x124a
+#define regSPI_WF_LIFETIME_CNTL_BASE_IDX                                                                0
+#define regSPI_WF_LIFETIME_LIMIT_0                                                                      0x124b
+#define regSPI_WF_LIFETIME_LIMIT_0_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_2                                                                      0x124d
+#define regSPI_WF_LIFETIME_LIMIT_2_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_3                                                                      0x124e
+#define regSPI_WF_LIFETIME_LIMIT_3_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_4                                                                      0x124f
+#define regSPI_WF_LIFETIME_LIMIT_4_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_5                                                                      0x1250
+#define regSPI_WF_LIFETIME_LIMIT_5_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_STATUS_0                                                                     0x1255
+#define regSPI_WF_LIFETIME_STATUS_0_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_2                                                                     0x1257
+#define regSPI_WF_LIFETIME_STATUS_2_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_4                                                                     0x1259
+#define regSPI_WF_LIFETIME_STATUS_4_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_6                                                                     0x125b
+#define regSPI_WF_LIFETIME_STATUS_6_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_7                                                                     0x125c
+#define regSPI_WF_LIFETIME_STATUS_7_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_9                                                                     0x125e
+#define regSPI_WF_LIFETIME_STATUS_9_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_11                                                                    0x1260
+#define regSPI_WF_LIFETIME_STATUS_11_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_13                                                                    0x1262
+#define regSPI_WF_LIFETIME_STATUS_13_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_14                                                                    0x1263
+#define regSPI_WF_LIFETIME_STATUS_14_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_15                                                                    0x1264
+#define regSPI_WF_LIFETIME_STATUS_15_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_16                                                                    0x1265
+#define regSPI_WF_LIFETIME_STATUS_16_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_17                                                                    0x1266
+#define regSPI_WF_LIFETIME_STATUS_17_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_18                                                                    0x1267
+#define regSPI_WF_LIFETIME_STATUS_18_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_19                                                                    0x1268
+#define regSPI_WF_LIFETIME_STATUS_19_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_20                                                                    0x1269
+#define regSPI_WF_LIFETIME_STATUS_20_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_DEBUG                                                                        0x126a
+#define regSPI_WF_LIFETIME_DEBUG_BASE_IDX                                                               0
+#define regSPI_WF_LIFETIME_STATUS_21                                                                    0x126b
+#define regSPI_WF_LIFETIME_STATUS_21_BASE_IDX                                                           0
+#define regSPI_WGP_WORK_PENDING                                                                         0x126c
+#define regSPI_WGP_WORK_PENDING_BASE_IDX                                                                0
+#define regSPI_CREST_MODE                                                                               0x126d
+#define regSPI_CREST_MODE_BASE_IDX                                                                      0
+#define regSPI_SLAVE_DEBUG_BUSY                                                                         0x1273
+#define regSPI_SLAVE_DEBUG_BUSY_BASE_IDX                                                                0
+#define regSPI_LB_CTR_CTRL                                                                              0x1274
+#define regSPI_LB_CTR_CTRL_BASE_IDX                                                                     0
+#define regSPI_LB_WGP_MASK                                                                              0x1275
+#define regSPI_LB_WGP_MASK_BASE_IDX                                                                     0
+#define regSPI_LB_DATA_REG                                                                              0x1276
+#define regSPI_LB_DATA_REG_BASE_IDX                                                                     0
+#define regSPI_PG_ENABLE_STATIC_WGP_MASK                                                                0x1277
+#define regSPI_PG_ENABLE_STATIC_WGP_MASK_BASE_IDX                                                       0
+#define regSPI_GDS_CREDITS                                                                              0x1278
+#define regSPI_GDS_CREDITS_BASE_IDX                                                                     0
+#define regSPI_SX_EXPORT_BUFFER_SIZES                                                                   0x1279
+#define regSPI_SX_EXPORT_BUFFER_SIZES_BASE_IDX                                                          0
+#define regSPI_SX_SCOREBOARD_BUFFER_SIZES                                                               0x127a
+#define regSPI_SX_SCOREBOARD_BUFFER_SIZES_BASE_IDX                                                      0
+#define regSPI_CSQ_WF_ACTIVE_STATUS                                                                     0x127b
+#define regSPI_CSQ_WF_ACTIVE_STATUS_BASE_IDX                                                            0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_0                                                                    0x127c
+#define regSPI_CSQ_WF_ACTIVE_COUNT_0_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_1                                                                    0x127d
+#define regSPI_CSQ_WF_ACTIVE_COUNT_1_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_2                                                                    0x127e
+#define regSPI_CSQ_WF_ACTIVE_COUNT_2_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_3                                                                    0x127f
+#define regSPI_CSQ_WF_ACTIVE_COUNT_3_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_4                                                                    0x1280
+#define regSPI_CSQ_WF_ACTIVE_COUNT_4_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_5                                                                    0x1281
+#define regSPI_CSQ_WF_ACTIVE_COUNT_5_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_6                                                                    0x1282
+#define regSPI_CSQ_WF_ACTIVE_COUNT_6_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_7                                                                    0x1283
+#define regSPI_CSQ_WF_ACTIVE_COUNT_7_BASE_IDX                                                           0
+#define regSPI_LB_DATA_WAVES                                                                            0x1284
+#define regSPI_LB_DATA_WAVES_BASE_IDX                                                                   0
+#define regSPI_LB_DATA_PERWGP_WAVE_HSGS                                                                 0x1285
+#define regSPI_LB_DATA_PERWGP_WAVE_HSGS_BASE_IDX                                                        0
+#define regSPI_LB_DATA_PERWGP_WAVE_PS                                                                   0x1286
+#define regSPI_LB_DATA_PERWGP_WAVE_PS_BASE_IDX                                                          0
+#define regSPI_LB_DATA_PERWGP_WAVE_CS                                                                   0x1287
+#define regSPI_LB_DATA_PERWGP_WAVE_CS_BASE_IDX                                                          0
+#define regSPI_WF_ACTIVE_COUNT_GFX                                                                      0x1288
+#define regSPI_WF_ACTIVE_COUNT_GFX_BASE_IDX                                                             0
+#define regSPI_WF_ACTIVE_COUNT_HPG                                                                      0x1289
+#define regSPI_WF_ACTIVE_COUNT_HPG_BASE_IDX                                                             0
+#define regSPIS_DEBUG_READ                                                                              0x128a
+#define regSPIS_DEBUG_READ_BASE_IDX                                                                     0
+#define regBCI_DEBUG_READ                                                                               0x128b
+#define regBCI_DEBUG_READ_BASE_IDX                                                                      0
+#define regSPI_P0_TRAP_SCREEN_PSBA_LO                                                                   0x128c
+#define regSPI_P0_TRAP_SCREEN_PSBA_LO_BASE_IDX                                                          0
+#define regSPI_P0_TRAP_SCREEN_PSBA_HI                                                                   0x128d
+#define regSPI_P0_TRAP_SCREEN_PSBA_HI_BASE_IDX                                                          0
+#define regSPI_P0_TRAP_SCREEN_PSMA_LO                                                                   0x128e
+#define regSPI_P0_TRAP_SCREEN_PSMA_LO_BASE_IDX                                                          0
+#define regSPI_P0_TRAP_SCREEN_PSMA_HI                                                                   0x128f
+#define regSPI_P0_TRAP_SCREEN_PSMA_HI_BASE_IDX                                                          0
+#define regSPI_P0_TRAP_SCREEN_GPR_MIN                                                                   0x1290
+#define regSPI_P0_TRAP_SCREEN_GPR_MIN_BASE_IDX                                                          0
+#define regSPI_P1_TRAP_SCREEN_PSBA_LO                                                                   0x1291
+#define regSPI_P1_TRAP_SCREEN_PSBA_LO_BASE_IDX                                                          0
+#define regSPI_P1_TRAP_SCREEN_PSBA_HI                                                                   0x1292
+#define regSPI_P1_TRAP_SCREEN_PSBA_HI_BASE_IDX                                                          0
+#define regSPI_P1_TRAP_SCREEN_PSMA_LO                                                                   0x1293
+#define regSPI_P1_TRAP_SCREEN_PSMA_LO_BASE_IDX                                                          0
+#define regSPI_P1_TRAP_SCREEN_PSMA_HI                                                                   0x1294
+#define regSPI_P1_TRAP_SCREEN_PSMA_HI_BASE_IDX                                                          0
+#define regSPI_P1_TRAP_SCREEN_GPR_MIN                                                                   0x1295
+#define regSPI_P1_TRAP_SCREEN_GPR_MIN_BASE_IDX                                                          0
+#define regSPI_GFX_CRAWLER_CONFIG                                                                       0x1296
+#define regSPI_GFX_CRAWLER_CONFIG_BASE_IDX                                                              0
+#define regSPI_CS_CRAWLER_CONFIG                                                                        0x1297
+#define regSPI_CS_CRAWLER_CONFIG_BASE_IDX                                                               0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_tpdec
+// base address: 0x9400
+#define regTD_CNTL                                                                                      0x12c5
+#define regTD_CNTL_BASE_IDX                                                                             0
+#define regTD_STATUS                                                                                    0x12c6
+#define regTD_STATUS_BASE_IDX                                                                           0
+#define regTD_DEBUG_DATA                                                                                0x12c9
+#define regTD_DEBUG_DATA_BASE_IDX                                                                       0
+#define regTD_POWER_CNTL                                                                                0x12ca
+#define regTD_POWER_CNTL_BASE_IDX                                                                       0
+#define regTD_CNTL2                                                                                     0x12cb
+#define regTD_CNTL2_BASE_IDX                                                                            0
+#define regTD_DSM_CNTL                                                                                  0x12cf
+#define regTD_DSM_CNTL_BASE_IDX                                                                         0
+#define regTD_DSM_CNTL2                                                                                 0x12d0
+#define regTD_DSM_CNTL2_BASE_IDX                                                                        0
+#define regTD_SCRATCH                                                                                   0x12d3
+#define regTD_SCRATCH_BASE_IDX                                                                          0
+#define regTA_CNTL                                                                                      0x12e1
+#define regTA_CNTL_BASE_IDX                                                                             0
+#define regTA_CNTL_AUX                                                                                  0x12e2
+#define regTA_CNTL_AUX_BASE_IDX                                                                         0
+#define regTA_CNTL2                                                                                     0x12e5
+#define regTA_CNTL2_BASE_IDX                                                                            0
+#define regTA_STATUS                                                                                    0x12e8
+#define regTA_STATUS_BASE_IDX                                                                           0
+#define regTA_SCRATCH                                                                                   0x1304
+#define regTA_SCRATCH_BASE_IDX                                                                          0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_rbdec
+// base address: 0x9800
+#define regDB_DEBUG                                                                                     0x13ac
+#define regDB_DEBUG_BASE_IDX                                                                            0
+#define regDB_DEBUG2                                                                                    0x13ad
+#define regDB_DEBUG2_BASE_IDX                                                                           0
+#define regDB_DEBUG3                                                                                    0x13ae
+#define regDB_DEBUG3_BASE_IDX                                                                           0
+#define regDB_DEBUG4                                                                                    0x13af
+#define regDB_DEBUG4_BASE_IDX                                                                           0
+#define regDB_CREDIT_LIMIT                                                                              0x13b4
+#define regDB_CREDIT_LIMIT_BASE_IDX                                                                     0
+#define regDB_WATERMARKS                                                                                0x13b5
+#define regDB_WATERMARKS_BASE_IDX                                                                       0
+#define regDB_FREE_CACHELINES                                                                           0x13b7
+#define regDB_FREE_CACHELINES_BASE_IDX                                                                  0
+#define regDB_FIFO_DEPTH1                                                                               0x13b8
+#define regDB_FIFO_DEPTH1_BASE_IDX                                                                      0
+#define regDB_FIFO_DEPTH2                                                                               0x13b9
+#define regDB_FIFO_DEPTH2_BASE_IDX                                                                      0
+#define regDB_RING_CONTROL                                                                              0x13bb
+#define regDB_RING_CONTROL_BASE_IDX                                                                     0
+#define regDB_MEM_ARB_WATERMARKS                                                                        0x13bc
+#define regDB_MEM_ARB_WATERMARKS_BASE_IDX                                                               0
+#define regDB_FIFO_DEPTH3                                                                               0x13bd
+#define regDB_FIFO_DEPTH3_BASE_IDX                                                                      0
+#define regDB_DEBUG6                                                                                    0x13be
+#define regDB_DEBUG6_BASE_IDX                                                                           0
+#define regDB_EXCEPTION_CONTROL                                                                         0x13bf
+#define regDB_EXCEPTION_CONTROL_BASE_IDX                                                                0
+#define regDB_DEBUG7                                                                                    0x13d0
+#define regDB_DEBUG7_BASE_IDX                                                                           0
+#define regDB_DEBUG5                                                                                    0x13d1
+#define regDB_DEBUG5_BASE_IDX                                                                           0
+#define regDB_MEM_CONFIG                                                                                0x13d2
+#define regDB_MEM_CONFIG_BASE_IDX                                                                       0
+#define regDB_ARB_CONFIG                                                                                0x13d3
+#define regDB_ARB_CONFIG_BASE_IDX                                                                       0
+#define regDB_DFD_INDIRECT_SEL                                                                          0x13d4
+#define regDB_DFD_INDIRECT_SEL_BASE_IDX                                                                 0
+#define regDB_DFD_INDIRECT_DAT                                                                          0x13d5
+#define regDB_DFD_INDIRECT_DAT_BASE_IDX                                                                 0
+#define regDB_SUMMARIZER_TIMEOUTS                                                                       0x13d6
+#define regDB_SUMMARIZER_TIMEOUTS_BASE_IDX                                                              0
+#define regDB_FGCG_SRAMS_CLK_CTRL                                                                       0x13d7
+#define regDB_FGCG_SRAMS_CLK_CTRL_BASE_IDX                                                              0
+#define regDB_FGCG_INTERFACES_CLK_CTRL                                                                  0x13d8
+#define regDB_FGCG_INTERFACES_CLK_CTRL_BASE_IDX                                                         0
+#define regDB_FIFO_DEPTH4                                                                               0x13d9
+#define regDB_FIFO_DEPTH4_BASE_IDX                                                                      0
+#define regCC_RB_BACKEND_DISABLE                                                                        0x13dd
+#define regCC_RB_BACKEND_DISABLE_BASE_IDX                                                               0
+#define regGB_ADDR_CONFIG_1                                                                             0x13de
+#define regGB_ADDR_CONFIG_1_BASE_IDX                                                                    0
+#define regGB_BACKEND_MAP                                                                               0x13df
+#define regGB_BACKEND_MAP_BASE_IDX                                                                      0
+#define regGB_GPU_ID                                                                                    0x13e0
+#define regGB_GPU_ID_BASE_IDX                                                                           0
+#define regGB_ADDR_CONFIG_READ                                                                          0x13e2
+#define regGB_ADDR_CONFIG_READ_BASE_IDX                                                                 0
+#define regCB_HW_CONTROL_4                                                                              0x1422
+#define regCB_HW_CONTROL_4_BASE_IDX                                                                     0
+#define regCB_HW_CONTROL_3                                                                              0x1423
+#define regCB_HW_CONTROL_3_BASE_IDX                                                                     0
+#define regCB_HW_CONTROL                                                                                0x1424
+#define regCB_HW_CONTROL_BASE_IDX                                                                       0
+#define regCB_HW_CONTROL_1                                                                              0x1425
+#define regCB_HW_CONTROL_1_BASE_IDX                                                                     0
+#define regCB_HW_CONTROL_2                                                                              0x1426
+#define regCB_HW_CONTROL_2_BASE_IDX                                                                     0
+#define regCB_HW_MEM_ARBITER_CTL                                                                        0x1428
+#define regCB_HW_MEM_ARBITER_CTL_BASE_IDX                                                               0
+#define regCB_FGCG_SRAM_OVERRIDE                                                                        0x142a
+#define regCB_FGCG_SRAM_OVERRIDE_BASE_IDX                                                               0
+#define regCB_CACHE_EVICT_POINTS                                                                        0x142e
+#define regCB_CACHE_EVICT_POINTS_BASE_IDX                                                               0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_spipdec2
+// base address: 0x9c80
+#define regSPI_PQEV_CTRL                                                                                0x14c0
+#define regSPI_PQEV_CTRL_BASE_IDX                                                                       0
+#define regSPI_EXP_THROTTLE_CTRL                                                                        0x14c3
+#define regSPI_EXP_THROTTLE_CTRL_BASE_IDX                                                               0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_rmi_gfx_se_rmidec
+// base address: 0x2e200
+#define regRMI_GENERAL_CNTL                                                                             0x1880
+#define regRMI_GENERAL_CNTL_BASE_IDX                                                                    1
+#define regRMI_GENERAL_CNTL1                                                                            0x1881
+#define regRMI_GENERAL_CNTL1_BASE_IDX                                                                   1
+#define regRMI_GENERAL_STATUS                                                                           0x1882
+#define regRMI_GENERAL_STATUS_BASE_IDX                                                                  1
+#define regRMI_SUBBLOCK_STATUS0                                                                         0x1883
+#define regRMI_SUBBLOCK_STATUS0_BASE_IDX                                                                1
+#define regRMI_SUBBLOCK_STATUS1                                                                         0x1884
+#define regRMI_SUBBLOCK_STATUS1_BASE_IDX                                                                1
+#define regRMI_SUBBLOCK_STATUS2                                                                         0x1885
+#define regRMI_SUBBLOCK_STATUS2_BASE_IDX                                                                1
+#define regRMI_SUBBLOCK_STATUS3                                                                         0x1886
+#define regRMI_SUBBLOCK_STATUS3_BASE_IDX                                                                1
+#define regRMI_XBAR_CONFIG                                                                              0x1887
+#define regRMI_XBAR_CONFIG_BASE_IDX                                                                     1
+#define regRMI_PROBE_POP_LOGIC_CNTL                                                                     0x1888
+#define regRMI_PROBE_POP_LOGIC_CNTL_BASE_IDX                                                            1
+#define regRMI_UTC_XNACK_N_MISC_CNTL                                                                    0x1889
+#define regRMI_UTC_XNACK_N_MISC_CNTL_BASE_IDX                                                           1
+#define regRMI_DEMUX_CNTL                                                                               0x188a
+#define regRMI_DEMUX_CNTL_BASE_IDX                                                                      1
+#define regRMI_UTCL1_CNTL1                                                                              0x188b
+#define regRMI_UTCL1_CNTL1_BASE_IDX                                                                     1
+#define regRMI_UTCL1_CNTL2                                                                              0x188c
+#define regRMI_UTCL1_CNTL2_BASE_IDX                                                                     1
+#define regRMI_UTC_UNIT_CONFIG                                                                          0x188d
+#define regRMI_UTC_UNIT_CONFIG_BASE_IDX                                                                 1
+#define regRMI_TCIW_FORMATTER0_CNTL                                                                     0x188e
+#define regRMI_TCIW_FORMATTER0_CNTL_BASE_IDX                                                            1
+#define regRMI_TCIW_FORMATTER1_CNTL                                                                     0x188f
+#define regRMI_TCIW_FORMATTER1_CNTL_BASE_IDX                                                            1
+#define regRMI_SCOREBOARD_CNTL                                                                          0x1890
+#define regRMI_SCOREBOARD_CNTL_BASE_IDX                                                                 1
+#define regRMI_SCOREBOARD_STATUS0                                                                       0x1891
+#define regRMI_SCOREBOARD_STATUS0_BASE_IDX                                                              1
+#define regRMI_SCOREBOARD_STATUS1                                                                       0x1892
+#define regRMI_SCOREBOARD_STATUS1_BASE_IDX                                                              1
+#define regRMI_SCOREBOARD_STATUS2                                                                       0x1893
+#define regRMI_SCOREBOARD_STATUS2_BASE_IDX                                                              1
+#define regRMI_XBAR_ARBITER_CONFIG                                                                      0x1894
+#define regRMI_XBAR_ARBITER_CONFIG_BASE_IDX                                                             1
+#define regRMI_XBAR_ARBITER_CONFIG_1                                                                    0x1895
+#define regRMI_XBAR_ARBITER_CONFIG_1_BASE_IDX                                                           1
+#define regRMI_CLOCK_CNTRL                                                                              0x1896
+#define regRMI_CLOCK_CNTRL_BASE_IDX                                                                     1
+#define regRMI_UTCL1_STATUS                                                                             0x1897
+#define regRMI_UTCL1_STATUS_BASE_IDX                                                                    1
+#define regRMI_RB_GLX_CID_MAP                                                                           0x1898
+#define regRMI_RB_GLX_CID_MAP_BASE_IDX                                                                  1
+#define regRMI_XNACK_DEBUG                                                                              0x189e
+#define regRMI_XNACK_DEBUG_BASE_IDX                                                                     1
+#define regRMI_SPARE                                                                                    0x189f
+#define regRMI_SPARE_BASE_IDX                                                                           1
+#define regRMI_SPARE_1                                                                                  0x18a0
+#define regRMI_SPARE_1_BASE_IDX                                                                         1
+#define regRMI_SPARE_2                                                                                  0x18a1
+#define regRMI_SPARE_2_BASE_IDX                                                                         1
+#define regCC_RMI_REDUNDANCY                                                                            0x18a2
+#define regCC_RMI_REDUNDANCY_BASE_IDX                                                                   1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_utcl1dec
+// base address: 0x9fb0
+#define regUTCL1_CTRL_1                                                                                 0x158c
+#define regUTCL1_CTRL_1_BASE_IDX                                                                        0
+#define regUTCL1_HASH_CTRL                                                                              0x158e
+#define regUTCL1_HASH_CTRL_BASE_IDX                                                                     0
+#define regUTCL1_ALOG                                                                                   0x158f
+#define regUTCL1_ALOG_BASE_IDX                                                                          0
+#define regUTCL1_STATUS                                                                                 0x1594
+#define regUTCL1_STATUS_BASE_IDX                                                                        0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_shdec
+// base address: 0xb000
+#define regSPI_SHADER_PGM_CHKSUM_PS                                                                     0x19a5
+#define regSPI_SHADER_PGM_CHKSUM_PS_BASE_IDX                                                            0
+#define regSPI_SHADER_PGM_RSRC3_PS                                                                      0x19a6
+#define regSPI_SHADER_PGM_RSRC3_PS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_RSRC4_PS                                                                      0x19a7
+#define regSPI_SHADER_PGM_RSRC4_PS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_LO_PS                                                                         0x19a8
+#define regSPI_SHADER_PGM_LO_PS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_HI_PS                                                                         0x19a9
+#define regSPI_SHADER_PGM_HI_PS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_RSRC1_PS                                                                      0x19aa
+#define regSPI_SHADER_PGM_RSRC1_PS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_RSRC2_PS                                                                      0x19ab
+#define regSPI_SHADER_PGM_RSRC2_PS_BASE_IDX                                                             0
+#define regSPI_SHADER_USER_DATA_PS_0                                                                    0x19ac
+#define regSPI_SHADER_USER_DATA_PS_0_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_1                                                                    0x19ad
+#define regSPI_SHADER_USER_DATA_PS_1_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_2                                                                    0x19ae
+#define regSPI_SHADER_USER_DATA_PS_2_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_3                                                                    0x19af
+#define regSPI_SHADER_USER_DATA_PS_3_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_4                                                                    0x19b0
+#define regSPI_SHADER_USER_DATA_PS_4_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_5                                                                    0x19b1
+#define regSPI_SHADER_USER_DATA_PS_5_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_6                                                                    0x19b2
+#define regSPI_SHADER_USER_DATA_PS_6_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_7                                                                    0x19b3
+#define regSPI_SHADER_USER_DATA_PS_7_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_8                                                                    0x19b4
+#define regSPI_SHADER_USER_DATA_PS_8_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_9                                                                    0x19b5
+#define regSPI_SHADER_USER_DATA_PS_9_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_10                                                                   0x19b6
+#define regSPI_SHADER_USER_DATA_PS_10_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_11                                                                   0x19b7
+#define regSPI_SHADER_USER_DATA_PS_11_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_12                                                                   0x19b8
+#define regSPI_SHADER_USER_DATA_PS_12_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_13                                                                   0x19b9
+#define regSPI_SHADER_USER_DATA_PS_13_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_14                                                                   0x19ba
+#define regSPI_SHADER_USER_DATA_PS_14_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_15                                                                   0x19bb
+#define regSPI_SHADER_USER_DATA_PS_15_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_16                                                                   0x19bc
+#define regSPI_SHADER_USER_DATA_PS_16_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_17                                                                   0x19bd
+#define regSPI_SHADER_USER_DATA_PS_17_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_18                                                                   0x19be
+#define regSPI_SHADER_USER_DATA_PS_18_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_19                                                                   0x19bf
+#define regSPI_SHADER_USER_DATA_PS_19_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_20                                                                   0x19c0
+#define regSPI_SHADER_USER_DATA_PS_20_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_21                                                                   0x19c1
+#define regSPI_SHADER_USER_DATA_PS_21_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_22                                                                   0x19c2
+#define regSPI_SHADER_USER_DATA_PS_22_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_23                                                                   0x19c3
+#define regSPI_SHADER_USER_DATA_PS_23_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_24                                                                   0x19c4
+#define regSPI_SHADER_USER_DATA_PS_24_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_25                                                                   0x19c5
+#define regSPI_SHADER_USER_DATA_PS_25_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_26                                                                   0x19c6
+#define regSPI_SHADER_USER_DATA_PS_26_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_27                                                                   0x19c7
+#define regSPI_SHADER_USER_DATA_PS_27_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_28                                                                   0x19c8
+#define regSPI_SHADER_USER_DATA_PS_28_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_29                                                                   0x19c9
+#define regSPI_SHADER_USER_DATA_PS_29_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_30                                                                   0x19ca
+#define regSPI_SHADER_USER_DATA_PS_30_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_31                                                                   0x19cb
+#define regSPI_SHADER_USER_DATA_PS_31_BASE_IDX                                                          0
+#define regSPI_SHADER_REQ_CTRL_PS                                                                       0x19d0
+#define regSPI_SHADER_REQ_CTRL_PS_BASE_IDX                                                              0
+#define regSPI_SHADER_GS_OUT_CONFIG_PS                                                                  0x19d1
+#define regSPI_SHADER_GS_OUT_CONFIG_PS_BASE_IDX                                                         0
+#define regSPI_SHADER_USER_ACCUM_PS_0                                                                   0x19d2
+#define regSPI_SHADER_USER_ACCUM_PS_0_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_ACCUM_PS_1                                                                   0x19d3
+#define regSPI_SHADER_USER_ACCUM_PS_1_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_ACCUM_PS_2                                                                   0x19d4
+#define regSPI_SHADER_USER_ACCUM_PS_2_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_ACCUM_PS_3                                                                   0x19d5
+#define regSPI_SHADER_USER_ACCUM_PS_3_BASE_IDX                                                          0
+#define regSPI_SHADER_PGM_CHKSUM_GS                                                                     0x1a20
+#define regSPI_SHADER_PGM_CHKSUM_GS_BASE_IDX                                                            0
+#define regSPI_SHADER_USER_DATA_ADDR_LO_GS                                                              0x1a22
+#define regSPI_SHADER_USER_DATA_ADDR_LO_GS_BASE_IDX                                                     0
+#define regSPI_SHADER_USER_DATA_ADDR_HI_GS                                                              0x1a23
+#define regSPI_SHADER_USER_DATA_ADDR_HI_GS_BASE_IDX                                                     0
+#define regSPI_SHADER_PGM_LO_GS                                                                         0x1a24
+#define regSPI_SHADER_PGM_LO_GS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_HI_GS                                                                         0x1a25
+#define regSPI_SHADER_PGM_HI_GS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_HI_ES                                                                         0x1a26
+#define regSPI_SHADER_PGM_HI_ES_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_RSRC3_GS                                                                      0x1a27
+#define regSPI_SHADER_PGM_RSRC3_GS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_RSRC4_GS                                                                      0x1a28
+#define regSPI_SHADER_PGM_RSRC4_GS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_LO_ES                                                                         0x1a29
+#define regSPI_SHADER_PGM_LO_ES_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_RSRC1_GS                                                                      0x1a2a
+#define regSPI_SHADER_PGM_RSRC1_GS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_RSRC2_GS                                                                      0x1a2b
+#define regSPI_SHADER_PGM_RSRC2_GS_BASE_IDX                                                             0
+#define regSPI_SHADER_USER_DATA_GS_0                                                                    0x1a2c
+#define regSPI_SHADER_USER_DATA_GS_0_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_1                                                                    0x1a2d
+#define regSPI_SHADER_USER_DATA_GS_1_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_2                                                                    0x1a2e
+#define regSPI_SHADER_USER_DATA_GS_2_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_3                                                                    0x1a2f
+#define regSPI_SHADER_USER_DATA_GS_3_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_4                                                                    0x1a30
+#define regSPI_SHADER_USER_DATA_GS_4_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_5                                                                    0x1a31
+#define regSPI_SHADER_USER_DATA_GS_5_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_6                                                                    0x1a32
+#define regSPI_SHADER_USER_DATA_GS_6_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_7                                                                    0x1a33
+#define regSPI_SHADER_USER_DATA_GS_7_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_8                                                                    0x1a34
+#define regSPI_SHADER_USER_DATA_GS_8_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_9                                                                    0x1a35
+#define regSPI_SHADER_USER_DATA_GS_9_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_10                                                                   0x1a36
+#define regSPI_SHADER_USER_DATA_GS_10_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_11                                                                   0x1a37
+#define regSPI_SHADER_USER_DATA_GS_11_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_12                                                                   0x1a38
+#define regSPI_SHADER_USER_DATA_GS_12_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_13                                                                   0x1a39
+#define regSPI_SHADER_USER_DATA_GS_13_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_14                                                                   0x1a3a
+#define regSPI_SHADER_USER_DATA_GS_14_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_15                                                                   0x1a3b
+#define regSPI_SHADER_USER_DATA_GS_15_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_16                                                                   0x1a3c
+#define regSPI_SHADER_USER_DATA_GS_16_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_17                                                                   0x1a3d
+#define regSPI_SHADER_USER_DATA_GS_17_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_18                                                                   0x1a3e
+#define regSPI_SHADER_USER_DATA_GS_18_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_19                                                                   0x1a3f
+#define regSPI_SHADER_USER_DATA_GS_19_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_20                                                                   0x1a40
+#define regSPI_SHADER_USER_DATA_GS_20_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_21                                                                   0x1a41
+#define regSPI_SHADER_USER_DATA_GS_21_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_22                                                                   0x1a42
+#define regSPI_SHADER_USER_DATA_GS_22_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_23                                                                   0x1a43
+#define regSPI_SHADER_USER_DATA_GS_23_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_24                                                                   0x1a44
+#define regSPI_SHADER_USER_DATA_GS_24_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_25                                                                   0x1a45
+#define regSPI_SHADER_USER_DATA_GS_25_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_26                                                                   0x1a46
+#define regSPI_SHADER_USER_DATA_GS_26_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_27                                                                   0x1a47
+#define regSPI_SHADER_USER_DATA_GS_27_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_28                                                                   0x1a48
+#define regSPI_SHADER_USER_DATA_GS_28_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_29                                                                   0x1a49
+#define regSPI_SHADER_USER_DATA_GS_29_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_30                                                                   0x1a4a
+#define regSPI_SHADER_USER_DATA_GS_30_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_31                                                                   0x1a4b
+#define regSPI_SHADER_USER_DATA_GS_31_BASE_IDX                                                          0
+#define regSPI_SHADER_GS_MESHLET_DIM                                                                    0x1a4c
+#define regSPI_SHADER_GS_MESHLET_DIM_BASE_IDX                                                           0
+#define regSPI_SHADER_GS_MESHLET_EXP_ALLOC                                                              0x1a4d
+#define regSPI_SHADER_GS_MESHLET_EXP_ALLOC_BASE_IDX                                                     0
+#define regSPI_SHADER_GS_MESHLET_CTRL                                                                   0x1a4e
+#define regSPI_SHADER_GS_MESHLET_CTRL_BASE_IDX                                                          0
+#define regSPI_SHADER_REQ_CTRL_ESGS                                                                     0x1a50
+#define regSPI_SHADER_REQ_CTRL_ESGS_BASE_IDX                                                            0
+#define regSPI_SHADER_GS_OUT_CONFIG_PS_GS                                                               0x1a51
+#define regSPI_SHADER_GS_OUT_CONFIG_PS_GS_BASE_IDX                                                      0
+#define regSPI_SHADER_USER_ACCUM_ESGS_0                                                                 0x1a52
+#define regSPI_SHADER_USER_ACCUM_ESGS_0_BASE_IDX                                                        0
+#define regSPI_SHADER_USER_ACCUM_ESGS_1                                                                 0x1a53
+#define regSPI_SHADER_USER_ACCUM_ESGS_1_BASE_IDX                                                        0
+#define regSPI_SHADER_USER_ACCUM_ESGS_2                                                                 0x1a54
+#define regSPI_SHADER_USER_ACCUM_ESGS_2_BASE_IDX                                                        0
+#define regSPI_SHADER_USER_ACCUM_ESGS_3                                                                 0x1a55
+#define regSPI_SHADER_USER_ACCUM_ESGS_3_BASE_IDX                                                        0
+#define regSPI_SHADER_PGM_CHKSUM_HS                                                                     0x1aa0
+#define regSPI_SHADER_PGM_CHKSUM_HS_BASE_IDX                                                            0
+#define regSPI_SHADER_USER_DATA_ADDR_LO_HS                                                              0x1aa2
+#define regSPI_SHADER_USER_DATA_ADDR_LO_HS_BASE_IDX                                                     0
+#define regSPI_SHADER_USER_DATA_ADDR_HI_HS                                                              0x1aa3
+#define regSPI_SHADER_USER_DATA_ADDR_HI_HS_BASE_IDX                                                     0
+#define regSPI_SHADER_PGM_LO_HS                                                                         0x1aa4
+#define regSPI_SHADER_PGM_LO_HS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_HI_HS                                                                         0x1aa5
+#define regSPI_SHADER_PGM_HI_HS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_HI_LS                                                                         0x1aa6
+#define regSPI_SHADER_PGM_HI_LS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_RSRC3_HS                                                                      0x1aa7
+#define regSPI_SHADER_PGM_RSRC3_HS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_RSRC4_HS                                                                      0x1aa8
+#define regSPI_SHADER_PGM_RSRC4_HS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_LO_LS                                                                         0x1aa9
+#define regSPI_SHADER_PGM_LO_LS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_RSRC1_HS                                                                      0x1aaa
+#define regSPI_SHADER_PGM_RSRC1_HS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_RSRC2_HS                                                                      0x1aab
+#define regSPI_SHADER_PGM_RSRC2_HS_BASE_IDX                                                             0
+#define regSPI_SHADER_USER_DATA_HS_0                                                                    0x1aac
+#define regSPI_SHADER_USER_DATA_HS_0_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_1                                                                    0x1aad
+#define regSPI_SHADER_USER_DATA_HS_1_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_2                                                                    0x1aae
+#define regSPI_SHADER_USER_DATA_HS_2_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_3                                                                    0x1aaf
+#define regSPI_SHADER_USER_DATA_HS_3_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_4                                                                    0x1ab0
+#define regSPI_SHADER_USER_DATA_HS_4_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_5                                                                    0x1ab1
+#define regSPI_SHADER_USER_DATA_HS_5_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_6                                                                    0x1ab2
+#define regSPI_SHADER_USER_DATA_HS_6_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_7                                                                    0x1ab3
+#define regSPI_SHADER_USER_DATA_HS_7_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_8                                                                    0x1ab4
+#define regSPI_SHADER_USER_DATA_HS_8_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_9                                                                    0x1ab5
+#define regSPI_SHADER_USER_DATA_HS_9_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_10                                                                   0x1ab6
+#define regSPI_SHADER_USER_DATA_HS_10_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_11                                                                   0x1ab7
+#define regSPI_SHADER_USER_DATA_HS_11_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_12                                                                   0x1ab8
+#define regSPI_SHADER_USER_DATA_HS_12_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_13                                                                   0x1ab9
+#define regSPI_SHADER_USER_DATA_HS_13_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_14                                                                   0x1aba
+#define regSPI_SHADER_USER_DATA_HS_14_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_15                                                                   0x1abb
+#define regSPI_SHADER_USER_DATA_HS_15_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_16                                                                   0x1abc
+#define regSPI_SHADER_USER_DATA_HS_16_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_17                                                                   0x1abd
+#define regSPI_SHADER_USER_DATA_HS_17_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_18                                                                   0x1abe
+#define regSPI_SHADER_USER_DATA_HS_18_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_19                                                                   0x1abf
+#define regSPI_SHADER_USER_DATA_HS_19_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_20                                                                   0x1ac0
+#define regSPI_SHADER_USER_DATA_HS_20_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_21                                                                   0x1ac1
+#define regSPI_SHADER_USER_DATA_HS_21_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_22                                                                   0x1ac2
+#define regSPI_SHADER_USER_DATA_HS_22_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_23                                                                   0x1ac3
+#define regSPI_SHADER_USER_DATA_HS_23_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_24                                                                   0x1ac4
+#define regSPI_SHADER_USER_DATA_HS_24_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_25                                                                   0x1ac5
+#define regSPI_SHADER_USER_DATA_HS_25_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_26                                                                   0x1ac6
+#define regSPI_SHADER_USER_DATA_HS_26_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_27                                                                   0x1ac7
+#define regSPI_SHADER_USER_DATA_HS_27_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_28                                                                   0x1ac8
+#define regSPI_SHADER_USER_DATA_HS_28_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_29                                                                   0x1ac9
+#define regSPI_SHADER_USER_DATA_HS_29_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_30                                                                   0x1aca
+#define regSPI_SHADER_USER_DATA_HS_30_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_31                                                                   0x1acb
+#define regSPI_SHADER_USER_DATA_HS_31_BASE_IDX                                                          0
+#define regSPI_SHADER_REQ_CTRL_LSHS                                                                     0x1ad0
+#define regSPI_SHADER_REQ_CTRL_LSHS_BASE_IDX                                                            0
+#define regSPI_SHADER_USER_ACCUM_LSHS_0                                                                 0x1ad2
+#define regSPI_SHADER_USER_ACCUM_LSHS_0_BASE_IDX                                                        0
+#define regSPI_SHADER_USER_ACCUM_LSHS_1                                                                 0x1ad3
+#define regSPI_SHADER_USER_ACCUM_LSHS_1_BASE_IDX                                                        0
+#define regSPI_SHADER_USER_ACCUM_LSHS_2                                                                 0x1ad4
+#define regSPI_SHADER_USER_ACCUM_LSHS_2_BASE_IDX                                                        0
+#define regSPI_SHADER_USER_ACCUM_LSHS_3                                                                 0x1ad5
+#define regSPI_SHADER_USER_ACCUM_LSHS_3_BASE_IDX                                                        0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_spipdec
+// base address: 0xc700
+#define regSPI_ARB_PRIORITY                                                                             0x1f60
+#define regSPI_ARB_PRIORITY_BASE_IDX                                                                    0
+#define regSPI_ARB_CYCLES_0                                                                             0x1f61
+#define regSPI_ARB_CYCLES_0_BASE_IDX                                                                    0
+#define regSPI_ARB_CYCLES_1                                                                             0x1f62
+#define regSPI_ARB_CYCLES_1_BASE_IDX                                                                    0
+#define regSPI_WCL_PIPE_PERCENT_GFX                                                                     0x1f67
+#define regSPI_WCL_PIPE_PERCENT_GFX_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_HP3D                                                                    0x1f68
+#define regSPI_WCL_PIPE_PERCENT_HP3D_BASE_IDX                                                           0
+#define regSPI_WCL_PIPE_PERCENT_CS0                                                                     0x1f69
+#define regSPI_WCL_PIPE_PERCENT_CS0_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS1                                                                     0x1f6a
+#define regSPI_WCL_PIPE_PERCENT_CS1_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS2                                                                     0x1f6b
+#define regSPI_WCL_PIPE_PERCENT_CS2_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS3                                                                     0x1f6c
+#define regSPI_WCL_PIPE_PERCENT_CS3_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS4                                                                     0x1f6d
+#define regSPI_WCL_PIPE_PERCENT_CS4_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS5                                                                     0x1f6e
+#define regSPI_WCL_PIPE_PERCENT_CS5_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS6                                                                     0x1f6f
+#define regSPI_WCL_PIPE_PERCENT_CS6_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS7                                                                     0x1f70
+#define regSPI_WCL_PIPE_PERCENT_CS7_BASE_IDX                                                            0
+#define regSPI_USER_ACCUM_VMID_CNTL                                                                     0x1f71
+#define regSPI_USER_ACCUM_VMID_CNTL_BASE_IDX                                                            0
+#define regSPI_GDBG_PER_VMID_CNTL                                                                       0x1f72
+#define regSPI_GDBG_PER_VMID_CNTL_BASE_IDX                                                              0
+#define regSPI_COMPUTE_QUEUE_RESET                                                                      0x1f73
+#define regSPI_COMPUTE_QUEUE_RESET_BASE_IDX                                                             0
+#define regSPI_COMPUTE_WF_CTX_SAVE                                                                      0x1f74
+#define regSPI_COMPUTE_WF_CTX_SAVE_BASE_IDX                                                             0
+#define regSPI_SAVE_RESTORE_STATUS                                                                      0x1f75
+#define regSPI_SAVE_RESTORE_STATUS_BASE_IDX                                                             0
+#define regSPI_CONFIG_DIDT_CNTL                                                                         0x1f76
+#define regSPI_CONFIG_DIDT_CNTL_BASE_IDX                                                                0
+#define regSPI_CONFIG_DIDT_TYPEMASK                                                                     0x1f77
+#define regSPI_CONFIG_DIDT_TYPEMASK_BASE_IDX                                                            0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_tcpdec
+// base address: 0xca80
+#define regTCP_WATCH0_ADDR_H                                                                            0x2048
+#define regTCP_WATCH0_ADDR_H_BASE_IDX                                                                   0
+#define regTCP_WATCH0_ADDR_L                                                                            0x2049
+#define regTCP_WATCH0_ADDR_L_BASE_IDX                                                                   0
+#define regTCP_WATCH0_CNTL                                                                              0x204a
+#define regTCP_WATCH0_CNTL_BASE_IDX                                                                     0
+#define regTCP_WATCH1_ADDR_H                                                                            0x204b
+#define regTCP_WATCH1_ADDR_H_BASE_IDX                                                                   0
+#define regTCP_WATCH1_ADDR_L                                                                            0x204c
+#define regTCP_WATCH1_ADDR_L_BASE_IDX                                                                   0
+#define regTCP_WATCH1_CNTL                                                                              0x204d
+#define regTCP_WATCH1_CNTL_BASE_IDX                                                                     0
+#define regTCP_WATCH2_ADDR_H                                                                            0x204e
+#define regTCP_WATCH2_ADDR_H_BASE_IDX                                                                   0
+#define regTCP_WATCH2_ADDR_L                                                                            0x204f
+#define regTCP_WATCH2_ADDR_L_BASE_IDX                                                                   0
+#define regTCP_WATCH2_CNTL                                                                              0x2050
+#define regTCP_WATCH2_CNTL_BASE_IDX                                                                     0
+#define regTCP_WATCH3_ADDR_H                                                                            0x2051
+#define regTCP_WATCH3_ADDR_H_BASE_IDX                                                                   0
+#define regTCP_WATCH3_ADDR_L                                                                            0x2052
+#define regTCP_WATCH3_ADDR_L_BASE_IDX                                                                   0
+#define regTCP_WATCH3_CNTL                                                                              0x2053
+#define regTCP_WATCH3_CNTL_BASE_IDX                                                                     0
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_gfxdec0
+// base address: 0x28000
+#define regDB_RENDER_CONTROL                                                                            0x0000
+#define regDB_RENDER_CONTROL_BASE_IDX                                                                   1
+#define regDB_DEPTH_VIEW                                                                                0x0001
+#define regDB_DEPTH_VIEW_BASE_IDX                                                                       1
+#define regDB_DEPTH_VIEW1                                                                               0x0002
+#define regDB_DEPTH_VIEW1_BASE_IDX                                                                      1
+#define regDB_RENDER_OVERRIDE                                                                           0x0003
+#define regDB_RENDER_OVERRIDE_BASE_IDX                                                                  1
+#define regDB_RENDER_OVERRIDE2                                                                          0x0004
+#define regDB_RENDER_OVERRIDE2_BASE_IDX                                                                 1
+#define regDB_DEPTH_SIZE_XY                                                                             0x0005
+#define regDB_DEPTH_SIZE_XY_BASE_IDX                                                                    1
+#define regDB_Z_INFO                                                                                    0x0006
+#define regDB_Z_INFO_BASE_IDX                                                                           1
+#define regDB_STENCIL_INFO                                                                              0x0007
+#define regDB_STENCIL_INFO_BASE_IDX                                                                     1
+#define regDB_Z_READ_BASE                                                                               0x0008
+#define regDB_Z_READ_BASE_BASE_IDX                                                                      1
+#define regDB_Z_READ_BASE_HI                                                                            0x0009
+#define regDB_Z_READ_BASE_HI_BASE_IDX                                                                   1
+#define regDB_Z_WRITE_BASE                                                                              0x000a
+#define regDB_Z_WRITE_BASE_BASE_IDX                                                                     1
+#define regDB_Z_WRITE_BASE_HI                                                                           0x000b
+#define regDB_Z_WRITE_BASE_HI_BASE_IDX                                                                  1
+#define regDB_STENCIL_READ_BASE                                                                         0x000c
+#define regDB_STENCIL_READ_BASE_BASE_IDX                                                                1
+#define regDB_STENCIL_READ_BASE_HI                                                                      0x000d
+#define regDB_STENCIL_READ_BASE_HI_BASE_IDX                                                             1
+#define regDB_STENCIL_WRITE_BASE                                                                        0x000e
+#define regDB_STENCIL_WRITE_BASE_BASE_IDX                                                               1
+#define regDB_STENCIL_WRITE_BASE_HI                                                                     0x000f
+#define regDB_STENCIL_WRITE_BASE_HI_BASE_IDX                                                            1
+#define regDB_GL1_INTERFACE_CONTROL                                                                     0x0010
+#define regDB_GL1_INTERFACE_CONTROL_BASE_IDX                                                            1
+#define regDB_MEM_TEMPORAL                                                                              0x0012
+#define regDB_MEM_TEMPORAL_BASE_IDX                                                                     1
+#define regDB_DEPTH_BOUNDS_MIN                                                                          0x0014
+#define regDB_DEPTH_BOUNDS_MIN_BASE_IDX                                                                 1
+#define regDB_DEPTH_BOUNDS_MAX                                                                          0x0015
+#define regDB_DEPTH_BOUNDS_MAX_BASE_IDX                                                                 1
+#define regDB_COUNT_CONTROL                                                                             0x0018
+#define regDB_COUNT_CONTROL_BASE_IDX                                                                    1
+#define regDB_VIEWPORT_CONTROL                                                                          0x0019
+#define regDB_VIEWPORT_CONTROL_BASE_IDX                                                                 1
+#define regDB_SPI_VRS_CENTER_LOCATION                                                                   0x001a
+#define regDB_SPI_VRS_CENTER_LOCATION_BASE_IDX                                                          1
+#define regDB_SHADER_CONTROL                                                                            0x001b
+#define regDB_SHADER_CONTROL_BASE_IDX                                                                   1
+#define regDB_DEPTH_CONTROL                                                                             0x001c
+#define regDB_DEPTH_CONTROL_BASE_IDX                                                                    1
+#define regDB_STENCIL_CONTROL                                                                           0x001d
+#define regDB_STENCIL_CONTROL_BASE_IDX                                                                  1
+#define regDB_EQAA                                                                                      0x001e
+#define regDB_EQAA_BASE_IDX                                                                             1
+#define regDB_ALPHA_TO_MASK                                                                             0x001f
+#define regDB_ALPHA_TO_MASK_BASE_IDX                                                                    1
+#define regTA_BC_BASE_ADDR                                                                              0x0020
+#define regTA_BC_BASE_ADDR_BASE_IDX                                                                     1
+#define regTA_BC_BASE_ADDR_HI                                                                           0x0021
+#define regTA_BC_BASE_ADDR_HI_BASE_IDX                                                                  1
+#define regDB_STENCIL_REF                                                                               0x0022
+#define regDB_STENCIL_REF_BASE_IDX                                                                      1
+#define regDB_STENCIL_OPVAL                                                                             0x0023
+#define regDB_STENCIL_OPVAL_BASE_IDX                                                                    1
+#define regDB_STENCIL_READ_MASK                                                                         0x0024
+#define regDB_STENCIL_READ_MASK_BASE_IDX                                                                1
+#define regDB_STENCIL_WRITE_MASK                                                                        0x0025
+#define regDB_STENCIL_WRITE_MASK_BASE_IDX                                                               1
+#define regSC_MEM_TEMPORAL                                                                              0x003e
+#define regSC_MEM_TEMPORAL_BASE_IDX                                                                     1
+#define regSC_MEM_SPEC_READ                                                                             0x003f
+#define regSC_MEM_SPEC_READ_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_0_TL                                                                             0x0040
+#define regPA_SC_VPORT_0_TL_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_0_BR                                                                             0x0041
+#define regPA_SC_VPORT_0_BR_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_1_TL                                                                             0x0042
+#define regPA_SC_VPORT_1_TL_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_1_BR                                                                             0x0043
+#define regPA_SC_VPORT_1_BR_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_2_TL                                                                             0x0044
+#define regPA_SC_VPORT_2_TL_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_2_BR                                                                             0x0045
+#define regPA_SC_VPORT_2_BR_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_3_TL                                                                             0x0046
+#define regPA_SC_VPORT_3_TL_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_3_BR                                                                             0x0047
+#define regPA_SC_VPORT_3_BR_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_4_TL                                                                             0x0048
+#define regPA_SC_VPORT_4_TL_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_4_BR                                                                             0x0049
+#define regPA_SC_VPORT_4_BR_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_5_TL                                                                             0x004a
+#define regPA_SC_VPORT_5_TL_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_5_BR                                                                             0x004b
+#define regPA_SC_VPORT_5_BR_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_6_TL                                                                             0x004c
+#define regPA_SC_VPORT_6_TL_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_6_BR                                                                             0x004d
+#define regPA_SC_VPORT_6_BR_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_7_TL                                                                             0x004e
+#define regPA_SC_VPORT_7_TL_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_7_BR                                                                             0x004f
+#define regPA_SC_VPORT_7_BR_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_8_TL                                                                             0x0050
+#define regPA_SC_VPORT_8_TL_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_8_BR                                                                             0x0051
+#define regPA_SC_VPORT_8_BR_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_9_TL                                                                             0x0052
+#define regPA_SC_VPORT_9_TL_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_9_BR                                                                             0x0053
+#define regPA_SC_VPORT_9_BR_BASE_IDX                                                                    1
+#define regPA_SC_VPORT_10_TL                                                                            0x0054
+#define regPA_SC_VPORT_10_TL_BASE_IDX                                                                   1
+#define regPA_SC_VPORT_10_BR                                                                            0x0055
+#define regPA_SC_VPORT_10_BR_BASE_IDX                                                                   1
+#define regPA_SC_VPORT_11_TL                                                                            0x0056
+#define regPA_SC_VPORT_11_TL_BASE_IDX                                                                   1
+#define regPA_SC_VPORT_11_BR                                                                            0x0057
+#define regPA_SC_VPORT_11_BR_BASE_IDX                                                                   1
+#define regPA_SC_VPORT_12_TL                                                                            0x0058
+#define regPA_SC_VPORT_12_TL_BASE_IDX                                                                   1
+#define regPA_SC_VPORT_12_BR                                                                            0x0059
+#define regPA_SC_VPORT_12_BR_BASE_IDX                                                                   1
+#define regPA_SC_VPORT_13_TL                                                                            0x005a
+#define regPA_SC_VPORT_13_TL_BASE_IDX                                                                   1
+#define regPA_SC_VPORT_13_BR                                                                            0x005b
+#define regPA_SC_VPORT_13_BR_BASE_IDX                                                                   1
+#define regPA_SC_VPORT_14_TL                                                                            0x005c
+#define regPA_SC_VPORT_14_TL_BASE_IDX                                                                   1
+#define regPA_SC_VPORT_14_BR                                                                            0x005d
+#define regPA_SC_VPORT_14_BR_BASE_IDX                                                                   1
+#define regPA_SC_VPORT_15_TL                                                                            0x005e
+#define regPA_SC_VPORT_15_TL_BASE_IDX                                                                   1
+#define regPA_SC_VPORT_15_BR                                                                            0x005f
+#define regPA_SC_VPORT_15_BR_BASE_IDX                                                                   1
+#define regPA_SC_SCREEN_SCISSOR_TL                                                                      0x0060
+#define regPA_SC_SCREEN_SCISSOR_TL_BASE_IDX                                                             1
+#define regPA_SC_SCREEN_SCISSOR_BR                                                                      0x0061
+#define regPA_SC_SCREEN_SCISSOR_BR_BASE_IDX                                                             1
+#define regPA_SC_WINDOW_OFFSET                                                                          0x0080
+#define regPA_SC_WINDOW_OFFSET_BASE_IDX                                                                 1
+#define regPA_SC_WINDOW_SCISSOR_TL                                                                      0x0081
+#define regPA_SC_WINDOW_SCISSOR_TL_BASE_IDX                                                             1
+#define regPA_SC_WINDOW_SCISSOR_BR                                                                      0x0082
+#define regPA_SC_WINDOW_SCISSOR_BR_BASE_IDX                                                             1
+#define regPA_SC_CLIPRECT_RULE                                                                          0x0083
+#define regPA_SC_CLIPRECT_RULE_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_0_TL                                                                          0x0084
+#define regPA_SC_CLIPRECT_0_TL_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_0_BR                                                                          0x0085
+#define regPA_SC_CLIPRECT_0_BR_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_1_TL                                                                          0x0086
+#define regPA_SC_CLIPRECT_1_TL_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_1_BR                                                                          0x0087
+#define regPA_SC_CLIPRECT_1_BR_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_2_TL                                                                          0x0088
+#define regPA_SC_CLIPRECT_2_TL_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_2_BR                                                                          0x0089
+#define regPA_SC_CLIPRECT_2_BR_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_3_TL                                                                          0x008a
+#define regPA_SC_CLIPRECT_3_TL_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_3_BR                                                                          0x008b
+#define regPA_SC_CLIPRECT_3_BR_BASE_IDX                                                                 1
+#define regPA_SC_EDGERULE                                                                               0x008c
+#define regPA_SC_EDGERULE_BASE_IDX                                                                      1
+#define regPA_SU_HARDWARE_SCREEN_OFFSET                                                                 0x008d
+#define regPA_SU_HARDWARE_SCREEN_OFFSET_BASE_IDX                                                        1
+#define regPA_SC_GENERIC_SCISSOR_TL                                                                     0x0090
+#define regPA_SC_GENERIC_SCISSOR_TL_BASE_IDX                                                            1
+#define regPA_SC_GENERIC_SCISSOR_BR                                                                     0x0091
+#define regPA_SC_GENERIC_SCISSOR_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_0_TL                                                                     0x0094
+#define regPA_SC_VPORT_SCISSOR_0_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_0_BR                                                                     0x0095
+#define regPA_SC_VPORT_SCISSOR_0_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_1_TL                                                                     0x0096
+#define regPA_SC_VPORT_SCISSOR_1_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_1_BR                                                                     0x0097
+#define regPA_SC_VPORT_SCISSOR_1_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_2_TL                                                                     0x0098
+#define regPA_SC_VPORT_SCISSOR_2_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_2_BR                                                                     0x0099
+#define regPA_SC_VPORT_SCISSOR_2_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_3_TL                                                                     0x009a
+#define regPA_SC_VPORT_SCISSOR_3_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_3_BR                                                                     0x009b
+#define regPA_SC_VPORT_SCISSOR_3_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_4_TL                                                                     0x009c
+#define regPA_SC_VPORT_SCISSOR_4_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_4_BR                                                                     0x009d
+#define regPA_SC_VPORT_SCISSOR_4_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_5_TL                                                                     0x009e
+#define regPA_SC_VPORT_SCISSOR_5_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_5_BR                                                                     0x009f
+#define regPA_SC_VPORT_SCISSOR_5_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_6_TL                                                                     0x00a0
+#define regPA_SC_VPORT_SCISSOR_6_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_6_BR                                                                     0x00a1
+#define regPA_SC_VPORT_SCISSOR_6_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_7_TL                                                                     0x00a2
+#define regPA_SC_VPORT_SCISSOR_7_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_7_BR                                                                     0x00a3
+#define regPA_SC_VPORT_SCISSOR_7_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_8_TL                                                                     0x00a4
+#define regPA_SC_VPORT_SCISSOR_8_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_8_BR                                                                     0x00a5
+#define regPA_SC_VPORT_SCISSOR_8_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_9_TL                                                                     0x00a6
+#define regPA_SC_VPORT_SCISSOR_9_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_9_BR                                                                     0x00a7
+#define regPA_SC_VPORT_SCISSOR_9_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_10_TL                                                                    0x00a8
+#define regPA_SC_VPORT_SCISSOR_10_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_10_BR                                                                    0x00a9
+#define regPA_SC_VPORT_SCISSOR_10_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_11_TL                                                                    0x00aa
+#define regPA_SC_VPORT_SCISSOR_11_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_11_BR                                                                    0x00ab
+#define regPA_SC_VPORT_SCISSOR_11_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_12_TL                                                                    0x00ac
+#define regPA_SC_VPORT_SCISSOR_12_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_12_BR                                                                    0x00ad
+#define regPA_SC_VPORT_SCISSOR_12_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_13_TL                                                                    0x00ae
+#define regPA_SC_VPORT_SCISSOR_13_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_13_BR                                                                    0x00af
+#define regPA_SC_VPORT_SCISSOR_13_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_14_TL                                                                    0x00b0
+#define regPA_SC_VPORT_SCISSOR_14_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_14_BR                                                                    0x00b1
+#define regPA_SC_VPORT_SCISSOR_14_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_15_TL                                                                    0x00b2
+#define regPA_SC_VPORT_SCISSOR_15_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_15_BR                                                                    0x00b3
+#define regPA_SC_VPORT_SCISSOR_15_BR_BASE_IDX                                                           1
+#define regPA_CL_UCP_0_X                                                                                0x00b4
+#define regPA_CL_UCP_0_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_0_Y                                                                                0x00b5
+#define regPA_CL_UCP_0_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_0_Z                                                                                0x00b6
+#define regPA_CL_UCP_0_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_0_W                                                                                0x00b7
+#define regPA_CL_UCP_0_W_BASE_IDX                                                                       1
+#define regPA_CL_UCP_1_X                                                                                0x00b8
+#define regPA_CL_UCP_1_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_1_Y                                                                                0x00b9
+#define regPA_CL_UCP_1_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_1_Z                                                                                0x00ba
+#define regPA_CL_UCP_1_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_1_W                                                                                0x00bb
+#define regPA_CL_UCP_1_W_BASE_IDX                                                                       1
+#define regPA_CL_UCP_2_X                                                                                0x00bc
+#define regPA_CL_UCP_2_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_2_Y                                                                                0x00bd
+#define regPA_CL_UCP_2_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_2_Z                                                                                0x00be
+#define regPA_CL_UCP_2_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_2_W                                                                                0x00bf
+#define regPA_CL_UCP_2_W_BASE_IDX                                                                       1
+#define regPA_CL_UCP_3_X                                                                                0x00c0
+#define regPA_CL_UCP_3_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_3_Y                                                                                0x00c1
+#define regPA_CL_UCP_3_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_3_Z                                                                                0x00c2
+#define regPA_CL_UCP_3_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_3_W                                                                                0x00c3
+#define regPA_CL_UCP_3_W_BASE_IDX                                                                       1
+#define regPA_CL_UCP_4_X                                                                                0x00c4
+#define regPA_CL_UCP_4_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_4_Y                                                                                0x00c5
+#define regPA_CL_UCP_4_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_4_Z                                                                                0x00c6
+#define regPA_CL_UCP_4_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_4_W                                                                                0x00c7
+#define regPA_CL_UCP_4_W_BASE_IDX                                                                       1
+#define regPA_CL_UCP_5_X                                                                                0x00c8
+#define regPA_CL_UCP_5_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_5_Y                                                                                0x00c9
+#define regPA_CL_UCP_5_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_5_Z                                                                                0x00ca
+#define regPA_CL_UCP_5_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_5_W                                                                                0x00cb
+#define regPA_CL_UCP_5_W_BASE_IDX                                                                       1
+#define regPA_CL_PROG_NEAR_CLIP_Z                                                                       0x00cc
+#define regPA_CL_PROG_NEAR_CLIP_Z_BASE_IDX                                                              1
+#define regPA_RATE_CNTL                                                                                 0x00cd
+#define regPA_RATE_CNTL_BASE_IDX                                                                        1
+#define regPA_SC_RASTER_CONFIG                                                                          0x00d4
+#define regPA_SC_RASTER_CONFIG_BASE_IDX                                                                 1
+#define regPA_SC_RASTER_CONFIG_1                                                                        0x00d5
+#define regPA_SC_RASTER_CONFIG_1_BASE_IDX                                                               1
+#define regPA_SC_SCREEN_EXTENT_CONTROL                                                                  0x00d6
+#define regPA_SC_SCREEN_EXTENT_CONTROL_BASE_IDX                                                         1
+#define regPA_SC_TILE_STEERING_OVERRIDE                                                                 0x00d7
+#define regPA_SC_TILE_STEERING_OVERRIDE_BASE_IDX                                                        1
+#define regPA_SC_CLIPRECT_0_EXT                                                                         0x00dd
+#define regPA_SC_CLIPRECT_0_EXT_BASE_IDX                                                                1
+#define regPA_SC_CLIPRECT_1_EXT                                                                         0x00de
+#define regPA_SC_CLIPRECT_1_EXT_BASE_IDX                                                                1
+#define regPA_SC_CLIPRECT_2_EXT                                                                         0x00df
+#define regPA_SC_CLIPRECT_2_EXT_BASE_IDX                                                                1
+#define regPA_SC_CLIPRECT_3_EXT                                                                         0x00e0
+#define regPA_SC_CLIPRECT_3_EXT_BASE_IDX                                                                1
+#define regPA_SC_VRS_OVERRIDE_CNTL                                                                      0x00f4
+#define regPA_SC_VRS_OVERRIDE_CNTL_BASE_IDX                                                             1
+#define regPA_SC_VRS_RATE_FEEDBACK_BASE                                                                 0x00f5
+#define regPA_SC_VRS_RATE_FEEDBACK_BASE_BASE_IDX                                                        1
+#define regPA_SC_VRS_RATE_FEEDBACK_BASE_EXT                                                             0x00f6
+#define regPA_SC_VRS_RATE_FEEDBACK_BASE_EXT_BASE_IDX                                                    1
+#define regPA_SC_VRS_RATE_FEEDBACK_SIZE_XY                                                              0x00f7
+#define regPA_SC_VRS_RATE_FEEDBACK_SIZE_XY_BASE_IDX                                                     1
+#define regPA_SC_VRS_INFO                                                                               0x00f8
+#define regPA_SC_VRS_INFO_BASE_IDX                                                                      1
+#define regPA_SC_VRS_RATE_BASE                                                                          0x00fc
+#define regPA_SC_VRS_RATE_BASE_BASE_IDX                                                                 1
+#define regPA_SC_VRS_RATE_BASE_EXT                                                                      0x00fd
+#define regPA_SC_VRS_RATE_BASE_EXT_BASE_IDX                                                             1
+#define regPA_SC_VRS_RATE_SIZE_XY                                                                       0x00fe
+#define regPA_SC_VRS_RATE_SIZE_XY_BASE_IDX                                                              1
+#define regCB_RMI_GL2_CACHE_CONTROL                                                                     0x0104
+#define regCB_RMI_GL2_CACHE_CONTROL_BASE_IDX                                                            1
+#define regCB_BLEND_RED                                                                                 0x0105
+#define regCB_BLEND_RED_BASE_IDX                                                                        1
+#define regCB_BLEND_GREEN                                                                               0x0106
+#define regCB_BLEND_GREEN_BASE_IDX                                                                      1
+#define regCB_BLEND_BLUE                                                                                0x0107
+#define regCB_BLEND_BLUE_BASE_IDX                                                                       1
+#define regCB_BLEND_ALPHA                                                                               0x0108
+#define regCB_BLEND_ALPHA_BASE_IDX                                                                      1
+#define regPA_CL_GB_VERT_CLIP_ADJ                                                                       0x010b
+#define regPA_CL_GB_VERT_CLIP_ADJ_BASE_IDX                                                              1
+#define regPA_CL_GB_VERT_DISC_ADJ                                                                       0x010c
+#define regPA_CL_GB_VERT_DISC_ADJ_BASE_IDX                                                              1
+#define regPA_CL_GB_HORZ_CLIP_ADJ                                                                       0x010d
+#define regPA_CL_GB_HORZ_CLIP_ADJ_BASE_IDX                                                              1
+#define regPA_CL_GB_HORZ_DISC_ADJ                                                                       0x010e
+#define regPA_CL_GB_HORZ_DISC_ADJ_BASE_IDX                                                              1
+#define regPA_CL_VPORT_XSCALE                                                                           0x010f
+#define regPA_CL_VPORT_XSCALE_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_XOFFSET                                                                          0x0110
+#define regPA_CL_VPORT_XOFFSET_BASE_IDX                                                                 1
+#define regPA_CL_VPORT_YSCALE                                                                           0x0111
+#define regPA_CL_VPORT_YSCALE_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_YOFFSET                                                                          0x0112
+#define regPA_CL_VPORT_YOFFSET_BASE_IDX                                                                 1
+#define regPA_CL_VPORT_ZSCALE                                                                           0x0113
+#define regPA_CL_VPORT_ZSCALE_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_ZOFFSET                                                                          0x0114
+#define regPA_CL_VPORT_ZOFFSET_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMIN_0                                                                           0x0115
+#define regPA_SC_VPORT_ZMIN_0_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_0                                                                           0x0116
+#define regPA_SC_VPORT_ZMAX_0_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_XSCALE_1                                                                         0x0117
+#define regPA_CL_VPORT_XSCALE_1_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_1                                                                        0x0118
+#define regPA_CL_VPORT_XOFFSET_1_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_1                                                                         0x0119
+#define regPA_CL_VPORT_YSCALE_1_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_1                                                                        0x011a
+#define regPA_CL_VPORT_YOFFSET_1_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_1                                                                         0x011b
+#define regPA_CL_VPORT_ZSCALE_1_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_1                                                                        0x011c
+#define regPA_CL_VPORT_ZOFFSET_1_BASE_IDX                                                               1
+#define regPA_SC_VPORT_ZMIN_1                                                                           0x011d
+#define regPA_SC_VPORT_ZMIN_1_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_1                                                                           0x011e
+#define regPA_SC_VPORT_ZMAX_1_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_XSCALE_2                                                                         0x011f
+#define regPA_CL_VPORT_XSCALE_2_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_2                                                                        0x0120
+#define regPA_CL_VPORT_XOFFSET_2_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_2                                                                         0x0121
+#define regPA_CL_VPORT_YSCALE_2_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_2                                                                        0x0122
+#define regPA_CL_VPORT_YOFFSET_2_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_2                                                                         0x0123
+#define regPA_CL_VPORT_ZSCALE_2_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_2                                                                        0x0124
+#define regPA_CL_VPORT_ZOFFSET_2_BASE_IDX                                                               1
+#define regPA_SC_VPORT_ZMIN_2                                                                           0x0125
+#define regPA_SC_VPORT_ZMIN_2_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_2                                                                           0x0126
+#define regPA_SC_VPORT_ZMAX_2_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_XSCALE_3                                                                         0x0127
+#define regPA_CL_VPORT_XSCALE_3_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_3                                                                        0x0128
+#define regPA_CL_VPORT_XOFFSET_3_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_3                                                                         0x0129
+#define regPA_CL_VPORT_YSCALE_3_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_3                                                                        0x012a
+#define regPA_CL_VPORT_YOFFSET_3_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_3                                                                         0x012b
+#define regPA_CL_VPORT_ZSCALE_3_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_3                                                                        0x012c
+#define regPA_CL_VPORT_ZOFFSET_3_BASE_IDX                                                               1
+#define regPA_SC_VPORT_ZMIN_3                                                                           0x012d
+#define regPA_SC_VPORT_ZMIN_3_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_3                                                                           0x012e
+#define regPA_SC_VPORT_ZMAX_3_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_XSCALE_4                                                                         0x012f
+#define regPA_CL_VPORT_XSCALE_4_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_4                                                                        0x0130
+#define regPA_CL_VPORT_XOFFSET_4_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_4                                                                         0x0131
+#define regPA_CL_VPORT_YSCALE_4_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_4                                                                        0x0132
+#define regPA_CL_VPORT_YOFFSET_4_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_4                                                                         0x0133
+#define regPA_CL_VPORT_ZSCALE_4_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_4                                                                        0x0134
+#define regPA_CL_VPORT_ZOFFSET_4_BASE_IDX                                                               1
+#define regPA_SC_VPORT_ZMIN_4                                                                           0x0135
+#define regPA_SC_VPORT_ZMIN_4_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_4                                                                           0x0136
+#define regPA_SC_VPORT_ZMAX_4_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_XSCALE_5                                                                         0x0137
+#define regPA_CL_VPORT_XSCALE_5_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_5                                                                        0x0138
+#define regPA_CL_VPORT_XOFFSET_5_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_5                                                                         0x0139
+#define regPA_CL_VPORT_YSCALE_5_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_5                                                                        0x013a
+#define regPA_CL_VPORT_YOFFSET_5_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_5                                                                         0x013b
+#define regPA_CL_VPORT_ZSCALE_5_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_5                                                                        0x013c
+#define regPA_CL_VPORT_ZOFFSET_5_BASE_IDX                                                               1
+#define regPA_SC_VPORT_ZMIN_5                                                                           0x013d
+#define regPA_SC_VPORT_ZMIN_5_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_5                                                                           0x013e
+#define regPA_SC_VPORT_ZMAX_5_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_XSCALE_6                                                                         0x013f
+#define regPA_CL_VPORT_XSCALE_6_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_6                                                                        0x0140
+#define regPA_CL_VPORT_XOFFSET_6_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_6                                                                         0x0141
+#define regPA_CL_VPORT_YSCALE_6_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_6                                                                        0x0142
+#define regPA_CL_VPORT_YOFFSET_6_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_6                                                                         0x0143
+#define regPA_CL_VPORT_ZSCALE_6_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_6                                                                        0x0144
+#define regPA_CL_VPORT_ZOFFSET_6_BASE_IDX                                                               1
+#define regPA_SC_VPORT_ZMIN_6                                                                           0x0145
+#define regPA_SC_VPORT_ZMIN_6_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_6                                                                           0x0146
+#define regPA_SC_VPORT_ZMAX_6_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_XSCALE_7                                                                         0x0147
+#define regPA_CL_VPORT_XSCALE_7_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_7                                                                        0x0148
+#define regPA_CL_VPORT_XOFFSET_7_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_7                                                                         0x0149
+#define regPA_CL_VPORT_YSCALE_7_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_7                                                                        0x014a
+#define regPA_CL_VPORT_YOFFSET_7_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_7                                                                         0x014b
+#define regPA_CL_VPORT_ZSCALE_7_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_7                                                                        0x014c
+#define regPA_CL_VPORT_ZOFFSET_7_BASE_IDX                                                               1
+#define regPA_SC_VPORT_ZMIN_7                                                                           0x014d
+#define regPA_SC_VPORT_ZMIN_7_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_7                                                                           0x014e
+#define regPA_SC_VPORT_ZMAX_7_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_XSCALE_8                                                                         0x014f
+#define regPA_CL_VPORT_XSCALE_8_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_8                                                                        0x0150
+#define regPA_CL_VPORT_XOFFSET_8_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_8                                                                         0x0151
+#define regPA_CL_VPORT_YSCALE_8_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_8                                                                        0x0152
+#define regPA_CL_VPORT_YOFFSET_8_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_8                                                                         0x0153
+#define regPA_CL_VPORT_ZSCALE_8_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_8                                                                        0x0154
+#define regPA_CL_VPORT_ZOFFSET_8_BASE_IDX                                                               1
+#define regPA_SC_VPORT_ZMIN_8                                                                           0x0155
+#define regPA_SC_VPORT_ZMIN_8_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_8                                                                           0x0156
+#define regPA_SC_VPORT_ZMAX_8_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_XSCALE_9                                                                         0x0157
+#define regPA_CL_VPORT_XSCALE_9_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_9                                                                        0x0158
+#define regPA_CL_VPORT_XOFFSET_9_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_9                                                                         0x0159
+#define regPA_CL_VPORT_YSCALE_9_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_9                                                                        0x015a
+#define regPA_CL_VPORT_YOFFSET_9_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_9                                                                         0x015b
+#define regPA_CL_VPORT_ZSCALE_9_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_9                                                                        0x015c
+#define regPA_CL_VPORT_ZOFFSET_9_BASE_IDX                                                               1
+#define regPA_SC_VPORT_ZMIN_9                                                                           0x015d
+#define regPA_SC_VPORT_ZMIN_9_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_9                                                                           0x015e
+#define regPA_SC_VPORT_ZMAX_9_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_XSCALE_10                                                                        0x015f
+#define regPA_CL_VPORT_XSCALE_10_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_10                                                                       0x0160
+#define regPA_CL_VPORT_XOFFSET_10_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_10                                                                        0x0161
+#define regPA_CL_VPORT_YSCALE_10_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_10                                                                       0x0162
+#define regPA_CL_VPORT_YOFFSET_10_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_10                                                                        0x0163
+#define regPA_CL_VPORT_ZSCALE_10_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_10                                                                       0x0164
+#define regPA_CL_VPORT_ZOFFSET_10_BASE_IDX                                                              1
+#define regPA_SC_VPORT_ZMIN_10                                                                          0x0165
+#define regPA_SC_VPORT_ZMIN_10_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_10                                                                          0x0166
+#define regPA_SC_VPORT_ZMAX_10_BASE_IDX                                                                 1
+#define regPA_CL_VPORT_XSCALE_11                                                                        0x0167
+#define regPA_CL_VPORT_XSCALE_11_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_11                                                                       0x0168
+#define regPA_CL_VPORT_XOFFSET_11_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_11                                                                        0x0169
+#define regPA_CL_VPORT_YSCALE_11_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_11                                                                       0x016a
+#define regPA_CL_VPORT_YOFFSET_11_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_11                                                                        0x016b
+#define regPA_CL_VPORT_ZSCALE_11_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_11                                                                       0x016c
+#define regPA_CL_VPORT_ZOFFSET_11_BASE_IDX                                                              1
+#define regPA_SC_VPORT_ZMIN_11                                                                          0x016d
+#define regPA_SC_VPORT_ZMIN_11_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_11                                                                          0x016e
+#define regPA_SC_VPORT_ZMAX_11_BASE_IDX                                                                 1
+#define regPA_CL_VPORT_XSCALE_12                                                                        0x016f
+#define regPA_CL_VPORT_XSCALE_12_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_12                                                                       0x0170
+#define regPA_CL_VPORT_XOFFSET_12_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_12                                                                        0x0171
+#define regPA_CL_VPORT_YSCALE_12_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_12                                                                       0x0172
+#define regPA_CL_VPORT_YOFFSET_12_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_12                                                                        0x0173
+#define regPA_CL_VPORT_ZSCALE_12_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_12                                                                       0x0174
+#define regPA_CL_VPORT_ZOFFSET_12_BASE_IDX                                                              1
+#define regPA_SC_VPORT_ZMIN_12                                                                          0x0175
+#define regPA_SC_VPORT_ZMIN_12_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_12                                                                          0x0176
+#define regPA_SC_VPORT_ZMAX_12_BASE_IDX                                                                 1
+#define regPA_CL_VPORT_XSCALE_13                                                                        0x0177
+#define regPA_CL_VPORT_XSCALE_13_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_13                                                                       0x0178
+#define regPA_CL_VPORT_XOFFSET_13_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_13                                                                        0x0179
+#define regPA_CL_VPORT_YSCALE_13_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_13                                                                       0x017a
+#define regPA_CL_VPORT_YOFFSET_13_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_13                                                                        0x017b
+#define regPA_CL_VPORT_ZSCALE_13_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_13                                                                       0x017c
+#define regPA_CL_VPORT_ZOFFSET_13_BASE_IDX                                                              1
+#define regPA_SC_VPORT_ZMIN_13                                                                          0x017d
+#define regPA_SC_VPORT_ZMIN_13_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_13                                                                          0x017e
+#define regPA_SC_VPORT_ZMAX_13_BASE_IDX                                                                 1
+#define regPA_CL_VPORT_XSCALE_14                                                                        0x017f
+#define regPA_CL_VPORT_XSCALE_14_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_14                                                                       0x0180
+#define regPA_CL_VPORT_XOFFSET_14_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_14                                                                        0x0181
+#define regPA_CL_VPORT_YSCALE_14_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_14                                                                       0x0182
+#define regPA_CL_VPORT_YOFFSET_14_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_14                                                                        0x0183
+#define regPA_CL_VPORT_ZSCALE_14_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_14                                                                       0x0184
+#define regPA_CL_VPORT_ZOFFSET_14_BASE_IDX                                                              1
+#define regPA_SC_VPORT_ZMIN_14                                                                          0x0185
+#define regPA_SC_VPORT_ZMIN_14_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_14                                                                          0x0186
+#define regPA_SC_VPORT_ZMAX_14_BASE_IDX                                                                 1
+#define regPA_CL_VPORT_XSCALE_15                                                                        0x0187
+#define regPA_CL_VPORT_XSCALE_15_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_15                                                                       0x0188
+#define regPA_CL_VPORT_XOFFSET_15_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_15                                                                        0x0189
+#define regPA_CL_VPORT_YSCALE_15_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_15                                                                       0x018a
+#define regPA_CL_VPORT_YOFFSET_15_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_15                                                                        0x018b
+#define regPA_CL_VPORT_ZSCALE_15_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_15                                                                       0x018c
+#define regPA_CL_VPORT_ZOFFSET_15_BASE_IDX                                                              1
+#define regPA_SC_VPORT_ZMIN_15                                                                          0x018d
+#define regPA_SC_VPORT_ZMIN_15_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_15                                                                          0x018e
+#define regPA_SC_VPORT_ZMAX_15_BASE_IDX                                                                 1
+#define regSPI_PS_IN_CONTROL                                                                            0x0190
+#define regSPI_PS_IN_CONTROL_BASE_IDX                                                                   1
+#define regSPI_INTERP_CONTROL_0                                                                         0x0191
+#define regSPI_INTERP_CONTROL_0_BASE_IDX                                                                1
+#define regSPI_SHADER_IDX_FORMAT                                                                        0x0192
+#define regSPI_SHADER_IDX_FORMAT_BASE_IDX                                                               1
+#define regSPI_SHADER_POS_FORMAT                                                                        0x0193
+#define regSPI_SHADER_POS_FORMAT_BASE_IDX                                                               1
+#define regSPI_SHADER_Z_FORMAT                                                                          0x0194
+#define regSPI_SHADER_Z_FORMAT_BASE_IDX                                                                 1
+#define regSPI_SHADER_COL_FORMAT                                                                        0x0195
+#define regSPI_SHADER_COL_FORMAT_BASE_IDX                                                               1
+#define regSPI_BARYC_CNTL                                                                               0x0196
+#define regSPI_BARYC_CNTL_BASE_IDX                                                                      1
+#define regSPI_PS_INPUT_ENA                                                                             0x0197
+#define regSPI_PS_INPUT_ENA_BASE_IDX                                                                    1
+#define regSPI_PS_INPUT_ADDR                                                                            0x0198
+#define regSPI_PS_INPUT_ADDR_BASE_IDX                                                                   1
+#define regSPI_PS_INPUT_CNTL_0                                                                          0x0199
+#define regSPI_PS_INPUT_CNTL_0_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_1                                                                          0x019a
+#define regSPI_PS_INPUT_CNTL_1_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_2                                                                          0x019b
+#define regSPI_PS_INPUT_CNTL_2_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_3                                                                          0x019c
+#define regSPI_PS_INPUT_CNTL_3_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_4                                                                          0x019d
+#define regSPI_PS_INPUT_CNTL_4_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_5                                                                          0x019e
+#define regSPI_PS_INPUT_CNTL_5_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_6                                                                          0x019f
+#define regSPI_PS_INPUT_CNTL_6_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_7                                                                          0x01a0
+#define regSPI_PS_INPUT_CNTL_7_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_8                                                                          0x01a1
+#define regSPI_PS_INPUT_CNTL_8_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_9                                                                          0x01a2
+#define regSPI_PS_INPUT_CNTL_9_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_10                                                                         0x01a3
+#define regSPI_PS_INPUT_CNTL_10_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_11                                                                         0x01a4
+#define regSPI_PS_INPUT_CNTL_11_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_12                                                                         0x01a5
+#define regSPI_PS_INPUT_CNTL_12_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_13                                                                         0x01a6
+#define regSPI_PS_INPUT_CNTL_13_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_14                                                                         0x01a7
+#define regSPI_PS_INPUT_CNTL_14_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_15                                                                         0x01a8
+#define regSPI_PS_INPUT_CNTL_15_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_16                                                                         0x01a9
+#define regSPI_PS_INPUT_CNTL_16_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_17                                                                         0x01aa
+#define regSPI_PS_INPUT_CNTL_17_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_18                                                                         0x01ab
+#define regSPI_PS_INPUT_CNTL_18_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_19                                                                         0x01ac
+#define regSPI_PS_INPUT_CNTL_19_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_20                                                                         0x01ad
+#define regSPI_PS_INPUT_CNTL_20_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_21                                                                         0x01ae
+#define regSPI_PS_INPUT_CNTL_21_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_22                                                                         0x01af
+#define regSPI_PS_INPUT_CNTL_22_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_23                                                                         0x01b0
+#define regSPI_PS_INPUT_CNTL_23_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_24                                                                         0x01b1
+#define regSPI_PS_INPUT_CNTL_24_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_25                                                                         0x01b2
+#define regSPI_PS_INPUT_CNTL_25_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_26                                                                         0x01b3
+#define regSPI_PS_INPUT_CNTL_26_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_27                                                                         0x01b4
+#define regSPI_PS_INPUT_CNTL_27_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_28                                                                         0x01b5
+#define regSPI_PS_INPUT_CNTL_28_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_29                                                                         0x01b6
+#define regSPI_PS_INPUT_CNTL_29_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_30                                                                         0x01b7
+#define regSPI_PS_INPUT_CNTL_30_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_31                                                                         0x01b8
+#define regSPI_PS_INPUT_CNTL_31_BASE_IDX                                                                1
+#define regSPI_BARYC_SSAA_CNTL                                                                          0x01b9
+#define regSPI_BARYC_SSAA_CNTL_BASE_IDX                                                                 1
+#define regSPI_TMPRING_SIZE                                                                             0x01ba
+#define regSPI_TMPRING_SIZE_BASE_IDX                                                                    1
+#define regSPI_GFX_SCRATCH_BASE_LO                                                                      0x01bb
+#define regSPI_GFX_SCRATCH_BASE_LO_BASE_IDX                                                             1
+#define regSPI_GFX_SCRATCH_BASE_HI                                                                      0x01bc
+#define regSPI_GFX_SCRATCH_BASE_HI_BASE_IDX                                                             1
+#define regSX_PS_DOWNCONVERT_CONTROL                                                                    0x01d4
+#define regSX_PS_DOWNCONVERT_CONTROL_BASE_IDX                                                           1
+#define regSX_PS_DOWNCONVERT                                                                            0x01d5
+#define regSX_PS_DOWNCONVERT_BASE_IDX                                                                   1
+#define regSX_BLEND_OPT_EPSILON                                                                         0x01d6
+#define regSX_BLEND_OPT_EPSILON_BASE_IDX                                                                1
+#define regSX_BLEND_OPT_CONTROL                                                                         0x01d7
+#define regSX_BLEND_OPT_CONTROL_BASE_IDX                                                                1
+#define regSX_MRT0_BLEND_OPT                                                                            0x01d8
+#define regSX_MRT0_BLEND_OPT_BASE_IDX                                                                   1
+#define regSX_MRT1_BLEND_OPT                                                                            0x01d9
+#define regSX_MRT1_BLEND_OPT_BASE_IDX                                                                   1
+#define regSX_MRT2_BLEND_OPT                                                                            0x01da
+#define regSX_MRT2_BLEND_OPT_BASE_IDX                                                                   1
+#define regSX_MRT3_BLEND_OPT                                                                            0x01db
+#define regSX_MRT3_BLEND_OPT_BASE_IDX                                                                   1
+#define regSX_MRT4_BLEND_OPT                                                                            0x01dc
+#define regSX_MRT4_BLEND_OPT_BASE_IDX                                                                   1
+#define regSX_MRT5_BLEND_OPT                                                                            0x01dd
+#define regSX_MRT5_BLEND_OPT_BASE_IDX                                                                   1
+#define regSX_MRT6_BLEND_OPT                                                                            0x01de
+#define regSX_MRT6_BLEND_OPT_BASE_IDX                                                                   1
+#define regSX_MRT7_BLEND_OPT                                                                            0x01df
+#define regSX_MRT7_BLEND_OPT_BASE_IDX                                                                   1
+#define regCB_BLEND0_CONTROL                                                                            0x01e0
+#define regCB_BLEND0_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND1_CONTROL                                                                            0x01e1
+#define regCB_BLEND1_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND2_CONTROL                                                                            0x01e2
+#define regCB_BLEND2_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND3_CONTROL                                                                            0x01e3
+#define regCB_BLEND3_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND4_CONTROL                                                                            0x01e4
+#define regCB_BLEND4_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND5_CONTROL                                                                            0x01e5
+#define regCB_BLEND5_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND6_CONTROL                                                                            0x01e6
+#define regCB_BLEND6_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND7_CONTROL                                                                            0x01e7
+#define regCB_BLEND7_CONTROL_BASE_IDX                                                                   1
+#define regPA_CL_POINT_X_RAD                                                                            0x01f5
+#define regPA_CL_POINT_X_RAD_BASE_IDX                                                                   1
+#define regPA_CL_POINT_Y_RAD                                                                            0x01f6
+#define regPA_CL_POINT_Y_RAD_BASE_IDX                                                                   1
+#define regPA_CL_POINT_SIZE                                                                             0x01f7
+#define regPA_CL_POINT_SIZE_BASE_IDX                                                                    1
+#define regPA_CL_POINT_CULL_RAD                                                                         0x01f8
+#define regPA_CL_POINT_CULL_RAD_BASE_IDX                                                                1
+#define regGE_MAX_OUTPUT_PER_SUBGROUP                                                                   0x01ff
+#define regGE_MAX_OUTPUT_PER_SUBGROUP_BASE_IDX                                                          1
+#define regPA_CL_CLIP_CNTL                                                                              0x0204
+#define regPA_CL_CLIP_CNTL_BASE_IDX                                                                     1
+#define regPA_CL_VTE_CNTL                                                                               0x0205
+#define regPA_CL_VTE_CNTL_BASE_IDX                                                                      1
+#define regPA_CL_VS_OUT_CNTL                                                                            0x0206
+#define regPA_CL_VS_OUT_CNTL_BASE_IDX                                                                   1
+#define regPA_SU_SC_MODE_CNTL                                                                           0x0207
+#define regPA_SU_SC_MODE_CNTL_BASE_IDX                                                                  1
+#define regPA_CL_NANINF_CNTL                                                                            0x0208
+#define regPA_CL_NANINF_CNTL_BASE_IDX                                                                   1
+#define regPA_SU_LINE_STIPPLE_CNTL                                                                      0x0209
+#define regPA_SU_LINE_STIPPLE_CNTL_BASE_IDX                                                             1
+#define regPA_SU_LINE_STIPPLE_SCALE                                                                     0x020a
+#define regPA_SU_LINE_STIPPLE_SCALE_BASE_IDX                                                            1
+#define regPA_SU_PRIM_FILTER_CNTL                                                                       0x020b
+#define regPA_SU_PRIM_FILTER_CNTL_BASE_IDX                                                              1
+#define regPA_SU_SMALL_PRIM_FILTER_CNTL                                                                 0x020c
+#define regPA_SU_SMALL_PRIM_FILTER_CNTL_BASE_IDX                                                        1
+#define regPA_CL_NGG_CNTL                                                                               0x020e
+#define regPA_CL_NGG_CNTL_BASE_IDX                                                                      1
+#define regPA_SU_OVER_RASTERIZATION_CNTL                                                                0x020f
+#define regPA_SU_OVER_RASTERIZATION_CNTL_BASE_IDX                                                       1
+#define regPA_STEREO_CNTL                                                                               0x0210
+#define regPA_STEREO_CNTL_BASE_IDX                                                                      1
+#define regPA_STATE_STEREO_X                                                                            0x0211
+#define regPA_STATE_STEREO_X_BASE_IDX                                                                   1
+#define regPA_CL_VRS_CNTL                                                                               0x0212
+#define regPA_CL_VRS_CNTL_BASE_IDX                                                                      1
+#define regCB_TARGET_MASK                                                                               0x0214
+#define regCB_TARGET_MASK_BASE_IDX                                                                      1
+#define regCB_SHADER_MASK                                                                               0x0215
+#define regCB_SHADER_MASK_BASE_IDX                                                                      1
+#define regCB_COLOR_CONTROL                                                                             0x0216
+#define regCB_COLOR_CONTROL_BASE_IDX                                                                    1
+#define regPA_SU_POINT_SIZE                                                                             0x0280
+#define regPA_SU_POINT_SIZE_BASE_IDX                                                                    1
+#define regPA_SU_POINT_MINMAX                                                                           0x0281
+#define regPA_SU_POINT_MINMAX_BASE_IDX                                                                  1
+#define regPA_SU_LINE_CNTL                                                                              0x0282
+#define regPA_SU_LINE_CNTL_BASE_IDX                                                                     1
+#define regPA_SC_LINE_STIPPLE                                                                           0x0283
+#define regPA_SC_LINE_STIPPLE_BASE_IDX                                                                  1
+#define regPA_SC_LINE_STIPPLE_RESET                                                                     0x0291
+#define regPA_SC_LINE_STIPPLE_RESET_BASE_IDX                                                            1
+#define regPA_SC_MODE_CNTL_0                                                                            0x0292
+#define regPA_SC_MODE_CNTL_0_BASE_IDX                                                                   1
+#define regPA_SC_MODE_CNTL_1                                                                            0x0293
+#define regPA_SC_MODE_CNTL_1_BASE_IDX                                                                   1
+#define regGE_SE_ENHANCE                                                                                0x0294
+#define regGE_SE_ENHANCE_BASE_IDX                                                                       1
+#define regVGT_REUSE_OFF                                                                                0x02a7
+#define regVGT_REUSE_OFF_BASE_IDX                                                                       1
+#define regVGT_DRAW_PAYLOAD_CNTL                                                                        0x02a8
+#define regVGT_DRAW_PAYLOAD_CNTL_BASE_IDX                                                               1
+#define regDB_HTILE_SURFACE                                                                             0x02af
+#define regDB_HTILE_SURFACE_BASE_IDX                                                                    1
+#define regDB_SRESULTS_COMPARE_STATE0                                                                   0x02b0
+#define regDB_SRESULTS_COMPARE_STATE0_BASE_IDX                                                          1
+#define regDB_SRESULTS_COMPARE_STATE1                                                                   0x02b1
+#define regDB_SRESULTS_COMPARE_STATE1_BASE_IDX                                                          1
+#define regVGT_GS_MAX_VERT_OUT                                                                          0x02ce
+#define regVGT_GS_MAX_VERT_OUT_BASE_IDX                                                                 1
+#define regVGT_GS_INSTANCE_CNT                                                                          0x02cf
+#define regVGT_GS_INSTANCE_CNT_BASE_IDX                                                                 1
+#define regGE_NGG_SUBGRP_CNTL                                                                           0x02d3
+#define regGE_NGG_SUBGRP_CNTL_BASE_IDX                                                                  1
+#define regPA_SU_POLY_OFFSET_DB_FMT_CNTL                                                                0x02de
+#define regPA_SU_POLY_OFFSET_DB_FMT_CNTL_BASE_IDX                                                       1
+#define regPA_SU_POLY_OFFSET_CLAMP                                                                      0x02df
+#define regPA_SU_POLY_OFFSET_CLAMP_BASE_IDX                                                             1
+#define regPA_SU_POLY_OFFSET_FRONT_SCALE                                                                0x02e0
+#define regPA_SU_POLY_OFFSET_FRONT_SCALE_BASE_IDX                                                       1
+#define regPA_SU_POLY_OFFSET_FRONT_OFFSET                                                               0x02e1
+#define regPA_SU_POLY_OFFSET_FRONT_OFFSET_BASE_IDX                                                      1
+#define regPA_SU_POLY_OFFSET_BACK_SCALE                                                                 0x02e2
+#define regPA_SU_POLY_OFFSET_BACK_SCALE_BASE_IDX                                                        1
+#define regPA_SU_POLY_OFFSET_BACK_OFFSET                                                                0x02e3
+#define regPA_SU_POLY_OFFSET_BACK_OFFSET_BASE_IDX                                                       1
+#define regPA_SC_HIZ_INFO                                                                               0x02e5
+#define regPA_SC_HIZ_INFO_BASE_IDX                                                                      1
+#define regPA_SC_HIS_INFO                                                                               0x02e6
+#define regPA_SC_HIS_INFO_BASE_IDX                                                                      1
+#define regPA_SC_HIZ_BASE                                                                               0x02e7
+#define regPA_SC_HIZ_BASE_BASE_IDX                                                                      1
+#define regPA_SC_HIZ_BASE_EXT                                                                           0x02e8
+#define regPA_SC_HIZ_BASE_EXT_BASE_IDX                                                                  1
+#define regPA_SC_HIZ_SIZE_XY                                                                            0x02e9
+#define regPA_SC_HIZ_SIZE_XY_BASE_IDX                                                                   1
+#define regPA_SC_HIS_BASE                                                                               0x02ea
+#define regPA_SC_HIS_BASE_BASE_IDX                                                                      1
+#define regPA_SC_HIS_BASE_EXT                                                                           0x02eb
+#define regPA_SC_HIS_BASE_EXT_BASE_IDX                                                                  1
+#define regPA_SC_HIS_SIZE_XY                                                                            0x02ec
+#define regPA_SC_HIS_SIZE_XY_BASE_IDX                                                                   1
+#define regPA_SC_BINNER_OUTPUT_TIMEOUT_CNTL                                                             0x02ed
+#define regPA_SC_BINNER_OUTPUT_TIMEOUT_CNTL_BASE_IDX                                                    1
+#define regPA_SC_BINNER_DYNAMIC_BATCH_LIMIT                                                             0x02ee
+#define regPA_SC_BINNER_DYNAMIC_BATCH_LIMIT_BASE_IDX                                                    1
+#define regPA_SC_HISZ_CONTROL                                                                           0x02ef
+#define regPA_SC_HISZ_CONTROL_BASE_IDX                                                                  1
+#define regPA_SC_HISZ_RENDER_OVERRIDE                                                                   0x02f0
+#define regPA_SC_HISZ_RENDER_OVERRIDE_BASE_IDX                                                          1
+#define regPA_SC_LINE_CNTL                                                                              0x02f7
+#define regPA_SC_LINE_CNTL_BASE_IDX                                                                     1
+#define regPA_SC_AA_CONFIG                                                                              0x02f8
+#define regPA_SC_AA_CONFIG_BASE_IDX                                                                     1
+#define regPA_SU_VTX_CNTL                                                                               0x02f9
+#define regPA_SU_VTX_CNTL_BASE_IDX                                                                      1
+#define regPA_SC_CENTROID_PRIORITY_0                                                                    0x02fc
+#define regPA_SC_CENTROID_PRIORITY_0_BASE_IDX                                                           1
+#define regPA_SC_CENTROID_PRIORITY_1                                                                    0x02fd
+#define regPA_SC_CENTROID_PRIORITY_1_BASE_IDX                                                           1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0                                                            0x02fe
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1                                                            0x02ff
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2                                                            0x0300
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3                                                            0x0301
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0                                                            0x0302
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1                                                            0x0303
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2                                                            0x0304
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3                                                            0x0305
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0                                                            0x0306
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1                                                            0x0307
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2                                                            0x0308
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3                                                            0x0309
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0                                                            0x030a
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1                                                            0x030b
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2                                                            0x030c
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3                                                            0x030d
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3_BASE_IDX                                                   1
+#define regPA_SC_AA_MASK_X0Y0_X1Y0                                                                      0x030e
+#define regPA_SC_AA_MASK_X0Y0_X1Y0_BASE_IDX                                                             1
+#define regPA_SC_AA_MASK_X0Y1_X1Y1                                                                      0x030f
+#define regPA_SC_AA_MASK_X0Y1_X1Y1_BASE_IDX                                                             1
+#define regPA_SC_BINNER_OUTPUT_TIMEOUT_COUNTER                                                          0x0310
+#define regPA_SC_BINNER_OUTPUT_TIMEOUT_COUNTER_BASE_IDX                                                 1
+#define regPA_SC_BINNER_CNTL_0                                                                          0x0311
+#define regPA_SC_BINNER_CNTL_0_BASE_IDX                                                                 1
+#define regPA_SC_BINNER_CNTL_1                                                                          0x0312
+#define regPA_SC_BINNER_CNTL_1_BASE_IDX                                                                 1
+#define regPA_SC_BINNER_CNTL_2                                                                          0x0313
+#define regPA_SC_BINNER_CNTL_2_BASE_IDX                                                                 1
+#define regPA_SC_NGG_MODE_CNTL                                                                          0x0314
+#define regPA_SC_NGG_MODE_CNTL_BASE_IDX                                                                 1
+#define regPA_SC_CONSERVATIVE_RASTERIZATION_CNTL                                                        0x0315
+#define regPA_SC_CONSERVATIVE_RASTERIZATION_CNTL_BASE_IDX                                               1
+#define regPA_SC_SHADER_CONTROL                                                                         0x0316
+#define regPA_SC_SHADER_CONTROL_BASE_IDX                                                                1
+#define regPA_SC_SAMPLE_PROPERTIES                                                                      0x0317
+#define regPA_SC_SAMPLE_PROPERTIES_BASE_IDX                                                             1
+#define regCB_COLOR0_BASE                                                                               0x0318
+#define regCB_COLOR0_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR0_VIEW                                                                               0x0319
+#define regCB_COLOR0_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR0_VIEW2                                                                              0x031a
+#define regCB_COLOR0_VIEW2_BASE_IDX                                                                     1
+#define regCB_COLOR0_ATTRIB                                                                             0x031b
+#define regCB_COLOR0_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR0_FDCC_CONTROL                                                                       0x031c
+#define regCB_COLOR0_FDCC_CONTROL_BASE_IDX                                                              1
+#define regCB_COLOR0_ATTRIB2                                                                            0x031e
+#define regCB_COLOR0_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR0_ATTRIB3                                                                            0x031f
+#define regCB_COLOR0_ATTRIB3_BASE_IDX                                                                   1
+#define regCB_COLOR1_BASE                                                                               0x0321
+#define regCB_COLOR1_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR1_VIEW                                                                               0x0322
+#define regCB_COLOR1_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR1_VIEW2                                                                              0x0323
+#define regCB_COLOR1_VIEW2_BASE_IDX                                                                     1
+#define regCB_COLOR1_ATTRIB                                                                             0x0324
+#define regCB_COLOR1_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR1_FDCC_CONTROL                                                                       0x0325
+#define regCB_COLOR1_FDCC_CONTROL_BASE_IDX                                                              1
+#define regCB_COLOR1_ATTRIB2                                                                            0x0327
+#define regCB_COLOR1_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR1_ATTRIB3                                                                            0x0328
+#define regCB_COLOR1_ATTRIB3_BASE_IDX                                                                   1
+#define regCB_COLOR2_BASE                                                                               0x032a
+#define regCB_COLOR2_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR2_VIEW                                                                               0x032b
+#define regCB_COLOR2_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR2_VIEW2                                                                              0x032c
+#define regCB_COLOR2_VIEW2_BASE_IDX                                                                     1
+#define regCB_COLOR2_ATTRIB                                                                             0x032d
+#define regCB_COLOR2_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR2_FDCC_CONTROL                                                                       0x032e
+#define regCB_COLOR2_FDCC_CONTROL_BASE_IDX                                                              1
+#define regCB_COLOR2_ATTRIB2                                                                            0x0330
+#define regCB_COLOR2_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR2_ATTRIB3                                                                            0x0331
+#define regCB_COLOR2_ATTRIB3_BASE_IDX                                                                   1
+#define regCB_COLOR3_BASE                                                                               0x0333
+#define regCB_COLOR3_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR3_VIEW                                                                               0x0334
+#define regCB_COLOR3_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR3_VIEW2                                                                              0x0335
+#define regCB_COLOR3_VIEW2_BASE_IDX                                                                     1
+#define regCB_COLOR3_ATTRIB                                                                             0x0336
+#define regCB_COLOR3_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR3_FDCC_CONTROL                                                                       0x0337
+#define regCB_COLOR3_FDCC_CONTROL_BASE_IDX                                                              1
+#define regCB_COLOR3_ATTRIB2                                                                            0x0339
+#define regCB_COLOR3_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR3_ATTRIB3                                                                            0x033a
+#define regCB_COLOR3_ATTRIB3_BASE_IDX                                                                   1
+#define regCB_COLOR4_BASE                                                                               0x033c
+#define regCB_COLOR4_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR4_VIEW                                                                               0x033d
+#define regCB_COLOR4_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR4_VIEW2                                                                              0x033e
+#define regCB_COLOR4_VIEW2_BASE_IDX                                                                     1
+#define regCB_COLOR4_ATTRIB                                                                             0x033f
+#define regCB_COLOR4_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR4_FDCC_CONTROL                                                                       0x0340
+#define regCB_COLOR4_FDCC_CONTROL_BASE_IDX                                                              1
+#define regCB_COLOR4_ATTRIB2                                                                            0x0342
+#define regCB_COLOR4_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR4_ATTRIB3                                                                            0x0343
+#define regCB_COLOR4_ATTRIB3_BASE_IDX                                                                   1
+#define regCB_COLOR5_BASE                                                                               0x0345
+#define regCB_COLOR5_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR5_VIEW                                                                               0x0346
+#define regCB_COLOR5_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR5_VIEW2                                                                              0x0347
+#define regCB_COLOR5_VIEW2_BASE_IDX                                                                     1
+#define regCB_COLOR5_ATTRIB                                                                             0x0348
+#define regCB_COLOR5_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR5_FDCC_CONTROL                                                                       0x0349
+#define regCB_COLOR5_FDCC_CONTROL_BASE_IDX                                                              1
+#define regCB_COLOR5_ATTRIB2                                                                            0x034b
+#define regCB_COLOR5_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR5_ATTRIB3                                                                            0x034c
+#define regCB_COLOR5_ATTRIB3_BASE_IDX                                                                   1
+#define regCB_COLOR6_BASE                                                                               0x034e
+#define regCB_COLOR6_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR6_VIEW                                                                               0x034f
+#define regCB_COLOR6_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR6_VIEW2                                                                              0x0350
+#define regCB_COLOR6_VIEW2_BASE_IDX                                                                     1
+#define regCB_COLOR6_ATTRIB                                                                             0x0351
+#define regCB_COLOR6_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR6_FDCC_CONTROL                                                                       0x0352
+#define regCB_COLOR6_FDCC_CONTROL_BASE_IDX                                                              1
+#define regCB_COLOR6_ATTRIB2                                                                            0x0354
+#define regCB_COLOR6_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR6_ATTRIB3                                                                            0x0355
+#define regCB_COLOR6_ATTRIB3_BASE_IDX                                                                   1
+#define regCB_COLOR7_BASE                                                                               0x0357
+#define regCB_COLOR7_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR7_VIEW                                                                               0x0358
+#define regCB_COLOR7_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR7_VIEW2                                                                              0x0359
+#define regCB_COLOR7_VIEW2_BASE_IDX                                                                     1
+#define regCB_COLOR7_ATTRIB                                                                             0x035a
+#define regCB_COLOR7_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR7_FDCC_CONTROL                                                                       0x035b
+#define regCB_COLOR7_FDCC_CONTROL_BASE_IDX                                                              1
+#define regCB_COLOR7_ATTRIB2                                                                            0x035d
+#define regCB_COLOR7_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR7_ATTRIB3                                                                            0x035e
+#define regCB_COLOR7_ATTRIB3_BASE_IDX                                                                   1
+#define regCB_COLOR0_BASE_EXT                                                                           0x0390
+#define regCB_COLOR0_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR1_BASE_EXT                                                                           0x0391
+#define regCB_COLOR1_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR2_BASE_EXT                                                                           0x0392
+#define regCB_COLOR2_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR3_BASE_EXT                                                                           0x0393
+#define regCB_COLOR3_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR4_BASE_EXT                                                                           0x0394
+#define regCB_COLOR4_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR5_BASE_EXT                                                                           0x0395
+#define regCB_COLOR5_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR6_BASE_EXT                                                                           0x0396
+#define regCB_COLOR6_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR7_BASE_EXT                                                                           0x0397
+#define regCB_COLOR7_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR0_INFO                                                                               0x03b0
+#define regCB_COLOR0_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR1_INFO                                                                               0x03b1
+#define regCB_COLOR1_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR2_INFO                                                                               0x03b2
+#define regCB_COLOR2_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR3_INFO                                                                               0x03b3
+#define regCB_COLOR3_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR4_INFO                                                                               0x03b4
+#define regCB_COLOR4_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR5_INFO                                                                               0x03b5
+#define regCB_COLOR5_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR6_INFO                                                                               0x03b6
+#define regCB_COLOR6_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR7_INFO                                                                               0x03b7
+#define regCB_COLOR7_INFO_BASE_IDX                                                                      1
+#define regCB_MEM0_INFO                                                                                 0x03c0
+#define regCB_MEM0_INFO_BASE_IDX                                                                        1
+#define regCB_MEM1_INFO                                                                                 0x03c1
+#define regCB_MEM1_INFO_BASE_IDX                                                                        1
+#define regCB_MEM2_INFO                                                                                 0x03c2
+#define regCB_MEM2_INFO_BASE_IDX                                                                        1
+#define regCB_MEM3_INFO                                                                                 0x03c3
+#define regCB_MEM3_INFO_BASE_IDX                                                                        1
+#define regCB_MEM4_INFO                                                                                 0x03c4
+#define regCB_MEM4_INFO_BASE_IDX                                                                        1
+#define regCB_MEM5_INFO                                                                                 0x03c5
+#define regCB_MEM5_INFO_BASE_IDX                                                                        1
+#define regCB_MEM6_INFO                                                                                 0x03c6
+#define regCB_MEM6_INFO_BASE_IDX                                                                        1
+#define regCB_MEM7_INFO                                                                                 0x03c7
+#define regCB_MEM7_INFO_BASE_IDX                                                                        1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pfvf_padec
+// base address: 0x2a500
+#define regPA_SC_VRS_SURFACE_CNTL                                                                       0x0940
+#define regPA_SC_VRS_SURFACE_CNTL_BASE_IDX                                                              1
+#define regPA_SC_ENHANCE                                                                                0x0941
+#define regPA_SC_ENHANCE_BASE_IDX                                                                       1
+#define regPA_SC_ENHANCE_1                                                                              0x0942
+#define regPA_SC_ENHANCE_1_BASE_IDX                                                                     1
+#define regPA_SC_ENHANCE_2                                                                              0x0943
+#define regPA_SC_ENHANCE_2_BASE_IDX                                                                     1
+#define regPA_SC_ENHANCE_3                                                                              0x0944
+#define regPA_SC_ENHANCE_3_BASE_IDX                                                                     1
+#define regPA_SC_BINNER_CNTL_OVERRIDE                                                                   0x0946
+#define regPA_SC_BINNER_CNTL_OVERRIDE_BASE_IDX                                                          1
+#define regPA_SC_PBB_OVERRIDE_FLAG                                                                      0x0947
+#define regPA_SC_PBB_OVERRIDE_FLAG_BASE_IDX                                                             1
+#define regPA_SC_DSM_CNTL                                                                               0x0948
+#define regPA_SC_DSM_CNTL_BASE_IDX                                                                      1
+#define regPA_SC_TILE_STEERING_CREST_OVERRIDE                                                           0x0949
+#define regPA_SC_TILE_STEERING_CREST_OVERRIDE_BASE_IDX                                                  1
+#define regPA_SC_FIFO_SIZE                                                                              0x094a
+#define regPA_SC_FIFO_SIZE_BASE_IDX                                                                     1
+#define regPA_SC_IF_FIFO_SIZE                                                                           0x094b
+#define regPA_SC_IF_FIFO_SIZE_BASE_IDX                                                                  1
+#define regPA_SC_PACKER_WAVE_ID_CNTL                                                                    0x094c
+#define regPA_SC_PACKER_WAVE_ID_CNTL_BASE_IDX                                                           1
+#define regPA_SC_ATM_CNTL                                                                               0x094d
+#define regPA_SC_ATM_CNTL_BASE_IDX                                                                      1
+#define regPA_SC_PKR_WAVE_TABLE_CNTL                                                                    0x094e
+#define regPA_SC_PKR_WAVE_TABLE_CNTL_BASE_IDX                                                           1
+#define regPA_SC_FORCE_EOV_MAX_CNTS                                                                     0x094f
+#define regPA_SC_FORCE_EOV_MAX_CNTS_BASE_IDX                                                            1
+#define regPA_SC_BINNER_EVENT_CNTL_0                                                                    0x0950
+#define regPA_SC_BINNER_EVENT_CNTL_0_BASE_IDX                                                           1
+#define regPA_SC_BINNER_EVENT_CNTL_1                                                                    0x0951
+#define regPA_SC_BINNER_EVENT_CNTL_1_BASE_IDX                                                           1
+#define regPA_SC_BINNER_EVENT_CNTL_2                                                                    0x0952
+#define regPA_SC_BINNER_EVENT_CNTL_2_BASE_IDX                                                           1
+#define regPA_SC_BINNER_EVENT_CNTL_3                                                                    0x0953
+#define regPA_SC_BINNER_EVENT_CNTL_3_BASE_IDX                                                           1
+#define regPA_SC_BINNER_TIMEOUT_COUNTER                                                                 0x0954
+#define regPA_SC_BINNER_TIMEOUT_COUNTER_BASE_IDX                                                        1
+#define regPA_SC_BINNER_PERF_CNTL_0                                                                     0x0955
+#define regPA_SC_BINNER_PERF_CNTL_0_BASE_IDX                                                            1
+#define regPA_SC_BINNER_PERF_CNTL_1                                                                     0x0956
+#define regPA_SC_BINNER_PERF_CNTL_1_BASE_IDX                                                            1
+#define regPA_SC_BINNER_PERF_CNTL_2                                                                     0x0957
+#define regPA_SC_BINNER_PERF_CNTL_2_BASE_IDX                                                            1
+#define regPA_SC_BINNER_PERF_CNTL_3                                                                     0x0958
+#define regPA_SC_BINNER_PERF_CNTL_3_BASE_IDX                                                            1
+#define regPA_SC_P3D_TRAP_SCREEN_HV_LOCK                                                                0x095b
+#define regPA_SC_P3D_TRAP_SCREEN_HV_LOCK_BASE_IDX                                                       1
+#define regPA_SC_HP3D_TRAP_SCREEN_HV_LOCK                                                               0x095c
+#define regPA_SC_HP3D_TRAP_SCREEN_HV_LOCK_BASE_IDX                                                      1
+#define regPA_SC_TRAP_SCREEN_HV_LOCK                                                                    0x095d
+#define regPA_SC_TRAP_SCREEN_HV_LOCK_BASE_IDX                                                           1
+#define regPA_PH_INTERFACE_FIFO_SIZE                                                                    0x095e
+#define regPA_PH_INTERFACE_FIFO_SIZE_BASE_IDX                                                           1
+#define regPA_PH_ENHANCE                                                                                0x095f
+#define regPA_PH_ENHANCE_BASE_IDX                                                                       1
+#define regPA_SC_VRS_SURFACE_CNTL_1                                                                     0x0960
+#define regPA_SC_VRS_SURFACE_CNTL_1_BASE_IDX                                                            1
+#define regPA_SC_HIZ_SURFACE_CNTL                                                                       0x0961
+#define regPA_SC_HIZ_SURFACE_CNTL_BASE_IDX                                                              1
+#define regPA_SC_HIS_SURFACE_CNTL                                                                       0x0962
+#define regPA_SC_HIS_SURFACE_CNTL_BASE_IDX                                                              1
+#define regPA_SC_HIZ_DEBUG                                                                              0x0963
+#define regPA_SC_HIZ_DEBUG_BASE_IDX                                                                     1
+#define regPA_SC_HIS_DEBUG                                                                              0x0964
+#define regPA_SC_HIS_DEBUG_BASE_IDX                                                                     1
+#define regSC_MEM_SCOPE                                                                                 0x0965
+#define regSC_MEM_SCOPE_BASE_IDX                                                                        1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pfvf_sqdec
+// base address: 0x2a780
+#define regSQ_RUNTIME_CONFIG                                                                            0x09e0
+#define regSQ_RUNTIME_CONFIG_BASE_IDX                                                                   1
+#define regSQ_DEBUG_STS_GLOBAL                                                                          0x09e1
+#define regSQ_DEBUG_STS_GLOBAL_BASE_IDX                                                                 1
+#define regSQ_DEBUG_STS_GLOBAL2                                                                         0x09e2
+#define regSQ_DEBUG_STS_GLOBAL2_BASE_IDX                                                                1
+#define regSH_MEM_BASES                                                                                 0x09e3
+#define regSH_MEM_BASES_BASE_IDX                                                                        1
+#define regSH_MEM_CONFIG                                                                                0x09e4
+#define regSH_MEM_CONFIG_BASE_IDX                                                                       1
+#define regSQ_DEBUG                                                                                     0x09e5
+#define regSQ_DEBUG_BASE_IDX                                                                            1
+#define regSQ_SHADER_TBA_LO                                                                             0x09e6
+#define regSQ_SHADER_TBA_LO_BASE_IDX                                                                    1
+#define regSQ_SHADER_TBA_HI                                                                             0x09e7
+#define regSQ_SHADER_TBA_HI_BASE_IDX                                                                    1
+#define regSQ_SHADER_TMA_LO                                                                             0x09e8
+#define regSQ_SHADER_TMA_LO_BASE_IDX                                                                    1
+#define regSQ_SHADER_TMA_HI                                                                             0x09e9
+#define regSQ_SHADER_TMA_HI_BASE_IDX                                                                    1
+#define regSQ_IND_INDEX_USER                                                                            0x09ea
+#define regSQ_IND_INDEX_USER_BASE_IDX                                                                   1
+#define regSQ_CMD_USER                                                                                  0x09eb
+#define regSQ_CMD_USER_BASE_IDX                                                                         1
+#define regSQ_IND_DATA_USER                                                                             0x09ec
+#define regSQ_IND_DATA_USER_BASE_IDX                                                                    1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pfonly_spidec
+// base address: 0x2e500
+#define regSPI_CDBG_SYS_GFX                                                                             0x1940
+#define regSPI_CDBG_SYS_GFX_BASE_IDX                                                                    1
+#define regSPI_CDBG_SYS_HP3D                                                                            0x1941
+#define regSPI_CDBG_SYS_HP3D_BASE_IDX                                                                   1
+#define regSPI_CDBG_SYS_CS0                                                                             0x1942
+#define regSPI_CDBG_SYS_CS0_BASE_IDX                                                                    1
+#define regSPI_GDBG_WAVE_CNTL                                                                           0x1943
+#define regSPI_GDBG_WAVE_CNTL_BASE_IDX                                                                  1
+#define regSPI_GDBG_TRAP_CONFIG                                                                         0x1944
+#define regSPI_GDBG_TRAP_CONFIG_BASE_IDX                                                                1
+#define regSPI_GDBG_WAVE_CNTL3                                                                          0x1945
+#define regSPI_GDBG_WAVE_CNTL3_BASE_IDX                                                                 1
+#define regSPI_RESET_DEBUG                                                                              0x1946
+#define regSPI_RESET_DEBUG_BASE_IDX                                                                     1
+#define regGDS_COMPUTE_MAX_WAVE_ID                                                                      0x1947
+#define regGDS_COMPUTE_MAX_WAVE_ID_BASE_IDX                                                             1
+#define regSPI_ARB_CNTL_0                                                                               0x1949
+#define regSPI_ARB_CNTL_0_BASE_IDX                                                                      1
+#define regSPI_FEATURE_CTRL                                                                             0x194a
+#define regSPI_FEATURE_CTRL_BASE_IDX                                                                    1
+#define regSPI_SHADER_RSRC_LIMIT_CTRL                                                                   0x194b
+#define regSPI_SHADER_RSRC_LIMIT_CTRL_BASE_IDX                                                          1
+#define regPC_CONFIG_CNTL_0                                                                             0x194c
+#define regPC_CONFIG_CNTL_0_BASE_IDX                                                                    1
+#define regPC_CONFIG_CNTL_1                                                                             0x194d
+#define regPC_CONFIG_CNTL_1_BASE_IDX                                                                    1
+#define regSPI_COMPUTE_WF_CTX_SAVE_STATUS                                                               0x194e
+#define regSPI_COMPUTE_WF_CTX_SAVE_STATUS_BASE_IDX                                                      1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pfonly_utcl1dec
+// base address: 0x2e600
+#define regUTCL1_CTRL_0                                                                                 0x1980
+#define regUTCL1_CTRL_0_BASE_IDX                                                                        1
+#define regUTCL1_UTCL0_INVREQ_DISABLE                                                                   0x1981
+#define regUTCL1_UTCL0_INVREQ_DISABLE_BASE_IDX                                                          1
+#define regUTCL1_CTRL_2                                                                                 0x1982
+#define regUTCL1_CTRL_2_BASE_IDX                                                                        1
+#define regUTCL1_FIFO_SIZING                                                                            0x1983
+#define regUTCL1_FIFO_SIZING_BASE_IDX                                                                   1
+#define regGCRD_SA0_TARGETS_DISABLE                                                                     0x1984
+#define regGCRD_SA0_TARGETS_DISABLE_BASE_IDX                                                            1
+#define regGCRD_SA1_TARGETS_DISABLE                                                                     0x1985
+#define regGCRD_SA1_TARGETS_DISABLE_BASE_IDX                                                            1
+#define regGCRD_CREDIT_SAFE                                                                             0x1986
+#define regGCRD_CREDIT_SAFE_BASE_IDX                                                                    1
+#define regUTCL1_IDENTITY_MODE0                                                                         0x1987
+#define regUTCL1_IDENTITY_MODE0_BASE_IDX                                                                1
+#define regUTCL1_IDENTITY_MODE1                                                                         0x1988
+#define regUTCL1_IDENTITY_MODE1_BASE_IDX                                                                1
+#define regUTCL1_IDENTITY_MODE2                                                                         0x1989
+#define regUTCL1_IDENTITY_MODE2_BASE_IDX                                                                1
+#define regUTCL1_IDENTITY_MODE3                                                                         0x198a
+#define regUTCL1_IDENTITY_MODE3_BASE_IDX                                                                1
+#define regUTCL1_IDENTITY_MODE4                                                                         0x198b
+#define regUTCL1_IDENTITY_MODE4_BASE_IDX                                                                1
+#define regUTCL1_IDENTITY_MODE5                                                                         0x198c
+#define regUTCL1_IDENTITY_MODE5_BASE_IDX                                                                1
+#define regUTCL1_IDENTITY_MODE6                                                                         0x198d
+#define regUTCL1_IDENTITY_MODE6_BASE_IDX                                                                1
+#define regUTCL1_IDENTITY_MODE7                                                                         0x198e
+#define regUTCL1_IDENTITY_MODE7_BASE_IDX                                                                1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pfonly_tcpdec
+// base address: 0x2ef40
+#define regTCP_INVALIDATE                                                                               0x1bd0
+#define regTCP_INVALIDATE_BASE_IDX                                                                      1
+#define regTCP_STATUS                                                                                   0x1bd1
+#define regTCP_STATUS_BASE_IDX                                                                          1
+#define regTCP_CNTL                                                                                     0x1bd2
+#define regTCP_CNTL_BASE_IDX                                                                            1
+#define regTCP_CNTL2                                                                                    0x1bd3
+#define regTCP_CNTL2_BASE_IDX                                                                           1
+#define regTCP_CREDIT                                                                                   0x1bd4
+#define regTCP_CREDIT_BASE_IDX                                                                          1
+#define regTCP_DEBUG_DATA                                                                               0x1bd6
+#define regTCP_DEBUG_DATA_BASE_IDX                                                                      1
+#define regTCP_COMPRESSION_CNTL                                                                         0x1bd7
+#define regTCP_COMPRESSION_CNTL_BASE_IDX                                                                1
+#define regTCP_ARB                                                                                      0x1bd8
+#define regTCP_ARB_BASE_IDX                                                                             1
+#define regTCP_UTCL0_CNTL1                                                                              0x1bd9
+#define regTCP_UTCL0_CNTL1_BASE_IDX                                                                     1
+#define regTCP_UTCL0_CNTL2                                                                              0x1bda
+#define regTCP_UTCL0_CNTL2_BASE_IDX                                                                     1
+#define regTCP_UTCL0_STATUS                                                                             0x1bdb
+#define regTCP_UTCL0_STATUS_BASE_IDX                                                                    1
+#define regTCP_RQID_HASH_CNTL                                                                           0x1bdc
+#define regTCP_RQID_HASH_CNTL_BASE_IDX                                                                  1
+#define regTCP_SET_HASH_CFG                                                                             0x1bdd
+#define regTCP_SET_HASH_CFG_BASE_IDX                                                                    1
+#define regTCP_SET_HASH_MASK_64_SET0                                                                    0x1bde
+#define regTCP_SET_HASH_MASK_64_SET0_BASE_IDX                                                           1
+#define regTCP_SET_HASH_MASK_64_SET1                                                                    0x1bdf
+#define regTCP_SET_HASH_MASK_64_SET1_BASE_IDX                                                           1
+#define regTCP_SET_HASH_MASK_64_SET2                                                                    0x1be0
+#define regTCP_SET_HASH_MASK_64_SET2_BASE_IDX                                                           1
+#define regTCP_SET_HASH_MASK_64_SET3                                                                    0x1be1
+#define regTCP_SET_HASH_MASK_64_SET3_BASE_IDX                                                           1
+#define regTCP_SET_HASH_MASK_128_SET0                                                                   0x1be2
+#define regTCP_SET_HASH_MASK_128_SET0_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_128_SET1                                                                   0x1be3
+#define regTCP_SET_HASH_MASK_128_SET1_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_128_SET2                                                                   0x1be4
+#define regTCP_SET_HASH_MASK_128_SET2_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_128_SET3                                                                   0x1be5
+#define regTCP_SET_HASH_MASK_128_SET3_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_128_SET4                                                                   0x1be6
+#define regTCP_SET_HASH_MASK_128_SET4_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_192_SET0                                                                   0x1be7
+#define regTCP_SET_HASH_MASK_192_SET0_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_192_SET1                                                                   0x1be8
+#define regTCP_SET_HASH_MASK_192_SET1_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_192_SET2                                                                   0x1be9
+#define regTCP_SET_HASH_MASK_192_SET2_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_192_SET3                                                                   0x1bea
+#define regTCP_SET_HASH_MASK_192_SET3_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_256_SET0                                                                   0x1beb
+#define regTCP_SET_HASH_MASK_256_SET0_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_256_SET1                                                                   0x1bec
+#define regTCP_SET_HASH_MASK_256_SET1_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_256_SET2                                                                   0x1bed
+#define regTCP_SET_HASH_MASK_256_SET2_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_256_SET3                                                                   0x1bee
+#define regTCP_SET_HASH_MASK_256_SET3_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_256_SET4                                                                   0x1bef
+#define regTCP_SET_HASH_MASK_256_SET4_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_256_SET5                                                                   0x1bf0
+#define regTCP_SET_HASH_MASK_256_SET5_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_320_SET0                                                                   0x1bf1
+#define regTCP_SET_HASH_MASK_320_SET0_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_320_SET1                                                                   0x1bf2
+#define regTCP_SET_HASH_MASK_320_SET1_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_320_SET2                                                                   0x1bf3
+#define regTCP_SET_HASH_MASK_320_SET2_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_320_SET3                                                                   0x1bf4
+#define regTCP_SET_HASH_MASK_320_SET3_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_384_SET0                                                                   0x1bf5
+#define regTCP_SET_HASH_MASK_384_SET0_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_384_SET1                                                                   0x1bf6
+#define regTCP_SET_HASH_MASK_384_SET1_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_384_SET2                                                                   0x1bf7
+#define regTCP_SET_HASH_MASK_384_SET2_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_384_SET3                                                                   0x1bf8
+#define regTCP_SET_HASH_MASK_384_SET3_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_384_SET4                                                                   0x1bf9
+#define regTCP_SET_HASH_MASK_384_SET4_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_448_SET0                                                                   0x1bfa
+#define regTCP_SET_HASH_MASK_448_SET0_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_448_SET1                                                                   0x1bfb
+#define regTCP_SET_HASH_MASK_448_SET1_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_448_SET2                                                                   0x1bfc
+#define regTCP_SET_HASH_MASK_448_SET2_BASE_IDX                                                          1
+#define regTCP_SET_HASH_MASK_448_SET3                                                                   0x1bfd
+#define regTCP_SET_HASH_MASK_448_SET3_BASE_IDX                                                          1
+#define regTCP_CNTL3                                                                                    0x1bfe
+#define regTCP_CNTL3_BASE_IDX                                                                           1
+#define regLDS_CONFIG                                                                                   0x1bff
+#define regLDS_CONFIG_BASE_IDX                                                                          1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pfonly2_spidec
+// base address: 0x2f000
+#define regSPI_RESOURCE_RESERVE_CU_0                                                                    0x1c00
+#define regSPI_RESOURCE_RESERVE_CU_0_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_1                                                                    0x1c01
+#define regSPI_RESOURCE_RESERVE_CU_1_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_2                                                                    0x1c02
+#define regSPI_RESOURCE_RESERVE_CU_2_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_3                                                                    0x1c03
+#define regSPI_RESOURCE_RESERVE_CU_3_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_4                                                                    0x1c04
+#define regSPI_RESOURCE_RESERVE_CU_4_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_5                                                                    0x1c05
+#define regSPI_RESOURCE_RESERVE_CU_5_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_6                                                                    0x1c06
+#define regSPI_RESOURCE_RESERVE_CU_6_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_7                                                                    0x1c07
+#define regSPI_RESOURCE_RESERVE_CU_7_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_8                                                                    0x1c08
+#define regSPI_RESOURCE_RESERVE_CU_8_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_9                                                                    0x1c09
+#define regSPI_RESOURCE_RESERVE_CU_9_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_10                                                                   0x1c0a
+#define regSPI_RESOURCE_RESERVE_CU_10_BASE_IDX                                                          1
+#define regSPI_RESOURCE_RESERVE_CU_11                                                                   0x1c0b
+#define regSPI_RESOURCE_RESERVE_CU_11_BASE_IDX                                                          1
+#define regSPI_RESOURCE_RESERVE_CU_12                                                                   0x1c0c
+#define regSPI_RESOURCE_RESERVE_CU_12_BASE_IDX                                                          1
+#define regSPI_RESOURCE_RESERVE_CU_13                                                                   0x1c0d
+#define regSPI_RESOURCE_RESERVE_CU_13_BASE_IDX                                                          1
+#define regSPI_RESOURCE_RESERVE_CU_14                                                                   0x1c0e
+#define regSPI_RESOURCE_RESERVE_CU_14_BASE_IDX                                                          1
+#define regSPI_RESOURCE_RESERVE_CU_15                                                                   0x1c0f
+#define regSPI_RESOURCE_RESERVE_CU_15_BASE_IDX                                                          1
+#define regSPI_RESOURCE_RESERVE_EN_CU_0                                                                 0x1c10
+#define regSPI_RESOURCE_RESERVE_EN_CU_0_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_1                                                                 0x1c11
+#define regSPI_RESOURCE_RESERVE_EN_CU_1_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_2                                                                 0x1c12
+#define regSPI_RESOURCE_RESERVE_EN_CU_2_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_3                                                                 0x1c13
+#define regSPI_RESOURCE_RESERVE_EN_CU_3_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_4                                                                 0x1c14
+#define regSPI_RESOURCE_RESERVE_EN_CU_4_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_5                                                                 0x1c15
+#define regSPI_RESOURCE_RESERVE_EN_CU_5_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_6                                                                 0x1c16
+#define regSPI_RESOURCE_RESERVE_EN_CU_6_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_7                                                                 0x1c17
+#define regSPI_RESOURCE_RESERVE_EN_CU_7_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_8                                                                 0x1c18
+#define regSPI_RESOURCE_RESERVE_EN_CU_8_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_9                                                                 0x1c19
+#define regSPI_RESOURCE_RESERVE_EN_CU_9_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_10                                                                0x1c1a
+#define regSPI_RESOURCE_RESERVE_EN_CU_10_BASE_IDX                                                       1
+#define regSPI_RESOURCE_RESERVE_EN_CU_11                                                                0x1c1b
+#define regSPI_RESOURCE_RESERVE_EN_CU_11_BASE_IDX                                                       1
+#define regSPI_RESOURCE_RESERVE_EN_CU_12                                                                0x1c1c
+#define regSPI_RESOURCE_RESERVE_EN_CU_12_BASE_IDX                                                       1
+#define regSPI_RESOURCE_RESERVE_EN_CU_13                                                                0x1c1d
+#define regSPI_RESOURCE_RESERVE_EN_CU_13_BASE_IDX                                                       1
+#define regSPI_RESOURCE_RESERVE_EN_CU_14                                                                0x1c1e
+#define regSPI_RESOURCE_RESERVE_EN_CU_14_BASE_IDX                                                       1
+#define regSPI_RESOURCE_RESERVE_EN_CU_15                                                                0x1c1f
+#define regSPI_RESOURCE_RESERVE_EN_CU_15_BASE_IDX                                                       1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pfonly2_tcpdec
+// base address: 0x2f4c0
+#define regTCP_UTCL0_THRASHING_CTRL                                                                     0x1d30
+#define regTCP_UTCL0_THRASHING_CTRL_BASE_IDX                                                            1
+#define regTCP_UTCL0_RETRY_THRESHOLD_DOWN_COUNTER                                                       0x1d31
+#define regTCP_UTCL0_RETRY_THRESHOLD_DOWN_COUNTER_BASE_IDX                                              1
+#define regTCP_UTCL0_RETRY_THRESHOLD_UP_COUNTER                                                         0x1d32
+#define regTCP_UTCL0_RETRY_THRESHOLD_UP_COUNTER_BASE_IDX                                                1
+#define regTCP_UTCL0_XNACK_RETRY                                                                        0x1d33
+#define regTCP_UTCL0_XNACK_RETRY_BASE_IDX                                                               1
+#define regTCP_UTCL0_RETRY_TIMER_THRESHOLD_LO                                                           0x1d34
+#define regTCP_UTCL0_RETRY_TIMER_THRESHOLD_LO_BASE_IDX                                                  1
+#define regTCP_UTCL0_RETRY_TIMER_THRESHOLD_HI                                                           0x1d35
+#define regTCP_UTCL0_RETRY_TIMER_THRESHOLD_HI_BASE_IDX                                                  1
+#define regTXA_CNTL                                                                                     0x1d36
+#define regTXA_CNTL_BASE_IDX                                                                            1
+#define regTXA_CNTL_AUX                                                                                 0x1d37
+#define regTXA_CNTL_AUX_BASE_IDX                                                                        1
+#define regTXA_CNTL2                                                                                    0x1d38
+#define regTXA_CNTL2_BASE_IDX                                                                           1
+#define regTXA_STATUS                                                                                   0x1d39
+#define regTXA_STATUS_BASE_IDX                                                                          1
+#define regTXA_TDM_ARB_CNTL                                                                             0x1d3a
+#define regTXA_TDM_ARB_CNTL_BASE_IDX                                                                    1
+#define regTCP_CREDIT2                                                                                  0x1d3b
+#define regTCP_CREDIT2_BASE_IDX                                                                         1
+#define regVC_CONGESTION_CONTROL                                                                        0x1d3c
+#define regVC_CONGESTION_CONTROL_BASE_IDX                                                               1
+#define regTXA_TDM_CNTL                                                                                 0x1d3d
+#define regTXA_TDM_CNTL_BASE_IDX                                                                        1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_gfxudec
+// base address: 0x30000
+#define regVGT_TF_RING_SIZE                                                                             0x224e
+#define regVGT_TF_RING_SIZE_BASE_IDX                                                                    1
+#define regVGT_HS_OFFCHIP_PARAM                                                                         0x224f
+#define regVGT_HS_OFFCHIP_PARAM_BASE_IDX                                                                1
+#define regGE_POS_RING_BASE                                                                             0x2268
+#define regGE_POS_RING_BASE_BASE_IDX                                                                    1
+#define regGE_POS_RING_SIZE                                                                             0x2269
+#define regGE_POS_RING_SIZE_BASE_IDX                                                                    1
+#define regGE_PRIM_RING_BASE                                                                            0x226a
+#define regGE_PRIM_RING_BASE_BASE_IDX                                                                   1
+#define regGE_PRIM_RING_SIZE                                                                            0x226b
+#define regGE_PRIM_RING_SIZE_BASE_IDX                                                                   1
+#define regPA_SU_LINE_STIPPLE_VALUE                                                                     0x2280
+#define regPA_SU_LINE_STIPPLE_VALUE_BASE_IDX                                                            1
+#define regPA_SC_LINE_STIPPLE_STATE                                                                     0x2281
+#define regPA_SC_LINE_STIPPLE_STATE_BASE_IDX                                                            1
+#define regPA_SC_SCREEN_EXTENT_MIN_0                                                                    0x2284
+#define regPA_SC_SCREEN_EXTENT_MIN_0_BASE_IDX                                                           1
+#define regPA_SC_SCREEN_EXTENT_MAX_0                                                                    0x2285
+#define regPA_SC_SCREEN_EXTENT_MAX_0_BASE_IDX                                                           1
+#define regPA_SC_SCREEN_EXTENT_MIN_1                                                                    0x2286
+#define regPA_SC_SCREEN_EXTENT_MIN_1_BASE_IDX                                                           1
+#define regPA_SC_SCREEN_EXTENT_MAX_1                                                                    0x228b
+#define regPA_SC_SCREEN_EXTENT_MAX_1_BASE_IDX                                                           1
+#define regPA_SC_P3D_TRAP_SCREEN_HV_EN                                                                  0x22a0
+#define regPA_SC_P3D_TRAP_SCREEN_HV_EN_BASE_IDX                                                         1
+#define regPA_SC_P3D_TRAP_SCREEN_H                                                                      0x22a1
+#define regPA_SC_P3D_TRAP_SCREEN_H_BASE_IDX                                                             1
+#define regPA_SC_P3D_TRAP_SCREEN_V                                                                      0x22a2
+#define regPA_SC_P3D_TRAP_SCREEN_V_BASE_IDX                                                             1
+#define regPA_SC_P3D_TRAP_SCREEN_OCCURRENCE                                                             0x22a3
+#define regPA_SC_P3D_TRAP_SCREEN_OCCURRENCE_BASE_IDX                                                    1
+#define regPA_SC_P3D_TRAP_SCREEN_COUNT                                                                  0x22a4
+#define regPA_SC_P3D_TRAP_SCREEN_COUNT_BASE_IDX                                                         1
+#define regPA_SC_HP3D_TRAP_SCREEN_HV_EN                                                                 0x22a8
+#define regPA_SC_HP3D_TRAP_SCREEN_HV_EN_BASE_IDX                                                        1
+#define regPA_SC_HP3D_TRAP_SCREEN_H                                                                     0x22a9
+#define regPA_SC_HP3D_TRAP_SCREEN_H_BASE_IDX                                                            1
+#define regPA_SC_HP3D_TRAP_SCREEN_V                                                                     0x22aa
+#define regPA_SC_HP3D_TRAP_SCREEN_V_BASE_IDX                                                            1
+#define regPA_SC_HP3D_TRAP_SCREEN_OCCURRENCE                                                            0x22ab
+#define regPA_SC_HP3D_TRAP_SCREEN_OCCURRENCE_BASE_IDX                                                   1
+#define regPA_SC_HP3D_TRAP_SCREEN_COUNT                                                                 0x22ac
+#define regPA_SC_HP3D_TRAP_SCREEN_COUNT_BASE_IDX                                                        1
+#define regPA_SC_TRAP_SCREEN_HV_EN                                                                      0x22b0
+#define regPA_SC_TRAP_SCREEN_HV_EN_BASE_IDX                                                             1
+#define regPA_SC_TRAP_SCREEN_H                                                                          0x22b1
+#define regPA_SC_TRAP_SCREEN_H_BASE_IDX                                                                 1
+#define regPA_SC_TRAP_SCREEN_V                                                                          0x22b2
+#define regPA_SC_TRAP_SCREEN_V_BASE_IDX                                                                 1
+#define regPA_SC_TRAP_SCREEN_OCCURRENCE                                                                 0x22b3
+#define regPA_SC_TRAP_SCREEN_OCCURRENCE_BASE_IDX                                                        1
+#define regPA_SC_TRAP_SCREEN_COUNT                                                                      0x22b4
+#define regPA_SC_TRAP_SCREEN_COUNT_BASE_IDX                                                             1
+#define regSQ_THREAD_TRACE_USERDATA_0                                                                   0x2340
+#define regSQ_THREAD_TRACE_USERDATA_0_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_1                                                                   0x2341
+#define regSQ_THREAD_TRACE_USERDATA_1_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_2                                                                   0x2342
+#define regSQ_THREAD_TRACE_USERDATA_2_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_3                                                                   0x2343
+#define regSQ_THREAD_TRACE_USERDATA_3_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_4                                                                   0x2344
+#define regSQ_THREAD_TRACE_USERDATA_4_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_5                                                                   0x2345
+#define regSQ_THREAD_TRACE_USERDATA_5_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_6                                                                   0x2346
+#define regSQ_THREAD_TRACE_USERDATA_6_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_7                                                                   0x2347
+#define regSQ_THREAD_TRACE_USERDATA_7_BASE_IDX                                                          1
+#define regSQC_CACHES                                                                                   0x2348
+#define regSQC_CACHES_BASE_IDX                                                                          1
+#define regTA_CS_BC_BASE_ADDR                                                                           0x2380
+#define regTA_CS_BC_BASE_ADDR_BASE_IDX                                                                  1
+#define regTA_CS_BC_BASE_ADDR_HI                                                                        0x2381
+#define regTA_CS_BC_BASE_ADDR_HI_BASE_IDX                                                               1
+#define regDB_OCCLUSION_COUNT0_LOW                                                                      0x23c0
+#define regDB_OCCLUSION_COUNT0_LOW_BASE_IDX                                                             1
+#define regDB_OCCLUSION_COUNT0_HI                                                                       0x23c1
+#define regDB_OCCLUSION_COUNT0_HI_BASE_IDX                                                              1
+#define regDB_OCCLUSION_COUNT1_LOW                                                                      0x23c2
+#define regDB_OCCLUSION_COUNT1_LOW_BASE_IDX                                                             1
+#define regDB_OCCLUSION_COUNT1_HI                                                                       0x23c3
+#define regDB_OCCLUSION_COUNT1_HI_BASE_IDX                                                              1
+#define regDB_OCCLUSION_COUNT2_LOW                                                                      0x23c4
+#define regDB_OCCLUSION_COUNT2_LOW_BASE_IDX                                                             1
+#define regDB_OCCLUSION_COUNT2_HI                                                                       0x23c5
+#define regDB_OCCLUSION_COUNT2_HI_BASE_IDX                                                              1
+#define regDB_OCCLUSION_COUNT3_LOW                                                                      0x23c6
+#define regDB_OCCLUSION_COUNT3_LOW_BASE_IDX                                                             1
+#define regDB_OCCLUSION_COUNT3_HI                                                                       0x23c7
+#define regDB_OCCLUSION_COUNT3_HI_BASE_IDX                                                              1
+#define regSPI_CONFIG_CNTL                                                                              0x2440
+#define regSPI_CONFIG_CNTL_BASE_IDX                                                                     1
+#define regSPI_CONFIG_CNTL_1                                                                            0x2441
+#define regSPI_CONFIG_CNTL_1_BASE_IDX                                                                   1
+#define regSPI_CONFIG_CNTL_2                                                                            0x2442
+#define regSPI_CONFIG_CNTL_2_BASE_IDX                                                                   1
+#define regSPI_GS_THROTTLE_CNTL1                                                                        0x2444
+#define regSPI_GS_THROTTLE_CNTL1_BASE_IDX                                                               1
+#define regSPI_GS_THROTTLE_CNTL2                                                                        0x2445
+#define regSPI_GS_THROTTLE_CNTL2_BASE_IDX                                                               1
+#define regSPI_ATTRIBUTE_RING_BASE                                                                      0x2446
+#define regSPI_ATTRIBUTE_RING_BASE_BASE_IDX                                                             1
+#define regSPI_ATTRIBUTE_RING_SIZE                                                                      0x2447
+#define regSPI_ATTRIBUTE_RING_SIZE_BASE_IDX                                                             1
+#define regSPI_SQG_EVENT_CTL                                                                            0x2448
+#define regSPI_SQG_EVENT_CTL_BASE_IDX                                                                   1
+#define regSPI_COMPUTE_WGS_CONTROL                                                                      0x2449
+#define regSPI_COMPUTE_WGS_CONTROL_BASE_IDX                                                             1
+#define regSPI_GRP_LAUNCH_GUARANTEE_ENABLE                                                              0x244a
+#define regSPI_GRP_LAUNCH_GUARANTEE_ENABLE_BASE_IDX                                                     1
+#define regSPI_GRP_LAUNCH_GUARANTEE_CTRL                                                                0x244b
+#define regSPI_GRP_LAUNCH_GUARANTEE_CTRL_BASE_IDX                                                       1
+#define regSPI_GOG_ALLOCATION_CTRL                                                                      0x244c
+#define regSPI_GOG_ALLOCATION_CTRL_BASE_IDX                                                             1
+#define regSPI_TCP_CNTL                                                                                 0x244d
+#define regSPI_TCP_CNTL_BASE_IDX                                                                        1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_comp_wgsdec
+// base address: 0x31a00
+#define regWGS_COMPUTE_DISPATCH_INITIATOR                                                               0x2680
+#define regWGS_COMPUTE_DISPATCH_INITIATOR_BASE_IDX                                                      1
+#define regWGS_COMPUTE_DIM_X                                                                            0x2681
+#define regWGS_COMPUTE_DIM_X_BASE_IDX                                                                   1
+#define regWGS_COMPUTE_DIM_Y                                                                            0x2682
+#define regWGS_COMPUTE_DIM_Y_BASE_IDX                                                                   1
+#define regWGS_COMPUTE_DIM_Z                                                                            0x2683
+#define regWGS_COMPUTE_DIM_Z_BASE_IDX                                                                   1
+#define regWGS_COMPUTE_START_X                                                                          0x2684
+#define regWGS_COMPUTE_START_X_BASE_IDX                                                                 1
+#define regWGS_COMPUTE_START_Y                                                                          0x2685
+#define regWGS_COMPUTE_START_Y_BASE_IDX                                                                 1
+#define regWGS_COMPUTE_START_Z                                                                          0x2686
+#define regWGS_COMPUTE_START_Z_BASE_IDX                                                                 1
+#define regWGS_COMPUTE_NUM_THREAD_X                                                                     0x2687
+#define regWGS_COMPUTE_NUM_THREAD_X_BASE_IDX                                                            1
+#define regWGS_COMPUTE_NUM_THREAD_Y                                                                     0x2688
+#define regWGS_COMPUTE_NUM_THREAD_Y_BASE_IDX                                                            1
+#define regWGS_COMPUTE_NUM_THREAD_Z                                                                     0x2689
+#define regWGS_COMPUTE_NUM_THREAD_Z_BASE_IDX                                                            1
+#define regWGS_COMPUTE_PIPELINESTAT_ENABLE                                                              0x268a
+#define regWGS_COMPUTE_PIPELINESTAT_ENABLE_BASE_IDX                                                     1
+#define regWGS_COMPUTE_PERFCOUNT_ENABLE                                                                 0x268b
+#define regWGS_COMPUTE_PERFCOUNT_ENABLE_BASE_IDX                                                        1
+#define regWGS_COMPUTE_PGM_LO                                                                           0x268c
+#define regWGS_COMPUTE_PGM_LO_BASE_IDX                                                                  1
+#define regWGS_COMPUTE_PGM_HI                                                                           0x268d
+#define regWGS_COMPUTE_PGM_HI_BASE_IDX                                                                  1
+#define regWGS_COMPUTE_DISPATCH_PKT_ADDR_LO                                                             0x268e
+#define regWGS_COMPUTE_DISPATCH_PKT_ADDR_LO_BASE_IDX                                                    1
+#define regWGS_COMPUTE_DISPATCH_PKT_ADDR_HI                                                             0x268f
+#define regWGS_COMPUTE_DISPATCH_PKT_ADDR_HI_BASE_IDX                                                    1
+#define regWGS_COMPUTE_DISPATCH_SCRATCH_BASE_LO                                                         0x2690
+#define regWGS_COMPUTE_DISPATCH_SCRATCH_BASE_LO_BASE_IDX                                                1
+#define regWGS_COMPUTE_DISPATCH_SCRATCH_BASE_HI                                                         0x2691
+#define regWGS_COMPUTE_DISPATCH_SCRATCH_BASE_HI_BASE_IDX                                                1
+#define regWGS_COMPUTE_PGM_RSRC1                                                                        0x2692
+#define regWGS_COMPUTE_PGM_RSRC1_BASE_IDX                                                               1
+#define regWGS_COMPUTE_PGM_RSRC2                                                                        0x2693
+#define regWGS_COMPUTE_PGM_RSRC2_BASE_IDX                                                               1
+#define regWGS_COMPUTE_VMID                                                                             0x2694
+#define regWGS_COMPUTE_VMID_BASE_IDX                                                                    1
+#define regWGS_COMPUTE_RESOURCE_LIMITS                                                                  0x2695
+#define regWGS_COMPUTE_RESOURCE_LIMITS_BASE_IDX                                                         1
+#define regWGS_COMPUTE_DESTINATION_EN_SE0                                                               0x2696
+#define regWGS_COMPUTE_DESTINATION_EN_SE0_BASE_IDX                                                      1
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE0                                                           0x2696
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE0_BASE_IDX                                                  1
+#define regWGS_COMPUTE_DESTINATION_EN_SE1                                                               0x2697
+#define regWGS_COMPUTE_DESTINATION_EN_SE1_BASE_IDX                                                      1
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE1                                                           0x2697
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE1_BASE_IDX                                                  1
+#define regWGS_COMPUTE_TMPRING_SIZE                                                                     0x2698
+#define regWGS_COMPUTE_TMPRING_SIZE_BASE_IDX                                                            1
+#define regWGS_COMPUTE_DESTINATION_EN_SE2                                                               0x2699
+#define regWGS_COMPUTE_DESTINATION_EN_SE2_BASE_IDX                                                      1
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE2                                                           0x2699
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE2_BASE_IDX                                                  1
+#define regWGS_COMPUTE_DESTINATION_EN_SE3                                                               0x269a
+#define regWGS_COMPUTE_DESTINATION_EN_SE3_BASE_IDX                                                      1
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE3                                                           0x269a
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE3_BASE_IDX                                                  1
+#define regWGS_COMPUTE_RESTART_X                                                                        0x269b
+#define regWGS_COMPUTE_RESTART_X_BASE_IDX                                                               1
+#define regWGS_COMPUTE_RESTART_Y                                                                        0x269c
+#define regWGS_COMPUTE_RESTART_Y_BASE_IDX                                                               1
+#define regWGS_COMPUTE_RESTART_Z                                                                        0x269d
+#define regWGS_COMPUTE_RESTART_Z_BASE_IDX                                                               1
+#define regWGS_COMPUTE_THREAD_TRACE_ENABLE                                                              0x269e
+#define regWGS_COMPUTE_THREAD_TRACE_ENABLE_BASE_IDX                                                     1
+#define regWGS_COMPUTE_MISC_RESERVED                                                                    0x269f
+#define regWGS_COMPUTE_MISC_RESERVED_BASE_IDX                                                           1
+#define regWGS_COMPUTE_DISPATCH_ID                                                                      0x26a0
+#define regWGS_COMPUTE_DISPATCH_ID_BASE_IDX                                                             1
+#define regWGS_COMPUTE_THREADGROUP_ID                                                                   0x26a1
+#define regWGS_COMPUTE_THREADGROUP_ID_BASE_IDX                                                          1
+#define regWGS_COMPUTE_REQ_CTRL                                                                         0x26a2
+#define regWGS_COMPUTE_REQ_CTRL_BASE_IDX                                                                1
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE8                                                           0x26a3
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE8_BASE_IDX                                                  1
+#define regWGS_COMPUTE_USER_ACCUM_0                                                                     0x26a4
+#define regWGS_COMPUTE_USER_ACCUM_0_BASE_IDX                                                            1
+#define regWGS_COMPUTE_USER_ACCUM_1                                                                     0x26a5
+#define regWGS_COMPUTE_USER_ACCUM_1_BASE_IDX                                                            1
+#define regWGS_COMPUTE_USER_ACCUM_2                                                                     0x26a6
+#define regWGS_COMPUTE_USER_ACCUM_2_BASE_IDX                                                            1
+#define regWGS_COMPUTE_USER_ACCUM_3                                                                     0x26a7
+#define regWGS_COMPUTE_USER_ACCUM_3_BASE_IDX                                                            1
+#define regWGS_COMPUTE_PGM_RSRC3                                                                        0x26a8
+#define regWGS_COMPUTE_PGM_RSRC3_BASE_IDX                                                               1
+#define regWGS_COMPUTE_DDID_INDEX                                                                       0x26a9
+#define regWGS_COMPUTE_DDID_INDEX_BASE_IDX                                                              1
+#define regWGS_COMPUTE_SHADER_CHKSUM                                                                    0x26aa
+#define regWGS_COMPUTE_SHADER_CHKSUM_BASE_IDX                                                           1
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE4                                                           0x26ab
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE4_BASE_IDX                                                  1
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE5                                                           0x26ac
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE5_BASE_IDX                                                  1
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE6                                                           0x26ad
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE6_BASE_IDX                                                  1
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE7                                                           0x26ae
+#define regWGS_COMPUTE_STATIC_THREAD_MGMT_SE7_BASE_IDX                                                  1
+#define regWGS_COMPUTE_DISPATCH_INTERLEAVE                                                              0x26af
+#define regWGS_COMPUTE_DISPATCH_INTERLEAVE_BASE_IDX                                                     1
+#define regWGS_COMPUTE_RELAUNCH                                                                         0x26b0
+#define regWGS_COMPUTE_RELAUNCH_BASE_IDX                                                                1
+#define regWGS_COMPUTE_WAVE_RESTORE_ADDR_LO                                                             0x26b1
+#define regWGS_COMPUTE_WAVE_RESTORE_ADDR_LO_BASE_IDX                                                    1
+#define regWGS_COMPUTE_WAVE_RESTORE_ADDR_HI                                                             0x26b2
+#define regWGS_COMPUTE_WAVE_RESTORE_ADDR_HI_BASE_IDX                                                    1
+#define regWGS_COMPUTE_RELAUNCH2                                                                        0x26b3
+#define regWGS_COMPUTE_RELAUNCH2_BASE_IDX                                                               1
+#define regWGS_COMPUTE_PRESCALED_DIM_X                                                                  0x26b5
+#define regWGS_COMPUTE_PRESCALED_DIM_X_BASE_IDX                                                         1
+#define regWGS_COMPUTE_PRESCALED_DIM_Y                                                                  0x26b6
+#define regWGS_COMPUTE_PRESCALED_DIM_Y_BASE_IDX                                                         1
+#define regWGS_COMPUTE_PRESCALED_DIM_Z                                                                  0x26b7
+#define regWGS_COMPUTE_PRESCALED_DIM_Z_BASE_IDX                                                         1
+#define regWGS_COMPUTE_USER_DATA_0                                                                      0x26c0
+#define regWGS_COMPUTE_USER_DATA_0_BASE_IDX                                                             1
+#define regWGS_COMPUTE_USER_DATA_1                                                                      0x26c1
+#define regWGS_COMPUTE_USER_DATA_1_BASE_IDX                                                             1
+#define regWGS_COMPUTE_USER_DATA_2                                                                      0x26c2
+#define regWGS_COMPUTE_USER_DATA_2_BASE_IDX                                                             1
+#define regWGS_COMPUTE_USER_DATA_3                                                                      0x26c3
+#define regWGS_COMPUTE_USER_DATA_3_BASE_IDX                                                             1
+#define regWGS_COMPUTE_USER_DATA_4                                                                      0x26c4
+#define regWGS_COMPUTE_USER_DATA_4_BASE_IDX                                                             1
+#define regWGS_COMPUTE_USER_DATA_5                                                                      0x26c5
+#define regWGS_COMPUTE_USER_DATA_5_BASE_IDX                                                             1
+#define regWGS_COMPUTE_USER_DATA_6                                                                      0x26c6
+#define regWGS_COMPUTE_USER_DATA_6_BASE_IDX                                                             1
+#define regWGS_COMPUTE_USER_DATA_7                                                                      0x26c7
+#define regWGS_COMPUTE_USER_DATA_7_BASE_IDX                                                             1
+#define regWGS_COMPUTE_USER_DATA_8                                                                      0x26c8
+#define regWGS_COMPUTE_USER_DATA_8_BASE_IDX                                                             1
+#define regWGS_COMPUTE_USER_DATA_9                                                                      0x26c9
+#define regWGS_COMPUTE_USER_DATA_9_BASE_IDX                                                             1
+#define regWGS_COMPUTE_USER_DATA_10                                                                     0x26ca
+#define regWGS_COMPUTE_USER_DATA_10_BASE_IDX                                                            1
+#define regWGS_COMPUTE_USER_DATA_11                                                                     0x26cb
+#define regWGS_COMPUTE_USER_DATA_11_BASE_IDX                                                            1
+#define regWGS_COMPUTE_USER_DATA_12                                                                     0x26cc
+#define regWGS_COMPUTE_USER_DATA_12_BASE_IDX                                                            1
+#define regWGS_COMPUTE_USER_DATA_13                                                                     0x26cd
+#define regWGS_COMPUTE_USER_DATA_13_BASE_IDX                                                            1
+#define regWGS_COMPUTE_USER_DATA_14                                                                     0x26ce
+#define regWGS_COMPUTE_USER_DATA_14_BASE_IDX                                                            1
+#define regWGS_COMPUTE_USER_DATA_15                                                                     0x26cf
+#define regWGS_COMPUTE_USER_DATA_15_BASE_IDX                                                            1
+#define regWGS_COMPUTE_DISPATCH_TUNNEL                                                                  0x26fd
+#define regWGS_COMPUTE_DISPATCH_TUNNEL_BASE_IDX                                                         1
+#define regWGS_COMPUTE_DISPATCH_END                                                                     0x26fe
+#define regWGS_COMPUTE_DISPATCH_END_BASE_IDX                                                            1
+#define regWGS_COMPUTE_NOWHERE                                                                          0x26ff
+#define regWGS_COMPUTE_NOWHERE_BASE_IDX                                                                 1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_wgsdec
+// base address: 0x31c00
+#define regWGS_MGCG_SYNC_CNTL                                                                           0x2708
+#define regWGS_MGCG_SYNC_CNTL_BASE_IDX                                                                  1
+#define regWGS_INT_INFO                                                                                 0x2709
+#define regWGS_INT_INFO_BASE_IDX                                                                        1
+#define regWGS_INT_CNTXID_LO                                                                            0x270a
+#define regWGS_INT_CNTXID_LO_BASE_IDX                                                                   1
+#define regWGS_INT_CNTXID_HI                                                                            0x270b
+#define regWGS_INT_CNTXID_HI_BASE_IDX                                                                   1
+#define regWGS_ERROR                                                                                    0x270c
+#define regWGS_ERROR_BASE_IDX                                                                           1
+#define regWGS_ENDIAN_SWAP                                                                              0x270d
+#define regWGS_ENDIAN_SWAP_BASE_IDX                                                                     1
+#define regWGS_GEN_BASE_LO                                                                              0x2710
+#define regWGS_GEN_BASE_LO_BASE_IDX                                                                     1
+#define regWGS_GEN_BASE_HI                                                                              0x2711
+#define regWGS_GEN_BASE_HI_BASE_IDX                                                                     1
+#define regCGTT_WGS_CLK_CTRL                                                                            0x2712
+#define regCGTT_WGS_CLK_CTRL_BASE_IDX                                                                   1
+#define regWGS_ME1_UCODE_ADDR                                                                           0x2713
+#define regWGS_ME1_UCODE_ADDR_BASE_IDX                                                                  1
+#define regWGS_ME1_UCODE_DATA                                                                           0x2714
+#define regWGS_ME1_UCODE_DATA_BASE_IDX                                                                  1
+#define regWGS_ME1_UCODE_CHKSUM                                                                         0x2717
+#define regWGS_ME1_UCODE_CHKSUM_BASE_IDX                                                                1
+#define regWGS_SUSPEND_CTX_SAVE_BASE_ADDR_LO                                                            0x2719
+#define regWGS_SUSPEND_CTX_SAVE_BASE_ADDR_LO_BASE_IDX                                                   1
+#define regWGS_SUSPEND_CTX_SAVE_BASE_ADDR_HI                                                            0x271a
+#define regWGS_SUSPEND_CTX_SAVE_BASE_ADDR_HI_BASE_IDX                                                   1
+#define regWGS_SUSPEND_CTX_SAVE_CONTROL                                                                 0x271b
+#define regWGS_SUSPEND_CTX_SAVE_CONTROL_BASE_IDX                                                        1
+#define regWGS_SUSPEND_CNTL_STACK_OFFSET                                                                0x271c
+#define regWGS_SUSPEND_CNTL_STACK_OFFSET_BASE_IDX                                                       1
+#define regWGS_SUSPEND_CNTL_STACK_SIZE                                                                  0x271d
+#define regWGS_SUSPEND_CNTL_STACK_SIZE_BASE_IDX                                                         1
+#define regWGS_SUSPEND_WG_STATE_OFFSET                                                                  0x271e
+#define regWGS_SUSPEND_WG_STATE_OFFSET_BASE_IDX                                                         1
+#define regWGS_SUSPEND_CTX_SAVE_SIZE                                                                    0x271f
+#define regWGS_SUSPEND_CTX_SAVE_SIZE_BASE_IDX                                                           1
+#define regWGS_OS_PIPES                                                                                 0x2720
+#define regWGS_OS_PIPES_BASE_IDX                                                                        1
+#define regWGS_SUSPEND_RESUME_REQ                                                                       0x2721
+#define regWGS_SUSPEND_RESUME_REQ_BASE_IDX                                                              1
+#define regWGS_IQ_WAIT_TIME3                                                                            0x2723
+#define regWGS_IQ_WAIT_TIME3_BASE_IDX                                                                   1
+#define regWGS_DDID_BASE_ADDR_LO                                                                        0x2724
+#define regWGS_DDID_BASE_ADDR_LO_BASE_IDX                                                               1
+#define regWGS_DDID_BASE_ADDR_HI                                                                        0x2725
+#define regWGS_DDID_BASE_ADDR_HI_BASE_IDX                                                               1
+#define regWGS_DDID_CNTL                                                                                0x2726
+#define regWGS_DDID_CNTL_BASE_IDX                                                                       1
+#define regWGS_RS64_PRGRM_CNTR_START                                                                    0x2727
+#define regWGS_RS64_PRGRM_CNTR_START_BASE_IDX                                                           1
+#define regWGS_MTVEC_LO                                                                                 0x2728
+#define regWGS_MTVEC_LO_BASE_IDX                                                                        1
+#define regWGS_MTVEC_HI                                                                                 0x2729
+#define regWGS_MTVEC_HI_BASE_IDX                                                                        1
+#define regWGS_RS64_CNTL                                                                                0x272b
+#define regWGS_RS64_CNTL_BASE_IDX                                                                       1
+#define regWGS_MIE_LO                                                                                   0x272c
+#define regWGS_MIE_LO_BASE_IDX                                                                          1
+#define regWGS_MIE_HI                                                                                   0x272d
+#define regWGS_MIE_HI_BASE_IDX                                                                          1
+#define regWGS_RS64_INTERRUPT                                                                           0x272e
+#define regWGS_RS64_INTERRUPT_BASE_IDX                                                                  1
+#define regWGS_RS64_INSTR_PNTR                                                                          0x272f
+#define regWGS_RS64_INSTR_PNTR_BASE_IDX                                                                 1
+#define regWGS_MIP_LO                                                                                   0x2730
+#define regWGS_MIP_LO_BASE_IDX                                                                          1
+#define regWGS_MIP_HI                                                                                   0x2731
+#define regWGS_MIP_HI_BASE_IDX                                                                          1
+#define regWGS_DC_BASE_CNTL                                                                             0x2732
+#define regWGS_DC_BASE_CNTL_BASE_IDX                                                                    1
+#define regWGS_DC_OP_CNTL                                                                               0x2733
+#define regWGS_DC_OP_CNTL_BASE_IDX                                                                      1
+#define regWGS_MTIMECMP_LO                                                                              0x2734
+#define regWGS_MTIMECMP_LO_BASE_IDX                                                                     1
+#define regWGS_MTIMECMP_HI                                                                              0x2735
+#define regWGS_MTIMECMP_HI_BASE_IDX                                                                     1
+#define regWGS_GP0_LO                                                                                   0x2737
+#define regWGS_GP0_LO_BASE_IDX                                                                          1
+#define regWGS_GP0_HI                                                                                   0x2738
+#define regWGS_GP0_HI_BASE_IDX                                                                          1
+#define regWGS_GP1_LO                                                                                   0x2739
+#define regWGS_GP1_LO_BASE_IDX                                                                          1
+#define regWGS_GP1_HI                                                                                   0x273a
+#define regWGS_GP1_HI_BASE_IDX                                                                          1
+#define regWGS_GP2_LO                                                                                   0x273b
+#define regWGS_GP2_LO_BASE_IDX                                                                          1
+#define regWGS_GP2_HI                                                                                   0x273c
+#define regWGS_GP2_HI_BASE_IDX                                                                          1
+#define regWGS_GP3_LO                                                                                   0x273d
+#define regWGS_GP3_LO_BASE_IDX                                                                          1
+#define regWGS_GP3_HI                                                                                   0x273e
+#define regWGS_GP3_HI_BASE_IDX                                                                          1
+#define regWGS_GP4_LO                                                                                   0x273f
+#define regWGS_GP4_LO_BASE_IDX                                                                          1
+#define regWGS_GP4_HI                                                                                   0x2740
+#define regWGS_GP4_HI_BASE_IDX                                                                          1
+#define regWGS_GP5_LO                                                                                   0x2741
+#define regWGS_GP5_LO_BASE_IDX                                                                          1
+#define regWGS_GP5_HI                                                                                   0x2742
+#define regWGS_GP5_HI_BASE_IDX                                                                          1
+#define regWGS_GP6_LO                                                                                   0x2743
+#define regWGS_GP6_LO_BASE_IDX                                                                          1
+#define regWGS_GP6_HI                                                                                   0x2744
+#define regWGS_GP6_HI_BASE_IDX                                                                          1
+#define regWGS_GP7_LO                                                                                   0x2745
+#define regWGS_GP7_LO_BASE_IDX                                                                          1
+#define regWGS_GP7_HI                                                                                   0x2746
+#define regWGS_GP7_HI_BASE_IDX                                                                          1
+#define regWGS_GP8_LO                                                                                   0x2747
+#define regWGS_GP8_LO_BASE_IDX                                                                          1
+#define regWGS_GP8_HI                                                                                   0x2748
+#define regWGS_GP8_HI_BASE_IDX                                                                          1
+#define regWGS_GP9_LO                                                                                   0x2749
+#define regWGS_GP9_LO_BASE_IDX                                                                          1
+#define regWGS_GP9_HI                                                                                   0x274a
+#define regWGS_GP9_HI_BASE_IDX                                                                          1
+#define regWGS_DM_INDEX_ADDR                                                                            0x274b
+#define regWGS_DM_INDEX_ADDR_BASE_IDX                                                                   1
+#define regWGS_DM_INDEX_DATA                                                                            0x274c
+#define regWGS_DM_INDEX_DATA_BASE_IDX                                                                   1
+#define regWGS_LOCAL_BASE0_LO                                                                           0x274e
+#define regWGS_LOCAL_BASE0_LO_BASE_IDX                                                                  1
+#define regWGS_LOCAL_BASE0_HI                                                                           0x274f
+#define regWGS_LOCAL_BASE0_HI_BASE_IDX                                                                  1
+#define regWGS_LOCAL_MASK0_LO                                                                           0x2750
+#define regWGS_LOCAL_MASK0_LO_BASE_IDX                                                                  1
+#define regWGS_LOCAL_MASK0_HI                                                                           0x2751
+#define regWGS_LOCAL_MASK0_HI_BASE_IDX                                                                  1
+#define regWGS_LOCAL_APERTURE                                                                           0x2752
+#define regWGS_LOCAL_APERTURE_BASE_IDX                                                                  1
+#define regWGS_LOCAL_INSTR_BASE_LO                                                                      0x2753
+#define regWGS_LOCAL_INSTR_BASE_LO_BASE_IDX                                                             1
+#define regWGS_LOCAL_INSTR_BASE_HI                                                                      0x2754
+#define regWGS_LOCAL_INSTR_BASE_HI_BASE_IDX                                                             1
+#define regWGS_LOCAL_INSTR_MASK_LO                                                                      0x2755
+#define regWGS_LOCAL_INSTR_MASK_LO_BASE_IDX                                                             1
+#define regWGS_LOCAL_INSTR_MASK_HI                                                                      0x2756
+#define regWGS_LOCAL_INSTR_MASK_HI_BASE_IDX                                                             1
+#define regWGS_LOCAL_INSTR_APERTURE                                                                     0x2757
+#define regWGS_LOCAL_INSTR_APERTURE_BASE_IDX                                                            1
+#define regWGS_LOCAL_SCRATCH_APERTURE                                                                   0x2758
+#define regWGS_LOCAL_SCRATCH_APERTURE_BASE_IDX                                                          1
+#define regWGS_LOCAL_SCRATCH_BASE_LO                                                                    0x2759
+#define regWGS_LOCAL_SCRATCH_BASE_LO_BASE_IDX                                                           1
+#define regWGS_LOCAL_SCRATCH_BASE_HI                                                                    0x275a
+#define regWGS_LOCAL_SCRATCH_BASE_HI_BASE_IDX                                                           1
+#define regWGS_RS64_PERFCOUNT_CNTL                                                                      0x275b
+#define regWGS_RS64_PERFCOUNT_CNTL_BASE_IDX                                                             1
+#define regWGS_RS64_PENDING_INTERRUPT                                                                   0x275c
+#define regWGS_RS64_PENDING_INTERRUPT_BASE_IDX                                                          1
+#define regWGS_RS64_EXCEPTION_STATUS                                                                    0x275e
+#define regWGS_RS64_EXCEPTION_STATUS_BASE_IDX                                                           1
+#define regWGS_RS64_PRGRM_CNTR_START_HI                                                                 0x275f
+#define regWGS_RS64_PRGRM_CNTR_START_HI_BASE_IDX                                                        1
+#define regWGS_DC_BASE_LO                                                                               0x2799
+#define regWGS_DC_BASE_LO_BASE_IDX                                                                      1
+#define regWGS_MDBASE_LO                                                                                0x2799
+#define regWGS_MDBASE_LO_BASE_IDX                                                                       1
+#define regWGS_DC_BASE_HI                                                                               0x279a
+#define regWGS_DC_BASE_HI_BASE_IDX                                                                      1
+#define regWGS_MDBASE_HI                                                                                0x279a
+#define regWGS_MDBASE_HI_BASE_IDX                                                                       1
+#define regWGS_MIBOUND_LO                                                                               0x279b
+#define regWGS_MIBOUND_LO_BASE_IDX                                                                      1
+#define regWGS_MIBOUND_HI                                                                               0x279c
+#define regWGS_MIBOUND_HI_BASE_IDX                                                                      1
+#define regWGS_MDBOUND_LO                                                                               0x279d
+#define regWGS_MDBOUND_LO_BASE_IDX                                                                      1
+#define regWGS_MDBOUND_HI                                                                               0x279e
+#define regWGS_MDBOUND_HI_BASE_IDX                                                                      1
+#define regWGS_IC_OP_CNTL                                                                               0x279f
+#define regWGS_IC_OP_CNTL_BASE_IDX                                                                      1
+#define regWGS_INT_STATUS                                                                               0x27a3
+#define regWGS_INT_STATUS_BASE_IDX                                                                      1
+#define regWGS_IH_FREE_COUNT                                                                            0x27a4
+#define regWGS_IH_FREE_COUNT_BASE_IDX                                                                   1
+#define regWGS_ME1_PIPE_PRIORITY_CNTS                                                                   0x27a6
+#define regWGS_ME1_PIPE_PRIORITY_CNTS_BASE_IDX                                                          1
+#define regWGS_ME1_PIPE0_PRIORITY                                                                       0x27a7
+#define regWGS_ME1_PIPE0_PRIORITY_BASE_IDX                                                              1
+#define regWGS_ME1_PIPE1_PRIORITY                                                                       0x27a8
+#define regWGS_ME1_PIPE1_PRIORITY_BASE_IDX                                                              1
+#define regWGS_ME1_PIPE2_PRIORITY                                                                       0x27a9
+#define regWGS_ME1_PIPE2_PRIORITY_BASE_IDX                                                              1
+#define regWGS_ME1_PIPE3_PRIORITY                                                                       0x27aa
+#define regWGS_ME1_PIPE3_PRIORITY_BASE_IDX                                                              1
+#define regWGS_IQ_WAIT_TIME1                                                                            0x27b4
+#define regWGS_IQ_WAIT_TIME1_BASE_IDX                                                                   1
+#define regWGS_IQ_WAIT_TIME2                                                                            0x27b5
+#define regWGS_IQ_WAIT_TIME2_BASE_IDX                                                                   1
+#define regWGS_IC_BASE_LO                                                                               0x27b7
+#define regWGS_IC_BASE_LO_BASE_IDX                                                                      1
+#define regWGS_IC_BASE_HI                                                                               0x27b8
+#define regWGS_IC_BASE_HI_BASE_IDX                                                                      1
+#define regWGS_IC_BASE_CNTL                                                                             0x27b9
+#define regWGS_IC_BASE_CNTL_BASE_IDX                                                                    1
+#define regWGS_UCODE_VERS                                                                               0x27bb
+#define regWGS_UCODE_VERS_BASE_IDX                                                                      1
+#define regWGS_BUSY_HYSTERESIS                                                                          0x27bd
+#define regWGS_BUSY_HYSTERESIS_BASE_IDX                                                                 1
+#define regWGS_STATUS                                                                                   0x27d0
+#define regWGS_STATUS_BASE_IDX                                                                          1
+#define regWGS_BUSY_STAT                                                                                0x27d1
+#define regWGS_BUSY_STAT_BASE_IDX                                                                       1
+#define regWGS_STALLED_STAT1                                                                            0x27d2
+#define regWGS_STALLED_STAT1_BASE_IDX                                                                   1
+#define regWGS_BUSY_STAT2                                                                               0x27d3
+#define regWGS_BUSY_STAT2_BASE_IDX                                                                      1
+#define regWGS_ME1_HEADER_DUMP                                                                          0x27d4
+#define regWGS_ME1_HEADER_DUMP_BASE_IDX                                                                 1
+#define regWGS_SCRATCH_INDEX                                                                            0x27d6
+#define regWGS_SCRATCH_INDEX_BASE_IDX                                                                   1
+#define regWGS_SCRATCH_DATA                                                                             0x27d7
+#define regWGS_SCRATCH_DATA_BASE_IDX                                                                    1
+#define regWGS_LATENCY_STATS_SELECT                                                                     0x27d8
+#define regWGS_LATENCY_STATS_SELECT_BASE_IDX                                                            1
+#define regWGS_LATENCY_STATS_DATA                                                                       0x27d9
+#define regWGS_LATENCY_STATS_DATA_BASE_IDX                                                              1
+#define regWGS_TC_PERF_COUNTER_WINDOW_SELECT                                                            0x27da
+#define regWGS_TC_PERF_COUNTER_WINDOW_SELECT_BASE_IDX                                                   1
+#define regWGS_DATA1                                                                                    0x27db
+#define regWGS_DATA1_BASE_IDX                                                                           1
+#define regWGS_DATA2                                                                                    0x27dc
+#define regWGS_DATA2_BASE_IDX                                                                           1
+#define regWGS_DATA3                                                                                    0x27dd
+#define regWGS_DATA3_BASE_IDX                                                                           1
+#define regWGS_DATA4                                                                                    0x27de
+#define regWGS_DATA4_BASE_IDX                                                                           1
+#define regWGS_DATA5                                                                                    0x27df
+#define regWGS_DATA5_BASE_IDX                                                                           1
+#define regWGS_DATA6                                                                                    0x27e0
+#define regWGS_DATA6_BASE_IDX                                                                           1
+#define regWGS_DATA7                                                                                    0x27e1
+#define regWGS_DATA7_BASE_IDX                                                                           1
+#define regWGS_DATA8                                                                                    0x27e2
+#define regWGS_DATA8_BASE_IDX                                                                           1
+#define regWGS_DATA9                                                                                    0x27e3
+#define regWGS_DATA9_BASE_IDX                                                                           1
+#define regWGS_DATA10                                                                                   0x27e4
+#define regWGS_DATA10_BASE_IDX                                                                          1
+#define regWGS_DATA11                                                                                   0x27e5
+#define regWGS_DATA11_BASE_IDX                                                                          1
+#define regWGS_DATA12                                                                                   0x27e6
+#define regWGS_DATA12_BASE_IDX                                                                          1
+#define regWGS_DATA13                                                                                   0x27e7
+#define regWGS_DATA13_BASE_IDX                                                                          1
+#define regWGS_DATA14                                                                                   0x27e8
+#define regWGS_DATA14_BASE_IDX                                                                          1
+#define regWGS_DATA15                                                                                   0x27e9
+#define regWGS_DATA15_BASE_IDX                                                                          1
+#define regWGS_DATA16                                                                                   0x27ea
+#define regWGS_DATA16_BASE_IDX                                                                          1
+#define regWGS_METADATA_BASE_ADDR_LO                                                                    0x27eb
+#define regWGS_METADATA_BASE_ADDR_LO_BASE_IDX                                                           1
+#define regWGS_METADATA_BASE_ADDR_HI                                                                    0x27ec
+#define regWGS_METADATA_BASE_ADDR_HI_BASE_IDX                                                           1
+#define regWGS_METADATA_CNTL                                                                            0x27ed
+#define regWGS_METADATA_CNTL_BASE_IDX                                                                   1
+#define regWGS_IQ_TIMER                                                                                 0x27ee
+#define regWGS_IQ_TIMER_BASE_IDX                                                                        1
+#define regWGS_HALT_HYST_COUNT                                                                          0x27ef
+#define regWGS_HALT_HYST_COUNT_BASE_IDX                                                                 1
+#define regWGS_IQ_TIMER_MESSAGE                                                                         0x27f0
+#define regWGS_IQ_TIMER_MESSAGE_BASE_IDX                                                                1
+#define regWGS_RS64_THREAD_CNTL                                                                         0x27f6
+#define regWGS_RS64_THREAD_CNTL_BASE_IDX                                                                1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_gl1dec
+// base address: 0x33400
+#define regGL1_ARB_CTRL                                                                                 0x2d00
+#define regGL1_ARB_CTRL_BASE_IDX                                                                        1
+#define regGL1_DRAM_BURST_MASK                                                                          0x2d02
+#define regGL1_DRAM_BURST_MASK_BASE_IDX                                                                 1
+#define regGL1_ARB_STATUS                                                                               0x2d03
+#define regGL1_ARB_STATUS_BASE_IDX                                                                      1
+#define regGL1_DRAM_BURST_CTRL                                                                          0x2d04
+#define regGL1_DRAM_BURST_CTRL_BASE_IDX                                                                 1
+#define regGL1I_GL1R_REP_FGCG_OVERRIDE                                                                  0x2d05
+#define regGL1I_GL1R_REP_FGCG_OVERRIDE_BASE_IDX                                                         1
+#define regGL1A_GL1C_CREDITS                                                                            0x2d08
+#define regGL1A_GL1C_CREDITS_BASE_IDX                                                                   1
+#define regGL1A_CLIENT_FREE_DELAY                                                                       0x2d09
+#define regGL1A_CLIENT_FREE_DELAY_BASE_IDX                                                              1
+#define regGL1X_ARB_CTRL                                                                                0x2d20
+#define regGL1X_ARB_CTRL_BASE_IDX                                                                       1
+#define regGL1X_DRAM_BURST_MASK                                                                         0x2d22
+#define regGL1X_DRAM_BURST_MASK_BASE_IDX                                                                1
+#define regGL1X_ARB_STATUS                                                                              0x2d23
+#define regGL1X_ARB_STATUS_BASE_IDX                                                                     1
+#define regGL1X_DRAM_BURST_CTRL                                                                         0x2d24
+#define regGL1X_DRAM_BURST_CTRL_BASE_IDX                                                                1
+#define regGL1XI_GL1XR_REP_FGCG_OVERRIDE                                                                0x2d25
+#define regGL1XI_GL1XR_REP_FGCG_OVERRIDE_BASE_IDX                                                       1
+#define regGL1XA_GL1XC_CREDITS                                                                          0x2d28
+#define regGL1XA_GL1XC_CREDITS_BASE_IDX                                                                 1
+#define regGL1XA_CLIENT_FREE_DELAY                                                                      0x2d29
+#define regGL1XA_CLIENT_FREE_DELAY_BASE_IDX                                                             1
+#define regGL1XA_COMPRESSION_MODE                                                                       0x2d2a
+#define regGL1XA_COMPRESSION_MODE_BASE_IDX                                                              1
+#define regGL1XA_COMPRESSOR_OVERRIDE                                                                    0x2d2b
+#define regGL1XA_COMPRESSOR_OVERRIDE_BASE_IDX                                                           1
+#define regGL1C_CTRL                                                                                    0x2d40
+#define regGL1C_CTRL_BASE_IDX                                                                           1
+#define regGL1C_STATUS                                                                                  0x2d41
+#define regGL1C_STATUS_BASE_IDX                                                                         1
+#define regGL1C_CTRL2                                                                                   0x2d46
+#define regGL1C_CTRL2_BASE_IDX                                                                          1
+#define regGL1XC_CTRL                                                                                   0x2d47
+#define regGL1XC_CTRL_BASE_IDX                                                                          1
+#define regGL1XC_STATUS                                                                                 0x2d48
+#define regGL1XC_STATUS_BASE_IDX                                                                        1
+#define regGL1XC_UTCL0_CNTL1                                                                            0x2d49
+#define regGL1XC_UTCL0_CNTL1_BASE_IDX                                                                   1
+#define regGL1XC_UTCL0_CNTL2                                                                            0x2d4a
+#define regGL1XC_UTCL0_CNTL2_BASE_IDX                                                                   1
+#define regGL1XC_UTCL0_STATUS                                                                           0x2d4b
+#define regGL1XC_UTCL0_STATUS_BASE_IDX                                                                  1
+#define regGL1XC_UTCL0_RETRY                                                                            0x2d4c
+#define regGL1XC_UTCL0_RETRY_BASE_IDX                                                                   1
+#define regGL1XC_CTRL2                                                                                  0x2d4d
+#define regGL1XC_CTRL2_BASE_IDX                                                                         1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pfonly_secacdec
+// base address: 0x2f1c0
+#define regSE_CAC_CTRL_1                                                                                0x1c70
+#define regSE_CAC_CTRL_1_BASE_IDX                                                                       1
+#define regSE_CAC_CTRL_2                                                                                0x1c71
+#define regSE_CAC_CTRL_2_BASE_IDX                                                                       1
+#define regSE_CAC_SOFT_CTRL                                                                             0x1c72
+#define regSE_CAC_SOFT_CTRL_BASE_IDX                                                                    1
+#define regSE_CAC_OVR_VAL_LOWER                                                                         0x1c74
+#define regSE_CAC_OVR_VAL_LOWER_BASE_IDX                                                                1
+#define regSE_CAC_OVR_VAL_UPPER                                                                         0x1c75
+#define regSE_CAC_OVR_VAL_UPPER_BASE_IDX                                                                1
+#define regSE_CAC_WINDOW_AGGR_VALUE_LO                                                                  0x1c76
+#define regSE_CAC_WINDOW_AGGR_VALUE_LO_BASE_IDX                                                         1
+#define regSE_CAC_WINDOW_AGGR_VALUE_HI                                                                  0x1c77
+#define regSE_CAC_WINDOW_AGGR_VALUE_HI_BASE_IDX                                                         1
+#define regSE_CAC_WINDOW_GFXCLK_CYCLE                                                                   0x1c78
+#define regSE_CAC_WINDOW_GFXCLK_CYCLE_BASE_IDX                                                          1
+#define regDIDT_EDC_CTRL                                                                                0x1c7c
+#define regDIDT_EDC_CTRL_BASE_IDX                                                                       1
+#define regDIDT_EDC_THROTTLE_CTRL                                                                       0x1c7d
+#define regDIDT_EDC_THROTTLE_CTRL_BASE_IDX                                                              1
+#define regDIDT_EDC_THRESHOLD                                                                           0x1c7e
+#define regDIDT_EDC_THRESHOLD_BASE_IDX                                                                  1
+#define regDIDT_EDC_STRETCH_THRESHOLD                                                                   0x1c7f
+#define regDIDT_EDC_STRETCH_THRESHOLD_BASE_IDX                                                          1
+#define regDIDT_EDC_CLK_COUNTER_THRESHOLD                                                               0x1c80
+#define regDIDT_EDC_CLK_COUNTER_THRESHOLD_BASE_IDX                                                      1
+#define regDIDT_EDC_STALL_PATTERN_1_2                                                                   0x1c81
+#define regDIDT_EDC_STALL_PATTERN_1_2_BASE_IDX                                                          1
+#define regDIDT_EDC_STALL_PATTERN_3_4                                                                   0x1c82
+#define regDIDT_EDC_STALL_PATTERN_3_4_BASE_IDX                                                          1
+#define regDIDT_EDC_STALL_PATTERN_5_6                                                                   0x1c83
+#define regDIDT_EDC_STALL_PATTERN_5_6_BASE_IDX                                                          1
+#define regDIDT_EDC_STALL_PATTERN_7                                                                     0x1c84
+#define regDIDT_EDC_STALL_PATTERN_7_BASE_IDX                                                            1
+#define regDIDT_EDC_STATUS                                                                              0x1c85
+#define regDIDT_EDC_STATUS_BASE_IDX                                                                     1
+#define regDIDT_EDC_OVERFLOW                                                                            0x1c86
+#define regDIDT_EDC_OVERFLOW_BASE_IDX                                                                   1
+#define regDIDT_EDC_ROLLING_POWER_DELTA                                                                 0x1c87
+#define regDIDT_EDC_ROLLING_POWER_DELTA_BASE_IDX                                                        1
+#define regDIDT_EDC_AVG_POWER_DELTA_HI                                                                  0x1c88
+#define regDIDT_EDC_AVG_POWER_DELTA_HI_BASE_IDX                                                         1
+#define regDIDT_EDC_AVG_POWER_DELTA_LO                                                                  0x1c89
+#define regDIDT_EDC_AVG_POWER_DELTA_LO_BASE_IDX                                                         1
+#define regDIDT_EDC_STALL_PERF_COUNTER                                                                  0x1c8a
+#define regDIDT_EDC_STALL_PERF_COUNTER_BASE_IDX                                                         1
+#define regDIDT_THRESHOLD                                                                               0x1c8c
+#define regDIDT_THRESHOLD_BASE_IDX                                                                      1
+#define regSE_CAC_WEIGHT_LDS1_0                                                                         0x1c93
+#define regSE_CAC_WEIGHT_LDS1_0_BASE_IDX                                                                1
+#define regSE_CAC_WEIGHT_LDS1_1                                                                         0x1c94
+#define regSE_CAC_WEIGHT_LDS1_1_BASE_IDX                                                                1
+#define regSE_CAC_WEIGHT_LDS1_2                                                                         0x1c95
+#define regSE_CAC_WEIGHT_LDS1_2_BASE_IDX                                                                1
+#define regSE_CAC_WEIGHT_LDS1_3                                                                         0x1c96
+#define regSE_CAC_WEIGHT_LDS1_3_BASE_IDX                                                                1
+#define regSE_CAC_WEIGHT_TCP_0                                                                          0x1c9d
+#define regSE_CAC_WEIGHT_TCP_0_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_TCP_1                                                                          0x1c9e
+#define regSE_CAC_WEIGHT_TCP_1_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_TCP_2                                                                          0x1c9f
+#define regSE_CAC_WEIGHT_TCP_2_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_TCP_3                                                                          0x1ca0
+#define regSE_CAC_WEIGHT_TCP_3_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_SQ_0                                                                           0x1ca1
+#define regSE_CAC_WEIGHT_SQ_0_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_SQ_1                                                                           0x1ca2
+#define regSE_CAC_WEIGHT_SQ_1_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_SQ_2                                                                           0x1ca3
+#define regSE_CAC_WEIGHT_SQ_2_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_SP_0                                                                           0x1ca4
+#define regSE_CAC_WEIGHT_SP_0_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_SP_1                                                                           0x1ca5
+#define regSE_CAC_WEIGHT_SP_1_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_SP_2                                                                           0x1ca6
+#define regSE_CAC_WEIGHT_SP_2_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_LDS0_0                                                                         0x1ca7
+#define regSE_CAC_WEIGHT_LDS0_0_BASE_IDX                                                                1
+#define regSE_CAC_WEIGHT_LDS0_1                                                                         0x1ca8
+#define regSE_CAC_WEIGHT_LDS0_1_BASE_IDX                                                                1
+#define regSE_CAC_WEIGHT_LDS0_2                                                                         0x1ca9
+#define regSE_CAC_WEIGHT_LDS0_2_BASE_IDX                                                                1
+#define regSE_CAC_WEIGHT_LDS0_3                                                                         0x1caa
+#define regSE_CAC_WEIGHT_LDS0_3_BASE_IDX                                                                1
+#define regSE_CAC_WEIGHT_SQC_0                                                                          0x1cac
+#define regSE_CAC_WEIGHT_SQC_0_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_SQC_1                                                                          0x1cad
+#define regSE_CAC_WEIGHT_SQC_1_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_CU_0                                                                           0x1cae
+#define regSE_CAC_WEIGHT_CU_0_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_UTCL1_0                                                                        0x1cc3
+#define regSE_CAC_WEIGHT_UTCL1_0_BASE_IDX                                                               1
+#define regSE_CAC_WEIGHT_GL1C_0                                                                         0x1cc4
+#define regSE_CAC_WEIGHT_GL1C_0_BASE_IDX                                                                1
+#define regSE_CAC_WEIGHT_GL1C_1                                                                         0x1cc5
+#define regSE_CAC_WEIGHT_GL1C_1_BASE_IDX                                                                1
+#define regSE_CAC_WEIGHT_SPI_0                                                                          0x1cc6
+#define regSE_CAC_WEIGHT_SPI_0_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_SPI_1                                                                          0x1cc7
+#define regSE_CAC_WEIGHT_SPI_1_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_SPI_2                                                                          0x1cc8
+#define regSE_CAC_WEIGHT_SPI_2_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_TCP_4                                                                          0x1cdc
+#define regSE_CAC_WEIGHT_TCP_4_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_TCP_5                                                                          0x1cdd
+#define regSE_CAC_WEIGHT_TCP_5_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_TCP_6                                                                          0x1cde
+#define regSE_CAC_WEIGHT_TCP_6_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_TCP_7                                                                          0x1cdf
+#define regSE_CAC_WEIGHT_TCP_7_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_TCP_8                                                                          0x1ce0
+#define regSE_CAC_WEIGHT_TCP_8_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_TCP_9                                                                          0x1ce1
+#define regSE_CAC_WEIGHT_TCP_9_BASE_IDX                                                                 1
+#define regSE_CAC_IND_INDEX                                                                             0x1d2e
+#define regSE_CAC_IND_INDEX_BASE_IDX                                                                    1
+#define regSE_CAC_IND_DATA                                                                              0x1d2f
+#define regSE_CAC_IND_DATA_BASE_IDX                                                                     1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_perfddec
+// base address: 0x34000
+#define regGE2_SE_PERFCOUNTER0_LO                                                                       0x30b4
+#define regGE2_SE_PERFCOUNTER0_LO_BASE_IDX                                                              1
+#define regGE2_SE_PERFCOUNTER0_HI                                                                       0x30b5
+#define regGE2_SE_PERFCOUNTER0_HI_BASE_IDX                                                              1
+#define regGE2_SE_PERFCOUNTER1_LO                                                                       0x30b6
+#define regGE2_SE_PERFCOUNTER1_LO_BASE_IDX                                                              1
+#define regGE2_SE_PERFCOUNTER1_HI                                                                       0x30b7
+#define regGE2_SE_PERFCOUNTER1_HI_BASE_IDX                                                              1
+#define regGE2_SE_PERFCOUNTER2_LO                                                                       0x30b8
+#define regGE2_SE_PERFCOUNTER2_LO_BASE_IDX                                                              1
+#define regGE2_SE_PERFCOUNTER2_HI                                                                       0x30b9
+#define regGE2_SE_PERFCOUNTER2_HI_BASE_IDX                                                              1
+#define regGE2_SE_PERFCOUNTER3_LO                                                                       0x30ba
+#define regGE2_SE_PERFCOUNTER3_LO_BASE_IDX                                                              1
+#define regGE2_SE_PERFCOUNTER3_HI                                                                       0x30bb
+#define regGE2_SE_PERFCOUNTER3_HI_BASE_IDX                                                              1
+#define regGRBMH_PERFCOUNTER0_LO                                                                        0x30f8
+#define regGRBMH_PERFCOUNTER0_LO_BASE_IDX                                                               1
+#define regGRBMH_PERFCOUNTER0_HI                                                                        0x30f9
+#define regGRBMH_PERFCOUNTER0_HI_BASE_IDX                                                               1
+#define regGRBMH_PERFCOUNTER1_LO                                                                        0x30fa
+#define regGRBMH_PERFCOUNTER1_LO_BASE_IDX                                                               1
+#define regGRBMH_PERFCOUNTER1_HI                                                                        0x30fb
+#define regGRBMH_PERFCOUNTER1_HI_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER0_LO                                                                        0x3100
+#define regPA_SU_PERFCOUNTER0_LO_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER0_HI                                                                        0x3101
+#define regPA_SU_PERFCOUNTER0_HI_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER1_LO                                                                        0x3102
+#define regPA_SU_PERFCOUNTER1_LO_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER1_HI                                                                        0x3103
+#define regPA_SU_PERFCOUNTER1_HI_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER2_LO                                                                        0x3104
+#define regPA_SU_PERFCOUNTER2_LO_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER2_HI                                                                        0x3105
+#define regPA_SU_PERFCOUNTER2_HI_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER3_LO                                                                        0x3106
+#define regPA_SU_PERFCOUNTER3_LO_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER3_HI                                                                        0x3107
+#define regPA_SU_PERFCOUNTER3_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER0_LO                                                                        0x3140
+#define regPA_SC_PERFCOUNTER0_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER0_HI                                                                        0x3141
+#define regPA_SC_PERFCOUNTER0_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER1_LO                                                                        0x3142
+#define regPA_SC_PERFCOUNTER1_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER1_HI                                                                        0x3143
+#define regPA_SC_PERFCOUNTER1_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER2_LO                                                                        0x3144
+#define regPA_SC_PERFCOUNTER2_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER2_HI                                                                        0x3145
+#define regPA_SC_PERFCOUNTER2_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER3_LO                                                                        0x3146
+#define regPA_SC_PERFCOUNTER3_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER3_HI                                                                        0x3147
+#define regPA_SC_PERFCOUNTER3_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER4_LO                                                                        0x3148
+#define regPA_SC_PERFCOUNTER4_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER4_HI                                                                        0x3149
+#define regPA_SC_PERFCOUNTER4_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER5_LO                                                                        0x314a
+#define regPA_SC_PERFCOUNTER5_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER5_HI                                                                        0x314b
+#define regPA_SC_PERFCOUNTER5_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER6_LO                                                                        0x314c
+#define regPA_SC_PERFCOUNTER6_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER6_HI                                                                        0x314d
+#define regPA_SC_PERFCOUNTER6_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER7_LO                                                                        0x314e
+#define regPA_SC_PERFCOUNTER7_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER7_HI                                                                        0x314f
+#define regPA_SC_PERFCOUNTER7_HI_BASE_IDX                                                               1
+#define regSPI_PERFCOUNTER0_HI                                                                          0x3180
+#define regSPI_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER0_LO                                                                          0x3181
+#define regSPI_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER1_HI                                                                          0x3182
+#define regSPI_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER1_LO                                                                          0x3183
+#define regSPI_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER2_HI                                                                          0x3184
+#define regSPI_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER2_LO                                                                          0x3185
+#define regSPI_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER3_HI                                                                          0x3186
+#define regSPI_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER3_LO                                                                          0x3187
+#define regSPI_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER4_HI                                                                          0x3188
+#define regSPI_PERFCOUNTER4_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER4_LO                                                                          0x3189
+#define regSPI_PERFCOUNTER4_LO_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER5_HI                                                                          0x318a
+#define regSPI_PERFCOUNTER5_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER5_LO                                                                          0x318b
+#define regSPI_PERFCOUNTER5_LO_BASE_IDX                                                                 1
+#define regPC_PERFCOUNTER0_HI                                                                           0x318c
+#define regPC_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regPC_PERFCOUNTER0_LO                                                                           0x318d
+#define regPC_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regPC_PERFCOUNTER1_HI                                                                           0x318e
+#define regPC_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regPC_PERFCOUNTER1_LO                                                                           0x318f
+#define regPC_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regPC_PERFCOUNTER2_HI                                                                           0x3190
+#define regPC_PERFCOUNTER2_HI_BASE_IDX                                                                  1
+#define regPC_PERFCOUNTER2_LO                                                                           0x3191
+#define regPC_PERFCOUNTER2_LO_BASE_IDX                                                                  1
+#define regPC_PERFCOUNTER3_HI                                                                           0x3192
+#define regPC_PERFCOUNTER3_HI_BASE_IDX                                                                  1
+#define regPC_PERFCOUNTER3_LO                                                                           0x3193
+#define regPC_PERFCOUNTER3_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER0_LO                                                                           0x31c0
+#define regSQ_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER1_LO                                                                           0x31c2
+#define regSQ_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER2_LO                                                                           0x31c4
+#define regSQ_PERFCOUNTER2_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER3_LO                                                                           0x31c6
+#define regSQ_PERFCOUNTER3_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER4_LO                                                                           0x31c8
+#define regSQ_PERFCOUNTER4_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER5_LO                                                                           0x31ca
+#define regSQ_PERFCOUNTER5_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER6_LO                                                                           0x31cc
+#define regSQ_PERFCOUNTER6_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER7_LO                                                                           0x31ce
+#define regSQ_PERFCOUNTER7_LO_BASE_IDX                                                                  1
+#define regSQG_PERFCOUNTER0_LO                                                                          0x31e4
+#define regSQG_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER0_HI                                                                          0x31e5
+#define regSQG_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER1_LO                                                                          0x31e6
+#define regSQG_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER1_HI                                                                          0x31e7
+#define regSQG_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER2_LO                                                                          0x31e8
+#define regSQG_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER2_HI                                                                          0x31e9
+#define regSQG_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER3_LO                                                                          0x31ea
+#define regSQG_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER3_HI                                                                          0x31eb
+#define regSQG_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER4_LO                                                                          0x31ec
+#define regSQG_PERFCOUNTER4_LO_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER4_HI                                                                          0x31ed
+#define regSQG_PERFCOUNTER4_HI_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER5_LO                                                                          0x31ee
+#define regSQG_PERFCOUNTER5_LO_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER5_HI                                                                          0x31ef
+#define regSQG_PERFCOUNTER5_HI_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER6_LO                                                                          0x31f0
+#define regSQG_PERFCOUNTER6_LO_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER6_HI                                                                          0x31f1
+#define regSQG_PERFCOUNTER6_HI_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER7_LO                                                                          0x31f2
+#define regSQG_PERFCOUNTER7_LO_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER7_HI                                                                          0x31f3
+#define regSQG_PERFCOUNTER7_HI_BASE_IDX                                                                 1
+#define regSX_PERFCOUNTER0_LO                                                                           0x3240
+#define regSX_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER0_HI                                                                           0x3241
+#define regSX_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER1_LO                                                                           0x3242
+#define regSX_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER1_HI                                                                           0x3243
+#define regSX_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER2_LO                                                                           0x3244
+#define regSX_PERFCOUNTER2_LO_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER2_HI                                                                           0x3245
+#define regSX_PERFCOUNTER2_HI_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER3_LO                                                                           0x3246
+#define regSX_PERFCOUNTER3_LO_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER3_HI                                                                           0x3247
+#define regSX_PERFCOUNTER3_HI_BASE_IDX                                                                  1
+#define regTA_PERFCOUNTER0_LO                                                                           0x32c0
+#define regTA_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regTA_PERFCOUNTER0_HI                                                                           0x32c1
+#define regTA_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regTA_PERFCOUNTER1_LO                                                                           0x32c2
+#define regTA_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regTA_PERFCOUNTER1_HI                                                                           0x32c3
+#define regTA_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regTD_PERFCOUNTER0_LO                                                                           0x3300
+#define regTD_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regTD_PERFCOUNTER0_HI                                                                           0x3301
+#define regTD_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regTD_PERFCOUNTER1_LO                                                                           0x3302
+#define regTD_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regTD_PERFCOUNTER1_HI                                                                           0x3303
+#define regTD_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regTCP_PERFCOUNTER0_LO                                                                          0x3340
+#define regTCP_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER0_HI                                                                          0x3341
+#define regTCP_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER1_LO                                                                          0x3342
+#define regTCP_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER1_HI                                                                          0x3343
+#define regTCP_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER2_LO                                                                          0x3344
+#define regTCP_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER2_HI                                                                          0x3345
+#define regTCP_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER3_LO                                                                          0x3346
+#define regTCP_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER3_HI                                                                          0x3347
+#define regTCP_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER4_LO                                                                          0x3348
+#define regTCP_PERFCOUNTER4_LO_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER4_HI                                                                          0x3349
+#define regTCP_PERFCOUNTER4_HI_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER5_LO                                                                          0x334a
+#define regTCP_PERFCOUNTER5_LO_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER5_HI                                                                          0x334b
+#define regTCP_PERFCOUNTER5_HI_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER6_LO                                                                          0x334c
+#define regTCP_PERFCOUNTER6_LO_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER6_HI                                                                          0x334d
+#define regTCP_PERFCOUNTER6_HI_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER7_LO                                                                          0x334e
+#define regTCP_PERFCOUNTER7_LO_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER7_HI                                                                          0x334f
+#define regTCP_PERFCOUNTER7_HI_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER_FILTER                                                                       0x3350
+#define regTCP_PERFCOUNTER_FILTER_BASE_IDX                                                              1
+#define regTCP_PERFCOUNTER_FILTER2                                                                      0x3351
+#define regTCP_PERFCOUNTER_FILTER2_BASE_IDX                                                             1
+#define regTCP_PERFCOUNTER_FILTER_EN                                                                    0x3352
+#define regTCP_PERFCOUNTER_FILTER_EN_BASE_IDX                                                           1
+#define regTCP_PERFCOUNTER_SET_DEFINE                                                                   0x3353
+#define regTCP_PERFCOUNTER_SET_DEFINE_BASE_IDX                                                          1
+#define regGL1C_PERFCOUNTER0_LO                                                                         0x33a0
+#define regGL1C_PERFCOUNTER0_LO_BASE_IDX                                                                1
+#define regGL1C_PERFCOUNTER0_HI                                                                         0x33a1
+#define regGL1C_PERFCOUNTER0_HI_BASE_IDX                                                                1
+#define regGL1C_PERFCOUNTER1_LO                                                                         0x33a2
+#define regGL1C_PERFCOUNTER1_LO_BASE_IDX                                                                1
+#define regGL1C_PERFCOUNTER1_HI                                                                         0x33a3
+#define regGL1C_PERFCOUNTER1_HI_BASE_IDX                                                                1
+#define regGL1C_PERFCOUNTER2_LO                                                                         0x33a4
+#define regGL1C_PERFCOUNTER2_LO_BASE_IDX                                                                1
+#define regGL1C_PERFCOUNTER2_HI                                                                         0x33a5
+#define regGL1C_PERFCOUNTER2_HI_BASE_IDX                                                                1
+#define regGL1C_PERFCOUNTER3_LO                                                                         0x33a6
+#define regGL1C_PERFCOUNTER3_LO_BASE_IDX                                                                1
+#define regGL1C_PERFCOUNTER3_HI                                                                         0x33a7
+#define regGL1C_PERFCOUNTER3_HI_BASE_IDX                                                                1
+#define regGL1XC_PERFCOUNTER0_LO                                                                        0x33a8
+#define regGL1XC_PERFCOUNTER0_LO_BASE_IDX                                                               1
+#define regGL1XC_PERFCOUNTER0_HI                                                                        0x33a9
+#define regGL1XC_PERFCOUNTER0_HI_BASE_IDX                                                               1
+#define regGL1XC_PERFCOUNTER1_LO                                                                        0x33aa
+#define regGL1XC_PERFCOUNTER1_LO_BASE_IDX                                                               1
+#define regGL1XC_PERFCOUNTER1_HI                                                                        0x33ab
+#define regGL1XC_PERFCOUNTER1_HI_BASE_IDX                                                               1
+#define regGL1XC_PERFCOUNTER2_LO                                                                        0x33ac
+#define regGL1XC_PERFCOUNTER2_LO_BASE_IDX                                                               1
+#define regGL1XC_PERFCOUNTER2_HI                                                                        0x33ad
+#define regGL1XC_PERFCOUNTER2_HI_BASE_IDX                                                               1
+#define regGL1XC_PERFCOUNTER3_LO                                                                        0x33ae
+#define regGL1XC_PERFCOUNTER3_LO_BASE_IDX                                                               1
+#define regGL1XC_PERFCOUNTER3_HI                                                                        0x33af
+#define regGL1XC_PERFCOUNTER3_HI_BASE_IDX                                                               1
+#define regCB_PERFCOUNTER0_LO                                                                           0x3406
+#define regCB_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER0_HI                                                                           0x3407
+#define regCB_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER1_LO                                                                           0x3408
+#define regCB_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER1_HI                                                                           0x3409
+#define regCB_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER2_LO                                                                           0x340a
+#define regCB_PERFCOUNTER2_LO_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER2_HI                                                                           0x340b
+#define regCB_PERFCOUNTER2_HI_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER3_LO                                                                           0x340c
+#define regCB_PERFCOUNTER3_LO_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER3_HI                                                                           0x340d
+#define regCB_PERFCOUNTER3_HI_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER0_LO                                                                           0x3440
+#define regDB_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER0_HI                                                                           0x3441
+#define regDB_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER1_LO                                                                           0x3442
+#define regDB_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER1_HI                                                                           0x3443
+#define regDB_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER2_LO                                                                           0x3444
+#define regDB_PERFCOUNTER2_LO_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER2_HI                                                                           0x3445
+#define regDB_PERFCOUNTER2_HI_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER3_LO                                                                           0x3446
+#define regDB_PERFCOUNTER3_LO_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER3_HI                                                                           0x3447
+#define regDB_PERFCOUNTER3_HI_BASE_IDX                                                                  1
+#define regRMI_PERFCOUNTER0_LO                                                                          0x34c0
+#define regRMI_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER0_HI                                                                          0x34c1
+#define regRMI_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER1_LO                                                                          0x34c2
+#define regRMI_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER1_HI                                                                          0x34c3
+#define regRMI_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER2_LO                                                                          0x34c4
+#define regRMI_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER2_HI                                                                          0x34c5
+#define regRMI_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER3_LO                                                                          0x34c6
+#define regRMI_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER3_HI                                                                          0x34c7
+#define regRMI_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regPA_PH_PERFCOUNTER0_LO                                                                        0x3580
+#define regPA_PH_PERFCOUNTER0_LO_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER0_HI                                                                        0x3581
+#define regPA_PH_PERFCOUNTER0_HI_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER1_LO                                                                        0x3582
+#define regPA_PH_PERFCOUNTER1_LO_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER1_HI                                                                        0x3583
+#define regPA_PH_PERFCOUNTER1_HI_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER2_LO                                                                        0x3584
+#define regPA_PH_PERFCOUNTER2_LO_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER2_HI                                                                        0x3585
+#define regPA_PH_PERFCOUNTER2_HI_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER3_LO                                                                        0x3586
+#define regPA_PH_PERFCOUNTER3_LO_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER3_HI                                                                        0x3587
+#define regPA_PH_PERFCOUNTER3_HI_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER4_LO                                                                        0x3588
+#define regPA_PH_PERFCOUNTER4_LO_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER4_HI                                                                        0x3589
+#define regPA_PH_PERFCOUNTER4_HI_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER5_LO                                                                        0x358a
+#define regPA_PH_PERFCOUNTER5_LO_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER5_HI                                                                        0x358b
+#define regPA_PH_PERFCOUNTER5_HI_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER6_LO                                                                        0x358c
+#define regPA_PH_PERFCOUNTER6_LO_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER6_HI                                                                        0x358d
+#define regPA_PH_PERFCOUNTER6_HI_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER7_LO                                                                        0x358e
+#define regPA_PH_PERFCOUNTER7_LO_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER7_HI                                                                        0x358f
+#define regPA_PH_PERFCOUNTER7_HI_BASE_IDX                                                               1
+#define regUTCL1_PERFCOUNTER0_LO                                                                        0x35a0
+#define regUTCL1_PERFCOUNTER0_LO_BASE_IDX                                                               1
+#define regUTCL1_PERFCOUNTER0_HI                                                                        0x35a1
+#define regUTCL1_PERFCOUNTER0_HI_BASE_IDX                                                               1
+#define regUTCL1_PERFCOUNTER1_LO                                                                        0x35a2
+#define regUTCL1_PERFCOUNTER1_LO_BASE_IDX                                                               1
+#define regUTCL1_PERFCOUNTER1_HI                                                                        0x35a3
+#define regUTCL1_PERFCOUNTER1_HI_BASE_IDX                                                               1
+#define regUTCL1_PERFCOUNTER2_LO                                                                        0x35a4
+#define regUTCL1_PERFCOUNTER2_LO_BASE_IDX                                                               1
+#define regUTCL1_PERFCOUNTER2_HI                                                                        0x35a5
+#define regUTCL1_PERFCOUNTER2_HI_BASE_IDX                                                               1
+#define regUTCL1_PERFCOUNTER3_LO                                                                        0x35a6
+#define regUTCL1_PERFCOUNTER3_LO_BASE_IDX                                                               1
+#define regUTCL1_PERFCOUNTER3_HI                                                                        0x35a7
+#define regUTCL1_PERFCOUNTER3_HI_BASE_IDX                                                               1
+#define regWGS_PERFCOUNTER1_LO                                                                          0x35b0
+#define regWGS_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regWGS_PERFCOUNTER1_HI                                                                          0x35b1
+#define regWGS_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regWGS_PERFCOUNTER0_LO                                                                          0x35b2
+#define regWGS_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regWGS_PERFCOUNTER0_HI                                                                          0x35b3
+#define regWGS_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regWGS_PERFMON_CNTL                                                                             0x35b4
+#define regWGS_PERFMON_CNTL_BASE_IDX                                                                    1
+#define regGL1A_PERFCOUNTER0_LO                                                                         0x35c0
+#define regGL1A_PERFCOUNTER0_LO_BASE_IDX                                                                1
+#define regGL1A_PERFCOUNTER0_HI                                                                         0x35c1
+#define regGL1A_PERFCOUNTER0_HI_BASE_IDX                                                                1
+#define regGL1A_PERFCOUNTER1_LO                                                                         0x35c2
+#define regGL1A_PERFCOUNTER1_LO_BASE_IDX                                                                1
+#define regGL1A_PERFCOUNTER1_HI                                                                         0x35c3
+#define regGL1A_PERFCOUNTER1_HI_BASE_IDX                                                                1
+#define regGL1A_PERFCOUNTER2_LO                                                                         0x35c4
+#define regGL1A_PERFCOUNTER2_LO_BASE_IDX                                                                1
+#define regGL1A_PERFCOUNTER2_HI                                                                         0x35c5
+#define regGL1A_PERFCOUNTER2_HI_BASE_IDX                                                                1
+#define regGL1A_PERFCOUNTER3_LO                                                                         0x35c6
+#define regGL1A_PERFCOUNTER3_LO_BASE_IDX                                                                1
+#define regGL1A_PERFCOUNTER3_HI                                                                         0x35c7
+#define regGL1A_PERFCOUNTER3_HI_BASE_IDX                                                                1
+#define regGL1XA_PERFCOUNTER0_LO                                                                        0x35c8
+#define regGL1XA_PERFCOUNTER0_LO_BASE_IDX                                                               1
+#define regGL1XA_PERFCOUNTER0_HI                                                                        0x35c9
+#define regGL1XA_PERFCOUNTER0_HI_BASE_IDX                                                               1
+#define regGL1XA_PERFCOUNTER1_LO                                                                        0x35ca
+#define regGL1XA_PERFCOUNTER1_LO_BASE_IDX                                                               1
+#define regGL1XA_PERFCOUNTER1_HI                                                                        0x35cb
+#define regGL1XA_PERFCOUNTER1_HI_BASE_IDX                                                               1
+#define regGL1XA_PERFCOUNTER2_LO                                                                        0x35cc
+#define regGL1XA_PERFCOUNTER2_LO_BASE_IDX                                                               1
+#define regGL1XA_PERFCOUNTER2_HI                                                                        0x35cd
+#define regGL1XA_PERFCOUNTER2_HI_BASE_IDX                                                               1
+#define regGL1XA_PERFCOUNTER3_LO                                                                        0x35ce
+#define regGL1XA_PERFCOUNTER3_LO_BASE_IDX                                                               1
+#define regGL1XA_PERFCOUNTER3_HI                                                                        0x35cf
+#define regGL1XA_PERFCOUNTER3_HI_BASE_IDX                                                               1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_perfsdec
+// base address: 0x36000
+#define regGE2_SE_PERFCOUNTER0_SELECT                                                                   0x38b4
+#define regGE2_SE_PERFCOUNTER0_SELECT_BASE_IDX                                                          1
+#define regGE2_SE_PERFCOUNTER0_SELECT1                                                                  0x38b5
+#define regGE2_SE_PERFCOUNTER0_SELECT1_BASE_IDX                                                         1
+#define regGE2_SE_PERFCOUNTER1_SELECT                                                                   0x38b6
+#define regGE2_SE_PERFCOUNTER1_SELECT_BASE_IDX                                                          1
+#define regGE2_SE_PERFCOUNTER1_SELECT1                                                                  0x38b7
+#define regGE2_SE_PERFCOUNTER1_SELECT1_BASE_IDX                                                         1
+#define regGE2_SE_PERFCOUNTER2_SELECT                                                                   0x38b8
+#define regGE2_SE_PERFCOUNTER2_SELECT_BASE_IDX                                                          1
+#define regGE2_SE_PERFCOUNTER2_SELECT1                                                                  0x38b9
+#define regGE2_SE_PERFCOUNTER2_SELECT1_BASE_IDX                                                         1
+#define regGE2_SE_PERFCOUNTER3_SELECT                                                                   0x38ba
+#define regGE2_SE_PERFCOUNTER3_SELECT_BASE_IDX                                                          1
+#define regGE2_SE_PERFCOUNTER3_SELECT1                                                                  0x38bb
+#define regGE2_SE_PERFCOUNTER3_SELECT1_BASE_IDX                                                         1
+#define regGRBMH_PERFCOUNTER0_SELECT                                                                    0x38f8
+#define regGRBMH_PERFCOUNTER0_SELECT_BASE_IDX                                                           1
+#define regGRBMH_PERFCOUNTER1_SELECT                                                                    0x38f9
+#define regGRBMH_PERFCOUNTER1_SELECT_BASE_IDX                                                           1
+#define regPA_SU_PERFCOUNTER0_SELECT                                                                    0x3900
+#define regPA_SU_PERFCOUNTER0_SELECT_BASE_IDX                                                           1
+#define regPA_SU_PERFCOUNTER0_SELECT1                                                                   0x3901
+#define regPA_SU_PERFCOUNTER0_SELECT1_BASE_IDX                                                          1
+#define regPA_SU_PERFCOUNTER1_SELECT                                                                    0x3902
+#define regPA_SU_PERFCOUNTER1_SELECT_BASE_IDX                                                           1
+#define regPA_SU_PERFCOUNTER1_SELECT1                                                                   0x3903
+#define regPA_SU_PERFCOUNTER1_SELECT1_BASE_IDX                                                          1
+#define regPA_SU_PERFCOUNTER2_SELECT                                                                    0x3904
+#define regPA_SU_PERFCOUNTER2_SELECT_BASE_IDX                                                           1
+#define regPA_SU_PERFCOUNTER2_SELECT1                                                                   0x3905
+#define regPA_SU_PERFCOUNTER2_SELECT1_BASE_IDX                                                          1
+#define regPA_SU_PERFCOUNTER3_SELECT                                                                    0x3906
+#define regPA_SU_PERFCOUNTER3_SELECT_BASE_IDX                                                           1
+#define regPA_SU_PERFCOUNTER3_SELECT1                                                                   0x3907
+#define regPA_SU_PERFCOUNTER3_SELECT1_BASE_IDX                                                          1
+#define regPA_SC_PERFCOUNTER0_SELECT                                                                    0x3940
+#define regPA_SC_PERFCOUNTER0_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER0_SELECT1                                                                   0x3941
+#define regPA_SC_PERFCOUNTER0_SELECT1_BASE_IDX                                                          1
+#define regPA_SC_PERFCOUNTER1_SELECT                                                                    0x3942
+#define regPA_SC_PERFCOUNTER1_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER2_SELECT                                                                    0x3943
+#define regPA_SC_PERFCOUNTER2_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER3_SELECT                                                                    0x3944
+#define regPA_SC_PERFCOUNTER3_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER4_SELECT                                                                    0x3945
+#define regPA_SC_PERFCOUNTER4_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER5_SELECT                                                                    0x3946
+#define regPA_SC_PERFCOUNTER5_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER6_SELECT                                                                    0x3947
+#define regPA_SC_PERFCOUNTER6_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER7_SELECT                                                                    0x3948
+#define regPA_SC_PERFCOUNTER7_SELECT_BASE_IDX                                                           1
+#define regSPI_PERFCOUNTER0_SELECT                                                                      0x3980
+#define regSPI_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER1_SELECT                                                                      0x3981
+#define regSPI_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER2_SELECT                                                                      0x3982
+#define regSPI_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER3_SELECT                                                                      0x3983
+#define regSPI_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER4_SELECT                                                                      0x3984
+#define regSPI_PERFCOUNTER4_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER5_SELECT                                                                      0x3985
+#define regSPI_PERFCOUNTER5_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER0_SELECT1                                                                     0x3986
+#define regSPI_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regSPI_PERFCOUNTER1_SELECT1                                                                     0x3987
+#define regSPI_PERFCOUNTER1_SELECT1_BASE_IDX                                                            1
+#define regSPI_PERFCOUNTER2_SELECT1                                                                     0x3988
+#define regSPI_PERFCOUNTER2_SELECT1_BASE_IDX                                                            1
+#define regSPI_PERFCOUNTER3_SELECT1                                                                     0x3989
+#define regSPI_PERFCOUNTER3_SELECT1_BASE_IDX                                                            1
+#define regSPI_PERFCOUNTER4_SELECT1                                                                     0x398a
+#define regSPI_PERFCOUNTER4_SELECT1_BASE_IDX                                                            1
+#define regSPI_PERFCOUNTER5_SELECT1                                                                     0x398b
+#define regSPI_PERFCOUNTER5_SELECT1_BASE_IDX                                                            1
+#define regSPI_PERFCOUNTER_BINS                                                                         0x398c
+#define regSPI_PERFCOUNTER_BINS_BASE_IDX                                                                1
+#define regPC_PERFCOUNTER0_SELECT                                                                       0x3990
+#define regPC_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regPC_PERFCOUNTER1_SELECT                                                                       0x3991
+#define regPC_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regPC_PERFCOUNTER2_SELECT                                                                       0x3992
+#define regPC_PERFCOUNTER2_SELECT_BASE_IDX                                                              1
+#define regPC_PERFCOUNTER3_SELECT                                                                       0x3993
+#define regPC_PERFCOUNTER3_SELECT_BASE_IDX                                                              1
+#define regPC_PERFCOUNTER0_SELECT1                                                                      0x3994
+#define regPC_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regPC_PERFCOUNTER1_SELECT1                                                                      0x3995
+#define regPC_PERFCOUNTER1_SELECT1_BASE_IDX                                                             1
+#define regPC_PERFCOUNTER2_SELECT1                                                                      0x3996
+#define regPC_PERFCOUNTER2_SELECT1_BASE_IDX                                                             1
+#define regPC_PERFCOUNTER3_SELECT1                                                                      0x3997
+#define regPC_PERFCOUNTER3_SELECT1_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER0_SELECT                                                                       0x39c0
+#define regSQ_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER1_SELECT                                                                       0x39c1
+#define regSQ_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER2_SELECT                                                                       0x39c2
+#define regSQ_PERFCOUNTER2_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER3_SELECT                                                                       0x39c3
+#define regSQ_PERFCOUNTER3_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER4_SELECT                                                                       0x39c4
+#define regSQ_PERFCOUNTER4_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER5_SELECT                                                                       0x39c5
+#define regSQ_PERFCOUNTER5_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER6_SELECT                                                                       0x39c6
+#define regSQ_PERFCOUNTER6_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER7_SELECT                                                                       0x39c7
+#define regSQ_PERFCOUNTER7_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER8_SELECT                                                                       0x39c8
+#define regSQ_PERFCOUNTER8_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER9_SELECT                                                                       0x39c9
+#define regSQ_PERFCOUNTER9_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER10_SELECT                                                                      0x39ca
+#define regSQ_PERFCOUNTER10_SELECT_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER11_SELECT                                                                      0x39cb
+#define regSQ_PERFCOUNTER11_SELECT_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER12_SELECT                                                                      0x39cc
+#define regSQ_PERFCOUNTER12_SELECT_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER13_SELECT                                                                      0x39cd
+#define regSQ_PERFCOUNTER13_SELECT_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER14_SELECT                                                                      0x39ce
+#define regSQ_PERFCOUNTER14_SELECT_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER15_SELECT                                                                      0x39cf
+#define regSQ_PERFCOUNTER15_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER0_SELECT                                                                      0x39d0
+#define regSQG_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER1_SELECT                                                                      0x39d1
+#define regSQG_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER2_SELECT                                                                      0x39d2
+#define regSQG_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER3_SELECT                                                                      0x39d3
+#define regSQG_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER4_SELECT                                                                      0x39d4
+#define regSQG_PERFCOUNTER4_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER5_SELECT                                                                      0x39d5
+#define regSQG_PERFCOUNTER5_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER6_SELECT                                                                      0x39d6
+#define regSQG_PERFCOUNTER6_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER7_SELECT                                                                      0x39d7
+#define regSQG_PERFCOUNTER7_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER_CTRL                                                                         0x39d8
+#define regSQG_PERFCOUNTER_CTRL_BASE_IDX                                                                1
+#define regSQG_PERFCOUNTER_CTRL2                                                                        0x39da
+#define regSQG_PERFCOUNTER_CTRL2_BASE_IDX                                                               1
+#define regSQG_PERF_SAMPLE_FINISH                                                                       0x39db
+#define regSQG_PERF_SAMPLE_FINISH_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER_CTRL                                                                          0x39e0
+#define regSQ_PERFCOUNTER_CTRL_BASE_IDX                                                                 1
+#define regSQ_PERFCOUNTER_CTRL2                                                                         0x39e2
+#define regSQ_PERFCOUNTER_CTRL2_BASE_IDX                                                                1
+#define regSQ_THREAD_TRACE_BUF0_SIZE                                                                    0x39e6
+#define regSQ_THREAD_TRACE_BUF0_SIZE_BASE_IDX                                                           1
+#define regSQ_THREAD_TRACE_BUF0_BASE_LO                                                                 0x39e7
+#define regSQ_THREAD_TRACE_BUF0_BASE_LO_BASE_IDX                                                        1
+#define regSQ_THREAD_TRACE_BUF0_BASE_HI                                                                 0x39e8
+#define regSQ_THREAD_TRACE_BUF0_BASE_HI_BASE_IDX                                                        1
+#define regSQ_THREAD_TRACE_BUF1_SIZE                                                                    0x39e9
+#define regSQ_THREAD_TRACE_BUF1_SIZE_BASE_IDX                                                           1
+#define regSQ_THREAD_TRACE_BUF1_BASE_LO                                                                 0x39ea
+#define regSQ_THREAD_TRACE_BUF1_BASE_LO_BASE_IDX                                                        1
+#define regSQ_THREAD_TRACE_BUF1_BASE_HI                                                                 0x39eb
+#define regSQ_THREAD_TRACE_BUF1_BASE_HI_BASE_IDX                                                        1
+#define regSQ_THREAD_TRACE_CTRL                                                                         0x39ec
+#define regSQ_THREAD_TRACE_CTRL_BASE_IDX                                                                1
+#define regSQ_THREAD_TRACE_MASK                                                                         0x39ed
+#define regSQ_THREAD_TRACE_MASK_BASE_IDX                                                                1
+#define regSQ_THREAD_TRACE_TOKEN_MASK                                                                   0x39ee
+#define regSQ_THREAD_TRACE_TOKEN_MASK_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_WPTR                                                                         0x39ef
+#define regSQ_THREAD_TRACE_WPTR_BASE_IDX                                                                1
+#define regSQ_THREAD_TRACE_HALT                                                                         0x39f0
+#define regSQ_THREAD_TRACE_HALT_BASE_IDX                                                                1
+#define regSQ_THREAD_TRACE_POWEROFF_RESTORE_1                                                           0x39f1
+#define regSQ_THREAD_TRACE_POWEROFF_RESTORE_1_BASE_IDX                                                  1
+#define regSQ_THREAD_TRACE_STATUS                                                                       0x39f4
+#define regSQ_THREAD_TRACE_STATUS_BASE_IDX                                                              1
+#define regSQ_THREAD_TRACE_STATUS2                                                                      0x39f5
+#define regSQ_THREAD_TRACE_STATUS2_BASE_IDX                                                             1
+#define regSQ_THREAD_TRACE_GFX_DRAW_CNTR                                                                0x39f6
+#define regSQ_THREAD_TRACE_GFX_DRAW_CNTR_BASE_IDX                                                       1
+#define regSQ_THREAD_TRACE_GFX_MARKER_CNTR                                                              0x39f7
+#define regSQ_THREAD_TRACE_GFX_MARKER_CNTR_BASE_IDX                                                     1
+#define regSQ_THREAD_TRACE_HP3D_DRAW_CNTR                                                               0x39f8
+#define regSQ_THREAD_TRACE_HP3D_DRAW_CNTR_BASE_IDX                                                      1
+#define regSQ_THREAD_TRACE_HP3D_MARKER_CNTR                                                             0x39f9
+#define regSQ_THREAD_TRACE_HP3D_MARKER_CNTR_BASE_IDX                                                    1
+#define regSQ_THREAD_TRACE_DROPPED_CNTR                                                                 0x39fa
+#define regSQ_THREAD_TRACE_DROPPED_CNTR_BASE_IDX                                                        1
+#define regSQ_THREAD_TRACE_FINISH_DONE_DEBUG                                                            0x39fb
+#define regSQ_THREAD_TRACE_FINISH_DONE_DEBUG_BASE_IDX                                                   1
+#define regSX_PERFCOUNTER0_SELECT                                                                       0x3a40
+#define regSX_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regSX_PERFCOUNTER1_SELECT                                                                       0x3a41
+#define regSX_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regSX_PERFCOUNTER2_SELECT                                                                       0x3a42
+#define regSX_PERFCOUNTER2_SELECT_BASE_IDX                                                              1
+#define regSX_PERFCOUNTER3_SELECT                                                                       0x3a43
+#define regSX_PERFCOUNTER3_SELECT_BASE_IDX                                                              1
+#define regSX_PERFCOUNTER0_SELECT1                                                                      0x3a44
+#define regSX_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regSX_PERFCOUNTER1_SELECT1                                                                      0x3a45
+#define regSX_PERFCOUNTER1_SELECT1_BASE_IDX                                                             1
+#define regSX_PERFCOUNTER2_SELECT1                                                                      0x3a46
+#define regSX_PERFCOUNTER2_SELECT1_BASE_IDX                                                             1
+#define regSX_PERFCOUNTER3_SELECT1                                                                      0x3a47
+#define regSX_PERFCOUNTER3_SELECT1_BASE_IDX                                                             1
+#define regTA_PERFCOUNTER0_SELECT                                                                       0x3ac0
+#define regTA_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regTA_PERFCOUNTER0_SELECT1                                                                      0x3ac1
+#define regTA_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regTA_PERFCOUNTER1_SELECT                                                                       0x3ac2
+#define regTA_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regTD_PERFCOUNTER0_SELECT                                                                       0x3b00
+#define regTD_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regTD_PERFCOUNTER0_SELECT1                                                                      0x3b01
+#define regTD_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regTD_PERFCOUNTER1_SELECT                                                                       0x3b02
+#define regTD_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regTCP_PERFCOUNTER0_SELECT                                                                      0x3b40
+#define regTCP_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regTCP_PERFCOUNTER0_SELECT1                                                                     0x3b41
+#define regTCP_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regTCP_PERFCOUNTER1_SELECT                                                                      0x3b42
+#define regTCP_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regTCP_PERFCOUNTER1_SELECT1                                                                     0x3b43
+#define regTCP_PERFCOUNTER1_SELECT1_BASE_IDX                                                            1
+#define regTCP_PERFCOUNTER2_SELECT                                                                      0x3b44
+#define regTCP_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regTCP_PERFCOUNTER2_SELECT1                                                                     0x3b45
+#define regTCP_PERFCOUNTER2_SELECT1_BASE_IDX                                                            1
+#define regTCP_PERFCOUNTER3_SELECT                                                                      0x3b46
+#define regTCP_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regTCP_PERFCOUNTER3_SELECT1                                                                     0x3b47
+#define regTCP_PERFCOUNTER3_SELECT1_BASE_IDX                                                            1
+#define regTCP_PERFCOUNTER4_SELECT                                                                      0x3b4a
+#define regTCP_PERFCOUNTER4_SELECT_BASE_IDX                                                             1
+#define regTCP_PERFCOUNTER5_SELECT                                                                      0x3b4b
+#define regTCP_PERFCOUNTER5_SELECT_BASE_IDX                                                             1
+#define regTCP_PERFCOUNTER6_SELECT                                                                      0x3b4c
+#define regTCP_PERFCOUNTER6_SELECT_BASE_IDX                                                             1
+#define regTCP_PERFCOUNTER7_SELECT                                                                      0x3b4d
+#define regTCP_PERFCOUNTER7_SELECT_BASE_IDX                                                             1
+#define regGL1C_PERFCOUNTER0_SELECT                                                                     0x3ba0
+#define regGL1C_PERFCOUNTER0_SELECT_BASE_IDX                                                            1
+#define regGL1C_PERFCOUNTER0_SELECT1                                                                    0x3ba1
+#define regGL1C_PERFCOUNTER0_SELECT1_BASE_IDX                                                           1
+#define regGL1C_PERFCOUNTER1_SELECT                                                                     0x3ba2
+#define regGL1C_PERFCOUNTER1_SELECT_BASE_IDX                                                            1
+#define regGL1C_PERFCOUNTER1_SELECT1                                                                    0x3ba3
+#define regGL1C_PERFCOUNTER1_SELECT1_BASE_IDX                                                           1
+#define regGL1C_PERFCOUNTER2_SELECT                                                                     0x3ba4
+#define regGL1C_PERFCOUNTER2_SELECT_BASE_IDX                                                            1
+#define regGL1C_PERFCOUNTER2_SELECT1                                                                    0x3ba5
+#define regGL1C_PERFCOUNTER2_SELECT1_BASE_IDX                                                           1
+#define regGL1C_PERFCOUNTER3_SELECT                                                                     0x3ba6
+#define regGL1C_PERFCOUNTER3_SELECT_BASE_IDX                                                            1
+#define regGL1C_PERFCOUNTER3_SELECT1                                                                    0x3ba7
+#define regGL1C_PERFCOUNTER3_SELECT1_BASE_IDX                                                           1
+#define regGL1XC_PERFCOUNTER0_SELECT                                                                    0x3ba8
+#define regGL1XC_PERFCOUNTER0_SELECT_BASE_IDX                                                           1
+#define regGL1XC_PERFCOUNTER0_SELECT1                                                                   0x3ba9
+#define regGL1XC_PERFCOUNTER0_SELECT1_BASE_IDX                                                          1
+#define regGL1XC_PERFCOUNTER1_SELECT                                                                    0x3baa
+#define regGL1XC_PERFCOUNTER1_SELECT_BASE_IDX                                                           1
+#define regGL1XC_PERFCOUNTER1_SELECT1                                                                   0x3bab
+#define regGL1XC_PERFCOUNTER1_SELECT1_BASE_IDX                                                          1
+#define regGL1XC_PERFCOUNTER2_SELECT                                                                    0x3bac
+#define regGL1XC_PERFCOUNTER2_SELECT_BASE_IDX                                                           1
+#define regGL1XC_PERFCOUNTER2_SELECT1                                                                   0x3bad
+#define regGL1XC_PERFCOUNTER2_SELECT1_BASE_IDX                                                          1
+#define regGL1XC_PERFCOUNTER3_SELECT                                                                    0x3bae
+#define regGL1XC_PERFCOUNTER3_SELECT_BASE_IDX                                                           1
+#define regGL1XC_PERFCOUNTER3_SELECT1                                                                   0x3baf
+#define regGL1XC_PERFCOUNTER3_SELECT1_BASE_IDX                                                          1
+#define regCB_PERFCOUNTER_FILTER                                                                        0x3c00
+#define regCB_PERFCOUNTER_FILTER_BASE_IDX                                                               1
+#define regCB_PERFCOUNTER0_SELECT                                                                       0x3c01
+#define regCB_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regCB_PERFCOUNTER0_SELECT1                                                                      0x3c02
+#define regCB_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regCB_PERFCOUNTER1_SELECT                                                                       0x3c03
+#define regCB_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regCB_PERFCOUNTER2_SELECT                                                                       0x3c04
+#define regCB_PERFCOUNTER2_SELECT_BASE_IDX                                                              1
+#define regCB_PERFCOUNTER3_SELECT                                                                       0x3c05
+#define regCB_PERFCOUNTER3_SELECT_BASE_IDX                                                              1
+#define regDB_PERFCOUNTER0_SELECT                                                                       0x3c40
+#define regDB_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regDB_PERFCOUNTER0_SELECT1                                                                      0x3c41
+#define regDB_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regDB_PERFCOUNTER1_SELECT                                                                       0x3c42
+#define regDB_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regDB_PERFCOUNTER1_SELECT1                                                                      0x3c43
+#define regDB_PERFCOUNTER1_SELECT1_BASE_IDX                                                             1
+#define regDB_PERFCOUNTER2_SELECT                                                                       0x3c44
+#define regDB_PERFCOUNTER2_SELECT_BASE_IDX                                                              1
+#define regDB_PERFCOUNTER2_SELECT1                                                                      0x3c45
+#define regDB_PERFCOUNTER2_SELECT1_BASE_IDX                                                             1
+#define regDB_PERFCOUNTER3_SELECT                                                                       0x3c46
+#define regDB_PERFCOUNTER3_SELECT_BASE_IDX                                                              1
+#define regDB_PERFCOUNTER3_SELECT1                                                                      0x3c47
+#define regDB_PERFCOUNTER3_SELECT1_BASE_IDX                                                             1
+#define regRMI_PERFCOUNTER0_SELECT                                                                      0x3d00
+#define regRMI_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regRMI_PERFCOUNTER0_SELECT1                                                                     0x3d01
+#define regRMI_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regRMI_PERFCOUNTER1_SELECT                                                                      0x3d02
+#define regRMI_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regRMI_PERFCOUNTER2_SELECT                                                                      0x3d03
+#define regRMI_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regRMI_PERFCOUNTER2_SELECT1                                                                     0x3d04
+#define regRMI_PERFCOUNTER2_SELECT1_BASE_IDX                                                            1
+#define regRMI_PERFCOUNTER3_SELECT                                                                      0x3d05
+#define regRMI_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regRMI_PERF_COUNTER_CNTL                                                                        0x3d06
+#define regRMI_PERF_COUNTER_CNTL_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER0_SELECT                                                                    0x3d80
+#define regPA_PH_PERFCOUNTER0_SELECT_BASE_IDX                                                           1
+#define regPA_PH_PERFCOUNTER0_SELECT1                                                                   0x3d81
+#define regPA_PH_PERFCOUNTER0_SELECT1_BASE_IDX                                                          1
+#define regPA_PH_PERFCOUNTER1_SELECT                                                                    0x3d82
+#define regPA_PH_PERFCOUNTER1_SELECT_BASE_IDX                                                           1
+#define regPA_PH_PERFCOUNTER2_SELECT                                                                    0x3d83
+#define regPA_PH_PERFCOUNTER2_SELECT_BASE_IDX                                                           1
+#define regPA_PH_PERFCOUNTER3_SELECT                                                                    0x3d84
+#define regPA_PH_PERFCOUNTER3_SELECT_BASE_IDX                                                           1
+#define regPA_PH_PERFCOUNTER4_SELECT                                                                    0x3d85
+#define regPA_PH_PERFCOUNTER4_SELECT_BASE_IDX                                                           1
+#define regPA_PH_PERFCOUNTER5_SELECT                                                                    0x3d86
+#define regPA_PH_PERFCOUNTER5_SELECT_BASE_IDX                                                           1
+#define regPA_PH_PERFCOUNTER6_SELECT                                                                    0x3d87
+#define regPA_PH_PERFCOUNTER6_SELECT_BASE_IDX                                                           1
+#define regPA_PH_PERFCOUNTER7_SELECT                                                                    0x3d88
+#define regPA_PH_PERFCOUNTER7_SELECT_BASE_IDX                                                           1
+#define regPA_PH_PERFCOUNTER1_SELECT1                                                                   0x3d90
+#define regPA_PH_PERFCOUNTER1_SELECT1_BASE_IDX                                                          1
+#define regPA_PH_PERFCOUNTER2_SELECT1                                                                   0x3d91
+#define regPA_PH_PERFCOUNTER2_SELECT1_BASE_IDX                                                          1
+#define regPA_PH_PERFCOUNTER3_SELECT1                                                                   0x3d92
+#define regPA_PH_PERFCOUNTER3_SELECT1_BASE_IDX                                                          1
+#define regUTCL1_PERFCOUNTER0_SELECT                                                                    0x3da0
+#define regUTCL1_PERFCOUNTER0_SELECT_BASE_IDX                                                           1
+#define regUTCL1_PERFCOUNTER0_SELECT1                                                                   0x3da1
+#define regUTCL1_PERFCOUNTER0_SELECT1_BASE_IDX                                                          1
+#define regUTCL1_PERFCOUNTER1_SELECT                                                                    0x3da2
+#define regUTCL1_PERFCOUNTER1_SELECT_BASE_IDX                                                           1
+#define regUTCL1_PERFCOUNTER1_SELECT1                                                                   0x3da3
+#define regUTCL1_PERFCOUNTER1_SELECT1_BASE_IDX                                                          1
+#define regUTCL1_PERFCOUNTER2_SELECT                                                                    0x3da4
+#define regUTCL1_PERFCOUNTER2_SELECT_BASE_IDX                                                           1
+#define regUTCL1_PERFCOUNTER3_SELECT                                                                    0x3da5
+#define regUTCL1_PERFCOUNTER3_SELECT_BASE_IDX                                                           1
+#define regWGS_PERFCOUNTER1_SELECT                                                                      0x3db0
+#define regWGS_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regWGS_PERFCOUNTER0_SELECT1                                                                     0x3db1
+#define regWGS_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regWGS_PERFCOUNTER0_SELECT                                                                      0x3db3
+#define regWGS_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regGL1A_PERFCOUNTER0_SELECT                                                                     0x3dc0
+#define regGL1A_PERFCOUNTER0_SELECT_BASE_IDX                                                            1
+#define regGL1A_PERFCOUNTER0_SELECT1                                                                    0x3dc1
+#define regGL1A_PERFCOUNTER0_SELECT1_BASE_IDX                                                           1
+#define regGL1A_PERFCOUNTER1_SELECT                                                                     0x3dc2
+#define regGL1A_PERFCOUNTER1_SELECT_BASE_IDX                                                            1
+#define regGL1A_PERFCOUNTER1_SELECT1                                                                    0x3dc3
+#define regGL1A_PERFCOUNTER1_SELECT1_BASE_IDX                                                           1
+#define regGL1A_PERFCOUNTER2_SELECT                                                                     0x3dc4
+#define regGL1A_PERFCOUNTER2_SELECT_BASE_IDX                                                            1
+#define regGL1A_PERFCOUNTER2_SELECT1                                                                    0x3dc5
+#define regGL1A_PERFCOUNTER2_SELECT1_BASE_IDX                                                           1
+#define regGL1A_PERFCOUNTER3_SELECT                                                                     0x3dc6
+#define regGL1A_PERFCOUNTER3_SELECT_BASE_IDX                                                            1
+#define regGL1A_PERFCOUNTER3_SELECT1                                                                    0x3dc7
+#define regGL1A_PERFCOUNTER3_SELECT1_BASE_IDX                                                           1
+#define regGL1XA_PERFCOUNTER0_SELECT                                                                    0x3dc8
+#define regGL1XA_PERFCOUNTER0_SELECT_BASE_IDX                                                           1
+#define regGL1XA_PERFCOUNTER0_SELECT1                                                                   0x3dc9
+#define regGL1XA_PERFCOUNTER0_SELECT1_BASE_IDX                                                          1
+#define regGL1XA_PERFCOUNTER1_SELECT                                                                    0x3dca
+#define regGL1XA_PERFCOUNTER1_SELECT_BASE_IDX                                                           1
+#define regGL1XA_PERFCOUNTER1_SELECT1                                                                   0x3dcb
+#define regGL1XA_PERFCOUNTER1_SELECT1_BASE_IDX                                                          1
+#define regGL1XA_PERFCOUNTER2_SELECT                                                                    0x3dcc
+#define regGL1XA_PERFCOUNTER2_SELECT_BASE_IDX                                                           1
+#define regGL1XA_PERFCOUNTER2_SELECT1                                                                   0x3dcd
+#define regGL1XA_PERFCOUNTER2_SELECT1_BASE_IDX                                                          1
+#define regGL1XA_PERFCOUNTER3_SELECT                                                                    0x3dce
+#define regGL1XA_PERFCOUNTER3_SELECT_BASE_IDX                                                           1
+#define regGL1XA_PERFCOUNTER3_SELECT1                                                                   0x3dcf
+#define regGL1XA_PERFCOUNTER3_SELECT1_BASE_IDX                                                          1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pwrdec
+// base address: 0x3c000
+#define regGFX_ICG_SPI_RA0_CLK_CTRL                                                                     0x507a
+#define regGFX_ICG_SPI_RA0_CLK_CTRL_BASE_IDX                                                            1
+#define regGFX_ICG_SPI_RA1_CLK_CTRL                                                                     0x507b
+#define regGFX_ICG_SPI_RA1_CLK_CTRL_BASE_IDX                                                            1
+#define regGFX_ICG_SPI_CS_CTRL                                                                          0x507c
+#define regGFX_ICG_SPI_CS_CTRL_BASE_IDX                                                                 1
+#define regGFX_ICG_SPI_PS_CTRL                                                                          0x507d
+#define regGFX_ICG_SPI_PS_CTRL_BASE_IDX                                                                 1
+#define regGFX_ICG_SPIS_CTRL                                                                            0x507e
+#define regGFX_ICG_SPIS_CTRL_BASE_IDX                                                                   1
+#define regCGTX_SPI_DEBUG_CLK_CTRL                                                                      0x507f
+#define regCGTX_SPI_DEBUG_CLK_CTRL_BASE_IDX                                                             1
+#define regGFX_ICG_SPI_CTRL                                                                             0x5080
+#define regGFX_ICG_SPI_CTRL_BASE_IDX                                                                    1
+#define regGFX_ICG_PC_CLK_CTRL                                                                          0x5081
+#define regGFX_ICG_PC_CLK_CTRL_BASE_IDX                                                                 1
+#define regGFX_ICG_BCI_CTRL                                                                             0x5082
+#define regGFX_ICG_BCI_CTRL_BASE_IDX                                                                    1
+#define regCGTT_VGT_CLK_CTRL                                                                            0x5084
+#define regCGTT_VGT_CLK_CTRL_BASE_IDX                                                                   1
+#define regCGTT_GS_NGG_CLK_CTRL                                                                         0x5087
+#define regCGTT_GS_NGG_CLK_CTRL_BASE_IDX                                                                1
+#define regCGTT_PA_CLK_CTRL                                                                             0x5088
+#define regCGTT_PA_CLK_CTRL_BASE_IDX                                                                    1
+#define regCGTT_SQ_CLK_CTRL                                                                             0x508c
+#define regCGTT_SQ_CLK_CTRL_BASE_IDX                                                                    1
+#define regCGTT_SQG_CLK_CTRL                                                                            0x508d
+#define regCGTT_SQG_CLK_CTRL_BASE_IDX                                                                   1
+#define regSQ_ALU_CLK_CTRL                                                                              0x508e
+#define regSQ_ALU_CLK_CTRL_BASE_IDX                                                                     1
+#define regSQ_TEX_CLK_CTRL                                                                              0x508f
+#define regSQ_TEX_CLK_CTRL_BASE_IDX                                                                     1
+#define regSQ_LDS_CLK_CTRL                                                                              0x5090
+#define regSQ_LDS_CLK_CTRL_BASE_IDX                                                                     1
+#define regSQ_CLK_CTRL                                                                                  0x5091
+#define regSQ_CLK_CTRL_BASE_IDX                                                                         1
+#define regICG_SQ_CLK_CTRL                                                                              0x5092
+#define regICG_SQ_CLK_CTRL_BASE_IDX                                                                     1
+#define regICG_SP_CLK_CTRL                                                                              0x5093
+#define regICG_SP_CLK_CTRL_BASE_IDX                                                                     1
+#define regGFX_ICG_SX_CLK_CTRL0                                                                         0x5094
+#define regGFX_ICG_SX_CLK_CTRL0_BASE_IDX                                                                1
+#define regGFX_ICG_SX_CLK_CTRL1                                                                         0x5095
+#define regGFX_ICG_SX_CLK_CTRL1_BASE_IDX                                                                1
+#define regGFX_ICG_SX_CLK_CTRL2                                                                         0x5096
+#define regGFX_ICG_SX_CLK_CTRL2_BASE_IDX                                                                1
+#define regGFX_ICG_SX_CLK_CTRL3                                                                         0x5097
+#define regGFX_ICG_SX_CLK_CTRL3_BASE_IDX                                                                1
+#define regGFX_ICG_SX_CLK_CTRL4                                                                         0x5098
+#define regGFX_ICG_SX_CLK_CTRL4_BASE_IDX                                                                1
+#define regGFX_ICG_TA_CTRL                                                                              0x509e
+#define regGFX_ICG_TA_CTRL_BASE_IDX                                                                     1
+#define regGFX_ICG_TD_CTRL                                                                              0x509f
+#define regGFX_ICG_TD_CTRL_BASE_IDX                                                                     1
+#define regDB_CGTT_CLK_CTRL_0                                                                           0x50a4
+#define regDB_CGTT_CLK_CTRL_0_BASE_IDX                                                                  1
+#define regGFX_ICG_CB_CTRL                                                                              0x50a9
+#define regGFX_ICG_CB_CTRL_BASE_IDX                                                                     1
+#define regGFX_ICG_RMI_CTRL                                                                             0x50c0
+#define regGFX_ICG_RMI_CTRL_BASE_IDX                                                                    1
+#define regGFX_ICG_SE_CAC_CLK_CTRL                                                                      0x50d0
+#define regGFX_ICG_SE_CAC_CLK_CTRL_BASE_IDX                                                             1
+#define regCGTT_PH_CLK_CTRL0                                                                            0x50f8
+#define regCGTT_PH_CLK_CTRL0_BASE_IDX                                                                   1
+#define regCGTT_PH_CLK_CTRL1                                                                            0x50f9
+#define regCGTT_PH_CLK_CTRL1_BASE_IDX                                                                   1
+#define regCGTT_PH_CLK_CTRL2                                                                            0x50fa
+#define regCGTT_PH_CLK_CTRL2_BASE_IDX                                                                   1
+#define regCGTT_PH_CLK_CTRL3                                                                            0x50fb
+#define regCGTT_PH_CLK_CTRL3_BASE_IDX                                                                   1
+#define regGFX_ICG_TCP_CTRL                                                                             0x5101
+#define regGFX_ICG_TCP_CTRL_BASE_IDX                                                                    1
+#define regGFX_ICG_TCP_CTRL2                                                                            0x5102
+#define regGFX_ICG_TCP_CTRL2_BASE_IDX                                                                   1
+#define regICG_LDS_CLK_CTRL                                                                             0x5114
+#define regICG_LDS_CLK_CTRL_BASE_IDX                                                                    1
+#define regGFX_ICG_UTCL1_CTRL                                                                           0x511c
+#define regGFX_ICG_UTCL1_CTRL_BASE_IDX                                                                  1
+#define regGFX_ICG_GRBMH_CTRL                                                                           0x5120
+#define regGFX_ICG_GRBMH_CTRL_BASE_IDX                                                                  1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_sc_pwrdec
+// base address: 0x3c344
+#define regCGTT_SC_CLK_CTRL0                                                                            0x50d1
+#define regCGTT_SC_CLK_CTRL0_BASE_IDX                                                                   1
+#define regCGTT_SC_CLK_CTRL1                                                                            0x50d2
+#define regCGTT_SC_CLK_CTRL1_BASE_IDX                                                                   1
+#define regCGTT_SC_CLK_CTRL2                                                                            0x50d3
+#define regCGTT_SC_CLK_CTRL2_BASE_IDX                                                                   1
+#define regCGTT_SC_CLK_CTRL3                                                                            0x50d4
+#define regCGTT_SC_CLK_CTRL3_BASE_IDX                                                                   1
+#define regCGTT_SC_CLK_CTRL4                                                                            0x50d5
+#define regCGTT_SC_CLK_CTRL4_BASE_IDX                                                                   1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_gl1_pwrdec
+// base address: 0x3c364
+#define regICG_GL1C_CLK_CTRL                                                                            0x50d9
+#define regICG_GL1C_CLK_CTRL_BASE_IDX                                                                   1
+#define regGL1I_GL1R_MGCG_OVERRIDE                                                                      0x50da
+#define regGL1I_GL1R_MGCG_OVERRIDE_BASE_IDX                                                             1
+#define regGL1XI_GL1XR_MGCG_OVERRIDE                                                                    0x50db
+#define regGL1XI_GL1XR_MGCG_OVERRIDE_BASE_IDX                                                           1
+#define regICG_GL1XC_CLK_CTRL                                                                           0x50dc
+#define regICG_GL1XC_CLK_CTRL_BASE_IDX                                                                  1
+#define regICG_GL1A_CTRL                                                                                0x50dd
+#define regICG_GL1A_CTRL_BASE_IDX                                                                       1
+#define regICG_GL1XA_CTRL                                                                               0x50de
+#define regICG_GL1XA_CTRL_BASE_IDX                                                                      1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_hypdec
+// base address: 0x3e000
+#define regGL1_PIPE_STEER_LSB                                                                           0x5b84
+#define regGL1_PIPE_STEER_LSB_BASE_IDX                                                                  1
+#define regGL1_PIPE_STEER_MSB                                                                           0x5b85
+#define regGL1_PIPE_STEER_MSB_BASE_IDX                                                                  1
+#define regGL1_HASH_CFG                                                                                 0x5b86
+#define regGL1_HASH_CFG_BASE_IDX                                                                        1
+#define regGC_USER_SHADER_ARRAY_CONFIG                                                                  0x5b90
+#define regGC_USER_SHADER_ARRAY_CONFIG_BASE_IDX                                                         1
+#define regGC_USER_RB_BACKEND_DISABLE                                                                   0x5b94
+#define regGC_USER_RB_BACKEND_DISABLE_BASE_IDX                                                          1
+#define regGC_USER_RMI_REDUNDANCY                                                                       0x5b95
+#define regGC_USER_RMI_REDUNDANCY_BASE_IDX                                                              1
+#define regGC_USER_SHADER_RATE_CONFIG_1                                                                 0x5b97
+#define regGC_USER_SHADER_RATE_CONFIG_1_BASE_IDX                                                        1
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_grbmh_hypdec
+// base address: 0x3ee80
+#define regGRBMH_WGP_SA0_REMAP_CNTL                                                                     0x5ba0
+#define regGRBMH_WGP_SA0_REMAP_CNTL_BASE_IDX                                                            1
+#define regGRBMH_WGP_SA1_REMAP_CNTL                                                                     0x5ba1
+#define regGRBMH_WGP_SA1_REMAP_CNTL_BASE_IDX                                                            1
+#define regGRBMH_RB_SA0_REMAP_CNTL                                                                      0x5ba2
+#define regGRBMH_RB_SA0_REMAP_CNTL_BASE_IDX                                                             1
+#define regGRBMH_RB_SA1_REMAP_CNTL                                                                      0x5ba3
+#define regGRBMH_RB_SA1_REMAP_CNTL_BASE_IDX                                                             1
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvmsharedpfdec
+// base address: 0xa000
+#define regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB                                                     0x15a0
+#define regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_BASE_IDX                                            0
+#define regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB                                                     0x15a1
+#define regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_BASE_IDX                                            0
+#define regGCUTCL2_ICG_CTRL                                                                             0x15a2
+#define regGCUTCL2_ICG_CTRL_BASE_IDX                                                                    0
+#define regGCMC_SHARED_ACTIVE_FCN_ID                                                                    0x15a3
+#define regGCMC_SHARED_ACTIVE_FCN_ID_BASE_IDX                                                           0
+#define regGCUTCL2_CGTT_BUSY_CTRL                                                                       0x15a4
+#define regGCUTCL2_CGTT_BUSY_CTRL_BASE_IDX                                                              0
+#define regGCUTCL2_GROUP_RET_FAULT_STATUS                                                               0x15a5
+#define regGCUTCL2_GROUP_RET_FAULT_STATUS_BASE_IDX                                                      0
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvml2pfdec
+// base address: 0xa090
+#define regGCVM_L2_CNTL                                                                                 0x15c4
+#define regGCVM_L2_CNTL_BASE_IDX                                                                        0
+#define regGCVM_L2_CNTL2                                                                                0x15c5
+#define regGCVM_L2_CNTL2_BASE_IDX                                                                       0
+#define regGCVM_L2_CNTL3                                                                                0x15c6
+#define regGCVM_L2_CNTL3_BASE_IDX                                                                       0
+#define regGCVM_L2_STATUS                                                                               0x15c7
+#define regGCVM_L2_STATUS_BASE_IDX                                                                      0
+#define regGCVM_DUMMY_PAGE_FAULT_CNTL                                                                   0x15c8
+#define regGCVM_DUMMY_PAGE_FAULT_CNTL_BASE_IDX                                                          0
+#define regGCVM_DUMMY_PAGE_FAULT_ADDR_LO32                                                              0x15c9
+#define regGCVM_DUMMY_PAGE_FAULT_ADDR_LO32_BASE_IDX                                                     0
+#define regGCVM_DUMMY_PAGE_FAULT_ADDR_HI32                                                              0x15ca
+#define regGCVM_DUMMY_PAGE_FAULT_ADDR_HI32_BASE_IDX                                                     0
+#define regGCVM_INVALIDATE_CNTL                                                                         0x15cb
+#define regGCVM_INVALIDATE_CNTL_BASE_IDX                                                                0
+#define regGCVM_L2_PROTECTION_FAULT_CNTL_LO32                                                           0x15cc
+#define regGCVM_L2_PROTECTION_FAULT_CNTL_LO32_BASE_IDX                                                  0
+#define regGCVM_L2_PROTECTION_FAULT_CNTL_HI32                                                           0x15cd
+#define regGCVM_L2_PROTECTION_FAULT_CNTL_HI32_BASE_IDX                                                  0
+#define regGCVM_L2_PROTECTION_FAULT_CNTL2                                                               0x15ce
+#define regGCVM_L2_PROTECTION_FAULT_CNTL2_BASE_IDX                                                      0
+#define regGCVM_L2_PROTECTION_FAULT_MM_CNTL3                                                            0x15cf
+#define regGCVM_L2_PROTECTION_FAULT_MM_CNTL3_BASE_IDX                                                   0
+#define regGCVM_L2_PROTECTION_FAULT_MM_CNTL4                                                            0x15d0
+#define regGCVM_L2_PROTECTION_FAULT_MM_CNTL4_BASE_IDX                                                   0
+#define regGCVM_L2_PROTECTION_FAULT_STATUS_LO32                                                         0x15d1
+#define regGCVM_L2_PROTECTION_FAULT_STATUS_LO32_BASE_IDX                                                0
+#define regGCVM_L2_PROTECTION_FAULT_STATUS_HI32                                                         0x15d2
+#define regGCVM_L2_PROTECTION_FAULT_STATUS_HI32_BASE_IDX                                                0
+#define regGCVM_L2_PROTECTION_FAULT_ADDR_LO32                                                           0x15d3
+#define regGCVM_L2_PROTECTION_FAULT_ADDR_LO32_BASE_IDX                                                  0
+#define regGCVM_L2_PROTECTION_FAULT_ADDR_HI32                                                           0x15d4
+#define regGCVM_L2_PROTECTION_FAULT_ADDR_HI32_BASE_IDX                                                  0
+#define regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32                                                   0x15d5
+#define regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32_BASE_IDX                                          0
+#define regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32                                                   0x15d6
+#define regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32_BASE_IDX                                          0
+#define regGCVM_DEBUG                                                                                   0x15d7
+#define regGCVM_DEBUG_BASE_IDX                                                                          0
+#define regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32                                             0x15d8
+#define regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32_BASE_IDX                                    0
+#define regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32                                             0x15d9
+#define regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32_BASE_IDX                                    0
+#define regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32                                            0x15da
+#define regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32_BASE_IDX                                   0
+#define regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32                                            0x15db
+#define regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32_BASE_IDX                                   0
+#define regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32                                                0x15dc
+#define regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32_BASE_IDX                                       0
+#define regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32                                                0x15dd
+#define regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32_BASE_IDX                                       0
+#define regGCVM_L2_CNTL4                                                                                0x15de
+#define regGCVM_L2_CNTL4_BASE_IDX                                                                       0
+#define regGCVM_L2_MM_GROUP_RT_CLASSES                                                                  0x15df
+#define regGCVM_L2_MM_GROUP_RT_CLASSES_BASE_IDX                                                         0
+#define regGCVM_L2_BANK_SELECT_RESERVED_CID                                                             0x15e0
+#define regGCVM_L2_BANK_SELECT_RESERVED_CID_BASE_IDX                                                    0
+#define regGCVM_L2_BANK_SELECT_RESERVED_CID2                                                            0x15e1
+#define regGCVM_L2_BANK_SELECT_RESERVED_CID2_BASE_IDX                                                   0
+#define regGCVM_L2_CACHE_PARITY_CNTL                                                                    0x15e2
+#define regGCVM_L2_CACHE_PARITY_CNTL_BASE_IDX                                                           0
+#define regGCVM_L2_ICG_CTRL                                                                             0x15e3
+#define regGCVM_L2_ICG_CTRL_BASE_IDX                                                                    0
+#define regGCVM_L2_CNTL5                                                                                0x15e4
+#define regGCVM_L2_CNTL5_BASE_IDX                                                                       0
+#define regGCVM_L2_GCR_CNTL                                                                             0x15e5
+#define regGCVM_L2_GCR_CNTL_BASE_IDX                                                                    0
+#define regGCVML2_WALKER_MACRO_THROTTLE_TIME                                                            0x15e6
+#define regGCVML2_WALKER_MACRO_THROTTLE_TIME_BASE_IDX                                                   0
+#define regGCVML2_WALKER_MACRO_THROTTLE_FETCH_LIMIT                                                     0x15e7
+#define regGCVML2_WALKER_MACRO_THROTTLE_FETCH_LIMIT_BASE_IDX                                            0
+#define regGCVML2_WALKER_MICRO_THROTTLE_TIME                                                            0x15e8
+#define regGCVML2_WALKER_MICRO_THROTTLE_TIME_BASE_IDX                                                   0
+#define regGCVML2_WALKER_MICRO_THROTTLE_FETCH_LIMIT                                                     0x15e9
+#define regGCVML2_WALKER_MICRO_THROTTLE_FETCH_LIMIT_BASE_IDX                                            0
+#define regGCVM_L2_CGTT_BUSY_CTRL                                                                       0x15ea
+#define regGCVM_L2_CGTT_BUSY_CTRL_BASE_IDX                                                              0
+#define regGCVM_L2_PTE_CACHE_DUMP_CNTL                                                                  0x15eb
+#define regGCVM_L2_PTE_CACHE_DUMP_CNTL_BASE_IDX                                                         0
+#define regGCVM_L2_PTE_CACHE_DUMP_READ                                                                  0x15ec
+#define regGCVM_L2_PTE_CACHE_DUMP_READ_BASE_IDX                                                         0
+#define regGCVML2_MAIN_DEBUG                                                                            0x15ed
+#define regGCVML2_MAIN_DEBUG_BASE_IDX                                                                   0
+#define regGCVML2_WALKER_DEBUG                                                                          0x15ee
+#define regGCVML2_WALKER_DEBUG_BASE_IDX                                                                 0
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ADDR_LO32                                        0x15ef
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ADDR_LO32_BASE_IDX                               0
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ADDR_HI32                                        0x15f0
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ADDR_HI32_BASE_IDX                               0
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR                                             0x15f1
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR_BASE_IDX                                    0
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ADDR_LO32                                       0x15f2
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ADDR_LO32_BASE_IDX                              0
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ADDR_HI32                                       0x15f3
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ADDR_HI32_BASE_IDX                              0
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR                                            0x15f4
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR_BASE_IDX                                   0
+#define regGCVM_L2_BANK_SELECT_MASKS                                                                    0x15f5
+#define regGCVM_L2_BANK_SELECT_MASKS_BASE_IDX                                                           0
+#define regGCUTCL2_CREDIT_SAFETY_GROUP_RET_CDC                                                          0x15f6
+#define regGCUTCL2_CREDIT_SAFETY_GROUP_RET_CDC_BASE_IDX                                                 0
+#define regGCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC                                               0x15f7
+#define regGCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC_BASE_IDX                                      0
+#define regGCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC                                             0x15f8
+#define regGCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC_BASE_IDX                                    0
+#define regGCVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT                                                      0x15f9
+#define regGCVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT_BASE_IDX                                             0
+#define regGCVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ                                                      0x15fa
+#define regGCVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ_BASE_IDX                                             0
+#define regGCVM_L2_UTCL2_FED_VMID_STS                                                                   0x15fb
+#define regGCVM_L2_UTCL2_FED_VMID_STS_BASE_IDX                                                          0
+#define regGCVM_L2_UTCL2_FED_ACK_REG                                                                    0x15fc
+#define regGCVM_L2_UTCL2_FED_ACK_REG_BASE_IDX                                                           0
+#define regGCVML2_IH_FAULT_INTERRUPT_CNTL                                                               0x15fd
+#define regGCVML2_IH_FAULT_INTERRUPT_CNTL_BASE_IDX                                                      0
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcl2tlbpfdec
+// base address: 0xa1d0
+#define regGC_L2TLB_TLB0_STATUS                                                                         0x1615
+#define regGC_L2TLB_TLB0_STATUS_BASE_IDX                                                                0
+#define regGC_L2TLB_TLB0_RETRY_CNTL                                                                     0x1616
+#define regGC_L2TLB_TLB0_RETRY_CNTL_BASE_IDX                                                            0
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvmsharedvcdec
+// base address: 0xa210
+#define regGCMC_VM_FB_LOCATION_BASE_LO32                                                                0x1624
+#define regGCMC_VM_FB_LOCATION_BASE_LO32_BASE_IDX                                                       0
+#define regGCMC_VM_FB_LOCATION_BASE_HI32                                                                0x1625
+#define regGCMC_VM_FB_LOCATION_BASE_HI32_BASE_IDX                                                       0
+#define regGCMC_VM_FB_LOCATION_TOP_LO32                                                                 0x1626
+#define regGCMC_VM_FB_LOCATION_TOP_LO32_BASE_IDX                                                        0
+#define regGCMC_VM_FB_LOCATION_TOP_HI32                                                                 0x1627
+#define regGCMC_VM_FB_LOCATION_TOP_HI32_BASE_IDX                                                        0
+#define regGCMC_VM_AGP_TOP_LO32                                                                         0x1628
+#define regGCMC_VM_AGP_TOP_LO32_BASE_IDX                                                                0
+#define regGCMC_VM_AGP_TOP_HI32                                                                         0x1629
+#define regGCMC_VM_AGP_TOP_HI32_BASE_IDX                                                                0
+#define regGCMC_VM_AGP_BOT_LO32                                                                         0x162a
+#define regGCMC_VM_AGP_BOT_LO32_BASE_IDX                                                                0
+#define regGCMC_VM_AGP_BOT_HI32                                                                         0x162b
+#define regGCMC_VM_AGP_BOT_HI32_BASE_IDX                                                                0
+#define regGCMC_VM_AGP_BASE_LO32                                                                        0x162c
+#define regGCMC_VM_AGP_BASE_LO32_BASE_IDX                                                               0
+#define regGCMC_VM_AGP_BASE_HI32                                                                        0x162d
+#define regGCMC_VM_AGP_BASE_HI32_BASE_IDX                                                               0
+#define regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32                                                        0x162e
+#define regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32_BASE_IDX                                               0
+#define regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32                                                        0x162f
+#define regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32_BASE_IDX                                               0
+#define regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32                                                       0x1630
+#define regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32_BASE_IDX                                              0
+#define regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32                                                       0x1631
+#define regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32_BASE_IDX                                              0
+#define regGCMC_VM_MX_L1_TLB_CNTL                                                                       0x1632
+#define regGCMC_VM_MX_L1_TLB_CNTL_BASE_IDX                                                              0
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvml2vcdec
+// base address: 0xa250
+#define regGCVM_CONTEXT0_CNTL                                                                           0x1634
+#define regGCVM_CONTEXT0_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT1_CNTL                                                                           0x1635
+#define regGCVM_CONTEXT1_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT2_CNTL                                                                           0x1636
+#define regGCVM_CONTEXT2_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT3_CNTL                                                                           0x1637
+#define regGCVM_CONTEXT3_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT4_CNTL                                                                           0x1638
+#define regGCVM_CONTEXT4_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT5_CNTL                                                                           0x1639
+#define regGCVM_CONTEXT5_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT6_CNTL                                                                           0x163a
+#define regGCVM_CONTEXT6_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT7_CNTL                                                                           0x163b
+#define regGCVM_CONTEXT7_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT8_CNTL                                                                           0x163c
+#define regGCVM_CONTEXT8_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT9_CNTL                                                                           0x163d
+#define regGCVM_CONTEXT9_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT10_CNTL                                                                          0x163e
+#define regGCVM_CONTEXT10_CNTL_BASE_IDX                                                                 0
+#define regGCVM_CONTEXT11_CNTL                                                                          0x163f
+#define regGCVM_CONTEXT11_CNTL_BASE_IDX                                                                 0
+#define regGCVM_CONTEXT12_CNTL                                                                          0x1640
+#define regGCVM_CONTEXT12_CNTL_BASE_IDX                                                                 0
+#define regGCVM_CONTEXT13_CNTL                                                                          0x1641
+#define regGCVM_CONTEXT13_CNTL_BASE_IDX                                                                 0
+#define regGCVM_CONTEXT14_CNTL                                                                          0x1642
+#define regGCVM_CONTEXT14_CNTL_BASE_IDX                                                                 0
+#define regGCVM_CONTEXT15_CNTL                                                                          0x1643
+#define regGCVM_CONTEXT15_CNTL_BASE_IDX                                                                 0
+#define regGCVM_CONTEXTS_DISABLE                                                                        0x1644
+#define regGCVM_CONTEXTS_DISABLE_BASE_IDX                                                               0
+#define regGCVM_INVALIDATE_ENG0_SEM                                                                     0x1645
+#define regGCVM_INVALIDATE_ENG0_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG1_SEM                                                                     0x1646
+#define regGCVM_INVALIDATE_ENG1_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG2_SEM                                                                     0x1647
+#define regGCVM_INVALIDATE_ENG2_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG3_SEM                                                                     0x1648
+#define regGCVM_INVALIDATE_ENG3_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG4_SEM                                                                     0x1649
+#define regGCVM_INVALIDATE_ENG4_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG5_SEM                                                                     0x164a
+#define regGCVM_INVALIDATE_ENG5_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG6_SEM                                                                     0x164b
+#define regGCVM_INVALIDATE_ENG6_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG7_SEM                                                                     0x164c
+#define regGCVM_INVALIDATE_ENG7_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG8_SEM                                                                     0x164d
+#define regGCVM_INVALIDATE_ENG8_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG9_SEM                                                                     0x164e
+#define regGCVM_INVALIDATE_ENG9_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG10_SEM                                                                    0x164f
+#define regGCVM_INVALIDATE_ENG10_SEM_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG11_SEM                                                                    0x1650
+#define regGCVM_INVALIDATE_ENG11_SEM_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG12_SEM                                                                    0x1651
+#define regGCVM_INVALIDATE_ENG12_SEM_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG13_SEM                                                                    0x1652
+#define regGCVM_INVALIDATE_ENG13_SEM_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG14_SEM                                                                    0x1653
+#define regGCVM_INVALIDATE_ENG14_SEM_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG15_SEM                                                                    0x1654
+#define regGCVM_INVALIDATE_ENG15_SEM_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG16_SEM                                                                    0x1655
+#define regGCVM_INVALIDATE_ENG16_SEM_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG17_SEM                                                                    0x1656
+#define regGCVM_INVALIDATE_ENG17_SEM_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG0_REQ                                                                     0x1657
+#define regGCVM_INVALIDATE_ENG0_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG1_REQ                                                                     0x1658
+#define regGCVM_INVALIDATE_ENG1_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG2_REQ                                                                     0x1659
+#define regGCVM_INVALIDATE_ENG2_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG3_REQ                                                                     0x165a
+#define regGCVM_INVALIDATE_ENG3_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG4_REQ                                                                     0x165b
+#define regGCVM_INVALIDATE_ENG4_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG5_REQ                                                                     0x165c
+#define regGCVM_INVALIDATE_ENG5_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG6_REQ                                                                     0x165d
+#define regGCVM_INVALIDATE_ENG6_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG7_REQ                                                                     0x165e
+#define regGCVM_INVALIDATE_ENG7_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG8_REQ                                                                     0x165f
+#define regGCVM_INVALIDATE_ENG8_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG9_REQ                                                                     0x1660
+#define regGCVM_INVALIDATE_ENG9_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG10_REQ                                                                    0x1661
+#define regGCVM_INVALIDATE_ENG10_REQ_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG11_REQ                                                                    0x1662
+#define regGCVM_INVALIDATE_ENG11_REQ_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG12_REQ                                                                    0x1663
+#define regGCVM_INVALIDATE_ENG12_REQ_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG13_REQ                                                                    0x1664
+#define regGCVM_INVALIDATE_ENG13_REQ_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG14_REQ                                                                    0x1665
+#define regGCVM_INVALIDATE_ENG14_REQ_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG15_REQ                                                                    0x1666
+#define regGCVM_INVALIDATE_ENG15_REQ_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG16_REQ                                                                    0x1667
+#define regGCVM_INVALIDATE_ENG16_REQ_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG17_REQ                                                                    0x1668
+#define regGCVM_INVALIDATE_ENG17_REQ_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG0_ACK                                                                     0x1669
+#define regGCVM_INVALIDATE_ENG0_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG1_ACK                                                                     0x166a
+#define regGCVM_INVALIDATE_ENG1_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG2_ACK                                                                     0x166b
+#define regGCVM_INVALIDATE_ENG2_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG3_ACK                                                                     0x166c
+#define regGCVM_INVALIDATE_ENG3_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG4_ACK                                                                     0x166d
+#define regGCVM_INVALIDATE_ENG4_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG5_ACK                                                                     0x166e
+#define regGCVM_INVALIDATE_ENG5_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG6_ACK                                                                     0x166f
+#define regGCVM_INVALIDATE_ENG6_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG7_ACK                                                                     0x1670
+#define regGCVM_INVALIDATE_ENG7_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG8_ACK                                                                     0x1671
+#define regGCVM_INVALIDATE_ENG8_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG9_ACK                                                                     0x1672
+#define regGCVM_INVALIDATE_ENG9_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG10_ACK                                                                    0x1673
+#define regGCVM_INVALIDATE_ENG10_ACK_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG11_ACK                                                                    0x1674
+#define regGCVM_INVALIDATE_ENG11_ACK_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG12_ACK                                                                    0x1675
+#define regGCVM_INVALIDATE_ENG12_ACK_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG13_ACK                                                                    0x1676
+#define regGCVM_INVALIDATE_ENG13_ACK_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG14_ACK                                                                    0x1677
+#define regGCVM_INVALIDATE_ENG14_ACK_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG15_ACK                                                                    0x1678
+#define regGCVM_INVALIDATE_ENG15_ACK_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG16_ACK                                                                    0x1679
+#define regGCVM_INVALIDATE_ENG16_ACK_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG17_ACK                                                                    0x167a
+#define regGCVM_INVALIDATE_ENG17_ACK_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32                                                         0x167b
+#define regGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32                                                         0x167c
+#define regGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32                                                         0x167d
+#define regGCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG1_ADDR_RANGE_HI32                                                         0x167e
+#define regGCVM_INVALIDATE_ENG1_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG2_ADDR_RANGE_LO32                                                         0x167f
+#define regGCVM_INVALIDATE_ENG2_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG2_ADDR_RANGE_HI32                                                         0x1680
+#define regGCVM_INVALIDATE_ENG2_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG3_ADDR_RANGE_LO32                                                         0x1681
+#define regGCVM_INVALIDATE_ENG3_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG3_ADDR_RANGE_HI32                                                         0x1682
+#define regGCVM_INVALIDATE_ENG3_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG4_ADDR_RANGE_LO32                                                         0x1683
+#define regGCVM_INVALIDATE_ENG4_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG4_ADDR_RANGE_HI32                                                         0x1684
+#define regGCVM_INVALIDATE_ENG4_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG5_ADDR_RANGE_LO32                                                         0x1685
+#define regGCVM_INVALIDATE_ENG5_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG5_ADDR_RANGE_HI32                                                         0x1686
+#define regGCVM_INVALIDATE_ENG5_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG6_ADDR_RANGE_LO32                                                         0x1687
+#define regGCVM_INVALIDATE_ENG6_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG6_ADDR_RANGE_HI32                                                         0x1688
+#define regGCVM_INVALIDATE_ENG6_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG7_ADDR_RANGE_LO32                                                         0x1689
+#define regGCVM_INVALIDATE_ENG7_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG7_ADDR_RANGE_HI32                                                         0x168a
+#define regGCVM_INVALIDATE_ENG7_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG8_ADDR_RANGE_LO32                                                         0x168b
+#define regGCVM_INVALIDATE_ENG8_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG8_ADDR_RANGE_HI32                                                         0x168c
+#define regGCVM_INVALIDATE_ENG8_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG9_ADDR_RANGE_LO32                                                         0x168d
+#define regGCVM_INVALIDATE_ENG9_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG9_ADDR_RANGE_HI32                                                         0x168e
+#define regGCVM_INVALIDATE_ENG9_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG10_ADDR_RANGE_LO32                                                        0x168f
+#define regGCVM_INVALIDATE_ENG10_ADDR_RANGE_LO32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG10_ADDR_RANGE_HI32                                                        0x1690
+#define regGCVM_INVALIDATE_ENG10_ADDR_RANGE_HI32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG11_ADDR_RANGE_LO32                                                        0x1691
+#define regGCVM_INVALIDATE_ENG11_ADDR_RANGE_LO32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG11_ADDR_RANGE_HI32                                                        0x1692
+#define regGCVM_INVALIDATE_ENG11_ADDR_RANGE_HI32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG12_ADDR_RANGE_LO32                                                        0x1693
+#define regGCVM_INVALIDATE_ENG12_ADDR_RANGE_LO32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG12_ADDR_RANGE_HI32                                                        0x1694
+#define regGCVM_INVALIDATE_ENG12_ADDR_RANGE_HI32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG13_ADDR_RANGE_LO32                                                        0x1695
+#define regGCVM_INVALIDATE_ENG13_ADDR_RANGE_LO32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG13_ADDR_RANGE_HI32                                                        0x1696
+#define regGCVM_INVALIDATE_ENG13_ADDR_RANGE_HI32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG14_ADDR_RANGE_LO32                                                        0x1697
+#define regGCVM_INVALIDATE_ENG14_ADDR_RANGE_LO32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG14_ADDR_RANGE_HI32                                                        0x1698
+#define regGCVM_INVALIDATE_ENG14_ADDR_RANGE_HI32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG15_ADDR_RANGE_LO32                                                        0x1699
+#define regGCVM_INVALIDATE_ENG15_ADDR_RANGE_LO32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG15_ADDR_RANGE_HI32                                                        0x169a
+#define regGCVM_INVALIDATE_ENG15_ADDR_RANGE_HI32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG16_ADDR_RANGE_LO32                                                        0x169b
+#define regGCVM_INVALIDATE_ENG16_ADDR_RANGE_LO32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG16_ADDR_RANGE_HI32                                                        0x169c
+#define regGCVM_INVALIDATE_ENG16_ADDR_RANGE_HI32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG17_ADDR_RANGE_LO32                                                        0x169d
+#define regGCVM_INVALIDATE_ENG17_ADDR_RANGE_LO32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG17_ADDR_RANGE_HI32                                                        0x169e
+#define regGCVM_INVALIDATE_ENG17_ADDR_RANGE_HI32_BASE_IDX                                               0
+#define regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32                                                      0x169f
+#define regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32                                                      0x16a0
+#define regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32                                                      0x16a1
+#define regGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32                                                      0x16a2
+#define regGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32                                                      0x16a3
+#define regGCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32                                                      0x16a4
+#define regGCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32                                                      0x16a5
+#define regGCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32                                                      0x16a6
+#define regGCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32                                                      0x16a7
+#define regGCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32                                                      0x16a8
+#define regGCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32                                                      0x16a9
+#define regGCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32                                                      0x16aa
+#define regGCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32                                                      0x16ab
+#define regGCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32                                                      0x16ac
+#define regGCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32                                                      0x16ad
+#define regGCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32                                                      0x16ae
+#define regGCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32                                                      0x16af
+#define regGCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32                                                      0x16b0
+#define regGCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32                                                      0x16b1
+#define regGCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32                                                      0x16b2
+#define regGCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32                                                     0x16b3
+#define regGCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32                                                     0x16b4
+#define regGCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32                                                     0x16b5
+#define regGCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32                                                     0x16b6
+#define regGCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32                                                     0x16b7
+#define regGCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32                                                     0x16b8
+#define regGCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32                                                     0x16b9
+#define regGCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32                                                     0x16ba
+#define regGCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32                                                     0x16bb
+#define regGCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32                                                     0x16bc
+#define regGCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32                                                     0x16bd
+#define regGCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32                                                     0x16be
+#define regGCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32                                                     0x16bf
+#define regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32                                                     0x16c0
+#define regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32                                                     0x16c1
+#define regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32                                                     0x16c2
+#define regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32                                                     0x16c3
+#define regGCVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32                                                     0x16c4
+#define regGCVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32                                                     0x16c5
+#define regGCVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32                                                     0x16c6
+#define regGCVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32                                                     0x16c7
+#define regGCVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32                                                     0x16c8
+#define regGCVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32                                                     0x16c9
+#define regGCVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32                                                     0x16ca
+#define regGCVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32                                                     0x16cb
+#define regGCVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32                                                     0x16cc
+#define regGCVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32                                                     0x16cd
+#define regGCVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32                                                     0x16ce
+#define regGCVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32                                                     0x16cf
+#define regGCVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32                                                     0x16d0
+#define regGCVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32                                                     0x16d1
+#define regGCVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32                                                     0x16d2
+#define regGCVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32                                                    0x16d3
+#define regGCVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           0
+#define regGCVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32                                                    0x16d4
+#define regGCVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           0
+#define regGCVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32                                                    0x16d5
+#define regGCVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           0
+#define regGCVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32                                                    0x16d6
+#define regGCVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           0
+#define regGCVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32                                                    0x16d7
+#define regGCVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           0
+#define regGCVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32                                                    0x16d8
+#define regGCVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           0
+#define regGCVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32                                                    0x16d9
+#define regGCVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           0
+#define regGCVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32                                                    0x16da
+#define regGCVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           0
+#define regGCVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32                                                    0x16db
+#define regGCVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           0
+#define regGCVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32                                                    0x16dc
+#define regGCVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           0
+#define regGCVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32                                                    0x16dd
+#define regGCVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           0
+#define regGCVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32                                                    0x16de
+#define regGCVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           0
+#define regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32                                                       0x16df
+#define regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32                                                       0x16e0
+#define regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32                                                       0x16e1
+#define regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32                                                       0x16e2
+#define regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32                                                       0x16e3
+#define regGCVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32                                                       0x16e4
+#define regGCVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32                                                       0x16e5
+#define regGCVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32                                                       0x16e6
+#define regGCVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32                                                       0x16e7
+#define regGCVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32                                                       0x16e8
+#define regGCVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32                                                       0x16e9
+#define regGCVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32                                                       0x16ea
+#define regGCVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32                                                       0x16eb
+#define regGCVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32                                                       0x16ec
+#define regGCVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32                                                       0x16ed
+#define regGCVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32                                                       0x16ee
+#define regGCVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32                                                       0x16ef
+#define regGCVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32                                                       0x16f0
+#define regGCVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32                                                       0x16f1
+#define regGCVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32                                                       0x16f2
+#define regGCVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32                                                      0x16f3
+#define regGCVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32                                                      0x16f4
+#define regGCVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32                                                      0x16f5
+#define regGCVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32                                                      0x16f6
+#define regGCVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32                                                      0x16f7
+#define regGCVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32                                                      0x16f8
+#define regGCVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32                                                      0x16f9
+#define regGCVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32                                                      0x16fa
+#define regGCVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32                                                      0x16fb
+#define regGCVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32                                                      0x16fc
+#define regGCVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32                                                      0x16fd
+#define regGCVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32                                                      0x16fe
+#define regGCVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                                    0x16ff
+#define regGCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                           0
+#define regGCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x1700
+#define regGCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x1701
+#define regGCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x1702
+#define regGCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x1703
+#define regGCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x1704
+#define regGCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x1705
+#define regGCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x1706
+#define regGCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x1707
+#define regGCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x1708
+#define regGCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x1709
+#define regGCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x170a
+#define regGCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 0
+#define regGCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x170b
+#define regGCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 0
+#define regGCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x170c
+#define regGCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 0
+#define regGCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x170d
+#define regGCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 0
+#define regGCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x170e
+#define regGCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 0
+#define regGCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x170f
+#define regGCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 0
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvmsharedvfdec
+// base address: 0xa5d0
+#define regGCMC_VM_PCIE_ATOMIC_SUPPORTED                                                                0x1714
+#define regGCMC_VM_PCIE_ATOMIC_SUPPORTED_BASE_IDX                                                       0
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcl2tlbvfdec
+// base address: 0xa5e0
+#define regGC_L2TLB_TLB0_EN_RETRY_ON_ATOMIC                                                             0x1718
+#define regGC_L2TLB_TLB0_EN_RETRY_ON_ATOMIC_BASE_IDX                                                    0
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvml2prdec
+// base address: 0x35380
+#define regGCMC_VM_L2_PERFCOUNTER_LO                                                                    0x34e0
+#define regGCMC_VM_L2_PERFCOUNTER_LO_BASE_IDX                                                           1
+#define regGCMC_VM_L2_PERFCOUNTER_HI                                                                    0x34e1
+#define regGCMC_VM_L2_PERFCOUNTER_HI_BASE_IDX                                                           1
+#define regGCUTCL2_PERFCOUNTER_LO                                                                       0x34e2
+#define regGCUTCL2_PERFCOUNTER_LO_BASE_IDX                                                              1
+#define regGCUTCL2_PERFCOUNTER_HI                                                                       0x34e3
+#define regGCUTCL2_PERFCOUNTER_HI_BASE_IDX                                                              1
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcatcl2prdec
+// base address: 0x353c0
+#define regGC_ATC_L2_PERFCOUNTER_LO                                                                     0x34f0
+#define regGC_ATC_L2_PERFCOUNTER_LO_BASE_IDX                                                            1
+#define regGC_ATC_L2_PERFCOUNTER_HI                                                                     0x34f1
+#define regGC_ATC_L2_PERFCOUNTER_HI_BASE_IDX                                                            1
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcl2tlbprdec
+// base address: 0x353f0
+#define regGC_L2TLB_PERFCOUNTER_LO                                                                      0x34fc
+#define regGC_L2TLB_PERFCOUNTER_LO_BASE_IDX                                                             1
+#define regGC_L2TLB_PERFCOUNTER_HI                                                                      0x34fd
+#define regGC_L2TLB_PERFCOUNTER_HI_BASE_IDX                                                             1
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvml2pldec
+// base address: 0x37b00
+#define regGCMC_VM_L2_PERFCOUNTER0_CFG                                                                  0x3ec0
+#define regGCMC_VM_L2_PERFCOUNTER0_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER1_CFG                                                                  0x3ec1
+#define regGCMC_VM_L2_PERFCOUNTER1_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER2_CFG                                                                  0x3ec2
+#define regGCMC_VM_L2_PERFCOUNTER2_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER3_CFG                                                                  0x3ec3
+#define regGCMC_VM_L2_PERFCOUNTER3_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER4_CFG                                                                  0x3ec4
+#define regGCMC_VM_L2_PERFCOUNTER4_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER5_CFG                                                                  0x3ec5
+#define regGCMC_VM_L2_PERFCOUNTER5_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER6_CFG                                                                  0x3ec6
+#define regGCMC_VM_L2_PERFCOUNTER6_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER7_CFG                                                                  0x3ec7
+#define regGCMC_VM_L2_PERFCOUNTER7_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER8_CFG                                                                  0x3ec8
+#define regGCMC_VM_L2_PERFCOUNTER8_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER9_CFG                                                                  0x3ec9
+#define regGCMC_VM_L2_PERFCOUNTER9_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER10_CFG                                                                 0x3eca
+#define regGCMC_VM_L2_PERFCOUNTER10_CFG_BASE_IDX                                                        1
+#define regGCMC_VM_L2_PERFCOUNTER11_CFG                                                                 0x3ecb
+#define regGCMC_VM_L2_PERFCOUNTER11_CFG_BASE_IDX                                                        1
+#define regGCMC_VM_L2_PERFCOUNTER12_CFG                                                                 0x3ecc
+#define regGCMC_VM_L2_PERFCOUNTER12_CFG_BASE_IDX                                                        1
+#define regGCMC_VM_L2_PERFCOUNTER13_CFG                                                                 0x3ecd
+#define regGCMC_VM_L2_PERFCOUNTER13_CFG_BASE_IDX                                                        1
+#define regGCMC_VM_L2_PERFCOUNTER14_CFG                                                                 0x3ece
+#define regGCMC_VM_L2_PERFCOUNTER14_CFG_BASE_IDX                                                        1
+#define regGCMC_VM_L2_PERFCOUNTER15_CFG                                                                 0x3ecf
+#define regGCMC_VM_L2_PERFCOUNTER15_CFG_BASE_IDX                                                        1
+#define regGCMC_VM_L2_PERFCOUNTER_RSLT_CNTL                                                             0x3ed0
+#define regGCMC_VM_L2_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                    1
+#define regGCUTCL2_PERFCOUNTER0_CFG                                                                     0x3ed1
+#define regGCUTCL2_PERFCOUNTER0_CFG_BASE_IDX                                                            1
+#define regGCUTCL2_PERFCOUNTER1_CFG                                                                     0x3ed2
+#define regGCUTCL2_PERFCOUNTER1_CFG_BASE_IDX                                                            1
+#define regGCUTCL2_PERFCOUNTER2_CFG                                                                     0x3ed3
+#define regGCUTCL2_PERFCOUNTER2_CFG_BASE_IDX                                                            1
+#define regGCUTCL2_PERFCOUNTER3_CFG                                                                     0x3ed4
+#define regGCUTCL2_PERFCOUNTER3_CFG_BASE_IDX                                                            1
+#define regGCUTCL2_PERFCOUNTER_RSLT_CNTL                                                                0x3ed5
+#define regGCUTCL2_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                       1
+#define regGCUTCL2_PERFCOUNTER_EVENTS_GROUP_SELECT                                                      0x3ed6
+#define regGCUTCL2_PERFCOUNTER_EVENTS_GROUP_SELECT_BASE_IDX                                             1
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcatcl2pldec
+// base address: 0x37b60
+#define regGC_ATC_L2_PERFCOUNTER0_CFG                                                                   0x3ed8
+#define regGC_ATC_L2_PERFCOUNTER0_CFG_BASE_IDX                                                          1
+#define regGC_ATC_L2_PERFCOUNTER1_CFG                                                                   0x3ed9
+#define regGC_ATC_L2_PERFCOUNTER1_CFG_BASE_IDX                                                          1
+#define regGC_ATC_L2_PERFCOUNTER2_CFG                                                                   0x3eda
+#define regGC_ATC_L2_PERFCOUNTER2_CFG_BASE_IDX                                                          1
+#define regGC_ATC_L2_PERFCOUNTER3_CFG                                                                   0x3edb
+#define regGC_ATC_L2_PERFCOUNTER3_CFG_BASE_IDX                                                          1
+#define regGC_ATC_L2_PERFCOUNTER4_CFG                                                                   0x3edc
+#define regGC_ATC_L2_PERFCOUNTER4_CFG_BASE_IDX                                                          1
+#define regGC_ATC_L2_PERFCOUNTER5_CFG                                                                   0x3edd
+#define regGC_ATC_L2_PERFCOUNTER5_CFG_BASE_IDX                                                          1
+#define regGC_ATC_L2_PERFCOUNTER6_CFG                                                                   0x3ede
+#define regGC_ATC_L2_PERFCOUNTER6_CFG_BASE_IDX                                                          1
+#define regGC_ATC_L2_PERFCOUNTER7_CFG                                                                   0x3edf
+#define regGC_ATC_L2_PERFCOUNTER7_CFG_BASE_IDX                                                          1
+#define regGC_ATC_L2_PERFCOUNTER8_CFG                                                                   0x3ee0
+#define regGC_ATC_L2_PERFCOUNTER8_CFG_BASE_IDX                                                          1
+#define regGC_ATC_L2_PERFCOUNTER9_CFG                                                                   0x3ee1
+#define regGC_ATC_L2_PERFCOUNTER9_CFG_BASE_IDX                                                          1
+#define regGC_ATC_L2_PERFCOUNTER10_CFG                                                                  0x3ee2
+#define regGC_ATC_L2_PERFCOUNTER10_CFG_BASE_IDX                                                         1
+#define regGC_ATC_L2_PERFCOUNTER11_CFG                                                                  0x3ee3
+#define regGC_ATC_L2_PERFCOUNTER11_CFG_BASE_IDX                                                         1
+#define regGC_ATC_L2_PERFCOUNTER12_CFG                                                                  0x3ee4
+#define regGC_ATC_L2_PERFCOUNTER12_CFG_BASE_IDX                                                         1
+#define regGC_ATC_L2_PERFCOUNTER13_CFG                                                                  0x3ee5
+#define regGC_ATC_L2_PERFCOUNTER13_CFG_BASE_IDX                                                         1
+#define regGC_ATC_L2_PERFCOUNTER14_CFG                                                                  0x3ee6
+#define regGC_ATC_L2_PERFCOUNTER14_CFG_BASE_IDX                                                         1
+#define regGC_ATC_L2_PERFCOUNTER15_CFG                                                                  0x3ee7
+#define regGC_ATC_L2_PERFCOUNTER15_CFG_BASE_IDX                                                         1
+#define regGC_ATC_L2_PERFCOUNTER_RSLT_CNTL                                                              0x3ee8
+#define regGC_ATC_L2_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                     1
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcl2tlbpldec
+// base address: 0x37c50
+#define regGC_L2TLB_PERFCOUNTER0_CFG                                                                    0x3f14
+#define regGC_L2TLB_PERFCOUNTER0_CFG_BASE_IDX                                                           1
+#define regGC_L2TLB_PERFCOUNTER1_CFG                                                                    0x3f15
+#define regGC_L2TLB_PERFCOUNTER1_CFG_BASE_IDX                                                           1
+#define regGC_L2TLB_PERFCOUNTER2_CFG                                                                    0x3f16
+#define regGC_L2TLB_PERFCOUNTER2_CFG_BASE_IDX                                                           1
+#define regGC_L2TLB_PERFCOUNTER3_CFG                                                                    0x3f17
+#define regGC_L2TLB_PERFCOUNTER3_CFG_BASE_IDX                                                           1
+#define regGC_L2TLB_PERFCOUNTER_RSLT_CNTL                                                               0x3f18
+#define regGC_L2TLB_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                      1
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvmsharedhvdec
+// base address: 0x3e800
+#define regGCVM_PCIE_ATS_CNTL                                                                           0x5a00
+#define regGCVM_PCIE_ATS_CNTL_BASE_IDX                                                                  1
+#define regGCMC_VM_NB_MMIOBASE                                                                          0x5a01
+#define regGCMC_VM_NB_MMIOBASE_BASE_IDX                                                                 1
+#define regGCMC_VM_NB_MMIOLIMIT                                                                         0x5a02
+#define regGCMC_VM_NB_MMIOLIMIT_BASE_IDX                                                                1
+#define regGCMC_VM_NB_PCI_CTRL                                                                          0x5a03
+#define regGCMC_VM_NB_PCI_CTRL_BASE_IDX                                                                 1
+#define regGCMC_VM_NB_PCI_ARB                                                                           0x5a04
+#define regGCMC_VM_NB_PCI_ARB_BASE_IDX                                                                  1
+#define regGCMC_VM_NB_TOP_OF_DRAM_SLOT1                                                                 0x5a05
+#define regGCMC_VM_NB_TOP_OF_DRAM_SLOT1_BASE_IDX                                                        1
+#define regGCMC_VM_NB_LOWER_TOP_OF_DRAM2                                                                0x5a06
+#define regGCMC_VM_NB_LOWER_TOP_OF_DRAM2_BASE_IDX                                                       1
+#define regGCMC_VM_NB_UPPER_TOP_OF_DRAM2                                                                0x5a07
+#define regGCMC_VM_NB_UPPER_TOP_OF_DRAM2_BASE_IDX                                                       1
+#define regGCMC_VM_STEERING                                                                             0x5a08
+#define regGCMC_VM_STEERING_BASE_IDX                                                                    1
+#define regGCMC_SHARED_VIRT_RESET_REQ                                                                   0x5a09
+#define regGCMC_SHARED_VIRT_RESET_REQ_BASE_IDX                                                          1
+#define regGCMC_VM_XGMI_LFB_CNTL                                                                        0x5a0a
+#define regGCMC_VM_XGMI_LFB_CNTL_BASE_IDX                                                               1
+#define regGCMC_VM_XGMI_LFB_SIZE                                                                        0x5a0b
+#define regGCMC_VM_XGMI_LFB_SIZE_BASE_IDX                                                               1
+#define regGCMC_VM_HOST_MAPPING                                                                         0x5a0c
+#define regGCMC_VM_HOST_MAPPING_BASE_IDX                                                                1
+#define regGCMC_VM_XGMI_GPUIOV_ENABLE                                                                   0x5a0d
+#define regGCMC_VM_XGMI_GPUIOV_ENABLE_BASE_IDX                                                          1
+#define regGCMC_VM_CACHEABLE_DRAM_ADDRESS_START                                                         0x5a0e
+#define regGCMC_VM_CACHEABLE_DRAM_ADDRESS_START_BASE_IDX                                                1
+#define regGCMC_VM_CACHEABLE_DRAM_ADDRESS_END                                                           0x5a0f
+#define regGCMC_VM_CACHEABLE_DRAM_ADDRESS_END_BASE_IDX                                                  1
+#define regGCMC_VM_LOCAL_SYSMEM_ADDRESS_START                                                           0x5a10
+#define regGCMC_VM_LOCAL_SYSMEM_ADDRESS_START_BASE_IDX                                                  1
+#define regGCMC_VM_LOCAL_SYSMEM_ADDRESS_END                                                             0x5a11
+#define regGCMC_VM_LOCAL_SYSMEM_ADDRESS_END_BASE_IDX                                                    1
+#define regGCMC_VM_LPDDR_ADDRESS_START                                                                  0x5a12
+#define regGCMC_VM_LPDDR_ADDRESS_START_BASE_IDX                                                         1
+#define regGCMC_VM_LPDDR_ADDRESS_END                                                                    0x5a13
+#define regGCMC_VM_LPDDR_ADDRESS_END_BASE_IDX                                                           1
+#define regGCMC_VM_APT_CNTL                                                                             0x5a14
+#define regGCMC_VM_APT_CNTL_BASE_IDX                                                                    1
+#define regGCUTCL2_HARVEST_BYPASS_GROUPS                                                                0x5a15
+#define regGCUTCL2_HARVEST_BYPASS_GROUPS_BASE_IDX                                                       1
+#define regGCUTCL2_ROUTER_DEBUG                                                                         0x5a16
+#define regGCUTCL2_ROUTER_DEBUG_BASE_IDX                                                                1
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcatcl2dec
+// base address: 0x3e9f0
+#define regGC_ATC_L2_CNTL                                                                               0x5a7c
+#define regGC_ATC_L2_CNTL_BASE_IDX                                                                      1
+#define regGC_ATC_L2_CNTL2                                                                              0x5a7d
+#define regGC_ATC_L2_CNTL2_BASE_IDX                                                                     1
+#define regGC_ATC_L2_CACHE_DATA0                                                                        0x5a80
+#define regGC_ATC_L2_CACHE_DATA0_BASE_IDX                                                               1
+#define regGC_ATC_L2_CACHE_DATA1                                                                        0x5a81
+#define regGC_ATC_L2_CACHE_DATA1_BASE_IDX                                                               1
+#define regGC_ATC_L2_CACHE_DATA2                                                                        0x5a82
+#define regGC_ATC_L2_CACHE_DATA2_BASE_IDX                                                               1
+#define regGC_ATC_L2_CACHE_DATA3                                                                        0x5a83
+#define regGC_ATC_L2_CACHE_DATA3_BASE_IDX                                                               1
+#define regGC_ATC_L2_CACHE_DATA4                                                                        0x5a84
+#define regGC_ATC_L2_CACHE_DATA4_BASE_IDX                                                               1
+#define regGC_ATC_L2_CNTL3                                                                              0x5a85
+#define regGC_ATC_L2_CNTL3_BASE_IDX                                                                     1
+#define regGC_ATC_L2_STATUS                                                                             0x5a86
+#define regGC_ATC_L2_STATUS_BASE_IDX                                                                    1
+#define regGC_ATC_L2_MISC_CG                                                                            0x5a87
+#define regGC_ATC_L2_MISC_CG_BASE_IDX                                                                   1
+#define regGC_ATC_L2_ICG_CTRL                                                                           0x5a88
+#define regGC_ATC_L2_ICG_CTRL_BASE_IDX                                                                  1
+#define regGC_ATC_L2_SDPPORT_CTRL                                                                       0x5a8b
+#define regGC_ATC_L2_SDPPORT_CTRL_BASE_IDX                                                              1
+#define regGC_ATC_L2_MISC_DEBUG                                                                         0x5a8c
+#define regGC_ATC_L2_MISC_DEBUG_BASE_IDX                                                                1
+#define regGC_ATC_L2_BANK_SELECT_MASKS                                                                  0x5a8d
+#define regGC_ATC_L2_BANK_SELECT_MASKS_BASE_IDX                                                         1
+#define regGC_ATC_L2_FAULT_CNTL                                                                         0x5a8e
+#define regGC_ATC_L2_FAULT_CNTL_BASE_IDX                                                                1
+#define regGC_ATC_L2_FAULT_STATUS                                                                       0x5a8f
+#define regGC_ATC_L2_FAULT_STATUS_BASE_IDX                                                              1
+#define regGC_ATC_L2_FAULT_ADDR_LO                                                                      0x5a90
+#define regGC_ATC_L2_FAULT_ADDR_LO_BASE_IDX                                                             1
+#define regGC_ATC_L2_FAULT_ADDR_HI                                                                      0x5a91
+#define regGC_ATC_L2_FAULT_ADDR_HI_BASE_IDX                                                             1
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcl2tlbhvdec
+// base address: 0x3eb20
+#define regGC_L2TLB_TMZ_CNTL                                                                            0x5ac9
+#define regGC_L2TLB_TMZ_CNTL_BASE_IDX                                                                   1
+#define regGC_UTCL2_L2TLB_CREDIT_SAFETY_FETCH_RDREQ                                                     0x5aca
+#define regGC_UTCL2_L2TLB_CREDIT_SAFETY_FETCH_RDREQ_BASE_IDX                                            1
+#define regGC_L2TLB_GCC_MTYPE_ENCODING_CNTL                                                             0x5acb
+#define regGC_L2TLB_GCC_MTYPE_ENCODING_CNTL_BASE_IDX                                                    1
+#define regGC_L2TLB_FB_COMP_EN_CNTL                                                                     0x5acc
+#define regGC_L2TLB_FB_COMP_EN_CNTL_BASE_IDX                                                            1
+#define regGC_L2TLB_RETRY_TIMEOUT_STATUS_0                                                              0x5acd
+#define regGC_L2TLB_RETRY_TIMEOUT_STATUS_0_BASE_IDX                                                     1
+#define regGC_L2TLB_RETRY_TIMEOUT_STATUS_1                                                              0x5ace
+#define regGC_L2TLB_RETRY_TIMEOUT_STATUS_1_BASE_IDX                                                     1
+#define regGC_L2TLB_RETRY_TIMEOUT_STATUS_2                                                              0x5acf
+#define regGC_L2TLB_RETRY_TIMEOUT_STATUS_2_BASE_IDX                                                     1
+#define regGC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS                                                 0x5ad0
+#define regGC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS_BASE_IDX                                        1
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvml2pspdec
+// base address: 0x3f900
+#define regGCUTCL2_TRANSLATION_BYPASS_BY_VMID                                                           0x5e40
+#define regGCUTCL2_TRANSLATION_BYPASS_BY_VMID_BASE_IDX                                                  1
+#define regGCVML2_SEC_MASTER                                                                            0x5e41
+#define regGCVML2_SEC_MASTER_BASE_IDX                                                                   1
+#define regGCVM_IOMMU_GPU_HOST_TRANSLATION_ENABLE                                                       0x5e42
+#define regGCVM_IOMMU_GPU_HOST_TRANSLATION_ENABLE_BASE_IDX                                              1
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_CNTL                                                     0x5e43
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_CNTL_BASE_IDX                                            1
+#define regGCVM_IOMMU_CONTROL_REGISTER                                                                  0x5e44
+#define regGCVM_IOMMU_CONTROL_REGISTER_BASE_IDX                                                         1
+#define regGCVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER                                         0x5e45
+#define regGCVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER_BASE_IDX                                1
+#define regGCUTC_TRANSLATION_FAULT_CNTL0                                                                0x5e46
+#define regGCUTC_TRANSLATION_FAULT_CNTL0_BASE_IDX                                                       1
+#define regGCUTC_TRANSLATION_FAULT_CNTL1                                                                0x5e47
+#define regGCUTC_TRANSLATION_FAULT_CNTL1_BASE_IDX                                                       1
+#define regGCUTCL2_COMP_EN_OVERRIDES                                                                    0x5e49
+#define regGCUTCL2_COMP_EN_OVERRIDES_BASE_IDX                                                           1
+#define regGCUTCL2_ROUTER_CNTL                                                                          0x5e4a
+#define regGCUTCL2_ROUTER_CNTL_BASE_IDX                                                                 1
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvmsharedpspdec
+// base address: 0x3f950
+#define regGCMC_VM_FB_SIZE_OFFSET_VF0                                                                   0x5e54
+#define regGCMC_VM_FB_SIZE_OFFSET_VF0_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF1                                                                   0x5e55
+#define regGCMC_VM_FB_SIZE_OFFSET_VF1_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF2                                                                   0x5e56
+#define regGCMC_VM_FB_SIZE_OFFSET_VF2_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF3                                                                   0x5e57
+#define regGCMC_VM_FB_SIZE_OFFSET_VF3_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF4                                                                   0x5e58
+#define regGCMC_VM_FB_SIZE_OFFSET_VF4_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF5                                                                   0x5e59
+#define regGCMC_VM_FB_SIZE_OFFSET_VF5_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF6                                                                   0x5e5a
+#define regGCMC_VM_FB_SIZE_OFFSET_VF6_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF7                                                                   0x5e5b
+#define regGCMC_VM_FB_SIZE_OFFSET_VF7_BASE_IDX                                                          1
+#define regGCMC_VM_FB_OFFSET                                                                            0x5e5c
+#define regGCMC_VM_FB_OFFSET_BASE_IDX                                                                   1
+#define regGCMC_VM_LOCAL_FB_ADDRESS_START                                                               0x5e5d
+#define regGCMC_VM_LOCAL_FB_ADDRESS_START_BASE_IDX                                                      1
+#define regGCMC_VM_LOCAL_FB_ADDRESS_END                                                                 0x5e5e
+#define regGCMC_VM_LOCAL_FB_ADDRESS_END_BASE_IDX                                                        1
+#define regGCMC_VM_LOCAL_FB_ADDRESS_LOCK_CNTL                                                           0x5e5f
+#define regGCMC_VM_LOCAL_FB_ADDRESS_LOCK_CNTL_BASE_IDX                                                  1
+#define regGCUTCL2_MISC_CNTL                                                                            0x5e64
+#define regGCUTCL2_MISC_CNTL_BASE_IDX                                                                   1
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcl2tlbpspdec
+// base address: 0x3fa00
+#define regGC_L2TLB_TLB0_CAM_ECC_CNTL                                                                   0x5e80
+#define regGC_L2TLB_TLB0_CAM_ECC_CNTL_BASE_IDX                                                          1
+
+
+// addressBlock: gfx_se_sqind
+// base address: 0x0
+#define ixSQ_DEBUG_STS_LOCAL                                                                           0x0008
+#define ixSQ_DEBUG_CTRL_LOCAL                                                                          0x0009
+#define ixSQ_WAVE_ACTIVE                                                                               0x000a
+#define ixSQ_WAVE_VALID_AND_IDLE                                                                       0x000b
+#define ixSQ_WAVE_MODE                                                                                 0x0101
+#define ixSQ_WAVE_STATUS                                                                               0x0102
+#define ixSQ_WAVE_STATE_PRIV                                                                           0x0104
+#define ixSQ_WAVE_GPR_ALLOC                                                                            0x0105
+#define ixSQ_WAVE_LDS_ALLOC                                                                            0x0106
+#define ixSQ_WAVE_IB_STS                                                                               0x0107
+#define ixSQ_PERF_SNAPSHOT_DATA                                                                        0x010a
+#define ixSQ_PERF_SNAPSHOT_PC_LO                                                                       0x010b
+#define ixSQ_PERF_SNAPSHOT_PC_HI                                                                       0x010c
+#define ixSQ_WAVE_IB_DBG1                                                                              0x010d
+#define ixSQ_WAVE_FLUSH_IB                                                                             0x010e
+#define ixSQ_PERF_SNAPSHOT_DATA1                                                                       0x010f
+#define ixSQ_PERF_SNAPSHOT_DATA2                                                                       0x0110
+#define ixSQ_WAVE_EXCP_FLAG_PRIV                                                                       0x0111
+#define ixSQ_WAVE_EXCP_FLAG_USER                                                                       0x0112
+#define ixSQ_WAVE_TRAP_CTRL                                                                            0x0113
+#define ixSQ_WAVE_SCRATCH_BASE_LO                                                                      0x0114
+#define ixSQ_WAVE_SCRATCH_BASE_HI                                                                      0x0115
+#define ixSQ_WAVE_HW_ID1                                                                               0x0117
+#define ixSQ_WAVE_HW_ID2                                                                               0x0118
+#define ixSQ_WAVE_SCHED_MODE                                                                           0x011a
+#define ixSQ_WAVE_IB_STS2                                                                              0x011c
+#define ixSQ_WAVE_DVGPR_ALLOC_LO                                                                       0x011f
+#define ixSQ_WAVE_DVGPR_ALLOC_HI                                                                       0x0120
+#define ixSQ_WAVE_XNACK_STATE_PRIV                                                                     0x0121
+#define ixSQ_WAVE_XNACK_MASK                                                                           0x0122
+#define ixSQ_WAVE_PC_LO                                                                                0x0140
+#define ixSQ_WAVE_PC_HI                                                                                0x0141
+#define ixSQ_WAVE_TTMP0                                                                                0x026c
+#define ixSQ_WAVE_TTMP1                                                                                0x026d
+#define ixSQ_WAVE_TTMP2                                                                                0x026e
+#define ixSQ_WAVE_TTMP3                                                                                0x026f
+#define ixSQ_WAVE_TTMP4                                                                                0x0270
+#define ixSQ_WAVE_TTMP5                                                                                0x0271
+#define ixSQ_WAVE_TTMP6                                                                                0x0272
+#define ixSQ_WAVE_TTMP7                                                                                0x0273
+#define ixSQ_WAVE_TTMP8                                                                                0x0274
+#define ixSQ_WAVE_TTMP9                                                                                0x0275
+#define ixSQ_WAVE_TTMP10                                                                               0x0276
+#define ixSQ_WAVE_TTMP11                                                                               0x0277
+#define ixSQ_WAVE_TTMP12                                                                               0x0278
+#define ixSQ_WAVE_TTMP13                                                                               0x0279
+#define ixSQ_WAVE_TTMP14                                                                               0x027a
+#define ixSQ_WAVE_TTMP15                                                                               0x027b
+#define ixSQ_WAVE_M0                                                                                   0x027d
+#define ixSQ_WAVE_EXEC_LO                                                                              0x027e
+#define ixSQ_WAVE_EXEC_HI                                                                              0x027f
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_sh_mask.h
new file mode 100644
index 000000000000..4db680044a83
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_sh_mask.h
@@ -0,0 +1,44620 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _gc_12_1_0_SH_MASK_HEADER
+#define _gc_12_1_0_SH_MASK_HEADER
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmadec
+//SDMA0_SDMA_DEC_START
+#define SDMA0_SDMA_DEC_START__START__SHIFT                                                                    0x0
+#define SDMA0_SDMA_DEC_START__START_MASK                                                                      0xFFFFFFFFL
+//SDMA0_SDMA_MCU_MISC_CNTL
+#define SDMA0_SDMA_MCU_MISC_CNTL__MCU_WAKEUP__SHIFT                                                           0x0
+#define SDMA0_SDMA_MCU_MISC_CNTL__MCU_WAKEUP_MASK                                                             0x00000001L
+//SDMA0_SDMA_UCODE_REV
+#define SDMA0_SDMA_UCODE_REV__P4_CL__SHIFT                                                                    0x0
+#define SDMA0_SDMA_UCODE_REV__DEBUG_VER__SHIFT                                                                0x18
+#define SDMA0_SDMA_UCODE_REV__VARIANT_ID__SHIFT                                                               0x1c
+#define SDMA0_SDMA_UCODE_REV__P4_CL_MASK                                                                      0x00FFFFFFL
+#define SDMA0_SDMA_UCODE_REV__DEBUG_VER_MASK                                                                  0x0F000000L
+#define SDMA0_SDMA_UCODE_REV__VARIANT_ID_MASK                                                                 0xF0000000L
+//SDMA0_SDMA_GLOBAL_TIMESTAMP_LO
+#define SDMA0_SDMA_GLOBAL_TIMESTAMP_LO__DATA__SHIFT                                                           0x0
+#define SDMA0_SDMA_GLOBAL_TIMESTAMP_LO__DATA_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_GLOBAL_TIMESTAMP_HI
+#define SDMA0_SDMA_GLOBAL_TIMESTAMP_HI__DATA__SHIFT                                                           0x0
+#define SDMA0_SDMA_GLOBAL_TIMESTAMP_HI__DATA_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_POWER_CNTL
+#define SDMA0_SDMA_POWER_CNTL__LS_ENABLE__SHIFT                                                               0x8
+#define SDMA0_SDMA_POWER_CNTL__LS_ENABLE_MASK                                                                 0x00000100L
+//SDMA0_SDMA_CNTL
+#define SDMA0_SDMA_CNTL__TRAP_ENABLE__SHIFT                                                                   0x0
+#define SDMA0_SDMA_CNTL__RESERVED__SHIFT                                                                      0x2
+#define SDMA0_SDMA_CNTL__DATA_SWAP_ENABLE__SHIFT                                                              0x3
+#define SDMA0_SDMA_CNTL__FENCE_SWAP_ENABLE__SHIFT                                                             0x4
+#define SDMA0_SDMA_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                         0x5
+#define SDMA0_SDMA_CNTL__PIO_DONE_ACK_ENABLE__SHIFT                                                           0x6
+#define SDMA0_SDMA_CNTL__CP_MES_SYNC_ENABLE__SHIFT                                                            0x7
+#define SDMA0_SDMA_CNTL__TMZ_MIDCMD_PREEMPT_ENABLE__SHIFT                                                     0x8
+#define SDMA0_SDMA_CNTL__MIDCMD_EXPIRE_ENABLE__SHIFT                                                          0x9
+#define SDMA0_SDMA_CNTL__CP_MES_INT_ENABLE__SHIFT                                                             0xa
+#define SDMA0_SDMA_CNTL__PAGE_RETRY_TIMEOUT_INT_ENABLE__SHIFT                                                 0xb
+#define SDMA0_SDMA_CNTL__PAGE_NULL_INT_ENABLE__SHIFT                                                          0xc
+#define SDMA0_SDMA_CNTL__PAGE_FAULT_INT_ENABLE__SHIFT                                                         0xd
+#define SDMA0_SDMA_CNTL__CH_PERFCNT_ENABLE__SHIFT                                                             0x10
+#define SDMA0_SDMA_CNTL__MIDCMD_WORLDSWITCH_ENABLE__SHIFT                                                     0x11
+#define SDMA0_SDMA_CNTL__CTXEMPTY_INT_ENABLE__SHIFT                                                           0x1c
+#define SDMA0_SDMA_CNTL__FROZEN_INT_ENABLE__SHIFT                                                             0x1d
+#define SDMA0_SDMA_CNTL__IB_PREEMPT_INT_ENABLE__SHIFT                                                         0x1e
+#define SDMA0_SDMA_CNTL__RB_PREEMPT_INT_ENABLE__SHIFT                                                         0x1f
+#define SDMA0_SDMA_CNTL__TRAP_ENABLE_MASK                                                                     0x00000001L
+#define SDMA0_SDMA_CNTL__RESERVED_MASK                                                                        0x00000004L
+#define SDMA0_SDMA_CNTL__DATA_SWAP_ENABLE_MASK                                                                0x00000008L
+#define SDMA0_SDMA_CNTL__FENCE_SWAP_ENABLE_MASK                                                               0x00000010L
+#define SDMA0_SDMA_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                           0x00000020L
+#define SDMA0_SDMA_CNTL__PIO_DONE_ACK_ENABLE_MASK                                                             0x00000040L
+#define SDMA0_SDMA_CNTL__CP_MES_SYNC_ENABLE_MASK                                                              0x00000080L
+#define SDMA0_SDMA_CNTL__TMZ_MIDCMD_PREEMPT_ENABLE_MASK                                                       0x00000100L
+#define SDMA0_SDMA_CNTL__MIDCMD_EXPIRE_ENABLE_MASK                                                            0x00000200L
+#define SDMA0_SDMA_CNTL__CP_MES_INT_ENABLE_MASK                                                               0x00000400L
+#define SDMA0_SDMA_CNTL__PAGE_RETRY_TIMEOUT_INT_ENABLE_MASK                                                   0x00000800L
+#define SDMA0_SDMA_CNTL__PAGE_NULL_INT_ENABLE_MASK                                                            0x00001000L
+#define SDMA0_SDMA_CNTL__PAGE_FAULT_INT_ENABLE_MASK                                                           0x00002000L
+#define SDMA0_SDMA_CNTL__CH_PERFCNT_ENABLE_MASK                                                               0x00010000L
+#define SDMA0_SDMA_CNTL__MIDCMD_WORLDSWITCH_ENABLE_MASK                                                       0x00020000L
+#define SDMA0_SDMA_CNTL__CTXEMPTY_INT_ENABLE_MASK                                                             0x10000000L
+#define SDMA0_SDMA_CNTL__FROZEN_INT_ENABLE_MASK                                                               0x20000000L
+#define SDMA0_SDMA_CNTL__IB_PREEMPT_INT_ENABLE_MASK                                                           0x40000000L
+#define SDMA0_SDMA_CNTL__RB_PREEMPT_INT_ENABLE_MASK                                                           0x80000000L
+//SDMA0_SDMA_CHICKEN_BITS
+#define SDMA0_SDMA_CHICKEN_BITS__SRBM_POLL_RETRYING__SHIFT                                                    0x5
+#define SDMA0_SDMA_CHICKEN_BITS__RD_BURST__SHIFT                                                              0x6
+#define SDMA0_SDMA_CHICKEN_BITS__WR_BURST__SHIFT                                                              0x8
+#define SDMA0_SDMA_CHICKEN_BITS__COMBINE_256B_WAIT_CYCLE__SHIFT                                               0xa
+#define SDMA0_SDMA_CHICKEN_BITS__WR_COMBINE_256B_ENABLE__SHIFT                                                0xe
+#define SDMA0_SDMA_CHICKEN_BITS__RD_COMBINE_256B_ENABLE__SHIFT                                                0xf
+#define SDMA0_SDMA_CHICKEN_BITS__OVERLAP_ENABLE__SHIFT                                                        0x10
+#define SDMA0_SDMA_CHICKEN_BITS__T2L_256B_ENABLE__SHIFT                                                       0x12
+#define SDMA0_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_GCR_FGCG__SHIFT                                                0x13
+#define SDMA0_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_GRBM_FGCG__SHIFT                                               0x14
+#define SDMA0_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_CH_FGCG__SHIFT                                                 0x15
+#define SDMA0_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_UTCL2_INVREQ_FGCG__SHIFT                                       0x16
+#define SDMA0_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_UTCL1_FGCG__SHIFT                                              0x17
+#define SDMA0_SDMA_CHICKEN_BITS__CG_STATUS_OUTPUT__SHIFT                                                      0x18
+#define SDMA0_SDMA_CHICKEN_BITS__SW_FREEZE_ENABLE__SHIFT                                                      0x19
+#define SDMA0_SDMA_CHICKEN_BITS__DRAM_ECC_COPY_MODE_CNTL__SHIFT                                               0x1a
+#define SDMA0_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_REG_ADDR_CHECK__SHIFT                                          0x1b
+#define SDMA0_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_CANE_FGCG__SHIFT                                               0x1c
+#define SDMA0_SDMA_CHICKEN_BITS__RESERVED__SHIFT                                                              0x1d
+#define SDMA0_SDMA_CHICKEN_BITS__SRBM_POLL_RETRYING_MASK                                                      0x00000020L
+#define SDMA0_SDMA_CHICKEN_BITS__RD_BURST_MASK                                                                0x000000C0L
+#define SDMA0_SDMA_CHICKEN_BITS__WR_BURST_MASK                                                                0x00000300L
+#define SDMA0_SDMA_CHICKEN_BITS__COMBINE_256B_WAIT_CYCLE_MASK                                                 0x00003C00L
+#define SDMA0_SDMA_CHICKEN_BITS__WR_COMBINE_256B_ENABLE_MASK                                                  0x00004000L
+#define SDMA0_SDMA_CHICKEN_BITS__RD_COMBINE_256B_ENABLE_MASK                                                  0x00008000L
+#define SDMA0_SDMA_CHICKEN_BITS__OVERLAP_ENABLE_MASK                                                          0x00010000L
+#define SDMA0_SDMA_CHICKEN_BITS__T2L_256B_ENABLE_MASK                                                         0x00040000L
+#define SDMA0_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_GCR_FGCG_MASK                                                  0x00080000L
+#define SDMA0_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_GRBM_FGCG_MASK                                                 0x00100000L
+#define SDMA0_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_CH_FGCG_MASK                                                   0x00200000L
+#define SDMA0_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_UTCL2_INVREQ_FGCG_MASK                                         0x00400000L
+#define SDMA0_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_UTCL1_FGCG_MASK                                                0x00800000L
+#define SDMA0_SDMA_CHICKEN_BITS__CG_STATUS_OUTPUT_MASK                                                        0x01000000L
+#define SDMA0_SDMA_CHICKEN_BITS__SW_FREEZE_ENABLE_MASK                                                        0x02000000L
+#define SDMA0_SDMA_CHICKEN_BITS__DRAM_ECC_COPY_MODE_CNTL_MASK                                                 0x04000000L
+#define SDMA0_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_REG_ADDR_CHECK_MASK                                            0x08000000L
+#define SDMA0_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_CANE_FGCG_MASK                                                 0x10000000L
+#define SDMA0_SDMA_CHICKEN_BITS__RESERVED_MASK                                                                0xE0000000L
+//SDMA0_SDMA_CACHE_CNTL
+#define SDMA0_SDMA_CACHE_CNTL__RD_TEMPORAL_HINT__SHIFT                                                        0x0
+#define SDMA0_SDMA_CACHE_CNTL__WR_TEMPORAL_HINT__SHIFT                                                        0x3
+#define SDMA0_SDMA_CACHE_CNTL__RD_SCOPE__SHIFT                                                                0xc
+#define SDMA0_SDMA_CACHE_CNTL__WR_SCOPE__SHIFT                                                                0xe
+#define SDMA0_SDMA_CACHE_CNTL__RD_TEMPORAL_HINT_MASK                                                          0x00000007L
+#define SDMA0_SDMA_CACHE_CNTL__WR_TEMPORAL_HINT_MASK                                                          0x00000038L
+#define SDMA0_SDMA_CACHE_CNTL__RD_SCOPE_MASK                                                                  0x00003000L
+#define SDMA0_SDMA_CACHE_CNTL__WR_SCOPE_MASK                                                                  0x0000C000L
+//SDMA0_SDMA_RB_RPTR_FETCH
+#define SDMA0_SDMA_RB_RPTR_FETCH__OFFSET__SHIFT                                                               0x2
+#define SDMA0_SDMA_RB_RPTR_FETCH__OFFSET_MASK                                                                 0xFFFFFFFCL
+//SDMA0_SDMA_RB_RPTR_FETCH_HI
+#define SDMA0_SDMA_RB_RPTR_FETCH_HI__OFFSET__SHIFT                                                            0x0
+#define SDMA0_SDMA_RB_RPTR_FETCH_HI__OFFSET_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_IB_OFFSET_FETCH
+#define SDMA0_SDMA_IB_OFFSET_FETCH__OFFSET__SHIFT                                                             0x2
+#define SDMA0_SDMA_IB_OFFSET_FETCH__OFFSET_MASK                                                               0x003FFFFCL
+//SDMA0_SDMA_PROGRAM
+#define SDMA0_SDMA_PROGRAM__STREAM__SHIFT                                                                     0x0
+#define SDMA0_SDMA_PROGRAM__STREAM_MASK                                                                       0xFFFFFFFFL
+//SDMA0_SDMA_STATUS_REG
+#define SDMA0_SDMA_STATUS_REG__IDLE__SHIFT                                                                    0x0
+#define SDMA0_SDMA_STATUS_REG__REG_IDLE__SHIFT                                                                0x1
+#define SDMA0_SDMA_STATUS_REG__RB_EMPTY__SHIFT                                                                0x2
+#define SDMA0_SDMA_STATUS_REG__RESERVED_3__SHIFT                                                              0x3
+#define SDMA0_SDMA_STATUS_REG__RB_CMD_IDLE__SHIFT                                                             0x4
+#define SDMA0_SDMA_STATUS_REG__RB_CMD_FULL__SHIFT                                                             0x5
+#define SDMA0_SDMA_STATUS_REG__IB_CMD_IDLE__SHIFT                                                             0x6
+#define SDMA0_SDMA_STATUS_REG__IB_CMD_FULL__SHIFT                                                             0x7
+#define SDMA0_SDMA_STATUS_REG__CORE_IDLE__SHIFT                                                               0x8
+#define SDMA0_SDMA_STATUS_REG__INSIDE_IB__SHIFT                                                               0x9
+#define SDMA0_SDMA_STATUS_REG__FETCH_IDLE__SHIFT                                                              0xa
+#define SDMA0_SDMA_STATUS_REG__CGCG_FENCE__SHIFT                                                              0xb
+#define SDMA0_SDMA_STATUS_REG__PACKET_READY__SHIFT                                                            0xc
+#define SDMA0_SDMA_STATUS_REG__MC_WR_IDLE__SHIFT                                                              0xd
+#define SDMA0_SDMA_STATUS_REG__SRBM_IDLE__SHIFT                                                               0xe
+#define SDMA0_SDMA_STATUS_REG__CONTEXT_EMPTY__SHIFT                                                           0xf
+#define SDMA0_SDMA_STATUS_REG__DELTA_RPTR_FULL__SHIFT                                                         0x10
+#define SDMA0_SDMA_STATUS_REG__RB_MC_RREQ_IDLE__SHIFT                                                         0x11
+#define SDMA0_SDMA_STATUS_REG__IB_MC_RREQ_IDLE__SHIFT                                                         0x12
+#define SDMA0_SDMA_STATUS_REG__MC_RD_IDLE__SHIFT                                                              0x13
+#define SDMA0_SDMA_STATUS_REG__DELTA_RPTR_EMPTY__SHIFT                                                        0x14
+#define SDMA0_SDMA_STATUS_REG__RESERVED_24_21__SHIFT                                                          0x15
+#define SDMA0_SDMA_STATUS_REG__EXEC_ENG_IDLE__SHIFT                                                           0x19
+#define SDMA0_SDMA_STATUS_REG__PROC_CNTL_IDLE__SHIFT                                                          0x1a
+#define SDMA0_SDMA_STATUS_REG__UCODE_INIT_DONE__SHIFT                                                         0x1b
+#define SDMA0_SDMA_STATUS_REG__RESERVED_29_28__SHIFT                                                          0x1c
+#define SDMA0_SDMA_STATUS_REG__INT_IDLE__SHIFT                                                                0x1e
+#define SDMA0_SDMA_STATUS_REG__INT_REQ_STALL__SHIFT                                                           0x1f
+#define SDMA0_SDMA_STATUS_REG__IDLE_MASK                                                                      0x00000001L
+#define SDMA0_SDMA_STATUS_REG__REG_IDLE_MASK                                                                  0x00000002L
+#define SDMA0_SDMA_STATUS_REG__RB_EMPTY_MASK                                                                  0x00000004L
+#define SDMA0_SDMA_STATUS_REG__RESERVED_3_MASK                                                                0x00000008L
+#define SDMA0_SDMA_STATUS_REG__RB_CMD_IDLE_MASK                                                               0x00000010L
+#define SDMA0_SDMA_STATUS_REG__RB_CMD_FULL_MASK                                                               0x00000020L
+#define SDMA0_SDMA_STATUS_REG__IB_CMD_IDLE_MASK                                                               0x00000040L
+#define SDMA0_SDMA_STATUS_REG__IB_CMD_FULL_MASK                                                               0x00000080L
+#define SDMA0_SDMA_STATUS_REG__CORE_IDLE_MASK                                                                 0x00000100L
+#define SDMA0_SDMA_STATUS_REG__INSIDE_IB_MASK                                                                 0x00000200L
+#define SDMA0_SDMA_STATUS_REG__FETCH_IDLE_MASK                                                                0x00000400L
+#define SDMA0_SDMA_STATUS_REG__CGCG_FENCE_MASK                                                                0x00000800L
+#define SDMA0_SDMA_STATUS_REG__PACKET_READY_MASK                                                              0x00001000L
+#define SDMA0_SDMA_STATUS_REG__MC_WR_IDLE_MASK                                                                0x00002000L
+#define SDMA0_SDMA_STATUS_REG__SRBM_IDLE_MASK                                                                 0x00004000L
+#define SDMA0_SDMA_STATUS_REG__CONTEXT_EMPTY_MASK                                                             0x00008000L
+#define SDMA0_SDMA_STATUS_REG__DELTA_RPTR_FULL_MASK                                                           0x00010000L
+#define SDMA0_SDMA_STATUS_REG__RB_MC_RREQ_IDLE_MASK                                                           0x00020000L
+#define SDMA0_SDMA_STATUS_REG__IB_MC_RREQ_IDLE_MASK                                                           0x00040000L
+#define SDMA0_SDMA_STATUS_REG__MC_RD_IDLE_MASK                                                                0x00080000L
+#define SDMA0_SDMA_STATUS_REG__DELTA_RPTR_EMPTY_MASK                                                          0x00100000L
+#define SDMA0_SDMA_STATUS_REG__RESERVED_24_21_MASK                                                            0x01E00000L
+#define SDMA0_SDMA_STATUS_REG__EXEC_ENG_IDLE_MASK                                                             0x02000000L
+#define SDMA0_SDMA_STATUS_REG__PROC_CNTL_IDLE_MASK                                                            0x04000000L
+#define SDMA0_SDMA_STATUS_REG__UCODE_INIT_DONE_MASK                                                           0x08000000L
+#define SDMA0_SDMA_STATUS_REG__RESERVED_29_28_MASK                                                            0x30000000L
+#define SDMA0_SDMA_STATUS_REG__INT_IDLE_MASK                                                                  0x40000000L
+#define SDMA0_SDMA_STATUS_REG__INT_REQ_STALL_MASK                                                             0x80000000L
+//SDMA0_SDMA_STATUS1_REG
+#define SDMA0_SDMA_STATUS1_REG__CE_WREQ_IDLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_STATUS1_REG__CE_WR_IDLE__SHIFT                                                             0x1
+#define SDMA0_SDMA_STATUS1_REG__CE_SPLIT_RD_IDLE__SHIFT                                                       0x2
+#define SDMA0_SDMA_STATUS1_REG__CE_RREQ_IDLE__SHIFT                                                           0x3
+#define SDMA0_SDMA_STATUS1_REG__CE_OUT_IDLE__SHIFT                                                            0x4
+#define SDMA0_SDMA_STATUS1_REG__CE_IN_IDLE__SHIFT                                                             0x5
+#define SDMA0_SDMA_STATUS1_REG__CE_DST_IDLE__SHIFT                                                            0x6
+#define SDMA0_SDMA_STATUS1_REG__CE_WR_TRACK_IDLE__SHIFT                                                       0x7
+#define SDMA0_SDMA_STATUS1_REG__CE_SPLIT_WR_IDLE__SHIFT                                                       0x8
+#define SDMA0_SDMA_STATUS1_REG__CE_CMD_IDLE__SHIFT                                                            0x9
+#define SDMA0_SDMA_STATUS1_REG__CE_AFIFO_FULL__SHIFT                                                          0xa
+#define SDMA0_SDMA_STATUS1_REG__CE_INFO_FULL__SHIFT                                                           0xb
+#define SDMA0_SDMA_STATUS1_REG__CE_INFO1_FULL__SHIFT                                                          0xc
+#define SDMA0_SDMA_STATUS1_REG__EX_START__SHIFT                                                               0xd
+#define SDMA0_SDMA_STATUS1_REG__CE_RD_STALL__SHIFT                                                            0xf
+#define SDMA0_SDMA_STATUS1_REG__CE_WR_STALL__SHIFT                                                            0x10
+#define SDMA0_SDMA_STATUS1_REG__SEC_INTR_STATUS__SHIFT                                                        0x11
+#define SDMA0_SDMA_STATUS1_REG__WPTR_POLL_IDLE__SHIFT                                                         0x12
+#define SDMA0_SDMA_STATUS1_REG__SDMA_IDLE__SHIFT                                                              0x13
+#define SDMA0_SDMA_STATUS1_REG__IC_FETCH_IDLE__SHIFT                                                          0x14
+#define SDMA0_SDMA_STATUS1_REG__IC_FETCH_PAGE_FAULT__SHIFT                                                    0x15
+#define SDMA0_SDMA_STATUS1_REG__IC_FETCH_PAGE_RETRY_TIMEOUT__SHIFT                                            0x16
+#define SDMA0_SDMA_STATUS1_REG__IC_FETCH_PAGE_NULL__SHIFT                                                     0x17
+#define SDMA0_SDMA_STATUS1_REG__MCU_FW_STACK_OVERFLOW__SHIFT                                                  0x18
+#define SDMA0_SDMA_STATUS1_REG__CE_WREQ_IDLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_STATUS1_REG__CE_WR_IDLE_MASK                                                               0x00000002L
+#define SDMA0_SDMA_STATUS1_REG__CE_SPLIT_RD_IDLE_MASK                                                         0x00000004L
+#define SDMA0_SDMA_STATUS1_REG__CE_RREQ_IDLE_MASK                                                             0x00000008L
+#define SDMA0_SDMA_STATUS1_REG__CE_OUT_IDLE_MASK                                                              0x00000010L
+#define SDMA0_SDMA_STATUS1_REG__CE_IN_IDLE_MASK                                                               0x00000020L
+#define SDMA0_SDMA_STATUS1_REG__CE_DST_IDLE_MASK                                                              0x00000040L
+#define SDMA0_SDMA_STATUS1_REG__CE_WR_TRACK_IDLE_MASK                                                         0x00000080L
+#define SDMA0_SDMA_STATUS1_REG__CE_SPLIT_WR_IDLE_MASK                                                         0x00000100L
+#define SDMA0_SDMA_STATUS1_REG__CE_CMD_IDLE_MASK                                                              0x00000200L
+#define SDMA0_SDMA_STATUS1_REG__CE_AFIFO_FULL_MASK                                                            0x00000400L
+#define SDMA0_SDMA_STATUS1_REG__CE_INFO_FULL_MASK                                                             0x00000800L
+#define SDMA0_SDMA_STATUS1_REG__CE_INFO1_FULL_MASK                                                            0x00001000L
+#define SDMA0_SDMA_STATUS1_REG__EX_START_MASK                                                                 0x00002000L
+#define SDMA0_SDMA_STATUS1_REG__CE_RD_STALL_MASK                                                              0x00008000L
+#define SDMA0_SDMA_STATUS1_REG__CE_WR_STALL_MASK                                                              0x00010000L
+#define SDMA0_SDMA_STATUS1_REG__SEC_INTR_STATUS_MASK                                                          0x00020000L
+#define SDMA0_SDMA_STATUS1_REG__WPTR_POLL_IDLE_MASK                                                           0x00040000L
+#define SDMA0_SDMA_STATUS1_REG__SDMA_IDLE_MASK                                                                0x00080000L
+#define SDMA0_SDMA_STATUS1_REG__IC_FETCH_IDLE_MASK                                                            0x00100000L
+#define SDMA0_SDMA_STATUS1_REG__IC_FETCH_PAGE_FAULT_MASK                                                      0x00200000L
+#define SDMA0_SDMA_STATUS1_REG__IC_FETCH_PAGE_RETRY_TIMEOUT_MASK                                              0x00400000L
+#define SDMA0_SDMA_STATUS1_REG__IC_FETCH_PAGE_NULL_MASK                                                       0x00800000L
+#define SDMA0_SDMA_STATUS1_REG__MCU_FW_STACK_OVERFLOW_MASK                                                    0x03000000L
+//SDMA0_SDMA_CNTL1
+#define SDMA0_SDMA_CNTL1__WPTR_POLL_FREQUENCY__SHIFT                                                          0x2
+#define SDMA0_SDMA_CNTL1__WPTR_POLL_FREQUENCY_MASK                                                            0x0000FFFCL
+//SDMA0_SDMA_HBM_PAGE_CONFIG
+#define SDMA0_SDMA_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT__SHIFT                                                 0x0
+#define SDMA0_SDMA_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT_MASK                                                   0x00000003L
+//SDMA0_SDMA_FREEZE
+#define SDMA0_SDMA_FREEZE__FROZEN__SHIFT                                                                      0x5
+#define SDMA0_SDMA_FREEZE__MCU_FREEZE__SHIFT                                                                  0x6
+#define SDMA0_SDMA_FREEZE__IMU_FSM_STATE__SHIFT                                                               0x8
+#define SDMA0_SDMA_FREEZE__EXTERNAL_FROZEN__SHIFT                                                             0xc
+#define SDMA0_SDMA_FREEZE__FROZEN_MASK                                                                        0x00000020L
+#define SDMA0_SDMA_FREEZE__MCU_FREEZE_MASK                                                                    0x00000040L
+#define SDMA0_SDMA_FREEZE__IMU_FSM_STATE_MASK                                                                 0x00000300L
+#define SDMA0_SDMA_FREEZE__EXTERNAL_FROZEN_MASK                                                               0x00001000L
+//SDMA0_SDMA_PROCESS_QUANTUM0
+#define SDMA0_SDMA_PROCESS_QUANTUM0__PROCESS0_QUANTUM__SHIFT                                                  0x0
+#define SDMA0_SDMA_PROCESS_QUANTUM0__PROCESS1_QUANTUM__SHIFT                                                  0x8
+#define SDMA0_SDMA_PROCESS_QUANTUM0__PROCESS2_QUANTUM__SHIFT                                                  0x10
+#define SDMA0_SDMA_PROCESS_QUANTUM0__PROCESS3_QUANTUM__SHIFT                                                  0x18
+#define SDMA0_SDMA_PROCESS_QUANTUM0__PROCESS0_QUANTUM_MASK                                                    0x000000FFL
+#define SDMA0_SDMA_PROCESS_QUANTUM0__PROCESS1_QUANTUM_MASK                                                    0x0000FF00L
+#define SDMA0_SDMA_PROCESS_QUANTUM0__PROCESS2_QUANTUM_MASK                                                    0x00FF0000L
+#define SDMA0_SDMA_PROCESS_QUANTUM0__PROCESS3_QUANTUM_MASK                                                    0xFF000000L
+//SDMA0_SDMA_PROCESS_QUANTUM1
+#define SDMA0_SDMA_PROCESS_QUANTUM1__PROCESS4_QUANTUM__SHIFT                                                  0x0
+#define SDMA0_SDMA_PROCESS_QUANTUM1__PROCESS5_QUANTUM__SHIFT                                                  0x8
+#define SDMA0_SDMA_PROCESS_QUANTUM1__PROCESS6_QUANTUM__SHIFT                                                  0x10
+#define SDMA0_SDMA_PROCESS_QUANTUM1__PROCESS7_QUANTUM__SHIFT                                                  0x18
+#define SDMA0_SDMA_PROCESS_QUANTUM1__PROCESS4_QUANTUM_MASK                                                    0x000000FFL
+#define SDMA0_SDMA_PROCESS_QUANTUM1__PROCESS5_QUANTUM_MASK                                                    0x0000FF00L
+#define SDMA0_SDMA_PROCESS_QUANTUM1__PROCESS6_QUANTUM_MASK                                                    0x00FF0000L
+#define SDMA0_SDMA_PROCESS_QUANTUM1__PROCESS7_QUANTUM_MASK                                                    0xFF000000L
+//SDMA0_SDMA_WATCHDOG_CNTL
+#define SDMA0_SDMA_WATCHDOG_CNTL__QUEUE_HANG_COUNT__SHIFT                                                     0x0
+#define SDMA0_SDMA_WATCHDOG_CNTL__CMD_TIMEOUT_COUNT__SHIFT                                                    0x8
+#define SDMA0_SDMA_WATCHDOG_CNTL__QUEUE_HANG_COUNT_MASK                                                       0x000000FFL
+#define SDMA0_SDMA_WATCHDOG_CNTL__CMD_TIMEOUT_COUNT_MASK                                                      0x0000FF00L
+//SDMA0_SDMA_QUEUE_STATUS0
+#define SDMA0_SDMA_QUEUE_STATUS0__QUEUE0_STATUS__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE_STATUS0__QUEUE1_STATUS__SHIFT                                                        0x4
+#define SDMA0_SDMA_QUEUE_STATUS0__QUEUE2_STATUS__SHIFT                                                        0x8
+#define SDMA0_SDMA_QUEUE_STATUS0__QUEUE3_STATUS__SHIFT                                                        0xc
+#define SDMA0_SDMA_QUEUE_STATUS0__QUEUE4_STATUS__SHIFT                                                        0x10
+#define SDMA0_SDMA_QUEUE_STATUS0__QUEUE5_STATUS__SHIFT                                                        0x14
+#define SDMA0_SDMA_QUEUE_STATUS0__QUEUE6_STATUS__SHIFT                                                        0x18
+#define SDMA0_SDMA_QUEUE_STATUS0__QUEUE7_STATUS__SHIFT                                                        0x1c
+#define SDMA0_SDMA_QUEUE_STATUS0__QUEUE0_STATUS_MASK                                                          0x0000000FL
+#define SDMA0_SDMA_QUEUE_STATUS0__QUEUE1_STATUS_MASK                                                          0x000000F0L
+#define SDMA0_SDMA_QUEUE_STATUS0__QUEUE2_STATUS_MASK                                                          0x00000F00L
+#define SDMA0_SDMA_QUEUE_STATUS0__QUEUE3_STATUS_MASK                                                          0x0000F000L
+#define SDMA0_SDMA_QUEUE_STATUS0__QUEUE4_STATUS_MASK                                                          0x000F0000L
+#define SDMA0_SDMA_QUEUE_STATUS0__QUEUE5_STATUS_MASK                                                          0x00F00000L
+#define SDMA0_SDMA_QUEUE_STATUS0__QUEUE6_STATUS_MASK                                                          0x0F000000L
+#define SDMA0_SDMA_QUEUE_STATUS0__QUEUE7_STATUS_MASK                                                          0xF0000000L
+//SDMA0_SDMA_QUEUE_STATUS1
+#define SDMA0_SDMA_QUEUE_STATUS1__QUEUE8_STATUS__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE_STATUS1__QUEUE9_STATUS__SHIFT                                                        0x4
+#define SDMA0_SDMA_QUEUE_STATUS1__QUEUE8_STATUS_MASK                                                          0x0000000FL
+#define SDMA0_SDMA_QUEUE_STATUS1__QUEUE9_STATUS_MASK                                                          0x000000F0L
+//SDMA0_SDMA_ID
+#define SDMA0_SDMA_ID__DEVICE_ID__SHIFT                                                                       0x0
+#define SDMA0_SDMA_ID__DEVICE_ID_MASK                                                                         0x000000FFL
+//SDMA0_SDMA_VERSION
+#define SDMA0_SDMA_VERSION__MINVER__SHIFT                                                                     0x0
+#define SDMA0_SDMA_VERSION__MAJVER__SHIFT                                                                     0x8
+#define SDMA0_SDMA_VERSION__REV__SHIFT                                                                        0x10
+#define SDMA0_SDMA_VERSION__MINVER_MASK                                                                       0x0000007FL
+#define SDMA0_SDMA_VERSION__MAJVER_MASK                                                                       0x00007F00L
+#define SDMA0_SDMA_VERSION__REV_MASK                                                                          0x003F0000L
+//SDMA0_SDMA_STATUS2_REG
+#define SDMA0_SDMA_STATUS2_REG__ID__SHIFT                                                                     0x0
+#define SDMA0_SDMA_STATUS2_REG__TH0MCU_INSTR_PTR__SHIFT                                                       0x2
+#define SDMA0_SDMA_STATUS2_REG__CMD_OP__SHIFT                                                                 0x10
+#define SDMA0_SDMA_STATUS2_REG__ID_MASK                                                                       0x00000003L
+#define SDMA0_SDMA_STATUS2_REG__TH0MCU_INSTR_PTR_MASK                                                         0x0000FFFCL
+#define SDMA0_SDMA_STATUS2_REG__CMD_OP_MASK                                                                   0xFFFF0000L
+//SDMA0_SDMA_ATOMIC_CNTL
+#define SDMA0_SDMA_ATOMIC_CNTL__LOOP_TIMER__SHIFT                                                             0x0
+#define SDMA0_SDMA_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE__SHIFT                                                  0x1f
+#define SDMA0_SDMA_ATOMIC_CNTL__LOOP_TIMER_MASK                                                               0x7FFFFFFFL
+#define SDMA0_SDMA_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE_MASK                                                    0x80000000L
+//SDMA0_SDMA_ATOMIC_PERMS_CNTL
+#define SDMA0_SDMA_ATOMIC_PERMS_CNTL__EN_RETRY_ON_ATOMIC__SHIFT                                               0x0
+#define SDMA0_SDMA_ATOMIC_PERMS_CNTL__PCIE_ATOMIC_SUPPORTED__SHIFT                                            0x1
+#define SDMA0_SDMA_ATOMIC_PERMS_CNTL__EN_RETRY_ON_ATOMIC_MASK                                                 0x00000001L
+#define SDMA0_SDMA_ATOMIC_PERMS_CNTL__PCIE_ATOMIC_SUPPORTED_MASK                                              0x00000002L
+//SDMA0_SDMA_ATOMIC_PREOP_LO
+#define SDMA0_SDMA_ATOMIC_PREOP_LO__DATA__SHIFT                                                               0x0
+#define SDMA0_SDMA_ATOMIC_PREOP_LO__DATA_MASK                                                                 0xFFFFFFFFL
+//SDMA0_SDMA_ATOMIC_PREOP_HI
+#define SDMA0_SDMA_ATOMIC_PREOP_HI__DATA__SHIFT                                                               0x0
+#define SDMA0_SDMA_ATOMIC_PREOP_HI__DATA_MASK                                                                 0xFFFFFFFFL
+//SDMA0_SDMA_DCC_CNTL
+#define SDMA0_SDMA_DCC_CNTL__DCC_FORCE_BYPASS__SHIFT                                                          0x0
+#define SDMA0_SDMA_DCC_CNTL__DCC_RD_NOPTE_OVERRIDE_0__SHIFT                                                   0x1
+#define SDMA0_SDMA_DCC_CNTL__DCC_RD_NOPTE_COMP_EN_0__SHIFT                                                    0x2
+#define SDMA0_SDMA_DCC_CNTL__DCC_WR_NOPTE_OVERRIDE_0__SHIFT                                                   0x3
+#define SDMA0_SDMA_DCC_CNTL__DCC_WR_NOPTE_COMP_EN_0__SHIFT                                                    0x4
+#define SDMA0_SDMA_DCC_CNTL__DCC_RD_NOPTE_OVERRIDE_1__SHIFT                                                   0x5
+#define SDMA0_SDMA_DCC_CNTL__DCC_RD_NOPTE_COMP_EN_1__SHIFT                                                    0x6
+#define SDMA0_SDMA_DCC_CNTL__DCC_WR_NOPTE_OVERRIDE_1__SHIFT                                                   0x7
+#define SDMA0_SDMA_DCC_CNTL__DCC_WR_NOPTE_COMP_EN_1__SHIFT                                                    0x8
+#define SDMA0_SDMA_DCC_CNTL__DCC_RD_NOPTE_OVERRIDE_2__SHIFT                                                   0x9
+#define SDMA0_SDMA_DCC_CNTL__DCC_RD_NOPTE_COMP_EN_2__SHIFT                                                    0xa
+#define SDMA0_SDMA_DCC_CNTL__DCC_WR_NOPTE_OVERRIDE_2__SHIFT                                                   0xb
+#define SDMA0_SDMA_DCC_CNTL__DCC_WR_NOPTE_COMP_EN_2__SHIFT                                                    0xc
+#define SDMA0_SDMA_DCC_CNTL__DCC_RD_NOPTE_OVERRIDE_3__SHIFT                                                   0xd
+#define SDMA0_SDMA_DCC_CNTL__DCC_RD_NOPTE_COMP_EN_3__SHIFT                                                    0xe
+#define SDMA0_SDMA_DCC_CNTL__DCC_WR_NOPTE_OVERRIDE_3__SHIFT                                                   0xf
+#define SDMA0_SDMA_DCC_CNTL__DCC_WR_NOPTE_COMP_EN_3__SHIFT                                                    0x10
+#define SDMA0_SDMA_DCC_CNTL__DCC_FORCE_BYPASS_MASK                                                            0x00000001L
+#define SDMA0_SDMA_DCC_CNTL__DCC_RD_NOPTE_OVERRIDE_0_MASK                                                     0x00000002L
+#define SDMA0_SDMA_DCC_CNTL__DCC_RD_NOPTE_COMP_EN_0_MASK                                                      0x00000004L
+#define SDMA0_SDMA_DCC_CNTL__DCC_WR_NOPTE_OVERRIDE_0_MASK                                                     0x00000008L
+#define SDMA0_SDMA_DCC_CNTL__DCC_WR_NOPTE_COMP_EN_0_MASK                                                      0x00000010L
+#define SDMA0_SDMA_DCC_CNTL__DCC_RD_NOPTE_OVERRIDE_1_MASK                                                     0x00000020L
+#define SDMA0_SDMA_DCC_CNTL__DCC_RD_NOPTE_COMP_EN_1_MASK                                                      0x00000040L
+#define SDMA0_SDMA_DCC_CNTL__DCC_WR_NOPTE_OVERRIDE_1_MASK                                                     0x00000080L
+#define SDMA0_SDMA_DCC_CNTL__DCC_WR_NOPTE_COMP_EN_1_MASK                                                      0x00000100L
+#define SDMA0_SDMA_DCC_CNTL__DCC_RD_NOPTE_OVERRIDE_2_MASK                                                     0x00000200L
+#define SDMA0_SDMA_DCC_CNTL__DCC_RD_NOPTE_COMP_EN_2_MASK                                                      0x00000400L
+#define SDMA0_SDMA_DCC_CNTL__DCC_WR_NOPTE_OVERRIDE_2_MASK                                                     0x00000800L
+#define SDMA0_SDMA_DCC_CNTL__DCC_WR_NOPTE_COMP_EN_2_MASK                                                      0x00001000L
+#define SDMA0_SDMA_DCC_CNTL__DCC_RD_NOPTE_OVERRIDE_3_MASK                                                     0x00002000L
+#define SDMA0_SDMA_DCC_CNTL__DCC_RD_NOPTE_COMP_EN_3_MASK                                                      0x00004000L
+#define SDMA0_SDMA_DCC_CNTL__DCC_WR_NOPTE_OVERRIDE_3_MASK                                                     0x00008000L
+#define SDMA0_SDMA_DCC_CNTL__DCC_WR_NOPTE_COMP_EN_3_MASK                                                      0x00010000L
+//SDMA0_SDMA_UTCL1_WARMUPL2_CNTL
+#define SDMA0_SDMA_UTCL1_WARMUPL2_CNTL__WARMUP_EN__SHIFT                                                      0x0
+#define SDMA0_SDMA_UTCL1_WARMUPL2_CNTL__INTERVAL_VA__SHIFT                                                    0x1
+#define SDMA0_SDMA_UTCL1_WARMUPL2_CNTL__GRANU__SHIFT                                                          0x8
+#define SDMA0_SDMA_UTCL1_WARMUPL2_CNTL__STEP_PER_INTERVAL__SHIFT                                              0x10
+#define SDMA0_SDMA_UTCL1_WARMUPL2_CNTL__WARMUP_EN_MASK                                                        0x00000001L
+#define SDMA0_SDMA_UTCL1_WARMUPL2_CNTL__INTERVAL_VA_MASK                                                      0x0000007EL
+#define SDMA0_SDMA_UTCL1_WARMUPL2_CNTL__GRANU_MASK                                                            0x00007F00L
+#define SDMA0_SDMA_UTCL1_WARMUPL2_CNTL__STEP_PER_INTERVAL_MASK                                                0x00070000L
+//SDMA0_SDMA_UTCL1_CNTL
+#define SDMA0_SDMA_UTCL1_CNTL__REDO_DELAY__SHIFT                                                              0x0
+#define SDMA0_SDMA_UTCL1_CNTL__PAGE_WAIT_DELAY__SHIFT                                                         0x5
+#define SDMA0_SDMA_UTCL1_CNTL__RESP_MODE__SHIFT                                                               0x9
+#define SDMA0_SDMA_UTCL1_CNTL__FORCE_INVALIDATION__SHIFT                                                      0xb
+#define SDMA0_SDMA_UTCL1_CNTL__FORCE_INVREQ_HEAVY__SHIFT                                                      0xc
+#define SDMA0_SDMA_UTCL1_CNTL__WR_EXE_PERMS_CTRL__SHIFT                                                       0xd
+#define SDMA0_SDMA_UTCL1_CNTL__RD_EXE_PERMS_CTRL__SHIFT                                                       0xe
+#define SDMA0_SDMA_UTCL1_CNTL__INVACK_DELAY__SHIFT                                                            0xf
+#define SDMA0_SDMA_UTCL1_CNTL__REQL2_CREDIT__SHIFT                                                            0x13
+#define SDMA0_SDMA_UTCL1_CNTL__REQL2_5CYC_CREDIT__SHIFT                                                       0x19
+#define SDMA0_SDMA_UTCL1_CNTL__REDO_DELAY_MASK                                                                0x0000001FL
+#define SDMA0_SDMA_UTCL1_CNTL__PAGE_WAIT_DELAY_MASK                                                           0x000001E0L
+#define SDMA0_SDMA_UTCL1_CNTL__RESP_MODE_MASK                                                                 0x00000600L
+#define SDMA0_SDMA_UTCL1_CNTL__FORCE_INVALIDATION_MASK                                                        0x00000800L
+#define SDMA0_SDMA_UTCL1_CNTL__FORCE_INVREQ_HEAVY_MASK                                                        0x00001000L
+#define SDMA0_SDMA_UTCL1_CNTL__WR_EXE_PERMS_CTRL_MASK                                                         0x00002000L
+#define SDMA0_SDMA_UTCL1_CNTL__RD_EXE_PERMS_CTRL_MASK                                                         0x00004000L
+#define SDMA0_SDMA_UTCL1_CNTL__INVACK_DELAY_MASK                                                              0x00078000L
+#define SDMA0_SDMA_UTCL1_CNTL__REQL2_CREDIT_MASK                                                              0x01F80000L
+#define SDMA0_SDMA_UTCL1_CNTL__REQL2_5CYC_CREDIT_MASK                                                         0x7E000000L
+//SDMA0_SDMA_UTCL1_WATERMK
+#define SDMA0_SDMA_UTCL1_WATERMK__WR_REQ_FIFO_WATERMK__SHIFT                                                  0x0
+#define SDMA0_SDMA_UTCL1_WATERMK__WR_REQ_FIFO_DEPTH_STEP__SHIFT                                               0x4
+#define SDMA0_SDMA_UTCL1_WATERMK__RD_REQ_FIFO_WATERMK__SHIFT                                                  0x6
+#define SDMA0_SDMA_UTCL1_WATERMK__RD_REQ_FIFO_DEPTH_STEP__SHIFT                                               0xa
+#define SDMA0_SDMA_UTCL1_WATERMK__WR_PAGE_FIFO_WATERMK__SHIFT                                                 0xc
+#define SDMA0_SDMA_UTCL1_WATERMK__WR_PAGE_FIFO_DEPTH_STEP__SHIFT                                              0x10
+#define SDMA0_SDMA_UTCL1_WATERMK__RD_PAGE_FIFO_WATERMK__SHIFT                                                 0x12
+#define SDMA0_SDMA_UTCL1_WATERMK__RD_PAGE_FIFO_DEPTH_STEP__SHIFT                                              0x16
+#define SDMA0_SDMA_UTCL1_WATERMK__WR_REQ_FIFO_WATERMK_MASK                                                    0x0000000FL
+#define SDMA0_SDMA_UTCL1_WATERMK__WR_REQ_FIFO_DEPTH_STEP_MASK                                                 0x00000030L
+#define SDMA0_SDMA_UTCL1_WATERMK__RD_REQ_FIFO_WATERMK_MASK                                                    0x000003C0L
+#define SDMA0_SDMA_UTCL1_WATERMK__RD_REQ_FIFO_DEPTH_STEP_MASK                                                 0x00000C00L
+#define SDMA0_SDMA_UTCL1_WATERMK__WR_PAGE_FIFO_WATERMK_MASK                                                   0x0000F000L
+#define SDMA0_SDMA_UTCL1_WATERMK__WR_PAGE_FIFO_DEPTH_STEP_MASK                                                0x00030000L
+#define SDMA0_SDMA_UTCL1_WATERMK__RD_PAGE_FIFO_WATERMK_MASK                                                   0x003C0000L
+#define SDMA0_SDMA_UTCL1_WATERMK__RD_PAGE_FIFO_DEPTH_STEP_MASK                                                0x00C00000L
+//SDMA0_SDMA_UTCL1_TIMEOUT
+#define SDMA0_SDMA_UTCL1_TIMEOUT__XNACK_LIMIT__SHIFT                                                          0x0
+#define SDMA0_SDMA_UTCL1_TIMEOUT__XNACK_LIMIT_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_UTCL1_PAGE
+#define SDMA0_SDMA_UTCL1_PAGE__INVALID_ADDR__SHIFT                                                            0x0
+#define SDMA0_SDMA_UTCL1_PAGE__REQ_TYPE__SHIFT                                                                0x1
+#define SDMA0_SDMA_UTCL1_PAGE__USE_MTYPE__SHIFT                                                               0x7
+#define SDMA0_SDMA_UTCL1_PAGE__USE_PT_SNOOP__SHIFT                                                            0xa
+#define SDMA0_SDMA_UTCL1_PAGE__USE_IO__SHIFT                                                                  0xb
+#define SDMA0_SDMA_UTCL1_PAGE__RD_L2_POLICY__SHIFT                                                            0xc
+#define SDMA0_SDMA_UTCL1_PAGE__WR_L2_POLICY__SHIFT                                                            0xe
+#define SDMA0_SDMA_UTCL1_PAGE__USE_BC__SHIFT                                                                  0x16
+#define SDMA0_SDMA_UTCL1_PAGE__ADDR_IS_PA__SHIFT                                                              0x17
+#define SDMA0_SDMA_UTCL1_PAGE__FORCE_BE_CMD_SNOOP_VALUE__SHIFT                                                0x18
+#define SDMA0_SDMA_UTCL1_PAGE__FORCE_BE_CMD_SNOOP_EN__SHIFT                                                   0x19
+#define SDMA0_SDMA_UTCL1_PAGE__MTYPE_NO_PTE_OVERRIDE_VALUE__SHIFT                                             0x1b
+#define SDMA0_SDMA_UTCL1_PAGE__MTYPE_NO_PTE_OVERRIDE_EN__SHIFT                                                0x1e
+#define SDMA0_SDMA_UTCL1_PAGE__INVALID_ADDR_MASK                                                              0x00000001L
+#define SDMA0_SDMA_UTCL1_PAGE__REQ_TYPE_MASK                                                                  0x0000003EL
+#define SDMA0_SDMA_UTCL1_PAGE__USE_MTYPE_MASK                                                                 0x00000380L
+#define SDMA0_SDMA_UTCL1_PAGE__USE_PT_SNOOP_MASK                                                              0x00000400L
+#define SDMA0_SDMA_UTCL1_PAGE__USE_IO_MASK                                                                    0x00000800L
+#define SDMA0_SDMA_UTCL1_PAGE__RD_L2_POLICY_MASK                                                              0x00003000L
+#define SDMA0_SDMA_UTCL1_PAGE__WR_L2_POLICY_MASK                                                              0x0000C000L
+#define SDMA0_SDMA_UTCL1_PAGE__USE_BC_MASK                                                                    0x00400000L
+#define SDMA0_SDMA_UTCL1_PAGE__ADDR_IS_PA_MASK                                                                0x00800000L
+#define SDMA0_SDMA_UTCL1_PAGE__FORCE_BE_CMD_SNOOP_VALUE_MASK                                                  0x01000000L
+#define SDMA0_SDMA_UTCL1_PAGE__FORCE_BE_CMD_SNOOP_EN_MASK                                                     0x02000000L
+#define SDMA0_SDMA_UTCL1_PAGE__MTYPE_NO_PTE_OVERRIDE_VALUE_MASK                                               0x18000000L
+#define SDMA0_SDMA_UTCL1_PAGE__MTYPE_NO_PTE_OVERRIDE_EN_MASK                                                  0x40000000L
+//SDMA0_SDMA_EXTERNAL_FROZEN
+#define SDMA0_SDMA_EXTERNAL_FROZEN__THRESHOLD__SHIFT                                                          0x0
+#define SDMA0_SDMA_EXTERNAL_FROZEN__THRESHOLD_MASK                                                            0x0000FFFFL
+//SDMA0_SDMA_FREEZE_TRIGGER
+#define SDMA0_SDMA_FREEZE_TRIGGER__CORE_FREEZE__SHIFT                                                         0x0
+#define SDMA0_SDMA_FREEZE_TRIGGER__CORE_FREEZE_MASK                                                           0x00000001L
+//SDMA0_SDMA_UTCL1_CACHE_CNTL
+#define SDMA0_SDMA_UTCL1_CACHE_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                              0x0
+#define SDMA0_SDMA_UTCL1_CACHE_CNTL__MTYPE_NO_PTE_OVERRIDE_EN__SHIFT                                          0x14
+#define SDMA0_SDMA_UTCL1_CACHE_CNTL__MTYPE_NO_PTE_OVERRIDE_VALUE__SHIFT                                       0x15
+#define SDMA0_SDMA_UTCL1_CACHE_CNTL__INVALIDATE_IC__SHIFT                                                     0x19
+#define SDMA0_SDMA_UTCL1_CACHE_CNTL__INVALIDATE_GU__SHIFT                                                     0x1a
+#define SDMA0_SDMA_UTCL1_CACHE_CNTL__FRAG_LIMIT_MODE__SHIFT                                                   0x1b
+#define SDMA0_SDMA_UTCL1_CACHE_CNTL__FORCE_SNOOP__SHIFT                                                       0x1c
+#define SDMA0_SDMA_UTCL1_CACHE_CNTL__STRICT_PERMISSION__SHIFT                                                 0x1d
+#define SDMA0_SDMA_UTCL1_CACHE_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                0x000FFFFFL
+#define SDMA0_SDMA_UTCL1_CACHE_CNTL__MTYPE_NO_PTE_OVERRIDE_EN_MASK                                            0x00100000L
+#define SDMA0_SDMA_UTCL1_CACHE_CNTL__MTYPE_NO_PTE_OVERRIDE_VALUE_MASK                                         0x00600000L
+#define SDMA0_SDMA_UTCL1_CACHE_CNTL__INVALIDATE_IC_MASK                                                       0x02000000L
+#define SDMA0_SDMA_UTCL1_CACHE_CNTL__INVALIDATE_GU_MASK                                                       0x04000000L
+#define SDMA0_SDMA_UTCL1_CACHE_CNTL__FRAG_LIMIT_MODE_MASK                                                     0x08000000L
+#define SDMA0_SDMA_UTCL1_CACHE_CNTL__FORCE_SNOOP_MASK                                                         0x10000000L
+#define SDMA0_SDMA_UTCL1_CACHE_CNTL__STRICT_PERMISSION_MASK                                                   0x20000000L
+//SDMA0_SDMA_UTCL1_RD_STATUS
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_DATA_FIFO_EMPTY__SHIFT                                              0x0
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_VA_REQ_SEND_FIFO_EMPTY__SHIFT                                       0x1
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_REG_ENTRY_EMPTY__SHIFT                                              0x2
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_PAGE_FIFO_EMPTY__SHIFT                                              0x3
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_REQ_FIFO_EMPTY__SHIFT                                               0x4
+#define SDMA0_SDMA_UTCL1_RD_STATUS__RD_VA_FIFO_EMPTY__SHIFT                                                   0x5
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_VA_REQ_SEND_FIFO_FULL__SHIFT                                        0x7
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_REG_ENTRY_FULL__SHIFT                                               0x8
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_PAGE_FIFO_FULL__SHIFT                                               0x9
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_REQ_FIFO_FULL__SHIFT                                                0xa
+#define SDMA0_SDMA_UTCL1_RD_STATUS__RD_VA_FIFO_FULL__SHIFT                                                    0xb
+#define SDMA0_SDMA_UTCL1_RD_STATUS__L2_INTF_RD_IDLE__SHIFT                                                    0x10
+#define SDMA0_SDMA_UTCL1_RD_STATUS__RD_REQRET_IDLE__SHIFT                                                     0x11
+#define SDMA0_SDMA_UTCL1_RD_STATUS__RD_REQ_IDLE__SHIFT                                                        0x12
+#define SDMA0_SDMA_UTCL1_RD_STATUS__UTCL1_WARMUP_IDLE__SHIFT                                                  0x1a
+#define SDMA0_SDMA_UTCL1_RD_STATUS__UTCL2_1CYC_IDLE__SHIFT                                                    0x1b
+#define SDMA0_SDMA_UTCL1_RD_STATUS__UTCL2_1CYC_WARMUP_IDLE__SHIFT                                             0x1c
+#define SDMA0_SDMA_UTCL1_RD_STATUS__UTCL2_5CYC_CACHE1_IDLE__SHIFT                                             0x1d
+#define SDMA0_SDMA_UTCL1_RD_STATUS__UTCL2_5CYC_CACHE0_IDLE__SHIFT                                             0x1e
+#define SDMA0_SDMA_UTCL1_RD_STATUS__UTCL2_IDLE__SHIFT                                                         0x1f
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_DATA_FIFO_EMPTY_MASK                                                0x00000001L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_VA_REQ_SEND_FIFO_EMPTY_MASK                                         0x00000002L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_REG_ENTRY_EMPTY_MASK                                                0x00000004L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_PAGE_FIFO_EMPTY_MASK                                                0x00000008L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_REQ_FIFO_EMPTY_MASK                                                 0x00000010L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__RD_VA_FIFO_EMPTY_MASK                                                     0x00000020L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_VA_REQ_SEND_FIFO_FULL_MASK                                          0x00000080L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_REG_ENTRY_FULL_MASK                                                 0x00000100L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_PAGE_FIFO_FULL_MASK                                                 0x00000200L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__CE_RD_REQ_FIFO_FULL_MASK                                                  0x00000400L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__RD_VA_FIFO_FULL_MASK                                                      0x00000800L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__L2_INTF_RD_IDLE_MASK                                                      0x00010000L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__RD_REQRET_IDLE_MASK                                                       0x00020000L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__RD_REQ_IDLE_MASK                                                          0x00040000L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__UTCL1_WARMUP_IDLE_MASK                                                    0x04000000L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__UTCL2_1CYC_IDLE_MASK                                                      0x08000000L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__UTCL2_1CYC_WARMUP_IDLE_MASK                                               0x10000000L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__UTCL2_5CYC_CACHE1_IDLE_MASK                                               0x20000000L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__UTCL2_5CYC_CACHE0_IDLE_MASK                                               0x40000000L
+#define SDMA0_SDMA_UTCL1_RD_STATUS__UTCL2_IDLE_MASK                                                           0x80000000L
+//SDMA0_SDMA_UTCL1_WR_STATUS
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_DATA_FIFO_EMPTY__SHIFT                                              0x0
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_VA_REQ_SEND_FIFO_EMPTY__SHIFT                                       0x1
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_REG_ENTRY_EMPTY__SHIFT                                              0x2
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_PAGE_FIFO_EMPTY__SHIFT                                              0x3
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_REQ_FIFO_EMPTY__SHIFT                                               0x4
+#define SDMA0_SDMA_UTCL1_WR_STATUS__WR_VA_FIFO_EMPTY__SHIFT                                                   0x5
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_DATA_FIFO_FULL__SHIFT                                               0x6
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_VA_REQ_SEND_FIFO_FULL__SHIFT                                        0x7
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_REG_ENTRY_FULL__SHIFT                                               0x8
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_PAGE_FIFO_FULL__SHIFT                                               0x9
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_REQ_FIFO_FULL__SHIFT                                                0xa
+#define SDMA0_SDMA_UTCL1_WR_STATUS__WR_VA_FIFO_FULL__SHIFT                                                    0xb
+#define SDMA0_SDMA_UTCL1_WR_STATUS__L2_INTF_WR_IDLE__SHIFT                                                    0x10
+#define SDMA0_SDMA_UTCL1_WR_STATUS__WR_REQRET_IDLE__SHIFT                                                     0x11
+#define SDMA0_SDMA_UTCL1_WR_STATUS__WR_REQ_IDLE__SHIFT                                                        0x12
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_DATA_FIFO_EMPTY_MASK                                                0x00000001L
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_VA_REQ_SEND_FIFO_EMPTY_MASK                                         0x00000002L
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_REG_ENTRY_EMPTY_MASK                                                0x00000004L
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_PAGE_FIFO_EMPTY_MASK                                                0x00000008L
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_REQ_FIFO_EMPTY_MASK                                                 0x00000010L
+#define SDMA0_SDMA_UTCL1_WR_STATUS__WR_VA_FIFO_EMPTY_MASK                                                     0x00000020L
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_DATA_FIFO_FULL_MASK                                                 0x00000040L
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_VA_REQ_SEND_FIFO_FULL_MASK                                          0x00000080L
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_REG_ENTRY_FULL_MASK                                                 0x00000100L
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_PAGE_FIFO_FULL_MASK                                                 0x00000200L
+#define SDMA0_SDMA_UTCL1_WR_STATUS__CE_WR_REQ_FIFO_FULL_MASK                                                  0x00000400L
+#define SDMA0_SDMA_UTCL1_WR_STATUS__WR_VA_FIFO_FULL_MASK                                                      0x00000800L
+#define SDMA0_SDMA_UTCL1_WR_STATUS__L2_INTF_WR_IDLE_MASK                                                      0x00010000L
+#define SDMA0_SDMA_UTCL1_WR_STATUS__WR_REQRET_IDLE_MASK                                                       0x00020000L
+#define SDMA0_SDMA_UTCL1_WR_STATUS__WR_REQ_IDLE_MASK                                                          0x00040000L
+//SDMA0_SDMA_UTCL1_INV0
+#define SDMA0_SDMA_UTCL1_INV0__INV_PROC_BUSY__SHIFT                                                           0x0
+#define SDMA0_SDMA_UTCL1_INV0__GPUVM_FRAG_SIZE__SHIFT                                                         0x1
+#define SDMA0_SDMA_UTCL1_INV0__GPUVM_VMID__SHIFT                                                              0x7
+#define SDMA0_SDMA_UTCL1_INV0__GPUVM_MODE__SHIFT                                                              0xb
+#define SDMA0_SDMA_UTCL1_INV0__GPUVM_HIGH__SHIFT                                                              0xd
+#define SDMA0_SDMA_UTCL1_INV0__GPUVM_TAG__SHIFT                                                               0xe
+#define SDMA0_SDMA_UTCL1_INV0__GPUVM_VMID_HIGH__SHIFT                                                         0x12
+#define SDMA0_SDMA_UTCL1_INV0__GPUVM_VMID_LOW__SHIFT                                                          0x16
+#define SDMA0_SDMA_UTCL1_INV0__INV_TYPE__SHIFT                                                                0x1a
+#define SDMA0_SDMA_UTCL1_INV0__INV_PROC_BUSY_MASK                                                             0x00000001L
+#define SDMA0_SDMA_UTCL1_INV0__GPUVM_FRAG_SIZE_MASK                                                           0x0000007EL
+#define SDMA0_SDMA_UTCL1_INV0__GPUVM_VMID_MASK                                                                0x00000780L
+#define SDMA0_SDMA_UTCL1_INV0__GPUVM_MODE_MASK                                                                0x00001800L
+#define SDMA0_SDMA_UTCL1_INV0__GPUVM_HIGH_MASK                                                                0x00002000L
+#define SDMA0_SDMA_UTCL1_INV0__GPUVM_TAG_MASK                                                                 0x0003C000L
+#define SDMA0_SDMA_UTCL1_INV0__GPUVM_VMID_HIGH_MASK                                                           0x003C0000L
+#define SDMA0_SDMA_UTCL1_INV0__GPUVM_VMID_LOW_MASK                                                            0x03C00000L
+#define SDMA0_SDMA_UTCL1_INV0__INV_TYPE_MASK                                                                  0x0C000000L
+//SDMA0_SDMA_UTCL1_INV1
+#define SDMA0_SDMA_UTCL1_INV1__INV_ADDR_LO__SHIFT                                                             0x0
+#define SDMA0_SDMA_UTCL1_INV1__INV_ADDR_LO_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_UTCL1_INV2
+#define SDMA0_SDMA_UTCL1_INV2__CPF_VMID__SHIFT                                                                0x0
+#define SDMA0_SDMA_UTCL1_INV2__CPF_FLUSH_TYPE__SHIFT                                                          0x10
+#define SDMA0_SDMA_UTCL1_INV2__CPF_FRAG_SIZE__SHIFT                                                           0x11
+#define SDMA0_SDMA_UTCL1_INV2__CPF_VMID_MASK                                                                  0x0000FFFFL
+#define SDMA0_SDMA_UTCL1_INV2__CPF_FLUSH_TYPE_MASK                                                            0x00010000L
+#define SDMA0_SDMA_UTCL1_INV2__CPF_FRAG_SIZE_MASK                                                             0x007E0000L
+//SDMA0_SDMA_UTCL1_RD_XNACK0
+#define SDMA0_SDMA_UTCL1_RD_XNACK0__XNACK_FAULT_ADDR_LO__SHIFT                                                0x0
+#define SDMA0_SDMA_UTCL1_RD_XNACK0__XNACK_FAULT_ADDR_LO_MASK                                                  0xFFFFFFFFL
+//SDMA0_SDMA_UTCL1_RD_XNACK1
+#define SDMA0_SDMA_UTCL1_RD_XNACK1__XNACK_FAULT_ADDR_HI__SHIFT                                                0x0
+#define SDMA0_SDMA_UTCL1_RD_XNACK1__XNACK_FAULT_VMID__SHIFT                                                   0xd
+#define SDMA0_SDMA_UTCL1_RD_XNACK1__XNACK_FAULT_VECTOR__SHIFT                                                 0x11
+#define SDMA0_SDMA_UTCL1_RD_XNACK1__XNACK_NULL_VECTOR__SHIFT                                                  0x14
+#define SDMA0_SDMA_UTCL1_RD_XNACK1__XNACK_TIMEOUT_VECTOR__SHIFT                                               0x17
+#define SDMA0_SDMA_UTCL1_RD_XNACK1__XNACK_FAULT_FLAG__SHIFT                                                   0x1a
+#define SDMA0_SDMA_UTCL1_RD_XNACK1__XNACK_NULL_FLAG__SHIFT                                                    0x1b
+#define SDMA0_SDMA_UTCL1_RD_XNACK1__XNACK_TIMEOUT_FLAG__SHIFT                                                 0x1c
+#define SDMA0_SDMA_UTCL1_RD_XNACK1__XNACK_FAULT_ADDR_HI_MASK                                                  0x00001FFFL
+#define SDMA0_SDMA_UTCL1_RD_XNACK1__XNACK_FAULT_VMID_MASK                                                     0x0001E000L
+#define SDMA0_SDMA_UTCL1_RD_XNACK1__XNACK_FAULT_VECTOR_MASK                                                   0x000E0000L
+#define SDMA0_SDMA_UTCL1_RD_XNACK1__XNACK_NULL_VECTOR_MASK                                                    0x00700000L
+#define SDMA0_SDMA_UTCL1_RD_XNACK1__XNACK_TIMEOUT_VECTOR_MASK                                                 0x03800000L
+#define SDMA0_SDMA_UTCL1_RD_XNACK1__XNACK_FAULT_FLAG_MASK                                                     0x04000000L
+#define SDMA0_SDMA_UTCL1_RD_XNACK1__XNACK_NULL_FLAG_MASK                                                      0x08000000L
+#define SDMA0_SDMA_UTCL1_RD_XNACK1__XNACK_TIMEOUT_FLAG_MASK                                                   0x10000000L
+//SDMA0_SDMA_UTCL1_WR_XNACK0
+#define SDMA0_SDMA_UTCL1_WR_XNACK0__XNACK_FAULT_ADDR_LO__SHIFT                                                0x0
+#define SDMA0_SDMA_UTCL1_WR_XNACK0__XNACK_FAULT_ADDR_LO_MASK                                                  0xFFFFFFFFL
+//SDMA0_SDMA_UTCL1_WR_XNACK1
+#define SDMA0_SDMA_UTCL1_WR_XNACK1__XNACK_FAULT_ADDR_HI__SHIFT                                                0x0
+#define SDMA0_SDMA_UTCL1_WR_XNACK1__XNACK_FAULT_VMID__SHIFT                                                   0xd
+#define SDMA0_SDMA_UTCL1_WR_XNACK1__XNACK_FAULT_VECTOR__SHIFT                                                 0x11
+#define SDMA0_SDMA_UTCL1_WR_XNACK1__XNACK_NULL_VECTOR__SHIFT                                                  0x14
+#define SDMA0_SDMA_UTCL1_WR_XNACK1__XNACK_TIMEOUT_VECTOR__SHIFT                                               0x17
+#define SDMA0_SDMA_UTCL1_WR_XNACK1__XNACK_FAULT_FLAG__SHIFT                                                   0x1a
+#define SDMA0_SDMA_UTCL1_WR_XNACK1__XNACK_NULL_FLAG__SHIFT                                                    0x1b
+#define SDMA0_SDMA_UTCL1_WR_XNACK1__XNACK_TIMEOUT_FLAG__SHIFT                                                 0x1c
+#define SDMA0_SDMA_UTCL1_WR_XNACK1__XNACK_FAULT_ADDR_HI_MASK                                                  0x00001FFFL
+#define SDMA0_SDMA_UTCL1_WR_XNACK1__XNACK_FAULT_VMID_MASK                                                     0x0001E000L
+#define SDMA0_SDMA_UTCL1_WR_XNACK1__XNACK_FAULT_VECTOR_MASK                                                   0x000E0000L
+#define SDMA0_SDMA_UTCL1_WR_XNACK1__XNACK_NULL_VECTOR_MASK                                                    0x00700000L
+#define SDMA0_SDMA_UTCL1_WR_XNACK1__XNACK_TIMEOUT_VECTOR_MASK                                                 0x03800000L
+#define SDMA0_SDMA_UTCL1_WR_XNACK1__XNACK_FAULT_FLAG_MASK                                                     0x04000000L
+#define SDMA0_SDMA_UTCL1_WR_XNACK1__XNACK_NULL_FLAG_MASK                                                      0x08000000L
+#define SDMA0_SDMA_UTCL1_WR_XNACK1__XNACK_TIMEOUT_FLAG_MASK                                                   0x10000000L
+//SDMA0_SDMA_CHICKEN_BITS_2
+#define SDMA0_SDMA_CHICKEN_BITS_2__MCU_CMD_PROC_DELAY__SHIFT                                                  0x0
+#define SDMA0_SDMA_CHICKEN_BITS_2__MCU_SEND_POSTCODE_EN__SHIFT                                                0x4
+#define SDMA0_SDMA_CHICKEN_BITS_2__MCU_DEBUG_MODE__SHIFT                                                      0x5
+#define SDMA0_SDMA_CHICKEN_BITS_2__RESERVED_7_6__SHIFT                                                        0x6
+#define SDMA0_SDMA_CHICKEN_BITS_2__WPTR_POLL_OUTSTANDING__SHIFT                                               0x8
+#define SDMA0_SDMA_CHICKEN_BITS_2__RESERVED_14_12__SHIFT                                                      0xc
+#define SDMA0_SDMA_CHICKEN_BITS_2__RESERVED_15__SHIFT                                                         0xf
+#define SDMA0_SDMA_CHICKEN_BITS_2__RB_FIFO_WATERMARK__SHIFT                                                   0x10
+#define SDMA0_SDMA_CHICKEN_BITS_2__IB_FIFO_WATERMARK__SHIFT                                                   0x12
+#define SDMA0_SDMA_CHICKEN_BITS_2__CH_WR_WATERMARK_MARGIN__SHIFT                                              0x14
+#define SDMA0_SDMA_CHICKEN_BITS_2__CH_RD_WATERMARK__SHIFT                                                     0x17
+#define SDMA0_SDMA_CHICKEN_BITS_2__CH_WR_WATERMARK__SHIFT                                                     0x19
+#define SDMA0_SDMA_CHICKEN_BITS_2__RESERVED_30__SHIFT                                                         0x1e
+#define SDMA0_SDMA_CHICKEN_BITS_2__PIO_VFID_SOURCE__SHIFT                                                     0x1f
+#define SDMA0_SDMA_CHICKEN_BITS_2__MCU_CMD_PROC_DELAY_MASK                                                    0x0000000FL
+#define SDMA0_SDMA_CHICKEN_BITS_2__MCU_SEND_POSTCODE_EN_MASK                                                  0x00000010L
+#define SDMA0_SDMA_CHICKEN_BITS_2__MCU_DEBUG_MODE_MASK                                                        0x00000020L
+#define SDMA0_SDMA_CHICKEN_BITS_2__RESERVED_7_6_MASK                                                          0x000000C0L
+#define SDMA0_SDMA_CHICKEN_BITS_2__WPTR_POLL_OUTSTANDING_MASK                                                 0x00000F00L
+#define SDMA0_SDMA_CHICKEN_BITS_2__RESERVED_14_12_MASK                                                        0x00007000L
+#define SDMA0_SDMA_CHICKEN_BITS_2__RESERVED_15_MASK                                                           0x00008000L
+#define SDMA0_SDMA_CHICKEN_BITS_2__RB_FIFO_WATERMARK_MASK                                                     0x00030000L
+#define SDMA0_SDMA_CHICKEN_BITS_2__IB_FIFO_WATERMARK_MASK                                                     0x000C0000L
+#define SDMA0_SDMA_CHICKEN_BITS_2__CH_WR_WATERMARK_MARGIN_MASK                                                0x00700000L
+#define SDMA0_SDMA_CHICKEN_BITS_2__CH_RD_WATERMARK_MASK                                                       0x01800000L
+#define SDMA0_SDMA_CHICKEN_BITS_2__CH_WR_WATERMARK_MASK                                                       0x3E000000L
+#define SDMA0_SDMA_CHICKEN_BITS_2__RESERVED_30_MASK                                                           0x40000000L
+#define SDMA0_SDMA_CHICKEN_BITS_2__PIO_VFID_SOURCE_MASK                                                       0x80000000L
+//SDMA0_SDMA_STATUS3_REG
+#define SDMA0_SDMA_STATUS3_REG__PREV_VM_CMD__SHIFT                                                            0x10
+#define SDMA0_SDMA_STATUS3_REG__EXCEPTION_IDLE__SHIFT                                                         0x14
+#define SDMA0_SDMA_STATUS3_REG__AQL_PREV_CMD_IDLE__SHIFT                                                      0x15
+#define SDMA0_SDMA_STATUS3_REG__TLBI_IDLE__SHIFT                                                              0x16
+#define SDMA0_SDMA_STATUS3_REG__GCR_IDLE__SHIFT                                                               0x17
+#define SDMA0_SDMA_STATUS3_REG__INVREQ_IDLE__SHIFT                                                            0x18
+#define SDMA0_SDMA_STATUS3_REG__QUEUE_ID_MATCH__SHIFT                                                         0x19
+#define SDMA0_SDMA_STATUS3_REG__INT_QUEUE_ID__SHIFT                                                           0x1a
+#define SDMA0_SDMA_STATUS3_REG__TMZ_MTYPE_STATUS__SHIFT                                                       0x1e
+#define SDMA0_SDMA_STATUS3_REG__PREV_VM_CMD_MASK                                                              0x000F0000L
+#define SDMA0_SDMA_STATUS3_REG__EXCEPTION_IDLE_MASK                                                           0x00100000L
+#define SDMA0_SDMA_STATUS3_REG__AQL_PREV_CMD_IDLE_MASK                                                        0x00200000L
+#define SDMA0_SDMA_STATUS3_REG__TLBI_IDLE_MASK                                                                0x00400000L
+#define SDMA0_SDMA_STATUS3_REG__GCR_IDLE_MASK                                                                 0x00800000L
+#define SDMA0_SDMA_STATUS3_REG__INVREQ_IDLE_MASK                                                              0x01000000L
+#define SDMA0_SDMA_STATUS3_REG__QUEUE_ID_MATCH_MASK                                                           0x02000000L
+#define SDMA0_SDMA_STATUS3_REG__INT_QUEUE_ID_MASK                                                             0x3C000000L
+#define SDMA0_SDMA_STATUS3_REG__TMZ_MTYPE_STATUS_MASK                                                         0xC0000000L
+//SDMA0_SDMA_GLOBAL_QUANTUM
+#define SDMA0_SDMA_GLOBAL_QUANTUM__GLOBAL_FOCUS_QUANTUM__SHIFT                                                0x0
+#define SDMA0_SDMA_GLOBAL_QUANTUM__GLOBAL_NORMAL_QUANTUM__SHIFT                                               0x8
+#define SDMA0_SDMA_GLOBAL_QUANTUM__GLOBAL_FOCUS_QUANTUM_MASK                                                  0x000000FFL
+#define SDMA0_SDMA_GLOBAL_QUANTUM__GLOBAL_NORMAL_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA0_SDMA_ERROR_LOG
+#define SDMA0_SDMA_ERROR_LOG__OVERRIDE__SHIFT                                                                 0x0
+#define SDMA0_SDMA_ERROR_LOG__STATUS__SHIFT                                                                   0x10
+#define SDMA0_SDMA_ERROR_LOG__OVERRIDE_MASK                                                                   0x0000FFFFL
+#define SDMA0_SDMA_ERROR_LOG__STATUS_MASK                                                                     0xFFFF0000L
+//SDMA0_SDMA_PUB_DUMMY_REG0
+#define SDMA0_SDMA_PUB_DUMMY_REG0__VALUE__SHIFT                                                               0x0
+#define SDMA0_SDMA_PUB_DUMMY_REG0__VALUE_MASK                                                                 0xFFFFFFFFL
+//SDMA0_SDMA_PUB_DUMMY_REG1
+#define SDMA0_SDMA_PUB_DUMMY_REG1__VALUE__SHIFT                                                               0x0
+#define SDMA0_SDMA_PUB_DUMMY_REG1__VALUE_MASK                                                                 0xFFFFFFFFL
+//SDMA0_SDMA_PUB_DUMMY_REG2
+#define SDMA0_SDMA_PUB_DUMMY_REG2__VALUE__SHIFT                                                               0x0
+#define SDMA0_SDMA_PUB_DUMMY_REG2__VALUE_MASK                                                                 0xFFFFFFFFL
+//SDMA0_SDMA_PUB_DUMMY_REG3
+#define SDMA0_SDMA_PUB_DUMMY_REG3__VALUE__SHIFT                                                               0x0
+#define SDMA0_SDMA_PUB_DUMMY_REG3__VALUE_MASK                                                                 0xFFFFFFFFL
+//SDMA0_SDMA_MCU_COUNTER
+#define SDMA0_SDMA_MCU_COUNTER__VALUE__SHIFT                                                                  0x0
+#define SDMA0_SDMA_MCU_COUNTER__VALUE_MASK                                                                    0xFFFFFFFFL
+//SDMA0_SDMA_CRD_CNTL
+#define SDMA0_SDMA_CRD_CNTL__CH_WRREQ_CREDIT__SHIFT                                                           0x13
+#define SDMA0_SDMA_CRD_CNTL__CH_RDREQ_CREDIT__SHIFT                                                           0x19
+#define SDMA0_SDMA_CRD_CNTL__CH_WRREQ_CREDIT_MASK                                                             0x01F80000L
+#define SDMA0_SDMA_CRD_CNTL__CH_RDREQ_CREDIT_MASK                                                             0x7E000000L
+//SDMA0_SDMA_RLC_CGCG_CTRL
+#define SDMA0_SDMA_RLC_CGCG_CTRL__CGCG_INT_ENABLE__SHIFT                                                      0x1
+#define SDMA0_SDMA_RLC_CGCG_CTRL__MCU_CGCG_ALLOW__SHIFT                                                       0x4
+#define SDMA0_SDMA_RLC_CGCG_CTRL__CGCG_IDLE_HYSTERESIS__SHIFT                                                 0x10
+#define SDMA0_SDMA_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK                                                        0x00000002L
+#define SDMA0_SDMA_RLC_CGCG_CTRL__MCU_CGCG_ALLOW_MASK                                                         0x00000010L
+#define SDMA0_SDMA_RLC_CGCG_CTRL__CGCG_IDLE_HYSTERESIS_MASK                                                   0xFFFF0000L
+//SDMA0_SDMA_GPU_IOV_VIOLATION_LOG
+#define SDMA0_SDMA_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT                                             0x0
+#define SDMA0_SDMA_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT                                    0x1
+#define SDMA0_SDMA_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT                                                      0x2
+#define SDMA0_SDMA_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION__SHIFT                                              0x14
+#define SDMA0_SDMA_GPU_IOV_VIOLATION_LOG__VF__SHIFT                                                           0x15
+#define SDMA0_SDMA_GPU_IOV_VIOLATION_LOG__VFID__SHIFT                                                         0x16
+#define SDMA0_SDMA_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK                                               0x00000001L
+#define SDMA0_SDMA_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK                                      0x00000002L
+#define SDMA0_SDMA_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK                                                        0x000FFFFCL
+#define SDMA0_SDMA_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION_MASK                                                0x00100000L
+#define SDMA0_SDMA_GPU_IOV_VIOLATION_LOG__VF_MASK                                                             0x00200000L
+#define SDMA0_SDMA_GPU_IOV_VIOLATION_LOG__VFID_MASK                                                           0x07C00000L
+//SDMA0_SDMA_AQL_STATUS
+#define SDMA0_SDMA_AQL_STATUS__COMPLETE_SIGNAL_EMPTY__SHIFT                                                   0x0
+#define SDMA0_SDMA_AQL_STATUS__INVALID_CMD_EMPTY__SHIFT                                                       0x1
+#define SDMA0_SDMA_AQL_STATUS__COMPLETE_SIGNAL_EMPTY_MASK                                                     0x00000001L
+#define SDMA0_SDMA_AQL_STATUS__INVALID_CMD_EMPTY_MASK                                                         0x00000002L
+//SDMA0_SDMA_TLBI_GCR_CNTL
+#define SDMA0_SDMA_TLBI_GCR_CNTL__TLBI_CMD_DW__SHIFT                                                          0x0
+#define SDMA0_SDMA_TLBI_GCR_CNTL__GCR_CMD_DW__SHIFT                                                           0x4
+#define SDMA0_SDMA_TLBI_GCR_CNTL__TLBI_CREDIT__SHIFT                                                          0x10
+#define SDMA0_SDMA_TLBI_GCR_CNTL__GCR_CREDIT__SHIFT                                                           0x18
+#define SDMA0_SDMA_TLBI_GCR_CNTL__TLBI_CMD_DW_MASK                                                            0x0000000FL
+#define SDMA0_SDMA_TLBI_GCR_CNTL__GCR_CMD_DW_MASK                                                             0x000000F0L
+#define SDMA0_SDMA_TLBI_GCR_CNTL__TLBI_CREDIT_MASK                                                            0x00FF0000L
+#define SDMA0_SDMA_TLBI_GCR_CNTL__GCR_CREDIT_MASK                                                             0xFF000000L
+//SDMA0_SDMA_INT_STATUS
+#define SDMA0_SDMA_INT_STATUS__DATA__SHIFT                                                                    0x0
+#define SDMA0_SDMA_INT_STATUS__DATA_MASK                                                                      0xFFFFFFFFL
+//SDMA0_SDMA_GPU_IOV_VIOLATION_LOG2
+#define SDMA0_SDMA_GPU_IOV_VIOLATION_LOG2__INITIATOR_ID__SHIFT                                                0x0
+#define SDMA0_SDMA_GPU_IOV_VIOLATION_LOG2__INITIATOR_ID_MASK                                                  0x000003FFL
+//SDMA0_SDMA_INVALID_ADDR_LO
+#define SDMA0_SDMA_INVALID_ADDR_LO__VALUE__SHIFT                                                              0x0
+#define SDMA0_SDMA_INVALID_ADDR_LO__VALUE_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_INVALID_ADDR_HI
+#define SDMA0_SDMA_INVALID_ADDR_HI__VALUE__SHIFT                                                              0x0
+#define SDMA0_SDMA_INVALID_ADDR_HI__VALUE_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_INVALID_ADDR_SOURCE
+#define SDMA0_SDMA_INVALID_ADDR_SOURCE__ID__SHIFT                                                             0x0
+#define SDMA0_SDMA_INVALID_ADDR_SOURCE__VMID__SHIFT                                                           0x6
+#define SDMA0_SDMA_INVALID_ADDR_SOURCE__VFID__SHIFT                                                           0xa
+#define SDMA0_SDMA_INVALID_ADDR_SOURCE__VF__SHIFT                                                             0xf
+#define SDMA0_SDMA_INVALID_ADDR_SOURCE__ID_MASK                                                               0x0000003FL
+#define SDMA0_SDMA_INVALID_ADDR_SOURCE__VMID_MASK                                                             0x000003C0L
+#define SDMA0_SDMA_INVALID_ADDR_SOURCE__VFID_MASK                                                             0x00007C00L
+#define SDMA0_SDMA_INVALID_ADDR_SOURCE__VF_MASK                                                               0x00008000L
+//SDMA0_SDMA_CLOCK_GATING_STATUS
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__PTR_MGCG_CLK_STATUS__SHIFT                                            0x8
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__PIO_MGCG_CLK_STATUS__SHIFT                                            0x9
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__MCU_MGCG_CLK_STATUS__SHIFT                                            0xa
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__COPY_ENG_MGCG_CLK_STATUS__SHIFT                                       0xb
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__SERVE_ENG_MGCG_CLK_STATUS__SHIFT                                      0xc
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__CMD_FETCH_MGCG_CLK_STATUS__SHIFT                                      0xd
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__GU_MEMREQ_MGCG_CLK_STATUS__SHIFT                                      0xe
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__INV_MGCG_CLK_STATUS__SHIFT                                            0xf
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__GU_CACHE_MGCG_CLK_STATUS__SHIFT                                       0x10
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__IC_CACHE_MGCG_CLK_STATUS__SHIFT                                       0x11
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__MEM_CHNL_MGCG_CLK_STATUS__SHIFT                                       0x12
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__MEM_CHNL_CESE_MGCG_CLK_STATUS__SHIFT                                  0x13
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__FE_ENG_MGCG_CLK_STATUS__SHIFT                                         0x14
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__MEM_CHNL_CESEFE_MGCG_CLK_STATUS__SHIFT                                0x15
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__MEM_CHNL_FE_WARMUP_MGCG_CLK_STATUS__SHIFT                             0x16
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__PTR_MGCG_CLK_STATUS_MASK                                              0x00000100L
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__PIO_MGCG_CLK_STATUS_MASK                                              0x00000200L
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__MCU_MGCG_CLK_STATUS_MASK                                              0x00000400L
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__COPY_ENG_MGCG_CLK_STATUS_MASK                                         0x00000800L
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__SERVE_ENG_MGCG_CLK_STATUS_MASK                                        0x00001000L
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__CMD_FETCH_MGCG_CLK_STATUS_MASK                                        0x00002000L
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__GU_MEMREQ_MGCG_CLK_STATUS_MASK                                        0x00004000L
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__INV_MGCG_CLK_STATUS_MASK                                              0x00008000L
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__GU_CACHE_MGCG_CLK_STATUS_MASK                                         0x00010000L
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__IC_CACHE_MGCG_CLK_STATUS_MASK                                         0x00020000L
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__MEM_CHNL_MGCG_CLK_STATUS_MASK                                         0x00040000L
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__MEM_CHNL_CESE_MGCG_CLK_STATUS_MASK                                    0x00080000L
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__FE_ENG_MGCG_CLK_STATUS_MASK                                           0x00100000L
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__MEM_CHNL_CESEFE_MGCG_CLK_STATUS_MASK                                  0x00200000L
+#define SDMA0_SDMA_CLOCK_GATING_STATUS__MEM_CHNL_FE_WARMUP_MGCG_CLK_STATUS_MASK                               0x00400000L
+//SDMA0_SDMA_STATUS4_REG
+#define SDMA0_SDMA_STATUS4_REG__IDLE__SHIFT                                                                   0x0
+#define SDMA0_SDMA_STATUS4_REG__IH_OUTSTANDING__SHIFT                                                         0x2
+#define SDMA0_SDMA_STATUS4_REG__RESERVED__SHIFT                                                               0x3
+#define SDMA0_SDMA_STATUS4_REG__CH_RD_OUTSTANDING__SHIFT                                                      0x4
+#define SDMA0_SDMA_STATUS4_REG__CH_WR_OUTSTANDING__SHIFT                                                      0x5
+#define SDMA0_SDMA_STATUS4_REG__GCR_OUTSTANDING__SHIFT                                                        0x6
+#define SDMA0_SDMA_STATUS4_REG__TLBI_OUTSTANDING__SHIFT                                                       0x7
+#define SDMA0_SDMA_STATUS4_REG__UTCL2_RD_OUTSTANDING__SHIFT                                                   0x8
+#define SDMA0_SDMA_STATUS4_REG__UTCL2_WR_OUTSTANDING__SHIFT                                                   0x9
+#define SDMA0_SDMA_STATUS4_REG__CMDFETCH_OUTSTANDING__SHIFT                                                   0xa
+#define SDMA0_SDMA_STATUS4_REG__RESERVED_11__SHIFT                                                            0xb
+#define SDMA0_SDMA_STATUS4_REG__RESERVED_13_12__SHIFT                                                         0xc
+#define SDMA0_SDMA_STATUS4_REG__RESERVED_15_14__SHIFT                                                         0xe
+#define SDMA0_SDMA_STATUS4_REG__ACTIVE_QUEUE_ID__SHIFT                                                        0x10
+#define SDMA0_SDMA_STATUS4_REG__SRIOV_WATING_RLCV_CMD__SHIFT                                                  0x14
+#define SDMA0_SDMA_STATUS4_REG__SRIOV_SDMA_EXECUTING_CMD__SHIFT                                               0x15
+#define SDMA0_SDMA_STATUS4_REG__UTCL2_RD_XNACK_FAULT__SHIFT                                                   0x16
+#define SDMA0_SDMA_STATUS4_REG__UTCL2_RD_XNACK_NULL__SHIFT                                                    0x17
+#define SDMA0_SDMA_STATUS4_REG__UTCL2_RD_XNACK_TIMEOUT__SHIFT                                                 0x18
+#define SDMA0_SDMA_STATUS4_REG__UTCL2_WR_XNACK_FAULT__SHIFT                                                   0x19
+#define SDMA0_SDMA_STATUS4_REG__UTCL2_WR_XNACK_NULL__SHIFT                                                    0x1a
+#define SDMA0_SDMA_STATUS4_REG__UTCL2_WR_XNACK_TIMEOUT__SHIFT                                                 0x1b
+#define SDMA0_SDMA_STATUS4_REG__IDLE_MASK                                                                     0x00000001L
+#define SDMA0_SDMA_STATUS4_REG__IH_OUTSTANDING_MASK                                                           0x00000004L
+#define SDMA0_SDMA_STATUS4_REG__RESERVED_MASK                                                                 0x00000008L
+#define SDMA0_SDMA_STATUS4_REG__CH_RD_OUTSTANDING_MASK                                                        0x00000010L
+#define SDMA0_SDMA_STATUS4_REG__CH_WR_OUTSTANDING_MASK                                                        0x00000020L
+#define SDMA0_SDMA_STATUS4_REG__GCR_OUTSTANDING_MASK                                                          0x00000040L
+#define SDMA0_SDMA_STATUS4_REG__TLBI_OUTSTANDING_MASK                                                         0x00000080L
+#define SDMA0_SDMA_STATUS4_REG__UTCL2_RD_OUTSTANDING_MASK                                                     0x00000100L
+#define SDMA0_SDMA_STATUS4_REG__UTCL2_WR_OUTSTANDING_MASK                                                     0x00000200L
+#define SDMA0_SDMA_STATUS4_REG__CMDFETCH_OUTSTANDING_MASK                                                     0x00000400L
+#define SDMA0_SDMA_STATUS4_REG__RESERVED_11_MASK                                                              0x00000800L
+#define SDMA0_SDMA_STATUS4_REG__RESERVED_13_12_MASK                                                           0x00003000L
+#define SDMA0_SDMA_STATUS4_REG__RESERVED_15_14_MASK                                                           0x0000C000L
+#define SDMA0_SDMA_STATUS4_REG__ACTIVE_QUEUE_ID_MASK                                                          0x000F0000L
+#define SDMA0_SDMA_STATUS4_REG__SRIOV_WATING_RLCV_CMD_MASK                                                    0x00100000L
+#define SDMA0_SDMA_STATUS4_REG__SRIOV_SDMA_EXECUTING_CMD_MASK                                                 0x00200000L
+#define SDMA0_SDMA_STATUS4_REG__UTCL2_RD_XNACK_FAULT_MASK                                                     0x00400000L
+#define SDMA0_SDMA_STATUS4_REG__UTCL2_RD_XNACK_NULL_MASK                                                      0x00800000L
+#define SDMA0_SDMA_STATUS4_REG__UTCL2_RD_XNACK_TIMEOUT_MASK                                                   0x01000000L
+#define SDMA0_SDMA_STATUS4_REG__UTCL2_WR_XNACK_FAULT_MASK                                                     0x02000000L
+#define SDMA0_SDMA_STATUS4_REG__UTCL2_WR_XNACK_NULL_MASK                                                      0x04000000L
+#define SDMA0_SDMA_STATUS4_REG__UTCL2_WR_XNACK_TIMEOUT_MASK                                                   0x08000000L
+//SDMA0_SDMA_SCRATCH_RAM_DATA
+#define SDMA0_SDMA_SCRATCH_RAM_DATA__DATA__SHIFT                                                              0x0
+#define SDMA0_SDMA_SCRATCH_RAM_DATA__DATA_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_SCRATCH_RAM_ADDR
+#define SDMA0_SDMA_SCRATCH_RAM_ADDR__ADDR__SHIFT                                                              0x0
+#define SDMA0_SDMA_SCRATCH_RAM_ADDR__ADDR_MASK                                                                0x0000007FL
+//SDMA0_SDMA_TIMESTAMP_CNTL
+#define SDMA0_SDMA_TIMESTAMP_CNTL__CAPTURE__SHIFT                                                             0x0
+#define SDMA0_SDMA_TIMESTAMP_CNTL__CAPTURE_MASK                                                               0x00000001L
+//SDMA0_SDMA_STATUS5_REG
+#define SDMA0_SDMA_STATUS5_REG__QUEUE0_RB_ENABLE_STATUS__SHIFT                                                0x0
+#define SDMA0_SDMA_STATUS5_REG__QUEUE1_RB_ENABLE_STATUS__SHIFT                                                0x1
+#define SDMA0_SDMA_STATUS5_REG__QUEUE2_RB_ENABLE_STATUS__SHIFT                                                0x2
+#define SDMA0_SDMA_STATUS5_REG__QUEUE3_RB_ENABLE_STATUS__SHIFT                                                0x3
+#define SDMA0_SDMA_STATUS5_REG__QUEUE4_RB_ENABLE_STATUS__SHIFT                                                0x4
+#define SDMA0_SDMA_STATUS5_REG__QUEUE5_RB_ENABLE_STATUS__SHIFT                                                0x5
+#define SDMA0_SDMA_STATUS5_REG__QUEUE6_RB_ENABLE_STATUS__SHIFT                                                0x6
+#define SDMA0_SDMA_STATUS5_REG__QUEUE7_RB_ENABLE_STATUS__SHIFT                                                0x7
+#define SDMA0_SDMA_STATUS5_REG__QUEUE8_RB_ENABLE_STATUS__SHIFT                                                0x8
+#define SDMA0_SDMA_STATUS5_REG__QUEUE9_RB_ENABLE_STATUS__SHIFT                                                0x9
+#define SDMA0_SDMA_STATUS5_REG__QUEUE0_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x10
+#define SDMA0_SDMA_STATUS5_REG__QUEUE1_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x11
+#define SDMA0_SDMA_STATUS5_REG__QUEUE2_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x12
+#define SDMA0_SDMA_STATUS5_REG__QUEUE3_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x13
+#define SDMA0_SDMA_STATUS5_REG__QUEUE4_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x14
+#define SDMA0_SDMA_STATUS5_REG__QUEUE5_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x15
+#define SDMA0_SDMA_STATUS5_REG__QUEUE6_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x16
+#define SDMA0_SDMA_STATUS5_REG__QUEUE7_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x17
+#define SDMA0_SDMA_STATUS5_REG__QUEUE8_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x18
+#define SDMA0_SDMA_STATUS5_REG__QUEUE9_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x19
+#define SDMA0_SDMA_STATUS5_REG__QUEUE0_RB_ENABLE_STATUS_MASK                                                  0x00000001L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE1_RB_ENABLE_STATUS_MASK                                                  0x00000002L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE2_RB_ENABLE_STATUS_MASK                                                  0x00000004L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE3_RB_ENABLE_STATUS_MASK                                                  0x00000008L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE4_RB_ENABLE_STATUS_MASK                                                  0x00000010L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE5_RB_ENABLE_STATUS_MASK                                                  0x00000020L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE6_RB_ENABLE_STATUS_MASK                                                  0x00000040L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE7_RB_ENABLE_STATUS_MASK                                                  0x00000080L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE8_RB_ENABLE_STATUS_MASK                                                  0x00000100L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE9_RB_ENABLE_STATUS_MASK                                                  0x00000200L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE0_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x00010000L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE1_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x00020000L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE2_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x00040000L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE3_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x00080000L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE4_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x00100000L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE5_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x00200000L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE6_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x00400000L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE7_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x00800000L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE8_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x01000000L
+#define SDMA0_SDMA_STATUS5_REG__QUEUE9_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x02000000L
+//SDMA0_SDMA_QUEUE_RESET_REQ
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE0_RESET__SHIFT                                                       0x0
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE1_RESET__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE2_RESET__SHIFT                                                       0x2
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE3_RESET__SHIFT                                                       0x3
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE4_RESET__SHIFT                                                       0x4
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE5_RESET__SHIFT                                                       0x5
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE6_RESET__SHIFT                                                       0x6
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE7_RESET__SHIFT                                                       0x7
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE8_RESET__SHIFT                                                       0x8
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE9_RESET__SHIFT                                                       0x9
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE0_RESET_MASK                                                         0x00000001L
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE1_RESET_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE2_RESET_MASK                                                         0x00000004L
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE3_RESET_MASK                                                         0x00000008L
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE4_RESET_MASK                                                         0x00000010L
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE5_RESET_MASK                                                         0x00000020L
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE6_RESET_MASK                                                         0x00000040L
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE7_RESET_MASK                                                         0x00000080L
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE8_RESET_MASK                                                         0x00000100L
+#define SDMA0_SDMA_QUEUE_RESET_REQ__QUEUE9_RESET_MASK                                                         0x00000200L
+//SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE0_DEQUEUE_REQ__SHIFT                                           0x0
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE1_DEQUEUE_REQ__SHIFT                                           0x1
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE2_DEQUEUE_REQ__SHIFT                                           0x2
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE3_DEQUEUE_REQ__SHIFT                                           0x3
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE4_DEQUEUE_REQ__SHIFT                                           0x4
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE5_DEQUEUE_REQ__SHIFT                                           0x5
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE6_DEQUEUE_REQ__SHIFT                                           0x6
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE7_DEQUEUE_REQ__SHIFT                                           0x7
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE8_DEQUEUE_REQ__SHIFT                                           0x8
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE9_DEQUEUE_REQ__SHIFT                                           0x9
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE0_DEQUEUE_REQ_MASK                                             0x00000001L
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE1_DEQUEUE_REQ_MASK                                             0x00000002L
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE2_DEQUEUE_REQ_MASK                                             0x00000004L
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE3_DEQUEUE_REQ_MASK                                             0x00000008L
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE4_DEQUEUE_REQ_MASK                                             0x00000010L
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE5_DEQUEUE_REQ_MASK                                             0x00000020L
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE6_DEQUEUE_REQ_MASK                                             0x00000040L
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE7_DEQUEUE_REQ_MASK                                             0x00000080L
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE8_DEQUEUE_REQ_MASK                                             0x00000100L
+#define SDMA0_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE9_DEQUEUE_REQ_MASK                                             0x00000200L
+//SDMA0_SDMA_STATUS6_REG
+#define SDMA0_SDMA_STATUS6_REG__ID__SHIFT                                                                     0x0
+#define SDMA0_SDMA_STATUS6_REG__TH1MCU_INSTR_PTR__SHIFT                                                       0x2
+#define SDMA0_SDMA_STATUS6_REG__TH1_EXCEPTION__SHIFT                                                          0x10
+#define SDMA0_SDMA_STATUS6_REG__ID_MASK                                                                       0x00000003L
+#define SDMA0_SDMA_STATUS6_REG__TH1MCU_INSTR_PTR_MASK                                                         0x0000FFFCL
+#define SDMA0_SDMA_STATUS6_REG__TH1_EXCEPTION_MASK                                                            0xFFFF0000L
+//SDMA0_SDMA_STATUS7_REG
+#define SDMA0_SDMA_STATUS7_REG__BLT_REQ_DROP__SHIFT                                                           0x0
+#define SDMA0_SDMA_STATUS7_REG__BLT_REQ_DROP_MASK                                                             0x00000001L
+//SDMA0_SDMA_STATUS8_REG
+#define SDMA0_SDMA_STATUS8_REG__LD_CTXSW_COND__SHIFT                                                          0x0
+#define SDMA0_SDMA_STATUS8_REG__LD_CTXSW_COND_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_CE_CTRL
+#define SDMA0_SDMA_CE_CTRL__RD_LUT_WATERMARK__SHIFT                                                           0x0
+#define SDMA0_SDMA_CE_CTRL__RD_LUT_DEPTH__SHIFT                                                               0x3
+#define SDMA0_SDMA_CE_CTRL__WR_AFIFO_WATERMARK__SHIFT                                                         0x5
+#define SDMA0_SDMA_CE_CTRL__RESERVED__SHIFT                                                                   0x9
+#define SDMA0_SDMA_CE_CTRL__RD_LUT_WATERMARK_MASK                                                             0x00000007L
+#define SDMA0_SDMA_CE_CTRL__RD_LUT_DEPTH_MASK                                                                 0x00000018L
+#define SDMA0_SDMA_CE_CTRL__WR_AFIFO_WATERMARK_MASK                                                           0x000000E0L
+#define SDMA0_SDMA_CE_CTRL__RESERVED_MASK                                                                     0xFFFFFE00L
+//SDMA0_SDMA_RAS_STATUS
+#define SDMA0_SDMA_RAS_STATUS__RB_FETCH_ECC__SHIFT                                                            0x0
+#define SDMA0_SDMA_RAS_STATUS__IB_FETCH_ECC__SHIFT                                                            0x1
+#define SDMA0_SDMA_RAS_STATUS__MCU_DATA_ECC__SHIFT                                                            0x2
+#define SDMA0_SDMA_RAS_STATUS__WPTR_POLL_ECC__SHIFT                                                           0x3
+#define SDMA0_SDMA_RAS_STATUS__COPY_DATA_ECC__SHIFT                                                           0x4
+#define SDMA0_SDMA_RAS_STATUS__INSTR_FETCH_ECC__SHIFT                                                         0x5
+#define SDMA0_SDMA_RAS_STATUS__ATOMIC_ECC__SHIFT                                                              0x6
+#define SDMA0_SDMA_RAS_STATUS__FRONTEND_ECC__SHIFT                                                            0x7
+#define SDMA0_SDMA_RAS_STATUS__SRAM_ECC__SHIFT                                                                0x10
+#define SDMA0_SDMA_RAS_STATUS__BUS_PARITY__SHIFT                                                              0x12
+#define SDMA0_SDMA_RAS_STATUS__VMID__SHIFT                                                                    0x14
+#define SDMA0_SDMA_RAS_STATUS__VFID__SHIFT                                                                    0x18
+#define SDMA0_SDMA_RAS_STATUS__VF__SHIFT                                                                      0x1d
+#define SDMA0_SDMA_RAS_STATUS__RB_FETCH_ECC_MASK                                                              0x00000001L
+#define SDMA0_SDMA_RAS_STATUS__IB_FETCH_ECC_MASK                                                              0x00000002L
+#define SDMA0_SDMA_RAS_STATUS__MCU_DATA_ECC_MASK                                                              0x00000004L
+#define SDMA0_SDMA_RAS_STATUS__WPTR_POLL_ECC_MASK                                                             0x00000008L
+#define SDMA0_SDMA_RAS_STATUS__COPY_DATA_ECC_MASK                                                             0x00000010L
+#define SDMA0_SDMA_RAS_STATUS__INSTR_FETCH_ECC_MASK                                                           0x00000020L
+#define SDMA0_SDMA_RAS_STATUS__ATOMIC_ECC_MASK                                                                0x00000040L
+#define SDMA0_SDMA_RAS_STATUS__FRONTEND_ECC_MASK                                                              0x00000080L
+#define SDMA0_SDMA_RAS_STATUS__SRAM_ECC_MASK                                                                  0x00010000L
+#define SDMA0_SDMA_RAS_STATUS__BUS_PARITY_MASK                                                                0x00040000L
+#define SDMA0_SDMA_RAS_STATUS__VMID_MASK                                                                      0x00F00000L
+#define SDMA0_SDMA_RAS_STATUS__VFID_MASK                                                                      0x1F000000L
+#define SDMA0_SDMA_RAS_STATUS__VF_MASK                                                                        0x20000000L
+//SDMA0_SDMA_POISON_INFO
+#define SDMA0_SDMA_POISON_INFO__VMID__SHIFT                                                                   0x0
+#define SDMA0_SDMA_POISON_INFO__VFID__SHIFT                                                                   0x4
+#define SDMA0_SDMA_POISON_INFO__VF__SHIFT                                                                     0x9
+#define SDMA0_SDMA_POISON_INFO__VMID_MASK                                                                     0x0000000FL
+#define SDMA0_SDMA_POISON_INFO__VFID_MASK                                                                     0x000001F0L
+#define SDMA0_SDMA_POISON_INFO__VF_MASK                                                                       0x00000200L
+//SDMA0_SDMA_MEMHUB_CNTL
+#define SDMA0_SDMA_MEMHUB_CNTL__SPECULATIVE_DATA_READ__SHIFT                                                  0x0
+#define SDMA0_SDMA_MEMHUB_CNTL__SPECULATIVE_DATA_READ_MASK                                                    0x00000003L
+//SDMA0_SDMA_QUEUE0_RB_CNTL
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA0_SDMA_QUEUE0_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA0_SDMA_QUEUE0_RB_BASE
+#define SDMA0_SDMA_QUEUE0_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE0_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_RB_BASE_HI
+#define SDMA0_SDMA_QUEUE0_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE0_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA0_SDMA_QUEUE0_RB_RPTR
+#define SDMA0_SDMA_QUEUE0_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE0_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_RB_RPTR_HI
+#define SDMA0_SDMA_QUEUE0_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE0_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_RB_WPTR
+#define SDMA0_SDMA_QUEUE0_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE0_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_RB_WPTR_HI
+#define SDMA0_SDMA_QUEUE0_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE0_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_RB_RPTR_ADDR_LO
+#define SDMA0_SDMA_QUEUE0_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_SDMA_QUEUE0_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE0_RB_RPTR_ADDR_HI
+#define SDMA0_SDMA_QUEUE0_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE0_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_IB_CNTL
+#define SDMA0_SDMA_QUEUE0_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE0_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA0_SDMA_QUEUE0_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE0_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA0_SDMA_QUEUE0_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA0_SDMA_QUEUE0_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE0_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA0_SDMA_QUEUE0_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE0_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA0_SDMA_QUEUE0_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA0_SDMA_QUEUE0_IB_RPTR
+#define SDMA0_SDMA_QUEUE0_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA0_SDMA_QUEUE0_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA0_SDMA_QUEUE0_IB_OFFSET
+#define SDMA0_SDMA_QUEUE0_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE0_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA0_SDMA_QUEUE0_IB_BASE_LO
+#define SDMA0_SDMA_QUEUE0_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_SDMA_QUEUE0_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE0_IB_BASE_HI
+#define SDMA0_SDMA_QUEUE0_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE0_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_IB_SIZE
+#define SDMA0_SDMA_QUEUE0_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE0_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA0_SDMA_QUEUE0_DOORBELL
+#define SDMA0_SDMA_QUEUE0_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA0_SDMA_QUEUE0_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA0_SDMA_QUEUE0_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA0_SDMA_QUEUE0_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA0_SDMA_QUEUE0_DOORBELL_LOG
+#define SDMA0_SDMA_QUEUE0_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA0_SDMA_QUEUE0_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA0_SDMA_QUEUE0_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA0_SDMA_QUEUE0_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE0_DOORBELL_OFFSET
+#define SDMA0_SDMA_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA0_SDMA_QUEUE0_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA0_SDMA_QUEUE0_CSA_ADDR_LO
+#define SDMA0_SDMA_QUEUE0_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE0_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE0_CSA_ADDR_HI
+#define SDMA0_SDMA_QUEUE0_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE0_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_SCHEDULE_CNTL
+#define SDMA0_SDMA_QUEUE0_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE0_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA0_SDMA_QUEUE0_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA0_SDMA_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA0_SDMA_QUEUE0_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA0_SDMA_QUEUE0_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA0_SDMA_QUEUE0_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA0_SDMA_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA0_SDMA_QUEUE0_IB_SUB_REMAIN
+#define SDMA0_SDMA_QUEUE0_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE0_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA0_SDMA_QUEUE0_PREEMPT
+#define SDMA0_SDMA_QUEUE0_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE0_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA0_SDMA_QUEUE0_DUMMY_REG
+#define SDMA0_SDMA_QUEUE0_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE0_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA0_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA0_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_RB_AQL_CNTL
+#define SDMA0_SDMA_QUEUE0_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE0_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA0_SDMA_QUEUE0_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA0_SDMA_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA0_SDMA_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA0_SDMA_QUEUE0_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA0_SDMA_QUEUE0_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE0_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA0_SDMA_QUEUE0_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA0_SDMA_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA0_SDMA_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA0_SDMA_QUEUE0_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA0_SDMA_QUEUE0_MINOR_PTR_UPDATE
+#define SDMA0_SDMA_QUEUE0_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE0_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA0_SDMA_QUEUE0_MIDCMD_CNTL
+#define SDMA0_SDMA_QUEUE0_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE0_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE0_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA0_SDMA_QUEUE0_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA0_SDMA_QUEUE0_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE0_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE0_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA0_SDMA_QUEUE0_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA0_SDMA_QUEUE0_MIDCMD_DATA0
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_MIDCMD_DATA1
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_MIDCMD_DATA2
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_MIDCMD_DATA3
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_MIDCMD_DATA4
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_MIDCMD_DATA5
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_MIDCMD_DATA6
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_MIDCMD_DATA7
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_MIDCMD_DATA8
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_MIDCMD_DATA9
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_MIDCMD_DATA10
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE0_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_UTILIZATION_LO
+#define SDMA0_SDMA_QUEUE0_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE0_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_UTILIZATION_HI
+#define SDMA0_SDMA_QUEUE0_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE0_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_WAIT_UNSATISFIED_THD
+#define SDMA0_SDMA_QUEUE0_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA0_SDMA_QUEUE0_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA0_SDMA_QUEUE0_MQD_BASE_ADDR_LO
+#define SDMA0_SDMA_QUEUE0_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA0_SDMA_QUEUE0_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE0_MQD_BASE_ADDR_HI
+#define SDMA0_SDMA_QUEUE0_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA0_SDMA_QUEUE0_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_MQD_CONTROL
+#define SDMA0_SDMA_QUEUE0_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE0_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA0_SDMA_QUEUE0_CONTEXT_STATUS
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA0_SDMA_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA0_SDMA_QUEUE0_DUMMY_REG1
+#define SDMA0_SDMA_QUEUE0_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE0_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE0_DUMMY_REG2
+#define SDMA0_SDMA_QUEUE0_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE0_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_RB_CNTL
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA0_SDMA_QUEUE1_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA0_SDMA_QUEUE1_RB_BASE
+#define SDMA0_SDMA_QUEUE1_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE1_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_RB_BASE_HI
+#define SDMA0_SDMA_QUEUE1_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE1_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA0_SDMA_QUEUE1_RB_RPTR
+#define SDMA0_SDMA_QUEUE1_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE1_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_RB_RPTR_HI
+#define SDMA0_SDMA_QUEUE1_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE1_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_RB_WPTR
+#define SDMA0_SDMA_QUEUE1_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE1_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_RB_WPTR_HI
+#define SDMA0_SDMA_QUEUE1_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE1_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_RB_RPTR_ADDR_LO
+#define SDMA0_SDMA_QUEUE1_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_SDMA_QUEUE1_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE1_RB_RPTR_ADDR_HI
+#define SDMA0_SDMA_QUEUE1_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE1_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_IB_CNTL
+#define SDMA0_SDMA_QUEUE1_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE1_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA0_SDMA_QUEUE1_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE1_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA0_SDMA_QUEUE1_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA0_SDMA_QUEUE1_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE1_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA0_SDMA_QUEUE1_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE1_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA0_SDMA_QUEUE1_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA0_SDMA_QUEUE1_IB_RPTR
+#define SDMA0_SDMA_QUEUE1_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA0_SDMA_QUEUE1_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA0_SDMA_QUEUE1_IB_OFFSET
+#define SDMA0_SDMA_QUEUE1_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE1_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA0_SDMA_QUEUE1_IB_BASE_LO
+#define SDMA0_SDMA_QUEUE1_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_SDMA_QUEUE1_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE1_IB_BASE_HI
+#define SDMA0_SDMA_QUEUE1_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE1_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_IB_SIZE
+#define SDMA0_SDMA_QUEUE1_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE1_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA0_SDMA_QUEUE1_DOORBELL
+#define SDMA0_SDMA_QUEUE1_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA0_SDMA_QUEUE1_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA0_SDMA_QUEUE1_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA0_SDMA_QUEUE1_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA0_SDMA_QUEUE1_DOORBELL_LOG
+#define SDMA0_SDMA_QUEUE1_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA0_SDMA_QUEUE1_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA0_SDMA_QUEUE1_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA0_SDMA_QUEUE1_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE1_DOORBELL_OFFSET
+#define SDMA0_SDMA_QUEUE1_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA0_SDMA_QUEUE1_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA0_SDMA_QUEUE1_CSA_ADDR_LO
+#define SDMA0_SDMA_QUEUE1_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE1_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE1_CSA_ADDR_HI
+#define SDMA0_SDMA_QUEUE1_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE1_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_SCHEDULE_CNTL
+#define SDMA0_SDMA_QUEUE1_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE1_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA0_SDMA_QUEUE1_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA0_SDMA_QUEUE1_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA0_SDMA_QUEUE1_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA0_SDMA_QUEUE1_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA0_SDMA_QUEUE1_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA0_SDMA_QUEUE1_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA0_SDMA_QUEUE1_IB_SUB_REMAIN
+#define SDMA0_SDMA_QUEUE1_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE1_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA0_SDMA_QUEUE1_PREEMPT
+#define SDMA0_SDMA_QUEUE1_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE1_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA0_SDMA_QUEUE1_DUMMY_REG
+#define SDMA0_SDMA_QUEUE1_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE1_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA0_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA0_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_RB_AQL_CNTL
+#define SDMA0_SDMA_QUEUE1_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE1_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA0_SDMA_QUEUE1_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA0_SDMA_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA0_SDMA_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA0_SDMA_QUEUE1_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA0_SDMA_QUEUE1_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE1_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA0_SDMA_QUEUE1_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA0_SDMA_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA0_SDMA_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA0_SDMA_QUEUE1_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA0_SDMA_QUEUE1_MINOR_PTR_UPDATE
+#define SDMA0_SDMA_QUEUE1_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE1_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA0_SDMA_QUEUE1_MIDCMD_CNTL
+#define SDMA0_SDMA_QUEUE1_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE1_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE1_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA0_SDMA_QUEUE1_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA0_SDMA_QUEUE1_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE1_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE1_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA0_SDMA_QUEUE1_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA0_SDMA_QUEUE1_MIDCMD_DATA0
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_MIDCMD_DATA1
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_MIDCMD_DATA2
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_MIDCMD_DATA3
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_MIDCMD_DATA4
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_MIDCMD_DATA5
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_MIDCMD_DATA6
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_MIDCMD_DATA7
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_MIDCMD_DATA8
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_MIDCMD_DATA9
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_MIDCMD_DATA10
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE1_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_UTILIZATION_LO
+#define SDMA0_SDMA_QUEUE1_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE1_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_UTILIZATION_HI
+#define SDMA0_SDMA_QUEUE1_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE1_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_WAIT_UNSATISFIED_THD
+#define SDMA0_SDMA_QUEUE1_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA0_SDMA_QUEUE1_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA0_SDMA_QUEUE1_MQD_BASE_ADDR_LO
+#define SDMA0_SDMA_QUEUE1_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA0_SDMA_QUEUE1_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE1_MQD_BASE_ADDR_HI
+#define SDMA0_SDMA_QUEUE1_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA0_SDMA_QUEUE1_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_MQD_CONTROL
+#define SDMA0_SDMA_QUEUE1_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE1_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA0_SDMA_QUEUE1_CONTEXT_STATUS
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA0_SDMA_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA0_SDMA_QUEUE1_DUMMY_REG1
+#define SDMA0_SDMA_QUEUE1_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE1_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE1_DUMMY_REG2
+#define SDMA0_SDMA_QUEUE1_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE1_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_RB_CNTL
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA0_SDMA_QUEUE2_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA0_SDMA_QUEUE2_RB_BASE
+#define SDMA0_SDMA_QUEUE2_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE2_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_RB_BASE_HI
+#define SDMA0_SDMA_QUEUE2_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE2_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA0_SDMA_QUEUE2_RB_RPTR
+#define SDMA0_SDMA_QUEUE2_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE2_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_RB_RPTR_HI
+#define SDMA0_SDMA_QUEUE2_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE2_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_RB_WPTR
+#define SDMA0_SDMA_QUEUE2_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE2_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_RB_WPTR_HI
+#define SDMA0_SDMA_QUEUE2_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE2_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_RB_RPTR_ADDR_LO
+#define SDMA0_SDMA_QUEUE2_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_SDMA_QUEUE2_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE2_RB_RPTR_ADDR_HI
+#define SDMA0_SDMA_QUEUE2_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE2_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_IB_CNTL
+#define SDMA0_SDMA_QUEUE2_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE2_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA0_SDMA_QUEUE2_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE2_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA0_SDMA_QUEUE2_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA0_SDMA_QUEUE2_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE2_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA0_SDMA_QUEUE2_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE2_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA0_SDMA_QUEUE2_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA0_SDMA_QUEUE2_IB_RPTR
+#define SDMA0_SDMA_QUEUE2_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA0_SDMA_QUEUE2_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA0_SDMA_QUEUE2_IB_OFFSET
+#define SDMA0_SDMA_QUEUE2_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE2_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA0_SDMA_QUEUE2_IB_BASE_LO
+#define SDMA0_SDMA_QUEUE2_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_SDMA_QUEUE2_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE2_IB_BASE_HI
+#define SDMA0_SDMA_QUEUE2_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE2_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_IB_SIZE
+#define SDMA0_SDMA_QUEUE2_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE2_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA0_SDMA_QUEUE2_DOORBELL
+#define SDMA0_SDMA_QUEUE2_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA0_SDMA_QUEUE2_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA0_SDMA_QUEUE2_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA0_SDMA_QUEUE2_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA0_SDMA_QUEUE2_DOORBELL_LOG
+#define SDMA0_SDMA_QUEUE2_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA0_SDMA_QUEUE2_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA0_SDMA_QUEUE2_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA0_SDMA_QUEUE2_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE2_DOORBELL_OFFSET
+#define SDMA0_SDMA_QUEUE2_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA0_SDMA_QUEUE2_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA0_SDMA_QUEUE2_CSA_ADDR_LO
+#define SDMA0_SDMA_QUEUE2_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE2_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE2_CSA_ADDR_HI
+#define SDMA0_SDMA_QUEUE2_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE2_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_SCHEDULE_CNTL
+#define SDMA0_SDMA_QUEUE2_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE2_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA0_SDMA_QUEUE2_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA0_SDMA_QUEUE2_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA0_SDMA_QUEUE2_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA0_SDMA_QUEUE2_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA0_SDMA_QUEUE2_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA0_SDMA_QUEUE2_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA0_SDMA_QUEUE2_IB_SUB_REMAIN
+#define SDMA0_SDMA_QUEUE2_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE2_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA0_SDMA_QUEUE2_PREEMPT
+#define SDMA0_SDMA_QUEUE2_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE2_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA0_SDMA_QUEUE2_DUMMY_REG
+#define SDMA0_SDMA_QUEUE2_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE2_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA0_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA0_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_RB_AQL_CNTL
+#define SDMA0_SDMA_QUEUE2_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE2_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA0_SDMA_QUEUE2_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA0_SDMA_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA0_SDMA_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA0_SDMA_QUEUE2_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA0_SDMA_QUEUE2_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE2_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA0_SDMA_QUEUE2_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA0_SDMA_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA0_SDMA_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA0_SDMA_QUEUE2_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA0_SDMA_QUEUE2_MINOR_PTR_UPDATE
+#define SDMA0_SDMA_QUEUE2_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE2_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA0_SDMA_QUEUE2_MIDCMD_CNTL
+#define SDMA0_SDMA_QUEUE2_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE2_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE2_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA0_SDMA_QUEUE2_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA0_SDMA_QUEUE2_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE2_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE2_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA0_SDMA_QUEUE2_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA0_SDMA_QUEUE2_MIDCMD_DATA0
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_MIDCMD_DATA1
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_MIDCMD_DATA2
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_MIDCMD_DATA3
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_MIDCMD_DATA4
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_MIDCMD_DATA5
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_MIDCMD_DATA6
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_MIDCMD_DATA7
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_MIDCMD_DATA8
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_MIDCMD_DATA9
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_MIDCMD_DATA10
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE2_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_UTILIZATION_LO
+#define SDMA0_SDMA_QUEUE2_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE2_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_UTILIZATION_HI
+#define SDMA0_SDMA_QUEUE2_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE2_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_WAIT_UNSATISFIED_THD
+#define SDMA0_SDMA_QUEUE2_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA0_SDMA_QUEUE2_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA0_SDMA_QUEUE2_MQD_BASE_ADDR_LO
+#define SDMA0_SDMA_QUEUE2_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA0_SDMA_QUEUE2_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE2_MQD_BASE_ADDR_HI
+#define SDMA0_SDMA_QUEUE2_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA0_SDMA_QUEUE2_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_MQD_CONTROL
+#define SDMA0_SDMA_QUEUE2_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE2_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA0_SDMA_QUEUE2_CONTEXT_STATUS
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA0_SDMA_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA0_SDMA_QUEUE2_DUMMY_REG1
+#define SDMA0_SDMA_QUEUE2_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE2_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE2_DUMMY_REG2
+#define SDMA0_SDMA_QUEUE2_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE2_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_RB_CNTL
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA0_SDMA_QUEUE3_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA0_SDMA_QUEUE3_RB_BASE
+#define SDMA0_SDMA_QUEUE3_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE3_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_RB_BASE_HI
+#define SDMA0_SDMA_QUEUE3_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE3_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA0_SDMA_QUEUE3_RB_RPTR
+#define SDMA0_SDMA_QUEUE3_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE3_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_RB_RPTR_HI
+#define SDMA0_SDMA_QUEUE3_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE3_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_RB_WPTR
+#define SDMA0_SDMA_QUEUE3_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE3_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_RB_WPTR_HI
+#define SDMA0_SDMA_QUEUE3_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE3_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_RB_RPTR_ADDR_LO
+#define SDMA0_SDMA_QUEUE3_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_SDMA_QUEUE3_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE3_RB_RPTR_ADDR_HI
+#define SDMA0_SDMA_QUEUE3_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE3_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_IB_CNTL
+#define SDMA0_SDMA_QUEUE3_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE3_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA0_SDMA_QUEUE3_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE3_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA0_SDMA_QUEUE3_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA0_SDMA_QUEUE3_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE3_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA0_SDMA_QUEUE3_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE3_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA0_SDMA_QUEUE3_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA0_SDMA_QUEUE3_IB_RPTR
+#define SDMA0_SDMA_QUEUE3_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA0_SDMA_QUEUE3_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA0_SDMA_QUEUE3_IB_OFFSET
+#define SDMA0_SDMA_QUEUE3_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE3_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA0_SDMA_QUEUE3_IB_BASE_LO
+#define SDMA0_SDMA_QUEUE3_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_SDMA_QUEUE3_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE3_IB_BASE_HI
+#define SDMA0_SDMA_QUEUE3_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE3_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_IB_SIZE
+#define SDMA0_SDMA_QUEUE3_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE3_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA0_SDMA_QUEUE3_DOORBELL
+#define SDMA0_SDMA_QUEUE3_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA0_SDMA_QUEUE3_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA0_SDMA_QUEUE3_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA0_SDMA_QUEUE3_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA0_SDMA_QUEUE3_DOORBELL_LOG
+#define SDMA0_SDMA_QUEUE3_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA0_SDMA_QUEUE3_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA0_SDMA_QUEUE3_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA0_SDMA_QUEUE3_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE3_DOORBELL_OFFSET
+#define SDMA0_SDMA_QUEUE3_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA0_SDMA_QUEUE3_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA0_SDMA_QUEUE3_CSA_ADDR_LO
+#define SDMA0_SDMA_QUEUE3_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE3_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE3_CSA_ADDR_HI
+#define SDMA0_SDMA_QUEUE3_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE3_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_SCHEDULE_CNTL
+#define SDMA0_SDMA_QUEUE3_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE3_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA0_SDMA_QUEUE3_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA0_SDMA_QUEUE3_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA0_SDMA_QUEUE3_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA0_SDMA_QUEUE3_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA0_SDMA_QUEUE3_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA0_SDMA_QUEUE3_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA0_SDMA_QUEUE3_IB_SUB_REMAIN
+#define SDMA0_SDMA_QUEUE3_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE3_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA0_SDMA_QUEUE3_PREEMPT
+#define SDMA0_SDMA_QUEUE3_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE3_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA0_SDMA_QUEUE3_DUMMY_REG
+#define SDMA0_SDMA_QUEUE3_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE3_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA0_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA0_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_RB_AQL_CNTL
+#define SDMA0_SDMA_QUEUE3_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE3_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA0_SDMA_QUEUE3_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA0_SDMA_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA0_SDMA_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA0_SDMA_QUEUE3_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA0_SDMA_QUEUE3_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE3_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA0_SDMA_QUEUE3_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA0_SDMA_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA0_SDMA_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA0_SDMA_QUEUE3_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA0_SDMA_QUEUE3_MINOR_PTR_UPDATE
+#define SDMA0_SDMA_QUEUE3_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE3_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA0_SDMA_QUEUE3_MIDCMD_CNTL
+#define SDMA0_SDMA_QUEUE3_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE3_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE3_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA0_SDMA_QUEUE3_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA0_SDMA_QUEUE3_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE3_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE3_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA0_SDMA_QUEUE3_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA0_SDMA_QUEUE3_MIDCMD_DATA0
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_MIDCMD_DATA1
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_MIDCMD_DATA2
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_MIDCMD_DATA3
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_MIDCMD_DATA4
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_MIDCMD_DATA5
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_MIDCMD_DATA6
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_MIDCMD_DATA7
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_MIDCMD_DATA8
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_MIDCMD_DATA9
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_MIDCMD_DATA10
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE3_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_UTILIZATION_LO
+#define SDMA0_SDMA_QUEUE3_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE3_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_UTILIZATION_HI
+#define SDMA0_SDMA_QUEUE3_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE3_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_WAIT_UNSATISFIED_THD
+#define SDMA0_SDMA_QUEUE3_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA0_SDMA_QUEUE3_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA0_SDMA_QUEUE3_MQD_BASE_ADDR_LO
+#define SDMA0_SDMA_QUEUE3_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA0_SDMA_QUEUE3_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE3_MQD_BASE_ADDR_HI
+#define SDMA0_SDMA_QUEUE3_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA0_SDMA_QUEUE3_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_MQD_CONTROL
+#define SDMA0_SDMA_QUEUE3_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE3_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA0_SDMA_QUEUE3_CONTEXT_STATUS
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA0_SDMA_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA0_SDMA_QUEUE3_DUMMY_REG1
+#define SDMA0_SDMA_QUEUE3_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE3_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE3_DUMMY_REG2
+#define SDMA0_SDMA_QUEUE3_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE3_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_RB_CNTL
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA0_SDMA_QUEUE4_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA0_SDMA_QUEUE4_RB_BASE
+#define SDMA0_SDMA_QUEUE4_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE4_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_RB_BASE_HI
+#define SDMA0_SDMA_QUEUE4_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE4_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA0_SDMA_QUEUE4_RB_RPTR
+#define SDMA0_SDMA_QUEUE4_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE4_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_RB_RPTR_HI
+#define SDMA0_SDMA_QUEUE4_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE4_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_RB_WPTR
+#define SDMA0_SDMA_QUEUE4_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE4_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_RB_WPTR_HI
+#define SDMA0_SDMA_QUEUE4_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE4_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_RB_RPTR_ADDR_LO
+#define SDMA0_SDMA_QUEUE4_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_SDMA_QUEUE4_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE4_RB_RPTR_ADDR_HI
+#define SDMA0_SDMA_QUEUE4_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE4_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_IB_CNTL
+#define SDMA0_SDMA_QUEUE4_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE4_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA0_SDMA_QUEUE4_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE4_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA0_SDMA_QUEUE4_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA0_SDMA_QUEUE4_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE4_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA0_SDMA_QUEUE4_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE4_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA0_SDMA_QUEUE4_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA0_SDMA_QUEUE4_IB_RPTR
+#define SDMA0_SDMA_QUEUE4_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA0_SDMA_QUEUE4_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA0_SDMA_QUEUE4_IB_OFFSET
+#define SDMA0_SDMA_QUEUE4_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE4_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA0_SDMA_QUEUE4_IB_BASE_LO
+#define SDMA0_SDMA_QUEUE4_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_SDMA_QUEUE4_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE4_IB_BASE_HI
+#define SDMA0_SDMA_QUEUE4_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE4_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_IB_SIZE
+#define SDMA0_SDMA_QUEUE4_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE4_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA0_SDMA_QUEUE4_DOORBELL
+#define SDMA0_SDMA_QUEUE4_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA0_SDMA_QUEUE4_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA0_SDMA_QUEUE4_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA0_SDMA_QUEUE4_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA0_SDMA_QUEUE4_DOORBELL_LOG
+#define SDMA0_SDMA_QUEUE4_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA0_SDMA_QUEUE4_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA0_SDMA_QUEUE4_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA0_SDMA_QUEUE4_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE4_DOORBELL_OFFSET
+#define SDMA0_SDMA_QUEUE4_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA0_SDMA_QUEUE4_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA0_SDMA_QUEUE4_CSA_ADDR_LO
+#define SDMA0_SDMA_QUEUE4_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE4_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE4_CSA_ADDR_HI
+#define SDMA0_SDMA_QUEUE4_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE4_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_SCHEDULE_CNTL
+#define SDMA0_SDMA_QUEUE4_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE4_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA0_SDMA_QUEUE4_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA0_SDMA_QUEUE4_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA0_SDMA_QUEUE4_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA0_SDMA_QUEUE4_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA0_SDMA_QUEUE4_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA0_SDMA_QUEUE4_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA0_SDMA_QUEUE4_IB_SUB_REMAIN
+#define SDMA0_SDMA_QUEUE4_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE4_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA0_SDMA_QUEUE4_PREEMPT
+#define SDMA0_SDMA_QUEUE4_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE4_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA0_SDMA_QUEUE4_DUMMY_REG
+#define SDMA0_SDMA_QUEUE4_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE4_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA0_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA0_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_RB_AQL_CNTL
+#define SDMA0_SDMA_QUEUE4_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE4_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA0_SDMA_QUEUE4_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA0_SDMA_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA0_SDMA_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA0_SDMA_QUEUE4_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA0_SDMA_QUEUE4_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE4_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA0_SDMA_QUEUE4_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA0_SDMA_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA0_SDMA_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA0_SDMA_QUEUE4_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA0_SDMA_QUEUE4_MINOR_PTR_UPDATE
+#define SDMA0_SDMA_QUEUE4_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE4_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA0_SDMA_QUEUE4_MIDCMD_CNTL
+#define SDMA0_SDMA_QUEUE4_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE4_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE4_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA0_SDMA_QUEUE4_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA0_SDMA_QUEUE4_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE4_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE4_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA0_SDMA_QUEUE4_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA0_SDMA_QUEUE4_MIDCMD_DATA0
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_MIDCMD_DATA1
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_MIDCMD_DATA2
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_MIDCMD_DATA3
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_MIDCMD_DATA4
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_MIDCMD_DATA5
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_MIDCMD_DATA6
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_MIDCMD_DATA7
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_MIDCMD_DATA8
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_MIDCMD_DATA9
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_MIDCMD_DATA10
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE4_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_UTILIZATION_LO
+#define SDMA0_SDMA_QUEUE4_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE4_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_UTILIZATION_HI
+#define SDMA0_SDMA_QUEUE4_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE4_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_WAIT_UNSATISFIED_THD
+#define SDMA0_SDMA_QUEUE4_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA0_SDMA_QUEUE4_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA0_SDMA_QUEUE4_MQD_BASE_ADDR_LO
+#define SDMA0_SDMA_QUEUE4_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA0_SDMA_QUEUE4_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE4_MQD_BASE_ADDR_HI
+#define SDMA0_SDMA_QUEUE4_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA0_SDMA_QUEUE4_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_MQD_CONTROL
+#define SDMA0_SDMA_QUEUE4_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE4_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA0_SDMA_QUEUE4_CONTEXT_STATUS
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA0_SDMA_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA0_SDMA_QUEUE4_DUMMY_REG1
+#define SDMA0_SDMA_QUEUE4_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE4_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE4_DUMMY_REG2
+#define SDMA0_SDMA_QUEUE4_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE4_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_RB_CNTL
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA0_SDMA_QUEUE5_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA0_SDMA_QUEUE5_RB_BASE
+#define SDMA0_SDMA_QUEUE5_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE5_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_RB_BASE_HI
+#define SDMA0_SDMA_QUEUE5_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE5_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA0_SDMA_QUEUE5_RB_RPTR
+#define SDMA0_SDMA_QUEUE5_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE5_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_RB_RPTR_HI
+#define SDMA0_SDMA_QUEUE5_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE5_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_RB_WPTR
+#define SDMA0_SDMA_QUEUE5_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE5_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_RB_WPTR_HI
+#define SDMA0_SDMA_QUEUE5_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE5_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_RB_RPTR_ADDR_LO
+#define SDMA0_SDMA_QUEUE5_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_SDMA_QUEUE5_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE5_RB_RPTR_ADDR_HI
+#define SDMA0_SDMA_QUEUE5_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE5_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_IB_CNTL
+#define SDMA0_SDMA_QUEUE5_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE5_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA0_SDMA_QUEUE5_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE5_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA0_SDMA_QUEUE5_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA0_SDMA_QUEUE5_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE5_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA0_SDMA_QUEUE5_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE5_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA0_SDMA_QUEUE5_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA0_SDMA_QUEUE5_IB_RPTR
+#define SDMA0_SDMA_QUEUE5_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA0_SDMA_QUEUE5_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA0_SDMA_QUEUE5_IB_OFFSET
+#define SDMA0_SDMA_QUEUE5_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE5_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA0_SDMA_QUEUE5_IB_BASE_LO
+#define SDMA0_SDMA_QUEUE5_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_SDMA_QUEUE5_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE5_IB_BASE_HI
+#define SDMA0_SDMA_QUEUE5_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE5_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_IB_SIZE
+#define SDMA0_SDMA_QUEUE5_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE5_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA0_SDMA_QUEUE5_DOORBELL
+#define SDMA0_SDMA_QUEUE5_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA0_SDMA_QUEUE5_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA0_SDMA_QUEUE5_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA0_SDMA_QUEUE5_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA0_SDMA_QUEUE5_DOORBELL_LOG
+#define SDMA0_SDMA_QUEUE5_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA0_SDMA_QUEUE5_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA0_SDMA_QUEUE5_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA0_SDMA_QUEUE5_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE5_DOORBELL_OFFSET
+#define SDMA0_SDMA_QUEUE5_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA0_SDMA_QUEUE5_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA0_SDMA_QUEUE5_CSA_ADDR_LO
+#define SDMA0_SDMA_QUEUE5_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE5_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE5_CSA_ADDR_HI
+#define SDMA0_SDMA_QUEUE5_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE5_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_SCHEDULE_CNTL
+#define SDMA0_SDMA_QUEUE5_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE5_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA0_SDMA_QUEUE5_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA0_SDMA_QUEUE5_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA0_SDMA_QUEUE5_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA0_SDMA_QUEUE5_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA0_SDMA_QUEUE5_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA0_SDMA_QUEUE5_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA0_SDMA_QUEUE5_IB_SUB_REMAIN
+#define SDMA0_SDMA_QUEUE5_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE5_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA0_SDMA_QUEUE5_PREEMPT
+#define SDMA0_SDMA_QUEUE5_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE5_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA0_SDMA_QUEUE5_DUMMY_REG
+#define SDMA0_SDMA_QUEUE5_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE5_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA0_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA0_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_RB_AQL_CNTL
+#define SDMA0_SDMA_QUEUE5_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE5_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA0_SDMA_QUEUE5_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA0_SDMA_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA0_SDMA_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA0_SDMA_QUEUE5_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA0_SDMA_QUEUE5_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE5_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA0_SDMA_QUEUE5_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA0_SDMA_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA0_SDMA_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA0_SDMA_QUEUE5_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA0_SDMA_QUEUE5_MINOR_PTR_UPDATE
+#define SDMA0_SDMA_QUEUE5_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE5_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA0_SDMA_QUEUE5_MIDCMD_CNTL
+#define SDMA0_SDMA_QUEUE5_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE5_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE5_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA0_SDMA_QUEUE5_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA0_SDMA_QUEUE5_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE5_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE5_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA0_SDMA_QUEUE5_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA0_SDMA_QUEUE5_MIDCMD_DATA0
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_MIDCMD_DATA1
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_MIDCMD_DATA2
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_MIDCMD_DATA3
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_MIDCMD_DATA4
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_MIDCMD_DATA5
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_MIDCMD_DATA6
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_MIDCMD_DATA7
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_MIDCMD_DATA8
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_MIDCMD_DATA9
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_MIDCMD_DATA10
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE5_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_UTILIZATION_LO
+#define SDMA0_SDMA_QUEUE5_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE5_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_UTILIZATION_HI
+#define SDMA0_SDMA_QUEUE5_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE5_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_WAIT_UNSATISFIED_THD
+#define SDMA0_SDMA_QUEUE5_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA0_SDMA_QUEUE5_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA0_SDMA_QUEUE5_MQD_BASE_ADDR_LO
+#define SDMA0_SDMA_QUEUE5_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA0_SDMA_QUEUE5_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE5_MQD_BASE_ADDR_HI
+#define SDMA0_SDMA_QUEUE5_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA0_SDMA_QUEUE5_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_MQD_CONTROL
+#define SDMA0_SDMA_QUEUE5_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE5_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA0_SDMA_QUEUE5_CONTEXT_STATUS
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA0_SDMA_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA0_SDMA_QUEUE5_DUMMY_REG1
+#define SDMA0_SDMA_QUEUE5_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE5_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE5_DUMMY_REG2
+#define SDMA0_SDMA_QUEUE5_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE5_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_RB_CNTL
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA0_SDMA_QUEUE6_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA0_SDMA_QUEUE6_RB_BASE
+#define SDMA0_SDMA_QUEUE6_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE6_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_RB_BASE_HI
+#define SDMA0_SDMA_QUEUE6_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE6_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA0_SDMA_QUEUE6_RB_RPTR
+#define SDMA0_SDMA_QUEUE6_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE6_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_RB_RPTR_HI
+#define SDMA0_SDMA_QUEUE6_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE6_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_RB_WPTR
+#define SDMA0_SDMA_QUEUE6_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE6_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_RB_WPTR_HI
+#define SDMA0_SDMA_QUEUE6_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE6_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_RB_RPTR_ADDR_LO
+#define SDMA0_SDMA_QUEUE6_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_SDMA_QUEUE6_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE6_RB_RPTR_ADDR_HI
+#define SDMA0_SDMA_QUEUE6_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE6_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_IB_CNTL
+#define SDMA0_SDMA_QUEUE6_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE6_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA0_SDMA_QUEUE6_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE6_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA0_SDMA_QUEUE6_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA0_SDMA_QUEUE6_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE6_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA0_SDMA_QUEUE6_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE6_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA0_SDMA_QUEUE6_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA0_SDMA_QUEUE6_IB_RPTR
+#define SDMA0_SDMA_QUEUE6_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA0_SDMA_QUEUE6_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA0_SDMA_QUEUE6_IB_OFFSET
+#define SDMA0_SDMA_QUEUE6_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE6_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA0_SDMA_QUEUE6_IB_BASE_LO
+#define SDMA0_SDMA_QUEUE6_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_SDMA_QUEUE6_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE6_IB_BASE_HI
+#define SDMA0_SDMA_QUEUE6_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE6_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_IB_SIZE
+#define SDMA0_SDMA_QUEUE6_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE6_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA0_SDMA_QUEUE6_DOORBELL
+#define SDMA0_SDMA_QUEUE6_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA0_SDMA_QUEUE6_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA0_SDMA_QUEUE6_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA0_SDMA_QUEUE6_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA0_SDMA_QUEUE6_DOORBELL_LOG
+#define SDMA0_SDMA_QUEUE6_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA0_SDMA_QUEUE6_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA0_SDMA_QUEUE6_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA0_SDMA_QUEUE6_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE6_DOORBELL_OFFSET
+#define SDMA0_SDMA_QUEUE6_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA0_SDMA_QUEUE6_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA0_SDMA_QUEUE6_CSA_ADDR_LO
+#define SDMA0_SDMA_QUEUE6_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE6_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE6_CSA_ADDR_HI
+#define SDMA0_SDMA_QUEUE6_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE6_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_SCHEDULE_CNTL
+#define SDMA0_SDMA_QUEUE6_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE6_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA0_SDMA_QUEUE6_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA0_SDMA_QUEUE6_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA0_SDMA_QUEUE6_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA0_SDMA_QUEUE6_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA0_SDMA_QUEUE6_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA0_SDMA_QUEUE6_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA0_SDMA_QUEUE6_IB_SUB_REMAIN
+#define SDMA0_SDMA_QUEUE6_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE6_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA0_SDMA_QUEUE6_PREEMPT
+#define SDMA0_SDMA_QUEUE6_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE6_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA0_SDMA_QUEUE6_DUMMY_REG
+#define SDMA0_SDMA_QUEUE6_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE6_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA0_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA0_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_RB_AQL_CNTL
+#define SDMA0_SDMA_QUEUE6_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE6_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA0_SDMA_QUEUE6_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA0_SDMA_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA0_SDMA_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA0_SDMA_QUEUE6_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA0_SDMA_QUEUE6_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE6_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA0_SDMA_QUEUE6_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA0_SDMA_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA0_SDMA_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA0_SDMA_QUEUE6_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA0_SDMA_QUEUE6_MINOR_PTR_UPDATE
+#define SDMA0_SDMA_QUEUE6_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE6_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA0_SDMA_QUEUE6_MIDCMD_CNTL
+#define SDMA0_SDMA_QUEUE6_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE6_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE6_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA0_SDMA_QUEUE6_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA0_SDMA_QUEUE6_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE6_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE6_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA0_SDMA_QUEUE6_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA0_SDMA_QUEUE6_MIDCMD_DATA0
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_MIDCMD_DATA1
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_MIDCMD_DATA2
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_MIDCMD_DATA3
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_MIDCMD_DATA4
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_MIDCMD_DATA5
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_MIDCMD_DATA6
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_MIDCMD_DATA7
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_MIDCMD_DATA8
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_MIDCMD_DATA9
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_MIDCMD_DATA10
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE6_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_UTILIZATION_LO
+#define SDMA0_SDMA_QUEUE6_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE6_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_UTILIZATION_HI
+#define SDMA0_SDMA_QUEUE6_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE6_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_WAIT_UNSATISFIED_THD
+#define SDMA0_SDMA_QUEUE6_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA0_SDMA_QUEUE6_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA0_SDMA_QUEUE6_MQD_BASE_ADDR_LO
+#define SDMA0_SDMA_QUEUE6_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA0_SDMA_QUEUE6_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE6_MQD_BASE_ADDR_HI
+#define SDMA0_SDMA_QUEUE6_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA0_SDMA_QUEUE6_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_MQD_CONTROL
+#define SDMA0_SDMA_QUEUE6_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE6_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA0_SDMA_QUEUE6_CONTEXT_STATUS
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA0_SDMA_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA0_SDMA_QUEUE6_DUMMY_REG1
+#define SDMA0_SDMA_QUEUE6_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE6_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE6_DUMMY_REG2
+#define SDMA0_SDMA_QUEUE6_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE6_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_RB_CNTL
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA0_SDMA_QUEUE7_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA0_SDMA_QUEUE7_RB_BASE
+#define SDMA0_SDMA_QUEUE7_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE7_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_RB_BASE_HI
+#define SDMA0_SDMA_QUEUE7_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE7_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA0_SDMA_QUEUE7_RB_RPTR
+#define SDMA0_SDMA_QUEUE7_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE7_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_RB_RPTR_HI
+#define SDMA0_SDMA_QUEUE7_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE7_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_RB_WPTR
+#define SDMA0_SDMA_QUEUE7_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE7_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_RB_WPTR_HI
+#define SDMA0_SDMA_QUEUE7_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE7_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_RB_RPTR_ADDR_LO
+#define SDMA0_SDMA_QUEUE7_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_SDMA_QUEUE7_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE7_RB_RPTR_ADDR_HI
+#define SDMA0_SDMA_QUEUE7_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE7_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_IB_CNTL
+#define SDMA0_SDMA_QUEUE7_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE7_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA0_SDMA_QUEUE7_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE7_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA0_SDMA_QUEUE7_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA0_SDMA_QUEUE7_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE7_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA0_SDMA_QUEUE7_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE7_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA0_SDMA_QUEUE7_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA0_SDMA_QUEUE7_IB_RPTR
+#define SDMA0_SDMA_QUEUE7_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA0_SDMA_QUEUE7_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA0_SDMA_QUEUE7_IB_OFFSET
+#define SDMA0_SDMA_QUEUE7_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE7_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA0_SDMA_QUEUE7_IB_BASE_LO
+#define SDMA0_SDMA_QUEUE7_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_SDMA_QUEUE7_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE7_IB_BASE_HI
+#define SDMA0_SDMA_QUEUE7_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE7_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_IB_SIZE
+#define SDMA0_SDMA_QUEUE7_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE7_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA0_SDMA_QUEUE7_DOORBELL
+#define SDMA0_SDMA_QUEUE7_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA0_SDMA_QUEUE7_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA0_SDMA_QUEUE7_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA0_SDMA_QUEUE7_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA0_SDMA_QUEUE7_DOORBELL_LOG
+#define SDMA0_SDMA_QUEUE7_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA0_SDMA_QUEUE7_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA0_SDMA_QUEUE7_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA0_SDMA_QUEUE7_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE7_DOORBELL_OFFSET
+#define SDMA0_SDMA_QUEUE7_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA0_SDMA_QUEUE7_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA0_SDMA_QUEUE7_CSA_ADDR_LO
+#define SDMA0_SDMA_QUEUE7_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE7_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE7_CSA_ADDR_HI
+#define SDMA0_SDMA_QUEUE7_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE7_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_SCHEDULE_CNTL
+#define SDMA0_SDMA_QUEUE7_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE7_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA0_SDMA_QUEUE7_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA0_SDMA_QUEUE7_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA0_SDMA_QUEUE7_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA0_SDMA_QUEUE7_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA0_SDMA_QUEUE7_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA0_SDMA_QUEUE7_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA0_SDMA_QUEUE7_IB_SUB_REMAIN
+#define SDMA0_SDMA_QUEUE7_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE7_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA0_SDMA_QUEUE7_PREEMPT
+#define SDMA0_SDMA_QUEUE7_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE7_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA0_SDMA_QUEUE7_DUMMY_REG
+#define SDMA0_SDMA_QUEUE7_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE7_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA0_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA0_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_RB_AQL_CNTL
+#define SDMA0_SDMA_QUEUE7_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE7_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA0_SDMA_QUEUE7_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA0_SDMA_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA0_SDMA_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA0_SDMA_QUEUE7_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA0_SDMA_QUEUE7_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE7_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA0_SDMA_QUEUE7_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA0_SDMA_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA0_SDMA_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA0_SDMA_QUEUE7_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA0_SDMA_QUEUE7_MINOR_PTR_UPDATE
+#define SDMA0_SDMA_QUEUE7_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE7_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA0_SDMA_QUEUE7_MIDCMD_CNTL
+#define SDMA0_SDMA_QUEUE7_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE7_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE7_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA0_SDMA_QUEUE7_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA0_SDMA_QUEUE7_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE7_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE7_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA0_SDMA_QUEUE7_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA0_SDMA_QUEUE7_MIDCMD_DATA0
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_MIDCMD_DATA1
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_MIDCMD_DATA2
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_MIDCMD_DATA3
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_MIDCMD_DATA4
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_MIDCMD_DATA5
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_MIDCMD_DATA6
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_MIDCMD_DATA7
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_MIDCMD_DATA8
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_MIDCMD_DATA9
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_MIDCMD_DATA10
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE7_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_UTILIZATION_LO
+#define SDMA0_SDMA_QUEUE7_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE7_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_UTILIZATION_HI
+#define SDMA0_SDMA_QUEUE7_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE7_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_WAIT_UNSATISFIED_THD
+#define SDMA0_SDMA_QUEUE7_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA0_SDMA_QUEUE7_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA0_SDMA_QUEUE7_MQD_BASE_ADDR_LO
+#define SDMA0_SDMA_QUEUE7_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA0_SDMA_QUEUE7_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE7_MQD_BASE_ADDR_HI
+#define SDMA0_SDMA_QUEUE7_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA0_SDMA_QUEUE7_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_MQD_CONTROL
+#define SDMA0_SDMA_QUEUE7_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE7_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA0_SDMA_QUEUE7_CONTEXT_STATUS
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA0_SDMA_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA0_SDMA_QUEUE7_DUMMY_REG1
+#define SDMA0_SDMA_QUEUE7_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE7_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE7_DUMMY_REG2
+#define SDMA0_SDMA_QUEUE7_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE7_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_RB_CNTL
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA0_SDMA_QUEUE8_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA0_SDMA_QUEUE8_RB_BASE
+#define SDMA0_SDMA_QUEUE8_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE8_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_RB_BASE_HI
+#define SDMA0_SDMA_QUEUE8_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE8_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA0_SDMA_QUEUE8_RB_RPTR
+#define SDMA0_SDMA_QUEUE8_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE8_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_RB_RPTR_HI
+#define SDMA0_SDMA_QUEUE8_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE8_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_RB_WPTR
+#define SDMA0_SDMA_QUEUE8_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE8_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_RB_WPTR_HI
+#define SDMA0_SDMA_QUEUE8_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE8_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_RB_RPTR_ADDR_LO
+#define SDMA0_SDMA_QUEUE8_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_SDMA_QUEUE8_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE8_RB_RPTR_ADDR_HI
+#define SDMA0_SDMA_QUEUE8_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE8_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_IB_CNTL
+#define SDMA0_SDMA_QUEUE8_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE8_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA0_SDMA_QUEUE8_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE8_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA0_SDMA_QUEUE8_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA0_SDMA_QUEUE8_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE8_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA0_SDMA_QUEUE8_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE8_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA0_SDMA_QUEUE8_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA0_SDMA_QUEUE8_IB_RPTR
+#define SDMA0_SDMA_QUEUE8_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA0_SDMA_QUEUE8_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA0_SDMA_QUEUE8_IB_OFFSET
+#define SDMA0_SDMA_QUEUE8_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE8_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA0_SDMA_QUEUE8_IB_BASE_LO
+#define SDMA0_SDMA_QUEUE8_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_SDMA_QUEUE8_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE8_IB_BASE_HI
+#define SDMA0_SDMA_QUEUE8_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE8_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_IB_SIZE
+#define SDMA0_SDMA_QUEUE8_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE8_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA0_SDMA_QUEUE8_DOORBELL
+#define SDMA0_SDMA_QUEUE8_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA0_SDMA_QUEUE8_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA0_SDMA_QUEUE8_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA0_SDMA_QUEUE8_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA0_SDMA_QUEUE8_DOORBELL_LOG
+#define SDMA0_SDMA_QUEUE8_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA0_SDMA_QUEUE8_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA0_SDMA_QUEUE8_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA0_SDMA_QUEUE8_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE8_DOORBELL_OFFSET
+#define SDMA0_SDMA_QUEUE8_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA0_SDMA_QUEUE8_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA0_SDMA_QUEUE8_CSA_ADDR_LO
+#define SDMA0_SDMA_QUEUE8_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE8_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE8_CSA_ADDR_HI
+#define SDMA0_SDMA_QUEUE8_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE8_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_SCHEDULE_CNTL
+#define SDMA0_SDMA_QUEUE8_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE8_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA0_SDMA_QUEUE8_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA0_SDMA_QUEUE8_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA0_SDMA_QUEUE8_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA0_SDMA_QUEUE8_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA0_SDMA_QUEUE8_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA0_SDMA_QUEUE8_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA0_SDMA_QUEUE8_IB_SUB_REMAIN
+#define SDMA0_SDMA_QUEUE8_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE8_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA0_SDMA_QUEUE8_PREEMPT
+#define SDMA0_SDMA_QUEUE8_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE8_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA0_SDMA_QUEUE8_DUMMY_REG
+#define SDMA0_SDMA_QUEUE8_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE8_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA0_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA0_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_RB_AQL_CNTL
+#define SDMA0_SDMA_QUEUE8_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE8_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA0_SDMA_QUEUE8_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA0_SDMA_QUEUE8_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA0_SDMA_QUEUE8_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA0_SDMA_QUEUE8_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA0_SDMA_QUEUE8_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE8_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA0_SDMA_QUEUE8_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA0_SDMA_QUEUE8_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA0_SDMA_QUEUE8_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA0_SDMA_QUEUE8_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA0_SDMA_QUEUE8_MINOR_PTR_UPDATE
+#define SDMA0_SDMA_QUEUE8_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE8_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA0_SDMA_QUEUE8_MIDCMD_CNTL
+#define SDMA0_SDMA_QUEUE8_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE8_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE8_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA0_SDMA_QUEUE8_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA0_SDMA_QUEUE8_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE8_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE8_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA0_SDMA_QUEUE8_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA0_SDMA_QUEUE8_MIDCMD_DATA0
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_MIDCMD_DATA1
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_MIDCMD_DATA2
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_MIDCMD_DATA3
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_MIDCMD_DATA4
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_MIDCMD_DATA5
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_MIDCMD_DATA6
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_MIDCMD_DATA7
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_MIDCMD_DATA8
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_MIDCMD_DATA9
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_MIDCMD_DATA10
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE8_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_UTILIZATION_LO
+#define SDMA0_SDMA_QUEUE8_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE8_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_UTILIZATION_HI
+#define SDMA0_SDMA_QUEUE8_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE8_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_WAIT_UNSATISFIED_THD
+#define SDMA0_SDMA_QUEUE8_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA0_SDMA_QUEUE8_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA0_SDMA_QUEUE8_MQD_BASE_ADDR_LO
+#define SDMA0_SDMA_QUEUE8_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA0_SDMA_QUEUE8_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE8_MQD_BASE_ADDR_HI
+#define SDMA0_SDMA_QUEUE8_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA0_SDMA_QUEUE8_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_MQD_CONTROL
+#define SDMA0_SDMA_QUEUE8_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE8_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA0_SDMA_QUEUE8_CONTEXT_STATUS
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA0_SDMA_QUEUE8_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA0_SDMA_QUEUE8_DUMMY_REG1
+#define SDMA0_SDMA_QUEUE8_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE8_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE8_DUMMY_REG2
+#define SDMA0_SDMA_QUEUE8_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE8_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_RB_CNTL
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA0_SDMA_QUEUE9_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA0_SDMA_QUEUE9_RB_BASE
+#define SDMA0_SDMA_QUEUE9_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE9_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_RB_BASE_HI
+#define SDMA0_SDMA_QUEUE9_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE9_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA0_SDMA_QUEUE9_RB_RPTR
+#define SDMA0_SDMA_QUEUE9_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE9_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_RB_RPTR_HI
+#define SDMA0_SDMA_QUEUE9_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE9_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_RB_WPTR
+#define SDMA0_SDMA_QUEUE9_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA0_SDMA_QUEUE9_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_RB_WPTR_HI
+#define SDMA0_SDMA_QUEUE9_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE9_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_RB_RPTR_ADDR_LO
+#define SDMA0_SDMA_QUEUE9_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_SDMA_QUEUE9_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE9_RB_RPTR_ADDR_HI
+#define SDMA0_SDMA_QUEUE9_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE9_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_IB_CNTL
+#define SDMA0_SDMA_QUEUE9_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA0_SDMA_QUEUE9_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA0_SDMA_QUEUE9_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE9_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA0_SDMA_QUEUE9_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA0_SDMA_QUEUE9_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_SDMA_QUEUE9_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA0_SDMA_QUEUE9_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE9_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA0_SDMA_QUEUE9_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA0_SDMA_QUEUE9_IB_RPTR
+#define SDMA0_SDMA_QUEUE9_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA0_SDMA_QUEUE9_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA0_SDMA_QUEUE9_IB_OFFSET
+#define SDMA0_SDMA_QUEUE9_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE9_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA0_SDMA_QUEUE9_IB_BASE_LO
+#define SDMA0_SDMA_QUEUE9_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_SDMA_QUEUE9_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE9_IB_BASE_HI
+#define SDMA0_SDMA_QUEUE9_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE9_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_IB_SIZE
+#define SDMA0_SDMA_QUEUE9_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA0_SDMA_QUEUE9_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA0_SDMA_QUEUE9_DOORBELL
+#define SDMA0_SDMA_QUEUE9_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA0_SDMA_QUEUE9_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA0_SDMA_QUEUE9_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA0_SDMA_QUEUE9_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA0_SDMA_QUEUE9_DOORBELL_LOG
+#define SDMA0_SDMA_QUEUE9_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA0_SDMA_QUEUE9_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA0_SDMA_QUEUE9_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA0_SDMA_QUEUE9_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE9_DOORBELL_OFFSET
+#define SDMA0_SDMA_QUEUE9_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA0_SDMA_QUEUE9_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA0_SDMA_QUEUE9_CSA_ADDR_LO
+#define SDMA0_SDMA_QUEUE9_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA0_SDMA_QUEUE9_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE9_CSA_ADDR_HI
+#define SDMA0_SDMA_QUEUE9_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE9_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_SCHEDULE_CNTL
+#define SDMA0_SDMA_QUEUE9_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE9_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA0_SDMA_QUEUE9_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA0_SDMA_QUEUE9_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA0_SDMA_QUEUE9_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA0_SDMA_QUEUE9_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA0_SDMA_QUEUE9_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA0_SDMA_QUEUE9_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA0_SDMA_QUEUE9_IB_SUB_REMAIN
+#define SDMA0_SDMA_QUEUE9_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE9_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA0_SDMA_QUEUE9_PREEMPT
+#define SDMA0_SDMA_QUEUE9_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE9_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA0_SDMA_QUEUE9_DUMMY_REG
+#define SDMA0_SDMA_QUEUE9_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA0_SDMA_QUEUE9_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA0_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA0_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_RB_AQL_CNTL
+#define SDMA0_SDMA_QUEUE9_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE9_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA0_SDMA_QUEUE9_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA0_SDMA_QUEUE9_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA0_SDMA_QUEUE9_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA0_SDMA_QUEUE9_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA0_SDMA_QUEUE9_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE9_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA0_SDMA_QUEUE9_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA0_SDMA_QUEUE9_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA0_SDMA_QUEUE9_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA0_SDMA_QUEUE9_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA0_SDMA_QUEUE9_MINOR_PTR_UPDATE
+#define SDMA0_SDMA_QUEUE9_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE9_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA0_SDMA_QUEUE9_MIDCMD_CNTL
+#define SDMA0_SDMA_QUEUE9_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA0_SDMA_QUEUE9_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE9_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA0_SDMA_QUEUE9_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA0_SDMA_QUEUE9_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA0_SDMA_QUEUE9_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE9_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA0_SDMA_QUEUE9_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA0_SDMA_QUEUE9_MIDCMD_DATA0
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_MIDCMD_DATA1
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_MIDCMD_DATA2
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_MIDCMD_DATA3
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_MIDCMD_DATA4
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_MIDCMD_DATA5
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_MIDCMD_DATA6
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_MIDCMD_DATA7
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_MIDCMD_DATA8
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_MIDCMD_DATA9
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_MIDCMD_DATA10
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA0_SDMA_QUEUE9_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_UTILIZATION_LO
+#define SDMA0_SDMA_QUEUE9_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE9_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_UTILIZATION_HI
+#define SDMA0_SDMA_QUEUE9_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA0_SDMA_QUEUE9_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_WAIT_UNSATISFIED_THD
+#define SDMA0_SDMA_QUEUE9_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA0_SDMA_QUEUE9_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA0_SDMA_QUEUE9_MQD_BASE_ADDR_LO
+#define SDMA0_SDMA_QUEUE9_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA0_SDMA_QUEUE9_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA0_SDMA_QUEUE9_MQD_BASE_ADDR_HI
+#define SDMA0_SDMA_QUEUE9_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA0_SDMA_QUEUE9_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_MQD_CONTROL
+#define SDMA0_SDMA_QUEUE9_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE9_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA0_SDMA_QUEUE9_CONTEXT_STATUS
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA0_SDMA_QUEUE9_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA0_SDMA_QUEUE9_DUMMY_REG1
+#define SDMA0_SDMA_QUEUE9_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE9_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA0_SDMA_QUEUE9_DUMMY_REG2
+#define SDMA0_SDMA_QUEUE9_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA0_SDMA_QUEUE9_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmahypdec
+//SDMA0_SDMA_VM_CTX_LO
+#define SDMA0_SDMA_VM_CTX_LO__ADDR__SHIFT                                                                     0x2
+#define SDMA0_SDMA_VM_CTX_LO__ADDR_MASK                                                                       0xFFFFFFFCL
+//SDMA0_SDMA_VM_CTX_HI
+#define SDMA0_SDMA_VM_CTX_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA0_SDMA_VM_CTX_HI__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA0_SDMA_ACTIVE_FCN_ID
+#define SDMA0_SDMA_ACTIVE_FCN_ID__VFID__SHIFT                                                                 0x0
+#define SDMA0_SDMA_ACTIVE_FCN_ID__VF__SHIFT                                                                   0x7
+#define SDMA0_SDMA_ACTIVE_FCN_ID__RESERVED__SHIFT                                                             0x8
+#define SDMA0_SDMA_ACTIVE_FCN_ID__VFID_MASK                                                                   0x0000001FL
+#define SDMA0_SDMA_ACTIVE_FCN_ID__VF_MASK                                                                     0x00000080L
+#define SDMA0_SDMA_ACTIVE_FCN_ID__RESERVED_MASK                                                               0xFFFFFF00L
+//SDMA0_SDMA_VM_CTX_CNTL
+#define SDMA0_SDMA_VM_CTX_CNTL__PRIV__SHIFT                                                                   0x0
+#define SDMA0_SDMA_VM_CTX_CNTL__VMID__SHIFT                                                                   0x4
+#define SDMA0_SDMA_VM_CTX_CNTL__MEM_PHY__SHIFT                                                                0x8
+#define SDMA0_SDMA_VM_CTX_CNTL__MEM_MTYPE__SHIFT                                                              0xa
+#define SDMA0_SDMA_VM_CTX_CNTL__BUSY_STATUS_REPORT_ENABLE__SHIFT                                              0x10
+#define SDMA0_SDMA_VM_CTX_CNTL__PRIV_MASK                                                                     0x00000001L
+#define SDMA0_SDMA_VM_CTX_CNTL__VMID_MASK                                                                     0x000000F0L
+#define SDMA0_SDMA_VM_CTX_CNTL__MEM_PHY_MASK                                                                  0x00000300L
+#define SDMA0_SDMA_VM_CTX_CNTL__MEM_MTYPE_MASK                                                                0x00000C00L
+#define SDMA0_SDMA_VM_CTX_CNTL__BUSY_STATUS_REPORT_ENABLE_MASK                                                0x00010000L
+//SDMA0_SDMA_VIRT_RESET_REQ
+#define SDMA0_SDMA_VIRT_RESET_REQ__VF__SHIFT                                                                  0x0
+#define SDMA0_SDMA_VIRT_RESET_REQ__PF__SHIFT                                                                  0x1f
+#define SDMA0_SDMA_VIRT_RESET_REQ__VF_MASK                                                                    0x7FFFFFFFL
+#define SDMA0_SDMA_VIRT_RESET_REQ__PF_MASK                                                                    0x80000000L
+//SDMA0_SDMA_CONTEXT_REG_TYPE0
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_CNTL__SHIFT                                              0x0
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_BASE__SHIFT                                              0x1
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_BASE_HI__SHIFT                                           0x2
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_RPTR__SHIFT                                              0x3
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_RPTR_HI__SHIFT                                           0x4
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_WPTR__SHIFT                                              0x5
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_WPTR_HI__SHIFT                                           0x6
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_RPTR_ADDR_LO__SHIFT                                      0x7
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_RPTR_ADDR_HI__SHIFT                                      0x8
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_CNTL__SHIFT                                              0x9
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_RPTR__SHIFT                                              0xa
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_OFFSET__SHIFT                                            0xb
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_BASE_LO__SHIFT                                           0xc
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_BASE_HI__SHIFT                                           0xd
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_SIZE__SHIFT                                              0xe
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_DOORBELL__SHIFT                                             0xf
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_DOORBELL_LOG__SHIFT                                         0x10
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_DOORBELL_OFFSET__SHIFT                                      0x11
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_CSA_ADDR_LO__SHIFT                                          0x12
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_CSA_ADDR_HI__SHIFT                                          0x13
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_SCHEDULE_CNTL__SHIFT                                        0x14
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_SUB_REMAIN__SHIFT                                        0x15
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_PREEMPT__SHIFT                                              0x16
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_DUMMY_REG__SHIFT                                            0x17
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_WPTR_POLL_ADDR_LO__SHIFT                                 0x18
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_WPTR_POLL_ADDR_HI__SHIFT                                 0x19
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_AQL_CNTL__SHIFT                                          0x1a
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_MINOR_PTR_UPDATE__SHIFT                                     0x1b
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__SHIFT                                0x1e
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_MIDCMD_CNTL__SHIFT                                          0x1f
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_CNTL_MASK                                                0x00000001L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_BASE_MASK                                                0x00000002L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_BASE_HI_MASK                                             0x00000004L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_RPTR_MASK                                                0x00000008L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_RPTR_HI_MASK                                             0x00000010L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_WPTR_MASK                                                0x00000020L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_WPTR_HI_MASK                                             0x00000040L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_RPTR_ADDR_LO_MASK                                        0x00000080L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_RPTR_ADDR_HI_MASK                                        0x00000100L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_CNTL_MASK                                                0x00000200L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_RPTR_MASK                                                0x00000400L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_OFFSET_MASK                                              0x00000800L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_BASE_LO_MASK                                             0x00001000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_BASE_HI_MASK                                             0x00002000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_SIZE_MASK                                                0x00004000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_DOORBELL_MASK                                               0x00008000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_DOORBELL_LOG_MASK                                           0x00010000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_DOORBELL_OFFSET_MASK                                        0x00020000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_CSA_ADDR_LO_MASK                                            0x00040000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_CSA_ADDR_HI_MASK                                            0x00080000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_SCHEDULE_CNTL_MASK                                          0x00100000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_SUB_REMAIN_MASK                                          0x00200000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_PREEMPT_MASK                                                0x00400000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_DUMMY_REG_MASK                                              0x00800000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_WPTR_POLL_ADDR_LO_MASK                                   0x01000000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_WPTR_POLL_ADDR_HI_MASK                                   0x02000000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_AQL_CNTL_MASK                                            0x04000000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_MINOR_PTR_UPDATE_MASK                                       0x08000000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_CONTEXT_SWITCH_STATUS_MASK                                  0x40000000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_MIDCMD_CNTL_MASK                                            0x80000000L
+//SDMA0_SDMA_CONTEXT_REG_TYPE1
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA0__SHIFT                                         0x0
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA1__SHIFT                                         0x1
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA2__SHIFT                                         0x2
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA3__SHIFT                                         0x3
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA4__SHIFT                                         0x4
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA5__SHIFT                                         0x5
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA6__SHIFT                                         0x6
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA7__SHIFT                                         0x7
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA8__SHIFT                                         0x8
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA9__SHIFT                                         0x9
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA10__SHIFT                                        0xa
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_UTILIZATION_LO__SHIFT                                       0xb
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_UTILIZATION_HI__SHIFT                                       0xc
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_WAIT_UNSATISFIED_THD__SHIFT                                 0xd
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MQD_BASE_ADDR_LO__SHIFT                                     0xe
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MQD_BASE_ADDR_HI__SHIFT                                     0xf
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MQD_CONTROL__SHIFT                                          0x10
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_CONTEXT_STATUS__SHIFT                                       0x11
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_DUMMY_REG1__SHIFT                                           0x12
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_DUMMY_REG2__SHIFT                                           0x13
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__RESERVED__SHIFT                                                         0x14
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA0_MASK                                           0x00000001L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA1_MASK                                           0x00000002L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA2_MASK                                           0x00000004L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA3_MASK                                           0x00000008L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA4_MASK                                           0x00000010L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA5_MASK                                           0x00000020L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA6_MASK                                           0x00000040L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA7_MASK                                           0x00000080L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA8_MASK                                           0x00000100L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA9_MASK                                           0x00000200L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA10_MASK                                          0x00000400L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_UTILIZATION_LO_MASK                                         0x00000800L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_UTILIZATION_HI_MASK                                         0x00001000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_WAIT_UNSATISFIED_THD_MASK                                   0x00002000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MQD_BASE_ADDR_LO_MASK                                       0x00004000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MQD_BASE_ADDR_HI_MASK                                       0x00008000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MQD_CONTROL_MASK                                            0x00010000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_CONTEXT_STATUS_MASK                                         0x00020000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_DUMMY_REG1_MASK                                             0x00040000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_DUMMY_REG2_MASK                                             0x00080000L
+#define SDMA0_SDMA_CONTEXT_REG_TYPE1__RESERVED_MASK                                                           0xFFF00000L
+//SDMA0_SDMA_CONTEXT_REG_TYPE2
+#define SDMA0_SDMA_CONTEXT_REG_TYPE2__RESERVED__SHIFT                                                         0x0
+#define SDMA0_SDMA_CONTEXT_REG_TYPE2__RESERVED_MASK                                                           0xFFFFFFFFL
+//SDMA0_SDMA_PUB_REG_TYPE0
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_DEC_START__SHIFT                                                       0x0
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_MCU_MISC_CNTL__SHIFT                                                   0x1
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_UCODE_REV__SHIFT                                                       0x3
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_GLOBAL_TIMESTAMP_LO__SHIFT                                             0x5
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_GLOBAL_TIMESTAMP_HI__SHIFT                                             0x6
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_POWER_CNTL__SHIFT                                                      0xc
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_CNTL__SHIFT                                                            0xd
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_CHICKEN_BITS__SHIFT                                                    0xe
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_CACHE_CNTL__SHIFT                                                      0xf
+#define SDMA0_SDMA_PUB_REG_TYPE0__RESERVED__SHIFT                                                             0x15
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_DEC_START_MASK                                                         0x00000001L
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_MCU_MISC_CNTL_MASK                                                     0x00000002L
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_UCODE_REV_MASK                                                         0x00000008L
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_GLOBAL_TIMESTAMP_LO_MASK                                               0x00000020L
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_GLOBAL_TIMESTAMP_HI_MASK                                               0x00000040L
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_POWER_CNTL_MASK                                                        0x00001000L
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_CNTL_MASK                                                              0x00002000L
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_CHICKEN_BITS_MASK                                                      0x00004000L
+#define SDMA0_SDMA_PUB_REG_TYPE0__SDMA_CACHE_CNTL_MASK                                                        0x00008000L
+#define SDMA0_SDMA_PUB_REG_TYPE0__RESERVED_MASK                                                               0xFFE00000L
+//SDMA0_SDMA_PUB_REG_TYPE1
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_RB_RPTR_FETCH__SHIFT                                                   0x0
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_RB_RPTR_FETCH_HI__SHIFT                                                0x1
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_IB_OFFSET_FETCH__SHIFT                                                 0x2
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_PROGRAM__SHIFT                                                         0x3
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_STATUS_REG__SHIFT                                                      0x4
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_STATUS1_REG__SHIFT                                                     0x5
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_CNTL1__SHIFT                                                           0x6
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_HBM_PAGE_CONFIG__SHIFT                                                 0x7
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_FREEZE__SHIFT                                                          0x8
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_PROCESS_QUANTUM0__SHIFT                                                0x9
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_PROCESS_QUANTUM1__SHIFT                                                0xa
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_WATCHDOG_CNTL__SHIFT                                                   0xb
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_QUEUE_STATUS0__SHIFT                                                   0xc
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_QUEUE_STATUS1__SHIFT                                                   0xd
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_ID__SHIFT                                                              0xe
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_VERSION__SHIFT                                                         0xf
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_STATUS2_REG__SHIFT                                                     0x10
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_CNTL__SHIFT                                                     0x11
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_PERMS_CNTL__SHIFT                                               0x12
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_PREOP_LO__SHIFT                                                 0x13
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_PREOP_HI__SHIFT                                                 0x14
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_DCC_CNTL__SHIFT                                                        0x15
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_WARMUPL2_CNTL__SHIFT                                             0x16
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_CNTL__SHIFT                                                      0x17
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_WATERMK__SHIFT                                                   0x18
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_TIMEOUT__SHIFT                                                   0x19
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_PAGE__SHIFT                                                      0x1a
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_EXTERNAL_FROZEN__SHIFT                                                 0x1b
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_FREEZE_TRIGGER__SHIFT                                                  0x1c
+#define SDMA0_SDMA_PUB_REG_TYPE1__RESERVED__SHIFT                                                             0x1d
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_RB_RPTR_FETCH_MASK                                                     0x00000001L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_RB_RPTR_FETCH_HI_MASK                                                  0x00000002L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_IB_OFFSET_FETCH_MASK                                                   0x00000004L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_PROGRAM_MASK                                                           0x00000008L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_STATUS_REG_MASK                                                        0x00000010L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_STATUS1_REG_MASK                                                       0x00000020L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_CNTL1_MASK                                                             0x00000040L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_HBM_PAGE_CONFIG_MASK                                                   0x00000080L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_FREEZE_MASK                                                            0x00000100L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_PROCESS_QUANTUM0_MASK                                                  0x00000200L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_PROCESS_QUANTUM1_MASK                                                  0x00000400L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_WATCHDOG_CNTL_MASK                                                     0x00000800L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_QUEUE_STATUS0_MASK                                                     0x00001000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_QUEUE_STATUS1_MASK                                                     0x00002000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_ID_MASK                                                                0x00004000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_VERSION_MASK                                                           0x00008000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_STATUS2_REG_MASK                                                       0x00010000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_CNTL_MASK                                                       0x00020000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_PERMS_CNTL_MASK                                                 0x00040000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_PREOP_LO_MASK                                                   0x00080000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_PREOP_HI_MASK                                                   0x00100000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_DCC_CNTL_MASK                                                          0x00200000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_WARMUPL2_CNTL_MASK                                               0x00400000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_CNTL_MASK                                                        0x00800000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_WATERMK_MASK                                                     0x01000000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_TIMEOUT_MASK                                                     0x02000000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_PAGE_MASK                                                        0x04000000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_EXTERNAL_FROZEN_MASK                                                   0x08000000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__SDMA_FREEZE_TRIGGER_MASK                                                    0x10000000L
+#define SDMA0_SDMA_PUB_REG_TYPE1__RESERVED_MASK                                                               0xE0000000L
+//SDMA0_SDMA_PUB_REG_TYPE2
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_CACHE_CNTL__SHIFT                                                0x0
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_RD_STATUS__SHIFT                                                 0x1
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_WR_STATUS__SHIFT                                                 0x2
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV0__SHIFT                                                      0x3
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV1__SHIFT                                                      0x4
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV2__SHIFT                                                      0x5
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_RD_XNACK0__SHIFT                                                 0x6
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_RD_XNACK1__SHIFT                                                 0x7
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_WR_XNACK0__SHIFT                                                 0x8
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_WR_XNACK1__SHIFT                                                 0x9
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_CHICKEN_BITS_2__SHIFT                                                  0xa
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_STATUS3_REG__SHIFT                                                     0xb
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_GLOBAL_QUANTUM__SHIFT                                                  0xc
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_ERROR_LOG__SHIFT                                                       0xd
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG0__SHIFT                                                  0xe
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG1__SHIFT                                                  0xf
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG2__SHIFT                                                  0x10
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG3__SHIFT                                                  0x11
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_MCU_COUNTER__SHIFT                                                     0x12
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_CRD_CNTL__SHIFT                                                        0x13
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_RLC_CGCG_CTRL__SHIFT                                                   0x14
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_GPU_IOV_VIOLATION_LOG__SHIFT                                           0x15
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_AQL_STATUS__SHIFT                                                      0x17
+#define SDMA0_SDMA_PUB_REG_TYPE2__RESERVED__SHIFT                                                             0x1f
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_CACHE_CNTL_MASK                                                  0x00000001L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_RD_STATUS_MASK                                                   0x00000002L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_WR_STATUS_MASK                                                   0x00000004L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV0_MASK                                                        0x00000008L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV1_MASK                                                        0x00000010L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV2_MASK                                                        0x00000020L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_RD_XNACK0_MASK                                                   0x00000040L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_RD_XNACK1_MASK                                                   0x00000080L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_WR_XNACK0_MASK                                                   0x00000100L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_WR_XNACK1_MASK                                                   0x00000200L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_CHICKEN_BITS_2_MASK                                                    0x00000400L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_STATUS3_REG_MASK                                                       0x00000800L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_GLOBAL_QUANTUM_MASK                                                    0x00001000L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_ERROR_LOG_MASK                                                         0x00002000L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG0_MASK                                                    0x00004000L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG1_MASK                                                    0x00008000L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG2_MASK                                                    0x00010000L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG3_MASK                                                    0x00020000L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_MCU_COUNTER_MASK                                                       0x00040000L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_CRD_CNTL_MASK                                                          0x00080000L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_RLC_CGCG_CTRL_MASK                                                     0x00100000L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_GPU_IOV_VIOLATION_LOG_MASK                                             0x00200000L
+#define SDMA0_SDMA_PUB_REG_TYPE2__SDMA_AQL_STATUS_MASK                                                        0x00800000L
+#define SDMA0_SDMA_PUB_REG_TYPE2__RESERVED_MASK                                                               0x80000000L
+//SDMA0_SDMA_PUB_REG_TYPE3
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_TLBI_GCR_CNTL__SHIFT                                                   0x0
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_INT_STATUS__SHIFT                                                      0x1
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_GPU_IOV_VIOLATION_LOG2__SHIFT                                          0x2
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_INVALID_ADDR_LO__SHIFT                                                 0x3
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_INVALID_ADDR_HI__SHIFT                                                 0x4
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_INVALID_ADDR_SRC__SHIFT                                                0x5
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_CLOCK_GATING_STATUS__SHIFT                                             0x6
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_STATUS4_REG__SHIFT                                                     0x7
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_SCRATCH_RAM_DATA__SHIFT                                                0x8
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_SCRATCH_RAM_ADDR__SHIFT                                                0x9
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_TIMESTAMP_CNTL__SHIFT                                                  0xa
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_STATUS5_REG__SHIFT                                                     0xb
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_QUEUE_RESET_REQ__SHIFT                                                 0xc
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_QUEUE_DEQUEUE_REQUEST__SHIFT                                           0xd
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_STATUS6_REG__SHIFT                                                     0xe
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_CE_CTRL__SHIFT                                                         0xf
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_RAS_STATUS__SHIFT                                                      0x10
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_POISON_INFO__SHIFT                                                     0x11
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_MEMHUB_CNTL__SHIFT                                                     0x12
+#define SDMA0_SDMA_PUB_REG_TYPE3__RESERVED__SHIFT                                                             0x13
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_TLBI_GCR_CNTL_MASK                                                     0x00000001L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_INT_STATUS_MASK                                                        0x00000002L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_GPU_IOV_VIOLATION_LOG2_MASK                                            0x00000004L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_INVALID_ADDR_LO_MASK                                                   0x00000008L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_INVALID_ADDR_HI_MASK                                                   0x00000010L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_INVALID_ADDR_SRC_MASK                                                  0x00000020L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_CLOCK_GATING_STATUS_MASK                                               0x00000040L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_STATUS4_REG_MASK                                                       0x00000080L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_SCRATCH_RAM_DATA_MASK                                                  0x00000100L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_SCRATCH_RAM_ADDR_MASK                                                  0x00000200L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_TIMESTAMP_CNTL_MASK                                                    0x00000400L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_STATUS5_REG_MASK                                                       0x00000800L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_QUEUE_RESET_REQ_MASK                                                   0x00001000L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_QUEUE_DEQUEUE_REQUEST_MASK                                             0x00002000L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_STATUS6_REG_MASK                                                       0x00004000L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_CE_CTRL_MASK                                                           0x00008000L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_RAS_STATUS_MASK                                                        0x00010000L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_POISON_INFO_MASK                                                       0x00020000L
+#define SDMA0_SDMA_PUB_REG_TYPE3__SDMA_MEMHUB_CNTL_MASK                                                       0x00040000L
+#define SDMA0_SDMA_PUB_REG_TYPE3__RESERVED_MASK                                                               0xFFF80000L
+//SDMA0_SDMA_VM_CNTL
+#define SDMA0_SDMA_VM_CNTL__CMD__SHIFT                                                                        0x0
+#define SDMA0_SDMA_VM_CNTL__CMD_MASK                                                                          0x0000000FL
+//SDMA0_SDMA_MCU_CNTL
+#define SDMA0_SDMA_MCU_CNTL__HALT__SHIFT                                                                      0x0
+#define SDMA0_SDMA_MCU_CNTL__RESET__SHIFT                                                                     0x1
+#define SDMA0_SDMA_MCU_CNTL__DBG_SELECT_BITS__SHIFT                                                           0x2
+#define SDMA0_SDMA_MCU_CNTL__HALT_MASK                                                                        0x00000001L
+#define SDMA0_SDMA_MCU_CNTL__RESET_MASK                                                                       0x00000002L
+#define SDMA0_SDMA_MCU_CNTL__DBG_SELECT_BITS_MASK                                                             0x000000FCL
+//SDMA0_SDMA_IC_BASE_LO
+#define SDMA0_SDMA_IC_BASE_LO__IC_BASE_LO__SHIFT                                                              0xc
+#define SDMA0_SDMA_IC_BASE_LO__IC_BASE_LO_MASK                                                                0xFFFFF000L
+//SDMA0_SDMA_IC_BASE_HI
+#define SDMA0_SDMA_IC_BASE_HI__IC_BASE_HI__SHIFT                                                              0x0
+#define SDMA0_SDMA_IC_BASE_HI__IC_BASE_HI_MASK                                                                0xFFFFFFFFL
+//SDMA0_SDMA_IC_BASE_CNTL
+#define SDMA0_SDMA_IC_BASE_CNTL__VMID__SHIFT                                                                  0x0
+#define SDMA0_SDMA_IC_BASE_CNTL__MTYPE__SHIFT                                                                 0x4
+#define SDMA0_SDMA_IC_BASE_CNTL__EXE_DISABLE__SHIFT                                                           0x17
+#define SDMA0_SDMA_IC_BASE_CNTL__TEMPORAL_HINT__SHIFT                                                         0x18
+#define SDMA0_SDMA_IC_BASE_CNTL__SCOPE__SHIFT                                                                 0x1c
+#define SDMA0_SDMA_IC_BASE_CNTL__VMID_MASK                                                                    0x0000000FL
+#define SDMA0_SDMA_IC_BASE_CNTL__MTYPE_MASK                                                                   0x00000030L
+#define SDMA0_SDMA_IC_BASE_CNTL__EXE_DISABLE_MASK                                                             0x00800000L
+#define SDMA0_SDMA_IC_BASE_CNTL__TEMPORAL_HINT_MASK                                                           0x07000000L
+#define SDMA0_SDMA_IC_BASE_CNTL__SCOPE_MASK                                                                   0x30000000L
+//SDMA0_SDMA_IC_OP_CNTL
+#define SDMA0_SDMA_IC_OP_CNTL__INVALIDATE_CACHE__SHIFT                                                        0x0
+#define SDMA0_SDMA_IC_OP_CNTL__PRIME_ICACHE__SHIFT                                                            0x4
+#define SDMA0_SDMA_IC_OP_CNTL__ICACHE_PRIMED__SHIFT                                                           0x5
+#define SDMA0_SDMA_IC_OP_CNTL__INVALIDATE_CACHE_MASK                                                          0x00000001L
+#define SDMA0_SDMA_IC_OP_CNTL__PRIME_ICACHE_MASK                                                              0x00000010L
+#define SDMA0_SDMA_IC_OP_CNTL__ICACHE_PRIMED_MASK                                                             0x00000020L
+//SDMA0_SDMA_IC_CNTL
+#define SDMA0_SDMA_IC_CNTL__CID_SEL__SHIFT                                                                    0x0
+#define SDMA0_SDMA_IC_CNTL__GPA__SHIFT                                                                        0x2
+#define SDMA0_SDMA_IC_CNTL__UCODE_VF_OVERRIDE__SHIFT                                                          0x4
+#define SDMA0_SDMA_IC_CNTL__AUTO_PRIME_ICACHE__SHIFT                                                          0x5
+#define SDMA0_SDMA_IC_CNTL__CID_SEL_MASK                                                                      0x00000001L
+#define SDMA0_SDMA_IC_CNTL__GPA_MASK                                                                          0x0000000CL
+#define SDMA0_SDMA_IC_CNTL__UCODE_VF_OVERRIDE_MASK                                                            0x00000010L
+#define SDMA0_SDMA_IC_CNTL__AUTO_PRIME_ICACHE_MASK                                                            0x00000020L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmapspdec
+//SDMA0_SDMA_MCU_DM_FROM_RST_ADDR_OFFSET
+#define SDMA0_SDMA_MCU_DM_FROM_RST_ADDR_OFFSET__DATA__SHIFT                                                   0x0
+#define SDMA0_SDMA_MCU_DM_FROM_RST_ADDR_OFFSET__DATA_MASK                                                     0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmaperfsdec
+//SDMA0_SDMA_PERFCNT_PERFCOUNTER0_CFG
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                  0x0
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                              0x8
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                 0x18
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                    0x1c
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                     0x1d
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                    0x000000FFL
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                0x0000FF00L
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                   0x0F000000L
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER0_CFG__ENABLE_MASK                                                      0x10000000L
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER0_CFG__CLEAR_MASK                                                       0x20000000L
+//SDMA0_SDMA_PERFCNT_PERFCOUNTER1_CFG
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                  0x0
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                              0x8
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                 0x18
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                    0x1c
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                     0x1d
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                    0x000000FFL
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                0x0000FF00L
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                   0x0F000000L
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER1_CFG__ENABLE_MASK                                                      0x10000000L
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER1_CFG__CLEAR_MASK                                                       0x20000000L
+//SDMA0_SDMA_PERFCNT_PERFCOUNTER2_CFG
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                  0x0
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                              0x8
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                 0x18
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                    0x1c
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                     0x1d
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                    0x000000FFL
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                0x0000FF00L
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                   0x0F000000L
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER2_CFG__ENABLE_MASK                                                      0x10000000L
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER2_CFG__CLEAR_MASK                                                       0x20000000L
+//SDMA0_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                  0x0
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                        0x8
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                         0x10
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                           0x18
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                            0x19
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                 0x1a
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                    0x0000000FL
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                          0x0000FF00L
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                           0x00FF0000L
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                             0x01000000L
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                              0x02000000L
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                   0x04000000L
+//SDMA0_SDMA_PERFCNT_MISC_CNTL
+#define SDMA0_SDMA_PERFCNT_MISC_CNTL__CMD_OP__SHIFT                                                           0x0
+#define SDMA0_SDMA_PERFCNT_MISC_CNTL__CMD_OP_MASK                                                             0x0000FFFFL
+//SDMA0_SDMA_PERFCOUNTER0_SELECT
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                       0x0
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                      0xa
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                       0x14
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT__CNTR_MODE1__SHIFT                                                     0x18
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT__CNTR_MODE0__SHIFT                                                     0x1c
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                         0x000003FFL
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                        0x000FFC00L
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                         0x00F00000L
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT__CNTR_MODE1_MASK                                                       0x0F000000L
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT__CNTR_MODE0_MASK                                                       0xF0000000L
+//SDMA0_SDMA_PERFCOUNTER0_SELECT1
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                     0xa
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT1__CNTR_MODE3__SHIFT                                                    0x18
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT1__CNTR_MODE2__SHIFT                                                    0x1c
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                       0x000003FFL
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                       0x000FFC00L
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT1__CNTR_MODE3_MASK                                                      0x0F000000L
+#define SDMA0_SDMA_PERFCOUNTER0_SELECT1__CNTR_MODE2_MASK                                                      0xF0000000L
+//SDMA0_SDMA_PERFCOUNTER1_SELECT
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                       0x0
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                      0xa
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                       0x14
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT__CNTR_MODE1__SHIFT                                                     0x18
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT__CNTR_MODE0__SHIFT                                                     0x1c
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                         0x000003FFL
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                        0x000FFC00L
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                         0x00F00000L
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT__CNTR_MODE1_MASK                                                       0x0F000000L
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT__CNTR_MODE0_MASK                                                       0xF0000000L
+//SDMA0_SDMA_PERFCOUNTER1_SELECT1
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                     0xa
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT1__CNTR_MODE3__SHIFT                                                    0x18
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT1__CNTR_MODE2__SHIFT                                                    0x1c
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                       0x000003FFL
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                       0x000FFC00L
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT1__CNTR_MODE3_MASK                                                      0x0F000000L
+#define SDMA0_SDMA_PERFCOUNTER1_SELECT1__CNTR_MODE2_MASK                                                      0xF0000000L
+//SDMA0_SDMA_PERFCOUNTER2_SELECT
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                       0x0
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                      0xa
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT__SPM_MODE__SHIFT                                                       0x14
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT__CNTR_MODE1__SHIFT                                                     0x18
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT__CNTR_MODE0__SHIFT                                                     0x1c
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                         0x000003FFL
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                        0x000FFC00L
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT__SPM_MODE_MASK                                                         0x00F00000L
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT__CNTR_MODE1_MASK                                                       0x0F000000L
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT__CNTR_MODE0_MASK                                                       0xF0000000L
+//SDMA0_SDMA_PERFCOUNTER2_SELECT1
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                     0xa
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT1__CNTR_MODE3__SHIFT                                                    0x18
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT1__CNTR_MODE2__SHIFT                                                    0x1c
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                       0x000003FFL
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                       0x000FFC00L
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT1__CNTR_MODE3_MASK                                                      0x0F000000L
+#define SDMA0_SDMA_PERFCOUNTER2_SELECT1__CNTR_MODE2_MASK                                                      0xF0000000L
+//SDMA0_SDMA_PERFCOUNTER3_SELECT
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                       0x0
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                      0xa
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT__SPM_MODE__SHIFT                                                       0x14
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT__CNTR_MODE1__SHIFT                                                     0x18
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT__CNTR_MODE0__SHIFT                                                     0x1c
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                         0x000003FFL
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                        0x000FFC00L
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT__SPM_MODE_MASK                                                         0x00F00000L
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT__CNTR_MODE1_MASK                                                       0x0F000000L
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT__CNTR_MODE0_MASK                                                       0xF0000000L
+//SDMA0_SDMA_PERFCOUNTER3_SELECT1
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                     0xa
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT1__CNTR_MODE3__SHIFT                                                    0x18
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT1__CNTR_MODE2__SHIFT                                                    0x1c
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                       0x000003FFL
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                       0x000FFC00L
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT1__CNTR_MODE3_MASK                                                      0x0F000000L
+#define SDMA0_SDMA_PERFCOUNTER3_SELECT1__CNTR_MODE2_MASK                                                      0xF0000000L
+//SDMA0_SDMA_PERFCOUNTER4_SELECT
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                       0x0
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT__PERF_SEL1__SHIFT                                                      0xa
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT__SPM_MODE__SHIFT                                                       0x14
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT__CNTR_MODE1__SHIFT                                                     0x18
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT__CNTR_MODE0__SHIFT                                                     0x1c
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                         0x000003FFL
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT__PERF_SEL1_MASK                                                        0x000FFC00L
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT__SPM_MODE_MASK                                                         0x00F00000L
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT__CNTR_MODE1_MASK                                                       0x0F000000L
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT__CNTR_MODE0_MASK                                                       0xF0000000L
+//SDMA0_SDMA_PERFCOUNTER4_SELECT1
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT1__PERF_SEL2__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT1__PERF_SEL3__SHIFT                                                     0xa
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT1__CNTR_MODE3__SHIFT                                                    0x18
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT1__CNTR_MODE2__SHIFT                                                    0x1c
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT1__PERF_SEL2_MASK                                                       0x000003FFL
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT1__PERF_SEL3_MASK                                                       0x000FFC00L
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT1__CNTR_MODE3_MASK                                                      0x0F000000L
+#define SDMA0_SDMA_PERFCOUNTER4_SELECT1__CNTR_MODE2_MASK                                                      0xF0000000L
+//SDMA0_SDMA_PERFCOUNTER5_SELECT
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                       0x0
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT__PERF_SEL1__SHIFT                                                      0xa
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT__SPM_MODE__SHIFT                                                       0x14
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT__CNTR_MODE1__SHIFT                                                     0x18
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT__CNTR_MODE0__SHIFT                                                     0x1c
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                         0x000003FFL
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT__PERF_SEL1_MASK                                                        0x000FFC00L
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT__SPM_MODE_MASK                                                         0x00F00000L
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT__CNTR_MODE1_MASK                                                       0x0F000000L
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT__CNTR_MODE0_MASK                                                       0xF0000000L
+//SDMA0_SDMA_PERFCOUNTER5_SELECT1
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT1__PERF_SEL2__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT1__PERF_SEL3__SHIFT                                                     0xa
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT1__CNTR_MODE3__SHIFT                                                    0x18
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT1__CNTR_MODE2__SHIFT                                                    0x1c
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT1__PERF_SEL2_MASK                                                       0x000003FFL
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT1__PERF_SEL3_MASK                                                       0x000FFC00L
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT1__CNTR_MODE3_MASK                                                      0x0F000000L
+#define SDMA0_SDMA_PERFCOUNTER5_SELECT1__CNTR_MODE2_MASK                                                      0xF0000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmaperfddec
+//SDMA0_SDMA_PERFCNT_PERFCOUNTER_LO
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                  0x0
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_LO__COUNTER_LO_MASK                                                    0xFFFFFFFFL
+//SDMA0_SDMA_PERFCNT_PERFCOUNTER_HI
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                  0x0
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                               0x10
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_HI__COUNTER_HI_MASK                                                    0x0000FFFFL
+#define SDMA0_SDMA_PERFCNT_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                 0xFFFF0000L
+//SDMA0_SDMA_PERFCOUNTER0_LO
+#define SDMA0_SDMA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                       0xFFFFFFFFL
+//SDMA0_SDMA_PERFCOUNTER0_HI
+#define SDMA0_SDMA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                       0xFFFFFFFFL
+//SDMA0_SDMA_PERFCOUNTER1_LO
+#define SDMA0_SDMA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                       0xFFFFFFFFL
+//SDMA0_SDMA_PERFCOUNTER1_HI
+#define SDMA0_SDMA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                       0xFFFFFFFFL
+//SDMA0_SDMA_PERFCOUNTER2_LO
+#define SDMA0_SDMA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                       0xFFFFFFFFL
+//SDMA0_SDMA_PERFCOUNTER2_HI
+#define SDMA0_SDMA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                       0xFFFFFFFFL
+//SDMA0_SDMA_PERFCOUNTER3_LO
+#define SDMA0_SDMA_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                       0xFFFFFFFFL
+//SDMA0_SDMA_PERFCOUNTER3_HI
+#define SDMA0_SDMA_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                       0xFFFFFFFFL
+//SDMA0_SDMA_PERFCOUNTER4_LO
+#define SDMA0_SDMA_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                       0xFFFFFFFFL
+//SDMA0_SDMA_PERFCOUNTER4_HI
+#define SDMA0_SDMA_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                       0xFFFFFFFFL
+//SDMA0_SDMA_PERFCOUNTER5_LO
+#define SDMA0_SDMA_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                       0xFFFFFFFFL
+//SDMA0_SDMA_PERFCOUNTER5_HI
+#define SDMA0_SDMA_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                     0x0
+#define SDMA0_SDMA_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                       0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmapwrdec
+//SDMA0_GFX_ICG_SDMA_CTRL
+#define SDMA0_GFX_ICG_SDMA_CTRL__REG_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                            0x1
+#define SDMA0_GFX_ICG_SDMA_CTRL__PTR_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                            0x2
+#define SDMA0_GFX_ICG_SDMA_CTRL__PIO_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                            0x3
+#define SDMA0_GFX_ICG_SDMA_CTRL__MCU_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                            0x4
+#define SDMA0_GFX_ICG_SDMA_CTRL__COPY_ENG_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                       0x5
+#define SDMA0_GFX_ICG_SDMA_CTRL__SERVE_ENG_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                      0x6
+#define SDMA0_GFX_ICG_SDMA_CTRL__CMD_FETCH_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                      0x7
+#define SDMA0_GFX_ICG_SDMA_CTRL__GU_MEMREQ_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                      0x8
+#define SDMA0_GFX_ICG_SDMA_CTRL__INV_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                            0x9
+#define SDMA0_GFX_ICG_SDMA_CTRL__GU_CACHE_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                       0xa
+#define SDMA0_GFX_ICG_SDMA_CTRL__IC_CACHE_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                       0xb
+#define SDMA0_GFX_ICG_SDMA_CTRL__MEM_CHNL_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                       0xc
+#define SDMA0_GFX_ICG_SDMA_CTRL__PERF_CNTR_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                      0xd
+#define SDMA0_GFX_ICG_SDMA_CTRL__CORE_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                           0xe
+#define SDMA0_GFX_ICG_SDMA_CTRL__MEM_CHNL_CESE_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                  0xf
+#define SDMA0_GFX_ICG_SDMA_CTRL__FE_ENG_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                         0x10
+#define SDMA0_GFX_ICG_SDMA_CTRL__MEM_CHNL_CESEFE_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                0x11
+#define SDMA0_GFX_ICG_SDMA_CTRL__MEM_CHNL_FE_WARMUP_MGCG_CLK_SOFT_OVERRIDE__SHIFT                             0x12
+#define SDMA0_GFX_ICG_SDMA_CTRL__MGCG_CLK_HYST__SHIFT                                                         0x18
+#define SDMA0_GFX_ICG_SDMA_CTRL__REG_MGCG_CLK_SOFT_OVERRIDE_MASK                                              0x00000002L
+#define SDMA0_GFX_ICG_SDMA_CTRL__PTR_MGCG_CLK_SOFT_OVERRIDE_MASK                                              0x00000004L
+#define SDMA0_GFX_ICG_SDMA_CTRL__PIO_MGCG_CLK_SOFT_OVERRIDE_MASK                                              0x00000008L
+#define SDMA0_GFX_ICG_SDMA_CTRL__MCU_MGCG_CLK_SOFT_OVERRIDE_MASK                                              0x00000010L
+#define SDMA0_GFX_ICG_SDMA_CTRL__COPY_ENG_MGCG_CLK_SOFT_OVERRIDE_MASK                                         0x00000020L
+#define SDMA0_GFX_ICG_SDMA_CTRL__SERVE_ENG_MGCG_CLK_SOFT_OVERRIDE_MASK                                        0x00000040L
+#define SDMA0_GFX_ICG_SDMA_CTRL__CMD_FETCH_MGCG_CLK_SOFT_OVERRIDE_MASK                                        0x00000080L
+#define SDMA0_GFX_ICG_SDMA_CTRL__GU_MEMREQ_MGCG_CLK_SOFT_OVERRIDE_MASK                                        0x00000100L
+#define SDMA0_GFX_ICG_SDMA_CTRL__INV_MGCG_CLK_SOFT_OVERRIDE_MASK                                              0x00000200L
+#define SDMA0_GFX_ICG_SDMA_CTRL__GU_CACHE_MGCG_CLK_SOFT_OVERRIDE_MASK                                         0x00000400L
+#define SDMA0_GFX_ICG_SDMA_CTRL__IC_CACHE_MGCG_CLK_SOFT_OVERRIDE_MASK                                         0x00000800L
+#define SDMA0_GFX_ICG_SDMA_CTRL__MEM_CHNL_MGCG_CLK_SOFT_OVERRIDE_MASK                                         0x00001000L
+#define SDMA0_GFX_ICG_SDMA_CTRL__PERF_CNTR_MGCG_CLK_SOFT_OVERRIDE_MASK                                        0x00002000L
+#define SDMA0_GFX_ICG_SDMA_CTRL__CORE_MGCG_CLK_SOFT_OVERRIDE_MASK                                             0x00004000L
+#define SDMA0_GFX_ICG_SDMA_CTRL__MEM_CHNL_CESE_MGCG_CLK_SOFT_OVERRIDE_MASK                                    0x00008000L
+#define SDMA0_GFX_ICG_SDMA_CTRL__FE_ENG_MGCG_CLK_SOFT_OVERRIDE_MASK                                           0x00010000L
+#define SDMA0_GFX_ICG_SDMA_CTRL__MEM_CHNL_CESEFE_MGCG_CLK_SOFT_OVERRIDE_MASK                                  0x00020000L
+#define SDMA0_GFX_ICG_SDMA_CTRL__MEM_CHNL_FE_WARMUP_MGCG_CLK_SOFT_OVERRIDE_MASK                               0x00040000L
+#define SDMA0_GFX_ICG_SDMA_CTRL__MGCG_CLK_HYST_MASK                                                           0xFF000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmadec:1
+//SDMA1_SDMA_DEC_START
+#define SDMA1_SDMA_DEC_START__START__SHIFT                                                                    0x0
+#define SDMA1_SDMA_DEC_START__START_MASK                                                                      0xFFFFFFFFL
+//SDMA1_SDMA_MCU_MISC_CNTL
+#define SDMA1_SDMA_MCU_MISC_CNTL__MCU_WAKEUP__SHIFT                                                           0x0
+#define SDMA1_SDMA_MCU_MISC_CNTL__MCU_WAKEUP_MASK                                                             0x00000001L
+//SDMA1_SDMA_UCODE_REV
+#define SDMA1_SDMA_UCODE_REV__P4_CL__SHIFT                                                                    0x0
+#define SDMA1_SDMA_UCODE_REV__DEBUG_VER__SHIFT                                                                0x18
+#define SDMA1_SDMA_UCODE_REV__VARIANT_ID__SHIFT                                                               0x1c
+#define SDMA1_SDMA_UCODE_REV__P4_CL_MASK                                                                      0x00FFFFFFL
+#define SDMA1_SDMA_UCODE_REV__DEBUG_VER_MASK                                                                  0x0F000000L
+#define SDMA1_SDMA_UCODE_REV__VARIANT_ID_MASK                                                                 0xF0000000L
+//SDMA1_SDMA_GLOBAL_TIMESTAMP_LO
+#define SDMA1_SDMA_GLOBAL_TIMESTAMP_LO__DATA__SHIFT                                                           0x0
+#define SDMA1_SDMA_GLOBAL_TIMESTAMP_LO__DATA_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_GLOBAL_TIMESTAMP_HI
+#define SDMA1_SDMA_GLOBAL_TIMESTAMP_HI__DATA__SHIFT                                                           0x0
+#define SDMA1_SDMA_GLOBAL_TIMESTAMP_HI__DATA_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_POWER_CNTL
+#define SDMA1_SDMA_POWER_CNTL__LS_ENABLE__SHIFT                                                               0x8
+#define SDMA1_SDMA_POWER_CNTL__LS_ENABLE_MASK                                                                 0x00000100L
+//SDMA1_SDMA_CNTL
+#define SDMA1_SDMA_CNTL__TRAP_ENABLE__SHIFT                                                                   0x0
+#define SDMA1_SDMA_CNTL__RESERVED__SHIFT                                                                      0x2
+#define SDMA1_SDMA_CNTL__DATA_SWAP_ENABLE__SHIFT                                                              0x3
+#define SDMA1_SDMA_CNTL__FENCE_SWAP_ENABLE__SHIFT                                                             0x4
+#define SDMA1_SDMA_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                         0x5
+#define SDMA1_SDMA_CNTL__PIO_DONE_ACK_ENABLE__SHIFT                                                           0x6
+#define SDMA1_SDMA_CNTL__CP_MES_SYNC_ENABLE__SHIFT                                                            0x7
+#define SDMA1_SDMA_CNTL__TMZ_MIDCMD_PREEMPT_ENABLE__SHIFT                                                     0x8
+#define SDMA1_SDMA_CNTL__MIDCMD_EXPIRE_ENABLE__SHIFT                                                          0x9
+#define SDMA1_SDMA_CNTL__CP_MES_INT_ENABLE__SHIFT                                                             0xa
+#define SDMA1_SDMA_CNTL__PAGE_RETRY_TIMEOUT_INT_ENABLE__SHIFT                                                 0xb
+#define SDMA1_SDMA_CNTL__PAGE_NULL_INT_ENABLE__SHIFT                                                          0xc
+#define SDMA1_SDMA_CNTL__PAGE_FAULT_INT_ENABLE__SHIFT                                                         0xd
+#define SDMA1_SDMA_CNTL__CH_PERFCNT_ENABLE__SHIFT                                                             0x10
+#define SDMA1_SDMA_CNTL__MIDCMD_WORLDSWITCH_ENABLE__SHIFT                                                     0x11
+#define SDMA1_SDMA_CNTL__CTXEMPTY_INT_ENABLE__SHIFT                                                           0x1c
+#define SDMA1_SDMA_CNTL__FROZEN_INT_ENABLE__SHIFT                                                             0x1d
+#define SDMA1_SDMA_CNTL__IB_PREEMPT_INT_ENABLE__SHIFT                                                         0x1e
+#define SDMA1_SDMA_CNTL__RB_PREEMPT_INT_ENABLE__SHIFT                                                         0x1f
+#define SDMA1_SDMA_CNTL__TRAP_ENABLE_MASK                                                                     0x00000001L
+#define SDMA1_SDMA_CNTL__RESERVED_MASK                                                                        0x00000004L
+#define SDMA1_SDMA_CNTL__DATA_SWAP_ENABLE_MASK                                                                0x00000008L
+#define SDMA1_SDMA_CNTL__FENCE_SWAP_ENABLE_MASK                                                               0x00000010L
+#define SDMA1_SDMA_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                           0x00000020L
+#define SDMA1_SDMA_CNTL__PIO_DONE_ACK_ENABLE_MASK                                                             0x00000040L
+#define SDMA1_SDMA_CNTL__CP_MES_SYNC_ENABLE_MASK                                                              0x00000080L
+#define SDMA1_SDMA_CNTL__TMZ_MIDCMD_PREEMPT_ENABLE_MASK                                                       0x00000100L
+#define SDMA1_SDMA_CNTL__MIDCMD_EXPIRE_ENABLE_MASK                                                            0x00000200L
+#define SDMA1_SDMA_CNTL__CP_MES_INT_ENABLE_MASK                                                               0x00000400L
+#define SDMA1_SDMA_CNTL__PAGE_RETRY_TIMEOUT_INT_ENABLE_MASK                                                   0x00000800L
+#define SDMA1_SDMA_CNTL__PAGE_NULL_INT_ENABLE_MASK                                                            0x00001000L
+#define SDMA1_SDMA_CNTL__PAGE_FAULT_INT_ENABLE_MASK                                                           0x00002000L
+#define SDMA1_SDMA_CNTL__CH_PERFCNT_ENABLE_MASK                                                               0x00010000L
+#define SDMA1_SDMA_CNTL__MIDCMD_WORLDSWITCH_ENABLE_MASK                                                       0x00020000L
+#define SDMA1_SDMA_CNTL__CTXEMPTY_INT_ENABLE_MASK                                                             0x10000000L
+#define SDMA1_SDMA_CNTL__FROZEN_INT_ENABLE_MASK                                                               0x20000000L
+#define SDMA1_SDMA_CNTL__IB_PREEMPT_INT_ENABLE_MASK                                                           0x40000000L
+#define SDMA1_SDMA_CNTL__RB_PREEMPT_INT_ENABLE_MASK                                                           0x80000000L
+//SDMA1_SDMA_CHICKEN_BITS
+#define SDMA1_SDMA_CHICKEN_BITS__SRBM_POLL_RETRYING__SHIFT                                                    0x5
+#define SDMA1_SDMA_CHICKEN_BITS__RD_BURST__SHIFT                                                              0x6
+#define SDMA1_SDMA_CHICKEN_BITS__WR_BURST__SHIFT                                                              0x8
+#define SDMA1_SDMA_CHICKEN_BITS__COMBINE_256B_WAIT_CYCLE__SHIFT                                               0xa
+#define SDMA1_SDMA_CHICKEN_BITS__WR_COMBINE_256B_ENABLE__SHIFT                                                0xe
+#define SDMA1_SDMA_CHICKEN_BITS__RD_COMBINE_256B_ENABLE__SHIFT                                                0xf
+#define SDMA1_SDMA_CHICKEN_BITS__OVERLAP_ENABLE__SHIFT                                                        0x10
+#define SDMA1_SDMA_CHICKEN_BITS__T2L_256B_ENABLE__SHIFT                                                       0x12
+#define SDMA1_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_GCR_FGCG__SHIFT                                                0x13
+#define SDMA1_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_GRBM_FGCG__SHIFT                                               0x14
+#define SDMA1_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_CH_FGCG__SHIFT                                                 0x15
+#define SDMA1_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_UTCL2_INVREQ_FGCG__SHIFT                                       0x16
+#define SDMA1_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_UTCL1_FGCG__SHIFT                                              0x17
+#define SDMA1_SDMA_CHICKEN_BITS__CG_STATUS_OUTPUT__SHIFT                                                      0x18
+#define SDMA1_SDMA_CHICKEN_BITS__SW_FREEZE_ENABLE__SHIFT                                                      0x19
+#define SDMA1_SDMA_CHICKEN_BITS__DRAM_ECC_COPY_MODE_CNTL__SHIFT                                               0x1a
+#define SDMA1_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_REG_ADDR_CHECK__SHIFT                                          0x1b
+#define SDMA1_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_CANE_FGCG__SHIFT                                               0x1c
+#define SDMA1_SDMA_CHICKEN_BITS__RESERVED__SHIFT                                                              0x1d
+#define SDMA1_SDMA_CHICKEN_BITS__SRBM_POLL_RETRYING_MASK                                                      0x00000020L
+#define SDMA1_SDMA_CHICKEN_BITS__RD_BURST_MASK                                                                0x000000C0L
+#define SDMA1_SDMA_CHICKEN_BITS__WR_BURST_MASK                                                                0x00000300L
+#define SDMA1_SDMA_CHICKEN_BITS__COMBINE_256B_WAIT_CYCLE_MASK                                                 0x00003C00L
+#define SDMA1_SDMA_CHICKEN_BITS__WR_COMBINE_256B_ENABLE_MASK                                                  0x00004000L
+#define SDMA1_SDMA_CHICKEN_BITS__RD_COMBINE_256B_ENABLE_MASK                                                  0x00008000L
+#define SDMA1_SDMA_CHICKEN_BITS__OVERLAP_ENABLE_MASK                                                          0x00010000L
+#define SDMA1_SDMA_CHICKEN_BITS__T2L_256B_ENABLE_MASK                                                         0x00040000L
+#define SDMA1_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_GCR_FGCG_MASK                                                  0x00080000L
+#define SDMA1_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_GRBM_FGCG_MASK                                                 0x00100000L
+#define SDMA1_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_CH_FGCG_MASK                                                   0x00200000L
+#define SDMA1_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_UTCL2_INVREQ_FGCG_MASK                                         0x00400000L
+#define SDMA1_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_UTCL1_FGCG_MASK                                                0x00800000L
+#define SDMA1_SDMA_CHICKEN_BITS__CG_STATUS_OUTPUT_MASK                                                        0x01000000L
+#define SDMA1_SDMA_CHICKEN_BITS__SW_FREEZE_ENABLE_MASK                                                        0x02000000L
+#define SDMA1_SDMA_CHICKEN_BITS__DRAM_ECC_COPY_MODE_CNTL_MASK                                                 0x04000000L
+#define SDMA1_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_REG_ADDR_CHECK_MASK                                            0x08000000L
+#define SDMA1_SDMA_CHICKEN_BITS__SOFT_OVERRIDE_CANE_FGCG_MASK                                                 0x10000000L
+#define SDMA1_SDMA_CHICKEN_BITS__RESERVED_MASK                                                                0xE0000000L
+//SDMA1_SDMA_CACHE_CNTL
+#define SDMA1_SDMA_CACHE_CNTL__RD_TEMPORAL_HINT__SHIFT                                                        0x0
+#define SDMA1_SDMA_CACHE_CNTL__WR_TEMPORAL_HINT__SHIFT                                                        0x3
+#define SDMA1_SDMA_CACHE_CNTL__RD_SCOPE__SHIFT                                                                0xc
+#define SDMA1_SDMA_CACHE_CNTL__WR_SCOPE__SHIFT                                                                0xe
+#define SDMA1_SDMA_CACHE_CNTL__RD_TEMPORAL_HINT_MASK                                                          0x00000007L
+#define SDMA1_SDMA_CACHE_CNTL__WR_TEMPORAL_HINT_MASK                                                          0x00000038L
+#define SDMA1_SDMA_CACHE_CNTL__RD_SCOPE_MASK                                                                  0x00003000L
+#define SDMA1_SDMA_CACHE_CNTL__WR_SCOPE_MASK                                                                  0x0000C000L
+//SDMA1_SDMA_RB_RPTR_FETCH
+#define SDMA1_SDMA_RB_RPTR_FETCH__OFFSET__SHIFT                                                               0x2
+#define SDMA1_SDMA_RB_RPTR_FETCH__OFFSET_MASK                                                                 0xFFFFFFFCL
+//SDMA1_SDMA_RB_RPTR_FETCH_HI
+#define SDMA1_SDMA_RB_RPTR_FETCH_HI__OFFSET__SHIFT                                                            0x0
+#define SDMA1_SDMA_RB_RPTR_FETCH_HI__OFFSET_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_IB_OFFSET_FETCH
+#define SDMA1_SDMA_IB_OFFSET_FETCH__OFFSET__SHIFT                                                             0x2
+#define SDMA1_SDMA_IB_OFFSET_FETCH__OFFSET_MASK                                                               0x003FFFFCL
+//SDMA1_SDMA_PROGRAM
+#define SDMA1_SDMA_PROGRAM__STREAM__SHIFT                                                                     0x0
+#define SDMA1_SDMA_PROGRAM__STREAM_MASK                                                                       0xFFFFFFFFL
+//SDMA1_SDMA_STATUS_REG
+#define SDMA1_SDMA_STATUS_REG__IDLE__SHIFT                                                                    0x0
+#define SDMA1_SDMA_STATUS_REG__REG_IDLE__SHIFT                                                                0x1
+#define SDMA1_SDMA_STATUS_REG__RB_EMPTY__SHIFT                                                                0x2
+#define SDMA1_SDMA_STATUS_REG__RESERVED_3__SHIFT                                                              0x3
+#define SDMA1_SDMA_STATUS_REG__RB_CMD_IDLE__SHIFT                                                             0x4
+#define SDMA1_SDMA_STATUS_REG__RB_CMD_FULL__SHIFT                                                             0x5
+#define SDMA1_SDMA_STATUS_REG__IB_CMD_IDLE__SHIFT                                                             0x6
+#define SDMA1_SDMA_STATUS_REG__IB_CMD_FULL__SHIFT                                                             0x7
+#define SDMA1_SDMA_STATUS_REG__CORE_IDLE__SHIFT                                                               0x8
+#define SDMA1_SDMA_STATUS_REG__INSIDE_IB__SHIFT                                                               0x9
+#define SDMA1_SDMA_STATUS_REG__FETCH_IDLE__SHIFT                                                              0xa
+#define SDMA1_SDMA_STATUS_REG__CGCG_FENCE__SHIFT                                                              0xb
+#define SDMA1_SDMA_STATUS_REG__PACKET_READY__SHIFT                                                            0xc
+#define SDMA1_SDMA_STATUS_REG__MC_WR_IDLE__SHIFT                                                              0xd
+#define SDMA1_SDMA_STATUS_REG__SRBM_IDLE__SHIFT                                                               0xe
+#define SDMA1_SDMA_STATUS_REG__CONTEXT_EMPTY__SHIFT                                                           0xf
+#define SDMA1_SDMA_STATUS_REG__DELTA_RPTR_FULL__SHIFT                                                         0x10
+#define SDMA1_SDMA_STATUS_REG__RB_MC_RREQ_IDLE__SHIFT                                                         0x11
+#define SDMA1_SDMA_STATUS_REG__IB_MC_RREQ_IDLE__SHIFT                                                         0x12
+#define SDMA1_SDMA_STATUS_REG__MC_RD_IDLE__SHIFT                                                              0x13
+#define SDMA1_SDMA_STATUS_REG__DELTA_RPTR_EMPTY__SHIFT                                                        0x14
+#define SDMA1_SDMA_STATUS_REG__RESERVED_24_21__SHIFT                                                          0x15
+#define SDMA1_SDMA_STATUS_REG__EXEC_ENG_IDLE__SHIFT                                                           0x19
+#define SDMA1_SDMA_STATUS_REG__PROC_CNTL_IDLE__SHIFT                                                          0x1a
+#define SDMA1_SDMA_STATUS_REG__UCODE_INIT_DONE__SHIFT                                                         0x1b
+#define SDMA1_SDMA_STATUS_REG__RESERVED_29_28__SHIFT                                                          0x1c
+#define SDMA1_SDMA_STATUS_REG__INT_IDLE__SHIFT                                                                0x1e
+#define SDMA1_SDMA_STATUS_REG__INT_REQ_STALL__SHIFT                                                           0x1f
+#define SDMA1_SDMA_STATUS_REG__IDLE_MASK                                                                      0x00000001L
+#define SDMA1_SDMA_STATUS_REG__REG_IDLE_MASK                                                                  0x00000002L
+#define SDMA1_SDMA_STATUS_REG__RB_EMPTY_MASK                                                                  0x00000004L
+#define SDMA1_SDMA_STATUS_REG__RESERVED_3_MASK                                                                0x00000008L
+#define SDMA1_SDMA_STATUS_REG__RB_CMD_IDLE_MASK                                                               0x00000010L
+#define SDMA1_SDMA_STATUS_REG__RB_CMD_FULL_MASK                                                               0x00000020L
+#define SDMA1_SDMA_STATUS_REG__IB_CMD_IDLE_MASK                                                               0x00000040L
+#define SDMA1_SDMA_STATUS_REG__IB_CMD_FULL_MASK                                                               0x00000080L
+#define SDMA1_SDMA_STATUS_REG__CORE_IDLE_MASK                                                                 0x00000100L
+#define SDMA1_SDMA_STATUS_REG__INSIDE_IB_MASK                                                                 0x00000200L
+#define SDMA1_SDMA_STATUS_REG__FETCH_IDLE_MASK                                                                0x00000400L
+#define SDMA1_SDMA_STATUS_REG__CGCG_FENCE_MASK                                                                0x00000800L
+#define SDMA1_SDMA_STATUS_REG__PACKET_READY_MASK                                                              0x00001000L
+#define SDMA1_SDMA_STATUS_REG__MC_WR_IDLE_MASK                                                                0x00002000L
+#define SDMA1_SDMA_STATUS_REG__SRBM_IDLE_MASK                                                                 0x00004000L
+#define SDMA1_SDMA_STATUS_REG__CONTEXT_EMPTY_MASK                                                             0x00008000L
+#define SDMA1_SDMA_STATUS_REG__DELTA_RPTR_FULL_MASK                                                           0x00010000L
+#define SDMA1_SDMA_STATUS_REG__RB_MC_RREQ_IDLE_MASK                                                           0x00020000L
+#define SDMA1_SDMA_STATUS_REG__IB_MC_RREQ_IDLE_MASK                                                           0x00040000L
+#define SDMA1_SDMA_STATUS_REG__MC_RD_IDLE_MASK                                                                0x00080000L
+#define SDMA1_SDMA_STATUS_REG__DELTA_RPTR_EMPTY_MASK                                                          0x00100000L
+#define SDMA1_SDMA_STATUS_REG__RESERVED_24_21_MASK                                                            0x01E00000L
+#define SDMA1_SDMA_STATUS_REG__EXEC_ENG_IDLE_MASK                                                             0x02000000L
+#define SDMA1_SDMA_STATUS_REG__PROC_CNTL_IDLE_MASK                                                            0x04000000L
+#define SDMA1_SDMA_STATUS_REG__UCODE_INIT_DONE_MASK                                                           0x08000000L
+#define SDMA1_SDMA_STATUS_REG__RESERVED_29_28_MASK                                                            0x30000000L
+#define SDMA1_SDMA_STATUS_REG__INT_IDLE_MASK                                                                  0x40000000L
+#define SDMA1_SDMA_STATUS_REG__INT_REQ_STALL_MASK                                                             0x80000000L
+//SDMA1_SDMA_STATUS1_REG
+#define SDMA1_SDMA_STATUS1_REG__CE_WREQ_IDLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_STATUS1_REG__CE_WR_IDLE__SHIFT                                                             0x1
+#define SDMA1_SDMA_STATUS1_REG__CE_SPLIT_RD_IDLE__SHIFT                                                       0x2
+#define SDMA1_SDMA_STATUS1_REG__CE_RREQ_IDLE__SHIFT                                                           0x3
+#define SDMA1_SDMA_STATUS1_REG__CE_OUT_IDLE__SHIFT                                                            0x4
+#define SDMA1_SDMA_STATUS1_REG__CE_IN_IDLE__SHIFT                                                             0x5
+#define SDMA1_SDMA_STATUS1_REG__CE_DST_IDLE__SHIFT                                                            0x6
+#define SDMA1_SDMA_STATUS1_REG__CE_WR_TRACK_IDLE__SHIFT                                                       0x7
+#define SDMA1_SDMA_STATUS1_REG__CE_SPLIT_WR_IDLE__SHIFT                                                       0x8
+#define SDMA1_SDMA_STATUS1_REG__CE_CMD_IDLE__SHIFT                                                            0x9
+#define SDMA1_SDMA_STATUS1_REG__CE_AFIFO_FULL__SHIFT                                                          0xa
+#define SDMA1_SDMA_STATUS1_REG__CE_INFO_FULL__SHIFT                                                           0xb
+#define SDMA1_SDMA_STATUS1_REG__CE_INFO1_FULL__SHIFT                                                          0xc
+#define SDMA1_SDMA_STATUS1_REG__EX_START__SHIFT                                                               0xd
+#define SDMA1_SDMA_STATUS1_REG__CE_RD_STALL__SHIFT                                                            0xf
+#define SDMA1_SDMA_STATUS1_REG__CE_WR_STALL__SHIFT                                                            0x10
+#define SDMA1_SDMA_STATUS1_REG__SEC_INTR_STATUS__SHIFT                                                        0x11
+#define SDMA1_SDMA_STATUS1_REG__WPTR_POLL_IDLE__SHIFT                                                         0x12
+#define SDMA1_SDMA_STATUS1_REG__SDMA_IDLE__SHIFT                                                              0x13
+#define SDMA1_SDMA_STATUS1_REG__IC_FETCH_IDLE__SHIFT                                                          0x14
+#define SDMA1_SDMA_STATUS1_REG__IC_FETCH_PAGE_FAULT__SHIFT                                                    0x15
+#define SDMA1_SDMA_STATUS1_REG__IC_FETCH_PAGE_RETRY_TIMEOUT__SHIFT                                            0x16
+#define SDMA1_SDMA_STATUS1_REG__IC_FETCH_PAGE_NULL__SHIFT                                                     0x17
+#define SDMA1_SDMA_STATUS1_REG__MCU_FW_STACK_OVERFLOW__SHIFT                                                  0x18
+#define SDMA1_SDMA_STATUS1_REG__CE_WREQ_IDLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_STATUS1_REG__CE_WR_IDLE_MASK                                                               0x00000002L
+#define SDMA1_SDMA_STATUS1_REG__CE_SPLIT_RD_IDLE_MASK                                                         0x00000004L
+#define SDMA1_SDMA_STATUS1_REG__CE_RREQ_IDLE_MASK                                                             0x00000008L
+#define SDMA1_SDMA_STATUS1_REG__CE_OUT_IDLE_MASK                                                              0x00000010L
+#define SDMA1_SDMA_STATUS1_REG__CE_IN_IDLE_MASK                                                               0x00000020L
+#define SDMA1_SDMA_STATUS1_REG__CE_DST_IDLE_MASK                                                              0x00000040L
+#define SDMA1_SDMA_STATUS1_REG__CE_WR_TRACK_IDLE_MASK                                                         0x00000080L
+#define SDMA1_SDMA_STATUS1_REG__CE_SPLIT_WR_IDLE_MASK                                                         0x00000100L
+#define SDMA1_SDMA_STATUS1_REG__CE_CMD_IDLE_MASK                                                              0x00000200L
+#define SDMA1_SDMA_STATUS1_REG__CE_AFIFO_FULL_MASK                                                            0x00000400L
+#define SDMA1_SDMA_STATUS1_REG__CE_INFO_FULL_MASK                                                             0x00000800L
+#define SDMA1_SDMA_STATUS1_REG__CE_INFO1_FULL_MASK                                                            0x00001000L
+#define SDMA1_SDMA_STATUS1_REG__EX_START_MASK                                                                 0x00002000L
+#define SDMA1_SDMA_STATUS1_REG__CE_RD_STALL_MASK                                                              0x00008000L
+#define SDMA1_SDMA_STATUS1_REG__CE_WR_STALL_MASK                                                              0x00010000L
+#define SDMA1_SDMA_STATUS1_REG__SEC_INTR_STATUS_MASK                                                          0x00020000L
+#define SDMA1_SDMA_STATUS1_REG__WPTR_POLL_IDLE_MASK                                                           0x00040000L
+#define SDMA1_SDMA_STATUS1_REG__SDMA_IDLE_MASK                                                                0x00080000L
+#define SDMA1_SDMA_STATUS1_REG__IC_FETCH_IDLE_MASK                                                            0x00100000L
+#define SDMA1_SDMA_STATUS1_REG__IC_FETCH_PAGE_FAULT_MASK                                                      0x00200000L
+#define SDMA1_SDMA_STATUS1_REG__IC_FETCH_PAGE_RETRY_TIMEOUT_MASK                                              0x00400000L
+#define SDMA1_SDMA_STATUS1_REG__IC_FETCH_PAGE_NULL_MASK                                                       0x00800000L
+#define SDMA1_SDMA_STATUS1_REG__MCU_FW_STACK_OVERFLOW_MASK                                                    0x03000000L
+//SDMA1_SDMA_CNTL1
+#define SDMA1_SDMA_CNTL1__WPTR_POLL_FREQUENCY__SHIFT                                                          0x2
+#define SDMA1_SDMA_CNTL1__WPTR_POLL_FREQUENCY_MASK                                                            0x0000FFFCL
+//SDMA1_SDMA_HBM_PAGE_CONFIG
+#define SDMA1_SDMA_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT__SHIFT                                                 0x0
+#define SDMA1_SDMA_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT_MASK                                                   0x00000003L
+//SDMA1_SDMA_FREEZE
+#define SDMA1_SDMA_FREEZE__FROZEN__SHIFT                                                                      0x5
+#define SDMA1_SDMA_FREEZE__MCU_FREEZE__SHIFT                                                                  0x6
+#define SDMA1_SDMA_FREEZE__IMU_FSM_STATE__SHIFT                                                               0x8
+#define SDMA1_SDMA_FREEZE__EXTERNAL_FROZEN__SHIFT                                                             0xc
+#define SDMA1_SDMA_FREEZE__FROZEN_MASK                                                                        0x00000020L
+#define SDMA1_SDMA_FREEZE__MCU_FREEZE_MASK                                                                    0x00000040L
+#define SDMA1_SDMA_FREEZE__IMU_FSM_STATE_MASK                                                                 0x00000300L
+#define SDMA1_SDMA_FREEZE__EXTERNAL_FROZEN_MASK                                                               0x00001000L
+//SDMA1_SDMA_PROCESS_QUANTUM0
+#define SDMA1_SDMA_PROCESS_QUANTUM0__PROCESS0_QUANTUM__SHIFT                                                  0x0
+#define SDMA1_SDMA_PROCESS_QUANTUM0__PROCESS1_QUANTUM__SHIFT                                                  0x8
+#define SDMA1_SDMA_PROCESS_QUANTUM0__PROCESS2_QUANTUM__SHIFT                                                  0x10
+#define SDMA1_SDMA_PROCESS_QUANTUM0__PROCESS3_QUANTUM__SHIFT                                                  0x18
+#define SDMA1_SDMA_PROCESS_QUANTUM0__PROCESS0_QUANTUM_MASK                                                    0x000000FFL
+#define SDMA1_SDMA_PROCESS_QUANTUM0__PROCESS1_QUANTUM_MASK                                                    0x0000FF00L
+#define SDMA1_SDMA_PROCESS_QUANTUM0__PROCESS2_QUANTUM_MASK                                                    0x00FF0000L
+#define SDMA1_SDMA_PROCESS_QUANTUM0__PROCESS3_QUANTUM_MASK                                                    0xFF000000L
+//SDMA1_SDMA_PROCESS_QUANTUM1
+#define SDMA1_SDMA_PROCESS_QUANTUM1__PROCESS4_QUANTUM__SHIFT                                                  0x0
+#define SDMA1_SDMA_PROCESS_QUANTUM1__PROCESS5_QUANTUM__SHIFT                                                  0x8
+#define SDMA1_SDMA_PROCESS_QUANTUM1__PROCESS6_QUANTUM__SHIFT                                                  0x10
+#define SDMA1_SDMA_PROCESS_QUANTUM1__PROCESS7_QUANTUM__SHIFT                                                  0x18
+#define SDMA1_SDMA_PROCESS_QUANTUM1__PROCESS4_QUANTUM_MASK                                                    0x000000FFL
+#define SDMA1_SDMA_PROCESS_QUANTUM1__PROCESS5_QUANTUM_MASK                                                    0x0000FF00L
+#define SDMA1_SDMA_PROCESS_QUANTUM1__PROCESS6_QUANTUM_MASK                                                    0x00FF0000L
+#define SDMA1_SDMA_PROCESS_QUANTUM1__PROCESS7_QUANTUM_MASK                                                    0xFF000000L
+//SDMA1_SDMA_WATCHDOG_CNTL
+#define SDMA1_SDMA_WATCHDOG_CNTL__QUEUE_HANG_COUNT__SHIFT                                                     0x0
+#define SDMA1_SDMA_WATCHDOG_CNTL__CMD_TIMEOUT_COUNT__SHIFT                                                    0x8
+#define SDMA1_SDMA_WATCHDOG_CNTL__QUEUE_HANG_COUNT_MASK                                                       0x000000FFL
+#define SDMA1_SDMA_WATCHDOG_CNTL__CMD_TIMEOUT_COUNT_MASK                                                      0x0000FF00L
+//SDMA1_SDMA_QUEUE_STATUS0
+#define SDMA1_SDMA_QUEUE_STATUS0__QUEUE0_STATUS__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE_STATUS0__QUEUE1_STATUS__SHIFT                                                        0x4
+#define SDMA1_SDMA_QUEUE_STATUS0__QUEUE2_STATUS__SHIFT                                                        0x8
+#define SDMA1_SDMA_QUEUE_STATUS0__QUEUE3_STATUS__SHIFT                                                        0xc
+#define SDMA1_SDMA_QUEUE_STATUS0__QUEUE4_STATUS__SHIFT                                                        0x10
+#define SDMA1_SDMA_QUEUE_STATUS0__QUEUE5_STATUS__SHIFT                                                        0x14
+#define SDMA1_SDMA_QUEUE_STATUS0__QUEUE6_STATUS__SHIFT                                                        0x18
+#define SDMA1_SDMA_QUEUE_STATUS0__QUEUE7_STATUS__SHIFT                                                        0x1c
+#define SDMA1_SDMA_QUEUE_STATUS0__QUEUE0_STATUS_MASK                                                          0x0000000FL
+#define SDMA1_SDMA_QUEUE_STATUS0__QUEUE1_STATUS_MASK                                                          0x000000F0L
+#define SDMA1_SDMA_QUEUE_STATUS0__QUEUE2_STATUS_MASK                                                          0x00000F00L
+#define SDMA1_SDMA_QUEUE_STATUS0__QUEUE3_STATUS_MASK                                                          0x0000F000L
+#define SDMA1_SDMA_QUEUE_STATUS0__QUEUE4_STATUS_MASK                                                          0x000F0000L
+#define SDMA1_SDMA_QUEUE_STATUS0__QUEUE5_STATUS_MASK                                                          0x00F00000L
+#define SDMA1_SDMA_QUEUE_STATUS0__QUEUE6_STATUS_MASK                                                          0x0F000000L
+#define SDMA1_SDMA_QUEUE_STATUS0__QUEUE7_STATUS_MASK                                                          0xF0000000L
+//SDMA1_SDMA_QUEUE_STATUS1
+#define SDMA1_SDMA_QUEUE_STATUS1__QUEUE8_STATUS__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE_STATUS1__QUEUE9_STATUS__SHIFT                                                        0x4
+#define SDMA1_SDMA_QUEUE_STATUS1__QUEUE8_STATUS_MASK                                                          0x0000000FL
+#define SDMA1_SDMA_QUEUE_STATUS1__QUEUE9_STATUS_MASK                                                          0x000000F0L
+//SDMA1_SDMA_ID
+#define SDMA1_SDMA_ID__DEVICE_ID__SHIFT                                                                       0x0
+#define SDMA1_SDMA_ID__DEVICE_ID_MASK                                                                         0x000000FFL
+//SDMA1_SDMA_VERSION
+#define SDMA1_SDMA_VERSION__MINVER__SHIFT                                                                     0x0
+#define SDMA1_SDMA_VERSION__MAJVER__SHIFT                                                                     0x8
+#define SDMA1_SDMA_VERSION__REV__SHIFT                                                                        0x10
+#define SDMA1_SDMA_VERSION__MINVER_MASK                                                                       0x0000007FL
+#define SDMA1_SDMA_VERSION__MAJVER_MASK                                                                       0x00007F00L
+#define SDMA1_SDMA_VERSION__REV_MASK                                                                          0x003F0000L
+//SDMA1_SDMA_STATUS2_REG
+#define SDMA1_SDMA_STATUS2_REG__ID__SHIFT                                                                     0x0
+#define SDMA1_SDMA_STATUS2_REG__TH0MCU_INSTR_PTR__SHIFT                                                       0x2
+#define SDMA1_SDMA_STATUS2_REG__CMD_OP__SHIFT                                                                 0x10
+#define SDMA1_SDMA_STATUS2_REG__ID_MASK                                                                       0x00000003L
+#define SDMA1_SDMA_STATUS2_REG__TH0MCU_INSTR_PTR_MASK                                                         0x0000FFFCL
+#define SDMA1_SDMA_STATUS2_REG__CMD_OP_MASK                                                                   0xFFFF0000L
+//SDMA1_SDMA_ATOMIC_CNTL
+#define SDMA1_SDMA_ATOMIC_CNTL__LOOP_TIMER__SHIFT                                                             0x0
+#define SDMA1_SDMA_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE__SHIFT                                                  0x1f
+#define SDMA1_SDMA_ATOMIC_CNTL__LOOP_TIMER_MASK                                                               0x7FFFFFFFL
+#define SDMA1_SDMA_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE_MASK                                                    0x80000000L
+//SDMA1_SDMA_ATOMIC_PERMS_CNTL
+#define SDMA1_SDMA_ATOMIC_PERMS_CNTL__EN_RETRY_ON_ATOMIC__SHIFT                                               0x0
+#define SDMA1_SDMA_ATOMIC_PERMS_CNTL__PCIE_ATOMIC_SUPPORTED__SHIFT                                            0x1
+#define SDMA1_SDMA_ATOMIC_PERMS_CNTL__EN_RETRY_ON_ATOMIC_MASK                                                 0x00000001L
+#define SDMA1_SDMA_ATOMIC_PERMS_CNTL__PCIE_ATOMIC_SUPPORTED_MASK                                              0x00000002L
+//SDMA1_SDMA_ATOMIC_PREOP_LO
+#define SDMA1_SDMA_ATOMIC_PREOP_LO__DATA__SHIFT                                                               0x0
+#define SDMA1_SDMA_ATOMIC_PREOP_LO__DATA_MASK                                                                 0xFFFFFFFFL
+//SDMA1_SDMA_ATOMIC_PREOP_HI
+#define SDMA1_SDMA_ATOMIC_PREOP_HI__DATA__SHIFT                                                               0x0
+#define SDMA1_SDMA_ATOMIC_PREOP_HI__DATA_MASK                                                                 0xFFFFFFFFL
+//SDMA1_SDMA_DCC_CNTL
+#define SDMA1_SDMA_DCC_CNTL__DCC_FORCE_BYPASS__SHIFT                                                          0x0
+#define SDMA1_SDMA_DCC_CNTL__DCC_RD_NOPTE_OVERRIDE_0__SHIFT                                                   0x1
+#define SDMA1_SDMA_DCC_CNTL__DCC_RD_NOPTE_COMP_EN_0__SHIFT                                                    0x2
+#define SDMA1_SDMA_DCC_CNTL__DCC_WR_NOPTE_OVERRIDE_0__SHIFT                                                   0x3
+#define SDMA1_SDMA_DCC_CNTL__DCC_WR_NOPTE_COMP_EN_0__SHIFT                                                    0x4
+#define SDMA1_SDMA_DCC_CNTL__DCC_RD_NOPTE_OVERRIDE_1__SHIFT                                                   0x5
+#define SDMA1_SDMA_DCC_CNTL__DCC_RD_NOPTE_COMP_EN_1__SHIFT                                                    0x6
+#define SDMA1_SDMA_DCC_CNTL__DCC_WR_NOPTE_OVERRIDE_1__SHIFT                                                   0x7
+#define SDMA1_SDMA_DCC_CNTL__DCC_WR_NOPTE_COMP_EN_1__SHIFT                                                    0x8
+#define SDMA1_SDMA_DCC_CNTL__DCC_RD_NOPTE_OVERRIDE_2__SHIFT                                                   0x9
+#define SDMA1_SDMA_DCC_CNTL__DCC_RD_NOPTE_COMP_EN_2__SHIFT                                                    0xa
+#define SDMA1_SDMA_DCC_CNTL__DCC_WR_NOPTE_OVERRIDE_2__SHIFT                                                   0xb
+#define SDMA1_SDMA_DCC_CNTL__DCC_WR_NOPTE_COMP_EN_2__SHIFT                                                    0xc
+#define SDMA1_SDMA_DCC_CNTL__DCC_RD_NOPTE_OVERRIDE_3__SHIFT                                                   0xd
+#define SDMA1_SDMA_DCC_CNTL__DCC_RD_NOPTE_COMP_EN_3__SHIFT                                                    0xe
+#define SDMA1_SDMA_DCC_CNTL__DCC_WR_NOPTE_OVERRIDE_3__SHIFT                                                   0xf
+#define SDMA1_SDMA_DCC_CNTL__DCC_WR_NOPTE_COMP_EN_3__SHIFT                                                    0x10
+#define SDMA1_SDMA_DCC_CNTL__DCC_FORCE_BYPASS_MASK                                                            0x00000001L
+#define SDMA1_SDMA_DCC_CNTL__DCC_RD_NOPTE_OVERRIDE_0_MASK                                                     0x00000002L
+#define SDMA1_SDMA_DCC_CNTL__DCC_RD_NOPTE_COMP_EN_0_MASK                                                      0x00000004L
+#define SDMA1_SDMA_DCC_CNTL__DCC_WR_NOPTE_OVERRIDE_0_MASK                                                     0x00000008L
+#define SDMA1_SDMA_DCC_CNTL__DCC_WR_NOPTE_COMP_EN_0_MASK                                                      0x00000010L
+#define SDMA1_SDMA_DCC_CNTL__DCC_RD_NOPTE_OVERRIDE_1_MASK                                                     0x00000020L
+#define SDMA1_SDMA_DCC_CNTL__DCC_RD_NOPTE_COMP_EN_1_MASK                                                      0x00000040L
+#define SDMA1_SDMA_DCC_CNTL__DCC_WR_NOPTE_OVERRIDE_1_MASK                                                     0x00000080L
+#define SDMA1_SDMA_DCC_CNTL__DCC_WR_NOPTE_COMP_EN_1_MASK                                                      0x00000100L
+#define SDMA1_SDMA_DCC_CNTL__DCC_RD_NOPTE_OVERRIDE_2_MASK                                                     0x00000200L
+#define SDMA1_SDMA_DCC_CNTL__DCC_RD_NOPTE_COMP_EN_2_MASK                                                      0x00000400L
+#define SDMA1_SDMA_DCC_CNTL__DCC_WR_NOPTE_OVERRIDE_2_MASK                                                     0x00000800L
+#define SDMA1_SDMA_DCC_CNTL__DCC_WR_NOPTE_COMP_EN_2_MASK                                                      0x00001000L
+#define SDMA1_SDMA_DCC_CNTL__DCC_RD_NOPTE_OVERRIDE_3_MASK                                                     0x00002000L
+#define SDMA1_SDMA_DCC_CNTL__DCC_RD_NOPTE_COMP_EN_3_MASK                                                      0x00004000L
+#define SDMA1_SDMA_DCC_CNTL__DCC_WR_NOPTE_OVERRIDE_3_MASK                                                     0x00008000L
+#define SDMA1_SDMA_DCC_CNTL__DCC_WR_NOPTE_COMP_EN_3_MASK                                                      0x00010000L
+//SDMA1_SDMA_UTCL1_WARMUPL2_CNTL
+#define SDMA1_SDMA_UTCL1_WARMUPL2_CNTL__WARMUP_EN__SHIFT                                                      0x0
+#define SDMA1_SDMA_UTCL1_WARMUPL2_CNTL__INTERVAL_VA__SHIFT                                                    0x1
+#define SDMA1_SDMA_UTCL1_WARMUPL2_CNTL__GRANU__SHIFT                                                          0x8
+#define SDMA1_SDMA_UTCL1_WARMUPL2_CNTL__STEP_PER_INTERVAL__SHIFT                                              0x10
+#define SDMA1_SDMA_UTCL1_WARMUPL2_CNTL__WARMUP_EN_MASK                                                        0x00000001L
+#define SDMA1_SDMA_UTCL1_WARMUPL2_CNTL__INTERVAL_VA_MASK                                                      0x0000007EL
+#define SDMA1_SDMA_UTCL1_WARMUPL2_CNTL__GRANU_MASK                                                            0x00007F00L
+#define SDMA1_SDMA_UTCL1_WARMUPL2_CNTL__STEP_PER_INTERVAL_MASK                                                0x00070000L
+//SDMA1_SDMA_UTCL1_CNTL
+#define SDMA1_SDMA_UTCL1_CNTL__REDO_DELAY__SHIFT                                                              0x0
+#define SDMA1_SDMA_UTCL1_CNTL__PAGE_WAIT_DELAY__SHIFT                                                         0x5
+#define SDMA1_SDMA_UTCL1_CNTL__RESP_MODE__SHIFT                                                               0x9
+#define SDMA1_SDMA_UTCL1_CNTL__FORCE_INVALIDATION__SHIFT                                                      0xb
+#define SDMA1_SDMA_UTCL1_CNTL__FORCE_INVREQ_HEAVY__SHIFT                                                      0xc
+#define SDMA1_SDMA_UTCL1_CNTL__WR_EXE_PERMS_CTRL__SHIFT                                                       0xd
+#define SDMA1_SDMA_UTCL1_CNTL__RD_EXE_PERMS_CTRL__SHIFT                                                       0xe
+#define SDMA1_SDMA_UTCL1_CNTL__INVACK_DELAY__SHIFT                                                            0xf
+#define SDMA1_SDMA_UTCL1_CNTL__REQL2_CREDIT__SHIFT                                                            0x13
+#define SDMA1_SDMA_UTCL1_CNTL__REQL2_5CYC_CREDIT__SHIFT                                                       0x19
+#define SDMA1_SDMA_UTCL1_CNTL__REDO_DELAY_MASK                                                                0x0000001FL
+#define SDMA1_SDMA_UTCL1_CNTL__PAGE_WAIT_DELAY_MASK                                                           0x000001E0L
+#define SDMA1_SDMA_UTCL1_CNTL__RESP_MODE_MASK                                                                 0x00000600L
+#define SDMA1_SDMA_UTCL1_CNTL__FORCE_INVALIDATION_MASK                                                        0x00000800L
+#define SDMA1_SDMA_UTCL1_CNTL__FORCE_INVREQ_HEAVY_MASK                                                        0x00001000L
+#define SDMA1_SDMA_UTCL1_CNTL__WR_EXE_PERMS_CTRL_MASK                                                         0x00002000L
+#define SDMA1_SDMA_UTCL1_CNTL__RD_EXE_PERMS_CTRL_MASK                                                         0x00004000L
+#define SDMA1_SDMA_UTCL1_CNTL__INVACK_DELAY_MASK                                                              0x00078000L
+#define SDMA1_SDMA_UTCL1_CNTL__REQL2_CREDIT_MASK                                                              0x01F80000L
+#define SDMA1_SDMA_UTCL1_CNTL__REQL2_5CYC_CREDIT_MASK                                                         0x7E000000L
+//SDMA1_SDMA_UTCL1_WATERMK
+#define SDMA1_SDMA_UTCL1_WATERMK__WR_REQ_FIFO_WATERMK__SHIFT                                                  0x0
+#define SDMA1_SDMA_UTCL1_WATERMK__WR_REQ_FIFO_DEPTH_STEP__SHIFT                                               0x4
+#define SDMA1_SDMA_UTCL1_WATERMK__RD_REQ_FIFO_WATERMK__SHIFT                                                  0x6
+#define SDMA1_SDMA_UTCL1_WATERMK__RD_REQ_FIFO_DEPTH_STEP__SHIFT                                               0xa
+#define SDMA1_SDMA_UTCL1_WATERMK__WR_PAGE_FIFO_WATERMK__SHIFT                                                 0xc
+#define SDMA1_SDMA_UTCL1_WATERMK__WR_PAGE_FIFO_DEPTH_STEP__SHIFT                                              0x10
+#define SDMA1_SDMA_UTCL1_WATERMK__RD_PAGE_FIFO_WATERMK__SHIFT                                                 0x12
+#define SDMA1_SDMA_UTCL1_WATERMK__RD_PAGE_FIFO_DEPTH_STEP__SHIFT                                              0x16
+#define SDMA1_SDMA_UTCL1_WATERMK__WR_REQ_FIFO_WATERMK_MASK                                                    0x0000000FL
+#define SDMA1_SDMA_UTCL1_WATERMK__WR_REQ_FIFO_DEPTH_STEP_MASK                                                 0x00000030L
+#define SDMA1_SDMA_UTCL1_WATERMK__RD_REQ_FIFO_WATERMK_MASK                                                    0x000003C0L
+#define SDMA1_SDMA_UTCL1_WATERMK__RD_REQ_FIFO_DEPTH_STEP_MASK                                                 0x00000C00L
+#define SDMA1_SDMA_UTCL1_WATERMK__WR_PAGE_FIFO_WATERMK_MASK                                                   0x0000F000L
+#define SDMA1_SDMA_UTCL1_WATERMK__WR_PAGE_FIFO_DEPTH_STEP_MASK                                                0x00030000L
+#define SDMA1_SDMA_UTCL1_WATERMK__RD_PAGE_FIFO_WATERMK_MASK                                                   0x003C0000L
+#define SDMA1_SDMA_UTCL1_WATERMK__RD_PAGE_FIFO_DEPTH_STEP_MASK                                                0x00C00000L
+//SDMA1_SDMA_UTCL1_TIMEOUT
+#define SDMA1_SDMA_UTCL1_TIMEOUT__XNACK_LIMIT__SHIFT                                                          0x0
+#define SDMA1_SDMA_UTCL1_TIMEOUT__XNACK_LIMIT_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_UTCL1_PAGE
+#define SDMA1_SDMA_UTCL1_PAGE__INVALID_ADDR__SHIFT                                                            0x0
+#define SDMA1_SDMA_UTCL1_PAGE__REQ_TYPE__SHIFT                                                                0x1
+#define SDMA1_SDMA_UTCL1_PAGE__USE_MTYPE__SHIFT                                                               0x7
+#define SDMA1_SDMA_UTCL1_PAGE__USE_PT_SNOOP__SHIFT                                                            0xa
+#define SDMA1_SDMA_UTCL1_PAGE__USE_IO__SHIFT                                                                  0xb
+#define SDMA1_SDMA_UTCL1_PAGE__RD_L2_POLICY__SHIFT                                                            0xc
+#define SDMA1_SDMA_UTCL1_PAGE__WR_L2_POLICY__SHIFT                                                            0xe
+#define SDMA1_SDMA_UTCL1_PAGE__USE_BC__SHIFT                                                                  0x16
+#define SDMA1_SDMA_UTCL1_PAGE__ADDR_IS_PA__SHIFT                                                              0x17
+#define SDMA1_SDMA_UTCL1_PAGE__FORCE_BE_CMD_SNOOP_VALUE__SHIFT                                                0x18
+#define SDMA1_SDMA_UTCL1_PAGE__FORCE_BE_CMD_SNOOP_EN__SHIFT                                                   0x19
+#define SDMA1_SDMA_UTCL1_PAGE__MTYPE_NO_PTE_OVERRIDE_VALUE__SHIFT                                             0x1b
+#define SDMA1_SDMA_UTCL1_PAGE__MTYPE_NO_PTE_OVERRIDE_EN__SHIFT                                                0x1e
+#define SDMA1_SDMA_UTCL1_PAGE__INVALID_ADDR_MASK                                                              0x00000001L
+#define SDMA1_SDMA_UTCL1_PAGE__REQ_TYPE_MASK                                                                  0x0000003EL
+#define SDMA1_SDMA_UTCL1_PAGE__USE_MTYPE_MASK                                                                 0x00000380L
+#define SDMA1_SDMA_UTCL1_PAGE__USE_PT_SNOOP_MASK                                                              0x00000400L
+#define SDMA1_SDMA_UTCL1_PAGE__USE_IO_MASK                                                                    0x00000800L
+#define SDMA1_SDMA_UTCL1_PAGE__RD_L2_POLICY_MASK                                                              0x00003000L
+#define SDMA1_SDMA_UTCL1_PAGE__WR_L2_POLICY_MASK                                                              0x0000C000L
+#define SDMA1_SDMA_UTCL1_PAGE__USE_BC_MASK                                                                    0x00400000L
+#define SDMA1_SDMA_UTCL1_PAGE__ADDR_IS_PA_MASK                                                                0x00800000L
+#define SDMA1_SDMA_UTCL1_PAGE__FORCE_BE_CMD_SNOOP_VALUE_MASK                                                  0x01000000L
+#define SDMA1_SDMA_UTCL1_PAGE__FORCE_BE_CMD_SNOOP_EN_MASK                                                     0x02000000L
+#define SDMA1_SDMA_UTCL1_PAGE__MTYPE_NO_PTE_OVERRIDE_VALUE_MASK                                               0x18000000L
+#define SDMA1_SDMA_UTCL1_PAGE__MTYPE_NO_PTE_OVERRIDE_EN_MASK                                                  0x40000000L
+//SDMA1_SDMA_EXTERNAL_FROZEN
+#define SDMA1_SDMA_EXTERNAL_FROZEN__THRESHOLD__SHIFT                                                          0x0
+#define SDMA1_SDMA_EXTERNAL_FROZEN__THRESHOLD_MASK                                                            0x0000FFFFL
+//SDMA1_SDMA_FREEZE_TRIGGER
+#define SDMA1_SDMA_FREEZE_TRIGGER__CORE_FREEZE__SHIFT                                                         0x0
+#define SDMA1_SDMA_FREEZE_TRIGGER__CORE_FREEZE_MASK                                                           0x00000001L
+//SDMA1_SDMA_UTCL1_CACHE_CNTL
+#define SDMA1_SDMA_UTCL1_CACHE_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                              0x0
+#define SDMA1_SDMA_UTCL1_CACHE_CNTL__MTYPE_NO_PTE_OVERRIDE_EN__SHIFT                                          0x14
+#define SDMA1_SDMA_UTCL1_CACHE_CNTL__MTYPE_NO_PTE_OVERRIDE_VALUE__SHIFT                                       0x15
+#define SDMA1_SDMA_UTCL1_CACHE_CNTL__INVALIDATE_IC__SHIFT                                                     0x19
+#define SDMA1_SDMA_UTCL1_CACHE_CNTL__INVALIDATE_GU__SHIFT                                                     0x1a
+#define SDMA1_SDMA_UTCL1_CACHE_CNTL__FRAG_LIMIT_MODE__SHIFT                                                   0x1b
+#define SDMA1_SDMA_UTCL1_CACHE_CNTL__FORCE_SNOOP__SHIFT                                                       0x1c
+#define SDMA1_SDMA_UTCL1_CACHE_CNTL__STRICT_PERMISSION__SHIFT                                                 0x1d
+#define SDMA1_SDMA_UTCL1_CACHE_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                0x000FFFFFL
+#define SDMA1_SDMA_UTCL1_CACHE_CNTL__MTYPE_NO_PTE_OVERRIDE_EN_MASK                                            0x00100000L
+#define SDMA1_SDMA_UTCL1_CACHE_CNTL__MTYPE_NO_PTE_OVERRIDE_VALUE_MASK                                         0x00600000L
+#define SDMA1_SDMA_UTCL1_CACHE_CNTL__INVALIDATE_IC_MASK                                                       0x02000000L
+#define SDMA1_SDMA_UTCL1_CACHE_CNTL__INVALIDATE_GU_MASK                                                       0x04000000L
+#define SDMA1_SDMA_UTCL1_CACHE_CNTL__FRAG_LIMIT_MODE_MASK                                                     0x08000000L
+#define SDMA1_SDMA_UTCL1_CACHE_CNTL__FORCE_SNOOP_MASK                                                         0x10000000L
+#define SDMA1_SDMA_UTCL1_CACHE_CNTL__STRICT_PERMISSION_MASK                                                   0x20000000L
+//SDMA1_SDMA_UTCL1_RD_STATUS
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_DATA_FIFO_EMPTY__SHIFT                                              0x0
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_VA_REQ_SEND_FIFO_EMPTY__SHIFT                                       0x1
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_REG_ENTRY_EMPTY__SHIFT                                              0x2
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_PAGE_FIFO_EMPTY__SHIFT                                              0x3
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_REQ_FIFO_EMPTY__SHIFT                                               0x4
+#define SDMA1_SDMA_UTCL1_RD_STATUS__RD_VA_FIFO_EMPTY__SHIFT                                                   0x5
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_VA_REQ_SEND_FIFO_FULL__SHIFT                                        0x7
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_REG_ENTRY_FULL__SHIFT                                               0x8
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_PAGE_FIFO_FULL__SHIFT                                               0x9
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_REQ_FIFO_FULL__SHIFT                                                0xa
+#define SDMA1_SDMA_UTCL1_RD_STATUS__RD_VA_FIFO_FULL__SHIFT                                                    0xb
+#define SDMA1_SDMA_UTCL1_RD_STATUS__L2_INTF_RD_IDLE__SHIFT                                                    0x10
+#define SDMA1_SDMA_UTCL1_RD_STATUS__RD_REQRET_IDLE__SHIFT                                                     0x11
+#define SDMA1_SDMA_UTCL1_RD_STATUS__RD_REQ_IDLE__SHIFT                                                        0x12
+#define SDMA1_SDMA_UTCL1_RD_STATUS__UTCL1_WARMUP_IDLE__SHIFT                                                  0x1a
+#define SDMA1_SDMA_UTCL1_RD_STATUS__UTCL2_1CYC_IDLE__SHIFT                                                    0x1b
+#define SDMA1_SDMA_UTCL1_RD_STATUS__UTCL2_1CYC_WARMUP_IDLE__SHIFT                                             0x1c
+#define SDMA1_SDMA_UTCL1_RD_STATUS__UTCL2_5CYC_CACHE1_IDLE__SHIFT                                             0x1d
+#define SDMA1_SDMA_UTCL1_RD_STATUS__UTCL2_5CYC_CACHE0_IDLE__SHIFT                                             0x1e
+#define SDMA1_SDMA_UTCL1_RD_STATUS__UTCL2_IDLE__SHIFT                                                         0x1f
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_DATA_FIFO_EMPTY_MASK                                                0x00000001L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_VA_REQ_SEND_FIFO_EMPTY_MASK                                         0x00000002L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_REG_ENTRY_EMPTY_MASK                                                0x00000004L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_PAGE_FIFO_EMPTY_MASK                                                0x00000008L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_REQ_FIFO_EMPTY_MASK                                                 0x00000010L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__RD_VA_FIFO_EMPTY_MASK                                                     0x00000020L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_VA_REQ_SEND_FIFO_FULL_MASK                                          0x00000080L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_REG_ENTRY_FULL_MASK                                                 0x00000100L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_PAGE_FIFO_FULL_MASK                                                 0x00000200L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__CE_RD_REQ_FIFO_FULL_MASK                                                  0x00000400L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__RD_VA_FIFO_FULL_MASK                                                      0x00000800L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__L2_INTF_RD_IDLE_MASK                                                      0x00010000L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__RD_REQRET_IDLE_MASK                                                       0x00020000L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__RD_REQ_IDLE_MASK                                                          0x00040000L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__UTCL1_WARMUP_IDLE_MASK                                                    0x04000000L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__UTCL2_1CYC_IDLE_MASK                                                      0x08000000L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__UTCL2_1CYC_WARMUP_IDLE_MASK                                               0x10000000L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__UTCL2_5CYC_CACHE1_IDLE_MASK                                               0x20000000L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__UTCL2_5CYC_CACHE0_IDLE_MASK                                               0x40000000L
+#define SDMA1_SDMA_UTCL1_RD_STATUS__UTCL2_IDLE_MASK                                                           0x80000000L
+//SDMA1_SDMA_UTCL1_WR_STATUS
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_DATA_FIFO_EMPTY__SHIFT                                              0x0
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_VA_REQ_SEND_FIFO_EMPTY__SHIFT                                       0x1
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_REG_ENTRY_EMPTY__SHIFT                                              0x2
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_PAGE_FIFO_EMPTY__SHIFT                                              0x3
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_REQ_FIFO_EMPTY__SHIFT                                               0x4
+#define SDMA1_SDMA_UTCL1_WR_STATUS__WR_VA_FIFO_EMPTY__SHIFT                                                   0x5
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_DATA_FIFO_FULL__SHIFT                                               0x6
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_VA_REQ_SEND_FIFO_FULL__SHIFT                                        0x7
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_REG_ENTRY_FULL__SHIFT                                               0x8
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_PAGE_FIFO_FULL__SHIFT                                               0x9
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_REQ_FIFO_FULL__SHIFT                                                0xa
+#define SDMA1_SDMA_UTCL1_WR_STATUS__WR_VA_FIFO_FULL__SHIFT                                                    0xb
+#define SDMA1_SDMA_UTCL1_WR_STATUS__L2_INTF_WR_IDLE__SHIFT                                                    0x10
+#define SDMA1_SDMA_UTCL1_WR_STATUS__WR_REQRET_IDLE__SHIFT                                                     0x11
+#define SDMA1_SDMA_UTCL1_WR_STATUS__WR_REQ_IDLE__SHIFT                                                        0x12
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_DATA_FIFO_EMPTY_MASK                                                0x00000001L
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_VA_REQ_SEND_FIFO_EMPTY_MASK                                         0x00000002L
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_REG_ENTRY_EMPTY_MASK                                                0x00000004L
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_PAGE_FIFO_EMPTY_MASK                                                0x00000008L
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_REQ_FIFO_EMPTY_MASK                                                 0x00000010L
+#define SDMA1_SDMA_UTCL1_WR_STATUS__WR_VA_FIFO_EMPTY_MASK                                                     0x00000020L
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_DATA_FIFO_FULL_MASK                                                 0x00000040L
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_VA_REQ_SEND_FIFO_FULL_MASK                                          0x00000080L
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_REG_ENTRY_FULL_MASK                                                 0x00000100L
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_PAGE_FIFO_FULL_MASK                                                 0x00000200L
+#define SDMA1_SDMA_UTCL1_WR_STATUS__CE_WR_REQ_FIFO_FULL_MASK                                                  0x00000400L
+#define SDMA1_SDMA_UTCL1_WR_STATUS__WR_VA_FIFO_FULL_MASK                                                      0x00000800L
+#define SDMA1_SDMA_UTCL1_WR_STATUS__L2_INTF_WR_IDLE_MASK                                                      0x00010000L
+#define SDMA1_SDMA_UTCL1_WR_STATUS__WR_REQRET_IDLE_MASK                                                       0x00020000L
+#define SDMA1_SDMA_UTCL1_WR_STATUS__WR_REQ_IDLE_MASK                                                          0x00040000L
+//SDMA1_SDMA_UTCL1_INV0
+#define SDMA1_SDMA_UTCL1_INV0__INV_PROC_BUSY__SHIFT                                                           0x0
+#define SDMA1_SDMA_UTCL1_INV0__GPUVM_FRAG_SIZE__SHIFT                                                         0x1
+#define SDMA1_SDMA_UTCL1_INV0__GPUVM_VMID__SHIFT                                                              0x7
+#define SDMA1_SDMA_UTCL1_INV0__GPUVM_MODE__SHIFT                                                              0xb
+#define SDMA1_SDMA_UTCL1_INV0__GPUVM_HIGH__SHIFT                                                              0xd
+#define SDMA1_SDMA_UTCL1_INV0__GPUVM_TAG__SHIFT                                                               0xe
+#define SDMA1_SDMA_UTCL1_INV0__GPUVM_VMID_HIGH__SHIFT                                                         0x12
+#define SDMA1_SDMA_UTCL1_INV0__GPUVM_VMID_LOW__SHIFT                                                          0x16
+#define SDMA1_SDMA_UTCL1_INV0__INV_TYPE__SHIFT                                                                0x1a
+#define SDMA1_SDMA_UTCL1_INV0__INV_PROC_BUSY_MASK                                                             0x00000001L
+#define SDMA1_SDMA_UTCL1_INV0__GPUVM_FRAG_SIZE_MASK                                                           0x0000007EL
+#define SDMA1_SDMA_UTCL1_INV0__GPUVM_VMID_MASK                                                                0x00000780L
+#define SDMA1_SDMA_UTCL1_INV0__GPUVM_MODE_MASK                                                                0x00001800L
+#define SDMA1_SDMA_UTCL1_INV0__GPUVM_HIGH_MASK                                                                0x00002000L
+#define SDMA1_SDMA_UTCL1_INV0__GPUVM_TAG_MASK                                                                 0x0003C000L
+#define SDMA1_SDMA_UTCL1_INV0__GPUVM_VMID_HIGH_MASK                                                           0x003C0000L
+#define SDMA1_SDMA_UTCL1_INV0__GPUVM_VMID_LOW_MASK                                                            0x03C00000L
+#define SDMA1_SDMA_UTCL1_INV0__INV_TYPE_MASK                                                                  0x0C000000L
+//SDMA1_SDMA_UTCL1_INV1
+#define SDMA1_SDMA_UTCL1_INV1__INV_ADDR_LO__SHIFT                                                             0x0
+#define SDMA1_SDMA_UTCL1_INV1__INV_ADDR_LO_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_UTCL1_INV2
+#define SDMA1_SDMA_UTCL1_INV2__CPF_VMID__SHIFT                                                                0x0
+#define SDMA1_SDMA_UTCL1_INV2__CPF_FLUSH_TYPE__SHIFT                                                          0x10
+#define SDMA1_SDMA_UTCL1_INV2__CPF_FRAG_SIZE__SHIFT                                                           0x11
+#define SDMA1_SDMA_UTCL1_INV2__CPF_VMID_MASK                                                                  0x0000FFFFL
+#define SDMA1_SDMA_UTCL1_INV2__CPF_FLUSH_TYPE_MASK                                                            0x00010000L
+#define SDMA1_SDMA_UTCL1_INV2__CPF_FRAG_SIZE_MASK                                                             0x007E0000L
+//SDMA1_SDMA_UTCL1_RD_XNACK0
+#define SDMA1_SDMA_UTCL1_RD_XNACK0__XNACK_FAULT_ADDR_LO__SHIFT                                                0x0
+#define SDMA1_SDMA_UTCL1_RD_XNACK0__XNACK_FAULT_ADDR_LO_MASK                                                  0xFFFFFFFFL
+//SDMA1_SDMA_UTCL1_RD_XNACK1
+#define SDMA1_SDMA_UTCL1_RD_XNACK1__XNACK_FAULT_ADDR_HI__SHIFT                                                0x0
+#define SDMA1_SDMA_UTCL1_RD_XNACK1__XNACK_FAULT_VMID__SHIFT                                                   0xd
+#define SDMA1_SDMA_UTCL1_RD_XNACK1__XNACK_FAULT_VECTOR__SHIFT                                                 0x11
+#define SDMA1_SDMA_UTCL1_RD_XNACK1__XNACK_NULL_VECTOR__SHIFT                                                  0x14
+#define SDMA1_SDMA_UTCL1_RD_XNACK1__XNACK_TIMEOUT_VECTOR__SHIFT                                               0x17
+#define SDMA1_SDMA_UTCL1_RD_XNACK1__XNACK_FAULT_FLAG__SHIFT                                                   0x1a
+#define SDMA1_SDMA_UTCL1_RD_XNACK1__XNACK_NULL_FLAG__SHIFT                                                    0x1b
+#define SDMA1_SDMA_UTCL1_RD_XNACK1__XNACK_TIMEOUT_FLAG__SHIFT                                                 0x1c
+#define SDMA1_SDMA_UTCL1_RD_XNACK1__XNACK_FAULT_ADDR_HI_MASK                                                  0x00001FFFL
+#define SDMA1_SDMA_UTCL1_RD_XNACK1__XNACK_FAULT_VMID_MASK                                                     0x0001E000L
+#define SDMA1_SDMA_UTCL1_RD_XNACK1__XNACK_FAULT_VECTOR_MASK                                                   0x000E0000L
+#define SDMA1_SDMA_UTCL1_RD_XNACK1__XNACK_NULL_VECTOR_MASK                                                    0x00700000L
+#define SDMA1_SDMA_UTCL1_RD_XNACK1__XNACK_TIMEOUT_VECTOR_MASK                                                 0x03800000L
+#define SDMA1_SDMA_UTCL1_RD_XNACK1__XNACK_FAULT_FLAG_MASK                                                     0x04000000L
+#define SDMA1_SDMA_UTCL1_RD_XNACK1__XNACK_NULL_FLAG_MASK                                                      0x08000000L
+#define SDMA1_SDMA_UTCL1_RD_XNACK1__XNACK_TIMEOUT_FLAG_MASK                                                   0x10000000L
+//SDMA1_SDMA_UTCL1_WR_XNACK0
+#define SDMA1_SDMA_UTCL1_WR_XNACK0__XNACK_FAULT_ADDR_LO__SHIFT                                                0x0
+#define SDMA1_SDMA_UTCL1_WR_XNACK0__XNACK_FAULT_ADDR_LO_MASK                                                  0xFFFFFFFFL
+//SDMA1_SDMA_UTCL1_WR_XNACK1
+#define SDMA1_SDMA_UTCL1_WR_XNACK1__XNACK_FAULT_ADDR_HI__SHIFT                                                0x0
+#define SDMA1_SDMA_UTCL1_WR_XNACK1__XNACK_FAULT_VMID__SHIFT                                                   0xd
+#define SDMA1_SDMA_UTCL1_WR_XNACK1__XNACK_FAULT_VECTOR__SHIFT                                                 0x11
+#define SDMA1_SDMA_UTCL1_WR_XNACK1__XNACK_NULL_VECTOR__SHIFT                                                  0x14
+#define SDMA1_SDMA_UTCL1_WR_XNACK1__XNACK_TIMEOUT_VECTOR__SHIFT                                               0x17
+#define SDMA1_SDMA_UTCL1_WR_XNACK1__XNACK_FAULT_FLAG__SHIFT                                                   0x1a
+#define SDMA1_SDMA_UTCL1_WR_XNACK1__XNACK_NULL_FLAG__SHIFT                                                    0x1b
+#define SDMA1_SDMA_UTCL1_WR_XNACK1__XNACK_TIMEOUT_FLAG__SHIFT                                                 0x1c
+#define SDMA1_SDMA_UTCL1_WR_XNACK1__XNACK_FAULT_ADDR_HI_MASK                                                  0x00001FFFL
+#define SDMA1_SDMA_UTCL1_WR_XNACK1__XNACK_FAULT_VMID_MASK                                                     0x0001E000L
+#define SDMA1_SDMA_UTCL1_WR_XNACK1__XNACK_FAULT_VECTOR_MASK                                                   0x000E0000L
+#define SDMA1_SDMA_UTCL1_WR_XNACK1__XNACK_NULL_VECTOR_MASK                                                    0x00700000L
+#define SDMA1_SDMA_UTCL1_WR_XNACK1__XNACK_TIMEOUT_VECTOR_MASK                                                 0x03800000L
+#define SDMA1_SDMA_UTCL1_WR_XNACK1__XNACK_FAULT_FLAG_MASK                                                     0x04000000L
+#define SDMA1_SDMA_UTCL1_WR_XNACK1__XNACK_NULL_FLAG_MASK                                                      0x08000000L
+#define SDMA1_SDMA_UTCL1_WR_XNACK1__XNACK_TIMEOUT_FLAG_MASK                                                   0x10000000L
+//SDMA1_SDMA_CHICKEN_BITS_2
+#define SDMA1_SDMA_CHICKEN_BITS_2__MCU_CMD_PROC_DELAY__SHIFT                                                  0x0
+#define SDMA1_SDMA_CHICKEN_BITS_2__MCU_SEND_POSTCODE_EN__SHIFT                                                0x4
+#define SDMA1_SDMA_CHICKEN_BITS_2__MCU_DEBUG_MODE__SHIFT                                                      0x5
+#define SDMA1_SDMA_CHICKEN_BITS_2__RESERVED_7_6__SHIFT                                                        0x6
+#define SDMA1_SDMA_CHICKEN_BITS_2__WPTR_POLL_OUTSTANDING__SHIFT                                               0x8
+#define SDMA1_SDMA_CHICKEN_BITS_2__RESERVED_14_12__SHIFT                                                      0xc
+#define SDMA1_SDMA_CHICKEN_BITS_2__RESERVED_15__SHIFT                                                         0xf
+#define SDMA1_SDMA_CHICKEN_BITS_2__RB_FIFO_WATERMARK__SHIFT                                                   0x10
+#define SDMA1_SDMA_CHICKEN_BITS_2__IB_FIFO_WATERMARK__SHIFT                                                   0x12
+#define SDMA1_SDMA_CHICKEN_BITS_2__CH_WR_WATERMARK_MARGIN__SHIFT                                              0x14
+#define SDMA1_SDMA_CHICKEN_BITS_2__CH_RD_WATERMARK__SHIFT                                                     0x17
+#define SDMA1_SDMA_CHICKEN_BITS_2__CH_WR_WATERMARK__SHIFT                                                     0x19
+#define SDMA1_SDMA_CHICKEN_BITS_2__RESERVED_30__SHIFT                                                         0x1e
+#define SDMA1_SDMA_CHICKEN_BITS_2__PIO_VFID_SOURCE__SHIFT                                                     0x1f
+#define SDMA1_SDMA_CHICKEN_BITS_2__MCU_CMD_PROC_DELAY_MASK                                                    0x0000000FL
+#define SDMA1_SDMA_CHICKEN_BITS_2__MCU_SEND_POSTCODE_EN_MASK                                                  0x00000010L
+#define SDMA1_SDMA_CHICKEN_BITS_2__MCU_DEBUG_MODE_MASK                                                        0x00000020L
+#define SDMA1_SDMA_CHICKEN_BITS_2__RESERVED_7_6_MASK                                                          0x000000C0L
+#define SDMA1_SDMA_CHICKEN_BITS_2__WPTR_POLL_OUTSTANDING_MASK                                                 0x00000F00L
+#define SDMA1_SDMA_CHICKEN_BITS_2__RESERVED_14_12_MASK                                                        0x00007000L
+#define SDMA1_SDMA_CHICKEN_BITS_2__RESERVED_15_MASK                                                           0x00008000L
+#define SDMA1_SDMA_CHICKEN_BITS_2__RB_FIFO_WATERMARK_MASK                                                     0x00030000L
+#define SDMA1_SDMA_CHICKEN_BITS_2__IB_FIFO_WATERMARK_MASK                                                     0x000C0000L
+#define SDMA1_SDMA_CHICKEN_BITS_2__CH_WR_WATERMARK_MARGIN_MASK                                                0x00700000L
+#define SDMA1_SDMA_CHICKEN_BITS_2__CH_RD_WATERMARK_MASK                                                       0x01800000L
+#define SDMA1_SDMA_CHICKEN_BITS_2__CH_WR_WATERMARK_MASK                                                       0x3E000000L
+#define SDMA1_SDMA_CHICKEN_BITS_2__RESERVED_30_MASK                                                           0x40000000L
+#define SDMA1_SDMA_CHICKEN_BITS_2__PIO_VFID_SOURCE_MASK                                                       0x80000000L
+//SDMA1_SDMA_STATUS3_REG
+#define SDMA1_SDMA_STATUS3_REG__PREV_VM_CMD__SHIFT                                                            0x10
+#define SDMA1_SDMA_STATUS3_REG__EXCEPTION_IDLE__SHIFT                                                         0x14
+#define SDMA1_SDMA_STATUS3_REG__AQL_PREV_CMD_IDLE__SHIFT                                                      0x15
+#define SDMA1_SDMA_STATUS3_REG__TLBI_IDLE__SHIFT                                                              0x16
+#define SDMA1_SDMA_STATUS3_REG__GCR_IDLE__SHIFT                                                               0x17
+#define SDMA1_SDMA_STATUS3_REG__INVREQ_IDLE__SHIFT                                                            0x18
+#define SDMA1_SDMA_STATUS3_REG__QUEUE_ID_MATCH__SHIFT                                                         0x19
+#define SDMA1_SDMA_STATUS3_REG__INT_QUEUE_ID__SHIFT                                                           0x1a
+#define SDMA1_SDMA_STATUS3_REG__TMZ_MTYPE_STATUS__SHIFT                                                       0x1e
+#define SDMA1_SDMA_STATUS3_REG__PREV_VM_CMD_MASK                                                              0x000F0000L
+#define SDMA1_SDMA_STATUS3_REG__EXCEPTION_IDLE_MASK                                                           0x00100000L
+#define SDMA1_SDMA_STATUS3_REG__AQL_PREV_CMD_IDLE_MASK                                                        0x00200000L
+#define SDMA1_SDMA_STATUS3_REG__TLBI_IDLE_MASK                                                                0x00400000L
+#define SDMA1_SDMA_STATUS3_REG__GCR_IDLE_MASK                                                                 0x00800000L
+#define SDMA1_SDMA_STATUS3_REG__INVREQ_IDLE_MASK                                                              0x01000000L
+#define SDMA1_SDMA_STATUS3_REG__QUEUE_ID_MATCH_MASK                                                           0x02000000L
+#define SDMA1_SDMA_STATUS3_REG__INT_QUEUE_ID_MASK                                                             0x3C000000L
+#define SDMA1_SDMA_STATUS3_REG__TMZ_MTYPE_STATUS_MASK                                                         0xC0000000L
+//SDMA1_SDMA_GLOBAL_QUANTUM
+#define SDMA1_SDMA_GLOBAL_QUANTUM__GLOBAL_FOCUS_QUANTUM__SHIFT                                                0x0
+#define SDMA1_SDMA_GLOBAL_QUANTUM__GLOBAL_NORMAL_QUANTUM__SHIFT                                               0x8
+#define SDMA1_SDMA_GLOBAL_QUANTUM__GLOBAL_FOCUS_QUANTUM_MASK                                                  0x000000FFL
+#define SDMA1_SDMA_GLOBAL_QUANTUM__GLOBAL_NORMAL_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA1_SDMA_ERROR_LOG
+#define SDMA1_SDMA_ERROR_LOG__OVERRIDE__SHIFT                                                                 0x0
+#define SDMA1_SDMA_ERROR_LOG__STATUS__SHIFT                                                                   0x10
+#define SDMA1_SDMA_ERROR_LOG__OVERRIDE_MASK                                                                   0x0000FFFFL
+#define SDMA1_SDMA_ERROR_LOG__STATUS_MASK                                                                     0xFFFF0000L
+//SDMA1_SDMA_PUB_DUMMY_REG0
+#define SDMA1_SDMA_PUB_DUMMY_REG0__VALUE__SHIFT                                                               0x0
+#define SDMA1_SDMA_PUB_DUMMY_REG0__VALUE_MASK                                                                 0xFFFFFFFFL
+//SDMA1_SDMA_PUB_DUMMY_REG1
+#define SDMA1_SDMA_PUB_DUMMY_REG1__VALUE__SHIFT                                                               0x0
+#define SDMA1_SDMA_PUB_DUMMY_REG1__VALUE_MASK                                                                 0xFFFFFFFFL
+//SDMA1_SDMA_PUB_DUMMY_REG2
+#define SDMA1_SDMA_PUB_DUMMY_REG2__VALUE__SHIFT                                                               0x0
+#define SDMA1_SDMA_PUB_DUMMY_REG2__VALUE_MASK                                                                 0xFFFFFFFFL
+//SDMA1_SDMA_PUB_DUMMY_REG3
+#define SDMA1_SDMA_PUB_DUMMY_REG3__VALUE__SHIFT                                                               0x0
+#define SDMA1_SDMA_PUB_DUMMY_REG3__VALUE_MASK                                                                 0xFFFFFFFFL
+//SDMA1_SDMA_MCU_COUNTER
+#define SDMA1_SDMA_MCU_COUNTER__VALUE__SHIFT                                                                  0x0
+#define SDMA1_SDMA_MCU_COUNTER__VALUE_MASK                                                                    0xFFFFFFFFL
+//SDMA1_SDMA_CRD_CNTL
+#define SDMA1_SDMA_CRD_CNTL__CH_WRREQ_CREDIT__SHIFT                                                           0x13
+#define SDMA1_SDMA_CRD_CNTL__CH_RDREQ_CREDIT__SHIFT                                                           0x19
+#define SDMA1_SDMA_CRD_CNTL__CH_WRREQ_CREDIT_MASK                                                             0x01F80000L
+#define SDMA1_SDMA_CRD_CNTL__CH_RDREQ_CREDIT_MASK                                                             0x7E000000L
+//SDMA1_SDMA_RLC_CGCG_CTRL
+#define SDMA1_SDMA_RLC_CGCG_CTRL__CGCG_INT_ENABLE__SHIFT                                                      0x1
+#define SDMA1_SDMA_RLC_CGCG_CTRL__MCU_CGCG_ALLOW__SHIFT                                                       0x4
+#define SDMA1_SDMA_RLC_CGCG_CTRL__CGCG_IDLE_HYSTERESIS__SHIFT                                                 0x10
+#define SDMA1_SDMA_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK                                                        0x00000002L
+#define SDMA1_SDMA_RLC_CGCG_CTRL__MCU_CGCG_ALLOW_MASK                                                         0x00000010L
+#define SDMA1_SDMA_RLC_CGCG_CTRL__CGCG_IDLE_HYSTERESIS_MASK                                                   0xFFFF0000L
+//SDMA1_SDMA_GPU_IOV_VIOLATION_LOG
+#define SDMA1_SDMA_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT                                             0x0
+#define SDMA1_SDMA_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT                                    0x1
+#define SDMA1_SDMA_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT                                                      0x2
+#define SDMA1_SDMA_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION__SHIFT                                              0x14
+#define SDMA1_SDMA_GPU_IOV_VIOLATION_LOG__VF__SHIFT                                                           0x15
+#define SDMA1_SDMA_GPU_IOV_VIOLATION_LOG__VFID__SHIFT                                                         0x16
+#define SDMA1_SDMA_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK                                               0x00000001L
+#define SDMA1_SDMA_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK                                      0x00000002L
+#define SDMA1_SDMA_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK                                                        0x000FFFFCL
+#define SDMA1_SDMA_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION_MASK                                                0x00100000L
+#define SDMA1_SDMA_GPU_IOV_VIOLATION_LOG__VF_MASK                                                             0x00200000L
+#define SDMA1_SDMA_GPU_IOV_VIOLATION_LOG__VFID_MASK                                                           0x07C00000L
+//SDMA1_SDMA_AQL_STATUS
+#define SDMA1_SDMA_AQL_STATUS__COMPLETE_SIGNAL_EMPTY__SHIFT                                                   0x0
+#define SDMA1_SDMA_AQL_STATUS__INVALID_CMD_EMPTY__SHIFT                                                       0x1
+#define SDMA1_SDMA_AQL_STATUS__COMPLETE_SIGNAL_EMPTY_MASK                                                     0x00000001L
+#define SDMA1_SDMA_AQL_STATUS__INVALID_CMD_EMPTY_MASK                                                         0x00000002L
+//SDMA1_SDMA_TLBI_GCR_CNTL
+#define SDMA1_SDMA_TLBI_GCR_CNTL__TLBI_CMD_DW__SHIFT                                                          0x0
+#define SDMA1_SDMA_TLBI_GCR_CNTL__GCR_CMD_DW__SHIFT                                                           0x4
+#define SDMA1_SDMA_TLBI_GCR_CNTL__TLBI_CREDIT__SHIFT                                                          0x10
+#define SDMA1_SDMA_TLBI_GCR_CNTL__GCR_CREDIT__SHIFT                                                           0x18
+#define SDMA1_SDMA_TLBI_GCR_CNTL__TLBI_CMD_DW_MASK                                                            0x0000000FL
+#define SDMA1_SDMA_TLBI_GCR_CNTL__GCR_CMD_DW_MASK                                                             0x000000F0L
+#define SDMA1_SDMA_TLBI_GCR_CNTL__TLBI_CREDIT_MASK                                                            0x00FF0000L
+#define SDMA1_SDMA_TLBI_GCR_CNTL__GCR_CREDIT_MASK                                                             0xFF000000L
+//SDMA1_SDMA_INT_STATUS
+#define SDMA1_SDMA_INT_STATUS__DATA__SHIFT                                                                    0x0
+#define SDMA1_SDMA_INT_STATUS__DATA_MASK                                                                      0xFFFFFFFFL
+//SDMA1_SDMA_GPU_IOV_VIOLATION_LOG2
+#define SDMA1_SDMA_GPU_IOV_VIOLATION_LOG2__INITIATOR_ID__SHIFT                                                0x0
+#define SDMA1_SDMA_GPU_IOV_VIOLATION_LOG2__INITIATOR_ID_MASK                                                  0x000003FFL
+//SDMA1_SDMA_INVALID_ADDR_LO
+#define SDMA1_SDMA_INVALID_ADDR_LO__VALUE__SHIFT                                                              0x0
+#define SDMA1_SDMA_INVALID_ADDR_LO__VALUE_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_INVALID_ADDR_HI
+#define SDMA1_SDMA_INVALID_ADDR_HI__VALUE__SHIFT                                                              0x0
+#define SDMA1_SDMA_INVALID_ADDR_HI__VALUE_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_INVALID_ADDR_SOURCE
+#define SDMA1_SDMA_INVALID_ADDR_SOURCE__ID__SHIFT                                                             0x0
+#define SDMA1_SDMA_INVALID_ADDR_SOURCE__VMID__SHIFT                                                           0x6
+#define SDMA1_SDMA_INVALID_ADDR_SOURCE__VFID__SHIFT                                                           0xa
+#define SDMA1_SDMA_INVALID_ADDR_SOURCE__VF__SHIFT                                                             0xf
+#define SDMA1_SDMA_INVALID_ADDR_SOURCE__ID_MASK                                                               0x0000003FL
+#define SDMA1_SDMA_INVALID_ADDR_SOURCE__VMID_MASK                                                             0x000003C0L
+#define SDMA1_SDMA_INVALID_ADDR_SOURCE__VFID_MASK                                                             0x00007C00L
+#define SDMA1_SDMA_INVALID_ADDR_SOURCE__VF_MASK                                                               0x00008000L
+//SDMA1_SDMA_CLOCK_GATING_STATUS
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__PTR_MGCG_CLK_STATUS__SHIFT                                            0x8
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__PIO_MGCG_CLK_STATUS__SHIFT                                            0x9
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__MCU_MGCG_CLK_STATUS__SHIFT                                            0xa
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__COPY_ENG_MGCG_CLK_STATUS__SHIFT                                       0xb
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__SERVE_ENG_MGCG_CLK_STATUS__SHIFT                                      0xc
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__CMD_FETCH_MGCG_CLK_STATUS__SHIFT                                      0xd
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__GU_MEMREQ_MGCG_CLK_STATUS__SHIFT                                      0xe
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__INV_MGCG_CLK_STATUS__SHIFT                                            0xf
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__GU_CACHE_MGCG_CLK_STATUS__SHIFT                                       0x10
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__IC_CACHE_MGCG_CLK_STATUS__SHIFT                                       0x11
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__MEM_CHNL_MGCG_CLK_STATUS__SHIFT                                       0x12
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__MEM_CHNL_CESE_MGCG_CLK_STATUS__SHIFT                                  0x13
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__FE_ENG_MGCG_CLK_STATUS__SHIFT                                         0x14
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__MEM_CHNL_CESEFE_MGCG_CLK_STATUS__SHIFT                                0x15
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__MEM_CHNL_FE_WARMUP_MGCG_CLK_STATUS__SHIFT                             0x16
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__PTR_MGCG_CLK_STATUS_MASK                                              0x00000100L
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__PIO_MGCG_CLK_STATUS_MASK                                              0x00000200L
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__MCU_MGCG_CLK_STATUS_MASK                                              0x00000400L
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__COPY_ENG_MGCG_CLK_STATUS_MASK                                         0x00000800L
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__SERVE_ENG_MGCG_CLK_STATUS_MASK                                        0x00001000L
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__CMD_FETCH_MGCG_CLK_STATUS_MASK                                        0x00002000L
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__GU_MEMREQ_MGCG_CLK_STATUS_MASK                                        0x00004000L
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__INV_MGCG_CLK_STATUS_MASK                                              0x00008000L
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__GU_CACHE_MGCG_CLK_STATUS_MASK                                         0x00010000L
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__IC_CACHE_MGCG_CLK_STATUS_MASK                                         0x00020000L
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__MEM_CHNL_MGCG_CLK_STATUS_MASK                                         0x00040000L
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__MEM_CHNL_CESE_MGCG_CLK_STATUS_MASK                                    0x00080000L
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__FE_ENG_MGCG_CLK_STATUS_MASK                                           0x00100000L
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__MEM_CHNL_CESEFE_MGCG_CLK_STATUS_MASK                                  0x00200000L
+#define SDMA1_SDMA_CLOCK_GATING_STATUS__MEM_CHNL_FE_WARMUP_MGCG_CLK_STATUS_MASK                               0x00400000L
+//SDMA1_SDMA_STATUS4_REG
+#define SDMA1_SDMA_STATUS4_REG__IDLE__SHIFT                                                                   0x0
+#define SDMA1_SDMA_STATUS4_REG__IH_OUTSTANDING__SHIFT                                                         0x2
+#define SDMA1_SDMA_STATUS4_REG__RESERVED__SHIFT                                                               0x3
+#define SDMA1_SDMA_STATUS4_REG__CH_RD_OUTSTANDING__SHIFT                                                      0x4
+#define SDMA1_SDMA_STATUS4_REG__CH_WR_OUTSTANDING__SHIFT                                                      0x5
+#define SDMA1_SDMA_STATUS4_REG__GCR_OUTSTANDING__SHIFT                                                        0x6
+#define SDMA1_SDMA_STATUS4_REG__TLBI_OUTSTANDING__SHIFT                                                       0x7
+#define SDMA1_SDMA_STATUS4_REG__UTCL2_RD_OUTSTANDING__SHIFT                                                   0x8
+#define SDMA1_SDMA_STATUS4_REG__UTCL2_WR_OUTSTANDING__SHIFT                                                   0x9
+#define SDMA1_SDMA_STATUS4_REG__CMDFETCH_OUTSTANDING__SHIFT                                                   0xa
+#define SDMA1_SDMA_STATUS4_REG__RESERVED_11__SHIFT                                                            0xb
+#define SDMA1_SDMA_STATUS4_REG__RESERVED_13_12__SHIFT                                                         0xc
+#define SDMA1_SDMA_STATUS4_REG__RESERVED_15_14__SHIFT                                                         0xe
+#define SDMA1_SDMA_STATUS4_REG__ACTIVE_QUEUE_ID__SHIFT                                                        0x10
+#define SDMA1_SDMA_STATUS4_REG__SRIOV_WATING_RLCV_CMD__SHIFT                                                  0x14
+#define SDMA1_SDMA_STATUS4_REG__SRIOV_SDMA_EXECUTING_CMD__SHIFT                                               0x15
+#define SDMA1_SDMA_STATUS4_REG__UTCL2_RD_XNACK_FAULT__SHIFT                                                   0x16
+#define SDMA1_SDMA_STATUS4_REG__UTCL2_RD_XNACK_NULL__SHIFT                                                    0x17
+#define SDMA1_SDMA_STATUS4_REG__UTCL2_RD_XNACK_TIMEOUT__SHIFT                                                 0x18
+#define SDMA1_SDMA_STATUS4_REG__UTCL2_WR_XNACK_FAULT__SHIFT                                                   0x19
+#define SDMA1_SDMA_STATUS4_REG__UTCL2_WR_XNACK_NULL__SHIFT                                                    0x1a
+#define SDMA1_SDMA_STATUS4_REG__UTCL2_WR_XNACK_TIMEOUT__SHIFT                                                 0x1b
+#define SDMA1_SDMA_STATUS4_REG__IDLE_MASK                                                                     0x00000001L
+#define SDMA1_SDMA_STATUS4_REG__IH_OUTSTANDING_MASK                                                           0x00000004L
+#define SDMA1_SDMA_STATUS4_REG__RESERVED_MASK                                                                 0x00000008L
+#define SDMA1_SDMA_STATUS4_REG__CH_RD_OUTSTANDING_MASK                                                        0x00000010L
+#define SDMA1_SDMA_STATUS4_REG__CH_WR_OUTSTANDING_MASK                                                        0x00000020L
+#define SDMA1_SDMA_STATUS4_REG__GCR_OUTSTANDING_MASK                                                          0x00000040L
+#define SDMA1_SDMA_STATUS4_REG__TLBI_OUTSTANDING_MASK                                                         0x00000080L
+#define SDMA1_SDMA_STATUS4_REG__UTCL2_RD_OUTSTANDING_MASK                                                     0x00000100L
+#define SDMA1_SDMA_STATUS4_REG__UTCL2_WR_OUTSTANDING_MASK                                                     0x00000200L
+#define SDMA1_SDMA_STATUS4_REG__CMDFETCH_OUTSTANDING_MASK                                                     0x00000400L
+#define SDMA1_SDMA_STATUS4_REG__RESERVED_11_MASK                                                              0x00000800L
+#define SDMA1_SDMA_STATUS4_REG__RESERVED_13_12_MASK                                                           0x00003000L
+#define SDMA1_SDMA_STATUS4_REG__RESERVED_15_14_MASK                                                           0x0000C000L
+#define SDMA1_SDMA_STATUS4_REG__ACTIVE_QUEUE_ID_MASK                                                          0x000F0000L
+#define SDMA1_SDMA_STATUS4_REG__SRIOV_WATING_RLCV_CMD_MASK                                                    0x00100000L
+#define SDMA1_SDMA_STATUS4_REG__SRIOV_SDMA_EXECUTING_CMD_MASK                                                 0x00200000L
+#define SDMA1_SDMA_STATUS4_REG__UTCL2_RD_XNACK_FAULT_MASK                                                     0x00400000L
+#define SDMA1_SDMA_STATUS4_REG__UTCL2_RD_XNACK_NULL_MASK                                                      0x00800000L
+#define SDMA1_SDMA_STATUS4_REG__UTCL2_RD_XNACK_TIMEOUT_MASK                                                   0x01000000L
+#define SDMA1_SDMA_STATUS4_REG__UTCL2_WR_XNACK_FAULT_MASK                                                     0x02000000L
+#define SDMA1_SDMA_STATUS4_REG__UTCL2_WR_XNACK_NULL_MASK                                                      0x04000000L
+#define SDMA1_SDMA_STATUS4_REG__UTCL2_WR_XNACK_TIMEOUT_MASK                                                   0x08000000L
+//SDMA1_SDMA_SCRATCH_RAM_DATA
+#define SDMA1_SDMA_SCRATCH_RAM_DATA__DATA__SHIFT                                                              0x0
+#define SDMA1_SDMA_SCRATCH_RAM_DATA__DATA_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_SCRATCH_RAM_ADDR
+#define SDMA1_SDMA_SCRATCH_RAM_ADDR__ADDR__SHIFT                                                              0x0
+#define SDMA1_SDMA_SCRATCH_RAM_ADDR__ADDR_MASK                                                                0x0000007FL
+//SDMA1_SDMA_TIMESTAMP_CNTL
+#define SDMA1_SDMA_TIMESTAMP_CNTL__CAPTURE__SHIFT                                                             0x0
+#define SDMA1_SDMA_TIMESTAMP_CNTL__CAPTURE_MASK                                                               0x00000001L
+//SDMA1_SDMA_STATUS5_REG
+#define SDMA1_SDMA_STATUS5_REG__QUEUE0_RB_ENABLE_STATUS__SHIFT                                                0x0
+#define SDMA1_SDMA_STATUS5_REG__QUEUE1_RB_ENABLE_STATUS__SHIFT                                                0x1
+#define SDMA1_SDMA_STATUS5_REG__QUEUE2_RB_ENABLE_STATUS__SHIFT                                                0x2
+#define SDMA1_SDMA_STATUS5_REG__QUEUE3_RB_ENABLE_STATUS__SHIFT                                                0x3
+#define SDMA1_SDMA_STATUS5_REG__QUEUE4_RB_ENABLE_STATUS__SHIFT                                                0x4
+#define SDMA1_SDMA_STATUS5_REG__QUEUE5_RB_ENABLE_STATUS__SHIFT                                                0x5
+#define SDMA1_SDMA_STATUS5_REG__QUEUE6_RB_ENABLE_STATUS__SHIFT                                                0x6
+#define SDMA1_SDMA_STATUS5_REG__QUEUE7_RB_ENABLE_STATUS__SHIFT                                                0x7
+#define SDMA1_SDMA_STATUS5_REG__QUEUE8_RB_ENABLE_STATUS__SHIFT                                                0x8
+#define SDMA1_SDMA_STATUS5_REG__QUEUE9_RB_ENABLE_STATUS__SHIFT                                                0x9
+#define SDMA1_SDMA_STATUS5_REG__QUEUE0_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x10
+#define SDMA1_SDMA_STATUS5_REG__QUEUE1_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x11
+#define SDMA1_SDMA_STATUS5_REG__QUEUE2_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x12
+#define SDMA1_SDMA_STATUS5_REG__QUEUE3_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x13
+#define SDMA1_SDMA_STATUS5_REG__QUEUE4_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x14
+#define SDMA1_SDMA_STATUS5_REG__QUEUE5_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x15
+#define SDMA1_SDMA_STATUS5_REG__QUEUE6_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x16
+#define SDMA1_SDMA_STATUS5_REG__QUEUE7_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x17
+#define SDMA1_SDMA_STATUS5_REG__QUEUE8_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x18
+#define SDMA1_SDMA_STATUS5_REG__QUEUE9_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                        0x19
+#define SDMA1_SDMA_STATUS5_REG__QUEUE0_RB_ENABLE_STATUS_MASK                                                  0x00000001L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE1_RB_ENABLE_STATUS_MASK                                                  0x00000002L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE2_RB_ENABLE_STATUS_MASK                                                  0x00000004L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE3_RB_ENABLE_STATUS_MASK                                                  0x00000008L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE4_RB_ENABLE_STATUS_MASK                                                  0x00000010L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE5_RB_ENABLE_STATUS_MASK                                                  0x00000020L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE6_RB_ENABLE_STATUS_MASK                                                  0x00000040L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE7_RB_ENABLE_STATUS_MASK                                                  0x00000080L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE8_RB_ENABLE_STATUS_MASK                                                  0x00000100L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE9_RB_ENABLE_STATUS_MASK                                                  0x00000200L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE0_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x00010000L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE1_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x00020000L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE2_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x00040000L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE3_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x00080000L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE4_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x00100000L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE5_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x00200000L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE6_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x00400000L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE7_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x00800000L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE8_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x01000000L
+#define SDMA1_SDMA_STATUS5_REG__QUEUE9_WPTR_POLL_PAGE_EXCEPTION_MASK                                          0x02000000L
+//SDMA1_SDMA_QUEUE_RESET_REQ
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE0_RESET__SHIFT                                                       0x0
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE1_RESET__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE2_RESET__SHIFT                                                       0x2
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE3_RESET__SHIFT                                                       0x3
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE4_RESET__SHIFT                                                       0x4
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE5_RESET__SHIFT                                                       0x5
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE6_RESET__SHIFT                                                       0x6
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE7_RESET__SHIFT                                                       0x7
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE8_RESET__SHIFT                                                       0x8
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE9_RESET__SHIFT                                                       0x9
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE0_RESET_MASK                                                         0x00000001L
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE1_RESET_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE2_RESET_MASK                                                         0x00000004L
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE3_RESET_MASK                                                         0x00000008L
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE4_RESET_MASK                                                         0x00000010L
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE5_RESET_MASK                                                         0x00000020L
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE6_RESET_MASK                                                         0x00000040L
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE7_RESET_MASK                                                         0x00000080L
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE8_RESET_MASK                                                         0x00000100L
+#define SDMA1_SDMA_QUEUE_RESET_REQ__QUEUE9_RESET_MASK                                                         0x00000200L
+//SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE0_DEQUEUE_REQ__SHIFT                                           0x0
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE1_DEQUEUE_REQ__SHIFT                                           0x1
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE2_DEQUEUE_REQ__SHIFT                                           0x2
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE3_DEQUEUE_REQ__SHIFT                                           0x3
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE4_DEQUEUE_REQ__SHIFT                                           0x4
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE5_DEQUEUE_REQ__SHIFT                                           0x5
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE6_DEQUEUE_REQ__SHIFT                                           0x6
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE7_DEQUEUE_REQ__SHIFT                                           0x7
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE8_DEQUEUE_REQ__SHIFT                                           0x8
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE9_DEQUEUE_REQ__SHIFT                                           0x9
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE0_DEQUEUE_REQ_MASK                                             0x00000001L
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE1_DEQUEUE_REQ_MASK                                             0x00000002L
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE2_DEQUEUE_REQ_MASK                                             0x00000004L
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE3_DEQUEUE_REQ_MASK                                             0x00000008L
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE4_DEQUEUE_REQ_MASK                                             0x00000010L
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE5_DEQUEUE_REQ_MASK                                             0x00000020L
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE6_DEQUEUE_REQ_MASK                                             0x00000040L
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE7_DEQUEUE_REQ_MASK                                             0x00000080L
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE8_DEQUEUE_REQ_MASK                                             0x00000100L
+#define SDMA1_SDMA_QUEUE_DEQUEUE_REQUEST__QUEUE9_DEQUEUE_REQ_MASK                                             0x00000200L
+//SDMA1_SDMA_STATUS6_REG
+#define SDMA1_SDMA_STATUS6_REG__ID__SHIFT                                                                     0x0
+#define SDMA1_SDMA_STATUS6_REG__TH1MCU_INSTR_PTR__SHIFT                                                       0x2
+#define SDMA1_SDMA_STATUS6_REG__TH1_EXCEPTION__SHIFT                                                          0x10
+#define SDMA1_SDMA_STATUS6_REG__ID_MASK                                                                       0x00000003L
+#define SDMA1_SDMA_STATUS6_REG__TH1MCU_INSTR_PTR_MASK                                                         0x0000FFFCL
+#define SDMA1_SDMA_STATUS6_REG__TH1_EXCEPTION_MASK                                                            0xFFFF0000L
+//SDMA1_SDMA_STATUS7_REG
+#define SDMA1_SDMA_STATUS7_REG__BLT_REQ_DROP__SHIFT                                                           0x0
+#define SDMA1_SDMA_STATUS7_REG__BLT_REQ_DROP_MASK                                                             0x00000001L
+//SDMA1_SDMA_STATUS8_REG
+#define SDMA1_SDMA_STATUS8_REG__LD_CTXSW_COND__SHIFT                                                          0x0
+#define SDMA1_SDMA_STATUS8_REG__LD_CTXSW_COND_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_CE_CTRL
+#define SDMA1_SDMA_CE_CTRL__RD_LUT_WATERMARK__SHIFT                                                           0x0
+#define SDMA1_SDMA_CE_CTRL__RD_LUT_DEPTH__SHIFT                                                               0x3
+#define SDMA1_SDMA_CE_CTRL__WR_AFIFO_WATERMARK__SHIFT                                                         0x5
+#define SDMA1_SDMA_CE_CTRL__RESERVED__SHIFT                                                                   0x9
+#define SDMA1_SDMA_CE_CTRL__RD_LUT_WATERMARK_MASK                                                             0x00000007L
+#define SDMA1_SDMA_CE_CTRL__RD_LUT_DEPTH_MASK                                                                 0x00000018L
+#define SDMA1_SDMA_CE_CTRL__WR_AFIFO_WATERMARK_MASK                                                           0x000000E0L
+#define SDMA1_SDMA_CE_CTRL__RESERVED_MASK                                                                     0xFFFFFE00L
+//SDMA1_SDMA_RAS_STATUS
+#define SDMA1_SDMA_RAS_STATUS__RB_FETCH_ECC__SHIFT                                                            0x0
+#define SDMA1_SDMA_RAS_STATUS__IB_FETCH_ECC__SHIFT                                                            0x1
+#define SDMA1_SDMA_RAS_STATUS__MCU_DATA_ECC__SHIFT                                                            0x2
+#define SDMA1_SDMA_RAS_STATUS__WPTR_POLL_ECC__SHIFT                                                           0x3
+#define SDMA1_SDMA_RAS_STATUS__COPY_DATA_ECC__SHIFT                                                           0x4
+#define SDMA1_SDMA_RAS_STATUS__INSTR_FETCH_ECC__SHIFT                                                         0x5
+#define SDMA1_SDMA_RAS_STATUS__ATOMIC_ECC__SHIFT                                                              0x6
+#define SDMA1_SDMA_RAS_STATUS__FRONTEND_ECC__SHIFT                                                            0x7
+#define SDMA1_SDMA_RAS_STATUS__SRAM_ECC__SHIFT                                                                0x10
+#define SDMA1_SDMA_RAS_STATUS__BUS_PARITY__SHIFT                                                              0x12
+#define SDMA1_SDMA_RAS_STATUS__VMID__SHIFT                                                                    0x14
+#define SDMA1_SDMA_RAS_STATUS__VFID__SHIFT                                                                    0x18
+#define SDMA1_SDMA_RAS_STATUS__VF__SHIFT                                                                      0x1d
+#define SDMA1_SDMA_RAS_STATUS__RB_FETCH_ECC_MASK                                                              0x00000001L
+#define SDMA1_SDMA_RAS_STATUS__IB_FETCH_ECC_MASK                                                              0x00000002L
+#define SDMA1_SDMA_RAS_STATUS__MCU_DATA_ECC_MASK                                                              0x00000004L
+#define SDMA1_SDMA_RAS_STATUS__WPTR_POLL_ECC_MASK                                                             0x00000008L
+#define SDMA1_SDMA_RAS_STATUS__COPY_DATA_ECC_MASK                                                             0x00000010L
+#define SDMA1_SDMA_RAS_STATUS__INSTR_FETCH_ECC_MASK                                                           0x00000020L
+#define SDMA1_SDMA_RAS_STATUS__ATOMIC_ECC_MASK                                                                0x00000040L
+#define SDMA1_SDMA_RAS_STATUS__FRONTEND_ECC_MASK                                                              0x00000080L
+#define SDMA1_SDMA_RAS_STATUS__SRAM_ECC_MASK                                                                  0x00010000L
+#define SDMA1_SDMA_RAS_STATUS__BUS_PARITY_MASK                                                                0x00040000L
+#define SDMA1_SDMA_RAS_STATUS__VMID_MASK                                                                      0x00F00000L
+#define SDMA1_SDMA_RAS_STATUS__VFID_MASK                                                                      0x1F000000L
+#define SDMA1_SDMA_RAS_STATUS__VF_MASK                                                                        0x20000000L
+//SDMA1_SDMA_POISON_INFO
+#define SDMA1_SDMA_POISON_INFO__VMID__SHIFT                                                                   0x0
+#define SDMA1_SDMA_POISON_INFO__VFID__SHIFT                                                                   0x4
+#define SDMA1_SDMA_POISON_INFO__VF__SHIFT                                                                     0x9
+#define SDMA1_SDMA_POISON_INFO__VMID_MASK                                                                     0x0000000FL
+#define SDMA1_SDMA_POISON_INFO__VFID_MASK                                                                     0x000001F0L
+#define SDMA1_SDMA_POISON_INFO__VF_MASK                                                                       0x00000200L
+//SDMA1_SDMA_MEMHUB_CNTL
+#define SDMA1_SDMA_MEMHUB_CNTL__SPECULATIVE_DATA_READ__SHIFT                                                  0x0
+#define SDMA1_SDMA_MEMHUB_CNTL__SPECULATIVE_DATA_READ_MASK                                                    0x00000003L
+//SDMA1_SDMA_QUEUE0_RB_CNTL
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA1_SDMA_QUEUE0_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA1_SDMA_QUEUE0_RB_BASE
+#define SDMA1_SDMA_QUEUE0_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE0_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_RB_BASE_HI
+#define SDMA1_SDMA_QUEUE0_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE0_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA1_SDMA_QUEUE0_RB_RPTR
+#define SDMA1_SDMA_QUEUE0_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE0_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_RB_RPTR_HI
+#define SDMA1_SDMA_QUEUE0_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE0_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_RB_WPTR
+#define SDMA1_SDMA_QUEUE0_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE0_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_RB_WPTR_HI
+#define SDMA1_SDMA_QUEUE0_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE0_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_RB_RPTR_ADDR_LO
+#define SDMA1_SDMA_QUEUE0_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_SDMA_QUEUE0_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE0_RB_RPTR_ADDR_HI
+#define SDMA1_SDMA_QUEUE0_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE0_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_IB_CNTL
+#define SDMA1_SDMA_QUEUE0_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE0_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA1_SDMA_QUEUE0_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE0_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA1_SDMA_QUEUE0_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA1_SDMA_QUEUE0_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE0_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA1_SDMA_QUEUE0_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE0_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA1_SDMA_QUEUE0_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA1_SDMA_QUEUE0_IB_RPTR
+#define SDMA1_SDMA_QUEUE0_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA1_SDMA_QUEUE0_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA1_SDMA_QUEUE0_IB_OFFSET
+#define SDMA1_SDMA_QUEUE0_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE0_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA1_SDMA_QUEUE0_IB_BASE_LO
+#define SDMA1_SDMA_QUEUE0_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_SDMA_QUEUE0_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE0_IB_BASE_HI
+#define SDMA1_SDMA_QUEUE0_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE0_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_IB_SIZE
+#define SDMA1_SDMA_QUEUE0_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE0_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA1_SDMA_QUEUE0_DOORBELL
+#define SDMA1_SDMA_QUEUE0_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA1_SDMA_QUEUE0_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA1_SDMA_QUEUE0_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA1_SDMA_QUEUE0_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA1_SDMA_QUEUE0_DOORBELL_LOG
+#define SDMA1_SDMA_QUEUE0_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA1_SDMA_QUEUE0_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA1_SDMA_QUEUE0_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA1_SDMA_QUEUE0_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE0_DOORBELL_OFFSET
+#define SDMA1_SDMA_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA1_SDMA_QUEUE0_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA1_SDMA_QUEUE0_CSA_ADDR_LO
+#define SDMA1_SDMA_QUEUE0_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE0_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE0_CSA_ADDR_HI
+#define SDMA1_SDMA_QUEUE0_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE0_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_SCHEDULE_CNTL
+#define SDMA1_SDMA_QUEUE0_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE0_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA1_SDMA_QUEUE0_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA1_SDMA_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA1_SDMA_QUEUE0_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA1_SDMA_QUEUE0_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA1_SDMA_QUEUE0_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA1_SDMA_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA1_SDMA_QUEUE0_IB_SUB_REMAIN
+#define SDMA1_SDMA_QUEUE0_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE0_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA1_SDMA_QUEUE0_PREEMPT
+#define SDMA1_SDMA_QUEUE0_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE0_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA1_SDMA_QUEUE0_DUMMY_REG
+#define SDMA1_SDMA_QUEUE0_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE0_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA1_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA1_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_RB_AQL_CNTL
+#define SDMA1_SDMA_QUEUE0_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE0_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA1_SDMA_QUEUE0_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA1_SDMA_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA1_SDMA_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA1_SDMA_QUEUE0_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA1_SDMA_QUEUE0_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE0_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA1_SDMA_QUEUE0_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA1_SDMA_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA1_SDMA_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA1_SDMA_QUEUE0_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA1_SDMA_QUEUE0_MINOR_PTR_UPDATE
+#define SDMA1_SDMA_QUEUE0_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE0_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA1_SDMA_QUEUE0_MIDCMD_CNTL
+#define SDMA1_SDMA_QUEUE0_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE0_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE0_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA1_SDMA_QUEUE0_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA1_SDMA_QUEUE0_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE0_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE0_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA1_SDMA_QUEUE0_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA1_SDMA_QUEUE0_MIDCMD_DATA0
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_MIDCMD_DATA1
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_MIDCMD_DATA2
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_MIDCMD_DATA3
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_MIDCMD_DATA4
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_MIDCMD_DATA5
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_MIDCMD_DATA6
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_MIDCMD_DATA7
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_MIDCMD_DATA8
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_MIDCMD_DATA9
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_MIDCMD_DATA10
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE0_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_UTILIZATION_LO
+#define SDMA1_SDMA_QUEUE0_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE0_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_UTILIZATION_HI
+#define SDMA1_SDMA_QUEUE0_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE0_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_WAIT_UNSATISFIED_THD
+#define SDMA1_SDMA_QUEUE0_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA1_SDMA_QUEUE0_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA1_SDMA_QUEUE0_MQD_BASE_ADDR_LO
+#define SDMA1_SDMA_QUEUE0_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA1_SDMA_QUEUE0_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE0_MQD_BASE_ADDR_HI
+#define SDMA1_SDMA_QUEUE0_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA1_SDMA_QUEUE0_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_MQD_CONTROL
+#define SDMA1_SDMA_QUEUE0_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE0_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA1_SDMA_QUEUE0_CONTEXT_STATUS
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA1_SDMA_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA1_SDMA_QUEUE0_DUMMY_REG1
+#define SDMA1_SDMA_QUEUE0_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE0_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE0_DUMMY_REG2
+#define SDMA1_SDMA_QUEUE0_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE0_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_RB_CNTL
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA1_SDMA_QUEUE1_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA1_SDMA_QUEUE1_RB_BASE
+#define SDMA1_SDMA_QUEUE1_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE1_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_RB_BASE_HI
+#define SDMA1_SDMA_QUEUE1_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE1_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA1_SDMA_QUEUE1_RB_RPTR
+#define SDMA1_SDMA_QUEUE1_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE1_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_RB_RPTR_HI
+#define SDMA1_SDMA_QUEUE1_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE1_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_RB_WPTR
+#define SDMA1_SDMA_QUEUE1_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE1_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_RB_WPTR_HI
+#define SDMA1_SDMA_QUEUE1_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE1_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_RB_RPTR_ADDR_LO
+#define SDMA1_SDMA_QUEUE1_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_SDMA_QUEUE1_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE1_RB_RPTR_ADDR_HI
+#define SDMA1_SDMA_QUEUE1_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE1_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_IB_CNTL
+#define SDMA1_SDMA_QUEUE1_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE1_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA1_SDMA_QUEUE1_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE1_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA1_SDMA_QUEUE1_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA1_SDMA_QUEUE1_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE1_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA1_SDMA_QUEUE1_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE1_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA1_SDMA_QUEUE1_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA1_SDMA_QUEUE1_IB_RPTR
+#define SDMA1_SDMA_QUEUE1_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA1_SDMA_QUEUE1_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA1_SDMA_QUEUE1_IB_OFFSET
+#define SDMA1_SDMA_QUEUE1_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE1_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA1_SDMA_QUEUE1_IB_BASE_LO
+#define SDMA1_SDMA_QUEUE1_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_SDMA_QUEUE1_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE1_IB_BASE_HI
+#define SDMA1_SDMA_QUEUE1_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE1_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_IB_SIZE
+#define SDMA1_SDMA_QUEUE1_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE1_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA1_SDMA_QUEUE1_DOORBELL
+#define SDMA1_SDMA_QUEUE1_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA1_SDMA_QUEUE1_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA1_SDMA_QUEUE1_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA1_SDMA_QUEUE1_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA1_SDMA_QUEUE1_DOORBELL_LOG
+#define SDMA1_SDMA_QUEUE1_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA1_SDMA_QUEUE1_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA1_SDMA_QUEUE1_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA1_SDMA_QUEUE1_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE1_DOORBELL_OFFSET
+#define SDMA1_SDMA_QUEUE1_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA1_SDMA_QUEUE1_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA1_SDMA_QUEUE1_CSA_ADDR_LO
+#define SDMA1_SDMA_QUEUE1_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE1_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE1_CSA_ADDR_HI
+#define SDMA1_SDMA_QUEUE1_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE1_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_SCHEDULE_CNTL
+#define SDMA1_SDMA_QUEUE1_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE1_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA1_SDMA_QUEUE1_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA1_SDMA_QUEUE1_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA1_SDMA_QUEUE1_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA1_SDMA_QUEUE1_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA1_SDMA_QUEUE1_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA1_SDMA_QUEUE1_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA1_SDMA_QUEUE1_IB_SUB_REMAIN
+#define SDMA1_SDMA_QUEUE1_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE1_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA1_SDMA_QUEUE1_PREEMPT
+#define SDMA1_SDMA_QUEUE1_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE1_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA1_SDMA_QUEUE1_DUMMY_REG
+#define SDMA1_SDMA_QUEUE1_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE1_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA1_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA1_SDMA_QUEUE1_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_RB_AQL_CNTL
+#define SDMA1_SDMA_QUEUE1_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE1_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA1_SDMA_QUEUE1_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA1_SDMA_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA1_SDMA_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA1_SDMA_QUEUE1_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA1_SDMA_QUEUE1_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE1_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA1_SDMA_QUEUE1_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA1_SDMA_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA1_SDMA_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA1_SDMA_QUEUE1_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA1_SDMA_QUEUE1_MINOR_PTR_UPDATE
+#define SDMA1_SDMA_QUEUE1_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE1_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA1_SDMA_QUEUE1_MIDCMD_CNTL
+#define SDMA1_SDMA_QUEUE1_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE1_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE1_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA1_SDMA_QUEUE1_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA1_SDMA_QUEUE1_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE1_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE1_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA1_SDMA_QUEUE1_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA1_SDMA_QUEUE1_MIDCMD_DATA0
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_MIDCMD_DATA1
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_MIDCMD_DATA2
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_MIDCMD_DATA3
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_MIDCMD_DATA4
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_MIDCMD_DATA5
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_MIDCMD_DATA6
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_MIDCMD_DATA7
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_MIDCMD_DATA8
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_MIDCMD_DATA9
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_MIDCMD_DATA10
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE1_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_UTILIZATION_LO
+#define SDMA1_SDMA_QUEUE1_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE1_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_UTILIZATION_HI
+#define SDMA1_SDMA_QUEUE1_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE1_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_WAIT_UNSATISFIED_THD
+#define SDMA1_SDMA_QUEUE1_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA1_SDMA_QUEUE1_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA1_SDMA_QUEUE1_MQD_BASE_ADDR_LO
+#define SDMA1_SDMA_QUEUE1_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA1_SDMA_QUEUE1_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE1_MQD_BASE_ADDR_HI
+#define SDMA1_SDMA_QUEUE1_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA1_SDMA_QUEUE1_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_MQD_CONTROL
+#define SDMA1_SDMA_QUEUE1_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE1_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA1_SDMA_QUEUE1_CONTEXT_STATUS
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA1_SDMA_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA1_SDMA_QUEUE1_DUMMY_REG1
+#define SDMA1_SDMA_QUEUE1_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE1_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE1_DUMMY_REG2
+#define SDMA1_SDMA_QUEUE1_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE1_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_RB_CNTL
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA1_SDMA_QUEUE2_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA1_SDMA_QUEUE2_RB_BASE
+#define SDMA1_SDMA_QUEUE2_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE2_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_RB_BASE_HI
+#define SDMA1_SDMA_QUEUE2_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE2_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA1_SDMA_QUEUE2_RB_RPTR
+#define SDMA1_SDMA_QUEUE2_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE2_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_RB_RPTR_HI
+#define SDMA1_SDMA_QUEUE2_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE2_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_RB_WPTR
+#define SDMA1_SDMA_QUEUE2_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE2_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_RB_WPTR_HI
+#define SDMA1_SDMA_QUEUE2_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE2_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_RB_RPTR_ADDR_LO
+#define SDMA1_SDMA_QUEUE2_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_SDMA_QUEUE2_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE2_RB_RPTR_ADDR_HI
+#define SDMA1_SDMA_QUEUE2_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE2_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_IB_CNTL
+#define SDMA1_SDMA_QUEUE2_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE2_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA1_SDMA_QUEUE2_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE2_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA1_SDMA_QUEUE2_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA1_SDMA_QUEUE2_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE2_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA1_SDMA_QUEUE2_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE2_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA1_SDMA_QUEUE2_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA1_SDMA_QUEUE2_IB_RPTR
+#define SDMA1_SDMA_QUEUE2_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA1_SDMA_QUEUE2_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA1_SDMA_QUEUE2_IB_OFFSET
+#define SDMA1_SDMA_QUEUE2_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE2_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA1_SDMA_QUEUE2_IB_BASE_LO
+#define SDMA1_SDMA_QUEUE2_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_SDMA_QUEUE2_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE2_IB_BASE_HI
+#define SDMA1_SDMA_QUEUE2_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE2_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_IB_SIZE
+#define SDMA1_SDMA_QUEUE2_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE2_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA1_SDMA_QUEUE2_DOORBELL
+#define SDMA1_SDMA_QUEUE2_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA1_SDMA_QUEUE2_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA1_SDMA_QUEUE2_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA1_SDMA_QUEUE2_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA1_SDMA_QUEUE2_DOORBELL_LOG
+#define SDMA1_SDMA_QUEUE2_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA1_SDMA_QUEUE2_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA1_SDMA_QUEUE2_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA1_SDMA_QUEUE2_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE2_DOORBELL_OFFSET
+#define SDMA1_SDMA_QUEUE2_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA1_SDMA_QUEUE2_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA1_SDMA_QUEUE2_CSA_ADDR_LO
+#define SDMA1_SDMA_QUEUE2_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE2_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE2_CSA_ADDR_HI
+#define SDMA1_SDMA_QUEUE2_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE2_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_SCHEDULE_CNTL
+#define SDMA1_SDMA_QUEUE2_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE2_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA1_SDMA_QUEUE2_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA1_SDMA_QUEUE2_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA1_SDMA_QUEUE2_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA1_SDMA_QUEUE2_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA1_SDMA_QUEUE2_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA1_SDMA_QUEUE2_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA1_SDMA_QUEUE2_IB_SUB_REMAIN
+#define SDMA1_SDMA_QUEUE2_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE2_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA1_SDMA_QUEUE2_PREEMPT
+#define SDMA1_SDMA_QUEUE2_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE2_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA1_SDMA_QUEUE2_DUMMY_REG
+#define SDMA1_SDMA_QUEUE2_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE2_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA1_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA1_SDMA_QUEUE2_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_RB_AQL_CNTL
+#define SDMA1_SDMA_QUEUE2_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE2_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA1_SDMA_QUEUE2_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA1_SDMA_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA1_SDMA_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA1_SDMA_QUEUE2_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA1_SDMA_QUEUE2_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE2_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA1_SDMA_QUEUE2_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA1_SDMA_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA1_SDMA_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA1_SDMA_QUEUE2_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA1_SDMA_QUEUE2_MINOR_PTR_UPDATE
+#define SDMA1_SDMA_QUEUE2_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE2_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA1_SDMA_QUEUE2_MIDCMD_CNTL
+#define SDMA1_SDMA_QUEUE2_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE2_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE2_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA1_SDMA_QUEUE2_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA1_SDMA_QUEUE2_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE2_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE2_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA1_SDMA_QUEUE2_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA1_SDMA_QUEUE2_MIDCMD_DATA0
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_MIDCMD_DATA1
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_MIDCMD_DATA2
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_MIDCMD_DATA3
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_MIDCMD_DATA4
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_MIDCMD_DATA5
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_MIDCMD_DATA6
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_MIDCMD_DATA7
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_MIDCMD_DATA8
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_MIDCMD_DATA9
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_MIDCMD_DATA10
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE2_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_UTILIZATION_LO
+#define SDMA1_SDMA_QUEUE2_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE2_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_UTILIZATION_HI
+#define SDMA1_SDMA_QUEUE2_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE2_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_WAIT_UNSATISFIED_THD
+#define SDMA1_SDMA_QUEUE2_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA1_SDMA_QUEUE2_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA1_SDMA_QUEUE2_MQD_BASE_ADDR_LO
+#define SDMA1_SDMA_QUEUE2_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA1_SDMA_QUEUE2_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE2_MQD_BASE_ADDR_HI
+#define SDMA1_SDMA_QUEUE2_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA1_SDMA_QUEUE2_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_MQD_CONTROL
+#define SDMA1_SDMA_QUEUE2_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE2_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA1_SDMA_QUEUE2_CONTEXT_STATUS
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA1_SDMA_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA1_SDMA_QUEUE2_DUMMY_REG1
+#define SDMA1_SDMA_QUEUE2_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE2_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE2_DUMMY_REG2
+#define SDMA1_SDMA_QUEUE2_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE2_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_RB_CNTL
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA1_SDMA_QUEUE3_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA1_SDMA_QUEUE3_RB_BASE
+#define SDMA1_SDMA_QUEUE3_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE3_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_RB_BASE_HI
+#define SDMA1_SDMA_QUEUE3_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE3_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA1_SDMA_QUEUE3_RB_RPTR
+#define SDMA1_SDMA_QUEUE3_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE3_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_RB_RPTR_HI
+#define SDMA1_SDMA_QUEUE3_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE3_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_RB_WPTR
+#define SDMA1_SDMA_QUEUE3_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE3_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_RB_WPTR_HI
+#define SDMA1_SDMA_QUEUE3_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE3_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_RB_RPTR_ADDR_LO
+#define SDMA1_SDMA_QUEUE3_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_SDMA_QUEUE3_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE3_RB_RPTR_ADDR_HI
+#define SDMA1_SDMA_QUEUE3_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE3_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_IB_CNTL
+#define SDMA1_SDMA_QUEUE3_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE3_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA1_SDMA_QUEUE3_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE3_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA1_SDMA_QUEUE3_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA1_SDMA_QUEUE3_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE3_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA1_SDMA_QUEUE3_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE3_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA1_SDMA_QUEUE3_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA1_SDMA_QUEUE3_IB_RPTR
+#define SDMA1_SDMA_QUEUE3_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA1_SDMA_QUEUE3_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA1_SDMA_QUEUE3_IB_OFFSET
+#define SDMA1_SDMA_QUEUE3_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE3_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA1_SDMA_QUEUE3_IB_BASE_LO
+#define SDMA1_SDMA_QUEUE3_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_SDMA_QUEUE3_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE3_IB_BASE_HI
+#define SDMA1_SDMA_QUEUE3_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE3_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_IB_SIZE
+#define SDMA1_SDMA_QUEUE3_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE3_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA1_SDMA_QUEUE3_DOORBELL
+#define SDMA1_SDMA_QUEUE3_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA1_SDMA_QUEUE3_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA1_SDMA_QUEUE3_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA1_SDMA_QUEUE3_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA1_SDMA_QUEUE3_DOORBELL_LOG
+#define SDMA1_SDMA_QUEUE3_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA1_SDMA_QUEUE3_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA1_SDMA_QUEUE3_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA1_SDMA_QUEUE3_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE3_DOORBELL_OFFSET
+#define SDMA1_SDMA_QUEUE3_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA1_SDMA_QUEUE3_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA1_SDMA_QUEUE3_CSA_ADDR_LO
+#define SDMA1_SDMA_QUEUE3_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE3_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE3_CSA_ADDR_HI
+#define SDMA1_SDMA_QUEUE3_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE3_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_SCHEDULE_CNTL
+#define SDMA1_SDMA_QUEUE3_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE3_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA1_SDMA_QUEUE3_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA1_SDMA_QUEUE3_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA1_SDMA_QUEUE3_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA1_SDMA_QUEUE3_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA1_SDMA_QUEUE3_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA1_SDMA_QUEUE3_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA1_SDMA_QUEUE3_IB_SUB_REMAIN
+#define SDMA1_SDMA_QUEUE3_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE3_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA1_SDMA_QUEUE3_PREEMPT
+#define SDMA1_SDMA_QUEUE3_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE3_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA1_SDMA_QUEUE3_DUMMY_REG
+#define SDMA1_SDMA_QUEUE3_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE3_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA1_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA1_SDMA_QUEUE3_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_RB_AQL_CNTL
+#define SDMA1_SDMA_QUEUE3_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE3_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA1_SDMA_QUEUE3_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA1_SDMA_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA1_SDMA_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA1_SDMA_QUEUE3_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA1_SDMA_QUEUE3_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE3_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA1_SDMA_QUEUE3_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA1_SDMA_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA1_SDMA_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA1_SDMA_QUEUE3_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA1_SDMA_QUEUE3_MINOR_PTR_UPDATE
+#define SDMA1_SDMA_QUEUE3_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE3_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA1_SDMA_QUEUE3_MIDCMD_CNTL
+#define SDMA1_SDMA_QUEUE3_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE3_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE3_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA1_SDMA_QUEUE3_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA1_SDMA_QUEUE3_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE3_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE3_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA1_SDMA_QUEUE3_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA1_SDMA_QUEUE3_MIDCMD_DATA0
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_MIDCMD_DATA1
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_MIDCMD_DATA2
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_MIDCMD_DATA3
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_MIDCMD_DATA4
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_MIDCMD_DATA5
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_MIDCMD_DATA6
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_MIDCMD_DATA7
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_MIDCMD_DATA8
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_MIDCMD_DATA9
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_MIDCMD_DATA10
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE3_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_UTILIZATION_LO
+#define SDMA1_SDMA_QUEUE3_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE3_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_UTILIZATION_HI
+#define SDMA1_SDMA_QUEUE3_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE3_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_WAIT_UNSATISFIED_THD
+#define SDMA1_SDMA_QUEUE3_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA1_SDMA_QUEUE3_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA1_SDMA_QUEUE3_MQD_BASE_ADDR_LO
+#define SDMA1_SDMA_QUEUE3_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA1_SDMA_QUEUE3_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE3_MQD_BASE_ADDR_HI
+#define SDMA1_SDMA_QUEUE3_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA1_SDMA_QUEUE3_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_MQD_CONTROL
+#define SDMA1_SDMA_QUEUE3_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE3_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA1_SDMA_QUEUE3_CONTEXT_STATUS
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA1_SDMA_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA1_SDMA_QUEUE3_DUMMY_REG1
+#define SDMA1_SDMA_QUEUE3_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE3_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE3_DUMMY_REG2
+#define SDMA1_SDMA_QUEUE3_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE3_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_RB_CNTL
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA1_SDMA_QUEUE4_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA1_SDMA_QUEUE4_RB_BASE
+#define SDMA1_SDMA_QUEUE4_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE4_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_RB_BASE_HI
+#define SDMA1_SDMA_QUEUE4_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE4_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA1_SDMA_QUEUE4_RB_RPTR
+#define SDMA1_SDMA_QUEUE4_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE4_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_RB_RPTR_HI
+#define SDMA1_SDMA_QUEUE4_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE4_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_RB_WPTR
+#define SDMA1_SDMA_QUEUE4_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE4_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_RB_WPTR_HI
+#define SDMA1_SDMA_QUEUE4_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE4_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_RB_RPTR_ADDR_LO
+#define SDMA1_SDMA_QUEUE4_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_SDMA_QUEUE4_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE4_RB_RPTR_ADDR_HI
+#define SDMA1_SDMA_QUEUE4_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE4_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_IB_CNTL
+#define SDMA1_SDMA_QUEUE4_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE4_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA1_SDMA_QUEUE4_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE4_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA1_SDMA_QUEUE4_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA1_SDMA_QUEUE4_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE4_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA1_SDMA_QUEUE4_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE4_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA1_SDMA_QUEUE4_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA1_SDMA_QUEUE4_IB_RPTR
+#define SDMA1_SDMA_QUEUE4_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA1_SDMA_QUEUE4_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA1_SDMA_QUEUE4_IB_OFFSET
+#define SDMA1_SDMA_QUEUE4_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE4_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA1_SDMA_QUEUE4_IB_BASE_LO
+#define SDMA1_SDMA_QUEUE4_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_SDMA_QUEUE4_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE4_IB_BASE_HI
+#define SDMA1_SDMA_QUEUE4_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE4_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_IB_SIZE
+#define SDMA1_SDMA_QUEUE4_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE4_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA1_SDMA_QUEUE4_DOORBELL
+#define SDMA1_SDMA_QUEUE4_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA1_SDMA_QUEUE4_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA1_SDMA_QUEUE4_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA1_SDMA_QUEUE4_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA1_SDMA_QUEUE4_DOORBELL_LOG
+#define SDMA1_SDMA_QUEUE4_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA1_SDMA_QUEUE4_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA1_SDMA_QUEUE4_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA1_SDMA_QUEUE4_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE4_DOORBELL_OFFSET
+#define SDMA1_SDMA_QUEUE4_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA1_SDMA_QUEUE4_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA1_SDMA_QUEUE4_CSA_ADDR_LO
+#define SDMA1_SDMA_QUEUE4_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE4_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE4_CSA_ADDR_HI
+#define SDMA1_SDMA_QUEUE4_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE4_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_SCHEDULE_CNTL
+#define SDMA1_SDMA_QUEUE4_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE4_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA1_SDMA_QUEUE4_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA1_SDMA_QUEUE4_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA1_SDMA_QUEUE4_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA1_SDMA_QUEUE4_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA1_SDMA_QUEUE4_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA1_SDMA_QUEUE4_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA1_SDMA_QUEUE4_IB_SUB_REMAIN
+#define SDMA1_SDMA_QUEUE4_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE4_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA1_SDMA_QUEUE4_PREEMPT
+#define SDMA1_SDMA_QUEUE4_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE4_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA1_SDMA_QUEUE4_DUMMY_REG
+#define SDMA1_SDMA_QUEUE4_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE4_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA1_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA1_SDMA_QUEUE4_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_RB_AQL_CNTL
+#define SDMA1_SDMA_QUEUE4_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE4_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA1_SDMA_QUEUE4_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA1_SDMA_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA1_SDMA_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA1_SDMA_QUEUE4_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA1_SDMA_QUEUE4_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE4_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA1_SDMA_QUEUE4_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA1_SDMA_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA1_SDMA_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA1_SDMA_QUEUE4_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA1_SDMA_QUEUE4_MINOR_PTR_UPDATE
+#define SDMA1_SDMA_QUEUE4_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE4_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA1_SDMA_QUEUE4_MIDCMD_CNTL
+#define SDMA1_SDMA_QUEUE4_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE4_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE4_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA1_SDMA_QUEUE4_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA1_SDMA_QUEUE4_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE4_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE4_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA1_SDMA_QUEUE4_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA1_SDMA_QUEUE4_MIDCMD_DATA0
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_MIDCMD_DATA1
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_MIDCMD_DATA2
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_MIDCMD_DATA3
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_MIDCMD_DATA4
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_MIDCMD_DATA5
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_MIDCMD_DATA6
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_MIDCMD_DATA7
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_MIDCMD_DATA8
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_MIDCMD_DATA9
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_MIDCMD_DATA10
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE4_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_UTILIZATION_LO
+#define SDMA1_SDMA_QUEUE4_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE4_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_UTILIZATION_HI
+#define SDMA1_SDMA_QUEUE4_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE4_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_WAIT_UNSATISFIED_THD
+#define SDMA1_SDMA_QUEUE4_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA1_SDMA_QUEUE4_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA1_SDMA_QUEUE4_MQD_BASE_ADDR_LO
+#define SDMA1_SDMA_QUEUE4_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA1_SDMA_QUEUE4_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE4_MQD_BASE_ADDR_HI
+#define SDMA1_SDMA_QUEUE4_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA1_SDMA_QUEUE4_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_MQD_CONTROL
+#define SDMA1_SDMA_QUEUE4_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE4_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA1_SDMA_QUEUE4_CONTEXT_STATUS
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA1_SDMA_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA1_SDMA_QUEUE4_DUMMY_REG1
+#define SDMA1_SDMA_QUEUE4_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE4_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE4_DUMMY_REG2
+#define SDMA1_SDMA_QUEUE4_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE4_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_RB_CNTL
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA1_SDMA_QUEUE5_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA1_SDMA_QUEUE5_RB_BASE
+#define SDMA1_SDMA_QUEUE5_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE5_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_RB_BASE_HI
+#define SDMA1_SDMA_QUEUE5_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE5_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA1_SDMA_QUEUE5_RB_RPTR
+#define SDMA1_SDMA_QUEUE5_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE5_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_RB_RPTR_HI
+#define SDMA1_SDMA_QUEUE5_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE5_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_RB_WPTR
+#define SDMA1_SDMA_QUEUE5_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE5_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_RB_WPTR_HI
+#define SDMA1_SDMA_QUEUE5_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE5_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_RB_RPTR_ADDR_LO
+#define SDMA1_SDMA_QUEUE5_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_SDMA_QUEUE5_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE5_RB_RPTR_ADDR_HI
+#define SDMA1_SDMA_QUEUE5_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE5_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_IB_CNTL
+#define SDMA1_SDMA_QUEUE5_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE5_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA1_SDMA_QUEUE5_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE5_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA1_SDMA_QUEUE5_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA1_SDMA_QUEUE5_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE5_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA1_SDMA_QUEUE5_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE5_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA1_SDMA_QUEUE5_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA1_SDMA_QUEUE5_IB_RPTR
+#define SDMA1_SDMA_QUEUE5_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA1_SDMA_QUEUE5_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA1_SDMA_QUEUE5_IB_OFFSET
+#define SDMA1_SDMA_QUEUE5_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE5_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA1_SDMA_QUEUE5_IB_BASE_LO
+#define SDMA1_SDMA_QUEUE5_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_SDMA_QUEUE5_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE5_IB_BASE_HI
+#define SDMA1_SDMA_QUEUE5_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE5_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_IB_SIZE
+#define SDMA1_SDMA_QUEUE5_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE5_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA1_SDMA_QUEUE5_DOORBELL
+#define SDMA1_SDMA_QUEUE5_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA1_SDMA_QUEUE5_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA1_SDMA_QUEUE5_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA1_SDMA_QUEUE5_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA1_SDMA_QUEUE5_DOORBELL_LOG
+#define SDMA1_SDMA_QUEUE5_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA1_SDMA_QUEUE5_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA1_SDMA_QUEUE5_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA1_SDMA_QUEUE5_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE5_DOORBELL_OFFSET
+#define SDMA1_SDMA_QUEUE5_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA1_SDMA_QUEUE5_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA1_SDMA_QUEUE5_CSA_ADDR_LO
+#define SDMA1_SDMA_QUEUE5_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE5_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE5_CSA_ADDR_HI
+#define SDMA1_SDMA_QUEUE5_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE5_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_SCHEDULE_CNTL
+#define SDMA1_SDMA_QUEUE5_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE5_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA1_SDMA_QUEUE5_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA1_SDMA_QUEUE5_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA1_SDMA_QUEUE5_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA1_SDMA_QUEUE5_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA1_SDMA_QUEUE5_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA1_SDMA_QUEUE5_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA1_SDMA_QUEUE5_IB_SUB_REMAIN
+#define SDMA1_SDMA_QUEUE5_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE5_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA1_SDMA_QUEUE5_PREEMPT
+#define SDMA1_SDMA_QUEUE5_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE5_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA1_SDMA_QUEUE5_DUMMY_REG
+#define SDMA1_SDMA_QUEUE5_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE5_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA1_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA1_SDMA_QUEUE5_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_RB_AQL_CNTL
+#define SDMA1_SDMA_QUEUE5_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE5_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA1_SDMA_QUEUE5_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA1_SDMA_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA1_SDMA_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA1_SDMA_QUEUE5_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA1_SDMA_QUEUE5_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE5_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA1_SDMA_QUEUE5_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA1_SDMA_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA1_SDMA_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA1_SDMA_QUEUE5_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA1_SDMA_QUEUE5_MINOR_PTR_UPDATE
+#define SDMA1_SDMA_QUEUE5_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE5_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA1_SDMA_QUEUE5_MIDCMD_CNTL
+#define SDMA1_SDMA_QUEUE5_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE5_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE5_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA1_SDMA_QUEUE5_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA1_SDMA_QUEUE5_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE5_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE5_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA1_SDMA_QUEUE5_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA1_SDMA_QUEUE5_MIDCMD_DATA0
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_MIDCMD_DATA1
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_MIDCMD_DATA2
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_MIDCMD_DATA3
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_MIDCMD_DATA4
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_MIDCMD_DATA5
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_MIDCMD_DATA6
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_MIDCMD_DATA7
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_MIDCMD_DATA8
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_MIDCMD_DATA9
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_MIDCMD_DATA10
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE5_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_UTILIZATION_LO
+#define SDMA1_SDMA_QUEUE5_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE5_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_UTILIZATION_HI
+#define SDMA1_SDMA_QUEUE5_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE5_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_WAIT_UNSATISFIED_THD
+#define SDMA1_SDMA_QUEUE5_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA1_SDMA_QUEUE5_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA1_SDMA_QUEUE5_MQD_BASE_ADDR_LO
+#define SDMA1_SDMA_QUEUE5_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA1_SDMA_QUEUE5_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE5_MQD_BASE_ADDR_HI
+#define SDMA1_SDMA_QUEUE5_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA1_SDMA_QUEUE5_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_MQD_CONTROL
+#define SDMA1_SDMA_QUEUE5_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE5_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA1_SDMA_QUEUE5_CONTEXT_STATUS
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA1_SDMA_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA1_SDMA_QUEUE5_DUMMY_REG1
+#define SDMA1_SDMA_QUEUE5_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE5_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE5_DUMMY_REG2
+#define SDMA1_SDMA_QUEUE5_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE5_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_RB_CNTL
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA1_SDMA_QUEUE6_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA1_SDMA_QUEUE6_RB_BASE
+#define SDMA1_SDMA_QUEUE6_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE6_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_RB_BASE_HI
+#define SDMA1_SDMA_QUEUE6_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE6_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA1_SDMA_QUEUE6_RB_RPTR
+#define SDMA1_SDMA_QUEUE6_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE6_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_RB_RPTR_HI
+#define SDMA1_SDMA_QUEUE6_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE6_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_RB_WPTR
+#define SDMA1_SDMA_QUEUE6_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE6_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_RB_WPTR_HI
+#define SDMA1_SDMA_QUEUE6_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE6_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_RB_RPTR_ADDR_LO
+#define SDMA1_SDMA_QUEUE6_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_SDMA_QUEUE6_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE6_RB_RPTR_ADDR_HI
+#define SDMA1_SDMA_QUEUE6_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE6_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_IB_CNTL
+#define SDMA1_SDMA_QUEUE6_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE6_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA1_SDMA_QUEUE6_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE6_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA1_SDMA_QUEUE6_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA1_SDMA_QUEUE6_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE6_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA1_SDMA_QUEUE6_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE6_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA1_SDMA_QUEUE6_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA1_SDMA_QUEUE6_IB_RPTR
+#define SDMA1_SDMA_QUEUE6_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA1_SDMA_QUEUE6_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA1_SDMA_QUEUE6_IB_OFFSET
+#define SDMA1_SDMA_QUEUE6_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE6_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA1_SDMA_QUEUE6_IB_BASE_LO
+#define SDMA1_SDMA_QUEUE6_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_SDMA_QUEUE6_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE6_IB_BASE_HI
+#define SDMA1_SDMA_QUEUE6_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE6_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_IB_SIZE
+#define SDMA1_SDMA_QUEUE6_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE6_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA1_SDMA_QUEUE6_DOORBELL
+#define SDMA1_SDMA_QUEUE6_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA1_SDMA_QUEUE6_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA1_SDMA_QUEUE6_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA1_SDMA_QUEUE6_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA1_SDMA_QUEUE6_DOORBELL_LOG
+#define SDMA1_SDMA_QUEUE6_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA1_SDMA_QUEUE6_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA1_SDMA_QUEUE6_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA1_SDMA_QUEUE6_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE6_DOORBELL_OFFSET
+#define SDMA1_SDMA_QUEUE6_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA1_SDMA_QUEUE6_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA1_SDMA_QUEUE6_CSA_ADDR_LO
+#define SDMA1_SDMA_QUEUE6_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE6_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE6_CSA_ADDR_HI
+#define SDMA1_SDMA_QUEUE6_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE6_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_SCHEDULE_CNTL
+#define SDMA1_SDMA_QUEUE6_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE6_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA1_SDMA_QUEUE6_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA1_SDMA_QUEUE6_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA1_SDMA_QUEUE6_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA1_SDMA_QUEUE6_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA1_SDMA_QUEUE6_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA1_SDMA_QUEUE6_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA1_SDMA_QUEUE6_IB_SUB_REMAIN
+#define SDMA1_SDMA_QUEUE6_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE6_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA1_SDMA_QUEUE6_PREEMPT
+#define SDMA1_SDMA_QUEUE6_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE6_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA1_SDMA_QUEUE6_DUMMY_REG
+#define SDMA1_SDMA_QUEUE6_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE6_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA1_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA1_SDMA_QUEUE6_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_RB_AQL_CNTL
+#define SDMA1_SDMA_QUEUE6_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE6_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA1_SDMA_QUEUE6_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA1_SDMA_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA1_SDMA_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA1_SDMA_QUEUE6_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA1_SDMA_QUEUE6_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE6_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA1_SDMA_QUEUE6_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA1_SDMA_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA1_SDMA_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA1_SDMA_QUEUE6_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA1_SDMA_QUEUE6_MINOR_PTR_UPDATE
+#define SDMA1_SDMA_QUEUE6_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE6_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA1_SDMA_QUEUE6_MIDCMD_CNTL
+#define SDMA1_SDMA_QUEUE6_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE6_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE6_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA1_SDMA_QUEUE6_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA1_SDMA_QUEUE6_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE6_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE6_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA1_SDMA_QUEUE6_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA1_SDMA_QUEUE6_MIDCMD_DATA0
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_MIDCMD_DATA1
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_MIDCMD_DATA2
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_MIDCMD_DATA3
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_MIDCMD_DATA4
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_MIDCMD_DATA5
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_MIDCMD_DATA6
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_MIDCMD_DATA7
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_MIDCMD_DATA8
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_MIDCMD_DATA9
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_MIDCMD_DATA10
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE6_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_UTILIZATION_LO
+#define SDMA1_SDMA_QUEUE6_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE6_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_UTILIZATION_HI
+#define SDMA1_SDMA_QUEUE6_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE6_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_WAIT_UNSATISFIED_THD
+#define SDMA1_SDMA_QUEUE6_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA1_SDMA_QUEUE6_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA1_SDMA_QUEUE6_MQD_BASE_ADDR_LO
+#define SDMA1_SDMA_QUEUE6_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA1_SDMA_QUEUE6_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE6_MQD_BASE_ADDR_HI
+#define SDMA1_SDMA_QUEUE6_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA1_SDMA_QUEUE6_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_MQD_CONTROL
+#define SDMA1_SDMA_QUEUE6_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE6_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA1_SDMA_QUEUE6_CONTEXT_STATUS
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA1_SDMA_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA1_SDMA_QUEUE6_DUMMY_REG1
+#define SDMA1_SDMA_QUEUE6_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE6_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE6_DUMMY_REG2
+#define SDMA1_SDMA_QUEUE6_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE6_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_RB_CNTL
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA1_SDMA_QUEUE7_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA1_SDMA_QUEUE7_RB_BASE
+#define SDMA1_SDMA_QUEUE7_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE7_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_RB_BASE_HI
+#define SDMA1_SDMA_QUEUE7_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE7_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA1_SDMA_QUEUE7_RB_RPTR
+#define SDMA1_SDMA_QUEUE7_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE7_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_RB_RPTR_HI
+#define SDMA1_SDMA_QUEUE7_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE7_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_RB_WPTR
+#define SDMA1_SDMA_QUEUE7_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE7_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_RB_WPTR_HI
+#define SDMA1_SDMA_QUEUE7_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE7_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_RB_RPTR_ADDR_LO
+#define SDMA1_SDMA_QUEUE7_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_SDMA_QUEUE7_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE7_RB_RPTR_ADDR_HI
+#define SDMA1_SDMA_QUEUE7_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE7_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_IB_CNTL
+#define SDMA1_SDMA_QUEUE7_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE7_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA1_SDMA_QUEUE7_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE7_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA1_SDMA_QUEUE7_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA1_SDMA_QUEUE7_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE7_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA1_SDMA_QUEUE7_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE7_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA1_SDMA_QUEUE7_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA1_SDMA_QUEUE7_IB_RPTR
+#define SDMA1_SDMA_QUEUE7_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA1_SDMA_QUEUE7_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA1_SDMA_QUEUE7_IB_OFFSET
+#define SDMA1_SDMA_QUEUE7_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE7_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA1_SDMA_QUEUE7_IB_BASE_LO
+#define SDMA1_SDMA_QUEUE7_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_SDMA_QUEUE7_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE7_IB_BASE_HI
+#define SDMA1_SDMA_QUEUE7_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE7_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_IB_SIZE
+#define SDMA1_SDMA_QUEUE7_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE7_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA1_SDMA_QUEUE7_DOORBELL
+#define SDMA1_SDMA_QUEUE7_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA1_SDMA_QUEUE7_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA1_SDMA_QUEUE7_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA1_SDMA_QUEUE7_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA1_SDMA_QUEUE7_DOORBELL_LOG
+#define SDMA1_SDMA_QUEUE7_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA1_SDMA_QUEUE7_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA1_SDMA_QUEUE7_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA1_SDMA_QUEUE7_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE7_DOORBELL_OFFSET
+#define SDMA1_SDMA_QUEUE7_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA1_SDMA_QUEUE7_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA1_SDMA_QUEUE7_CSA_ADDR_LO
+#define SDMA1_SDMA_QUEUE7_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE7_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE7_CSA_ADDR_HI
+#define SDMA1_SDMA_QUEUE7_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE7_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_SCHEDULE_CNTL
+#define SDMA1_SDMA_QUEUE7_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE7_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA1_SDMA_QUEUE7_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA1_SDMA_QUEUE7_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA1_SDMA_QUEUE7_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA1_SDMA_QUEUE7_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA1_SDMA_QUEUE7_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA1_SDMA_QUEUE7_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA1_SDMA_QUEUE7_IB_SUB_REMAIN
+#define SDMA1_SDMA_QUEUE7_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE7_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA1_SDMA_QUEUE7_PREEMPT
+#define SDMA1_SDMA_QUEUE7_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE7_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA1_SDMA_QUEUE7_DUMMY_REG
+#define SDMA1_SDMA_QUEUE7_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE7_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA1_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA1_SDMA_QUEUE7_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_RB_AQL_CNTL
+#define SDMA1_SDMA_QUEUE7_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE7_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA1_SDMA_QUEUE7_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA1_SDMA_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA1_SDMA_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA1_SDMA_QUEUE7_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA1_SDMA_QUEUE7_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE7_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA1_SDMA_QUEUE7_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA1_SDMA_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA1_SDMA_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA1_SDMA_QUEUE7_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA1_SDMA_QUEUE7_MINOR_PTR_UPDATE
+#define SDMA1_SDMA_QUEUE7_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE7_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA1_SDMA_QUEUE7_MIDCMD_CNTL
+#define SDMA1_SDMA_QUEUE7_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE7_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE7_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA1_SDMA_QUEUE7_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA1_SDMA_QUEUE7_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE7_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE7_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA1_SDMA_QUEUE7_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA1_SDMA_QUEUE7_MIDCMD_DATA0
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_MIDCMD_DATA1
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_MIDCMD_DATA2
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_MIDCMD_DATA3
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_MIDCMD_DATA4
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_MIDCMD_DATA5
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_MIDCMD_DATA6
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_MIDCMD_DATA7
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_MIDCMD_DATA8
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_MIDCMD_DATA9
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_MIDCMD_DATA10
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE7_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_UTILIZATION_LO
+#define SDMA1_SDMA_QUEUE7_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE7_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_UTILIZATION_HI
+#define SDMA1_SDMA_QUEUE7_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE7_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_WAIT_UNSATISFIED_THD
+#define SDMA1_SDMA_QUEUE7_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA1_SDMA_QUEUE7_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA1_SDMA_QUEUE7_MQD_BASE_ADDR_LO
+#define SDMA1_SDMA_QUEUE7_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA1_SDMA_QUEUE7_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE7_MQD_BASE_ADDR_HI
+#define SDMA1_SDMA_QUEUE7_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA1_SDMA_QUEUE7_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_MQD_CONTROL
+#define SDMA1_SDMA_QUEUE7_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE7_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA1_SDMA_QUEUE7_CONTEXT_STATUS
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA1_SDMA_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA1_SDMA_QUEUE7_DUMMY_REG1
+#define SDMA1_SDMA_QUEUE7_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE7_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE7_DUMMY_REG2
+#define SDMA1_SDMA_QUEUE7_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE7_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_RB_CNTL
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA1_SDMA_QUEUE8_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA1_SDMA_QUEUE8_RB_BASE
+#define SDMA1_SDMA_QUEUE8_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE8_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_RB_BASE_HI
+#define SDMA1_SDMA_QUEUE8_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE8_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA1_SDMA_QUEUE8_RB_RPTR
+#define SDMA1_SDMA_QUEUE8_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE8_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_RB_RPTR_HI
+#define SDMA1_SDMA_QUEUE8_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE8_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_RB_WPTR
+#define SDMA1_SDMA_QUEUE8_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE8_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_RB_WPTR_HI
+#define SDMA1_SDMA_QUEUE8_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE8_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_RB_RPTR_ADDR_LO
+#define SDMA1_SDMA_QUEUE8_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_SDMA_QUEUE8_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE8_RB_RPTR_ADDR_HI
+#define SDMA1_SDMA_QUEUE8_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE8_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_IB_CNTL
+#define SDMA1_SDMA_QUEUE8_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE8_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA1_SDMA_QUEUE8_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE8_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA1_SDMA_QUEUE8_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA1_SDMA_QUEUE8_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE8_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA1_SDMA_QUEUE8_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE8_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA1_SDMA_QUEUE8_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA1_SDMA_QUEUE8_IB_RPTR
+#define SDMA1_SDMA_QUEUE8_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA1_SDMA_QUEUE8_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA1_SDMA_QUEUE8_IB_OFFSET
+#define SDMA1_SDMA_QUEUE8_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE8_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA1_SDMA_QUEUE8_IB_BASE_LO
+#define SDMA1_SDMA_QUEUE8_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_SDMA_QUEUE8_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE8_IB_BASE_HI
+#define SDMA1_SDMA_QUEUE8_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE8_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_IB_SIZE
+#define SDMA1_SDMA_QUEUE8_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE8_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA1_SDMA_QUEUE8_DOORBELL
+#define SDMA1_SDMA_QUEUE8_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA1_SDMA_QUEUE8_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA1_SDMA_QUEUE8_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA1_SDMA_QUEUE8_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA1_SDMA_QUEUE8_DOORBELL_LOG
+#define SDMA1_SDMA_QUEUE8_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA1_SDMA_QUEUE8_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA1_SDMA_QUEUE8_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA1_SDMA_QUEUE8_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE8_DOORBELL_OFFSET
+#define SDMA1_SDMA_QUEUE8_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA1_SDMA_QUEUE8_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA1_SDMA_QUEUE8_CSA_ADDR_LO
+#define SDMA1_SDMA_QUEUE8_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE8_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE8_CSA_ADDR_HI
+#define SDMA1_SDMA_QUEUE8_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE8_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_SCHEDULE_CNTL
+#define SDMA1_SDMA_QUEUE8_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE8_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA1_SDMA_QUEUE8_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA1_SDMA_QUEUE8_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA1_SDMA_QUEUE8_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA1_SDMA_QUEUE8_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA1_SDMA_QUEUE8_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA1_SDMA_QUEUE8_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA1_SDMA_QUEUE8_IB_SUB_REMAIN
+#define SDMA1_SDMA_QUEUE8_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE8_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA1_SDMA_QUEUE8_PREEMPT
+#define SDMA1_SDMA_QUEUE8_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE8_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA1_SDMA_QUEUE8_DUMMY_REG
+#define SDMA1_SDMA_QUEUE8_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE8_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA1_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA1_SDMA_QUEUE8_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_RB_AQL_CNTL
+#define SDMA1_SDMA_QUEUE8_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE8_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA1_SDMA_QUEUE8_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA1_SDMA_QUEUE8_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA1_SDMA_QUEUE8_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA1_SDMA_QUEUE8_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA1_SDMA_QUEUE8_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE8_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA1_SDMA_QUEUE8_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA1_SDMA_QUEUE8_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA1_SDMA_QUEUE8_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA1_SDMA_QUEUE8_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA1_SDMA_QUEUE8_MINOR_PTR_UPDATE
+#define SDMA1_SDMA_QUEUE8_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE8_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA1_SDMA_QUEUE8_MIDCMD_CNTL
+#define SDMA1_SDMA_QUEUE8_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE8_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE8_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA1_SDMA_QUEUE8_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA1_SDMA_QUEUE8_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE8_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE8_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA1_SDMA_QUEUE8_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA1_SDMA_QUEUE8_MIDCMD_DATA0
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_MIDCMD_DATA1
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_MIDCMD_DATA2
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_MIDCMD_DATA3
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_MIDCMD_DATA4
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_MIDCMD_DATA5
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_MIDCMD_DATA6
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_MIDCMD_DATA7
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_MIDCMD_DATA8
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_MIDCMD_DATA9
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_MIDCMD_DATA10
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE8_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_UTILIZATION_LO
+#define SDMA1_SDMA_QUEUE8_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE8_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_UTILIZATION_HI
+#define SDMA1_SDMA_QUEUE8_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE8_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_WAIT_UNSATISFIED_THD
+#define SDMA1_SDMA_QUEUE8_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA1_SDMA_QUEUE8_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA1_SDMA_QUEUE8_MQD_BASE_ADDR_LO
+#define SDMA1_SDMA_QUEUE8_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA1_SDMA_QUEUE8_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE8_MQD_BASE_ADDR_HI
+#define SDMA1_SDMA_QUEUE8_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA1_SDMA_QUEUE8_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_MQD_CONTROL
+#define SDMA1_SDMA_QUEUE8_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE8_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA1_SDMA_QUEUE8_CONTEXT_STATUS
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA1_SDMA_QUEUE8_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA1_SDMA_QUEUE8_DUMMY_REG1
+#define SDMA1_SDMA_QUEUE8_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE8_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE8_DUMMY_REG2
+#define SDMA1_SDMA_QUEUE8_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE8_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_RB_CNTL
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__RB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__RB_SIZE__SHIFT                                                             0x1
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                      0x9
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                               0xa
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT                                                0xb
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                               0xc
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                          0xd
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                0x10
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__RB_PRIV__SHIFT                                                             0x17
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__RB_VMID__SHIFT                                                             0x18
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__RB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__RB_SIZE_MASK                                                               0x0000003EL
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__RB_SWAP_ENABLE_MASK                                                        0x00000200L
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                 0x00000400L
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__MCU_WPTR_POLL_ENABLE_MASK                                                  0x00000800L
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                 0x00001000L
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                            0x00002000L
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                  0x001F0000L
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__RB_PRIV_MASK                                                               0x00800000L
+#define SDMA1_SDMA_QUEUE9_RB_CNTL__RB_VMID_MASK                                                               0x0F000000L
+//SDMA1_SDMA_QUEUE9_RB_BASE
+#define SDMA1_SDMA_QUEUE9_RB_BASE__ADDR__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE9_RB_BASE__ADDR_MASK                                                                  0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_RB_BASE_HI
+#define SDMA1_SDMA_QUEUE9_RB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE9_RB_BASE_HI__ADDR_MASK                                                               0x00FFFFFFL
+//SDMA1_SDMA_QUEUE9_RB_RPTR
+#define SDMA1_SDMA_QUEUE9_RB_RPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE9_RB_RPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_RB_RPTR_HI
+#define SDMA1_SDMA_QUEUE9_RB_RPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE9_RB_RPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_RB_WPTR
+#define SDMA1_SDMA_QUEUE9_RB_WPTR__OFFSET__SHIFT                                                              0x0
+#define SDMA1_SDMA_QUEUE9_RB_WPTR__OFFSET_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_RB_WPTR_HI
+#define SDMA1_SDMA_QUEUE9_RB_WPTR_HI__OFFSET__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE9_RB_WPTR_HI__OFFSET_MASK                                                             0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_RB_RPTR_ADDR_LO
+#define SDMA1_SDMA_QUEUE9_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_SDMA_QUEUE9_RB_RPTR_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE9_RB_RPTR_ADDR_HI
+#define SDMA1_SDMA_QUEUE9_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE9_RB_RPTR_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_IB_CNTL
+#define SDMA1_SDMA_QUEUE9_IB_CNTL__IB_ENABLE__SHIFT                                                           0x0
+#define SDMA1_SDMA_QUEUE9_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                      0x4
+#define SDMA1_SDMA_QUEUE9_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE9_IB_CNTL__CMD_VMID__SHIFT                                                            0x10
+#define SDMA1_SDMA_QUEUE9_IB_CNTL__IB_PRIV__SHIFT                                                             0x1f
+#define SDMA1_SDMA_QUEUE9_IB_CNTL__IB_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_SDMA_QUEUE9_IB_CNTL__IB_SWAP_ENABLE_MASK                                                        0x00000010L
+#define SDMA1_SDMA_QUEUE9_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE9_IB_CNTL__CMD_VMID_MASK                                                              0x000F0000L
+#define SDMA1_SDMA_QUEUE9_IB_CNTL__IB_PRIV_MASK                                                               0x80000000L
+//SDMA1_SDMA_QUEUE9_IB_RPTR
+#define SDMA1_SDMA_QUEUE9_IB_RPTR__OFFSET__SHIFT                                                              0x2
+#define SDMA1_SDMA_QUEUE9_IB_RPTR__OFFSET_MASK                                                                0x003FFFFCL
+//SDMA1_SDMA_QUEUE9_IB_OFFSET
+#define SDMA1_SDMA_QUEUE9_IB_OFFSET__OFFSET__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE9_IB_OFFSET__OFFSET_MASK                                                              0x003FFFFCL
+//SDMA1_SDMA_QUEUE9_IB_BASE_LO
+#define SDMA1_SDMA_QUEUE9_IB_BASE_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_SDMA_QUEUE9_IB_BASE_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE9_IB_BASE_HI
+#define SDMA1_SDMA_QUEUE9_IB_BASE_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE9_IB_BASE_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_IB_SIZE
+#define SDMA1_SDMA_QUEUE9_IB_SIZE__SIZE__SHIFT                                                                0x0
+#define SDMA1_SDMA_QUEUE9_IB_SIZE__SIZE_MASK                                                                  0x000FFFFFL
+//SDMA1_SDMA_QUEUE9_DOORBELL
+#define SDMA1_SDMA_QUEUE9_DOORBELL__ENABLE__SHIFT                                                             0x1c
+#define SDMA1_SDMA_QUEUE9_DOORBELL__CAPTURED__SHIFT                                                           0x1e
+#define SDMA1_SDMA_QUEUE9_DOORBELL__ENABLE_MASK                                                               0x10000000L
+#define SDMA1_SDMA_QUEUE9_DOORBELL__CAPTURED_MASK                                                             0x40000000L
+//SDMA1_SDMA_QUEUE9_DOORBELL_LOG
+#define SDMA1_SDMA_QUEUE9_DOORBELL_LOG__BE_ERROR__SHIFT                                                       0x0
+#define SDMA1_SDMA_QUEUE9_DOORBELL_LOG__DATA__SHIFT                                                           0x2
+#define SDMA1_SDMA_QUEUE9_DOORBELL_LOG__BE_ERROR_MASK                                                         0x00000001L
+#define SDMA1_SDMA_QUEUE9_DOORBELL_LOG__DATA_MASK                                                             0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE9_DOORBELL_OFFSET
+#define SDMA1_SDMA_QUEUE9_DOORBELL_OFFSET__OFFSET__SHIFT                                                      0x2
+#define SDMA1_SDMA_QUEUE9_DOORBELL_OFFSET__OFFSET_MASK                                                        0x0FFFFFFCL
+//SDMA1_SDMA_QUEUE9_CSA_ADDR_LO
+#define SDMA1_SDMA_QUEUE9_CSA_ADDR_LO__ADDR__SHIFT                                                            0x2
+#define SDMA1_SDMA_QUEUE9_CSA_ADDR_LO__ADDR_MASK                                                              0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE9_CSA_ADDR_HI
+#define SDMA1_SDMA_QUEUE9_CSA_ADDR_HI__ADDR__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE9_CSA_ADDR_HI__ADDR_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_SCHEDULE_CNTL
+#define SDMA1_SDMA_QUEUE9_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE9_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                    0x2
+#define SDMA1_SDMA_QUEUE9_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                      0x6
+#define SDMA1_SDMA_QUEUE9_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                               0x8
+#define SDMA1_SDMA_QUEUE9_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                       0x00000003L
+#define SDMA1_SDMA_QUEUE9_SCHEDULE_CNTL__PROCESS_ID_MASK                                                      0x0000001CL
+#define SDMA1_SDMA_QUEUE9_SCHEDULE_CNTL__LOCAL_ID_MASK                                                        0x000000C0L
+#define SDMA1_SDMA_QUEUE9_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                 0x0000FF00L
+//SDMA1_SDMA_QUEUE9_IB_SUB_REMAIN
+#define SDMA1_SDMA_QUEUE9_IB_SUB_REMAIN__SIZE__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE9_IB_SUB_REMAIN__SIZE_MASK                                                            0x00003FFFL
+//SDMA1_SDMA_QUEUE9_PREEMPT
+#define SDMA1_SDMA_QUEUE9_PREEMPT__IB_PREEMPT__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE9_PREEMPT__IB_PREEMPT_MASK                                                            0x00000001L
+//SDMA1_SDMA_QUEUE9_DUMMY_REG
+#define SDMA1_SDMA_QUEUE9_DUMMY_REG__DUMMY__SHIFT                                                             0x0
+#define SDMA1_SDMA_QUEUE9_DUMMY_REG__DUMMY_MASK                                                               0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                   0x2
+#define SDMA1_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                     0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                   0x0
+#define SDMA1_SDMA_QUEUE9_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                     0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_RB_AQL_CNTL
+#define SDMA1_SDMA_QUEUE9_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE9_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                 0x1
+#define SDMA1_SDMA_QUEUE9_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                     0x8
+#define SDMA1_SDMA_QUEUE9_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                           0x10
+#define SDMA1_SDMA_QUEUE9_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                     0x11
+#define SDMA1_SDMA_QUEUE9_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                  0x12
+#define SDMA1_SDMA_QUEUE9_RB_AQL_CNTL__AQL_ENABLE_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE9_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                   0x000000FEL
+#define SDMA1_SDMA_QUEUE9_RB_AQL_CNTL__PACKET_STEP_MASK                                                       0x0000FF00L
+#define SDMA1_SDMA_QUEUE9_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                             0x00010000L
+#define SDMA1_SDMA_QUEUE9_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                       0x00020000L
+#define SDMA1_SDMA_QUEUE9_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                    0x00040000L
+//SDMA1_SDMA_QUEUE9_MINOR_PTR_UPDATE
+#define SDMA1_SDMA_QUEUE9_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE9_MINOR_PTR_UPDATE__ENABLE_MASK                                                       0x00000001L
+//SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS__SHIFT                                     0x0
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION__SHIFT                                     0x1
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION__SHIFT                                        0x2
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION__SHIFT                                 0x4
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION__SHIFT                                  0x5
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION__SHIFT                              0x6
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION__SHIFT                                    0x7
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION__SHIFT                                    0x8
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION__SHIFT                                0x9
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__RB_PREEMPT_STATUS_MASK                                       0x00000001L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__VM_HOLE_EXCEPTION_MASK                                       0x00000002L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__PAGE_EXCEPTION_MASK                                          0x00000004L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__CMD_TIMEOUT_EXCEPTION_MASK                                   0x00000010L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__QUEUE_HANG_EXCEPTION_MASK                                    0x00000020L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__DOORBELL_ERROR_EXCEPTION_MASK                                0x00000040L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__SRAM_ECC_EXCEPTION_MASK                                      0x00000080L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__DRAM_ECC_EXCEPTION_MASK                                      0x00000100L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_SWITCH_STATUS__WPTR_LT_RPTR_EXCEPTION_MASK                                  0x00000200L
+//SDMA1_SDMA_QUEUE9_MIDCMD_CNTL
+#define SDMA1_SDMA_QUEUE9_MIDCMD_CNTL__DATA_VALID__SHIFT                                                      0x0
+#define SDMA1_SDMA_QUEUE9_MIDCMD_CNTL__COPY_MODE__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE9_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                     0x4
+#define SDMA1_SDMA_QUEUE9_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                   0x8
+#define SDMA1_SDMA_QUEUE9_MIDCMD_CNTL__DATA_VALID_MASK                                                        0x00000001L
+#define SDMA1_SDMA_QUEUE9_MIDCMD_CNTL__COPY_MODE_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE9_MIDCMD_CNTL__SPLIT_STATE_MASK                                                       0x000000F0L
+#define SDMA1_SDMA_QUEUE9_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                     0x00000100L
+//SDMA1_SDMA_QUEUE9_MIDCMD_DATA0
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA0__DATA0__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA0__DATA0_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_MIDCMD_DATA1
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA1__DATA1__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA1__DATA1_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_MIDCMD_DATA2
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA2__DATA2__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA2__DATA2_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_MIDCMD_DATA3
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA3__DATA3__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA3__DATA3_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_MIDCMD_DATA4
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA4__DATA4__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA4__DATA4_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_MIDCMD_DATA5
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA5__DATA5__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA5__DATA5_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_MIDCMD_DATA6
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA6__DATA6__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA6__DATA6_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_MIDCMD_DATA7
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA7__DATA7__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA7__DATA7_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_MIDCMD_DATA8
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA8__DATA8__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA8__DATA8_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_MIDCMD_DATA9
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA9__DATA9__SHIFT                                                          0x0
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA9__DATA9_MASK                                                            0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_MIDCMD_DATA10
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA10__DATA10__SHIFT                                                        0x0
+#define SDMA1_SDMA_QUEUE9_MIDCMD_DATA10__DATA10_MASK                                                          0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_UTILIZATION_LO
+#define SDMA1_SDMA_QUEUE9_UTILIZATION_LO__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE9_UTILIZATION_LO__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_UTILIZATION_HI
+#define SDMA1_SDMA_QUEUE9_UTILIZATION_HI__DATA__SHIFT                                                         0x0
+#define SDMA1_SDMA_QUEUE9_UTILIZATION_HI__DATA_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_WAIT_UNSATISFIED_THD
+#define SDMA1_SDMA_QUEUE9_WAIT_UNSATISFIED_THD__THRESHOLD__SHIFT                                              0x0
+#define SDMA1_SDMA_QUEUE9_WAIT_UNSATISFIED_THD__THRESHOLD_MASK                                                0x0000001FL
+//SDMA1_SDMA_QUEUE9_MQD_BASE_ADDR_LO
+#define SDMA1_SDMA_QUEUE9_MQD_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                               0x2
+#define SDMA1_SDMA_QUEUE9_MQD_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//SDMA1_SDMA_QUEUE9_MQD_BASE_ADDR_HI
+#define SDMA1_SDMA_QUEUE9_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                               0x0
+#define SDMA1_SDMA_QUEUE9_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_MQD_CONTROL
+#define SDMA1_SDMA_QUEUE9_MQD_CONTROL__VMID__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE9_MQD_CONTROL__VMID_MASK                                                              0x0000000FL
+//SDMA1_SDMA_QUEUE9_CONTEXT_STATUS
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__SELECTED__SHIFT                                                     0x0
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__USE_IB__SHIFT                                                       0x1
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__IDLE__SHIFT                                                         0x2
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__EXPIRED__SHIFT                                                      0x3
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__EXCEPTION__SHIFT                                                    0x4
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                   0x7
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__VF_STATUS__SHIFT                                                    0x8
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__PRIV_VIOLATION__SHIFT                                               0x9
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                              0xa
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                 0xb
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                          0xc
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                       0x10
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__SELECTED_MASK                                                       0x00000001L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__USE_IB_MASK                                                         0x00000002L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__IDLE_MASK                                                           0x00000004L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__EXPIRED_MASK                                                        0x00000008L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__EXCEPTION_MASK                                                      0x00000070L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                     0x00000080L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__VF_STATUS_MASK                                                      0x00000100L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__PRIV_VIOLATION_MASK                                                 0x00000200L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                0x00000400L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                   0x00000800L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                            0x00001000L
+#define SDMA1_SDMA_QUEUE9_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                         0x00FF0000L
+//SDMA1_SDMA_QUEUE9_DUMMY_REG1
+#define SDMA1_SDMA_QUEUE9_DUMMY_REG1__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE9_DUMMY_REG1__DUMMY_MASK                                                              0xFFFFFFFFL
+//SDMA1_SDMA_QUEUE9_DUMMY_REG2
+#define SDMA1_SDMA_QUEUE9_DUMMY_REG2__DUMMY__SHIFT                                                            0x0
+#define SDMA1_SDMA_QUEUE9_DUMMY_REG2__DUMMY_MASK                                                              0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmahypdec:1
+//SDMA1_SDMA_VM_CTX_LO
+#define SDMA1_SDMA_VM_CTX_LO__ADDR__SHIFT                                                                     0x2
+#define SDMA1_SDMA_VM_CTX_LO__ADDR_MASK                                                                       0xFFFFFFFCL
+//SDMA1_SDMA_VM_CTX_HI
+#define SDMA1_SDMA_VM_CTX_HI__ADDR__SHIFT                                                                     0x0
+#define SDMA1_SDMA_VM_CTX_HI__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA1_SDMA_ACTIVE_FCN_ID
+#define SDMA1_SDMA_ACTIVE_FCN_ID__VFID__SHIFT                                                                 0x0
+#define SDMA1_SDMA_ACTIVE_FCN_ID__VF__SHIFT                                                                   0x7
+#define SDMA1_SDMA_ACTIVE_FCN_ID__RESERVED__SHIFT                                                             0x8
+#define SDMA1_SDMA_ACTIVE_FCN_ID__VFID_MASK                                                                   0x0000001FL
+#define SDMA1_SDMA_ACTIVE_FCN_ID__VF_MASK                                                                     0x00000080L
+#define SDMA1_SDMA_ACTIVE_FCN_ID__RESERVED_MASK                                                               0xFFFFFF00L
+//SDMA1_SDMA_VM_CTX_CNTL
+#define SDMA1_SDMA_VM_CTX_CNTL__PRIV__SHIFT                                                                   0x0
+#define SDMA1_SDMA_VM_CTX_CNTL__VMID__SHIFT                                                                   0x4
+#define SDMA1_SDMA_VM_CTX_CNTL__MEM_PHY__SHIFT                                                                0x8
+#define SDMA1_SDMA_VM_CTX_CNTL__MEM_MTYPE__SHIFT                                                              0xa
+#define SDMA1_SDMA_VM_CTX_CNTL__BUSY_STATUS_REPORT_ENABLE__SHIFT                                              0x10
+#define SDMA1_SDMA_VM_CTX_CNTL__PRIV_MASK                                                                     0x00000001L
+#define SDMA1_SDMA_VM_CTX_CNTL__VMID_MASK                                                                     0x000000F0L
+#define SDMA1_SDMA_VM_CTX_CNTL__MEM_PHY_MASK                                                                  0x00000300L
+#define SDMA1_SDMA_VM_CTX_CNTL__MEM_MTYPE_MASK                                                                0x00000C00L
+#define SDMA1_SDMA_VM_CTX_CNTL__BUSY_STATUS_REPORT_ENABLE_MASK                                                0x00010000L
+//SDMA1_SDMA_VIRT_RESET_REQ
+#define SDMA1_SDMA_VIRT_RESET_REQ__VF__SHIFT                                                                  0x0
+#define SDMA1_SDMA_VIRT_RESET_REQ__PF__SHIFT                                                                  0x1f
+#define SDMA1_SDMA_VIRT_RESET_REQ__VF_MASK                                                                    0x7FFFFFFFL
+#define SDMA1_SDMA_VIRT_RESET_REQ__PF_MASK                                                                    0x80000000L
+//SDMA1_SDMA_CONTEXT_REG_TYPE0
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_CNTL__SHIFT                                              0x0
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_BASE__SHIFT                                              0x1
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_BASE_HI__SHIFT                                           0x2
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_RPTR__SHIFT                                              0x3
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_RPTR_HI__SHIFT                                           0x4
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_WPTR__SHIFT                                              0x5
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_WPTR_HI__SHIFT                                           0x6
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_RPTR_ADDR_LO__SHIFT                                      0x7
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_RPTR_ADDR_HI__SHIFT                                      0x8
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_CNTL__SHIFT                                              0x9
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_RPTR__SHIFT                                              0xa
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_OFFSET__SHIFT                                            0xb
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_BASE_LO__SHIFT                                           0xc
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_BASE_HI__SHIFT                                           0xd
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_SIZE__SHIFT                                              0xe
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_DOORBELL__SHIFT                                             0xf
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_DOORBELL_LOG__SHIFT                                         0x10
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_DOORBELL_OFFSET__SHIFT                                      0x11
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_CSA_ADDR_LO__SHIFT                                          0x12
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_CSA_ADDR_HI__SHIFT                                          0x13
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_SCHEDULE_CNTL__SHIFT                                        0x14
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_SUB_REMAIN__SHIFT                                        0x15
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_PREEMPT__SHIFT                                              0x16
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_DUMMY_REG__SHIFT                                            0x17
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_WPTR_POLL_ADDR_LO__SHIFT                                 0x18
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_WPTR_POLL_ADDR_HI__SHIFT                                 0x19
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_AQL_CNTL__SHIFT                                          0x1a
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_MINOR_PTR_UPDATE__SHIFT                                     0x1b
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_CONTEXT_SWITCH_STATUS__SHIFT                                0x1e
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_MIDCMD_CNTL__SHIFT                                          0x1f
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_CNTL_MASK                                                0x00000001L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_BASE_MASK                                                0x00000002L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_BASE_HI_MASK                                             0x00000004L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_RPTR_MASK                                                0x00000008L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_RPTR_HI_MASK                                             0x00000010L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_WPTR_MASK                                                0x00000020L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_WPTR_HI_MASK                                             0x00000040L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_RPTR_ADDR_LO_MASK                                        0x00000080L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_RPTR_ADDR_HI_MASK                                        0x00000100L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_CNTL_MASK                                                0x00000200L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_RPTR_MASK                                                0x00000400L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_OFFSET_MASK                                              0x00000800L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_BASE_LO_MASK                                             0x00001000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_BASE_HI_MASK                                             0x00002000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_SIZE_MASK                                                0x00004000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_DOORBELL_MASK                                               0x00008000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_DOORBELL_LOG_MASK                                           0x00010000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_DOORBELL_OFFSET_MASK                                        0x00020000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_CSA_ADDR_LO_MASK                                            0x00040000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_CSA_ADDR_HI_MASK                                            0x00080000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_SCHEDULE_CNTL_MASK                                          0x00100000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_IB_SUB_REMAIN_MASK                                          0x00200000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_PREEMPT_MASK                                                0x00400000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_DUMMY_REG_MASK                                              0x00800000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_WPTR_POLL_ADDR_LO_MASK                                   0x01000000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_WPTR_POLL_ADDR_HI_MASK                                   0x02000000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_RB_AQL_CNTL_MASK                                            0x04000000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_MINOR_PTR_UPDATE_MASK                                       0x08000000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_CONTEXT_SWITCH_STATUS_MASK                                  0x40000000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE0__SDMA_QUEUE0_MIDCMD_CNTL_MASK                                            0x80000000L
+//SDMA1_SDMA_CONTEXT_REG_TYPE1
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA0__SHIFT                                         0x0
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA1__SHIFT                                         0x1
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA2__SHIFT                                         0x2
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA3__SHIFT                                         0x3
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA4__SHIFT                                         0x4
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA5__SHIFT                                         0x5
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA6__SHIFT                                         0x6
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA7__SHIFT                                         0x7
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA8__SHIFT                                         0x8
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA9__SHIFT                                         0x9
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA10__SHIFT                                        0xa
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_UTILIZATION_LO__SHIFT                                       0xb
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_UTILIZATION_HI__SHIFT                                       0xc
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_WAIT_UNSATISFIED_THD__SHIFT                                 0xd
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MQD_BASE_ADDR_LO__SHIFT                                     0xe
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MQD_BASE_ADDR_HI__SHIFT                                     0xf
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MQD_CONTROL__SHIFT                                          0x10
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_CONTEXT_STATUS__SHIFT                                       0x11
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_DUMMY_REG1__SHIFT                                           0x12
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_DUMMY_REG2__SHIFT                                           0x13
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__RESERVED__SHIFT                                                         0x14
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA0_MASK                                           0x00000001L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA1_MASK                                           0x00000002L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA2_MASK                                           0x00000004L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA3_MASK                                           0x00000008L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA4_MASK                                           0x00000010L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA5_MASK                                           0x00000020L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA6_MASK                                           0x00000040L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA7_MASK                                           0x00000080L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA8_MASK                                           0x00000100L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA9_MASK                                           0x00000200L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MIDCMD_DATA10_MASK                                          0x00000400L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_UTILIZATION_LO_MASK                                         0x00000800L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_UTILIZATION_HI_MASK                                         0x00001000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_WAIT_UNSATISFIED_THD_MASK                                   0x00002000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MQD_BASE_ADDR_LO_MASK                                       0x00004000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MQD_BASE_ADDR_HI_MASK                                       0x00008000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_MQD_CONTROL_MASK                                            0x00010000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_CONTEXT_STATUS_MASK                                         0x00020000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_DUMMY_REG1_MASK                                             0x00040000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__SDMA_QUEUE0_DUMMY_REG2_MASK                                             0x00080000L
+#define SDMA1_SDMA_CONTEXT_REG_TYPE1__RESERVED_MASK                                                           0xFFF00000L
+//SDMA1_SDMA_CONTEXT_REG_TYPE2
+#define SDMA1_SDMA_CONTEXT_REG_TYPE2__RESERVED__SHIFT                                                         0x0
+#define SDMA1_SDMA_CONTEXT_REG_TYPE2__RESERVED_MASK                                                           0xFFFFFFFFL
+//SDMA1_SDMA_PUB_REG_TYPE0
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_DEC_START__SHIFT                                                       0x0
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_MCU_MISC_CNTL__SHIFT                                                   0x1
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_UCODE_REV__SHIFT                                                       0x3
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_GLOBAL_TIMESTAMP_LO__SHIFT                                             0x5
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_GLOBAL_TIMESTAMP_HI__SHIFT                                             0x6
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_POWER_CNTL__SHIFT                                                      0xc
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_CNTL__SHIFT                                                            0xd
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_CHICKEN_BITS__SHIFT                                                    0xe
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_CACHE_CNTL__SHIFT                                                      0xf
+#define SDMA1_SDMA_PUB_REG_TYPE0__RESERVED__SHIFT                                                             0x15
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_DEC_START_MASK                                                         0x00000001L
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_MCU_MISC_CNTL_MASK                                                     0x00000002L
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_UCODE_REV_MASK                                                         0x00000008L
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_GLOBAL_TIMESTAMP_LO_MASK                                               0x00000020L
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_GLOBAL_TIMESTAMP_HI_MASK                                               0x00000040L
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_POWER_CNTL_MASK                                                        0x00001000L
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_CNTL_MASK                                                              0x00002000L
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_CHICKEN_BITS_MASK                                                      0x00004000L
+#define SDMA1_SDMA_PUB_REG_TYPE0__SDMA_CACHE_CNTL_MASK                                                        0x00008000L
+#define SDMA1_SDMA_PUB_REG_TYPE0__RESERVED_MASK                                                               0xFFE00000L
+//SDMA1_SDMA_PUB_REG_TYPE1
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_RB_RPTR_FETCH__SHIFT                                                   0x0
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_RB_RPTR_FETCH_HI__SHIFT                                                0x1
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_IB_OFFSET_FETCH__SHIFT                                                 0x2
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_PROGRAM__SHIFT                                                         0x3
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_STATUS_REG__SHIFT                                                      0x4
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_STATUS1_REG__SHIFT                                                     0x5
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_CNTL1__SHIFT                                                           0x6
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_HBM_PAGE_CONFIG__SHIFT                                                 0x7
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_FREEZE__SHIFT                                                          0x8
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_PROCESS_QUANTUM0__SHIFT                                                0x9
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_PROCESS_QUANTUM1__SHIFT                                                0xa
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_WATCHDOG_CNTL__SHIFT                                                   0xb
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_QUEUE_STATUS0__SHIFT                                                   0xc
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_QUEUE_STATUS1__SHIFT                                                   0xd
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_ID__SHIFT                                                              0xe
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_VERSION__SHIFT                                                         0xf
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_STATUS2_REG__SHIFT                                                     0x10
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_CNTL__SHIFT                                                     0x11
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_PERMS_CNTL__SHIFT                                               0x12
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_PREOP_LO__SHIFT                                                 0x13
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_PREOP_HI__SHIFT                                                 0x14
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_DCC_CNTL__SHIFT                                                        0x15
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_WARMUPL2_CNTL__SHIFT                                             0x16
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_CNTL__SHIFT                                                      0x17
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_WATERMK__SHIFT                                                   0x18
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_TIMEOUT__SHIFT                                                   0x19
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_PAGE__SHIFT                                                      0x1a
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_EXTERNAL_FROZEN__SHIFT                                                 0x1b
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_FREEZE_TRIGGER__SHIFT                                                  0x1c
+#define SDMA1_SDMA_PUB_REG_TYPE1__RESERVED__SHIFT                                                             0x1d
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_RB_RPTR_FETCH_MASK                                                     0x00000001L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_RB_RPTR_FETCH_HI_MASK                                                  0x00000002L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_IB_OFFSET_FETCH_MASK                                                   0x00000004L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_PROGRAM_MASK                                                           0x00000008L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_STATUS_REG_MASK                                                        0x00000010L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_STATUS1_REG_MASK                                                       0x00000020L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_CNTL1_MASK                                                             0x00000040L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_HBM_PAGE_CONFIG_MASK                                                   0x00000080L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_FREEZE_MASK                                                            0x00000100L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_PROCESS_QUANTUM0_MASK                                                  0x00000200L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_PROCESS_QUANTUM1_MASK                                                  0x00000400L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_WATCHDOG_CNTL_MASK                                                     0x00000800L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_QUEUE_STATUS0_MASK                                                     0x00001000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_QUEUE_STATUS1_MASK                                                     0x00002000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_ID_MASK                                                                0x00004000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_VERSION_MASK                                                           0x00008000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_STATUS2_REG_MASK                                                       0x00010000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_CNTL_MASK                                                       0x00020000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_PERMS_CNTL_MASK                                                 0x00040000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_PREOP_LO_MASK                                                   0x00080000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_ATOMIC_PREOP_HI_MASK                                                   0x00100000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_DCC_CNTL_MASK                                                          0x00200000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_WARMUPL2_CNTL_MASK                                               0x00400000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_CNTL_MASK                                                        0x00800000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_WATERMK_MASK                                                     0x01000000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_TIMEOUT_MASK                                                     0x02000000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_UTCL1_PAGE_MASK                                                        0x04000000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_EXTERNAL_FROZEN_MASK                                                   0x08000000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__SDMA_FREEZE_TRIGGER_MASK                                                    0x10000000L
+#define SDMA1_SDMA_PUB_REG_TYPE1__RESERVED_MASK                                                               0xE0000000L
+//SDMA1_SDMA_PUB_REG_TYPE2
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_CACHE_CNTL__SHIFT                                                0x0
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_RD_STATUS__SHIFT                                                 0x1
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_WR_STATUS__SHIFT                                                 0x2
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV0__SHIFT                                                      0x3
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV1__SHIFT                                                      0x4
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV2__SHIFT                                                      0x5
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_RD_XNACK0__SHIFT                                                 0x6
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_RD_XNACK1__SHIFT                                                 0x7
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_WR_XNACK0__SHIFT                                                 0x8
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_WR_XNACK1__SHIFT                                                 0x9
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_CHICKEN_BITS_2__SHIFT                                                  0xa
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_STATUS3_REG__SHIFT                                                     0xb
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_GLOBAL_QUANTUM__SHIFT                                                  0xc
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_ERROR_LOG__SHIFT                                                       0xd
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG0__SHIFT                                                  0xe
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG1__SHIFT                                                  0xf
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG2__SHIFT                                                  0x10
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG3__SHIFT                                                  0x11
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_MCU_COUNTER__SHIFT                                                     0x12
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_CRD_CNTL__SHIFT                                                        0x13
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_RLC_CGCG_CTRL__SHIFT                                                   0x14
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_GPU_IOV_VIOLATION_LOG__SHIFT                                           0x15
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_AQL_STATUS__SHIFT                                                      0x17
+#define SDMA1_SDMA_PUB_REG_TYPE2__RESERVED__SHIFT                                                             0x1f
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_CACHE_CNTL_MASK                                                  0x00000001L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_RD_STATUS_MASK                                                   0x00000002L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_WR_STATUS_MASK                                                   0x00000004L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV0_MASK                                                        0x00000008L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV1_MASK                                                        0x00000010L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_INV2_MASK                                                        0x00000020L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_RD_XNACK0_MASK                                                   0x00000040L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_RD_XNACK1_MASK                                                   0x00000080L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_WR_XNACK0_MASK                                                   0x00000100L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_UTCL1_WR_XNACK1_MASK                                                   0x00000200L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_CHICKEN_BITS_2_MASK                                                    0x00000400L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_STATUS3_REG_MASK                                                       0x00000800L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_GLOBAL_QUANTUM_MASK                                                    0x00001000L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_ERROR_LOG_MASK                                                         0x00002000L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG0_MASK                                                    0x00004000L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG1_MASK                                                    0x00008000L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG2_MASK                                                    0x00010000L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_PUB_DUMMY_REG3_MASK                                                    0x00020000L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_MCU_COUNTER_MASK                                                       0x00040000L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_CRD_CNTL_MASK                                                          0x00080000L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_RLC_CGCG_CTRL_MASK                                                     0x00100000L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_GPU_IOV_VIOLATION_LOG_MASK                                             0x00200000L
+#define SDMA1_SDMA_PUB_REG_TYPE2__SDMA_AQL_STATUS_MASK                                                        0x00800000L
+#define SDMA1_SDMA_PUB_REG_TYPE2__RESERVED_MASK                                                               0x80000000L
+//SDMA1_SDMA_PUB_REG_TYPE3
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_TLBI_GCR_CNTL__SHIFT                                                   0x0
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_INT_STATUS__SHIFT                                                      0x1
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_GPU_IOV_VIOLATION_LOG2__SHIFT                                          0x2
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_INVALID_ADDR_LO__SHIFT                                                 0x3
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_INVALID_ADDR_HI__SHIFT                                                 0x4
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_INVALID_ADDR_SRC__SHIFT                                                0x5
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_CLOCK_GATING_STATUS__SHIFT                                             0x6
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_STATUS4_REG__SHIFT                                                     0x7
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_SCRATCH_RAM_DATA__SHIFT                                                0x8
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_SCRATCH_RAM_ADDR__SHIFT                                                0x9
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_TIMESTAMP_CNTL__SHIFT                                                  0xa
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_STATUS5_REG__SHIFT                                                     0xb
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_QUEUE_RESET_REQ__SHIFT                                                 0xc
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_QUEUE_DEQUEUE_REQUEST__SHIFT                                           0xd
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_STATUS6_REG__SHIFT                                                     0xe
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_CE_CTRL__SHIFT                                                         0xf
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_RAS_STATUS__SHIFT                                                      0x10
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_POISON_INFO__SHIFT                                                     0x11
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_MEMHUB_CNTL__SHIFT                                                     0x12
+#define SDMA1_SDMA_PUB_REG_TYPE3__RESERVED__SHIFT                                                             0x13
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_TLBI_GCR_CNTL_MASK                                                     0x00000001L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_INT_STATUS_MASK                                                        0x00000002L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_GPU_IOV_VIOLATION_LOG2_MASK                                            0x00000004L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_INVALID_ADDR_LO_MASK                                                   0x00000008L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_INVALID_ADDR_HI_MASK                                                   0x00000010L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_INVALID_ADDR_SRC_MASK                                                  0x00000020L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_CLOCK_GATING_STATUS_MASK                                               0x00000040L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_STATUS4_REG_MASK                                                       0x00000080L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_SCRATCH_RAM_DATA_MASK                                                  0x00000100L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_SCRATCH_RAM_ADDR_MASK                                                  0x00000200L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_TIMESTAMP_CNTL_MASK                                                    0x00000400L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_STATUS5_REG_MASK                                                       0x00000800L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_QUEUE_RESET_REQ_MASK                                                   0x00001000L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_QUEUE_DEQUEUE_REQUEST_MASK                                             0x00002000L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_STATUS6_REG_MASK                                                       0x00004000L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_CE_CTRL_MASK                                                           0x00008000L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_RAS_STATUS_MASK                                                        0x00010000L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_POISON_INFO_MASK                                                       0x00020000L
+#define SDMA1_SDMA_PUB_REG_TYPE3__SDMA_MEMHUB_CNTL_MASK                                                       0x00040000L
+#define SDMA1_SDMA_PUB_REG_TYPE3__RESERVED_MASK                                                               0xFFF80000L
+//SDMA1_SDMA_VM_CNTL
+#define SDMA1_SDMA_VM_CNTL__CMD__SHIFT                                                                        0x0
+#define SDMA1_SDMA_VM_CNTL__CMD_MASK                                                                          0x0000000FL
+//SDMA1_SDMA_MCU_CNTL
+#define SDMA1_SDMA_MCU_CNTL__HALT__SHIFT                                                                      0x0
+#define SDMA1_SDMA_MCU_CNTL__RESET__SHIFT                                                                     0x1
+#define SDMA1_SDMA_MCU_CNTL__DBG_SELECT_BITS__SHIFT                                                           0x2
+#define SDMA1_SDMA_MCU_CNTL__HALT_MASK                                                                        0x00000001L
+#define SDMA1_SDMA_MCU_CNTL__RESET_MASK                                                                       0x00000002L
+#define SDMA1_SDMA_MCU_CNTL__DBG_SELECT_BITS_MASK                                                             0x000000FCL
+//SDMA1_SDMA_IC_BASE_LO
+#define SDMA1_SDMA_IC_BASE_LO__IC_BASE_LO__SHIFT                                                              0xc
+#define SDMA1_SDMA_IC_BASE_LO__IC_BASE_LO_MASK                                                                0xFFFFF000L
+//SDMA1_SDMA_IC_BASE_HI
+#define SDMA1_SDMA_IC_BASE_HI__IC_BASE_HI__SHIFT                                                              0x0
+#define SDMA1_SDMA_IC_BASE_HI__IC_BASE_HI_MASK                                                                0xFFFFFFFFL
+//SDMA1_SDMA_IC_BASE_CNTL
+#define SDMA1_SDMA_IC_BASE_CNTL__VMID__SHIFT                                                                  0x0
+#define SDMA1_SDMA_IC_BASE_CNTL__MTYPE__SHIFT                                                                 0x4
+#define SDMA1_SDMA_IC_BASE_CNTL__EXE_DISABLE__SHIFT                                                           0x17
+#define SDMA1_SDMA_IC_BASE_CNTL__TEMPORAL_HINT__SHIFT                                                         0x18
+#define SDMA1_SDMA_IC_BASE_CNTL__SCOPE__SHIFT                                                                 0x1c
+#define SDMA1_SDMA_IC_BASE_CNTL__VMID_MASK                                                                    0x0000000FL
+#define SDMA1_SDMA_IC_BASE_CNTL__MTYPE_MASK                                                                   0x00000030L
+#define SDMA1_SDMA_IC_BASE_CNTL__EXE_DISABLE_MASK                                                             0x00800000L
+#define SDMA1_SDMA_IC_BASE_CNTL__TEMPORAL_HINT_MASK                                                           0x07000000L
+#define SDMA1_SDMA_IC_BASE_CNTL__SCOPE_MASK                                                                   0x30000000L
+//SDMA1_SDMA_IC_OP_CNTL
+#define SDMA1_SDMA_IC_OP_CNTL__INVALIDATE_CACHE__SHIFT                                                        0x0
+#define SDMA1_SDMA_IC_OP_CNTL__PRIME_ICACHE__SHIFT                                                            0x4
+#define SDMA1_SDMA_IC_OP_CNTL__ICACHE_PRIMED__SHIFT                                                           0x5
+#define SDMA1_SDMA_IC_OP_CNTL__INVALIDATE_CACHE_MASK                                                          0x00000001L
+#define SDMA1_SDMA_IC_OP_CNTL__PRIME_ICACHE_MASK                                                              0x00000010L
+#define SDMA1_SDMA_IC_OP_CNTL__ICACHE_PRIMED_MASK                                                             0x00000020L
+//SDMA1_SDMA_IC_CNTL
+#define SDMA1_SDMA_IC_CNTL__CID_SEL__SHIFT                                                                    0x0
+#define SDMA1_SDMA_IC_CNTL__GPA__SHIFT                                                                        0x2
+#define SDMA1_SDMA_IC_CNTL__UCODE_VF_OVERRIDE__SHIFT                                                          0x4
+#define SDMA1_SDMA_IC_CNTL__AUTO_PRIME_ICACHE__SHIFT                                                          0x5
+#define SDMA1_SDMA_IC_CNTL__CID_SEL_MASK                                                                      0x00000001L
+#define SDMA1_SDMA_IC_CNTL__GPA_MASK                                                                          0x0000000CL
+#define SDMA1_SDMA_IC_CNTL__UCODE_VF_OVERRIDE_MASK                                                            0x00000010L
+#define SDMA1_SDMA_IC_CNTL__AUTO_PRIME_ICACHE_MASK                                                            0x00000020L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmapspdec:1
+//SDMA1_SDMA_MCU_DM_FROM_RST_ADDR_OFFSET
+#define SDMA1_SDMA_MCU_DM_FROM_RST_ADDR_OFFSET__DATA__SHIFT                                                   0x0
+#define SDMA1_SDMA_MCU_DM_FROM_RST_ADDR_OFFSET__DATA_MASK                                                     0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmaperfsdec:1
+//SDMA1_SDMA_PERFCNT_PERFCOUNTER0_CFG
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                  0x0
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                              0x8
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                 0x18
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                    0x1c
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                     0x1d
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                    0x000000FFL
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                0x0000FF00L
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                   0x0F000000L
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER0_CFG__ENABLE_MASK                                                      0x10000000L
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER0_CFG__CLEAR_MASK                                                       0x20000000L
+//SDMA1_SDMA_PERFCNT_PERFCOUNTER1_CFG
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                  0x0
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                              0x8
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                 0x18
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                    0x1c
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                     0x1d
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                    0x000000FFL
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                0x0000FF00L
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                   0x0F000000L
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER1_CFG__ENABLE_MASK                                                      0x10000000L
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER1_CFG__CLEAR_MASK                                                       0x20000000L
+//SDMA1_SDMA_PERFCNT_PERFCOUNTER2_CFG
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                  0x0
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                              0x8
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                 0x18
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                    0x1c
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                     0x1d
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                    0x000000FFL
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                0x0000FF00L
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                   0x0F000000L
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER2_CFG__ENABLE_MASK                                                      0x10000000L
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER2_CFG__CLEAR_MASK                                                       0x20000000L
+//SDMA1_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                  0x0
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                        0x8
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                         0x10
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                           0x18
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                            0x19
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                 0x1a
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                    0x0000000FL
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                          0x0000FF00L
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                           0x00FF0000L
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                             0x01000000L
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                              0x02000000L
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                   0x04000000L
+//SDMA1_SDMA_PERFCNT_MISC_CNTL
+#define SDMA1_SDMA_PERFCNT_MISC_CNTL__CMD_OP__SHIFT                                                           0x0
+#define SDMA1_SDMA_PERFCNT_MISC_CNTL__CMD_OP_MASK                                                             0x0000FFFFL
+//SDMA1_SDMA_PERFCOUNTER0_SELECT
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                       0x0
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                      0xa
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                       0x14
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT__CNTR_MODE1__SHIFT                                                     0x18
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT__CNTR_MODE0__SHIFT                                                     0x1c
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                         0x000003FFL
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                        0x000FFC00L
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                         0x00F00000L
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT__CNTR_MODE1_MASK                                                       0x0F000000L
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT__CNTR_MODE0_MASK                                                       0xF0000000L
+//SDMA1_SDMA_PERFCOUNTER0_SELECT1
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                     0xa
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT1__CNTR_MODE3__SHIFT                                                    0x18
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT1__CNTR_MODE2__SHIFT                                                    0x1c
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                       0x000003FFL
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                       0x000FFC00L
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT1__CNTR_MODE3_MASK                                                      0x0F000000L
+#define SDMA1_SDMA_PERFCOUNTER0_SELECT1__CNTR_MODE2_MASK                                                      0xF0000000L
+//SDMA1_SDMA_PERFCOUNTER1_SELECT
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                       0x0
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                      0xa
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                       0x14
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT__CNTR_MODE1__SHIFT                                                     0x18
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT__CNTR_MODE0__SHIFT                                                     0x1c
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                         0x000003FFL
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                        0x000FFC00L
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                         0x00F00000L
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT__CNTR_MODE1_MASK                                                       0x0F000000L
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT__CNTR_MODE0_MASK                                                       0xF0000000L
+//SDMA1_SDMA_PERFCOUNTER1_SELECT1
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                     0xa
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT1__CNTR_MODE3__SHIFT                                                    0x18
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT1__CNTR_MODE2__SHIFT                                                    0x1c
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                       0x000003FFL
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                       0x000FFC00L
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT1__CNTR_MODE3_MASK                                                      0x0F000000L
+#define SDMA1_SDMA_PERFCOUNTER1_SELECT1__CNTR_MODE2_MASK                                                      0xF0000000L
+//SDMA1_SDMA_PERFCOUNTER2_SELECT
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                       0x0
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                      0xa
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT__SPM_MODE__SHIFT                                                       0x14
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT__CNTR_MODE1__SHIFT                                                     0x18
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT__CNTR_MODE0__SHIFT                                                     0x1c
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                         0x000003FFL
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                        0x000FFC00L
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT__SPM_MODE_MASK                                                         0x00F00000L
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT__CNTR_MODE1_MASK                                                       0x0F000000L
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT__CNTR_MODE0_MASK                                                       0xF0000000L
+//SDMA1_SDMA_PERFCOUNTER2_SELECT1
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                     0xa
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT1__CNTR_MODE3__SHIFT                                                    0x18
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT1__CNTR_MODE2__SHIFT                                                    0x1c
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                       0x000003FFL
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                       0x000FFC00L
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT1__CNTR_MODE3_MASK                                                      0x0F000000L
+#define SDMA1_SDMA_PERFCOUNTER2_SELECT1__CNTR_MODE2_MASK                                                      0xF0000000L
+//SDMA1_SDMA_PERFCOUNTER3_SELECT
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                       0x0
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                      0xa
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT__SPM_MODE__SHIFT                                                       0x14
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT__CNTR_MODE1__SHIFT                                                     0x18
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT__CNTR_MODE0__SHIFT                                                     0x1c
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                         0x000003FFL
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                        0x000FFC00L
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT__SPM_MODE_MASK                                                         0x00F00000L
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT__CNTR_MODE1_MASK                                                       0x0F000000L
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT__CNTR_MODE0_MASK                                                       0xF0000000L
+//SDMA1_SDMA_PERFCOUNTER3_SELECT1
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                     0xa
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT1__CNTR_MODE3__SHIFT                                                    0x18
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT1__CNTR_MODE2__SHIFT                                                    0x1c
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                       0x000003FFL
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                       0x000FFC00L
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT1__CNTR_MODE3_MASK                                                      0x0F000000L
+#define SDMA1_SDMA_PERFCOUNTER3_SELECT1__CNTR_MODE2_MASK                                                      0xF0000000L
+//SDMA1_SDMA_PERFCOUNTER4_SELECT
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                       0x0
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT__PERF_SEL1__SHIFT                                                      0xa
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT__SPM_MODE__SHIFT                                                       0x14
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT__CNTR_MODE1__SHIFT                                                     0x18
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT__CNTR_MODE0__SHIFT                                                     0x1c
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                         0x000003FFL
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT__PERF_SEL1_MASK                                                        0x000FFC00L
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT__SPM_MODE_MASK                                                         0x00F00000L
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT__CNTR_MODE1_MASK                                                       0x0F000000L
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT__CNTR_MODE0_MASK                                                       0xF0000000L
+//SDMA1_SDMA_PERFCOUNTER4_SELECT1
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT1__PERF_SEL2__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT1__PERF_SEL3__SHIFT                                                     0xa
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT1__CNTR_MODE3__SHIFT                                                    0x18
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT1__CNTR_MODE2__SHIFT                                                    0x1c
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT1__PERF_SEL2_MASK                                                       0x000003FFL
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT1__PERF_SEL3_MASK                                                       0x000FFC00L
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT1__CNTR_MODE3_MASK                                                      0x0F000000L
+#define SDMA1_SDMA_PERFCOUNTER4_SELECT1__CNTR_MODE2_MASK                                                      0xF0000000L
+//SDMA1_SDMA_PERFCOUNTER5_SELECT
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                       0x0
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT__PERF_SEL1__SHIFT                                                      0xa
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT__SPM_MODE__SHIFT                                                       0x14
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT__CNTR_MODE1__SHIFT                                                     0x18
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT__CNTR_MODE0__SHIFT                                                     0x1c
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                         0x000003FFL
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT__PERF_SEL1_MASK                                                        0x000FFC00L
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT__SPM_MODE_MASK                                                         0x00F00000L
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT__CNTR_MODE1_MASK                                                       0x0F000000L
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT__CNTR_MODE0_MASK                                                       0xF0000000L
+//SDMA1_SDMA_PERFCOUNTER5_SELECT1
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT1__PERF_SEL2__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT1__PERF_SEL3__SHIFT                                                     0xa
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT1__CNTR_MODE3__SHIFT                                                    0x18
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT1__CNTR_MODE2__SHIFT                                                    0x1c
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT1__PERF_SEL2_MASK                                                       0x000003FFL
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT1__PERF_SEL3_MASK                                                       0x000FFC00L
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT1__CNTR_MODE3_MASK                                                      0x0F000000L
+#define SDMA1_SDMA_PERFCOUNTER5_SELECT1__CNTR_MODE2_MASK                                                      0xF0000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmaperfddec:1
+//SDMA1_SDMA_PERFCNT_PERFCOUNTER_LO
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                  0x0
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_LO__COUNTER_LO_MASK                                                    0xFFFFFFFFL
+//SDMA1_SDMA_PERFCNT_PERFCOUNTER_HI
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                  0x0
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                               0x10
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_HI__COUNTER_HI_MASK                                                    0x0000FFFFL
+#define SDMA1_SDMA_PERFCNT_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                 0xFFFF0000L
+//SDMA1_SDMA_PERFCOUNTER0_LO
+#define SDMA1_SDMA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                       0xFFFFFFFFL
+//SDMA1_SDMA_PERFCOUNTER0_HI
+#define SDMA1_SDMA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                       0xFFFFFFFFL
+//SDMA1_SDMA_PERFCOUNTER1_LO
+#define SDMA1_SDMA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                       0xFFFFFFFFL
+//SDMA1_SDMA_PERFCOUNTER1_HI
+#define SDMA1_SDMA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                       0xFFFFFFFFL
+//SDMA1_SDMA_PERFCOUNTER2_LO
+#define SDMA1_SDMA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                       0xFFFFFFFFL
+//SDMA1_SDMA_PERFCOUNTER2_HI
+#define SDMA1_SDMA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                       0xFFFFFFFFL
+//SDMA1_SDMA_PERFCOUNTER3_LO
+#define SDMA1_SDMA_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                       0xFFFFFFFFL
+//SDMA1_SDMA_PERFCOUNTER3_HI
+#define SDMA1_SDMA_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                       0xFFFFFFFFL
+//SDMA1_SDMA_PERFCOUNTER4_LO
+#define SDMA1_SDMA_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                       0xFFFFFFFFL
+//SDMA1_SDMA_PERFCOUNTER4_HI
+#define SDMA1_SDMA_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                       0xFFFFFFFFL
+//SDMA1_SDMA_PERFCOUNTER5_LO
+#define SDMA1_SDMA_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                       0xFFFFFFFFL
+//SDMA1_SDMA_PERFCOUNTER5_HI
+#define SDMA1_SDMA_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                     0x0
+#define SDMA1_SDMA_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                       0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_sdma_sdmapwrdec:1
+//SDMA1_GFX_ICG_SDMA_CTRL
+#define SDMA1_GFX_ICG_SDMA_CTRL__REG_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                            0x1
+#define SDMA1_GFX_ICG_SDMA_CTRL__PTR_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                            0x2
+#define SDMA1_GFX_ICG_SDMA_CTRL__PIO_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                            0x3
+#define SDMA1_GFX_ICG_SDMA_CTRL__MCU_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                            0x4
+#define SDMA1_GFX_ICG_SDMA_CTRL__COPY_ENG_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                       0x5
+#define SDMA1_GFX_ICG_SDMA_CTRL__SERVE_ENG_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                      0x6
+#define SDMA1_GFX_ICG_SDMA_CTRL__CMD_FETCH_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                      0x7
+#define SDMA1_GFX_ICG_SDMA_CTRL__GU_MEMREQ_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                      0x8
+#define SDMA1_GFX_ICG_SDMA_CTRL__INV_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                            0x9
+#define SDMA1_GFX_ICG_SDMA_CTRL__GU_CACHE_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                       0xa
+#define SDMA1_GFX_ICG_SDMA_CTRL__IC_CACHE_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                       0xb
+#define SDMA1_GFX_ICG_SDMA_CTRL__MEM_CHNL_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                       0xc
+#define SDMA1_GFX_ICG_SDMA_CTRL__PERF_CNTR_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                      0xd
+#define SDMA1_GFX_ICG_SDMA_CTRL__CORE_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                           0xe
+#define SDMA1_GFX_ICG_SDMA_CTRL__MEM_CHNL_CESE_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                  0xf
+#define SDMA1_GFX_ICG_SDMA_CTRL__FE_ENG_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                         0x10
+#define SDMA1_GFX_ICG_SDMA_CTRL__MEM_CHNL_CESEFE_MGCG_CLK_SOFT_OVERRIDE__SHIFT                                0x11
+#define SDMA1_GFX_ICG_SDMA_CTRL__MEM_CHNL_FE_WARMUP_MGCG_CLK_SOFT_OVERRIDE__SHIFT                             0x12
+#define SDMA1_GFX_ICG_SDMA_CTRL__MGCG_CLK_HYST__SHIFT                                                         0x18
+#define SDMA1_GFX_ICG_SDMA_CTRL__REG_MGCG_CLK_SOFT_OVERRIDE_MASK                                              0x00000002L
+#define SDMA1_GFX_ICG_SDMA_CTRL__PTR_MGCG_CLK_SOFT_OVERRIDE_MASK                                              0x00000004L
+#define SDMA1_GFX_ICG_SDMA_CTRL__PIO_MGCG_CLK_SOFT_OVERRIDE_MASK                                              0x00000008L
+#define SDMA1_GFX_ICG_SDMA_CTRL__MCU_MGCG_CLK_SOFT_OVERRIDE_MASK                                              0x00000010L
+#define SDMA1_GFX_ICG_SDMA_CTRL__COPY_ENG_MGCG_CLK_SOFT_OVERRIDE_MASK                                         0x00000020L
+#define SDMA1_GFX_ICG_SDMA_CTRL__SERVE_ENG_MGCG_CLK_SOFT_OVERRIDE_MASK                                        0x00000040L
+#define SDMA1_GFX_ICG_SDMA_CTRL__CMD_FETCH_MGCG_CLK_SOFT_OVERRIDE_MASK                                        0x00000080L
+#define SDMA1_GFX_ICG_SDMA_CTRL__GU_MEMREQ_MGCG_CLK_SOFT_OVERRIDE_MASK                                        0x00000100L
+#define SDMA1_GFX_ICG_SDMA_CTRL__INV_MGCG_CLK_SOFT_OVERRIDE_MASK                                              0x00000200L
+#define SDMA1_GFX_ICG_SDMA_CTRL__GU_CACHE_MGCG_CLK_SOFT_OVERRIDE_MASK                                         0x00000400L
+#define SDMA1_GFX_ICG_SDMA_CTRL__IC_CACHE_MGCG_CLK_SOFT_OVERRIDE_MASK                                         0x00000800L
+#define SDMA1_GFX_ICG_SDMA_CTRL__MEM_CHNL_MGCG_CLK_SOFT_OVERRIDE_MASK                                         0x00001000L
+#define SDMA1_GFX_ICG_SDMA_CTRL__PERF_CNTR_MGCG_CLK_SOFT_OVERRIDE_MASK                                        0x00002000L
+#define SDMA1_GFX_ICG_SDMA_CTRL__CORE_MGCG_CLK_SOFT_OVERRIDE_MASK                                             0x00004000L
+#define SDMA1_GFX_ICG_SDMA_CTRL__MEM_CHNL_CESE_MGCG_CLK_SOFT_OVERRIDE_MASK                                    0x00008000L
+#define SDMA1_GFX_ICG_SDMA_CTRL__FE_ENG_MGCG_CLK_SOFT_OVERRIDE_MASK                                           0x00010000L
+#define SDMA1_GFX_ICG_SDMA_CTRL__MEM_CHNL_CESEFE_MGCG_CLK_SOFT_OVERRIDE_MASK                                  0x00020000L
+#define SDMA1_GFX_ICG_SDMA_CTRL__MEM_CHNL_FE_WARMUP_MGCG_CLK_SOFT_OVERRIDE_MASK                               0x00040000L
+#define SDMA1_GFX_ICG_SDMA_CTRL__MGCG_CLK_HYST_MASK                                                           0xFF000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_grbmdec
+//GRBM_CNTL
+#define GRBM_CNTL__READ_TIMEOUT__SHIFT                                                                        0x0
+#define GRBM_CNTL__SED_READ_TIMEOUT__SHIFT                                                                    0x10
+#define GRBM_CNTL__REPORT_LAST_RDERR__SHIFT                                                                   0x1f
+#define GRBM_CNTL__READ_TIMEOUT_MASK                                                                          0x0000FFFFL
+#define GRBM_CNTL__SED_READ_TIMEOUT_MASK                                                                      0x0FFF0000L
+#define GRBM_CNTL__REPORT_LAST_RDERR_MASK                                                                     0x80000000L
+//GRBM_SKEW_CNTL
+#define GRBM_SKEW_CNTL__SKEW_TOP_THRESHOLD__SHIFT                                                             0x0
+#define GRBM_SKEW_CNTL__SKEW_COUNT__SHIFT                                                                     0x6
+#define GRBM_SKEW_CNTL__SKEW_TOP_THRESHOLD_MASK                                                               0x0000003FL
+#define GRBM_SKEW_CNTL__SKEW_COUNT_MASK                                                                       0x00000FC0L
+//GRBM_STATUS2
+#define GRBM_STATUS2__ME0PIPE1_CMDFIFO_AVAIL__SHIFT                                                           0x0
+#define GRBM_STATUS2__ME0PIPE1_CF_RQ_PENDING__SHIFT                                                           0x4
+#define GRBM_STATUS2__ME0PIPE1_PF_RQ_PENDING__SHIFT                                                           0x5
+#define GRBM_STATUS2__ME1PIPE0_RQ_PENDING__SHIFT                                                              0x6
+#define GRBM_STATUS2__ME1PIPE1_RQ_PENDING__SHIFT                                                              0x7
+#define GRBM_STATUS2__ME1PIPE2_RQ_PENDING__SHIFT                                                              0x8
+#define GRBM_STATUS2__ME1PIPE3_RQ_PENDING__SHIFT                                                              0x9
+#define GRBM_STATUS2__RLC_RQ_PENDING__SHIFT                                                                   0xe
+#define GRBM_STATUS2__UTCL2_BUSY__SHIFT                                                                       0xf
+#define GRBM_STATUS2__CANE_BUSY__SHIFT                                                                        0x10
+#define GRBM_STATUS2__UTCL2_RQ_PENDING__SHIFT                                                                 0x12
+#define GRBM_STATUS2__SDMA_SCH_RQ_PENDING__SHIFT                                                              0x13
+#define GRBM_STATUS2__CANE_LINK_BUSY__SHIFT                                                                   0x14
+#define GRBM_STATUS2__SDMA_BUSY__SHIFT                                                                        0x15
+#define GRBM_STATUS2__SDMA0_RQ_PENDING__SHIFT                                                                 0x16
+#define GRBM_STATUS2__SDMA1_RQ_PENDING__SHIFT                                                                 0x17
+#define GRBM_STATUS2__RLC_BUSY__SHIFT                                                                         0x1a
+#define GRBM_STATUS2__TCP_BUSY__SHIFT                                                                         0x1b
+#define GRBM_STATUS2__CPF_BUSY__SHIFT                                                                         0x1c
+#define GRBM_STATUS2__CPC_BUSY__SHIFT                                                                         0x1d
+#define GRBM_STATUS2__CPG_BUSY__SHIFT                                                                         0x1e
+#define GRBM_STATUS2__ME0PIPE1_CMDFIFO_AVAIL_MASK                                                             0x0000000FL
+#define GRBM_STATUS2__ME0PIPE1_CF_RQ_PENDING_MASK                                                             0x00000010L
+#define GRBM_STATUS2__ME0PIPE1_PF_RQ_PENDING_MASK                                                             0x00000020L
+#define GRBM_STATUS2__ME1PIPE0_RQ_PENDING_MASK                                                                0x00000040L
+#define GRBM_STATUS2__ME1PIPE1_RQ_PENDING_MASK                                                                0x00000080L
+#define GRBM_STATUS2__ME1PIPE2_RQ_PENDING_MASK                                                                0x00000100L
+#define GRBM_STATUS2__ME1PIPE3_RQ_PENDING_MASK                                                                0x00000200L
+#define GRBM_STATUS2__RLC_RQ_PENDING_MASK                                                                     0x00004000L
+#define GRBM_STATUS2__UTCL2_BUSY_MASK                                                                         0x00008000L
+#define GRBM_STATUS2__CANE_BUSY_MASK                                                                          0x00010000L
+#define GRBM_STATUS2__UTCL2_RQ_PENDING_MASK                                                                   0x00040000L
+#define GRBM_STATUS2__SDMA_SCH_RQ_PENDING_MASK                                                                0x00080000L
+#define GRBM_STATUS2__CANE_LINK_BUSY_MASK                                                                     0x00100000L
+#define GRBM_STATUS2__SDMA_BUSY_MASK                                                                          0x00200000L
+#define GRBM_STATUS2__SDMA0_RQ_PENDING_MASK                                                                   0x00400000L
+#define GRBM_STATUS2__SDMA1_RQ_PENDING_MASK                                                                   0x00800000L
+#define GRBM_STATUS2__RLC_BUSY_MASK                                                                           0x04000000L
+#define GRBM_STATUS2__TCP_BUSY_MASK                                                                           0x08000000L
+#define GRBM_STATUS2__CPF_BUSY_MASK                                                                           0x10000000L
+#define GRBM_STATUS2__CPC_BUSY_MASK                                                                           0x20000000L
+#define GRBM_STATUS2__CPG_BUSY_MASK                                                                           0x40000000L
+//GRBM_PWR_CNTL
+#define GRBM_PWR_CNTL__ALL_REQ_TYPE__SHIFT                                                                    0x0
+#define GRBM_PWR_CNTL__GFX_REQ_TYPE__SHIFT                                                                    0x2
+#define GRBM_PWR_CNTL__ALL_RSP_TYPE__SHIFT                                                                    0x4
+#define GRBM_PWR_CNTL__GFX_RSP_TYPE__SHIFT                                                                    0x6
+#define GRBM_PWR_CNTL__GFX_REQ_EN__SHIFT                                                                      0xe
+#define GRBM_PWR_CNTL__ALL_REQ_EN__SHIFT                                                                      0xf
+#define GRBM_PWR_CNTL__ALL_REQ_TYPE_MASK                                                                      0x00000003L
+#define GRBM_PWR_CNTL__GFX_REQ_TYPE_MASK                                                                      0x0000000CL
+#define GRBM_PWR_CNTL__ALL_RSP_TYPE_MASK                                                                      0x00000030L
+#define GRBM_PWR_CNTL__GFX_RSP_TYPE_MASK                                                                      0x000000C0L
+#define GRBM_PWR_CNTL__GFX_REQ_EN_MASK                                                                        0x00004000L
+#define GRBM_PWR_CNTL__ALL_REQ_EN_MASK                                                                        0x00008000L
+//GRBM_STATUS
+#define GRBM_STATUS__ME0PIPE0_CMDFIFO_AVAIL__SHIFT                                                            0x0
+#define GRBM_STATUS__RSMU_RQ_PENDING__SHIFT                                                                   0x5
+#define GRBM_STATUS__SDMA_RQ_PENDING__SHIFT                                                                   0x6
+#define GRBM_STATUS__ME0PIPE0_CF_RQ_PENDING__SHIFT                                                            0x7
+#define GRBM_STATUS__ME0PIPE0_PF_RQ_PENDING__SHIFT                                                            0x8
+#define GRBM_STATUS__DB_CLEAN__SHIFT                                                                          0xc
+#define GRBM_STATUS__CB_CLEAN__SHIFT                                                                          0xd
+#define GRBM_STATUS__TA_BUSY__SHIFT                                                                           0xe
+#define GRBM_STATUS__GE_BUSY_NO_DMA__SHIFT                                                                    0x10
+#define GRBM_STATUS__SX_BUSY__SHIFT                                                                           0x14
+#define GRBM_STATUS__GE_BUSY__SHIFT                                                                           0x15
+#define GRBM_STATUS__SPI_BUSY__SHIFT                                                                          0x16
+#define GRBM_STATUS__BCI_BUSY__SHIFT                                                                          0x17
+#define GRBM_STATUS__SC_BUSY__SHIFT                                                                           0x18
+#define GRBM_STATUS__PA_BUSY__SHIFT                                                                           0x19
+#define GRBM_STATUS__DB_BUSY__SHIFT                                                                           0x1a
+#define GRBM_STATUS__ANY_ACTIVE__SHIFT                                                                        0x1b
+#define GRBM_STATUS__CP_COHERENCY_BUSY__SHIFT                                                                 0x1c
+#define GRBM_STATUS__CP_BUSY__SHIFT                                                                           0x1d
+#define GRBM_STATUS__CB_BUSY__SHIFT                                                                           0x1e
+#define GRBM_STATUS__GUI_ACTIVE__SHIFT                                                                        0x1f
+#define GRBM_STATUS__ME0PIPE0_CMDFIFO_AVAIL_MASK                                                              0x0000000FL
+#define GRBM_STATUS__RSMU_RQ_PENDING_MASK                                                                     0x00000020L
+#define GRBM_STATUS__SDMA_RQ_PENDING_MASK                                                                     0x00000040L
+#define GRBM_STATUS__ME0PIPE0_CF_RQ_PENDING_MASK                                                              0x00000080L
+#define GRBM_STATUS__ME0PIPE0_PF_RQ_PENDING_MASK                                                              0x00000100L
+#define GRBM_STATUS__DB_CLEAN_MASK                                                                            0x00001000L
+#define GRBM_STATUS__CB_CLEAN_MASK                                                                            0x00002000L
+#define GRBM_STATUS__TA_BUSY_MASK                                                                             0x00004000L
+#define GRBM_STATUS__GE_BUSY_NO_DMA_MASK                                                                      0x00010000L
+#define GRBM_STATUS__SX_BUSY_MASK                                                                             0x00100000L
+#define GRBM_STATUS__GE_BUSY_MASK                                                                             0x00200000L
+#define GRBM_STATUS__SPI_BUSY_MASK                                                                            0x00400000L
+#define GRBM_STATUS__BCI_BUSY_MASK                                                                            0x00800000L
+#define GRBM_STATUS__SC_BUSY_MASK                                                                             0x01000000L
+#define GRBM_STATUS__PA_BUSY_MASK                                                                             0x02000000L
+#define GRBM_STATUS__DB_BUSY_MASK                                                                             0x04000000L
+#define GRBM_STATUS__ANY_ACTIVE_MASK                                                                          0x08000000L
+#define GRBM_STATUS__CP_COHERENCY_BUSY_MASK                                                                   0x10000000L
+#define GRBM_STATUS__CP_BUSY_MASK                                                                             0x20000000L
+#define GRBM_STATUS__CB_BUSY_MASK                                                                             0x40000000L
+#define GRBM_STATUS__GUI_ACTIVE_MASK                                                                          0x80000000L
+//GRBM_STATUS_SE0
+#define GRBM_STATUS_SE0__DB_CLEAN__SHIFT                                                                      0x1
+#define GRBM_STATUS_SE0__CB_CLEAN__SHIFT                                                                      0x2
+#define GRBM_STATUS_SE0__UTCL1_BUSY__SHIFT                                                                    0x3
+#define GRBM_STATUS_SE0__TCP_BUSY__SHIFT                                                                      0x4
+#define GRBM_STATUS_SE0__GL1CC_BUSY__SHIFT                                                                    0x5
+#define GRBM_STATUS_SE0__GL1XCC_BUSY__SHIFT                                                                   0x6
+#define GRBM_STATUS_SE0__WGS_BUSY__SHIFT                                                                      0x14
+#define GRBM_STATUS_SE0__BCI_BUSY__SHIFT                                                                      0x16
+#define GRBM_STATUS_SE0__PA_BUSY__SHIFT                                                                       0x18
+#define GRBM_STATUS_SE0__TA_BUSY__SHIFT                                                                       0x19
+#define GRBM_STATUS_SE0__SX_BUSY__SHIFT                                                                       0x1a
+#define GRBM_STATUS_SE0__SPI_BUSY__SHIFT                                                                      0x1b
+#define GRBM_STATUS_SE0__SC_BUSY__SHIFT                                                                       0x1d
+#define GRBM_STATUS_SE0__DB_BUSY__SHIFT                                                                       0x1e
+#define GRBM_STATUS_SE0__CB_BUSY__SHIFT                                                                       0x1f
+#define GRBM_STATUS_SE0__DB_CLEAN_MASK                                                                        0x00000002L
+#define GRBM_STATUS_SE0__CB_CLEAN_MASK                                                                        0x00000004L
+#define GRBM_STATUS_SE0__UTCL1_BUSY_MASK                                                                      0x00000008L
+#define GRBM_STATUS_SE0__TCP_BUSY_MASK                                                                        0x00000010L
+#define GRBM_STATUS_SE0__GL1CC_BUSY_MASK                                                                      0x00000020L
+#define GRBM_STATUS_SE0__GL1XCC_BUSY_MASK                                                                     0x00000040L
+#define GRBM_STATUS_SE0__WGS_BUSY_MASK                                                                        0x00100000L
+#define GRBM_STATUS_SE0__BCI_BUSY_MASK                                                                        0x00400000L
+#define GRBM_STATUS_SE0__PA_BUSY_MASK                                                                         0x01000000L
+#define GRBM_STATUS_SE0__TA_BUSY_MASK                                                                         0x02000000L
+#define GRBM_STATUS_SE0__SX_BUSY_MASK                                                                         0x04000000L
+#define GRBM_STATUS_SE0__SPI_BUSY_MASK                                                                        0x08000000L
+#define GRBM_STATUS_SE0__SC_BUSY_MASK                                                                         0x20000000L
+#define GRBM_STATUS_SE0__DB_BUSY_MASK                                                                         0x40000000L
+#define GRBM_STATUS_SE0__CB_BUSY_MASK                                                                         0x80000000L
+//GRBM_STATUS_SE1
+#define GRBM_STATUS_SE1__DB_CLEAN__SHIFT                                                                      0x1
+#define GRBM_STATUS_SE1__CB_CLEAN__SHIFT                                                                      0x2
+#define GRBM_STATUS_SE1__UTCL1_BUSY__SHIFT                                                                    0x3
+#define GRBM_STATUS_SE1__TCP_BUSY__SHIFT                                                                      0x4
+#define GRBM_STATUS_SE1__GL1CC_BUSY__SHIFT                                                                    0x5
+#define GRBM_STATUS_SE1__GL1XCC_BUSY__SHIFT                                                                   0x6
+#define GRBM_STATUS_SE1__WGS_BUSY__SHIFT                                                                      0x14
+#define GRBM_STATUS_SE1__BCI_BUSY__SHIFT                                                                      0x16
+#define GRBM_STATUS_SE1__PA_BUSY__SHIFT                                                                       0x18
+#define GRBM_STATUS_SE1__TA_BUSY__SHIFT                                                                       0x19
+#define GRBM_STATUS_SE1__SX_BUSY__SHIFT                                                                       0x1a
+#define GRBM_STATUS_SE1__SPI_BUSY__SHIFT                                                                      0x1b
+#define GRBM_STATUS_SE1__SC_BUSY__SHIFT                                                                       0x1d
+#define GRBM_STATUS_SE1__DB_BUSY__SHIFT                                                                       0x1e
+#define GRBM_STATUS_SE1__CB_BUSY__SHIFT                                                                       0x1f
+#define GRBM_STATUS_SE1__DB_CLEAN_MASK                                                                        0x00000002L
+#define GRBM_STATUS_SE1__CB_CLEAN_MASK                                                                        0x00000004L
+#define GRBM_STATUS_SE1__UTCL1_BUSY_MASK                                                                      0x00000008L
+#define GRBM_STATUS_SE1__TCP_BUSY_MASK                                                                        0x00000010L
+#define GRBM_STATUS_SE1__GL1CC_BUSY_MASK                                                                      0x00000020L
+#define GRBM_STATUS_SE1__GL1XCC_BUSY_MASK                                                                     0x00000040L
+#define GRBM_STATUS_SE1__WGS_BUSY_MASK                                                                        0x00100000L
+#define GRBM_STATUS_SE1__BCI_BUSY_MASK                                                                        0x00400000L
+#define GRBM_STATUS_SE1__PA_BUSY_MASK                                                                         0x01000000L
+#define GRBM_STATUS_SE1__TA_BUSY_MASK                                                                         0x02000000L
+#define GRBM_STATUS_SE1__SX_BUSY_MASK                                                                         0x04000000L
+#define GRBM_STATUS_SE1__SPI_BUSY_MASK                                                                        0x08000000L
+#define GRBM_STATUS_SE1__SC_BUSY_MASK                                                                         0x20000000L
+#define GRBM_STATUS_SE1__DB_BUSY_MASK                                                                         0x40000000L
+#define GRBM_STATUS_SE1__CB_BUSY_MASK                                                                         0x80000000L
+//GRBM_STATUS3
+#define GRBM_STATUS3__GRBM_RLC_INTR_CREDIT_PENDING__SHIFT                                                     0x5
+#define GRBM_STATUS3__GRBM_CPF_INTR_CREDIT_PENDING__SHIFT                                                     0x7
+#define GRBM_STATUS3__MESPIPE0_RQ_PENDING__SHIFT                                                              0x8
+#define GRBM_STATUS3__MESPIPE1_RQ_PENDING__SHIFT                                                              0x9
+#define GRBM_STATUS3__CH_BUSY__SHIFT                                                                          0xe
+#define GRBM_STATUS3__GLARB_BUSY__SHIFT                                                                       0xf
+#define GRBM_STATUS3__GL1CC_BUSY__SHIFT                                                                       0x10
+#define GRBM_STATUS3__WGS_BUSY__SHIFT                                                                         0x11
+#define GRBM_STATUS3__GL1XCC_BUSY__SHIFT                                                                      0x1b
+#define GRBM_STATUS3__UTCL1_BUSY__SHIFT                                                                       0x1e
+#define GRBM_STATUS3__PMM_BUSY__SHIFT                                                                         0x1f
+#define GRBM_STATUS3__GRBM_RLC_INTR_CREDIT_PENDING_MASK                                                       0x00000020L
+#define GRBM_STATUS3__GRBM_CPF_INTR_CREDIT_PENDING_MASK                                                       0x00000080L
+#define GRBM_STATUS3__MESPIPE0_RQ_PENDING_MASK                                                                0x00000100L
+#define GRBM_STATUS3__MESPIPE1_RQ_PENDING_MASK                                                                0x00000200L
+#define GRBM_STATUS3__CH_BUSY_MASK                                                                            0x00004000L
+#define GRBM_STATUS3__GLARB_BUSY_MASK                                                                         0x00008000L
+#define GRBM_STATUS3__GL1CC_BUSY_MASK                                                                         0x00010000L
+#define GRBM_STATUS3__WGS_BUSY_MASK                                                                           0x00020000L
+#define GRBM_STATUS3__GL1XCC_BUSY_MASK                                                                        0x08000000L
+#define GRBM_STATUS3__UTCL1_BUSY_MASK                                                                         0x40000000L
+#define GRBM_STATUS3__PMM_BUSY_MASK                                                                           0x80000000L
+//GRBM_SOFT_RESET
+#define GRBM_SOFT_RESET__SOFT_RESET_CP__SHIFT                                                                 0x0
+#define GRBM_SOFT_RESET__SOFT_RESET_RLC__SHIFT                                                                0x2
+#define GRBM_SOFT_RESET__SOFT_RESET_UTCL2__SHIFT                                                              0xf
+#define GRBM_SOFT_RESET__SOFT_RESET_GFX__SHIFT                                                                0x10
+#define GRBM_SOFT_RESET__SOFT_RESET_CPF__SHIFT                                                                0x11
+#define GRBM_SOFT_RESET__SOFT_RESET_CPC__SHIFT                                                                0x12
+#define GRBM_SOFT_RESET__SOFT_RESET_CPG__SHIFT                                                                0x13
+#define GRBM_SOFT_RESET__SOFT_RESET_CAC__SHIFT                                                                0x14
+#define GRBM_SOFT_RESET__SOFT_RESET_CANE__SHIFT                                                               0x15
+#define GRBM_SOFT_RESET__SOFT_RESET_EA__SHIFT                                                                 0x16
+#define GRBM_SOFT_RESET__SOFT_RESET_SDMA0__SHIFT                                                              0x17
+#define GRBM_SOFT_RESET__SOFT_RESET_SDMA1__SHIFT                                                              0x18
+#define GRBM_SOFT_RESET__SOFT_RESET_CP_MASK                                                                   0x00000001L
+#define GRBM_SOFT_RESET__SOFT_RESET_RLC_MASK                                                                  0x00000004L
+#define GRBM_SOFT_RESET__SOFT_RESET_UTCL2_MASK                                                                0x00008000L
+#define GRBM_SOFT_RESET__SOFT_RESET_GFX_MASK                                                                  0x00010000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CPF_MASK                                                                  0x00020000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CPC_MASK                                                                  0x00040000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CPG_MASK                                                                  0x00080000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CAC_MASK                                                                  0x00100000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CANE_MASK                                                                 0x00200000L
+#define GRBM_SOFT_RESET__SOFT_RESET_EA_MASK                                                                   0x00400000L
+#define GRBM_SOFT_RESET__SOFT_RESET_SDMA0_MASK                                                                0x00800000L
+#define GRBM_SOFT_RESET__SOFT_RESET_SDMA1_MASK                                                                0x01000000L
+//GRBM_FAO_CNT
+#define GRBM_FAO_CNT__GRBM_FAO_CNT__SHIFT                                                                     0x0
+#define GRBM_FAO_CNT__GRBM_FAO_CNT_MASK                                                                       0x0000FFFFL
+//GRBM_GFX_CLKEN_CNTL
+#define GRBM_GFX_CLKEN_CNTL__PREFIX_DELAY_CNT__SHIFT                                                          0x0
+#define GRBM_GFX_CLKEN_CNTL__POST_DELAY_CNT__SHIFT                                                            0x8
+#define GRBM_GFX_CLKEN_CNTL__PREFIX_DELAY_CNT_MASK                                                            0x0000000FL
+#define GRBM_GFX_CLKEN_CNTL__POST_DELAY_CNT_MASK                                                              0x00001F00L
+//GRBM_WAIT_IDLE_CLOCKS
+#define GRBM_WAIT_IDLE_CLOCKS__WAIT_IDLE_CLOCKS__SHIFT                                                        0x0
+#define GRBM_WAIT_IDLE_CLOCKS__WAIT_IDLE_CLOCKS_MASK                                                          0x000000FFL
+//GRBM_READ_ERROR
+#define GRBM_READ_ERROR__READ_ADDRESS__SHIFT                                                                  0x2
+#define GRBM_READ_ERROR__READ_PIPEID__SHIFT                                                                   0x14
+#define GRBM_READ_ERROR__READ_MEID__SHIFT                                                                     0x16
+#define GRBM_READ_ERROR__READ_ERROR__SHIFT                                                                    0x1f
+#define GRBM_READ_ERROR__READ_ADDRESS_MASK                                                                    0x000FFFFCL
+#define GRBM_READ_ERROR__READ_PIPEID_MASK                                                                     0x00300000L
+#define GRBM_READ_ERROR__READ_MEID_MASK                                                                       0x00C00000L
+#define GRBM_READ_ERROR__READ_ERROR_MASK                                                                      0x80000000L
+//GRBM_READ_ERROR2
+#define GRBM_READ_ERROR2__READ_REQUESTER_MESPIPE0__SHIFT                                                      0x9
+#define GRBM_READ_ERROR2__READ_REQUESTER_MESPIPE1__SHIFT                                                      0xa
+#define GRBM_READ_ERROR2__READ_REQUESTER_MESPIPE2__SHIFT                                                      0xb
+#define GRBM_READ_ERROR2__READ_REQUESTER_MESPIPE3__SHIFT                                                      0xc
+#define GRBM_READ_ERROR2__READ_REQUESTER_SDMA0__SHIFT                                                         0xd
+#define GRBM_READ_ERROR2__READ_REQUESTER_SDMA1__SHIFT                                                         0xe
+#define GRBM_READ_ERROR2__READ_REQUESTER_RSMU__SHIFT                                                          0x11
+#define GRBM_READ_ERROR2__READ_REQUESTER_RLC__SHIFT                                                           0x12
+#define GRBM_READ_ERROR2__READ_REQUESTER_GDS_DMA__SHIFT                                                       0x13
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_CF__SHIFT                                                   0x14
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_PF__SHIFT                                                   0x15
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_CF__SHIFT                                                   0x16
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_PF__SHIFT                                                   0x17
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE0__SHIFT                                                      0x18
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE1__SHIFT                                                      0x19
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE2__SHIFT                                                      0x1a
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE3__SHIFT                                                      0x1b
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE0__SHIFT                                                      0x1c
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE1__SHIFT                                                      0x1d
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE2__SHIFT                                                      0x1e
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE3__SHIFT                                                      0x1f
+#define GRBM_READ_ERROR2__READ_REQUESTER_MESPIPE0_MASK                                                        0x00000200L
+#define GRBM_READ_ERROR2__READ_REQUESTER_MESPIPE1_MASK                                                        0x00000400L
+#define GRBM_READ_ERROR2__READ_REQUESTER_MESPIPE2_MASK                                                        0x00000800L
+#define GRBM_READ_ERROR2__READ_REQUESTER_MESPIPE3_MASK                                                        0x00001000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_SDMA0_MASK                                                           0x00002000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_SDMA1_MASK                                                           0x00004000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_RSMU_MASK                                                            0x00020000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_RLC_MASK                                                             0x00040000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_GDS_DMA_MASK                                                         0x00080000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_CF_MASK                                                     0x00100000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_PF_MASK                                                     0x00200000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_CF_MASK                                                     0x00400000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_PF_MASK                                                     0x00800000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE0_MASK                                                        0x01000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE1_MASK                                                        0x02000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE2_MASK                                                        0x04000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE3_MASK                                                        0x08000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE0_MASK                                                        0x10000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE1_MASK                                                        0x20000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE2_MASK                                                        0x40000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE3_MASK                                                        0x80000000L
+//GRBM_INT_CNTL
+#define GRBM_INT_CNTL__RDERR_INT_ENABLE__SHIFT                                                                0x0
+#define GRBM_INT_CNTL__GUI_IDLE_INT_ENABLE__SHIFT                                                             0x13
+#define GRBM_INT_CNTL__RDERR_INT_ENABLE_MASK                                                                  0x00000001L
+#define GRBM_INT_CNTL__GUI_IDLE_INT_ENABLE_MASK                                                               0x00080000L
+//GRBM_TRAP_OP
+#define GRBM_TRAP_OP__RW__SHIFT                                                                               0x0
+#define GRBM_TRAP_OP__RW_MASK                                                                                 0x00000001L
+//GRBM_TRAP_ADDR
+#define GRBM_TRAP_ADDR__DATA__SHIFT                                                                           0x0
+#define GRBM_TRAP_ADDR__DATA_MASK                                                                             0x0003FFFFL
+//GRBM_TRAP_ADDR_MSK
+#define GRBM_TRAP_ADDR_MSK__DATA__SHIFT                                                                       0x0
+#define GRBM_TRAP_ADDR_MSK__DATA_MASK                                                                         0x0003FFFFL
+//GRBM_TRAP_WD
+#define GRBM_TRAP_WD__DATA__SHIFT                                                                             0x0
+#define GRBM_TRAP_WD__DATA_MASK                                                                               0xFFFFFFFFL
+//GRBM_TRAP_WD_MSK
+#define GRBM_TRAP_WD_MSK__DATA__SHIFT                                                                         0x0
+#define GRBM_TRAP_WD_MSK__DATA_MASK                                                                           0xFFFFFFFFL
+//GRBM_DSM_BYPASS
+#define GRBM_DSM_BYPASS__BYPASS_BITS__SHIFT                                                                   0x0
+#define GRBM_DSM_BYPASS__BYPASS_EN__SHIFT                                                                     0x2
+#define GRBM_DSM_BYPASS__BYPASS_BITS_MASK                                                                     0x00000003L
+#define GRBM_DSM_BYPASS__BYPASS_EN_MASK                                                                       0x00000004L
+//GRBM_WRITE_ERROR
+#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RLC__SHIFT                                                          0x0
+#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RSMU__SHIFT                                                         0x1
+#define GRBM_WRITE_ERROR__WRITE_SSRCID__SHIFT                                                                 0x2
+#define GRBM_WRITE_ERROR__WRITE_VFID__SHIFT                                                                   0x7
+#define GRBM_WRITE_ERROR__WRITE_VF__SHIFT                                                                     0xc
+#define GRBM_WRITE_ERROR__WRITE_VMID__SHIFT                                                                   0xd
+#define GRBM_WRITE_ERROR__TMZ__SHIFT                                                                          0x11
+#define GRBM_WRITE_ERROR__WRITE_PIPEID__SHIFT                                                                 0x14
+#define GRBM_WRITE_ERROR__WRITE_MEID__SHIFT                                                                   0x16
+#define GRBM_WRITE_ERROR__WRITE_ERROR__SHIFT                                                                  0x1f
+#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RLC_MASK                                                            0x00000001L
+#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RSMU_MASK                                                           0x00000002L
+#define GRBM_WRITE_ERROR__WRITE_SSRCID_MASK                                                                   0x0000003CL
+#define GRBM_WRITE_ERROR__WRITE_VFID_MASK                                                                     0x00000F80L
+#define GRBM_WRITE_ERROR__WRITE_VF_MASK                                                                       0x00001000L
+#define GRBM_WRITE_ERROR__WRITE_VMID_MASK                                                                     0x0001E000L
+#define GRBM_WRITE_ERROR__TMZ_MASK                                                                            0x00020000L
+#define GRBM_WRITE_ERROR__WRITE_PIPEID_MASK                                                                   0x00300000L
+#define GRBM_WRITE_ERROR__WRITE_MEID_MASK                                                                     0x00C00000L
+#define GRBM_WRITE_ERROR__WRITE_ERROR_MASK                                                                    0x80000000L
+//GRBM_IOV_ERROR_FIFO
+#define GRBM_IOV_ERROR_FIFO__IOV_ADDR__SHIFT                                                                  0x0
+#define GRBM_IOV_ERROR_FIFO__IOV_VFID__SHIFT                                                                  0x12
+#define GRBM_IOV_ERROR_FIFO__IOV_SSRCID__SHIFT                                                                0x18
+#define GRBM_IOV_ERROR_FIFO__IOV_OP__SHIFT                                                                    0x1c
+#define GRBM_IOV_ERROR_FIFO__IOV_VF__SHIFT                                                                    0x1d
+#define GRBM_IOV_ERROR_FIFO__FIFO_OVERFLOW__SHIFT                                                             0x1e
+#define GRBM_IOV_ERROR_FIFO__READ_VALID__SHIFT                                                                0x1f
+#define GRBM_IOV_ERROR_FIFO__IOV_ADDR_MASK                                                                    0x0003FFFFL
+#define GRBM_IOV_ERROR_FIFO__IOV_VFID_MASK                                                                    0x00FC0000L
+#define GRBM_IOV_ERROR_FIFO__IOV_SSRCID_MASK                                                                  0x0F000000L
+#define GRBM_IOV_ERROR_FIFO__IOV_OP_MASK                                                                      0x10000000L
+#define GRBM_IOV_ERROR_FIFO__IOV_VF_MASK                                                                      0x20000000L
+#define GRBM_IOV_ERROR_FIFO__FIFO_OVERFLOW_MASK                                                               0x40000000L
+#define GRBM_IOV_ERROR_FIFO__READ_VALID_MASK                                                                  0x80000000L
+//GRBM_CHIP_REVISION
+#define GRBM_CHIP_REVISION__CHIP_REVISION__SHIFT                                                              0x0
+#define GRBM_CHIP_REVISION__CHIP_REVISION_MASK                                                                0x000000FFL
+//GRBM_RSMU_CFG
+#define GRBM_RSMU_CFG__APERTURE_ID__SHIFT                                                                     0x0
+#define GRBM_RSMU_CFG__QOS__SHIFT                                                                             0xc
+#define GRBM_RSMU_CFG__POSTED_WR__SHIFT                                                                       0x10
+#define GRBM_RSMU_CFG__DEBUG_MASK__SHIFT                                                                      0x11
+#define GRBM_RSMU_CFG__APERTURE_ID_MASK                                                                       0x00000FFFL
+#define GRBM_RSMU_CFG__QOS_MASK                                                                               0x0000F000L
+#define GRBM_RSMU_CFG__POSTED_WR_MASK                                                                         0x00010000L
+#define GRBM_RSMU_CFG__DEBUG_MASK_MASK                                                                        0x00020000L
+//GRBM_IH_CREDIT
+#define GRBM_IH_CREDIT__CREDIT_VALUE__SHIFT                                                                   0x0
+#define GRBM_IH_CREDIT__IH_CLIENT_ID__SHIFT                                                                   0x10
+#define GRBM_IH_CREDIT__CREDIT_VALUE_MASK                                                                     0x00000003L
+#define GRBM_IH_CREDIT__IH_CLIENT_ID_MASK                                                                     0x00FF0000L
+//GRBM_PWR_CNTL2
+#define GRBM_PWR_CNTL2__PWR_REQUEST_HALT__SHIFT                                                               0x10
+#define GRBM_PWR_CNTL2__PWR_GFX3D_REQUEST_HALT__SHIFT                                                         0x14
+#define GRBM_PWR_CNTL2__PWR_REQUEST_HALT_MASK                                                                 0x00010000L
+#define GRBM_PWR_CNTL2__PWR_GFX3D_REQUEST_HALT_MASK                                                           0x00100000L
+//GRBM_UTCL2_INVAL_RANGE_START
+#define GRBM_UTCL2_INVAL_RANGE_START__DATA__SHIFT                                                             0x0
+#define GRBM_UTCL2_INVAL_RANGE_START__DATA_MASK                                                               0x0003FFFFL
+//GRBM_UTCL2_INVAL_RANGE_END
+#define GRBM_UTCL2_INVAL_RANGE_END__DATA__SHIFT                                                               0x0
+#define GRBM_UTCL2_INVAL_RANGE_END__DATA_MASK                                                                 0x0003FFFFL
+//GRBM_RSMU_READ_ERROR
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ADDRESS__SHIFT                                                        0x2
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_VF__SHIFT                                                             0x14
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_VFID__SHIFT                                                           0x15
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ERROR_TYPE__SHIFT                                                     0x1b
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ERROR__SHIFT                                                          0x1f
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ADDRESS_MASK                                                          0x000FFFFCL
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_VF_MASK                                                               0x00100000L
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_VFID_MASK                                                             0x07E00000L
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ERROR_TYPE_MASK                                                       0x08000000L
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ERROR_MASK                                                            0x80000000L
+//GRBM_INVALID_PIPE
+#define GRBM_INVALID_PIPE__ADDR__SHIFT                                                                        0x2
+#define GRBM_INVALID_PIPE__PIPEID__SHIFT                                                                      0x14
+#define GRBM_INVALID_PIPE__MEID__SHIFT                                                                        0x16
+#define GRBM_INVALID_PIPE__QUEUEID__SHIFT                                                                     0x18
+#define GRBM_INVALID_PIPE__SSRCID__SHIFT                                                                      0x1b
+#define GRBM_INVALID_PIPE__INVALID_PIPE__SHIFT                                                                0x1f
+#define GRBM_INVALID_PIPE__ADDR_MASK                                                                          0x000FFFFCL
+#define GRBM_INVALID_PIPE__PIPEID_MASK                                                                        0x00300000L
+#define GRBM_INVALID_PIPE__MEID_MASK                                                                          0x00C00000L
+#define GRBM_INVALID_PIPE__QUEUEID_MASK                                                                       0x07000000L
+#define GRBM_INVALID_PIPE__SSRCID_MASK                                                                        0x78000000L
+#define GRBM_INVALID_PIPE__INVALID_PIPE_MASK                                                                  0x80000000L
+//GRBM_FENCE_RANGE0
+#define GRBM_FENCE_RANGE0__START__SHIFT                                                                       0x0
+#define GRBM_FENCE_RANGE0__END__SHIFT                                                                         0x10
+#define GRBM_FENCE_RANGE0__START_MASK                                                                         0x0000FFFFL
+#define GRBM_FENCE_RANGE0__END_MASK                                                                           0xFFFF0000L
+//GRBM_FENCE_RANGE1
+#define GRBM_FENCE_RANGE1__START__SHIFT                                                                       0x0
+#define GRBM_FENCE_RANGE1__END__SHIFT                                                                         0x10
+#define GRBM_FENCE_RANGE1__START_MASK                                                                         0x0000FFFFL
+#define GRBM_FENCE_RANGE1__END_MASK                                                                           0xFFFF0000L
+//GRBM_CHICKEN_BITS0
+#define GRBM_CHICKEN_BITS0__GRBM_SDMA0_reg_fgcg_chick_bit__SHIFT                                              0x0
+#define GRBM_CHICKEN_BITS0__GRBM_SDMA1_reg_fgcg_chick_bit__SHIFT                                              0x1
+#define GRBM_CHICKEN_BITS0__GRBM_CPG_reg_fgcg_chick_bit__SHIFT                                                0x2
+#define GRBM_CHICKEN_BITS0__GRBM_CPF_reg_fgcg_chick_bit__SHIFT                                                0x3
+#define GRBM_CHICKEN_BITS0__GRBM_UTCL2_reg_fgcg_chick_bit__SHIFT                                              0x5
+#define GRBM_CHICKEN_BITS0__GRBM_TARG0_mcd_reg_clken_chick_bit__SHIFT                                         0x6
+#define GRBM_CHICKEN_BITS0__GRBM_TARG1_targvf_reg_clken_chick_bit__SHIFT                                      0x7
+#define GRBM_CHICKEN_BITS0__GRBM_TARG2_targvf_reg_clken_chick_bit__SHIFT                                      0x8
+#define GRBM_CHICKEN_BITS0__GRBM_TARG3_targvf_reg_clken_chick_bit__SHIFT                                      0x9
+#define GRBM_CHICKEN_BITS0__GRBM_TARG4_targvf_reg_clken_chick_bit__SHIFT                                      0xa
+#define GRBM_CHICKEN_BITS0__GRBM_TARG5_reg_clken_chick_bit__SHIFT                                             0xb
+#define GRBM_CHICKEN_BITS0__GRBM_TARG6_reg_clken_chick_bit__SHIFT                                             0xc
+#define GRBM_CHICKEN_BITS0__GRBM_TARG7_reg_clken_chick_bit__SHIFT                                             0xd
+#define GRBM_CHICKEN_BITS0__GRBM_TARG8_reg_clken_chick_bit__SHIFT                                             0xe
+#define GRBM_CHICKEN_BITS0__GRBM_TARG9_reg_clken_chick_bit__SHIFT                                             0xf
+#define GRBM_CHICKEN_BITS0__GRBM_TARG10_reg_clken_chick_bit__SHIFT                                            0x10
+#define GRBM_CHICKEN_BITS0__GRBM_TARG11_reg_clken_chick_bit__SHIFT                                            0x11
+#define GRBM_CHICKEN_BITS0__GRBM_GRBMHSE0_reg_clken_chick_bit__SHIFT                                          0x12
+#define GRBM_CHICKEN_BITS0__GRBM_GRBMHSE1_reg_clken_chick_bit__SHIFT                                          0x13
+#define GRBM_CHICKEN_BITS0__GRBM_GRBMHSE2_reg_clken_chick_bit__SHIFT                                          0x14
+#define GRBM_CHICKEN_BITS0__GRBM_GRBMHSE3_reg_clken_chick_bit__SHIFT                                          0x15
+#define GRBM_CHICKEN_BITS0__GRBM_GRBMHSE4_reg_clken_chick_bit__SHIFT                                          0x16
+#define GRBM_CHICKEN_BITS0__GRBM_GRBMHSE5_reg_clken_chick_bit__SHIFT                                          0x17
+#define GRBM_CHICKEN_BITS0__GRBM_GRBMHSE6_reg_clken_chick_bit__SHIFT                                          0x18
+#define GRBM_CHICKEN_BITS0__GRBM_GRBMHSE7_reg_clken_chick_bit__SHIFT                                          0x19
+#define GRBM_CHICKEN_BITS0__GRBM_CPC_reg_clken_chick_bit__SHIFT                                               0x1c
+#define GRBM_CHICKEN_BITS0__GRBM_RLC_reg_clken_chick_bit__SHIFT                                               0x1f
+#define GRBM_CHICKEN_BITS0__GRBM_SDMA0_reg_fgcg_chick_bit_MASK                                                0x00000001L
+#define GRBM_CHICKEN_BITS0__GRBM_SDMA1_reg_fgcg_chick_bit_MASK                                                0x00000002L
+#define GRBM_CHICKEN_BITS0__GRBM_CPG_reg_fgcg_chick_bit_MASK                                                  0x00000004L
+#define GRBM_CHICKEN_BITS0__GRBM_CPF_reg_fgcg_chick_bit_MASK                                                  0x00000008L
+#define GRBM_CHICKEN_BITS0__GRBM_UTCL2_reg_fgcg_chick_bit_MASK                                                0x00000020L
+#define GRBM_CHICKEN_BITS0__GRBM_TARG0_mcd_reg_clken_chick_bit_MASK                                           0x00000040L
+#define GRBM_CHICKEN_BITS0__GRBM_TARG1_targvf_reg_clken_chick_bit_MASK                                        0x00000080L
+#define GRBM_CHICKEN_BITS0__GRBM_TARG2_targvf_reg_clken_chick_bit_MASK                                        0x00000100L
+#define GRBM_CHICKEN_BITS0__GRBM_TARG3_targvf_reg_clken_chick_bit_MASK                                        0x00000200L
+#define GRBM_CHICKEN_BITS0__GRBM_TARG4_targvf_reg_clken_chick_bit_MASK                                        0x00000400L
+#define GRBM_CHICKEN_BITS0__GRBM_TARG5_reg_clken_chick_bit_MASK                                               0x00000800L
+#define GRBM_CHICKEN_BITS0__GRBM_TARG6_reg_clken_chick_bit_MASK                                               0x00001000L
+#define GRBM_CHICKEN_BITS0__GRBM_TARG7_reg_clken_chick_bit_MASK                                               0x00002000L
+#define GRBM_CHICKEN_BITS0__GRBM_TARG8_reg_clken_chick_bit_MASK                                               0x00004000L
+#define GRBM_CHICKEN_BITS0__GRBM_TARG9_reg_clken_chick_bit_MASK                                               0x00008000L
+#define GRBM_CHICKEN_BITS0__GRBM_TARG10_reg_clken_chick_bit_MASK                                              0x00010000L
+#define GRBM_CHICKEN_BITS0__GRBM_TARG11_reg_clken_chick_bit_MASK                                              0x00020000L
+#define GRBM_CHICKEN_BITS0__GRBM_GRBMHSE0_reg_clken_chick_bit_MASK                                            0x00040000L
+#define GRBM_CHICKEN_BITS0__GRBM_GRBMHSE1_reg_clken_chick_bit_MASK                                            0x00080000L
+#define GRBM_CHICKEN_BITS0__GRBM_GRBMHSE2_reg_clken_chick_bit_MASK                                            0x00100000L
+#define GRBM_CHICKEN_BITS0__GRBM_GRBMHSE3_reg_clken_chick_bit_MASK                                            0x00200000L
+#define GRBM_CHICKEN_BITS0__GRBM_GRBMHSE4_reg_clken_chick_bit_MASK                                            0x00400000L
+#define GRBM_CHICKEN_BITS0__GRBM_GRBMHSE5_reg_clken_chick_bit_MASK                                            0x00800000L
+#define GRBM_CHICKEN_BITS0__GRBM_GRBMHSE6_reg_clken_chick_bit_MASK                                            0x01000000L
+#define GRBM_CHICKEN_BITS0__GRBM_GRBMHSE7_reg_clken_chick_bit_MASK                                            0x02000000L
+#define GRBM_CHICKEN_BITS0__GRBM_CPC_reg_clken_chick_bit_MASK                                                 0x10000000L
+#define GRBM_CHICKEN_BITS0__GRBM_RLC_reg_clken_chick_bit_MASK                                                 0x80000000L
+//GRBM_CHICKEN_BITS1
+#define GRBM_CHICKEN_BITS1__GRBM_UTCL2MCA_reg_clken_chick_bit__SHIFT                                          0xd
+#define GRBM_CHICKEN_BITS1__GRBM_MCA0_reg_clken_chick_bit__SHIFT                                              0xe
+#define GRBM_CHICKEN_BITS1__GRBM_MCA1_reg_clken_chick_bit__SHIFT                                              0xf
+#define GRBM_CHICKEN_BITS1__GRBM_MCA2_reg_clken_chick_bit__SHIFT                                              0x10
+#define GRBM_CHICKEN_BITS1__GRBM_MCA3_reg_clken_chick_bit__SHIFT                                              0x11
+#define GRBM_CHICKEN_BITS1__GRBM_MCA4_reg_clken_chick_bit__SHIFT                                              0x12
+#define GRBM_CHICKEN_BITS1__GRBM_MCA5_reg_clken_chick_bit__SHIFT                                              0x13
+#define GRBM_CHICKEN_BITS1__GRBM_MCA6_reg_clken_chick_bit__SHIFT                                              0x14
+#define GRBM_CHICKEN_BITS1__GRBM_MCA7_reg_clken_chick_bit__SHIFT                                              0x15
+#define GRBM_CHICKEN_BITS1__GRBM_MCA8_reg_clken_chick_bit__SHIFT                                              0x16
+#define GRBM_CHICKEN_BITS1__GRBM_MCA9_reg_clken_chick_bit__SHIFT                                              0x17
+#define GRBM_CHICKEN_BITS1__GRBM_MCA10_reg_clken_chick_bit__SHIFT                                             0x18
+#define GRBM_CHICKEN_BITS1__GRBM_UTCL2MCA_reg_clken_chick_bit_MASK                                            0x00002000L
+#define GRBM_CHICKEN_BITS1__GRBM_MCA0_reg_clken_chick_bit_MASK                                                0x00004000L
+#define GRBM_CHICKEN_BITS1__GRBM_MCA1_reg_clken_chick_bit_MASK                                                0x00008000L
+#define GRBM_CHICKEN_BITS1__GRBM_MCA2_reg_clken_chick_bit_MASK                                                0x00010000L
+#define GRBM_CHICKEN_BITS1__GRBM_MCA3_reg_clken_chick_bit_MASK                                                0x00020000L
+#define GRBM_CHICKEN_BITS1__GRBM_MCA4_reg_clken_chick_bit_MASK                                                0x00040000L
+#define GRBM_CHICKEN_BITS1__GRBM_MCA5_reg_clken_chick_bit_MASK                                                0x00080000L
+#define GRBM_CHICKEN_BITS1__GRBM_MCA6_reg_clken_chick_bit_MASK                                                0x00100000L
+#define GRBM_CHICKEN_BITS1__GRBM_MCA7_reg_clken_chick_bit_MASK                                                0x00200000L
+#define GRBM_CHICKEN_BITS1__GRBM_MCA8_reg_clken_chick_bit_MASK                                                0x00400000L
+#define GRBM_CHICKEN_BITS1__GRBM_MCA9_reg_clken_chick_bit_MASK                                                0x00800000L
+#define GRBM_CHICKEN_BITS1__GRBM_MCA10_reg_clken_chick_bit_MASK                                               0x01000000L
+//CC_GC_FULL_SA_UNIT_DISABLE
+#define CC_GC_FULL_SA_UNIT_DISABLE__SA_DISABLE__SHIFT                                                         0x8
+#define CC_GC_FULL_SA_UNIT_DISABLE__SA_DISABLE_MASK                                                           0x03FFFF00L
+//GRBM_SCRATCH_REG0
+#define GRBM_SCRATCH_REG0__SCRATCH_REG0__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG0__SCRATCH_REG0_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG1
+#define GRBM_SCRATCH_REG1__SCRATCH_REG1__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG1__SCRATCH_REG1_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG2
+#define GRBM_SCRATCH_REG2__SCRATCH_REG2__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG2__SCRATCH_REG2_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG3
+#define GRBM_SCRATCH_REG3__SCRATCH_REG3__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG3__SCRATCH_REG3_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG4
+#define GRBM_SCRATCH_REG4__SCRATCH_REG4__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG4__SCRATCH_REG4_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG5
+#define GRBM_SCRATCH_REG5__SCRATCH_REG5__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG5__SCRATCH_REG5_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG6
+#define GRBM_SCRATCH_REG6__SCRATCH_REG6__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG6__SCRATCH_REG6_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG7
+#define GRBM_SCRATCH_REG7__SCRATCH_REG7__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG7__SCRATCH_REG7_MASK                                                                  0xFFFFFFFFL
+//VIOLATION_DATA_ASYNC_VF_PROG
+#define VIOLATION_DATA_ASYNC_VF_PROG__SSRCID__SHIFT                                                           0x0
+#define VIOLATION_DATA_ASYNC_VF_PROG__VFID__SHIFT                                                             0x4
+#define VIOLATION_DATA_ASYNC_VF_PROG__VIOLATION_ERROR__SHIFT                                                  0x1f
+#define VIOLATION_DATA_ASYNC_VF_PROG__SSRCID_MASK                                                             0x0000000FL
+#define VIOLATION_DATA_ASYNC_VF_PROG__VFID_MASK                                                               0x000003F0L
+#define VIOLATION_DATA_ASYNC_VF_PROG__VIOLATION_ERROR_MASK                                                    0x80000000L
+//GRBM_INTF_CNTL
+#define GRBM_INTF_CNTL__GRBM_BRIDGE_DISABLE__SHIFT                                                            0x0
+#define GRBM_INTF_CNTL__GRBM_BRIDGE_DISABLE_MASK                                                              0x00000001L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_cpdec
+//CP_CPC_DEBUG_CNTL
+#define CP_CPC_DEBUG_CNTL__DEBUG_INDX__SHIFT                                                                  0x0
+#define CP_CPC_DEBUG_CNTL__DEBUG_INDX_MASK                                                                    0x0000007FL
+//CP_CPC_DEBUG_DATA
+#define CP_CPC_DEBUG_DATA__DEBUG_DATA__SHIFT                                                                  0x0
+#define CP_CPC_DEBUG_DATA__DEBUG_DATA_MASK                                                                    0xFFFFFFFFL
+//CP_CPF_DEBUG_CNTL
+#define CP_CPF_DEBUG_CNTL__DEBUG_INDX__SHIFT                                                                  0x0
+#define CP_CPF_DEBUG_CNTL__DEBUG_INDX_MASK                                                                    0x0000007FL
+//CP_CPF_DEBUG_DATA
+#define CP_CPF_DEBUG_DATA__DEBUG_DATA__SHIFT                                                                  0x0
+#define CP_CPF_DEBUG_DATA__DEBUG_DATA_MASK                                                                    0xFFFFFFFFL
+//CP_CPC_STATUS
+#define CP_CPC_STATUS__MEC1_BUSY__SHIFT                                                                       0x0
+#define CP_CPC_STATUS__MEC2_BUSY__SHIFT                                                                       0x1
+#define CP_CPC_STATUS__DC0_BUSY__SHIFT                                                                        0x2
+#define CP_CPC_STATUS__DC1_BUSY__SHIFT                                                                        0x3
+#define CP_CPC_STATUS__RCIU1_BUSY__SHIFT                                                                      0x4
+#define CP_CPC_STATUS__RCIU2_BUSY__SHIFT                                                                      0x5
+#define CP_CPC_STATUS__ROQ1_BUSY__SHIFT                                                                       0x6
+#define CP_CPC_STATUS__ROQ2_BUSY__SHIFT                                                                       0x7
+#define CP_CPC_STATUS__TCIU_BUSY__SHIFT                                                                       0xa
+#define CP_CPC_STATUS__SCRATCH_RAM_BUSY__SHIFT                                                                0xb
+#define CP_CPC_STATUS__QU_BUSY__SHIFT                                                                         0xc
+#define CP_CPC_STATUS__UTCL2IU_BUSY__SHIFT                                                                    0xd
+#define CP_CPC_STATUS__SAVE_RESTORE_BUSY__SHIFT                                                               0xe
+#define CP_CPC_STATUS__GCRIU_BUSY__SHIFT                                                                      0xf
+#define CP_CPC_STATUS__MES_BUSY__SHIFT                                                                        0x10
+#define CP_CPC_STATUS__MES_SCRATCH_RAM_BUSY__SHIFT                                                            0x11
+#define CP_CPC_STATUS__RCIU3_BUSY__SHIFT                                                                      0x12
+#define CP_CPC_STATUS__MES_INSTRUCTION_CACHE_BUSY__SHIFT                                                      0x13
+#define CP_CPC_STATUS__MES_DATA_CACHE_BUSY__SHIFT                                                             0x14
+#define CP_CPC_STATUS__MEC_DATA_CACHE_BUSY__SHIFT                                                             0x15
+#define CP_CPC_STATUS__CANEIU_BUSY__SHIFT                                                                     0x16
+#define CP_CPC_STATUS__CPF_CPC_BUSY__SHIFT                                                                    0x1d
+#define CP_CPC_STATUS__CPG_CPC_BUSY__SHIFT                                                                    0x1e
+#define CP_CPC_STATUS__CPC_BUSY__SHIFT                                                                        0x1f
+#define CP_CPC_STATUS__MEC1_BUSY_MASK                                                                         0x00000001L
+#define CP_CPC_STATUS__MEC2_BUSY_MASK                                                                         0x00000002L
+#define CP_CPC_STATUS__DC0_BUSY_MASK                                                                          0x00000004L
+#define CP_CPC_STATUS__DC1_BUSY_MASK                                                                          0x00000008L
+#define CP_CPC_STATUS__RCIU1_BUSY_MASK                                                                        0x00000010L
+#define CP_CPC_STATUS__RCIU2_BUSY_MASK                                                                        0x00000020L
+#define CP_CPC_STATUS__ROQ1_BUSY_MASK                                                                         0x00000040L
+#define CP_CPC_STATUS__ROQ2_BUSY_MASK                                                                         0x00000080L
+#define CP_CPC_STATUS__TCIU_BUSY_MASK                                                                         0x00000400L
+#define CP_CPC_STATUS__SCRATCH_RAM_BUSY_MASK                                                                  0x00000800L
+#define CP_CPC_STATUS__QU_BUSY_MASK                                                                           0x00001000L
+#define CP_CPC_STATUS__UTCL2IU_BUSY_MASK                                                                      0x00002000L
+#define CP_CPC_STATUS__SAVE_RESTORE_BUSY_MASK                                                                 0x00004000L
+#define CP_CPC_STATUS__GCRIU_BUSY_MASK                                                                        0x00008000L
+#define CP_CPC_STATUS__MES_BUSY_MASK                                                                          0x00010000L
+#define CP_CPC_STATUS__MES_SCRATCH_RAM_BUSY_MASK                                                              0x00020000L
+#define CP_CPC_STATUS__RCIU3_BUSY_MASK                                                                        0x00040000L
+#define CP_CPC_STATUS__MES_INSTRUCTION_CACHE_BUSY_MASK                                                        0x00080000L
+#define CP_CPC_STATUS__MES_DATA_CACHE_BUSY_MASK                                                               0x00100000L
+#define CP_CPC_STATUS__MEC_DATA_CACHE_BUSY_MASK                                                               0x00200000L
+#define CP_CPC_STATUS__CANEIU_BUSY_MASK                                                                       0x00400000L
+#define CP_CPC_STATUS__CPF_CPC_BUSY_MASK                                                                      0x20000000L
+#define CP_CPC_STATUS__CPG_CPC_BUSY_MASK                                                                      0x40000000L
+#define CP_CPC_STATUS__CPC_BUSY_MASK                                                                          0x80000000L
+//CP_CPC_BUSY_STAT
+#define CP_CPC_BUSY_STAT__MEC1_LOAD_BUSY__SHIFT                                                               0x0
+#define CP_CPC_BUSY_STAT__MEC1_MUTEX_BUSY__SHIFT                                                              0x2
+#define CP_CPC_BUSY_STAT__MEC1_MESSAGE_BUSY__SHIFT                                                            0x3
+#define CP_CPC_BUSY_STAT__MEC1_EOP_QUEUE_BUSY__SHIFT                                                          0x4
+#define CP_CPC_BUSY_STAT__MEC1_IQ_QUEUE_BUSY__SHIFT                                                           0x5
+#define CP_CPC_BUSY_STAT__MEC1_IB_QUEUE_BUSY__SHIFT                                                           0x6
+#define CP_CPC_BUSY_STAT__MEC1_TC_BUSY__SHIFT                                                                 0x7
+#define CP_CPC_BUSY_STAT__MEC1_DMA_BUSY__SHIFT                                                                0x8
+#define CP_CPC_BUSY_STAT__MEC1_PARTIAL_FLUSH_BUSY__SHIFT                                                      0x9
+#define CP_CPC_BUSY_STAT__MEC1_PIPE0_BUSY__SHIFT                                                              0xa
+#define CP_CPC_BUSY_STAT__MEC1_PIPE1_BUSY__SHIFT                                                              0xb
+#define CP_CPC_BUSY_STAT__MEC1_PIPE2_BUSY__SHIFT                                                              0xc
+#define CP_CPC_BUSY_STAT__MEC1_PIPE3_BUSY__SHIFT                                                              0xd
+#define CP_CPC_BUSY_STAT__MEC2_LOAD_BUSY__SHIFT                                                               0x10
+#define CP_CPC_BUSY_STAT__MEC2_MUTEX_BUSY__SHIFT                                                              0x12
+#define CP_CPC_BUSY_STAT__MEC2_MESSAGE_BUSY__SHIFT                                                            0x13
+#define CP_CPC_BUSY_STAT__MEC2_EOP_QUEUE_BUSY__SHIFT                                                          0x14
+#define CP_CPC_BUSY_STAT__MEC2_IQ_QUEUE_BUSY__SHIFT                                                           0x15
+#define CP_CPC_BUSY_STAT__MEC2_IB_QUEUE_BUSY__SHIFT                                                           0x16
+#define CP_CPC_BUSY_STAT__MEC2_TC_BUSY__SHIFT                                                                 0x17
+#define CP_CPC_BUSY_STAT__MEC2_DMA_BUSY__SHIFT                                                                0x18
+#define CP_CPC_BUSY_STAT__MEC2_PARTIAL_FLUSH_BUSY__SHIFT                                                      0x19
+#define CP_CPC_BUSY_STAT__MEC2_PIPE0_BUSY__SHIFT                                                              0x1a
+#define CP_CPC_BUSY_STAT__MEC2_PIPE1_BUSY__SHIFT                                                              0x1b
+#define CP_CPC_BUSY_STAT__MEC2_PIPE2_BUSY__SHIFT                                                              0x1c
+#define CP_CPC_BUSY_STAT__MEC2_PIPE3_BUSY__SHIFT                                                              0x1d
+#define CP_CPC_BUSY_STAT__MEC1_LOAD_BUSY_MASK                                                                 0x00000001L
+#define CP_CPC_BUSY_STAT__MEC1_MUTEX_BUSY_MASK                                                                0x00000004L
+#define CP_CPC_BUSY_STAT__MEC1_MESSAGE_BUSY_MASK                                                              0x00000008L
+#define CP_CPC_BUSY_STAT__MEC1_EOP_QUEUE_BUSY_MASK                                                            0x00000010L
+#define CP_CPC_BUSY_STAT__MEC1_IQ_QUEUE_BUSY_MASK                                                             0x00000020L
+#define CP_CPC_BUSY_STAT__MEC1_IB_QUEUE_BUSY_MASK                                                             0x00000040L
+#define CP_CPC_BUSY_STAT__MEC1_TC_BUSY_MASK                                                                   0x00000080L
+#define CP_CPC_BUSY_STAT__MEC1_DMA_BUSY_MASK                                                                  0x00000100L
+#define CP_CPC_BUSY_STAT__MEC1_PARTIAL_FLUSH_BUSY_MASK                                                        0x00000200L
+#define CP_CPC_BUSY_STAT__MEC1_PIPE0_BUSY_MASK                                                                0x00000400L
+#define CP_CPC_BUSY_STAT__MEC1_PIPE1_BUSY_MASK                                                                0x00000800L
+#define CP_CPC_BUSY_STAT__MEC1_PIPE2_BUSY_MASK                                                                0x00001000L
+#define CP_CPC_BUSY_STAT__MEC1_PIPE3_BUSY_MASK                                                                0x00002000L
+#define CP_CPC_BUSY_STAT__MEC2_LOAD_BUSY_MASK                                                                 0x00010000L
+#define CP_CPC_BUSY_STAT__MEC2_MUTEX_BUSY_MASK                                                                0x00040000L
+#define CP_CPC_BUSY_STAT__MEC2_MESSAGE_BUSY_MASK                                                              0x00080000L
+#define CP_CPC_BUSY_STAT__MEC2_EOP_QUEUE_BUSY_MASK                                                            0x00100000L
+#define CP_CPC_BUSY_STAT__MEC2_IQ_QUEUE_BUSY_MASK                                                             0x00200000L
+#define CP_CPC_BUSY_STAT__MEC2_IB_QUEUE_BUSY_MASK                                                             0x00400000L
+#define CP_CPC_BUSY_STAT__MEC2_TC_BUSY_MASK                                                                   0x00800000L
+#define CP_CPC_BUSY_STAT__MEC2_DMA_BUSY_MASK                                                                  0x01000000L
+#define CP_CPC_BUSY_STAT__MEC2_PARTIAL_FLUSH_BUSY_MASK                                                        0x02000000L
+#define CP_CPC_BUSY_STAT__MEC2_PIPE0_BUSY_MASK                                                                0x04000000L
+#define CP_CPC_BUSY_STAT__MEC2_PIPE1_BUSY_MASK                                                                0x08000000L
+#define CP_CPC_BUSY_STAT__MEC2_PIPE2_BUSY_MASK                                                                0x10000000L
+#define CP_CPC_BUSY_STAT__MEC2_PIPE3_BUSY_MASK                                                                0x20000000L
+//CP_CPC_STALLED_STAT1
+#define CP_CPC_STALLED_STAT1__RCIU_TX_FREE_STALL__SHIFT                                                       0x3
+#define CP_CPC_STALLED_STAT1__RCIU_PRIV_VIOLATION__SHIFT                                                      0x4
+#define CP_CPC_STALLED_STAT1__TCIU_TX_FREE_STALL__SHIFT                                                       0x6
+#define CP_CPC_STALLED_STAT1__TCIU_WAITING_ON_TAGS__SHIFT                                                     0x7
+#define CP_CPC_STALLED_STAT1__MEC1_DECODING_PACKET__SHIFT                                                     0x8
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU__SHIFT                                                        0x9
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_READ__SHIFT                                                   0xa
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_ROQ_DATA__SHIFT                                                    0xd
+#define CP_CPC_STALLED_STAT1__MEC2_DECODING_PACKET__SHIFT                                                     0x10
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU__SHIFT                                                        0x11
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_READ__SHIFT                                                   0x12
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_ROQ_DATA__SHIFT                                                    0x15
+#define CP_CPC_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE__SHIFT                                                  0x16
+#define CP_CPC_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS__SHIFT                                                  0x17
+#define CP_CPC_STALLED_STAT1__UTCL1_WAITING_ON_TRANS__SHIFT                                                   0x18
+#define CP_CPC_STALLED_STAT1__GCRIU_WAITING_ON_FREE__SHIFT                                                    0x19
+#define CP_CPC_STALLED_STAT1__RCIU_TX_FREE_STALL_MASK                                                         0x00000008L
+#define CP_CPC_STALLED_STAT1__RCIU_PRIV_VIOLATION_MASK                                                        0x00000010L
+#define CP_CPC_STALLED_STAT1__TCIU_TX_FREE_STALL_MASK                                                         0x00000040L
+#define CP_CPC_STALLED_STAT1__TCIU_WAITING_ON_TAGS_MASK                                                       0x00000080L
+#define CP_CPC_STALLED_STAT1__MEC1_DECODING_PACKET_MASK                                                       0x00000100L
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_MASK                                                          0x00000200L
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_READ_MASK                                                     0x00000400L
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_ROQ_DATA_MASK                                                      0x00002000L
+#define CP_CPC_STALLED_STAT1__MEC2_DECODING_PACKET_MASK                                                       0x00010000L
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_MASK                                                          0x00020000L
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_READ_MASK                                                     0x00040000L
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_ROQ_DATA_MASK                                                      0x00200000L
+#define CP_CPC_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE_MASK                                                    0x00400000L
+#define CP_CPC_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS_MASK                                                    0x00800000L
+#define CP_CPC_STALLED_STAT1__UTCL1_WAITING_ON_TRANS_MASK                                                     0x01000000L
+#define CP_CPC_STALLED_STAT1__GCRIU_WAITING_ON_FREE_MASK                                                      0x02000000L
+//CP_CPF_STATUS
+#define CP_CPF_STATUS__POST_WPTR_GFX_BUSY__SHIFT                                                              0x0
+#define CP_CPF_STATUS__CSF_BUSY__SHIFT                                                                        0x1
+#define CP_CPF_STATUS__ROQ_ALIGN_BUSY__SHIFT                                                                  0x4
+#define CP_CPF_STATUS__ROQ_RING_BUSY__SHIFT                                                                   0x5
+#define CP_CPF_STATUS__ROQ_INDIRECT1_BUSY__SHIFT                                                              0x6
+#define CP_CPF_STATUS__ROQ_INDIRECT2_BUSY__SHIFT                                                              0x7
+#define CP_CPF_STATUS__ROQ_STATE_BUSY__SHIFT                                                                  0x8
+#define CP_CPF_STATUS__ROQ_CE_RING_BUSY__SHIFT                                                                0x9
+#define CP_CPF_STATUS__ROQ_CE_INDIRECT1_BUSY__SHIFT                                                           0xa
+#define CP_CPF_STATUS__ROQ_CE_INDIRECT2_BUSY__SHIFT                                                           0xb
+#define CP_CPF_STATUS__INTERRUPT_BUSY__SHIFT                                                                  0xd
+#define CP_CPF_STATUS__TCIU_BUSY__SHIFT                                                                       0xe
+#define CP_CPF_STATUS__HQD_BUSY__SHIFT                                                                        0xf
+#define CP_CPF_STATUS__PRT_BUSY__SHIFT                                                                        0x10
+#define CP_CPF_STATUS__UTCL2IU_BUSY__SHIFT                                                                    0x11
+#define CP_CPF_STATUS__RCIU_BUSY__SHIFT                                                                       0x12
+#define CP_CPF_STATUS__RCIU_GFX_BUSY__SHIFT                                                                   0x13
+#define CP_CPF_STATUS__RCIU_CMP_BUSY__SHIFT                                                                   0x14
+#define CP_CPF_STATUS__ROQ_DATA_BUSY__SHIFT                                                                   0x15
+#define CP_CPF_STATUS__ROQ_CE_DATA_BUSY__SHIFT                                                                0x16
+#define CP_CPF_STATUS__GCRIU_BUSY__SHIFT                                                                      0x17
+#define CP_CPF_STATUS__MES_HQD_BUSY__SHIFT                                                                    0x18
+#define CP_CPF_STATUS__CPF_GFX_BUSY__SHIFT                                                                    0x1a
+#define CP_CPF_STATUS__CPF_CMP_BUSY__SHIFT                                                                    0x1b
+#define CP_CPF_STATUS__GRBM_CPF_STAT_BUSY__SHIFT                                                              0x1c
+#define CP_CPF_STATUS__CPC_CPF_BUSY__SHIFT                                                                    0x1e
+#define CP_CPF_STATUS__CPF_BUSY__SHIFT                                                                        0x1f
+#define CP_CPF_STATUS__POST_WPTR_GFX_BUSY_MASK                                                                0x00000001L
+#define CP_CPF_STATUS__CSF_BUSY_MASK                                                                          0x00000002L
+#define CP_CPF_STATUS__ROQ_ALIGN_BUSY_MASK                                                                    0x00000010L
+#define CP_CPF_STATUS__ROQ_RING_BUSY_MASK                                                                     0x00000020L
+#define CP_CPF_STATUS__ROQ_INDIRECT1_BUSY_MASK                                                                0x00000040L
+#define CP_CPF_STATUS__ROQ_INDIRECT2_BUSY_MASK                                                                0x00000080L
+#define CP_CPF_STATUS__ROQ_STATE_BUSY_MASK                                                                    0x00000100L
+#define CP_CPF_STATUS__ROQ_CE_RING_BUSY_MASK                                                                  0x00000200L
+#define CP_CPF_STATUS__ROQ_CE_INDIRECT1_BUSY_MASK                                                             0x00000400L
+#define CP_CPF_STATUS__ROQ_CE_INDIRECT2_BUSY_MASK                                                             0x00000800L
+#define CP_CPF_STATUS__INTERRUPT_BUSY_MASK                                                                    0x00002000L
+#define CP_CPF_STATUS__TCIU_BUSY_MASK                                                                         0x00004000L
+#define CP_CPF_STATUS__HQD_BUSY_MASK                                                                          0x00008000L
+#define CP_CPF_STATUS__PRT_BUSY_MASK                                                                          0x00010000L
+#define CP_CPF_STATUS__UTCL2IU_BUSY_MASK                                                                      0x00020000L
+#define CP_CPF_STATUS__RCIU_BUSY_MASK                                                                         0x00040000L
+#define CP_CPF_STATUS__RCIU_GFX_BUSY_MASK                                                                     0x00080000L
+#define CP_CPF_STATUS__RCIU_CMP_BUSY_MASK                                                                     0x00100000L
+#define CP_CPF_STATUS__ROQ_DATA_BUSY_MASK                                                                     0x00200000L
+#define CP_CPF_STATUS__ROQ_CE_DATA_BUSY_MASK                                                                  0x00400000L
+#define CP_CPF_STATUS__GCRIU_BUSY_MASK                                                                        0x00800000L
+#define CP_CPF_STATUS__MES_HQD_BUSY_MASK                                                                      0x01000000L
+#define CP_CPF_STATUS__CPF_GFX_BUSY_MASK                                                                      0x04000000L
+#define CP_CPF_STATUS__CPF_CMP_BUSY_MASK                                                                      0x08000000L
+#define CP_CPF_STATUS__GRBM_CPF_STAT_BUSY_MASK                                                                0x30000000L
+#define CP_CPF_STATUS__CPC_CPF_BUSY_MASK                                                                      0x40000000L
+#define CP_CPF_STATUS__CPF_BUSY_MASK                                                                          0x80000000L
+//CP_CPF_BUSY_STAT
+#define CP_CPF_BUSY_STAT__REG_BUS_FIFO_BUSY__SHIFT                                                            0x0
+#define CP_CPF_BUSY_STAT__CSF_RING_BUSY__SHIFT                                                                0x1
+#define CP_CPF_BUSY_STAT__CSF_INDIRECT1_BUSY__SHIFT                                                           0x2
+#define CP_CPF_BUSY_STAT__CSF_INDIRECT2_BUSY__SHIFT                                                           0x3
+#define CP_CPF_BUSY_STAT__CSF_STATE_BUSY__SHIFT                                                               0x4
+#define CP_CPF_BUSY_STAT__CSF_CE_INDR1_BUSY__SHIFT                                                            0x5
+#define CP_CPF_BUSY_STAT__CSF_CE_INDR2_BUSY__SHIFT                                                            0x6
+#define CP_CPF_BUSY_STAT__CSF_ARBITER_BUSY__SHIFT                                                             0x7
+#define CP_CPF_BUSY_STAT__CSF_INPUT_BUSY__SHIFT                                                               0x8
+#define CP_CPF_BUSY_STAT__CSF_DATA_BUSY__SHIFT                                                                0x9
+#define CP_CPF_BUSY_STAT__CSF_CE_DATA_BUSY__SHIFT                                                             0xa
+#define CP_CPF_BUSY_STAT__HPD_PROCESSING_EOP_BUSY__SHIFT                                                      0xb
+#define CP_CPF_BUSY_STAT__HQD_DISPATCH_BUSY__SHIFT                                                            0xc
+#define CP_CPF_BUSY_STAT__HQD_IQ_TIMER_BUSY__SHIFT                                                            0xd
+#define CP_CPF_BUSY_STAT__HQD_DMA_OFFLOAD_BUSY__SHIFT                                                         0xe
+#define CP_CPF_BUSY_STAT__HQD_MESSAGE_BUSY__SHIFT                                                             0x11
+#define CP_CPF_BUSY_STAT__HQD_PQ_FETCHER_BUSY__SHIFT                                                          0x12
+#define CP_CPF_BUSY_STAT__HQD_IB_FETCHER_BUSY__SHIFT                                                          0x13
+#define CP_CPF_BUSY_STAT__HQD_IQ_FETCHER_BUSY__SHIFT                                                          0x14
+#define CP_CPF_BUSY_STAT__HQD_EOP_FETCHER_BUSY__SHIFT                                                         0x15
+#define CP_CPF_BUSY_STAT__HQD_CONSUMED_RPTR_BUSY__SHIFT                                                       0x16
+#define CP_CPF_BUSY_STAT__HQD_FETCHER_ARB_BUSY__SHIFT                                                         0x17
+#define CP_CPF_BUSY_STAT__HQD_ROQ_ALIGN_BUSY__SHIFT                                                           0x18
+#define CP_CPF_BUSY_STAT__HQD_ROQ_EOP_BUSY__SHIFT                                                             0x19
+#define CP_CPF_BUSY_STAT__HQD_ROQ_IQ_BUSY__SHIFT                                                              0x1a
+#define CP_CPF_BUSY_STAT__HQD_ROQ_PQ_BUSY__SHIFT                                                              0x1b
+#define CP_CPF_BUSY_STAT__HQD_ROQ_IB_BUSY__SHIFT                                                              0x1c
+#define CP_CPF_BUSY_STAT__HQD_WPTR_POLL_BUSY__SHIFT                                                           0x1d
+#define CP_CPF_BUSY_STAT__HQD_PQ_BUSY__SHIFT                                                                  0x1e
+#define CP_CPF_BUSY_STAT__HQD_IB_BUSY__SHIFT                                                                  0x1f
+#define CP_CPF_BUSY_STAT__REG_BUS_FIFO_BUSY_MASK                                                              0x00000001L
+#define CP_CPF_BUSY_STAT__CSF_RING_BUSY_MASK                                                                  0x00000002L
+#define CP_CPF_BUSY_STAT__CSF_INDIRECT1_BUSY_MASK                                                             0x00000004L
+#define CP_CPF_BUSY_STAT__CSF_INDIRECT2_BUSY_MASK                                                             0x00000008L
+#define CP_CPF_BUSY_STAT__CSF_STATE_BUSY_MASK                                                                 0x00000010L
+#define CP_CPF_BUSY_STAT__CSF_CE_INDR1_BUSY_MASK                                                              0x00000020L
+#define CP_CPF_BUSY_STAT__CSF_CE_INDR2_BUSY_MASK                                                              0x00000040L
+#define CP_CPF_BUSY_STAT__CSF_ARBITER_BUSY_MASK                                                               0x00000080L
+#define CP_CPF_BUSY_STAT__CSF_INPUT_BUSY_MASK                                                                 0x00000100L
+#define CP_CPF_BUSY_STAT__CSF_DATA_BUSY_MASK                                                                  0x00000200L
+#define CP_CPF_BUSY_STAT__CSF_CE_DATA_BUSY_MASK                                                               0x00000400L
+#define CP_CPF_BUSY_STAT__HPD_PROCESSING_EOP_BUSY_MASK                                                        0x00000800L
+#define CP_CPF_BUSY_STAT__HQD_DISPATCH_BUSY_MASK                                                              0x00001000L
+#define CP_CPF_BUSY_STAT__HQD_IQ_TIMER_BUSY_MASK                                                              0x00002000L
+#define CP_CPF_BUSY_STAT__HQD_DMA_OFFLOAD_BUSY_MASK                                                           0x00004000L
+#define CP_CPF_BUSY_STAT__HQD_MESSAGE_BUSY_MASK                                                               0x00020000L
+#define CP_CPF_BUSY_STAT__HQD_PQ_FETCHER_BUSY_MASK                                                            0x00040000L
+#define CP_CPF_BUSY_STAT__HQD_IB_FETCHER_BUSY_MASK                                                            0x00080000L
+#define CP_CPF_BUSY_STAT__HQD_IQ_FETCHER_BUSY_MASK                                                            0x00100000L
+#define CP_CPF_BUSY_STAT__HQD_EOP_FETCHER_BUSY_MASK                                                           0x00200000L
+#define CP_CPF_BUSY_STAT__HQD_CONSUMED_RPTR_BUSY_MASK                                                         0x00400000L
+#define CP_CPF_BUSY_STAT__HQD_FETCHER_ARB_BUSY_MASK                                                           0x00800000L
+#define CP_CPF_BUSY_STAT__HQD_ROQ_ALIGN_BUSY_MASK                                                             0x01000000L
+#define CP_CPF_BUSY_STAT__HQD_ROQ_EOP_BUSY_MASK                                                               0x02000000L
+#define CP_CPF_BUSY_STAT__HQD_ROQ_IQ_BUSY_MASK                                                                0x04000000L
+#define CP_CPF_BUSY_STAT__HQD_ROQ_PQ_BUSY_MASK                                                                0x08000000L
+#define CP_CPF_BUSY_STAT__HQD_ROQ_IB_BUSY_MASK                                                                0x10000000L
+#define CP_CPF_BUSY_STAT__HQD_WPTR_POLL_BUSY_MASK                                                             0x20000000L
+#define CP_CPF_BUSY_STAT__HQD_PQ_BUSY_MASK                                                                    0x40000000L
+#define CP_CPF_BUSY_STAT__HQD_IB_BUSY_MASK                                                                    0x80000000L
+//CP_CPF_STALLED_STAT1
+#define CP_CPF_STALLED_STAT1__RING_FETCHING_DATA__SHIFT                                                       0x0
+#define CP_CPF_STALLED_STAT1__INDR1_FETCHING_DATA__SHIFT                                                      0x1
+#define CP_CPF_STALLED_STAT1__INDR2_FETCHING_DATA__SHIFT                                                      0x2
+#define CP_CPF_STALLED_STAT1__STATE_FETCHING_DATA__SHIFT                                                      0x3
+#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_FREE__SHIFT                                                     0x5
+#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_TAGS__SHIFT                                                     0x6
+#define CP_CPF_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE__SHIFT                                                  0x7
+#define CP_CPF_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS__SHIFT                                                  0x8
+#define CP_CPF_STALLED_STAT1__GFX_UTCL1_WAITING_ON_TRANS__SHIFT                                               0x9
+#define CP_CPF_STALLED_STAT1__CMP_UTCL1_WAITING_ON_TRANS__SHIFT                                               0xa
+#define CP_CPF_STALLED_STAT1__RCIU_WAITING_ON_FREE__SHIFT                                                     0xb
+#define CP_CPF_STALLED_STAT1__DATA_FETCHING_DATA__SHIFT                                                       0xc
+#define CP_CPF_STALLED_STAT1__GCRIU_WAIT_ON_FREE__SHIFT                                                       0xd
+#define CP_CPF_STALLED_STAT1__RING_FETCHING_DATA_MASK                                                         0x00000001L
+#define CP_CPF_STALLED_STAT1__INDR1_FETCHING_DATA_MASK                                                        0x00000002L
+#define CP_CPF_STALLED_STAT1__INDR2_FETCHING_DATA_MASK                                                        0x00000004L
+#define CP_CPF_STALLED_STAT1__STATE_FETCHING_DATA_MASK                                                        0x00000008L
+#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_FREE_MASK                                                       0x00000020L
+#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_TAGS_MASK                                                       0x00000040L
+#define CP_CPF_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE_MASK                                                    0x00000080L
+#define CP_CPF_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS_MASK                                                    0x00000100L
+#define CP_CPF_STALLED_STAT1__GFX_UTCL1_WAITING_ON_TRANS_MASK                                                 0x00000200L
+#define CP_CPF_STALLED_STAT1__CMP_UTCL1_WAITING_ON_TRANS_MASK                                                 0x00000400L
+#define CP_CPF_STALLED_STAT1__RCIU_WAITING_ON_FREE_MASK                                                       0x00000800L
+#define CP_CPF_STALLED_STAT1__DATA_FETCHING_DATA_MASK                                                         0x00001000L
+#define CP_CPF_STALLED_STAT1__GCRIU_WAIT_ON_FREE_MASK                                                         0x00002000L
+//CP_CPC_BUSY_STAT2
+#define CP_CPC_BUSY_STAT2__MES_LOAD_BUSY__SHIFT                                                               0x0
+#define CP_CPC_BUSY_STAT2__MES_MUTEX_BUSY__SHIFT                                                              0x2
+#define CP_CPC_BUSY_STAT2__MES_MESSAGE_BUSY__SHIFT                                                            0x3
+#define CP_CPC_BUSY_STAT2__MES_TC_BUSY__SHIFT                                                                 0x7
+#define CP_CPC_BUSY_STAT2__MES_DMA_BUSY__SHIFT                                                                0x8
+#define CP_CPC_BUSY_STAT2__MES_PIPE0_BUSY__SHIFT                                                              0xa
+#define CP_CPC_BUSY_STAT2__MES_PIPE1_BUSY__SHIFT                                                              0xb
+#define CP_CPC_BUSY_STAT2__MES_PIPE2_BUSY__SHIFT                                                              0xc
+#define CP_CPC_BUSY_STAT2__MES_PIPE3_BUSY__SHIFT                                                              0xd
+#define CP_CPC_BUSY_STAT2__MES_PIPE0_DC_BUSY__SHIFT                                                           0xe
+#define CP_CPC_BUSY_STAT2__MES_PIPE1_DC_BUSY__SHIFT                                                           0xf
+#define CP_CPC_BUSY_STAT2__MES_PIPE2_DC_BUSY__SHIFT                                                           0x10
+#define CP_CPC_BUSY_STAT2__MES_PIPE3_DC_BUSY__SHIFT                                                           0x11
+#define CP_CPC_BUSY_STAT2__MEC1_PIPE0_DC_BUSY__SHIFT                                                          0x12
+#define CP_CPC_BUSY_STAT2__MEC1_PIPE1_DC_BUSY__SHIFT                                                          0x13
+#define CP_CPC_BUSY_STAT2__MEC1_PIPE2_DC_BUSY__SHIFT                                                          0x14
+#define CP_CPC_BUSY_STAT2__MEC1_PIPE3_DC_BUSY__SHIFT                                                          0x15
+#define CP_CPC_BUSY_STAT2__MES_LOAD_BUSY_MASK                                                                 0x00000001L
+#define CP_CPC_BUSY_STAT2__MES_MUTEX_BUSY_MASK                                                                0x00000004L
+#define CP_CPC_BUSY_STAT2__MES_MESSAGE_BUSY_MASK                                                              0x00000008L
+#define CP_CPC_BUSY_STAT2__MES_TC_BUSY_MASK                                                                   0x00000080L
+#define CP_CPC_BUSY_STAT2__MES_DMA_BUSY_MASK                                                                  0x00000100L
+#define CP_CPC_BUSY_STAT2__MES_PIPE0_BUSY_MASK                                                                0x00000400L
+#define CP_CPC_BUSY_STAT2__MES_PIPE1_BUSY_MASK                                                                0x00000800L
+#define CP_CPC_BUSY_STAT2__MES_PIPE2_BUSY_MASK                                                                0x00001000L
+#define CP_CPC_BUSY_STAT2__MES_PIPE3_BUSY_MASK                                                                0x00002000L
+#define CP_CPC_BUSY_STAT2__MES_PIPE0_DC_BUSY_MASK                                                             0x00004000L
+#define CP_CPC_BUSY_STAT2__MES_PIPE1_DC_BUSY_MASK                                                             0x00008000L
+#define CP_CPC_BUSY_STAT2__MES_PIPE2_DC_BUSY_MASK                                                             0x00010000L
+#define CP_CPC_BUSY_STAT2__MES_PIPE3_DC_BUSY_MASK                                                             0x00020000L
+#define CP_CPC_BUSY_STAT2__MEC1_PIPE0_DC_BUSY_MASK                                                            0x00040000L
+#define CP_CPC_BUSY_STAT2__MEC1_PIPE1_DC_BUSY_MASK                                                            0x00080000L
+#define CP_CPC_BUSY_STAT2__MEC1_PIPE2_DC_BUSY_MASK                                                            0x00100000L
+#define CP_CPC_BUSY_STAT2__MEC1_PIPE3_DC_BUSY_MASK                                                            0x00200000L
+//CP_CPC_GRBM_FREE_COUNT
+#define CP_CPC_GRBM_FREE_COUNT__FREE_COUNT__SHIFT                                                             0x0
+#define CP_CPC_GRBM_FREE_COUNT__FREE_COUNT_MASK                                                               0x0000003FL
+//CP_CPC_PRIV_VIOLATION_ADDR
+#define CP_CPC_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_STATUS__SHIFT                                              0x0
+#define CP_CPC_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_OP__SHIFT                                                  0x1
+#define CP_CPC_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_ADDR__SHIFT                                                0x2
+#define CP_CPC_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_STATUS_MASK                                                0x00000001L
+#define CP_CPC_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_OP_MASK                                                    0x00000002L
+#define CP_CPC_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_ADDR_MASK                                                  0xFFFFFFFCL
+//CP_CPC_PRIV_VIOLATION_ADDR_HI
+#define CP_CPC_PRIV_VIOLATION_ADDR_HI__PRIV_VIOLATION_ADDR__SHIFT                                             0x0
+#define CP_CPC_PRIV_VIOLATION_ADDR_HI__PRIV_VIOLATION_ADDR_MASK                                               0x000000FFL
+//CP_MEC_ME1_HEADER_DUMP
+#define CP_MEC_ME1_HEADER_DUMP__HEADER_DUMP__SHIFT                                                            0x0
+#define CP_MEC_ME1_HEADER_DUMP__HEADER_DUMP_MASK                                                              0xFFFFFFFFL
+//CP_CPC_SCRATCH_INDEX
+#define CP_CPC_SCRATCH_INDEX__SCRATCH_INDEX__SHIFT                                                            0x0
+#define CP_CPC_SCRATCH_INDEX__SCRATCH_INDEX_64BIT_MODE__SHIFT                                                 0x1f
+#define CP_CPC_SCRATCH_INDEX__SCRATCH_INDEX_MASK                                                              0x000001FFL
+#define CP_CPC_SCRATCH_INDEX__SCRATCH_INDEX_64BIT_MODE_MASK                                                   0x80000000L
+//CP_CPC_SCRATCH_DATA
+#define CP_CPC_SCRATCH_DATA__SCRATCH_DATA__SHIFT                                                              0x0
+#define CP_CPC_SCRATCH_DATA__SCRATCH_DATA_MASK                                                                0xFFFFFFFFL
+//CP_CPF_GRBM_FREE_COUNT
+#define CP_CPF_GRBM_FREE_COUNT__FREE_COUNT__SHIFT                                                             0x0
+#define CP_CPF_GRBM_FREE_COUNT__FREE_COUNT_MASK                                                               0x00000007L
+//CP_CPF_BUSY_STAT2
+#define CP_CPF_BUSY_STAT2__CP_SDMA_CPG_BUSY__SHIFT                                                            0x0
+#define CP_CPF_BUSY_STAT2__CP_SDMA_CPC_BUSY__SHIFT                                                            0x1
+#define CP_CPF_BUSY_STAT2__MES_HQD_DISPATCH_BUSY__SHIFT                                                       0xc
+#define CP_CPF_BUSY_STAT2__MES_HQD_DMA_OFFLOAD_BUSY__SHIFT                                                    0xe
+#define CP_CPF_BUSY_STAT2__MES_HQD_MESSAGE_BUSY__SHIFT                                                        0x11
+#define CP_CPF_BUSY_STAT2__MES_HQD_PQ_FETCHER_BUSY__SHIFT                                                     0x12
+#define CP_CPF_BUSY_STAT2__MES_HQD_CONSUMED_RPTR_BUSY__SHIFT                                                  0x16
+#define CP_CPF_BUSY_STAT2__MES_HQD_FETCHER_ARB_BUSY__SHIFT                                                    0x17
+#define CP_CPF_BUSY_STAT2__MES_HQD_ROQ_ALIGN_BUSY__SHIFT                                                      0x18
+#define CP_CPF_BUSY_STAT2__MES_HQD_ROQ_PQ_BUSY__SHIFT                                                         0x1b
+#define CP_CPF_BUSY_STAT2__MES_HQD_PQ_BUSY__SHIFT                                                             0x1e
+#define CP_CPF_BUSY_STAT2__MES_UNMAPPED_DOORBELL_BUSY__SHIFT                                                  0x1f
+#define CP_CPF_BUSY_STAT2__CP_SDMA_CPG_BUSY_MASK                                                              0x00000001L
+#define CP_CPF_BUSY_STAT2__CP_SDMA_CPC_BUSY_MASK                                                              0x00000002L
+#define CP_CPF_BUSY_STAT2__MES_HQD_DISPATCH_BUSY_MASK                                                         0x00001000L
+#define CP_CPF_BUSY_STAT2__MES_HQD_DMA_OFFLOAD_BUSY_MASK                                                      0x00004000L
+#define CP_CPF_BUSY_STAT2__MES_HQD_MESSAGE_BUSY_MASK                                                          0x00020000L
+#define CP_CPF_BUSY_STAT2__MES_HQD_PQ_FETCHER_BUSY_MASK                                                       0x00040000L
+#define CP_CPF_BUSY_STAT2__MES_HQD_CONSUMED_RPTR_BUSY_MASK                                                    0x00400000L
+#define CP_CPF_BUSY_STAT2__MES_HQD_FETCHER_ARB_BUSY_MASK                                                      0x00800000L
+#define CP_CPF_BUSY_STAT2__MES_HQD_ROQ_ALIGN_BUSY_MASK                                                        0x01000000L
+#define CP_CPF_BUSY_STAT2__MES_HQD_ROQ_PQ_BUSY_MASK                                                           0x08000000L
+#define CP_CPF_BUSY_STAT2__MES_HQD_PQ_BUSY_MASK                                                               0x40000000L
+#define CP_CPF_BUSY_STAT2__MES_UNMAPPED_DOORBELL_BUSY_MASK                                                    0x80000000L
+//CP_CPC_HALT_HYST_COUNT
+#define CP_CPC_HALT_HYST_COUNT__COUNT__SHIFT                                                                  0x0
+#define CP_CPC_HALT_HYST_COUNT__COUNT_MASK                                                                    0x0000000FL
+//CP_STALLED_STAT3
+#define CP_STALLED_STAT3__CE_TO_CSF_NOT_RDY_TO_RCV__SHIFT                                                     0x0
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_FETCHER_NOT_RDY_TO_RCV__SHIFT                                        0x1
+#define CP_STALLED_STAT3__CE_WAITING_ON_DATA_FROM_RAM_INIT_FETCHER__SHIFT                                     0x2
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_NOT_RDY__SHIFT                                                       0x3
+#define CP_STALLED_STAT3__CE_TO_RAM_DUMP_NOT_RDY__SHIFT                                                       0x4
+#define CP_STALLED_STAT3__CE_TO_RAM_WRITE_NOT_RDY__SHIFT                                                      0x5
+#define CP_STALLED_STAT3__CE_TO_INC_FIFO_NOT_RDY_TO_RCV__SHIFT                                                0x6
+#define CP_STALLED_STAT3__CE_TO_WR_FIFO_NOT_RDY_TO_RCV__SHIFT                                                 0x7
+#define CP_STALLED_STAT3__CE_WAITING_ON_BUFFER_DATA__SHIFT                                                    0xa
+#define CP_STALLED_STAT3__CE_WAITING_ON_CE_BUFFER_FLAG__SHIFT                                                 0xb
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER__SHIFT                                                     0xc
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_UNDERFLOW__SHIFT                                           0xd
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_FREE__SHIFT                                                         0xe
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_TAGS__SHIFT                                                         0xf
+#define CP_STALLED_STAT3__CE_STALLED_ON_TC_WR_CONFIRM__SHIFT                                                  0x10
+#define CP_STALLED_STAT3__CE_STALLED_ON_ATOMIC_RTN_DATA__SHIFT                                                0x11
+#define CP_STALLED_STAT3__UTCL2IU_WAITING_ON_FREE__SHIFT                                                      0x12
+#define CP_STALLED_STAT3__UTCL2IU_WAITING_ON_TAGS__SHIFT                                                      0x13
+#define CP_STALLED_STAT3__UTCL1_WAITING_ON_TRANS__SHIFT                                                       0x14
+#define CP_STALLED_STAT3__GCRIU_WAITING_ON_FREE__SHIFT                                                        0x15
+#define CP_STALLED_STAT3__CE_TO_CSF_NOT_RDY_TO_RCV_MASK                                                       0x00000001L
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_FETCHER_NOT_RDY_TO_RCV_MASK                                          0x00000002L
+#define CP_STALLED_STAT3__CE_WAITING_ON_DATA_FROM_RAM_INIT_FETCHER_MASK                                       0x00000004L
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_NOT_RDY_MASK                                                         0x00000008L
+#define CP_STALLED_STAT3__CE_TO_RAM_DUMP_NOT_RDY_MASK                                                         0x00000010L
+#define CP_STALLED_STAT3__CE_TO_RAM_WRITE_NOT_RDY_MASK                                                        0x00000020L
+#define CP_STALLED_STAT3__CE_TO_INC_FIFO_NOT_RDY_TO_RCV_MASK                                                  0x00000040L
+#define CP_STALLED_STAT3__CE_TO_WR_FIFO_NOT_RDY_TO_RCV_MASK                                                   0x00000080L
+#define CP_STALLED_STAT3__CE_WAITING_ON_BUFFER_DATA_MASK                                                      0x00000400L
+#define CP_STALLED_STAT3__CE_WAITING_ON_CE_BUFFER_FLAG_MASK                                                   0x00000800L
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_MASK                                                       0x00001000L
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_UNDERFLOW_MASK                                             0x00002000L
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_FREE_MASK                                                           0x00004000L
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_TAGS_MASK                                                           0x00008000L
+#define CP_STALLED_STAT3__CE_STALLED_ON_TC_WR_CONFIRM_MASK                                                    0x00010000L
+#define CP_STALLED_STAT3__CE_STALLED_ON_ATOMIC_RTN_DATA_MASK                                                  0x00020000L
+#define CP_STALLED_STAT3__UTCL2IU_WAITING_ON_FREE_MASK                                                        0x00040000L
+#define CP_STALLED_STAT3__UTCL2IU_WAITING_ON_TAGS_MASK                                                        0x00080000L
+#define CP_STALLED_STAT3__UTCL1_WAITING_ON_TRANS_MASK                                                         0x00100000L
+#define CP_STALLED_STAT3__GCRIU_WAITING_ON_FREE_MASK                                                          0x00200000L
+//CP_STALLED_STAT1
+#define CP_STALLED_STAT1__RBIU_TO_DMA_NOT_RDY_TO_RCV__SHIFT                                                   0x0
+#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV_R0__SHIFT                                              0x4
+#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV_R1__SHIFT                                              0x5
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_CE_BUFFER_FLAG__SHIFT                                                 0xa
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_DE_BUFFER_FLAG__SHIFT                                                 0xb
+#define CP_STALLED_STAT1__ME_STALLED_ON_TC_WR_CONFIRM__SHIFT                                                  0xc
+#define CP_STALLED_STAT1__ME_STALLED_ON_ATOMIC_RTN_DATA__SHIFT                                                0xd
+#define CP_STALLED_STAT1__ME_WAITING_ON_TC_READ_DATA__SHIFT                                                   0xe
+#define CP_STALLED_STAT1__ME_WAITING_ON_REG_READ_DATA__SHIFT                                                  0xf
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GDS_FREE__SHIFT                                                     0x17
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GRBM_FREE__SHIFT                                                    0x18
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_VGT_FREE__SHIFT                                                     0x19
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_ME_READ__SHIFT                                                      0x1a
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_DMA_READ__SHIFT                                                     0x1b
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_APPEND_READ__SHIFT                                                  0x1c
+#define CP_STALLED_STAT1__RCIU_HALTED_BY_REG_VIOLATION__SHIFT                                                 0x1d
+#define CP_STALLED_STAT1__RBIU_TO_DMA_NOT_RDY_TO_RCV_MASK                                                     0x00000001L
+#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV_R0_MASK                                                0x00000010L
+#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV_R1_MASK                                                0x00000020L
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_CE_BUFFER_FLAG_MASK                                                   0x00000400L
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_DE_BUFFER_FLAG_MASK                                                   0x00000800L
+#define CP_STALLED_STAT1__ME_STALLED_ON_TC_WR_CONFIRM_MASK                                                    0x00001000L
+#define CP_STALLED_STAT1__ME_STALLED_ON_ATOMIC_RTN_DATA_MASK                                                  0x00002000L
+#define CP_STALLED_STAT1__ME_WAITING_ON_TC_READ_DATA_MASK                                                     0x00004000L
+#define CP_STALLED_STAT1__ME_WAITING_ON_REG_READ_DATA_MASK                                                    0x00008000L
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GDS_FREE_MASK                                                       0x00800000L
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GRBM_FREE_MASK                                                      0x01000000L
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_VGT_FREE_MASK                                                       0x02000000L
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_ME_READ_MASK                                                        0x04000000L
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_DMA_READ_MASK                                                       0x08000000L
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_APPEND_READ_MASK                                                    0x10000000L
+#define CP_STALLED_STAT1__RCIU_HALTED_BY_REG_VIOLATION_MASK                                                   0x20000000L
+//CP_STALLED_STAT2
+#define CP_STALLED_STAT2__PFP_TO_CSF_NOT_RDY_TO_RCV__SHIFT                                                    0x0
+#define CP_STALLED_STAT2__PFP_TO_MEQ_NOT_RDY_TO_RCV__SHIFT                                                    0x1
+#define CP_STALLED_STAT2__PFP_TO_RCIU_NOT_RDY_TO_RCV__SHIFT                                                   0x2
+#define CP_STALLED_STAT2__PFP_TO_VGT_WRITES_PENDING__SHIFT                                                    0x4
+#define CP_STALLED_STAT2__PFP_RCIU_READ_PENDING__SHIFT                                                        0x5
+#define CP_STALLED_STAT2__PFP_TO_MEQ_DDID_NOT_RDY_TO_RCV__SHIFT                                               0x6
+#define CP_STALLED_STAT2__PFP_WAITING_ON_BUFFER_DATA__SHIFT                                                   0x8
+#define CP_STALLED_STAT2__ME_WAIT_ON_CE_COUNTER__SHIFT                                                        0x9
+#define CP_STALLED_STAT2__ME_WAIT_ON_AVAIL_BUFFER__SHIFT                                                      0xa
+#define CP_STALLED_STAT2__GFX_CNTX_NOT_AVAIL_TO_ME__SHIFT                                                     0xb
+#define CP_STALLED_STAT2__ME_RCIU_NOT_RDY_TO_RCV__SHIFT                                                       0xc
+#define CP_STALLED_STAT2__ME_TO_CONST_NOT_RDY_TO_RCV__SHIFT                                                   0xd
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_PFP__SHIFT                                                     0xe
+#define CP_STALLED_STAT2__ME_WAITING_ON_PARTIAL_FLUSH__SHIFT                                                  0xf
+#define CP_STALLED_STAT2__MEQ_TO_ME_NOT_RDY_TO_RCV__SHIFT                                                     0x10
+#define CP_STALLED_STAT2__STQ_TO_ME_NOT_RDY_TO_RCV__SHIFT                                                     0x11
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_STQ__SHIFT                                                     0x12
+#define CP_STALLED_STAT2__PFP_STALLED_ON_TC_WR_CONFIRM__SHIFT                                                 0x13
+#define CP_STALLED_STAT2__PFP_STALLED_ON_ATOMIC_RTN_DATA__SHIFT                                               0x14
+#define CP_STALLED_STAT2__QU_STALLED_ON_EOP_DONE_PULSE__SHIFT                                                 0x15
+#define CP_STALLED_STAT2__QU_STALLED_ON_EOP_DONE_WR_CONFIRM__SHIFT                                            0x16
+#define CP_STALLED_STAT2__STRMO_WR_OF_PRIM_DATA_PENDING__SHIFT                                                0x17
+#define CP_STALLED_STAT2__PIPE_STATS_WR_DATA_PENDING__SHIFT                                                   0x18
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_CS_DONE__SHIFT                                                   0x19
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_PS_DONE__SHIFT                                                   0x1a
+#define CP_STALLED_STAT2__APPEND_WAIT_ON_WR_CONFIRM__SHIFT                                                    0x1b
+#define CP_STALLED_STAT2__APPEND_ACTIVE_PARTITION__SHIFT                                                      0x1c
+#define CP_STALLED_STAT2__APPEND_WAITING_TO_SEND_MEMWRITE__SHIFT                                              0x1d
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_IDLE_CNTXS__SHIFT                                                   0x1e
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_ALL_CLEAN__SHIFT                                                    0x1f
+#define CP_STALLED_STAT2__PFP_TO_CSF_NOT_RDY_TO_RCV_MASK                                                      0x00000001L
+#define CP_STALLED_STAT2__PFP_TO_MEQ_NOT_RDY_TO_RCV_MASK                                                      0x00000002L
+#define CP_STALLED_STAT2__PFP_TO_RCIU_NOT_RDY_TO_RCV_MASK                                                     0x00000004L
+#define CP_STALLED_STAT2__PFP_TO_VGT_WRITES_PENDING_MASK                                                      0x00000010L
+#define CP_STALLED_STAT2__PFP_RCIU_READ_PENDING_MASK                                                          0x00000020L
+#define CP_STALLED_STAT2__PFP_TO_MEQ_DDID_NOT_RDY_TO_RCV_MASK                                                 0x00000040L
+#define CP_STALLED_STAT2__PFP_WAITING_ON_BUFFER_DATA_MASK                                                     0x00000100L
+#define CP_STALLED_STAT2__ME_WAIT_ON_CE_COUNTER_MASK                                                          0x00000200L
+#define CP_STALLED_STAT2__ME_WAIT_ON_AVAIL_BUFFER_MASK                                                        0x00000400L
+#define CP_STALLED_STAT2__GFX_CNTX_NOT_AVAIL_TO_ME_MASK                                                       0x00000800L
+#define CP_STALLED_STAT2__ME_RCIU_NOT_RDY_TO_RCV_MASK                                                         0x00001000L
+#define CP_STALLED_STAT2__ME_TO_CONST_NOT_RDY_TO_RCV_MASK                                                     0x00002000L
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_PFP_MASK                                                       0x00004000L
+#define CP_STALLED_STAT2__ME_WAITING_ON_PARTIAL_FLUSH_MASK                                                    0x00008000L
+#define CP_STALLED_STAT2__MEQ_TO_ME_NOT_RDY_TO_RCV_MASK                                                       0x00010000L
+#define CP_STALLED_STAT2__STQ_TO_ME_NOT_RDY_TO_RCV_MASK                                                       0x00020000L
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_STQ_MASK                                                       0x00040000L
+#define CP_STALLED_STAT2__PFP_STALLED_ON_TC_WR_CONFIRM_MASK                                                   0x00080000L
+#define CP_STALLED_STAT2__PFP_STALLED_ON_ATOMIC_RTN_DATA_MASK                                                 0x00100000L
+#define CP_STALLED_STAT2__QU_STALLED_ON_EOP_DONE_PULSE_MASK                                                   0x00200000L
+#define CP_STALLED_STAT2__QU_STALLED_ON_EOP_DONE_WR_CONFIRM_MASK                                              0x00400000L
+#define CP_STALLED_STAT2__STRMO_WR_OF_PRIM_DATA_PENDING_MASK                                                  0x00800000L
+#define CP_STALLED_STAT2__PIPE_STATS_WR_DATA_PENDING_MASK                                                     0x01000000L
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_CS_DONE_MASK                                                     0x02000000L
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_PS_DONE_MASK                                                     0x04000000L
+#define CP_STALLED_STAT2__APPEND_WAIT_ON_WR_CONFIRM_MASK                                                      0x08000000L
+#define CP_STALLED_STAT2__APPEND_ACTIVE_PARTITION_MASK                                                        0x10000000L
+#define CP_STALLED_STAT2__APPEND_WAITING_TO_SEND_MEMWRITE_MASK                                                0x20000000L
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_IDLE_CNTXS_MASK                                                     0x40000000L
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_ALL_CLEAN_MASK                                                      0x80000000L
+//CP_BUSY_STAT
+#define CP_BUSY_STAT__REG_BUS_FIFO_BUSY__SHIFT                                                                0x0
+#define CP_BUSY_STAT__PFP_DATA_CACHE_BUSY__SHIFT                                                              0x1
+#define CP_BUSY_STAT__ME_DATA_CACHE_BUSY__SHIFT                                                               0x2
+#define CP_BUSY_STAT__COHER_CNT_NEQ_ZERO__SHIFT                                                               0x6
+#define CP_BUSY_STAT__PFP_PARSING_PACKETS__SHIFT                                                              0x7
+#define CP_BUSY_STAT__ME_PARSING_PACKETS__SHIFT                                                               0x8
+#define CP_BUSY_STAT__RCIU_PFP_BUSY__SHIFT                                                                    0x9
+#define CP_BUSY_STAT__RCIU_ME_BUSY__SHIFT                                                                     0xa
+#define CP_BUSY_STAT__GFX_CONTEXT_BUSY__SHIFT                                                                 0xf
+#define CP_BUSY_STAT__ME_PARSER_BUSY__SHIFT                                                                   0x11
+#define CP_BUSY_STAT__EOP_DONE_BUSY__SHIFT                                                                    0x12
+#define CP_BUSY_STAT__STRM_OUT_BUSY__SHIFT                                                                    0x13
+#define CP_BUSY_STAT__PIPE_STATS_BUSY__SHIFT                                                                  0x14
+#define CP_BUSY_STAT__RCIU_CE_BUSY__SHIFT                                                                     0x15
+#define CP_BUSY_STAT__CE_PARSING_PACKETS__SHIFT                                                               0x16
+#define CP_BUSY_STAT__PFP_PIPE0_DC_BUSY__SHIFT                                                                0x17
+#define CP_BUSY_STAT__ME_PIPE0_DC_BUSY__SHIFT                                                                 0x18
+#define CP_BUSY_STAT__PFP_PIPE1_DC_BUSY__SHIFT                                                                0x19
+#define CP_BUSY_STAT__ME_PIPE1_DC_BUSY__SHIFT                                                                 0x1a
+#define CP_BUSY_STAT__REG_BUS_FIFO_BUSY_MASK                                                                  0x00000001L
+#define CP_BUSY_STAT__PFP_DATA_CACHE_BUSY_MASK                                                                0x00000002L
+#define CP_BUSY_STAT__ME_DATA_CACHE_BUSY_MASK                                                                 0x00000004L
+#define CP_BUSY_STAT__COHER_CNT_NEQ_ZERO_MASK                                                                 0x00000040L
+#define CP_BUSY_STAT__PFP_PARSING_PACKETS_MASK                                                                0x00000080L
+#define CP_BUSY_STAT__ME_PARSING_PACKETS_MASK                                                                 0x00000100L
+#define CP_BUSY_STAT__RCIU_PFP_BUSY_MASK                                                                      0x00000200L
+#define CP_BUSY_STAT__RCIU_ME_BUSY_MASK                                                                       0x00000400L
+#define CP_BUSY_STAT__GFX_CONTEXT_BUSY_MASK                                                                   0x00008000L
+#define CP_BUSY_STAT__ME_PARSER_BUSY_MASK                                                                     0x00020000L
+#define CP_BUSY_STAT__EOP_DONE_BUSY_MASK                                                                      0x00040000L
+#define CP_BUSY_STAT__STRM_OUT_BUSY_MASK                                                                      0x00080000L
+#define CP_BUSY_STAT__PIPE_STATS_BUSY_MASK                                                                    0x00100000L
+#define CP_BUSY_STAT__RCIU_CE_BUSY_MASK                                                                       0x00200000L
+#define CP_BUSY_STAT__CE_PARSING_PACKETS_MASK                                                                 0x00400000L
+#define CP_BUSY_STAT__PFP_PIPE0_DC_BUSY_MASK                                                                  0x00800000L
+#define CP_BUSY_STAT__ME_PIPE0_DC_BUSY_MASK                                                                   0x01000000L
+#define CP_BUSY_STAT__PFP_PIPE1_DC_BUSY_MASK                                                                  0x02000000L
+#define CP_BUSY_STAT__ME_PIPE1_DC_BUSY_MASK                                                                   0x04000000L
+//CP_STAT
+#define CP_STAT__ROQ_DB_BUSY__SHIFT                                                                           0x5
+#define CP_STAT__ROQ_CE_DB_BUSY__SHIFT                                                                        0x6
+#define CP_STAT__ROQ_RING_BUSY__SHIFT                                                                         0x9
+#define CP_STAT__ROQ_INDIRECT1_BUSY__SHIFT                                                                    0xa
+#define CP_STAT__ROQ_INDIRECT2_BUSY__SHIFT                                                                    0xb
+#define CP_STAT__ROQ_STATE_BUSY__SHIFT                                                                        0xc
+#define CP_STAT__DC_BUSY__SHIFT                                                                               0xd
+#define CP_STAT__UTCL2IU_BUSY__SHIFT                                                                          0xe
+#define CP_STAT__PFP_BUSY__SHIFT                                                                              0xf
+#define CP_STAT__MEQ_BUSY__SHIFT                                                                              0x10
+#define CP_STAT__ME_BUSY__SHIFT                                                                               0x11
+#define CP_STAT__QUERY_BUSY__SHIFT                                                                            0x12
+#define CP_STAT__INTERRUPT_BUSY__SHIFT                                                                        0x14
+#define CP_STAT__SURFACE_SYNC_BUSY__SHIFT                                                                     0x15
+#define CP_STAT__DMA_BUSY__SHIFT                                                                              0x16
+#define CP_STAT__RCIU_BUSY__SHIFT                                                                             0x17
+#define CP_STAT__SCRATCH_RAM_BUSY__SHIFT                                                                      0x18
+#define CP_STAT__GCRIU_BUSY__SHIFT                                                                            0x19
+#define CP_STAT__CE_BUSY__SHIFT                                                                               0x1a
+#define CP_STAT__TCIU_BUSY__SHIFT                                                                             0x1b
+#define CP_STAT__ROQ_CE_RING_BUSY__SHIFT                                                                      0x1c
+#define CP_STAT__ROQ_CE_INDIRECT1_BUSY__SHIFT                                                                 0x1d
+#define CP_STAT__ROQ_CE_INDIRECT2_BUSY__SHIFT                                                                 0x1e
+#define CP_STAT__CP_BUSY__SHIFT                                                                               0x1f
+#define CP_STAT__ROQ_DB_BUSY_MASK                                                                             0x00000020L
+#define CP_STAT__ROQ_CE_DB_BUSY_MASK                                                                          0x00000040L
+#define CP_STAT__ROQ_RING_BUSY_MASK                                                                           0x00000200L
+#define CP_STAT__ROQ_INDIRECT1_BUSY_MASK                                                                      0x00000400L
+#define CP_STAT__ROQ_INDIRECT2_BUSY_MASK                                                                      0x00000800L
+#define CP_STAT__ROQ_STATE_BUSY_MASK                                                                          0x00001000L
+#define CP_STAT__DC_BUSY_MASK                                                                                 0x00002000L
+#define CP_STAT__UTCL2IU_BUSY_MASK                                                                            0x00004000L
+#define CP_STAT__PFP_BUSY_MASK                                                                                0x00008000L
+#define CP_STAT__MEQ_BUSY_MASK                                                                                0x00010000L
+#define CP_STAT__ME_BUSY_MASK                                                                                 0x00020000L
+#define CP_STAT__QUERY_BUSY_MASK                                                                              0x00040000L
+#define CP_STAT__INTERRUPT_BUSY_MASK                                                                          0x00100000L
+#define CP_STAT__SURFACE_SYNC_BUSY_MASK                                                                       0x00200000L
+#define CP_STAT__DMA_BUSY_MASK                                                                                0x00400000L
+#define CP_STAT__RCIU_BUSY_MASK                                                                               0x00800000L
+#define CP_STAT__SCRATCH_RAM_BUSY_MASK                                                                        0x01000000L
+#define CP_STAT__GCRIU_BUSY_MASK                                                                              0x02000000L
+#define CP_STAT__CE_BUSY_MASK                                                                                 0x04000000L
+#define CP_STAT__TCIU_BUSY_MASK                                                                               0x08000000L
+#define CP_STAT__ROQ_CE_RING_BUSY_MASK                                                                        0x10000000L
+#define CP_STAT__ROQ_CE_INDIRECT1_BUSY_MASK                                                                   0x20000000L
+#define CP_STAT__ROQ_CE_INDIRECT2_BUSY_MASK                                                                   0x40000000L
+#define CP_STAT__CP_BUSY_MASK                                                                                 0x80000000L
+//CP_ME_HEADER_DUMP
+#define CP_ME_HEADER_DUMP__ME_HEADER_DUMP__SHIFT                                                              0x0
+#define CP_ME_HEADER_DUMP__ME_HEADER_DUMP_MASK                                                                0xFFFFFFFFL
+//CP_PFP_HEADER_DUMP
+#define CP_PFP_HEADER_DUMP__PFP_HEADER_DUMP__SHIFT                                                            0x0
+#define CP_PFP_HEADER_DUMP__PFP_HEADER_DUMP_MASK                                                              0xFFFFFFFFL
+//CP_GRBM_FREE_COUNT
+#define CP_GRBM_FREE_COUNT__FREE_COUNT__SHIFT                                                                 0x0
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_PFP__SHIFT                                                             0x10
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_MASK                                                                   0x0000003FL
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_PFP_MASK                                                               0x003F0000L
+//CP_PFP_INSTR_PNTR
+#define CP_PFP_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                  0x0
+#define CP_PFP_INSTR_PNTR__INSTR_PNTR_MASK                                                                    0x0000FFFFL
+//CP_ME_INSTR_PNTR
+#define CP_ME_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                   0x0
+#define CP_ME_INSTR_PNTR__INSTR_PNTR_MASK                                                                     0x0000FFFFL
+//CP_CSF_STAT
+#define CP_CSF_STAT__BUFFER_REQUEST_COUNT__SHIFT                                                              0x8
+#define CP_CSF_STAT__BUFFER_REQUEST_COUNT_MASK                                                                0x0001FF00L
+//CP_CNTX_STAT
+#define CP_CNTX_STAT__ACTIVE_HP3D_CONTEXTS__SHIFT                                                             0x0
+#define CP_CNTX_STAT__CURRENT_HP3D_CONTEXT__SHIFT                                                             0x8
+#define CP_CNTX_STAT__ACTIVE_GFX_CONTEXTS__SHIFT                                                              0x14
+#define CP_CNTX_STAT__CURRENT_GFX_CONTEXT__SHIFT                                                              0x1c
+#define CP_CNTX_STAT__ACTIVE_HP3D_CONTEXTS_MASK                                                               0x000000FFL
+#define CP_CNTX_STAT__CURRENT_HP3D_CONTEXT_MASK                                                               0x00000700L
+#define CP_CNTX_STAT__ACTIVE_GFX_CONTEXTS_MASK                                                                0x0FF00000L
+#define CP_CNTX_STAT__CURRENT_GFX_CONTEXT_MASK                                                                0x70000000L
+//CP_ME_PREEMPTION
+#define CP_ME_PREEMPTION__OBSOLETE__SHIFT                                                                     0x0
+#define CP_ME_PREEMPTION__OBSOLETE_MASK                                                                       0x00000001L
+//CP_RB0_RPTR
+#define CP_RB0_RPTR__RB_RPTR__SHIFT                                                                           0x0
+#define CP_RB0_RPTR__RB_RPTR_MASK                                                                             0x000FFFFFL
+//CP_RB_RPTR
+#define CP_RB_RPTR__RB_RPTR__SHIFT                                                                            0x0
+#define CP_RB_RPTR__RB_RPTR_MASK                                                                              0x000FFFFFL
+//CP_RB_WPTR_DELAY
+#define CP_RB_WPTR_DELAY__PRE_WRITE_TIMER__SHIFT                                                              0x0
+#define CP_RB_WPTR_DELAY__PRE_WRITE_LIMIT__SHIFT                                                              0x1c
+#define CP_RB_WPTR_DELAY__PRE_WRITE_TIMER_MASK                                                                0x0FFFFFFFL
+#define CP_RB_WPTR_DELAY__PRE_WRITE_LIMIT_MASK                                                                0xF0000000L
+//CP_RB_WPTR_POLL_CNTL
+#define CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT                                                           0x0
+#define CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                          0x10
+#define CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK                                                             0x0000FFFFL
+#define CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                            0xFFFF0000L
+//CP_ROQ1_THRESHOLDS
+#define CP_ROQ1_THRESHOLDS__RB1_START__SHIFT                                                                  0x0
+#define CP_ROQ1_THRESHOLDS__R0_IB1_START__SHIFT                                                               0xa
+#define CP_ROQ1_THRESHOLDS__R1_IB1_START__SHIFT                                                               0x14
+#define CP_ROQ1_THRESHOLDS__RB1_START_MASK                                                                    0x000003FFL
+#define CP_ROQ1_THRESHOLDS__R0_IB1_START_MASK                                                                 0x000FFC00L
+#define CP_ROQ1_THRESHOLDS__R1_IB1_START_MASK                                                                 0x3FF00000L
+//CP_ROQ2_THRESHOLDS
+#define CP_ROQ2_THRESHOLDS__R0_IB2_START__SHIFT                                                               0x0
+#define CP_ROQ2_THRESHOLDS__R1_IB2_START__SHIFT                                                               0xa
+#define CP_ROQ2_THRESHOLDS__R0_IB2_START_MASK                                                                 0x000003FFL
+#define CP_ROQ2_THRESHOLDS__R1_IB2_START_MASK                                                                 0x000FFC00L
+//CP_STQ_THRESHOLDS
+#define CP_STQ_THRESHOLDS__STQ0_START__SHIFT                                                                  0x0
+#define CP_STQ_THRESHOLDS__STQ1_START__SHIFT                                                                  0x8
+#define CP_STQ_THRESHOLDS__STQ2_START__SHIFT                                                                  0x10
+#define CP_STQ_THRESHOLDS__STQ0_START_MASK                                                                    0x000000FFL
+#define CP_STQ_THRESHOLDS__STQ1_START_MASK                                                                    0x0000FF00L
+#define CP_STQ_THRESHOLDS__STQ2_START_MASK                                                                    0x00FF0000L
+//CP_MEQ_THRESHOLDS
+#define CP_MEQ_THRESHOLDS__MEQ1_START__SHIFT                                                                  0x0
+#define CP_MEQ_THRESHOLDS__MEQ2_START__SHIFT                                                                  0x8
+#define CP_MEQ_THRESHOLDS__MEQ1_START_MASK                                                                    0x000000FFL
+#define CP_MEQ_THRESHOLDS__MEQ2_START_MASK                                                                    0x0000FF00L
+//CP_ROQ_AVAIL
+#define CP_ROQ_AVAIL__ROQ_CNT_RING__SHIFT                                                                     0x0
+#define CP_ROQ_AVAIL__ROQ_CNT_IB1__SHIFT                                                                      0x10
+#define CP_ROQ_AVAIL__ROQ_CNT_RING_MASK                                                                       0x00000FFFL
+#define CP_ROQ_AVAIL__ROQ_CNT_IB1_MASK                                                                        0x0FFF0000L
+//CP_STQ_AVAIL
+#define CP_STQ_AVAIL__STQ_CNT__SHIFT                                                                          0x0
+#define CP_STQ_AVAIL__STQ_CNT_MASK                                                                            0x000003FFL
+//CP_ROQ2_AVAIL
+#define CP_ROQ2_AVAIL__ROQ_CNT_IB2__SHIFT                                                                     0x0
+#define CP_ROQ2_AVAIL__ROQ_CNT_DB__SHIFT                                                                      0x10
+#define CP_ROQ2_AVAIL__ROQ_CNT_IB2_MASK                                                                       0x00000FFFL
+#define CP_ROQ2_AVAIL__ROQ_CNT_DB_MASK                                                                        0x0FFF0000L
+//CP_MEQ_AVAIL
+#define CP_MEQ_AVAIL__MEQ_CNT__SHIFT                                                                          0x0
+#define CP_MEQ_AVAIL__MEQ_CNT_MASK                                                                            0x000003FFL
+//CP_CMD_INDEX
+#define CP_CMD_INDEX__CMD_INDEX__SHIFT                                                                        0x0
+#define CP_CMD_INDEX__CMD_ME_SEL__SHIFT                                                                       0xc
+#define CP_CMD_INDEX__CMD_QUEUE_SEL__SHIFT                                                                    0x10
+#define CP_CMD_INDEX__CMD_INDEX_MASK                                                                          0x000007FFL
+#define CP_CMD_INDEX__CMD_ME_SEL_MASK                                                                         0x00003000L
+#define CP_CMD_INDEX__CMD_QUEUE_SEL_MASK                                                                      0x00070000L
+//CP_CMD_DATA
+#define CP_CMD_DATA__CMD_DATA__SHIFT                                                                          0x0
+#define CP_CMD_DATA__CMD_DATA_MASK                                                                            0xFFFFFFFFL
+//CP_ROQ_RB_STAT
+#define CP_ROQ_RB_STAT__ROQ_RPTR_PRIMARY__SHIFT                                                               0x0
+#define CP_ROQ_RB_STAT__ROQ_WPTR_PRIMARY__SHIFT                                                               0x10
+#define CP_ROQ_RB_STAT__ROQ_RPTR_PRIMARY_MASK                                                                 0x00000FFFL
+#define CP_ROQ_RB_STAT__ROQ_WPTR_PRIMARY_MASK                                                                 0x0FFF0000L
+//CP_ROQ_IB1_STAT
+#define CP_ROQ_IB1_STAT__ROQ_RPTR_INDIRECT1__SHIFT                                                            0x0
+#define CP_ROQ_IB1_STAT__ROQ_WPTR_INDIRECT1__SHIFT                                                            0x10
+#define CP_ROQ_IB1_STAT__ROQ_RPTR_INDIRECT1_MASK                                                              0x00000FFFL
+#define CP_ROQ_IB1_STAT__ROQ_WPTR_INDIRECT1_MASK                                                              0x0FFF0000L
+//CP_ROQ_IB2_STAT
+#define CP_ROQ_IB2_STAT__ROQ_RPTR_INDIRECT2__SHIFT                                                            0x0
+#define CP_ROQ_IB2_STAT__ROQ_WPTR_INDIRECT2__SHIFT                                                            0x10
+#define CP_ROQ_IB2_STAT__ROQ_RPTR_INDIRECT2_MASK                                                              0x00000FFFL
+#define CP_ROQ_IB2_STAT__ROQ_WPTR_INDIRECT2_MASK                                                              0x0FFF0000L
+//CP_STQ_STAT
+#define CP_STQ_STAT__STQ_RPTR__SHIFT                                                                          0x0
+#define CP_STQ_STAT__STQ_RPTR_MASK                                                                            0x000003FFL
+//CP_STQ_WR_STAT
+#define CP_STQ_WR_STAT__STQ_WPTR__SHIFT                                                                       0x0
+#define CP_STQ_WR_STAT__STQ_WPTR_MASK                                                                         0x000003FFL
+//CP_MEQ_STAT
+#define CP_MEQ_STAT__MEQ_RPTR__SHIFT                                                                          0x0
+#define CP_MEQ_STAT__MEQ_WPTR__SHIFT                                                                          0x10
+#define CP_MEQ_STAT__MEQ_RPTR_MASK                                                                            0x000003FFL
+#define CP_MEQ_STAT__MEQ_WPTR_MASK                                                                            0x03FF0000L
+//CP_ROQ3_THRESHOLDS
+#define CP_ROQ3_THRESHOLDS__R0_DB_START__SHIFT                                                                0x0
+#define CP_ROQ3_THRESHOLDS__R1_DB_START__SHIFT                                                                0xa
+#define CP_ROQ3_THRESHOLDS__R0_DB_START_MASK                                                                  0x000003FFL
+#define CP_ROQ3_THRESHOLDS__R1_DB_START_MASK                                                                  0x000FFC00L
+//CP_ROQ_DB_STAT
+#define CP_ROQ_DB_STAT__ROQ_RPTR_DB__SHIFT                                                                    0x0
+#define CP_ROQ_DB_STAT__ROQ_WPTR_DB__SHIFT                                                                    0x10
+#define CP_ROQ_DB_STAT__ROQ_RPTR_DB_MASK                                                                      0x00000FFFL
+#define CP_ROQ_DB_STAT__ROQ_WPTR_DB_MASK                                                                      0x0FFF0000L
+//CP_INT_STAT_DEBUG
+#define CP_INT_STAT_DEBUG__RESUME_INT_ASSERTED__SHIFT                                                         0x8
+#define CP_INT_STAT_DEBUG__SUSPEND_INT_ASSERTED__SHIFT                                                        0x9
+#define CP_INT_STAT_DEBUG__DMA_WATCH_INT_ASSERTED__SHIFT                                                      0xa
+#define CP_INT_STAT_DEBUG__CP_VM_DOORBELL_WR_INT_ASSERTED__SHIFT                                              0xb
+#define CP_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT                                                   0xe
+#define CP_INT_STAT_DEBUG__FUE_INT_STATUS_DEBUG__SHIFT                                                        0xf
+#define CP_INT_STAT_DEBUG__GPF_INT_ASSERTED__SHIFT                                                            0x10
+#define CP_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT                                               0x11
+#define CP_INT_STAT_DEBUG__CMP_BUSY_INT_ASSERTED__SHIFT                                                       0x12
+#define CP_INT_STAT_DEBUG__CNTX_BUSY_INT_ASSERTED__SHIFT                                                      0x13
+#define CP_INT_STAT_DEBUG__CNTX_EMPTY_INT_ASSERTED__SHIFT                                                     0x14
+#define CP_INT_STAT_DEBUG__GFX_IDLE_INT_ASSERTED__SHIFT                                                       0x15
+#define CP_INT_STAT_DEBUG__PRIV_INSTR_INT_ASSERTED__SHIFT                                                     0x16
+#define CP_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                       0x17
+#define CP_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT                                                   0x18
+#define CP_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT                                                     0x1a
+#define CP_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT                                             0x1b
+#define CP_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT                                                       0x1d
+#define CP_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT                                                       0x1e
+#define CP_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT                                                       0x1f
+#define CP_INT_STAT_DEBUG__RESUME_INT_ASSERTED_MASK                                                           0x00000100L
+#define CP_INT_STAT_DEBUG__SUSPEND_INT_ASSERTED_MASK                                                          0x00000200L
+#define CP_INT_STAT_DEBUG__DMA_WATCH_INT_ASSERTED_MASK                                                        0x00000400L
+#define CP_INT_STAT_DEBUG__CP_VM_DOORBELL_WR_INT_ASSERTED_MASK                                                0x00000800L
+#define CP_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK                                                     0x00004000L
+#define CP_INT_STAT_DEBUG__FUE_INT_STATUS_DEBUG_MASK                                                          0x00008000L
+#define CP_INT_STAT_DEBUG__GPF_INT_ASSERTED_MASK                                                              0x00010000L
+#define CP_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK                                                 0x00020000L
+#define CP_INT_STAT_DEBUG__CMP_BUSY_INT_ASSERTED_MASK                                                         0x00040000L
+#define CP_INT_STAT_DEBUG__CNTX_BUSY_INT_ASSERTED_MASK                                                        0x00080000L
+#define CP_INT_STAT_DEBUG__CNTX_EMPTY_INT_ASSERTED_MASK                                                       0x00100000L
+#define CP_INT_STAT_DEBUG__GFX_IDLE_INT_ASSERTED_MASK                                                         0x00200000L
+#define CP_INT_STAT_DEBUG__PRIV_INSTR_INT_ASSERTED_MASK                                                       0x00400000L
+#define CP_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                         0x00800000L
+#define CP_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK                                                     0x01000000L
+#define CP_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK                                                       0x04000000L
+#define CP_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK                                               0x08000000L
+#define CP_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK                                                         0x20000000L
+#define CP_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK                                                         0x40000000L
+#define CP_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK                                                         0x80000000L
+//CP_DEBUG_CNTL
+#define CP_DEBUG_CNTL__DEBUG_INDX__SHIFT                                                                      0x0
+#define CP_DEBUG_CNTL__DEBUG_INDX_MASK                                                                        0x0000007FL
+//CP_DEBUG_DATA
+#define CP_DEBUG_DATA__DEBUG_DATA__SHIFT                                                                      0x0
+#define CP_DEBUG_DATA__DEBUG_DATA_MASK                                                                        0xFFFFFFFFL
+//CP_PRIV_VIOLATION_ADDR
+#define CP_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_STATUS__SHIFT                                                  0x0
+#define CP_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_OP__SHIFT                                                      0x1
+#define CP_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_ADDR__SHIFT                                                    0x2
+#define CP_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_STATUS_MASK                                                    0x00000001L
+#define CP_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_OP_MASK                                                        0x00000002L
+#define CP_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_ADDR_MASK                                                      0xFFFFFFFCL
+//CP_PRIV_VIOLATION_ADDR_HI
+#define CP_PRIV_VIOLATION_ADDR_HI__PRIV_VIOLATION_ADDR_HI__SHIFT                                              0x0
+#define CP_PRIV_VIOLATION_ADDR_HI__PRIV_VIOLATION_ADDR_HI_MASK                                                0x000000FFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_padec
+//VGT_DMA_DATA_FIFO_DEPTH
+#define VGT_DMA_DATA_FIFO_DEPTH__DMA_DATA_FIFO_DEPTH__SHIFT                                                   0x0
+#define VGT_DMA_DATA_FIFO_DEPTH__DMA_DATA_FIFO_DEPTH_MASK                                                     0x000003FFL
+//VGT_DMA_REQ_FIFO_DEPTH
+#define VGT_DMA_REQ_FIFO_DEPTH__DMA_REQ_FIFO_DEPTH__SHIFT                                                     0x0
+#define VGT_DMA_REQ_FIFO_DEPTH__DMA_REQ_FIFO_DEPTH_MASK                                                       0x0000003FL
+//VGT_DRAW_INIT_FIFO_DEPTH
+#define VGT_DRAW_INIT_FIFO_DEPTH__DRAW_INIT_FIFO_DEPTH__SHIFT                                                 0x0
+#define VGT_DRAW_INIT_FIFO_DEPTH__DRAW_INIT_FIFO_DEPTH_MASK                                                   0x0000003FL
+//VGT_MC_LAT_CNTL
+#define VGT_MC_LAT_CNTL__MC_TIME_STAMP_RES__SHIFT                                                             0x0
+#define VGT_MC_LAT_CNTL__MC_TIME_STAMP_RES_MASK                                                               0x0000000FL
+//IA_UTCL1_STATUS_2
+#define IA_UTCL1_STATUS_2__FAULT_DETECTED__SHIFT                                                              0x0
+#define IA_UTCL1_STATUS_2__RETRY_DETECTED__SHIFT                                                              0x1
+#define IA_UTCL1_STATUS_2__PRT_DETECTED__SHIFT                                                                0x2
+#define IA_UTCL1_STATUS_2__FAULT_VMID__SHIFT                                                                  0x4
+#define IA_UTCL1_STATUS_2__FAULT_UTCL1ID__SHIFT                                                               0x8
+#define IA_UTCL1_STATUS_2__FAULT_INSTANCEID__SHIFT                                                            0xe
+#define IA_UTCL1_STATUS_2__RETRY_UTCL1ID__SHIFT                                                               0x10
+#define IA_UTCL1_STATUS_2__RETRY_INSTANCEID__SHIFT                                                            0x16
+#define IA_UTCL1_STATUS_2__PRT_UTCL1ID__SHIFT                                                                 0x18
+#define IA_UTCL1_STATUS_2__PRT_INSTANCEID__SHIFT                                                              0x1e
+#define IA_UTCL1_STATUS_2__FAULT_DETECTED_MASK                                                                0x00000001L
+#define IA_UTCL1_STATUS_2__RETRY_DETECTED_MASK                                                                0x00000002L
+#define IA_UTCL1_STATUS_2__PRT_DETECTED_MASK                                                                  0x00000004L
+#define IA_UTCL1_STATUS_2__FAULT_VMID_MASK                                                                    0x000000F0L
+#define IA_UTCL1_STATUS_2__FAULT_UTCL1ID_MASK                                                                 0x00003F00L
+#define IA_UTCL1_STATUS_2__FAULT_INSTANCEID_MASK                                                              0x0000C000L
+#define IA_UTCL1_STATUS_2__RETRY_UTCL1ID_MASK                                                                 0x003F0000L
+#define IA_UTCL1_STATUS_2__RETRY_INSTANCEID_MASK                                                              0x00C00000L
+#define IA_UTCL1_STATUS_2__PRT_UTCL1ID_MASK                                                                   0x3F000000L
+#define IA_UTCL1_STATUS_2__PRT_INSTANCEID_MASK                                                                0xC0000000L
+//GE_WD_CNTL_STATUS
+#define GE_WD_CNTL_STATUS__DIST_BUSY__SHIFT                                                                   0x0
+#define GE_WD_CNTL_STATUS__DIST_BE_BUSY__SHIFT                                                                0x1
+#define GE_WD_CNTL_STATUS__GE_UTCL1_BUSY__SHIFT                                                               0x2
+#define GE_WD_CNTL_STATUS__WD_TE11_BUSY__SHIFT                                                                0x3
+#define GE_WD_CNTL_STATUS__PC_MANAGER_BUSY__SHIFT                                                             0x4
+#define GE_WD_CNTL_STATUS__WLC_BUSY__SHIFT                                                                    0x5
+#define GE_WD_CNTL_STATUS__DIST_BUSY_MASK                                                                     0x00000001L
+#define GE_WD_CNTL_STATUS__DIST_BE_BUSY_MASK                                                                  0x00000002L
+#define GE_WD_CNTL_STATUS__GE_UTCL1_BUSY_MASK                                                                 0x00000004L
+#define GE_WD_CNTL_STATUS__WD_TE11_BUSY_MASK                                                                  0x00000008L
+#define GE_WD_CNTL_STATUS__PC_MANAGER_BUSY_MASK                                                               0x00000010L
+#define GE_WD_CNTL_STATUS__WLC_BUSY_MASK                                                                      0x00000020L
+//WD_UTCL1_CNTL
+#define WD_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                            0x0
+#define WD_UTCL1_CNTL__VMID_RESET_MODE__SHIFT                                                                 0x17
+#define WD_UTCL1_CNTL__DROP_MODE__SHIFT                                                                       0x18
+#define WD_UTCL1_CNTL__BYPASS__SHIFT                                                                          0x19
+#define WD_UTCL1_CNTL__INVALIDATE__SHIFT                                                                      0x1a
+#define WD_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                 0x1b
+#define WD_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                     0x1c
+#define WD_UTCL1_CNTL__MTYPE_OVERRIDE__SHIFT                                                                  0x1d
+#define WD_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                              0x000FFFFFL
+#define WD_UTCL1_CNTL__VMID_RESET_MODE_MASK                                                                   0x00800000L
+#define WD_UTCL1_CNTL__DROP_MODE_MASK                                                                         0x01000000L
+#define WD_UTCL1_CNTL__BYPASS_MASK                                                                            0x02000000L
+#define WD_UTCL1_CNTL__INVALIDATE_MASK                                                                        0x04000000L
+#define WD_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                   0x08000000L
+#define WD_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                       0x10000000L
+#define WD_UTCL1_CNTL__MTYPE_OVERRIDE_MASK                                                                    0x20000000L
+//WD_UTCL1_STATUS
+#define WD_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                                0x0
+#define WD_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                                0x1
+#define WD_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                  0x2
+#define WD_UTCL1_STATUS__FAULT_VMID__SHIFT                                                                    0x4
+#define WD_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                 0x8
+#define WD_UTCL1_STATUS__FAULT_INSTANCEID__SHIFT                                                              0xe
+#define WD_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                 0x10
+#define WD_UTCL1_STATUS__RETRY_INSTANCEID__SHIFT                                                              0x16
+#define WD_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                   0x18
+#define WD_UTCL1_STATUS__PRT_INSTANCEID__SHIFT                                                                0x1e
+#define WD_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                  0x00000001L
+#define WD_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                  0x00000002L
+#define WD_UTCL1_STATUS__PRT_DETECTED_MASK                                                                    0x00000004L
+#define WD_UTCL1_STATUS__FAULT_VMID_MASK                                                                      0x000000F0L
+#define WD_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                   0x00003F00L
+#define WD_UTCL1_STATUS__FAULT_INSTANCEID_MASK                                                                0x0000C000L
+#define WD_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                   0x003F0000L
+#define WD_UTCL1_STATUS__RETRY_INSTANCEID_MASK                                                                0x00C00000L
+#define WD_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                     0x3F000000L
+#define WD_UTCL1_STATUS__PRT_INSTANCEID_MASK                                                                  0xC0000000L
+//IA_UTCL1_CNTL
+#define IA_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                            0x0
+#define IA_UTCL1_CNTL__VMID_RESET_MODE__SHIFT                                                                 0x17
+#define IA_UTCL1_CNTL__DROP_MODE__SHIFT                                                                       0x18
+#define IA_UTCL1_CNTL__BYPASS__SHIFT                                                                          0x19
+#define IA_UTCL1_CNTL__INVALIDATE__SHIFT                                                                      0x1a
+#define IA_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                 0x1b
+#define IA_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                     0x1c
+#define IA_UTCL1_CNTL__MTYPE_OVERRIDE__SHIFT                                                                  0x1d
+#define IA_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                              0x000FFFFFL
+#define IA_UTCL1_CNTL__VMID_RESET_MODE_MASK                                                                   0x00800000L
+#define IA_UTCL1_CNTL__DROP_MODE_MASK                                                                         0x01000000L
+#define IA_UTCL1_CNTL__BYPASS_MASK                                                                            0x02000000L
+#define IA_UTCL1_CNTL__INVALIDATE_MASK                                                                        0x04000000L
+#define IA_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                   0x08000000L
+#define IA_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                       0x10000000L
+#define IA_UTCL1_CNTL__MTYPE_OVERRIDE_MASK                                                                    0x20000000L
+//IA_UTCL1_STATUS
+#define IA_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                                0x0
+#define IA_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                                0x1
+#define IA_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                  0x2
+#define IA_UTCL1_STATUS__FAULT_VMID__SHIFT                                                                    0x4
+#define IA_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                 0x8
+#define IA_UTCL1_STATUS__FAULT_INSTANCEID__SHIFT                                                              0xe
+#define IA_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                 0x10
+#define IA_UTCL1_STATUS__RETRY_INSTANCEID__SHIFT                                                              0x16
+#define IA_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                   0x18
+#define IA_UTCL1_STATUS__PRT_INSTANCEID__SHIFT                                                                0x1e
+#define IA_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                  0x00000001L
+#define IA_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                  0x00000002L
+#define IA_UTCL1_STATUS__PRT_DETECTED_MASK                                                                    0x00000004L
+#define IA_UTCL1_STATUS__FAULT_VMID_MASK                                                                      0x000000F0L
+#define IA_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                   0x00003F00L
+#define IA_UTCL1_STATUS__FAULT_INSTANCEID_MASK                                                                0x0000C000L
+#define IA_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                   0x003F0000L
+#define IA_UTCL1_STATUS__RETRY_INSTANCEID_MASK                                                                0x00C00000L
+#define IA_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                     0x3F000000L
+#define IA_UTCL1_STATUS__PRT_INSTANCEID_MASK                                                                  0xC0000000L
+//CC_GC_SA_UNIT_DISABLE
+#define CC_GC_SA_UNIT_DISABLE__SA_DISABLE__SHIFT                                                              0x8
+#define CC_GC_SA_UNIT_DISABLE__SA_DISABLE_MASK                                                                0x00FFFF00L
+//GE_FED_STATUS
+#define GE_FED_STATUS__DMA_C0_FED_ERROR__SHIFT                                                                0x0
+#define GE_FED_STATUS__DMA_C1_FED_ERROR__SHIFT                                                                0x1
+#define GE_FED_STATUS__TF_REQ_FED_ERROR__SHIFT                                                                0x2
+#define GE_FED_STATUS__DMA_C0_FED_ERROR_MASK                                                                  0x00000001L
+#define GE_FED_STATUS__DMA_C1_FED_ERROR_MASK                                                                  0x00000002L
+#define GE_FED_STATUS__TF_REQ_FED_ERROR_MASK                                                                  0x00000004L
+//GE_PRIV_CONTROL
+#define GE_PRIV_CONTROL__RESERVED__SHIFT                                                                      0x0
+#define GE_PRIV_CONTROL__CLAMP_PRIMGRP_SIZE__SHIFT                                                            0x1
+#define GE_PRIV_CONTROL__FGCG_OVERRIDE__SHIFT                                                                 0xf
+#define GE_PRIV_CONTROL__CLAMP_HS_OFFCHIP_PER_SE_OVERRIDE__SHIFT                                              0x10
+#define GE_PRIV_CONTROL__MIN_ATTR_GRPS__SHIFT                                                                 0x12
+#define GE_PRIV_CONTROL__RESERVED_MASK                                                                        0x00000001L
+#define GE_PRIV_CONTROL__CLAMP_PRIMGRP_SIZE_MASK                                                              0x000003FEL
+#define GE_PRIV_CONTROL__FGCG_OVERRIDE_MASK                                                                   0x00008000L
+#define GE_PRIV_CONTROL__CLAMP_HS_OFFCHIP_PER_SE_OVERRIDE_MASK                                                0x00010000L
+#define GE_PRIV_CONTROL__MIN_ATTR_GRPS_MASK                                                                   0x003C0000L
+//GE_STATUS
+#define GE_STATUS__PERFCOUNTER_STATUS__SHIFT                                                                  0x0
+#define GE_STATUS__THREAD_TRACE_STATUS__SHIFT                                                                 0x1
+#define GE_STATUS__PERFCOUNTER_STATUS_MASK                                                                    0x00000001L
+#define GE_STATUS__THREAD_TRACE_STATUS_MASK                                                                   0x00000002L
+//VGT_GS_MAX_WAVE_ID
+#define VGT_GS_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT                                                                0x0
+#define VGT_GS_MAX_WAVE_ID__MAX_WAVE_ID_MASK                                                                  0x00000FFFL
+//GFX_PIPE_CONTROL
+#define GFX_PIPE_CONTROL__HYSTERESIS_CNT__SHIFT                                                               0x0
+#define GFX_PIPE_CONTROL__CONTEXT_SUSPEND_EN__SHIFT                                                           0x10
+#define GFX_PIPE_CONTROL__CONTEXT_SUSPEND_STALL_EN__SHIFT                                                     0x11
+#define GFX_PIPE_CONTROL__HYSTERESIS_CNT_MASK                                                                 0x00001FFFL
+#define GFX_PIPE_CONTROL__CONTEXT_SUSPEND_EN_MASK                                                             0x00010000L
+#define GFX_PIPE_CONTROL__CONTEXT_SUSPEND_STALL_EN_MASK                                                       0x00020000L
+//VGT_RESET_DEBUG
+#define VGT_RESET_DEBUG__GS_DISABLE__SHIFT                                                                    0x0
+#define VGT_RESET_DEBUG__TESS_DISABLE__SHIFT                                                                  0x1
+#define VGT_RESET_DEBUG__WD_DISABLE__SHIFT                                                                    0x2
+#define VGT_RESET_DEBUG__DISABLE_TE11_DIST_PIPE0__SHIFT                                                       0x3
+#define VGT_RESET_DEBUG__DISABLE_TE11_DIST_PIPE1__SHIFT                                                       0x4
+#define VGT_RESET_DEBUG__ENABLE_VMID_RESET_UTCL1__SHIFT                                                       0x5
+#define VGT_RESET_DEBUG__DISABLE_PREFETCH__SHIFT                                                              0x6
+#define VGT_RESET_DEBUG__DISABLE_SWITCH_MODE_STALL_FIX__SHIFT                                                 0x7
+#define VGT_RESET_DEBUG__DISABLE_SENDING_MULTIPLE_SE_IN_PD__SHIFT                                             0x8
+#define VGT_RESET_DEBUG__ENABLE_DIST_STALL_TESS_ON_OFF__SHIFT                                                 0x9
+#define VGT_RESET_DEBUG__DISABLE_PATCH_OPTIMIZATION__SHIFT                                                    0xa
+#define VGT_RESET_DEBUG__ENABLE_DIST_STALL_TESS_OFF_ON__SHIFT                                                 0xb
+#define VGT_RESET_DEBUG__DISABLE_MERGE_GRP_PERF_FIX__SHIFT                                                    0xc
+#define VGT_RESET_DEBUG__DISABLE_MESH_SHADER_ATTR_PACKING__SHIFT                                              0xd
+#define VGT_RESET_DEBUG__ENABLE_SMALL_INST_PACK_ADJ_GS_OFF__SHIFT                                             0xe
+#define VGT_RESET_DEBUG__DISABLE_PATCH_DIST_LAST_DONUT_SE_SWITCH_LOGIC__SHIFT                                 0xf
+#define VGT_RESET_DEBUG__DISABLE_GE1_FED_OPT__SHIFT                                                           0x10
+#define VGT_RESET_DEBUG__DISABLE_GE2_DIST_FED_OPT__SHIFT                                                      0x11
+#define VGT_RESET_DEBUG__SPARE__SHIFT                                                                         0x12
+#define VGT_RESET_DEBUG__GS_DISABLE_MASK                                                                      0x00000001L
+#define VGT_RESET_DEBUG__TESS_DISABLE_MASK                                                                    0x00000002L
+#define VGT_RESET_DEBUG__WD_DISABLE_MASK                                                                      0x00000004L
+#define VGT_RESET_DEBUG__DISABLE_TE11_DIST_PIPE0_MASK                                                         0x00000008L
+#define VGT_RESET_DEBUG__DISABLE_TE11_DIST_PIPE1_MASK                                                         0x00000010L
+#define VGT_RESET_DEBUG__ENABLE_VMID_RESET_UTCL1_MASK                                                         0x00000020L
+#define VGT_RESET_DEBUG__DISABLE_PREFETCH_MASK                                                                0x00000040L
+#define VGT_RESET_DEBUG__DISABLE_SWITCH_MODE_STALL_FIX_MASK                                                   0x00000080L
+#define VGT_RESET_DEBUG__DISABLE_SENDING_MULTIPLE_SE_IN_PD_MASK                                               0x00000100L
+#define VGT_RESET_DEBUG__ENABLE_DIST_STALL_TESS_ON_OFF_MASK                                                   0x00000200L
+#define VGT_RESET_DEBUG__DISABLE_PATCH_OPTIMIZATION_MASK                                                      0x00000400L
+#define VGT_RESET_DEBUG__ENABLE_DIST_STALL_TESS_OFF_ON_MASK                                                   0x00000800L
+#define VGT_RESET_DEBUG__DISABLE_MERGE_GRP_PERF_FIX_MASK                                                      0x00001000L
+#define VGT_RESET_DEBUG__DISABLE_MESH_SHADER_ATTR_PACKING_MASK                                                0x00002000L
+#define VGT_RESET_DEBUG__ENABLE_SMALL_INST_PACK_ADJ_GS_OFF_MASK                                               0x00004000L
+#define VGT_RESET_DEBUG__DISABLE_PATCH_DIST_LAST_DONUT_SE_SWITCH_LOGIC_MASK                                   0x00008000L
+#define VGT_RESET_DEBUG__DISABLE_GE1_FED_OPT_MASK                                                             0x00010000L
+#define VGT_RESET_DEBUG__DISABLE_GE2_DIST_FED_OPT_MASK                                                        0x00020000L
+#define VGT_RESET_DEBUG__SPARE_MASK                                                                           0xFFFC0000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_shdec
+//COMPUTE_DISPATCH_INITIATOR
+#define COMPUTE_DISPATCH_INITIATOR__COMPUTE_SHADER_EN__SHIFT                                                  0x0
+#define COMPUTE_DISPATCH_INITIATOR__PARTIAL_TG_EN__SHIFT                                                      0x1
+#define COMPUTE_DISPATCH_INITIATOR__FORCE_START_AT_000__SHIFT                                                 0x2
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_ENBL__SHIFT                                                0x3
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_MODE__SHIFT                                                0x4
+#define COMPUTE_DISPATCH_INITIATOR__USE_THREAD_DIMENSIONS__SHIFT                                              0x5
+#define COMPUTE_DISPATCH_INITIATOR__ORDER_MODE__SHIFT                                                         0x6
+#define COMPUTE_DISPATCH_INITIATOR__SCALAR_L0_INV__SHIFT                                                      0xa
+#define COMPUTE_DISPATCH_INITIATOR__VECTOR_L0_INV__SHIFT                                                      0xb
+#define COMPUTE_DISPATCH_INITIATOR__PING_PONG_EN__SHIFT                                                       0xc
+#define COMPUTE_DISPATCH_INITIATOR__TUNNEL_ENABLE__SHIFT                                                      0xd
+#define COMPUTE_DISPATCH_INITIATOR__RESTORE__SHIFT                                                            0xe
+#define COMPUTE_DISPATCH_INITIATOR__CS_W32_EN__SHIFT                                                          0xf
+#define COMPUTE_DISPATCH_INITIATOR__AMP_SHADER_EN__SHIFT                                                      0x10
+#define COMPUTE_DISPATCH_INITIATOR__DISABLE_DISP_PREMPT_EN__SHIFT                                             0x11
+#define COMPUTE_DISPATCH_INITIATOR__INTERLEAVE_2D_EN__SHIFT                                                   0x12
+#define COMPUTE_DISPATCH_INITIATOR__WGS_DISPATCH__SHIFT                                                       0x13
+#define COMPUTE_DISPATCH_INITIATOR__CLUSTER_EN__SHIFT                                                         0x14
+#define COMPUTE_DISPATCH_INITIATOR__TTRACE_QUEUE_ID__SHIFT                                                    0x1d
+#define COMPUTE_DISPATCH_INITIATOR__COMPUTE_SHADER_EN_MASK                                                    0x00000001L
+#define COMPUTE_DISPATCH_INITIATOR__PARTIAL_TG_EN_MASK                                                        0x00000002L
+#define COMPUTE_DISPATCH_INITIATOR__FORCE_START_AT_000_MASK                                                   0x00000004L
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_ENBL_MASK                                                  0x00000008L
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_MODE_MASK                                                  0x00000010L
+#define COMPUTE_DISPATCH_INITIATOR__USE_THREAD_DIMENSIONS_MASK                                                0x00000020L
+#define COMPUTE_DISPATCH_INITIATOR__ORDER_MODE_MASK                                                           0x000000C0L
+#define COMPUTE_DISPATCH_INITIATOR__SCALAR_L0_INV_MASK                                                        0x00000400L
+#define COMPUTE_DISPATCH_INITIATOR__VECTOR_L0_INV_MASK                                                        0x00000800L
+#define COMPUTE_DISPATCH_INITIATOR__PING_PONG_EN_MASK                                                         0x00001000L
+#define COMPUTE_DISPATCH_INITIATOR__TUNNEL_ENABLE_MASK                                                        0x00002000L
+#define COMPUTE_DISPATCH_INITIATOR__RESTORE_MASK                                                              0x00004000L
+#define COMPUTE_DISPATCH_INITIATOR__CS_W32_EN_MASK                                                            0x00008000L
+#define COMPUTE_DISPATCH_INITIATOR__AMP_SHADER_EN_MASK                                                        0x00010000L
+#define COMPUTE_DISPATCH_INITIATOR__DISABLE_DISP_PREMPT_EN_MASK                                               0x00020000L
+#define COMPUTE_DISPATCH_INITIATOR__INTERLEAVE_2D_EN_MASK                                                     0x00040000L
+#define COMPUTE_DISPATCH_INITIATOR__WGS_DISPATCH_MASK                                                         0x00080000L
+#define COMPUTE_DISPATCH_INITIATOR__CLUSTER_EN_MASK                                                           0x00100000L
+#define COMPUTE_DISPATCH_INITIATOR__TTRACE_QUEUE_ID_MASK                                                      0xE0000000L
+//COMPUTE_DIM_X
+#define COMPUTE_DIM_X__SIZE__SHIFT                                                                            0x0
+#define COMPUTE_DIM_X__SIZE_MASK                                                                              0xFFFFFFFFL
+//COMPUTE_DIM_Y
+#define COMPUTE_DIM_Y__SIZE__SHIFT                                                                            0x0
+#define COMPUTE_DIM_Y__SIZE_MASK                                                                              0xFFFFFFFFL
+//COMPUTE_DIM_Z
+#define COMPUTE_DIM_Z__SIZE__SHIFT                                                                            0x0
+#define COMPUTE_DIM_Z__SIZE_MASK                                                                              0xFFFFFFFFL
+//COMPUTE_START_X
+#define COMPUTE_START_X__START__SHIFT                                                                         0x0
+#define COMPUTE_START_X__START_MASK                                                                           0xFFFFFFFFL
+//COMPUTE_START_Y
+#define COMPUTE_START_Y__START__SHIFT                                                                         0x0
+#define COMPUTE_START_Y__START_MASK                                                                           0xFFFFFFFFL
+//COMPUTE_START_Z
+#define COMPUTE_START_Z__START__SHIFT                                                                         0x0
+#define COMPUTE_START_Z__START_MASK                                                                           0xFFFFFFFFL
+//COMPUTE_NUM_THREAD_X
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_FULL__SHIFT                                                          0x0
+#define COMPUTE_NUM_THREAD_X__INTERLEAVE_BITS_X__SHIFT                                                        0xd
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_PARTIAL__SHIFT                                                       0x10
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_FULL_MASK                                                            0x00001FFFL
+#define COMPUTE_NUM_THREAD_X__INTERLEAVE_BITS_X_MASK                                                          0x0000E000L
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_PARTIAL_MASK                                                         0xFFFF0000L
+//COMPUTE_NUM_THREAD_Y
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_FULL__SHIFT                                                          0x0
+#define COMPUTE_NUM_THREAD_Y__INTERLEAVE_BITS_Y__SHIFT                                                        0xd
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_PARTIAL__SHIFT                                                       0x10
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_FULL_MASK                                                            0x00001FFFL
+#define COMPUTE_NUM_THREAD_Y__INTERLEAVE_BITS_Y_MASK                                                          0x0000E000L
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_PARTIAL_MASK                                                         0xFFFF0000L
+//COMPUTE_NUM_THREAD_Z
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_FULL__SHIFT                                                          0x0
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_PARTIAL__SHIFT                                                       0x10
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_FULL_MASK                                                            0x0000FFFFL
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_PARTIAL_MASK                                                         0xFFFF0000L
+//COMPUTE_PIPELINESTAT_ENABLE
+#define COMPUTE_PIPELINESTAT_ENABLE__PIPELINESTAT_ENABLE__SHIFT                                               0x0
+#define COMPUTE_PIPELINESTAT_ENABLE__PIPELINESTAT_ENABLE_MASK                                                 0x00000001L
+//COMPUTE_PERFCOUNT_ENABLE
+#define COMPUTE_PERFCOUNT_ENABLE__PERFCOUNT_ENABLE__SHIFT                                                     0x0
+#define COMPUTE_PERFCOUNT_ENABLE__PERFCOUNT_ENABLE_MASK                                                       0x00000001L
+//COMPUTE_PGM_LO
+#define COMPUTE_PGM_LO__DATA__SHIFT                                                                           0x0
+#define COMPUTE_PGM_LO__DATA_MASK                                                                             0xFFFFFFFFL
+//COMPUTE_PGM_HI
+#define COMPUTE_PGM_HI__DATA__SHIFT                                                                           0x0
+#define COMPUTE_PGM_HI__DATA_MASK                                                                             0x0001FFFFL
+//COMPUTE_DISPATCH_PKT_ADDR_LO
+#define COMPUTE_DISPATCH_PKT_ADDR_LO__DATA__SHIFT                                                             0x0
+#define COMPUTE_DISPATCH_PKT_ADDR_LO__DATA_MASK                                                               0xFFFFFFFFL
+//COMPUTE_DISPATCH_PKT_ADDR_HI
+#define COMPUTE_DISPATCH_PKT_ADDR_HI__DATA__SHIFT                                                             0x0
+#define COMPUTE_DISPATCH_PKT_ADDR_HI__DATA_MASK                                                               0x000000FFL
+//COMPUTE_DISPATCH_SCRATCH_BASE_LO
+#define COMPUTE_DISPATCH_SCRATCH_BASE_LO__DATA__SHIFT                                                         0x0
+#define COMPUTE_DISPATCH_SCRATCH_BASE_LO__DATA_MASK                                                           0xFFFFFFFFL
+//COMPUTE_DISPATCH_SCRATCH_BASE_HI
+#define COMPUTE_DISPATCH_SCRATCH_BASE_HI__DATA__SHIFT                                                         0x0
+#define COMPUTE_DISPATCH_SCRATCH_BASE_HI__DATA_MASK                                                           0x0001FFFFL
+//COMPUTE_PGM_RSRC1
+#define COMPUTE_PGM_RSRC1__VGPRS__SHIFT                                                                       0x0
+#define COMPUTE_PGM_RSRC1__SGPRS__SHIFT                                                                       0x6
+#define COMPUTE_PGM_RSRC1__PRIORITY__SHIFT                                                                    0xa
+#define COMPUTE_PGM_RSRC1__FLOAT_MODE__SHIFT                                                                  0xc
+#define COMPUTE_PGM_RSRC1__PRIV__SHIFT                                                                        0x14
+#define COMPUTE_PGM_RSRC1__WG_RR_EN__SHIFT                                                                    0x15
+#define COMPUTE_PGM_RSRC1__DEBUG_MODE__SHIFT                                                                  0x16
+#define COMPUTE_PGM_RSRC1__DISABLE_PERF__SHIFT                                                                0x17
+#define COMPUTE_PGM_RSRC1__BULKY__SHIFT                                                                       0x18
+#define COMPUTE_PGM_RSRC1__CDBG_USER__SHIFT                                                                   0x19
+#define COMPUTE_PGM_RSRC1__FP16_OVFL__SHIFT                                                                   0x1a
+#define COMPUTE_PGM_RSRC1__FLAT_SCRATCH_IS_NV__SHIFT                                                          0x1b
+#define COMPUTE_PGM_RSRC1__WGP_MODE__SHIFT                                                                    0x1d
+#define COMPUTE_PGM_RSRC1__MEM_ORDERED__SHIFT                                                                 0x1e
+#define COMPUTE_PGM_RSRC1__FWD_PROGRESS__SHIFT                                                                0x1f
+#define COMPUTE_PGM_RSRC1__VGPRS_MASK                                                                         0x0000003FL
+#define COMPUTE_PGM_RSRC1__SGPRS_MASK                                                                         0x000003C0L
+#define COMPUTE_PGM_RSRC1__PRIORITY_MASK                                                                      0x00000C00L
+#define COMPUTE_PGM_RSRC1__FLOAT_MODE_MASK                                                                    0x000FF000L
+#define COMPUTE_PGM_RSRC1__PRIV_MASK                                                                          0x00100000L
+#define COMPUTE_PGM_RSRC1__WG_RR_EN_MASK                                                                      0x00200000L
+#define COMPUTE_PGM_RSRC1__DEBUG_MODE_MASK                                                                    0x00400000L
+#define COMPUTE_PGM_RSRC1__DISABLE_PERF_MASK                                                                  0x00800000L
+#define COMPUTE_PGM_RSRC1__BULKY_MASK                                                                         0x01000000L
+#define COMPUTE_PGM_RSRC1__CDBG_USER_MASK                                                                     0x02000000L
+#define COMPUTE_PGM_RSRC1__FP16_OVFL_MASK                                                                     0x04000000L
+#define COMPUTE_PGM_RSRC1__FLAT_SCRATCH_IS_NV_MASK                                                            0x08000000L
+#define COMPUTE_PGM_RSRC1__WGP_MODE_MASK                                                                      0x20000000L
+#define COMPUTE_PGM_RSRC1__MEM_ORDERED_MASK                                                                   0x40000000L
+#define COMPUTE_PGM_RSRC1__FWD_PROGRESS_MASK                                                                  0x80000000L
+//COMPUTE_PGM_RSRC2
+#define COMPUTE_PGM_RSRC2__SCRATCH_EN__SHIFT                                                                  0x0
+#define COMPUTE_PGM_RSRC2__USER_SGPR__SHIFT                                                                   0x1
+#define COMPUTE_PGM_RSRC2__TGID_X_EN__SHIFT                                                                   0x7
+#define COMPUTE_PGM_RSRC2__TGID_Y_EN__SHIFT                                                                   0x8
+#define COMPUTE_PGM_RSRC2__TGID_Z_EN__SHIFT                                                                   0x9
+#define COMPUTE_PGM_RSRC2__TG_SIZE_EN__SHIFT                                                                  0xa
+#define COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT__SHIFT                                                              0xb
+#define COMPUTE_PGM_RSRC2__EXCP_EN_MSB__SHIFT                                                                 0xd
+#define COMPUTE_PGM_RSRC2__LDS_SIZE__SHIFT                                                                    0xf
+#define COMPUTE_PGM_RSRC2__EXCP_EN__SHIFT                                                                     0x18
+#define COMPUTE_PGM_RSRC2__WGP_TAKEOVER__SHIFT                                                                0x1f
+#define COMPUTE_PGM_RSRC2__SCRATCH_EN_MASK                                                                    0x00000001L
+#define COMPUTE_PGM_RSRC2__USER_SGPR_MASK                                                                     0x0000007EL
+#define COMPUTE_PGM_RSRC2__TGID_X_EN_MASK                                                                     0x00000080L
+#define COMPUTE_PGM_RSRC2__TGID_Y_EN_MASK                                                                     0x00000100L
+#define COMPUTE_PGM_RSRC2__TGID_Z_EN_MASK                                                                     0x00000200L
+#define COMPUTE_PGM_RSRC2__TG_SIZE_EN_MASK                                                                    0x00000400L
+#define COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT_MASK                                                                0x00001800L
+#define COMPUTE_PGM_RSRC2__EXCP_EN_MSB_MASK                                                                   0x00006000L
+#define COMPUTE_PGM_RSRC2__LDS_SIZE_MASK                                                                      0x00FF8000L
+#define COMPUTE_PGM_RSRC2__EXCP_EN_MASK                                                                       0x7F000000L
+#define COMPUTE_PGM_RSRC2__WGP_TAKEOVER_MASK                                                                  0x80000000L
+//COMPUTE_VMID
+#define COMPUTE_VMID__DATA__SHIFT                                                                             0x0
+#define COMPUTE_VMID__DATA_MASK                                                                               0x0000000FL
+//COMPUTE_RESOURCE_LIMITS
+#define COMPUTE_RESOURCE_LIMITS__WAVES_PER_SH__SHIFT                                                          0x0
+#define COMPUTE_RESOURCE_LIMITS__TG_PER_CU__SHIFT                                                             0xc
+#define COMPUTE_RESOURCE_LIMITS__LOCK_THRESHOLD__SHIFT                                                        0x10
+#define COMPUTE_RESOURCE_LIMITS__SIMD_DEST_CNTL__SHIFT                                                        0x16
+#define COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST__SHIFT                                                       0x17
+#define COMPUTE_RESOURCE_LIMITS__CU_GROUP_COUNT__SHIFT                                                        0x18
+#define COMPUTE_RESOURCE_LIMITS__CLUSTER_PER_SE__SHIFT                                                        0x1b
+#define COMPUTE_RESOURCE_LIMITS__SIMD_WALK_PATTERN__SHIFT                                                     0x1f
+#define COMPUTE_RESOURCE_LIMITS__WAVES_PER_SH_MASK                                                            0x000003FFL
+#define COMPUTE_RESOURCE_LIMITS__TG_PER_CU_MASK                                                               0x0000F000L
+#define COMPUTE_RESOURCE_LIMITS__LOCK_THRESHOLD_MASK                                                          0x003F0000L
+#define COMPUTE_RESOURCE_LIMITS__SIMD_DEST_CNTL_MASK                                                          0x00400000L
+#define COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST_MASK                                                         0x00800000L
+#define COMPUTE_RESOURCE_LIMITS__CU_GROUP_COUNT_MASK                                                          0x07000000L
+#define COMPUTE_RESOURCE_LIMITS__CLUSTER_PER_SE_MASK                                                          0x78000000L
+#define COMPUTE_RESOURCE_LIMITS__SIMD_WALK_PATTERN_MASK                                                       0x80000000L
+//COMPUTE_TMPRING_SIZE
+#define COMPUTE_TMPRING_SIZE__WAVES__SHIFT                                                                    0x0
+#define COMPUTE_TMPRING_SIZE__WAVESIZE__SHIFT                                                                 0xc
+#define COMPUTE_TMPRING_SIZE__WAVES_MASK                                                                      0x00000FFFL
+#define COMPUTE_TMPRING_SIZE__WAVESIZE_MASK                                                                   0x3FFFF000L
+//COMPUTE_RESTART_X
+#define COMPUTE_RESTART_X__RESTART__SHIFT                                                                     0x0
+#define COMPUTE_RESTART_X__RESTART_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_RESTART_Y
+#define COMPUTE_RESTART_Y__RESTART__SHIFT                                                                     0x0
+#define COMPUTE_RESTART_Y__RESTART_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_RESTART_Z
+#define COMPUTE_RESTART_Z__RESTART__SHIFT                                                                     0x0
+#define COMPUTE_RESTART_Z__RESTART_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_THREAD_TRACE_ENABLE
+#define COMPUTE_THREAD_TRACE_ENABLE__THREAD_TRACE_ENABLE__SHIFT                                               0x0
+#define COMPUTE_THREAD_TRACE_ENABLE__THREAD_TRACE_ENABLE_MASK                                                 0x00000001L
+//COMPUTE_MISC_RESERVED
+#define COMPUTE_MISC_RESERVED__SEND_SEID__SHIFT                                                               0x0
+#define COMPUTE_MISC_RESERVED__RESERVED4__SHIFT                                                               0x4
+#define COMPUTE_MISC_RESERVED__WAVE_ID_BASE__SHIFT                                                            0x5
+#define COMPUTE_MISC_RESERVED__SEND_SEID_MASK                                                                 0x0000000FL
+#define COMPUTE_MISC_RESERVED__RESERVED4_MASK                                                                 0x00000010L
+#define COMPUTE_MISC_RESERVED__WAVE_ID_BASE_MASK                                                              0x0001FFE0L
+//COMPUTE_DISPATCH_ID
+#define COMPUTE_DISPATCH_ID__DISPATCH_ID__SHIFT                                                               0x0
+#define COMPUTE_DISPATCH_ID__DISPATCH_ID_MASK                                                                 0xFFFFFFFFL
+//COMPUTE_THREADGROUP_ID
+#define COMPUTE_THREADGROUP_ID__THREADGROUP_ID__SHIFT                                                         0x0
+#define COMPUTE_THREADGROUP_ID__THREADGROUP_ID_MASK                                                           0xFFFFFFFFL
+//COMPUTE_REQ_CTRL
+#define COMPUTE_REQ_CTRL__SOFT_GROUPING_EN__SHIFT                                                             0x0
+#define COMPUTE_REQ_CTRL__NUMBER_OF_REQUESTS_PER_CU__SHIFT                                                    0x1
+#define COMPUTE_REQ_CTRL__SOFT_GROUPING_ALLOCATION_TIMEOUT__SHIFT                                             0x5
+#define COMPUTE_REQ_CTRL__HARD_LOCK_HYSTERESIS__SHIFT                                                         0x9
+#define COMPUTE_REQ_CTRL__HARD_LOCK_LOW_THRESHOLD__SHIFT                                                      0xa
+#define COMPUTE_REQ_CTRL__PRODUCER_REQUEST_LOCKOUT__SHIFT                                                     0xf
+#define COMPUTE_REQ_CTRL__GLOBAL_SCANNING_EN__SHIFT                                                           0x10
+#define COMPUTE_REQ_CTRL__ALLOCATION_RATE_THROTTLING_THRESHOLD__SHIFT                                         0x11
+#define COMPUTE_REQ_CTRL__DEDICATED_PREALLOCATION_BUFFER_LIMIT__SHIFT                                         0x14
+#define COMPUTE_REQ_CTRL__SOFT_GROUPING_EN_MASK                                                               0x00000001L
+#define COMPUTE_REQ_CTRL__NUMBER_OF_REQUESTS_PER_CU_MASK                                                      0x0000001EL
+#define COMPUTE_REQ_CTRL__SOFT_GROUPING_ALLOCATION_TIMEOUT_MASK                                               0x000001E0L
+#define COMPUTE_REQ_CTRL__HARD_LOCK_HYSTERESIS_MASK                                                           0x00000200L
+#define COMPUTE_REQ_CTRL__HARD_LOCK_LOW_THRESHOLD_MASK                                                        0x00007C00L
+#define COMPUTE_REQ_CTRL__PRODUCER_REQUEST_LOCKOUT_MASK                                                       0x00008000L
+#define COMPUTE_REQ_CTRL__GLOBAL_SCANNING_EN_MASK                                                             0x00010000L
+#define COMPUTE_REQ_CTRL__ALLOCATION_RATE_THROTTLING_THRESHOLD_MASK                                           0x000E0000L
+#define COMPUTE_REQ_CTRL__DEDICATED_PREALLOCATION_BUFFER_LIMIT_MASK                                           0x07F00000L
+//COMPUTE_USER_ACCUM_0
+#define COMPUTE_USER_ACCUM_0__CONTRIBUTION__SHIFT                                                             0x0
+#define COMPUTE_USER_ACCUM_0__CONTRIBUTION_MASK                                                               0x0000007FL
+//COMPUTE_USER_ACCUM_1
+#define COMPUTE_USER_ACCUM_1__CONTRIBUTION__SHIFT                                                             0x0
+#define COMPUTE_USER_ACCUM_1__CONTRIBUTION_MASK                                                               0x0000007FL
+//COMPUTE_USER_ACCUM_2
+#define COMPUTE_USER_ACCUM_2__CONTRIBUTION__SHIFT                                                             0x0
+#define COMPUTE_USER_ACCUM_2__CONTRIBUTION_MASK                                                               0x0000007FL
+//COMPUTE_USER_ACCUM_3
+#define COMPUTE_USER_ACCUM_3__CONTRIBUTION__SHIFT                                                             0x0
+#define COMPUTE_USER_ACCUM_3__CONTRIBUTION_MASK                                                               0x0000007FL
+//COMPUTE_PGM_RSRC3
+#define COMPUTE_PGM_RSRC3__SHARED_VGPR_CNT__SHIFT                                                             0x0
+#define COMPUTE_PGM_RSRC3__INST_PREF_SIZE__SHIFT                                                              0x4
+#define COMPUTE_PGM_RSRC3__GLG_EN__SHIFT                                                                      0xd
+#define COMPUTE_PGM_RSRC3__NAMED_BAR_CNT__SHIFT                                                               0xe
+#define COMPUTE_PGM_RSRC3__DYNAMIC_VGPR__SHIFT                                                                0x11
+#define COMPUTE_PGM_RSRC3__TCP_SPLIT__SHIFT                                                                   0x12
+#define COMPUTE_PGM_RSRC3__DIDT_THROTTLE__SHIFT                                                               0x15
+#define COMPUTE_PGM_RSRC3__IMAGE_OP__SHIFT                                                                    0x1f
+#define COMPUTE_PGM_RSRC3__SHARED_VGPR_CNT_MASK                                                               0x0000000FL
+#define COMPUTE_PGM_RSRC3__INST_PREF_SIZE_MASK                                                                0x00000FF0L
+#define COMPUTE_PGM_RSRC3__GLG_EN_MASK                                                                        0x00002000L
+#define COMPUTE_PGM_RSRC3__NAMED_BAR_CNT_MASK                                                                 0x0001C000L
+#define COMPUTE_PGM_RSRC3__DYNAMIC_VGPR_MASK                                                                  0x00020000L
+#define COMPUTE_PGM_RSRC3__TCP_SPLIT_MASK                                                                     0x001C0000L
+#define COMPUTE_PGM_RSRC3__DIDT_THROTTLE_MASK                                                                 0x00200000L
+#define COMPUTE_PGM_RSRC3__IMAGE_OP_MASK                                                                      0x80000000L
+//COMPUTE_DDID_INDEX
+#define COMPUTE_DDID_INDEX__INDEX__SHIFT                                                                      0x0
+#define COMPUTE_DDID_INDEX__INDEX_MASK                                                                        0x000007FFL
+//COMPUTE_SHADER_CHKSUM
+#define COMPUTE_SHADER_CHKSUM__CHECKSUM__SHIFT                                                                0x0
+#define COMPUTE_SHADER_CHKSUM__CHECKSUM_MASK                                                                  0xFFFFFFFFL
+//COMPUTE_DISPATCH_INTERLEAVE
+#define COMPUTE_DISPATCH_INTERLEAVE__INTERLEAVE_1D__SHIFT                                                     0x0
+#define COMPUTE_DISPATCH_INTERLEAVE__INTERLEAVE_2D_X_SIZE__SHIFT                                              0x10
+#define COMPUTE_DISPATCH_INTERLEAVE__INTERLEAVE_2D_Y_SIZE__SHIFT                                              0x15
+#define COMPUTE_DISPATCH_INTERLEAVE__INTERLEAVE_Z_SIZE__SHIFT                                                 0x1a
+#define COMPUTE_DISPATCH_INTERLEAVE__INTERLEAVE_1D_MASK                                                       0x000003FFL
+#define COMPUTE_DISPATCH_INTERLEAVE__INTERLEAVE_2D_X_SIZE_MASK                                                0x001F0000L
+#define COMPUTE_DISPATCH_INTERLEAVE__INTERLEAVE_2D_Y_SIZE_MASK                                                0x03E00000L
+#define COMPUTE_DISPATCH_INTERLEAVE__INTERLEAVE_Z_SIZE_MASK                                                   0x7C000000L
+//COMPUTE_CURRENT_LOGICAL_XCC_ID
+#define COMPUTE_CURRENT_LOGICAL_XCC_ID__CURRENT_LOGICAL_XCC_ID__SHIFT                                         0x0
+#define COMPUTE_CURRENT_LOGICAL_XCC_ID__CURRENT_LOGICAL_XCC_ID_MASK                                           0x0000000FL
+//COMPUTE_RESTART_CG_TG_ID
+#define COMPUTE_RESTART_CG_TG_ID__RESTART_CG_TG_ID__SHIFT                                                     0x0
+#define COMPUTE_RESTART_CG_TG_ID__RESTART_CG_TG_ID_MASK                                                       0x000FFFFFL
+//COMPUTE_TG_CHUNK_SIZE
+#define COMPUTE_TG_CHUNK_SIZE__TG_CHUNK_SIZE__SHIFT                                                           0x0
+#define COMPUTE_TG_CHUNK_SIZE__TG_CHUNK_SIZE_MASK                                                             0x0000FFFFL
+//COMPUTE_RESTORE_TG_CHUNK_SIZE
+#define COMPUTE_RESTORE_TG_CHUNK_SIZE__RESTORE_TG_CHUNK_SIZE__SHIFT                                           0x0
+#define COMPUTE_RESTORE_TG_CHUNK_SIZE__RESTORE_TG_CHUNK_SIZE_MASK                                             0x0000FFFFL
+//COMPUTE_SPARE_1
+#define COMPUTE_SPARE_1__SIZE__SHIFT                                                                          0x0
+#define COMPUTE_SPARE_1__SIZE_MASK                                                                            0xFFFFFFFFL
+//COMPUTE_SPARE_2
+#define COMPUTE_SPARE_2__SIZE__SHIFT                                                                          0x0
+#define COMPUTE_SPARE_2__SIZE_MASK                                                                            0xFFFFFFFFL
+//COMPUTE_PRESCALED_DIM_X
+#define COMPUTE_PRESCALED_DIM_X__SIZE__SHIFT                                                                  0x0
+#define COMPUTE_PRESCALED_DIM_X__SIZE_MASK                                                                    0xFFFFFFFFL
+//COMPUTE_PRESCALED_DIM_Y
+#define COMPUTE_PRESCALED_DIM_Y__SIZE__SHIFT                                                                  0x0
+#define COMPUTE_PRESCALED_DIM_Y__SIZE_MASK                                                                    0xFFFFFFFFL
+//COMPUTE_PRESCALED_DIM_Z
+#define COMPUTE_PRESCALED_DIM_Z__SIZE__SHIFT                                                                  0x0
+#define COMPUTE_PRESCALED_DIM_Z__SIZE_MASK                                                                    0xFFFFFFFFL
+//COMPUTE_DESTINATION_EN_SE0
+#define COMPUTE_DESTINATION_EN_SE0__CU_EN__SHIFT                                                              0x0
+#define COMPUTE_DESTINATION_EN_SE0__CU_EN_MASK                                                                0xFFFFFFFFL
+//COMPUTE_STATIC_THREAD_MGMT_SE0
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_DESTINATION_EN_SE1
+#define COMPUTE_DESTINATION_EN_SE1__CU_EN__SHIFT                                                              0x0
+#define COMPUTE_DESTINATION_EN_SE1__CU_EN_MASK                                                                0xFFFFFFFFL
+//COMPUTE_STATIC_THREAD_MGMT_SE1
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_DESTINATION_EN_SE2
+#define COMPUTE_DESTINATION_EN_SE2__CU_EN__SHIFT                                                              0x0
+#define COMPUTE_DESTINATION_EN_SE2__CU_EN_MASK                                                                0xFFFFFFFFL
+//COMPUTE_STATIC_THREAD_MGMT_SE2
+#define COMPUTE_STATIC_THREAD_MGMT_SE2__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE2__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE2__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE2__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_DESTINATION_EN_SE3
+#define COMPUTE_DESTINATION_EN_SE3__CU_EN__SHIFT                                                              0x0
+#define COMPUTE_DESTINATION_EN_SE3__CU_EN_MASK                                                                0xFFFFFFFFL
+//COMPUTE_STATIC_THREAD_MGMT_SE3
+#define COMPUTE_STATIC_THREAD_MGMT_SE3__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE3__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE3__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE3__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_STATIC_THREAD_MGMT_SE4
+#define COMPUTE_STATIC_THREAD_MGMT_SE4__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE4__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE4__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE4__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_STATIC_THREAD_MGMT_SE5
+#define COMPUTE_STATIC_THREAD_MGMT_SE5__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE5__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE5__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE5__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_STATIC_THREAD_MGMT_SE6
+#define COMPUTE_STATIC_THREAD_MGMT_SE6__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE6__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE6__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE6__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_STATIC_THREAD_MGMT_SE7
+#define COMPUTE_STATIC_THREAD_MGMT_SE7__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE7__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE7__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE7__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_STATIC_THREAD_MGMT_SE8
+#define COMPUTE_STATIC_THREAD_MGMT_SE8__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE8__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE8__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE8__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_USER_DATA_0
+#define COMPUTE_USER_DATA_0__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_0__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_1
+#define COMPUTE_USER_DATA_1__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_1__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_2
+#define COMPUTE_USER_DATA_2__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_2__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_3
+#define COMPUTE_USER_DATA_3__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_3__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_4
+#define COMPUTE_USER_DATA_4__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_4__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_5
+#define COMPUTE_USER_DATA_5__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_5__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_6
+#define COMPUTE_USER_DATA_6__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_6__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_7
+#define COMPUTE_USER_DATA_7__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_7__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_8
+#define COMPUTE_USER_DATA_8__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_8__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_9
+#define COMPUTE_USER_DATA_9__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_9__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_10
+#define COMPUTE_USER_DATA_10__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_10__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_11
+#define COMPUTE_USER_DATA_11__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_11__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_12
+#define COMPUTE_USER_DATA_12__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_12__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_13
+#define COMPUTE_USER_DATA_13__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_13__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_14
+#define COMPUTE_USER_DATA_14__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_14__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_15
+#define COMPUTE_USER_DATA_15__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_15__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_16
+#define COMPUTE_USER_DATA_16__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_16__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_17
+#define COMPUTE_USER_DATA_17__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_17__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_18
+#define COMPUTE_USER_DATA_18__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_18__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_19
+#define COMPUTE_USER_DATA_19__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_19__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_20
+#define COMPUTE_USER_DATA_20__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_20__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_21
+#define COMPUTE_USER_DATA_21__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_21__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_22
+#define COMPUTE_USER_DATA_22__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_22__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_23
+#define COMPUTE_USER_DATA_23__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_23__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_24
+#define COMPUTE_USER_DATA_24__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_24__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_25
+#define COMPUTE_USER_DATA_25__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_25__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_26
+#define COMPUTE_USER_DATA_26__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_26__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_27
+#define COMPUTE_USER_DATA_27__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_27__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_28
+#define COMPUTE_USER_DATA_28__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_28__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_29
+#define COMPUTE_USER_DATA_29__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_29__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_30
+#define COMPUTE_USER_DATA_30__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_30__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_31
+#define COMPUTE_USER_DATA_31__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_31__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_RELAUNCH
+#define COMPUTE_RELAUNCH__PAYLOAD__SHIFT                                                                      0x0
+#define COMPUTE_RELAUNCH__IS_EVENT__SHIFT                                                                     0x1e
+#define COMPUTE_RELAUNCH__IS_STATE__SHIFT                                                                     0x1f
+#define COMPUTE_RELAUNCH__PAYLOAD_MASK                                                                        0x3FFFFFFFL
+#define COMPUTE_RELAUNCH__IS_EVENT_MASK                                                                       0x40000000L
+#define COMPUTE_RELAUNCH__IS_STATE_MASK                                                                       0x80000000L
+//COMPUTE_RELAUNCH_STATE_PAYLOAD
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__VGPRS__SHIFT                                                          0x0
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__WGP_TAKEOVER__SHIFT                                                   0x6
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__LDS_SPLIT__SHIFT                                                      0x7
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__LDS_SIZE__SHIFT                                                       0xa
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__TG_PER_CU__SHIFT                                                      0x13
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__SCRATCH_EN__SHIFT                                                     0x17
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__W32_EN__SHIFT                                                         0x18
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__WGP_CENTRIC__SHIFT                                                    0x19
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__NAMED_BAR_CNT__SHIFT                                                  0x1a
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__DYNAMIC_VGPR__SHIFT                                                   0x1d
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__IS_EVENT__SHIFT                                                       0x1e
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__IS_STATE__SHIFT                                                       0x1f
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__VGPRS_MASK                                                            0x0000003FL
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__WGP_TAKEOVER_MASK                                                     0x00000040L
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__LDS_SPLIT_MASK                                                        0x00000380L
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__LDS_SIZE_MASK                                                         0x0007FC00L
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__TG_PER_CU_MASK                                                        0x00780000L
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__SCRATCH_EN_MASK                                                       0x00800000L
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__W32_EN_MASK                                                           0x01000000L
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__WGP_CENTRIC_MASK                                                      0x02000000L
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__NAMED_BAR_CNT_MASK                                                    0x1C000000L
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__DYNAMIC_VGPR_MASK                                                     0x20000000L
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__IS_EVENT_MASK                                                         0x40000000L
+#define COMPUTE_RELAUNCH_STATE_PAYLOAD__IS_STATE_MASK                                                         0x80000000L
+//COMPUTE_WAVE_RESTORE_ADDR_LO
+#define COMPUTE_WAVE_RESTORE_ADDR_LO__ADDR__SHIFT                                                             0x0
+#define COMPUTE_WAVE_RESTORE_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFFL
+//COMPUTE_WAVE_RESTORE_ADDR_HI
+#define COMPUTE_WAVE_RESTORE_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define COMPUTE_WAVE_RESTORE_ADDR_HI__ADDR_MASK                                                               0x01FFFFFFL
+//COMPUTE_RELAUNCH2
+#define COMPUTE_RELAUNCH2__PAYLOAD__SHIFT                                                                     0x0
+#define COMPUTE_RELAUNCH2__IS_EVENT__SHIFT                                                                    0x1e
+#define COMPUTE_RELAUNCH2__IS_STATE__SHIFT                                                                    0x1f
+#define COMPUTE_RELAUNCH2__PAYLOAD_MASK                                                                       0x3FFFFFFFL
+#define COMPUTE_RELAUNCH2__IS_EVENT_MASK                                                                      0x40000000L
+#define COMPUTE_RELAUNCH2__IS_STATE_MASK                                                                      0x80000000L
+//COMPUTE_RELAUNCH2_STATE_PAYLOAD
+#define COMPUTE_RELAUNCH2_STATE_PAYLOAD__USER_ACCUM0__SHIFT                                                   0x0
+#define COMPUTE_RELAUNCH2_STATE_PAYLOAD__USER_ACCUM1__SHIFT                                                   0x7
+#define COMPUTE_RELAUNCH2_STATE_PAYLOAD__USER_ACCUM2__SHIFT                                                   0xe
+#define COMPUTE_RELAUNCH2_STATE_PAYLOAD__USER_ACCUM3__SHIFT                                                   0x15
+#define COMPUTE_RELAUNCH2_STATE_PAYLOAD__BULKY__SHIFT                                                         0x1c
+#define COMPUTE_RELAUNCH2_STATE_PAYLOAD__IS_EVENT__SHIFT                                                      0x1e
+#define COMPUTE_RELAUNCH2_STATE_PAYLOAD__IS_STATE__SHIFT                                                      0x1f
+#define COMPUTE_RELAUNCH2_STATE_PAYLOAD__USER_ACCUM0_MASK                                                     0x0000007FL
+#define COMPUTE_RELAUNCH2_STATE_PAYLOAD__USER_ACCUM1_MASK                                                     0x00003F80L
+#define COMPUTE_RELAUNCH2_STATE_PAYLOAD__USER_ACCUM2_MASK                                                     0x001FC000L
+#define COMPUTE_RELAUNCH2_STATE_PAYLOAD__USER_ACCUM3_MASK                                                     0x0FE00000L
+#define COMPUTE_RELAUNCH2_STATE_PAYLOAD__BULKY_MASK                                                           0x10000000L
+#define COMPUTE_RELAUNCH2_STATE_PAYLOAD__IS_EVENT_MASK                                                        0x40000000L
+#define COMPUTE_RELAUNCH2_STATE_PAYLOAD__IS_STATE_MASK                                                        0x80000000L
+//COMPUTE_DISPATCH_TUNNEL
+#define COMPUTE_DISPATCH_TUNNEL__OFF_DELAY__SHIFT                                                             0x0
+#define COMPUTE_DISPATCH_TUNNEL__OFF_DELAY_MASK                                                               0x00001FFFL
+//COMPUTE_DISPATCH_END
+#define COMPUTE_DISPATCH_END__DATA__SHIFT                                                                     0x0
+#define COMPUTE_DISPATCH_END__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_NOWHERE
+#define COMPUTE_NOWHERE__DATA__SHIFT                                                                          0x0
+#define COMPUTE_NOWHERE__DATA_MASK                                                                            0xFFFFFFFFL
+//SH_RESERVED_REG0
+#define SH_RESERVED_REG0__DATA__SHIFT                                                                         0x0
+#define SH_RESERVED_REG0__DATA_MASK                                                                           0xFFFFFFFFL
+//SH_RESERVED_REG1
+#define SH_RESERVED_REG1__DATA__SHIFT                                                                         0x0
+#define SH_RESERVED_REG1__DATA_MASK                                                                           0xFFFFFFFFL
+//COMPUTE_PREALLOC_CR_DB_BUFFER_SIZE
+#define COMPUTE_PREALLOC_CR_DB_BUFFER_SIZE__CR_DB_OVERRIDE__SHIFT                                             0x0
+#define COMPUTE_PREALLOC_CR_DB_BUFFER_SIZE__CR_DB_OVERRIDE_RESV__SHIFT                                        0x7
+#define COMPUTE_PREALLOC_CR_DB_BUFFER_SIZE__CR_DB_OVERRIDE_EN__SHIFT                                          0x1f
+#define COMPUTE_PREALLOC_CR_DB_BUFFER_SIZE__CR_DB_OVERRIDE_MASK                                               0x0000007FL
+#define COMPUTE_PREALLOC_CR_DB_BUFFER_SIZE__CR_DB_OVERRIDE_RESV_MASK                                          0x7FFFFF80L
+#define COMPUTE_PREALLOC_CR_DB_BUFFER_SIZE__CR_DB_OVERRIDE_EN_MASK                                            0x80000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pfonly_gccacdec
+//GC_CAC_CTRL_1
+#define GC_CAC_CTRL_1__CAC_WINDOW__SHIFT                                                                      0x0
+#define GC_CAC_CTRL_1__TDP_WINDOW__SHIFT                                                                      0x8
+#define GC_CAC_CTRL_1__CAC_WINDOW_MASK                                                                        0x000000FFL
+#define GC_CAC_CTRL_1__TDP_WINDOW_MASK                                                                        0xFFFFFF00L
+//GC_CAC_CTRL_2
+#define GC_CAC_CTRL_2__CAC_ENABLE__SHIFT                                                                      0x0
+#define GC_CAC_CTRL_2__GC_LCAC_ENABLE__SHIFT                                                                  0x1
+#define GC_CAC_CTRL_2__GC_CAC_INDEX_AUTO_INCR_EN__SHIFT                                                       0x2
+#define GC_CAC_CTRL_2__TOGGLE_EN__SHIFT                                                                       0x3
+#define GC_CAC_CTRL_2__INTR_EN__SHIFT                                                                         0x4
+#define GC_CAC_CTRL_2__CAC_COUNTER_SNAP_SEL__SHIFT                                                            0x5
+#define GC_CAC_CTRL_2__SE_AGGR_ACC_EN__SHIFT                                                                  0x6
+#define GC_CAC_CTRL_2__GC_AGGR_ACC_EN__SHIFT                                                                  0xf
+#define GC_CAC_CTRL_2__GC_LCAC_OVR_EN__SHIFT                                                                  0x10
+#define GC_CAC_CTRL_2__CAC_SOFT_CTRL_ENABLE__SHIFT                                                            0x11
+#define GC_CAC_CTRL_2__CAC_INTR_MAX_HYSTERESIS__SHIFT                                                         0x12
+#define GC_CAC_CTRL_2__CAC_TDP_AGGR_MODE__SHIFT                                                               0x18
+#define GC_CAC_CTRL_2__LOCAL_REP_FGCG_OVERRIDE__SHIFT                                                         0x19
+#define GC_CAC_CTRL_2__CAC_ENABLE_MASK                                                                        0x00000001L
+#define GC_CAC_CTRL_2__GC_LCAC_ENABLE_MASK                                                                    0x00000002L
+#define GC_CAC_CTRL_2__GC_CAC_INDEX_AUTO_INCR_EN_MASK                                                         0x00000004L
+#define GC_CAC_CTRL_2__TOGGLE_EN_MASK                                                                         0x00000008L
+#define GC_CAC_CTRL_2__INTR_EN_MASK                                                                           0x00000010L
+#define GC_CAC_CTRL_2__CAC_COUNTER_SNAP_SEL_MASK                                                              0x00000020L
+#define GC_CAC_CTRL_2__SE_AGGR_ACC_EN_MASK                                                                    0x00007FC0L
+#define GC_CAC_CTRL_2__GC_AGGR_ACC_EN_MASK                                                                    0x00008000L
+#define GC_CAC_CTRL_2__GC_LCAC_OVR_EN_MASK                                                                    0x00010000L
+#define GC_CAC_CTRL_2__CAC_SOFT_CTRL_ENABLE_MASK                                                              0x00020000L
+#define GC_CAC_CTRL_2__CAC_INTR_MAX_HYSTERESIS_MASK                                                           0x00FC0000L
+#define GC_CAC_CTRL_2__CAC_TDP_AGGR_MODE_MASK                                                                 0x01000000L
+#define GC_CAC_CTRL_2__LOCAL_REP_FGCG_OVERRIDE_MASK                                                           0x02000000L
+//GC_CAC_AGGR_LOWER
+#define GC_CAC_AGGR_LOWER__GC_AGGR_31_0__SHIFT                                                                0x0
+#define GC_CAC_AGGR_LOWER__GC_AGGR_31_0_MASK                                                                  0xFFFFFFFFL
+//GC_CAC_AGGR_UPPER
+#define GC_CAC_AGGR_UPPER__GC_AGGR_63_32__SHIFT                                                               0x0
+#define GC_CAC_AGGR_UPPER__GC_AGGR_63_32_MASK                                                                 0xFFFFFFFFL
+//SE0_CAC_AGGR_LOWER
+#define SE0_CAC_AGGR_LOWER__SE0_AGGR_31_0__SHIFT                                                              0x0
+#define SE0_CAC_AGGR_LOWER__SE0_AGGR_31_0_MASK                                                                0xFFFFFFFFL
+//SE0_CAC_AGGR_UPPER
+#define SE0_CAC_AGGR_UPPER__SE0_AGGR_63_32__SHIFT                                                             0x0
+#define SE0_CAC_AGGR_UPPER__SE0_AGGR_63_32_MASK                                                               0xFFFFFFFFL
+//SE1_CAC_AGGR_LOWER
+#define SE1_CAC_AGGR_LOWER__SE1_AGGR_31_0__SHIFT                                                              0x0
+#define SE1_CAC_AGGR_LOWER__SE1_AGGR_31_0_MASK                                                                0xFFFFFFFFL
+//SE1_CAC_AGGR_UPPER
+#define SE1_CAC_AGGR_UPPER__SE1_AGGR_63_32__SHIFT                                                             0x0
+#define SE1_CAC_AGGR_UPPER__SE1_AGGR_63_32_MASK                                                               0xFFFFFFFFL
+//GC_CAC_AGGR_GFXCLK_CYCLE
+#define GC_CAC_AGGR_GFXCLK_CYCLE__GC_AGGR_GFXCLK_CYCLE__SHIFT                                                 0x0
+#define GC_CAC_AGGR_GFXCLK_CYCLE__GC_AGGR_GFXCLK_CYCLE_MASK                                                   0xFFFFFFFFL
+//SE0_CAC_AGGR_GFXCLK_CYCLE
+#define SE0_CAC_AGGR_GFXCLK_CYCLE__SE0_AGGR_GFXCLK_CYCLE__SHIFT                                               0x0
+#define SE0_CAC_AGGR_GFXCLK_CYCLE__SE0_AGGR_GFXCLK_CYCLE_MASK                                                 0xFFFFFFFFL
+//SE1_CAC_AGGR_GFXCLK_CYCLE
+#define SE1_CAC_AGGR_GFXCLK_CYCLE__SE1_AGGR_GFXCLK_CYCLE__SHIFT                                               0x0
+#define SE1_CAC_AGGR_GFXCLK_CYCLE__SE1_AGGR_GFXCLK_CYCLE_MASK                                                 0xFFFFFFFFL
+//GC_CAC_OVR_VAL_LOWER
+#define GC_CAC_OVR_VAL_LOWER__GC_LCAC_OVR_VAL_LOWER__SHIFT                                                    0x0
+#define GC_CAC_OVR_VAL_LOWER__GC_LCAC_OVR_VAL_LOWER_MASK                                                      0xFFFFFFFFL
+//GC_CAC_OVR_VAL_UPPER
+#define GC_CAC_OVR_VAL_UPPER__GC_LCAC_OVR_VAL_UPPER__SHIFT                                                    0x0
+#define GC_CAC_OVR_VAL_UPPER__GC_LCAC_OVR_VAL_UPPER_MASK                                                      0xFFFFFFFFL
+//GC_EDC_CTRL
+#define GC_EDC_CTRL__EDC_EN__SHIFT                                                                            0x0
+#define GC_EDC_CTRL__EDC_SW_RST__SHIFT                                                                        0x1
+#define GC_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT                                                               0x2
+#define GC_EDC_CTRL__EDC_FORCE_STALL__SHIFT                                                                   0x3
+#define GC_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT                                                       0x4
+#define GC_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT                                                          0xa
+#define GC_EDC_CTRL__EDC_THROTTLE_PATTERN_BIT_NUMS__SHIFT                                                     0xb
+#define GC_EDC_CTRL__EDC_LEVEL_SEL__SHIFT                                                                     0xf
+#define GC_EDC_CTRL__EDC_ALGORITHM_MODE__SHIFT                                                                0x10
+#define GC_EDC_CTRL__EDC_AVGDIV__SHIFT                                                                        0x11
+#define GC_EDC_CTRL__PSM_THROTTLE_SRC_SEL__SHIFT                                                              0x15
+#define GC_EDC_CTRL__THROTTLE_SRC0_MASK__SHIFT                                                                0x18
+#define GC_EDC_CTRL__THROTTLE_SRC1_MASK__SHIFT                                                                0x19
+#define GC_EDC_CTRL__THROTTLE_SRC2_MASK__SHIFT                                                                0x1a
+#define GC_EDC_CTRL__THROTTLE_SRC3_MASK__SHIFT                                                                0x1b
+#define GC_EDC_CTRL__EDC_CREDIT_SHIFT_BIT_NUMS__SHIFT                                                         0x1c
+#define GC_EDC_CTRL__EDC_EN_MASK                                                                              0x00000001L
+#define GC_EDC_CTRL__EDC_SW_RST_MASK                                                                          0x00000002L
+#define GC_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK                                                                 0x00000004L
+#define GC_EDC_CTRL__EDC_FORCE_STALL_MASK                                                                     0x00000008L
+#define GC_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK                                                         0x000003F0L
+#define GC_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK                                                            0x00000400L
+#define GC_EDC_CTRL__EDC_THROTTLE_PATTERN_BIT_NUMS_MASK                                                       0x00007800L
+#define GC_EDC_CTRL__EDC_LEVEL_SEL_MASK                                                                       0x00008000L
+#define GC_EDC_CTRL__EDC_ALGORITHM_MODE_MASK                                                                  0x00010000L
+#define GC_EDC_CTRL__EDC_AVGDIV_MASK                                                                          0x001E0000L
+#define GC_EDC_CTRL__PSM_THROTTLE_SRC_SEL_MASK                                                                0x00E00000L
+#define GC_EDC_CTRL__THROTTLE_SRC0_MASK_MASK                                                                  0x01000000L
+#define GC_EDC_CTRL__THROTTLE_SRC1_MASK_MASK                                                                  0x02000000L
+#define GC_EDC_CTRL__THROTTLE_SRC2_MASK_MASK                                                                  0x04000000L
+#define GC_EDC_CTRL__THROTTLE_SRC3_MASK_MASK                                                                  0x08000000L
+#define GC_EDC_CTRL__EDC_CREDIT_SHIFT_BIT_NUMS_MASK                                                           0xF0000000L
+//GC_EDC_STRETCH_CTRL
+#define GC_EDC_STRETCH_CTRL__EDC_STRETCH_EN__SHIFT                                                            0x0
+#define GC_EDC_STRETCH_CTRL__EDC_STRETCH_DELAY__SHIFT                                                         0x1
+#define GC_EDC_STRETCH_CTRL__EDC_UNSTRETCH_DELAY__SHIFT                                                       0xa
+#define GC_EDC_STRETCH_CTRL__ROLLING_POWER_MODE__SHIFT                                                        0x13
+#define GC_EDC_STRETCH_CTRL__EDC_STRETCH_EN_MASK                                                              0x00000001L
+#define GC_EDC_STRETCH_CTRL__EDC_STRETCH_DELAY_MASK                                                           0x000003FEL
+#define GC_EDC_STRETCH_CTRL__EDC_UNSTRETCH_DELAY_MASK                                                         0x0007FC00L
+#define GC_EDC_STRETCH_CTRL__ROLLING_POWER_MODE_MASK                                                          0x00080000L
+//GC_EDC_THRESHOLD_LO
+#define GC_EDC_THRESHOLD_LO__EDC_THRESHOLD_LO__SHIFT                                                          0x0
+#define GC_EDC_THRESHOLD_LO__EDC_THRESHOLD_LO_MASK                                                            0xFFFFFFFFL
+//GC_EDC_THRESHOLD_HI
+#define GC_EDC_THRESHOLD_HI__EDC_THRESHOLD_HI__SHIFT                                                          0x0
+#define GC_EDC_THRESHOLD_HI__EDC_THRESHOLD_HI_MASK                                                            0xFFFFFFFFL
+//GC_EDC_STRETCH_THRESHOLD_LO
+#define GC_EDC_STRETCH_THRESHOLD_LO__EDC_STRETCH_THRESHOLD_LO__SHIFT                                          0x0
+#define GC_EDC_STRETCH_THRESHOLD_LO__EDC_STRETCH_THRESHOLD_LO_MASK                                            0xFFFFFFFFL
+//GC_EDC_STRETCH_THRESHOLD_HI
+#define GC_EDC_STRETCH_THRESHOLD_HI__EDC_STRETCH_THRESHOLD_HI__SHIFT                                          0x0
+#define GC_EDC_STRETCH_THRESHOLD_HI__EDC_STRETCH_THRESHOLD_HI_MASK                                            0xFFFFFFFFL
+//EDC_HYSTERESIS_CNTL
+#define EDC_HYSTERESIS_CNTL__MAX_HYSTERESIS__SHIFT                                                            0x0
+#define EDC_HYSTERESIS_CNTL__EDC_AGGR_TIMER__SHIFT                                                            0x8
+#define EDC_HYSTERESIS_CNTL__PATTERN_EXTEND_EN__SHIFT                                                         0x10
+#define EDC_HYSTERESIS_CNTL__PATTERN_EXTEND_MODE__SHIFT                                                       0x11
+#define EDC_HYSTERESIS_CNTL__EDC_AGGR_MODE__SHIFT                                                             0x14
+#define EDC_HYSTERESIS_CNTL__MAX_HYSTERESIS_MASK                                                              0x000000FFL
+#define EDC_HYSTERESIS_CNTL__EDC_AGGR_TIMER_MASK                                                              0x0000FF00L
+#define EDC_HYSTERESIS_CNTL__PATTERN_EXTEND_EN_MASK                                                           0x00010000L
+#define EDC_HYSTERESIS_CNTL__PATTERN_EXTEND_MODE_MASK                                                         0x000E0000L
+#define EDC_HYSTERESIS_CNTL__EDC_AGGR_MODE_MASK                                                               0x00100000L
+//GC_THROTTLE_CTRL
+#define GC_THROTTLE_CTRL__THROTTLE_CTRL_SW_RST__SHIFT                                                         0x0
+#define GC_THROTTLE_CTRL__GC_EDC_STALL_EN__SHIFT                                                              0x1
+#define GC_THROTTLE_CTRL__PWRBRK_STALL_EN__SHIFT                                                              0x2
+#define GC_THROTTLE_CTRL__PWRBRK_POLARITY_CNTL__SHIFT                                                         0x3
+#define GC_THROTTLE_CTRL__PCC_STALL_EN__SHIFT                                                                 0x4
+#define GC_THROTTLE_CTRL__PATTERN_MODE__SHIFT                                                                 0x5
+#define GC_THROTTLE_CTRL__GC_EDC_ONLY_MODE__SHIFT                                                             0x6
+#define GC_THROTTLE_CTRL__GC_EDC_OVERRIDE__SHIFT                                                              0x7
+#define GC_THROTTLE_CTRL__PCC_OVERRIDE__SHIFT                                                                 0x8
+#define GC_THROTTLE_CTRL__PWRBRK_OVERRIDE__SHIFT                                                              0x9
+#define GC_THROTTLE_CTRL__GC_EDC_PERF_COUNTER_EN__SHIFT                                                       0xa
+#define GC_THROTTLE_CTRL__PCC_PERF_COUNTER_EN__SHIFT                                                          0xb
+#define GC_THROTTLE_CTRL__PWRBRK_PERF_COUNTER_EN__SHIFT                                                       0xc
+#define GC_THROTTLE_CTRL__RELEASE_STEP_INTERVAL__SHIFT                                                        0xd
+#define GC_THROTTLE_CTRL__FIXED_PATTERN_PERF_COUNTER_EN__SHIFT                                                0x17
+#define GC_THROTTLE_CTRL__FIXED_PATTERN_LOG_INDEX__SHIFT                                                      0x18
+#define GC_THROTTLE_CTRL__LUT_HW_UPDATE__SHIFT                                                                0x1d
+#define GC_THROTTLE_CTRL__THROTTLE_CTRL_CLK_EN_OVERRIDE__SHIFT                                                0x1e
+#define GC_THROTTLE_CTRL__PCC_POLARITY_CNTL__SHIFT                                                            0x1f
+#define GC_THROTTLE_CTRL__THROTTLE_CTRL_SW_RST_MASK                                                           0x00000001L
+#define GC_THROTTLE_CTRL__GC_EDC_STALL_EN_MASK                                                                0x00000002L
+#define GC_THROTTLE_CTRL__PWRBRK_STALL_EN_MASK                                                                0x00000004L
+#define GC_THROTTLE_CTRL__PWRBRK_POLARITY_CNTL_MASK                                                           0x00000008L
+#define GC_THROTTLE_CTRL__PCC_STALL_EN_MASK                                                                   0x00000010L
+#define GC_THROTTLE_CTRL__PATTERN_MODE_MASK                                                                   0x00000020L
+#define GC_THROTTLE_CTRL__GC_EDC_ONLY_MODE_MASK                                                               0x00000040L
+#define GC_THROTTLE_CTRL__GC_EDC_OVERRIDE_MASK                                                                0x00000080L
+#define GC_THROTTLE_CTRL__PCC_OVERRIDE_MASK                                                                   0x00000100L
+#define GC_THROTTLE_CTRL__PWRBRK_OVERRIDE_MASK                                                                0x00000200L
+#define GC_THROTTLE_CTRL__GC_EDC_PERF_COUNTER_EN_MASK                                                         0x00000400L
+#define GC_THROTTLE_CTRL__PCC_PERF_COUNTER_EN_MASK                                                            0x00000800L
+#define GC_THROTTLE_CTRL__PWRBRK_PERF_COUNTER_EN_MASK                                                         0x00001000L
+#define GC_THROTTLE_CTRL__RELEASE_STEP_INTERVAL_MASK                                                          0x007FE000L
+#define GC_THROTTLE_CTRL__FIXED_PATTERN_PERF_COUNTER_EN_MASK                                                  0x00800000L
+#define GC_THROTTLE_CTRL__FIXED_PATTERN_LOG_INDEX_MASK                                                        0x1F000000L
+#define GC_THROTTLE_CTRL__LUT_HW_UPDATE_MASK                                                                  0x20000000L
+#define GC_THROTTLE_CTRL__THROTTLE_CTRL_CLK_EN_OVERRIDE_MASK                                                  0x40000000L
+#define GC_THROTTLE_CTRL__PCC_POLARITY_CNTL_MASK                                                              0x80000000L
+//GC_THROTTLE_CTRL1
+#define GC_THROTTLE_CTRL1__PCC_FP_PROGRAM_STEP_EN__SHIFT                                                      0x0
+#define GC_THROTTLE_CTRL1__PCC_PROGRAM_MIN_STEP__SHIFT                                                        0x1
+#define GC_THROTTLE_CTRL1__PCC_PROGRAM_MAX_STEP__SHIFT                                                        0x5
+#define GC_THROTTLE_CTRL1__PCC_PROGRAM_UPWARDS_STEP_SIZE__SHIFT                                               0xa
+#define GC_THROTTLE_CTRL1__PWRBRK_FP_PROGRAM_STEP_EN__SHIFT                                                   0xd
+#define GC_THROTTLE_CTRL1__PWRBRK_PROGRAM_MIN_STEP__SHIFT                                                     0xe
+#define GC_THROTTLE_CTRL1__PWRBRK_PROGRAM_MAX_STEP__SHIFT                                                     0x12
+#define GC_THROTTLE_CTRL1__PWRBRK_PROGRAM_UPWARDS_STEP_SIZE__SHIFT                                            0x17
+#define GC_THROTTLE_CTRL1__FIXED_PATTERN_SELECT__SHIFT                                                        0x1a
+#define GC_THROTTLE_CTRL1__GC_EDC_STRETCH_PERF_COUNTER_EN__SHIFT                                              0x1e
+#define GC_THROTTLE_CTRL1__GC_EDC_UNSTRETCH_PERF_COUNTER_EN__SHIFT                                            0x1f
+#define GC_THROTTLE_CTRL1__PCC_FP_PROGRAM_STEP_EN_MASK                                                        0x00000001L
+#define GC_THROTTLE_CTRL1__PCC_PROGRAM_MIN_STEP_MASK                                                          0x0000001EL
+#define GC_THROTTLE_CTRL1__PCC_PROGRAM_MAX_STEP_MASK                                                          0x000003E0L
+#define GC_THROTTLE_CTRL1__PCC_PROGRAM_UPWARDS_STEP_SIZE_MASK                                                 0x00001C00L
+#define GC_THROTTLE_CTRL1__PWRBRK_FP_PROGRAM_STEP_EN_MASK                                                     0x00002000L
+#define GC_THROTTLE_CTRL1__PWRBRK_PROGRAM_MIN_STEP_MASK                                                       0x0003C000L
+#define GC_THROTTLE_CTRL1__PWRBRK_PROGRAM_MAX_STEP_MASK                                                       0x007C0000L
+#define GC_THROTTLE_CTRL1__PWRBRK_PROGRAM_UPWARDS_STEP_SIZE_MASK                                              0x03800000L
+#define GC_THROTTLE_CTRL1__FIXED_PATTERN_SELECT_MASK                                                          0x0C000000L
+#define GC_THROTTLE_CTRL1__GC_EDC_STRETCH_PERF_COUNTER_EN_MASK                                                0x40000000L
+#define GC_THROTTLE_CTRL1__GC_EDC_UNSTRETCH_PERF_COUNTER_EN_MASK                                              0x80000000L
+//GC_THROTTLE_CTRL2
+#define GC_THROTTLE_CTRL2__EDC_FP_PROGRAM_STEP_EN__SHIFT                                                      0x0
+#define GC_THROTTLE_CTRL2__EDC_PROGRAM_MIN_STEP__SHIFT                                                        0x1
+#define GC_THROTTLE_CTRL2__EDC_PROGRAM_MAX_STEP__SHIFT                                                        0x5
+#define GC_THROTTLE_CTRL2__EDC_PROGRAM_UPWARDS_STEP_SIZE__SHIFT                                               0xa
+#define GC_THROTTLE_CTRL2__PATTERN_COUNTER_NO_RESTART__SHIFT                                                  0xd
+#define GC_THROTTLE_CTRL2__EDC_THROTTLE_LEVEL_COUNTER_EN__SHIFT                                               0xe
+#define GC_THROTTLE_CTRL2__EDC_POWER_DELTA_ALGO2_EN__SHIFT                                                    0xf
+#define GC_THROTTLE_CTRL2__EDC_FP_PROGRAM_STEP_EN_MASK                                                        0x00000001L
+#define GC_THROTTLE_CTRL2__EDC_PROGRAM_MIN_STEP_MASK                                                          0x0000001EL
+#define GC_THROTTLE_CTRL2__EDC_PROGRAM_MAX_STEP_MASK                                                          0x000003E0L
+#define GC_THROTTLE_CTRL2__EDC_PROGRAM_UPWARDS_STEP_SIZE_MASK                                                 0x00001C00L
+#define GC_THROTTLE_CTRL2__PATTERN_COUNTER_NO_RESTART_MASK                                                    0x00002000L
+#define GC_THROTTLE_CTRL2__EDC_THROTTLE_LEVEL_COUNTER_EN_MASK                                                 0x00004000L
+#define GC_THROTTLE_CTRL2__EDC_POWER_DELTA_ALGO2_EN_MASK                                                      0x00008000L
+//EDC_STALL_PATTERN_CTRL
+#define EDC_STALL_PATTERN_CTRL__EDC_STEP_INTERVAL__SHIFT                                                      0x0
+#define EDC_STALL_PATTERN_CTRL__EDC_BEGIN_STEP__SHIFT                                                         0xa
+#define EDC_STALL_PATTERN_CTRL__EDC_END_STEP__SHIFT                                                           0xf
+#define EDC_STALL_PATTERN_CTRL__EDC_DITHER_MODE__SHIFT                                                        0x14
+#define EDC_STALL_PATTERN_CTRL__EDC_STEP_INTERVAL_MASK                                                        0x000003FFL
+#define EDC_STALL_PATTERN_CTRL__EDC_BEGIN_STEP_MASK                                                           0x00007C00L
+#define EDC_STALL_PATTERN_CTRL__EDC_END_STEP_MASK                                                             0x000F8000L
+#define EDC_STALL_PATTERN_CTRL__EDC_DITHER_MODE_MASK                                                          0x00100000L
+//PCC_STALL_PATTERN_CTRL
+#define PCC_STALL_PATTERN_CTRL__PCC_STEP_INTERVAL__SHIFT                                                      0x0
+#define PCC_STALL_PATTERN_CTRL__PCC_BEGIN_STEP__SHIFT                                                         0xa
+#define PCC_STALL_PATTERN_CTRL__PCC_END_STEP__SHIFT                                                           0xf
+#define PCC_STALL_PATTERN_CTRL__PCC_THROTTLE_PATTERN_BIT_NUMS__SHIFT                                          0x14
+#define PCC_STALL_PATTERN_CTRL__PCC_INST_THROT_INCR__SHIFT                                                    0x18
+#define PCC_STALL_PATTERN_CTRL__PCC_INST_THROT_DECR__SHIFT                                                    0x19
+#define PCC_STALL_PATTERN_CTRL__PCC_DITHER_MODE__SHIFT                                                        0x1a
+#define PCC_STALL_PATTERN_CTRL__PCC_STEP_INTERVAL_MASK                                                        0x000003FFL
+#define PCC_STALL_PATTERN_CTRL__PCC_BEGIN_STEP_MASK                                                           0x00007C00L
+#define PCC_STALL_PATTERN_CTRL__PCC_END_STEP_MASK                                                             0x000F8000L
+#define PCC_STALL_PATTERN_CTRL__PCC_THROTTLE_PATTERN_BIT_NUMS_MASK                                            0x00F00000L
+#define PCC_STALL_PATTERN_CTRL__PCC_INST_THROT_INCR_MASK                                                      0x01000000L
+#define PCC_STALL_PATTERN_CTRL__PCC_INST_THROT_DECR_MASK                                                      0x02000000L
+#define PCC_STALL_PATTERN_CTRL__PCC_DITHER_MODE_MASK                                                          0x04000000L
+//PWRBRK_STALL_PATTERN_CTRL
+#define PWRBRK_STALL_PATTERN_CTRL__PWRBRK_STEP_INTERVAL__SHIFT                                                0x0
+#define PWRBRK_STALL_PATTERN_CTRL__PWRBRK_BEGIN_STEP__SHIFT                                                   0xa
+#define PWRBRK_STALL_PATTERN_CTRL__PWRBRK_END_STEP__SHIFT                                                     0xf
+#define PWRBRK_STALL_PATTERN_CTRL__PWRBRK_THROTTLE_PATTERN_BIT_NUMS__SHIFT                                    0x14
+#define PWRBRK_STALL_PATTERN_CTRL__PWRBRK_STEP_INTERVAL_MASK                                                  0x000003FFL
+#define PWRBRK_STALL_PATTERN_CTRL__PWRBRK_BEGIN_STEP_MASK                                                     0x00007C00L
+#define PWRBRK_STALL_PATTERN_CTRL__PWRBRK_END_STEP_MASK                                                       0x000F8000L
+#define PWRBRK_STALL_PATTERN_CTRL__PWRBRK_THROTTLE_PATTERN_BIT_NUMS_MASK                                      0x00F00000L
+//EDC_STALL_PATTERN_1_2
+#define EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1__SHIFT                                                     0x0
+#define EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2__SHIFT                                                     0x10
+#define EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1_MASK                                                       0x00007FFFL
+#define EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2_MASK                                                       0x7FFF0000L
+//EDC_STALL_PATTERN_3_4
+#define EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3__SHIFT                                                     0x0
+#define EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4__SHIFT                                                     0x10
+#define EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3_MASK                                                       0x00007FFFL
+#define EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4_MASK                                                       0x7FFF0000L
+//EDC_STALL_PATTERN_5_6
+#define EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5__SHIFT                                                     0x0
+#define EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6__SHIFT                                                     0x10
+#define EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5_MASK                                                       0x00007FFFL
+#define EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6_MASK                                                       0x7FFF0000L
+//EDC_STALL_PATTERN_7
+#define EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7__SHIFT                                                       0x0
+#define EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_0__SHIFT                                                       0x10
+#define EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7_MASK                                                         0x00007FFFL
+#define EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_0_MASK                                                         0x7FFF0000L
+//PCC_STALL_PATTERN_1_2
+#define PCC_STALL_PATTERN_1_2__PCC_STALL_PATTERN_1__SHIFT                                                     0x0
+#define PCC_STALL_PATTERN_1_2__PCC_STALL_PATTERN_2__SHIFT                                                     0x10
+#define PCC_STALL_PATTERN_1_2__PCC_STALL_PATTERN_1_MASK                                                       0x00007FFFL
+#define PCC_STALL_PATTERN_1_2__PCC_STALL_PATTERN_2_MASK                                                       0x7FFF0000L
+//PCC_STALL_PATTERN_3_4
+#define PCC_STALL_PATTERN_3_4__PCC_STALL_PATTERN_3__SHIFT                                                     0x0
+#define PCC_STALL_PATTERN_3_4__PCC_STALL_PATTERN_4__SHIFT                                                     0x10
+#define PCC_STALL_PATTERN_3_4__PCC_STALL_PATTERN_3_MASK                                                       0x00007FFFL
+#define PCC_STALL_PATTERN_3_4__PCC_STALL_PATTERN_4_MASK                                                       0x7FFF0000L
+//PCC_STALL_PATTERN_5_6
+#define PCC_STALL_PATTERN_5_6__PCC_STALL_PATTERN_5__SHIFT                                                     0x0
+#define PCC_STALL_PATTERN_5_6__PCC_STALL_PATTERN_6__SHIFT                                                     0x10
+#define PCC_STALL_PATTERN_5_6__PCC_STALL_PATTERN_5_MASK                                                       0x00007FFFL
+#define PCC_STALL_PATTERN_5_6__PCC_STALL_PATTERN_6_MASK                                                       0x7FFF0000L
+//PCC_STALL_PATTERN_7
+#define PCC_STALL_PATTERN_7__PCC_STALL_PATTERN_7__SHIFT                                                       0x0
+#define PCC_STALL_PATTERN_7__PCC_STALL_PATTERN_7_MASK                                                         0x00007FFFL
+//PWRBRK_STALL_PATTERN_1_2
+#define PWRBRK_STALL_PATTERN_1_2__PWRBRK_STALL_PATTERN_1__SHIFT                                               0x0
+#define PWRBRK_STALL_PATTERN_1_2__PWRBRK_STALL_PATTERN_2__SHIFT                                               0x10
+#define PWRBRK_STALL_PATTERN_1_2__PWRBRK_STALL_PATTERN_1_MASK                                                 0x00007FFFL
+#define PWRBRK_STALL_PATTERN_1_2__PWRBRK_STALL_PATTERN_2_MASK                                                 0x7FFF0000L
+//PWRBRK_STALL_PATTERN_3_4
+#define PWRBRK_STALL_PATTERN_3_4__PWRBRK_STALL_PATTERN_3__SHIFT                                               0x0
+#define PWRBRK_STALL_PATTERN_3_4__PWRBRK_STALL_PATTERN_4__SHIFT                                               0x10
+#define PWRBRK_STALL_PATTERN_3_4__PWRBRK_STALL_PATTERN_3_MASK                                                 0x00007FFFL
+#define PWRBRK_STALL_PATTERN_3_4__PWRBRK_STALL_PATTERN_4_MASK                                                 0x7FFF0000L
+//PWRBRK_STALL_PATTERN_5_6
+#define PWRBRK_STALL_PATTERN_5_6__PWRBRK_STALL_PATTERN_5__SHIFT                                               0x0
+#define PWRBRK_STALL_PATTERN_5_6__PWRBRK_STALL_PATTERN_6__SHIFT                                               0x10
+#define PWRBRK_STALL_PATTERN_5_6__PWRBRK_STALL_PATTERN_5_MASK                                                 0x00007FFFL
+#define PWRBRK_STALL_PATTERN_5_6__PWRBRK_STALL_PATTERN_6_MASK                                                 0x7FFF0000L
+//PWRBRK_STALL_PATTERN_7
+#define PWRBRK_STALL_PATTERN_7__PWRBRK_STALL_PATTERN_7__SHIFT                                                 0x0
+#define PWRBRK_STALL_PATTERN_7__PWRBRK_STALL_PATTERN_7_MASK                                                   0x00007FFFL
+//DIDT_STALL_PATTERN_CTRL
+#define DIDT_STALL_PATTERN_CTRL__DIDT_DROOP_CTRL_EN__SHIFT                                                    0x0
+#define DIDT_STALL_PATTERN_CTRL__DIDT_DROOP_SW_RST__SHIFT                                                     0x1
+#define DIDT_STALL_PATTERN_CTRL__DIDT_DROOP_CLK_EN_OVERRIDE__SHIFT                                            0x2
+#define DIDT_STALL_PATTERN_CTRL__DIDT_STALL_PATTERN_BIT_NUMS__SHIFT                                           0x3
+#define DIDT_STALL_PATTERN_CTRL__DIDT_PATTERN_EXTEND_EN__SHIFT                                                0x7
+#define DIDT_STALL_PATTERN_CTRL__DIDT_PATTERN_EXTEND_MODE__SHIFT                                              0x8
+#define DIDT_STALL_PATTERN_CTRL__DIDT_DROOP_STRETCH_EN__SHIFT                                                 0xb
+#define DIDT_STALL_PATTERN_CTRL__DIDT_MAX_HYSTERESIS__SHIFT                                                   0xc
+#define DIDT_STALL_PATTERN_CTRL__DIDT_PERF_COUNTER_EN__SHIFT                                                  0x14
+#define DIDT_STALL_PATTERN_CTRL__PSM_DIDT_THROTTLE_SRC_SEL__SHIFT                                             0x15
+#define DIDT_STALL_PATTERN_CTRL__DIDT_THROTTLE_SRC0_MASK__SHIFT                                               0x18
+#define DIDT_STALL_PATTERN_CTRL__DIDT_THROTTLE_SRC1_MASK__SHIFT                                               0x19
+#define DIDT_STALL_PATTERN_CTRL__DIDT_THROTTLE_SRC2_MASK__SHIFT                                               0x1a
+#define DIDT_STALL_PATTERN_CTRL__DIDT_THROTTLE_SRC3_MASK__SHIFT                                               0x1b
+#define DIDT_STALL_PATTERN_CTRL__DIDT_DROOP_CTRL_EN_MASK                                                      0x00000001L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_DROOP_SW_RST_MASK                                                       0x00000002L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_DROOP_CLK_EN_OVERRIDE_MASK                                              0x00000004L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_STALL_PATTERN_BIT_NUMS_MASK                                             0x00000078L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_PATTERN_EXTEND_EN_MASK                                                  0x00000080L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_PATTERN_EXTEND_MODE_MASK                                                0x00000700L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_DROOP_STRETCH_EN_MASK                                                   0x00000800L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_MAX_HYSTERESIS_MASK                                                     0x000FF000L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_PERF_COUNTER_EN_MASK                                                    0x00100000L
+#define DIDT_STALL_PATTERN_CTRL__PSM_DIDT_THROTTLE_SRC_SEL_MASK                                               0x00E00000L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_THROTTLE_SRC0_MASK_MASK                                                 0x01000000L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_THROTTLE_SRC1_MASK_MASK                                                 0x02000000L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_THROTTLE_SRC2_MASK_MASK                                                 0x04000000L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_THROTTLE_SRC3_MASK_MASK                                                 0x08000000L
+//DIDT_STALL_PATTERN_1_2
+#define DIDT_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1__SHIFT                                                   0x0
+#define DIDT_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2__SHIFT                                                   0x10
+#define DIDT_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1_MASK                                                     0x00007FFFL
+#define DIDT_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2_MASK                                                     0x7FFF0000L
+//DIDT_STALL_PATTERN_3_4
+#define DIDT_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3__SHIFT                                                   0x0
+#define DIDT_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4__SHIFT                                                   0x10
+#define DIDT_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3_MASK                                                     0x00007FFFL
+#define DIDT_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4_MASK                                                     0x7FFF0000L
+//DIDT_STALL_PATTERN_5_6
+#define DIDT_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5__SHIFT                                                   0x0
+#define DIDT_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6__SHIFT                                                   0x10
+#define DIDT_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5_MASK                                                     0x00007FFFL
+#define DIDT_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6_MASK                                                     0x7FFF0000L
+//DIDT_STALL_PATTERN_7
+#define DIDT_STALL_PATTERN_7__DIDT_STALL_PATTERN_7__SHIFT                                                     0x0
+#define DIDT_STALL_PATTERN_7__DIDT_STALL_PATTERN_7_MASK                                                       0x00007FFFL
+//PCC_PWRBRK_HYSTERESIS_CTRL
+#define PCC_PWRBRK_HYSTERESIS_CTRL__PCC_MAX_HYSTERESIS__SHIFT                                                 0x0
+#define PCC_PWRBRK_HYSTERESIS_CTRL__PWRBRK_MAX_HYSTERESIS__SHIFT                                              0x8
+#define PCC_PWRBRK_HYSTERESIS_CTRL__PCC_MAX_HYSTERESIS_MASK                                                   0x000000FFL
+#define PCC_PWRBRK_HYSTERESIS_CTRL__PWRBRK_MAX_HYSTERESIS_MASK                                                0x0000FF00L
+//EDC_STRETCH_PERF_COUNTER
+#define EDC_STRETCH_PERF_COUNTER__STRETCH_PERF_COUNTER__SHIFT                                                 0x0
+#define EDC_STRETCH_PERF_COUNTER__STRETCH_PERF_COUNTER_MASK                                                   0xFFFFFFFFL
+//EDC_UNSTRETCH_PERF_COUNTER
+#define EDC_UNSTRETCH_PERF_COUNTER__UNSTRETCH_PERF_COUNTER__SHIFT                                             0x0
+#define EDC_UNSTRETCH_PERF_COUNTER__UNSTRETCH_PERF_COUNTER_MASK                                               0xFFFFFFFFL
+//EDC_STRETCH_NUM_PERF_COUNTER
+#define EDC_STRETCH_NUM_PERF_COUNTER__STRETCH_NUM_PERF_COUNTER__SHIFT                                         0x0
+#define EDC_STRETCH_NUM_PERF_COUNTER__STRETCH_NUM_PERF_COUNTER_MASK                                           0xFFFFFFFFL
+//GC_EDC_STATUS
+#define GC_EDC_STATUS__EDC_THROTTLE_LEVEL__SHIFT                                                              0x0
+#define GC_EDC_STATUS__GPIO_IN_0__SHIFT                                                                       0x3
+#define GC_EDC_STATUS__GPIO_IN_1__SHIFT                                                                       0x4
+#define GC_EDC_STATUS__EDC_THROTTLE_LEVEL_MASK                                                                0x00000007L
+#define GC_EDC_STATUS__GPIO_IN_0_MASK                                                                         0x00000008L
+#define GC_EDC_STATUS__GPIO_IN_1_MASK                                                                         0x00000010L
+//GC_EDC_OVERFLOW
+#define GC_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW__SHIFT                                              0x0
+#define GC_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER__SHIFT                                           0x1
+#define GC_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW_MASK                                                0x00000001L
+#define GC_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER_MASK                                             0x0001FFFEL
+//GC_EDC_ROLLING_POWER_DELTA_LO
+#define GC_EDC_ROLLING_POWER_DELTA_LO__EDC_ROLLING_POWER_DELTA_LO__SHIFT                                      0x0
+#define GC_EDC_ROLLING_POWER_DELTA_LO__EDC_ROLLING_POWER_DELTA_LO_MASK                                        0xFFFFFFFFL
+//GC_EDC_ROLLING_POWER_DELTA_HI
+#define GC_EDC_ROLLING_POWER_DELTA_HI__EDC_ROLLING_POWER_DELTA_HI__SHIFT                                      0x0
+#define GC_EDC_ROLLING_POWER_DELTA_HI__EDC_ROLLING_POWER_DELTA_HI_MASK                                        0xFFFFFFFFL
+//GC_EDC_AVG_POWER_DELTA_LO
+#define GC_EDC_AVG_POWER_DELTA_LO__EDC_AVG_POWER_DELTA_LO__SHIFT                                              0x0
+#define GC_EDC_AVG_POWER_DELTA_LO__EDC_AVG_POWER_DELTA_LO_MASK                                                0xFFFFFFFFL
+//GC_EDC_AVG_POWER_DELTA_HI
+#define GC_EDC_AVG_POWER_DELTA_HI__EDC_AVG_POWER_DELTA_HI__SHIFT                                              0x0
+#define GC_EDC_AVG_POWER_DELTA_HI__EDC_AVG_POWER_DELTA_HI_MASK                                                0xFFFFFFFFL
+//GC_THROTTLE_STATUS
+#define GC_THROTTLE_STATUS__FSM_STATE__SHIFT                                                                  0x0
+#define GC_THROTTLE_STATUS__PATTERN_INDEX__SHIFT                                                              0x4
+#define GC_THROTTLE_STATUS__FSM_STATE_MASK                                                                    0x0000000FL
+#define GC_THROTTLE_STATUS__PATTERN_INDEX_MASK                                                                0x000001F0L
+//EDC_PERF_COUNTER
+#define EDC_PERF_COUNTER__EDC_PERF_COUNTER__SHIFT                                                             0x0
+#define EDC_PERF_COUNTER__EDC_PERF_COUNTER_MASK                                                               0xFFFFFFFFL
+//PCC_PERF_COUNTER
+#define PCC_PERF_COUNTER__PCC_PERF_COUNTER__SHIFT                                                             0x0
+#define PCC_PERF_COUNTER__PCC_PERF_COUNTER_MASK                                                               0xFFFFFFFFL
+//PWRBRK_PERF_COUNTER
+#define PWRBRK_PERF_COUNTER__PWRBRK_PERF_COUNTER__SHIFT                                                       0x0
+#define PWRBRK_PERF_COUNTER__PWRBRK_PERF_COUNTER_MASK                                                         0xFFFFFFFFL
+//EDC_HYSTERESIS_STAT
+#define EDC_HYSTERESIS_STAT__HYSTERESIS_CNT__SHIFT                                                            0x0
+#define EDC_HYSTERESIS_STAT__EDC_STATUS__SHIFT                                                                0x8
+#define EDC_HYSTERESIS_STAT__EDC_THRESHOLD_STAT__SHIFT                                                        0x9
+#define EDC_HYSTERESIS_STAT__HYSTERESIS_CNT_MASK                                                              0x000000FFL
+#define EDC_HYSTERESIS_STAT__EDC_STATUS_MASK                                                                  0x00000100L
+#define EDC_HYSTERESIS_STAT__EDC_THRESHOLD_STAT_MASK                                                          0x00000200L
+//DIDT_HYSTERESIS_STAT
+#define DIDT_HYSTERESIS_STAT__DIDT_HYSTERESIS_CNT__SHIFT                                                      0x0
+#define DIDT_HYSTERESIS_STAT__DIDT_DROOP_STATUS__SHIFT                                                        0x8
+#define DIDT_HYSTERESIS_STAT__DIDT_HYSTERESIS_CNT_MASK                                                        0x000000FFL
+#define DIDT_HYSTERESIS_STAT__DIDT_DROOP_STATUS_MASK                                                          0x00000100L
+//DIDT_PERF_COUNTER
+#define DIDT_PERF_COUNTER__DIDT_PERF_COUNTER__SHIFT                                                           0x0
+#define DIDT_PERF_COUNTER__DIDT_PERF_COUNTER_MASK                                                             0xFFFFFFFFL
+//GC_EDC_CLK_MONITOR_CTRL
+#define GC_EDC_CLK_MONITOR_CTRL__EDC_CLK_MONITOR_EN__SHIFT                                                    0x0
+#define GC_EDC_CLK_MONITOR_CTRL__EDC_CLK_MONITOR_INTERVAL__SHIFT                                              0x1
+#define GC_EDC_CLK_MONITOR_CTRL__EDC_CLK_MONITOR_THRESHOLD__SHIFT                                             0x5
+#define GC_EDC_CLK_MONITOR_CTRL__EDC_CLK_MONITOR_EN_MASK                                                      0x00000001L
+#define GC_EDC_CLK_MONITOR_CTRL__EDC_CLK_MONITOR_INTERVAL_MASK                                                0x0000001EL
+#define GC_EDC_CLK_MONITOR_CTRL__EDC_CLK_MONITOR_THRESHOLD_MASK                                               0x0001FFE0L
+//GC_CAC_SOFT_CTRL
+#define GC_CAC_SOFT_CTRL__CAC_SOFT_SNAP__SHIFT                                                                0x0
+#define GC_CAC_SOFT_CTRL__CAC_SOFT_SNAP_MASK                                                                  0x00000001L
+//GC_CAC_WEIGHT_CP_0
+#define GC_CAC_WEIGHT_CP_0__WEIGHT_CP_SIG0__SHIFT                                                             0x0
+#define GC_CAC_WEIGHT_CP_0__WEIGHT_CP_SIG1__SHIFT                                                             0x10
+#define GC_CAC_WEIGHT_CP_0__WEIGHT_CP_SIG0_MASK                                                               0x0000FFFFL
+#define GC_CAC_WEIGHT_CP_0__WEIGHT_CP_SIG1_MASK                                                               0xFFFF0000L
+//GC_CAC_WEIGHT_CP_1
+#define GC_CAC_WEIGHT_CP_1__WEIGHT_CP_SIG2__SHIFT                                                             0x0
+#define GC_CAC_WEIGHT_CP_1__WEIGHT_CP_SIG2_MASK                                                               0x0000FFFFL
+//GC_CAC_WEIGHT_PMM_0
+#define GC_CAC_WEIGHT_PMM_0__WEIGHT_PMM_SIG0__SHIFT                                                           0x0
+#define GC_CAC_WEIGHT_PMM_0__WEIGHT_PMM_SIG0_MASK                                                             0x0000FFFFL
+//GC_CAC_WEIGHT_SDMA_0
+#define GC_CAC_WEIGHT_SDMA_0__WEIGHT_SDMA_SIG0__SHIFT                                                         0x0
+#define GC_CAC_WEIGHT_SDMA_0__WEIGHT_SDMA_SIG1__SHIFT                                                         0x10
+#define GC_CAC_WEIGHT_SDMA_0__WEIGHT_SDMA_SIG0_MASK                                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_SDMA_0__WEIGHT_SDMA_SIG1_MASK                                                           0xFFFF0000L
+//GC_CAC_WEIGHT_SDMA_1
+#define GC_CAC_WEIGHT_SDMA_1__WEIGHT_SDMA_SIG2__SHIFT                                                         0x0
+#define GC_CAC_WEIGHT_SDMA_1__WEIGHT_SDMA_SIG3__SHIFT                                                         0x10
+#define GC_CAC_WEIGHT_SDMA_1__WEIGHT_SDMA_SIG2_MASK                                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_SDMA_1__WEIGHT_SDMA_SIG3_MASK                                                           0xFFFF0000L
+//GC_CAC_WEIGHT_SDMA_2
+#define GC_CAC_WEIGHT_SDMA_2__WEIGHT_SDMA_SIG4__SHIFT                                                         0x0
+#define GC_CAC_WEIGHT_SDMA_2__WEIGHT_SDMA_SIG5__SHIFT                                                         0x10
+#define GC_CAC_WEIGHT_SDMA_2__WEIGHT_SDMA_SIG4_MASK                                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_SDMA_2__WEIGHT_SDMA_SIG5_MASK                                                           0xFFFF0000L
+//GC_CAC_WEIGHT_SDMA_3
+#define GC_CAC_WEIGHT_SDMA_3__WEIGHT_SDMA_SIG6__SHIFT                                                         0x0
+#define GC_CAC_WEIGHT_SDMA_3__WEIGHT_SDMA_SIG7__SHIFT                                                         0x10
+#define GC_CAC_WEIGHT_SDMA_3__WEIGHT_SDMA_SIG6_MASK                                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_SDMA_3__WEIGHT_SDMA_SIG7_MASK                                                           0xFFFF0000L
+//GC_CAC_WEIGHT_SDMA_4
+#define GC_CAC_WEIGHT_SDMA_4__WEIGHT_SDMA_SIG8__SHIFT                                                         0x0
+#define GC_CAC_WEIGHT_SDMA_4__WEIGHT_SDMA_SIG9__SHIFT                                                         0x10
+#define GC_CAC_WEIGHT_SDMA_4__WEIGHT_SDMA_SIG8_MASK                                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_SDMA_4__WEIGHT_SDMA_SIG9_MASK                                                           0xFFFF0000L
+//GC_CAC_WEIGHT_SDMA_5
+#define GC_CAC_WEIGHT_SDMA_5__WEIGHT_SDMA_SIG10__SHIFT                                                        0x0
+#define GC_CAC_WEIGHT_SDMA_5__WEIGHT_SDMA_SIG11__SHIFT                                                        0x10
+#define GC_CAC_WEIGHT_SDMA_5__WEIGHT_SDMA_SIG10_MASK                                                          0x0000FFFFL
+#define GC_CAC_WEIGHT_SDMA_5__WEIGHT_SDMA_SIG11_MASK                                                          0xFFFF0000L
+//GC_CAC_WEIGHT_CHC_0
+#define GC_CAC_WEIGHT_CHC_0__WEIGHT_CHC_SIG0__SHIFT                                                           0x0
+#define GC_CAC_WEIGHT_CHC_0__WEIGHT_CHC_SIG1__SHIFT                                                           0x10
+#define GC_CAC_WEIGHT_CHC_0__WEIGHT_CHC_SIG0_MASK                                                             0x0000FFFFL
+#define GC_CAC_WEIGHT_CHC_0__WEIGHT_CHC_SIG1_MASK                                                             0xFFFF0000L
+//GC_CAC_WEIGHT_CHC_1
+#define GC_CAC_WEIGHT_CHC_1__WEIGHT_CHC_SIG2__SHIFT                                                           0x0
+#define GC_CAC_WEIGHT_CHC_1__WEIGHT_CHC_SIG2_MASK                                                             0x0000FFFFL
+//GC_CAC_WEIGHT_RLC_0
+#define GC_CAC_WEIGHT_RLC_0__WEIGHT_RLC_SIG0__SHIFT                                                           0x0
+#define GC_CAC_WEIGHT_RLC_0__WEIGHT_RLC_SIG0_MASK                                                             0x0000FFFFL
+//GC_CAC_WEIGHT_GRBM_0
+#define GC_CAC_WEIGHT_GRBM_0__WEIGHT_GRBM_SIG0__SHIFT                                                         0x0
+#define GC_CAC_WEIGHT_GRBM_0__WEIGHT_GRBM_SIG1__SHIFT                                                         0x10
+#define GC_CAC_WEIGHT_GRBM_0__WEIGHT_GRBM_SIG0_MASK                                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_GRBM_0__WEIGHT_GRBM_SIG1_MASK                                                           0xFFFF0000L
+//GC_CAC_WEIGHT_SDMA_6
+#define GC_CAC_WEIGHT_SDMA_6__WEIGHT_SDMA_SIG12__SHIFT                                                        0x0
+#define GC_CAC_WEIGHT_SDMA_6__WEIGHT_SDMA_SIG13__SHIFT                                                        0x10
+#define GC_CAC_WEIGHT_SDMA_6__WEIGHT_SDMA_SIG12_MASK                                                          0x0000FFFFL
+#define GC_CAC_WEIGHT_SDMA_6__WEIGHT_SDMA_SIG13_MASK                                                          0xFFFF0000L
+//GC_CAC_WEIGHT_SDMA_7
+#define GC_CAC_WEIGHT_SDMA_7__WEIGHT_SDMA_SIG14__SHIFT                                                        0x0
+#define GC_CAC_WEIGHT_SDMA_7__WEIGHT_SDMA_SIG15__SHIFT                                                        0x10
+#define GC_CAC_WEIGHT_SDMA_7__WEIGHT_SDMA_SIG14_MASK                                                          0x0000FFFFL
+#define GC_CAC_WEIGHT_SDMA_7__WEIGHT_SDMA_SIG15_MASK                                                          0xFFFF0000L
+//GC_CAC_WEIGHT_GLARB_0
+#define GC_CAC_WEIGHT_GLARB_0__WEIGHT_GLARB_SIG0__SHIFT                                                       0x0
+#define GC_CAC_WEIGHT_GLARB_0__WEIGHT_GLARB_SIG1__SHIFT                                                       0x10
+#define GC_CAC_WEIGHT_GLARB_0__WEIGHT_GLARB_SIG0_MASK                                                         0x0000FFFFL
+#define GC_CAC_WEIGHT_GLARB_0__WEIGHT_GLARB_SIG1_MASK                                                         0xFFFF0000L
+//GC_CAC_WEIGHT_GLARB_1
+#define GC_CAC_WEIGHT_GLARB_1__WEIGHT_GLARB_SIG2__SHIFT                                                       0x0
+#define GC_CAC_WEIGHT_GLARB_1__WEIGHT_GLARB_SIG2_MASK                                                         0x0000FFFFL
+//XCD_TOTAL_CAC_AGGR_LOWER
+#define XCD_TOTAL_CAC_AGGR_LOWER__XCD_TOTAL_AGGR_31_0__SHIFT                                                  0x0
+#define XCD_TOTAL_CAC_AGGR_LOWER__XCD_TOTAL_AGGR_31_0_MASK                                                    0xFFFFFFFFL
+//XCD_TOTAL_CAC_AGGR_UPPER
+#define XCD_TOTAL_CAC_AGGR_UPPER__XCD_TOTAL_AGGR_63_32__SHIFT                                                 0x0
+#define XCD_TOTAL_CAC_AGGR_UPPER__XCD_TOTAL_AGGR_63_32_MASK                                                   0xFFFFFFFFL
+//GC_EDC_WEIGHTED_DATA_MULTIPLIER
+#define GC_EDC_WEIGHTED_DATA_MULTIPLIER__CAC_SCALE__SHIFT                                                     0x0
+#define GC_EDC_WEIGHTED_DATA_MULTIPLIER__CAC_SCALING_EN__SHIFT                                                0x10
+#define GC_EDC_WEIGHTED_DATA_MULTIPLIER__CAC_SCALE_MASK                                                       0x0000FFFFL
+#define GC_EDC_WEIGHTED_DATA_MULTIPLIER__CAC_SCALING_EN_MASK                                                  0x00010000L
+//GC_CAC_IND_INDEX
+#define GC_CAC_IND_INDEX__GC_CAC_IND_ADDR__SHIFT                                                              0x0
+#define GC_CAC_IND_INDEX__GC_CAC_IND_ADDR_MASK                                                                0xFFFFFFFFL
+//GC_CAC_IND_DATA
+#define GC_CAC_IND_DATA__GC_CAC_IND_DATA__SHIFT                                                               0x0
+#define GC_CAC_IND_DATA__GC_CAC_IND_DATA_MASK                                                                 0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_gc_ea_cpwd_gceadec
+//GC_EA_CPWD_VC_MAP
+#define GC_EA_CPWD_VC_MAP__DRAM_VC__SHIFT                                                                     0x0
+#define GC_EA_CPWD_VC_MAP__IO_RD_VC__SHIFT                                                                    0x3
+#define GC_EA_CPWD_VC_MAP__IO_WR_VC__SHIFT                                                                    0x6
+#define GC_EA_CPWD_VC_MAP__GMI_VC__SHIFT                                                                      0x9
+#define GC_EA_CPWD_VC_MAP__NPA_VC__SHIFT                                                                      0xc
+#define GC_EA_CPWD_VC_MAP__NPAS_VC__SHIFT                                                                     0xf
+#define GC_EA_CPWD_VC_MAP__LPDDR_VC__SHIFT                                                                    0x12
+#define GC_EA_CPWD_VC_MAP__RSP_ACK_TRCKR_CHECK_EN_ON_RSP_CREDIT__SHIFT                                        0x15
+#define GC_EA_CPWD_VC_MAP__PRB_RSP_SEND_WO_PRB_RTN_DAT_AVAIL__SHIFT                                           0x16
+#define GC_EA_CPWD_VC_MAP__REG_ATOMIC_RPC_CMD_SELECT__SHIFT                                                   0x17
+#define GC_EA_CPWD_VC_MAP__RSP_ACK_TRACKER_FIFO_SPACE__SHIFT                                                  0x18
+#define GC_EA_CPWD_VC_MAP__DRAM_VC_MASK                                                                       0x00000007L
+#define GC_EA_CPWD_VC_MAP__IO_RD_VC_MASK                                                                      0x00000038L
+#define GC_EA_CPWD_VC_MAP__IO_WR_VC_MASK                                                                      0x000001C0L
+#define GC_EA_CPWD_VC_MAP__GMI_VC_MASK                                                                        0x00000E00L
+#define GC_EA_CPWD_VC_MAP__NPA_VC_MASK                                                                        0x00007000L
+#define GC_EA_CPWD_VC_MAP__NPAS_VC_MASK                                                                       0x00038000L
+#define GC_EA_CPWD_VC_MAP__LPDDR_VC_MASK                                                                      0x001C0000L
+#define GC_EA_CPWD_VC_MAP__RSP_ACK_TRCKR_CHECK_EN_ON_RSP_CREDIT_MASK                                          0x00200000L
+#define GC_EA_CPWD_VC_MAP__PRB_RSP_SEND_WO_PRB_RTN_DAT_AVAIL_MASK                                             0x00400000L
+#define GC_EA_CPWD_VC_MAP__REG_ATOMIC_RPC_CMD_SELECT_MASK                                                     0x00800000L
+#define GC_EA_CPWD_VC_MAP__RSP_ACK_TRACKER_FIFO_SPACE_MASK                                                    0xFF000000L
+//GC_EA_CPWD_SDP_ARB_FINAL
+#define GC_EA_CPWD_SDP_ARB_FINAL__DRAM_BURST_LIMIT__SHIFT                                                     0x0
+#define GC_EA_CPWD_SDP_ARB_FINAL__LPDDR_BURST_LIMIT__SHIFT                                                    0x5
+#define GC_EA_CPWD_SDP_ARB_FINAL__IO_BURST_LIMIT__SHIFT                                                       0xa
+#define GC_EA_CPWD_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER__SHIFT                                               0xf
+#define GC_EA_CPWD_SDP_ARB_FINAL__ERREVENT_ON_ERROR__SHIFT                                                    0x11
+#define GC_EA_CPWD_SDP_ARB_FINAL__HALTREQ_ON_ERROR__SHIFT                                                     0x12
+#define GC_EA_CPWD_SDP_ARB_FINAL__GMI_BURST_LIMIT__SHIFT                                                      0x13
+#define GC_EA_CPWD_SDP_ARB_FINAL__HDM_BURST_LIMIT__SHIFT                                                      0x18
+#define GC_EA_CPWD_SDP_ARB_FINAL__DRAM_BURST_LIMIT_MASK                                                       0x0000001FL
+#define GC_EA_CPWD_SDP_ARB_FINAL__LPDDR_BURST_LIMIT_MASK                                                      0x000003E0L
+#define GC_EA_CPWD_SDP_ARB_FINAL__IO_BURST_LIMIT_MASK                                                         0x00007C00L
+#define GC_EA_CPWD_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER_MASK                                                 0x00018000L
+#define GC_EA_CPWD_SDP_ARB_FINAL__ERREVENT_ON_ERROR_MASK                                                      0x00020000L
+#define GC_EA_CPWD_SDP_ARB_FINAL__HALTREQ_ON_ERROR_MASK                                                       0x00040000L
+#define GC_EA_CPWD_SDP_ARB_FINAL__GMI_BURST_LIMIT_MASK                                                        0x00F80000L
+#define GC_EA_CPWD_SDP_ARB_FINAL__HDM_BURST_LIMIT_MASK                                                        0x1F000000L
+//GC_EA_CPWD_SDP_PRIORITY
+#define GC_EA_CPWD_SDP_PRIORITY__DRAM_PRIORITY__SHIFT                                                         0x0
+#define GC_EA_CPWD_SDP_PRIORITY__IO_RD_PRIORITY__SHIFT                                                        0x4
+#define GC_EA_CPWD_SDP_PRIORITY__IO_WR_PRIORITY__SHIFT                                                        0x8
+#define GC_EA_CPWD_SDP_PRIORITY__MAM_WR_PRIORITY__SHIFT                                                       0xc
+#define GC_EA_CPWD_SDP_PRIORITY__GMI_PRIORITY__SHIFT                                                          0x10
+#define GC_EA_CPWD_SDP_PRIORITY__HDM_PRIORITY__SHIFT                                                          0x14
+#define GC_EA_CPWD_SDP_PRIORITY__LPDDR_PRIORITY__SHIFT                                                        0x18
+#define GC_EA_CPWD_SDP_PRIORITY__DRAM_PRIORITY_MASK                                                           0x0000000FL
+#define GC_EA_CPWD_SDP_PRIORITY__IO_RD_PRIORITY_MASK                                                          0x000000F0L
+#define GC_EA_CPWD_SDP_PRIORITY__IO_WR_PRIORITY_MASK                                                          0x00000F00L
+#define GC_EA_CPWD_SDP_PRIORITY__MAM_WR_PRIORITY_MASK                                                         0x0000F000L
+#define GC_EA_CPWD_SDP_PRIORITY__GMI_PRIORITY_MASK                                                            0x000F0000L
+#define GC_EA_CPWD_SDP_PRIORITY__HDM_PRIORITY_MASK                                                            0x00F00000L
+#define GC_EA_CPWD_SDP_PRIORITY__LPDDR_PRIORITY_MASK                                                          0x0F000000L
+//GC_EA_CPWD_SDP_CREDITS
+#define GC_EA_CPWD_SDP_CREDITS__TAG_LIMIT__SHIFT                                                              0x0
+#define GC_EA_CPWD_SDP_CREDITS__WR_RESP_CREDITS__SHIFT                                                        0x9
+#define GC_EA_CPWD_SDP_CREDITS__RD_RESP_CREDITS__SHIFT                                                        0x10
+#define GC_EA_CPWD_SDP_CREDITS__PRB_REQ_CREDITS__SHIFT                                                        0x17
+#define GC_EA_CPWD_SDP_CREDITS__TAG_LIMIT_MASK                                                                0x000001FFL
+#define GC_EA_CPWD_SDP_CREDITS__WR_RESP_CREDITS_MASK                                                          0x0000FE00L
+#define GC_EA_CPWD_SDP_CREDITS__RD_RESP_CREDITS_MASK                                                          0x007F0000L
+#define GC_EA_CPWD_SDP_CREDITS__PRB_REQ_CREDITS_MASK                                                          0x3F800000L
+//GC_EA_CPWD_SDP_TAG_RESERVE0
+#define GC_EA_CPWD_SDP_TAG_RESERVE0__VC0__SHIFT                                                               0x0
+#define GC_EA_CPWD_SDP_TAG_RESERVE0__VC1__SHIFT                                                               0x9
+#define GC_EA_CPWD_SDP_TAG_RESERVE0__VC2__SHIFT                                                               0x12
+#define GC_EA_CPWD_SDP_TAG_RESERVE0__VC0_MASK                                                                 0x000001FFL
+#define GC_EA_CPWD_SDP_TAG_RESERVE0__VC1_MASK                                                                 0x0003FE00L
+#define GC_EA_CPWD_SDP_TAG_RESERVE0__VC2_MASK                                                                 0x07FC0000L
+//GC_EA_CPWD_SDP_TAG_RESERVE1
+#define GC_EA_CPWD_SDP_TAG_RESERVE1__VC3__SHIFT                                                               0x0
+#define GC_EA_CPWD_SDP_TAG_RESERVE1__VC4__SHIFT                                                               0x9
+#define GC_EA_CPWD_SDP_TAG_RESERVE1__VC5__SHIFT                                                               0x12
+#define GC_EA_CPWD_SDP_TAG_RESERVE1__VC3_MASK                                                                 0x000001FFL
+#define GC_EA_CPWD_SDP_TAG_RESERVE1__VC4_MASK                                                                 0x0003FE00L
+#define GC_EA_CPWD_SDP_TAG_RESERVE1__VC5_MASK                                                                 0x07FC0000L
+//GC_EA_CPWD_SDP_TAG_RESERVE2
+#define GC_EA_CPWD_SDP_TAG_RESERVE2__VC6__SHIFT                                                               0x0
+#define GC_EA_CPWD_SDP_TAG_RESERVE2__VC7__SHIFT                                                               0x9
+#define GC_EA_CPWD_SDP_TAG_RESERVE2__VC6_MASK                                                                 0x000001FFL
+#define GC_EA_CPWD_SDP_TAG_RESERVE2__VC7_MASK                                                                 0x0003FE00L
+//GC_EA_CPWD_SDP_VCC_RESERVE0
+#define GC_EA_CPWD_SDP_VCC_RESERVE0__VC0_CREDITS__SHIFT                                                       0x0
+#define GC_EA_CPWD_SDP_VCC_RESERVE0__VC1_CREDITS__SHIFT                                                       0x6
+#define GC_EA_CPWD_SDP_VCC_RESERVE0__VC2_CREDITS__SHIFT                                                       0xc
+#define GC_EA_CPWD_SDP_VCC_RESERVE0__VC3_CREDITS__SHIFT                                                       0x12
+#define GC_EA_CPWD_SDP_VCC_RESERVE0__VC4_CREDITS__SHIFT                                                       0x18
+#define GC_EA_CPWD_SDP_VCC_RESERVE0__VC0_CREDITS_MASK                                                         0x0000003FL
+#define GC_EA_CPWD_SDP_VCC_RESERVE0__VC1_CREDITS_MASK                                                         0x00000FC0L
+#define GC_EA_CPWD_SDP_VCC_RESERVE0__VC2_CREDITS_MASK                                                         0x0003F000L
+#define GC_EA_CPWD_SDP_VCC_RESERVE0__VC3_CREDITS_MASK                                                         0x00FC0000L
+#define GC_EA_CPWD_SDP_VCC_RESERVE0__VC4_CREDITS_MASK                                                         0x3F000000L
+//GC_EA_CPWD_SDP_VCC_RESERVE1
+#define GC_EA_CPWD_SDP_VCC_RESERVE1__VC5_CREDITS__SHIFT                                                       0x0
+#define GC_EA_CPWD_SDP_VCC_RESERVE1__VC6_CREDITS__SHIFT                                                       0x6
+#define GC_EA_CPWD_SDP_VCC_RESERVE1__VC7_CREDITS__SHIFT                                                       0xc
+#define GC_EA_CPWD_SDP_VCC_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                   0x1f
+#define GC_EA_CPWD_SDP_VCC_RESERVE1__VC5_CREDITS_MASK                                                         0x0000003FL
+#define GC_EA_CPWD_SDP_VCC_RESERVE1__VC6_CREDITS_MASK                                                         0x00000FC0L
+#define GC_EA_CPWD_SDP_VCC_RESERVE1__VC7_CREDITS_MASK                                                         0x0003F000L
+#define GC_EA_CPWD_SDP_VCC_RESERVE1__DISTRIBUTE_POOL_MASK                                                     0x80000000L
+//GC_EA_CPWD_SDP_VCD_RESERVE0
+#define GC_EA_CPWD_SDP_VCD_RESERVE0__VC0_CREDITS__SHIFT                                                       0x0
+#define GC_EA_CPWD_SDP_VCD_RESERVE0__VC1_CREDITS__SHIFT                                                       0x6
+#define GC_EA_CPWD_SDP_VCD_RESERVE0__VC2_CREDITS__SHIFT                                                       0xc
+#define GC_EA_CPWD_SDP_VCD_RESERVE0__VC3_CREDITS__SHIFT                                                       0x12
+#define GC_EA_CPWD_SDP_VCD_RESERVE0__VC4_CREDITS__SHIFT                                                       0x18
+#define GC_EA_CPWD_SDP_VCD_RESERVE0__VC0_CREDITS_MASK                                                         0x0000003FL
+#define GC_EA_CPWD_SDP_VCD_RESERVE0__VC1_CREDITS_MASK                                                         0x00000FC0L
+#define GC_EA_CPWD_SDP_VCD_RESERVE0__VC2_CREDITS_MASK                                                         0x0003F000L
+#define GC_EA_CPWD_SDP_VCD_RESERVE0__VC3_CREDITS_MASK                                                         0x00FC0000L
+#define GC_EA_CPWD_SDP_VCD_RESERVE0__VC4_CREDITS_MASK                                                         0x3F000000L
+//GC_EA_CPWD_SDP_VCD_RESERVE1
+#define GC_EA_CPWD_SDP_VCD_RESERVE1__VC5_CREDITS__SHIFT                                                       0x0
+#define GC_EA_CPWD_SDP_VCD_RESERVE1__VC6_CREDITS__SHIFT                                                       0x6
+#define GC_EA_CPWD_SDP_VCD_RESERVE1__VC7_CREDITS__SHIFT                                                       0xc
+#define GC_EA_CPWD_SDP_VCD_RESERVE1__PRB_CREDITS__SHIFT                                                       0x12
+#define GC_EA_CPWD_SDP_VCD_RESERVE1__POOL_CREDITS__SHIFT                                                      0x18
+#define GC_EA_CPWD_SDP_VCD_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                   0x1f
+#define GC_EA_CPWD_SDP_VCD_RESERVE1__VC5_CREDITS_MASK                                                         0x0000003FL
+#define GC_EA_CPWD_SDP_VCD_RESERVE1__VC6_CREDITS_MASK                                                         0x00000FC0L
+#define GC_EA_CPWD_SDP_VCD_RESERVE1__VC7_CREDITS_MASK                                                         0x0003F000L
+#define GC_EA_CPWD_SDP_VCD_RESERVE1__PRB_CREDITS_MASK                                                         0x00FC0000L
+#define GC_EA_CPWD_SDP_VCD_RESERVE1__POOL_CREDITS_MASK                                                        0x3F000000L
+#define GC_EA_CPWD_SDP_VCD_RESERVE1__DISTRIBUTE_POOL_MASK                                                     0x80000000L
+//GC_EA_CPWD_SDP_REQ_CNTL
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ__SHIFT                                             0x0
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE__SHIFT                                            0x1
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC__SHIFT                                           0x2
+#define GC_EA_CPWD_SDP_REQ_CNTL__INNER_DOMAIN_MODE__SHIFT                                                     0x3
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_READ__SHIFT                                                  0x4
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_WRITE__SHIFT                                                 0x6
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_ATOMIC__SHIFT                                                0x8
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM__SHIFT                                               0xa
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_GMI__SHIFT                                                0xb
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_HDM__SHIFT                                                0xc
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_LPDDR__SHIFT                                              0xd
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ_MASK                                               0x00000001L
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE_MASK                                              0x00000002L
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC_MASK                                             0x00000004L
+#define GC_EA_CPWD_SDP_REQ_CNTL__INNER_DOMAIN_MODE_MASK                                                       0x00000008L
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_READ_MASK                                                    0x00000030L
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_WRITE_MASK                                                   0x000000C0L
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_ATOMIC_MASK                                                  0x00000300L
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM_MASK                                                 0x00000400L
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_GMI_MASK                                                  0x00000800L
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_HDM_MASK                                                  0x00001000L
+#define GC_EA_CPWD_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_LPDDR_MASK                                                0x00002000L
+//GC_EA_CPWD_MISC
+#define GC_EA_CPWD_MISC__LINKMGR_DYNAMIC_MODE__SHIFT                                                          0x0
+#define GC_EA_CPWD_MISC__LINKMGR_HALT_THRESHOLD__SHIFT                                                        0x2
+#define GC_EA_CPWD_MISC__LINKMGR_RECONNECT_DELAY__SHIFT                                                       0x4
+#define GC_EA_CPWD_MISC__LINKMGR_IDLE_THRESHOLD__SHIFT                                                        0x6
+#define GC_EA_CPWD_MISC__LINKMGR_CREDITRESET_CGCG_IGNORE__SHIFT                                               0xb
+#define GC_EA_CPWD_MISC__RSP_ACK_TRACKER_FIFO_SPACE_RSP_PATH__SHIFT                                           0xc
+#define GC_EA_CPWD_MISC__RSP_ACK_PRB_LINK_BUSY__SHIFT                                                         0x14
+#define GC_EA_CPWD_MISC__RSP_ACK_TAG_RELEASE__SHIFT                                                           0x15
+#define GC_EA_CPWD_MISC__LINKMGR_DYNAMIC_MODE_MASK                                                            0x00000003L
+#define GC_EA_CPWD_MISC__LINKMGR_HALT_THRESHOLD_MASK                                                          0x0000000CL
+#define GC_EA_CPWD_MISC__LINKMGR_RECONNECT_DELAY_MASK                                                         0x00000030L
+#define GC_EA_CPWD_MISC__LINKMGR_IDLE_THRESHOLD_MASK                                                          0x000007C0L
+#define GC_EA_CPWD_MISC__LINKMGR_CREDITRESET_CGCG_IGNORE_MASK                                                 0x00000800L
+#define GC_EA_CPWD_MISC__RSP_ACK_TRACKER_FIFO_SPACE_RSP_PATH_MASK                                             0x000FF000L
+#define GC_EA_CPWD_MISC__RSP_ACK_PRB_LINK_BUSY_MASK                                                           0x00100000L
+#define GC_EA_CPWD_MISC__RSP_ACK_TAG_RELEASE_MASK                                                             0x00200000L
+//GC_EA_CPWD_ERR_STATUS
+#define GC_EA_CPWD_ERR_STATUS__SDP_RDRSP_STATUS__SHIFT                                                        0x0
+#define GC_EA_CPWD_ERR_STATUS__SDP_WRRSP_STATUS__SHIFT                                                        0x4
+#define GC_EA_CPWD_ERR_STATUS__SDP_RDRSP_DATASTATUS__SHIFT                                                    0x8
+#define GC_EA_CPWD_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR__SHIFT                                              0xa
+#define GC_EA_CPWD_ERR_STATUS__SDP_RDRSP_COMP_KEY_PARITY_ERROR__SHIFT                                         0xb
+#define GC_EA_CPWD_ERR_STATUS__CLEAR_ERROR_STATUS__SHIFT                                                      0xc
+#define GC_EA_CPWD_ERR_STATUS__BUSY_ON_ERROR__SHIFT                                                           0xd
+#define GC_EA_CPWD_ERR_STATUS__IGNORE_RDRSP_FED__SHIFT                                                        0xe
+#define GC_EA_CPWD_ERR_STATUS__FUE_FLAG__SHIFT                                                                0xf
+#define GC_EA_CPWD_ERR_STATUS__SDP_RDRSP_STATUS_MASK                                                          0x0000000FL
+#define GC_EA_CPWD_ERR_STATUS__SDP_WRRSP_STATUS_MASK                                                          0x000000F0L
+#define GC_EA_CPWD_ERR_STATUS__SDP_RDRSP_DATASTATUS_MASK                                                      0x00000300L
+#define GC_EA_CPWD_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR_MASK                                                0x00000400L
+#define GC_EA_CPWD_ERR_STATUS__SDP_RDRSP_COMP_KEY_PARITY_ERROR_MASK                                           0x00000800L
+#define GC_EA_CPWD_ERR_STATUS__CLEAR_ERROR_STATUS_MASK                                                        0x00001000L
+#define GC_EA_CPWD_ERR_STATUS__BUSY_ON_ERROR_MASK                                                             0x00002000L
+#define GC_EA_CPWD_ERR_STATUS__IGNORE_RDRSP_FED_MASK                                                          0x00004000L
+#define GC_EA_CPWD_ERR_STATUS__FUE_FLAG_MASK                                                                  0x00008000L
+//GC_EA_CPWD_MISC2
+#define GC_EA_CPWD_MISC2__BLOCK_REQUESTS__SHIFT                                                               0x0
+#define GC_EA_CPWD_MISC2__REQUESTS_BLOCKED__SHIFT                                                             0x1
+#define GC_EA_CPWD_MISC2__FGCLKEN_OVERRIDE__SHIFT                                                             0x2
+#define GC_EA_CPWD_MISC2__LINKMGR_CRBUSY_MASK__SHIFT                                                          0x3
+#define GC_EA_CPWD_MISC2__RDRET_FED_MASK__SHIFT                                                               0x4
+#define GC_EA_CPWD_MISC2__BLOCK_REQUESTS_MASK                                                                 0x00000001L
+#define GC_EA_CPWD_MISC2__REQUESTS_BLOCKED_MASK                                                               0x00000002L
+#define GC_EA_CPWD_MISC2__FGCLKEN_OVERRIDE_MASK                                                               0x00000004L
+#define GC_EA_CPWD_MISC2__LINKMGR_CRBUSY_MASK_MASK                                                            0x00000008L
+#define GC_EA_CPWD_MISC2__RDRET_FED_MASK_MASK                                                                 0x00000010L
+//GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0__VC0_CREDITS_RECEIVED__SHIFT                                      0x0
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0__VC1_CREDITS_RECEIVED__SHIFT                                      0x7
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0__VC2_CREDITS_RECEIVED__SHIFT                                      0xe
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0__VC3_CREDITS_RECEIVED__SHIFT                                      0x15
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0__VC4_CREDITS_RECEIVED__SHIFT                                      0x1c
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0__VC0_CREDITS_RECEIVED_MASK                                        0x0000007FL
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0__VC1_CREDITS_RECEIVED_MASK                                        0x00003F80L
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0__VC2_CREDITS_RECEIVED_MASK                                        0x001FC000L
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0__VC3_CREDITS_RECEIVED_MASK                                        0x0FE00000L
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0__VC4_CREDITS_RECEIVED_MASK                                        0xF0000000L
+//GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0_WRITE
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0_WRITE__VC0_CREDITS_RECEIVED__SHIFT                                0x0
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0_WRITE__VC1_CREDITS_RECEIVED__SHIFT                                0x7
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0_WRITE__VC2_CREDITS_RECEIVED__SHIFT                                0xe
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0_WRITE__VC3_CREDITS_RECEIVED__SHIFT                                0x15
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0_WRITE__VC4_CREDITS_RECEIVED__SHIFT                                0x1c
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0_WRITE__VC0_CREDITS_RECEIVED_MASK                                  0x0000007FL
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0_WRITE__VC1_CREDITS_RECEIVED_MASK                                  0x00003F80L
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0_WRITE__VC2_CREDITS_RECEIVED_MASK                                  0x001FC000L
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0_WRITE__VC3_CREDITS_RECEIVED_MASK                                  0x0FE00000L
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS0_WRITE__VC4_CREDITS_RECEIVED_MASK                                  0xF0000000L
+//GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1__VC4_CREDITS_RECEIVED__SHIFT                                      0x0
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1__VC5_CREDITS_RECEIVED__SHIFT                                      0x3
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1__VC6_CREDITS_RECEIVED__SHIFT                                      0xa
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1__VC7_CREDITS_RECEIVED__SHIFT                                      0x11
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1__POOL_CREDITS_RECEIVED__SHIFT                                     0x18
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1__VC4_CREDITS_RECEIVED_MASK                                        0x00000007L
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1__VC5_CREDITS_RECEIVED_MASK                                        0x000003F8L
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1__VC6_CREDITS_RECEIVED_MASK                                        0x0001FC00L
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1__VC7_CREDITS_RECEIVED_MASK                                        0x00FE0000L
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1__POOL_CREDITS_RECEIVED_MASK                                       0x7F000000L
+//GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1_WRITE
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1_WRITE__VC4_CREDITS_RECEIVED__SHIFT                                0x0
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1_WRITE__VC5_CREDITS_RECEIVED__SHIFT                                0x3
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1_WRITE__VC6_CREDITS_RECEIVED__SHIFT                                0xa
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1_WRITE__VC7_CREDITS_RECEIVED__SHIFT                                0x11
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1_WRITE__POOL_CREDITS_RECEIVED__SHIFT                               0x18
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1_WRITE__VC4_CREDITS_RECEIVED_MASK                                  0x00000007L
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1_WRITE__VC5_CREDITS_RECEIVED_MASK                                  0x000003F8L
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1_WRITE__VC6_CREDITS_RECEIVED_MASK                                  0x0001FC00L
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1_WRITE__VC7_CREDITS_RECEIVED_MASK                                  0x00FE0000L
+#define GC_EA_CPWD_SDP_BACKDOOR_CMDCREDITS1_WRITE__POOL_CREDITS_RECEIVED_MASK                                 0x7F000000L
+//GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0__VC0_CREDITS_RECEIVED__SHIFT                                     0x0
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0__VC1_CREDITS_RECEIVED__SHIFT                                     0x7
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0__VC2_CREDITS_RECEIVED__SHIFT                                     0xe
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0__VC3_CREDITS_RECEIVED__SHIFT                                     0x15
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0__VC4_CREDITS_RECEIVED__SHIFT                                     0x1c
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0__VC0_CREDITS_RECEIVED_MASK                                       0x0000007FL
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0__VC1_CREDITS_RECEIVED_MASK                                       0x00003F80L
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0__VC2_CREDITS_RECEIVED_MASK                                       0x001FC000L
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0__VC3_CREDITS_RECEIVED_MASK                                       0x0FE00000L
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0__VC4_CREDITS_RECEIVED_MASK                                       0xF0000000L
+//GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0_WRITE
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0_WRITE__VC0_CREDITS_RECEIVED__SHIFT                               0x0
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0_WRITE__VC1_CREDITS_RECEIVED__SHIFT                               0x7
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0_WRITE__VC2_CREDITS_RECEIVED__SHIFT                               0xe
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0_WRITE__VC3_CREDITS_RECEIVED__SHIFT                               0x15
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0_WRITE__VC4_CREDITS_RECEIVED__SHIFT                               0x1c
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0_WRITE__VC0_CREDITS_RECEIVED_MASK                                 0x0000007FL
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0_WRITE__VC1_CREDITS_RECEIVED_MASK                                 0x00003F80L
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0_WRITE__VC2_CREDITS_RECEIVED_MASK                                 0x001FC000L
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0_WRITE__VC3_CREDITS_RECEIVED_MASK                                 0x0FE00000L
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS0_WRITE__VC4_CREDITS_RECEIVED_MASK                                 0xF0000000L
+//GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1__VC4_CREDITS_RECEIVED__SHIFT                                     0x0
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1__VC5_CREDITS_RECEIVED__SHIFT                                     0x3
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1__VC6_CREDITS_RECEIVED__SHIFT                                     0xa
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1__VC7_CREDITS_RECEIVED__SHIFT                                     0x11
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1__POOL_CREDITS_RECEIVED__SHIFT                                    0x18
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1__VC4_CREDITS_RECEIVED_MASK                                       0x00000007L
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1__VC5_CREDITS_RECEIVED_MASK                                       0x000003F8L
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1__VC6_CREDITS_RECEIVED_MASK                                       0x0001FC00L
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1__VC7_CREDITS_RECEIVED_MASK                                       0x00FE0000L
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1__POOL_CREDITS_RECEIVED_MASK                                      0x7F000000L
+//GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1_WRITE
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1_WRITE__VC4_CREDITS_RECEIVED__SHIFT                               0x0
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1_WRITE__VC5_CREDITS_RECEIVED__SHIFT                               0x3
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1_WRITE__VC6_CREDITS_RECEIVED__SHIFT                               0xa
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1_WRITE__VC7_CREDITS_RECEIVED__SHIFT                               0x11
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1_WRITE__POOL_CREDITS_RECEIVED__SHIFT                              0x18
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1_WRITE__VC4_CREDITS_RECEIVED_MASK                                 0x00000007L
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1_WRITE__VC5_CREDITS_RECEIVED_MASK                                 0x000003F8L
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1_WRITE__VC6_CREDITS_RECEIVED_MASK                                 0x0001FC00L
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1_WRITE__VC7_CREDITS_RECEIVED_MASK                                 0x00FE0000L
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS1_WRITE__POOL_CREDITS_RECEIVED_MASK                                0x7F000000L
+//GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2__PRBR_CREDITS_RECEIVED__SHIFT                                    0x0
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2__PRBRSP_CREDITS_RECEIVED__SHIFT                                  0x7
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2__RSP_ACK_CREDITS_RECEIVED__SHIFT                                 0xe
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2__PRBR_CREDITS_RECEIVED_MASK                                      0x0000007FL
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2__PRBRSP_CREDITS_RECEIVED_MASK                                    0x00003F80L
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2__RSP_ACK_CREDITS_RECEIVED_MASK                                   0x001FC000L
+//GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2_WRITE
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2_WRITE__PRBR_CREDITS_RECEIVED__SHIFT                              0x0
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2_WRITE__PRBRSP_CREDITS_RECEIVED__SHIFT                            0x7
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2_WRITE__RSP_ACK_CREDITS_RECEIVED__SHIFT                           0xe
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2_WRITE__PRBR_CREDITS_RECEIVED_MASK                                0x0000007FL
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2_WRITE__PRBRSP_CREDITS_RECEIVED_MASK                              0x00003F80L
+#define GC_EA_CPWD_SDP_BACKDOOR_DATACREDITS2_WRITE__RSP_ACK_CREDITS_RECEIVED_MASK                             0x001FC000L
+//GC_EA_CPWD_SDP_BACKDOOR_MISCCTL
+#define GC_EA_CPWD_SDP_BACKDOOR_MISCCTL__SDP_ORIGCLKCTL__SHIFT                                                0x0
+#define GC_EA_CPWD_SDP_BACKDOOR_MISCCTL__SDP_ORIGCLKCTL_MASK                                                  0x00000001L
+//GC_EA_CPWD_SDP_BACKDOOR_MISCCTL_WRITE
+#define GC_EA_CPWD_SDP_BACKDOOR_MISCCTL_WRITE__SDP_ORIGCLKCTL__SHIFT                                          0x0
+#define GC_EA_CPWD_SDP_BACKDOOR_MISCCTL_WRITE__SDP_ORIGCLKCTL_MASK                                            0x00000001L
+//GC_EA_CPWD_INVALID_OPCODE
+#define GC_EA_CPWD_INVALID_OPCODE__INVALID_COMMAND__SHIFT                                                     0x0
+#define GC_EA_CPWD_INVALID_OPCODE__BAD_OPCODE__SHIFT                                                          0x1
+#define GC_EA_CPWD_INVALID_OPCODE__BAD_OPCODE_VFID__SHIFT                                                     0xc
+#define GC_EA_CPWD_INVALID_OPCODE__INVALID_COMMAND_MASK                                                       0x00000001L
+#define GC_EA_CPWD_INVALID_OPCODE__BAD_OPCODE_MASK                                                            0x000001FEL
+#define GC_EA_CPWD_INVALID_OPCODE__BAD_OPCODE_VFID_MASK                                                       0x0001F000L
+//GC_EA_CPWD_SDP_POISON_ERR_INJ_CTRL
+#define GC_EA_CPWD_SDP_POISON_ERR_INJ_CTRL__EGRESS_POISON_ERR_INJ_CNT__SHIFT                                  0x0
+#define GC_EA_CPWD_SDP_POISON_ERR_INJ_CTRL__EGRESS_POISON_ERR_INJ_EN__SHIFT                                   0x7
+#define GC_EA_CPWD_SDP_POISON_ERR_INJ_CTRL__INGRESS_POISON_ERR_INJ_CNT__SHIFT                                 0x8
+#define GC_EA_CPWD_SDP_POISON_ERR_INJ_CTRL__INGRESS_POISON_ERR_INJ_EN__SHIFT                                  0xf
+#define GC_EA_CPWD_SDP_POISON_ERR_INJ_CTRL__EGRESS_POISON_ERR_INJ_CNT_MASK                                    0x0000007FL
+#define GC_EA_CPWD_SDP_POISON_ERR_INJ_CTRL__EGRESS_POISON_ERR_INJ_EN_MASK                                     0x00000080L
+#define GC_EA_CPWD_SDP_POISON_ERR_INJ_CTRL__INGRESS_POISON_ERR_INJ_CNT_MASK                                   0x00007F00L
+#define GC_EA_CPWD_SDP_POISON_ERR_INJ_CTRL__INGRESS_POISON_ERR_INJ_EN_MASK                                    0x00008000L
+//GC_EA_CPWD_SDP_POISON_ERR_LOG
+#define GC_EA_CPWD_SDP_POISON_ERR_LOG__EGRESS_POISON_ERR_CNT_LOG__SHIFT                                       0x0
+#define GC_EA_CPWD_SDP_POISON_ERR_LOG__INGRESS_POISON_ERR_CNT_LOG__SHIFT                                      0x8
+#define GC_EA_CPWD_SDP_POISON_ERR_LOG__EGRESS_POISON_ERR_CNT_LOG_MASK                                         0x000000FFL
+#define GC_EA_CPWD_SDP_POISON_ERR_LOG__INGRESS_POISON_ERR_CNT_LOG_MASK                                        0x0000FF00L
+//GC_EA_CPWD_SDP_PARITY_ERR_INJ_CTRL
+#define GC_EA_CPWD_SDP_PARITY_ERR_INJ_CTRL__EGRESS_ORIGDATA_PARITY_ERR_INJ_CNT__SHIFT                         0x0
+#define GC_EA_CPWD_SDP_PARITY_ERR_INJ_CTRL__EGRESS_ORIGDATA_PARITY_ERR_INJ_EN__SHIFT                          0x7
+#define GC_EA_CPWD_SDP_PARITY_ERR_INJ_CTRL__INGRESS_RDRSPDATA_PARITY_ERR_INJ_CNT__SHIFT                       0x8
+#define GC_EA_CPWD_SDP_PARITY_ERR_INJ_CTRL__INGRESS_RDRSPDATA_PARITY_ERR_INJ_EN__SHIFT                        0xf
+#define GC_EA_CPWD_SDP_PARITY_ERR_INJ_CTRL__EGRESS_ORIGDATA_PARITY_ERR_INJ_CNT_MASK                           0x0000007FL
+#define GC_EA_CPWD_SDP_PARITY_ERR_INJ_CTRL__EGRESS_ORIGDATA_PARITY_ERR_INJ_EN_MASK                            0x00000080L
+#define GC_EA_CPWD_SDP_PARITY_ERR_INJ_CTRL__INGRESS_RDRSPDATA_PARITY_ERR_INJ_CNT_MASK                         0x00007F00L
+#define GC_EA_CPWD_SDP_PARITY_ERR_INJ_CTRL__INGRESS_RDRSPDATA_PARITY_ERR_INJ_EN_MASK                          0x00008000L
+//GC_EA_CPWD_SDP_PARITY_ERR_LOG
+#define GC_EA_CPWD_SDP_PARITY_ERR_LOG__EGRESS_ORIGDATA_PARITY_ERR_CNT_LOG__SHIFT                              0x0
+#define GC_EA_CPWD_SDP_PARITY_ERR_LOG__INGRESS_RDRSPDATA_PARITY_ERR_CNT_LOG__SHIFT                            0x8
+#define GC_EA_CPWD_SDP_PARITY_ERR_LOG__EGRESS_ORIGDATA_PARITY_ERR_CNT_LOG_MASK                                0x000000FFL
+#define GC_EA_CPWD_SDP_PARITY_ERR_LOG__INGRESS_RDRSPDATA_PARITY_ERR_CNT_LOG_MASK                              0x0000FF00L
+//GC_EA_CPWD_SDP_ENABLE
+#define GC_EA_CPWD_SDP_ENABLE__ENABLE__SHIFT                                                                  0x0
+#define GC_EA_CPWD_SDP_ENABLE__EARLY_CREDIT_REQUEST__SHIFT                                                    0x1
+#define GC_EA_CPWD_SDP_ENABLE__ENABLE_MASK                                                                    0x00000001L
+#define GC_EA_CPWD_SDP_ENABLE__EARLY_CREDIT_REQUEST_MASK                                                      0x00000002L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_gcrdec
+//GCR_PIO_CNTL
+#define GCR_PIO_CNTL__GCR_DATA_INDEX__SHIFT                                                                   0x0
+#define GCR_PIO_CNTL__GCR_REG_DONE__SHIFT                                                                     0x3
+#define GCR_PIO_CNTL__GCR_REG_RESET__SHIFT                                                                    0x4
+#define GCR_PIO_CNTL__GCR_PIO_RSP_TAG__SHIFT                                                                  0x10
+#define GCR_PIO_CNTL__GCR_PIO_RSP_DONE__SHIFT                                                                 0x1e
+#define GCR_PIO_CNTL__GCR_READY__SHIFT                                                                        0x1f
+#define GCR_PIO_CNTL__GCR_DATA_INDEX_MASK                                                                     0x00000007L
+#define GCR_PIO_CNTL__GCR_REG_DONE_MASK                                                                       0x00000008L
+#define GCR_PIO_CNTL__GCR_REG_RESET_MASK                                                                      0x00000010L
+#define GCR_PIO_CNTL__GCR_PIO_RSP_TAG_MASK                                                                    0x00FF0000L
+#define GCR_PIO_CNTL__GCR_PIO_RSP_DONE_MASK                                                                   0x40000000L
+#define GCR_PIO_CNTL__GCR_READY_MASK                                                                          0x80000000L
+//GCR_PIO_DATA
+#define GCR_PIO_DATA__GCR_DATA__SHIFT                                                                         0x0
+#define GCR_PIO_DATA__GCR_DATA_MASK                                                                           0xFFFFFFFFL
+//GCR_NHTOE_CNTL
+#define GCR_NHTOE_CNTL__GCR_DATA_INDEX__SHIFT                                                                 0x0
+#define GCR_NHTOE_CNTL__GCR_REG_DONE__SHIFT                                                                   0x3
+#define GCR_NHTOE_CNTL__GCR_REG_RESET__SHIFT                                                                  0x4
+#define GCR_NHTOE_CNTL__GCR_NHTOE_RSP_TAG__SHIFT                                                              0x10
+#define GCR_NHTOE_CNTL__GCR_NHTOE_RSP_DONE__SHIFT                                                             0x1e
+#define GCR_NHTOE_CNTL__GCR_READY__SHIFT                                                                      0x1f
+#define GCR_NHTOE_CNTL__GCR_DATA_INDEX_MASK                                                                   0x00000007L
+#define GCR_NHTOE_CNTL__GCR_REG_DONE_MASK                                                                     0x00000008L
+#define GCR_NHTOE_CNTL__GCR_REG_RESET_MASK                                                                    0x00000010L
+#define GCR_NHTOE_CNTL__GCR_NHTOE_RSP_TAG_MASK                                                                0x00FF0000L
+#define GCR_NHTOE_CNTL__GCR_NHTOE_RSP_DONE_MASK                                                               0x40000000L
+#define GCR_NHTOE_CNTL__GCR_READY_MASK                                                                        0x80000000L
+//GCR_NHTOE_DATA
+#define GCR_NHTOE_DATA__GCR_DATA__SHIFT                                                                       0x0
+#define GCR_NHTOE_DATA__GCR_DATA_MASK                                                                         0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_cppdec
+//CPC_INT_ADDR1
+#define CPC_INT_ADDR1__ADDR__SHIFT                                                                            0x0
+#define CPC_INT_ADDR1__ADDR_MASK                                                                              0x000001FFL
+//CP_CU_MASK_ADDR_LO
+#define CP_CU_MASK_ADDR_LO__ADDR_LO__SHIFT                                                                    0x2
+#define CP_CU_MASK_ADDR_LO__ADDR_LO_MASK                                                                      0xFFFFFFFCL
+//CP_CU_MASK_ADDR_HI
+#define CP_CU_MASK_ADDR_HI__ADDR_HI__SHIFT                                                                    0x0
+#define CP_CU_MASK_ADDR_HI__ADDR_HI_MASK                                                                      0xFFFFFFFFL
+//CP_CU_MASK_CNTL
+#define CP_CU_MASK_CNTL__POLICY__SHIFT                                                                        0x0
+#define CP_CU_MASK_CNTL__SCOPE__SHIFT                                                                         0x2
+#define CP_CU_MASK_CNTL__POLICY_MASK                                                                          0x00000003L
+#define CP_CU_MASK_CNTL__SCOPE_MASK                                                                           0x0000000CL
+//CP_EOPQ_WAIT_TIME
+#define CP_EOPQ_WAIT_TIME__WAIT_TIME__SHIFT                                                                   0x0
+#define CP_EOPQ_WAIT_TIME__SCALE_COUNT__SHIFT                                                                 0xa
+#define CP_EOPQ_WAIT_TIME__WAIT_TIME_MASK                                                                     0x000003FFL
+#define CP_EOPQ_WAIT_TIME__SCALE_COUNT_MASK                                                                   0x0003FC00L
+//CP_CPC_MGCG_SYNC_CNTL
+#define CP_CPC_MGCG_SYNC_CNTL__COOLDOWN_PERIOD__SHIFT                                                         0x0
+#define CP_CPC_MGCG_SYNC_CNTL__WARMUP_PERIOD__SHIFT                                                           0x8
+#define CP_CPC_MGCG_SYNC_CNTL__COOLDOWN_PERIOD_MASK                                                           0x000000FFL
+#define CP_CPC_MGCG_SYNC_CNTL__WARMUP_PERIOD_MASK                                                             0x0000FF00L
+//CPC_INT_INFO
+#define CPC_INT_INFO__ADDR_HI__SHIFT                                                                          0x0
+#define CPC_INT_INFO__TYPE__SHIFT                                                                             0x10
+#define CPC_INT_INFO__VMID__SHIFT                                                                             0x14
+#define CPC_INT_INFO__QUEUE_ID__SHIFT                                                                         0x1c
+#define CPC_INT_INFO__ADDR_HI_MASK                                                                            0x0000FFFFL
+#define CPC_INT_INFO__TYPE_MASK                                                                               0x00010000L
+#define CPC_INT_INFO__VMID_MASK                                                                               0x00F00000L
+#define CPC_INT_INFO__QUEUE_ID_MASK                                                                           0x70000000L
+//CP_VIRT_STATUS
+#define CP_VIRT_STATUS__VF__SHIFT                                                                             0x0
+#define CP_VIRT_STATUS__PF__SHIFT                                                                             0x1f
+#define CP_VIRT_STATUS__VF_MASK                                                                               0x00FFFFFFL
+#define CP_VIRT_STATUS__PF_MASK                                                                               0x80000000L
+//CPC_INT_ADDR
+#define CPC_INT_ADDR__ADDR__SHIFT                                                                             0x0
+#define CPC_INT_ADDR__ADDR_MASK                                                                               0xFFFFFFFFL
+//CPC_INT_PASID
+#define CPC_INT_PASID__PASID__SHIFT                                                                           0x0
+#define CPC_INT_PASID__BYPASS_PASID__SHIFT                                                                    0x10
+#define CPC_INT_PASID__PASID_MASK                                                                             0x0000FFFFL
+#define CPC_INT_PASID__BYPASS_PASID_MASK                                                                      0x00010000L
+//CP_GFX_ERROR
+#define CP_GFX_ERROR__ME_INSTR_CACHE_UTCL1_ERROR__SHIFT                                                       0x0
+#define CP_GFX_ERROR__PFP_INSTR_CACHE_UTCL1_ERROR__SHIFT                                                      0x1
+#define CP_GFX_ERROR__DDID_DRAW_UTCL1_ERROR__SHIFT                                                            0x2
+#define CP_GFX_ERROR__DDID_DISPATCH_UTCL1_ERROR__SHIFT                                                        0x3
+#define CP_GFX_ERROR__SUA_ERROR__SHIFT                                                                        0x4
+#define CP_GFX_ERROR__DATA_FETCHER_UTCL1_ERROR__SHIFT                                                         0x6
+#define CP_GFX_ERROR__QU_EOP_UTCL1_ERROR__SHIFT                                                               0x9
+#define CP_GFX_ERROR__QU_PIPE_UTCL1_ERROR__SHIFT                                                              0xa
+#define CP_GFX_ERROR__QU_READ_UTCL1_ERROR__SHIFT                                                              0xb
+#define CP_GFX_ERROR__SYNC_MEMRD_UTCL1_ERROR__SHIFT                                                           0xc
+#define CP_GFX_ERROR__SYNC_MEMWR_UTCL1_ERROR__SHIFT                                                           0xd
+#define CP_GFX_ERROR__SHADOW_UTCL1_ERROR__SHIFT                                                               0xe
+#define CP_GFX_ERROR__APPEND_UTCL1_ERROR__SHIFT                                                               0xf
+#define CP_GFX_ERROR__DMA_SRC_UTCL1_ERROR__SHIFT                                                              0x12
+#define CP_GFX_ERROR__DMA_DST_UTCL1_ERROR__SHIFT                                                              0x13
+#define CP_GFX_ERROR__PFP_TC_UTCL1_ERROR__SHIFT                                                               0x14
+#define CP_GFX_ERROR__ME_TC_UTCL1_ERROR__SHIFT                                                                0x15
+#define CP_GFX_ERROR__PRT_LOD_UTCL1_ERROR__SHIFT                                                              0x17
+#define CP_GFX_ERROR__RDPTR_RPT_UTCL1_ERROR__SHIFT                                                            0x18
+#define CP_GFX_ERROR__RB_FETCHER_UTCL1_ERROR__SHIFT                                                           0x19
+#define CP_GFX_ERROR__I1_FETCHER_UTCL1_ERROR__SHIFT                                                           0x1a
+#define CP_GFX_ERROR__I2_FETCHER_UTCL1_ERROR__SHIFT                                                           0x1b
+#define CP_GFX_ERROR__ST_FETCHER_UTCL1_ERROR__SHIFT                                                           0x1e
+#define CP_GFX_ERROR__RESERVED__SHIFT                                                                         0x1f
+#define CP_GFX_ERROR__ME_INSTR_CACHE_UTCL1_ERROR_MASK                                                         0x00000001L
+#define CP_GFX_ERROR__PFP_INSTR_CACHE_UTCL1_ERROR_MASK                                                        0x00000002L
+#define CP_GFX_ERROR__DDID_DRAW_UTCL1_ERROR_MASK                                                              0x00000004L
+#define CP_GFX_ERROR__DDID_DISPATCH_UTCL1_ERROR_MASK                                                          0x00000008L
+#define CP_GFX_ERROR__SUA_ERROR_MASK                                                                          0x00000010L
+#define CP_GFX_ERROR__DATA_FETCHER_UTCL1_ERROR_MASK                                                           0x00000040L
+#define CP_GFX_ERROR__QU_EOP_UTCL1_ERROR_MASK                                                                 0x00000200L
+#define CP_GFX_ERROR__QU_PIPE_UTCL1_ERROR_MASK                                                                0x00000400L
+#define CP_GFX_ERROR__QU_READ_UTCL1_ERROR_MASK                                                                0x00000800L
+#define CP_GFX_ERROR__SYNC_MEMRD_UTCL1_ERROR_MASK                                                             0x00001000L
+#define CP_GFX_ERROR__SYNC_MEMWR_UTCL1_ERROR_MASK                                                             0x00002000L
+#define CP_GFX_ERROR__SHADOW_UTCL1_ERROR_MASK                                                                 0x00004000L
+#define CP_GFX_ERROR__APPEND_UTCL1_ERROR_MASK                                                                 0x00008000L
+#define CP_GFX_ERROR__DMA_SRC_UTCL1_ERROR_MASK                                                                0x00040000L
+#define CP_GFX_ERROR__DMA_DST_UTCL1_ERROR_MASK                                                                0x00080000L
+#define CP_GFX_ERROR__PFP_TC_UTCL1_ERROR_MASK                                                                 0x00100000L
+#define CP_GFX_ERROR__ME_TC_UTCL1_ERROR_MASK                                                                  0x00200000L
+#define CP_GFX_ERROR__PRT_LOD_UTCL1_ERROR_MASK                                                                0x00800000L
+#define CP_GFX_ERROR__RDPTR_RPT_UTCL1_ERROR_MASK                                                              0x01000000L
+#define CP_GFX_ERROR__RB_FETCHER_UTCL1_ERROR_MASK                                                             0x02000000L
+#define CP_GFX_ERROR__I1_FETCHER_UTCL1_ERROR_MASK                                                             0x04000000L
+#define CP_GFX_ERROR__I2_FETCHER_UTCL1_ERROR_MASK                                                             0x08000000L
+#define CP_GFX_ERROR__ST_FETCHER_UTCL1_ERROR_MASK                                                             0x40000000L
+#define CP_GFX_ERROR__RESERVED_MASK                                                                           0x80000000L
+//CPG_UTCL1_CNTL
+#define CPG_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                           0x0
+#define CPG_UTCL1_CNTL__VMID_RESET_MODE__SHIFT                                                                0x17
+#define CPG_UTCL1_CNTL__DROP_MODE__SHIFT                                                                      0x18
+#define CPG_UTCL1_CNTL__INVALIDATE__SHIFT                                                                     0x1a
+#define CPG_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                0x1b
+#define CPG_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                    0x1c
+#define CPG_UTCL1_CNTL__IGNORE_PTE_PERMISSION__SHIFT                                                          0x1d
+#define CPG_UTCL1_CNTL__MTYPE_NO_PTE_MODE__SHIFT                                                              0x1e
+#define CPG_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                             0x000FFFFFL
+#define CPG_UTCL1_CNTL__VMID_RESET_MODE_MASK                                                                  0x00800000L
+#define CPG_UTCL1_CNTL__DROP_MODE_MASK                                                                        0x01000000L
+#define CPG_UTCL1_CNTL__INVALIDATE_MASK                                                                       0x04000000L
+#define CPG_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                  0x08000000L
+#define CPG_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                      0x10000000L
+#define CPG_UTCL1_CNTL__IGNORE_PTE_PERMISSION_MASK                                                            0x20000000L
+#define CPG_UTCL1_CNTL__MTYPE_NO_PTE_MODE_MASK                                                                0x40000000L
+//CPC_UTCL1_CNTL
+#define CPC_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                           0x0
+#define CPC_UTCL1_CNTL__EN_RETRY_ON_ATOMIC__SHIFT                                                             0x15
+#define CPC_UTCL1_CNTL__PCIE_ATOMIC_SUPPORTED__SHIFT                                                          0x16
+#define CPC_UTCL1_CNTL__DROP_MODE__SHIFT                                                                      0x18
+#define CPC_UTCL1_CNTL__INVALIDATE__SHIFT                                                                     0x1a
+#define CPC_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                0x1b
+#define CPC_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                    0x1c
+#define CPC_UTCL1_CNTL__IGNORE_PTE_PERMISSION__SHIFT                                                          0x1d
+#define CPC_UTCL1_CNTL__MTYPE_NO_PTE_MODE__SHIFT                                                              0x1e
+#define CPC_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                             0x000FFFFFL
+#define CPC_UTCL1_CNTL__EN_RETRY_ON_ATOMIC_MASK                                                               0x00200000L
+#define CPC_UTCL1_CNTL__PCIE_ATOMIC_SUPPORTED_MASK                                                            0x00400000L
+#define CPC_UTCL1_CNTL__DROP_MODE_MASK                                                                        0x01000000L
+#define CPC_UTCL1_CNTL__INVALIDATE_MASK                                                                       0x04000000L
+#define CPC_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                  0x08000000L
+#define CPC_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                      0x10000000L
+#define CPC_UTCL1_CNTL__IGNORE_PTE_PERMISSION_MASK                                                            0x20000000L
+#define CPC_UTCL1_CNTL__MTYPE_NO_PTE_MODE_MASK                                                                0x40000000L
+//CPF_UTCL1_CNTL
+#define CPF_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                           0x0
+#define CPF_UTCL1_CNTL__VMID_RESET_MODE__SHIFT                                                                0x17
+#define CPF_UTCL1_CNTL__DROP_MODE__SHIFT                                                                      0x18
+#define CPF_UTCL1_CNTL__INVALIDATE__SHIFT                                                                     0x1a
+#define CPF_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                0x1b
+#define CPF_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                    0x1c
+#define CPF_UTCL1_CNTL__IGNORE_PTE_PERMISSION__SHIFT                                                          0x1d
+#define CPF_UTCL1_CNTL__MTYPE_NO_PTE_MODE__SHIFT                                                              0x1e
+#define CPF_UTCL1_CNTL__FORCE_NO_EXE__SHIFT                                                                   0x1f
+#define CPF_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                             0x000FFFFFL
+#define CPF_UTCL1_CNTL__VMID_RESET_MODE_MASK                                                                  0x00800000L
+#define CPF_UTCL1_CNTL__DROP_MODE_MASK                                                                        0x01000000L
+#define CPF_UTCL1_CNTL__INVALIDATE_MASK                                                                       0x04000000L
+#define CPF_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                  0x08000000L
+#define CPF_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                      0x10000000L
+#define CPF_UTCL1_CNTL__IGNORE_PTE_PERMISSION_MASK                                                            0x20000000L
+#define CPF_UTCL1_CNTL__MTYPE_NO_PTE_MODE_MASK                                                                0x40000000L
+#define CPF_UTCL1_CNTL__FORCE_NO_EXE_MASK                                                                     0x80000000L
+//CP_AQL_SMM_STATUS
+#define CP_AQL_SMM_STATUS__AQL_QUEUE_SMM__SHIFT                                                               0x0
+#define CP_AQL_SMM_STATUS__AQL_QUEUE_SMM_MASK                                                                 0xFFFFFFFFL
+//CP_RB0_BASE
+#define CP_RB0_BASE__RB_BASE__SHIFT                                                                           0x0
+#define CP_RB0_BASE__RB_BASE_MASK                                                                             0xFFFFFFFFL
+//CP_RB_BASE
+#define CP_RB_BASE__RB_BASE__SHIFT                                                                            0x0
+#define CP_RB_BASE__RB_BASE_MASK                                                                              0xFFFFFFFFL
+//CP_RB0_CNTL
+#define CP_RB0_CNTL__RB_BUFSZ__SHIFT                                                                          0x0
+#define CP_RB0_CNTL__TMZ_STATE__SHIFT                                                                         0x6
+#define CP_RB0_CNTL__TMZ_MATCH__SHIFT                                                                         0x7
+#define CP_RB0_CNTL__RB_BLKSZ__SHIFT                                                                          0x8
+#define CP_RB0_CNTL__RB_NON_PRIV__SHIFT                                                                       0xf
+#define CP_RB0_CNTL__SCOPE__SHIFT                                                                             0x12
+#define CP_RB0_CNTL__MIN_AVAILSZ__SHIFT                                                                       0x14
+#define CP_RB0_CNTL__MIN_IB_AVAILSZ__SHIFT                                                                    0x16
+#define CP_RB0_CNTL__CACHE_POLICY__SHIFT                                                                      0x18
+#define CP_RB0_CNTL__RB_NO_UPDATE__SHIFT                                                                      0x1b
+#define CP_RB0_CNTL__RB_EXE__SHIFT                                                                            0x1c
+#define CP_RB0_CNTL__KMD_QUEUE__SHIFT                                                                         0x1d
+#define CP_RB0_CNTL__RB_RPTR_WR_ENA__SHIFT                                                                    0x1f
+#define CP_RB0_CNTL__RB_BUFSZ_MASK                                                                            0x0000003FL
+#define CP_RB0_CNTL__TMZ_STATE_MASK                                                                           0x00000040L
+#define CP_RB0_CNTL__TMZ_MATCH_MASK                                                                           0x00000080L
+#define CP_RB0_CNTL__RB_BLKSZ_MASK                                                                            0x00003F00L
+#define CP_RB0_CNTL__RB_NON_PRIV_MASK                                                                         0x00008000L
+#define CP_RB0_CNTL__SCOPE_MASK                                                                               0x000C0000L
+#define CP_RB0_CNTL__MIN_AVAILSZ_MASK                                                                         0x00300000L
+#define CP_RB0_CNTL__MIN_IB_AVAILSZ_MASK                                                                      0x00C00000L
+#define CP_RB0_CNTL__CACHE_POLICY_MASK                                                                        0x03000000L
+#define CP_RB0_CNTL__RB_NO_UPDATE_MASK                                                                        0x08000000L
+#define CP_RB0_CNTL__RB_EXE_MASK                                                                              0x10000000L
+#define CP_RB0_CNTL__KMD_QUEUE_MASK                                                                           0x20000000L
+#define CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK                                                                      0x80000000L
+//CP_RB_CNTL
+#define CP_RB_CNTL__RB_BUFSZ__SHIFT                                                                           0x0
+#define CP_RB_CNTL__TMZ_STATE__SHIFT                                                                          0x6
+#define CP_RB_CNTL__TMZ_MATCH__SHIFT                                                                          0x7
+#define CP_RB_CNTL__RB_BLKSZ__SHIFT                                                                           0x8
+#define CP_RB_CNTL__RB_NON_PRIV__SHIFT                                                                        0xf
+#define CP_RB_CNTL__SCOPE__SHIFT                                                                              0x12
+#define CP_RB_CNTL__MIN_AVAILSZ__SHIFT                                                                        0x14
+#define CP_RB_CNTL__MIN_IB_AVAILSZ__SHIFT                                                                     0x16
+#define CP_RB_CNTL__CACHE_POLICY__SHIFT                                                                       0x18
+#define CP_RB_CNTL__RB_NO_UPDATE__SHIFT                                                                       0x1b
+#define CP_RB_CNTL__RB_EXE__SHIFT                                                                             0x1c
+#define CP_RB_CNTL__KMD_QUEUE__SHIFT                                                                          0x1d
+#define CP_RB_CNTL__RB_RPTR_WR_ENA__SHIFT                                                                     0x1f
+#define CP_RB_CNTL__RB_BUFSZ_MASK                                                                             0x0000003FL
+#define CP_RB_CNTL__TMZ_STATE_MASK                                                                            0x00000040L
+#define CP_RB_CNTL__TMZ_MATCH_MASK                                                                            0x00000080L
+#define CP_RB_CNTL__RB_BLKSZ_MASK                                                                             0x00003F00L
+#define CP_RB_CNTL__RB_NON_PRIV_MASK                                                                          0x00008000L
+#define CP_RB_CNTL__SCOPE_MASK                                                                                0x000C0000L
+#define CP_RB_CNTL__MIN_AVAILSZ_MASK                                                                          0x00300000L
+#define CP_RB_CNTL__MIN_IB_AVAILSZ_MASK                                                                       0x00C00000L
+#define CP_RB_CNTL__CACHE_POLICY_MASK                                                                         0x03000000L
+#define CP_RB_CNTL__RB_NO_UPDATE_MASK                                                                         0x08000000L
+#define CP_RB_CNTL__RB_EXE_MASK                                                                               0x10000000L
+#define CP_RB_CNTL__KMD_QUEUE_MASK                                                                            0x20000000L
+#define CP_RB_CNTL__RB_RPTR_WR_ENA_MASK                                                                       0x80000000L
+//CP_RB_RPTR_WR
+#define CP_RB_RPTR_WR__RB_RPTR_WR__SHIFT                                                                      0x0
+#define CP_RB_RPTR_WR__RB_RPTR_WR_MASK                                                                        0x000FFFFFL
+//CP_RB0_RPTR_ADDR
+#define CP_RB0_RPTR_ADDR__RB_RPTR_ADDR__SHIFT                                                                 0x2
+#define CP_RB0_RPTR_ADDR__RB_RPTR_ADDR_MASK                                                                   0xFFFFFFFCL
+//CP_RB_RPTR_ADDR
+#define CP_RB_RPTR_ADDR__RB_RPTR_ADDR__SHIFT                                                                  0x2
+#define CP_RB_RPTR_ADDR__RB_RPTR_ADDR_MASK                                                                    0xFFFFFFFCL
+//CP_RB0_RPTR_ADDR_HI
+#define CP_RB0_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT                                                           0x0
+#define CP_RB0_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK                                                             0x0000FFFFL
+//CP_RB_RPTR_ADDR_HI
+#define CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT                                                            0x0
+#define CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK                                                              0x0000FFFFL
+//CP_RB0_BUFSZ_MASK
+#define CP_RB0_BUFSZ_MASK__DATA__SHIFT                                                                        0x0
+#define CP_RB0_BUFSZ_MASK__DATA_MASK                                                                          0x000FFFFFL
+//CP_RB_BUFSZ_MASK
+#define CP_RB_BUFSZ_MASK__DATA__SHIFT                                                                         0x0
+#define CP_RB_BUFSZ_MASK__DATA_MASK                                                                           0x000FFFFFL
+//CP_ME3_INT_STAT_DEBUG
+#define CP_ME3_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT                                                 0x1a
+#define CP_ME3_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT                                                   0x1d
+#define CP_ME3_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT                                                   0x1e
+#define CP_ME3_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT                                                   0x1f
+#define CP_ME3_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK                                                   0x04000000L
+#define CP_ME3_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK                                                     0x20000000L
+#define CP_ME3_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK                                                     0x40000000L
+#define CP_ME3_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK                                                     0x80000000L
+//GC_PRIV_MODE
+#define GC_PRIV_MODE__MC_PRIV_MODE__SHIFT                                                                     0x0
+#define GC_PRIV_MODE__MC_PRIV_MODE_MASK                                                                       0x00000001L
+//CP_INT_CNTL
+#define CP_INT_CNTL__RESUME_INT_ENABLE__SHIFT                                                                 0x8
+#define CP_INT_CNTL__SUSPEND_INT_ENABLE__SHIFT                                                                0x9
+#define CP_INT_CNTL__DMA_WATCH_INT_ENABLE__SHIFT                                                              0xa
+#define CP_INT_CNTL__CP_VM_DOORBELL_WR_INT_ENABLE__SHIFT                                                      0xb
+#define CP_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                           0xe
+#define CP_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                                    0x10
+#define CP_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                       0x11
+#define CP_INT_CNTL__CMP_BUSY_INT_ENABLE__SHIFT                                                               0x12
+#define CP_INT_CNTL__CNTX_BUSY_INT_ENABLE__SHIFT                                                              0x13
+#define CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE__SHIFT                                                             0x14
+#define CP_INT_CNTL__GFX_IDLE_INT_ENABLE__SHIFT                                                               0x15
+#define CP_INT_CNTL__PRIV_INSTR_INT_ENABLE__SHIFT                                                             0x16
+#define CP_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                               0x17
+#define CP_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                           0x18
+#define CP_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                             0x1a
+#define CP_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                                     0x1b
+#define CP_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                               0x1d
+#define CP_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                               0x1e
+#define CP_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                               0x1f
+#define CP_INT_CNTL__RESUME_INT_ENABLE_MASK                                                                   0x00000100L
+#define CP_INT_CNTL__SUSPEND_INT_ENABLE_MASK                                                                  0x00000200L
+#define CP_INT_CNTL__DMA_WATCH_INT_ENABLE_MASK                                                                0x00000400L
+#define CP_INT_CNTL__CP_VM_DOORBELL_WR_INT_ENABLE_MASK                                                        0x00000800L
+#define CP_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                             0x00004000L
+#define CP_INT_CNTL__GPF_INT_ENABLE_MASK                                                                      0x00010000L
+#define CP_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                         0x00020000L
+#define CP_INT_CNTL__CMP_BUSY_INT_ENABLE_MASK                                                                 0x00040000L
+#define CP_INT_CNTL__CNTX_BUSY_INT_ENABLE_MASK                                                                0x00080000L
+#define CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE_MASK                                                               0x00100000L
+#define CP_INT_CNTL__GFX_IDLE_INT_ENABLE_MASK                                                                 0x00200000L
+#define CP_INT_CNTL__PRIV_INSTR_INT_ENABLE_MASK                                                               0x00400000L
+#define CP_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                                 0x00800000L
+#define CP_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                             0x01000000L
+#define CP_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                               0x04000000L
+#define CP_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                       0x08000000L
+#define CP_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                                 0x20000000L
+#define CP_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                                 0x40000000L
+#define CP_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                                 0x80000000L
+//CP_INT_STATUS
+#define CP_INT_STATUS__RESUME_INT_STAT__SHIFT                                                                 0x8
+#define CP_INT_STATUS__SUSPEND_INT_STAT__SHIFT                                                                0x9
+#define CP_INT_STATUS__DMA_WATCH_INT_STAT__SHIFT                                                              0xa
+#define CP_INT_STATUS__CP_VM_DOORBELL_WR_INT_STAT__SHIFT                                                      0xb
+#define CP_INT_STATUS__CP_ECC_ERROR_INT_STAT__SHIFT                                                           0xe
+#define CP_INT_STATUS__GPF_INT_STAT__SHIFT                                                                    0x10
+#define CP_INT_STATUS__WRM_POLL_TIMEOUT_INT_STAT__SHIFT                                                       0x11
+#define CP_INT_STATUS__CMP_BUSY_INT_STAT__SHIFT                                                               0x12
+#define CP_INT_STATUS__CNTX_BUSY_INT_STAT__SHIFT                                                              0x13
+#define CP_INT_STATUS__CNTX_EMPTY_INT_STAT__SHIFT                                                             0x14
+#define CP_INT_STATUS__GFX_IDLE_INT_STAT__SHIFT                                                               0x15
+#define CP_INT_STATUS__PRIV_INSTR_INT_STAT__SHIFT                                                             0x16
+#define CP_INT_STATUS__PRIV_REG_INT_STAT__SHIFT                                                               0x17
+#define CP_INT_STATUS__OPCODE_ERROR_INT_STAT__SHIFT                                                           0x18
+#define CP_INT_STATUS__TIME_STAMP_INT_STAT__SHIFT                                                             0x1a
+#define CP_INT_STATUS__RESERVED_BIT_ERROR_INT_STAT__SHIFT                                                     0x1b
+#define CP_INT_STATUS__GENERIC2_INT_STAT__SHIFT                                                               0x1d
+#define CP_INT_STATUS__GENERIC1_INT_STAT__SHIFT                                                               0x1e
+#define CP_INT_STATUS__GENERIC0_INT_STAT__SHIFT                                                               0x1f
+#define CP_INT_STATUS__RESUME_INT_STAT_MASK                                                                   0x00000100L
+#define CP_INT_STATUS__SUSPEND_INT_STAT_MASK                                                                  0x00000200L
+#define CP_INT_STATUS__DMA_WATCH_INT_STAT_MASK                                                                0x00000400L
+#define CP_INT_STATUS__CP_VM_DOORBELL_WR_INT_STAT_MASK                                                        0x00000800L
+#define CP_INT_STATUS__CP_ECC_ERROR_INT_STAT_MASK                                                             0x00004000L
+#define CP_INT_STATUS__GPF_INT_STAT_MASK                                                                      0x00010000L
+#define CP_INT_STATUS__WRM_POLL_TIMEOUT_INT_STAT_MASK                                                         0x00020000L
+#define CP_INT_STATUS__CMP_BUSY_INT_STAT_MASK                                                                 0x00040000L
+#define CP_INT_STATUS__CNTX_BUSY_INT_STAT_MASK                                                                0x00080000L
+#define CP_INT_STATUS__CNTX_EMPTY_INT_STAT_MASK                                                               0x00100000L
+#define CP_INT_STATUS__GFX_IDLE_INT_STAT_MASK                                                                 0x00200000L
+#define CP_INT_STATUS__PRIV_INSTR_INT_STAT_MASK                                                               0x00400000L
+#define CP_INT_STATUS__PRIV_REG_INT_STAT_MASK                                                                 0x00800000L
+#define CP_INT_STATUS__OPCODE_ERROR_INT_STAT_MASK                                                             0x01000000L
+#define CP_INT_STATUS__TIME_STAMP_INT_STAT_MASK                                                               0x04000000L
+#define CP_INT_STATUS__RESERVED_BIT_ERROR_INT_STAT_MASK                                                       0x08000000L
+#define CP_INT_STATUS__GENERIC2_INT_STAT_MASK                                                                 0x20000000L
+#define CP_INT_STATUS__GENERIC1_INT_STAT_MASK                                                                 0x40000000L
+#define CP_INT_STATUS__GENERIC0_INT_STAT_MASK                                                                 0x80000000L
+//CP_DEVICE_ID
+#define CP_DEVICE_ID__DEVICE_ID__SHIFT                                                                        0x0
+#define CP_DEVICE_ID__DEVICE_ID_MASK                                                                          0x000000FFL
+//CP_ME0_PIPE_PRIORITY_CNTS
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                       0x0
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                      0x8
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                      0x10
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                       0x18
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                         0x000000FFL
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                        0x0000FF00L
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                        0x00FF0000L
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                         0xFF000000L
+//CP_RING_PRIORITY_CNTS
+#define CP_RING_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                           0x0
+#define CP_RING_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                          0x8
+#define CP_RING_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                          0x10
+#define CP_RING_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                           0x18
+#define CP_RING_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                             0x000000FFL
+#define CP_RING_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                            0x0000FF00L
+#define CP_RING_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                            0x00FF0000L
+#define CP_RING_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                             0xFF000000L
+//CP_ME0_PIPE0_PRIORITY
+#define CP_ME0_PIPE0_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME0_PIPE0_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_RING0_PRIORITY
+#define CP_RING0_PRIORITY__PRIORITY__SHIFT                                                                    0x0
+#define CP_RING0_PRIORITY__PRIORITY_MASK                                                                      0x00000003L
+//CP_FATAL_ERROR
+#define CP_FATAL_ERROR__CPF_FATAL_ERROR__SHIFT                                                                0x0
+#define CP_FATAL_ERROR__CPG_FATAL_ERROR__SHIFT                                                                0x1
+#define CP_FATAL_ERROR__GFX_HALT_PROC__SHIFT                                                                  0x2
+#define CP_FATAL_ERROR__DIS_CPG_FATAL_ERROR__SHIFT                                                            0x3
+#define CP_FATAL_ERROR__CPG_TAG_FATAL_ERROR_EN__SHIFT                                                         0x4
+#define CP_FATAL_ERROR__CPF_FATAL_ERROR_MASK                                                                  0x00000001L
+#define CP_FATAL_ERROR__CPG_FATAL_ERROR_MASK                                                                  0x00000002L
+#define CP_FATAL_ERROR__GFX_HALT_PROC_MASK                                                                    0x00000004L
+#define CP_FATAL_ERROR__DIS_CPG_FATAL_ERROR_MASK                                                              0x00000008L
+#define CP_FATAL_ERROR__CPG_TAG_FATAL_ERROR_EN_MASK                                                           0x00000010L
+//CP_RB_VMID
+#define CP_RB_VMID__RB0_VMID__SHIFT                                                                           0x0
+#define CP_RB_VMID__RB1_VMID__SHIFT                                                                           0x8
+#define CP_RB_VMID__RB2_VMID__SHIFT                                                                           0x10
+#define CP_RB_VMID__RB0_VMID_MASK                                                                             0x0000000FL
+#define CP_RB_VMID__RB1_VMID_MASK                                                                             0x00000F00L
+#define CP_RB_VMID__RB2_VMID_MASK                                                                             0x000F0000L
+//CP_ME0_PIPE0_VMID
+#define CP_ME0_PIPE0_VMID__VMID__SHIFT                                                                        0x0
+#define CP_ME0_PIPE0_VMID__VMID_MASK                                                                          0x0000000FL
+//CP_RB0_WPTR
+#define CP_RB0_WPTR__RB_WPTR__SHIFT                                                                           0x0
+#define CP_RB0_WPTR__RB_WPTR_MASK                                                                             0xFFFFFFFFL
+//CP_RB_WPTR
+#define CP_RB_WPTR__RB_WPTR__SHIFT                                                                            0x0
+#define CP_RB_WPTR__RB_WPTR_MASK                                                                              0xFFFFFFFFL
+//CP_RB0_WPTR_HI
+#define CP_RB0_WPTR_HI__RB_WPTR__SHIFT                                                                        0x0
+#define CP_RB0_WPTR_HI__RB_WPTR_MASK                                                                          0xFFFFFFFFL
+//CP_RB_WPTR_HI
+#define CP_RB_WPTR_HI__RB_WPTR__SHIFT                                                                         0x0
+#define CP_RB_WPTR_HI__RB_WPTR_MASK                                                                           0xFFFFFFFFL
+//CP_PROCESS_QUANTUM
+#define CP_PROCESS_QUANTUM__QUANTUM_DURATION__SHIFT                                                           0x0
+#define CP_PROCESS_QUANTUM__TIMER_EXPIRED__SHIFT                                                              0x1c
+#define CP_PROCESS_QUANTUM__QUANTUM_SCALE__SHIFT                                                              0x1d
+#define CP_PROCESS_QUANTUM__QUANTUM_EN__SHIFT                                                                 0x1f
+#define CP_PROCESS_QUANTUM__QUANTUM_DURATION_MASK                                                             0x0FFFFFFFL
+#define CP_PROCESS_QUANTUM__TIMER_EXPIRED_MASK                                                                0x10000000L
+#define CP_PROCESS_QUANTUM__QUANTUM_SCALE_MASK                                                                0x60000000L
+#define CP_PROCESS_QUANTUM__QUANTUM_EN_MASK                                                                   0x80000000L
+//CP_RB_DOORBELL_RANGE_LOWER
+#define CP_RB_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER__SHIFT                                               0x2
+#define CP_RB_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER_MASK                                                 0x00000FFCL
+//CP_RB_DOORBELL_RANGE_UPPER
+#define CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER__SHIFT                                               0x2
+#define CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK                                                 0x00000FFCL
+//CP_MEC_DOORBELL_RANGE_LOWER
+#define CP_MEC_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER__SHIFT                                              0x2
+#define CP_MEC_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER_MASK                                                0x00000FFCL
+//CP_MEC_DOORBELL_RANGE_UPPER
+#define CP_MEC_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER__SHIFT                                              0x2
+#define CP_MEC_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK                                                0x00000FFCL
+//CPG_UTCL1_ERROR
+#define CPG_UTCL1_ERROR__ERROR_DETECTED_HALT__SHIFT                                                           0x0
+#define CPG_UTCL1_ERROR__ERROR_DETECTED_HALT_MASK                                                             0x00000001L
+//CPC_UTCL1_ERROR
+#define CPC_UTCL1_ERROR__ERROR_DETECTED_HALT__SHIFT                                                           0x0
+#define CPC_UTCL1_ERROR__ERROR_DETECTED_HALT_MASK                                                             0x00000001L
+//CPF_FED_STATUS
+#define CPF_FED_STATUS__FED_ERROR__SHIFT                                                                      0x0
+#define CPF_FED_STATUS__FED_ERROR_ID__SHIFT                                                                   0x1
+#define CPF_FED_STATUS__FED_RING_ID__SHIFT                                                                    0x18
+#define CPF_FED_STATUS__FED_VF_ID__SHIFT                                                                      0x1a
+#define CPF_FED_STATUS__FED_VF_PF__SHIFT                                                                      0x1f
+#define CPF_FED_STATUS__FED_ERROR_MASK                                                                        0x00000001L
+#define CPF_FED_STATUS__FED_ERROR_ID_MASK                                                                     0x0000003EL
+#define CPF_FED_STATUS__FED_RING_ID_MASK                                                                      0x03000000L
+#define CPF_FED_STATUS__FED_VF_ID_MASK                                                                        0x7C000000L
+#define CPF_FED_STATUS__FED_VF_PF_MASK                                                                        0x80000000L
+//CPC_FED_STATUS
+#define CPC_FED_STATUS__FED_ERROR__SHIFT                                                                      0x0
+#define CPC_FED_STATUS__FED_ERROR_ID__SHIFT                                                                   0x1
+#define CPC_FED_STATUS__FED_RING_ID__SHIFT                                                                    0x18
+#define CPC_FED_STATUS__FED_VF_ID__SHIFT                                                                      0x1a
+#define CPC_FED_STATUS__FED_VF_PF__SHIFT                                                                      0x1f
+#define CPC_FED_STATUS__FED_ERROR_MASK                                                                        0x00000001L
+#define CPC_FED_STATUS__FED_ERROR_ID_MASK                                                                     0x0000003EL
+#define CPC_FED_STATUS__FED_RING_ID_MASK                                                                      0x03000000L
+#define CPC_FED_STATUS__FED_VF_ID_MASK                                                                        0x7C000000L
+#define CPC_FED_STATUS__FED_VF_PF_MASK                                                                        0x80000000L
+//CPG_FED_STATUS
+#define CPG_FED_STATUS__FED_ERROR__SHIFT                                                                      0x0
+#define CPG_FED_STATUS__FED_ERROR_ID__SHIFT                                                                   0x1
+#define CPG_FED_STATUS__FED_RING_ID__SHIFT                                                                    0x18
+#define CPG_FED_STATUS__FED_VF_ID__SHIFT                                                                      0x1a
+#define CPG_FED_STATUS__FED_VF_PF__SHIFT                                                                      0x1f
+#define CPG_FED_STATUS__FED_ERROR_MASK                                                                        0x00000001L
+#define CPG_FED_STATUS__FED_ERROR_ID_MASK                                                                     0x0000003EL
+#define CPG_FED_STATUS__FED_RING_ID_MASK                                                                      0x03000000L
+#define CPG_FED_STATUS__FED_VF_ID_MASK                                                                        0x7C000000L
+#define CPG_FED_STATUS__FED_VF_PF_MASK                                                                        0x80000000L
+//CP_IB1_BUFFER_COUNT
+#define CP_IB1_BUFFER_COUNT__COUNT__SHIFT                                                                     0x0
+#define CP_IB1_BUFFER_COUNT__COUNT_MASK                                                                       0x000FFFFFL
+//CP_IB2_BUFFER_COUNT
+#define CP_IB2_BUFFER_COUNT__COUNT__SHIFT                                                                     0x0
+#define CP_IB2_BUFFER_COUNT__COUNT_MASK                                                                       0x000FFFFFL
+//CP_INT_CNTL_RING0
+#define CP_INT_CNTL_RING0__RESUME_INT_ENABLE__SHIFT                                                           0x8
+#define CP_INT_CNTL_RING0__SUSPEND_INT_ENABLE__SHIFT                                                          0x9
+#define CP_INT_CNTL_RING0__DMA_WATCH_INT_ENABLE__SHIFT                                                        0xa
+#define CP_INT_CNTL_RING0__CP_VM_DOORBELL_WR_INT_ENABLE__SHIFT                                                0xb
+#define CP_INT_CNTL_RING0__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                     0xe
+#define CP_INT_CNTL_RING0__GPF_INT_ENABLE__SHIFT                                                              0x10
+#define CP_INT_CNTL_RING0__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                 0x11
+#define CP_INT_CNTL_RING0__CMP_BUSY_INT_ENABLE__SHIFT                                                         0x12
+#define CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE__SHIFT                                                        0x13
+#define CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE__SHIFT                                                       0x14
+#define CP_INT_CNTL_RING0__GFX_IDLE_INT_ENABLE__SHIFT                                                         0x15
+#define CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE__SHIFT                                                       0x16
+#define CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE__SHIFT                                                         0x17
+#define CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE__SHIFT                                                     0x18
+#define CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE__SHIFT                                                       0x1a
+#define CP_INT_CNTL_RING0__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                               0x1b
+#define CP_INT_CNTL_RING0__GENERIC2_INT_ENABLE__SHIFT                                                         0x1d
+#define CP_INT_CNTL_RING0__GENERIC1_INT_ENABLE__SHIFT                                                         0x1e
+#define CP_INT_CNTL_RING0__GENERIC0_INT_ENABLE__SHIFT                                                         0x1f
+#define CP_INT_CNTL_RING0__RESUME_INT_ENABLE_MASK                                                             0x00000100L
+#define CP_INT_CNTL_RING0__SUSPEND_INT_ENABLE_MASK                                                            0x00000200L
+#define CP_INT_CNTL_RING0__DMA_WATCH_INT_ENABLE_MASK                                                          0x00000400L
+#define CP_INT_CNTL_RING0__CP_VM_DOORBELL_WR_INT_ENABLE_MASK                                                  0x00000800L
+#define CP_INT_CNTL_RING0__CP_ECC_ERROR_INT_ENABLE_MASK                                                       0x00004000L
+#define CP_INT_CNTL_RING0__GPF_INT_ENABLE_MASK                                                                0x00010000L
+#define CP_INT_CNTL_RING0__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                   0x00020000L
+#define CP_INT_CNTL_RING0__CMP_BUSY_INT_ENABLE_MASK                                                           0x00040000L
+#define CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE_MASK                                                          0x00080000L
+#define CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE_MASK                                                         0x00100000L
+#define CP_INT_CNTL_RING0__GFX_IDLE_INT_ENABLE_MASK                                                           0x00200000L
+#define CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK                                                         0x00400000L
+#define CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK                                                           0x00800000L
+#define CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK                                                       0x01000000L
+#define CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK                                                         0x04000000L
+#define CP_INT_CNTL_RING0__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                 0x08000000L
+#define CP_INT_CNTL_RING0__GENERIC2_INT_ENABLE_MASK                                                           0x20000000L
+#define CP_INT_CNTL_RING0__GENERIC1_INT_ENABLE_MASK                                                           0x40000000L
+#define CP_INT_CNTL_RING0__GENERIC0_INT_ENABLE_MASK                                                           0x80000000L
+//CP_DEBUG_2
+#define CP_DEBUG_2__CPG_CHIU_HDM_OVERRIDE__SHIFT                                                              0x9
+#define CP_DEBUG_2__CPG_CHIU_TEE_OVERRIDE__SHIFT                                                              0xa
+#define CP_DEBUG_2__HEADER_TRAP_DIS__SHIFT                                                                    0xb
+#define CP_DEBUG_2__RB_PACKET_INJECTOR_DISABLE__SHIFT                                                         0xe
+#define CP_DEBUG_2__CNTX_DONE_COPY_STATE_DISABLE__SHIFT                                                       0xf
+#define CP_DEBUG_2__NOP_DISCARD_DISABLE__SHIFT                                                                0x10
+#define CP_DEBUG_2__DC_INTERLEAVE_DISABLE__SHIFT                                                              0x11
+#define CP_DEBUG_2__BC_LOOKUP_CB_DB_FLUSH_DISABLE__SHIFT                                                      0x1b
+#define CP_DEBUG_2__DC_FORCE_CLK_EN__SHIFT                                                                    0x1c
+#define CP_DEBUG_2__DC_DISABLE_BROADCAST__SHIFT                                                               0x1d
+#define CP_DEBUG_2__NOT_EOP_HW_DETECT_DISABLE__SHIFT                                                          0x1e
+#define CP_DEBUG_2__PFP_DDID_HW_DETECT_DISABLE__SHIFT                                                         0x1f
+#define CP_DEBUG_2__CPG_CHIU_HDM_OVERRIDE_MASK                                                                0x00000200L
+#define CP_DEBUG_2__CPG_CHIU_TEE_OVERRIDE_MASK                                                                0x00000400L
+#define CP_DEBUG_2__HEADER_TRAP_DIS_MASK                                                                      0x00000800L
+#define CP_DEBUG_2__RB_PACKET_INJECTOR_DISABLE_MASK                                                           0x00004000L
+#define CP_DEBUG_2__CNTX_DONE_COPY_STATE_DISABLE_MASK                                                         0x00008000L
+#define CP_DEBUG_2__NOP_DISCARD_DISABLE_MASK                                                                  0x00010000L
+#define CP_DEBUG_2__DC_INTERLEAVE_DISABLE_MASK                                                                0x00020000L
+#define CP_DEBUG_2__BC_LOOKUP_CB_DB_FLUSH_DISABLE_MASK                                                        0x08000000L
+#define CP_DEBUG_2__DC_FORCE_CLK_EN_MASK                                                                      0x10000000L
+#define CP_DEBUG_2__DC_DISABLE_BROADCAST_MASK                                                                 0x20000000L
+#define CP_DEBUG_2__NOT_EOP_HW_DETECT_DISABLE_MASK                                                            0x40000000L
+#define CP_DEBUG_2__PFP_DDID_HW_DETECT_DISABLE_MASK                                                           0x80000000L
+//CP_INT_STATUS_RING0
+#define CP_INT_STATUS_RING0__RESUME_INT_STAT__SHIFT                                                           0x8
+#define CP_INT_STATUS_RING0__SUSPEND_INT_STAT__SHIFT                                                          0x9
+#define CP_INT_STATUS_RING0__DMA_WATCH_INT_STAT__SHIFT                                                        0xa
+#define CP_INT_STATUS_RING0__CP_VM_DOORBELL_WR_INT_STAT__SHIFT                                                0xb
+#define CP_INT_STATUS_RING0__CP_ECC_ERROR_INT_STAT__SHIFT                                                     0xe
+#define CP_INT_STATUS_RING0__GPF_INT_STAT__SHIFT                                                              0x10
+#define CP_INT_STATUS_RING0__WRM_POLL_TIMEOUT_INT_STAT__SHIFT                                                 0x11
+#define CP_INT_STATUS_RING0__CMP_BUSY_INT_STAT__SHIFT                                                         0x12
+#define CP_INT_STATUS_RING0__GCNTX_BUSY_INT_STAT__SHIFT                                                       0x13
+#define CP_INT_STATUS_RING0__CNTX_EMPTY_INT_STAT__SHIFT                                                       0x14
+#define CP_INT_STATUS_RING0__GFX_IDLE_INT_STAT__SHIFT                                                         0x15
+#define CP_INT_STATUS_RING0__PRIV_INSTR_INT_STAT__SHIFT                                                       0x16
+#define CP_INT_STATUS_RING0__PRIV_REG_INT_STAT__SHIFT                                                         0x17
+#define CP_INT_STATUS_RING0__OPCODE_ERROR_INT_STAT__SHIFT                                                     0x18
+#define CP_INT_STATUS_RING0__TIME_STAMP_INT_STAT__SHIFT                                                       0x1a
+#define CP_INT_STATUS_RING0__RESERVED_BIT_ERROR_INT_STAT__SHIFT                                               0x1b
+#define CP_INT_STATUS_RING0__GENERIC2_INT_STAT__SHIFT                                                         0x1d
+#define CP_INT_STATUS_RING0__GENERIC1_INT_STAT__SHIFT                                                         0x1e
+#define CP_INT_STATUS_RING0__GENERIC0_INT_STAT__SHIFT                                                         0x1f
+#define CP_INT_STATUS_RING0__RESUME_INT_STAT_MASK                                                             0x00000100L
+#define CP_INT_STATUS_RING0__SUSPEND_INT_STAT_MASK                                                            0x00000200L
+#define CP_INT_STATUS_RING0__DMA_WATCH_INT_STAT_MASK                                                          0x00000400L
+#define CP_INT_STATUS_RING0__CP_VM_DOORBELL_WR_INT_STAT_MASK                                                  0x00000800L
+#define CP_INT_STATUS_RING0__CP_ECC_ERROR_INT_STAT_MASK                                                       0x00004000L
+#define CP_INT_STATUS_RING0__GPF_INT_STAT_MASK                                                                0x00010000L
+#define CP_INT_STATUS_RING0__WRM_POLL_TIMEOUT_INT_STAT_MASK                                                   0x00020000L
+#define CP_INT_STATUS_RING0__CMP_BUSY_INT_STAT_MASK                                                           0x00040000L
+#define CP_INT_STATUS_RING0__GCNTX_BUSY_INT_STAT_MASK                                                         0x00080000L
+#define CP_INT_STATUS_RING0__CNTX_EMPTY_INT_STAT_MASK                                                         0x00100000L
+#define CP_INT_STATUS_RING0__GFX_IDLE_INT_STAT_MASK                                                           0x00200000L
+#define CP_INT_STATUS_RING0__PRIV_INSTR_INT_STAT_MASK                                                         0x00400000L
+#define CP_INT_STATUS_RING0__PRIV_REG_INT_STAT_MASK                                                           0x00800000L
+#define CP_INT_STATUS_RING0__OPCODE_ERROR_INT_STAT_MASK                                                       0x01000000L
+#define CP_INT_STATUS_RING0__TIME_STAMP_INT_STAT_MASK                                                         0x04000000L
+#define CP_INT_STATUS_RING0__RESERVED_BIT_ERROR_INT_STAT_MASK                                                 0x08000000L
+#define CP_INT_STATUS_RING0__GENERIC2_INT_STAT_MASK                                                           0x20000000L
+#define CP_INT_STATUS_RING0__GENERIC1_INT_STAT_MASK                                                           0x40000000L
+#define CP_INT_STATUS_RING0__GENERIC0_INT_STAT_MASK                                                           0x80000000L
+//CP_ME_F32_INTERRUPT
+#define CP_ME_F32_INTERRUPT__ECC_ERROR_INT__SHIFT                                                             0x0
+#define CP_ME_F32_INTERRUPT__TIME_STAMP_INT__SHIFT                                                            0x1
+#define CP_ME_F32_INTERRUPT__ME_F32_INT_2__SHIFT                                                              0x2
+#define CP_ME_F32_INTERRUPT__ME_F32_INT_3__SHIFT                                                              0x3
+#define CP_ME_F32_INTERRUPT__ECC_ERROR_INT_MASK                                                               0x00000001L
+#define CP_ME_F32_INTERRUPT__TIME_STAMP_INT_MASK                                                              0x00000002L
+#define CP_ME_F32_INTERRUPT__ME_F32_INT_2_MASK                                                                0x00000004L
+#define CP_ME_F32_INTERRUPT__ME_F32_INT_3_MASK                                                                0x00000008L
+//CP_PFP_F32_INTERRUPT
+#define CP_PFP_F32_INTERRUPT__ECC_ERROR_INT__SHIFT                                                            0x0
+#define CP_PFP_F32_INTERRUPT__PRIV_REG_INT__SHIFT                                                             0x1
+#define CP_PFP_F32_INTERRUPT__RESERVED_BIT_ERR_INT__SHIFT                                                     0x2
+#define CP_PFP_F32_INTERRUPT__PFP_F32_INT_3__SHIFT                                                            0x3
+#define CP_PFP_F32_INTERRUPT__ECC_ERROR_INT_MASK                                                              0x00000001L
+#define CP_PFP_F32_INTERRUPT__PRIV_REG_INT_MASK                                                               0x00000002L
+#define CP_PFP_F32_INTERRUPT__RESERVED_BIT_ERR_INT_MASK                                                       0x00000004L
+#define CP_PFP_F32_INTERRUPT__PFP_F32_INT_3_MASK                                                              0x00000008L
+//CP_MEC1_F32_INTERRUPT
+#define CP_MEC1_F32_INTERRUPT__EDC_ROQ_FED_INT__SHIFT                                                         0x0
+#define CP_MEC1_F32_INTERRUPT__PRIV_REG_INT__SHIFT                                                            0x1
+#define CP_MEC1_F32_INTERRUPT__RESERVED_BIT_ERR_INT__SHIFT                                                    0x2
+#define CP_MEC1_F32_INTERRUPT__EDC_TC_FED_INT__SHIFT                                                          0x3
+#define CP_MEC1_F32_INTERRUPT__EDC_GDS_FED_INT__SHIFT                                                         0x4
+#define CP_MEC1_F32_INTERRUPT__EDC_SCRATCH_FED_INT__SHIFT                                                     0x5
+#define CP_MEC1_F32_INTERRUPT__WAVE_RESTORE_INT__SHIFT                                                        0x6
+#define CP_MEC1_F32_INTERRUPT__SUA_VIOLATION_INT__SHIFT                                                       0x7
+#define CP_MEC1_F32_INTERRUPT__EDC_DMA_FED_INT__SHIFT                                                         0x8
+#define CP_MEC1_F32_INTERRUPT__IQ_TIMER_INT__SHIFT                                                            0x9
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_CPF__SHIFT                                                             0xa
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_DMA__SHIFT                                                             0xb
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_CPC__SHIFT                                                             0xc
+#define CP_MEC1_F32_INTERRUPT__EDC_SR_MEM_FED_INT__SHIFT                                                      0xd
+#define CP_MEC1_F32_INTERRUPT__QUEUE_MESSAGE_INT__SHIFT                                                       0xe
+#define CP_MEC1_F32_INTERRUPT__FATAL_EDC_ERROR_INT__SHIFT                                                     0xf
+#define CP_MEC1_F32_INTERRUPT__EDC_ROQ_FED_INT_MASK                                                           0x00000001L
+#define CP_MEC1_F32_INTERRUPT__PRIV_REG_INT_MASK                                                              0x00000002L
+#define CP_MEC1_F32_INTERRUPT__RESERVED_BIT_ERR_INT_MASK                                                      0x00000004L
+#define CP_MEC1_F32_INTERRUPT__EDC_TC_FED_INT_MASK                                                            0x00000008L
+#define CP_MEC1_F32_INTERRUPT__EDC_GDS_FED_INT_MASK                                                           0x00000010L
+#define CP_MEC1_F32_INTERRUPT__EDC_SCRATCH_FED_INT_MASK                                                       0x00000020L
+#define CP_MEC1_F32_INTERRUPT__WAVE_RESTORE_INT_MASK                                                          0x00000040L
+#define CP_MEC1_F32_INTERRUPT__SUA_VIOLATION_INT_MASK                                                         0x00000080L
+#define CP_MEC1_F32_INTERRUPT__EDC_DMA_FED_INT_MASK                                                           0x00000100L
+#define CP_MEC1_F32_INTERRUPT__IQ_TIMER_INT_MASK                                                              0x00000200L
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_CPF_MASK                                                               0x00000400L
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_DMA_MASK                                                               0x00000800L
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_CPC_MASK                                                               0x00001000L
+#define CP_MEC1_F32_INTERRUPT__EDC_SR_MEM_FED_INT_MASK                                                        0x00002000L
+#define CP_MEC1_F32_INTERRUPT__QUEUE_MESSAGE_INT_MASK                                                         0x00004000L
+#define CP_MEC1_F32_INTERRUPT__FATAL_EDC_ERROR_INT_MASK                                                       0x00008000L
+//CP_PWR_CNTL
+#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE0__SHIFT                                                            0x0
+#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE1__SHIFT                                                            0x1
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE0__SHIFT                                                            0x8
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE1__SHIFT                                                            0x9
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE2__SHIFT                                                            0xa
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE3__SHIFT                                                            0xb
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE0__SHIFT                                                            0x10
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE1__SHIFT                                                            0x11
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE2__SHIFT                                                            0x12
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE3__SHIFT                                                            0x13
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME3_PIPE0__SHIFT                                                            0x14
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME3_PIPE1__SHIFT                                                            0x15
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME3_PIPE2__SHIFT                                                            0x16
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME3_PIPE3__SHIFT                                                            0x17
+#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE0_MASK                                                              0x00000001L
+#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE1_MASK                                                              0x00000002L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE0_MASK                                                              0x00000100L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE1_MASK                                                              0x00000200L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE2_MASK                                                              0x00000400L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE3_MASK                                                              0x00000800L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE0_MASK                                                              0x00010000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE1_MASK                                                              0x00020000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE2_MASK                                                              0x00040000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE3_MASK                                                              0x00080000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME3_PIPE0_MASK                                                              0x00100000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME3_PIPE1_MASK                                                              0x00200000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME3_PIPE2_MASK                                                              0x00400000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME3_PIPE3_MASK                                                              0x00800000L
+//CP_ECC_FIRSTOCCURRENCE
+#define CP_ECC_FIRSTOCCURRENCE__INTERFACE__SHIFT                                                              0x0
+#define CP_ECC_FIRSTOCCURRENCE__CLIENT__SHIFT                                                                 0x4
+#define CP_ECC_FIRSTOCCURRENCE__ME__SHIFT                                                                     0x8
+#define CP_ECC_FIRSTOCCURRENCE__PIPE__SHIFT                                                                   0xa
+#define CP_ECC_FIRSTOCCURRENCE__VMID__SHIFT                                                                   0x10
+#define CP_ECC_FIRSTOCCURRENCE__INTERFACE_MASK                                                                0x00000003L
+#define CP_ECC_FIRSTOCCURRENCE__CLIENT_MASK                                                                   0x000000F0L
+#define CP_ECC_FIRSTOCCURRENCE__ME_MASK                                                                       0x00000300L
+#define CP_ECC_FIRSTOCCURRENCE__PIPE_MASK                                                                     0x00000C00L
+#define CP_ECC_FIRSTOCCURRENCE__VMID_MASK                                                                     0x000F0000L
+//CP_ECC_FIRSTOCCURRENCE_RING0
+#define CP_ECC_FIRSTOCCURRENCE_RING0__OBSOLETE__SHIFT                                                         0x0
+#define CP_ECC_FIRSTOCCURRENCE_RING0__OBSOLETE_MASK                                                           0xFFFFFFFFL
+//GB_EDC_MODE
+#define GB_EDC_MODE__FORCE_SEC_ON_DED__SHIFT                                                                  0xf
+#define GB_EDC_MODE__COUNT_FED_OUT__SHIFT                                                                     0x10
+#define GB_EDC_MODE__GATE_FUE__SHIFT                                                                          0x11
+#define GB_EDC_MODE__DED_MODE__SHIFT                                                                          0x14
+#define GB_EDC_MODE__PROP_FED__SHIFT                                                                          0x1d
+#define GB_EDC_MODE__BYPASS__SHIFT                                                                            0x1f
+#define GB_EDC_MODE__FORCE_SEC_ON_DED_MASK                                                                    0x00008000L
+#define GB_EDC_MODE__COUNT_FED_OUT_MASK                                                                       0x00010000L
+#define GB_EDC_MODE__GATE_FUE_MASK                                                                            0x00020000L
+#define GB_EDC_MODE__DED_MODE_MASK                                                                            0x00300000L
+#define GB_EDC_MODE__PROP_FED_MASK                                                                            0x20000000L
+#define GB_EDC_MODE__BYPASS_MASK                                                                              0x80000000L
+//CP_DEBUG
+#define CP_DEBUG__PERFMON_RING_SEL__SHIFT                                                                     0x0
+#define CP_DEBUG__DEBUG_BUS_SELECT_BITS__SHIFT                                                                0x2
+#define CP_DEBUG__DEBUG_BUS_FLOP_EN__SHIFT                                                                    0x8
+#define CP_DEBUG__CPG_REPEATER_FGCG_OVERRIDE__SHIFT                                                           0x9
+#define CP_DEBUG__PACKET_FILTER_DISABLE__SHIFT                                                                0xa
+#define CP_DEBUG__NOT_EOP_PREEMPT_DISABLE__SHIFT                                                              0xb
+#define CP_DEBUG__CPG_CHIU_RO_DISABLE__SHIFT                                                                  0xc
+#define CP_DEBUG__CPG_GCR_CNTL_BYPASS__SHIFT                                                                  0xd
+#define CP_DEBUG__CPG_RAM_CLK_GATING_DISABLE__SHIFT                                                           0xe
+#define CP_DEBUG__CPG_UTCL1_ERROR_HALT_DISABLE__SHIFT                                                         0xf
+#define CP_DEBUG__SURFSYNC_CNTX_RDADDR__SHIFT                                                                 0x10
+#define CP_DEBUG__CPG_DATA_POISONING_INT_DISABLE__SHIFT                                                       0x13
+#define CP_DEBUG__PRIV_VIOLATION_WRITE_DISABLE__SHIFT                                                         0x14
+#define CP_DEBUG__INTERRUPT_DISABLE__SHIFT                                                                    0x16
+#define CP_DEBUG__PREDICATE_DISABLE__SHIFT                                                                    0x17
+#define CP_DEBUG__UNDERFLOW_BUSY_DISABLE__SHIFT                                                               0x18
+#define CP_DEBUG__OVERFLOW_BUSY_DISABLE__SHIFT                                                                0x19
+#define CP_DEBUG__EVENT_FILT_DISABLE__SHIFT                                                                   0x1a
+#define CP_DEBUG__CPG_CHIU_MTYPE_OVERRIDE__SHIFT                                                              0x1b
+#define CP_DEBUG__CPG_TC_ONE_CYCLE_WRITE_DISABLE__SHIFT                                                       0x1c
+#define CP_DEBUG__CS_STATE_FILT_DISABLE__SHIFT                                                                0x1d
+#define CP_DEBUG__CS_PIPELINE_RESET_DISABLE__SHIFT                                                            0x1e
+#define CP_DEBUG__IB_PACKET_INJECTOR_DISABLE__SHIFT                                                           0x1f
+#define CP_DEBUG__PERFMON_RING_SEL_MASK                                                                       0x00000003L
+#define CP_DEBUG__DEBUG_BUS_SELECT_BITS_MASK                                                                  0x000000FCL
+#define CP_DEBUG__DEBUG_BUS_FLOP_EN_MASK                                                                      0x00000100L
+#define CP_DEBUG__CPG_REPEATER_FGCG_OVERRIDE_MASK                                                             0x00000200L
+#define CP_DEBUG__PACKET_FILTER_DISABLE_MASK                                                                  0x00000400L
+#define CP_DEBUG__NOT_EOP_PREEMPT_DISABLE_MASK                                                                0x00000800L
+#define CP_DEBUG__CPG_CHIU_RO_DISABLE_MASK                                                                    0x00001000L
+#define CP_DEBUG__CPG_GCR_CNTL_BYPASS_MASK                                                                    0x00002000L
+#define CP_DEBUG__CPG_RAM_CLK_GATING_DISABLE_MASK                                                             0x00004000L
+#define CP_DEBUG__CPG_UTCL1_ERROR_HALT_DISABLE_MASK                                                           0x00008000L
+#define CP_DEBUG__SURFSYNC_CNTX_RDADDR_MASK                                                                   0x00070000L
+#define CP_DEBUG__CPG_DATA_POISONING_INT_DISABLE_MASK                                                         0x00080000L
+#define CP_DEBUG__PRIV_VIOLATION_WRITE_DISABLE_MASK                                                           0x00100000L
+#define CP_DEBUG__INTERRUPT_DISABLE_MASK                                                                      0x00400000L
+#define CP_DEBUG__PREDICATE_DISABLE_MASK                                                                      0x00800000L
+#define CP_DEBUG__UNDERFLOW_BUSY_DISABLE_MASK                                                                 0x01000000L
+#define CP_DEBUG__OVERFLOW_BUSY_DISABLE_MASK                                                                  0x02000000L
+#define CP_DEBUG__EVENT_FILT_DISABLE_MASK                                                                     0x04000000L
+#define CP_DEBUG__CPG_CHIU_MTYPE_OVERRIDE_MASK                                                                0x08000000L
+#define CP_DEBUG__CPG_TC_ONE_CYCLE_WRITE_DISABLE_MASK                                                         0x10000000L
+#define CP_DEBUG__CS_STATE_FILT_DISABLE_MASK                                                                  0x20000000L
+#define CP_DEBUG__CS_PIPELINE_RESET_DISABLE_MASK                                                              0x40000000L
+#define CP_DEBUG__IB_PACKET_INJECTOR_DISABLE_MASK                                                             0x80000000L
+//CP_CPF_DEBUG
+#define CP_CPF_DEBUG__PRIVATE_REG_ACC_DISABLE__SHIFT                                                          0x6
+#define CP_CPF_DEBUG__CPF_CHIU_TEE_OVERRIDE__SHIFT                                                            0x7
+#define CP_CPF_DEBUG__CPF_CHIU_HDM_OVERRIDE__SHIFT                                                            0xd
+#define CP_CPF_DEBUG__DEBUG_BUS_FLOP_EN__SHIFT                                                                0xe
+#define CP_CPF_DEBUG__MES_DOORBELL_HIT_BUSY_OVERRIDE__SHIFT                                                   0xf
+#define CP_CPF_DEBUG__CPF_REPEATER_FGCG_OVERRIDE__SHIFT                                                       0x10
+#define CP_CPF_DEBUG__CPF_GCR_CNTL_BYPASS__SHIFT                                                              0x11
+#define CP_CPF_DEBUG__CPF_RAM_CLK_GATING_DISABLE__SHIFT                                                       0x12
+#define CP_CPF_DEBUG__CPF_DATA_POISONING_INT_DISABLE__SHIFT                                                   0x13
+#define CP_CPF_DEBUG__CLOCK_ACTIVE_DELAY_OVERRIDE__SHIFT                                                      0x16
+#define CP_CPF_DEBUG__CLOCK_ACTIVE_OVERRIDE__SHIFT                                                            0x17
+#define CP_CPF_DEBUG__UNDERFLOW_BUSY_DISABLE__SHIFT                                                           0x18
+#define CP_CPF_DEBUG__OVERFLOW_BUSY_DISABLE__SHIFT                                                            0x19
+#define CP_CPF_DEBUG__CE_FETCHER_DISABLE__SHIFT                                                               0x1b
+#define CP_CPF_DEBUG__CPF_PRIORITY_YIELD_ACTIVE_DIS__SHIFT                                                    0x1d
+#define CP_CPF_DEBUG__CPF_CHIU_MTYPE_OVERRIDE__SHIFT                                                          0x1e
+#define CP_CPF_DEBUG__DBGU_TRIGGER__SHIFT                                                                     0x1f
+#define CP_CPF_DEBUG__PRIVATE_REG_ACC_DISABLE_MASK                                                            0x00000040L
+#define CP_CPF_DEBUG__CPF_CHIU_TEE_OVERRIDE_MASK                                                              0x00000080L
+#define CP_CPF_DEBUG__CPF_CHIU_HDM_OVERRIDE_MASK                                                              0x00002000L
+#define CP_CPF_DEBUG__DEBUG_BUS_FLOP_EN_MASK                                                                  0x00004000L
+#define CP_CPF_DEBUG__MES_DOORBELL_HIT_BUSY_OVERRIDE_MASK                                                     0x00008000L
+#define CP_CPF_DEBUG__CPF_REPEATER_FGCG_OVERRIDE_MASK                                                         0x00010000L
+#define CP_CPF_DEBUG__CPF_GCR_CNTL_BYPASS_MASK                                                                0x00020000L
+#define CP_CPF_DEBUG__CPF_RAM_CLK_GATING_DISABLE_MASK                                                         0x00040000L
+#define CP_CPF_DEBUG__CPF_DATA_POISONING_INT_DISABLE_MASK                                                     0x00080000L
+#define CP_CPF_DEBUG__CLOCK_ACTIVE_DELAY_OVERRIDE_MASK                                                        0x00400000L
+#define CP_CPF_DEBUG__CLOCK_ACTIVE_OVERRIDE_MASK                                                              0x00800000L
+#define CP_CPF_DEBUG__UNDERFLOW_BUSY_DISABLE_MASK                                                             0x01000000L
+#define CP_CPF_DEBUG__OVERFLOW_BUSY_DISABLE_MASK                                                              0x02000000L
+#define CP_CPF_DEBUG__CE_FETCHER_DISABLE_MASK                                                                 0x08000000L
+#define CP_CPF_DEBUG__CPF_PRIORITY_YIELD_ACTIVE_DIS_MASK                                                      0x20000000L
+#define CP_CPF_DEBUG__CPF_CHIU_MTYPE_OVERRIDE_MASK                                                            0x40000000L
+#define CP_CPF_DEBUG__DBGU_TRIGGER_MASK                                                                       0x80000000L
+//CP_CPC_DEBUG
+#define CP_CPC_DEBUG__PIPE_SELECT__SHIFT                                                                      0x0
+#define CP_CPC_DEBUG__ME_SELECT__SHIFT                                                                        0x2
+#define CP_CPC_DEBUG__CPC_CHIU_TEE_OVERRIDE__SHIFT                                                            0x3
+#define CP_CPC_DEBUG__ADC_INTERLEAVE_DISABLE__SHIFT                                                           0x4
+#define CP_CPC_DEBUG__CPC_CHIU_HDM_OVERRIDE__SHIFT                                                            0x5
+#define CP_CPC_DEBUG__DISP_MODE1_FAST_SKIP_DISABLE__SHIFT                                                     0x6
+#define CP_CPC_DEBUG__DISABLE_PER_QUEUE_ADC_STATE_STORE__SHIFT                                                0xb
+#define CP_CPC_DEBUG__ENABLE_CONF_CS_DIST__SHIFT                                                              0xd
+#define CP_CPC_DEBUG__DEBUG_BUS_FLOP_EN__SHIFT                                                                0xe
+#define CP_CPC_DEBUG__CPC_REPEATER_FGCG_OVERRIDE__SHIFT                                                       0xf
+#define CP_CPC_DEBUG__CPC_GCR_CNTL_BYPASS__SHIFT                                                              0x11
+#define CP_CPC_DEBUG__CPC_RAM_CLK_GATING_DISABLE__SHIFT                                                       0x12
+#define CP_CPC_DEBUG__CPC_DATA_POISONING_INT_DISABLE__SHIFT                                                   0x13
+#define CP_CPC_DEBUG__PRIV_VIOLATION_WRITE_DISABLE__SHIFT                                                     0x14
+#define CP_CPC_DEBUG__UCODE_ECC_ERROR_DISABLE__SHIFT                                                          0x15
+#define CP_CPC_DEBUG__INTERRUPT_DISABLE__SHIFT                                                                0x16
+#define CP_CPC_DEBUG__CPC_CHIU_RO_DISABLE__SHIFT                                                              0x17
+#define CP_CPC_DEBUG__UNDERFLOW_BUSY_DISABLE__SHIFT                                                           0x18
+#define CP_CPC_DEBUG__OVERFLOW_BUSY_DISABLE__SHIFT                                                            0x19
+#define CP_CPC_DEBUG__EVENT_FILT_DISABLE__SHIFT                                                               0x1a
+#define CP_CPC_DEBUG__CPC_TC_ONE_CYCLE_WRITE_DISABLE__SHIFT                                                   0x1c
+#define CP_CPC_DEBUG__CS_STATE_FILT_DISABLE__SHIFT                                                            0x1d
+#define CP_CPC_DEBUG__CPC_CHIU_MTYPE_OVERRIDE__SHIFT                                                          0x1e
+#define CP_CPC_DEBUG__ME2_UCODE_RAM_ENABLE__SHIFT                                                             0x1f
+#define CP_CPC_DEBUG__PIPE_SELECT_MASK                                                                        0x00000003L
+#define CP_CPC_DEBUG__ME_SELECT_MASK                                                                          0x00000004L
+#define CP_CPC_DEBUG__CPC_CHIU_TEE_OVERRIDE_MASK                                                              0x00000008L
+#define CP_CPC_DEBUG__ADC_INTERLEAVE_DISABLE_MASK                                                             0x00000010L
+#define CP_CPC_DEBUG__CPC_CHIU_HDM_OVERRIDE_MASK                                                              0x00000020L
+#define CP_CPC_DEBUG__DISP_MODE1_FAST_SKIP_DISABLE_MASK                                                       0x00000040L
+#define CP_CPC_DEBUG__DISABLE_PER_QUEUE_ADC_STATE_STORE_MASK                                                  0x00000800L
+#define CP_CPC_DEBUG__ENABLE_CONF_CS_DIST_MASK                                                                0x00002000L
+#define CP_CPC_DEBUG__DEBUG_BUS_FLOP_EN_MASK                                                                  0x00004000L
+#define CP_CPC_DEBUG__CPC_REPEATER_FGCG_OVERRIDE_MASK                                                         0x00008000L
+#define CP_CPC_DEBUG__CPC_GCR_CNTL_BYPASS_MASK                                                                0x00020000L
+#define CP_CPC_DEBUG__CPC_RAM_CLK_GATING_DISABLE_MASK                                                         0x00040000L
+#define CP_CPC_DEBUG__CPC_DATA_POISONING_INT_DISABLE_MASK                                                     0x00080000L
+#define CP_CPC_DEBUG__PRIV_VIOLATION_WRITE_DISABLE_MASK                                                       0x00100000L
+#define CP_CPC_DEBUG__UCODE_ECC_ERROR_DISABLE_MASK                                                            0x00200000L
+#define CP_CPC_DEBUG__INTERRUPT_DISABLE_MASK                                                                  0x00400000L
+#define CP_CPC_DEBUG__CPC_CHIU_RO_DISABLE_MASK                                                                0x00800000L
+#define CP_CPC_DEBUG__UNDERFLOW_BUSY_DISABLE_MASK                                                             0x01000000L
+#define CP_CPC_DEBUG__OVERFLOW_BUSY_DISABLE_MASK                                                              0x02000000L
+#define CP_CPC_DEBUG__EVENT_FILT_DISABLE_MASK                                                                 0x04000000L
+#define CP_CPC_DEBUG__CPC_TC_ONE_CYCLE_WRITE_DISABLE_MASK                                                     0x10000000L
+#define CP_CPC_DEBUG__CS_STATE_FILT_DISABLE_MASK                                                              0x20000000L
+#define CP_CPC_DEBUG__CPC_CHIU_MTYPE_OVERRIDE_MASK                                                            0x40000000L
+#define CP_CPC_DEBUG__ME2_UCODE_RAM_ENABLE_MASK                                                               0x80000000L
+//CP_PQ_WPTR_POLL_CNTL
+#define CP_PQ_WPTR_POLL_CNTL__PERIOD__SHIFT                                                                   0x0
+#define CP_PQ_WPTR_POLL_CNTL__DISABLE_PEND_REQ_ONE_SHOT__SHIFT                                                0x1d
+#define CP_PQ_WPTR_POLL_CNTL__POLL_ACTIVE__SHIFT                                                              0x1e
+#define CP_PQ_WPTR_POLL_CNTL__EN__SHIFT                                                                       0x1f
+#define CP_PQ_WPTR_POLL_CNTL__PERIOD_MASK                                                                     0x000000FFL
+#define CP_PQ_WPTR_POLL_CNTL__DISABLE_PEND_REQ_ONE_SHOT_MASK                                                  0x20000000L
+#define CP_PQ_WPTR_POLL_CNTL__POLL_ACTIVE_MASK                                                                0x40000000L
+#define CP_PQ_WPTR_POLL_CNTL__EN_MASK                                                                         0x80000000L
+//CP_PQ_WPTR_POLL_CNTL1
+#define CP_PQ_WPTR_POLL_CNTL1__QUEUE_MASK__SHIFT                                                              0x0
+#define CP_PQ_WPTR_POLL_CNTL1__QUEUE_MASK_MASK                                                                0xFFFFFFFFL
+//CP_ME1_PIPE0_INT_CNTL
+#define CP_ME1_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME1_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME1_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME1_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME1_PIPE0_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME1_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME1_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME1_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME1_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME1_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME1_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME1_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME1_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME1_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME1_PIPE0_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME1_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME1_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME1_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME1_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME1_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME1_PIPE1_INT_CNTL
+#define CP_ME1_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME1_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME1_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME1_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME1_PIPE1_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME1_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME1_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME1_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME1_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME1_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME1_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME1_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME1_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME1_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME1_PIPE1_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME1_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME1_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME1_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME1_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME1_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME1_PIPE2_INT_CNTL
+#define CP_ME1_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME1_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME1_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME1_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME1_PIPE2_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME1_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME1_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME1_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME1_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME1_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME1_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME1_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME1_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME1_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME1_PIPE2_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME1_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME1_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME1_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME1_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME1_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME1_PIPE3_INT_CNTL
+#define CP_ME1_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME1_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME1_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME1_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME1_PIPE3_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME1_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME1_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME1_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME1_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME1_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME1_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME1_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME1_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME1_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME1_PIPE3_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME1_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME1_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME1_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME1_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME1_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME1_PIPE0_INT_STATUS
+#define CP_ME1_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME1_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME1_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME1_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME1_PIPE0_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME1_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME1_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME1_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME1_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME1_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME1_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME1_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME1_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME1_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME1_PIPE0_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME1_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME1_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME1_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME1_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME1_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME1_PIPE1_INT_STATUS
+#define CP_ME1_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME1_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME1_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME1_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME1_PIPE1_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME1_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME1_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME1_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME1_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME1_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME1_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME1_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME1_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME1_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME1_PIPE1_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME1_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME1_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME1_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME1_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME1_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME1_PIPE2_INT_STATUS
+#define CP_ME1_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME1_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME1_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME1_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME1_PIPE2_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME1_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME1_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME1_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME1_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME1_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME1_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME1_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME1_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME1_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME1_PIPE2_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME1_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME1_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME1_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME1_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME1_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME1_PIPE3_INT_STATUS
+#define CP_ME1_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME1_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME1_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME1_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME1_PIPE3_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME1_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME1_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME1_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME1_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME1_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME1_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME1_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME1_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME1_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME1_PIPE3_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME1_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME1_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME1_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME1_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME1_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME1_INT_STAT_DEBUG
+#define CP_ME1_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED__SHIFT                                           0xc
+#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT                                            0xd
+#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT                                               0xe
+#define CP_ME1_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS__SHIFT                                                0xf
+#define CP_ME1_INT_STAT_DEBUG__GPF_INT_ASSERTED__SHIFT                                                        0x10
+#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT                                           0x11
+#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                   0x17
+#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT                                               0x18
+#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT                                                 0x1a
+#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT                                         0x1b
+#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT                                                   0x1d
+#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT                                                   0x1e
+#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT                                                   0x1f
+#define CP_ME1_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED_MASK                                             0x00001000L
+#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK                                              0x00002000L
+#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK                                                 0x00004000L
+#define CP_ME1_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS_MASK                                                  0x00008000L
+#define CP_ME1_INT_STAT_DEBUG__GPF_INT_ASSERTED_MASK                                                          0x00010000L
+#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK                                             0x00020000L
+#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                     0x00800000L
+#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK                                                 0x01000000L
+#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK                                                   0x04000000L
+#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK                                           0x08000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK                                                     0x20000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK                                                     0x40000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK                                                     0x80000000L
+//CP_GFX_QUEUE_INDEX
+#define CP_GFX_QUEUE_INDEX__QUEUE_ACCESS__SHIFT                                                               0x0
+#define CP_GFX_QUEUE_INDEX__PIPE_ID__SHIFT                                                                    0x4
+#define CP_GFX_QUEUE_INDEX__QUEUE_ID__SHIFT                                                                   0x8
+#define CP_GFX_QUEUE_INDEX__QUEUE_ACCESS_MASK                                                                 0x00000001L
+#define CP_GFX_QUEUE_INDEX__PIPE_ID_MASK                                                                      0x00000030L
+#define CP_GFX_QUEUE_INDEX__QUEUE_ID_MASK                                                                     0x00000700L
+//CC_GC_EDC_CONFIG
+#define CC_GC_EDC_CONFIG__DIS_EDC__SHIFT                                                                      0x1
+#define CC_GC_EDC_CONFIG__DIS_EDC_MASK                                                                        0x00000002L
+//CP_ME1_PIPE_PRIORITY_CNTS
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                       0x0
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                      0x8
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                      0x10
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                       0x18
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                         0x000000FFL
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                        0x0000FF00L
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                        0x00FF0000L
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                         0xFF000000L
+//CP_ME1_PIPE0_PRIORITY
+#define CP_ME1_PIPE0_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME1_PIPE0_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME1_PIPE1_PRIORITY
+#define CP_ME1_PIPE1_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME1_PIPE1_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME1_PIPE2_PRIORITY
+#define CP_ME1_PIPE2_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME1_PIPE2_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME1_PIPE3_PRIORITY
+#define CP_ME1_PIPE3_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME1_PIPE3_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_PFP_PRGRM_CNTR_START
+#define CP_PFP_PRGRM_CNTR_START__IP_START__SHIFT                                                              0x0
+#define CP_PFP_PRGRM_CNTR_START__IP_START_MASK                                                                0xFFFFFFFFL
+//CP_ME_PRGRM_CNTR_START
+#define CP_ME_PRGRM_CNTR_START__IP_START__SHIFT                                                               0x0
+#define CP_ME_PRGRM_CNTR_START__IP_START_MASK                                                                 0xFFFFFFFFL
+//CP_MEC1_PRGRM_CNTR_START
+#define CP_MEC1_PRGRM_CNTR_START__IP_START__SHIFT                                                             0x0
+#define CP_MEC1_PRGRM_CNTR_START__IP_START_MASK                                                               0x000FFFFFL
+//CP_PFP_INTR_ROUTINE_START
+#define CP_PFP_INTR_ROUTINE_START__IR_START__SHIFT                                                            0x0
+#define CP_PFP_INTR_ROUTINE_START__IR_START_MASK                                                              0xFFFFFFFFL
+//CP_ME_INTR_ROUTINE_START
+#define CP_ME_INTR_ROUTINE_START__IR_START__SHIFT                                                             0x0
+#define CP_ME_INTR_ROUTINE_START__IR_START_MASK                                                               0xFFFFFFFFL
+//CP_MEC1_INTR_ROUTINE_START
+#define CP_MEC1_INTR_ROUTINE_START__IR_START__SHIFT                                                           0x0
+#define CP_MEC1_INTR_ROUTINE_START__IR_START_MASK                                                             0x000FFFFFL
+//CP_CONTEXT_CNTL
+#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_GE_CNTX__SHIFT                                                          0x0
+#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_PIPE_CNTX__SHIFT                                                        0x4
+#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_GE_CNTX__SHIFT                                                          0x10
+#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_PIPE_CNTX__SHIFT                                                        0x14
+#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_GE_CNTX_MASK                                                            0x0000000FL
+#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_PIPE_CNTX_MASK                                                          0x000000F0L
+#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_GE_CNTX_MASK                                                            0x000F0000L
+#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_PIPE_CNTX_MASK                                                          0x00F00000L
+//CP_MAX_CONTEXT
+#define CP_MAX_CONTEXT__MAX_CONTEXT__SHIFT                                                                    0x0
+#define CP_MAX_CONTEXT__MAX_CONTEXT_MASK                                                                      0x00000007L
+//CP_IQ_WAIT_TIME1
+#define CP_IQ_WAIT_TIME1__IB_OFFLOAD__SHIFT                                                                   0x0
+#define CP_IQ_WAIT_TIME1__ATOMIC_OFFLOAD__SHIFT                                                               0x8
+#define CP_IQ_WAIT_TIME1__WRM_OFFLOAD__SHIFT                                                                  0x10
+#define CP_IQ_WAIT_TIME1__GWS__SHIFT                                                                          0x18
+#define CP_IQ_WAIT_TIME1__IB_OFFLOAD_MASK                                                                     0x000000FFL
+#define CP_IQ_WAIT_TIME1__ATOMIC_OFFLOAD_MASK                                                                 0x0000FF00L
+#define CP_IQ_WAIT_TIME1__WRM_OFFLOAD_MASK                                                                    0x00FF0000L
+#define CP_IQ_WAIT_TIME1__GWS_MASK                                                                            0xFF000000L
+//CP_IQ_WAIT_TIME2
+#define CP_IQ_WAIT_TIME2__QUE_SLEEP__SHIFT                                                                    0x0
+#define CP_IQ_WAIT_TIME2__SCH_WAVE__SHIFT                                                                     0x8
+#define CP_IQ_WAIT_TIME2__DEQ_RETRY__SHIFT                                                                    0x18
+#define CP_IQ_WAIT_TIME2__QUE_SLEEP_MASK                                                                      0x000000FFL
+#define CP_IQ_WAIT_TIME2__SCH_WAVE_MASK                                                                       0x0000FF00L
+#define CP_IQ_WAIT_TIME2__DEQ_RETRY_MASK                                                                      0xFF000000L
+//CP_RB0_BASE_HI
+#define CP_RB0_BASE_HI__RB_BASE_HI__SHIFT                                                                     0x0
+#define CP_RB0_BASE_HI__RB_BASE_HI_MASK                                                                       0x000000FFL
+//CP_VMID_RESET
+#define CP_VMID_RESET__RESET_REQUEST__SHIFT                                                                   0x0
+#define CP_VMID_RESET__PIPE0_QUEUES__SHIFT                                                                    0x10
+#define CP_VMID_RESET__PIPE1_QUEUES__SHIFT                                                                    0x18
+#define CP_VMID_RESET__RESET_REQUEST_MASK                                                                     0x0000FFFFL
+#define CP_VMID_RESET__PIPE0_QUEUES_MASK                                                                      0x00FF0000L
+#define CP_VMID_RESET__PIPE1_QUEUES_MASK                                                                      0xFF000000L
+//CPC_INT_CNTL
+#define CPC_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                                      0xc
+#define CPC_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                                       0xd
+#define CPC_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                          0xe
+#define CPC_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                         0xf
+#define CPC_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                                   0x10
+#define CPC_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                      0x11
+#define CPC_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                              0x17
+#define CPC_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                          0x18
+#define CPC_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                            0x1a
+#define CPC_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                                    0x1b
+#define CPC_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                              0x1d
+#define CPC_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                              0x1e
+#define CPC_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                              0x1f
+#define CPC_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                                        0x00001000L
+#define CPC_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                         0x00002000L
+#define CPC_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                            0x00004000L
+#define CPC_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                           0x00008000L
+#define CPC_INT_CNTL__GPF_INT_ENABLE_MASK                                                                     0x00010000L
+#define CPC_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                        0x00020000L
+#define CPC_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                                0x00800000L
+#define CPC_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                            0x01000000L
+#define CPC_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                              0x04000000L
+#define CPC_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                      0x08000000L
+#define CPC_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                                0x20000000L
+#define CPC_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                                0x40000000L
+#define CPC_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                                0x80000000L
+//CPC_INT_STATUS
+#define CPC_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                                    0xc
+#define CPC_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                                     0xd
+#define CPC_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                                        0xe
+#define CPC_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                                       0xf
+#define CPC_INT_STATUS__GPF_INT_STATUS__SHIFT                                                                 0x10
+#define CPC_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                                    0x11
+#define CPC_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                            0x17
+#define CPC_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                                        0x18
+#define CPC_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                          0x1a
+#define CPC_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                                  0x1b
+#define CPC_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                            0x1d
+#define CPC_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                            0x1e
+#define CPC_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                            0x1f
+#define CPC_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                                      0x00001000L
+#define CPC_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                                       0x00002000L
+#define CPC_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                          0x00004000L
+#define CPC_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                         0x00008000L
+#define CPC_INT_STATUS__GPF_INT_STATUS_MASK                                                                   0x00010000L
+#define CPC_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                                      0x00020000L
+#define CPC_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                              0x00800000L
+#define CPC_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                          0x01000000L
+#define CPC_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                            0x04000000L
+#define CPC_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                                    0x08000000L
+#define CPC_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                              0x20000000L
+#define CPC_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                              0x40000000L
+#define CPC_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                              0x80000000L
+//CP_VMID_PREEMPT
+#define CP_VMID_PREEMPT__PREEMPT_REQUEST__SHIFT                                                               0x0
+#define CP_VMID_PREEMPT__VIRT_COMMAND__SHIFT                                                                  0x10
+#define CP_VMID_PREEMPT__PREEMPT_REQUEST_MASK                                                                 0x0000FFFFL
+#define CP_VMID_PREEMPT__VIRT_COMMAND_MASK                                                                    0x000F0000L
+//CPC_INT_CNTX_ID
+#define CPC_INT_CNTX_ID__CNTX_ID__SHIFT                                                                       0x0
+#define CPC_INT_CNTX_ID__CNTX_ID_MASK                                                                         0xFFFFFFFFL
+//CP_PQ_STATUS
+#define CP_PQ_STATUS__DOORBELL_UPDATED__SHIFT                                                                 0x0
+#define CP_PQ_STATUS__DOORBELL_ENABLE__SHIFT                                                                  0x1
+#define CP_PQ_STATUS__DOORBELL_UPDATED_EN__SHIFT                                                              0x2
+#define CP_PQ_STATUS__DOORBELL_UPDATED_MODE__SHIFT                                                            0x3
+#define CP_PQ_STATUS__DOORBELL_UPDATED_MASK                                                                   0x00000001L
+#define CP_PQ_STATUS__DOORBELL_ENABLE_MASK                                                                    0x00000002L
+#define CP_PQ_STATUS__DOORBELL_UPDATED_EN_MASK                                                                0x00000004L
+#define CP_PQ_STATUS__DOORBELL_UPDATED_MODE_MASK                                                              0x00000008L
+//CP_PFP_PRGRM_CNTR_START_HI
+#define CP_PFP_PRGRM_CNTR_START_HI__IP_START__SHIFT                                                           0x0
+#define CP_PFP_PRGRM_CNTR_START_HI__IP_START_MASK                                                             0x3FFFFFFFL
+//CP_MAX_DRAW_COUNT
+#define CP_MAX_DRAW_COUNT__MAX_DRAW_COUNT__SHIFT                                                              0x0
+#define CP_MAX_DRAW_COUNT__MAX_DRAW_COUNT_MASK                                                                0xFFFFFFFFL
+//CP_VMID_STATUS
+#define CP_VMID_STATUS__PREEMPT_DE_STATUS__SHIFT                                                              0x0
+#define CP_VMID_STATUS__PREEMPT_CE_STATUS__SHIFT                                                              0x10
+#define CP_VMID_STATUS__PREEMPT_DE_STATUS_MASK                                                                0x0000FFFFL
+#define CP_VMID_STATUS__PREEMPT_CE_STATUS_MASK                                                                0xFFFF0000L
+//CPC_SUSPEND_CTX_SAVE_BASE_ADDR_LO
+#define CPC_SUSPEND_CTX_SAVE_BASE_ADDR_LO__ADDR__SHIFT                                                        0xc
+#define CPC_SUSPEND_CTX_SAVE_BASE_ADDR_LO__ADDR_MASK                                                          0xFFFFF000L
+//CPC_SUSPEND_CTX_SAVE_BASE_ADDR_HI
+#define CPC_SUSPEND_CTX_SAVE_BASE_ADDR_HI__ADDR_HI__SHIFT                                                     0x0
+#define CPC_SUSPEND_CTX_SAVE_BASE_ADDR_HI__ADDR_HI_MASK                                                       0x01FFFFFFL
+//CPC_SUSPEND_CTX_SAVE_CONTROL
+#define CPC_SUSPEND_CTX_SAVE_CONTROL__POLICY__SHIFT                                                           0x3
+#define CPC_SUSPEND_CTX_SAVE_CONTROL__SCOPE__SHIFT                                                            0x5
+#define CPC_SUSPEND_CTX_SAVE_CONTROL__EXE_DISABLE__SHIFT                                                      0x17
+#define CPC_SUSPEND_CTX_SAVE_CONTROL__POLICY_MASK                                                             0x00000018L
+#define CPC_SUSPEND_CTX_SAVE_CONTROL__SCOPE_MASK                                                              0x00000060L
+#define CPC_SUSPEND_CTX_SAVE_CONTROL__EXE_DISABLE_MASK                                                        0x00800000L
+//CPC_SUSPEND_CNTL_STACK_OFFSET
+#define CPC_SUSPEND_CNTL_STACK_OFFSET__OFFSET__SHIFT                                                          0x2
+#define CPC_SUSPEND_CNTL_STACK_OFFSET__OFFSET_MASK                                                            0x0000FFFCL
+//CPC_SUSPEND_CNTL_STACK_SIZE
+#define CPC_SUSPEND_CNTL_STACK_SIZE__SIZE__SHIFT                                                              0xc
+#define CPC_SUSPEND_CNTL_STACK_SIZE__SIZE_MASK                                                                0x0000F000L
+//CPC_SUSPEND_WG_STATE_OFFSET
+#define CPC_SUSPEND_WG_STATE_OFFSET__OFFSET__SHIFT                                                            0x2
+#define CPC_SUSPEND_WG_STATE_OFFSET__OFFSET_MASK                                                              0x03FFFFFCL
+//CPC_SUSPEND_CTX_SAVE_SIZE
+#define CPC_SUSPEND_CTX_SAVE_SIZE__SIZE__SHIFT                                                                0xc
+#define CPC_SUSPEND_CTX_SAVE_SIZE__SIZE_MASK                                                                  0x03FFF000L
+//CPC_OS_PIPES
+#define CPC_OS_PIPES__OS_PIPES__SHIFT                                                                         0x0
+#define CPC_OS_PIPES__OS_PIPES_MASK                                                                           0x000000FFL
+//CP_SUSPEND_RESUME_REQ
+#define CP_SUSPEND_RESUME_REQ__SUSPEND_REQ__SHIFT                                                             0x0
+#define CP_SUSPEND_RESUME_REQ__RESUME_REQ__SHIFT                                                              0x1
+#define CP_SUSPEND_RESUME_REQ__SUSPEND_REQ_MASK                                                               0x00000001L
+#define CP_SUSPEND_RESUME_REQ__RESUME_REQ_MASK                                                                0x00000002L
+//CP_SUSPEND_CNTL
+#define CP_SUSPEND_CNTL__SUSPEND_MODE__SHIFT                                                                  0x0
+#define CP_SUSPEND_CNTL__SUSPEND_ENABLE__SHIFT                                                                0x1
+#define CP_SUSPEND_CNTL__RESUME_LOCK__SHIFT                                                                   0x2
+#define CP_SUSPEND_CNTL__ACE_SUSPEND_ACTIVE__SHIFT                                                            0x3
+#define CP_SUSPEND_CNTL__SUSPEND_MODE_MASK                                                                    0x00000001L
+#define CP_SUSPEND_CNTL__SUSPEND_ENABLE_MASK                                                                  0x00000002L
+#define CP_SUSPEND_CNTL__RESUME_LOCK_MASK                                                                     0x00000004L
+#define CP_SUSPEND_CNTL__ACE_SUSPEND_ACTIVE_MASK                                                              0x00000008L
+//CP_IQ_WAIT_TIME3
+#define CP_IQ_WAIT_TIME3__SUSPEND_QUE__SHIFT                                                                  0x0
+#define CP_IQ_WAIT_TIME3__SUSPEND_QUE_MASK                                                                    0x000000FFL
+//CPC_DDID_BASE_ADDR_LO
+#define CPC_DDID_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                                            0x6
+#define CPC_DDID_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                              0xFFFFFFC0L
+//CP_DDID_BASE_ADDR_LO
+#define CP_DDID_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                                             0x6
+#define CP_DDID_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                               0xFFFFFFC0L
+//CPC_DDID_BASE_ADDR_HI
+#define CPC_DDID_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                            0x0
+#define CPC_DDID_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                              0x01FFFFFFL
+//CP_DDID_BASE_ADDR_HI
+#define CP_DDID_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                             0x0
+#define CP_DDID_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                               0x01FFFFFFL
+//CPC_DDID_CNTL
+#define CPC_DDID_CNTL__THRESHOLD__SHIFT                                                                       0x0
+#define CPC_DDID_CNTL__SIZE__SHIFT                                                                            0x10
+#define CPC_DDID_CNTL__NO_RING_MEMORY__SHIFT                                                                  0x13
+#define CPC_DDID_CNTL__SCOPE__SHIFT                                                                           0x18
+#define CPC_DDID_CNTL__POLICY__SHIFT                                                                          0x1c
+#define CPC_DDID_CNTL__MODE__SHIFT                                                                            0x1e
+#define CPC_DDID_CNTL__ENABLE__SHIFT                                                                          0x1f
+#define CPC_DDID_CNTL__THRESHOLD_MASK                                                                         0x000000FFL
+#define CPC_DDID_CNTL__SIZE_MASK                                                                              0x00010000L
+#define CPC_DDID_CNTL__NO_RING_MEMORY_MASK                                                                    0x00080000L
+#define CPC_DDID_CNTL__SCOPE_MASK                                                                             0x03000000L
+#define CPC_DDID_CNTL__POLICY_MASK                                                                            0x30000000L
+#define CPC_DDID_CNTL__MODE_MASK                                                                              0x40000000L
+#define CPC_DDID_CNTL__ENABLE_MASK                                                                            0x80000000L
+//CP_DDID_CNTL
+#define CP_DDID_CNTL__THRESHOLD__SHIFT                                                                        0x0
+#define CP_DDID_CNTL__SIZE__SHIFT                                                                             0x10
+#define CP_DDID_CNTL__NO_RING_MEMORY__SHIFT                                                                   0x13
+#define CP_DDID_CNTL__VMID__SHIFT                                                                             0x14
+#define CP_DDID_CNTL__VMID_SEL__SHIFT                                                                         0x18
+#define CP_DDID_CNTL__SCOPE__SHIFT                                                                            0x1a
+#define CP_DDID_CNTL__POLICY__SHIFT                                                                           0x1c
+#define CP_DDID_CNTL__MODE__SHIFT                                                                             0x1e
+#define CP_DDID_CNTL__ENABLE__SHIFT                                                                           0x1f
+#define CP_DDID_CNTL__THRESHOLD_MASK                                                                          0x000000FFL
+#define CP_DDID_CNTL__SIZE_MASK                                                                               0x00010000L
+#define CP_DDID_CNTL__NO_RING_MEMORY_MASK                                                                     0x00080000L
+#define CP_DDID_CNTL__VMID_MASK                                                                               0x00F00000L
+#define CP_DDID_CNTL__VMID_SEL_MASK                                                                           0x01000000L
+#define CP_DDID_CNTL__SCOPE_MASK                                                                              0x0C000000L
+#define CP_DDID_CNTL__POLICY_MASK                                                                             0x30000000L
+#define CP_DDID_CNTL__MODE_MASK                                                                               0x40000000L
+#define CP_DDID_CNTL__ENABLE_MASK                                                                             0x80000000L
+//CP_GFX_DDID_INFLIGHT_COUNT
+#define CP_GFX_DDID_INFLIGHT_COUNT__COUNT__SHIFT                                                              0x0
+#define CP_GFX_DDID_INFLIGHT_COUNT__COUNT_MASK                                                                0x0000FFFFL
+//CP_GFX_DDID_WPTR
+#define CP_GFX_DDID_WPTR__COUNT__SHIFT                                                                        0x0
+#define CP_GFX_DDID_WPTR__COUNT_MASK                                                                          0x0000FFFFL
+//CP_GFX_DDID_RPTR
+#define CP_GFX_DDID_RPTR__COUNT__SHIFT                                                                        0x0
+#define CP_GFX_DDID_RPTR__COUNT_MASK                                                                          0x0000FFFFL
+//CP_GFX_DDID_DELTA_RPT_COUNT
+#define CP_GFX_DDID_DELTA_RPT_COUNT__COUNT__SHIFT                                                             0x0
+#define CP_GFX_DDID_DELTA_RPT_COUNT__COUNT_MASK                                                               0x000000FFL
+//CP_GFX_HPD_STATUS0
+#define CP_GFX_HPD_STATUS0__QUEUE_STATE__SHIFT                                                                0x0
+#define CP_GFX_HPD_STATUS0__MAPPED_QUEUE__SHIFT                                                               0x5
+#define CP_GFX_HPD_STATUS0__QUEUE_AVAILABLE__SHIFT                                                            0x8
+#define CP_GFX_HPD_STATUS0__FORCE_MAPPED_QUEUE__SHIFT                                                         0x10
+#define CP_GFX_HPD_STATUS0__FORCE_QUEUE_STATE__SHIFT                                                          0x14
+#define CP_GFX_HPD_STATUS0__SUSPEND_REQ__SHIFT                                                                0x1c
+#define CP_GFX_HPD_STATUS0__ENABLE_OVERIDE_QUEUEID__SHIFT                                                     0x1d
+#define CP_GFX_HPD_STATUS0__OVERIDE_QUEUEID__SHIFT                                                            0x1e
+#define CP_GFX_HPD_STATUS0__FORCE_QUEUE__SHIFT                                                                0x1f
+#define CP_GFX_HPD_STATUS0__QUEUE_STATE_MASK                                                                  0x0000001FL
+#define CP_GFX_HPD_STATUS0__MAPPED_QUEUE_MASK                                                                 0x000000E0L
+#define CP_GFX_HPD_STATUS0__QUEUE_AVAILABLE_MASK                                                              0x0000FF00L
+#define CP_GFX_HPD_STATUS0__FORCE_MAPPED_QUEUE_MASK                                                           0x00070000L
+#define CP_GFX_HPD_STATUS0__FORCE_QUEUE_STATE_MASK                                                            0x01F00000L
+#define CP_GFX_HPD_STATUS0__SUSPEND_REQ_MASK                                                                  0x10000000L
+#define CP_GFX_HPD_STATUS0__ENABLE_OVERIDE_QUEUEID_MASK                                                       0x20000000L
+#define CP_GFX_HPD_STATUS0__OVERIDE_QUEUEID_MASK                                                              0x40000000L
+#define CP_GFX_HPD_STATUS0__FORCE_QUEUE_MASK                                                                  0x80000000L
+//CP_GFX_HPD_CONTROL0
+#define CP_GFX_HPD_CONTROL0__SUSPEND_ENABLE__SHIFT                                                            0x0
+#define CP_GFX_HPD_CONTROL0__PIPE_HOLDING__SHIFT                                                              0x4
+#define CP_GFX_HPD_CONTROL0__SUSPEND_ENABLE_MASK                                                              0x00000001L
+#define CP_GFX_HPD_CONTROL0__PIPE_HOLDING_MASK                                                                0x00000010L
+//CP_GFX_HPD_OSPRE_FENCE_ADDR_LO
+#define CP_GFX_HPD_OSPRE_FENCE_ADDR_LO__ADDR_LO__SHIFT                                                        0x2
+#define CP_GFX_HPD_OSPRE_FENCE_ADDR_LO__ADDR_LO_MASK                                                          0xFFFFFFFCL
+//CP_GFX_HPD_OSPRE_FENCE_ADDR_HI
+#define CP_GFX_HPD_OSPRE_FENCE_ADDR_HI__ADDR_HI__SHIFT                                                        0x0
+#define CP_GFX_HPD_OSPRE_FENCE_ADDR_HI__RSVD__SHIFT                                                           0x10
+#define CP_GFX_HPD_OSPRE_FENCE_ADDR_HI__ADDR_HI_MASK                                                          0x0000FFFFL
+#define CP_GFX_HPD_OSPRE_FENCE_ADDR_HI__RSVD_MASK                                                             0xFFFF0000L
+//CP_GFX_HPD_OSPRE_FENCE_DATA_LO
+#define CP_GFX_HPD_OSPRE_FENCE_DATA_LO__DATA_LO__SHIFT                                                        0x0
+#define CP_GFX_HPD_OSPRE_FENCE_DATA_LO__DATA_LO_MASK                                                          0xFFFFFFFFL
+//CP_GFX_HPD_OSPRE_FENCE_DATA_HI
+#define CP_GFX_HPD_OSPRE_FENCE_DATA_HI__DATA_HI__SHIFT                                                        0x0
+#define CP_GFX_HPD_OSPRE_FENCE_DATA_HI__DATA_HI_MASK                                                          0xFFFFFFFFL
+//CP_GFX_INDEX_MUTEX
+#define CP_GFX_INDEX_MUTEX__REQUEST__SHIFT                                                                    0x0
+#define CP_GFX_INDEX_MUTEX__CLIENTID__SHIFT                                                                   0x1
+#define CP_GFX_INDEX_MUTEX__REQUEST_MASK                                                                      0x00000001L
+#define CP_GFX_INDEX_MUTEX__CLIENTID_MASK                                                                     0x0000000EL
+//CP_ME_PRGRM_CNTR_START_HI
+#define CP_ME_PRGRM_CNTR_START_HI__IP_START__SHIFT                                                            0x0
+#define CP_ME_PRGRM_CNTR_START_HI__IP_START_MASK                                                              0x3FFFFFFFL
+//CP_PFP_INTR_ROUTINE_START_HI
+#define CP_PFP_INTR_ROUTINE_START_HI__IR_START__SHIFT                                                         0x0
+#define CP_PFP_INTR_ROUTINE_START_HI__IR_START_MASK                                                           0x3FFFFFFFL
+//CP_ME_INTR_ROUTINE_START_HI
+#define CP_ME_INTR_ROUTINE_START_HI__IR_START__SHIFT                                                          0x0
+#define CP_ME_INTR_ROUTINE_START_HI__IR_START_MASK                                                            0x3FFFFFFFL
+//CP_GFX_MQD_BASE_ADDR
+#define CP_GFX_MQD_BASE_ADDR__BASE_ADDR__SHIFT                                                                0x2
+#define CP_GFX_MQD_BASE_ADDR__BASE_ADDR_MASK                                                                  0xFFFFFFFCL
+//CP_GFX_MQD_BASE_ADDR_HI
+#define CP_GFX_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                          0x0
+#define CP_GFX_MQD_BASE_ADDR_HI__APP_VMID__SHIFT                                                              0x1c
+#define CP_GFX_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                            0x0000FFFFL
+#define CP_GFX_MQD_BASE_ADDR_HI__APP_VMID_MASK                                                                0xF0000000L
+//CP_GFX_HQD_ACTIVE
+#define CP_GFX_HQD_ACTIVE__ACTIVE__SHIFT                                                                      0x0
+#define CP_GFX_HQD_ACTIVE__ACTIVE_MASK                                                                        0x00000001L
+//CP_GFX_HQD_VMID
+#define CP_GFX_HQD_VMID__VMID__SHIFT                                                                          0x0
+#define CP_GFX_HQD_VMID__VMID_MASK                                                                            0x0000000FL
+//CP_GFX_HQD_QUEUE_PRIORITY
+#define CP_GFX_HQD_QUEUE_PRIORITY__PRIORITY_LEVEL__SHIFT                                                      0x0
+#define CP_GFX_HQD_QUEUE_PRIORITY__PRIORITY_LEVEL_MASK                                                        0x0000000FL
+//CP_GFX_HQD_QUANTUM
+#define CP_GFX_HQD_QUANTUM__QUANTUM_EN__SHIFT                                                                 0x0
+#define CP_GFX_HQD_QUANTUM__QUANTUM_SCALE__SHIFT                                                              0x3
+#define CP_GFX_HQD_QUANTUM__QUANTUM_DURATION__SHIFT                                                           0x8
+#define CP_GFX_HQD_QUANTUM__QUANTUM_ACTIVE__SHIFT                                                             0x1f
+#define CP_GFX_HQD_QUANTUM__QUANTUM_EN_MASK                                                                   0x00000001L
+#define CP_GFX_HQD_QUANTUM__QUANTUM_SCALE_MASK                                                                0x00000018L
+#define CP_GFX_HQD_QUANTUM__QUANTUM_DURATION_MASK                                                             0x0000FF00L
+#define CP_GFX_HQD_QUANTUM__QUANTUM_ACTIVE_MASK                                                               0x80000000L
+//CP_GFX_HQD_BASE
+#define CP_GFX_HQD_BASE__RB_BASE__SHIFT                                                                       0x0
+#define CP_GFX_HQD_BASE__RB_BASE_MASK                                                                         0xFFFFFFFFL
+//CP_GFX_HQD_BASE_HI
+#define CP_GFX_HQD_BASE_HI__RB_BASE_HI__SHIFT                                                                 0x0
+#define CP_GFX_HQD_BASE_HI__RB_BASE_HI_MASK                                                                   0x000000FFL
+//CP_GFX_HQD_RPTR
+#define CP_GFX_HQD_RPTR__RB_RPTR__SHIFT                                                                       0x0
+#define CP_GFX_HQD_RPTR__RB_RPTR_MASK                                                                         0x000FFFFFL
+//CP_GFX_HQD_RPTR_ADDR
+#define CP_GFX_HQD_RPTR_ADDR__RB_RPTR_ADDR__SHIFT                                                             0x2
+#define CP_GFX_HQD_RPTR_ADDR__RB_RPTR_ADDR_MASK                                                               0xFFFFFFFCL
+//CP_GFX_HQD_RPTR_ADDR_HI
+#define CP_GFX_HQD_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT                                                       0x0
+#define CP_GFX_HQD_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK                                                         0x0000FFFFL
+//CP_RB_WPTR_POLL_ADDR_LO
+#define CP_RB_WPTR_POLL_ADDR_LO__RB_WPTR_POLL_ADDR_LO__SHIFT                                                  0x2
+#define CP_RB_WPTR_POLL_ADDR_LO__RB_WPTR_POLL_ADDR_LO_MASK                                                    0xFFFFFFFCL
+//CP_RB_WPTR_POLL_ADDR_HI
+#define CP_RB_WPTR_POLL_ADDR_HI__RB_WPTR_POLL_ADDR_HI__SHIFT                                                  0x0
+#define CP_RB_WPTR_POLL_ADDR_HI__RB_WPTR_POLL_ADDR_HI_MASK                                                    0x0000FFFFL
+//CP_RB_DOORBELL_CONTROL
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT                                                      0x1
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT                                                        0x2
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_EN__SHIFT                                                            0x1e
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_HIT__SHIFT                                                           0x1f
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_BIF_DROP_MASK                                                        0x00000002L
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK                                                          0x0FFFFFFCL
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_EN_MASK                                                              0x40000000L
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_HIT_MASK                                                             0x80000000L
+//CP_GFX_HQD_OFFSET
+#define CP_GFX_HQD_OFFSET__RB_OFFSET__SHIFT                                                                   0x0
+#define CP_GFX_HQD_OFFSET__DISABLE_RB_OFFSET__SHIFT                                                           0x1f
+#define CP_GFX_HQD_OFFSET__RB_OFFSET_MASK                                                                     0x000FFFFFL
+#define CP_GFX_HQD_OFFSET__DISABLE_RB_OFFSET_MASK                                                             0x80000000L
+//CP_GFX_HQD_CNTL
+#define CP_GFX_HQD_CNTL__RB_BUFSZ__SHIFT                                                                      0x0
+#define CP_GFX_HQD_CNTL__TMZ_STATE__SHIFT                                                                     0x6
+#define CP_GFX_HQD_CNTL__TMZ_MATCH__SHIFT                                                                     0x7
+#define CP_GFX_HQD_CNTL__RB_BLKSZ__SHIFT                                                                      0x8
+#define CP_GFX_HQD_CNTL__RB_NON_PRIV__SHIFT                                                                   0xf
+#define CP_GFX_HQD_CNTL__BUF_SWAP__SHIFT                                                                      0x10
+#define CP_GFX_HQD_CNTL__SCOPE__SHIFT                                                                         0x12
+#define CP_GFX_HQD_CNTL__MIN_AVAILSZ__SHIFT                                                                   0x14
+#define CP_GFX_HQD_CNTL__MIN_IB_AVAILSZ__SHIFT                                                                0x16
+#define CP_GFX_HQD_CNTL__CACHE_POLICY__SHIFT                                                                  0x18
+#define CP_GFX_HQD_CNTL__RB_NO_UPDATE__SHIFT                                                                  0x1b
+#define CP_GFX_HQD_CNTL__RB_EXE__SHIFT                                                                        0x1c
+#define CP_GFX_HQD_CNTL__KMD_QUEUE__SHIFT                                                                     0x1d
+#define CP_GFX_HQD_CNTL__RB_RPTR_WR_ENA__SHIFT                                                                0x1f
+#define CP_GFX_HQD_CNTL__RB_BUFSZ_MASK                                                                        0x0000003FL
+#define CP_GFX_HQD_CNTL__TMZ_STATE_MASK                                                                       0x00000040L
+#define CP_GFX_HQD_CNTL__TMZ_MATCH_MASK                                                                       0x00000080L
+#define CP_GFX_HQD_CNTL__RB_BLKSZ_MASK                                                                        0x00003F00L
+#define CP_GFX_HQD_CNTL__RB_NON_PRIV_MASK                                                                     0x00008000L
+#define CP_GFX_HQD_CNTL__BUF_SWAP_MASK                                                                        0x00030000L
+#define CP_GFX_HQD_CNTL__SCOPE_MASK                                                                           0x000C0000L
+#define CP_GFX_HQD_CNTL__MIN_AVAILSZ_MASK                                                                     0x00300000L
+#define CP_GFX_HQD_CNTL__MIN_IB_AVAILSZ_MASK                                                                  0x00C00000L
+#define CP_GFX_HQD_CNTL__CACHE_POLICY_MASK                                                                    0x03000000L
+#define CP_GFX_HQD_CNTL__RB_NO_UPDATE_MASK                                                                    0x08000000L
+#define CP_GFX_HQD_CNTL__RB_EXE_MASK                                                                          0x10000000L
+#define CP_GFX_HQD_CNTL__KMD_QUEUE_MASK                                                                       0x20000000L
+#define CP_GFX_HQD_CNTL__RB_RPTR_WR_ENA_MASK                                                                  0x80000000L
+//CP_GFX_HQD_CSMD_RPTR
+#define CP_GFX_HQD_CSMD_RPTR__RB_RPTR__SHIFT                                                                  0x0
+#define CP_GFX_HQD_CSMD_RPTR__RB_RPTR_MASK                                                                    0x000FFFFFL
+//CP_GFX_HQD_WPTR
+#define CP_GFX_HQD_WPTR__RB_WPTR__SHIFT                                                                       0x0
+#define CP_GFX_HQD_WPTR__RB_WPTR_MASK                                                                         0xFFFFFFFFL
+//CP_GFX_HQD_WPTR_HI
+#define CP_GFX_HQD_WPTR_HI__RB_WPTR__SHIFT                                                                    0x0
+#define CP_GFX_HQD_WPTR_HI__RB_WPTR_MASK                                                                      0xFFFFFFFFL
+//CP_GFX_HQD_DEQUEUE_REQUEST
+#define CP_GFX_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ__SHIFT                                                        0x0
+#define CP_GFX_HQD_DEQUEUE_REQUEST__REQ_TYPE__SHIFT                                                           0x1
+#define CP_GFX_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND__SHIFT                                                        0x4
+#define CP_GFX_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_EN__SHIFT                                                     0x9
+#define CP_GFX_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_EN__SHIFT                                                     0xa
+#define CP_GFX_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_MASK                                                          0x00000001L
+#define CP_GFX_HQD_DEQUEUE_REQUEST__REQ_TYPE_MASK                                                             0x0000000EL
+#define CP_GFX_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK                                                          0x00000010L
+#define CP_GFX_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_EN_MASK                                                       0x00000200L
+#define CP_GFX_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_EN_MASK                                                       0x00000400L
+//CP_GFX_HQD_MAPPED
+#define CP_GFX_HQD_MAPPED__MAPPED__SHIFT                                                                      0x0
+#define CP_GFX_HQD_MAPPED__MAPPED_MASK                                                                        0x00000001L
+//CP_GFX_HQD_QUE_MGR_CONTROL
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_IDLE_QUEUE_DISCONNECT__SHIFT                                      0x0
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_CONNECT_HANDSHAKE__SHIFT                                          0x4
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_FETCHER_DISCONNECT__SHIFT                                         0x5
+#define CP_GFX_HQD_QUE_MGR_CONTROL__FORCE_QUEUE_ACTIVE_EN__SHIFT                                              0x6
+#define CP_GFX_HQD_QUE_MGR_CONTROL__FORCE_ALLOW_DB_UPDATE_EN__SHIFT                                           0x7
+#define CP_GFX_HQD_QUE_MGR_CONTROL__FORCE_QUEUE__SHIFT                                                        0x8
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_OFFSET_UPDATE__SHIFT                                              0xb
+#define CP_GFX_HQD_QUE_MGR_CONTROL__PRIORITY_PREEMPT_DISABLE__SHIFT                                           0xd
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_QUEUE_MGR__SHIFT                                                  0xf
+#define CP_GFX_HQD_QUE_MGR_CONTROL__ENABLE_IDLE_MESSAGE__SHIFT                                                0x10
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_SWITCH_MESSAGE_IDLE__SHIFT                                        0x11
+#define CP_GFX_HQD_QUE_MGR_CONTROL__ENABLE_SWITCH_MSG_PREEMPT__SHIFT                                          0x12
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_MAPPED_QUEUE_IDLE_MSG__SHIFT                                      0x17
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_IDLE_QUEUE_DISCONNECT_MASK                                        0x00000001L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_CONNECT_HANDSHAKE_MASK                                            0x00000010L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_FETCHER_DISCONNECT_MASK                                           0x00000020L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__FORCE_QUEUE_ACTIVE_EN_MASK                                                0x00000040L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__FORCE_ALLOW_DB_UPDATE_EN_MASK                                             0x00000080L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__FORCE_QUEUE_MASK                                                          0x00000700L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_OFFSET_UPDATE_MASK                                                0x00000800L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__PRIORITY_PREEMPT_DISABLE_MASK                                             0x00002000L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_QUEUE_MGR_MASK                                                    0x00008000L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__ENABLE_IDLE_MESSAGE_MASK                                                  0x00010000L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_SWITCH_MESSAGE_IDLE_MASK                                          0x00020000L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__ENABLE_SWITCH_MSG_PREEMPT_MASK                                            0x00040000L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_MAPPED_QUEUE_IDLE_MSG_MASK                                        0x00800000L
+//CP_GFX_HQD_IQ_TIMER
+#define CP_GFX_HQD_IQ_TIMER__WAIT_TIME__SHIFT                                                                 0x0
+#define CP_GFX_HQD_IQ_TIMER__RETRY_TYPE__SHIFT                                                                0x8
+#define CP_GFX_HQD_IQ_TIMER__IMMEDIATE_EXPIRE__SHIFT                                                          0xb
+#define CP_GFX_HQD_IQ_TIMER__INTERRUPT_TYPE__SHIFT                                                            0xc
+#define CP_GFX_HQD_IQ_TIMER__CLOCK_COUNT__SHIFT                                                               0xe
+#define CP_GFX_HQD_IQ_TIMER__QUANTUM_TIMER__SHIFT                                                             0x16
+#define CP_GFX_HQD_IQ_TIMER__QUEUE_TYPE__SHIFT                                                                0x1b
+#define CP_GFX_HQD_IQ_TIMER__REARM_TIMER__SHIFT                                                               0x1c
+#define CP_GFX_HQD_IQ_TIMER__ACTIVE__SHIFT                                                                    0x1f
+#define CP_GFX_HQD_IQ_TIMER__WAIT_TIME_MASK                                                                   0x000000FFL
+#define CP_GFX_HQD_IQ_TIMER__RETRY_TYPE_MASK                                                                  0x00000700L
+#define CP_GFX_HQD_IQ_TIMER__IMMEDIATE_EXPIRE_MASK                                                            0x00000800L
+#define CP_GFX_HQD_IQ_TIMER__INTERRUPT_TYPE_MASK                                                              0x00003000L
+#define CP_GFX_HQD_IQ_TIMER__CLOCK_COUNT_MASK                                                                 0x0000C000L
+#define CP_GFX_HQD_IQ_TIMER__QUANTUM_TIMER_MASK                                                               0x00400000L
+#define CP_GFX_HQD_IQ_TIMER__QUEUE_TYPE_MASK                                                                  0x08000000L
+#define CP_GFX_HQD_IQ_TIMER__REARM_TIMER_MASK                                                                 0x10000000L
+#define CP_GFX_HQD_IQ_TIMER__ACTIVE_MASK                                                                      0x80000000L
+//CP_GFX_HQD_HQ_STATUS0
+#define CP_GFX_HQD_HQ_STATUS0__DEQUEUE_STATUS__SHIFT                                                          0x0
+#define CP_GFX_HQD_HQ_STATUS0__OS_PREEMPT_STATUS__SHIFT                                                       0x4
+#define CP_GFX_HQD_HQ_STATUS0__PREEMPT_ACK__SHIFT                                                             0x6
+#define CP_GFX_HQD_HQ_STATUS0__QUEUE_IDLE__SHIFT                                                              0x1e
+#define CP_GFX_HQD_HQ_STATUS0__DEQUEUE_STATUS_MASK                                                            0x00000001L
+#define CP_GFX_HQD_HQ_STATUS0__OS_PREEMPT_STATUS_MASK                                                         0x00000030L
+#define CP_GFX_HQD_HQ_STATUS0__PREEMPT_ACK_MASK                                                               0x00000040L
+#define CP_GFX_HQD_HQ_STATUS0__QUEUE_IDLE_MASK                                                                0x40000000L
+//CP_GFX_HQD_HQ_CONTROL0
+#define CP_GFX_HQD_HQ_CONTROL0__COMMAND__SHIFT                                                                0x0
+#define CP_GFX_HQD_HQ_CONTROL0__SPARES__SHIFT                                                                 0x4
+#define CP_GFX_HQD_HQ_CONTROL0__COMMAND_MASK                                                                  0x0000000FL
+#define CP_GFX_HQD_HQ_CONTROL0__SPARES_MASK                                                                   0x000000F0L
+//CP_GFX_MQD_CONTROL
+#define CP_GFX_MQD_CONTROL__VMID__SHIFT                                                                       0x0
+#define CP_GFX_MQD_CONTROL__PRIV_STATE__SHIFT                                                                 0x8
+#define CP_GFX_MQD_CONTROL__PROCESSING_MQD__SHIFT                                                             0xc
+#define CP_GFX_MQD_CONTROL__PROCESSING_MQD_EN__SHIFT                                                          0xd
+#define CP_GFX_MQD_CONTROL__EXE_DISABLE__SHIFT                                                                0x17
+#define CP_GFX_MQD_CONTROL__CACHE_POLICY__SHIFT                                                               0x18
+#define CP_GFX_MQD_CONTROL__SCOPE__SHIFT                                                                      0x1a
+#define CP_GFX_MQD_CONTROL__VMID_MASK                                                                         0x0000000FL
+#define CP_GFX_MQD_CONTROL__PRIV_STATE_MASK                                                                   0x00000100L
+#define CP_GFX_MQD_CONTROL__PROCESSING_MQD_MASK                                                               0x00001000L
+#define CP_GFX_MQD_CONTROL__PROCESSING_MQD_EN_MASK                                                            0x00002000L
+#define CP_GFX_MQD_CONTROL__EXE_DISABLE_MASK                                                                  0x00800000L
+#define CP_GFX_MQD_CONTROL__CACHE_POLICY_MASK                                                                 0x03000000L
+#define CP_GFX_MQD_CONTROL__SCOPE_MASK                                                                        0x0C000000L
+//CP_HQD_GFX_CONTROL
+#define CP_HQD_GFX_CONTROL__MESSAGE__SHIFT                                                                    0x0
+#define CP_HQD_GFX_CONTROL__MISC__SHIFT                                                                       0x4
+#define CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN__SHIFT                                                          0xf
+#define CP_HQD_GFX_CONTROL__MESSAGE_MASK                                                                      0x0000000FL
+#define CP_HQD_GFX_CONTROL__MISC_MASK                                                                         0x00007FF0L
+#define CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN_MASK                                                            0x00008000L
+//CP_HQD_GFX_STATUS
+#define CP_HQD_GFX_STATUS__STATUS__SHIFT                                                                      0x0
+#define CP_HQD_GFX_STATUS__STATUS_MASK                                                                        0x0000FFFFL
+//CP_DMA_WATCH0_ADDR_LO
+#define CP_DMA_WATCH0_ADDR_LO__RSVD__SHIFT                                                                    0x0
+#define CP_DMA_WATCH0_ADDR_LO__ADDR_LO__SHIFT                                                                 0x8
+#define CP_DMA_WATCH0_ADDR_LO__RSVD_MASK                                                                      0x000000FFL
+#define CP_DMA_WATCH0_ADDR_LO__ADDR_LO_MASK                                                                   0xFFFFFF00L
+//CP_DMA_WATCH0_ADDR_HI
+#define CP_DMA_WATCH0_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CP_DMA_WATCH0_ADDR_HI__RSVD__SHIFT                                                                    0x19
+#define CP_DMA_WATCH0_ADDR_HI__ADDR_HI_MASK                                                                   0x01FFFFFFL
+#define CP_DMA_WATCH0_ADDR_HI__RSVD_MASK                                                                      0xFE000000L
+//CP_DMA_WATCH0_MASK
+#define CP_DMA_WATCH0_MASK__RSVD__SHIFT                                                                       0x0
+#define CP_DMA_WATCH0_MASK__MASK__SHIFT                                                                       0x8
+#define CP_DMA_WATCH0_MASK__RSVD_MASK                                                                         0x000000FFL
+#define CP_DMA_WATCH0_MASK__MASK_MASK                                                                         0xFFFFFF00L
+//CP_DMA_WATCH0_CNTL
+#define CP_DMA_WATCH0_CNTL__VMID__SHIFT                                                                       0x0
+#define CP_DMA_WATCH0_CNTL__RSVD1__SHIFT                                                                      0x4
+#define CP_DMA_WATCH0_CNTL__WATCH_READS__SHIFT                                                                0x8
+#define CP_DMA_WATCH0_CNTL__WATCH_WRITES__SHIFT                                                               0x9
+#define CP_DMA_WATCH0_CNTL__ANY_VMID__SHIFT                                                                   0xa
+#define CP_DMA_WATCH0_CNTL__RSVD2__SHIFT                                                                      0xb
+#define CP_DMA_WATCH0_CNTL__VMID_MASK                                                                         0x0000000FL
+#define CP_DMA_WATCH0_CNTL__RSVD1_MASK                                                                        0x000000F0L
+#define CP_DMA_WATCH0_CNTL__WATCH_READS_MASK                                                                  0x00000100L
+#define CP_DMA_WATCH0_CNTL__WATCH_WRITES_MASK                                                                 0x00000200L
+#define CP_DMA_WATCH0_CNTL__ANY_VMID_MASK                                                                     0x00000400L
+#define CP_DMA_WATCH0_CNTL__RSVD2_MASK                                                                        0xFFFFF800L
+//CP_DMA_WATCH1_ADDR_LO
+#define CP_DMA_WATCH1_ADDR_LO__RSVD__SHIFT                                                                    0x0
+#define CP_DMA_WATCH1_ADDR_LO__ADDR_LO__SHIFT                                                                 0x8
+#define CP_DMA_WATCH1_ADDR_LO__RSVD_MASK                                                                      0x000000FFL
+#define CP_DMA_WATCH1_ADDR_LO__ADDR_LO_MASK                                                                   0xFFFFFF00L
+//CP_DMA_WATCH1_ADDR_HI
+#define CP_DMA_WATCH1_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CP_DMA_WATCH1_ADDR_HI__RSVD__SHIFT                                                                    0x19
+#define CP_DMA_WATCH1_ADDR_HI__ADDR_HI_MASK                                                                   0x01FFFFFFL
+#define CP_DMA_WATCH1_ADDR_HI__RSVD_MASK                                                                      0xFE000000L
+//CP_DMA_WATCH1_MASK
+#define CP_DMA_WATCH1_MASK__RSVD__SHIFT                                                                       0x0
+#define CP_DMA_WATCH1_MASK__MASK__SHIFT                                                                       0x8
+#define CP_DMA_WATCH1_MASK__RSVD_MASK                                                                         0x000000FFL
+#define CP_DMA_WATCH1_MASK__MASK_MASK                                                                         0xFFFFFF00L
+//CP_DMA_WATCH1_CNTL
+#define CP_DMA_WATCH1_CNTL__VMID__SHIFT                                                                       0x0
+#define CP_DMA_WATCH1_CNTL__RSVD1__SHIFT                                                                      0x4
+#define CP_DMA_WATCH1_CNTL__WATCH_READS__SHIFT                                                                0x8
+#define CP_DMA_WATCH1_CNTL__WATCH_WRITES__SHIFT                                                               0x9
+#define CP_DMA_WATCH1_CNTL__ANY_VMID__SHIFT                                                                   0xa
+#define CP_DMA_WATCH1_CNTL__RSVD2__SHIFT                                                                      0xb
+#define CP_DMA_WATCH1_CNTL__VMID_MASK                                                                         0x0000000FL
+#define CP_DMA_WATCH1_CNTL__RSVD1_MASK                                                                        0x000000F0L
+#define CP_DMA_WATCH1_CNTL__WATCH_READS_MASK                                                                  0x00000100L
+#define CP_DMA_WATCH1_CNTL__WATCH_WRITES_MASK                                                                 0x00000200L
+#define CP_DMA_WATCH1_CNTL__ANY_VMID_MASK                                                                     0x00000400L
+#define CP_DMA_WATCH1_CNTL__RSVD2_MASK                                                                        0xFFFFF800L
+//CP_DMA_WATCH2_ADDR_LO
+#define CP_DMA_WATCH2_ADDR_LO__RSVD__SHIFT                                                                    0x0
+#define CP_DMA_WATCH2_ADDR_LO__ADDR_LO__SHIFT                                                                 0x8
+#define CP_DMA_WATCH2_ADDR_LO__RSVD_MASK                                                                      0x000000FFL
+#define CP_DMA_WATCH2_ADDR_LO__ADDR_LO_MASK                                                                   0xFFFFFF00L
+//CP_DMA_WATCH2_ADDR_HI
+#define CP_DMA_WATCH2_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CP_DMA_WATCH2_ADDR_HI__RSVD__SHIFT                                                                    0x19
+#define CP_DMA_WATCH2_ADDR_HI__ADDR_HI_MASK                                                                   0x01FFFFFFL
+#define CP_DMA_WATCH2_ADDR_HI__RSVD_MASK                                                                      0xFE000000L
+//CP_DMA_WATCH2_MASK
+#define CP_DMA_WATCH2_MASK__RSVD__SHIFT                                                                       0x0
+#define CP_DMA_WATCH2_MASK__MASK__SHIFT                                                                       0x8
+#define CP_DMA_WATCH2_MASK__RSVD_MASK                                                                         0x000000FFL
+#define CP_DMA_WATCH2_MASK__MASK_MASK                                                                         0xFFFFFF00L
+//CP_DMA_WATCH2_CNTL
+#define CP_DMA_WATCH2_CNTL__VMID__SHIFT                                                                       0x0
+#define CP_DMA_WATCH2_CNTL__RSVD1__SHIFT                                                                      0x4
+#define CP_DMA_WATCH2_CNTL__WATCH_READS__SHIFT                                                                0x8
+#define CP_DMA_WATCH2_CNTL__WATCH_WRITES__SHIFT                                                               0x9
+#define CP_DMA_WATCH2_CNTL__ANY_VMID__SHIFT                                                                   0xa
+#define CP_DMA_WATCH2_CNTL__RSVD2__SHIFT                                                                      0xb
+#define CP_DMA_WATCH2_CNTL__VMID_MASK                                                                         0x0000000FL
+#define CP_DMA_WATCH2_CNTL__RSVD1_MASK                                                                        0x000000F0L
+#define CP_DMA_WATCH2_CNTL__WATCH_READS_MASK                                                                  0x00000100L
+#define CP_DMA_WATCH2_CNTL__WATCH_WRITES_MASK                                                                 0x00000200L
+#define CP_DMA_WATCH2_CNTL__ANY_VMID_MASK                                                                     0x00000400L
+#define CP_DMA_WATCH2_CNTL__RSVD2_MASK                                                                        0xFFFFF800L
+//CP_DMA_WATCH3_ADDR_LO
+#define CP_DMA_WATCH3_ADDR_LO__RSVD__SHIFT                                                                    0x0
+#define CP_DMA_WATCH3_ADDR_LO__ADDR_LO__SHIFT                                                                 0x8
+#define CP_DMA_WATCH3_ADDR_LO__RSVD_MASK                                                                      0x000000FFL
+#define CP_DMA_WATCH3_ADDR_LO__ADDR_LO_MASK                                                                   0xFFFFFF00L
+//CP_DMA_WATCH3_ADDR_HI
+#define CP_DMA_WATCH3_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CP_DMA_WATCH3_ADDR_HI__RSVD__SHIFT                                                                    0x19
+#define CP_DMA_WATCH3_ADDR_HI__ADDR_HI_MASK                                                                   0x01FFFFFFL
+#define CP_DMA_WATCH3_ADDR_HI__RSVD_MASK                                                                      0xFE000000L
+//CP_DMA_WATCH3_MASK
+#define CP_DMA_WATCH3_MASK__RSVD__SHIFT                                                                       0x0
+#define CP_DMA_WATCH3_MASK__MASK__SHIFT                                                                       0x8
+#define CP_DMA_WATCH3_MASK__RSVD_MASK                                                                         0x000000FFL
+#define CP_DMA_WATCH3_MASK__MASK_MASK                                                                         0xFFFFFF00L
+//CP_DMA_WATCH3_CNTL
+#define CP_DMA_WATCH3_CNTL__VMID__SHIFT                                                                       0x0
+#define CP_DMA_WATCH3_CNTL__RSVD1__SHIFT                                                                      0x4
+#define CP_DMA_WATCH3_CNTL__WATCH_READS__SHIFT                                                                0x8
+#define CP_DMA_WATCH3_CNTL__WATCH_WRITES__SHIFT                                                               0x9
+#define CP_DMA_WATCH3_CNTL__ANY_VMID__SHIFT                                                                   0xa
+#define CP_DMA_WATCH3_CNTL__RSVD2__SHIFT                                                                      0xb
+#define CP_DMA_WATCH3_CNTL__VMID_MASK                                                                         0x0000000FL
+#define CP_DMA_WATCH3_CNTL__RSVD1_MASK                                                                        0x000000F0L
+#define CP_DMA_WATCH3_CNTL__WATCH_READS_MASK                                                                  0x00000100L
+#define CP_DMA_WATCH3_CNTL__WATCH_WRITES_MASK                                                                 0x00000200L
+#define CP_DMA_WATCH3_CNTL__ANY_VMID_MASK                                                                     0x00000400L
+#define CP_DMA_WATCH3_CNTL__RSVD2_MASK                                                                        0xFFFFF800L
+//CP_DMA_WATCH_STAT_ADDR_LO
+#define CP_DMA_WATCH_STAT_ADDR_LO__ADDR_LO__SHIFT                                                             0x2
+#define CP_DMA_WATCH_STAT_ADDR_LO__ADDR_LO_MASK                                                               0xFFFFFFFCL
+//CP_DMA_WATCH_STAT_ADDR_HI
+#define CP_DMA_WATCH_STAT_ADDR_HI__ADDR_HI__SHIFT                                                             0x0
+#define CP_DMA_WATCH_STAT_ADDR_HI__ADDR_HI_MASK                                                               0x01FFFFFFL
+//CP_DMA_WATCH_STAT
+#define CP_DMA_WATCH_STAT__VMID__SHIFT                                                                        0x0
+#define CP_DMA_WATCH_STAT__QUEUE_ID__SHIFT                                                                    0x4
+#define CP_DMA_WATCH_STAT__CLIENT_ID__SHIFT                                                                   0x8
+#define CP_DMA_WATCH_STAT__PIPE__SHIFT                                                                        0xc
+#define CP_DMA_WATCH_STAT__WATCH_ID__SHIFT                                                                    0x10
+#define CP_DMA_WATCH_STAT__RD_WR__SHIFT                                                                       0x14
+#define CP_DMA_WATCH_STAT__TRAP_FLAG__SHIFT                                                                   0x1f
+#define CP_DMA_WATCH_STAT__VMID_MASK                                                                          0x0000000FL
+#define CP_DMA_WATCH_STAT__QUEUE_ID_MASK                                                                      0x00000070L
+#define CP_DMA_WATCH_STAT__CLIENT_ID_MASK                                                                     0x00000700L
+#define CP_DMA_WATCH_STAT__PIPE_MASK                                                                          0x00003000L
+#define CP_DMA_WATCH_STAT__WATCH_ID_MASK                                                                      0x00030000L
+#define CP_DMA_WATCH_STAT__RD_WR_MASK                                                                         0x00100000L
+#define CP_DMA_WATCH_STAT__TRAP_FLAG_MASK                                                                     0x80000000L
+//CP_PFP_JT_STAT
+#define CP_PFP_JT_STAT__JT_LOADED__SHIFT                                                                      0x0
+#define CP_PFP_JT_STAT__WR_MASK__SHIFT                                                                        0x10
+#define CP_PFP_JT_STAT__JT_LOADED_MASK                                                                        0x00000003L
+#define CP_PFP_JT_STAT__WR_MASK_MASK                                                                          0x00030000L
+//CP_MEC_JT_STAT
+#define CP_MEC_JT_STAT__JT_LOADED__SHIFT                                                                      0x0
+#define CP_MEC_JT_STAT__WR_MASK__SHIFT                                                                        0x10
+#define CP_MEC_JT_STAT__JT_LOADED_MASK                                                                        0x000000FFL
+#define CP_MEC_JT_STAT__WR_MASK_MASK                                                                          0x00FF0000L
+//CP_CPC_BUSY_HYSTERESIS
+#define CP_CPC_BUSY_HYSTERESIS__CAC_ACTIVE__SHIFT                                                             0x0
+#define CP_CPC_BUSY_HYSTERESIS__CPC_BUSY__SHIFT                                                               0x8
+#define CP_CPC_BUSY_HYSTERESIS__CAC_ACTIVE_MASK                                                               0x000000FFL
+#define CP_CPC_BUSY_HYSTERESIS__CPC_BUSY_MASK                                                                 0x0000FF00L
+//CP_CPF_BUSY_HYSTERESIS1
+#define CP_CPF_BUSY_HYSTERESIS1__CAC_ACTIVE__SHIFT                                                            0x0
+#define CP_CPF_BUSY_HYSTERESIS1__CPF_BUSY__SHIFT                                                              0x8
+#define CP_CPF_BUSY_HYSTERESIS1__CORE_BUSY__SHIFT                                                             0x10
+#define CP_CPF_BUSY_HYSTERESIS1__GFX_BUSY__SHIFT                                                              0x18
+#define CP_CPF_BUSY_HYSTERESIS1__CAC_ACTIVE_MASK                                                              0x000000FFL
+#define CP_CPF_BUSY_HYSTERESIS1__CPF_BUSY_MASK                                                                0x0000FF00L
+#define CP_CPF_BUSY_HYSTERESIS1__CORE_BUSY_MASK                                                               0x00FF0000L
+#define CP_CPF_BUSY_HYSTERESIS1__GFX_BUSY_MASK                                                                0xFF000000L
+//CP_CPF_BUSY_HYSTERESIS2
+#define CP_CPF_BUSY_HYSTERESIS2__CMP_BUSY__SHIFT                                                              0x0
+#define CP_CPF_BUSY_HYSTERESIS2__CMP_BUSY_MASK                                                                0x000000FFL
+//CP_CPG_BUSY_HYSTERESIS1
+#define CP_CPG_BUSY_HYSTERESIS1__CAC_ACTIVE__SHIFT                                                            0x0
+#define CP_CPG_BUSY_HYSTERESIS1__CP_BUSY__SHIFT                                                               0x8
+#define CP_CPG_BUSY_HYSTERESIS1__DMA_BUSY__SHIFT                                                              0x10
+#define CP_CPG_BUSY_HYSTERESIS1__GFX_BUSY__SHIFT                                                              0x18
+#define CP_CPG_BUSY_HYSTERESIS1__CAC_ACTIVE_MASK                                                              0x000000FFL
+#define CP_CPG_BUSY_HYSTERESIS1__CP_BUSY_MASK                                                                 0x0000FF00L
+#define CP_CPG_BUSY_HYSTERESIS1__DMA_BUSY_MASK                                                                0x00FF0000L
+#define CP_CPG_BUSY_HYSTERESIS1__GFX_BUSY_MASK                                                                0xFF000000L
+//CP_CPG_BUSY_HYSTERESIS2
+#define CP_CPG_BUSY_HYSTERESIS2__CMP_BUSY__SHIFT                                                              0x0
+#define CP_CPG_BUSY_HYSTERESIS2__SPI_CLOCK_0__SHIFT                                                           0x8
+#define CP_CPG_BUSY_HYSTERESIS2__CMP_BUSY_MASK                                                                0x000000FFL
+#define CP_CPG_BUSY_HYSTERESIS2__SPI_CLOCK_0_MASK                                                             0x0000FF00L
+//CPF_FED_ERROR_ADDR_LO
+#define CPF_FED_ERROR_ADDR_LO__ADDR_LO__SHIFT                                                                 0x5
+#define CPF_FED_ERROR_ADDR_LO__ADDR_LO_MASK                                                                   0xFFFFFFE0L
+//CPF_FED_ERROR_ADDR_HI
+#define CPF_FED_ERROR_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CPF_FED_ERROR_ADDR_HI__ADDR_HI_MASK                                                                   0x01FFFFFFL
+//CPC_FED_ERROR_ADDR_LO
+#define CPC_FED_ERROR_ADDR_LO__ADDR_LO__SHIFT                                                                 0x5
+#define CPC_FED_ERROR_ADDR_LO__ADDR_LO_MASK                                                                   0xFFFFFFE0L
+//CPC_FED_ERROR_ADDR_HI
+#define CPC_FED_ERROR_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CPC_FED_ERROR_ADDR_HI__ADDR_HI_MASK                                                                   0x0000FFFFL
+//CPG_FED_ERROR_ADDR_LO
+#define CPG_FED_ERROR_ADDR_LO__ADDR_LO__SHIFT                                                                 0x5
+#define CPG_FED_ERROR_ADDR_LO__ADDR_LO_MASK                                                                   0xFFFFFFE0L
+//CPG_FED_ERROR_ADDR_HI
+#define CPG_FED_ERROR_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CPG_FED_ERROR_ADDR_HI__ADDR_HI_MASK                                                                   0x01FFFFFFL
+//CP_RB_DOORBELL_CLEAR
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUEUE__SHIFT                                                             0x0
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUE_DOORBELL_EN_CLEAR__SHIFT                                             0x8
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUE_DOORBELL_HIT_CLEAR__SHIFT                                            0x9
+#define CP_RB_DOORBELL_CLEAR__MASTER_DOORBELL_EN_CLEAR__SHIFT                                                 0xa
+#define CP_RB_DOORBELL_CLEAR__MASTER_DOORBELL_HIT_CLEAR__SHIFT                                                0xb
+#define CP_RB_DOORBELL_CLEAR__QUEUES_DOORBELL_EN_CLEAR__SHIFT                                                 0xc
+#define CP_RB_DOORBELL_CLEAR__QUEUES_DOORBELL_HIT_CLEAR__SHIFT                                                0xd
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUEUE_MASK                                                               0x00000007L
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUE_DOORBELL_EN_CLEAR_MASK                                               0x00000100L
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUE_DOORBELL_HIT_CLEAR_MASK                                              0x00000200L
+#define CP_RB_DOORBELL_CLEAR__MASTER_DOORBELL_EN_CLEAR_MASK                                                   0x00000400L
+#define CP_RB_DOORBELL_CLEAR__MASTER_DOORBELL_HIT_CLEAR_MASK                                                  0x00000800L
+#define CP_RB_DOORBELL_CLEAR__QUEUES_DOORBELL_EN_CLEAR_MASK                                                   0x00001000L
+#define CP_RB_DOORBELL_CLEAR__QUEUES_DOORBELL_HIT_CLEAR_MASK                                                  0x00002000L
+//CP_RB0_ACTIVE
+#define CP_RB0_ACTIVE__ACTIVE__SHIFT                                                                          0x0
+#define CP_RB0_ACTIVE__ACTIVE_MASK                                                                            0x00000001L
+//CP_RB_ACTIVE
+#define CP_RB_ACTIVE__ACTIVE__SHIFT                                                                           0x0
+#define CP_RB_ACTIVE__ACTIVE_MASK                                                                             0x00000001L
+//CP_RB_STATUS
+#define CP_RB_STATUS__DOORBELL_UPDATED__SHIFT                                                                 0x0
+#define CP_RB_STATUS__DOORBELL_ENABLE__SHIFT                                                                  0x1
+#define CP_RB_STATUS__DOORBELL_UPDATED_MASK                                                                   0x00000001L
+#define CP_RB_STATUS__DOORBELL_ENABLE_MASK                                                                    0x00000002L
+//CPG_RCIU_CAM_INDEX
+#define CPG_RCIU_CAM_INDEX__INDEX__SHIFT                                                                      0x0
+#define CPG_RCIU_CAM_INDEX__INDEX_MASK                                                                        0x0000001FL
+//CPG_RCIU_CAM_DATA
+#define CPG_RCIU_CAM_DATA__DATA__SHIFT                                                                        0x0
+#define CPG_RCIU_CAM_DATA__DATA_MASK                                                                          0xFFFFFFFFL
+//CPG_RCIU_CAM_DATA_PHASE0
+#define CPG_RCIU_CAM_DATA_PHASE0__ADDR__SHIFT                                                                 0x0
+#define CPG_RCIU_CAM_DATA_PHASE0__PIPE0_EN__SHIFT                                                             0x18
+#define CPG_RCIU_CAM_DATA_PHASE0__PIPE1_EN__SHIFT                                                             0x19
+#define CPG_RCIU_CAM_DATA_PHASE0__SKIP_WR__SHIFT                                                              0x1f
+#define CPG_RCIU_CAM_DATA_PHASE0__ADDR_MASK                                                                   0x0003FFFFL
+#define CPG_RCIU_CAM_DATA_PHASE0__PIPE0_EN_MASK                                                               0x01000000L
+#define CPG_RCIU_CAM_DATA_PHASE0__PIPE1_EN_MASK                                                               0x02000000L
+#define CPG_RCIU_CAM_DATA_PHASE0__SKIP_WR_MASK                                                                0x80000000L
+//CPG_RCIU_CAM_DATA_PHASE1
+#define CPG_RCIU_CAM_DATA_PHASE1__MASK__SHIFT                                                                 0x0
+#define CPG_RCIU_CAM_DATA_PHASE1__MASK_MASK                                                                   0xFFFFFFFFL
+//CPG_RCIU_CAM_DATA_PHASE2
+#define CPG_RCIU_CAM_DATA_PHASE2__VALUE__SHIFT                                                                0x0
+#define CPG_RCIU_CAM_DATA_PHASE2__VALUE_MASK                                                                  0xFFFFFFFFL
+//CPG_RCIU_CAM_DATA_PHASE3
+#define CPG_RCIU_CAM_DATA_PHASE3__ADDR_HI__SHIFT                                                              0x0
+#define CPG_RCIU_CAM_DATA_PHASE3__ADDR_HI_MASK                                                                0x000FFFFFL
+//CP_GPU_TIMESTAMP_OFFSET_LO
+#define CP_GPU_TIMESTAMP_OFFSET_LO__OFFSET_LO__SHIFT                                                          0x0
+#define CP_GPU_TIMESTAMP_OFFSET_LO__OFFSET_LO_MASK                                                            0xFFFFFFFFL
+//CP_GPU_TIMESTAMP_OFFSET_HI
+#define CP_GPU_TIMESTAMP_OFFSET_HI__OFFSET_HI__SHIFT                                                          0x0
+#define CP_GPU_TIMESTAMP_OFFSET_HI__OFFSET_HI_MASK                                                            0xFFFFFFFFL
+//CP_SDMA_DMA_DONE
+#define CP_SDMA_DMA_DONE__SDMA_ID__SHIFT                                                                      0x0
+#define CP_SDMA_DMA_DONE__SDMA_ID_MASK                                                                        0x0000000FL
+//CP_PFP_SDMA_CS
+#define CP_PFP_SDMA_CS__REQUEST_GRANT__SHIFT                                                                  0x0
+#define CP_PFP_SDMA_CS__SDMA_ID__SHIFT                                                                        0x4
+#define CP_PFP_SDMA_CS__REQUEST_POSITION__SHIFT                                                               0x8
+#define CP_PFP_SDMA_CS__SDMA_COUNT__SHIFT                                                                     0xc
+#define CP_PFP_SDMA_CS__REQUEST_GRANT_MASK                                                                    0x00000001L
+#define CP_PFP_SDMA_CS__SDMA_ID_MASK                                                                          0x000000F0L
+#define CP_PFP_SDMA_CS__REQUEST_POSITION_MASK                                                                 0x00000F00L
+#define CP_PFP_SDMA_CS__SDMA_COUNT_MASK                                                                       0x00003000L
+//CP_ME_SDMA_CS
+#define CP_ME_SDMA_CS__REQUEST_GRANT__SHIFT                                                                   0x0
+#define CP_ME_SDMA_CS__SDMA_ID__SHIFT                                                                         0x4
+#define CP_ME_SDMA_CS__REQUEST_POSITION__SHIFT                                                                0x8
+#define CP_ME_SDMA_CS__SDMA_COUNT__SHIFT                                                                      0xc
+#define CP_ME_SDMA_CS__REQUEST_GRANT_MASK                                                                     0x00000001L
+#define CP_ME_SDMA_CS__SDMA_ID_MASK                                                                           0x000000F0L
+#define CP_ME_SDMA_CS__REQUEST_POSITION_MASK                                                                  0x00000F00L
+#define CP_ME_SDMA_CS__SDMA_COUNT_MASK                                                                        0x00003000L
+//CPF_GCR_CNTL
+#define CPF_GCR_CNTL__GCR_GL_CMD__SHIFT                                                                       0x0
+#define CPF_GCR_CNTL__GCR_GL_CMD_MASK                                                                         0x0007FFFFL
+//CPG_UTCL1_STATUS
+#define CPG_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define CPG_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define CPG_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define CPG_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                0x8
+#define CPG_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                0x10
+#define CPG_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                  0x18
+#define CPG_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define CPG_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define CPG_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+#define CPG_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                  0x00003F00L
+#define CPG_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                  0x003F0000L
+#define CPG_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                    0x3F000000L
+//CPC_UTCL1_STATUS
+#define CPC_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define CPC_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define CPC_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define CPC_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                0x8
+#define CPC_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                0x10
+#define CPC_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                  0x18
+#define CPC_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define CPC_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define CPC_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+#define CPC_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                  0x00003F00L
+#define CPC_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                  0x003F0000L
+#define CPC_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                    0x3F000000L
+//CPF_UTCL1_STATUS
+#define CPF_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define CPF_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define CPF_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define CPF_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                0x8
+#define CPF_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                0x10
+#define CPF_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                  0x18
+#define CPF_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define CPF_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define CPF_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+#define CPF_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                  0x00003F00L
+#define CPF_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                  0x003F0000L
+#define CPF_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                    0x3F000000L
+//CP_SD_CNTL
+#define CP_SD_CNTL__CPF_EN__SHIFT                                                                             0x0
+#define CP_SD_CNTL__CPG_EN__SHIFT                                                                             0x1
+#define CP_SD_CNTL__CPC_EN__SHIFT                                                                             0x2
+#define CP_SD_CNTL__RLC_EN__SHIFT                                                                             0x3
+#define CP_SD_CNTL__GE_EN__SHIFT                                                                              0x5
+#define CP_SD_CNTL__UTCL1_EN__SHIFT                                                                           0x6
+#define CP_SD_CNTL__EA_EN__SHIFT                                                                              0x9
+#define CP_SD_CNTL__SDMA_EN__SHIFT                                                                            0xa
+#define CP_SD_CNTL__SD_VMIDVEC_OVERRIDE__SHIFT                                                                0x1f
+#define CP_SD_CNTL__CPF_EN_MASK                                                                               0x00000001L
+#define CP_SD_CNTL__CPG_EN_MASK                                                                               0x00000002L
+#define CP_SD_CNTL__CPC_EN_MASK                                                                               0x00000004L
+#define CP_SD_CNTL__RLC_EN_MASK                                                                               0x00000008L
+#define CP_SD_CNTL__GE_EN_MASK                                                                                0x00000020L
+#define CP_SD_CNTL__UTCL1_EN_MASK                                                                             0x00000040L
+#define CP_SD_CNTL__EA_EN_MASK                                                                                0x00000200L
+#define CP_SD_CNTL__SDMA_EN_MASK                                                                              0x00000400L
+#define CP_SD_CNTL__SD_VMIDVEC_OVERRIDE_MASK                                                                  0x80000000L
+//CP_SOFT_RESET_CNTL
+#define CP_SOFT_RESET_CNTL__CMP_ONLY_SOFT_RESET__SHIFT                                                        0x0
+#define CP_SOFT_RESET_CNTL__GFX_ONLY_SOFT_RESET__SHIFT                                                        0x1
+#define CP_SOFT_RESET_CNTL__CMP_HQD_REG_RESET__SHIFT                                                          0x2
+#define CP_SOFT_RESET_CNTL__CMP_INTR_REG_RESET__SHIFT                                                         0x3
+#define CP_SOFT_RESET_CNTL__CMP_HQD_QUEUE_DOORBELL_RESET__SHIFT                                               0x4
+#define CP_SOFT_RESET_CNTL__GFX_RB_DOORBELL_RESET__SHIFT                                                      0x5
+#define CP_SOFT_RESET_CNTL__GFX_INTR_REG_RESET__SHIFT                                                         0x6
+#define CP_SOFT_RESET_CNTL__GFX_HQD_REG_RESET__SHIFT                                                          0x7
+#define CP_SOFT_RESET_CNTL__CMP_ONLY_SOFT_RESET_MASK                                                          0x00000001L
+#define CP_SOFT_RESET_CNTL__GFX_ONLY_SOFT_RESET_MASK                                                          0x00000002L
+#define CP_SOFT_RESET_CNTL__CMP_HQD_REG_RESET_MASK                                                            0x00000004L
+#define CP_SOFT_RESET_CNTL__CMP_INTR_REG_RESET_MASK                                                           0x00000008L
+#define CP_SOFT_RESET_CNTL__CMP_HQD_QUEUE_DOORBELL_RESET_MASK                                                 0x00000010L
+#define CP_SOFT_RESET_CNTL__GFX_RB_DOORBELL_RESET_MASK                                                        0x00000020L
+#define CP_SOFT_RESET_CNTL__GFX_INTR_REG_RESET_MASK                                                           0x00000040L
+#define CP_SOFT_RESET_CNTL__GFX_HQD_REG_RESET_MASK                                                            0x00000080L
+//CP_CPC_GFX_CNTL
+#define CP_CPC_GFX_CNTL__QUEUEID__SHIFT                                                                       0x0
+#define CP_CPC_GFX_CNTL__PIPEID__SHIFT                                                                        0x3
+#define CP_CPC_GFX_CNTL__MEID__SHIFT                                                                          0x5
+#define CP_CPC_GFX_CNTL__VALID__SHIFT                                                                         0x7
+#define CP_CPC_GFX_CNTL__QUEUEID_MASK                                                                         0x00000007L
+#define CP_CPC_GFX_CNTL__PIPEID_MASK                                                                          0x00000018L
+#define CP_CPC_GFX_CNTL__MEID_MASK                                                                            0x00000060L
+#define CP_CPC_GFX_CNTL__VALID_MASK                                                                           0x00000080L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_cpphqddec
+//CP_HPD_UTCL1_CNTL
+#define CP_HPD_UTCL1_CNTL__SELECT__SHIFT                                                                      0x0
+#define CP_HPD_UTCL1_CNTL__DISABLE_ERROR_REPORT__SHIFT                                                        0xa
+#define CP_HPD_UTCL1_CNTL__SELECT_MASK                                                                        0x0000000FL
+#define CP_HPD_UTCL1_CNTL__DISABLE_ERROR_REPORT_MASK                                                          0x00000400L
+//CP_HPD_UTCL1_ERROR
+#define CP_HPD_UTCL1_ERROR__ADDR_HI__SHIFT                                                                    0x0
+#define CP_HPD_UTCL1_ERROR__TYPE__SHIFT                                                                       0x1b
+#define CP_HPD_UTCL1_ERROR__VMID__SHIFT                                                                       0x1c
+#define CP_HPD_UTCL1_ERROR__ADDR_HI_MASK                                                                      0x01FFFFFFL
+#define CP_HPD_UTCL1_ERROR__TYPE_MASK                                                                         0x08000000L
+#define CP_HPD_UTCL1_ERROR__VMID_MASK                                                                         0xF0000000L
+//CP_HPD_UTCL1_ERROR_ADDR
+#define CP_HPD_UTCL1_ERROR_ADDR__ADDR__SHIFT                                                                  0xc
+#define CP_HPD_UTCL1_ERROR_ADDR__ADDR_MASK                                                                    0xFFFFF000L
+//CP_MQD_BASE_ADDR
+#define CP_MQD_BASE_ADDR__BASE_ADDR__SHIFT                                                                    0x2
+#define CP_MQD_BASE_ADDR__BASE_ADDR_MASK                                                                      0xFFFFFFFCL
+//CP_MQD_BASE_ADDR_HI
+#define CP_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                              0x0
+#define CP_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                                0x01FFFFFFL
+//CP_HQD_ACTIVE
+#define CP_HQD_ACTIVE__ACTIVE__SHIFT                                                                          0x0
+#define CP_HQD_ACTIVE__BUSY_GATE__SHIFT                                                                       0x1
+#define CP_HQD_ACTIVE__ACTIVE_MASK                                                                            0x00000001L
+#define CP_HQD_ACTIVE__BUSY_GATE_MASK                                                                         0x00000002L
+//CP_HQD_VMID
+#define CP_HQD_VMID__VMID__SHIFT                                                                              0x0
+#define CP_HQD_VMID__IB_VMID__SHIFT                                                                           0x8
+#define CP_HQD_VMID__VQID__SHIFT                                                                              0x10
+#define CP_HQD_VMID__VMID_MASK                                                                                0x0000000FL
+#define CP_HQD_VMID__IB_VMID_MASK                                                                             0x00000F00L
+#define CP_HQD_VMID__VQID_MASK                                                                                0x03FF0000L
+//CP_HQD_PERSISTENT_STATE
+#define CP_HQD_PERSISTENT_STATE__PRELOAD_REQ__SHIFT                                                           0x0
+#define CP_HQD_PERSISTENT_STATE__TMZ_CONNECT_OVERRIDE__SHIFT                                                  0x1
+#define CP_HQD_PERSISTENT_STATE__WGS_QUEUE__SHIFT                                                             0x4
+#define CP_HQD_PERSISTENT_STATE__SUSPEND_STATUS__SHIFT                                                        0x7
+#define CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT                                                          0x8
+#define CP_HQD_PERSISTENT_STATE__TMZ_SWITCH_EXEMPT__SHIFT                                                     0x12
+#define CP_HQD_PERSISTENT_STATE__TMZ_MATCH_DIS__SHIFT                                                         0x13
+#define CP_HQD_PERSISTENT_STATE__WPP_CLAMP_EN__SHIFT                                                          0x14
+#define CP_HQD_PERSISTENT_STATE__WPP_SWITCH_QOS_EN__SHIFT                                                     0x15
+#define CP_HQD_PERSISTENT_STATE__IQ_SWITCH_QOS_EN__SHIFT                                                      0x16
+#define CP_HQD_PERSISTENT_STATE__IB_SWITCH_QOS_EN__SHIFT                                                      0x17
+#define CP_HQD_PERSISTENT_STATE__EOP_SWITCH_QOS_EN__SHIFT                                                     0x18
+#define CP_HQD_PERSISTENT_STATE__PQ_SWITCH_QOS_EN__SHIFT                                                      0x19
+#define CP_HQD_PERSISTENT_STATE__TC_OFFLOAD_QOS_EN__SHIFT                                                     0x1a
+#define CP_HQD_PERSISTENT_STATE__CACHE_FULL_PACKET_EN__SHIFT                                                  0x1b
+#define CP_HQD_PERSISTENT_STATE__RESTORE_ACTIVE__SHIFT                                                        0x1c
+#define CP_HQD_PERSISTENT_STATE__RELAUNCH_WAVES__SHIFT                                                        0x1d
+#define CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT                                                          0x1e
+#define CP_HQD_PERSISTENT_STATE__DISP_ACTIVE__SHIFT                                                           0x1f
+#define CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK                                                             0x00000001L
+#define CP_HQD_PERSISTENT_STATE__TMZ_CONNECT_OVERRIDE_MASK                                                    0x00000002L
+#define CP_HQD_PERSISTENT_STATE__WGS_QUEUE_MASK                                                               0x00000010L
+#define CP_HQD_PERSISTENT_STATE__SUSPEND_STATUS_MASK                                                          0x00000080L
+#define CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE_MASK                                                            0x0003FF00L
+#define CP_HQD_PERSISTENT_STATE__TMZ_SWITCH_EXEMPT_MASK                                                       0x00040000L
+#define CP_HQD_PERSISTENT_STATE__TMZ_MATCH_DIS_MASK                                                           0x00080000L
+#define CP_HQD_PERSISTENT_STATE__WPP_CLAMP_EN_MASK                                                            0x00100000L
+#define CP_HQD_PERSISTENT_STATE__WPP_SWITCH_QOS_EN_MASK                                                       0x00200000L
+#define CP_HQD_PERSISTENT_STATE__IQ_SWITCH_QOS_EN_MASK                                                        0x00400000L
+#define CP_HQD_PERSISTENT_STATE__IB_SWITCH_QOS_EN_MASK                                                        0x00800000L
+#define CP_HQD_PERSISTENT_STATE__EOP_SWITCH_QOS_EN_MASK                                                       0x01000000L
+#define CP_HQD_PERSISTENT_STATE__PQ_SWITCH_QOS_EN_MASK                                                        0x02000000L
+#define CP_HQD_PERSISTENT_STATE__TC_OFFLOAD_QOS_EN_MASK                                                       0x04000000L
+#define CP_HQD_PERSISTENT_STATE__CACHE_FULL_PACKET_EN_MASK                                                    0x08000000L
+#define CP_HQD_PERSISTENT_STATE__RESTORE_ACTIVE_MASK                                                          0x10000000L
+#define CP_HQD_PERSISTENT_STATE__RELAUNCH_WAVES_MASK                                                          0x20000000L
+#define CP_HQD_PERSISTENT_STATE__QSWITCH_MODE_MASK                                                            0x40000000L
+#define CP_HQD_PERSISTENT_STATE__DISP_ACTIVE_MASK                                                             0x80000000L
+//CP_HQD_PIPE_PRIORITY
+#define CP_HQD_PIPE_PRIORITY__PIPE_PRIORITY__SHIFT                                                            0x0
+#define CP_HQD_PIPE_PRIORITY__PIPE_PRIORITY_MASK                                                              0x00000003L
+//CP_HQD_QUEUE_PRIORITY
+#define CP_HQD_QUEUE_PRIORITY__PRIORITY_LEVEL__SHIFT                                                          0x0
+#define CP_HQD_QUEUE_PRIORITY__PRIORITY_LEVEL_MASK                                                            0x0000000FL
+//CP_HQD_QUANTUM
+#define CP_HQD_QUANTUM__QUANTUM_EN__SHIFT                                                                     0x0
+#define CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT                                                                  0x4
+#define CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT                                                               0x8
+#define CP_HQD_QUANTUM__QUANTUM_ACTIVE__SHIFT                                                                 0x1f
+#define CP_HQD_QUANTUM__QUANTUM_EN_MASK                                                                       0x00000001L
+#define CP_HQD_QUANTUM__QUANTUM_SCALE_MASK                                                                    0x00000010L
+#define CP_HQD_QUANTUM__QUANTUM_DURATION_MASK                                                                 0x00003F00L
+#define CP_HQD_QUANTUM__QUANTUM_ACTIVE_MASK                                                                   0x80000000L
+//CP_HQD_PQ_BASE
+#define CP_HQD_PQ_BASE__ADDR__SHIFT                                                                           0x0
+#define CP_HQD_PQ_BASE__ADDR_MASK                                                                             0xFFFFFFFFL
+//CP_HQD_PQ_BASE_HI
+#define CP_HQD_PQ_BASE_HI__ADDR_HI__SHIFT                                                                     0x0
+#define CP_HQD_PQ_BASE_HI__ADDR_HI_MASK                                                                       0x0001FFFFL
+//CP_HQD_PQ_RPTR
+#define CP_HQD_PQ_RPTR__CONSUMED_OFFSET__SHIFT                                                                0x0
+#define CP_HQD_PQ_RPTR__CONSUMED_OFFSET_MASK                                                                  0xFFFFFFFFL
+//CP_HQD_PQ_RPTR_REPORT_ADDR
+#define CP_HQD_PQ_RPTR_REPORT_ADDR__RPTR_REPORT_ADDR__SHIFT                                                   0x2
+#define CP_HQD_PQ_RPTR_REPORT_ADDR__RPTR_REPORT_ADDR_MASK                                                     0xFFFFFFFCL
+//CP_HQD_PQ_RPTR_REPORT_ADDR_HI
+#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI__RPTR_REPORT_ADDR_HI__SHIFT                                             0x0
+#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI__RPTR_REPORT_ADDR_HI_MASK                                               0x01FFFFFFL
+//CP_HQD_PQ_WPTR_POLL_ADDR
+#define CP_HQD_PQ_WPTR_POLL_ADDR__WPTR_ADDR__SHIFT                                                            0x3
+#define CP_HQD_PQ_WPTR_POLL_ADDR__WPTR_ADDR_MASK                                                              0xFFFFFFF8L
+//CP_HQD_PQ_WPTR_POLL_ADDR_HI
+#define CP_HQD_PQ_WPTR_POLL_ADDR_HI__WPTR_ADDR_HI__SHIFT                                                      0x0
+#define CP_HQD_PQ_WPTR_POLL_ADDR_HI__WPTR_ADDR_HI_MASK                                                        0x01FFFFFFL
+//CP_HQD_PQ_DOORBELL_CONTROL
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_MODE__SHIFT                                                      0x0
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT                                                  0x1
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT                                                    0x2
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SOURCE__SHIFT                                                    0x1c
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SCHD_HIT__SHIFT                                                  0x1d
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN__SHIFT                                                        0x1e
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_HIT__SHIFT                                                       0x1f
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_MODE_MASK                                                        0x00000001L
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP_MASK                                                    0x00000002L
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK                                                      0x0FFFFFFCL
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SOURCE_MASK                                                      0x10000000L
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SCHD_HIT_MASK                                                    0x20000000L
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN_MASK                                                          0x40000000L
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_HIT_MASK                                                         0x80000000L
+//CP_HQD_PQ_CONTROL
+#define CP_HQD_PQ_CONTROL__QUEUE_SIZE__SHIFT                                                                  0x0
+#define CP_HQD_PQ_CONTROL__WPTR_CARRY__SHIFT                                                                  0x6
+#define CP_HQD_PQ_CONTROL__RPTR_CARRY__SHIFT                                                                  0x7
+#define CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT                                                             0x8
+#define CP_HQD_PQ_CONTROL__QUEUE_FULL_EN__SHIFT                                                               0xe
+#define CP_HQD_PQ_CONTROL__PQ_EMPTY__SHIFT                                                                    0xf
+#define CP_HQD_PQ_CONTROL__SCOPE__SHIFT                                                                       0x10
+#define CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT                                                             0x12
+#define CP_HQD_PQ_CONTROL__MIN_AVAIL_SIZE__SHIFT                                                              0x14
+#define CP_HQD_PQ_CONTROL__TMZ__SHIFT                                                                         0x16
+#define CP_HQD_PQ_CONTROL__EXE_DISABLE__SHIFT                                                                 0x17
+#define CP_HQD_PQ_CONTROL__CACHE_POLICY__SHIFT                                                                0x18
+#define CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR__SHIFT                                                              0x1b
+#define CP_HQD_PQ_CONTROL__UNORD_DISPATCH__SHIFT                                                              0x1c
+#define CP_HQD_PQ_CONTROL__TUNNEL_DISPATCH__SHIFT                                                             0x1d
+#define CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT                                                                  0x1e
+#define CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT                                                                   0x1f
+#define CP_HQD_PQ_CONTROL__QUEUE_SIZE_MASK                                                                    0x0000003FL
+#define CP_HQD_PQ_CONTROL__WPTR_CARRY_MASK                                                                    0x00000040L
+#define CP_HQD_PQ_CONTROL__RPTR_CARRY_MASK                                                                    0x00000080L
+#define CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE_MASK                                                               0x00003F00L
+#define CP_HQD_PQ_CONTROL__QUEUE_FULL_EN_MASK                                                                 0x00004000L
+#define CP_HQD_PQ_CONTROL__PQ_EMPTY_MASK                                                                      0x00008000L
+#define CP_HQD_PQ_CONTROL__SCOPE_MASK                                                                         0x00030000L
+#define CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR_MASK                                                               0x000C0000L
+#define CP_HQD_PQ_CONTROL__MIN_AVAIL_SIZE_MASK                                                                0x00300000L
+#define CP_HQD_PQ_CONTROL__TMZ_MASK                                                                           0x00400000L
+#define CP_HQD_PQ_CONTROL__EXE_DISABLE_MASK                                                                   0x00800000L
+#define CP_HQD_PQ_CONTROL__CACHE_POLICY_MASK                                                                  0x03000000L
+#define CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK                                                                0x08000000L
+#define CP_HQD_PQ_CONTROL__UNORD_DISPATCH_MASK                                                                0x10000000L
+#define CP_HQD_PQ_CONTROL__TUNNEL_DISPATCH_MASK                                                               0x20000000L
+#define CP_HQD_PQ_CONTROL__PRIV_STATE_MASK                                                                    0x40000000L
+#define CP_HQD_PQ_CONTROL__KMD_QUEUE_MASK                                                                     0x80000000L
+//CP_HQD_IB_BASE_ADDR
+#define CP_HQD_IB_BASE_ADDR__IB_BASE_ADDR__SHIFT                                                              0x2
+#define CP_HQD_IB_BASE_ADDR__IB_BASE_ADDR_MASK                                                                0xFFFFFFFCL
+//CP_HQD_IB_BASE_ADDR_HI
+#define CP_HQD_IB_BASE_ADDR_HI__IB_BASE_ADDR_HI__SHIFT                                                        0x0
+#define CP_HQD_IB_BASE_ADDR_HI__IB_BASE_ADDR_HI_MASK                                                          0x01FFFFFFL
+//CP_HQD_IB_RPTR
+#define CP_HQD_IB_RPTR__CONSUMED_OFFSET__SHIFT                                                                0x0
+#define CP_HQD_IB_RPTR__CONSUMED_OFFSET_MASK                                                                  0x000FFFFFL
+//CP_HQD_IB_CONTROL
+#define CP_HQD_IB_CONTROL__IB_SIZE__SHIFT                                                                     0x0
+#define CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT                                                           0x14
+#define CP_HQD_IB_CONTROL__IB_EXE_DISABLE__SHIFT                                                              0x17
+#define CP_HQD_IB_CONTROL__IB_CACHE_POLICY__SHIFT                                                             0x18
+#define CP_HQD_IB_CONTROL__IB_SCOPE__SHIFT                                                                    0x1a
+#define CP_HQD_IB_CONTROL__IB_PRIV_STATE__SHIFT                                                               0x1e
+#define CP_HQD_IB_CONTROL__PROCESSING_IB__SHIFT                                                               0x1f
+#define CP_HQD_IB_CONTROL__IB_SIZE_MASK                                                                       0x000FFFFFL
+#define CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE_MASK                                                             0x00300000L
+#define CP_HQD_IB_CONTROL__IB_EXE_DISABLE_MASK                                                                0x00800000L
+#define CP_HQD_IB_CONTROL__IB_CACHE_POLICY_MASK                                                               0x03000000L
+#define CP_HQD_IB_CONTROL__IB_SCOPE_MASK                                                                      0x0C000000L
+#define CP_HQD_IB_CONTROL__IB_PRIV_STATE_MASK                                                                 0x40000000L
+#define CP_HQD_IB_CONTROL__PROCESSING_IB_MASK                                                                 0x80000000L
+//CP_HQD_IQ_TIMER
+#define CP_HQD_IQ_TIMER__WAIT_TIME__SHIFT                                                                     0x0
+#define CP_HQD_IQ_TIMER__RETRY_TYPE__SHIFT                                                                    0x8
+#define CP_HQD_IQ_TIMER__IMMEDIATE_EXPIRE__SHIFT                                                              0xb
+#define CP_HQD_IQ_TIMER__INTERRUPT_TYPE__SHIFT                                                                0xc
+#define CP_HQD_IQ_TIMER__CLOCK_COUNT__SHIFT                                                                   0xe
+#define CP_HQD_IQ_TIMER__INTERRUPT_SIZE__SHIFT                                                                0x10
+#define CP_HQD_IQ_TIMER__QUANTUM_TIMER__SHIFT                                                                 0x16
+#define CP_HQD_IQ_TIMER__EXE_DISABLE__SHIFT                                                                   0x17
+#define CP_HQD_IQ_TIMER__QUEUE_TYPE__SHIFT                                                                    0x1b
+#define CP_HQD_IQ_TIMER__REARM_TIMER__SHIFT                                                                   0x1c
+#define CP_HQD_IQ_TIMER__PROCESS_IQ_EN__SHIFT                                                                 0x1d
+#define CP_HQD_IQ_TIMER__PROCESSING_IQ__SHIFT                                                                 0x1e
+#define CP_HQD_IQ_TIMER__ACTIVE__SHIFT                                                                        0x1f
+#define CP_HQD_IQ_TIMER__WAIT_TIME_MASK                                                                       0x000000FFL
+#define CP_HQD_IQ_TIMER__RETRY_TYPE_MASK                                                                      0x00000700L
+#define CP_HQD_IQ_TIMER__IMMEDIATE_EXPIRE_MASK                                                                0x00000800L
+#define CP_HQD_IQ_TIMER__INTERRUPT_TYPE_MASK                                                                  0x00003000L
+#define CP_HQD_IQ_TIMER__CLOCK_COUNT_MASK                                                                     0x0000C000L
+#define CP_HQD_IQ_TIMER__INTERRUPT_SIZE_MASK                                                                  0x003F0000L
+#define CP_HQD_IQ_TIMER__QUANTUM_TIMER_MASK                                                                   0x00400000L
+#define CP_HQD_IQ_TIMER__EXE_DISABLE_MASK                                                                     0x00800000L
+#define CP_HQD_IQ_TIMER__QUEUE_TYPE_MASK                                                                      0x08000000L
+#define CP_HQD_IQ_TIMER__REARM_TIMER_MASK                                                                     0x10000000L
+#define CP_HQD_IQ_TIMER__PROCESS_IQ_EN_MASK                                                                   0x20000000L
+#define CP_HQD_IQ_TIMER__PROCESSING_IQ_MASK                                                                   0x40000000L
+#define CP_HQD_IQ_TIMER__ACTIVE_MASK                                                                          0x80000000L
+//CP_HQD_IQ_RPTR
+#define CP_HQD_IQ_RPTR__OFFSET__SHIFT                                                                         0x0
+#define CP_HQD_IQ_RPTR__OFFSET_MASK                                                                           0x0000003FL
+//CP_HQD_DEQUEUE_REQUEST
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ__SHIFT                                                            0x0
+#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND__SHIFT                                                            0x4
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_INT__SHIFT                                                            0x8
+#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_EN__SHIFT                                                         0x9
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_EN__SHIFT                                                         0xa
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_MASK                                                              0x0000000FL
+#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK                                                              0x00000010L
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_INT_MASK                                                              0x00000100L
+#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_EN_MASK                                                           0x00000200L
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_EN_MASK                                                           0x00000400L
+//CP_HQD_DMA_OFFLOAD
+#define CP_HQD_DMA_OFFLOAD__DMA_OFFLOAD__SHIFT                                                                0x0
+#define CP_HQD_DMA_OFFLOAD__DMA_OFFLOAD_EN__SHIFT                                                             0x1
+#define CP_HQD_DMA_OFFLOAD__AQL_OFFLOAD__SHIFT                                                                0x2
+#define CP_HQD_DMA_OFFLOAD__AQL_OFFLOAD_EN__SHIFT                                                             0x3
+#define CP_HQD_DMA_OFFLOAD__EOP_OFFLOAD__SHIFT                                                                0x4
+#define CP_HQD_DMA_OFFLOAD__EOP_OFFLOAD_EN__SHIFT                                                             0x5
+#define CP_HQD_DMA_OFFLOAD__DMA_OFFLOAD_MASK                                                                  0x00000001L
+#define CP_HQD_DMA_OFFLOAD__DMA_OFFLOAD_EN_MASK                                                               0x00000002L
+#define CP_HQD_DMA_OFFLOAD__AQL_OFFLOAD_MASK                                                                  0x00000004L
+#define CP_HQD_DMA_OFFLOAD__AQL_OFFLOAD_EN_MASK                                                               0x00000008L
+#define CP_HQD_DMA_OFFLOAD__EOP_OFFLOAD_MASK                                                                  0x00000010L
+#define CP_HQD_DMA_OFFLOAD__EOP_OFFLOAD_EN_MASK                                                               0x00000020L
+//CP_HQD_OFFLOAD
+#define CP_HQD_OFFLOAD__DMA_OFFLOAD__SHIFT                                                                    0x0
+#define CP_HQD_OFFLOAD__DMA_OFFLOAD_EN__SHIFT                                                                 0x1
+#define CP_HQD_OFFLOAD__AQL_OFFLOAD__SHIFT                                                                    0x2
+#define CP_HQD_OFFLOAD__AQL_OFFLOAD_EN__SHIFT                                                                 0x3
+#define CP_HQD_OFFLOAD__EOP_OFFLOAD__SHIFT                                                                    0x4
+#define CP_HQD_OFFLOAD__EOP_OFFLOAD_EN__SHIFT                                                                 0x5
+#define CP_HQD_OFFLOAD__DMA_OFFLOAD_MASK                                                                      0x00000001L
+#define CP_HQD_OFFLOAD__DMA_OFFLOAD_EN_MASK                                                                   0x00000002L
+#define CP_HQD_OFFLOAD__AQL_OFFLOAD_MASK                                                                      0x00000004L
+#define CP_HQD_OFFLOAD__AQL_OFFLOAD_EN_MASK                                                                   0x00000008L
+#define CP_HQD_OFFLOAD__EOP_OFFLOAD_MASK                                                                      0x00000010L
+#define CP_HQD_OFFLOAD__EOP_OFFLOAD_EN_MASK                                                                   0x00000020L
+//CP_HQD_MSG_TYPE
+#define CP_HQD_MSG_TYPE__ACTION__SHIFT                                                                        0x0
+#define CP_HQD_MSG_TYPE__SAVE_STATE__SHIFT                                                                    0x4
+#define CP_HQD_MSG_TYPE__ACTION_MASK                                                                          0x00000007L
+#define CP_HQD_MSG_TYPE__SAVE_STATE_MASK                                                                      0x00000070L
+//CP_HQD_ATOMIC0_PREOP_LO
+#define CP_HQD_ATOMIC0_PREOP_LO__ATOMIC0_PREOP_LO__SHIFT                                                      0x0
+#define CP_HQD_ATOMIC0_PREOP_LO__ATOMIC0_PREOP_LO_MASK                                                        0xFFFFFFFFL
+//CP_HQD_ATOMIC0_PREOP_HI
+#define CP_HQD_ATOMIC0_PREOP_HI__ATOMIC0_PREOP_HI__SHIFT                                                      0x0
+#define CP_HQD_ATOMIC0_PREOP_HI__ATOMIC0_PREOP_HI_MASK                                                        0xFFFFFFFFL
+//CP_HQD_ATOMIC1_PREOP_LO
+#define CP_HQD_ATOMIC1_PREOP_LO__ATOMIC1_PREOP_LO__SHIFT                                                      0x0
+#define CP_HQD_ATOMIC1_PREOP_LO__ATOMIC1_PREOP_LO_MASK                                                        0xFFFFFFFFL
+//CP_HQD_ATOMIC1_PREOP_HI
+#define CP_HQD_ATOMIC1_PREOP_HI__ATOMIC1_PREOP_HI__SHIFT                                                      0x0
+#define CP_HQD_ATOMIC1_PREOP_HI__ATOMIC1_PREOP_HI_MASK                                                        0xFFFFFFFFL
+//CP_HQD_HQ_SCHEDULER0
+#define CP_HQD_HQ_SCHEDULER0__CWSR__SHIFT                                                                     0x0
+#define CP_HQD_HQ_SCHEDULER0__SAVE_STATUS__SHIFT                                                              0x1
+#define CP_HQD_HQ_SCHEDULER0__RSRV__SHIFT                                                                     0x2
+#define CP_HQD_HQ_SCHEDULER0__STATIC_QUEUE__SHIFT                                                             0x3
+#define CP_HQD_HQ_SCHEDULER0__QUEUE_RUN_ONCE__SHIFT                                                           0x6
+#define CP_HQD_HQ_SCHEDULER0__SCRATCH_RAM_INIT__SHIFT                                                         0x7
+#define CP_HQD_HQ_SCHEDULER0__TCL2_DIRTY__SHIFT                                                               0x8
+#define CP_HQD_HQ_SCHEDULER0__C_INHERIT_VMID__SHIFT                                                           0x9
+#define CP_HQD_HQ_SCHEDULER0__QUEUE_SCHEDULER_TYPE__SHIFT                                                     0xa
+#define CP_HQD_HQ_SCHEDULER0__C_QUEUE_USE_GWS__SHIFT                                                          0xd
+#define CP_HQD_HQ_SCHEDULER0__C_QUEUE_DEBUG_EN__SHIFT                                                         0xe
+#define CP_HQD_HQ_SCHEDULER0__QUEUE_SLOT_CONNECTED__SHIFT                                                     0xf
+#define CP_HQD_HQ_SCHEDULER0__MES_INTERRUPT_ENABLED__SHIFT                                                    0x14
+#define CP_HQD_HQ_SCHEDULER0__MES_INTERRUPT_PIPE__SHIFT                                                       0x15
+#define CP_HQD_HQ_SCHEDULER0__CONCURRENT_PROCESS_COUNT__SHIFT                                                 0x18
+#define CP_HQD_HQ_SCHEDULER0__QUEUE_IDLE__SHIFT                                                               0x1e
+#define CP_HQD_HQ_SCHEDULER0__DB_UPDATED_MSG_EN__SHIFT                                                        0x1f
+#define CP_HQD_HQ_SCHEDULER0__CWSR_MASK                                                                       0x00000001L
+#define CP_HQD_HQ_SCHEDULER0__SAVE_STATUS_MASK                                                                0x00000002L
+#define CP_HQD_HQ_SCHEDULER0__RSRV_MASK                                                                       0x00000004L
+#define CP_HQD_HQ_SCHEDULER0__STATIC_QUEUE_MASK                                                               0x00000038L
+#define CP_HQD_HQ_SCHEDULER0__QUEUE_RUN_ONCE_MASK                                                             0x00000040L
+#define CP_HQD_HQ_SCHEDULER0__SCRATCH_RAM_INIT_MASK                                                           0x00000080L
+#define CP_HQD_HQ_SCHEDULER0__TCL2_DIRTY_MASK                                                                 0x00000100L
+#define CP_HQD_HQ_SCHEDULER0__C_INHERIT_VMID_MASK                                                             0x00000200L
+#define CP_HQD_HQ_SCHEDULER0__QUEUE_SCHEDULER_TYPE_MASK                                                       0x00001C00L
+#define CP_HQD_HQ_SCHEDULER0__C_QUEUE_USE_GWS_MASK                                                            0x00002000L
+#define CP_HQD_HQ_SCHEDULER0__C_QUEUE_DEBUG_EN_MASK                                                           0x00004000L
+#define CP_HQD_HQ_SCHEDULER0__QUEUE_SLOT_CONNECTED_MASK                                                       0x00008000L
+#define CP_HQD_HQ_SCHEDULER0__MES_INTERRUPT_ENABLED_MASK                                                      0x00100000L
+#define CP_HQD_HQ_SCHEDULER0__MES_INTERRUPT_PIPE_MASK                                                         0x00600000L
+#define CP_HQD_HQ_SCHEDULER0__CONCURRENT_PROCESS_COUNT_MASK                                                   0x0F000000L
+#define CP_HQD_HQ_SCHEDULER0__QUEUE_IDLE_MASK                                                                 0x40000000L
+#define CP_HQD_HQ_SCHEDULER0__DB_UPDATED_MSG_EN_MASK                                                          0x80000000L
+//CP_HQD_HQ_STATUS0
+#define CP_HQD_HQ_STATUS0__CWSR__SHIFT                                                                        0x0
+#define CP_HQD_HQ_STATUS0__SAVE_STATUS__SHIFT                                                                 0x1
+#define CP_HQD_HQ_STATUS0__RSRV__SHIFT                                                                        0x2
+#define CP_HQD_HQ_STATUS0__STATIC_QUEUE__SHIFT                                                                0x3
+#define CP_HQD_HQ_STATUS0__QUEUE_RUN_ONCE__SHIFT                                                              0x6
+#define CP_HQD_HQ_STATUS0__SCRATCH_RAM_INIT__SHIFT                                                            0x7
+#define CP_HQD_HQ_STATUS0__TCL2_DIRTY__SHIFT                                                                  0x8
+#define CP_HQD_HQ_STATUS0__C_INHERIT_VMID__SHIFT                                                              0x9
+#define CP_HQD_HQ_STATUS0__QUEUE_SCHEDULER_TYPE__SHIFT                                                        0xa
+#define CP_HQD_HQ_STATUS0__C_QUEUE_USE_GWS__SHIFT                                                             0xd
+#define CP_HQD_HQ_STATUS0__C_QUEUE_DEBUG_EN__SHIFT                                                            0xe
+#define CP_HQD_HQ_STATUS0__QUEUE_SLOT_CONNECTED__SHIFT                                                        0xf
+#define CP_HQD_HQ_STATUS0__MES_INTERRUPT_ENABLED__SHIFT                                                       0x14
+#define CP_HQD_HQ_STATUS0__MES_INTERRUPT_PIPE__SHIFT                                                          0x15
+#define CP_HQD_HQ_STATUS0__DFWX_QUEUE_SUPPORTED__SHIFT                                                        0x17
+#define CP_HQD_HQ_STATUS0__CONCURRENT_PROCESS_COUNT__SHIFT                                                    0x18
+#define CP_HQD_HQ_STATUS0__PLATFORM_ATOMICS_SUPPORTED__SHIFT                                                  0x1d
+#define CP_HQD_HQ_STATUS0__QUEUE_IDLE__SHIFT                                                                  0x1e
+#define CP_HQD_HQ_STATUS0__DB_UPDATED_MSG_EN__SHIFT                                                           0x1f
+#define CP_HQD_HQ_STATUS0__CWSR_MASK                                                                          0x00000001L
+#define CP_HQD_HQ_STATUS0__SAVE_STATUS_MASK                                                                   0x00000002L
+#define CP_HQD_HQ_STATUS0__RSRV_MASK                                                                          0x00000004L
+#define CP_HQD_HQ_STATUS0__STATIC_QUEUE_MASK                                                                  0x00000038L
+#define CP_HQD_HQ_STATUS0__QUEUE_RUN_ONCE_MASK                                                                0x00000040L
+#define CP_HQD_HQ_STATUS0__SCRATCH_RAM_INIT_MASK                                                              0x00000080L
+#define CP_HQD_HQ_STATUS0__TCL2_DIRTY_MASK                                                                    0x00000100L
+#define CP_HQD_HQ_STATUS0__C_INHERIT_VMID_MASK                                                                0x00000200L
+#define CP_HQD_HQ_STATUS0__QUEUE_SCHEDULER_TYPE_MASK                                                          0x00001C00L
+#define CP_HQD_HQ_STATUS0__C_QUEUE_USE_GWS_MASK                                                               0x00002000L
+#define CP_HQD_HQ_STATUS0__C_QUEUE_DEBUG_EN_MASK                                                              0x00004000L
+#define CP_HQD_HQ_STATUS0__QUEUE_SLOT_CONNECTED_MASK                                                          0x00008000L
+#define CP_HQD_HQ_STATUS0__MES_INTERRUPT_ENABLED_MASK                                                         0x00100000L
+#define CP_HQD_HQ_STATUS0__MES_INTERRUPT_PIPE_MASK                                                            0x00600000L
+#define CP_HQD_HQ_STATUS0__DFWX_QUEUE_SUPPORTED_MASK                                                          0x00800000L
+#define CP_HQD_HQ_STATUS0__CONCURRENT_PROCESS_COUNT_MASK                                                      0x0F000000L
+#define CP_HQD_HQ_STATUS0__PLATFORM_ATOMICS_SUPPORTED_MASK                                                    0x20000000L
+#define CP_HQD_HQ_STATUS0__QUEUE_IDLE_MASK                                                                    0x40000000L
+#define CP_HQD_HQ_STATUS0__DB_UPDATED_MSG_EN_MASK                                                             0x80000000L
+//CP_HQD_HQ_CONTROL0
+#define CP_HQD_HQ_CONTROL0__CONTROL__SHIFT                                                                    0x0
+#define CP_HQD_HQ_CONTROL0__CONTROL_MASK                                                                      0xFFFFFFFFL
+//CP_HQD_HQ_SCHEDULER1
+#define CP_HQD_HQ_SCHEDULER1__SCHEDULER__SHIFT                                                                0x0
+#define CP_HQD_HQ_SCHEDULER1__SCHEDULER_MASK                                                                  0xFFFFFFFFL
+//CP_MQD_CONTROL
+#define CP_MQD_CONTROL__VMID__SHIFT                                                                           0x0
+#define CP_MQD_CONTROL__PRIV_STATE__SHIFT                                                                     0x8
+#define CP_MQD_CONTROL__PROCESSING_MQD__SHIFT                                                                 0xc
+#define CP_MQD_CONTROL__PROCESSING_MQD_EN__SHIFT                                                              0xd
+#define CP_MQD_CONTROL__EXE_DISABLE__SHIFT                                                                    0x17
+#define CP_MQD_CONTROL__CACHE_POLICY__SHIFT                                                                   0x18
+#define CP_MQD_CONTROL__SCOPE__SHIFT                                                                          0x1a
+#define CP_MQD_CONTROL__VMID_MASK                                                                             0x0000000FL
+#define CP_MQD_CONTROL__PRIV_STATE_MASK                                                                       0x00000100L
+#define CP_MQD_CONTROL__PROCESSING_MQD_MASK                                                                   0x00001000L
+#define CP_MQD_CONTROL__PROCESSING_MQD_EN_MASK                                                                0x00002000L
+#define CP_MQD_CONTROL__EXE_DISABLE_MASK                                                                      0x00800000L
+#define CP_MQD_CONTROL__CACHE_POLICY_MASK                                                                     0x03000000L
+#define CP_MQD_CONTROL__SCOPE_MASK                                                                            0x0C000000L
+//CP_HQD_HQ_STATUS1
+#define CP_HQD_HQ_STATUS1__STATUS__SHIFT                                                                      0x0
+#define CP_HQD_HQ_STATUS1__STATUS_MASK                                                                        0xFFFFFFFFL
+//CP_HQD_HQ_CONTROL1
+#define CP_HQD_HQ_CONTROL1__CONTROL__SHIFT                                                                    0x0
+#define CP_HQD_HQ_CONTROL1__CONTROL_MASK                                                                      0xFFFFFFFFL
+//CP_HQD_EOP_BASE_ADDR
+#define CP_HQD_EOP_BASE_ADDR__BASE_ADDR__SHIFT                                                                0x0
+#define CP_HQD_EOP_BASE_ADDR__BASE_ADDR_MASK                                                                  0xFFFFFFFFL
+//CP_HQD_EOP_BASE_ADDR_HI
+#define CP_HQD_EOP_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                          0x0
+#define CP_HQD_EOP_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                            0x0001FFFFL
+//CP_HQD_EOP_CONTROL
+#define CP_HQD_EOP_CONTROL__EOP_SIZE__SHIFT                                                                   0x0
+#define CP_HQD_EOP_CONTROL__PROCESSING_EOP__SHIFT                                                             0x8
+#define CP_HQD_EOP_CONTROL__PROCESS_EOP_EN__SHIFT                                                             0xc
+#define CP_HQD_EOP_CONTROL__PROCESSING_EOPIB__SHIFT                                                           0xd
+#define CP_HQD_EOP_CONTROL__PROCESS_EOPIB_EN__SHIFT                                                           0xe
+#define CP_HQD_EOP_CONTROL__SCOPE__SHIFT                                                                      0x13
+#define CP_HQD_EOP_CONTROL__HALT_FETCHER__SHIFT                                                               0x15
+#define CP_HQD_EOP_CONTROL__HALT_FETCHER_EN__SHIFT                                                            0x16
+#define CP_HQD_EOP_CONTROL__EXE_DISABLE__SHIFT                                                                0x17
+#define CP_HQD_EOP_CONTROL__CACHE_POLICY__SHIFT                                                               0x18
+#define CP_HQD_EOP_CONTROL__EOP_SIZE_MASK                                                                     0x0000003FL
+#define CP_HQD_EOP_CONTROL__PROCESSING_EOP_MASK                                                               0x00000100L
+#define CP_HQD_EOP_CONTROL__PROCESS_EOP_EN_MASK                                                               0x00001000L
+#define CP_HQD_EOP_CONTROL__PROCESSING_EOPIB_MASK                                                             0x00002000L
+#define CP_HQD_EOP_CONTROL__PROCESS_EOPIB_EN_MASK                                                             0x00004000L
+#define CP_HQD_EOP_CONTROL__SCOPE_MASK                                                                        0x00180000L
+#define CP_HQD_EOP_CONTROL__HALT_FETCHER_MASK                                                                 0x00200000L
+#define CP_HQD_EOP_CONTROL__HALT_FETCHER_EN_MASK                                                              0x00400000L
+#define CP_HQD_EOP_CONTROL__EXE_DISABLE_MASK                                                                  0x00800000L
+#define CP_HQD_EOP_CONTROL__CACHE_POLICY_MASK                                                                 0x03000000L
+//CP_HQD_EOP_RPTR
+#define CP_HQD_EOP_RPTR__RPTR__SHIFT                                                                          0x0
+#define CP_HQD_EOP_RPTR__RESET_FETCHER__SHIFT                                                                 0x1c
+#define CP_HQD_EOP_RPTR__DEQUEUE_PEND__SHIFT                                                                  0x1d
+#define CP_HQD_EOP_RPTR__RPTR_EQ_CSMD_WPTR__SHIFT                                                             0x1e
+#define CP_HQD_EOP_RPTR__INIT_FETCHER__SHIFT                                                                  0x1f
+#define CP_HQD_EOP_RPTR__RPTR_MASK                                                                            0x00001FFFL
+#define CP_HQD_EOP_RPTR__RESET_FETCHER_MASK                                                                   0x10000000L
+#define CP_HQD_EOP_RPTR__DEQUEUE_PEND_MASK                                                                    0x20000000L
+#define CP_HQD_EOP_RPTR__RPTR_EQ_CSMD_WPTR_MASK                                                               0x40000000L
+#define CP_HQD_EOP_RPTR__INIT_FETCHER_MASK                                                                    0x80000000L
+//CP_HQD_EOP_WPTR
+#define CP_HQD_EOP_WPTR__WPTR__SHIFT                                                                          0x0
+#define CP_HQD_EOP_WPTR__EOP_EMPTY__SHIFT                                                                     0xf
+#define CP_HQD_EOP_WPTR__EOP_AVAIL__SHIFT                                                                     0x10
+#define CP_HQD_EOP_WPTR__WPTR_MASK                                                                            0x00001FFFL
+#define CP_HQD_EOP_WPTR__EOP_EMPTY_MASK                                                                       0x00008000L
+#define CP_HQD_EOP_WPTR__EOP_AVAIL_MASK                                                                       0x1FFF0000L
+//CP_HQD_EOP_EVENTS
+#define CP_HQD_EOP_EVENTS__EVENT_COUNT__SHIFT                                                                 0x0
+#define CP_HQD_EOP_EVENTS__RESTORE_EOP__SHIFT                                                                 0xf
+#define CP_HQD_EOP_EVENTS__CS_PARTIAL_FLUSH_PEND__SHIFT                                                       0x10
+#define CP_HQD_EOP_EVENTS__EVENT_COUNT_MASK                                                                   0x00000FFFL
+#define CP_HQD_EOP_EVENTS__RESTORE_EOP_MASK                                                                   0x00008000L
+#define CP_HQD_EOP_EVENTS__CS_PARTIAL_FLUSH_PEND_MASK                                                         0x00010000L
+//CP_HQD_CTX_SAVE_BASE_ADDR_LO
+#define CP_HQD_CTX_SAVE_BASE_ADDR_LO__ADDR__SHIFT                                                             0xc
+#define CP_HQD_CTX_SAVE_BASE_ADDR_LO__ADDR_MASK                                                               0xFFFFF000L
+//CP_HQD_CTX_SAVE_BASE_ADDR_HI
+#define CP_HQD_CTX_SAVE_BASE_ADDR_HI__ADDR_HI__SHIFT                                                          0x0
+#define CP_HQD_CTX_SAVE_BASE_ADDR_HI__ADDR_HI_MASK                                                            0x01FFFFFFL
+//CP_HQD_CTX_SAVE_CONTROL
+#define CP_HQD_CTX_SAVE_CONTROL__POLICY__SHIFT                                                                0x3
+#define CP_HQD_CTX_SAVE_CONTROL__SCOPE__SHIFT                                                                 0x8
+#define CP_HQD_CTX_SAVE_CONTROL__EXE_DISABLE__SHIFT                                                           0x17
+#define CP_HQD_CTX_SAVE_CONTROL__POLICY_MASK                                                                  0x00000018L
+#define CP_HQD_CTX_SAVE_CONTROL__SCOPE_MASK                                                                   0x00000300L
+#define CP_HQD_CTX_SAVE_CONTROL__EXE_DISABLE_MASK                                                             0x00800000L
+//CP_HQD_CNTL_STACK_OFFSET
+#define CP_HQD_CNTL_STACK_OFFSET__OFFSET__SHIFT                                                               0x2
+#define CP_HQD_CNTL_STACK_OFFSET__OFFSET_MASK                                                                 0x0000FFFCL
+//CP_HQD_CNTL_STACK_SIZE
+#define CP_HQD_CNTL_STACK_SIZE__SIZE__SHIFT                                                                   0xc
+#define CP_HQD_CNTL_STACK_SIZE__SIZE_MASK                                                                     0x0000F000L
+//CP_HQD_WG_STATE_OFFSET
+#define CP_HQD_WG_STATE_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define CP_HQD_WG_STATE_OFFSET__OFFSET_MASK                                                                   0x03FFFFFCL
+//CP_HQD_CTX_SAVE_SIZE
+#define CP_HQD_CTX_SAVE_SIZE__SIZE__SHIFT                                                                     0xc
+#define CP_HQD_CTX_SAVE_SIZE__SIZE_MASK                                                                       0x03FFF000L
+//CP_HQD_GDS_RESOURCE_STATE
+#define CP_HQD_GDS_RESOURCE_STATE__OA_REQUIRED__SHIFT                                                         0x0
+#define CP_HQD_GDS_RESOURCE_STATE__OA_ACQUIRED__SHIFT                                                         0x1
+#define CP_HQD_GDS_RESOURCE_STATE__GWS_SIZE__SHIFT                                                            0x4
+#define CP_HQD_GDS_RESOURCE_STATE__GWS_PNTR__SHIFT                                                            0xc
+#define CP_HQD_GDS_RESOURCE_STATE__OA_REQUIRED_MASK                                                           0x00000001L
+#define CP_HQD_GDS_RESOURCE_STATE__OA_ACQUIRED_MASK                                                           0x00000002L
+#define CP_HQD_GDS_RESOURCE_STATE__GWS_SIZE_MASK                                                              0x000003F0L
+#define CP_HQD_GDS_RESOURCE_STATE__GWS_PNTR_MASK                                                              0x0003F000L
+//CP_HQD_ERROR
+#define CP_HQD_ERROR__EDC_ERROR_ID__SHIFT                                                                     0x0
+#define CP_HQD_ERROR__SUA_ERROR__SHIFT                                                                        0x4
+#define CP_HQD_ERROR__AQL_ERROR__SHIFT                                                                        0x5
+#define CP_HQD_ERROR__PQ_UTCL1_ERROR__SHIFT                                                                   0x8
+#define CP_HQD_ERROR__IB_UTCL1_ERROR__SHIFT                                                                   0x9
+#define CP_HQD_ERROR__EOP_UTCL1_ERROR__SHIFT                                                                  0xa
+#define CP_HQD_ERROR__IQ_UTCL1_ERROR__SHIFT                                                                   0xb
+#define CP_HQD_ERROR__RRPT_UTCL1_ERROR__SHIFT                                                                 0xc
+#define CP_HQD_ERROR__WPP_UTCL1_ERROR__SHIFT                                                                  0xd
+#define CP_HQD_ERROR__DMA_SRC_UTCL1_ERROR__SHIFT                                                              0xf
+#define CP_HQD_ERROR__DMA_DST_UTCL1_ERROR__SHIFT                                                              0x10
+#define CP_HQD_ERROR__SR_UTCL1_ERROR__SHIFT                                                                   0x11
+#define CP_HQD_ERROR__QU_UTCL1_ERROR__SHIFT                                                                   0x12
+#define CP_HQD_ERROR__TC_UTCL1_ERROR__SHIFT                                                                   0x13
+#define CP_HQD_ERROR__EDC_ERROR_ID_MASK                                                                       0x0000000FL
+#define CP_HQD_ERROR__SUA_ERROR_MASK                                                                          0x00000010L
+#define CP_HQD_ERROR__AQL_ERROR_MASK                                                                          0x00000020L
+#define CP_HQD_ERROR__PQ_UTCL1_ERROR_MASK                                                                     0x00000100L
+#define CP_HQD_ERROR__IB_UTCL1_ERROR_MASK                                                                     0x00000200L
+#define CP_HQD_ERROR__EOP_UTCL1_ERROR_MASK                                                                    0x00000400L
+#define CP_HQD_ERROR__IQ_UTCL1_ERROR_MASK                                                                     0x00000800L
+#define CP_HQD_ERROR__RRPT_UTCL1_ERROR_MASK                                                                   0x00001000L
+#define CP_HQD_ERROR__WPP_UTCL1_ERROR_MASK                                                                    0x00002000L
+#define CP_HQD_ERROR__DMA_SRC_UTCL1_ERROR_MASK                                                                0x00008000L
+#define CP_HQD_ERROR__DMA_DST_UTCL1_ERROR_MASK                                                                0x00010000L
+#define CP_HQD_ERROR__SR_UTCL1_ERROR_MASK                                                                     0x00020000L
+#define CP_HQD_ERROR__QU_UTCL1_ERROR_MASK                                                                     0x00040000L
+#define CP_HQD_ERROR__TC_UTCL1_ERROR_MASK                                                                     0x00080000L
+//CP_HQD_EOP_WPTR_MEM
+#define CP_HQD_EOP_WPTR_MEM__WPTR__SHIFT                                                                      0x0
+#define CP_HQD_EOP_WPTR_MEM__WPTR_MASK                                                                        0x00001FFFL
+//CP_HQD_AQL_CONTROL
+#define CP_HQD_AQL_CONTROL__CONTROL0__SHIFT                                                                   0x0
+#define CP_HQD_AQL_CONTROL__CONTROL0_EN__SHIFT                                                                0xf
+#define CP_HQD_AQL_CONTROL__CONTROL1__SHIFT                                                                   0x10
+#define CP_HQD_AQL_CONTROL__CONTROL1_EN__SHIFT                                                                0x1f
+#define CP_HQD_AQL_CONTROL__CONTROL0_MASK                                                                     0x00007FFFL
+#define CP_HQD_AQL_CONTROL__CONTROL0_EN_MASK                                                                  0x00008000L
+#define CP_HQD_AQL_CONTROL__CONTROL1_MASK                                                                     0x7FFF0000L
+#define CP_HQD_AQL_CONTROL__CONTROL1_EN_MASK                                                                  0x80000000L
+//CP_HQD_PQ_WPTR_LO
+#define CP_HQD_PQ_WPTR_LO__OFFSET__SHIFT                                                                      0x0
+#define CP_HQD_PQ_WPTR_LO__OFFSET_MASK                                                                        0xFFFFFFFFL
+//CP_HQD_PQ_WPTR_HI
+#define CP_HQD_PQ_WPTR_HI__DATA__SHIFT                                                                        0x0
+#define CP_HQD_PQ_WPTR_HI__DATA_MASK                                                                          0xFFFFFFFFL
+//CP_HQD_SUSPEND_CNTL_STACK_OFFSET
+#define CP_HQD_SUSPEND_CNTL_STACK_OFFSET__OFFSET__SHIFT                                                       0x2
+#define CP_HQD_SUSPEND_CNTL_STACK_OFFSET__OFFSET_MASK                                                         0x0000FFFCL
+//CP_HQD_SUSPEND_CNTL_STACK_DW_CNT
+#define CP_HQD_SUSPEND_CNTL_STACK_DW_CNT__CNT__SHIFT                                                          0x0
+#define CP_HQD_SUSPEND_CNTL_STACK_DW_CNT__CNT_MASK                                                            0x00003FFFL
+//CP_HQD_SUSPEND_WG_STATE_OFFSET
+#define CP_HQD_SUSPEND_WG_STATE_OFFSET__OFFSET__SHIFT                                                         0x2
+#define CP_HQD_SUSPEND_WG_STATE_OFFSET__OFFSET_MASK                                                           0x03FFFFFCL
+//CP_HQD_DDID_RPTR
+#define CP_HQD_DDID_RPTR__RPTR__SHIFT                                                                         0x0
+#define CP_HQD_DDID_RPTR__RPTR_MASK                                                                           0x000007FFL
+//CP_HQD_DDID_WPTR
+#define CP_HQD_DDID_WPTR__WPTR__SHIFT                                                                         0x0
+#define CP_HQD_DDID_WPTR__WPTR_MASK                                                                           0x000007FFL
+//CP_HQD_DDID_INFLIGHT_COUNT
+#define CP_HQD_DDID_INFLIGHT_COUNT__COUNT__SHIFT                                                              0x0
+#define CP_HQD_DDID_INFLIGHT_COUNT__COUNT_MASK                                                                0x0000FFFFL
+//CP_HQD_DDID_DELTA_RPT_COUNT
+#define CP_HQD_DDID_DELTA_RPT_COUNT__COUNT__SHIFT                                                             0x0
+#define CP_HQD_DDID_DELTA_RPT_COUNT__COUNT_MASK                                                               0x000000FFL
+//CP_HQD_DEQUEUE_STATUS
+#define CP_HQD_DEQUEUE_STATUS__DEQUEUE_STAT__SHIFT                                                            0x0
+#define CP_HQD_DEQUEUE_STATUS__SUSPEND_REQ_PEND__SHIFT                                                        0x4
+#define CP_HQD_DEQUEUE_STATUS__SUSPEND_REQ_PEND_EN__SHIFT                                                     0x9
+#define CP_HQD_DEQUEUE_STATUS__DEQUEUE_STAT_EN__SHIFT                                                         0xa
+#define CP_HQD_DEQUEUE_STATUS__DEQUEUE_STAT_MASK                                                              0x0000000FL
+#define CP_HQD_DEQUEUE_STATUS__SUSPEND_REQ_PEND_MASK                                                          0x00000010L
+#define CP_HQD_DEQUEUE_STATUS__SUSPEND_REQ_PEND_EN_MASK                                                       0x00000200L
+#define CP_HQD_DEQUEUE_STATUS__DEQUEUE_STAT_EN_MASK                                                           0x00000400L
+//CP_HQD_AQL_CONTROL_1
+#define CP_HQD_AQL_CONTROL_1__RESERVED__SHIFT                                                                 0x0
+#define CP_HQD_AQL_CONTROL_1__RESERVED_MASK                                                                   0xFFFFFFFFL
+//CP_HQD_AQL_DISPATCH_ID
+#define CP_HQD_AQL_DISPATCH_ID__CONSUMED_OFFSET__SHIFT                                                        0x0
+#define CP_HQD_AQL_DISPATCH_ID__CONSUMED_OFFSET_MASK                                                          0xFFFFFFFFL
+//CP_HQD_AQL_DISPATCH_ID_HI
+#define CP_HQD_AQL_DISPATCH_ID_HI__CONSUMED_OFFSET__SHIFT                                                     0x0
+#define CP_HQD_AQL_DISPATCH_ID_HI__CONSUMED_OFFSET_MASK                                                       0xFFFFFFFFL
+//CP_HQD_KD_BASE
+#define CP_HQD_KD_BASE__ADDR__SHIFT                                                                           0x0
+#define CP_HQD_KD_BASE__ADDR_MASK                                                                             0xFFFFFFFFL
+//CP_HQD_KD_BASE_HI
+#define CP_HQD_KD_BASE_HI__ADDR_HI__SHIFT                                                                     0x0
+#define CP_HQD_KD_BASE_HI__ADDR_HI_MASK                                                                       0x0001FFFFL
+//CP_HQD_KD_CNTL
+#define CP_HQD_KD_CNTL__KD_SIZE__SHIFT                                                                        0x0
+#define CP_HQD_KD_CNTL__KD_FETCHER_ENABLE__SHIFT                                                              0x10
+#define CP_HQD_KD_CNTL__EXE_DISABLE__SHIFT                                                                    0x17
+#define CP_HQD_KD_CNTL__CACHE_POLICY__SHIFT                                                                   0x18
+#define CP_HQD_KD_CNTL__SCOPE__SHIFT                                                                          0x1a
+#define CP_HQD_KD_CNTL__PRIV_STATE__SHIFT                                                                     0x1e
+#define CP_HQD_KD_CNTL__KD_SIZE_MASK                                                                          0x00000003L
+#define CP_HQD_KD_CNTL__KD_FETCHER_ENABLE_MASK                                                                0x00010000L
+#define CP_HQD_KD_CNTL__EXE_DISABLE_MASK                                                                      0x00800000L
+#define CP_HQD_KD_CNTL__CACHE_POLICY_MASK                                                                     0x03000000L
+#define CP_HQD_KD_CNTL__SCOPE_MASK                                                                            0x0C000000L
+#define CP_HQD_KD_CNTL__PRIV_STATE_MASK                                                                       0x40000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_gfxdec0
+//COHER_DEST_BASE_HI_0
+#define COHER_DEST_BASE_HI_0__DEST_BASE_HI_256B__SHIFT                                                        0x0
+#define COHER_DEST_BASE_HI_0__DEST_BASE_HI_256B_MASK                                                          0x000000FFL
+//COHER_DEST_BASE_HI_1
+#define COHER_DEST_BASE_HI_1__DEST_BASE_HI_256B__SHIFT                                                        0x0
+#define COHER_DEST_BASE_HI_1__DEST_BASE_HI_256B_MASK                                                          0x000000FFL
+//COHER_DEST_BASE_HI_2
+#define COHER_DEST_BASE_HI_2__DEST_BASE_HI_256B__SHIFT                                                        0x0
+#define COHER_DEST_BASE_HI_2__DEST_BASE_HI_256B_MASK                                                          0x000000FFL
+//COHER_DEST_BASE_HI_3
+#define COHER_DEST_BASE_HI_3__DEST_BASE_HI_256B__SHIFT                                                        0x0
+#define COHER_DEST_BASE_HI_3__DEST_BASE_HI_256B_MASK                                                          0x000000FFL
+//COHER_DEST_BASE_2
+#define COHER_DEST_BASE_2__DEST_BASE_256B__SHIFT                                                              0x0
+#define COHER_DEST_BASE_2__DEST_BASE_256B_MASK                                                                0xFFFFFFFFL
+//COHER_DEST_BASE_3
+#define COHER_DEST_BASE_3__DEST_BASE_256B__SHIFT                                                              0x0
+#define COHER_DEST_BASE_3__DEST_BASE_256B_MASK                                                                0xFFFFFFFFL
+//COHER_DEST_BASE_0
+#define COHER_DEST_BASE_0__DEST_BASE_256B__SHIFT                                                              0x0
+#define COHER_DEST_BASE_0__DEST_BASE_256B_MASK                                                                0xFFFFFFFFL
+//COHER_DEST_BASE_1
+#define COHER_DEST_BASE_1__DEST_BASE_256B__SHIFT                                                              0x0
+#define COHER_DEST_BASE_1__DEST_BASE_256B_MASK                                                                0xFFFFFFFFL
+//CP_PERFMON_CNTX_CNTL
+#define CP_PERFMON_CNTX_CNTL__PERFMON_ENABLE__SHIFT                                                           0x1f
+#define CP_PERFMON_CNTX_CNTL__PERFMON_ENABLE_MASK                                                             0x80000000L
+//CP_PIPEID
+#define CP_PIPEID__PIPE_ID__SHIFT                                                                             0x0
+#define CP_PIPEID__PIPE_ID_MASK                                                                               0x00000003L
+//CP_RINGID
+#define CP_RINGID__RINGID__SHIFT                                                                              0x0
+#define CP_RINGID__RINGID_MASK                                                                                0x00000003L
+//CP_VMID
+#define CP_VMID__VMID__SHIFT                                                                                  0x0
+#define CP_VMID__VMID_MASK                                                                                    0x0000000FL
+//CONTEXT_RESERVED_REG0
+#define CONTEXT_RESERVED_REG0__DATA__SHIFT                                                                    0x0
+#define CONTEXT_RESERVED_REG0__DATA_MASK                                                                      0xFFFFFFFFL
+//CONTEXT_RESERVED_REG1
+#define CONTEXT_RESERVED_REG1__DATA__SHIFT                                                                    0x0
+#define CONTEXT_RESERVED_REG1__DATA_MASK                                                                      0xFFFFFFFFL
+//VGT_MULTI_PRIM_IB_RESET_INDX
+#define VGT_MULTI_PRIM_IB_RESET_INDX__RESET_INDX__SHIFT                                                       0x0
+#define VGT_MULTI_PRIM_IB_RESET_INDX__RESET_INDX_MASK                                                         0xFFFFFFFFL
+//GFX_COPY_STATE
+#define GFX_COPY_STATE__SRC_STATE_ID__SHIFT                                                                   0x0
+#define GFX_COPY_STATE__SRC_STATE_ID_MASK                                                                     0x00000007L
+//VGT_DMA_BASE_HI
+#define VGT_DMA_BASE_HI__BASE_ADDR__SHIFT                                                                     0x0
+#define VGT_DMA_BASE_HI__BASE_ADDR_MASK                                                                       0x0000FFFFL
+//VGT_DMA_BASE
+#define VGT_DMA_BASE__BASE_ADDR__SHIFT                                                                        0x0
+#define VGT_DMA_BASE__BASE_ADDR_MASK                                                                          0xFFFFFFFFL
+//VGT_DRAW_INITIATOR
+#define VGT_DRAW_INITIATOR__SOURCE_SELECT__SHIFT                                                              0x0
+#define VGT_DRAW_INITIATOR__NOT_EOP__SHIFT                                                                    0x5
+#define VGT_DRAW_INITIATOR__USE_OPAQUE__SHIFT                                                                 0x6
+#define VGT_DRAW_INITIATOR__REG_RT_INDEX__SHIFT                                                               0x1d
+#define VGT_DRAW_INITIATOR__SOURCE_SELECT_MASK                                                                0x00000003L
+#define VGT_DRAW_INITIATOR__NOT_EOP_MASK                                                                      0x00000020L
+#define VGT_DRAW_INITIATOR__USE_OPAQUE_MASK                                                                   0x00000040L
+#define VGT_DRAW_INITIATOR__REG_RT_INDEX_MASK                                                                 0xE0000000L
+//VGT_EVENT_ADDRESS_REG
+#define VGT_EVENT_ADDRESS_REG__ADDRESS_LOW__SHIFT                                                             0x0
+#define VGT_EVENT_ADDRESS_REG__ADDRESS_LOW_MASK                                                               0x0FFFFFFFL
+//VGT_HOS_MAX_TESS_LEVEL
+#define VGT_HOS_MAX_TESS_LEVEL__MAX_TESS__SHIFT                                                               0x0
+#define VGT_HOS_MAX_TESS_LEVEL__MAX_TESS_MASK                                                                 0xFFFFFFFFL
+//VGT_HOS_MIN_TESS_LEVEL
+#define VGT_HOS_MIN_TESS_LEVEL__MIN_TESS__SHIFT                                                               0x0
+#define VGT_HOS_MIN_TESS_LEVEL__MIN_TESS_MASK                                                                 0xFFFFFFFFL
+//GE_IA_ENHANCE
+#define GE_IA_ENHANCE__MISC__SHIFT                                                                            0x0
+#define GE_IA_ENHANCE__MISC_MASK                                                                              0xFFFFFFFFL
+//VGT_DMA_SIZE
+#define VGT_DMA_SIZE__NUM_INDICES__SHIFT                                                                      0x0
+#define VGT_DMA_SIZE__NUM_INDICES_MASK                                                                        0xFFFFFFFFL
+//VGT_DMA_MAX_SIZE
+#define VGT_DMA_MAX_SIZE__MAX_SIZE__SHIFT                                                                     0x0
+#define VGT_DMA_MAX_SIZE__MAX_SIZE_MASK                                                                       0xFFFFFFFFL
+//VGT_DMA_INDEX_TYPE
+#define VGT_DMA_INDEX_TYPE__INDEX_TYPE__SHIFT                                                                 0x0
+#define VGT_DMA_INDEX_TYPE__RDREQ_POLICY__SHIFT                                                               0x6
+#define VGT_DMA_INDEX_TYPE__NOT_EOP__SHIFT                                                                    0x9
+#define VGT_DMA_INDEX_TYPE__MTYPE__SHIFT                                                                      0xb
+#define VGT_DMA_INDEX_TYPE__DISABLE_INSTANCE_PACKING__SHIFT                                                   0xe
+#define VGT_DMA_INDEX_TYPE__TEMPORAL__SHIFT                                                                   0xf
+#define VGT_DMA_INDEX_TYPE__SPEC_DATA_READ__SHIFT                                                             0x11
+#define VGT_DMA_INDEX_TYPE__INDEX_TYPE_MASK                                                                   0x00000003L
+#define VGT_DMA_INDEX_TYPE__RDREQ_POLICY_MASK                                                                 0x000000C0L
+#define VGT_DMA_INDEX_TYPE__NOT_EOP_MASK                                                                      0x00000200L
+#define VGT_DMA_INDEX_TYPE__MTYPE_MASK                                                                        0x00003800L
+#define VGT_DMA_INDEX_TYPE__DISABLE_INSTANCE_PACKING_MASK                                                     0x00004000L
+#define VGT_DMA_INDEX_TYPE__TEMPORAL_MASK                                                                     0x00018000L
+#define VGT_DMA_INDEX_TYPE__SPEC_DATA_READ_MASK                                                               0x00060000L
+//GE_WD_ENHANCE
+#define GE_WD_ENHANCE__MISC__SHIFT                                                                            0x0
+#define GE_WD_ENHANCE__MISC_MASK                                                                              0xFFFFFFFFL
+//VGT_DMA_NUM_INSTANCES
+#define VGT_DMA_NUM_INSTANCES__NUM_INSTANCES__SHIFT                                                           0x0
+#define VGT_DMA_NUM_INSTANCES__NUM_INSTANCES_MASK                                                             0xFFFFFFFFL
+//VGT_EVENT_INITIATOR
+#define VGT_EVENT_INITIATOR__EVENT_TYPE__SHIFT                                                                0x0
+#define VGT_EVENT_INITIATOR__ADDRESS_HI__SHIFT                                                                0xa
+#define VGT_EVENT_INITIATOR__EXTENDED_EVENT__SHIFT                                                            0x1b
+#define VGT_EVENT_INITIATOR__EVENT_TYPE_MASK                                                                  0x0000003FL
+#define VGT_EVENT_INITIATOR__ADDRESS_HI_MASK                                                                  0x07FFFC00L
+#define VGT_EVENT_INITIATOR__EXTENDED_EVENT_MASK                                                              0x08000000L
+//VGT_SHADER_STAGES_EN
+#define VGT_SHADER_STAGES_EN__HS_EN__SHIFT                                                                    0x2
+#define VGT_SHADER_STAGES_EN__GS_EN__SHIFT                                                                    0x5
+#define VGT_SHADER_STAGES_EN__GS_FAST_LAUNCH__SHIFT                                                           0x13
+#define VGT_SHADER_STAGES_EN__HS_W32_EN__SHIFT                                                                0x15
+#define VGT_SHADER_STAGES_EN__GS_W32_EN__SHIFT                                                                0x16
+#define VGT_SHADER_STAGES_EN__NGG_WAVE_ID_EN__SHIFT                                                           0x18
+#define VGT_SHADER_STAGES_EN__PRIMGEN_PASSTHRU_NO_MSG__SHIFT                                                  0x1a
+#define VGT_SHADER_STAGES_EN__HS_EN_MASK                                                                      0x00000004L
+#define VGT_SHADER_STAGES_EN__GS_EN_MASK                                                                      0x00000020L
+#define VGT_SHADER_STAGES_EN__GS_FAST_LAUNCH_MASK                                                             0x00080000L
+#define VGT_SHADER_STAGES_EN__HS_W32_EN_MASK                                                                  0x00200000L
+#define VGT_SHADER_STAGES_EN__GS_W32_EN_MASK                                                                  0x00400000L
+#define VGT_SHADER_STAGES_EN__NGG_WAVE_ID_EN_MASK                                                             0x01000000L
+#define VGT_SHADER_STAGES_EN__PRIMGEN_PASSTHRU_NO_MSG_MASK                                                    0x04000000L
+//VGT_TF_PARAM
+#define VGT_TF_PARAM__TYPE__SHIFT                                                                             0x0
+#define VGT_TF_PARAM__PARTITIONING__SHIFT                                                                     0x2
+#define VGT_TF_PARAM__TOPOLOGY__SHIFT                                                                         0x5
+#define VGT_TF_PARAM__DISABLE_DONUTS__SHIFT                                                                   0xe
+#define VGT_TF_PARAM__TEMPORAL__SHIFT                                                                         0xf
+#define VGT_TF_PARAM__DISTRIBUTION_MODE__SHIFT                                                                0x11
+#define VGT_TF_PARAM__DETECT_ONE__SHIFT                                                                       0x13
+#define VGT_TF_PARAM__DETECT_ZERO__SHIFT                                                                      0x14
+#define VGT_TF_PARAM__MTYPE__SHIFT                                                                            0x17
+#define VGT_TF_PARAM__SPEC_DATA_READ__SHIFT                                                                   0x1c
+#define VGT_TF_PARAM__TYPE_MASK                                                                               0x00000003L
+#define VGT_TF_PARAM__PARTITIONING_MASK                                                                       0x0000001CL
+#define VGT_TF_PARAM__TOPOLOGY_MASK                                                                           0x000000E0L
+#define VGT_TF_PARAM__DISABLE_DONUTS_MASK                                                                     0x00004000L
+#define VGT_TF_PARAM__TEMPORAL_MASK                                                                           0x00018000L
+#define VGT_TF_PARAM__DISTRIBUTION_MODE_MASK                                                                  0x00060000L
+#define VGT_TF_PARAM__DETECT_ONE_MASK                                                                         0x00080000L
+#define VGT_TF_PARAM__DETECT_ZERO_MASK                                                                        0x00100000L
+#define VGT_TF_PARAM__MTYPE_MASK                                                                              0x03800000L
+#define VGT_TF_PARAM__SPEC_DATA_READ_MASK                                                                     0x30000000L
+//VGT_STRMOUT_DRAW_OPAQUE_OFFSET
+#define VGT_STRMOUT_DRAW_OPAQUE_OFFSET__OFFSET__SHIFT                                                         0x0
+#define VGT_STRMOUT_DRAW_OPAQUE_OFFSET__OFFSET_MASK                                                           0xFFFFFFFFL
+//VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE
+#define VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE__SIZE__SHIFT                                               0x0
+#define VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE__SIZE_MASK                                                 0xFFFFFFFFL
+//VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE
+#define VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE__VERTEX_STRIDE__SHIFT                                           0x0
+#define VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE__VERTEX_STRIDE_MASK                                             0x000001FFL
+//VGT_TESS_DISTRIBUTION
+#define VGT_TESS_DISTRIBUTION__ACCUM_ISOLINE__SHIFT                                                           0x0
+#define VGT_TESS_DISTRIBUTION__ACCUM_TRI__SHIFT                                                               0x8
+#define VGT_TESS_DISTRIBUTION__ACCUM_QUAD__SHIFT                                                              0x10
+#define VGT_TESS_DISTRIBUTION__DONUT_SPLIT__SHIFT                                                             0x18
+#define VGT_TESS_DISTRIBUTION__TRAP_SPLIT__SHIFT                                                              0x1d
+#define VGT_TESS_DISTRIBUTION__ACCUM_ISOLINE_MASK                                                             0x000000FFL
+#define VGT_TESS_DISTRIBUTION__ACCUM_TRI_MASK                                                                 0x0000FF00L
+#define VGT_TESS_DISTRIBUTION__ACCUM_QUAD_MASK                                                                0x00FF0000L
+#define VGT_TESS_DISTRIBUTION__DONUT_SPLIT_MASK                                                               0x1F000000L
+#define VGT_TESS_DISTRIBUTION__TRAP_SPLIT_MASK                                                                0xE0000000L
+//VGT_LS_HS_CONFIG
+#define VGT_LS_HS_CONFIG__NUM_PATCHES__SHIFT                                                                  0x0
+#define VGT_LS_HS_CONFIG__HS_NUM_OUTPUT_CP__SHIFT                                                             0xe
+#define VGT_LS_HS_CONFIG__NUM_PATCHES_MASK                                                                    0x000000FFL
+#define VGT_LS_HS_CONFIG__HS_NUM_OUTPUT_CP_MASK                                                               0x000FC000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pfvf_cpdec
+//CONFIG_RESERVED_REG0
+#define CONFIG_RESERVED_REG0__DATA__SHIFT                                                                     0x0
+#define CONFIG_RESERVED_REG0__DATA_MASK                                                                       0xFFFFFFFFL
+//CONFIG_RESERVED_REG1
+#define CONFIG_RESERVED_REG1__DATA__SHIFT                                                                     0x0
+#define CONFIG_RESERVED_REG1__DATA_MASK                                                                       0xFFFFFFFFL
+//CP_MEC_CNTL
+#define CP_MEC_CNTL__MEC_ME1_PIPE0_RESET__SHIFT                                                               0x10
+#define CP_MEC_CNTL__MEC_ME1_PIPE1_RESET__SHIFT                                                               0x11
+#define CP_MEC_CNTL__MEC_ME1_PIPE2_RESET__SHIFT                                                               0x12
+#define CP_MEC_CNTL__MEC_ME1_PIPE3_RESET__SHIFT                                                               0x13
+#define CP_MEC_CNTL__MEC_INVALIDATE_ICACHE__SHIFT                                                             0x1b
+#define CP_MEC_CNTL__MEC_ME1_HALT__SHIFT                                                                      0x1e
+#define CP_MEC_CNTL__MEC_ME1_STEP__SHIFT                                                                      0x1f
+#define CP_MEC_CNTL__MEC_ME1_PIPE0_RESET_MASK                                                                 0x00010000L
+#define CP_MEC_CNTL__MEC_ME1_PIPE1_RESET_MASK                                                                 0x00020000L
+#define CP_MEC_CNTL__MEC_ME1_PIPE2_RESET_MASK                                                                 0x00040000L
+#define CP_MEC_CNTL__MEC_ME1_PIPE3_RESET_MASK                                                                 0x00080000L
+#define CP_MEC_CNTL__MEC_INVALIDATE_ICACHE_MASK                                                               0x08000000L
+#define CP_MEC_CNTL__MEC_ME1_HALT_MASK                                                                        0x40000000L
+#define CP_MEC_CNTL__MEC_ME1_STEP_MASK                                                                        0x80000000L
+//CP_ME_CNTL
+#define CP_ME_CNTL__CE_INVALIDATE_ICACHE__SHIFT                                                               0x4
+#define CP_ME_CNTL__PFP_INVALIDATE_ICACHE__SHIFT                                                              0x6
+#define CP_ME_CNTL__ME_INVALIDATE_ICACHE__SHIFT                                                               0x8
+#define CP_ME_CNTL__PFP_PIPE0_DISABLE__SHIFT                                                                  0xc
+#define CP_ME_CNTL__PFP_PIPE1_DISABLE__SHIFT                                                                  0xd
+#define CP_ME_CNTL__ME_PIPE0_DISABLE__SHIFT                                                                   0xe
+#define CP_ME_CNTL__ME_PIPE1_DISABLE__SHIFT                                                                   0xf
+#define CP_ME_CNTL__CE_PIPE0_RESET__SHIFT                                                                     0x10
+#define CP_ME_CNTL__CE_PIPE1_RESET__SHIFT                                                                     0x11
+#define CP_ME_CNTL__PFP_PIPE0_RESET__SHIFT                                                                    0x12
+#define CP_ME_CNTL__PFP_PIPE1_RESET__SHIFT                                                                    0x13
+#define CP_ME_CNTL__ME_PIPE0_RESET__SHIFT                                                                     0x14
+#define CP_ME_CNTL__ME_PIPE1_RESET__SHIFT                                                                     0x15
+#define CP_ME_CNTL__CE_HALT__SHIFT                                                                            0x18
+#define CP_ME_CNTL__CE_STEP__SHIFT                                                                            0x19
+#define CP_ME_CNTL__PFP_HALT__SHIFT                                                                           0x1a
+#define CP_ME_CNTL__PFP_STEP__SHIFT                                                                           0x1b
+#define CP_ME_CNTL__ME_HALT__SHIFT                                                                            0x1c
+#define CP_ME_CNTL__ME_STEP__SHIFT                                                                            0x1d
+#define CP_ME_CNTL__CE_INVALIDATE_ICACHE_MASK                                                                 0x00000010L
+#define CP_ME_CNTL__PFP_INVALIDATE_ICACHE_MASK                                                                0x00000040L
+#define CP_ME_CNTL__ME_INVALIDATE_ICACHE_MASK                                                                 0x00000100L
+#define CP_ME_CNTL__PFP_PIPE0_DISABLE_MASK                                                                    0x00001000L
+#define CP_ME_CNTL__PFP_PIPE1_DISABLE_MASK                                                                    0x00002000L
+#define CP_ME_CNTL__ME_PIPE0_DISABLE_MASK                                                                     0x00004000L
+#define CP_ME_CNTL__ME_PIPE1_DISABLE_MASK                                                                     0x00008000L
+#define CP_ME_CNTL__CE_PIPE0_RESET_MASK                                                                       0x00010000L
+#define CP_ME_CNTL__CE_PIPE1_RESET_MASK                                                                       0x00020000L
+#define CP_ME_CNTL__PFP_PIPE0_RESET_MASK                                                                      0x00040000L
+#define CP_ME_CNTL__PFP_PIPE1_RESET_MASK                                                                      0x00080000L
+#define CP_ME_CNTL__ME_PIPE0_RESET_MASK                                                                       0x00100000L
+#define CP_ME_CNTL__ME_PIPE1_RESET_MASK                                                                       0x00200000L
+#define CP_ME_CNTL__CE_HALT_MASK                                                                              0x01000000L
+#define CP_ME_CNTL__CE_STEP_MASK                                                                              0x02000000L
+#define CP_ME_CNTL__PFP_HALT_MASK                                                                             0x04000000L
+#define CP_ME_CNTL__PFP_STEP_MASK                                                                             0x08000000L
+#define CP_ME_CNTL__ME_HALT_MASK                                                                              0x10000000L
+#define CP_ME_CNTL__ME_STEP_MASK                                                                              0x20000000L
+//CP_UNMAPPED_QUEUE0
+#define CP_UNMAPPED_QUEUE0__HIT__SHIFT                                                                        0x0
+#define CP_UNMAPPED_QUEUE0__HIT_MASK                                                                          0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE1
+#define CP_UNMAPPED_QUEUE1__HIT__SHIFT                                                                        0x0
+#define CP_UNMAPPED_QUEUE1__HIT_MASK                                                                          0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE2
+#define CP_UNMAPPED_QUEUE2__HIT__SHIFT                                                                        0x0
+#define CP_UNMAPPED_QUEUE2__HIT_MASK                                                                          0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE3
+#define CP_UNMAPPED_QUEUE3__HIT__SHIFT                                                                        0x0
+#define CP_UNMAPPED_QUEUE3__HIT_MASK                                                                          0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE4
+#define CP_UNMAPPED_QUEUE4__HIT__SHIFT                                                                        0x0
+#define CP_UNMAPPED_QUEUE4__HIT_MASK                                                                          0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE5
+#define CP_UNMAPPED_QUEUE5__HIT__SHIFT                                                                        0x0
+#define CP_UNMAPPED_QUEUE5__HIT_MASK                                                                          0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE6
+#define CP_UNMAPPED_QUEUE6__HIT__SHIFT                                                                        0x0
+#define CP_UNMAPPED_QUEUE6__HIT_MASK                                                                          0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE7
+#define CP_UNMAPPED_QUEUE7__HIT__SHIFT                                                                        0x0
+#define CP_UNMAPPED_QUEUE7__HIT_MASK                                                                          0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE8
+#define CP_UNMAPPED_QUEUE8__HIT__SHIFT                                                                        0x0
+#define CP_UNMAPPED_QUEUE8__HIT_MASK                                                                          0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE9
+#define CP_UNMAPPED_QUEUE9__HIT__SHIFT                                                                        0x0
+#define CP_UNMAPPED_QUEUE9__HIT_MASK                                                                          0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE10
+#define CP_UNMAPPED_QUEUE10__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE10__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE11
+#define CP_UNMAPPED_QUEUE11__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE11__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE12
+#define CP_UNMAPPED_QUEUE12__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE12__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE13
+#define CP_UNMAPPED_QUEUE13__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE13__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE14
+#define CP_UNMAPPED_QUEUE14__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE14__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE15
+#define CP_UNMAPPED_QUEUE15__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE15__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE16
+#define CP_UNMAPPED_QUEUE16__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE16__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE17
+#define CP_UNMAPPED_QUEUE17__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE17__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE18
+#define CP_UNMAPPED_QUEUE18__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE18__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE19
+#define CP_UNMAPPED_QUEUE19__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE19__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE20
+#define CP_UNMAPPED_QUEUE20__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE20__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE21
+#define CP_UNMAPPED_QUEUE21__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE21__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE22
+#define CP_UNMAPPED_QUEUE22__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE22__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE23
+#define CP_UNMAPPED_QUEUE23__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE23__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE24
+#define CP_UNMAPPED_QUEUE24__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE24__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE25
+#define CP_UNMAPPED_QUEUE25__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE25__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE26
+#define CP_UNMAPPED_QUEUE26__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE26__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE27
+#define CP_UNMAPPED_QUEUE27__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE27__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE28
+#define CP_UNMAPPED_QUEUE28__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE28__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE29
+#define CP_UNMAPPED_QUEUE29__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE29__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE30
+#define CP_UNMAPPED_QUEUE30__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE30__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE31
+#define CP_UNMAPPED_QUEUE31__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE31__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE32
+#define CP_UNMAPPED_QUEUE32__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE32__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE33
+#define CP_UNMAPPED_QUEUE33__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE33__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE34
+#define CP_UNMAPPED_QUEUE34__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE34__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE35
+#define CP_UNMAPPED_QUEUE35__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE35__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE36
+#define CP_UNMAPPED_QUEUE36__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE36__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE37
+#define CP_UNMAPPED_QUEUE37__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE37__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE38
+#define CP_UNMAPPED_QUEUE38__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE38__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE39
+#define CP_UNMAPPED_QUEUE39__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE39__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE40
+#define CP_UNMAPPED_QUEUE40__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE40__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE41
+#define CP_UNMAPPED_QUEUE41__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE41__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE42
+#define CP_UNMAPPED_QUEUE42__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE42__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE43
+#define CP_UNMAPPED_QUEUE43__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE43__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE44
+#define CP_UNMAPPED_QUEUE44__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE44__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE45
+#define CP_UNMAPPED_QUEUE45__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE45__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE46
+#define CP_UNMAPPED_QUEUE46__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE46__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE47
+#define CP_UNMAPPED_QUEUE47__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE47__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE48
+#define CP_UNMAPPED_QUEUE48__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE48__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE49
+#define CP_UNMAPPED_QUEUE49__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE49__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE50
+#define CP_UNMAPPED_QUEUE50__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE50__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE51
+#define CP_UNMAPPED_QUEUE51__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE51__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE52
+#define CP_UNMAPPED_QUEUE52__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE52__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE53
+#define CP_UNMAPPED_QUEUE53__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE53__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE54
+#define CP_UNMAPPED_QUEUE54__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE54__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE55
+#define CP_UNMAPPED_QUEUE55__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE55__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE56
+#define CP_UNMAPPED_QUEUE56__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE56__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE57
+#define CP_UNMAPPED_QUEUE57__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE57__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE58
+#define CP_UNMAPPED_QUEUE58__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE58__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE59
+#define CP_UNMAPPED_QUEUE59__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE59__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE60
+#define CP_UNMAPPED_QUEUE60__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE60__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE61
+#define CP_UNMAPPED_QUEUE61__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE61__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE62
+#define CP_UNMAPPED_QUEUE62__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE62__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE63
+#define CP_UNMAPPED_QUEUE63__HIT__SHIFT                                                                       0x0
+#define CP_UNMAPPED_QUEUE63__HIT_MASK                                                                         0xFFFFFFFFL
+//CP_UNMAPPED_DOORBELL
+#define CP_UNMAPPED_DOORBELL__ENABLE__SHIFT                                                                   0x0
+#define CP_UNMAPPED_DOORBELL__DBELL_MSG_BLOCK__SHIFT                                                          0x1
+#define CP_UNMAPPED_DOORBELL__CLEAR_ALL__SHIFT                                                                0x2
+#define CP_UNMAPPED_DOORBELL__QUEUE_LSB__SHIFT                                                                0x4
+#define CP_UNMAPPED_DOORBELL__PROC_LSB__SHIFT                                                                 0x8
+#define CP_UNMAPPED_DOORBELL__ENABLE_MASK                                                                     0x00000001L
+#define CP_UNMAPPED_DOORBELL__DBELL_MSG_BLOCK_MASK                                                            0x00000002L
+#define CP_UNMAPPED_DOORBELL__CLEAR_ALL_MASK                                                                  0x00000004L
+#define CP_UNMAPPED_DOORBELL__QUEUE_LSB_MASK                                                                  0x000000F0L
+#define CP_UNMAPPED_DOORBELL__PROC_LSB_MASK                                                                   0x00001F00L
+//CP_UNMAPPED_QUEUE_BANK0
+#define CP_UNMAPPED_QUEUE_BANK0__BANK_HIT__SHIFT                                                              0x0
+#define CP_UNMAPPED_QUEUE_BANK0__BANK_HIT_MASK                                                                0xFFFFFFFFL
+//CP_UNMAPPED_QUEUE_BANK1
+#define CP_UNMAPPED_QUEUE_BANK1__BANK_HIT__SHIFT                                                              0x0
+#define CP_UNMAPPED_QUEUE_BANK1__BANK_HIT_MASK                                                                0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pfvf_grbmdec
+//GRBM_GFX_CNTL
+#define GRBM_GFX_CNTL__PIPEID__SHIFT                                                                          0x0
+#define GRBM_GFX_CNTL__MEID__SHIFT                                                                            0x2
+#define GRBM_GFX_CNTL__VMID__SHIFT                                                                            0x4
+#define GRBM_GFX_CNTL__QUEUEID__SHIFT                                                                         0x8
+#define GRBM_GFX_CNTL__CTXID__SHIFT                                                                           0xb
+#define GRBM_GFX_CNTL__PIPEID_MASK                                                                            0x00000003L
+#define GRBM_GFX_CNTL__MEID_MASK                                                                              0x0000000CL
+#define GRBM_GFX_CNTL__VMID_MASK                                                                              0x000000F0L
+#define GRBM_GFX_CNTL__QUEUEID_MASK                                                                           0x00000700L
+#define GRBM_GFX_CNTL__CTXID_MASK                                                                             0x00003800L
+//GRBM_NOWHERE
+#define GRBM_NOWHERE__DATA__SHIFT                                                                             0x0
+#define GRBM_NOWHERE__DATA_MASK                                                                               0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pfonly_cpdec
+//CP_FETCHER_SOURCE
+#define CP_FETCHER_SOURCE__ME_SRC__SHIFT                                                                      0x0
+#define CP_FETCHER_SOURCE__ME_SRC_MASK                                                                        0x00000001L
+//CP_DFY_CNTL
+#define CP_DFY_CNTL__SCOPE__SHIFT                                                                             0x6
+#define CP_DFY_CNTL__POLICY__SHIFT                                                                            0x8
+#define CP_DFY_CNTL__MTYPE__SHIFT                                                                             0xc
+#define CP_DFY_CNTL__SPEC_DATA_READ__SHIFT                                                                    0xf
+#define CP_DFY_CNTL__REPEATER_FGCG_DISABLE__SHIFT                                                             0x19
+#define CP_DFY_CNTL__TPI_SDP_SEL__SHIFT                                                                       0x1a
+#define CP_DFY_CNTL__LFSR_RESET__SHIFT                                                                        0x1c
+#define CP_DFY_CNTL__MODE__SHIFT                                                                              0x1d
+#define CP_DFY_CNTL__ENABLE__SHIFT                                                                            0x1f
+#define CP_DFY_CNTL__SCOPE_MASK                                                                               0x000000C0L
+#define CP_DFY_CNTL__POLICY_MASK                                                                              0x00000300L
+#define CP_DFY_CNTL__MTYPE_MASK                                                                               0x00003000L
+#define CP_DFY_CNTL__SPEC_DATA_READ_MASK                                                                      0x00018000L
+#define CP_DFY_CNTL__REPEATER_FGCG_DISABLE_MASK                                                               0x02000000L
+#define CP_DFY_CNTL__TPI_SDP_SEL_MASK                                                                         0x04000000L
+#define CP_DFY_CNTL__LFSR_RESET_MASK                                                                          0x10000000L
+#define CP_DFY_CNTL__MODE_MASK                                                                                0x60000000L
+#define CP_DFY_CNTL__ENABLE_MASK                                                                              0x80000000L
+//CP_DFY_STAT
+#define CP_DFY_STAT__BURST_COUNT__SHIFT                                                                       0x0
+#define CP_DFY_STAT__TAGS_PENDING__SHIFT                                                                      0x10
+#define CP_DFY_STAT__BUSY__SHIFT                                                                              0x1f
+#define CP_DFY_STAT__BURST_COUNT_MASK                                                                         0x0000FFFFL
+#define CP_DFY_STAT__TAGS_PENDING_MASK                                                                        0x07FF0000L
+#define CP_DFY_STAT__BUSY_MASK                                                                                0x80000000L
+//CP_DFY_ADDR_HI
+#define CP_DFY_ADDR_HI__ADDR_HI__SHIFT                                                                        0x0
+#define CP_DFY_ADDR_HI__ADDR_HI_MASK                                                                          0xFFFFFFFFL
+//CP_DFY_ADDR_LO
+#define CP_DFY_ADDR_LO__ADDR_LO__SHIFT                                                                        0x5
+#define CP_DFY_ADDR_LO__ADDR_LO_MASK                                                                          0xFFFFFFE0L
+//CP_DFY_DATA_0
+#define CP_DFY_DATA_0__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_0__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_1
+#define CP_DFY_DATA_1__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_1__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_2
+#define CP_DFY_DATA_2__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_2__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_3
+#define CP_DFY_DATA_3__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_3__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_4
+#define CP_DFY_DATA_4__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_4__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_5
+#define CP_DFY_DATA_5__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_5__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_6
+#define CP_DFY_DATA_6__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_6__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_7
+#define CP_DFY_DATA_7__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_7__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_8
+#define CP_DFY_DATA_8__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_8__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_9
+#define CP_DFY_DATA_9__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_9__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_10
+#define CP_DFY_DATA_10__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_10__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_DATA_11
+#define CP_DFY_DATA_11__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_11__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_DATA_12
+#define CP_DFY_DATA_12__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_12__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_DATA_13
+#define CP_DFY_DATA_13__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_13__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_DATA_14
+#define CP_DFY_DATA_14__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_14__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_DATA_15
+#define CP_DFY_DATA_15__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_15__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_CMD
+#define CP_DFY_CMD__SIZE__SHIFT                                                                               0x10
+#define CP_DFY_CMD__SIZE_MASK                                                                                 0xFFFF0000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pfonly_cpphqddec
+//CP_HPD_MES_ROQ_OFFSETS
+#define CP_HPD_MES_ROQ_OFFSETS__IQ_OFFSET__SHIFT                                                              0x0
+#define CP_HPD_MES_ROQ_OFFSETS__PQ_OFFSET__SHIFT                                                              0x8
+#define CP_HPD_MES_ROQ_OFFSETS__IB_OFFSET__SHIFT                                                              0x10
+#define CP_HPD_MES_ROQ_OFFSETS__IQ_OFFSET_MASK                                                                0x00000007L
+#define CP_HPD_MES_ROQ_OFFSETS__PQ_OFFSET_MASK                                                                0x00003F00L
+#define CP_HPD_MES_ROQ_OFFSETS__IB_OFFSET_MASK                                                                0x007F0000L
+//CP_HPD_ROQ_OFFSETS
+#define CP_HPD_ROQ_OFFSETS__IQ_OFFSET__SHIFT                                                                  0x0
+#define CP_HPD_ROQ_OFFSETS__PQ_OFFSET__SHIFT                                                                  0x8
+#define CP_HPD_ROQ_OFFSETS__IB_OFFSET__SHIFT                                                                  0x10
+#define CP_HPD_ROQ_OFFSETS__KD_OFFSET__SHIFT                                                                  0x18
+#define CP_HPD_ROQ_OFFSETS__IQ_OFFSET_MASK                                                                    0x00000007L
+#define CP_HPD_ROQ_OFFSETS__PQ_OFFSET_MASK                                                                    0x00003F00L
+#define CP_HPD_ROQ_OFFSETS__IB_OFFSET_MASK                                                                    0x007F0000L
+#define CP_HPD_ROQ_OFFSETS__KD_OFFSET_MASK                                                                    0xFF000000L
+//CP_HPD_STATUS0
+#define CP_HPD_STATUS0__QUEUE_STATE__SHIFT                                                                    0x0
+#define CP_HPD_STATUS0__MAPPED_QUEUE__SHIFT                                                                   0x5
+#define CP_HPD_STATUS0__QUEUE_AVAILABLE__SHIFT                                                                0x8
+#define CP_HPD_STATUS0__FETCHING_MQD__SHIFT                                                                   0x10
+#define CP_HPD_STATUS0__PEND_TXFER_SIZE_PQIB__SHIFT                                                           0x11
+#define CP_HPD_STATUS0__PEND_TXFER_SIZE_IQ__SHIFT                                                             0x12
+#define CP_HPD_STATUS0__ENABLE_MSG_NO_DISC__SHIFT                                                             0x13
+#define CP_HPD_STATUS0__FORCE_QUEUE_STATE__SHIFT                                                              0x14
+#define CP_HPD_STATUS0__MASTER_QUEUE_IDLE_DIS__SHIFT                                                          0x1b
+#define CP_HPD_STATUS0__ENABLE_OFFLOAD_CHECK__SHIFT                                                           0x1c
+#define CP_HPD_STATUS0__FREEZE_QUEUE_STATE__SHIFT                                                             0x1e
+#define CP_HPD_STATUS0__FORCE_QUEUE__SHIFT                                                                    0x1f
+#define CP_HPD_STATUS0__QUEUE_STATE_MASK                                                                      0x0000001FL
+#define CP_HPD_STATUS0__MAPPED_QUEUE_MASK                                                                     0x000000E0L
+#define CP_HPD_STATUS0__QUEUE_AVAILABLE_MASK                                                                  0x0000FF00L
+#define CP_HPD_STATUS0__FETCHING_MQD_MASK                                                                     0x00010000L
+#define CP_HPD_STATUS0__PEND_TXFER_SIZE_PQIB_MASK                                                             0x00020000L
+#define CP_HPD_STATUS0__PEND_TXFER_SIZE_IQ_MASK                                                               0x00040000L
+#define CP_HPD_STATUS0__ENABLE_MSG_NO_DISC_MASK                                                               0x00080000L
+#define CP_HPD_STATUS0__FORCE_QUEUE_STATE_MASK                                                                0x01F00000L
+#define CP_HPD_STATUS0__MASTER_QUEUE_IDLE_DIS_MASK                                                            0x08000000L
+#define CP_HPD_STATUS0__ENABLE_OFFLOAD_CHECK_MASK                                                             0x30000000L
+#define CP_HPD_STATUS0__FREEZE_QUEUE_STATE_MASK                                                               0x40000000L
+#define CP_HPD_STATUS0__FORCE_QUEUE_MASK                                                                      0x80000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pfonly_gcrdec
+//GCR_GENERAL_CNTL
+#define GCR_GENERAL_CNTL__FORCE_4K_L2_RESP__SHIFT                                                             0x0
+#define GCR_GENERAL_CNTL__REDUCE_HALF_MAIN_WQ__SHIFT                                                          0x1
+#define GCR_GENERAL_CNTL__REDUCE_HALF_PHY_WQ__SHIFT                                                           0x2
+#define GCR_GENERAL_CNTL__FORCE_INV_ALL__SHIFT                                                                0x3
+#define GCR_GENERAL_CNTL__HI_PRIORITY_CNTL__SHIFT                                                             0x4
+#define GCR_GENERAL_CNTL__HI_PRIORITY_DISABLE__SHIFT                                                          0x6
+#define GCR_GENERAL_CNTL__BIG_PAGE_FILTER_DISABLE__SHIFT                                                      0x7
+#define GCR_GENERAL_CNTL__PERF_CNTR_ENABLE__SHIFT                                                             0x8
+#define GCR_GENERAL_CNTL__FORCE_SINGLE_WQ__SHIFT                                                              0x9
+#define GCR_GENERAL_CNTL__UTCL2_REQ_PERM__SHIFT                                                               0xa
+#define GCR_GENERAL_CNTL__TARGET_MGCG_CLKEN_DIS__SHIFT                                                        0xd
+#define GCR_GENERAL_CNTL__MIXED_RANGE_MODE_DIS__SHIFT                                                         0xe
+#define GCR_GENERAL_CNTL__ENABLE_16K_UTCL2_REQ__SHIFT                                                         0xf
+#define GCR_GENERAL_CNTL__DISABLE_FGCG__SHIFT                                                                 0x10
+#define GCR_GENERAL_CNTL__UTCL2_REQ_LIMIT__SHIFT                                                              0x11
+#define GCR_GENERAL_CNTL__CLIENT_ID__SHIFT                                                                    0x14
+#define GCR_GENERAL_CNTL__FORCE_4K_L2_RESP_MASK                                                               0x00000001L
+#define GCR_GENERAL_CNTL__REDUCE_HALF_MAIN_WQ_MASK                                                            0x00000002L
+#define GCR_GENERAL_CNTL__REDUCE_HALF_PHY_WQ_MASK                                                             0x00000004L
+#define GCR_GENERAL_CNTL__FORCE_INV_ALL_MASK                                                                  0x00000008L
+#define GCR_GENERAL_CNTL__HI_PRIORITY_CNTL_MASK                                                               0x00000030L
+#define GCR_GENERAL_CNTL__HI_PRIORITY_DISABLE_MASK                                                            0x00000040L
+#define GCR_GENERAL_CNTL__BIG_PAGE_FILTER_DISABLE_MASK                                                        0x00000080L
+#define GCR_GENERAL_CNTL__PERF_CNTR_ENABLE_MASK                                                               0x00000100L
+#define GCR_GENERAL_CNTL__FORCE_SINGLE_WQ_MASK                                                                0x00000200L
+#define GCR_GENERAL_CNTL__UTCL2_REQ_PERM_MASK                                                                 0x00001C00L
+#define GCR_GENERAL_CNTL__TARGET_MGCG_CLKEN_DIS_MASK                                                          0x00002000L
+#define GCR_GENERAL_CNTL__MIXED_RANGE_MODE_DIS_MASK                                                           0x00004000L
+#define GCR_GENERAL_CNTL__ENABLE_16K_UTCL2_REQ_MASK                                                           0x00008000L
+#define GCR_GENERAL_CNTL__DISABLE_FGCG_MASK                                                                   0x00010000L
+#define GCR_GENERAL_CNTL__UTCL2_REQ_LIMIT_MASK                                                                0x000E0000L
+#define GCR_GENERAL_CNTL__CLIENT_ID_MASK                                                                      0x1FF00000L
+//GCR_TARGET_DISABLE
+#define GCR_TARGET_DISABLE__DISABLE_SE0_PHY__SHIFT                                                            0x0
+#define GCR_TARGET_DISABLE__DISABLE_SE0_VIRT__SHIFT                                                           0x1
+#define GCR_TARGET_DISABLE__DISABLE_SE1_PHY__SHIFT                                                            0x2
+#define GCR_TARGET_DISABLE__DISABLE_SE1_VIRT__SHIFT                                                           0x3
+#define GCR_TARGET_DISABLE__DISABLE_SE2_PHY__SHIFT                                                            0x4
+#define GCR_TARGET_DISABLE__DISABLE_SE2_VIRT__SHIFT                                                           0x5
+#define GCR_TARGET_DISABLE__DISABLE_SE3_PHY__SHIFT                                                            0x6
+#define GCR_TARGET_DISABLE__DISABLE_SE3_VIRT__SHIFT                                                           0x7
+#define GCR_TARGET_DISABLE__DISABLE_GL2A0_PHY__SHIFT                                                          0x8
+#define GCR_TARGET_DISABLE__DISABLE_GL2A1_PHY__SHIFT                                                          0x9
+#define GCR_TARGET_DISABLE__DISABLE_GL2A2_PHY__SHIFT                                                          0xa
+#define GCR_TARGET_DISABLE__DISABLE_GL2A3_PHY__SHIFT                                                          0xb
+#define GCR_TARGET_DISABLE__SE0_INACTIVE_STATUS__SHIFT                                                        0x10
+#define GCR_TARGET_DISABLE__SE1_INACTIVE_STATUS__SHIFT                                                        0x11
+#define GCR_TARGET_DISABLE__SE2_INACTIVE_STATUS__SHIFT                                                        0x12
+#define GCR_TARGET_DISABLE__SE3_INACTIVE_STATUS__SHIFT                                                        0x13
+#define GCR_TARGET_DISABLE__GL2A0_DISABLE_STATUS__SHIFT                                                       0x18
+#define GCR_TARGET_DISABLE__GL2A1_DISABLE_STATUS__SHIFT                                                       0x19
+#define GCR_TARGET_DISABLE__GL2A2_DISABLE_STATUS__SHIFT                                                       0x1a
+#define GCR_TARGET_DISABLE__GL2A3_DISABLE_STATUS__SHIFT                                                       0x1b
+#define GCR_TARGET_DISABLE__DISABLE_SE0_PHY_MASK                                                              0x00000001L
+#define GCR_TARGET_DISABLE__DISABLE_SE0_VIRT_MASK                                                             0x00000002L
+#define GCR_TARGET_DISABLE__DISABLE_SE1_PHY_MASK                                                              0x00000004L
+#define GCR_TARGET_DISABLE__DISABLE_SE1_VIRT_MASK                                                             0x00000008L
+#define GCR_TARGET_DISABLE__DISABLE_SE2_PHY_MASK                                                              0x00000010L
+#define GCR_TARGET_DISABLE__DISABLE_SE2_VIRT_MASK                                                             0x00000020L
+#define GCR_TARGET_DISABLE__DISABLE_SE3_PHY_MASK                                                              0x00000040L
+#define GCR_TARGET_DISABLE__DISABLE_SE3_VIRT_MASK                                                             0x00000080L
+#define GCR_TARGET_DISABLE__DISABLE_GL2A0_PHY_MASK                                                            0x00000100L
+#define GCR_TARGET_DISABLE__DISABLE_GL2A1_PHY_MASK                                                            0x00000200L
+#define GCR_TARGET_DISABLE__DISABLE_GL2A2_PHY_MASK                                                            0x00000400L
+#define GCR_TARGET_DISABLE__DISABLE_GL2A3_PHY_MASK                                                            0x00000800L
+#define GCR_TARGET_DISABLE__SE0_INACTIVE_STATUS_MASK                                                          0x00010000L
+#define GCR_TARGET_DISABLE__SE1_INACTIVE_STATUS_MASK                                                          0x00020000L
+#define GCR_TARGET_DISABLE__SE2_INACTIVE_STATUS_MASK                                                          0x00040000L
+#define GCR_TARGET_DISABLE__SE3_INACTIVE_STATUS_MASK                                                          0x00080000L
+#define GCR_TARGET_DISABLE__GL2A0_DISABLE_STATUS_MASK                                                         0x01000000L
+#define GCR_TARGET_DISABLE__GL2A1_DISABLE_STATUS_MASK                                                         0x02000000L
+#define GCR_TARGET_DISABLE__GL2A2_DISABLE_STATUS_MASK                                                         0x04000000L
+#define GCR_TARGET_DISABLE__GL2A3_DISABLE_STATUS_MASK                                                         0x08000000L
+//GCR_CMD_STATUS
+#define GCR_CMD_STATUS__GCR_CONTROL__SHIFT                                                                    0x0
+#define GCR_CMD_STATUS__GCR_SRC__SHIFT                                                                        0x13
+#define GCR_CMD_STATUS__GCR_TLB_SHOOTDOWN__SHIFT                                                              0x17
+#define GCR_CMD_STATUS__GCR_TLB_SHOOTDOWN_VMID__SHIFT                                                         0x18
+#define GCR_CMD_STATUS__UTCL2_NACK_STATUS__SHIFT                                                              0x1c
+#define GCR_CMD_STATUS__GCR_SEQ_OP_ERROR__SHIFT                                                               0x1e
+#define GCR_CMD_STATUS__UTCL2_NACK_ERROR__SHIFT                                                               0x1f
+#define GCR_CMD_STATUS__GCR_CONTROL_MASK                                                                      0x0007FFFFL
+#define GCR_CMD_STATUS__GCR_SRC_MASK                                                                          0x00780000L
+#define GCR_CMD_STATUS__GCR_TLB_SHOOTDOWN_MASK                                                                0x00800000L
+#define GCR_CMD_STATUS__GCR_TLB_SHOOTDOWN_VMID_MASK                                                           0x0F000000L
+#define GCR_CMD_STATUS__UTCL2_NACK_STATUS_MASK                                                                0x30000000L
+#define GCR_CMD_STATUS__GCR_SEQ_OP_ERROR_MASK                                                                 0x40000000L
+#define GCR_CMD_STATUS__UTCL2_NACK_ERROR_MASK                                                                 0x80000000L
+//GCR_SPARE
+#define GCR_SPARE__SPARE_BIT_1__SHIFT                                                                         0x1
+#define GCR_SPARE__SPARE_BIT_2__SHIFT                                                                         0x2
+#define GCR_SPARE__SPARE_BIT_3__SHIFT                                                                         0x3
+#define GCR_SPARE__SPARE_BIT_4__SHIFT                                                                         0x4
+#define GCR_SPARE__SPARE_BIT_5__SHIFT                                                                         0x5
+#define GCR_SPARE__SPARE_BIT_6__SHIFT                                                                         0x6
+#define GCR_SPARE__SPARE_BIT_7__SHIFT                                                                         0x7
+#define GCR_SPARE__UTCL2_REQ_CREDIT__SHIFT                                                                    0x8
+#define GCR_SPARE__GCRD_GL2A_REQ_CREDIT__SHIFT                                                                0x10
+#define GCR_SPARE__GCRD_SE_REQ_CREDIT__SHIFT                                                                  0x14
+#define GCR_SPARE__SPARE_BIT_31_24__SHIFT                                                                     0x18
+#define GCR_SPARE__SPARE_BIT_1_MASK                                                                           0x00000002L
+#define GCR_SPARE__SPARE_BIT_2_MASK                                                                           0x00000004L
+#define GCR_SPARE__SPARE_BIT_3_MASK                                                                           0x00000008L
+#define GCR_SPARE__SPARE_BIT_4_MASK                                                                           0x00000010L
+#define GCR_SPARE__SPARE_BIT_5_MASK                                                                           0x00000020L
+#define GCR_SPARE__SPARE_BIT_6_MASK                                                                           0x00000040L
+#define GCR_SPARE__SPARE_BIT_7_MASK                                                                           0x00000080L
+#define GCR_SPARE__UTCL2_REQ_CREDIT_MASK                                                                      0x0000FF00L
+#define GCR_SPARE__GCRD_GL2A_REQ_CREDIT_MASK                                                                  0x000F0000L
+#define GCR_SPARE__GCRD_SE_REQ_CREDIT_MASK                                                                    0x00F00000L
+#define GCR_SPARE__SPARE_BIT_31_24_MASK                                                                       0xFF000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_gfxudec
+//CP_EOP_DONE_ADDR_LO
+#define CP_EOP_DONE_ADDR_LO__ADDR_LO__SHIFT                                                                   0x2
+#define CP_EOP_DONE_ADDR_LO__ADDR_LO_MASK                                                                     0xFFFFFFFCL
+//CP_EOP_DONE_ADDR_HI
+#define CP_EOP_DONE_ADDR_HI__ADDR_HI__SHIFT                                                                   0x0
+#define CP_EOP_DONE_ADDR_HI__ADDR_HI_MASK                                                                     0x0000FFFFL
+//CP_EOP_DONE_DATA_LO
+#define CP_EOP_DONE_DATA_LO__DATA_LO__SHIFT                                                                   0x0
+#define CP_EOP_DONE_DATA_LO__DATA_LO_MASK                                                                     0xFFFFFFFFL
+//CP_EOP_DONE_DATA_HI
+#define CP_EOP_DONE_DATA_HI__DATA_HI__SHIFT                                                                   0x0
+#define CP_EOP_DONE_DATA_HI__DATA_HI_MASK                                                                     0xFFFFFFFFL
+//CP_EOP_LAST_FENCE_LO
+#define CP_EOP_LAST_FENCE_LO__LAST_FENCE_LO__SHIFT                                                            0x0
+#define CP_EOP_LAST_FENCE_LO__LAST_FENCE_LO_MASK                                                              0xFFFFFFFFL
+//CP_EOP_LAST_FENCE_HI
+#define CP_EOP_LAST_FENCE_HI__LAST_FENCE_HI__SHIFT                                                            0x0
+#define CP_EOP_LAST_FENCE_HI__LAST_FENCE_HI_MASK                                                              0xFFFFFFFFL
+//CP_PIPE_STATS_ADDR_LO
+#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_LO__SHIFT                                                      0x2
+#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_LO_MASK                                                        0xFFFFFFFCL
+//CP_PIPE_STATS_ADDR_HI
+#define CP_PIPE_STATS_ADDR_HI__PIPE_STATS_ADDR_HI__SHIFT                                                      0x0
+#define CP_PIPE_STATS_ADDR_HI__PIPE_STATS_ADDR_HI_MASK                                                        0x0000FFFFL
+//CP_VGT_IAVERT_COUNT_LO
+#define CP_VGT_IAVERT_COUNT_LO__IAVERT_COUNT_LO__SHIFT                                                        0x0
+#define CP_VGT_IAVERT_COUNT_LO__IAVERT_COUNT_LO_MASK                                                          0xFFFFFFFFL
+//CP_VGT_IAVERT_COUNT_HI
+#define CP_VGT_IAVERT_COUNT_HI__IAVERT_COUNT_HI__SHIFT                                                        0x0
+#define CP_VGT_IAVERT_COUNT_HI__IAVERT_COUNT_HI_MASK                                                          0xFFFFFFFFL
+//CP_VGT_IAPRIM_COUNT_LO
+#define CP_VGT_IAPRIM_COUNT_LO__IAPRIM_COUNT_LO__SHIFT                                                        0x0
+#define CP_VGT_IAPRIM_COUNT_LO__IAPRIM_COUNT_LO_MASK                                                          0xFFFFFFFFL
+//CP_VGT_IAPRIM_COUNT_HI
+#define CP_VGT_IAPRIM_COUNT_HI__IAPRIM_COUNT_HI__SHIFT                                                        0x0
+#define CP_VGT_IAPRIM_COUNT_HI__IAPRIM_COUNT_HI_MASK                                                          0xFFFFFFFFL
+//CP_VGT_GSPRIM_COUNT_LO
+#define CP_VGT_GSPRIM_COUNT_LO__GSPRIM_COUNT_LO__SHIFT                                                        0x0
+#define CP_VGT_GSPRIM_COUNT_LO__GSPRIM_COUNT_LO_MASK                                                          0xFFFFFFFFL
+//CP_VGT_GSPRIM_COUNT_HI
+#define CP_VGT_GSPRIM_COUNT_HI__GSPRIM_COUNT_HI__SHIFT                                                        0x0
+#define CP_VGT_GSPRIM_COUNT_HI__GSPRIM_COUNT_HI_MASK                                                          0xFFFFFFFFL
+//CP_VGT_VSINVOC_COUNT_LO
+#define CP_VGT_VSINVOC_COUNT_LO__VSINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_VSINVOC_COUNT_LO__VSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_VSINVOC_COUNT_HI
+#define CP_VGT_VSINVOC_COUNT_HI__VSINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_VSINVOC_COUNT_HI__VSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_VGT_GSINVOC_COUNT_LO
+#define CP_VGT_GSINVOC_COUNT_LO__GSINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_GSINVOC_COUNT_LO__GSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_GSINVOC_COUNT_HI
+#define CP_VGT_GSINVOC_COUNT_HI__GSINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_GSINVOC_COUNT_HI__GSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_VGT_HSINVOC_COUNT_LO
+#define CP_VGT_HSINVOC_COUNT_LO__HSINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_HSINVOC_COUNT_LO__HSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_HSINVOC_COUNT_HI
+#define CP_VGT_HSINVOC_COUNT_HI__HSINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_HSINVOC_COUNT_HI__HSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_VGT_DSINVOC_COUNT_LO
+#define CP_VGT_DSINVOC_COUNT_LO__DSINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_DSINVOC_COUNT_LO__DSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_DSINVOC_COUNT_HI
+#define CP_VGT_DSINVOC_COUNT_HI__DSINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_DSINVOC_COUNT_HI__DSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_PA_CINVOC_COUNT_LO
+#define CP_PA_CINVOC_COUNT_LO__CINVOC_COUNT_LO__SHIFT                                                         0x0
+#define CP_PA_CINVOC_COUNT_LO__CINVOC_COUNT_LO_MASK                                                           0xFFFFFFFFL
+//CP_PA_CINVOC_COUNT_HI
+#define CP_PA_CINVOC_COUNT_HI__CINVOC_COUNT_HI__SHIFT                                                         0x0
+#define CP_PA_CINVOC_COUNT_HI__CINVOC_COUNT_HI_MASK                                                           0xFFFFFFFFL
+//CP_PA_CPRIM_COUNT_LO
+#define CP_PA_CPRIM_COUNT_LO__CPRIM_COUNT_LO__SHIFT                                                           0x0
+#define CP_PA_CPRIM_COUNT_LO__CPRIM_COUNT_LO_MASK                                                             0xFFFFFFFFL
+//CP_PA_CPRIM_COUNT_HI
+#define CP_PA_CPRIM_COUNT_HI__CPRIM_COUNT_HI__SHIFT                                                           0x0
+#define CP_PA_CPRIM_COUNT_HI__CPRIM_COUNT_HI_MASK                                                             0xFFFFFFFFL
+//CP_SC_PSINVOC_COUNT0_LO
+#define CP_SC_PSINVOC_COUNT0_LO__PSINVOC_COUNT0_LO__SHIFT                                                     0x0
+#define CP_SC_PSINVOC_COUNT0_LO__PSINVOC_COUNT0_LO_MASK                                                       0xFFFFFFFFL
+//CP_SC_PSINVOC_COUNT0_HI
+#define CP_SC_PSINVOC_COUNT0_HI__PSINVOC_COUNT0_HI__SHIFT                                                     0x0
+#define CP_SC_PSINVOC_COUNT0_HI__PSINVOC_COUNT0_HI_MASK                                                       0xFFFFFFFFL
+//CP_SC_PSINVOC_COUNT1_LO
+#define CP_SC_PSINVOC_COUNT1_LO__OBSOLETE__SHIFT                                                              0x0
+#define CP_SC_PSINVOC_COUNT1_LO__OBSOLETE_MASK                                                                0xFFFFFFFFL
+//CP_SC_PSINVOC_COUNT1_HI
+#define CP_SC_PSINVOC_COUNT1_HI__OBSOLETE__SHIFT                                                              0x0
+#define CP_SC_PSINVOC_COUNT1_HI__OBSOLETE_MASK                                                                0xFFFFFFFFL
+//CP_VGT_CSINVOC_COUNT_LO
+#define CP_VGT_CSINVOC_COUNT_LO__CSINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_CSINVOC_COUNT_LO__CSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_CSINVOC_COUNT_HI
+#define CP_VGT_CSINVOC_COUNT_HI__CSINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_CSINVOC_COUNT_HI__CSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_VGT_ASINVOC_COUNT_LO
+#define CP_VGT_ASINVOC_COUNT_LO__ASINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_ASINVOC_COUNT_LO__ASINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_ASINVOC_COUNT_HI
+#define CP_VGT_ASINVOC_COUNT_HI__ASINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_ASINVOC_COUNT_HI__ASINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_PIPE_STATS_CONTROL
+#define CP_PIPE_STATS_CONTROL__SCOPE__SHIFT                                                                   0x17
+#define CP_PIPE_STATS_CONTROL__CACHE_POLICY__SHIFT                                                            0x19
+#define CP_PIPE_STATS_CONTROL__SCOPE_MASK                                                                     0x01800000L
+#define CP_PIPE_STATS_CONTROL__CACHE_POLICY_MASK                                                              0x06000000L
+//SCRATCH_REG0
+#define SCRATCH_REG0__SCRATCH_REG0__SHIFT                                                                     0x0
+#define SCRATCH_REG0__SCRATCH_REG0_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG1
+#define SCRATCH_REG1__SCRATCH_REG1__SHIFT                                                                     0x0
+#define SCRATCH_REG1__SCRATCH_REG1_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG2
+#define SCRATCH_REG2__SCRATCH_REG2__SHIFT                                                                     0x0
+#define SCRATCH_REG2__SCRATCH_REG2_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG3
+#define SCRATCH_REG3__SCRATCH_REG3__SHIFT                                                                     0x0
+#define SCRATCH_REG3__SCRATCH_REG3_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG4
+#define SCRATCH_REG4__SCRATCH_REG4__SHIFT                                                                     0x0
+#define SCRATCH_REG4__SCRATCH_REG4_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG5
+#define SCRATCH_REG5__SCRATCH_REG5__SHIFT                                                                     0x0
+#define SCRATCH_REG5__SCRATCH_REG5_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG6
+#define SCRATCH_REG6__SCRATCH_REG6__SHIFT                                                                     0x0
+#define SCRATCH_REG6__SCRATCH_REG6_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG7
+#define SCRATCH_REG7__SCRATCH_REG7__SHIFT                                                                     0x0
+#define SCRATCH_REG7__SCRATCH_REG7_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG_ATOMIC
+#define SCRATCH_REG_ATOMIC__IMMED__SHIFT                                                                      0x0
+#define SCRATCH_REG_ATOMIC__ID__SHIFT                                                                         0x18
+#define SCRATCH_REG_ATOMIC__reserved27__SHIFT                                                                 0x1b
+#define SCRATCH_REG_ATOMIC__OP__SHIFT                                                                         0x1c
+#define SCRATCH_REG_ATOMIC__reserved31__SHIFT                                                                 0x1f
+#define SCRATCH_REG_ATOMIC__IMMED_MASK                                                                        0x00FFFFFFL
+#define SCRATCH_REG_ATOMIC__ID_MASK                                                                           0x07000000L
+#define SCRATCH_REG_ATOMIC__reserved27_MASK                                                                   0x08000000L
+#define SCRATCH_REG_ATOMIC__OP_MASK                                                                           0x70000000L
+#define SCRATCH_REG_ATOMIC__reserved31_MASK                                                                   0x80000000L
+//SCRATCH_REG_CMPSWAP_ATOMIC
+#define SCRATCH_REG_CMPSWAP_ATOMIC__IMMED_COMPARE__SHIFT                                                      0x0
+#define SCRATCH_REG_CMPSWAP_ATOMIC__IMMED_REPLACE__SHIFT                                                      0xc
+#define SCRATCH_REG_CMPSWAP_ATOMIC__ID__SHIFT                                                                 0x18
+#define SCRATCH_REG_CMPSWAP_ATOMIC__reserved27__SHIFT                                                         0x1b
+#define SCRATCH_REG_CMPSWAP_ATOMIC__OP__SHIFT                                                                 0x1c
+#define SCRATCH_REG_CMPSWAP_ATOMIC__reserved31__SHIFT                                                         0x1f
+#define SCRATCH_REG_CMPSWAP_ATOMIC__IMMED_COMPARE_MASK                                                        0x00000FFFL
+#define SCRATCH_REG_CMPSWAP_ATOMIC__IMMED_REPLACE_MASK                                                        0x00FFF000L
+#define SCRATCH_REG_CMPSWAP_ATOMIC__ID_MASK                                                                   0x07000000L
+#define SCRATCH_REG_CMPSWAP_ATOMIC__reserved27_MASK                                                           0x08000000L
+#define SCRATCH_REG_CMPSWAP_ATOMIC__OP_MASK                                                                   0x70000000L
+#define SCRATCH_REG_CMPSWAP_ATOMIC__reserved31_MASK                                                           0x80000000L
+//CP_APPEND_DDID_CNT
+#define CP_APPEND_DDID_CNT__DATA__SHIFT                                                                       0x0
+#define CP_APPEND_DDID_CNT__DATA_MASK                                                                         0x000000FFL
+//CP_APPEND_DATA_HI
+#define CP_APPEND_DATA_HI__DATA__SHIFT                                                                        0x0
+#define CP_APPEND_DATA_HI__DATA_MASK                                                                          0xFFFFFFFFL
+//CP_APPEND_LAST_CS_FENCE_HI
+#define CP_APPEND_LAST_CS_FENCE_HI__LAST_FENCE__SHIFT                                                         0x0
+#define CP_APPEND_LAST_CS_FENCE_HI__LAST_FENCE_MASK                                                           0xFFFFFFFFL
+//CP_APPEND_LAST_PS_FENCE_HI
+#define CP_APPEND_LAST_PS_FENCE_HI__LAST_FENCE__SHIFT                                                         0x0
+#define CP_APPEND_LAST_PS_FENCE_HI__LAST_FENCE_MASK                                                           0xFFFFFFFFL
+//CP_PFP_ATOMIC_PREOP_LO
+#define CP_PFP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT                                                        0x0
+#define CP_PFP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK                                                          0xFFFFFFFFL
+//CP_PFP_ATOMIC_PREOP_HI
+#define CP_PFP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT                                                        0x0
+#define CP_PFP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK                                                          0xFFFFFFFFL
+//CP_APPEND_ADDR_LO
+#define CP_APPEND_ADDR_LO__MEM_ADDR_LO__SHIFT                                                                 0x2
+#define CP_APPEND_ADDR_LO__MEM_ADDR_LO_MASK                                                                   0xFFFFFFFCL
+//CP_APPEND_ADDR_HI
+#define CP_APPEND_ADDR_HI__MEM_ADDR_HI__SHIFT                                                                 0x0
+#define CP_APPEND_ADDR_HI__CS_PS_SEL__SHIFT                                                                   0x10
+#define CP_APPEND_ADDR_HI__FENCE_SIZE__SHIFT                                                                  0x12
+#define CP_APPEND_ADDR_HI__PWS_ENABLE__SHIFT                                                                  0x13
+#define CP_APPEND_ADDR_HI__CACHE_POLICY__SHIFT                                                                0x19
+#define CP_APPEND_ADDR_HI__SCOPE__SHIFT                                                                       0x1b
+#define CP_APPEND_ADDR_HI__COMMAND__SHIFT                                                                     0x1d
+#define CP_APPEND_ADDR_HI__MEM_ADDR_HI_MASK                                                                   0x0000FFFFL
+#define CP_APPEND_ADDR_HI__CS_PS_SEL_MASK                                                                     0x00030000L
+#define CP_APPEND_ADDR_HI__FENCE_SIZE_MASK                                                                    0x00040000L
+#define CP_APPEND_ADDR_HI__PWS_ENABLE_MASK                                                                    0x00080000L
+#define CP_APPEND_ADDR_HI__CACHE_POLICY_MASK                                                                  0x06000000L
+#define CP_APPEND_ADDR_HI__SCOPE_MASK                                                                         0x18000000L
+#define CP_APPEND_ADDR_HI__COMMAND_MASK                                                                       0xE0000000L
+//CP_APPEND_DATA
+#define CP_APPEND_DATA__DATA__SHIFT                                                                           0x0
+#define CP_APPEND_DATA__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_APPEND_DATA_LO
+#define CP_APPEND_DATA_LO__DATA__SHIFT                                                                        0x0
+#define CP_APPEND_DATA_LO__DATA_MASK                                                                          0xFFFFFFFFL
+//CP_APPEND_LAST_CS_FENCE
+#define CP_APPEND_LAST_CS_FENCE__LAST_FENCE__SHIFT                                                            0x0
+#define CP_APPEND_LAST_CS_FENCE__LAST_FENCE_MASK                                                              0xFFFFFFFFL
+//CP_APPEND_LAST_CS_FENCE_LO
+#define CP_APPEND_LAST_CS_FENCE_LO__LAST_FENCE__SHIFT                                                         0x0
+#define CP_APPEND_LAST_CS_FENCE_LO__LAST_FENCE_MASK                                                           0xFFFFFFFFL
+//CP_APPEND_LAST_PS_FENCE
+#define CP_APPEND_LAST_PS_FENCE__LAST_FENCE__SHIFT                                                            0x0
+#define CP_APPEND_LAST_PS_FENCE__LAST_FENCE_MASK                                                              0xFFFFFFFFL
+//CP_APPEND_LAST_PS_FENCE_LO
+#define CP_APPEND_LAST_PS_FENCE_LO__LAST_FENCE__SHIFT                                                         0x0
+#define CP_APPEND_LAST_PS_FENCE_LO__LAST_FENCE_MASK                                                           0xFFFFFFFFL
+//CP_ATOMIC_PREOP_LO
+#define CP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT                                                            0x0
+#define CP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK                                                              0xFFFFFFFFL
+//CP_ME_ATOMIC_PREOP_LO
+#define CP_ME_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT                                                         0x0
+#define CP_ME_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK                                                           0xFFFFFFFFL
+//CP_ATOMIC_PREOP_HI
+#define CP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT                                                            0x0
+#define CP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK                                                              0xFFFFFFFFL
+//CP_ME_ATOMIC_PREOP_HI
+#define CP_ME_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT                                                         0x0
+#define CP_ME_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK                                                           0xFFFFFFFFL
+//CP_ME_MC_WADDR_LO
+#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_LO__SHIFT                                                              0x2
+#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_LO_MASK                                                                0xFFFFFFFCL
+//CP_ME_MC_WADDR_HI
+#define CP_ME_MC_WADDR_HI__ME_MC_WADDR_HI__SHIFT                                                              0x0
+#define CP_ME_MC_WADDR_HI__WRITE_CONFIRM__SHIFT                                                               0x11
+#define CP_ME_MC_WADDR_HI__WRITE64__SHIFT                                                                     0x12
+#define CP_ME_MC_WADDR_HI__SCOPE__SHIFT                                                                       0x14
+#define CP_ME_MC_WADDR_HI__CACHE_POLICY__SHIFT                                                                0x16
+#define CP_ME_MC_WADDR_HI__VMID__SHIFT                                                                        0x18
+#define CP_ME_MC_WADDR_HI__RINGID__SHIFT                                                                      0x1c
+#define CP_ME_MC_WADDR_HI__PRIVILEGE__SHIFT                                                                   0x1f
+#define CP_ME_MC_WADDR_HI__ME_MC_WADDR_HI_MASK                                                                0x0000FFFFL
+#define CP_ME_MC_WADDR_HI__WRITE_CONFIRM_MASK                                                                 0x00020000L
+#define CP_ME_MC_WADDR_HI__WRITE64_MASK                                                                       0x00040000L
+#define CP_ME_MC_WADDR_HI__SCOPE_MASK                                                                         0x00300000L
+#define CP_ME_MC_WADDR_HI__CACHE_POLICY_MASK                                                                  0x00C00000L
+#define CP_ME_MC_WADDR_HI__VMID_MASK                                                                          0x0F000000L
+#define CP_ME_MC_WADDR_HI__RINGID_MASK                                                                        0x30000000L
+#define CP_ME_MC_WADDR_HI__PRIVILEGE_MASK                                                                     0x80000000L
+//CP_ME_MC_WDATA_LO
+#define CP_ME_MC_WDATA_LO__ME_MC_WDATA_LO__SHIFT                                                              0x0
+#define CP_ME_MC_WDATA_LO__ME_MC_WDATA_LO_MASK                                                                0xFFFFFFFFL
+//CP_ME_MC_WDATA_HI
+#define CP_ME_MC_WDATA_HI__ME_MC_WDATA_HI__SHIFT                                                              0x0
+#define CP_ME_MC_WDATA_HI__ME_MC_WDATA_HI_MASK                                                                0xFFFFFFFFL
+//CP_ME_MC_RADDR_LO
+#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_LO__SHIFT                                                              0x2
+#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_LO_MASK                                                                0xFFFFFFFCL
+//CP_ME_MC_RADDR_HI
+#define CP_ME_MC_RADDR_HI__ME_MC_RADDR_HI__SHIFT                                                              0x0
+#define CP_ME_MC_RADDR_HI__SIZE__SHIFT                                                                        0x10
+#define CP_ME_MC_RADDR_HI__SCOPE__SHIFT                                                                       0x14
+#define CP_ME_MC_RADDR_HI__CACHE_POLICY__SHIFT                                                                0x16
+#define CP_ME_MC_RADDR_HI__VMID__SHIFT                                                                        0x18
+#define CP_ME_MC_RADDR_HI__PRIVILEGE__SHIFT                                                                   0x1f
+#define CP_ME_MC_RADDR_HI__ME_MC_RADDR_HI_MASK                                                                0x0000FFFFL
+#define CP_ME_MC_RADDR_HI__SIZE_MASK                                                                          0x000F0000L
+#define CP_ME_MC_RADDR_HI__SCOPE_MASK                                                                         0x00300000L
+#define CP_ME_MC_RADDR_HI__CACHE_POLICY_MASK                                                                  0x00C00000L
+#define CP_ME_MC_RADDR_HI__VMID_MASK                                                                          0x0F000000L
+#define CP_ME_MC_RADDR_HI__PRIVILEGE_MASK                                                                     0x80000000L
+//CP_WAIT_REG_MEM_TIMEOUT
+#define CP_WAIT_REG_MEM_TIMEOUT__WAIT_REG_MEM_TIMEOUT__SHIFT                                                  0x0
+#define CP_WAIT_REG_MEM_TIMEOUT__WAIT_REG_MEM_TIMEOUT_MASK                                                    0xFFFFFFFFL
+//CP_DMA_PFP_CONTROL
+#define CP_DMA_PFP_CONTROL__VMID__SHIFT                                                                       0x0
+#define CP_DMA_PFP_CONTROL__TMZ__SHIFT                                                                        0x4
+#define CP_DMA_PFP_CONTROL__MEMLOG_CLEAR__SHIFT                                                               0xa
+#define CP_DMA_PFP_CONTROL__SRC_CACHE_POLICY__SHIFT                                                           0xd
+#define CP_DMA_PFP_CONTROL__SRC_SCOPE__SHIFT                                                                  0xf
+#define CP_DMA_PFP_CONTROL__DST_SELECT__SHIFT                                                                 0x14
+#define CP_DMA_PFP_CONTROL__DST_CACHE_POLICY__SHIFT                                                           0x19
+#define CP_DMA_PFP_CONTROL__DST_SCOPE__SHIFT                                                                  0x1b
+#define CP_DMA_PFP_CONTROL__SRC_SELECT__SHIFT                                                                 0x1d
+#define CP_DMA_PFP_CONTROL__VMID_MASK                                                                         0x0000000FL
+#define CP_DMA_PFP_CONTROL__TMZ_MASK                                                                          0x00000010L
+#define CP_DMA_PFP_CONTROL__MEMLOG_CLEAR_MASK                                                                 0x00000400L
+#define CP_DMA_PFP_CONTROL__SRC_CACHE_POLICY_MASK                                                             0x00006000L
+#define CP_DMA_PFP_CONTROL__SRC_SCOPE_MASK                                                                    0x00018000L
+#define CP_DMA_PFP_CONTROL__DST_SELECT_MASK                                                                   0x00300000L
+#define CP_DMA_PFP_CONTROL__DST_CACHE_POLICY_MASK                                                             0x06000000L
+#define CP_DMA_PFP_CONTROL__DST_SCOPE_MASK                                                                    0x18000000L
+#define CP_DMA_PFP_CONTROL__SRC_SELECT_MASK                                                                   0x60000000L
+//CP_DMA_ME_CONTROL
+#define CP_DMA_ME_CONTROL__VMID__SHIFT                                                                        0x0
+#define CP_DMA_ME_CONTROL__TMZ__SHIFT                                                                         0x4
+#define CP_DMA_ME_CONTROL__MEMLOG_CLEAR__SHIFT                                                                0xa
+#define CP_DMA_ME_CONTROL__SRC_CACHE_POLICY__SHIFT                                                            0xd
+#define CP_DMA_ME_CONTROL__SRC_SCOPE__SHIFT                                                                   0xf
+#define CP_DMA_ME_CONTROL__DST_SELECT__SHIFT                                                                  0x14
+#define CP_DMA_ME_CONTROL__DST_CACHE_POLICY__SHIFT                                                            0x19
+#define CP_DMA_ME_CONTROL__DST_SCOPE__SHIFT                                                                   0x1b
+#define CP_DMA_ME_CONTROL__SRC_SELECT__SHIFT                                                                  0x1d
+#define CP_DMA_ME_CONTROL__VMID_MASK                                                                          0x0000000FL
+#define CP_DMA_ME_CONTROL__TMZ_MASK                                                                           0x00000010L
+#define CP_DMA_ME_CONTROL__MEMLOG_CLEAR_MASK                                                                  0x00000400L
+#define CP_DMA_ME_CONTROL__SRC_CACHE_POLICY_MASK                                                              0x00006000L
+#define CP_DMA_ME_CONTROL__SRC_SCOPE_MASK                                                                     0x00018000L
+#define CP_DMA_ME_CONTROL__DST_SELECT_MASK                                                                    0x00300000L
+#define CP_DMA_ME_CONTROL__DST_CACHE_POLICY_MASK                                                              0x06000000L
+#define CP_DMA_ME_CONTROL__DST_SCOPE_MASK                                                                     0x18000000L
+#define CP_DMA_ME_CONTROL__SRC_SELECT_MASK                                                                    0x60000000L
+//CP_DMA_ME_SRC_ADDR
+#define CP_DMA_ME_SRC_ADDR__SRC_ADDR__SHIFT                                                                   0x0
+#define CP_DMA_ME_SRC_ADDR__SRC_ADDR_MASK                                                                     0xFFFFFFFFL
+//CP_DMA_ME_SRC_ADDR_HI
+#define CP_DMA_ME_SRC_ADDR_HI__SRC_ADDR_HI__SHIFT                                                             0x0
+#define CP_DMA_ME_SRC_ADDR_HI__SRC_TRANS_ERROR_TYPE__SHIFT                                                    0x1b
+#define CP_DMA_ME_SRC_ADDR_HI__SRC_TRANS_ERROR_VMID__SHIFT                                                    0x1c
+#define CP_DMA_ME_SRC_ADDR_HI__SRC_ADDR_HI_MASK                                                               0x01FFFFFFL
+#define CP_DMA_ME_SRC_ADDR_HI__SRC_TRANS_ERROR_TYPE_MASK                                                      0x08000000L
+#define CP_DMA_ME_SRC_ADDR_HI__SRC_TRANS_ERROR_VMID_MASK                                                      0xF0000000L
+//CP_DMA_ME_DST_ADDR
+#define CP_DMA_ME_DST_ADDR__DST_ADDR__SHIFT                                                                   0x0
+#define CP_DMA_ME_DST_ADDR__DST_ADDR_MASK                                                                     0xFFFFFFFFL
+//CP_DMA_ME_DST_ADDR_HI
+#define CP_DMA_ME_DST_ADDR_HI__DST_ADDR_HI__SHIFT                                                             0x0
+#define CP_DMA_ME_DST_ADDR_HI__DST_TRANS_ERROR_TYPE__SHIFT                                                    0x1b
+#define CP_DMA_ME_DST_ADDR_HI__DST_TRANS_ERROR_VMID__SHIFT                                                    0x1c
+#define CP_DMA_ME_DST_ADDR_HI__DST_ADDR_HI_MASK                                                               0x01FFFFFFL
+#define CP_DMA_ME_DST_ADDR_HI__DST_TRANS_ERROR_TYPE_MASK                                                      0x08000000L
+#define CP_DMA_ME_DST_ADDR_HI__DST_TRANS_ERROR_VMID_MASK                                                      0xF0000000L
+//CP_DMA_ME_COMMAND
+#define CP_DMA_ME_COMMAND__BYTE_COUNT__SHIFT                                                                  0x0
+#define CP_DMA_ME_COMMAND__SAS__SHIFT                                                                         0x1a
+#define CP_DMA_ME_COMMAND__DAS__SHIFT                                                                         0x1b
+#define CP_DMA_ME_COMMAND__SAIC__SHIFT                                                                        0x1c
+#define CP_DMA_ME_COMMAND__DAIC__SHIFT                                                                        0x1d
+#define CP_DMA_ME_COMMAND__RAW_WAIT__SHIFT                                                                    0x1e
+#define CP_DMA_ME_COMMAND__DIS_WC__SHIFT                                                                      0x1f
+#define CP_DMA_ME_COMMAND__BYTE_COUNT_MASK                                                                    0x03FFFFFFL
+#define CP_DMA_ME_COMMAND__SAS_MASK                                                                           0x04000000L
+#define CP_DMA_ME_COMMAND__DAS_MASK                                                                           0x08000000L
+#define CP_DMA_ME_COMMAND__SAIC_MASK                                                                          0x10000000L
+#define CP_DMA_ME_COMMAND__DAIC_MASK                                                                          0x20000000L
+#define CP_DMA_ME_COMMAND__RAW_WAIT_MASK                                                                      0x40000000L
+#define CP_DMA_ME_COMMAND__DIS_WC_MASK                                                                        0x80000000L
+//CP_DMA_PFP_SRC_ADDR
+#define CP_DMA_PFP_SRC_ADDR__SRC_ADDR__SHIFT                                                                  0x0
+#define CP_DMA_PFP_SRC_ADDR__SRC_ADDR_MASK                                                                    0xFFFFFFFFL
+//CP_DMA_PFP_SRC_ADDR_HI
+#define CP_DMA_PFP_SRC_ADDR_HI__SRC_ADDR_HI__SHIFT                                                            0x0
+#define CP_DMA_PFP_SRC_ADDR_HI__SRC_ADDR_HI_MASK                                                              0x0000FFFFL
+//CP_DMA_PFP_DST_ADDR
+#define CP_DMA_PFP_DST_ADDR__DST_ADDR__SHIFT                                                                  0x0
+#define CP_DMA_PFP_DST_ADDR__DST_ADDR_MASK                                                                    0xFFFFFFFFL
+//CP_DMA_PFP_DST_ADDR_HI
+#define CP_DMA_PFP_DST_ADDR_HI__DST_ADDR_HI__SHIFT                                                            0x0
+#define CP_DMA_PFP_DST_ADDR_HI__DST_ADDR_HI_MASK                                                              0x0000FFFFL
+//CP_DMA_PFP_COMMAND
+#define CP_DMA_PFP_COMMAND__BYTE_COUNT__SHIFT                                                                 0x0
+#define CP_DMA_PFP_COMMAND__SAS__SHIFT                                                                        0x1a
+#define CP_DMA_PFP_COMMAND__DAS__SHIFT                                                                        0x1b
+#define CP_DMA_PFP_COMMAND__SAIC__SHIFT                                                                       0x1c
+#define CP_DMA_PFP_COMMAND__DAIC__SHIFT                                                                       0x1d
+#define CP_DMA_PFP_COMMAND__RAW_WAIT__SHIFT                                                                   0x1e
+#define CP_DMA_PFP_COMMAND__DIS_WC__SHIFT                                                                     0x1f
+#define CP_DMA_PFP_COMMAND__BYTE_COUNT_MASK                                                                   0x03FFFFFFL
+#define CP_DMA_PFP_COMMAND__SAS_MASK                                                                          0x04000000L
+#define CP_DMA_PFP_COMMAND__DAS_MASK                                                                          0x08000000L
+#define CP_DMA_PFP_COMMAND__SAIC_MASK                                                                         0x10000000L
+#define CP_DMA_PFP_COMMAND__DAIC_MASK                                                                         0x20000000L
+#define CP_DMA_PFP_COMMAND__RAW_WAIT_MASK                                                                     0x40000000L
+#define CP_DMA_PFP_COMMAND__DIS_WC_MASK                                                                       0x80000000L
+//CP_DMA_CNTL
+#define CP_DMA_CNTL__UTCL1_FAULT_CONTROL__SHIFT                                                               0x0
+#define CP_DMA_CNTL__WATCH_CONTROL__SHIFT                                                                     0x1
+#define CP_DMA_CNTL__MIN_AVAILSZ__SHIFT                                                                       0x4
+#define CP_DMA_CNTL__SPECULATIVE_DATA_READ__SHIFT                                                             0x6
+#define CP_DMA_CNTL__BUFFER_DEPTH__SHIFT                                                                      0x10
+#define CP_DMA_CNTL__PIO_FIFO_EMPTY__SHIFT                                                                    0x1c
+#define CP_DMA_CNTL__PIO_FIFO_FULL__SHIFT                                                                     0x1d
+#define CP_DMA_CNTL__PIO_COUNT__SHIFT                                                                         0x1e
+#define CP_DMA_CNTL__UTCL1_FAULT_CONTROL_MASK                                                                 0x00000001L
+#define CP_DMA_CNTL__WATCH_CONTROL_MASK                                                                       0x00000002L
+#define CP_DMA_CNTL__MIN_AVAILSZ_MASK                                                                         0x00000030L
+#define CP_DMA_CNTL__SPECULATIVE_DATA_READ_MASK                                                               0x000000C0L
+#define CP_DMA_CNTL__BUFFER_DEPTH_MASK                                                                        0x01FF0000L
+#define CP_DMA_CNTL__PIO_FIFO_EMPTY_MASK                                                                      0x10000000L
+#define CP_DMA_CNTL__PIO_FIFO_FULL_MASK                                                                       0x20000000L
+#define CP_DMA_CNTL__PIO_COUNT_MASK                                                                           0xC0000000L
+//CP_DMA_READ_TAGS
+#define CP_DMA_READ_TAGS__DMA_READ_TAG__SHIFT                                                                 0x0
+#define CP_DMA_READ_TAGS__DMA_READ_TAG_VALID__SHIFT                                                           0x1c
+#define CP_DMA_READ_TAGS__DMA_READ_TAG_MASK                                                                   0x03FFFFFFL
+#define CP_DMA_READ_TAGS__DMA_READ_TAG_VALID_MASK                                                             0x10000000L
+//CP_PFP_IB_CONTROL
+#define CP_PFP_IB_CONTROL__IB_EN__SHIFT                                                                       0x0
+#define CP_PFP_IB_CONTROL__IB_EN_MASK                                                                         0x000000FFL
+//CP_PFP_LOAD_CONTROL
+#define CP_PFP_LOAD_CONTROL__CONFIG_REG_EN__SHIFT                                                             0x0
+#define CP_PFP_LOAD_CONTROL__CNTX_REG_EN__SHIFT                                                               0x1
+#define CP_PFP_LOAD_CONTROL__UCONFIG_REG_EN__SHIFT                                                            0xf
+#define CP_PFP_LOAD_CONTROL__SH_GFX_REG_EN__SHIFT                                                             0x10
+#define CP_PFP_LOAD_CONTROL__SH_CS_REG_EN__SHIFT                                                              0x18
+#define CP_PFP_LOAD_CONTROL__LOAD_ORDINAL__SHIFT                                                              0x1f
+#define CP_PFP_LOAD_CONTROL__CONFIG_REG_EN_MASK                                                               0x00000001L
+#define CP_PFP_LOAD_CONTROL__CNTX_REG_EN_MASK                                                                 0x00000002L
+#define CP_PFP_LOAD_CONTROL__UCONFIG_REG_EN_MASK                                                              0x00008000L
+#define CP_PFP_LOAD_CONTROL__SH_GFX_REG_EN_MASK                                                               0x00010000L
+#define CP_PFP_LOAD_CONTROL__SH_CS_REG_EN_MASK                                                                0x01000000L
+#define CP_PFP_LOAD_CONTROL__LOAD_ORDINAL_MASK                                                                0x80000000L
+//CP_SCRATCH_INDEX
+#define CP_SCRATCH_INDEX__SCRATCH_INDEX__SHIFT                                                                0x0
+#define CP_SCRATCH_INDEX__SCRATCH_INDEX_64BIT_MODE__SHIFT                                                     0x1f
+#define CP_SCRATCH_INDEX__SCRATCH_INDEX_MASK                                                                  0x000001FFL
+#define CP_SCRATCH_INDEX__SCRATCH_INDEX_64BIT_MODE_MASK                                                       0x80000000L
+//CP_SCRATCH_DATA
+#define CP_SCRATCH_DATA__SCRATCH_DATA__SHIFT                                                                  0x0
+#define CP_SCRATCH_DATA__SCRATCH_DATA_MASK                                                                    0xFFFFFFFFL
+//CP_RB_OFFSET
+#define CP_RB_OFFSET__RB_OFFSET__SHIFT                                                                        0x0
+#define CP_RB_OFFSET__RB_OFFSET_MASK                                                                          0x000FFFFFL
+//CP_IB1_OFFSET
+#define CP_IB1_OFFSET__IB1_OFFSET__SHIFT                                                                      0x0
+#define CP_IB1_OFFSET__IB1_OFFSET_MASK                                                                        0x000FFFFFL
+//CP_IB2_OFFSET
+#define CP_IB2_OFFSET__IB2_OFFSET__SHIFT                                                                      0x0
+#define CP_IB2_OFFSET__IB2_OFFSET_MASK                                                                        0x000FFFFFL
+//CP_IB1_PREAMBLE_BEGIN
+#define CP_IB1_PREAMBLE_BEGIN__IB1_PREAMBLE_BEGIN__SHIFT                                                      0x0
+#define CP_IB1_PREAMBLE_BEGIN__IB1_PREAMBLE_BEGIN_MASK                                                        0x000FFFFFL
+//CP_IB1_PREAMBLE_END
+#define CP_IB1_PREAMBLE_END__IB1_PREAMBLE_END__SHIFT                                                          0x0
+#define CP_IB1_PREAMBLE_END__IB1_PREAMBLE_END_MASK                                                            0x000FFFFFL
+//CP_IB2_PREAMBLE_BEGIN
+#define CP_IB2_PREAMBLE_BEGIN__IB2_PREAMBLE_BEGIN__SHIFT                                                      0x0
+#define CP_IB2_PREAMBLE_BEGIN__IB2_PREAMBLE_BEGIN_MASK                                                        0x000FFFFFL
+//CP_IB2_PREAMBLE_END
+#define CP_IB2_PREAMBLE_END__IB2_PREAMBLE_END__SHIFT                                                          0x0
+#define CP_IB2_PREAMBLE_END__IB2_PREAMBLE_END_MASK                                                            0x000FFFFFL
+//CP_DMA_ME_CMD_ADDR_LO
+#define CP_DMA_ME_CMD_ADDR_LO__RSVD__SHIFT                                                                    0x0
+#define CP_DMA_ME_CMD_ADDR_LO__ADDR_LO__SHIFT                                                                 0x2
+#define CP_DMA_ME_CMD_ADDR_LO__RSVD_MASK                                                                      0x00000003L
+#define CP_DMA_ME_CMD_ADDR_LO__ADDR_LO_MASK                                                                   0xFFFFFFFCL
+//CP_DMA_ME_CMD_ADDR_HI
+#define CP_DMA_ME_CMD_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CP_DMA_ME_CMD_ADDR_HI__RSVD__SHIFT                                                                    0x19
+#define CP_DMA_ME_CMD_ADDR_HI__ADDR_HI_MASK                                                                   0x01FFFFFFL
+#define CP_DMA_ME_CMD_ADDR_HI__RSVD_MASK                                                                      0xFE000000L
+//CP_DMA_PFP_CMD_ADDR_LO
+#define CP_DMA_PFP_CMD_ADDR_LO__RSVD__SHIFT                                                                   0x0
+#define CP_DMA_PFP_CMD_ADDR_LO__ADDR_LO__SHIFT                                                                0x2
+#define CP_DMA_PFP_CMD_ADDR_LO__RSVD_MASK                                                                     0x00000003L
+#define CP_DMA_PFP_CMD_ADDR_LO__ADDR_LO_MASK                                                                  0xFFFFFFFCL
+//CP_DMA_PFP_CMD_ADDR_HI
+#define CP_DMA_PFP_CMD_ADDR_HI__ADDR_HI__SHIFT                                                                0x0
+#define CP_DMA_PFP_CMD_ADDR_HI__RSVD__SHIFT                                                                   0x10
+#define CP_DMA_PFP_CMD_ADDR_HI__ADDR_HI_MASK                                                                  0x0000FFFFL
+#define CP_DMA_PFP_CMD_ADDR_HI__RSVD_MASK                                                                     0xFFFF0000L
+//UCONFIG_RESERVED_REG0
+#define UCONFIG_RESERVED_REG0__DATA__SHIFT                                                                    0x0
+#define UCONFIG_RESERVED_REG0__DATA_MASK                                                                      0xFFFFFFFFL
+//UCONFIG_RESERVED_REG1
+#define UCONFIG_RESERVED_REG1__DATA__SHIFT                                                                    0x0
+#define UCONFIG_RESERVED_REG1__DATA_MASK                                                                      0xFFFFFFFFL
+//CP_PA_MSPRIM_COUNT_LO
+#define CP_PA_MSPRIM_COUNT_LO__MSPRIM_COUNT_LO__SHIFT                                                         0x0
+#define CP_PA_MSPRIM_COUNT_LO__MSPRIM_COUNT_LO_MASK                                                           0xFFFFFFFFL
+//CP_PA_MSPRIM_COUNT_HI
+#define CP_PA_MSPRIM_COUNT_HI__MSPRIM_COUNT_HI__SHIFT                                                         0x0
+#define CP_PA_MSPRIM_COUNT_HI__MSPRIM_COUNT_HI_MASK                                                           0xFFFFFFFFL
+//CP_GE_MSINVOC_COUNT_LO
+#define CP_GE_MSINVOC_COUNT_LO__MSINVOC_COUNT_LO__SHIFT                                                       0x0
+#define CP_GE_MSINVOC_COUNT_LO__MSINVOC_COUNT_LO_MASK                                                         0xFFFFFFFFL
+//CP_GE_MSINVOC_COUNT_HI
+#define CP_GE_MSINVOC_COUNT_HI__MSINVOC_COUNT_HI__SHIFT                                                       0x0
+#define CP_GE_MSINVOC_COUNT_HI__MSINVOC_COUNT_HI_MASK                                                         0xFFFFFFFFL
+//CP_IB1_CMD_BUFSZ
+#define CP_IB1_CMD_BUFSZ__IB1_CMD_REQSZ__SHIFT                                                                0x0
+#define CP_IB1_CMD_BUFSZ__IB1_CMD_REQSZ_MASK                                                                  0x000FFFFFL
+//CP_IB2_CMD_BUFSZ
+#define CP_IB2_CMD_BUFSZ__IB2_CMD_REQSZ__SHIFT                                                                0x0
+#define CP_IB2_CMD_BUFSZ__IB2_CMD_REQSZ_MASK                                                                  0x000FFFFFL
+//CP_ST_CMD_BUFSZ
+#define CP_ST_CMD_BUFSZ__ST_CMD_REQSZ__SHIFT                                                                  0x0
+#define CP_ST_CMD_BUFSZ__ST_CMD_REQSZ_MASK                                                                    0x000FFFFFL
+//CP_IB1_BASE_LO
+#define CP_IB1_BASE_LO__IB1_BASE_LO__SHIFT                                                                    0x2
+#define CP_IB1_BASE_LO__IB1_BASE_LO_MASK                                                                      0xFFFFFFFCL
+//CP_IB1_BASE_HI
+#define CP_IB1_BASE_HI__IB1_BASE_HI__SHIFT                                                                    0x0
+#define CP_IB1_BASE_HI__IB1_BASE_HI_MASK                                                                      0x0000FFFFL
+//CP_IB1_BUFSZ
+#define CP_IB1_BUFSZ__IB1_BUFSZ__SHIFT                                                                        0x0
+#define CP_IB1_BUFSZ__IB1_BUFSZ_MASK                                                                          0x000FFFFFL
+//CP_IB2_BASE_LO
+#define CP_IB2_BASE_LO__IB2_BASE_LO__SHIFT                                                                    0x2
+#define CP_IB2_BASE_LO__IB2_BASE_LO_MASK                                                                      0xFFFFFFFCL
+//CP_IB2_BASE_HI
+#define CP_IB2_BASE_HI__IB2_BASE_HI__SHIFT                                                                    0x0
+#define CP_IB2_BASE_HI__IB2_BASE_HI_MASK                                                                      0x0000FFFFL
+//CP_IB2_BUFSZ
+#define CP_IB2_BUFSZ__IB2_BUFSZ__SHIFT                                                                        0x0
+#define CP_IB2_BUFSZ__IB2_BUFSZ_MASK                                                                          0x000FFFFFL
+//CP_ST_BASE_LO
+#define CP_ST_BASE_LO__ST_BASE_LO__SHIFT                                                                      0x2
+#define CP_ST_BASE_LO__ST_BASE_LO_MASK                                                                        0xFFFFFFFCL
+//CP_ST_BASE_HI
+#define CP_ST_BASE_HI__ST_BASE_HI__SHIFT                                                                      0x0
+#define CP_ST_BASE_HI__ST_BASE_HI_MASK                                                                        0x0000FFFFL
+//CP_ST_BUFSZ
+#define CP_ST_BUFSZ__ST_BUFSZ__SHIFT                                                                          0x0
+#define CP_ST_BUFSZ__ST_BUFSZ_MASK                                                                            0x000FFFFFL
+//CP_EOP_DONE_EVENT_CNTL
+#define CP_EOP_DONE_EVENT_CNTL__SCOPE__SHIFT                                                                  0x7
+#define CP_EOP_DONE_EVENT_CNTL__GCR_CNTL__SHIFT                                                               0xc
+#define CP_EOP_DONE_EVENT_CNTL__CACHE_POLICY__SHIFT                                                           0x19
+#define CP_EOP_DONE_EVENT_CNTL__EXECUTE__SHIFT                                                                0x1c
+#define CP_EOP_DONE_EVENT_CNTL__GLK_INV__SHIFT                                                                0x1e
+#define CP_EOP_DONE_EVENT_CNTL__PWS_ENABLE__SHIFT                                                             0x1f
+#define CP_EOP_DONE_EVENT_CNTL__SCOPE_MASK                                                                    0x00000180L
+#define CP_EOP_DONE_EVENT_CNTL__GCR_CNTL_MASK                                                                 0x01FFF000L
+#define CP_EOP_DONE_EVENT_CNTL__CACHE_POLICY_MASK                                                             0x06000000L
+#define CP_EOP_DONE_EVENT_CNTL__EXECUTE_MASK                                                                  0x10000000L
+#define CP_EOP_DONE_EVENT_CNTL__GLK_INV_MASK                                                                  0x40000000L
+#define CP_EOP_DONE_EVENT_CNTL__PWS_ENABLE_MASK                                                               0x80000000L
+//CP_EOP_DONE_DATA_CNTL
+#define CP_EOP_DONE_DATA_CNTL__DST_SEL__SHIFT                                                                 0x10
+#define CP_EOP_DONE_DATA_CNTL__ACTION_PIPE_ID__SHIFT                                                          0x14
+#define CP_EOP_DONE_DATA_CNTL__ACTION_ID__SHIFT                                                               0x16
+#define CP_EOP_DONE_DATA_CNTL__INT_SEL__SHIFT                                                                 0x18
+#define CP_EOP_DONE_DATA_CNTL__DATA_SEL__SHIFT                                                                0x1d
+#define CP_EOP_DONE_DATA_CNTL__DST_SEL_MASK                                                                   0x00030000L
+#define CP_EOP_DONE_DATA_CNTL__ACTION_PIPE_ID_MASK                                                            0x00300000L
+#define CP_EOP_DONE_DATA_CNTL__ACTION_ID_MASK                                                                 0x00C00000L
+#define CP_EOP_DONE_DATA_CNTL__INT_SEL_MASK                                                                   0x07000000L
+#define CP_EOP_DONE_DATA_CNTL__DATA_SEL_MASK                                                                  0xE0000000L
+//CP_EOP_DONE_CNTX_ID
+#define CP_EOP_DONE_CNTX_ID__CNTX_ID__SHIFT                                                                   0x0
+#define CP_EOP_DONE_CNTX_ID__CNTX_ID_MASK                                                                     0xFFFFFFFFL
+//CP_DB_BASE_LO
+#define CP_DB_BASE_LO__DB_BASE_LO__SHIFT                                                                      0x2
+#define CP_DB_BASE_LO__DB_BASE_LO_MASK                                                                        0xFFFFFFFCL
+//CP_DB_BASE_HI
+#define CP_DB_BASE_HI__DB_BASE_HI__SHIFT                                                                      0x0
+#define CP_DB_BASE_HI__DB_BASE_HI_MASK                                                                        0x0000FFFFL
+//CP_DB_BUFSZ
+#define CP_DB_BUFSZ__DB_BUFSZ__SHIFT                                                                          0x0
+#define CP_DB_BUFSZ__DB_BUFSZ_MASK                                                                            0x000FFFFFL
+//CP_DB_CMD_BUFSZ
+#define CP_DB_CMD_BUFSZ__DB_CMD_REQSZ__SHIFT                                                                  0x0
+#define CP_DB_CMD_BUFSZ__DB_CMD_REQSZ_MASK                                                                    0x000FFFFFL
+//CP_PFP_COMPLETION_STATUS
+#define CP_PFP_COMPLETION_STATUS__STATUS__SHIFT                                                               0x0
+#define CP_PFP_COMPLETION_STATUS__STATUS_MASK                                                                 0x00000003L
+//CP_PRED_NOT_VISIBLE
+#define CP_PRED_NOT_VISIBLE__NOT_VISIBLE__SHIFT                                                               0x0
+#define CP_PRED_NOT_VISIBLE__NOT_VISIBLE_MASK                                                                 0x00000001L
+//CP_PFP_METADATA_BASE_ADDR
+#define CP_PFP_METADATA_BASE_ADDR__ADDR_LO__SHIFT                                                             0x0
+#define CP_PFP_METADATA_BASE_ADDR__ADDR_LO_MASK                                                               0xFFFFFFFFL
+//CP_PFP_METADATA_BASE_ADDR_HI
+#define CP_PFP_METADATA_BASE_ADDR_HI__ADDR_HI__SHIFT                                                          0x0
+#define CP_PFP_METADATA_BASE_ADDR_HI__ADDR_HI_MASK                                                            0x0000FFFFL
+//CP_DRAW_INDX_INDR_ADDR
+#define CP_DRAW_INDX_INDR_ADDR__ADDR_LO__SHIFT                                                                0x0
+#define CP_DRAW_INDX_INDR_ADDR__ADDR_LO_MASK                                                                  0xFFFFFFFFL
+//CP_DRAW_INDX_INDR_ADDR_HI
+#define CP_DRAW_INDX_INDR_ADDR_HI__ADDR_HI__SHIFT                                                             0x0
+#define CP_DRAW_INDX_INDR_ADDR_HI__ADDR_HI_MASK                                                               0x0000FFFFL
+//CP_DISPATCH_INDR_ADDR
+#define CP_DISPATCH_INDR_ADDR__ADDR_LO__SHIFT                                                                 0x0
+#define CP_DISPATCH_INDR_ADDR__ADDR_LO_MASK                                                                   0xFFFFFFFFL
+//CP_DISPATCH_INDR_ADDR_HI
+#define CP_DISPATCH_INDR_ADDR_HI__ADDR_HI__SHIFT                                                              0x0
+#define CP_DISPATCH_INDR_ADDR_HI__ADDR_HI_MASK                                                                0x0000FFFFL
+//CP_INDEX_BASE_ADDR
+#define CP_INDEX_BASE_ADDR__ADDR_LO__SHIFT                                                                    0x0
+#define CP_INDEX_BASE_ADDR__ADDR_LO_MASK                                                                      0xFFFFFFFFL
+//CP_INDEX_BASE_ADDR_HI
+#define CP_INDEX_BASE_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CP_INDEX_BASE_ADDR_HI__ADDR_HI_MASK                                                                   0x0000FFFFL
+//CP_INDEX_TYPE
+#define CP_INDEX_TYPE__INDEX_TYPE__SHIFT                                                                      0x0
+#define CP_INDEX_TYPE__INDEX_TYPE_MASK                                                                        0x00000003L
+//CP_SAMPLE_STATUS
+#define CP_SAMPLE_STATUS__Z_PASS_ACITVE__SHIFT                                                                0x0
+#define CP_SAMPLE_STATUS__STREAMOUT_ACTIVE__SHIFT                                                             0x1
+#define CP_SAMPLE_STATUS__PIPELINE_ACTIVE__SHIFT                                                              0x2
+#define CP_SAMPLE_STATUS__STIPPLE_ACTIVE__SHIFT                                                               0x3
+#define CP_SAMPLE_STATUS__VGT_BUFFERS_ACTIVE__SHIFT                                                           0x4
+#define CP_SAMPLE_STATUS__SCREEN_EXT_ACTIVE__SHIFT                                                            0x5
+#define CP_SAMPLE_STATUS__DRAW_INDIRECT_ACTIVE__SHIFT                                                         0x6
+#define CP_SAMPLE_STATUS__DISP_INDIRECT_ACTIVE__SHIFT                                                         0x7
+#define CP_SAMPLE_STATUS__Z_PASS_ACITVE_MASK                                                                  0x00000001L
+#define CP_SAMPLE_STATUS__STREAMOUT_ACTIVE_MASK                                                               0x00000002L
+#define CP_SAMPLE_STATUS__PIPELINE_ACTIVE_MASK                                                                0x00000004L
+#define CP_SAMPLE_STATUS__STIPPLE_ACTIVE_MASK                                                                 0x00000008L
+#define CP_SAMPLE_STATUS__VGT_BUFFERS_ACTIVE_MASK                                                             0x00000010L
+#define CP_SAMPLE_STATUS__SCREEN_EXT_ACTIVE_MASK                                                              0x00000020L
+#define CP_SAMPLE_STATUS__DRAW_INDIRECT_ACTIVE_MASK                                                           0x00000040L
+#define CP_SAMPLE_STATUS__DISP_INDIRECT_ACTIVE_MASK                                                           0x00000080L
+//CP_ME_COHER_CNTL
+#define CP_ME_COHER_CNTL__DEST_BASE_0_ENA__SHIFT                                                              0x0
+#define CP_ME_COHER_CNTL__DEST_BASE_1_ENA__SHIFT                                                              0x1
+#define CP_ME_COHER_CNTL__CB0_DEST_BASE_ENA__SHIFT                                                            0x6
+#define CP_ME_COHER_CNTL__CB1_DEST_BASE_ENA__SHIFT                                                            0x7
+#define CP_ME_COHER_CNTL__CB2_DEST_BASE_ENA__SHIFT                                                            0x8
+#define CP_ME_COHER_CNTL__CB3_DEST_BASE_ENA__SHIFT                                                            0x9
+#define CP_ME_COHER_CNTL__CB4_DEST_BASE_ENA__SHIFT                                                            0xa
+#define CP_ME_COHER_CNTL__CB5_DEST_BASE_ENA__SHIFT                                                            0xb
+#define CP_ME_COHER_CNTL__CB6_DEST_BASE_ENA__SHIFT                                                            0xc
+#define CP_ME_COHER_CNTL__CB7_DEST_BASE_ENA__SHIFT                                                            0xd
+#define CP_ME_COHER_CNTL__DB_DEST_BASE_ENA__SHIFT                                                             0xe
+#define CP_ME_COHER_CNTL__DEST_BASE_2_ENA__SHIFT                                                              0x13
+#define CP_ME_COHER_CNTL__DEST_BASE_3_ENA__SHIFT                                                              0x15
+#define CP_ME_COHER_CNTL__DEST_BASE_0_ENA_MASK                                                                0x00000001L
+#define CP_ME_COHER_CNTL__DEST_BASE_1_ENA_MASK                                                                0x00000002L
+#define CP_ME_COHER_CNTL__CB0_DEST_BASE_ENA_MASK                                                              0x00000040L
+#define CP_ME_COHER_CNTL__CB1_DEST_BASE_ENA_MASK                                                              0x00000080L
+#define CP_ME_COHER_CNTL__CB2_DEST_BASE_ENA_MASK                                                              0x00000100L
+#define CP_ME_COHER_CNTL__CB3_DEST_BASE_ENA_MASK                                                              0x00000200L
+#define CP_ME_COHER_CNTL__CB4_DEST_BASE_ENA_MASK                                                              0x00000400L
+#define CP_ME_COHER_CNTL__CB5_DEST_BASE_ENA_MASK                                                              0x00000800L
+#define CP_ME_COHER_CNTL__CB6_DEST_BASE_ENA_MASK                                                              0x00001000L
+#define CP_ME_COHER_CNTL__CB7_DEST_BASE_ENA_MASK                                                              0x00002000L
+#define CP_ME_COHER_CNTL__DB_DEST_BASE_ENA_MASK                                                               0x00004000L
+#define CP_ME_COHER_CNTL__DEST_BASE_2_ENA_MASK                                                                0x00080000L
+#define CP_ME_COHER_CNTL__DEST_BASE_3_ENA_MASK                                                                0x00200000L
+//CP_ME_COHER_SIZE
+#define CP_ME_COHER_SIZE__COHER_SIZE_256B__SHIFT                                                              0x0
+#define CP_ME_COHER_SIZE__COHER_SIZE_256B_MASK                                                                0xFFFFFFFFL
+//CP_ME_COHER_SIZE_HI
+#define CP_ME_COHER_SIZE_HI__COHER_SIZE_HI_256B__SHIFT                                                        0x0
+#define CP_ME_COHER_SIZE_HI__COHER_SIZE_HI_256B_MASK                                                          0x000000FFL
+//CP_ME_COHER_BASE
+#define CP_ME_COHER_BASE__COHER_BASE_256B__SHIFT                                                              0x0
+#define CP_ME_COHER_BASE__COHER_BASE_256B_MASK                                                                0xFFFFFFFFL
+//CP_ME_COHER_BASE_HI
+#define CP_ME_COHER_BASE_HI__COHER_BASE_HI_256B__SHIFT                                                        0x0
+#define CP_ME_COHER_BASE_HI__COHER_BASE_HI_256B_MASK                                                          0x000000FFL
+//CP_ME_COHER_STATUS
+#define CP_ME_COHER_STATUS__MATCHING_GFX_CNTX__SHIFT                                                          0x0
+#define CP_ME_COHER_STATUS__STATUS__SHIFT                                                                     0x1f
+#define CP_ME_COHER_STATUS__MATCHING_GFX_CNTX_MASK                                                            0x000000FFL
+#define CP_ME_COHER_STATUS__STATUS_MASK                                                                       0x80000000L
+//RLC_GPM_PERF_COUNT_0
+#define RLC_GPM_PERF_COUNT_0__FEATURE_SEL__SHIFT                                                              0x0
+#define RLC_GPM_PERF_COUNT_0__SE_INDEX__SHIFT                                                                 0x4
+#define RLC_GPM_PERF_COUNT_0__SA_INDEX__SHIFT                                                                 0x8
+#define RLC_GPM_PERF_COUNT_0__WGP_INDEX__SHIFT                                                                0xc
+#define RLC_GPM_PERF_COUNT_0__EVENT_SEL__SHIFT                                                                0x10
+#define RLC_GPM_PERF_COUNT_0__UNUSED__SHIFT                                                                   0x12
+#define RLC_GPM_PERF_COUNT_0__ENABLE__SHIFT                                                                   0x14
+#define RLC_GPM_PERF_COUNT_0__RESERVED__SHIFT                                                                 0x15
+#define RLC_GPM_PERF_COUNT_0__FEATURE_SEL_MASK                                                                0x0000000FL
+#define RLC_GPM_PERF_COUNT_0__SE_INDEX_MASK                                                                   0x000000F0L
+#define RLC_GPM_PERF_COUNT_0__SA_INDEX_MASK                                                                   0x00000F00L
+#define RLC_GPM_PERF_COUNT_0__WGP_INDEX_MASK                                                                  0x0000F000L
+#define RLC_GPM_PERF_COUNT_0__EVENT_SEL_MASK                                                                  0x00030000L
+#define RLC_GPM_PERF_COUNT_0__UNUSED_MASK                                                                     0x000C0000L
+#define RLC_GPM_PERF_COUNT_0__ENABLE_MASK                                                                     0x00100000L
+#define RLC_GPM_PERF_COUNT_0__RESERVED_MASK                                                                   0xFFE00000L
+//RLC_GPM_PERF_COUNT_1
+#define RLC_GPM_PERF_COUNT_1__FEATURE_SEL__SHIFT                                                              0x0
+#define RLC_GPM_PERF_COUNT_1__SE_INDEX__SHIFT                                                                 0x4
+#define RLC_GPM_PERF_COUNT_1__SA_INDEX__SHIFT                                                                 0x8
+#define RLC_GPM_PERF_COUNT_1__WGP_INDEX__SHIFT                                                                0xc
+#define RLC_GPM_PERF_COUNT_1__EVENT_SEL__SHIFT                                                                0x10
+#define RLC_GPM_PERF_COUNT_1__UNUSED__SHIFT                                                                   0x12
+#define RLC_GPM_PERF_COUNT_1__ENABLE__SHIFT                                                                   0x14
+#define RLC_GPM_PERF_COUNT_1__RESERVED__SHIFT                                                                 0x15
+#define RLC_GPM_PERF_COUNT_1__FEATURE_SEL_MASK                                                                0x0000000FL
+#define RLC_GPM_PERF_COUNT_1__SE_INDEX_MASK                                                                   0x000000F0L
+#define RLC_GPM_PERF_COUNT_1__SA_INDEX_MASK                                                                   0x00000F00L
+#define RLC_GPM_PERF_COUNT_1__WGP_INDEX_MASK                                                                  0x0000F000L
+#define RLC_GPM_PERF_COUNT_1__EVENT_SEL_MASK                                                                  0x00030000L
+#define RLC_GPM_PERF_COUNT_1__UNUSED_MASK                                                                     0x000C0000L
+#define RLC_GPM_PERF_COUNT_1__ENABLE_MASK                                                                     0x00100000L
+#define RLC_GPM_PERF_COUNT_1__RESERVED_MASK                                                                   0xFFE00000L
+//GRBM_GFX_INDEX
+#define GRBM_GFX_INDEX__INSTANCE_INDEX__SHIFT                                                                 0x0
+#define GRBM_GFX_INDEX__SA_INDEX__SHIFT                                                                       0x8
+#define GRBM_GFX_INDEX__SE_INDEX__SHIFT                                                                       0x10
+#define GRBM_GFX_INDEX__SA_BROADCAST_WRITES__SHIFT                                                            0x1d
+#define GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES__SHIFT                                                      0x1e
+#define GRBM_GFX_INDEX__SE_BROADCAST_WRITES__SHIFT                                                            0x1f
+#define GRBM_GFX_INDEX__INSTANCE_INDEX_MASK                                                                   0x0000007FL
+#define GRBM_GFX_INDEX__SA_INDEX_MASK                                                                         0x00000300L
+#define GRBM_GFX_INDEX__SE_INDEX_MASK                                                                         0x000F0000L
+#define GRBM_GFX_INDEX__SA_BROADCAST_WRITES_MASK                                                              0x20000000L
+#define GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK                                                        0x40000000L
+#define GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK                                                              0x80000000L
+//GRBM_NOWHERE_2
+#define GRBM_NOWHERE_2__DATA__SHIFT                                                                           0x0
+#define GRBM_NOWHERE_2__DATA_MASK                                                                             0xFFFFFFFFL
+//VGT_PRIMITIVE_TYPE
+#define VGT_PRIMITIVE_TYPE__PRIM_TYPE__SHIFT                                                                  0x0
+#define VGT_PRIMITIVE_TYPE__NUM_INPUT_CP__SHIFT                                                               0x6
+#define VGT_PRIMITIVE_TYPE__PRIMS_PER_SUBGROUP__SHIFT                                                         0xc
+#define VGT_PRIMITIVE_TYPE__PRIM_TYPE_MASK                                                                    0x0000003FL
+#define VGT_PRIMITIVE_TYPE__NUM_INPUT_CP_MASK                                                                 0x00000FC0L
+#define VGT_PRIMITIVE_TYPE__PRIMS_PER_SUBGROUP_MASK                                                           0x001FF000L
+//VGT_INDEX_TYPE
+#define VGT_INDEX_TYPE__INDEX_TYPE__SHIFT                                                                     0x0
+#define VGT_INDEX_TYPE__DISABLE_INSTANCE_PACKING__SHIFT                                                       0xe
+#define VGT_INDEX_TYPE__INDEX_TYPE_MASK                                                                       0x00000003L
+#define VGT_INDEX_TYPE__DISABLE_INSTANCE_PACKING_MASK                                                         0x00004000L
+//GE_MIN_VTX_INDX
+#define GE_MIN_VTX_INDX__MIN_INDX__SHIFT                                                                      0x0
+#define GE_MIN_VTX_INDX__MIN_INDX_MASK                                                                        0xFFFFFFFFL
+//GE_INDX_OFFSET
+#define GE_INDX_OFFSET__INDX_OFFSET__SHIFT                                                                    0x0
+#define GE_INDX_OFFSET__INDX_OFFSET_MASK                                                                      0xFFFFFFFFL
+//GE_MULTI_PRIM_IB_RESET_EN
+#define GE_MULTI_PRIM_IB_RESET_EN__RESET_EN__SHIFT                                                            0x0
+#define GE_MULTI_PRIM_IB_RESET_EN__MATCH_ALL_BITS__SHIFT                                                      0x1
+#define GE_MULTI_PRIM_IB_RESET_EN__DISABLE_FOR_AUTO_INDEX__SHIFT                                              0x2
+#define GE_MULTI_PRIM_IB_RESET_EN__RESET_EN_MASK                                                              0x00000001L
+#define GE_MULTI_PRIM_IB_RESET_EN__MATCH_ALL_BITS_MASK                                                        0x00000002L
+#define GE_MULTI_PRIM_IB_RESET_EN__DISABLE_FOR_AUTO_INDEX_MASK                                                0x00000004L
+//VGT_NUM_INDICES
+#define VGT_NUM_INDICES__NUM_INDICES__SHIFT                                                                   0x0
+#define VGT_NUM_INDICES__NUM_INDICES_MASK                                                                     0xFFFFFFFFL
+//VGT_NUM_INSTANCES
+#define VGT_NUM_INSTANCES__NUM_INSTANCES__SHIFT                                                               0x0
+#define VGT_NUM_INSTANCES__NUM_INSTANCES_MASK                                                                 0xFFFFFFFFL
+//VGT_TF_MEMORY_BASE
+#define VGT_TF_MEMORY_BASE__BASE__SHIFT                                                                       0x0
+#define VGT_TF_MEMORY_BASE__BASE_MASK                                                                         0xFFFFFFFFL
+//GE_GS_THROTTLE
+#define GE_GS_THROTTLE__T0__SHIFT                                                                             0x0
+#define GE_GS_THROTTLE__T1__SHIFT                                                                             0x3
+#define GE_GS_THROTTLE__T2__SHIFT                                                                             0x6
+#define GE_GS_THROTTLE__STALL_CYCLES__SHIFT                                                                   0x9
+#define GE_GS_THROTTLE__FACTOR1__SHIFT                                                                        0x10
+#define GE_GS_THROTTLE__FACTOR2__SHIFT                                                                        0x13
+#define GE_GS_THROTTLE__ENABLE_THROTTLE__SHIFT                                                                0x16
+#define GE_GS_THROTTLE__NUM_INIT_GRPS__SHIFT                                                                  0x17
+#define GE_GS_THROTTLE__T0_MASK                                                                               0x00000007L
+#define GE_GS_THROTTLE__T1_MASK                                                                               0x00000038L
+#define GE_GS_THROTTLE__T2_MASK                                                                               0x000001C0L
+#define GE_GS_THROTTLE__STALL_CYCLES_MASK                                                                     0x0000FE00L
+#define GE_GS_THROTTLE__FACTOR1_MASK                                                                          0x00070000L
+#define GE_GS_THROTTLE__FACTOR2_MASK                                                                          0x00380000L
+#define GE_GS_THROTTLE__ENABLE_THROTTLE_MASK                                                                  0x00400000L
+#define GE_GS_THROTTLE__NUM_INIT_GRPS_MASK                                                                    0x7F800000L
+//GE_MAX_VTX_INDX
+#define GE_MAX_VTX_INDX__MAX_INDX__SHIFT                                                                      0x0
+#define GE_MAX_VTX_INDX__MAX_INDX_MASK                                                                        0xFFFFFFFFL
+//VGT_INSTANCE_BASE_ID
+#define VGT_INSTANCE_BASE_ID__INSTANCE_BASE_ID__SHIFT                                                         0x0
+#define VGT_INSTANCE_BASE_ID__INSTANCE_BASE_ID_MASK                                                           0xFFFFFFFFL
+//GE_CNTL
+#define GE_CNTL__PRIMS_PER_SUBGRP__SHIFT                                                                      0x0
+#define GE_CNTL__VERTS_PER_SUBGRP__SHIFT                                                                      0x9
+#define GE_CNTL__BREAK_SUBGRP_AT_EOI__SHIFT                                                                   0x12
+#define GE_CNTL__PACKET_TO_ONE_PA__SHIFT                                                                      0x13
+#define GE_CNTL__BREAK_PRIMGRP_AT_EOI__SHIFT                                                                  0x14
+#define GE_CNTL__PRIM_GRP_SIZE__SHIFT                                                                         0x15
+#define GE_CNTL__GCR_DISABLE__SHIFT                                                                           0x1e
+#define GE_CNTL__DIS_PG_SIZE_ADJUST_FOR_STRIP__SHIFT                                                          0x1f
+#define GE_CNTL__PRIMS_PER_SUBGRP_MASK                                                                        0x000001FFL
+#define GE_CNTL__VERTS_PER_SUBGRP_MASK                                                                        0x0003FE00L
+#define GE_CNTL__BREAK_SUBGRP_AT_EOI_MASK                                                                     0x00040000L
+#define GE_CNTL__PACKET_TO_ONE_PA_MASK                                                                        0x00080000L
+#define GE_CNTL__BREAK_PRIMGRP_AT_EOI_MASK                                                                    0x00100000L
+#define GE_CNTL__PRIM_GRP_SIZE_MASK                                                                           0x3FE00000L
+#define GE_CNTL__GCR_DISABLE_MASK                                                                             0x40000000L
+#define GE_CNTL__DIS_PG_SIZE_ADJUST_FOR_STRIP_MASK                                                            0x80000000L
+//GE_USER_VGPR1
+#define GE_USER_VGPR1__DATA__SHIFT                                                                            0x0
+#define GE_USER_VGPR1__DATA_MASK                                                                              0xFFFFFFFFL
+//GE_USER_VGPR2
+#define GE_USER_VGPR2__DATA__SHIFT                                                                            0x0
+#define GE_USER_VGPR2__DATA_MASK                                                                              0xFFFFFFFFL
+//GE_USER_VGPR3
+#define GE_USER_VGPR3__DATA__SHIFT                                                                            0x0
+#define GE_USER_VGPR3__DATA_MASK                                                                              0xFFFFFFFFL
+//GE_STEREO_CNTL
+#define GE_STEREO_CNTL__RT_SLICE__SHIFT                                                                       0x0
+#define GE_STEREO_CNTL__VIEWPORT__SHIFT                                                                       0x3
+#define GE_STEREO_CNTL__UNUSED__SHIFT                                                                         0x7
+#define GE_STEREO_CNTL__EN_STEREO__SHIFT                                                                      0x8
+#define GE_STEREO_CNTL__RT_SLICE_MASK                                                                         0x00000007L
+#define GE_STEREO_CNTL__VIEWPORT_MASK                                                                         0x00000078L
+#define GE_STEREO_CNTL__UNUSED_MASK                                                                           0x00000080L
+#define GE_STEREO_CNTL__EN_STEREO_MASK                                                                        0x00000100L
+//GE_USER_VGPR_EN
+#define GE_USER_VGPR_EN__EN_USER_VGPR1__SHIFT                                                                 0x0
+#define GE_USER_VGPR_EN__EN_USER_VGPR2__SHIFT                                                                 0x1
+#define GE_USER_VGPR_EN__EN_USER_VGPR3__SHIFT                                                                 0x2
+#define GE_USER_VGPR_EN__EN_USER_VGPR1_MASK                                                                   0x00000001L
+#define GE_USER_VGPR_EN__EN_USER_VGPR2_MASK                                                                   0x00000002L
+#define GE_USER_VGPR_EN__EN_USER_VGPR3_MASK                                                                   0x00000004L
+//VGT_PRIMITIVEID_EN
+#define VGT_PRIMITIVEID_EN__NGG_DISABLE_PROVOK_REUSE__SHIFT                                                   0x2
+#define VGT_PRIMITIVEID_EN__NGG_DISABLE_PROVOK_REUSE_MASK                                                     0x00000004L
+//GE_VRS_RATE
+#define GE_VRS_RATE__RATE_X__SHIFT                                                                            0x0
+#define GE_VRS_RATE__RATE_Y__SHIFT                                                                            0x4
+#define GE_VRS_RATE__RATE_X_MASK                                                                              0x00000003L
+#define GE_VRS_RATE__RATE_Y_MASK                                                                              0x00000030L
+//GE_GS_FAST_LAUNCH_WG_DIM
+#define GE_GS_FAST_LAUNCH_WG_DIM__GS_FL_DIM_X__SHIFT                                                          0x0
+#define GE_GS_FAST_LAUNCH_WG_DIM__GS_FL_DIM_Y__SHIFT                                                          0x10
+#define GE_GS_FAST_LAUNCH_WG_DIM__GS_FL_DIM_X_MASK                                                            0x0000FFFFL
+#define GE_GS_FAST_LAUNCH_WG_DIM__GS_FL_DIM_Y_MASK                                                            0xFFFF0000L
+//GE_GS_FAST_LAUNCH_WG_DIM_1
+#define GE_GS_FAST_LAUNCH_WG_DIM_1__GS_FL_DIM_Z__SHIFT                                                        0x0
+#define GE_GS_FAST_LAUNCH_WG_DIM_1__GS_FL_DIM_Z_MASK                                                          0x0000FFFFL
+//VGT_GS_OUT_PRIM_TYPE
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE__SHIFT                                                             0x0
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_MASK                                                               0x0000003FL
+//VGT_TF_MEMORY_BASE_HI
+#define VGT_TF_MEMORY_BASE_HI__BASE_HI__SHIFT                                                                 0x0
+#define VGT_TF_MEMORY_BASE_HI__BASE_HI_MASK                                                                   0x000000FFL
+//GE_GS_ORDERED_ID_BASE
+#define GE_GS_ORDERED_ID_BASE__BASE__SHIFT                                                                    0x0
+#define GE_GS_ORDERED_ID_BASE__BASE_MASK                                                                      0x00000FFFL
+//VGT_PRIMITIVEID_RESET
+#define VGT_PRIMITIVEID_RESET__VALUE__SHIFT                                                                   0x0
+#define VGT_PRIMITIVEID_RESET__VALUE_MASK                                                                     0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_cprs64dec
+//CP_MES_PRGRM_CNTR_START
+#define CP_MES_PRGRM_CNTR_START__IP_START__SHIFT                                                              0x0
+#define CP_MES_PRGRM_CNTR_START__IP_START_MASK                                                                0xFFFFFFFFL
+//CP_MES_INTR_ROUTINE_START
+#define CP_MES_INTR_ROUTINE_START__IR_START__SHIFT                                                            0x0
+#define CP_MES_INTR_ROUTINE_START__IR_START_MASK                                                              0xFFFFFFFFL
+//CP_MES_MTVEC_LO
+#define CP_MES_MTVEC_LO__ADDR_LO__SHIFT                                                                       0x0
+#define CP_MES_MTVEC_LO__ADDR_LO_MASK                                                                         0xFFFFFFFFL
+//CP_MES_INTR_ROUTINE_START_HI
+#define CP_MES_INTR_ROUTINE_START_HI__IR_START__SHIFT                                                         0x0
+#define CP_MES_INTR_ROUTINE_START_HI__IR_START_MASK                                                           0xFFFFFFFFL
+//CP_MES_MTVEC_HI
+#define CP_MES_MTVEC_HI__ADDR_LO__SHIFT                                                                       0x0
+#define CP_MES_MTVEC_HI__ADDR_LO_MASK                                                                         0xFFFFFFFFL
+//CP_MES_CNTL
+#define CP_MES_CNTL__MES_INVALIDATE_ICACHE__SHIFT                                                             0x4
+#define CP_MES_CNTL__MES_PIPE0_RESET__SHIFT                                                                   0x10
+#define CP_MES_CNTL__MES_PIPE1_RESET__SHIFT                                                                   0x11
+#define CP_MES_CNTL__MES_PIPE2_RESET__SHIFT                                                                   0x12
+#define CP_MES_CNTL__MES_PIPE3_RESET__SHIFT                                                                   0x13
+#define CP_MES_CNTL__MES_PIPE0_ACTIVE__SHIFT                                                                  0x1a
+#define CP_MES_CNTL__MES_PIPE1_ACTIVE__SHIFT                                                                  0x1b
+#define CP_MES_CNTL__MES_PIPE2_ACTIVE__SHIFT                                                                  0x1c
+#define CP_MES_CNTL__MES_PIPE3_ACTIVE__SHIFT                                                                  0x1d
+#define CP_MES_CNTL__MES_HALT__SHIFT                                                                          0x1e
+#define CP_MES_CNTL__MES_STEP__SHIFT                                                                          0x1f
+#define CP_MES_CNTL__MES_INVALIDATE_ICACHE_MASK                                                               0x00000010L
+#define CP_MES_CNTL__MES_PIPE0_RESET_MASK                                                                     0x00010000L
+#define CP_MES_CNTL__MES_PIPE1_RESET_MASK                                                                     0x00020000L
+#define CP_MES_CNTL__MES_PIPE2_RESET_MASK                                                                     0x00040000L
+#define CP_MES_CNTL__MES_PIPE3_RESET_MASK                                                                     0x00080000L
+#define CP_MES_CNTL__MES_PIPE0_ACTIVE_MASK                                                                    0x04000000L
+#define CP_MES_CNTL__MES_PIPE1_ACTIVE_MASK                                                                    0x08000000L
+#define CP_MES_CNTL__MES_PIPE2_ACTIVE_MASK                                                                    0x10000000L
+#define CP_MES_CNTL__MES_PIPE3_ACTIVE_MASK                                                                    0x20000000L
+#define CP_MES_CNTL__MES_HALT_MASK                                                                            0x40000000L
+#define CP_MES_CNTL__MES_STEP_MASK                                                                            0x80000000L
+//CP_MES_PIPE_PRIORITY_CNTS
+#define CP_MES_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                       0x0
+#define CP_MES_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                      0x8
+#define CP_MES_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                      0x10
+#define CP_MES_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                       0x18
+#define CP_MES_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                         0x000000FFL
+#define CP_MES_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                        0x0000FF00L
+#define CP_MES_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                        0x00FF0000L
+#define CP_MES_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                         0xFF000000L
+//CP_MES_PIPE0_PRIORITY
+#define CP_MES_PIPE0_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_MES_PIPE0_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_MES_PIPE1_PRIORITY
+#define CP_MES_PIPE1_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_MES_PIPE1_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_MES_PIPE2_PRIORITY
+#define CP_MES_PIPE2_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_MES_PIPE2_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_MES_PIPE3_PRIORITY
+#define CP_MES_PIPE3_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_MES_PIPE3_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_MES_HEADER_DUMP
+#define CP_MES_HEADER_DUMP__HEADER_DUMP__SHIFT                                                                0x0
+#define CP_MES_HEADER_DUMP__HEADER_DUMP_MASK                                                                  0xFFFFFFFFL
+//CP_MES_MIE_LO
+#define CP_MES_MIE_LO__MES_INT__SHIFT                                                                         0x0
+#define CP_MES_MIE_LO__MES_INT_MASK                                                                           0xFFFFFFFFL
+//CP_MES_MIE_HI
+#define CP_MES_MIE_HI__MES_INT__SHIFT                                                                         0x0
+#define CP_MES_MIE_HI__MES_INT_MASK                                                                           0xFFFFFFFFL
+//CP_MES_INTERRUPT
+#define CP_MES_INTERRUPT__MES_INT__SHIFT                                                                      0x0
+#define CP_MES_INTERRUPT__MES_INT_MASK                                                                        0xFFFFFFFFL
+//CP_MES_SCRATCH_INDEX
+#define CP_MES_SCRATCH_INDEX__SCRATCH_INDEX__SHIFT                                                            0x0
+#define CP_MES_SCRATCH_INDEX__SCRATCH_INDEX_64BIT_MODE__SHIFT                                                 0x1f
+#define CP_MES_SCRATCH_INDEX__SCRATCH_INDEX_MASK                                                              0x000001FFL
+#define CP_MES_SCRATCH_INDEX__SCRATCH_INDEX_64BIT_MODE_MASK                                                   0x80000000L
+//CP_MES_SCRATCH_DATA
+#define CP_MES_SCRATCH_DATA__SCRATCH_DATA__SHIFT                                                              0x0
+#define CP_MES_SCRATCH_DATA__SCRATCH_DATA_MASK                                                                0xFFFFFFFFL
+//CP_MES_INSTR_PNTR
+#define CP_MES_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                  0x0
+#define CP_MES_INSTR_PNTR__INSTR_PNTR_MASK                                                                    0x000FFFFFL
+//CP_MES_MSCRATCH_HI
+#define CP_MES_MSCRATCH_HI__DATA__SHIFT                                                                       0x0
+#define CP_MES_MSCRATCH_HI__DATA_MASK                                                                         0xFFFFFFFFL
+//CP_MES_MSCRATCH_LO
+#define CP_MES_MSCRATCH_LO__DATA__SHIFT                                                                       0x0
+#define CP_MES_MSCRATCH_LO__DATA_MASK                                                                         0xFFFFFFFFL
+//CP_MES_MSTATUS_LO
+#define CP_MES_MSTATUS_LO__STATUS_LO__SHIFT                                                                   0x0
+#define CP_MES_MSTATUS_LO__STATUS_LO_MASK                                                                     0xFFFFFFFFL
+//CP_MES_MSTATUS_HI
+#define CP_MES_MSTATUS_HI__STATUS_HI__SHIFT                                                                   0x0
+#define CP_MES_MSTATUS_HI__STATUS_HI_MASK                                                                     0xFFFFFFFFL
+//CP_MES_MEPC_LO
+#define CP_MES_MEPC_LO__MEPC_LO__SHIFT                                                                        0x0
+#define CP_MES_MEPC_LO__MEPC_LO_MASK                                                                          0xFFFFFFFFL
+//CP_MES_MEPC_HI
+#define CP_MES_MEPC_HI__MEPC_HI__SHIFT                                                                        0x0
+#define CP_MES_MEPC_HI__MEPC_HI_MASK                                                                          0xFFFFFFFFL
+//CP_MES_MCAUSE_LO
+#define CP_MES_MCAUSE_LO__CAUSE_LO__SHIFT                                                                     0x0
+#define CP_MES_MCAUSE_LO__CAUSE_LO_MASK                                                                       0xFFFFFFFFL
+//CP_MES_MCAUSE_HI
+#define CP_MES_MCAUSE_HI__CAUSE_HI__SHIFT                                                                     0x0
+#define CP_MES_MCAUSE_HI__CAUSE_HI_MASK                                                                       0xFFFFFFFFL
+//CP_MES_MBADADDR_LO
+#define CP_MES_MBADADDR_LO__ADDR_LO__SHIFT                                                                    0x0
+#define CP_MES_MBADADDR_LO__ADDR_LO_MASK                                                                      0xFFFFFFFFL
+//CP_MES_MBADADDR_HI
+#define CP_MES_MBADADDR_HI__ADDR_HI__SHIFT                                                                    0x0
+#define CP_MES_MBADADDR_HI__ADDR_HI_MASK                                                                      0xFFFFFFFFL
+//CP_MES_MIP_LO
+#define CP_MES_MIP_LO__MIP_LO__SHIFT                                                                          0x0
+#define CP_MES_MIP_LO__MIP_LO_MASK                                                                            0xFFFFFFFFL
+//CP_MES_MIP_HI
+#define CP_MES_MIP_HI__MIP_HI__SHIFT                                                                          0x0
+#define CP_MES_MIP_HI__MIP_HI_MASK                                                                            0xFFFFFFFFL
+//CP_MES_IC_OP_CNTL
+#define CP_MES_IC_OP_CNTL__INVALIDATE_CACHE__SHIFT                                                            0x0
+#define CP_MES_IC_OP_CNTL__PRIME_ICACHE__SHIFT                                                                0x4
+#define CP_MES_IC_OP_CNTL__ICACHE_PRIMED__SHIFT                                                               0x5
+#define CP_MES_IC_OP_CNTL__INVALIDATE_CACHE_MASK                                                              0x00000001L
+#define CP_MES_IC_OP_CNTL__PRIME_ICACHE_MASK                                                                  0x00000010L
+#define CP_MES_IC_OP_CNTL__ICACHE_PRIMED_MASK                                                                 0x00000020L
+//CP_MES_MCYCLE_LO
+#define CP_MES_MCYCLE_LO__CYCLE_LO__SHIFT                                                                     0x0
+#define CP_MES_MCYCLE_LO__CYCLE_LO_MASK                                                                       0xFFFFFFFFL
+//CP_MES_MCYCLE_HI
+#define CP_MES_MCYCLE_HI__CYCLE_HI__SHIFT                                                                     0x0
+#define CP_MES_MCYCLE_HI__CYCLE_HI_MASK                                                                       0xFFFFFFFFL
+//CP_MES_MTIME_LO
+#define CP_MES_MTIME_LO__TIME_LO__SHIFT                                                                       0x0
+#define CP_MES_MTIME_LO__TIME_LO_MASK                                                                         0xFFFFFFFFL
+//CP_MES_MTIME_HI
+#define CP_MES_MTIME_HI__TIME_HI__SHIFT                                                                       0x0
+#define CP_MES_MTIME_HI__TIME_HI_MASK                                                                         0xFFFFFFFFL
+//CP_MES_MINSTRET_LO
+#define CP_MES_MINSTRET_LO__INSTRET_LO__SHIFT                                                                 0x0
+#define CP_MES_MINSTRET_LO__INSTRET_LO_MASK                                                                   0xFFFFFFFFL
+//CP_MES_MINSTRET_HI
+#define CP_MES_MINSTRET_HI__INSTRET_HI__SHIFT                                                                 0x0
+#define CP_MES_MINSTRET_HI__INSTRET_HI_MASK                                                                   0xFFFFFFFFL
+//CP_MES_MISA_LO
+#define CP_MES_MISA_LO__MISA_LO__SHIFT                                                                        0x0
+#define CP_MES_MISA_LO__MISA_LO_MASK                                                                          0xFFFFFFFFL
+//CP_MES_MISA_HI
+#define CP_MES_MISA_HI__MISA_HI__SHIFT                                                                        0x0
+#define CP_MES_MISA_HI__MISA_HI_MASK                                                                          0xFFFFFFFFL
+//CP_MES_MVENDORID_LO
+#define CP_MES_MVENDORID_LO__MVENDORID_LO__SHIFT                                                              0x0
+#define CP_MES_MVENDORID_LO__MVENDORID_LO_MASK                                                                0xFFFFFFFFL
+//CP_MES_MVENDORID_HI
+#define CP_MES_MVENDORID_HI__MVENDORID_HI__SHIFT                                                              0x0
+#define CP_MES_MVENDORID_HI__MVENDORID_HI_MASK                                                                0xFFFFFFFFL
+//CP_MES_MARCHID_LO
+#define CP_MES_MARCHID_LO__MARCHID_LO__SHIFT                                                                  0x0
+#define CP_MES_MARCHID_LO__MARCHID_LO_MASK                                                                    0xFFFFFFFFL
+//CP_MES_MARCHID_HI
+#define CP_MES_MARCHID_HI__MARCHID_HI__SHIFT                                                                  0x0
+#define CP_MES_MARCHID_HI__MARCHID_HI_MASK                                                                    0xFFFFFFFFL
+//CP_MES_MIMPID_LO
+#define CP_MES_MIMPID_LO__MIMPID_LO__SHIFT                                                                    0x0
+#define CP_MES_MIMPID_LO__MIMPID_LO_MASK                                                                      0xFFFFFFFFL
+//CP_MES_MIMPID_HI
+#define CP_MES_MIMPID_HI__MIMPID_HI__SHIFT                                                                    0x0
+#define CP_MES_MIMPID_HI__MIMPID_HI_MASK                                                                      0xFFFFFFFFL
+//CP_MES_MHARTID_LO
+#define CP_MES_MHARTID_LO__MHARTID_LO__SHIFT                                                                  0x0
+#define CP_MES_MHARTID_LO__MHARTID_LO_MASK                                                                    0xFFFFFFFFL
+//CP_MES_MHARTID_HI
+#define CP_MES_MHARTID_HI__MHARTID_HI__SHIFT                                                                  0x0
+#define CP_MES_MHARTID_HI__MHARTID_HI_MASK                                                                    0xFFFFFFFFL
+//CP_MES_DC_BASE_CNTL
+#define CP_MES_DC_BASE_CNTL__VMID__SHIFT                                                                      0x0
+#define CP_MES_DC_BASE_CNTL__CACHE_POLICY__SHIFT                                                              0x18
+#define CP_MES_DC_BASE_CNTL__SCOPE__SHIFT                                                                     0x1a
+#define CP_MES_DC_BASE_CNTL__VMID_MASK                                                                        0x0000000FL
+#define CP_MES_DC_BASE_CNTL__CACHE_POLICY_MASK                                                                0x03000000L
+#define CP_MES_DC_BASE_CNTL__SCOPE_MASK                                                                       0x0C000000L
+//CP_MES_DC_OP_CNTL
+#define CP_MES_DC_OP_CNTL__INVALIDATE_DCACHE__SHIFT                                                           0x0
+#define CP_MES_DC_OP_CNTL__INVALIDATE_DCACHE_COMPLETE__SHIFT                                                  0x1
+#define CP_MES_DC_OP_CNTL__BYPASS_ALL__SHIFT                                                                  0x2
+#define CP_MES_DC_OP_CNTL__INVALIDATE_DCACHE_MASK                                                             0x00000001L
+#define CP_MES_DC_OP_CNTL__INVALIDATE_DCACHE_COMPLETE_MASK                                                    0x00000002L
+#define CP_MES_DC_OP_CNTL__BYPASS_ALL_MASK                                                                    0x00000004L
+//CP_MES_MTIMECMP_LO
+#define CP_MES_MTIMECMP_LO__TIME_LO__SHIFT                                                                    0x0
+#define CP_MES_MTIMECMP_LO__TIME_LO_MASK                                                                      0xFFFFFFFFL
+//CP_MES_MTIMECMP_HI
+#define CP_MES_MTIMECMP_HI__TIME_HI__SHIFT                                                                    0x0
+#define CP_MES_MTIMECMP_HI__TIME_HI_MASK                                                                      0xFFFFFFFFL
+//CP_MES_PROCESS_QUANTUM_PIPE0
+#define CP_MES_PROCESS_QUANTUM_PIPE0__QUANTUM_DURATION__SHIFT                                                 0x0
+#define CP_MES_PROCESS_QUANTUM_PIPE0__TIMER_EXPIRED__SHIFT                                                    0x1c
+#define CP_MES_PROCESS_QUANTUM_PIPE0__QUANTUM_SCALE__SHIFT                                                    0x1d
+#define CP_MES_PROCESS_QUANTUM_PIPE0__QUANTUM_EN__SHIFT                                                       0x1f
+#define CP_MES_PROCESS_QUANTUM_PIPE0__QUANTUM_DURATION_MASK                                                   0x0FFFFFFFL
+#define CP_MES_PROCESS_QUANTUM_PIPE0__TIMER_EXPIRED_MASK                                                      0x10000000L
+#define CP_MES_PROCESS_QUANTUM_PIPE0__QUANTUM_SCALE_MASK                                                      0x60000000L
+#define CP_MES_PROCESS_QUANTUM_PIPE0__QUANTUM_EN_MASK                                                         0x80000000L
+//CP_MES_PROCESS_QUANTUM_PIPE1
+#define CP_MES_PROCESS_QUANTUM_PIPE1__QUANTUM_DURATION__SHIFT                                                 0x0
+#define CP_MES_PROCESS_QUANTUM_PIPE1__TIMER_EXPIRED__SHIFT                                                    0x1c
+#define CP_MES_PROCESS_QUANTUM_PIPE1__QUANTUM_SCALE__SHIFT                                                    0x1d
+#define CP_MES_PROCESS_QUANTUM_PIPE1__QUANTUM_EN__SHIFT                                                       0x1f
+#define CP_MES_PROCESS_QUANTUM_PIPE1__QUANTUM_DURATION_MASK                                                   0x0FFFFFFFL
+#define CP_MES_PROCESS_QUANTUM_PIPE1__TIMER_EXPIRED_MASK                                                      0x10000000L
+#define CP_MES_PROCESS_QUANTUM_PIPE1__QUANTUM_SCALE_MASK                                                      0x60000000L
+#define CP_MES_PROCESS_QUANTUM_PIPE1__QUANTUM_EN_MASK                                                         0x80000000L
+//CP_MES_DOORBELL_CONTROL1
+#define CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET__SHIFT                                                      0x2
+#define CP_MES_DOORBELL_CONTROL1__DOORBELL_EN__SHIFT                                                          0x1e
+#define CP_MES_DOORBELL_CONTROL1__DOORBELL_HIT__SHIFT                                                         0x1f
+#define CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET_MASK                                                        0x0FFFFFFCL
+#define CP_MES_DOORBELL_CONTROL1__DOORBELL_EN_MASK                                                            0x40000000L
+#define CP_MES_DOORBELL_CONTROL1__DOORBELL_HIT_MASK                                                           0x80000000L
+//CP_MES_DOORBELL_CONTROL2
+#define CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET__SHIFT                                                      0x2
+#define CP_MES_DOORBELL_CONTROL2__DOORBELL_EN__SHIFT                                                          0x1e
+#define CP_MES_DOORBELL_CONTROL2__DOORBELL_HIT__SHIFT                                                         0x1f
+#define CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET_MASK                                                        0x0FFFFFFCL
+#define CP_MES_DOORBELL_CONTROL2__DOORBELL_EN_MASK                                                            0x40000000L
+#define CP_MES_DOORBELL_CONTROL2__DOORBELL_HIT_MASK                                                           0x80000000L
+//CP_MES_DOORBELL_CONTROL3
+#define CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET__SHIFT                                                      0x2
+#define CP_MES_DOORBELL_CONTROL3__DOORBELL_EN__SHIFT                                                          0x1e
+#define CP_MES_DOORBELL_CONTROL3__DOORBELL_HIT__SHIFT                                                         0x1f
+#define CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET_MASK                                                        0x0FFFFFFCL
+#define CP_MES_DOORBELL_CONTROL3__DOORBELL_EN_MASK                                                            0x40000000L
+#define CP_MES_DOORBELL_CONTROL3__DOORBELL_HIT_MASK                                                           0x80000000L
+//CP_MES_DOORBELL_CONTROL4
+#define CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET__SHIFT                                                      0x2
+#define CP_MES_DOORBELL_CONTROL4__DOORBELL_EN__SHIFT                                                          0x1e
+#define CP_MES_DOORBELL_CONTROL4__DOORBELL_HIT__SHIFT                                                         0x1f
+#define CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET_MASK                                                        0x0FFFFFFCL
+#define CP_MES_DOORBELL_CONTROL4__DOORBELL_EN_MASK                                                            0x40000000L
+#define CP_MES_DOORBELL_CONTROL4__DOORBELL_HIT_MASK                                                           0x80000000L
+//CP_MES_DOORBELL_CONTROL5
+#define CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET__SHIFT                                                      0x2
+#define CP_MES_DOORBELL_CONTROL5__DOORBELL_EN__SHIFT                                                          0x1e
+#define CP_MES_DOORBELL_CONTROL5__DOORBELL_HIT__SHIFT                                                         0x1f
+#define CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET_MASK                                                        0x0FFFFFFCL
+#define CP_MES_DOORBELL_CONTROL5__DOORBELL_EN_MASK                                                            0x40000000L
+#define CP_MES_DOORBELL_CONTROL5__DOORBELL_HIT_MASK                                                           0x80000000L
+//CP_MES_DOORBELL_CONTROL6
+#define CP_MES_DOORBELL_CONTROL6__DOORBELL_OFFSET__SHIFT                                                      0x2
+#define CP_MES_DOORBELL_CONTROL6__DOORBELL_EN__SHIFT                                                          0x1e
+#define CP_MES_DOORBELL_CONTROL6__DOORBELL_HIT__SHIFT                                                         0x1f
+#define CP_MES_DOORBELL_CONTROL6__DOORBELL_OFFSET_MASK                                                        0x0FFFFFFCL
+#define CP_MES_DOORBELL_CONTROL6__DOORBELL_EN_MASK                                                            0x40000000L
+#define CP_MES_DOORBELL_CONTROL6__DOORBELL_HIT_MASK                                                           0x80000000L
+//CP_MES_GP0_LO
+#define CP_MES_GP0_LO__PG_VIRT_HALTED__SHIFT                                                                  0x0
+#define CP_MES_GP0_LO__DATA__SHIFT                                                                            0x1
+#define CP_MES_GP0_LO__PG_VIRT_HALTED_MASK                                                                    0x00000001L
+#define CP_MES_GP0_LO__DATA_MASK                                                                              0xFFFFFFFEL
+//CP_MES_GP0_HI
+#define CP_MES_GP0_HI__M_RET_ADDR__SHIFT                                                                      0x0
+#define CP_MES_GP0_HI__M_RET_ADDR_MASK                                                                        0xFFFFFFFFL
+//CP_MES_GP1_LO
+#define CP_MES_GP1_LO__RD_WR_SELECT_LO__SHIFT                                                                 0x0
+#define CP_MES_GP1_LO__RD_WR_SELECT_LO_MASK                                                                   0xFFFFFFFFL
+//CP_MES_GP1_HI
+#define CP_MES_GP1_HI__RD_WR_SELECT_HI__SHIFT                                                                 0x0
+#define CP_MES_GP1_HI__RD_WR_SELECT_HI_MASK                                                                   0xFFFFFFFFL
+//CP_MES_GP2_LO
+#define CP_MES_GP2_LO__STACK_PNTR_LO__SHIFT                                                                   0x0
+#define CP_MES_GP2_LO__STACK_PNTR_LO_MASK                                                                     0xFFFFFFFFL
+//CP_MES_GP2_HI
+#define CP_MES_GP2_HI__STACK_PNTR_HI__SHIFT                                                                   0x0
+#define CP_MES_GP2_HI__STACK_PNTR_HI_MASK                                                                     0xFFFFFFFFL
+//CP_MES_GP3_LO
+#define CP_MES_GP3_LO__DATA__SHIFT                                                                            0x0
+#define CP_MES_GP3_LO__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MES_GP3_HI
+#define CP_MES_GP3_HI__DATA__SHIFT                                                                            0x0
+#define CP_MES_GP3_HI__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MES_GP4_LO
+#define CP_MES_GP4_LO__DATA__SHIFT                                                                            0x0
+#define CP_MES_GP4_LO__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MES_GP4_HI
+#define CP_MES_GP4_HI__DATA__SHIFT                                                                            0x0
+#define CP_MES_GP4_HI__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MES_GP5_LO
+#define CP_MES_GP5_LO__PG_VIRT_HALTED__SHIFT                                                                  0x0
+#define CP_MES_GP5_LO__DATA__SHIFT                                                                            0x1
+#define CP_MES_GP5_LO__PG_VIRT_HALTED_MASK                                                                    0x00000001L
+#define CP_MES_GP5_LO__DATA_MASK                                                                              0xFFFFFFFEL
+//CP_MES_GP5_HI
+#define CP_MES_GP5_HI__M_RET_ADDR__SHIFT                                                                      0x0
+#define CP_MES_GP5_HI__M_RET_ADDR_MASK                                                                        0xFFFFFFFFL
+//CP_MES_GP6_LO
+#define CP_MES_GP6_LO__RD_WR_SELECT_LO__SHIFT                                                                 0x0
+#define CP_MES_GP6_LO__RD_WR_SELECT_LO_MASK                                                                   0xFFFFFFFFL
+//CP_MES_GP6_HI
+#define CP_MES_GP6_HI__RD_WR_SELECT_HI__SHIFT                                                                 0x0
+#define CP_MES_GP6_HI__RD_WR_SELECT_HI_MASK                                                                   0xFFFFFFFFL
+//CP_MES_GP7_LO
+#define CP_MES_GP7_LO__STACK_PNTR_LO__SHIFT                                                                   0x0
+#define CP_MES_GP7_LO__STACK_PNTR_LO_MASK                                                                     0xFFFFFFFFL
+//CP_MES_GP7_HI
+#define CP_MES_GP7_HI__STACK_PNTR_HI__SHIFT                                                                   0x0
+#define CP_MES_GP7_HI__STACK_PNTR_HI_MASK                                                                     0xFFFFFFFFL
+//CP_MES_GP8_LO
+#define CP_MES_GP8_LO__DATA__SHIFT                                                                            0x0
+#define CP_MES_GP8_LO__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MES_GP8_HI
+#define CP_MES_GP8_HI__DATA__SHIFT                                                                            0x0
+#define CP_MES_GP8_HI__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MES_GP9_LO
+#define CP_MES_GP9_LO__DATA__SHIFT                                                                            0x0
+#define CP_MES_GP9_LO__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MES_GP9_HI
+#define CP_MES_GP9_HI__DATA__SHIFT                                                                            0x0
+#define CP_MES_GP9_HI__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MES_LOCAL_BASE0_LO
+#define CP_MES_LOCAL_BASE0_LO__BASE0_LO__SHIFT                                                                0x10
+#define CP_MES_LOCAL_BASE0_LO__BASE0_LO_MASK                                                                  0xFFFF0000L
+//CP_MES_LOCAL_BASE0_HI
+#define CP_MES_LOCAL_BASE0_HI__BASE0_HI__SHIFT                                                                0x0
+#define CP_MES_LOCAL_BASE0_HI__BASE0_HI_MASK                                                                  0x01FFFFFFL
+//CP_MES_LOCAL_MASK0_LO
+#define CP_MES_LOCAL_MASK0_LO__MASK0_LO__SHIFT                                                                0x10
+#define CP_MES_LOCAL_MASK0_LO__MASK0_LO_MASK                                                                  0xFFFF0000L
+//CP_MES_LOCAL_MASK0_HI
+#define CP_MES_LOCAL_MASK0_HI__MASK0_HI__SHIFT                                                                0x0
+#define CP_MES_LOCAL_MASK0_HI__MASK0_HI_MASK                                                                  0x01FFFFFFL
+//CP_MES_LOCAL_APERTURE
+#define CP_MES_LOCAL_APERTURE__APERTURE__SHIFT                                                                0x0
+#define CP_MES_LOCAL_APERTURE__SCOPE__SHIFT                                                                   0x6
+#define CP_MES_LOCAL_APERTURE__TEMPORAL__SHIFT                                                                0x8
+#define CP_MES_LOCAL_APERTURE__APERTURE_MASK                                                                  0x00000007L
+#define CP_MES_LOCAL_APERTURE__SCOPE_MASK                                                                     0x000000C0L
+#define CP_MES_LOCAL_APERTURE__TEMPORAL_MASK                                                                  0x00000700L
+//CP_MES_LOCAL_INSTR_BASE_LO
+#define CP_MES_LOCAL_INSTR_BASE_LO__BASE_LO__SHIFT                                                            0x10
+#define CP_MES_LOCAL_INSTR_BASE_LO__BASE_LO_MASK                                                              0xFFFF0000L
+//CP_MES_LOCAL_INSTR_BASE_HI
+#define CP_MES_LOCAL_INSTR_BASE_HI__BASE_HI__SHIFT                                                            0x0
+#define CP_MES_LOCAL_INSTR_BASE_HI__BASE_HI_MASK                                                              0x01FFFFFFL
+//CP_MES_LOCAL_INSTR_MASK_LO
+#define CP_MES_LOCAL_INSTR_MASK_LO__MASK_LO__SHIFT                                                            0x10
+#define CP_MES_LOCAL_INSTR_MASK_LO__MASK_LO_MASK                                                              0xFFFF0000L
+//CP_MES_LOCAL_INSTR_MASK_HI
+#define CP_MES_LOCAL_INSTR_MASK_HI__MASK_HI__SHIFT                                                            0x0
+#define CP_MES_LOCAL_INSTR_MASK_HI__MASK_HI_MASK                                                              0x01FFFFFFL
+//CP_MES_LOCAL_INSTR_APERTURE
+#define CP_MES_LOCAL_INSTR_APERTURE__APERTURE__SHIFT                                                          0x0
+#define CP_MES_LOCAL_INSTR_APERTURE__SCOPE__SHIFT                                                             0x6
+#define CP_MES_LOCAL_INSTR_APERTURE__TEMPORAL__SHIFT                                                          0x8
+#define CP_MES_LOCAL_INSTR_APERTURE__APERTURE_MASK                                                            0x00000007L
+#define CP_MES_LOCAL_INSTR_APERTURE__SCOPE_MASK                                                               0x000000C0L
+#define CP_MES_LOCAL_INSTR_APERTURE__TEMPORAL_MASK                                                            0x00000700L
+//CP_MES_LOCAL_SCRATCH_APERTURE
+#define CP_MES_LOCAL_SCRATCH_APERTURE__APERTURE__SHIFT                                                        0x0
+#define CP_MES_LOCAL_SCRATCH_APERTURE__APERTURE_MASK                                                          0x00000007L
+//CP_MES_LOCAL_SCRATCH_BASE_LO
+#define CP_MES_LOCAL_SCRATCH_BASE_LO__BASE_LO__SHIFT                                                          0x10
+#define CP_MES_LOCAL_SCRATCH_BASE_LO__BASE_LO_MASK                                                            0xFFFF0000L
+//CP_MES_LOCAL_SCRATCH_BASE_HI
+#define CP_MES_LOCAL_SCRATCH_BASE_HI__BASE_HI__SHIFT                                                          0x0
+#define CP_MES_LOCAL_SCRATCH_BASE_HI__BASE_HI_MASK                                                            0x01FFFFFFL
+//CP_MES_PERFCOUNT_CNTL
+#define CP_MES_PERFCOUNT_CNTL__EVENT_SEL__SHIFT                                                               0x0
+#define CP_MES_PERFCOUNT_CNTL__EVENT_SEL_MASK                                                                 0x0000001FL
+//CP_MES_PENDING_INTERRUPT
+#define CP_MES_PENDING_INTERRUPT__PENDING_INTERRUPT__SHIFT                                                    0x0
+#define CP_MES_PENDING_INTERRUPT__PENDING_INTERRUPT_MASK                                                      0xFFFFFFFFL
+//CP_MES_RS64_EXCEPTION_STATUS
+#define CP_MES_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_ILLEGAL_INSTRUCTION__SHIFT                               0x0
+#define CP_MES_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_MISALIGNED_ADDR__SHIFT                                   0x1
+#define CP_MES_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_UNALIGNED_INSTRUTCION__SHIFT                             0x2
+#define CP_MES_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_PAGE_FAULT__SHIFT                                        0x3
+#define CP_MES_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_INSTRUCTION_ADDR__SHIFT                                  0x4
+#define CP_MES_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_ILLEGAL_INSTRUCTION_MASK                                 0x00000001L
+#define CP_MES_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_MISALIGNED_ADDR_MASK                                     0x00000002L
+#define CP_MES_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_UNALIGNED_INSTRUTCION_MASK                               0x00000004L
+#define CP_MES_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_PAGE_FAULT_MASK                                          0x00000008L
+#define CP_MES_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_INSTRUCTION_ADDR_MASK                                    0x07FFFFF0L
+//CP_MES_PRGRM_CNTR_START_HI
+#define CP_MES_PRGRM_CNTR_START_HI__IP_START__SHIFT                                                           0x0
+#define CP_MES_PRGRM_CNTR_START_HI__IP_START_MASK                                                             0x3FFFFFFFL
+//CP_MES_INTERRUPT_DATA_16
+#define CP_MES_INTERRUPT_DATA_16__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_16__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_17
+#define CP_MES_INTERRUPT_DATA_17__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_17__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_18
+#define CP_MES_INTERRUPT_DATA_18__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_18__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_19
+#define CP_MES_INTERRUPT_DATA_19__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_19__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_20
+#define CP_MES_INTERRUPT_DATA_20__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_20__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_21
+#define CP_MES_INTERRUPT_DATA_21__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_21__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_22
+#define CP_MES_INTERRUPT_DATA_22__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_22__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_23
+#define CP_MES_INTERRUPT_DATA_23__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_23__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_24
+#define CP_MES_INTERRUPT_DATA_24__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_24__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_25
+#define CP_MES_INTERRUPT_DATA_25__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_25__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_26
+#define CP_MES_INTERRUPT_DATA_26__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_26__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_27
+#define CP_MES_INTERRUPT_DATA_27__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_27__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_28
+#define CP_MES_INTERRUPT_DATA_28__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_28__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_29
+#define CP_MES_INTERRUPT_DATA_29__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_29__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_30
+#define CP_MES_INTERRUPT_DATA_30__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_30__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_31
+#define CP_MES_INTERRUPT_DATA_31__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_31__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_METADATA_CNTL
+#define CP_MES_METADATA_CNTL__SCOPE__SHIFT                                                                    0x6
+#define CP_MES_METADATA_CNTL__TEMPORAL__SHIFT                                                                 0x8
+#define CP_MES_METADATA_CNTL__SCOPE_MASK                                                                      0x000000C0L
+#define CP_MES_METADATA_CNTL__TEMPORAL_MASK                                                                   0x00000700L
+//CP_MEC_RS64_PRGRM_CNTR_START
+#define CP_MEC_RS64_PRGRM_CNTR_START__IP_START__SHIFT                                                         0x0
+#define CP_MEC_RS64_PRGRM_CNTR_START__IP_START_MASK                                                           0xFFFFFFFFL
+//CP_MEC_MTVEC_LO
+#define CP_MEC_MTVEC_LO__ADDR_LO__SHIFT                                                                       0x0
+#define CP_MEC_MTVEC_LO__ADDR_LO_MASK                                                                         0xFFFFFFFFL
+//CP_MEC_MTVEC_HI
+#define CP_MEC_MTVEC_HI__ADDR_LO__SHIFT                                                                       0x0
+#define CP_MEC_MTVEC_HI__ADDR_LO_MASK                                                                         0xFFFFFFFFL
+//CP_MEC_RS64_CNTL
+#define CP_MEC_RS64_CNTL__SPARE__SHIFT                                                                        0x0
+#define CP_MEC_RS64_CNTL__MEC_INVALIDATE_ICACHE__SHIFT                                                        0x4
+#define CP_MEC_RS64_CNTL__MEC_PIPE0_RESET__SHIFT                                                              0x10
+#define CP_MEC_RS64_CNTL__MEC_PIPE1_RESET__SHIFT                                                              0x11
+#define CP_MEC_RS64_CNTL__MEC_PIPE2_RESET__SHIFT                                                              0x12
+#define CP_MEC_RS64_CNTL__MEC_PIPE3_RESET__SHIFT                                                              0x13
+#define CP_MEC_RS64_CNTL__MEC_PIPE0_ACTIVE__SHIFT                                                             0x1a
+#define CP_MEC_RS64_CNTL__MEC_PIPE1_ACTIVE__SHIFT                                                             0x1b
+#define CP_MEC_RS64_CNTL__MEC_PIPE2_ACTIVE__SHIFT                                                             0x1c
+#define CP_MEC_RS64_CNTL__MEC_PIPE3_ACTIVE__SHIFT                                                             0x1d
+#define CP_MEC_RS64_CNTL__MEC_HALT__SHIFT                                                                     0x1e
+#define CP_MEC_RS64_CNTL__MEC_STEP__SHIFT                                                                     0x1f
+#define CP_MEC_RS64_CNTL__SPARE_MASK                                                                          0x0000000FL
+#define CP_MEC_RS64_CNTL__MEC_INVALIDATE_ICACHE_MASK                                                          0x00000010L
+#define CP_MEC_RS64_CNTL__MEC_PIPE0_RESET_MASK                                                                0x00010000L
+#define CP_MEC_RS64_CNTL__MEC_PIPE1_RESET_MASK                                                                0x00020000L
+#define CP_MEC_RS64_CNTL__MEC_PIPE2_RESET_MASK                                                                0x00040000L
+#define CP_MEC_RS64_CNTL__MEC_PIPE3_RESET_MASK                                                                0x00080000L
+#define CP_MEC_RS64_CNTL__MEC_PIPE0_ACTIVE_MASK                                                               0x04000000L
+#define CP_MEC_RS64_CNTL__MEC_PIPE1_ACTIVE_MASK                                                               0x08000000L
+#define CP_MEC_RS64_CNTL__MEC_PIPE2_ACTIVE_MASK                                                               0x10000000L
+#define CP_MEC_RS64_CNTL__MEC_PIPE3_ACTIVE_MASK                                                               0x20000000L
+#define CP_MEC_RS64_CNTL__MEC_HALT_MASK                                                                       0x40000000L
+#define CP_MEC_RS64_CNTL__MEC_STEP_MASK                                                                       0x80000000L
+//CP_MEC_MIE_LO
+#define CP_MEC_MIE_LO__MEC_INT__SHIFT                                                                         0x0
+#define CP_MEC_MIE_LO__MEC_INT_MASK                                                                           0xFFFFFFFFL
+//CP_MEC_MIE_HI
+#define CP_MEC_MIE_HI__MEC_INT__SHIFT                                                                         0x0
+#define CP_MEC_MIE_HI__MEC_INT_MASK                                                                           0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT
+#define CP_MEC_RS64_INTERRUPT__MEC_INT__SHIFT                                                                 0x0
+#define CP_MEC_RS64_INTERRUPT__MEC_INT_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_RS64_INSTR_PNTR
+#define CP_MEC_RS64_INSTR_PNTR__INSTR_PNTR__SHIFT                                                             0x0
+#define CP_MEC_RS64_INSTR_PNTR__INSTR_PNTR_MASK                                                               0x000FFFFFL
+//CP_MEC_MIP_LO
+#define CP_MEC_MIP_LO__MIP_LO__SHIFT                                                                          0x0
+#define CP_MEC_MIP_LO__MIP_LO_MASK                                                                            0xFFFFFFFFL
+//CP_MEC_MIP_HI
+#define CP_MEC_MIP_HI__MIP_HI__SHIFT                                                                          0x0
+#define CP_MEC_MIP_HI__MIP_HI_MASK                                                                            0xFFFFFFFFL
+//CP_MEC_DC_BASE_CNTL
+#define CP_MEC_DC_BASE_CNTL__VMID__SHIFT                                                                      0x0
+#define CP_MEC_DC_BASE_CNTL__CACHE_POLICY__SHIFT                                                              0x18
+#define CP_MEC_DC_BASE_CNTL__VMID_MASK                                                                        0x0000000FL
+#define CP_MEC_DC_BASE_CNTL__CACHE_POLICY_MASK                                                                0x03000000L
+//CP_MEC_DC_OP_CNTL
+#define CP_MEC_DC_OP_CNTL__INVALIDATE_DCACHE__SHIFT                                                           0x0
+#define CP_MEC_DC_OP_CNTL__INVALIDATE_DCACHE_COMPLETE__SHIFT                                                  0x1
+#define CP_MEC_DC_OP_CNTL__BYPASS_ALL__SHIFT                                                                  0x2
+#define CP_MEC_DC_OP_CNTL__INVALIDATE_DCACHE_MASK                                                             0x00000001L
+#define CP_MEC_DC_OP_CNTL__INVALIDATE_DCACHE_COMPLETE_MASK                                                    0x00000002L
+#define CP_MEC_DC_OP_CNTL__BYPASS_ALL_MASK                                                                    0x00000004L
+//CP_MEC_MTIMECMP_LO
+#define CP_MEC_MTIMECMP_LO__TIME_LO__SHIFT                                                                    0x0
+#define CP_MEC_MTIMECMP_LO__TIME_LO_MASK                                                                      0xFFFFFFFFL
+//CP_MEC_MTIMECMP_HI
+#define CP_MEC_MTIMECMP_HI__TIME_HI__SHIFT                                                                    0x0
+#define CP_MEC_MTIMECMP_HI__TIME_HI_MASK                                                                      0xFFFFFFFFL
+//CP_MEC_GP0_LO
+#define CP_MEC_GP0_LO__PG_VIRT_HALTED__SHIFT                                                                  0x0
+#define CP_MEC_GP0_LO__DATA__SHIFT                                                                            0x1
+#define CP_MEC_GP0_LO__PG_VIRT_HALTED_MASK                                                                    0x00000001L
+#define CP_MEC_GP0_LO__DATA_MASK                                                                              0xFFFFFFFEL
+//CP_MEC_GP0_HI
+#define CP_MEC_GP0_HI__M_RET_ADDR__SHIFT                                                                      0x0
+#define CP_MEC_GP0_HI__M_RET_ADDR_MASK                                                                        0xFFFFFFFFL
+//CP_MEC_GP1_LO
+#define CP_MEC_GP1_LO__RD_WR_SELECT_LO__SHIFT                                                                 0x0
+#define CP_MEC_GP1_LO__RD_WR_SELECT_LO_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_GP1_HI
+#define CP_MEC_GP1_HI__RD_WR_SELECT_HI__SHIFT                                                                 0x0
+#define CP_MEC_GP1_HI__RD_WR_SELECT_HI_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_GP2_LO
+#define CP_MEC_GP2_LO__STACK_PNTR_LO__SHIFT                                                                   0x0
+#define CP_MEC_GP2_LO__STACK_PNTR_LO_MASK                                                                     0xFFFFFFFFL
+//CP_MEC_GP2_HI
+#define CP_MEC_GP2_HI__STACK_PNTR_HI__SHIFT                                                                   0x0
+#define CP_MEC_GP2_HI__STACK_PNTR_HI_MASK                                                                     0xFFFFFFFFL
+//CP_MEC_GP3_LO
+#define CP_MEC_GP3_LO__DATA__SHIFT                                                                            0x0
+#define CP_MEC_GP3_LO__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MEC_GP3_HI
+#define CP_MEC_GP3_HI__DATA__SHIFT                                                                            0x0
+#define CP_MEC_GP3_HI__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MEC_GP4_LO
+#define CP_MEC_GP4_LO__DATA__SHIFT                                                                            0x0
+#define CP_MEC_GP4_LO__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MEC_GP4_HI
+#define CP_MEC_GP4_HI__DATA__SHIFT                                                                            0x0
+#define CP_MEC_GP4_HI__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MEC_GP5_LO
+#define CP_MEC_GP5_LO__PG_VIRT_HALTED__SHIFT                                                                  0x0
+#define CP_MEC_GP5_LO__DATA__SHIFT                                                                            0x1
+#define CP_MEC_GP5_LO__PG_VIRT_HALTED_MASK                                                                    0x00000001L
+#define CP_MEC_GP5_LO__DATA_MASK                                                                              0xFFFFFFFEL
+//CP_MEC_GP5_HI
+#define CP_MEC_GP5_HI__M_RET_ADDR__SHIFT                                                                      0x0
+#define CP_MEC_GP5_HI__M_RET_ADDR_MASK                                                                        0xFFFFFFFFL
+//CP_MEC_GP6_LO
+#define CP_MEC_GP6_LO__RD_WR_SELECT_LO__SHIFT                                                                 0x0
+#define CP_MEC_GP6_LO__RD_WR_SELECT_LO_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_GP6_HI
+#define CP_MEC_GP6_HI__RD_WR_SELECT_HI__SHIFT                                                                 0x0
+#define CP_MEC_GP6_HI__RD_WR_SELECT_HI_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_GP7_LO
+#define CP_MEC_GP7_LO__STACK_PNTR_LO__SHIFT                                                                   0x0
+#define CP_MEC_GP7_LO__STACK_PNTR_LO_MASK                                                                     0xFFFFFFFFL
+//CP_MEC_GP7_HI
+#define CP_MEC_GP7_HI__STACK_PNTR_HI__SHIFT                                                                   0x0
+#define CP_MEC_GP7_HI__STACK_PNTR_HI_MASK                                                                     0xFFFFFFFFL
+//CP_MEC_GP8_LO
+#define CP_MEC_GP8_LO__DATA__SHIFT                                                                            0x0
+#define CP_MEC_GP8_LO__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MEC_GP8_HI
+#define CP_MEC_GP8_HI__DATA__SHIFT                                                                            0x0
+#define CP_MEC_GP8_HI__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MEC_GP9_LO
+#define CP_MEC_GP9_LO__DATA__SHIFT                                                                            0x0
+#define CP_MEC_GP9_LO__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MEC_GP9_HI
+#define CP_MEC_GP9_HI__DATA__SHIFT                                                                            0x0
+#define CP_MEC_GP9_HI__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MEC_LOCAL_BASE0_LO
+#define CP_MEC_LOCAL_BASE0_LO__BASE0_LO__SHIFT                                                                0x10
+#define CP_MEC_LOCAL_BASE0_LO__BASE0_LO_MASK                                                                  0xFFFF0000L
+//CP_MEC_LOCAL_BASE0_HI
+#define CP_MEC_LOCAL_BASE0_HI__BASE0_HI__SHIFT                                                                0x0
+#define CP_MEC_LOCAL_BASE0_HI__BASE0_HI_MASK                                                                  0x01FFFFFFL
+//CP_MEC_LOCAL_MASK0_LO
+#define CP_MEC_LOCAL_MASK0_LO__MASK0_LO__SHIFT                                                                0x10
+#define CP_MEC_LOCAL_MASK0_LO__MASK0_LO_MASK                                                                  0xFFFF0000L
+//CP_MEC_LOCAL_MASK0_HI
+#define CP_MEC_LOCAL_MASK0_HI__MASK0_HI__SHIFT                                                                0x0
+#define CP_MEC_LOCAL_MASK0_HI__MASK0_HI_MASK                                                                  0x01FFFFFFL
+//CP_MEC_LOCAL_APERTURE
+#define CP_MEC_LOCAL_APERTURE__APERTURE__SHIFT                                                                0x0
+#define CP_MEC_LOCAL_APERTURE__SCOPE__SHIFT                                                                   0x6
+#define CP_MEC_LOCAL_APERTURE__TEMPORAL__SHIFT                                                                0x8
+#define CP_MEC_LOCAL_APERTURE__APERTURE_MASK                                                                  0x00000007L
+#define CP_MEC_LOCAL_APERTURE__SCOPE_MASK                                                                     0x000000C0L
+#define CP_MEC_LOCAL_APERTURE__TEMPORAL_MASK                                                                  0x00000700L
+//CP_MEC_LOCAL_INSTR_BASE_LO
+#define CP_MEC_LOCAL_INSTR_BASE_LO__BASE_LO__SHIFT                                                            0x10
+#define CP_MEC_LOCAL_INSTR_BASE_LO__BASE_LO_MASK                                                              0xFFFF0000L
+//CP_MEC_LOCAL_INSTR_BASE_HI
+#define CP_MEC_LOCAL_INSTR_BASE_HI__BASE_HI__SHIFT                                                            0x0
+#define CP_MEC_LOCAL_INSTR_BASE_HI__BASE_HI_MASK                                                              0x01FFFFFFL
+//CP_MEC_LOCAL_INSTR_MASK_LO
+#define CP_MEC_LOCAL_INSTR_MASK_LO__MASK_LO__SHIFT                                                            0x10
+#define CP_MEC_LOCAL_INSTR_MASK_LO__MASK_LO_MASK                                                              0xFFFF0000L
+//CP_MEC_LOCAL_INSTR_MASK_HI
+#define CP_MEC_LOCAL_INSTR_MASK_HI__MASK_HI__SHIFT                                                            0x0
+#define CP_MEC_LOCAL_INSTR_MASK_HI__MASK_HI_MASK                                                              0x01FFFFFFL
+//CP_MEC_LOCAL_INSTR_APERTURE
+#define CP_MEC_LOCAL_INSTR_APERTURE__APERTURE__SHIFT                                                          0x0
+#define CP_MEC_LOCAL_INSTR_APERTURE__SCOPE__SHIFT                                                             0x6
+#define CP_MEC_LOCAL_INSTR_APERTURE__TEMPORAL__SHIFT                                                          0x8
+#define CP_MEC_LOCAL_INSTR_APERTURE__APERTURE_MASK                                                            0x00000007L
+#define CP_MEC_LOCAL_INSTR_APERTURE__SCOPE_MASK                                                               0x000000C0L
+#define CP_MEC_LOCAL_INSTR_APERTURE__TEMPORAL_MASK                                                            0x00000700L
+//CP_MEC_LOCAL_SCRATCH_APERTURE
+#define CP_MEC_LOCAL_SCRATCH_APERTURE__APERTURE__SHIFT                                                        0x0
+#define CP_MEC_LOCAL_SCRATCH_APERTURE__APERTURE_MASK                                                          0x00000007L
+//CP_MEC_LOCAL_SCRATCH_BASE_LO
+#define CP_MEC_LOCAL_SCRATCH_BASE_LO__BASE_LO__SHIFT                                                          0x10
+#define CP_MEC_LOCAL_SCRATCH_BASE_LO__BASE_LO_MASK                                                            0xFFFF0000L
+//CP_MEC_LOCAL_SCRATCH_BASE_HI
+#define CP_MEC_LOCAL_SCRATCH_BASE_HI__BASE_HI__SHIFT                                                          0x0
+#define CP_MEC_LOCAL_SCRATCH_BASE_HI__BASE_HI_MASK                                                            0x01FFFFFFL
+//CP_MEC_RS64_PERFCOUNT_CNTL
+#define CP_MEC_RS64_PERFCOUNT_CNTL__EVENT_SEL__SHIFT                                                          0x0
+#define CP_MEC_RS64_PERFCOUNT_CNTL__EVENT_SEL_MASK                                                            0x0000001FL
+//CP_MEC_RS64_PENDING_INTERRUPT
+#define CP_MEC_RS64_PENDING_INTERRUPT__PENDING_INTERRUPT__SHIFT                                               0x0
+#define CP_MEC_RS64_PENDING_INTERRUPT__PENDING_INTERRUPT_MASK                                                 0xFFFFFFFFL
+//CP_MEC_RS64_EXCEPTION_STATUS
+#define CP_MEC_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_ILLEGAL_INSTRUCTION__SHIFT                               0x0
+#define CP_MEC_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_MISALIGNED_ADDR__SHIFT                                   0x1
+#define CP_MEC_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_UNALIGNED_INSTRUTCION__SHIFT                             0x2
+#define CP_MEC_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_PAGE_FAULT__SHIFT                                        0x3
+#define CP_MEC_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_INSTRUCTION_ADDR__SHIFT                                  0x4
+#define CP_MEC_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_ILLEGAL_INSTRUCTION_MASK                                 0x00000001L
+#define CP_MEC_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_MISALIGNED_ADDR_MASK                                     0x00000002L
+#define CP_MEC_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_UNALIGNED_INSTRUTCION_MASK                               0x00000004L
+#define CP_MEC_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_PAGE_FAULT_MASK                                          0x00000008L
+#define CP_MEC_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_INSTRUCTION_ADDR_MASK                                    0x07FFFFF0L
+//CP_MEC_RS64_PRGRM_CNTR_START_HI
+#define CP_MEC_RS64_PRGRM_CNTR_START_HI__IP_START__SHIFT                                                      0x0
+#define CP_MEC_RS64_PRGRM_CNTR_START_HI__IP_START_MASK                                                        0x3FFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_16
+#define CP_MEC_RS64_INTERRUPT_DATA_16__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_16__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_17
+#define CP_MEC_RS64_INTERRUPT_DATA_17__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_17__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_18
+#define CP_MEC_RS64_INTERRUPT_DATA_18__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_18__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_19
+#define CP_MEC_RS64_INTERRUPT_DATA_19__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_19__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_20
+#define CP_MEC_RS64_INTERRUPT_DATA_20__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_20__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_21
+#define CP_MEC_RS64_INTERRUPT_DATA_21__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_21__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_22
+#define CP_MEC_RS64_INTERRUPT_DATA_22__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_22__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_23
+#define CP_MEC_RS64_INTERRUPT_DATA_23__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_23__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_24
+#define CP_MEC_RS64_INTERRUPT_DATA_24__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_24__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_25
+#define CP_MEC_RS64_INTERRUPT_DATA_25__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_25__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_26
+#define CP_MEC_RS64_INTERRUPT_DATA_26__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_26__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_27
+#define CP_MEC_RS64_INTERRUPT_DATA_27__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_27__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_28
+#define CP_MEC_RS64_INTERRUPT_DATA_28__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_28__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_29
+#define CP_MEC_RS64_INTERRUPT_DATA_29__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_29__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_30
+#define CP_MEC_RS64_INTERRUPT_DATA_30__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_30__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_31
+#define CP_MEC_RS64_INTERRUPT_DATA_31__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_31__DATA_MASK                                                              0xFFFFFFFFL
+//CP_CPC_IC_OP_CNTL
+#define CP_CPC_IC_OP_CNTL__INVALIDATE_CACHE__SHIFT                                                            0x0
+#define CP_CPC_IC_OP_CNTL__INVALIDATE_CACHE_COMPLETE__SHIFT                                                   0x1
+#define CP_CPC_IC_OP_CNTL__RESERVED__SHIFT                                                                    0x2
+#define CP_CPC_IC_OP_CNTL__PRIME_START_PC__SHIFT                                                              0x3
+#define CP_CPC_IC_OP_CNTL__PRIME_ICACHE__SHIFT                                                                0x4
+#define CP_CPC_IC_OP_CNTL__ICACHE_PRIMED__SHIFT                                                               0x5
+#define CP_CPC_IC_OP_CNTL__INVALIDATE_CACHE_MASK                                                              0x00000001L
+#define CP_CPC_IC_OP_CNTL__INVALIDATE_CACHE_COMPLETE_MASK                                                     0x00000002L
+#define CP_CPC_IC_OP_CNTL__RESERVED_MASK                                                                      0x00000004L
+#define CP_CPC_IC_OP_CNTL__PRIME_START_PC_MASK                                                                0x00000008L
+#define CP_CPC_IC_OP_CNTL__PRIME_ICACHE_MASK                                                                  0x00000010L
+#define CP_CPC_IC_OP_CNTL__ICACHE_PRIMED_MASK                                                                 0x00000020L
+//CP_GFX_RS64_INTERRUPT0
+#define CP_GFX_RS64_INTERRUPT0__ME_INT__SHIFT                                                                 0x0
+#define CP_GFX_RS64_INTERRUPT0__ME_INT_MASK                                                                   0xFFFFFFFFL
+//CP_GFX_RS64_INTR_EN0
+#define CP_GFX_RS64_INTR_EN0__ME_INT__SHIFT                                                                   0x0
+#define CP_GFX_RS64_INTR_EN0__ME_INT_MASK                                                                     0xFFFFFFFFL
+//CP_GFX_RS64_INTR_EN1
+#define CP_GFX_RS64_INTR_EN1__ME_INT__SHIFT                                                                   0x0
+#define CP_GFX_RS64_INTR_EN1__ME_INT_MASK                                                                     0xFFFFFFFFL
+//CP_GFX_RS64_DC_BASE_CNTL
+#define CP_GFX_RS64_DC_BASE_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_GFX_RS64_DC_BASE_CNTL__CACHE_POLICY__SHIFT                                                         0x18
+#define CP_GFX_RS64_DC_BASE_CNTL__SCOPE__SHIFT                                                                0x1a
+#define CP_GFX_RS64_DC_BASE_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_GFX_RS64_DC_BASE_CNTL__CACHE_POLICY_MASK                                                           0x03000000L
+#define CP_GFX_RS64_DC_BASE_CNTL__SCOPE_MASK                                                                  0x0C000000L
+//CP_GFX_RS64_DC_OP_CNTL
+#define CP_GFX_RS64_DC_OP_CNTL__INVALIDATE_DCACHE__SHIFT                                                      0x0
+#define CP_GFX_RS64_DC_OP_CNTL__INVALIDATE_DCACHE_COMPLETE__SHIFT                                             0x1
+#define CP_GFX_RS64_DC_OP_CNTL__BYPASS_ALL__SHIFT                                                             0x2
+#define CP_GFX_RS64_DC_OP_CNTL__DEPRECATED__SHIFT                                                             0x3
+#define CP_GFX_RS64_DC_OP_CNTL__INVALIDATE_DCACHE_MASK                                                        0x00000001L
+#define CP_GFX_RS64_DC_OP_CNTL__INVALIDATE_DCACHE_COMPLETE_MASK                                               0x00000002L
+#define CP_GFX_RS64_DC_OP_CNTL__BYPASS_ALL_MASK                                                               0x00000004L
+#define CP_GFX_RS64_DC_OP_CNTL__DEPRECATED_MASK                                                               0x00000008L
+//CP_GFX_RS64_LOCAL_BASE0_LO
+#define CP_GFX_RS64_LOCAL_BASE0_LO__BASE0_LO__SHIFT                                                           0x10
+#define CP_GFX_RS64_LOCAL_BASE0_LO__BASE0_LO_MASK                                                             0xFFFF0000L
+//CP_GFX_RS64_LOCAL_BASE0_HI
+#define CP_GFX_RS64_LOCAL_BASE0_HI__BASE0_HI__SHIFT                                                           0x0
+#define CP_GFX_RS64_LOCAL_BASE0_HI__BASE0_HI_MASK                                                             0x0000FFFFL
+//CP_GFX_RS64_LOCAL_MASK0_LO
+#define CP_GFX_RS64_LOCAL_MASK0_LO__MASK0_LO__SHIFT                                                           0x10
+#define CP_GFX_RS64_LOCAL_MASK0_LO__MASK0_LO_MASK                                                             0xFFFF0000L
+//CP_GFX_RS64_LOCAL_MASK0_HI
+#define CP_GFX_RS64_LOCAL_MASK0_HI__MASK0_HI__SHIFT                                                           0x0
+#define CP_GFX_RS64_LOCAL_MASK0_HI__MASK0_HI_MASK                                                             0x0000FFFFL
+//CP_GFX_RS64_LOCAL_APERTURE
+#define CP_GFX_RS64_LOCAL_APERTURE__APERTURE__SHIFT                                                           0x0
+#define CP_GFX_RS64_LOCAL_APERTURE__SCOPE__SHIFT                                                              0x6
+#define CP_GFX_RS64_LOCAL_APERTURE__TEMPORAL__SHIFT                                                           0x8
+#define CP_GFX_RS64_LOCAL_APERTURE__APERTURE_MASK                                                             0x00000007L
+#define CP_GFX_RS64_LOCAL_APERTURE__SCOPE_MASK                                                                0x000000C0L
+#define CP_GFX_RS64_LOCAL_APERTURE__TEMPORAL_MASK                                                             0x00000700L
+//CP_GFX_RS64_LOCAL_INSTR_BASE_LO
+#define CP_GFX_RS64_LOCAL_INSTR_BASE_LO__BASE_LO__SHIFT                                                       0x10
+#define CP_GFX_RS64_LOCAL_INSTR_BASE_LO__BASE_LO_MASK                                                         0xFFFF0000L
+//CP_GFX_RS64_LOCAL_INSTR_BASE_HI
+#define CP_GFX_RS64_LOCAL_INSTR_BASE_HI__BASE_HI__SHIFT                                                       0x0
+#define CP_GFX_RS64_LOCAL_INSTR_BASE_HI__BASE_HI_MASK                                                         0x0000FFFFL
+//CP_GFX_RS64_LOCAL_INSTR_MASK_LO
+#define CP_GFX_RS64_LOCAL_INSTR_MASK_LO__MASK_LO__SHIFT                                                       0x10
+#define CP_GFX_RS64_LOCAL_INSTR_MASK_LO__MASK_LO_MASK                                                         0xFFFF0000L
+//CP_GFX_RS64_LOCAL_INSTR_MASK_HI
+#define CP_GFX_RS64_LOCAL_INSTR_MASK_HI__MASK_HI__SHIFT                                                       0x0
+#define CP_GFX_RS64_LOCAL_INSTR_MASK_HI__MASK_HI_MASK                                                         0x0000FFFFL
+//CP_GFX_RS64_LOCAL_INSTR_APERTURE
+#define CP_GFX_RS64_LOCAL_INSTR_APERTURE__APERTURE__SHIFT                                                     0x0
+#define CP_GFX_RS64_LOCAL_INSTR_APERTURE__SCOPE__SHIFT                                                        0x6
+#define CP_GFX_RS64_LOCAL_INSTR_APERTURE__TEMPORAL__SHIFT                                                     0x8
+#define CP_GFX_RS64_LOCAL_INSTR_APERTURE__APERTURE_MASK                                                       0x00000007L
+#define CP_GFX_RS64_LOCAL_INSTR_APERTURE__SCOPE_MASK                                                          0x000000C0L
+#define CP_GFX_RS64_LOCAL_INSTR_APERTURE__TEMPORAL_MASK                                                       0x00000700L
+//CP_GFX_RS64_LOCAL_SCRATCH_APERTURE
+#define CP_GFX_RS64_LOCAL_SCRATCH_APERTURE__APERTURE__SHIFT                                                   0x0
+#define CP_GFX_RS64_LOCAL_SCRATCH_APERTURE__APERTURE_MASK                                                     0x00000007L
+//CP_GFX_RS64_LOCAL_SCRATCH_BASE_LO
+#define CP_GFX_RS64_LOCAL_SCRATCH_BASE_LO__BASE_LO__SHIFT                                                     0x10
+#define CP_GFX_RS64_LOCAL_SCRATCH_BASE_LO__BASE_LO_MASK                                                       0xFFFF0000L
+//CP_GFX_RS64_LOCAL_SCRATCH_BASE_HI
+#define CP_GFX_RS64_LOCAL_SCRATCH_BASE_HI__BASE_HI__SHIFT                                                     0x0
+#define CP_GFX_RS64_LOCAL_SCRATCH_BASE_HI__BASE_HI_MASK                                                       0x0000FFFFL
+//CP_PFP_RS64_EXCEPTION_STATUS
+#define CP_PFP_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_ILLEGAL_INSTRUCTION__SHIFT                               0x0
+#define CP_PFP_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_MISALIGNED_ADDR__SHIFT                                   0x1
+#define CP_PFP_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_UNALIGNED_INSTRUTCION__SHIFT                             0x2
+#define CP_PFP_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_PAGE_FAULT__SHIFT                                        0x3
+#define CP_PFP_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_INSTRUCTION_ADDR__SHIFT                                  0x4
+#define CP_PFP_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_ILLEGAL_INSTRUCTION_MASK                                 0x00000001L
+#define CP_PFP_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_MISALIGNED_ADDR_MASK                                     0x00000002L
+#define CP_PFP_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_UNALIGNED_INSTRUTCION_MASK                               0x00000004L
+#define CP_PFP_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_PAGE_FAULT_MASK                                          0x00000008L
+#define CP_PFP_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_INSTRUCTION_ADDR_MASK                                    0x07FFFFF0L
+//CP_GFX_RS64_PERFCOUNT_CNTL0
+#define CP_GFX_RS64_PERFCOUNT_CNTL0__EVENT_SEL__SHIFT                                                         0x0
+#define CP_GFX_RS64_PERFCOUNT_CNTL0__EVENT_SEL_MASK                                                           0x0000001FL
+//CP_GFX_RS64_PERFCOUNT_CNTL1
+#define CP_GFX_RS64_PERFCOUNT_CNTL1__EVENT_SEL__SHIFT                                                         0x0
+#define CP_GFX_RS64_PERFCOUNT_CNTL1__EVENT_SEL_MASK                                                           0x0000001FL
+//CP_GFX_RS64_MIP_LO0
+#define CP_GFX_RS64_MIP_LO0__MIP_LO__SHIFT                                                                    0x0
+#define CP_GFX_RS64_MIP_LO0__MIP_LO_MASK                                                                      0xFFFFFFFFL
+//CP_GFX_RS64_MIP_LO1
+#define CP_GFX_RS64_MIP_LO1__MIP_LO__SHIFT                                                                    0x0
+#define CP_GFX_RS64_MIP_LO1__MIP_LO_MASK                                                                      0xFFFFFFFFL
+//CP_GFX_RS64_MIP_HI0
+#define CP_GFX_RS64_MIP_HI0__MIP_HI__SHIFT                                                                    0x0
+#define CP_GFX_RS64_MIP_HI0__MIP_HI_MASK                                                                      0xFFFFFFFFL
+//CP_GFX_RS64_MIP_HI1
+#define CP_GFX_RS64_MIP_HI1__MIP_HI__SHIFT                                                                    0x0
+#define CP_GFX_RS64_MIP_HI1__MIP_HI_MASK                                                                      0xFFFFFFFFL
+//CP_GFX_RS64_MTIMECMP_LO0
+#define CP_GFX_RS64_MTIMECMP_LO0__TIME_LO__SHIFT                                                              0x0
+#define CP_GFX_RS64_MTIMECMP_LO0__TIME_LO_MASK                                                                0xFFFFFFFFL
+//CP_GFX_RS64_MTIMECMP_LO1
+#define CP_GFX_RS64_MTIMECMP_LO1__TIME_LO__SHIFT                                                              0x0
+#define CP_GFX_RS64_MTIMECMP_LO1__TIME_LO_MASK                                                                0xFFFFFFFFL
+//CP_GFX_RS64_MTIMECMP_HI0
+#define CP_GFX_RS64_MTIMECMP_HI0__TIME_HI__SHIFT                                                              0x0
+#define CP_GFX_RS64_MTIMECMP_HI0__TIME_HI_MASK                                                                0xFFFFFFFFL
+//CP_GFX_RS64_MTIMECMP_HI1
+#define CP_GFX_RS64_MTIMECMP_HI1__TIME_HI__SHIFT                                                              0x0
+#define CP_GFX_RS64_MTIMECMP_HI1__TIME_HI_MASK                                                                0xFFFFFFFFL
+//CP_GFX_RS64_GP0_LO0
+#define CP_GFX_RS64_GP0_LO0__PG_VIRT_HALTED__SHIFT                                                            0x0
+#define CP_GFX_RS64_GP0_LO0__DATA__SHIFT                                                                      0x1
+#define CP_GFX_RS64_GP0_LO0__PG_VIRT_HALTED_MASK                                                              0x00000001L
+#define CP_GFX_RS64_GP0_LO0__DATA_MASK                                                                        0xFFFFFFFEL
+//CP_GFX_RS64_GP0_LO1
+#define CP_GFX_RS64_GP0_LO1__PG_VIRT_HALTED__SHIFT                                                            0x0
+#define CP_GFX_RS64_GP0_LO1__DATA__SHIFT                                                                      0x1
+#define CP_GFX_RS64_GP0_LO1__PG_VIRT_HALTED_MASK                                                              0x00000001L
+#define CP_GFX_RS64_GP0_LO1__DATA_MASK                                                                        0xFFFFFFFEL
+//CP_GFX_RS64_GP0_HI0
+#define CP_GFX_RS64_GP0_HI0__M_RET_ADDR__SHIFT                                                                0x0
+#define CP_GFX_RS64_GP0_HI0__M_RET_ADDR_MASK                                                                  0xFFFFFFFFL
+//CP_GFX_RS64_GP0_HI1
+#define CP_GFX_RS64_GP0_HI1__M_RET_ADDR__SHIFT                                                                0x0
+#define CP_GFX_RS64_GP0_HI1__M_RET_ADDR_MASK                                                                  0xFFFFFFFFL
+//CP_GFX_RS64_GP1_LO0
+#define CP_GFX_RS64_GP1_LO0__RD_WR_SELECT_LO__SHIFT                                                           0x0
+#define CP_GFX_RS64_GP1_LO0__RD_WR_SELECT_LO_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_GP1_LO1
+#define CP_GFX_RS64_GP1_LO1__RD_WR_SELECT_LO__SHIFT                                                           0x0
+#define CP_GFX_RS64_GP1_LO1__RD_WR_SELECT_LO_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_GP1_HI0
+#define CP_GFX_RS64_GP1_HI0__RD_WR_SELECT_HI__SHIFT                                                           0x0
+#define CP_GFX_RS64_GP1_HI0__RD_WR_SELECT_HI_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_GP1_HI1
+#define CP_GFX_RS64_GP1_HI1__RD_WR_SELECT_HI__SHIFT                                                           0x0
+#define CP_GFX_RS64_GP1_HI1__RD_WR_SELECT_HI_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_GP2_LO0
+#define CP_GFX_RS64_GP2_LO0__STACK_PNTR_LO__SHIFT                                                             0x0
+#define CP_GFX_RS64_GP2_LO0__STACK_PNTR_LO_MASK                                                               0xFFFFFFFFL
+//CP_GFX_RS64_GP2_LO1
+#define CP_GFX_RS64_GP2_LO1__STACK_PNTR_LO__SHIFT                                                             0x0
+#define CP_GFX_RS64_GP2_LO1__STACK_PNTR_LO_MASK                                                               0xFFFFFFFFL
+//CP_GFX_RS64_GP2_HI0
+#define CP_GFX_RS64_GP2_HI0__STACK_PNTR_HI__SHIFT                                                             0x0
+#define CP_GFX_RS64_GP2_HI0__STACK_PNTR_HI_MASK                                                               0xFFFFFFFFL
+//CP_GFX_RS64_GP2_HI1
+#define CP_GFX_RS64_GP2_HI1__STACK_PNTR_HI__SHIFT                                                             0x0
+#define CP_GFX_RS64_GP2_HI1__STACK_PNTR_HI_MASK                                                               0xFFFFFFFFL
+//CP_GFX_RS64_GP3_LO0
+#define CP_GFX_RS64_GP3_LO0__DATA__SHIFT                                                                      0x0
+#define CP_GFX_RS64_GP3_LO0__DATA_MASK                                                                        0xFFFFFFFFL
+//CP_GFX_RS64_GP3_LO1
+#define CP_GFX_RS64_GP3_LO1__DATA__SHIFT                                                                      0x0
+#define CP_GFX_RS64_GP3_LO1__DATA_MASK                                                                        0xFFFFFFFFL
+//CP_GFX_RS64_GP3_HI0
+#define CP_GFX_RS64_GP3_HI0__DATA__SHIFT                                                                      0x0
+#define CP_GFX_RS64_GP3_HI0__DATA_MASK                                                                        0xFFFFFFFFL
+//CP_GFX_RS64_GP3_HI1
+#define CP_GFX_RS64_GP3_HI1__DATA__SHIFT                                                                      0x0
+#define CP_GFX_RS64_GP3_HI1__DATA_MASK                                                                        0xFFFFFFFFL
+//CP_GFX_RS64_GP4_LO0
+#define CP_GFX_RS64_GP4_LO0__DATA__SHIFT                                                                      0x0
+#define CP_GFX_RS64_GP4_LO0__DATA_MASK                                                                        0xFFFFFFFFL
+//CP_GFX_RS64_GP4_LO1
+#define CP_GFX_RS64_GP4_LO1__DATA__SHIFT                                                                      0x0
+#define CP_GFX_RS64_GP4_LO1__DATA_MASK                                                                        0xFFFFFFFFL
+//CP_GFX_RS64_GP4_HI0
+#define CP_GFX_RS64_GP4_HI0__DATA__SHIFT                                                                      0x0
+#define CP_GFX_RS64_GP4_HI0__DATA_MASK                                                                        0xFFFFFFFFL
+//CP_GFX_RS64_GP4_HI1
+#define CP_GFX_RS64_GP4_HI1__DATA__SHIFT                                                                      0x0
+#define CP_GFX_RS64_GP4_HI1__DATA_MASK                                                                        0xFFFFFFFFL
+//CP_GFX_RS64_GP5_LO0
+#define CP_GFX_RS64_GP5_LO0__PG_VIRT_HALTED__SHIFT                                                            0x0
+#define CP_GFX_RS64_GP5_LO0__DATA__SHIFT                                                                      0x1
+#define CP_GFX_RS64_GP5_LO0__PG_VIRT_HALTED_MASK                                                              0x00000001L
+#define CP_GFX_RS64_GP5_LO0__DATA_MASK                                                                        0xFFFFFFFEL
+//CP_GFX_RS64_GP5_LO1
+#define CP_GFX_RS64_GP5_LO1__PG_VIRT_HALTED__SHIFT                                                            0x0
+#define CP_GFX_RS64_GP5_LO1__DATA__SHIFT                                                                      0x1
+#define CP_GFX_RS64_GP5_LO1__PG_VIRT_HALTED_MASK                                                              0x00000001L
+#define CP_GFX_RS64_GP5_LO1__DATA_MASK                                                                        0xFFFFFFFEL
+//CP_GFX_RS64_GP5_HI0
+#define CP_GFX_RS64_GP5_HI0__M_RET_ADDR__SHIFT                                                                0x0
+#define CP_GFX_RS64_GP5_HI0__M_RET_ADDR_MASK                                                                  0xFFFFFFFFL
+//CP_GFX_RS64_GP5_HI1
+#define CP_GFX_RS64_GP5_HI1__M_RET_ADDR__SHIFT                                                                0x0
+#define CP_GFX_RS64_GP5_HI1__M_RET_ADDR_MASK                                                                  0xFFFFFFFFL
+//CP_GFX_RS64_GP6_LO
+#define CP_GFX_RS64_GP6_LO__RD_WR_SELECT_LO__SHIFT                                                            0x0
+#define CP_GFX_RS64_GP6_LO__RD_WR_SELECT_LO_MASK                                                              0xFFFFFFFFL
+//CP_GFX_RS64_GP6_HI
+#define CP_GFX_RS64_GP6_HI__RD_WR_SELECT_HI__SHIFT                                                            0x0
+#define CP_GFX_RS64_GP6_HI__RD_WR_SELECT_HI_MASK                                                              0xFFFFFFFFL
+//CP_GFX_RS64_GP7_LO
+#define CP_GFX_RS64_GP7_LO__STACK_PNTR_LO__SHIFT                                                              0x0
+#define CP_GFX_RS64_GP7_LO__STACK_PNTR_LO_MASK                                                                0xFFFFFFFFL
+//CP_GFX_RS64_GP7_HI
+#define CP_GFX_RS64_GP7_HI__STACK_PNTR_HI__SHIFT                                                              0x0
+#define CP_GFX_RS64_GP7_HI__STACK_PNTR_HI_MASK                                                                0xFFFFFFFFL
+//CP_GFX_RS64_GP8_LO
+#define CP_GFX_RS64_GP8_LO__DATA__SHIFT                                                                       0x0
+#define CP_GFX_RS64_GP8_LO__DATA_MASK                                                                         0xFFFFFFFFL
+//CP_GFX_RS64_GP8_HI
+#define CP_GFX_RS64_GP8_HI__DATA__SHIFT                                                                       0x0
+#define CP_GFX_RS64_GP8_HI__DATA_MASK                                                                         0xFFFFFFFFL
+//CP_GFX_RS64_GP9_LO
+#define CP_GFX_RS64_GP9_LO__DATA__SHIFT                                                                       0x0
+#define CP_GFX_RS64_GP9_LO__DATA_MASK                                                                         0xFFFFFFFFL
+//CP_GFX_RS64_GP9_HI
+#define CP_GFX_RS64_GP9_HI__DATA__SHIFT                                                                       0x0
+#define CP_GFX_RS64_GP9_HI__DATA_MASK                                                                         0xFFFFFFFFL
+//CP_GFX_RS64_INSTR_PNTR0
+#define CP_GFX_RS64_INSTR_PNTR0__INSTR_PNTR__SHIFT                                                            0x0
+#define CP_GFX_RS64_INSTR_PNTR0__INSTR_PNTR_MASK                                                              0x000FFFFFL
+//CP_GFX_RS64_INSTR_PNTR1
+#define CP_GFX_RS64_INSTR_PNTR1__INSTR_PNTR__SHIFT                                                            0x0
+#define CP_GFX_RS64_INSTR_PNTR1__INSTR_PNTR_MASK                                                              0x000FFFFFL
+//CP_GFX_RS64_PENDING_INTERRUPT0
+#define CP_GFX_RS64_PENDING_INTERRUPT0__PENDING_INTERRUPT__SHIFT                                              0x0
+#define CP_GFX_RS64_PENDING_INTERRUPT0__PENDING_INTERRUPT_MASK                                                0xFFFFFFFFL
+//CP_GFX_RS64_PENDING_INTERRUPT1
+#define CP_GFX_RS64_PENDING_INTERRUPT1__PENDING_INTERRUPT__SHIFT                                              0x0
+#define CP_GFX_RS64_PENDING_INTERRUPT1__PENDING_INTERRUPT_MASK                                                0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE0_BASE0
+#define CP_GFX_RS64_DC_APERTURE0_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE0_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE0_MASK0
+#define CP_GFX_RS64_DC_APERTURE0_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE0_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE0_CNTL0
+#define CP_GFX_RS64_DC_APERTURE0_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE0_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE0_CNTL0__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE0_CNTL0__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE0_CNTL0__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE0_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE0_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE0_CNTL0__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE0_CNTL0__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE0_CNTL0__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE1_BASE0
+#define CP_GFX_RS64_DC_APERTURE1_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE1_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE1_MASK0
+#define CP_GFX_RS64_DC_APERTURE1_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE1_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE1_CNTL0
+#define CP_GFX_RS64_DC_APERTURE1_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE1_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE1_CNTL0__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE1_CNTL0__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE1_CNTL0__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE1_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE1_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE1_CNTL0__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE1_CNTL0__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE1_CNTL0__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE2_BASE0
+#define CP_GFX_RS64_DC_APERTURE2_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE2_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE2_MASK0
+#define CP_GFX_RS64_DC_APERTURE2_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE2_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE2_CNTL0
+#define CP_GFX_RS64_DC_APERTURE2_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE2_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE2_CNTL0__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE2_CNTL0__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE2_CNTL0__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE2_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE2_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE2_CNTL0__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE2_CNTL0__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE2_CNTL0__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE3_BASE0
+#define CP_GFX_RS64_DC_APERTURE3_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE3_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE3_MASK0
+#define CP_GFX_RS64_DC_APERTURE3_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE3_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE3_CNTL0
+#define CP_GFX_RS64_DC_APERTURE3_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE3_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE3_CNTL0__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE3_CNTL0__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE3_CNTL0__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE3_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE3_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE3_CNTL0__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE3_CNTL0__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE3_CNTL0__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE4_BASE0
+#define CP_GFX_RS64_DC_APERTURE4_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE4_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE4_MASK0
+#define CP_GFX_RS64_DC_APERTURE4_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE4_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE4_CNTL0
+#define CP_GFX_RS64_DC_APERTURE4_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE4_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE4_CNTL0__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE4_CNTL0__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE4_CNTL0__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE4_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE4_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE4_CNTL0__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE4_CNTL0__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE4_CNTL0__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE5_BASE0
+#define CP_GFX_RS64_DC_APERTURE5_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE5_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE5_MASK0
+#define CP_GFX_RS64_DC_APERTURE5_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE5_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE5_CNTL0
+#define CP_GFX_RS64_DC_APERTURE5_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE5_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE5_CNTL0__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE5_CNTL0__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE5_CNTL0__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE5_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE5_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE5_CNTL0__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE5_CNTL0__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE5_CNTL0__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE6_BASE0
+#define CP_GFX_RS64_DC_APERTURE6_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE6_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE6_MASK0
+#define CP_GFX_RS64_DC_APERTURE6_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE6_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE6_CNTL0
+#define CP_GFX_RS64_DC_APERTURE6_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE6_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE6_CNTL0__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE6_CNTL0__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE6_CNTL0__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE6_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE6_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE6_CNTL0__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE6_CNTL0__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE6_CNTL0__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE7_BASE0
+#define CP_GFX_RS64_DC_APERTURE7_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE7_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE7_MASK0
+#define CP_GFX_RS64_DC_APERTURE7_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE7_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE7_CNTL0
+#define CP_GFX_RS64_DC_APERTURE7_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE7_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE7_CNTL0__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE7_CNTL0__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE7_CNTL0__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE7_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE7_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE7_CNTL0__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE7_CNTL0__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE7_CNTL0__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE8_BASE0
+#define CP_GFX_RS64_DC_APERTURE8_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE8_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE8_MASK0
+#define CP_GFX_RS64_DC_APERTURE8_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE8_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE8_CNTL0
+#define CP_GFX_RS64_DC_APERTURE8_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE8_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE8_CNTL0__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE8_CNTL0__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE8_CNTL0__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE8_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE8_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE8_CNTL0__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE8_CNTL0__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE8_CNTL0__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE9_BASE0
+#define CP_GFX_RS64_DC_APERTURE9_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE9_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE9_MASK0
+#define CP_GFX_RS64_DC_APERTURE9_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE9_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE9_CNTL0
+#define CP_GFX_RS64_DC_APERTURE9_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE9_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE9_CNTL0__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE9_CNTL0__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE9_CNTL0__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE9_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE9_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE9_CNTL0__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE9_CNTL0__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE9_CNTL0__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE10_BASE0
+#define CP_GFX_RS64_DC_APERTURE10_BASE0__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE10_BASE0__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE10_MASK0
+#define CP_GFX_RS64_DC_APERTURE10_MASK0__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE10_MASK0__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE10_CNTL0
+#define CP_GFX_RS64_DC_APERTURE10_CNTL0__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE10_CNTL0__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE10_CNTL0__ENABLE__SHIFT                                                        0x5
+#define CP_GFX_RS64_DC_APERTURE10_CNTL0__SCOPE__SHIFT                                                         0x6
+#define CP_GFX_RS64_DC_APERTURE10_CNTL0__TEMPORAL__SHIFT                                                      0x8
+#define CP_GFX_RS64_DC_APERTURE10_CNTL0__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE10_CNTL0__BYPASS_MODE_MASK                                                     0x00000010L
+#define CP_GFX_RS64_DC_APERTURE10_CNTL0__ENABLE_MASK                                                          0x00000020L
+#define CP_GFX_RS64_DC_APERTURE10_CNTL0__SCOPE_MASK                                                           0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE10_CNTL0__TEMPORAL_MASK                                                        0x00000700L
+//CP_GFX_RS64_DC_APERTURE11_BASE0
+#define CP_GFX_RS64_DC_APERTURE11_BASE0__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE11_BASE0__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE11_MASK0
+#define CP_GFX_RS64_DC_APERTURE11_MASK0__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE11_MASK0__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE11_CNTL0
+#define CP_GFX_RS64_DC_APERTURE11_CNTL0__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE11_CNTL0__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE11_CNTL0__ENABLE__SHIFT                                                        0x5
+#define CP_GFX_RS64_DC_APERTURE11_CNTL0__SCOPE__SHIFT                                                         0x6
+#define CP_GFX_RS64_DC_APERTURE11_CNTL0__TEMPORAL__SHIFT                                                      0x8
+#define CP_GFX_RS64_DC_APERTURE11_CNTL0__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE11_CNTL0__BYPASS_MODE_MASK                                                     0x00000010L
+#define CP_GFX_RS64_DC_APERTURE11_CNTL0__ENABLE_MASK                                                          0x00000020L
+#define CP_GFX_RS64_DC_APERTURE11_CNTL0__SCOPE_MASK                                                           0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE11_CNTL0__TEMPORAL_MASK                                                        0x00000700L
+//CP_GFX_RS64_DC_APERTURE12_BASE0
+#define CP_GFX_RS64_DC_APERTURE12_BASE0__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE12_BASE0__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE12_MASK0
+#define CP_GFX_RS64_DC_APERTURE12_MASK0__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE12_MASK0__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE12_CNTL0
+#define CP_GFX_RS64_DC_APERTURE12_CNTL0__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE12_CNTL0__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE12_CNTL0__ENABLE__SHIFT                                                        0x5
+#define CP_GFX_RS64_DC_APERTURE12_CNTL0__SCOPE__SHIFT                                                         0x6
+#define CP_GFX_RS64_DC_APERTURE12_CNTL0__TEMPORAL__SHIFT                                                      0x8
+#define CP_GFX_RS64_DC_APERTURE12_CNTL0__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE12_CNTL0__BYPASS_MODE_MASK                                                     0x00000010L
+#define CP_GFX_RS64_DC_APERTURE12_CNTL0__ENABLE_MASK                                                          0x00000020L
+#define CP_GFX_RS64_DC_APERTURE12_CNTL0__SCOPE_MASK                                                           0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE12_CNTL0__TEMPORAL_MASK                                                        0x00000700L
+//CP_GFX_RS64_DC_APERTURE13_BASE0
+#define CP_GFX_RS64_DC_APERTURE13_BASE0__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE13_BASE0__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE13_MASK0
+#define CP_GFX_RS64_DC_APERTURE13_MASK0__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE13_MASK0__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE13_CNTL0
+#define CP_GFX_RS64_DC_APERTURE13_CNTL0__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE13_CNTL0__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE13_CNTL0__ENABLE__SHIFT                                                        0x5
+#define CP_GFX_RS64_DC_APERTURE13_CNTL0__SCOPE__SHIFT                                                         0x6
+#define CP_GFX_RS64_DC_APERTURE13_CNTL0__TEMPORAL__SHIFT                                                      0x8
+#define CP_GFX_RS64_DC_APERTURE13_CNTL0__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE13_CNTL0__BYPASS_MODE_MASK                                                     0x00000010L
+#define CP_GFX_RS64_DC_APERTURE13_CNTL0__ENABLE_MASK                                                          0x00000020L
+#define CP_GFX_RS64_DC_APERTURE13_CNTL0__SCOPE_MASK                                                           0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE13_CNTL0__TEMPORAL_MASK                                                        0x00000700L
+//CP_GFX_RS64_DC_APERTURE14_BASE0
+#define CP_GFX_RS64_DC_APERTURE14_BASE0__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE14_BASE0__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE14_MASK0
+#define CP_GFX_RS64_DC_APERTURE14_MASK0__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE14_MASK0__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE14_CNTL0
+#define CP_GFX_RS64_DC_APERTURE14_CNTL0__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE14_CNTL0__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE14_CNTL0__ENABLE__SHIFT                                                        0x5
+#define CP_GFX_RS64_DC_APERTURE14_CNTL0__SCOPE__SHIFT                                                         0x6
+#define CP_GFX_RS64_DC_APERTURE14_CNTL0__TEMPORAL__SHIFT                                                      0x8
+#define CP_GFX_RS64_DC_APERTURE14_CNTL0__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE14_CNTL0__BYPASS_MODE_MASK                                                     0x00000010L
+#define CP_GFX_RS64_DC_APERTURE14_CNTL0__ENABLE_MASK                                                          0x00000020L
+#define CP_GFX_RS64_DC_APERTURE14_CNTL0__SCOPE_MASK                                                           0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE14_CNTL0__TEMPORAL_MASK                                                        0x00000700L
+//CP_GFX_RS64_DC_APERTURE15_BASE0
+#define CP_GFX_RS64_DC_APERTURE15_BASE0__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE15_BASE0__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE15_MASK0
+#define CP_GFX_RS64_DC_APERTURE15_MASK0__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE15_MASK0__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE15_CNTL0
+#define CP_GFX_RS64_DC_APERTURE15_CNTL0__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE15_CNTL0__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE15_CNTL0__ENABLE__SHIFT                                                        0x5
+#define CP_GFX_RS64_DC_APERTURE15_CNTL0__SCOPE__SHIFT                                                         0x6
+#define CP_GFX_RS64_DC_APERTURE15_CNTL0__TEMPORAL__SHIFT                                                      0x8
+#define CP_GFX_RS64_DC_APERTURE15_CNTL0__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE15_CNTL0__BYPASS_MODE_MASK                                                     0x00000010L
+#define CP_GFX_RS64_DC_APERTURE15_CNTL0__ENABLE_MASK                                                          0x00000020L
+#define CP_GFX_RS64_DC_APERTURE15_CNTL0__SCOPE_MASK                                                           0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE15_CNTL0__TEMPORAL_MASK                                                        0x00000700L
+//CP_GFX_RS64_DC_APERTURE0_BASE1
+#define CP_GFX_RS64_DC_APERTURE0_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE0_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE0_MASK1
+#define CP_GFX_RS64_DC_APERTURE0_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE0_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE0_CNTL1
+#define CP_GFX_RS64_DC_APERTURE0_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE0_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE0_CNTL1__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE0_CNTL1__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE0_CNTL1__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE0_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE0_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE0_CNTL1__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE0_CNTL1__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE0_CNTL1__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE1_BASE1
+#define CP_GFX_RS64_DC_APERTURE1_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE1_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE1_MASK1
+#define CP_GFX_RS64_DC_APERTURE1_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE1_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE1_CNTL1
+#define CP_GFX_RS64_DC_APERTURE1_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE1_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE1_CNTL1__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE1_CNTL1__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE1_CNTL1__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE1_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE1_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE1_CNTL1__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE1_CNTL1__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE1_CNTL1__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE2_BASE1
+#define CP_GFX_RS64_DC_APERTURE2_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE2_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE2_MASK1
+#define CP_GFX_RS64_DC_APERTURE2_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE2_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE2_CNTL1
+#define CP_GFX_RS64_DC_APERTURE2_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE2_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE2_CNTL1__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE2_CNTL1__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE2_CNTL1__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE2_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE2_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE2_CNTL1__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE2_CNTL1__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE2_CNTL1__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE3_BASE1
+#define CP_GFX_RS64_DC_APERTURE3_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE3_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE3_MASK1
+#define CP_GFX_RS64_DC_APERTURE3_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE3_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE3_CNTL1
+#define CP_GFX_RS64_DC_APERTURE3_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE3_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE3_CNTL1__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE3_CNTL1__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE3_CNTL1__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE3_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE3_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE3_CNTL1__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE3_CNTL1__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE3_CNTL1__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE4_BASE1
+#define CP_GFX_RS64_DC_APERTURE4_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE4_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE4_MASK1
+#define CP_GFX_RS64_DC_APERTURE4_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE4_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE4_CNTL1
+#define CP_GFX_RS64_DC_APERTURE4_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE4_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE4_CNTL1__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE4_CNTL1__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE4_CNTL1__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE4_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE4_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE4_CNTL1__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE4_CNTL1__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE4_CNTL1__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE5_BASE1
+#define CP_GFX_RS64_DC_APERTURE5_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE5_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE5_MASK1
+#define CP_GFX_RS64_DC_APERTURE5_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE5_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE5_CNTL1
+#define CP_GFX_RS64_DC_APERTURE5_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE5_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE5_CNTL1__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE5_CNTL1__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE5_CNTL1__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE5_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE5_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE5_CNTL1__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE5_CNTL1__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE5_CNTL1__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE6_BASE1
+#define CP_GFX_RS64_DC_APERTURE6_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE6_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE6_MASK1
+#define CP_GFX_RS64_DC_APERTURE6_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE6_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE6_CNTL1
+#define CP_GFX_RS64_DC_APERTURE6_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE6_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE6_CNTL1__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE6_CNTL1__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE6_CNTL1__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE6_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE6_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE6_CNTL1__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE6_CNTL1__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE6_CNTL1__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE7_BASE1
+#define CP_GFX_RS64_DC_APERTURE7_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE7_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE7_MASK1
+#define CP_GFX_RS64_DC_APERTURE7_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE7_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE7_CNTL1
+#define CP_GFX_RS64_DC_APERTURE7_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE7_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE7_CNTL1__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE7_CNTL1__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE7_CNTL1__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE7_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE7_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE7_CNTL1__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE7_CNTL1__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE7_CNTL1__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE8_BASE1
+#define CP_GFX_RS64_DC_APERTURE8_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE8_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE8_MASK1
+#define CP_GFX_RS64_DC_APERTURE8_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE8_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE8_CNTL1
+#define CP_GFX_RS64_DC_APERTURE8_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE8_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE8_CNTL1__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE8_CNTL1__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE8_CNTL1__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE8_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE8_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE8_CNTL1__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE8_CNTL1__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE8_CNTL1__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE9_BASE1
+#define CP_GFX_RS64_DC_APERTURE9_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE9_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE9_MASK1
+#define CP_GFX_RS64_DC_APERTURE9_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE9_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE9_CNTL1
+#define CP_GFX_RS64_DC_APERTURE9_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE9_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE9_CNTL1__ENABLE__SHIFT                                                         0x5
+#define CP_GFX_RS64_DC_APERTURE9_CNTL1__SCOPE__SHIFT                                                          0x6
+#define CP_GFX_RS64_DC_APERTURE9_CNTL1__TEMPORAL__SHIFT                                                       0x8
+#define CP_GFX_RS64_DC_APERTURE9_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE9_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+#define CP_GFX_RS64_DC_APERTURE9_CNTL1__ENABLE_MASK                                                           0x00000020L
+#define CP_GFX_RS64_DC_APERTURE9_CNTL1__SCOPE_MASK                                                            0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE9_CNTL1__TEMPORAL_MASK                                                         0x00000700L
+//CP_GFX_RS64_DC_APERTURE10_BASE1
+#define CP_GFX_RS64_DC_APERTURE10_BASE1__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE10_BASE1__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE10_MASK1
+#define CP_GFX_RS64_DC_APERTURE10_MASK1__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE10_MASK1__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE10_CNTL1
+#define CP_GFX_RS64_DC_APERTURE10_CNTL1__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE10_CNTL1__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE10_CNTL1__ENABLE__SHIFT                                                        0x5
+#define CP_GFX_RS64_DC_APERTURE10_CNTL1__SCOPE__SHIFT                                                         0x6
+#define CP_GFX_RS64_DC_APERTURE10_CNTL1__TEMPORAL__SHIFT                                                      0x8
+#define CP_GFX_RS64_DC_APERTURE10_CNTL1__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE10_CNTL1__BYPASS_MODE_MASK                                                     0x00000010L
+#define CP_GFX_RS64_DC_APERTURE10_CNTL1__ENABLE_MASK                                                          0x00000020L
+#define CP_GFX_RS64_DC_APERTURE10_CNTL1__SCOPE_MASK                                                           0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE10_CNTL1__TEMPORAL_MASK                                                        0x00000700L
+//CP_GFX_RS64_DC_APERTURE11_BASE1
+#define CP_GFX_RS64_DC_APERTURE11_BASE1__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE11_BASE1__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE11_MASK1
+#define CP_GFX_RS64_DC_APERTURE11_MASK1__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE11_MASK1__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE11_CNTL1
+#define CP_GFX_RS64_DC_APERTURE11_CNTL1__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE11_CNTL1__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE11_CNTL1__ENABLE__SHIFT                                                        0x5
+#define CP_GFX_RS64_DC_APERTURE11_CNTL1__SCOPE__SHIFT                                                         0x6
+#define CP_GFX_RS64_DC_APERTURE11_CNTL1__TEMPORAL__SHIFT                                                      0x8
+#define CP_GFX_RS64_DC_APERTURE11_CNTL1__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE11_CNTL1__BYPASS_MODE_MASK                                                     0x00000010L
+#define CP_GFX_RS64_DC_APERTURE11_CNTL1__ENABLE_MASK                                                          0x00000020L
+#define CP_GFX_RS64_DC_APERTURE11_CNTL1__SCOPE_MASK                                                           0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE11_CNTL1__TEMPORAL_MASK                                                        0x00000700L
+//CP_GFX_RS64_DC_APERTURE12_BASE1
+#define CP_GFX_RS64_DC_APERTURE12_BASE1__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE12_BASE1__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE12_MASK1
+#define CP_GFX_RS64_DC_APERTURE12_MASK1__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE12_MASK1__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE12_CNTL1
+#define CP_GFX_RS64_DC_APERTURE12_CNTL1__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE12_CNTL1__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE12_CNTL1__ENABLE__SHIFT                                                        0x5
+#define CP_GFX_RS64_DC_APERTURE12_CNTL1__SCOPE__SHIFT                                                         0x6
+#define CP_GFX_RS64_DC_APERTURE12_CNTL1__TEMPORAL__SHIFT                                                      0x8
+#define CP_GFX_RS64_DC_APERTURE12_CNTL1__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE12_CNTL1__BYPASS_MODE_MASK                                                     0x00000010L
+#define CP_GFX_RS64_DC_APERTURE12_CNTL1__ENABLE_MASK                                                          0x00000020L
+#define CP_GFX_RS64_DC_APERTURE12_CNTL1__SCOPE_MASK                                                           0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE12_CNTL1__TEMPORAL_MASK                                                        0x00000700L
+//CP_GFX_RS64_DC_APERTURE13_BASE1
+#define CP_GFX_RS64_DC_APERTURE13_BASE1__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE13_BASE1__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE13_MASK1
+#define CP_GFX_RS64_DC_APERTURE13_MASK1__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE13_MASK1__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE13_CNTL1
+#define CP_GFX_RS64_DC_APERTURE13_CNTL1__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE13_CNTL1__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE13_CNTL1__ENABLE__SHIFT                                                        0x5
+#define CP_GFX_RS64_DC_APERTURE13_CNTL1__SCOPE__SHIFT                                                         0x6
+#define CP_GFX_RS64_DC_APERTURE13_CNTL1__TEMPORAL__SHIFT                                                      0x8
+#define CP_GFX_RS64_DC_APERTURE13_CNTL1__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE13_CNTL1__BYPASS_MODE_MASK                                                     0x00000010L
+#define CP_GFX_RS64_DC_APERTURE13_CNTL1__ENABLE_MASK                                                          0x00000020L
+#define CP_GFX_RS64_DC_APERTURE13_CNTL1__SCOPE_MASK                                                           0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE13_CNTL1__TEMPORAL_MASK                                                        0x00000700L
+//CP_GFX_RS64_DC_APERTURE14_BASE1
+#define CP_GFX_RS64_DC_APERTURE14_BASE1__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE14_BASE1__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE14_MASK1
+#define CP_GFX_RS64_DC_APERTURE14_MASK1__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE14_MASK1__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE14_CNTL1
+#define CP_GFX_RS64_DC_APERTURE14_CNTL1__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE14_CNTL1__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE14_CNTL1__ENABLE__SHIFT                                                        0x5
+#define CP_GFX_RS64_DC_APERTURE14_CNTL1__SCOPE__SHIFT                                                         0x6
+#define CP_GFX_RS64_DC_APERTURE14_CNTL1__TEMPORAL__SHIFT                                                      0x8
+#define CP_GFX_RS64_DC_APERTURE14_CNTL1__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE14_CNTL1__BYPASS_MODE_MASK                                                     0x00000010L
+#define CP_GFX_RS64_DC_APERTURE14_CNTL1__ENABLE_MASK                                                          0x00000020L
+#define CP_GFX_RS64_DC_APERTURE14_CNTL1__SCOPE_MASK                                                           0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE14_CNTL1__TEMPORAL_MASK                                                        0x00000700L
+//CP_GFX_RS64_DC_APERTURE15_BASE1
+#define CP_GFX_RS64_DC_APERTURE15_BASE1__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE15_BASE1__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE15_MASK1
+#define CP_GFX_RS64_DC_APERTURE15_MASK1__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE15_MASK1__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE15_CNTL1
+#define CP_GFX_RS64_DC_APERTURE15_CNTL1__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE15_CNTL1__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE15_CNTL1__ENABLE__SHIFT                                                        0x5
+#define CP_GFX_RS64_DC_APERTURE15_CNTL1__SCOPE__SHIFT                                                         0x6
+#define CP_GFX_RS64_DC_APERTURE15_CNTL1__TEMPORAL__SHIFT                                                      0x8
+#define CP_GFX_RS64_DC_APERTURE15_CNTL1__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE15_CNTL1__BYPASS_MODE_MASK                                                     0x00000010L
+#define CP_GFX_RS64_DC_APERTURE15_CNTL1__ENABLE_MASK                                                          0x00000020L
+#define CP_GFX_RS64_DC_APERTURE15_CNTL1__SCOPE_MASK                                                           0x000000C0L
+#define CP_GFX_RS64_DC_APERTURE15_CNTL1__TEMPORAL_MASK                                                        0x00000700L
+//CP_ME_RS64_EXCEPTION_STATUS
+#define CP_ME_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_ILLEGAL_INSTRUCTION__SHIFT                                0x0
+#define CP_ME_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_MISALIGNED_ADDR__SHIFT                                    0x1
+#define CP_ME_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_UNALIGNED_INSTRUTCION__SHIFT                              0x2
+#define CP_ME_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_PAGE_FAULT__SHIFT                                         0x3
+#define CP_ME_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_INSTRUCTION_ADDR__SHIFT                                   0x4
+#define CP_ME_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_ILLEGAL_INSTRUCTION_MASK                                  0x00000001L
+#define CP_ME_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_MISALIGNED_ADDR_MASK                                      0x00000002L
+#define CP_ME_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_UNALIGNED_INSTRUTCION_MASK                                0x00000004L
+#define CP_ME_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_PAGE_FAULT_MASK                                           0x00000008L
+#define CP_ME_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_INSTRUCTION_ADDR_MASK                                     0x07FFFFF0L
+//CP_GFX_RS64_INTERRUPT1
+#define CP_GFX_RS64_INTERRUPT1__ME_INT__SHIFT                                                                 0x0
+#define CP_GFX_RS64_INTERRUPT1__ME_INT_MASK                                                                   0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_chdec
+//CH_ARB_CTRL
+#define CH_ARB_CTRL__NUM_MEM_PIPES__SHIFT                                                                     0x0
+#define CH_ARB_CTRL__FGCG_DISABLE__SHIFT                                                                      0x2
+#define CH_ARB_CTRL__PERF_CNTR_EN_OVERRIDE__SHIFT                                                             0x3
+#define CH_ARB_CTRL__CHICKEN_BITS__SHIFT                                                                      0x4
+#define CH_ARB_CTRL__NUM_MEM_PIPES_MASK                                                                       0x00000001L
+#define CH_ARB_CTRL__FGCG_DISABLE_MASK                                                                        0x00000004L
+#define CH_ARB_CTRL__PERF_CNTR_EN_OVERRIDE_MASK                                                               0x00000008L
+#define CH_ARB_CTRL__CHICKEN_BITS_MASK                                                                        0x00000FF0L
+//CH_DRAM_BURST_MASK
+#define CH_DRAM_BURST_MASK__DRAM_BURST_ADDR_MASK__SHIFT                                                       0x0
+#define CH_DRAM_BURST_MASK__DRAM_BURST_ADDR_MASK_MASK                                                         0x000000FFL
+//CH_ARB_STATUS
+#define CH_ARB_STATUS__REQ_ARB_BUSY__SHIFT                                                                    0x0
+#define CH_ARB_STATUS__RET_ARB_BUSY__SHIFT                                                                    0x1
+#define CH_ARB_STATUS__REQ_ARB_BUSY_MASK                                                                      0x00000001L
+#define CH_ARB_STATUS__RET_ARB_BUSY_MASK                                                                      0x00000002L
+//CH_DRAM_BURST_CTRL
+#define CH_DRAM_BURST_CTRL__MAX_DRAM_BURST__SHIFT                                                             0x0
+#define CH_DRAM_BURST_CTRL__BURST_DISABLE__SHIFT                                                              0x3
+#define CH_DRAM_BURST_CTRL__MAX_DRAM_BURST_MASK                                                               0x00000007L
+#define CH_DRAM_BURST_CTRL__BURST_DISABLE_MASK                                                                0x00000008L
+//CHA_CHC_CREDITS
+#define CHA_CHC_CREDITS__CHC_REQ_CREDITS__SHIFT                                                               0x0
+#define CHA_CHC_CREDITS__CHC_DATA_CREDITS__SHIFT                                                              0x8
+#define CHA_CHC_CREDITS__CHC_REQ_CREDITS_MASK                                                                 0x000000FFL
+#define CHA_CHC_CREDITS__CHC_DATA_CREDITS_MASK                                                                0x0000FF00L
+//CHA_CLIENT_FREE_DELAY
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_0_FREE_DELAY__SHIFT                                                0x0
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_1_FREE_DELAY__SHIFT                                                0x3
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_2_FREE_DELAY__SHIFT                                                0x6
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_3_FREE_DELAY__SHIFT                                                0x9
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_4_FREE_DELAY__SHIFT                                                0xc
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_0_FREE_DELAY_MASK                                                  0x00000007L
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_1_FREE_DELAY_MASK                                                  0x00000038L
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_2_FREE_DELAY_MASK                                                  0x000001C0L
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_3_FREE_DELAY_MASK                                                  0x00000E00L
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_4_FREE_DELAY_MASK                                                  0x00007000L
+//CHI_CHR_REP_FGCG_OVERRIDE
+#define CHI_CHR_REP_FGCG_OVERRIDE__CHA_CHIW_REP_FGCG_OVERRIDE__SHIFT                                          0x0
+#define CHI_CHR_REP_FGCG_OVERRIDE__CHA_CHIR_REP_FGCG_OVERRIDE__SHIFT                                          0x1
+#define CHI_CHR_REP_FGCG_OVERRIDE__CHA_CHR_SRC_REP_FGCG_OVERRIDE__SHIFT                                       0x2
+#define CHI_CHR_REP_FGCG_OVERRIDE__CHA_CHR_RET_REP_FGCG_OVERRIDE__SHIFT                                       0x3
+#define CHI_CHR_REP_FGCG_OVERRIDE__CHA_CHIW_REP_FGCG_OVERRIDE_MASK                                            0x00000001L
+#define CHI_CHR_REP_FGCG_OVERRIDE__CHA_CHIR_REP_FGCG_OVERRIDE_MASK                                            0x00000002L
+#define CHI_CHR_REP_FGCG_OVERRIDE__CHA_CHR_SRC_REP_FGCG_OVERRIDE_MASK                                         0x00000004L
+#define CHI_CHR_REP_FGCG_OVERRIDE__CHA_CHR_RET_REP_FGCG_OVERRIDE_MASK                                         0x00000008L
+//CHC_CTRL
+#define CHC_CTRL__BUFFER_DEPTH_MAX__SHIFT                                                                     0x0
+#define CHC_CTRL__GL2_REQ_CREDITS__SHIFT                                                                      0x4
+#define CHC_CTRL__GL2_DATA_CREDITS__SHIFT                                                                     0xb
+#define CHC_CTRL__TO_L1_REPEATER_FGCG_DISABLE__SHIFT                                                          0x12
+#define CHC_CTRL__TO_L2_REPEATER_FGCG_DISABLE__SHIFT                                                          0x13
+#define CHC_CTRL__DISABLE_CMPSWAP_DATA_REPLICATE__SHIFT                                                       0x14
+#define CHC_CTRL__BUFFER_DEPTH_MAX_MASK                                                                       0x0000000FL
+#define CHC_CTRL__GL2_REQ_CREDITS_MASK                                                                        0x000007F0L
+#define CHC_CTRL__GL2_DATA_CREDITS_MASK                                                                       0x0003F800L
+#define CHC_CTRL__TO_L1_REPEATER_FGCG_DISABLE_MASK                                                            0x00040000L
+#define CHC_CTRL__TO_L2_REPEATER_FGCG_DISABLE_MASK                                                            0x00080000L
+#define CHC_CTRL__DISABLE_CMPSWAP_DATA_REPLICATE_MASK                                                         0x00100000L
+//CHC_STATUS
+#define CHC_STATUS__INPUT_BUFFER_VC0_FIFO_FULL__SHIFT                                                         0x0
+#define CHC_STATUS__OUTPUT_FIFOS_BUSY__SHIFT                                                                  0x1
+#define CHC_STATUS__GL2_REQ_VC0_STALL__SHIFT                                                                  0x3
+#define CHC_STATUS__GL2_DATA_VC0_STALL__SHIFT                                                                 0x4
+#define CHC_STATUS__GL2_REQ_VC1_STALL__SHIFT                                                                  0x5
+#define CHC_STATUS__GL2_DATA_VC1_STALL__SHIFT                                                                 0x6
+#define CHC_STATUS__INPUT_BUFFER_VC0_BUSY__SHIFT                                                              0x7
+#define CHC_STATUS__SRC_DATA_FIFO_VC0_BUSY__SHIFT                                                             0x8
+#define CHC_STATUS__GL2_RH_BUSY__SHIFT                                                                        0x9
+#define CHC_STATUS__NUM_REQ_PENDING_FROM_L2__SHIFT                                                            0xa
+#define CHC_STATUS__VIRTUAL_FIFO_FULL_STALL__SHIFT                                                            0x15
+#define CHC_STATUS__REQUEST_TRACKER_BUFFER_STALL__SHIFT                                                       0x16
+#define CHC_STATUS__REQUEST_TRACKER_BUSY__SHIFT                                                               0x17
+#define CHC_STATUS__BUFFER_FULL__SHIFT                                                                        0x18
+#define CHC_STATUS__INPUT_BUFFER_VC0_FIFO_FULL_MASK                                                           0x00000001L
+#define CHC_STATUS__OUTPUT_FIFOS_BUSY_MASK                                                                    0x00000002L
+#define CHC_STATUS__GL2_REQ_VC0_STALL_MASK                                                                    0x00000008L
+#define CHC_STATUS__GL2_DATA_VC0_STALL_MASK                                                                   0x00000010L
+#define CHC_STATUS__GL2_REQ_VC1_STALL_MASK                                                                    0x00000020L
+#define CHC_STATUS__GL2_DATA_VC1_STALL_MASK                                                                   0x00000040L
+#define CHC_STATUS__INPUT_BUFFER_VC0_BUSY_MASK                                                                0x00000080L
+#define CHC_STATUS__SRC_DATA_FIFO_VC0_BUSY_MASK                                                               0x00000100L
+#define CHC_STATUS__GL2_RH_BUSY_MASK                                                                          0x00000200L
+#define CHC_STATUS__NUM_REQ_PENDING_FROM_L2_MASK                                                              0x001FFC00L
+#define CHC_STATUS__VIRTUAL_FIFO_FULL_STALL_MASK                                                              0x00200000L
+#define CHC_STATUS__REQUEST_TRACKER_BUFFER_STALL_MASK                                                         0x00400000L
+#define CHC_STATUS__REQUEST_TRACKER_BUSY_MASK                                                                 0x00800000L
+#define CHC_STATUS__BUFFER_FULL_MASK                                                                          0x01000000L
+//CHC_STATUS2
+#define CHC_STATUS2__DCC_OUT_ERROR_CODE__SHIFT                                                                0x0
+#define CHC_STATUS2__DCC_OUT_ERROR_CODE_MASK                                                                  0x00000FFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_glarbdec
+//GLARB_ARB_CTRL
+#define GLARB_ARB_CTRL__NUM_MEM_PIPES__SHIFT                                                                  0x0
+#define GLARB_ARB_CTRL__FGCG_DISABLE__SHIFT                                                                   0x4
+#define GLARB_ARB_CTRL__PERF_CNTR_EN_OVERRIDE__SHIFT                                                          0x5
+#define GLARB_ARB_CTRL__GCRD_TARGET_DISABLE__SHIFT                                                            0x6
+#define GLARB_ARB_CTRL__NPS_MODE__SHIFT                                                                       0x12
+#define GLARB_ARB_CTRL__CHICKEN_BITS__SHIFT                                                                   0x14
+#define GLARB_ARB_CTRL__NUM_MEM_PIPES_MASK                                                                    0x0000000FL
+#define GLARB_ARB_CTRL__FGCG_DISABLE_MASK                                                                     0x00000010L
+#define GLARB_ARB_CTRL__PERF_CNTR_EN_OVERRIDE_MASK                                                            0x00000020L
+#define GLARB_ARB_CTRL__GCRD_TARGET_DISABLE_MASK                                                              0x0003FFC0L
+#define GLARB_ARB_CTRL__NPS_MODE_MASK                                                                         0x000C0000L
+#define GLARB_ARB_CTRL__CHICKEN_BITS_MASK                                                                     0x01F00000L
+//GLARB_DRAM_BURST_MASK
+#define GLARB_DRAM_BURST_MASK__DRAM_BURST_ADDR_MASK__SHIFT                                                    0x0
+#define GLARB_DRAM_BURST_MASK__DRAM_BURST_ADDR_MASK_MASK                                                      0x000000FFL
+//GLARB_ARB_STATUS
+#define GLARB_ARB_STATUS__REQ_ARB_BUSY__SHIFT                                                                 0x0
+#define GLARB_ARB_STATUS__RET_ARB_BUSY__SHIFT                                                                 0x1
+#define GLARB_ARB_STATUS__REQ_ARB_BUSY_MASK                                                                   0x00000001L
+#define GLARB_ARB_STATUS__RET_ARB_BUSY_MASK                                                                   0x00000002L
+//GLARB_DRAM_BURST_CTRL
+#define GLARB_DRAM_BURST_CTRL__MAX_DRAM_BURST__SHIFT                                                          0x0
+#define GLARB_DRAM_BURST_CTRL__BURST_DISABLE__SHIFT                                                           0x3
+#define GLARB_DRAM_BURST_CTRL__GATHER_64B_MEMORY_BURST_DISABLE__SHIFT                                         0x4
+#define GLARB_DRAM_BURST_CTRL__GATHER_64B_BURST_DISABLE__SHIFT                                                0x5
+#define GLARB_DRAM_BURST_CTRL__GATHER_32B_MEMORY_BURST_DISABLE__SHIFT                                         0x6
+#define GLARB_DRAM_BURST_CTRL__GATHER_32B_BURST_DISABLE__SHIFT                                                0x7
+#define GLARB_DRAM_BURST_CTRL__WRITE_BURSTABLE_STALL_DISABLE__SHIFT                                           0x8
+#define GLARB_DRAM_BURST_CTRL__MAX_DRAM_BURST_MASK                                                            0x00000007L
+#define GLARB_DRAM_BURST_CTRL__BURST_DISABLE_MASK                                                             0x00000008L
+#define GLARB_DRAM_BURST_CTRL__GATHER_64B_MEMORY_BURST_DISABLE_MASK                                           0x00000010L
+#define GLARB_DRAM_BURST_CTRL__GATHER_64B_BURST_DISABLE_MASK                                                  0x00000020L
+#define GLARB_DRAM_BURST_CTRL__GATHER_32B_MEMORY_BURST_DISABLE_MASK                                           0x00000040L
+#define GLARB_DRAM_BURST_CTRL__GATHER_32B_BURST_DISABLE_MASK                                                  0x00000080L
+#define GLARB_DRAM_BURST_CTRL__WRITE_BURSTABLE_STALL_DISABLE_MASK                                             0x00000100L
+//GLARBA_GLARBC_CREDITS
+#define GLARBA_GLARBC_CREDITS__GLARBC_REQ_CREDITS__SHIFT                                                      0x0
+#define GLARBA_GLARBC_CREDITS__GLARBC_DATA_CREDITS__SHIFT                                                     0x8
+#define GLARBA_GLARBC_CREDITS__GLARBCG_REQ_CREDITS__SHIFT                                                     0x10
+#define GLARBA_GLARBC_CREDITS__GLARBC_REQ_CREDITS_MASK                                                        0x000000FFL
+#define GLARBA_GLARBC_CREDITS__GLARBC_DATA_CREDITS_MASK                                                       0x0000FF00L
+#define GLARBA_GLARBC_CREDITS__GLARBCG_REQ_CREDITS_MASK                                                       0x00FF0000L
+//GLARBA_CLIENT_FREE_DELAY
+#define GLARBA_CLIENT_FREE_DELAY__CLIENT_TYPE_0_FREE_DELAY__SHIFT                                             0x0
+#define GLARBA_CLIENT_FREE_DELAY__CLIENT_TYPE_1_FREE_DELAY__SHIFT                                             0x3
+#define GLARBA_CLIENT_FREE_DELAY__CLIENT_TYPE_2_FREE_DELAY__SHIFT                                             0x6
+#define GLARBA_CLIENT_FREE_DELAY__CLIENT_TYPE_3_FREE_DELAY__SHIFT                                             0x9
+#define GLARBA_CLIENT_FREE_DELAY__CLIENT_TYPE_4_FREE_DELAY__SHIFT                                             0xc
+#define GLARBA_CLIENT_FREE_DELAY__CLIENT_TYPE_0_FREE_DELAY_MASK                                               0x00000007L
+#define GLARBA_CLIENT_FREE_DELAY__CLIENT_TYPE_1_FREE_DELAY_MASK                                               0x00000038L
+#define GLARBA_CLIENT_FREE_DELAY__CLIENT_TYPE_2_FREE_DELAY_MASK                                               0x000001C0L
+#define GLARBA_CLIENT_FREE_DELAY__CLIENT_TYPE_3_FREE_DELAY_MASK                                               0x00000E00L
+#define GLARBA_CLIENT_FREE_DELAY__CLIENT_TYPE_4_FREE_DELAY_MASK                                               0x00007000L
+//GLARBI_GLARBR_REP_FGCG_OVERRIDE
+#define GLARBI_GLARBR_REP_FGCG_OVERRIDE__GLARBA_GLARBIW_REP_FGCG_OVERRIDE__SHIFT                              0x0
+#define GLARBI_GLARBR_REP_FGCG_OVERRIDE__GLARBA_GLARBIR_REP_FGCG_OVERRIDE__SHIFT                              0x1
+#define GLARBI_GLARBR_REP_FGCG_OVERRIDE__GLARBA_GLARBR_SRC_REP_FGCG_OVERRIDE__SHIFT                           0x2
+#define GLARBI_GLARBR_REP_FGCG_OVERRIDE__GLARBA_GLARBR_RET_REP_FGCG_OVERRIDE__SHIFT                           0x3
+#define GLARBI_GLARBR_REP_FGCG_OVERRIDE__GLARBA_GLARBIW_REP_FGCG_OVERRIDE_MASK                                0x00000001L
+#define GLARBI_GLARBR_REP_FGCG_OVERRIDE__GLARBA_GLARBIR_REP_FGCG_OVERRIDE_MASK                                0x00000002L
+#define GLARBI_GLARBR_REP_FGCG_OVERRIDE__GLARBA_GLARBR_SRC_REP_FGCG_OVERRIDE_MASK                             0x00000004L
+#define GLARBI_GLARBR_REP_FGCG_OVERRIDE__GLARBA_GLARBR_RET_REP_FGCG_OVERRIDE_MASK                             0x00000008L
+//GLARBC_CTRL
+#define GLARBC_CTRL__BUFFER_DEPTH_MAX__SHIFT                                                                  0x0
+#define GLARBC_CTRL__GL2_REQ_CREDITS__SHIFT                                                                   0x4
+#define GLARBC_CTRL__GL2_DATA_CREDITS__SHIFT                                                                  0xb
+#define GLARBC_CTRL__TO_L1_REPEATER_FGCG_DISABLE__SHIFT                                                       0x12
+#define GLARBC_CTRL__TO_L2_REPEATER_FGCG_DISABLE__SHIFT                                                       0x13
+#define GLARBC_CTRL__DISABLE_PERF_WR_DATA_ALLOC_COUNT__SHIFT                                                  0x14
+#define GLARBC_CTRL__OC_EA_REQ_D_CREDIT__SHIFT                                                                0x15
+#define GLARBC_CTRL__OC_EA_REQ_I_CREDIT__SHIFT                                                                0x1a
+#define GLARBC_CTRL__CREST_MODE_ENABLE__SHIFT                                                                 0x1f
+#define GLARBC_CTRL__BUFFER_DEPTH_MAX_MASK                                                                    0x0000000FL
+#define GLARBC_CTRL__GL2_REQ_CREDITS_MASK                                                                     0x000007F0L
+#define GLARBC_CTRL__GL2_DATA_CREDITS_MASK                                                                    0x0003F800L
+#define GLARBC_CTRL__TO_L1_REPEATER_FGCG_DISABLE_MASK                                                         0x00040000L
+#define GLARBC_CTRL__TO_L2_REPEATER_FGCG_DISABLE_MASK                                                         0x00080000L
+#define GLARBC_CTRL__DISABLE_PERF_WR_DATA_ALLOC_COUNT_MASK                                                    0x00100000L
+#define GLARBC_CTRL__OC_EA_REQ_D_CREDIT_MASK                                                                  0x03E00000L
+#define GLARBC_CTRL__OC_EA_REQ_I_CREDIT_MASK                                                                  0x7C000000L
+#define GLARBC_CTRL__CREST_MODE_ENABLE_MASK                                                                   0x80000000L
+//GLARBC_STATUS
+#define GLARBC_STATUS__INPUT_BUFFER_VC0_FIFO_FULL__SHIFT                                                      0x0
+#define GLARBC_STATUS__OUTPUT_FIFOS_BUSY__SHIFT                                                               0x1
+#define GLARBC_STATUS__SRC_DATA_FIFO_VC0_FULL__SHIFT                                                          0x2
+#define GLARBC_STATUS__GL2_REQ_VC0_STALL__SHIFT                                                               0x3
+#define GLARBC_STATUS__GL2_DATA_VC0_STALL__SHIFT                                                              0x4
+#define GLARBC_STATUS__GL2_REQ_VC1_STALL__SHIFT                                                               0x5
+#define GLARBC_STATUS__GL2_DATA_VC1_STALL__SHIFT                                                              0x6
+#define GLARBC_STATUS__INPUT_BUFFER_VC0_BUSY__SHIFT                                                           0x7
+#define GLARBC_STATUS__SRC_DATA_FIFO_VC0_BUSY__SHIFT                                                          0x8
+#define GLARBC_STATUS__GL2_RH_BUSY__SHIFT                                                                     0x9
+#define GLARBC_STATUS__NUM_REQ_PENDING_FROM_L2__SHIFT                                                         0xa
+#define GLARBC_STATUS__VIRTUAL_FIFO_FULL_STALL__SHIFT                                                         0x14
+#define GLARBC_STATUS__REQUEST_TRACKER_BUFFER_STALL__SHIFT                                                    0x15
+#define GLARBC_STATUS__REQUEST_TRACKER_BUSY__SHIFT                                                            0x16
+#define GLARBC_STATUS__BUFFER_FULL__SHIFT                                                                     0x17
+#define GLARBC_STATUS__DCC_OUT_ERROR_CODE__SHIFT                                                              0x18
+#define GLARBC_STATUS__INPUT_BUFFER_VC0_FIFO_FULL_MASK                                                        0x00000001L
+#define GLARBC_STATUS__OUTPUT_FIFOS_BUSY_MASK                                                                 0x00000002L
+#define GLARBC_STATUS__SRC_DATA_FIFO_VC0_FULL_MASK                                                            0x00000004L
+#define GLARBC_STATUS__GL2_REQ_VC0_STALL_MASK                                                                 0x00000008L
+#define GLARBC_STATUS__GL2_DATA_VC0_STALL_MASK                                                                0x00000010L
+#define GLARBC_STATUS__GL2_REQ_VC1_STALL_MASK                                                                 0x00000020L
+#define GLARBC_STATUS__GL2_DATA_VC1_STALL_MASK                                                                0x00000040L
+#define GLARBC_STATUS__INPUT_BUFFER_VC0_BUSY_MASK                                                             0x00000080L
+#define GLARBC_STATUS__SRC_DATA_FIFO_VC0_BUSY_MASK                                                            0x00000100L
+#define GLARBC_STATUS__GL2_RH_BUSY_MASK                                                                       0x00000200L
+#define GLARBC_STATUS__NUM_REQ_PENDING_FROM_L2_MASK                                                           0x000FFC00L
+#define GLARBC_STATUS__VIRTUAL_FIFO_FULL_STALL_MASK                                                           0x00100000L
+#define GLARBC_STATUS__REQUEST_TRACKER_BUFFER_STALL_MASK                                                      0x00200000L
+#define GLARBC_STATUS__REQUEST_TRACKER_BUSY_MASK                                                              0x00400000L
+#define GLARBC_STATUS__BUFFER_FULL_MASK                                                                       0x00800000L
+#define GLARBC_STATUS__DCC_OUT_ERROR_CODE_MASK                                                                0x1F000000L
+//GLARBC_CTRL2
+#define GLARBC_CTRL2__DCC_COMP_TO_CONSTANT_EN__SHIFT                                                          0x0
+#define GLARBC_CTRL2__DCC_COMP_TO_SINGLE_EN__SHIFT                                                            0x1
+#define GLARBC_CTRL2__DCC_COMP_ERROR_DETECTION_EN__SHIFT                                                      0x2
+#define GLARBC_CTRL2__DCC_CLEAR_ERROR_CODE__SHIFT                                                             0x6
+#define GLARBC_CTRL2__DCC_COMP_TRANSFER_SIZE_ENABLE__SHIFT                                                    0x7
+#define GLARBC_CTRL2__DCC_COMP_SKIP_LOW_COMP_RATIOS__SHIFT                                                    0xa
+#define GLARBC_CTRL2__DCC_COMP_TO_CONSTANT_EN_MASK                                                            0x00000001L
+#define GLARBC_CTRL2__DCC_COMP_TO_SINGLE_EN_MASK                                                              0x00000002L
+#define GLARBC_CTRL2__DCC_COMP_ERROR_DETECTION_EN_MASK                                                        0x0000003CL
+#define GLARBC_CTRL2__DCC_CLEAR_ERROR_CODE_MASK                                                               0x00000040L
+#define GLARBC_CTRL2__DCC_COMP_TRANSFER_SIZE_ENABLE_MASK                                                      0x00000380L
+#define GLARBC_CTRL2__DCC_COMP_SKIP_LOW_COMP_RATIOS_MASK                                                      0x00000400L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_perfddec
+//CPG_PERFCOUNTER1_LO
+#define CPG_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPG_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPG_PERFCOUNTER1_HI
+#define CPG_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPG_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPG_PERFCOUNTER0_LO
+#define CPG_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPG_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPG_PERFCOUNTER0_HI
+#define CPG_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPG_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPC_PERFCOUNTER1_LO
+#define CPC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPC_PERFCOUNTER1_HI
+#define CPC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPC_PERFCOUNTER0_LO
+#define CPC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPC_PERFCOUNTER0_HI
+#define CPC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPF_PERFCOUNTER1_LO
+#define CPF_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPF_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPF_PERFCOUNTER1_HI
+#define CPF_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPF_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPF_PERFCOUNTER0_LO
+#define CPF_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPF_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPF_PERFCOUNTER0_HI
+#define CPF_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPF_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPF_LATENCY_STATS_DATA
+#define CPF_LATENCY_STATS_DATA__DATA__SHIFT                                                                   0x0
+#define CPF_LATENCY_STATS_DATA__DATA_MASK                                                                     0xFFFFFFFFL
+//CPG_LATENCY_STATS_DATA
+#define CPG_LATENCY_STATS_DATA__DATA__SHIFT                                                                   0x0
+#define CPG_LATENCY_STATS_DATA__DATA_MASK                                                                     0xFFFFFFFFL
+//CPC_LATENCY_STATS_DATA
+#define CPC_LATENCY_STATS_DATA__DATA__SHIFT                                                                   0x0
+#define CPC_LATENCY_STATS_DATA__DATA_MASK                                                                     0xFFFFFFFFL
+//GRBM_PERFCOUNTER0_LO
+#define GRBM_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GRBM_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GRBM_PERFCOUNTER0_HI
+#define GRBM_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GRBM_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GRBM_PERFCOUNTER1_LO
+#define GRBM_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GRBM_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GRBM_PERFCOUNTER1_HI
+#define GRBM_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GRBM_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GE1_PERFCOUNTER0_LO
+#define GE1_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GE1_PERFCOUNTER0_HI
+#define GE1_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GE1_PERFCOUNTER1_LO
+#define GE1_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GE1_PERFCOUNTER1_HI
+#define GE1_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GE1_PERFCOUNTER2_LO
+#define GE1_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GE1_PERFCOUNTER2_HI
+#define GE1_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GE1_PERFCOUNTER3_LO
+#define GE1_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GE1_PERFCOUNTER3_HI
+#define GE1_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GE2_DIST_PERFCOUNTER0_LO
+#define GE2_DIST_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                         0xFFFFFFFFL
+//GE2_DIST_PERFCOUNTER0_HI
+#define GE2_DIST_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                         0xFFFFFFFFL
+//GE2_DIST_PERFCOUNTER1_LO
+#define GE2_DIST_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                         0xFFFFFFFFL
+//GE2_DIST_PERFCOUNTER1_HI
+#define GE2_DIST_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                         0xFFFFFFFFL
+//GE2_DIST_PERFCOUNTER2_LO
+#define GE2_DIST_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                         0xFFFFFFFFL
+//GE2_DIST_PERFCOUNTER2_HI
+#define GE2_DIST_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                         0xFFFFFFFFL
+//GE2_DIST_PERFCOUNTER3_LO
+#define GE2_DIST_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                         0xFFFFFFFFL
+//GE2_DIST_PERFCOUNTER3_HI
+#define GE2_DIST_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                         0xFFFFFFFFL
+//GC_EA_CPWD_PERFCOUNTER0_LO
+#define GC_EA_CPWD_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                     0x0
+#define GC_EA_CPWD_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                       0xFFFFFFFFL
+//GC_EA_CPWD_PERFCOUNTER0_HI
+#define GC_EA_CPWD_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                     0x0
+#define GC_EA_CPWD_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                       0xFFFFFFFFL
+//GC_EA_CPWD_PERFCOUNTER1_LO
+#define GC_EA_CPWD_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                     0x0
+#define GC_EA_CPWD_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                       0xFFFFFFFFL
+//GC_EA_CPWD_PERFCOUNTER1_HI
+#define GC_EA_CPWD_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                     0x0
+#define GC_EA_CPWD_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                       0xFFFFFFFFL
+//CHC_PERFCOUNTER0_LO
+#define CHC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CHC_PERFCOUNTER0_HI
+#define CHC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CHC_PERFCOUNTER1_LO
+#define CHC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CHC_PERFCOUNTER1_HI
+#define CHC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CHC_PERFCOUNTER2_LO
+#define CHC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CHC_PERFCOUNTER2_HI
+#define CHC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CHC_PERFCOUNTER3_LO
+#define CHC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CHC_PERFCOUNTER3_HI
+#define CHC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GLARBA_PERFCOUNTER0_LO
+#define GLARBA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                         0x0
+#define GLARBA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                           0xFFFFFFFFL
+//GLARBA_PERFCOUNTER0_HI
+#define GLARBA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                         0x0
+#define GLARBA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                           0xFFFFFFFFL
+//GLARBA_PERFCOUNTER1_LO
+#define GLARBA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                         0x0
+#define GLARBA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                           0xFFFFFFFFL
+//GLARBA_PERFCOUNTER1_HI
+#define GLARBA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                         0x0
+#define GLARBA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                           0xFFFFFFFFL
+//GLARBA_PERFCOUNTER2_LO
+#define GLARBA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                         0x0
+#define GLARBA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                           0xFFFFFFFFL
+//GLARBA_PERFCOUNTER2_HI
+#define GLARBA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                         0x0
+#define GLARBA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                           0xFFFFFFFFL
+//GLARBA_PERFCOUNTER3_LO
+#define GLARBA_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                         0x0
+#define GLARBA_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                           0xFFFFFFFFL
+//GLARBA_PERFCOUNTER3_HI
+#define GLARBA_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                         0x0
+#define GLARBA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                           0xFFFFFFFFL
+//GLARBC_PERFCOUNTER0_LO
+#define GLARBC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                         0x0
+#define GLARBC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                           0xFFFFFFFFL
+//GLARBC_PERFCOUNTER0_HI
+#define GLARBC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                         0x0
+#define GLARBC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                           0xFFFFFFFFL
+//GLARBC_PERFCOUNTER1_LO
+#define GLARBC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                         0x0
+#define GLARBC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                           0xFFFFFFFFL
+//GLARBC_PERFCOUNTER1_HI
+#define GLARBC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                         0x0
+#define GLARBC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                           0xFFFFFFFFL
+//GLARBC_PERFCOUNTER2_LO
+#define GLARBC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                         0x0
+#define GLARBC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                           0xFFFFFFFFL
+//GLARBC_PERFCOUNTER2_HI
+#define GLARBC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                         0x0
+#define GLARBC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                           0xFFFFFFFFL
+//GLARBC_PERFCOUNTER3_LO
+#define GLARBC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                         0x0
+#define GLARBC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                           0xFFFFFFFFL
+//GLARBC_PERFCOUNTER3_HI
+#define GLARBC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                         0x0
+#define GLARBC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                           0xFFFFFFFFL
+//RLC_PERFCOUNTER0_LO
+#define RLC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RLC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RLC_PERFCOUNTER0_HI
+#define RLC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RLC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//RLC_PERFCOUNTER1_LO
+#define RLC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RLC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RLC_PERFCOUNTER1_HI
+#define RLC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RLC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GCR_PERFCOUNTER0_LO
+#define GCR_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GCR_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GCR_PERFCOUNTER0_HI
+#define GCR_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GCR_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GCR_PERFCOUNTER1_LO
+#define GCR_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GCR_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GCR_PERFCOUNTER1_HI
+#define GCR_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GCR_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CHA_PERFCOUNTER0_LO
+#define CHA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CHA_PERFCOUNTER0_HI
+#define CHA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CHA_PERFCOUNTER1_LO
+#define CHA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CHA_PERFCOUNTER1_HI
+#define CHA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CHA_PERFCOUNTER2_LO
+#define CHA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CHA_PERFCOUNTER2_HI
+#define CHA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CHA_PERFCOUNTER3_LO
+#define CHA_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CHA_PERFCOUNTER3_HI
+#define CHA_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_perfsdec
+//CPG_PERFCOUNTER1_SELECT
+#define CPG_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CPG_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x1c
+#define CPG_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CPG_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0xF0000000L
+//CPG_PERFCOUNTER0_SELECT1
+#define CPG_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define CPG_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define CPG_PERFCOUNTER0_SELECT1__CNTR_MODE3__SHIFT                                                           0x18
+#define CPG_PERFCOUNTER0_SELECT1__CNTR_MODE2__SHIFT                                                           0x1c
+#define CPG_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define CPG_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define CPG_PERFCOUNTER0_SELECT1__CNTR_MODE3_MASK                                                             0x0F000000L
+#define CPG_PERFCOUNTER0_SELECT1__CNTR_MODE2_MASK                                                             0xF0000000L
+//CPG_PERFCOUNTER0_SELECT
+#define CPG_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CPG_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define CPG_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE1__SHIFT                                                            0x18
+#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
+#define CPG_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CPG_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define CPG_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
+#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
+//CPC_PERFCOUNTER1_SELECT
+#define CPC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CPC_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x1c
+#define CPC_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CPC_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0xF0000000L
+//CPC_PERFCOUNTER0_SELECT1
+#define CPC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define CPC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define CPC_PERFCOUNTER0_SELECT1__CNTR_MODE3__SHIFT                                                           0x18
+#define CPC_PERFCOUNTER0_SELECT1__CNTR_MODE2__SHIFT                                                           0x1c
+#define CPC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define CPC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define CPC_PERFCOUNTER0_SELECT1__CNTR_MODE3_MASK                                                             0x0F000000L
+#define CPC_PERFCOUNTER0_SELECT1__CNTR_MODE2_MASK                                                             0xF0000000L
+//CPF_PERFCOUNTER1_SELECT
+#define CPF_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CPF_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x1c
+#define CPF_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CPF_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0xF0000000L
+//CPF_PERFCOUNTER0_SELECT1
+#define CPF_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define CPF_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define CPF_PERFCOUNTER0_SELECT1__CNTR_MODE3__SHIFT                                                           0x18
+#define CPF_PERFCOUNTER0_SELECT1__CNTR_MODE2__SHIFT                                                           0x1c
+#define CPF_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define CPF_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define CPF_PERFCOUNTER0_SELECT1__CNTR_MODE3_MASK                                                             0x0F000000L
+#define CPF_PERFCOUNTER0_SELECT1__CNTR_MODE2_MASK                                                             0xF0000000L
+//CPF_PERFCOUNTER0_SELECT
+#define CPF_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CPF_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define CPF_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE1__SHIFT                                                            0x18
+#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
+#define CPF_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CPF_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define CPF_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
+#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
+//CP_PERFMON_CNTL
+#define CP_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                                 0x0
+#define CP_PERFMON_CNTL__SPM_PERFMON_STATE__SHIFT                                                             0x4
+#define CP_PERFMON_CNTL__PERFMON_ENABLE_MODE__SHIFT                                                           0x8
+#define CP_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT                                                         0xa
+#define CP_PERFMON_CNTL__PERFMON_STATE_MASK                                                                   0x0000000FL
+#define CP_PERFMON_CNTL__SPM_PERFMON_STATE_MASK                                                               0x000000F0L
+#define CP_PERFMON_CNTL__PERFMON_ENABLE_MODE_MASK                                                             0x00000300L
+#define CP_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK                                                           0x00000400L
+//CPC_PERFCOUNTER0_SELECT
+#define CPC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CPC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define CPC_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE1__SHIFT                                                            0x18
+#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
+#define CPC_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CPC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define CPC_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
+#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
+//CPF_TC_PERF_COUNTER_WINDOW_SELECT
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__INDEX__SHIFT                                                       0x0
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS__SHIFT                                                      0x1e
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE__SHIFT                                                      0x1f
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__INDEX_MASK                                                         0x00000007L
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS_MASK                                                        0x40000000L
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE_MASK                                                        0x80000000L
+//CPG_TC_PERF_COUNTER_WINDOW_SELECT
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__INDEX__SHIFT                                                       0x0
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS__SHIFT                                                      0x1e
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE__SHIFT                                                      0x1f
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__INDEX_MASK                                                         0x0000001FL
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS_MASK                                                        0x40000000L
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE_MASK                                                        0x80000000L
+//CPF_LATENCY_STATS_SELECT
+#define CPF_LATENCY_STATS_SELECT__INDEX__SHIFT                                                                0x0
+#define CPF_LATENCY_STATS_SELECT__CLEAR__SHIFT                                                                0x1e
+#define CPF_LATENCY_STATS_SELECT__ENABLE__SHIFT                                                               0x1f
+#define CPF_LATENCY_STATS_SELECT__INDEX_MASK                                                                  0x0000000FL
+#define CPF_LATENCY_STATS_SELECT__CLEAR_MASK                                                                  0x40000000L
+#define CPF_LATENCY_STATS_SELECT__ENABLE_MASK                                                                 0x80000000L
+//CPG_LATENCY_STATS_SELECT
+#define CPG_LATENCY_STATS_SELECT__INDEX__SHIFT                                                                0x0
+#define CPG_LATENCY_STATS_SELECT__CLEAR__SHIFT                                                                0x1e
+#define CPG_LATENCY_STATS_SELECT__ENABLE__SHIFT                                                               0x1f
+#define CPG_LATENCY_STATS_SELECT__INDEX_MASK                                                                  0x0000001FL
+#define CPG_LATENCY_STATS_SELECT__CLEAR_MASK                                                                  0x40000000L
+#define CPG_LATENCY_STATS_SELECT__ENABLE_MASK                                                                 0x80000000L
+//CPC_LATENCY_STATS_SELECT
+#define CPC_LATENCY_STATS_SELECT__INDEX__SHIFT                                                                0x0
+#define CPC_LATENCY_STATS_SELECT__CLEAR__SHIFT                                                                0x1e
+#define CPC_LATENCY_STATS_SELECT__ENABLE__SHIFT                                                               0x1f
+#define CPC_LATENCY_STATS_SELECT__INDEX_MASK                                                                  0x0000001FL
+#define CPC_LATENCY_STATS_SELECT__CLEAR_MASK                                                                  0x40000000L
+#define CPC_LATENCY_STATS_SELECT__ENABLE_MASK                                                                 0x80000000L
+//CPC_TC_PERF_COUNTER_WINDOW_SELECT
+#define CPC_TC_PERF_COUNTER_WINDOW_SELECT__INDEX__SHIFT                                                       0x0
+#define CPC_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS__SHIFT                                                      0x1e
+#define CPC_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE__SHIFT                                                      0x1f
+#define CPC_TC_PERF_COUNTER_WINDOW_SELECT__INDEX_MASK                                                         0x0000001FL
+#define CPC_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS_MASK                                                        0x40000000L
+#define CPC_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE_MASK                                                        0x80000000L
+//CP_DRAW_OBJECT
+#define CP_DRAW_OBJECT__OBJECT__SHIFT                                                                         0x0
+#define CP_DRAW_OBJECT__OBJECT_MASK                                                                           0xFFFFFFFFL
+//CP_DRAW_OBJECT_COUNTER
+#define CP_DRAW_OBJECT_COUNTER__COUNT__SHIFT                                                                  0x0
+#define CP_DRAW_OBJECT_COUNTER__COUNT_MASK                                                                    0x0000FFFFL
+//CP_DRAW_WINDOW_MASK_HI
+#define CP_DRAW_WINDOW_MASK_HI__WINDOW_MASK_HI__SHIFT                                                         0x0
+#define CP_DRAW_WINDOW_MASK_HI__WINDOW_MASK_HI_MASK                                                           0xFFFFFFFFL
+//CP_DRAW_WINDOW_HI
+#define CP_DRAW_WINDOW_HI__WINDOW_HI__SHIFT                                                                   0x0
+#define CP_DRAW_WINDOW_HI__WINDOW_HI_MASK                                                                     0xFFFFFFFFL
+//CP_DRAW_WINDOW_LO
+#define CP_DRAW_WINDOW_LO__MIN__SHIFT                                                                         0x0
+#define CP_DRAW_WINDOW_LO__MAX__SHIFT                                                                         0x10
+#define CP_DRAW_WINDOW_LO__MIN_MASK                                                                           0x0000FFFFL
+#define CP_DRAW_WINDOW_LO__MAX_MASK                                                                           0xFFFF0000L
+//CP_DRAW_WINDOW_CNTL
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MAX__SHIFT                                                0x0
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MIN__SHIFT                                                0x1
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_HI__SHIFT                                                    0x2
+#define CP_DRAW_WINDOW_CNTL__MODE__SHIFT                                                                      0x8
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MAX_MASK                                                  0x00000001L
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MIN_MASK                                                  0x00000002L
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_HI_MASK                                                      0x00000004L
+#define CP_DRAW_WINDOW_CNTL__MODE_MASK                                                                        0x00000100L
+//GRBM_PERFCOUNTER0_SELECT
+#define GRBM_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GRBM_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                           0xa
+#define GRBM_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                           0xb
+#define GRBM_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                            0xd
+#define GRBM_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                            0xe
+#define GRBM_PERFCOUNTER0_SELECT__GRBM_BUSY_USER_DEFINED_MASK__SHIFT                                          0x13
+#define GRBM_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                            0x14
+#define GRBM_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                            0x15
+#define GRBM_PERFCOUNTER0_SELECT__CP_BUSY_USER_DEFINED_MASK__SHIFT                                            0x16
+#define GRBM_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x19
+#define GRBM_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1a
+#define GRBM_PERFCOUNTER0_SELECT__TCP_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1b
+#define GRBM_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1c
+#define GRBM_PERFCOUNTER0_SELECT__UTCL2_BUSY_USER_DEFINED_MASK__SHIFT                                         0x1d
+#define GRBM_PERFCOUNTER0_SELECT__CANE_BUSY_USER_DEFINED_MASK__SHIFT                                          0x1e
+#define GRBM_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                               0x0000003FL
+#define GRBM_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                             0x00000400L
+#define GRBM_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                             0x00000800L
+#define GRBM_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                              0x00002000L
+#define GRBM_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                              0x00004000L
+#define GRBM_PERFCOUNTER0_SELECT__GRBM_BUSY_USER_DEFINED_MASK_MASK                                            0x00080000L
+#define GRBM_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                              0x00100000L
+#define GRBM_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                              0x00200000L
+#define GRBM_PERFCOUNTER0_SELECT__CP_BUSY_USER_DEFINED_MASK_MASK                                              0x00400000L
+#define GRBM_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                             0x02000000L
+#define GRBM_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK                                             0x04000000L
+#define GRBM_PERFCOUNTER0_SELECT__TCP_BUSY_USER_DEFINED_MASK_MASK                                             0x08000000L
+#define GRBM_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                             0x10000000L
+#define GRBM_PERFCOUNTER0_SELECT__UTCL2_BUSY_USER_DEFINED_MASK_MASK                                           0x20000000L
+#define GRBM_PERFCOUNTER0_SELECT__CANE_BUSY_USER_DEFINED_MASK_MASK                                            0x40000000L
+//GRBM_PERFCOUNTER1_SELECT
+#define GRBM_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GRBM_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                           0xa
+#define GRBM_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                           0xb
+#define GRBM_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                            0xd
+#define GRBM_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                            0xe
+#define GRBM_PERFCOUNTER1_SELECT__GRBM_BUSY_USER_DEFINED_MASK__SHIFT                                          0x13
+#define GRBM_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                            0x14
+#define GRBM_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                            0x15
+#define GRBM_PERFCOUNTER1_SELECT__CP_BUSY_USER_DEFINED_MASK__SHIFT                                            0x16
+#define GRBM_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x19
+#define GRBM_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1a
+#define GRBM_PERFCOUNTER1_SELECT__TCP_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1b
+#define GRBM_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1c
+#define GRBM_PERFCOUNTER1_SELECT__UTCL2_BUSY_USER_DEFINED_MASK__SHIFT                                         0x1d
+#define GRBM_PERFCOUNTER1_SELECT__CANE_BUSY_USER_DEFINED_MASK__SHIFT                                          0x1e
+#define GRBM_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                               0x0000003FL
+#define GRBM_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                             0x00000400L
+#define GRBM_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                             0x00000800L
+#define GRBM_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                              0x00002000L
+#define GRBM_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                              0x00004000L
+#define GRBM_PERFCOUNTER1_SELECT__GRBM_BUSY_USER_DEFINED_MASK_MASK                                            0x00080000L
+#define GRBM_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                              0x00100000L
+#define GRBM_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                              0x00200000L
+#define GRBM_PERFCOUNTER1_SELECT__CP_BUSY_USER_DEFINED_MASK_MASK                                              0x00400000L
+#define GRBM_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                             0x02000000L
+#define GRBM_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK                                             0x04000000L
+#define GRBM_PERFCOUNTER1_SELECT__TCP_BUSY_USER_DEFINED_MASK_MASK                                             0x08000000L
+#define GRBM_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                             0x10000000L
+#define GRBM_PERFCOUNTER1_SELECT__UTCL2_BUSY_USER_DEFINED_MASK_MASK                                           0x20000000L
+#define GRBM_PERFCOUNTER1_SELECT__CANE_BUSY_USER_DEFINED_MASK_MASK                                            0x40000000L
+//GRBM_PERFCOUNTER0_SELECT_HI
+#define GRBM_PERFCOUNTER0_SELECT_HI__UTCL1_BUSY_USER_DEFINED_MASK__SHIFT                                      0x1
+#define GRBM_PERFCOUNTER0_SELECT_HI__GLARB_BUSY_USER_DEFINED_MASK__SHIFT                                      0x2
+#define GRBM_PERFCOUNTER0_SELECT_HI__SDMA_BUSY_USER_DEFINED_MASK__SHIFT                                       0x3
+#define GRBM_PERFCOUNTER0_SELECT_HI__CH_BUSY_USER_DEFINED_MASK__SHIFT                                         0x4
+#define GRBM_PERFCOUNTER0_SELECT_HI__PMM_BUSY_USER_DEFINED_MASK__SHIFT                                        0x6
+#define GRBM_PERFCOUNTER0_SELECT_HI__GL1CC_BUSY_USER_DEFINED_MASK__SHIFT                                      0x8
+#define GRBM_PERFCOUNTER0_SELECT_HI__GL1XCC_BUSY_USER_DEFINED_MASK__SHIFT                                     0x9
+#define GRBM_PERFCOUNTER0_SELECT_HI__WGS_BUSY_USER_DEFINED_MASK__SHIFT                                        0xc
+#define GRBM_PERFCOUNTER0_SELECT_HI__CANE_STAT_LINK_BUSY_USER_DEFINED_MASK__SHIFT                             0xd
+#define GRBM_PERFCOUNTER0_SELECT_HI__UTCL1_BUSY_USER_DEFINED_MASK_MASK                                        0x00000002L
+#define GRBM_PERFCOUNTER0_SELECT_HI__GLARB_BUSY_USER_DEFINED_MASK_MASK                                        0x00000004L
+#define GRBM_PERFCOUNTER0_SELECT_HI__SDMA_BUSY_USER_DEFINED_MASK_MASK                                         0x00000008L
+#define GRBM_PERFCOUNTER0_SELECT_HI__CH_BUSY_USER_DEFINED_MASK_MASK                                           0x00000010L
+#define GRBM_PERFCOUNTER0_SELECT_HI__PMM_BUSY_USER_DEFINED_MASK_MASK                                          0x00000040L
+#define GRBM_PERFCOUNTER0_SELECT_HI__GL1CC_BUSY_USER_DEFINED_MASK_MASK                                        0x00000100L
+#define GRBM_PERFCOUNTER0_SELECT_HI__GL1XCC_BUSY_USER_DEFINED_MASK_MASK                                       0x00000200L
+#define GRBM_PERFCOUNTER0_SELECT_HI__WGS_BUSY_USER_DEFINED_MASK_MASK                                          0x00001000L
+#define GRBM_PERFCOUNTER0_SELECT_HI__CANE_STAT_LINK_BUSY_USER_DEFINED_MASK_MASK                               0x00002000L
+//GRBM_PERFCOUNTER1_SELECT_HI
+#define GRBM_PERFCOUNTER1_SELECT_HI__UTCL1_BUSY_USER_DEFINED_MASK__SHIFT                                      0x1
+#define GRBM_PERFCOUNTER1_SELECT_HI__GLARB_BUSY_USER_DEFINED_MASK__SHIFT                                      0x2
+#define GRBM_PERFCOUNTER1_SELECT_HI__SDMA_BUSY_USER_DEFINED_MASK__SHIFT                                       0x3
+#define GRBM_PERFCOUNTER1_SELECT_HI__CH_BUSY_USER_DEFINED_MASK__SHIFT                                         0x4
+#define GRBM_PERFCOUNTER1_SELECT_HI__PMM_BUSY_USER_DEFINED_MASK__SHIFT                                        0x6
+#define GRBM_PERFCOUNTER1_SELECT_HI__GL1CC_BUSY_USER_DEFINED_MASK__SHIFT                                      0x8
+#define GRBM_PERFCOUNTER1_SELECT_HI__GL1XCC_BUSY_USER_DEFINED_MASK__SHIFT                                     0x9
+#define GRBM_PERFCOUNTER1_SELECT_HI__WGS_BUSY_USER_DEFINED_MASK__SHIFT                                        0xc
+#define GRBM_PERFCOUNTER1_SELECT_HI__CANE_STAT_LINK_BUSY_USER_DEFINED_MASK__SHIFT                             0xd
+#define GRBM_PERFCOUNTER1_SELECT_HI__UTCL1_BUSY_USER_DEFINED_MASK_MASK                                        0x00000002L
+#define GRBM_PERFCOUNTER1_SELECT_HI__GLARB_BUSY_USER_DEFINED_MASK_MASK                                        0x00000004L
+#define GRBM_PERFCOUNTER1_SELECT_HI__SDMA_BUSY_USER_DEFINED_MASK_MASK                                         0x00000008L
+#define GRBM_PERFCOUNTER1_SELECT_HI__CH_BUSY_USER_DEFINED_MASK_MASK                                           0x00000010L
+#define GRBM_PERFCOUNTER1_SELECT_HI__PMM_BUSY_USER_DEFINED_MASK_MASK                                          0x00000040L
+#define GRBM_PERFCOUNTER1_SELECT_HI__GL1CC_BUSY_USER_DEFINED_MASK_MASK                                        0x00000100L
+#define GRBM_PERFCOUNTER1_SELECT_HI__GL1XCC_BUSY_USER_DEFINED_MASK_MASK                                       0x00000200L
+#define GRBM_PERFCOUNTER1_SELECT_HI__WGS_BUSY_USER_DEFINED_MASK_MASK                                          0x00001000L
+#define GRBM_PERFCOUNTER1_SELECT_HI__CANE_STAT_LINK_BUSY_USER_DEFINED_MASK_MASK                               0x00002000L
+//GE1_PERFCOUNTER0_SELECT
+#define GE1_PERFCOUNTER0_SELECT__PERF_SEL0__SHIFT                                                             0x0
+#define GE1_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GE1_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GE1_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GE1_PERFCOUNTER0_SELECT__PERF_MODE0__SHIFT                                                            0x1c
+#define GE1_PERFCOUNTER0_SELECT__PERF_SEL0_MASK                                                               0x000003FFL
+#define GE1_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GE1_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GE1_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GE1_PERFCOUNTER0_SELECT__PERF_MODE0_MASK                                                              0xF0000000L
+//GE1_PERFCOUNTER0_SELECT1
+#define GE1_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GE1_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GE1_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GE1_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GE1_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GE1_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GE1_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//GE1_PERFCOUNTER1_SELECT
+#define GE1_PERFCOUNTER1_SELECT__PERF_SEL0__SHIFT                                                             0x0
+#define GE1_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GE1_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GE1_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GE1_PERFCOUNTER1_SELECT__PERF_MODE0__SHIFT                                                            0x1c
+#define GE1_PERFCOUNTER1_SELECT__PERF_SEL0_MASK                                                               0x000003FFL
+#define GE1_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GE1_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GE1_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GE1_PERFCOUNTER1_SELECT__PERF_MODE0_MASK                                                              0xF0000000L
+//GE1_PERFCOUNTER1_SELECT1
+#define GE1_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GE1_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GE1_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GE1_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GE1_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GE1_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GE1_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//GE1_PERFCOUNTER2_SELECT
+#define GE1_PERFCOUNTER2_SELECT__PERF_SEL0__SHIFT                                                             0x0
+#define GE1_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GE1_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GE1_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GE1_PERFCOUNTER2_SELECT__PERF_MODE0__SHIFT                                                            0x1c
+#define GE1_PERFCOUNTER2_SELECT__PERF_SEL0_MASK                                                               0x000003FFL
+#define GE1_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GE1_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GE1_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GE1_PERFCOUNTER2_SELECT__PERF_MODE0_MASK                                                              0xF0000000L
+//GE1_PERFCOUNTER2_SELECT1
+#define GE1_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GE1_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GE1_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GE1_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GE1_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GE1_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GE1_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//GE1_PERFCOUNTER3_SELECT
+#define GE1_PERFCOUNTER3_SELECT__PERF_SEL0__SHIFT                                                             0x0
+#define GE1_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GE1_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GE1_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GE1_PERFCOUNTER3_SELECT__PERF_MODE0__SHIFT                                                            0x1c
+#define GE1_PERFCOUNTER3_SELECT__PERF_SEL0_MASK                                                               0x000003FFL
+#define GE1_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GE1_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GE1_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GE1_PERFCOUNTER3_SELECT__PERF_MODE0_MASK                                                              0xF0000000L
+//GE1_PERFCOUNTER3_SELECT1
+#define GE1_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GE1_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GE1_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GE1_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GE1_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GE1_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GE1_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//GE2_DIST_PERFCOUNTER0_SELECT
+#define GE2_DIST_PERFCOUNTER0_SELECT__PERF_SEL0__SHIFT                                                        0x0
+#define GE2_DIST_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                        0xa
+#define GE2_DIST_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                        0x14
+#define GE2_DIST_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                       0x18
+#define GE2_DIST_PERFCOUNTER0_SELECT__PERF_MODE0__SHIFT                                                       0x1c
+#define GE2_DIST_PERFCOUNTER0_SELECT__PERF_SEL0_MASK                                                          0x000003FFL
+#define GE2_DIST_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                          0x000FFC00L
+#define GE2_DIST_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                          0x00F00000L
+#define GE2_DIST_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                         0x0F000000L
+#define GE2_DIST_PERFCOUNTER0_SELECT__PERF_MODE0_MASK                                                         0xF0000000L
+//GE2_DIST_PERFCOUNTER0_SELECT1
+#define GE2_DIST_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                       0xa
+#define GE2_DIST_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                      0x18
+#define GE2_DIST_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                      0x1c
+#define GE2_DIST_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                         0x000003FFL
+#define GE2_DIST_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                         0x000FFC00L
+#define GE2_DIST_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                        0x0F000000L
+#define GE2_DIST_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                        0xF0000000L
+//GE2_DIST_PERFCOUNTER1_SELECT
+#define GE2_DIST_PERFCOUNTER1_SELECT__PERF_SEL0__SHIFT                                                        0x0
+#define GE2_DIST_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                        0xa
+#define GE2_DIST_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                        0x14
+#define GE2_DIST_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                       0x18
+#define GE2_DIST_PERFCOUNTER1_SELECT__PERF_MODE0__SHIFT                                                       0x1c
+#define GE2_DIST_PERFCOUNTER1_SELECT__PERF_SEL0_MASK                                                          0x000003FFL
+#define GE2_DIST_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                          0x000FFC00L
+#define GE2_DIST_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                          0x00F00000L
+#define GE2_DIST_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                         0x0F000000L
+#define GE2_DIST_PERFCOUNTER1_SELECT__PERF_MODE0_MASK                                                         0xF0000000L
+//GE2_DIST_PERFCOUNTER1_SELECT1
+#define GE2_DIST_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                       0xa
+#define GE2_DIST_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                      0x18
+#define GE2_DIST_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                      0x1c
+#define GE2_DIST_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                         0x000003FFL
+#define GE2_DIST_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                         0x000FFC00L
+#define GE2_DIST_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                        0x0F000000L
+#define GE2_DIST_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                        0xF0000000L
+//GE2_DIST_PERFCOUNTER2_SELECT
+#define GE2_DIST_PERFCOUNTER2_SELECT__PERF_SEL0__SHIFT                                                        0x0
+#define GE2_DIST_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                        0xa
+#define GE2_DIST_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                        0x14
+#define GE2_DIST_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                       0x18
+#define GE2_DIST_PERFCOUNTER2_SELECT__PERF_MODE0__SHIFT                                                       0x1c
+#define GE2_DIST_PERFCOUNTER2_SELECT__PERF_SEL0_MASK                                                          0x000003FFL
+#define GE2_DIST_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                          0x000FFC00L
+#define GE2_DIST_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                          0x00F00000L
+#define GE2_DIST_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                         0x0F000000L
+#define GE2_DIST_PERFCOUNTER2_SELECT__PERF_MODE0_MASK                                                         0xF0000000L
+//GE2_DIST_PERFCOUNTER2_SELECT1
+#define GE2_DIST_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                       0xa
+#define GE2_DIST_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                      0x18
+#define GE2_DIST_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                      0x1c
+#define GE2_DIST_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                         0x000003FFL
+#define GE2_DIST_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                         0x000FFC00L
+#define GE2_DIST_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                        0x0F000000L
+#define GE2_DIST_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                        0xF0000000L
+//GE2_DIST_PERFCOUNTER3_SELECT
+#define GE2_DIST_PERFCOUNTER3_SELECT__PERF_SEL0__SHIFT                                                        0x0
+#define GE2_DIST_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                        0xa
+#define GE2_DIST_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                        0x14
+#define GE2_DIST_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                       0x18
+#define GE2_DIST_PERFCOUNTER3_SELECT__PERF_MODE0__SHIFT                                                       0x1c
+#define GE2_DIST_PERFCOUNTER3_SELECT__PERF_SEL0_MASK                                                          0x000003FFL
+#define GE2_DIST_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                          0x000FFC00L
+#define GE2_DIST_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                          0x00F00000L
+#define GE2_DIST_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                         0x0F000000L
+#define GE2_DIST_PERFCOUNTER3_SELECT__PERF_MODE0_MASK                                                         0xF0000000L
+//GE2_DIST_PERFCOUNTER3_SELECT1
+#define GE2_DIST_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                       0xa
+#define GE2_DIST_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                      0x18
+#define GE2_DIST_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                      0x1c
+#define GE2_DIST_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                         0x000003FFL
+#define GE2_DIST_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                         0x000FFC00L
+#define GE2_DIST_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                        0x0F000000L
+#define GE2_DIST_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                        0xF0000000L
+//GC_EA_CPWD_PERFCOUNTER0_SELECT
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                       0x0
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                      0xa
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                      0x14
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                     0x18
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                      0x1c
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                         0x000003FFL
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                        0x000FFC00L
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                        0x00F00000L
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                       0x0F000000L
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                        0xF0000000L
+//GC_EA_CPWD_PERFCOUNTER0_SELECT1
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                     0x0
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                     0xa
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                    0x18
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                    0x1c
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                       0x000003FFL
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                       0x000FFC00L
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                      0x0F000000L
+#define GC_EA_CPWD_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                      0xF0000000L
+//GC_EA_CPWD_PERFCOUNTER1_SELECT
+#define GC_EA_CPWD_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                       0x0
+#define GC_EA_CPWD_PERFCOUNTER1_SELECT__COUNTER_MODE__SHIFT                                                   0x1c
+#define GC_EA_CPWD_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                         0x000003FFL
+#define GC_EA_CPWD_PERFCOUNTER1_SELECT__COUNTER_MODE_MASK                                                     0xF0000000L
+//CHC_PERFCOUNTER0_SELECT
+#define CHC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CHC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define CHC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define CHC_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define CHC_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define CHC_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CHC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define CHC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define CHC_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define CHC_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//CHC_PERFCOUNTER0_SELECT1
+#define CHC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define CHC_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define CHC_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define CHC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define CHC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define CHC_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define CHC_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//CHC_PERFCOUNTER1_SELECT
+#define CHC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CHC_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define CHC_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define CHC_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define CHC_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define CHC_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CHC_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define CHC_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define CHC_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define CHC_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//CHC_PERFCOUNTER1_SELECT1
+#define CHC_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define CHC_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define CHC_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define CHC_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define CHC_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define CHC_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define CHC_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//CHC_PERFCOUNTER2_SELECT
+#define CHC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CHC_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define CHC_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define CHC_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define CHC_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define CHC_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CHC_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define CHC_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define CHC_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define CHC_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//CHC_PERFCOUNTER2_SELECT1
+#define CHC_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define CHC_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define CHC_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define CHC_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define CHC_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define CHC_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define CHC_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//CHC_PERFCOUNTER3_SELECT
+#define CHC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CHC_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define CHC_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define CHC_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define CHC_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define CHC_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CHC_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define CHC_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define CHC_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define CHC_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//CHC_PERFCOUNTER3_SELECT1
+#define CHC_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define CHC_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define CHC_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define CHC_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define CHC_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define CHC_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define CHC_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//GLARBA_PERFCOUNTER0_SELECT
+#define GLARBA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                           0x0
+#define GLARBA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                          0xa
+#define GLARBA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                          0x14
+#define GLARBA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                         0x18
+#define GLARBA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                          0x1c
+#define GLARBA_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                             0x000003FFL
+#define GLARBA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                            0x000FFC00L
+#define GLARBA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                            0x00F00000L
+#define GLARBA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                           0x0F000000L
+#define GLARBA_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                            0xF0000000L
+//GLARBA_PERFCOUNTER0_SELECT1
+#define GLARBA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                         0x0
+#define GLARBA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                         0xa
+#define GLARBA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                        0x18
+#define GLARBA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                        0x1c
+#define GLARBA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                           0x000003FFL
+#define GLARBA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                           0x000FFC00L
+#define GLARBA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                          0x0F000000L
+#define GLARBA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                          0xF0000000L
+//GLARBA_PERFCOUNTER1_SELECT
+#define GLARBA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                           0x0
+#define GLARBA_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                          0xa
+#define GLARBA_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                          0x14
+#define GLARBA_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                         0x18
+#define GLARBA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                          0x1c
+#define GLARBA_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                             0x000003FFL
+#define GLARBA_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                            0x000FFC00L
+#define GLARBA_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                            0x00F00000L
+#define GLARBA_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                           0x0F000000L
+#define GLARBA_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                            0xF0000000L
+//GLARBA_PERFCOUNTER1_SELECT1
+#define GLARBA_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                         0x0
+#define GLARBA_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                         0xa
+#define GLARBA_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                        0x18
+#define GLARBA_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                        0x1c
+#define GLARBA_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                           0x000003FFL
+#define GLARBA_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                           0x000FFC00L
+#define GLARBA_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                          0x0F000000L
+#define GLARBA_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                          0xF0000000L
+//GLARBA_PERFCOUNTER2_SELECT
+#define GLARBA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                           0x0
+#define GLARBA_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                          0xa
+#define GLARBA_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                          0x14
+#define GLARBA_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                         0x18
+#define GLARBA_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                          0x1c
+#define GLARBA_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                             0x000003FFL
+#define GLARBA_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                            0x000FFC00L
+#define GLARBA_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                            0x00F00000L
+#define GLARBA_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                           0x0F000000L
+#define GLARBA_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                            0xF0000000L
+//GLARBA_PERFCOUNTER2_SELECT1
+#define GLARBA_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                         0x0
+#define GLARBA_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                         0xa
+#define GLARBA_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                        0x18
+#define GLARBA_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                        0x1c
+#define GLARBA_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                           0x000003FFL
+#define GLARBA_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                           0x000FFC00L
+#define GLARBA_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                          0x0F000000L
+#define GLARBA_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                          0xF0000000L
+//GLARBA_PERFCOUNTER3_SELECT
+#define GLARBA_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                           0x0
+#define GLARBA_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                          0xa
+#define GLARBA_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                          0x14
+#define GLARBA_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                         0x18
+#define GLARBA_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                          0x1c
+#define GLARBA_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                             0x000003FFL
+#define GLARBA_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                            0x000FFC00L
+#define GLARBA_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                            0x00F00000L
+#define GLARBA_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                           0x0F000000L
+#define GLARBA_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                            0xF0000000L
+//GLARBA_PERFCOUNTER3_SELECT1
+#define GLARBA_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                         0x0
+#define GLARBA_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                         0xa
+#define GLARBA_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                        0x18
+#define GLARBA_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                        0x1c
+#define GLARBA_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                           0x000003FFL
+#define GLARBA_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                           0x000FFC00L
+#define GLARBA_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                          0x0F000000L
+#define GLARBA_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                          0xF0000000L
+//GLARBC_PERFCOUNTER0_SELECT
+#define GLARBC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                           0x0
+#define GLARBC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                          0xa
+#define GLARBC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                          0x14
+#define GLARBC_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                         0x18
+#define GLARBC_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                          0x1c
+#define GLARBC_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                             0x000003FFL
+#define GLARBC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                            0x000FFC00L
+#define GLARBC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                            0x00F00000L
+#define GLARBC_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                           0x0F000000L
+#define GLARBC_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                            0xF0000000L
+//GLARBC_PERFCOUNTER0_SELECT1
+#define GLARBC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                         0x0
+#define GLARBC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                         0xa
+#define GLARBC_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                        0x18
+#define GLARBC_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                        0x1c
+#define GLARBC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                           0x000003FFL
+#define GLARBC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                           0x000FFC00L
+#define GLARBC_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                          0x0F000000L
+#define GLARBC_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                          0xF0000000L
+//GLARBC_PERFCOUNTER1_SELECT
+#define GLARBC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                           0x0
+#define GLARBC_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                          0xa
+#define GLARBC_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                          0x14
+#define GLARBC_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                         0x18
+#define GLARBC_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                          0x1c
+#define GLARBC_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                             0x000003FFL
+#define GLARBC_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                            0x000FFC00L
+#define GLARBC_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                            0x00F00000L
+#define GLARBC_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                           0x0F000000L
+#define GLARBC_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                            0xF0000000L
+//GLARBC_PERFCOUNTER1_SELECT1
+#define GLARBC_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                         0x0
+#define GLARBC_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                         0xa
+#define GLARBC_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                        0x18
+#define GLARBC_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                        0x1c
+#define GLARBC_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                           0x000003FFL
+#define GLARBC_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                           0x000FFC00L
+#define GLARBC_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                          0x0F000000L
+#define GLARBC_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                          0xF0000000L
+//GLARBC_PERFCOUNTER2_SELECT
+#define GLARBC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                           0x0
+#define GLARBC_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                          0xa
+#define GLARBC_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                          0x14
+#define GLARBC_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                         0x18
+#define GLARBC_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                          0x1c
+#define GLARBC_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                             0x000003FFL
+#define GLARBC_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                            0x000FFC00L
+#define GLARBC_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                            0x00F00000L
+#define GLARBC_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                           0x0F000000L
+#define GLARBC_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                            0xF0000000L
+//GLARBC_PERFCOUNTER2_SELECT1
+#define GLARBC_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                         0x0
+#define GLARBC_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                         0xa
+#define GLARBC_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                        0x18
+#define GLARBC_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                        0x1c
+#define GLARBC_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                           0x000003FFL
+#define GLARBC_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                           0x000FFC00L
+#define GLARBC_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                          0x0F000000L
+#define GLARBC_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                          0xF0000000L
+//GLARBC_PERFCOUNTER3_SELECT
+#define GLARBC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                           0x0
+#define GLARBC_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                          0xa
+#define GLARBC_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                          0x14
+#define GLARBC_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                         0x18
+#define GLARBC_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                          0x1c
+#define GLARBC_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                             0x000003FFL
+#define GLARBC_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                            0x000FFC00L
+#define GLARBC_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                            0x00F00000L
+#define GLARBC_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                           0x0F000000L
+#define GLARBC_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                            0xF0000000L
+//GLARBC_PERFCOUNTER3_SELECT1
+#define GLARBC_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                         0x0
+#define GLARBC_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                         0xa
+#define GLARBC_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                        0x18
+#define GLARBC_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                        0x1c
+#define GLARBC_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                           0x000003FFL
+#define GLARBC_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                           0x000FFC00L
+#define GLARBC_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                          0x0F000000L
+#define GLARBC_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                          0xF0000000L
+//RLC_SPM_PERFMON_CNTL
+#define RLC_SPM_PERFMON_CNTL__RESERVED__SHIFT                                                                 0x0
+#define RLC_SPM_PERFMON_CNTL__PERFMON_RING_MODE__SHIFT                                                        0xc
+#define RLC_SPM_PERFMON_CNTL__PERFMON_SAMPLE_INTERVAL_START_MODE__SHIFT                                       0xe
+#define RLC_SPM_PERFMON_CNTL__PERFMON_SAMPLE_INTERVAL_TYPE__SHIFT                                             0xf
+#define RLC_SPM_PERFMON_CNTL__PERFMON_SAMPLE_INTERVAL__SHIFT                                                  0x10
+#define RLC_SPM_PERFMON_CNTL__RESERVED_MASK                                                                   0x00000FFFL
+#define RLC_SPM_PERFMON_CNTL__PERFMON_RING_MODE_MASK                                                          0x00003000L
+#define RLC_SPM_PERFMON_CNTL__PERFMON_SAMPLE_INTERVAL_START_MODE_MASK                                         0x00004000L
+#define RLC_SPM_PERFMON_CNTL__PERFMON_SAMPLE_INTERVAL_TYPE_MASK                                               0x00008000L
+#define RLC_SPM_PERFMON_CNTL__PERFMON_SAMPLE_INTERVAL_MASK                                                    0xFFFF0000L
+//RLC_SPM_PERFMON_RING_BASE_LO
+#define RLC_SPM_PERFMON_RING_BASE_LO__RING_BASE_LO__SHIFT                                                     0x0
+#define RLC_SPM_PERFMON_RING_BASE_LO__RING_BASE_LO_MASK                                                       0xFFFFFFFFL
+//RLC_SPM_PERFMON_RING_BASE_HI
+#define RLC_SPM_PERFMON_RING_BASE_HI__RING_BASE_HI__SHIFT                                                     0x0
+#define RLC_SPM_PERFMON_RING_BASE_HI__RESERVED__SHIFT                                                         0x19
+#define RLC_SPM_PERFMON_RING_BASE_HI__RING_BASE_HI_MASK                                                       0x01FFFFFFL
+#define RLC_SPM_PERFMON_RING_BASE_HI__RESERVED_MASK                                                           0xFE000000L
+//RLC_SPM_PERFMON_RING_SIZE
+#define RLC_SPM_PERFMON_RING_SIZE__RING_BASE_SIZE__SHIFT                                                      0x0
+#define RLC_SPM_PERFMON_RING_SIZE__RING_BASE_SIZE_MASK                                                        0xFFFFFFFFL
+//RLC_SPM_RING_WRPTR
+#define RLC_SPM_RING_WRPTR__RESERVED__SHIFT                                                                   0x0
+#define RLC_SPM_RING_WRPTR__PERFMON_RING_WRPTR__SHIFT                                                         0x5
+#define RLC_SPM_RING_WRPTR__RESERVED_MASK                                                                     0x0000001FL
+#define RLC_SPM_RING_WRPTR__PERFMON_RING_WRPTR_MASK                                                           0xFFFFFFE0L
+//RLC_SPM_RING_RDPTR
+#define RLC_SPM_RING_RDPTR__PERFMON_RING_RDPTR__SHIFT                                                         0x0
+#define RLC_SPM_RING_RDPTR__PERFMON_RING_RDPTR_MASK                                                           0xFFFFFFFFL
+//RLC_SPM_SEGMENT_THRESHOLD
+#define RLC_SPM_SEGMENT_THRESHOLD__NUM_SEGMENT_THRESHOLD__SHIFT                                               0x0
+#define RLC_SPM_SEGMENT_THRESHOLD__RESERVED__SHIFT                                                            0x8
+#define RLC_SPM_SEGMENT_THRESHOLD__NUM_SEGMENT_THRESHOLD_MASK                                                 0x000000FFL
+#define RLC_SPM_SEGMENT_THRESHOLD__RESERVED_MASK                                                              0xFFFFFF00L
+//RLC_SPM_PERFMON_SEGMENT_SIZE
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__TOTAL_NUM_SEGMENT__SHIFT                                                0x0
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__GLOBAL_NUM_SEGMENT__SHIFT                                               0x10
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE_NUM_SEGMENT__SHIFT                                                   0x18
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__TOTAL_NUM_SEGMENT_MASK                                                  0x0000FFFFL
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__GLOBAL_NUM_SEGMENT_MASK                                                 0x00FF0000L
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE_NUM_SEGMENT_MASK                                                     0xFF000000L
+//RLC_SPM_GLOBAL_MUXSEL_ADDR
+#define RLC_SPM_GLOBAL_MUXSEL_ADDR__ADDR__SHIFT                                                               0x0
+#define RLC_SPM_GLOBAL_MUXSEL_ADDR__ADDR_MASK                                                                 0x000003FFL
+//RLC_SPM_GLOBAL_MUXSEL_DATA
+#define RLC_SPM_GLOBAL_MUXSEL_DATA__SEL0__SHIFT                                                               0x0
+#define RLC_SPM_GLOBAL_MUXSEL_DATA__SEL1__SHIFT                                                               0x10
+#define RLC_SPM_GLOBAL_MUXSEL_DATA__SEL0_MASK                                                                 0x0000FFFFL
+#define RLC_SPM_GLOBAL_MUXSEL_DATA__SEL1_MASK                                                                 0xFFFF0000L
+//RLC_SPM_SE_MUXSEL_ADDR
+#define RLC_SPM_SE_MUXSEL_ADDR__ADDR__SHIFT                                                                   0x0
+#define RLC_SPM_SE_MUXSEL_ADDR__ADDR_MASK                                                                     0x000003FFL
+//RLC_SPM_SE_MUXSEL_DATA
+#define RLC_SPM_SE_MUXSEL_DATA__SEL0__SHIFT                                                                   0x0
+#define RLC_SPM_SE_MUXSEL_DATA__SEL1__SHIFT                                                                   0x10
+#define RLC_SPM_SE_MUXSEL_DATA__SEL0_MASK                                                                     0x0000FFFFL
+#define RLC_SPM_SE_MUXSEL_DATA__SEL1_MASK                                                                     0xFFFF0000L
+//RLC_SPM_SE_USER_DATA_0
+#define RLC_SPM_SE_USER_DATA_0__LO__SHIFT                                                                     0x0
+#define RLC_SPM_SE_USER_DATA_0__HI__SHIFT                                                                     0x10
+#define RLC_SPM_SE_USER_DATA_0__LO_MASK                                                                       0x0000FFFFL
+#define RLC_SPM_SE_USER_DATA_0__HI_MASK                                                                       0xFFFF0000L
+//RLC_SPM_SE_USER_DATA_1
+#define RLC_SPM_SE_USER_DATA_1__LO__SHIFT                                                                     0x0
+#define RLC_SPM_SE_USER_DATA_1__HI__SHIFT                                                                     0x10
+#define RLC_SPM_SE_USER_DATA_1__LO_MASK                                                                       0x0000FFFFL
+#define RLC_SPM_SE_USER_DATA_1__HI_MASK                                                                       0xFFFF0000L
+//RLC_SPM_SE_USER_DATA_2
+#define RLC_SPM_SE_USER_DATA_2__LO__SHIFT                                                                     0x0
+#define RLC_SPM_SE_USER_DATA_2__HI__SHIFT                                                                     0x10
+#define RLC_SPM_SE_USER_DATA_2__LO_MASK                                                                       0x0000FFFFL
+#define RLC_SPM_SE_USER_DATA_2__HI_MASK                                                                       0xFFFF0000L
+//RLC_SPM_SE_USER_DATA_3
+#define RLC_SPM_SE_USER_DATA_3__LO__SHIFT                                                                     0x0
+#define RLC_SPM_SE_USER_DATA_3__HI__SHIFT                                                                     0x10
+#define RLC_SPM_SE_USER_DATA_3__LO_MASK                                                                       0x0000FFFFL
+#define RLC_SPM_SE_USER_DATA_3__HI_MASK                                                                       0xFFFF0000L
+//RLC_SPM_ACCUM_DATARAM_ADDR
+#define RLC_SPM_ACCUM_DATARAM_ADDR__ADDR__SHIFT                                                               0x0
+#define RLC_SPM_ACCUM_DATARAM_ADDR__ADDR_MASK                                                                 0x0000007FL
+//RLC_SPM_ACCUM_DATARAM_DATA
+#define RLC_SPM_ACCUM_DATARAM_DATA__DATA__SHIFT                                                               0x0
+#define RLC_SPM_ACCUM_DATARAM_DATA__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_SPM_ACCUM_SWA_DATARAM_ADDR
+#define RLC_SPM_ACCUM_SWA_DATARAM_ADDR__ADDR__SHIFT                                                           0x0
+#define RLC_SPM_ACCUM_SWA_DATARAM_ADDR__ADDR_MASK                                                             0x0000007FL
+//RLC_SPM_ACCUM_SWA_DATARAM_DATA
+#define RLC_SPM_ACCUM_SWA_DATARAM_DATA__DATA__SHIFT                                                           0x0
+#define RLC_SPM_ACCUM_SWA_DATARAM_DATA__DATA_MASK                                                             0xFFFFFFFFL
+//RLC_SPM_ACCUM_CTRLRAM_ADDR
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR__ADDR__SHIFT                                                               0x0
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR__ADDR_MASK                                                                 0x000007FFL
+//RLC_SPM_ACCUM_CTRLRAM_DATA
+#define RLC_SPM_ACCUM_CTRLRAM_DATA__DATA__SHIFT                                                               0x0
+#define RLC_SPM_ACCUM_CTRLRAM_DATA__DATA_MASK                                                                 0x000000FFL
+//RLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET__global_offset__SHIFT                                               0x0
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET__spmwithaccum_se_offset__SHIFT                                      0x8
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET__spmwithaccum_global_offset__SHIFT                                  0x10
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET__global_offset_MASK                                                 0x000000FFL
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET__spmwithaccum_se_offset_MASK                                        0x0000FF00L
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET__spmwithaccum_global_offset_MASK                                    0x00FF0000L
+//RLC_SPM_ACCUM_STATUS
+#define RLC_SPM_ACCUM_STATUS__NumbSamplesCompleted__SHIFT                                                     0x0
+#define RLC_SPM_ACCUM_STATUS__AccumDone__SHIFT                                                                0x8
+#define RLC_SPM_ACCUM_STATUS__SpmDone__SHIFT                                                                  0x9
+#define RLC_SPM_ACCUM_STATUS__AccumOverflow__SHIFT                                                            0xa
+#define RLC_SPM_ACCUM_STATUS__AccumArmed__SHIFT                                                               0xb
+#define RLC_SPM_ACCUM_STATUS__SequenceInProgress__SHIFT                                                       0xc
+#define RLC_SPM_ACCUM_STATUS__FinalSequenceInProgress__SHIFT                                                  0xd
+#define RLC_SPM_ACCUM_STATUS__AllFifosEmpty__SHIFT                                                            0xe
+#define RLC_SPM_ACCUM_STATUS__FSMIsIdle__SHIFT                                                                0xf
+#define RLC_SPM_ACCUM_STATUS__SwaAccumDone__SHIFT                                                             0x10
+#define RLC_SPM_ACCUM_STATUS__SwaSpmDone__SHIFT                                                               0x11
+#define RLC_SPM_ACCUM_STATUS__SwaAccumOverflow__SHIFT                                                         0x12
+#define RLC_SPM_ACCUM_STATUS__SwaAccumArmed__SHIFT                                                            0x13
+#define RLC_SPM_ACCUM_STATUS__AllSegsDone__SHIFT                                                              0x14
+#define RLC_SPM_ACCUM_STATUS__RearmSwaPending__SHIFT                                                          0x15
+#define RLC_SPM_ACCUM_STATUS__RearmSppPending__SHIFT                                                          0x16
+#define RLC_SPM_ACCUM_STATUS__MultiSampleAborted__SHIFT                                                       0x17
+#define RLC_SPM_ACCUM_STATUS__NumbSamplesCompleted_MASK                                                       0x000000FFL
+#define RLC_SPM_ACCUM_STATUS__AccumDone_MASK                                                                  0x00000100L
+#define RLC_SPM_ACCUM_STATUS__SpmDone_MASK                                                                    0x00000200L
+#define RLC_SPM_ACCUM_STATUS__AccumOverflow_MASK                                                              0x00000400L
+#define RLC_SPM_ACCUM_STATUS__AccumArmed_MASK                                                                 0x00000800L
+#define RLC_SPM_ACCUM_STATUS__SequenceInProgress_MASK                                                         0x00001000L
+#define RLC_SPM_ACCUM_STATUS__FinalSequenceInProgress_MASK                                                    0x00002000L
+#define RLC_SPM_ACCUM_STATUS__AllFifosEmpty_MASK                                                              0x00004000L
+#define RLC_SPM_ACCUM_STATUS__FSMIsIdle_MASK                                                                  0x00008000L
+#define RLC_SPM_ACCUM_STATUS__SwaAccumDone_MASK                                                               0x00010000L
+#define RLC_SPM_ACCUM_STATUS__SwaSpmDone_MASK                                                                 0x00020000L
+#define RLC_SPM_ACCUM_STATUS__SwaAccumOverflow_MASK                                                           0x00040000L
+#define RLC_SPM_ACCUM_STATUS__SwaAccumArmed_MASK                                                              0x00080000L
+#define RLC_SPM_ACCUM_STATUS__AllSegsDone_MASK                                                                0x00100000L
+#define RLC_SPM_ACCUM_STATUS__RearmSwaPending_MASK                                                            0x00200000L
+#define RLC_SPM_ACCUM_STATUS__RearmSppPending_MASK                                                            0x00400000L
+#define RLC_SPM_ACCUM_STATUS__MultiSampleAborted_MASK                                                         0x00800000L
+//RLC_SPM_ACCUM_CTRL
+#define RLC_SPM_ACCUM_CTRL__StrobeResetPerfMonitors__SHIFT                                                    0x0
+#define RLC_SPM_ACCUM_CTRL__StrobeStartAccumulation__SHIFT                                                    0x1
+#define RLC_SPM_ACCUM_CTRL__StrobeRearmAccum__SHIFT                                                           0x2
+#define RLC_SPM_ACCUM_CTRL__StrobeResetSpmBlock__SHIFT                                                        0x3
+#define RLC_SPM_ACCUM_CTRL__StrobeStartSpm__SHIFT                                                             0x4
+#define RLC_SPM_ACCUM_CTRL__StrobeRearmSwaAccum__SHIFT                                                        0x8
+#define RLC_SPM_ACCUM_CTRL__StrobeStartSwa__SHIFT                                                             0x9
+#define RLC_SPM_ACCUM_CTRL__StrobePerfmonSampleWires__SHIFT                                                   0xa
+#define RLC_SPM_ACCUM_CTRL__StrobeResetPerfMonitors_MASK                                                      0x00000001L
+#define RLC_SPM_ACCUM_CTRL__StrobeStartAccumulation_MASK                                                      0x00000002L
+#define RLC_SPM_ACCUM_CTRL__StrobeRearmAccum_MASK                                                             0x00000004L
+#define RLC_SPM_ACCUM_CTRL__StrobeResetSpmBlock_MASK                                                          0x00000008L
+#define RLC_SPM_ACCUM_CTRL__StrobeStartSpm_MASK                                                               0x000000F0L
+#define RLC_SPM_ACCUM_CTRL__StrobeRearmSwaAccum_MASK                                                          0x00000100L
+#define RLC_SPM_ACCUM_CTRL__StrobeStartSwa_MASK                                                               0x00000200L
+#define RLC_SPM_ACCUM_CTRL__StrobePerfmonSampleWires_MASK                                                     0x00000400L
+//RLC_SPM_ACCUM_MODE
+#define RLC_SPM_ACCUM_MODE__EnableAccum__SHIFT                                                                0x0
+#define RLC_SPM_ACCUM_MODE__EnableSpmWithAccumMode__SHIFT                                                     0x1
+#define RLC_SPM_ACCUM_MODE__EnableSPPMode__SHIFT                                                              0x2
+#define RLC_SPM_ACCUM_MODE__AutoResetPerfmonDisable__SHIFT                                                    0x3
+#define RLC_SPM_ACCUM_MODE__RESERVED_4__SHIFT                                                                 0x4
+#define RLC_SPM_ACCUM_MODE__AutoAccumEn__SHIFT                                                                0x5
+#define RLC_SPM_ACCUM_MODE__SwaAutoAccumEn__SHIFT                                                             0x6
+#define RLC_SPM_ACCUM_MODE__AutoSpmEn__SHIFT                                                                  0x7
+#define RLC_SPM_ACCUM_MODE__SwaAutoSpmEn__SHIFT                                                               0x8
+#define RLC_SPM_ACCUM_MODE__Globals_LoadOverride__SHIFT                                                       0x9
+#define RLC_SPM_ACCUM_MODE__Globals_SwaLoadOverride__SHIFT                                                    0xa
+#define RLC_SPM_ACCUM_MODE__SE0_LoadOverride__SHIFT                                                           0xb
+#define RLC_SPM_ACCUM_MODE__SE0_SwaLoadOverride__SHIFT                                                        0xc
+#define RLC_SPM_ACCUM_MODE__SE1_LoadOverride__SHIFT                                                           0xd
+#define RLC_SPM_ACCUM_MODE__SE1_SwaLoadOverride__SHIFT                                                        0xe
+#define RLC_SPM_ACCUM_MODE__RESERVED_16_15__SHIFT                                                             0xf
+#define RLC_SPM_ACCUM_MODE__RESERVED_18_17__SHIFT                                                             0x11
+#define RLC_SPM_ACCUM_MODE__RESERVED_20_19__SHIFT                                                             0x13
+#define RLC_SPM_ACCUM_MODE__RESERVED_22_21__SHIFT                                                             0x15
+#define RLC_SPM_ACCUM_MODE__RESERVED_24_23__SHIFT                                                             0x17
+#define RLC_SPM_ACCUM_MODE__RESERVED_26_25__SHIFT                                                             0x19
+#define RLC_SPM_ACCUM_MODE__RESERVED_31_27__SHIFT                                                             0x1b
+#define RLC_SPM_ACCUM_MODE__EnableAccum_MASK                                                                  0x00000001L
+#define RLC_SPM_ACCUM_MODE__EnableSpmWithAccumMode_MASK                                                       0x00000002L
+#define RLC_SPM_ACCUM_MODE__EnableSPPMode_MASK                                                                0x00000004L
+#define RLC_SPM_ACCUM_MODE__AutoResetPerfmonDisable_MASK                                                      0x00000008L
+#define RLC_SPM_ACCUM_MODE__RESERVED_4_MASK                                                                   0x00000010L
+#define RLC_SPM_ACCUM_MODE__AutoAccumEn_MASK                                                                  0x00000020L
+#define RLC_SPM_ACCUM_MODE__SwaAutoAccumEn_MASK                                                               0x00000040L
+#define RLC_SPM_ACCUM_MODE__AutoSpmEn_MASK                                                                    0x00000080L
+#define RLC_SPM_ACCUM_MODE__SwaAutoSpmEn_MASK                                                                 0x00000100L
+#define RLC_SPM_ACCUM_MODE__Globals_LoadOverride_MASK                                                         0x00000200L
+#define RLC_SPM_ACCUM_MODE__Globals_SwaLoadOverride_MASK                                                      0x00000400L
+#define RLC_SPM_ACCUM_MODE__SE0_LoadOverride_MASK                                                             0x00000800L
+#define RLC_SPM_ACCUM_MODE__SE0_SwaLoadOverride_MASK                                                          0x00001000L
+#define RLC_SPM_ACCUM_MODE__SE1_LoadOverride_MASK                                                             0x00002000L
+#define RLC_SPM_ACCUM_MODE__SE1_SwaLoadOverride_MASK                                                          0x00004000L
+#define RLC_SPM_ACCUM_MODE__RESERVED_16_15_MASK                                                               0x00018000L
+#define RLC_SPM_ACCUM_MODE__RESERVED_18_17_MASK                                                               0x00060000L
+#define RLC_SPM_ACCUM_MODE__RESERVED_20_19_MASK                                                               0x00180000L
+#define RLC_SPM_ACCUM_MODE__RESERVED_22_21_MASK                                                               0x00600000L
+#define RLC_SPM_ACCUM_MODE__RESERVED_24_23_MASK                                                               0x01800000L
+#define RLC_SPM_ACCUM_MODE__RESERVED_26_25_MASK                                                               0x06000000L
+#define RLC_SPM_ACCUM_MODE__RESERVED_31_27_MASK                                                               0xF8000000L
+//RLC_SPM_ACCUM_THRESHOLD
+#define RLC_SPM_ACCUM_THRESHOLD__Threshold__SHIFT                                                             0x0
+#define RLC_SPM_ACCUM_THRESHOLD__Threshold_MASK                                                               0x0000FFFFL
+//RLC_SPM_ACCUM_SAMPLES_REQUESTED
+#define RLC_SPM_ACCUM_SAMPLES_REQUESTED__SamplesRequested__SHIFT                                              0x0
+#define RLC_SPM_ACCUM_SAMPLES_REQUESTED__SamplesRequested_MASK                                                0x000000FFL
+//RLC_SPM_ACCUM_DATARAM_WRCOUNT
+#define RLC_SPM_ACCUM_DATARAM_WRCOUNT__DataRamWrCount__SHIFT                                                  0x0
+#define RLC_SPM_ACCUM_DATARAM_WRCOUNT__DataRamWrCount_MASK                                                    0x0007FFFFL
+//RLC_SPM_ACCUM_DATARAM_32BITCNTRS_REGIONS
+#define RLC_SPM_ACCUM_DATARAM_32BITCNTRS_REGIONS__spp_addr_region__SHIFT                                      0x0
+#define RLC_SPM_ACCUM_DATARAM_32BITCNTRS_REGIONS__swa_addr_region__SHIFT                                      0x8
+#define RLC_SPM_ACCUM_DATARAM_32BITCNTRS_REGIONS__spp_addr_region_MASK                                        0x000000FFL
+#define RLC_SPM_ACCUM_DATARAM_32BITCNTRS_REGIONS__swa_addr_region_MASK                                        0x0000FF00L
+//RLC_SPM_PAUSE
+#define RLC_SPM_PAUSE__PAUSE__SHIFT                                                                           0x0
+#define RLC_SPM_PAUSE__PAUSED__SHIFT                                                                          0x1
+#define RLC_SPM_PAUSE__PAUSE_MASK                                                                             0x00000001L
+#define RLC_SPM_PAUSE__PAUSED_MASK                                                                            0x00000002L
+//RLC_SPM_STATUS
+#define RLC_SPM_STATUS__CTL_BUSY__SHIFT                                                                       0x0
+#define RLC_SPM_STATUS__RSPM_REG_BUSY__SHIFT                                                                  0x1
+#define RLC_SPM_STATUS__SPM_RSPM_BUSY__SHIFT                                                                  0x2
+#define RLC_SPM_STATUS__SPM_RSPM_IO_BUSY__SHIFT                                                               0x3
+#define RLC_SPM_STATUS__SE_RSPM_IO_BUSY__SHIFT                                                                0x4
+#define RLC_SPM_STATUS__ACCUM_BUSY__SHIFT                                                                     0xf
+#define RLC_SPM_STATUS__FSM_MASTER_STATE__SHIFT                                                               0x10
+#define RLC_SPM_STATUS__FSM_MEMORY_STATE__SHIFT                                                               0x14
+#define RLC_SPM_STATUS__CTL_REQ_STATE__SHIFT                                                                  0x18
+#define RLC_SPM_STATUS__CTL_RET_STATE__SHIFT                                                                  0x1a
+#define RLC_SPM_STATUS__CTL_BUSY_MASK                                                                         0x00000001L
+#define RLC_SPM_STATUS__RSPM_REG_BUSY_MASK                                                                    0x00000002L
+#define RLC_SPM_STATUS__SPM_RSPM_BUSY_MASK                                                                    0x00000004L
+#define RLC_SPM_STATUS__SPM_RSPM_IO_BUSY_MASK                                                                 0x00000008L
+#define RLC_SPM_STATUS__SE_RSPM_IO_BUSY_MASK                                                                  0x00000FF0L
+#define RLC_SPM_STATUS__ACCUM_BUSY_MASK                                                                       0x00008000L
+#define RLC_SPM_STATUS__FSM_MASTER_STATE_MASK                                                                 0x000F0000L
+#define RLC_SPM_STATUS__FSM_MEMORY_STATE_MASK                                                                 0x00F00000L
+#define RLC_SPM_STATUS__CTL_REQ_STATE_MASK                                                                    0x03000000L
+#define RLC_SPM_STATUS__CTL_RET_STATE_MASK                                                                    0x04000000L
+//RLC_SPM_GFXCLOCK_LOWCOUNT
+#define RLC_SPM_GFXCLOCK_LOWCOUNT__GFXCLOCK_LOWCOUNT__SHIFT                                                   0x0
+#define RLC_SPM_GFXCLOCK_LOWCOUNT__GFXCLOCK_LOWCOUNT_MASK                                                     0xFFFFFFFFL
+//RLC_SPM_GFXCLOCK_HIGHCOUNT
+#define RLC_SPM_GFXCLOCK_HIGHCOUNT__GFXCLOCK_HIGHCOUNT__SHIFT                                                 0x0
+#define RLC_SPM_GFXCLOCK_HIGHCOUNT__GFXCLOCK_HIGHCOUNT_MASK                                                   0xFFFFFFFFL
+//RLC_SPM_GTS_TRIGGER_VALUE_LO
+#define RLC_SPM_GTS_TRIGGER_VALUE_LO__VALUE_LO__SHIFT                                                         0x0
+#define RLC_SPM_GTS_TRIGGER_VALUE_LO__VALUE_LO_MASK                                                           0xFFFFFFFFL
+//RLC_SPM_GTS_TRIGGER_VALUE_HI
+#define RLC_SPM_GTS_TRIGGER_VALUE_HI__VALUE_HI__SHIFT                                                         0x0
+#define RLC_SPM_GTS_TRIGGER_VALUE_HI__VALUE_HI_MASK                                                           0x00FFFFFFL
+//RLC_SPM_GLOBAL_USER_DATA_0
+#define RLC_SPM_GLOBAL_USER_DATA_0__LO__SHIFT                                                                 0x0
+#define RLC_SPM_GLOBAL_USER_DATA_0__HI__SHIFT                                                                 0x10
+#define RLC_SPM_GLOBAL_USER_DATA_0__LO_MASK                                                                   0x0000FFFFL
+#define RLC_SPM_GLOBAL_USER_DATA_0__HI_MASK                                                                   0xFFFF0000L
+//RLC_SPM_GLOBAL_USER_DATA_1
+#define RLC_SPM_GLOBAL_USER_DATA_1__LO__SHIFT                                                                 0x0
+#define RLC_SPM_GLOBAL_USER_DATA_1__HI__SHIFT                                                                 0x10
+#define RLC_SPM_GLOBAL_USER_DATA_1__LO_MASK                                                                   0x0000FFFFL
+#define RLC_SPM_GLOBAL_USER_DATA_1__HI_MASK                                                                   0xFFFF0000L
+//RLC_SPM_GLOBAL_USER_DATA_2
+#define RLC_SPM_GLOBAL_USER_DATA_2__LO__SHIFT                                                                 0x0
+#define RLC_SPM_GLOBAL_USER_DATA_2__HI__SHIFT                                                                 0x10
+#define RLC_SPM_GLOBAL_USER_DATA_2__LO_MASK                                                                   0x0000FFFFL
+#define RLC_SPM_GLOBAL_USER_DATA_2__HI_MASK                                                                   0xFFFF0000L
+//RLC_SPM_GLOBAL_USER_DATA_3
+#define RLC_SPM_GLOBAL_USER_DATA_3__LO__SHIFT                                                                 0x0
+#define RLC_SPM_GLOBAL_USER_DATA_3__HI__SHIFT                                                                 0x10
+#define RLC_SPM_GLOBAL_USER_DATA_3__LO_MASK                                                                   0x0000FFFFL
+#define RLC_SPM_GLOBAL_USER_DATA_3__HI_MASK                                                                   0xFFFF0000L
+//RLC_SPM_MODE
+#define RLC_SPM_MODE__MODE__SHIFT                                                                             0x0
+#define RLC_SPM_MODE__MODE_MASK                                                                               0x00000001L
+//RLC_SPM_RSPM_REQ_DATA
+#define RLC_SPM_RSPM_REQ_DATA__DATA__SHIFT                                                                    0x0
+#define RLC_SPM_RSPM_REQ_DATA__DATA_MASK                                                                      0x0000000FL
+//RLC_SPM_RSPM_REQ_OP
+#define RLC_SPM_RSPM_REQ_OP__OP__SHIFT                                                                        0x0
+#define RLC_SPM_RSPM_REQ_OP__OP_MASK                                                                          0x0000000FL
+//RLC_SPM_RSPM_RET_DATA
+#define RLC_SPM_RSPM_RET_DATA__DATA__SHIFT                                                                    0x0
+#define RLC_SPM_RSPM_RET_DATA__DATA_MASK                                                                      0xFFFFFFFFL
+//RLC_SPM_RSPM_RET_OP
+#define RLC_SPM_RSPM_RET_OP__OP__SHIFT                                                                        0x0
+#define RLC_SPM_RSPM_RET_OP__VALID__SHIFT                                                                     0x8
+#define RLC_SPM_RSPM_RET_OP__OP_MASK                                                                          0x0000000FL
+#define RLC_SPM_RSPM_RET_OP__VALID_MASK                                                                       0x00000100L
+//RLC_SPM_SE_RSPM_REQ_DATA
+#define RLC_SPM_SE_RSPM_REQ_DATA__DATA__SHIFT                                                                 0x0
+#define RLC_SPM_SE_RSPM_REQ_DATA__DATA_MASK                                                                   0x0000000FL
+//RLC_SPM_SE_RSPM_REQ_OP
+#define RLC_SPM_SE_RSPM_REQ_OP__OP__SHIFT                                                                     0x0
+#define RLC_SPM_SE_RSPM_REQ_OP__OP_MASK                                                                       0x0000000FL
+//RLC_SPM_SE_RSPM_RET_DATA
+#define RLC_SPM_SE_RSPM_RET_DATA__DATA__SHIFT                                                                 0x0
+#define RLC_SPM_SE_RSPM_RET_DATA__DATA_MASK                                                                   0xFFFFFFFFL
+//RLC_SPM_SE_RSPM_RET_OP
+#define RLC_SPM_SE_RSPM_RET_OP__OP__SHIFT                                                                     0x0
+#define RLC_SPM_SE_RSPM_RET_OP__VALID__SHIFT                                                                  0x8
+#define RLC_SPM_SE_RSPM_RET_OP__OP_MASK                                                                       0x0000000FL
+#define RLC_SPM_SE_RSPM_RET_OP__VALID_MASK                                                                    0x00000100L
+//RLC_SPM_RSPM_CMD
+#define RLC_SPM_RSPM_CMD__CMD__SHIFT                                                                          0x0
+#define RLC_SPM_RSPM_CMD__CMD_MASK                                                                            0x0000000FL
+//RLC_SPM_RSPM_CMD_ACK
+#define RLC_SPM_RSPM_CMD_ACK__SE0_ACK__SHIFT                                                                  0x0
+#define RLC_SPM_RSPM_CMD_ACK__SE1_ACK__SHIFT                                                                  0x1
+#define RLC_SPM_RSPM_CMD_ACK__SE2_ACK__SHIFT                                                                  0x2
+#define RLC_SPM_RSPM_CMD_ACK__SE3_ACK__SHIFT                                                                  0x3
+#define RLC_SPM_RSPM_CMD_ACK__SE4_ACK__SHIFT                                                                  0x4
+#define RLC_SPM_RSPM_CMD_ACK__SE5_ACK__SHIFT                                                                  0x5
+#define RLC_SPM_RSPM_CMD_ACK__SE6_ACK__SHIFT                                                                  0x6
+#define RLC_SPM_RSPM_CMD_ACK__SE7_ACK__SHIFT                                                                  0x7
+#define RLC_SPM_RSPM_CMD_ACK__SPM_ACK__SHIFT                                                                  0x8
+#define RLC_SPM_RSPM_CMD_ACK__SE0_ACK_MASK                                                                    0x00000001L
+#define RLC_SPM_RSPM_CMD_ACK__SE1_ACK_MASK                                                                    0x00000002L
+#define RLC_SPM_RSPM_CMD_ACK__SE2_ACK_MASK                                                                    0x00000004L
+#define RLC_SPM_RSPM_CMD_ACK__SE3_ACK_MASK                                                                    0x00000008L
+#define RLC_SPM_RSPM_CMD_ACK__SE4_ACK_MASK                                                                    0x00000010L
+#define RLC_SPM_RSPM_CMD_ACK__SE5_ACK_MASK                                                                    0x00000020L
+#define RLC_SPM_RSPM_CMD_ACK__SE6_ACK_MASK                                                                    0x00000040L
+#define RLC_SPM_RSPM_CMD_ACK__SE7_ACK_MASK                                                                    0x00000080L
+#define RLC_SPM_RSPM_CMD_ACK__SPM_ACK_MASK                                                                    0x00000100L
+//RLC_SPM_SPARE
+#define RLC_SPM_SPARE__SPARE__SHIFT                                                                           0x0
+#define RLC_SPM_SPARE__SPARE_MASK                                                                             0xFFFFFFFFL
+//RLC_PERFMON_CNTL
+#define RLC_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                                0x0
+#define RLC_PERFMON_CNTL__RESERVED_9_3__SHIFT                                                                 0x3
+#define RLC_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT                                                        0xa
+#define RLC_PERFMON_CNTL__RESERVED__SHIFT                                                                     0xb
+#define RLC_PERFMON_CNTL__PERFMON_STATE_MASK                                                                  0x00000007L
+#define RLC_PERFMON_CNTL__RESERVED_9_3_MASK                                                                   0x000003F8L
+#define RLC_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK                                                          0x00000400L
+#define RLC_PERFMON_CNTL__RESERVED_MASK                                                                       0xFFFFF800L
+//RLC_PERFCOUNTER0_SELECT
+#define RLC_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT__SHIFT                                                    0x0
+#define RLC_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT_MASK                                                      0x000000FFL
+//RLC_PERFCOUNTER1_SELECT
+#define RLC_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT__SHIFT                                                    0x0
+#define RLC_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT_MASK                                                      0x000000FFL
+//RLC_GPU_IOV_PERF_CNT_CNTL
+#define RLC_GPU_IOV_PERF_CNT_CNTL__ENABLE__SHIFT                                                              0x0
+#define RLC_GPU_IOV_PERF_CNT_CNTL__MODE_SELECT__SHIFT                                                         0x1
+#define RLC_GPU_IOV_PERF_CNT_CNTL__RESET__SHIFT                                                               0x2
+#define RLC_GPU_IOV_PERF_CNT_CNTL__RESERVED__SHIFT                                                            0x3
+#define RLC_GPU_IOV_PERF_CNT_CNTL__ENABLE_MASK                                                                0x00000001L
+#define RLC_GPU_IOV_PERF_CNT_CNTL__MODE_SELECT_MASK                                                           0x00000002L
+#define RLC_GPU_IOV_PERF_CNT_CNTL__RESET_MASK                                                                 0x00000004L
+#define RLC_GPU_IOV_PERF_CNT_CNTL__RESERVED_MASK                                                              0xFFFFFFF8L
+//RLC_GPU_IOV_PERF_CNT_WR_ADDR
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__VFID__SHIFT                                                             0x0
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__CNT_ID__SHIFT                                                           0x5
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__RESERVED__SHIFT                                                         0x7
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__VFID_MASK                                                               0x0000001FL
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__CNT_ID_MASK                                                             0x00000060L
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__RESERVED_MASK                                                           0xFFFFFF80L
+//RLC_GPU_IOV_PERF_CNT_WR_DATA
+#define RLC_GPU_IOV_PERF_CNT_WR_DATA__DATA__SHIFT                                                             0x0
+#define RLC_GPU_IOV_PERF_CNT_WR_DATA__DATA_MASK                                                               0xFFFFFFFFL
+//RLC_GPU_IOV_PERF_CNT_RD_ADDR
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__VFID__SHIFT                                                             0x0
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__CNT_ID__SHIFT                                                           0x5
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__RESERVED__SHIFT                                                         0x7
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__VFID_MASK                                                               0x0000001FL
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__CNT_ID_MASK                                                             0x00000060L
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__RESERVED_MASK                                                           0xFFFFFF80L
+//RLC_GPU_IOV_PERF_CNT_RD_DATA
+#define RLC_GPU_IOV_PERF_CNT_RD_DATA__DATA__SHIFT                                                             0x0
+#define RLC_GPU_IOV_PERF_CNT_RD_DATA__DATA_MASK                                                               0xFFFFFFFFL
+//GCR_PERFCOUNTER0_SELECT
+#define GCR_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define GCR_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GCR_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GCR_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GCR_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define GCR_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define GCR_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GCR_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GCR_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GCR_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//GCR_PERFCOUNTER0_SELECT1
+#define GCR_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GCR_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GCR_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GCR_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GCR_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GCR_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GCR_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GCR_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//GCR_PERFCOUNTER1_SELECT
+#define GCR_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define GCR_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GCR_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GCR_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GCR_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define GCR_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define GCR_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GCR_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GCR_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GCR_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//GCR_PERFCOUNTER1_SELECT1
+#define GCR_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GCR_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GCR_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GCR_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GCR_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GCR_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GCR_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GCR_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//CHA_PERFCOUNTER0_SELECT
+#define CHA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CHA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define CHA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define CHA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define CHA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define CHA_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CHA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define CHA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define CHA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define CHA_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//CHA_PERFCOUNTER0_SELECT1
+#define CHA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define CHA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define CHA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define CHA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define CHA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define CHA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define CHA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//CHA_PERFCOUNTER1_SELECT
+#define CHA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CHA_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define CHA_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define CHA_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define CHA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define CHA_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CHA_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define CHA_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define CHA_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define CHA_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//CHA_PERFCOUNTER1_SELECT1
+#define CHA_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define CHA_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define CHA_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define CHA_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define CHA_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define CHA_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define CHA_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//CHA_PERFCOUNTER2_SELECT
+#define CHA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CHA_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define CHA_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define CHA_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define CHA_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define CHA_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CHA_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define CHA_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define CHA_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define CHA_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//CHA_PERFCOUNTER2_SELECT1
+#define CHA_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define CHA_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define CHA_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define CHA_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define CHA_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define CHA_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define CHA_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//CHA_PERFCOUNTER3_SELECT
+#define CHA_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CHA_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define CHA_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define CHA_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define CHA_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define CHA_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CHA_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define CHA_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define CHA_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define CHA_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//CHA_PERFCOUNTER3_SELECT1
+#define CHA_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define CHA_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define CHA_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define CHA_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define CHA_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define CHA_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define CHA_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_gdfll_xvmin_xvmin_xvmin_reg_blk
+//XVMIN_XVMIN_WR_DATA
+#define XVMIN_XVMIN_WR_DATA__XVMINDATA__SHIFT                                                                 0x0
+#define XVMIN_XVMIN_WR_DATA__XVMINDATA_MASK                                                                   0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_hypdec
+//RLC_GPU_IOV_VF_ENABLE
+#define RLC_GPU_IOV_VF_ENABLE__VF_ENABLE__SHIFT                                                               0x0
+#define RLC_GPU_IOV_VF_ENABLE__RESERVED__SHIFT                                                                0x1
+#define RLC_GPU_IOV_VF_ENABLE__VF_NUM__SHIFT                                                                  0x10
+#define RLC_GPU_IOV_VF_ENABLE__VF_ENABLE_MASK                                                                 0x00000001L
+#define RLC_GPU_IOV_VF_ENABLE__RESERVED_MASK                                                                  0x0000FFFEL
+#define RLC_GPU_IOV_VF_ENABLE__VF_NUM_MASK                                                                    0xFFFF0000L
+//RLC_FED_DRVR_STATUS
+#define RLC_FED_DRVR_STATUS__PENDING__SHIFT                                                                   0x0
+#define RLC_FED_DRVR_STATUS__PENDING_MASK                                                                     0x00000007L
+//RLC_GPU_IOV_CFG_REG6
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_SIZE__SHIFT                                                               0x0
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_LOCATION__SHIFT                                                           0x7
+#define RLC_GPU_IOV_CFG_REG6__RESERVED__SHIFT                                                                 0x8
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_OFFSET__SHIFT                                                             0xa
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_SIZE_MASK                                                                 0x0000007FL
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_LOCATION_MASK                                                             0x00000080L
+#define RLC_GPU_IOV_CFG_REG6__RESERVED_MASK                                                                   0x00000300L
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_OFFSET_MASK                                                               0xFFFFFC00L
+//RLC_SDMA0_STATUS
+#define RLC_SDMA0_STATUS__STATUS__SHIFT                                                                       0x0
+#define RLC_SDMA0_STATUS__STATUS_MASK                                                                         0xFFFFFFFFL
+//RLC_SDMA1_STATUS
+#define RLC_SDMA1_STATUS__STATUS__SHIFT                                                                       0x0
+#define RLC_SDMA1_STATUS__STATUS_MASK                                                                         0xFFFFFFFFL
+//RLC_SDMA2_STATUS
+#define RLC_SDMA2_STATUS__STATUS__SHIFT                                                                       0x0
+#define RLC_SDMA2_STATUS__STATUS_MASK                                                                         0xFFFFFFFFL
+//RLC_SDMA3_STATUS
+#define RLC_SDMA3_STATUS__STATUS__SHIFT                                                                       0x0
+#define RLC_SDMA3_STATUS__STATUS_MASK                                                                         0xFFFFFFFFL
+//RLC_SDMA0_BUSY_STATUS
+#define RLC_SDMA0_BUSY_STATUS__BUSY_STATUS__SHIFT                                                             0x0
+#define RLC_SDMA0_BUSY_STATUS__BUSY_STATUS_MASK                                                               0xFFFFFFFFL
+//RLC_SDMA1_BUSY_STATUS
+#define RLC_SDMA1_BUSY_STATUS__BUSY_STATUS__SHIFT                                                             0x0
+#define RLC_SDMA1_BUSY_STATUS__BUSY_STATUS_MASK                                                               0xFFFFFFFFL
+//RLC_SDMA2_BUSY_STATUS
+#define RLC_SDMA2_BUSY_STATUS__BUSY_STATUS__SHIFT                                                             0x0
+#define RLC_SDMA2_BUSY_STATUS__BUSY_STATUS_MASK                                                               0xFFFFFFFFL
+//RLC_SDMA3_BUSY_STATUS
+#define RLC_SDMA3_BUSY_STATUS__BUSY_STATUS__SHIFT                                                             0x0
+#define RLC_SDMA3_BUSY_STATUS__BUSY_STATUS_MASK                                                               0xFFFFFFFFL
+//RLC_GPU_IOV_CFG_REG8
+#define RLC_GPU_IOV_CFG_REG8__VM_BUSY_STATUS__SHIFT                                                           0x0
+#define RLC_GPU_IOV_CFG_REG8__VM_BUSY_STATUS_MASK                                                             0xFFFFFFFFL
+//RLC_GPU_IOV_VF_DOORBELL_STATUS
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS__SHIFT                                             0x0
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS__PF_DOORBELL_STATUS__SHIFT                                             0x1f
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_MASK                                               0x7FFFFFFFL
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS__PF_DOORBELL_STATUS_MASK                                               0x80000000L
+//RLC_GPU_IOV_VF_DOORBELL_STATUS_SET
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__VF_DOORBELL_STATUS_SET__SHIFT                                     0x0
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__PF_DOORBELL_STATUS_SET__SHIFT                                     0x1f
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__VF_DOORBELL_STATUS_SET_MASK                                       0x7FFFFFFFL
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__PF_DOORBELL_STATUS_SET_MASK                                       0x80000000L
+//RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__VF_DOORBELL_STATUS_CLR__SHIFT                                     0x0
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__PF_DOORBELL_STATUS_CLR__SHIFT                                     0x1f
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__VF_DOORBELL_STATUS_CLR_MASK                                       0x7FFFFFFFL
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__PF_DOORBELL_STATUS_CLR_MASK                                       0x80000000L
+//RLC_GPU_IOV_VF_MASK
+#define RLC_GPU_IOV_VF_MASK__VF_MASK__SHIFT                                                                   0x0
+#define RLC_GPU_IOV_VF_MASK__RESERVED__SHIFT                                                                  0x1f
+#define RLC_GPU_IOV_VF_MASK__VF_MASK_MASK                                                                     0x7FFFFFFFL
+#define RLC_GPU_IOV_VF_MASK__RESERVED_MASK                                                                    0x80000000L
+//RLC_HYP_SEMAPHORE_0
+#define RLC_HYP_SEMAPHORE_0__CLIENT_ID__SHIFT                                                                 0x0
+#define RLC_HYP_SEMAPHORE_0__CLIENT_ID_MASK                                                                   0x0000001FL
+//RLC_HYP_SEMAPHORE_1
+#define RLC_HYP_SEMAPHORE_1__CLIENT_ID__SHIFT                                                                 0x0
+#define RLC_HYP_SEMAPHORE_1__CLIENT_ID_MASK                                                                   0x0000001FL
+//RLC_BUSY_CLK_CNTL
+#define RLC_BUSY_CLK_CNTL__BUSY_OFF_LATENCY__SHIFT                                                            0x0
+#define RLC_BUSY_CLK_CNTL__RESERVED__SHIFT                                                                    0x6
+#define RLC_BUSY_CLK_CNTL__GRBM_BUSY_OFF_LATENCY__SHIFT                                                       0x8
+#define RLC_BUSY_CLK_CNTL__BUSY_OFF_LATENCY_MASK                                                              0x0000003FL
+#define RLC_BUSY_CLK_CNTL__RESERVED_MASK                                                                      0x000000C0L
+#define RLC_BUSY_CLK_CNTL__GRBM_BUSY_OFF_LATENCY_MASK                                                         0x00003F00L
+//RLC_CLK_CNTL
+#define RLC_CLK_CNTL__RLC_SRM_ICG_OVERRIDE__SHIFT                                                             0x0
+#define RLC_CLK_CNTL__RLC_IMU_ICG_OVERRIDE__SHIFT                                                             0x1
+#define RLC_CLK_CNTL__RLC_SPM_ICG_OVERRIDE__SHIFT                                                             0x2
+#define RLC_CLK_CNTL__RLC_SPM_RSPM_ICG_OVERRIDE__SHIFT                                                        0x3
+#define RLC_CLK_CNTL__RLC_GPM_ICG_OVERRIDE__SHIFT                                                             0x4
+#define RLC_CLK_CNTL__RLC_CMN_ICG_OVERRIDE__SHIFT                                                             0x5
+#define RLC_CLK_CNTL__RLC_TC_ICG_OVERRIDE__SHIFT                                                              0x6
+#define RLC_CLK_CNTL__RLC_REG_ICG_OVERRIDE__SHIFT                                                             0x7
+#define RLC_CLK_CNTL__RLC_SRAM_CLK_GATER_OVERRIDE__SHIFT                                                      0x8
+#define RLC_CLK_CNTL__RESERVED_9__SHIFT                                                                       0x9
+#define RLC_CLK_CNTL__RLC_SPP_ICG_OVERRIDE__SHIFT                                                             0xa
+#define RLC_CLK_CNTL__RESERVED_11__SHIFT                                                                      0xb
+#define RLC_CLK_CNTL__RLC_TC_FGCG_REP_OVERRIDE__SHIFT                                                         0xc
+#define RLC_CLK_CNTL__RLC_DFLL_ICG_OVERRIDE__SHIFT                                                            0xd
+#define RLC_CLK_CNTL__RESERVED_15__SHIFT                                                                      0xf
+#define RLC_CLK_CNTL__RLC_LX6_CORE_ICG_OVERRIDE__SHIFT                                                        0x10
+#define RLC_CLK_CNTL__RLC_LX6_ICG_OVERRIDE__SHIFT                                                             0x11
+#define RLC_CLK_CNTL__RLC_UTCL2_FGCG_OVERRIDE__SHIFT                                                          0x12
+#define RLC_CLK_CNTL__RLC_IH_GASKET_ICG_OVERRIDE__SHIFT                                                       0x13
+#define RLC_CLK_CNTL__RLC_BRIDGE_ICG_OVERRIDE__SHIFT                                                          0x14
+#define RLC_CLK_CNTL__RLC_DMA_ICG_OVERRIDE__SHIFT                                                             0x15
+#define RLC_CLK_CNTL__RESERVED__SHIFT                                                                         0x16
+#define RLC_CLK_CNTL__RLC_SRM_ICG_OVERRIDE_MASK                                                               0x00000001L
+#define RLC_CLK_CNTL__RLC_IMU_ICG_OVERRIDE_MASK                                                               0x00000002L
+#define RLC_CLK_CNTL__RLC_SPM_ICG_OVERRIDE_MASK                                                               0x00000004L
+#define RLC_CLK_CNTL__RLC_SPM_RSPM_ICG_OVERRIDE_MASK                                                          0x00000008L
+#define RLC_CLK_CNTL__RLC_GPM_ICG_OVERRIDE_MASK                                                               0x00000010L
+#define RLC_CLK_CNTL__RLC_CMN_ICG_OVERRIDE_MASK                                                               0x00000020L
+#define RLC_CLK_CNTL__RLC_TC_ICG_OVERRIDE_MASK                                                                0x00000040L
+#define RLC_CLK_CNTL__RLC_REG_ICG_OVERRIDE_MASK                                                               0x00000080L
+#define RLC_CLK_CNTL__RLC_SRAM_CLK_GATER_OVERRIDE_MASK                                                        0x00000100L
+#define RLC_CLK_CNTL__RESERVED_9_MASK                                                                         0x00000200L
+#define RLC_CLK_CNTL__RLC_SPP_ICG_OVERRIDE_MASK                                                               0x00000400L
+#define RLC_CLK_CNTL__RESERVED_11_MASK                                                                        0x00000800L
+#define RLC_CLK_CNTL__RLC_TC_FGCG_REP_OVERRIDE_MASK                                                           0x00001000L
+#define RLC_CLK_CNTL__RLC_DFLL_ICG_OVERRIDE_MASK                                                              0x00002000L
+#define RLC_CLK_CNTL__RESERVED_15_MASK                                                                        0x00008000L
+#define RLC_CLK_CNTL__RLC_LX6_CORE_ICG_OVERRIDE_MASK                                                          0x00010000L
+#define RLC_CLK_CNTL__RLC_LX6_ICG_OVERRIDE_MASK                                                               0x00020000L
+#define RLC_CLK_CNTL__RLC_UTCL2_FGCG_OVERRIDE_MASK                                                            0x00040000L
+#define RLC_CLK_CNTL__RLC_IH_GASKET_ICG_OVERRIDE_MASK                                                         0x00080000L
+#define RLC_CLK_CNTL__RLC_BRIDGE_ICG_OVERRIDE_MASK                                                            0x00100000L
+#define RLC_CLK_CNTL__RLC_DMA_ICG_OVERRIDE_MASK                                                               0x00200000L
+#define RLC_CLK_CNTL__RESERVED_MASK                                                                           0xFFC00000L
+//RLC_GPU_IOV_SCH_BLOCK
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_ID__SHIFT                                                            0x0
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_Ver__SHIFT                                                           0x4
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_Size__SHIFT                                                          0x8
+#define RLC_GPU_IOV_SCH_BLOCK__RESERVED__SHIFT                                                                0x10
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_ID_MASK                                                              0x0000000FL
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_Ver_MASK                                                             0x000000F0L
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_Size_MASK                                                            0x0000FF00L
+#define RLC_GPU_IOV_SCH_BLOCK__RESERVED_MASK                                                                  0xFFFF0000L
+//RLC_GPU_IOV_CFG_REG1
+#define RLC_GPU_IOV_CFG_REG1__CMD_TYPE__SHIFT                                                                 0x0
+#define RLC_GPU_IOV_CFG_REG1__CMD_EXECUTE__SHIFT                                                              0x4
+#define RLC_GPU_IOV_CFG_REG1__CMD_EXECUTE_INTR_EN__SHIFT                                                      0x5
+#define RLC_GPU_IOV_CFG_REG1__RESERVED__SHIFT                                                                 0x6
+#define RLC_GPU_IOV_CFG_REG1__FCN_ID__SHIFT                                                                   0x8
+#define RLC_GPU_IOV_CFG_REG1__PF_VF__SHIFT                                                                    0xf
+#define RLC_GPU_IOV_CFG_REG1__NEXT_FCN_ID__SHIFT                                                              0x10
+#define RLC_GPU_IOV_CFG_REG1__RESERVED1__SHIFT                                                                0x18
+#define RLC_GPU_IOV_CFG_REG1__CMD_TYPE_MASK                                                                   0x0000000FL
+#define RLC_GPU_IOV_CFG_REG1__CMD_EXECUTE_MASK                                                                0x00000010L
+#define RLC_GPU_IOV_CFG_REG1__CMD_EXECUTE_INTR_EN_MASK                                                        0x00000020L
+#define RLC_GPU_IOV_CFG_REG1__RESERVED_MASK                                                                   0x000000C0L
+#define RLC_GPU_IOV_CFG_REG1__FCN_ID_MASK                                                                     0x00007F00L
+#define RLC_GPU_IOV_CFG_REG1__PF_VF_MASK                                                                      0x00008000L
+#define RLC_GPU_IOV_CFG_REG1__NEXT_FCN_ID_MASK                                                                0x00FF0000L
+#define RLC_GPU_IOV_CFG_REG1__RESERVED1_MASK                                                                  0xFF000000L
+//RLC_GPU_IOV_CFG_REG2
+#define RLC_GPU_IOV_CFG_REG2__CMD_STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_CFG_REG2__CMD_STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_VM_BUSY_STATUS
+#define RLC_GPU_IOV_VM_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                     0x0
+#define RLC_GPU_IOV_VM_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                       0xFFFFFFFFL
+//RLC_GPU_IOV_SCH_0
+#define RLC_GPU_IOV_SCH_0__ACTIVE_FUNCTIONS__SHIFT                                                            0x0
+#define RLC_GPU_IOV_SCH_0__ACTIVE_FUNCTIONS_MASK                                                              0xFFFFFFFFL
+//RLC_GPU_IOV_ACTIVE_FCN_ID
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__VF_ID__SHIFT                                                               0x0
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__RESERVED_6_5__SHIFT                                                        0x5
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__PF_VF__SHIFT                                                               0x7
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__ACTIVE_FCN_ID_STATUS__SHIFT                                                0x8
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__RESERVED_31_12__SHIFT                                                      0xc
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__VF_ID_MASK                                                                 0x0000001FL
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__RESERVED_6_5_MASK                                                          0x00000060L
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__PF_VF_MASK                                                                 0x00000080L
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__ACTIVE_FCN_ID_STATUS_MASK                                                  0x00000F00L
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__RESERVED_31_12_MASK                                                        0xFFFFF000L
+//RLC_GPU_IOV_SCH_3
+#define RLC_GPU_IOV_SCH_3__Time_Quanta_Def__SHIFT                                                             0x0
+#define RLC_GPU_IOV_SCH_3__Time_Quanta_Def_MASK                                                               0xFFFFFFFFL
+//RLC_GPU_IOV_SCH_1
+#define RLC_GPU_IOV_SCH_1__DATA__SHIFT                                                                        0x0
+#define RLC_GPU_IOV_SCH_1__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPU_IOV_SCH_2
+#define RLC_GPU_IOV_SCH_2__DATA__SHIFT                                                                        0x0
+#define RLC_GPU_IOV_SCH_2__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_IH_COOKIE
+#define RLC_IH_COOKIE__DATA__SHIFT                                                                            0x0
+#define RLC_IH_COOKIE__DATA_MASK                                                                              0xFFFFFFFFL
+//RLC_IH_COOKIE_CNTL
+#define RLC_IH_COOKIE_CNTL__CREDIT__SHIFT                                                                     0x0
+#define RLC_IH_COOKIE_CNTL__RESET_COUNTER__SHIFT                                                              0x2
+#define RLC_IH_COOKIE_CNTL__CREDIT_MASK                                                                       0x00000003L
+#define RLC_IH_COOKIE_CNTL__RESET_COUNTER_MASK                                                                0x00000004L
+//RLC_HYP_RLCG_UCODE_CHKSUM
+#define RLC_HYP_RLCG_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                        0x0
+#define RLC_HYP_RLCG_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                          0xFFFFFFFFL
+//RLC_GPU_IOV_SMU_RESPONSE
+#define RLC_GPU_IOV_SMU_RESPONSE__RESP__SHIFT                                                                 0x0
+#define RLC_GPU_IOV_SMU_RESPONSE__RESP_MASK                                                                   0xFFFFFFFFL
+//RLC_GPU_IOV_VIRT_RESET_REQ
+#define RLC_GPU_IOV_VIRT_RESET_REQ__VF_FLR__SHIFT                                                             0x0
+#define RLC_GPU_IOV_VIRT_RESET_REQ__SOFT_PF_FLR__SHIFT                                                        0x1f
+#define RLC_GPU_IOV_VIRT_RESET_REQ__VF_FLR_MASK                                                               0x7FFFFFFFL
+#define RLC_GPU_IOV_VIRT_RESET_REQ__SOFT_PF_FLR_MASK                                                          0x80000000L
+//RLC_GPU_IOV_RLC_RESPONSE
+#define RLC_GPU_IOV_RLC_RESPONSE__RESP__SHIFT                                                                 0x0
+#define RLC_GPU_IOV_RLC_RESPONSE__RESP_MASK                                                                   0xFFFFFFFFL
+//RLC_HYP_SEMAPHORE_2
+#define RLC_HYP_SEMAPHORE_2__CLIENT_ID__SHIFT                                                                 0x0
+#define RLC_HYP_SEMAPHORE_2__CLIENT_ID_MASK                                                                   0x0000001FL
+//RLC_HYP_SEMAPHORE_3
+#define RLC_HYP_SEMAPHORE_3__CLIENT_ID__SHIFT                                                                 0x0
+#define RLC_HYP_SEMAPHORE_3__CLIENT_ID_MASK                                                                   0x0000001FL
+//RLC_GPM_UCODE_ADDR
+#define RLC_GPM_UCODE_ADDR__UCODE_ADDR__SHIFT                                                                 0x0
+#define RLC_GPM_UCODE_ADDR__RESERVED__SHIFT                                                                   0xe
+#define RLC_GPM_UCODE_ADDR__UCODE_ADDR_MASK                                                                   0x00003FFFL
+#define RLC_GPM_UCODE_ADDR__RESERVED_MASK                                                                     0xFFFFC000L
+//RLC_GPM_UCODE_DATA
+#define RLC_GPM_UCODE_DATA__UCODE_DATA__SHIFT                                                                 0x0
+#define RLC_GPM_UCODE_DATA__UCODE_DATA_MASK                                                                   0xFFFFFFFFL
+//RLC_GPM_IRAM_ADDR
+#define RLC_GPM_IRAM_ADDR__ADDR__SHIFT                                                                        0x0
+#define RLC_GPM_IRAM_ADDR__ADDR_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_IRAM_DATA
+#define RLC_GPM_IRAM_DATA__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_IRAM_DATA__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_LX6_DRAM_ADDR
+#define RLC_LX6_DRAM_ADDR__ADDR__SHIFT                                                                        0x0
+#define RLC_LX6_DRAM_ADDR__ADDR_MASK                                                                          0x000007FFL
+//RLC_LX6_DRAM_DATA
+#define RLC_LX6_DRAM_DATA__DATA__SHIFT                                                                        0x0
+#define RLC_LX6_DRAM_DATA__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_LX6_IRAM_ADDR
+#define RLC_LX6_IRAM_ADDR__ADDR__SHIFT                                                                        0x0
+#define RLC_LX6_IRAM_ADDR__ADDR_MASK                                                                          0x00000FFFL
+//RLC_LX6_IRAM_DATA
+#define RLC_LX6_IRAM_DATA__DATA__SHIFT                                                                        0x0
+#define RLC_LX6_IRAM_DATA__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_SCRATCH_ADDR
+#define RLC_GPM_SCRATCH_ADDR__ADDR__SHIFT                                                                     0x0
+#define RLC_GPM_SCRATCH_ADDR__ADDR_MASK                                                                       0x0000FFFFL
+//RLC_GPM_SCRATCH_DATA
+#define RLC_GPM_SCRATCH_DATA__DATA__SHIFT                                                                     0x0
+#define RLC_GPM_SCRATCH_DATA__DATA_MASK                                                                       0xFFFFFFFFL
+//RLC_SRM_DRAM_ADDR
+#define RLC_SRM_DRAM_ADDR__ADDR__SHIFT                                                                        0x0
+#define RLC_SRM_DRAM_ADDR__RESERVED__SHIFT                                                                    0xd
+#define RLC_SRM_DRAM_ADDR__ADDR_MASK                                                                          0x00001FFFL
+#define RLC_SRM_DRAM_ADDR__RESERVED_MASK                                                                      0xFFFFE000L
+//RLC_SRM_DRAM_DATA
+#define RLC_SRM_DRAM_DATA__DATA__SHIFT                                                                        0x0
+#define RLC_SRM_DRAM_DATA__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_SRM_ARAM_ADDR
+#define RLC_SRM_ARAM_ADDR__ADDR__SHIFT                                                                        0x0
+#define RLC_SRM_ARAM_ADDR__RESERVED__SHIFT                                                                    0xd
+#define RLC_SRM_ARAM_ADDR__ADDR_MASK                                                                          0x00001FFFL
+#define RLC_SRM_ARAM_ADDR__RESERVED_MASK                                                                      0xFFFFE000L
+//RLC_SRM_ARAM_DATA
+#define RLC_SRM_ARAM_DATA__DATA__SHIFT                                                                        0x0
+#define RLC_SRM_ARAM_DATA__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GTS_OFFSET_LSB
+#define RLC_GTS_OFFSET_LSB__DATA__SHIFT                                                                       0x0
+#define RLC_GTS_OFFSET_LSB__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GTS_OFFSET_MSB
+#define RLC_GTS_OFFSET_MSB__DATA__SHIFT                                                                       0x0
+#define RLC_GTS_OFFSET_MSB__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GTS_OFFSET_SNAP_LSB
+#define RLC_GTS_OFFSET_SNAP_LSB__DATA__SHIFT                                                                  0x0
+#define RLC_GTS_OFFSET_SNAP_LSB__DATA_MASK                                                                    0xFFFFFFFFL
+//RLC_GTS_OFFSET_SNAP_MSB
+#define RLC_GTS_OFFSET_SNAP_MSB__DATA__SHIFT                                                                  0x0
+#define RLC_GTS_OFFSET_SNAP_MSB__DATA_MASK                                                                    0xFFFFFFFFL
+//RLC_GAP_PWROK
+#define RLC_GAP_PWROK__GAP_PWROK__SHIFT                                                                       0x0
+#define RLC_GAP_PWROK__GAP_PWROK_MASK                                                                         0x00000001L
+//RLC_GAP_PWROK_COUNT
+#define RLC_GAP_PWROK_COUNT__COUNT__SHIFT                                                                     0x0
+#define RLC_GAP_PWROK_COUNT__COUNT_MASK                                                                       0x0000000FL
+//GLARB_HASH_CFG
+#define GLARB_HASH_CFG__HPA_HASH_EN__SHIFT                                                                    0x0
+#define GLARB_HASH_CFG__CHANNEL_HASH_EN__SHIFT                                                                0x13
+#define GLARB_HASH_CFG__RESERVED__SHIFT                                                                       0x14
+#define GLARB_HASH_CFG__HPA_HASH_EN_MASK                                                                      0x00000001L
+#define GLARB_HASH_CFG__CHANNEL_HASH_EN_MASK                                                                  0x00080000L
+#define GLARB_HASH_CFG__RESERVED_MASK                                                                         0xFFF00000L
+//CH_PIPE_STEER
+#define CH_PIPE_STEER__PIPE0__SHIFT                                                                           0x0
+#define CH_PIPE_STEER__PIPE1__SHIFT                                                                           0x1
+#define CH_PIPE_STEER__MODE__SHIFT                                                                            0x2
+#define CH_PIPE_STEER__PIPE0_MASK                                                                             0x00000001L
+#define CH_PIPE_STEER__PIPE1_MASK                                                                             0x00000002L
+#define CH_PIPE_STEER__MODE_MASK                                                                              0x00000004L
+//GLARB_PIPE_STEER_LSB
+#define GLARB_PIPE_STEER_LSB__PIPE0__SHIFT                                                                    0x0
+#define GLARB_PIPE_STEER_LSB__PIPE1__SHIFT                                                                    0x4
+#define GLARB_PIPE_STEER_LSB__PIPE2__SHIFT                                                                    0x8
+#define GLARB_PIPE_STEER_LSB__PIPE3__SHIFT                                                                    0xc
+#define GLARB_PIPE_STEER_LSB__PIPE4__SHIFT                                                                    0x10
+#define GLARB_PIPE_STEER_LSB__PIPE5__SHIFT                                                                    0x14
+#define GLARB_PIPE_STEER_LSB__PIPE6__SHIFT                                                                    0x18
+#define GLARB_PIPE_STEER_LSB__PIPE7__SHIFT                                                                    0x1c
+#define GLARB_PIPE_STEER_LSB__PIPE0_MASK                                                                      0x0000000FL
+#define GLARB_PIPE_STEER_LSB__PIPE1_MASK                                                                      0x000000F0L
+#define GLARB_PIPE_STEER_LSB__PIPE2_MASK                                                                      0x00000F00L
+#define GLARB_PIPE_STEER_LSB__PIPE3_MASK                                                                      0x0000F000L
+#define GLARB_PIPE_STEER_LSB__PIPE4_MASK                                                                      0x000F0000L
+#define GLARB_PIPE_STEER_LSB__PIPE5_MASK                                                                      0x00F00000L
+#define GLARB_PIPE_STEER_LSB__PIPE6_MASK                                                                      0x0F000000L
+#define GLARB_PIPE_STEER_LSB__PIPE7_MASK                                                                      0xF0000000L
+//GLARB_PIPE_STEER_MSB
+#define GLARB_PIPE_STEER_MSB__PIPE8__SHIFT                                                                    0x0
+#define GLARB_PIPE_STEER_MSB__PIPE9__SHIFT                                                                    0x4
+#define GLARB_PIPE_STEER_MSB__PIPE10__SHIFT                                                                   0x8
+#define GLARB_PIPE_STEER_MSB__PIPE11__SHIFT                                                                   0xc
+#define GLARB_PIPE_STEER_MSB__MODE__SHIFT                                                                     0x10
+#define GLARB_PIPE_STEER_MSB__PIPE8_MASK                                                                      0x0000000FL
+#define GLARB_PIPE_STEER_MSB__PIPE9_MASK                                                                      0x000000F0L
+#define GLARB_PIPE_STEER_MSB__PIPE10_MASK                                                                     0x00000F00L
+#define GLARB_PIPE_STEER_MSB__PIPE11_MASK                                                                     0x0000F000L
+#define GLARB_PIPE_STEER_MSB__MODE_MASK                                                                       0x00010000L
+//CH_HASH_CFG
+#define CH_HASH_CFG__CHANNEL_FLIP__SHIFT                                                                      0x12
+#define CH_HASH_CFG__CHANNEL_HASH_EN__SHIFT                                                                   0x13
+#define CH_HASH_CFG__RESERVED__SHIFT                                                                          0x14
+#define CH_HASH_CFG__CHANNEL_FLIP_MASK                                                                        0x00040000L
+#define CH_HASH_CFG__CHANNEL_HASH_EN_MASK                                                                     0x00080000L
+#define CH_HASH_CFG__RESERVED_MASK                                                                            0xFFF00000L
+//GLARB_LPDDR_PIPE_STEER_LSB
+#define GLARB_LPDDR_PIPE_STEER_LSB__PIPE0__SHIFT                                                              0x0
+#define GLARB_LPDDR_PIPE_STEER_LSB__PIPE1__SHIFT                                                              0x4
+#define GLARB_LPDDR_PIPE_STEER_LSB__PIPE2__SHIFT                                                              0x8
+#define GLARB_LPDDR_PIPE_STEER_LSB__PIPE3__SHIFT                                                              0xc
+#define GLARB_LPDDR_PIPE_STEER_LSB__PIPE4__SHIFT                                                              0x10
+#define GLARB_LPDDR_PIPE_STEER_LSB__PIPE5__SHIFT                                                              0x14
+#define GLARB_LPDDR_PIPE_STEER_LSB__PIPE6__SHIFT                                                              0x18
+#define GLARB_LPDDR_PIPE_STEER_LSB__PIPE7__SHIFT                                                              0x1c
+#define GLARB_LPDDR_PIPE_STEER_LSB__PIPE0_MASK                                                                0x0000000FL
+#define GLARB_LPDDR_PIPE_STEER_LSB__PIPE1_MASK                                                                0x000000F0L
+#define GLARB_LPDDR_PIPE_STEER_LSB__PIPE2_MASK                                                                0x00000F00L
+#define GLARB_LPDDR_PIPE_STEER_LSB__PIPE3_MASK                                                                0x0000F000L
+#define GLARB_LPDDR_PIPE_STEER_LSB__PIPE4_MASK                                                                0x000F0000L
+#define GLARB_LPDDR_PIPE_STEER_LSB__PIPE5_MASK                                                                0x00F00000L
+#define GLARB_LPDDR_PIPE_STEER_LSB__PIPE6_MASK                                                                0x0F000000L
+#define GLARB_LPDDR_PIPE_STEER_LSB__PIPE7_MASK                                                                0xF0000000L
+//GLARB_LPDDR_PIPE_STEER_MSB
+#define GLARB_LPDDR_PIPE_STEER_MSB__PIPE8__SHIFT                                                              0x0
+#define GLARB_LPDDR_PIPE_STEER_MSB__PIPE9__SHIFT                                                              0x4
+#define GLARB_LPDDR_PIPE_STEER_MSB__PIPE10__SHIFT                                                             0x8
+#define GLARB_LPDDR_PIPE_STEER_MSB__PIPE11__SHIFT                                                             0xc
+#define GLARB_LPDDR_PIPE_STEER_MSB__PIPE8_MASK                                                                0x0000000FL
+#define GLARB_LPDDR_PIPE_STEER_MSB__PIPE9_MASK                                                                0x000000F0L
+#define GLARB_LPDDR_PIPE_STEER_MSB__PIPE10_MASK                                                               0x00000F00L
+#define GLARB_LPDDR_PIPE_STEER_MSB__PIPE11_MASK                                                               0x0000F000L
+//GLARB_LOCAL_CAC
+#define GLARB_LOCAL_CAC__THRESHOLD__SHIFT                                                                     0x0
+#define GLARB_LOCAL_CAC__OVERRIDE_EN__SHIFT                                                                   0x10
+#define GLARB_LOCAL_CAC__OVERRIDE_VAL__SHIFT                                                                  0x11
+#define GLARB_LOCAL_CAC__THRESHOLD_MASK                                                                       0x0000FFFFL
+#define GLARB_LOCAL_CAC__OVERRIDE_EN_MASK                                                                     0x00010000L
+#define GLARB_LOCAL_CAC__OVERRIDE_VAL_MASK                                                                    0x1FFE0000L
+//GLARB_AID_SEL_CFG
+#define GLARB_AID_SEL_CFG__AS_SEL__SHIFT                                                                      0x0
+#define GLARB_AID_SEL_CFG__RESERVED__SHIFT                                                                    0x5
+#define GLARB_AID_SEL_CFG__AS_SEL_MASK                                                                        0x0000001FL
+#define GLARB_AID_SEL_CFG__RESERVED_MASK                                                                      0xFFFFFFE0L
+//GC_USER_FULL_SA_UNIT_DISABLE
+#define GC_USER_FULL_SA_UNIT_DISABLE__SA_DISABLE__SHIFT                                                       0x8
+#define GC_USER_FULL_SA_UNIT_DISABLE__SA_DISABLE_MASK                                                         0x03FFFF00L
+//GC_USER_SA_UNIT_DISABLE
+#define GC_USER_SA_UNIT_DISABLE__SA_DISABLE__SHIFT                                                            0x8
+#define GC_USER_SA_UNIT_DISABLE__SA_DISABLE_MASK                                                              0x00FFFF00L
+//GC_USER_HBM_DISABLE
+#define GC_USER_HBM_DISABLE__GC_USER_HBM_DISABLE__SHIFT                                                       0x10
+#define GC_USER_HBM_DISABLE__GC_USER_HBM_DISABLE_MASK                                                         0x0FFF0000L
+//RLC_GPU_IOV_SDMA0_STATUS
+#define RLC_GPU_IOV_SDMA0_STATUS__STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_SDMA0_STATUS__STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA1_STATUS
+#define RLC_GPU_IOV_SDMA1_STATUS__STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_SDMA1_STATUS__STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA2_STATUS
+#define RLC_GPU_IOV_SDMA2_STATUS__STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_SDMA2_STATUS__STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA3_STATUS
+#define RLC_GPU_IOV_SDMA3_STATUS__STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_SDMA3_STATUS__STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA4_STATUS
+#define RLC_GPU_IOV_SDMA4_STATUS__STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_SDMA4_STATUS__STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA5_STATUS
+#define RLC_GPU_IOV_SDMA5_STATUS__STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_SDMA5_STATUS__STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA6_STATUS
+#define RLC_GPU_IOV_SDMA6_STATUS__STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_SDMA6_STATUS__STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA7_STATUS
+#define RLC_GPU_IOV_SDMA7_STATUS__STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_SDMA7_STATUS__STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA0_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA0_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA0_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA1_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA1_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA1_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA2_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA2_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA2_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA3_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA3_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA3_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA4_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA4_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA4_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA5_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA5_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA5_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA6_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA6_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA6_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA7_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA7_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA7_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_cphypdec
+//CP_HYP_CONTEXT_RANGE_BASE
+#define CP_HYP_CONTEXT_RANGE_BASE__BASE__SHIFT                                                                0x0
+#define CP_HYP_CONTEXT_RANGE_BASE__BASE_MASK                                                                  0x0003FFFFL
+//CP_HYP_CONTEXT_RANGE_END
+#define CP_HYP_CONTEXT_RANGE_END__END__SHIFT                                                                  0x0
+#define CP_HYP_CONTEXT_RANGE_END__END_MASK                                                                    0x0003FFFFL
+//CP_HYP_PFP_UCODE_ADDR
+#define CP_HYP_PFP_UCODE_ADDR__UCODE_ADDR__SHIFT                                                              0x0
+#define CP_HYP_PFP_UCODE_ADDR__PIPE_SEL__SHIFT                                                                0x1f
+#define CP_HYP_PFP_UCODE_ADDR__UCODE_ADDR_MASK                                                                0x000000FFL
+#define CP_HYP_PFP_UCODE_ADDR__PIPE_SEL_MASK                                                                  0x80000000L
+//CP_PFP_UCODE_ADDR
+#define CP_PFP_UCODE_ADDR__UCODE_ADDR__SHIFT                                                                  0x0
+#define CP_PFP_UCODE_ADDR__PIPE_SEL__SHIFT                                                                    0x1f
+#define CP_PFP_UCODE_ADDR__UCODE_ADDR_MASK                                                                    0x000000FFL
+#define CP_PFP_UCODE_ADDR__PIPE_SEL_MASK                                                                      0x80000000L
+//CP_HYP_PFP_UCODE_DATA
+#define CP_HYP_PFP_UCODE_DATA__UCODE_DATA__SHIFT                                                              0x0
+#define CP_HYP_PFP_UCODE_DATA__UCODE_DATA_MASK                                                                0xFFFFFFFFL
+//CP_PFP_UCODE_DATA
+#define CP_PFP_UCODE_DATA__UCODE_DATA__SHIFT                                                                  0x0
+#define CP_PFP_UCODE_DATA__UCODE_DATA_MASK                                                                    0xFFFFFFFFL
+//CP_HYP_ME_UCODE_ADDR
+#define CP_HYP_ME_UCODE_ADDR__UCODE_ADDR__SHIFT                                                               0x0
+#define CP_HYP_ME_UCODE_ADDR__PIPE_SEL__SHIFT                                                                 0x1f
+#define CP_HYP_ME_UCODE_ADDR__UCODE_ADDR_MASK                                                                 0x000000FFL
+#define CP_HYP_ME_UCODE_ADDR__PIPE_SEL_MASK                                                                   0x80000000L
+//CP_ME_RAM_RADDR
+#define CP_ME_RAM_RADDR__ME_RAM_RADDR__SHIFT                                                                  0x0
+#define CP_ME_RAM_RADDR__PIPE_SEL__SHIFT                                                                      0x1f
+#define CP_ME_RAM_RADDR__ME_RAM_RADDR_MASK                                                                    0x000000FFL
+#define CP_ME_RAM_RADDR__PIPE_SEL_MASK                                                                        0x80000000L
+//CP_ME_RAM_WADDR
+#define CP_ME_RAM_WADDR__ME_RAM_WADDR__SHIFT                                                                  0x0
+#define CP_ME_RAM_WADDR__PIPE_SEL__SHIFT                                                                      0x1f
+#define CP_ME_RAM_WADDR__ME_RAM_WADDR_MASK                                                                    0x000000FFL
+#define CP_ME_RAM_WADDR__PIPE_SEL_MASK                                                                        0x80000000L
+//CP_HYP_ME_UCODE_DATA
+#define CP_HYP_ME_UCODE_DATA__UCODE_DATA__SHIFT                                                               0x0
+#define CP_HYP_ME_UCODE_DATA__UCODE_DATA_MASK                                                                 0xFFFFFFFFL
+//CP_ME_RAM_DATA
+#define CP_ME_RAM_DATA__ME_RAM_DATA__SHIFT                                                                    0x0
+#define CP_ME_RAM_DATA__ME_RAM_DATA_MASK                                                                      0xFFFFFFFFL
+//CP_HYP_MEC1_UCODE_ADDR
+#define CP_HYP_MEC1_UCODE_ADDR__UCODE_ADDR__SHIFT                                                             0x0
+#define CP_HYP_MEC1_UCODE_ADDR__PIPE_SEL__SHIFT                                                               0x1e
+#define CP_HYP_MEC1_UCODE_ADDR__UCODE_ADDR_MASK                                                               0x000000FFL
+#define CP_HYP_MEC1_UCODE_ADDR__PIPE_SEL_MASK                                                                 0xC0000000L
+//CP_MEC_ME1_UCODE_ADDR
+#define CP_MEC_ME1_UCODE_ADDR__UCODE_ADDR__SHIFT                                                              0x0
+#define CP_MEC_ME1_UCODE_ADDR__PIPE_SEL__SHIFT                                                                0x1e
+#define CP_MEC_ME1_UCODE_ADDR__UCODE_ADDR_MASK                                                                0x000000FFL
+#define CP_MEC_ME1_UCODE_ADDR__PIPE_SEL_MASK                                                                  0xC0000000L
+//CP_HYP_MEC1_UCODE_DATA
+#define CP_HYP_MEC1_UCODE_DATA__UCODE_DATA__SHIFT                                                             0x0
+#define CP_HYP_MEC1_UCODE_DATA__UCODE_DATA_MASK                                                               0xFFFFFFFFL
+//CP_MEC_ME1_UCODE_DATA
+#define CP_MEC_ME1_UCODE_DATA__UCODE_DATA__SHIFT                                                              0x0
+#define CP_MEC_ME1_UCODE_DATA__UCODE_DATA_MASK                                                                0xFFFFFFFFL
+//CP_HYP_PFP_UCODE_CHKSUM
+#define CP_HYP_PFP_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                          0x0
+#define CP_HYP_PFP_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                            0xFFFFFFFFL
+//CP_HYP_ME_UCODE_CHKSUM
+#define CP_HYP_ME_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                           0x0
+#define CP_HYP_ME_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                             0xFFFFFFFFL
+//CP_HYP_MEC_ME1_UCODE_CHKSUM
+#define CP_HYP_MEC_ME1_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                      0x0
+#define CP_HYP_MEC_ME1_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                        0xFFFFFFFFL
+//CP_PFP_IC_BASE_LO
+#define CP_PFP_IC_BASE_LO__IC_BASE_LO__SHIFT                                                                  0xc
+#define CP_PFP_IC_BASE_LO__IC_BASE_LO_MASK                                                                    0xFFFFF000L
+//CP_PFP_IC_BASE_HI
+#define CP_PFP_IC_BASE_HI__IC_BASE_HI__SHIFT                                                                  0x0
+#define CP_PFP_IC_BASE_HI__IC_BASE_HI_MASK                                                                    0x0000FFFFL
+//CP_PFP_IC_BASE_CNTL
+#define CP_PFP_IC_BASE_CNTL__VMID__SHIFT                                                                      0x0
+#define CP_PFP_IC_BASE_CNTL__EXE_DISABLE__SHIFT                                                               0x17
+#define CP_PFP_IC_BASE_CNTL__CACHE_POLICY__SHIFT                                                              0x18
+#define CP_PFP_IC_BASE_CNTL__SCOPE__SHIFT                                                                     0x1a
+#define CP_PFP_IC_BASE_CNTL__VMID_MASK                                                                        0x0000000FL
+#define CP_PFP_IC_BASE_CNTL__EXE_DISABLE_MASK                                                                 0x00800000L
+#define CP_PFP_IC_BASE_CNTL__CACHE_POLICY_MASK                                                                0x03000000L
+#define CP_PFP_IC_BASE_CNTL__SCOPE_MASK                                                                       0x0C000000L
+//CP_PFP_IC_OP_CNTL
+#define CP_PFP_IC_OP_CNTL__INVALIDATE_CACHE__SHIFT                                                            0x0
+#define CP_PFP_IC_OP_CNTL__INVALIDATE_CACHE_COMPLETE__SHIFT                                                   0x1
+#define CP_PFP_IC_OP_CNTL__PRIME_START_PC__SHIFT                                                              0x3
+#define CP_PFP_IC_OP_CNTL__PRIME_ICACHE__SHIFT                                                                0x4
+#define CP_PFP_IC_OP_CNTL__ICACHE_PRIMED__SHIFT                                                               0x5
+#define CP_PFP_IC_OP_CNTL__INVALIDATE_CACHE_MASK                                                              0x00000001L
+#define CP_PFP_IC_OP_CNTL__INVALIDATE_CACHE_COMPLETE_MASK                                                     0x00000002L
+#define CP_PFP_IC_OP_CNTL__PRIME_START_PC_MASK                                                                0x00000008L
+#define CP_PFP_IC_OP_CNTL__PRIME_ICACHE_MASK                                                                  0x00000010L
+#define CP_PFP_IC_OP_CNTL__ICACHE_PRIMED_MASK                                                                 0x00000020L
+//CP_ME_IC_BASE_LO
+#define CP_ME_IC_BASE_LO__IC_BASE_LO__SHIFT                                                                   0xc
+#define CP_ME_IC_BASE_LO__IC_BASE_LO_MASK                                                                     0xFFFFF000L
+//CP_ME_IC_BASE_HI
+#define CP_ME_IC_BASE_HI__IC_BASE_HI__SHIFT                                                                   0x0
+#define CP_ME_IC_BASE_HI__IC_BASE_HI_MASK                                                                     0x0000FFFFL
+//CP_ME_IC_BASE_CNTL
+#define CP_ME_IC_BASE_CNTL__VMID__SHIFT                                                                       0x0
+#define CP_ME_IC_BASE_CNTL__EXE_DISABLE__SHIFT                                                                0x17
+#define CP_ME_IC_BASE_CNTL__CACHE_POLICY__SHIFT                                                               0x18
+#define CP_ME_IC_BASE_CNTL__SCOPE__SHIFT                                                                      0x1a
+#define CP_ME_IC_BASE_CNTL__VMID_MASK                                                                         0x0000000FL
+#define CP_ME_IC_BASE_CNTL__EXE_DISABLE_MASK                                                                  0x00800000L
+#define CP_ME_IC_BASE_CNTL__CACHE_POLICY_MASK                                                                 0x03000000L
+#define CP_ME_IC_BASE_CNTL__SCOPE_MASK                                                                        0x0C000000L
+//CP_ME_IC_OP_CNTL
+#define CP_ME_IC_OP_CNTL__INVALIDATE_CACHE__SHIFT                                                             0x0
+#define CP_ME_IC_OP_CNTL__INVALIDATE_CACHE_COMPLETE__SHIFT                                                    0x1
+#define CP_ME_IC_OP_CNTL__PRIME_START_PC__SHIFT                                                               0x3
+#define CP_ME_IC_OP_CNTL__PRIME_ICACHE__SHIFT                                                                 0x4
+#define CP_ME_IC_OP_CNTL__ICACHE_PRIMED__SHIFT                                                                0x5
+#define CP_ME_IC_OP_CNTL__INVALIDATE_CACHE_MASK                                                               0x00000001L
+#define CP_ME_IC_OP_CNTL__INVALIDATE_CACHE_COMPLETE_MASK                                                      0x00000002L
+#define CP_ME_IC_OP_CNTL__PRIME_START_PC_MASK                                                                 0x00000008L
+#define CP_ME_IC_OP_CNTL__PRIME_ICACHE_MASK                                                                   0x00000010L
+#define CP_ME_IC_OP_CNTL__ICACHE_PRIMED_MASK                                                                  0x00000020L
+//CP_CPC_IC_BASE_LO
+#define CP_CPC_IC_BASE_LO__IC_BASE_LO__SHIFT                                                                  0xc
+#define CP_CPC_IC_BASE_LO__IC_BASE_LO_MASK                                                                    0xFFFFF000L
+//CP_CPC_IC_BASE_HI
+#define CP_CPC_IC_BASE_HI__IC_BASE_HI__SHIFT                                                                  0x0
+#define CP_CPC_IC_BASE_HI__IC_BASE_HI_MASK                                                                    0x01FFFFFFL
+//CP_CPC_IC_BASE_CNTL
+#define CP_CPC_IC_BASE_CNTL__VMID__SHIFT                                                                      0x0
+#define CP_CPC_IC_BASE_CNTL__PER_PIPE__SHIFT                                                                  0x5
+#define CP_CPC_IC_BASE_CNTL__SCOPE__SHIFT                                                                     0x6
+#define CP_CPC_IC_BASE_CNTL__TEMPORAL__SHIFT                                                                  0x8
+#define CP_CPC_IC_BASE_CNTL__EXE_DISABLE__SHIFT                                                               0x17
+#define CP_CPC_IC_BASE_CNTL__CACHE_POLICY__SHIFT                                                              0x18
+#define CP_CPC_IC_BASE_CNTL__VMID_MASK                                                                        0x0000000FL
+#define CP_CPC_IC_BASE_CNTL__PER_PIPE_MASK                                                                    0x00000020L
+#define CP_CPC_IC_BASE_CNTL__SCOPE_MASK                                                                       0x000000C0L
+#define CP_CPC_IC_BASE_CNTL__TEMPORAL_MASK                                                                    0x00000700L
+#define CP_CPC_IC_BASE_CNTL__EXE_DISABLE_MASK                                                                 0x00800000L
+#define CP_CPC_IC_BASE_CNTL__CACHE_POLICY_MASK                                                                0x03000000L
+//CP_MES_IC_BASE_LO
+#define CP_MES_IC_BASE_LO__IC_BASE_LO__SHIFT                                                                  0xc
+#define CP_MES_IC_BASE_LO__IC_BASE_LO_MASK                                                                    0xFFFFF000L
+//CP_MES_MIBASE_LO
+#define CP_MES_MIBASE_LO__IC_BASE_LO__SHIFT                                                                   0xc
+#define CP_MES_MIBASE_LO__IC_BASE_LO_MASK                                                                     0xFFFFF000L
+//CP_MES_IC_BASE_HI
+#define CP_MES_IC_BASE_HI__IC_BASE_HI__SHIFT                                                                  0x0
+#define CP_MES_IC_BASE_HI__IC_BASE_HI_MASK                                                                    0x01FFFFFFL
+//CP_MES_MIBASE_HI
+#define CP_MES_MIBASE_HI__IC_BASE_HI__SHIFT                                                                   0x0
+#define CP_MES_MIBASE_HI__IC_BASE_HI_MASK                                                                     0x0000FFFFL
+//CP_MES_IC_BASE_CNTL
+#define CP_MES_IC_BASE_CNTL__VMID__SHIFT                                                                      0x0
+#define CP_MES_IC_BASE_CNTL__EXE_DISABLE__SHIFT                                                               0x17
+#define CP_MES_IC_BASE_CNTL__CACHE_POLICY__SHIFT                                                              0x18
+#define CP_MES_IC_BASE_CNTL__SCOPE__SHIFT                                                                     0x1a
+#define CP_MES_IC_BASE_CNTL__VMID_MASK                                                                        0x0000000FL
+#define CP_MES_IC_BASE_CNTL__EXE_DISABLE_MASK                                                                 0x00800000L
+#define CP_MES_IC_BASE_CNTL__CACHE_POLICY_MASK                                                                0x03000000L
+#define CP_MES_IC_BASE_CNTL__SCOPE_MASK                                                                       0x0C000000L
+//CP_MES_DC_BASE_LO
+#define CP_MES_DC_BASE_LO__DC_BASE_LO__SHIFT                                                                  0x10
+#define CP_MES_DC_BASE_LO__DC_BASE_LO_MASK                                                                    0xFFFF0000L
+//CP_MES_MDBASE_LO
+#define CP_MES_MDBASE_LO__BASE_LO__SHIFT                                                                      0x10
+#define CP_MES_MDBASE_LO__BASE_LO_MASK                                                                        0xFFFF0000L
+//CP_MES_DC_BASE_HI
+#define CP_MES_DC_BASE_HI__DC_BASE_HI__SHIFT                                                                  0x0
+#define CP_MES_DC_BASE_HI__DC_BASE_HI_MASK                                                                    0x01FFFFFFL
+//CP_MES_MDBASE_HI
+#define CP_MES_MDBASE_HI__BASE_HI__SHIFT                                                                      0x0
+#define CP_MES_MDBASE_HI__BASE_HI_MASK                                                                        0x01FFFFFFL
+//CP_MES_MIBOUND_LO
+#define CP_MES_MIBOUND_LO__BOUND_LO__SHIFT                                                                    0x0
+#define CP_MES_MIBOUND_LO__BOUND_LO_MASK                                                                      0xFFFFFFFFL
+//CP_MES_MIBOUND_HI
+#define CP_MES_MIBOUND_HI__BOUND_HI__SHIFT                                                                    0x0
+#define CP_MES_MIBOUND_HI__BOUND_HI_MASK                                                                      0xFFFFFFFFL
+//CP_MES_MDBOUND_LO
+#define CP_MES_MDBOUND_LO__BOUND_LO__SHIFT                                                                    0x0
+#define CP_MES_MDBOUND_LO__BOUND_LO_MASK                                                                      0xFFFFFFFFL
+//CP_MES_MDBOUND_HI
+#define CP_MES_MDBOUND_HI__BOUND_HI__SHIFT                                                                    0x0
+#define CP_MES_MDBOUND_HI__BOUND_HI_MASK                                                                      0xFFFFFFFFL
+//CP_HYP_PFP_UCODE_VERS
+#define CP_HYP_PFP_UCODE_VERS__ENGINE__SHIFT                                                                  0x0
+#define CP_HYP_PFP_UCODE_VERS__COMMON__SHIFT                                                                  0xa
+#define CP_HYP_PFP_UCODE_VERS__HEADER__SHIFT                                                                  0x14
+#define CP_HYP_PFP_UCODE_VERS__STEP__SHIFT                                                                    0x1e
+#define CP_HYP_PFP_UCODE_VERS__ENGINE_MASK                                                                    0x000003FFL
+#define CP_HYP_PFP_UCODE_VERS__COMMON_MASK                                                                    0x000FFC00L
+#define CP_HYP_PFP_UCODE_VERS__HEADER_MASK                                                                    0x3FF00000L
+#define CP_HYP_PFP_UCODE_VERS__STEP_MASK                                                                      0xC0000000L
+//CP_HYP_ME_UCODE_VERS
+#define CP_HYP_ME_UCODE_VERS__ENGINE__SHIFT                                                                   0x0
+#define CP_HYP_ME_UCODE_VERS__COMMON__SHIFT                                                                   0xa
+#define CP_HYP_ME_UCODE_VERS__HEADER__SHIFT                                                                   0x14
+#define CP_HYP_ME_UCODE_VERS__STEP__SHIFT                                                                     0x1e
+#define CP_HYP_ME_UCODE_VERS__ENGINE_MASK                                                                     0x000003FFL
+#define CP_HYP_ME_UCODE_VERS__COMMON_MASK                                                                     0x000FFC00L
+#define CP_HYP_ME_UCODE_VERS__HEADER_MASK                                                                     0x3FF00000L
+#define CP_HYP_ME_UCODE_VERS__STEP_MASK                                                                       0xC0000000L
+//CP_GFX_RS64_DC_BASE0_LO
+#define CP_GFX_RS64_DC_BASE0_LO__DC_BASE_LO__SHIFT                                                            0x10
+#define CP_GFX_RS64_DC_BASE0_LO__DC_BASE_LO_MASK                                                              0xFFFF0000L
+//CP_GFX_RS64_DC_BASE1_LO
+#define CP_GFX_RS64_DC_BASE1_LO__DC_BASE_LO__SHIFT                                                            0x10
+#define CP_GFX_RS64_DC_BASE1_LO__DC_BASE_LO_MASK                                                              0xFFFF0000L
+//CP_GFX_RS64_DC_BASE0_HI
+#define CP_GFX_RS64_DC_BASE0_HI__DC_BASE_HI__SHIFT                                                            0x0
+#define CP_GFX_RS64_DC_BASE0_HI__DC_BASE_HI_MASK                                                              0x0000FFFFL
+//CP_GFX_RS64_DC_BASE1_HI
+#define CP_GFX_RS64_DC_BASE1_HI__DC_BASE_HI__SHIFT                                                            0x0
+#define CP_GFX_RS64_DC_BASE1_HI__DC_BASE_HI_MASK                                                              0x0000FFFFL
+//CP_GFX_RS64_MIBOUND_LO
+#define CP_GFX_RS64_MIBOUND_LO__BOUND__SHIFT                                                                  0x0
+#define CP_GFX_RS64_MIBOUND_LO__BOUND_MASK                                                                    0xFFFFFFFFL
+//CP_GFX_RS64_MIBOUND_HI
+#define CP_GFX_RS64_MIBOUND_HI__BOUND__SHIFT                                                                  0x0
+#define CP_GFX_RS64_MIBOUND_HI__BOUND_MASK                                                                    0xFFFFFFFFL
+//CP_MEC_DC_BASE_LO
+#define CP_MEC_DC_BASE_LO__DC_BASE_LO__SHIFT                                                                  0x10
+#define CP_MEC_DC_BASE_LO__DC_BASE_LO_MASK                                                                    0xFFFF0000L
+//CP_MEC_MDBASE_LO
+#define CP_MEC_MDBASE_LO__BASE_LO__SHIFT                                                                      0x10
+#define CP_MEC_MDBASE_LO__BASE_LO_MASK                                                                        0xFFFF0000L
+//CP_MEC_DC_BASE_HI
+#define CP_MEC_DC_BASE_HI__DC_BASE_HI__SHIFT                                                                  0x0
+#define CP_MEC_DC_BASE_HI__DC_BASE_HI_MASK                                                                    0x01FFFFFFL
+//CP_MEC_MDBASE_HI
+#define CP_MEC_MDBASE_HI__BASE_HI__SHIFT                                                                      0x0
+#define CP_MEC_MDBASE_HI__BASE_HI_MASK                                                                        0x01FFFFFFL
+//CP_MEC_MIBOUND_LO
+#define CP_MEC_MIBOUND_LO__BOUND_LO__SHIFT                                                                    0x0
+#define CP_MEC_MIBOUND_LO__BOUND_LO_MASK                                                                      0xFFFFFFFFL
+//CP_MEC_MIBOUND_HI
+#define CP_MEC_MIBOUND_HI__BOUND_HI__SHIFT                                                                    0x0
+#define CP_MEC_MIBOUND_HI__BOUND_HI_MASK                                                                      0xFFFFFFFFL
+//CP_MEC_MDBOUND_LO
+#define CP_MEC_MDBOUND_LO__BOUND_LO__SHIFT                                                                    0x0
+#define CP_MEC_MDBOUND_LO__BOUND_LO_MASK                                                                      0xFFFFFFFFL
+//CP_MEC_MDBOUND_HI
+#define CP_MEC_MDBOUND_HI__BOUND_HI__SHIFT                                                                    0x0
+#define CP_MEC_MDBOUND_HI__BOUND_HI_MASK                                                                      0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_grbm_hypdec
+//GRBM_GFX_INDEX_SR_SELECT
+#define GRBM_GFX_INDEX_SR_SELECT__INDEX__SHIFT                                                                0x0
+#define GRBM_GFX_INDEX_SR_SELECT__VF_PF__SHIFT                                                                0x1f
+#define GRBM_GFX_INDEX_SR_SELECT__INDEX_MASK                                                                  0x00000007L
+#define GRBM_GFX_INDEX_SR_SELECT__VF_PF_MASK                                                                  0x80000000L
+//GRBM_GFX_INDEX_SR_DATA
+#define GRBM_GFX_INDEX_SR_DATA__INSTANCE_INDEX__SHIFT                                                         0x0
+#define GRBM_GFX_INDEX_SR_DATA__SA_INDEX__SHIFT                                                               0x8
+#define GRBM_GFX_INDEX_SR_DATA__SE_INDEX__SHIFT                                                               0x10
+#define GRBM_GFX_INDEX_SR_DATA__SA_BROADCAST_WRITES__SHIFT                                                    0x1d
+#define GRBM_GFX_INDEX_SR_DATA__INSTANCE_BROADCAST_WRITES__SHIFT                                              0x1e
+#define GRBM_GFX_INDEX_SR_DATA__SE_BROADCAST_WRITES__SHIFT                                                    0x1f
+#define GRBM_GFX_INDEX_SR_DATA__INSTANCE_INDEX_MASK                                                           0x0000007FL
+#define GRBM_GFX_INDEX_SR_DATA__SA_INDEX_MASK                                                                 0x00000300L
+#define GRBM_GFX_INDEX_SR_DATA__SE_INDEX_MASK                                                                 0x000F0000L
+#define GRBM_GFX_INDEX_SR_DATA__SA_BROADCAST_WRITES_MASK                                                      0x20000000L
+#define GRBM_GFX_INDEX_SR_DATA__INSTANCE_BROADCAST_WRITES_MASK                                                0x40000000L
+#define GRBM_GFX_INDEX_SR_DATA__SE_BROADCAST_WRITES_MASK                                                      0x80000000L
+//GRBM_GFX_CNTL_SR_SELECT
+#define GRBM_GFX_CNTL_SR_SELECT__INDEX__SHIFT                                                                 0x0
+#define GRBM_GFX_CNTL_SR_SELECT__VF_PF__SHIFT                                                                 0x1f
+#define GRBM_GFX_CNTL_SR_SELECT__INDEX_MASK                                                                   0x00000007L
+#define GRBM_GFX_CNTL_SR_SELECT__VF_PF_MASK                                                                   0x80000000L
+//GRBM_GFX_CNTL_SR_DATA
+#define GRBM_GFX_CNTL_SR_DATA__PIPEID__SHIFT                                                                  0x0
+#define GRBM_GFX_CNTL_SR_DATA__MEID__SHIFT                                                                    0x2
+#define GRBM_GFX_CNTL_SR_DATA__VMID__SHIFT                                                                    0x4
+#define GRBM_GFX_CNTL_SR_DATA__QUEUEID__SHIFT                                                                 0x8
+#define GRBM_GFX_CNTL_SR_DATA__CTXID__SHIFT                                                                   0xb
+#define GRBM_GFX_CNTL_SR_DATA__PIPEID_MASK                                                                    0x00000003L
+#define GRBM_GFX_CNTL_SR_DATA__MEID_MASK                                                                      0x0000000CL
+#define GRBM_GFX_CNTL_SR_DATA__VMID_MASK                                                                      0x000000F0L
+#define GRBM_GFX_CNTL_SR_DATA__QUEUEID_MASK                                                                   0x00000700L
+#define GRBM_GFX_CNTL_SR_DATA__CTXID_MASK                                                                     0x00003800L
+//GC_IH_COOKIE_0_PTR_HIGH
+#define GC_IH_COOKIE_0_PTR_HIGH__ADDR__SHIFT                                                                  0x0
+#define GC_IH_COOKIE_0_PTR_HIGH__ADDR_MASK                                                                    0x0000FFFFL
+//GC_IH_COOKIE_0_PTR
+#define GC_IH_COOKIE_0_PTR__ADDR__SHIFT                                                                       0x0
+#define GC_IH_COOKIE_0_PTR__ADDR_MASK                                                                         0xFFFFFFFFL
+//GRBM_SE_REMAP_CNTL
+#define GRBM_SE_REMAP_CNTL__SE0_REMAP_EN__SHIFT                                                               0x0
+#define GRBM_SE_REMAP_CNTL__SE0_REMAP__SHIFT                                                                  0x1
+#define GRBM_SE_REMAP_CNTL__SE1_REMAP_EN__SHIFT                                                               0x4
+#define GRBM_SE_REMAP_CNTL__SE1_REMAP__SHIFT                                                                  0x5
+#define GRBM_SE_REMAP_CNTL__SE2_REMAP_EN__SHIFT                                                               0x8
+#define GRBM_SE_REMAP_CNTL__SE2_REMAP__SHIFT                                                                  0x9
+#define GRBM_SE_REMAP_CNTL__SE3_REMAP_EN__SHIFT                                                               0xc
+#define GRBM_SE_REMAP_CNTL__SE3_REMAP__SHIFT                                                                  0xd
+#define GRBM_SE_REMAP_CNTL__SE4_REMAP_EN__SHIFT                                                               0x10
+#define GRBM_SE_REMAP_CNTL__SE4_REMAP__SHIFT                                                                  0x11
+#define GRBM_SE_REMAP_CNTL__SE5_REMAP_EN__SHIFT                                                               0x14
+#define GRBM_SE_REMAP_CNTL__SE5_REMAP__SHIFT                                                                  0x15
+#define GRBM_SE_REMAP_CNTL__SE6_REMAP_EN__SHIFT                                                               0x18
+#define GRBM_SE_REMAP_CNTL__SE6_REMAP__SHIFT                                                                  0x19
+#define GRBM_SE_REMAP_CNTL__SE7_REMAP_EN__SHIFT                                                               0x1c
+#define GRBM_SE_REMAP_CNTL__SE7_REMAP__SHIFT                                                                  0x1d
+#define GRBM_SE_REMAP_CNTL__SE0_REMAP_EN_MASK                                                                 0x00000001L
+#define GRBM_SE_REMAP_CNTL__SE0_REMAP_MASK                                                                    0x0000000EL
+#define GRBM_SE_REMAP_CNTL__SE1_REMAP_EN_MASK                                                                 0x00000010L
+#define GRBM_SE_REMAP_CNTL__SE1_REMAP_MASK                                                                    0x000000E0L
+#define GRBM_SE_REMAP_CNTL__SE2_REMAP_EN_MASK                                                                 0x00000100L
+#define GRBM_SE_REMAP_CNTL__SE2_REMAP_MASK                                                                    0x00000E00L
+#define GRBM_SE_REMAP_CNTL__SE3_REMAP_EN_MASK                                                                 0x00001000L
+#define GRBM_SE_REMAP_CNTL__SE3_REMAP_MASK                                                                    0x0000E000L
+#define GRBM_SE_REMAP_CNTL__SE4_REMAP_EN_MASK                                                                 0x00010000L
+#define GRBM_SE_REMAP_CNTL__SE4_REMAP_MASK                                                                    0x000E0000L
+#define GRBM_SE_REMAP_CNTL__SE5_REMAP_EN_MASK                                                                 0x00100000L
+#define GRBM_SE_REMAP_CNTL__SE5_REMAP_MASK                                                                    0x00E00000L
+#define GRBM_SE_REMAP_CNTL__SE6_REMAP_EN_MASK                                                                 0x01000000L
+#define GRBM_SE_REMAP_CNTL__SE6_REMAP_MASK                                                                    0x0E000000L
+#define GRBM_SE_REMAP_CNTL__SE7_REMAP_EN_MASK                                                                 0x10000000L
+#define GRBM_SE_REMAP_CNTL__SE7_REMAP_MASK                                                                    0xE0000000L
+//GRBM_SA_REMAP_CNTL
+#define GRBM_SA_REMAP_CNTL__SE0_SA_REMAP__SHIFT                                                               0x0
+#define GRBM_SA_REMAP_CNTL__SE1_SA_REMAP__SHIFT                                                               0x2
+#define GRBM_SA_REMAP_CNTL__SE2_SA_REMAP__SHIFT                                                               0x4
+#define GRBM_SA_REMAP_CNTL__SE3_SA_REMAP__SHIFT                                                               0x6
+#define GRBM_SA_REMAP_CNTL__SE4_SA_REMAP__SHIFT                                                               0x8
+#define GRBM_SA_REMAP_CNTL__SE5_SA_REMAP__SHIFT                                                               0xa
+#define GRBM_SA_REMAP_CNTL__SE6_SA_REMAP__SHIFT                                                               0xc
+#define GRBM_SA_REMAP_CNTL__SE7_SA_REMAP__SHIFT                                                               0xe
+#define GRBM_SA_REMAP_CNTL__SE0_SA_REMAP_MASK                                                                 0x00000003L
+#define GRBM_SA_REMAP_CNTL__SE1_SA_REMAP_MASK                                                                 0x0000000CL
+#define GRBM_SA_REMAP_CNTL__SE2_SA_REMAP_MASK                                                                 0x00000030L
+#define GRBM_SA_REMAP_CNTL__SE3_SA_REMAP_MASK                                                                 0x000000C0L
+#define GRBM_SA_REMAP_CNTL__SE4_SA_REMAP_MASK                                                                 0x00000300L
+#define GRBM_SA_REMAP_CNTL__SE5_SA_REMAP_MASK                                                                 0x00000C00L
+#define GRBM_SA_REMAP_CNTL__SE6_SA_REMAP_MASK                                                                 0x00003000L
+#define GRBM_SA_REMAP_CNTL__SE7_SA_REMAP_MASK                                                                 0x0000C000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_rlcdec
+//RLC_CNTL
+#define RLC_CNTL__RLC_ENABLE_F32__SHIFT                                                                       0x0
+#define RLC_CNTL__FORCE_RETRY__SHIFT                                                                          0x1
+#define RLC_CNTL__READ_CACHE_DISABLE__SHIFT                                                                   0x2
+#define RLC_CNTL__RLC_STEP_F32__SHIFT                                                                         0x3
+#define RLC_CNTL__POISON_INTR_ON_IH_COOKIE_OR_IH_GASKET__SHIFT                                                0x4
+#define RLC_CNTL__POISON_INTERRUPT_TO_PF__SHIFT                                                               0x5
+#define RLC_CNTL__RESERVED__SHIFT                                                                             0x6
+#define RLC_CNTL__RLC_ENABLE_F32_MASK                                                                         0x00000001L
+#define RLC_CNTL__FORCE_RETRY_MASK                                                                            0x00000002L
+#define RLC_CNTL__READ_CACHE_DISABLE_MASK                                                                     0x00000004L
+#define RLC_CNTL__RLC_STEP_F32_MASK                                                                           0x00000008L
+#define RLC_CNTL__POISON_INTR_ON_IH_COOKIE_OR_IH_GASKET_MASK                                                  0x00000010L
+#define RLC_CNTL__POISON_INTERRUPT_TO_PF_MASK                                                                 0x00000020L
+#define RLC_CNTL__RESERVED_MASK                                                                               0xFFFFFFC0L
+//RLC_F32_UCODE_VERSION
+#define RLC_F32_UCODE_VERSION__THREAD0_VERSION__SHIFT                                                         0x0
+#define RLC_F32_UCODE_VERSION__THREAD1_VERSION__SHIFT                                                         0xa
+#define RLC_F32_UCODE_VERSION__THREAD2_VERSION__SHIFT                                                         0x14
+#define RLC_F32_UCODE_VERSION__THREAD0_VERSION_MASK                                                           0x000003FFL
+#define RLC_F32_UCODE_VERSION__THREAD1_VERSION_MASK                                                           0x000FFC00L
+#define RLC_F32_UCODE_VERSION__THREAD2_VERSION_MASK                                                           0x3FF00000L
+//RLC_STAT
+#define RLC_STAT__RLC_BUSY__SHIFT                                                                             0x0
+#define RLC_STAT__RLC_SRM_BUSY__SHIFT                                                                         0x1
+#define RLC_STAT__RLC_GPM_BUSY__SHIFT                                                                         0x2
+#define RLC_STAT__RLC_SPM_BUSY__SHIFT                                                                         0x3
+#define RLC_STAT__MC_BUSY__SHIFT                                                                              0x4
+#define RLC_STAT__RLC_THREAD_0_BUSY__SHIFT                                                                    0x5
+#define RLC_STAT__RLC_THREAD_1_BUSY__SHIFT                                                                    0x6
+#define RLC_STAT__RLC_THREAD_2_BUSY__SHIFT                                                                    0x7
+#define RLC_STAT__RLC_DMA_BUSY__SHIFT                                                                         0x8
+#define RLC_STAT__RESERVED__SHIFT                                                                             0x9
+#define RLC_STAT__RLC_BUSY_MASK                                                                               0x00000001L
+#define RLC_STAT__RLC_SRM_BUSY_MASK                                                                           0x00000002L
+#define RLC_STAT__RLC_GPM_BUSY_MASK                                                                           0x00000004L
+#define RLC_STAT__RLC_SPM_BUSY_MASK                                                                           0x00000008L
+#define RLC_STAT__MC_BUSY_MASK                                                                                0x00000010L
+#define RLC_STAT__RLC_THREAD_0_BUSY_MASK                                                                      0x00000020L
+#define RLC_STAT__RLC_THREAD_1_BUSY_MASK                                                                      0x00000040L
+#define RLC_STAT__RLC_THREAD_2_BUSY_MASK                                                                      0x00000080L
+#define RLC_STAT__RLC_DMA_BUSY_MASK                                                                           0x00000100L
+#define RLC_STAT__RESERVED_MASK                                                                               0xFFFFFE00L
+//RLC_ACTIVE_MASK
+#define RLC_ACTIVE_MASK__SE__SHIFT                                                                            0x0
+#define RLC_ACTIVE_MASK__SE_MASK                                                                              0x000000FFL
+//RLC_GFX_SE_STATUS
+#define RLC_GFX_SE_STATUS__SQG_TTRACE_HALT__SHIFT                                                             0x0
+#define RLC_GFX_SE_STATUS__SQG_TTRACE_HALT_MASK                                                               0x00000003L
+//RLC_REFCLOCK_TIMESTAMP_LSB
+#define RLC_REFCLOCK_TIMESTAMP_LSB__TIMESTAMP_LSB__SHIFT                                                      0x0
+#define RLC_REFCLOCK_TIMESTAMP_LSB__TIMESTAMP_LSB_MASK                                                        0xFFFFFFFFL
+//RLC_REFCLOCK_TIMESTAMP_MSB
+#define RLC_REFCLOCK_TIMESTAMP_MSB__TIMESTAMP_MSB__SHIFT                                                      0x0
+#define RLC_REFCLOCK_TIMESTAMP_MSB__TIMESTAMP_MSB_MASK                                                        0xFFFFFFFFL
+//RLC_GPM_TIMER_INT_0
+#define RLC_GPM_TIMER_INT_0__TIMER__SHIFT                                                                     0x0
+#define RLC_GPM_TIMER_INT_0__TIMER_MASK                                                                       0xFFFFFFFFL
+//RLC_GPM_TIMER_INT_1
+#define RLC_GPM_TIMER_INT_1__TIMER__SHIFT                                                                     0x0
+#define RLC_GPM_TIMER_INT_1__TIMER_MASK                                                                       0xFFFFFFFFL
+//RLC_GPM_TIMER_INT_2
+#define RLC_GPM_TIMER_INT_2__TIMER__SHIFT                                                                     0x0
+#define RLC_GPM_TIMER_INT_2__TIMER_MASK                                                                       0xFFFFFFFFL
+//RLC_GPM_TIMER_INT_3
+#define RLC_GPM_TIMER_INT_3__TIMER__SHIFT                                                                     0x0
+#define RLC_GPM_TIMER_INT_3__TIMER_MASK                                                                       0xFFFFFFFFL
+//RLC_GPM_TIMER_INT_4
+#define RLC_GPM_TIMER_INT_4__TIMER__SHIFT                                                                     0x0
+#define RLC_GPM_TIMER_INT_4__TIMER_MASK                                                                       0xFFFFFFFFL
+//RLC_GPM_TIMER_CTRL
+#define RLC_GPM_TIMER_CTRL__TIMER_0_EN__SHIFT                                                                 0x0
+#define RLC_GPM_TIMER_CTRL__TIMER_1_EN__SHIFT                                                                 0x1
+#define RLC_GPM_TIMER_CTRL__TIMER_2_EN__SHIFT                                                                 0x2
+#define RLC_GPM_TIMER_CTRL__TIMER_3_EN__SHIFT                                                                 0x3
+#define RLC_GPM_TIMER_CTRL__TIMER_4_EN__SHIFT                                                                 0x4
+#define RLC_GPM_TIMER_CTRL__RESERVED_1__SHIFT                                                                 0x5
+#define RLC_GPM_TIMER_CTRL__TIMER_0_AUTO_REARM__SHIFT                                                         0x8
+#define RLC_GPM_TIMER_CTRL__TIMER_1_AUTO_REARM__SHIFT                                                         0x9
+#define RLC_GPM_TIMER_CTRL__TIMER_2_AUTO_REARM__SHIFT                                                         0xa
+#define RLC_GPM_TIMER_CTRL__TIMER_3_AUTO_REARM__SHIFT                                                         0xb
+#define RLC_GPM_TIMER_CTRL__TIMER_4_AUTO_REARM__SHIFT                                                         0xc
+#define RLC_GPM_TIMER_CTRL__RESERVED_2__SHIFT                                                                 0xd
+#define RLC_GPM_TIMER_CTRL__TIMER_0_INT_CLEAR__SHIFT                                                          0x10
+#define RLC_GPM_TIMER_CTRL__TIMER_1_INT_CLEAR__SHIFT                                                          0x11
+#define RLC_GPM_TIMER_CTRL__TIMER_2_INT_CLEAR__SHIFT                                                          0x12
+#define RLC_GPM_TIMER_CTRL__TIMER_3_INT_CLEAR__SHIFT                                                          0x13
+#define RLC_GPM_TIMER_CTRL__TIMER_4_INT_CLEAR__SHIFT                                                          0x14
+#define RLC_GPM_TIMER_CTRL__RESERVED__SHIFT                                                                   0x15
+#define RLC_GPM_TIMER_CTRL__TIMER_0_EN_MASK                                                                   0x00000001L
+#define RLC_GPM_TIMER_CTRL__TIMER_1_EN_MASK                                                                   0x00000002L
+#define RLC_GPM_TIMER_CTRL__TIMER_2_EN_MASK                                                                   0x00000004L
+#define RLC_GPM_TIMER_CTRL__TIMER_3_EN_MASK                                                                   0x00000008L
+#define RLC_GPM_TIMER_CTRL__TIMER_4_EN_MASK                                                                   0x00000010L
+#define RLC_GPM_TIMER_CTRL__RESERVED_1_MASK                                                                   0x000000E0L
+#define RLC_GPM_TIMER_CTRL__TIMER_0_AUTO_REARM_MASK                                                           0x00000100L
+#define RLC_GPM_TIMER_CTRL__TIMER_1_AUTO_REARM_MASK                                                           0x00000200L
+#define RLC_GPM_TIMER_CTRL__TIMER_2_AUTO_REARM_MASK                                                           0x00000400L
+#define RLC_GPM_TIMER_CTRL__TIMER_3_AUTO_REARM_MASK                                                           0x00000800L
+#define RLC_GPM_TIMER_CTRL__TIMER_4_AUTO_REARM_MASK                                                           0x00001000L
+#define RLC_GPM_TIMER_CTRL__RESERVED_2_MASK                                                                   0x0000E000L
+#define RLC_GPM_TIMER_CTRL__TIMER_0_INT_CLEAR_MASK                                                            0x00010000L
+#define RLC_GPM_TIMER_CTRL__TIMER_1_INT_CLEAR_MASK                                                            0x00020000L
+#define RLC_GPM_TIMER_CTRL__TIMER_2_INT_CLEAR_MASK                                                            0x00040000L
+#define RLC_GPM_TIMER_CTRL__TIMER_3_INT_CLEAR_MASK                                                            0x00080000L
+#define RLC_GPM_TIMER_CTRL__TIMER_4_INT_CLEAR_MASK                                                            0x00100000L
+#define RLC_GPM_TIMER_CTRL__RESERVED_MASK                                                                     0xFFE00000L
+//RLC_GPM_TIMER_STAT
+#define RLC_GPM_TIMER_STAT__TIMER_0_STAT__SHIFT                                                               0x0
+#define RLC_GPM_TIMER_STAT__TIMER_1_STAT__SHIFT                                                               0x1
+#define RLC_GPM_TIMER_STAT__TIMER_2_STAT__SHIFT                                                               0x2
+#define RLC_GPM_TIMER_STAT__TIMER_3_STAT__SHIFT                                                               0x3
+#define RLC_GPM_TIMER_STAT__TIMER_4_STAT__SHIFT                                                               0x4
+#define RLC_GPM_TIMER_STAT__RESERVED_1__SHIFT                                                                 0x5
+#define RLC_GPM_TIMER_STAT__TIMER_0_ENABLE_SYNC__SHIFT                                                        0x8
+#define RLC_GPM_TIMER_STAT__TIMER_1_ENABLE_SYNC__SHIFT                                                        0x9
+#define RLC_GPM_TIMER_STAT__TIMER_2_ENABLE_SYNC__SHIFT                                                        0xa
+#define RLC_GPM_TIMER_STAT__TIMER_3_ENABLE_SYNC__SHIFT                                                        0xb
+#define RLC_GPM_TIMER_STAT__TIMER_4_ENABLE_SYNC__SHIFT                                                        0xc
+#define RLC_GPM_TIMER_STAT__RESERVED_2__SHIFT                                                                 0xd
+#define RLC_GPM_TIMER_STAT__TIMER_0_AUTO_REARM_SYNC__SHIFT                                                    0x10
+#define RLC_GPM_TIMER_STAT__TIMER_1_AUTO_REARM_SYNC__SHIFT                                                    0x11
+#define RLC_GPM_TIMER_STAT__TIMER_2_AUTO_REARM_SYNC__SHIFT                                                    0x12
+#define RLC_GPM_TIMER_STAT__TIMER_3_AUTO_REARM_SYNC__SHIFT                                                    0x13
+#define RLC_GPM_TIMER_STAT__TIMER_4_AUTO_REARM_SYNC__SHIFT                                                    0x14
+#define RLC_GPM_TIMER_STAT__RESERVED__SHIFT                                                                   0x15
+#define RLC_GPM_TIMER_STAT__TIMER_0_STAT_MASK                                                                 0x00000001L
+#define RLC_GPM_TIMER_STAT__TIMER_1_STAT_MASK                                                                 0x00000002L
+#define RLC_GPM_TIMER_STAT__TIMER_2_STAT_MASK                                                                 0x00000004L
+#define RLC_GPM_TIMER_STAT__TIMER_3_STAT_MASK                                                                 0x00000008L
+#define RLC_GPM_TIMER_STAT__TIMER_4_STAT_MASK                                                                 0x00000010L
+#define RLC_GPM_TIMER_STAT__RESERVED_1_MASK                                                                   0x000000E0L
+#define RLC_GPM_TIMER_STAT__TIMER_0_ENABLE_SYNC_MASK                                                          0x00000100L
+#define RLC_GPM_TIMER_STAT__TIMER_1_ENABLE_SYNC_MASK                                                          0x00000200L
+#define RLC_GPM_TIMER_STAT__TIMER_2_ENABLE_SYNC_MASK                                                          0x00000400L
+#define RLC_GPM_TIMER_STAT__TIMER_3_ENABLE_SYNC_MASK                                                          0x00000800L
+#define RLC_GPM_TIMER_STAT__TIMER_4_ENABLE_SYNC_MASK                                                          0x00001000L
+#define RLC_GPM_TIMER_STAT__RESERVED_2_MASK                                                                   0x0000E000L
+#define RLC_GPM_TIMER_STAT__TIMER_0_AUTO_REARM_SYNC_MASK                                                      0x00010000L
+#define RLC_GPM_TIMER_STAT__TIMER_1_AUTO_REARM_SYNC_MASK                                                      0x00020000L
+#define RLC_GPM_TIMER_STAT__TIMER_2_AUTO_REARM_SYNC_MASK                                                      0x00040000L
+#define RLC_GPM_TIMER_STAT__TIMER_3_AUTO_REARM_SYNC_MASK                                                      0x00080000L
+#define RLC_GPM_TIMER_STAT__TIMER_4_AUTO_REARM_SYNC_MASK                                                      0x00100000L
+#define RLC_GPM_TIMER_STAT__RESERVED_MASK                                                                     0xFFE00000L
+//RLC_GPM_LEGACY_INT_STAT
+#define RLC_GPM_LEGACY_INT_STAT__SPP_PVT_INT_CHANGED__SHIFT                                                   0x0
+#define RLC_GPM_LEGACY_INT_STAT__RESERVED__SHIFT                                                              0x1
+#define RLC_GPM_LEGACY_INT_STAT__RLC_EOF_INT_CHANGED__SHIFT                                                   0x2
+#define RLC_GPM_LEGACY_INT_STAT__RLC_PG_CNTL_CHANGED__SHIFT                                                   0x3
+#define RLC_GPM_LEGACY_INT_STAT__STORE_LOAD_TIMER3_EXPIRED_T0__SHIFT                                          0x4
+#define RLC_GPM_LEGACY_INT_STAT__SPP_PVT_INT_CHANGED_MASK                                                     0x00000001L
+#define RLC_GPM_LEGACY_INT_STAT__RESERVED_MASK                                                                0x00000002L
+#define RLC_GPM_LEGACY_INT_STAT__RLC_EOF_INT_CHANGED_MASK                                                     0x00000004L
+#define RLC_GPM_LEGACY_INT_STAT__RLC_PG_CNTL_CHANGED_MASK                                                     0x00000008L
+#define RLC_GPM_LEGACY_INT_STAT__STORE_LOAD_TIMER3_EXPIRED_T0_MASK                                            0x00000010L
+//RLC_GPM_LEGACY_INT_CLEAR
+#define RLC_GPM_LEGACY_INT_CLEAR__SPP_PVT_INT_CHANGED__SHIFT                                                  0x0
+#define RLC_GPM_LEGACY_INT_CLEAR__RESERVED__SHIFT                                                             0x1
+#define RLC_GPM_LEGACY_INT_CLEAR__RLC_EOF_INT_CHANGED__SHIFT                                                  0x2
+#define RLC_GPM_LEGACY_INT_CLEAR__RLC_PG_CNTL_CHANGED__SHIFT                                                  0x3
+#define RLC_GPM_LEGACY_INT_CLEAR__RESERVED_4__SHIFT                                                           0x4
+#define RLC_GPM_LEGACY_INT_CLEAR__SPP_PVT_INT_CHANGED_MASK                                                    0x00000001L
+#define RLC_GPM_LEGACY_INT_CLEAR__RESERVED_MASK                                                               0x00000002L
+#define RLC_GPM_LEGACY_INT_CLEAR__RLC_EOF_INT_CHANGED_MASK                                                    0x00000004L
+#define RLC_GPM_LEGACY_INT_CLEAR__RLC_PG_CNTL_CHANGED_MASK                                                    0x00000008L
+#define RLC_GPM_LEGACY_INT_CLEAR__RESERVED_4_MASK                                                             0x00000010L
+//RLC_INT_STAT
+#define RLC_INT_STAT__LAST_CP_RLC_INT_ID__SHIFT                                                               0x0
+#define RLC_INT_STAT__CP_RLC_INT_PENDING__SHIFT                                                               0x8
+#define RLC_INT_STAT__RESERVED__SHIFT                                                                         0x9
+#define RLC_INT_STAT__LAST_CP_RLC_INT_ID_MASK                                                                 0x000000FFL
+#define RLC_INT_STAT__CP_RLC_INT_PENDING_MASK                                                                 0x00000100L
+#define RLC_INT_STAT__RESERVED_MASK                                                                           0xFFFFFE00L
+//RLC_MGCG_CTRL
+#define RLC_MGCG_CTRL__MGCG_EN__SHIFT                                                                         0x0
+#define RLC_MGCG_CTRL__SILICON_EN__SHIFT                                                                      0x1
+#define RLC_MGCG_CTRL__SIMULATION_EN__SHIFT                                                                   0x2
+#define RLC_MGCG_CTRL__ON_DELAY__SHIFT                                                                        0x3
+#define RLC_MGCG_CTRL__OFF_HYSTERESIS__SHIFT                                                                  0x7
+#define RLC_MGCG_CTRL__SPARE__SHIFT                                                                           0xf
+#define RLC_MGCG_CTRL__MGCG_EN_MASK                                                                           0x00000001L
+#define RLC_MGCG_CTRL__SILICON_EN_MASK                                                                        0x00000002L
+#define RLC_MGCG_CTRL__SIMULATION_EN_MASK                                                                     0x00000004L
+#define RLC_MGCG_CTRL__ON_DELAY_MASK                                                                          0x00000078L
+#define RLC_MGCG_CTRL__OFF_HYSTERESIS_MASK                                                                    0x00007F80L
+#define RLC_MGCG_CTRL__SPARE_MASK                                                                             0xFFFF8000L
+//RLC_MCA_DF_RASINTRCTRL_REG_CTRL
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_VMID__SHIFT                                              0x0
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_SRCID__SHIFT                                             0x4
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_CTXID__SHIFT                                             0x8
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_ME_ID__SHIFT                                             0xb
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_PIPE_ID__SHIFT                                           0xd
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_QUEUE_ID__SHIFT                                          0xf
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_TMZ__SHIFT                                               0x12
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_VF__SHIFT                                                0x13
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_VFID__SHIFT                                              0x14
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_VMID_MASK                                                0x0000000FL
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_SRCID_MASK                                               0x000000F0L
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_CTXID_MASK                                               0x00000700L
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_ME_ID_MASK                                               0x00001800L
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_PIPE_ID_MASK                                             0x00006000L
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_QUEUE_ID_MASK                                            0x00038000L
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_TMZ_MASK                                                 0x00040000L
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_VF_MASK                                                  0x00080000L
+#define RLC_MCA_DF_RASINTRCTRL_REG_CTRL__MCA_GRBM_WR_VFID_MASK                                                0x01F00000L
+//RLC_JUMP_TABLE_RESTORE
+#define RLC_JUMP_TABLE_RESTORE__ADDR__SHIFT                                                                   0x0
+#define RLC_JUMP_TABLE_RESTORE__ADDR_MASK                                                                     0xFFFFFFFFL
+//RLC_PG_DELAY_2
+#define RLC_PG_DELAY_2__SERDES_TIMEOUT_VALUE__SHIFT                                                           0x0
+#define RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT                                                               0x8
+#define RLC_PG_DELAY_2__PERWGP_TIMEOUT_VALUE__SHIFT                                                           0x10
+#define RLC_PG_DELAY_2__SERDES_TIMEOUT_VALUE_MASK                                                             0x000000FFL
+#define RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK                                                                 0x0000FF00L
+#define RLC_PG_DELAY_2__PERWGP_TIMEOUT_VALUE_MASK                                                             0xFFFF0000L
+//RLC_GPU_CLOCK_COUNT_LSB
+#define RLC_GPU_CLOCK_COUNT_LSB__GPU_CLOCKS_LSB__SHIFT                                                        0x0
+#define RLC_GPU_CLOCK_COUNT_LSB__GPU_CLOCKS_LSB_MASK                                                          0xFFFFFFFFL
+//RLC_GPU_CLOCK_COUNT_MSB
+#define RLC_GPU_CLOCK_COUNT_MSB__GPU_CLOCKS_MSB__SHIFT                                                        0x0
+#define RLC_GPU_CLOCK_COUNT_MSB__GPU_CLOCKS_MSB_MASK                                                          0xFFFFFFFFL
+//RLC_CAPTURE_GPU_CLOCK_COUNT
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__CAPTURE__SHIFT                                                           0x0
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__RESERVED__SHIFT                                                          0x1
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__CAPTURE_MASK                                                             0x00000001L
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__RESERVED_MASK                                                            0xFFFFFFFEL
+//RLC_UCODE_CNTL
+#define RLC_UCODE_CNTL__RLC_UCODE_FLAGS__SHIFT                                                                0x0
+#define RLC_UCODE_CNTL__RLC_UCODE_FLAGS_MASK                                                                  0xFFFFFFFFL
+//RLC_GPM_THREAD_RESET
+#define RLC_GPM_THREAD_RESET__THREAD0_RESET__SHIFT                                                            0x0
+#define RLC_GPM_THREAD_RESET__THREAD1_RESET__SHIFT                                                            0x1
+#define RLC_GPM_THREAD_RESET__THREAD2_RESET__SHIFT                                                            0x2
+#define RLC_GPM_THREAD_RESET__THREAD3_RESET__SHIFT                                                            0x3
+#define RLC_GPM_THREAD_RESET__RESERVED__SHIFT                                                                 0x4
+#define RLC_GPM_THREAD_RESET__THREAD0_RESET_MASK                                                              0x00000001L
+#define RLC_GPM_THREAD_RESET__THREAD1_RESET_MASK                                                              0x00000002L
+#define RLC_GPM_THREAD_RESET__THREAD2_RESET_MASK                                                              0x00000004L
+#define RLC_GPM_THREAD_RESET__THREAD3_RESET_MASK                                                              0x00000008L
+#define RLC_GPM_THREAD_RESET__RESERVED_MASK                                                                   0xFFFFFFF0L
+//RLC_GPM_CP_DMA_COMPLETE_T0
+#define RLC_GPM_CP_DMA_COMPLETE_T0__DATA__SHIFT                                                               0x0
+#define RLC_GPM_CP_DMA_COMPLETE_T0__RESERVED__SHIFT                                                           0x1
+#define RLC_GPM_CP_DMA_COMPLETE_T0__DATA_MASK                                                                 0x00000001L
+#define RLC_GPM_CP_DMA_COMPLETE_T0__RESERVED_MASK                                                             0xFFFFFFFEL
+//RLC_GPM_CP_DMA_COMPLETE_T1
+#define RLC_GPM_CP_DMA_COMPLETE_T1__DATA__SHIFT                                                               0x0
+#define RLC_GPM_CP_DMA_COMPLETE_T1__RESERVED__SHIFT                                                           0x1
+#define RLC_GPM_CP_DMA_COMPLETE_T1__DATA_MASK                                                                 0x00000001L
+#define RLC_GPM_CP_DMA_COMPLETE_T1__RESERVED_MASK                                                             0xFFFFFFFEL
+//RLC_GPM_THREAD_INVALIDATE_CACHE
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__THREAD0_INVALIDATE_CACHE__SHIFT                                      0x0
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__THREAD1_INVALIDATE_CACHE__SHIFT                                      0x1
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__THREAD2_INVALIDATE_CACHE__SHIFT                                      0x2
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__THREAD3_INVALIDATE_CACHE__SHIFT                                      0x3
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__RESERVED__SHIFT                                                      0x4
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__THREAD0_INVALIDATE_CACHE_MASK                                        0x00000001L
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__THREAD1_INVALIDATE_CACHE_MASK                                        0x00000002L
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__THREAD2_INVALIDATE_CACHE_MASK                                        0x00000004L
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__THREAD3_INVALIDATE_CACHE_MASK                                        0x00000008L
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__RESERVED_MASK                                                        0xFFFFFFF0L
+//RLC_MCA_DF_RASINTRCTRL_REG_LO
+#define RLC_MCA_DF_RASINTRCTRL_REG_LO__ADDR__SHIFT                                                            0x0
+#define RLC_MCA_DF_RASINTRCTRL_REG_LO__APERTURE__SHIFT                                                        0x14
+#define RLC_MCA_DF_RASINTRCTRL_REG_LO__ADDR_MASK                                                              0x000FFFFFL
+#define RLC_MCA_DF_RASINTRCTRL_REG_LO__APERTURE_MASK                                                          0xFFF00000L
+//RLC_MCA_DF_RASINTRCTRL_REG_HI
+#define RLC_MCA_DF_RASINTRCTRL_REG_HI__MCM_ADDR__SHIFT                                                        0x0
+#define RLC_MCA_DF_RASINTRCTRL_REG_HI__MCM_ADDR_MASK                                                          0x0000FFFFL
+//RLC_MCA_INTR_CTRL
+#define RLC_MCA_INTR_CTRL__MCA_INTR_HYSTERISIS_CNTR__SHIFT                                                    0x0
+#define RLC_MCA_INTR_CTRL__MCA_INTR_BY_RLC_FW__SHIFT                                                          0x10
+#define RLC_MCA_INTR_CTRL__MCA_INTR_HYSTERISIS_CNTR_MASK                                                      0x0000FFFFL
+#define RLC_MCA_INTR_CTRL__MCA_INTR_BY_RLC_FW_MASK                                                            0x00010000L
+//RLC_MCA_INTR_INFO
+#define RLC_MCA_INTR_INFO__MCA_INTREQ_INFO__SHIFT                                                             0x0
+#define RLC_MCA_INTR_INFO__MCA_FatalOut_INFO__SHIFT                                                           0x8
+#define RLC_MCA_INTR_INFO__MCA_MASTERCOREID_REG_OFFSET__SHIFT                                                 0x10
+#define RLC_MCA_INTR_INFO__MCA_INTERRUPT_SERVICED_BY_RLC_FW__SHIFT                                            0x1f
+#define RLC_MCA_INTR_INFO__MCA_INTREQ_INFO_MASK                                                               0x0000000FL
+#define RLC_MCA_INTR_INFO__MCA_FatalOut_INFO_MASK                                                             0x00000100L
+#define RLC_MCA_INTR_INFO__MCA_MASTERCOREID_REG_OFFSET_MASK                                                   0x01FF0000L
+#define RLC_MCA_INTR_INFO__MCA_INTERRUPT_SERVICED_BY_RLC_FW_MASK                                              0x80000000L
+//RLC_CLK_COUNT_GFXCLK_LSB
+#define RLC_CLK_COUNT_GFXCLK_LSB__COUNTER__SHIFT                                                              0x0
+#define RLC_CLK_COUNT_GFXCLK_LSB__COUNTER_MASK                                                                0xFFFFFFFFL
+//RLC_CLK_COUNT_GFXCLK_MSB
+#define RLC_CLK_COUNT_GFXCLK_MSB__COUNTER__SHIFT                                                              0x0
+#define RLC_CLK_COUNT_GFXCLK_MSB__COUNTER_MASK                                                                0xFFFFFFFFL
+//RLC_CLK_COUNT_REFCLK_LSB
+#define RLC_CLK_COUNT_REFCLK_LSB__COUNTER__SHIFT                                                              0x0
+#define RLC_CLK_COUNT_REFCLK_LSB__COUNTER_MASK                                                                0xFFFFFFFFL
+//RLC_CLK_COUNT_REFCLK_MSB
+#define RLC_CLK_COUNT_REFCLK_MSB__COUNTER__SHIFT                                                              0x0
+#define RLC_CLK_COUNT_REFCLK_MSB__COUNTER_MASK                                                                0xFFFFFFFFL
+//RLC_CLK_COUNT_CTRL
+#define RLC_CLK_COUNT_CTRL__GFXCLK_RUN__SHIFT                                                                 0x0
+#define RLC_CLK_COUNT_CTRL__GFXCLK_RESET__SHIFT                                                               0x1
+#define RLC_CLK_COUNT_CTRL__GFXCLK_SAMPLE__SHIFT                                                              0x2
+#define RLC_CLK_COUNT_CTRL__REFCLK_RUN__SHIFT                                                                 0x3
+#define RLC_CLK_COUNT_CTRL__REFCLK_RESET__SHIFT                                                               0x4
+#define RLC_CLK_COUNT_CTRL__REFCLK_SAMPLE__SHIFT                                                              0x5
+#define RLC_CLK_COUNT_CTRL__GFXCLK_RUN_MASK                                                                   0x00000001L
+#define RLC_CLK_COUNT_CTRL__GFXCLK_RESET_MASK                                                                 0x00000002L
+#define RLC_CLK_COUNT_CTRL__GFXCLK_SAMPLE_MASK                                                                0x00000004L
+#define RLC_CLK_COUNT_CTRL__REFCLK_RUN_MASK                                                                   0x00000008L
+#define RLC_CLK_COUNT_CTRL__REFCLK_RESET_MASK                                                                 0x00000010L
+#define RLC_CLK_COUNT_CTRL__REFCLK_SAMPLE_MASK                                                                0x00000020L
+//RLC_CLK_COUNT_STAT
+#define RLC_CLK_COUNT_STAT__GFXCLK_VALID__SHIFT                                                               0x0
+#define RLC_CLK_COUNT_STAT__REFCLK_VALID__SHIFT                                                               0x1
+#define RLC_CLK_COUNT_STAT__REFCLK_RUN_RESYNC__SHIFT                                                          0x2
+#define RLC_CLK_COUNT_STAT__REFCLK_RESET_RESYNC__SHIFT                                                        0x3
+#define RLC_CLK_COUNT_STAT__REFCLK_SAMPLE_RESYNC__SHIFT                                                       0x4
+#define RLC_CLK_COUNT_STAT__RESERVED__SHIFT                                                                   0x5
+#define RLC_CLK_COUNT_STAT__GFXCLK_VALID_MASK                                                                 0x00000001L
+#define RLC_CLK_COUNT_STAT__REFCLK_VALID_MASK                                                                 0x00000002L
+#define RLC_CLK_COUNT_STAT__REFCLK_RUN_RESYNC_MASK                                                            0x00000004L
+#define RLC_CLK_COUNT_STAT__REFCLK_RESET_RESYNC_MASK                                                          0x00000008L
+#define RLC_CLK_COUNT_STAT__REFCLK_SAMPLE_RESYNC_MASK                                                         0x00000010L
+#define RLC_CLK_COUNT_STAT__RESERVED_MASK                                                                     0xFFFFFFE0L
+//RLC_RLCG_DOORBELL_CNTL
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_0_MODE__SHIFT                                                        0x0
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_1_MODE__SHIFT                                                        0x2
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_2_MODE__SHIFT                                                        0x4
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_3_MODE__SHIFT                                                        0x6
+#define RLC_RLCG_DOORBELL_CNTL__RESERVED__SHIFT                                                               0x8
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_ID__SHIFT                                                            0x10
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_ID_EN__SHIFT                                                         0x15
+#define RLC_RLCG_DOORBELL_CNTL__RESERVED_31_22__SHIFT                                                         0x16
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_0_MODE_MASK                                                          0x00000003L
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_1_MODE_MASK                                                          0x0000000CL
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_2_MODE_MASK                                                          0x00000030L
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_3_MODE_MASK                                                          0x000000C0L
+#define RLC_RLCG_DOORBELL_CNTL__RESERVED_MASK                                                                 0x0000FF00L
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_ID_MASK                                                              0x001F0000L
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_ID_EN_MASK                                                           0x00200000L
+#define RLC_RLCG_DOORBELL_CNTL__RESERVED_31_22_MASK                                                           0xFFC00000L
+//RLC_RLCG_DOORBELL_STAT
+#define RLC_RLCG_DOORBELL_STAT__DOORBELL_0_VALID__SHIFT                                                       0x0
+#define RLC_RLCG_DOORBELL_STAT__DOORBELL_1_VALID__SHIFT                                                       0x1
+#define RLC_RLCG_DOORBELL_STAT__DOORBELL_2_VALID__SHIFT                                                       0x2
+#define RLC_RLCG_DOORBELL_STAT__DOORBELL_3_VALID__SHIFT                                                       0x3
+#define RLC_RLCG_DOORBELL_STAT__DOORBELL_0_VALID_MASK                                                         0x00000001L
+#define RLC_RLCG_DOORBELL_STAT__DOORBELL_1_VALID_MASK                                                         0x00000002L
+#define RLC_RLCG_DOORBELL_STAT__DOORBELL_2_VALID_MASK                                                         0x00000004L
+#define RLC_RLCG_DOORBELL_STAT__DOORBELL_3_VALID_MASK                                                         0x00000008L
+//RLC_RLCG_DOORBELL_0_DATA_LO
+#define RLC_RLCG_DOORBELL_0_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCG_DOORBELL_0_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCG_DOORBELL_0_DATA_HI
+#define RLC_RLCG_DOORBELL_0_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCG_DOORBELL_0_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCG_DOORBELL_1_DATA_LO
+#define RLC_RLCG_DOORBELL_1_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCG_DOORBELL_1_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCG_DOORBELL_1_DATA_HI
+#define RLC_RLCG_DOORBELL_1_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCG_DOORBELL_1_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCG_DOORBELL_2_DATA_LO
+#define RLC_RLCG_DOORBELL_2_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCG_DOORBELL_2_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCG_DOORBELL_2_DATA_HI
+#define RLC_RLCG_DOORBELL_2_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCG_DOORBELL_2_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCG_DOORBELL_3_DATA_LO
+#define RLC_RLCG_DOORBELL_3_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCG_DOORBELL_3_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCG_DOORBELL_3_DATA_HI
+#define RLC_RLCG_DOORBELL_3_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCG_DOORBELL_3_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_GPU_CLOCK_32_RES_SEL
+#define RLC_GPU_CLOCK_32_RES_SEL__RES_SEL__SHIFT                                                              0x0
+#define RLC_GPU_CLOCK_32_RES_SEL__RESERVED__SHIFT                                                             0x6
+#define RLC_GPU_CLOCK_32_RES_SEL__RES_SEL_MASK                                                                0x0000003FL
+#define RLC_GPU_CLOCK_32_RES_SEL__RESERVED_MASK                                                               0xFFFFFFC0L
+//RLC_GPU_CLOCK_32
+#define RLC_GPU_CLOCK_32__GPU_CLOCK_32__SHIFT                                                                 0x0
+#define RLC_GPU_CLOCK_32__GPU_CLOCK_32_MASK                                                                   0xFFFFFFFFL
+//RLC_PG_CNTL
+#define RLC_PG_CNTL__GFX_POWER_GATING_ENABLE__SHIFT                                                           0x0
+#define RLC_PG_CNTL__GFX_POWER_GATING_SRC__SHIFT                                                              0x1
+#define RLC_PG_CNTL__DYN_PER_WGP_PG_ENABLE__SHIFT                                                             0x2
+#define RLC_PG_CNTL__STATIC_PER_WGP_PG_ENABLE__SHIFT                                                          0x3
+#define RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE__SHIFT                                                            0x4
+#define RLC_PG_CNTL__RESERVED__SHIFT                                                                          0x5
+#define RLC_PG_CNTL__MEM_DS_DISABLE__SHIFT                                                                    0xd
+#define RLC_PG_CNTL__PG_OVERRIDE__SHIFT                                                                       0xe
+#define RLC_PG_CNTL__CP_PG_DISABLE__SHIFT                                                                     0xf
+#define RLC_PG_CNTL__CHUB_HANDSHAKE_ENABLE__SHIFT                                                             0x10
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE__SHIFT                                                     0x11
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE__SHIFT                                                     0x12
+#define RLC_PG_CNTL__RESERVED1__SHIFT                                                                         0x13
+#define RLC_PG_CNTL__Ultra_Low_Voltage_Enable__SHIFT                                                          0x15
+#define RLC_PG_CNTL__RESERVED2__SHIFT                                                                         0x16
+#define RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE__SHIFT                                                             0x17
+#define RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK                                                             0x00000001L
+#define RLC_PG_CNTL__GFX_POWER_GATING_SRC_MASK                                                                0x00000002L
+#define RLC_PG_CNTL__DYN_PER_WGP_PG_ENABLE_MASK                                                               0x00000004L
+#define RLC_PG_CNTL__STATIC_PER_WGP_PG_ENABLE_MASK                                                            0x00000008L
+#define RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE_MASK                                                              0x00000010L
+#define RLC_PG_CNTL__RESERVED_MASK                                                                            0x00001FE0L
+#define RLC_PG_CNTL__MEM_DS_DISABLE_MASK                                                                      0x00002000L
+#define RLC_PG_CNTL__PG_OVERRIDE_MASK                                                                         0x00004000L
+#define RLC_PG_CNTL__CP_PG_DISABLE_MASK                                                                       0x00008000L
+#define RLC_PG_CNTL__CHUB_HANDSHAKE_ENABLE_MASK                                                               0x00010000L
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK                                                       0x00020000L
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK                                                       0x00040000L
+#define RLC_PG_CNTL__RESERVED1_MASK                                                                           0x00180000L
+#define RLC_PG_CNTL__Ultra_Low_Voltage_Enable_MASK                                                            0x00200000L
+#define RLC_PG_CNTL__RESERVED2_MASK                                                                           0x00400000L
+#define RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK                                                               0x00800000L
+//RLC_GPM_THREAD_PRIORITY
+#define RLC_GPM_THREAD_PRIORITY__THREAD0_PRIORITY__SHIFT                                                      0x0
+#define RLC_GPM_THREAD_PRIORITY__THREAD1_PRIORITY__SHIFT                                                      0x8
+#define RLC_GPM_THREAD_PRIORITY__THREAD2_PRIORITY__SHIFT                                                      0x10
+#define RLC_GPM_THREAD_PRIORITY__THREAD3_PRIORITY__SHIFT                                                      0x18
+#define RLC_GPM_THREAD_PRIORITY__THREAD0_PRIORITY_MASK                                                        0x000000FFL
+#define RLC_GPM_THREAD_PRIORITY__THREAD1_PRIORITY_MASK                                                        0x0000FF00L
+#define RLC_GPM_THREAD_PRIORITY__THREAD2_PRIORITY_MASK                                                        0x00FF0000L
+#define RLC_GPM_THREAD_PRIORITY__THREAD3_PRIORITY_MASK                                                        0xFF000000L
+//RLC_GPM_THREAD_ENABLE
+#define RLC_GPM_THREAD_ENABLE__THREAD0_ENABLE__SHIFT                                                          0x0
+#define RLC_GPM_THREAD_ENABLE__THREAD1_ENABLE__SHIFT                                                          0x1
+#define RLC_GPM_THREAD_ENABLE__THREAD2_ENABLE__SHIFT                                                          0x2
+#define RLC_GPM_THREAD_ENABLE__THREAD3_ENABLE__SHIFT                                                          0x3
+#define RLC_GPM_THREAD_ENABLE__RESERVED__SHIFT                                                                0x4
+#define RLC_GPM_THREAD_ENABLE__THREAD0_ENABLE_MASK                                                            0x00000001L
+#define RLC_GPM_THREAD_ENABLE__THREAD1_ENABLE_MASK                                                            0x00000002L
+#define RLC_GPM_THREAD_ENABLE__THREAD2_ENABLE_MASK                                                            0x00000004L
+#define RLC_GPM_THREAD_ENABLE__THREAD3_ENABLE_MASK                                                            0x00000008L
+#define RLC_GPM_THREAD_ENABLE__RESERVED_MASK                                                                  0xFFFFFFF0L
+//RLC_RLCG_DOORBELL_RANGE
+#define RLC_RLCG_DOORBELL_RANGE__LOWER_ADDR_RESERVED__SHIFT                                                   0x0
+#define RLC_RLCG_DOORBELL_RANGE__LOWER_ADDR__SHIFT                                                            0x2
+#define RLC_RLCG_DOORBELL_RANGE__RESERVED_15_12__SHIFT                                                        0xc
+#define RLC_RLCG_DOORBELL_RANGE__UPPER_ADDR_RESERVED__SHIFT                                                   0x10
+#define RLC_RLCG_DOORBELL_RANGE__UPPER_ADDR__SHIFT                                                            0x12
+#define RLC_RLCG_DOORBELL_RANGE__RESERVED_31_28__SHIFT                                                        0x1c
+#define RLC_RLCG_DOORBELL_RANGE__LOWER_ADDR_RESERVED_MASK                                                     0x00000003L
+#define RLC_RLCG_DOORBELL_RANGE__LOWER_ADDR_MASK                                                              0x00000FFCL
+#define RLC_RLCG_DOORBELL_RANGE__RESERVED_15_12_MASK                                                          0x0000F000L
+#define RLC_RLCG_DOORBELL_RANGE__UPPER_ADDR_RESERVED_MASK                                                     0x00030000L
+#define RLC_RLCG_DOORBELL_RANGE__UPPER_ADDR_MASK                                                              0x0FFC0000L
+#define RLC_RLCG_DOORBELL_RANGE__RESERVED_31_28_MASK                                                          0xF0000000L
+//RLC_CGTT_MGCG_OVERRIDE
+#define RLC_CGTT_MGCG_OVERRIDE__RLC_REPEATER_FGCG_OVERRIDE__SHIFT                                             0x0
+#define RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE__SHIFT                                                 0x1
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE__SHIFT                                                    0x2
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE__SHIFT                                                    0x3
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE__SHIFT                                                    0x4
+#define RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE__SHIFT                                                0x5
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE__SHIFT                                                    0x6
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE__SHIFT                                                0x7
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE__SHIFT                                                    0x8
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE__SHIFT                                           0x9
+#define RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE__SHIFT                                                    0xa
+#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_31_11__SHIFT                                                         0xb
+#define RLC_CGTT_MGCG_OVERRIDE__RLC_REPEATER_FGCG_OVERRIDE_MASK                                               0x00000001L
+#define RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK                                                   0x00000002L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK                                                      0x00000004L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK                                                      0x00000008L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK                                                      0x00000010L
+#define RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK                                                  0x00000020L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK                                                      0x00000040L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK                                                  0x00000080L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK                                                      0x00000100L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK                                             0x00000200L
+#define RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK                                                      0x00000400L
+#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_31_11_MASK                                                           0xFFFFF800L
+//RLC_CGCG_CGLS_CTRL
+#define RLC_CGCG_CGLS_CTRL__CGCG_EN__SHIFT                                                                    0x0
+#define RLC_CGCG_CGLS_CTRL__CGLS_EN__SHIFT                                                                    0x1
+#define RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT                                                   0x2
+#define RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT                                                    0x8
+#define RLC_CGCG_CGLS_CTRL__CGCG_CONTROLLER__SHIFT                                                            0x1b
+#define RLC_CGCG_CGLS_CTRL__CGCG_REG_CTRL__SHIFT                                                              0x1c
+#define RLC_CGCG_CGLS_CTRL__SLEEP_MODE__SHIFT                                                                 0x1d
+#define RLC_CGCG_CGLS_CTRL__SIM_SILICON_EN__SHIFT                                                             0x1f
+#define RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK                                                                      0x00000001L
+#define RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK                                                                      0x00000002L
+#define RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK                                                     0x000000FCL
+#define RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK                                                      0x07FFFF00L
+#define RLC_CGCG_CGLS_CTRL__CGCG_CONTROLLER_MASK                                                              0x08000000L
+#define RLC_CGCG_CGLS_CTRL__CGCG_REG_CTRL_MASK                                                                0x10000000L
+#define RLC_CGCG_CGLS_CTRL__SLEEP_MODE_MASK                                                                   0x60000000L
+#define RLC_CGCG_CGLS_CTRL__SIM_SILICON_EN_MASK                                                               0x80000000L
+//RLC_CGCG_RAMP_CTRL
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_START_UNIT__SHIFT                                                        0x0
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_STEP_UNIT__SHIFT                                                         0x4
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_START_UNIT__SHIFT                                                          0x8
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_STEP_UNIT__SHIFT                                                           0xc
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_CNT__SHIFT                                                             0x10
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_UNIT__SHIFT                                                            0x1c
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_START_UNIT_MASK                                                          0x0000000FL
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_STEP_UNIT_MASK                                                           0x000000F0L
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_START_UNIT_MASK                                                            0x00000F00L
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_STEP_UNIT_MASK                                                             0x0000F000L
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_CNT_MASK                                                               0x0FFF0000L
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_UNIT_MASK                                                              0xF0000000L
+//RLC_DYN_PG_STATUS
+#define RLC_DYN_PG_STATUS__PG_STATUS_WGP_MASK__SHIFT                                                          0x0
+#define RLC_DYN_PG_STATUS__PG_STATUS_WGP_MASK_MASK                                                            0xFFFFFFFFL
+//RLC_DYN_PG_REQUEST
+#define RLC_DYN_PG_REQUEST__PG_REQUEST_WGP_MASK__SHIFT                                                        0x0
+#define RLC_DYN_PG_REQUEST__PG_REQUEST_WGP_MASK_MASK                                                          0xFFFFFFFFL
+//RLC_PG_DELAY
+#define RLC_PG_DELAY__POWER_UP_DELAY__SHIFT                                                                   0x0
+#define RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT                                                                 0x8
+#define RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT                                                              0x10
+#define RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT                                                                  0x18
+#define RLC_PG_DELAY__POWER_UP_DELAY_MASK                                                                     0x000000FFL
+#define RLC_PG_DELAY__POWER_DOWN_DELAY_MASK                                                                   0x0000FF00L
+#define RLC_PG_DELAY__CMD_PROPAGATE_DELAY_MASK                                                                0x00FF0000L
+#define RLC_PG_DELAY__MEM_SLEEP_DELAY_MASK                                                                    0xFF000000L
+//RLC_CGCG_CGLS_CTRL_2
+#define RLC_CGCG_CGLS_CTRL_2__CANE_STAT_BUSY_MASK__SHIFT                                                      0x0
+#define RLC_CGCG_CGLS_CTRL_2__RESERVED__SHIFT                                                                 0x1
+#define RLC_CGCG_CGLS_CTRL_2__CANE_STAT_BUSY_MASK_MASK                                                        0x00000001L
+#define RLC_CGCG_CGLS_CTRL_2__RESERVED_MASK                                                                   0xFFFFFFFEL
+//RLC_PG_ALWAYS_ON_WGP_MASK
+#define RLC_PG_ALWAYS_ON_WGP_MASK__AON_WGP_MASK__SHIFT                                                        0x0
+#define RLC_PG_ALWAYS_ON_WGP_MASK__AON_WGP_MASK_MASK                                                          0xFFFFFFFFL
+//RLC_MAX_PG_WGP
+#define RLC_MAX_PG_WGP__MAX_POWERED_UP_WGP__SHIFT                                                             0x0
+#define RLC_MAX_PG_WGP__SPARE__SHIFT                                                                          0x8
+#define RLC_MAX_PG_WGP__MAX_POWERED_UP_WGP_MASK                                                               0x000000FFL
+#define RLC_MAX_PG_WGP__SPARE_MASK                                                                            0xFFFFFF00L
+//RLC_AUTO_PG_CTRL
+#define RLC_AUTO_PG_CTRL__AUTO_PG_EN__SHIFT                                                                   0x0
+#define RLC_AUTO_PG_CTRL__AUTO_GRBM_REG_SAVE_ON_IDLE_EN__SHIFT                                                0x1
+#define RLC_AUTO_PG_CTRL__AUTO_WAKE_UP_EN__SHIFT                                                              0x2
+#define RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT                                             0x3
+#define RLC_AUTO_PG_CTRL__PG_AFTER_GRBM_REG_SAVE_THRESHOLD__SHIFT                                             0x13
+#define RLC_AUTO_PG_CTRL__AUTO_PG_EN_MASK                                                                     0x00000001L
+#define RLC_AUTO_PG_CTRL__AUTO_GRBM_REG_SAVE_ON_IDLE_EN_MASK                                                  0x00000002L
+#define RLC_AUTO_PG_CTRL__AUTO_WAKE_UP_EN_MASK                                                                0x00000004L
+#define RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK                                               0x0007FFF8L
+#define RLC_AUTO_PG_CTRL__PG_AFTER_GRBM_REG_SAVE_THRESHOLD_MASK                                               0xFFF80000L
+//RLC_SERDES_RD_INDEX
+#define RLC_SERDES_RD_INDEX__DATA_REG_ID__SHIFT                                                               0x0
+#define RLC_SERDES_RD_INDEX__SPARE__SHIFT                                                                     0x2
+#define RLC_SERDES_RD_INDEX__DATA_REG_ID_MASK                                                                 0x00000003L
+#define RLC_SERDES_RD_INDEX__SPARE_MASK                                                                       0xFFFFFFFCL
+//RLC_SERDES_RD_DATA_0
+#define RLC_SERDES_RD_DATA_0__DATA__SHIFT                                                                     0x0
+#define RLC_SERDES_RD_DATA_0__DATA_MASK                                                                       0xFFFFFFFFL
+//RLC_SERDES_RD_DATA_1
+#define RLC_SERDES_RD_DATA_1__DATA__SHIFT                                                                     0x0
+#define RLC_SERDES_RD_DATA_1__DATA_MASK                                                                       0xFFFFFFFFL
+//RLC_SERDES_RD_DATA_2
+#define RLC_SERDES_RD_DATA_2__DATA__SHIFT                                                                     0x0
+#define RLC_SERDES_RD_DATA_2__DATA_MASK                                                                       0xFFFFFFFFL
+//RLC_SERDES_RD_DATA_3
+#define RLC_SERDES_RD_DATA_3__DATA__SHIFT                                                                     0x0
+#define RLC_SERDES_RD_DATA_3__DATA_MASK                                                                       0xFFFFFFFFL
+//RLC_SERDES_MASK
+#define RLC_SERDES_MASK__GC_CENTER_HUB_0__SHIFT                                                               0x0
+#define RLC_SERDES_MASK__GC_CENTER_HUB_1__SHIFT                                                               0x1
+#define RLC_SERDES_MASK__GC_CENTER_HUB_0_MASK                                                                 0x00000001L
+#define RLC_SERDES_MASK__GC_CENTER_HUB_1_MASK                                                                 0x00000002L
+//RLC_SERDES_CTRL
+#define RLC_SERDES_CTRL__BPM_BROADCAST__SHIFT                                                                 0x0
+#define RLC_SERDES_CTRL__BPM_REG_WRITE__SHIFT                                                                 0x1
+#define RLC_SERDES_CTRL__BPM_LONG_CMD__SHIFT                                                                  0x2
+#define RLC_SERDES_CTRL__BPM_ADDR__SHIFT                                                                      0x4
+#define RLC_SERDES_CTRL__REG_ADDR__SHIFT                                                                      0x10
+#define RLC_SERDES_CTRL__BPM_BROADCAST_MASK                                                                   0x000001L
+#define RLC_SERDES_CTRL__BPM_REG_WRITE_MASK                                                                   0x000002L
+#define RLC_SERDES_CTRL__BPM_LONG_CMD_MASK                                                                    0x000004L
+#define RLC_SERDES_CTRL__BPM_ADDR_MASK                                                                        0x007FF0L
+#define RLC_SERDES_CTRL__REG_ADDR_MASK                                                                        0xFF0000L
+//RLC_SERDES_DATA
+#define RLC_SERDES_DATA__DATA__SHIFT                                                                          0x0
+#define RLC_SERDES_DATA__DATA_MASK                                                                            0xFFFFFFFFL
+//RLC_SERDES_BUSY
+#define RLC_SERDES_BUSY__GC_CENTER_HUB_0__SHIFT                                                               0x0
+#define RLC_SERDES_BUSY__GC_CENTER_HUB_1__SHIFT                                                               0x1
+#define RLC_SERDES_BUSY__RESERVED__SHIFT                                                                      0x2
+#define RLC_SERDES_BUSY__RD_FIFO_NOT_EMPTY__SHIFT                                                             0x1e
+#define RLC_SERDES_BUSY__RD_PENDING__SHIFT                                                                    0x1f
+#define RLC_SERDES_BUSY__GC_CENTER_HUB_0_MASK                                                                 0x00000001L
+#define RLC_SERDES_BUSY__GC_CENTER_HUB_1_MASK                                                                 0x00000002L
+#define RLC_SERDES_BUSY__RESERVED_MASK                                                                        0x3FFFFFFCL
+#define RLC_SERDES_BUSY__RD_FIFO_NOT_EMPTY_MASK                                                               0x40000000L
+#define RLC_SERDES_BUSY__RD_PENDING_MASK                                                                      0x80000000L
+//RLC_GPM_GENERAL_0
+#define RLC_GPM_GENERAL_0__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_0__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_1
+#define RLC_GPM_GENERAL_1__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_1__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_2
+#define RLC_GPM_GENERAL_2__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_2__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_3
+#define RLC_GPM_GENERAL_3__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_3__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_4
+#define RLC_GPM_GENERAL_4__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_4__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_5
+#define RLC_GPM_GENERAL_5__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_5__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_6
+#define RLC_GPM_GENERAL_6__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_6__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_7
+#define RLC_GPM_GENERAL_7__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_7__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_STATIC_PG_STATUS
+#define RLC_STATIC_PG_STATUS__PG_STATUS_WGP_MASK__SHIFT                                                       0x0
+#define RLC_STATIC_PG_STATUS__PG_STATUS_WGP_MASK_MASK                                                         0xFFFFFFFFL
+//RLC_GPM_GENERAL_16
+#define RLC_GPM_GENERAL_16__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_16__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_PG_DELAY_3
+#define RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT                                                        0x0
+#define RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK                                                          0xFFFFFFFFL
+//RLC_GPR_REG1
+#define RLC_GPR_REG1__DATA__SHIFT                                                                             0x0
+#define RLC_GPR_REG1__DATA_MASK                                                                               0xFFFFFFFFL
+//RLC_GPR_REG2
+#define RLC_GPR_REG2__DATA__SHIFT                                                                             0x0
+#define RLC_GPR_REG2__DATA_MASK                                                                               0xFFFFFFFFL
+//RLC_GPM_INT_DISABLE_TH0
+#define RLC_GPM_INT_DISABLE_TH0__DISABLE_INT__SHIFT                                                           0x0
+#define RLC_GPM_INT_DISABLE_TH0__DISABLE_INT_MASK                                                             0xFFFFFFFFL
+//RLC_GPM_LEGACY_INT_DISABLE
+#define RLC_GPM_LEGACY_INT_DISABLE__SPP_PVT_INT_CHANGED__SHIFT                                                0x0
+#define RLC_GPM_LEGACY_INT_DISABLE__RESERVED__SHIFT                                                           0x1
+#define RLC_GPM_LEGACY_INT_DISABLE__RLC_EOF_INT_CHANGED__SHIFT                                                0x2
+#define RLC_GPM_LEGACY_INT_DISABLE__RLC_PG_CNTL_CHANGED__SHIFT                                                0x3
+#define RLC_GPM_LEGACY_INT_DISABLE__STORE_LOAD_TIMER3_EXPIRED_T0__SHIFT                                       0x4
+#define RLC_GPM_LEGACY_INT_DISABLE__SPP_PVT_INT_CHANGED_MASK                                                  0x00000001L
+#define RLC_GPM_LEGACY_INT_DISABLE__RESERVED_MASK                                                             0x00000002L
+#define RLC_GPM_LEGACY_INT_DISABLE__RLC_EOF_INT_CHANGED_MASK                                                  0x00000004L
+#define RLC_GPM_LEGACY_INT_DISABLE__RLC_PG_CNTL_CHANGED_MASK                                                  0x00000008L
+#define RLC_GPM_LEGACY_INT_DISABLE__STORE_LOAD_TIMER3_EXPIRED_T0_MASK                                         0x00000010L
+//RLC_GPM_INT_FORCE_TH0
+#define RLC_GPM_INT_FORCE_TH0__FORCE_INT__SHIFT                                                               0x0
+#define RLC_GPM_INT_FORCE_TH0__FORCE_INT_MASK                                                                 0xFFFFFFFFL
+//RLC_SRM_CNTL
+#define RLC_SRM_CNTL__SRM_ENABLE__SHIFT                                                                       0x0
+#define RLC_SRM_CNTL__AUTO_INCR_ADDR__SHIFT                                                                   0x1
+#define RLC_SRM_CNTL__SRM_GPM_FIFO_RESET__SHIFT                                                               0x2
+#define RLC_SRM_CNTL__RESERVED__SHIFT                                                                         0x3
+#define RLC_SRM_CNTL__SRM_ENABLE_MASK                                                                         0x00000001L
+#define RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK                                                                     0x00000002L
+#define RLC_SRM_CNTL__SRM_GPM_FIFO_RESET_MASK                                                                 0x00000004L
+#define RLC_SRM_CNTL__RESERVED_MASK                                                                           0xFFFFFFF8L
+//RLC_SRM_GPM_COMMAND_STATUS
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_EMPTY__SHIFT                                                         0x0
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_FULL__SHIFT                                                          0x1
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_OVERFLOW__SHIFT                                                      0x2
+#define RLC_SRM_GPM_COMMAND_STATUS__RESERVED__SHIFT                                                           0x3
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_EMPTY_MASK                                                           0x00000001L
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_FULL_MASK                                                            0x00000002L
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_OVERFLOW_MASK                                                        0x00000004L
+#define RLC_SRM_GPM_COMMAND_STATUS__RESERVED_MASK                                                             0xFFFFFFF8L
+//RLC_SRM_INDEX_CNTL_ADDR_0
+#define RLC_SRM_INDEX_CNTL_ADDR_0__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_0__ADDRESS_MASK                                                               0x0003FFFFL
+//RLC_SRM_INDEX_CNTL_ADDR_1
+#define RLC_SRM_INDEX_CNTL_ADDR_1__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_1__ADDRESS_MASK                                                               0x0003FFFFL
+//RLC_SRM_INDEX_CNTL_ADDR_2
+#define RLC_SRM_INDEX_CNTL_ADDR_2__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_2__ADDRESS_MASK                                                               0x0003FFFFL
+//RLC_SRM_INDEX_CNTL_ADDR_3
+#define RLC_SRM_INDEX_CNTL_ADDR_3__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_3__ADDRESS_MASK                                                               0x0003FFFFL
+//RLC_SRM_INDEX_CNTL_ADDR_4
+#define RLC_SRM_INDEX_CNTL_ADDR_4__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_4__ADDRESS_MASK                                                               0x0003FFFFL
+//RLC_SRM_INDEX_CNTL_ADDR_5
+#define RLC_SRM_INDEX_CNTL_ADDR_5__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_5__ADDRESS_MASK                                                               0x0003FFFFL
+//RLC_SRM_INDEX_CNTL_ADDR_6
+#define RLC_SRM_INDEX_CNTL_ADDR_6__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_6__ADDRESS_MASK                                                               0x0003FFFFL
+//RLC_SRM_INDEX_CNTL_ADDR_7
+#define RLC_SRM_INDEX_CNTL_ADDR_7__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_7__ADDRESS_MASK                                                               0x0003FFFFL
+//RLC_SRM_INDEX_CNTL_DATA_0
+#define RLC_SRM_INDEX_CNTL_DATA_0__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_0__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_1
+#define RLC_SRM_INDEX_CNTL_DATA_1__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_1__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_2
+#define RLC_SRM_INDEX_CNTL_DATA_2__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_2__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_3
+#define RLC_SRM_INDEX_CNTL_DATA_3__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_3__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_4
+#define RLC_SRM_INDEX_CNTL_DATA_4__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_4__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_5
+#define RLC_SRM_INDEX_CNTL_DATA_5__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_5__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_6
+#define RLC_SRM_INDEX_CNTL_DATA_6__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_6__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_7
+#define RLC_SRM_INDEX_CNTL_DATA_7__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_7__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_STAT
+#define RLC_SRM_STAT__SRM_BUSY__SHIFT                                                                         0x0
+#define RLC_SRM_STAT__RESERVED__SHIFT                                                                         0x1
+#define RLC_SRM_STAT__SRM_BUSY_MASK                                                                           0x00000001L
+#define RLC_SRM_STAT__RESERVED_MASK                                                                           0xFFFFFFFEL
+//RLC_LX6_UTCL1_ERROR_2
+#define RLC_LX6_UTCL1_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                             0x0
+#define RLC_LX6_UTCL1_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                               0xFFFFFFFFL
+//RLC_GPM_GENERAL_8
+#define RLC_GPM_GENERAL_8__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_8__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_9
+#define RLC_GPM_GENERAL_9__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_9__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_10
+#define RLC_GPM_GENERAL_10__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_10__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GPM_GENERAL_11
+#define RLC_GPM_GENERAL_11__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_11__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GPM_GENERAL_12
+#define RLC_GPM_GENERAL_12__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_12__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GPM_UTCL1_CNTL_0
+#define RLC_GPM_UTCL1_CNTL_0__XNACK_REDO_TIMER_CNT__SHIFT                                                     0x0
+#define RLC_GPM_UTCL1_CNTL_0__RESERVED_23_20__SHIFT                                                           0x14
+#define RLC_GPM_UTCL1_CNTL_0__DROP_MODE__SHIFT                                                                0x18
+#define RLC_GPM_UTCL1_CNTL_0__BYPASS__SHIFT                                                                   0x19
+#define RLC_GPM_UTCL1_CNTL_0__INVALIDATE__SHIFT                                                               0x1a
+#define RLC_GPM_UTCL1_CNTL_0__FRAG_LIMIT_MODE__SHIFT                                                          0x1b
+#define RLC_GPM_UTCL1_CNTL_0__FORCE_SNOOP__SHIFT                                                              0x1c
+#define RLC_GPM_UTCL1_CNTL_0__RESERVED__SHIFT                                                                 0x1d
+#define RLC_GPM_UTCL1_CNTL_0__XNACK_REDO_TIMER_CNT_MASK                                                       0x000FFFFFL
+#define RLC_GPM_UTCL1_CNTL_0__RESERVED_23_20_MASK                                                             0x00F00000L
+#define RLC_GPM_UTCL1_CNTL_0__DROP_MODE_MASK                                                                  0x01000000L
+#define RLC_GPM_UTCL1_CNTL_0__BYPASS_MASK                                                                     0x02000000L
+#define RLC_GPM_UTCL1_CNTL_0__INVALIDATE_MASK                                                                 0x04000000L
+#define RLC_GPM_UTCL1_CNTL_0__FRAG_LIMIT_MODE_MASK                                                            0x08000000L
+#define RLC_GPM_UTCL1_CNTL_0__FORCE_SNOOP_MASK                                                                0x10000000L
+#define RLC_GPM_UTCL1_CNTL_0__RESERVED_MASK                                                                   0xE0000000L
+//RLC_SPM_UTCL1_CNTL
+#define RLC_SPM_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                       0x0
+#define RLC_SPM_UTCL1_CNTL__RESERVED_23_20__SHIFT                                                             0x14
+#define RLC_SPM_UTCL1_CNTL__DROP_MODE__SHIFT                                                                  0x18
+#define RLC_SPM_UTCL1_CNTL__BYPASS__SHIFT                                                                     0x19
+#define RLC_SPM_UTCL1_CNTL__INVALIDATE__SHIFT                                                                 0x1a
+#define RLC_SPM_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                            0x1b
+#define RLC_SPM_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                0x1c
+#define RLC_SPM_UTCL1_CNTL__RESERVED__SHIFT                                                                   0x1d
+#define RLC_SPM_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                         0x000FFFFFL
+#define RLC_SPM_UTCL1_CNTL__RESERVED_23_20_MASK                                                               0x00F00000L
+#define RLC_SPM_UTCL1_CNTL__DROP_MODE_MASK                                                                    0x01000000L
+#define RLC_SPM_UTCL1_CNTL__BYPASS_MASK                                                                       0x02000000L
+#define RLC_SPM_UTCL1_CNTL__INVALIDATE_MASK                                                                   0x04000000L
+#define RLC_SPM_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                              0x08000000L
+#define RLC_SPM_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                  0x10000000L
+#define RLC_SPM_UTCL1_CNTL__RESERVED_MASK                                                                     0xE0000000L
+//RLC_UTCL1_STATUS_2
+#define RLC_UTCL1_STATUS_2__GPM_TH0_UTCL1_BUSY__SHIFT                                                         0x0
+#define RLC_UTCL1_STATUS_2__GPM_TH1_UTCL1_BUSY__SHIFT                                                         0x1
+#define RLC_UTCL1_STATUS_2__GPM_TH2_UTCL1_BUSY__SHIFT                                                         0x2
+#define RLC_UTCL1_STATUS_2__SPM_UTCL1_BUSY__SHIFT                                                             0x3
+#define RLC_UTCL1_STATUS_2__LX6_UTCL1_BUSY__SHIFT                                                             0x4
+#define RLC_UTCL1_STATUS_2__DMA_UTCL1_BUSY__SHIFT                                                             0x5
+#define RLC_UTCL1_STATUS_2__SRM_UTCL1_BUSY__SHIFT                                                             0x6
+#define RLC_UTCL1_STATUS_2__DLG_UTCL1_BUSY__SHIFT                                                             0x7
+#define RLC_UTCL1_STATUS_2__GPM_TH0_UTCL1_StallOnTrans__SHIFT                                                 0x8
+#define RLC_UTCL1_STATUS_2__GPM_TH1_UTCL1_StallOnTrans__SHIFT                                                 0x9
+#define RLC_UTCL1_STATUS_2__GPM_TH2_UTCL1_StallOnTrans__SHIFT                                                 0xa
+#define RLC_UTCL1_STATUS_2__SPM_UTCL1_StallOnTrans__SHIFT                                                     0xb
+#define RLC_UTCL1_STATUS_2__LX6_UTCL1_StallOnTrans__SHIFT                                                     0xc
+#define RLC_UTCL1_STATUS_2__DMA_UTCL1_StallOnTrans__SHIFT                                                     0xd
+#define RLC_UTCL1_STATUS_2__SRM_UTCL1_StallOnTrans__SHIFT                                                     0xe
+#define RLC_UTCL1_STATUS_2__DLG_UTCL1_StallOnTrans__SHIFT                                                     0xf
+#define RLC_UTCL1_STATUS_2__RESERVED__SHIFT                                                                   0x10
+#define RLC_UTCL1_STATUS_2__GPM_TH0_UTCL1_BUSY_MASK                                                           0x00000001L
+#define RLC_UTCL1_STATUS_2__GPM_TH1_UTCL1_BUSY_MASK                                                           0x00000002L
+#define RLC_UTCL1_STATUS_2__GPM_TH2_UTCL1_BUSY_MASK                                                           0x00000004L
+#define RLC_UTCL1_STATUS_2__SPM_UTCL1_BUSY_MASK                                                               0x00000008L
+#define RLC_UTCL1_STATUS_2__LX6_UTCL1_BUSY_MASK                                                               0x00000010L
+#define RLC_UTCL1_STATUS_2__DMA_UTCL1_BUSY_MASK                                                               0x00000020L
+#define RLC_UTCL1_STATUS_2__SRM_UTCL1_BUSY_MASK                                                               0x00000040L
+#define RLC_UTCL1_STATUS_2__DLG_UTCL1_BUSY_MASK                                                               0x00000080L
+#define RLC_UTCL1_STATUS_2__GPM_TH0_UTCL1_StallOnTrans_MASK                                                   0x00000100L
+#define RLC_UTCL1_STATUS_2__GPM_TH1_UTCL1_StallOnTrans_MASK                                                   0x00000200L
+#define RLC_UTCL1_STATUS_2__GPM_TH2_UTCL1_StallOnTrans_MASK                                                   0x00000400L
+#define RLC_UTCL1_STATUS_2__SPM_UTCL1_StallOnTrans_MASK                                                       0x00000800L
+#define RLC_UTCL1_STATUS_2__LX6_UTCL1_StallOnTrans_MASK                                                       0x00001000L
+#define RLC_UTCL1_STATUS_2__DMA_UTCL1_StallOnTrans_MASK                                                       0x00002000L
+#define RLC_UTCL1_STATUS_2__SRM_UTCL1_StallOnTrans_MASK                                                       0x00004000L
+#define RLC_UTCL1_STATUS_2__DLG_UTCL1_StallOnTrans_MASK                                                       0x00008000L
+#define RLC_UTCL1_STATUS_2__RESERVED_MASK                                                                     0xFFFF0000L
+//RLC_SPM_UTCL1_ERROR_1
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqError__SHIFT                                                     0x0
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqErrorVmid__SHIFT                                                 0x2
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                             0x6
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqError_MASK                                                       0x00000003L
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqErrorVmid_MASK                                                   0x0000003CL
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                               0x0007FFC0L
+//RLC_SPM_UTCL1_ERROR_2
+#define RLC_SPM_UTCL1_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                             0x0
+#define RLC_SPM_UTCL1_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                               0xFFFFFFFFL
+//RLC_GPM_UTCL1_TH0_ERROR_1
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqError__SHIFT                                                 0x0
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqErrorVmid__SHIFT                                             0x2
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                         0x6
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqError_MASK                                                   0x00000003L
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqErrorVmid_MASK                                               0x0000003CL
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                           0x0007FFC0L
+//RLC_GPM_UTCL1_TH0_ERROR_2
+#define RLC_GPM_UTCL1_TH0_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                         0x0
+#define RLC_GPM_UTCL1_TH0_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                           0xFFFFFFFFL
+//RLC_SEMAPHORE_0
+#define RLC_SEMAPHORE_0__CLIENT_ID__SHIFT                                                                     0x0
+#define RLC_SEMAPHORE_0__CLIENT_ID_MASK                                                                       0x0000001FL
+//RLC_SEMAPHORE_1
+#define RLC_SEMAPHORE_1__CLIENT_ID__SHIFT                                                                     0x0
+#define RLC_SEMAPHORE_1__CLIENT_ID_MASK                                                                       0x0000001FL
+//RLC_SEMAPHORE_2
+#define RLC_SEMAPHORE_2__CLIENT_ID__SHIFT                                                                     0x0
+#define RLC_SEMAPHORE_2__CLIENT_ID_MASK                                                                       0x0000001FL
+//RLC_SEMAPHORE_3
+#define RLC_SEMAPHORE_3__CLIENT_ID__SHIFT                                                                     0x0
+#define RLC_SEMAPHORE_3__CLIENT_ID_MASK                                                                       0x0000001FL
+//RLC_SRM_UTCL1_CNTL
+#define RLC_SRM_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                       0x0
+#define RLC_SRM_UTCL1_CNTL__RESERVED_23_20__SHIFT                                                             0x14
+#define RLC_SRM_UTCL1_CNTL__DROP_MODE__SHIFT                                                                  0x18
+#define RLC_SRM_UTCL1_CNTL__BYPASS__SHIFT                                                                     0x19
+#define RLC_SRM_UTCL1_CNTL__INVALIDATE__SHIFT                                                                 0x1a
+#define RLC_SRM_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                            0x1b
+#define RLC_SRM_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                0x1c
+#define RLC_SRM_UTCL1_CNTL__RESERVED__SHIFT                                                                   0x1d
+#define RLC_SRM_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                         0x000FFFFFL
+#define RLC_SRM_UTCL1_CNTL__RESERVED_23_20_MASK                                                               0x00F00000L
+#define RLC_SRM_UTCL1_CNTL__DROP_MODE_MASK                                                                    0x01000000L
+#define RLC_SRM_UTCL1_CNTL__BYPASS_MASK                                                                       0x02000000L
+#define RLC_SRM_UTCL1_CNTL__INVALIDATE_MASK                                                                   0x04000000L
+#define RLC_SRM_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                              0x08000000L
+#define RLC_SRM_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                  0x10000000L
+#define RLC_SRM_UTCL1_CNTL__RESERVED_MASK                                                                     0xE0000000L
+//RLC_SRM_UTCL1_ERROR_1
+#define RLC_SRM_UTCL1_ERROR_1__Translated_ReqError__SHIFT                                                     0x0
+#define RLC_SRM_UTCL1_ERROR_1__Translated_ReqErrorVmid__SHIFT                                                 0x2
+#define RLC_SRM_UTCL1_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                             0x6
+#define RLC_SRM_UTCL1_ERROR_1__Translated_ReqError_MASK                                                       0x00000003L
+#define RLC_SRM_UTCL1_ERROR_1__Translated_ReqErrorVmid_MASK                                                   0x0000003CL
+#define RLC_SRM_UTCL1_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                               0x0007FFC0L
+//RLC_SRM_UTCL1_ERROR_2
+#define RLC_SRM_UTCL1_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                             0x0
+#define RLC_SRM_UTCL1_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                               0xFFFFFFFFL
+//RLC_DLG_UTCL1_ERROR_2
+#define RLC_DLG_UTCL1_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                             0x0
+#define RLC_DLG_UTCL1_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                               0xFFFFFFFFL
+//RLC_UTCL1_STATUS
+#define RLC_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define RLC_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define RLC_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define RLC_UTCL1_STATUS__RESERVED__SHIFT                                                                     0x3
+#define RLC_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                0x8
+#define RLC_UTCL1_STATUS__RESERVED_1__SHIFT                                                                   0xe
+#define RLC_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                0x10
+#define RLC_UTCL1_STATUS__RESERVED_2__SHIFT                                                                   0x16
+#define RLC_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                  0x18
+#define RLC_UTCL1_STATUS__RESERVED_3__SHIFT                                                                   0x1e
+#define RLC_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define RLC_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define RLC_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+#define RLC_UTCL1_STATUS__RESERVED_MASK                                                                       0x000000F8L
+#define RLC_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                  0x00003F00L
+#define RLC_UTCL1_STATUS__RESERVED_1_MASK                                                                     0x0000C000L
+#define RLC_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                  0x003F0000L
+#define RLC_UTCL1_STATUS__RESERVED_2_MASK                                                                     0x00C00000L
+#define RLC_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                    0x3F000000L
+#define RLC_UTCL1_STATUS__RESERVED_3_MASK                                                                     0xC0000000L
+//RLC_R2I_CNTL_0
+#define RLC_R2I_CNTL_0__Data__SHIFT                                                                           0x0
+#define RLC_R2I_CNTL_0__Data_MASK                                                                             0xFFFFFFFFL
+//RLC_R2I_CNTL_1
+#define RLC_R2I_CNTL_1__Data__SHIFT                                                                           0x0
+#define RLC_R2I_CNTL_1__Data_MASK                                                                             0xFFFFFFFFL
+//RLC_R2I_CNTL_2
+#define RLC_R2I_CNTL_2__Data__SHIFT                                                                           0x0
+#define RLC_R2I_CNTL_2__Data_MASK                                                                             0xFFFFFFFFL
+//RLC_R2I_CNTL_3
+#define RLC_R2I_CNTL_3__Data__SHIFT                                                                           0x0
+#define RLC_R2I_CNTL_3__Data_MASK                                                                             0xFFFFFFFFL
+//RLC_GPM_INT_STAT_TH0
+#define RLC_GPM_INT_STAT_TH0__STATUS__SHIFT                                                                   0x0
+#define RLC_GPM_INT_STAT_TH0__STATUS_MASK                                                                     0xFFFFFFFFL
+//RLC_GPM_GENERAL_13
+#define RLC_GPM_GENERAL_13__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_13__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GPM_GENERAL_14
+#define RLC_GPM_GENERAL_14__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_14__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GPM_GENERAL_15
+#define RLC_GPM_GENERAL_15__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_15__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_LX6_UTCL1_ERROR_1
+#define RLC_LX6_UTCL1_ERROR_1__Translated_ReqError__SHIFT                                                     0x0
+#define RLC_LX6_UTCL1_ERROR_1__Translated_ReqErrorVmid__SHIFT                                                 0x2
+#define RLC_LX6_UTCL1_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                             0x6
+#define RLC_LX6_UTCL1_ERROR_1__Translated_ReqError_MASK                                                       0x00000003L
+#define RLC_LX6_UTCL1_ERROR_1__Translated_ReqErrorVmid_MASK                                                   0x0000003CL
+#define RLC_LX6_UTCL1_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                               0x0007FFC0L
+//RLC_LX6_UTCL1_CNTL
+#define RLC_LX6_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                       0x0
+#define RLC_LX6_UTCL1_CNTL__RESERVED_23_20__SHIFT                                                             0x14
+#define RLC_LX6_UTCL1_CNTL__DROP_MODE__SHIFT                                                                  0x18
+#define RLC_LX6_UTCL1_CNTL__BYPASS__SHIFT                                                                     0x19
+#define RLC_LX6_UTCL1_CNTL__INVALIDATE__SHIFT                                                                 0x1a
+#define RLC_LX6_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                            0x1b
+#define RLC_LX6_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                0x1c
+#define RLC_LX6_UTCL1_CNTL__RESERVED__SHIFT                                                                   0x1d
+#define RLC_LX6_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                         0x000FFFFFL
+#define RLC_LX6_UTCL1_CNTL__RESERVED_23_20_MASK                                                               0x00F00000L
+#define RLC_LX6_UTCL1_CNTL__DROP_MODE_MASK                                                                    0x01000000L
+#define RLC_LX6_UTCL1_CNTL__BYPASS_MASK                                                                       0x02000000L
+#define RLC_LX6_UTCL1_CNTL__INVALIDATE_MASK                                                                   0x04000000L
+#define RLC_LX6_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                              0x08000000L
+#define RLC_LX6_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                  0x10000000L
+#define RLC_LX6_UTCL1_CNTL__RESERVED_MASK                                                                     0xE0000000L
+//RLC_DMA_UTCL1_ERROR_1
+#define RLC_DMA_UTCL1_ERROR_1__Translated_ReqError__SHIFT                                                     0x0
+#define RLC_DMA_UTCL1_ERROR_1__Translated_ReqErrorVmid__SHIFT                                                 0x2
+#define RLC_DMA_UTCL1_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                             0x6
+#define RLC_DMA_UTCL1_ERROR_1__Translated_ReqError_MASK                                                       0x00000003L
+#define RLC_DMA_UTCL1_ERROR_1__Translated_ReqErrorVmid_MASK                                                   0x0000003CL
+#define RLC_DMA_UTCL1_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                               0x0007FFC0L
+//RLC_DMA_UTCL1_ERROR_2
+#define RLC_DMA_UTCL1_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                             0x0
+#define RLC_DMA_UTCL1_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                               0xFFFFFFFFL
+//RLC_CAPTURE_GPU_CLOCK_COUNT_1
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_1__CAPTURE__SHIFT                                                         0x0
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_1__RESERVED__SHIFT                                                        0x1
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_1__CAPTURE_MASK                                                           0x00000001L
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_1__RESERVED_MASK                                                          0xFFFFFFFEL
+//RLC_GPU_CLOCK_COUNT_LSB_2
+#define RLC_GPU_CLOCK_COUNT_LSB_2__GPU_CLOCKS_LSB__SHIFT                                                      0x0
+#define RLC_GPU_CLOCK_COUNT_LSB_2__GPU_CLOCKS_LSB_MASK                                                        0xFFFFFFFFL
+//RLC_GPU_CLOCK_COUNT_MSB_2
+#define RLC_GPU_CLOCK_COUNT_MSB_2__GPU_CLOCKS_MSB__SHIFT                                                      0x0
+#define RLC_GPU_CLOCK_COUNT_MSB_2__GPU_CLOCKS_MSB_MASK                                                        0xFFFFFFFFL
+//RLC_CAPTURE_GPU_CLOCK_COUNT_2
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_2__CAPTURE__SHIFT                                                         0x0
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_2__RESERVED__SHIFT                                                        0x1
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_2__CAPTURE_MASK                                                           0x00000001L
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_2__RESERVED_MASK                                                          0xFFFFFFFEL
+//RLC_RLCV_DOORBELL_RANGE
+#define RLC_RLCV_DOORBELL_RANGE__LOWER_ADDR_RESERVED__SHIFT                                                   0x0
+#define RLC_RLCV_DOORBELL_RANGE__LOWER_ADDR__SHIFT                                                            0x2
+#define RLC_RLCV_DOORBELL_RANGE__UPPER_ADDR_RESERVED__SHIFT                                                   0x10
+#define RLC_RLCV_DOORBELL_RANGE__UPPER_ADDR__SHIFT                                                            0x12
+#define RLC_RLCV_DOORBELL_RANGE__RESERVED__SHIFT                                                              0x1c
+#define RLC_RLCV_DOORBELL_RANGE__LOWER_ADDR_RESERVED_MASK                                                     0x00000003L
+#define RLC_RLCV_DOORBELL_RANGE__LOWER_ADDR_MASK                                                              0x00000FFCL
+#define RLC_RLCV_DOORBELL_RANGE__UPPER_ADDR_RESERVED_MASK                                                     0x00030000L
+#define RLC_RLCV_DOORBELL_RANGE__UPPER_ADDR_MASK                                                              0x0FFC0000L
+#define RLC_RLCV_DOORBELL_RANGE__RESERVED_MASK                                                                0xF0000000L
+//RLC_RLCV_DOORBELL_CNTL
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_0_MODE__SHIFT                                                        0x0
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_1_MODE__SHIFT                                                        0x2
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_2_MODE__SHIFT                                                        0x4
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_3_MODE__SHIFT                                                        0x6
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_ID__SHIFT                                                            0x10
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_ID_EN__SHIFT                                                         0x15
+#define RLC_RLCV_DOORBELL_CNTL__RESERVED__SHIFT                                                               0x16
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_0_MODE_MASK                                                          0x00000003L
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_1_MODE_MASK                                                          0x0000000CL
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_2_MODE_MASK                                                          0x00000030L
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_3_MODE_MASK                                                          0x000000C0L
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_ID_MASK                                                              0x001F0000L
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_ID_EN_MASK                                                           0x00200000L
+#define RLC_RLCV_DOORBELL_CNTL__RESERVED_MASK                                                                 0xFFC00000L
+//RLC_RLCV_DOORBELL_STAT
+#define RLC_RLCV_DOORBELL_STAT__DOORBELL_0_VALID__SHIFT                                                       0x0
+#define RLC_RLCV_DOORBELL_STAT__DOORBELL_1_VALID__SHIFT                                                       0x1
+#define RLC_RLCV_DOORBELL_STAT__DOORBELL_2_VALID__SHIFT                                                       0x2
+#define RLC_RLCV_DOORBELL_STAT__DOORBELL_3_VALID__SHIFT                                                       0x3
+#define RLC_RLCV_DOORBELL_STAT__DOORBELL_0_VALID_MASK                                                         0x00000001L
+#define RLC_RLCV_DOORBELL_STAT__DOORBELL_1_VALID_MASK                                                         0x00000002L
+#define RLC_RLCV_DOORBELL_STAT__DOORBELL_2_VALID_MASK                                                         0x00000004L
+#define RLC_RLCV_DOORBELL_STAT__DOORBELL_3_VALID_MASK                                                         0x00000008L
+//RLC_RLCV_DOORBELL_0_DATA_LO
+#define RLC_RLCV_DOORBELL_0_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCV_DOORBELL_0_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCV_DOORBELL_0_DATA_HI
+#define RLC_RLCV_DOORBELL_0_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCV_DOORBELL_0_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCV_DOORBELL_1_DATA_LO
+#define RLC_RLCV_DOORBELL_1_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCV_DOORBELL_1_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCV_DOORBELL_1_DATA_HI
+#define RLC_RLCV_DOORBELL_1_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCV_DOORBELL_1_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCV_DOORBELL_2_DATA_LO
+#define RLC_RLCV_DOORBELL_2_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCV_DOORBELL_2_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCV_DOORBELL_2_DATA_HI
+#define RLC_RLCV_DOORBELL_2_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCV_DOORBELL_2_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCV_DOORBELL_3_DATA_LO
+#define RLC_RLCV_DOORBELL_3_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCV_DOORBELL_3_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCV_DOORBELL_3_DATA_HI
+#define RLC_RLCV_DOORBELL_3_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCV_DOORBELL_3_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_GPU_CLOCK_COUNT_LSB_1
+#define RLC_GPU_CLOCK_COUNT_LSB_1__GPU_CLOCKS_LSB__SHIFT                                                      0x0
+#define RLC_GPU_CLOCK_COUNT_LSB_1__GPU_CLOCKS_LSB_MASK                                                        0xFFFFFFFFL
+//RLC_GPU_CLOCK_COUNT_MSB_1
+#define RLC_GPU_CLOCK_COUNT_MSB_1__GPU_CLOCKS_MSB__SHIFT                                                      0x0
+#define RLC_GPU_CLOCK_COUNT_MSB_1__GPU_CLOCKS_MSB_MASK                                                        0xFFFFFFFFL
+//RLC_RLCV_SPARE_INT
+#define RLC_RLCV_SPARE_INT__INTERRUPT__SHIFT                                                                  0x0
+#define RLC_RLCV_SPARE_INT__RESERVED__SHIFT                                                                   0x1
+#define RLC_RLCV_SPARE_INT__INTERRUPT_MASK                                                                    0x00000001L
+#define RLC_RLCV_SPARE_INT__RESERVED_MASK                                                                     0xFFFFFFFEL
+//RLC_SMU_CLK_REQ
+#define RLC_SMU_CLK_REQ__VALID__SHIFT                                                                         0x0
+#define RLC_SMU_CLK_REQ__VALID_MASK                                                                           0x00000001L
+//RLC_SPARE
+#define RLC_SPARE__SPARE__SHIFT                                                                               0x0
+#define RLC_SPARE__SPARE_MASK                                                                                 0xFFFFFFFFL
+//RLC_SPP_CTRL
+#define RLC_SPP_CTRL__ENABLE__SHIFT                                                                           0x0
+#define RLC_SPP_CTRL__ENABLE_PPROF__SHIFT                                                                     0x1
+#define RLC_SPP_CTRL__ENABLE_PWR_OPT__SHIFT                                                                   0x2
+#define RLC_SPP_CTRL__PAUSE__SHIFT                                                                            0x3
+#define RLC_SPP_CTRL__ENABLE_MASK                                                                             0x00000001L
+#define RLC_SPP_CTRL__ENABLE_PPROF_MASK                                                                       0x00000002L
+#define RLC_SPP_CTRL__ENABLE_PWR_OPT_MASK                                                                     0x00000004L
+#define RLC_SPP_CTRL__PAUSE_MASK                                                                              0x00000008L
+//RLC_SPP_SHADER_PROFILE_EN
+#define RLC_SPP_SHADER_PROFILE_EN__PS_ENABLE__SHIFT                                                           0x0
+#define RLC_SPP_SHADER_PROFILE_EN__GS_ENABLE__SHIFT                                                           0x2
+#define RLC_SPP_SHADER_PROFILE_EN__HS_ENABLE__SHIFT                                                           0x3
+#define RLC_SPP_SHADER_PROFILE_EN__CSG_ENABLE__SHIFT                                                          0x4
+#define RLC_SPP_SHADER_PROFILE_EN__CS_ENABLE__SHIFT                                                           0x5
+#define RLC_SPP_SHADER_PROFILE_EN__PS_START_CONDITION__SHIFT                                                  0xb
+#define RLC_SPP_SHADER_PROFILE_EN__CSG_START_CONDITION__SHIFT                                                 0xc
+#define RLC_SPP_SHADER_PROFILE_EN__CS_START_CONDITION__SHIFT                                                  0xd
+#define RLC_SPP_SHADER_PROFILE_EN__FORCE_MISS__SHIFT                                                          0xe
+#define RLC_SPP_SHADER_PROFILE_EN__FORCE_UNLOCKED__SHIFT                                                      0xf
+#define RLC_SPP_SHADER_PROFILE_EN__ENABLE_PROF_INFO_LOCK__SHIFT                                               0x10
+#define RLC_SPP_SHADER_PROFILE_EN__PS_ENABLE_MASK                                                             0x00000001L
+#define RLC_SPP_SHADER_PROFILE_EN__GS_ENABLE_MASK                                                             0x00000004L
+#define RLC_SPP_SHADER_PROFILE_EN__HS_ENABLE_MASK                                                             0x00000008L
+#define RLC_SPP_SHADER_PROFILE_EN__CSG_ENABLE_MASK                                                            0x00000010L
+#define RLC_SPP_SHADER_PROFILE_EN__CS_ENABLE_MASK                                                             0x00000020L
+#define RLC_SPP_SHADER_PROFILE_EN__PS_START_CONDITION_MASK                                                    0x00000800L
+#define RLC_SPP_SHADER_PROFILE_EN__CSG_START_CONDITION_MASK                                                   0x00001000L
+#define RLC_SPP_SHADER_PROFILE_EN__CS_START_CONDITION_MASK                                                    0x00002000L
+#define RLC_SPP_SHADER_PROFILE_EN__FORCE_MISS_MASK                                                            0x00004000L
+#define RLC_SPP_SHADER_PROFILE_EN__FORCE_UNLOCKED_MASK                                                        0x00008000L
+#define RLC_SPP_SHADER_PROFILE_EN__ENABLE_PROF_INFO_LOCK_MASK                                                 0x00010000L
+//RLC_SPP_SSF_CAPTURE_EN
+#define RLC_SPP_SSF_CAPTURE_EN__PS_ENABLE__SHIFT                                                              0x0
+#define RLC_SPP_SSF_CAPTURE_EN__GS_ENABLE__SHIFT                                                              0x2
+#define RLC_SPP_SSF_CAPTURE_EN__HS_ENABLE__SHIFT                                                              0x3
+#define RLC_SPP_SSF_CAPTURE_EN__CSG_ENABLE__SHIFT                                                             0x4
+#define RLC_SPP_SSF_CAPTURE_EN__CS_ENABLE__SHIFT                                                              0x5
+#define RLC_SPP_SSF_CAPTURE_EN__PS_ENABLE_MASK                                                                0x00000001L
+#define RLC_SPP_SSF_CAPTURE_EN__GS_ENABLE_MASK                                                                0x00000004L
+#define RLC_SPP_SSF_CAPTURE_EN__HS_ENABLE_MASK                                                                0x00000008L
+#define RLC_SPP_SSF_CAPTURE_EN__CSG_ENABLE_MASK                                                               0x00000010L
+#define RLC_SPP_SSF_CAPTURE_EN__CS_ENABLE_MASK                                                                0x00000020L
+//RLC_SPP_SSF_THRESHOLD_0
+#define RLC_SPP_SSF_THRESHOLD_0__PS_THRESHOLD__SHIFT                                                          0x0
+#define RLC_SPP_SSF_THRESHOLD_0__RESERVED__SHIFT                                                              0x10
+#define RLC_SPP_SSF_THRESHOLD_0__PS_THRESHOLD_MASK                                                            0x0000FFFFL
+#define RLC_SPP_SSF_THRESHOLD_0__RESERVED_MASK                                                                0xFFFF0000L
+//RLC_SPP_SSF_THRESHOLD_1
+#define RLC_SPP_SSF_THRESHOLD_1__GS_THRESHOLD__SHIFT                                                          0x0
+#define RLC_SPP_SSF_THRESHOLD_1__HS_THRESHOLD__SHIFT                                                          0x10
+#define RLC_SPP_SSF_THRESHOLD_1__GS_THRESHOLD_MASK                                                            0x0000FFFFL
+#define RLC_SPP_SSF_THRESHOLD_1__HS_THRESHOLD_MASK                                                            0xFFFF0000L
+//RLC_SPP_SSF_THRESHOLD_2
+#define RLC_SPP_SSF_THRESHOLD_2__CSG_THRESHOLD__SHIFT                                                         0x0
+#define RLC_SPP_SSF_THRESHOLD_2__CS_THRESHOLD__SHIFT                                                          0x10
+#define RLC_SPP_SSF_THRESHOLD_2__CSG_THRESHOLD_MASK                                                           0x0000FFFFL
+#define RLC_SPP_SSF_THRESHOLD_2__CS_THRESHOLD_MASK                                                            0xFFFF0000L
+//RLC_SPP_INFLIGHT_RD_ADDR
+#define RLC_SPP_INFLIGHT_RD_ADDR__ADDR__SHIFT                                                                 0x0
+#define RLC_SPP_INFLIGHT_RD_ADDR__ADDR_MASK                                                                   0x0000001FL
+//RLC_SPP_INFLIGHT_RD_DATA
+#define RLC_SPP_INFLIGHT_RD_DATA__DATA__SHIFT                                                                 0x0
+#define RLC_SPP_INFLIGHT_RD_DATA__DATA_MASK                                                                   0xFFFFFFFFL
+//RLC_SPP_PROF_INFO_1
+#define RLC_SPP_PROF_INFO_1__SH_ID__SHIFT                                                                     0x0
+#define RLC_SPP_PROF_INFO_1__SH_ID_MASK                                                                       0xFFFFFFFFL
+//RLC_SPP_PROF_INFO_2
+#define RLC_SPP_PROF_INFO_2__SH_TYPE__SHIFT                                                                   0x0
+#define RLC_SPP_PROF_INFO_2__CAM_HIT__SHIFT                                                                   0x5
+#define RLC_SPP_PROF_INFO_2__CAM_LOCK__SHIFT                                                                  0x6
+#define RLC_SPP_PROF_INFO_2__CAM_CONFLICT__SHIFT                                                              0x7
+#define RLC_SPP_PROF_INFO_2__SH_TYPE_MASK                                                                     0x0000001FL
+#define RLC_SPP_PROF_INFO_2__CAM_HIT_MASK                                                                     0x00000020L
+#define RLC_SPP_PROF_INFO_2__CAM_LOCK_MASK                                                                    0x00000040L
+#define RLC_SPP_PROF_INFO_2__CAM_CONFLICT_MASK                                                                0x00000080L
+//RLC_SPP_GLOBAL_SH_ID
+#define RLC_SPP_GLOBAL_SH_ID__SH_ID__SHIFT                                                                    0x0
+#define RLC_SPP_GLOBAL_SH_ID__SH_ID_MASK                                                                      0xFFFFFFFFL
+//RLC_SPP_GLOBAL_SH_ID_VALID
+#define RLC_SPP_GLOBAL_SH_ID_VALID__VALID__SHIFT                                                              0x0
+#define RLC_SPP_GLOBAL_SH_ID_VALID__VALID_MASK                                                                0x00000001L
+//RLC_SPP_STATUS
+#define RLC_SPP_STATUS__RESERVED_0__SHIFT                                                                     0x0
+#define RLC_SPP_STATUS__SSF_BUSY__SHIFT                                                                       0x1
+#define RLC_SPP_STATUS__EVENT_ARB_BUSY__SHIFT                                                                 0x2
+#define RLC_SPP_STATUS__SPP_BUSY__SHIFT                                                                       0x1f
+#define RLC_SPP_STATUS__RESERVED_0_MASK                                                                       0x00000001L
+#define RLC_SPP_STATUS__SSF_BUSY_MASK                                                                         0x00000002L
+#define RLC_SPP_STATUS__EVENT_ARB_BUSY_MASK                                                                   0x00000004L
+#define RLC_SPP_STATUS__SPP_BUSY_MASK                                                                         0x80000000L
+//RLC_SPP_PVT_STAT_0
+#define RLC_SPP_PVT_STAT_0__LEVEL_0_COUNTER__SHIFT                                                            0x0
+#define RLC_SPP_PVT_STAT_0__LEVEL_1_COUNTER__SHIFT                                                            0x8
+#define RLC_SPP_PVT_STAT_0__LEVEL_2_COUNTER__SHIFT                                                            0x10
+#define RLC_SPP_PVT_STAT_0__LEVEL_3_COUNTER__SHIFT                                                            0x18
+#define RLC_SPP_PVT_STAT_0__LEVEL_0_COUNTER_MASK                                                              0x0000007FL
+#define RLC_SPP_PVT_STAT_0__LEVEL_1_COUNTER_MASK                                                              0x00007F00L
+#define RLC_SPP_PVT_STAT_0__LEVEL_2_COUNTER_MASK                                                              0x007F0000L
+#define RLC_SPP_PVT_STAT_0__LEVEL_3_COUNTER_MASK                                                              0x7F000000L
+//RLC_SPP_PVT_STAT_1
+#define RLC_SPP_PVT_STAT_1__LEVEL_4_COUNTER__SHIFT                                                            0x0
+#define RLC_SPP_PVT_STAT_1__LEVEL_5_COUNTER__SHIFT                                                            0x8
+#define RLC_SPP_PVT_STAT_1__LEVEL_6_COUNTER__SHIFT                                                            0x10
+#define RLC_SPP_PVT_STAT_1__LEVEL_7_COUNTER__SHIFT                                                            0x18
+#define RLC_SPP_PVT_STAT_1__LEVEL_4_COUNTER_MASK                                                              0x0000007FL
+#define RLC_SPP_PVT_STAT_1__LEVEL_5_COUNTER_MASK                                                              0x00007F00L
+#define RLC_SPP_PVT_STAT_1__LEVEL_6_COUNTER_MASK                                                              0x007F0000L
+#define RLC_SPP_PVT_STAT_1__LEVEL_7_COUNTER_MASK                                                              0x7F000000L
+//RLC_SPP_PVT_STAT_2
+#define RLC_SPP_PVT_STAT_2__LEVEL_8_COUNTER__SHIFT                                                            0x0
+#define RLC_SPP_PVT_STAT_2__LEVEL_9_COUNTER__SHIFT                                                            0x8
+#define RLC_SPP_PVT_STAT_2__LEVEL_10_COUNTER__SHIFT                                                           0x10
+#define RLC_SPP_PVT_STAT_2__LEVEL_11_COUNTER__SHIFT                                                           0x18
+#define RLC_SPP_PVT_STAT_2__LEVEL_8_COUNTER_MASK                                                              0x0000007FL
+#define RLC_SPP_PVT_STAT_2__LEVEL_9_COUNTER_MASK                                                              0x00007F00L
+#define RLC_SPP_PVT_STAT_2__LEVEL_10_COUNTER_MASK                                                             0x007F0000L
+#define RLC_SPP_PVT_STAT_2__LEVEL_11_COUNTER_MASK                                                             0x7F000000L
+//RLC_SPP_PVT_STAT_3
+#define RLC_SPP_PVT_STAT_3__LEVEL_12_COUNTER__SHIFT                                                           0x0
+#define RLC_SPP_PVT_STAT_3__LEVEL_13_COUNTER__SHIFT                                                           0x8
+#define RLC_SPP_PVT_STAT_3__LEVEL_14_COUNTER__SHIFT                                                           0x10
+#define RLC_SPP_PVT_STAT_3__LEVEL_15_COUNTER__SHIFT                                                           0x18
+#define RLC_SPP_PVT_STAT_3__LEVEL_12_COUNTER_MASK                                                             0x0000007FL
+#define RLC_SPP_PVT_STAT_3__LEVEL_13_COUNTER_MASK                                                             0x00007F00L
+#define RLC_SPP_PVT_STAT_3__LEVEL_14_COUNTER_MASK                                                             0x007F0000L
+#define RLC_SPP_PVT_STAT_3__LEVEL_15_COUNTER_MASK                                                             0x7F000000L
+//RLC_SPP_PVT_LEVEL_MAX
+#define RLC_SPP_PVT_LEVEL_MAX__LEVEL__SHIFT                                                                   0x0
+#define RLC_SPP_PVT_LEVEL_MAX__LEVEL_MASK                                                                     0x0000000FL
+//RLC_SPP_STALL_STATE_UPDATE
+#define RLC_SPP_STALL_STATE_UPDATE__STALL__SHIFT                                                              0x0
+#define RLC_SPP_STALL_STATE_UPDATE__ENABLE__SHIFT                                                             0x1
+#define RLC_SPP_STALL_STATE_UPDATE__STALL_MASK                                                                0x00000001L
+#define RLC_SPP_STALL_STATE_UPDATE__ENABLE_MASK                                                               0x00000002L
+//RLC_SPP_PBB_INFO
+#define RLC_SPP_PBB_INFO__PIPE0_OVERRIDE__SHIFT                                                               0x0
+#define RLC_SPP_PBB_INFO__PIPE0_OVERRIDE_VALID__SHIFT                                                         0x1
+#define RLC_SPP_PBB_INFO__PIPE1_OVERRIDE__SHIFT                                                               0x2
+#define RLC_SPP_PBB_INFO__PIPE1_OVERRIDE_VALID__SHIFT                                                         0x3
+#define RLC_SPP_PBB_INFO__PIPE0_OVERRIDE_MASK                                                                 0x00000001L
+#define RLC_SPP_PBB_INFO__PIPE0_OVERRIDE_VALID_MASK                                                           0x00000002L
+#define RLC_SPP_PBB_INFO__PIPE1_OVERRIDE_MASK                                                                 0x00000004L
+#define RLC_SPP_PBB_INFO__PIPE1_OVERRIDE_VALID_MASK                                                           0x00000008L
+//RLC_SPP_RESET
+#define RLC_SPP_RESET__SSF_RESET__SHIFT                                                                       0x0
+#define RLC_SPP_RESET__EVENT_ARB_RESET__SHIFT                                                                 0x1
+#define RLC_SPP_RESET__CAM_RESET__SHIFT                                                                       0x2
+#define RLC_SPP_RESET__PVT_RESET__SHIFT                                                                       0x3
+#define RLC_SPP_RESET__SSF_RESET_MASK                                                                         0x00000001L
+#define RLC_SPP_RESET__EVENT_ARB_RESET_MASK                                                                   0x00000002L
+#define RLC_SPP_RESET__CAM_RESET_MASK                                                                         0x00000004L
+#define RLC_SPP_RESET__PVT_RESET_MASK                                                                         0x00000008L
+//RLC_CAC_MASK_CNTL
+#define RLC_CAC_MASK_CNTL__RLC_CAC_MASK__SHIFT                                                                0x0
+#define RLC_CAC_MASK_CNTL__RLC_CAC_MASK_MASK                                                                  0xFFFFFFFFL
+//RLC_POWER_RESIDENCY_CNTR_CTRL
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__RESET__SHIFT                                                           0x0
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__ENABLE__SHIFT                                                          0x1
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__RESET_ACK__SHIFT                                                       0x2
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__ENABLE_ACK__SHIFT                                                      0x3
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW__SHIFT                                                0x4
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__RESERVED__SHIFT                                                        0x5
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__RESET_MASK                                                             0x00000001L
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__ENABLE_MASK                                                            0x00000002L
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__RESET_ACK_MASK                                                         0x00000004L
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__ENABLE_ACK_MASK                                                        0x00000008L
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW_MASK                                                  0x00000010L
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__RESERVED_MASK                                                          0xFFFFFFE0L
+//RLC_CLK_RESIDENCY_CNTR_CTRL
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__RESET__SHIFT                                                             0x0
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__ENABLE__SHIFT                                                            0x1
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__RESET_ACK__SHIFT                                                         0x2
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__ENABLE_ACK__SHIFT                                                        0x3
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW__SHIFT                                                  0x4
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__RESERVED__SHIFT                                                          0x5
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__RESET_MASK                                                               0x00000001L
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__ENABLE_MASK                                                              0x00000002L
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__RESET_ACK_MASK                                                           0x00000004L
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__ENABLE_ACK_MASK                                                          0x00000008L
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW_MASK                                                    0x00000010L
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__RESERVED_MASK                                                            0xFFFFFFE0L
+//RLC_DS_RESIDENCY_CNTR_CTRL
+#define RLC_DS_RESIDENCY_CNTR_CTRL__RESET__SHIFT                                                              0x0
+#define RLC_DS_RESIDENCY_CNTR_CTRL__ENABLE__SHIFT                                                             0x1
+#define RLC_DS_RESIDENCY_CNTR_CTRL__RESET_ACK__SHIFT                                                          0x2
+#define RLC_DS_RESIDENCY_CNTR_CTRL__ENABLE_ACK__SHIFT                                                         0x3
+#define RLC_DS_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW__SHIFT                                                   0x4
+#define RLC_DS_RESIDENCY_CNTR_CTRL__RESERVED__SHIFT                                                           0x5
+#define RLC_DS_RESIDENCY_CNTR_CTRL__RESET_MASK                                                                0x00000001L
+#define RLC_DS_RESIDENCY_CNTR_CTRL__ENABLE_MASK                                                               0x00000002L
+#define RLC_DS_RESIDENCY_CNTR_CTRL__RESET_ACK_MASK                                                            0x00000004L
+#define RLC_DS_RESIDENCY_CNTR_CTRL__ENABLE_ACK_MASK                                                           0x00000008L
+#define RLC_DS_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW_MASK                                                     0x00000010L
+#define RLC_DS_RESIDENCY_CNTR_CTRL__RESERVED_MASK                                                             0xFFFFFFE0L
+//RLC_ULV_RESIDENCY_CNTR_CTRL
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__RESET__SHIFT                                                             0x0
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__ENABLE__SHIFT                                                            0x1
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__RESET_ACK__SHIFT                                                         0x2
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__ENABLE_ACK__SHIFT                                                        0x3
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW__SHIFT                                                  0x4
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__RESERVED__SHIFT                                                          0x5
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__RESET_MASK                                                               0x00000001L
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__ENABLE_MASK                                                              0x00000002L
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__RESET_ACK_MASK                                                           0x00000004L
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__ENABLE_ACK_MASK                                                          0x00000008L
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW_MASK                                                    0x00000010L
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__RESERVED_MASK                                                            0xFFFFFFE0L
+//RLC_PCC_RESIDENCY_CNTR_CTRL
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__RESET__SHIFT                                                             0x0
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__ENABLE__SHIFT                                                            0x1
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__RESET_ACK__SHIFT                                                         0x2
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__ENABLE_ACK__SHIFT                                                        0x3
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW__SHIFT                                                  0x4
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__RESERVED__SHIFT                                                          0x5
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__RESET_MASK                                                               0x00000001L
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__ENABLE_MASK                                                              0x00000002L
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__RESET_ACK_MASK                                                           0x00000004L
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__ENABLE_ACK_MASK                                                          0x00000008L
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW_MASK                                                    0x00000010L
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__RESERVED_MASK                                                            0xFFFFFFE0L
+//RLC_GENERAL_RESIDENCY_CNTR_CTRL
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__RESET__SHIFT                                                         0x0
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__ENABLE__SHIFT                                                        0x1
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__RESET_ACK__SHIFT                                                     0x2
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__ENABLE_ACK__SHIFT                                                    0x3
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW__SHIFT                                              0x4
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__RESERVED__SHIFT                                                      0x5
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__RESET_MASK                                                           0x00000001L
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__ENABLE_MASK                                                          0x00000002L
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__RESET_ACK_MASK                                                       0x00000004L
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__ENABLE_ACK_MASK                                                      0x00000008L
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW_MASK                                                0x00000010L
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__RESERVED_MASK                                                        0xFFFFFFE0L
+//RLC_POWER_RESIDENCY_EVENT_CNTR
+#define RLC_POWER_RESIDENCY_EVENT_CNTR__DATA__SHIFT                                                           0x0
+#define RLC_POWER_RESIDENCY_EVENT_CNTR__DATA_MASK                                                             0xFFFFFFFFL
+//RLC_CLK_RESIDENCY_EVENT_CNTR
+#define RLC_CLK_RESIDENCY_EVENT_CNTR__DATA__SHIFT                                                             0x0
+#define RLC_CLK_RESIDENCY_EVENT_CNTR__DATA_MASK                                                               0xFFFFFFFFL
+//RLC_DS_RESIDENCY_EVENT_CNTR
+#define RLC_DS_RESIDENCY_EVENT_CNTR__DATA__SHIFT                                                              0x0
+#define RLC_DS_RESIDENCY_EVENT_CNTR__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_ULV_RESIDENCY_EVENT_CNTR
+#define RLC_ULV_RESIDENCY_EVENT_CNTR__DATA__SHIFT                                                             0x0
+#define RLC_ULV_RESIDENCY_EVENT_CNTR__DATA_MASK                                                               0xFFFFFFFFL
+//RLC_PCC_RESIDENCY_EVENT_CNTR
+#define RLC_PCC_RESIDENCY_EVENT_CNTR__DATA__SHIFT                                                             0x0
+#define RLC_PCC_RESIDENCY_EVENT_CNTR__DATA_MASK                                                               0xFFFFFFFFL
+//RLC_GENERAL_RESIDENCY_EVENT_CNTR
+#define RLC_GENERAL_RESIDENCY_EVENT_CNTR__DATA__SHIFT                                                         0x0
+#define RLC_GENERAL_RESIDENCY_EVENT_CNTR__DATA_MASK                                                           0xFFFFFFFFL
+//RLC_POWER_RESIDENCY_REF_CNTR
+#define RLC_POWER_RESIDENCY_REF_CNTR__DATA__SHIFT                                                             0x0
+#define RLC_POWER_RESIDENCY_REF_CNTR__DATA_MASK                                                               0xFFFFFFFFL
+//RLC_CLK_RESIDENCY_REF_CNTR
+#define RLC_CLK_RESIDENCY_REF_CNTR__DATA__SHIFT                                                               0x0
+#define RLC_CLK_RESIDENCY_REF_CNTR__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_DS_RESIDENCY_REF_CNTR
+#define RLC_DS_RESIDENCY_REF_CNTR__DATA__SHIFT                                                                0x0
+#define RLC_DS_RESIDENCY_REF_CNTR__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_ULV_RESIDENCY_REF_CNTR
+#define RLC_ULV_RESIDENCY_REF_CNTR__DATA__SHIFT                                                               0x0
+#define RLC_ULV_RESIDENCY_REF_CNTR__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_PCC_RESIDENCY_REF_CNTR
+#define RLC_PCC_RESIDENCY_REF_CNTR__DATA__SHIFT                                                               0x0
+#define RLC_PCC_RESIDENCY_REF_CNTR__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_GENERAL_RESIDENCY_REF_CNTR
+#define RLC_GENERAL_RESIDENCY_REF_CNTR__DATA__SHIFT                                                           0x0
+#define RLC_GENERAL_RESIDENCY_REF_CNTR__DATA_MASK                                                             0xFFFFFFFFL
+//RLC_GFX_IH_CLIENT_CTRL
+#define RLC_GFX_IH_CLIENT_CTRL__SE_INTERRUPT_MASK__SHIFT                                                      0x0
+#define RLC_GFX_IH_CLIENT_CTRL__SDMA_INTERRUPT_MASK__SHIFT                                                    0x8
+#define RLC_GFX_IH_CLIENT_CTRL__UTCL2_INTERRUPT_MASK__SHIFT                                                   0xc
+#define RLC_GFX_IH_CLIENT_CTRL__RESERVED_13__SHIFT                                                            0xd
+#define RLC_GFX_IH_CLIENT_CTRL__FED_INTERRUPT_MASK__SHIFT                                                     0xe
+#define RLC_GFX_IH_CLIENT_CTRL__RESERVED_15__SHIFT                                                            0xf
+#define RLC_GFX_IH_CLIENT_CTRL__SE_INTERRUPT_ERROR_CLEAR__SHIFT                                               0x10
+#define RLC_GFX_IH_CLIENT_CTRL__SDMA_INTERRUPT_ERROR_CLEAR__SHIFT                                             0x18
+#define RLC_GFX_IH_CLIENT_CTRL__UTCL2_INTERRUPT_ERROR_CLEAR__SHIFT                                            0x1c
+#define RLC_GFX_IH_CLIENT_CTRL__RESERVED_29__SHIFT                                                            0x1d
+#define RLC_GFX_IH_CLIENT_CTRL__FED_INTERRUPT_ERROR_CLEAR__SHIFT                                              0x1e
+#define RLC_GFX_IH_CLIENT_CTRL__IH_HALT__SHIFT                                                                0x1f
+#define RLC_GFX_IH_CLIENT_CTRL__SE_INTERRUPT_MASK_MASK                                                        0x000000FFL
+#define RLC_GFX_IH_CLIENT_CTRL__SDMA_INTERRUPT_MASK_MASK                                                      0x00000F00L
+#define RLC_GFX_IH_CLIENT_CTRL__UTCL2_INTERRUPT_MASK_MASK                                                     0x00001000L
+#define RLC_GFX_IH_CLIENT_CTRL__RESERVED_13_MASK                                                              0x00002000L
+#define RLC_GFX_IH_CLIENT_CTRL__FED_INTERRUPT_MASK_MASK                                                       0x00004000L
+#define RLC_GFX_IH_CLIENT_CTRL__RESERVED_15_MASK                                                              0x00008000L
+#define RLC_GFX_IH_CLIENT_CTRL__SE_INTERRUPT_ERROR_CLEAR_MASK                                                 0x00FF0000L
+#define RLC_GFX_IH_CLIENT_CTRL__SDMA_INTERRUPT_ERROR_CLEAR_MASK                                               0x0F000000L
+#define RLC_GFX_IH_CLIENT_CTRL__UTCL2_INTERRUPT_ERROR_CLEAR_MASK                                              0x10000000L
+#define RLC_GFX_IH_CLIENT_CTRL__RESERVED_29_MASK                                                              0x20000000L
+#define RLC_GFX_IH_CLIENT_CTRL__FED_INTERRUPT_ERROR_CLEAR_MASK                                                0x40000000L
+#define RLC_GFX_IH_CLIENT_CTRL__IH_HALT_MASK                                                                  0x80000000L
+//RLC_GFX_IH_ARBITER_STAT
+#define RLC_GFX_IH_ARBITER_STAT__CLIENT_GRANTED__SHIFT                                                        0x0
+#define RLC_GFX_IH_ARBITER_STAT__RESERVED__SHIFT                                                              0x10
+#define RLC_GFX_IH_ARBITER_STAT__LAST_CLIENT_GRANTED__SHIFT                                                   0x1c
+#define RLC_GFX_IH_ARBITER_STAT__CLIENT_GRANTED_MASK                                                          0x0000FFFFL
+#define RLC_GFX_IH_ARBITER_STAT__RESERVED_MASK                                                                0x0FFF0000L
+#define RLC_GFX_IH_ARBITER_STAT__LAST_CLIENT_GRANTED_MASK                                                     0xF0000000L
+//RLC_GFX_IH_CLIENT_SE_STAT_L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_BUFFER_LEVEL__SHIFT                                                  0x0
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_BUFFER_LOADING__SHIFT                                                0x4
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_PROTOCOL_ERROR__SHIFT                                                0x5
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_BUFFER_OVERFLOW__SHIFT                                               0x6
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_RESERVED__SHIFT                                                      0x7
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_BUFFER_LEVEL__SHIFT                                                  0x8
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_BUFFER_LOADING__SHIFT                                                0xc
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_PROTOCOL_ERROR__SHIFT                                                0xd
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_BUFFER_OVERFLOW__SHIFT                                               0xe
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_RESERVED__SHIFT                                                      0xf
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_BUFFER_LEVEL__SHIFT                                                  0x10
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_BUFFER_LOADING__SHIFT                                                0x14
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_PROTOCOL_ERROR__SHIFT                                                0x15
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_BUFFER_OVERFLOW__SHIFT                                               0x16
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_RESERVED__SHIFT                                                      0x17
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_BUFFER_LEVEL__SHIFT                                                  0x18
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_BUFFER_LOADING__SHIFT                                                0x1c
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_PROTOCOL_ERROR__SHIFT                                                0x1d
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_BUFFER_OVERFLOW__SHIFT                                               0x1e
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_RESERVED__SHIFT                                                      0x1f
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_BUFFER_LEVEL_MASK                                                    0x0000000FL
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_BUFFER_LOADING_MASK                                                  0x00000010L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_PROTOCOL_ERROR_MASK                                                  0x00000020L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_BUFFER_OVERFLOW_MASK                                                 0x00000040L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_RESERVED_MASK                                                        0x00000080L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_BUFFER_LEVEL_MASK                                                    0x00000F00L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_BUFFER_LOADING_MASK                                                  0x00001000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_PROTOCOL_ERROR_MASK                                                  0x00002000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_BUFFER_OVERFLOW_MASK                                                 0x00004000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_RESERVED_MASK                                                        0x00008000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_BUFFER_LEVEL_MASK                                                    0x000F0000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_BUFFER_LOADING_MASK                                                  0x00100000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_PROTOCOL_ERROR_MASK                                                  0x00200000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_BUFFER_OVERFLOW_MASK                                                 0x00400000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_RESERVED_MASK                                                        0x00800000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_BUFFER_LEVEL_MASK                                                    0x0F000000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_BUFFER_LOADING_MASK                                                  0x10000000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_PROTOCOL_ERROR_MASK                                                  0x20000000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_BUFFER_OVERFLOW_MASK                                                 0x40000000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_RESERVED_MASK                                                        0x80000000L
+//RLC_GFX_IH_CLIENT_SE_STAT_H
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_BUFFER_LEVEL__SHIFT                                                  0x0
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_BUFFER_LOADING__SHIFT                                                0x4
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_PROTOCOL_ERROR__SHIFT                                                0x5
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_BUFFER_OVERFLOW__SHIFT                                               0x6
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_RESERVED__SHIFT                                                      0x7
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_BUFFER_LEVEL__SHIFT                                                  0x8
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_BUFFER_LOADING__SHIFT                                                0xc
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_PROTOCOL_ERROR__SHIFT                                                0xd
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_BUFFER_OVERFLOW__SHIFT                                               0xe
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_RESERVED__SHIFT                                                      0xf
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_BUFFER_LEVEL__SHIFT                                                  0x10
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_BUFFER_LOADING__SHIFT                                                0x14
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_PROTOCOL_ERROR__SHIFT                                                0x15
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_BUFFER_OVERFLOW__SHIFT                                               0x16
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_RESERVED__SHIFT                                                      0x17
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_BUFFER_LEVEL__SHIFT                                                  0x18
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_BUFFER_LOADING__SHIFT                                                0x1c
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_PROTOCOL_ERROR__SHIFT                                                0x1d
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_BUFFER_OVERFLOW__SHIFT                                               0x1e
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_RESERVED__SHIFT                                                      0x1f
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_BUFFER_LEVEL_MASK                                                    0x0000000FL
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_BUFFER_LOADING_MASK                                                  0x00000010L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_PROTOCOL_ERROR_MASK                                                  0x00000020L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_BUFFER_OVERFLOW_MASK                                                 0x00000040L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_RESERVED_MASK                                                        0x00000080L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_BUFFER_LEVEL_MASK                                                    0x00000F00L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_BUFFER_LOADING_MASK                                                  0x00001000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_PROTOCOL_ERROR_MASK                                                  0x00002000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_BUFFER_OVERFLOW_MASK                                                 0x00004000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_RESERVED_MASK                                                        0x00008000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_BUFFER_LEVEL_MASK                                                    0x000F0000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_BUFFER_LOADING_MASK                                                  0x00100000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_PROTOCOL_ERROR_MASK                                                  0x00200000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_BUFFER_OVERFLOW_MASK                                                 0x00400000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_RESERVED_MASK                                                        0x00800000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_BUFFER_LEVEL_MASK                                                    0x0F000000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_BUFFER_LOADING_MASK                                                  0x10000000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_PROTOCOL_ERROR_MASK                                                  0x20000000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_BUFFER_OVERFLOW_MASK                                                 0x40000000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_RESERVED_MASK                                                        0x80000000L
+//RLC_GFX_IH_CLIENT_SDMA_STAT
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_BUFFER_LEVEL__SHIFT                                                0x0
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_BUFFER_LOADING__SHIFT                                              0x4
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_PROTOCOL_ERROR__SHIFT                                              0x5
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_BUFFER_OVERFLOW__SHIFT                                             0x6
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_RESERVED__SHIFT                                                    0x7
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_BUFFER_LEVEL__SHIFT                                                0x8
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_BUFFER_LOADING__SHIFT                                              0xc
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_PROTOCOL_ERROR__SHIFT                                              0xd
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_BUFFER_OVERFLOW__SHIFT                                             0xe
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_RESERVED__SHIFT                                                    0xf
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_BUFFER_LEVEL__SHIFT                                                0x10
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_BUFFER_LOADING__SHIFT                                              0x14
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_PROTOCOL_ERROR__SHIFT                                              0x15
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_BUFFER_OVERFLOW__SHIFT                                             0x16
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_RESERVED__SHIFT                                                    0x17
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_BUFFER_LEVEL__SHIFT                                                0x18
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_BUFFER_LOADING__SHIFT                                              0x1c
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_PROTOCOL_ERROR__SHIFT                                              0x1d
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_BUFFER_OVERFLOW__SHIFT                                             0x1e
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_RESERVED__SHIFT                                                    0x1f
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_BUFFER_LEVEL_MASK                                                  0x0000000FL
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_BUFFER_LOADING_MASK                                                0x00000010L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_PROTOCOL_ERROR_MASK                                                0x00000020L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_BUFFER_OVERFLOW_MASK                                               0x00000040L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_RESERVED_MASK                                                      0x00000080L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_BUFFER_LEVEL_MASK                                                  0x00000F00L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_BUFFER_LOADING_MASK                                                0x00001000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_PROTOCOL_ERROR_MASK                                                0x00002000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_BUFFER_OVERFLOW_MASK                                               0x00004000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_RESERVED_MASK                                                      0x00008000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_BUFFER_LEVEL_MASK                                                  0x000F0000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_BUFFER_LOADING_MASK                                                0x00100000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_PROTOCOL_ERROR_MASK                                                0x00200000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_BUFFER_OVERFLOW_MASK                                               0x00400000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_RESERVED_MASK                                                      0x00800000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_BUFFER_LEVEL_MASK                                                  0x0F000000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_BUFFER_LOADING_MASK                                                0x10000000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_PROTOCOL_ERROR_MASK                                                0x20000000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_BUFFER_OVERFLOW_MASK                                               0x40000000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_RESERVED_MASK                                                      0x80000000L
+//RLC_GFX_IH_CLIENT_OTHER_STAT
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__UTCL2_RESERVED__SHIFT                                                   0x0
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__PMM_RESERVED__SHIFT                                                     0x8
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_BUFFER_LEVEL__SHIFT                                                 0x10
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_BUFFER_LOADING__SHIFT                                               0x14
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_PROTOCOL_ERROR__SHIFT                                               0x15
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_BUFFER_OVERFLOW__SHIFT                                              0x16
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_RESERVED__SHIFT                                                     0x17
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__RESERVED_31_24__SHIFT                                                   0x18
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__UTCL2_RESERVED_MASK                                                     0x000000FFL
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__PMM_RESERVED_MASK                                                       0x0000FF00L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_BUFFER_LEVEL_MASK                                                   0x000F0000L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_BUFFER_LOADING_MASK                                                 0x00100000L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_PROTOCOL_ERROR_MASK                                                 0x00200000L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_BUFFER_OVERFLOW_MASK                                                0x00400000L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_RESERVED_MASK                                                       0x00800000L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__RESERVED_31_24_MASK                                                     0xFF000000L
+//RLC_SPM_GLOBAL_DELAY_IND_ADDR
+#define RLC_SPM_GLOBAL_DELAY_IND_ADDR__ADDR__SHIFT                                                            0x0
+#define RLC_SPM_GLOBAL_DELAY_IND_ADDR__ADDR_MASK                                                              0x00000FFFL
+//RLC_SPM_GLOBAL_DELAY_IND_DATA
+#define RLC_SPM_GLOBAL_DELAY_IND_DATA__DATA__SHIFT                                                            0x0
+#define RLC_SPM_GLOBAL_DELAY_IND_DATA__DATA_MASK                                                              0x0000003FL
+//RLC_SPM_SE_DELAY_IND_ADDR
+#define RLC_SPM_SE_DELAY_IND_ADDR__ADDR__SHIFT                                                                0x0
+#define RLC_SPM_SE_DELAY_IND_ADDR__ADDR_MASK                                                                  0x00000FFFL
+//RLC_SPM_SE_DELAY_IND_DATA
+#define RLC_SPM_SE_DELAY_IND_DATA__DATA__SHIFT                                                                0x0
+#define RLC_SPM_SE_DELAY_IND_DATA__DATA_MASK                                                                  0x0000003FL
+//RLC_SPM_GLOBAL_BLK_EN_MASK_IND_ADDR
+#define RLC_SPM_GLOBAL_BLK_EN_MASK_IND_ADDR__ADDR__SHIFT                                                      0x0
+#define RLC_SPM_GLOBAL_BLK_EN_MASK_IND_ADDR__ADDR_MASK                                                        0x00000FFFL
+//RLC_SPM_GLOBAL_BLK_EN_MASK_IND_DATA
+#define RLC_SPM_GLOBAL_BLK_EN_MASK_IND_DATA__DATA__SHIFT                                                      0x0
+#define RLC_SPM_GLOBAL_BLK_EN_MASK_IND_DATA__DATA_MASK                                                        0xFFFFFFFFL
+//RLC_SPM_SE_BLK_EN_MASK_IND_ADDR
+#define RLC_SPM_SE_BLK_EN_MASK_IND_ADDR__ADDR__SHIFT                                                          0x0
+#define RLC_SPM_SE_BLK_EN_MASK_IND_ADDR__ADDR_MASK                                                            0x00000FFFL
+//RLC_SPM_SE_BLK_EN_MASK_IND_DATA
+#define RLC_SPM_SE_BLK_EN_MASK_IND_DATA__DATA__SHIFT                                                          0x0
+#define RLC_SPM_SE_BLK_EN_MASK_IND_DATA__DATA_MASK                                                            0xFFFFFFFFL
+//RLC_GFX_IH_22BEAT_STAT
+#define RLC_GFX_IH_22BEAT_STAT__UTCL2_BUFFER_LEVEL__SHIFT                                                     0x0
+#define RLC_GFX_IH_22BEAT_STAT__UTCL2_BUFFER_LOADING__SHIFT                                                   0x8
+#define RLC_GFX_IH_22BEAT_STAT__UTCL2_PROTOCOL_ERROR__SHIFT                                                   0x9
+#define RLC_GFX_IH_22BEAT_STAT__UTCL2_BUFFER_OVERFLOW__SHIFT                                                  0xa
+#define RLC_GFX_IH_22BEAT_STAT__UTCL2_RESERVED__SHIFT                                                         0xb
+#define RLC_GFX_IH_22BEAT_STAT__UTCL2_BUFFER_LEVEL_MASK                                                       0x000000FFL
+#define RLC_GFX_IH_22BEAT_STAT__UTCL2_BUFFER_LOADING_MASK                                                     0x00000100L
+#define RLC_GFX_IH_22BEAT_STAT__UTCL2_PROTOCOL_ERROR_MASK                                                     0x00000200L
+#define RLC_GFX_IH_22BEAT_STAT__UTCL2_BUFFER_OVERFLOW_MASK                                                    0x00000400L
+#define RLC_GFX_IH_22BEAT_STAT__UTCL2_RESERVED_MASK                                                           0xFFFFF800L
+//RLC_GFX_IH_ARBITER_STAT_22BEAT
+#define RLC_GFX_IH_ARBITER_STAT_22BEAT__CLIENT_GRANTED__SHIFT                                                 0x0
+#define RLC_GFX_IH_ARBITER_STAT_22BEAT__RESERVED__SHIFT                                                       0x10
+#define RLC_GFX_IH_ARBITER_STAT_22BEAT__LAST_CLIENT_GRANTED__SHIFT                                            0x1c
+#define RLC_GFX_IH_ARBITER_STAT_22BEAT__CLIENT_GRANTED_MASK                                                   0x0000FFFFL
+#define RLC_GFX_IH_ARBITER_STAT_22BEAT__RESERVED_MASK                                                         0x0FFF0000L
+#define RLC_GFX_IH_ARBITER_STAT_22BEAT__LAST_CLIENT_GRANTED_MASK                                              0xF0000000L
+//RLC_LX6_CNTL
+#define RLC_LX6_CNTL__BRESET__SHIFT                                                                           0x0
+#define RLC_LX6_CNTL__RUNSTALL__SHIFT                                                                         0x1
+#define RLC_LX6_CNTL__PDEBUG_ENABLE__SHIFT                                                                    0x2
+#define RLC_LX6_CNTL__STAT_VECTOR_SEL__SHIFT                                                                  0x3
+#define RLC_LX6_CNTL__BRESET_MASK                                                                             0x00000001L
+#define RLC_LX6_CNTL__RUNSTALL_MASK                                                                           0x00000002L
+#define RLC_LX6_CNTL__PDEBUG_ENABLE_MASK                                                                      0x00000004L
+#define RLC_LX6_CNTL__STAT_VECTOR_SEL_MASK                                                                    0x00000008L
+//RLC_LX6_STATUS
+#define RLC_LX6_STATUS__CORE0_CORE_BUSY__SHIFT                                                                0x0
+#define RLC_LX6_STATUS__CORE0_PIF_GASKET_BUSY__SHIFT                                                          0x1
+#define RLC_LX6_STATUS__CORE0_INT_PENDING__SHIFT                                                              0x2
+#define RLC_LX6_STATUS__CORE0_GRBMT_BUSY__SHIFT                                                               0x3
+#define RLC_LX6_STATUS__CORE1_CORE_BUSY__SHIFT                                                                0x4
+#define RLC_LX6_STATUS__CORE1_PIF_GASKET_BUSY__SHIFT                                                          0x5
+#define RLC_LX6_STATUS__CORE1_INT_PENDING__SHIFT                                                              0x6
+#define RLC_LX6_STATUS__CORE1_GRBMT_BUSY__SHIFT                                                               0x7
+#define RLC_LX6_STATUS__GRBMT_BUSY__SHIFT                                                                     0x8
+#define RLC_LX6_STATUS__CORE0_CORE_BUSY_MASK                                                                  0x00000001L
+#define RLC_LX6_STATUS__CORE0_PIF_GASKET_BUSY_MASK                                                            0x00000002L
+#define RLC_LX6_STATUS__CORE0_INT_PENDING_MASK                                                                0x00000004L
+#define RLC_LX6_STATUS__CORE0_GRBMT_BUSY_MASK                                                                 0x00000008L
+#define RLC_LX6_STATUS__CORE1_CORE_BUSY_MASK                                                                  0x00000010L
+#define RLC_LX6_STATUS__CORE1_PIF_GASKET_BUSY_MASK                                                            0x00000020L
+#define RLC_LX6_STATUS__CORE1_INT_PENDING_MASK                                                                0x00000040L
+#define RLC_LX6_STATUS__CORE1_GRBMT_BUSY_MASK                                                                 0x00000080L
+#define RLC_LX6_STATUS__GRBMT_BUSY_MASK                                                                       0x00000100L
+//RLC_LX6_FW_STATUS
+#define RLC_LX6_FW_STATUS__STATUS__SHIFT                                                                      0x0
+#define RLC_LX6_FW_STATUS__STATUS_MASK                                                                        0xFFFFFFFFL
+//RLC_LX6_FW_VERSION
+#define RLC_LX6_FW_VERSION__VERSION__SHIFT                                                                    0x0
+#define RLC_LX6_FW_VERSION__VERSION_MASK                                                                      0xFFFFFFFFL
+//RLC_XT_CORE_STATUS
+#define RLC_XT_CORE_STATUS__P_WAIT_MODE__SHIFT                                                                0x0
+#define RLC_XT_CORE_STATUS__P_FATAL_ERROR__SHIFT                                                              0x1
+#define RLC_XT_CORE_STATUS__DOUBLE_EXCEPTION_ERROR__SHIFT                                                     0x2
+#define RLC_XT_CORE_STATUS__P_WAIT_MODE_MASK                                                                  0x00000001L
+#define RLC_XT_CORE_STATUS__P_FATAL_ERROR_MASK                                                                0x00000002L
+#define RLC_XT_CORE_STATUS__DOUBLE_EXCEPTION_ERROR_MASK                                                       0x00000004L
+//RLC_XT_CORE_INTERRUPT
+#define RLC_XT_CORE_INTERRUPT__EXTINT1__SHIFT                                                                 0x0
+#define RLC_XT_CORE_INTERRUPT__EXTINT2__SHIFT                                                                 0x1a
+#define RLC_XT_CORE_INTERRUPT__NMI__SHIFT                                                                     0x1b
+#define RLC_XT_CORE_INTERRUPT__EXTINT1_MASK                                                                   0x03FFFFFFL
+#define RLC_XT_CORE_INTERRUPT__EXTINT2_MASK                                                                   0x04000000L
+#define RLC_XT_CORE_INTERRUPT__NMI_MASK                                                                       0x08000000L
+//RLC_XT_CORE_FAULT_INFO
+#define RLC_XT_CORE_FAULT_INFO__FAULT_INFO__SHIFT                                                             0x0
+#define RLC_XT_CORE_FAULT_INFO__FAULT_INFO_MASK                                                               0xFFFFFFFFL
+//RLC_XT_CORE_ALT_RESET_VEC
+#define RLC_XT_CORE_ALT_RESET_VEC__ALT_RESET_VEC__SHIFT                                                       0x0
+#define RLC_XT_CORE_ALT_RESET_VEC__ALT_RESET_VEC_MASK                                                         0xFFFFFFFFL
+//RLC_XT_CORE_RESERVED
+#define RLC_XT_CORE_RESERVED__RESERVED__SHIFT                                                                 0x0
+#define RLC_XT_CORE_RESERVED__RESERVED_MASK                                                                   0xFFFFFFFFL
+//RLC_XT_INT_VEC_FORCE
+#define RLC_XT_INT_VEC_FORCE__NUM_0__SHIFT                                                                    0x0
+#define RLC_XT_INT_VEC_FORCE__NUM_1__SHIFT                                                                    0x1
+#define RLC_XT_INT_VEC_FORCE__NUM_2__SHIFT                                                                    0x2
+#define RLC_XT_INT_VEC_FORCE__NUM_3__SHIFT                                                                    0x3
+#define RLC_XT_INT_VEC_FORCE__NUM_4__SHIFT                                                                    0x4
+#define RLC_XT_INT_VEC_FORCE__NUM_5__SHIFT                                                                    0x5
+#define RLC_XT_INT_VEC_FORCE__NUM_6__SHIFT                                                                    0x6
+#define RLC_XT_INT_VEC_FORCE__NUM_7__SHIFT                                                                    0x7
+#define RLC_XT_INT_VEC_FORCE__NUM_8__SHIFT                                                                    0x8
+#define RLC_XT_INT_VEC_FORCE__NUM_9__SHIFT                                                                    0x9
+#define RLC_XT_INT_VEC_FORCE__NUM_10__SHIFT                                                                   0xa
+#define RLC_XT_INT_VEC_FORCE__NUM_11__SHIFT                                                                   0xb
+#define RLC_XT_INT_VEC_FORCE__NUM_12__SHIFT                                                                   0xc
+#define RLC_XT_INT_VEC_FORCE__NUM_13__SHIFT                                                                   0xd
+#define RLC_XT_INT_VEC_FORCE__NUM_14__SHIFT                                                                   0xe
+#define RLC_XT_INT_VEC_FORCE__NUM_15__SHIFT                                                                   0xf
+#define RLC_XT_INT_VEC_FORCE__NUM_16__SHIFT                                                                   0x10
+#define RLC_XT_INT_VEC_FORCE__NUM_17__SHIFT                                                                   0x11
+#define RLC_XT_INT_VEC_FORCE__NUM_18__SHIFT                                                                   0x12
+#define RLC_XT_INT_VEC_FORCE__NUM_19__SHIFT                                                                   0x13
+#define RLC_XT_INT_VEC_FORCE__NUM_20__SHIFT                                                                   0x14
+#define RLC_XT_INT_VEC_FORCE__NUM_21__SHIFT                                                                   0x15
+#define RLC_XT_INT_VEC_FORCE__NUM_22__SHIFT                                                                   0x16
+#define RLC_XT_INT_VEC_FORCE__NUM_23__SHIFT                                                                   0x17
+#define RLC_XT_INT_VEC_FORCE__NUM_24__SHIFT                                                                   0x18
+#define RLC_XT_INT_VEC_FORCE__NUM_25__SHIFT                                                                   0x19
+#define RLC_XT_INT_VEC_FORCE__NUM_0_MASK                                                                      0x00000001L
+#define RLC_XT_INT_VEC_FORCE__NUM_1_MASK                                                                      0x00000002L
+#define RLC_XT_INT_VEC_FORCE__NUM_2_MASK                                                                      0x00000004L
+#define RLC_XT_INT_VEC_FORCE__NUM_3_MASK                                                                      0x00000008L
+#define RLC_XT_INT_VEC_FORCE__NUM_4_MASK                                                                      0x00000010L
+#define RLC_XT_INT_VEC_FORCE__NUM_5_MASK                                                                      0x00000020L
+#define RLC_XT_INT_VEC_FORCE__NUM_6_MASK                                                                      0x00000040L
+#define RLC_XT_INT_VEC_FORCE__NUM_7_MASK                                                                      0x00000080L
+#define RLC_XT_INT_VEC_FORCE__NUM_8_MASK                                                                      0x00000100L
+#define RLC_XT_INT_VEC_FORCE__NUM_9_MASK                                                                      0x00000200L
+#define RLC_XT_INT_VEC_FORCE__NUM_10_MASK                                                                     0x00000400L
+#define RLC_XT_INT_VEC_FORCE__NUM_11_MASK                                                                     0x00000800L
+#define RLC_XT_INT_VEC_FORCE__NUM_12_MASK                                                                     0x00001000L
+#define RLC_XT_INT_VEC_FORCE__NUM_13_MASK                                                                     0x00002000L
+#define RLC_XT_INT_VEC_FORCE__NUM_14_MASK                                                                     0x00004000L
+#define RLC_XT_INT_VEC_FORCE__NUM_15_MASK                                                                     0x00008000L
+#define RLC_XT_INT_VEC_FORCE__NUM_16_MASK                                                                     0x00010000L
+#define RLC_XT_INT_VEC_FORCE__NUM_17_MASK                                                                     0x00020000L
+#define RLC_XT_INT_VEC_FORCE__NUM_18_MASK                                                                     0x00040000L
+#define RLC_XT_INT_VEC_FORCE__NUM_19_MASK                                                                     0x00080000L
+#define RLC_XT_INT_VEC_FORCE__NUM_20_MASK                                                                     0x00100000L
+#define RLC_XT_INT_VEC_FORCE__NUM_21_MASK                                                                     0x00200000L
+#define RLC_XT_INT_VEC_FORCE__NUM_22_MASK                                                                     0x00400000L
+#define RLC_XT_INT_VEC_FORCE__NUM_23_MASK                                                                     0x00800000L
+#define RLC_XT_INT_VEC_FORCE__NUM_24_MASK                                                                     0x01000000L
+#define RLC_XT_INT_VEC_FORCE__NUM_25_MASK                                                                     0x02000000L
+//RLC_XT_INT_VEC_CLEAR
+#define RLC_XT_INT_VEC_CLEAR__NUM_0__SHIFT                                                                    0x0
+#define RLC_XT_INT_VEC_CLEAR__NUM_1__SHIFT                                                                    0x1
+#define RLC_XT_INT_VEC_CLEAR__NUM_2__SHIFT                                                                    0x2
+#define RLC_XT_INT_VEC_CLEAR__NUM_3__SHIFT                                                                    0x3
+#define RLC_XT_INT_VEC_CLEAR__NUM_4__SHIFT                                                                    0x4
+#define RLC_XT_INT_VEC_CLEAR__NUM_5__SHIFT                                                                    0x5
+#define RLC_XT_INT_VEC_CLEAR__NUM_6__SHIFT                                                                    0x6
+#define RLC_XT_INT_VEC_CLEAR__NUM_7__SHIFT                                                                    0x7
+#define RLC_XT_INT_VEC_CLEAR__NUM_8__SHIFT                                                                    0x8
+#define RLC_XT_INT_VEC_CLEAR__NUM_9__SHIFT                                                                    0x9
+#define RLC_XT_INT_VEC_CLEAR__NUM_10__SHIFT                                                                   0xa
+#define RLC_XT_INT_VEC_CLEAR__NUM_11__SHIFT                                                                   0xb
+#define RLC_XT_INT_VEC_CLEAR__NUM_12__SHIFT                                                                   0xc
+#define RLC_XT_INT_VEC_CLEAR__NUM_13__SHIFT                                                                   0xd
+#define RLC_XT_INT_VEC_CLEAR__NUM_14__SHIFT                                                                   0xe
+#define RLC_XT_INT_VEC_CLEAR__NUM_15__SHIFT                                                                   0xf
+#define RLC_XT_INT_VEC_CLEAR__NUM_16__SHIFT                                                                   0x10
+#define RLC_XT_INT_VEC_CLEAR__NUM_17__SHIFT                                                                   0x11
+#define RLC_XT_INT_VEC_CLEAR__NUM_18__SHIFT                                                                   0x12
+#define RLC_XT_INT_VEC_CLEAR__NUM_19__SHIFT                                                                   0x13
+#define RLC_XT_INT_VEC_CLEAR__NUM_20__SHIFT                                                                   0x14
+#define RLC_XT_INT_VEC_CLEAR__NUM_21__SHIFT                                                                   0x15
+#define RLC_XT_INT_VEC_CLEAR__NUM_22__SHIFT                                                                   0x16
+#define RLC_XT_INT_VEC_CLEAR__NUM_23__SHIFT                                                                   0x17
+#define RLC_XT_INT_VEC_CLEAR__NUM_24__SHIFT                                                                   0x18
+#define RLC_XT_INT_VEC_CLEAR__NUM_25__SHIFT                                                                   0x19
+#define RLC_XT_INT_VEC_CLEAR__NUM_0_MASK                                                                      0x00000001L
+#define RLC_XT_INT_VEC_CLEAR__NUM_1_MASK                                                                      0x00000002L
+#define RLC_XT_INT_VEC_CLEAR__NUM_2_MASK                                                                      0x00000004L
+#define RLC_XT_INT_VEC_CLEAR__NUM_3_MASK                                                                      0x00000008L
+#define RLC_XT_INT_VEC_CLEAR__NUM_4_MASK                                                                      0x00000010L
+#define RLC_XT_INT_VEC_CLEAR__NUM_5_MASK                                                                      0x00000020L
+#define RLC_XT_INT_VEC_CLEAR__NUM_6_MASK                                                                      0x00000040L
+#define RLC_XT_INT_VEC_CLEAR__NUM_7_MASK                                                                      0x00000080L
+#define RLC_XT_INT_VEC_CLEAR__NUM_8_MASK                                                                      0x00000100L
+#define RLC_XT_INT_VEC_CLEAR__NUM_9_MASK                                                                      0x00000200L
+#define RLC_XT_INT_VEC_CLEAR__NUM_10_MASK                                                                     0x00000400L
+#define RLC_XT_INT_VEC_CLEAR__NUM_11_MASK                                                                     0x00000800L
+#define RLC_XT_INT_VEC_CLEAR__NUM_12_MASK                                                                     0x00001000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_13_MASK                                                                     0x00002000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_14_MASK                                                                     0x00004000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_15_MASK                                                                     0x00008000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_16_MASK                                                                     0x00010000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_17_MASK                                                                     0x00020000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_18_MASK                                                                     0x00040000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_19_MASK                                                                     0x00080000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_20_MASK                                                                     0x00100000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_21_MASK                                                                     0x00200000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_22_MASK                                                                     0x00400000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_23_MASK                                                                     0x00800000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_24_MASK                                                                     0x01000000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_25_MASK                                                                     0x02000000L
+//RLC_XT_INT_VEC_MUX_SEL
+#define RLC_XT_INT_VEC_MUX_SEL__MUX_SEL__SHIFT                                                                0x0
+#define RLC_XT_INT_VEC_MUX_SEL__MUX_SEL_MASK                                                                  0x0000001FL
+//RLC_XT_INT_VEC_MUX_INT_SEL
+#define RLC_XT_INT_VEC_MUX_INT_SEL__INT_SEL__SHIFT                                                            0x0
+#define RLC_XT_INT_VEC_MUX_INT_SEL__INT_SEL_MASK                                                              0x0000003FL
+//RLC_GPU_CLOCK_COUNT_SPM_LSB
+#define RLC_GPU_CLOCK_COUNT_SPM_LSB__GPU_CLOCKS_LSB__SHIFT                                                    0x0
+#define RLC_GPU_CLOCK_COUNT_SPM_LSB__GPU_CLOCKS_LSB_MASK                                                      0xFFFFFFFFL
+//RLC_GPU_CLOCK_COUNT_SPM_MSB
+#define RLC_GPU_CLOCK_COUNT_SPM_MSB__GPU_CLOCKS_MSB__SHIFT                                                    0x0
+#define RLC_GPU_CLOCK_COUNT_SPM_MSB__GPU_CLOCKS_MSB_MASK                                                      0xFFFFFFFFL
+//RLC_SPM_THREAD_TRACE_CTRL
+#define RLC_SPM_THREAD_TRACE_CTRL__THREAD_TRACE_INT_EN__SHIFT                                                 0x0
+#define RLC_SPM_THREAD_TRACE_CTRL__THREAD_TRACE_INT_EN_MASK                                                   0x00000001L
+//RLC_SPP_CAM_ADDR
+#define RLC_SPP_CAM_ADDR__ADDR__SHIFT                                                                         0x0
+#define RLC_SPP_CAM_ADDR__ADDR_MASK                                                                           0x000000FFL
+//RLC_SPP_CAM_DATA
+#define RLC_SPP_CAM_DATA__DATA__SHIFT                                                                         0x0
+#define RLC_SPP_CAM_DATA__TAG__SHIFT                                                                          0x8
+#define RLC_SPP_CAM_DATA__DATA_MASK                                                                           0x000000FFL
+#define RLC_SPP_CAM_DATA__TAG_MASK                                                                            0xFFFFFF00L
+//RLC_SPP_CAM_EXT_ADDR
+#define RLC_SPP_CAM_EXT_ADDR__ADDR__SHIFT                                                                     0x0
+#define RLC_SPP_CAM_EXT_ADDR__ADDR_MASK                                                                       0x000000FFL
+//RLC_SPP_CAM_EXT_DATA
+#define RLC_SPP_CAM_EXT_DATA__VALID__SHIFT                                                                    0x0
+#define RLC_SPP_CAM_EXT_DATA__LOCK__SHIFT                                                                     0x1
+#define RLC_SPP_CAM_EXT_DATA__VALID_MASK                                                                      0x00000001L
+#define RLC_SPP_CAM_EXT_DATA__LOCK_MASK                                                                       0x00000002L
+//RLC_CPAXI_DOORBELL_MON_CTRL
+#define RLC_CPAXI_DOORBELL_MON_CTRL__EN__SHIFT                                                                0x0
+#define RLC_CPAXI_DOORBELL_MON_CTRL__ID__SHIFT                                                                0x1
+#define RLC_CPAXI_DOORBELL_MON_CTRL__EN_MASK                                                                  0x00000001L
+#define RLC_CPAXI_DOORBELL_MON_CTRL__ID_MASK                                                                  0x0000003EL
+//RLC_CPAXI_DOORBELL_MON_STAT
+#define RLC_CPAXI_DOORBELL_MON_STAT__ID_MATCH__SHIFT                                                          0x0
+#define RLC_CPAXI_DOORBELL_MON_STAT__MATCH_CLEAR__SHIFT                                                       0x1
+#define RLC_CPAXI_DOORBELL_MON_STAT__ADDR__SHIFT                                                              0x2
+#define RLC_CPAXI_DOORBELL_MON_STAT__ID_MATCH_MASK                                                            0x00000001L
+#define RLC_CPAXI_DOORBELL_MON_STAT__MATCH_CLEAR_MASK                                                         0x00000002L
+#define RLC_CPAXI_DOORBELL_MON_STAT__ADDR_MASK                                                                0x0FFFFFFCL
+//RLC_CPAXI_DOORBELL_MON_DATA_LSB
+#define RLC_CPAXI_DOORBELL_MON_DATA_LSB__DATA__SHIFT                                                          0x0
+#define RLC_CPAXI_DOORBELL_MON_DATA_LSB__DATA_MASK                                                            0xFFFFFFFFL
+//RLC_CPAXI_DOORBELL_MON_DATA_MSB
+#define RLC_CPAXI_DOORBELL_MON_DATA_MSB__DATA__SHIFT                                                          0x0
+#define RLC_CPAXI_DOORBELL_MON_DATA_MSB__DATA_MASK                                                            0xFFFFFFFFL
+//RLC_XT_DOORBELL_RANGE
+#define RLC_XT_DOORBELL_RANGE__LOWER_ADDR_RESERVED__SHIFT                                                     0x0
+#define RLC_XT_DOORBELL_RANGE__LOWER_ADDR__SHIFT                                                              0x2
+#define RLC_XT_DOORBELL_RANGE__RESERVED_15_12__SHIFT                                                          0xc
+#define RLC_XT_DOORBELL_RANGE__UPPER_ADDR_RESERVED__SHIFT                                                     0x10
+#define RLC_XT_DOORBELL_RANGE__UPPER_ADDR__SHIFT                                                              0x12
+#define RLC_XT_DOORBELL_RANGE__RESERVED_31_28__SHIFT                                                          0x1c
+#define RLC_XT_DOORBELL_RANGE__LOWER_ADDR_RESERVED_MASK                                                       0x00000003L
+#define RLC_XT_DOORBELL_RANGE__LOWER_ADDR_MASK                                                                0x00000FFCL
+#define RLC_XT_DOORBELL_RANGE__RESERVED_15_12_MASK                                                            0x0000F000L
+#define RLC_XT_DOORBELL_RANGE__UPPER_ADDR_RESERVED_MASK                                                       0x00030000L
+#define RLC_XT_DOORBELL_RANGE__UPPER_ADDR_MASK                                                                0x0FFC0000L
+#define RLC_XT_DOORBELL_RANGE__RESERVED_31_28_MASK                                                            0xF0000000L
+//RLC_XT_DOORBELL_CNTL
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_0_MODE__SHIFT                                                          0x0
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_1_MODE__SHIFT                                                          0x2
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_2_MODE__SHIFT                                                          0x4
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_3_MODE__SHIFT                                                          0x6
+#define RLC_XT_DOORBELL_CNTL__RESERVED_15_8__SHIFT                                                            0x8
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_ID__SHIFT                                                              0x10
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_ID_EN__SHIFT                                                           0x15
+#define RLC_XT_DOORBELL_CNTL__RESERVED_31_22__SHIFT                                                           0x16
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_0_MODE_MASK                                                            0x00000003L
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_1_MODE_MASK                                                            0x0000000CL
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_2_MODE_MASK                                                            0x00000030L
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_3_MODE_MASK                                                            0x000000C0L
+#define RLC_XT_DOORBELL_CNTL__RESERVED_15_8_MASK                                                              0x0000FF00L
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_ID_MASK                                                                0x001F0000L
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_ID_EN_MASK                                                             0x00200000L
+#define RLC_XT_DOORBELL_CNTL__RESERVED_31_22_MASK                                                             0xFFC00000L
+//RLC_XT_DOORBELL_STAT
+#define RLC_XT_DOORBELL_STAT__DOORBELL_0_VALID__SHIFT                                                         0x0
+#define RLC_XT_DOORBELL_STAT__DOORBELL_1_VALID__SHIFT                                                         0x1
+#define RLC_XT_DOORBELL_STAT__DOORBELL_2_VALID__SHIFT                                                         0x2
+#define RLC_XT_DOORBELL_STAT__DOORBELL_3_VALID__SHIFT                                                         0x3
+#define RLC_XT_DOORBELL_STAT__DOORBELL_0_VALID_MASK                                                           0x00000001L
+#define RLC_XT_DOORBELL_STAT__DOORBELL_1_VALID_MASK                                                           0x00000002L
+#define RLC_XT_DOORBELL_STAT__DOORBELL_2_VALID_MASK                                                           0x00000004L
+#define RLC_XT_DOORBELL_STAT__DOORBELL_3_VALID_MASK                                                           0x00000008L
+//RLC_XT_DOORBELL_0_DATA_LO
+#define RLC_XT_DOORBELL_0_DATA_LO__DATA__SHIFT                                                                0x0
+#define RLC_XT_DOORBELL_0_DATA_LO__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_XT_DOORBELL_0_DATA_HI
+#define RLC_XT_DOORBELL_0_DATA_HI__DATA__SHIFT                                                                0x0
+#define RLC_XT_DOORBELL_0_DATA_HI__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_XT_DOORBELL_1_DATA_LO
+#define RLC_XT_DOORBELL_1_DATA_LO__DATA__SHIFT                                                                0x0
+#define RLC_XT_DOORBELL_1_DATA_LO__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_XT_DOORBELL_1_DATA_HI
+#define RLC_XT_DOORBELL_1_DATA_HI__DATA__SHIFT                                                                0x0
+#define RLC_XT_DOORBELL_1_DATA_HI__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_XT_DOORBELL_2_DATA_LO
+#define RLC_XT_DOORBELL_2_DATA_LO__DATA__SHIFT                                                                0x0
+#define RLC_XT_DOORBELL_2_DATA_LO__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_XT_DOORBELL_2_DATA_HI
+#define RLC_XT_DOORBELL_2_DATA_HI__DATA__SHIFT                                                                0x0
+#define RLC_XT_DOORBELL_2_DATA_HI__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_XT_DOORBELL_3_DATA_LO
+#define RLC_XT_DOORBELL_3_DATA_LO__DATA__SHIFT                                                                0x0
+#define RLC_XT_DOORBELL_3_DATA_LO__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_XT_DOORBELL_3_DATA_HI
+#define RLC_XT_DOORBELL_3_DATA_HI__DATA__SHIFT                                                                0x0
+#define RLC_XT_DOORBELL_3_DATA_HI__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_MEM_SLP_CNTL
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN__SHIFT                                                                0x0
+#define RLC_MEM_SLP_CNTL__RLC_MEM_DS_EN__SHIFT                                                                0x1
+#define RLC_MEM_SLP_CNTL__RLC_SRM_MEM_LS_OVERRIDE__SHIFT                                                      0x2
+#define RLC_MEM_SLP_CNTL__RLC_SRM_MEM_DS_OVERRIDE__SHIFT                                                      0x3
+#define RLC_MEM_SLP_CNTL__RLC_SPM_MEM_LS_OVERRIDE__SHIFT                                                      0x4
+#define RLC_MEM_SLP_CNTL__RLC_SPM_MEM_DS_OVERRIDE__SHIFT                                                      0x5
+#define RLC_MEM_SLP_CNTL__RESERVED__SHIFT                                                                     0x6
+#define RLC_MEM_SLP_CNTL__RLC_LS_DS_BUSY_OVERRIDE__SHIFT                                                      0x7
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_ON_DELAY__SHIFT                                                          0x8
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_OFF_DELAY__SHIFT                                                         0x10
+#define RLC_MEM_SLP_CNTL__RLC_SPP_MEM_LS_OVERRIDE__SHIFT                                                      0x18
+#define RLC_MEM_SLP_CNTL__RLC_SPP_MEM_DS_OVERRIDE__SHIFT                                                      0x19
+#define RLC_MEM_SLP_CNTL__RLC_DMA_MEM_LS_OVERRIDE__SHIFT                                                      0x1a
+#define RLC_MEM_SLP_CNTL__RLC_DMA_MEM_DS_OVERRIDE__SHIFT                                                      0x1b
+#define RLC_MEM_SLP_CNTL__RLC_TC_MEM_LS_OVERRIDE__SHIFT                                                       0x1c
+#define RLC_MEM_SLP_CNTL__RLC_TC_MEM_DS_OVERRIDE__SHIFT                                                       0x1d
+#define RLC_MEM_SLP_CNTL__RESERVED1__SHIFT                                                                    0x1e
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK                                                                  0x00000001L
+#define RLC_MEM_SLP_CNTL__RLC_MEM_DS_EN_MASK                                                                  0x00000002L
+#define RLC_MEM_SLP_CNTL__RLC_SRM_MEM_LS_OVERRIDE_MASK                                                        0x00000004L
+#define RLC_MEM_SLP_CNTL__RLC_SRM_MEM_DS_OVERRIDE_MASK                                                        0x00000008L
+#define RLC_MEM_SLP_CNTL__RLC_SPM_MEM_LS_OVERRIDE_MASK                                                        0x00000010L
+#define RLC_MEM_SLP_CNTL__RLC_SPM_MEM_DS_OVERRIDE_MASK                                                        0x00000020L
+#define RLC_MEM_SLP_CNTL__RESERVED_MASK                                                                       0x00000040L
+#define RLC_MEM_SLP_CNTL__RLC_LS_DS_BUSY_OVERRIDE_MASK                                                        0x00000080L
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_ON_DELAY_MASK                                                            0x0000FF00L
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_OFF_DELAY_MASK                                                           0x00FF0000L
+#define RLC_MEM_SLP_CNTL__RLC_SPP_MEM_LS_OVERRIDE_MASK                                                        0x01000000L
+#define RLC_MEM_SLP_CNTL__RLC_SPP_MEM_DS_OVERRIDE_MASK                                                        0x02000000L
+#define RLC_MEM_SLP_CNTL__RLC_DMA_MEM_LS_OVERRIDE_MASK                                                        0x04000000L
+#define RLC_MEM_SLP_CNTL__RLC_DMA_MEM_DS_OVERRIDE_MASK                                                        0x08000000L
+#define RLC_MEM_SLP_CNTL__RLC_TC_MEM_LS_OVERRIDE_MASK                                                         0x10000000L
+#define RLC_MEM_SLP_CNTL__RLC_TC_MEM_DS_OVERRIDE_MASK                                                         0x20000000L
+#define RLC_MEM_SLP_CNTL__RESERVED1_MASK                                                                      0xC0000000L
+//RLC_RLCV_SAFE_MODE
+#define RLC_RLCV_SAFE_MODE__CMD__SHIFT                                                                        0x0
+#define RLC_RLCV_SAFE_MODE__MESSAGE__SHIFT                                                                    0x1
+#define RLC_RLCV_SAFE_MODE__RESERVED1__SHIFT                                                                  0x5
+#define RLC_RLCV_SAFE_MODE__RESPONSE__SHIFT                                                                   0x8
+#define RLC_RLCV_SAFE_MODE__RESERVED__SHIFT                                                                   0xc
+#define RLC_RLCV_SAFE_MODE__CMD_MASK                                                                          0x00000001L
+#define RLC_RLCV_SAFE_MODE__MESSAGE_MASK                                                                      0x0000001EL
+#define RLC_RLCV_SAFE_MODE__RESERVED1_MASK                                                                    0x000000E0L
+#define RLC_RLCV_SAFE_MODE__RESPONSE_MASK                                                                     0x00000F00L
+#define RLC_RLCV_SAFE_MODE__RESERVED_MASK                                                                     0xFFFFF000L
+//RLC_SMU_SAFE_MODE
+#define RLC_SMU_SAFE_MODE__CMD__SHIFT                                                                         0x0
+#define RLC_SMU_SAFE_MODE__MESSAGE__SHIFT                                                                     0x1
+#define RLC_SMU_SAFE_MODE__RESERVED1__SHIFT                                                                   0x5
+#define RLC_SMU_SAFE_MODE__RESPONSE__SHIFT                                                                    0x8
+#define RLC_SMU_SAFE_MODE__RESERVED__SHIFT                                                                    0xc
+#define RLC_SMU_SAFE_MODE__CMD_MASK                                                                           0x00000001L
+#define RLC_SMU_SAFE_MODE__MESSAGE_MASK                                                                       0x0000001EL
+#define RLC_SMU_SAFE_MODE__RESERVED1_MASK                                                                     0x000000E0L
+#define RLC_SMU_SAFE_MODE__RESPONSE_MASK                                                                      0x00000F00L
+#define RLC_SMU_SAFE_MODE__RESERVED_MASK                                                                      0xFFFFF000L
+//RLC_RLCV_COMMAND
+#define RLC_RLCV_COMMAND__CMD__SHIFT                                                                          0x0
+#define RLC_RLCV_COMMAND__RESERVED__SHIFT                                                                     0x4
+#define RLC_RLCV_COMMAND__CMD_MASK                                                                            0x0000000FL
+#define RLC_RLCV_COMMAND__RESERVED_MASK                                                                       0xFFFFFFF0L
+//RLC_SMU_MESSAGE
+#define RLC_SMU_MESSAGE__CMD__SHIFT                                                                           0x0
+#define RLC_SMU_MESSAGE__CMD_MASK                                                                             0xFFFFFFFFL
+//RLC_SMU_MESSAGE_1
+#define RLC_SMU_MESSAGE_1__CMD__SHIFT                                                                         0x0
+#define RLC_SMU_MESSAGE_1__CMD_MASK                                                                           0xFFFFFFFFL
+//RLC_SMU_MESSAGE_2
+#define RLC_SMU_MESSAGE_2__CMD__SHIFT                                                                         0x0
+#define RLC_SMU_MESSAGE_2__CMD_MASK                                                                           0xFFFFFFFFL
+//RLC_SRM_GPM_COMMAND
+#define RLC_SRM_GPM_COMMAND__OP__SHIFT                                                                        0x0
+#define RLC_SRM_GPM_COMMAND__INDEX_CNTL__SHIFT                                                                0x1
+#define RLC_SRM_GPM_COMMAND__INDEX_CNTL_NUM__SHIFT                                                            0x2
+#define RLC_SRM_GPM_COMMAND__SIZE__SHIFT                                                                      0x5
+#define RLC_SRM_GPM_COMMAND__START_OFFSET__SHIFT                                                              0x12
+#define RLC_SRM_GPM_COMMAND__RESERVED__SHIFT                                                                  0x1f
+#define RLC_SRM_GPM_COMMAND__OP_MASK                                                                          0x00000001L
+#define RLC_SRM_GPM_COMMAND__INDEX_CNTL_MASK                                                                  0x00000002L
+#define RLC_SRM_GPM_COMMAND__INDEX_CNTL_NUM_MASK                                                              0x0000001CL
+#define RLC_SRM_GPM_COMMAND__SIZE_MASK                                                                        0x0003FFE0L
+#define RLC_SRM_GPM_COMMAND__START_OFFSET_MASK                                                                0x7FFC0000L
+#define RLC_SRM_GPM_COMMAND__RESERVED_MASK                                                                    0x80000000L
+//RLC_SRM_GPM_ABORT
+#define RLC_SRM_GPM_ABORT__ABORT__SHIFT                                                                       0x0
+#define RLC_SRM_GPM_ABORT__RESERVED__SHIFT                                                                    0x1
+#define RLC_SRM_GPM_ABORT__ABORT_MASK                                                                         0x00000001L
+#define RLC_SRM_GPM_ABORT__RESERVED_MASK                                                                      0xFFFFFFFEL
+//RLC_SMU_COMMAND
+#define RLC_SMU_COMMAND__CMD__SHIFT                                                                           0x0
+#define RLC_SMU_COMMAND__CMD_MASK                                                                             0xFFFFFFFFL
+//RLC_SMU_ARGUMENT_1
+#define RLC_SMU_ARGUMENT_1__ARG__SHIFT                                                                        0x0
+#define RLC_SMU_ARGUMENT_1__ARG_MASK                                                                          0xFFFFFFFFL
+//RLC_SMU_ARGUMENT_2
+#define RLC_SMU_ARGUMENT_2__ARG__SHIFT                                                                        0x0
+#define RLC_SMU_ARGUMENT_2__ARG_MASK                                                                          0xFFFFFFFFL
+//RLC_SMU_ARGUMENT_3
+#define RLC_SMU_ARGUMENT_3__ARG__SHIFT                                                                        0x0
+#define RLC_SMU_ARGUMENT_3__ARG_MASK                                                                          0xFFFFFFFFL
+//RLC_SMU_ARGUMENT_4
+#define RLC_SMU_ARGUMENT_4__ARG__SHIFT                                                                        0x0
+#define RLC_SMU_ARGUMENT_4__ARG_MASK                                                                          0xFFFFFFFFL
+//RLC_SMU_ARGUMENT_5
+#define RLC_SMU_ARGUMENT_5__ARG__SHIFT                                                                        0x0
+#define RLC_SMU_ARGUMENT_5__ARG_MASK                                                                          0xFFFFFFFFL
+//RLC_IMU_BOOTLOAD_ADDR_HI
+#define RLC_IMU_BOOTLOAD_ADDR_HI__ADDR_HI__SHIFT                                                              0x0
+#define RLC_IMU_BOOTLOAD_ADDR_HI__ADDR_HI_MASK                                                                0xFFFFFFFFL
+//RLC_IMU_BOOTLOAD_ADDR_LO
+#define RLC_IMU_BOOTLOAD_ADDR_LO__ADDR_LO__SHIFT                                                              0x0
+#define RLC_IMU_BOOTLOAD_ADDR_LO__ADDR_LO_MASK                                                                0xFFFFFFFFL
+//RLC_IMU_BOOTLOAD_SIZE
+#define RLC_IMU_BOOTLOAD_SIZE__SIZE__SHIFT                                                                    0x0
+#define RLC_IMU_BOOTLOAD_SIZE__RESERVED__SHIFT                                                                0x1a
+#define RLC_IMU_BOOTLOAD_SIZE__SIZE_MASK                                                                      0x03FFFFFFL
+#define RLC_IMU_BOOTLOAD_SIZE__RESERVED_MASK                                                                  0xFC000000L
+//RLC_IMU_MISC
+#define RLC_IMU_MISC__THROTTLE_GFX__SHIFT                                                                     0x0
+#define RLC_IMU_MISC__EARLY_MGCG__SHIFT                                                                       0x1
+#define RLC_IMU_MISC__RESERVED__SHIFT                                                                         0x2
+#define RLC_IMU_MISC__THROTTLE_GFX_MASK                                                                       0x00000001L
+#define RLC_IMU_MISC__EARLY_MGCG_MASK                                                                         0x00000002L
+#define RLC_IMU_MISC__RESERVED_MASK                                                                           0xFFFFFFFCL
+//RLC_IMU_RESET_VECTOR
+#define RLC_IMU_RESET_VECTOR__COLD_BOOT_EXIT__SHIFT                                                           0x0
+#define RLC_IMU_RESET_VECTOR__VDDGFX_EXIT__SHIFT                                                              0x1
+#define RLC_IMU_RESET_VECTOR__VECTOR__SHIFT                                                                   0x2
+#define RLC_IMU_RESET_VECTOR__RESERVED__SHIFT                                                                 0x8
+#define RLC_IMU_RESET_VECTOR__COLD_BOOT_EXIT_MASK                                                             0x00000001L
+#define RLC_IMU_RESET_VECTOR__VDDGFX_EXIT_MASK                                                                0x00000002L
+#define RLC_IMU_RESET_VECTOR__VECTOR_MASK                                                                     0x000000FCL
+#define RLC_IMU_RESET_VECTOR__RESERVED_MASK                                                                   0xFFFFFF00L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_rlcsdec
+//RLC_RLCS_DEC_START
+//RLC_RLCS_DEC_DUMP_ADDR
+//RLC_RLCS_EXCEPTION_REG_1
+#define RLC_RLCS_EXCEPTION_REG_1__ADDR__SHIFT                                                                 0x0
+#define RLC_RLCS_EXCEPTION_REG_1__RESERVED__SHIFT                                                             0x12
+#define RLC_RLCS_EXCEPTION_REG_1__ADDR_MASK                                                                   0x0003FFFFL
+#define RLC_RLCS_EXCEPTION_REG_1__RESERVED_MASK                                                               0xFFFC0000L
+//RLC_RLCS_EXCEPTION_REG_2
+#define RLC_RLCS_EXCEPTION_REG_2__ADDR__SHIFT                                                                 0x0
+#define RLC_RLCS_EXCEPTION_REG_2__RESERVED__SHIFT                                                             0x12
+#define RLC_RLCS_EXCEPTION_REG_2__ADDR_MASK                                                                   0x0003FFFFL
+#define RLC_RLCS_EXCEPTION_REG_2__RESERVED_MASK                                                               0xFFFC0000L
+//RLC_RLCS_EXCEPTION_REG_3
+#define RLC_RLCS_EXCEPTION_REG_3__ADDR__SHIFT                                                                 0x0
+#define RLC_RLCS_EXCEPTION_REG_3__RESERVED__SHIFT                                                             0x12
+#define RLC_RLCS_EXCEPTION_REG_3__ADDR_MASK                                                                   0x0003FFFFL
+#define RLC_RLCS_EXCEPTION_REG_3__RESERVED_MASK                                                               0xFFFC0000L
+//RLC_RLCS_EXCEPTION_REG_4
+#define RLC_RLCS_EXCEPTION_REG_4__ADDR__SHIFT                                                                 0x0
+#define RLC_RLCS_EXCEPTION_REG_4__RESERVED__SHIFT                                                             0x12
+#define RLC_RLCS_EXCEPTION_REG_4__ADDR_MASK                                                                   0x0003FFFFL
+#define RLC_RLCS_EXCEPTION_REG_4__RESERVED_MASK                                                               0xFFFC0000L
+//RLC_RLCS_FENCE_CNTL
+#define RLC_RLCS_FENCE_CNTL__MODE__SHIFT                                                                      0x0
+#define RLC_RLCS_FENCE_CNTL__SRC_ID_EXC_EN__SHIFT                                                             0x2
+#define RLC_RLCS_FENCE_CNTL__SRC_ID_EXC__SHIFT                                                                0x4
+#define RLC_RLCS_FENCE_CNTL__MODE_MASK                                                                        0x00000003L
+#define RLC_RLCS_FENCE_CNTL__SRC_ID_EXC_EN_MASK                                                               0x00000004L
+#define RLC_RLCS_FENCE_CNTL__SRC_ID_EXC_MASK                                                                  0x000000F0L
+//RLC_RLCS_CGCG_REQUEST
+#define RLC_RLCS_CGCG_REQUEST__CGCG_REQUEST__SHIFT                                                            0x0
+#define RLC_RLCS_CGCG_REQUEST__RESERVED__SHIFT                                                                0x1
+#define RLC_RLCS_CGCG_REQUEST__CGCG_REQUEST_MASK                                                              0x00000001L
+#define RLC_RLCS_CGCG_REQUEST__RESERVED_MASK                                                                  0xFFFFFFFEL
+//RLC_RLCS_CGCG_STATUS
+#define RLC_RLCS_CGCG_STATUS__CGCG_RAMP_STATUS__SHIFT                                                         0x0
+#define RLC_RLCS_CGCG_STATUS__GFX_CLK_STATUS__SHIFT                                                           0x2
+#define RLC_RLCS_CGCG_STATUS__RESERVED__SHIFT                                                                 0x3
+#define RLC_RLCS_CGCG_STATUS__CGCG_RAMP_STATUS_MASK                                                           0x00000003L
+#define RLC_RLCS_CGCG_STATUS__GFX_CLK_STATUS_MASK                                                             0x00000004L
+#define RLC_RLCS_CGCG_STATUS__RESERVED_MASK                                                                   0xFFFFFFF8L
+//RLC_RLCS_SOC_DS_CNTL
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_ALLOW__SHIFT                                                         0x0
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_RLC_BUSY_MASK__SHIFT                                                 0x1
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_CP_BUSY_MASK__SHIFT                                                  0x2
+#define RLC_RLCS_SOC_DS_CNTL__RESERVED_4_3__SHIFT                                                             0x3
+#define RLC_RLCS_SOC_DS_CNTL__RESERVED_5__SHIFT                                                               0x5
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_GFX_PWR_STALLED_MASK__SHIFT                                          0x6
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_NON3D_PWR_STALLED_MASK__SHIFT                                        0x7
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_0_BUSY_MASK__SHIFT                                              0x10
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_1_BUSY_MASK__SHIFT                                              0x11
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_2_BUSY_MASK__SHIFT                                              0x12
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_3_BUSY_MASK__SHIFT                                              0x13
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_4_BUSY_MASK__SHIFT                                              0x14
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_5_BUSY_MASK__SHIFT                                              0x15
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_6_BUSY_MASK__SHIFT                                              0x16
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_7_BUSY_MASK__SHIFT                                              0x17
+#define RLC_RLCS_SOC_DS_CNTL__RESERVED_31_24__SHIFT                                                           0x18
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_ALLOW_MASK                                                           0x00000001L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_RLC_BUSY_MASK_MASK                                                   0x00000002L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_CP_BUSY_MASK_MASK                                                    0x00000004L
+#define RLC_RLCS_SOC_DS_CNTL__RESERVED_4_3_MASK                                                               0x00000018L
+#define RLC_RLCS_SOC_DS_CNTL__RESERVED_5_MASK                                                                 0x00000020L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_GFX_PWR_STALLED_MASK_MASK                                            0x00000040L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_NON3D_PWR_STALLED_MASK_MASK                                          0x00000080L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_0_BUSY_MASK_MASK                                                0x00010000L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_1_BUSY_MASK_MASK                                                0x00020000L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_2_BUSY_MASK_MASK                                                0x00040000L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_3_BUSY_MASK_MASK                                                0x00080000L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_4_BUSY_MASK_MASK                                                0x00100000L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_5_BUSY_MASK_MASK                                                0x00200000L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_6_BUSY_MASK_MASK                                                0x00400000L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_7_BUSY_MASK_MASK                                                0x00800000L
+#define RLC_RLCS_SOC_DS_CNTL__RESERVED_31_24_MASK                                                             0xFF000000L
+//RLC_RLCS_GFX_DS_CNTL
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_ALLOW__SHIFT                                                         0x0
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_RLC_BUSY_MASK__SHIFT                                                 0x1
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_CP_BUSY_MASK__SHIFT                                                  0x2
+#define RLC_RLCS_GFX_DS_CNTL__RESERVED_4_3__SHIFT                                                             0x3
+#define RLC_RLCS_GFX_DS_CNTL__RESERVED_5__SHIFT                                                               0x5
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_GFX_PWR_STALLED_MASK__SHIFT                                          0x6
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_NON3D_PWR_STALLED_MASK__SHIFT                                        0x7
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_IMU_DISABLE_MASK__SHIFT                                              0x8
+#define RLC_RLCS_GFX_DS_CNTL__RESERVED_15_9__SHIFT                                                            0x9
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_0_BUSY_MASK__SHIFT                                              0x10
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_1_BUSY_MASK__SHIFT                                              0x11
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_2_BUSY_MASK__SHIFT                                              0x12
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_3_BUSY_MASK__SHIFT                                              0x13
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_4_BUSY_MASK__SHIFT                                              0x14
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_5_BUSY_MASK__SHIFT                                              0x15
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_6_BUSY_MASK__SHIFT                                              0x16
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_7_BUSY_MASK__SHIFT                                              0x17
+#define RLC_RLCS_GFX_DS_CNTL__RESERVED__SHIFT                                                                 0x18
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_ALLOW_MASK                                                           0x00000001L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_RLC_BUSY_MASK_MASK                                                   0x00000002L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_CP_BUSY_MASK_MASK                                                    0x00000004L
+#define RLC_RLCS_GFX_DS_CNTL__RESERVED_4_3_MASK                                                               0x00000018L
+#define RLC_RLCS_GFX_DS_CNTL__RESERVED_5_MASK                                                                 0x00000020L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_GFX_PWR_STALLED_MASK_MASK                                            0x00000040L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_NON3D_PWR_STALLED_MASK_MASK                                          0x00000080L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_IMU_DISABLE_MASK_MASK                                                0x00000100L
+#define RLC_RLCS_GFX_DS_CNTL__RESERVED_15_9_MASK                                                              0x0000FE00L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_0_BUSY_MASK_MASK                                                0x00010000L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_1_BUSY_MASK_MASK                                                0x00020000L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_2_BUSY_MASK_MASK                                                0x00040000L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_3_BUSY_MASK_MASK                                                0x00080000L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_4_BUSY_MASK_MASK                                                0x00100000L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_5_BUSY_MASK_MASK                                                0x00200000L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_6_BUSY_MASK_MASK                                                0x00400000L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_7_BUSY_MASK_MASK                                                0x00800000L
+#define RLC_RLCS_GFX_DS_CNTL__RESERVED_MASK                                                                   0xFF000000L
+//RLC_RLCS_GFX_DS_ALLOW_MASK_CNTL
+//RLC_GPM_STAT
+#define RLC_GPM_STAT__RLC_BUSY__SHIFT                                                                         0x0
+#define RLC_GPM_STAT__GFX_POWER_STATUS__SHIFT                                                                 0x1
+#define RLC_GPM_STAT__GFX_CLOCK_STATUS__SHIFT                                                                 0x2
+#define RLC_GPM_STAT__GFX_LS_STATUS__SHIFT                                                                    0x3
+#define RLC_GPM_STAT__GFX_PIPELINE_POWER_STATUS__SHIFT                                                        0x4
+#define RLC_GPM_STAT__CNTX_IDLE_BEING_PROCESSED__SHIFT                                                        0x5
+#define RLC_GPM_STAT__CNTX_BUSY_BEING_PROCESSED__SHIFT                                                        0x6
+#define RLC_GPM_STAT__GFX_IDLE_BEING_PROCESSED__SHIFT                                                         0x7
+#define RLC_GPM_STAT__CMP_BUSY_BEING_PROCESSED__SHIFT                                                         0x8
+#define RLC_GPM_STAT__SAVING_REGISTERS__SHIFT                                                                 0x9
+#define RLC_GPM_STAT__RESTORING_REGISTERS__SHIFT                                                              0xa
+#define RLC_GPM_STAT__GFX3D_BLOCKS_CHANGING_POWER_STATE__SHIFT                                                0xb
+#define RLC_GPM_STAT__CMP_BLOCKS_CHANGING_POWER_STATE__SHIFT                                                  0xc
+#define RLC_GPM_STAT__STATIC_WGP_POWERING_UP__SHIFT                                                           0xd
+#define RLC_GPM_STAT__STATIC_WGP_POWERING_DOWN__SHIFT                                                         0xe
+#define RLC_GPM_STAT__DYN_WGP_POWERING_UP__SHIFT                                                              0xf
+#define RLC_GPM_STAT__DYN_WGP_POWERING_DOWN__SHIFT                                                            0x10
+#define RLC_GPM_STAT__ABORTED_PD_SEQUENCE__SHIFT                                                              0x11
+#define RLC_GPM_STAT__CMP_power_status__SHIFT                                                                 0x12
+#define RLC_GPM_STAT__RESERVED_1__SHIFT                                                                       0x13
+#define RLC_GPM_STAT__MGCG_OVERRIDE_STATUS__SHIFT                                                             0x15
+#define RLC_GPM_STAT__RLC_EXEC_ROM_CODE__SHIFT                                                                0x16
+#define RLC_GPM_STAT__FGCG_OVERRIDE_STATUS__SHIFT                                                             0x17
+#define RLC_GPM_STAT__PG_ERROR_STATUS__SHIFT                                                                  0x18
+#define RLC_GPM_STAT__RLC_BUSY_MASK                                                                           0x00000001L
+#define RLC_GPM_STAT__GFX_POWER_STATUS_MASK                                                                   0x00000002L
+#define RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK                                                                   0x00000004L
+#define RLC_GPM_STAT__GFX_LS_STATUS_MASK                                                                      0x00000008L
+#define RLC_GPM_STAT__GFX_PIPELINE_POWER_STATUS_MASK                                                          0x00000010L
+#define RLC_GPM_STAT__CNTX_IDLE_BEING_PROCESSED_MASK                                                          0x00000020L
+#define RLC_GPM_STAT__CNTX_BUSY_BEING_PROCESSED_MASK                                                          0x00000040L
+#define RLC_GPM_STAT__GFX_IDLE_BEING_PROCESSED_MASK                                                           0x00000080L
+#define RLC_GPM_STAT__CMP_BUSY_BEING_PROCESSED_MASK                                                           0x00000100L
+#define RLC_GPM_STAT__SAVING_REGISTERS_MASK                                                                   0x00000200L
+#define RLC_GPM_STAT__RESTORING_REGISTERS_MASK                                                                0x00000400L
+#define RLC_GPM_STAT__GFX3D_BLOCKS_CHANGING_POWER_STATE_MASK                                                  0x00000800L
+#define RLC_GPM_STAT__CMP_BLOCKS_CHANGING_POWER_STATE_MASK                                                    0x00001000L
+#define RLC_GPM_STAT__STATIC_WGP_POWERING_UP_MASK                                                             0x00002000L
+#define RLC_GPM_STAT__STATIC_WGP_POWERING_DOWN_MASK                                                           0x00004000L
+#define RLC_GPM_STAT__DYN_WGP_POWERING_UP_MASK                                                                0x00008000L
+#define RLC_GPM_STAT__DYN_WGP_POWERING_DOWN_MASK                                                              0x00010000L
+#define RLC_GPM_STAT__ABORTED_PD_SEQUENCE_MASK                                                                0x00020000L
+#define RLC_GPM_STAT__CMP_power_status_MASK                                                                   0x00040000L
+#define RLC_GPM_STAT__RESERVED_1_MASK                                                                         0x00180000L
+#define RLC_GPM_STAT__MGCG_OVERRIDE_STATUS_MASK                                                               0x00200000L
+#define RLC_GPM_STAT__RLC_EXEC_ROM_CODE_MASK                                                                  0x00400000L
+#define RLC_GPM_STAT__FGCG_OVERRIDE_STATUS_MASK                                                               0x00800000L
+#define RLC_GPM_STAT__PG_ERROR_STATUS_MASK                                                                    0xFF000000L
+//RLC_RLCS_GPM_STAT
+#define RLC_RLCS_GPM_STAT__RLC_BUSY__SHIFT                                                                    0x0
+#define RLC_RLCS_GPM_STAT__GFX_POWER_STATUS__SHIFT                                                            0x1
+#define RLC_RLCS_GPM_STAT__GFX_CLOCK_STATUS__SHIFT                                                            0x2
+#define RLC_RLCS_GPM_STAT__GFX_LS_STATUS__SHIFT                                                               0x3
+#define RLC_RLCS_GPM_STAT__GFX_PIPELINE_POWER_STATUS__SHIFT                                                   0x4
+#define RLC_RLCS_GPM_STAT__CNTX_IDLE_BEING_PROCESSED__SHIFT                                                   0x5
+#define RLC_RLCS_GPM_STAT__CNTX_BUSY_BEING_PROCESSED__SHIFT                                                   0x6
+#define RLC_RLCS_GPM_STAT__GFX_IDLE_BEING_PROCESSED__SHIFT                                                    0x7
+#define RLC_RLCS_GPM_STAT__CMP_BUSY_BEING_PROCESSED__SHIFT                                                    0x8
+#define RLC_RLCS_GPM_STAT__SAVING_REGISTERS__SHIFT                                                            0x9
+#define RLC_RLCS_GPM_STAT__RESTORING_REGISTERS__SHIFT                                                         0xa
+#define RLC_RLCS_GPM_STAT__GFX3D_BLOCKS_CHANGING_POWER_STATE__SHIFT                                           0xb
+#define RLC_RLCS_GPM_STAT__CMP_BLOCKS_CHANGING_POWER_STATE__SHIFT                                             0xc
+#define RLC_RLCS_GPM_STAT__STATIC_WGP_POWERING_UP__SHIFT                                                      0xd
+#define RLC_RLCS_GPM_STAT__STATIC_WGP_POWERING_DOWN__SHIFT                                                    0xe
+#define RLC_RLCS_GPM_STAT__DYN_WGP_POWERING_UP__SHIFT                                                         0xf
+#define RLC_RLCS_GPM_STAT__DYN_WGP_POWERING_DOWN__SHIFT                                                       0x10
+#define RLC_RLCS_GPM_STAT__ABORTED_PD_SEQUENCE__SHIFT                                                         0x11
+#define RLC_RLCS_GPM_STAT__CMP_POWER_STATUS__SHIFT                                                            0x12
+#define RLC_RLCS_GPM_STAT__RESERVED_1__SHIFT                                                                  0x13
+#define RLC_RLCS_GPM_STAT__MGCG_OVERRIDE_STATUS__SHIFT                                                        0x15
+#define RLC_RLCS_GPM_STAT__RLC_EXEC_ROM_CODE__SHIFT                                                           0x16
+#define RLC_RLCS_GPM_STAT__FGCG_OVERRIDE_STATUS__SHIFT                                                        0x17
+#define RLC_RLCS_GPM_STAT__PG_ERROR_STATUS__SHIFT                                                             0x18
+#define RLC_RLCS_GPM_STAT__RLC_BUSY_MASK                                                                      0x00000001L
+#define RLC_RLCS_GPM_STAT__GFX_POWER_STATUS_MASK                                                              0x00000002L
+#define RLC_RLCS_GPM_STAT__GFX_CLOCK_STATUS_MASK                                                              0x00000004L
+#define RLC_RLCS_GPM_STAT__GFX_LS_STATUS_MASK                                                                 0x00000008L
+#define RLC_RLCS_GPM_STAT__GFX_PIPELINE_POWER_STATUS_MASK                                                     0x00000010L
+#define RLC_RLCS_GPM_STAT__CNTX_IDLE_BEING_PROCESSED_MASK                                                     0x00000020L
+#define RLC_RLCS_GPM_STAT__CNTX_BUSY_BEING_PROCESSED_MASK                                                     0x00000040L
+#define RLC_RLCS_GPM_STAT__GFX_IDLE_BEING_PROCESSED_MASK                                                      0x00000080L
+#define RLC_RLCS_GPM_STAT__CMP_BUSY_BEING_PROCESSED_MASK                                                      0x00000100L
+#define RLC_RLCS_GPM_STAT__SAVING_REGISTERS_MASK                                                              0x00000200L
+#define RLC_RLCS_GPM_STAT__RESTORING_REGISTERS_MASK                                                           0x00000400L
+#define RLC_RLCS_GPM_STAT__GFX3D_BLOCKS_CHANGING_POWER_STATE_MASK                                             0x00000800L
+#define RLC_RLCS_GPM_STAT__CMP_BLOCKS_CHANGING_POWER_STATE_MASK                                               0x00001000L
+#define RLC_RLCS_GPM_STAT__STATIC_WGP_POWERING_UP_MASK                                                        0x00002000L
+#define RLC_RLCS_GPM_STAT__STATIC_WGP_POWERING_DOWN_MASK                                                      0x00004000L
+#define RLC_RLCS_GPM_STAT__DYN_WGP_POWERING_UP_MASK                                                           0x00008000L
+#define RLC_RLCS_GPM_STAT__DYN_WGP_POWERING_DOWN_MASK                                                         0x00010000L
+#define RLC_RLCS_GPM_STAT__ABORTED_PD_SEQUENCE_MASK                                                           0x00020000L
+#define RLC_RLCS_GPM_STAT__CMP_POWER_STATUS_MASK                                                              0x00040000L
+#define RLC_RLCS_GPM_STAT__RESERVED_1_MASK                                                                    0x00180000L
+#define RLC_RLCS_GPM_STAT__MGCG_OVERRIDE_STATUS_MASK                                                          0x00200000L
+#define RLC_RLCS_GPM_STAT__RLC_EXEC_ROM_CODE_MASK                                                             0x00400000L
+#define RLC_RLCS_GPM_STAT__FGCG_OVERRIDE_STATUS_MASK                                                          0x00800000L
+#define RLC_RLCS_GPM_STAT__PG_ERROR_STATUS_MASK                                                               0xFF000000L
+//RLC_RLCS_ABORTED_PD_SEQUENCE
+#define RLC_RLCS_ABORTED_PD_SEQUENCE__APS__SHIFT                                                              0x0
+#define RLC_RLCS_ABORTED_PD_SEQUENCE__RESERVED__SHIFT                                                         0x10
+#define RLC_RLCS_ABORTED_PD_SEQUENCE__APS_MASK                                                                0x0000FFFFL
+#define RLC_RLCS_ABORTED_PD_SEQUENCE__RESERVED_MASK                                                           0xFFFF0000L
+//RLC_RLCS_IOV_CMD_STATUS
+#define RLC_RLCS_IOV_CMD_STATUS__DATA__SHIFT                                                                  0x0
+#define RLC_RLCS_IOV_CMD_STATUS__DATA_MASK                                                                    0xFFFFFFFFL
+//RLC_RLCS_IOV_CNTX_LOC_SIZE
+#define RLC_RLCS_IOV_CNTX_LOC_SIZE__DATA__SHIFT                                                               0x0
+#define RLC_RLCS_IOV_CNTX_LOC_SIZE__RESERVED__SHIFT                                                           0x8
+#define RLC_RLCS_IOV_CNTX_LOC_SIZE__DATA_MASK                                                                 0x000000FFL
+#define RLC_RLCS_IOV_CNTX_LOC_SIZE__RESERVED_MASK                                                             0xFFFFFF00L
+//RLC_RLCS_IOV_SCH_BLOCK
+#define RLC_RLCS_IOV_SCH_BLOCK__DATA__SHIFT                                                                   0x0
+#define RLC_RLCS_IOV_SCH_BLOCK__DATA_MASK                                                                     0xFFFFFFFFL
+//RLC_RLCS_IOV_VM_BUSY_STATUS
+#define RLC_RLCS_IOV_VM_BUSY_STATUS__DATA__SHIFT                                                              0x0
+#define RLC_RLCS_IOV_VM_BUSY_STATUS__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCS_GPM_STAT_2
+#define RLC_RLCS_GPM_STAT_2__TC_TRANS_ERROR__SHIFT                                                            0x0
+#define RLC_RLCS_GPM_STAT_2__RLC_PWR_NON3D_STALLED__SHIFT                                                     0x1
+#define RLC_RLCS_GPM_STAT_2__GFX_PWR_STALLED_STATUS__SHIFT                                                    0x2
+#define RLC_RLCS_GPM_STAT_2__GFX_ULV_STATUS__SHIFT                                                            0x3
+#define RLC_RLCS_GPM_STAT_2__GFX_GENERAL_STATUS__SHIFT                                                        0x4
+#define RLC_RLCS_GPM_STAT_2__RESERVED__SHIFT                                                                  0x5
+#define RLC_RLCS_GPM_STAT_2__TC_TRANS_ERROR_MASK                                                              0x00000001L
+#define RLC_RLCS_GPM_STAT_2__RLC_PWR_NON3D_STALLED_MASK                                                       0x00000002L
+#define RLC_RLCS_GPM_STAT_2__GFX_PWR_STALLED_STATUS_MASK                                                      0x00000004L
+#define RLC_RLCS_GPM_STAT_2__GFX_ULV_STATUS_MASK                                                              0x00000008L
+#define RLC_RLCS_GPM_STAT_2__GFX_GENERAL_STATUS_MASK                                                          0x00000010L
+#define RLC_RLCS_GPM_STAT_2__RESERVED_MASK                                                                    0xFFFFFFE0L
+//RLC_RLCS_GRBM_SOFT_RESET
+#define RLC_RLCS_GRBM_SOFT_RESET__RESET__SHIFT                                                                0x0
+#define RLC_RLCS_GRBM_SOFT_RESET__RESERVED__SHIFT                                                             0x1
+#define RLC_RLCS_GRBM_SOFT_RESET__RESET_MASK                                                                  0x00000001L
+#define RLC_RLCS_GRBM_SOFT_RESET__RESERVED_MASK                                                               0xFFFFFFFEL
+//RLC_RLCS_PG_CHANGE_STATUS
+#define RLC_RLCS_PG_CHANGE_STATUS__PG_CNTL_CHANGED__SHIFT                                                     0x0
+#define RLC_RLCS_PG_CHANGE_STATUS__PG_REG_CHANGED__SHIFT                                                      0x1
+#define RLC_RLCS_PG_CHANGE_STATUS__DYN_PG_STATUS_CHANGED__SHIFT                                               0x2
+#define RLC_RLCS_PG_CHANGE_STATUS__DYN_PG_REQ_CHANGED__SHIFT                                                  0x3
+#define RLC_RLCS_PG_CHANGE_STATUS__RESERVED__SHIFT                                                            0x4
+#define RLC_RLCS_PG_CHANGE_STATUS__PG_CNTL_CHANGED_MASK                                                       0x00000001L
+#define RLC_RLCS_PG_CHANGE_STATUS__PG_REG_CHANGED_MASK                                                        0x00000002L
+#define RLC_RLCS_PG_CHANGE_STATUS__DYN_PG_STATUS_CHANGED_MASK                                                 0x00000004L
+#define RLC_RLCS_PG_CHANGE_STATUS__DYN_PG_REQ_CHANGED_MASK                                                    0x00000008L
+#define RLC_RLCS_PG_CHANGE_STATUS__RESERVED_MASK                                                              0xFFFFFFF0L
+//RLC_RLCS_PG_CHANGE_READ
+#define RLC_RLCS_PG_CHANGE_READ__RESERVED__SHIFT                                                              0x0
+#define RLC_RLCS_PG_CHANGE_READ__PG_REG_CHANGED__SHIFT                                                        0x1
+#define RLC_RLCS_PG_CHANGE_READ__DYN_PG_STATUS_CHANGED__SHIFT                                                 0x2
+#define RLC_RLCS_PG_CHANGE_READ__DYN_PG_REQ_CHANGED__SHIFT                                                    0x3
+#define RLC_RLCS_PG_CHANGE_READ__RESERVED_MASK                                                                0x00000001L
+#define RLC_RLCS_PG_CHANGE_READ__PG_REG_CHANGED_MASK                                                          0x00000002L
+#define RLC_RLCS_PG_CHANGE_READ__DYN_PG_STATUS_CHANGED_MASK                                                   0x00000004L
+#define RLC_RLCS_PG_CHANGE_READ__DYN_PG_REQ_CHANGED_MASK                                                      0x00000008L
+//RLC_RLCS_IH_SEMAPHORE
+#define RLC_RLCS_IH_SEMAPHORE__CLIENT_ID__SHIFT                                                               0x0
+#define RLC_RLCS_IH_SEMAPHORE__CLIENT_ID_MASK                                                                 0x0000001FL
+//RLC_RLCS_IH_COOKIE_SEMAPHORE
+#define RLC_RLCS_IH_COOKIE_SEMAPHORE__CLIENT_ID__SHIFT                                                        0x0
+#define RLC_RLCS_IH_COOKIE_SEMAPHORE__CLIENT_ID_MASK                                                          0x0000001FL
+//RLC_RLCS_CP_INT_CTRL_1
+#define RLC_RLCS_CP_INT_CTRL_1__INTERRUPT_ACK__SHIFT                                                          0x0
+#define RLC_RLCS_CP_INT_CTRL_1__RESERVED__SHIFT                                                               0x1
+#define RLC_RLCS_CP_INT_CTRL_1__INTERRUPT_ACK_MASK                                                            0x00000001L
+#define RLC_RLCS_CP_INT_CTRL_1__RESERVED_MASK                                                                 0xFFFFFFFEL
+//RLC_RLCS_CP_INT_CTRL_2
+#define RLC_RLCS_CP_INT_CTRL_2__IDLE_AUTO_ACK_EN__SHIFT                                                       0x0
+#define RLC_RLCS_CP_INT_CTRL_2__BUSY_AUTO_ACK_EN__SHIFT                                                       0x1
+#define RLC_RLCS_CP_INT_CTRL_2__IDLE_AUTO_ACK_ACTIVE__SHIFT                                                   0x2
+#define RLC_RLCS_CP_INT_CTRL_2__BUSY_AUTO_ACK_ACTIVE__SHIFT                                                   0x3
+#define RLC_RLCS_CP_INT_CTRL_2__INTERRUPT_PENDING__SHIFT                                                      0x4
+#define RLC_RLCS_CP_INT_CTRL_2__RESERVED__SHIFT                                                               0x5
+#define RLC_RLCS_CP_INT_CTRL_2__IDLE_AUTO_ACK_EN_MASK                                                         0x00000001L
+#define RLC_RLCS_CP_INT_CTRL_2__BUSY_AUTO_ACK_EN_MASK                                                         0x00000002L
+#define RLC_RLCS_CP_INT_CTRL_2__IDLE_AUTO_ACK_ACTIVE_MASK                                                     0x00000004L
+#define RLC_RLCS_CP_INT_CTRL_2__BUSY_AUTO_ACK_ACTIVE_MASK                                                     0x00000008L
+#define RLC_RLCS_CP_INT_CTRL_2__INTERRUPT_PENDING_MASK                                                        0x00000010L
+#define RLC_RLCS_CP_INT_CTRL_2__RESERVED_MASK                                                                 0xFFFFFFE0L
+//RLC_RLCS_CP_INT_INFO_1
+#define RLC_RLCS_CP_INT_INFO_1__INTERRUPT_INFO_1__SHIFT                                                       0x0
+#define RLC_RLCS_CP_INT_INFO_1__INTERRUPT_INFO_1_MASK                                                         0xFFFFFFFFL
+//RLC_RLCS_CP_INT_INFO_2
+#define RLC_RLCS_CP_INT_INFO_2__INTERRUPT_INFO_2__SHIFT                                                       0x0
+#define RLC_RLCS_CP_INT_INFO_2__INTERRUPT_ID__SHIFT                                                           0x10
+#define RLC_RLCS_CP_INT_INFO_2__RESERVED__SHIFT                                                               0x19
+#define RLC_RLCS_CP_INT_INFO_2__INTERRUPT_INFO_2_MASK                                                         0x0000FFFFL
+#define RLC_RLCS_CP_INT_INFO_2__INTERRUPT_ID_MASK                                                             0x01FF0000L
+#define RLC_RLCS_CP_INT_INFO_2__RESERVED_MASK                                                                 0xFE000000L
+//RLC_RLCS_SPM_INT_CTRL
+#define RLC_RLCS_SPM_INT_CTRL__INTERRUPT_ACK__SHIFT                                                           0x0
+#define RLC_RLCS_SPM_INT_CTRL__RESERVED__SHIFT                                                                0x1
+#define RLC_RLCS_SPM_INT_CTRL__INTERRUPT_ACK_MASK                                                             0x00000001L
+#define RLC_RLCS_SPM_INT_CTRL__RESERVED_MASK                                                                  0xFFFFFFFEL
+//RLC_RLCS_SPM_INT_INFO_1
+#define RLC_RLCS_SPM_INT_INFO_1__INTERRUPT_INFO_1__SHIFT                                                      0x0
+#define RLC_RLCS_SPM_INT_INFO_1__INTERRUPT_INFO_1_MASK                                                        0xFFFFFFFFL
+//RLC_RLCS_SPM_INT_INFO_2
+#define RLC_RLCS_SPM_INT_INFO_2__INTERRUPT_INFO_2__SHIFT                                                      0x0
+#define RLC_RLCS_SPM_INT_INFO_2__INTERRUPT_ID__SHIFT                                                          0x10
+#define RLC_RLCS_SPM_INT_INFO_2__RESERVED__SHIFT                                                              0x19
+#define RLC_RLCS_SPM_INT_INFO_2__INTERRUPT_INFO_2_MASK                                                        0x0000FFFFL
+#define RLC_RLCS_SPM_INT_INFO_2__INTERRUPT_ID_MASK                                                            0x01FF0000L
+#define RLC_RLCS_SPM_INT_INFO_2__RESERVED_MASK                                                                0xFE000000L
+//RLC_RLCS_DSM_TRIG
+#define RLC_RLCS_DSM_TRIG__START__SHIFT                                                                       0x0
+#define RLC_RLCS_DSM_TRIG__RESERVED__SHIFT                                                                    0x1
+#define RLC_RLCS_DSM_TRIG__START_MASK                                                                         0x00000001L
+#define RLC_RLCS_DSM_TRIG__RESERVED_MASK                                                                      0xFFFFFFFEL
+//RLC_RLCS_BOOTLOAD_STATUS
+#define RLC_RLCS_BOOTLOAD_STATUS__GFX_FUSE_DIST_DONE__SHIFT                                                   0x0
+#define RLC_RLCS_BOOTLOAD_STATUS__GFX_INIT_DONE__SHIFT                                                        0x1
+#define RLC_RLCS_BOOTLOAD_STATUS__RLC_GPM_IRAM_LOADED__SHIFT                                                  0x4
+#define RLC_RLCS_BOOTLOAD_STATUS__RLC_GPM_IRAM_DONE__SHIFT                                                    0x5
+#define RLC_RLCS_BOOTLOAD_STATUS__STATUS_6_30__SHIFT                                                          0x6
+#define RLC_RLCS_BOOTLOAD_STATUS__BOOTLOAD_COMPLETE__SHIFT                                                    0x1f
+#define RLC_RLCS_BOOTLOAD_STATUS__GFX_FUSE_DIST_DONE_MASK                                                     0x00000001L
+#define RLC_RLCS_BOOTLOAD_STATUS__GFX_INIT_DONE_MASK                                                          0x00000002L
+#define RLC_RLCS_BOOTLOAD_STATUS__RLC_GPM_IRAM_LOADED_MASK                                                    0x00000010L
+#define RLC_RLCS_BOOTLOAD_STATUS__RLC_GPM_IRAM_DONE_MASK                                                      0x00000020L
+#define RLC_RLCS_BOOTLOAD_STATUS__STATUS_6_30_MASK                                                            0x7FFFFFC0L
+#define RLC_RLCS_BOOTLOAD_STATUS__BOOTLOAD_COMPLETE_MASK                                                      0x80000000L
+//RLC_RLCS_GRBM_IDLE_BUSY_STAT
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__GRBM_RLC_GC_STAT_IDLE__SHIFT                                            0x0
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_0_BUSY__SHIFT                                                      0x10
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_1_BUSY__SHIFT                                                      0x11
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_2_BUSY__SHIFT                                                      0x12
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_3_BUSY__SHIFT                                                      0x13
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_4_BUSY__SHIFT                                                      0x14
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_5_BUSY__SHIFT                                                      0x15
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_6_BUSY__SHIFT                                                      0x16
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_7_BUSY__SHIFT                                                      0x17
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_0_BUSY_CHANGED__SHIFT                                              0x18
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_1_BUSY_CHANGED__SHIFT                                              0x19
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_2_BUSY_CHANGED__SHIFT                                              0x1a
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_3_BUSY_CHANGED__SHIFT                                              0x1b
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_4_BUSY_CHANGED__SHIFT                                              0x1c
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_5_BUSY_CHANGED__SHIFT                                              0x1d
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_6_BUSY_CHANGED__SHIFT                                              0x1e
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_7_BUSY_CHANGED__SHIFT                                              0x1f
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__GRBM_RLC_GC_STAT_IDLE_MASK                                              0x00000003L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_0_BUSY_MASK                                                        0x00010000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_1_BUSY_MASK                                                        0x00020000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_2_BUSY_MASK                                                        0x00040000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_3_BUSY_MASK                                                        0x00080000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_4_BUSY_MASK                                                        0x00100000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_5_BUSY_MASK                                                        0x00200000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_6_BUSY_MASK                                                        0x00400000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_7_BUSY_MASK                                                        0x00800000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_0_BUSY_CHANGED_MASK                                                0x01000000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_1_BUSY_CHANGED_MASK                                                0x02000000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_2_BUSY_CHANGED_MASK                                                0x04000000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_3_BUSY_CHANGED_MASK                                                0x08000000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_4_BUSY_CHANGED_MASK                                                0x10000000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_5_BUSY_CHANGED_MASK                                                0x20000000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_6_BUSY_CHANGED_MASK                                                0x40000000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_7_BUSY_CHANGED_MASK                                                0x80000000L
+//RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA0_BUSY_INT_CLEAR__SHIFT                                         0x0
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA1_BUSY_INT_CLEAR__SHIFT                                         0x1
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA2_BUSY_INT_CLEAR__SHIFT                                         0x2
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA3_BUSY_INT_CLEAR__SHIFT                                         0x3
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA4_BUSY_INT_CLEAR__SHIFT                                         0x4
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA5_BUSY_INT_CLEAR__SHIFT                                         0x5
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA6_BUSY_INT_CLEAR__SHIFT                                         0x6
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA7_BUSY_INT_CLEAR__SHIFT                                         0x7
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA0_BUSY_INT_CLEAR_MASK                                           0x00000001L
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA1_BUSY_INT_CLEAR_MASK                                           0x00000002L
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA2_BUSY_INT_CLEAR_MASK                                           0x00000004L
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA3_BUSY_INT_CLEAR_MASK                                           0x00000008L
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA4_BUSY_INT_CLEAR_MASK                                           0x00000010L
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA5_BUSY_INT_CLEAR_MASK                                           0x00000020L
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA6_BUSY_INT_CLEAR_MASK                                           0x00000040L
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA7_BUSY_INT_CLEAR_MASK                                           0x00000080L
+//RLC_RLCS_CMP_IDLE_CNTL
+#define RLC_RLCS_CMP_IDLE_CNTL__INT_CLEAR__SHIFT                                                              0x0
+#define RLC_RLCS_CMP_IDLE_CNTL__CMP_IDLE_HYST__SHIFT                                                          0x1
+#define RLC_RLCS_CMP_IDLE_CNTL__CMP_IDLE__SHIFT                                                               0x2
+#define RLC_RLCS_CMP_IDLE_CNTL__MAX_HYSTERESIS__SHIFT                                                         0x3
+#define RLC_RLCS_CMP_IDLE_CNTL__HYSTERESIS_CNT__SHIFT                                                         0xb
+#define RLC_RLCS_CMP_IDLE_CNTL__RESERVED__SHIFT                                                               0x13
+#define RLC_RLCS_CMP_IDLE_CNTL__INT_CLEAR_MASK                                                                0x00000001L
+#define RLC_RLCS_CMP_IDLE_CNTL__CMP_IDLE_HYST_MASK                                                            0x00000002L
+#define RLC_RLCS_CMP_IDLE_CNTL__CMP_IDLE_MASK                                                                 0x00000004L
+#define RLC_RLCS_CMP_IDLE_CNTL__MAX_HYSTERESIS_MASK                                                           0x000007F8L
+#define RLC_RLCS_CMP_IDLE_CNTL__HYSTERESIS_CNT_MASK                                                           0x0007F800L
+#define RLC_RLCS_CMP_IDLE_CNTL__RESERVED_MASK                                                                 0xFFF80000L
+//RLC_RLCS_GENERAL_0
+#define RLC_RLCS_GENERAL_0__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_0__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_1
+#define RLC_RLCS_GENERAL_1__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_1__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_2
+#define RLC_RLCS_GENERAL_2__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_2__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_3
+#define RLC_RLCS_GENERAL_3__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_3__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_4
+#define RLC_RLCS_GENERAL_4__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_4__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_5
+#define RLC_RLCS_GENERAL_5__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_5__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_6
+#define RLC_RLCS_GENERAL_6__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_6__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_7
+#define RLC_RLCS_GENERAL_7__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_7__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_8
+#define RLC_RLCS_GENERAL_8__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_8__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_9
+#define RLC_RLCS_GENERAL_9__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_9__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_10
+#define RLC_RLCS_GENERAL_10__DATA__SHIFT                                                                      0x0
+#define RLC_RLCS_GENERAL_10__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_RLCS_GENERAL_11
+#define RLC_RLCS_GENERAL_11__DATA__SHIFT                                                                      0x0
+#define RLC_RLCS_GENERAL_11__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_RLCS_GENERAL_12
+#define RLC_RLCS_GENERAL_12__DATA__SHIFT                                                                      0x0
+#define RLC_RLCS_GENERAL_12__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_RLCS_GENERAL_13
+#define RLC_RLCS_GENERAL_13__DATA__SHIFT                                                                      0x0
+#define RLC_RLCS_GENERAL_13__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_RLCS_GENERAL_14
+#define RLC_RLCS_GENERAL_14__DATA__SHIFT                                                                      0x0
+#define RLC_RLCS_GENERAL_14__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_RLCS_GENERAL_15
+#define RLC_RLCS_GENERAL_15__DATA__SHIFT                                                                      0x0
+#define RLC_RLCS_GENERAL_15__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_RLCS_GENERAL_16
+#define RLC_RLCS_GENERAL_16__DATA__SHIFT                                                                      0x0
+#define RLC_RLCS_GENERAL_16__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_RLCS_AUXILIARY_REG_1
+#define RLC_RLCS_AUXILIARY_REG_1__ADDR__SHIFT                                                                 0x0
+#define RLC_RLCS_AUXILIARY_REG_1__RESERVED__SHIFT                                                             0x12
+#define RLC_RLCS_AUXILIARY_REG_1__ADDR_MASK                                                                   0x0003FFFFL
+#define RLC_RLCS_AUXILIARY_REG_1__RESERVED_MASK                                                               0xFFFC0000L
+//RLC_RLCS_AUXILIARY_REG_2
+#define RLC_RLCS_AUXILIARY_REG_2__ADDR__SHIFT                                                                 0x0
+#define RLC_RLCS_AUXILIARY_REG_2__RESERVED__SHIFT                                                             0x12
+#define RLC_RLCS_AUXILIARY_REG_2__ADDR_MASK                                                                   0x0003FFFFL
+#define RLC_RLCS_AUXILIARY_REG_2__RESERVED_MASK                                                               0xFFFC0000L
+//RLC_RLCS_AUXILIARY_REG_3
+#define RLC_RLCS_AUXILIARY_REG_3__ADDR__SHIFT                                                                 0x0
+#define RLC_RLCS_AUXILIARY_REG_3__RESERVED__SHIFT                                                             0x12
+#define RLC_RLCS_AUXILIARY_REG_3__ADDR_MASK                                                                   0x0003FFFFL
+#define RLC_RLCS_AUXILIARY_REG_3__RESERVED_MASK                                                               0xFFFC0000L
+//RLC_RLCS_AUXILIARY_REG_4
+#define RLC_RLCS_AUXILIARY_REG_4__ADDR__SHIFT                                                                 0x0
+#define RLC_RLCS_AUXILIARY_REG_4__RESERVED__SHIFT                                                             0x12
+#define RLC_RLCS_AUXILIARY_REG_4__ADDR_MASK                                                                   0x0003FFFFL
+#define RLC_RLCS_AUXILIARY_REG_4__RESERVED_MASK                                                               0xFFFC0000L
+//RLC_RLCS_SPM_SQTT_MODE
+#define RLC_RLCS_SPM_SQTT_MODE__MODE__SHIFT                                                                   0x0
+#define RLC_RLCS_SPM_SQTT_MODE__MODE_MASK                                                                     0x00000001L
+//RLC_RLCS_CP_DMA_SRCID_OVER
+#define RLC_RLCS_CP_DMA_SRCID_OVER__SRCID_OVERRIDE__SHIFT                                                     0x0
+#define RLC_RLCS_CP_DMA_SRCID_OVER__SRCID_OVERRIDE_MASK                                                       0x00000001L
+//RLC_RLCS_BOOTLOAD_ID_STATUS1
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_0_LOADED__SHIFT                                                      0x0
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_1_LOADED__SHIFT                                                      0x1
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_2_LOADED__SHIFT                                                      0x2
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_3_LOADED__SHIFT                                                      0x3
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_4_LOADED__SHIFT                                                      0x4
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_5_LOADED__SHIFT                                                      0x5
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_6_LOADED__SHIFT                                                      0x6
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_7_LOADED__SHIFT                                                      0x7
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_8_LOADED__SHIFT                                                      0x8
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_9_LOADED__SHIFT                                                      0x9
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_10_LOADED__SHIFT                                                     0xa
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_11_LOADED__SHIFT                                                     0xb
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_12_LOADED__SHIFT                                                     0xc
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_13_LOADED__SHIFT                                                     0xd
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_14_LOADED__SHIFT                                                     0xe
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_15_LOADED__SHIFT                                                     0xf
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_16_LOADED__SHIFT                                                     0x10
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_17_LOADED__SHIFT                                                     0x11
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_18_LOADED__SHIFT                                                     0x12
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_19_LOADED__SHIFT                                                     0x13
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_20_LOADED__SHIFT                                                     0x14
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_21_LOADED__SHIFT                                                     0x15
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_22_LOADED__SHIFT                                                     0x16
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_23_LOADED__SHIFT                                                     0x17
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_24_LOADED__SHIFT                                                     0x18
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_25_LOADED__SHIFT                                                     0x19
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_26_LOADED__SHIFT                                                     0x1a
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_27_LOADED__SHIFT                                                     0x1b
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_28_LOADED__SHIFT                                                     0x1c
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_29_LOADED__SHIFT                                                     0x1d
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_30_LOADED__SHIFT                                                     0x1e
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_31_LOADED__SHIFT                                                     0x1f
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_0_LOADED_MASK                                                        0x00000001L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_1_LOADED_MASK                                                        0x00000002L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_2_LOADED_MASK                                                        0x00000004L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_3_LOADED_MASK                                                        0x00000008L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_4_LOADED_MASK                                                        0x00000010L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_5_LOADED_MASK                                                        0x00000020L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_6_LOADED_MASK                                                        0x00000040L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_7_LOADED_MASK                                                        0x00000080L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_8_LOADED_MASK                                                        0x00000100L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_9_LOADED_MASK                                                        0x00000200L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_10_LOADED_MASK                                                       0x00000400L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_11_LOADED_MASK                                                       0x00000800L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_12_LOADED_MASK                                                       0x00001000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_13_LOADED_MASK                                                       0x00002000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_14_LOADED_MASK                                                       0x00004000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_15_LOADED_MASK                                                       0x00008000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_16_LOADED_MASK                                                       0x00010000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_17_LOADED_MASK                                                       0x00020000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_18_LOADED_MASK                                                       0x00040000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_19_LOADED_MASK                                                       0x00080000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_20_LOADED_MASK                                                       0x00100000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_21_LOADED_MASK                                                       0x00200000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_22_LOADED_MASK                                                       0x00400000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_23_LOADED_MASK                                                       0x00800000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_24_LOADED_MASK                                                       0x01000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_25_LOADED_MASK                                                       0x02000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_26_LOADED_MASK                                                       0x04000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_27_LOADED_MASK                                                       0x08000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_28_LOADED_MASK                                                       0x10000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_29_LOADED_MASK                                                       0x20000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_30_LOADED_MASK                                                       0x40000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_31_LOADED_MASK                                                       0x80000000L
+//RLC_RLCS_BOOTLOAD_ID_STATUS2
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_32_LOADED__SHIFT                                                     0x0
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_33_LOADED__SHIFT                                                     0x1
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_34_LOADED__SHIFT                                                     0x2
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_35_LOADED__SHIFT                                                     0x3
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_36_LOADED__SHIFT                                                     0x4
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_37_LOADED__SHIFT                                                     0x5
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_38_LOADED__SHIFT                                                     0x6
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_39_LOADED__SHIFT                                                     0x7
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_40_LOADED__SHIFT                                                     0x8
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_41_LOADED__SHIFT                                                     0x9
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_42_LOADED__SHIFT                                                     0xa
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_43_LOADED__SHIFT                                                     0xb
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_44_LOADED__SHIFT                                                     0xc
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_45_LOADED__SHIFT                                                     0xd
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_46_LOADED__SHIFT                                                     0xe
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_47_LOADED__SHIFT                                                     0xf
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_48_LOADED__SHIFT                                                     0x10
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_49_LOADED__SHIFT                                                     0x11
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_50_LOADED__SHIFT                                                     0x12
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_51_LOADED__SHIFT                                                     0x13
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_52_LOADED__SHIFT                                                     0x14
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_53_LOADED__SHIFT                                                     0x15
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_54_LOADED__SHIFT                                                     0x16
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_55_LOADED__SHIFT                                                     0x17
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_56_LOADED__SHIFT                                                     0x18
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_57_LOADED__SHIFT                                                     0x19
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_58_LOADED__SHIFT                                                     0x1a
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_59_LOADED__SHIFT                                                     0x1b
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_60_LOADED__SHIFT                                                     0x1c
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_61_LOADED__SHIFT                                                     0x1d
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_62_LOADED__SHIFT                                                     0x1e
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_63_LOADED__SHIFT                                                     0x1f
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_32_LOADED_MASK                                                       0x00000001L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_33_LOADED_MASK                                                       0x00000002L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_34_LOADED_MASK                                                       0x00000004L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_35_LOADED_MASK                                                       0x00000008L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_36_LOADED_MASK                                                       0x00000010L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_37_LOADED_MASK                                                       0x00000020L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_38_LOADED_MASK                                                       0x00000040L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_39_LOADED_MASK                                                       0x00000080L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_40_LOADED_MASK                                                       0x00000100L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_41_LOADED_MASK                                                       0x00000200L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_42_LOADED_MASK                                                       0x00000400L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_43_LOADED_MASK                                                       0x00000800L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_44_LOADED_MASK                                                       0x00001000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_45_LOADED_MASK                                                       0x00002000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_46_LOADED_MASK                                                       0x00004000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_47_LOADED_MASK                                                       0x00008000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_48_LOADED_MASK                                                       0x00010000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_49_LOADED_MASK                                                       0x00020000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_50_LOADED_MASK                                                       0x00040000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_51_LOADED_MASK                                                       0x00080000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_52_LOADED_MASK                                                       0x00100000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_53_LOADED_MASK                                                       0x00200000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_54_LOADED_MASK                                                       0x00400000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_55_LOADED_MASK                                                       0x00800000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_56_LOADED_MASK                                                       0x01000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_57_LOADED_MASK                                                       0x02000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_58_LOADED_MASK                                                       0x04000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_59_LOADED_MASK                                                       0x08000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_60_LOADED_MASK                                                       0x10000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_61_LOADED_MASK                                                       0x20000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_62_LOADED_MASK                                                       0x40000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_63_LOADED_MASK                                                       0x80000000L
+//RLC_RLCS_KMD_LOG_CNTL1
+#define RLC_RLCS_KMD_LOG_CNTL1__DATA__SHIFT                                                                   0x0
+#define RLC_RLCS_KMD_LOG_CNTL1__DATA_MASK                                                                     0xFFFFFFFFL
+//RLC_RLCS_KMD_LOG_CNTL2
+#define RLC_RLCS_KMD_LOG_CNTL2__DATA__SHIFT                                                                   0x0
+#define RLC_RLCS_KMD_LOG_CNTL2__DATA_MASK                                                                     0xFFFFFFFFL
+//RLC_RLCS_GPM_LEGACY_INT_STAT
+#define RLC_RLCS_GPM_LEGACY_INT_STAT__RESERVED__SHIFT                                                         0x0
+#define RLC_RLCS_GPM_LEGACY_INT_STAT__RESERVED_MASK                                                           0x00000001L
+//RLC_RLCS_GPM_LEGACY_INT_DISABLE
+#define RLC_RLCS_GPM_LEGACY_INT_DISABLE__RESERVED__SHIFT                                                      0x0
+#define RLC_RLCS_GPM_LEGACY_INT_DISABLE__RESERVED_MASK                                                        0x00000001L
+//RLC_RLCS_GCR_DATA_0
+#define RLC_RLCS_GCR_DATA_0__PHASE_0__SHIFT                                                                   0x0
+#define RLC_RLCS_GCR_DATA_0__PHASE_1__SHIFT                                                                   0x10
+#define RLC_RLCS_GCR_DATA_0__PHASE_0_MASK                                                                     0x0000FFFFL
+#define RLC_RLCS_GCR_DATA_0__PHASE_1_MASK                                                                     0xFFFF0000L
+//RLC_RLCS_GCR_DATA_1
+#define RLC_RLCS_GCR_DATA_1__PHASE_2__SHIFT                                                                   0x0
+#define RLC_RLCS_GCR_DATA_1__PHASE_3__SHIFT                                                                   0x10
+#define RLC_RLCS_GCR_DATA_1__PHASE_2_MASK                                                                     0x0000FFFFL
+#define RLC_RLCS_GCR_DATA_1__PHASE_3_MASK                                                                     0xFFFF0000L
+//RLC_RLCS_GCR_DATA_2
+#define RLC_RLCS_GCR_DATA_2__PHASE_4__SHIFT                                                                   0x0
+#define RLC_RLCS_GCR_DATA_2__PHASE_5__SHIFT                                                                   0x10
+#define RLC_RLCS_GCR_DATA_2__PHASE_4_MASK                                                                     0x0000FFFFL
+#define RLC_RLCS_GCR_DATA_2__PHASE_5_MASK                                                                     0xFFFF0000L
+//RLC_RLCS_GCR_DATA_3
+#define RLC_RLCS_GCR_DATA_3__PHASE_6__SHIFT                                                                   0x0
+#define RLC_RLCS_GCR_DATA_3__PHASE_7__SHIFT                                                                   0x10
+#define RLC_RLCS_GCR_DATA_3__PHASE_6_MASK                                                                     0x0000FFFFL
+#define RLC_RLCS_GCR_DATA_3__PHASE_7_MASK                                                                     0xFFFF0000L
+//RLC_RLCS_GCR_DATA_4
+#define RLC_RLCS_GCR_DATA_4__PHASE_8__SHIFT                                                                   0x0
+#define RLC_RLCS_GCR_DATA_4__PHASE_9__SHIFT                                                                   0x10
+#define RLC_RLCS_GCR_DATA_4__PHASE_8_MASK                                                                     0x0000FFFFL
+#define RLC_RLCS_GCR_DATA_4__PHASE_9_MASK                                                                     0xFFFF0000L
+//RLC_RLCS_GCR_STATUS
+#define RLC_RLCS_GCR_STATUS__GCR_BUSY__SHIFT                                                                  0x0
+#define RLC_RLCS_GCR_STATUS__GCR_OUT_COUNT__SHIFT                                                             0x1
+#define RLC_RLCS_GCR_STATUS__RESERVED_2__SHIFT                                                                0x5
+#define RLC_RLCS_GCR_STATUS__GCRIU_CLI_RSP_TAG__SHIFT                                                         0x8
+#define RLC_RLCS_GCR_STATUS__RESERVED__SHIFT                                                                  0x10
+#define RLC_RLCS_GCR_STATUS__GCR_BUSY_MASK                                                                    0x00000001L
+#define RLC_RLCS_GCR_STATUS__GCR_OUT_COUNT_MASK                                                               0x0000001EL
+#define RLC_RLCS_GCR_STATUS__RESERVED_2_MASK                                                                  0x000000E0L
+#define RLC_RLCS_GCR_STATUS__GCRIU_CLI_RSP_TAG_MASK                                                           0x0000FF00L
+#define RLC_RLCS_GCR_STATUS__RESERVED_MASK                                                                    0xFFFF0000L
+//RLC_RLCS_PERFMON_CLK_CNTL_UCODE
+#define RLC_RLCS_PERFMON_CLK_CNTL_UCODE__PERFMON_CLOCK_STATE__SHIFT                                           0x0
+#define RLC_RLCS_PERFMON_CLK_CNTL_UCODE__PERFMON_CLOCK_STATE_MASK                                             0x00000001L
+//RLC_RLCS_UTCL2_CNTL
+#define RLC_RLCS_UTCL2_CNTL__MTYPE_NO_PTE_MODE__SHIFT                                                         0x0
+#define RLC_RLCS_UTCL2_CNTL__GPA_OVERRIDE__SHIFT                                                              0x1
+#define RLC_RLCS_UTCL2_CNTL__VF_OVERRIDE__SHIFT                                                               0x2
+#define RLC_RLCS_UTCL2_CNTL__GPA_OVERRIDE_VALUE__SHIFT                                                        0x3
+#define RLC_RLCS_UTCL2_CNTL__VF_OVERRIDE_VALUE__SHIFT                                                         0x5
+#define RLC_RLCS_UTCL2_CNTL__IGNORE_PTE_PERMISSION__SHIFT                                                     0x6
+#define RLC_RLCS_UTCL2_CNTL__RESERVED__SHIFT                                                                  0x7
+#define RLC_RLCS_UTCL2_CNTL__MTYPE_NO_PTE_MODE_MASK                                                           0x00000001L
+#define RLC_RLCS_UTCL2_CNTL__GPA_OVERRIDE_MASK                                                                0x00000002L
+#define RLC_RLCS_UTCL2_CNTL__VF_OVERRIDE_MASK                                                                 0x00000004L
+#define RLC_RLCS_UTCL2_CNTL__GPA_OVERRIDE_VALUE_MASK                                                          0x00000018L
+#define RLC_RLCS_UTCL2_CNTL__VF_OVERRIDE_VALUE_MASK                                                           0x00000020L
+#define RLC_RLCS_UTCL2_CNTL__IGNORE_PTE_PERMISSION_MASK                                                       0x00000040L
+#define RLC_RLCS_UTCL2_CNTL__RESERVED_MASK                                                                    0xFFFFFF80L
+//RLC_RLCS_IMU_RLC_MSG_DATA0
+#define RLC_RLCS_IMU_RLC_MSG_DATA0__DATA__SHIFT                                                               0x0
+#define RLC_RLCS_IMU_RLC_MSG_DATA0__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_RLCS_IMU_RLC_MSG_DATA1
+#define RLC_RLCS_IMU_RLC_MSG_DATA1__DATA__SHIFT                                                               0x0
+#define RLC_RLCS_IMU_RLC_MSG_DATA1__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_RLCS_IMU_RLC_MSG_DATA2
+#define RLC_RLCS_IMU_RLC_MSG_DATA2__DATA__SHIFT                                                               0x0
+#define RLC_RLCS_IMU_RLC_MSG_DATA2__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_RLCS_IMU_RLC_MSG_DATA3
+#define RLC_RLCS_IMU_RLC_MSG_DATA3__DATA__SHIFT                                                               0x0
+#define RLC_RLCS_IMU_RLC_MSG_DATA3__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_RLCS_IMU_RLC_MSG_DATA4
+#define RLC_RLCS_IMU_RLC_MSG_DATA4__DATA__SHIFT                                                               0x0
+#define RLC_RLCS_IMU_RLC_MSG_DATA4__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_RLCS_IMU_RLC_MSG_CONTROL
+#define RLC_RLCS_IMU_RLC_MSG_CONTROL__DATA__SHIFT                                                             0x0
+#define RLC_RLCS_IMU_RLC_MSG_CONTROL__DATA_MASK                                                               0xFFFFFFFFL
+//RLC_RLCS_IMU_RLC_MSG_CNTL
+#define RLC_RLCS_IMU_RLC_MSG_CNTL__DONETOG__SHIFT                                                             0x0
+#define RLC_RLCS_IMU_RLC_MSG_CNTL__CHGTOG__SHIFT                                                              0x1
+#define RLC_RLCS_IMU_RLC_MSG_CNTL__RESERVED__SHIFT                                                            0x2
+#define RLC_RLCS_IMU_RLC_MSG_CNTL__DONETOG_MASK                                                               0x00000001L
+#define RLC_RLCS_IMU_RLC_MSG_CNTL__CHGTOG_MASK                                                                0x00000002L
+#define RLC_RLCS_IMU_RLC_MSG_CNTL__RESERVED_MASK                                                              0xFFFFFFFCL
+//RLC_RLCS_RLC_IMU_MSG_DATA0
+#define RLC_RLCS_RLC_IMU_MSG_DATA0__DATA__SHIFT                                                               0x0
+#define RLC_RLCS_RLC_IMU_MSG_DATA0__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_RLCS_RLC_IMU_MSG_CONTROL
+#define RLC_RLCS_RLC_IMU_MSG_CONTROL__DATA__SHIFT                                                             0x0
+#define RLC_RLCS_RLC_IMU_MSG_CONTROL__DATA_MASK                                                               0xFFFFFFFFL
+//RLC_RLCS_RLC_IMU_MSG_CNTL
+#define RLC_RLCS_RLC_IMU_MSG_CNTL__CHGTOG__SHIFT                                                              0x0
+#define RLC_RLCS_RLC_IMU_MSG_CNTL__DONETOG__SHIFT                                                             0x1
+#define RLC_RLCS_RLC_IMU_MSG_CNTL__RESERVED__SHIFT                                                            0x2
+#define RLC_RLCS_RLC_IMU_MSG_CNTL__CHGTOG_MASK                                                                0x00000001L
+#define RLC_RLCS_RLC_IMU_MSG_CNTL__DONETOG_MASK                                                               0x00000002L
+#define RLC_RLCS_RLC_IMU_MSG_CNTL__RESERVED_MASK                                                              0xFFFFFFFCL
+//RLC_RLCS_IMU_RLC_MUTEX_CNTL
+#define RLC_RLCS_IMU_RLC_MUTEX_CNTL__REQ__SHIFT                                                               0x0
+#define RLC_RLCS_IMU_RLC_MUTEX_CNTL__ACQUIRE__SHIFT                                                           0x1
+#define RLC_RLCS_IMU_RLC_MUTEX_CNTL__RESERVED__SHIFT                                                          0x2
+#define RLC_RLCS_IMU_RLC_MUTEX_CNTL__REQ_MASK                                                                 0x00000001L
+#define RLC_RLCS_IMU_RLC_MUTEX_CNTL__ACQUIRE_MASK                                                             0x00000002L
+#define RLC_RLCS_IMU_RLC_MUTEX_CNTL__RESERVED_MASK                                                            0xFFFFFFFCL
+//RLC_RLCS_IMU_RLC_STATUS
+#define RLC_RLCS_IMU_RLC_STATUS__ALLOW_GFXOFF__SHIFT                                                          0x0
+#define RLC_RLCS_IMU_RLC_STATUS__ALLOW_FA_DCS__SHIFT                                                          0x1
+#define RLC_RLCS_IMU_RLC_STATUS__STATUS_14_2__SHIFT                                                           0x2
+#define RLC_RLCS_IMU_RLC_STATUS__DISABLE_GFXCLK_DS__SHIFT                                                     0xf
+#define RLC_RLCS_IMU_RLC_STATUS__RESERVED__SHIFT                                                              0x10
+#define RLC_RLCS_IMU_RLC_STATUS__ALLOW_GFXOFF_MASK                                                            0x00000001L
+#define RLC_RLCS_IMU_RLC_STATUS__ALLOW_FA_DCS_MASK                                                            0x00000002L
+#define RLC_RLCS_IMU_RLC_STATUS__STATUS_14_2_MASK                                                             0x00007FFCL
+#define RLC_RLCS_IMU_RLC_STATUS__DISABLE_GFXCLK_DS_MASK                                                       0x00008000L
+#define RLC_RLCS_IMU_RLC_STATUS__RESERVED_MASK                                                                0xFFFF0000L
+//RLC_RLCS_RLC_IMU_STATUS
+#define RLC_RLCS_RLC_IMU_STATUS__PWR_DOWN_ACTIVE__SHIFT                                                       0x0
+#define RLC_RLCS_RLC_IMU_STATUS__RLC_ALIVE__SHIFT                                                             0x1
+#define RLC_RLCS_RLC_IMU_STATUS__STATUS_3_2__SHIFT                                                            0x2
+#define RLC_RLCS_RLC_IMU_STATUS__RESERVED__SHIFT                                                              0x4
+#define RLC_RLCS_RLC_IMU_STATUS__PWR_DOWN_ACTIVE_MASK                                                         0x00000001L
+#define RLC_RLCS_RLC_IMU_STATUS__RLC_ALIVE_MASK                                                               0x00000002L
+#define RLC_RLCS_RLC_IMU_STATUS__STATUS_3_2_MASK                                                              0x0000000CL
+#define RLC_RLCS_RLC_IMU_STATUS__RESERVED_MASK                                                                0xFFFFFFF0L
+//RLC_RLCS_IMU_RAM_DATA_1
+#define RLC_RLCS_IMU_RAM_DATA_1__DATA__SHIFT                                                                  0x0
+#define RLC_RLCS_IMU_RAM_DATA_1__DATA_MASK                                                                    0xFFFFFFFFL
+//RLC_RLCS_IMU_RAM_ADDR_1_LSB
+#define RLC_RLCS_IMU_RAM_ADDR_1_LSB__DATA__SHIFT                                                              0x0
+#define RLC_RLCS_IMU_RAM_ADDR_1_LSB__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCS_IMU_RAM_ADDR_1_MSB
+#define RLC_RLCS_IMU_RAM_ADDR_1_MSB__DATA__SHIFT                                                              0x0
+#define RLC_RLCS_IMU_RAM_ADDR_1_MSB__RESERVED__SHIFT                                                          0x10
+#define RLC_RLCS_IMU_RAM_ADDR_1_MSB__DATA_MASK                                                                0x0000FFFFL
+#define RLC_RLCS_IMU_RAM_ADDR_1_MSB__RESERVED_MASK                                                            0xFFFF0000L
+//RLC_RLCS_IMU_RAM_DATA_0
+#define RLC_RLCS_IMU_RAM_DATA_0__DATA__SHIFT                                                                  0x0
+#define RLC_RLCS_IMU_RAM_DATA_0__DATA_MASK                                                                    0xFFFFFFFFL
+//RLC_RLCS_IMU_RAM_ADDR_0_LSB
+#define RLC_RLCS_IMU_RAM_ADDR_0_LSB__DATA__SHIFT                                                              0x0
+#define RLC_RLCS_IMU_RAM_ADDR_0_LSB__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCS_IMU_RAM_ADDR_0_MSB
+#define RLC_RLCS_IMU_RAM_ADDR_0_MSB__DATA__SHIFT                                                              0x0
+#define RLC_RLCS_IMU_RAM_ADDR_0_MSB__RESERVED__SHIFT                                                          0x10
+#define RLC_RLCS_IMU_RAM_ADDR_0_MSB__DATA_MASK                                                                0x0000FFFFL
+#define RLC_RLCS_IMU_RAM_ADDR_0_MSB__RESERVED_MASK                                                            0xFFFF0000L
+//RLC_RLCS_IMU_RAM_CNTL
+#define RLC_RLCS_IMU_RAM_CNTL__REQTOG__SHIFT                                                                  0x0
+#define RLC_RLCS_IMU_RAM_CNTL__ACKTOG__SHIFT                                                                  0x1
+#define RLC_RLCS_IMU_RAM_CNTL__RESERVED__SHIFT                                                                0x2
+#define RLC_RLCS_IMU_RAM_CNTL__REQTOG_MASK                                                                    0x00000001L
+#define RLC_RLCS_IMU_RAM_CNTL__ACKTOG_MASK                                                                    0x00000002L
+#define RLC_RLCS_IMU_RAM_CNTL__RESERVED_MASK                                                                  0xFFFFFFFCL
+//RLC_RLCS_IMU_GFX_DOORBELL_FENCE
+#define RLC_RLCS_IMU_GFX_DOORBELL_FENCE__ENABLE__SHIFT                                                        0x0
+#define RLC_RLCS_IMU_GFX_DOORBELL_FENCE__ACK__SHIFT                                                           0x1
+#define RLC_RLCS_IMU_GFX_DOORBELL_FENCE__RESERVED__SHIFT                                                      0x2
+#define RLC_RLCS_IMU_GFX_DOORBELL_FENCE__ENABLE_MASK                                                          0x00000001L
+#define RLC_RLCS_IMU_GFX_DOORBELL_FENCE__ACK_MASK                                                             0x00000002L
+#define RLC_RLCS_IMU_GFX_DOORBELL_FENCE__RESERVED_MASK                                                        0xFFFFFFFCL
+//RLC_RLCS_SDMA_INT_CNTL_1
+#define RLC_RLCS_SDMA_INT_CNTL_1__INTERRUPT_ACK__SHIFT                                                        0x0
+#define RLC_RLCS_SDMA_INT_CNTL_1__RESP_ID__SHIFT                                                              0x1
+#define RLC_RLCS_SDMA_INT_CNTL_1__RESERVED__SHIFT                                                             0x2
+#define RLC_RLCS_SDMA_INT_CNTL_1__INTERRUPT_ACK_MASK                                                          0x00000001L
+#define RLC_RLCS_SDMA_INT_CNTL_1__RESP_ID_MASK                                                                0x00000002L
+#define RLC_RLCS_SDMA_INT_CNTL_1__RESERVED_MASK                                                               0xFFFFFFFCL
+//RLC_RLCS_SDMA_INT_CNTL_2
+#define RLC_RLCS_SDMA_INT_CNTL_2__AUTO_ACK_EN__SHIFT                                                          0x0
+#define RLC_RLCS_SDMA_INT_CNTL_2__AUTO_ACK_ACTIVE__SHIFT                                                      0x1
+#define RLC_RLCS_SDMA_INT_CNTL_2__RESERVED__SHIFT                                                             0x2
+#define RLC_RLCS_SDMA_INT_CNTL_2__AUTO_ACK_EN_MASK                                                            0x00000001L
+#define RLC_RLCS_SDMA_INT_CNTL_2__AUTO_ACK_ACTIVE_MASK                                                        0x00000002L
+#define RLC_RLCS_SDMA_INT_CNTL_2__RESERVED_MASK                                                               0xFFFFFFFCL
+//RLC_RLCS_SDMA_INT_STAT
+#define RLC_RLCS_SDMA_INT_STAT__REQ_IDLE_HIST__SHIFT                                                          0x0
+#define RLC_RLCS_SDMA_INT_STAT__REQ_BUSY_HIST__SHIFT                                                          0x8
+#define RLC_RLCS_SDMA_INT_STAT__LAST_SDMA_RLC_INT_ID__SHIFT                                                   0x10
+#define RLC_RLCS_SDMA_INT_STAT__SDMA_RLC_INT_PENDING__SHIFT                                                   0x11
+#define RLC_RLCS_SDMA_INT_STAT__RESERVED__SHIFT                                                               0x12
+#define RLC_RLCS_SDMA_INT_STAT__REQ_IDLE_HIST_MASK                                                            0x000000FFL
+#define RLC_RLCS_SDMA_INT_STAT__REQ_BUSY_HIST_MASK                                                            0x0000FF00L
+#define RLC_RLCS_SDMA_INT_STAT__LAST_SDMA_RLC_INT_ID_MASK                                                     0x00010000L
+#define RLC_RLCS_SDMA_INT_STAT__SDMA_RLC_INT_PENDING_MASK                                                     0x00020000L
+#define RLC_RLCS_SDMA_INT_STAT__RESERVED_MASK                                                                 0xFFFC0000L
+//RLC_RLCS_SDMA_INT_INFO
+#define RLC_RLCS_SDMA_INT_INFO__REQ_IDLE_TO_FW__SHIFT                                                         0x0
+#define RLC_RLCS_SDMA_INT_INFO__REQ_BUSY_TO_FW__SHIFT                                                         0x8
+#define RLC_RLCS_SDMA_INT_INFO__INTERRUPT_ID__SHIFT                                                           0x10
+#define RLC_RLCS_SDMA_INT_INFO__RESERVED__SHIFT                                                               0x11
+#define RLC_RLCS_SDMA_INT_INFO__REQ_IDLE_TO_FW_MASK                                                           0x000000FFL
+#define RLC_RLCS_SDMA_INT_INFO__REQ_BUSY_TO_FW_MASK                                                           0x0000FF00L
+#define RLC_RLCS_SDMA_INT_INFO__INTERRUPT_ID_MASK                                                             0x00010000L
+#define RLC_RLCS_SDMA_INT_INFO__RESERVED_MASK                                                                 0xFFFE0000L
+//RLC_RLCS_GFX_MEM_POWER_CTRL_0
+#define RLC_RLCS_GFX_MEM_POWER_CTRL_0__DATA__SHIFT                                                            0x0
+#define RLC_RLCS_GFX_MEM_POWER_CTRL_0__DATA_MASK                                                              0xFFFFFFFFL
+//RLC_RLCS_GFX_MEM_POWER_CTRL_1
+#define RLC_RLCS_GFX_MEM_POWER_CTRL_1__DATA__SHIFT                                                            0x0
+#define RLC_RLCS_GFX_MEM_POWER_CTRL_1__DATA_MASK                                                              0xFFFFFFFFL
+//RLC_RLCS_GFX_MEM_POWER_CTRL_2
+#define RLC_RLCS_GFX_MEM_POWER_CTRL_2__DATA__SHIFT                                                            0x0
+#define RLC_RLCS_GFX_MEM_POWER_CTRL_2__DATA_MASK                                                              0xFFFFFFFFL
+//RLC_RLCS_IH_CTRL_1
+#define RLC_RLCS_IH_CTRL_1__IH_CONTEXT_ID_1__SHIFT                                                            0x0
+#define RLC_RLCS_IH_CTRL_1__IH_CONTEXT_ID_1_MASK                                                              0xFFFFFFFFL
+//RLC_RLCS_IH_CTRL_2
+#define RLC_RLCS_IH_CTRL_2__IH_CONTEXT_ID_2__SHIFT                                                            0x0
+#define RLC_RLCS_IH_CTRL_2__IH_RING_ID__SHIFT                                                                 0x8
+#define RLC_RLCS_IH_CTRL_2__IH_VM_ID__SHIFT                                                                   0x10
+#define RLC_RLCS_IH_CTRL_2__RESERVED__SHIFT                                                                   0x14
+#define RLC_RLCS_IH_CTRL_2__IH_CONTEXT_ID_2_MASK                                                              0x000000FFL
+#define RLC_RLCS_IH_CTRL_2__IH_RING_ID_MASK                                                                   0x0000FF00L
+#define RLC_RLCS_IH_CTRL_2__IH_VM_ID_MASK                                                                     0x000F0000L
+#define RLC_RLCS_IH_CTRL_2__RESERVED_MASK                                                                     0xFFF00000L
+//RLC_RLCS_IH_CTRL_3
+#define RLC_RLCS_IH_CTRL_3__IH_SOURCE_ID__SHIFT                                                               0x0
+#define RLC_RLCS_IH_CTRL_3__IH_VF_ID__SHIFT                                                                   0x8
+#define RLC_RLCS_IH_CTRL_3__IH_VF__SHIFT                                                                      0xd
+#define RLC_RLCS_IH_CTRL_3__RESERVED__SHIFT                                                                   0xe
+#define RLC_RLCS_IH_CTRL_3__IH_SOURCE_ID_MASK                                                                 0x000000FFL
+#define RLC_RLCS_IH_CTRL_3__IH_VF_ID_MASK                                                                     0x00001F00L
+#define RLC_RLCS_IH_CTRL_3__IH_VF_MASK                                                                        0x00002000L
+#define RLC_RLCS_IH_CTRL_3__RESERVED_MASK                                                                     0xFFFFC000L
+//RLC_RLCS_IH_STATUS
+#define RLC_RLCS_IH_STATUS__IH_CREDIT_COUNT__SHIFT                                                            0x0
+#define RLC_RLCS_IH_STATUS__IH_BUSY__SHIFT                                                                    0x6
+#define RLC_RLCS_IH_STATUS__IH_WRITE_DONE__SHIFT                                                              0x7
+#define RLC_RLCS_IH_STATUS__RESERVED__SHIFT                                                                   0x8
+#define RLC_RLCS_IH_STATUS__IH_CREDIT_COUNT_MASK                                                              0x0000003FL
+#define RLC_RLCS_IH_STATUS__IH_BUSY_MASK                                                                      0x00000040L
+#define RLC_RLCS_IH_STATUS__IH_WRITE_DONE_MASK                                                                0x00000080L
+#define RLC_RLCS_IH_STATUS__RESERVED_MASK                                                                     0xFFFFFF00L
+//RLC_RLCS_FED_STATUS
+#define RLC_RLCS_FED_STATUS__RLC_FED_ERR__SHIFT                                                               0x0
+#define RLC_RLCS_FED_STATUS__UTCL2_FED_ERR__SHIFT                                                             0x1
+#define RLC_RLCS_FED_STATUS__GE_FED_ERR__SHIFT                                                                0x2
+#define RLC_RLCS_FED_STATUS__CPC_FED_ERR__SHIFT                                                               0x3
+#define RLC_RLCS_FED_STATUS__CPF_FED_ERR__SHIFT                                                               0x4
+#define RLC_RLCS_FED_STATUS__CPG_FED_ERR__SHIFT                                                               0x5
+#define RLC_RLCS_FED_STATUS__SDMA0_FED_ERR__SHIFT                                                             0x6
+#define RLC_RLCS_FED_STATUS__SDMA1_FED_ERR__SHIFT                                                             0x7
+#define RLC_RLCS_FED_STATUS__RESERVED_9_8__SHIFT                                                              0x8
+#define RLC_RLCS_FED_STATUS__SE0_FED_ERR__SHIFT                                                               0xa
+#define RLC_RLCS_FED_STATUS__SE1_FED_ERR__SHIFT                                                               0xb
+#define RLC_RLCS_FED_STATUS__CANE_FED_ERR__SHIFT                                                              0xc
+#define RLC_RLCS_FED_STATUS__RESERVED_17_13__SHIFT                                                            0xd
+#define RLC_RLCS_FED_STATUS__AID0_FED_ERR__SHIFT                                                              0x12
+#define RLC_RLCS_FED_STATUS__AID1_FED_ERR__SHIFT                                                              0x13
+#define RLC_RLCS_FED_STATUS__RLC_FED_ERR_MASK                                                                 0x00000001L
+#define RLC_RLCS_FED_STATUS__UTCL2_FED_ERR_MASK                                                               0x00000002L
+#define RLC_RLCS_FED_STATUS__GE_FED_ERR_MASK                                                                  0x00000004L
+#define RLC_RLCS_FED_STATUS__CPC_FED_ERR_MASK                                                                 0x00000008L
+#define RLC_RLCS_FED_STATUS__CPF_FED_ERR_MASK                                                                 0x00000010L
+#define RLC_RLCS_FED_STATUS__CPG_FED_ERR_MASK                                                                 0x00000020L
+#define RLC_RLCS_FED_STATUS__SDMA0_FED_ERR_MASK                                                               0x00000040L
+#define RLC_RLCS_FED_STATUS__SDMA1_FED_ERR_MASK                                                               0x00000080L
+#define RLC_RLCS_FED_STATUS__RESERVED_9_8_MASK                                                                0x00000300L
+#define RLC_RLCS_FED_STATUS__SE0_FED_ERR_MASK                                                                 0x00000400L
+#define RLC_RLCS_FED_STATUS__SE1_FED_ERR_MASK                                                                 0x00000800L
+#define RLC_RLCS_FED_STATUS__CANE_FED_ERR_MASK                                                                0x00001000L
+#define RLC_RLCS_FED_STATUS__RESERVED_17_13_MASK                                                              0x0003E000L
+#define RLC_RLCS_FED_STATUS__AID0_FED_ERR_MASK                                                                0x00040000L
+#define RLC_RLCS_FED_STATUS__AID1_FED_ERR_MASK                                                                0x00080000L
+//RLC_RLCS_FED_RESP
+#define RLC_RLCS_FED_RESP__RLC_FED_ACK__SHIFT                                                                 0x0
+#define RLC_RLCS_FED_RESP__UTCL2_FED_ACK__SHIFT                                                               0x1
+#define RLC_RLCS_FED_RESP__GE_FED_ACK__SHIFT                                                                  0x2
+#define RLC_RLCS_FED_RESP__CPC_FED_ACK__SHIFT                                                                 0x3
+#define RLC_RLCS_FED_RESP__CPF_FED_ACK__SHIFT                                                                 0x4
+#define RLC_RLCS_FED_RESP__CPG_FED_ACK__SHIFT                                                                 0x5
+#define RLC_RLCS_FED_RESP__SDMA0_FED_ACK__SHIFT                                                               0x6
+#define RLC_RLCS_FED_RESP__SDMA1_FED_ACK__SHIFT                                                               0x7
+#define RLC_RLCS_FED_RESP__CANE_FED_ACK__SHIFT                                                                0xc
+#define RLC_RLCS_FED_RESP__SE0_FED_ACK__SHIFT                                                                 0xd
+#define RLC_RLCS_FED_RESP__SE1_FED_ACK__SHIFT                                                                 0xe
+#define RLC_RLCS_FED_RESP__RLC_FED_ACK_MASK                                                                   0x00000001L
+#define RLC_RLCS_FED_RESP__UTCL2_FED_ACK_MASK                                                                 0x00000002L
+#define RLC_RLCS_FED_RESP__GE_FED_ACK_MASK                                                                    0x00000004L
+#define RLC_RLCS_FED_RESP__CPC_FED_ACK_MASK                                                                   0x00000008L
+#define RLC_RLCS_FED_RESP__CPF_FED_ACK_MASK                                                                   0x00000010L
+#define RLC_RLCS_FED_RESP__CPG_FED_ACK_MASK                                                                   0x00000020L
+#define RLC_RLCS_FED_RESP__SDMA0_FED_ACK_MASK                                                                 0x00000040L
+#define RLC_RLCS_FED_RESP__SDMA1_FED_ACK_MASK                                                                 0x00000080L
+#define RLC_RLCS_FED_RESP__CANE_FED_ACK_MASK                                                                  0x00001000L
+#define RLC_RLCS_FED_RESP__SE0_FED_ACK_MASK                                                                   0x00002000L
+#define RLC_RLCS_FED_RESP__SE1_FED_ACK_MASK                                                                   0x00004000L
+//RLC_RLCS_FED_INT_MASK
+#define RLC_RLCS_FED_INT_MASK__RLC__SHIFT                                                                     0x0
+#define RLC_RLCS_FED_INT_MASK__UTCL2__SHIFT                                                                   0x1
+#define RLC_RLCS_FED_INT_MASK__GE__SHIFT                                                                      0x2
+#define RLC_RLCS_FED_INT_MASK__CPC__SHIFT                                                                     0x3
+#define RLC_RLCS_FED_INT_MASK__CPF__SHIFT                                                                     0x4
+#define RLC_RLCS_FED_INT_MASK__CPG__SHIFT                                                                     0x5
+#define RLC_RLCS_FED_INT_MASK__SDMA0__SHIFT                                                                   0x6
+#define RLC_RLCS_FED_INT_MASK__SDMA1__SHIFT                                                                   0x7
+#define RLC_RLCS_FED_INT_MASK__RESERVED_9_8__SHIFT                                                            0x8
+#define RLC_RLCS_FED_INT_MASK__SE0__SHIFT                                                                     0xa
+#define RLC_RLCS_FED_INT_MASK__SE1__SHIFT                                                                     0xb
+#define RLC_RLCS_FED_INT_MASK__CANE__SHIFT                                                                    0xc
+#define RLC_RLCS_FED_INT_MASK__RESERVED_17_13__SHIFT                                                          0xd
+#define RLC_RLCS_FED_INT_MASK__AID0__SHIFT                                                                    0x12
+#define RLC_RLCS_FED_INT_MASK__AID1__SHIFT                                                                    0x13
+#define RLC_RLCS_FED_INT_MASK__RLC_MASK                                                                       0x00000001L
+#define RLC_RLCS_FED_INT_MASK__UTCL2_MASK                                                                     0x00000002L
+#define RLC_RLCS_FED_INT_MASK__GE_MASK                                                                        0x00000004L
+#define RLC_RLCS_FED_INT_MASK__CPC_MASK                                                                       0x00000008L
+#define RLC_RLCS_FED_INT_MASK__CPF_MASK                                                                       0x00000010L
+#define RLC_RLCS_FED_INT_MASK__CPG_MASK                                                                       0x00000020L
+#define RLC_RLCS_FED_INT_MASK__SDMA0_MASK                                                                     0x00000040L
+#define RLC_RLCS_FED_INT_MASK__SDMA1_MASK                                                                     0x00000080L
+#define RLC_RLCS_FED_INT_MASK__RESERVED_9_8_MASK                                                              0x00000300L
+#define RLC_RLCS_FED_INT_MASK__SE0_MASK                                                                       0x00000400L
+#define RLC_RLCS_FED_INT_MASK__SE1_MASK                                                                       0x00000800L
+#define RLC_RLCS_FED_INT_MASK__CANE_MASK                                                                      0x00001000L
+#define RLC_RLCS_FED_INT_MASK__RESERVED_17_13_MASK                                                            0x0003E000L
+#define RLC_RLCS_FED_INT_MASK__AID0_MASK                                                                      0x00040000L
+#define RLC_RLCS_FED_INT_MASK__AID1_MASK                                                                      0x00080000L
+//RLC_RLCS_MEM_FED
+#define RLC_RLCS_MEM_FED__RESERVED_3_0__SHIFT                                                                 0x0
+#define RLC_RLCS_MEM_FED__LX6_RD_ERR__SHIFT                                                                   0x4
+#define RLC_RLCS_MEM_FED__TH2_RD_ERR__SHIFT                                                                   0x5
+#define RLC_RLCS_MEM_FED__TH1_RD_ERR__SHIFT                                                                   0x6
+#define RLC_RLCS_MEM_FED__TH0_RD_ERR__SHIFT                                                                   0x7
+#define RLC_RLCS_MEM_FED__DMA_RD_ERR__SHIFT                                                                   0x8
+#define RLC_RLCS_MEM_FED__SRM_RD_ERR__SHIFT                                                                   0x9
+#define RLC_RLCS_MEM_FED__RESERVED_11_10__SHIFT                                                               0xa
+#define RLC_RLCS_MEM_FED__LX6_TR_ERR__SHIFT                                                                   0xc
+#define RLC_RLCS_MEM_FED__TH2_TR_ERR__SHIFT                                                                   0xd
+#define RLC_RLCS_MEM_FED__TH1_TR_ERR__SHIFT                                                                   0xe
+#define RLC_RLCS_MEM_FED__TH0_TR_ERR__SHIFT                                                                   0xf
+#define RLC_RLCS_MEM_FED__SPM_TR_ERR__SHIFT                                                                   0x10
+#define RLC_RLCS_MEM_FED__DMA_TR_ERR__SHIFT                                                                   0x11
+#define RLC_RLCS_MEM_FED__DLG_TR_ERR__SHIFT                                                                   0x12
+#define RLC_RLCS_MEM_FED__SRM_TR_ERR__SHIFT                                                                   0x13
+#define RLC_RLCS_MEM_FED__RESERVED_3_0_MASK                                                                   0x0000000FL
+#define RLC_RLCS_MEM_FED__LX6_RD_ERR_MASK                                                                     0x00000010L
+#define RLC_RLCS_MEM_FED__TH2_RD_ERR_MASK                                                                     0x00000020L
+#define RLC_RLCS_MEM_FED__TH1_RD_ERR_MASK                                                                     0x00000040L
+#define RLC_RLCS_MEM_FED__TH0_RD_ERR_MASK                                                                     0x00000080L
+#define RLC_RLCS_MEM_FED__DMA_RD_ERR_MASK                                                                     0x00000100L
+#define RLC_RLCS_MEM_FED__SRM_RD_ERR_MASK                                                                     0x00000200L
+#define RLC_RLCS_MEM_FED__RESERVED_11_10_MASK                                                                 0x00000C00L
+#define RLC_RLCS_MEM_FED__LX6_TR_ERR_MASK                                                                     0x00001000L
+#define RLC_RLCS_MEM_FED__TH2_TR_ERR_MASK                                                                     0x00002000L
+#define RLC_RLCS_MEM_FED__TH1_TR_ERR_MASK                                                                     0x00004000L
+#define RLC_RLCS_MEM_FED__TH0_TR_ERR_MASK                                                                     0x00008000L
+#define RLC_RLCS_MEM_FED__SPM_TR_ERR_MASK                                                                     0x00010000L
+#define RLC_RLCS_MEM_FED__DMA_TR_ERR_MASK                                                                     0x00020000L
+#define RLC_RLCS_MEM_FED__DLG_TR_ERR_MASK                                                                     0x00040000L
+#define RLC_RLCS_MEM_FED__SRM_TR_ERR_MASK                                                                     0x00080000L
+//RLC_RLCS_SE_PWR_CTRL
+#define RLC_RLCS_SE_PWR_CTRL__SE_GFXCLK_CLKEN__SHIFT                                                          0x0
+#define RLC_RLCS_SE_PWR_CTRL__SE_GFX_HARD_RESETB__SHIFT                                                       0x8
+#define RLC_RLCS_SE_PWR_CTRL__SE_EA_HARD_RESETB__SHIFT                                                        0x10
+#define RLC_RLCS_SE_PWR_CTRL__SE_GFXCLK_CLKEN_MASK                                                            0x00000003L
+#define RLC_RLCS_SE_PWR_CTRL__SE_GFX_HARD_RESETB_MASK                                                         0x00000300L
+#define RLC_RLCS_SE_PWR_CTRL__SE_EA_HARD_RESETB_MASK                                                          0x00030000L
+//RLC_UTCL2_FED_STATUS_SNAP
+#define RLC_UTCL2_FED_STATUS_SNAP__UTCL2_FED_STATUS_SNAP__SHIFT                                               0x0
+#define RLC_UTCL2_FED_STATUS_SNAP__UTCL2_FED_STATUS_SNAP_MASK                                                 0xFFFFFFFFL
+//RLC_SE0_FED_STATUS_SNAP
+#define RLC_SE0_FED_STATUS_SNAP__SE0_FED_STATUS_SNAP__SHIFT                                                   0x0
+#define RLC_SE0_FED_STATUS_SNAP__SE0_FED_STATUS_SNAP_MASK                                                     0xFFFFFFFFL
+//RLC_SE1_FED_STATUS_SNAP
+#define RLC_SE1_FED_STATUS_SNAP__SE1_FED_STATUS_SNAP__SHIFT                                                   0x0
+#define RLC_SE1_FED_STATUS_SNAP__SE1_FED_STATUS_SNAP_MASK                                                     0xFFFFFFFFL
+//RLC_CANE_FED_STATUS_SNAP
+#define RLC_CANE_FED_STATUS_SNAP__CANE_FED_STATUS_SNAP__SHIFT                                                 0x0
+#define RLC_CANE_FED_STATUS_SNAP__CANE_FED_STATUS_SNAP_MASK                                                   0xFFFFFFFFL
+//RLC_CONSOLIDATED_FED_STS
+#define RLC_CONSOLIDATED_FED_STS__RLC_CONSOLIDATED_FED_STS__SHIFT                                             0x0
+#define RLC_CONSOLIDATED_FED_STS__NEED_TO_APPLY_FLR__SHIFT                                                    0x10
+#define RLC_CONSOLIDATED_FED_STS__RESERVED__SHIFT                                                             0x11
+#define RLC_CONSOLIDATED_FED_STS__RLC_CONSOLIDATED_FED_STS_MASK                                               0x0000FFFFL
+#define RLC_CONSOLIDATED_FED_STS__NEED_TO_APPLY_FLR_MASK                                                      0x00010000L
+#define RLC_CONSOLIDATED_FED_STS__RESERVED_MASK                                                               0xFFFE0000L
+//RLC_HOST_FED_ACK
+#define RLC_HOST_FED_ACK__RLC_HOST_FED_ACK__SHIFT                                                             0x0
+#define RLC_HOST_FED_ACK__RESERVED__SHIFT                                                                     0x10
+#define RLC_HOST_FED_ACK__RLC_HOST_FED_ACK_MASK                                                               0x0000FFFFL
+#define RLC_HOST_FED_ACK__RESERVED_MASK                                                                       0xFFFF0000L
+//RLC_RLCS_SB_RLC_ROM_STATUS
+#define RLC_RLCS_SB_RLC_ROM_STATUS__STATUS__SHIFT                                                             0x0
+#define RLC_RLCS_SB_RLC_ROM_STATUS__STATUS_MASK                                                               0x00000003L
+//RLC_RLCS_SB_ROM_STATUS
+#define RLC_RLCS_SB_ROM_STATUS__RESERVED__SHIFT                                                               0x1f
+#define RLC_RLCS_SB_ROM_STATUS__RESERVED_MASK                                                                 0x80000000L
+//RLC_RLCS_UTCL2_BUSY_CNTL
+#define RLC_RLCS_UTCL2_BUSY_CNTL__AUTO_HDSHK__SHIFT                                                           0x0
+#define RLC_RLCS_UTCL2_BUSY_CNTL__ACK__SHIFT                                                                  0x1
+#define RLC_RLCS_UTCL2_BUSY_CNTL__SPARE__SHIFT                                                                0x2
+#define RLC_RLCS_UTCL2_BUSY_CNTL__HW_CHK_DIS__SHIFT                                                           0x4
+#define RLC_RLCS_UTCL2_BUSY_CNTL__AUTO_HDSHK_MASK                                                             0x00000001L
+#define RLC_RLCS_UTCL2_BUSY_CNTL__ACK_MASK                                                                    0x00000002L
+#define RLC_RLCS_UTCL2_BUSY_CNTL__SPARE_MASK                                                                  0x0000000CL
+#define RLC_RLCS_UTCL2_BUSY_CNTL__HW_CHK_DIS_MASK                                                             0x00000070L
+//RLC_RLCS_UTCL2_BUSY_STAT
+#define RLC_RLCS_UTCL2_BUSY_STAT__REQ_STATUS__SHIFT                                                           0x0
+#define RLC_RLCS_UTCL2_BUSY_STAT__ACK_STATUS__SHIFT                                                           0x1
+#define RLC_RLCS_UTCL2_BUSY_STAT__REQ_STATUS_MASK                                                             0x00000001L
+#define RLC_RLCS_UTCL2_BUSY_STAT__ACK_STATUS_MASK                                                             0x00000002L
+//RLC_RLCS_DEC_END
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pfvfdec_rlc
+//RLC_SAFE_MODE
+#define RLC_SAFE_MODE__CMD__SHIFT                                                                             0x0
+#define RLC_SAFE_MODE__MESSAGE__SHIFT                                                                         0x1
+#define RLC_SAFE_MODE__RESERVED1__SHIFT                                                                       0x5
+#define RLC_SAFE_MODE__RESPONSE__SHIFT                                                                        0x8
+#define RLC_SAFE_MODE__RESERVED__SHIFT                                                                        0xc
+#define RLC_SAFE_MODE__CMD_MASK                                                                               0x00000001L
+#define RLC_SAFE_MODE__MESSAGE_MASK                                                                           0x0000001EL
+#define RLC_SAFE_MODE__RESERVED1_MASK                                                                         0x000000E0L
+#define RLC_SAFE_MODE__RESPONSE_MASK                                                                          0x00000F00L
+#define RLC_SAFE_MODE__RESERVED_MASK                                                                          0xFFFFF000L
+//RLC_SPM_SAMPLE_CNT
+#define RLC_SPM_SAMPLE_CNT__COUNT__SHIFT                                                                      0x0
+#define RLC_SPM_SAMPLE_CNT__COUNT_MASK                                                                        0xFFFFFFFFL
+//RLC_SPM_MC_CNTL
+#define RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT                                                                  0x0
+#define RLC_SPM_MC_CNTL__RLC_SPM_SDR__SHIFT                                                                   0x4
+#define RLC_SPM_MC_CNTL__RLC_SPM_PERF_CNTR__SHIFT                                                             0x6
+#define RLC_SPM_MC_CNTL__RLC_SPM_FED__SHIFT                                                                   0x7
+#define RLC_SPM_MC_CNTL__RLC_SPM_TEMPORAL__SHIFT                                                              0x8
+#define RLC_SPM_MC_CNTL__RLC_SPM_COMP__SHIFT                                                                  0xb
+#define RLC_SPM_MC_CNTL__RLC_SPM_COMP_OVER__SHIFT                                                             0xd
+#define RLC_SPM_MC_CNTL__RLC_SPM_RO__SHIFT                                                                    0xe
+#define RLC_SPM_MC_CNTL__RLC_SPM_NOFILL__SHIFT                                                                0xf
+#define RLC_SPM_MC_CNTL__RLC_SPM_MTYPE_OVER__SHIFT                                                            0x10
+#define RLC_SPM_MC_CNTL__RLC_SPM_MTYPE__SHIFT                                                                 0x11
+#define RLC_SPM_MC_CNTL__RESERVED__SHIFT                                                                      0x13
+#define RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK                                                                    0x0000000FL
+#define RLC_SPM_MC_CNTL__RLC_SPM_SDR_MASK                                                                     0x00000030L
+#define RLC_SPM_MC_CNTL__RLC_SPM_PERF_CNTR_MASK                                                               0x00000040L
+#define RLC_SPM_MC_CNTL__RLC_SPM_FED_MASK                                                                     0x00000080L
+#define RLC_SPM_MC_CNTL__RLC_SPM_TEMPORAL_MASK                                                                0x00000700L
+#define RLC_SPM_MC_CNTL__RLC_SPM_COMP_MASK                                                                    0x00001800L
+#define RLC_SPM_MC_CNTL__RLC_SPM_COMP_OVER_MASK                                                               0x00002000L
+#define RLC_SPM_MC_CNTL__RLC_SPM_RO_MASK                                                                      0x00004000L
+#define RLC_SPM_MC_CNTL__RLC_SPM_NOFILL_MASK                                                                  0x00008000L
+#define RLC_SPM_MC_CNTL__RLC_SPM_MTYPE_OVER_MASK                                                              0x00010000L
+#define RLC_SPM_MC_CNTL__RLC_SPM_MTYPE_MASK                                                                   0x00060000L
+#define RLC_SPM_MC_CNTL__RESERVED_MASK                                                                        0xFFF80000L
+//RLC_SPM_INT_CNTL
+#define RLC_SPM_INT_CNTL__RLC_SPM_INT_CNTL__SHIFT                                                             0x0
+#define RLC_SPM_INT_CNTL__RESERVED__SHIFT                                                                     0x1
+#define RLC_SPM_INT_CNTL__RLC_SPM_INT_CNTL_MASK                                                               0x00000001L
+#define RLC_SPM_INT_CNTL__RESERVED_MASK                                                                       0xFFFFFFFEL
+//RLC_SPM_INT_STATUS
+#define RLC_SPM_INT_STATUS__RLC_SPM_INT_STATUS__SHIFT                                                         0x0
+#define RLC_SPM_INT_STATUS__RESERVED__SHIFT                                                                   0x1
+#define RLC_SPM_INT_STATUS__RLC_SPM_INT_STATUS_MASK                                                           0x00000001L
+#define RLC_SPM_INT_STATUS__RESERVED_MASK                                                                     0xFFFFFFFEL
+//RLC_SPM_INT_INFO_1
+#define RLC_SPM_INT_INFO_1__INTERRUPT_INFO_1__SHIFT                                                           0x0
+#define RLC_SPM_INT_INFO_1__INTERRUPT_INFO_1_MASK                                                             0xFFFFFFFFL
+//RLC_SPM_INT_INFO_2
+#define RLC_SPM_INT_INFO_2__INTERRUPT_INFO_2__SHIFT                                                           0x0
+#define RLC_SPM_INT_INFO_2__INTERRUPT_ID__SHIFT                                                               0x10
+#define RLC_SPM_INT_INFO_2__RESERVED__SHIFT                                                                   0x18
+#define RLC_SPM_INT_INFO_2__INTERRUPT_INFO_2_MASK                                                             0x0000FFFFL
+#define RLC_SPM_INT_INFO_2__INTERRUPT_ID_MASK                                                                 0x00FF0000L
+#define RLC_SPM_INT_INFO_2__RESERVED_MASK                                                                     0xFF000000L
+//RLC_CSIB_ADDR_LO
+#define RLC_CSIB_ADDR_LO__ADDRESS__SHIFT                                                                      0x0
+#define RLC_CSIB_ADDR_LO__ADDRESS_MASK                                                                        0xFFFFFFFFL
+//RLC_CSIB_ADDR_HI
+#define RLC_CSIB_ADDR_HI__ADDRESS__SHIFT                                                                      0x0
+#define RLC_CSIB_ADDR_HI__ADDRESS_MASK                                                                        0x01FFFFFFL
+//RLC_CSIB_LENGTH
+#define RLC_CSIB_LENGTH__LENGTH__SHIFT                                                                        0x0
+#define RLC_CSIB_LENGTH__LENGTH_MASK                                                                          0xFFFFFFFFL
+//RLC_CP_SCHEDULERS
+#define RLC_CP_SCHEDULERS__scheduler0__SHIFT                                                                  0x0
+#define RLC_CP_SCHEDULERS__scheduler1__SHIFT                                                                  0x8
+#define RLC_CP_SCHEDULERS__scheduler0_MASK                                                                    0x000000FFL
+#define RLC_CP_SCHEDULERS__scheduler1_MASK                                                                    0x0000FF00L
+//RLC_CP_EOF_INT
+#define RLC_CP_EOF_INT__INTERRUPT__SHIFT                                                                      0x0
+#define RLC_CP_EOF_INT__RESERVED__SHIFT                                                                       0x1
+#define RLC_CP_EOF_INT__INTERRUPT_MASK                                                                        0x00000001L
+#define RLC_CP_EOF_INT__RESERVED_MASK                                                                         0xFFFFFFFEL
+//RLC_CP_EOF_INT_CNTL
+#define RLC_CP_EOF_INT_CNTL__DATA__SHIFT                                                                      0x0
+#define RLC_CP_EOF_INT_CNTL__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_SPARE_INT_0
+#define RLC_SPARE_INT_0__DATA__SHIFT                                                                          0x0
+#define RLC_SPARE_INT_0__PROCESSING__SHIFT                                                                    0x1e
+#define RLC_SPARE_INT_0__COMPLETE__SHIFT                                                                      0x1f
+#define RLC_SPARE_INT_0__DATA_MASK                                                                            0x3FFFFFFFL
+#define RLC_SPARE_INT_0__PROCESSING_MASK                                                                      0x40000000L
+#define RLC_SPARE_INT_0__COMPLETE_MASK                                                                        0x80000000L
+//RLC_SPARE_INT_1
+#define RLC_SPARE_INT_1__DATA__SHIFT                                                                          0x0
+#define RLC_SPARE_INT_1__PROCESSING__SHIFT                                                                    0x1e
+#define RLC_SPARE_INT_1__COMPLETE__SHIFT                                                                      0x1f
+#define RLC_SPARE_INT_1__DATA_MASK                                                                            0x3FFFFFFFL
+#define RLC_SPARE_INT_1__PROCESSING_MASK                                                                      0x40000000L
+#define RLC_SPARE_INT_1__COMPLETE_MASK                                                                        0x80000000L
+//RLC_SPARE_INT_2
+#define RLC_SPARE_INT_2__DATA__SHIFT                                                                          0x0
+#define RLC_SPARE_INT_2__PROCESSING__SHIFT                                                                    0x1e
+#define RLC_SPARE_INT_2__COMPLETE__SHIFT                                                                      0x1f
+#define RLC_SPARE_INT_2__DATA_MASK                                                                            0x3FFFFFFFL
+#define RLC_SPARE_INT_2__PROCESSING_MASK                                                                      0x40000000L
+#define RLC_SPARE_INT_2__COMPLETE_MASK                                                                        0x80000000L
+//RLC_RLCV_SPARE_INT_1
+#define RLC_RLCV_SPARE_INT_1__INTERRUPT__SHIFT                                                                0x0
+#define RLC_RLCV_SPARE_INT_1__RESERVED__SHIFT                                                                 0x1
+#define RLC_RLCV_SPARE_INT_1__INTERRUPT_MASK                                                                  0x00000001L
+#define RLC_RLCV_SPARE_INT_1__RESERVED_MASK                                                                   0xFFFFFFFEL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pwrdec
+//GRBMX_GLARB_BUSY_MASK
+#define GRBMX_GLARB_BUSY_MASK__GLARB0C_BUSY_MASK__SHIFT                                                       0x0
+#define GRBMX_GLARB_BUSY_MASK__GLARB0A_BUSY_MASK__SHIFT                                                       0xf
+#define GRBMX_GLARB_BUSY_MASK__GLARB1C_BUSY_MASK__SHIFT                                                       0x10
+#define GRBMX_GLARB_BUSY_MASK__GLARB1A_BUSY_MASK__SHIFT                                                       0x1f
+#define GRBMX_GLARB_BUSY_MASK__GLARB0C_BUSY_MASK_MASK                                                         0x00000FFFL
+#define GRBMX_GLARB_BUSY_MASK__GLARB0A_BUSY_MASK_MASK                                                         0x00008000L
+#define GRBMX_GLARB_BUSY_MASK__GLARB1C_BUSY_MASK_MASK                                                         0x0FFF0000L
+#define GRBMX_GLARB_BUSY_MASK__GLARB1A_BUSY_MASK_MASK                                                         0x80000000L
+//CGTT_IA_CLK_CTRL
+#define CGTT_IA_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_IA_CLK_CTRL__PERF_ENABLE__SHIFT                                                                  0xf
+#define CGTT_IA_CLK_CTRL__DBG_ENABLE__SHIFT                                                                   0x10
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                               0x18
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                               0x19
+#define CGTT_IA_CLK_CTRL__DIST_OVERRIDE__SHIFT                                                                0x1a
+#define CGTT_IA_CLK_CTRL__PERF_OVERRIDE__SHIFT                                                                0x1b
+#define CGTT_IA_CLK_CTRL__PCM_OVERRIDE__SHIFT                                                                 0x1c
+#define CGTT_IA_CLK_CTRL__TESS_DIST_OVERRIDE__SHIFT                                                           0x1d
+#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                                0x1e
+#define CGTT_IA_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
+#define CGTT_IA_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_IA_CLK_CTRL__PERF_ENABLE_MASK                                                                    0x00008000L
+#define CGTT_IA_CLK_CTRL__DBG_ENABLE_MASK                                                                     0x00010000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                 0x01000000L
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                                 0x02000000L
+#define CGTT_IA_CLK_CTRL__DIST_OVERRIDE_MASK                                                                  0x04000000L
+#define CGTT_IA_CLK_CTRL__PERF_OVERRIDE_MASK                                                                  0x08000000L
+#define CGTT_IA_CLK_CTRL__PCM_OVERRIDE_MASK                                                                   0x10000000L
+#define CGTT_IA_CLK_CTRL__TESS_DIST_OVERRIDE_MASK                                                             0x20000000L
+#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE_MASK                                                                  0x40000000L
+#define CGTT_IA_CLK_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
+//CGTT_WD_CLK_CTRL
+#define CGTT_WD_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define CGTT_WD_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_WD_CLK_CTRL__PERF_ENABLE__SHIFT                                                                  0xf
+#define CGTT_WD_CLK_CTRL__DBG_ENABLE__SHIFT                                                                   0x10
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_WD_CLK_CTRL__FE_OUT_OVERRIDE__SHIFT                                                              0x17
+#define CGTT_WD_CLK_CTRL__UNUSED__SHIFT                                                                       0x18
+#define CGTT_WD_CLK_CTRL__DMA_PROC0_OVERRIDE__SHIFT                                                           0x19
+#define CGTT_WD_CLK_CTRL__DMA_PROC1_OVERRIDE__SHIFT                                                           0x1a
+#define CGTT_WD_CLK_CTRL__PERF_OVERRIDE__SHIFT                                                                0x1b
+#define CGTT_WD_CLK_CTRL__DMA_OVERRIDE__SHIFT                                                                 0x1c
+#define CGTT_WD_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                                0x1d
+#define CGTT_WD_CLK_CTRL__RBIU_INPUT_OVERRIDE__SHIFT                                                          0x1e
+#define CGTT_WD_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
+#define CGTT_WD_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define CGTT_WD_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_WD_CLK_CTRL__PERF_ENABLE_MASK                                                                    0x00008000L
+#define CGTT_WD_CLK_CTRL__DBG_ENABLE_MASK                                                                     0x00010000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_WD_CLK_CTRL__FE_OUT_OVERRIDE_MASK                                                                0x00800000L
+#define CGTT_WD_CLK_CTRL__UNUSED_MASK                                                                         0x01000000L
+#define CGTT_WD_CLK_CTRL__DMA_PROC0_OVERRIDE_MASK                                                             0x02000000L
+#define CGTT_WD_CLK_CTRL__DMA_PROC1_OVERRIDE_MASK                                                             0x04000000L
+#define CGTT_WD_CLK_CTRL__PERF_OVERRIDE_MASK                                                                  0x08000000L
+#define CGTT_WD_CLK_CTRL__DMA_OVERRIDE_MASK                                                                   0x10000000L
+#define CGTT_WD_CLK_CTRL__CORE_OVERRIDE_MASK                                                                  0x20000000L
+#define CGTT_WD_CLK_CTRL__RBIU_INPUT_OVERRIDE_MASK                                                            0x40000000L
+#define CGTT_WD_CLK_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
+//CGTT_CP_CLK_CTRL
+#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_CP_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                                0xf
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                         0x10
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                        0x1d
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                            0x1e
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                            0x1f
+#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_CP_CLK_CTRL__MGLS_OVERRIDE_MASK                                                                  0x00008000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                           0x00010000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                          0x20000000L
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                              0x40000000L
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                              0x80000000L
+//CGTT_CPF_CLK_CTRL
+#define CGTT_CPF_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_CPF_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                               0xf
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                       0x1a
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PRT__SHIFT                                                           0x1b
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_CMP__SHIFT                                                           0x1c
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_GFX__SHIFT                                                           0x1d
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                           0x1e
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                           0x1f
+#define CGTT_CPF_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_CPF_CLK_CTRL__MGLS_OVERRIDE_MASK                                                                 0x00008000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                         0x04000000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PRT_MASK                                                             0x08000000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_CMP_MASK                                                             0x10000000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_GFX_MASK                                                             0x20000000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                             0x40000000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                             0x80000000L
+//CGTT_CPC_CLK_CTRL
+#define CGTT_CPC_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_CPC_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                               0xf
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                       0x1d
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                           0x1e
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                           0x1f
+#define CGTT_CPC_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_CPC_CLK_CTRL__MGLS_OVERRIDE_MASK                                                                 0x00008000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                         0x20000000L
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                             0x40000000L
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                             0x80000000L
+//CGTT_RLC_CLK_CTRL
+#define CGTT_RLC_CLK_CTRL__RESERVED__SHIFT                                                                    0x0
+#define CGTT_RLC_CLK_CTRL__RESERVED_MASK                                                                      0xFFFFFFFFL
+//GFX_ICG_GCR_CTRL
+#define GFX_ICG_GCR_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define GFX_ICG_GCR_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define GFX_ICG_GCR_CTRL__SOFT_OVERRIDE6__SHIFT                                                               0x19
+#define GFX_ICG_GCR_CTRL__SOFT_OVERRIDE5__SHIFT                                                               0x1a
+#define GFX_ICG_GCR_CTRL__SOFT_OVERRIDE4__SHIFT                                                               0x1b
+#define GFX_ICG_GCR_CTRL__SOFT_OVERRIDE3__SHIFT                                                               0x1c
+#define GFX_ICG_GCR_CTRL__SOFT_OVERRIDE2__SHIFT                                                               0x1d
+#define GFX_ICG_GCR_CTRL__SOFT_OVERRIDE1__SHIFT                                                               0x1e
+#define GFX_ICG_GCR_CTRL__SOFT_OVERRIDE0__SHIFT                                                               0x1f
+#define GFX_ICG_GCR_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define GFX_ICG_GCR_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define GFX_ICG_GCR_CTRL__SOFT_OVERRIDE6_MASK                                                                 0x02000000L
+#define GFX_ICG_GCR_CTRL__SOFT_OVERRIDE5_MASK                                                                 0x04000000L
+#define GFX_ICG_GCR_CTRL__SOFT_OVERRIDE4_MASK                                                                 0x08000000L
+#define GFX_ICG_GCR_CTRL__SOFT_OVERRIDE3_MASK                                                                 0x10000000L
+#define GFX_ICG_GCR_CTRL__SOFT_OVERRIDE2_MASK                                                                 0x20000000L
+#define GFX_ICG_GCR_CTRL__SOFT_OVERRIDE1_MASK                                                                 0x40000000L
+#define GFX_ICG_GCR_CTRL__SOFT_OVERRIDE0_MASK                                                                 0x80000000L
+//GC_EA_CPWD_ICG_CTRL
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_RETURN__SHIFT                                                      0x0
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_DRAM_FE__SHIFT                                                     0x1
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_IO_FE__SHIFT                                                       0x2
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_REGISTER__SHIFT                                                    0x3
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                     0x4
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_MAM__SHIFT                                                         0x5
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_CREST__SHIFT                                                       0x6
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_GMI__SHIFT                                                         0x7
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_HDM__SHIFT                                                         0x8
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_PRB__SHIFT                                                         0x9
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_LPDDR__SHIFT                                                       0xa
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_RETURN_MASK                                                        0x00000001L
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_DRAM_FE_MASK                                                       0x00000002L
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_IO_FE_MASK                                                         0x00000004L
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_REGISTER_MASK                                                      0x00000008L
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                       0x00000010L
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_MAM_MASK                                                           0x00000020L
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_CREST_MASK                                                         0x00000040L
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_GMI_MASK                                                           0x00000080L
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_HDM_MASK                                                           0x00000100L
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_PRB_MASK                                                           0x00000200L
+#define GC_EA_CPWD_ICG_CTRL__SOFT_OVERRIDE_LPDDR_MASK                                                         0x00000400L
+//GFX_ICG_GC_CAC_CLK_CTRL
+#define GFX_ICG_GC_CAC_CLK_CTRL__GC_CAC_DYNAMIC_ICG_OVERRIDE__SHIFT                                           0x0
+#define GFX_ICG_GC_CAC_CLK_CTRL__GC_CAC_REG_ICG_OVERRIDE__SHIFT                                               0x1
+#define GFX_ICG_GC_CAC_CLK_CTRL__GC_CAC_DYNAMIC_ICG_OVERRIDE_MASK                                             0x00000001L
+#define GFX_ICG_GC_CAC_CLK_CTRL__GC_CAC_REG_ICG_OVERRIDE_MASK                                                 0x00000002L
+//GFX_ICG_GRBM_CTRL
+#define GFX_ICG_GRBM_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define GFX_ICG_GRBM_CTRL__SOFT_OVERRIDE_SE__SHIFT                                                            0x10
+#define GFX_ICG_GRBM_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                       0x1d
+#define GFX_ICG_GRBM_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                           0x1e
+#define GFX_ICG_GRBM_CTRL__OFF_HYSTERESIS_MASK                                                                0x000003F0L
+#define GFX_ICG_GRBM_CTRL__SOFT_OVERRIDE_SE_MASK                                                              0x00FF0000L
+#define GFX_ICG_GRBM_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                         0x20000000L
+#define GFX_ICG_GRBM_CTRL__SOFT_OVERRIDE_DYN_MASK                                                             0x40000000L
+//CC_GC_HBM_DISABLE
+#define CC_GC_HBM_DISABLE__CC_GC_HBM_DISABLE__SHIFT                                                           0x10
+#define CC_GC_HBM_DISABLE__CC_GC_HBM_DISABLE_MASK                                                             0x0FFF0000L
+//GRBMX_LPDDR_DISABLE_0
+#define GRBMX_LPDDR_DISABLE_0__DISABLE__SHIFT                                                                 0x10
+#define GRBMX_LPDDR_DISABLE_0__DISABLE_MASK                                                                   0xFFFF0000L
+//GRBMX_LPDDR_DISABLE_1
+#define GRBMX_LPDDR_DISABLE_1__DISABLE__SHIFT                                                                 0x10
+#define GRBMX_LPDDR_DISABLE_1__DISABLE_MASK                                                                   0xFFFF0000L
+//GLARBI_GLARBR_MGCG_OVERRIDE
+#define GLARBI_GLARBR_MGCG_OVERRIDE__GLARBA_GLARBIR_MGCG_RET_DCLK_OVERRIDE__SHIFT                             0x0
+#define GLARBI_GLARBR_MGCG_OVERRIDE__GLARBA_GLARBIW_MGCG_RET_DCLK_OVERRIDE__SHIFT                             0x1
+#define GLARBI_GLARBR_MGCG_OVERRIDE__GLARBA_GLARBIW_MGCG_SRC_DCLK_OVERRIDE__SHIFT                             0x2
+#define GLARBI_GLARBR_MGCG_OVERRIDE__GLARBA_GLARBIR_MGCG_RET_DCLK_OVERRIDE_MASK                               0x00000001L
+#define GLARBI_GLARBR_MGCG_OVERRIDE__GLARBA_GLARBIW_MGCG_RET_DCLK_OVERRIDE_MASK                               0x00000002L
+#define GLARBI_GLARBR_MGCG_OVERRIDE__GLARBA_GLARBIW_MGCG_SRC_DCLK_OVERRIDE_MASK                               0x00000004L
+//ICG_GLARBA_CTRL
+#define ICG_GLARBA_CTRL__REG_CLK_OVERRIDE__SHIFT                                                              0x0
+#define ICG_GLARBA_CTRL__REQ_CLI_CLK_OVERRIDE__SHIFT                                                          0x1
+#define ICG_GLARBA_CTRL__REQ_ARB_CLK_OVERRIDE__SHIFT                                                          0x2
+#define ICG_GLARBA_CTRL__RET_CLK_OVERRIDE__SHIFT                                                              0x3
+#define ICG_GLARBA_CTRL__REQ_CREDIT_CLK_OVERRIDE__SHIFT                                                       0x4
+#define ICG_GLARBA_CTRL__PERFMON_CLK_OVERRIDE__SHIFT                                                          0x5
+#define ICG_GLARBA_CTRL__LOCAL_CAC_CLK_OVERRIDE__SHIFT                                                        0x6
+#define ICG_GLARBA_CTRL__GCRD_CLK_OVERRIDE__SHIFT                                                             0x7
+#define ICG_GLARBA_CTRL__REG_CLK_OVERRIDE_MASK                                                                0x00000001L
+#define ICG_GLARBA_CTRL__REQ_CLI_CLK_OVERRIDE_MASK                                                            0x00000002L
+#define ICG_GLARBA_CTRL__REQ_ARB_CLK_OVERRIDE_MASK                                                            0x00000004L
+#define ICG_GLARBA_CTRL__RET_CLK_OVERRIDE_MASK                                                                0x00000008L
+#define ICG_GLARBA_CTRL__REQ_CREDIT_CLK_OVERRIDE_MASK                                                         0x00000010L
+#define ICG_GLARBA_CTRL__PERFMON_CLK_OVERRIDE_MASK                                                            0x00000020L
+#define ICG_GLARBA_CTRL__LOCAL_CAC_CLK_OVERRIDE_MASK                                                          0x00000040L
+#define ICG_GLARBA_CTRL__GCRD_CLK_OVERRIDE_MASK                                                               0x00000080L
+//ICG_GLARBC_CLK_CTRL
+#define ICG_GLARBC_CLK_CTRL__GLOBAL_CLK_OVERRIDE__SHIFT                                                       0x0
+#define ICG_GLARBC_CLK_CTRL__GLOBAL_NONHARVESTABLE_CLK_OVERRIDE__SHIFT                                        0x1
+#define ICG_GLARBC_CLK_CTRL__REQUEST_CLK_OVERRIDE__SHIFT                                                      0x2
+#define ICG_GLARBC_CLK_CTRL__SRC_DATA_CLK_OVERRIDE__SHIFT                                                     0x3
+#define ICG_GLARBC_CLK_CTRL__RETURN_CLK_OVERRIDE__SHIFT                                                       0x4
+#define ICG_GLARBC_CLK_CTRL__GRBM_CLK_OVERRIDE__SHIFT                                                         0x5
+#define ICG_GLARBC_CLK_CTRL__PERF_CLK_OVERRIDE__SHIFT                                                         0x6
+#define ICG_GLARBC_CLK_CTRL__GCR_CLK_OVERRIDE__SHIFT                                                          0x7
+#define ICG_GLARBC_CLK_CTRL__CREST_RET_CLK_OVERRIDE__SHIFT                                                    0x8
+#define ICG_GLARBC_CLK_CTRL__GLOBAL_CLK_OVERRIDE_MASK                                                         0x00000001L
+#define ICG_GLARBC_CLK_CTRL__GLOBAL_NONHARVESTABLE_CLK_OVERRIDE_MASK                                          0x00000002L
+#define ICG_GLARBC_CLK_CTRL__REQUEST_CLK_OVERRIDE_MASK                                                        0x00000004L
+#define ICG_GLARBC_CLK_CTRL__SRC_DATA_CLK_OVERRIDE_MASK                                                       0x00000008L
+#define ICG_GLARBC_CLK_CTRL__RETURN_CLK_OVERRIDE_MASK                                                         0x00000010L
+#define ICG_GLARBC_CLK_CTRL__GRBM_CLK_OVERRIDE_MASK                                                           0x00000020L
+#define ICG_GLARBC_CLK_CTRL__PERF_CLK_OVERRIDE_MASK                                                           0x00000040L
+#define ICG_GLARBC_CLK_CTRL__GCR_CLK_OVERRIDE_MASK                                                            0x00000080L
+#define ICG_GLARBC_CLK_CTRL__CREST_RET_CLK_OVERRIDE_MASK                                                      0x00000100L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_pspdec
+//GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0__CID0_SECLEVEL__SHIFT                                              0x0
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0__CID1_SECLEVEL__SHIFT                                              0x4
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0__CID2_SECLEVEL__SHIFT                                              0x8
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0__CID3_SECLEVEL__SHIFT                                              0xc
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0__CID4_SECLEVEL__SHIFT                                              0x10
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0__CID5_SECLEVEL__SHIFT                                              0x14
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0__CID6_SECLEVEL__SHIFT                                              0x18
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0__CID7_SECLEVEL__SHIFT                                              0x1c
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0__CID0_SECLEVEL_MASK                                                0x0000000FL
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0__CID1_SECLEVEL_MASK                                                0x000000F0L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0__CID2_SECLEVEL_MASK                                                0x00000F00L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0__CID3_SECLEVEL_MASK                                                0x0000F000L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0__CID4_SECLEVEL_MASK                                                0x000F0000L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0__CID5_SECLEVEL_MASK                                                0x00F00000L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0__CID6_SECLEVEL_MASK                                                0x0F000000L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP0__CID7_SECLEVEL_MASK                                                0xF0000000L
+//GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1__CID8_SECLEVEL__SHIFT                                              0x0
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1__CID9_SECLEVEL__SHIFT                                              0x4
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1__CID10_SECLEVEL__SHIFT                                             0x8
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1__CID11_SECLEVEL__SHIFT                                             0xc
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1__CID12_SECLEVEL__SHIFT                                             0x10
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1__CID13_SECLEVEL__SHIFT                                             0x14
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1__CID14_SECLEVEL__SHIFT                                             0x18
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1__CID15_SECLEVEL__SHIFT                                             0x1c
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1__CID8_SECLEVEL_MASK                                                0x0000000FL
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1__CID9_SECLEVEL_MASK                                                0x000000F0L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1__CID10_SECLEVEL_MASK                                               0x00000F00L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1__CID11_SECLEVEL_MASK                                               0x0000F000L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1__CID12_SECLEVEL_MASK                                               0x000F0000L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1__CID13_SECLEVEL_MASK                                               0x00F00000L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1__CID14_SECLEVEL_MASK                                               0x0F000000L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP1__CID15_SECLEVEL_MASK                                               0xF0000000L
+//GC_EA_CPWD_SDP_ERR_CTRL
+#define GC_EA_CPWD_SDP_ERR_CTRL__INJ_GC_TO_SDP_ERREVENT__SHIFT                                                0x0
+#define GC_EA_CPWD_SDP_ERR_CTRL__INJ_SDP_TO_GC_UER__SHIFT                                                     0x1
+#define GC_EA_CPWD_SDP_ERR_CTRL__SDP_READ_RESPONSE_PARITY_EN__SHIFT                                           0x2
+#define GC_EA_CPWD_SDP_ERR_CTRL__SDP_WRITE_RESPONSE_PARITY_EN__SHIFT                                          0x3
+#define GC_EA_CPWD_SDP_ERR_CTRL__SDP_PROBE_PARITY_EN__SHIFT                                                   0x4
+#define GC_EA_CPWD_SDP_ERR_CTRL__SDP_PROBE_RSP_PARITY_EN__SHIFT                                               0x5
+#define GC_EA_CPWD_SDP_ERR_CTRL__SDP_ACKCH_PARITY_EN__SHIFT                                                   0x6
+#define GC_EA_CPWD_SDP_ERR_CTRL__INJ_GC_TO_SDP_ERREVENT_MASK                                                  0x00000001L
+#define GC_EA_CPWD_SDP_ERR_CTRL__INJ_SDP_TO_GC_UER_MASK                                                       0x00000002L
+#define GC_EA_CPWD_SDP_ERR_CTRL__SDP_READ_RESPONSE_PARITY_EN_MASK                                             0x00000004L
+#define GC_EA_CPWD_SDP_ERR_CTRL__SDP_WRITE_RESPONSE_PARITY_EN_MASK                                            0x00000008L
+#define GC_EA_CPWD_SDP_ERR_CTRL__SDP_PROBE_PARITY_EN_MASK                                                     0x00000010L
+#define GC_EA_CPWD_SDP_ERR_CTRL__SDP_PROBE_RSP_PARITY_EN_MASK                                                 0x00000020L
+#define GC_EA_CPWD_SDP_ERR_CTRL__SDP_ACKCH_PARITY_EN_MASK                                                     0x00000040L
+//GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2__CID0_SECLEVEL__SHIFT                                              0x0
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2__CID1_SECLEVEL__SHIFT                                              0x4
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2__CID2_SECLEVEL__SHIFT                                              0x8
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2__CID3_SECLEVEL__SHIFT                                              0xc
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2__CID4_SECLEVEL__SHIFT                                              0x10
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2__CID5_SECLEVEL__SHIFT                                              0x14
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2__CID6_SECLEVEL__SHIFT                                              0x18
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2__CID7_SECLEVEL__SHIFT                                              0x1c
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2__CID0_SECLEVEL_MASK                                                0x0000000FL
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2__CID1_SECLEVEL_MASK                                                0x000000F0L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2__CID2_SECLEVEL_MASK                                                0x00000F00L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2__CID3_SECLEVEL_MASK                                                0x0000F000L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2__CID4_SECLEVEL_MASK                                                0x000F0000L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2__CID5_SECLEVEL_MASK                                                0x00F00000L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2__CID6_SECLEVEL_MASK                                                0x0F000000L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP2__CID7_SECLEVEL_MASK                                                0xF0000000L
+//GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3__CID8_SECLEVEL__SHIFT                                              0x0
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3__CID9_SECLEVEL__SHIFT                                              0x4
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3__CID10_SECLEVEL__SHIFT                                             0x8
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3__CID11_SECLEVEL__SHIFT                                             0xc
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3__CID12_SECLEVEL__SHIFT                                             0x10
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3__CID13_SECLEVEL__SHIFT                                             0x14
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3__CID14_SECLEVEL__SHIFT                                             0x18
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3__CID15_SECLEVEL__SHIFT                                             0x1c
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3__CID8_SECLEVEL_MASK                                                0x0000000FL
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3__CID9_SECLEVEL_MASK                                                0x000000F0L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3__CID10_SECLEVEL_MASK                                               0x00000F00L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3__CID11_SECLEVEL_MASK                                               0x0000F000L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3__CID12_SECLEVEL_MASK                                               0x000F0000L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3__CID13_SECLEVEL_MASK                                               0x00F00000L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3__CID14_SECLEVEL_MASK                                               0x0F000000L
+#define GC_EA_CPWD_SDP_SECLEVEL_NONIO_MAP3__CID15_SECLEVEL_MASK                                               0xF0000000L
+//GRBM_SRCID_CAM_INDEX
+#define GRBM_SRCID_CAM_INDEX__INDEX__SHIFT                                                                    0x0
+#define GRBM_SRCID_CAM_INDEX__INDEX_MASK                                                                      0x00000007L
+//GRBM_SRCID_CAM_DATA
+#define GRBM_SRCID_CAM_DATA__PROG_SRCID__SHIFT                                                                0x0
+#define GRBM_SRCID_CAM_DATA__TRUST_LEVEL__SHIFT                                                               0x18
+#define GRBM_SRCID_CAM_DATA__PROG_SRCID_MASK                                                                  0x0001FFFFL
+#define GRBM_SRCID_CAM_DATA__TRUST_LEVEL_MASK                                                                 0x0F000000L
+//GRBM_IOV_RANGE_ENABLE
+#define GRBM_IOV_RANGE_ENABLE__ENABLE__SHIFT                                                                  0x0
+#define GRBM_IOV_RANGE_ENABLE__ENABLE_MASK                                                                    0xFFFFFFFFL
+//GRBM_IOV_RANGE_READ_BLOCK_ENABLE
+#define GRBM_IOV_RANGE_READ_BLOCK_ENABLE__ENABLE__SHIFT                                                       0x0
+#define GRBM_IOV_RANGE_READ_BLOCK_ENABLE__ENABLE_MASK                                                         0xFFFFFFFFL
+//GRBM_SEC_CNTL
+#define GRBM_SEC_CNTL__DEBUG_ENABLE__SHIFT                                                                    0x0
+#define GRBM_SEC_CNTL__DEBUG_ENABLE_MASK                                                                      0x00000001L
+//GRBM_IOV_SPARE_RANGE_START
+#define GRBM_IOV_SPARE_RANGE_START__START__SHIFT                                                              0x0
+#define GRBM_IOV_SPARE_RANGE_START__START_MASK                                                                0x0003FFFFL
+//GRBM_IOV_SPARE_RANGE_END
+#define GRBM_IOV_SPARE_RANGE_END__END__SHIFT                                                                  0x0
+#define GRBM_IOV_SPARE_RANGE_END__END_MASK                                                                    0x0003FFFFL
+//GRBM_CAM_INDEX
+#define GRBM_CAM_INDEX__CAM_INDEX__SHIFT                                                                      0x0
+#define GRBM_CAM_INDEX__CAM_INDEX_MASK                                                                        0x0000000FL
+//GRBM_CAM_DATA
+#define GRBM_CAM_DATA__CAM_ADDR__SHIFT                                                                        0x0
+#define GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT                                                                   0x10
+#define GRBM_CAM_DATA__CAM_ADDR_MASK                                                                          0x0000FFFFL
+#define GRBM_CAM_DATA__CAM_REMAPADDR_MASK                                                                     0xFFFF0000L
+//GRBM_CAM_DATA_UPPER
+#define GRBM_CAM_DATA_UPPER__CAM_ADDR__SHIFT                                                                  0x0
+#define GRBM_CAM_DATA_UPPER__CAM_REMAPADDR__SHIFT                                                             0x10
+#define GRBM_CAM_DATA_UPPER__CAM_ADDR_MASK                                                                    0x00000003L
+#define GRBM_CAM_DATA_UPPER__CAM_REMAPADDR_MASK                                                               0x00030000L
+//RLC_REG_SEC_INT_STATUS
+#define RLC_REG_SEC_INT_STATUS__FWL_VIOL_COUNT__SHIFT                                                         0x0
+#define RLC_REG_SEC_INT_STATUS__FWL_VIOL_COUNT_OVERFLOW__SHIFT                                                0x10
+#define RLC_REG_SEC_INT_STATUS__RESERVED__SHIFT                                                               0x11
+#define RLC_REG_SEC_INT_STATUS__FWL_VIOL_COUNT_MASK                                                           0x0000FFFFL
+#define RLC_REG_SEC_INT_STATUS__FWL_VIOL_COUNT_OVERFLOW_MASK                                                  0x00010000L
+#define RLC_REG_SEC_INT_STATUS__RESERVED_MASK                                                                 0xFFFE0000L
+//RLC_FWL_FIRST_VIOL_ADDR
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_ADDR__SHIFT                                                             0x0
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_OP__SHIFT                                                               0x12
+#define RLC_FWL_FIRST_VIOL_ADDR__RESERVED__SHIFT                                                              0x13
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_ADDR_MASK                                                               0x0003FFFFL
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_OP_MASK                                                                 0x00040000L
+#define RLC_FWL_FIRST_VIOL_ADDR__RESERVED_MASK                                                                0xFFF80000L
+//RLC_FWL_FIRST_VIOL_ADDR_HI
+#define RLC_FWL_FIRST_VIOL_ADDR_HI__ADDR_HI__SHIFT                                                            0x0
+#define RLC_FWL_FIRST_VIOL_ADDR_HI__RESERVED__SHIFT                                                           0x1c
+#define RLC_FWL_FIRST_VIOL_ADDR_HI__ADDR_HI_MASK                                                              0x0FFFFFFFL
+#define RLC_FWL_FIRST_VIOL_ADDR_HI__RESERVED_MASK                                                             0xF0000000L
+//RLC_UTC_BYPASS_CNTL
+#define RLC_UTC_BYPASS_CNTL__SPM__SHIFT                                                                       0x0
+#define RLC_UTC_BYPASS_CNTL__TH0__SHIFT                                                                       0x1
+#define RLC_UTC_BYPASS_CNTL__TH1__SHIFT                                                                       0x2
+#define RLC_UTC_BYPASS_CNTL__TH2__SHIFT                                                                       0x3
+#define RLC_UTC_BYPASS_CNTL__LX6__SHIFT                                                                       0x4
+#define RLC_UTC_BYPASS_CNTL__DMA__SHIFT                                                                       0x5
+#define RLC_UTC_BYPASS_CNTL__SRM__SHIFT                                                                       0x6
+#define RLC_UTC_BYPASS_CNTL__DLG__SHIFT                                                                       0x7
+#define RLC_UTC_BYPASS_CNTL__RESERVED__SHIFT                                                                  0x8
+#define RLC_UTC_BYPASS_CNTL__SPM_MASK                                                                         0x00000001L
+#define RLC_UTC_BYPASS_CNTL__TH0_MASK                                                                         0x00000002L
+#define RLC_UTC_BYPASS_CNTL__TH1_MASK                                                                         0x00000004L
+#define RLC_UTC_BYPASS_CNTL__TH2_MASK                                                                         0x00000008L
+#define RLC_UTC_BYPASS_CNTL__LX6_MASK                                                                         0x00000010L
+#define RLC_UTC_BYPASS_CNTL__DMA_MASK                                                                         0x00000020L
+#define RLC_UTC_BYPASS_CNTL__SRM_MASK                                                                         0x00000040L
+#define RLC_UTC_BYPASS_CNTL__DLG_MASK                                                                         0x00000080L
+#define RLC_UTC_BYPASS_CNTL__RESERVED_MASK                                                                    0xFFFFFF00L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_cp_pspdec
+//CP_MES_DM_INDEX_ADDR
+#define CP_MES_DM_INDEX_ADDR__ADDR__SHIFT                                                                     0x0
+#define CP_MES_DM_INDEX_ADDR__ADDR_MASK                                                                       0xFFFFFFFFL
+//CP_MES_DM_INDEX_DATA
+#define CP_MES_DM_INDEX_DATA__DATA__SHIFT                                                                     0x0
+#define CP_MES_DM_INDEX_DATA__DATA_MASK                                                                       0xFFFFFFFFL
+//CP_MEC_DM_INDEX_ADDR
+#define CP_MEC_DM_INDEX_ADDR__ADDR__SHIFT                                                                     0x0
+#define CP_MEC_DM_INDEX_ADDR__ADDR_MASK                                                                       0xFFFFFFFFL
+//CP_MEC_DM_INDEX_DATA
+#define CP_MEC_DM_INDEX_DATA__DATA__SHIFT                                                                     0x0
+#define CP_MEC_DM_INDEX_DATA__DATA_MASK                                                                       0xFFFFFFFFL
+//CP_GFX_RS64_DM_INDEX_ADDR
+#define CP_GFX_RS64_DM_INDEX_ADDR__ADDR__SHIFT                                                                0x0
+#define CP_GFX_RS64_DM_INDEX_ADDR__ADDR_MASK                                                                  0xFFFFFFFFL
+//CP_GFX_RS64_DM_INDEX_DATA
+#define CP_GFX_RS64_DM_INDEX_DATA__DATA__SHIFT                                                                0x0
+#define CP_GFX_RS64_DM_INDEX_DATA__DATA_MASK                                                                  0xFFFFFFFFL
+//CPG_PSP_DEBUG
+#define CPG_PSP_DEBUG__PRIV_VIOLATION_CNTL__SHIFT                                                             0x0
+#define CPG_PSP_DEBUG__VMID_VIOLATION_CNTL__SHIFT                                                             0x2
+#define CPG_PSP_DEBUG__GPA_OVERRIDE__SHIFT                                                                    0x3
+#define CPG_PSP_DEBUG__UCODE_VF_OVERRIDE__SHIFT                                                               0x4
+#define CPG_PSP_DEBUG__MTYPE_TMZ_OVERRIDE__SHIFT                                                              0x5
+#define CPG_PSP_DEBUG__SECURE_REG_OVERRIDE__SHIFT                                                             0x6
+#define CPG_PSP_DEBUG__PRIV_VIOLATION_CNTL_MASK                                                               0x00000003L
+#define CPG_PSP_DEBUG__VMID_VIOLATION_CNTL_MASK                                                               0x00000004L
+#define CPG_PSP_DEBUG__GPA_OVERRIDE_MASK                                                                      0x00000008L
+#define CPG_PSP_DEBUG__UCODE_VF_OVERRIDE_MASK                                                                 0x00000010L
+#define CPG_PSP_DEBUG__MTYPE_TMZ_OVERRIDE_MASK                                                                0x00000020L
+#define CPG_PSP_DEBUG__SECURE_REG_OVERRIDE_MASK                                                               0x00000040L
+//CPC_PSP_DEBUG
+#define CPC_PSP_DEBUG__PRIV_VIOLATION_CNTL__SHIFT                                                             0x0
+#define CPC_PSP_DEBUG__GPA_OVERRIDE__SHIFT                                                                    0x3
+#define CPC_PSP_DEBUG__UCODE_VF_OVERRIDE__SHIFT                                                               0x4
+#define CPC_PSP_DEBUG__MTYPE_TMZ_OVERRIDE__SHIFT                                                              0x5
+#define CPC_PSP_DEBUG__SECURE_REG_OVERRIDE__SHIFT                                                             0x6
+#define CPC_PSP_DEBUG__PRIV_VIOLATION_CNTL_MASK                                                               0x00000003L
+#define CPC_PSP_DEBUG__GPA_OVERRIDE_MASK                                                                      0x00000008L
+#define CPC_PSP_DEBUG__UCODE_VF_OVERRIDE_MASK                                                                 0x00000010L
+#define CPC_PSP_DEBUG__MTYPE_TMZ_OVERRIDE_MASK                                                                0x00000020L
+#define CPC_PSP_DEBUG__SECURE_REG_OVERRIDE_MASK                                                               0x00000040L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_ch_pwrdec
+//CHI_CHR_MGCG_OVERRIDE
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHIR_MGCG_RET_DCLK_OVERRIDE__SHIFT                                         0x0
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHIW_MGCG_RET_DCLK_OVERRIDE__SHIFT                                         0x1
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHIW_MGCG_SRC_DCLK_OVERRIDE__SHIFT                                         0x2
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHIR_MGCG_RET_DCLK_OVERRIDE_MASK                                           0x00000001L
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHIW_MGCG_RET_DCLK_OVERRIDE_MASK                                           0x00000002L
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHIW_MGCG_SRC_DCLK_OVERRIDE_MASK                                           0x00000004L
+//ICG_CHA_CTRL
+#define ICG_CHA_CTRL__REG_CLK_OVERRIDE__SHIFT                                                                 0x0
+#define ICG_CHA_CTRL__REQ_CLI_CLK_OVERRIDE__SHIFT                                                             0x1
+#define ICG_CHA_CTRL__REQ_ARB_CLK_OVERRIDE__SHIFT                                                             0x2
+#define ICG_CHA_CTRL__RET_CLK_OVERRIDE__SHIFT                                                                 0x3
+#define ICG_CHA_CTRL__REQ_CREDIT_CLK_OVERRIDE__SHIFT                                                          0x4
+#define ICG_CHA_CTRL__PERFMON_CLK_OVERRIDE__SHIFT                                                             0x5
+#define ICG_CHA_CTRL__REG_CLK_OVERRIDE_MASK                                                                   0x00000001L
+#define ICG_CHA_CTRL__REQ_CLI_CLK_OVERRIDE_MASK                                                               0x00000002L
+#define ICG_CHA_CTRL__REQ_ARB_CLK_OVERRIDE_MASK                                                               0x00000004L
+#define ICG_CHA_CTRL__RET_CLK_OVERRIDE_MASK                                                                   0x00000008L
+#define ICG_CHA_CTRL__REQ_CREDIT_CLK_OVERRIDE_MASK                                                            0x00000010L
+#define ICG_CHA_CTRL__PERFMON_CLK_OVERRIDE_MASK                                                               0x00000020L
+//ICG_CHC_CLK_CTRL
+#define ICG_CHC_CLK_CTRL__GLOBAL_CLK_OVERRIDE__SHIFT                                                          0x0
+#define ICG_CHC_CLK_CTRL__GLOBAL_NONHARVESTABLE_CLK_OVERRIDE__SHIFT                                           0x1
+#define ICG_CHC_CLK_CTRL__REQUEST_CLK_OVERRIDE__SHIFT                                                         0x2
+#define ICG_CHC_CLK_CTRL__SRC_DATA_CLK_OVERRIDE__SHIFT                                                        0x3
+#define ICG_CHC_CLK_CTRL__RETURN_CLK_OVERRIDE__SHIFT                                                          0x4
+#define ICG_CHC_CLK_CTRL__GRBM_CLK_OVERRIDE__SHIFT                                                            0x5
+#define ICG_CHC_CLK_CTRL__PERF_CLK_OVERRIDE__SHIFT                                                            0x6
+#define ICG_CHC_CLK_CTRL__OC_IREQ_CLK_OVERRIDE__SHIFT                                                         0x7
+#define ICG_CHC_CLK_CTRL__OC_OREQ_CLK_OVERRIDE__SHIFT                                                         0x8
+#define ICG_CHC_CLK_CTRL__DCC_COMP_CLK_OVERRIDE__SHIFT                                                        0x9
+#define ICG_CHC_CLK_CTRL__GLOBAL_CLK_OVERRIDE_MASK                                                            0x00000001L
+#define ICG_CHC_CLK_CTRL__GLOBAL_NONHARVESTABLE_CLK_OVERRIDE_MASK                                             0x00000002L
+#define ICG_CHC_CLK_CTRL__REQUEST_CLK_OVERRIDE_MASK                                                           0x00000004L
+#define ICG_CHC_CLK_CTRL__SRC_DATA_CLK_OVERRIDE_MASK                                                          0x00000008L
+#define ICG_CHC_CLK_CTRL__RETURN_CLK_OVERRIDE_MASK                                                            0x00000010L
+#define ICG_CHC_CLK_CTRL__GRBM_CLK_OVERRIDE_MASK                                                              0x00000020L
+#define ICG_CHC_CLK_CTRL__PERF_CLK_OVERRIDE_MASK                                                              0x00000040L
+#define ICG_CHC_CLK_CTRL__OC_IREQ_CLK_OVERRIDE_MASK                                                           0x00000080L
+#define ICG_CHC_CLK_CTRL__OC_OREQ_CLK_OVERRIDE_MASK                                                           0x00000100L
+#define ICG_CHC_CLK_CTRL__DCC_COMP_CLK_OVERRIDE_MASK                                                          0x00000200L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_gfx_imu_cpwd_gfx_imudec
+//GFX_IMU_C2PMSG_0
+#define GFX_IMU_C2PMSG_0__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_0__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_1
+#define GFX_IMU_C2PMSG_1__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_1__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_2
+#define GFX_IMU_C2PMSG_2__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_2__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_3
+#define GFX_IMU_C2PMSG_3__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_3__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_4
+#define GFX_IMU_C2PMSG_4__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_4__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_5
+#define GFX_IMU_C2PMSG_5__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_5__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_6
+#define GFX_IMU_C2PMSG_6__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_6__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_7
+#define GFX_IMU_C2PMSG_7__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_7__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_8
+#define GFX_IMU_C2PMSG_8__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_8__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_9
+#define GFX_IMU_C2PMSG_9__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_9__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_10
+#define GFX_IMU_C2PMSG_10__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_10__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_11
+#define GFX_IMU_C2PMSG_11__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_11__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_12
+#define GFX_IMU_C2PMSG_12__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_12__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_13
+#define GFX_IMU_C2PMSG_13__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_13__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_14
+#define GFX_IMU_C2PMSG_14__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_14__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_15
+#define GFX_IMU_C2PMSG_15__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_15__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_16
+#define GFX_IMU_C2PMSG_16__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_16__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_17
+#define GFX_IMU_C2PMSG_17__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_17__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_18
+#define GFX_IMU_C2PMSG_18__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_18__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_19
+#define GFX_IMU_C2PMSG_19__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_19__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_20
+#define GFX_IMU_C2PMSG_20__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_20__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_21
+#define GFX_IMU_C2PMSG_21__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_21__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_22
+#define GFX_IMU_C2PMSG_22__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_22__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_23
+#define GFX_IMU_C2PMSG_23__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_23__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_24
+#define GFX_IMU_C2PMSG_24__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_24__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_25
+#define GFX_IMU_C2PMSG_25__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_25__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_26
+#define GFX_IMU_C2PMSG_26__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_26__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_27
+#define GFX_IMU_C2PMSG_27__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_27__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_28
+#define GFX_IMU_C2PMSG_28__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_28__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_29
+#define GFX_IMU_C2PMSG_29__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_29__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_30
+#define GFX_IMU_C2PMSG_30__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_30__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_31
+#define GFX_IMU_C2PMSG_31__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_31__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_32
+#define GFX_IMU_C2PMSG_32__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_32__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_33
+#define GFX_IMU_C2PMSG_33__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_33__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_34
+#define GFX_IMU_C2PMSG_34__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_34__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_35
+#define GFX_IMU_C2PMSG_35__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_35__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_36
+#define GFX_IMU_C2PMSG_36__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_36__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_37
+#define GFX_IMU_C2PMSG_37__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_37__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_38
+#define GFX_IMU_C2PMSG_38__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_38__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_39
+#define GFX_IMU_C2PMSG_39__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_39__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_40
+#define GFX_IMU_C2PMSG_40__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_40__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_41
+#define GFX_IMU_C2PMSG_41__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_41__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_42
+#define GFX_IMU_C2PMSG_42__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_42__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_43
+#define GFX_IMU_C2PMSG_43__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_43__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_44
+#define GFX_IMU_C2PMSG_44__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_44__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_45
+#define GFX_IMU_C2PMSG_45__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_45__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_46
+#define GFX_IMU_C2PMSG_46__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_46__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_47
+#define GFX_IMU_C2PMSG_47__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_47__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_MSG_FLAGS
+#define GFX_IMU_MSG_FLAGS__STATUS__SHIFT                                                                      0x0
+#define GFX_IMU_MSG_FLAGS__STATUS_MASK                                                                        0xFFFFFFFFL
+//GFX_IMU_C2PMSG_ACCESS_CTRL0
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC0__SHIFT                                                              0x0
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC1__SHIFT                                                              0x3
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC2__SHIFT                                                              0x6
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC3__SHIFT                                                              0x9
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC4__SHIFT                                                              0xc
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC5__SHIFT                                                              0xf
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC6__SHIFT                                                              0x12
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC7__SHIFT                                                              0x15
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC0_MASK                                                                0x00000007L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC1_MASK                                                                0x00000038L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC2_MASK                                                                0x000001C0L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC3_MASK                                                                0x00000E00L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC4_MASK                                                                0x00007000L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC5_MASK                                                                0x00038000L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC6_MASK                                                                0x001C0000L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC7_MASK                                                                0x00E00000L
+//GFX_IMU_C2PMSG_ACCESS_CTRL1
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC8_15__SHIFT                                                           0x0
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC16_23__SHIFT                                                          0x3
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC24_31__SHIFT                                                          0x6
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC32_39__SHIFT                                                          0x9
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC40_47__SHIFT                                                          0xc
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC8_15_MASK                                                             0x00000007L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC16_23_MASK                                                            0x00000038L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC24_31_MASK                                                            0x000001C0L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC32_39_MASK                                                            0x00000E00L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC40_47_MASK                                                            0x00007000L
+//GFX_IMU_PWRMGT_IRQ_CTRL
+#define GFX_IMU_PWRMGT_IRQ_CTRL__REQ__SHIFT                                                                   0x0
+#define GFX_IMU_PWRMGT_IRQ_CTRL__REQ_MASK                                                                     0x00000001L
+//GFX_IMU_MP1_MUTEX
+#define GFX_IMU_MP1_MUTEX__MUTEX__SHIFT                                                                       0x0
+#define GFX_IMU_MP1_MUTEX__MUTEX_MASK                                                                         0x00000003L
+//GFX_IMU_RLC_DATA_4
+#define GFX_IMU_RLC_DATA_4__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_RLC_DATA_4__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_RLC_DATA_3
+#define GFX_IMU_RLC_DATA_3__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_RLC_DATA_3__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_RLC_DATA_2
+#define GFX_IMU_RLC_DATA_2__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_RLC_DATA_2__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_RLC_DATA_1
+#define GFX_IMU_RLC_DATA_1__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_RLC_DATA_1__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_RLC_DATA_0
+#define GFX_IMU_RLC_DATA_0__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_RLC_DATA_0__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_RLC_CMD
+#define GFX_IMU_RLC_CMD__CMD__SHIFT                                                                           0x0
+#define GFX_IMU_RLC_CMD__CMD_MASK                                                                             0xFFFFFFFFL
+//GFX_IMU_RLC_MUTEX
+#define GFX_IMU_RLC_MUTEX__MUTEX__SHIFT                                                                       0x0
+#define GFX_IMU_RLC_MUTEX__MUTEX_MASK                                                                         0x00000003L
+//GFX_IMU_RLC_MSG_STATUS
+#define GFX_IMU_RLC_MSG_STATUS__IMU2RLC_BUSY__SHIFT                                                           0x0
+#define GFX_IMU_RLC_MSG_STATUS__IMU2RLC_MSG_ERROR__SHIFT                                                      0x1
+#define GFX_IMU_RLC_MSG_STATUS__RLC2IMU_MSGDONE__SHIFT                                                        0x10
+#define GFX_IMU_RLC_MSG_STATUS__RLC2IMU_CHGTOG__SHIFT                                                         0x1e
+#define GFX_IMU_RLC_MSG_STATUS__RLC2IMU_DONETOG__SHIFT                                                        0x1f
+#define GFX_IMU_RLC_MSG_STATUS__IMU2RLC_BUSY_MASK                                                             0x00000001L
+#define GFX_IMU_RLC_MSG_STATUS__IMU2RLC_MSG_ERROR_MASK                                                        0x00000002L
+#define GFX_IMU_RLC_MSG_STATUS__RLC2IMU_MSGDONE_MASK                                                          0x00010000L
+#define GFX_IMU_RLC_MSG_STATUS__RLC2IMU_CHGTOG_MASK                                                           0x40000000L
+#define GFX_IMU_RLC_MSG_STATUS__RLC2IMU_DONETOG_MASK                                                          0x80000000L
+//RLC_GFX_IMU_DATA_0
+#define RLC_GFX_IMU_DATA_0__DATA__SHIFT                                                                       0x0
+#define RLC_GFX_IMU_DATA_0__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GFX_IMU_CMD
+#define RLC_GFX_IMU_CMD__CMD__SHIFT                                                                           0x0
+#define RLC_GFX_IMU_CMD__CMD_MASK                                                                             0xFFFFFFFFL
+//GFX_IMU_RLC_STATUS
+#define GFX_IMU_RLC_STATUS__PD_ACTIVE__SHIFT                                                                  0x0
+#define GFX_IMU_RLC_STATUS__RLC_ALIVE__SHIFT                                                                  0x1
+#define GFX_IMU_RLC_STATUS__TBD2__SHIFT                                                                       0x2
+#define GFX_IMU_RLC_STATUS__TBD3__SHIFT                                                                       0x3
+#define GFX_IMU_RLC_STATUS__PD_ACTIVE_MASK                                                                    0x00000001L
+#define GFX_IMU_RLC_STATUS__RLC_ALIVE_MASK                                                                    0x00000002L
+#define GFX_IMU_RLC_STATUS__TBD2_MASK                                                                         0x00000004L
+#define GFX_IMU_RLC_STATUS__TBD3_MASK                                                                         0x00000008L
+//GFX_IMU_STATUS
+#define GFX_IMU_STATUS__ALLOW_GFXOFF__SHIFT                                                                   0x0
+#define GFX_IMU_STATUS__ALLOW_FA_DCS__SHIFT                                                                   0x1
+#define GFX_IMU_STATUS__TBD4__SHIFT                                                                           0x4
+#define GFX_IMU_STATUS__TBD5__SHIFT                                                                           0x5
+#define GFX_IMU_STATUS__TBD6__SHIFT                                                                           0x6
+#define GFX_IMU_STATUS__TBD7__SHIFT                                                                           0x7
+#define GFX_IMU_STATUS__TBD8__SHIFT                                                                           0x8
+#define GFX_IMU_STATUS__TBD9__SHIFT                                                                           0x9
+#define GFX_IMU_STATUS__TBD10__SHIFT                                                                          0xa
+#define GFX_IMU_STATUS__TBD11__SHIFT                                                                          0xb
+#define GFX_IMU_STATUS__TBD12__SHIFT                                                                          0xc
+#define GFX_IMU_STATUS__TBD13__SHIFT                                                                          0xd
+#define GFX_IMU_STATUS__TBD14__SHIFT                                                                          0xe
+#define GFX_IMU_STATUS__DISABLE_GFXCLK_DS__SHIFT                                                              0xf
+#define GFX_IMU_STATUS__ALLOW_GFXOFF_MASK                                                                     0x00000001L
+#define GFX_IMU_STATUS__ALLOW_FA_DCS_MASK                                                                     0x00000002L
+#define GFX_IMU_STATUS__TBD4_MASK                                                                             0x00000010L
+#define GFX_IMU_STATUS__TBD5_MASK                                                                             0x00000020L
+#define GFX_IMU_STATUS__TBD6_MASK                                                                             0x00000040L
+#define GFX_IMU_STATUS__TBD7_MASK                                                                             0x00000080L
+#define GFX_IMU_STATUS__TBD8_MASK                                                                             0x00000100L
+#define GFX_IMU_STATUS__TBD9_MASK                                                                             0x00000200L
+#define GFX_IMU_STATUS__TBD10_MASK                                                                            0x00000400L
+#define GFX_IMU_STATUS__TBD11_MASK                                                                            0x00000800L
+#define GFX_IMU_STATUS__TBD12_MASK                                                                            0x00001000L
+#define GFX_IMU_STATUS__TBD13_MASK                                                                            0x00002000L
+#define GFX_IMU_STATUS__TBD14_MASK                                                                            0x00004000L
+#define GFX_IMU_STATUS__DISABLE_GFXCLK_DS_MASK                                                                0x00008000L
+//GFX_IMU_SOC_DATA
+#define GFX_IMU_SOC_DATA__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_SOC_DATA__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_SOC_ADDR
+#define GFX_IMU_SOC_ADDR__ADDR__SHIFT                                                                         0x0
+#define GFX_IMU_SOC_ADDR__ADDR_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_SOC_REQ
+#define GFX_IMU_SOC_REQ__REQ_BUSY__SHIFT                                                                      0x0
+#define GFX_IMU_SOC_REQ__R_W__SHIFT                                                                           0x1
+#define GFX_IMU_SOC_REQ__ERR__SHIFT                                                                           0x1f
+#define GFX_IMU_SOC_REQ__REQ_BUSY_MASK                                                                        0x00000001L
+#define GFX_IMU_SOC_REQ__R_W_MASK                                                                             0x00000002L
+#define GFX_IMU_SOC_REQ__ERR_MASK                                                                             0x80000000L
+//GFX_IMU_VF_CTRL
+#define GFX_IMU_VF_CTRL__VF__SHIFT                                                                            0x0
+#define GFX_IMU_VF_CTRL__VFID__SHIFT                                                                          0x1
+#define GFX_IMU_VF_CTRL__QOS__SHIFT                                                                           0x7
+#define GFX_IMU_VF_CTRL__VF_MASK                                                                              0x00000001L
+#define GFX_IMU_VF_CTRL__VFID_MASK                                                                            0x0000007EL
+#define GFX_IMU_VF_CTRL__QOS_MASK                                                                             0x00000780L
+//GFX_IMU_SCRATCH_0
+#define GFX_IMU_SCRATCH_0__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_0__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_1
+#define GFX_IMU_SCRATCH_1__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_1__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_2
+#define GFX_IMU_SCRATCH_2__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_2__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_3
+#define GFX_IMU_SCRATCH_3__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_3__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_4
+#define GFX_IMU_SCRATCH_4__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_4__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_5
+#define GFX_IMU_SCRATCH_5__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_5__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_6
+#define GFX_IMU_SCRATCH_6__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_6__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_7
+#define GFX_IMU_SCRATCH_7__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_7__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_8
+#define GFX_IMU_SCRATCH_8__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_8__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_9
+#define GFX_IMU_SCRATCH_9__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_9__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_10
+#define GFX_IMU_SCRATCH_10__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_SCRATCH_10__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_SCRATCH_11
+#define GFX_IMU_SCRATCH_11__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_SCRATCH_11__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_SCRATCH_12
+#define GFX_IMU_SCRATCH_12__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_SCRATCH_12__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_SCRATCH_13
+#define GFX_IMU_SCRATCH_13__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_SCRATCH_13__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_SCRATCH_14
+#define GFX_IMU_SCRATCH_14__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_SCRATCH_14__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_SCRATCH_15
+#define GFX_IMU_SCRATCH_15__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_SCRATCH_15__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_FW_GTS_LO
+#define GFX_IMU_FW_GTS_LO__TSTAMP_LO__SHIFT                                                                   0x0
+#define GFX_IMU_FW_GTS_LO__TSTAMP_LO_MASK                                                                     0xFFFFFFFFL
+//GFX_IMU_FW_GTS_HI
+#define GFX_IMU_FW_GTS_HI__TSTAMP_HI__SHIFT                                                                   0x0
+#define GFX_IMU_FW_GTS_HI__TSTAMP_HI_MASK                                                                     0x00FFFFFFL
+//GFX_IMU_GTS_OFFSET_LO
+#define GFX_IMU_GTS_OFFSET_LO__GTS_OFFSET_LO__SHIFT                                                           0x0
+#define GFX_IMU_GTS_OFFSET_LO__GTS_OFFSET_LO_MASK                                                             0xFFFFFFFFL
+//GFX_IMU_GTS_OFFSET_HI
+#define GFX_IMU_GTS_OFFSET_HI__GTS_OFFSET_HI__SHIFT                                                           0x0
+#define GFX_IMU_GTS_OFFSET_HI__GTS_OFFSET_HI_MASK                                                             0x00FFFFFFL
+//GFX_IMU_RLC_GTS_OFFSET_LO
+#define GFX_IMU_RLC_GTS_OFFSET_LO__GTS_OFFSET_LO__SHIFT                                                       0x0
+#define GFX_IMU_RLC_GTS_OFFSET_LO__GTS_OFFSET_LO_MASK                                                         0xFFFFFFFFL
+//GFX_IMU_RLC_GTS_OFFSET_HI
+#define GFX_IMU_RLC_GTS_OFFSET_HI__GTS_OFFSET_HI__SHIFT                                                       0x0
+#define GFX_IMU_RLC_GTS_OFFSET_HI__GTS_OFFSET_HI_MASK                                                         0x00FFFFFFL
+//GFX_IMU_CORE_INT_STATUS
+#define GFX_IMU_CORE_INT_STATUS__INTERRUPT_24__SHIFT                                                          0x18
+#define GFX_IMU_CORE_INT_STATUS__INTERRUPT_25__SHIFT                                                          0x19
+#define GFX_IMU_CORE_INT_STATUS__INTERRUPT_29__SHIFT                                                          0x1d
+#define GFX_IMU_CORE_INT_STATUS__INTERRUPT_24_MASK                                                            0x01000000L
+#define GFX_IMU_CORE_INT_STATUS__INTERRUPT_25_MASK                                                            0x02000000L
+#define GFX_IMU_CORE_INT_STATUS__INTERRUPT_29_MASK                                                            0x20000000L
+//GFX_IMU_PIC_INT_MASK
+#define GFX_IMU_PIC_INT_MASK__MASK_0__SHIFT                                                                   0x0
+#define GFX_IMU_PIC_INT_MASK__MASK_1__SHIFT                                                                   0x1
+#define GFX_IMU_PIC_INT_MASK__MASK_2__SHIFT                                                                   0x2
+#define GFX_IMU_PIC_INT_MASK__MASK_3__SHIFT                                                                   0x3
+#define GFX_IMU_PIC_INT_MASK__MASK_4__SHIFT                                                                   0x4
+#define GFX_IMU_PIC_INT_MASK__MASK_5__SHIFT                                                                   0x5
+#define GFX_IMU_PIC_INT_MASK__MASK_6__SHIFT                                                                   0x6
+#define GFX_IMU_PIC_INT_MASK__MASK_7__SHIFT                                                                   0x7
+#define GFX_IMU_PIC_INT_MASK__MASK_8__SHIFT                                                                   0x8
+#define GFX_IMU_PIC_INT_MASK__MASK_9__SHIFT                                                                   0x9
+#define GFX_IMU_PIC_INT_MASK__MASK_10__SHIFT                                                                  0xa
+#define GFX_IMU_PIC_INT_MASK__MASK_11__SHIFT                                                                  0xb
+#define GFX_IMU_PIC_INT_MASK__MASK_12__SHIFT                                                                  0xc
+#define GFX_IMU_PIC_INT_MASK__MASK_13__SHIFT                                                                  0xd
+#define GFX_IMU_PIC_INT_MASK__MASK_14__SHIFT                                                                  0xe
+#define GFX_IMU_PIC_INT_MASK__MASK_15__SHIFT                                                                  0xf
+#define GFX_IMU_PIC_INT_MASK__MASK_16__SHIFT                                                                  0x10
+#define GFX_IMU_PIC_INT_MASK__MASK_17__SHIFT                                                                  0x11
+#define GFX_IMU_PIC_INT_MASK__MASK_18__SHIFT                                                                  0x12
+#define GFX_IMU_PIC_INT_MASK__MASK_19__SHIFT                                                                  0x13
+#define GFX_IMU_PIC_INT_MASK__MASK_20__SHIFT                                                                  0x14
+#define GFX_IMU_PIC_INT_MASK__MASK_21__SHIFT                                                                  0x15
+#define GFX_IMU_PIC_INT_MASK__MASK_22__SHIFT                                                                  0x16
+#define GFX_IMU_PIC_INT_MASK__MASK_23__SHIFT                                                                  0x17
+#define GFX_IMU_PIC_INT_MASK__MASK_24__SHIFT                                                                  0x18
+#define GFX_IMU_PIC_INT_MASK__MASK_25__SHIFT                                                                  0x19
+#define GFX_IMU_PIC_INT_MASK__MASK_26__SHIFT                                                                  0x1a
+#define GFX_IMU_PIC_INT_MASK__MASK_27__SHIFT                                                                  0x1b
+#define GFX_IMU_PIC_INT_MASK__MASK_28__SHIFT                                                                  0x1c
+#define GFX_IMU_PIC_INT_MASK__MASK_29__SHIFT                                                                  0x1d
+#define GFX_IMU_PIC_INT_MASK__MASK_30__SHIFT                                                                  0x1e
+#define GFX_IMU_PIC_INT_MASK__MASK_31__SHIFT                                                                  0x1f
+#define GFX_IMU_PIC_INT_MASK__MASK_0_MASK                                                                     0x00000001L
+#define GFX_IMU_PIC_INT_MASK__MASK_1_MASK                                                                     0x00000002L
+#define GFX_IMU_PIC_INT_MASK__MASK_2_MASK                                                                     0x00000004L
+#define GFX_IMU_PIC_INT_MASK__MASK_3_MASK                                                                     0x00000008L
+#define GFX_IMU_PIC_INT_MASK__MASK_4_MASK                                                                     0x00000010L
+#define GFX_IMU_PIC_INT_MASK__MASK_5_MASK                                                                     0x00000020L
+#define GFX_IMU_PIC_INT_MASK__MASK_6_MASK                                                                     0x00000040L
+#define GFX_IMU_PIC_INT_MASK__MASK_7_MASK                                                                     0x00000080L
+#define GFX_IMU_PIC_INT_MASK__MASK_8_MASK                                                                     0x00000100L
+#define GFX_IMU_PIC_INT_MASK__MASK_9_MASK                                                                     0x00000200L
+#define GFX_IMU_PIC_INT_MASK__MASK_10_MASK                                                                    0x00000400L
+#define GFX_IMU_PIC_INT_MASK__MASK_11_MASK                                                                    0x00000800L
+#define GFX_IMU_PIC_INT_MASK__MASK_12_MASK                                                                    0x00001000L
+#define GFX_IMU_PIC_INT_MASK__MASK_13_MASK                                                                    0x00002000L
+#define GFX_IMU_PIC_INT_MASK__MASK_14_MASK                                                                    0x00004000L
+#define GFX_IMU_PIC_INT_MASK__MASK_15_MASK                                                                    0x00008000L
+#define GFX_IMU_PIC_INT_MASK__MASK_16_MASK                                                                    0x00010000L
+#define GFX_IMU_PIC_INT_MASK__MASK_17_MASK                                                                    0x00020000L
+#define GFX_IMU_PIC_INT_MASK__MASK_18_MASK                                                                    0x00040000L
+#define GFX_IMU_PIC_INT_MASK__MASK_19_MASK                                                                    0x00080000L
+#define GFX_IMU_PIC_INT_MASK__MASK_20_MASK                                                                    0x00100000L
+#define GFX_IMU_PIC_INT_MASK__MASK_21_MASK                                                                    0x00200000L
+#define GFX_IMU_PIC_INT_MASK__MASK_22_MASK                                                                    0x00400000L
+#define GFX_IMU_PIC_INT_MASK__MASK_23_MASK                                                                    0x00800000L
+#define GFX_IMU_PIC_INT_MASK__MASK_24_MASK                                                                    0x01000000L
+#define GFX_IMU_PIC_INT_MASK__MASK_25_MASK                                                                    0x02000000L
+#define GFX_IMU_PIC_INT_MASK__MASK_26_MASK                                                                    0x04000000L
+#define GFX_IMU_PIC_INT_MASK__MASK_27_MASK                                                                    0x08000000L
+#define GFX_IMU_PIC_INT_MASK__MASK_28_MASK                                                                    0x10000000L
+#define GFX_IMU_PIC_INT_MASK__MASK_29_MASK                                                                    0x20000000L
+#define GFX_IMU_PIC_INT_MASK__MASK_30_MASK                                                                    0x40000000L
+#define GFX_IMU_PIC_INT_MASK__MASK_31_MASK                                                                    0x80000000L
+//GFX_IMU_PIC_INT_LVL
+#define GFX_IMU_PIC_INT_LVL__LVL_0__SHIFT                                                                     0x0
+#define GFX_IMU_PIC_INT_LVL__LVL_1__SHIFT                                                                     0x1
+#define GFX_IMU_PIC_INT_LVL__LVL_2__SHIFT                                                                     0x2
+#define GFX_IMU_PIC_INT_LVL__LVL_3__SHIFT                                                                     0x3
+#define GFX_IMU_PIC_INT_LVL__LVL_4__SHIFT                                                                     0x4
+#define GFX_IMU_PIC_INT_LVL__LVL_5__SHIFT                                                                     0x5
+#define GFX_IMU_PIC_INT_LVL__LVL_6__SHIFT                                                                     0x6
+#define GFX_IMU_PIC_INT_LVL__LVL_7__SHIFT                                                                     0x7
+#define GFX_IMU_PIC_INT_LVL__LVL_8__SHIFT                                                                     0x8
+#define GFX_IMU_PIC_INT_LVL__LVL_9__SHIFT                                                                     0x9
+#define GFX_IMU_PIC_INT_LVL__LVL_10__SHIFT                                                                    0xa
+#define GFX_IMU_PIC_INT_LVL__LVL_11__SHIFT                                                                    0xb
+#define GFX_IMU_PIC_INT_LVL__LVL_12__SHIFT                                                                    0xc
+#define GFX_IMU_PIC_INT_LVL__LVL_13__SHIFT                                                                    0xd
+#define GFX_IMU_PIC_INT_LVL__LVL_14__SHIFT                                                                    0xe
+#define GFX_IMU_PIC_INT_LVL__LVL_15__SHIFT                                                                    0xf
+#define GFX_IMU_PIC_INT_LVL__LVL_16__SHIFT                                                                    0x10
+#define GFX_IMU_PIC_INT_LVL__LVL_17__SHIFT                                                                    0x11
+#define GFX_IMU_PIC_INT_LVL__LVL_18__SHIFT                                                                    0x12
+#define GFX_IMU_PIC_INT_LVL__LVL_19__SHIFT                                                                    0x13
+#define GFX_IMU_PIC_INT_LVL__LVL_20__SHIFT                                                                    0x14
+#define GFX_IMU_PIC_INT_LVL__LVL_21__SHIFT                                                                    0x15
+#define GFX_IMU_PIC_INT_LVL__LVL_22__SHIFT                                                                    0x16
+#define GFX_IMU_PIC_INT_LVL__LVL_23__SHIFT                                                                    0x17
+#define GFX_IMU_PIC_INT_LVL__LVL_24__SHIFT                                                                    0x18
+#define GFX_IMU_PIC_INT_LVL__LVL_25__SHIFT                                                                    0x19
+#define GFX_IMU_PIC_INT_LVL__LVL_26__SHIFT                                                                    0x1a
+#define GFX_IMU_PIC_INT_LVL__LVL_27__SHIFT                                                                    0x1b
+#define GFX_IMU_PIC_INT_LVL__LVL_28__SHIFT                                                                    0x1c
+#define GFX_IMU_PIC_INT_LVL__LVL_29__SHIFT                                                                    0x1d
+#define GFX_IMU_PIC_INT_LVL__LVL_30__SHIFT                                                                    0x1e
+#define GFX_IMU_PIC_INT_LVL__LVL_31__SHIFT                                                                    0x1f
+#define GFX_IMU_PIC_INT_LVL__LVL_0_MASK                                                                       0x00000001L
+#define GFX_IMU_PIC_INT_LVL__LVL_1_MASK                                                                       0x00000002L
+#define GFX_IMU_PIC_INT_LVL__LVL_2_MASK                                                                       0x00000004L
+#define GFX_IMU_PIC_INT_LVL__LVL_3_MASK                                                                       0x00000008L
+#define GFX_IMU_PIC_INT_LVL__LVL_4_MASK                                                                       0x00000010L
+#define GFX_IMU_PIC_INT_LVL__LVL_5_MASK                                                                       0x00000020L
+#define GFX_IMU_PIC_INT_LVL__LVL_6_MASK                                                                       0x00000040L
+#define GFX_IMU_PIC_INT_LVL__LVL_7_MASK                                                                       0x00000080L
+#define GFX_IMU_PIC_INT_LVL__LVL_8_MASK                                                                       0x00000100L
+#define GFX_IMU_PIC_INT_LVL__LVL_9_MASK                                                                       0x00000200L
+#define GFX_IMU_PIC_INT_LVL__LVL_10_MASK                                                                      0x00000400L
+#define GFX_IMU_PIC_INT_LVL__LVL_11_MASK                                                                      0x00000800L
+#define GFX_IMU_PIC_INT_LVL__LVL_12_MASK                                                                      0x00001000L
+#define GFX_IMU_PIC_INT_LVL__LVL_13_MASK                                                                      0x00002000L
+#define GFX_IMU_PIC_INT_LVL__LVL_14_MASK                                                                      0x00004000L
+#define GFX_IMU_PIC_INT_LVL__LVL_15_MASK                                                                      0x00008000L
+#define GFX_IMU_PIC_INT_LVL__LVL_16_MASK                                                                      0x00010000L
+#define GFX_IMU_PIC_INT_LVL__LVL_17_MASK                                                                      0x00020000L
+#define GFX_IMU_PIC_INT_LVL__LVL_18_MASK                                                                      0x00040000L
+#define GFX_IMU_PIC_INT_LVL__LVL_19_MASK                                                                      0x00080000L
+#define GFX_IMU_PIC_INT_LVL__LVL_20_MASK                                                                      0x00100000L
+#define GFX_IMU_PIC_INT_LVL__LVL_21_MASK                                                                      0x00200000L
+#define GFX_IMU_PIC_INT_LVL__LVL_22_MASK                                                                      0x00400000L
+#define GFX_IMU_PIC_INT_LVL__LVL_23_MASK                                                                      0x00800000L
+#define GFX_IMU_PIC_INT_LVL__LVL_24_MASK                                                                      0x01000000L
+#define GFX_IMU_PIC_INT_LVL__LVL_25_MASK                                                                      0x02000000L
+#define GFX_IMU_PIC_INT_LVL__LVL_26_MASK                                                                      0x04000000L
+#define GFX_IMU_PIC_INT_LVL__LVL_27_MASK                                                                      0x08000000L
+#define GFX_IMU_PIC_INT_LVL__LVL_28_MASK                                                                      0x10000000L
+#define GFX_IMU_PIC_INT_LVL__LVL_29_MASK                                                                      0x20000000L
+#define GFX_IMU_PIC_INT_LVL__LVL_30_MASK                                                                      0x40000000L
+#define GFX_IMU_PIC_INT_LVL__LVL_31_MASK                                                                      0x80000000L
+//GFX_IMU_PIC_INT_EDGE
+#define GFX_IMU_PIC_INT_EDGE__EDGE_0__SHIFT                                                                   0x0
+#define GFX_IMU_PIC_INT_EDGE__EDGE_1__SHIFT                                                                   0x1
+#define GFX_IMU_PIC_INT_EDGE__EDGE_2__SHIFT                                                                   0x2
+#define GFX_IMU_PIC_INT_EDGE__EDGE_3__SHIFT                                                                   0x3
+#define GFX_IMU_PIC_INT_EDGE__EDGE_4__SHIFT                                                                   0x4
+#define GFX_IMU_PIC_INT_EDGE__EDGE_5__SHIFT                                                                   0x5
+#define GFX_IMU_PIC_INT_EDGE__EDGE_6__SHIFT                                                                   0x6
+#define GFX_IMU_PIC_INT_EDGE__EDGE_7__SHIFT                                                                   0x7
+#define GFX_IMU_PIC_INT_EDGE__EDGE_8__SHIFT                                                                   0x8
+#define GFX_IMU_PIC_INT_EDGE__EDGE_9__SHIFT                                                                   0x9
+#define GFX_IMU_PIC_INT_EDGE__EDGE_10__SHIFT                                                                  0xa
+#define GFX_IMU_PIC_INT_EDGE__EDGE_11__SHIFT                                                                  0xb
+#define GFX_IMU_PIC_INT_EDGE__EDGE_12__SHIFT                                                                  0xc
+#define GFX_IMU_PIC_INT_EDGE__EDGE_13__SHIFT                                                                  0xd
+#define GFX_IMU_PIC_INT_EDGE__EDGE_14__SHIFT                                                                  0xe
+#define GFX_IMU_PIC_INT_EDGE__EDGE_15__SHIFT                                                                  0xf
+#define GFX_IMU_PIC_INT_EDGE__EDGE_16__SHIFT                                                                  0x10
+#define GFX_IMU_PIC_INT_EDGE__EDGE_17__SHIFT                                                                  0x11
+#define GFX_IMU_PIC_INT_EDGE__EDGE_18__SHIFT                                                                  0x12
+#define GFX_IMU_PIC_INT_EDGE__EDGE_19__SHIFT                                                                  0x13
+#define GFX_IMU_PIC_INT_EDGE__EDGE_20__SHIFT                                                                  0x14
+#define GFX_IMU_PIC_INT_EDGE__EDGE_21__SHIFT                                                                  0x15
+#define GFX_IMU_PIC_INT_EDGE__EDGE_22__SHIFT                                                                  0x16
+#define GFX_IMU_PIC_INT_EDGE__EDGE_23__SHIFT                                                                  0x17
+#define GFX_IMU_PIC_INT_EDGE__EDGE_24__SHIFT                                                                  0x18
+#define GFX_IMU_PIC_INT_EDGE__EDGE_25__SHIFT                                                                  0x19
+#define GFX_IMU_PIC_INT_EDGE__EDGE_26__SHIFT                                                                  0x1a
+#define GFX_IMU_PIC_INT_EDGE__EDGE_27__SHIFT                                                                  0x1b
+#define GFX_IMU_PIC_INT_EDGE__EDGE_28__SHIFT                                                                  0x1c
+#define GFX_IMU_PIC_INT_EDGE__EDGE_29__SHIFT                                                                  0x1d
+#define GFX_IMU_PIC_INT_EDGE__EDGE_30__SHIFT                                                                  0x1e
+#define GFX_IMU_PIC_INT_EDGE__EDGE_31__SHIFT                                                                  0x1f
+#define GFX_IMU_PIC_INT_EDGE__EDGE_0_MASK                                                                     0x00000001L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_1_MASK                                                                     0x00000002L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_2_MASK                                                                     0x00000004L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_3_MASK                                                                     0x00000008L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_4_MASK                                                                     0x00000010L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_5_MASK                                                                     0x00000020L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_6_MASK                                                                     0x00000040L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_7_MASK                                                                     0x00000080L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_8_MASK                                                                     0x00000100L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_9_MASK                                                                     0x00000200L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_10_MASK                                                                    0x00000400L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_11_MASK                                                                    0x00000800L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_12_MASK                                                                    0x00001000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_13_MASK                                                                    0x00002000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_14_MASK                                                                    0x00004000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_15_MASK                                                                    0x00008000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_16_MASK                                                                    0x00010000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_17_MASK                                                                    0x00020000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_18_MASK                                                                    0x00040000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_19_MASK                                                                    0x00080000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_20_MASK                                                                    0x00100000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_21_MASK                                                                    0x00200000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_22_MASK                                                                    0x00400000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_23_MASK                                                                    0x00800000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_24_MASK                                                                    0x01000000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_25_MASK                                                                    0x02000000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_26_MASK                                                                    0x04000000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_27_MASK                                                                    0x08000000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_28_MASK                                                                    0x10000000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_29_MASK                                                                    0x20000000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_30_MASK                                                                    0x40000000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_31_MASK                                                                    0x80000000L
+//GFX_IMU_PIC_INT_PRI_0
+#define GFX_IMU_PIC_INT_PRI_0__PRI_0__SHIFT                                                                   0x0
+#define GFX_IMU_PIC_INT_PRI_0__PRI_1__SHIFT                                                                   0x8
+#define GFX_IMU_PIC_INT_PRI_0__PRI_2__SHIFT                                                                   0x10
+#define GFX_IMU_PIC_INT_PRI_0__PRI_3__SHIFT                                                                   0x18
+#define GFX_IMU_PIC_INT_PRI_0__PRI_0_MASK                                                                     0x000000FFL
+#define GFX_IMU_PIC_INT_PRI_0__PRI_1_MASK                                                                     0x0000FF00L
+#define GFX_IMU_PIC_INT_PRI_0__PRI_2_MASK                                                                     0x00FF0000L
+#define GFX_IMU_PIC_INT_PRI_0__PRI_3_MASK                                                                     0xFF000000L
+//GFX_IMU_PIC_INT_PRI_1
+#define GFX_IMU_PIC_INT_PRI_1__PRI_4__SHIFT                                                                   0x0
+#define GFX_IMU_PIC_INT_PRI_1__PRI_5__SHIFT                                                                   0x8
+#define GFX_IMU_PIC_INT_PRI_1__PRI_6__SHIFT                                                                   0x10
+#define GFX_IMU_PIC_INT_PRI_1__PRI_7__SHIFT                                                                   0x18
+#define GFX_IMU_PIC_INT_PRI_1__PRI_4_MASK                                                                     0x000000FFL
+#define GFX_IMU_PIC_INT_PRI_1__PRI_5_MASK                                                                     0x0000FF00L
+#define GFX_IMU_PIC_INT_PRI_1__PRI_6_MASK                                                                     0x00FF0000L
+#define GFX_IMU_PIC_INT_PRI_1__PRI_7_MASK                                                                     0xFF000000L
+//GFX_IMU_PIC_INT_PRI_2
+#define GFX_IMU_PIC_INT_PRI_2__PRI_8__SHIFT                                                                   0x0
+#define GFX_IMU_PIC_INT_PRI_2__PRI_9__SHIFT                                                                   0x8
+#define GFX_IMU_PIC_INT_PRI_2__PRI_10__SHIFT                                                                  0x10
+#define GFX_IMU_PIC_INT_PRI_2__PRI_11__SHIFT                                                                  0x18
+#define GFX_IMU_PIC_INT_PRI_2__PRI_8_MASK                                                                     0x000000FFL
+#define GFX_IMU_PIC_INT_PRI_2__PRI_9_MASK                                                                     0x0000FF00L
+#define GFX_IMU_PIC_INT_PRI_2__PRI_10_MASK                                                                    0x00FF0000L
+#define GFX_IMU_PIC_INT_PRI_2__PRI_11_MASK                                                                    0xFF000000L
+//GFX_IMU_PIC_INT_PRI_3
+#define GFX_IMU_PIC_INT_PRI_3__PRI_12__SHIFT                                                                  0x0
+#define GFX_IMU_PIC_INT_PRI_3__PRI_13__SHIFT                                                                  0x8
+#define GFX_IMU_PIC_INT_PRI_3__PRI_14__SHIFT                                                                  0x10
+#define GFX_IMU_PIC_INT_PRI_3__PRI_15__SHIFT                                                                  0x18
+#define GFX_IMU_PIC_INT_PRI_3__PRI_12_MASK                                                                    0x000000FFL
+#define GFX_IMU_PIC_INT_PRI_3__PRI_13_MASK                                                                    0x0000FF00L
+#define GFX_IMU_PIC_INT_PRI_3__PRI_14_MASK                                                                    0x00FF0000L
+#define GFX_IMU_PIC_INT_PRI_3__PRI_15_MASK                                                                    0xFF000000L
+//GFX_IMU_PIC_INT_PRI_4
+#define GFX_IMU_PIC_INT_PRI_4__PRI_16__SHIFT                                                                  0x0
+#define GFX_IMU_PIC_INT_PRI_4__PRI_17__SHIFT                                                                  0x8
+#define GFX_IMU_PIC_INT_PRI_4__PRI_18__SHIFT                                                                  0x10
+#define GFX_IMU_PIC_INT_PRI_4__PRI_19__SHIFT                                                                  0x18
+#define GFX_IMU_PIC_INT_PRI_4__PRI_16_MASK                                                                    0x000000FFL
+#define GFX_IMU_PIC_INT_PRI_4__PRI_17_MASK                                                                    0x0000FF00L
+#define GFX_IMU_PIC_INT_PRI_4__PRI_18_MASK                                                                    0x00FF0000L
+#define GFX_IMU_PIC_INT_PRI_4__PRI_19_MASK                                                                    0xFF000000L
+//GFX_IMU_PIC_INT_PRI_5
+#define GFX_IMU_PIC_INT_PRI_5__PRI_20__SHIFT                                                                  0x0
+#define GFX_IMU_PIC_INT_PRI_5__PRI_21__SHIFT                                                                  0x8
+#define GFX_IMU_PIC_INT_PRI_5__PRI_22__SHIFT                                                                  0x10
+#define GFX_IMU_PIC_INT_PRI_5__PRI_23__SHIFT                                                                  0x18
+#define GFX_IMU_PIC_INT_PRI_5__PRI_20_MASK                                                                    0x000000FFL
+#define GFX_IMU_PIC_INT_PRI_5__PRI_21_MASK                                                                    0x0000FF00L
+#define GFX_IMU_PIC_INT_PRI_5__PRI_22_MASK                                                                    0x00FF0000L
+#define GFX_IMU_PIC_INT_PRI_5__PRI_23_MASK                                                                    0xFF000000L
+//GFX_IMU_PIC_INT_PRI_6
+#define GFX_IMU_PIC_INT_PRI_6__PRI_24__SHIFT                                                                  0x0
+#define GFX_IMU_PIC_INT_PRI_6__PRI_25__SHIFT                                                                  0x8
+#define GFX_IMU_PIC_INT_PRI_6__PRI_26__SHIFT                                                                  0x10
+#define GFX_IMU_PIC_INT_PRI_6__PRI_27__SHIFT                                                                  0x18
+#define GFX_IMU_PIC_INT_PRI_6__PRI_24_MASK                                                                    0x000000FFL
+#define GFX_IMU_PIC_INT_PRI_6__PRI_25_MASK                                                                    0x0000FF00L
+#define GFX_IMU_PIC_INT_PRI_6__PRI_26_MASK                                                                    0x00FF0000L
+#define GFX_IMU_PIC_INT_PRI_6__PRI_27_MASK                                                                    0xFF000000L
+//GFX_IMU_PIC_INT_PRI_7
+#define GFX_IMU_PIC_INT_PRI_7__PRI_28__SHIFT                                                                  0x0
+#define GFX_IMU_PIC_INT_PRI_7__PRI_29__SHIFT                                                                  0x8
+#define GFX_IMU_PIC_INT_PRI_7__PRI_30__SHIFT                                                                  0x10
+#define GFX_IMU_PIC_INT_PRI_7__PRI_31__SHIFT                                                                  0x18
+#define GFX_IMU_PIC_INT_PRI_7__PRI_28_MASK                                                                    0x000000FFL
+#define GFX_IMU_PIC_INT_PRI_7__PRI_29_MASK                                                                    0x0000FF00L
+#define GFX_IMU_PIC_INT_PRI_7__PRI_30_MASK                                                                    0x00FF0000L
+#define GFX_IMU_PIC_INT_PRI_7__PRI_31_MASK                                                                    0xFF000000L
+//GFX_IMU_PIC_INT_STATUS
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS0__SHIFT                                                            0x0
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS1__SHIFT                                                            0x1
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS2__SHIFT                                                            0x2
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS3__SHIFT                                                            0x3
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS4__SHIFT                                                            0x4
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS5__SHIFT                                                            0x5
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS6__SHIFT                                                            0x6
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS7__SHIFT                                                            0x7
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS8__SHIFT                                                            0x8
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS9__SHIFT                                                            0x9
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS10__SHIFT                                                           0xa
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS11__SHIFT                                                           0xb
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS12__SHIFT                                                           0xc
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS13__SHIFT                                                           0xd
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS14__SHIFT                                                           0xe
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS15__SHIFT                                                           0xf
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS16__SHIFT                                                           0x10
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS17__SHIFT                                                           0x11
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS18__SHIFT                                                           0x12
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS19__SHIFT                                                           0x13
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS20__SHIFT                                                           0x14
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS21__SHIFT                                                           0x15
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS22__SHIFT                                                           0x16
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS23__SHIFT                                                           0x17
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS24__SHIFT                                                           0x18
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS25__SHIFT                                                           0x19
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS26__SHIFT                                                           0x1a
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS27__SHIFT                                                           0x1b
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS28__SHIFT                                                           0x1c
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS29__SHIFT                                                           0x1d
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS30__SHIFT                                                           0x1e
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS31__SHIFT                                                           0x1f
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS0_MASK                                                              0x00000001L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS1_MASK                                                              0x00000002L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS2_MASK                                                              0x00000004L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS3_MASK                                                              0x00000008L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS4_MASK                                                              0x00000010L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS5_MASK                                                              0x00000020L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS6_MASK                                                              0x00000040L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS7_MASK                                                              0x00000080L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS8_MASK                                                              0x00000100L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS9_MASK                                                              0x00000200L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS10_MASK                                                             0x00000400L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS11_MASK                                                             0x00000800L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS12_MASK                                                             0x00001000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS13_MASK                                                             0x00002000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS14_MASK                                                             0x00004000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS15_MASK                                                             0x00008000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS16_MASK                                                             0x00010000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS17_MASK                                                             0x00020000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS18_MASK                                                             0x00040000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS19_MASK                                                             0x00080000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS20_MASK                                                             0x00100000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS21_MASK                                                             0x00200000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS22_MASK                                                             0x00400000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS23_MASK                                                             0x00800000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS24_MASK                                                             0x01000000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS25_MASK                                                             0x02000000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS26_MASK                                                             0x04000000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS27_MASK                                                             0x08000000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS28_MASK                                                             0x10000000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS29_MASK                                                             0x20000000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS30_MASK                                                             0x40000000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS31_MASK                                                             0x80000000L
+//GFX_IMU_PIC_INTR
+#define GFX_IMU_PIC_INTR__INTR_n__SHIFT                                                                       0x0
+#define GFX_IMU_PIC_INTR__INTR_n_MASK                                                                         0x00000001L
+//GFX_IMU_PIC_INTR_ID
+#define GFX_IMU_PIC_INTR_ID__INTR_n__SHIFT                                                                    0x0
+#define GFX_IMU_PIC_INTR_ID__INTR_n_MASK                                                                      0x000000FFL
+//GFX_IMU_IH_CTRL_1
+#define GFX_IMU_IH_CTRL_1__CONTEXT_ID__SHIFT                                                                  0x0
+#define GFX_IMU_IH_CTRL_1__CONTEXT_ID_MASK                                                                    0xFFFFFFFFL
+//GFX_IMU_IH_CTRL_2
+#define GFX_IMU_IH_CTRL_2__CONTEXT_ID__SHIFT                                                                  0x0
+#define GFX_IMU_IH_CTRL_2__RING_ID__SHIFT                                                                     0x8
+#define GFX_IMU_IH_CTRL_2__VM_ID__SHIFT                                                                       0x10
+#define GFX_IMU_IH_CTRL_2__SRSTB__SHIFT                                                                       0x1f
+#define GFX_IMU_IH_CTRL_2__CONTEXT_ID_MASK                                                                    0x000000FFL
+#define GFX_IMU_IH_CTRL_2__RING_ID_MASK                                                                       0x0000FF00L
+#define GFX_IMU_IH_CTRL_2__VM_ID_MASK                                                                         0x000F0000L
+#define GFX_IMU_IH_CTRL_2__SRSTB_MASK                                                                         0x80000000L
+//GFX_IMU_IH_CTRL_3
+#define GFX_IMU_IH_CTRL_3__SOURCE_ID__SHIFT                                                                   0x0
+#define GFX_IMU_IH_CTRL_3__VF_ID__SHIFT                                                                       0x8
+#define GFX_IMU_IH_CTRL_3__VF__SHIFT                                                                          0xd
+#define GFX_IMU_IH_CTRL_3__SOURCE_ID_MASK                                                                     0x000000FFL
+#define GFX_IMU_IH_CTRL_3__VF_ID_MASK                                                                         0x00001F00L
+#define GFX_IMU_IH_CTRL_3__VF_MASK                                                                            0x00002000L
+//GFX_IMU_IH_STATUS
+#define GFX_IMU_IH_STATUS__IH_BUSY__SHIFT                                                                     0x0
+#define GFX_IMU_IH_STATUS__IH_BUSY_MASK                                                                       0x00000001L
+//GFX_IMU_FUSESTRAP
+//GFX_IMU_GFXCLK_BYPASS_CTRL
+#define GFX_IMU_GFXCLK_BYPASS_CTRL__BYPASS_SEL__SHIFT                                                         0x0
+#define GFX_IMU_GFXCLK_BYPASS_CTRL__BYPASS_SEL_MASK                                                           0x00000001L
+//GFX_IMU_CLK_CTRL
+#define GFX_IMU_CLK_CTRL__CG_OVR__SHIFT                                                                       0x0
+#define GFX_IMU_CLK_CTRL__CG_OVR_CORE__SHIFT                                                                  0x1
+#define GFX_IMU_CLK_CTRL__GFXBYPASSCLK_DIV__SHIFT                                                             0x10
+#define GFX_IMU_CLK_CTRL__COOLDOWN_PERIOD__SHIFT                                                              0x1c
+#define GFX_IMU_CLK_CTRL__CG_OVR_MASK                                                                         0x00000001L
+#define GFX_IMU_CLK_CTRL__CG_OVR_CORE_MASK                                                                    0x00000002L
+#define GFX_IMU_CLK_CTRL__GFXBYPASSCLK_DIV_MASK                                                               0x007F0000L
+#define GFX_IMU_CLK_CTRL__COOLDOWN_PERIOD_MASK                                                                0xF0000000L
+//GFX_IMU_DOORBELL_CONTROL
+#define GFX_IMU_DOORBELL_CONTROL__OVR_EN__SHIFT                                                               0x0
+#define GFX_IMU_DOORBELL_CONTROL__FENCE_EN_OVR__SHIFT                                                         0x1
+#define GFX_IMU_DOORBELL_CONTROL__FENCE_EN_STATUS__SHIFT                                                      0x1f
+#define GFX_IMU_DOORBELL_CONTROL__OVR_EN_MASK                                                                 0x00000001L
+#define GFX_IMU_DOORBELL_CONTROL__FENCE_EN_OVR_MASK                                                           0x00000002L
+#define GFX_IMU_DOORBELL_CONTROL__FENCE_EN_STATUS_MASK                                                        0x80000000L
+//GFX_IMU_RLC_CG_CTRL
+#define GFX_IMU_RLC_CG_CTRL__FORCE_CGCG__SHIFT                                                                0x0
+#define GFX_IMU_RLC_CG_CTRL__MGCG_EARLY_EN__SHIFT                                                             0x1
+#define GFX_IMU_RLC_CG_CTRL__FORCE_CGCG_MASK                                                                  0x00000001L
+#define GFX_IMU_RLC_CG_CTRL__MGCG_EARLY_EN_MASK                                                               0x00000002L
+//GFX_IMU_RLC_THROTTLE_GFX
+#define GFX_IMU_RLC_THROTTLE_GFX__THROTTLE_EN__SHIFT                                                          0x0
+#define GFX_IMU_RLC_THROTTLE_GFX__THROTTLE_EN_MASK                                                            0x00000001L
+//GFX_IMU_RLC_RESET_VECTOR
+#define GFX_IMU_RLC_RESET_VECTOR__COLD_VS_GFXOFF__SHIFT                                                       0x0
+#define GFX_IMU_RLC_RESET_VECTOR__WARM_RESET_EXIT__SHIFT                                                      0x2
+#define GFX_IMU_RLC_RESET_VECTOR__VF_FLR_EXIT__SHIFT                                                          0x3
+#define GFX_IMU_RLC_RESET_VECTOR__VECTOR__SHIFT                                                               0x6
+#define GFX_IMU_RLC_RESET_VECTOR__COLD_VS_GFXOFF_MASK                                                         0x00000001L
+#define GFX_IMU_RLC_RESET_VECTOR__WARM_RESET_EXIT_MASK                                                        0x00000004L
+#define GFX_IMU_RLC_RESET_VECTOR__VF_FLR_EXIT_MASK                                                            0x00000008L
+#define GFX_IMU_RLC_RESET_VECTOR__VECTOR_MASK                                                                 0x000000C0L
+//GFX_IMU_RLC_OVERRIDE
+#define GFX_IMU_RLC_OVERRIDE__DS_ALLOW__SHIFT                                                                 0x0
+#define GFX_IMU_RLC_OVERRIDE__DS_ALLOW_MASK                                                                   0x00000001L
+//GFX_IMU_DPM_CONTROL
+#define GFX_IMU_DPM_CONTROL__ACC_RESET__SHIFT                                                                 0x0
+#define GFX_IMU_DPM_CONTROL__ACC_START__SHIFT                                                                 0x1
+#define GFX_IMU_DPM_CONTROL__BUSY_MASK__SHIFT                                                                 0x2
+#define GFX_IMU_DPM_CONTROL__ACC_RESET_MASK                                                                   0x00000001L
+#define GFX_IMU_DPM_CONTROL__ACC_START_MASK                                                                   0x00000002L
+#define GFX_IMU_DPM_CONTROL__BUSY_MASK_MASK                                                                   0x0003FFFCL
+//GFX_IMU_DPM_ACC
+#define GFX_IMU_DPM_ACC__COUNT__SHIFT                                                                         0x0
+#define GFX_IMU_DPM_ACC__COUNT_MASK                                                                           0x00FFFFFFL
+//GFX_IMU_DPM_REF_COUNTER
+#define GFX_IMU_DPM_REF_COUNTER__COUNT__SHIFT                                                                 0x0
+#define GFX_IMU_DPM_REF_COUNTER__COUNT_MASK                                                                   0x00FFFFFFL
+//GFX_IMU_RLC_RAM_INDEX
+#define GFX_IMU_RLC_RAM_INDEX__INDEX__SHIFT                                                                   0x0
+#define GFX_IMU_RLC_RAM_INDEX__RLC_INDEX__SHIFT                                                               0x10
+#define GFX_IMU_RLC_RAM_INDEX__RAM_VALID__SHIFT                                                               0x1f
+#define GFX_IMU_RLC_RAM_INDEX__INDEX_MASK                                                                     0x000000FFL
+#define GFX_IMU_RLC_RAM_INDEX__RLC_INDEX_MASK                                                                 0x00FF0000L
+#define GFX_IMU_RLC_RAM_INDEX__RAM_VALID_MASK                                                                 0x80000000L
+//GFX_IMU_RLC_RAM_ADDR_HIGH
+#define GFX_IMU_RLC_RAM_ADDR_HIGH__ADDR_MSB__SHIFT                                                            0x0
+#define GFX_IMU_RLC_RAM_ADDR_HIGH__ADDR_MSB_MASK                                                              0x0000FFFFL
+//GFX_IMU_RLC_RAM_ADDR_LOW
+#define GFX_IMU_RLC_RAM_ADDR_LOW__ADDR_LSB__SHIFT                                                             0x0
+#define GFX_IMU_RLC_RAM_ADDR_LOW__ADDR_LSB_MASK                                                               0xFFFFFFFFL
+//GFX_IMU_RLC_RAM_DATA
+#define GFX_IMU_RLC_RAM_DATA__DATA__SHIFT                                                                     0x0
+#define GFX_IMU_RLC_RAM_DATA__DATA_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_FENCE_CTRL
+#define GFX_IMU_FENCE_CTRL__ENABLED__SHIFT                                                                    0x0
+#define GFX_IMU_FENCE_CTRL__ARM_LOG__SHIFT                                                                    0x1
+#define GFX_IMU_FENCE_CTRL__FLUSH_ARBITER_CREDITS__SHIFT                                                      0x3
+#define GFX_IMU_FENCE_CTRL__GFX_REG_FENCE_OVR_EN__SHIFT                                                       0x8
+#define GFX_IMU_FENCE_CTRL__GFX_REG_FENCE_OVR__SHIFT                                                          0x9
+#define GFX_IMU_FENCE_CTRL__ENABLED_MASK                                                                      0x00000001L
+#define GFX_IMU_FENCE_CTRL__ARM_LOG_MASK                                                                      0x00000002L
+#define GFX_IMU_FENCE_CTRL__FLUSH_ARBITER_CREDITS_MASK                                                        0x00000008L
+#define GFX_IMU_FENCE_CTRL__GFX_REG_FENCE_OVR_EN_MASK                                                         0x00000100L
+#define GFX_IMU_FENCE_CTRL__GFX_REG_FENCE_OVR_MASK                                                            0x00000200L
+//GFX_IMU_FENCE_LOG_INIT
+#define GFX_IMU_FENCE_LOG_INIT__UNIT_ID__SHIFT                                                                0x0
+#define GFX_IMU_FENCE_LOG_INIT__INITIATOR_ID__SHIFT                                                           0x7
+#define GFX_IMU_FENCE_LOG_INIT__UNIT_ID_MASK                                                                  0x0000007FL
+#define GFX_IMU_FENCE_LOG_INIT__INITIATOR_ID_MASK                                                             0x0001FF80L
+//GFX_IMU_FENCE_LOG_ADDR
+#define GFX_IMU_FENCE_LOG_ADDR__ADDR__SHIFT                                                                   0x2
+#define GFX_IMU_FENCE_LOG_ADDR__ADDR_MASK                                                                     0x000FFFFCL
+//GFX_IMU_PROGRAM_CTR
+#define GFX_IMU_PROGRAM_CTR__PC__SHIFT                                                                        0x0
+#define GFX_IMU_PROGRAM_CTR__PC_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_CORE_CTRL
+#define GFX_IMU_CORE_CTRL__CRESET__SHIFT                                                                      0x0
+#define GFX_IMU_CORE_CTRL__CSTALL__SHIFT                                                                      0x1
+#define GFX_IMU_CORE_CTRL__CDBGENABLE__SHIFT                                                                  0x2
+#define GFX_IMU_CORE_CTRL__DRESET__SHIFT                                                                      0x3
+#define GFX_IMU_CORE_CTRL__HALT_ON_RESET__SHIFT                                                               0x4
+#define GFX_IMU_CORE_CTRL__BREAK_IN__SHIFT                                                                    0x8
+#define GFX_IMU_CORE_CTRL__BREAK_OUT_ACK__SHIFT                                                               0x9
+#define GFX_IMU_CORE_CTRL__CRESET_MASK                                                                        0x00000001L
+#define GFX_IMU_CORE_CTRL__CSTALL_MASK                                                                        0x00000002L
+#define GFX_IMU_CORE_CTRL__CDBGENABLE_MASK                                                                    0x00000004L
+#define GFX_IMU_CORE_CTRL__DRESET_MASK                                                                        0x00000008L
+#define GFX_IMU_CORE_CTRL__HALT_ON_RESET_MASK                                                                 0x00000010L
+#define GFX_IMU_CORE_CTRL__BREAK_IN_MASK                                                                      0x00000100L
+#define GFX_IMU_CORE_CTRL__BREAK_OUT_ACK_MASK                                                                 0x00000200L
+//GFX_IMU_CORE_STATUS
+#define GFX_IMU_CORE_STATUS__CBUSY__SHIFT                                                                     0x0
+#define GFX_IMU_CORE_STATUS__PWAIT_MODE__SHIFT                                                                0x1
+#define GFX_IMU_CORE_STATUS__CINTLEVEL__SHIFT                                                                 0x4
+#define GFX_IMU_CORE_STATUS__BREAK_IN_ACK__SHIFT                                                              0x8
+#define GFX_IMU_CORE_STATUS__BREAK_OUT__SHIFT                                                                 0x9
+#define GFX_IMU_CORE_STATUS__P_FATAL_ERROR__SHIFT                                                             0xb
+#define GFX_IMU_CORE_STATUS__FAULT_SEVERITY_LEVEL__SHIFT                                                      0x18
+#define GFX_IMU_CORE_STATUS__FAULT_TYPE__SHIFT                                                                0x1c
+#define GFX_IMU_CORE_STATUS__CBUSY_MASK                                                                       0x00000001L
+#define GFX_IMU_CORE_STATUS__PWAIT_MODE_MASK                                                                  0x00000002L
+#define GFX_IMU_CORE_STATUS__CINTLEVEL_MASK                                                                   0x000000F0L
+#define GFX_IMU_CORE_STATUS__BREAK_IN_ACK_MASK                                                                0x00000100L
+#define GFX_IMU_CORE_STATUS__BREAK_OUT_MASK                                                                   0x00000200L
+#define GFX_IMU_CORE_STATUS__P_FATAL_ERROR_MASK                                                               0x00000800L
+#define GFX_IMU_CORE_STATUS__FAULT_SEVERITY_LEVEL_MASK                                                        0x0F000000L
+#define GFX_IMU_CORE_STATUS__FAULT_TYPE_MASK                                                                  0xF0000000L
+//GFX_IMU_PWROKRAW
+#define GFX_IMU_PWROKRAW__PWROKRAW__SHIFT                                                                     0x0
+#define GFX_IMU_PWROKRAW__PWROKRAW_MASK                                                                       0x00000001L
+//GFX_IMU_PWROK
+#define GFX_IMU_PWROK__PWROK__SHIFT                                                                           0x0
+#define GFX_IMU_PWROK__PWROK_MASK                                                                             0x00000001L
+//GFX_IMU_GAP_PWROK
+#define GFX_IMU_GAP_PWROK__GAP_PWROK__SHIFT                                                                   0x0
+#define GFX_IMU_GAP_PWROK__GAP_PWROK_MASK                                                                     0x00000001L
+//GFX_IMU_RESETn
+#define GFX_IMU_RESETn__Cpl_RESETn__SHIFT                                                                     0x0
+#define GFX_IMU_RESETn__Cpl_RESETn_MASK                                                                       0x00000001L
+//GFX_IMU_GFX_RESET_CTRL
+#define GFX_IMU_GFX_RESET_CTRL__HARD_RESETB__SHIFT                                                            0x0
+#define GFX_IMU_GFX_RESET_CTRL__SDMA_RESETB__SHIFT                                                            0x3
+#define GFX_IMU_GFX_RESET_CTRL__GRBM_RESETB__SHIFT                                                            0x4
+#define GFX_IMU_GFX_RESET_CTRL__DFLL_SRESETB__SHIFT                                                           0x5
+#define GFX_IMU_GFX_RESET_CTRL__HARD_RESETB_MASK                                                              0x00000001L
+#define GFX_IMU_GFX_RESET_CTRL__SDMA_RESETB_MASK                                                              0x00000008L
+#define GFX_IMU_GFX_RESET_CTRL__GRBM_RESETB_MASK                                                              0x00000010L
+#define GFX_IMU_GFX_RESET_CTRL__DFLL_SRESETB_MASK                                                             0x00000020L
+//GFX_IMU_VDCI_RESET_CTRL
+#define GFX_IMU_VDCI_RESET_CTRL__SOC2GFX_VDCI_RESETn__SHIFT                                                   0x0
+#define GFX_IMU_VDCI_RESET_CTRL__SOC_EA_SDF_VDCI_RESET__SHIFT                                                 0x1
+#define GFX_IMU_VDCI_RESET_CTRL__SOC_UTCL2_ATHUB_VDCI_RESET__SHIFT                                            0x2
+#define GFX_IMU_VDCI_RESET_CTRL__IMU2GFX_VDCI_RESETn__SHIFT                                                   0x4
+#define GFX_IMU_VDCI_RESET_CTRL__SOC2GFX_VDCI_RESETn_MASK                                                     0x00000001L
+#define GFX_IMU_VDCI_RESET_CTRL__SOC_EA_SDF_VDCI_RESET_MASK                                                   0x00000002L
+#define GFX_IMU_VDCI_RESET_CTRL__SOC_UTCL2_ATHUB_VDCI_RESET_MASK                                              0x00000004L
+#define GFX_IMU_VDCI_RESET_CTRL__IMU2GFX_VDCI_RESETn_MASK                                                     0x00000010L
+//GFX_IMU_GFX_ISO_CTRL
+#define GFX_IMU_GFX_ISO_CTRL__GFX2IMU_ISOn__SHIFT                                                             0x0
+#define GFX_IMU_GFX_ISO_CTRL__SOC_EA_SDF_VDCI_ISOn_EN__SHIFT                                                  0x1
+#define GFX_IMU_GFX_ISO_CTRL__SOC_UTCL2_ATHUB_VDCI_ISOn_EN__SHIFT                                             0x2
+#define GFX_IMU_GFX_ISO_CTRL__GFX2SOC_ISOn__SHIFT                                                             0x3
+#define GFX_IMU_GFX_ISO_CTRL__GFX2SOC_CLK_ISOn__SHIFT                                                         0x4
+#define GFX_IMU_GFX_ISO_CTRL__GFX2IMU_ISOn_MASK                                                               0x00000001L
+#define GFX_IMU_GFX_ISO_CTRL__SOC_EA_SDF_VDCI_ISOn_EN_MASK                                                    0x00000002L
+#define GFX_IMU_GFX_ISO_CTRL__SOC_UTCL2_ATHUB_VDCI_ISOn_EN_MASK                                               0x00000004L
+#define GFX_IMU_GFX_ISO_CTRL__GFX2SOC_ISOn_MASK                                                               0x00000008L
+#define GFX_IMU_GFX_ISO_CTRL__GFX2SOC_CLK_ISOn_MASK                                                           0x00000010L
+//GFX_IMU_TIMER0_CTRL0
+#define GFX_IMU_TIMER0_CTRL0__START_STOP__SHIFT                                                               0x0
+#define GFX_IMU_TIMER0_CTRL0__CLEAR__SHIFT                                                                    0x8
+#define GFX_IMU_TIMER0_CTRL0__UP_DOWN__SHIFT                                                                  0x10
+#define GFX_IMU_TIMER0_CTRL0__PULSE_EN__SHIFT                                                                 0x18
+#define GFX_IMU_TIMER0_CTRL0__START_STOP_MASK                                                                 0x00000001L
+#define GFX_IMU_TIMER0_CTRL0__CLEAR_MASK                                                                      0x00000100L
+#define GFX_IMU_TIMER0_CTRL0__UP_DOWN_MASK                                                                    0x00010000L
+#define GFX_IMU_TIMER0_CTRL0__PULSE_EN_MASK                                                                   0x01000000L
+//GFX_IMU_TIMER0_CTRL1
+#define GFX_IMU_TIMER0_CTRL1__PWM_EN__SHIFT                                                                   0x0
+#define GFX_IMU_TIMER0_CTRL1__TS_MODE__SHIFT                                                                  0x8
+#define GFX_IMU_TIMER0_CTRL1__SAT_EN__SHIFT                                                                   0x10
+#define GFX_IMU_TIMER0_CTRL1__PWM_EN_MASK                                                                     0x00000001L
+#define GFX_IMU_TIMER0_CTRL1__TS_MODE_MASK                                                                    0x00000100L
+#define GFX_IMU_TIMER0_CTRL1__SAT_EN_MASK                                                                     0x00010000L
+//GFX_IMU_TIMER0_CMP_AUTOINC
+#define GFX_IMU_TIMER0_CMP_AUTOINC__AUTOINC_EN0__SHIFT                                                        0x0
+#define GFX_IMU_TIMER0_CMP_AUTOINC__AUTOINC_EN1__SHIFT                                                        0x1
+#define GFX_IMU_TIMER0_CMP_AUTOINC__AUTOINC_EN2__SHIFT                                                        0x2
+#define GFX_IMU_TIMER0_CMP_AUTOINC__AUTOINC_EN3__SHIFT                                                        0x3
+#define GFX_IMU_TIMER0_CMP_AUTOINC__AUTOINC_EN0_MASK                                                          0x00000001L
+#define GFX_IMU_TIMER0_CMP_AUTOINC__AUTOINC_EN1_MASK                                                          0x00000002L
+#define GFX_IMU_TIMER0_CMP_AUTOINC__AUTOINC_EN2_MASK                                                          0x00000004L
+#define GFX_IMU_TIMER0_CMP_AUTOINC__AUTOINC_EN3_MASK                                                          0x00000008L
+//GFX_IMU_TIMER0_CMP_INTEN
+#define GFX_IMU_TIMER0_CMP_INTEN__INT_EN0__SHIFT                                                              0x0
+#define GFX_IMU_TIMER0_CMP_INTEN__INT_EN1__SHIFT                                                              0x1
+#define GFX_IMU_TIMER0_CMP_INTEN__INT_EN2__SHIFT                                                              0x2
+#define GFX_IMU_TIMER0_CMP_INTEN__INT_EN3__SHIFT                                                              0x3
+#define GFX_IMU_TIMER0_CMP_INTEN__INT_EN0_MASK                                                                0x00000001L
+#define GFX_IMU_TIMER0_CMP_INTEN__INT_EN1_MASK                                                                0x00000002L
+#define GFX_IMU_TIMER0_CMP_INTEN__INT_EN2_MASK                                                                0x00000004L
+#define GFX_IMU_TIMER0_CMP_INTEN__INT_EN3_MASK                                                                0x00000008L
+//GFX_IMU_TIMER0_CMP0
+#define GFX_IMU_TIMER0_CMP0__VALUE__SHIFT                                                                     0x0
+#define GFX_IMU_TIMER0_CMP0__VALUE_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_TIMER0_CMP1
+#define GFX_IMU_TIMER0_CMP1__VALUE__SHIFT                                                                     0x0
+#define GFX_IMU_TIMER0_CMP1__VALUE_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_TIMER0_CMP3
+#define GFX_IMU_TIMER0_CMP3__VALUE__SHIFT                                                                     0x0
+#define GFX_IMU_TIMER0_CMP3__VALUE_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_TIMER0_VALUE
+#define GFX_IMU_TIMER0_VALUE__VALUE__SHIFT                                                                    0x0
+#define GFX_IMU_TIMER0_VALUE__VALUE_MASK                                                                      0xFFFFFFFFL
+//GFX_IMU_TIMER1_CTRL0
+#define GFX_IMU_TIMER1_CTRL0__START_STOP__SHIFT                                                               0x0
+#define GFX_IMU_TIMER1_CTRL0__CLEAR__SHIFT                                                                    0x8
+#define GFX_IMU_TIMER1_CTRL0__UP_DOWN__SHIFT                                                                  0x10
+#define GFX_IMU_TIMER1_CTRL0__PULSE_EN__SHIFT                                                                 0x18
+#define GFX_IMU_TIMER1_CTRL0__START_STOP_MASK                                                                 0x00000001L
+#define GFX_IMU_TIMER1_CTRL0__CLEAR_MASK                                                                      0x00000100L
+#define GFX_IMU_TIMER1_CTRL0__UP_DOWN_MASK                                                                    0x00010000L
+#define GFX_IMU_TIMER1_CTRL0__PULSE_EN_MASK                                                                   0x01000000L
+//GFX_IMU_TIMER1_CTRL1
+#define GFX_IMU_TIMER1_CTRL1__PWM_EN__SHIFT                                                                   0x0
+#define GFX_IMU_TIMER1_CTRL1__TS_MODE__SHIFT                                                                  0x8
+#define GFX_IMU_TIMER1_CTRL1__SAT_EN__SHIFT                                                                   0x10
+#define GFX_IMU_TIMER1_CTRL1__PWM_EN_MASK                                                                     0x00000001L
+#define GFX_IMU_TIMER1_CTRL1__TS_MODE_MASK                                                                    0x00000100L
+#define GFX_IMU_TIMER1_CTRL1__SAT_EN_MASK                                                                     0x00010000L
+//GFX_IMU_TIMER1_CMP_AUTOINC
+#define GFX_IMU_TIMER1_CMP_AUTOINC__AUTOINC_EN0__SHIFT                                                        0x0
+#define GFX_IMU_TIMER1_CMP_AUTOINC__AUTOINC_EN1__SHIFT                                                        0x1
+#define GFX_IMU_TIMER1_CMP_AUTOINC__AUTOINC_EN2__SHIFT                                                        0x2
+#define GFX_IMU_TIMER1_CMP_AUTOINC__AUTOINC_EN3__SHIFT                                                        0x3
+#define GFX_IMU_TIMER1_CMP_AUTOINC__AUTOINC_EN0_MASK                                                          0x00000001L
+#define GFX_IMU_TIMER1_CMP_AUTOINC__AUTOINC_EN1_MASK                                                          0x00000002L
+#define GFX_IMU_TIMER1_CMP_AUTOINC__AUTOINC_EN2_MASK                                                          0x00000004L
+#define GFX_IMU_TIMER1_CMP_AUTOINC__AUTOINC_EN3_MASK                                                          0x00000008L
+//GFX_IMU_TIMER1_CMP_INTEN
+#define GFX_IMU_TIMER1_CMP_INTEN__INT_EN0__SHIFT                                                              0x0
+#define GFX_IMU_TIMER1_CMP_INTEN__INT_EN1__SHIFT                                                              0x1
+#define GFX_IMU_TIMER1_CMP_INTEN__INT_EN2__SHIFT                                                              0x2
+#define GFX_IMU_TIMER1_CMP_INTEN__INT_EN3__SHIFT                                                              0x3
+#define GFX_IMU_TIMER1_CMP_INTEN__INT_EN0_MASK                                                                0x00000001L
+#define GFX_IMU_TIMER1_CMP_INTEN__INT_EN1_MASK                                                                0x00000002L
+#define GFX_IMU_TIMER1_CMP_INTEN__INT_EN2_MASK                                                                0x00000004L
+#define GFX_IMU_TIMER1_CMP_INTEN__INT_EN3_MASK                                                                0x00000008L
+//GFX_IMU_TIMER1_CMP0
+#define GFX_IMU_TIMER1_CMP0__VALUE__SHIFT                                                                     0x0
+#define GFX_IMU_TIMER1_CMP0__VALUE_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_TIMER1_CMP1
+#define GFX_IMU_TIMER1_CMP1__VALUE__SHIFT                                                                     0x0
+#define GFX_IMU_TIMER1_CMP1__VALUE_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_TIMER1_CMP3
+#define GFX_IMU_TIMER1_CMP3__VALUE__SHIFT                                                                     0x0
+#define GFX_IMU_TIMER1_CMP3__VALUE_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_TIMER1_VALUE
+#define GFX_IMU_TIMER1_VALUE__VALUE__SHIFT                                                                    0x0
+#define GFX_IMU_TIMER1_VALUE__VALUE_MASK                                                                      0xFFFFFFFFL
+//GFX_IMU_FUSE_CTRL
+#define GFX_IMU_FUSE_CTRL__DIV_OVR__SHIFT                                                                     0x0
+#define GFX_IMU_FUSE_CTRL__DIV_OVR_EN__SHIFT                                                                  0x5
+#define GFX_IMU_FUSE_CTRL__FORCE_DONE__SHIFT                                                                  0x6
+#define GFX_IMU_FUSE_CTRL__DIV_OVR_MASK                                                                       0x0000001FL
+#define GFX_IMU_FUSE_CTRL__DIV_OVR_EN_MASK                                                                    0x00000020L
+#define GFX_IMU_FUSE_CTRL__FORCE_DONE_MASK                                                                    0x00000040L
+//GFX_IMU_D_RAM_ADDR
+#define GFX_IMU_D_RAM_ADDR__ADDR__SHIFT                                                                       0x2
+#define GFX_IMU_D_RAM_ADDR__ADDR_MASK                                                                         0x0000FFFCL
+//GFX_IMU_D_RAM_DATA
+#define GFX_IMU_D_RAM_DATA__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_D_RAM_DATA__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_GFX_IH_GASKET_CTRL
+#define GFX_IMU_GFX_IH_GASKET_CTRL__SRSTB__SHIFT                                                              0x0
+#define GFX_IMU_GFX_IH_GASKET_CTRL__BUFFER_LEVEL__SHIFT                                                       0x10
+#define GFX_IMU_GFX_IH_GASKET_CTRL__BUFFER_OVERFLOW__SHIFT                                                    0x14
+#define GFX_IMU_GFX_IH_GASKET_CTRL__SRSTB_MASK                                                                0x00000001L
+#define GFX_IMU_GFX_IH_GASKET_CTRL__BUFFER_LEVEL_MASK                                                         0x000F0000L
+#define GFX_IMU_GFX_IH_GASKET_CTRL__BUFFER_OVERFLOW_MASK                                                      0x00100000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_gfx_imu_cpwd_gfx_imu_pspdec
+//GFX_IMU_RLC_BOOTLOADER_ADDR_HI
+#define GFX_IMU_RLC_BOOTLOADER_ADDR_HI__ADDR_HI__SHIFT                                                        0x0
+#define GFX_IMU_RLC_BOOTLOADER_ADDR_HI__ADDR_HI_MASK                                                          0xFFFFFFFFL
+//GFX_IMU_RLC_BOOTLOADER_ADDR_LO
+#define GFX_IMU_RLC_BOOTLOADER_ADDR_LO__ADDR_LO__SHIFT                                                        0x0
+#define GFX_IMU_RLC_BOOTLOADER_ADDR_LO__ADDR_LO_MASK                                                          0xFFFFFFFFL
+//GFX_IMU_RLC_BOOTLOADER_SIZE
+#define GFX_IMU_RLC_BOOTLOADER_SIZE__SIZE__SHIFT                                                              0x0
+#define GFX_IMU_RLC_BOOTLOADER_SIZE__SIZE_MASK                                                                0x03FFFFFFL
+//GFX_IMU_I_RAM_ADDR
+#define GFX_IMU_I_RAM_ADDR__ADDR__SHIFT                                                                       0x2
+#define GFX_IMU_I_RAM_ADDR__ADDR_MASK                                                                         0x0000FFFCL
+//GFX_IMU_I_RAM_DATA
+#define GFX_IMU_I_RAM_DATA__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_I_RAM_DATA__DATA_MASK                                                                         0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_grbmhdec
+//GRBMH_CNTL
+#define GRBMH_CNTL__READ_TIMEOUT__SHIFT                                                                       0x0
+#define GRBMH_CNTL__REPORT_LAST_RDERR__SHIFT                                                                  0x1f
+#define GRBMH_CNTL__READ_TIMEOUT_MASK                                                                         0x000000FFL
+#define GRBMH_CNTL__REPORT_LAST_RDERR_MASK                                                                    0x80000000L
+//GRBMH_INTF_CNTL
+#define GRBMH_INTF_CNTL__RSMUSE_PATH_DISABLE__SHIFT                                                           0x0
+#define GRBMH_INTF_CNTL__GRBM_PATH_DISABLE__SHIFT                                                             0x1
+#define GRBMH_INTF_CNTL__RSMUSE_PATH_DISABLE_MASK                                                             0x00000001L
+#define GRBMH_INTF_CNTL__GRBM_PATH_DISABLE_MASK                                                               0x00000002L
+//GRBMH_STATUS
+#define GRBMH_STATUS__SC_CLEAN__SHIFT                                                                         0x0
+#define GRBMH_STATUS__DB_CLEAN__SHIFT                                                                         0x1
+#define GRBMH_STATUS__CB_CLEAN__SHIFT                                                                         0x2
+#define GRBMH_STATUS__UTCL1_BUSY__SHIFT                                                                       0x3
+#define GRBMH_STATUS__TCP_BUSY__SHIFT                                                                         0x4
+#define GRBMH_STATUS__GL1CC_BUSY__SHIFT                                                                       0x5
+#define GRBMH_STATUS__GL1XCC_BUSY__SHIFT                                                                      0x6
+#define GRBMH_STATUS__GE_BUSY__SHIFT                                                                          0x9
+#define GRBMH_STATUS__RLC_BUSY__SHIFT                                                                         0xa
+#define GRBMH_STATUS__WGS_BUSY__SHIFT                                                                         0xb
+#define GRBMH_STATUS__EA_LINK_BUSY__SHIFT                                                                     0xc
+#define GRBMH_STATUS__EA_BUSY__SHIFT                                                                          0xd
+#define GRBMH_STATUS__GL2C_BUSY__SHIFT                                                                        0xe
+#define GRBMH_STATUS__GL2A_BUSY__SHIFT                                                                        0xf
+#define GRBMH_STATUS__SC_BUSY__SHIFT                                                                          0x11
+#define GRBMH_STATUS__GL1A_BUSY__SHIFT                                                                        0x12
+#define GRBMH_STATUS__BCI_BUSY__SHIFT                                                                         0x14
+#define GRBMH_STATUS__SQG_BUSY__SHIFT                                                                         0x17
+#define GRBMH_STATUS__PA_BUSY__SHIFT                                                                          0x18
+#define GRBMH_STATUS__TA_BUSY__SHIFT                                                                          0x19
+#define GRBMH_STATUS__SX_BUSY__SHIFT                                                                          0x1a
+#define GRBMH_STATUS__SPI_BUSY__SHIFT                                                                         0x1b
+#define GRBMH_STATUS__DB_BUSY__SHIFT                                                                          0x1e
+#define GRBMH_STATUS__CB_BUSY__SHIFT                                                                          0x1f
+#define GRBMH_STATUS__SC_CLEAN_MASK                                                                           0x00000001L
+#define GRBMH_STATUS__DB_CLEAN_MASK                                                                           0x00000002L
+#define GRBMH_STATUS__CB_CLEAN_MASK                                                                           0x00000004L
+#define GRBMH_STATUS__UTCL1_BUSY_MASK                                                                         0x00000008L
+#define GRBMH_STATUS__TCP_BUSY_MASK                                                                           0x00000010L
+#define GRBMH_STATUS__GL1CC_BUSY_MASK                                                                         0x00000020L
+#define GRBMH_STATUS__GL1XCC_BUSY_MASK                                                                        0x00000040L
+#define GRBMH_STATUS__GE_BUSY_MASK                                                                            0x00000200L
+#define GRBMH_STATUS__RLC_BUSY_MASK                                                                           0x00000400L
+#define GRBMH_STATUS__WGS_BUSY_MASK                                                                           0x00000800L
+#define GRBMH_STATUS__EA_LINK_BUSY_MASK                                                                       0x00001000L
+#define GRBMH_STATUS__EA_BUSY_MASK                                                                            0x00002000L
+#define GRBMH_STATUS__GL2C_BUSY_MASK                                                                          0x00004000L
+#define GRBMH_STATUS__GL2A_BUSY_MASK                                                                          0x00008000L
+#define GRBMH_STATUS__SC_BUSY_MASK                                                                            0x00020000L
+#define GRBMH_STATUS__GL1A_BUSY_MASK                                                                          0x00040000L
+#define GRBMH_STATUS__BCI_BUSY_MASK                                                                           0x00100000L
+#define GRBMH_STATUS__SQG_BUSY_MASK                                                                           0x00800000L
+#define GRBMH_STATUS__PA_BUSY_MASK                                                                            0x01000000L
+#define GRBMH_STATUS__TA_BUSY_MASK                                                                            0x02000000L
+#define GRBMH_STATUS__SX_BUSY_MASK                                                                            0x04000000L
+#define GRBMH_STATUS__SPI_BUSY_MASK                                                                           0x08000000L
+#define GRBMH_STATUS__DB_BUSY_MASK                                                                            0x40000000L
+#define GRBMH_STATUS__CB_BUSY_MASK                                                                            0x80000000L
+//GRBMH_FGCG0_TARG
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG0_CHICK_BIT__SHIFT                                                   0x0
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG1_CHICK_BIT__SHIFT                                                   0x1
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG2_CHICK_BIT__SHIFT                                                   0x2
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG3_CHICK_BIT__SHIFT                                                   0x3
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG4_CHICK_BIT__SHIFT                                                   0x4
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG5_CHICK_BIT__SHIFT                                                   0x5
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG6_CHICK_BIT__SHIFT                                                   0x6
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG7_CHICK_BIT__SHIFT                                                   0x7
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG8_CHICK_BIT__SHIFT                                                   0x8
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG9_CHICK_BIT__SHIFT                                                   0x9
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG10_CHICK_BIT__SHIFT                                                  0xa
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG11_CHICK_BIT__SHIFT                                                  0xb
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG12_CHICK_BIT__SHIFT                                                  0xc
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG13_CHICK_BIT__SHIFT                                                  0xd
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG14_CHICK_BIT__SHIFT                                                  0xe
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG15_CHICK_BIT__SHIFT                                                  0xf
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG16_CHICK_BIT__SHIFT                                                  0x10
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG17_CHICK_BIT__SHIFT                                                  0x11
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG18_CHICK_BIT__SHIFT                                                  0x12
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG19_CHICK_BIT__SHIFT                                                  0x13
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG20_CHICK_BIT__SHIFT                                                  0x14
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG21_CHICK_BIT__SHIFT                                                  0x15
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG22_CHICK_BIT__SHIFT                                                  0x16
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG23_CHICK_BIT__SHIFT                                                  0x17
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG24_CHICK_BIT__SHIFT                                                  0x18
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG25_CHICK_BIT__SHIFT                                                  0x19
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG26_CHICK_BIT__SHIFT                                                  0x1a
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG27_CHICK_BIT__SHIFT                                                  0x1b
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG28_CHICK_BIT__SHIFT                                                  0x1c
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG29_CHICK_BIT__SHIFT                                                  0x1d
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG30_CHICK_BIT__SHIFT                                                  0x1e
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG31_CHICK_BIT__SHIFT                                                  0x1f
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG0_CHICK_BIT_MASK                                                     0x00000001L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG1_CHICK_BIT_MASK                                                     0x00000002L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG2_CHICK_BIT_MASK                                                     0x00000004L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG3_CHICK_BIT_MASK                                                     0x00000008L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG4_CHICK_BIT_MASK                                                     0x00000010L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG5_CHICK_BIT_MASK                                                     0x00000020L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG6_CHICK_BIT_MASK                                                     0x00000040L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG7_CHICK_BIT_MASK                                                     0x00000080L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG8_CHICK_BIT_MASK                                                     0x00000100L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG9_CHICK_BIT_MASK                                                     0x00000200L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG10_CHICK_BIT_MASK                                                    0x00000400L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG11_CHICK_BIT_MASK                                                    0x00000800L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG12_CHICK_BIT_MASK                                                    0x00001000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG13_CHICK_BIT_MASK                                                    0x00002000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG14_CHICK_BIT_MASK                                                    0x00004000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG15_CHICK_BIT_MASK                                                    0x00008000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG16_CHICK_BIT_MASK                                                    0x00010000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG17_CHICK_BIT_MASK                                                    0x00020000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG18_CHICK_BIT_MASK                                                    0x00040000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG19_CHICK_BIT_MASK                                                    0x00080000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG20_CHICK_BIT_MASK                                                    0x00100000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG21_CHICK_BIT_MASK                                                    0x00200000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG22_CHICK_BIT_MASK                                                    0x00400000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG23_CHICK_BIT_MASK                                                    0x00800000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG24_CHICK_BIT_MASK                                                    0x01000000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG25_CHICK_BIT_MASK                                                    0x02000000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG26_CHICK_BIT_MASK                                                    0x04000000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG27_CHICK_BIT_MASK                                                    0x08000000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG28_CHICK_BIT_MASK                                                    0x10000000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG29_CHICK_BIT_MASK                                                    0x20000000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG30_CHICK_BIT_MASK                                                    0x40000000L
+#define GRBMH_FGCG0_TARG__FGCG_GRBMH_TARG31_CHICK_BIT_MASK                                                    0x80000000L
+//GRBMH_SOFT_RESET
+#define GRBMH_SOFT_RESET__SOFT_RESET_GFX__SHIFT                                                               0x0
+#define GRBMH_SOFT_RESET__SOFT_RESET_SX__SHIFT                                                                0x1
+#define GRBMH_SOFT_RESET__SOFT_RESET_EA__SHIFT                                                                0x2
+#define GRBMH_SOFT_RESET__SOFT_RESET_WGS__SHIFT                                                               0x4
+#define GRBMH_SOFT_RESET__SOFT_RESET_RLCSE__SHIFT                                                             0x5
+#define GRBMH_SOFT_RESET__SOFT_RESET_WGPCAC__SHIFT                                                            0x6
+#define GRBMH_SOFT_RESET__SOFT_RESET_CAC__SHIFT                                                               0x7
+#define GRBMH_SOFT_RESET__SOFT_RESET_TA__SHIFT                                                                0x9
+#define GRBMH_SOFT_RESET__SOFT_RESET_GFX_MASK                                                                 0x00000001L
+#define GRBMH_SOFT_RESET__SOFT_RESET_SX_MASK                                                                  0x00000002L
+#define GRBMH_SOFT_RESET__SOFT_RESET_EA_MASK                                                                  0x00000004L
+#define GRBMH_SOFT_RESET__SOFT_RESET_WGS_MASK                                                                 0x00000010L
+#define GRBMH_SOFT_RESET__SOFT_RESET_RLCSE_MASK                                                               0x00000020L
+#define GRBMH_SOFT_RESET__SOFT_RESET_WGPCAC_MASK                                                              0x00000040L
+#define GRBMH_SOFT_RESET__SOFT_RESET_CAC_MASK                                                                 0x00000080L
+#define GRBMH_SOFT_RESET__SOFT_RESET_TA_MASK                                                                  0x00000200L
+//GRBMH_READ_ERROR
+#define GRBMH_READ_ERROR__READ_ADDRESS__SHIFT                                                                 0x2
+#define GRBMH_READ_ERROR__READ_PIPEID__SHIFT                                                                  0x14
+#define GRBMH_READ_ERROR__READ_MEID__SHIFT                                                                    0x16
+#define GRBMH_READ_ERROR__READ_REQUESTER_RLC__SHIFT                                                           0x1b
+#define GRBMH_READ_ERROR__READ_REQUESTER_AID_GFX_PIPE0__SHIFT                                                 0x1c
+#define GRBMH_READ_ERROR__READ_REQUESTER_AID_NBP_PIPE__SHIFT                                                  0x1e
+#define GRBMH_READ_ERROR__READ_ERROR__SHIFT                                                                   0x1f
+#define GRBMH_READ_ERROR__READ_ADDRESS_MASK                                                                   0x000FFFFCL
+#define GRBMH_READ_ERROR__READ_PIPEID_MASK                                                                    0x00300000L
+#define GRBMH_READ_ERROR__READ_MEID_MASK                                                                      0x00C00000L
+#define GRBMH_READ_ERROR__READ_REQUESTER_RLC_MASK                                                             0x08000000L
+#define GRBMH_READ_ERROR__READ_REQUESTER_AID_GFX_PIPE0_MASK                                                   0x10000000L
+#define GRBMH_READ_ERROR__READ_REQUESTER_AID_NBP_PIPE_MASK                                                    0x40000000L
+#define GRBMH_READ_ERROR__READ_ERROR_MASK                                                                     0x80000000L
+//GRBMH_GFX_CLKEN_CNTL
+#define GRBMH_GFX_CLKEN_CNTL__PREFIX_DELAY_CNT__SHIFT                                                         0x0
+#define GRBMH_GFX_CLKEN_CNTL__POST_DELAY_CNT__SHIFT                                                           0x8
+#define GRBMH_GFX_CLKEN_CNTL__PREFIX_DELAY_CNT_MASK                                                           0x0000000FL
+#define GRBMH_GFX_CLKEN_CNTL__POST_DELAY_CNT_MASK                                                             0x00001F00L
+//GRBMH_FGCG2_MISC
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_SPI_CHICK_BIT__SHIFT                                                     0x0
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_SQG_CHICK_BIT__SHIFT                                                     0x1
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_WGS_CHICK_BIT__SHIFT                                                     0x2
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_GESE_CHICK_BIT__SHIFT                                                    0x3
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_WGP0_CHICK_BIT__SHIFT                                                    0x4
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_WGP1_CHICK_BIT__SHIFT                                                    0x5
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_MSFLLRS0_CHICK_BIT__SHIFT                                                0x6
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_MSFLLRS1_CHICK_BIT__SHIFT                                                0x7
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_WGP4_CHICK_BIT__SHIFT                                                    0x8
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_WGP5_CHICK_BIT__SHIFT                                                    0x9
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_WGP6_CHICK_BIT__SHIFT                                                    0xa
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_WGP7_CHICK_BIT__SHIFT                                                    0xb
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_RB0_CHICK_BIT__SHIFT                                                     0xc
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_RB1_CHICK_BIT__SHIFT                                                     0xd
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_RB2_CHICK_BIT__SHIFT                                                     0xe
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_RB3_CHICK_BIT__SHIFT                                                     0xf
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_RB4_CHICK_BIT__SHIFT                                                     0x10
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_RB5_CHICK_BIT__SHIFT                                                     0x11
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_RB6_CHICK_BIT__SHIFT                                                     0x12
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_RB7_CHICK_BIT__SHIFT                                                     0x13
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_WGP8_CHICK_BIT__SHIFT                                                    0x14
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_SPI_CHICK_BIT_MASK                                                       0x00000001L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_SQG_CHICK_BIT_MASK                                                       0x00000002L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_WGS_CHICK_BIT_MASK                                                       0x00000004L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_GESE_CHICK_BIT_MASK                                                      0x00000008L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_WGP0_CHICK_BIT_MASK                                                      0x00000010L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_WGP1_CHICK_BIT_MASK                                                      0x00000020L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_MSFLLRS0_CHICK_BIT_MASK                                                  0x00000040L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_MSFLLRS1_CHICK_BIT_MASK                                                  0x00000080L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_WGP4_CHICK_BIT_MASK                                                      0x00000100L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_WGP5_CHICK_BIT_MASK                                                      0x00000200L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_WGP6_CHICK_BIT_MASK                                                      0x00000400L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_WGP7_CHICK_BIT_MASK                                                      0x00000800L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_RB0_CHICK_BIT_MASK                                                       0x00001000L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_RB1_CHICK_BIT_MASK                                                       0x00002000L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_RB2_CHICK_BIT_MASK                                                       0x00004000L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_RB3_CHICK_BIT_MASK                                                       0x00008000L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_RB4_CHICK_BIT_MASK                                                       0x00010000L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_RB5_CHICK_BIT_MASK                                                       0x00020000L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_RB6_CHICK_BIT_MASK                                                       0x00040000L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_RB7_CHICK_BIT_MASK                                                       0x00080000L
+#define GRBMH_FGCG2_MISC__FGCG_GRBMH_WGP8_CHICK_BIT_MASK                                                      0x00100000L
+//GRBMH_FGCG1_TARGVF
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF0_CHICK_BIT__SHIFT                                               0x0
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF1_CHICK_BIT__SHIFT                                               0x1
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF2_CHICK_BIT__SHIFT                                               0x2
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF3_CHICK_BIT__SHIFT                                               0x3
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF4_CHICK_BIT__SHIFT                                               0x4
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF5_CHICK_BIT__SHIFT                                               0x5
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF6_CHICK_BIT__SHIFT                                               0x6
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF7_CHICK_BIT__SHIFT                                               0x7
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF8_CHICK_BIT__SHIFT                                               0x8
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF9_CHICK_BIT__SHIFT                                               0x9
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF10_CHICK_BIT__SHIFT                                              0xa
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF11_CHICK_BIT__SHIFT                                              0xb
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF12_CHICK_BIT__SHIFT                                              0xc
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF13_CHICK_BIT__SHIFT                                              0xd
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF14_CHICK_BIT__SHIFT                                              0xe
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF15_CHICK_BIT__SHIFT                                              0xf
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF16_CHICK_BIT__SHIFT                                              0x10
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF17_CHICK_BIT__SHIFT                                              0x11
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF18_CHICK_BIT__SHIFT                                              0x12
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF19_CHICK_BIT__SHIFT                                              0x13
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF20_CHICK_BIT__SHIFT                                              0x14
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF21_CHICK_BIT__SHIFT                                              0x15
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF22_CHICK_BIT__SHIFT                                              0x16
+#define GRBMH_FGCG1_TARGVF__AID_READ_ERROR_CHICK_BIT__SHIFT                                                   0x17
+#define GRBMH_FGCG1_TARGVF__SE_STRAP_MATCH_ENABLE_CHICK_BIT__SHIFT                                            0x18
+#define GRBMH_FGCG1_TARGVF__GRBMH_RLCSE_reg_clken_CHICK_BIT__SHIFT                                            0x19
+#define GRBMH_FGCG1_TARGVF__GRBM_GRBMH_reg_clken_CHICK_BIT__SHIFT                                             0x1a
+#define GRBMH_FGCG1_TARGVF__AID_FGCG_CHICK_BIT__SHIFT                                                         0x1b
+#define GRBMH_FGCG1_TARGVF__GRBMH_GRBM_fgcg_CHICK_BIT__SHIFT                                                  0x1c
+#define GRBMH_FGCG1_TARGVF__GRBMH_SPI_reg_clken_CHICK_BIT__SHIFT                                              0x1d
+#define GRBMH_FGCG1_TARGVF__GRBM_WGS_reg_clken_CHICK_BIT__SHIFT                                               0x1e
+#define GRBMH_FGCG1_TARGVF__GRBMH_ALWAYSON_reg_clken_CHICK_BIT__SHIFT                                         0x1f
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF0_CHICK_BIT_MASK                                                 0x00000001L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF1_CHICK_BIT_MASK                                                 0x00000002L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF2_CHICK_BIT_MASK                                                 0x00000004L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF3_CHICK_BIT_MASK                                                 0x00000008L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF4_CHICK_BIT_MASK                                                 0x00000010L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF5_CHICK_BIT_MASK                                                 0x00000020L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF6_CHICK_BIT_MASK                                                 0x00000040L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF7_CHICK_BIT_MASK                                                 0x00000080L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF8_CHICK_BIT_MASK                                                 0x00000100L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF9_CHICK_BIT_MASK                                                 0x00000200L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF10_CHICK_BIT_MASK                                                0x00000400L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF11_CHICK_BIT_MASK                                                0x00000800L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF12_CHICK_BIT_MASK                                                0x00001000L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF13_CHICK_BIT_MASK                                                0x00002000L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF14_CHICK_BIT_MASK                                                0x00004000L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF15_CHICK_BIT_MASK                                                0x00008000L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF16_CHICK_BIT_MASK                                                0x00010000L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF17_CHICK_BIT_MASK                                                0x00020000L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF18_CHICK_BIT_MASK                                                0x00040000L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF19_CHICK_BIT_MASK                                                0x00080000L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF20_CHICK_BIT_MASK                                                0x00100000L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF21_CHICK_BIT_MASK                                                0x00200000L
+#define GRBMH_FGCG1_TARGVF__FGCG_GRBMH_TARGVF22_CHICK_BIT_MASK                                                0x00400000L
+#define GRBMH_FGCG1_TARGVF__AID_READ_ERROR_CHICK_BIT_MASK                                                     0x00800000L
+#define GRBMH_FGCG1_TARGVF__SE_STRAP_MATCH_ENABLE_CHICK_BIT_MASK                                              0x01000000L
+#define GRBMH_FGCG1_TARGVF__GRBMH_RLCSE_reg_clken_CHICK_BIT_MASK                                              0x02000000L
+#define GRBMH_FGCG1_TARGVF__GRBM_GRBMH_reg_clken_CHICK_BIT_MASK                                               0x04000000L
+#define GRBMH_FGCG1_TARGVF__AID_FGCG_CHICK_BIT_MASK                                                           0x08000000L
+#define GRBMH_FGCG1_TARGVF__GRBMH_GRBM_fgcg_CHICK_BIT_MASK                                                    0x10000000L
+#define GRBMH_FGCG1_TARGVF__GRBMH_SPI_reg_clken_CHICK_BIT_MASK                                                0x20000000L
+#define GRBMH_FGCG1_TARGVF__GRBM_WGS_reg_clken_CHICK_BIT_MASK                                                 0x40000000L
+#define GRBMH_FGCG1_TARGVF__GRBMH_ALWAYSON_reg_clken_CHICK_BIT_MASK                                           0x80000000L
+//GRBMH_NOWHERE
+#define GRBMH_NOWHERE__DATA__SHIFT                                                                            0x0
+#define GRBMH_NOWHERE__DATA_MASK                                                                              0xFFFFFFFFL
+//GRBMH_INVALID_PIPE
+#define GRBMH_INVALID_PIPE__ADDR__SHIFT                                                                       0x2
+#define GRBMH_INVALID_PIPE__PIPEID__SHIFT                                                                     0x14
+#define GRBMH_INVALID_PIPE__MEID__SHIFT                                                                       0x16
+#define GRBMH_INVALID_PIPE__QUEUEID__SHIFT                                                                    0x18
+#define GRBMH_INVALID_PIPE__SSRCID__SHIFT                                                                     0x1b
+#define GRBMH_INVALID_PIPE__INVALID_PIPE__SHIFT                                                               0x1f
+#define GRBMH_INVALID_PIPE__ADDR_MASK                                                                         0x000FFFFCL
+#define GRBMH_INVALID_PIPE__PIPEID_MASK                                                                       0x00300000L
+#define GRBMH_INVALID_PIPE__MEID_MASK                                                                         0x00C00000L
+#define GRBMH_INVALID_PIPE__QUEUEID_MASK                                                                      0x07000000L
+#define GRBMH_INVALID_PIPE__SSRCID_MASK                                                                       0x78000000L
+#define GRBMH_INVALID_PIPE__INVALID_PIPE_MASK                                                                 0x80000000L
+//GRBMH_SYNC
+#define GRBMH_SYNC__GFX_PIPE0_PERFMON_SYNC__SHIFT                                                             0x0
+#define GRBMH_SYNC__GFX_PIPE0_SYNC__SHIFT                                                                     0x10
+#define GRBMH_SYNC__GFX_SYNC_SET_CLR__SHIFT                                                                   0x1f
+#define GRBMH_SYNC__GFX_PIPE0_PERFMON_SYNC_MASK                                                               0x00000001L
+#define GRBMH_SYNC__GFX_PIPE0_SYNC_MASK                                                                       0x00010000L
+#define GRBMH_SYNC__GFX_SYNC_SET_CLR_MASK                                                                     0x80000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_padec
+//GE_RATE_CNTL_1
+#define GE_RATE_CNTL_1__ADD_X_CLKS_LS_VERT__SHIFT                                                             0x0
+#define GE_RATE_CNTL_1__AFTER_Y_TRANS_LS_VERT__SHIFT                                                          0x4
+#define GE_RATE_CNTL_1__ADD_X_CLKS_HS_VERT__SHIFT                                                             0x8
+#define GE_RATE_CNTL_1__AFTER_Y_TRANS_HS_VERT__SHIFT                                                          0xc
+#define GE_RATE_CNTL_1__ADD_X_CLKS_ES_VERT__SHIFT                                                             0x10
+#define GE_RATE_CNTL_1__AFTER_Y_TRANS_ES_VERT__SHIFT                                                          0x14
+#define GE_RATE_CNTL_1__ADD_X_CLKS_GS_PRIM__SHIFT                                                             0x18
+#define GE_RATE_CNTL_1__AFTER_Y_TRANS_GS_PRIM__SHIFT                                                          0x1c
+#define GE_RATE_CNTL_1__ADD_X_CLKS_LS_VERT_MASK                                                               0x0000000FL
+#define GE_RATE_CNTL_1__AFTER_Y_TRANS_LS_VERT_MASK                                                            0x000000F0L
+#define GE_RATE_CNTL_1__ADD_X_CLKS_HS_VERT_MASK                                                               0x00000F00L
+#define GE_RATE_CNTL_1__AFTER_Y_TRANS_HS_VERT_MASK                                                            0x0000F000L
+#define GE_RATE_CNTL_1__ADD_X_CLKS_ES_VERT_MASK                                                               0x000F0000L
+#define GE_RATE_CNTL_1__AFTER_Y_TRANS_ES_VERT_MASK                                                            0x00F00000L
+#define GE_RATE_CNTL_1__ADD_X_CLKS_GS_PRIM_MASK                                                               0x0F000000L
+#define GE_RATE_CNTL_1__AFTER_Y_TRANS_GS_PRIM_MASK                                                            0xF0000000L
+//GE_RATE_CNTL_2
+#define GE_RATE_CNTL_2__ADD_X_CLKS_MERGED_HS_GS__SHIFT                                                        0x10
+#define GE_RATE_CNTL_2__ADD_X_CLKS_MERGED_LS_ES__SHIFT                                                        0x14
+#define GE_RATE_CNTL_2__MERGED_HS_GS_MODE__SHIFT                                                              0x18
+#define GE_RATE_CNTL_2__MERGED_LS_ES_MODE__SHIFT                                                              0x19
+#define GE_RATE_CNTL_2__ENABLE_RATE_CNTL__SHIFT                                                               0x1a
+#define GE_RATE_CNTL_2__SWAP_PRIORITY__SHIFT                                                                  0x1b
+#define GE_RATE_CNTL_2__ADD_X_CLKS_MERGED_HS_GS_MASK                                                          0x000F0000L
+#define GE_RATE_CNTL_2__ADD_X_CLKS_MERGED_LS_ES_MASK                                                          0x00F00000L
+#define GE_RATE_CNTL_2__MERGED_HS_GS_MODE_MASK                                                                0x01000000L
+#define GE_RATE_CNTL_2__MERGED_LS_ES_MODE_MASK                                                                0x02000000L
+#define GE_RATE_CNTL_2__ENABLE_RATE_CNTL_MASK                                                                 0x04000000L
+#define GE_RATE_CNTL_2__SWAP_PRIORITY_MASK                                                                    0x08000000L
+//CC_GC_SHADER_ARRAY_CONFIG
+#define CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT                                                       0x10
+#define CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK                                                         0xFFFF0000L
+//GE_SE_CNTL_STATUS
+#define GE_SE_CNTL_STATUS__TE_BUSY__SHIFT                                                                     0x0
+#define GE_SE_CNTL_STATUS__NGG_BUSY__SHIFT                                                                    0x1
+#define GE_SE_CNTL_STATUS__HS_BUSY__SHIFT                                                                     0x2
+#define GE_SE_CNTL_STATUS__TE_BUSY_MASK                                                                       0x00000001L
+#define GE_SE_CNTL_STATUS__NGG_BUSY_MASK                                                                      0x00000002L
+#define GE_SE_CNTL_STATUS__HS_BUSY_MASK                                                                       0x00000004L
+//GE_SPI_IF_SAFE_REG
+#define GE_SPI_IF_SAFE_REG__GE_SPI_LS_ES_DATA__SHIFT                                                          0x0
+#define GE_SPI_IF_SAFE_REG__GE_SPI_HS_GS_DATA__SHIFT                                                          0x6
+#define GE_SPI_IF_SAFE_REG__GE_SPI_GRP__SHIFT                                                                 0xc
+#define GE_SPI_IF_SAFE_REG__GE_SPI_LS_ES_DATA_MASK                                                            0x0000003FL
+#define GE_SPI_IF_SAFE_REG__GE_SPI_HS_GS_DATA_MASK                                                            0x00000FC0L
+#define GE_SPI_IF_SAFE_REG__GE_SPI_GRP_MASK                                                                   0x0003F000L
+//GE_PA_IF_SAFE_REG
+#define GE_PA_IF_SAFE_REG__GE_PA_CSB__SHIFT                                                                   0x0
+#define GE_PA_IF_SAFE_REG__GE_PA_PAYLOAD__SHIFT                                                               0xa
+#define GE_PA_IF_SAFE_REG__GE_PA_CSB_MASK                                                                     0x000003FFL
+#define GE_PA_IF_SAFE_REG__GE_PA_PAYLOAD_MASK                                                                 0x000FFC00L
+//PA_SU_DEBUG_CNTL
+#define PA_SU_DEBUG_CNTL__SU_DEBUG_INDX__SHIFT                                                                0x0
+#define PA_SU_DEBUG_CNTL__SU_DEBUG_INDX_MASK                                                                  0x0000003FL
+//PA_CL_CNTL_STATUS
+#define PA_CL_CNTL_STATUS__CL_BUSY__SHIFT                                                                     0x1f
+#define PA_CL_CNTL_STATUS__CL_BUSY_MASK                                                                       0x80000000L
+//PA_CL_ENHANCE
+#define PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA__SHIFT                                                            0x0
+#define PA_CL_ENHANCE__NUM_CLIP_SEQ__SHIFT                                                                    0x1
+#define PA_CL_ENHANCE__CLIPPED_PRIM_SEQ_STALL__SHIFT                                                          0x3
+#define PA_CL_ENHANCE__VE_NAN_PROC_DISABLE__SHIFT                                                             0x4
+#define PA_CL_ENHANCE__XTRA_DEBUG_REG_SEL__SHIFT                                                              0x5
+#define PA_CL_ENHANCE__IGNORE_PIPELINE_RESET__SHIFT                                                           0x6
+#define PA_CL_ENHANCE__KILL_INNER_EDGE_FLAGS__SHIFT                                                           0x7
+#define PA_CL_ENHANCE__NGG_PA_TO_ALL_SC__SHIFT                                                                0x8
+#define PA_CL_ENHANCE__TC_LATENCY_TIME_STAMP_RESOLUTION__SHIFT                                                0x9
+#define PA_CL_ENHANCE__NGG_BYPASS_PRIM_FILTER__SHIFT                                                          0xb
+#define PA_CL_ENHANCE__NGG_SIDEBAND_MEMORY_DEPTH__SHIFT                                                       0xc
+#define PA_CL_ENHANCE__NGG_PRIM_INDICES_FIFO_DEPTH__SHIFT                                                     0xe
+#define PA_CL_ENHANCE__PROG_NEAR_CLIP_PLANE_ENABLE__SHIFT                                                     0x11
+#define PA_CL_ENHANCE__POLY_INNER_EDGE_FLAG_DISABLE__SHIFT                                                    0x12
+#define PA_CL_ENHANCE__TC_REQUEST_PERF_CNTR_ENABLE__SHIFT                                                     0x13
+#define PA_CL_ENHANCE__DISABLE_PA_PH_INTF_FINE_CLOCK_GATE__SHIFT                                              0x14
+#define PA_CL_ENHANCE__DISABLE_PA_SX_REQ_INTF_FINE_CLOCK_GATE__SHIFT                                          0x15
+#define PA_CL_ENHANCE__ENABLE_PA_RATE_CNTL__SHIFT                                                             0x16
+#define PA_CL_ENHANCE__CLAMP_NEGATIVE_BB_TO_ZERO__SHIFT                                                       0x17
+#define PA_CL_ENHANCE__PA_W_GL1X_SRC_CLK_OVERRIDE__SHIFT                                                      0x18
+#define PA_CL_ENHANCE__PA_W_GL1X_REQ_CLK_OVERRIDE__SHIFT                                                      0x19
+#define PA_CL_ENHANCE__PA_R_GL1X_REQ_CLK_OVERRIDE__SHIFT                                                      0x1a
+#define PA_CL_ENHANCE__PAF_GEWD_CSB_CLK_OVERRIDE__SHIFT                                                       0x1b
+#define PA_CL_ENHANCE__BROADCAST_PMODE_PRIMS__SHIFT                                                           0x1c
+#define PA_CL_ENHANCE__BROADCAST_PERP_ENDCAP_PRIMS__SHIFT                                                     0x1d
+#define PA_CL_ENHANCE__ECO_SPARE1__SHIFT                                                                      0x1e
+#define PA_CL_ENHANCE__ECO_SPARE0__SHIFT                                                                      0x1f
+#define PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA_MASK                                                              0x00000001L
+#define PA_CL_ENHANCE__NUM_CLIP_SEQ_MASK                                                                      0x00000006L
+#define PA_CL_ENHANCE__CLIPPED_PRIM_SEQ_STALL_MASK                                                            0x00000008L
+#define PA_CL_ENHANCE__VE_NAN_PROC_DISABLE_MASK                                                               0x00000010L
+#define PA_CL_ENHANCE__XTRA_DEBUG_REG_SEL_MASK                                                                0x00000020L
+#define PA_CL_ENHANCE__IGNORE_PIPELINE_RESET_MASK                                                             0x00000040L
+#define PA_CL_ENHANCE__KILL_INNER_EDGE_FLAGS_MASK                                                             0x00000080L
+#define PA_CL_ENHANCE__NGG_PA_TO_ALL_SC_MASK                                                                  0x00000100L
+#define PA_CL_ENHANCE__TC_LATENCY_TIME_STAMP_RESOLUTION_MASK                                                  0x00000600L
+#define PA_CL_ENHANCE__NGG_BYPASS_PRIM_FILTER_MASK                                                            0x00000800L
+#define PA_CL_ENHANCE__NGG_SIDEBAND_MEMORY_DEPTH_MASK                                                         0x00003000L
+#define PA_CL_ENHANCE__NGG_PRIM_INDICES_FIFO_DEPTH_MASK                                                       0x0001C000L
+#define PA_CL_ENHANCE__PROG_NEAR_CLIP_PLANE_ENABLE_MASK                                                       0x00020000L
+#define PA_CL_ENHANCE__POLY_INNER_EDGE_FLAG_DISABLE_MASK                                                      0x00040000L
+#define PA_CL_ENHANCE__TC_REQUEST_PERF_CNTR_ENABLE_MASK                                                       0x00080000L
+#define PA_CL_ENHANCE__DISABLE_PA_PH_INTF_FINE_CLOCK_GATE_MASK                                                0x00100000L
+#define PA_CL_ENHANCE__DISABLE_PA_SX_REQ_INTF_FINE_CLOCK_GATE_MASK                                            0x00200000L
+#define PA_CL_ENHANCE__ENABLE_PA_RATE_CNTL_MASK                                                               0x00400000L
+#define PA_CL_ENHANCE__CLAMP_NEGATIVE_BB_TO_ZERO_MASK                                                         0x00800000L
+#define PA_CL_ENHANCE__PA_W_GL1X_SRC_CLK_OVERRIDE_MASK                                                        0x01000000L
+#define PA_CL_ENHANCE__PA_W_GL1X_REQ_CLK_OVERRIDE_MASK                                                        0x02000000L
+#define PA_CL_ENHANCE__PA_R_GL1X_REQ_CLK_OVERRIDE_MASK                                                        0x04000000L
+#define PA_CL_ENHANCE__PAF_GEWD_CSB_CLK_OVERRIDE_MASK                                                         0x08000000L
+#define PA_CL_ENHANCE__BROADCAST_PMODE_PRIMS_MASK                                                             0x10000000L
+#define PA_CL_ENHANCE__BROADCAST_PERP_ENDCAP_PRIMS_MASK                                                       0x20000000L
+#define PA_CL_ENHANCE__ECO_SPARE1_MASK                                                                        0x40000000L
+#define PA_CL_ENHANCE__ECO_SPARE0_MASK                                                                        0x80000000L
+//PA_CL_RESET_DEBUG
+#define PA_CL_RESET_DEBUG__CL_TRIV_DISC_DISABLE__SHIFT                                                        0x0
+#define PA_CL_RESET_DEBUG__DISABLE_PAF_WR_SHUNT__SHIFT                                                        0x1
+#define PA_CL_RESET_DEBUG__DISABLE_PAB_RD_SHUNT__SHIFT                                                        0x2
+#define PA_CL_RESET_DEBUG__CL_TRIV_DISC_DISABLE_MASK                                                          0x00000001L
+#define PA_CL_RESET_DEBUG__DISABLE_PAF_WR_SHUNT_MASK                                                          0x00000002L
+#define PA_CL_RESET_DEBUG__DISABLE_PAB_RD_SHUNT_MASK                                                          0x00000004L
+//PA_SU_CNTL_STATUS
+#define PA_SU_CNTL_STATUS__SU_BUSY__SHIFT                                                                     0x1f
+#define PA_SU_CNTL_STATUS__SU_BUSY_MASK                                                                       0x80000000L
+//PA_SC_FIFO_DEPTH_CNTL
+#define PA_SC_FIFO_DEPTH_CNTL__DEPTH__SHIFT                                                                   0x0
+#define PA_SC_FIFO_DEPTH_CNTL__DEPTH_MASK                                                                     0x000003FFL
+//PA_PH_DEBUG_CNTL
+#define PA_PH_DEBUG_CNTL__PH_DEBUG_INDX__SHIFT                                                                0x0
+#define PA_PH_DEBUG_CNTL__PH_DEBUG_CLEAR_ASSERT_ON_ERROR_BITS__SHIFT                                          0x8
+#define PA_PH_DEBUG_CNTL__PH_DEBUG_CLEAR_EVENT_HISTORY_DATA__SHIFT                                            0x9
+#define PA_PH_DEBUG_CNTL__PH_DEBUG_FILTER_VPZ_FLUSH_DFSM_AND_SOP_EVENTS_IN_EVENT_HISTORY_DATA__SHIFT          0xa
+#define PA_PH_DEBUG_CNTL__PH_DEBUG_FILTER_DEBUG_CNTL_EVENTS_IN_EVENT_HISTORY_DATA__SHIFT                      0xb
+#define PA_PH_DEBUG_CNTL__PH_DEBUG_FILTER_EVENT_0__SHIFT                                                      0xc
+#define PA_PH_DEBUG_CNTL__PH_DEBUG_FILTER_EVENT_1__SHIFT                                                      0x12
+#define PA_PH_DEBUG_CNTL__PH_DEBUG_FILTER_EVENT_2__SHIFT                                                      0x18
+#define PA_PH_DEBUG_CNTL__PH_DEBUG_INDX_MASK                                                                  0x000000FFL
+#define PA_PH_DEBUG_CNTL__PH_DEBUG_CLEAR_ASSERT_ON_ERROR_BITS_MASK                                            0x00000100L
+#define PA_PH_DEBUG_CNTL__PH_DEBUG_CLEAR_EVENT_HISTORY_DATA_MASK                                              0x00000200L
+#define PA_PH_DEBUG_CNTL__PH_DEBUG_FILTER_VPZ_FLUSH_DFSM_AND_SOP_EVENTS_IN_EVENT_HISTORY_DATA_MASK            0x00000400L
+#define PA_PH_DEBUG_CNTL__PH_DEBUG_FILTER_DEBUG_CNTL_EVENTS_IN_EVENT_HISTORY_DATA_MASK                        0x00000800L
+#define PA_PH_DEBUG_CNTL__PH_DEBUG_FILTER_EVENT_0_MASK                                                        0x0003F000L
+#define PA_PH_DEBUG_CNTL__PH_DEBUG_FILTER_EVENT_1_MASK                                                        0x00FC0000L
+#define PA_PH_DEBUG_CNTL__PH_DEBUG_FILTER_EVENT_2_MASK                                                        0x3F000000L
+//PA_SC_DEBUG_CNTL
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_INDX__SHIFT                                                                0x0
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_CLEAR_ASSERT_ON_ERROR_BITS__SHIFT                                          0x8
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_CLEAR_EVENT_HISTORY_DATA__SHIFT                                            0x9
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_FILTER_VPZ_FLUSH_DFSM_AND_SOP_EVENTS_IN_EVENT_HISTORY_DATA__SHIFT          0xa
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_FILTER_DEBUG_CNTL_EVENTS_IN_EVENT_HISTORY_DATA__SHIFT                      0xb
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_FILTER_EVENT_0__SHIFT                                                      0xc
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_FILTER_EVENT_1__SHIFT                                                      0x12
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_FILTER_EVENT_2__SHIFT                                                      0x18
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_BUS_FLOP_EN__SHIFT                                                         0x1e
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_INDX_MASK                                                                  0x000000FFL
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_CLEAR_ASSERT_ON_ERROR_BITS_MASK                                            0x00000100L
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_CLEAR_EVENT_HISTORY_DATA_MASK                                              0x00000200L
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_FILTER_VPZ_FLUSH_DFSM_AND_SOP_EVENTS_IN_EVENT_HISTORY_DATA_MASK            0x00000400L
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_FILTER_DEBUG_CNTL_EVENTS_IN_EVENT_HISTORY_DATA_MASK                        0x00000800L
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_FILTER_EVENT_0_MASK                                                        0x0003F000L
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_FILTER_EVENT_1_MASK                                                        0x00FC0000L
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_FILTER_EVENT_2_MASK                                                        0x3F000000L
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_BUS_FLOP_EN_MASK                                                           0x40000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_sqdec
+//SQ_CONFIG
+#define SQ_CONFIG__ECO_SPARE__SHIFT                                                                           0x0
+#define SQ_CONFIG__DISABLE_SGPR_RD_PERSIST__SHIFT                                                             0x7
+#define SQ_CONFIG__NEW_TRANS_ARB_SCHEME__SHIFT                                                                0x8
+#define SQ_CONFIG__DISABLE_VMEM_EXEC_ZERO_SKIP__SHIFT                                                         0x9
+#define SQ_CONFIG__DISABLE_SGPR_RD_KILL__SHIFT                                                                0xa
+#define SQ_CONFIG__REPLAY_SLEEP_CNT__SHIFT                                                                    0xb
+#define SQ_CONFIG__ENABLE_HIPRIO_ON_EXP_RDY_GS__SHIFT                                                         0x12
+#define SQ_CONFIG__PRIO_VAL_ON_EXP_RDY_GS__SHIFT                                                              0x13
+#define SQ_CONFIG__WCLK_HYSTERESIS_CNT__SHIFT                                                                 0x15
+#define SQ_CONFIG__DISABLE_ILLEGAL_OPCODE_DETECTION__SHIFT                                                    0x17
+#define SQ_CONFIG__DISABLE_ILLEGAL_TO_NOP_DETECTION__SHIFT                                                    0x18
+#define SQ_CONFIG__DISABLE_ILLEGAL_EXPORT_DETECTION__SHIFT                                                    0x19
+#define SQ_CONFIG__DISABLE_ILLEGAL_CLAUSE_DETECTION__SHIFT                                                    0x1a
+#define SQ_CONFIG__DISABLE_END_CLAUSE_TX__SHIFT                                                               0x1b
+#define SQ_CONFIG__DISABLE_DIFF_WAVE_VALU_COISSUE__SHIFT                                                      0x1c
+#define SQ_CONFIG__DISABLE_IHP_STALL__SHIFT                                                                   0x1d
+#define SQ_CONFIG__DISABLE_SP_SINGLE_ISSUE_WAVE64_TRANS__SHIFT                                                0x1e
+#define SQ_CONFIG__DISABLE_ISC_PREFETCH_LIMITER__SHIFT                                                        0x1f
+#define SQ_CONFIG__ECO_SPARE_MASK                                                                             0x0000007FL
+#define SQ_CONFIG__DISABLE_SGPR_RD_PERSIST_MASK                                                               0x00000080L
+#define SQ_CONFIG__NEW_TRANS_ARB_SCHEME_MASK                                                                  0x00000100L
+#define SQ_CONFIG__DISABLE_VMEM_EXEC_ZERO_SKIP_MASK                                                           0x00000200L
+#define SQ_CONFIG__DISABLE_SGPR_RD_KILL_MASK                                                                  0x00000400L
+#define SQ_CONFIG__REPLAY_SLEEP_CNT_MASK                                                                      0x0003F800L
+#define SQ_CONFIG__ENABLE_HIPRIO_ON_EXP_RDY_GS_MASK                                                           0x00040000L
+#define SQ_CONFIG__PRIO_VAL_ON_EXP_RDY_GS_MASK                                                                0x00180000L
+#define SQ_CONFIG__WCLK_HYSTERESIS_CNT_MASK                                                                   0x00600000L
+#define SQ_CONFIG__DISABLE_ILLEGAL_OPCODE_DETECTION_MASK                                                      0x00800000L
+#define SQ_CONFIG__DISABLE_ILLEGAL_TO_NOP_DETECTION_MASK                                                      0x01000000L
+#define SQ_CONFIG__DISABLE_ILLEGAL_EXPORT_DETECTION_MASK                                                      0x02000000L
+#define SQ_CONFIG__DISABLE_ILLEGAL_CLAUSE_DETECTION_MASK                                                      0x04000000L
+#define SQ_CONFIG__DISABLE_END_CLAUSE_TX_MASK                                                                 0x08000000L
+#define SQ_CONFIG__DISABLE_DIFF_WAVE_VALU_COISSUE_MASK                                                        0x10000000L
+#define SQ_CONFIG__DISABLE_IHP_STALL_MASK                                                                     0x20000000L
+#define SQ_CONFIG__DISABLE_SP_SINGLE_ISSUE_WAVE64_TRANS_MASK                                                  0x40000000L
+#define SQ_CONFIG__DISABLE_ISC_PREFETCH_LIMITER_MASK                                                          0x80000000L
+//SQC_CONFIG
+#define SQC_CONFIG__INST_CACHE_SIZE__SHIFT                                                                    0x0
+#define SQC_CONFIG__DATA_CACHE_SIZE__SHIFT                                                                    0x2
+#define SQC_CONFIG__MISS_FIFO_DEPTH__SHIFT                                                                    0x4
+#define SQC_CONFIG__HIT_FIFO_DEPTH__SHIFT                                                                     0x6
+#define SQC_CONFIG__FORCE_ALWAYS_MISS__SHIFT                                                                  0x7
+#define SQC_CONFIG__FORCE_IN_ORDER__SHIFT                                                                     0x8
+#define SQC_CONFIG__PER_VMID_INV_DISABLE__SHIFT                                                               0x9
+#define SQC_CONFIG__EVICT_LRU__SHIFT                                                                          0xa
+#define SQC_CONFIG__FORCE_2_BANK__SHIFT                                                                       0xc
+#define SQC_CONFIG__FORCE_1_BANK__SHIFT                                                                       0xd
+#define SQC_CONFIG__LS_DISABLE_CLOCKS__SHIFT                                                                  0xe
+#define SQC_CONFIG__CACHE_CTRL_GCR_FIX_DISABLE__SHIFT                                                         0x16
+#define SQC_CONFIG__CACHE_CTRL_ALMOST_MAX_INFLIGHT_CONFIG__SHIFT                                              0x17
+#define SQC_CONFIG__PIT_PM_STATUS_FGCG_OVERRIDE__SHIFT                                                        0x1a
+#define SQC_CONFIG__EXEC_POP_CNT_25PCT__SHIFT                                                                 0x1b
+#define SQC_CONFIG__SQC_SQ_INV_REG_GCR_SEL__SHIFT                                                             0x1c
+#define SQC_CONFIG__SPARE__SHIFT                                                                              0x1d
+#define SQC_CONFIG__INST_CACHE_SIZE_MASK                                                                      0x00000003L
+#define SQC_CONFIG__DATA_CACHE_SIZE_MASK                                                                      0x0000000CL
+#define SQC_CONFIG__MISS_FIFO_DEPTH_MASK                                                                      0x00000030L
+#define SQC_CONFIG__HIT_FIFO_DEPTH_MASK                                                                       0x00000040L
+#define SQC_CONFIG__FORCE_ALWAYS_MISS_MASK                                                                    0x00000080L
+#define SQC_CONFIG__FORCE_IN_ORDER_MASK                                                                       0x00000100L
+#define SQC_CONFIG__PER_VMID_INV_DISABLE_MASK                                                                 0x00000200L
+#define SQC_CONFIG__EVICT_LRU_MASK                                                                            0x00000C00L
+#define SQC_CONFIG__FORCE_2_BANK_MASK                                                                         0x00001000L
+#define SQC_CONFIG__FORCE_1_BANK_MASK                                                                         0x00002000L
+#define SQC_CONFIG__LS_DISABLE_CLOCKS_MASK                                                                    0x003FC000L
+#define SQC_CONFIG__CACHE_CTRL_GCR_FIX_DISABLE_MASK                                                           0x00400000L
+#define SQC_CONFIG__CACHE_CTRL_ALMOST_MAX_INFLIGHT_CONFIG_MASK                                                0x03800000L
+#define SQC_CONFIG__PIT_PM_STATUS_FGCG_OVERRIDE_MASK                                                          0x04000000L
+#define SQC_CONFIG__EXEC_POP_CNT_25PCT_MASK                                                                   0x08000000L
+#define SQC_CONFIG__SQC_SQ_INV_REG_GCR_SEL_MASK                                                               0x10000000L
+#define SQC_CONFIG__SPARE_MASK                                                                                0xE0000000L
+//SQ_RANDOM_WAVE_PRI
+#define SQ_RANDOM_WAVE_PRI__RET__SHIFT                                                                        0x0
+#define SQ_RANDOM_WAVE_PRI__RUI__SHIFT                                                                        0x7
+#define SQ_RANDOM_WAVE_PRI__RNG__SHIFT                                                                        0xa
+#define SQ_RANDOM_WAVE_PRI__FORCE_IB_ARB_PRIO_MSK_VALID__SHIFT                                                0x1f
+#define SQ_RANDOM_WAVE_PRI__RET_MASK                                                                          0x0000007FL
+#define SQ_RANDOM_WAVE_PRI__RUI_MASK                                                                          0x00000380L
+#define SQ_RANDOM_WAVE_PRI__RNG_MASK                                                                          0x00FFFC00L
+#define SQ_RANDOM_WAVE_PRI__FORCE_IB_ARB_PRIO_MSK_VALID_MASK                                                  0x80000000L
+//SQG_STATUS
+#define SQG_STATUS__REG_BUSY__SHIFT                                                                           0x0
+#define SQG_STATUS__POWEROFF_RESTORE__SHIFT                                                                   0x1
+#define SQG_STATUS__REG_BUSY_MASK                                                                             0x00000001L
+#define SQG_STATUS__POWEROFF_RESTORE_MASK                                                                     0x00000002L
+//SQ_FIFO_SIZES
+#define SQ_FIFO_SIZES__INTERRUPT_FIFO_SIZE__SHIFT                                                             0x0
+#define SQ_FIFO_SIZES__TTRACE_FIFO_SIZE__SHIFT                                                                0x8
+#define SQ_FIFO_SIZES__EXPORT_BUF_GS_RESERVED__SHIFT                                                          0xc
+#define SQ_FIFO_SIZES__EXPORT_BUF_PS_RESERVED__SHIFT                                                          0xe
+#define SQ_FIFO_SIZES__EXPORT_BUF_REDUCE__SHIFT                                                               0x10
+#define SQ_FIFO_SIZES__VMEM_DATA_FIFO_SIZE__SHIFT                                                             0x12
+#define SQ_FIFO_SIZES__EXPORT_BUF_PRIMPOS_LIMIT__SHIFT                                                        0x14
+#define SQ_FIFO_SIZES__INTERRUPT_FIFO_SIZE_MASK                                                               0x0000000FL
+#define SQ_FIFO_SIZES__TTRACE_FIFO_SIZE_MASK                                                                  0x00000300L
+#define SQ_FIFO_SIZES__EXPORT_BUF_GS_RESERVED_MASK                                                            0x00003000L
+#define SQ_FIFO_SIZES__EXPORT_BUF_PS_RESERVED_MASK                                                            0x0000C000L
+#define SQ_FIFO_SIZES__EXPORT_BUF_REDUCE_MASK                                                                 0x00030000L
+#define SQ_FIFO_SIZES__VMEM_DATA_FIFO_SIZE_MASK                                                               0x000C0000L
+#define SQ_FIFO_SIZES__EXPORT_BUF_PRIMPOS_LIMIT_MASK                                                          0x00300000L
+//SQ_DSM_CNTL
+#define SQ_DSM_CNTL__WAVEFRONT_STALL_0__SHIFT                                                                 0x0
+#define SQ_DSM_CNTL__WAVEFRONT_STALL_1__SHIFT                                                                 0x1
+#define SQ_DSM_CNTL__SPI_BACKPRESSURE_0__SHIFT                                                                0x2
+#define SQ_DSM_CNTL__SPI_BACKPRESSURE_1__SHIFT                                                                0x3
+#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA0__SHIFT                                                      0x8
+#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA1__SHIFT                                                      0x9
+#define SQ_DSM_CNTL__SGPR_ENABLE_SINGLE_WRITE__SHIFT                                                          0xa
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA0__SHIFT                                                       0x10
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA1__SHIFT                                                       0x11
+#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE01__SHIFT                                                         0x12
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA2__SHIFT                                                       0x13
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA3__SHIFT                                                       0x14
+#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE23__SHIFT                                                         0x15
+#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA0__SHIFT                                                        0x18
+#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA1__SHIFT                                                        0x19
+#define SQ_DSM_CNTL__SP_ENABLE_SINGLE_WRITE__SHIFT                                                            0x1a
+#define SQ_DSM_CNTL__WAVEFRONT_STALL_0_MASK                                                                   0x00000001L
+#define SQ_DSM_CNTL__WAVEFRONT_STALL_1_MASK                                                                   0x00000002L
+#define SQ_DSM_CNTL__SPI_BACKPRESSURE_0_MASK                                                                  0x00000004L
+#define SQ_DSM_CNTL__SPI_BACKPRESSURE_1_MASK                                                                  0x00000008L
+#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA0_MASK                                                        0x00000100L
+#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA1_MASK                                                        0x00000200L
+#define SQ_DSM_CNTL__SGPR_ENABLE_SINGLE_WRITE_MASK                                                            0x00000400L
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA0_MASK                                                         0x00010000L
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA1_MASK                                                         0x00020000L
+#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE01_MASK                                                           0x00040000L
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA2_MASK                                                         0x00080000L
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA3_MASK                                                         0x00100000L
+#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE23_MASK                                                           0x00200000L
+#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA0_MASK                                                          0x01000000L
+#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA1_MASK                                                          0x02000000L
+#define SQ_DSM_CNTL__SP_ENABLE_SINGLE_WRITE_MASK                                                              0x04000000L
+//SQ_DSM_CNTL2
+#define SQ_DSM_CNTL2__SGPR_ENABLE_ERROR_INJECT__SHIFT                                                         0x0
+#define SQ_DSM_CNTL2__SGPR_SELECT_INJECT_DELAY__SHIFT                                                         0x2
+#define SQ_DSM_CNTL2__LDS_D_ENABLE_ERROR_INJECT__SHIFT                                                        0x3
+#define SQ_DSM_CNTL2__LDS_D_SELECT_INJECT_DELAY__SHIFT                                                        0x5
+#define SQ_DSM_CNTL2__LDS_I_ENABLE_ERROR_INJECT__SHIFT                                                        0x6
+#define SQ_DSM_CNTL2__LDS_I_SELECT_INJECT_DELAY__SHIFT                                                        0x8
+#define SQ_DSM_CNTL2__SP_ENABLE_ERROR_INJECT__SHIFT                                                           0x9
+#define SQ_DSM_CNTL2__SP_SELECT_INJECT_DELAY__SHIFT                                                           0xb
+#define SQ_DSM_CNTL2__LDS_INJECT_DELAY__SHIFT                                                                 0xe
+#define SQ_DSM_CNTL2__SP_INJECT_DELAY__SHIFT                                                                  0x14
+#define SQ_DSM_CNTL2__SQ_INJECT_DELAY__SHIFT                                                                  0x1a
+#define SQ_DSM_CNTL2__SGPR_ENABLE_ERROR_INJECT_MASK                                                           0x00000003L
+#define SQ_DSM_CNTL2__SGPR_SELECT_INJECT_DELAY_MASK                                                           0x00000004L
+#define SQ_DSM_CNTL2__LDS_D_ENABLE_ERROR_INJECT_MASK                                                          0x00000018L
+#define SQ_DSM_CNTL2__LDS_D_SELECT_INJECT_DELAY_MASK                                                          0x00000020L
+#define SQ_DSM_CNTL2__LDS_I_ENABLE_ERROR_INJECT_MASK                                                          0x000000C0L
+#define SQ_DSM_CNTL2__LDS_I_SELECT_INJECT_DELAY_MASK                                                          0x00000100L
+#define SQ_DSM_CNTL2__SP_ENABLE_ERROR_INJECT_MASK                                                             0x00000600L
+#define SQ_DSM_CNTL2__SP_SELECT_INJECT_DELAY_MASK                                                             0x00000800L
+#define SQ_DSM_CNTL2__LDS_INJECT_DELAY_MASK                                                                   0x000FC000L
+#define SQ_DSM_CNTL2__SP_INJECT_DELAY_MASK                                                                    0x03F00000L
+#define SQ_DSM_CNTL2__SQ_INJECT_DELAY_MASK                                                                    0xFC000000L
+//SQ_PIT_WEIGHT
+#define SQ_PIT_WEIGHT__PIT_WEIGHT_GRP0__SHIFT                                                                 0x0
+#define SQ_PIT_WEIGHT__PIT_WEIGHT_GRP1__SHIFT                                                                 0x3
+#define SQ_PIT_WEIGHT__PIT_WEIGHT_GRP2__SHIFT                                                                 0x6
+#define SQ_PIT_WEIGHT__PIT_WEIGHT_GRP3__SHIFT                                                                 0x9
+#define SQ_PIT_WEIGHT__PIT_WEIGHT_GRP4__SHIFT                                                                 0xc
+#define SQ_PIT_WEIGHT__PIT_WEIGHT_GRP5__SHIFT                                                                 0xf
+#define SQ_PIT_WEIGHT__PIT_WEIGHT_GRP6__SHIFT                                                                 0x12
+#define SQ_PIT_WEIGHT__PIT_WEIGHT_GRP7__SHIFT                                                                 0x15
+#define SQ_PIT_WEIGHT__UNUSED__SHIFT                                                                          0x18
+#define SQ_PIT_WEIGHT__PIT_WEIGHT_GRP0_MASK                                                                   0x00000007L
+#define SQ_PIT_WEIGHT__PIT_WEIGHT_GRP1_MASK                                                                   0x00000038L
+#define SQ_PIT_WEIGHT__PIT_WEIGHT_GRP2_MASK                                                                   0x000001C0L
+#define SQ_PIT_WEIGHT__PIT_WEIGHT_GRP3_MASK                                                                   0x00000E00L
+#define SQ_PIT_WEIGHT__PIT_WEIGHT_GRP4_MASK                                                                   0x00007000L
+#define SQ_PIT_WEIGHT__PIT_WEIGHT_GRP5_MASK                                                                   0x00038000L
+#define SQ_PIT_WEIGHT__PIT_WEIGHT_GRP6_MASK                                                                   0x001C0000L
+#define SQ_PIT_WEIGHT__PIT_WEIGHT_GRP7_MASK                                                                   0x00E00000L
+#define SQ_PIT_WEIGHT__UNUSED_MASK                                                                            0xFF000000L
+//SQG_THREAD_TRACE_CONFIG
+#define SQG_THREAD_TRACE_CONFIG__ALL_VMID__SHIFT                                                              0x0
+#define SQG_THREAD_TRACE_CONFIG__ALL_VMID_MASK                                                                0x00000001L
+//SP_CONFIG
+#define SP_CONFIG__ECO_SPARE__SHIFT                                                                           0x0
+#define SP_CONFIG__MGCG_SYNC_PULSE__SHIFT                                                                     0x1
+#define SP_CONFIG__DISABLE_MGCG_OVERSHOOT_MITIGATION__SHIFT                                                   0x2
+#define SP_CONFIG__DISABLE_TRANS_COEXEC__SHIFT                                                                0x3
+#define SP_CONFIG__CAC_COUNTER_OVERRIDE__SHIFT                                                                0x4
+#define SP_CONFIG__SP_SX_EXPVDATA_FGCG_OVERRIDE__SHIFT                                                        0x5
+#define SP_CONFIG__DISABLE_XDL_COEXEC__SHIFT                                                                  0x6
+#define SP_CONFIG__ECO_SPARE_2__SHIFT                                                                         0x7
+#define SP_CONFIG__ECO_SPARE_MASK                                                                             0x00000001L
+#define SP_CONFIG__MGCG_SYNC_PULSE_MASK                                                                       0x00000002L
+#define SP_CONFIG__DISABLE_MGCG_OVERSHOOT_MITIGATION_MASK                                                     0x00000004L
+#define SP_CONFIG__DISABLE_TRANS_COEXEC_MASK                                                                  0x00000008L
+#define SP_CONFIG__CAC_COUNTER_OVERRIDE_MASK                                                                  0x00000010L
+#define SP_CONFIG__SP_SX_EXPVDATA_FGCG_OVERRIDE_MASK                                                          0x00000020L
+#define SP_CONFIG__DISABLE_XDL_COEXEC_MASK                                                                    0x00000040L
+#define SP_CONFIG__ECO_SPARE_2_MASK                                                                           0x00000380L
+//SQ_ARB_CONFIG
+#define SQ_ARB_CONFIG__WG_RR_INTERVAL__SHIFT                                                                  0x0
+#define SQ_ARB_CONFIG__FWD_PROG_INTERVAL__SHIFT                                                               0x4
+#define SQ_ARB_CONFIG__WG_RR_INTERVAL_MASK                                                                    0x00000003L
+#define SQ_ARB_CONFIG__FWD_PROG_INTERVAL_MASK                                                                 0x00000030L
+//SQ_DYN_VGPR
+#define SQ_DYN_VGPR__WAVE_LIMIT__SHIFT                                                                        0x0
+#define SQ_DYN_VGPR__FWD_PROGRESS__SHIFT                                                                      0x4
+#define SQ_DYN_VGPR__MAX_BLOCK_ALLOC__SHIFT                                                                   0x5
+#define SQ_DYN_VGPR__BLOCK_SIZE__SHIFT                                                                        0x8
+#define SQ_DYN_VGPR__WAVE_LIMIT_MASK                                                                          0x0000000FL
+#define SQ_DYN_VGPR__FWD_PROGRESS_MASK                                                                        0x00000010L
+#define SQ_DYN_VGPR__MAX_BLOCK_ALLOC_MASK                                                                     0x000000E0L
+#define SQ_DYN_VGPR__BLOCK_SIZE_MASK                                                                          0x00000100L
+//SQ_SHADER_CYCLES_LO
+#define SQ_SHADER_CYCLES_LO__CYCLES_LO__SHIFT                                                                 0x0
+#define SQ_SHADER_CYCLES_LO__CYCLES_LO_MASK                                                                   0xFFFFFFFFL
+//SQ_SHADER_CYCLES_HI
+#define SQ_SHADER_CYCLES_HI__CYCLES_HI__SHIFT                                                                 0x0
+#define SQ_SHADER_CYCLES_HI__CYCLES_HI_MASK                                                                   0x0FFFFFFFL
+//SQ_DEBUG_HOST_TRAP_STATUS
+#define SQ_DEBUG_HOST_TRAP_STATUS__PENDING_COUNT__SHIFT                                                       0x0
+#define SQ_DEBUG_HOST_TRAP_STATUS__PENDING_COUNT_MASK                                                         0x0000007FL
+//SQG_GL1X_CTRL
+#define SQG_GL1X_CTRL__TEMPORAL__SHIFT                                                                        0x0
+#define SQG_GL1X_CTRL__SCOPE__SHIFT                                                                           0x4
+#define SQG_GL1X_CTRL__TEMPORAL_MASK                                                                          0x00000007L
+#define SQG_GL1X_CTRL__SCOPE_MASK                                                                             0x00000030L
+//SQG_GL1X_STATUS
+#define SQG_GL1X_STATUS__ACK_ERR_DETECTED__SHIFT                                                              0x0
+#define SQG_GL1X_STATUS__XNACK_ERR_DETECTED__SHIFT                                                            0x1
+#define SQG_GL1X_STATUS__ACK_ERR_DETECTED_MASK                                                                0x00000001L
+#define SQG_GL1X_STATUS__XNACK_ERR_DETECTED_MASK                                                              0x00000002L
+//SQG_CONFIG
+#define SQG_CONFIG__SQG_ICPFT_EN__SHIFT                                                                       0x0
+#define SQG_CONFIG__SQG_ICPFT_CLR__SHIFT                                                                      0x1
+#define SQG_CONFIG__XNACK_INTR_MASK__SHIFT                                                                    0x10
+#define SQG_CONFIG__SQG_ICPFT_EN_MASK                                                                         0x00000001L
+#define SQG_CONFIG__SQG_ICPFT_CLR_MASK                                                                        0x00000002L
+#define SQG_CONFIG__XNACK_INTR_MASK_MASK                                                                      0xFFFF0000L
+//SQ_PERF_SNAPSHOT_CTRL
+#define SQ_PERF_SNAPSHOT_CTRL__TIMER_ON_OFF__SHIFT                                                            0x0
+#define SQ_PERF_SNAPSHOT_CTRL__VMID_MASK__SHIFT                                                               0x1
+#define SQ_PERF_SNAPSHOT_CTRL__COUNT_SEL__SHIFT                                                               0x11
+#define SQ_PERF_SNAPSHOT_CTRL__COUNT_INTERVAL__SHIFT                                                          0x12
+#define SQ_PERF_SNAPSHOT_CTRL__TEST_MODE__SHIFT                                                               0x16
+#define SQ_PERF_SNAPSHOT_CTRL__RANDOM_SEED__SHIFT                                                             0x17
+#define SQ_PERF_SNAPSHOT_CTRL__TIMER_ON_OFF_MASK                                                              0x00000001L
+#define SQ_PERF_SNAPSHOT_CTRL__VMID_MASK_MASK                                                                 0x0001FFFEL
+#define SQ_PERF_SNAPSHOT_CTRL__COUNT_SEL_MASK                                                                 0x00020000L
+#define SQ_PERF_SNAPSHOT_CTRL__COUNT_INTERVAL_MASK                                                            0x003C0000L
+#define SQ_PERF_SNAPSHOT_CTRL__TEST_MODE_MASK                                                                 0x00400000L
+#define SQ_PERF_SNAPSHOT_CTRL__RANDOM_SEED_MASK                                                               0x07800000L
+//CC_GC_SHADER_RATE_CONFIG_1
+#define CC_GC_SHADER_RATE_CONFIG_1__SHADER_RATE__SHIFT                                                        0x1
+#define CC_GC_SHADER_RATE_CONFIG_1__SHADER_RATE_MASK                                                          0x000007FEL
+//SQ_INTERRUPT_AUTO_MASK
+#define SQ_INTERRUPT_AUTO_MASK__MASK__SHIFT                                                                   0x0
+#define SQ_INTERRUPT_AUTO_MASK__MASK_MASK                                                                     0x00FFFFFFL
+//SQ_INTERRUPT_MSG_CTRL
+#define SQ_INTERRUPT_MSG_CTRL__STALL__SHIFT                                                                   0x0
+#define SQ_INTERRUPT_MSG_CTRL__STALL_MASK                                                                     0x00000001L
+//SQ_WATCH0_ADDR_H
+#define SQ_WATCH0_ADDR_H__ADDR__SHIFT                                                                         0x0
+#define SQ_WATCH0_ADDR_H__ADDR_MASK                                                                           0x01FFFFFFL
+//SQ_WATCH0_ADDR_L
+#define SQ_WATCH0_ADDR_L__ADDR__SHIFT                                                                         0x6
+#define SQ_WATCH0_ADDR_L__ADDR_MASK                                                                           0xFFFFFFC0L
+//SQ_WATCH0_CNTL
+#define SQ_WATCH0_CNTL__MASK__SHIFT                                                                           0x0
+#define SQ_WATCH0_CNTL__VMID__SHIFT                                                                           0x18
+#define SQ_WATCH0_CNTL__VALID__SHIFT                                                                          0x1f
+#define SQ_WATCH0_CNTL__MASK_MASK                                                                             0x00FFFFFFL
+#define SQ_WATCH0_CNTL__VMID_MASK                                                                             0x0F000000L
+#define SQ_WATCH0_CNTL__VALID_MASK                                                                            0x80000000L
+//SQ_WATCH1_ADDR_H
+#define SQ_WATCH1_ADDR_H__ADDR__SHIFT                                                                         0x0
+#define SQ_WATCH1_ADDR_H__ADDR_MASK                                                                           0x01FFFFFFL
+//SQ_WATCH1_ADDR_L
+#define SQ_WATCH1_ADDR_L__ADDR__SHIFT                                                                         0x6
+#define SQ_WATCH1_ADDR_L__ADDR_MASK                                                                           0xFFFFFFC0L
+//SQ_WATCH1_CNTL
+#define SQ_WATCH1_CNTL__MASK__SHIFT                                                                           0x0
+#define SQ_WATCH1_CNTL__VMID__SHIFT                                                                           0x18
+#define SQ_WATCH1_CNTL__VALID__SHIFT                                                                          0x1f
+#define SQ_WATCH1_CNTL__MASK_MASK                                                                             0x00FFFFFFL
+#define SQ_WATCH1_CNTL__VMID_MASK                                                                             0x0F000000L
+#define SQ_WATCH1_CNTL__VALID_MASK                                                                            0x80000000L
+//SQ_WATCH2_ADDR_H
+#define SQ_WATCH2_ADDR_H__ADDR__SHIFT                                                                         0x0
+#define SQ_WATCH2_ADDR_H__ADDR_MASK                                                                           0x01FFFFFFL
+//SQ_WATCH2_ADDR_L
+#define SQ_WATCH2_ADDR_L__ADDR__SHIFT                                                                         0x6
+#define SQ_WATCH2_ADDR_L__ADDR_MASK                                                                           0xFFFFFFC0L
+//SQ_WATCH2_CNTL
+#define SQ_WATCH2_CNTL__MASK__SHIFT                                                                           0x0
+#define SQ_WATCH2_CNTL__VMID__SHIFT                                                                           0x18
+#define SQ_WATCH2_CNTL__VALID__SHIFT                                                                          0x1f
+#define SQ_WATCH2_CNTL__MASK_MASK                                                                             0x00FFFFFFL
+#define SQ_WATCH2_CNTL__VMID_MASK                                                                             0x0F000000L
+#define SQ_WATCH2_CNTL__VALID_MASK                                                                            0x80000000L
+//SQ_WATCH3_ADDR_H
+#define SQ_WATCH3_ADDR_H__ADDR__SHIFT                                                                         0x0
+#define SQ_WATCH3_ADDR_H__ADDR_MASK                                                                           0x01FFFFFFL
+//SQ_WATCH3_ADDR_L
+#define SQ_WATCH3_ADDR_L__ADDR__SHIFT                                                                         0x6
+#define SQ_WATCH3_ADDR_L__ADDR_MASK                                                                           0xFFFFFFC0L
+//SQ_WATCH3_CNTL
+#define SQ_WATCH3_CNTL__MASK__SHIFT                                                                           0x0
+#define SQ_WATCH3_CNTL__VMID__SHIFT                                                                           0x18
+#define SQ_WATCH3_CNTL__VALID__SHIFT                                                                          0x1f
+#define SQ_WATCH3_CNTL__MASK_MASK                                                                             0x00FFFFFFL
+#define SQ_WATCH3_CNTL__VMID_MASK                                                                             0x0F000000L
+#define SQ_WATCH3_CNTL__VALID_MASK                                                                            0x80000000L
+//SQG_UTCL0_STATUS
+#define SQG_UTCL0_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define SQG_UTCL0_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define SQG_UTCL0_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define SQG_UTCL0_STATUS__RESERVED__SHIFT                                                                     0x3
+#define SQG_UTCL0_STATUS__UNUSED__SHIFT                                                                       0x8
+#define SQG_UTCL0_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define SQG_UTCL0_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define SQG_UTCL0_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+#define SQG_UTCL0_STATUS__RESERVED_MASK                                                                       0x000000F8L
+#define SQG_UTCL0_STATUS__UNUSED_MASK                                                                         0xFFFFFF00L
+//SQG_UTCL0_CNTL1
+#define SQG_UTCL0_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                              0x0
+#define SQG_UTCL0_CNTL1__GPUVM_64K_DEFAULT__SHIFT                                                             0x1
+#define SQG_UTCL0_CNTL1__GPUVM_PERM_MODE__SHIFT                                                               0x2
+#define SQG_UTCL0_CNTL1__RESP_MODE__SHIFT                                                                     0x3
+#define SQG_UTCL0_CNTL1__RESP_FAULT_MODE__SHIFT                                                               0x5
+#define SQG_UTCL0_CNTL1__CLIENTID__SHIFT                                                                      0x7
+#define SQG_UTCL0_CNTL1__REG_INV_VMID__SHIFT                                                                  0x13
+#define SQG_UTCL0_CNTL1__REG_INV_TOGGLE__SHIFT                                                                0x18
+#define SQG_UTCL0_CNTL1__ATOMIC_REQUEST_ENABLE__SHIFT                                                         0x19
+#define SQG_UTCL0_CNTL1__FORCE_MISS__SHIFT                                                                    0x1a
+#define SQG_UTCL0_CNTL1__FORCE_IN_ORDER__SHIFT                                                                0x1b
+#define SQG_UTCL0_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                        0x1c
+#define SQG_UTCL0_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                        0x1e
+#define SQG_UTCL0_CNTL1__FORCE_4K_L2_RESP_MASK                                                                0x00000001L
+#define SQG_UTCL0_CNTL1__GPUVM_64K_DEFAULT_MASK                                                               0x00000002L
+#define SQG_UTCL0_CNTL1__GPUVM_PERM_MODE_MASK                                                                 0x00000004L
+#define SQG_UTCL0_CNTL1__RESP_MODE_MASK                                                                       0x00000018L
+#define SQG_UTCL0_CNTL1__RESP_FAULT_MODE_MASK                                                                 0x00000060L
+#define SQG_UTCL0_CNTL1__CLIENTID_MASK                                                                        0x0000FF80L
+#define SQG_UTCL0_CNTL1__REG_INV_VMID_MASK                                                                    0x00780000L
+#define SQG_UTCL0_CNTL1__REG_INV_TOGGLE_MASK                                                                  0x01000000L
+#define SQG_UTCL0_CNTL1__ATOMIC_REQUEST_ENABLE_MASK                                                           0x02000000L
+#define SQG_UTCL0_CNTL1__FORCE_MISS_MASK                                                                      0x04000000L
+#define SQG_UTCL0_CNTL1__FORCE_IN_ORDER_MASK                                                                  0x08000000L
+#define SQG_UTCL0_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                          0x30000000L
+#define SQG_UTCL0_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                          0xC0000000L
+//SQG_UTCL0_CNTL2
+#define SQG_UTCL0_CNTL2__SPARE__SHIFT                                                                         0x0
+#define SQG_UTCL0_CNTL2__COMP_SYNC_DISABLE__SHIFT                                                             0x8
+#define SQG_UTCL0_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                                0x9
+#define SQG_UTCL0_CNTL2__ANY_LINE_VALID__SHIFT                                                                0xa
+#define SQG_UTCL0_CNTL2__FORCE_SNOOP__SHIFT                                                                   0xe
+#define SQG_UTCL0_CNTL2__DISABLE_BURST__SHIFT                                                                 0x11
+#define SQG_UTCL0_CNTL2__ENABLE_PERF_EVENT_RD_WR__SHIFT                                                       0x12
+#define SQG_UTCL0_CNTL2__PERF_EVENT_RD_WR__SHIFT                                                              0x13
+#define SQG_UTCL0_CNTL2__ENABLE_PERF_EVENT_VMID__SHIFT                                                        0x14
+#define SQG_UTCL0_CNTL2__PERF_EVENT_VMID__SHIFT                                                               0x15
+#define SQG_UTCL0_CNTL2__FORCE_FRAG_2M_TO_64K__SHIFT                                                          0x1a
+#define SQG_UTCL0_CNTL2__FGCG_DISABLE__SHIFT                                                                  0x1e
+#define SQG_UTCL0_CNTL2__BIG_PAGE_DISABLE__SHIFT                                                              0x1f
+#define SQG_UTCL0_CNTL2__SPARE_MASK                                                                           0x000000FFL
+#define SQG_UTCL0_CNTL2__COMP_SYNC_DISABLE_MASK                                                               0x00000100L
+#define SQG_UTCL0_CNTL2__MTYPE_OVRD_DIS_MASK                                                                  0x00000200L
+#define SQG_UTCL0_CNTL2__ANY_LINE_VALID_MASK                                                                  0x00000400L
+#define SQG_UTCL0_CNTL2__FORCE_SNOOP_MASK                                                                     0x00004000L
+#define SQG_UTCL0_CNTL2__DISABLE_BURST_MASK                                                                   0x00020000L
+#define SQG_UTCL0_CNTL2__ENABLE_PERF_EVENT_RD_WR_MASK                                                         0x00040000L
+#define SQG_UTCL0_CNTL2__PERF_EVENT_RD_WR_MASK                                                                0x00080000L
+#define SQG_UTCL0_CNTL2__ENABLE_PERF_EVENT_VMID_MASK                                                          0x00100000L
+#define SQG_UTCL0_CNTL2__PERF_EVENT_VMID_MASK                                                                 0x01E00000L
+#define SQG_UTCL0_CNTL2__FORCE_FRAG_2M_TO_64K_MASK                                                            0x04000000L
+#define SQG_UTCL0_CNTL2__FGCG_DISABLE_MASK                                                                    0x40000000L
+#define SQG_UTCL0_CNTL2__BIG_PAGE_DISABLE_MASK                                                                0x80000000L
+//SQG_UTCL0_RETRY
+#define SQG_UTCL0_RETRY__INCR__SHIFT                                                                          0x0
+#define SQG_UTCL0_RETRY__COUNT__SHIFT                                                                         0x8
+#define SQG_UTCL0_RETRY__INCR_MASK                                                                            0x000000FFL
+#define SQG_UTCL0_RETRY__COUNT_MASK                                                                           0x00000F00L
+//SQG_VMID_FED_STATUS
+#define SQG_VMID_FED_STATUS__VMID_MASK__SHIFT                                                                 0x0
+#define SQG_VMID_FED_STATUS__VMID_MASK_MASK                                                                   0x0000FFFFL
+//SQG_VMID_FED_CLEAR
+#define SQG_VMID_FED_CLEAR__VMID_MASK__SHIFT                                                                  0x0
+#define SQG_VMID_FED_CLEAR__VMID_MASK_MASK                                                                    0x0000FFFFL
+//SQG_SE_PIT_CONTROL
+#define SQG_SE_PIT_CONTROL__PIT_PM_ENABLE__SHIFT                                                              0x0
+#define SQG_SE_PIT_CONTROL__PIT_PM_PRIMING_COOLDOWN__SHIFT                                                    0x1
+#define SQG_SE_PIT_CONTROL__PIT_PM_STATUS_AVG_LENGTH__SHIFT                                                   0x6
+#define SQG_SE_PIT_CONTROL__PIT_PM_IDLE_THRESHOLD__SHIFT                                                      0x8
+#define SQG_SE_PIT_CONTROL__PIT_PM_IDLE_CYCLES__SHIFT                                                         0xc
+#define SQG_SE_PIT_CONTROL__PIT_PM_ACTIVE_THRESHOLD__SHIFT                                                    0x12
+#define SQG_SE_PIT_CONTROL__PIT_PM_ACTIVE_CYCLES__SHIFT                                                       0x16
+#define SQG_SE_PIT_CONTROL__PIT_PM_ENABLE_MASK                                                                0x00000001L
+#define SQG_SE_PIT_CONTROL__PIT_PM_PRIMING_COOLDOWN_MASK                                                      0x00000006L
+#define SQG_SE_PIT_CONTROL__PIT_PM_STATUS_AVG_LENGTH_MASK                                                     0x000000C0L
+#define SQG_SE_PIT_CONTROL__PIT_PM_IDLE_THRESHOLD_MASK                                                        0x00000F00L
+#define SQG_SE_PIT_CONTROL__PIT_PM_IDLE_CYCLES_MASK                                                           0x0003F000L
+#define SQG_SE_PIT_CONTROL__PIT_PM_ACTIVE_THRESHOLD_MASK                                                      0x003C0000L
+#define SQG_SE_PIT_CONTROL__PIT_PM_ACTIVE_CYCLES_MASK                                                         0x0FC00000L
+//SQG_UTCL0_RETRY_TIMER_THRESHOLD_HI
+#define SQG_UTCL0_RETRY_TIMER_THRESHOLD_HI__VALUE__SHIFT                                                      0x0
+#define SQG_UTCL0_RETRY_TIMER_THRESHOLD_HI__VALUE_MASK                                                        0xFFFFFFFFL
+//SQG_UTCL0_RETRY_TIMER_THRESHOLD_LO
+#define SQG_UTCL0_RETRY_TIMER_THRESHOLD_LO__VALUE__SHIFT                                                      0x0
+#define SQG_UTCL0_RETRY_TIMER_THRESHOLD_LO__VALUE_MASK                                                        0xFFFFFFFFL
+//SQG_WS_INSERTION_COUNT
+#define SQG_WS_INSERTION_COUNT__COUNT__SHIFT                                                                  0x0
+#define SQG_WS_INSERTION_COUNT__COUNT_MASK                                                                    0x0000FFFFL
+//SQG_WS_SPI_STATUS
+#define SQG_WS_SPI_STATUS__COUNT__SHIFT                                                                       0x0
+#define SQG_WS_SPI_STATUS__WS_COUNT__SHIFT                                                                    0x10
+#define SQG_WS_SPI_STATUS__COUNT_MASK                                                                         0x0000FFFFL
+#define SQG_WS_SPI_STATUS__WS_COUNT_MASK                                                                      0xFFFF0000L
+//SQ_IND_INDEX
+#define SQ_IND_INDEX__WAVE_ID__SHIFT                                                                          0x0
+#define SQ_IND_INDEX__WORKITEM_ID__SHIFT                                                                      0x5
+#define SQ_IND_INDEX__AUTO_INCR__SHIFT                                                                        0xb
+#define SQ_IND_INDEX__INDEX__SHIFT                                                                            0x10
+#define SQ_IND_INDEX__WAVE_ID_MASK                                                                            0x0000001FL
+#define SQ_IND_INDEX__WORKITEM_ID_MASK                                                                        0x000007E0L
+#define SQ_IND_INDEX__AUTO_INCR_MASK                                                                          0x00000800L
+#define SQ_IND_INDEX__INDEX_MASK                                                                              0xFFFF0000L
+//SQ_IND_DATA
+#define SQ_IND_DATA__DATA__SHIFT                                                                              0x0
+#define SQ_IND_DATA__DATA_MASK                                                                                0xFFFFFFFFL
+//SQ_CMD
+#define SQ_CMD__CMD__SHIFT                                                                                    0x0
+#define SQ_CMD__MODE__SHIFT                                                                                   0x4
+#define SQ_CMD__CHECK_VMID__SHIFT                                                                             0x7
+#define SQ_CMD__DATA__SHIFT                                                                                   0x8
+#define SQ_CMD__WAVE_ID__SHIFT                                                                                0x10
+#define SQ_CMD__QUEUE_ID__SHIFT                                                                               0x18
+#define SQ_CMD__VM_ID__SHIFT                                                                                  0x1c
+#define SQ_CMD__CMD_MASK                                                                                      0x0000000FL
+#define SQ_CMD__MODE_MASK                                                                                     0x00000070L
+#define SQ_CMD__CHECK_VMID_MASK                                                                               0x00000080L
+#define SQ_CMD__DATA_MASK                                                                                     0x00000F00L
+#define SQ_CMD__WAVE_ID_MASK                                                                                  0x001F0000L
+#define SQ_CMD__QUEUE_ID_MASK                                                                                 0x07000000L
+#define SQ_CMD__VM_ID_MASK                                                                                    0xF0000000L
+//SQC_WSM_CONTROL
+#define SQC_WSM_CONTROL__WS_GO__SHIFT                                                                         0x0
+#define SQC_WSM_CONTROL__WS_INTENSITY__SHIFT                                                                  0x1
+#define SQC_WSM_CONTROL__WS_PRIORITY_MODE__SHIFT                                                              0x3
+#define SQC_WSM_CONTROL__WS_PERIOD__SHIFT                                                                     0x4
+#define SQC_WSM_CONTROL__WS_RATE__SHIFT                                                                       0xc
+#define SQC_WSM_CONTROL__WS_PERIOD_COUNT__SHIFT                                                               0x14
+#define SQC_WSM_CONTROL__WS_GO_MASK                                                                           0x00000001L
+#define SQC_WSM_CONTROL__WS_INTENSITY_MASK                                                                    0x00000006L
+#define SQC_WSM_CONTROL__WS_PRIORITY_MODE_MASK                                                                0x00000008L
+#define SQC_WSM_CONTROL__WS_PERIOD_MASK                                                                       0x00000FF0L
+#define SQC_WSM_CONTROL__WS_RATE_MASK                                                                         0x000FF000L
+#define SQC_WSM_CONTROL__WS_PERIOD_COUNT_MASK                                                                 0x00F00000L
+//SQC_PIT_CONTROL0
+#define SQC_PIT_CONTROL0__PIT_PRIMED_US_DIDT_THRESHOLD_2__SHIFT                                               0x0
+#define SQC_PIT_CONTROL0__PIT_PRIMED_US_DIDT_THRESHOLD_4__SHIFT                                               0x3
+#define SQC_PIT_CONTROL0__PIT_PRIMED_OS_DIDT_THRESHOLD_2__SHIFT                                               0x6
+#define SQC_PIT_CONTROL0__PIT_PRIMED_OS_DIDT_THRESHOLD_4__SHIFT                                               0x9
+#define SQC_PIT_CONTROL0__PIT_PRIMED_US_THROTTLE_LEVEL__SHIFT                                                 0xc
+#define SQC_PIT_CONTROL0__PIT_PRIMED_US_THROTTLE_RATE__SHIFT                                                  0xf
+#define SQC_PIT_CONTROL0__PIT_PRIMED_US_THROTTLE_CYCLES__SHIFT                                                0x11
+#define SQC_PIT_CONTROL0__PIT_PRIMED_OS_THROTTLE_LEVEL__SHIFT                                                 0x15
+#define SQC_PIT_CONTROL0__PIT_PRIMED_OS_THROTTLE_RATE__SHIFT                                                  0x18
+#define SQC_PIT_CONTROL0__PIT_PRIMED_OS_THROTTLE_CYCLES__SHIFT                                                0x1a
+#define SQC_PIT_CONTROL0__PIT_PRIMED_US_DIDT_THRESHOLD_2_MASK                                                 0x00000007L
+#define SQC_PIT_CONTROL0__PIT_PRIMED_US_DIDT_THRESHOLD_4_MASK                                                 0x00000038L
+#define SQC_PIT_CONTROL0__PIT_PRIMED_OS_DIDT_THRESHOLD_2_MASK                                                 0x000001C0L
+#define SQC_PIT_CONTROL0__PIT_PRIMED_OS_DIDT_THRESHOLD_4_MASK                                                 0x00000E00L
+#define SQC_PIT_CONTROL0__PIT_PRIMED_US_THROTTLE_LEVEL_MASK                                                   0x00007000L
+#define SQC_PIT_CONTROL0__PIT_PRIMED_US_THROTTLE_RATE_MASK                                                    0x00018000L
+#define SQC_PIT_CONTROL0__PIT_PRIMED_US_THROTTLE_CYCLES_MASK                                                  0x001E0000L
+#define SQC_PIT_CONTROL0__PIT_PRIMED_OS_THROTTLE_LEVEL_MASK                                                   0x00E00000L
+#define SQC_PIT_CONTROL0__PIT_PRIMED_OS_THROTTLE_RATE_MASK                                                    0x03000000L
+#define SQC_PIT_CONTROL0__PIT_PRIMED_OS_THROTTLE_CYCLES_MASK                                                  0x3C000000L
+//SQC_PIT_CONTROL1
+#define SQC_PIT_CONTROL1__LOCAL_THROTTLE_EN__SHIFT                                                            0x0
+#define SQC_PIT_CONTROL1__LOCAL_US_DIDT_THRESHOLD_2__SHIFT                                                    0x1
+#define SQC_PIT_CONTROL1__LOCAL_US_DIDT_THRESHOLD_4__SHIFT                                                    0x4
+#define SQC_PIT_CONTROL1__LOCAL_OS_DIDT_THRESHOLD_2__SHIFT                                                    0x7
+#define SQC_PIT_CONTROL1__LOCAL_OS_DIDT_THRESHOLD_4__SHIFT                                                    0xa
+#define SQC_PIT_CONTROL1__LOCAL_US_THROTTLE_RATE__SHIFT                                                       0xd
+#define SQC_PIT_CONTROL1__LOCAL_US_THROTTLE_CYCLES__SHIFT                                                     0xf
+#define SQC_PIT_CONTROL1__LOCAL_OS_THROTTLE_LEVEL__SHIFT                                                      0x13
+#define SQC_PIT_CONTROL1__LOCAL_OS_THROTTLE_RATE__SHIFT                                                       0x16
+#define SQC_PIT_CONTROL1__LOCAL_OS_THROTTLE_CYCLES__SHIFT                                                     0x18
+#define SQC_PIT_CONTROL1__LOCAL_THROTTLE_EN_MASK                                                              0x00000001L
+#define SQC_PIT_CONTROL1__LOCAL_US_DIDT_THRESHOLD_2_MASK                                                      0x0000000EL
+#define SQC_PIT_CONTROL1__LOCAL_US_DIDT_THRESHOLD_4_MASK                                                      0x00000070L
+#define SQC_PIT_CONTROL1__LOCAL_OS_DIDT_THRESHOLD_2_MASK                                                      0x00000380L
+#define SQC_PIT_CONTROL1__LOCAL_OS_DIDT_THRESHOLD_4_MASK                                                      0x00001C00L
+#define SQC_PIT_CONTROL1__LOCAL_US_THROTTLE_RATE_MASK                                                         0x00006000L
+#define SQC_PIT_CONTROL1__LOCAL_US_THROTTLE_CYCLES_MASK                                                       0x00078000L
+#define SQC_PIT_CONTROL1__LOCAL_OS_THROTTLE_LEVEL_MASK                                                        0x00380000L
+#define SQC_PIT_CONTROL1__LOCAL_OS_THROTTLE_RATE_MASK                                                         0x00C00000L
+#define SQC_PIT_CONTROL1__LOCAL_OS_THROTTLE_CYCLES_MASK                                                       0x0F000000L
+//SQC_PIT_CONTROL2
+#define SQC_PIT_CONTROL2__PIT_STATUS_AVG_WINDOW__SHIFT                                                        0x0
+#define SQC_PIT_CONTROL2__PIT_STATUS_AVG_IDLE_THRESHOLD__SHIFT                                                0x2
+#define SQC_PIT_CONTROL2__PIT_STATUS_AVG_ACTIVE_THRESHOLD__SHIFT                                              0x8
+#define SQC_PIT_CONTROL2__PIT_PF_LEVEL__SHIFT                                                                 0xe
+#define SQC_PIT_CONTROL2__PIT_PF_CYCLES__SHIFT                                                                0x13
+#define SQC_PIT_CONTROL2__PIT_PF_CYCLES_MIN__SHIFT                                                            0x17
+#define SQC_PIT_CONTROL2__PIT_PRIMED_US_INITIAL_DELAY_SA_STEP__SHIFT                                          0x1b
+#define SQC_PIT_CONTROL2__PIT_PRIMED_US_INITIAL_DELAY_WGP_STEP__SHIFT                                         0x1d
+#define SQC_PIT_CONTROL2__PIT_STATUS_AVG_WINDOW_MASK                                                          0x00000003L
+#define SQC_PIT_CONTROL2__PIT_STATUS_AVG_IDLE_THRESHOLD_MASK                                                  0x000000FCL
+#define SQC_PIT_CONTROL2__PIT_STATUS_AVG_ACTIVE_THRESHOLD_MASK                                                0x00003F00L
+#define SQC_PIT_CONTROL2__PIT_PF_LEVEL_MASK                                                                   0x0007C000L
+#define SQC_PIT_CONTROL2__PIT_PF_CYCLES_MASK                                                                  0x00780000L
+#define SQC_PIT_CONTROL2__PIT_PF_CYCLES_MIN_MASK                                                              0x07800000L
+#define SQC_PIT_CONTROL2__PIT_PRIMED_US_INITIAL_DELAY_SA_STEP_MASK                                            0x18000000L
+#define SQC_PIT_CONTROL2__PIT_PRIMED_US_INITIAL_DELAY_WGP_STEP_MASK                                           0x60000000L
+//SQC_UTCL0_RETRY
+#define SQC_UTCL0_RETRY__INCR__SHIFT                                                                          0x0
+#define SQC_UTCL0_RETRY__COUNT__SHIFT                                                                         0x8
+#define SQC_UTCL0_RETRY__INCR_MASK                                                                            0x000000FFL
+#define SQC_UTCL0_RETRY__COUNT_MASK                                                                           0x00000F00L
+//SQC_MISC_CONFIG
+#define SQC_MISC_CONFIG__SQC_SQ_MGCG_OVERSHOOT_PROG_DELAY__SHIFT                                              0x0
+#define SQC_MISC_CONFIG__SQC_SPI_TTRACE_FGCG_OVERRIDE__SHIFT                                                  0x5
+#define SQC_MISC_CONFIG__SQ_SPI_MSG_FGCG_OVERRIDE__SHIFT                                                      0x6
+#define SQC_MISC_CONFIG__SPI_SQ_EXPALLOC_FGCG_OVERRIDE__SHIFT                                                 0x7
+#define SQC_MISC_CONFIG__SQC_SQ_DATA_RET_FGCG_OVERRIDE__SHIFT                                                 0x8
+#define SQC_MISC_CONFIG__SQC_SQ_INST_RET_FGCG_OVERRIDE__SHIFT                                                 0x9
+#define SQC_MISC_CONFIG__SQC_GCR_RSP_FGCG_OVERRIDE__SHIFT                                                     0xa
+#define SQC_MISC_CONFIG__ICLK_MGCG_DISABLE__SHIFT                                                             0xb
+#define SQC_MISC_CONFIG__ICLK_BANK_MGCG_DISABLE__SHIFT                                                        0xc
+#define SQC_MISC_CONFIG__DCLK_MGCG_DISABLE__SHIFT                                                             0xd
+#define SQC_MISC_CONFIG__GCLK_MGCG_DISABLE__SHIFT                                                             0xe
+#define SQC_MISC_CONFIG__MCLK_MGCG_DISABLE__SHIFT                                                             0xf
+#define SQC_MISC_CONFIG__PCLK_MGCG_DISABLE__SHIFT                                                             0x10
+#define SQC_MISC_CONFIG__BCLK_MGCG_DISABLE__SHIFT                                                             0x11
+#define SQC_MISC_CONFIG__SQC_TA_RESET_FGCG_OVERRIDE__SHIFT                                                    0x12
+#define SQC_MISC_CONFIG__SQC_MCA_FGCG_OVERRIDE__SHIFT                                                         0x13
+#define SQC_MISC_CONFIG__DCLK_BANK_MGCG_DISABLE__SHIFT                                                        0x14
+#define SQC_MISC_CONFIG__SQC_SQ_BARRIER_DONE_FGCG_OVERRIDE__SHIFT                                             0x15
+#define SQC_MISC_CONFIG__SQC_SQ_MSGDONE_FGCG_OVERRIDE__SHIFT                                                  0x16
+#define SQC_MISC_CONFIG__CMCLK_MGCG_DISABLE__SHIFT                                                            0x17
+#define SQC_MISC_CONFIG__SQC_GL1_CLKEN_OVERRIDE__SHIFT                                                        0x18
+#define SQC_MISC_CONFIG__SQC_CORE_OVERRIDE__SHIFT                                                             0x19
+#define SQC_MISC_CONFIG__ICLK_HMF_BS_MGCG_DISABLE__SHIFT                                                      0x1a
+#define SQC_MISC_CONFIG__ICLK_CC_MGCG_DISABLE__SHIFT                                                          0x1b
+#define SQC_MISC_CONFIG__DCLK_HMF_BS_MGCG_DISABLE__SHIFT                                                      0x1c
+#define SQC_MISC_CONFIG__DCLK_CC_MGCG_DISABLE__SHIFT                                                          0x1d
+#define SQC_MISC_CONFIG__SQC_SQ_INVALIDATE_FGCG_DISABLE__SHIFT                                                0x1e
+#define SQC_MISC_CONFIG__SQC_UTCL0_FGCG_OVERRIDE__SHIFT                                                       0x1f
+#define SQC_MISC_CONFIG__SQC_SQ_MGCG_OVERSHOOT_PROG_DELAY_MASK                                                0x0000001FL
+#define SQC_MISC_CONFIG__SQC_SPI_TTRACE_FGCG_OVERRIDE_MASK                                                    0x00000020L
+#define SQC_MISC_CONFIG__SQ_SPI_MSG_FGCG_OVERRIDE_MASK                                                        0x00000040L
+#define SQC_MISC_CONFIG__SPI_SQ_EXPALLOC_FGCG_OVERRIDE_MASK                                                   0x00000080L
+#define SQC_MISC_CONFIG__SQC_SQ_DATA_RET_FGCG_OVERRIDE_MASK                                                   0x00000100L
+#define SQC_MISC_CONFIG__SQC_SQ_INST_RET_FGCG_OVERRIDE_MASK                                                   0x00000200L
+#define SQC_MISC_CONFIG__SQC_GCR_RSP_FGCG_OVERRIDE_MASK                                                       0x00000400L
+#define SQC_MISC_CONFIG__ICLK_MGCG_DISABLE_MASK                                                               0x00000800L
+#define SQC_MISC_CONFIG__ICLK_BANK_MGCG_DISABLE_MASK                                                          0x00001000L
+#define SQC_MISC_CONFIG__DCLK_MGCG_DISABLE_MASK                                                               0x00002000L
+#define SQC_MISC_CONFIG__GCLK_MGCG_DISABLE_MASK                                                               0x00004000L
+#define SQC_MISC_CONFIG__MCLK_MGCG_DISABLE_MASK                                                               0x00008000L
+#define SQC_MISC_CONFIG__PCLK_MGCG_DISABLE_MASK                                                               0x00010000L
+#define SQC_MISC_CONFIG__BCLK_MGCG_DISABLE_MASK                                                               0x00020000L
+#define SQC_MISC_CONFIG__SQC_TA_RESET_FGCG_OVERRIDE_MASK                                                      0x00040000L
+#define SQC_MISC_CONFIG__SQC_MCA_FGCG_OVERRIDE_MASK                                                           0x00080000L
+#define SQC_MISC_CONFIG__DCLK_BANK_MGCG_DISABLE_MASK                                                          0x00100000L
+#define SQC_MISC_CONFIG__SQC_SQ_BARRIER_DONE_FGCG_OVERRIDE_MASK                                               0x00200000L
+#define SQC_MISC_CONFIG__SQC_SQ_MSGDONE_FGCG_OVERRIDE_MASK                                                    0x00400000L
+#define SQC_MISC_CONFIG__CMCLK_MGCG_DISABLE_MASK                                                              0x00800000L
+#define SQC_MISC_CONFIG__SQC_GL1_CLKEN_OVERRIDE_MASK                                                          0x01000000L
+#define SQC_MISC_CONFIG__SQC_CORE_OVERRIDE_MASK                                                               0x02000000L
+#define SQC_MISC_CONFIG__ICLK_HMF_BS_MGCG_DISABLE_MASK                                                        0x04000000L
+#define SQC_MISC_CONFIG__ICLK_CC_MGCG_DISABLE_MASK                                                            0x08000000L
+#define SQC_MISC_CONFIG__DCLK_HMF_BS_MGCG_DISABLE_MASK                                                        0x10000000L
+#define SQC_MISC_CONFIG__DCLK_CC_MGCG_DISABLE_MASK                                                            0x20000000L
+#define SQC_MISC_CONFIG__SQC_SQ_INVALIDATE_FGCG_DISABLE_MASK                                                  0x40000000L
+#define SQC_MISC_CONFIG__SQC_UTCL0_FGCG_OVERRIDE_MASK                                                         0x80000000L
+//SQ_TIMEOUT_CONFIG
+#define SQ_TIMEOUT_CONFIG__PERIOD_SEL__SHIFT                                                                  0x0
+#define SQ_TIMEOUT_CONFIG__TIMEOUT_FATAL_DISABLE__SHIFT                                                       0x6
+#define SQ_TIMEOUT_CONFIG__PERIOD_SEL_MASK                                                                    0x0000003FL
+#define SQ_TIMEOUT_CONFIG__TIMEOUT_FATAL_DISABLE_MASK                                                         0x00000040L
+//SQ_TIMEOUT_STATUS_SIMD01
+#define SQ_TIMEOUT_STATUS_SIMD01__WAVE_TIMEOUT__SHIFT                                                         0x0
+#define SQ_TIMEOUT_STATUS_SIMD01__WAVE_TIMEOUT_MASK                                                           0xFFFFFFFFL
+//SQ_TIMEOUT_STATUS_SIMD23
+#define SQ_TIMEOUT_STATUS_SIMD23__WAVE_TIMEOUT__SHIFT                                                         0x0
+#define SQ_TIMEOUT_STATUS_SIMD23__WAVE_TIMEOUT_MASK                                                           0xFFFFFFFFL
+//SQC_UTCL0_STATUS
+#define SQC_UTCL0_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define SQC_UTCL0_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define SQC_UTCL0_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define SQC_UTCL0_STATUS__RESERVED__SHIFT                                                                     0x3
+#define SQC_UTCL0_STATUS__UNUSED__SHIFT                                                                       0x8
+#define SQC_UTCL0_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define SQC_UTCL0_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define SQC_UTCL0_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+#define SQC_UTCL0_STATUS__RESERVED_MASK                                                                       0x000000F8L
+#define SQC_UTCL0_STATUS__UNUSED_MASK                                                                         0xFFFFFF00L
+//SQC_UTCL0_CNTL1
+#define SQC_UTCL0_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                              0x0
+#define SQC_UTCL0_CNTL1__GPUVM_64K_DEFAULT__SHIFT                                                             0x1
+#define SQC_UTCL0_CNTL1__GPUVM_PERM_MODE__SHIFT                                                               0x2
+#define SQC_UTCL0_CNTL1__RESP_MODE__SHIFT                                                                     0x3
+#define SQC_UTCL0_CNTL1__RESP_FAULT_MODE__SHIFT                                                               0x5
+#define SQC_UTCL0_CNTL1__CLIENTID__SHIFT                                                                      0x7
+#define SQC_UTCL0_CNTL1__REG_INV_VMID__SHIFT                                                                  0x13
+#define SQC_UTCL0_CNTL1__REG_INV_TOGGLE__SHIFT                                                                0x18
+#define SQC_UTCL0_CNTL1__ATOMIC_REQUEST_ENABLE__SHIFT                                                         0x19
+#define SQC_UTCL0_CNTL1__FORCE_MISS__SHIFT                                                                    0x1a
+#define SQC_UTCL0_CNTL1__FORCE_IN_ORDER__SHIFT                                                                0x1b
+#define SQC_UTCL0_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                        0x1c
+#define SQC_UTCL0_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                        0x1e
+#define SQC_UTCL0_CNTL1__FORCE_4K_L2_RESP_MASK                                                                0x00000001L
+#define SQC_UTCL0_CNTL1__GPUVM_64K_DEFAULT_MASK                                                               0x00000002L
+#define SQC_UTCL0_CNTL1__GPUVM_PERM_MODE_MASK                                                                 0x00000004L
+#define SQC_UTCL0_CNTL1__RESP_MODE_MASK                                                                       0x00000018L
+#define SQC_UTCL0_CNTL1__RESP_FAULT_MODE_MASK                                                                 0x00000060L
+#define SQC_UTCL0_CNTL1__CLIENTID_MASK                                                                        0x0000FF80L
+#define SQC_UTCL0_CNTL1__REG_INV_VMID_MASK                                                                    0x00780000L
+#define SQC_UTCL0_CNTL1__REG_INV_TOGGLE_MASK                                                                  0x01000000L
+#define SQC_UTCL0_CNTL1__ATOMIC_REQUEST_ENABLE_MASK                                                           0x02000000L
+#define SQC_UTCL0_CNTL1__FORCE_MISS_MASK                                                                      0x04000000L
+#define SQC_UTCL0_CNTL1__FORCE_IN_ORDER_MASK                                                                  0x08000000L
+#define SQC_UTCL0_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                          0x30000000L
+#define SQC_UTCL0_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                          0xC0000000L
+//SQC_UTCL0_CNTL2
+#define SQC_UTCL0_CNTL2__SPARE__SHIFT                                                                         0x0
+#define SQC_UTCL0_CNTL2__COMP_SYNC_DISABLE__SHIFT                                                             0x8
+#define SQC_UTCL0_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                                0x9
+#define SQC_UTCL0_CNTL2__ANY_LINE_VALID__SHIFT                                                                0xa
+#define SQC_UTCL0_CNTL2__FORCE_SNOOP__SHIFT                                                                   0xe
+#define SQC_UTCL0_CNTL2__DISABLE_BURST__SHIFT                                                                 0x11
+#define SQC_UTCL0_CNTL2__ENABLE_PERF_EVENT_RD_WR__SHIFT                                                       0x12
+#define SQC_UTCL0_CNTL2__PERF_EVENT_RD_WR__SHIFT                                                              0x13
+#define SQC_UTCL0_CNTL2__ENABLE_PERF_EVENT_VMID__SHIFT                                                        0x14
+#define SQC_UTCL0_CNTL2__PERF_EVENT_VMID__SHIFT                                                               0x15
+#define SQC_UTCL0_CNTL2__FORCE_FRAG_2M_TO_64K__SHIFT                                                          0x1a
+#define SQC_UTCL0_CNTL2__FGCG_DISABLE__SHIFT                                                                  0x1e
+#define SQC_UTCL0_CNTL2__BIG_PAGE_DISABLE__SHIFT                                                              0x1f
+#define SQC_UTCL0_CNTL2__SPARE_MASK                                                                           0x000000FFL
+#define SQC_UTCL0_CNTL2__COMP_SYNC_DISABLE_MASK                                                               0x00000100L
+#define SQC_UTCL0_CNTL2__MTYPE_OVRD_DIS_MASK                                                                  0x00000200L
+#define SQC_UTCL0_CNTL2__ANY_LINE_VALID_MASK                                                                  0x00000400L
+#define SQC_UTCL0_CNTL2__FORCE_SNOOP_MASK                                                                     0x00004000L
+#define SQC_UTCL0_CNTL2__DISABLE_BURST_MASK                                                                   0x00020000L
+#define SQC_UTCL0_CNTL2__ENABLE_PERF_EVENT_RD_WR_MASK                                                         0x00040000L
+#define SQC_UTCL0_CNTL2__PERF_EVENT_RD_WR_MASK                                                                0x00080000L
+#define SQC_UTCL0_CNTL2__ENABLE_PERF_EVENT_VMID_MASK                                                          0x00100000L
+#define SQC_UTCL0_CNTL2__PERF_EVENT_VMID_MASK                                                                 0x01E00000L
+#define SQC_UTCL0_CNTL2__FORCE_FRAG_2M_TO_64K_MASK                                                            0x04000000L
+#define SQC_UTCL0_CNTL2__FGCG_DISABLE_MASK                                                                    0x40000000L
+#define SQC_UTCL0_CNTL2__BIG_PAGE_DISABLE_MASK                                                                0x80000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_shsdec
+//SX_DEBUG_BUSY
+#define SX_DEBUG_BUSY__COL_WRCTRL1_VALIDQ3__SHIFT                                                             0x0
+#define SX_DEBUG_BUSY__COL_WRCTRL1_VALIDQ2__SHIFT                                                             0x1
+#define SX_DEBUG_BUSY__COL_WRCTRL1_VALIDQ1__SHIFT                                                             0x2
+#define SX_DEBUG_BUSY__COL_WRCTRL1_VALID__SHIFT                                                               0x3
+#define SX_DEBUG_BUSY__COL_WRCTRL0_VALIDQ3__SHIFT                                                             0x4
+#define SX_DEBUG_BUSY__COL_WRCTRL0_VALIDQ2__SHIFT                                                             0x5
+#define SX_DEBUG_BUSY__COL_WRCTRL0_VALIDQ1__SHIFT                                                             0x6
+#define SX_DEBUG_BUSY__COL_WRCTRL0_VALID__SHIFT                                                               0x7
+#define SX_DEBUG_BUSY__VDATA1_VALID__SHIFT                                                                    0x9
+#define SX_DEBUG_BUSY__VDATA0_VALID__SHIFT                                                                    0xa
+#define SX_DEBUG_BUSY__CMD_BUSYORVAL__SHIFT                                                                   0xb
+#define SX_DEBUG_BUSY__ADDR_BUSYORVAL__SHIFT                                                                  0xc
+#define SX_DEBUG_BUSY__SX_SX_IN_VALID__SHIFT                                                                  0xd
+#define SX_DEBUG_BUSY__SX_SX_OUT_VALID__SHIFT                                                                 0xe
+#define SX_DEBUG_BUSY__RESERVED__SHIFT                                                                        0xf
+#define SX_DEBUG_BUSY__COL_WRCTRL1_VALIDQ3_MASK                                                               0x00000001L
+#define SX_DEBUG_BUSY__COL_WRCTRL1_VALIDQ2_MASK                                                               0x00000002L
+#define SX_DEBUG_BUSY__COL_WRCTRL1_VALIDQ1_MASK                                                               0x00000004L
+#define SX_DEBUG_BUSY__COL_WRCTRL1_VALID_MASK                                                                 0x00000008L
+#define SX_DEBUG_BUSY__COL_WRCTRL0_VALIDQ3_MASK                                                               0x00000010L
+#define SX_DEBUG_BUSY__COL_WRCTRL0_VALIDQ2_MASK                                                               0x00000020L
+#define SX_DEBUG_BUSY__COL_WRCTRL0_VALIDQ1_MASK                                                               0x00000040L
+#define SX_DEBUG_BUSY__COL_WRCTRL0_VALID_MASK                                                                 0x00000080L
+#define SX_DEBUG_BUSY__VDATA1_VALID_MASK                                                                      0x00000200L
+#define SX_DEBUG_BUSY__VDATA0_VALID_MASK                                                                      0x00000400L
+#define SX_DEBUG_BUSY__CMD_BUSYORVAL_MASK                                                                     0x00000800L
+#define SX_DEBUG_BUSY__ADDR_BUSYORVAL_MASK                                                                    0x00001000L
+#define SX_DEBUG_BUSY__SX_SX_IN_VALID_MASK                                                                    0x00002000L
+#define SX_DEBUG_BUSY__SX_SX_OUT_VALID_MASK                                                                   0x00004000L
+#define SX_DEBUG_BUSY__RESERVED_MASK                                                                          0xFFFF8000L
+//SX_DEBUG_BUSY_2
+#define SX_DEBUG_BUSY_2__COL_SCBD0_BUSY__SHIFT                                                                0x0
+#define SX_DEBUG_BUSY_2__COL_REQ3_FREECNT_NE0__SHIFT                                                          0x1
+#define SX_DEBUG_BUSY_2__COL_REQ3_IDLE__SHIFT                                                                 0x2
+#define SX_DEBUG_BUSY_2__COL_REQ3_BUSY__SHIFT                                                                 0x3
+#define SX_DEBUG_BUSY_2__COL_REQ2_FREECNT_NE0__SHIFT                                                          0x4
+#define SX_DEBUG_BUSY_2__COL_REQ2_IDLE__SHIFT                                                                 0x5
+#define SX_DEBUG_BUSY_2__COL_REQ2_BUSY__SHIFT                                                                 0x6
+#define SX_DEBUG_BUSY_2__COL_REQ1_FREECNT_NE0__SHIFT                                                          0x7
+#define SX_DEBUG_BUSY_2__COL_REQ1_IDLE__SHIFT                                                                 0x8
+#define SX_DEBUG_BUSY_2__COL_REQ1_BUSY__SHIFT                                                                 0x9
+#define SX_DEBUG_BUSY_2__COL_REQ0_FREECNT_NE0__SHIFT                                                          0xa
+#define SX_DEBUG_BUSY_2__COL_REQ0_IDLE__SHIFT                                                                 0xb
+#define SX_DEBUG_BUSY_2__COL_REQ0_BUSY__SHIFT                                                                 0xc
+#define SX_DEBUG_BUSY_2__COL_DBIF3_SENDFREE_BUSY__SHIFT                                                       0xd
+#define SX_DEBUG_BUSY_2__COL_DBIF3_FIFO_BUSY__SHIFT                                                           0xe
+#define SX_DEBUG_BUSY_2__COL_DBIF3_QUAD_FREE__SHIFT                                                           0xf
+#define SX_DEBUG_BUSY_2__COL_DBIF2_SENDFREE_BUSY__SHIFT                                                       0x10
+#define SX_DEBUG_BUSY_2__COL_DBIF2_FIFO_BUSY__SHIFT                                                           0x11
+#define SX_DEBUG_BUSY_2__COL_DBIF2_QUAD_FREE__SHIFT                                                           0x12
+#define SX_DEBUG_BUSY_2__COL_DBIF1_SENDFREE_BUSY__SHIFT                                                       0x13
+#define SX_DEBUG_BUSY_2__COL_DBIF1_FIFO_BUSY__SHIFT                                                           0x14
+#define SX_DEBUG_BUSY_2__COL_DBIF1_QUAD_FREE__SHIFT                                                           0x15
+#define SX_DEBUG_BUSY_2__COL_DBIF0_SENDFREE_BUSY__SHIFT                                                       0x16
+#define SX_DEBUG_BUSY_2__COL_DBIF0_FIFO_BUSY__SHIFT                                                           0x17
+#define SX_DEBUG_BUSY_2__COL_DBIF0_QUAD_FREE__SHIFT                                                           0x18
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL3_BUSY__SHIFT                                                     0x19
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL2_BUSY__SHIFT                                                     0x1a
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL1_BUSY__SHIFT                                                     0x1b
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL0_BUSY__SHIFT                                                     0x1c
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL3_BUSY__SHIFT                                                     0x1d
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL2_BUSY__SHIFT                                                     0x1e
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL1_BUSY__SHIFT                                                     0x1f
+#define SX_DEBUG_BUSY_2__COL_SCBD0_BUSY_MASK                                                                  0x00000001L
+#define SX_DEBUG_BUSY_2__COL_REQ3_FREECNT_NE0_MASK                                                            0x00000002L
+#define SX_DEBUG_BUSY_2__COL_REQ3_IDLE_MASK                                                                   0x00000004L
+#define SX_DEBUG_BUSY_2__COL_REQ3_BUSY_MASK                                                                   0x00000008L
+#define SX_DEBUG_BUSY_2__COL_REQ2_FREECNT_NE0_MASK                                                            0x00000010L
+#define SX_DEBUG_BUSY_2__COL_REQ2_IDLE_MASK                                                                   0x00000020L
+#define SX_DEBUG_BUSY_2__COL_REQ2_BUSY_MASK                                                                   0x00000040L
+#define SX_DEBUG_BUSY_2__COL_REQ1_FREECNT_NE0_MASK                                                            0x00000080L
+#define SX_DEBUG_BUSY_2__COL_REQ1_IDLE_MASK                                                                   0x00000100L
+#define SX_DEBUG_BUSY_2__COL_REQ1_BUSY_MASK                                                                   0x00000200L
+#define SX_DEBUG_BUSY_2__COL_REQ0_FREECNT_NE0_MASK                                                            0x00000400L
+#define SX_DEBUG_BUSY_2__COL_REQ0_IDLE_MASK                                                                   0x00000800L
+#define SX_DEBUG_BUSY_2__COL_REQ0_BUSY_MASK                                                                   0x00001000L
+#define SX_DEBUG_BUSY_2__COL_DBIF3_SENDFREE_BUSY_MASK                                                         0x00002000L
+#define SX_DEBUG_BUSY_2__COL_DBIF3_FIFO_BUSY_MASK                                                             0x00004000L
+#define SX_DEBUG_BUSY_2__COL_DBIF3_QUAD_FREE_MASK                                                             0x00008000L
+#define SX_DEBUG_BUSY_2__COL_DBIF2_SENDFREE_BUSY_MASK                                                         0x00010000L
+#define SX_DEBUG_BUSY_2__COL_DBIF2_FIFO_BUSY_MASK                                                             0x00020000L
+#define SX_DEBUG_BUSY_2__COL_DBIF2_QUAD_FREE_MASK                                                             0x00040000L
+#define SX_DEBUG_BUSY_2__COL_DBIF1_SENDFREE_BUSY_MASK                                                         0x00080000L
+#define SX_DEBUG_BUSY_2__COL_DBIF1_FIFO_BUSY_MASK                                                             0x00100000L
+#define SX_DEBUG_BUSY_2__COL_DBIF1_QUAD_FREE_MASK                                                             0x00200000L
+#define SX_DEBUG_BUSY_2__COL_DBIF0_SENDFREE_BUSY_MASK                                                         0x00400000L
+#define SX_DEBUG_BUSY_2__COL_DBIF0_FIFO_BUSY_MASK                                                             0x00800000L
+#define SX_DEBUG_BUSY_2__COL_DBIF0_QUAD_FREE_MASK                                                             0x01000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL3_BUSY_MASK                                                       0x02000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL2_BUSY_MASK                                                       0x04000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL1_BUSY_MASK                                                       0x08000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL0_BUSY_MASK                                                       0x10000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL3_BUSY_MASK                                                       0x20000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL2_BUSY_MASK                                                       0x40000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL1_BUSY_MASK                                                       0x80000000L
+//SX_DEBUG_BUSY_3
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK2_VAL0_BUSY__SHIFT                                                     0x0
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL3_BUSY__SHIFT                                                     0x1
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL2_BUSY__SHIFT                                                     0x2
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL1_BUSY__SHIFT                                                     0x3
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL0_BUSY__SHIFT                                                     0x4
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL3_BUSY__SHIFT                                                     0x5
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL2_BUSY__SHIFT                                                     0x6
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL1_BUSY__SHIFT                                                     0x7
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL0_BUSY__SHIFT                                                     0x8
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL3_BUSY__SHIFT                                                     0x9
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL2_BUSY__SHIFT                                                     0xa
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL1_BUSY__SHIFT                                                     0xb
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL0_BUSY__SHIFT                                                     0xc
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL3_BUSY__SHIFT                                                     0xd
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL2_BUSY__SHIFT                                                     0xe
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL1_BUSY__SHIFT                                                     0xf
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL0_BUSY__SHIFT                                                     0x10
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL3_BUSY__SHIFT                                                     0x11
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL2_BUSY__SHIFT                                                     0x12
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL1_BUSY__SHIFT                                                     0x13
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL0_BUSY__SHIFT                                                     0x14
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL3_BUSY__SHIFT                                                     0x15
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL2_BUSY__SHIFT                                                     0x16
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL1_BUSY__SHIFT                                                     0x17
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL0_BUSY__SHIFT                                                     0x18
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL3_BUSY__SHIFT                                                     0x19
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL2_BUSY__SHIFT                                                     0x1a
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL1_BUSY__SHIFT                                                     0x1b
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL0_BUSY__SHIFT                                                     0x1c
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL3_BUSY__SHIFT                                                     0x1d
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL2_BUSY__SHIFT                                                     0x1e
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL1_BUSY__SHIFT                                                     0x1f
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK2_VAL0_BUSY_MASK                                                       0x00000001L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL3_BUSY_MASK                                                       0x00000002L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL2_BUSY_MASK                                                       0x00000004L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL1_BUSY_MASK                                                       0x00000008L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL0_BUSY_MASK                                                       0x00000010L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL3_BUSY_MASK                                                       0x00000020L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL2_BUSY_MASK                                                       0x00000040L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL1_BUSY_MASK                                                       0x00000080L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL0_BUSY_MASK                                                       0x00000100L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL3_BUSY_MASK                                                       0x00000200L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL2_BUSY_MASK                                                       0x00000400L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL1_BUSY_MASK                                                       0x00000800L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL0_BUSY_MASK                                                       0x00001000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL3_BUSY_MASK                                                       0x00002000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL2_BUSY_MASK                                                       0x00004000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL1_BUSY_MASK                                                       0x00008000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL0_BUSY_MASK                                                       0x00010000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL3_BUSY_MASK                                                       0x00020000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL2_BUSY_MASK                                                       0x00040000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL1_BUSY_MASK                                                       0x00080000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL0_BUSY_MASK                                                       0x00100000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL3_BUSY_MASK                                                       0x00200000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL2_BUSY_MASK                                                       0x00400000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL1_BUSY_MASK                                                       0x00800000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL0_BUSY_MASK                                                       0x01000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL3_BUSY_MASK                                                       0x02000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL2_BUSY_MASK                                                       0x04000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL1_BUSY_MASK                                                       0x08000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL0_BUSY_MASK                                                       0x10000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL3_BUSY_MASK                                                       0x20000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL2_BUSY_MASK                                                       0x40000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL1_BUSY_MASK                                                       0x80000000L
+//SX_DEBUG_BUSY_4
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK2_VAL0_BUSY__SHIFT                                                     0x0
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL3_BUSY__SHIFT                                                     0x1
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL2_BUSY__SHIFT                                                     0x2
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL1_BUSY__SHIFT                                                     0x3
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL0_BUSY__SHIFT                                                     0x4
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL3_BUSY__SHIFT                                                     0x5
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL2_BUSY__SHIFT                                                     0x6
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL1_BUSY__SHIFT                                                     0x7
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL0_BUSY__SHIFT                                                     0x8
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL3_BUSY__SHIFT                                                     0x9
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL2_BUSY__SHIFT                                                     0xa
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL1_BUSY__SHIFT                                                     0xb
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL0_BUSY__SHIFT                                                     0xc
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL3_BUSY__SHIFT                                                     0xd
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL2_BUSY__SHIFT                                                     0xe
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL1_BUSY__SHIFT                                                     0xf
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL0_BUSY__SHIFT                                                     0x10
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL3_BUSY__SHIFT                                                     0x11
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL2_BUSY__SHIFT                                                     0x12
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL1_BUSY__SHIFT                                                     0x13
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL0_BUSY__SHIFT                                                     0x14
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL3_BUSY__SHIFT                                                     0x15
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL2_BUSY__SHIFT                                                     0x16
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL1_BUSY__SHIFT                                                     0x17
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL0_BUSY__SHIFT                                                     0x18
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK7_VAL3_BUSY__SHIFT                                                     0x19
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK7_VAL2_BUSY__SHIFT                                                     0x1a
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK7_VAL1_BUSY__SHIFT                                                     0x1b
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK7_VAL0_BUSY__SHIFT                                                     0x1c
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK6_VAL3_BUSY__SHIFT                                                     0x1d
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK6_VAL2_BUSY__SHIFT                                                     0x1e
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK6_VAL1_BUSY__SHIFT                                                     0x1f
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK2_VAL0_BUSY_MASK                                                       0x00000001L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL3_BUSY_MASK                                                       0x00000002L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL2_BUSY_MASK                                                       0x00000004L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL1_BUSY_MASK                                                       0x00000008L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL0_BUSY_MASK                                                       0x00000010L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL3_BUSY_MASK                                                       0x00000020L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL2_BUSY_MASK                                                       0x00000040L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL1_BUSY_MASK                                                       0x00000080L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL0_BUSY_MASK                                                       0x00000100L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL3_BUSY_MASK                                                       0x00000200L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL2_BUSY_MASK                                                       0x00000400L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL1_BUSY_MASK                                                       0x00000800L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL0_BUSY_MASK                                                       0x00001000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL3_BUSY_MASK                                                       0x00002000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL2_BUSY_MASK                                                       0x00004000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL1_BUSY_MASK                                                       0x00008000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL0_BUSY_MASK                                                       0x00010000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL3_BUSY_MASK                                                       0x00020000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL2_BUSY_MASK                                                       0x00040000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL1_BUSY_MASK                                                       0x00080000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL0_BUSY_MASK                                                       0x00100000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL3_BUSY_MASK                                                       0x00200000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL2_BUSY_MASK                                                       0x00400000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL1_BUSY_MASK                                                       0x00800000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL0_BUSY_MASK                                                       0x01000000L
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK7_VAL3_BUSY_MASK                                                       0x02000000L
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK7_VAL2_BUSY_MASK                                                       0x04000000L
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK7_VAL1_BUSY_MASK                                                       0x08000000L
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK7_VAL0_BUSY_MASK                                                       0x10000000L
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK6_VAL3_BUSY_MASK                                                       0x20000000L
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK6_VAL2_BUSY_MASK                                                       0x40000000L
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK6_VAL1_BUSY_MASK                                                       0x80000000L
+//SX_DEBUG_1
+#define SX_DEBUG_1__SX_DB_QUAD_CREDIT__SHIFT                                                                  0x0
+#define SX_DEBUG_1__ENABLE_FIFO_DEBUG_WRITE__SHIFT                                                            0x7
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_DONT_RD_DST__SHIFT                                                      0x8
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_BYPASS__SHIFT                                                           0x9
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_DISCARD_PIXEL__SHIFT                                                    0xa
+#define SX_DEBUG_1__DISABLE_QUAD_PAIR_OPT__SHIFT                                                              0xb
+#define SX_DEBUG_1__DISABLE_PIX_EN_ZERO_OPT__SHIFT                                                            0xc
+#define SX_DEBUG_1__DISABLE_REP_FGCG__SHIFT                                                                   0xd
+#define SX_DEBUG_1__DISABLE_RAM_FGCG__SHIFT                                                                   0xf
+#define SX_DEBUG_1__DISABLE_COL_VAL_READ_OPT__SHIFT                                                           0x11
+#define SX_DEBUG_1__DISABLE_BC_RB_PLUS__SHIFT                                                                 0x12
+#define SX_DEBUG_1__DISABLE_NATIVE_DOWNCVT_FMT_MAPPING__SHIFT                                                 0x13
+#define SX_DEBUG_1__DISABLE_SCBD_READ_PWR_OPT__SHIFT                                                          0x14
+#define SX_DEBUG_1__DISABLE_DOWNCVT_PWR_OPT__SHIFT                                                            0x16
+#define SX_DEBUG_1__DISABLE_POS_BUFF_REUSE_OPT__SHIFT                                                         0x17
+#define SX_DEBUG_1__DISABLE_DBIF_PIX_ENABLE_FGCG__SHIFT                                                       0x18
+#define SX_DEBUG_1__DEBUG_DATA__SHIFT                                                                         0x19
+#define SX_DEBUG_1__SX_DB_QUAD_CREDIT_MASK                                                                    0x0000007FL
+#define SX_DEBUG_1__ENABLE_FIFO_DEBUG_WRITE_MASK                                                              0x00000080L
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_DONT_RD_DST_MASK                                                        0x00000100L
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_BYPASS_MASK                                                             0x00000200L
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_DISCARD_PIXEL_MASK                                                      0x00000400L
+#define SX_DEBUG_1__DISABLE_QUAD_PAIR_OPT_MASK                                                                0x00000800L
+#define SX_DEBUG_1__DISABLE_PIX_EN_ZERO_OPT_MASK                                                              0x00001000L
+#define SX_DEBUG_1__DISABLE_REP_FGCG_MASK                                                                     0x00002000L
+#define SX_DEBUG_1__DISABLE_RAM_FGCG_MASK                                                                     0x00008000L
+#define SX_DEBUG_1__DISABLE_COL_VAL_READ_OPT_MASK                                                             0x00020000L
+#define SX_DEBUG_1__DISABLE_BC_RB_PLUS_MASK                                                                   0x00040000L
+#define SX_DEBUG_1__DISABLE_NATIVE_DOWNCVT_FMT_MAPPING_MASK                                                   0x00080000L
+#define SX_DEBUG_1__DISABLE_SCBD_READ_PWR_OPT_MASK                                                            0x00100000L
+#define SX_DEBUG_1__DISABLE_DOWNCVT_PWR_OPT_MASK                                                              0x00400000L
+#define SX_DEBUG_1__DISABLE_POS_BUFF_REUSE_OPT_MASK                                                           0x00800000L
+#define SX_DEBUG_1__DISABLE_DBIF_PIX_ENABLE_FGCG_MASK                                                         0x01000000L
+#define SX_DEBUG_1__DEBUG_DATA_MASK                                                                           0xFE000000L
+//SX_DEBUG_BUSY_5
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK6_VAL0_BUSY__SHIFT                                                     0x0
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK5_VAL3_BUSY__SHIFT                                                     0x1
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK5_VAL2_BUSY__SHIFT                                                     0x2
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK5_VAL1_BUSY__SHIFT                                                     0x3
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK5_VAL0_BUSY__SHIFT                                                     0x4
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK4_VAL3_BUSY__SHIFT                                                     0x5
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK4_VAL2_BUSY__SHIFT                                                     0x6
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK4_VAL1_BUSY__SHIFT                                                     0x7
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK4_VAL0_BUSY__SHIFT                                                     0x8
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK7_VAL3_BUSY__SHIFT                                                     0x9
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK7_VAL2_BUSY__SHIFT                                                     0xa
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK7_VAL1_BUSY__SHIFT                                                     0xb
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK7_VAL0_BUSY__SHIFT                                                     0xc
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK6_VAL3_BUSY__SHIFT                                                     0xd
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK6_VAL2_BUSY__SHIFT                                                     0xe
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK6_VAL1_BUSY__SHIFT                                                     0xf
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK6_VAL0_BUSY__SHIFT                                                     0x10
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK5_VAL3_BUSY__SHIFT                                                     0x11
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK5_VAL2_BUSY__SHIFT                                                     0x12
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK5_VAL1_BUSY__SHIFT                                                     0x13
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK5_VAL0_BUSY__SHIFT                                                     0x14
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK4_VAL3_BUSY__SHIFT                                                     0x15
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK4_VAL2_BUSY__SHIFT                                                     0x16
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK4_VAL1_BUSY__SHIFT                                                     0x17
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK4_VAL0_BUSY__SHIFT                                                     0x18
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK7_VAL3_BUSY__SHIFT                                                     0x19
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK7_VAL2_BUSY__SHIFT                                                     0x1a
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK7_VAL1_BUSY__SHIFT                                                     0x1b
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK7_VAL0_BUSY__SHIFT                                                     0x1c
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK6_VAL3_BUSY__SHIFT                                                     0x1d
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK6_VAL2_BUSY__SHIFT                                                     0x1e
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK6_VAL1_BUSY__SHIFT                                                     0x1f
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK6_VAL0_BUSY_MASK                                                       0x00000001L
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK5_VAL3_BUSY_MASK                                                       0x00000002L
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK5_VAL2_BUSY_MASK                                                       0x00000004L
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK5_VAL1_BUSY_MASK                                                       0x00000008L
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK5_VAL0_BUSY_MASK                                                       0x00000010L
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK4_VAL3_BUSY_MASK                                                       0x00000020L
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK4_VAL2_BUSY_MASK                                                       0x00000040L
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK4_VAL1_BUSY_MASK                                                       0x00000080L
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK4_VAL0_BUSY_MASK                                                       0x00000100L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK7_VAL3_BUSY_MASK                                                       0x00000200L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK7_VAL2_BUSY_MASK                                                       0x00000400L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK7_VAL1_BUSY_MASK                                                       0x00000800L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK7_VAL0_BUSY_MASK                                                       0x00001000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK6_VAL3_BUSY_MASK                                                       0x00002000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK6_VAL2_BUSY_MASK                                                       0x00004000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK6_VAL1_BUSY_MASK                                                       0x00008000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK6_VAL0_BUSY_MASK                                                       0x00010000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK5_VAL3_BUSY_MASK                                                       0x00020000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK5_VAL2_BUSY_MASK                                                       0x00040000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK5_VAL1_BUSY_MASK                                                       0x00080000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK5_VAL0_BUSY_MASK                                                       0x00100000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK4_VAL3_BUSY_MASK                                                       0x00200000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK4_VAL2_BUSY_MASK                                                       0x00400000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK4_VAL1_BUSY_MASK                                                       0x00800000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK4_VAL0_BUSY_MASK                                                       0x01000000L
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK7_VAL3_BUSY_MASK                                                       0x02000000L
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK7_VAL2_BUSY_MASK                                                       0x04000000L
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK7_VAL1_BUSY_MASK                                                       0x08000000L
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK7_VAL0_BUSY_MASK                                                       0x10000000L
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK6_VAL3_BUSY_MASK                                                       0x20000000L
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK6_VAL2_BUSY_MASK                                                       0x40000000L
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK6_VAL1_BUSY_MASK                                                       0x80000000L
+//SX_DEBUG_BUSY_6
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK6_VAL0_BUSY__SHIFT                                                     0x0
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK5_VAL3_BUSY__SHIFT                                                     0x1
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK5_VAL2_BUSY__SHIFT                                                     0x2
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK5_VAL1_BUSY__SHIFT                                                     0x3
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK5_VAL0_BUSY__SHIFT                                                     0x4
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK4_VAL3_BUSY__SHIFT                                                     0x5
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK4_VAL2_BUSY__SHIFT                                                     0x6
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK4_VAL1_BUSY__SHIFT                                                     0x7
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK4_VAL0_BUSY__SHIFT                                                     0x8
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK7_VAL3_BUSY__SHIFT                                                     0x9
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK7_VAL2_BUSY__SHIFT                                                     0xa
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK7_VAL1_BUSY__SHIFT                                                     0xb
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK7_VAL0_BUSY__SHIFT                                                     0xc
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK6_VAL3_BUSY__SHIFT                                                     0xd
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK6_VAL2_BUSY__SHIFT                                                     0xe
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK6_VAL1_BUSY__SHIFT                                                     0xf
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK6_VAL0_BUSY__SHIFT                                                     0x10
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK5_VAL3_BUSY__SHIFT                                                     0x11
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK5_VAL2_BUSY__SHIFT                                                     0x12
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK5_VAL1_BUSY__SHIFT                                                     0x13
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK5_VAL0_BUSY__SHIFT                                                     0x14
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK4_VAL3_BUSY__SHIFT                                                     0x15
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK4_VAL2_BUSY__SHIFT                                                     0x16
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK4_VAL1_BUSY__SHIFT                                                     0x17
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK4_VAL0_BUSY__SHIFT                                                     0x18
+#define SX_DEBUG_BUSY_6__COL_REQ3_CREDIT_BUSY__SHIFT                                                          0x19
+#define SX_DEBUG_BUSY_6__COL_REQ3_FLOP_BUSY__SHIFT                                                            0x1a
+#define SX_DEBUG_BUSY_6__COL_REQ2_CREDIT_BUSY__SHIFT                                                          0x1b
+#define SX_DEBUG_BUSY_6__COL_REQ2_FLOP_BUSY__SHIFT                                                            0x1c
+#define SX_DEBUG_BUSY_6__COL_REQ1_CREDIT_BUSY__SHIFT                                                          0x1d
+#define SX_DEBUG_BUSY_6__COL_REQ1_FLOP_BUSY__SHIFT                                                            0x1e
+#define SX_DEBUG_BUSY_6__COL_REQ0_CREDIT_BUSY__SHIFT                                                          0x1f
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK6_VAL0_BUSY_MASK                                                       0x00000001L
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK5_VAL3_BUSY_MASK                                                       0x00000002L
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK5_VAL2_BUSY_MASK                                                       0x00000004L
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK5_VAL1_BUSY_MASK                                                       0x00000008L
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK5_VAL0_BUSY_MASK                                                       0x00000010L
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK4_VAL3_BUSY_MASK                                                       0x00000020L
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK4_VAL2_BUSY_MASK                                                       0x00000040L
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK4_VAL1_BUSY_MASK                                                       0x00000080L
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK4_VAL0_BUSY_MASK                                                       0x00000100L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK7_VAL3_BUSY_MASK                                                       0x00000200L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK7_VAL2_BUSY_MASK                                                       0x00000400L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK7_VAL1_BUSY_MASK                                                       0x00000800L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK7_VAL0_BUSY_MASK                                                       0x00001000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK6_VAL3_BUSY_MASK                                                       0x00002000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK6_VAL2_BUSY_MASK                                                       0x00004000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK6_VAL1_BUSY_MASK                                                       0x00008000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK6_VAL0_BUSY_MASK                                                       0x00010000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK5_VAL3_BUSY_MASK                                                       0x00020000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK5_VAL2_BUSY_MASK                                                       0x00040000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK5_VAL1_BUSY_MASK                                                       0x00080000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK5_VAL0_BUSY_MASK                                                       0x00100000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK4_VAL3_BUSY_MASK                                                       0x00200000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK4_VAL2_BUSY_MASK                                                       0x00400000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK4_VAL1_BUSY_MASK                                                       0x00800000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK4_VAL0_BUSY_MASK                                                       0x01000000L
+#define SX_DEBUG_BUSY_6__COL_REQ3_CREDIT_BUSY_MASK                                                            0x02000000L
+#define SX_DEBUG_BUSY_6__COL_REQ3_FLOP_BUSY_MASK                                                              0x04000000L
+#define SX_DEBUG_BUSY_6__COL_REQ2_CREDIT_BUSY_MASK                                                            0x08000000L
+#define SX_DEBUG_BUSY_6__COL_REQ2_FLOP_BUSY_MASK                                                              0x10000000L
+#define SX_DEBUG_BUSY_6__COL_REQ1_CREDIT_BUSY_MASK                                                            0x20000000L
+#define SX_DEBUG_BUSY_6__COL_REQ1_FLOP_BUSY_MASK                                                              0x40000000L
+#define SX_DEBUG_BUSY_6__COL_REQ0_CREDIT_BUSY_MASK                                                            0x80000000L
+//SX_DEBUG_BUSY_7
+#define SX_DEBUG_BUSY_7__COL_REQ0_FLOP_BUSY__SHIFT                                                            0x0
+#define SX_DEBUG_BUSY_7__COL_SCBD1_BUSY__SHIFT                                                                0x1
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ1__SHIFT                                                       0x2
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ1_ADJ__SHIFT                                                   0x3
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ2__SHIFT                                                       0x4
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ3__SHIFT                                                       0x5
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ4__SHIFT                                                       0x6
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ5__SHIFT                                                       0x7
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALID_OUT__SHIFT                                                     0x8
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ1__SHIFT                                                       0x9
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ1_ADJ__SHIFT                                                   0xa
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ2__SHIFT                                                       0xb
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ3__SHIFT                                                       0xc
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ4__SHIFT                                                       0xd
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ5__SHIFT                                                       0xe
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALID_OUT__SHIFT                                                     0xf
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ1__SHIFT                                                       0x10
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ1_ADJ__SHIFT                                                   0x11
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ2__SHIFT                                                       0x12
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ3__SHIFT                                                       0x13
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ4__SHIFT                                                       0x14
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ5__SHIFT                                                       0x15
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALID_OUT__SHIFT                                                     0x16
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ1__SHIFT                                                       0x17
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ1_ADJ__SHIFT                                                   0x18
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ2__SHIFT                                                       0x19
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ3__SHIFT                                                       0x1a
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ4__SHIFT                                                       0x1b
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ5__SHIFT                                                       0x1c
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALID_OUT__SHIFT                                                     0x1d
+#define SX_DEBUG_BUSY_7__RESERVED__SHIFT                                                                      0x1e
+#define SX_DEBUG_BUSY_7__COL_REQ0_FLOP_BUSY_MASK                                                              0x00000001L
+#define SX_DEBUG_BUSY_7__COL_SCBD1_BUSY_MASK                                                                  0x00000002L
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ1_MASK                                                         0x00000004L
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ1_ADJ_MASK                                                     0x00000008L
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ2_MASK                                                         0x00000010L
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ3_MASK                                                         0x00000020L
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ4_MASK                                                         0x00000040L
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ5_MASK                                                         0x00000080L
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALID_OUT_MASK                                                       0x00000100L
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ1_MASK                                                         0x00000200L
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ1_ADJ_MASK                                                     0x00000400L
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ2_MASK                                                         0x00000800L
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ3_MASK                                                         0x00001000L
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ4_MASK                                                         0x00002000L
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ5_MASK                                                         0x00004000L
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALID_OUT_MASK                                                       0x00008000L
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ1_MASK                                                         0x00010000L
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ1_ADJ_MASK                                                     0x00020000L
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ2_MASK                                                         0x00040000L
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ3_MASK                                                         0x00080000L
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ4_MASK                                                         0x00100000L
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ5_MASK                                                         0x00200000L
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALID_OUT_MASK                                                       0x00400000L
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ1_MASK                                                         0x00800000L
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ1_ADJ_MASK                                                     0x01000000L
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ2_MASK                                                         0x02000000L
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ3_MASK                                                         0x04000000L
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ4_MASK                                                         0x08000000L
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ5_MASK                                                         0x10000000L
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALID_OUT_MASK                                                       0x20000000L
+#define SX_DEBUG_BUSY_7__RESERVED_MASK                                                                        0xC0000000L
+//SX_DEBUG_BUSY_8
+#define SX_DEBUG_BUSY_8__POS_BANK7VAL3_BUSY__SHIFT                                                            0x0
+#define SX_DEBUG_BUSY_8__POS_BANK7VAL2_BUSY__SHIFT                                                            0x1
+#define SX_DEBUG_BUSY_8__POS_BANK7VAL1_BUSY__SHIFT                                                            0x2
+#define SX_DEBUG_BUSY_8__POS_BANK7VAL0_BUSY__SHIFT                                                            0x3
+#define SX_DEBUG_BUSY_8__POS_BANK6VAL3_BUSY__SHIFT                                                            0x4
+#define SX_DEBUG_BUSY_8__POS_BANK6VAL2_BUSY__SHIFT                                                            0x5
+#define SX_DEBUG_BUSY_8__POS_BANK6VAL1_BUSY__SHIFT                                                            0x6
+#define SX_DEBUG_BUSY_8__POS_BANK6VAL0_BUSY__SHIFT                                                            0x7
+#define SX_DEBUG_BUSY_8__POS_BANK5VAL3_BUSY__SHIFT                                                            0x8
+#define SX_DEBUG_BUSY_8__POS_BANK5VAL2_BUSY__SHIFT                                                            0x9
+#define SX_DEBUG_BUSY_8__POS_BANK5VAL1_BUSY__SHIFT                                                            0xa
+#define SX_DEBUG_BUSY_8__POS_BANK5VAL0_BUSY__SHIFT                                                            0xb
+#define SX_DEBUG_BUSY_8__POS_BANK4VAL3_BUSY__SHIFT                                                            0xc
+#define SX_DEBUG_BUSY_8__POS_BANK4VAL2_BUSY__SHIFT                                                            0xd
+#define SX_DEBUG_BUSY_8__POS_BANK4VAL1_BUSY__SHIFT                                                            0xe
+#define SX_DEBUG_BUSY_8__POS_BANK4VAL0_BUSY__SHIFT                                                            0xf
+#define SX_DEBUG_BUSY_8__POS_BANK3VAL3_BUSY__SHIFT                                                            0x10
+#define SX_DEBUG_BUSY_8__POS_BANK3VAL2_BUSY__SHIFT                                                            0x11
+#define SX_DEBUG_BUSY_8__POS_BANK3VAL1_BUSY__SHIFT                                                            0x12
+#define SX_DEBUG_BUSY_8__POS_BANK3VAL0_BUSY__SHIFT                                                            0x13
+#define SX_DEBUG_BUSY_8__POS_BANK2VAL3_BUSY__SHIFT                                                            0x14
+#define SX_DEBUG_BUSY_8__POS_BANK2VAL2_BUSY__SHIFT                                                            0x15
+#define SX_DEBUG_BUSY_8__POS_BANK2VAL1_BUSY__SHIFT                                                            0x16
+#define SX_DEBUG_BUSY_8__POS_BANK2VAL0_BUSY__SHIFT                                                            0x17
+#define SX_DEBUG_BUSY_8__POS_BANK1VAL3_BUSY__SHIFT                                                            0x18
+#define SX_DEBUG_BUSY_8__POS_BANK1VAL2_BUSY__SHIFT                                                            0x19
+#define SX_DEBUG_BUSY_8__POS_BANK1VAL1_BUSY__SHIFT                                                            0x1a
+#define SX_DEBUG_BUSY_8__POS_BANK1VAL0_BUSY__SHIFT                                                            0x1b
+#define SX_DEBUG_BUSY_8__POS_BANK0VAL3_BUSY__SHIFT                                                            0x1c
+#define SX_DEBUG_BUSY_8__POS_BANK0VAL2_BUSY__SHIFT                                                            0x1d
+#define SX_DEBUG_BUSY_8__POS_BANK0VAL1_BUSY__SHIFT                                                            0x1e
+#define SX_DEBUG_BUSY_8__POS_BANK0VAL0_BUSY__SHIFT                                                            0x1f
+#define SX_DEBUG_BUSY_8__POS_BANK7VAL3_BUSY_MASK                                                              0x00000001L
+#define SX_DEBUG_BUSY_8__POS_BANK7VAL2_BUSY_MASK                                                              0x00000002L
+#define SX_DEBUG_BUSY_8__POS_BANK7VAL1_BUSY_MASK                                                              0x00000004L
+#define SX_DEBUG_BUSY_8__POS_BANK7VAL0_BUSY_MASK                                                              0x00000008L
+#define SX_DEBUG_BUSY_8__POS_BANK6VAL3_BUSY_MASK                                                              0x00000010L
+#define SX_DEBUG_BUSY_8__POS_BANK6VAL2_BUSY_MASK                                                              0x00000020L
+#define SX_DEBUG_BUSY_8__POS_BANK6VAL1_BUSY_MASK                                                              0x00000040L
+#define SX_DEBUG_BUSY_8__POS_BANK6VAL0_BUSY_MASK                                                              0x00000080L
+#define SX_DEBUG_BUSY_8__POS_BANK5VAL3_BUSY_MASK                                                              0x00000100L
+#define SX_DEBUG_BUSY_8__POS_BANK5VAL2_BUSY_MASK                                                              0x00000200L
+#define SX_DEBUG_BUSY_8__POS_BANK5VAL1_BUSY_MASK                                                              0x00000400L
+#define SX_DEBUG_BUSY_8__POS_BANK5VAL0_BUSY_MASK                                                              0x00000800L
+#define SX_DEBUG_BUSY_8__POS_BANK4VAL3_BUSY_MASK                                                              0x00001000L
+#define SX_DEBUG_BUSY_8__POS_BANK4VAL2_BUSY_MASK                                                              0x00002000L
+#define SX_DEBUG_BUSY_8__POS_BANK4VAL1_BUSY_MASK                                                              0x00004000L
+#define SX_DEBUG_BUSY_8__POS_BANK4VAL0_BUSY_MASK                                                              0x00008000L
+#define SX_DEBUG_BUSY_8__POS_BANK3VAL3_BUSY_MASK                                                              0x00010000L
+#define SX_DEBUG_BUSY_8__POS_BANK3VAL2_BUSY_MASK                                                              0x00020000L
+#define SX_DEBUG_BUSY_8__POS_BANK3VAL1_BUSY_MASK                                                              0x00040000L
+#define SX_DEBUG_BUSY_8__POS_BANK3VAL0_BUSY_MASK                                                              0x00080000L
+#define SX_DEBUG_BUSY_8__POS_BANK2VAL3_BUSY_MASK                                                              0x00100000L
+#define SX_DEBUG_BUSY_8__POS_BANK2VAL2_BUSY_MASK                                                              0x00200000L
+#define SX_DEBUG_BUSY_8__POS_BANK2VAL1_BUSY_MASK                                                              0x00400000L
+#define SX_DEBUG_BUSY_8__POS_BANK2VAL0_BUSY_MASK                                                              0x00800000L
+#define SX_DEBUG_BUSY_8__POS_BANK1VAL3_BUSY_MASK                                                              0x01000000L
+#define SX_DEBUG_BUSY_8__POS_BANK1VAL2_BUSY_MASK                                                              0x02000000L
+#define SX_DEBUG_BUSY_8__POS_BANK1VAL1_BUSY_MASK                                                              0x04000000L
+#define SX_DEBUG_BUSY_8__POS_BANK1VAL0_BUSY_MASK                                                              0x08000000L
+#define SX_DEBUG_BUSY_8__POS_BANK0VAL3_BUSY_MASK                                                              0x10000000L
+#define SX_DEBUG_BUSY_8__POS_BANK0VAL2_BUSY_MASK                                                              0x20000000L
+#define SX_DEBUG_BUSY_8__POS_BANK0VAL1_BUSY_MASK                                                              0x40000000L
+#define SX_DEBUG_BUSY_8__POS_BANK0VAL0_BUSY_MASK                                                              0x80000000L
+//SX_DEBUG_BUSY_9
+#define SX_DEBUG_BUSY_9__IDX_BANK7VAL3_BUSY__SHIFT                                                            0x0
+#define SX_DEBUG_BUSY_9__IDX_BANK7VAL2_BUSY__SHIFT                                                            0x1
+#define SX_DEBUG_BUSY_9__IDX_BANK7VAL1_BUSY__SHIFT                                                            0x2
+#define SX_DEBUG_BUSY_9__IDX_BANK7VAL0_BUSY__SHIFT                                                            0x3
+#define SX_DEBUG_BUSY_9__IDX_BANK6VAL3_BUSY__SHIFT                                                            0x4
+#define SX_DEBUG_BUSY_9__IDX_BANK6VAL2_BUSY__SHIFT                                                            0x5
+#define SX_DEBUG_BUSY_9__IDX_BANK6VAL1_BUSY__SHIFT                                                            0x6
+#define SX_DEBUG_BUSY_9__IDX_BANK6VAL0_BUSY__SHIFT                                                            0x7
+#define SX_DEBUG_BUSY_9__IDX_BANK5VAL3_BUSY__SHIFT                                                            0x8
+#define SX_DEBUG_BUSY_9__IDX_BANK5VAL2_BUSY__SHIFT                                                            0x9
+#define SX_DEBUG_BUSY_9__IDX_BANK5VAL1_BUSY__SHIFT                                                            0xa
+#define SX_DEBUG_BUSY_9__IDX_BANK5VAL0_BUSY__SHIFT                                                            0xb
+#define SX_DEBUG_BUSY_9__IDX_BANK4VAL3_BUSY__SHIFT                                                            0xc
+#define SX_DEBUG_BUSY_9__IDX_BANK4VAL2_BUSY__SHIFT                                                            0xd
+#define SX_DEBUG_BUSY_9__IDX_BANK4VAL1_BUSY__SHIFT                                                            0xe
+#define SX_DEBUG_BUSY_9__IDX_BANK4VAL0_BUSY__SHIFT                                                            0xf
+#define SX_DEBUG_BUSY_9__IDX_BANK3VAL3_BUSY__SHIFT                                                            0x10
+#define SX_DEBUG_BUSY_9__IDX_BANK3VAL2_BUSY__SHIFT                                                            0x11
+#define SX_DEBUG_BUSY_9__IDX_BANK3VAL1_BUSY__SHIFT                                                            0x12
+#define SX_DEBUG_BUSY_9__IDX_BANK3VAL0_BUSY__SHIFT                                                            0x13
+#define SX_DEBUG_BUSY_9__IDX_BANK2VAL3_BUSY__SHIFT                                                            0x14
+#define SX_DEBUG_BUSY_9__IDX_BANK2VAL2_BUSY__SHIFT                                                            0x15
+#define SX_DEBUG_BUSY_9__IDX_BANK2VAL1_BUSY__SHIFT                                                            0x16
+#define SX_DEBUG_BUSY_9__IDX_BANK2VAL0_BUSY__SHIFT                                                            0x17
+#define SX_DEBUG_BUSY_9__IDX_BANK1VAL3_BUSY__SHIFT                                                            0x18
+#define SX_DEBUG_BUSY_9__IDX_BANK1VAL2_BUSY__SHIFT                                                            0x19
+#define SX_DEBUG_BUSY_9__IDX_BANK1VAL1_BUSY__SHIFT                                                            0x1a
+#define SX_DEBUG_BUSY_9__IDX_BANK1VAL0_BUSY__SHIFT                                                            0x1b
+#define SX_DEBUG_BUSY_9__IDX_BANK0VAL3_BUSY__SHIFT                                                            0x1c
+#define SX_DEBUG_BUSY_9__IDX_BANK0VAL2_BUSY__SHIFT                                                            0x1d
+#define SX_DEBUG_BUSY_9__IDX_BANK0VAL1_BUSY__SHIFT                                                            0x1e
+#define SX_DEBUG_BUSY_9__IDX_BANK0VAL0_BUSY__SHIFT                                                            0x1f
+#define SX_DEBUG_BUSY_9__IDX_BANK7VAL3_BUSY_MASK                                                              0x00000001L
+#define SX_DEBUG_BUSY_9__IDX_BANK7VAL2_BUSY_MASK                                                              0x00000002L
+#define SX_DEBUG_BUSY_9__IDX_BANK7VAL1_BUSY_MASK                                                              0x00000004L
+#define SX_DEBUG_BUSY_9__IDX_BANK7VAL0_BUSY_MASK                                                              0x00000008L
+#define SX_DEBUG_BUSY_9__IDX_BANK6VAL3_BUSY_MASK                                                              0x00000010L
+#define SX_DEBUG_BUSY_9__IDX_BANK6VAL2_BUSY_MASK                                                              0x00000020L
+#define SX_DEBUG_BUSY_9__IDX_BANK6VAL1_BUSY_MASK                                                              0x00000040L
+#define SX_DEBUG_BUSY_9__IDX_BANK6VAL0_BUSY_MASK                                                              0x00000080L
+#define SX_DEBUG_BUSY_9__IDX_BANK5VAL3_BUSY_MASK                                                              0x00000100L
+#define SX_DEBUG_BUSY_9__IDX_BANK5VAL2_BUSY_MASK                                                              0x00000200L
+#define SX_DEBUG_BUSY_9__IDX_BANK5VAL1_BUSY_MASK                                                              0x00000400L
+#define SX_DEBUG_BUSY_9__IDX_BANK5VAL0_BUSY_MASK                                                              0x00000800L
+#define SX_DEBUG_BUSY_9__IDX_BANK4VAL3_BUSY_MASK                                                              0x00001000L
+#define SX_DEBUG_BUSY_9__IDX_BANK4VAL2_BUSY_MASK                                                              0x00002000L
+#define SX_DEBUG_BUSY_9__IDX_BANK4VAL1_BUSY_MASK                                                              0x00004000L
+#define SX_DEBUG_BUSY_9__IDX_BANK4VAL0_BUSY_MASK                                                              0x00008000L
+#define SX_DEBUG_BUSY_9__IDX_BANK3VAL3_BUSY_MASK                                                              0x00010000L
+#define SX_DEBUG_BUSY_9__IDX_BANK3VAL2_BUSY_MASK                                                              0x00020000L
+#define SX_DEBUG_BUSY_9__IDX_BANK3VAL1_BUSY_MASK                                                              0x00040000L
+#define SX_DEBUG_BUSY_9__IDX_BANK3VAL0_BUSY_MASK                                                              0x00080000L
+#define SX_DEBUG_BUSY_9__IDX_BANK2VAL3_BUSY_MASK                                                              0x00100000L
+#define SX_DEBUG_BUSY_9__IDX_BANK2VAL2_BUSY_MASK                                                              0x00200000L
+#define SX_DEBUG_BUSY_9__IDX_BANK2VAL1_BUSY_MASK                                                              0x00400000L
+#define SX_DEBUG_BUSY_9__IDX_BANK2VAL0_BUSY_MASK                                                              0x00800000L
+#define SX_DEBUG_BUSY_9__IDX_BANK1VAL3_BUSY_MASK                                                              0x01000000L
+#define SX_DEBUG_BUSY_9__IDX_BANK1VAL2_BUSY_MASK                                                              0x02000000L
+#define SX_DEBUG_BUSY_9__IDX_BANK1VAL1_BUSY_MASK                                                              0x04000000L
+#define SX_DEBUG_BUSY_9__IDX_BANK1VAL0_BUSY_MASK                                                              0x08000000L
+#define SX_DEBUG_BUSY_9__IDX_BANK0VAL3_BUSY_MASK                                                              0x10000000L
+#define SX_DEBUG_BUSY_9__IDX_BANK0VAL2_BUSY_MASK                                                              0x20000000L
+#define SX_DEBUG_BUSY_9__IDX_BANK0VAL1_BUSY_MASK                                                              0x40000000L
+#define SX_DEBUG_BUSY_9__IDX_BANK0VAL0_BUSY_MASK                                                              0x80000000L
+//SX_DEBUG_BUSY_10
+#define SX_DEBUG_BUSY_10__POS_SCBD_BUSY__SHIFT                                                                0x0
+#define SX_DEBUG_BUSY_10__POS_FREE_OR_VALIDS__SHIFT                                                           0x1
+#define SX_DEBUG_BUSY_10__POS_REQUESTER_BUSY__SHIFT                                                           0x2
+#define SX_DEBUG_BUSY_10__PA_SX_BUSY__SHIFT                                                                   0x3
+#define SX_DEBUG_BUSY_10__POS_WRCTRL1_VALIDQ3__SHIFT                                                          0x4
+#define SX_DEBUG_BUSY_10__POS_WRCTRL1_VALIDQ2__SHIFT                                                          0x5
+#define SX_DEBUG_BUSY_10__POS_WRCTRL1_VALIDQ1__SHIFT                                                          0x6
+#define SX_DEBUG_BUSY_10__IDX_SCBD_BUSY__SHIFT                                                                0x7
+#define SX_DEBUG_BUSY_10__IDX_FREE_OR_VALIDS__SHIFT                                                           0x8
+#define SX_DEBUG_BUSY_10__IDX_REQUESTER_BUSY__SHIFT                                                           0x9
+#define SX_DEBUG_BUSY_10__PA_SX_IDX_BUSY__SHIFT                                                               0xa
+#define SX_DEBUG_BUSY_10__IDX_WRCTRL1_VALIDQ3__SHIFT                                                          0xb
+#define SX_DEBUG_BUSY_10__IDX_WRCTRL1_VALIDQ2__SHIFT                                                          0xc
+#define SX_DEBUG_BUSY_10__IDX_WRCTRL1_VALIDQ1__SHIFT                                                          0xd
+#define SX_DEBUG_BUSY_10__RESERVED__SHIFT                                                                     0xe
+#define SX_DEBUG_BUSY_10__POS_SCBD_BUSY_MASK                                                                  0x00000001L
+#define SX_DEBUG_BUSY_10__POS_FREE_OR_VALIDS_MASK                                                             0x00000002L
+#define SX_DEBUG_BUSY_10__POS_REQUESTER_BUSY_MASK                                                             0x00000004L
+#define SX_DEBUG_BUSY_10__PA_SX_BUSY_MASK                                                                     0x00000008L
+#define SX_DEBUG_BUSY_10__POS_WRCTRL1_VALIDQ3_MASK                                                            0x00000010L
+#define SX_DEBUG_BUSY_10__POS_WRCTRL1_VALIDQ2_MASK                                                            0x00000020L
+#define SX_DEBUG_BUSY_10__POS_WRCTRL1_VALIDQ1_MASK                                                            0x00000040L
+#define SX_DEBUG_BUSY_10__IDX_SCBD_BUSY_MASK                                                                  0x00000080L
+#define SX_DEBUG_BUSY_10__IDX_FREE_OR_VALIDS_MASK                                                             0x00000100L
+#define SX_DEBUG_BUSY_10__IDX_REQUESTER_BUSY_MASK                                                             0x00000200L
+#define SX_DEBUG_BUSY_10__PA_SX_IDX_BUSY_MASK                                                                 0x00000400L
+#define SX_DEBUG_BUSY_10__IDX_WRCTRL1_VALIDQ3_MASK                                                            0x00000800L
+#define SX_DEBUG_BUSY_10__IDX_WRCTRL1_VALIDQ2_MASK                                                            0x00001000L
+#define SX_DEBUG_BUSY_10__IDX_WRCTRL1_VALIDQ1_MASK                                                            0x00002000L
+#define SX_DEBUG_BUSY_10__RESERVED_MASK                                                                       0xFFFFC000L
+//SPI_PS_MAX_WAVE_ID
+#define SPI_PS_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT                                                                0x0
+#define SPI_PS_MAX_WAVE_ID__MAX_COLLISION_WAVE_ID__SHIFT                                                      0x10
+#define SPI_PS_MAX_WAVE_ID__MAX_WAVE_ID_MASK                                                                  0x00000FFFL
+#define SPI_PS_MAX_WAVE_ID__MAX_COLLISION_WAVE_ID_MASK                                                        0x03FF0000L
+//SPI_SCRATCH_ADDR_STATUS
+#define SPI_SCRATCH_ADDR_STATUS__OVERFLOW_DETECTED__SHIFT                                                     0x1
+#define SPI_SCRATCH_ADDR_STATUS__ME_ID__SHIFT                                                                 0x2
+#define SPI_SCRATCH_ADDR_STATUS__PIPE_ID__SHIFT                                                               0x4
+#define SPI_SCRATCH_ADDR_STATUS__OVERFLOW_DETECTED_MASK                                                       0x00000002L
+#define SPI_SCRATCH_ADDR_STATUS__ME_ID_MASK                                                                   0x0000000CL
+#define SPI_SCRATCH_ADDR_STATUS__PIPE_ID_MASK                                                                 0x00000030L
+//SPI_GFX_CNTL
+#define SPI_GFX_CNTL__RESET_COUNTS__SHIFT                                                                     0x0
+#define SPI_GFX_CNTL__RESET_COUNTS_MASK                                                                       0x00000001L
+//SPI_DEBUG_CNTL_2
+#define SPI_DEBUG_CNTL_2__ECO_SPARE_0__SHIFT                                                                  0x0
+#define SPI_DEBUG_CNTL_2__ECO_SPARE_1__SHIFT                                                                  0x1
+#define SPI_DEBUG_CNTL_2__ECO_SPARE_2__SHIFT                                                                  0x2
+#define SPI_DEBUG_CNTL_2__ECO_SPARE_3__SHIFT                                                                  0x3
+#define SPI_DEBUG_CNTL_2__ECO_SPARE_4__SHIFT                                                                  0x4
+#define SPI_DEBUG_CNTL_2__ECO_SPARE_5__SHIFT                                                                  0x5
+#define SPI_DEBUG_CNTL_2__ECO_SPARE_6__SHIFT                                                                  0x6
+#define SPI_DEBUG_CNTL_2__ECO_SPARE_7__SHIFT                                                                  0x7
+#define SPI_DEBUG_CNTL_2__DISABLE_INTRA_PRIM_CONFLICT__SHIFT                                                  0x8
+#define SPI_DEBUG_CNTL_2__DISABLE_PS_AGE_SORT__SHIFT                                                          0x9
+#define SPI_DEBUG_CNTL_2__DISABLE_VSGS_AGE_SORT__SHIFT                                                        0xa
+#define SPI_DEBUG_CNTL_2__PA_CSB_DEPTH__SHIFT                                                                 0xb
+#define SPI_DEBUG_CNTL_2__DISABLE_PSUD_SAME_ADDR_OPT__SHIFT                                                   0xf
+#define SPI_DEBUG_CNTL_2__SPI_S_WAVE_WR_CTL_BUSY__SHIFT                                                       0x10
+#define SPI_DEBUG_CNTL_2__DISABLE_EARLY_COL_QUEUE_RD__SHIFT                                                   0x11
+#define SPI_DEBUG_CNTL_2__DISABLE_EGM_SAME_ADDR_OPT__SHIFT                                                    0x12
+#define SPI_DEBUG_CNTL_2__SPI_S_WB_WCT_BUSY__SHIFT                                                            0x16
+#define SPI_DEBUG_CNTL_2__DISABLE_CSG_CRAWLER_ACTIVE_FGCG_OPT__SHIFT                                          0x17
+#define SPI_DEBUG_CNTL_2__DISABLE_CSC_CRAWLER_ACTIVE_FGCG_OPT__SHIFT                                          0x18
+#define SPI_DEBUG_CNTL_2__ECO_SPARE_0_MASK                                                                    0x00000001L
+#define SPI_DEBUG_CNTL_2__ECO_SPARE_1_MASK                                                                    0x00000002L
+#define SPI_DEBUG_CNTL_2__ECO_SPARE_2_MASK                                                                    0x00000004L
+#define SPI_DEBUG_CNTL_2__ECO_SPARE_3_MASK                                                                    0x00000008L
+#define SPI_DEBUG_CNTL_2__ECO_SPARE_4_MASK                                                                    0x00000010L
+#define SPI_DEBUG_CNTL_2__ECO_SPARE_5_MASK                                                                    0x00000020L
+#define SPI_DEBUG_CNTL_2__ECO_SPARE_6_MASK                                                                    0x00000040L
+#define SPI_DEBUG_CNTL_2__ECO_SPARE_7_MASK                                                                    0x00000080L
+#define SPI_DEBUG_CNTL_2__DISABLE_INTRA_PRIM_CONFLICT_MASK                                                    0x00000100L
+#define SPI_DEBUG_CNTL_2__DISABLE_PS_AGE_SORT_MASK                                                            0x00000200L
+#define SPI_DEBUG_CNTL_2__DISABLE_VSGS_AGE_SORT_MASK                                                          0x00000400L
+#define SPI_DEBUG_CNTL_2__PA_CSB_DEPTH_MASK                                                                   0x00007800L
+#define SPI_DEBUG_CNTL_2__DISABLE_PSUD_SAME_ADDR_OPT_MASK                                                     0x00008000L
+#define SPI_DEBUG_CNTL_2__SPI_S_WAVE_WR_CTL_BUSY_MASK                                                         0x00010000L
+#define SPI_DEBUG_CNTL_2__DISABLE_EARLY_COL_QUEUE_RD_MASK                                                     0x00020000L
+#define SPI_DEBUG_CNTL_2__DISABLE_EGM_SAME_ADDR_OPT_MASK                                                      0x00040000L
+#define SPI_DEBUG_CNTL_2__SPI_S_WB_WCT_BUSY_MASK                                                              0x00400000L
+#define SPI_DEBUG_CNTL_2__DISABLE_CSG_CRAWLER_ACTIVE_FGCG_OPT_MASK                                            0x00800000L
+#define SPI_DEBUG_CNTL_2__DISABLE_CSC_CRAWLER_ACTIVE_FGCG_OPT_MASK                                            0x01000000L
+//SPI_DEBUG_CNTL_3
+#define SPI_DEBUG_CNTL_3__CSC_PUSH_CREDITS__SHIFT                                                             0x0
+#define SPI_DEBUG_CNTL_3__CSC_POP_CREDITS__SHIFT                                                              0x5
+#define SPI_DEBUG_CNTL_3__CSC_PUSH_CREDITS_MASK                                                               0x0000001FL
+#define SPI_DEBUG_CNTL_3__CSC_POP_CREDITS_MASK                                                                0x000003E0L
+//SPI_DEBUG_CNTL
+#define SPI_DEBUG_CNTL__DEBUG_GFX_PIPE_SEL__SHIFT                                                             0x0
+#define SPI_DEBUG_CNTL__DEBUG_THREAD_TYPE_SEL__SHIFT                                                          0x1
+#define SPI_DEBUG_CNTL__DEBUG_GROUP_SEL__SHIFT                                                                0x4
+#define SPI_DEBUG_CNTL__DEBUG_SIMD_SEL__SHIFT                                                                 0xa
+#define SPI_DEBUG_CNTL__DEBUG_SH_SEL__SHIFT                                                                   0x10
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_0__SHIFT                                                                0x11
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_1__SHIFT                                                                0x12
+#define SPI_DEBUG_CNTL__PS_PSTNT_STATE_PIPELINE_ENABLE__SHIFT                                                 0x13
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_3__SHIFT                                                                0x14
+#define SPI_DEBUG_CNTL__CGTS_VBUS_SP0_OVERRIDE__SHIFT                                                         0x15
+#define SPI_DEBUG_CNTL__CGTS_VBUS_SP1_OVERRIDE__SHIFT                                                         0x16
+#define SPI_DEBUG_CNTL__ENABLE_FIFO_DEBUG_WRITE__SHIFT                                                        0x17
+#define SPI_DEBUG_CNTL__CGTT_LEGACY_MODE__SHIFT                                                               0x18
+#define SPI_DEBUG_CNTL__DEBUG_PIPE_SEL__SHIFT                                                                 0x19
+#define SPI_DEBUG_CNTL__DEBUG_PIXEL_PIPE_SEL__SHIFT                                                           0x1c
+#define SPI_DEBUG_CNTL__BCI_PIPE_PER_STAGE_CG_OVERRIDE__SHIFT                                                 0x1e
+#define SPI_DEBUG_CNTL__DEBUG_REG_EN__SHIFT                                                                   0x1f
+#define SPI_DEBUG_CNTL__DEBUG_GFX_PIPE_SEL_MASK                                                               0x00000001L
+#define SPI_DEBUG_CNTL__DEBUG_THREAD_TYPE_SEL_MASK                                                            0x0000000EL
+#define SPI_DEBUG_CNTL__DEBUG_GROUP_SEL_MASK                                                                  0x000003F0L
+#define SPI_DEBUG_CNTL__DEBUG_SIMD_SEL_MASK                                                                   0x0000FC00L
+#define SPI_DEBUG_CNTL__DEBUG_SH_SEL_MASK                                                                     0x00010000L
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_0_MASK                                                                  0x00020000L
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_1_MASK                                                                  0x00040000L
+#define SPI_DEBUG_CNTL__PS_PSTNT_STATE_PIPELINE_ENABLE_MASK                                                   0x00080000L
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_3_MASK                                                                  0x00100000L
+#define SPI_DEBUG_CNTL__CGTS_VBUS_SP0_OVERRIDE_MASK                                                           0x00200000L
+#define SPI_DEBUG_CNTL__CGTS_VBUS_SP1_OVERRIDE_MASK                                                           0x00400000L
+#define SPI_DEBUG_CNTL__ENABLE_FIFO_DEBUG_WRITE_MASK                                                          0x00800000L
+#define SPI_DEBUG_CNTL__CGTT_LEGACY_MODE_MASK                                                                 0x01000000L
+#define SPI_DEBUG_CNTL__DEBUG_PIPE_SEL_MASK                                                                   0x0E000000L
+#define SPI_DEBUG_CNTL__DEBUG_PIXEL_PIPE_SEL_MASK                                                             0x30000000L
+#define SPI_DEBUG_CNTL__BCI_PIPE_PER_STAGE_CG_OVERRIDE_MASK                                                   0x40000000L
+#define SPI_DEBUG_CNTL__DEBUG_REG_EN_MASK                                                                     0x80000000L
+//SPI_DEBUG_READ
+#define SPI_DEBUG_READ__DATA__SHIFT                                                                           0x0
+#define SPI_DEBUG_READ__DATA_MASK                                                                             0xFFFFFFFFL
+//SPI_DSM_CNTL
+#define SPI_DSM_CNTL__SPI_SR_MEM_DSM_IRRITATOR_DATA__SHIFT                                                    0x0
+#define SPI_DSM_CNTL__SPI_SR_MEM_ENABLE_SINGLE_WRITE__SHIFT                                                   0x2
+#define SPI_DSM_CNTL__SPI_SR_MEM_DSM_IRRITATOR_DATA_MASK                                                      0x00000003L
+#define SPI_DSM_CNTL__SPI_SR_MEM_ENABLE_SINGLE_WRITE_MASK                                                     0x00000004L
+//SPI_DSM_CNTL2
+#define SPI_DSM_CNTL2__SPI_SR_MEM_ENABLE_ERROR_INJECT__SHIFT                                                  0x0
+#define SPI_DSM_CNTL2__SPI_SR_MEM_SELECT_INJECT_DELAY__SHIFT                                                  0x2
+#define SPI_DSM_CNTL2__SPI_SR_MEM_INJECT_DELAY__SHIFT                                                         0x3
+#define SPI_DSM_CNTL2__SPI_SR_MEM_ENABLE_ERROR_INJECT_MASK                                                    0x00000003L
+#define SPI_DSM_CNTL2__SPI_SR_MEM_SELECT_INJECT_DELAY_MASK                                                    0x00000004L
+#define SPI_DSM_CNTL2__SPI_SR_MEM_INJECT_DELAY_MASK                                                           0x000001F8L
+//SPI_EDC_CNT
+#define SPI_EDC_CNT__SPI_SR_MEM_SED_COUNT__SHIFT                                                              0x0
+#define SPI_EDC_CNT__SPI_SR_MEM_SED_COUNT_MASK                                                                0x00000003L
+//SPIRA_DEBUG_READ
+#define SPIRA_DEBUG_READ__DATA__SHIFT                                                                         0x0
+#define SPIRA_DEBUG_READ__DATA_MASK                                                                           0xFFFFFFFFL
+//SPI_DEBUG_BUSY
+#define SPI_DEBUG_BUSY__HS_BUSY__SHIFT                                                                        0x0
+#define SPI_DEBUG_BUSY__GS_BUSY__SHIFT                                                                        0x1
+#define SPI_DEBUG_BUSY__PS0_BUSY__SHIFT                                                                       0x2
+#define SPI_DEBUG_BUSY__PS1_BUSY__SHIFT                                                                       0x3
+#define SPI_DEBUG_BUSY__PS2_BUSY__SHIFT                                                                       0x4
+#define SPI_DEBUG_BUSY__PS3_BUSY__SHIFT                                                                       0x5
+#define SPI_DEBUG_BUSY__CSG0_BUSY__SHIFT                                                                      0x6
+#define SPI_DEBUG_BUSY__CSG1_BUSY__SHIFT                                                                      0x7
+#define SPI_DEBUG_BUSY__CS0_BUSY__SHIFT                                                                       0x8
+#define SPI_DEBUG_BUSY__CS1_BUSY__SHIFT                                                                       0x9
+#define SPI_DEBUG_BUSY__CS2_BUSY__SHIFT                                                                       0xa
+#define SPI_DEBUG_BUSY__CS3_BUSY__SHIFT                                                                       0xb
+#define SPI_DEBUG_BUSY__CS4_BUSY__SHIFT                                                                       0xc
+#define SPI_DEBUG_BUSY__CS5_BUSY__SHIFT                                                                       0xd
+#define SPI_DEBUG_BUSY__CS6_BUSY__SHIFT                                                                       0xe
+#define SPI_DEBUG_BUSY__CS7_BUSY__SHIFT                                                                       0xf
+#define SPI_DEBUG_BUSY__PC_DEALLOC_BUSY__SHIFT                                                                0x12
+#define SPI_DEBUG_BUSY__OFC_LDS_BUSY__SHIFT                                                                   0x13
+#define SPI_DEBUG_BUSY__EVENT_CLCTR_BUSY__SHIFT                                                               0x14
+#define SPI_DEBUG_BUSY__GRBM_BUSY__SHIFT                                                                      0x15
+#define SPI_DEBUG_BUSY__SPIS_BUSY__SHIFT                                                                      0x16
+#define SPI_DEBUG_BUSY__RSRC_ALLOC_BUSY__SHIFT                                                                0x17
+#define SPI_DEBUG_BUSY__PWS_BUSY__SHIFT                                                                       0x18
+#define SPI_DEBUG_BUSY__SPP_BUSY__SHIFT                                                                       0x19
+#define SPI_DEBUG_BUSY__HS_BUSY_MASK                                                                          0x00000001L
+#define SPI_DEBUG_BUSY__GS_BUSY_MASK                                                                          0x00000002L
+#define SPI_DEBUG_BUSY__PS0_BUSY_MASK                                                                         0x00000004L
+#define SPI_DEBUG_BUSY__PS1_BUSY_MASK                                                                         0x00000008L
+#define SPI_DEBUG_BUSY__PS2_BUSY_MASK                                                                         0x00000010L
+#define SPI_DEBUG_BUSY__PS3_BUSY_MASK                                                                         0x00000020L
+#define SPI_DEBUG_BUSY__CSG0_BUSY_MASK                                                                        0x00000040L
+#define SPI_DEBUG_BUSY__CSG1_BUSY_MASK                                                                        0x00000080L
+#define SPI_DEBUG_BUSY__CS0_BUSY_MASK                                                                         0x00000100L
+#define SPI_DEBUG_BUSY__CS1_BUSY_MASK                                                                         0x00000200L
+#define SPI_DEBUG_BUSY__CS2_BUSY_MASK                                                                         0x00000400L
+#define SPI_DEBUG_BUSY__CS3_BUSY_MASK                                                                         0x00000800L
+#define SPI_DEBUG_BUSY__CS4_BUSY_MASK                                                                         0x00001000L
+#define SPI_DEBUG_BUSY__CS5_BUSY_MASK                                                                         0x00002000L
+#define SPI_DEBUG_BUSY__CS6_BUSY_MASK                                                                         0x00004000L
+#define SPI_DEBUG_BUSY__CS7_BUSY_MASK                                                                         0x00008000L
+#define SPI_DEBUG_BUSY__PC_DEALLOC_BUSY_MASK                                                                  0x00040000L
+#define SPI_DEBUG_BUSY__OFC_LDS_BUSY_MASK                                                                     0x00080000L
+#define SPI_DEBUG_BUSY__EVENT_CLCTR_BUSY_MASK                                                                 0x00100000L
+#define SPI_DEBUG_BUSY__GRBM_BUSY_MASK                                                                        0x00200000L
+#define SPI_DEBUG_BUSY__SPIS_BUSY_MASK                                                                        0x00400000L
+#define SPI_DEBUG_BUSY__RSRC_ALLOC_BUSY_MASK                                                                  0x00800000L
+#define SPI_DEBUG_BUSY__PWS_BUSY_MASK                                                                         0x01000000L
+#define SPI_DEBUG_BUSY__SPP_BUSY_MASK                                                                         0x02000000L
+//SPI_CONFIG_PS_CU_EN
+#define SPI_CONFIG_PS_CU_EN__PKR_OFFSET__SHIFT                                                                0x0
+#define SPI_CONFIG_PS_CU_EN__PKR2_OFFSET__SHIFT                                                               0x4
+#define SPI_CONFIG_PS_CU_EN__PKR3_OFFSET__SHIFT                                                               0x8
+#define SPI_CONFIG_PS_CU_EN__PKR_OFFSET_MASK                                                                  0x0000000FL
+#define SPI_CONFIG_PS_CU_EN__PKR2_OFFSET_MASK                                                                 0x000000F0L
+#define SPI_CONFIG_PS_CU_EN__PKR3_OFFSET_MASK                                                                 0x00000F00L
+//SPI_CONFIG_CU_MASK_GFX0
+#define SPI_CONFIG_CU_MASK_GFX0__HS_CU_EN__SHIFT                                                              0x0
+#define SPI_CONFIG_CU_MASK_GFX0__GS_CU_EN__SHIFT                                                              0x10
+#define SPI_CONFIG_CU_MASK_GFX0__HS_CU_EN_MASK                                                                0x0000FFFFL
+#define SPI_CONFIG_CU_MASK_GFX0__GS_CU_EN_MASK                                                                0xFFFF0000L
+//SPI_CONFIG_CU_MASK_HP3D0
+#define SPI_CONFIG_CU_MASK_HP3D0__HS_CU_EN__SHIFT                                                             0x0
+#define SPI_CONFIG_CU_MASK_HP3D0__GS_CU_EN__SHIFT                                                             0x10
+#define SPI_CONFIG_CU_MASK_HP3D0__HS_CU_EN_MASK                                                               0x0000FFFFL
+#define SPI_CONFIG_CU_MASK_HP3D0__GS_CU_EN_MASK                                                               0xFFFF0000L
+//SPI_CONFIG_CU_MASK_GFX1
+#define SPI_CONFIG_CU_MASK_GFX1__PS_CU_EN__SHIFT                                                              0x0
+#define SPI_CONFIG_CU_MASK_GFX1__CSG_CU_EN__SHIFT                                                             0x10
+#define SPI_CONFIG_CU_MASK_GFX1__PS_CU_EN_MASK                                                                0x0000FFFFL
+#define SPI_CONFIG_CU_MASK_GFX1__CSG_CU_EN_MASK                                                               0xFFFF0000L
+//SPI_CONFIG_CU_MASK_HP3D1
+#define SPI_CONFIG_CU_MASK_HP3D1__PS_CU_EN__SHIFT                                                             0x0
+#define SPI_CONFIG_CU_MASK_HP3D1__CSG_CU_EN__SHIFT                                                            0x10
+#define SPI_CONFIG_CU_MASK_HP3D1__PS_CU_EN_MASK                                                               0x0000FFFFL
+#define SPI_CONFIG_CU_MASK_HP3D1__CSG_CU_EN_MASK                                                              0xFFFF0000L
+//SPI_CONFIG_CU_MASK_CS0
+#define SPI_CONFIG_CU_MASK_CS0__CU_EN_SA0__SHIFT                                                              0x0
+#define SPI_CONFIG_CU_MASK_CS0__CU_EN_SA1__SHIFT                                                              0x10
+#define SPI_CONFIG_CU_MASK_CS0__CU_EN_SA0_MASK                                                                0x0000FFFFL
+#define SPI_CONFIG_CU_MASK_CS0__CU_EN_SA1_MASK                                                                0xFFFF0000L
+//SPI_CONFIG_CU_MASK_CS1
+#define SPI_CONFIG_CU_MASK_CS1__CU_EN_SA0__SHIFT                                                              0x0
+#define SPI_CONFIG_CU_MASK_CS1__CU_EN_SA1__SHIFT                                                              0x10
+#define SPI_CONFIG_CU_MASK_CS1__CU_EN_SA0_MASK                                                                0x0000FFFFL
+#define SPI_CONFIG_CU_MASK_CS1__CU_EN_SA1_MASK                                                                0xFFFF0000L
+//SPI_CONFIG_CU_MASK_CS2
+#define SPI_CONFIG_CU_MASK_CS2__CU_EN_SA0__SHIFT                                                              0x0
+#define SPI_CONFIG_CU_MASK_CS2__CU_EN_SA1__SHIFT                                                              0x10
+#define SPI_CONFIG_CU_MASK_CS2__CU_EN_SA0_MASK                                                                0x0000FFFFL
+#define SPI_CONFIG_CU_MASK_CS2__CU_EN_SA1_MASK                                                                0xFFFF0000L
+//SPI_CONFIG_CU_MASK_CS3
+#define SPI_CONFIG_CU_MASK_CS3__CU_EN_SA0__SHIFT                                                              0x0
+#define SPI_CONFIG_CU_MASK_CS3__CU_EN_SA1__SHIFT                                                              0x10
+#define SPI_CONFIG_CU_MASK_CS3__CU_EN_SA0_MASK                                                                0x0000FFFFL
+#define SPI_CONFIG_CU_MASK_CS3__CU_EN_SA1_MASK                                                                0xFFFF0000L
+//SPI_CONFIG_CU_MASK_CS4
+#define SPI_CONFIG_CU_MASK_CS4__CU_EN_SA0__SHIFT                                                              0x0
+#define SPI_CONFIG_CU_MASK_CS4__CU_EN_SA1__SHIFT                                                              0x10
+#define SPI_CONFIG_CU_MASK_CS4__CU_EN_SA0_MASK                                                                0x0000FFFFL
+#define SPI_CONFIG_CU_MASK_CS4__CU_EN_SA1_MASK                                                                0xFFFF0000L
+//SPI_CONFIG_CU_MASK_CS5
+#define SPI_CONFIG_CU_MASK_CS5__CU_EN_SA0__SHIFT                                                              0x0
+#define SPI_CONFIG_CU_MASK_CS5__CU_EN_SA1__SHIFT                                                              0x10
+#define SPI_CONFIG_CU_MASK_CS5__CU_EN_SA0_MASK                                                                0x0000FFFFL
+#define SPI_CONFIG_CU_MASK_CS5__CU_EN_SA1_MASK                                                                0xFFFF0000L
+//SPI_CONFIG_CU_MASK_CS6
+#define SPI_CONFIG_CU_MASK_CS6__CU_EN_SA0__SHIFT                                                              0x0
+#define SPI_CONFIG_CU_MASK_CS6__CU_EN_SA1__SHIFT                                                              0x10
+#define SPI_CONFIG_CU_MASK_CS6__CU_EN_SA0_MASK                                                                0x0000FFFFL
+#define SPI_CONFIG_CU_MASK_CS6__CU_EN_SA1_MASK                                                                0xFFFF0000L
+//SPI_CONFIG_CU_MASK_CS7
+#define SPI_CONFIG_CU_MASK_CS7__CU_EN_SA0__SHIFT                                                              0x0
+#define SPI_CONFIG_CU_MASK_CS7__CU_EN_SA1__SHIFT                                                              0x10
+#define SPI_CONFIG_CU_MASK_CS7__CU_EN_SA0_MASK                                                                0x0000FFFFL
+#define SPI_CONFIG_CU_MASK_CS7__CU_EN_SA1_MASK                                                                0xFFFF0000L
+//SPI_WF_LIFETIME_CNTL
+#define SPI_WF_LIFETIME_CNTL__SAMPLE_PERIOD__SHIFT                                                            0x0
+#define SPI_WF_LIFETIME_CNTL__EN__SHIFT                                                                       0x4
+#define SPI_WF_LIFETIME_CNTL__SAMPLE_PERIOD_MASK                                                              0x0000000FL
+#define SPI_WF_LIFETIME_CNTL__EN_MASK                                                                         0x00000010L
+//SPI_WF_LIFETIME_LIMIT_0
+#define SPI_WF_LIFETIME_LIMIT_0__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_0__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_0__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_0__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_2
+#define SPI_WF_LIFETIME_LIMIT_2__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_2__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_2__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_2__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_3
+#define SPI_WF_LIFETIME_LIMIT_3__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_3__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_3__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_3__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_4
+#define SPI_WF_LIFETIME_LIMIT_4__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_4__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_4__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_4__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_5
+#define SPI_WF_LIFETIME_LIMIT_5__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_5__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_5__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_5__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_STATUS_0
+#define SPI_WF_LIFETIME_STATUS_0__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_0__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_0__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_0__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_2
+#define SPI_WF_LIFETIME_STATUS_2__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_2__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_2__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_2__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_4
+#define SPI_WF_LIFETIME_STATUS_4__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_4__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_4__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_4__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_6
+#define SPI_WF_LIFETIME_STATUS_6__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_6__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_6__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_6__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_7
+#define SPI_WF_LIFETIME_STATUS_7__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_7__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_7__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_7__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_9
+#define SPI_WF_LIFETIME_STATUS_9__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_9__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_9__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_9__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_11
+#define SPI_WF_LIFETIME_STATUS_11__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_11__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_11__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_11__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_13
+#define SPI_WF_LIFETIME_STATUS_13__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_13__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_13__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_13__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_14
+#define SPI_WF_LIFETIME_STATUS_14__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_14__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_14__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_14__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_15
+#define SPI_WF_LIFETIME_STATUS_15__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_15__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_15__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_15__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_16
+#define SPI_WF_LIFETIME_STATUS_16__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_16__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_16__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_16__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_17
+#define SPI_WF_LIFETIME_STATUS_17__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_17__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_17__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_17__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_18
+#define SPI_WF_LIFETIME_STATUS_18__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_18__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_18__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_18__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_19
+#define SPI_WF_LIFETIME_STATUS_19__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_19__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_19__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_19__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_20
+#define SPI_WF_LIFETIME_STATUS_20__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_20__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_20__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_20__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_DEBUG
+#define SPI_WF_LIFETIME_DEBUG__START_VALUE__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_DEBUG__OVERRIDE_EN__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_DEBUG__START_VALUE_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_DEBUG__OVERRIDE_EN_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_21
+#define SPI_WF_LIFETIME_STATUS_21__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_21__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_21__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_21__INT_SENT_MASK                                                              0x80000000L
+//SPI_WGP_WORK_PENDING
+#define SPI_WGP_WORK_PENDING__SPI_WGP_WORK_PENDING__SHIFT                                                     0x0
+#define SPI_WGP_WORK_PENDING__SA0_WGP8_WORK_PENDING__SHIFT                                                    0x10
+#define SPI_WGP_WORK_PENDING__RESERVED__SHIFT                                                                 0x11
+#define SPI_WGP_WORK_PENDING__SPI_WGP_WORK_PENDING_MASK                                                       0x0000FFFFL
+#define SPI_WGP_WORK_PENDING__SA0_WGP8_WORK_PENDING_MASK                                                      0x00010000L
+#define SPI_WGP_WORK_PENDING__RESERVED_MASK                                                                   0xFFFE0000L
+//SPI_CREST_MODE
+#define SPI_CREST_MODE__ENABLE_CREST__SHIFT                                                                   0x0
+#define SPI_CREST_MODE__ENABLE_CREST_MASK                                                                     0x00000001L
+//SPI_SLAVE_DEBUG_BUSY
+#define SPI_SLAVE_DEBUG_BUSY__LS_VTX_BUSY__SHIFT                                                              0x0
+#define SPI_SLAVE_DEBUG_BUSY__HS_VTX_BUSY__SHIFT                                                              0x1
+#define SPI_SLAVE_DEBUG_BUSY__ES_VTX_BUSY__SHIFT                                                              0x2
+#define SPI_SLAVE_DEBUG_BUSY__GS_VTX_BUSY__SHIFT                                                              0x3
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC00_BUSY__SHIFT                                                           0x4
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC01_BUSY__SHIFT                                                           0x5
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC00_BUSY__SHIFT                                                           0x6
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC01_BUSY__SHIFT                                                           0x7
+#define SPI_SLAVE_DEBUG_BUSY__WAVEBUFFER_BUSY__SHIFT                                                          0x8
+#define SPI_SLAVE_DEBUG_BUSY__WAVE_WR_WCTL_BUSY__SHIFT                                                        0x9
+#define SPI_SLAVE_DEBUG_BUSY__EVENT_CNTL_BUSY__SHIFT                                                          0xa
+#define SPI_SLAVE_DEBUG_BUSY__SAVE_CTX_BUSY__SHIFT                                                            0xb
+#define SPI_SLAVE_DEBUG_BUSY__WR_CTL_MUX_BUSY__SHIFT                                                          0xc
+#define SPI_SLAVE_DEBUG_BUSY__LS_VTX_BUSY_MASK                                                                0x00000001L
+#define SPI_SLAVE_DEBUG_BUSY__HS_VTX_BUSY_MASK                                                                0x00000002L
+#define SPI_SLAVE_DEBUG_BUSY__ES_VTX_BUSY_MASK                                                                0x00000004L
+#define SPI_SLAVE_DEBUG_BUSY__GS_VTX_BUSY_MASK                                                                0x00000008L
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC00_BUSY_MASK                                                             0x00000010L
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC01_BUSY_MASK                                                             0x00000020L
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC00_BUSY_MASK                                                             0x00000040L
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC01_BUSY_MASK                                                             0x00000080L
+#define SPI_SLAVE_DEBUG_BUSY__WAVEBUFFER_BUSY_MASK                                                            0x00000100L
+#define SPI_SLAVE_DEBUG_BUSY__WAVE_WR_WCTL_BUSY_MASK                                                          0x00000200L
+#define SPI_SLAVE_DEBUG_BUSY__EVENT_CNTL_BUSY_MASK                                                            0x00000400L
+#define SPI_SLAVE_DEBUG_BUSY__SAVE_CTX_BUSY_MASK                                                              0x00000800L
+#define SPI_SLAVE_DEBUG_BUSY__WR_CTL_MUX_BUSY_MASK                                                            0x00001000L
+//SPI_LB_CTR_CTRL
+#define SPI_LB_CTR_CTRL__LOAD__SHIFT                                                                          0x0
+#define SPI_LB_CTR_CTRL__WAVES_SELECT__SHIFT                                                                  0x1
+#define SPI_LB_CTR_CTRL__CLEAR_ON_READ__SHIFT                                                                 0x3
+#define SPI_LB_CTR_CTRL__RESET_COUNTS__SHIFT                                                                  0x4
+#define SPI_LB_CTR_CTRL__LOAD_MASK                                                                            0x00000001L
+#define SPI_LB_CTR_CTRL__WAVES_SELECT_MASK                                                                    0x00000006L
+#define SPI_LB_CTR_CTRL__CLEAR_ON_READ_MASK                                                                   0x00000008L
+#define SPI_LB_CTR_CTRL__RESET_COUNTS_MASK                                                                    0x00000010L
+//SPI_LB_WGP_MASK
+#define SPI_LB_WGP_MASK__WGP_MASK__SHIFT                                                                      0x0
+#define SPI_LB_WGP_MASK__WGP_MASK_MASK                                                                        0x0000FFFFL
+//SPI_LB_DATA_REG
+#define SPI_LB_DATA_REG__CNT_DATA__SHIFT                                                                      0x0
+#define SPI_LB_DATA_REG__CNT_DATA_MASK                                                                        0xFFFFFFFFL
+//SPI_PG_ENABLE_STATIC_WGP_MASK
+#define SPI_PG_ENABLE_STATIC_WGP_MASK__WGP_MASK__SHIFT                                                        0x0
+#define SPI_PG_ENABLE_STATIC_WGP_MASK__WGP_MASK_MASK                                                          0x0000FFFFL
+//SPI_GDS_CREDITS
+#define SPI_GDS_CREDITS__DS_DATA_CREDITS__SHIFT                                                               0x0
+#define SPI_GDS_CREDITS__DS_CMD_CREDITS__SHIFT                                                                0x8
+#define SPI_GDS_CREDITS__DS_DATA_CREDITS_MASK                                                                 0x000000FFL
+#define SPI_GDS_CREDITS__DS_CMD_CREDITS_MASK                                                                  0x0000FF00L
+//SPI_SX_EXPORT_BUFFER_SIZES
+#define SPI_SX_EXPORT_BUFFER_SIZES__COLOR_BUFFER_SIZE__SHIFT                                                  0x0
+#define SPI_SX_EXPORT_BUFFER_SIZES__POSITION_BUFFER_SIZE__SHIFT                                               0x10
+#define SPI_SX_EXPORT_BUFFER_SIZES__COLOR_BUFFER_SIZE_MASK                                                    0x0000FFFFL
+#define SPI_SX_EXPORT_BUFFER_SIZES__POSITION_BUFFER_SIZE_MASK                                                 0xFFFF0000L
+//SPI_SX_SCOREBOARD_BUFFER_SIZES
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__COLOR_SCOREBOARD_SIZE__SHIFT                                          0x0
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__POSITION_SCOREBOARD_SIZE__SHIFT                                       0x10
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__COLOR_SCOREBOARD_SIZE_MASK                                            0x0000FFFFL
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__POSITION_SCOREBOARD_SIZE_MASK                                         0xFFFF0000L
+//SPI_CSQ_WF_ACTIVE_STATUS
+#define SPI_CSQ_WF_ACTIVE_STATUS__ACTIVE__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_STATUS__ACTIVE_MASK                                                                 0xFFFFFFFFL
+//SPI_CSQ_WF_ACTIVE_COUNT_0
+#define SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_0__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK                                                                 0x000007FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_0__EVENTS_MASK                                                                0x07FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_1
+#define SPI_CSQ_WF_ACTIVE_COUNT_1__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_1__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_1__COUNT_MASK                                                                 0x000007FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_1__EVENTS_MASK                                                                0x07FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_2
+#define SPI_CSQ_WF_ACTIVE_COUNT_2__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_2__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_2__COUNT_MASK                                                                 0x000007FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_2__EVENTS_MASK                                                                0x07FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_3
+#define SPI_CSQ_WF_ACTIVE_COUNT_3__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_3__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_3__COUNT_MASK                                                                 0x000007FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_3__EVENTS_MASK                                                                0x07FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_4
+#define SPI_CSQ_WF_ACTIVE_COUNT_4__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_4__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_4__COUNT_MASK                                                                 0x000007FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_4__EVENTS_MASK                                                                0x07FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_5
+#define SPI_CSQ_WF_ACTIVE_COUNT_5__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_5__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_5__COUNT_MASK                                                                 0x000007FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_5__EVENTS_MASK                                                                0x07FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_6
+#define SPI_CSQ_WF_ACTIVE_COUNT_6__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_6__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_6__COUNT_MASK                                                                 0x000007FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_6__EVENTS_MASK                                                                0x07FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_7
+#define SPI_CSQ_WF_ACTIVE_COUNT_7__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_7__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_7__COUNT_MASK                                                                 0x000007FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_7__EVENTS_MASK                                                                0x07FF0000L
+//SPI_LB_DATA_WAVES
+#define SPI_LB_DATA_WAVES__COUNT0__SHIFT                                                                      0x0
+#define SPI_LB_DATA_WAVES__COUNT1__SHIFT                                                                      0x10
+#define SPI_LB_DATA_WAVES__COUNT0_MASK                                                                        0x0000FFFFL
+#define SPI_LB_DATA_WAVES__COUNT1_MASK                                                                        0xFFFF0000L
+//SPI_LB_DATA_PERWGP_WAVE_HSGS
+#define SPI_LB_DATA_PERWGP_WAVE_HSGS__WGP_USED_HS__SHIFT                                                      0x0
+#define SPI_LB_DATA_PERWGP_WAVE_HSGS__WGP_USED_GS__SHIFT                                                      0x10
+#define SPI_LB_DATA_PERWGP_WAVE_HSGS__WGP_USED_HS_MASK                                                        0x0000FFFFL
+#define SPI_LB_DATA_PERWGP_WAVE_HSGS__WGP_USED_GS_MASK                                                        0xFFFF0000L
+//SPI_LB_DATA_PERWGP_WAVE_PS
+#define SPI_LB_DATA_PERWGP_WAVE_PS__WGP_USED_PS__SHIFT                                                        0x0
+#define SPI_LB_DATA_PERWGP_WAVE_PS__WGP_USED_PS_MASK                                                          0x0000FFFFL
+//SPI_LB_DATA_PERWGP_WAVE_CS
+#define SPI_LB_DATA_PERWGP_WAVE_CS__ACTIVE__SHIFT                                                             0x0
+#define SPI_LB_DATA_PERWGP_WAVE_CS__ACTIVE_MASK                                                               0x0000FFFFL
+//SPI_WF_ACTIVE_COUNT_GFX
+#define SPI_WF_ACTIVE_COUNT_GFX__WF_ALLOCATED__SHIFT                                                          0x0
+#define SPI_WF_ACTIVE_COUNT_GFX__WF_ACTIVE__SHIFT                                                             0x8
+#define SPI_WF_ACTIVE_COUNT_GFX__WF_ALLOCATED_MASK                                                            0x000000FFL
+#define SPI_WF_ACTIVE_COUNT_GFX__WF_ACTIVE_MASK                                                               0x00FFFF00L
+//SPI_WF_ACTIVE_COUNT_HPG
+#define SPI_WF_ACTIVE_COUNT_HPG__WF_ALLOCATED__SHIFT                                                          0x0
+#define SPI_WF_ACTIVE_COUNT_HPG__WF_ACTIVE__SHIFT                                                             0x8
+#define SPI_WF_ACTIVE_COUNT_HPG__WF_ALLOCATED_MASK                                                            0x000000FFL
+#define SPI_WF_ACTIVE_COUNT_HPG__WF_ACTIVE_MASK                                                               0x00FFFF00L
+//SPIS_DEBUG_READ
+#define SPIS_DEBUG_READ__DATA__SHIFT                                                                          0x0
+#define SPIS_DEBUG_READ__DATA_MASK                                                                            0xFFFFFFFFL
+//BCI_DEBUG_READ
+#define BCI_DEBUG_READ__DATA__SHIFT                                                                           0x0
+#define BCI_DEBUG_READ__DATA_MASK                                                                             0x00FFFFFFL
+//SPI_P0_TRAP_SCREEN_PSBA_LO
+#define SPI_P0_TRAP_SCREEN_PSBA_LO__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P0_TRAP_SCREEN_PSBA_LO__MEM_BASE_MASK                                                             0xFFFFFFFFL
+//SPI_P0_TRAP_SCREEN_PSBA_HI
+#define SPI_P0_TRAP_SCREEN_PSBA_HI__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P0_TRAP_SCREEN_PSBA_HI__MEM_BASE_MASK                                                             0x000000FFL
+//SPI_P0_TRAP_SCREEN_PSMA_LO
+#define SPI_P0_TRAP_SCREEN_PSMA_LO__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P0_TRAP_SCREEN_PSMA_LO__MEM_BASE_MASK                                                             0xFFFFFFFFL
+//SPI_P0_TRAP_SCREEN_PSMA_HI
+#define SPI_P0_TRAP_SCREEN_PSMA_HI__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P0_TRAP_SCREEN_PSMA_HI__MEM_BASE_MASK                                                             0x000000FFL
+//SPI_P0_TRAP_SCREEN_GPR_MIN
+#define SPI_P0_TRAP_SCREEN_GPR_MIN__VGPR_MIN__SHIFT                                                           0x0
+#define SPI_P0_TRAP_SCREEN_GPR_MIN__SGPR_MIN__SHIFT                                                           0x6
+#define SPI_P0_TRAP_SCREEN_GPR_MIN__VGPR_MIN_MASK                                                             0x0000003FL
+#define SPI_P0_TRAP_SCREEN_GPR_MIN__SGPR_MIN_MASK                                                             0x000003C0L
+//SPI_P1_TRAP_SCREEN_PSBA_LO
+#define SPI_P1_TRAP_SCREEN_PSBA_LO__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P1_TRAP_SCREEN_PSBA_LO__MEM_BASE_MASK                                                             0xFFFFFFFFL
+//SPI_P1_TRAP_SCREEN_PSBA_HI
+#define SPI_P1_TRAP_SCREEN_PSBA_HI__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P1_TRAP_SCREEN_PSBA_HI__MEM_BASE_MASK                                                             0x000000FFL
+//SPI_P1_TRAP_SCREEN_PSMA_LO
+#define SPI_P1_TRAP_SCREEN_PSMA_LO__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P1_TRAP_SCREEN_PSMA_LO__MEM_BASE_MASK                                                             0xFFFFFFFFL
+//SPI_P1_TRAP_SCREEN_PSMA_HI
+#define SPI_P1_TRAP_SCREEN_PSMA_HI__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P1_TRAP_SCREEN_PSMA_HI__MEM_BASE_MASK                                                             0x000000FFL
+//SPI_P1_TRAP_SCREEN_GPR_MIN
+#define SPI_P1_TRAP_SCREEN_GPR_MIN__VGPR_MIN__SHIFT                                                           0x0
+#define SPI_P1_TRAP_SCREEN_GPR_MIN__SGPR_MIN__SHIFT                                                           0x6
+#define SPI_P1_TRAP_SCREEN_GPR_MIN__VGPR_MIN_MASK                                                             0x0000003FL
+#define SPI_P1_TRAP_SCREEN_GPR_MIN__SGPR_MIN_MASK                                                             0x000003C0L
+//SPI_GFX_CRAWLER_CONFIG
+#define SPI_GFX_CRAWLER_CONFIG__PS_DEPTH__SHIFT                                                               0x0
+#define SPI_GFX_CRAWLER_CONFIG__GS_DEPTH__SHIFT                                                               0x5
+#define SPI_GFX_CRAWLER_CONFIG__HS_DEPTH__SHIFT                                                               0xb
+#define SPI_GFX_CRAWLER_CONFIG__PS_ALLOC_DEPTH__SHIFT                                                         0x11
+#define SPI_GFX_CRAWLER_CONFIG__PS_LDS_DONE_DEPTH__SHIFT                                                      0x16
+#define SPI_GFX_CRAWLER_CONFIG__PS_LDS_DONE_CNTL__SHIFT                                                       0x19
+#define SPI_GFX_CRAWLER_CONFIG__RA_PSWAVE_CREDITS__SHIFT                                                      0x1a
+#define SPI_GFX_CRAWLER_CONFIG__PS_DEPTH_MASK                                                                 0x0000001FL
+#define SPI_GFX_CRAWLER_CONFIG__GS_DEPTH_MASK                                                                 0x000007E0L
+#define SPI_GFX_CRAWLER_CONFIG__HS_DEPTH_MASK                                                                 0x0001F800L
+#define SPI_GFX_CRAWLER_CONFIG__PS_ALLOC_DEPTH_MASK                                                           0x003E0000L
+#define SPI_GFX_CRAWLER_CONFIG__PS_LDS_DONE_DEPTH_MASK                                                        0x01C00000L
+#define SPI_GFX_CRAWLER_CONFIG__PS_LDS_DONE_CNTL_MASK                                                         0x02000000L
+#define SPI_GFX_CRAWLER_CONFIG__RA_PSWAVE_CREDITS_MASK                                                        0x1C000000L
+//SPI_CS_CRAWLER_CONFIG
+#define SPI_CS_CRAWLER_CONFIG__CSG_DEPTH__SHIFT                                                               0x0
+#define SPI_CS_CRAWLER_CONFIG__CSC_DEPTH__SHIFT                                                               0x6
+#define SPI_CS_CRAWLER_CONFIG__CSG_DEPTH_MASK                                                                 0x0000003FL
+#define SPI_CS_CRAWLER_CONFIG__CSC_DEPTH_MASK                                                                 0x00000FC0L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_tpdec
+//TD_CNTL
+#define TD_CNTL__DISABLE_MEDIAN_CALC_FOR_CUBECORNER_PHANTOM_TEXELS__SHIFT                                     0x0
+#define TD_CNTL__PRESERVE_VGPR_ON_UTC_ERROR__SHIFT                                                            0xd
+#define TD_CNTL__GATHER4_FLOAT_MODE__SHIFT                                                                    0x10
+#define TD_CNTL__FORCE_RT_BVH4_ARBITER_TO_PING_PONG__SHIFT                                                    0x11
+#define TD_CNTL__GATHER4_DX9_MODE__SHIFT                                                                      0x13
+#define TD_CNTL__DISABLE_POWER_THROTTLE__SHIFT                                                                0x14
+#define TD_CNTL__ENABLE_ROUND_TO_ZERO__SHIFT                                                                  0x15
+#define TD_CNTL__DISABLE_ROUND_TO_ZERO_FOR_LARGE_FLOAT_TO_SMALL_FLOAT__SHIFT                                  0x16
+#define TD_CNTL__DISABLE_2BIT_SIGNED_FORMAT__SHIFT                                                            0x17
+#define TD_CNTL__ARBITER_ROUND_ROBIN__SHIFT                                                                   0x18
+#define TD_CNTL__ARBITER_OLDEST_PRIORITY__SHIFT                                                               0x19
+#define TD_CNTL__DONE_SCOREBOARD_DEPTH__SHIFT                                                                 0x1a
+#define TD_CNTL__DISABLE_MEDIAN_CALC_FOR_CUBECORNER_PHANTOM_TEXELS_MASK                                       0x00000001L
+#define TD_CNTL__PRESERVE_VGPR_ON_UTC_ERROR_MASK                                                              0x00002000L
+#define TD_CNTL__GATHER4_FLOAT_MODE_MASK                                                                      0x00010000L
+#define TD_CNTL__FORCE_RT_BVH4_ARBITER_TO_PING_PONG_MASK                                                      0x00020000L
+#define TD_CNTL__GATHER4_DX9_MODE_MASK                                                                        0x00080000L
+#define TD_CNTL__DISABLE_POWER_THROTTLE_MASK                                                                  0x00100000L
+#define TD_CNTL__ENABLE_ROUND_TO_ZERO_MASK                                                                    0x00200000L
+#define TD_CNTL__DISABLE_ROUND_TO_ZERO_FOR_LARGE_FLOAT_TO_SMALL_FLOAT_MASK                                    0x00400000L
+#define TD_CNTL__DISABLE_2BIT_SIGNED_FORMAT_MASK                                                              0x00800000L
+#define TD_CNTL__ARBITER_ROUND_ROBIN_MASK                                                                     0x01000000L
+#define TD_CNTL__ARBITER_OLDEST_PRIORITY_MASK                                                                 0x02000000L
+#define TD_CNTL__DONE_SCOREBOARD_DEPTH_MASK                                                                   0xFC000000L
+//TD_STATUS
+#define TD_STATUS__BUSY__SHIFT                                                                                0x1f
+#define TD_STATUS__BUSY_MASK                                                                                  0x80000000L
+//TD_DEBUG_DATA
+#define TD_DEBUG_DATA__DATA__SHIFT                                                                            0x0
+#define TD_DEBUG_DATA__DATA_MASK                                                                              0xFFFFFFFFL
+//TD_POWER_CNTL
+#define TD_POWER_CNTL__DISABLE_NOFILTER_FORMATTER_POWER_OPT__SHIFT                                            0x6
+#define TD_POWER_CNTL__FORCE_NOFILTER_D16_FORMATTERS_ON__SHIFT                                                0x7
+#define TD_POWER_CNTL__ENABLE_DEBUG_REG__SHIFT                                                                0x8
+#define TD_POWER_CNTL__DISABLE_NOFILTER_FORMATTER_POWER_OPT_MASK                                              0x00000040L
+#define TD_POWER_CNTL__FORCE_NOFILTER_D16_FORMATTERS_ON_MASK                                                  0x00000080L
+#define TD_POWER_CNTL__ENABLE_DEBUG_REG_MASK                                                                  0x00000100L
+//TD_CNTL2
+#define TD_CNTL2__LDS_RETURN_FIFO_CREDIT__SHIFT                                                               0x0
+#define TD_CNTL2__MULTI_CYCLE_16FP__SHIFT                                                                     0x3
+#define TD_CNTL2__DISABLE_BLEND_PRT_FOR_LOADS__SHIFT                                                          0x4
+#define TD_CNTL2__LDS_RETURN_FIFO_CREDIT_MASK                                                                 0x00000007L
+#define TD_CNTL2__MULTI_CYCLE_16FP_MASK                                                                       0x00000008L
+#define TD_CNTL2__DISABLE_BLEND_PRT_FOR_LOADS_MASK                                                            0x00000010L
+//TD_DSM_CNTL
+//TD_DSM_CNTL2
+//TD_SCRATCH
+#define TD_SCRATCH__SCRATCH__SHIFT                                                                            0x0
+#define TD_SCRATCH__SCRATCH_MASK                                                                              0xFFFFFFFFL
+//TA_CNTL
+#define TA_CNTL__TA_SQ_XNACK_FGCG_DISABLE__SHIFT                                                              0x0
+#define TA_CNTL__TA_INPUT_RDATA_PER_BANK_FGCG_OVERRIDE__SHIFT                                                 0x2
+#define TA_CNTL__TA_INPUT_CFIFO_VEC64_OPT_OVERRIDE__SHIFT                                                     0x3
+#define TA_CNTL__ALIGNER_CREDIT__SHIFT                                                                        0x8
+#define TA_CNTL__TD_FIFO_CREDIT__SHIFT                                                                        0x16
+#define TA_CNTL__TA_SQ_XNACK_FGCG_DISABLE_MASK                                                                0x00000001L
+#define TA_CNTL__TA_INPUT_RDATA_PER_BANK_FGCG_OVERRIDE_MASK                                                   0x00000004L
+#define TA_CNTL__TA_INPUT_CFIFO_VEC64_OPT_OVERRIDE_MASK                                                       0x00000008L
+#define TA_CNTL__ALIGNER_CREDIT_MASK                                                                          0x00001F00L
+#define TA_CNTL__TD_FIFO_CREDIT_MASK                                                                          0xFFC00000L
+//TA_CNTL_AUX
+#define TA_CNTL_AUX__SCOAL_DSWIZZLE_N__SHIFT                                                                  0x0
+#define TA_CNTL_AUX__DEPTH_AS_PITCH_DIS__SHIFT                                                                0x1
+#define TA_CNTL_AUX__CORNER_SAMPLES_MIN_DIM__SHIFT                                                            0x2
+#define TA_CNTL_AUX__OVERRIDE_QUAD_MODE_DIS__SHIFT                                                            0x3
+#define TA_CNTL_AUX__DERIV_ADJUST_DIS__SHIFT                                                                  0x4
+#define TA_CNTL_AUX__TFAULT_EN_OVERRIDE__SHIFT                                                                0x5
+#define TA_CNTL_AUX__GATHERH_DST_SEL__SHIFT                                                                   0x6
+#define TA_CNTL_AUX__DISABLE_GATHER4_BC_SWIZZLE__SHIFT                                                        0x7
+#define TA_CNTL_AUX__ANISO_MAG_STEP_CLAMP__SHIFT                                                              0x8
+#define TA_CNTL_AUX__AUTO_ALIGN_FORMAT__SHIFT                                                                 0x9
+#define TA_CNTL_AUX__ANISO_HALF_THRESH__SHIFT                                                                 0xa
+#define TA_CNTL_AUX__ANISO_ERROR_FP_VBIAS__SHIFT                                                              0xc
+#define TA_CNTL_AUX__ANISO_STEP_ORDER__SHIFT                                                                  0xd
+#define TA_CNTL_AUX__ANISO_STEP__SHIFT                                                                        0xe
+#define TA_CNTL_AUX__MINMAG_UNNORM__SHIFT                                                                     0xf
+#define TA_CNTL_AUX__ANISO_WEIGHT_MODE__SHIFT                                                                 0x10
+#define TA_CNTL_AUX__ANISO_RATIO_LUT__SHIFT                                                                   0x11
+#define TA_CNTL_AUX__ANISO_TAP__SHIFT                                                                         0x12
+#define TA_CNTL_AUX__DETERMINISM_RESERVED_DISABLE__SHIFT                                                      0x14
+#define TA_CNTL_AUX__DETERMINISM_OPCODE_STRICT_DISABLE__SHIFT                                                 0x15
+#define TA_CNTL_AUX__DETERMINISM_MISC_DISABLE__SHIFT                                                          0x16
+#define TA_CNTL_AUX__DETERMINISM_SAMPLE_C_DFMT_DISABLE__SHIFT                                                 0x17
+#define TA_CNTL_AUX__DETERMINISM_SAMPLER_MSAA_DISABLE__SHIFT                                                  0x18
+#define TA_CNTL_AUX__DETERMINISM_WRITEOP_READFMT_DISABLE__SHIFT                                               0x19
+#define TA_CNTL_AUX__DETERMINISM_DFMT_NFMT_DISABLE__SHIFT                                                     0x1a
+#define TA_CNTL_AUX__CUBEMAP_SLICE_CLAMP__SHIFT                                                               0x1c
+#define TA_CNTL_AUX__TRUNC_SMALL_NEG__SHIFT                                                                   0x1d
+#define TA_CNTL_AUX__ARRAY_ROUND_MODE__SHIFT                                                                  0x1e
+#define TA_CNTL_AUX__SCOAL_DSWIZZLE_N_MASK                                                                    0x00000001L
+#define TA_CNTL_AUX__DEPTH_AS_PITCH_DIS_MASK                                                                  0x00000002L
+#define TA_CNTL_AUX__CORNER_SAMPLES_MIN_DIM_MASK                                                              0x00000004L
+#define TA_CNTL_AUX__OVERRIDE_QUAD_MODE_DIS_MASK                                                              0x00000008L
+#define TA_CNTL_AUX__DERIV_ADJUST_DIS_MASK                                                                    0x00000010L
+#define TA_CNTL_AUX__TFAULT_EN_OVERRIDE_MASK                                                                  0x00000020L
+#define TA_CNTL_AUX__GATHERH_DST_SEL_MASK                                                                     0x00000040L
+#define TA_CNTL_AUX__DISABLE_GATHER4_BC_SWIZZLE_MASK                                                          0x00000080L
+#define TA_CNTL_AUX__ANISO_MAG_STEP_CLAMP_MASK                                                                0x00000100L
+#define TA_CNTL_AUX__AUTO_ALIGN_FORMAT_MASK                                                                   0x00000200L
+#define TA_CNTL_AUX__ANISO_HALF_THRESH_MASK                                                                   0x00000C00L
+#define TA_CNTL_AUX__ANISO_ERROR_FP_VBIAS_MASK                                                                0x00001000L
+#define TA_CNTL_AUX__ANISO_STEP_ORDER_MASK                                                                    0x00002000L
+#define TA_CNTL_AUX__ANISO_STEP_MASK                                                                          0x00004000L
+#define TA_CNTL_AUX__MINMAG_UNNORM_MASK                                                                       0x00008000L
+#define TA_CNTL_AUX__ANISO_WEIGHT_MODE_MASK                                                                   0x00010000L
+#define TA_CNTL_AUX__ANISO_RATIO_LUT_MASK                                                                     0x00020000L
+#define TA_CNTL_AUX__ANISO_TAP_MASK                                                                           0x00040000L
+#define TA_CNTL_AUX__DETERMINISM_RESERVED_DISABLE_MASK                                                        0x00100000L
+#define TA_CNTL_AUX__DETERMINISM_OPCODE_STRICT_DISABLE_MASK                                                   0x00200000L
+#define TA_CNTL_AUX__DETERMINISM_MISC_DISABLE_MASK                                                            0x00400000L
+#define TA_CNTL_AUX__DETERMINISM_SAMPLE_C_DFMT_DISABLE_MASK                                                   0x00800000L
+#define TA_CNTL_AUX__DETERMINISM_SAMPLER_MSAA_DISABLE_MASK                                                    0x01000000L
+#define TA_CNTL_AUX__DETERMINISM_WRITEOP_READFMT_DISABLE_MASK                                                 0x02000000L
+#define TA_CNTL_AUX__DETERMINISM_DFMT_NFMT_DISABLE_MASK                                                       0x04000000L
+#define TA_CNTL_AUX__CUBEMAP_SLICE_CLAMP_MASK                                                                 0x10000000L
+#define TA_CNTL_AUX__TRUNC_SMALL_NEG_MASK                                                                     0x20000000L
+#define TA_CNTL_AUX__ARRAY_ROUND_MODE_MASK                                                                    0xC0000000L
+//TA_CNTL2
+#define TA_CNTL2__STORE_COMPONENT_MODE__SHIFT                                                                 0x0
+#define TA_CNTL2__MAX_RQ_ID__SHIFT                                                                            0x4
+#define TA_CNTL2__ELEMSIZE_HASH_DIS__SHIFT                                                                    0x11
+#define TA_CNTL2__TRUNCATE_COORD_MODE__SHIFT                                                                  0x12
+#define TA_CNTL2__ELIMINATE_UNLIT_QUAD_DIS__SHIFT                                                             0x13
+#define TA_CNTL2__PRTPLUS_ACCUM_MODE__SHIFT                                                                   0x14
+#define TA_CNTL2__STORE_COMPONENT_MODE_MASK                                                                   0x00000001L
+#define TA_CNTL2__MAX_RQ_ID_MASK                                                                              0x00000070L
+#define TA_CNTL2__ELEMSIZE_HASH_DIS_MASK                                                                      0x00020000L
+#define TA_CNTL2__TRUNCATE_COORD_MODE_MASK                                                                    0x00040000L
+#define TA_CNTL2__ELIMINATE_UNLIT_QUAD_DIS_MASK                                                               0x00080000L
+#define TA_CNTL2__PRTPLUS_ACCUM_MODE_MASK                                                                     0x00300000L
+//TA_STATUS
+#define TA_STATUS__FG_PFIFO_EMPTYB__SHIFT                                                                     0xc
+#define TA_STATUS__FG_LFIFO_EMPTYB__SHIFT                                                                     0xd
+#define TA_STATUS__FG_SFIFO_EMPTYB__SHIFT                                                                     0xe
+#define TA_STATUS__FL_PFIFO_EMPTYB__SHIFT                                                                     0x10
+#define TA_STATUS__FL_LFIFO_EMPTYB__SHIFT                                                                     0x11
+#define TA_STATUS__FL_SFIFO_EMPTYB__SHIFT                                                                     0x12
+#define TA_STATUS__FA_PFIFO_EMPTYB__SHIFT                                                                     0x14
+#define TA_STATUS__FA_LFIFO_EMPTYB__SHIFT                                                                     0x15
+#define TA_STATUS__FA_SFIFO_EMPTYB__SHIFT                                                                     0x16
+#define TA_STATUS__IN_BUSY__SHIFT                                                                             0x18
+#define TA_STATUS__FG_BUSY__SHIFT                                                                             0x19
+#define TA_STATUS__LA_BUSY__SHIFT                                                                             0x1a
+#define TA_STATUS__FL_BUSY__SHIFT                                                                             0x1b
+#define TA_STATUS__TA_BUSY__SHIFT                                                                             0x1c
+#define TA_STATUS__FA_BUSY__SHIFT                                                                             0x1d
+#define TA_STATUS__AL_BUSY__SHIFT                                                                             0x1e
+#define TA_STATUS__BUSY__SHIFT                                                                                0x1f
+#define TA_STATUS__FG_PFIFO_EMPTYB_MASK                                                                       0x00001000L
+#define TA_STATUS__FG_LFIFO_EMPTYB_MASK                                                                       0x00002000L
+#define TA_STATUS__FG_SFIFO_EMPTYB_MASK                                                                       0x00004000L
+#define TA_STATUS__FL_PFIFO_EMPTYB_MASK                                                                       0x00010000L
+#define TA_STATUS__FL_LFIFO_EMPTYB_MASK                                                                       0x00020000L
+#define TA_STATUS__FL_SFIFO_EMPTYB_MASK                                                                       0x00040000L
+#define TA_STATUS__FA_PFIFO_EMPTYB_MASK                                                                       0x00100000L
+#define TA_STATUS__FA_LFIFO_EMPTYB_MASK                                                                       0x00200000L
+#define TA_STATUS__FA_SFIFO_EMPTYB_MASK                                                                       0x00400000L
+#define TA_STATUS__IN_BUSY_MASK                                                                               0x01000000L
+#define TA_STATUS__FG_BUSY_MASK                                                                               0x02000000L
+#define TA_STATUS__LA_BUSY_MASK                                                                               0x04000000L
+#define TA_STATUS__FL_BUSY_MASK                                                                               0x08000000L
+#define TA_STATUS__TA_BUSY_MASK                                                                               0x10000000L
+#define TA_STATUS__FA_BUSY_MASK                                                                               0x20000000L
+#define TA_STATUS__AL_BUSY_MASK                                                                               0x40000000L
+#define TA_STATUS__BUSY_MASK                                                                                  0x80000000L
+//TA_SCRATCH
+#define TA_SCRATCH__SCRATCH__SHIFT                                                                            0x0
+#define TA_SCRATCH__SCRATCH_MASK                                                                              0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_rbdec
+//DB_DEBUG
+#define DB_DEBUG__DEBUG_STENCIL_COMPRESS_DISABLE__SHIFT                                                       0x0
+#define DB_DEBUG__DEBUG_DEPTH_COMPRESS_DISABLE__SHIFT                                                         0x1
+#define DB_DEBUG__ENABLE_COMPRESSION_ON_BYPASS__SHIFT                                                         0x2
+#define DB_DEBUG__DISABLE_TILE_RATE_1XAA__SHIFT                                                               0x3
+#define DB_DEBUG__FORCE_Z_MODE__SHIFT                                                                         0x4
+#define DB_DEBUG__DEBUG_FORCE_DEPTH_READ__SHIFT                                                               0x6
+#define DB_DEBUG__DEBUG_FORCE_STENCIL_READ__SHIFT                                                             0x7
+#define DB_DEBUG__DEBUG_FORCE_HIZ_ENABLE__SHIFT                                                               0x8
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE0__SHIFT                                                              0xa
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE1__SHIFT                                                              0xc
+#define DB_DEBUG__DEBUG_FAST_Z_DISABLE__SHIFT                                                                 0xe
+#define DB_DEBUG__DEBUG_FAST_STENCIL_DISABLE__SHIFT                                                           0xf
+#define DB_DEBUG__DEBUG_NOOP_CULL_DISABLE__SHIFT                                                              0x10
+#define DB_DEBUG__DEPTH_CACHE_FORCE_MISS__SHIFT                                                               0x12
+#define DB_DEBUG__DEBUG_FORCE_FULL_Z_RANGE__SHIFT                                                             0x13
+#define DB_DEBUG__ZPASS_COUNTS_LOOK_AT_PIPE_STAT_EVENTS__SHIFT                                                0x16
+#define DB_DEBUG__DISABLE_VPORT_ZPLANE_OPTIMIZATION__SHIFT                                                    0x17
+#define DB_DEBUG__DECOMPRESS_AFTER_N_ZPLANES__SHIFT                                                           0x18
+#define DB_DEBUG__ONE_FREE_IN_FLIGHT__SHIFT                                                                   0x1c
+#define DB_DEBUG__FORCE_MISS_IF_NOT_INFLIGHT__SHIFT                                                           0x1d
+#define DB_DEBUG__RESERVED_FIELD_1__SHIFT                                                                     0x1e
+#define DB_DEBUG__DEBUG_STENCIL_COMPRESS_DISABLE_MASK                                                         0x00000001L
+#define DB_DEBUG__DEBUG_DEPTH_COMPRESS_DISABLE_MASK                                                           0x00000002L
+#define DB_DEBUG__ENABLE_COMPRESSION_ON_BYPASS_MASK                                                           0x00000004L
+#define DB_DEBUG__DISABLE_TILE_RATE_1XAA_MASK                                                                 0x00000008L
+#define DB_DEBUG__FORCE_Z_MODE_MASK                                                                           0x00000030L
+#define DB_DEBUG__DEBUG_FORCE_DEPTH_READ_MASK                                                                 0x00000040L
+#define DB_DEBUG__DEBUG_FORCE_STENCIL_READ_MASK                                                               0x00000080L
+#define DB_DEBUG__DEBUG_FORCE_HIZ_ENABLE_MASK                                                                 0x00000300L
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE0_MASK                                                                0x00000C00L
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE1_MASK                                                                0x00003000L
+#define DB_DEBUG__DEBUG_FAST_Z_DISABLE_MASK                                                                   0x00004000L
+#define DB_DEBUG__DEBUG_FAST_STENCIL_DISABLE_MASK                                                             0x00008000L
+#define DB_DEBUG__DEBUG_NOOP_CULL_DISABLE_MASK                                                                0x00010000L
+#define DB_DEBUG__DEPTH_CACHE_FORCE_MISS_MASK                                                                 0x00040000L
+#define DB_DEBUG__DEBUG_FORCE_FULL_Z_RANGE_MASK                                                               0x00180000L
+#define DB_DEBUG__ZPASS_COUNTS_LOOK_AT_PIPE_STAT_EVENTS_MASK                                                  0x00400000L
+#define DB_DEBUG__DISABLE_VPORT_ZPLANE_OPTIMIZATION_MASK                                                      0x00800000L
+#define DB_DEBUG__DECOMPRESS_AFTER_N_ZPLANES_MASK                                                             0x0F000000L
+#define DB_DEBUG__ONE_FREE_IN_FLIGHT_MASK                                                                     0x10000000L
+#define DB_DEBUG__FORCE_MISS_IF_NOT_INFLIGHT_MASK                                                             0x20000000L
+#define DB_DEBUG__RESERVED_FIELD_1_MASK                                                                       0xC0000000L
+//DB_DEBUG2
+#define DB_DEBUG2__TRAP_ENABLE__SHIFT                                                                         0x1
+#define DB_DEBUG2__DTR_ROUND_ROBIN_ARB__SHIFT                                                                 0x3
+#define DB_DEBUG2__DTR_PREZ_STALLS_FOR_ETF_ROOM__SHIFT                                                        0x4
+#define DB_DEBUG2__DISABLE_PREZL_FIFO_STALL__SHIFT                                                            0x5
+#define DB_DEBUG2__DISABLE_PREZL_FIFO_STALL_REZ__SHIFT                                                        0x6
+#define DB_DEBUG2__ENABLE_VIEWPORT_STALL_ON_ALL__SHIFT                                                        0x7
+#define DB_DEBUG2__OPTIMIZE_HIZ_MATCHES_FB_DISABLE__SHIFT                                                     0x8
+#define DB_DEBUG2__CLK_OFF_DELAY__SHIFT                                                                       0x9
+#define DB_DEBUG2__FORCE_PERF_COUNTERS_ON__SHIFT                                                              0xe
+#define DB_DEBUG2__FL_FLUSH_ONE_STILE_AT_A_TIME__SHIFT                                                        0x10
+#define DB_DEBUG2__DISABLE_NULL_EOT_FORWARDING__SHIFT                                                         0x11
+#define DB_DEBUG2__DISABLE_DTT_DATA_FORWARDING__SHIFT                                                         0x12
+#define DB_DEBUG2__DISABLE_QUAD_COHERENCY_STALL__SHIFT                                                        0x13
+#define DB_DEBUG2__DISABLE_FULL_TILE_WAVE_BREAK__SHIFT                                                        0x14
+#define DB_DEBUG2__ENABLE_FULL_TILE_WAVE_BREAK_FOR_ALL_TILES__SHIFT                                           0x15
+#define DB_DEBUG2__FL_DISABLE_PLANE_REPACK__SHIFT                                                             0x1a
+#define DB_DEBUG2__DEBUG_BUS_FLOP_EN__SHIFT                                                                   0x1b
+#define DB_DEBUG2__ENABLE_PREZ_OF_REZ_SUMM__SHIFT                                                             0x1c
+#define DB_DEBUG2__DISABLE_PREZL_VIEWPORT_STALL__SHIFT                                                        0x1d
+#define DB_DEBUG2__DISABLE_SINGLE_STENCIL_QUAD_SUMM__SHIFT                                                    0x1e
+#define DB_DEBUG2__DISABLE_WRITE_STALL_ON_RDWR_CONFLICT__SHIFT                                                0x1f
+#define DB_DEBUG2__TRAP_ENABLE_MASK                                                                           0x00000002L
+#define DB_DEBUG2__DTR_ROUND_ROBIN_ARB_MASK                                                                   0x00000008L
+#define DB_DEBUG2__DTR_PREZ_STALLS_FOR_ETF_ROOM_MASK                                                          0x00000010L
+#define DB_DEBUG2__DISABLE_PREZL_FIFO_STALL_MASK                                                              0x00000020L
+#define DB_DEBUG2__DISABLE_PREZL_FIFO_STALL_REZ_MASK                                                          0x00000040L
+#define DB_DEBUG2__ENABLE_VIEWPORT_STALL_ON_ALL_MASK                                                          0x00000080L
+#define DB_DEBUG2__OPTIMIZE_HIZ_MATCHES_FB_DISABLE_MASK                                                       0x00000100L
+#define DB_DEBUG2__CLK_OFF_DELAY_MASK                                                                         0x00003E00L
+#define DB_DEBUG2__FORCE_PERF_COUNTERS_ON_MASK                                                                0x00004000L
+#define DB_DEBUG2__FL_FLUSH_ONE_STILE_AT_A_TIME_MASK                                                          0x00010000L
+#define DB_DEBUG2__DISABLE_NULL_EOT_FORWARDING_MASK                                                           0x00020000L
+#define DB_DEBUG2__DISABLE_DTT_DATA_FORWARDING_MASK                                                           0x00040000L
+#define DB_DEBUG2__DISABLE_QUAD_COHERENCY_STALL_MASK                                                          0x00080000L
+#define DB_DEBUG2__DISABLE_FULL_TILE_WAVE_BREAK_MASK                                                          0x00100000L
+#define DB_DEBUG2__ENABLE_FULL_TILE_WAVE_BREAK_FOR_ALL_TILES_MASK                                             0x00200000L
+#define DB_DEBUG2__FL_DISABLE_PLANE_REPACK_MASK                                                               0x04000000L
+#define DB_DEBUG2__DEBUG_BUS_FLOP_EN_MASK                                                                     0x08000000L
+#define DB_DEBUG2__ENABLE_PREZ_OF_REZ_SUMM_MASK                                                               0x10000000L
+#define DB_DEBUG2__DISABLE_PREZL_VIEWPORT_STALL_MASK                                                          0x20000000L
+#define DB_DEBUG2__DISABLE_SINGLE_STENCIL_QUAD_SUMM_MASK                                                      0x40000000L
+#define DB_DEBUG2__DISABLE_WRITE_STALL_ON_RDWR_CONFLICT_MASK                                                  0x80000000L
+//DB_DEBUG3
+#define DB_DEBUG3__DISABLE_RELOAD_CONTEXT_DRAW_DATA__SHIFT                                                    0x1
+#define DB_DEBUG3__FORCE_DB_IS_GOOD__SHIFT                                                                    0x2
+#define DB_DEBUG3__DISABLE_TL_SSO_NULL_SUPPRESSION__SHIFT                                                     0x3
+#define DB_DEBUG3__DISABLE_HIZ_ON_VPORT_CLAMP__SHIFT                                                          0x4
+#define DB_DEBUG3__DISABLE_ZCMP_DIRTY_SUPPRESSION__SHIFT                                                      0x8
+#define DB_DEBUG3__DISABLE_RECOMP_TO_1ZPLANE_WITHOUT_FASTOP__SHIFT                                            0xa
+#define DB_DEBUG3__DISABLE_OP_DF_BYPASS__SHIFT                                                                0xd
+#define DB_DEBUG3__DISABLE_OP_DF_WRITE_COMBINE__SHIFT                                                         0xe
+#define DB_DEBUG3__DISABLE_OP_DF_DIRECT_FEEDBACK__SHIFT                                                       0xf
+#define DB_DEBUG3__DISABLE_SLOCS_PER_CTXT_MATCH__SHIFT                                                        0x10
+#define DB_DEBUG3__SLOW_PREZ_TO_A2M_OMASK_RATE__SHIFT                                                         0x11
+#define DB_DEBUG3__ENABLE_RECOMP_ZDIRTY_SUPPRESSION_OPT__SHIFT                                                0x15
+#define DB_DEBUG3__DISABLE_RAM_READ_SUPPRESION_ON_FWD__SHIFT                                                  0x17
+#define DB_DEBUG3__ENABLE_DB_PROCESS_RESET__SHIFT                                                             0x1a
+#define DB_DEBUG3__DISABLE_OVERRASTERIZATION_FIX__SHIFT                                                       0x1b
+#define DB_DEBUG3__DISABLE_MULTIDTAG_FL_PANIC_REQUIREMENT__SHIFT                                              0x1f
+#define DB_DEBUG3__DISABLE_RELOAD_CONTEXT_DRAW_DATA_MASK                                                      0x00000002L
+#define DB_DEBUG3__FORCE_DB_IS_GOOD_MASK                                                                      0x00000004L
+#define DB_DEBUG3__DISABLE_TL_SSO_NULL_SUPPRESSION_MASK                                                       0x00000008L
+#define DB_DEBUG3__DISABLE_HIZ_ON_VPORT_CLAMP_MASK                                                            0x00000010L
+#define DB_DEBUG3__DISABLE_ZCMP_DIRTY_SUPPRESSION_MASK                                                        0x00000100L
+#define DB_DEBUG3__DISABLE_RECOMP_TO_1ZPLANE_WITHOUT_FASTOP_MASK                                              0x00000400L
+#define DB_DEBUG3__DISABLE_OP_DF_BYPASS_MASK                                                                  0x00002000L
+#define DB_DEBUG3__DISABLE_OP_DF_WRITE_COMBINE_MASK                                                           0x00004000L
+#define DB_DEBUG3__DISABLE_OP_DF_DIRECT_FEEDBACK_MASK                                                         0x00008000L
+#define DB_DEBUG3__DISABLE_SLOCS_PER_CTXT_MATCH_MASK                                                          0x00010000L
+#define DB_DEBUG3__SLOW_PREZ_TO_A2M_OMASK_RATE_MASK                                                           0x00020000L
+#define DB_DEBUG3__ENABLE_RECOMP_ZDIRTY_SUPPRESSION_OPT_MASK                                                  0x00200000L
+#define DB_DEBUG3__DISABLE_RAM_READ_SUPPRESION_ON_FWD_MASK                                                    0x00800000L
+#define DB_DEBUG3__ENABLE_DB_PROCESS_RESET_MASK                                                               0x04000000L
+#define DB_DEBUG3__DISABLE_OVERRASTERIZATION_FIX_MASK                                                         0x08000000L
+#define DB_DEBUG3__DISABLE_MULTIDTAG_FL_PANIC_REQUIREMENT_MASK                                                0x80000000L
+//DB_DEBUG4
+#define DB_DEBUG4__DISABLE_QC_Z_MASK_SUMMATION__SHIFT                                                         0x0
+#define DB_DEBUG4__DISABLE_QC_STENCIL_MASK_SUMMATION__SHIFT                                                   0x1
+#define DB_DEBUG4__DISABLE_RESUMM_TO_SINGLE_STENCIL__SHIFT                                                    0x2
+#define DB_DEBUG4__DISABLE_PREZ_POSTZ_DTILE_CONFLICT_STALL__SHIFT                                             0x3
+#define DB_DEBUG4__DISABLE_SEPARATE_OP_PIPE_CLK__SHIFT                                                        0x4
+#define DB_DEBUG4__DISABLE_SEPARATE_SX_CLK__SHIFT                                                             0x5
+#define DB_DEBUG4__DISABLE_1PLANE_PMASK_OPTIMIZATION__SHIFT                                                   0x7
+#define DB_DEBUG4__DISABLE_SEPARATE_DBG_CLK__SHIFT                                                            0x8
+#define DB_DEBUG4__ENABLE_A2M_DQUAD_OPTIMIZATION__SHIFT                                                       0xc
+#define DB_DEBUG4__DISABLE_DTT_FAST_HTILENACK_LOOKUP__SHIFT                                                   0xd
+#define DB_DEBUG4__DISABLE_DYNAMIC_RAM_LIGHT_SLEEP_MODE__SHIFT                                                0xf
+#define DB_DEBUG4__DISABLE_HIZ_TS_COLLISION_DETECT__SHIFT                                                     0x10
+#define DB_DEBUG4__LATE_ACK_SCOREBOARD_MULTIPLE_SLOT__SHIFT                                                   0x1e
+#define DB_DEBUG4__DISABLE_QC_Z_MASK_SUMMATION_MASK                                                           0x00000001L
+#define DB_DEBUG4__DISABLE_QC_STENCIL_MASK_SUMMATION_MASK                                                     0x00000002L
+#define DB_DEBUG4__DISABLE_RESUMM_TO_SINGLE_STENCIL_MASK                                                      0x00000004L
+#define DB_DEBUG4__DISABLE_PREZ_POSTZ_DTILE_CONFLICT_STALL_MASK                                               0x00000008L
+#define DB_DEBUG4__DISABLE_SEPARATE_OP_PIPE_CLK_MASK                                                          0x00000010L
+#define DB_DEBUG4__DISABLE_SEPARATE_SX_CLK_MASK                                                               0x00000020L
+#define DB_DEBUG4__DISABLE_1PLANE_PMASK_OPTIMIZATION_MASK                                                     0x00000080L
+#define DB_DEBUG4__DISABLE_SEPARATE_DBG_CLK_MASK                                                              0x00000100L
+#define DB_DEBUG4__ENABLE_A2M_DQUAD_OPTIMIZATION_MASK                                                         0x00001000L
+#define DB_DEBUG4__DISABLE_DTT_FAST_HTILENACK_LOOKUP_MASK                                                     0x00002000L
+#define DB_DEBUG4__DISABLE_DYNAMIC_RAM_LIGHT_SLEEP_MODE_MASK                                                  0x00008000L
+#define DB_DEBUG4__DISABLE_HIZ_TS_COLLISION_DETECT_MASK                                                       0x00010000L
+#define DB_DEBUG4__LATE_ACK_SCOREBOARD_MULTIPLE_SLOT_MASK                                                     0x40000000L
+//DB_CREDIT_LIMIT
+#define DB_CREDIT_LIMIT__DB_SC_UPDATE_CREDITS__SHIFT                                                          0x0
+#define DB_CREDIT_LIMIT__DB_SC_QUAD_CREDITS__SHIFT                                                            0x5
+#define DB_CREDIT_LIMIT__DB_CB_EXPORT_CREDITS__SHIFT                                                          0xa
+#define DB_CREDIT_LIMIT__DB_SC_WAVE_CREDITS__SHIFT                                                            0xd
+#define DB_CREDIT_LIMIT__DB_SC_FREE_WAVE_CREDITS__SHIFT                                                       0x12
+#define DB_CREDIT_LIMIT__DB_SC_UPDATE_CREDITS_MASK                                                            0x0000001FL
+#define DB_CREDIT_LIMIT__DB_SC_QUAD_CREDITS_MASK                                                              0x000003E0L
+#define DB_CREDIT_LIMIT__DB_CB_EXPORT_CREDITS_MASK                                                            0x00001C00L
+#define DB_CREDIT_LIMIT__DB_SC_WAVE_CREDITS_MASK                                                              0x0003E000L
+#define DB_CREDIT_LIMIT__DB_SC_FREE_WAVE_CREDITS_MASK                                                         0x007C0000L
+//DB_WATERMARKS
+#define DB_WATERMARKS__DEPTH_FREE__SHIFT                                                                      0x0
+#define DB_WATERMARKS__DEPTH_FLUSH__SHIFT                                                                     0x8
+#define DB_WATERMARKS__DEPTH_PENDING_FREE__SHIFT                                                              0x10
+#define DB_WATERMARKS__DEPTH_CACHELINE_FREE__SHIFT                                                            0x18
+#define DB_WATERMARKS__DEPTH_FREE_MASK                                                                        0x000000FFL
+#define DB_WATERMARKS__DEPTH_FLUSH_MASK                                                                       0x0000FF00L
+#define DB_WATERMARKS__DEPTH_PENDING_FREE_MASK                                                                0x00FF0000L
+#define DB_WATERMARKS__DEPTH_CACHELINE_FREE_MASK                                                              0xFF000000L
+//DB_FREE_CACHELINES
+#define DB_FREE_CACHELINES__FREE_DTILE_DEPTH__SHIFT                                                           0x0
+#define DB_FREE_CACHELINES__FREE_PLANE_DEPTH__SHIFT                                                           0x8
+#define DB_FREE_CACHELINES__FREE_Z_DEPTH__SHIFT                                                               0x10
+#define DB_FREE_CACHELINES__FREE_DTILE_DEPTH_MASK                                                             0x000000FFL
+#define DB_FREE_CACHELINES__FREE_PLANE_DEPTH_MASK                                                             0x0000FF00L
+#define DB_FREE_CACHELINES__FREE_Z_DEPTH_MASK                                                                 0x00FF0000L
+//DB_FIFO_DEPTH1
+#define DB_FIFO_DEPTH1__MI_RDREQ_FIFO_DEPTH__SHIFT                                                            0x0
+#define DB_FIFO_DEPTH1__MI_WRREQ_FIFO_DEPTH__SHIFT                                                            0x8
+#define DB_FIFO_DEPTH1__QC_DEPTH__SHIFT                                                                       0x18
+#define DB_FIFO_DEPTH1__MI_RDREQ_FIFO_DEPTH_MASK                                                              0x000000FFL
+#define DB_FIFO_DEPTH1__MI_WRREQ_FIFO_DEPTH_MASK                                                              0x0000FF00L
+#define DB_FIFO_DEPTH1__QC_DEPTH_MASK                                                                         0xFF000000L
+//DB_FIFO_DEPTH2
+#define DB_FIFO_DEPTH2__EQUAD_FIFO_DEPTH__SHIFT                                                               0x0
+#define DB_FIFO_DEPTH2__ETILE_OP_FIFO_DEPTH__SHIFT                                                            0x8
+#define DB_FIFO_DEPTH2__LQUAD_FIFO_DEPTH__SHIFT                                                               0x10
+#define DB_FIFO_DEPTH2__LTILE_OP_FIFO_DEPTH__SHIFT                                                            0x19
+#define DB_FIFO_DEPTH2__EQUAD_FIFO_DEPTH_MASK                                                                 0x000000FFL
+#define DB_FIFO_DEPTH2__ETILE_OP_FIFO_DEPTH_MASK                                                              0x0000FF00L
+#define DB_FIFO_DEPTH2__LQUAD_FIFO_DEPTH_MASK                                                                 0x01FF0000L
+#define DB_FIFO_DEPTH2__LTILE_OP_FIFO_DEPTH_MASK                                                              0xFE000000L
+//DB_RING_CONTROL
+#define DB_RING_CONTROL__COUNTER_CONTROL__SHIFT                                                               0x0
+#define DB_RING_CONTROL__COUNTER_CONTROL_MASK                                                                 0x00000003L
+//DB_MEM_ARB_WATERMARKS
+#define DB_MEM_ARB_WATERMARKS__CLIENT0_WATERMARK__SHIFT                                                       0x0
+#define DB_MEM_ARB_WATERMARKS__CLIENT1_WATERMARK__SHIFT                                                       0x8
+#define DB_MEM_ARB_WATERMARKS__CLIENT2_WATERMARK__SHIFT                                                       0x10
+#define DB_MEM_ARB_WATERMARKS__CLIENT3_WATERMARK__SHIFT                                                       0x18
+#define DB_MEM_ARB_WATERMARKS__CLIENT0_WATERMARK_MASK                                                         0x00000007L
+#define DB_MEM_ARB_WATERMARKS__CLIENT1_WATERMARK_MASK                                                         0x00000700L
+#define DB_MEM_ARB_WATERMARKS__CLIENT2_WATERMARK_MASK                                                         0x00070000L
+#define DB_MEM_ARB_WATERMARKS__CLIENT3_WATERMARK_MASK                                                         0x07000000L
+//DB_FIFO_DEPTH3
+#define DB_FIFO_DEPTH3__LTILE_PROBE_FIFO_DEPTH__SHIFT                                                         0x0
+#define DB_FIFO_DEPTH3__OSB_WAVE_TABLE_DEPTH__SHIFT                                                           0x8
+#define DB_FIFO_DEPTH3__OREO_WAVE_HIDE_DEPTH__SHIFT                                                           0x10
+#define DB_FIFO_DEPTH3__QUAD_READ_REQS__SHIFT                                                                 0x18
+#define DB_FIFO_DEPTH3__LTILE_PROBE_FIFO_DEPTH_MASK                                                           0x000000FFL
+#define DB_FIFO_DEPTH3__OSB_WAVE_TABLE_DEPTH_MASK                                                             0x0000FF00L
+#define DB_FIFO_DEPTH3__OREO_WAVE_HIDE_DEPTH_MASK                                                             0x00FF0000L
+#define DB_FIFO_DEPTH3__QUAD_READ_REQS_MASK                                                                   0xFF000000L
+//DB_DEBUG6
+#define DB_DEBUG6__FORCE_DB_SC_WAVE_CONFLICT__SHIFT                                                           0x0
+#define DB_DEBUG6__FORCE_DB_SC_WAVE_HARD_CONFLICT__SHIFT                                                      0x1
+#define DB_DEBUG6__FORCE_DB_SC_QUAD_CONFLICT__SHIFT                                                           0x2
+#define DB_DEBUG6__OREO_TRANSITION_EVENT_ALL__SHIFT                                                           0x3
+#define DB_DEBUG6__OREO_TRANSITION_EVENT_ID__SHIFT                                                            0x4
+#define DB_DEBUG6__OREO_TRANSITION_EVENT_EN__SHIFT                                                            0xa
+#define DB_DEBUG6__NEVER_DB_SC_WAVE_CONFLICT__SHIFT                                                           0xb
+#define DB_DEBUG6__DISABLE_PWS_PLUS_STC_TAG_LIVENESS_STALL__SHIFT                                             0xc
+#define DB_DEBUG6__SET_DB_PERFMON_PWS_PIPE_ID__SHIFT                                                          0xd
+#define DB_DEBUG6__NEVER_DB_SC_WAVE_HARD_CONFLICT__SHIFT                                                      0xf
+#define DB_DEBUG6__FTWB_MAX_TIMEOUT_VAL__SHIFT                                                                0x10
+#define DB_DEBUG6__DISABLE_LQO_SMT_RAM_OPT__SHIFT                                                             0x18
+#define DB_DEBUG6__FORCE_MAX_STILES_IN_WAVE_CHECK__SHIFT                                                      0x19
+#define DB_DEBUG6__DISABLE_OSB_DEADLOCK_FIX__SHIFT                                                            0x1a
+#define DB_DEBUG6__DISABLE_OSB_DEADLOCK_WAIT_PANIC__SHIFT                                                     0x1b
+#define DB_DEBUG6__FORCE_ZC_WRITEMASK_TO_FULL__SHIFT                                                          0x1c
+#define DB_DEBUG6__SPARE_BITS_31_28__SHIFT                                                                    0x1d
+#define DB_DEBUG6__FORCE_DB_SC_WAVE_CONFLICT_MASK                                                             0x00000001L
+#define DB_DEBUG6__FORCE_DB_SC_WAVE_HARD_CONFLICT_MASK                                                        0x00000002L
+#define DB_DEBUG6__FORCE_DB_SC_QUAD_CONFLICT_MASK                                                             0x00000004L
+#define DB_DEBUG6__OREO_TRANSITION_EVENT_ALL_MASK                                                             0x00000008L
+#define DB_DEBUG6__OREO_TRANSITION_EVENT_ID_MASK                                                              0x000003F0L
+#define DB_DEBUG6__OREO_TRANSITION_EVENT_EN_MASK                                                              0x00000400L
+#define DB_DEBUG6__NEVER_DB_SC_WAVE_CONFLICT_MASK                                                             0x00000800L
+#define DB_DEBUG6__DISABLE_PWS_PLUS_STC_TAG_LIVENESS_STALL_MASK                                               0x00001000L
+#define DB_DEBUG6__SET_DB_PERFMON_PWS_PIPE_ID_MASK                                                            0x00006000L
+#define DB_DEBUG6__NEVER_DB_SC_WAVE_HARD_CONFLICT_MASK                                                        0x00008000L
+#define DB_DEBUG6__FTWB_MAX_TIMEOUT_VAL_MASK                                                                  0x00FF0000L
+#define DB_DEBUG6__DISABLE_LQO_SMT_RAM_OPT_MASK                                                               0x01000000L
+#define DB_DEBUG6__FORCE_MAX_STILES_IN_WAVE_CHECK_MASK                                                        0x02000000L
+#define DB_DEBUG6__DISABLE_OSB_DEADLOCK_FIX_MASK                                                              0x04000000L
+#define DB_DEBUG6__DISABLE_OSB_DEADLOCK_WAIT_PANIC_MASK                                                       0x08000000L
+#define DB_DEBUG6__FORCE_ZC_WRITEMASK_TO_FULL_MASK                                                            0x10000000L
+#define DB_DEBUG6__SPARE_BITS_31_28_MASK                                                                      0xE0000000L
+//DB_EXCEPTION_CONTROL
+#define DB_EXCEPTION_CONTROL__EARLY_Z_PANIC_DISABLE__SHIFT                                                    0x0
+#define DB_EXCEPTION_CONTROL__LATE_Z_PANIC_DISABLE__SHIFT                                                     0x1
+#define DB_EXCEPTION_CONTROL__RE_Z_PANIC_DISABLE__SHIFT                                                       0x2
+#define DB_EXCEPTION_CONTROL__AUTO_FLUSH_QUAD__SHIFT                                                          0x4
+#define DB_EXCEPTION_CONTROL__FORCE_SUMMARIZE__SHIFT                                                          0x8
+#define DB_EXCEPTION_CONTROL__CAM_FREE_WATERMARK__SHIFT                                                       0x18
+#define DB_EXCEPTION_CONTROL__EARLY_Z_PANIC_DISABLE_MASK                                                      0x00000001L
+#define DB_EXCEPTION_CONTROL__LATE_Z_PANIC_DISABLE_MASK                                                       0x00000002L
+#define DB_EXCEPTION_CONTROL__RE_Z_PANIC_DISABLE_MASK                                                         0x00000004L
+#define DB_EXCEPTION_CONTROL__AUTO_FLUSH_QUAD_MASK                                                            0x00000010L
+#define DB_EXCEPTION_CONTROL__FORCE_SUMMARIZE_MASK                                                            0x00000F00L
+#define DB_EXCEPTION_CONTROL__CAM_FREE_WATERMARK_MASK                                                         0xFF000000L
+//DB_DEBUG7
+#define DB_DEBUG7__SPARE_BITS__SHIFT                                                                          0x0
+#define DB_DEBUG7__SPARE_BITS_MASK                                                                            0xFFFFFFFFL
+//DB_DEBUG5
+#define DB_DEBUG5__DISABLE_2SRC_VRS_HARD_CONFLICT__SHIFT                                                      0x3
+#define DB_DEBUG5__DISABLE_FLQ_MCC_DTILEID_CHECK__SHIFT                                                       0x4
+#define DB_DEBUG5__DISABLE_NOZ_POWER_SAVINGS__SHIFT                                                           0x5
+#define DB_DEBUG5__DISABLE_MGCG_GATING_ON_SHADER_WAIT__SHIFT                                                  0x7
+#define DB_DEBUG5__DISABLE_VRS_1X2_2XAA__SHIFT                                                                0x8
+#define DB_DEBUG5__ENABLE_FULL_TILE_WAVE_BREAK_ON_COARSE__SHIFT                                               0x9
+#define DB_DEBUG5__DISABLE_PSL_AUTO_MODE_FIX__SHIFT                                                           0xd
+#define DB_DEBUG5__DISABLE_FORCE_ZMASK_EXPANDED__SHIFT                                                        0xe
+#define DB_DEBUG5__DISABLE_SEPARATE_LQO_CLK__SHIFT                                                            0xf
+#define DB_DEBUG5__DISABLE_Z_WITHOUT_PLANES_FLQ__SHIFT                                                        0x10
+#define DB_DEBUG5__PRESERVE_QMASK_FOR_POSTZ_OP_PIPE__SHIFT                                                    0x11
+#define DB_DEBUG5__Z_NACK_BEHAVIOR_ONLY_WHEN_Z_IS_PRT__SHIFT                                                  0x12
+#define DB_DEBUG5__S_NACK_BEHAVIOR_ONLY_WHEN_S_IS_PRT__SHIFT                                                  0x13
+#define DB_DEBUG5__DISABLE_RESIDENCY_CHECK_Z__SHIFT                                                           0x14
+#define DB_DEBUG5__DISABLE_RESIDENCY_CHECK_STENCIL__SHIFT                                                     0x15
+#define DB_DEBUG5__DISABLE_LQO_FTCQ_DUAL_QUAD_REGION_CHECK__SHIFT                                             0x16
+#define DB_DEBUG5__SPARE_BITS__SHIFT                                                                          0x18
+#define DB_DEBUG5__DISABLE_2SRC_VRS_HARD_CONFLICT_MASK                                                        0x00000008L
+#define DB_DEBUG5__DISABLE_FLQ_MCC_DTILEID_CHECK_MASK                                                         0x00000010L
+#define DB_DEBUG5__DISABLE_NOZ_POWER_SAVINGS_MASK                                                             0x00000020L
+#define DB_DEBUG5__DISABLE_MGCG_GATING_ON_SHADER_WAIT_MASK                                                    0x00000080L
+#define DB_DEBUG5__DISABLE_VRS_1X2_2XAA_MASK                                                                  0x00000100L
+#define DB_DEBUG5__ENABLE_FULL_TILE_WAVE_BREAK_ON_COARSE_MASK                                                 0x00000200L
+#define DB_DEBUG5__DISABLE_PSL_AUTO_MODE_FIX_MASK                                                             0x00002000L
+#define DB_DEBUG5__DISABLE_FORCE_ZMASK_EXPANDED_MASK                                                          0x00004000L
+#define DB_DEBUG5__DISABLE_SEPARATE_LQO_CLK_MASK                                                              0x00008000L
+#define DB_DEBUG5__DISABLE_Z_WITHOUT_PLANES_FLQ_MASK                                                          0x00010000L
+#define DB_DEBUG5__PRESERVE_QMASK_FOR_POSTZ_OP_PIPE_MASK                                                      0x00020000L
+#define DB_DEBUG5__Z_NACK_BEHAVIOR_ONLY_WHEN_Z_IS_PRT_MASK                                                    0x00040000L
+#define DB_DEBUG5__S_NACK_BEHAVIOR_ONLY_WHEN_S_IS_PRT_MASK                                                    0x00080000L
+#define DB_DEBUG5__DISABLE_RESIDENCY_CHECK_Z_MASK                                                             0x00100000L
+#define DB_DEBUG5__DISABLE_RESIDENCY_CHECK_STENCIL_MASK                                                       0x00200000L
+#define DB_DEBUG5__DISABLE_LQO_FTCQ_DUAL_QUAD_REGION_CHECK_MASK                                               0x00400000L
+#define DB_DEBUG5__SPARE_BITS_MASK                                                                            0xFF000000L
+//DB_MEM_CONFIG
+#define DB_MEM_CONFIG__Z_SCOPE__SHIFT                                                                         0x0
+#define DB_MEM_CONFIG__STENCIL_SCOPE__SHIFT                                                                   0x2
+#define DB_MEM_CONFIG__OCCLUSION_SCOPE__SHIFT                                                                 0x4
+#define DB_MEM_CONFIG__ENABLE_OVERRIDE_COMP_MODE_Z__SHIFT                                                     0x6
+#define DB_MEM_CONFIG__ENABLE_OVERRIDE_COMP_MODE_STENCIL__SHIFT                                               0x7
+#define DB_MEM_CONFIG__ENABLE_OVERRIDE_COMP_MODE_OCCLUSION__SHIFT                                             0x8
+#define DB_MEM_CONFIG__Z_OVERRIDE_COMPRESSION_MODE__SHIFT                                                     0x9
+#define DB_MEM_CONFIG__STENCIL_OVERRIDE_COMPRESSION_MODE__SHIFT                                               0xb
+#define DB_MEM_CONFIG__OCCLUSION_OVERRIDE_COMPRESSION_MODE__SHIFT                                             0xd
+#define DB_MEM_CONFIG__FL_DISABLE_SINGLE_COMPRESS__SHIFT                                                      0xf
+#define DB_MEM_CONFIG__Z_SCOPE_MASK                                                                           0x00000003L
+#define DB_MEM_CONFIG__STENCIL_SCOPE_MASK                                                                     0x0000000CL
+#define DB_MEM_CONFIG__OCCLUSION_SCOPE_MASK                                                                   0x00000030L
+#define DB_MEM_CONFIG__ENABLE_OVERRIDE_COMP_MODE_Z_MASK                                                       0x00000040L
+#define DB_MEM_CONFIG__ENABLE_OVERRIDE_COMP_MODE_STENCIL_MASK                                                 0x00000080L
+#define DB_MEM_CONFIG__ENABLE_OVERRIDE_COMP_MODE_OCCLUSION_MASK                                               0x00000100L
+#define DB_MEM_CONFIG__Z_OVERRIDE_COMPRESSION_MODE_MASK                                                       0x00000600L
+#define DB_MEM_CONFIG__STENCIL_OVERRIDE_COMPRESSION_MODE_MASK                                                 0x00001800L
+#define DB_MEM_CONFIG__OCCLUSION_OVERRIDE_COMPRESSION_MODE_MASK                                               0x00006000L
+#define DB_MEM_CONFIG__FL_DISABLE_SINGLE_COMPRESS_MASK                                                        0x00008000L
+//DB_ARB_CONFIG
+#define DB_ARB_CONFIG__ARB_MODE__SHIFT                                                                        0x0
+#define DB_ARB_CONFIG__CREDITS_MAX_RD__SHIFT                                                                  0x2
+#define DB_ARB_CONFIG__CREDITS_WEIGHT_RD__SHIFT                                                               0x7
+#define DB_ARB_CONFIG__CREDITS_MAX_WR__SHIFT                                                                  0xc
+#define DB_ARB_CONFIG__CREDITS_WEIGHT_WR__SHIFT                                                               0x11
+#define DB_ARB_CONFIG__ARB_MODE_MASK                                                                          0x00000003L
+#define DB_ARB_CONFIG__CREDITS_MAX_RD_MASK                                                                    0x0000007CL
+#define DB_ARB_CONFIG__CREDITS_WEIGHT_RD_MASK                                                                 0x00000F80L
+#define DB_ARB_CONFIG__CREDITS_MAX_WR_MASK                                                                    0x0001F000L
+#define DB_ARB_CONFIG__CREDITS_WEIGHT_WR_MASK                                                                 0x003E0000L
+//DB_DFD_INDIRECT_SEL
+#define DB_DFD_INDIRECT_SEL__DFD_INDEX__SHIFT                                                                 0x0
+#define DB_DFD_INDIRECT_SEL__DFD_INDEX_MASK                                                                   0x000000FFL
+//DB_DFD_INDIRECT_DAT
+#define DB_DFD_INDIRECT_DAT__DFD_DATA__SHIFT                                                                  0x0
+#define DB_DFD_INDIRECT_DAT__DFD_DATA_MASK                                                                    0xFFFFFFFFL
+//DB_SUMMARIZER_TIMEOUTS
+#define DB_SUMMARIZER_TIMEOUTS__SUMM_CNTL_EVICT_TIMEOUT__SHIFT                                                0x0
+#define DB_SUMMARIZER_TIMEOUTS__SUMM_EVICT_TIMEOUT__SHIFT                                                     0x10
+#define DB_SUMMARIZER_TIMEOUTS__SUMM_CNTL_EVICT_TIMEOUT_MASK                                                  0x00000FFFL
+#define DB_SUMMARIZER_TIMEOUTS__SUMM_EVICT_TIMEOUT_MASK                                                       0x0FFF0000L
+//DB_FGCG_SRAMS_CLK_CTRL
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE0__SHIFT                                                              0x0
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE1__SHIFT                                                              0x1
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE2__SHIFT                                                              0x2
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE3__SHIFT                                                              0x3
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE4__SHIFT                                                              0x4
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE5__SHIFT                                                              0x5
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE6__SHIFT                                                              0x6
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE7__SHIFT                                                              0x7
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE8__SHIFT                                                              0x8
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE9__SHIFT                                                              0x9
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE10__SHIFT                                                             0xa
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE11__SHIFT                                                             0xb
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE12__SHIFT                                                             0xc
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE13__SHIFT                                                             0xd
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE14__SHIFT                                                             0xe
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE15__SHIFT                                                             0xf
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE16__SHIFT                                                             0x10
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE17__SHIFT                                                             0x11
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE18__SHIFT                                                             0x12
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE19__SHIFT                                                             0x13
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE20__SHIFT                                                             0x14
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE22__SHIFT                                                             0x16
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE23__SHIFT                                                             0x17
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE24__SHIFT                                                             0x18
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE26__SHIFT                                                             0x1a
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE27__SHIFT                                                             0x1b
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE28__SHIFT                                                             0x1c
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE29__SHIFT                                                             0x1d
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE30__SHIFT                                                             0x1e
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE31__SHIFT                                                             0x1f
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE0_MASK                                                                0x00000001L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE1_MASK                                                                0x00000002L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE2_MASK                                                                0x00000004L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE3_MASK                                                                0x00000008L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE4_MASK                                                                0x00000010L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE5_MASK                                                                0x00000020L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE6_MASK                                                                0x00000040L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE7_MASK                                                                0x00000080L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE8_MASK                                                                0x00000100L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE9_MASK                                                                0x00000200L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE10_MASK                                                               0x00000400L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE11_MASK                                                               0x00000800L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE12_MASK                                                               0x00001000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE13_MASK                                                               0x00002000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE14_MASK                                                               0x00004000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE15_MASK                                                               0x00008000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE16_MASK                                                               0x00010000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE17_MASK                                                               0x00020000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE18_MASK                                                               0x00040000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE19_MASK                                                               0x00080000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE20_MASK                                                               0x00100000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE22_MASK                                                               0x00400000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE23_MASK                                                               0x00800000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE24_MASK                                                               0x01000000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE26_MASK                                                               0x04000000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE27_MASK                                                               0x08000000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE28_MASK                                                               0x10000000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE29_MASK                                                               0x20000000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE30_MASK                                                               0x40000000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE31_MASK                                                               0x80000000L
+//DB_FGCG_INTERFACES_CLK_CTRL
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_SC_QUAD_OVERRIDE__SHIFT                                               0x0
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_CB_EXPORT_OVERRIDE__SHIFT                                             0x2
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_GL1_COMP_REQ_OVERRIDE__SHIFT                                          0x3
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_GL1_SRC_OVERRIDE__SHIFT                                               0x4
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_SC_UPDATE_OVERRIDE__SHIFT                                             0x5
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_CB_RMIRET_OVERRIDE__SHIFT                                             0x6
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_SC_WAVE_OVERRIDE__SHIFT                                               0x7
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_SC_FREE_WAVE_OVERRIDE__SHIFT                                          0x8
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_SC_QUAD_OVERRIDE_MASK                                                 0x00000001L
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_CB_EXPORT_OVERRIDE_MASK                                               0x00000004L
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_GL1_COMP_REQ_OVERRIDE_MASK                                            0x00000008L
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_GL1_SRC_OVERRIDE_MASK                                                 0x00000010L
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_SC_UPDATE_OVERRIDE_MASK                                               0x00000020L
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_CB_RMIRET_OVERRIDE_MASK                                               0x00000040L
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_SC_WAVE_OVERRIDE_MASK                                                 0x00000080L
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_SC_FREE_WAVE_OVERRIDE_MASK                                            0x00000100L
+//DB_FIFO_DEPTH4
+#define DB_FIFO_DEPTH4__OSB_SQUAD_TABLE_DEPTH__SHIFT                                                          0x0
+#define DB_FIFO_DEPTH4__OSB_TILE_TABLE_DEPTH__SHIFT                                                           0x8
+#define DB_FIFO_DEPTH4__OSB_SCORE_BOARD_DEPTH__SHIFT                                                          0x10
+#define DB_FIFO_DEPTH4__OSB_EVENT_FIFO_DEPTH__SHIFT                                                           0x18
+#define DB_FIFO_DEPTH4__OSB_SQUAD_TABLE_DEPTH_MASK                                                            0x000000FFL
+#define DB_FIFO_DEPTH4__OSB_TILE_TABLE_DEPTH_MASK                                                             0x0000FF00L
+#define DB_FIFO_DEPTH4__OSB_SCORE_BOARD_DEPTH_MASK                                                            0x00FF0000L
+#define DB_FIFO_DEPTH4__OSB_EVENT_FIFO_DEPTH_MASK                                                             0xFF000000L
+//CC_RB_BACKEND_DISABLE
+#define CC_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT                                                         0x4
+#define CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK                                                           0x000000F0L
+//GB_ADDR_CONFIG_1
+#define GB_ADDR_CONFIG_1__NUM_PIPES__SHIFT                                                                    0x0
+#define GB_ADDR_CONFIG_1__PIPE_INTERLEAVE_SIZE__SHIFT                                                         0x3
+#define GB_ADDR_CONFIG_1__MAX_COMPRESSED_FRAGS__SHIFT                                                         0x6
+#define GB_ADDR_CONFIG_1__NUM_PKRS__SHIFT                                                                     0x8
+#define GB_ADDR_CONFIG_1__NUM_SHADER_ENGINES__SHIFT                                                           0x13
+#define GB_ADDR_CONFIG_1__NUM_RB_PER_SE__SHIFT                                                                0x1a
+#define GB_ADDR_CONFIG_1__NUM_PIPES_MASK                                                                      0x00000007L
+#define GB_ADDR_CONFIG_1__PIPE_INTERLEAVE_SIZE_MASK                                                           0x00000038L
+#define GB_ADDR_CONFIG_1__MAX_COMPRESSED_FRAGS_MASK                                                           0x000000C0L
+#define GB_ADDR_CONFIG_1__NUM_PKRS_MASK                                                                       0x00000700L
+#define GB_ADDR_CONFIG_1__NUM_SHADER_ENGINES_MASK                                                             0x00780000L
+#define GB_ADDR_CONFIG_1__NUM_RB_PER_SE_MASK                                                                  0x0C000000L
+//GB_BACKEND_MAP
+#define GB_BACKEND_MAP__BACKEND_MAP__SHIFT                                                                    0x0
+#define GB_BACKEND_MAP__BACKEND_MAP_MASK                                                                      0xFFFFFFFFL
+//GB_GPU_ID
+#define GB_GPU_ID__GPU_ID__SHIFT                                                                              0x0
+#define GB_GPU_ID__GPU_ID_MASK                                                                                0x0000000FL
+//GB_ADDR_CONFIG_READ
+#define GB_ADDR_CONFIG_READ__NUM_PIPES__SHIFT                                                                 0x0
+#define GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE__SHIFT                                                      0x3
+#define GB_ADDR_CONFIG_READ__MAX_COMPRESSED_FRAGS__SHIFT                                                      0x6
+#define GB_ADDR_CONFIG_READ__NUM_PKRS__SHIFT                                                                  0x8
+#define GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES__SHIFT                                                        0x13
+#define GB_ADDR_CONFIG_READ__NUM_RB_PER_SE__SHIFT                                                             0x1a
+#define GB_ADDR_CONFIG_READ__NUM_PIPES_MASK                                                                   0x00000007L
+#define GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE_MASK                                                        0x00000038L
+#define GB_ADDR_CONFIG_READ__MAX_COMPRESSED_FRAGS_MASK                                                        0x000000C0L
+#define GB_ADDR_CONFIG_READ__NUM_PKRS_MASK                                                                    0x00000700L
+#define GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES_MASK                                                          0x00780000L
+#define GB_ADDR_CONFIG_READ__NUM_RB_PER_SE_MASK                                                               0x0C000000L
+//CB_HW_CONTROL_4
+#define CB_HW_CONTROL_4__ENABLE_READ_RESIDENCY_TIMEOUT_CNTR__SHIFT                                            0x0
+#define CB_HW_CONTROL_4__THRESHOLD_READ_RESIDENCY_TIMEOUT_CNTR__SHIFT                                         0x1
+#define CB_HW_CONTROL_4__DISABLE_FRAGOP_MULTI_FRAGMENT__SHIFT                                                 0xe
+#define CB_HW_CONTROL_4__ENABLE_FRAGOP_STALLING_ON_RAW_HAZARD__SHIFT                                          0x10
+#define CB_HW_CONTROL_4__ENABLE_FRAGOP_STALLING_ON_COARSE_RAW_HAZARD__SHIFT                                   0x11
+#define CB_HW_CONTROL_4__ENABLE_FRAGOP_STALLING_ON_DS_RAW_HAZARD__SHIFT                                       0x12
+#define CB_HW_CONTROL_4__ENABLE_READ_RESIDENCY_TIMEOUT_CNTR_MASK                                              0x00000001L
+#define CB_HW_CONTROL_4__THRESHOLD_READ_RESIDENCY_TIMEOUT_CNTR_MASK                                           0x00000006L
+#define CB_HW_CONTROL_4__DISABLE_FRAGOP_MULTI_FRAGMENT_MASK                                                   0x00004000L
+#define CB_HW_CONTROL_4__ENABLE_FRAGOP_STALLING_ON_RAW_HAZARD_MASK                                            0x00010000L
+#define CB_HW_CONTROL_4__ENABLE_FRAGOP_STALLING_ON_COARSE_RAW_HAZARD_MASK                                     0x00020000L
+#define CB_HW_CONTROL_4__ENABLE_FRAGOP_STALLING_ON_DS_RAW_HAZARD_MASK                                         0x00040000L
+//CB_HW_CONTROL_3
+#define CB_HW_CONTROL_3__FORCE_GLX_REQ_CLKEN_HIGH__SHIFT                                                      0x3
+#define CB_HW_CONTROL_3__FORCE_GLX_SRC_CLKEN_HIGH__SHIFT                                                      0x4
+#define CB_HW_CONTROL_3__SPLIT_ALL_FAST_MODE_TRANSFERS__SHIFT                                                 0x6
+#define CB_HW_CONTROL_3__DISABLE_SHADER_BLEND_OPTS__SHIFT                                                     0x7
+#define CB_HW_CONTROL_3__DISABLE_FMASK_OPT_WA_AND_FULLY_COVERED__SHIFT                                        0x15
+#define CB_HW_CONTROL_3__DISABLE_READ_DAW_OPT_OVERRIDE_ROH_COMP__SHIFT                                        0x19
+#define CB_HW_CONTROL_3__DISABLE_READ_DAW_OPT_INSERT_BUBBLES_B2B__SHIFT                                       0x1a
+#define CB_HW_CONTROL_3__DISABLE_READ_DAW_CLEAR_KEY_OVERRIDE_FROM_ILLEGAL_DECODE__SHIFT                       0x1b
+#define CB_HW_CONTROL_3__DISABLE_READ_DAW_WAIT_SECOND_64B__SHIFT                                              0x1c
+#define CB_HW_CONTROL_3__DISABLE_READ_DAW_OPT_256B_CLEARS_BECOME_1FRAG__SHIFT                                 0x1d
+#define CB_HW_CONTROL_3__FORCE_GLX_REQ_CLKEN_HIGH_MASK                                                        0x00000008L
+#define CB_HW_CONTROL_3__FORCE_GLX_SRC_CLKEN_HIGH_MASK                                                        0x00000010L
+#define CB_HW_CONTROL_3__SPLIT_ALL_FAST_MODE_TRANSFERS_MASK                                                   0x00000040L
+#define CB_HW_CONTROL_3__DISABLE_SHADER_BLEND_OPTS_MASK                                                       0x00000080L
+#define CB_HW_CONTROL_3__DISABLE_FMASK_OPT_WA_AND_FULLY_COVERED_MASK                                          0x00200000L
+#define CB_HW_CONTROL_3__DISABLE_READ_DAW_OPT_OVERRIDE_ROH_COMP_MASK                                          0x02000000L
+#define CB_HW_CONTROL_3__DISABLE_READ_DAW_OPT_INSERT_BUBBLES_B2B_MASK                                         0x04000000L
+#define CB_HW_CONTROL_3__DISABLE_READ_DAW_CLEAR_KEY_OVERRIDE_FROM_ILLEGAL_DECODE_MASK                         0x08000000L
+#define CB_HW_CONTROL_3__DISABLE_READ_DAW_WAIT_SECOND_64B_MASK                                                0x10000000L
+#define CB_HW_CONTROL_3__DISABLE_READ_DAW_OPT_256B_CLEARS_BECOME_1FRAG_MASK                                   0x20000000L
+//CB_HW_CONTROL
+#define CB_HW_CONTROL__DISABLE_GRBM_BUSY_CNTR__SHIFT                                                          0x0
+#define CB_HW_CONTROL__DISABLE_VRS_FILLRATE_OPTIMIZATION__SHIFT                                               0x1
+#define CB_HW_CONTROL__GLX_CREDITS__SHIFT                                                                     0x6
+#define CB_HW_CONTROL__NUM_CCC_SKID_FIFO_ENTRIES__SHIFT                                                       0xc
+#define CB_HW_CONTROL__DISABLE_EVICT_ILLEGAL_KEY_OVERRIDE__SHIFT                                              0xf
+#define CB_HW_CONTROL__FORCE_EVICT_ALL_VALID__SHIFT                                                           0x10
+#define CB_HW_CONTROL__FORCE_WAIT_EOP_DONE_FLUSH__SHIFT                                                       0x12
+#define CB_HW_CONTROL__FORCE_NEEDS_DST__SHIFT                                                                 0x13
+#define CB_HW_CONTROL__DISABLE_USE_OF_SET_HASH__SHIFT                                                         0x14
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_RESULT_EQ_DEST__SHIFT                                                0x15
+#define CB_HW_CONTROL__ENABLE_SINGLE_KEY_WR_OPT__SHIFT                                                        0x16
+#define CB_HW_CONTROL__DISABLE_POWER_OPT_HC__SHIFT                                                            0x17
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DONT_RD_DST__SHIFT                                                   0x18
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_BYPASS__SHIFT                                                        0x19
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DISCARD_PIXEL__SHIFT                                                 0x1a
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_WHEN_DISABLED_SRCALPHA_IS_USED__SHIFT                                0x1b
+#define CB_HW_CONTROL__DISABLE_MULTICYCLE_WRITES__SHIFT                                                       0x1c
+#define CB_HW_CONTROL__DISABLE_HOLE_COLLAPSE__SHIFT                                                           0x1d
+#define CB_HW_CONTROL__DISABLE_FMASK_REREAD_OPT__SHIFT                                                        0x1e
+#define CB_HW_CONTROL__EN_KEY_OVERRIDE__SHIFT                                                                 0x1f
+#define CB_HW_CONTROL__DISABLE_GRBM_BUSY_CNTR_MASK                                                            0x00000001L
+#define CB_HW_CONTROL__DISABLE_VRS_FILLRATE_OPTIMIZATION_MASK                                                 0x00000002L
+#define CB_HW_CONTROL__GLX_CREDITS_MASK                                                                       0x000003C0L
+#define CB_HW_CONTROL__NUM_CCC_SKID_FIFO_ENTRIES_MASK                                                         0x00007000L
+#define CB_HW_CONTROL__DISABLE_EVICT_ILLEGAL_KEY_OVERRIDE_MASK                                                0x00008000L
+#define CB_HW_CONTROL__FORCE_EVICT_ALL_VALID_MASK                                                             0x00010000L
+#define CB_HW_CONTROL__FORCE_WAIT_EOP_DONE_FLUSH_MASK                                                         0x00040000L
+#define CB_HW_CONTROL__FORCE_NEEDS_DST_MASK                                                                   0x00080000L
+#define CB_HW_CONTROL__DISABLE_USE_OF_SET_HASH_MASK                                                           0x00100000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_RESULT_EQ_DEST_MASK                                                  0x00200000L
+#define CB_HW_CONTROL__ENABLE_SINGLE_KEY_WR_OPT_MASK                                                          0x00400000L
+#define CB_HW_CONTROL__DISABLE_POWER_OPT_HC_MASK                                                              0x00800000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DONT_RD_DST_MASK                                                     0x01000000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_BYPASS_MASK                                                          0x02000000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DISCARD_PIXEL_MASK                                                   0x04000000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_WHEN_DISABLED_SRCALPHA_IS_USED_MASK                                  0x08000000L
+#define CB_HW_CONTROL__DISABLE_MULTICYCLE_WRITES_MASK                                                         0x10000000L
+#define CB_HW_CONTROL__DISABLE_HOLE_COLLAPSE_MASK                                                             0x20000000L
+#define CB_HW_CONTROL__DISABLE_FMASK_REREAD_OPT_MASK                                                          0x40000000L
+#define CB_HW_CONTROL__EN_KEY_OVERRIDE_MASK                                                                   0x80000000L
+//CB_HW_CONTROL_1
+#define CB_HW_CONTROL_1__CC_CACHE_NUM_TAGS__SHIFT                                                             0x0
+#define CB_HW_CONTROL_1__DISABLE_SRC_FIFO_BYP__SHIFT                                                          0x15
+#define CB_HW_CONTROL_1__DISABLE_RDLAT_FIFO_BYP__SHIFT                                                        0x16
+#define CB_HW_CONTROL_1__ENABLE_COMPRESSION_ON_BYPASS__SHIFT                                                  0x17
+#define CB_HW_CONTROL_1__COLOR_SCOPE__SHIFT                                                                   0x18
+#define CB_HW_CONTROL_1__GLX_NOFILL__SHIFT                                                                    0x1a
+#define CB_HW_CONTROL_1__GLX_VQID__SHIFT                                                                      0x1b
+#define CB_HW_CONTROL_1__GLX_PERF_CNTR_EN__SHIFT                                                              0x1f
+#define CB_HW_CONTROL_1__CC_CACHE_NUM_TAGS_MASK                                                               0x0000003FL
+#define CB_HW_CONTROL_1__DISABLE_SRC_FIFO_BYP_MASK                                                            0x00200000L
+#define CB_HW_CONTROL_1__DISABLE_RDLAT_FIFO_BYP_MASK                                                          0x00400000L
+#define CB_HW_CONTROL_1__ENABLE_COMPRESSION_ON_BYPASS_MASK                                                    0x00800000L
+#define CB_HW_CONTROL_1__COLOR_SCOPE_MASK                                                                     0x03000000L
+#define CB_HW_CONTROL_1__GLX_NOFILL_MASK                                                                      0x04000000L
+#define CB_HW_CONTROL_1__GLX_VQID_MASK                                                                        0x78000000L
+#define CB_HW_CONTROL_1__GLX_PERF_CNTR_EN_MASK                                                                0x80000000L
+//CB_HW_CONTROL_2
+#define CB_HW_CONTROL_2__RESERVED__SHIFT                                                                      0x0
+#define CB_HW_CONTROL_2__RESERVED_MASK                                                                        0x00000001L
+//CB_HW_MEM_ARBITER_CTL
+#define CB_HW_MEM_ARBITER_CTL__ARB_MODE__SHIFT                                                                0x0
+#define CB_HW_MEM_ARBITER_CTL__READ_CYC_WEIGHT__SHIFT                                                         0x1
+#define CB_HW_MEM_ARBITER_CTL__READ_CRED_CNT_MAX__SHIFT                                                       0x7
+#define CB_HW_MEM_ARBITER_CTL__WRITE_CYC_WEIGHT__SHIFT                                                        0xd
+#define CB_HW_MEM_ARBITER_CTL__WRITE_CRED_CNT_MAX__SHIFT                                                      0x13
+#define CB_HW_MEM_ARBITER_CTL__WRITE_CREDIT_MODE__SHIFT                                                       0x1b
+#define CB_HW_MEM_ARBITER_CTL__READ_CREDIT_MODE__SHIFT                                                        0x1c
+#define CB_HW_MEM_ARBITER_CTL__ARB_MODE_MASK                                                                  0x00000001L
+#define CB_HW_MEM_ARBITER_CTL__READ_CYC_WEIGHT_MASK                                                           0x0000003EL
+#define CB_HW_MEM_ARBITER_CTL__READ_CRED_CNT_MAX_MASK                                                         0x00000F80L
+#define CB_HW_MEM_ARBITER_CTL__WRITE_CYC_WEIGHT_MASK                                                          0x0003E000L
+#define CB_HW_MEM_ARBITER_CTL__WRITE_CRED_CNT_MAX_MASK                                                        0x00F80000L
+#define CB_HW_MEM_ARBITER_CTL__WRITE_CREDIT_MODE_MASK                                                         0x08000000L
+#define CB_HW_MEM_ARBITER_CTL__READ_CREDIT_MODE_MASK                                                          0x10000000L
+//CB_FGCG_SRAM_OVERRIDE
+#define CB_FGCG_SRAM_OVERRIDE__DISABLE_FGCG__SHIFT                                                            0x0
+#define CB_FGCG_SRAM_OVERRIDE__DISABLE_FGCG_MASK                                                              0x000007FFL
+//CB_CACHE_EVICT_POINTS
+#define CB_CACHE_EVICT_POINTS__CC_COLOR_EVICT_POINT__SHIFT                                                    0x0
+#define CB_CACHE_EVICT_POINTS__CC_FMASK_EVICT_POINT__SHIFT                                                    0x8
+#define CB_CACHE_EVICT_POINTS__CC_CACHE_EVICT_POINT__SHIFT                                                    0x18
+#define CB_CACHE_EVICT_POINTS__CC_COLOR_EVICT_POINT_MASK                                                      0x000000FFL
+#define CB_CACHE_EVICT_POINTS__CC_FMASK_EVICT_POINT_MASK                                                      0x0000FF00L
+#define CB_CACHE_EVICT_POINTS__CC_CACHE_EVICT_POINT_MASK                                                      0xFF000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_spipdec2
+//SPI_PQEV_CTRL
+#define SPI_PQEV_CTRL__SCAN_PERIOD__SHIFT                                                                     0x0
+#define SPI_PQEV_CTRL__QUEUE_DURATION__SHIFT                                                                  0xa
+#define SPI_PQEV_CTRL__COMPUTE_PIPE_EN__SHIFT                                                                 0x10
+#define SPI_PQEV_CTRL__SCAN_PERIOD_MASK                                                                       0x000003FFL
+#define SPI_PQEV_CTRL__QUEUE_DURATION_MASK                                                                    0x0000FC00L
+#define SPI_PQEV_CTRL__COMPUTE_PIPE_EN_MASK                                                                   0x00FF0000L
+//SPI_EXP_THROTTLE_CTRL
+#define SPI_EXP_THROTTLE_CTRL__ENABLE__SHIFT                                                                  0x0
+#define SPI_EXP_THROTTLE_CTRL__PERIOD__SHIFT                                                                  0x1
+#define SPI_EXP_THROTTLE_CTRL__UPSTEP__SHIFT                                                                  0x5
+#define SPI_EXP_THROTTLE_CTRL__DOWNSTEP__SHIFT                                                                0x9
+#define SPI_EXP_THROTTLE_CTRL__LOW_STALL_MON_HIST_COUNT__SHIFT                                                0xd
+#define SPI_EXP_THROTTLE_CTRL__HIGH_STALL_MON_HIST_COUNT__SHIFT                                               0x10
+#define SPI_EXP_THROTTLE_CTRL__EXP_STALL_THRESHOLD__SHIFT                                                     0x13
+#define SPI_EXP_THROTTLE_CTRL__SKEW_COUNT__SHIFT                                                              0x1a
+#define SPI_EXP_THROTTLE_CTRL__THROTTLE_RESET__SHIFT                                                          0x1d
+#define SPI_EXP_THROTTLE_CTRL__ENABLE_MASK                                                                    0x00000001L
+#define SPI_EXP_THROTTLE_CTRL__PERIOD_MASK                                                                    0x0000001EL
+#define SPI_EXP_THROTTLE_CTRL__UPSTEP_MASK                                                                    0x000001E0L
+#define SPI_EXP_THROTTLE_CTRL__DOWNSTEP_MASK                                                                  0x00001E00L
+#define SPI_EXP_THROTTLE_CTRL__LOW_STALL_MON_HIST_COUNT_MASK                                                  0x0000E000L
+#define SPI_EXP_THROTTLE_CTRL__HIGH_STALL_MON_HIST_COUNT_MASK                                                 0x00070000L
+#define SPI_EXP_THROTTLE_CTRL__EXP_STALL_THRESHOLD_MASK                                                       0x03F80000L
+#define SPI_EXP_THROTTLE_CTRL__SKEW_COUNT_MASK                                                                0x1C000000L
+#define SPI_EXP_THROTTLE_CTRL__THROTTLE_RESET_MASK                                                            0x20000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_rmi_gfx_se_rmidec
+//RMI_GENERAL_CNTL
+#define RMI_GENERAL_CNTL__BURST_DISABLE__SHIFT                                                                0x0
+#define RMI_GENERAL_CNTL__VMID_BYPASS_ENABLE__SHIFT                                                           0x1
+#define RMI_GENERAL_CNTL__RB0_HARVEST_EN__SHIFT                                                               0x13
+#define RMI_GENERAL_CNTL__LOOPBACK_DIS_BY_REQ_TYPE__SHIFT                                                     0x15
+#define RMI_GENERAL_CNTL__BURST_DISABLE_MASK                                                                  0x00000001L
+#define RMI_GENERAL_CNTL__VMID_BYPASS_ENABLE_MASK                                                             0x0001FFFEL
+#define RMI_GENERAL_CNTL__RB0_HARVEST_EN_MASK                                                                 0x00080000L
+#define RMI_GENERAL_CNTL__LOOPBACK_DIS_BY_REQ_TYPE_MASK                                                       0x01E00000L
+//RMI_GENERAL_CNTL1
+#define RMI_GENERAL_CNTL1__EARLY_WRACK_ENABLE_PER_MTYPE__SHIFT                                                0x0
+#define RMI_GENERAL_CNTL1__TCIW0_64B_RD_STALL_MODE__SHIFT                                                     0x4
+#define RMI_GENERAL_CNTL1__TCIW1_64B_RD_STALL_MODE__SHIFT                                                     0x6
+#define RMI_GENERAL_CNTL1__EARLY_WRACK_DISABLE_FOR_LOOPBACK__SHIFT                                            0x8
+#define RMI_GENERAL_CNTL1__POLICY_OVERRIDE_VALUE__SHIFT                                                       0x9
+#define RMI_GENERAL_CNTL1__POLICY_OVERRIDE__SHIFT                                                             0xb
+#define RMI_GENERAL_CNTL1__ARBITER_ADDRESS_CHANGE_ENABLE__SHIFT                                               0xe
+#define RMI_GENERAL_CNTL1__LAST_OF_BURST_INSERTION_DISABLE__SHIFT                                             0xf
+#define RMI_GENERAL_CNTL1__TCIW0_PRODUCER_CREDITS__SHIFT                                                      0x10
+#define RMI_GENERAL_CNTL1__TCIW1_PRODUCER_CREDITS__SHIFT                                                      0x16
+#define RMI_GENERAL_CNTL1__EARLY_WRACK_ENABLE_PER_MTYPE_MASK                                                  0x0000000FL
+#define RMI_GENERAL_CNTL1__TCIW0_64B_RD_STALL_MODE_MASK                                                       0x00000030L
+#define RMI_GENERAL_CNTL1__TCIW1_64B_RD_STALL_MODE_MASK                                                       0x000000C0L
+#define RMI_GENERAL_CNTL1__EARLY_WRACK_DISABLE_FOR_LOOPBACK_MASK                                              0x00000100L
+#define RMI_GENERAL_CNTL1__POLICY_OVERRIDE_VALUE_MASK                                                         0x00000600L
+#define RMI_GENERAL_CNTL1__POLICY_OVERRIDE_MASK                                                               0x00000800L
+#define RMI_GENERAL_CNTL1__ARBITER_ADDRESS_CHANGE_ENABLE_MASK                                                 0x00004000L
+#define RMI_GENERAL_CNTL1__LAST_OF_BURST_INSERTION_DISABLE_MASK                                               0x00008000L
+#define RMI_GENERAL_CNTL1__TCIW0_PRODUCER_CREDITS_MASK                                                        0x003F0000L
+#define RMI_GENERAL_CNTL1__TCIW1_PRODUCER_CREDITS_MASK                                                        0x0FC00000L
+//RMI_GENERAL_STATUS
+#define RMI_GENERAL_STATUS__GENERAL_RMI_ERRORS_COMBINED__SHIFT                                                0x0
+#define RMI_GENERAL_STATUS__SKID_FIFO_0_OVERFLOW_ERROR__SHIFT                                                 0x1
+#define RMI_GENERAL_STATUS__SKID_FIFO_0_UNDERFLOW_ERROR__SHIFT                                                0x2
+#define RMI_GENERAL_STATUS__SKID_FIFO_1_OVERFLOW_ERROR__SHIFT                                                 0x3
+#define RMI_GENERAL_STATUS__SKID_FIFO_1_UNDERFLOW_ERROR__SHIFT                                                0x4
+#define RMI_GENERAL_STATUS__RMI_XBAR_BUSY__SHIFT                                                              0x5
+#define RMI_GENERAL_STATUS__RESERVED_BIT_6__SHIFT                                                             0x6
+#define RMI_GENERAL_STATUS__RMI_SCOREBOARD_BUSY__SHIFT                                                        0x7
+#define RMI_GENERAL_STATUS__TCIW0_PRT_FIFO_BUSY__SHIFT                                                        0x8
+#define RMI_GENERAL_STATUS__TCIW_FRMTR0_BUSY__SHIFT                                                           0x9
+#define RMI_GENERAL_STATUS__TCIW_RTN_FRMTR0_BUSY__SHIFT                                                       0xa
+#define RMI_GENERAL_STATUS__WRREQ_CONSUMER_FIFO_0_BUSY__SHIFT                                                 0xb
+#define RMI_GENERAL_STATUS__RDREQ_CONSUMER_FIFO_0_BUSY__SHIFT                                                 0xc
+#define RMI_GENERAL_STATUS__TCIW1_PRT_FIFO_BUSY__SHIFT                                                        0xd
+#define RMI_GENERAL_STATUS__TCIW_FRMTR1_BUSY__SHIFT                                                           0xe
+#define RMI_GENERAL_STATUS__TCIW_RTN_FRMTR1_BUSY__SHIFT                                                       0xf
+#define RMI_GENERAL_STATUS__RESERVED_BIT_18__SHIFT                                                            0x12
+#define RMI_GENERAL_STATUS__RESERVED_BIT_19__SHIFT                                                            0x13
+#define RMI_GENERAL_STATUS__RESERVED_BIT_20__SHIFT                                                            0x14
+#define RMI_GENERAL_STATUS__RESERVED_BITS_28_21__SHIFT                                                        0x15
+#define RMI_GENERAL_STATUS__RESERVED_BIT_29__SHIFT                                                            0x1d
+#define RMI_GENERAL_STATUS__RESERVED_BIT_30__SHIFT                                                            0x1e
+#define RMI_GENERAL_STATUS__SKID_FIFO_FREESPACE_IS_ZERO_ERROR__SHIFT                                          0x1f
+#define RMI_GENERAL_STATUS__GENERAL_RMI_ERRORS_COMBINED_MASK                                                  0x00000001L
+#define RMI_GENERAL_STATUS__SKID_FIFO_0_OVERFLOW_ERROR_MASK                                                   0x00000002L
+#define RMI_GENERAL_STATUS__SKID_FIFO_0_UNDERFLOW_ERROR_MASK                                                  0x00000004L
+#define RMI_GENERAL_STATUS__SKID_FIFO_1_OVERFLOW_ERROR_MASK                                                   0x00000008L
+#define RMI_GENERAL_STATUS__SKID_FIFO_1_UNDERFLOW_ERROR_MASK                                                  0x00000010L
+#define RMI_GENERAL_STATUS__RMI_XBAR_BUSY_MASK                                                                0x00000020L
+#define RMI_GENERAL_STATUS__RESERVED_BIT_6_MASK                                                               0x00000040L
+#define RMI_GENERAL_STATUS__RMI_SCOREBOARD_BUSY_MASK                                                          0x00000080L
+#define RMI_GENERAL_STATUS__TCIW0_PRT_FIFO_BUSY_MASK                                                          0x00000100L
+#define RMI_GENERAL_STATUS__TCIW_FRMTR0_BUSY_MASK                                                             0x00000200L
+#define RMI_GENERAL_STATUS__TCIW_RTN_FRMTR0_BUSY_MASK                                                         0x00000400L
+#define RMI_GENERAL_STATUS__WRREQ_CONSUMER_FIFO_0_BUSY_MASK                                                   0x00000800L
+#define RMI_GENERAL_STATUS__RDREQ_CONSUMER_FIFO_0_BUSY_MASK                                                   0x00001000L
+#define RMI_GENERAL_STATUS__TCIW1_PRT_FIFO_BUSY_MASK                                                          0x00002000L
+#define RMI_GENERAL_STATUS__TCIW_FRMTR1_BUSY_MASK                                                             0x00004000L
+#define RMI_GENERAL_STATUS__TCIW_RTN_FRMTR1_BUSY_MASK                                                         0x00008000L
+#define RMI_GENERAL_STATUS__RESERVED_BIT_18_MASK                                                              0x00040000L
+#define RMI_GENERAL_STATUS__RESERVED_BIT_19_MASK                                                              0x00080000L
+#define RMI_GENERAL_STATUS__RESERVED_BIT_20_MASK                                                              0x00100000L
+#define RMI_GENERAL_STATUS__RESERVED_BITS_28_21_MASK                                                          0x1FE00000L
+#define RMI_GENERAL_STATUS__RESERVED_BIT_29_MASK                                                              0x20000000L
+#define RMI_GENERAL_STATUS__RESERVED_BIT_30_MASK                                                              0x40000000L
+#define RMI_GENERAL_STATUS__SKID_FIFO_FREESPACE_IS_ZERO_ERROR_MASK                                            0x80000000L
+//RMI_SUBBLOCK_STATUS0
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_NUM_USED_PROBE0__SHIFT                                     0x0
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_FULL_PROBE0__SHIFT                                         0x7
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_EMPTY_PROBE0__SHIFT                                        0x8
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_NUM_USED_PROBE1__SHIFT                                     0x9
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_FULL_PROBE1__SHIFT                                         0x10
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_EMPTY_PROBE1__SHIFT                                        0x11
+#define RMI_SUBBLOCK_STATUS0__TCIW0_INFLIGHT_CNT__SHIFT                                                       0x12
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_NUM_USED_PROBE0_MASK                                       0x0000007FL
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_FULL_PROBE0_MASK                                           0x00000080L
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_EMPTY_PROBE0_MASK                                          0x00000100L
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_NUM_USED_PROBE1_MASK                                       0x0000FE00L
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_FULL_PROBE1_MASK                                           0x00010000L
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_EMPTY_PROBE1_MASK                                          0x00020000L
+#define RMI_SUBBLOCK_STATUS0__TCIW0_INFLIGHT_CNT_MASK                                                         0x0FFC0000L
+//RMI_SUBBLOCK_STATUS1
+#define RMI_SUBBLOCK_STATUS1__SKID_FIFO_0_FREE_SPACE__SHIFT                                                   0x0
+#define RMI_SUBBLOCK_STATUS1__SKID_FIFO_1_FREE_SPACE__SHIFT                                                   0xa
+#define RMI_SUBBLOCK_STATUS1__TCIW1_INFLIGHT_CNT__SHIFT                                                       0x14
+#define RMI_SUBBLOCK_STATUS1__SKID_FIFO_0_FREE_SPACE_MASK                                                     0x000003FFL
+#define RMI_SUBBLOCK_STATUS1__SKID_FIFO_1_FREE_SPACE_MASK                                                     0x000FFC00L
+#define RMI_SUBBLOCK_STATUS1__TCIW1_INFLIGHT_CNT_MASK                                                         0x3FF00000L
+//RMI_SUBBLOCK_STATUS2
+#define RMI_SUBBLOCK_STATUS2__PRT_FIFO_0_NUM_USED__SHIFT                                                      0x0
+#define RMI_SUBBLOCK_STATUS2__PRT_FIFO_1_NUM_USED__SHIFT                                                      0x9
+#define RMI_SUBBLOCK_STATUS2__PRT_FIFO_0_NUM_USED_MASK                                                        0x000001FFL
+#define RMI_SUBBLOCK_STATUS2__PRT_FIFO_1_NUM_USED_MASK                                                        0x0003FE00L
+//RMI_SUBBLOCK_STATUS3
+#define RMI_SUBBLOCK_STATUS3__SKID_FIFO_0_FREE_SPACE_TOTAL__SHIFT                                             0x0
+#define RMI_SUBBLOCK_STATUS3__SKID_FIFO_1_FREE_SPACE_TOTAL__SHIFT                                             0xa
+#define RMI_SUBBLOCK_STATUS3__SKID_FIFO_0_FREE_SPACE_TOTAL_MASK                                               0x000003FFL
+#define RMI_SUBBLOCK_STATUS3__SKID_FIFO_1_FREE_SPACE_TOTAL_MASK                                               0x000FFC00L
+//RMI_XBAR_CONFIG
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_OVERRIDE__SHIFT                                                      0x0
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_REQ_TYPE_OVERRIDE__SHIFT                                             0x2
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_CB_DB_OVERRIDE__SHIFT                                                0x6
+#define RMI_XBAR_CONFIG__ARBITER_DIS__SHIFT                                                                   0x7
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_REQ__SHIFT                                                                0x8
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_REQ_OVERRIDE__SHIFT                                                       0xc
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_RB0__SHIFT                                                                0xd
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_OVERRIDE_MASK                                                        0x00000003L
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_REQ_TYPE_OVERRIDE_MASK                                               0x0000003CL
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_CB_DB_OVERRIDE_MASK                                                  0x00000040L
+#define RMI_XBAR_CONFIG__ARBITER_DIS_MASK                                                                     0x00000080L
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_REQ_MASK                                                                  0x00000F00L
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_REQ_OVERRIDE_MASK                                                         0x00001000L
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_RB0_MASK                                                                  0x00002000L
+//RMI_PROBE_POP_LOGIC_CNTL
+#define RMI_PROBE_POP_LOGIC_CNTL__EXT_LAT_FIFO_0_MAX_DEPTH__SHIFT                                             0x0
+#define RMI_PROBE_POP_LOGIC_CNTL__XLAT_COMBINE0_DIS__SHIFT                                                    0x7
+#define RMI_PROBE_POP_LOGIC_CNTL__REDUCE_MAX_XLAT_CHAIN_SIZE_BY_2__SHIFT                                      0x8
+#define RMI_PROBE_POP_LOGIC_CNTL__EXT_LAT_FIFO_1_MAX_DEPTH__SHIFT                                             0xa
+#define RMI_PROBE_POP_LOGIC_CNTL__XLAT_COMBINE1_DIS__SHIFT                                                    0x11
+#define RMI_PROBE_POP_LOGIC_CNTL__EXT_LAT_FIFO_0_MAX_DEPTH_MASK                                               0x0000007FL
+#define RMI_PROBE_POP_LOGIC_CNTL__XLAT_COMBINE0_DIS_MASK                                                      0x00000080L
+#define RMI_PROBE_POP_LOGIC_CNTL__REDUCE_MAX_XLAT_CHAIN_SIZE_BY_2_MASK                                        0x00000300L
+#define RMI_PROBE_POP_LOGIC_CNTL__EXT_LAT_FIFO_1_MAX_DEPTH_MASK                                               0x0001FC00L
+#define RMI_PROBE_POP_LOGIC_CNTL__XLAT_COMBINE1_DIS_MASK                                                      0x00020000L
+//RMI_UTC_XNACK_N_MISC_CNTL
+#define RMI_UTC_XNACK_N_MISC_CNTL__MASTER_XNACK_TIMER_INC__SHIFT                                              0x0
+#define RMI_UTC_XNACK_N_MISC_CNTL__IND_XNACK_TIMER_START_VALUE__SHIFT                                         0x8
+#define RMI_UTC_XNACK_N_MISC_CNTL__UTCL1_PERM_MODE__SHIFT                                                     0xc
+#define RMI_UTC_XNACK_N_MISC_CNTL__CP_VMID_RESET_REQUEST_DISABLE__SHIFT                                       0xd
+#define RMI_UTC_XNACK_N_MISC_CNTL__MASTER_XNACK_TIMER_INC_MASK                                                0x000000FFL
+#define RMI_UTC_XNACK_N_MISC_CNTL__IND_XNACK_TIMER_START_VALUE_MASK                                           0x00000F00L
+#define RMI_UTC_XNACK_N_MISC_CNTL__UTCL1_PERM_MODE_MASK                                                       0x00001000L
+#define RMI_UTC_XNACK_N_MISC_CNTL__CP_VMID_RESET_REQUEST_DISABLE_MASK                                         0x00002000L
+//RMI_DEMUX_CNTL
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_MODE_OVERRIDE_EN__SHIFT                                                    0x2
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_STALL_TIMER_START_VALUE__SHIFT                                             0x6
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_MODE__SHIFT                                                                0xe
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_MODE_OVERRIDE_EN__SHIFT                                                    0x12
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_STALL_TIMER_START_VALUE__SHIFT                                             0x16
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_MODE__SHIFT                                                                0x1e
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_MODE_OVERRIDE_EN_MASK                                                      0x00000004L
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_STALL_TIMER_START_VALUE_MASK                                               0x00003FC0L
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_MODE_MASK                                                                  0x0000C000L
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_MODE_OVERRIDE_EN_MASK                                                      0x00040000L
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_STALL_TIMER_START_VALUE_MASK                                               0x3FC00000L
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_MODE_MASK                                                                  0xC0000000L
+//RMI_UTCL1_CNTL1
+#define RMI_UTCL1_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                              0x0
+#define RMI_UTCL1_CNTL1__GPUVM_64K_DEF__SHIFT                                                                 0x1
+#define RMI_UTCL1_CNTL1__GPUVM_PERM_MODE__SHIFT                                                               0x2
+#define RMI_UTCL1_CNTL1__RESP_MODE__SHIFT                                                                     0x3
+#define RMI_UTCL1_CNTL1__RESP_FAULT_MODE__SHIFT                                                               0x5
+#define RMI_UTCL1_CNTL1__CLIENTID__SHIFT                                                                      0x7
+#define RMI_UTCL1_CNTL1__USERVM_DIS__SHIFT                                                                    0x10
+#define RMI_UTCL1_CNTL1__ENABLE_PUSH_LFIFO__SHIFT                                                             0x11
+#define RMI_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB__SHIFT                                                          0x12
+#define RMI_UTCL1_CNTL1__REG_INV_VMID__SHIFT                                                                  0x13
+#define RMI_UTCL1_CNTL1__REG_INV_ALL_VMID__SHIFT                                                              0x17
+#define RMI_UTCL1_CNTL1__REG_INV_TOGGLE__SHIFT                                                                0x18
+#define RMI_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID__SHIFT                                                    0x19
+#define RMI_UTCL1_CNTL1__FORCE_MISS__SHIFT                                                                    0x1a
+#define RMI_UTCL1_CNTL1__FORCE_IN_ORDER__SHIFT                                                                0x1b
+#define RMI_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                        0x1c
+#define RMI_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                        0x1e
+#define RMI_UTCL1_CNTL1__FORCE_4K_L2_RESP_MASK                                                                0x00000001L
+#define RMI_UTCL1_CNTL1__GPUVM_64K_DEF_MASK                                                                   0x00000002L
+#define RMI_UTCL1_CNTL1__GPUVM_PERM_MODE_MASK                                                                 0x00000004L
+#define RMI_UTCL1_CNTL1__RESP_MODE_MASK                                                                       0x00000018L
+#define RMI_UTCL1_CNTL1__RESP_FAULT_MODE_MASK                                                                 0x00000060L
+#define RMI_UTCL1_CNTL1__CLIENTID_MASK                                                                        0x0000FF80L
+#define RMI_UTCL1_CNTL1__USERVM_DIS_MASK                                                                      0x00010000L
+#define RMI_UTCL1_CNTL1__ENABLE_PUSH_LFIFO_MASK                                                               0x00020000L
+#define RMI_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB_MASK                                                            0x00040000L
+#define RMI_UTCL1_CNTL1__REG_INV_VMID_MASK                                                                    0x00780000L
+#define RMI_UTCL1_CNTL1__REG_INV_ALL_VMID_MASK                                                                0x00800000L
+#define RMI_UTCL1_CNTL1__REG_INV_TOGGLE_MASK                                                                  0x01000000L
+#define RMI_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID_MASK                                                      0x02000000L
+#define RMI_UTCL1_CNTL1__FORCE_MISS_MASK                                                                      0x04000000L
+#define RMI_UTCL1_CNTL1__FORCE_IN_ORDER_MASK                                                                  0x08000000L
+#define RMI_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                          0x30000000L
+#define RMI_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                          0xC0000000L
+//RMI_UTCL1_CNTL2
+#define RMI_UTCL1_CNTL2__UTC_SPARE__SHIFT                                                                     0x0
+#define RMI_UTCL1_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                                0x9
+#define RMI_UTCL1_CNTL2__LINE_VALID__SHIFT                                                                    0xa
+#define RMI_UTCL1_CNTL2__DIS_EDC__SHIFT                                                                       0xb
+#define RMI_UTCL1_CNTL2__GPUVM_INV_MODE__SHIFT                                                                0xc
+#define RMI_UTCL1_CNTL2__SHOOTDOWN_OPT__SHIFT                                                                 0xd
+#define RMI_UTCL1_CNTL2__FORCE_SNOOP__SHIFT                                                                   0xe
+#define RMI_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK__SHIFT                                                           0xf
+#define RMI_UTCL1_CNTL2__UTCL1_ARB_BURST_MODE__SHIFT                                                          0x10
+#define RMI_UTCL1_CNTL2__UTCL1_ENABLE_PERF_EVENT_RD_WR__SHIFT                                                 0x12
+#define RMI_UTCL1_CNTL2__UTCL1_PERF_EVENT_RD_WR__SHIFT                                                        0x13
+#define RMI_UTCL1_CNTL2__UTCL1_ENABLE_PERF_EVENT_VMID__SHIFT                                                  0x14
+#define RMI_UTCL1_CNTL2__UTCL1_PERF_EVENT_VMID__SHIFT                                                         0x15
+#define RMI_UTCL1_CNTL2__UTCL1_DIS_DUAL_L2_REQ__SHIFT                                                         0x19
+#define RMI_UTCL1_CNTL2__UTCL1_FORCE_FRAG_2M_TO_64K__SHIFT                                                    0x1a
+#define RMI_UTCL1_CNTL2__PERM_MODE_OVRD__SHIFT                                                                0x1b
+#define RMI_UTCL1_CNTL2__LINE_INVALIDATE_OPT__SHIFT                                                           0x1c
+#define RMI_UTCL1_CNTL2__GPUVM_16K_DEFAULT__SHIFT                                                             0x1d
+#define RMI_UTCL1_CNTL2__FGCG_DISABLE__SHIFT                                                                  0x1e
+#define RMI_UTCL1_CNTL2__RESERVED__SHIFT                                                                      0x1f
+#define RMI_UTCL1_CNTL2__UTC_SPARE_MASK                                                                       0x000000FFL
+#define RMI_UTCL1_CNTL2__MTYPE_OVRD_DIS_MASK                                                                  0x00000200L
+#define RMI_UTCL1_CNTL2__LINE_VALID_MASK                                                                      0x00000400L
+#define RMI_UTCL1_CNTL2__DIS_EDC_MASK                                                                         0x00000800L
+#define RMI_UTCL1_CNTL2__GPUVM_INV_MODE_MASK                                                                  0x00001000L
+#define RMI_UTCL1_CNTL2__SHOOTDOWN_OPT_MASK                                                                   0x00002000L
+#define RMI_UTCL1_CNTL2__FORCE_SNOOP_MASK                                                                     0x00004000L
+#define RMI_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK_MASK                                                             0x00008000L
+#define RMI_UTCL1_CNTL2__UTCL1_ARB_BURST_MODE_MASK                                                            0x00030000L
+#define RMI_UTCL1_CNTL2__UTCL1_ENABLE_PERF_EVENT_RD_WR_MASK                                                   0x00040000L
+#define RMI_UTCL1_CNTL2__UTCL1_PERF_EVENT_RD_WR_MASK                                                          0x00080000L
+#define RMI_UTCL1_CNTL2__UTCL1_ENABLE_PERF_EVENT_VMID_MASK                                                    0x00100000L
+#define RMI_UTCL1_CNTL2__UTCL1_PERF_EVENT_VMID_MASK                                                           0x01E00000L
+#define RMI_UTCL1_CNTL2__UTCL1_DIS_DUAL_L2_REQ_MASK                                                           0x02000000L
+#define RMI_UTCL1_CNTL2__UTCL1_FORCE_FRAG_2M_TO_64K_MASK                                                      0x04000000L
+#define RMI_UTCL1_CNTL2__PERM_MODE_OVRD_MASK                                                                  0x08000000L
+#define RMI_UTCL1_CNTL2__LINE_INVALIDATE_OPT_MASK                                                             0x10000000L
+#define RMI_UTCL1_CNTL2__GPUVM_16K_DEFAULT_MASK                                                               0x20000000L
+#define RMI_UTCL1_CNTL2__FGCG_DISABLE_MASK                                                                    0x40000000L
+#define RMI_UTCL1_CNTL2__RESERVED_MASK                                                                        0x80000000L
+//RMI_UTC_UNIT_CONFIG
+#define RMI_UTC_UNIT_CONFIG__TMZ_REQ_EN__SHIFT                                                                0x0
+#define RMI_UTC_UNIT_CONFIG__TMZ_REQ_EN_MASK                                                                  0x0000FFFFL
+//RMI_TCIW_FORMATTER0_CNTL
+#define RMI_TCIW_FORMATTER0_CNTL__TCIW0_MAX_ALLOWED_INFLIGHT_REQ__SHIFT                                       0x9
+#define RMI_TCIW_FORMATTER0_CNTL__RMI_IN0_REORDER_DIS__SHIFT                                                  0x1d
+#define RMI_TCIW_FORMATTER0_CNTL__ALL_FAULT_RET0_DATA__SHIFT                                                  0x1f
+#define RMI_TCIW_FORMATTER0_CNTL__TCIW0_MAX_ALLOWED_INFLIGHT_REQ_MASK                                         0x0007FE00L
+#define RMI_TCIW_FORMATTER0_CNTL__RMI_IN0_REORDER_DIS_MASK                                                    0x20000000L
+#define RMI_TCIW_FORMATTER0_CNTL__ALL_FAULT_RET0_DATA_MASK                                                    0x80000000L
+//RMI_TCIW_FORMATTER1_CNTL
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_DIS_OVERRIDE__SHIFT                                             0x0
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_TIME_OUT_WINDOW__SHIFT                                          0x1
+#define RMI_TCIW_FORMATTER1_CNTL__TCIW1_MAX_ALLOWED_INFLIGHT_REQ__SHIFT                                       0x9
+#define RMI_TCIW_FORMATTER1_CNTL__RMI_IN1_REORDER_DIS__SHIFT                                                  0x1d
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_DIS_AT_LAST_OF_BURST__SHIFT                                     0x1e
+#define RMI_TCIW_FORMATTER1_CNTL__ALL_FAULT_RET1_DATA__SHIFT                                                  0x1f
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_DIS_OVERRIDE_MASK                                               0x00000001L
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_TIME_OUT_WINDOW_MASK                                            0x000001FEL
+#define RMI_TCIW_FORMATTER1_CNTL__TCIW1_MAX_ALLOWED_INFLIGHT_REQ_MASK                                         0x0007FE00L
+#define RMI_TCIW_FORMATTER1_CNTL__RMI_IN1_REORDER_DIS_MASK                                                    0x20000000L
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_DIS_AT_LAST_OF_BURST_MASK                                       0x40000000L
+#define RMI_TCIW_FORMATTER1_CNTL__ALL_FAULT_RET1_DATA_MASK                                                    0x80000000L
+//RMI_SCOREBOARD_CNTL
+#define RMI_SCOREBOARD_CNTL__COMPLETE_RB0_FLUSH__SHIFT                                                        0x0
+#define RMI_SCOREBOARD_CNTL__REQ_IN_RE_EN_AFTER_FLUSH_RB0__SHIFT                                              0x1
+#define RMI_SCOREBOARD_CNTL__COMPLETE_RB1_FLUSH__SHIFT                                                        0x2
+#define RMI_SCOREBOARD_CNTL__REQ_IN_RE_EN_AFTER_FLUSH_RB1__SHIFT                                              0x3
+#define RMI_SCOREBOARD_CNTL__VMID_INVAL_FLUSH_TYPE_OVERRIDE_EN__SHIFT                                         0x5
+#define RMI_SCOREBOARD_CNTL__VMID_INVAL_FLUSH_TYPE_OVERRIDE_VALUE__SHIFT                                      0x6
+#define RMI_SCOREBOARD_CNTL__FORCE_VMID_INVAL_DONE_TIMER_START_VALUE__SHIFT                                   0x9
+#define RMI_SCOREBOARD_CNTL__COMPLETE_RB0_FLUSH_MASK                                                          0x00000001L
+#define RMI_SCOREBOARD_CNTL__REQ_IN_RE_EN_AFTER_FLUSH_RB0_MASK                                                0x00000002L
+#define RMI_SCOREBOARD_CNTL__COMPLETE_RB1_FLUSH_MASK                                                          0x00000004L
+#define RMI_SCOREBOARD_CNTL__REQ_IN_RE_EN_AFTER_FLUSH_RB1_MASK                                                0x00000008L
+#define RMI_SCOREBOARD_CNTL__VMID_INVAL_FLUSH_TYPE_OVERRIDE_EN_MASK                                           0x00000020L
+#define RMI_SCOREBOARD_CNTL__VMID_INVAL_FLUSH_TYPE_OVERRIDE_VALUE_MASK                                        0x00000040L
+#define RMI_SCOREBOARD_CNTL__FORCE_VMID_INVAL_DONE_TIMER_START_VALUE_MASK                                     0x001FFE00L
+//RMI_SCOREBOARD_STATUS0
+#define RMI_SCOREBOARD_STATUS0__CURRENT_SESSION_ID__SHIFT                                                     0x0
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_IN_PROG__SHIFT                                                    0x1
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_REQ_VMID__SHIFT                                                   0x2
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_UTC_DONE__SHIFT                                                   0x12
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_DONE__SHIFT                                                       0x13
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_FLUSH_TYPE__SHIFT                                                 0x14
+#define RMI_SCOREBOARD_STATUS0__FORCE_VMID_INV_DONE__SHIFT                                                    0x15
+#define RMI_SCOREBOARD_STATUS0__COUNTER_SELECT__SHIFT                                                         0x16
+#define RMI_SCOREBOARD_STATUS0__CURRENT_SESSION_ID_MASK                                                       0x00000001L
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_IN_PROG_MASK                                                      0x00000002L
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_REQ_VMID_MASK                                                     0x0003FFFCL
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_UTC_DONE_MASK                                                     0x00040000L
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_DONE_MASK                                                         0x00080000L
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_FLUSH_TYPE_MASK                                                   0x00100000L
+#define RMI_SCOREBOARD_STATUS0__FORCE_VMID_INV_DONE_MASK                                                      0x00200000L
+#define RMI_SCOREBOARD_STATUS0__COUNTER_SELECT_MASK                                                           0x07C00000L
+//RMI_SCOREBOARD_STATUS1
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_RB0__SHIFT                                                        0x0
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_UNDERFLOW_RB0__SHIFT                                              0xc
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_OVERFLOW_RB0__SHIFT                                               0xd
+#define RMI_SCOREBOARD_STATUS1__MULTI_VMID_INVAL_FROM_CP_DETECTED__SHIFT                                      0xe
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_RB1__SHIFT                                                        0xf
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_UNDERFLOW_RB1__SHIFT                                              0x1b
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_OVERFLOW_RB1__SHIFT                                               0x1c
+#define RMI_SCOREBOARD_STATUS1__COM_FLUSH_IN_PROG_RB1__SHIFT                                                  0x1d
+#define RMI_SCOREBOARD_STATUS1__COM_FLUSH_IN_PROG_RB0__SHIFT                                                  0x1e
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_RB0_MASK                                                          0x00000FFFL
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_UNDERFLOW_RB0_MASK                                                0x00001000L
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_OVERFLOW_RB0_MASK                                                 0x00002000L
+#define RMI_SCOREBOARD_STATUS1__MULTI_VMID_INVAL_FROM_CP_DETECTED_MASK                                        0x00004000L
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_RB1_MASK                                                          0x07FF8000L
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_UNDERFLOW_RB1_MASK                                                0x08000000L
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_OVERFLOW_RB1_MASK                                                 0x10000000L
+#define RMI_SCOREBOARD_STATUS1__COM_FLUSH_IN_PROG_RB1_MASK                                                    0x20000000L
+#define RMI_SCOREBOARD_STATUS1__COM_FLUSH_IN_PROG_RB0_MASK                                                    0x40000000L
+//RMI_SCOREBOARD_STATUS2
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_RB0__SHIFT                                                       0x0
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_UNDERFLOW_RB0__SHIFT                                             0xc
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_RB1__SHIFT                                                       0xd
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_UNDERFLOW_RB1__SHIFT                                             0x19
+#define RMI_SCOREBOARD_STATUS2__COM_FLUSH_DONE_RB1__SHIFT                                                     0x1a
+#define RMI_SCOREBOARD_STATUS2__COM_FLUSH_DONE_RB0__SHIFT                                                     0x1b
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_IN_PROG_RB0__SHIFT                                           0x1c
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_IN_PROG_RB1__SHIFT                                           0x1d
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_DONE_RB0__SHIFT                                              0x1e
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_DONE_RB1__SHIFT                                              0x1f
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_RB0_MASK                                                         0x00000FFFL
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_UNDERFLOW_RB0_MASK                                               0x00001000L
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_RB1_MASK                                                         0x01FFE000L
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_UNDERFLOW_RB1_MASK                                               0x02000000L
+#define RMI_SCOREBOARD_STATUS2__COM_FLUSH_DONE_RB1_MASK                                                       0x04000000L
+#define RMI_SCOREBOARD_STATUS2__COM_FLUSH_DONE_RB0_MASK                                                       0x08000000L
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_IN_PROG_RB0_MASK                                             0x10000000L
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_IN_PROG_RB1_MASK                                             0x20000000L
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_DONE_RB0_MASK                                                0x40000000L
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_DONE_RB1_MASK                                                0x80000000L
+//RMI_XBAR_ARBITER_CONFIG
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_MODE__SHIFT                                                        0x0
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_BREAK_LOB_ON_WEIGHTEDRR__SHIFT                                     0x2
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL__SHIFT                                                       0x3
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_BREAK_LOB_ON_IDLEIN__SHIFT                                         0x4
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_MODE_OVERRIDE_EN__SHIFT                                            0x5
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_TIMER_OVERRIDE__SHIFT                                        0x6
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_TIMER_START_VALUE__SHIFT                                     0x8
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_MODE__SHIFT                                                        0x10
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_BREAK_LOB_ON_WEIGHTEDRR__SHIFT                                     0x12
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL__SHIFT                                                       0x13
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_BREAK_LOB_ON_IDLEIN__SHIFT                                         0x14
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_MODE_OVERRIDE_EN__SHIFT                                            0x15
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_TIMER_OVERRIDE__SHIFT                                        0x16
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_TIMER_START_VALUE__SHIFT                                     0x18
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_MODE_MASK                                                          0x00000003L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_BREAK_LOB_ON_WEIGHTEDRR_MASK                                       0x00000004L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_MASK                                                         0x00000008L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_BREAK_LOB_ON_IDLEIN_MASK                                           0x00000010L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_MODE_OVERRIDE_EN_MASK                                              0x00000020L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_TIMER_OVERRIDE_MASK                                          0x000000C0L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_TIMER_START_VALUE_MASK                                       0x0000FF00L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_MODE_MASK                                                          0x00030000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_BREAK_LOB_ON_WEIGHTEDRR_MASK                                       0x00040000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_MASK                                                         0x00080000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_BREAK_LOB_ON_IDLEIN_MASK                                           0x00100000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_MODE_OVERRIDE_EN_MASK                                              0x00200000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_TIMER_OVERRIDE_MASK                                          0x00C00000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_TIMER_START_VALUE_MASK                                       0xFF000000L
+//RMI_XBAR_ARBITER_CONFIG_1
+#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB0_RD__SHIFT                                  0x0
+#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB0_WR__SHIFT                                  0x8
+#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB0_RD_MASK                                    0x000000FFL
+#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB0_WR_MASK                                    0x0000FF00L
+//RMI_CLOCK_CNTRL
+#define RMI_CLOCK_CNTRL__DYN_CLK_RB0_BUSY_MASK__SHIFT                                                         0x0
+#define RMI_CLOCK_CNTRL__DYN_CLK_CMN_BUSY_MASK__SHIFT                                                         0x5
+#define RMI_CLOCK_CNTRL__DYN_CLK_RB0_WAKEUP_MASK__SHIFT                                                       0xa
+#define RMI_CLOCK_CNTRL__DYN_CLK_CMN_WAKEUP_MASK__SHIFT                                                       0xf
+#define RMI_CLOCK_CNTRL__DYN_CLK_RB0_BUSY_MASK_MASK                                                           0x0000001FL
+#define RMI_CLOCK_CNTRL__DYN_CLK_CMN_BUSY_MASK_MASK                                                           0x000003E0L
+#define RMI_CLOCK_CNTRL__DYN_CLK_RB0_WAKEUP_MASK_MASK                                                         0x00007C00L
+#define RMI_CLOCK_CNTRL__DYN_CLK_CMN_WAKEUP_MASK_MASK                                                         0x000F8000L
+//RMI_UTCL1_STATUS
+#define RMI_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define RMI_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define RMI_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define RMI_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define RMI_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define RMI_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+//RMI_RB_GLX_CID_MAP
+#define RMI_RB_GLX_CID_MAP__CB_COLOR_MAP__SHIFT                                                               0x0
+#define RMI_RB_GLX_CID_MAP__CB_FMASK_MAP__SHIFT                                                               0x4
+#define RMI_RB_GLX_CID_MAP__CB_CMASK_MAP__SHIFT                                                               0x8
+#define RMI_RB_GLX_CID_MAP__CB_DCC_MAP__SHIFT                                                                 0xc
+#define RMI_RB_GLX_CID_MAP__DB_Z_MAP__SHIFT                                                                   0x10
+#define RMI_RB_GLX_CID_MAP__DB_S_MAP__SHIFT                                                                   0x14
+#define RMI_RB_GLX_CID_MAP__DB_TILE_MAP__SHIFT                                                                0x18
+#define RMI_RB_GLX_CID_MAP__DB_ZPCPSD_MAP__SHIFT                                                              0x1c
+#define RMI_RB_GLX_CID_MAP__CB_COLOR_MAP_MASK                                                                 0x0000000FL
+#define RMI_RB_GLX_CID_MAP__CB_FMASK_MAP_MASK                                                                 0x000000F0L
+#define RMI_RB_GLX_CID_MAP__CB_CMASK_MAP_MASK                                                                 0x00000F00L
+#define RMI_RB_GLX_CID_MAP__CB_DCC_MAP_MASK                                                                   0x0000F000L
+#define RMI_RB_GLX_CID_MAP__DB_Z_MAP_MASK                                                                     0x000F0000L
+#define RMI_RB_GLX_CID_MAP__DB_S_MAP_MASK                                                                     0x00F00000L
+#define RMI_RB_GLX_CID_MAP__DB_TILE_MAP_MASK                                                                  0x0F000000L
+#define RMI_RB_GLX_CID_MAP__DB_ZPCPSD_MAP_MASK                                                                0xF0000000L
+//RMI_XNACK_DEBUG
+#define RMI_XNACK_DEBUG__XNACK_PER_VMID__SHIFT                                                                0x0
+#define RMI_XNACK_DEBUG__XNACK_PER_VMID_MASK                                                                  0x0000FFFFL
+//RMI_SPARE
+#define RMI_SPARE__RMI_2_GL1_128B_READ_DISABLE__SHIFT                                                         0x1
+#define RMI_SPARE__RMI_2_GL1_REPEATER_FGCG_DISABLE__SHIFT                                                     0x2
+#define RMI_SPARE__RMI_2_RB_REPEATER_FGCG_DISABLE__SHIFT                                                      0x3
+#define RMI_SPARE__EARLY_WRITE_ACK_ENABLE_C_RW_NOA_RESOLVE_DIS__SHIFT                                         0x4
+#define RMI_SPARE__RMI_REORDER_BYPASS_CHANNEL_DIS__SHIFT                                                      0x5
+#define RMI_SPARE__XNACK_RETURN_DATA_OVERRIDE__SHIFT                                                          0x6
+#define RMI_SPARE__SPARE_BIT_7__SHIFT                                                                         0x7
+#define RMI_SPARE__NOFILL_RMI_CID_CC__SHIFT                                                                   0x8
+#define RMI_SPARE__NOFILL_RMI_CID_FC__SHIFT                                                                   0x9
+#define RMI_SPARE__NOFILL_RMI_CID_CM__SHIFT                                                                   0xa
+#define RMI_SPARE__NOFILL_RMI_CID_DC__SHIFT                                                                   0xb
+#define RMI_SPARE__NOFILL_RMI_CID_Z__SHIFT                                                                    0xc
+#define RMI_SPARE__NOFILL_RMI_CID_S__SHIFT                                                                    0xd
+#define RMI_SPARE__NOFILL_RMI_CID_TILE__SHIFT                                                                 0xe
+#define RMI_SPARE__SPARE_BIT_15_0__SHIFT                                                                      0xf
+#define RMI_SPARE__ARBITER_ADDRESS_MASK__SHIFT                                                                0x10
+#define RMI_SPARE__RMI_2_GL1_128B_READ_DISABLE_MASK                                                           0x00000002L
+#define RMI_SPARE__RMI_2_GL1_REPEATER_FGCG_DISABLE_MASK                                                       0x00000004L
+#define RMI_SPARE__RMI_2_RB_REPEATER_FGCG_DISABLE_MASK                                                        0x00000008L
+#define RMI_SPARE__EARLY_WRITE_ACK_ENABLE_C_RW_NOA_RESOLVE_DIS_MASK                                           0x00000010L
+#define RMI_SPARE__RMI_REORDER_BYPASS_CHANNEL_DIS_MASK                                                        0x00000020L
+#define RMI_SPARE__XNACK_RETURN_DATA_OVERRIDE_MASK                                                            0x00000040L
+#define RMI_SPARE__SPARE_BIT_7_MASK                                                                           0x00000080L
+#define RMI_SPARE__NOFILL_RMI_CID_CC_MASK                                                                     0x00000100L
+#define RMI_SPARE__NOFILL_RMI_CID_FC_MASK                                                                     0x00000200L
+#define RMI_SPARE__NOFILL_RMI_CID_CM_MASK                                                                     0x00000400L
+#define RMI_SPARE__NOFILL_RMI_CID_DC_MASK                                                                     0x00000800L
+#define RMI_SPARE__NOFILL_RMI_CID_Z_MASK                                                                      0x00001000L
+#define RMI_SPARE__NOFILL_RMI_CID_S_MASK                                                                      0x00002000L
+#define RMI_SPARE__NOFILL_RMI_CID_TILE_MASK                                                                   0x00004000L
+#define RMI_SPARE__SPARE_BIT_15_0_MASK                                                                        0x00008000L
+#define RMI_SPARE__ARBITER_ADDRESS_MASK_MASK                                                                  0xFFFF0000L
+//RMI_SPARE_1
+#define RMI_SPARE_1__EARLY_WRACK_FIFO_DISABLE__SHIFT                                                          0x0
+#define RMI_SPARE_1__SPARE_BIT_9__SHIFT                                                                       0x1
+#define RMI_SPARE_1__SPARE_BIT_10__SHIFT                                                                      0x2
+#define RMI_SPARE_1__SPARE_BIT_11__SHIFT                                                                      0x3
+#define RMI_SPARE_1__SPARE_BIT_12__SHIFT                                                                      0x4
+#define RMI_SPARE_1__SPARE_BIT_13__SHIFT                                                                      0x5
+#define RMI_SPARE_1__SPARE_BIT_14__SHIFT                                                                      0x6
+#define RMI_SPARE_1__SPARE_BIT_15__SHIFT                                                                      0x7
+#define RMI_SPARE_1__RMI_REORDER_DIS_BY_CID__SHIFT                                                            0x8
+#define RMI_SPARE_1__SPARE_BIT_16_1__SHIFT                                                                    0x10
+#define RMI_SPARE_1__EARLY_WRACK_FIFO_DISABLE_MASK                                                            0x00000001L
+#define RMI_SPARE_1__SPARE_BIT_9_MASK                                                                         0x00000002L
+#define RMI_SPARE_1__SPARE_BIT_10_MASK                                                                        0x00000004L
+#define RMI_SPARE_1__SPARE_BIT_11_MASK                                                                        0x00000008L
+#define RMI_SPARE_1__SPARE_BIT_12_MASK                                                                        0x00000010L
+#define RMI_SPARE_1__SPARE_BIT_13_MASK                                                                        0x00000020L
+#define RMI_SPARE_1__SPARE_BIT_14_MASK                                                                        0x00000040L
+#define RMI_SPARE_1__SPARE_BIT_15_MASK                                                                        0x00000080L
+#define RMI_SPARE_1__RMI_REORDER_DIS_BY_CID_MASK                                                              0x0000FF00L
+#define RMI_SPARE_1__SPARE_BIT_16_1_MASK                                                                      0xFFFF0000L
+//RMI_SPARE_2
+#define RMI_SPARE_2__ERROR_ZERO_BYTE_MASK_CID__SHIFT                                                          0x0
+#define RMI_SPARE_2__SPARE_BIT_8_2__SHIFT                                                                     0x10
+#define RMI_SPARE_2__SPARE_BIT_8_3__SHIFT                                                                     0x18
+#define RMI_SPARE_2__ERROR_ZERO_BYTE_MASK_CID_MASK                                                            0x0000FFFFL
+#define RMI_SPARE_2__SPARE_BIT_8_2_MASK                                                                       0x00FF0000L
+#define RMI_SPARE_2__SPARE_BIT_8_3_MASK                                                                       0xFF000000L
+//CC_RMI_REDUNDANCY
+#define CC_RMI_REDUNDANCY__REPAIR_EN_IN_0__SHIFT                                                              0x1
+#define CC_RMI_REDUNDANCY__REPAIR_EN_IN_1__SHIFT                                                              0x2
+#define CC_RMI_REDUNDANCY__REPAIR_RMI_OVERRIDE__SHIFT                                                         0x3
+#define CC_RMI_REDUNDANCY__REPAIR_ID_SWAP__SHIFT                                                              0x4
+#define CC_RMI_REDUNDANCY__REPAIR_EN_IN_0_MASK                                                                0x00000002L
+#define CC_RMI_REDUNDANCY__REPAIR_EN_IN_1_MASK                                                                0x00000004L
+#define CC_RMI_REDUNDANCY__REPAIR_RMI_OVERRIDE_MASK                                                           0x00000008L
+#define CC_RMI_REDUNDANCY__REPAIR_ID_SWAP_MASK                                                                0x00000010L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_utcl1dec
+//UTCL1_CTRL_1
+#define UTCL1_CTRL_1__UTCL1_CACHE_CORE_BYPASS__SHIFT                                                          0x0
+#define UTCL1_CTRL_1__UTCL1_TCP_BYPASS__SHIFT                                                                 0x1
+#define UTCL1_CTRL_1__UTCL1_SQCI_BYPASS__SHIFT                                                                0x2
+#define UTCL1_CTRL_1__UTCL1_SQCD_BYPASS__SHIFT                                                                0x3
+#define UTCL1_CTRL_1__UTCL1_RMI_BYPASS__SHIFT                                                                 0x4
+#define UTCL1_CTRL_1__UTCL1_SQG_BYPASS__SHIFT                                                                 0x5
+#define UTCL1_CTRL_1__UTCL1_FORCE_RANGE_INV_TO_VMID__SHIFT                                                    0x6
+#define UTCL1_CTRL_1__RESERVED_0__SHIFT                                                                       0x7
+#define UTCL1_CTRL_1__UTCL1_FORCE_INV_ALL_DONE__SHIFT                                                         0x8
+#define UTCL1_CTRL_1__UTCL1_PAGE_SIZE_1__SHIFT                                                                0x9
+#define UTCL1_CTRL_1__UTCL1_PAGE_SIZE_2__SHIFT                                                                0xb
+#define UTCL1_CTRL_1__UTCL1_PAGE_SIZE_3__SHIFT                                                                0xd
+#define UTCL1_CTRL_1__UTCL1_PAGE_SIZE_4__SHIFT                                                                0xf
+#define UTCL1_CTRL_1__RESERVED_1__SHIFT                                                                       0x11
+#define UTCL1_CTRL_1__UTCL1_CACHE_CORE_BYPASS_MASK                                                            0x00000001L
+#define UTCL1_CTRL_1__UTCL1_TCP_BYPASS_MASK                                                                   0x00000002L
+#define UTCL1_CTRL_1__UTCL1_SQCI_BYPASS_MASK                                                                  0x00000004L
+#define UTCL1_CTRL_1__UTCL1_SQCD_BYPASS_MASK                                                                  0x00000008L
+#define UTCL1_CTRL_1__UTCL1_RMI_BYPASS_MASK                                                                   0x00000010L
+#define UTCL1_CTRL_1__UTCL1_SQG_BYPASS_MASK                                                                   0x00000020L
+#define UTCL1_CTRL_1__UTCL1_FORCE_RANGE_INV_TO_VMID_MASK                                                      0x00000040L
+#define UTCL1_CTRL_1__RESERVED_0_MASK                                                                         0x00000080L
+#define UTCL1_CTRL_1__UTCL1_FORCE_INV_ALL_DONE_MASK                                                           0x00000100L
+#define UTCL1_CTRL_1__UTCL1_PAGE_SIZE_1_MASK                                                                  0x00000600L
+#define UTCL1_CTRL_1__UTCL1_PAGE_SIZE_2_MASK                                                                  0x00001800L
+#define UTCL1_CTRL_1__UTCL1_PAGE_SIZE_3_MASK                                                                  0x00006000L
+#define UTCL1_CTRL_1__UTCL1_PAGE_SIZE_4_MASK                                                                  0x00018000L
+#define UTCL1_CTRL_1__RESERVED_1_MASK                                                                         0xFFFE0000L
+//UTCL1_HASH_CTRL
+#define UTCL1_HASH_CTRL__UTCL1_BANK_SELECT_BASE__SHIFT                                                        0x0
+#define UTCL1_HASH_CTRL__UTCL1_BANK_MASK0__SHIFT                                                              0x5
+#define UTCL1_HASH_CTRL__UTCL1_BANK_MASK1__SHIFT                                                              0x9
+#define UTCL1_HASH_CTRL__UTCL1_WAY_SEL_MASK0__SHIFT                                                           0xd
+#define UTCL1_HASH_CTRL__UTCL1_WAY_SEL_MASK1__SHIFT                                                           0x11
+#define UTCL1_HASH_CTRL__UTCL1_WAY_SEL_MASK2__SHIFT                                                           0x15
+#define UTCL1_HASH_CTRL__UTCL1_XOR_ONLY_HIGHER_WAYS__SHIFT                                                    0x19
+#define UTCL1_HASH_CTRL__UTCL1_WAY_SELECT_OFFSET__SHIFT                                                       0x1a
+#define UTCL1_HASH_CTRL__RESERVED__SHIFT                                                                      0x1f
+#define UTCL1_HASH_CTRL__UTCL1_BANK_SELECT_BASE_MASK                                                          0x0000001FL
+#define UTCL1_HASH_CTRL__UTCL1_BANK_MASK0_MASK                                                                0x000001E0L
+#define UTCL1_HASH_CTRL__UTCL1_BANK_MASK1_MASK                                                                0x00001E00L
+#define UTCL1_HASH_CTRL__UTCL1_WAY_SEL_MASK0_MASK                                                             0x0001E000L
+#define UTCL1_HASH_CTRL__UTCL1_WAY_SEL_MASK1_MASK                                                             0x001E0000L
+#define UTCL1_HASH_CTRL__UTCL1_WAY_SEL_MASK2_MASK                                                             0x01E00000L
+#define UTCL1_HASH_CTRL__UTCL1_XOR_ONLY_HIGHER_WAYS_MASK                                                      0x02000000L
+#define UTCL1_HASH_CTRL__UTCL1_WAY_SELECT_OFFSET_MASK                                                         0x7C000000L
+#define UTCL1_HASH_CTRL__RESERVED_MASK                                                                        0x80000000L
+//UTCL1_ALOG
+#define UTCL1_ALOG__UTCL1_ALOG_MODE1_FILTER1_THRESHOLD__SHIFT                                                 0x0
+#define UTCL1_ALOG__UTCL1_ALOG_MODE1_FILTER2_BYPASS__SHIFT                                                    0x3
+#define UTCL1_ALOG__UTCL1_ALOG_ACTIVE__SHIFT                                                                  0x4
+#define UTCL1_ALOG__UTCL1_ALOG_MODE__SHIFT                                                                    0x5
+#define UTCL1_ALOG__UTCL1_ALOG_MODE2_LOCK_WINDOW__SHIFT                                                       0x6
+#define UTCL1_ALOG__UTCL1_ALOG_ONLY_MISS__SHIFT                                                               0x9
+#define UTCL1_ALOG__UTCL1_ALOG_MODE2_INTR_THRESHOLD__SHIFT                                                    0xa
+#define UTCL1_ALOG__UTCL1_ALOG_SPACE_EN__SHIFT                                                                0xc
+#define UTCL1_ALOG__UTCL1_ALOG_CLEAN__SHIFT                                                                   0xf
+#define UTCL1_ALOG__UTCL1_ALOG_IDLE__SHIFT                                                                    0x10
+#define UTCL1_ALOG__UTCL1_ALOG_TRACK_SEGMENT_SIZE__SHIFT                                                      0x11
+#define UTCL1_ALOG__UTCL1_ALOG_MODE1_FILTER1_BYPASS__SHIFT                                                    0x17
+#define UTCL1_ALOG__UTCL1_ALOG_MODE1_INTR_ON_ALLOC__SHIFT                                                     0x18
+#define UTCL1_ALOG__UTCL1_ALOG_MODE1_FILTER1_THRESHOLD_MASK                                                   0x00000007L
+#define UTCL1_ALOG__UTCL1_ALOG_MODE1_FILTER2_BYPASS_MASK                                                      0x00000008L
+#define UTCL1_ALOG__UTCL1_ALOG_ACTIVE_MASK                                                                    0x00000010L
+#define UTCL1_ALOG__UTCL1_ALOG_MODE_MASK                                                                      0x00000020L
+#define UTCL1_ALOG__UTCL1_ALOG_MODE2_LOCK_WINDOW_MASK                                                         0x000001C0L
+#define UTCL1_ALOG__UTCL1_ALOG_ONLY_MISS_MASK                                                                 0x00000200L
+#define UTCL1_ALOG__UTCL1_ALOG_MODE2_INTR_THRESHOLD_MASK                                                      0x00000C00L
+#define UTCL1_ALOG__UTCL1_ALOG_SPACE_EN_MASK                                                                  0x00007000L
+#define UTCL1_ALOG__UTCL1_ALOG_CLEAN_MASK                                                                     0x00008000L
+#define UTCL1_ALOG__UTCL1_ALOG_IDLE_MASK                                                                      0x00010000L
+#define UTCL1_ALOG__UTCL1_ALOG_TRACK_SEGMENT_SIZE_MASK                                                        0x007E0000L
+#define UTCL1_ALOG__UTCL1_ALOG_MODE1_FILTER1_BYPASS_MASK                                                      0x00800000L
+#define UTCL1_ALOG__UTCL1_ALOG_MODE1_INTR_ON_ALLOC_MASK                                                       0x01000000L
+//UTCL1_STATUS
+#define UTCL1_STATUS__UTCL1_HIT_PATH_BUSY__SHIFT                                                              0x0
+#define UTCL1_STATUS__UTCL1_MH_BUSY__SHIFT                                                                    0x1
+#define UTCL1_STATUS__UTCL1_INV_BUSY__SHIFT                                                                   0x2
+#define UTCL1_STATUS__UTCL1_PENDING_UTCL2_REQ__SHIFT                                                          0x3
+#define UTCL1_STATUS__UTCL1_PENDING_UTCL2_RET__SHIFT                                                          0x4
+#define UTCL1_STATUS__UTCL1_LAST_UTCL2_RET_XNACK__SHIFT                                                       0x5
+#define UTCL1_STATUS__UTCL1_RANGE_INV_IN_PROGRESS__SHIFT                                                      0x7
+#define UTCL1_STATUS__RESERVED__SHIFT                                                                         0x8
+#define UTCL1_STATUS__UTCL1_HIT_PATH_BUSY_MASK                                                                0x00000001L
+#define UTCL1_STATUS__UTCL1_MH_BUSY_MASK                                                                      0x00000002L
+#define UTCL1_STATUS__UTCL1_INV_BUSY_MASK                                                                     0x00000004L
+#define UTCL1_STATUS__UTCL1_PENDING_UTCL2_REQ_MASK                                                            0x00000008L
+#define UTCL1_STATUS__UTCL1_PENDING_UTCL2_RET_MASK                                                            0x00000010L
+#define UTCL1_STATUS__UTCL1_LAST_UTCL2_RET_XNACK_MASK                                                         0x00000060L
+#define UTCL1_STATUS__UTCL1_RANGE_INV_IN_PROGRESS_MASK                                                        0x00000080L
+#define UTCL1_STATUS__RESERVED_MASK                                                                           0x00000100L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_shdec
+//SPI_SHADER_PGM_CHKSUM_PS
+#define SPI_SHADER_PGM_CHKSUM_PS__CHECKSUM__SHIFT                                                             0x0
+#define SPI_SHADER_PGM_CHKSUM_PS__CHECKSUM_MASK                                                               0xFFFFFFFFL
+//SPI_SHADER_PGM_RSRC3_PS
+#define SPI_SHADER_PGM_RSRC3_PS__CU_EN__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC3_PS__CU_EN_MASK                                                                   0x0000FFFFL
+//SPI_SHADER_PGM_RSRC4_PS
+#define SPI_SHADER_PGM_RSRC4_PS__WAVE_LIMIT__SHIFT                                                            0x0
+#define SPI_SHADER_PGM_RSRC4_PS__LDS_GROUP_SIZE__SHIFT                                                        0xa
+#define SPI_SHADER_PGM_RSRC4_PS__DIDT_THROTTLE__SHIFT                                                         0xc
+#define SPI_SHADER_PGM_RSRC4_PS__INST_PREF_SIZE__SHIFT                                                        0x10
+#define SPI_SHADER_PGM_RSRC4_PS__IMAGE_OP__SHIFT                                                              0x1f
+#define SPI_SHADER_PGM_RSRC4_PS__WAVE_LIMIT_MASK                                                              0x000003FFL
+#define SPI_SHADER_PGM_RSRC4_PS__LDS_GROUP_SIZE_MASK                                                          0x00000C00L
+#define SPI_SHADER_PGM_RSRC4_PS__DIDT_THROTTLE_MASK                                                           0x00001000L
+#define SPI_SHADER_PGM_RSRC4_PS__INST_PREF_SIZE_MASK                                                          0x00FF0000L
+#define SPI_SHADER_PGM_RSRC4_PS__IMAGE_OP_MASK                                                                0x80000000L
+//SPI_SHADER_PGM_LO_PS
+#define SPI_SHADER_PGM_LO_PS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_LO_PS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_PS
+#define SPI_SHADER_PGM_HI_PS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_HI_PS__MEM_BASE_MASK                                                                   0x000000FFL
+//SPI_SHADER_PGM_RSRC1_PS
+#define SPI_SHADER_PGM_RSRC1_PS__VGPRS__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC1_PS__SGPRS__SHIFT                                                                 0x6
+#define SPI_SHADER_PGM_RSRC1_PS__PRIORITY__SHIFT                                                              0xa
+#define SPI_SHADER_PGM_RSRC1_PS__FLOAT_MODE__SHIFT                                                            0xc
+#define SPI_SHADER_PGM_RSRC1_PS__PRIV__SHIFT                                                                  0x14
+#define SPI_SHADER_PGM_RSRC1_PS__WG_RR_EN__SHIFT                                                              0x15
+#define SPI_SHADER_PGM_RSRC1_PS__DEBUG_MODE__SHIFT                                                            0x16
+#define SPI_SHADER_PGM_RSRC1_PS__DISABLE_PERF__SHIFT                                                          0x17
+#define SPI_SHADER_PGM_RSRC1_PS__CU_GROUP_DISABLE__SHIFT                                                      0x18
+#define SPI_SHADER_PGM_RSRC1_PS__FWD_PROGRESS__SHIFT                                                          0x1a
+#define SPI_SHADER_PGM_RSRC1_PS__LOAD_PROVOKING_VTX__SHIFT                                                    0x1b
+#define SPI_SHADER_PGM_RSRC1_PS__CDBG_USER__SHIFT                                                             0x1c
+#define SPI_SHADER_PGM_RSRC1_PS__FP16_OVFL__SHIFT                                                             0x1d
+#define SPI_SHADER_PGM_RSRC1_PS__VGPRS_MASK                                                                   0x0000003FL
+#define SPI_SHADER_PGM_RSRC1_PS__SGPRS_MASK                                                                   0x000003C0L
+#define SPI_SHADER_PGM_RSRC1_PS__PRIORITY_MASK                                                                0x00000C00L
+#define SPI_SHADER_PGM_RSRC1_PS__FLOAT_MODE_MASK                                                              0x000FF000L
+#define SPI_SHADER_PGM_RSRC1_PS__PRIV_MASK                                                                    0x00100000L
+#define SPI_SHADER_PGM_RSRC1_PS__WG_RR_EN_MASK                                                                0x00200000L
+#define SPI_SHADER_PGM_RSRC1_PS__DEBUG_MODE_MASK                                                              0x00400000L
+#define SPI_SHADER_PGM_RSRC1_PS__DISABLE_PERF_MASK                                                            0x00800000L
+#define SPI_SHADER_PGM_RSRC1_PS__CU_GROUP_DISABLE_MASK                                                        0x01000000L
+#define SPI_SHADER_PGM_RSRC1_PS__FWD_PROGRESS_MASK                                                            0x04000000L
+#define SPI_SHADER_PGM_RSRC1_PS__LOAD_PROVOKING_VTX_MASK                                                      0x08000000L
+#define SPI_SHADER_PGM_RSRC1_PS__CDBG_USER_MASK                                                               0x10000000L
+#define SPI_SHADER_PGM_RSRC1_PS__FP16_OVFL_MASK                                                               0x20000000L
+//SPI_SHADER_PGM_RSRC2_PS
+#define SPI_SHADER_PGM_RSRC2_PS__SCRATCH_EN__SHIFT                                                            0x0
+#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR__SHIFT                                                             0x1
+#define SPI_SHADER_PGM_RSRC2_PS__TRAP_PRESENT__SHIFT                                                          0x6
+#define SPI_SHADER_PGM_RSRC2_PS__WAVE_CNT_EN__SHIFT                                                           0x7
+#define SPI_SHADER_PGM_RSRC2_PS__EXTRA_LDS_SIZE__SHIFT                                                        0x8
+#define SPI_SHADER_PGM_RSRC2_PS__EXCP_EN__SHIFT                                                               0x10
+#define SPI_SHADER_PGM_RSRC2_PS__LOAD_COLLISION_WAVEID__SHIFT                                                 0x19
+#define SPI_SHADER_PGM_RSRC2_PS__LOAD_INTRAWAVE_COLLISION__SHIFT                                              0x1a
+#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR_MSB__SHIFT                                                         0x1b
+#define SPI_SHADER_PGM_RSRC2_PS__SHARED_VGPR_CNT__SHIFT                                                       0x1c
+#define SPI_SHADER_PGM_RSRC2_PS__SCRATCH_EN_MASK                                                              0x00000001L
+#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR_MASK                                                               0x0000003EL
+#define SPI_SHADER_PGM_RSRC2_PS__TRAP_PRESENT_MASK                                                            0x00000040L
+#define SPI_SHADER_PGM_RSRC2_PS__WAVE_CNT_EN_MASK                                                             0x00000080L
+#define SPI_SHADER_PGM_RSRC2_PS__EXTRA_LDS_SIZE_MASK                                                          0x0000FF00L
+#define SPI_SHADER_PGM_RSRC2_PS__EXCP_EN_MASK                                                                 0x01FF0000L
+#define SPI_SHADER_PGM_RSRC2_PS__LOAD_COLLISION_WAVEID_MASK                                                   0x02000000L
+#define SPI_SHADER_PGM_RSRC2_PS__LOAD_INTRAWAVE_COLLISION_MASK                                                0x04000000L
+#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR_MSB_MASK                                                           0x08000000L
+#define SPI_SHADER_PGM_RSRC2_PS__SHARED_VGPR_CNT_MASK                                                         0xF0000000L
+//SPI_SHADER_USER_DATA_PS_0
+#define SPI_SHADER_USER_DATA_PS_0__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_0__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_1
+#define SPI_SHADER_USER_DATA_PS_1__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_1__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_2
+#define SPI_SHADER_USER_DATA_PS_2__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_2__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_3
+#define SPI_SHADER_USER_DATA_PS_3__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_3__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_4
+#define SPI_SHADER_USER_DATA_PS_4__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_4__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_5
+#define SPI_SHADER_USER_DATA_PS_5__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_5__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_6
+#define SPI_SHADER_USER_DATA_PS_6__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_6__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_7
+#define SPI_SHADER_USER_DATA_PS_7__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_7__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_8
+#define SPI_SHADER_USER_DATA_PS_8__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_8__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_9
+#define SPI_SHADER_USER_DATA_PS_9__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_9__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_10
+#define SPI_SHADER_USER_DATA_PS_10__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_10__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_11
+#define SPI_SHADER_USER_DATA_PS_11__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_11__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_12
+#define SPI_SHADER_USER_DATA_PS_12__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_12__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_13
+#define SPI_SHADER_USER_DATA_PS_13__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_13__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_14
+#define SPI_SHADER_USER_DATA_PS_14__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_14__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_15
+#define SPI_SHADER_USER_DATA_PS_15__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_15__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_16
+#define SPI_SHADER_USER_DATA_PS_16__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_16__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_17
+#define SPI_SHADER_USER_DATA_PS_17__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_17__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_18
+#define SPI_SHADER_USER_DATA_PS_18__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_18__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_19
+#define SPI_SHADER_USER_DATA_PS_19__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_19__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_20
+#define SPI_SHADER_USER_DATA_PS_20__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_20__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_21
+#define SPI_SHADER_USER_DATA_PS_21__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_21__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_22
+#define SPI_SHADER_USER_DATA_PS_22__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_22__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_23
+#define SPI_SHADER_USER_DATA_PS_23__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_23__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_24
+#define SPI_SHADER_USER_DATA_PS_24__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_24__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_25
+#define SPI_SHADER_USER_DATA_PS_25__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_25__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_26
+#define SPI_SHADER_USER_DATA_PS_26__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_26__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_27
+#define SPI_SHADER_USER_DATA_PS_27__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_27__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_28
+#define SPI_SHADER_USER_DATA_PS_28__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_28__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_29
+#define SPI_SHADER_USER_DATA_PS_29__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_29__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_30
+#define SPI_SHADER_USER_DATA_PS_30__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_30__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_31
+#define SPI_SHADER_USER_DATA_PS_31__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_31__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_REQ_CTRL_PS
+#define SPI_SHADER_REQ_CTRL_PS__SOFT_GROUPING_EN__SHIFT                                                       0x0
+#define SPI_SHADER_REQ_CTRL_PS__NUMBER_OF_REQUESTS_PER_CU__SHIFT                                              0x1
+#define SPI_SHADER_REQ_CTRL_PS__SOFT_GROUPING_ALLOCATION_TIMEOUT__SHIFT                                       0x5
+#define SPI_SHADER_REQ_CTRL_PS__HARD_LOCK_HYSTERESIS__SHIFT                                                   0x9
+#define SPI_SHADER_REQ_CTRL_PS__HARD_LOCK_LOW_THRESHOLD__SHIFT                                                0xa
+#define SPI_SHADER_REQ_CTRL_PS__PRODUCER_REQUEST_LOCKOUT__SHIFT                                               0xf
+#define SPI_SHADER_REQ_CTRL_PS__GLOBAL_SCANNING_EN__SHIFT                                                     0x10
+#define SPI_SHADER_REQ_CTRL_PS__ALLOCATION_RATE_THROTTLING_THRESHOLD__SHIFT                                   0x11
+#define SPI_SHADER_REQ_CTRL_PS__SOFT_GROUPING_EN_MASK                                                         0x00000001L
+#define SPI_SHADER_REQ_CTRL_PS__NUMBER_OF_REQUESTS_PER_CU_MASK                                                0x0000001EL
+#define SPI_SHADER_REQ_CTRL_PS__SOFT_GROUPING_ALLOCATION_TIMEOUT_MASK                                         0x000001E0L
+#define SPI_SHADER_REQ_CTRL_PS__HARD_LOCK_HYSTERESIS_MASK                                                     0x00000200L
+#define SPI_SHADER_REQ_CTRL_PS__HARD_LOCK_LOW_THRESHOLD_MASK                                                  0x00007C00L
+#define SPI_SHADER_REQ_CTRL_PS__PRODUCER_REQUEST_LOCKOUT_MASK                                                 0x00008000L
+#define SPI_SHADER_REQ_CTRL_PS__GLOBAL_SCANNING_EN_MASK                                                       0x00010000L
+#define SPI_SHADER_REQ_CTRL_PS__ALLOCATION_RATE_THROTTLING_THRESHOLD_MASK                                     0x000E0000L
+//SPI_SHADER_GS_OUT_CONFIG_PS
+#define SPI_SHADER_GS_OUT_CONFIG_PS__VS_EXPORT_COUNT__SHIFT                                                   0x0
+#define SPI_SHADER_GS_OUT_CONFIG_PS__PRIM_EXPORT_COUNT__SHIFT                                                 0x5
+#define SPI_SHADER_GS_OUT_CONFIG_PS__NO_PC_EXPORT__SHIFT                                                      0xa
+#define SPI_SHADER_GS_OUT_CONFIG_PS__NUM_INTERP__SHIFT                                                        0xb
+#define SPI_SHADER_GS_OUT_CONFIG_PS__NUM_PRIM_INTERP__SHIFT                                                   0x11
+#define SPI_SHADER_GS_OUT_CONFIG_PS__VS_EXPORT_COUNT_MASK                                                     0x0000001FL
+#define SPI_SHADER_GS_OUT_CONFIG_PS__PRIM_EXPORT_COUNT_MASK                                                   0x000003E0L
+#define SPI_SHADER_GS_OUT_CONFIG_PS__NO_PC_EXPORT_MASK                                                        0x00000400L
+#define SPI_SHADER_GS_OUT_CONFIG_PS__NUM_INTERP_MASK                                                          0x0001F800L
+#define SPI_SHADER_GS_OUT_CONFIG_PS__NUM_PRIM_INTERP_MASK                                                     0x003E0000L
+//SPI_SHADER_USER_ACCUM_PS_0
+#define SPI_SHADER_USER_ACCUM_PS_0__CONTRIBUTION__SHIFT                                                       0x0
+#define SPI_SHADER_USER_ACCUM_PS_0__CONTRIBUTION_MASK                                                         0x0000007FL
+//SPI_SHADER_USER_ACCUM_PS_1
+#define SPI_SHADER_USER_ACCUM_PS_1__CONTRIBUTION__SHIFT                                                       0x0
+#define SPI_SHADER_USER_ACCUM_PS_1__CONTRIBUTION_MASK                                                         0x0000007FL
+//SPI_SHADER_USER_ACCUM_PS_2
+#define SPI_SHADER_USER_ACCUM_PS_2__CONTRIBUTION__SHIFT                                                       0x0
+#define SPI_SHADER_USER_ACCUM_PS_2__CONTRIBUTION_MASK                                                         0x0000007FL
+//SPI_SHADER_USER_ACCUM_PS_3
+#define SPI_SHADER_USER_ACCUM_PS_3__CONTRIBUTION__SHIFT                                                       0x0
+#define SPI_SHADER_USER_ACCUM_PS_3__CONTRIBUTION_MASK                                                         0x0000007FL
+//SPI_SHADER_PGM_CHKSUM_GS
+#define SPI_SHADER_PGM_CHKSUM_GS__CHECKSUM__SHIFT                                                             0x0
+#define SPI_SHADER_PGM_CHKSUM_GS__CHECKSUM_MASK                                                               0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ADDR_LO_GS
+#define SPI_SHADER_USER_DATA_ADDR_LO_GS__MEM_BASE__SHIFT                                                      0x0
+#define SPI_SHADER_USER_DATA_ADDR_LO_GS__MEM_BASE_MASK                                                        0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ADDR_HI_GS
+#define SPI_SHADER_USER_DATA_ADDR_HI_GS__MEM_BASE__SHIFT                                                      0x0
+#define SPI_SHADER_USER_DATA_ADDR_HI_GS__MEM_BASE_MASK                                                        0xFFFFFFFFL
+//SPI_SHADER_PGM_LO_GS
+#define SPI_SHADER_PGM_LO_GS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_LO_GS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_GS
+#define SPI_SHADER_PGM_HI_GS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_HI_GS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_ES
+#define SPI_SHADER_PGM_HI_ES__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_HI_ES__MEM_BASE_MASK                                                                   0x000000FFL
+//SPI_SHADER_PGM_RSRC3_GS
+#define SPI_SHADER_PGM_RSRC3_GS__CU_EN__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC3_GS__CU_EN_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_RSRC4_GS
+#define SPI_SHADER_PGM_RSRC4_GS__WAVE_LIMIT__SHIFT                                                            0x0
+#define SPI_SHADER_PGM_RSRC4_GS__GLG_EN_OVERRIDE__SHIFT                                                       0xa
+#define SPI_SHADER_PGM_RSRC4_GS__GLG_FORCE_DISABLE__SHIFT                                                     0xb
+#define SPI_SHADER_PGM_RSRC4_GS__DIDT_THROTTLE__SHIFT                                                         0xc
+#define SPI_SHADER_PGM_RSRC4_GS__PH_THROTTLE_EN__SHIFT                                                        0xe
+#define SPI_SHADER_PGM_RSRC4_GS__SPI_THROTTLE_EN__SHIFT                                                       0xf
+#define SPI_SHADER_PGM_RSRC4_GS__SPI_SHADER_LATE_ALLOC_GS__SHIFT                                              0x10
+#define SPI_SHADER_PGM_RSRC4_GS__INST_PREF_SIZE__SHIFT                                                        0x17
+#define SPI_SHADER_PGM_RSRC4_GS__IMAGE_OP__SHIFT                                                              0x1f
+#define SPI_SHADER_PGM_RSRC4_GS__WAVE_LIMIT_MASK                                                              0x000003FFL
+#define SPI_SHADER_PGM_RSRC4_GS__GLG_EN_OVERRIDE_MASK                                                         0x00000400L
+#define SPI_SHADER_PGM_RSRC4_GS__GLG_FORCE_DISABLE_MASK                                                       0x00000800L
+#define SPI_SHADER_PGM_RSRC4_GS__DIDT_THROTTLE_MASK                                                           0x00001000L
+#define SPI_SHADER_PGM_RSRC4_GS__PH_THROTTLE_EN_MASK                                                          0x00004000L
+#define SPI_SHADER_PGM_RSRC4_GS__SPI_THROTTLE_EN_MASK                                                         0x00008000L
+#define SPI_SHADER_PGM_RSRC4_GS__SPI_SHADER_LATE_ALLOC_GS_MASK                                                0x007F0000L
+#define SPI_SHADER_PGM_RSRC4_GS__INST_PREF_SIZE_MASK                                                          0x7F800000L
+#define SPI_SHADER_PGM_RSRC4_GS__IMAGE_OP_MASK                                                                0x80000000L
+//SPI_SHADER_PGM_LO_ES
+#define SPI_SHADER_PGM_LO_ES__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_LO_ES__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_RSRC1_GS
+#define SPI_SHADER_PGM_RSRC1_GS__VGPRS__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC1_GS__SGPRS__SHIFT                                                                 0x6
+#define SPI_SHADER_PGM_RSRC1_GS__PRIORITY__SHIFT                                                              0xa
+#define SPI_SHADER_PGM_RSRC1_GS__FLOAT_MODE__SHIFT                                                            0xc
+#define SPI_SHADER_PGM_RSRC1_GS__PRIV__SHIFT                                                                  0x14
+#define SPI_SHADER_PGM_RSRC1_GS__WG_RR_EN__SHIFT                                                              0x15
+#define SPI_SHADER_PGM_RSRC1_GS__DEBUG_MODE__SHIFT                                                            0x16
+#define SPI_SHADER_PGM_RSRC1_GS__DISABLE_PERF__SHIFT                                                          0x17
+#define SPI_SHADER_PGM_RSRC1_GS__CU_GROUP_ENABLE__SHIFT                                                       0x18
+#define SPI_SHADER_PGM_RSRC1_GS__FWD_PROGRESS__SHIFT                                                          0x1a
+#define SPI_SHADER_PGM_RSRC1_GS__WGP_MODE__SHIFT                                                              0x1b
+#define SPI_SHADER_PGM_RSRC1_GS__CDBG_USER__SHIFT                                                             0x1c
+#define SPI_SHADER_PGM_RSRC1_GS__GS_VGPR_COMP_CNT__SHIFT                                                      0x1d
+#define SPI_SHADER_PGM_RSRC1_GS__FP16_OVFL__SHIFT                                                             0x1f
+#define SPI_SHADER_PGM_RSRC1_GS__VGPRS_MASK                                                                   0x0000003FL
+#define SPI_SHADER_PGM_RSRC1_GS__SGPRS_MASK                                                                   0x000003C0L
+#define SPI_SHADER_PGM_RSRC1_GS__PRIORITY_MASK                                                                0x00000C00L
+#define SPI_SHADER_PGM_RSRC1_GS__FLOAT_MODE_MASK                                                              0x000FF000L
+#define SPI_SHADER_PGM_RSRC1_GS__PRIV_MASK                                                                    0x00100000L
+#define SPI_SHADER_PGM_RSRC1_GS__WG_RR_EN_MASK                                                                0x00200000L
+#define SPI_SHADER_PGM_RSRC1_GS__DEBUG_MODE_MASK                                                              0x00400000L
+#define SPI_SHADER_PGM_RSRC1_GS__DISABLE_PERF_MASK                                                            0x00800000L
+#define SPI_SHADER_PGM_RSRC1_GS__CU_GROUP_ENABLE_MASK                                                         0x01000000L
+#define SPI_SHADER_PGM_RSRC1_GS__FWD_PROGRESS_MASK                                                            0x04000000L
+#define SPI_SHADER_PGM_RSRC1_GS__WGP_MODE_MASK                                                                0x08000000L
+#define SPI_SHADER_PGM_RSRC1_GS__CDBG_USER_MASK                                                               0x10000000L
+#define SPI_SHADER_PGM_RSRC1_GS__GS_VGPR_COMP_CNT_MASK                                                        0x60000000L
+#define SPI_SHADER_PGM_RSRC1_GS__FP16_OVFL_MASK                                                               0x80000000L
+//SPI_SHADER_PGM_RSRC2_GS
+#define SPI_SHADER_PGM_RSRC2_GS__SCRATCH_EN__SHIFT                                                            0x0
+#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR__SHIFT                                                             0x1
+#define SPI_SHADER_PGM_RSRC2_GS__TRAP_PRESENT__SHIFT                                                          0x6
+#define SPI_SHADER_PGM_RSRC2_GS__EXCP_EN__SHIFT                                                               0x7
+#define SPI_SHADER_PGM_RSRC2_GS__ES_VGPR_COMP_CNT__SHIFT                                                      0x10
+#define SPI_SHADER_PGM_RSRC2_GS__OC_LDS_EN__SHIFT                                                             0x12
+#define SPI_SHADER_PGM_RSRC2_GS__LDS_SIZE__SHIFT                                                              0x13
+#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR_MSB__SHIFT                                                         0x1b
+#define SPI_SHADER_PGM_RSRC2_GS__SHARED_VGPR_CNT__SHIFT                                                       0x1c
+#define SPI_SHADER_PGM_RSRC2_GS__SCRATCH_EN_MASK                                                              0x00000001L
+#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR_MASK                                                               0x0000003EL
+#define SPI_SHADER_PGM_RSRC2_GS__TRAP_PRESENT_MASK                                                            0x00000040L
+#define SPI_SHADER_PGM_RSRC2_GS__EXCP_EN_MASK                                                                 0x0000FF80L
+#define SPI_SHADER_PGM_RSRC2_GS__ES_VGPR_COMP_CNT_MASK                                                        0x00030000L
+#define SPI_SHADER_PGM_RSRC2_GS__OC_LDS_EN_MASK                                                               0x00040000L
+#define SPI_SHADER_PGM_RSRC2_GS__LDS_SIZE_MASK                                                                0x07F80000L
+#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR_MSB_MASK                                                           0x08000000L
+#define SPI_SHADER_PGM_RSRC2_GS__SHARED_VGPR_CNT_MASK                                                         0xF0000000L
+//SPI_SHADER_USER_DATA_GS_0
+#define SPI_SHADER_USER_DATA_GS_0__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_0__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_1
+#define SPI_SHADER_USER_DATA_GS_1__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_1__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_2
+#define SPI_SHADER_USER_DATA_GS_2__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_2__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_3
+#define SPI_SHADER_USER_DATA_GS_3__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_3__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_4
+#define SPI_SHADER_USER_DATA_GS_4__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_4__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_5
+#define SPI_SHADER_USER_DATA_GS_5__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_5__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_6
+#define SPI_SHADER_USER_DATA_GS_6__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_6__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_7
+#define SPI_SHADER_USER_DATA_GS_7__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_7__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_8
+#define SPI_SHADER_USER_DATA_GS_8__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_8__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_9
+#define SPI_SHADER_USER_DATA_GS_9__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_9__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_10
+#define SPI_SHADER_USER_DATA_GS_10__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_10__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_11
+#define SPI_SHADER_USER_DATA_GS_11__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_11__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_12
+#define SPI_SHADER_USER_DATA_GS_12__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_12__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_13
+#define SPI_SHADER_USER_DATA_GS_13__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_13__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_14
+#define SPI_SHADER_USER_DATA_GS_14__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_14__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_15
+#define SPI_SHADER_USER_DATA_GS_15__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_15__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_16
+#define SPI_SHADER_USER_DATA_GS_16__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_16__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_17
+#define SPI_SHADER_USER_DATA_GS_17__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_17__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_18
+#define SPI_SHADER_USER_DATA_GS_18__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_18__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_19
+#define SPI_SHADER_USER_DATA_GS_19__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_19__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_20
+#define SPI_SHADER_USER_DATA_GS_20__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_20__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_21
+#define SPI_SHADER_USER_DATA_GS_21__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_21__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_22
+#define SPI_SHADER_USER_DATA_GS_22__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_22__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_23
+#define SPI_SHADER_USER_DATA_GS_23__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_23__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_24
+#define SPI_SHADER_USER_DATA_GS_24__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_24__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_25
+#define SPI_SHADER_USER_DATA_GS_25__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_25__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_26
+#define SPI_SHADER_USER_DATA_GS_26__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_26__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_27
+#define SPI_SHADER_USER_DATA_GS_27__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_27__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_28
+#define SPI_SHADER_USER_DATA_GS_28__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_28__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_29
+#define SPI_SHADER_USER_DATA_GS_29__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_29__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_30
+#define SPI_SHADER_USER_DATA_GS_30__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_30__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_31
+#define SPI_SHADER_USER_DATA_GS_31__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_31__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_GS_MESHLET_DIM
+#define SPI_SHADER_GS_MESHLET_DIM__MESHLET_NUM_THREAD_X__SHIFT                                                0x0
+#define SPI_SHADER_GS_MESHLET_DIM__MESHLET_NUM_THREAD_Y__SHIFT                                                0x8
+#define SPI_SHADER_GS_MESHLET_DIM__MESHLET_NUM_THREAD_Z__SHIFT                                                0x10
+#define SPI_SHADER_GS_MESHLET_DIM__MESHLET_THREADGROUP_SIZE__SHIFT                                            0x18
+#define SPI_SHADER_GS_MESHLET_DIM__MESHLET_NUM_THREAD_X_MASK                                                  0x000000FFL
+#define SPI_SHADER_GS_MESHLET_DIM__MESHLET_NUM_THREAD_Y_MASK                                                  0x0000FF00L
+#define SPI_SHADER_GS_MESHLET_DIM__MESHLET_NUM_THREAD_Z_MASK                                                  0x00FF0000L
+#define SPI_SHADER_GS_MESHLET_DIM__MESHLET_THREADGROUP_SIZE_MASK                                              0xFF000000L
+//SPI_SHADER_GS_MESHLET_EXP_ALLOC
+#define SPI_SHADER_GS_MESHLET_EXP_ALLOC__MAX_EXP_VERTS__SHIFT                                                 0x0
+#define SPI_SHADER_GS_MESHLET_EXP_ALLOC__MAX_EXP_PRIMS__SHIFT                                                 0x9
+#define SPI_SHADER_GS_MESHLET_EXP_ALLOC__MAX_EXP_VERTS_MASK                                                   0x000001FFL
+#define SPI_SHADER_GS_MESHLET_EXP_ALLOC__MAX_EXP_PRIMS_MASK                                                   0x0003FE00L
+//SPI_SHADER_GS_MESHLET_CTRL
+#define SPI_SHADER_GS_MESHLET_CTRL__INTERLEAVE_BITS_X__SHIFT                                                  0x0
+#define SPI_SHADER_GS_MESHLET_CTRL__INTERLEAVE_BITS_Y__SHIFT                                                  0x4
+#define SPI_SHADER_GS_MESHLET_CTRL__INTERLEAVE_BITS_X_MASK                                                    0x0000000FL
+#define SPI_SHADER_GS_MESHLET_CTRL__INTERLEAVE_BITS_Y_MASK                                                    0x000000F0L
+//SPI_SHADER_REQ_CTRL_ESGS
+#define SPI_SHADER_REQ_CTRL_ESGS__SOFT_GROUPING_EN__SHIFT                                                     0x0
+#define SPI_SHADER_REQ_CTRL_ESGS__NUMBER_OF_REQUESTS_PER_CU__SHIFT                                            0x1
+#define SPI_SHADER_REQ_CTRL_ESGS__SOFT_GROUPING_ALLOCATION_TIMEOUT__SHIFT                                     0x5
+#define SPI_SHADER_REQ_CTRL_ESGS__HARD_LOCK_HYSTERESIS__SHIFT                                                 0x9
+#define SPI_SHADER_REQ_CTRL_ESGS__HARD_LOCK_LOW_THRESHOLD__SHIFT                                              0xa
+#define SPI_SHADER_REQ_CTRL_ESGS__PRODUCER_REQUEST_LOCKOUT__SHIFT                                             0xf
+#define SPI_SHADER_REQ_CTRL_ESGS__GLOBAL_SCANNING_EN__SHIFT                                                   0x10
+#define SPI_SHADER_REQ_CTRL_ESGS__ALLOCATION_RATE_THROTTLING_THRESHOLD__SHIFT                                 0x11
+#define SPI_SHADER_REQ_CTRL_ESGS__SOFT_GROUPING_EN_MASK                                                       0x00000001L
+#define SPI_SHADER_REQ_CTRL_ESGS__NUMBER_OF_REQUESTS_PER_CU_MASK                                              0x0000001EL
+#define SPI_SHADER_REQ_CTRL_ESGS__SOFT_GROUPING_ALLOCATION_TIMEOUT_MASK                                       0x000001E0L
+#define SPI_SHADER_REQ_CTRL_ESGS__HARD_LOCK_HYSTERESIS_MASK                                                   0x00000200L
+#define SPI_SHADER_REQ_CTRL_ESGS__HARD_LOCK_LOW_THRESHOLD_MASK                                                0x00007C00L
+#define SPI_SHADER_REQ_CTRL_ESGS__PRODUCER_REQUEST_LOCKOUT_MASK                                               0x00008000L
+#define SPI_SHADER_REQ_CTRL_ESGS__GLOBAL_SCANNING_EN_MASK                                                     0x00010000L
+#define SPI_SHADER_REQ_CTRL_ESGS__ALLOCATION_RATE_THROTTLING_THRESHOLD_MASK                                   0x000E0000L
+//SPI_SHADER_GS_OUT_CONFIG_PS_GS
+#define SPI_SHADER_GS_OUT_CONFIG_PS_GS__VS_EXPORT_COUNT__SHIFT                                                0x0
+#define SPI_SHADER_GS_OUT_CONFIG_PS_GS__PRIM_EXPORT_COUNT__SHIFT                                              0x5
+#define SPI_SHADER_GS_OUT_CONFIG_PS_GS__NO_PC_EXPORT__SHIFT                                                   0xa
+#define SPI_SHADER_GS_OUT_CONFIG_PS_GS__NUM_INTERP__SHIFT                                                     0xb
+#define SPI_SHADER_GS_OUT_CONFIG_PS_GS__NUM_PRIM_INTERP__SHIFT                                                0x11
+#define SPI_SHADER_GS_OUT_CONFIG_PS_GS__VS_EXPORT_COUNT_MASK                                                  0x0000001FL
+#define SPI_SHADER_GS_OUT_CONFIG_PS_GS__PRIM_EXPORT_COUNT_MASK                                                0x000003E0L
+#define SPI_SHADER_GS_OUT_CONFIG_PS_GS__NO_PC_EXPORT_MASK                                                     0x00000400L
+#define SPI_SHADER_GS_OUT_CONFIG_PS_GS__NUM_INTERP_MASK                                                       0x0001F800L
+#define SPI_SHADER_GS_OUT_CONFIG_PS_GS__NUM_PRIM_INTERP_MASK                                                  0x003E0000L
+//SPI_SHADER_USER_ACCUM_ESGS_0
+#define SPI_SHADER_USER_ACCUM_ESGS_0__CONTRIBUTION__SHIFT                                                     0x0
+#define SPI_SHADER_USER_ACCUM_ESGS_0__CONTRIBUTION_MASK                                                       0x0000007FL
+//SPI_SHADER_USER_ACCUM_ESGS_1
+#define SPI_SHADER_USER_ACCUM_ESGS_1__CONTRIBUTION__SHIFT                                                     0x0
+#define SPI_SHADER_USER_ACCUM_ESGS_1__CONTRIBUTION_MASK                                                       0x0000007FL
+//SPI_SHADER_USER_ACCUM_ESGS_2
+#define SPI_SHADER_USER_ACCUM_ESGS_2__CONTRIBUTION__SHIFT                                                     0x0
+#define SPI_SHADER_USER_ACCUM_ESGS_2__CONTRIBUTION_MASK                                                       0x0000007FL
+//SPI_SHADER_USER_ACCUM_ESGS_3
+#define SPI_SHADER_USER_ACCUM_ESGS_3__CONTRIBUTION__SHIFT                                                     0x0
+#define SPI_SHADER_USER_ACCUM_ESGS_3__CONTRIBUTION_MASK                                                       0x0000007FL
+//SPI_SHADER_PGM_CHKSUM_HS
+#define SPI_SHADER_PGM_CHKSUM_HS__CHECKSUM__SHIFT                                                             0x0
+#define SPI_SHADER_PGM_CHKSUM_HS__CHECKSUM_MASK                                                               0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ADDR_LO_HS
+#define SPI_SHADER_USER_DATA_ADDR_LO_HS__MEM_BASE__SHIFT                                                      0x0
+#define SPI_SHADER_USER_DATA_ADDR_LO_HS__MEM_BASE_MASK                                                        0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ADDR_HI_HS
+#define SPI_SHADER_USER_DATA_ADDR_HI_HS__MEM_BASE__SHIFT                                                      0x0
+#define SPI_SHADER_USER_DATA_ADDR_HI_HS__MEM_BASE_MASK                                                        0xFFFFFFFFL
+//SPI_SHADER_PGM_LO_HS
+#define SPI_SHADER_PGM_LO_HS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_LO_HS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_HS
+#define SPI_SHADER_PGM_HI_HS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_HI_HS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_LS
+#define SPI_SHADER_PGM_HI_LS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_HI_LS__MEM_BASE_MASK                                                                   0x000000FFL
+//SPI_SHADER_PGM_RSRC3_HS
+#define SPI_SHADER_PGM_RSRC3_HS__CU_EN__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC3_HS__CU_EN_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_RSRC4_HS
+#define SPI_SHADER_PGM_RSRC4_HS__WAVE_LIMIT__SHIFT                                                            0x0
+#define SPI_SHADER_PGM_RSRC4_HS__GLG_EN_OVERRIDE__SHIFT                                                       0xa
+#define SPI_SHADER_PGM_RSRC4_HS__GLG_FORCE_DISABLE__SHIFT                                                     0xb
+#define SPI_SHADER_PGM_RSRC4_HS__DIDT_THROTTLE__SHIFT                                                         0xc
+#define SPI_SHADER_PGM_RSRC4_HS__INST_PREF_SIZE__SHIFT                                                        0x10
+#define SPI_SHADER_PGM_RSRC4_HS__IMAGE_OP__SHIFT                                                              0x1f
+#define SPI_SHADER_PGM_RSRC4_HS__WAVE_LIMIT_MASK                                                              0x000003FFL
+#define SPI_SHADER_PGM_RSRC4_HS__GLG_EN_OVERRIDE_MASK                                                         0x00000400L
+#define SPI_SHADER_PGM_RSRC4_HS__GLG_FORCE_DISABLE_MASK                                                       0x00000800L
+#define SPI_SHADER_PGM_RSRC4_HS__DIDT_THROTTLE_MASK                                                           0x00001000L
+#define SPI_SHADER_PGM_RSRC4_HS__INST_PREF_SIZE_MASK                                                          0x00FF0000L
+#define SPI_SHADER_PGM_RSRC4_HS__IMAGE_OP_MASK                                                                0x80000000L
+//SPI_SHADER_PGM_LO_LS
+#define SPI_SHADER_PGM_LO_LS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_LO_LS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_RSRC1_HS
+#define SPI_SHADER_PGM_RSRC1_HS__VGPRS__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC1_HS__SGPRS__SHIFT                                                                 0x6
+#define SPI_SHADER_PGM_RSRC1_HS__PRIORITY__SHIFT                                                              0xa
+#define SPI_SHADER_PGM_RSRC1_HS__FLOAT_MODE__SHIFT                                                            0xc
+#define SPI_SHADER_PGM_RSRC1_HS__PRIV__SHIFT                                                                  0x14
+#define SPI_SHADER_PGM_RSRC1_HS__WG_RR_EN__SHIFT                                                              0x15
+#define SPI_SHADER_PGM_RSRC1_HS__DEBUG_MODE__SHIFT                                                            0x16
+#define SPI_SHADER_PGM_RSRC1_HS__DISABLE_PERF__SHIFT                                                          0x17
+#define SPI_SHADER_PGM_RSRC1_HS__FWD_PROGRESS__SHIFT                                                          0x19
+#define SPI_SHADER_PGM_RSRC1_HS__WGP_MODE__SHIFT                                                              0x1a
+#define SPI_SHADER_PGM_RSRC1_HS__CDBG_USER__SHIFT                                                             0x1b
+#define SPI_SHADER_PGM_RSRC1_HS__LS_VGPR_COMP_CNT__SHIFT                                                      0x1c
+#define SPI_SHADER_PGM_RSRC1_HS__FP16_OVFL__SHIFT                                                             0x1e
+#define SPI_SHADER_PGM_RSRC1_HS__VGPRS_MASK                                                                   0x0000003FL
+#define SPI_SHADER_PGM_RSRC1_HS__SGPRS_MASK                                                                   0x000003C0L
+#define SPI_SHADER_PGM_RSRC1_HS__PRIORITY_MASK                                                                0x00000C00L
+#define SPI_SHADER_PGM_RSRC1_HS__FLOAT_MODE_MASK                                                              0x000FF000L
+#define SPI_SHADER_PGM_RSRC1_HS__PRIV_MASK                                                                    0x00100000L
+#define SPI_SHADER_PGM_RSRC1_HS__WG_RR_EN_MASK                                                                0x00200000L
+#define SPI_SHADER_PGM_RSRC1_HS__DEBUG_MODE_MASK                                                              0x00400000L
+#define SPI_SHADER_PGM_RSRC1_HS__DISABLE_PERF_MASK                                                            0x00800000L
+#define SPI_SHADER_PGM_RSRC1_HS__FWD_PROGRESS_MASK                                                            0x02000000L
+#define SPI_SHADER_PGM_RSRC1_HS__WGP_MODE_MASK                                                                0x04000000L
+#define SPI_SHADER_PGM_RSRC1_HS__CDBG_USER_MASK                                                               0x08000000L
+#define SPI_SHADER_PGM_RSRC1_HS__LS_VGPR_COMP_CNT_MASK                                                        0x30000000L
+#define SPI_SHADER_PGM_RSRC1_HS__FP16_OVFL_MASK                                                               0x40000000L
+//SPI_SHADER_PGM_RSRC2_HS
+#define SPI_SHADER_PGM_RSRC2_HS__SCRATCH_EN__SHIFT                                                            0x0
+#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR__SHIFT                                                             0x1
+#define SPI_SHADER_PGM_RSRC2_HS__TRAP_PRESENT__SHIFT                                                          0x6
+#define SPI_SHADER_PGM_RSRC2_HS__OC_LDS_EN__SHIFT                                                             0x7
+#define SPI_SHADER_PGM_RSRC2_HS__TG_SIZE_EN__SHIFT                                                            0x8
+#define SPI_SHADER_PGM_RSRC2_HS__EXCP_EN__SHIFT                                                               0x9
+#define SPI_SHADER_PGM_RSRC2_HS__LDS_SIZE__SHIFT                                                              0x12
+#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR_MSB__SHIFT                                                         0x1b
+#define SPI_SHADER_PGM_RSRC2_HS__SHARED_VGPR_CNT__SHIFT                                                       0x1c
+#define SPI_SHADER_PGM_RSRC2_HS__SCRATCH_EN_MASK                                                              0x00000001L
+#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR_MASK                                                               0x0000003EL
+#define SPI_SHADER_PGM_RSRC2_HS__TRAP_PRESENT_MASK                                                            0x00000040L
+#define SPI_SHADER_PGM_RSRC2_HS__OC_LDS_EN_MASK                                                               0x00000080L
+#define SPI_SHADER_PGM_RSRC2_HS__TG_SIZE_EN_MASK                                                              0x00000100L
+#define SPI_SHADER_PGM_RSRC2_HS__EXCP_EN_MASK                                                                 0x0003FE00L
+#define SPI_SHADER_PGM_RSRC2_HS__LDS_SIZE_MASK                                                                0x07FC0000L
+#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR_MSB_MASK                                                           0x08000000L
+#define SPI_SHADER_PGM_RSRC2_HS__SHARED_VGPR_CNT_MASK                                                         0xF0000000L
+//SPI_SHADER_USER_DATA_HS_0
+#define SPI_SHADER_USER_DATA_HS_0__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_0__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_1
+#define SPI_SHADER_USER_DATA_HS_1__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_1__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_2
+#define SPI_SHADER_USER_DATA_HS_2__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_2__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_3
+#define SPI_SHADER_USER_DATA_HS_3__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_3__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_4
+#define SPI_SHADER_USER_DATA_HS_4__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_4__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_5
+#define SPI_SHADER_USER_DATA_HS_5__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_5__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_6
+#define SPI_SHADER_USER_DATA_HS_6__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_6__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_7
+#define SPI_SHADER_USER_DATA_HS_7__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_7__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_8
+#define SPI_SHADER_USER_DATA_HS_8__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_8__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_9
+#define SPI_SHADER_USER_DATA_HS_9__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_9__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_10
+#define SPI_SHADER_USER_DATA_HS_10__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_10__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_11
+#define SPI_SHADER_USER_DATA_HS_11__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_11__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_12
+#define SPI_SHADER_USER_DATA_HS_12__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_12__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_13
+#define SPI_SHADER_USER_DATA_HS_13__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_13__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_14
+#define SPI_SHADER_USER_DATA_HS_14__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_14__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_15
+#define SPI_SHADER_USER_DATA_HS_15__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_15__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_16
+#define SPI_SHADER_USER_DATA_HS_16__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_16__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_17
+#define SPI_SHADER_USER_DATA_HS_17__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_17__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_18
+#define SPI_SHADER_USER_DATA_HS_18__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_18__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_19
+#define SPI_SHADER_USER_DATA_HS_19__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_19__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_20
+#define SPI_SHADER_USER_DATA_HS_20__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_20__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_21
+#define SPI_SHADER_USER_DATA_HS_21__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_21__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_22
+#define SPI_SHADER_USER_DATA_HS_22__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_22__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_23
+#define SPI_SHADER_USER_DATA_HS_23__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_23__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_24
+#define SPI_SHADER_USER_DATA_HS_24__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_24__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_25
+#define SPI_SHADER_USER_DATA_HS_25__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_25__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_26
+#define SPI_SHADER_USER_DATA_HS_26__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_26__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_27
+#define SPI_SHADER_USER_DATA_HS_27__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_27__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_28
+#define SPI_SHADER_USER_DATA_HS_28__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_28__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_29
+#define SPI_SHADER_USER_DATA_HS_29__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_29__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_30
+#define SPI_SHADER_USER_DATA_HS_30__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_30__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_31
+#define SPI_SHADER_USER_DATA_HS_31__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_31__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_REQ_CTRL_LSHS
+#define SPI_SHADER_REQ_CTRL_LSHS__SOFT_GROUPING_EN__SHIFT                                                     0x0
+#define SPI_SHADER_REQ_CTRL_LSHS__NUMBER_OF_REQUESTS_PER_CU__SHIFT                                            0x1
+#define SPI_SHADER_REQ_CTRL_LSHS__SOFT_GROUPING_ALLOCATION_TIMEOUT__SHIFT                                     0x5
+#define SPI_SHADER_REQ_CTRL_LSHS__HARD_LOCK_HYSTERESIS__SHIFT                                                 0x9
+#define SPI_SHADER_REQ_CTRL_LSHS__HARD_LOCK_LOW_THRESHOLD__SHIFT                                              0xa
+#define SPI_SHADER_REQ_CTRL_LSHS__PRODUCER_REQUEST_LOCKOUT__SHIFT                                             0xf
+#define SPI_SHADER_REQ_CTRL_LSHS__GLOBAL_SCANNING_EN__SHIFT                                                   0x10
+#define SPI_SHADER_REQ_CTRL_LSHS__ALLOCATION_RATE_THROTTLING_THRESHOLD__SHIFT                                 0x11
+#define SPI_SHADER_REQ_CTRL_LSHS__SOFT_GROUPING_EN_MASK                                                       0x00000001L
+#define SPI_SHADER_REQ_CTRL_LSHS__NUMBER_OF_REQUESTS_PER_CU_MASK                                              0x0000001EL
+#define SPI_SHADER_REQ_CTRL_LSHS__SOFT_GROUPING_ALLOCATION_TIMEOUT_MASK                                       0x000001E0L
+#define SPI_SHADER_REQ_CTRL_LSHS__HARD_LOCK_HYSTERESIS_MASK                                                   0x00000200L
+#define SPI_SHADER_REQ_CTRL_LSHS__HARD_LOCK_LOW_THRESHOLD_MASK                                                0x00007C00L
+#define SPI_SHADER_REQ_CTRL_LSHS__PRODUCER_REQUEST_LOCKOUT_MASK                                               0x00008000L
+#define SPI_SHADER_REQ_CTRL_LSHS__GLOBAL_SCANNING_EN_MASK                                                     0x00010000L
+#define SPI_SHADER_REQ_CTRL_LSHS__ALLOCATION_RATE_THROTTLING_THRESHOLD_MASK                                   0x000E0000L
+//SPI_SHADER_USER_ACCUM_LSHS_0
+#define SPI_SHADER_USER_ACCUM_LSHS_0__CONTRIBUTION__SHIFT                                                     0x0
+#define SPI_SHADER_USER_ACCUM_LSHS_0__CONTRIBUTION_MASK                                                       0x0000007FL
+//SPI_SHADER_USER_ACCUM_LSHS_1
+#define SPI_SHADER_USER_ACCUM_LSHS_1__CONTRIBUTION__SHIFT                                                     0x0
+#define SPI_SHADER_USER_ACCUM_LSHS_1__CONTRIBUTION_MASK                                                       0x0000007FL
+//SPI_SHADER_USER_ACCUM_LSHS_2
+#define SPI_SHADER_USER_ACCUM_LSHS_2__CONTRIBUTION__SHIFT                                                     0x0
+#define SPI_SHADER_USER_ACCUM_LSHS_2__CONTRIBUTION_MASK                                                       0x0000007FL
+//SPI_SHADER_USER_ACCUM_LSHS_3
+#define SPI_SHADER_USER_ACCUM_LSHS_3__CONTRIBUTION__SHIFT                                                     0x0
+#define SPI_SHADER_USER_ACCUM_LSHS_3__CONTRIBUTION_MASK                                                       0x0000007FL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_spipdec
+//SPI_ARB_PRIORITY
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS0__SHIFT                                                               0x0
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS1__SHIFT                                                               0x3
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS2__SHIFT                                                               0x6
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS3__SHIFT                                                               0x9
+#define SPI_ARB_PRIORITY__TS0_DUR_MULT__SHIFT                                                                 0xc
+#define SPI_ARB_PRIORITY__TS1_DUR_MULT__SHIFT                                                                 0xe
+#define SPI_ARB_PRIORITY__TS2_DUR_MULT__SHIFT                                                                 0x10
+#define SPI_ARB_PRIORITY__TS3_DUR_MULT__SHIFT                                                                 0x12
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS0_MASK                                                                 0x00000007L
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS1_MASK                                                                 0x00000038L
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS2_MASK                                                                 0x000001C0L
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS3_MASK                                                                 0x00000E00L
+#define SPI_ARB_PRIORITY__TS0_DUR_MULT_MASK                                                                   0x00003000L
+#define SPI_ARB_PRIORITY__TS1_DUR_MULT_MASK                                                                   0x0000C000L
+#define SPI_ARB_PRIORITY__TS2_DUR_MULT_MASK                                                                   0x00030000L
+#define SPI_ARB_PRIORITY__TS3_DUR_MULT_MASK                                                                   0x000C0000L
+//SPI_ARB_CYCLES_0
+#define SPI_ARB_CYCLES_0__TS0_DURATION__SHIFT                                                                 0x0
+#define SPI_ARB_CYCLES_0__TS1_DURATION__SHIFT                                                                 0x10
+#define SPI_ARB_CYCLES_0__TS0_DURATION_MASK                                                                   0x0000FFFFL
+#define SPI_ARB_CYCLES_0__TS1_DURATION_MASK                                                                   0xFFFF0000L
+//SPI_ARB_CYCLES_1
+#define SPI_ARB_CYCLES_1__TS2_DURATION__SHIFT                                                                 0x0
+#define SPI_ARB_CYCLES_1__TS3_DURATION__SHIFT                                                                 0x10
+#define SPI_ARB_CYCLES_1__TS2_DURATION_MASK                                                                   0x0000FFFFL
+#define SPI_ARB_CYCLES_1__TS3_DURATION_MASK                                                                   0xFFFF0000L
+//SPI_WCL_PIPE_PERCENT_GFX
+#define SPI_WCL_PIPE_PERCENT_GFX__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_GFX__HS_GRP_VALUE__SHIFT                                                         0xc
+#define SPI_WCL_PIPE_PERCENT_GFX__GS_GRP_VALUE__SHIFT                                                         0x16
+#define SPI_WCL_PIPE_PERCENT_GFX__VALUE_MASK                                                                  0x0000007FL
+#define SPI_WCL_PIPE_PERCENT_GFX__HS_GRP_VALUE_MASK                                                           0x0001F000L
+#define SPI_WCL_PIPE_PERCENT_GFX__GS_GRP_VALUE_MASK                                                           0x07C00000L
+//SPI_WCL_PIPE_PERCENT_HP3D
+#define SPI_WCL_PIPE_PERCENT_HP3D__VALUE__SHIFT                                                               0x0
+#define SPI_WCL_PIPE_PERCENT_HP3D__HS_GRP_VALUE__SHIFT                                                        0xc
+#define SPI_WCL_PIPE_PERCENT_HP3D__GS_GRP_VALUE__SHIFT                                                        0x16
+#define SPI_WCL_PIPE_PERCENT_HP3D__VALUE_MASK                                                                 0x0000007FL
+#define SPI_WCL_PIPE_PERCENT_HP3D__HS_GRP_VALUE_MASK                                                          0x0001F000L
+#define SPI_WCL_PIPE_PERCENT_HP3D__GS_GRP_VALUE_MASK                                                          0x07C00000L
+//SPI_WCL_PIPE_PERCENT_CS0
+#define SPI_WCL_PIPE_PERCENT_CS0__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS0__VALUE_MASK                                                                  0x0000007FL
+//SPI_WCL_PIPE_PERCENT_CS1
+#define SPI_WCL_PIPE_PERCENT_CS1__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS1__VALUE_MASK                                                                  0x0000007FL
+//SPI_WCL_PIPE_PERCENT_CS2
+#define SPI_WCL_PIPE_PERCENT_CS2__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS2__VALUE_MASK                                                                  0x0000007FL
+//SPI_WCL_PIPE_PERCENT_CS3
+#define SPI_WCL_PIPE_PERCENT_CS3__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS3__VALUE_MASK                                                                  0x0000007FL
+//SPI_WCL_PIPE_PERCENT_CS4
+#define SPI_WCL_PIPE_PERCENT_CS4__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS4__VALUE_MASK                                                                  0x0000007FL
+//SPI_WCL_PIPE_PERCENT_CS5
+#define SPI_WCL_PIPE_PERCENT_CS5__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS5__VALUE_MASK                                                                  0x0000007FL
+//SPI_WCL_PIPE_PERCENT_CS6
+#define SPI_WCL_PIPE_PERCENT_CS6__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS6__VALUE_MASK                                                                  0x0000007FL
+//SPI_WCL_PIPE_PERCENT_CS7
+#define SPI_WCL_PIPE_PERCENT_CS7__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS7__VALUE_MASK                                                                  0x0000007FL
+//SPI_USER_ACCUM_VMID_CNTL
+#define SPI_USER_ACCUM_VMID_CNTL__EN_USER_ACCUM__SHIFT                                                        0x0
+#define SPI_USER_ACCUM_VMID_CNTL__EN_USER_ACCUM_MASK                                                          0x0000000FL
+//SPI_GDBG_PER_VMID_CNTL
+#define SPI_GDBG_PER_VMID_CNTL__STALL_VMID__SHIFT                                                             0x0
+#define SPI_GDBG_PER_VMID_CNTL__LAUNCH_MODE__SHIFT                                                            0x1
+#define SPI_GDBG_PER_VMID_CNTL__TRAP_EN__SHIFT                                                                0x3
+#define SPI_GDBG_PER_VMID_CNTL__EXCP_EN__SHIFT                                                                0x4
+#define SPI_GDBG_PER_VMID_CNTL__EXCP_REPLACE__SHIFT                                                           0xd
+#define SPI_GDBG_PER_VMID_CNTL__TRAP_ON_START__SHIFT                                                          0xe
+#define SPI_GDBG_PER_VMID_CNTL__TRAP_ON_END__SHIFT                                                            0xf
+#define SPI_GDBG_PER_VMID_CNTL__STALL_VMID_MASK                                                               0x00000001L
+#define SPI_GDBG_PER_VMID_CNTL__LAUNCH_MODE_MASK                                                              0x00000006L
+#define SPI_GDBG_PER_VMID_CNTL__TRAP_EN_MASK                                                                  0x00000008L
+#define SPI_GDBG_PER_VMID_CNTL__EXCP_EN_MASK                                                                  0x00001FF0L
+#define SPI_GDBG_PER_VMID_CNTL__EXCP_REPLACE_MASK                                                             0x00002000L
+#define SPI_GDBG_PER_VMID_CNTL__TRAP_ON_START_MASK                                                            0x00004000L
+#define SPI_GDBG_PER_VMID_CNTL__TRAP_ON_END_MASK                                                              0x00008000L
+//SPI_COMPUTE_QUEUE_RESET
+#define SPI_COMPUTE_QUEUE_RESET__RESET__SHIFT                                                                 0x0
+#define SPI_COMPUTE_QUEUE_RESET__RESET_MASK                                                                   0x00000001L
+//SPI_COMPUTE_WF_CTX_SAVE
+#define SPI_COMPUTE_WF_CTX_SAVE__INITIATE__SHIFT                                                              0x0
+#define SPI_COMPUTE_WF_CTX_SAVE__DONE_INTERRUPT_EN__SHIFT                                                     0x2
+#define SPI_COMPUTE_WF_CTX_SAVE__SAVE_BUSY__SHIFT                                                             0x1f
+#define SPI_COMPUTE_WF_CTX_SAVE__INITIATE_MASK                                                                0x00000001L
+#define SPI_COMPUTE_WF_CTX_SAVE__DONE_INTERRUPT_EN_MASK                                                       0x00000004L
+#define SPI_COMPUTE_WF_CTX_SAVE__SAVE_BUSY_MASK                                                               0x80000000L
+//SPI_SAVE_RESTORE_STATUS
+#define SPI_SAVE_RESTORE_STATUS__PERFCOUNTER_EN__SHIFT                                                        0x0
+#define SPI_SAVE_RESTORE_STATUS__THREAD_TRACE_EN__SHIFT                                                       0x1
+#define SPI_SAVE_RESTORE_STATUS__PERFCOUNTER_EN_MASK                                                          0x00000001L
+#define SPI_SAVE_RESTORE_STATUS__THREAD_TRACE_EN_MASK                                                         0x00000002L
+//SPI_CONFIG_DIDT_CNTL
+#define SPI_CONFIG_DIDT_CNTL__DELAY_SELECT__SHIFT                                                             0x0
+#define SPI_CONFIG_DIDT_CNTL__INCR_EVENT__SHIFT                                                               0x6
+#define SPI_CONFIG_DIDT_CNTL__THRESHOLD__SHIFT                                                                0x8
+#define SPI_CONFIG_DIDT_CNTL__STALL_WAVES__SHIFT                                                              0xe
+#define SPI_CONFIG_DIDT_CNTL__STALL_RATE__SHIFT                                                               0x10
+#define SPI_CONFIG_DIDT_CNTL__THROTTLE_EN__SHIFT                                                              0x15
+#define SPI_CONFIG_DIDT_CNTL__DELAY_SELECT_MASK                                                               0x0000003FL
+#define SPI_CONFIG_DIDT_CNTL__INCR_EVENT_MASK                                                                 0x000000C0L
+#define SPI_CONFIG_DIDT_CNTL__THRESHOLD_MASK                                                                  0x00003F00L
+#define SPI_CONFIG_DIDT_CNTL__STALL_WAVES_MASK                                                                0x0000C000L
+#define SPI_CONFIG_DIDT_CNTL__STALL_RATE_MASK                                                                 0x001F0000L
+#define SPI_CONFIG_DIDT_CNTL__THROTTLE_EN_MASK                                                                0x00200000L
+//SPI_CONFIG_DIDT_TYPEMASK
+#define SPI_CONFIG_DIDT_TYPEMASK__GFX_TYPE_ENABLE__SHIFT                                                      0x0
+#define SPI_CONFIG_DIDT_TYPEMASK__ACE_TYPE_ENABLE__SHIFT                                                      0x4
+#define SPI_CONFIG_DIDT_TYPEMASK__GFX_TYPE_ENABLE_MASK                                                        0x0000000FL
+#define SPI_CONFIG_DIDT_TYPEMASK__ACE_TYPE_ENABLE_MASK                                                        0x00000FF0L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_tcpdec
+//TCP_WATCH0_ADDR_H
+#define TCP_WATCH0_ADDR_H__ADDR__SHIFT                                                                        0x0
+#define TCP_WATCH0_ADDR_H__ADDR_MASK                                                                          0x0000FFFFL
+//TCP_WATCH0_ADDR_L
+#define TCP_WATCH0_ADDR_L__ADDR__SHIFT                                                                        0x7
+#define TCP_WATCH0_ADDR_L__ADDR_MASK                                                                          0xFFFFFF80L
+//TCP_WATCH0_CNTL
+#define TCP_WATCH0_CNTL__MASK__SHIFT                                                                          0x0
+#define TCP_WATCH0_CNTL__VMID__SHIFT                                                                          0x18
+#define TCP_WATCH0_CNTL__MODE__SHIFT                                                                          0x1d
+#define TCP_WATCH0_CNTL__VALID__SHIFT                                                                         0x1f
+#define TCP_WATCH0_CNTL__MASK_MASK                                                                            0x007FFFFFL
+#define TCP_WATCH0_CNTL__VMID_MASK                                                                            0x0F000000L
+#define TCP_WATCH0_CNTL__MODE_MASK                                                                            0x60000000L
+#define TCP_WATCH0_CNTL__VALID_MASK                                                                           0x80000000L
+//TCP_WATCH1_ADDR_H
+#define TCP_WATCH1_ADDR_H__ADDR__SHIFT                                                                        0x0
+#define TCP_WATCH1_ADDR_H__ADDR_MASK                                                                          0x0000FFFFL
+//TCP_WATCH1_ADDR_L
+#define TCP_WATCH1_ADDR_L__ADDR__SHIFT                                                                        0x7
+#define TCP_WATCH1_ADDR_L__ADDR_MASK                                                                          0xFFFFFF80L
+//TCP_WATCH1_CNTL
+#define TCP_WATCH1_CNTL__MASK__SHIFT                                                                          0x0
+#define TCP_WATCH1_CNTL__VMID__SHIFT                                                                          0x18
+#define TCP_WATCH1_CNTL__MODE__SHIFT                                                                          0x1d
+#define TCP_WATCH1_CNTL__VALID__SHIFT                                                                         0x1f
+#define TCP_WATCH1_CNTL__MASK_MASK                                                                            0x007FFFFFL
+#define TCP_WATCH1_CNTL__VMID_MASK                                                                            0x0F000000L
+#define TCP_WATCH1_CNTL__MODE_MASK                                                                            0x60000000L
+#define TCP_WATCH1_CNTL__VALID_MASK                                                                           0x80000000L
+//TCP_WATCH2_ADDR_H
+#define TCP_WATCH2_ADDR_H__ADDR__SHIFT                                                                        0x0
+#define TCP_WATCH2_ADDR_H__ADDR_MASK                                                                          0x0000FFFFL
+//TCP_WATCH2_ADDR_L
+#define TCP_WATCH2_ADDR_L__ADDR__SHIFT                                                                        0x7
+#define TCP_WATCH2_ADDR_L__ADDR_MASK                                                                          0xFFFFFF80L
+//TCP_WATCH2_CNTL
+#define TCP_WATCH2_CNTL__MASK__SHIFT                                                                          0x0
+#define TCP_WATCH2_CNTL__VMID__SHIFT                                                                          0x18
+#define TCP_WATCH2_CNTL__MODE__SHIFT                                                                          0x1d
+#define TCP_WATCH2_CNTL__VALID__SHIFT                                                                         0x1f
+#define TCP_WATCH2_CNTL__MASK_MASK                                                                            0x007FFFFFL
+#define TCP_WATCH2_CNTL__VMID_MASK                                                                            0x0F000000L
+#define TCP_WATCH2_CNTL__MODE_MASK                                                                            0x60000000L
+#define TCP_WATCH2_CNTL__VALID_MASK                                                                           0x80000000L
+//TCP_WATCH3_ADDR_H
+#define TCP_WATCH3_ADDR_H__ADDR__SHIFT                                                                        0x0
+#define TCP_WATCH3_ADDR_H__ADDR_MASK                                                                          0x0000FFFFL
+//TCP_WATCH3_ADDR_L
+#define TCP_WATCH3_ADDR_L__ADDR__SHIFT                                                                        0x7
+#define TCP_WATCH3_ADDR_L__ADDR_MASK                                                                          0xFFFFFF80L
+//TCP_WATCH3_CNTL
+#define TCP_WATCH3_CNTL__MASK__SHIFT                                                                          0x0
+#define TCP_WATCH3_CNTL__VMID__SHIFT                                                                          0x18
+#define TCP_WATCH3_CNTL__MODE__SHIFT                                                                          0x1d
+#define TCP_WATCH3_CNTL__VALID__SHIFT                                                                         0x1f
+#define TCP_WATCH3_CNTL__MASK_MASK                                                                            0x007FFFFFL
+#define TCP_WATCH3_CNTL__VMID_MASK                                                                            0x0F000000L
+#define TCP_WATCH3_CNTL__MODE_MASK                                                                            0x60000000L
+#define TCP_WATCH3_CNTL__VALID_MASK                                                                           0x80000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_gfxdec0
+//DB_RENDER_CONTROL
+#define DB_RENDER_CONTROL__STENCIL_CLEAR_ENABLE__SHIFT                                                        0x1
+#define DB_RENDER_CONTROL__RESERVED_FIELD_1__SHIFT                                                            0x2
+#define DB_RENDER_CONTROL__RESERVED_FIELD_2__SHIFT                                                            0x3
+#define DB_RENDER_CONTROL__RESERVED_FIELD_4__SHIFT                                                            0x4
+#define DB_RENDER_CONTROL__STENCIL_COMPRESS_DISABLE__SHIFT                                                    0x5
+#define DB_RENDER_CONTROL__DEPTH_COMPRESS_DISABLE__SHIFT                                                      0x6
+#define DB_RENDER_CONTROL__COPY_CENTROID__SHIFT                                                               0x7
+#define DB_RENDER_CONTROL__COPY_SAMPLE__SHIFT                                                                 0x8
+#define DB_RENDER_CONTROL__DECOMPRESS_ENABLE__SHIFT                                                           0xc
+#define DB_RENDER_CONTROL__PS_INVOKE_DISABLE__SHIFT                                                           0xe
+#define DB_RENDER_CONTROL__OREO_MODE__SHIFT                                                                   0x10
+#define DB_RENDER_CONTROL__FORCE_OREO_MODE__SHIFT                                                             0x12
+#define DB_RENDER_CONTROL__FORCE_EXPORT_ORDER__SHIFT                                                          0x13
+#define DB_RENDER_CONTROL__MAX_ALLOWED_STILES_IN_WAVE__SHIFT                                                  0x14
+#define DB_RENDER_CONTROL__STENCIL_CLEAR_ENABLE_MASK                                                          0x00000002L
+#define DB_RENDER_CONTROL__RESERVED_FIELD_1_MASK                                                              0x00000004L
+#define DB_RENDER_CONTROL__RESERVED_FIELD_2_MASK                                                              0x00000008L
+#define DB_RENDER_CONTROL__RESERVED_FIELD_4_MASK                                                              0x00000010L
+#define DB_RENDER_CONTROL__STENCIL_COMPRESS_DISABLE_MASK                                                      0x00000020L
+#define DB_RENDER_CONTROL__DEPTH_COMPRESS_DISABLE_MASK                                                        0x00000040L
+#define DB_RENDER_CONTROL__COPY_CENTROID_MASK                                                                 0x00000080L
+#define DB_RENDER_CONTROL__COPY_SAMPLE_MASK                                                                   0x00000F00L
+#define DB_RENDER_CONTROL__DECOMPRESS_ENABLE_MASK                                                             0x00001000L
+#define DB_RENDER_CONTROL__PS_INVOKE_DISABLE_MASK                                                             0x00004000L
+#define DB_RENDER_CONTROL__OREO_MODE_MASK                                                                     0x00030000L
+#define DB_RENDER_CONTROL__FORCE_OREO_MODE_MASK                                                               0x00040000L
+#define DB_RENDER_CONTROL__FORCE_EXPORT_ORDER_MASK                                                            0x00080000L
+#define DB_RENDER_CONTROL__MAX_ALLOWED_STILES_IN_WAVE_MASK                                                    0x00F00000L
+//DB_DEPTH_VIEW
+#define DB_DEPTH_VIEW__SLICE_START__SHIFT                                                                     0x0
+#define DB_DEPTH_VIEW__SLICE_MAX__SHIFT                                                                       0x10
+#define DB_DEPTH_VIEW__SLICE_START_MASK                                                                       0x00003FFFL
+#define DB_DEPTH_VIEW__SLICE_MAX_MASK                                                                         0x3FFF0000L
+//DB_DEPTH_VIEW1
+#define DB_DEPTH_VIEW1__Z_READ_ONLY__SHIFT                                                                    0x18
+#define DB_DEPTH_VIEW1__STENCIL_READ_ONLY__SHIFT                                                              0x19
+#define DB_DEPTH_VIEW1__MIPID__SHIFT                                                                          0x1a
+#define DB_DEPTH_VIEW1__Z_READ_ONLY_MASK                                                                      0x01000000L
+#define DB_DEPTH_VIEW1__STENCIL_READ_ONLY_MASK                                                                0x02000000L
+#define DB_DEPTH_VIEW1__MIPID_MASK                                                                            0x7C000000L
+//DB_RENDER_OVERRIDE
+#define DB_RENDER_OVERRIDE__FORCE_HIZ_ENABLE__SHIFT                                                           0x0
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE0__SHIFT                                                          0x2
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE1__SHIFT                                                          0x4
+#define DB_RENDER_OVERRIDE__FORCE_SHADER_Z_ORDER__SHIFT                                                       0x6
+#define DB_RENDER_OVERRIDE__FAST_Z_DISABLE__SHIFT                                                             0x7
+#define DB_RENDER_OVERRIDE__FAST_STENCIL_DISABLE__SHIFT                                                       0x8
+#define DB_RENDER_OVERRIDE__NOOP_CULL_DISABLE__SHIFT                                                          0x9
+#define DB_RENDER_OVERRIDE__FORCE_COLOR_KILL__SHIFT                                                           0xa
+#define DB_RENDER_OVERRIDE__FORCE_Z_READ__SHIFT                                                               0xb
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_READ__SHIFT                                                         0xc
+#define DB_RENDER_OVERRIDE__FORCE_FULL_Z_RANGE__SHIFT                                                         0xd
+#define DB_RENDER_OVERRIDE__FORCE_Z_LIMIT_SUMM__SHIFT                                                         0x13
+#define DB_RENDER_OVERRIDE__MAX_TILES_IN_DTT__SHIFT                                                           0x15
+#define DB_RENDER_OVERRIDE__DISABLE_TILE_RATE_TILES__SHIFT                                                    0x1a
+#define DB_RENDER_OVERRIDE__FORCE_Z_DIRTY__SHIFT                                                              0x1b
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_DIRTY__SHIFT                                                        0x1c
+#define DB_RENDER_OVERRIDE__FORCE_Z_VALID__SHIFT                                                              0x1d
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_VALID__SHIFT                                                        0x1e
+#define DB_RENDER_OVERRIDE__PRESERVE_COMPRESSION__SHIFT                                                       0x1f
+#define DB_RENDER_OVERRIDE__FORCE_HIZ_ENABLE_MASK                                                             0x00000003L
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE0_MASK                                                            0x0000000CL
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE1_MASK                                                            0x00000030L
+#define DB_RENDER_OVERRIDE__FORCE_SHADER_Z_ORDER_MASK                                                         0x00000040L
+#define DB_RENDER_OVERRIDE__FAST_Z_DISABLE_MASK                                                               0x00000080L
+#define DB_RENDER_OVERRIDE__FAST_STENCIL_DISABLE_MASK                                                         0x00000100L
+#define DB_RENDER_OVERRIDE__NOOP_CULL_DISABLE_MASK                                                            0x00000200L
+#define DB_RENDER_OVERRIDE__FORCE_COLOR_KILL_MASK                                                             0x00000400L
+#define DB_RENDER_OVERRIDE__FORCE_Z_READ_MASK                                                                 0x00000800L
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_READ_MASK                                                           0x00001000L
+#define DB_RENDER_OVERRIDE__FORCE_FULL_Z_RANGE_MASK                                                           0x00006000L
+#define DB_RENDER_OVERRIDE__FORCE_Z_LIMIT_SUMM_MASK                                                           0x00180000L
+#define DB_RENDER_OVERRIDE__MAX_TILES_IN_DTT_MASK                                                             0x03E00000L
+#define DB_RENDER_OVERRIDE__DISABLE_TILE_RATE_TILES_MASK                                                      0x04000000L
+#define DB_RENDER_OVERRIDE__FORCE_Z_DIRTY_MASK                                                                0x08000000L
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_DIRTY_MASK                                                          0x10000000L
+#define DB_RENDER_OVERRIDE__FORCE_Z_VALID_MASK                                                                0x20000000L
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_VALID_MASK                                                          0x40000000L
+#define DB_RENDER_OVERRIDE__PRESERVE_COMPRESSION_MASK                                                         0x80000000L
+//DB_RENDER_OVERRIDE2
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_CONTROL__SHIFT                                              0x0
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_COUNTDOWN__SHIFT                                            0x2
+#define DB_RENDER_OVERRIDE2__RESERVED_FIELD_5__SHIFT                                                          0x5
+#define DB_RENDER_OVERRIDE2__RESERVED_FIELD_6__SHIFT                                                          0x6
+#define DB_RENDER_OVERRIDE2__DISABLE_COLOR_ON_VALIDATION__SHIFT                                               0x7
+#define DB_RENDER_OVERRIDE2__DECOMPRESS_Z_ON_FLUSH__SHIFT                                                     0x8
+#define DB_RENDER_OVERRIDE2__RESERVED_FIELD_1__SHIFT                                                          0x9
+#define DB_RENDER_OVERRIDE2__DEPTH_BOUNDS_HIER_DEPTH_DISABLE__SHIFT                                           0xa
+#define DB_RENDER_OVERRIDE2__FORCE_SUMM_Z_RANGE_TO_MAX__SHIFT                                                 0xb
+#define DB_RENDER_OVERRIDE2__FORCE_SUMM_STENCIL_RANGE_TO_MAX__SHIFT                                           0xc
+#define DB_RENDER_OVERRIDE2__RESERVED_FIELD_2__SHIFT                                                          0xd
+#define DB_RENDER_OVERRIDE2__PRESERVE_ZRANGE__SHIFT                                                           0x15
+#define DB_RENDER_OVERRIDE2__DISABLE_FAST_PASS__SHIFT                                                         0x17
+#define DB_RENDER_OVERRIDE2__ALLOW_PARTIAL_RES_HIER_KILL__SHIFT                                               0x19
+#define DB_RENDER_OVERRIDE2__CENTROID_COMPUTATION_MODE__SHIFT                                                 0x1b
+#define DB_RENDER_OVERRIDE2__DISABLE_NOZ__SHIFT                                                               0x1d
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_CONTROL_MASK                                                0x00000003L
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_COUNTDOWN_MASK                                              0x0000001CL
+#define DB_RENDER_OVERRIDE2__RESERVED_FIELD_5_MASK                                                            0x00000020L
+#define DB_RENDER_OVERRIDE2__RESERVED_FIELD_6_MASK                                                            0x00000040L
+#define DB_RENDER_OVERRIDE2__DISABLE_COLOR_ON_VALIDATION_MASK                                                 0x00000080L
+#define DB_RENDER_OVERRIDE2__DECOMPRESS_Z_ON_FLUSH_MASK                                                       0x00000100L
+#define DB_RENDER_OVERRIDE2__RESERVED_FIELD_1_MASK                                                            0x00000200L
+#define DB_RENDER_OVERRIDE2__DEPTH_BOUNDS_HIER_DEPTH_DISABLE_MASK                                             0x00000400L
+#define DB_RENDER_OVERRIDE2__FORCE_SUMM_Z_RANGE_TO_MAX_MASK                                                   0x00000800L
+#define DB_RENDER_OVERRIDE2__FORCE_SUMM_STENCIL_RANGE_TO_MAX_MASK                                             0x00001000L
+#define DB_RENDER_OVERRIDE2__RESERVED_FIELD_2_MASK                                                            0x001FE000L
+#define DB_RENDER_OVERRIDE2__PRESERVE_ZRANGE_MASK                                                             0x00200000L
+#define DB_RENDER_OVERRIDE2__DISABLE_FAST_PASS_MASK                                                           0x00800000L
+#define DB_RENDER_OVERRIDE2__ALLOW_PARTIAL_RES_HIER_KILL_MASK                                                 0x02000000L
+#define DB_RENDER_OVERRIDE2__CENTROID_COMPUTATION_MODE_MASK                                                   0x18000000L
+#define DB_RENDER_OVERRIDE2__DISABLE_NOZ_MASK                                                                 0x20000000L
+//DB_DEPTH_SIZE_XY
+#define DB_DEPTH_SIZE_XY__X_MAX__SHIFT                                                                        0x0
+#define DB_DEPTH_SIZE_XY__Y_MAX__SHIFT                                                                        0x10
+#define DB_DEPTH_SIZE_XY__X_MAX_MASK                                                                          0x0000FFFFL
+#define DB_DEPTH_SIZE_XY__Y_MAX_MASK                                                                          0xFFFF0000L
+//DB_Z_INFO
+#define DB_Z_INFO__FORMAT__SHIFT                                                                              0x0
+#define DB_Z_INFO__NUM_SAMPLES__SHIFT                                                                         0x2
+#define DB_Z_INFO__SW_MODE__SHIFT                                                                             0x4
+#define DB_Z_INFO__RESERVED_FIELD_2__SHIFT                                                                    0x9
+#define DB_Z_INFO__RESERVED_FIELD_11__SHIFT                                                                   0xb
+#define DB_Z_INFO__RESERVED_FIELD_12__SHIFT                                                                   0xc
+#define DB_Z_INFO__RESERVED_FIELD_1__SHIFT                                                                    0xd
+#define DB_Z_INFO__MAXMIP__SHIFT                                                                              0xf
+#define DB_Z_INFO__RESERVED_FIELD_20__SHIFT                                                                   0x14
+#define DB_Z_INFO__DECOMPRESS_ON_N_ZPLANES__SHIFT                                                             0x17
+#define DB_Z_INFO__RESERVED_FIELD_27__SHIFT                                                                   0x1b
+#define DB_Z_INFO__RESERVED_FIELD_28__SHIFT                                                                   0x1c
+#define DB_Z_INFO__TILE_SURFACE_ENABLE__SHIFT                                                                 0x1d
+#define DB_Z_INFO__FORMAT_MASK                                                                                0x00000003L
+#define DB_Z_INFO__NUM_SAMPLES_MASK                                                                           0x0000000CL
+#define DB_Z_INFO__SW_MODE_MASK                                                                               0x000001F0L
+#define DB_Z_INFO__RESERVED_FIELD_2_MASK                                                                      0x00000600L
+#define DB_Z_INFO__RESERVED_FIELD_11_MASK                                                                     0x00000800L
+#define DB_Z_INFO__RESERVED_FIELD_12_MASK                                                                     0x00001000L
+#define DB_Z_INFO__RESERVED_FIELD_1_MASK                                                                      0x00006000L
+#define DB_Z_INFO__MAXMIP_MASK                                                                                0x000F8000L
+#define DB_Z_INFO__RESERVED_FIELD_20_MASK                                                                     0x00100000L
+#define DB_Z_INFO__DECOMPRESS_ON_N_ZPLANES_MASK                                                               0x07800000L
+#define DB_Z_INFO__RESERVED_FIELD_27_MASK                                                                     0x08000000L
+#define DB_Z_INFO__RESERVED_FIELD_28_MASK                                                                     0x10000000L
+#define DB_Z_INFO__TILE_SURFACE_ENABLE_MASK                                                                   0x20000000L
+//DB_STENCIL_INFO
+#define DB_STENCIL_INFO__FORMAT__SHIFT                                                                        0x0
+#define DB_STENCIL_INFO__SW_MODE__SHIFT                                                                       0x4
+#define DB_STENCIL_INFO__RESERVED_FIELD_2__SHIFT                                                              0x9
+#define DB_STENCIL_INFO__RESERVED_FIELD_11__SHIFT                                                             0xb
+#define DB_STENCIL_INFO__RESERVED_FIELD_12__SHIFT                                                             0xc
+#define DB_STENCIL_INFO__RESERVED_FIELD_1__SHIFT                                                              0xd
+#define DB_STENCIL_INFO__RESERVED_FIELD_20__SHIFT                                                             0x14
+#define DB_STENCIL_INFO__RESERVED_FIELD_27__SHIFT                                                             0x1b
+#define DB_STENCIL_INFO__TILE_STENCIL_DISABLE__SHIFT                                                          0x1d
+#define DB_STENCIL_INFO__FORMAT_MASK                                                                          0x00000001L
+#define DB_STENCIL_INFO__SW_MODE_MASK                                                                         0x000001F0L
+#define DB_STENCIL_INFO__RESERVED_FIELD_2_MASK                                                                0x00000600L
+#define DB_STENCIL_INFO__RESERVED_FIELD_11_MASK                                                               0x00000800L
+#define DB_STENCIL_INFO__RESERVED_FIELD_12_MASK                                                               0x00001000L
+#define DB_STENCIL_INFO__RESERVED_FIELD_1_MASK                                                                0x0000E000L
+#define DB_STENCIL_INFO__RESERVED_FIELD_20_MASK                                                               0x00100000L
+#define DB_STENCIL_INFO__RESERVED_FIELD_27_MASK                                                               0x08000000L
+#define DB_STENCIL_INFO__TILE_STENCIL_DISABLE_MASK                                                            0x20000000L
+//DB_Z_READ_BASE
+#define DB_Z_READ_BASE__BASE_256B__SHIFT                                                                      0x0
+#define DB_Z_READ_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//DB_Z_READ_BASE_HI
+#define DB_Z_READ_BASE_HI__BASE_HI__SHIFT                                                                     0x0
+#define DB_Z_READ_BASE_HI__BASE_HI_MASK                                                                       0x000000FFL
+//DB_Z_WRITE_BASE
+#define DB_Z_WRITE_BASE__BASE_256B__SHIFT                                                                     0x0
+#define DB_Z_WRITE_BASE__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//DB_Z_WRITE_BASE_HI
+#define DB_Z_WRITE_BASE_HI__BASE_HI__SHIFT                                                                    0x0
+#define DB_Z_WRITE_BASE_HI__BASE_HI_MASK                                                                      0x000000FFL
+//DB_STENCIL_READ_BASE
+#define DB_STENCIL_READ_BASE__BASE_256B__SHIFT                                                                0x0
+#define DB_STENCIL_READ_BASE__BASE_256B_MASK                                                                  0xFFFFFFFFL
+//DB_STENCIL_READ_BASE_HI
+#define DB_STENCIL_READ_BASE_HI__BASE_HI__SHIFT                                                               0x0
+#define DB_STENCIL_READ_BASE_HI__BASE_HI_MASK                                                                 0x000000FFL
+//DB_STENCIL_WRITE_BASE
+#define DB_STENCIL_WRITE_BASE__BASE_256B__SHIFT                                                               0x0
+#define DB_STENCIL_WRITE_BASE__BASE_256B_MASK                                                                 0xFFFFFFFFL
+//DB_STENCIL_WRITE_BASE_HI
+#define DB_STENCIL_WRITE_BASE_HI__BASE_HI__SHIFT                                                              0x0
+#define DB_STENCIL_WRITE_BASE_HI__BASE_HI_MASK                                                                0x000000FFL
+//DB_GL1_INTERFACE_CONTROL
+#define DB_GL1_INTERFACE_CONTROL__Z_SPECULATIVE_READ__SHIFT                                                   0x0
+#define DB_GL1_INTERFACE_CONTROL__STENCIL_SPECULATIVE_READ__SHIFT                                             0x2
+#define DB_GL1_INTERFACE_CONTROL__Z_COMPRESSION_MODE__SHIFT                                                   0x4
+#define DB_GL1_INTERFACE_CONTROL__STENCIL_COMPRESSION_MODE__SHIFT                                             0x6
+#define DB_GL1_INTERFACE_CONTROL__OCCLUSION_COMPRESSION_MODE__SHIFT                                           0x8
+#define DB_GL1_INTERFACE_CONTROL__Z_SPECULATIVE_READ_MASK                                                     0x00000003L
+#define DB_GL1_INTERFACE_CONTROL__STENCIL_SPECULATIVE_READ_MASK                                               0x0000000CL
+#define DB_GL1_INTERFACE_CONTROL__Z_COMPRESSION_MODE_MASK                                                     0x00000030L
+#define DB_GL1_INTERFACE_CONTROL__STENCIL_COMPRESSION_MODE_MASK                                               0x000000C0L
+#define DB_GL1_INTERFACE_CONTROL__OCCLUSION_COMPRESSION_MODE_MASK                                             0x00000300L
+//DB_MEM_TEMPORAL
+#define DB_MEM_TEMPORAL__Z_TEMPORAL_READ__SHIFT                                                               0x0
+#define DB_MEM_TEMPORAL__Z_TEMPORAL_WRITE__SHIFT                                                              0x3
+#define DB_MEM_TEMPORAL__STENCIL_TEMPORAL_READ__SHIFT                                                         0x6
+#define DB_MEM_TEMPORAL__STENCIL_TEMPORAL_WRITE__SHIFT                                                        0x9
+#define DB_MEM_TEMPORAL__OCCLUSION_TEMPORAL_WRITE__SHIFT                                                      0xc
+#define DB_MEM_TEMPORAL__Z_TEMPORAL_READ_MASK                                                                 0x00000007L
+#define DB_MEM_TEMPORAL__Z_TEMPORAL_WRITE_MASK                                                                0x00000038L
+#define DB_MEM_TEMPORAL__STENCIL_TEMPORAL_READ_MASK                                                           0x000001C0L
+#define DB_MEM_TEMPORAL__STENCIL_TEMPORAL_WRITE_MASK                                                          0x00000E00L
+#define DB_MEM_TEMPORAL__OCCLUSION_TEMPORAL_WRITE_MASK                                                        0x00007000L
+//DB_DEPTH_BOUNDS_MIN
+#define DB_DEPTH_BOUNDS_MIN__MIN__SHIFT                                                                       0x0
+#define DB_DEPTH_BOUNDS_MIN__MIN_MASK                                                                         0xFFFFFFFFL
+//DB_DEPTH_BOUNDS_MAX
+#define DB_DEPTH_BOUNDS_MAX__MAX__SHIFT                                                                       0x0
+#define DB_DEPTH_BOUNDS_MAX__MAX_MASK                                                                         0xFFFFFFFFL
+//DB_COUNT_CONTROL
+#define DB_COUNT_CONTROL__PERFECT_ZPASS_COUNTS__SHIFT                                                         0x1
+#define DB_COUNT_CONTROL__DISABLE_CONSERVATIVE_ZPASS_COUNTS__SHIFT                                            0x2
+#define DB_COUNT_CONTROL__ENHANCED_CONSERVATIVE_ZPASS_COUNTS__SHIFT                                           0x3
+#define DB_COUNT_CONTROL__ZPASS_ENABLE__SHIFT                                                                 0x8
+#define DB_COUNT_CONTROL__ZFAIL_ENABLE__SHIFT                                                                 0xc
+#define DB_COUNT_CONTROL__SFAIL_ENABLE__SHIFT                                                                 0x10
+#define DB_COUNT_CONTROL__DBFAIL_ENABLE__SHIFT                                                                0x14
+#define DB_COUNT_CONTROL__SLICE_EVEN_ENABLE__SHIFT                                                            0x18
+#define DB_COUNT_CONTROL__SLICE_ODD_ENABLE__SHIFT                                                             0x1c
+#define DB_COUNT_CONTROL__PERFECT_ZPASS_COUNTS_MASK                                                           0x00000002L
+#define DB_COUNT_CONTROL__DISABLE_CONSERVATIVE_ZPASS_COUNTS_MASK                                              0x00000004L
+#define DB_COUNT_CONTROL__ENHANCED_CONSERVATIVE_ZPASS_COUNTS_MASK                                             0x00000008L
+#define DB_COUNT_CONTROL__ZPASS_ENABLE_MASK                                                                   0x00000F00L
+#define DB_COUNT_CONTROL__ZFAIL_ENABLE_MASK                                                                   0x0000F000L
+#define DB_COUNT_CONTROL__SFAIL_ENABLE_MASK                                                                   0x000F0000L
+#define DB_COUNT_CONTROL__DBFAIL_ENABLE_MASK                                                                  0x00F00000L
+#define DB_COUNT_CONTROL__SLICE_EVEN_ENABLE_MASK                                                              0x0F000000L
+#define DB_COUNT_CONTROL__SLICE_ODD_ENABLE_MASK                                                               0xF0000000L
+//DB_VIEWPORT_CONTROL
+#define DB_VIEWPORT_CONTROL__DISABLE_VIEWPORT_CLAMP__SHIFT                                                    0x0
+#define DB_VIEWPORT_CONTROL__DISABLE_VIEWPORT_CLAMP_MASK                                                      0x00000001L
+//DB_SPI_VRS_CENTER_LOCATION
+#define DB_SPI_VRS_CENTER_LOCATION__CENTER_X_OFFSET_1X1__SHIFT                                                0x0
+#define DB_SPI_VRS_CENTER_LOCATION__CENTER_Y_OFFSET_1X1__SHIFT                                                0x4
+#define DB_SPI_VRS_CENTER_LOCATION__CENTER_X_OFFSET_2X1__SHIFT                                                0x8
+#define DB_SPI_VRS_CENTER_LOCATION__CENTER_Y_OFFSET_2X1__SHIFT                                                0xc
+#define DB_SPI_VRS_CENTER_LOCATION__CENTER_X_OFFSET_1X2__SHIFT                                                0x10
+#define DB_SPI_VRS_CENTER_LOCATION__CENTER_Y_OFFSET_1X2__SHIFT                                                0x14
+#define DB_SPI_VRS_CENTER_LOCATION__CENTER_X_OFFSET_2X2__SHIFT                                                0x18
+#define DB_SPI_VRS_CENTER_LOCATION__CENTER_Y_OFFSET_2X2__SHIFT                                                0x1c
+#define DB_SPI_VRS_CENTER_LOCATION__CENTER_X_OFFSET_1X1_MASK                                                  0x0000000FL
+#define DB_SPI_VRS_CENTER_LOCATION__CENTER_Y_OFFSET_1X1_MASK                                                  0x000000F0L
+#define DB_SPI_VRS_CENTER_LOCATION__CENTER_X_OFFSET_2X1_MASK                                                  0x00000F00L
+#define DB_SPI_VRS_CENTER_LOCATION__CENTER_Y_OFFSET_2X1_MASK                                                  0x0000F000L
+#define DB_SPI_VRS_CENTER_LOCATION__CENTER_X_OFFSET_1X2_MASK                                                  0x000F0000L
+#define DB_SPI_VRS_CENTER_LOCATION__CENTER_Y_OFFSET_1X2_MASK                                                  0x00F00000L
+#define DB_SPI_VRS_CENTER_LOCATION__CENTER_X_OFFSET_2X2_MASK                                                  0x0F000000L
+#define DB_SPI_VRS_CENTER_LOCATION__CENTER_Y_OFFSET_2X2_MASK                                                  0xF0000000L
+//DB_SHADER_CONTROL
+#define DB_SHADER_CONTROL__Z_EXPORT_ENABLE__SHIFT                                                             0x0
+#define DB_SHADER_CONTROL__STENCIL_TEST_VAL_EXPORT_ENABLE__SHIFT                                              0x1
+#define DB_SHADER_CONTROL__STENCIL_OP_VAL_EXPORT_ENABLE__SHIFT                                                0x2
+#define DB_SHADER_CONTROL__Z_ORDER__SHIFT                                                                     0x4
+#define DB_SHADER_CONTROL__KILL_ENABLE__SHIFT                                                                 0x6
+#define DB_SHADER_CONTROL__COVERAGE_TO_MASK_ENABLE__SHIFT                                                     0x7
+#define DB_SHADER_CONTROL__MASK_EXPORT_ENABLE__SHIFT                                                          0x8
+#define DB_SHADER_CONTROL__EXEC_ON_HIER_FAIL__SHIFT                                                           0x9
+#define DB_SHADER_CONTROL__EXEC_ON_NOOP__SHIFT                                                                0xa
+#define DB_SHADER_CONTROL__ALPHA_TO_MASK_DISABLE__SHIFT                                                       0xb
+#define DB_SHADER_CONTROL__DEPTH_BEFORE_SHADER__SHIFT                                                         0xc
+#define DB_SHADER_CONTROL__CONSERVATIVE_Z_EXPORT__SHIFT                                                       0xd
+#define DB_SHADER_CONTROL__DUAL_QUAD_DISABLE__SHIFT                                                           0xf
+#define DB_SHADER_CONTROL__PRIMITIVE_ORDERED_PIXEL_SHADER__SHIFT                                              0x10
+#define DB_SHADER_CONTROL__PRE_SHADER_DEPTH_COVERAGE_ENABLE__SHIFT                                            0x17
+#define DB_SHADER_CONTROL__OREO_BLEND_ENABLE__SHIFT                                                           0x18
+#define DB_SHADER_CONTROL__OVERRIDE_INTRINSIC_RATE_ENABLE__SHIFT                                              0x19
+#define DB_SHADER_CONTROL__OVERRIDE_INTRINSIC_RATE__SHIFT                                                     0x1a
+#define DB_SHADER_CONTROL__Z_EXPORT_ENABLE_MASK                                                               0x00000001L
+#define DB_SHADER_CONTROL__STENCIL_TEST_VAL_EXPORT_ENABLE_MASK                                                0x00000002L
+#define DB_SHADER_CONTROL__STENCIL_OP_VAL_EXPORT_ENABLE_MASK                                                  0x00000004L
+#define DB_SHADER_CONTROL__Z_ORDER_MASK                                                                       0x00000030L
+#define DB_SHADER_CONTROL__KILL_ENABLE_MASK                                                                   0x00000040L
+#define DB_SHADER_CONTROL__COVERAGE_TO_MASK_ENABLE_MASK                                                       0x00000080L
+#define DB_SHADER_CONTROL__MASK_EXPORT_ENABLE_MASK                                                            0x00000100L
+#define DB_SHADER_CONTROL__EXEC_ON_HIER_FAIL_MASK                                                             0x00000200L
+#define DB_SHADER_CONTROL__EXEC_ON_NOOP_MASK                                                                  0x00000400L
+#define DB_SHADER_CONTROL__ALPHA_TO_MASK_DISABLE_MASK                                                         0x00000800L
+#define DB_SHADER_CONTROL__DEPTH_BEFORE_SHADER_MASK                                                           0x00001000L
+#define DB_SHADER_CONTROL__CONSERVATIVE_Z_EXPORT_MASK                                                         0x00006000L
+#define DB_SHADER_CONTROL__DUAL_QUAD_DISABLE_MASK                                                             0x00008000L
+#define DB_SHADER_CONTROL__PRIMITIVE_ORDERED_PIXEL_SHADER_MASK                                                0x00010000L
+#define DB_SHADER_CONTROL__PRE_SHADER_DEPTH_COVERAGE_ENABLE_MASK                                              0x00800000L
+#define DB_SHADER_CONTROL__OREO_BLEND_ENABLE_MASK                                                             0x01000000L
+#define DB_SHADER_CONTROL__OVERRIDE_INTRINSIC_RATE_ENABLE_MASK                                                0x02000000L
+#define DB_SHADER_CONTROL__OVERRIDE_INTRINSIC_RATE_MASK                                                       0x1C000000L
+//DB_DEPTH_CONTROL
+#define DB_DEPTH_CONTROL__STENCIL_ENABLE__SHIFT                                                               0x0
+#define DB_DEPTH_CONTROL__Z_ENABLE__SHIFT                                                                     0x1
+#define DB_DEPTH_CONTROL__Z_WRITE_ENABLE__SHIFT                                                               0x2
+#define DB_DEPTH_CONTROL__DEPTH_BOUNDS_ENABLE__SHIFT                                                          0x3
+#define DB_DEPTH_CONTROL__ZFUNC__SHIFT                                                                        0x4
+#define DB_DEPTH_CONTROL__BACKFACE_ENABLE__SHIFT                                                              0x7
+#define DB_DEPTH_CONTROL__STENCILFUNC__SHIFT                                                                  0x8
+#define DB_DEPTH_CONTROL__STENCILFUNC_BF__SHIFT                                                               0x14
+#define DB_DEPTH_CONTROL__RESERVED_FIELD_30__SHIFT                                                            0x1e
+#define DB_DEPTH_CONTROL__RESERVED_FIELD_31__SHIFT                                                            0x1f
+#define DB_DEPTH_CONTROL__STENCIL_ENABLE_MASK                                                                 0x00000001L
+#define DB_DEPTH_CONTROL__Z_ENABLE_MASK                                                                       0x00000002L
+#define DB_DEPTH_CONTROL__Z_WRITE_ENABLE_MASK                                                                 0x00000004L
+#define DB_DEPTH_CONTROL__DEPTH_BOUNDS_ENABLE_MASK                                                            0x00000008L
+#define DB_DEPTH_CONTROL__ZFUNC_MASK                                                                          0x00000070L
+#define DB_DEPTH_CONTROL__BACKFACE_ENABLE_MASK                                                                0x00000080L
+#define DB_DEPTH_CONTROL__STENCILFUNC_MASK                                                                    0x00000700L
+#define DB_DEPTH_CONTROL__STENCILFUNC_BF_MASK                                                                 0x00700000L
+#define DB_DEPTH_CONTROL__RESERVED_FIELD_30_MASK                                                              0x40000000L
+#define DB_DEPTH_CONTROL__RESERVED_FIELD_31_MASK                                                              0x80000000L
+//DB_STENCIL_CONTROL
+#define DB_STENCIL_CONTROL__STENCILFAIL__SHIFT                                                                0x0
+#define DB_STENCIL_CONTROL__STENCILZPASS__SHIFT                                                               0x4
+#define DB_STENCIL_CONTROL__STENCILZFAIL__SHIFT                                                               0x8
+#define DB_STENCIL_CONTROL__STENCILFAIL_BF__SHIFT                                                             0xc
+#define DB_STENCIL_CONTROL__STENCILZPASS_BF__SHIFT                                                            0x10
+#define DB_STENCIL_CONTROL__STENCILZFAIL_BF__SHIFT                                                            0x14
+#define DB_STENCIL_CONTROL__STENCILFAIL_MASK                                                                  0x0000000FL
+#define DB_STENCIL_CONTROL__STENCILZPASS_MASK                                                                 0x000000F0L
+#define DB_STENCIL_CONTROL__STENCILZFAIL_MASK                                                                 0x00000F00L
+#define DB_STENCIL_CONTROL__STENCILFAIL_BF_MASK                                                               0x0000F000L
+#define DB_STENCIL_CONTROL__STENCILZPASS_BF_MASK                                                              0x000F0000L
+#define DB_STENCIL_CONTROL__STENCILZFAIL_BF_MASK                                                              0x00F00000L
+//DB_EQAA
+#define DB_EQAA__RESERVED_FIELD_1__SHIFT                                                                      0x0
+#define DB_EQAA__MASK_EXPORT_NUM_SAMPLES__SHIFT                                                               0x8
+#define DB_EQAA__ALPHA_TO_MASK_NUM_SAMPLES__SHIFT                                                             0xc
+#define DB_EQAA__HIGH_QUALITY_INTERSECTIONS__SHIFT                                                            0x10
+#define DB_EQAA__RESERVED_FIELD_17__SHIFT                                                                     0x11
+#define DB_EQAA__RESERVED_FIELD_18__SHIFT                                                                     0x12
+#define DB_EQAA__RESERVED_FIELD_19__SHIFT                                                                     0x13
+#define DB_EQAA__STATIC_ANCHOR_ASSOCIATIONS__SHIFT                                                            0x14
+#define DB_EQAA__RESERVED_FIELD_21__SHIFT                                                                     0x15
+#define DB_EQAA__OVERRASTERIZATION_AMOUNT__SHIFT                                                              0x18
+#define DB_EQAA__ENABLE_POSTZ_OVERRASTERIZATION__SHIFT                                                        0x1b
+#define DB_EQAA__RESERVED_FIELD_1_MASK                                                                        0x00000007L
+#define DB_EQAA__MASK_EXPORT_NUM_SAMPLES_MASK                                                                 0x00000700L
+#define DB_EQAA__ALPHA_TO_MASK_NUM_SAMPLES_MASK                                                               0x00007000L
+#define DB_EQAA__HIGH_QUALITY_INTERSECTIONS_MASK                                                              0x00010000L
+#define DB_EQAA__RESERVED_FIELD_17_MASK                                                                       0x00020000L
+#define DB_EQAA__RESERVED_FIELD_18_MASK                                                                       0x00040000L
+#define DB_EQAA__RESERVED_FIELD_19_MASK                                                                       0x00080000L
+#define DB_EQAA__STATIC_ANCHOR_ASSOCIATIONS_MASK                                                              0x00100000L
+#define DB_EQAA__RESERVED_FIELD_21_MASK                                                                       0x00200000L
+#define DB_EQAA__OVERRASTERIZATION_AMOUNT_MASK                                                                0x07000000L
+#define DB_EQAA__ENABLE_POSTZ_OVERRASTERIZATION_MASK                                                          0x08000000L
+//DB_ALPHA_TO_MASK
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_ENABLE__SHIFT                                                         0x0
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET0__SHIFT                                                        0x8
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET1__SHIFT                                                        0xa
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET2__SHIFT                                                        0xc
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET3__SHIFT                                                        0xe
+#define DB_ALPHA_TO_MASK__OFFSET_ROUND__SHIFT                                                                 0x10
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_ENABLE_MASK                                                           0x00000001L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET0_MASK                                                          0x00000300L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET1_MASK                                                          0x00000C00L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET2_MASK                                                          0x00003000L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET3_MASK                                                          0x0000C000L
+#define DB_ALPHA_TO_MASK__OFFSET_ROUND_MASK                                                                   0x00010000L
+//TA_BC_BASE_ADDR
+#define TA_BC_BASE_ADDR__ADDRESS__SHIFT                                                                       0x0
+#define TA_BC_BASE_ADDR__ADDRESS_MASK                                                                         0xFFFFFFFFL
+//TA_BC_BASE_ADDR_HI
+#define TA_BC_BASE_ADDR_HI__ADDRESS__SHIFT                                                                    0x0
+#define TA_BC_BASE_ADDR_HI__ADDRESS_MASK                                                                      0x000000FFL
+//DB_STENCIL_REF
+#define DB_STENCIL_REF__TESTVAL__SHIFT                                                                        0x0
+#define DB_STENCIL_REF__TESTVAL_BF__SHIFT                                                                     0x8
+#define DB_STENCIL_REF__TESTVAL_MASK                                                                          0x000000FFL
+#define DB_STENCIL_REF__TESTVAL_BF_MASK                                                                       0x0000FF00L
+//DB_STENCIL_OPVAL
+#define DB_STENCIL_OPVAL__OPVAL__SHIFT                                                                        0x0
+#define DB_STENCIL_OPVAL__OPVAL_BF__SHIFT                                                                     0x8
+#define DB_STENCIL_OPVAL__OPVAL_MASK                                                                          0x000000FFL
+#define DB_STENCIL_OPVAL__OPVAL_BF_MASK                                                                       0x0000FF00L
+//DB_STENCIL_READ_MASK
+#define DB_STENCIL_READ_MASK__TESTMASK__SHIFT                                                                 0x0
+#define DB_STENCIL_READ_MASK__TESTMASK_BF__SHIFT                                                              0x8
+#define DB_STENCIL_READ_MASK__TESTMASK_MASK                                                                   0x000000FFL
+#define DB_STENCIL_READ_MASK__TESTMASK_BF_MASK                                                                0x0000FF00L
+//DB_STENCIL_WRITE_MASK
+#define DB_STENCIL_WRITE_MASK__WRITEMASK__SHIFT                                                               0x0
+#define DB_STENCIL_WRITE_MASK__WRITEMASK_BF__SHIFT                                                            0x8
+#define DB_STENCIL_WRITE_MASK__WRITEMASK_MASK                                                                 0x000000FFL
+#define DB_STENCIL_WRITE_MASK__WRITEMASK_BF_MASK                                                              0x0000FF00L
+//SC_MEM_TEMPORAL
+#define SC_MEM_TEMPORAL__VRS_TEMPORAL_READ__SHIFT                                                             0x0
+#define SC_MEM_TEMPORAL__VRS_TEMPORAL_WRITE__SHIFT                                                            0x3
+#define SC_MEM_TEMPORAL__HIZ_TEMPORAL_READ__SHIFT                                                             0x6
+#define SC_MEM_TEMPORAL__HIZ_TEMPORAL_WRITE__SHIFT                                                            0x9
+#define SC_MEM_TEMPORAL__HIS_TEMPORAL_READ__SHIFT                                                             0xc
+#define SC_MEM_TEMPORAL__HIS_TEMPORAL_WRITE__SHIFT                                                            0xf
+#define SC_MEM_TEMPORAL__VRS_TEMPORAL_READ_MASK                                                               0x00000007L
+#define SC_MEM_TEMPORAL__VRS_TEMPORAL_WRITE_MASK                                                              0x00000038L
+#define SC_MEM_TEMPORAL__HIZ_TEMPORAL_READ_MASK                                                               0x000001C0L
+#define SC_MEM_TEMPORAL__HIZ_TEMPORAL_WRITE_MASK                                                              0x00000E00L
+#define SC_MEM_TEMPORAL__HIS_TEMPORAL_READ_MASK                                                               0x00007000L
+#define SC_MEM_TEMPORAL__HIS_TEMPORAL_WRITE_MASK                                                              0x00038000L
+//SC_MEM_SPEC_READ
+#define SC_MEM_SPEC_READ__VRS_SPECULATIVE_READ__SHIFT                                                         0x0
+#define SC_MEM_SPEC_READ__HIZ_SPECULATIVE_READ__SHIFT                                                         0x2
+#define SC_MEM_SPEC_READ__HIS_SPECULATIVE_READ__SHIFT                                                         0x4
+#define SC_MEM_SPEC_READ__VRS_SPECULATIVE_READ_MASK                                                           0x00000003L
+#define SC_MEM_SPEC_READ__HIZ_SPECULATIVE_READ_MASK                                                           0x0000000CL
+#define SC_MEM_SPEC_READ__HIS_SPECULATIVE_READ_MASK                                                           0x00000030L
+//PA_SC_VPORT_0_TL
+#define PA_SC_VPORT_0_TL__TL_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_0_TL__TL_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_0_TL__TL_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_0_TL__TL_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_0_BR
+#define PA_SC_VPORT_0_BR__BR_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_0_BR__BR_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_0_BR__BR_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_0_BR__BR_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_1_TL
+#define PA_SC_VPORT_1_TL__TL_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_1_TL__TL_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_1_TL__TL_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_1_TL__TL_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_1_BR
+#define PA_SC_VPORT_1_BR__BR_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_1_BR__BR_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_1_BR__BR_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_1_BR__BR_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_2_TL
+#define PA_SC_VPORT_2_TL__TL_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_2_TL__TL_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_2_TL__TL_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_2_TL__TL_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_2_BR
+#define PA_SC_VPORT_2_BR__BR_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_2_BR__BR_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_2_BR__BR_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_2_BR__BR_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_3_TL
+#define PA_SC_VPORT_3_TL__TL_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_3_TL__TL_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_3_TL__TL_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_3_TL__TL_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_3_BR
+#define PA_SC_VPORT_3_BR__BR_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_3_BR__BR_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_3_BR__BR_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_3_BR__BR_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_4_TL
+#define PA_SC_VPORT_4_TL__TL_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_4_TL__TL_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_4_TL__TL_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_4_TL__TL_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_4_BR
+#define PA_SC_VPORT_4_BR__BR_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_4_BR__BR_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_4_BR__BR_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_4_BR__BR_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_5_TL
+#define PA_SC_VPORT_5_TL__TL_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_5_TL__TL_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_5_TL__TL_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_5_TL__TL_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_5_BR
+#define PA_SC_VPORT_5_BR__BR_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_5_BR__BR_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_5_BR__BR_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_5_BR__BR_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_6_TL
+#define PA_SC_VPORT_6_TL__TL_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_6_TL__TL_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_6_TL__TL_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_6_TL__TL_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_6_BR
+#define PA_SC_VPORT_6_BR__BR_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_6_BR__BR_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_6_BR__BR_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_6_BR__BR_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_7_TL
+#define PA_SC_VPORT_7_TL__TL_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_7_TL__TL_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_7_TL__TL_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_7_TL__TL_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_7_BR
+#define PA_SC_VPORT_7_BR__BR_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_7_BR__BR_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_7_BR__BR_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_7_BR__BR_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_8_TL
+#define PA_SC_VPORT_8_TL__TL_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_8_TL__TL_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_8_TL__TL_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_8_TL__TL_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_8_BR
+#define PA_SC_VPORT_8_BR__BR_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_8_BR__BR_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_8_BR__BR_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_8_BR__BR_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_9_TL
+#define PA_SC_VPORT_9_TL__TL_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_9_TL__TL_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_9_TL__TL_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_9_TL__TL_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_9_BR
+#define PA_SC_VPORT_9_BR__BR_X__SHIFT                                                                         0x0
+#define PA_SC_VPORT_9_BR__BR_Y__SHIFT                                                                         0x10
+#define PA_SC_VPORT_9_BR__BR_X_MASK                                                                           0x0000FFFFL
+#define PA_SC_VPORT_9_BR__BR_Y_MASK                                                                           0xFFFF0000L
+//PA_SC_VPORT_10_TL
+#define PA_SC_VPORT_10_TL__TL_X__SHIFT                                                                        0x0
+#define PA_SC_VPORT_10_TL__TL_Y__SHIFT                                                                        0x10
+#define PA_SC_VPORT_10_TL__TL_X_MASK                                                                          0x0000FFFFL
+#define PA_SC_VPORT_10_TL__TL_Y_MASK                                                                          0xFFFF0000L
+//PA_SC_VPORT_10_BR
+#define PA_SC_VPORT_10_BR__BR_X__SHIFT                                                                        0x0
+#define PA_SC_VPORT_10_BR__BR_Y__SHIFT                                                                        0x10
+#define PA_SC_VPORT_10_BR__BR_X_MASK                                                                          0x0000FFFFL
+#define PA_SC_VPORT_10_BR__BR_Y_MASK                                                                          0xFFFF0000L
+//PA_SC_VPORT_11_TL
+#define PA_SC_VPORT_11_TL__TL_X__SHIFT                                                                        0x0
+#define PA_SC_VPORT_11_TL__TL_Y__SHIFT                                                                        0x10
+#define PA_SC_VPORT_11_TL__TL_X_MASK                                                                          0x0000FFFFL
+#define PA_SC_VPORT_11_TL__TL_Y_MASK                                                                          0xFFFF0000L
+//PA_SC_VPORT_11_BR
+#define PA_SC_VPORT_11_BR__BR_X__SHIFT                                                                        0x0
+#define PA_SC_VPORT_11_BR__BR_Y__SHIFT                                                                        0x10
+#define PA_SC_VPORT_11_BR__BR_X_MASK                                                                          0x0000FFFFL
+#define PA_SC_VPORT_11_BR__BR_Y_MASK                                                                          0xFFFF0000L
+//PA_SC_VPORT_12_TL
+#define PA_SC_VPORT_12_TL__TL_X__SHIFT                                                                        0x0
+#define PA_SC_VPORT_12_TL__TL_Y__SHIFT                                                                        0x10
+#define PA_SC_VPORT_12_TL__TL_X_MASK                                                                          0x0000FFFFL
+#define PA_SC_VPORT_12_TL__TL_Y_MASK                                                                          0xFFFF0000L
+//PA_SC_VPORT_12_BR
+#define PA_SC_VPORT_12_BR__BR_X__SHIFT                                                                        0x0
+#define PA_SC_VPORT_12_BR__BR_Y__SHIFT                                                                        0x10
+#define PA_SC_VPORT_12_BR__BR_X_MASK                                                                          0x0000FFFFL
+#define PA_SC_VPORT_12_BR__BR_Y_MASK                                                                          0xFFFF0000L
+//PA_SC_VPORT_13_TL
+#define PA_SC_VPORT_13_TL__TL_X__SHIFT                                                                        0x0
+#define PA_SC_VPORT_13_TL__TL_Y__SHIFT                                                                        0x10
+#define PA_SC_VPORT_13_TL__TL_X_MASK                                                                          0x0000FFFFL
+#define PA_SC_VPORT_13_TL__TL_Y_MASK                                                                          0xFFFF0000L
+//PA_SC_VPORT_13_BR
+#define PA_SC_VPORT_13_BR__BR_X__SHIFT                                                                        0x0
+#define PA_SC_VPORT_13_BR__BR_Y__SHIFT                                                                        0x10
+#define PA_SC_VPORT_13_BR__BR_X_MASK                                                                          0x0000FFFFL
+#define PA_SC_VPORT_13_BR__BR_Y_MASK                                                                          0xFFFF0000L
+//PA_SC_VPORT_14_TL
+#define PA_SC_VPORT_14_TL__TL_X__SHIFT                                                                        0x0
+#define PA_SC_VPORT_14_TL__TL_Y__SHIFT                                                                        0x10
+#define PA_SC_VPORT_14_TL__TL_X_MASK                                                                          0x0000FFFFL
+#define PA_SC_VPORT_14_TL__TL_Y_MASK                                                                          0xFFFF0000L
+//PA_SC_VPORT_14_BR
+#define PA_SC_VPORT_14_BR__BR_X__SHIFT                                                                        0x0
+#define PA_SC_VPORT_14_BR__BR_Y__SHIFT                                                                        0x10
+#define PA_SC_VPORT_14_BR__BR_X_MASK                                                                          0x0000FFFFL
+#define PA_SC_VPORT_14_BR__BR_Y_MASK                                                                          0xFFFF0000L
+//PA_SC_VPORT_15_TL
+#define PA_SC_VPORT_15_TL__TL_X__SHIFT                                                                        0x0
+#define PA_SC_VPORT_15_TL__TL_Y__SHIFT                                                                        0x10
+#define PA_SC_VPORT_15_TL__TL_X_MASK                                                                          0x0000FFFFL
+#define PA_SC_VPORT_15_TL__TL_Y_MASK                                                                          0xFFFF0000L
+//PA_SC_VPORT_15_BR
+#define PA_SC_VPORT_15_BR__BR_X__SHIFT                                                                        0x0
+#define PA_SC_VPORT_15_BR__BR_Y__SHIFT                                                                        0x10
+#define PA_SC_VPORT_15_BR__BR_X_MASK                                                                          0x0000FFFFL
+#define PA_SC_VPORT_15_BR__BR_Y_MASK                                                                          0xFFFF0000L
+//PA_SC_SCREEN_SCISSOR_TL
+#define PA_SC_SCREEN_SCISSOR_TL__TL_X__SHIFT                                                                  0x0
+#define PA_SC_SCREEN_SCISSOR_TL__TL_Y__SHIFT                                                                  0x10
+#define PA_SC_SCREEN_SCISSOR_TL__TL_X_MASK                                                                    0x0000FFFFL
+#define PA_SC_SCREEN_SCISSOR_TL__TL_Y_MASK                                                                    0xFFFF0000L
+//PA_SC_SCREEN_SCISSOR_BR
+#define PA_SC_SCREEN_SCISSOR_BR__BR_X__SHIFT                                                                  0x0
+#define PA_SC_SCREEN_SCISSOR_BR__BR_Y__SHIFT                                                                  0x10
+#define PA_SC_SCREEN_SCISSOR_BR__BR_X_MASK                                                                    0x0000FFFFL
+#define PA_SC_SCREEN_SCISSOR_BR__BR_Y_MASK                                                                    0xFFFF0000L
+//PA_SC_WINDOW_OFFSET
+#define PA_SC_WINDOW_OFFSET__WINDOW_X_OFFSET__SHIFT                                                           0x0
+#define PA_SC_WINDOW_OFFSET__WINDOW_Y_OFFSET__SHIFT                                                           0x10
+#define PA_SC_WINDOW_OFFSET__WINDOW_X_OFFSET_MASK                                                             0x0000FFFFL
+#define PA_SC_WINDOW_OFFSET__WINDOW_Y_OFFSET_MASK                                                             0xFFFF0000L
+//PA_SC_WINDOW_SCISSOR_TL
+#define PA_SC_WINDOW_SCISSOR_TL__TL_X__SHIFT                                                                  0x0
+#define PA_SC_WINDOW_SCISSOR_TL__TL_Y__SHIFT                                                                  0x10
+#define PA_SC_WINDOW_SCISSOR_TL__TL_X_MASK                                                                    0x0000FFFFL
+#define PA_SC_WINDOW_SCISSOR_TL__TL_Y_MASK                                                                    0xFFFF0000L
+//PA_SC_WINDOW_SCISSOR_BR
+#define PA_SC_WINDOW_SCISSOR_BR__BR_X__SHIFT                                                                  0x0
+#define PA_SC_WINDOW_SCISSOR_BR__BR_Y__SHIFT                                                                  0x10
+#define PA_SC_WINDOW_SCISSOR_BR__BR_X_MASK                                                                    0x0000FFFFL
+#define PA_SC_WINDOW_SCISSOR_BR__BR_Y_MASK                                                                    0xFFFF0000L
+//PA_SC_CLIPRECT_RULE
+#define PA_SC_CLIPRECT_RULE__CLIP_RULE__SHIFT                                                                 0x0
+#define PA_SC_CLIPRECT_RULE__CLIP_RULE_MASK                                                                   0x0000FFFFL
+//PA_SC_CLIPRECT_0_TL
+#define PA_SC_CLIPRECT_0_TL__TL_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_0_TL__TL_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_0_TL__TL_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_0_TL__TL_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_0_BR
+#define PA_SC_CLIPRECT_0_BR__BR_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_0_BR__BR_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_0_BR__BR_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_0_BR__BR_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_1_TL
+#define PA_SC_CLIPRECT_1_TL__TL_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_1_TL__TL_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_1_TL__TL_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_1_TL__TL_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_1_BR
+#define PA_SC_CLIPRECT_1_BR__BR_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_1_BR__BR_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_1_BR__BR_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_1_BR__BR_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_2_TL
+#define PA_SC_CLIPRECT_2_TL__TL_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_2_TL__TL_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_2_TL__TL_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_2_TL__TL_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_2_BR
+#define PA_SC_CLIPRECT_2_BR__BR_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_2_BR__BR_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_2_BR__BR_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_2_BR__BR_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_3_TL
+#define PA_SC_CLIPRECT_3_TL__TL_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_3_TL__TL_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_3_TL__TL_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_3_TL__TL_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_3_BR
+#define PA_SC_CLIPRECT_3_BR__BR_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_3_BR__BR_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_3_BR__BR_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_3_BR__BR_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_EDGERULE
+#define PA_SC_EDGERULE__ER_TRI__SHIFT                                                                         0x0
+#define PA_SC_EDGERULE__ER_POINT__SHIFT                                                                       0x4
+#define PA_SC_EDGERULE__ER_RECT__SHIFT                                                                        0x8
+#define PA_SC_EDGERULE__ER_LINE_LR__SHIFT                                                                     0xc
+#define PA_SC_EDGERULE__ER_LINE_RL__SHIFT                                                                     0x12
+#define PA_SC_EDGERULE__ER_LINE_TB__SHIFT                                                                     0x18
+#define PA_SC_EDGERULE__ER_LINE_BT__SHIFT                                                                     0x1c
+#define PA_SC_EDGERULE__ER_TRI_MASK                                                                           0x0000000FL
+#define PA_SC_EDGERULE__ER_POINT_MASK                                                                         0x000000F0L
+#define PA_SC_EDGERULE__ER_RECT_MASK                                                                          0x00000F00L
+#define PA_SC_EDGERULE__ER_LINE_LR_MASK                                                                       0x0003F000L
+#define PA_SC_EDGERULE__ER_LINE_RL_MASK                                                                       0x00FC0000L
+#define PA_SC_EDGERULE__ER_LINE_TB_MASK                                                                       0x0F000000L
+#define PA_SC_EDGERULE__ER_LINE_BT_MASK                                                                       0xF0000000L
+//PA_SU_HARDWARE_SCREEN_OFFSET
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_X__SHIFT                                               0x0
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_Y__SHIFT                                               0x10
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_X_MASK                                                 0x00000FFFL
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_Y_MASK                                                 0x0FFF0000L
+//PA_SC_GENERIC_SCISSOR_TL
+#define PA_SC_GENERIC_SCISSOR_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_GENERIC_SCISSOR_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_GENERIC_SCISSOR_TL__TL_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_GENERIC_SCISSOR_TL__TL_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_GENERIC_SCISSOR_BR
+#define PA_SC_GENERIC_SCISSOR_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_GENERIC_SCISSOR_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_GENERIC_SCISSOR_BR__BR_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_GENERIC_SCISSOR_BR__BR_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_0_TL
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_0_BR
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_1_TL
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_1_BR
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_2_TL
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_2_BR
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_3_TL
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_3_BR
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_4_TL
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_4_BR
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_5_TL
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_5_BR
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_6_TL
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_6_BR
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_7_TL
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_7_BR
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_8_TL
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_8_BR
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_9_TL
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_9_BR
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_X_MASK                                                                   0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_Y_MASK                                                                   0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_10_TL
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_X_MASK                                                                  0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_Y_MASK                                                                  0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_10_BR
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_X_MASK                                                                  0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_Y_MASK                                                                  0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_11_TL
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_X_MASK                                                                  0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_Y_MASK                                                                  0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_11_BR
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_X_MASK                                                                  0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_Y_MASK                                                                  0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_12_TL
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_X_MASK                                                                  0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_Y_MASK                                                                  0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_12_BR
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_X_MASK                                                                  0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_Y_MASK                                                                  0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_13_TL
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_X_MASK                                                                  0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_Y_MASK                                                                  0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_13_BR
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_X_MASK                                                                  0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_Y_MASK                                                                  0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_14_TL
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_X_MASK                                                                  0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_Y_MASK                                                                  0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_14_BR
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_X_MASK                                                                  0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_Y_MASK                                                                  0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_15_TL
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_X_MASK                                                                  0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_Y_MASK                                                                  0xFFFF0000L
+//PA_SC_VPORT_SCISSOR_15_BR
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_X_MASK                                                                  0x0000FFFFL
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_Y_MASK                                                                  0xFFFF0000L
+//PA_CL_UCP_0_X
+#define PA_CL_UCP_0_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_0_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_0_Y
+#define PA_CL_UCP_0_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_0_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_0_Z
+#define PA_CL_UCP_0_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_0_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_0_W
+#define PA_CL_UCP_0_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_0_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_1_X
+#define PA_CL_UCP_1_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_1_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_1_Y
+#define PA_CL_UCP_1_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_1_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_1_Z
+#define PA_CL_UCP_1_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_1_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_1_W
+#define PA_CL_UCP_1_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_1_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_2_X
+#define PA_CL_UCP_2_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_2_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_2_Y
+#define PA_CL_UCP_2_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_2_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_2_Z
+#define PA_CL_UCP_2_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_2_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_2_W
+#define PA_CL_UCP_2_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_2_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_3_X
+#define PA_CL_UCP_3_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_3_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_3_Y
+#define PA_CL_UCP_3_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_3_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_3_Z
+#define PA_CL_UCP_3_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_3_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_3_W
+#define PA_CL_UCP_3_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_3_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_4_X
+#define PA_CL_UCP_4_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_4_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_4_Y
+#define PA_CL_UCP_4_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_4_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_4_Z
+#define PA_CL_UCP_4_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_4_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_4_W
+#define PA_CL_UCP_4_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_4_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_5_X
+#define PA_CL_UCP_5_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_5_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_5_Y
+#define PA_CL_UCP_5_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_5_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_5_Z
+#define PA_CL_UCP_5_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_5_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_5_W
+#define PA_CL_UCP_5_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_5_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_PROG_NEAR_CLIP_Z
+#define PA_CL_PROG_NEAR_CLIP_Z__DATA_REGISTER__SHIFT                                                          0x0
+#define PA_CL_PROG_NEAR_CLIP_Z__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
+//PA_RATE_CNTL
+#define PA_RATE_CNTL__VERTEX_RATE__SHIFT                                                                      0x0
+#define PA_RATE_CNTL__PRIM_RATE__SHIFT                                                                        0x4
+#define PA_RATE_CNTL__VERTEX_RATE_MASK                                                                        0x0000000FL
+#define PA_RATE_CNTL__PRIM_RATE_MASK                                                                          0x000000F0L
+//PA_SC_RASTER_CONFIG
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR0__SHIFT                                                               0x0
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR1__SHIFT                                                               0x2
+#define PA_SC_RASTER_CONFIG__RB_XSEL2__SHIFT                                                                  0x4
+#define PA_SC_RASTER_CONFIG__RB_XSEL__SHIFT                                                                   0x6
+#define PA_SC_RASTER_CONFIG__RB_YSEL__SHIFT                                                                   0x7
+#define PA_SC_RASTER_CONFIG__PKR_MAP__SHIFT                                                                   0x8
+#define PA_SC_RASTER_CONFIG__PKR_XSEL__SHIFT                                                                  0xa
+#define PA_SC_RASTER_CONFIG__PKR_YSEL__SHIFT                                                                  0xc
+#define PA_SC_RASTER_CONFIG__PKR_XSEL2__SHIFT                                                                 0xe
+#define PA_SC_RASTER_CONFIG__SC_MAP__SHIFT                                                                    0x10
+#define PA_SC_RASTER_CONFIG__SC_XSEL__SHIFT                                                                   0x12
+#define PA_SC_RASTER_CONFIG__SC_YSEL__SHIFT                                                                   0x14
+#define PA_SC_RASTER_CONFIG__SE_MAP__SHIFT                                                                    0x18
+#define PA_SC_RASTER_CONFIG__SE_XSEL__SHIFT                                                                   0x1a
+#define PA_SC_RASTER_CONFIG__SE_YSEL__SHIFT                                                                   0x1c
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR0_MASK                                                                 0x00000003L
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR1_MASK                                                                 0x0000000CL
+#define PA_SC_RASTER_CONFIG__RB_XSEL2_MASK                                                                    0x00000030L
+#define PA_SC_RASTER_CONFIG__RB_XSEL_MASK                                                                     0x00000040L
+#define PA_SC_RASTER_CONFIG__RB_YSEL_MASK                                                                     0x00000080L
+#define PA_SC_RASTER_CONFIG__PKR_MAP_MASK                                                                     0x00000300L
+#define PA_SC_RASTER_CONFIG__PKR_XSEL_MASK                                                                    0x00000C00L
+#define PA_SC_RASTER_CONFIG__PKR_YSEL_MASK                                                                    0x00003000L
+#define PA_SC_RASTER_CONFIG__PKR_XSEL2_MASK                                                                   0x0000C000L
+#define PA_SC_RASTER_CONFIG__SC_MAP_MASK                                                                      0x00030000L
+#define PA_SC_RASTER_CONFIG__SC_XSEL_MASK                                                                     0x000C0000L
+#define PA_SC_RASTER_CONFIG__SC_YSEL_MASK                                                                     0x00300000L
+#define PA_SC_RASTER_CONFIG__SE_MAP_MASK                                                                      0x03000000L
+#define PA_SC_RASTER_CONFIG__SE_XSEL_MASK                                                                     0x0C000000L
+#define PA_SC_RASTER_CONFIG__SE_YSEL_MASK                                                                     0x30000000L
+//PA_SC_RASTER_CONFIG_1
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_MAP__SHIFT                                                             0x0
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_XSEL__SHIFT                                                            0x2
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_YSEL__SHIFT                                                            0x4
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_MAP_MASK                                                               0x00000003L
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_XSEL_MASK                                                              0x0000000CL
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_YSEL_MASK                                                              0x00000030L
+//PA_SC_SCREEN_EXTENT_CONTROL
+#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_EVEN_ENABLE__SHIFT                                                 0x0
+#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_ODD_ENABLE__SHIFT                                                  0x2
+#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_EVEN_ENABLE_MASK                                                   0x00000003L
+#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_ODD_ENABLE_MASK                                                    0x0000000CL
+//PA_SC_TILE_STEERING_OVERRIDE
+#define PA_SC_TILE_STEERING_OVERRIDE__ENABLE__SHIFT                                                           0x0
+#define PA_SC_TILE_STEERING_OVERRIDE__NUM_SC__SHIFT                                                           0xc
+#define PA_SC_TILE_STEERING_OVERRIDE__NUM_RB_PER_SC__SHIFT                                                    0x10
+#define PA_SC_TILE_STEERING_OVERRIDE__NUM_PACKER_PER_SC__SHIFT                                                0x14
+#define PA_SC_TILE_STEERING_OVERRIDE__ENABLE_MASK                                                             0x00000001L
+#define PA_SC_TILE_STEERING_OVERRIDE__NUM_SC_MASK                                                             0x00003000L
+#define PA_SC_TILE_STEERING_OVERRIDE__NUM_RB_PER_SC_MASK                                                      0x00030000L
+#define PA_SC_TILE_STEERING_OVERRIDE__NUM_PACKER_PER_SC_MASK                                                  0x00300000L
+//PA_SC_CLIPRECT_0_EXT
+#define PA_SC_CLIPRECT_0_EXT__BR_X_EXT__SHIFT                                                                 0x0
+#define PA_SC_CLIPRECT_0_EXT__BR_Y_EXT__SHIFT                                                                 0x2
+#define PA_SC_CLIPRECT_0_EXT__TL_X_EXT__SHIFT                                                                 0x4
+#define PA_SC_CLIPRECT_0_EXT__TL_Y_EXT__SHIFT                                                                 0x6
+#define PA_SC_CLIPRECT_0_EXT__BR_X_EXT_MASK                                                                   0x00000003L
+#define PA_SC_CLIPRECT_0_EXT__BR_Y_EXT_MASK                                                                   0x0000000CL
+#define PA_SC_CLIPRECT_0_EXT__TL_X_EXT_MASK                                                                   0x00000030L
+#define PA_SC_CLIPRECT_0_EXT__TL_Y_EXT_MASK                                                                   0x000000C0L
+//PA_SC_CLIPRECT_1_EXT
+#define PA_SC_CLIPRECT_1_EXT__BR_X_EXT__SHIFT                                                                 0x0
+#define PA_SC_CLIPRECT_1_EXT__BR_Y_EXT__SHIFT                                                                 0x2
+#define PA_SC_CLIPRECT_1_EXT__TL_X_EXT__SHIFT                                                                 0x4
+#define PA_SC_CLIPRECT_1_EXT__TL_Y_EXT__SHIFT                                                                 0x6
+#define PA_SC_CLIPRECT_1_EXT__BR_X_EXT_MASK                                                                   0x00000003L
+#define PA_SC_CLIPRECT_1_EXT__BR_Y_EXT_MASK                                                                   0x0000000CL
+#define PA_SC_CLIPRECT_1_EXT__TL_X_EXT_MASK                                                                   0x00000030L
+#define PA_SC_CLIPRECT_1_EXT__TL_Y_EXT_MASK                                                                   0x000000C0L
+//PA_SC_CLIPRECT_2_EXT
+#define PA_SC_CLIPRECT_2_EXT__BR_X_EXT__SHIFT                                                                 0x0
+#define PA_SC_CLIPRECT_2_EXT__BR_Y_EXT__SHIFT                                                                 0x2
+#define PA_SC_CLIPRECT_2_EXT__TL_X_EXT__SHIFT                                                                 0x4
+#define PA_SC_CLIPRECT_2_EXT__TL_Y_EXT__SHIFT                                                                 0x6
+#define PA_SC_CLIPRECT_2_EXT__BR_X_EXT_MASK                                                                   0x00000003L
+#define PA_SC_CLIPRECT_2_EXT__BR_Y_EXT_MASK                                                                   0x0000000CL
+#define PA_SC_CLIPRECT_2_EXT__TL_X_EXT_MASK                                                                   0x00000030L
+#define PA_SC_CLIPRECT_2_EXT__TL_Y_EXT_MASK                                                                   0x000000C0L
+//PA_SC_CLIPRECT_3_EXT
+#define PA_SC_CLIPRECT_3_EXT__BR_X_EXT__SHIFT                                                                 0x0
+#define PA_SC_CLIPRECT_3_EXT__BR_Y_EXT__SHIFT                                                                 0x2
+#define PA_SC_CLIPRECT_3_EXT__TL_X_EXT__SHIFT                                                                 0x4
+#define PA_SC_CLIPRECT_3_EXT__TL_Y_EXT__SHIFT                                                                 0x6
+#define PA_SC_CLIPRECT_3_EXT__BR_X_EXT_MASK                                                                   0x00000003L
+#define PA_SC_CLIPRECT_3_EXT__BR_Y_EXT_MASK                                                                   0x0000000CL
+#define PA_SC_CLIPRECT_3_EXT__TL_X_EXT_MASK                                                                   0x00000030L
+#define PA_SC_CLIPRECT_3_EXT__TL_Y_EXT_MASK                                                                   0x000000C0L
+//PA_SC_VRS_OVERRIDE_CNTL
+#define PA_SC_VRS_OVERRIDE_CNTL__VRS_OVERRIDE_RATE_COMBINER_MODE__SHIFT                                       0x0
+#define PA_SC_VRS_OVERRIDE_CNTL__VRS_RATE__SHIFT                                                              0x4
+#define PA_SC_VRS_OVERRIDE_CNTL__VRS_SURFACE_ENABLE__SHIFT                                                    0xc
+#define PA_SC_VRS_OVERRIDE_CNTL__RATE_HINT_WRITE_BACK_ENABLE__SHIFT                                           0xd
+#define PA_SC_VRS_OVERRIDE_CNTL__VRS_FEEDBACK_RATE_OVERRIDE__SHIFT                                            0xe
+#define PA_SC_VRS_OVERRIDE_CNTL__VRS_OVERRIDE_RATE_COMBINER_MODE_MASK                                         0x00000007L
+#define PA_SC_VRS_OVERRIDE_CNTL__VRS_RATE_MASK                                                                0x000000F0L
+#define PA_SC_VRS_OVERRIDE_CNTL__VRS_SURFACE_ENABLE_MASK                                                      0x00001000L
+#define PA_SC_VRS_OVERRIDE_CNTL__RATE_HINT_WRITE_BACK_ENABLE_MASK                                             0x00002000L
+#define PA_SC_VRS_OVERRIDE_CNTL__VRS_FEEDBACK_RATE_OVERRIDE_MASK                                              0x00004000L
+//PA_SC_VRS_RATE_FEEDBACK_BASE
+#define PA_SC_VRS_RATE_FEEDBACK_BASE__BASE_256B__SHIFT                                                        0x0
+#define PA_SC_VRS_RATE_FEEDBACK_BASE__BASE_256B_MASK                                                          0xFFFFFFFFL
+//PA_SC_VRS_RATE_FEEDBACK_BASE_EXT
+#define PA_SC_VRS_RATE_FEEDBACK_BASE_EXT__BASE_256B__SHIFT                                                    0x0
+#define PA_SC_VRS_RATE_FEEDBACK_BASE_EXT__BASE_256B_MASK                                                      0x000000FFL
+//PA_SC_VRS_RATE_FEEDBACK_SIZE_XY
+#define PA_SC_VRS_RATE_FEEDBACK_SIZE_XY__X_MAX__SHIFT                                                         0x0
+#define PA_SC_VRS_RATE_FEEDBACK_SIZE_XY__Y_MAX__SHIFT                                                         0x10
+#define PA_SC_VRS_RATE_FEEDBACK_SIZE_XY__X_MAX_MASK                                                           0x00001FFFL
+#define PA_SC_VRS_RATE_FEEDBACK_SIZE_XY__Y_MAX_MASK                                                           0x1FFF0000L
+//PA_SC_VRS_INFO
+#define PA_SC_VRS_INFO__RATE_SW_MODE__SHIFT                                                                   0x0
+#define PA_SC_VRS_INFO__FEEDBACK_SW_MODE__SHIFT                                                               0x3
+#define PA_SC_VRS_INFO__RATE_SW_MODE_MASK                                                                     0x00000007L
+#define PA_SC_VRS_INFO__FEEDBACK_SW_MODE_MASK                                                                 0x00000038L
+//PA_SC_VRS_RATE_BASE
+#define PA_SC_VRS_RATE_BASE__BASE_256B__SHIFT                                                                 0x0
+#define PA_SC_VRS_RATE_BASE__BASE_256B_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VRS_RATE_BASE_EXT
+#define PA_SC_VRS_RATE_BASE_EXT__BASE_256B__SHIFT                                                             0x0
+#define PA_SC_VRS_RATE_BASE_EXT__TB_SYNC_SIM_ID__SHIFT                                                        0x1c
+#define PA_SC_VRS_RATE_BASE_EXT__BASE_256B_MASK                                                               0x000000FFL
+#define PA_SC_VRS_RATE_BASE_EXT__TB_SYNC_SIM_ID_MASK                                                          0xF0000000L
+//PA_SC_VRS_RATE_SIZE_XY
+#define PA_SC_VRS_RATE_SIZE_XY__X_MAX__SHIFT                                                                  0x0
+#define PA_SC_VRS_RATE_SIZE_XY__Y_MAX__SHIFT                                                                  0x10
+#define PA_SC_VRS_RATE_SIZE_XY__X_MAX_MASK                                                                    0x00001FFFL
+#define PA_SC_VRS_RATE_SIZE_XY__Y_MAX_MASK                                                                    0x1FFF0000L
+//CB_RMI_GL2_CACHE_CONTROL
+#define CB_RMI_GL2_CACHE_CONTROL__COLOR_WR_POLICY__SHIFT                                                      0x2
+#define CB_RMI_GL2_CACHE_CONTROL__COLOR_RD_POLICY__SHIFT                                                      0x16
+#define CB_RMI_GL2_CACHE_CONTROL__COLOR_WR_POLICY_MASK                                                        0x0000000CL
+#define CB_RMI_GL2_CACHE_CONTROL__COLOR_RD_POLICY_MASK                                                        0x00C00000L
+//CB_BLEND_RED
+#define CB_BLEND_RED__BLEND_RED__SHIFT                                                                        0x0
+#define CB_BLEND_RED__BLEND_RED_MASK                                                                          0xFFFFFFFFL
+//CB_BLEND_GREEN
+#define CB_BLEND_GREEN__BLEND_GREEN__SHIFT                                                                    0x0
+#define CB_BLEND_GREEN__BLEND_GREEN_MASK                                                                      0xFFFFFFFFL
+//CB_BLEND_BLUE
+#define CB_BLEND_BLUE__BLEND_BLUE__SHIFT                                                                      0x0
+#define CB_BLEND_BLUE__BLEND_BLUE_MASK                                                                        0xFFFFFFFFL
+//CB_BLEND_ALPHA
+#define CB_BLEND_ALPHA__BLEND_ALPHA__SHIFT                                                                    0x0
+#define CB_BLEND_ALPHA__BLEND_ALPHA_MASK                                                                      0xFFFFFFFFL
+//PA_CL_GB_VERT_CLIP_ADJ
+#define PA_CL_GB_VERT_CLIP_ADJ__DATA_REGISTER__SHIFT                                                          0x0
+#define PA_CL_GB_VERT_CLIP_ADJ__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
+//PA_CL_GB_VERT_DISC_ADJ
+#define PA_CL_GB_VERT_DISC_ADJ__DATA_REGISTER__SHIFT                                                          0x0
+#define PA_CL_GB_VERT_DISC_ADJ__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
+//PA_CL_GB_HORZ_CLIP_ADJ
+#define PA_CL_GB_HORZ_CLIP_ADJ__DATA_REGISTER__SHIFT                                                          0x0
+#define PA_CL_GB_HORZ_CLIP_ADJ__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
+//PA_CL_GB_HORZ_DISC_ADJ
+#define PA_CL_GB_HORZ_DISC_ADJ__DATA_REGISTER__SHIFT                                                          0x0
+#define PA_CL_GB_HORZ_DISC_ADJ__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE
+#define PA_CL_VPORT_XSCALE__VPORT_XSCALE__SHIFT                                                               0x0
+#define PA_CL_VPORT_XSCALE__VPORT_XSCALE_MASK                                                                 0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET
+#define PA_CL_VPORT_XOFFSET__VPORT_XOFFSET__SHIFT                                                             0x0
+#define PA_CL_VPORT_XOFFSET__VPORT_XOFFSET_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE
+#define PA_CL_VPORT_YSCALE__VPORT_YSCALE__SHIFT                                                               0x0
+#define PA_CL_VPORT_YSCALE__VPORT_YSCALE_MASK                                                                 0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET
+#define PA_CL_VPORT_YOFFSET__VPORT_YOFFSET__SHIFT                                                             0x0
+#define PA_CL_VPORT_YOFFSET__VPORT_YOFFSET_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE
+#define PA_CL_VPORT_ZSCALE__VPORT_ZSCALE__SHIFT                                                               0x0
+#define PA_CL_VPORT_ZSCALE__VPORT_ZSCALE_MASK                                                                 0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET
+#define PA_CL_VPORT_ZOFFSET__VPORT_ZOFFSET__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZOFFSET__VPORT_ZOFFSET_MASK                                                               0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_0
+#define PA_SC_VPORT_ZMIN_0__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_0__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_0
+#define PA_SC_VPORT_ZMAX_0__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_0__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_1
+#define PA_CL_VPORT_XSCALE_1__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_1__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_1
+#define PA_CL_VPORT_XOFFSET_1__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_1__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_1
+#define PA_CL_VPORT_YSCALE_1__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_1__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_1
+#define PA_CL_VPORT_YOFFSET_1__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_1__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_1
+#define PA_CL_VPORT_ZSCALE_1__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_1__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_1
+#define PA_CL_VPORT_ZOFFSET_1__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_1__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_1
+#define PA_SC_VPORT_ZMIN_1__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_1__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_1
+#define PA_SC_VPORT_ZMAX_1__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_1__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_2
+#define PA_CL_VPORT_XSCALE_2__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_2__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_2
+#define PA_CL_VPORT_XOFFSET_2__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_2__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_2
+#define PA_CL_VPORT_YSCALE_2__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_2__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_2
+#define PA_CL_VPORT_YOFFSET_2__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_2__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_2
+#define PA_CL_VPORT_ZSCALE_2__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_2__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_2
+#define PA_CL_VPORT_ZOFFSET_2__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_2__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_2
+#define PA_SC_VPORT_ZMIN_2__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_2__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_2
+#define PA_SC_VPORT_ZMAX_2__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_2__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_3
+#define PA_CL_VPORT_XSCALE_3__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_3__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_3
+#define PA_CL_VPORT_XOFFSET_3__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_3__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_3
+#define PA_CL_VPORT_YSCALE_3__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_3__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_3
+#define PA_CL_VPORT_YOFFSET_3__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_3__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_3
+#define PA_CL_VPORT_ZSCALE_3__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_3__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_3
+#define PA_CL_VPORT_ZOFFSET_3__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_3__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_3
+#define PA_SC_VPORT_ZMIN_3__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_3__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_3
+#define PA_SC_VPORT_ZMAX_3__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_3__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_4
+#define PA_CL_VPORT_XSCALE_4__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_4__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_4
+#define PA_CL_VPORT_XOFFSET_4__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_4__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_4
+#define PA_CL_VPORT_YSCALE_4__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_4__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_4
+#define PA_CL_VPORT_YOFFSET_4__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_4__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_4
+#define PA_CL_VPORT_ZSCALE_4__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_4__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_4
+#define PA_CL_VPORT_ZOFFSET_4__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_4__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_4
+#define PA_SC_VPORT_ZMIN_4__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_4__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_4
+#define PA_SC_VPORT_ZMAX_4__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_4__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_5
+#define PA_CL_VPORT_XSCALE_5__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_5__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_5
+#define PA_CL_VPORT_XOFFSET_5__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_5__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_5
+#define PA_CL_VPORT_YSCALE_5__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_5__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_5
+#define PA_CL_VPORT_YOFFSET_5__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_5__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_5
+#define PA_CL_VPORT_ZSCALE_5__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_5__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_5
+#define PA_CL_VPORT_ZOFFSET_5__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_5__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_5
+#define PA_SC_VPORT_ZMIN_5__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_5__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_5
+#define PA_SC_VPORT_ZMAX_5__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_5__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_6
+#define PA_CL_VPORT_XSCALE_6__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_6__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_6
+#define PA_CL_VPORT_XOFFSET_6__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_6__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_6
+#define PA_CL_VPORT_YSCALE_6__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_6__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_6
+#define PA_CL_VPORT_YOFFSET_6__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_6__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_6
+#define PA_CL_VPORT_ZSCALE_6__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_6__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_6
+#define PA_CL_VPORT_ZOFFSET_6__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_6__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_6
+#define PA_SC_VPORT_ZMIN_6__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_6__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_6
+#define PA_SC_VPORT_ZMAX_6__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_6__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_7
+#define PA_CL_VPORT_XSCALE_7__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_7__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_7
+#define PA_CL_VPORT_XOFFSET_7__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_7__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_7
+#define PA_CL_VPORT_YSCALE_7__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_7__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_7
+#define PA_CL_VPORT_YOFFSET_7__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_7__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_7
+#define PA_CL_VPORT_ZSCALE_7__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_7__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_7
+#define PA_CL_VPORT_ZOFFSET_7__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_7__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_7
+#define PA_SC_VPORT_ZMIN_7__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_7__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_7
+#define PA_SC_VPORT_ZMAX_7__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_7__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_8
+#define PA_CL_VPORT_XSCALE_8__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_8__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_8
+#define PA_CL_VPORT_XOFFSET_8__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_8__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_8
+#define PA_CL_VPORT_YSCALE_8__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_8__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_8
+#define PA_CL_VPORT_YOFFSET_8__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_8__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_8
+#define PA_CL_VPORT_ZSCALE_8__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_8__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_8
+#define PA_CL_VPORT_ZOFFSET_8__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_8__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_8
+#define PA_SC_VPORT_ZMIN_8__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_8__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_8
+#define PA_SC_VPORT_ZMAX_8__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_8__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_9
+#define PA_CL_VPORT_XSCALE_9__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_9__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_9
+#define PA_CL_VPORT_XOFFSET_9__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_9__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_9
+#define PA_CL_VPORT_YSCALE_9__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_9__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_9
+#define PA_CL_VPORT_YOFFSET_9__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_9__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_9
+#define PA_CL_VPORT_ZSCALE_9__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_9__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_9
+#define PA_CL_VPORT_ZOFFSET_9__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_9__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_9
+#define PA_SC_VPORT_ZMIN_9__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_9__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_9
+#define PA_SC_VPORT_ZMAX_9__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_9__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_10
+#define PA_CL_VPORT_XSCALE_10__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_10__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_10
+#define PA_CL_VPORT_XOFFSET_10__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_10__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_10
+#define PA_CL_VPORT_YSCALE_10__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_10__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_10
+#define PA_CL_VPORT_YOFFSET_10__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_10__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_10
+#define PA_CL_VPORT_ZSCALE_10__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_10__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_10
+#define PA_CL_VPORT_ZOFFSET_10__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_10__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_10
+#define PA_SC_VPORT_ZMIN_10__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_10__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_10
+#define PA_SC_VPORT_ZMAX_10__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_10__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_11
+#define PA_CL_VPORT_XSCALE_11__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_11__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_11
+#define PA_CL_VPORT_XOFFSET_11__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_11__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_11
+#define PA_CL_VPORT_YSCALE_11__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_11__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_11
+#define PA_CL_VPORT_YOFFSET_11__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_11__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_11
+#define PA_CL_VPORT_ZSCALE_11__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_11__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_11
+#define PA_CL_VPORT_ZOFFSET_11__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_11__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_11
+#define PA_SC_VPORT_ZMIN_11__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_11__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_11
+#define PA_SC_VPORT_ZMAX_11__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_11__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_12
+#define PA_CL_VPORT_XSCALE_12__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_12__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_12
+#define PA_CL_VPORT_XOFFSET_12__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_12__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_12
+#define PA_CL_VPORT_YSCALE_12__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_12__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_12
+#define PA_CL_VPORT_YOFFSET_12__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_12__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_12
+#define PA_CL_VPORT_ZSCALE_12__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_12__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_12
+#define PA_CL_VPORT_ZOFFSET_12__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_12__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_12
+#define PA_SC_VPORT_ZMIN_12__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_12__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_12
+#define PA_SC_VPORT_ZMAX_12__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_12__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_13
+#define PA_CL_VPORT_XSCALE_13__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_13__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_13
+#define PA_CL_VPORT_XOFFSET_13__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_13__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_13
+#define PA_CL_VPORT_YSCALE_13__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_13__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_13
+#define PA_CL_VPORT_YOFFSET_13__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_13__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_13
+#define PA_CL_VPORT_ZSCALE_13__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_13__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_13
+#define PA_CL_VPORT_ZOFFSET_13__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_13__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_13
+#define PA_SC_VPORT_ZMIN_13__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_13__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_13
+#define PA_SC_VPORT_ZMAX_13__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_13__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_14
+#define PA_CL_VPORT_XSCALE_14__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_14__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_14
+#define PA_CL_VPORT_XOFFSET_14__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_14__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_14
+#define PA_CL_VPORT_YSCALE_14__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_14__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_14
+#define PA_CL_VPORT_YOFFSET_14__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_14__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_14
+#define PA_CL_VPORT_ZSCALE_14__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_14__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_14
+#define PA_CL_VPORT_ZOFFSET_14__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_14__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_14
+#define PA_SC_VPORT_ZMIN_14__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_14__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_14
+#define PA_SC_VPORT_ZMAX_14__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_14__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_15
+#define PA_CL_VPORT_XSCALE_15__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_15__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_15
+#define PA_CL_VPORT_XOFFSET_15__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_15__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_15
+#define PA_CL_VPORT_YSCALE_15__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_15__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_15
+#define PA_CL_VPORT_YOFFSET_15__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_15__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_15
+#define PA_CL_VPORT_ZSCALE_15__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_15__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_15
+#define PA_CL_VPORT_ZOFFSET_15__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_15__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_15
+#define PA_SC_VPORT_ZMIN_15__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_15__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_15
+#define PA_SC_VPORT_ZMAX_15__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_15__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//SPI_PS_IN_CONTROL
+#define SPI_PS_IN_CONTROL__PARAM_GEN__SHIFT                                                                   0x6
+#define SPI_PS_IN_CONTROL__BC_OPTIMIZE_DISABLE__SHIFT                                                         0xe
+#define SPI_PS_IN_CONTROL__PS_W32_EN__SHIFT                                                                   0xf
+#define SPI_PS_IN_CONTROL__PARAM_GEN_MASK                                                                     0x00000040L
+#define SPI_PS_IN_CONTROL__BC_OPTIMIZE_DISABLE_MASK                                                           0x00004000L
+#define SPI_PS_IN_CONTROL__PS_W32_EN_MASK                                                                     0x00008000L
+//SPI_INTERP_CONTROL_0
+#define SPI_INTERP_CONTROL_0__FLAT_SHADE_ENA__SHIFT                                                           0x0
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_ENA__SHIFT                                                           0x1
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_X__SHIFT                                                        0x2
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Y__SHIFT                                                        0x5
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Z__SHIFT                                                        0x8
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_W__SHIFT                                                        0xb
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_TOP_1__SHIFT                                                         0xe
+#define SPI_INTERP_CONTROL_0__FLAT_SHADE_ENA_MASK                                                             0x00000001L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_ENA_MASK                                                             0x00000002L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_X_MASK                                                          0x0000001CL
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Y_MASK                                                          0x000000E0L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Z_MASK                                                          0x00000700L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_W_MASK                                                          0x00003800L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_TOP_1_MASK                                                           0x00004000L
+//SPI_SHADER_IDX_FORMAT
+#define SPI_SHADER_IDX_FORMAT__IDX0_EXPORT_FORMAT__SHIFT                                                      0x0
+#define SPI_SHADER_IDX_FORMAT__IDX0_EXPORT_FORMAT_MASK                                                        0x0000000FL
+//SPI_SHADER_POS_FORMAT
+#define SPI_SHADER_POS_FORMAT__POS0_EXPORT_FORMAT__SHIFT                                                      0x0
+#define SPI_SHADER_POS_FORMAT__POS1_EXPORT_FORMAT__SHIFT                                                      0x4
+#define SPI_SHADER_POS_FORMAT__POS2_EXPORT_FORMAT__SHIFT                                                      0x8
+#define SPI_SHADER_POS_FORMAT__POS3_EXPORT_FORMAT__SHIFT                                                      0xc
+#define SPI_SHADER_POS_FORMAT__POS4_EXPORT_FORMAT__SHIFT                                                      0x10
+#define SPI_SHADER_POS_FORMAT__POS0_EXPORT_FORMAT_MASK                                                        0x0000000FL
+#define SPI_SHADER_POS_FORMAT__POS1_EXPORT_FORMAT_MASK                                                        0x000000F0L
+#define SPI_SHADER_POS_FORMAT__POS2_EXPORT_FORMAT_MASK                                                        0x00000F00L
+#define SPI_SHADER_POS_FORMAT__POS3_EXPORT_FORMAT_MASK                                                        0x0000F000L
+#define SPI_SHADER_POS_FORMAT__POS4_EXPORT_FORMAT_MASK                                                        0x000F0000L
+//SPI_SHADER_Z_FORMAT
+#define SPI_SHADER_Z_FORMAT__Z_EXPORT_FORMAT__SHIFT                                                           0x0
+#define SPI_SHADER_Z_FORMAT__Z_EXPORT_FORMAT_MASK                                                             0x0000000FL
+//SPI_SHADER_COL_FORMAT
+#define SPI_SHADER_COL_FORMAT__COL0_EXPORT_FORMAT__SHIFT                                                      0x0
+#define SPI_SHADER_COL_FORMAT__COL1_EXPORT_FORMAT__SHIFT                                                      0x4
+#define SPI_SHADER_COL_FORMAT__COL2_EXPORT_FORMAT__SHIFT                                                      0x8
+#define SPI_SHADER_COL_FORMAT__COL3_EXPORT_FORMAT__SHIFT                                                      0xc
+#define SPI_SHADER_COL_FORMAT__COL4_EXPORT_FORMAT__SHIFT                                                      0x10
+#define SPI_SHADER_COL_FORMAT__COL5_EXPORT_FORMAT__SHIFT                                                      0x14
+#define SPI_SHADER_COL_FORMAT__COL6_EXPORT_FORMAT__SHIFT                                                      0x18
+#define SPI_SHADER_COL_FORMAT__COL7_EXPORT_FORMAT__SHIFT                                                      0x1c
+#define SPI_SHADER_COL_FORMAT__COL0_EXPORT_FORMAT_MASK                                                        0x0000000FL
+#define SPI_SHADER_COL_FORMAT__COL1_EXPORT_FORMAT_MASK                                                        0x000000F0L
+#define SPI_SHADER_COL_FORMAT__COL2_EXPORT_FORMAT_MASK                                                        0x00000F00L
+#define SPI_SHADER_COL_FORMAT__COL3_EXPORT_FORMAT_MASK                                                        0x0000F000L
+#define SPI_SHADER_COL_FORMAT__COL4_EXPORT_FORMAT_MASK                                                        0x000F0000L
+#define SPI_SHADER_COL_FORMAT__COL5_EXPORT_FORMAT_MASK                                                        0x00F00000L
+#define SPI_SHADER_COL_FORMAT__COL6_EXPORT_FORMAT_MASK                                                        0x0F000000L
+#define SPI_SHADER_COL_FORMAT__COL7_EXPORT_FORMAT_MASK                                                        0xF0000000L
+//SPI_BARYC_CNTL
+#define SPI_BARYC_CNTL__PERSP_CENTER_CNTL__SHIFT                                                              0x0
+#define SPI_BARYC_CNTL__PERSP_CENTROID_CNTL__SHIFT                                                            0x4
+#define SPI_BARYC_CNTL__LINEAR_CENTER_CNTL__SHIFT                                                             0x8
+#define SPI_BARYC_CNTL__LINEAR_CENTROID_CNTL__SHIFT                                                           0xc
+#define SPI_BARYC_CNTL__POS_FLOAT_LOCATION__SHIFT                                                             0x10
+#define SPI_BARYC_CNTL__POS_FLOAT_ULC__SHIFT                                                                  0x14
+#define SPI_BARYC_CNTL__FRONT_FACE_ALL_BITS__SHIFT                                                            0x18
+#define SPI_BARYC_CNTL__PERSP_CENTER_CNTL_MASK                                                                0x00000001L
+#define SPI_BARYC_CNTL__PERSP_CENTROID_CNTL_MASK                                                              0x00000010L
+#define SPI_BARYC_CNTL__LINEAR_CENTER_CNTL_MASK                                                               0x00000100L
+#define SPI_BARYC_CNTL__LINEAR_CENTROID_CNTL_MASK                                                             0x00001000L
+#define SPI_BARYC_CNTL__POS_FLOAT_LOCATION_MASK                                                               0x00030000L
+#define SPI_BARYC_CNTL__POS_FLOAT_ULC_MASK                                                                    0x00100000L
+#define SPI_BARYC_CNTL__FRONT_FACE_ALL_BITS_MASK                                                              0x01000000L
+//SPI_PS_INPUT_ENA
+#define SPI_PS_INPUT_ENA__PERSP_SAMPLE_ENA__SHIFT                                                             0x0
+#define SPI_PS_INPUT_ENA__PERSP_CENTER_ENA__SHIFT                                                             0x1
+#define SPI_PS_INPUT_ENA__PERSP_CENTROID_ENA__SHIFT                                                           0x2
+#define SPI_PS_INPUT_ENA__PERSP_PULL_MODEL_ENA__SHIFT                                                         0x3
+#define SPI_PS_INPUT_ENA__LINEAR_SAMPLE_ENA__SHIFT                                                            0x4
+#define SPI_PS_INPUT_ENA__LINEAR_CENTER_ENA__SHIFT                                                            0x5
+#define SPI_PS_INPUT_ENA__LINEAR_CENTROID_ENA__SHIFT                                                          0x6
+#define SPI_PS_INPUT_ENA__LINE_STIPPLE_TEX_ENA__SHIFT                                                         0x7
+#define SPI_PS_INPUT_ENA__POS_X_FLOAT_ENA__SHIFT                                                              0x8
+#define SPI_PS_INPUT_ENA__POS_Y_FLOAT_ENA__SHIFT                                                              0x9
+#define SPI_PS_INPUT_ENA__POS_Z_FLOAT_ENA__SHIFT                                                              0xa
+#define SPI_PS_INPUT_ENA__POS_W_FLOAT_ENA__SHIFT                                                              0xb
+#define SPI_PS_INPUT_ENA__FRONT_FACE_ENA__SHIFT                                                               0xc
+#define SPI_PS_INPUT_ENA__ANCILLARY_ENA__SHIFT                                                                0xd
+#define SPI_PS_INPUT_ENA__SAMPLE_COVERAGE_ENA__SHIFT                                                          0xe
+#define SPI_PS_INPUT_ENA__POS_FIXED_PT_ENA__SHIFT                                                             0xf
+#define SPI_PS_INPUT_ENA__COVERAGE_TO_SHADER_SELECT__SHIFT                                                    0x10
+#define SPI_PS_INPUT_ENA__PERSP_SAMPLE_ENA_MASK                                                               0x00000001L
+#define SPI_PS_INPUT_ENA__PERSP_CENTER_ENA_MASK                                                               0x00000002L
+#define SPI_PS_INPUT_ENA__PERSP_CENTROID_ENA_MASK                                                             0x00000004L
+#define SPI_PS_INPUT_ENA__PERSP_PULL_MODEL_ENA_MASK                                                           0x00000008L
+#define SPI_PS_INPUT_ENA__LINEAR_SAMPLE_ENA_MASK                                                              0x00000010L
+#define SPI_PS_INPUT_ENA__LINEAR_CENTER_ENA_MASK                                                              0x00000020L
+#define SPI_PS_INPUT_ENA__LINEAR_CENTROID_ENA_MASK                                                            0x00000040L
+#define SPI_PS_INPUT_ENA__LINE_STIPPLE_TEX_ENA_MASK                                                           0x00000080L
+#define SPI_PS_INPUT_ENA__POS_X_FLOAT_ENA_MASK                                                                0x00000100L
+#define SPI_PS_INPUT_ENA__POS_Y_FLOAT_ENA_MASK                                                                0x00000200L
+#define SPI_PS_INPUT_ENA__POS_Z_FLOAT_ENA_MASK                                                                0x00000400L
+#define SPI_PS_INPUT_ENA__POS_W_FLOAT_ENA_MASK                                                                0x00000800L
+#define SPI_PS_INPUT_ENA__FRONT_FACE_ENA_MASK                                                                 0x00001000L
+#define SPI_PS_INPUT_ENA__ANCILLARY_ENA_MASK                                                                  0x00002000L
+#define SPI_PS_INPUT_ENA__SAMPLE_COVERAGE_ENA_MASK                                                            0x00004000L
+#define SPI_PS_INPUT_ENA__POS_FIXED_PT_ENA_MASK                                                               0x00008000L
+#define SPI_PS_INPUT_ENA__COVERAGE_TO_SHADER_SELECT_MASK                                                      0x00030000L
+//SPI_PS_INPUT_ADDR
+#define SPI_PS_INPUT_ADDR__PERSP_SAMPLE_ENA__SHIFT                                                            0x0
+#define SPI_PS_INPUT_ADDR__PERSP_CENTER_ENA__SHIFT                                                            0x1
+#define SPI_PS_INPUT_ADDR__PERSP_CENTROID_ENA__SHIFT                                                          0x2
+#define SPI_PS_INPUT_ADDR__PERSP_PULL_MODEL_ENA__SHIFT                                                        0x3
+#define SPI_PS_INPUT_ADDR__LINEAR_SAMPLE_ENA__SHIFT                                                           0x4
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTER_ENA__SHIFT                                                           0x5
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTROID_ENA__SHIFT                                                         0x6
+#define SPI_PS_INPUT_ADDR__LINE_STIPPLE_TEX_ENA__SHIFT                                                        0x7
+#define SPI_PS_INPUT_ADDR__POS_X_FLOAT_ENA__SHIFT                                                             0x8
+#define SPI_PS_INPUT_ADDR__POS_Y_FLOAT_ENA__SHIFT                                                             0x9
+#define SPI_PS_INPUT_ADDR__POS_Z_FLOAT_ENA__SHIFT                                                             0xa
+#define SPI_PS_INPUT_ADDR__POS_W_FLOAT_ENA__SHIFT                                                             0xb
+#define SPI_PS_INPUT_ADDR__FRONT_FACE_ENA__SHIFT                                                              0xc
+#define SPI_PS_INPUT_ADDR__ANCILLARY_ENA__SHIFT                                                               0xd
+#define SPI_PS_INPUT_ADDR__SAMPLE_COVERAGE_ENA__SHIFT                                                         0xe
+#define SPI_PS_INPUT_ADDR__POS_FIXED_PT_ENA__SHIFT                                                            0xf
+#define SPI_PS_INPUT_ADDR__PERSP_SAMPLE_ENA_MASK                                                              0x00000001L
+#define SPI_PS_INPUT_ADDR__PERSP_CENTER_ENA_MASK                                                              0x00000002L
+#define SPI_PS_INPUT_ADDR__PERSP_CENTROID_ENA_MASK                                                            0x00000004L
+#define SPI_PS_INPUT_ADDR__PERSP_PULL_MODEL_ENA_MASK                                                          0x00000008L
+#define SPI_PS_INPUT_ADDR__LINEAR_SAMPLE_ENA_MASK                                                             0x00000010L
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTER_ENA_MASK                                                             0x00000020L
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTROID_ENA_MASK                                                           0x00000040L
+#define SPI_PS_INPUT_ADDR__LINE_STIPPLE_TEX_ENA_MASK                                                          0x00000080L
+#define SPI_PS_INPUT_ADDR__POS_X_FLOAT_ENA_MASK                                                               0x00000100L
+#define SPI_PS_INPUT_ADDR__POS_Y_FLOAT_ENA_MASK                                                               0x00000200L
+#define SPI_PS_INPUT_ADDR__POS_Z_FLOAT_ENA_MASK                                                               0x00000400L
+#define SPI_PS_INPUT_ADDR__POS_W_FLOAT_ENA_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_ADDR__FRONT_FACE_ENA_MASK                                                                0x00001000L
+#define SPI_PS_INPUT_ADDR__ANCILLARY_ENA_MASK                                                                 0x00002000L
+#define SPI_PS_INPUT_ADDR__SAMPLE_COVERAGE_ENA_MASK                                                           0x00004000L
+#define SPI_PS_INPUT_ADDR__POS_FIXED_PT_ENA_MASK                                                              0x00008000L
+//SPI_PS_INPUT_CNTL_0
+#define SPI_PS_INPUT_CNTL_0__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_0__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_0__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_0__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_0__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_0__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_0__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_0__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_0__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_0__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_0__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_0__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_0__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_0__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_0__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_0__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_0__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_0__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_1
+#define SPI_PS_INPUT_CNTL_1__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_1__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_1__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_1__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_1__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_1__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_1__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_1__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_1__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_1__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_1__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_1__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_1__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_1__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_1__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_1__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_1__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_1__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_2
+#define SPI_PS_INPUT_CNTL_2__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_2__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_2__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_2__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_2__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_2__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_2__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_2__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_2__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_2__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_2__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_2__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_2__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_2__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_2__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_2__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_2__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_2__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_3
+#define SPI_PS_INPUT_CNTL_3__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_3__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_3__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_3__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_3__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_3__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_3__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_3__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_3__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_3__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_3__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_3__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_3__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_3__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_3__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_3__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_3__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_3__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_4
+#define SPI_PS_INPUT_CNTL_4__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_4__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_4__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_4__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_4__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_4__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_4__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_4__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_4__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_4__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_4__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_4__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_4__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_4__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_4__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_4__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_4__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_4__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_5
+#define SPI_PS_INPUT_CNTL_5__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_5__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_5__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_5__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_5__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_5__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_5__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_5__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_5__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_5__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_5__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_5__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_5__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_5__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_5__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_5__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_5__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_5__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_6
+#define SPI_PS_INPUT_CNTL_6__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_6__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_6__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_6__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_6__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_6__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_6__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_6__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_6__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_6__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_6__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_6__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_6__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_6__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_6__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_6__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_6__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_6__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_7
+#define SPI_PS_INPUT_CNTL_7__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_7__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_7__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_7__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_7__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_7__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_7__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_7__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_7__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_7__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_7__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_7__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_7__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_7__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_7__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_7__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_7__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_7__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_8
+#define SPI_PS_INPUT_CNTL_8__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_8__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_8__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_8__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_8__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_8__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_8__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_8__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_8__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_8__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_8__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_8__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_8__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_8__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_8__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_8__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_8__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_8__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_9
+#define SPI_PS_INPUT_CNTL_9__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_9__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_9__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_9__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_9__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_9__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_9__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_9__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_9__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_9__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_9__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_9__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_9__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_9__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_9__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_9__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_9__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_9__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_10
+#define SPI_PS_INPUT_CNTL_10__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_10__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_10__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_10__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_10__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_10__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_10__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_10__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_10__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_10__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_10__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_10__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_10__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_10__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_10__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_10__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_10__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_10__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_11
+#define SPI_PS_INPUT_CNTL_11__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_11__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_11__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_11__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_11__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_11__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_11__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_11__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_11__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_11__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_11__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_11__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_11__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_11__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_11__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_11__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_11__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_11__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_12
+#define SPI_PS_INPUT_CNTL_12__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_12__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_12__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_12__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_12__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_12__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_12__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_12__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_12__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_12__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_12__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_12__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_12__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_12__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_12__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_12__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_12__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_12__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_13
+#define SPI_PS_INPUT_CNTL_13__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_13__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_13__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_13__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_13__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_13__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_13__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_13__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_13__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_13__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_13__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_13__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_13__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_13__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_13__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_13__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_13__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_13__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_14
+#define SPI_PS_INPUT_CNTL_14__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_14__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_14__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_14__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_14__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_14__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_14__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_14__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_14__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_14__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_14__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_14__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_14__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_14__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_14__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_14__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_14__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_14__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_15
+#define SPI_PS_INPUT_CNTL_15__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_15__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_15__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_15__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_15__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_15__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_15__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_15__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_15__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_15__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_15__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_15__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_15__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_15__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_15__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_15__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_15__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_15__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_16
+#define SPI_PS_INPUT_CNTL_16__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_16__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_16__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_16__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_16__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_16__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_16__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_16__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_16__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_16__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_16__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_16__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_16__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_16__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_16__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_16__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_16__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_16__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_17
+#define SPI_PS_INPUT_CNTL_17__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_17__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_17__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_17__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_17__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_17__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_17__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_17__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_17__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_17__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_17__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_17__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_17__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_17__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_17__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_17__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_17__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_17__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_18
+#define SPI_PS_INPUT_CNTL_18__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_18__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_18__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_18__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_18__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_18__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_18__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_18__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_18__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_18__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_18__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_18__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_18__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_18__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_18__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_18__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_18__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_18__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_19
+#define SPI_PS_INPUT_CNTL_19__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_19__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_19__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_19__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_19__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_19__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_19__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_19__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_19__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_19__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_19__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_19__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_19__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_19__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_19__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_19__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_19__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_19__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_20
+#define SPI_PS_INPUT_CNTL_20__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_20__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_20__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_20__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_20__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_20__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_20__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_20__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_20__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_20__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_20__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_20__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_20__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_20__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_20__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_20__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_20__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_20__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_21
+#define SPI_PS_INPUT_CNTL_21__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_21__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_21__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_21__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_21__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_21__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_21__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_21__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_21__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_21__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_21__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_21__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_21__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_21__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_21__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_21__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_21__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_21__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_22
+#define SPI_PS_INPUT_CNTL_22__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_22__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_22__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_22__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_22__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_22__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_22__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_22__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_22__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_22__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_22__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_22__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_22__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_22__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_22__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_22__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_22__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_22__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_23
+#define SPI_PS_INPUT_CNTL_23__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_23__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_23__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_23__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_23__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_23__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_23__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_23__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_23__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_23__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_23__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_23__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_23__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_23__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_23__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_23__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_23__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_23__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_24
+#define SPI_PS_INPUT_CNTL_24__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_24__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_24__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_24__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_24__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_24__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_24__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_24__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_24__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_24__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_24__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_24__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_24__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_24__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_24__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_24__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_24__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_24__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_25
+#define SPI_PS_INPUT_CNTL_25__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_25__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_25__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_25__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_25__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_25__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_25__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_25__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_25__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_25__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_25__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_25__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_25__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_25__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_25__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_25__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_25__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_25__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_26
+#define SPI_PS_INPUT_CNTL_26__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_26__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_26__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_26__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_26__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_26__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_26__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_26__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_26__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_26__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_26__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_26__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_26__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_26__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_26__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_26__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_26__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_26__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_27
+#define SPI_PS_INPUT_CNTL_27__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_27__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_27__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_27__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_27__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_27__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_27__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_27__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_27__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_27__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_27__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_27__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_27__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_27__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_27__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_27__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_27__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_27__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_28
+#define SPI_PS_INPUT_CNTL_28__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_28__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_28__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_28__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_28__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_28__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_28__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_28__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_28__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_28__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_28__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_28__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_28__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_28__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_28__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_28__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_28__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_28__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_29
+#define SPI_PS_INPUT_CNTL_29__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_29__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_29__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_29__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_29__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_29__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_29__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_29__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_29__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_29__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_29__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_29__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_29__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_29__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_29__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_29__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_29__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_29__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_30
+#define SPI_PS_INPUT_CNTL_30__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_30__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_30__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_30__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_30__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_30__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_30__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_30__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_30__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_30__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_30__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_30__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_30__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_30__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_30__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_30__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_30__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_30__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_31
+#define SPI_PS_INPUT_CNTL_31__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_31__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_31__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_31__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_31__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_31__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_31__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_31__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_31__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_31__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_31__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_31__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_31__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_31__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_31__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_31__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_31__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_31__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_BARYC_SSAA_CNTL
+#define SPI_BARYC_SSAA_CNTL__CENTER_SSAA_MODE__SHIFT                                                          0x0
+#define SPI_BARYC_SSAA_CNTL__CENTROID_SSAA_MODE__SHIFT                                                        0x1
+#define SPI_BARYC_SSAA_CNTL__COVERED_CENTROID_IS_CENTER__SHIFT                                                0x2
+#define SPI_BARYC_SSAA_CNTL__CENTER_SSAA_MODE_MASK                                                            0x00000001L
+#define SPI_BARYC_SSAA_CNTL__CENTROID_SSAA_MODE_MASK                                                          0x00000002L
+#define SPI_BARYC_SSAA_CNTL__COVERED_CENTROID_IS_CENTER_MASK                                                  0x00000004L
+//SPI_TMPRING_SIZE
+#define SPI_TMPRING_SIZE__WAVES__SHIFT                                                                        0x0
+#define SPI_TMPRING_SIZE__WAVESIZE__SHIFT                                                                     0xc
+#define SPI_TMPRING_SIZE__WAVES_MASK                                                                          0x00000FFFL
+#define SPI_TMPRING_SIZE__WAVESIZE_MASK                                                                       0x3FFFF000L
+//SPI_GFX_SCRATCH_BASE_LO
+#define SPI_GFX_SCRATCH_BASE_LO__DATA__SHIFT                                                                  0x0
+#define SPI_GFX_SCRATCH_BASE_LO__DATA_MASK                                                                    0xFFFFFFFFL
+//SPI_GFX_SCRATCH_BASE_HI
+#define SPI_GFX_SCRATCH_BASE_HI__DATA__SHIFT                                                                  0x0
+#define SPI_GFX_SCRATCH_BASE_HI__DATA_MASK                                                                    0x000000FFL
+//SX_PS_DOWNCONVERT_CONTROL
+#define SX_PS_DOWNCONVERT_CONTROL__MRT0_FMT_MAPPING_DISABLE__SHIFT                                            0x0
+#define SX_PS_DOWNCONVERT_CONTROL__MRT1_FMT_MAPPING_DISABLE__SHIFT                                            0x1
+#define SX_PS_DOWNCONVERT_CONTROL__MRT2_FMT_MAPPING_DISABLE__SHIFT                                            0x2
+#define SX_PS_DOWNCONVERT_CONTROL__MRT3_FMT_MAPPING_DISABLE__SHIFT                                            0x3
+#define SX_PS_DOWNCONVERT_CONTROL__MRT4_FMT_MAPPING_DISABLE__SHIFT                                            0x4
+#define SX_PS_DOWNCONVERT_CONTROL__MRT5_FMT_MAPPING_DISABLE__SHIFT                                            0x5
+#define SX_PS_DOWNCONVERT_CONTROL__MRT6_FMT_MAPPING_DISABLE__SHIFT                                            0x6
+#define SX_PS_DOWNCONVERT_CONTROL__MRT7_FMT_MAPPING_DISABLE__SHIFT                                            0x7
+#define SX_PS_DOWNCONVERT_CONTROL__MRT0_FMT_MAPPING_DISABLE_MASK                                              0x00000001L
+#define SX_PS_DOWNCONVERT_CONTROL__MRT1_FMT_MAPPING_DISABLE_MASK                                              0x00000002L
+#define SX_PS_DOWNCONVERT_CONTROL__MRT2_FMT_MAPPING_DISABLE_MASK                                              0x00000004L
+#define SX_PS_DOWNCONVERT_CONTROL__MRT3_FMT_MAPPING_DISABLE_MASK                                              0x00000008L
+#define SX_PS_DOWNCONVERT_CONTROL__MRT4_FMT_MAPPING_DISABLE_MASK                                              0x00000010L
+#define SX_PS_DOWNCONVERT_CONTROL__MRT5_FMT_MAPPING_DISABLE_MASK                                              0x00000020L
+#define SX_PS_DOWNCONVERT_CONTROL__MRT6_FMT_MAPPING_DISABLE_MASK                                              0x00000040L
+#define SX_PS_DOWNCONVERT_CONTROL__MRT7_FMT_MAPPING_DISABLE_MASK                                              0x00000080L
+//SX_PS_DOWNCONVERT
+#define SX_PS_DOWNCONVERT__MRT0__SHIFT                                                                        0x0
+#define SX_PS_DOWNCONVERT__MRT1__SHIFT                                                                        0x4
+#define SX_PS_DOWNCONVERT__MRT2__SHIFT                                                                        0x8
+#define SX_PS_DOWNCONVERT__MRT3__SHIFT                                                                        0xc
+#define SX_PS_DOWNCONVERT__MRT4__SHIFT                                                                        0x10
+#define SX_PS_DOWNCONVERT__MRT5__SHIFT                                                                        0x14
+#define SX_PS_DOWNCONVERT__MRT6__SHIFT                                                                        0x18
+#define SX_PS_DOWNCONVERT__MRT7__SHIFT                                                                        0x1c
+#define SX_PS_DOWNCONVERT__MRT0_MASK                                                                          0x0000000FL
+#define SX_PS_DOWNCONVERT__MRT1_MASK                                                                          0x000000F0L
+#define SX_PS_DOWNCONVERT__MRT2_MASK                                                                          0x00000F00L
+#define SX_PS_DOWNCONVERT__MRT3_MASK                                                                          0x0000F000L
+#define SX_PS_DOWNCONVERT__MRT4_MASK                                                                          0x000F0000L
+#define SX_PS_DOWNCONVERT__MRT5_MASK                                                                          0x00F00000L
+#define SX_PS_DOWNCONVERT__MRT6_MASK                                                                          0x0F000000L
+#define SX_PS_DOWNCONVERT__MRT7_MASK                                                                          0xF0000000L
+//SX_BLEND_OPT_EPSILON
+#define SX_BLEND_OPT_EPSILON__MRT0_EPSILON__SHIFT                                                             0x0
+#define SX_BLEND_OPT_EPSILON__MRT1_EPSILON__SHIFT                                                             0x4
+#define SX_BLEND_OPT_EPSILON__MRT2_EPSILON__SHIFT                                                             0x8
+#define SX_BLEND_OPT_EPSILON__MRT3_EPSILON__SHIFT                                                             0xc
+#define SX_BLEND_OPT_EPSILON__MRT4_EPSILON__SHIFT                                                             0x10
+#define SX_BLEND_OPT_EPSILON__MRT5_EPSILON__SHIFT                                                             0x14
+#define SX_BLEND_OPT_EPSILON__MRT6_EPSILON__SHIFT                                                             0x18
+#define SX_BLEND_OPT_EPSILON__MRT7_EPSILON__SHIFT                                                             0x1c
+#define SX_BLEND_OPT_EPSILON__MRT0_EPSILON_MASK                                                               0x0000000FL
+#define SX_BLEND_OPT_EPSILON__MRT1_EPSILON_MASK                                                               0x000000F0L
+#define SX_BLEND_OPT_EPSILON__MRT2_EPSILON_MASK                                                               0x00000F00L
+#define SX_BLEND_OPT_EPSILON__MRT3_EPSILON_MASK                                                               0x0000F000L
+#define SX_BLEND_OPT_EPSILON__MRT4_EPSILON_MASK                                                               0x000F0000L
+#define SX_BLEND_OPT_EPSILON__MRT5_EPSILON_MASK                                                               0x00F00000L
+#define SX_BLEND_OPT_EPSILON__MRT6_EPSILON_MASK                                                               0x0F000000L
+#define SX_BLEND_OPT_EPSILON__MRT7_EPSILON_MASK                                                               0xF0000000L
+//SX_BLEND_OPT_CONTROL
+#define SX_BLEND_OPT_CONTROL__MRT0_COLOR_OPT_DISABLE__SHIFT                                                   0x0
+#define SX_BLEND_OPT_CONTROL__MRT0_ALPHA_OPT_DISABLE__SHIFT                                                   0x1
+#define SX_BLEND_OPT_CONTROL__MRT1_COLOR_OPT_DISABLE__SHIFT                                                   0x4
+#define SX_BLEND_OPT_CONTROL__MRT1_ALPHA_OPT_DISABLE__SHIFT                                                   0x5
+#define SX_BLEND_OPT_CONTROL__MRT2_COLOR_OPT_DISABLE__SHIFT                                                   0x8
+#define SX_BLEND_OPT_CONTROL__MRT2_ALPHA_OPT_DISABLE__SHIFT                                                   0x9
+#define SX_BLEND_OPT_CONTROL__MRT3_COLOR_OPT_DISABLE__SHIFT                                                   0xc
+#define SX_BLEND_OPT_CONTROL__MRT3_ALPHA_OPT_DISABLE__SHIFT                                                   0xd
+#define SX_BLEND_OPT_CONTROL__MRT4_COLOR_OPT_DISABLE__SHIFT                                                   0x10
+#define SX_BLEND_OPT_CONTROL__MRT4_ALPHA_OPT_DISABLE__SHIFT                                                   0x11
+#define SX_BLEND_OPT_CONTROL__MRT5_COLOR_OPT_DISABLE__SHIFT                                                   0x14
+#define SX_BLEND_OPT_CONTROL__MRT5_ALPHA_OPT_DISABLE__SHIFT                                                   0x15
+#define SX_BLEND_OPT_CONTROL__MRT6_COLOR_OPT_DISABLE__SHIFT                                                   0x18
+#define SX_BLEND_OPT_CONTROL__MRT6_ALPHA_OPT_DISABLE__SHIFT                                                   0x19
+#define SX_BLEND_OPT_CONTROL__MRT7_COLOR_OPT_DISABLE__SHIFT                                                   0x1c
+#define SX_BLEND_OPT_CONTROL__MRT7_ALPHA_OPT_DISABLE__SHIFT                                                   0x1d
+#define SX_BLEND_OPT_CONTROL__PIXEN_ZERO_OPT_DISABLE__SHIFT                                                   0x1f
+#define SX_BLEND_OPT_CONTROL__MRT0_COLOR_OPT_DISABLE_MASK                                                     0x00000001L
+#define SX_BLEND_OPT_CONTROL__MRT0_ALPHA_OPT_DISABLE_MASK                                                     0x00000002L
+#define SX_BLEND_OPT_CONTROL__MRT1_COLOR_OPT_DISABLE_MASK                                                     0x00000010L
+#define SX_BLEND_OPT_CONTROL__MRT1_ALPHA_OPT_DISABLE_MASK                                                     0x00000020L
+#define SX_BLEND_OPT_CONTROL__MRT2_COLOR_OPT_DISABLE_MASK                                                     0x00000100L
+#define SX_BLEND_OPT_CONTROL__MRT2_ALPHA_OPT_DISABLE_MASK                                                     0x00000200L
+#define SX_BLEND_OPT_CONTROL__MRT3_COLOR_OPT_DISABLE_MASK                                                     0x00001000L
+#define SX_BLEND_OPT_CONTROL__MRT3_ALPHA_OPT_DISABLE_MASK                                                     0x00002000L
+#define SX_BLEND_OPT_CONTROL__MRT4_COLOR_OPT_DISABLE_MASK                                                     0x00010000L
+#define SX_BLEND_OPT_CONTROL__MRT4_ALPHA_OPT_DISABLE_MASK                                                     0x00020000L
+#define SX_BLEND_OPT_CONTROL__MRT5_COLOR_OPT_DISABLE_MASK                                                     0x00100000L
+#define SX_BLEND_OPT_CONTROL__MRT5_ALPHA_OPT_DISABLE_MASK                                                     0x00200000L
+#define SX_BLEND_OPT_CONTROL__MRT6_COLOR_OPT_DISABLE_MASK                                                     0x01000000L
+#define SX_BLEND_OPT_CONTROL__MRT6_ALPHA_OPT_DISABLE_MASK                                                     0x02000000L
+#define SX_BLEND_OPT_CONTROL__MRT7_COLOR_OPT_DISABLE_MASK                                                     0x10000000L
+#define SX_BLEND_OPT_CONTROL__MRT7_ALPHA_OPT_DISABLE_MASK                                                     0x20000000L
+#define SX_BLEND_OPT_CONTROL__PIXEN_ZERO_OPT_DISABLE_MASK                                                     0x80000000L
+//SX_MRT0_BLEND_OPT
+#define SX_MRT0_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
+#define SX_MRT0_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
+#define SX_MRT0_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
+#define SX_MRT0_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
+#define SX_MRT0_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
+#define SX_MRT0_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
+#define SX_MRT0_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
+#define SX_MRT0_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
+#define SX_MRT0_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
+#define SX_MRT0_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
+#define SX_MRT0_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
+#define SX_MRT0_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
+//SX_MRT1_BLEND_OPT
+#define SX_MRT1_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
+#define SX_MRT1_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
+#define SX_MRT1_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
+#define SX_MRT1_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
+#define SX_MRT1_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
+#define SX_MRT1_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
+#define SX_MRT1_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
+#define SX_MRT1_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
+#define SX_MRT1_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
+#define SX_MRT1_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
+#define SX_MRT1_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
+#define SX_MRT1_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
+//SX_MRT2_BLEND_OPT
+#define SX_MRT2_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
+#define SX_MRT2_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
+#define SX_MRT2_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
+#define SX_MRT2_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
+#define SX_MRT2_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
+#define SX_MRT2_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
+#define SX_MRT2_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
+#define SX_MRT2_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
+#define SX_MRT2_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
+#define SX_MRT2_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
+#define SX_MRT2_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
+#define SX_MRT2_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
+//SX_MRT3_BLEND_OPT
+#define SX_MRT3_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
+#define SX_MRT3_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
+#define SX_MRT3_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
+#define SX_MRT3_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
+#define SX_MRT3_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
+#define SX_MRT3_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
+#define SX_MRT3_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
+#define SX_MRT3_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
+#define SX_MRT3_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
+#define SX_MRT3_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
+#define SX_MRT3_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
+#define SX_MRT3_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
+//SX_MRT4_BLEND_OPT
+#define SX_MRT4_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
+#define SX_MRT4_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
+#define SX_MRT4_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
+#define SX_MRT4_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
+#define SX_MRT4_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
+#define SX_MRT4_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
+#define SX_MRT4_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
+#define SX_MRT4_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
+#define SX_MRT4_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
+#define SX_MRT4_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
+#define SX_MRT4_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
+#define SX_MRT4_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
+//SX_MRT5_BLEND_OPT
+#define SX_MRT5_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
+#define SX_MRT5_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
+#define SX_MRT5_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
+#define SX_MRT5_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
+#define SX_MRT5_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
+#define SX_MRT5_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
+#define SX_MRT5_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
+#define SX_MRT5_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
+#define SX_MRT5_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
+#define SX_MRT5_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
+#define SX_MRT5_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
+#define SX_MRT5_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
+//SX_MRT6_BLEND_OPT
+#define SX_MRT6_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
+#define SX_MRT6_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
+#define SX_MRT6_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
+#define SX_MRT6_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
+#define SX_MRT6_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
+#define SX_MRT6_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
+#define SX_MRT6_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
+#define SX_MRT6_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
+#define SX_MRT6_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
+#define SX_MRT6_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
+#define SX_MRT6_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
+#define SX_MRT6_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
+//SX_MRT7_BLEND_OPT
+#define SX_MRT7_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
+#define SX_MRT7_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
+#define SX_MRT7_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
+#define SX_MRT7_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
+#define SX_MRT7_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
+#define SX_MRT7_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
+#define SX_MRT7_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
+#define SX_MRT7_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
+#define SX_MRT7_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
+#define SX_MRT7_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
+#define SX_MRT7_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
+#define SX_MRT7_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
+//CB_BLEND0_CONTROL
+#define CB_BLEND0_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND0_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND0_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND0_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND0_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND0_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND0_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND0_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND0_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND0_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND0_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND0_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND0_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND0_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND0_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND0_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND0_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND0_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND1_CONTROL
+#define CB_BLEND1_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND1_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND1_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND1_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND1_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND1_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND1_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND1_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND1_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND1_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND1_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND1_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND1_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND1_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND1_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND1_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND1_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND1_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND2_CONTROL
+#define CB_BLEND2_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND2_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND2_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND2_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND2_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND2_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND2_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND2_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND2_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND2_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND2_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND2_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND2_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND2_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND2_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND2_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND2_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND2_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND3_CONTROL
+#define CB_BLEND3_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND3_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND3_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND3_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND3_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND3_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND3_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND3_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND3_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND3_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND3_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND3_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND3_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND3_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND3_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND3_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND3_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND3_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND4_CONTROL
+#define CB_BLEND4_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND4_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND4_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND4_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND4_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND4_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND4_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND4_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND4_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND4_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND4_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND4_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND4_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND4_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND4_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND4_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND4_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND4_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND5_CONTROL
+#define CB_BLEND5_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND5_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND5_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND5_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND5_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND5_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND5_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND5_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND5_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND5_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND5_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND5_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND5_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND5_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND5_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND5_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND5_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND5_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND6_CONTROL
+#define CB_BLEND6_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND6_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND6_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND6_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND6_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND6_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND6_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND6_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND6_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND6_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND6_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND6_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND6_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND6_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND6_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND6_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND6_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND6_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND7_CONTROL
+#define CB_BLEND7_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND7_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND7_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND7_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND7_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND7_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND7_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND7_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND7_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND7_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND7_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND7_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND7_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND7_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND7_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND7_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND7_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND7_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//PA_CL_POINT_X_RAD
+#define PA_CL_POINT_X_RAD__DATA_REGISTER__SHIFT                                                               0x0
+#define PA_CL_POINT_X_RAD__DATA_REGISTER_MASK                                                                 0xFFFFFFFFL
+//PA_CL_POINT_Y_RAD
+#define PA_CL_POINT_Y_RAD__DATA_REGISTER__SHIFT                                                               0x0
+#define PA_CL_POINT_Y_RAD__DATA_REGISTER_MASK                                                                 0xFFFFFFFFL
+//PA_CL_POINT_SIZE
+#define PA_CL_POINT_SIZE__DATA_REGISTER__SHIFT                                                                0x0
+#define PA_CL_POINT_SIZE__DATA_REGISTER_MASK                                                                  0xFFFFFFFFL
+//PA_CL_POINT_CULL_RAD
+#define PA_CL_POINT_CULL_RAD__DATA_REGISTER__SHIFT                                                            0x0
+#define PA_CL_POINT_CULL_RAD__DATA_REGISTER_MASK                                                              0xFFFFFFFFL
+//GE_MAX_OUTPUT_PER_SUBGROUP
+#define GE_MAX_OUTPUT_PER_SUBGROUP__MAX_VERTS_PER_SUBGROUP__SHIFT                                             0x0
+#define GE_MAX_OUTPUT_PER_SUBGROUP__MAX_VERTS_PER_SUBGROUP_MASK                                               0x000003FFL
+//PA_CL_CLIP_CNTL
+#define PA_CL_CLIP_CNTL__UCP_ENA_0__SHIFT                                                                     0x0
+#define PA_CL_CLIP_CNTL__UCP_ENA_1__SHIFT                                                                     0x1
+#define PA_CL_CLIP_CNTL__UCP_ENA_2__SHIFT                                                                     0x2
+#define PA_CL_CLIP_CNTL__UCP_ENA_3__SHIFT                                                                     0x3
+#define PA_CL_CLIP_CNTL__UCP_ENA_4__SHIFT                                                                     0x4
+#define PA_CL_CLIP_CNTL__UCP_ENA_5__SHIFT                                                                     0x5
+#define PA_CL_CLIP_CNTL__PS_UCP_Y_SCALE_NEG__SHIFT                                                            0xd
+#define PA_CL_CLIP_CNTL__PS_UCP_MODE__SHIFT                                                                   0xe
+#define PA_CL_CLIP_CNTL__CLIP_DISABLE__SHIFT                                                                  0x10
+#define PA_CL_CLIP_CNTL__UCP_CULL_ONLY_ENA__SHIFT                                                             0x11
+#define PA_CL_CLIP_CNTL__BOUNDARY_EDGE_FLAG_ENA__SHIFT                                                        0x12
+#define PA_CL_CLIP_CNTL__DX_CLIP_SPACE_DEF__SHIFT                                                             0x13
+#define PA_CL_CLIP_CNTL__DIS_CLIP_ERR_DETECT__SHIFT                                                           0x14
+#define PA_CL_CLIP_CNTL__VTX_KILL_OR__SHIFT                                                                   0x15
+#define PA_CL_CLIP_CNTL__DX_RASTERIZATION_KILL__SHIFT                                                         0x16
+#define PA_CL_CLIP_CNTL__DX_LINEAR_ATTR_CLIP_ENA__SHIFT                                                       0x18
+#define PA_CL_CLIP_CNTL__VTE_VPORT_PROVOKE_DISABLE__SHIFT                                                     0x19
+#define PA_CL_CLIP_CNTL__ZCLIP_NEAR_DISABLE__SHIFT                                                            0x1a
+#define PA_CL_CLIP_CNTL__ZCLIP_FAR_DISABLE__SHIFT                                                             0x1b
+#define PA_CL_CLIP_CNTL__ZCLIP_PROG_NEAR_ENA__SHIFT                                                           0x1c
+#define PA_CL_CLIP_CNTL__UCP_ENA_0_MASK                                                                       0x00000001L
+#define PA_CL_CLIP_CNTL__UCP_ENA_1_MASK                                                                       0x00000002L
+#define PA_CL_CLIP_CNTL__UCP_ENA_2_MASK                                                                       0x00000004L
+#define PA_CL_CLIP_CNTL__UCP_ENA_3_MASK                                                                       0x00000008L
+#define PA_CL_CLIP_CNTL__UCP_ENA_4_MASK                                                                       0x00000010L
+#define PA_CL_CLIP_CNTL__UCP_ENA_5_MASK                                                                       0x00000020L
+#define PA_CL_CLIP_CNTL__PS_UCP_Y_SCALE_NEG_MASK                                                              0x00002000L
+#define PA_CL_CLIP_CNTL__PS_UCP_MODE_MASK                                                                     0x0000C000L
+#define PA_CL_CLIP_CNTL__CLIP_DISABLE_MASK                                                                    0x00010000L
+#define PA_CL_CLIP_CNTL__UCP_CULL_ONLY_ENA_MASK                                                               0x00020000L
+#define PA_CL_CLIP_CNTL__BOUNDARY_EDGE_FLAG_ENA_MASK                                                          0x00040000L
+#define PA_CL_CLIP_CNTL__DX_CLIP_SPACE_DEF_MASK                                                               0x00080000L
+#define PA_CL_CLIP_CNTL__DIS_CLIP_ERR_DETECT_MASK                                                             0x00100000L
+#define PA_CL_CLIP_CNTL__VTX_KILL_OR_MASK                                                                     0x00200000L
+#define PA_CL_CLIP_CNTL__DX_RASTERIZATION_KILL_MASK                                                           0x00400000L
+#define PA_CL_CLIP_CNTL__DX_LINEAR_ATTR_CLIP_ENA_MASK                                                         0x01000000L
+#define PA_CL_CLIP_CNTL__VTE_VPORT_PROVOKE_DISABLE_MASK                                                       0x02000000L
+#define PA_CL_CLIP_CNTL__ZCLIP_NEAR_DISABLE_MASK                                                              0x04000000L
+#define PA_CL_CLIP_CNTL__ZCLIP_FAR_DISABLE_MASK                                                               0x08000000L
+#define PA_CL_CLIP_CNTL__ZCLIP_PROG_NEAR_ENA_MASK                                                             0x10000000L
+//PA_CL_VTE_CNTL
+#define PA_CL_VTE_CNTL__VPORT_X_SCALE_ENA__SHIFT                                                              0x0
+#define PA_CL_VTE_CNTL__VPORT_X_OFFSET_ENA__SHIFT                                                             0x1
+#define PA_CL_VTE_CNTL__VPORT_Y_SCALE_ENA__SHIFT                                                              0x2
+#define PA_CL_VTE_CNTL__VPORT_Y_OFFSET_ENA__SHIFT                                                             0x3
+#define PA_CL_VTE_CNTL__VPORT_Z_SCALE_ENA__SHIFT                                                              0x4
+#define PA_CL_VTE_CNTL__VPORT_Z_OFFSET_ENA__SHIFT                                                             0x5
+#define PA_CL_VTE_CNTL__VTX_XY_FMT__SHIFT                                                                     0x8
+#define PA_CL_VTE_CNTL__VTX_Z_FMT__SHIFT                                                                      0x9
+#define PA_CL_VTE_CNTL__VTX_W0_FMT__SHIFT                                                                     0xa
+#define PA_CL_VTE_CNTL__PERFCOUNTER_REF__SHIFT                                                                0xb
+#define PA_CL_VTE_CNTL__VPORT_X_SCALE_ENA_MASK                                                                0x00000001L
+#define PA_CL_VTE_CNTL__VPORT_X_OFFSET_ENA_MASK                                                               0x00000002L
+#define PA_CL_VTE_CNTL__VPORT_Y_SCALE_ENA_MASK                                                                0x00000004L
+#define PA_CL_VTE_CNTL__VPORT_Y_OFFSET_ENA_MASK                                                               0x00000008L
+#define PA_CL_VTE_CNTL__VPORT_Z_SCALE_ENA_MASK                                                                0x00000010L
+#define PA_CL_VTE_CNTL__VPORT_Z_OFFSET_ENA_MASK                                                               0x00000020L
+#define PA_CL_VTE_CNTL__VTX_XY_FMT_MASK                                                                       0x00000100L
+#define PA_CL_VTE_CNTL__VTX_Z_FMT_MASK                                                                        0x00000200L
+#define PA_CL_VTE_CNTL__VTX_W0_FMT_MASK                                                                       0x00000400L
+#define PA_CL_VTE_CNTL__PERFCOUNTER_REF_MASK                                                                  0x00000800L
+//PA_CL_VS_OUT_CNTL
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_0__SHIFT                                                             0x0
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_1__SHIFT                                                             0x1
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_2__SHIFT                                                             0x2
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_3__SHIFT                                                             0x3
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_4__SHIFT                                                             0x4
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_5__SHIFT                                                             0x5
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_6__SHIFT                                                             0x6
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_7__SHIFT                                                             0x7
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_0__SHIFT                                                             0x8
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_1__SHIFT                                                             0x9
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_2__SHIFT                                                             0xa
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_3__SHIFT                                                             0xb
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_4__SHIFT                                                             0xc
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_5__SHIFT                                                             0xd
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_6__SHIFT                                                             0xe
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_7__SHIFT                                                             0xf
+#define PA_CL_VS_OUT_CNTL__USE_VTX_POINT_SIZE__SHIFT                                                          0x10
+#define PA_CL_VS_OUT_CNTL__USE_VTX_EDGE_FLAG__SHIFT                                                           0x11
+#define PA_CL_VS_OUT_CNTL__USE_VTX_RENDER_TARGET_INDX__SHIFT                                                  0x12
+#define PA_CL_VS_OUT_CNTL__USE_VTX_VIEWPORT_INDX__SHIFT                                                       0x13
+#define PA_CL_VS_OUT_CNTL__USE_VTX_KILL_FLAG__SHIFT                                                           0x14
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_VEC_ENA__SHIFT                                                         0x15
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST0_VEC_ENA__SHIFT                                                      0x16
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST1_VEC_ENA__SHIFT                                                      0x17
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_SIDE_BUS_ENA__SHIFT                                                    0x18
+#define PA_CL_VS_OUT_CNTL__USE_VTX_LINE_WIDTH__SHIFT                                                          0x1b
+#define PA_CL_VS_OUT_CNTL__USE_VTX_VRS_RATE__SHIFT                                                            0x1c
+#define PA_CL_VS_OUT_CNTL__BYPASS_VTX_RATE_COMBINER__SHIFT                                                    0x1d
+#define PA_CL_VS_OUT_CNTL__BYPASS_PRIM_RATE_COMBINER__SHIFT                                                   0x1e
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_0_MASK                                                               0x00000001L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_1_MASK                                                               0x00000002L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_2_MASK                                                               0x00000004L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_3_MASK                                                               0x00000008L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_4_MASK                                                               0x00000010L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_5_MASK                                                               0x00000020L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_6_MASK                                                               0x00000040L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_7_MASK                                                               0x00000080L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_0_MASK                                                               0x00000100L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_1_MASK                                                               0x00000200L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_2_MASK                                                               0x00000400L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_3_MASK                                                               0x00000800L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_4_MASK                                                               0x00001000L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_5_MASK                                                               0x00002000L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_6_MASK                                                               0x00004000L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_7_MASK                                                               0x00008000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_POINT_SIZE_MASK                                                            0x00010000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_EDGE_FLAG_MASK                                                             0x00020000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_RENDER_TARGET_INDX_MASK                                                    0x00040000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_VIEWPORT_INDX_MASK                                                         0x00080000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_KILL_FLAG_MASK                                                             0x00100000L
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_VEC_ENA_MASK                                                           0x00200000L
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST0_VEC_ENA_MASK                                                        0x00400000L
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST1_VEC_ENA_MASK                                                        0x00800000L
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_SIDE_BUS_ENA_MASK                                                      0x01000000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_LINE_WIDTH_MASK                                                            0x08000000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_VRS_RATE_MASK                                                              0x10000000L
+#define PA_CL_VS_OUT_CNTL__BYPASS_VTX_RATE_COMBINER_MASK                                                      0x20000000L
+#define PA_CL_VS_OUT_CNTL__BYPASS_PRIM_RATE_COMBINER_MASK                                                     0x40000000L
+//PA_SU_SC_MODE_CNTL
+#define PA_SU_SC_MODE_CNTL__CULL_FRONT__SHIFT                                                                 0x0
+#define PA_SU_SC_MODE_CNTL__CULL_BACK__SHIFT                                                                  0x1
+#define PA_SU_SC_MODE_CNTL__FACE__SHIFT                                                                       0x2
+#define PA_SU_SC_MODE_CNTL__POLY_MODE__SHIFT                                                                  0x3
+#define PA_SU_SC_MODE_CNTL__POLYMODE_FRONT_PTYPE__SHIFT                                                       0x5
+#define PA_SU_SC_MODE_CNTL__POLYMODE_BACK_PTYPE__SHIFT                                                        0x8
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_FRONT_ENABLE__SHIFT                                                   0xb
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_BACK_ENABLE__SHIFT                                                    0xc
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_PARA_ENABLE__SHIFT                                                    0xd
+#define PA_SU_SC_MODE_CNTL__VTX_WINDOW_OFFSET_ENABLE__SHIFT                                                   0x10
+#define PA_SU_SC_MODE_CNTL__PROVOKING_VTX_LAST__SHIFT                                                         0x13
+#define PA_SU_SC_MODE_CNTL__PERSP_CORR_DIS__SHIFT                                                             0x14
+#define PA_SU_SC_MODE_CNTL__MULTI_PRIM_IB_ENA__SHIFT                                                          0x15
+#define PA_SU_SC_MODE_CNTL__RIGHT_TRIANGLE_ALTERNATE_GRADIENT_REF__SHIFT                                      0x16
+#define PA_SU_SC_MODE_CNTL__NEW_QUAD_DECOMPOSITION__SHIFT                                                     0x17
+#define PA_SU_SC_MODE_CNTL__CULL_FRONT_MASK                                                                   0x00000001L
+#define PA_SU_SC_MODE_CNTL__CULL_BACK_MASK                                                                    0x00000002L
+#define PA_SU_SC_MODE_CNTL__FACE_MASK                                                                         0x00000004L
+#define PA_SU_SC_MODE_CNTL__POLY_MODE_MASK                                                                    0x00000018L
+#define PA_SU_SC_MODE_CNTL__POLYMODE_FRONT_PTYPE_MASK                                                         0x000000E0L
+#define PA_SU_SC_MODE_CNTL__POLYMODE_BACK_PTYPE_MASK                                                          0x00000700L
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_FRONT_ENABLE_MASK                                                     0x00000800L
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_BACK_ENABLE_MASK                                                      0x00001000L
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_PARA_ENABLE_MASK                                                      0x00002000L
+#define PA_SU_SC_MODE_CNTL__VTX_WINDOW_OFFSET_ENABLE_MASK                                                     0x00010000L
+#define PA_SU_SC_MODE_CNTL__PROVOKING_VTX_LAST_MASK                                                           0x00080000L
+#define PA_SU_SC_MODE_CNTL__PERSP_CORR_DIS_MASK                                                               0x00100000L
+#define PA_SU_SC_MODE_CNTL__MULTI_PRIM_IB_ENA_MASK                                                            0x00200000L
+#define PA_SU_SC_MODE_CNTL__RIGHT_TRIANGLE_ALTERNATE_GRADIENT_REF_MASK                                        0x00400000L
+#define PA_SU_SC_MODE_CNTL__NEW_QUAD_DECOMPOSITION_MASK                                                       0x00800000L
+//PA_CL_NANINF_CNTL
+#define PA_CL_NANINF_CNTL__VTE_XY_INF_DISCARD__SHIFT                                                          0x0
+#define PA_CL_NANINF_CNTL__VTE_Z_INF_DISCARD__SHIFT                                                           0x1
+#define PA_CL_NANINF_CNTL__VTE_W_INF_DISCARD__SHIFT                                                           0x2
+#define PA_CL_NANINF_CNTL__VTE_0XNANINF_IS_0__SHIFT                                                           0x3
+#define PA_CL_NANINF_CNTL__VTE_XY_NAN_RETAIN__SHIFT                                                           0x4
+#define PA_CL_NANINF_CNTL__VTE_Z_NAN_RETAIN__SHIFT                                                            0x5
+#define PA_CL_NANINF_CNTL__VTE_W_NAN_RETAIN__SHIFT                                                            0x6
+#define PA_CL_NANINF_CNTL__VTE_W_RECIP_NAN_IS_0__SHIFT                                                        0x7
+#define PA_CL_NANINF_CNTL__VS_XY_NAN_TO_INF__SHIFT                                                            0x8
+#define PA_CL_NANINF_CNTL__VS_XY_INF_RETAIN__SHIFT                                                            0x9
+#define PA_CL_NANINF_CNTL__VS_Z_NAN_TO_INF__SHIFT                                                             0xa
+#define PA_CL_NANINF_CNTL__VS_Z_INF_RETAIN__SHIFT                                                             0xb
+#define PA_CL_NANINF_CNTL__VS_W_NAN_TO_INF__SHIFT                                                             0xc
+#define PA_CL_NANINF_CNTL__VS_W_INF_RETAIN__SHIFT                                                             0xd
+#define PA_CL_NANINF_CNTL__VS_CLIP_DIST_INF_DISCARD__SHIFT                                                    0xe
+#define PA_CL_NANINF_CNTL__VTE_NO_OUTPUT_NEG_0__SHIFT                                                         0x14
+#define PA_CL_NANINF_CNTL__VTE_XY_INF_DISCARD_MASK                                                            0x00000001L
+#define PA_CL_NANINF_CNTL__VTE_Z_INF_DISCARD_MASK                                                             0x00000002L
+#define PA_CL_NANINF_CNTL__VTE_W_INF_DISCARD_MASK                                                             0x00000004L
+#define PA_CL_NANINF_CNTL__VTE_0XNANINF_IS_0_MASK                                                             0x00000008L
+#define PA_CL_NANINF_CNTL__VTE_XY_NAN_RETAIN_MASK                                                             0x00000010L
+#define PA_CL_NANINF_CNTL__VTE_Z_NAN_RETAIN_MASK                                                              0x00000020L
+#define PA_CL_NANINF_CNTL__VTE_W_NAN_RETAIN_MASK                                                              0x00000040L
+#define PA_CL_NANINF_CNTL__VTE_W_RECIP_NAN_IS_0_MASK                                                          0x00000080L
+#define PA_CL_NANINF_CNTL__VS_XY_NAN_TO_INF_MASK                                                              0x00000100L
+#define PA_CL_NANINF_CNTL__VS_XY_INF_RETAIN_MASK                                                              0x00000200L
+#define PA_CL_NANINF_CNTL__VS_Z_NAN_TO_INF_MASK                                                               0x00000400L
+#define PA_CL_NANINF_CNTL__VS_Z_INF_RETAIN_MASK                                                               0x00000800L
+#define PA_CL_NANINF_CNTL__VS_W_NAN_TO_INF_MASK                                                               0x00001000L
+#define PA_CL_NANINF_CNTL__VS_W_INF_RETAIN_MASK                                                               0x00002000L
+#define PA_CL_NANINF_CNTL__VS_CLIP_DIST_INF_DISCARD_MASK                                                      0x00004000L
+#define PA_CL_NANINF_CNTL__VTE_NO_OUTPUT_NEG_0_MASK                                                           0x00100000L
+//PA_SU_LINE_STIPPLE_CNTL
+#define PA_SU_LINE_STIPPLE_CNTL__LINE_STIPPLE_RESET__SHIFT                                                    0x0
+#define PA_SU_LINE_STIPPLE_CNTL__EXPAND_FULL_LENGTH__SHIFT                                                    0x2
+#define PA_SU_LINE_STIPPLE_CNTL__FRACTIONAL_ACCUM__SHIFT                                                      0x3
+#define PA_SU_LINE_STIPPLE_CNTL__LINE_STIPPLE_RESET_MASK                                                      0x00000003L
+#define PA_SU_LINE_STIPPLE_CNTL__EXPAND_FULL_LENGTH_MASK                                                      0x00000004L
+#define PA_SU_LINE_STIPPLE_CNTL__FRACTIONAL_ACCUM_MASK                                                        0x00000008L
+//PA_SU_LINE_STIPPLE_SCALE
+#define PA_SU_LINE_STIPPLE_SCALE__LINE_STIPPLE_SCALE__SHIFT                                                   0x0
+#define PA_SU_LINE_STIPPLE_SCALE__LINE_STIPPLE_SCALE_MASK                                                     0xFFFFFFFFL
+//PA_SU_PRIM_FILTER_CNTL
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE__SHIFT                                                0x0
+#define PA_SU_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE__SHIFT                                                    0x1
+#define PA_SU_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE__SHIFT                                                   0x2
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE__SHIFT                                               0x3
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_EXPAND_ENA__SHIFT                                                    0x4
+#define PA_SU_PRIM_FILTER_CNTL__LINE_EXPAND_ENA__SHIFT                                                        0x5
+#define PA_SU_PRIM_FILTER_CNTL__POINT_EXPAND_ENA__SHIFT                                                       0x6
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_EXPAND_ENA__SHIFT                                                   0x7
+#define PA_SU_PRIM_FILTER_CNTL__PRIM_EXPAND_CONSTANT__SHIFT                                                   0x8
+#define PA_SU_PRIM_FILTER_CNTL__XMAX_RIGHT_EXCLUSION__SHIFT                                                   0x1e
+#define PA_SU_PRIM_FILTER_CNTL__YMAX_BOTTOM_EXCLUSION__SHIFT                                                  0x1f
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE_MASK                                                  0x00000001L
+#define PA_SU_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE_MASK                                                      0x00000002L
+#define PA_SU_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE_MASK                                                     0x00000004L
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE_MASK                                                 0x00000008L
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_EXPAND_ENA_MASK                                                      0x00000010L
+#define PA_SU_PRIM_FILTER_CNTL__LINE_EXPAND_ENA_MASK                                                          0x00000020L
+#define PA_SU_PRIM_FILTER_CNTL__POINT_EXPAND_ENA_MASK                                                         0x00000040L
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_EXPAND_ENA_MASK                                                     0x00000080L
+#define PA_SU_PRIM_FILTER_CNTL__PRIM_EXPAND_CONSTANT_MASK                                                     0x0000FF00L
+#define PA_SU_PRIM_FILTER_CNTL__XMAX_RIGHT_EXCLUSION_MASK                                                     0x40000000L
+#define PA_SU_PRIM_FILTER_CNTL__YMAX_BOTTOM_EXCLUSION_MASK                                                    0x80000000L
+//PA_SU_SMALL_PRIM_FILTER_CNTL
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__SMALL_PRIM_FILTER_ENABLE__SHIFT                                         0x0
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE__SHIFT                                          0x1
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE__SHIFT                                              0x2
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE__SHIFT                                             0x3
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE__SHIFT                                         0x4
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__SC_1XMSAA_COMPATIBLE_DISABLE__SHIFT                                     0x6
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__SMALL_PRIM_FILTER_ENABLE_MASK                                           0x00000001L
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE_MASK                                            0x00000002L
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE_MASK                                                0x00000004L
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE_MASK                                               0x00000008L
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE_MASK                                           0x00000010L
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__SC_1XMSAA_COMPATIBLE_DISABLE_MASK                                       0x00000040L
+//PA_CL_NGG_CNTL
+#define PA_CL_NGG_CNTL__VERTEX_REUSE_OFF__SHIFT                                                               0x0
+#define PA_CL_NGG_CNTL__INDEX_BUF_EDGE_FLAG_ENA__SHIFT                                                        0x1
+#define PA_CL_NGG_CNTL__VERTEX_REUSE_DEPTH__SHIFT                                                             0x2
+#define PA_CL_NGG_CNTL__VERTEX_REUSE_OFF_MASK                                                                 0x00000001L
+#define PA_CL_NGG_CNTL__INDEX_BUF_EDGE_FLAG_ENA_MASK                                                          0x00000002L
+#define PA_CL_NGG_CNTL__VERTEX_REUSE_DEPTH_MASK                                                               0x000003FCL
+//PA_SU_OVER_RASTERIZATION_CNTL
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_TRIANGLES__SHIFT                                        0x0
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_LINES__SHIFT                                            0x1
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_POINTS__SHIFT                                           0x2
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_RECTANGLES__SHIFT                                       0x3
+#define PA_SU_OVER_RASTERIZATION_CNTL__USE_PROVOKING_ZW__SHIFT                                                0x4
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_TRIANGLES_MASK                                          0x00000001L
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_LINES_MASK                                              0x00000002L
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_POINTS_MASK                                             0x00000004L
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_RECTANGLES_MASK                                         0x00000008L
+#define PA_SU_OVER_RASTERIZATION_CNTL__USE_PROVOKING_ZW_MASK                                                  0x00000010L
+//PA_STEREO_CNTL
+#define PA_STEREO_CNTL__STEREO_MODE__SHIFT                                                                    0x1
+#define PA_STEREO_CNTL__RT_SLICE_MODE__SHIFT                                                                  0x5
+#define PA_STEREO_CNTL__RT_SLICE_OFFSET__SHIFT                                                                0x8
+#define PA_STEREO_CNTL__VP_ID_MODE__SHIFT                                                                     0x10
+#define PA_STEREO_CNTL__VP_ID_OFFSET__SHIFT                                                                   0x13
+#define PA_STEREO_CNTL__STEREO_MODE_MASK                                                                      0x0000001EL
+#define PA_STEREO_CNTL__RT_SLICE_MODE_MASK                                                                    0x000000E0L
+#define PA_STEREO_CNTL__RT_SLICE_OFFSET_MASK                                                                  0x00000F00L
+#define PA_STEREO_CNTL__VP_ID_MODE_MASK                                                                       0x00070000L
+#define PA_STEREO_CNTL__VP_ID_OFFSET_MASK                                                                     0x00780000L
+//PA_STATE_STEREO_X
+#define PA_STATE_STEREO_X__STEREO_X_OFFSET__SHIFT                                                             0x0
+#define PA_STATE_STEREO_X__STEREO_X_OFFSET_MASK                                                               0xFFFFFFFFL
+//PA_CL_VRS_CNTL
+#define PA_CL_VRS_CNTL__VERTEX_RATE_COMBINER_MODE__SHIFT                                                      0x0
+#define PA_CL_VRS_CNTL__PRIMITIVE_RATE_COMBINER_MODE__SHIFT                                                   0x3
+#define PA_CL_VRS_CNTL__HTILE_RATE_COMBINER_MODE__SHIFT                                                       0x6
+#define PA_CL_VRS_CNTL__SAMPLE_ITER_COMBINER_MODE__SHIFT                                                      0x9
+#define PA_CL_VRS_CNTL__EXPOSE_VRS_PIXELS_MASK__SHIFT                                                         0xd
+#define PA_CL_VRS_CNTL__CMASK_RATE_HINT_FORCE_ZERO__SHIFT                                                     0xe
+#define PA_CL_VRS_CNTL__SAMPLE_COVERAGE_ENCODING__SHIFT                                                       0xf
+#define PA_CL_VRS_CNTL__VERTEX_RATE_COMBINER_MODE_MASK                                                        0x00000007L
+#define PA_CL_VRS_CNTL__PRIMITIVE_RATE_COMBINER_MODE_MASK                                                     0x00000038L
+#define PA_CL_VRS_CNTL__HTILE_RATE_COMBINER_MODE_MASK                                                         0x000001C0L
+#define PA_CL_VRS_CNTL__SAMPLE_ITER_COMBINER_MODE_MASK                                                        0x00000E00L
+#define PA_CL_VRS_CNTL__EXPOSE_VRS_PIXELS_MASK_MASK                                                           0x00002000L
+#define PA_CL_VRS_CNTL__CMASK_RATE_HINT_FORCE_ZERO_MASK                                                       0x00004000L
+#define PA_CL_VRS_CNTL__SAMPLE_COVERAGE_ENCODING_MASK                                                         0x00008000L
+//CB_TARGET_MASK
+#define CB_TARGET_MASK__TARGET0_ENABLE__SHIFT                                                                 0x0
+#define CB_TARGET_MASK__TARGET1_ENABLE__SHIFT                                                                 0x4
+#define CB_TARGET_MASK__TARGET2_ENABLE__SHIFT                                                                 0x8
+#define CB_TARGET_MASK__TARGET3_ENABLE__SHIFT                                                                 0xc
+#define CB_TARGET_MASK__TARGET4_ENABLE__SHIFT                                                                 0x10
+#define CB_TARGET_MASK__TARGET5_ENABLE__SHIFT                                                                 0x14
+#define CB_TARGET_MASK__TARGET6_ENABLE__SHIFT                                                                 0x18
+#define CB_TARGET_MASK__TARGET7_ENABLE__SHIFT                                                                 0x1c
+#define CB_TARGET_MASK__TARGET0_ENABLE_MASK                                                                   0x0000000FL
+#define CB_TARGET_MASK__TARGET1_ENABLE_MASK                                                                   0x000000F0L
+#define CB_TARGET_MASK__TARGET2_ENABLE_MASK                                                                   0x00000F00L
+#define CB_TARGET_MASK__TARGET3_ENABLE_MASK                                                                   0x0000F000L
+#define CB_TARGET_MASK__TARGET4_ENABLE_MASK                                                                   0x000F0000L
+#define CB_TARGET_MASK__TARGET5_ENABLE_MASK                                                                   0x00F00000L
+#define CB_TARGET_MASK__TARGET6_ENABLE_MASK                                                                   0x0F000000L
+#define CB_TARGET_MASK__TARGET7_ENABLE_MASK                                                                   0xF0000000L
+//CB_SHADER_MASK
+#define CB_SHADER_MASK__OUTPUT0_ENABLE__SHIFT                                                                 0x0
+#define CB_SHADER_MASK__OUTPUT1_ENABLE__SHIFT                                                                 0x4
+#define CB_SHADER_MASK__OUTPUT2_ENABLE__SHIFT                                                                 0x8
+#define CB_SHADER_MASK__OUTPUT3_ENABLE__SHIFT                                                                 0xc
+#define CB_SHADER_MASK__OUTPUT4_ENABLE__SHIFT                                                                 0x10
+#define CB_SHADER_MASK__OUTPUT5_ENABLE__SHIFT                                                                 0x14
+#define CB_SHADER_MASK__OUTPUT6_ENABLE__SHIFT                                                                 0x18
+#define CB_SHADER_MASK__OUTPUT7_ENABLE__SHIFT                                                                 0x1c
+#define CB_SHADER_MASK__OUTPUT0_ENABLE_MASK                                                                   0x0000000FL
+#define CB_SHADER_MASK__OUTPUT1_ENABLE_MASK                                                                   0x000000F0L
+#define CB_SHADER_MASK__OUTPUT2_ENABLE_MASK                                                                   0x00000F00L
+#define CB_SHADER_MASK__OUTPUT3_ENABLE_MASK                                                                   0x0000F000L
+#define CB_SHADER_MASK__OUTPUT4_ENABLE_MASK                                                                   0x000F0000L
+#define CB_SHADER_MASK__OUTPUT5_ENABLE_MASK                                                                   0x00F00000L
+#define CB_SHADER_MASK__OUTPUT6_ENABLE_MASK                                                                   0x0F000000L
+#define CB_SHADER_MASK__OUTPUT7_ENABLE_MASK                                                                   0xF0000000L
+//CB_COLOR_CONTROL
+#define CB_COLOR_CONTROL__DISABLE_DUAL_QUAD__SHIFT                                                            0x0
+#define CB_COLOR_CONTROL__ENABLE_1FRAG_PS_INVOKE__SHIFT                                                       0x1
+#define CB_COLOR_CONTROL__DEGAMMA_ENABLE__SHIFT                                                               0x3
+#define CB_COLOR_CONTROL__MODE__SHIFT                                                                         0x4
+#define CB_COLOR_CONTROL__ROP3__SHIFT                                                                         0x10
+#define CB_COLOR_CONTROL__DISABLE_DUAL_QUAD_MASK                                                              0x00000001L
+#define CB_COLOR_CONTROL__ENABLE_1FRAG_PS_INVOKE_MASK                                                         0x00000002L
+#define CB_COLOR_CONTROL__DEGAMMA_ENABLE_MASK                                                                 0x00000008L
+#define CB_COLOR_CONTROL__MODE_MASK                                                                           0x00000070L
+#define CB_COLOR_CONTROL__ROP3_MASK                                                                           0x00FF0000L
+//PA_SU_POINT_SIZE
+#define PA_SU_POINT_SIZE__HEIGHT__SHIFT                                                                       0x0
+#define PA_SU_POINT_SIZE__WIDTH__SHIFT                                                                        0x10
+#define PA_SU_POINT_SIZE__HEIGHT_MASK                                                                         0x0000FFFFL
+#define PA_SU_POINT_SIZE__WIDTH_MASK                                                                          0xFFFF0000L
+//PA_SU_POINT_MINMAX
+#define PA_SU_POINT_MINMAX__MIN_SIZE__SHIFT                                                                   0x0
+#define PA_SU_POINT_MINMAX__MAX_SIZE__SHIFT                                                                   0x10
+#define PA_SU_POINT_MINMAX__MIN_SIZE_MASK                                                                     0x0000FFFFL
+#define PA_SU_POINT_MINMAX__MAX_SIZE_MASK                                                                     0xFFFF0000L
+//PA_SU_LINE_CNTL
+#define PA_SU_LINE_CNTL__WIDTH__SHIFT                                                                         0x0
+#define PA_SU_LINE_CNTL__WIDTH_MASK                                                                           0x0000FFFFL
+//PA_SC_LINE_STIPPLE
+#define PA_SC_LINE_STIPPLE__LINE_PATTERN__SHIFT                                                               0x0
+#define PA_SC_LINE_STIPPLE__REPEAT_COUNT__SHIFT                                                               0x10
+#define PA_SC_LINE_STIPPLE__PATTERN_BIT_ORDER__SHIFT                                                          0x1c
+#define PA_SC_LINE_STIPPLE__LINE_PATTERN_MASK                                                                 0x0000FFFFL
+#define PA_SC_LINE_STIPPLE__REPEAT_COUNT_MASK                                                                 0x00FF0000L
+#define PA_SC_LINE_STIPPLE__PATTERN_BIT_ORDER_MASK                                                            0x10000000L
+//PA_SC_LINE_STIPPLE_RESET
+#define PA_SC_LINE_STIPPLE_RESET__AUTO_RESET_CNTL__SHIFT                                                      0x0
+#define PA_SC_LINE_STIPPLE_RESET__AUTO_RESET_CNTL_MASK                                                        0x00000003L
+//PA_SC_MODE_CNTL_0
+#define PA_SC_MODE_CNTL_0__MSAA_ENABLE__SHIFT                                                                 0x0
+#define PA_SC_MODE_CNTL_0__VPORT_SCISSOR_ENABLE__SHIFT                                                        0x1
+#define PA_SC_MODE_CNTL_0__LINE_STIPPLE_ENABLE__SHIFT                                                         0x2
+#define PA_SC_MODE_CNTL_0__SEND_UNLIT_STILES_TO_PKR__SHIFT                                                    0x3
+#define PA_SC_MODE_CNTL_0__ALTERNATE_RBS_PER_TILE__SHIFT                                                      0x5
+#define PA_SC_MODE_CNTL_0__COARSE_TILE_STARTS_ON_EVEN_RB__SHIFT                                               0x6
+#define PA_SC_MODE_CNTL_0__IMPLICIT_VPORT_SCISSOR_ENABLE__SHIFT                                               0x7
+#define PA_SC_MODE_CNTL_0__MSAA_ENABLE_MASK                                                                   0x00000001L
+#define PA_SC_MODE_CNTL_0__VPORT_SCISSOR_ENABLE_MASK                                                          0x00000002L
+#define PA_SC_MODE_CNTL_0__LINE_STIPPLE_ENABLE_MASK                                                           0x00000004L
+#define PA_SC_MODE_CNTL_0__SEND_UNLIT_STILES_TO_PKR_MASK                                                      0x00000008L
+#define PA_SC_MODE_CNTL_0__ALTERNATE_RBS_PER_TILE_MASK                                                        0x00000020L
+#define PA_SC_MODE_CNTL_0__COARSE_TILE_STARTS_ON_EVEN_RB_MASK                                                 0x00000040L
+#define PA_SC_MODE_CNTL_0__IMPLICIT_VPORT_SCISSOR_ENABLE_MASK                                                 0x00000080L
+//PA_SC_MODE_CNTL_1
+#define PA_SC_MODE_CNTL_1__WALK_SIZE__SHIFT                                                                   0x0
+#define PA_SC_MODE_CNTL_1__WALK_ALIGNMENT__SHIFT                                                              0x1
+#define PA_SC_MODE_CNTL_1__WALK_ALIGN8_PRIM_FITS_ST__SHIFT                                                    0x2
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_ENABLE__SHIFT                                                           0x3
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_SIZE__SHIFT                                                             0x4
+#define PA_SC_MODE_CNTL_1__SUPERTILE_WALK_ORDER_ENABLE__SHIFT                                                 0x7
+#define PA_SC_MODE_CNTL_1__TILE_WALK_ORDER_ENABLE__SHIFT                                                      0x8
+#define PA_SC_MODE_CNTL_1__TILE_COVER_DISABLE__SHIFT                                                          0x9
+#define PA_SC_MODE_CNTL_1__TILE_COVER_NO_SCISSOR__SHIFT                                                       0xa
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_EXTENT__SHIFT                                                             0xb
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_OFFSET__SHIFT                                                             0xc
+#define PA_SC_MODE_CNTL_1__ZMM_RECT_EXTENT__SHIFT                                                             0xd
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_HI_Z__SHIFT                                                          0xe
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_DETAIL_MASK__SHIFT                                                   0xf
+#define PA_SC_MODE_CNTL_1__PS_ITER_SAMPLE__SHIFT                                                              0x10
+#define PA_SC_MODE_CNTL_1__MULTI_SHADER_ENGINE_PRIM_DISCARD_ENABLE__SHIFT                                     0x11
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_SUPERTILE_ENABLE__SHIFT                                                  0x12
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_ENABLE__SHIFT                                                      0x13
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE__SHIFT                                                             0x14
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_PRIM_DISCARD_ENABLE__SHIFT                                               0x18
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_CNTDWN_ENABLE__SHIFT                                                     0x19
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_REZ_ENABLE__SHIFT                                                        0x1a
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_PRIMITIVE_ENABLE__SHIFT                                               0x1b
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_WATER_MARK__SHIFT                                                     0x1c
+#define PA_SC_MODE_CNTL_1__WALK_SIZE_MASK                                                                     0x00000001L
+#define PA_SC_MODE_CNTL_1__WALK_ALIGNMENT_MASK                                                                0x00000002L
+#define PA_SC_MODE_CNTL_1__WALK_ALIGN8_PRIM_FITS_ST_MASK                                                      0x00000004L
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_ENABLE_MASK                                                             0x00000008L
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_SIZE_MASK                                                               0x00000070L
+#define PA_SC_MODE_CNTL_1__SUPERTILE_WALK_ORDER_ENABLE_MASK                                                   0x00000080L
+#define PA_SC_MODE_CNTL_1__TILE_WALK_ORDER_ENABLE_MASK                                                        0x00000100L
+#define PA_SC_MODE_CNTL_1__TILE_COVER_DISABLE_MASK                                                            0x00000200L
+#define PA_SC_MODE_CNTL_1__TILE_COVER_NO_SCISSOR_MASK                                                         0x00000400L
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_EXTENT_MASK                                                               0x00000800L
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_OFFSET_MASK                                                               0x00001000L
+#define PA_SC_MODE_CNTL_1__ZMM_RECT_EXTENT_MASK                                                               0x00002000L
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_HI_Z_MASK                                                            0x00004000L
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_DETAIL_MASK_MASK                                                     0x00008000L
+#define PA_SC_MODE_CNTL_1__PS_ITER_SAMPLE_MASK                                                                0x00010000L
+#define PA_SC_MODE_CNTL_1__MULTI_SHADER_ENGINE_PRIM_DISCARD_ENABLE_MASK                                       0x00020000L
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_SUPERTILE_ENABLE_MASK                                                    0x00040000L
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_ENABLE_MASK                                                        0x00080000L
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_MASK                                                               0x00F00000L
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_PRIM_DISCARD_ENABLE_MASK                                                 0x01000000L
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_CNTDWN_ENABLE_MASK                                                       0x02000000L
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_REZ_ENABLE_MASK                                                          0x04000000L
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_PRIMITIVE_ENABLE_MASK                                                 0x08000000L
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_WATER_MARK_MASK                                                       0x70000000L
+//GE_SE_ENHANCE
+#define GE_SE_ENHANCE__MISC__SHIFT                                                                            0x0
+#define GE_SE_ENHANCE__MISC_MASK                                                                              0xFFFFFFFFL
+//VGT_REUSE_OFF
+#define VGT_REUSE_OFF__REUSE_OFF__SHIFT                                                                       0x0
+#define VGT_REUSE_OFF__REUSE_OFF_MASK                                                                         0x00000001L
+//VGT_DRAW_PAYLOAD_CNTL
+#define VGT_DRAW_PAYLOAD_CNTL__EN_REG_RT_INDEX__SHIFT                                                         0x1
+#define VGT_DRAW_PAYLOAD_CNTL__EN_PRIM_PAYLOAD__SHIFT                                                         0x3
+#define VGT_DRAW_PAYLOAD_CNTL__EN_DRAW_VP__SHIFT                                                              0x4
+#define VGT_DRAW_PAYLOAD_CNTL__UNUSED__SHIFT                                                                  0x5
+#define VGT_DRAW_PAYLOAD_CNTL__EN_VRS_RATE__SHIFT                                                             0x6
+#define VGT_DRAW_PAYLOAD_CNTL__EN_REG_RT_INDEX_MASK                                                           0x00000002L
+#define VGT_DRAW_PAYLOAD_CNTL__EN_PRIM_PAYLOAD_MASK                                                           0x00000008L
+#define VGT_DRAW_PAYLOAD_CNTL__EN_DRAW_VP_MASK                                                                0x00000010L
+#define VGT_DRAW_PAYLOAD_CNTL__UNUSED_MASK                                                                    0x00000020L
+#define VGT_DRAW_PAYLOAD_CNTL__EN_VRS_RATE_MASK                                                               0x00000040L
+//DB_HTILE_SURFACE
+#define DB_HTILE_SURFACE__RESERVED_FIELD_1__SHIFT                                                             0x0
+#define DB_HTILE_SURFACE__RESERVED_FIELD_2__SHIFT                                                             0x2
+#define DB_HTILE_SURFACE__RESERVED_FIELD_3__SHIFT                                                             0x3
+#define DB_HTILE_SURFACE__RESERVED_FIELD_4__SHIFT                                                             0x4
+#define DB_HTILE_SURFACE__RESERVED_FIELD_5__SHIFT                                                             0xa
+#define DB_HTILE_SURFACE__DST_OUTSIDE_ZERO_TO_ONE__SHIFT                                                      0x10
+#define DB_HTILE_SURFACE__RESERVED_FIELD_6__SHIFT                                                             0x11
+#define DB_HTILE_SURFACE__RESERVED_FIELD_1_MASK                                                               0x00000001L
+#define DB_HTILE_SURFACE__RESERVED_FIELD_2_MASK                                                               0x00000004L
+#define DB_HTILE_SURFACE__RESERVED_FIELD_3_MASK                                                               0x00000008L
+#define DB_HTILE_SURFACE__RESERVED_FIELD_4_MASK                                                               0x000003F0L
+#define DB_HTILE_SURFACE__RESERVED_FIELD_5_MASK                                                               0x0000FC00L
+#define DB_HTILE_SURFACE__DST_OUTSIDE_ZERO_TO_ONE_MASK                                                        0x00010000L
+#define DB_HTILE_SURFACE__RESERVED_FIELD_6_MASK                                                               0x00020000L
+//DB_SRESULTS_COMPARE_STATE0
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREFUNC0__SHIFT                                                       0x0
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREVALUE0__SHIFT                                                      0x4
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREMASK0__SHIFT                                                       0xc
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREFUNC0_MASK                                                         0x00000007L
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREVALUE0_MASK                                                        0x00000FF0L
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREMASK0_MASK                                                         0x000FF000L
+//DB_SRESULTS_COMPARE_STATE1
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREFUNC1__SHIFT                                                       0x0
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREVALUE1__SHIFT                                                      0x4
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREMASK1__SHIFT                                                       0xc
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREFUNC1_MASK                                                         0x00000007L
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREVALUE1_MASK                                                        0x00000FF0L
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREMASK1_MASK                                                         0x000FF000L
+//VGT_GS_MAX_VERT_OUT
+#define VGT_GS_MAX_VERT_OUT__MAX_VERT_OUT__SHIFT                                                              0x0
+#define VGT_GS_MAX_VERT_OUT__MAX_VERT_OUT_MASK                                                                0x000007FFL
+//VGT_GS_INSTANCE_CNT
+#define VGT_GS_INSTANCE_CNT__ENABLE__SHIFT                                                                    0x0
+#define VGT_GS_INSTANCE_CNT__CNT__SHIFT                                                                       0x2
+#define VGT_GS_INSTANCE_CNT__EN_MAX_VERT_OUT_PER_GS_INSTANCE__SHIFT                                           0x1f
+#define VGT_GS_INSTANCE_CNT__ENABLE_MASK                                                                      0x00000001L
+#define VGT_GS_INSTANCE_CNT__CNT_MASK                                                                         0x000000FCL
+#define VGT_GS_INSTANCE_CNT__EN_MAX_VERT_OUT_PER_GS_INSTANCE_MASK                                             0x80000000L
+//GE_NGG_SUBGRP_CNTL
+#define GE_NGG_SUBGRP_CNTL__PRIM_AMP_FACTOR__SHIFT                                                            0x0
+#define GE_NGG_SUBGRP_CNTL__THDS_PER_SUBGRP__SHIFT                                                            0x9
+#define GE_NGG_SUBGRP_CNTL__PRIM_AMP_FACTOR_MASK                                                              0x000001FFL
+#define GE_NGG_SUBGRP_CNTL__THDS_PER_SUBGRP_MASK                                                              0x0003FE00L
+//PA_SU_POLY_OFFSET_DB_FMT_CNTL
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_NEG_NUM_DB_BITS__SHIFT                                     0x0
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_DB_IS_FLOAT_FMT__SHIFT                                     0x8
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_NEG_NUM_DB_BITS_MASK                                       0x000000FFL
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_DB_IS_FLOAT_FMT_MASK                                       0x00000100L
+//PA_SU_POLY_OFFSET_CLAMP
+#define PA_SU_POLY_OFFSET_CLAMP__CLAMP__SHIFT                                                                 0x0
+#define PA_SU_POLY_OFFSET_CLAMP__CLAMP_MASK                                                                   0xFFFFFFFFL
+//PA_SU_POLY_OFFSET_FRONT_SCALE
+#define PA_SU_POLY_OFFSET_FRONT_SCALE__SCALE__SHIFT                                                           0x0
+#define PA_SU_POLY_OFFSET_FRONT_SCALE__SCALE_MASK                                                             0xFFFFFFFFL
+//PA_SU_POLY_OFFSET_FRONT_OFFSET
+#define PA_SU_POLY_OFFSET_FRONT_OFFSET__OFFSET__SHIFT                                                         0x0
+#define PA_SU_POLY_OFFSET_FRONT_OFFSET__OFFSET_MASK                                                           0xFFFFFFFFL
+//PA_SU_POLY_OFFSET_BACK_SCALE
+#define PA_SU_POLY_OFFSET_BACK_SCALE__SCALE__SHIFT                                                            0x0
+#define PA_SU_POLY_OFFSET_BACK_SCALE__SCALE_MASK                                                              0xFFFFFFFFL
+//PA_SU_POLY_OFFSET_BACK_OFFSET
+#define PA_SU_POLY_OFFSET_BACK_OFFSET__OFFSET__SHIFT                                                          0x0
+#define PA_SU_POLY_OFFSET_BACK_OFFSET__OFFSET_MASK                                                            0xFFFFFFFFL
+//PA_SC_HIZ_INFO
+#define PA_SC_HIZ_INFO__SURFACE_ENABLE__SHIFT                                                                 0x0
+#define PA_SC_HIZ_INFO__FORMAT__SHIFT                                                                         0x1
+#define PA_SC_HIZ_INFO__SW_MODE__SHIFT                                                                        0x2
+#define PA_SC_HIZ_INFO__DST_OUTSIDE_ZERO_TO_ONE__SHIFT                                                        0x19
+#define PA_SC_HIZ_INFO__SURFACE_ENABLE_MASK                                                                   0x00000001L
+#define PA_SC_HIZ_INFO__FORMAT_MASK                                                                           0x00000002L
+#define PA_SC_HIZ_INFO__SW_MODE_MASK                                                                          0x0000001CL
+#define PA_SC_HIZ_INFO__DST_OUTSIDE_ZERO_TO_ONE_MASK                                                          0x02000000L
+//PA_SC_HIS_INFO
+#define PA_SC_HIS_INFO__SURFACE_ENABLE__SHIFT                                                                 0x0
+#define PA_SC_HIS_INFO__SW_MODE__SHIFT                                                                        0x1
+#define PA_SC_HIS_INFO__SURFACE_ENABLE_MASK                                                                   0x00000001L
+#define PA_SC_HIS_INFO__SW_MODE_MASK                                                                          0x0000000EL
+//PA_SC_HIZ_BASE
+#define PA_SC_HIZ_BASE__BASE_256B__SHIFT                                                                      0x0
+#define PA_SC_HIZ_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//PA_SC_HIZ_BASE_EXT
+#define PA_SC_HIZ_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define PA_SC_HIZ_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//PA_SC_HIZ_SIZE_XY
+#define PA_SC_HIZ_SIZE_XY__X_MAX__SHIFT                                                                       0x0
+#define PA_SC_HIZ_SIZE_XY__Y_MAX__SHIFT                                                                       0x10
+#define PA_SC_HIZ_SIZE_XY__X_MAX_MASK                                                                         0x00001FFFL
+#define PA_SC_HIZ_SIZE_XY__Y_MAX_MASK                                                                         0x1FFF0000L
+//PA_SC_HIS_BASE
+#define PA_SC_HIS_BASE__BASE_256B__SHIFT                                                                      0x0
+#define PA_SC_HIS_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//PA_SC_HIS_BASE_EXT
+#define PA_SC_HIS_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define PA_SC_HIS_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//PA_SC_HIS_SIZE_XY
+#define PA_SC_HIS_SIZE_XY__X_MAX__SHIFT                                                                       0x0
+#define PA_SC_HIS_SIZE_XY__Y_MAX__SHIFT                                                                       0x10
+#define PA_SC_HIS_SIZE_XY__X_MAX_MASK                                                                         0x00001FFFL
+#define PA_SC_HIS_SIZE_XY__Y_MAX_MASK                                                                         0x1FFF0000L
+//PA_SC_BINNER_OUTPUT_TIMEOUT_CNTL
+#define PA_SC_BINNER_OUTPUT_TIMEOUT_CNTL__THRESHOLD__SHIFT                                                    0x0
+#define PA_SC_BINNER_OUTPUT_TIMEOUT_CNTL__THRESHOLD_MASK                                                      0x0000FFFFL
+//PA_SC_BINNER_DYNAMIC_BATCH_LIMIT
+#define PA_SC_BINNER_DYNAMIC_BATCH_LIMIT__LIMIT__SHIFT                                                        0x0
+#define PA_SC_BINNER_DYNAMIC_BATCH_LIMIT__LIMIT_MASK                                                          0x000007FFL
+//PA_SC_HISZ_CONTROL
+#define PA_SC_HISZ_CONTROL__ROUND__SHIFT                                                                      0x0
+#define PA_SC_HISZ_CONTROL__CONSERVATIVE_Z_EXPORT__SHIFT                                                      0x3
+#define PA_SC_HISZ_CONTROL__ROUND_MASK                                                                        0x00000007L
+#define PA_SC_HISZ_CONTROL__CONSERVATIVE_Z_EXPORT_MASK                                                        0x00000018L
+//PA_SC_HISZ_RENDER_OVERRIDE
+#define PA_SC_HISZ_RENDER_OVERRIDE__FORCE_HIZ_ENABLE__SHIFT                                                   0x0
+#define PA_SC_HISZ_RENDER_OVERRIDE__FORCE_HIS_ENABLE__SHIFT                                                   0x2
+#define PA_SC_HISZ_RENDER_OVERRIDE__FAST_Z_DISABLE__SHIFT                                                     0x4
+#define PA_SC_HISZ_RENDER_OVERRIDE__FAST_STENCIL_DISABLE__SHIFT                                               0x5
+#define PA_SC_HISZ_RENDER_OVERRIDE__FORCE_FULL_Z_RANGE__SHIFT                                                 0x6
+#define PA_SC_HISZ_RENDER_OVERRIDE__DISABLE_TILE_RATE_TILES__SHIFT                                            0x8
+#define PA_SC_HISZ_RENDER_OVERRIDE__DEPTH_BOUNDS_HIER_DEPTH_DISABLE__SHIFT                                    0x9
+#define PA_SC_HISZ_RENDER_OVERRIDE__PRESERVE_ZRANGE__SHIFT                                                    0xa
+#define PA_SC_HISZ_RENDER_OVERRIDE__DISABLE_FAST_PASS__SHIFT                                                  0xb
+#define PA_SC_HISZ_RENDER_OVERRIDE__FORCE_HIZ_ENABLE_MASK                                                     0x00000003L
+#define PA_SC_HISZ_RENDER_OVERRIDE__FORCE_HIS_ENABLE_MASK                                                     0x0000000CL
+#define PA_SC_HISZ_RENDER_OVERRIDE__FAST_Z_DISABLE_MASK                                                       0x00000010L
+#define PA_SC_HISZ_RENDER_OVERRIDE__FAST_STENCIL_DISABLE_MASK                                                 0x00000020L
+#define PA_SC_HISZ_RENDER_OVERRIDE__FORCE_FULL_Z_RANGE_MASK                                                   0x000000C0L
+#define PA_SC_HISZ_RENDER_OVERRIDE__DISABLE_TILE_RATE_TILES_MASK                                              0x00000100L
+#define PA_SC_HISZ_RENDER_OVERRIDE__DEPTH_BOUNDS_HIER_DEPTH_DISABLE_MASK                                      0x00000200L
+#define PA_SC_HISZ_RENDER_OVERRIDE__PRESERVE_ZRANGE_MASK                                                      0x00000400L
+#define PA_SC_HISZ_RENDER_OVERRIDE__DISABLE_FAST_PASS_MASK                                                    0x00000800L
+//PA_SC_LINE_CNTL
+#define PA_SC_LINE_CNTL__EXPAND_LINE_WIDTH__SHIFT                                                             0x9
+#define PA_SC_LINE_CNTL__LAST_PIXEL__SHIFT                                                                    0xa
+#define PA_SC_LINE_CNTL__PERPENDICULAR_ENDCAP_ENA__SHIFT                                                      0xb
+#define PA_SC_LINE_CNTL__DX10_DIAMOND_TEST_ENA__SHIFT                                                         0xc
+#define PA_SC_LINE_CNTL__EXTRA_DX_DY_PRECISION__SHIFT                                                         0xd
+#define PA_SC_LINE_CNTL__EXPAND_LINE_WIDTH_MASK                                                               0x00000200L
+#define PA_SC_LINE_CNTL__LAST_PIXEL_MASK                                                                      0x00000400L
+#define PA_SC_LINE_CNTL__PERPENDICULAR_ENDCAP_ENA_MASK                                                        0x00000800L
+#define PA_SC_LINE_CNTL__DX10_DIAMOND_TEST_ENA_MASK                                                           0x00001000L
+#define PA_SC_LINE_CNTL__EXTRA_DX_DY_PRECISION_MASK                                                           0x00002000L
+//PA_SC_AA_CONFIG
+#define PA_SC_AA_CONFIG__MSAA_NUM_SAMPLES__SHIFT                                                              0x0
+#define PA_SC_AA_CONFIG__AA_MASK_CENTROID_DTMN__SHIFT                                                         0x4
+#define PA_SC_AA_CONFIG__MSAA_EXPOSED_SAMPLES__SHIFT                                                          0x14
+#define PA_SC_AA_CONFIG__DETAIL_TO_EXPOSED_MODE__SHIFT                                                        0x18
+#define PA_SC_AA_CONFIG__PS_ITER_SAMPLES__SHIFT                                                               0x1e
+#define PA_SC_AA_CONFIG__MSAA_NUM_SAMPLES_MASK                                                                0x00000007L
+#define PA_SC_AA_CONFIG__AA_MASK_CENTROID_DTMN_MASK                                                           0x00000010L
+#define PA_SC_AA_CONFIG__MSAA_EXPOSED_SAMPLES_MASK                                                            0x00700000L
+#define PA_SC_AA_CONFIG__DETAIL_TO_EXPOSED_MODE_MASK                                                          0x03000000L
+#define PA_SC_AA_CONFIG__PS_ITER_SAMPLES_MASK                                                                 0xC0000000L
+//PA_SU_VTX_CNTL
+#define PA_SU_VTX_CNTL__PIX_CENTER__SHIFT                                                                     0x0
+#define PA_SU_VTX_CNTL__ROUND_MODE__SHIFT                                                                     0x1
+#define PA_SU_VTX_CNTL__QUANT_MODE__SHIFT                                                                     0x3
+#define PA_SU_VTX_CNTL__PIX_CENTER_MASK                                                                       0x00000001L
+#define PA_SU_VTX_CNTL__ROUND_MODE_MASK                                                                       0x00000006L
+#define PA_SU_VTX_CNTL__QUANT_MODE_MASK                                                                       0x00000038L
+//PA_SC_CENTROID_PRIORITY_0
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_0__SHIFT                                                          0x0
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_1__SHIFT                                                          0x4
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_2__SHIFT                                                          0x8
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_3__SHIFT                                                          0xc
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_4__SHIFT                                                          0x10
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_5__SHIFT                                                          0x14
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_6__SHIFT                                                          0x18
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_7__SHIFT                                                          0x1c
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_0_MASK                                                            0x0000000FL
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_1_MASK                                                            0x000000F0L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_2_MASK                                                            0x00000F00L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_3_MASK                                                            0x0000F000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_4_MASK                                                            0x000F0000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_5_MASK                                                            0x00F00000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_6_MASK                                                            0x0F000000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_7_MASK                                                            0xF0000000L
+//PA_SC_CENTROID_PRIORITY_1
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_8__SHIFT                                                          0x0
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_9__SHIFT                                                          0x4
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_10__SHIFT                                                         0x8
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_11__SHIFT                                                         0xc
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_12__SHIFT                                                         0x10
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_13__SHIFT                                                         0x14
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_14__SHIFT                                                         0x18
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_15__SHIFT                                                         0x1c
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_8_MASK                                                            0x0000000FL
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_9_MASK                                                            0x000000F0L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_10_MASK                                                           0x00000F00L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_11_MASK                                                           0x0000F000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_12_MASK                                                           0x000F0000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_13_MASK                                                           0x00F00000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_14_MASK                                                           0x0F000000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_15_MASK                                                           0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_X__SHIFT                                                       0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_Y__SHIFT                                                       0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_X__SHIFT                                                       0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_Y__SHIFT                                                       0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_X_MASK                                                         0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_Y_MASK                                                         0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_X_MASK                                                         0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_Y_MASK                                                         0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_X__SHIFT                                                       0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_Y__SHIFT                                                       0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_X__SHIFT                                                       0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_Y__SHIFT                                                       0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_X_MASK                                                         0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_Y_MASK                                                         0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_X_MASK                                                         0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_Y_MASK                                                         0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_X__SHIFT                                                       0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_Y__SHIFT                                                       0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_X__SHIFT                                                       0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_Y__SHIFT                                                       0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_X_MASK                                                         0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_Y_MASK                                                         0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_X_MASK                                                         0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_Y_MASK                                                         0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_X__SHIFT                                                       0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_Y__SHIFT                                                       0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_X__SHIFT                                                       0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_Y__SHIFT                                                       0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_X_MASK                                                         0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_Y_MASK                                                         0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_X_MASK                                                         0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_Y_MASK                                                         0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_MASK_X0Y0_X1Y0
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X0Y0__SHIFT                                                          0x0
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X1Y0__SHIFT                                                          0x10
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X0Y0_MASK                                                            0x0000FFFFL
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X1Y0_MASK                                                            0xFFFF0000L
+//PA_SC_AA_MASK_X0Y1_X1Y1
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X0Y1__SHIFT                                                          0x0
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X1Y1__SHIFT                                                          0x10
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X0Y1_MASK                                                            0x0000FFFFL
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X1Y1_MASK                                                            0xFFFF0000L
+//PA_SC_BINNER_OUTPUT_TIMEOUT_COUNTER
+#define PA_SC_BINNER_OUTPUT_TIMEOUT_COUNTER__THRESHOLD__SHIFT                                                 0x0
+#define PA_SC_BINNER_OUTPUT_TIMEOUT_COUNTER__THRESHOLD_MASK                                                   0xFFFFFFFFL
+//PA_SC_BINNER_CNTL_0
+#define PA_SC_BINNER_CNTL_0__BINNING_MODE__SHIFT                                                              0x0
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_X__SHIFT                                                                0x2
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_Y__SHIFT                                                                0x3
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_X_EXTEND__SHIFT                                                         0x4
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_Y_EXTEND__SHIFT                                                         0x7
+#define PA_SC_BINNER_CNTL_0__CONTEXT_STATES_PER_BIN__SHIFT                                                    0xa
+#define PA_SC_BINNER_CNTL_0__PERSISTENT_STATES_PER_BIN__SHIFT                                                 0xd
+#define PA_SC_BINNER_CNTL_0__DISABLE_START_OF_PRIM__SHIFT                                                     0x12
+#define PA_SC_BINNER_CNTL_0__FPOVS_PER_BATCH__SHIFT                                                           0x13
+#define PA_SC_BINNER_CNTL_0__OPTIMAL_BIN_SELECTION__SHIFT                                                     0x1b
+#define PA_SC_BINNER_CNTL_0__FLUSH_ON_BINNING_TRANSITION__SHIFT                                               0x1c
+#define PA_SC_BINNER_CNTL_0__BIN_MAPPING_MODE__SHIFT                                                          0x1d
+#define PA_SC_BINNER_CNTL_0__RESERVED_31__SHIFT                                                               0x1f
+#define PA_SC_BINNER_CNTL_0__BINNING_MODE_MASK                                                                0x00000003L
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_X_MASK                                                                  0x00000004L
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_Y_MASK                                                                  0x00000008L
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_X_EXTEND_MASK                                                           0x00000070L
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_Y_EXTEND_MASK                                                           0x00000380L
+#define PA_SC_BINNER_CNTL_0__CONTEXT_STATES_PER_BIN_MASK                                                      0x00001C00L
+#define PA_SC_BINNER_CNTL_0__PERSISTENT_STATES_PER_BIN_MASK                                                   0x0003E000L
+#define PA_SC_BINNER_CNTL_0__DISABLE_START_OF_PRIM_MASK                                                       0x00040000L
+#define PA_SC_BINNER_CNTL_0__FPOVS_PER_BATCH_MASK                                                             0x07F80000L
+#define PA_SC_BINNER_CNTL_0__OPTIMAL_BIN_SELECTION_MASK                                                       0x08000000L
+#define PA_SC_BINNER_CNTL_0__FLUSH_ON_BINNING_TRANSITION_MASK                                                 0x10000000L
+#define PA_SC_BINNER_CNTL_0__BIN_MAPPING_MODE_MASK                                                            0x60000000L
+#define PA_SC_BINNER_CNTL_0__RESERVED_31_MASK                                                                 0x80000000L
+//PA_SC_BINNER_CNTL_1
+#define PA_SC_BINNER_CNTL_1__MAX_ALLOC_COUNT__SHIFT                                                           0x0
+#define PA_SC_BINNER_CNTL_1__MAX_PRIM_PER_BATCH__SHIFT                                                        0x10
+#define PA_SC_BINNER_CNTL_1__MAX_ALLOC_COUNT_MASK                                                             0x0000FFFFL
+#define PA_SC_BINNER_CNTL_1__MAX_PRIM_PER_BATCH_MASK                                                          0xFFFF0000L
+//PA_SC_BINNER_CNTL_2
+#define PA_SC_BINNER_CNTL_2__BIN_SIZE_X_MULT_BY_1P5X__SHIFT                                                   0x0
+#define PA_SC_BINNER_CNTL_2__BIN_SIZE_Y_MULT_BY_1P5X__SHIFT                                                   0x1
+#define PA_SC_BINNER_CNTL_2__ENABLE_LIGHT_VOLUME_RENDERING_OPTIMIZATION__SHIFT                                0x2
+#define PA_SC_BINNER_CNTL_2__DUAL_LIGHT_SHAFT_IN_DRAW__SHIFT                                                  0x3
+#define PA_SC_BINNER_CNTL_2__RESERVED_LIGHT_SHAFT_DRAW_CALL_LIMIT__SHIFT                                      0x4
+#define PA_SC_BINNER_CNTL_2__CONTEXT_DONE_EVENTS_PER_BIN__SHIFT                                               0x7
+#define PA_SC_BINNER_CNTL_2__ZPP_ENABLED__SHIFT                                                               0xb
+#define PA_SC_BINNER_CNTL_2__ZPP_OPTIMIZATION_ENABLED__SHIFT                                                  0xc
+#define PA_SC_BINNER_CNTL_2__ZPP_AREA_THRESHOLD__SHIFT                                                        0xd
+#define PA_SC_BINNER_CNTL_2__DISABLE_NOPCEXPORT_BREAKBATCH_CONDITION__SHIFT                                   0x15
+#define PA_SC_BINNER_CNTL_2__SBB_ENABLE__SHIFT                                                                0x16
+#define PA_SC_BINNER_CNTL_2__ENABLE_PING_PONG_BIN_ORDER__SHIFT                                                0x17
+#define PA_SC_BINNER_CNTL_2__PING_PONG_BIN_ORDER_FLIP__SHIFT                                                  0x18
+#define PA_SC_BINNER_CNTL_2__LIGHT_SHAFT_DRAW_CALL_LIMIT__SHIFT                                               0x1a
+#define PA_SC_BINNER_CNTL_2__BIN_SIZE_X_MULT_BY_1P5X_MASK                                                     0x00000001L
+#define PA_SC_BINNER_CNTL_2__BIN_SIZE_Y_MULT_BY_1P5X_MASK                                                     0x00000002L
+#define PA_SC_BINNER_CNTL_2__ENABLE_LIGHT_VOLUME_RENDERING_OPTIMIZATION_MASK                                  0x00000004L
+#define PA_SC_BINNER_CNTL_2__DUAL_LIGHT_SHAFT_IN_DRAW_MASK                                                    0x00000008L
+#define PA_SC_BINNER_CNTL_2__RESERVED_LIGHT_SHAFT_DRAW_CALL_LIMIT_MASK                                        0x00000070L
+#define PA_SC_BINNER_CNTL_2__CONTEXT_DONE_EVENTS_PER_BIN_MASK                                                 0x00000780L
+#define PA_SC_BINNER_CNTL_2__ZPP_ENABLED_MASK                                                                 0x00000800L
+#define PA_SC_BINNER_CNTL_2__ZPP_OPTIMIZATION_ENABLED_MASK                                                    0x00001000L
+#define PA_SC_BINNER_CNTL_2__ZPP_AREA_THRESHOLD_MASK                                                          0x001FE000L
+#define PA_SC_BINNER_CNTL_2__DISABLE_NOPCEXPORT_BREAKBATCH_CONDITION_MASK                                     0x00200000L
+#define PA_SC_BINNER_CNTL_2__SBB_ENABLE_MASK                                                                  0x00400000L
+#define PA_SC_BINNER_CNTL_2__ENABLE_PING_PONG_BIN_ORDER_MASK                                                  0x00800000L
+#define PA_SC_BINNER_CNTL_2__PING_PONG_BIN_ORDER_FLIP_MASK                                                    0x03000000L
+#define PA_SC_BINNER_CNTL_2__LIGHT_SHAFT_DRAW_CALL_LIMIT_MASK                                                 0x7C000000L
+//PA_SC_NGG_MODE_CNTL
+#define PA_SC_NGG_MODE_CNTL__MAX_DEALLOCS_IN_WAVE__SHIFT                                                      0x0
+#define PA_SC_NGG_MODE_CNTL__DISABLE_FPOG_AND_DEALLOC_CONFLICT__SHIFT                                         0xc
+#define PA_SC_NGG_MODE_CNTL__DISABLE_MAX_DEALLOC__SHIFT                                                       0xd
+#define PA_SC_NGG_MODE_CNTL__DISABLE_MAX_ATTRIBUTES__SHIFT                                                    0xe
+#define PA_SC_NGG_MODE_CNTL__MAX_FPOVS_IN_WAVE__SHIFT                                                         0x10
+#define PA_SC_NGG_MODE_CNTL__MAX_ATTRIBUTES_IN_WAVE__SHIFT                                                    0x18
+#define PA_SC_NGG_MODE_CNTL__MAX_DEALLOCS_IN_WAVE_MASK                                                        0x000007FFL
+#define PA_SC_NGG_MODE_CNTL__DISABLE_FPOG_AND_DEALLOC_CONFLICT_MASK                                           0x00001000L
+#define PA_SC_NGG_MODE_CNTL__DISABLE_MAX_DEALLOC_MASK                                                         0x00002000L
+#define PA_SC_NGG_MODE_CNTL__DISABLE_MAX_ATTRIBUTES_MASK                                                      0x00004000L
+#define PA_SC_NGG_MODE_CNTL__MAX_FPOVS_IN_WAVE_MASK                                                           0x00FF0000L
+#define PA_SC_NGG_MODE_CNTL__MAX_ATTRIBUTES_IN_WAVE_MASK                                                      0xFF000000L
+//PA_SC_CONSERVATIVE_RASTERIZATION_CNTL
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVER_RAST_ENABLE__SHIFT                                        0x0
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVER_RAST_SAMPLE_SELECT__SHIFT                                 0x1
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNDER_RAST_ENABLE__SHIFT                                       0x5
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNDER_RAST_SAMPLE_SELECT__SHIFT                                0x6
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__PBB_UNCERTAINTY_REGION_ENABLE__SHIFT                           0xa
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__ZMM_TRI_EXTENT__SHIFT                                          0xb
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__ZMM_TRI_OFFSET__SHIFT                                          0xc
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVERRIDE_OVER_RAST_INNER_TO_NORMAL__SHIFT                      0xd
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVERRIDE_UNDER_RAST_INNER_TO_NORMAL__SHIFT                     0xe
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__DEGENERATE_OVERRIDE_INNER_TO_NORMAL_DISABLE__SHIFT             0xf
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNCERTAINTY_REGION_MODE__SHIFT                                 0x10
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OUTER_UNCERTAINTY_EDGERULE_OVERRIDE__SHIFT                     0x12
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__INNER_UNCERTAINTY_EDGERULE_OVERRIDE__SHIFT                     0x13
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__NULL_SQUAD_AA_MASK_ENABLE__SHIFT                               0x14
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__COVERAGE_AA_MASK_ENABLE__SHIFT                                 0x15
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__PREZ_AA_MASK_ENABLE__SHIFT                                     0x16
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__POSTZ_AA_MASK_ENABLE__SHIFT                                    0x17
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__CENTROID_SAMPLE_OVERRIDE__SHIFT                                0x18
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNCERTAINTY_REGION_MULT__SHIFT                                 0x19
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNCERTAINTY_REGION_PBB_MULT__SHIFT                             0x1b
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVER_RAST_ENABLE_MASK                                          0x00000001L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVER_RAST_SAMPLE_SELECT_MASK                                   0x0000001EL
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNDER_RAST_ENABLE_MASK                                         0x00000020L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNDER_RAST_SAMPLE_SELECT_MASK                                  0x000003C0L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__PBB_UNCERTAINTY_REGION_ENABLE_MASK                             0x00000400L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__ZMM_TRI_EXTENT_MASK                                            0x00000800L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__ZMM_TRI_OFFSET_MASK                                            0x00001000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVERRIDE_OVER_RAST_INNER_TO_NORMAL_MASK                        0x00002000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVERRIDE_UNDER_RAST_INNER_TO_NORMAL_MASK                       0x00004000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__DEGENERATE_OVERRIDE_INNER_TO_NORMAL_DISABLE_MASK               0x00008000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNCERTAINTY_REGION_MODE_MASK                                   0x00030000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OUTER_UNCERTAINTY_EDGERULE_OVERRIDE_MASK                       0x00040000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__INNER_UNCERTAINTY_EDGERULE_OVERRIDE_MASK                       0x00080000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__NULL_SQUAD_AA_MASK_ENABLE_MASK                                 0x00100000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__COVERAGE_AA_MASK_ENABLE_MASK                                   0x00200000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__PREZ_AA_MASK_ENABLE_MASK                                       0x00400000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__POSTZ_AA_MASK_ENABLE_MASK                                      0x00800000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__CENTROID_SAMPLE_OVERRIDE_MASK                                  0x01000000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNCERTAINTY_REGION_MULT_MASK                                   0x06000000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNCERTAINTY_REGION_PBB_MULT_MASK                               0x18000000L
+//PA_SC_SHADER_CONTROL
+#define PA_SC_SHADER_CONTROL__REALIGN_DQUADS_AFTER_N_WAVES__SHIFT                                             0x0
+#define PA_SC_SHADER_CONTROL__LOAD_COLLISION_WAVEID__SHIFT                                                    0x2
+#define PA_SC_SHADER_CONTROL__LOAD_INTRAWAVE_COLLISION__SHIFT                                                 0x3
+#define PA_SC_SHADER_CONTROL__WAVE_BREAK_REGION_SIZE__SHIFT                                                   0x5
+#define PA_SC_SHADER_CONTROL__DISABLE_OREO_CONFLICT_QUAD__SHIFT                                               0x7
+#define PA_SC_SHADER_CONTROL__PS_ITER_SAMPLE__SHIFT                                                           0x8
+#define PA_SC_SHADER_CONTROL__REALIGN_DQUADS_AFTER_N_WAVES_MASK                                               0x00000003L
+#define PA_SC_SHADER_CONTROL__LOAD_COLLISION_WAVEID_MASK                                                      0x00000004L
+#define PA_SC_SHADER_CONTROL__LOAD_INTRAWAVE_COLLISION_MASK                                                   0x00000008L
+#define PA_SC_SHADER_CONTROL__WAVE_BREAK_REGION_SIZE_MASK                                                     0x00000060L
+#define PA_SC_SHADER_CONTROL__DISABLE_OREO_CONFLICT_QUAD_MASK                                                 0x00000080L
+#define PA_SC_SHADER_CONTROL__PS_ITER_SAMPLE_MASK                                                             0x00000100L
+//PA_SC_SAMPLE_PROPERTIES
+#define PA_SC_SAMPLE_PROPERTIES__MAX_SAMPLE_DIST__SHIFT                                                       0x0
+#define PA_SC_SAMPLE_PROPERTIES__MAX_SAMPLE_DIST_MASK                                                         0x0000000FL
+//CB_COLOR0_BASE
+#define CB_COLOR0_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR0_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR0_VIEW
+#define CB_COLOR0_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR0_VIEW__SLICE_MAX__SHIFT                                                                      0xe
+#define CB_COLOR0_VIEW__SLICE_START_MASK                                                                      0x00003FFFL
+#define CB_COLOR0_VIEW__SLICE_MAX_MASK                                                                        0x0FFFC000L
+//CB_COLOR0_VIEW2
+#define CB_COLOR0_VIEW2__MIP_LEVEL__SHIFT                                                                     0x0
+#define CB_COLOR0_VIEW2__MIP_LEVEL_MASK                                                                       0x0000001FL
+//CB_COLOR0_ATTRIB
+#define CB_COLOR0_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0x0
+#define CB_COLOR0_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x2
+#define CB_COLOR0_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX__SHIFT                                         0x3
+#define CB_COLOR0_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00000003L
+#define CB_COLOR0_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00000004L
+#define CB_COLOR0_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX_MASK                                           0x00000008L
+//CB_COLOR0_FDCC_CONTROL
+#define CB_COLOR0_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                            0x2
+#define CB_COLOR0_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x5
+#define CB_COLOR0_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE__SHIFT                                              0x18
+#define CB_COLOR0_FDCC_CONTROL__COMPRESSION_MODE__SHIFT                                                       0x1a
+#define CB_COLOR0_FDCC_CONTROL__ENABLE_MAX_COMP_FRAG_OVERRIDE__SHIFT                                          0x1c
+#define CB_COLOR0_FDCC_CONTROL__MAX_COMP_FRAGS__SHIFT                                                         0x1d
+#define CB_COLOR0_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                              0x00000004L
+#define CB_COLOR0_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000060L
+#define CB_COLOR0_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE_MASK                                                0x01000000L
+#define CB_COLOR0_FDCC_CONTROL__COMPRESSION_MODE_MASK                                                         0x0C000000L
+#define CB_COLOR0_FDCC_CONTROL__ENABLE_MAX_COMP_FRAG_OVERRIDE_MASK                                            0x10000000L
+#define CB_COLOR0_FDCC_CONTROL__MAX_COMP_FRAGS_MASK                                                           0x60000000L
+//CB_COLOR0_ATTRIB2
+#define CB_COLOR0_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR0_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0x10
+#define CB_COLOR0_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x0000FFFFL
+#define CB_COLOR0_ATTRIB2__MIP0_WIDTH_MASK                                                                    0xFFFF0000L
+//CB_COLOR0_ATTRIB3
+#define CB_COLOR0_ATTRIB3__MIP0_DEPTH__SHIFT                                                                  0x0
+#define CB_COLOR0_ATTRIB3__COLOR_SW_MODE__SHIFT                                                               0xf
+#define CB_COLOR0_ATTRIB3__MAX_MIP__SHIFT                                                                     0x13
+#define CB_COLOR0_ATTRIB3__RESOURCE_TYPE__SHIFT                                                               0x18
+#define CB_COLOR0_ATTRIB3__SPECULATIVE_READ__SHIFT                                                            0x1a
+#define CB_COLOR0_ATTRIB3__MIP0_DEPTH_MASK                                                                    0x00003FFFL
+#define CB_COLOR0_ATTRIB3__COLOR_SW_MODE_MASK                                                                 0x00038000L
+#define CB_COLOR0_ATTRIB3__MAX_MIP_MASK                                                                       0x00F80000L
+#define CB_COLOR0_ATTRIB3__RESOURCE_TYPE_MASK                                                                 0x03000000L
+#define CB_COLOR0_ATTRIB3__SPECULATIVE_READ_MASK                                                              0x0C000000L
+//CB_COLOR1_BASE
+#define CB_COLOR1_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR1_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR1_VIEW
+#define CB_COLOR1_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR1_VIEW__SLICE_MAX__SHIFT                                                                      0xe
+#define CB_COLOR1_VIEW__SLICE_START_MASK                                                                      0x00003FFFL
+#define CB_COLOR1_VIEW__SLICE_MAX_MASK                                                                        0x0FFFC000L
+//CB_COLOR1_VIEW2
+#define CB_COLOR1_VIEW2__MIP_LEVEL__SHIFT                                                                     0x0
+#define CB_COLOR1_VIEW2__MIP_LEVEL_MASK                                                                       0x0000001FL
+//CB_COLOR1_ATTRIB
+#define CB_COLOR1_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0x0
+#define CB_COLOR1_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x2
+#define CB_COLOR1_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX__SHIFT                                         0x3
+#define CB_COLOR1_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00000003L
+#define CB_COLOR1_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00000004L
+#define CB_COLOR1_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX_MASK                                           0x00000008L
+//CB_COLOR1_FDCC_CONTROL
+#define CB_COLOR1_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                            0x2
+#define CB_COLOR1_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x5
+#define CB_COLOR1_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE__SHIFT                                              0x18
+#define CB_COLOR1_FDCC_CONTROL__COMPRESSION_MODE__SHIFT                                                       0x1a
+#define CB_COLOR1_FDCC_CONTROL__ENABLE_MAX_COMP_FRAG_OVERRIDE__SHIFT                                          0x1c
+#define CB_COLOR1_FDCC_CONTROL__MAX_COMP_FRAGS__SHIFT                                                         0x1d
+#define CB_COLOR1_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                              0x00000004L
+#define CB_COLOR1_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000060L
+#define CB_COLOR1_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE_MASK                                                0x01000000L
+#define CB_COLOR1_FDCC_CONTROL__COMPRESSION_MODE_MASK                                                         0x0C000000L
+#define CB_COLOR1_FDCC_CONTROL__ENABLE_MAX_COMP_FRAG_OVERRIDE_MASK                                            0x10000000L
+#define CB_COLOR1_FDCC_CONTROL__MAX_COMP_FRAGS_MASK                                                           0x60000000L
+//CB_COLOR1_ATTRIB2
+#define CB_COLOR1_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR1_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0x10
+#define CB_COLOR1_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x0000FFFFL
+#define CB_COLOR1_ATTRIB2__MIP0_WIDTH_MASK                                                                    0xFFFF0000L
+//CB_COLOR1_ATTRIB3
+#define CB_COLOR1_ATTRIB3__MIP0_DEPTH__SHIFT                                                                  0x0
+#define CB_COLOR1_ATTRIB3__COLOR_SW_MODE__SHIFT                                                               0xf
+#define CB_COLOR1_ATTRIB3__MAX_MIP__SHIFT                                                                     0x13
+#define CB_COLOR1_ATTRIB3__RESOURCE_TYPE__SHIFT                                                               0x18
+#define CB_COLOR1_ATTRIB3__SPECULATIVE_READ__SHIFT                                                            0x1a
+#define CB_COLOR1_ATTRIB3__MIP0_DEPTH_MASK                                                                    0x00003FFFL
+#define CB_COLOR1_ATTRIB3__COLOR_SW_MODE_MASK                                                                 0x00038000L
+#define CB_COLOR1_ATTRIB3__MAX_MIP_MASK                                                                       0x00F80000L
+#define CB_COLOR1_ATTRIB3__RESOURCE_TYPE_MASK                                                                 0x03000000L
+#define CB_COLOR1_ATTRIB3__SPECULATIVE_READ_MASK                                                              0x0C000000L
+//CB_COLOR2_BASE
+#define CB_COLOR2_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR2_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR2_VIEW
+#define CB_COLOR2_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR2_VIEW__SLICE_MAX__SHIFT                                                                      0xe
+#define CB_COLOR2_VIEW__SLICE_START_MASK                                                                      0x00003FFFL
+#define CB_COLOR2_VIEW__SLICE_MAX_MASK                                                                        0x0FFFC000L
+//CB_COLOR2_VIEW2
+#define CB_COLOR2_VIEW2__MIP_LEVEL__SHIFT                                                                     0x0
+#define CB_COLOR2_VIEW2__MIP_LEVEL_MASK                                                                       0x0000001FL
+//CB_COLOR2_ATTRIB
+#define CB_COLOR2_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0x0
+#define CB_COLOR2_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x2
+#define CB_COLOR2_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX__SHIFT                                         0x3
+#define CB_COLOR2_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00000003L
+#define CB_COLOR2_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00000004L
+#define CB_COLOR2_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX_MASK                                           0x00000008L
+//CB_COLOR2_FDCC_CONTROL
+#define CB_COLOR2_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                            0x2
+#define CB_COLOR2_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x5
+#define CB_COLOR2_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE__SHIFT                                              0x18
+#define CB_COLOR2_FDCC_CONTROL__COMPRESSION_MODE__SHIFT                                                       0x1a
+#define CB_COLOR2_FDCC_CONTROL__ENABLE_MAX_COMP_FRAG_OVERRIDE__SHIFT                                          0x1c
+#define CB_COLOR2_FDCC_CONTROL__MAX_COMP_FRAGS__SHIFT                                                         0x1d
+#define CB_COLOR2_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                              0x00000004L
+#define CB_COLOR2_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000060L
+#define CB_COLOR2_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE_MASK                                                0x01000000L
+#define CB_COLOR2_FDCC_CONTROL__COMPRESSION_MODE_MASK                                                         0x0C000000L
+#define CB_COLOR2_FDCC_CONTROL__ENABLE_MAX_COMP_FRAG_OVERRIDE_MASK                                            0x10000000L
+#define CB_COLOR2_FDCC_CONTROL__MAX_COMP_FRAGS_MASK                                                           0x60000000L
+//CB_COLOR2_ATTRIB2
+#define CB_COLOR2_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR2_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0x10
+#define CB_COLOR2_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x0000FFFFL
+#define CB_COLOR2_ATTRIB2__MIP0_WIDTH_MASK                                                                    0xFFFF0000L
+//CB_COLOR2_ATTRIB3
+#define CB_COLOR2_ATTRIB3__MIP0_DEPTH__SHIFT                                                                  0x0
+#define CB_COLOR2_ATTRIB3__COLOR_SW_MODE__SHIFT                                                               0xf
+#define CB_COLOR2_ATTRIB3__MAX_MIP__SHIFT                                                                     0x13
+#define CB_COLOR2_ATTRIB3__RESOURCE_TYPE__SHIFT                                                               0x18
+#define CB_COLOR2_ATTRIB3__SPECULATIVE_READ__SHIFT                                                            0x1a
+#define CB_COLOR2_ATTRIB3__MIP0_DEPTH_MASK                                                                    0x00003FFFL
+#define CB_COLOR2_ATTRIB3__COLOR_SW_MODE_MASK                                                                 0x00038000L
+#define CB_COLOR2_ATTRIB3__MAX_MIP_MASK                                                                       0x00F80000L
+#define CB_COLOR2_ATTRIB3__RESOURCE_TYPE_MASK                                                                 0x03000000L
+#define CB_COLOR2_ATTRIB3__SPECULATIVE_READ_MASK                                                              0x0C000000L
+//CB_COLOR3_BASE
+#define CB_COLOR3_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR3_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR3_VIEW
+#define CB_COLOR3_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR3_VIEW__SLICE_MAX__SHIFT                                                                      0xe
+#define CB_COLOR3_VIEW__SLICE_START_MASK                                                                      0x00003FFFL
+#define CB_COLOR3_VIEW__SLICE_MAX_MASK                                                                        0x0FFFC000L
+//CB_COLOR3_VIEW2
+#define CB_COLOR3_VIEW2__MIP_LEVEL__SHIFT                                                                     0x0
+#define CB_COLOR3_VIEW2__MIP_LEVEL_MASK                                                                       0x0000001FL
+//CB_COLOR3_ATTRIB
+#define CB_COLOR3_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0x0
+#define CB_COLOR3_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x2
+#define CB_COLOR3_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX__SHIFT                                         0x3
+#define CB_COLOR3_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00000003L
+#define CB_COLOR3_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00000004L
+#define CB_COLOR3_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX_MASK                                           0x00000008L
+//CB_COLOR3_FDCC_CONTROL
+#define CB_COLOR3_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                            0x2
+#define CB_COLOR3_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x5
+#define CB_COLOR3_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE__SHIFT                                              0x18
+#define CB_COLOR3_FDCC_CONTROL__COMPRESSION_MODE__SHIFT                                                       0x1a
+#define CB_COLOR3_FDCC_CONTROL__ENABLE_MAX_COMP_FRAG_OVERRIDE__SHIFT                                          0x1c
+#define CB_COLOR3_FDCC_CONTROL__MAX_COMP_FRAGS__SHIFT                                                         0x1d
+#define CB_COLOR3_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                              0x00000004L
+#define CB_COLOR3_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000060L
+#define CB_COLOR3_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE_MASK                                                0x01000000L
+#define CB_COLOR3_FDCC_CONTROL__COMPRESSION_MODE_MASK                                                         0x0C000000L
+#define CB_COLOR3_FDCC_CONTROL__ENABLE_MAX_COMP_FRAG_OVERRIDE_MASK                                            0x10000000L
+#define CB_COLOR3_FDCC_CONTROL__MAX_COMP_FRAGS_MASK                                                           0x60000000L
+//CB_COLOR3_ATTRIB2
+#define CB_COLOR3_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR3_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0x10
+#define CB_COLOR3_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x0000FFFFL
+#define CB_COLOR3_ATTRIB2__MIP0_WIDTH_MASK                                                                    0xFFFF0000L
+//CB_COLOR3_ATTRIB3
+#define CB_COLOR3_ATTRIB3__MIP0_DEPTH__SHIFT                                                                  0x0
+#define CB_COLOR3_ATTRIB3__COLOR_SW_MODE__SHIFT                                                               0xf
+#define CB_COLOR3_ATTRIB3__MAX_MIP__SHIFT                                                                     0x13
+#define CB_COLOR3_ATTRIB3__RESOURCE_TYPE__SHIFT                                                               0x18
+#define CB_COLOR3_ATTRIB3__SPECULATIVE_READ__SHIFT                                                            0x1a
+#define CB_COLOR3_ATTRIB3__MIP0_DEPTH_MASK                                                                    0x00003FFFL
+#define CB_COLOR3_ATTRIB3__COLOR_SW_MODE_MASK                                                                 0x00038000L
+#define CB_COLOR3_ATTRIB3__MAX_MIP_MASK                                                                       0x00F80000L
+#define CB_COLOR3_ATTRIB3__RESOURCE_TYPE_MASK                                                                 0x03000000L
+#define CB_COLOR3_ATTRIB3__SPECULATIVE_READ_MASK                                                              0x0C000000L
+//CB_COLOR4_BASE
+#define CB_COLOR4_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR4_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR4_VIEW
+#define CB_COLOR4_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR4_VIEW__SLICE_MAX__SHIFT                                                                      0xe
+#define CB_COLOR4_VIEW__SLICE_START_MASK                                                                      0x00003FFFL
+#define CB_COLOR4_VIEW__SLICE_MAX_MASK                                                                        0x0FFFC000L
+//CB_COLOR4_VIEW2
+#define CB_COLOR4_VIEW2__MIP_LEVEL__SHIFT                                                                     0x0
+#define CB_COLOR4_VIEW2__MIP_LEVEL_MASK                                                                       0x0000001FL
+//CB_COLOR4_ATTRIB
+#define CB_COLOR4_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0x0
+#define CB_COLOR4_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x2
+#define CB_COLOR4_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX__SHIFT                                         0x3
+#define CB_COLOR4_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00000003L
+#define CB_COLOR4_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00000004L
+#define CB_COLOR4_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX_MASK                                           0x00000008L
+//CB_COLOR4_FDCC_CONTROL
+#define CB_COLOR4_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                            0x2
+#define CB_COLOR4_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x5
+#define CB_COLOR4_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE__SHIFT                                              0x18
+#define CB_COLOR4_FDCC_CONTROL__COMPRESSION_MODE__SHIFT                                                       0x1a
+#define CB_COLOR4_FDCC_CONTROL__ENABLE_MAX_COMP_FRAG_OVERRIDE__SHIFT                                          0x1c
+#define CB_COLOR4_FDCC_CONTROL__MAX_COMP_FRAGS__SHIFT                                                         0x1d
+#define CB_COLOR4_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                              0x00000004L
+#define CB_COLOR4_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000060L
+#define CB_COLOR4_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE_MASK                                                0x01000000L
+#define CB_COLOR4_FDCC_CONTROL__COMPRESSION_MODE_MASK                                                         0x0C000000L
+#define CB_COLOR4_FDCC_CONTROL__ENABLE_MAX_COMP_FRAG_OVERRIDE_MASK                                            0x10000000L
+#define CB_COLOR4_FDCC_CONTROL__MAX_COMP_FRAGS_MASK                                                           0x60000000L
+//CB_COLOR4_ATTRIB2
+#define CB_COLOR4_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR4_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0x10
+#define CB_COLOR4_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x0000FFFFL
+#define CB_COLOR4_ATTRIB2__MIP0_WIDTH_MASK                                                                    0xFFFF0000L
+//CB_COLOR4_ATTRIB3
+#define CB_COLOR4_ATTRIB3__MIP0_DEPTH__SHIFT                                                                  0x0
+#define CB_COLOR4_ATTRIB3__COLOR_SW_MODE__SHIFT                                                               0xf
+#define CB_COLOR4_ATTRIB3__MAX_MIP__SHIFT                                                                     0x13
+#define CB_COLOR4_ATTRIB3__RESOURCE_TYPE__SHIFT                                                               0x18
+#define CB_COLOR4_ATTRIB3__SPECULATIVE_READ__SHIFT                                                            0x1a
+#define CB_COLOR4_ATTRIB3__MIP0_DEPTH_MASK                                                                    0x00003FFFL
+#define CB_COLOR4_ATTRIB3__COLOR_SW_MODE_MASK                                                                 0x00038000L
+#define CB_COLOR4_ATTRIB3__MAX_MIP_MASK                                                                       0x00F80000L
+#define CB_COLOR4_ATTRIB3__RESOURCE_TYPE_MASK                                                                 0x03000000L
+#define CB_COLOR4_ATTRIB3__SPECULATIVE_READ_MASK                                                              0x0C000000L
+//CB_COLOR5_BASE
+#define CB_COLOR5_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR5_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR5_VIEW
+#define CB_COLOR5_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR5_VIEW__SLICE_MAX__SHIFT                                                                      0xe
+#define CB_COLOR5_VIEW__SLICE_START_MASK                                                                      0x00003FFFL
+#define CB_COLOR5_VIEW__SLICE_MAX_MASK                                                                        0x0FFFC000L
+//CB_COLOR5_VIEW2
+#define CB_COLOR5_VIEW2__MIP_LEVEL__SHIFT                                                                     0x0
+#define CB_COLOR5_VIEW2__MIP_LEVEL_MASK                                                                       0x0000001FL
+//CB_COLOR5_ATTRIB
+#define CB_COLOR5_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0x0
+#define CB_COLOR5_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x2
+#define CB_COLOR5_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX__SHIFT                                         0x3
+#define CB_COLOR5_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00000003L
+#define CB_COLOR5_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00000004L
+#define CB_COLOR5_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX_MASK                                           0x00000008L
+//CB_COLOR5_FDCC_CONTROL
+#define CB_COLOR5_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                            0x2
+#define CB_COLOR5_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x5
+#define CB_COLOR5_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE__SHIFT                                              0x18
+#define CB_COLOR5_FDCC_CONTROL__COMPRESSION_MODE__SHIFT                                                       0x1a
+#define CB_COLOR5_FDCC_CONTROL__ENABLE_MAX_COMP_FRAG_OVERRIDE__SHIFT                                          0x1c
+#define CB_COLOR5_FDCC_CONTROL__MAX_COMP_FRAGS__SHIFT                                                         0x1d
+#define CB_COLOR5_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                              0x00000004L
+#define CB_COLOR5_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000060L
+#define CB_COLOR5_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE_MASK                                                0x01000000L
+#define CB_COLOR5_FDCC_CONTROL__COMPRESSION_MODE_MASK                                                         0x0C000000L
+#define CB_COLOR5_FDCC_CONTROL__ENABLE_MAX_COMP_FRAG_OVERRIDE_MASK                                            0x10000000L
+#define CB_COLOR5_FDCC_CONTROL__MAX_COMP_FRAGS_MASK                                                           0x60000000L
+//CB_COLOR5_ATTRIB2
+#define CB_COLOR5_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR5_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0x10
+#define CB_COLOR5_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x0000FFFFL
+#define CB_COLOR5_ATTRIB2__MIP0_WIDTH_MASK                                                                    0xFFFF0000L
+//CB_COLOR5_ATTRIB3
+#define CB_COLOR5_ATTRIB3__MIP0_DEPTH__SHIFT                                                                  0x0
+#define CB_COLOR5_ATTRIB3__COLOR_SW_MODE__SHIFT                                                               0xf
+#define CB_COLOR5_ATTRIB3__MAX_MIP__SHIFT                                                                     0x13
+#define CB_COLOR5_ATTRIB3__RESOURCE_TYPE__SHIFT                                                               0x18
+#define CB_COLOR5_ATTRIB3__SPECULATIVE_READ__SHIFT                                                            0x1a
+#define CB_COLOR5_ATTRIB3__MIP0_DEPTH_MASK                                                                    0x00003FFFL
+#define CB_COLOR5_ATTRIB3__COLOR_SW_MODE_MASK                                                                 0x00038000L
+#define CB_COLOR5_ATTRIB3__MAX_MIP_MASK                                                                       0x00F80000L
+#define CB_COLOR5_ATTRIB3__RESOURCE_TYPE_MASK                                                                 0x03000000L
+#define CB_COLOR5_ATTRIB3__SPECULATIVE_READ_MASK                                                              0x0C000000L
+//CB_COLOR6_BASE
+#define CB_COLOR6_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR6_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR6_VIEW
+#define CB_COLOR6_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR6_VIEW__SLICE_MAX__SHIFT                                                                      0xe
+#define CB_COLOR6_VIEW__SLICE_START_MASK                                                                      0x00003FFFL
+#define CB_COLOR6_VIEW__SLICE_MAX_MASK                                                                        0x0FFFC000L
+//CB_COLOR6_VIEW2
+#define CB_COLOR6_VIEW2__MIP_LEVEL__SHIFT                                                                     0x0
+#define CB_COLOR6_VIEW2__MIP_LEVEL_MASK                                                                       0x0000001FL
+//CB_COLOR6_ATTRIB
+#define CB_COLOR6_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0x0
+#define CB_COLOR6_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x2
+#define CB_COLOR6_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX__SHIFT                                         0x3
+#define CB_COLOR6_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00000003L
+#define CB_COLOR6_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00000004L
+#define CB_COLOR6_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX_MASK                                           0x00000008L
+//CB_COLOR6_FDCC_CONTROL
+#define CB_COLOR6_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                            0x2
+#define CB_COLOR6_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x5
+#define CB_COLOR6_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE__SHIFT                                              0x18
+#define CB_COLOR6_FDCC_CONTROL__COMPRESSION_MODE__SHIFT                                                       0x1a
+#define CB_COLOR6_FDCC_CONTROL__ENABLE_MAX_COMP_FRAG_OVERRIDE__SHIFT                                          0x1c
+#define CB_COLOR6_FDCC_CONTROL__MAX_COMP_FRAGS__SHIFT                                                         0x1d
+#define CB_COLOR6_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                              0x00000004L
+#define CB_COLOR6_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000060L
+#define CB_COLOR6_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE_MASK                                                0x01000000L
+#define CB_COLOR6_FDCC_CONTROL__COMPRESSION_MODE_MASK                                                         0x0C000000L
+#define CB_COLOR6_FDCC_CONTROL__ENABLE_MAX_COMP_FRAG_OVERRIDE_MASK                                            0x10000000L
+#define CB_COLOR6_FDCC_CONTROL__MAX_COMP_FRAGS_MASK                                                           0x60000000L
+//CB_COLOR6_ATTRIB2
+#define CB_COLOR6_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR6_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0x10
+#define CB_COLOR6_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x0000FFFFL
+#define CB_COLOR6_ATTRIB2__MIP0_WIDTH_MASK                                                                    0xFFFF0000L
+//CB_COLOR6_ATTRIB3
+#define CB_COLOR6_ATTRIB3__MIP0_DEPTH__SHIFT                                                                  0x0
+#define CB_COLOR6_ATTRIB3__COLOR_SW_MODE__SHIFT                                                               0xf
+#define CB_COLOR6_ATTRIB3__MAX_MIP__SHIFT                                                                     0x13
+#define CB_COLOR6_ATTRIB3__RESOURCE_TYPE__SHIFT                                                               0x18
+#define CB_COLOR6_ATTRIB3__SPECULATIVE_READ__SHIFT                                                            0x1a
+#define CB_COLOR6_ATTRIB3__MIP0_DEPTH_MASK                                                                    0x00003FFFL
+#define CB_COLOR6_ATTRIB3__COLOR_SW_MODE_MASK                                                                 0x00038000L
+#define CB_COLOR6_ATTRIB3__MAX_MIP_MASK                                                                       0x00F80000L
+#define CB_COLOR6_ATTRIB3__RESOURCE_TYPE_MASK                                                                 0x03000000L
+#define CB_COLOR6_ATTRIB3__SPECULATIVE_READ_MASK                                                              0x0C000000L
+//CB_COLOR7_BASE
+#define CB_COLOR7_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR7_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR7_VIEW
+#define CB_COLOR7_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR7_VIEW__SLICE_MAX__SHIFT                                                                      0xe
+#define CB_COLOR7_VIEW__SLICE_START_MASK                                                                      0x00003FFFL
+#define CB_COLOR7_VIEW__SLICE_MAX_MASK                                                                        0x0FFFC000L
+//CB_COLOR7_VIEW2
+#define CB_COLOR7_VIEW2__MIP_LEVEL__SHIFT                                                                     0x0
+#define CB_COLOR7_VIEW2__MIP_LEVEL_MASK                                                                       0x0000001FL
+//CB_COLOR7_ATTRIB
+#define CB_COLOR7_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0x0
+#define CB_COLOR7_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x2
+#define CB_COLOR7_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX__SHIFT                                         0x3
+#define CB_COLOR7_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00000003L
+#define CB_COLOR7_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00000004L
+#define CB_COLOR7_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX_MASK                                           0x00000008L
+//CB_COLOR7_FDCC_CONTROL
+#define CB_COLOR7_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                            0x2
+#define CB_COLOR7_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x5
+#define CB_COLOR7_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE__SHIFT                                              0x18
+#define CB_COLOR7_FDCC_CONTROL__COMPRESSION_MODE__SHIFT                                                       0x1a
+#define CB_COLOR7_FDCC_CONTROL__ENABLE_MAX_COMP_FRAG_OVERRIDE__SHIFT                                          0x1c
+#define CB_COLOR7_FDCC_CONTROL__MAX_COMP_FRAGS__SHIFT                                                         0x1d
+#define CB_COLOR7_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                              0x00000004L
+#define CB_COLOR7_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000060L
+#define CB_COLOR7_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE_MASK                                                0x01000000L
+#define CB_COLOR7_FDCC_CONTROL__COMPRESSION_MODE_MASK                                                         0x0C000000L
+#define CB_COLOR7_FDCC_CONTROL__ENABLE_MAX_COMP_FRAG_OVERRIDE_MASK                                            0x10000000L
+#define CB_COLOR7_FDCC_CONTROL__MAX_COMP_FRAGS_MASK                                                           0x60000000L
+//CB_COLOR7_ATTRIB2
+#define CB_COLOR7_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR7_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0x10
+#define CB_COLOR7_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x0000FFFFL
+#define CB_COLOR7_ATTRIB2__MIP0_WIDTH_MASK                                                                    0xFFFF0000L
+//CB_COLOR7_ATTRIB3
+#define CB_COLOR7_ATTRIB3__MIP0_DEPTH__SHIFT                                                                  0x0
+#define CB_COLOR7_ATTRIB3__COLOR_SW_MODE__SHIFT                                                               0xf
+#define CB_COLOR7_ATTRIB3__MAX_MIP__SHIFT                                                                     0x13
+#define CB_COLOR7_ATTRIB3__RESOURCE_TYPE__SHIFT                                                               0x18
+#define CB_COLOR7_ATTRIB3__SPECULATIVE_READ__SHIFT                                                            0x1a
+#define CB_COLOR7_ATTRIB3__MIP0_DEPTH_MASK                                                                    0x00003FFFL
+#define CB_COLOR7_ATTRIB3__COLOR_SW_MODE_MASK                                                                 0x00038000L
+#define CB_COLOR7_ATTRIB3__MAX_MIP_MASK                                                                       0x00F80000L
+#define CB_COLOR7_ATTRIB3__RESOURCE_TYPE_MASK                                                                 0x03000000L
+#define CB_COLOR7_ATTRIB3__SPECULATIVE_READ_MASK                                                              0x0C000000L
+//CB_COLOR0_BASE_EXT
+#define CB_COLOR0_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR0_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR1_BASE_EXT
+#define CB_COLOR1_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR1_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR2_BASE_EXT
+#define CB_COLOR2_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR2_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR3_BASE_EXT
+#define CB_COLOR3_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR3_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR4_BASE_EXT
+#define CB_COLOR4_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR4_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR5_BASE_EXT
+#define CB_COLOR5_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR5_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR6_BASE_EXT
+#define CB_COLOR6_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR6_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR7_BASE_EXT
+#define CB_COLOR7_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR7_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR0_INFO
+#define CB_COLOR0_INFO__FORMAT__SHIFT                                                                         0x0
+#define CB_COLOR0_INFO__LINEAR_GENERAL__SHIFT                                                                 0x7
+#define CB_COLOR0_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR0_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR0_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR0_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR0_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR0_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR0_INFO__DISABLE_WA_FOR_PARTIAL_TARGET_MASK_ALL__SHIFT                                         0x13
+#define CB_COLOR0_INFO__FORMAT_MASK                                                                           0x0000001FL
+#define CB_COLOR0_INFO__LINEAR_GENERAL_MASK                                                                   0x00000080L
+#define CB_COLOR0_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR0_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR0_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR0_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR0_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR0_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR0_INFO__DISABLE_WA_FOR_PARTIAL_TARGET_MASK_ALL_MASK                                           0x00080000L
+//CB_COLOR1_INFO
+#define CB_COLOR1_INFO__FORMAT__SHIFT                                                                         0x0
+#define CB_COLOR1_INFO__LINEAR_GENERAL__SHIFT                                                                 0x7
+#define CB_COLOR1_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR1_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR1_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR1_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR1_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR1_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR1_INFO__DISABLE_WA_FOR_PARTIAL_TARGET_MASK_ALL__SHIFT                                         0x13
+#define CB_COLOR1_INFO__FORMAT_MASK                                                                           0x0000001FL
+#define CB_COLOR1_INFO__LINEAR_GENERAL_MASK                                                                   0x00000080L
+#define CB_COLOR1_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR1_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR1_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR1_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR1_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR1_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR1_INFO__DISABLE_WA_FOR_PARTIAL_TARGET_MASK_ALL_MASK                                           0x00080000L
+//CB_COLOR2_INFO
+#define CB_COLOR2_INFO__FORMAT__SHIFT                                                                         0x0
+#define CB_COLOR2_INFO__LINEAR_GENERAL__SHIFT                                                                 0x7
+#define CB_COLOR2_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR2_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR2_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR2_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR2_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR2_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR2_INFO__DISABLE_WA_FOR_PARTIAL_TARGET_MASK_ALL__SHIFT                                         0x13
+#define CB_COLOR2_INFO__FORMAT_MASK                                                                           0x0000001FL
+#define CB_COLOR2_INFO__LINEAR_GENERAL_MASK                                                                   0x00000080L
+#define CB_COLOR2_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR2_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR2_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR2_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR2_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR2_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR2_INFO__DISABLE_WA_FOR_PARTIAL_TARGET_MASK_ALL_MASK                                           0x00080000L
+//CB_COLOR3_INFO
+#define CB_COLOR3_INFO__FORMAT__SHIFT                                                                         0x0
+#define CB_COLOR3_INFO__LINEAR_GENERAL__SHIFT                                                                 0x7
+#define CB_COLOR3_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR3_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR3_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR3_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR3_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR3_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR3_INFO__DISABLE_WA_FOR_PARTIAL_TARGET_MASK_ALL__SHIFT                                         0x13
+#define CB_COLOR3_INFO__FORMAT_MASK                                                                           0x0000001FL
+#define CB_COLOR3_INFO__LINEAR_GENERAL_MASK                                                                   0x00000080L
+#define CB_COLOR3_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR3_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR3_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR3_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR3_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR3_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR3_INFO__DISABLE_WA_FOR_PARTIAL_TARGET_MASK_ALL_MASK                                           0x00080000L
+//CB_COLOR4_INFO
+#define CB_COLOR4_INFO__FORMAT__SHIFT                                                                         0x0
+#define CB_COLOR4_INFO__LINEAR_GENERAL__SHIFT                                                                 0x7
+#define CB_COLOR4_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR4_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR4_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR4_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR4_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR4_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR4_INFO__DISABLE_WA_FOR_PARTIAL_TARGET_MASK_ALL__SHIFT                                         0x13
+#define CB_COLOR4_INFO__FORMAT_MASK                                                                           0x0000001FL
+#define CB_COLOR4_INFO__LINEAR_GENERAL_MASK                                                                   0x00000080L
+#define CB_COLOR4_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR4_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR4_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR4_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR4_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR4_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR4_INFO__DISABLE_WA_FOR_PARTIAL_TARGET_MASK_ALL_MASK                                           0x00080000L
+//CB_COLOR5_INFO
+#define CB_COLOR5_INFO__FORMAT__SHIFT                                                                         0x0
+#define CB_COLOR5_INFO__LINEAR_GENERAL__SHIFT                                                                 0x7
+#define CB_COLOR5_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR5_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR5_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR5_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR5_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR5_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR5_INFO__DISABLE_WA_FOR_PARTIAL_TARGET_MASK_ALL__SHIFT                                         0x13
+#define CB_COLOR5_INFO__FORMAT_MASK                                                                           0x0000001FL
+#define CB_COLOR5_INFO__LINEAR_GENERAL_MASK                                                                   0x00000080L
+#define CB_COLOR5_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR5_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR5_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR5_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR5_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR5_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR5_INFO__DISABLE_WA_FOR_PARTIAL_TARGET_MASK_ALL_MASK                                           0x00080000L
+//CB_COLOR6_INFO
+#define CB_COLOR6_INFO__FORMAT__SHIFT                                                                         0x0
+#define CB_COLOR6_INFO__LINEAR_GENERAL__SHIFT                                                                 0x7
+#define CB_COLOR6_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR6_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR6_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR6_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR6_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR6_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR6_INFO__DISABLE_WA_FOR_PARTIAL_TARGET_MASK_ALL__SHIFT                                         0x13
+#define CB_COLOR6_INFO__FORMAT_MASK                                                                           0x0000001FL
+#define CB_COLOR6_INFO__LINEAR_GENERAL_MASK                                                                   0x00000080L
+#define CB_COLOR6_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR6_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR6_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR6_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR6_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR6_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR6_INFO__DISABLE_WA_FOR_PARTIAL_TARGET_MASK_ALL_MASK                                           0x00080000L
+//CB_COLOR7_INFO
+#define CB_COLOR7_INFO__FORMAT__SHIFT                                                                         0x0
+#define CB_COLOR7_INFO__LINEAR_GENERAL__SHIFT                                                                 0x7
+#define CB_COLOR7_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR7_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR7_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR7_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR7_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR7_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR7_INFO__DISABLE_WA_FOR_PARTIAL_TARGET_MASK_ALL__SHIFT                                         0x13
+#define CB_COLOR7_INFO__FORMAT_MASK                                                                           0x0000001FL
+#define CB_COLOR7_INFO__LINEAR_GENERAL_MASK                                                                   0x00000080L
+#define CB_COLOR7_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR7_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR7_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR7_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR7_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR7_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR7_INFO__DISABLE_WA_FOR_PARTIAL_TARGET_MASK_ALL_MASK                                           0x00080000L
+//CB_MEM0_INFO
+#define CB_MEM0_INFO__TEMPORAL_READ__SHIFT                                                                    0x0
+#define CB_MEM0_INFO__TEMPORAL_WRITE__SHIFT                                                                   0x3
+#define CB_MEM0_INFO__TEMPORAL_READ_MASK                                                                      0x00000007L
+#define CB_MEM0_INFO__TEMPORAL_WRITE_MASK                                                                     0x00000038L
+//CB_MEM1_INFO
+#define CB_MEM1_INFO__TEMPORAL_READ__SHIFT                                                                    0x0
+#define CB_MEM1_INFO__TEMPORAL_WRITE__SHIFT                                                                   0x3
+#define CB_MEM1_INFO__TEMPORAL_READ_MASK                                                                      0x00000007L
+#define CB_MEM1_INFO__TEMPORAL_WRITE_MASK                                                                     0x00000038L
+//CB_MEM2_INFO
+#define CB_MEM2_INFO__TEMPORAL_READ__SHIFT                                                                    0x0
+#define CB_MEM2_INFO__TEMPORAL_WRITE__SHIFT                                                                   0x3
+#define CB_MEM2_INFO__TEMPORAL_READ_MASK                                                                      0x00000007L
+#define CB_MEM2_INFO__TEMPORAL_WRITE_MASK                                                                     0x00000038L
+//CB_MEM3_INFO
+#define CB_MEM3_INFO__TEMPORAL_READ__SHIFT                                                                    0x0
+#define CB_MEM3_INFO__TEMPORAL_WRITE__SHIFT                                                                   0x3
+#define CB_MEM3_INFO__TEMPORAL_READ_MASK                                                                      0x00000007L
+#define CB_MEM3_INFO__TEMPORAL_WRITE_MASK                                                                     0x00000038L
+//CB_MEM4_INFO
+#define CB_MEM4_INFO__TEMPORAL_READ__SHIFT                                                                    0x0
+#define CB_MEM4_INFO__TEMPORAL_WRITE__SHIFT                                                                   0x3
+#define CB_MEM4_INFO__TEMPORAL_READ_MASK                                                                      0x00000007L
+#define CB_MEM4_INFO__TEMPORAL_WRITE_MASK                                                                     0x00000038L
+//CB_MEM5_INFO
+#define CB_MEM5_INFO__TEMPORAL_READ__SHIFT                                                                    0x0
+#define CB_MEM5_INFO__TEMPORAL_WRITE__SHIFT                                                                   0x3
+#define CB_MEM5_INFO__TEMPORAL_READ_MASK                                                                      0x00000007L
+#define CB_MEM5_INFO__TEMPORAL_WRITE_MASK                                                                     0x00000038L
+//CB_MEM6_INFO
+#define CB_MEM6_INFO__TEMPORAL_READ__SHIFT                                                                    0x0
+#define CB_MEM6_INFO__TEMPORAL_WRITE__SHIFT                                                                   0x3
+#define CB_MEM6_INFO__TEMPORAL_READ_MASK                                                                      0x00000007L
+#define CB_MEM6_INFO__TEMPORAL_WRITE_MASK                                                                     0x00000038L
+//CB_MEM7_INFO
+#define CB_MEM7_INFO__TEMPORAL_READ__SHIFT                                                                    0x0
+#define CB_MEM7_INFO__TEMPORAL_WRITE__SHIFT                                                                   0x3
+#define CB_MEM7_INFO__TEMPORAL_READ_MASK                                                                      0x00000007L
+#define CB_MEM7_INFO__TEMPORAL_WRITE_MASK                                                                     0x00000038L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pfvf_padec
+//PA_SC_VRS_SURFACE_CNTL
+#define PA_SC_VRS_SURFACE_CNTL__VRC_CONTEXT_DONE_SYNC_DISABLE__SHIFT                                          0x6
+#define PA_SC_VRS_SURFACE_CNTL__VRS_FEEDBACK_RATE_OVERRIDE__SHIFT                                             0x7
+#define PA_SC_VRS_SURFACE_CNTL__VRC_FLUSH_EVENT_MASK_DISABLE__SHIFT                                           0x8
+#define PA_SC_VRS_SURFACE_CNTL__VRC_PREFETCH_DISABLE__SHIFT                                                   0xd
+#define PA_SC_VRS_SURFACE_CNTL__VRC_FLUSH_NO_INV_DISABLE__SHIFT                                               0xe
+#define PA_SC_VRS_SURFACE_CNTL__VRC_NONSTALLING_FLUSH_DISABLE__SHIFT                                          0xf
+#define PA_SC_VRS_SURFACE_CNTL__VRC_PARTIAL_FLUSH_DISABLE__SHIFT                                              0x10
+#define PA_SC_VRS_SURFACE_CNTL__VRC_EOP_SYNC_DISABLE__SHIFT                                                   0x12
+#define PA_SC_VRS_SURFACE_CNTL__VRC_MAX_TAGS__SHIFT                                                           0x13
+#define PA_SC_VRS_SURFACE_CNTL__VRC_EVICT_POINT__SHIFT                                                        0x1a
+#define PA_SC_VRS_SURFACE_CNTL__VRC_CONTEXT_DONE_SYNC_DISABLE_MASK                                            0x00000040L
+#define PA_SC_VRS_SURFACE_CNTL__VRS_FEEDBACK_RATE_OVERRIDE_MASK                                               0x00000080L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_FLUSH_EVENT_MASK_DISABLE_MASK                                             0x00001F00L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_PREFETCH_DISABLE_MASK                                                     0x00002000L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_FLUSH_NO_INV_DISABLE_MASK                                                 0x00004000L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_NONSTALLING_FLUSH_DISABLE_MASK                                            0x00008000L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_PARTIAL_FLUSH_DISABLE_MASK                                                0x00010000L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_EOP_SYNC_DISABLE_MASK                                                     0x00040000L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_MAX_TAGS_MASK                                                             0x03F80000L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_EVICT_POINT_MASK                                                          0xFC000000L
+//PA_SC_ENHANCE
+#define PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER__SHIFT                                                       0x0
+#define PA_SC_ENHANCE__DISABLE_SC_DB_TILE_FIX__SHIFT                                                          0x1
+#define PA_SC_ENHANCE__DISABLE_AA_MASK_FULL_FIX__SHIFT                                                        0x2
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOCATIONS__SHIFT                                                  0x3
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOC_CENTROID__SHIFT                                               0x4
+#define PA_SC_ENHANCE__DISABLE_SCISSOR_FIX__SHIFT                                                             0x5
+#define PA_SC_ENHANCE__SEND_UNLIT_STILES_TO_PACKER__SHIFT                                                     0x6
+#define PA_SC_ENHANCE__DISABLE_DUALGRAD_PERF_OPTIMIZATION__SHIFT                                              0x7
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_PRIM__SHIFT                                                   0x8
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_SUPERTILE__SHIFT                                              0x9
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_TILE__SHIFT                                                   0xa
+#define PA_SC_ENHANCE__DISABLE_PA_SC_GUIDANCE__SHIFT                                                          0xb
+#define PA_SC_ENHANCE__DISABLE_EOV_ALL_CTRL_ONLY_COMBINATIONS__SHIFT                                          0xc
+#define PA_SC_ENHANCE__ENABLE_MULTICYCLE_BUBBLE_FREEZE__SHIFT                                                 0xd
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_PA_SC_GUIDANCE__SHIFT                                             0xe
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_POLY_MODE__SHIFT                                                   0xf
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EOP_SYNC_NULL_PRIMS_LAST__SHIFT                                   0x10
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_THRESHOLD_SWITCHING__SHIFT                                        0x11
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_THRESHOLD_SWITCH_AT_EOPG_ONLY__SHIFT                               0x12
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_DESIRED_FIFO_EMPTY_SWITCHING__SHIFT                               0x13
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_SELECTED_FIFO_EMPTY_SWITCHING__SHIFT                              0x14
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EMPTY_SWITCHING_HYSTERYSIS__SHIFT                                 0x15
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_DESIRED_FIFO_IS_NEXT_FEID__SHIFT                                   0x16
+#define PA_SC_ENHANCE__DISABLE_OOO_NO_EOPG_SKEW_DESIRED_FIFO_IS_CURRENT_FIFO__SHIFT                           0x17
+#define PA_SC_ENHANCE__OOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT__SHIFT                                          0x18
+#define PA_SC_ENHANCE__OOO_DISABLE_EOPG_SKEW_THRESHOLD_SWITCHING__SHIFT                                       0x19
+#define PA_SC_ENHANCE__DISABLE_EOP_LINE_STIPPLE_RESET__SHIFT                                                  0x1a
+#define PA_SC_ENHANCE__DISABLE_VPZ_EOP_LINE_STIPPLE_RESET__SHIFT                                              0x1b
+#define PA_SC_ENHANCE__IOO_DISABLE_SCAN_UNSELECTED_FIFOS_FOR_DUAL_GFX_RING_CHANGE__SHIFT                      0x1c
+#define PA_SC_ENHANCE__OOO_USE_ABSOLUTE_FIFO_COUNT_IN_THRESHOLD_SWITCHING__SHIFT                              0x1d
+#define PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER_MASK                                                         0x00000001L
+#define PA_SC_ENHANCE__DISABLE_SC_DB_TILE_FIX_MASK                                                            0x00000002L
+#define PA_SC_ENHANCE__DISABLE_AA_MASK_FULL_FIX_MASK                                                          0x00000004L
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOCATIONS_MASK                                                    0x00000008L
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOC_CENTROID_MASK                                                 0x00000010L
+#define PA_SC_ENHANCE__DISABLE_SCISSOR_FIX_MASK                                                               0x00000020L
+#define PA_SC_ENHANCE__SEND_UNLIT_STILES_TO_PACKER_MASK                                                       0x00000040L
+#define PA_SC_ENHANCE__DISABLE_DUALGRAD_PERF_OPTIMIZATION_MASK                                                0x00000080L
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_PRIM_MASK                                                     0x00000100L
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_SUPERTILE_MASK                                                0x00000200L
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_TILE_MASK                                                     0x00000400L
+#define PA_SC_ENHANCE__DISABLE_PA_SC_GUIDANCE_MASK                                                            0x00000800L
+#define PA_SC_ENHANCE__DISABLE_EOV_ALL_CTRL_ONLY_COMBINATIONS_MASK                                            0x00001000L
+#define PA_SC_ENHANCE__ENABLE_MULTICYCLE_BUBBLE_FREEZE_MASK                                                   0x00002000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_PA_SC_GUIDANCE_MASK                                               0x00004000L
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_POLY_MODE_MASK                                                     0x00008000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EOP_SYNC_NULL_PRIMS_LAST_MASK                                     0x00010000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_THRESHOLD_SWITCHING_MASK                                          0x00020000L
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_THRESHOLD_SWITCH_AT_EOPG_ONLY_MASK                                 0x00040000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_DESIRED_FIFO_EMPTY_SWITCHING_MASK                                 0x00080000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_SELECTED_FIFO_EMPTY_SWITCHING_MASK                                0x00100000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EMPTY_SWITCHING_HYSTERYSIS_MASK                                   0x00200000L
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_DESIRED_FIFO_IS_NEXT_FEID_MASK                                     0x00400000L
+#define PA_SC_ENHANCE__DISABLE_OOO_NO_EOPG_SKEW_DESIRED_FIFO_IS_CURRENT_FIFO_MASK                             0x00800000L
+#define PA_SC_ENHANCE__OOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT_MASK                                            0x01000000L
+#define PA_SC_ENHANCE__OOO_DISABLE_EOPG_SKEW_THRESHOLD_SWITCHING_MASK                                         0x02000000L
+#define PA_SC_ENHANCE__DISABLE_EOP_LINE_STIPPLE_RESET_MASK                                                    0x04000000L
+#define PA_SC_ENHANCE__DISABLE_VPZ_EOP_LINE_STIPPLE_RESET_MASK                                                0x08000000L
+#define PA_SC_ENHANCE__IOO_DISABLE_SCAN_UNSELECTED_FIFOS_FOR_DUAL_GFX_RING_CHANGE_MASK                        0x10000000L
+#define PA_SC_ENHANCE__OOO_USE_ABSOLUTE_FIFO_COUNT_IN_THRESHOLD_SWITCHING_MASK                                0x20000000L
+//PA_SC_ENHANCE_1
+#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE_ENABLE__SHIFT                                                0x0
+#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE__SHIFT                                                       0x1
+#define PA_SC_ENHANCE_1__DISABLE_SC_BINNING__SHIFT                                                            0x3
+#define PA_SC_ENHANCE_1__BYPASS_PBB__SHIFT                                                                    0x4
+#define PA_SC_ENHANCE_1__DISABLE_NONBINNED_LIVE_PRIM_DG1_LS0_CL0_EOPKT_POKE__SHIFT                            0x5
+#define PA_SC_ENHANCE_1__ECO_SPARE1__SHIFT                                                                    0x6
+#define PA_SC_ENHANCE_1__ECO_SPARE2__SHIFT                                                                    0x7
+#define PA_SC_ENHANCE_1__ECO_SPARE3__SHIFT                                                                    0x8
+#define PA_SC_ENHANCE_1__DISABLE_SC_PROCESS_RESET_PBB__SHIFT                                                  0x9
+#define PA_SC_ENHANCE_1__DISABLE_PBB_SCISSOR_OPT__SHIFT                                                       0xa
+#define PA_SC_ENHANCE_1__ENABLE_DFSM_FLUSH_EVENT_TO_FLUSH_POPS_CAM__SHIFT                                     0xb
+#define PA_SC_ENHANCE_1__DEBUG_PIXEL_PICKER_COUNT_PIXELS__SHIFT                                               0xd
+#define PA_SC_ENHANCE_1__DISABLE_SC_DB_STILE_INTF_FINE_CLOCK_GATE__SHIFT                                      0xe
+#define PA_SC_ENHANCE_1__DISABLE_PACKER_ODC_ENHANCE__SHIFT                                                    0x10
+#define PA_SC_ENHANCE_1__OPTIMAL_BIN_SELECTION__SHIFT                                                         0x12
+#define PA_SC_ENHANCE_1__DISABLE_FORCE_SOP_ALL_EVENTS__SHIFT                                                  0x13
+#define PA_SC_ENHANCE_1__DISABLE_PBB_CLK_OPTIMIZATION__SHIFT                                                  0x14
+#define PA_SC_ENHANCE_1__DISABLE_PBB_SCISSOR_CLK_OPTIMIZATION__SHIFT                                          0x15
+#define PA_SC_ENHANCE_1__DISABLE_PBB_BINNING_CLK_OPTIMIZATION__SHIFT                                          0x16
+#define PA_SC_ENHANCE_1__DISABLE_INTF_CG__SHIFT                                                               0x17
+#define PA_SC_ENHANCE_1__IOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT__SHIFT                                        0x18
+#define PA_SC_ENHANCE_1__DISABLE_SHADER_PROFILING_FOR_POWER__SHIFT                                            0x19
+#define PA_SC_ENHANCE_1__FLUSH_ON_BINNING_TRANSITION__SHIFT                                                   0x1a
+#define PA_SC_ENHANCE_1__DISABLE_QUAD_PROC_FDCE_ENHANCE__SHIFT                                                0x1b
+#define PA_SC_ENHANCE_1__DISABLE_SC_PS_PA_ARBITER_FIX__SHIFT                                                  0x1c
+#define PA_SC_ENHANCE_1__DISABLE_SC_PS_PA_ARBITER_FIX_1__SHIFT                                                0x1d
+#define PA_SC_ENHANCE_1__PASS_VPZ_EVENT_TO_SPI__SHIFT                                                         0x1e
+#define PA_SC_ENHANCE_1__RESERVED_31__SHIFT                                                                   0x1f
+#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE_ENABLE_MASK                                                  0x00000001L
+#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE_MASK                                                         0x00000006L
+#define PA_SC_ENHANCE_1__DISABLE_SC_BINNING_MASK                                                              0x00000008L
+#define PA_SC_ENHANCE_1__BYPASS_PBB_MASK                                                                      0x00000010L
+#define PA_SC_ENHANCE_1__DISABLE_NONBINNED_LIVE_PRIM_DG1_LS0_CL0_EOPKT_POKE_MASK                              0x00000020L
+#define PA_SC_ENHANCE_1__ECO_SPARE1_MASK                                                                      0x00000040L
+#define PA_SC_ENHANCE_1__ECO_SPARE2_MASK                                                                      0x00000080L
+#define PA_SC_ENHANCE_1__ECO_SPARE3_MASK                                                                      0x00000100L
+#define PA_SC_ENHANCE_1__DISABLE_SC_PROCESS_RESET_PBB_MASK                                                    0x00000200L
+#define PA_SC_ENHANCE_1__DISABLE_PBB_SCISSOR_OPT_MASK                                                         0x00000400L
+#define PA_SC_ENHANCE_1__ENABLE_DFSM_FLUSH_EVENT_TO_FLUSH_POPS_CAM_MASK                                       0x00000800L
+#define PA_SC_ENHANCE_1__DEBUG_PIXEL_PICKER_COUNT_PIXELS_MASK                                                 0x00002000L
+#define PA_SC_ENHANCE_1__DISABLE_SC_DB_STILE_INTF_FINE_CLOCK_GATE_MASK                                        0x00004000L
+#define PA_SC_ENHANCE_1__DISABLE_PACKER_ODC_ENHANCE_MASK                                                      0x00010000L
+#define PA_SC_ENHANCE_1__OPTIMAL_BIN_SELECTION_MASK                                                           0x00040000L
+#define PA_SC_ENHANCE_1__DISABLE_FORCE_SOP_ALL_EVENTS_MASK                                                    0x00080000L
+#define PA_SC_ENHANCE_1__DISABLE_PBB_CLK_OPTIMIZATION_MASK                                                    0x00100000L
+#define PA_SC_ENHANCE_1__DISABLE_PBB_SCISSOR_CLK_OPTIMIZATION_MASK                                            0x00200000L
+#define PA_SC_ENHANCE_1__DISABLE_PBB_BINNING_CLK_OPTIMIZATION_MASK                                            0x00400000L
+#define PA_SC_ENHANCE_1__DISABLE_INTF_CG_MASK                                                                 0x00800000L
+#define PA_SC_ENHANCE_1__IOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT_MASK                                          0x01000000L
+#define PA_SC_ENHANCE_1__DISABLE_SHADER_PROFILING_FOR_POWER_MASK                                              0x02000000L
+#define PA_SC_ENHANCE_1__FLUSH_ON_BINNING_TRANSITION_MASK                                                     0x04000000L
+#define PA_SC_ENHANCE_1__DISABLE_QUAD_PROC_FDCE_ENHANCE_MASK                                                  0x08000000L
+#define PA_SC_ENHANCE_1__DISABLE_SC_PS_PA_ARBITER_FIX_MASK                                                    0x10000000L
+#define PA_SC_ENHANCE_1__DISABLE_SC_PS_PA_ARBITER_FIX_1_MASK                                                  0x20000000L
+#define PA_SC_ENHANCE_1__PASS_VPZ_EVENT_TO_SPI_MASK                                                           0x40000000L
+#define PA_SC_ENHANCE_1__RESERVED_31_MASK                                                                     0x80000000L
+//PA_SC_ENHANCE_2
+#define PA_SC_ENHANCE_2__DISABLE_SC_BCI_QUAD_INTF_FINE_CLOCK_GATE__SHIFT                                      0x2
+#define PA_SC_ENHANCE_2__DISABLE_SC_BCI_PRIM_INTF_FINE_CLOCK_GATE__SHIFT                                      0x3
+#define PA_SC_ENHANCE_2__ENABLE_LPOV_WAVE_BREAK__SHIFT                                                        0x4
+#define PA_SC_ENHANCE_2__ENABLE_FPOV_WAVE_BREAK__SHIFT                                                        0x5
+#define PA_SC_ENHANCE_2__ENABLE_SC_SEND_DB_VPZ_FOR_EN_PRIM_PAYLOAD__SHIFT                                     0x7
+#define PA_SC_ENHANCE_2__DISABLE_BREAK_BATCH_ON_GFX_PIPE_SWITCH__SHIFT                                        0x8
+#define PA_SC_ENHANCE_2__DISABLE_FULL_TILE_WAVE_BREAK__SHIFT                                                  0x9
+#define PA_SC_ENHANCE_2__ENABLE_VPZ_INJECTION_BEFORE_NULL_PRIMS__SHIFT                                        0xa
+#define PA_SC_ENHANCE_2__PBB_TIMEOUT_THRESHOLD_MODE__SHIFT                                                    0xb
+#define PA_SC_ENHANCE_2__DISABLE_PACKER_GRAD_FDCE_ENHANCE__SHIFT                                              0xc
+#define PA_SC_ENHANCE_2__DISABLE_SC_SPI_INTF_EARLY_WAKEUP__SHIFT                                              0xd
+#define PA_SC_ENHANCE_2__DISABLE_SC_BCI_INTF_EARLY_WAKEUP__SHIFT                                              0xe
+#define PA_SC_ENHANCE_2__DISABLE_EXPOSED_GT_DETAIL_RATE_TILE_COV_ADJ__SHIFT                                   0xf
+#define PA_SC_ENHANCE_2__RESERVED_16__SHIFT                                                                   0x10
+#define PA_SC_ENHANCE_2__PBB_MAIN_CLK_REG_BUSY_WAKEUP__SHIFT                                                  0x11
+#define PA_SC_ENHANCE_2__DISABLE_BREAK_BATCH_ON_GFX_PIPELINE_RESET__SHIFT                                     0x12
+#define PA_SC_ENHANCE_2__RESERVED_22__SHIFT                                                                   0x16
+#define PA_SC_ENHANCE_2__PROCESS_RESET_FORCE_STILE_MASK_TO_ZERO__SHIFT                                        0x17
+#define PA_SC_ENHANCE_2__BREAK_WHEN_ONE_NULL_PRIM_BATCH__SHIFT                                                0x1a
+#define PA_SC_ENHANCE_2__NULL_PRIM_BREAK_BATCH_LIMIT__SHIFT                                                   0x1b
+#define PA_SC_ENHANCE_2__DISABLE_MAX_DEALLOC_FORCE_EOV_RESET_N_WAVES_COUNT__SHIFT                             0x1e
+#define PA_SC_ENHANCE_2__RSVD__SHIFT                                                                          0x1f
+#define PA_SC_ENHANCE_2__DISABLE_SC_BCI_QUAD_INTF_FINE_CLOCK_GATE_MASK                                        0x00000004L
+#define PA_SC_ENHANCE_2__DISABLE_SC_BCI_PRIM_INTF_FINE_CLOCK_GATE_MASK                                        0x00000008L
+#define PA_SC_ENHANCE_2__ENABLE_LPOV_WAVE_BREAK_MASK                                                          0x00000010L
+#define PA_SC_ENHANCE_2__ENABLE_FPOV_WAVE_BREAK_MASK                                                          0x00000020L
+#define PA_SC_ENHANCE_2__ENABLE_SC_SEND_DB_VPZ_FOR_EN_PRIM_PAYLOAD_MASK                                       0x00000080L
+#define PA_SC_ENHANCE_2__DISABLE_BREAK_BATCH_ON_GFX_PIPE_SWITCH_MASK                                          0x00000100L
+#define PA_SC_ENHANCE_2__DISABLE_FULL_TILE_WAVE_BREAK_MASK                                                    0x00000200L
+#define PA_SC_ENHANCE_2__ENABLE_VPZ_INJECTION_BEFORE_NULL_PRIMS_MASK                                          0x00000400L
+#define PA_SC_ENHANCE_2__PBB_TIMEOUT_THRESHOLD_MODE_MASK                                                      0x00000800L
+#define PA_SC_ENHANCE_2__DISABLE_PACKER_GRAD_FDCE_ENHANCE_MASK                                                0x00001000L
+#define PA_SC_ENHANCE_2__DISABLE_SC_SPI_INTF_EARLY_WAKEUP_MASK                                                0x00002000L
+#define PA_SC_ENHANCE_2__DISABLE_SC_BCI_INTF_EARLY_WAKEUP_MASK                                                0x00004000L
+#define PA_SC_ENHANCE_2__DISABLE_EXPOSED_GT_DETAIL_RATE_TILE_COV_ADJ_MASK                                     0x00008000L
+#define PA_SC_ENHANCE_2__RESERVED_16_MASK                                                                     0x00010000L
+#define PA_SC_ENHANCE_2__PBB_MAIN_CLK_REG_BUSY_WAKEUP_MASK                                                    0x00020000L
+#define PA_SC_ENHANCE_2__DISABLE_BREAK_BATCH_ON_GFX_PIPELINE_RESET_MASK                                       0x00040000L
+#define PA_SC_ENHANCE_2__RESERVED_22_MASK                                                                     0x00400000L
+#define PA_SC_ENHANCE_2__PROCESS_RESET_FORCE_STILE_MASK_TO_ZERO_MASK                                          0x00800000L
+#define PA_SC_ENHANCE_2__BREAK_WHEN_ONE_NULL_PRIM_BATCH_MASK                                                  0x04000000L
+#define PA_SC_ENHANCE_2__NULL_PRIM_BREAK_BATCH_LIMIT_MASK                                                     0x38000000L
+#define PA_SC_ENHANCE_2__DISABLE_MAX_DEALLOC_FORCE_EOV_RESET_N_WAVES_COUNT_MASK                               0x40000000L
+#define PA_SC_ENHANCE_2__RSVD_MASK                                                                            0x80000000L
+//PA_SC_ENHANCE_3
+#define PA_SC_ENHANCE_3__FORCE_USE_OF_SC_CENTROID_DATA__SHIFT                                                 0x0
+#define PA_SC_ENHANCE_3__ECO_SPARE2__SHIFT                                                                    0x1
+#define PA_SC_ENHANCE_3__DISABLE_RB_MASK_COPY_FOR_NONP2_SA_PAIR_HARVEST__SHIFT                                0x2
+#define PA_SC_ENHANCE_3__FORCE_PBB_WORKLOAD_MODE_TO_ZERO__SHIFT                                               0x3
+#define PA_SC_ENHANCE_3__DISABLE_PKR_BCI_QUAD_NEW_PRIM_DATA_LOAD_OPTIMIZATION__SHIFT                          0x4
+#define PA_SC_ENHANCE_3__DISABLE_CP_CONTEXT_DONE_PERFCOUNT_SAMPLE_EN__SHIFT                                   0x5
+#define PA_SC_ENHANCE_3__ENABLE_SINGLE_PA_EOPKT_FIRST_PHASE_FILTER__SHIFT                                     0x6
+#define PA_SC_ENHANCE_3__ENABLE_SINGLE_PA_EOPKT_LAST_PHASE_FILTER__SHIFT                                      0x7
+#define PA_SC_ENHANCE_3__ENABLE_SINGLE_PA_EOPKT_LAST_PHASE_FILTER_FOR_PBB_BINNED_PRIMS__SHIFT                 0x8
+#define PA_SC_ENHANCE_3__DISABLE_SET_VPZ_DIRTY_EOPKT_LAST_PHASE_ONLY__SHIFT                                   0x9
+#define PA_SC_ENHANCE_3__DISABLE_PBB_EOP_OPTIMIZATION_WITH_SAME_CONTEXT_BATCHES__SHIFT                        0xa
+#define PA_SC_ENHANCE_3__DISABLE_FAST_NULL_PRIM_OPTIMIZATION__SHIFT                                           0xb
+#define PA_SC_ENHANCE_3__USE_PBB_PRIM_STORAGE_WHEN_STALLED__SHIFT                                             0xc
+#define PA_SC_ENHANCE_3__DISABLE_LIGHT_VOLUME_RENDERING_OPTIMIZATION__SHIFT                                   0xd
+#define PA_SC_ENHANCE_3__DISABLE_ZPRE_PASS_OPTIMIZATION__SHIFT                                                0xe
+#define PA_SC_ENHANCE_3__DISABLE_EVENT_INCLUSION_IN_CONTEXT_STATES_PER_BIN__SHIFT                             0xf
+#define PA_SC_ENHANCE_3__DISABLE_PIXEL_WAIT_SYNC_COUNTERS__SHIFT                                              0x10
+#define PA_SC_ENHANCE_3__DISABLE_SC_CPG_PSINVOC_SEDC_ISOLATION_ACCUM__SHIFT                                   0x11
+#define PA_SC_ENHANCE_3__DISABLE_PKR_FORCE_EOV_MAX_REZ_CNT_FOR_SPI_BACKPRESSURE_ONLY__SHIFT                   0x14
+#define PA_SC_ENHANCE_3__DISABLE_PKR_FORCE_EOV_MAX_CLK_CNT_FOR_SPI_BACKPRESSURE_ONLY__SHIFT                   0x15
+#define PA_SC_ENHANCE_3__DO_NOT_INCLUDE_OREO_WAVEID_IN_FORCE_EOV_MAX_CNT_DISABLE__SHIFT                       0x16
+#define PA_SC_ENHANCE_3__DISABLE_PWS_PRE_DEPTH_WAIT_SYNC_VPZ_INSERTION__SHIFT                                 0x17
+#define PA_SC_ENHANCE_3__PKR_CNT_FORCE_EOV_AT_QS_EMPTY_ONLY__SHIFT                                            0x18
+#define PA_SC_ENHANCE_3__PKR_S0_FORCE_EOV_STALL__SHIFT                                                        0x19
+#define PA_SC_ENHANCE_3__PKR_S1_FORCE_EOV_STALL__SHIFT                                                        0x1a
+#define PA_SC_ENHANCE_3__APPLY_AA_MASK_AT_EXPOSED_RATE_FOR_VRS_COURSE_QUADS_WITH_CR__SHIFT                    0x1b
+#define PA_SC_ENHANCE_3__ECO_SPARE0__SHIFT                                                                    0x1c
+#define PA_SC_ENHANCE_3__ECO_SPARE1__SHIFT                                                                    0x1d
+#define PA_SC_ENHANCE_3__DISABLE_SC_GL1X_SRC_FINE_CLOCK_GATE__SHIFT                                           0x1e
+#define PA_SC_ENHANCE_3__DISABLE_SC_GL1X_REQ_FINE_CLOCK_GATE__SHIFT                                           0x1f
+#define PA_SC_ENHANCE_3__FORCE_USE_OF_SC_CENTROID_DATA_MASK                                                   0x00000001L
+#define PA_SC_ENHANCE_3__ECO_SPARE2_MASK                                                                      0x00000002L
+#define PA_SC_ENHANCE_3__DISABLE_RB_MASK_COPY_FOR_NONP2_SA_PAIR_HARVEST_MASK                                  0x00000004L
+#define PA_SC_ENHANCE_3__FORCE_PBB_WORKLOAD_MODE_TO_ZERO_MASK                                                 0x00000008L
+#define PA_SC_ENHANCE_3__DISABLE_PKR_BCI_QUAD_NEW_PRIM_DATA_LOAD_OPTIMIZATION_MASK                            0x00000010L
+#define PA_SC_ENHANCE_3__DISABLE_CP_CONTEXT_DONE_PERFCOUNT_SAMPLE_EN_MASK                                     0x00000020L
+#define PA_SC_ENHANCE_3__ENABLE_SINGLE_PA_EOPKT_FIRST_PHASE_FILTER_MASK                                       0x00000040L
+#define PA_SC_ENHANCE_3__ENABLE_SINGLE_PA_EOPKT_LAST_PHASE_FILTER_MASK                                        0x00000080L
+#define PA_SC_ENHANCE_3__ENABLE_SINGLE_PA_EOPKT_LAST_PHASE_FILTER_FOR_PBB_BINNED_PRIMS_MASK                   0x00000100L
+#define PA_SC_ENHANCE_3__DISABLE_SET_VPZ_DIRTY_EOPKT_LAST_PHASE_ONLY_MASK                                     0x00000200L
+#define PA_SC_ENHANCE_3__DISABLE_PBB_EOP_OPTIMIZATION_WITH_SAME_CONTEXT_BATCHES_MASK                          0x00000400L
+#define PA_SC_ENHANCE_3__DISABLE_FAST_NULL_PRIM_OPTIMIZATION_MASK                                             0x00000800L
+#define PA_SC_ENHANCE_3__USE_PBB_PRIM_STORAGE_WHEN_STALLED_MASK                                               0x00001000L
+#define PA_SC_ENHANCE_3__DISABLE_LIGHT_VOLUME_RENDERING_OPTIMIZATION_MASK                                     0x00002000L
+#define PA_SC_ENHANCE_3__DISABLE_ZPRE_PASS_OPTIMIZATION_MASK                                                  0x00004000L
+#define PA_SC_ENHANCE_3__DISABLE_EVENT_INCLUSION_IN_CONTEXT_STATES_PER_BIN_MASK                               0x00008000L
+#define PA_SC_ENHANCE_3__DISABLE_PIXEL_WAIT_SYNC_COUNTERS_MASK                                                0x00010000L
+#define PA_SC_ENHANCE_3__DISABLE_SC_CPG_PSINVOC_SEDC_ISOLATION_ACCUM_MASK                                     0x00020000L
+#define PA_SC_ENHANCE_3__DISABLE_PKR_FORCE_EOV_MAX_REZ_CNT_FOR_SPI_BACKPRESSURE_ONLY_MASK                     0x00100000L
+#define PA_SC_ENHANCE_3__DISABLE_PKR_FORCE_EOV_MAX_CLK_CNT_FOR_SPI_BACKPRESSURE_ONLY_MASK                     0x00200000L
+#define PA_SC_ENHANCE_3__DO_NOT_INCLUDE_OREO_WAVEID_IN_FORCE_EOV_MAX_CNT_DISABLE_MASK                         0x00400000L
+#define PA_SC_ENHANCE_3__DISABLE_PWS_PRE_DEPTH_WAIT_SYNC_VPZ_INSERTION_MASK                                   0x00800000L
+#define PA_SC_ENHANCE_3__PKR_CNT_FORCE_EOV_AT_QS_EMPTY_ONLY_MASK                                              0x01000000L
+#define PA_SC_ENHANCE_3__PKR_S0_FORCE_EOV_STALL_MASK                                                          0x02000000L
+#define PA_SC_ENHANCE_3__PKR_S1_FORCE_EOV_STALL_MASK                                                          0x04000000L
+#define PA_SC_ENHANCE_3__APPLY_AA_MASK_AT_EXPOSED_RATE_FOR_VRS_COURSE_QUADS_WITH_CR_MASK                      0x08000000L
+#define PA_SC_ENHANCE_3__ECO_SPARE0_MASK                                                                      0x10000000L
+#define PA_SC_ENHANCE_3__ECO_SPARE1_MASK                                                                      0x20000000L
+#define PA_SC_ENHANCE_3__DISABLE_SC_GL1X_SRC_FINE_CLOCK_GATE_MASK                                             0x40000000L
+#define PA_SC_ENHANCE_3__DISABLE_SC_GL1X_REQ_FINE_CLOCK_GATE_MASK                                             0x80000000L
+//PA_SC_BINNER_CNTL_OVERRIDE
+#define PA_SC_BINNER_CNTL_OVERRIDE__BINNING_MODE__SHIFT                                                       0x0
+#define PA_SC_BINNER_CNTL_OVERRIDE__CONTEXT_STATES_PER_BIN__SHIFT                                             0xa
+#define PA_SC_BINNER_CNTL_OVERRIDE__PERSISTENT_STATES_PER_BIN__SHIFT                                          0xd
+#define PA_SC_BINNER_CNTL_OVERRIDE__FPOVS_PER_BATCH__SHIFT                                                    0x13
+#define PA_SC_BINNER_CNTL_OVERRIDE__DIRECT_OVERRIDE_MODE__SHIFT                                               0x1b
+#define PA_SC_BINNER_CNTL_OVERRIDE__OVERRIDE__SHIFT                                                           0x1c
+#define PA_SC_BINNER_CNTL_OVERRIDE__BINNING_MODE_MASK                                                         0x00000003L
+#define PA_SC_BINNER_CNTL_OVERRIDE__CONTEXT_STATES_PER_BIN_MASK                                               0x00001C00L
+#define PA_SC_BINNER_CNTL_OVERRIDE__PERSISTENT_STATES_PER_BIN_MASK                                            0x0003E000L
+#define PA_SC_BINNER_CNTL_OVERRIDE__FPOVS_PER_BATCH_MASK                                                      0x07F80000L
+#define PA_SC_BINNER_CNTL_OVERRIDE__DIRECT_OVERRIDE_MODE_MASK                                                 0x08000000L
+#define PA_SC_BINNER_CNTL_OVERRIDE__OVERRIDE_MASK                                                             0xF0000000L
+//PA_SC_PBB_OVERRIDE_FLAG
+#define PA_SC_PBB_OVERRIDE_FLAG__OVERRIDE__SHIFT                                                              0x0
+#define PA_SC_PBB_OVERRIDE_FLAG__PIPE_ID__SHIFT                                                               0x1
+#define PA_SC_PBB_OVERRIDE_FLAG__OVERRIDE_MASK                                                                0x00000001L
+#define PA_SC_PBB_OVERRIDE_FLAG__PIPE_ID_MASK                                                                 0x00000002L
+//PA_SC_DSM_CNTL
+#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_0__SHIFT                                                                0x0
+#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_1__SHIFT                                                                0x1
+#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_0_MASK                                                                  0x00000001L
+#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_1_MASK                                                                  0x00000002L
+//PA_SC_TILE_STEERING_CREST_OVERRIDE
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__ONE_RB_MODE_ENABLE__SHIFT                                         0x0
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__SE_SELECT__SHIFT                                                  0x1
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__RB_SELECT__SHIFT                                                  0x5
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__SA_SELECT__SHIFT                                                  0x8
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__FORCE_TILE_STEERING_OVERRIDE_USE__SHIFT                           0x1f
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__ONE_RB_MODE_ENABLE_MASK                                           0x00000001L
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__SE_SELECT_MASK                                                    0x00000006L
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__RB_SELECT_MASK                                                    0x00000060L
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__SA_SELECT_MASK                                                    0x00000700L
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__FORCE_TILE_STEERING_OVERRIDE_USE_MASK                             0x80000000L
+//PA_SC_FIFO_SIZE
+#define PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT                                                    0x0
+#define PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT                                                     0x6
+#define PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT                                                         0xf
+#define PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT                                                      0x15
+#define PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE_MASK                                                      0x0000003FL
+#define PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE_MASK                                                       0x00007FC0L
+#define PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE_MASK                                                           0x001F8000L
+#define PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE_MASK                                                        0xFFE00000L
+//PA_SC_IF_FIFO_SIZE
+#define PA_SC_IF_FIFO_SIZE__SC_DB_TILE_IF_FIFO_SIZE__SHIFT                                                    0x0
+#define PA_SC_IF_FIFO_SIZE__SC_DB_QUAD_IF_FIFO_SIZE__SHIFT                                                    0x6
+#define PA_SC_IF_FIFO_SIZE__SC_SPI_IF_FIFO_SIZE__SHIFT                                                        0xc
+#define PA_SC_IF_FIFO_SIZE__SC_BCI_IF_FIFO_SIZE__SHIFT                                                        0x12
+#define PA_SC_IF_FIFO_SIZE__SC_DB_TILE_IF_FIFO_SIZE_MASK                                                      0x0000003FL
+#define PA_SC_IF_FIFO_SIZE__SC_DB_QUAD_IF_FIFO_SIZE_MASK                                                      0x00000FC0L
+#define PA_SC_IF_FIFO_SIZE__SC_SPI_IF_FIFO_SIZE_MASK                                                          0x0003F000L
+#define PA_SC_IF_FIFO_SIZE__SC_BCI_IF_FIFO_SIZE_MASK                                                          0x00FC0000L
+//PA_SC_PACKER_WAVE_ID_CNTL
+#define PA_SC_PACKER_WAVE_ID_CNTL__WAVES_IN_FLIGHT_LIMIT__SHIFT                                               0x0
+#define PA_SC_PACKER_WAVE_ID_CNTL__SC_DB_WAVE_IF_FIFO_SIZE__SHIFT                                             0xa
+#define PA_SC_PACKER_WAVE_ID_CNTL__DISABLE_SC_DB_WAVE_IF_FGCG_EN__SHIFT                                       0x10
+#define PA_SC_PACKER_WAVE_ID_CNTL__SC_SPI_WAVE_IF_FIFO_SIZE__SHIFT                                            0x11
+#define PA_SC_PACKER_WAVE_ID_CNTL__DISABLE_SC_SPI_WAVE_IF_FGCG_EN__SHIFT                                      0x17
+#define PA_SC_PACKER_WAVE_ID_CNTL__DEBUG_CONFLICT_QUAD__SHIFT                                                 0x18
+#define PA_SC_PACKER_WAVE_ID_CNTL__DISABLE_OREO_CONFLICT_QUAD__SHIFT                                          0x1f
+#define PA_SC_PACKER_WAVE_ID_CNTL__WAVES_IN_FLIGHT_LIMIT_MASK                                                 0x000001FFL
+#define PA_SC_PACKER_WAVE_ID_CNTL__SC_DB_WAVE_IF_FIFO_SIZE_MASK                                               0x0000FC00L
+#define PA_SC_PACKER_WAVE_ID_CNTL__DISABLE_SC_DB_WAVE_IF_FGCG_EN_MASK                                         0x00010000L
+#define PA_SC_PACKER_WAVE_ID_CNTL__SC_SPI_WAVE_IF_FIFO_SIZE_MASK                                              0x007E0000L
+#define PA_SC_PACKER_WAVE_ID_CNTL__DISABLE_SC_SPI_WAVE_IF_FGCG_EN_MASK                                        0x00800000L
+#define PA_SC_PACKER_WAVE_ID_CNTL__DEBUG_CONFLICT_QUAD_MASK                                                   0x0F000000L
+#define PA_SC_PACKER_WAVE_ID_CNTL__DISABLE_OREO_CONFLICT_QUAD_MASK                                            0x80000000L
+//PA_SC_ATM_CNTL
+#define PA_SC_ATM_CNTL__SC_PC_IF_SIZE__SHIFT                                                                  0x0
+#define PA_SC_ATM_CNTL__DISABLE_SC_PC_IF_FGCG_EN__SHIFT                                                       0x7
+#define PA_SC_ATM_CNTL__MAX_ATTRIBUTES_IN_WAVE__SHIFT                                                         0x8
+#define PA_SC_ATM_CNTL__DISABLE_MAX_ATTRIBUTES__SHIFT                                                         0x10
+#define PA_SC_ATM_CNTL__SELECT_MAX_ATTRIBUTES__SHIFT                                                          0x11
+#define PA_SC_ATM_CNTL__SC_PC_IF_SIZE_MASK                                                                    0x0000003FL
+#define PA_SC_ATM_CNTL__DISABLE_SC_PC_IF_FGCG_EN_MASK                                                         0x00000080L
+#define PA_SC_ATM_CNTL__MAX_ATTRIBUTES_IN_WAVE_MASK                                                           0x0000FF00L
+#define PA_SC_ATM_CNTL__DISABLE_MAX_ATTRIBUTES_MASK                                                           0x00010000L
+#define PA_SC_ATM_CNTL__SELECT_MAX_ATTRIBUTES_MASK                                                            0x00020000L
+//PA_SC_PKR_WAVE_TABLE_CNTL
+#define PA_SC_PKR_WAVE_TABLE_CNTL__SIZE__SHIFT                                                                0x0
+#define PA_SC_PKR_WAVE_TABLE_CNTL__SIZE_MASK                                                                  0x0000003FL
+//PA_SC_FORCE_EOV_MAX_CNTS
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT__SHIFT                                                0x0
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT__SHIFT                                                0x10
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT_MASK                                                  0x0000FFFFL
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT_MASK                                                  0xFFFF0000L
+//PA_SC_BINNER_EVENT_CNTL_0
+#define PA_SC_BINNER_EVENT_CNTL_0__RESERVED_0__SHIFT                                                          0x0
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS1__SHIFT                                              0x2
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS2__SHIFT                                              0x4
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS3__SHIFT                                              0x6
+#define PA_SC_BINNER_EVENT_CNTL_0__CACHE_FLUSH_TS__SHIFT                                                      0x8
+#define PA_SC_BINNER_EVENT_CNTL_0__CONTEXT_DONE__SHIFT                                                        0xa
+#define PA_SC_BINNER_EVENT_CNTL_0__CACHE_FLUSH__SHIFT                                                         0xc
+#define PA_SC_BINNER_EVENT_CNTL_0__CS_PARTIAL_FLUSH__SHIFT                                                    0xe
+#define PA_SC_BINNER_EVENT_CNTL_0__VGT_STREAMOUT_SYNC__SHIFT                                                  0x10
+#define PA_SC_BINNER_EVENT_CNTL_0__EVENT_STATE_CHANGE__SHIFT                                                  0x12
+#define PA_SC_BINNER_EVENT_CNTL_0__VGT_STREAMOUT_RESET__SHIFT                                                 0x14
+#define PA_SC_BINNER_EVENT_CNTL_0__END_OF_PIPE_INCR_DE__SHIFT                                                 0x16
+#define PA_SC_BINNER_EVENT_CNTL_0__END_OF_PIPE_IB_END__SHIFT                                                  0x18
+#define PA_SC_BINNER_EVENT_CNTL_0__RST_PIX_CNT__SHIFT                                                         0x1a
+#define PA_SC_BINNER_EVENT_CNTL_0__BREAK_BATCH__SHIFT                                                         0x1c
+#define PA_SC_BINNER_EVENT_CNTL_0__VS_PARTIAL_FLUSH__SHIFT                                                    0x1e
+#define PA_SC_BINNER_EVENT_CNTL_0__RESERVED_0_MASK                                                            0x00000003L
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS1_MASK                                                0x0000000CL
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS2_MASK                                                0x00000030L
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS3_MASK                                                0x000000C0L
+#define PA_SC_BINNER_EVENT_CNTL_0__CACHE_FLUSH_TS_MASK                                                        0x00000300L
+#define PA_SC_BINNER_EVENT_CNTL_0__CONTEXT_DONE_MASK                                                          0x00000C00L
+#define PA_SC_BINNER_EVENT_CNTL_0__CACHE_FLUSH_MASK                                                           0x00003000L
+#define PA_SC_BINNER_EVENT_CNTL_0__CS_PARTIAL_FLUSH_MASK                                                      0x0000C000L
+#define PA_SC_BINNER_EVENT_CNTL_0__VGT_STREAMOUT_SYNC_MASK                                                    0x00030000L
+#define PA_SC_BINNER_EVENT_CNTL_0__EVENT_STATE_CHANGE_MASK                                                    0x000C0000L
+#define PA_SC_BINNER_EVENT_CNTL_0__VGT_STREAMOUT_RESET_MASK                                                   0x00300000L
+#define PA_SC_BINNER_EVENT_CNTL_0__END_OF_PIPE_INCR_DE_MASK                                                   0x00C00000L
+#define PA_SC_BINNER_EVENT_CNTL_0__END_OF_PIPE_IB_END_MASK                                                    0x03000000L
+#define PA_SC_BINNER_EVENT_CNTL_0__RST_PIX_CNT_MASK                                                           0x0C000000L
+#define PA_SC_BINNER_EVENT_CNTL_0__BREAK_BATCH_MASK                                                           0x30000000L
+#define PA_SC_BINNER_EVENT_CNTL_0__VS_PARTIAL_FLUSH_MASK                                                      0xC0000000L
+//PA_SC_BINNER_EVENT_CNTL_1
+#define PA_SC_BINNER_EVENT_CNTL_1__PS_PARTIAL_FLUSH__SHIFT                                                    0x0
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_HS_OUTPUT__SHIFT                                                     0x2
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_DFSM__SHIFT                                                          0x4
+#define PA_SC_BINNER_EVENT_CNTL_1__RESET_TO_LOWEST_VGT__SHIFT                                                 0x6
+#define PA_SC_BINNER_EVENT_CNTL_1__CACHE_FLUSH_AND_INV_TS_EVENT__SHIFT                                        0x8
+#define PA_SC_BINNER_EVENT_CNTL_1__WAIT_SYNC__SHIFT                                                           0xa
+#define PA_SC_BINNER_EVENT_CNTL_1__CACHE_FLUSH_AND_INV_EVENT__SHIFT                                           0xc
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_START__SHIFT                                                   0xe
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_STOP__SHIFT                                                    0x10
+#define PA_SC_BINNER_EVENT_CNTL_1__PIPELINESTAT_START__SHIFT                                                  0x12
+#define PA_SC_BINNER_EVENT_CNTL_1__PIPELINESTAT_STOP__SHIFT                                                   0x14
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_SAMPLE__SHIFT                                                  0x16
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_ES_OUTPUT__SHIFT                                                     0x18
+#define PA_SC_BINNER_EVENT_CNTL_1__BIN_CONF_OVERRIDE_CHECK__SHIFT                                             0x1a
+#define PA_SC_BINNER_EVENT_CNTL_1__SAMPLE_PIPELINESTAT__SHIFT                                                 0x1c
+#define PA_SC_BINNER_EVENT_CNTL_1__SO_VGTSTREAMOUT_FLUSH__SHIFT                                               0x1e
+#define PA_SC_BINNER_EVENT_CNTL_1__PS_PARTIAL_FLUSH_MASK                                                      0x00000003L
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_HS_OUTPUT_MASK                                                       0x0000000CL
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_DFSM_MASK                                                            0x00000030L
+#define PA_SC_BINNER_EVENT_CNTL_1__RESET_TO_LOWEST_VGT_MASK                                                   0x000000C0L
+#define PA_SC_BINNER_EVENT_CNTL_1__CACHE_FLUSH_AND_INV_TS_EVENT_MASK                                          0x00000300L
+#define PA_SC_BINNER_EVENT_CNTL_1__WAIT_SYNC_MASK                                                             0x00000C00L
+#define PA_SC_BINNER_EVENT_CNTL_1__CACHE_FLUSH_AND_INV_EVENT_MASK                                             0x00003000L
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_START_MASK                                                     0x0000C000L
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_STOP_MASK                                                      0x00030000L
+#define PA_SC_BINNER_EVENT_CNTL_1__PIPELINESTAT_START_MASK                                                    0x000C0000L
+#define PA_SC_BINNER_EVENT_CNTL_1__PIPELINESTAT_STOP_MASK                                                     0x00300000L
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_SAMPLE_MASK                                                    0x00C00000L
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_ES_OUTPUT_MASK                                                       0x03000000L
+#define PA_SC_BINNER_EVENT_CNTL_1__BIN_CONF_OVERRIDE_CHECK_MASK                                               0x0C000000L
+#define PA_SC_BINNER_EVENT_CNTL_1__SAMPLE_PIPELINESTAT_MASK                                                   0x30000000L
+#define PA_SC_BINNER_EVENT_CNTL_1__SO_VGTSTREAMOUT_FLUSH_MASK                                                 0xC0000000L
+//PA_SC_BINNER_EVENT_CNTL_2
+#define PA_SC_BINNER_EVENT_CNTL_2__SAMPLE_STREAMOUTSTATS__SHIFT                                               0x0
+#define PA_SC_BINNER_EVENT_CNTL_2__RESET_VTX_CNT__SHIFT                                                       0x2
+#define PA_SC_BINNER_EVENT_CNTL_2__BLOCK_CONTEXT_DONE__SHIFT                                                  0x4
+#define PA_SC_BINNER_EVENT_CNTL_2__RESERVED_35__SHIFT                                                         0x6
+#define PA_SC_BINNER_EVENT_CNTL_2__VGT_FLUSH__SHIFT                                                           0x8
+#define PA_SC_BINNER_EVENT_CNTL_2__TGID_ROLLOVER__SHIFT                                                       0xa
+#define PA_SC_BINNER_EVENT_CNTL_2__SQ_NON_EVENT__SHIFT                                                        0xc
+#define PA_SC_BINNER_EVENT_CNTL_2__SC_SEND_DB_VPZ__SHIFT                                                      0xe
+#define PA_SC_BINNER_EVENT_CNTL_2__BOTTOM_OF_PIPE_TS__SHIFT                                                   0x10
+#define PA_SC_BINNER_EVENT_CNTL_2__RESERVED_41__SHIFT                                                         0x12
+#define PA_SC_BINNER_EVENT_CNTL_2__DB_CACHE_FLUSH_AND_INV__SHIFT                                              0x14
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_DB_DATA_TS__SHIFT                                            0x16
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_DB_META__SHIFT                                               0x18
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_CB_DATA_TS__SHIFT                                            0x1a
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_CB_META__SHIFT                                               0x1c
+#define PA_SC_BINNER_EVENT_CNTL_2__CS_DONE__SHIFT                                                             0x1e
+#define PA_SC_BINNER_EVENT_CNTL_2__SAMPLE_STREAMOUTSTATS_MASK                                                 0x00000003L
+#define PA_SC_BINNER_EVENT_CNTL_2__RESET_VTX_CNT_MASK                                                         0x0000000CL
+#define PA_SC_BINNER_EVENT_CNTL_2__BLOCK_CONTEXT_DONE_MASK                                                    0x00000030L
+#define PA_SC_BINNER_EVENT_CNTL_2__RESERVED_35_MASK                                                           0x000000C0L
+#define PA_SC_BINNER_EVENT_CNTL_2__VGT_FLUSH_MASK                                                             0x00000300L
+#define PA_SC_BINNER_EVENT_CNTL_2__TGID_ROLLOVER_MASK                                                         0x00000C00L
+#define PA_SC_BINNER_EVENT_CNTL_2__SQ_NON_EVENT_MASK                                                          0x00003000L
+#define PA_SC_BINNER_EVENT_CNTL_2__SC_SEND_DB_VPZ_MASK                                                        0x0000C000L
+#define PA_SC_BINNER_EVENT_CNTL_2__BOTTOM_OF_PIPE_TS_MASK                                                     0x00030000L
+#define PA_SC_BINNER_EVENT_CNTL_2__RESERVED_41_MASK                                                           0x000C0000L
+#define PA_SC_BINNER_EVENT_CNTL_2__DB_CACHE_FLUSH_AND_INV_MASK                                                0x00300000L
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_DB_DATA_TS_MASK                                              0x00C00000L
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_DB_META_MASK                                                 0x03000000L
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_CB_DATA_TS_MASK                                              0x0C000000L
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_CB_META_MASK                                                 0x30000000L
+#define PA_SC_BINNER_EVENT_CNTL_2__CS_DONE_MASK                                                               0xC0000000L
+//PA_SC_BINNER_EVENT_CNTL_3
+#define PA_SC_BINNER_EVENT_CNTL_3__PS_DONE__SHIFT                                                             0x0
+#define PA_SC_BINNER_EVENT_CNTL_3__FLUSH_AND_INV_CB_PIXEL_DATA__SHIFT                                         0x2
+#define PA_SC_BINNER_EVENT_CNTL_3__RESERVED_50__SHIFT                                                         0x4
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_START__SHIFT                                                  0x6
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_STOP__SHIFT                                                   0x8
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_MARKER__SHIFT                                                 0xa
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_DRAW__SHIFT                                                   0xc
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_FINISH__SHIFT                                                 0xe
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_CONTROL__SHIFT                                             0x10
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_DUMP__SHIFT                                                0x12
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_RESET__SHIFT                                               0x14
+#define PA_SC_BINNER_EVENT_CNTL_3__CONTEXT_SUSPEND__SHIFT                                                     0x16
+#define PA_SC_BINNER_EVENT_CNTL_3__OFFCHIP_HS_DEALLOC__SHIFT                                                  0x18
+#define PA_SC_BINNER_EVENT_CNTL_3__ENABLE_NGG_PIPELINE__SHIFT                                                 0x1a
+#define PA_SC_BINNER_EVENT_CNTL_3__ENABLE_PIPELINE_NOT_USED__SHIFT                                            0x1c
+#define PA_SC_BINNER_EVENT_CNTL_3__DRAW_DONE__SHIFT                                                           0x1e
+#define PA_SC_BINNER_EVENT_CNTL_3__PS_DONE_MASK                                                               0x00000003L
+#define PA_SC_BINNER_EVENT_CNTL_3__FLUSH_AND_INV_CB_PIXEL_DATA_MASK                                           0x0000000CL
+#define PA_SC_BINNER_EVENT_CNTL_3__RESERVED_50_MASK                                                           0x00000030L
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_START_MASK                                                    0x000000C0L
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_STOP_MASK                                                     0x00000300L
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_MARKER_MASK                                                   0x00000C00L
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_DRAW_MASK                                                     0x00003000L
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_FINISH_MASK                                                   0x0000C000L
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_CONTROL_MASK                                               0x00030000L
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_DUMP_MASK                                                  0x000C0000L
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_RESET_MASK                                                 0x00300000L
+#define PA_SC_BINNER_EVENT_CNTL_3__CONTEXT_SUSPEND_MASK                                                       0x00C00000L
+#define PA_SC_BINNER_EVENT_CNTL_3__OFFCHIP_HS_DEALLOC_MASK                                                    0x03000000L
+#define PA_SC_BINNER_EVENT_CNTL_3__ENABLE_NGG_PIPELINE_MASK                                                   0x0C000000L
+#define PA_SC_BINNER_EVENT_CNTL_3__ENABLE_PIPELINE_NOT_USED_MASK                                              0x30000000L
+#define PA_SC_BINNER_EVENT_CNTL_3__DRAW_DONE_MASK                                                             0xC0000000L
+//PA_SC_BINNER_TIMEOUT_COUNTER
+#define PA_SC_BINNER_TIMEOUT_COUNTER__THRESHOLD__SHIFT                                                        0x0
+#define PA_SC_BINNER_TIMEOUT_COUNTER__THRESHOLD_MASK                                                          0xFFFFFFFFL
+//PA_SC_BINNER_PERF_CNTL_0
+#define PA_SC_BINNER_PERF_CNTL_0__BIN_HIST_NUM_PRIMS_THRESHOLD__SHIFT                                         0x0
+#define PA_SC_BINNER_PERF_CNTL_0__BATCH_HIST_NUM_PRIMS_THRESHOLD__SHIFT                                       0xa
+#define PA_SC_BINNER_PERF_CNTL_0__BIN_HIST_NUM_CONTEXT_THRESHOLD__SHIFT                                       0x14
+#define PA_SC_BINNER_PERF_CNTL_0__BATCH_HIST_NUM_CONTEXT_THRESHOLD__SHIFT                                     0x17
+#define PA_SC_BINNER_PERF_CNTL_0__BIN_HIST_NUM_PRIMS_THRESHOLD_MASK                                           0x000003FFL
+#define PA_SC_BINNER_PERF_CNTL_0__BATCH_HIST_NUM_PRIMS_THRESHOLD_MASK                                         0x000FFC00L
+#define PA_SC_BINNER_PERF_CNTL_0__BIN_HIST_NUM_CONTEXT_THRESHOLD_MASK                                         0x00700000L
+#define PA_SC_BINNER_PERF_CNTL_0__BATCH_HIST_NUM_CONTEXT_THRESHOLD_MASK                                       0x03800000L
+//PA_SC_BINNER_PERF_CNTL_1
+#define PA_SC_BINNER_PERF_CNTL_1__BIN_HIST_NUM_PERSISTENT_STATE_THRESHOLD__SHIFT                              0x0
+#define PA_SC_BINNER_PERF_CNTL_1__BATCH_HIST_NUM_PERSISTENT_STATE_THRESHOLD__SHIFT                            0x5
+#define PA_SC_BINNER_PERF_CNTL_1__BATCH_HIST_NUM_TRIV_REJECTED_PRIMS_THRESHOLD__SHIFT                         0xa
+#define PA_SC_BINNER_PERF_CNTL_1__BIN_HIST_NUM_PERSISTENT_STATE_THRESHOLD_MASK                                0x0000001FL
+#define PA_SC_BINNER_PERF_CNTL_1__BATCH_HIST_NUM_PERSISTENT_STATE_THRESHOLD_MASK                              0x000003E0L
+#define PA_SC_BINNER_PERF_CNTL_1__BATCH_HIST_NUM_TRIV_REJECTED_PRIMS_THRESHOLD_MASK                           0x03FFFC00L
+//PA_SC_BINNER_PERF_CNTL_2
+#define PA_SC_BINNER_PERF_CNTL_2__BATCH_HIST_NUM_ROWS_PER_PRIM_THRESHOLD__SHIFT                               0x0
+#define PA_SC_BINNER_PERF_CNTL_2__BATCH_HIST_NUM_COLUMNS_PER_ROW_THRESHOLD__SHIFT                             0xb
+#define PA_SC_BINNER_PERF_CNTL_2__BATCH_HIST_NUM_ROWS_PER_PRIM_THRESHOLD_MASK                                 0x000007FFL
+#define PA_SC_BINNER_PERF_CNTL_2__BATCH_HIST_NUM_COLUMNS_PER_ROW_THRESHOLD_MASK                               0x003FF800L
+//PA_SC_BINNER_PERF_CNTL_3
+#define PA_SC_BINNER_PERF_CNTL_3__BATCH_HIST_NUM_PS_WAVE_BREAKS_THRESHOLD__SHIFT                              0x0
+#define PA_SC_BINNER_PERF_CNTL_3__BATCH_HIST_NUM_PS_WAVE_BREAKS_THRESHOLD_MASK                                0xFFFFFFFFL
+//PA_SC_P3D_TRAP_SCREEN_HV_LOCK
+#define PA_SC_P3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT                                         0x0
+#define PA_SC_P3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK                                           0x00000001L
+//PA_SC_HP3D_TRAP_SCREEN_HV_LOCK
+#define PA_SC_HP3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT                                        0x0
+#define PA_SC_HP3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK                                          0x00000001L
+//PA_SC_TRAP_SCREEN_HV_LOCK
+#define PA_SC_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT                                             0x0
+#define PA_SC_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK                                               0x00000001L
+//PA_PH_INTERFACE_FIFO_SIZE
+#define PA_PH_INTERFACE_FIFO_SIZE__PA_PH_IF_FIFO_SIZE__SHIFT                                                  0x0
+#define PA_PH_INTERFACE_FIFO_SIZE__PH_SC_IF_FIFO_SIZE__SHIFT                                                  0x10
+#define PA_PH_INTERFACE_FIFO_SIZE__PA_PH_IF_FIFO_SIZE_MASK                                                    0x000003FFL
+#define PA_PH_INTERFACE_FIFO_SIZE__PH_SC_IF_FIFO_SIZE_MASK                                                    0x003F0000L
+//PA_PH_ENHANCE
+#define PA_PH_ENHANCE__ECO_SPARE0__SHIFT                                                                      0x0
+#define PA_PH_ENHANCE__ECO_SPARE1__SHIFT                                                                      0x1
+#define PA_PH_ENHANCE__ECO_SPARE2__SHIFT                                                                      0x2
+#define PA_PH_ENHANCE__ECO_SPARE3__SHIFT                                                                      0x3
+#define PA_PH_ENHANCE__DISABLE_PH_SC_INTF_FINE_CLOCK_GATE__SHIFT                                              0x4
+#define PA_PH_ENHANCE__DISABLE_FOPKT__SHIFT                                                                   0x5
+#define PA_PH_ENHANCE__DISABLE_FOPKT_SCAN_POST_RESET__SHIFT                                                   0x6
+#define PA_PH_ENHANCE__DISABLE_PH_SC_INTF_CLKEN_CLOCK_GATE__SHIFT                                             0x7
+#define PA_PH_ENHANCE__DISABLE_PH_DEBUG_REG_FGCG__SHIFT                                                       0x8
+#define PA_PH_ENHANCE__DISABLE_PH_PERF_REG_FGCG__SHIFT                                                        0x9
+#define PA_PH_ENHANCE__ENABLE_PH_INTF_CLKEN_STRETCH__SHIFT                                                    0xa
+#define PA_PH_ENHANCE__DISABLE_USE_LAST_PH_ARBITER_PERFCOUNTER_SAMPLE_EVENT__SHIFT                            0xd
+#define PA_PH_ENHANCE__USE_PERFCOUNTER_START_STOP_EVENTS__SHIFT                                               0xe
+#define PA_PH_ENHANCE__FORCE_PH_PERFCOUNTER_SAMPLE_ENABLE_ON__SHIFT                                           0xf
+#define PA_PH_ENHANCE__PH_SPI_GE_THROTTLE_MODE__SHIFT                                                         0x10
+#define PA_PH_ENHANCE__PH_SPI_GE_THROTTLE_MODE_DISABLE__SHIFT                                                 0x11
+#define PA_PH_ENHANCE__PH_SPI_GE_THROTTLE_PERFCOUNTER_COUNT_MODE__SHIFT                                       0x12
+#define PA_PH_ENHANCE__ECO_SPARE0_MASK                                                                        0x00000001L
+#define PA_PH_ENHANCE__ECO_SPARE1_MASK                                                                        0x00000002L
+#define PA_PH_ENHANCE__ECO_SPARE2_MASK                                                                        0x00000004L
+#define PA_PH_ENHANCE__ECO_SPARE3_MASK                                                                        0x00000008L
+#define PA_PH_ENHANCE__DISABLE_PH_SC_INTF_FINE_CLOCK_GATE_MASK                                                0x00000010L
+#define PA_PH_ENHANCE__DISABLE_FOPKT_MASK                                                                     0x00000020L
+#define PA_PH_ENHANCE__DISABLE_FOPKT_SCAN_POST_RESET_MASK                                                     0x00000040L
+#define PA_PH_ENHANCE__DISABLE_PH_SC_INTF_CLKEN_CLOCK_GATE_MASK                                               0x00000080L
+#define PA_PH_ENHANCE__DISABLE_PH_DEBUG_REG_FGCG_MASK                                                         0x00000100L
+#define PA_PH_ENHANCE__DISABLE_PH_PERF_REG_FGCG_MASK                                                          0x00000200L
+#define PA_PH_ENHANCE__ENABLE_PH_INTF_CLKEN_STRETCH_MASK                                                      0x00001C00L
+#define PA_PH_ENHANCE__DISABLE_USE_LAST_PH_ARBITER_PERFCOUNTER_SAMPLE_EVENT_MASK                              0x00002000L
+#define PA_PH_ENHANCE__USE_PERFCOUNTER_START_STOP_EVENTS_MASK                                                 0x00004000L
+#define PA_PH_ENHANCE__FORCE_PH_PERFCOUNTER_SAMPLE_ENABLE_ON_MASK                                             0x00008000L
+#define PA_PH_ENHANCE__PH_SPI_GE_THROTTLE_MODE_MASK                                                           0x00010000L
+#define PA_PH_ENHANCE__PH_SPI_GE_THROTTLE_MODE_DISABLE_MASK                                                   0x00020000L
+#define PA_PH_ENHANCE__PH_SPI_GE_THROTTLE_PERFCOUNTER_COUNT_MODE_MASK                                         0x00040000L
+//PA_SC_VRS_SURFACE_CNTL_1
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE__SHIFT                                               0x0
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_SHADER_KILL_ENABLE__SHIFT                            0x1
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_MASK_OPS_ENABLE__SHIFT                               0x2
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_INTERFACE_RATE_16XAA__SHIFT                          0x3
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_Z_OR_STENCIL__SHIFT                                  0x4
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_PRE_SHADER_DEPTH_COVERAGE_ENABLED__SHIFT             0x5
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_POST_DEPTH_IMPORT__SHIFT                             0x6
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_POPS__SHIFT                                          0x7
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_INTERFACE_RATE_8XAA__SHIFT                           0x8
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_INTRINSIC_RATE_GT_4XAA__SHIFT                        0x9
+#define PA_SC_VRS_SURFACE_CNTL_1__USE_ONLY_VRS_RATE_FINE_CFG__SHIFT                                           0xa
+#define PA_SC_VRS_SURFACE_CNTL_1__DISABLE_SSAA_VRS_RATE_NORMALIZATION__SHIFT                                  0xc
+#define PA_SC_VRS_SURFACE_CNTL_1__DISABLE_PS_ITER_RATE_COMBINER_PASSTHRU_OVERRIDE__SHIFT                      0xf
+#define PA_SC_VRS_SURFACE_CNTL_1__DISABLE_CMASK_RATE_HINT_FORCE_ZERO_OVERRIDE__SHIFT                          0x13
+#define PA_SC_VRS_SURFACE_CNTL_1__DISABLE_SSAA_DETAIL_TO_EXPOSED_RATE_CLAMPING__SHIFT                         0x14
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_0__SHIFT                                                      0x15
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_1__SHIFT                                                      0x16
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_2__SHIFT                                                      0x17
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_3__SHIFT                                                      0x18
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_4__SHIFT                                                      0x19
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_5__SHIFT                                                      0x1a
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_6__SHIFT                                                      0x1b
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_7__SHIFT                                                      0x1c
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_8__SHIFT                                                      0x1d
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_9__SHIFT                                                      0x1e
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_10__SHIFT                                                     0x1f
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_MASK                                                 0x00000001L
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_SHADER_KILL_ENABLE_MASK                              0x00000002L
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_MASK_OPS_ENABLE_MASK                                 0x00000004L
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_INTERFACE_RATE_16XAA_MASK                            0x00000008L
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_Z_OR_STENCIL_MASK                                    0x00000010L
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_PRE_SHADER_DEPTH_COVERAGE_ENABLED_MASK               0x00000020L
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_POST_DEPTH_IMPORT_MASK                               0x00000040L
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_POPS_MASK                                            0x00000080L
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_INTERFACE_RATE_8XAA_MASK                             0x00000100L
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_INTRINSIC_RATE_GT_4XAA_MASK                          0x00000200L
+#define PA_SC_VRS_SURFACE_CNTL_1__USE_ONLY_VRS_RATE_FINE_CFG_MASK                                             0x00000400L
+#define PA_SC_VRS_SURFACE_CNTL_1__DISABLE_SSAA_VRS_RATE_NORMALIZATION_MASK                                    0x00001000L
+#define PA_SC_VRS_SURFACE_CNTL_1__DISABLE_PS_ITER_RATE_COMBINER_PASSTHRU_OVERRIDE_MASK                        0x00008000L
+#define PA_SC_VRS_SURFACE_CNTL_1__DISABLE_CMASK_RATE_HINT_FORCE_ZERO_OVERRIDE_MASK                            0x00080000L
+#define PA_SC_VRS_SURFACE_CNTL_1__DISABLE_SSAA_DETAIL_TO_EXPOSED_RATE_CLAMPING_MASK                           0x00100000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_0_MASK                                                        0x00200000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_1_MASK                                                        0x00400000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_2_MASK                                                        0x00800000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_3_MASK                                                        0x01000000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_4_MASK                                                        0x02000000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_5_MASK                                                        0x04000000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_6_MASK                                                        0x08000000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_7_MASK                                                        0x10000000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_8_MASK                                                        0x20000000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_9_MASK                                                        0x40000000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_10_MASK                                                       0x80000000L
+//PA_SC_HIZ_SURFACE_CNTL
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_OUTSTANDING_CONTEXT_FILTERING_DISABLE__SHIFT                              0x5
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_CONTEXT_DONE_SYNC_DISABLE__SHIFT                                          0x6
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_DB_CONTEXT_DONE_SYNC_DISABLE__SHIFT                                       0x7
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_FLUSH_EVENT_MASK_DISABLE__SHIFT                                           0x8
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_PREFETCH_DISABLE__SHIFT                                                   0xd
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_FLUSH_NO_INV_DISABLE__SHIFT                                               0xe
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_NONSTALLING_FLUSH_DISABLE__SHIFT                                          0xf
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_PARTIAL_FLUSH_DISABLE__SHIFT                                              0x10
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_EOP_SYNC_DISABLE__SHIFT                                                   0x12
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_MAX_TAGS__SHIFT                                                           0x13
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_EVICT_POINT__SHIFT                                                        0x1a
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_OUTSTANDING_CONTEXT_FILTERING_DISABLE_MASK                                0x00000020L
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_CONTEXT_DONE_SYNC_DISABLE_MASK                                            0x00000040L
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_DB_CONTEXT_DONE_SYNC_DISABLE_MASK                                         0x00000080L
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_FLUSH_EVENT_MASK_DISABLE_MASK                                             0x00001F00L
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_PREFETCH_DISABLE_MASK                                                     0x00002000L
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_FLUSH_NO_INV_DISABLE_MASK                                                 0x00004000L
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_NONSTALLING_FLUSH_DISABLE_MASK                                            0x00008000L
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_PARTIAL_FLUSH_DISABLE_MASK                                                0x00010000L
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_EOP_SYNC_DISABLE_MASK                                                     0x00040000L
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_MAX_TAGS_MASK                                                             0x03F80000L
+#define PA_SC_HIZ_SURFACE_CNTL__HZC_EVICT_POINT_MASK                                                          0xFC000000L
+//PA_SC_HIS_SURFACE_CNTL
+#define PA_SC_HIS_SURFACE_CNTL__HSC_OUTSTANDING_CONTEXT_FILTERING_DISABLE__SHIFT                              0x5
+#define PA_SC_HIS_SURFACE_CNTL__HSC_CONTEXT_DONE_SYNC_DISABLE__SHIFT                                          0x6
+#define PA_SC_HIS_SURFACE_CNTL__HSC_DB_CONTEXT_DONE_SYNC_DISABLE__SHIFT                                       0x7
+#define PA_SC_HIS_SURFACE_CNTL__HSC_FLUSH_EVENT_MASK_DISABLE__SHIFT                                           0x8
+#define PA_SC_HIS_SURFACE_CNTL__HSC_PREFETCH_DISABLE__SHIFT                                                   0xd
+#define PA_SC_HIS_SURFACE_CNTL__HSC_FLUSH_NO_INV_DISABLE__SHIFT                                               0xe
+#define PA_SC_HIS_SURFACE_CNTL__HSC_NONSTALLING_FLUSH_DISABLE__SHIFT                                          0xf
+#define PA_SC_HIS_SURFACE_CNTL__HSC_PARTIAL_FLUSH_DISABLE__SHIFT                                              0x10
+#define PA_SC_HIS_SURFACE_CNTL__HSC_EOP_SYNC_DISABLE__SHIFT                                                   0x12
+#define PA_SC_HIS_SURFACE_CNTL__HSC_MAX_TAGS__SHIFT                                                           0x13
+#define PA_SC_HIS_SURFACE_CNTL__HSC_EVICT_POINT__SHIFT                                                        0x1a
+#define PA_SC_HIS_SURFACE_CNTL__HSC_OUTSTANDING_CONTEXT_FILTERING_DISABLE_MASK                                0x00000020L
+#define PA_SC_HIS_SURFACE_CNTL__HSC_CONTEXT_DONE_SYNC_DISABLE_MASK                                            0x00000040L
+#define PA_SC_HIS_SURFACE_CNTL__HSC_DB_CONTEXT_DONE_SYNC_DISABLE_MASK                                         0x00000080L
+#define PA_SC_HIS_SURFACE_CNTL__HSC_FLUSH_EVENT_MASK_DISABLE_MASK                                             0x00001F00L
+#define PA_SC_HIS_SURFACE_CNTL__HSC_PREFETCH_DISABLE_MASK                                                     0x00002000L
+#define PA_SC_HIS_SURFACE_CNTL__HSC_FLUSH_NO_INV_DISABLE_MASK                                                 0x00004000L
+#define PA_SC_HIS_SURFACE_CNTL__HSC_NONSTALLING_FLUSH_DISABLE_MASK                                            0x00008000L
+#define PA_SC_HIS_SURFACE_CNTL__HSC_PARTIAL_FLUSH_DISABLE_MASK                                                0x00010000L
+#define PA_SC_HIS_SURFACE_CNTL__HSC_EOP_SYNC_DISABLE_MASK                                                     0x00040000L
+#define PA_SC_HIS_SURFACE_CNTL__HSC_MAX_TAGS_MASK                                                             0x03F80000L
+#define PA_SC_HIS_SURFACE_CNTL__HSC_EVICT_POINT_MASK                                                          0xFC000000L
+//PA_SC_HIZ_DEBUG
+#define PA_SC_HIZ_DEBUG__FORCE_Z_MODE__SHIFT                                                                  0x1
+#define PA_SC_HIZ_DEBUG__FORCE_DEPTH_READ__SHIFT                                                              0x3
+#define PA_SC_HIZ_DEBUG__FORCE_HIZ_ENABLE__SHIFT                                                              0x4
+#define PA_SC_HIZ_DEBUG__FAST_Z_DISABLE__SHIFT                                                                0x6
+#define PA_SC_HIZ_DEBUG__NOOP_CULL_DISABLE__SHIFT                                                             0x7
+#define PA_SC_HIZ_DEBUG__FORCE_TILE_OP__SHIFT                                                                 0x8
+#define PA_SC_HIZ_DEBUG__FORCE_HITEST_RESULTS__SHIFT                                                          0xc
+#define PA_SC_HIZ_DEBUG__DISABLE_4X_TILE_PICKING__SHIFT                                                       0x11
+#define PA_SC_HIZ_DEBUG__DISABLE_FAST_SET_WITHOUT_HIZ_SURFACE__SHIFT                                          0x12
+#define PA_SC_HIZ_DEBUG__FORCE_FULL_Z_RANGE__SHIFT                                                            0x13
+#define PA_SC_HIZ_DEBUG__DISABLE_VPORT_ZPLANE_OPTIMIZATION__SHIFT                                             0x15
+#define PA_SC_HIZ_DEBUG__DISABLE_HIZ_ON_VPORT_CLAMP__SHIFT                                                    0x16
+#define PA_SC_HIZ_DEBUG__DISABLE_VPZ_EVENT_FILTERING__SHIFT                                                   0x17
+#define PA_SC_HIZ_DEBUG__DISABLE_REPLICATE_DETAIL_LOCATIONS_TO_SURFACE_RATE_FOR_SURF_GT_DETAIL_RATE__SHIFT    0x18
+#define PA_SC_HIZ_DEBUG__DISABLE_SUMMARIZATION_MONOTONICITY__SHIFT                                            0x19
+#define PA_SC_HIZ_DEBUG__DISABLE_FAST_NO_OP_WITH_TILE_RATE__SHIFT                                             0x1a
+#define PA_SC_HIZ_DEBUG__FORCE_Z_MODE_MASK                                                                    0x00000006L
+#define PA_SC_HIZ_DEBUG__FORCE_DEPTH_READ_MASK                                                                0x00000008L
+#define PA_SC_HIZ_DEBUG__FORCE_HIZ_ENABLE_MASK                                                                0x00000030L
+#define PA_SC_HIZ_DEBUG__FAST_Z_DISABLE_MASK                                                                  0x00000040L
+#define PA_SC_HIZ_DEBUG__NOOP_CULL_DISABLE_MASK                                                               0x00000080L
+#define PA_SC_HIZ_DEBUG__FORCE_TILE_OP_MASK                                                                   0x00000F00L
+#define PA_SC_HIZ_DEBUG__FORCE_HITEST_RESULTS_MASK                                                            0x0001F000L
+#define PA_SC_HIZ_DEBUG__DISABLE_4X_TILE_PICKING_MASK                                                         0x00020000L
+#define PA_SC_HIZ_DEBUG__DISABLE_FAST_SET_WITHOUT_HIZ_SURFACE_MASK                                            0x00040000L
+#define PA_SC_HIZ_DEBUG__FORCE_FULL_Z_RANGE_MASK                                                              0x00180000L
+#define PA_SC_HIZ_DEBUG__DISABLE_VPORT_ZPLANE_OPTIMIZATION_MASK                                               0x00200000L
+#define PA_SC_HIZ_DEBUG__DISABLE_HIZ_ON_VPORT_CLAMP_MASK                                                      0x00400000L
+#define PA_SC_HIZ_DEBUG__DISABLE_VPZ_EVENT_FILTERING_MASK                                                     0x00800000L
+#define PA_SC_HIZ_DEBUG__DISABLE_REPLICATE_DETAIL_LOCATIONS_TO_SURFACE_RATE_FOR_SURF_GT_DETAIL_RATE_MASK      0x01000000L
+#define PA_SC_HIZ_DEBUG__DISABLE_SUMMARIZATION_MONOTONICITY_MASK                                              0x02000000L
+#define PA_SC_HIZ_DEBUG__DISABLE_FAST_NO_OP_WITH_TILE_RATE_MASK                                               0x04000000L
+//PA_SC_HIS_DEBUG
+#define PA_SC_HIS_DEBUG__FORCE_STENCIL_READ__SHIFT                                                            0x1
+#define PA_SC_HIS_DEBUG__FORCE_HIS_ENABLE__SHIFT                                                              0x2
+#define PA_SC_HIS_DEBUG__FAST_STENCIL_DISABLE__SHIFT                                                          0x6
+#define PA_SC_HIS_DEBUG__NOOP_CULL_DISABLE__SHIFT                                                             0x7
+#define PA_SC_HIS_DEBUG__DISABLE_FULLY_COVERED_PRE_HISZ__SHIFT                                                0x8
+#define PA_SC_HIS_DEBUG__DISABLE_FULLY_COVERED_POST_HISZ__SHIFT                                               0x9
+#define PA_SC_HIS_DEBUG__DROP_UNLIT_STILES_AFTER_DETAIL_WALK__SHIFT                                           0xa
+#define PA_SC_HIS_DEBUG__DROP_DUPLICATE_SC_DB_STILE_UPDATE_BEFORE_DETAIL_WALK__SHIFT                          0xb
+#define PA_SC_HIS_DEBUG__FULLY_COVERED_IS_STILE_QUAD_MASK_BASED__SHIFT                                        0xc
+#define PA_SC_HIS_DEBUG__FORCE_SURFACE_ENABLED_TO_HISZ__SHIFT                                                 0xd
+#define PA_SC_HIS_DEBUG__USE_DETAIL_RATE_IN_PRE_HISZ_STW__SHIFT                                               0xe
+#define PA_SC_HIS_DEBUG__DISABLE_SINGLE_STENCIL__SHIFT                                                        0xf
+#define PA_SC_HIS_DEBUG__DISABLE_SUMMARIZATION_MONOTONICITY__SHIFT                                            0x10
+#define PA_SC_HIS_DEBUG__DISABLE_FAST_NO_OP_WITH_TILE_RATE__SHIFT                                             0x11
+#define PA_SC_HIS_DEBUG__FORCE_STENCIL_READ_MASK                                                              0x00000002L
+#define PA_SC_HIS_DEBUG__FORCE_HIS_ENABLE_MASK                                                                0x0000000CL
+#define PA_SC_HIS_DEBUG__FAST_STENCIL_DISABLE_MASK                                                            0x00000040L
+#define PA_SC_HIS_DEBUG__NOOP_CULL_DISABLE_MASK                                                               0x00000080L
+#define PA_SC_HIS_DEBUG__DISABLE_FULLY_COVERED_PRE_HISZ_MASK                                                  0x00000100L
+#define PA_SC_HIS_DEBUG__DISABLE_FULLY_COVERED_POST_HISZ_MASK                                                 0x00000200L
+#define PA_SC_HIS_DEBUG__DROP_UNLIT_STILES_AFTER_DETAIL_WALK_MASK                                             0x00000400L
+#define PA_SC_HIS_DEBUG__DROP_DUPLICATE_SC_DB_STILE_UPDATE_BEFORE_DETAIL_WALK_MASK                            0x00000800L
+#define PA_SC_HIS_DEBUG__FULLY_COVERED_IS_STILE_QUAD_MASK_BASED_MASK                                          0x00001000L
+#define PA_SC_HIS_DEBUG__FORCE_SURFACE_ENABLED_TO_HISZ_MASK                                                   0x00002000L
+#define PA_SC_HIS_DEBUG__USE_DETAIL_RATE_IN_PRE_HISZ_STW_MASK                                                 0x00004000L
+#define PA_SC_HIS_DEBUG__DISABLE_SINGLE_STENCIL_MASK                                                          0x00008000L
+#define PA_SC_HIS_DEBUG__DISABLE_SUMMARIZATION_MONOTONICITY_MASK                                              0x00010000L
+#define PA_SC_HIS_DEBUG__DISABLE_FAST_NO_OP_WITH_TILE_RATE_MASK                                               0x00020000L
+//SC_MEM_SCOPE
+#define SC_MEM_SCOPE__VRS_RATE_SCOPE__SHIFT                                                                   0x0
+#define SC_MEM_SCOPE__VRS_FEEDBACK_SCOPE__SHIFT                                                               0x2
+#define SC_MEM_SCOPE__HIZ_SCOPE__SHIFT                                                                        0x4
+#define SC_MEM_SCOPE__HIS_SCOPE__SHIFT                                                                        0x6
+#define SC_MEM_SCOPE__VRS_RATE_SCOPE_MASK                                                                     0x00000003L
+#define SC_MEM_SCOPE__VRS_FEEDBACK_SCOPE_MASK                                                                 0x0000000CL
+#define SC_MEM_SCOPE__HIZ_SCOPE_MASK                                                                          0x00000030L
+#define SC_MEM_SCOPE__HIS_SCOPE_MASK                                                                          0x000000C0L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pfvf_sqdec
+//SQ_RUNTIME_CONFIG
+#define SQ_RUNTIME_CONFIG__UNUSED_REGISTER__SHIFT                                                             0x0
+#define SQ_RUNTIME_CONFIG__UNUSED_REGISTER_MASK                                                               0x00000001L
+//SQ_DEBUG_STS_GLOBAL
+#define SQ_DEBUG_STS_GLOBAL__BUSY__SHIFT                                                                      0x0
+#define SQ_DEBUG_STS_GLOBAL__INTERRUPT_BUSY__SHIFT                                                            0x1
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SA0__SHIFT                                                            0x4
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SA1__SHIFT                                                            0x10
+#define SQ_DEBUG_STS_GLOBAL__BUSY_MASK                                                                        0x00000001L
+#define SQ_DEBUG_STS_GLOBAL__INTERRUPT_BUSY_MASK                                                              0x00000002L
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SA0_MASK                                                              0x0000FFF0L
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SA1_MASK                                                              0x0FFF0000L
+//SQ_DEBUG_STS_GLOBAL2
+#define SQ_DEBUG_STS_GLOBAL2__REG_FIFO_LEVEL_GFX0__SHIFT                                                      0x0
+#define SQ_DEBUG_STS_GLOBAL2__REG_FIFO_LEVEL_GFX1__SHIFT                                                      0x8
+#define SQ_DEBUG_STS_GLOBAL2__REG_FIFO_LEVEL_COMPUTE__SHIFT                                                   0x10
+#define SQ_DEBUG_STS_GLOBAL2__REG_FIFO_LEVEL_GFX0_MASK                                                        0x000000FFL
+#define SQ_DEBUG_STS_GLOBAL2__REG_FIFO_LEVEL_GFX1_MASK                                                        0x0000FF00L
+#define SQ_DEBUG_STS_GLOBAL2__REG_FIFO_LEVEL_COMPUTE_MASK                                                     0x00FF0000L
+//SH_MEM_BASES
+#define SH_MEM_BASES__PRIVATE_BASE__SHIFT                                                                     0xa
+#define SH_MEM_BASES__SHARED_BASE__SHIFT                                                                      0x10
+#define SH_MEM_BASES__PRIVATE_BASE_MASK                                                                       0x0000FC00L
+#define SH_MEM_BASES__SHARED_BASE_MASK                                                                        0xFFFF0000L
+//SH_MEM_CONFIG
+#define SH_MEM_CONFIG__ADDRESS_MODE__SHIFT                                                                    0x0
+#define SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT                                                                  0x2
+#define SH_MEM_CONFIG__F8_MODE__SHIFT                                                                         0x8
+#define SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT                                                           0xe
+#define SH_MEM_CONFIG__ICACHE_USE_GL1__SHIFT                                                                  0x12
+#define SH_MEM_CONFIG__RETRY_DISABLE__SHIFT                                                                   0x13
+#define SH_MEM_CONFIG__RETRY_DEBUG__SHIFT                                                                     0x14
+#define SH_MEM_CONFIG__XCNT_DEBUG__SHIFT                                                                      0x15
+#define SH_MEM_CONFIG__ADDRESS_MODE_MASK                                                                      0x00000001L
+#define SH_MEM_CONFIG__ALIGNMENT_MODE_MASK                                                                    0x00000004L
+#define SH_MEM_CONFIG__F8_MODE_MASK                                                                           0x00000100L
+#define SH_MEM_CONFIG__INITIAL_INST_PREFETCH_MASK                                                             0x0000C000L
+#define SH_MEM_CONFIG__ICACHE_USE_GL1_MASK                                                                    0x00040000L
+#define SH_MEM_CONFIG__RETRY_DISABLE_MASK                                                                     0x00080000L
+#define SH_MEM_CONFIG__RETRY_DEBUG_MASK                                                                       0x00100000L
+#define SH_MEM_CONFIG__XCNT_DEBUG_MASK                                                                        0x00200000L
+//SQ_DEBUG
+#define SQ_DEBUG__SINGLE_MEMOP__SHIFT                                                                         0x0
+#define SQ_DEBUG__SINGLE_ALU_OP__SHIFT                                                                        0x1
+#define SQ_DEBUG__WAIT_DEP_CTR_ZERO__SHIFT                                                                    0x2
+#define SQ_DEBUG__SU_VDST_WKILL_DIS__SHIFT                                                                    0x3
+#define SQ_DEBUG__ADDR_OUT_OF_RANGE_REPORTING__SHIFT                                                          0x4
+#define SQ_DEBUG__DISABLE_VGPR_DEALLOC__SHIFT                                                                 0x5
+#define SQ_DEBUG__SINGLE_MEMOP_MASK                                                                           0x00000001L
+#define SQ_DEBUG__SINGLE_ALU_OP_MASK                                                                          0x00000002L
+#define SQ_DEBUG__WAIT_DEP_CTR_ZERO_MASK                                                                      0x00000004L
+#define SQ_DEBUG__SU_VDST_WKILL_DIS_MASK                                                                      0x00000008L
+#define SQ_DEBUG__ADDR_OUT_OF_RANGE_REPORTING_MASK                                                            0x00000010L
+#define SQ_DEBUG__DISABLE_VGPR_DEALLOC_MASK                                                                   0x00000020L
+//SQ_SHADER_TBA_LO
+#define SQ_SHADER_TBA_LO__ADDR_LO__SHIFT                                                                      0x0
+#define SQ_SHADER_TBA_LO__ADDR_LO_MASK                                                                        0xFFFFFFFFL
+//SQ_SHADER_TBA_HI
+#define SQ_SHADER_TBA_HI__ADDR_HI__SHIFT                                                                      0x0
+#define SQ_SHADER_TBA_HI__TRAP_EN__SHIFT                                                                      0x1f
+#define SQ_SHADER_TBA_HI__ADDR_HI_MASK                                                                        0x0001FFFFL
+#define SQ_SHADER_TBA_HI__TRAP_EN_MASK                                                                        0x80000000L
+//SQ_SHADER_TMA_LO
+#define SQ_SHADER_TMA_LO__ADDR_LO__SHIFT                                                                      0x0
+#define SQ_SHADER_TMA_LO__ADDR_LO_MASK                                                                        0xFFFFFFFFL
+//SQ_SHADER_TMA_HI
+#define SQ_SHADER_TMA_HI__ADDR_HI__SHIFT                                                                      0x0
+#define SQ_SHADER_TMA_HI__ADDR_HI_MASK                                                                        0x0001FFFFL
+//SQ_IND_INDEX_USER
+#define SQ_IND_INDEX_USER__WAVE_ID__SHIFT                                                                     0x0
+#define SQ_IND_INDEX_USER__WORKITEM_ID__SHIFT                                                                 0x5
+#define SQ_IND_INDEX_USER__AUTO_INCR__SHIFT                                                                   0xb
+#define SQ_IND_INDEX_USER__INDEX__SHIFT                                                                       0x10
+#define SQ_IND_INDEX_USER__WAVE_ID_MASK                                                                       0x0000001FL
+#define SQ_IND_INDEX_USER__WORKITEM_ID_MASK                                                                   0x000007E0L
+#define SQ_IND_INDEX_USER__AUTO_INCR_MASK                                                                     0x00000800L
+#define SQ_IND_INDEX_USER__INDEX_MASK                                                                         0xFFFF0000L
+//SQ_CMD_USER
+#define SQ_CMD_USER__CMD__SHIFT                                                                               0x0
+#define SQ_CMD_USER__MODE__SHIFT                                                                              0x4
+#define SQ_CMD_USER__CHECK_VMID__SHIFT                                                                        0x7
+#define SQ_CMD_USER__DATA__SHIFT                                                                              0x8
+#define SQ_CMD_USER__WAVE_ID__SHIFT                                                                           0x10
+#define SQ_CMD_USER__QUEUE_ID__SHIFT                                                                          0x18
+#define SQ_CMD_USER__VM_ID__SHIFT                                                                             0x1c
+#define SQ_CMD_USER__CMD_MASK                                                                                 0x0000000FL
+#define SQ_CMD_USER__MODE_MASK                                                                                0x00000070L
+#define SQ_CMD_USER__CHECK_VMID_MASK                                                                          0x00000080L
+#define SQ_CMD_USER__DATA_MASK                                                                                0x00000F00L
+#define SQ_CMD_USER__WAVE_ID_MASK                                                                             0x001F0000L
+#define SQ_CMD_USER__QUEUE_ID_MASK                                                                            0x07000000L
+#define SQ_CMD_USER__VM_ID_MASK                                                                               0xF0000000L
+//SQ_IND_DATA_USER
+#define SQ_IND_DATA_USER__DATA__SHIFT                                                                         0x0
+#define SQ_IND_DATA_USER__DATA_MASK                                                                           0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pfonly_spidec
+//SPI_CDBG_SYS_GFX
+#define SPI_CDBG_SYS_GFX__PS_EN__SHIFT                                                                        0x0
+#define SPI_CDBG_SYS_GFX__GS_EN__SHIFT                                                                        0x2
+#define SPI_CDBG_SYS_GFX__HS_EN__SHIFT                                                                        0x4
+#define SPI_CDBG_SYS_GFX__CS_EN__SHIFT                                                                        0x6
+#define SPI_CDBG_SYS_GFX__PS_EN_MASK                                                                          0x00000001L
+#define SPI_CDBG_SYS_GFX__GS_EN_MASK                                                                          0x00000004L
+#define SPI_CDBG_SYS_GFX__HS_EN_MASK                                                                          0x00000010L
+#define SPI_CDBG_SYS_GFX__CS_EN_MASK                                                                          0x00000040L
+//SPI_CDBG_SYS_HP3D
+#define SPI_CDBG_SYS_HP3D__PS_EN__SHIFT                                                                       0x0
+#define SPI_CDBG_SYS_HP3D__GS_EN__SHIFT                                                                       0x2
+#define SPI_CDBG_SYS_HP3D__HS_EN__SHIFT                                                                       0x4
+#define SPI_CDBG_SYS_HP3D__CS_EN__SHIFT                                                                       0x6
+#define SPI_CDBG_SYS_HP3D__PS_EN_MASK                                                                         0x00000001L
+#define SPI_CDBG_SYS_HP3D__GS_EN_MASK                                                                         0x00000004L
+#define SPI_CDBG_SYS_HP3D__HS_EN_MASK                                                                         0x00000010L
+#define SPI_CDBG_SYS_HP3D__CS_EN_MASK                                                                         0x00000040L
+//SPI_CDBG_SYS_CS0
+#define SPI_CDBG_SYS_CS0__PIPE0__SHIFT                                                                        0x0
+#define SPI_CDBG_SYS_CS0__PIPE1__SHIFT                                                                        0x8
+#define SPI_CDBG_SYS_CS0__PIPE2__SHIFT                                                                        0x10
+#define SPI_CDBG_SYS_CS0__PIPE3__SHIFT                                                                        0x18
+#define SPI_CDBG_SYS_CS0__PIPE0_MASK                                                                          0x000000FFL
+#define SPI_CDBG_SYS_CS0__PIPE1_MASK                                                                          0x0000FF00L
+#define SPI_CDBG_SYS_CS0__PIPE2_MASK                                                                          0x00FF0000L
+#define SPI_CDBG_SYS_CS0__PIPE3_MASK                                                                          0xFF000000L
+//SPI_GDBG_WAVE_CNTL
+#define SPI_GDBG_WAVE_CNTL__STALL_RA__SHIFT                                                                   0x0
+#define SPI_GDBG_WAVE_CNTL__STALL_LAUNCH__SHIFT                                                               0x1
+#define SPI_GDBG_WAVE_CNTL__STALL_STATUS__SHIFT                                                               0x2
+#define SPI_GDBG_WAVE_CNTL__STALL_RA_MASK                                                                     0x00000001L
+#define SPI_GDBG_WAVE_CNTL__STALL_LAUNCH_MASK                                                                 0x00000002L
+#define SPI_GDBG_WAVE_CNTL__STALL_STATUS_MASK                                                                 0x00000004L
+//SPI_GDBG_TRAP_CONFIG
+#define SPI_GDBG_TRAP_CONFIG__PIPE0_EN__SHIFT                                                                 0x0
+#define SPI_GDBG_TRAP_CONFIG__PIPE1_EN__SHIFT                                                                 0x8
+#define SPI_GDBG_TRAP_CONFIG__PIPE2_EN__SHIFT                                                                 0x10
+#define SPI_GDBG_TRAP_CONFIG__PIPE3_EN__SHIFT                                                                 0x18
+#define SPI_GDBG_TRAP_CONFIG__PIPE0_EN_MASK                                                                   0x000000FFL
+#define SPI_GDBG_TRAP_CONFIG__PIPE1_EN_MASK                                                                   0x0000FF00L
+#define SPI_GDBG_TRAP_CONFIG__PIPE2_EN_MASK                                                                   0x00FF0000L
+#define SPI_GDBG_TRAP_CONFIG__PIPE3_EN_MASK                                                                   0xFF000000L
+//SPI_GDBG_WAVE_CNTL3
+#define SPI_GDBG_WAVE_CNTL3__STALL_PS__SHIFT                                                                  0x0
+#define SPI_GDBG_WAVE_CNTL3__STALL_GS__SHIFT                                                                  0x2
+#define SPI_GDBG_WAVE_CNTL3__STALL_HS__SHIFT                                                                  0x3
+#define SPI_GDBG_WAVE_CNTL3__STALL_CSG__SHIFT                                                                 0x4
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS0__SHIFT                                                                 0x5
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS1__SHIFT                                                                 0x6
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS2__SHIFT                                                                 0x7
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS3__SHIFT                                                                 0x8
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS4__SHIFT                                                                 0x9
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS5__SHIFT                                                                 0xa
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS6__SHIFT                                                                 0xb
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS7__SHIFT                                                                 0xc
+#define SPI_GDBG_WAVE_CNTL3__STALL_DURATION__SHIFT                                                            0xd
+#define SPI_GDBG_WAVE_CNTL3__STALL_MULT__SHIFT                                                                0x1c
+#define SPI_GDBG_WAVE_CNTL3__STALL_PS_MASK                                                                    0x00000001L
+#define SPI_GDBG_WAVE_CNTL3__STALL_GS_MASK                                                                    0x00000004L
+#define SPI_GDBG_WAVE_CNTL3__STALL_HS_MASK                                                                    0x00000008L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CSG_MASK                                                                   0x00000010L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS0_MASK                                                                   0x00000020L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS1_MASK                                                                   0x00000040L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS2_MASK                                                                   0x00000080L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS3_MASK                                                                   0x00000100L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS4_MASK                                                                   0x00000200L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS5_MASK                                                                   0x00000400L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS6_MASK                                                                   0x00000800L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS7_MASK                                                                   0x00001000L
+#define SPI_GDBG_WAVE_CNTL3__STALL_DURATION_MASK                                                              0x0FFFE000L
+#define SPI_GDBG_WAVE_CNTL3__STALL_MULT_MASK                                                                  0x10000000L
+//SPI_RESET_DEBUG
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET__SHIFT                                                             0x0
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PER_VMID__SHIFT                                                    0x1
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_ALL_VMID__SHIFT                                                    0x2
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_RESOURCE__SHIFT                                                    0x3
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PRIORITY__SHIFT                                                    0x4
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_MASK                                                               0x00000001L
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PER_VMID_MASK                                                      0x00000002L
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_ALL_VMID_MASK                                                      0x00000004L
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_RESOURCE_MASK                                                      0x00000008L
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PRIORITY_MASK                                                      0x00000010L
+//GDS_COMPUTE_MAX_WAVE_ID
+#define GDS_COMPUTE_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT                                                           0x0
+#define GDS_COMPUTE_MAX_WAVE_ID__MAX_WAVE_ID_MASK                                                             0x00000FFFL
+//SPI_ARB_CNTL_0
+#define SPI_ARB_CNTL_0__EXP_ARB_COL_WT__SHIFT                                                                 0x0
+#define SPI_ARB_CNTL_0__EXP_ARB_POS_WT__SHIFT                                                                 0x4
+#define SPI_ARB_CNTL_0__EXP_ARB_GDS_WT__SHIFT                                                                 0x8
+#define SPI_ARB_CNTL_0__EXP_ARB_COL_WT_MASK                                                                   0x0000000FL
+#define SPI_ARB_CNTL_0__EXP_ARB_POS_WT_MASK                                                                   0x000000F0L
+#define SPI_ARB_CNTL_0__EXP_ARB_GDS_WT_MASK                                                                   0x00000F00L
+//SPI_FEATURE_CTRL
+#define SPI_FEATURE_CTRL__TUNNELING_WAVE_LIMIT__SHIFT                                                         0x0
+#define SPI_FEATURE_CTRL__RA_PROBE_IGNORE__SHIFT                                                              0x4
+#define SPI_FEATURE_CTRL__PS_THROTTLE_MAX_WAVE_LIMIT__SHIFT                                                   0x5
+#define SPI_FEATURE_CTRL__RA_PROBE_SKEW_WIF_CTRL__SHIFT                                                       0xb
+#define SPI_FEATURE_CTRL__RA_PROBE_SKEW_OOO_CTRL__SHIFT                                                       0xd
+#define SPI_FEATURE_CTRL__RA_PROBE_SKEW_DISABLE__SHIFT                                                        0xe
+#define SPI_FEATURE_CTRL__TUNNELING_WAVE_LIMIT_MASK                                                           0x0000000FL
+#define SPI_FEATURE_CTRL__RA_PROBE_IGNORE_MASK                                                                0x00000010L
+#define SPI_FEATURE_CTRL__PS_THROTTLE_MAX_WAVE_LIMIT_MASK                                                     0x000007E0L
+#define SPI_FEATURE_CTRL__RA_PROBE_SKEW_WIF_CTRL_MASK                                                         0x00001800L
+#define SPI_FEATURE_CTRL__RA_PROBE_SKEW_OOO_CTRL_MASK                                                         0x00002000L
+#define SPI_FEATURE_CTRL__RA_PROBE_SKEW_DISABLE_MASK                                                          0x00004000L
+//SPI_SHADER_RSRC_LIMIT_CTRL
+#define SPI_SHADER_RSRC_LIMIT_CTRL__WAVES_PER_SIMD32__SHIFT                                                   0x0
+#define SPI_SHADER_RSRC_LIMIT_CTRL__VGPR_PER_SIMD32__SHIFT                                                    0x5
+#define SPI_SHADER_RSRC_LIMIT_CTRL__VGPR_WRAP_DISABLE__SHIFT                                                  0xc
+#define SPI_SHADER_RSRC_LIMIT_CTRL__BARRIER_LIMIT__SHIFT                                                      0xd
+#define SPI_SHADER_RSRC_LIMIT_CTRL__BARRIER_LIMIT_HIERARCHY_LEVEL__SHIFT                                      0x13
+#define SPI_SHADER_RSRC_LIMIT_CTRL__NAMED_BARRIER_LIMIT__SHIFT                                                0x14
+#define SPI_SHADER_RSRC_LIMIT_CTRL__CLUSTER_LIMIT__SHIFT                                                      0x1c
+#define SPI_SHADER_RSRC_LIMIT_CTRL__PERFORMANCE_LIMIT_ENABLE__SHIFT                                           0x1f
+#define SPI_SHADER_RSRC_LIMIT_CTRL__WAVES_PER_SIMD32_MASK                                                     0x0000001FL
+#define SPI_SHADER_RSRC_LIMIT_CTRL__VGPR_PER_SIMD32_MASK                                                      0x00000FE0L
+#define SPI_SHADER_RSRC_LIMIT_CTRL__VGPR_WRAP_DISABLE_MASK                                                    0x00001000L
+#define SPI_SHADER_RSRC_LIMIT_CTRL__BARRIER_LIMIT_MASK                                                        0x0007E000L
+#define SPI_SHADER_RSRC_LIMIT_CTRL__BARRIER_LIMIT_HIERARCHY_LEVEL_MASK                                        0x00080000L
+#define SPI_SHADER_RSRC_LIMIT_CTRL__NAMED_BARRIER_LIMIT_MASK                                                  0x00F00000L
+#define SPI_SHADER_RSRC_LIMIT_CTRL__CLUSTER_LIMIT_MASK                                                        0x70000000L
+#define SPI_SHADER_RSRC_LIMIT_CTRL__PERFORMANCE_LIMIT_ENABLE_MASK                                             0x80000000L
+//PC_CONFIG_CNTL_0
+#define PC_CONFIG_CNTL_0__PQ_FIFO_DEPTH__SHIFT                                                                0x0
+#define PC_CONFIG_CNTL_0__READ_RET_DEPTH__SHIFT                                                               0x5
+#define PC_CONFIG_CNTL_0__MAX_PRIMS_PER_PROBE__SHIFT                                                          0xa
+#define PC_CONFIG_CNTL_0__GL1_CREDIT_COUNT__SHIFT                                                             0xe
+#define PC_CONFIG_CNTL_0__SC_PC_RATE_CNTL__SHIFT                                                              0x12
+#define PC_CONFIG_CNTL_0__MW_PQ_RATE_CNTL__SHIFT                                                              0x16
+#define PC_CONFIG_CNTL_0__PC_DEALLOC_TIMEOUT__SHIFT                                                           0x1a
+#define PC_CONFIG_CNTL_0__MW_DISABLE_EARLY_HIT__SHIFT                                                         0x1e
+#define PC_CONFIG_CNTL_0__DISABLE_DEALLOC_ON_TIMEOUT__SHIFT                                                   0x1f
+#define PC_CONFIG_CNTL_0__PQ_FIFO_DEPTH_MASK                                                                  0x0000001FL
+#define PC_CONFIG_CNTL_0__READ_RET_DEPTH_MASK                                                                 0x000003E0L
+#define PC_CONFIG_CNTL_0__MAX_PRIMS_PER_PROBE_MASK                                                            0x00003C00L
+#define PC_CONFIG_CNTL_0__GL1_CREDIT_COUNT_MASK                                                               0x0003C000L
+#define PC_CONFIG_CNTL_0__SC_PC_RATE_CNTL_MASK                                                                0x003C0000L
+#define PC_CONFIG_CNTL_0__MW_PQ_RATE_CNTL_MASK                                                                0x03C00000L
+#define PC_CONFIG_CNTL_0__PC_DEALLOC_TIMEOUT_MASK                                                             0x3C000000L
+#define PC_CONFIG_CNTL_0__MW_DISABLE_EARLY_HIT_MASK                                                           0x40000000L
+#define PC_CONFIG_CNTL_0__DISABLE_DEALLOC_ON_TIMEOUT_MASK                                                     0x80000000L
+//PC_CONFIG_CNTL_1
+#define PC_CONFIG_CNTL_1__DISABLE_LWC_SLOT_REUSE__SHIFT                                                       0x0
+#define PC_CONFIG_CNTL_1__DISABLE_LWC_WAVE_REUSE__SHIFT                                                       0x1
+#define PC_CONFIG_CNTL_1__LIMIT_BANK_ACCESS__SHIFT                                                            0x2
+#define PC_CONFIG_CNTL_1__FORCE_BANK_SERIALIZE__SHIFT                                                         0x3
+#define PC_CONFIG_CNTL_1__ENABLE_FIFO_DEBUG_WRITE__SHIFT                                                      0x4
+#define PC_CONFIG_CNTL_1__DEBUG_REG_EN__SHIFT                                                                 0x5
+#define PC_CONFIG_CNTL_1__DEBUG_GROUP_SEL__SHIFT                                                              0x6
+#define PC_CONFIG_CNTL_1__FORCE_SA_SERIALIZE__SHIFT                                                           0xc
+#define PC_CONFIG_CNTL_1__PC_GL1H_FGCG_OVERRIDE__SHIFT                                                        0xd
+#define PC_CONFIG_CNTL_1__PC_LDS_FGCG_OVERRIDE__SHIFT                                                         0xe
+#define PC_CONFIG_CNTL_1__PC_MAX_BCD__SHIFT                                                                   0xf
+#define PC_CONFIG_CNTL_1__DISABLE_DEALLOC_ON_EVENT__SHIFT                                                     0x11
+#define PC_CONFIG_CNTL_1__MAX_PSTATE_IDS__SHIFT                                                               0x12
+#define PC_CONFIG_CNTL_1__MAX_PC_SPI_PROBES__SHIFT                                                            0x14
+#define PC_CONFIG_CNTL_1__CMM_USE_POLICY__SHIFT                                                               0x18
+#define PC_CONFIG_CNTL_1__SPECULATIVE_DATA_READ__SHIFT                                                        0x1b
+#define PC_CONFIG_CNTL_1__CMM_SCOPE__SHIFT                                                                    0x1d
+#define PC_CONFIG_CNTL_1__DISABLE_DEALLOC_ON_PC_IDLE__SHIFT                                                   0x1f
+#define PC_CONFIG_CNTL_1__DISABLE_LWC_SLOT_REUSE_MASK                                                         0x00000001L
+#define PC_CONFIG_CNTL_1__DISABLE_LWC_WAVE_REUSE_MASK                                                         0x00000002L
+#define PC_CONFIG_CNTL_1__LIMIT_BANK_ACCESS_MASK                                                              0x00000004L
+#define PC_CONFIG_CNTL_1__FORCE_BANK_SERIALIZE_MASK                                                           0x00000008L
+#define PC_CONFIG_CNTL_1__ENABLE_FIFO_DEBUG_WRITE_MASK                                                        0x00000010L
+#define PC_CONFIG_CNTL_1__DEBUG_REG_EN_MASK                                                                   0x00000020L
+#define PC_CONFIG_CNTL_1__DEBUG_GROUP_SEL_MASK                                                                0x00000FC0L
+#define PC_CONFIG_CNTL_1__FORCE_SA_SERIALIZE_MASK                                                             0x00001000L
+#define PC_CONFIG_CNTL_1__PC_GL1H_FGCG_OVERRIDE_MASK                                                          0x00002000L
+#define PC_CONFIG_CNTL_1__PC_LDS_FGCG_OVERRIDE_MASK                                                           0x00004000L
+#define PC_CONFIG_CNTL_1__PC_MAX_BCD_MASK                                                                     0x00018000L
+#define PC_CONFIG_CNTL_1__DISABLE_DEALLOC_ON_EVENT_MASK                                                       0x00020000L
+#define PC_CONFIG_CNTL_1__MAX_PSTATE_IDS_MASK                                                                 0x000C0000L
+#define PC_CONFIG_CNTL_1__MAX_PC_SPI_PROBES_MASK                                                              0x00F00000L
+#define PC_CONFIG_CNTL_1__CMM_USE_POLICY_MASK                                                                 0x07000000L
+#define PC_CONFIG_CNTL_1__SPECULATIVE_DATA_READ_MASK                                                          0x18000000L
+#define PC_CONFIG_CNTL_1__CMM_SCOPE_MASK                                                                      0x60000000L
+#define PC_CONFIG_CNTL_1__DISABLE_DEALLOC_ON_PC_IDLE_MASK                                                     0x80000000L
+//SPI_COMPUTE_WF_CTX_SAVE_STATUS
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE0_SAVE_BUSY__SHIFT                                         0x0
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE1_SAVE_BUSY__SHIFT                                         0x1
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE2_SAVE_BUSY__SHIFT                                         0x2
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE3_SAVE_BUSY__SHIFT                                         0x3
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE4_SAVE_BUSY__SHIFT                                         0x4
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE5_SAVE_BUSY__SHIFT                                         0x5
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE6_SAVE_BUSY__SHIFT                                         0x6
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE7_SAVE_BUSY__SHIFT                                         0x7
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE0_SAVE_BUSY__SHIFT                                         0x8
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE1_SAVE_BUSY__SHIFT                                         0x9
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE2_SAVE_BUSY__SHIFT                                         0xa
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE3_SAVE_BUSY__SHIFT                                         0xb
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE4_SAVE_BUSY__SHIFT                                         0xc
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE5_SAVE_BUSY__SHIFT                                         0xd
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE6_SAVE_BUSY__SHIFT                                         0xe
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE7_SAVE_BUSY__SHIFT                                         0xf
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE0_SAVE_BUSY__SHIFT                                         0x10
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE1_SAVE_BUSY__SHIFT                                         0x11
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE2_SAVE_BUSY__SHIFT                                         0x12
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE3_SAVE_BUSY__SHIFT                                         0x13
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE4_SAVE_BUSY__SHIFT                                         0x14
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE5_SAVE_BUSY__SHIFT                                         0x15
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE6_SAVE_BUSY__SHIFT                                         0x16
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE7_SAVE_BUSY__SHIFT                                         0x17
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE0_SAVE_BUSY__SHIFT                                         0x18
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE1_SAVE_BUSY__SHIFT                                         0x19
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE2_SAVE_BUSY__SHIFT                                         0x1a
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE3_SAVE_BUSY__SHIFT                                         0x1b
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE4_SAVE_BUSY__SHIFT                                         0x1c
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE5_SAVE_BUSY__SHIFT                                         0x1d
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE6_SAVE_BUSY__SHIFT                                         0x1e
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE7_SAVE_BUSY__SHIFT                                         0x1f
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE0_SAVE_BUSY_MASK                                           0x00000001L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE1_SAVE_BUSY_MASK                                           0x00000002L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE2_SAVE_BUSY_MASK                                           0x00000004L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE3_SAVE_BUSY_MASK                                           0x00000008L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE4_SAVE_BUSY_MASK                                           0x00000010L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE5_SAVE_BUSY_MASK                                           0x00000020L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE6_SAVE_BUSY_MASK                                           0x00000040L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE7_SAVE_BUSY_MASK                                           0x00000080L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE0_SAVE_BUSY_MASK                                           0x00000100L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE1_SAVE_BUSY_MASK                                           0x00000200L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE2_SAVE_BUSY_MASK                                           0x00000400L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE3_SAVE_BUSY_MASK                                           0x00000800L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE4_SAVE_BUSY_MASK                                           0x00001000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE5_SAVE_BUSY_MASK                                           0x00002000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE6_SAVE_BUSY_MASK                                           0x00004000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE7_SAVE_BUSY_MASK                                           0x00008000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE0_SAVE_BUSY_MASK                                           0x00010000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE1_SAVE_BUSY_MASK                                           0x00020000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE2_SAVE_BUSY_MASK                                           0x00040000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE3_SAVE_BUSY_MASK                                           0x00080000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE4_SAVE_BUSY_MASK                                           0x00100000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE5_SAVE_BUSY_MASK                                           0x00200000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE6_SAVE_BUSY_MASK                                           0x00400000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE7_SAVE_BUSY_MASK                                           0x00800000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE0_SAVE_BUSY_MASK                                           0x01000000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE1_SAVE_BUSY_MASK                                           0x02000000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE2_SAVE_BUSY_MASK                                           0x04000000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE3_SAVE_BUSY_MASK                                           0x08000000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE4_SAVE_BUSY_MASK                                           0x10000000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE5_SAVE_BUSY_MASK                                           0x20000000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE6_SAVE_BUSY_MASK                                           0x40000000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE7_SAVE_BUSY_MASK                                           0x80000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pfonly_utcl1dec
+//UTCL1_CTRL_0
+#define UTCL1_CTRL_0__UTCL1_L0_REQ_VFIFO_DISABLE__SHIFT                                                       0x0
+#define UTCL1_CTRL_0__UTCL1_UTCL2_INVACK_CDC_FIFO_DISABLE__SHIFT                                              0x1
+#define UTCL1_CTRL_0__UTCL1_MH_B2B_DUPLICATES_DET_DISABLE__SHIFT                                              0x2
+#define UTCL1_CTRL_0__UTCL1_UTCL2_REQ_CREDITS_UNIFIED_IF__SHIFT                                               0x3
+#define UTCL1_CTRL_0__UTCL1_UTCL0_INVREQ_CREDITS__SHIFT                                                       0x9
+#define UTCL1_CTRL_0__UTCL1_LIMIT_INV_TO_ONE__SHIFT                                                           0xd
+#define UTCL1_CTRL_0__UTCL1_LIMIT_XLAT_TO_ONE__SHIFT                                                          0xe
+#define UTCL1_CTRL_0__UTCL1_UTCL2_FGCG_REPEATERS_OVERRIDE__SHIFT                                              0xf
+#define UTCL1_CTRL_0__UTCL1_INV_FILTER_VMID__SHIFT                                                            0x10
+#define UTCL1_CTRL_0__UTCL1_RANGE_INV_FORCE_CHK_ALL__SHIFT                                                    0x11
+#define UTCL1_CTRL_0__UTCL1_UTCL0_RET_FGCG_REPEATERS_OVERRIDE__SHIFT                                          0x12
+#define UTCL1_CTRL_0__UTCL1_UTCL0_INVREQ_FGCG_REPEATERS_OVERRIDE__SHIFT                                       0x13
+#define UTCL1_CTRL_0__GCRD_FGCG_DISABLE__SHIFT                                                                0x14
+#define UTCL1_CTRL_0__UTCL1_MH_RANGE_INV_TO_VMID_OVERRIDE__SHIFT                                              0x15
+#define UTCL1_CTRL_0__UTCL1_MH_DISABLE_DUPLICATES__SHIFT                                                      0x16
+#define UTCL1_CTRL_0__UTCL1_MH_DISABLE_REQUEST_SQUASHING__SHIFT                                               0x17
+#define UTCL1_CTRL_0__UTCL1_MH_DISABLE_RECENT_BUFFER__SHIFT                                                   0x18
+#define UTCL1_CTRL_0__UTCL1_XLAT_FAULT_LOCK_CTRL__SHIFT                                                       0x19
+#define UTCL1_CTRL_0__UTCL1_REDUCE_CC_SIZE__SHIFT                                                             0x1b
+#define UTCL1_CTRL_0__UTCL1_HIT_DETECTION_BASED_ON_PAGE_SIZE__SHIFT                                           0x1d
+#define UTCL1_CTRL_0__UTCL1_MH_DUPL_DETECT_ON_NATIVE_PG_SZ__SHIFT                                             0x1e
+#define UTCL1_CTRL_0__UTCL1_FORCE_INV_ALL__SHIFT                                                              0x1f
+#define UTCL1_CTRL_0__UTCL1_L0_REQ_VFIFO_DISABLE_MASK                                                         0x00000001L
+#define UTCL1_CTRL_0__UTCL1_UTCL2_INVACK_CDC_FIFO_DISABLE_MASK                                                0x00000002L
+#define UTCL1_CTRL_0__UTCL1_MH_B2B_DUPLICATES_DET_DISABLE_MASK                                                0x00000004L
+#define UTCL1_CTRL_0__UTCL1_UTCL2_REQ_CREDITS_UNIFIED_IF_MASK                                                 0x000001F8L
+#define UTCL1_CTRL_0__UTCL1_UTCL0_INVREQ_CREDITS_MASK                                                         0x00001E00L
+#define UTCL1_CTRL_0__UTCL1_LIMIT_INV_TO_ONE_MASK                                                             0x00002000L
+#define UTCL1_CTRL_0__UTCL1_LIMIT_XLAT_TO_ONE_MASK                                                            0x00004000L
+#define UTCL1_CTRL_0__UTCL1_UTCL2_FGCG_REPEATERS_OVERRIDE_MASK                                                0x00008000L
+#define UTCL1_CTRL_0__UTCL1_INV_FILTER_VMID_MASK                                                              0x00010000L
+#define UTCL1_CTRL_0__UTCL1_RANGE_INV_FORCE_CHK_ALL_MASK                                                      0x00020000L
+#define UTCL1_CTRL_0__UTCL1_UTCL0_RET_FGCG_REPEATERS_OVERRIDE_MASK                                            0x00040000L
+#define UTCL1_CTRL_0__UTCL1_UTCL0_INVREQ_FGCG_REPEATERS_OVERRIDE_MASK                                         0x00080000L
+#define UTCL1_CTRL_0__GCRD_FGCG_DISABLE_MASK                                                                  0x00100000L
+#define UTCL1_CTRL_0__UTCL1_MH_RANGE_INV_TO_VMID_OVERRIDE_MASK                                                0x00200000L
+#define UTCL1_CTRL_0__UTCL1_MH_DISABLE_DUPLICATES_MASK                                                        0x00400000L
+#define UTCL1_CTRL_0__UTCL1_MH_DISABLE_REQUEST_SQUASHING_MASK                                                 0x00800000L
+#define UTCL1_CTRL_0__UTCL1_MH_DISABLE_RECENT_BUFFER_MASK                                                     0x01000000L
+#define UTCL1_CTRL_0__UTCL1_XLAT_FAULT_LOCK_CTRL_MASK                                                         0x06000000L
+#define UTCL1_CTRL_0__UTCL1_REDUCE_CC_SIZE_MASK                                                               0x18000000L
+#define UTCL1_CTRL_0__UTCL1_HIT_DETECTION_BASED_ON_PAGE_SIZE_MASK                                             0x20000000L
+#define UTCL1_CTRL_0__UTCL1_MH_DUPL_DETECT_ON_NATIVE_PG_SZ_MASK                                               0x40000000L
+#define UTCL1_CTRL_0__UTCL1_FORCE_INV_ALL_MASK                                                                0x80000000L
+//UTCL1_UTCL0_INVREQ_DISABLE
+#define UTCL1_UTCL0_INVREQ_DISABLE__UTCL1_UTCL0_INVREQ_DISABLE__SHIFT                                         0x0
+#define UTCL1_UTCL0_INVREQ_DISABLE__UTCL1_UTCL0_INVREQ_DISABLE_MASK                                           0xFFFFFFFFL
+//UTCL1_CTRL_2
+#define UTCL1_CTRL_2__UTCL1_RNG_TO_VMID_INV_OVRD__SHIFT                                                       0x0
+#define UTCL1_CTRL_2__UTCL1_PMM_INTERRUPT_CREDITS_OVERRIDE__SHIFT                                             0x4
+#define UTCL1_CTRL_2__UTCL1_CACHE_WRITE_PERM__SHIFT                                                           0xa
+#define UTCL1_CTRL_2__UTCL1_PAGE_OVRD_DISABLE__SHIFT                                                          0xb
+#define UTCL1_CTRL_2__UTCL1_SPARE0__SHIFT                                                                     0xc
+#define UTCL1_CTRL_2__UTCL2_UTCL1_RET_FGCG_REPEATERS_OVERRIDE__SHIFT                                          0xd
+#define UTCL1_CTRL_2__UTCL1_IDENTITY_MODE_CONFIG_SELECTOR__SHIFT                                              0xe
+#define UTCL1_CTRL_2__UTCL1_UTCL2_REQ_CREDITS_SA01__SHIFT                                                     0x14
+#define UTCL1_CTRL_2__UTCL1_UTCL2_REQ_CREDITS_SAX__SHIFT                                                      0x1a
+#define UTCL1_CTRL_2__UTCL1_RNG_TO_VMID_INV_OVRD_MASK                                                         0x0000000FL
+#define UTCL1_CTRL_2__UTCL1_PMM_INTERRUPT_CREDITS_OVERRIDE_MASK                                               0x000003F0L
+#define UTCL1_CTRL_2__UTCL1_CACHE_WRITE_PERM_MASK                                                             0x00000400L
+#define UTCL1_CTRL_2__UTCL1_PAGE_OVRD_DISABLE_MASK                                                            0x00000800L
+#define UTCL1_CTRL_2__UTCL1_SPARE0_MASK                                                                       0x00001000L
+#define UTCL1_CTRL_2__UTCL2_UTCL1_RET_FGCG_REPEATERS_OVERRIDE_MASK                                            0x00002000L
+#define UTCL1_CTRL_2__UTCL1_IDENTITY_MODE_CONFIG_SELECTOR_MASK                                                0x000FC000L
+#define UTCL1_CTRL_2__UTCL1_UTCL2_REQ_CREDITS_SA01_MASK                                                       0x03F00000L
+#define UTCL1_CTRL_2__UTCL1_UTCL2_REQ_CREDITS_SAX_MASK                                                        0xFC000000L
+//UTCL1_FIFO_SIZING
+#define UTCL1_FIFO_SIZING__UTCL1_UTCL2_INVACK_CDC_FIFO_THRESH__SHIFT                                          0x0
+#define UTCL1_FIFO_SIZING__UTCL1_GENERAL_SIZING_CTRL_LOW__SHIFT                                               0x3
+#define UTCL1_FIFO_SIZING__UTCL1_GENERAL_SIZING_CTRL_HIGH__SHIFT                                              0x10
+#define UTCL1_FIFO_SIZING__UTCL1_UTCL2_INVACK_CDC_FIFO_THRESH_MASK                                            0x00000007L
+#define UTCL1_FIFO_SIZING__UTCL1_GENERAL_SIZING_CTRL_LOW_MASK                                                 0x0000FFF8L
+#define UTCL1_FIFO_SIZING__UTCL1_GENERAL_SIZING_CTRL_HIGH_MASK                                                0xFFFF0000L
+//GCRD_SA0_TARGETS_DISABLE
+#define GCRD_SA0_TARGETS_DISABLE__GCRD_SA0_TARGETS_DISABLE__SHIFT                                             0x0
+#define GCRD_SA0_TARGETS_DISABLE__GCRD_SA0_TARGETS_DISABLE_MASK                                               0x00FFFFFFL
+//GCRD_SA1_TARGETS_DISABLE
+#define GCRD_SA1_TARGETS_DISABLE__GCRD_SA1_TARGETS_DISABLE__SHIFT                                             0x0
+#define GCRD_SA1_TARGETS_DISABLE__GCRD_SA1_TARGETS_DISABLE_MASK                                               0x00FFFFFFL
+//GCRD_CREDIT_SAFE
+#define GCRD_CREDIT_SAFE__GCRD_CHAIN_CREDIT_SAFE_REG__SHIFT                                                   0x0
+#define GCRD_CREDIT_SAFE__GCRD_TARGET_CREDIT_SAFE_REG__SHIFT                                                  0x4
+#define GCRD_CREDIT_SAFE__GCRD_RSP_CREDIT_SAFE_REG__SHIFT                                                     0x8
+#define GCRD_CREDIT_SAFE__GCRD_CHAIN_CREDIT_SAFE_REG_MASK                                                     0x00000007L
+#define GCRD_CREDIT_SAFE__GCRD_TARGET_CREDIT_SAFE_REG_MASK                                                    0x00000070L
+#define GCRD_CREDIT_SAFE__GCRD_RSP_CREDIT_SAFE_REG_MASK                                                       0x00000F00L
+//UTCL1_IDENTITY_MODE0
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_SNOOP__SHIFT                                                    0x0
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_FRAG_SIZE__SHIFT                                                0x1
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_PERM_GRANTED__SHIFT                                             0x7
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_PERM_FAULT__SHIFT                                               0xb
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_XNACK__SHIFT                                                    0xc
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_PTE_TMZ__SHIFT                                                  0xe
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_NO_PTE__SHIFT                                                   0xf
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_SPA__SHIFT                                                      0x10
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_IOSTEER__SHIFT                                                  0x11
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_MTYPE__SHIFT                                                    0x12
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_D__SHIFT                                                        0x14
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_PREFETCH_CODE__SHIFT                                            0x15
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_TEE__SHIFT                                                      0x17
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_HDM__SHIFT                                                      0x18
+#define UTCL1_IDENTITY_MODE0__RESERVED__SHIFT                                                                 0x19
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_SNOOP_MASK                                                      0x00000001L
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_FRAG_SIZE_MASK                                                  0x0000007EL
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_PERM_GRANTED_MASK                                               0x00000780L
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_PERM_FAULT_MASK                                                 0x00000800L
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_XNACK_MASK                                                      0x00003000L
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_PTE_TMZ_MASK                                                    0x00004000L
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_NO_PTE_MASK                                                     0x00008000L
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_SPA_MASK                                                        0x00010000L
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_IOSTEER_MASK                                                    0x00020000L
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_MTYPE_MASK                                                      0x000C0000L
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_D_MASK                                                          0x00100000L
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_PREFETCH_CODE_MASK                                              0x00600000L
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_TEE_MASK                                                        0x00800000L
+#define UTCL1_IDENTITY_MODE0__UTCL1_UTCL0_RET_HDM_MASK                                                        0x01000000L
+#define UTCL1_IDENTITY_MODE0__RESERVED_MASK                                                                   0xFE000000L
+//UTCL1_IDENTITY_MODE1
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_SNOOP__SHIFT                                                    0x0
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_FRAG_SIZE__SHIFT                                                0x1
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_PERM_GRANTED__SHIFT                                             0x7
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_PERM_FAULT__SHIFT                                               0xb
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_XNACK__SHIFT                                                    0xc
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_PTE_TMZ__SHIFT                                                  0xe
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_NO_PTE__SHIFT                                                   0xf
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_SPA__SHIFT                                                      0x10
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_IOSTEER__SHIFT                                                  0x11
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_MTYPE__SHIFT                                                    0x12
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_D__SHIFT                                                        0x14
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_PREFETCH_CODE__SHIFT                                            0x15
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_TEE__SHIFT                                                      0x17
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_HDM__SHIFT                                                      0x18
+#define UTCL1_IDENTITY_MODE1__RESERVED__SHIFT                                                                 0x19
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_SNOOP_MASK                                                      0x00000001L
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_FRAG_SIZE_MASK                                                  0x0000007EL
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_PERM_GRANTED_MASK                                               0x00000780L
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_PERM_FAULT_MASK                                                 0x00000800L
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_XNACK_MASK                                                      0x00003000L
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_PTE_TMZ_MASK                                                    0x00004000L
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_NO_PTE_MASK                                                     0x00008000L
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_SPA_MASK                                                        0x00010000L
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_IOSTEER_MASK                                                    0x00020000L
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_MTYPE_MASK                                                      0x000C0000L
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_D_MASK                                                          0x00100000L
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_PREFETCH_CODE_MASK                                              0x00600000L
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_TEE_MASK                                                        0x00800000L
+#define UTCL1_IDENTITY_MODE1__UTCL1_UTCL0_RET_HDM_MASK                                                        0x01000000L
+#define UTCL1_IDENTITY_MODE1__RESERVED_MASK                                                                   0xFE000000L
+//UTCL1_IDENTITY_MODE2
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_SNOOP__SHIFT                                                    0x0
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_FRAG_SIZE__SHIFT                                                0x1
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_PERM_GRANTED__SHIFT                                             0x7
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_PERM_FAULT__SHIFT                                               0xb
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_XNACK__SHIFT                                                    0xc
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_PTE_TMZ__SHIFT                                                  0xe
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_NO_PTE__SHIFT                                                   0xf
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_SPA__SHIFT                                                      0x10
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_IOSTEER__SHIFT                                                  0x11
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_MTYPE__SHIFT                                                    0x12
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_D__SHIFT                                                        0x14
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_PREFETCH_CODE__SHIFT                                            0x15
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_TEE__SHIFT                                                      0x17
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_HDM__SHIFT                                                      0x18
+#define UTCL1_IDENTITY_MODE2__RESERVED__SHIFT                                                                 0x19
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_SNOOP_MASK                                                      0x00000001L
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_FRAG_SIZE_MASK                                                  0x0000007EL
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_PERM_GRANTED_MASK                                               0x00000780L
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_PERM_FAULT_MASK                                                 0x00000800L
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_XNACK_MASK                                                      0x00003000L
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_PTE_TMZ_MASK                                                    0x00004000L
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_NO_PTE_MASK                                                     0x00008000L
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_SPA_MASK                                                        0x00010000L
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_IOSTEER_MASK                                                    0x00020000L
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_MTYPE_MASK                                                      0x000C0000L
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_D_MASK                                                          0x00100000L
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_PREFETCH_CODE_MASK                                              0x00600000L
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_TEE_MASK                                                        0x00800000L
+#define UTCL1_IDENTITY_MODE2__UTCL1_UTCL0_RET_HDM_MASK                                                        0x01000000L
+#define UTCL1_IDENTITY_MODE2__RESERVED_MASK                                                                   0xFE000000L
+//UTCL1_IDENTITY_MODE3
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_SNOOP__SHIFT                                                    0x0
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_FRAG_SIZE__SHIFT                                                0x1
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_PERM_GRANTED__SHIFT                                             0x7
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_PERM_FAULT__SHIFT                                               0xb
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_XNACK__SHIFT                                                    0xc
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_PTE_TMZ__SHIFT                                                  0xe
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_NO_PTE__SHIFT                                                   0xf
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_SPA__SHIFT                                                      0x10
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_IOSTEER__SHIFT                                                  0x11
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_MTYPE__SHIFT                                                    0x12
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_D__SHIFT                                                        0x14
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_PREFETCH_CODE__SHIFT                                            0x15
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_TEE__SHIFT                                                      0x17
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_HDM__SHIFT                                                      0x18
+#define UTCL1_IDENTITY_MODE3__RESERVED__SHIFT                                                                 0x19
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_SNOOP_MASK                                                      0x00000001L
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_FRAG_SIZE_MASK                                                  0x0000007EL
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_PERM_GRANTED_MASK                                               0x00000780L
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_PERM_FAULT_MASK                                                 0x00000800L
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_XNACK_MASK                                                      0x00003000L
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_PTE_TMZ_MASK                                                    0x00004000L
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_NO_PTE_MASK                                                     0x00008000L
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_SPA_MASK                                                        0x00010000L
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_IOSTEER_MASK                                                    0x00020000L
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_MTYPE_MASK                                                      0x000C0000L
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_D_MASK                                                          0x00100000L
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_PREFETCH_CODE_MASK                                              0x00600000L
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_TEE_MASK                                                        0x00800000L
+#define UTCL1_IDENTITY_MODE3__UTCL1_UTCL0_RET_HDM_MASK                                                        0x01000000L
+#define UTCL1_IDENTITY_MODE3__RESERVED_MASK                                                                   0xFE000000L
+//UTCL1_IDENTITY_MODE4
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_SNOOP__SHIFT                                                    0x0
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_FRAG_SIZE__SHIFT                                                0x1
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_PERM_GRANTED__SHIFT                                             0x7
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_PERM_FAULT__SHIFT                                               0xb
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_XNACK__SHIFT                                                    0xc
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_PTE_TMZ__SHIFT                                                  0xe
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_NO_PTE__SHIFT                                                   0xf
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_SPA__SHIFT                                                      0x10
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_IOSTEER__SHIFT                                                  0x11
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_MTYPE__SHIFT                                                    0x12
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_D__SHIFT                                                        0x14
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_PREFETCH_CODE__SHIFT                                            0x15
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_TEE__SHIFT                                                      0x17
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_HDM__SHIFT                                                      0x18
+#define UTCL1_IDENTITY_MODE4__RESERVED__SHIFT                                                                 0x19
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_SNOOP_MASK                                                      0x00000001L
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_FRAG_SIZE_MASK                                                  0x0000007EL
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_PERM_GRANTED_MASK                                               0x00000780L
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_PERM_FAULT_MASK                                                 0x00000800L
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_XNACK_MASK                                                      0x00003000L
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_PTE_TMZ_MASK                                                    0x00004000L
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_NO_PTE_MASK                                                     0x00008000L
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_SPA_MASK                                                        0x00010000L
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_IOSTEER_MASK                                                    0x00020000L
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_MTYPE_MASK                                                      0x000C0000L
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_D_MASK                                                          0x00100000L
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_PREFETCH_CODE_MASK                                              0x00600000L
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_TEE_MASK                                                        0x00800000L
+#define UTCL1_IDENTITY_MODE4__UTCL1_UTCL0_RET_HDM_MASK                                                        0x01000000L
+#define UTCL1_IDENTITY_MODE4__RESERVED_MASK                                                                   0xFE000000L
+//UTCL1_IDENTITY_MODE5
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_SNOOP__SHIFT                                                    0x0
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_FRAG_SIZE__SHIFT                                                0x1
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_PERM_GRANTED__SHIFT                                             0x7
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_PERM_FAULT__SHIFT                                               0xb
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_XNACK__SHIFT                                                    0xc
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_PTE_TMZ__SHIFT                                                  0xe
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_NO_PTE__SHIFT                                                   0xf
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_SPA__SHIFT                                                      0x10
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_IOSTEER__SHIFT                                                  0x11
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_MTYPE__SHIFT                                                    0x12
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_D__SHIFT                                                        0x14
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_PREFETCH_CODE__SHIFT                                            0x15
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_TEE__SHIFT                                                      0x17
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_HDM__SHIFT                                                      0x18
+#define UTCL1_IDENTITY_MODE5__RESERVED__SHIFT                                                                 0x19
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_SNOOP_MASK                                                      0x00000001L
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_FRAG_SIZE_MASK                                                  0x0000007EL
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_PERM_GRANTED_MASK                                               0x00000780L
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_PERM_FAULT_MASK                                                 0x00000800L
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_XNACK_MASK                                                      0x00003000L
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_PTE_TMZ_MASK                                                    0x00004000L
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_NO_PTE_MASK                                                     0x00008000L
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_SPA_MASK                                                        0x00010000L
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_IOSTEER_MASK                                                    0x00020000L
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_MTYPE_MASK                                                      0x000C0000L
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_D_MASK                                                          0x00100000L
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_PREFETCH_CODE_MASK                                              0x00600000L
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_TEE_MASK                                                        0x00800000L
+#define UTCL1_IDENTITY_MODE5__UTCL1_UTCL0_RET_HDM_MASK                                                        0x01000000L
+#define UTCL1_IDENTITY_MODE5__RESERVED_MASK                                                                   0xFE000000L
+//UTCL1_IDENTITY_MODE6
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_SNOOP__SHIFT                                                    0x0
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_FRAG_SIZE__SHIFT                                                0x1
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_PERM_GRANTED__SHIFT                                             0x7
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_PERM_FAULT__SHIFT                                               0xb
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_XNACK__SHIFT                                                    0xc
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_PTE_TMZ__SHIFT                                                  0xe
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_NO_PTE__SHIFT                                                   0xf
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_SPA__SHIFT                                                      0x10
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_IOSTEER__SHIFT                                                  0x11
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_MTYPE__SHIFT                                                    0x12
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_D__SHIFT                                                        0x14
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_PREFETCH_CODE__SHIFT                                            0x15
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_TEE__SHIFT                                                      0x17
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_HDM__SHIFT                                                      0x18
+#define UTCL1_IDENTITY_MODE6__RESERVED__SHIFT                                                                 0x19
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_SNOOP_MASK                                                      0x00000001L
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_FRAG_SIZE_MASK                                                  0x0000007EL
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_PERM_GRANTED_MASK                                               0x00000780L
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_PERM_FAULT_MASK                                                 0x00000800L
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_XNACK_MASK                                                      0x00003000L
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_PTE_TMZ_MASK                                                    0x00004000L
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_NO_PTE_MASK                                                     0x00008000L
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_SPA_MASK                                                        0x00010000L
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_IOSTEER_MASK                                                    0x00020000L
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_MTYPE_MASK                                                      0x000C0000L
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_D_MASK                                                          0x00100000L
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_PREFETCH_CODE_MASK                                              0x00600000L
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_TEE_MASK                                                        0x00800000L
+#define UTCL1_IDENTITY_MODE6__UTCL1_UTCL0_RET_HDM_MASK                                                        0x01000000L
+#define UTCL1_IDENTITY_MODE6__RESERVED_MASK                                                                   0xFE000000L
+//UTCL1_IDENTITY_MODE7
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_SNOOP__SHIFT                                                    0x0
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_FRAG_SIZE__SHIFT                                                0x1
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_PERM_GRANTED__SHIFT                                             0x7
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_PERM_FAULT__SHIFT                                               0xb
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_XNACK__SHIFT                                                    0xc
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_PTE_TMZ__SHIFT                                                  0xe
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_NO_PTE__SHIFT                                                   0xf
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_SPA__SHIFT                                                      0x10
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_IOSTEER__SHIFT                                                  0x11
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_MTYPE__SHIFT                                                    0x12
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_D__SHIFT                                                        0x14
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_PREFETCH_CODE__SHIFT                                            0x15
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_TEE__SHIFT                                                      0x17
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_HDM__SHIFT                                                      0x18
+#define UTCL1_IDENTITY_MODE7__RESERVED__SHIFT                                                                 0x19
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_SNOOP_MASK                                                      0x00000001L
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_FRAG_SIZE_MASK                                                  0x0000007EL
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_PERM_GRANTED_MASK                                               0x00000780L
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_PERM_FAULT_MASK                                                 0x00000800L
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_XNACK_MASK                                                      0x00003000L
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_PTE_TMZ_MASK                                                    0x00004000L
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_NO_PTE_MASK                                                     0x00008000L
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_SPA_MASK                                                        0x00010000L
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_IOSTEER_MASK                                                    0x00020000L
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_MTYPE_MASK                                                      0x000C0000L
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_D_MASK                                                          0x00100000L
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_PREFETCH_CODE_MASK                                              0x00600000L
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_TEE_MASK                                                        0x00800000L
+#define UTCL1_IDENTITY_MODE7__UTCL1_UTCL0_RET_HDM_MASK                                                        0x01000000L
+#define UTCL1_IDENTITY_MODE7__RESERVED_MASK                                                                   0xFE000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pfonly_tcpdec
+//TCP_INVALIDATE
+#define TCP_INVALIDATE__START__SHIFT                                                                          0x0
+#define TCP_INVALIDATE__START_MASK                                                                            0x00000001L
+//TCP_STATUS
+#define TCP_STATUS__TCP_BUSY__SHIFT                                                                           0x0
+#define TCP_STATUS__INPUT_BUSY_PORT0__SHIFT                                                                   0x1
+#define TCP_STATUS__INPUT_BUSY_PORT1__SHIFT                                                                   0x2
+#define TCP_STATUS__ADRS_BUSY_PORT0__SHIFT                                                                    0x3
+#define TCP_STATUS__ADRS_BUSY_PORT1__SHIFT                                                                    0x4
+#define TCP_STATUS__TAGRAMS_BUSY__SHIFT                                                                       0x5
+#define TCP_STATUS__VM_BUSY__SHIFT                                                                            0x6
+#define TCP_STATUS__MEMIF_BUSY__SHIFT                                                                         0x7
+#define TCP_STATUS__GCR_BUSY__SHIFT                                                                           0x8
+#define TCP_STATUS__CNTRL_BUSY_PORT0__SHIFT                                                                   0x9
+#define TCP_STATUS__CNTRL_BUSY_PORT1__SHIFT                                                                   0xa
+#define TCP_STATUS__LFIFO_BUSY_PORT0__SHIFT                                                                   0xb
+#define TCP_STATUS__LFIFO_BUSY_PORT1__SHIFT                                                                   0xc
+#define TCP_STATUS__OFIFO_BUSY_PORT0__SHIFT                                                                   0xd
+#define TCP_STATUS__OFIFO_BUSY_PORT1__SHIFT                                                                   0xe
+#define TCP_STATUS__OFIFO_QUEUE_BUSY_PORT0__SHIFT                                                             0xf
+#define TCP_STATUS__OFIFO_QUEUE_BUSY_PORT1__SHIFT                                                             0x12
+#define TCP_STATUS__READ_BUSY_PORT1__SHIFT                                                                    0x15
+#define TCP_STATUS__READ_BUSY_PORT0__SHIFT                                                                    0x16
+#define TCP_STATUS__CL_PENDING__SHIFT                                                                         0x17
+#define TCP_STATUS__VRX_BUSY__SHIFT                                                                           0x18
+#define TCP_STATUS__PKR_BUSY_PORT0__SHIFT                                                                     0x19
+#define TCP_STATUS__PKR_BUSY_PORT1__SHIFT                                                                     0x1a
+#define TCP_STATUS__LDS_BUSY_PORT0__SHIFT                                                                     0x1b
+#define TCP_STATUS__LDS_BUSY_PORT1__SHIFT                                                                     0x1c
+#define TCP_STATUS__XNACK_PRT__SHIFT                                                                          0x1d
+#define TCP_STATUS__TCP_BUSY_MASK                                                                             0x00000001L
+#define TCP_STATUS__INPUT_BUSY_PORT0_MASK                                                                     0x00000002L
+#define TCP_STATUS__INPUT_BUSY_PORT1_MASK                                                                     0x00000004L
+#define TCP_STATUS__ADRS_BUSY_PORT0_MASK                                                                      0x00000008L
+#define TCP_STATUS__ADRS_BUSY_PORT1_MASK                                                                      0x00000010L
+#define TCP_STATUS__TAGRAMS_BUSY_MASK                                                                         0x00000020L
+#define TCP_STATUS__VM_BUSY_MASK                                                                              0x00000040L
+#define TCP_STATUS__MEMIF_BUSY_MASK                                                                           0x00000080L
+#define TCP_STATUS__GCR_BUSY_MASK                                                                             0x00000100L
+#define TCP_STATUS__CNTRL_BUSY_PORT0_MASK                                                                     0x00000200L
+#define TCP_STATUS__CNTRL_BUSY_PORT1_MASK                                                                     0x00000400L
+#define TCP_STATUS__LFIFO_BUSY_PORT0_MASK                                                                     0x00000800L
+#define TCP_STATUS__LFIFO_BUSY_PORT1_MASK                                                                     0x00001000L
+#define TCP_STATUS__OFIFO_BUSY_PORT0_MASK                                                                     0x00002000L
+#define TCP_STATUS__OFIFO_BUSY_PORT1_MASK                                                                     0x00004000L
+#define TCP_STATUS__OFIFO_QUEUE_BUSY_PORT0_MASK                                                               0x00018000L
+#define TCP_STATUS__OFIFO_QUEUE_BUSY_PORT1_MASK                                                               0x000C0000L
+#define TCP_STATUS__READ_BUSY_PORT1_MASK                                                                      0x00200000L
+#define TCP_STATUS__READ_BUSY_PORT0_MASK                                                                      0x00400000L
+#define TCP_STATUS__CL_PENDING_MASK                                                                           0x00800000L
+#define TCP_STATUS__VRX_BUSY_MASK                                                                             0x01000000L
+#define TCP_STATUS__PKR_BUSY_PORT0_MASK                                                                       0x02000000L
+#define TCP_STATUS__PKR_BUSY_PORT1_MASK                                                                       0x04000000L
+#define TCP_STATUS__LDS_BUSY_PORT0_MASK                                                                       0x08000000L
+#define TCP_STATUS__LDS_BUSY_PORT1_MASK                                                                       0x10000000L
+#define TCP_STATUS__XNACK_PRT_MASK                                                                            0x20000000L
+//TCP_CNTL
+#define TCP_CNTL__FORCE_HIT__SHIFT                                                                            0x0
+#define TCP_CNTL__FORCE_MISS__SHIFT                                                                           0x1
+#define TCP_CNTL__STORE_ATOMIC_COLLAPSE_CLAUSE_LIMIT__SHIFT                                                   0x2
+#define TCP_CNTL__FLAT_BUF_CACHE_SWIZZLE__SHIFT                                                               0x5
+#define TCP_CNTL__TD_DATA_EN_OVERRIDE__SHIFT                                                                  0x6
+#define TCP_CNTL__TCP_SPILL_CACHE_DISABLE__SHIFT                                                              0x7
+#define TCP_CNTL__DISABLE_WRITE_COMBINING__SHIFT                                                              0x9
+#define TCP_CNTL__FORCE_SCOPE_EOW__SHIFT                                                                      0xa
+#define TCP_CNTL__FORCE_TEMPORAL_EOW__SHIFT                                                                   0xb
+#define TCP_CNTL__DISABLE_DCOPY_SET_THRASH_OPT__SHIFT                                                         0xc
+#define TCP_CNTL__FORCE_EOW_TOTAL_CNT__SHIFT                                                                  0xf
+#define TCP_CNTL__FORCE_EOW_SET_CNT__SHIFT                                                                    0x16
+#define TCP_CNTL__DISABLE_FULL_CL_ACCESS__SHIFT                                                               0x1b
+#define TCP_CNTL__DISABLE_Z_MAP__SHIFT                                                                        0x1c
+#define TCP_CNTL__ENABLE_64B_SCRATCH_STRIDE__SHIFT                                                            0x1d
+#define TCP_CNTL__ASTC_VE_MSB_TOLERANT__SHIFT                                                                 0x1f
+#define TCP_CNTL__FORCE_HIT_MASK                                                                              0x00000001L
+#define TCP_CNTL__FORCE_MISS_MASK                                                                             0x00000002L
+#define TCP_CNTL__STORE_ATOMIC_COLLAPSE_CLAUSE_LIMIT_MASK                                                     0x0000001CL
+#define TCP_CNTL__FLAT_BUF_CACHE_SWIZZLE_MASK                                                                 0x00000020L
+#define TCP_CNTL__TD_DATA_EN_OVERRIDE_MASK                                                                    0x00000040L
+#define TCP_CNTL__TCP_SPILL_CACHE_DISABLE_MASK                                                                0x00000080L
+#define TCP_CNTL__DISABLE_WRITE_COMBINING_MASK                                                                0x00000200L
+#define TCP_CNTL__FORCE_SCOPE_EOW_MASK                                                                        0x00000400L
+#define TCP_CNTL__FORCE_TEMPORAL_EOW_MASK                                                                     0x00000800L
+#define TCP_CNTL__DISABLE_DCOPY_SET_THRASH_OPT_MASK                                                           0x00001000L
+#define TCP_CNTL__FORCE_EOW_TOTAL_CNT_MASK                                                                    0x001F8000L
+#define TCP_CNTL__FORCE_EOW_SET_CNT_MASK                                                                      0x07C00000L
+#define TCP_CNTL__DISABLE_FULL_CL_ACCESS_MASK                                                                 0x08000000L
+#define TCP_CNTL__DISABLE_Z_MAP_MASK                                                                          0x10000000L
+#define TCP_CNTL__ENABLE_64B_SCRATCH_STRIDE_MASK                                                              0x20000000L
+#define TCP_CNTL__ASTC_VE_MSB_TOLERANT_MASK                                                                   0x80000000L
+//TCP_CNTL2
+#define TCP_CNTL2__TCP_FMT_MGCG_DISABLE__SHIFT                                                                0x8
+#define TCP_CNTL2__TCPF_LATENCY_BYPASS_DISABLE__SHIFT                                                         0x9
+#define TCP_CNTL2__TCP_WRITE_DATA_MGCG_DISABLE__SHIFT                                                         0xa
+#define TCP_CNTL2__TCP_INNER_BLOCK_MGCG_DISABLE__SHIFT                                                        0xb
+#define TCP_CNTL2__TCP_ADRS_IMG_CALC_MGCG_DISABLE__SHIFT                                                      0xc
+#define TCP_CNTL2__V64_COMBINE_ENABLE__SHIFT                                                                  0xd
+#define TCP_CNTL2__TAGRAM_ADDR_SWIZZLE_DISABLE__SHIFT                                                         0xe
+#define TCP_CNTL2__TX_SP_READ_FGCG_DISABLE__SHIFT                                                             0xf
+#define TCP_CNTL2__POWER_OPT_DISABLE__SHIFT                                                                   0x10
+#define TCP_CNTL2__GCR_RSP_FGCG_DISABLE__SHIFT                                                                0x11
+#define TCP_CNTL2__PERF_EN_OVERRIDE__SHIFT                                                                    0x12
+#define TCP_CNTL2__TCP_GL1_REQ_CLKEN_DISABLE__SHIFT                                                           0x16
+#define TCP_CNTL2__TCP_GL1R_SRC_CLKEN_DISABLE__SHIFT                                                          0x17
+#define TCP_CNTL2__TCP_FORCE_2X_TO_LOAD__SHIFT                                                                0x18
+#define TCP_CNTL2__DISABLE_FLAT_BUF_DATARAM_SWIZZLE__SHIFT                                                    0x19
+#define TCP_CNTL2__VC_UTC_PERMISSIONS_ATOMIC_ALL_SCOPES__SHIFT                                                0x1a
+#define TCP_CNTL2__TAGRAM_XY_BIAS_OVERRIDE__SHIFT                                                             0x1b
+#define TCP_CNTL2__TCP_REQ_MGCG_DISABLE__SHIFT                                                                0x1d
+#define TCP_CNTL2__TCP_MISS_MGCG_DISABLE__SHIFT                                                               0x1e
+#define TCP_CNTL2__DISABLE_MIPMAP_PARAM_CALC_SELF_GATING__SHIFT                                               0x1f
+#define TCP_CNTL2__TCP_FMT_MGCG_DISABLE_MASK                                                                  0x00000100L
+#define TCP_CNTL2__TCPF_LATENCY_BYPASS_DISABLE_MASK                                                           0x00000200L
+#define TCP_CNTL2__TCP_WRITE_DATA_MGCG_DISABLE_MASK                                                           0x00000400L
+#define TCP_CNTL2__TCP_INNER_BLOCK_MGCG_DISABLE_MASK                                                          0x00000800L
+#define TCP_CNTL2__TCP_ADRS_IMG_CALC_MGCG_DISABLE_MASK                                                        0x00001000L
+#define TCP_CNTL2__V64_COMBINE_ENABLE_MASK                                                                    0x00002000L
+#define TCP_CNTL2__TAGRAM_ADDR_SWIZZLE_DISABLE_MASK                                                           0x00004000L
+#define TCP_CNTL2__TX_SP_READ_FGCG_DISABLE_MASK                                                               0x00008000L
+#define TCP_CNTL2__POWER_OPT_DISABLE_MASK                                                                     0x00010000L
+#define TCP_CNTL2__GCR_RSP_FGCG_DISABLE_MASK                                                                  0x00020000L
+#define TCP_CNTL2__PERF_EN_OVERRIDE_MASK                                                                      0x000C0000L
+#define TCP_CNTL2__TCP_GL1_REQ_CLKEN_DISABLE_MASK                                                             0x00400000L
+#define TCP_CNTL2__TCP_GL1R_SRC_CLKEN_DISABLE_MASK                                                            0x00800000L
+#define TCP_CNTL2__TCP_FORCE_2X_TO_LOAD_MASK                                                                  0x01000000L
+#define TCP_CNTL2__DISABLE_FLAT_BUF_DATARAM_SWIZZLE_MASK                                                      0x02000000L
+#define TCP_CNTL2__VC_UTC_PERMISSIONS_ATOMIC_ALL_SCOPES_MASK                                                  0x04000000L
+#define TCP_CNTL2__TAGRAM_XY_BIAS_OVERRIDE_MASK                                                               0x18000000L
+#define TCP_CNTL2__TCP_REQ_MGCG_DISABLE_MASK                                                                  0x20000000L
+#define TCP_CNTL2__TCP_MISS_MGCG_DISABLE_MASK                                                                 0x40000000L
+#define TCP_CNTL2__DISABLE_MIPMAP_PARAM_CALC_SELF_GATING_MASK                                                 0x80000000L
+//TCP_CREDIT
+#define TCP_CREDIT__LFIFO_RAM_DEPTH__SHIFT                                                                    0x0
+#define TCP_CREDIT__GL1_REQ_CREDIT__SHIFT                                                                     0xa
+#define TCP_CREDIT__REQ_FIFO_CREDIT__SHIFT                                                                    0x11
+#define TCP_CREDIT__TD_RAM_CREDIT__SHIFT                                                                      0x17
+#define TCP_CREDIT__TD_DATA_CREDIT__SHIFT                                                                     0x1d
+#define TCP_CREDIT__LFIFO_RAM_DEPTH_MASK                                                                      0x000003FFL
+#define TCP_CREDIT__GL1_REQ_CREDIT_MASK                                                                       0x0001FC00L
+#define TCP_CREDIT__REQ_FIFO_CREDIT_MASK                                                                      0x007E0000L
+#define TCP_CREDIT__TD_RAM_CREDIT_MASK                                                                        0x0F800000L
+#define TCP_CREDIT__TD_DATA_CREDIT_MASK                                                                       0xE0000000L
+//TCP_DEBUG_DATA
+#define TCP_DEBUG_DATA__DATA__SHIFT                                                                           0x0
+#define TCP_DEBUG_DATA__DATA_MASK                                                                             0xFFFFFFFFL
+//TCP_COMPRESSION_CNTL
+#define TCP_COMPRESSION_CNTL__IMAGE_COMPRESSION_OVERRIDE__SHIFT                                               0x0
+#define TCP_COMPRESSION_CNTL__IMAGE_BYPASS_COMPRESSION__SHIFT                                                 0x1
+#define TCP_COMPRESSION_CNTL__IMAGE_WRITE_COMPRESSION_DISABLE__SHIFT                                          0x2
+#define TCP_COMPRESSION_CNTL__BUFFER_COMPRESSION_OVERRIDE__SHIFT                                              0x3
+#define TCP_COMPRESSION_CNTL__BUFFER_BYPASS_COMPRESSION__SHIFT                                                0x4
+#define TCP_COMPRESSION_CNTL__BUFFER_WRITE_COMPRESSION_DISABLE__SHIFT                                         0x5
+#define TCP_COMPRESSION_CNTL__FLAT_COMPRESSION_OVERRIDE__SHIFT                                                0x6
+#define TCP_COMPRESSION_CNTL__FLAT_BYPASS_COMPRESSION__SHIFT                                                  0x7
+#define TCP_COMPRESSION_CNTL__FLAT_WRITE_COMPRESSION_DISABLE__SHIFT                                           0x8
+#define TCP_COMPRESSION_CNTL__BVH_BYPASS_COMPRESSION__SHIFT                                                   0x9
+#define TCP_COMPRESSION_CNTL__BUF_SPECULATIVE_DATA_READ__SHIFT                                                0xa
+#define TCP_COMPRESSION_CNTL__BUF_MAX_COMP_BLOCK_SIZE__SHIFT                                                  0xc
+#define TCP_COMPRESSION_CNTL__BUF_MAX_UNCOMP_BLOCK_SIZE__SHIFT                                                0xe
+#define TCP_COMPRESSION_CNTL__BVH_SPECULATIVE_DATA_READ__SHIFT                                                0xf
+#define TCP_COMPRESSION_CNTL__BVH_MAX_COMP_BLOCK_SIZE__SHIFT                                                  0x11
+#define TCP_COMPRESSION_CNTL__BVH_MAX_UNCOMP_BLOCK_SIZE__SHIFT                                                0x13
+#define TCP_COMPRESSION_CNTL__IMAGE_COMPRESSION_OVERRIDE_MASK                                                 0x00000001L
+#define TCP_COMPRESSION_CNTL__IMAGE_BYPASS_COMPRESSION_MASK                                                   0x00000002L
+#define TCP_COMPRESSION_CNTL__IMAGE_WRITE_COMPRESSION_DISABLE_MASK                                            0x00000004L
+#define TCP_COMPRESSION_CNTL__BUFFER_COMPRESSION_OVERRIDE_MASK                                                0x00000008L
+#define TCP_COMPRESSION_CNTL__BUFFER_BYPASS_COMPRESSION_MASK                                                  0x00000010L
+#define TCP_COMPRESSION_CNTL__BUFFER_WRITE_COMPRESSION_DISABLE_MASK                                           0x00000020L
+#define TCP_COMPRESSION_CNTL__FLAT_COMPRESSION_OVERRIDE_MASK                                                  0x00000040L
+#define TCP_COMPRESSION_CNTL__FLAT_BYPASS_COMPRESSION_MASK                                                    0x00000080L
+#define TCP_COMPRESSION_CNTL__FLAT_WRITE_COMPRESSION_DISABLE_MASK                                             0x00000100L
+#define TCP_COMPRESSION_CNTL__BVH_BYPASS_COMPRESSION_MASK                                                     0x00000200L
+#define TCP_COMPRESSION_CNTL__BUF_SPECULATIVE_DATA_READ_MASK                                                  0x00000C00L
+#define TCP_COMPRESSION_CNTL__BUF_MAX_COMP_BLOCK_SIZE_MASK                                                    0x00003000L
+#define TCP_COMPRESSION_CNTL__BUF_MAX_UNCOMP_BLOCK_SIZE_MASK                                                  0x00004000L
+#define TCP_COMPRESSION_CNTL__BVH_SPECULATIVE_DATA_READ_MASK                                                  0x00018000L
+#define TCP_COMPRESSION_CNTL__BVH_MAX_COMP_BLOCK_SIZE_MASK                                                    0x00060000L
+#define TCP_COMPRESSION_CNTL__BVH_MAX_UNCOMP_BLOCK_SIZE_MASK                                                  0x00080000L
+//TCP_ARB
+#define TCP_ARB__WEIGHT__SHIFT                                                                                0x0
+#define TCP_ARB__END_CLAUSE__SHIFT                                                                            0x3
+#define TCP_ARB__WEIGHT_MASK                                                                                  0x00000007L
+#define TCP_ARB__END_CLAUSE_MASK                                                                              0x00000008L
+//TCP_UTCL0_CNTL1
+#define TCP_UTCL0_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                              0x0
+#define TCP_UTCL0_CNTL1__GPUVM_64K_DEF__SHIFT                                                                 0x1
+#define TCP_UTCL0_CNTL1__GPUVM_PERM_MODE__SHIFT                                                               0x2
+#define TCP_UTCL0_CNTL1__RESP_MODE__SHIFT                                                                     0x3
+#define TCP_UTCL0_CNTL1__RESP_FAULT_MODE__SHIFT                                                               0x5
+#define TCP_UTCL0_CNTL1__CLIENTID__SHIFT                                                                      0x7
+#define TCP_UTCL0_CNTL1__REG_INV_VMID__SHIFT                                                                  0x11
+#define TCP_UTCL0_CNTL1__REG_INV_TOGGLE__SHIFT                                                                0x16
+#define TCP_UTCL0_CNTL1__ATOMIC_REQUESTER_EN__SHIFT                                                           0x17
+#define TCP_UTCL0_CNTL1__FORCE_MISS__SHIFT                                                                    0x18
+#define TCP_UTCL0_CNTL1__FORCE_IN_ORDER__SHIFT                                                                0x19
+#define TCP_UTCL0_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                        0x1a
+#define TCP_UTCL0_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                        0x1d
+#define TCP_UTCL0_CNTL1__FORCE_4K_L2_RESP_MASK                                                                0x00000001L
+#define TCP_UTCL0_CNTL1__GPUVM_64K_DEF_MASK                                                                   0x00000002L
+#define TCP_UTCL0_CNTL1__GPUVM_PERM_MODE_MASK                                                                 0x00000004L
+#define TCP_UTCL0_CNTL1__RESP_MODE_MASK                                                                       0x00000018L
+#define TCP_UTCL0_CNTL1__RESP_FAULT_MODE_MASK                                                                 0x00000060L
+#define TCP_UTCL0_CNTL1__CLIENTID_MASK                                                                        0x0000FF80L
+#define TCP_UTCL0_CNTL1__REG_INV_VMID_MASK                                                                    0x001E0000L
+#define TCP_UTCL0_CNTL1__REG_INV_TOGGLE_MASK                                                                  0x00400000L
+#define TCP_UTCL0_CNTL1__ATOMIC_REQUESTER_EN_MASK                                                             0x00800000L
+#define TCP_UTCL0_CNTL1__FORCE_MISS_MASK                                                                      0x01000000L
+#define TCP_UTCL0_CNTL1__FORCE_IN_ORDER_MASK                                                                  0x02000000L
+#define TCP_UTCL0_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                          0x1C000000L
+#define TCP_UTCL0_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                          0xE0000000L
+//TCP_UTCL0_CNTL2
+#define TCP_UTCL0_CNTL2__SPARE__SHIFT                                                                         0x0
+#define TCP_UTCL0_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                                0x9
+#define TCP_UTCL0_CNTL2__LINE_VALID__SHIFT                                                                    0xa
+#define TCP_UTCL0_CNTL2__FORCE_SNOOP__SHIFT                                                                   0xe
+#define TCP_UTCL0_CNTL2__DISABLE_BURST__SHIFT                                                                 0x11
+#define TCP_UTCL0_CNTL2__ENABLE_PERF_EVENT_RD_WR__SHIFT                                                       0x12
+#define TCP_UTCL0_CNTL2__PERF_EVENT_RD_WR__SHIFT                                                              0x13
+#define TCP_UTCL0_CNTL2__ENABLE_PERF_EVENT_VMID__SHIFT                                                        0x14
+#define TCP_UTCL0_CNTL2__PERF_EVENT_VMID__SHIFT                                                               0x15
+#define TCP_UTCL0_CNTL2__FORCE_FRAG_2M_TO_64K__SHIFT                                                          0x1a
+#define TCP_UTCL0_CNTL2__THRASHING_TIMEOUT_PROTECT_ENABLE__SHIFT                                              0x1b
+#define TCP_UTCL0_CNTL2__THRASHING_ENABLE__SHIFT                                                              0x1c
+#define TCP_UTCL0_CNTL2__DISABLE_SPECULATIVE_PREFETCH__SHIFT                                                  0x1d
+#define TCP_UTCL0_CNTL2__FGCG_DISABLE__SHIFT                                                                  0x1e
+#define TCP_UTCL0_CNTL2__DISABLE_ILLEGAL_PCIE_ATOMIC__SHIFT                                                   0x1f
+#define TCP_UTCL0_CNTL2__SPARE_MASK                                                                           0x000000FFL
+#define TCP_UTCL0_CNTL2__MTYPE_OVRD_DIS_MASK                                                                  0x00000200L
+#define TCP_UTCL0_CNTL2__LINE_VALID_MASK                                                                      0x00000400L
+#define TCP_UTCL0_CNTL2__FORCE_SNOOP_MASK                                                                     0x00004000L
+#define TCP_UTCL0_CNTL2__DISABLE_BURST_MASK                                                                   0x00020000L
+#define TCP_UTCL0_CNTL2__ENABLE_PERF_EVENT_RD_WR_MASK                                                         0x00040000L
+#define TCP_UTCL0_CNTL2__PERF_EVENT_RD_WR_MASK                                                                0x00080000L
+#define TCP_UTCL0_CNTL2__ENABLE_PERF_EVENT_VMID_MASK                                                          0x00100000L
+#define TCP_UTCL0_CNTL2__PERF_EVENT_VMID_MASK                                                                 0x01E00000L
+#define TCP_UTCL0_CNTL2__FORCE_FRAG_2M_TO_64K_MASK                                                            0x04000000L
+#define TCP_UTCL0_CNTL2__THRASHING_TIMEOUT_PROTECT_ENABLE_MASK                                                0x08000000L
+#define TCP_UTCL0_CNTL2__THRASHING_ENABLE_MASK                                                                0x10000000L
+#define TCP_UTCL0_CNTL2__DISABLE_SPECULATIVE_PREFETCH_MASK                                                    0x20000000L
+#define TCP_UTCL0_CNTL2__FGCG_DISABLE_MASK                                                                    0x40000000L
+#define TCP_UTCL0_CNTL2__DISABLE_ILLEGAL_PCIE_ATOMIC_MASK                                                     0x80000000L
+//TCP_UTCL0_STATUS
+#define TCP_UTCL0_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define TCP_UTCL0_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define TCP_UTCL0_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define TCP_UTCL0_STATUS__TIMEOUT_DETECTED__SHIFT                                                             0x3
+#define TCP_UTCL0_STATUS__VMID_BUSY__SHIFT                                                                    0x4
+#define TCP_UTCL0_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define TCP_UTCL0_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define TCP_UTCL0_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+#define TCP_UTCL0_STATUS__TIMEOUT_DETECTED_MASK                                                               0x00000008L
+#define TCP_UTCL0_STATUS__VMID_BUSY_MASK                                                                      0x000FFFF0L
+//TCP_RQID_HASH_CNTL
+#define TCP_RQID_HASH_CNTL__WAVE_ID_LEFT_MASK__SHIFT                                                          0x0
+#define TCP_RQID_HASH_CNTL__WAVE_ID_LEFT_SHIFT__SHIFT                                                         0x4
+#define TCP_RQID_HASH_CNTL__WAVE_ID_RIGHT_MASK__SHIFT                                                         0x7
+#define TCP_RQID_HASH_CNTL__WAVE_ID_RIGHT_SHIFT__SHIFT                                                        0xb
+#define TCP_RQID_HASH_CNTL__SIMD_ID_MASK__SHIFT                                                               0xe
+#define TCP_RQID_HASH_CNTL__SIMD_ID_SHIFT__SHIFT                                                              0xf
+#define TCP_RQID_HASH_CNTL__TYPE_MASK__SHIFT                                                                  0x12
+#define TCP_RQID_HASH_CNTL__TYPE_SHIFT__SHIFT                                                                 0x14
+#define TCP_RQID_HASH_CNTL__WAVE_ID_LEFT_MASK_MASK                                                            0x0000000FL
+#define TCP_RQID_HASH_CNTL__WAVE_ID_LEFT_SHIFT_MASK                                                           0x00000070L
+#define TCP_RQID_HASH_CNTL__WAVE_ID_RIGHT_MASK_MASK                                                           0x00000780L
+#define TCP_RQID_HASH_CNTL__WAVE_ID_RIGHT_SHIFT_MASK                                                          0x00003800L
+#define TCP_RQID_HASH_CNTL__SIMD_ID_MASK_MASK                                                                 0x00004000L
+#define TCP_RQID_HASH_CNTL__SIMD_ID_SHIFT_MASK                                                                0x00038000L
+#define TCP_RQID_HASH_CNTL__TYPE_MASK_MASK                                                                    0x000C0000L
+#define TCP_RQID_HASH_CNTL__TYPE_SHIFT_MASK                                                                   0x00700000L
+//TCP_SET_HASH_CFG
+#define TCP_SET_HASH_CFG__SHIFT_BY_FOR_192__SHIFT                                                             0x1
+#define TCP_SET_HASH_CFG__SHIFT_BY_FOR_320__SHIFT                                                             0x5
+#define TCP_SET_HASH_CFG__SHIFT_BY_FOR_384__SHIFT                                                             0x9
+#define TCP_SET_HASH_CFG__SHIFT_BY_FOR_448__SHIFT                                                             0xd
+#define TCP_SET_HASH_CFG__SHIFT_BY_FOR_192_MASK                                                               0x0000001EL
+#define TCP_SET_HASH_CFG__SHIFT_BY_FOR_320_MASK                                                               0x000001E0L
+#define TCP_SET_HASH_CFG__SHIFT_BY_FOR_384_MASK                                                               0x00001E00L
+#define TCP_SET_HASH_CFG__SHIFT_BY_FOR_448_MASK                                                               0x0001E000L
+//TCP_SET_HASH_MASK_64_SET0
+#define TCP_SET_HASH_MASK_64_SET0__MASK__SHIFT                                                                0x0
+#define TCP_SET_HASH_MASK_64_SET0__MASK_MASK                                                                  0x0007FFFFL
+//TCP_SET_HASH_MASK_64_SET1
+#define TCP_SET_HASH_MASK_64_SET1__MASK__SHIFT                                                                0x0
+#define TCP_SET_HASH_MASK_64_SET1__MASK_MASK                                                                  0x0007FFFFL
+//TCP_SET_HASH_MASK_64_SET2
+#define TCP_SET_HASH_MASK_64_SET2__MASK__SHIFT                                                                0x0
+#define TCP_SET_HASH_MASK_64_SET2__MASK_MASK                                                                  0x0007FFFFL
+//TCP_SET_HASH_MASK_64_SET3
+#define TCP_SET_HASH_MASK_64_SET3__MASK__SHIFT                                                                0x0
+#define TCP_SET_HASH_MASK_64_SET3__MASK_MASK                                                                  0x0007FFFFL
+//TCP_SET_HASH_MASK_128_SET0
+#define TCP_SET_HASH_MASK_128_SET0__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_128_SET0__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_128_SET1
+#define TCP_SET_HASH_MASK_128_SET1__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_128_SET1__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_128_SET2
+#define TCP_SET_HASH_MASK_128_SET2__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_128_SET2__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_128_SET3
+#define TCP_SET_HASH_MASK_128_SET3__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_128_SET3__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_128_SET4
+#define TCP_SET_HASH_MASK_128_SET4__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_128_SET4__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_192_SET0
+#define TCP_SET_HASH_MASK_192_SET0__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_192_SET0__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_192_SET1
+#define TCP_SET_HASH_MASK_192_SET1__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_192_SET1__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_192_SET2
+#define TCP_SET_HASH_MASK_192_SET2__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_192_SET2__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_192_SET3
+#define TCP_SET_HASH_MASK_192_SET3__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_192_SET3__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_256_SET0
+#define TCP_SET_HASH_MASK_256_SET0__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_256_SET0__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_256_SET1
+#define TCP_SET_HASH_MASK_256_SET1__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_256_SET1__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_256_SET2
+#define TCP_SET_HASH_MASK_256_SET2__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_256_SET2__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_256_SET3
+#define TCP_SET_HASH_MASK_256_SET3__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_256_SET3__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_256_SET4
+#define TCP_SET_HASH_MASK_256_SET4__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_256_SET4__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_256_SET5
+#define TCP_SET_HASH_MASK_256_SET5__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_256_SET5__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_320_SET0
+#define TCP_SET_HASH_MASK_320_SET0__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_320_SET0__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_320_SET1
+#define TCP_SET_HASH_MASK_320_SET1__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_320_SET1__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_320_SET2
+#define TCP_SET_HASH_MASK_320_SET2__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_320_SET2__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_320_SET3
+#define TCP_SET_HASH_MASK_320_SET3__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_320_SET3__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_384_SET0
+#define TCP_SET_HASH_MASK_384_SET0__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_384_SET0__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_384_SET1
+#define TCP_SET_HASH_MASK_384_SET1__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_384_SET1__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_384_SET2
+#define TCP_SET_HASH_MASK_384_SET2__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_384_SET2__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_384_SET3
+#define TCP_SET_HASH_MASK_384_SET3__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_384_SET3__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_384_SET4
+#define TCP_SET_HASH_MASK_384_SET4__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_384_SET4__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_448_SET0
+#define TCP_SET_HASH_MASK_448_SET0__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_448_SET0__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_448_SET1
+#define TCP_SET_HASH_MASK_448_SET1__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_448_SET1__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_448_SET2
+#define TCP_SET_HASH_MASK_448_SET2__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_448_SET2__MASK_MASK                                                                 0x0007FFFFL
+//TCP_SET_HASH_MASK_448_SET3
+#define TCP_SET_HASH_MASK_448_SET3__MASK__SHIFT                                                               0x0
+#define TCP_SET_HASH_MASK_448_SET3__MASK_MASK                                                                 0x0007FFFFL
+//TCP_CNTL3
+#define TCP_CNTL3__TCS_PORT_PRIORITY_EVENT__SHIFT                                                             0x0
+#define TCP_CNTL3__TCS_PORT_PRIORITY_WEIGHT__SHIFT                                                            0x2
+#define TCP_CNTL3__VRX_VC_PORT_PRIORITY_EVENT__SHIFT                                                          0x7
+#define TCP_CNTL3__VRX_VC_PORT_PRIORITY_WEIGHT__SHIFT                                                         0x9
+#define TCP_CNTL3__VRX_LDS_PORT_PRIORITY_EVENT__SHIFT                                                         0xe
+#define TCP_CNTL3__VRX_LDS_PORT_PRIORITY_WEIGHT__SHIFT                                                        0x10
+#define TCP_CNTL3__QM_CAC_OUTPUT__SHIFT                                                                       0x15
+#define TCP_CNTL3__DISABLE_COALESCING_FOR_PAGE_TRANSLATION__SHIFT                                             0x1a
+#define TCP_CNTL3__OOO_HWT_SHOOTDOWN_DRAIN_DIS__SHIFT                                                         0x1b
+#define TCP_CNTL3__ASYNC_INFLIGHT_VRX_SINGLE_PORT__SHIFT                                                      0x1c
+#define TCP_CNTL3__VC_DP_INV_STALL_EN__SHIFT                                                                  0x1d
+#define TCP_CNTL3__DISABLE_EARLY_WRITE_ACK__SHIFT                                                             0x1e
+#define TCP_CNTL3__CL_256B_COMBINE_DIS__SHIFT                                                                 0x1f
+#define TCP_CNTL3__TCS_PORT_PRIORITY_EVENT_MASK                                                               0x00000003L
+#define TCP_CNTL3__TCS_PORT_PRIORITY_WEIGHT_MASK                                                              0x0000007CL
+#define TCP_CNTL3__VRX_VC_PORT_PRIORITY_EVENT_MASK                                                            0x00000180L
+#define TCP_CNTL3__VRX_VC_PORT_PRIORITY_WEIGHT_MASK                                                           0x00003E00L
+#define TCP_CNTL3__VRX_LDS_PORT_PRIORITY_EVENT_MASK                                                           0x0000C000L
+#define TCP_CNTL3__VRX_LDS_PORT_PRIORITY_WEIGHT_MASK                                                          0x001F0000L
+#define TCP_CNTL3__QM_CAC_OUTPUT_MASK                                                                         0x00200000L
+#define TCP_CNTL3__DISABLE_COALESCING_FOR_PAGE_TRANSLATION_MASK                                               0x04000000L
+#define TCP_CNTL3__OOO_HWT_SHOOTDOWN_DRAIN_DIS_MASK                                                           0x08000000L
+#define TCP_CNTL3__ASYNC_INFLIGHT_VRX_SINGLE_PORT_MASK                                                        0x10000000L
+#define TCP_CNTL3__VC_DP_INV_STALL_EN_MASK                                                                    0x20000000L
+#define TCP_CNTL3__DISABLE_EARLY_WRITE_ACK_MASK                                                               0x40000000L
+#define TCP_CNTL3__CL_256B_COMBINE_DIS_MASK                                                                   0x80000000L
+//LDS_CONFIG
+#define LDS_CONFIG__CONF_BIT_1__SHIFT                                                                         0x0
+#define LDS_CONFIG__PC_CNTRL_OUT_FGCG_OVERRIDE__SHIFT                                                         0x1
+#define LDS_CONFIG__SP_TDDATA_FGCG_OVERRIDE__SHIFT                                                            0x2
+#define LDS_CONFIG__SQC_PERF_FGCG_OVERRIDE__SHIFT                                                             0x3
+#define LDS_CONFIG__CONF_BIT_5__SHIFT                                                                         0x4
+#define LDS_CONFIG__CONF_BIT_6__SHIFT                                                                         0x5
+#define LDS_CONFIG__CONF_BIT_7__SHIFT                                                                         0x6
+#define LDS_CONFIG__CONF_BIT_8__SHIFT                                                                         0x7
+#define LDS_CONFIG__PEND_CNT_WIDTH__SHIFT                                                                     0x8
+#define LDS_CONFIG__LDSI_ROUNDROBIN_ARB__SHIFT                                                                0xe
+#define LDS_CONFIG__CONF_BIT_1_MASK                                                                           0x00000001L
+#define LDS_CONFIG__PC_CNTRL_OUT_FGCG_OVERRIDE_MASK                                                           0x00000002L
+#define LDS_CONFIG__SP_TDDATA_FGCG_OVERRIDE_MASK                                                              0x00000004L
+#define LDS_CONFIG__SQC_PERF_FGCG_OVERRIDE_MASK                                                               0x00000008L
+#define LDS_CONFIG__CONF_BIT_5_MASK                                                                           0x00000010L
+#define LDS_CONFIG__CONF_BIT_6_MASK                                                                           0x00000020L
+#define LDS_CONFIG__CONF_BIT_7_MASK                                                                           0x00000040L
+#define LDS_CONFIG__CONF_BIT_8_MASK                                                                           0x00000080L
+#define LDS_CONFIG__PEND_CNT_WIDTH_MASK                                                                       0x00003F00L
+#define LDS_CONFIG__LDSI_ROUNDROBIN_ARB_MASK                                                                  0x00004000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pfonly2_spidec
+//SPI_RESOURCE_RESERVE_CU_0
+#define SPI_RESOURCE_RESERVE_CU_0__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_0__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_0__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_0__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_0__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_0__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_0__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_0__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_0__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_0__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_1
+#define SPI_RESOURCE_RESERVE_CU_1__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_1__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_1__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_1__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_1__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_1__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_1__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_1__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_1__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_1__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_2
+#define SPI_RESOURCE_RESERVE_CU_2__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_2__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_2__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_2__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_2__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_2__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_2__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_2__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_2__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_2__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_3
+#define SPI_RESOURCE_RESERVE_CU_3__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_3__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_3__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_3__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_3__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_3__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_3__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_3__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_3__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_3__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_4
+#define SPI_RESOURCE_RESERVE_CU_4__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_4__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_4__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_4__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_4__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_4__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_4__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_4__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_4__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_4__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_5
+#define SPI_RESOURCE_RESERVE_CU_5__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_5__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_5__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_5__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_5__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_5__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_5__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_5__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_5__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_5__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_6
+#define SPI_RESOURCE_RESERVE_CU_6__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_6__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_6__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_6__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_6__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_6__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_6__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_6__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_6__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_6__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_7
+#define SPI_RESOURCE_RESERVE_CU_7__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_7__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_7__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_7__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_7__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_7__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_7__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_7__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_7__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_7__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_8
+#define SPI_RESOURCE_RESERVE_CU_8__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_8__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_8__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_8__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_8__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_8__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_8__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_8__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_8__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_8__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_9
+#define SPI_RESOURCE_RESERVE_CU_9__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_9__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_9__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_9__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_9__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_9__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_9__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_9__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_9__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_9__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_10
+#define SPI_RESOURCE_RESERVE_CU_10__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_10__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_10__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_10__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_10__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_10__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_10__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_10__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_10__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_10__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_CU_11
+#define SPI_RESOURCE_RESERVE_CU_11__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_11__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_11__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_11__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_11__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_11__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_11__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_11__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_11__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_11__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_CU_12
+#define SPI_RESOURCE_RESERVE_CU_12__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_12__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_12__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_12__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_12__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_12__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_12__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_12__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_12__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_12__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_CU_13
+#define SPI_RESOURCE_RESERVE_CU_13__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_13__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_13__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_13__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_13__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_13__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_13__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_13__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_13__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_13__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_CU_14
+#define SPI_RESOURCE_RESERVE_CU_14__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_14__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_14__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_14__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_14__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_14__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_14__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_14__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_14__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_14__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_CU_15
+#define SPI_RESOURCE_RESERVE_CU_15__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_15__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_15__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_15__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_15__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_15__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_15__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_15__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_15__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_15__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_EN_CU_0
+#define SPI_RESOURCE_RESERVE_EN_CU_0__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_0__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_0__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_0__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_0__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_0__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_1
+#define SPI_RESOURCE_RESERVE_EN_CU_1__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_1__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_1__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_1__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_1__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_1__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_2
+#define SPI_RESOURCE_RESERVE_EN_CU_2__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_2__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_2__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_2__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_2__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_2__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_3
+#define SPI_RESOURCE_RESERVE_EN_CU_3__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_3__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_3__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_3__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_3__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_3__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_4
+#define SPI_RESOURCE_RESERVE_EN_CU_4__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_4__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_4__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_4__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_4__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_4__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_5
+#define SPI_RESOURCE_RESERVE_EN_CU_5__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_5__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_5__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_5__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_5__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_5__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_6
+#define SPI_RESOURCE_RESERVE_EN_CU_6__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_6__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_6__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_6__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_6__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_6__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_7
+#define SPI_RESOURCE_RESERVE_EN_CU_7__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_7__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_7__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_7__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_7__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_7__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_8
+#define SPI_RESOURCE_RESERVE_EN_CU_8__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_8__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_8__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_8__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_8__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_8__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_9
+#define SPI_RESOURCE_RESERVE_EN_CU_9__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_9__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_9__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_9__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_9__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_9__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_10
+#define SPI_RESOURCE_RESERVE_EN_CU_10__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_10__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_10__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_10__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_10__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_10__QUEUE_MASK_MASK                                                        0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_11
+#define SPI_RESOURCE_RESERVE_EN_CU_11__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_11__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_11__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_11__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_11__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_11__QUEUE_MASK_MASK                                                        0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_12
+#define SPI_RESOURCE_RESERVE_EN_CU_12__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_12__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_12__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_12__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_12__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_12__QUEUE_MASK_MASK                                                        0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_13
+#define SPI_RESOURCE_RESERVE_EN_CU_13__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_13__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_13__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_13__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_13__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_13__QUEUE_MASK_MASK                                                        0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_14
+#define SPI_RESOURCE_RESERVE_EN_CU_14__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_14__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_14__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_14__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_14__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_14__QUEUE_MASK_MASK                                                        0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_15
+#define SPI_RESOURCE_RESERVE_EN_CU_15__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_15__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_15__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_15__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_15__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_15__QUEUE_MASK_MASK                                                        0x00FF0000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pfonly2_tcpdec
+//TCP_UTCL0_THRASHING_CTRL
+#define TCP_UTCL0_THRASHING_CTRL__THRASHING_EN__SHIFT                                                         0x0
+#define TCP_UTCL0_THRASHING_CTRL__RETRY_FRAGMENT_THRESHOLD_DOWN_EN__SHIFT                                     0x2
+#define TCP_UTCL0_THRASHING_CTRL__RETRY_FRAGMENT_THRESHOLD_UP_EN__SHIFT                                       0x3
+#define TCP_UTCL0_THRASHING_CTRL__STALL_TIMEOUT_PROTECT_ENABLE__SHIFT                                         0x4
+#define TCP_UTCL0_THRASHING_CTRL__STALL_TARGET_FRAGMENT_SIZE__SHIFT                                           0x5
+#define TCP_UTCL0_THRASHING_CTRL__RETRY_TARGET_FRAGMENT_SIZE__SHIFT                                           0x9
+#define TCP_UTCL0_THRASHING_CTRL__THRASHING_EN_MASK                                                           0x00000003L
+#define TCP_UTCL0_THRASHING_CTRL__RETRY_FRAGMENT_THRESHOLD_DOWN_EN_MASK                                       0x00000004L
+#define TCP_UTCL0_THRASHING_CTRL__RETRY_FRAGMENT_THRESHOLD_UP_EN_MASK                                         0x00000008L
+#define TCP_UTCL0_THRASHING_CTRL__STALL_TIMEOUT_PROTECT_ENABLE_MASK                                           0x00000010L
+#define TCP_UTCL0_THRASHING_CTRL__STALL_TARGET_FRAGMENT_SIZE_MASK                                             0x000001E0L
+#define TCP_UTCL0_THRASHING_CTRL__RETRY_TARGET_FRAGMENT_SIZE_MASK                                             0x00001E00L
+//TCP_UTCL0_RETRY_THRESHOLD_DOWN_COUNTER
+#define TCP_UTCL0_RETRY_THRESHOLD_DOWN_COUNTER__DOWN_COUNTER__SHIFT                                           0x0
+#define TCP_UTCL0_RETRY_THRESHOLD_DOWN_COUNTER__DOWN_COUNTER_MASK                                             0xFFFFFFFFL
+//TCP_UTCL0_RETRY_THRESHOLD_UP_COUNTER
+#define TCP_UTCL0_RETRY_THRESHOLD_UP_COUNTER__UP_COUNTER__SHIFT                                               0x0
+#define TCP_UTCL0_RETRY_THRESHOLD_UP_COUNTER__UP_COUNTER_MASK                                                 0xFFFFFFFFL
+//TCP_UTCL0_XNACK_RETRY
+#define TCP_UTCL0_XNACK_RETRY__INCR__SHIFT                                                                    0x0
+#define TCP_UTCL0_XNACK_RETRY__COUNT__SHIFT                                                                   0x8
+#define TCP_UTCL0_XNACK_RETRY__INCR_MASK                                                                      0x000000FFL
+#define TCP_UTCL0_XNACK_RETRY__COUNT_MASK                                                                     0x00000F00L
+//TCP_UTCL0_RETRY_TIMER_THRESHOLD_LO
+#define TCP_UTCL0_RETRY_TIMER_THRESHOLD_LO__VALUE__SHIFT                                                      0x0
+#define TCP_UTCL0_RETRY_TIMER_THRESHOLD_LO__VALUE_MASK                                                        0xFFFFFFFFL
+//TCP_UTCL0_RETRY_TIMER_THRESHOLD_HI
+#define TCP_UTCL0_RETRY_TIMER_THRESHOLD_HI__VALUE__SHIFT                                                      0x0
+#define TCP_UTCL0_RETRY_TIMER_THRESHOLD_HI__VALUE_MASK                                                        0xFFFFFFFFL
+//TXA_CNTL
+#define TXA_CNTL__TA_SQ_XNACK_FGCG_DISABLE__SHIFT                                                             0x0
+#define TXA_CNTL__TXA_CFG_CHG_STALL__SHIFT                                                                    0x1
+#define TXA_CNTL__TXA_ALIGNER_LDS_B96_LOAD_CYCLING__SHIFT                                                     0x2
+#define TXA_CNTL__ALIGNER_CREDIT__SHIFT                                                                       0x8
+#define TXA_CNTL__ALIGNER_CREDIT_LDS__SHIFT                                                                   0x10
+#define TXA_CNTL__DISABLE_ASYNC_LOAD_DIRECT_COPY__SHIFT                                                       0x15
+#define TXA_CNTL__DISABLE_ASYNC_STORE_DIRECT_COPY__SHIFT                                                      0x16
+#define TXA_CNTL__INPUT_XNACK_RESET_CREDIT__SHIFT                                                             0x17
+#define TXA_CNTL__ENABLE_B128_DIRECT_COPY_STORE__SHIFT                                                        0x1a
+#define TXA_CNTL__TA_SQ_XNACK_FGCG_DISABLE_MASK                                                               0x00000001L
+#define TXA_CNTL__TXA_CFG_CHG_STALL_MASK                                                                      0x00000002L
+#define TXA_CNTL__TXA_ALIGNER_LDS_B96_LOAD_CYCLING_MASK                                                       0x00000004L
+#define TXA_CNTL__ALIGNER_CREDIT_MASK                                                                         0x00001F00L
+#define TXA_CNTL__ALIGNER_CREDIT_LDS_MASK                                                                     0x001F0000L
+#define TXA_CNTL__DISABLE_ASYNC_LOAD_DIRECT_COPY_MASK                                                         0x00200000L
+#define TXA_CNTL__DISABLE_ASYNC_STORE_DIRECT_COPY_MASK                                                        0x00400000L
+#define TXA_CNTL__INPUT_XNACK_RESET_CREDIT_MASK                                                               0x03800000L
+#define TXA_CNTL__ENABLE_B128_DIRECT_COPY_STORE_MASK                                                          0x04000000L
+//TXA_CNTL_AUX
+#define TXA_CNTL_AUX__AUTO_ALIGN_FORMAT__SHIFT                                                                0x9
+#define TXA_CNTL_AUX__DETERMINISM_RESERVED_DISABLE__SHIFT                                                     0x14
+#define TXA_CNTL_AUX__DETERMINISM_OPCODE_STRICT_DISABLE__SHIFT                                                0x15
+#define TXA_CNTL_AUX__DETERMINISM_MISC_DISABLE__SHIFT                                                         0x16
+#define TXA_CNTL_AUX__DETERMINISM_WRITEOP_READFMT_DISABLE__SHIFT                                              0x19
+#define TXA_CNTL_AUX__DETERMINISM_DFMT_NFMT_DISABLE__SHIFT                                                    0x1a
+#define TXA_CNTL_AUX__AUTO_ALIGN_FORMAT_MASK                                                                  0x00000200L
+#define TXA_CNTL_AUX__DETERMINISM_RESERVED_DISABLE_MASK                                                       0x00100000L
+#define TXA_CNTL_AUX__DETERMINISM_OPCODE_STRICT_DISABLE_MASK                                                  0x00200000L
+#define TXA_CNTL_AUX__DETERMINISM_MISC_DISABLE_MASK                                                           0x00400000L
+#define TXA_CNTL_AUX__DETERMINISM_WRITEOP_READFMT_DISABLE_MASK                                                0x02000000L
+#define TXA_CNTL_AUX__DETERMINISM_DFMT_NFMT_DISABLE_MASK                                                      0x04000000L
+//TXA_CNTL2
+#define TXA_CNTL2__ELEMSIZE_HASH_DIS__SHIFT                                                                   0x11
+#define TXA_CNTL2__ELEMSIZE_HASH_DIS_MASK                                                                     0x00020000L
+//TXA_STATUS
+#define TXA_STATUS__TXA0_BUSY__SHIFT                                                                          0x10
+#define TXA_STATUS__TXA1_BUSY__SHIFT                                                                          0x11
+#define TXA_STATUS__TXA0_BUSY_MASK                                                                            0x00010000L
+#define TXA_STATUS__TXA1_BUSY_MASK                                                                            0x00020000L
+//TXA_TDM_ARB_CNTL
+#define TXA_TDM_ARB_CNTL__INTERLEAVE_MODE__SHIFT                                                              0x0
+#define TXA_TDM_ARB_CNTL__INTERLEAVE_STEP__SHIFT                                                              0x2
+#define TXA_TDM_ARB_CNTL__PR_EN__SHIFT                                                                        0x6
+#define TXA_TDM_ARB_CNTL__BAL_TM_ON_BUS__SHIFT                                                                0x8
+#define TXA_TDM_ARB_CNTL__PR_RATE__SHIFT                                                                      0x9
+#define TXA_TDM_ARB_CNTL__PR_CNTX_RESTORE__SHIFT                                                              0xc
+#define TXA_TDM_ARB_CNTL__DIS_DIRECT_FETCH__SHIFT                                                             0xd
+#define TXA_TDM_ARB_CNTL__REPLAY_SLEEP_CNT__SHIFT                                                             0xe
+#define TXA_TDM_ARB_CNTL__INFLIGHT_CNT_MAX__SHIFT                                                             0x15
+#define TXA_TDM_ARB_CNTL__DIS_ASYNC_LOAD_DC__SHIFT                                                            0x1e
+#define TXA_TDM_ARB_CNTL__DIS_ASYNC_STORE_DC__SHIFT                                                           0x1f
+#define TXA_TDM_ARB_CNTL__INTERLEAVE_MODE_MASK                                                                0x00000003L
+#define TXA_TDM_ARB_CNTL__INTERLEAVE_STEP_MASK                                                                0x0000003CL
+#define TXA_TDM_ARB_CNTL__PR_EN_MASK                                                                          0x000000C0L
+#define TXA_TDM_ARB_CNTL__BAL_TM_ON_BUS_MASK                                                                  0x00000100L
+#define TXA_TDM_ARB_CNTL__PR_RATE_MASK                                                                        0x00000E00L
+#define TXA_TDM_ARB_CNTL__PR_CNTX_RESTORE_MASK                                                                0x00001000L
+#define TXA_TDM_ARB_CNTL__DIS_DIRECT_FETCH_MASK                                                               0x00002000L
+#define TXA_TDM_ARB_CNTL__REPLAY_SLEEP_CNT_MASK                                                               0x001FC000L
+#define TXA_TDM_ARB_CNTL__INFLIGHT_CNT_MAX_MASK                                                               0x3FE00000L
+#define TXA_TDM_ARB_CNTL__DIS_ASYNC_LOAD_DC_MASK                                                              0x40000000L
+#define TXA_TDM_ARB_CNTL__DIS_ASYNC_STORE_DC_MASK                                                             0x80000000L
+//TCP_CREDIT2
+#define TCP_CREDIT2__LDSO0_ARB0_HANDLER_CREDIT__SHIFT                                                         0x0
+#define TCP_CREDIT2__LDSO1_ARB1_HANDLER_CREDIT__SHIFT                                                         0x4
+#define TCP_CREDIT2__VRX_TXD0_HANDLER_CREDIT__SHIFT                                                           0x8
+#define TCP_CREDIT2__VRX_TXD1_HANDLER_CREDIT__SHIFT                                                           0xc
+#define TCP_CREDIT2__LDSO0_ARB0_HANDLER_CREDIT_MASK                                                           0x0000000FL
+#define TCP_CREDIT2__LDSO1_ARB1_HANDLER_CREDIT_MASK                                                           0x000000F0L
+#define TCP_CREDIT2__VRX_TXD0_HANDLER_CREDIT_MASK                                                             0x00000F00L
+#define TCP_CREDIT2__VRX_TXD1_HANDLER_CREDIT_MASK                                                             0x0000F000L
+//VC_CONGESTION_CONTROL
+#define VC_CONGESTION_CONTROL__LFIFO_LIMIT__SHIFT                                                             0x0
+#define VC_CONGESTION_CONTROL__SET_PENDING_LIMIT__SHIFT                                                       0xa
+#define VC_CONGESTION_CONTROL__UTCL0_LFIFO_LIMIT__SHIFT                                                       0xf
+#define VC_CONGESTION_CONTROL__LFIFO_LIMIT_MASK                                                               0x000003FFL
+#define VC_CONGESTION_CONTROL__SET_PENDING_LIMIT_MASK                                                         0x00007C00L
+#define VC_CONGESTION_CONTROL__UTCL0_LFIFO_LIMIT_MASK                                                         0x007F8000L
+//TXA_TDM_CNTL
+#define TXA_TDM_CNTL__DIS_DONE_INTERLEAVE__SHIFT                                                              0x0
+#define TXA_TDM_CNTL__DIS_CFG_STALL_UPD__SHIFT                                                                0x1
+#define TXA_TDM_CNTL__EN_ADS_SLOT_RELEASE__SHIFT                                                              0x2
+#define TXA_TDM_CNTL__EN_DC_STORE_B128__SHIFT                                                                 0x3
+#define TXA_TDM_CNTL__CTRL_SPARE0__SHIFT                                                                      0x4
+#define TXA_TDM_CNTL__CTRL_SPARE1__SHIFT                                                                      0x5
+#define TXA_TDM_CNTL__CTRL_SPARE2__SHIFT                                                                      0x6
+#define TXA_TDM_CNTL__CTRL_SPARE3__SHIFT                                                                      0x7
+#define TXA_TDM_CNTL__CTRL_SPARE4__SHIFT                                                                      0x8
+#define TXA_TDM_CNTL__DIS_DONE_INTERLEAVE_MASK                                                                0x00000001L
+#define TXA_TDM_CNTL__DIS_CFG_STALL_UPD_MASK                                                                  0x00000002L
+#define TXA_TDM_CNTL__EN_ADS_SLOT_RELEASE_MASK                                                                0x00000004L
+#define TXA_TDM_CNTL__EN_DC_STORE_B128_MASK                                                                   0x00000008L
+#define TXA_TDM_CNTL__CTRL_SPARE0_MASK                                                                        0x00000010L
+#define TXA_TDM_CNTL__CTRL_SPARE1_MASK                                                                        0x00000020L
+#define TXA_TDM_CNTL__CTRL_SPARE2_MASK                                                                        0x00000040L
+#define TXA_TDM_CNTL__CTRL_SPARE3_MASK                                                                        0x00000080L
+#define TXA_TDM_CNTL__CTRL_SPARE4_MASK                                                                        0x00000100L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_gfxudec
+//VGT_TF_RING_SIZE
+#define VGT_TF_RING_SIZE__SIZE__SHIFT                                                                         0x0
+#define VGT_TF_RING_SIZE__SIZE_MASK                                                                           0x0001FFFFL
+//VGT_HS_OFFCHIP_PARAM
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_BUFFERING__SHIFT                                                        0x0
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_GRANULARITY__SHIFT                                                      0xa
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_BUFFERING_MASK                                                          0x000003FFL
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_GRANULARITY_MASK                                                        0x00000C00L
+//GE_POS_RING_BASE
+#define GE_POS_RING_BASE__BASE__SHIFT                                                                         0x0
+#define GE_POS_RING_BASE__BASE_MASK                                                                           0xFFFFFFFFL
+//GE_POS_RING_SIZE
+#define GE_POS_RING_SIZE__MEM_SIZE__SHIFT                                                                     0x0
+#define GE_POS_RING_SIZE__MEM_SIZE_MASK                                                                       0x00003FFFL
+//GE_PRIM_RING_BASE
+#define GE_PRIM_RING_BASE__BASE__SHIFT                                                                        0x0
+#define GE_PRIM_RING_BASE__BASE_MASK                                                                          0xFFFFFFFFL
+//GE_PRIM_RING_SIZE
+#define GE_PRIM_RING_SIZE__MEM_SIZE__SHIFT                                                                    0x0
+#define GE_PRIM_RING_SIZE__SCOPE__SHIFT                                                                       0x10
+#define GE_PRIM_RING_SIZE__PAF_TEMPORAL__SHIFT                                                                0x12
+#define GE_PRIM_RING_SIZE__PAB_TEMPORAL__SHIFT                                                                0x15
+#define GE_PRIM_RING_SIZE__SPEC_DATA_READ__SHIFT                                                              0x18
+#define GE_PRIM_RING_SIZE__FORCE_SE_SCOPE__SHIFT                                                              0x1a
+#define GE_PRIM_RING_SIZE__PAB_NOFILL__SHIFT                                                                  0x1b
+#define GE_PRIM_RING_SIZE__MEM_SIZE_MASK                                                                      0x000007FFL
+#define GE_PRIM_RING_SIZE__SCOPE_MASK                                                                         0x00030000L
+#define GE_PRIM_RING_SIZE__PAF_TEMPORAL_MASK                                                                  0x001C0000L
+#define GE_PRIM_RING_SIZE__PAB_TEMPORAL_MASK                                                                  0x00E00000L
+#define GE_PRIM_RING_SIZE__SPEC_DATA_READ_MASK                                                                0x03000000L
+#define GE_PRIM_RING_SIZE__FORCE_SE_SCOPE_MASK                                                                0x04000000L
+#define GE_PRIM_RING_SIZE__PAB_NOFILL_MASK                                                                    0x08000000L
+//PA_SU_LINE_STIPPLE_VALUE
+#define PA_SU_LINE_STIPPLE_VALUE__LINE_STIPPLE_VALUE__SHIFT                                                   0x0
+#define PA_SU_LINE_STIPPLE_VALUE__LINE_STIPPLE_VALUE_MASK                                                     0x00FFFFFFL
+//PA_SC_LINE_STIPPLE_STATE
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_PTR__SHIFT                                                          0x0
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_COUNT__SHIFT                                                        0x8
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_PTR_MASK                                                            0x0000000FL
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_COUNT_MASK                                                          0x0000FF00L
+//PA_SC_SCREEN_EXTENT_MIN_0
+#define PA_SC_SCREEN_EXTENT_MIN_0__X__SHIFT                                                                   0x0
+#define PA_SC_SCREEN_EXTENT_MIN_0__Y__SHIFT                                                                   0x10
+#define PA_SC_SCREEN_EXTENT_MIN_0__X_MASK                                                                     0x0000FFFFL
+#define PA_SC_SCREEN_EXTENT_MIN_0__Y_MASK                                                                     0xFFFF0000L
+//PA_SC_SCREEN_EXTENT_MAX_0
+#define PA_SC_SCREEN_EXTENT_MAX_0__X__SHIFT                                                                   0x0
+#define PA_SC_SCREEN_EXTENT_MAX_0__Y__SHIFT                                                                   0x10
+#define PA_SC_SCREEN_EXTENT_MAX_0__X_MASK                                                                     0x0000FFFFL
+#define PA_SC_SCREEN_EXTENT_MAX_0__Y_MASK                                                                     0xFFFF0000L
+//PA_SC_SCREEN_EXTENT_MIN_1
+#define PA_SC_SCREEN_EXTENT_MIN_1__X__SHIFT                                                                   0x0
+#define PA_SC_SCREEN_EXTENT_MIN_1__Y__SHIFT                                                                   0x10
+#define PA_SC_SCREEN_EXTENT_MIN_1__X_MASK                                                                     0x0000FFFFL
+#define PA_SC_SCREEN_EXTENT_MIN_1__Y_MASK                                                                     0xFFFF0000L
+//PA_SC_SCREEN_EXTENT_MAX_1
+#define PA_SC_SCREEN_EXTENT_MAX_1__X__SHIFT                                                                   0x0
+#define PA_SC_SCREEN_EXTENT_MAX_1__Y__SHIFT                                                                   0x10
+#define PA_SC_SCREEN_EXTENT_MAX_1__X_MASK                                                                     0x0000FFFFL
+#define PA_SC_SCREEN_EXTENT_MAX_1__Y_MASK                                                                     0xFFFF0000L
+//PA_SC_P3D_TRAP_SCREEN_HV_EN
+#define PA_SC_P3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT                                              0x0
+#define PA_SC_P3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT                                       0x1
+#define PA_SC_P3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK                                                0x00000001L
+#define PA_SC_P3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK                                         0x00000002L
+//PA_SC_P3D_TRAP_SCREEN_H
+#define PA_SC_P3D_TRAP_SCREEN_H__X_COORD__SHIFT                                                               0x0
+#define PA_SC_P3D_TRAP_SCREEN_H__X_COORD_MASK                                                                 0x0000FFFFL
+//PA_SC_P3D_TRAP_SCREEN_V
+#define PA_SC_P3D_TRAP_SCREEN_V__Y_COORD__SHIFT                                                               0x0
+#define PA_SC_P3D_TRAP_SCREEN_V__Y_COORD_MASK                                                                 0x0000FFFFL
+//PA_SC_P3D_TRAP_SCREEN_OCCURRENCE
+#define PA_SC_P3D_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT                                                        0x0
+#define PA_SC_P3D_TRAP_SCREEN_OCCURRENCE__COUNT_MASK                                                          0x0000FFFFL
+//PA_SC_P3D_TRAP_SCREEN_COUNT
+#define PA_SC_P3D_TRAP_SCREEN_COUNT__COUNT__SHIFT                                                             0x0
+#define PA_SC_P3D_TRAP_SCREEN_COUNT__COUNT_MASK                                                               0x0000FFFFL
+//PA_SC_HP3D_TRAP_SCREEN_HV_EN
+#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT                                             0x0
+#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT                                      0x1
+#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK                                               0x00000001L
+#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK                                        0x00000002L
+//PA_SC_HP3D_TRAP_SCREEN_H
+#define PA_SC_HP3D_TRAP_SCREEN_H__X_COORD__SHIFT                                                              0x0
+#define PA_SC_HP3D_TRAP_SCREEN_H__X_COORD_MASK                                                                0x0000FFFFL
+//PA_SC_HP3D_TRAP_SCREEN_V
+#define PA_SC_HP3D_TRAP_SCREEN_V__Y_COORD__SHIFT                                                              0x0
+#define PA_SC_HP3D_TRAP_SCREEN_V__Y_COORD_MASK                                                                0x0000FFFFL
+//PA_SC_HP3D_TRAP_SCREEN_OCCURRENCE
+#define PA_SC_HP3D_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT                                                       0x0
+#define PA_SC_HP3D_TRAP_SCREEN_OCCURRENCE__COUNT_MASK                                                         0x0000FFFFL
+//PA_SC_HP3D_TRAP_SCREEN_COUNT
+#define PA_SC_HP3D_TRAP_SCREEN_COUNT__COUNT__SHIFT                                                            0x0
+#define PA_SC_HP3D_TRAP_SCREEN_COUNT__COUNT_MASK                                                              0x0000FFFFL
+//PA_SC_TRAP_SCREEN_HV_EN
+#define PA_SC_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT                                                  0x0
+#define PA_SC_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT                                           0x1
+#define PA_SC_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK                                                    0x00000001L
+#define PA_SC_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK                                             0x00000002L
+//PA_SC_TRAP_SCREEN_H
+#define PA_SC_TRAP_SCREEN_H__X_COORD__SHIFT                                                                   0x0
+#define PA_SC_TRAP_SCREEN_H__X_COORD_MASK                                                                     0x0000FFFFL
+//PA_SC_TRAP_SCREEN_V
+#define PA_SC_TRAP_SCREEN_V__Y_COORD__SHIFT                                                                   0x0
+#define PA_SC_TRAP_SCREEN_V__Y_COORD_MASK                                                                     0x0000FFFFL
+//PA_SC_TRAP_SCREEN_OCCURRENCE
+#define PA_SC_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT                                                            0x0
+#define PA_SC_TRAP_SCREEN_OCCURRENCE__COUNT_MASK                                                              0x0000FFFFL
+//PA_SC_TRAP_SCREEN_COUNT
+#define PA_SC_TRAP_SCREEN_COUNT__COUNT__SHIFT                                                                 0x0
+#define PA_SC_TRAP_SCREEN_COUNT__COUNT_MASK                                                                   0x0000FFFFL
+//SQ_THREAD_TRACE_USERDATA_0
+#define SQ_THREAD_TRACE_USERDATA_0__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_0__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_1
+#define SQ_THREAD_TRACE_USERDATA_1__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_1__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_2
+#define SQ_THREAD_TRACE_USERDATA_2__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_2__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_3
+#define SQ_THREAD_TRACE_USERDATA_3__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_3__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_4
+#define SQ_THREAD_TRACE_USERDATA_4__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_4__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_5
+#define SQ_THREAD_TRACE_USERDATA_5__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_5__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_6
+#define SQ_THREAD_TRACE_USERDATA_6__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_6__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_7
+#define SQ_THREAD_TRACE_USERDATA_7__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_7__DATA_MASK                                                                 0xFFFFFFFFL
+//SQC_CACHES
+#define SQC_CACHES__TARGET_INST__SHIFT                                                                        0x0
+#define SQC_CACHES__TARGET_DATA__SHIFT                                                                        0x1
+#define SQC_CACHES__INVALIDATE__SHIFT                                                                         0x2
+#define SQC_CACHES__COMPLETE__SHIFT                                                                           0x10
+#define SQC_CACHES__TARGET_INST_MASK                                                                          0x00000001L
+#define SQC_CACHES__TARGET_DATA_MASK                                                                          0x00000002L
+#define SQC_CACHES__INVALIDATE_MASK                                                                           0x00000004L
+#define SQC_CACHES__COMPLETE_MASK                                                                             0x00010000L
+//TA_CS_BC_BASE_ADDR
+#define TA_CS_BC_BASE_ADDR__ADDRESS__SHIFT                                                                    0x0
+#define TA_CS_BC_BASE_ADDR__ADDRESS_MASK                                                                      0xFFFFFFFFL
+//TA_CS_BC_BASE_ADDR_HI
+#define TA_CS_BC_BASE_ADDR_HI__ADDRESS__SHIFT                                                                 0x0
+#define TA_CS_BC_BASE_ADDR_HI__ADDRESS_MASK                                                                   0x000000FFL
+//DB_OCCLUSION_COUNT0_LOW
+#define DB_OCCLUSION_COUNT0_LOW__COUNT_LOW__SHIFT                                                             0x0
+#define DB_OCCLUSION_COUNT0_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
+//DB_OCCLUSION_COUNT0_HI
+#define DB_OCCLUSION_COUNT0_HI__COUNT_HI__SHIFT                                                               0x0
+#define DB_OCCLUSION_COUNT0_HI__COUNT_HI_MASK                                                                 0x7FFFFFFFL
+//DB_OCCLUSION_COUNT1_LOW
+#define DB_OCCLUSION_COUNT1_LOW__COUNT_LOW__SHIFT                                                             0x0
+#define DB_OCCLUSION_COUNT1_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
+//DB_OCCLUSION_COUNT1_HI
+#define DB_OCCLUSION_COUNT1_HI__COUNT_HI__SHIFT                                                               0x0
+#define DB_OCCLUSION_COUNT1_HI__COUNT_HI_MASK                                                                 0x7FFFFFFFL
+//DB_OCCLUSION_COUNT2_LOW
+#define DB_OCCLUSION_COUNT2_LOW__COUNT_LOW__SHIFT                                                             0x0
+#define DB_OCCLUSION_COUNT2_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
+//DB_OCCLUSION_COUNT2_HI
+#define DB_OCCLUSION_COUNT2_HI__COUNT_HI__SHIFT                                                               0x0
+#define DB_OCCLUSION_COUNT2_HI__COUNT_HI_MASK                                                                 0x7FFFFFFFL
+//DB_OCCLUSION_COUNT3_LOW
+#define DB_OCCLUSION_COUNT3_LOW__COUNT_LOW__SHIFT                                                             0x0
+#define DB_OCCLUSION_COUNT3_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
+//DB_OCCLUSION_COUNT3_HI
+#define DB_OCCLUSION_COUNT3_HI__COUNT_HI__SHIFT                                                               0x0
+#define DB_OCCLUSION_COUNT3_HI__COUNT_HI_MASK                                                                 0x7FFFFFFFL
+//SPI_CONFIG_CNTL
+#define SPI_CONFIG_CNTL__GPR_WRITE_PRIORITY__SHIFT                                                            0x0
+#define SPI_CONFIG_CNTL__EXP_PRIORITY_ORDER__SHIFT                                                            0x15
+#define SPI_CONFIG_CNTL__ENABLE_SQG_TOP_EVENTS__SHIFT                                                         0x18
+#define SPI_CONFIG_CNTL__ENABLE_SQG_BOP_EVENTS__SHIFT                                                         0x19
+#define SPI_CONFIG_CNTL__ALLOC_ARB_LRU_ENA__SHIFT                                                             0x1c
+#define SPI_CONFIG_CNTL__EXP_ARB_LRU_ENA__SHIFT                                                               0x1d
+#define SPI_CONFIG_CNTL__PS_PKR_PRIORITY_CNTL__SHIFT                                                          0x1e
+#define SPI_CONFIG_CNTL__GPR_WRITE_PRIORITY_MASK                                                              0x001FFFFFL
+#define SPI_CONFIG_CNTL__EXP_PRIORITY_ORDER_MASK                                                              0x00E00000L
+#define SPI_CONFIG_CNTL__ENABLE_SQG_TOP_EVENTS_MASK                                                           0x01000000L
+#define SPI_CONFIG_CNTL__ENABLE_SQG_BOP_EVENTS_MASK                                                           0x02000000L
+#define SPI_CONFIG_CNTL__ALLOC_ARB_LRU_ENA_MASK                                                               0x10000000L
+#define SPI_CONFIG_CNTL__EXP_ARB_LRU_ENA_MASK                                                                 0x20000000L
+#define SPI_CONFIG_CNTL__PS_PKR_PRIORITY_CNTL_MASK                                                            0xC0000000L
+//SPI_CONFIG_CNTL_1
+#define SPI_CONFIG_CNTL_1__VTX_DONE_DELAY__SHIFT                                                              0x0
+#define SPI_CONFIG_CNTL_1__INTERP_ONE_PRIM_PER_ROW__SHIFT                                                     0x4
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_ENABLE__SHIFT                                                             0x5
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_STRICT__SHIFT                                                             0x7
+#define SPI_CONFIG_CNTL_1__PS_GROUP_TIMEOUT_MODE__SHIFT                                                       0x8
+#define SPI_CONFIG_CNTL_1__OREO_EXPALLOC_STALL__SHIFT                                                         0x9
+#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_CNT__SHIFT                                                             0xa
+#define SPI_CONFIG_CNTL_1__CSC_PWR_SAVE_DISABLE__SHIFT                                                        0xe
+#define SPI_CONFIG_CNTL_1__CSG_PWR_SAVE_DISABLE__SHIFT                                                        0xf
+#define SPI_CONFIG_CNTL_1__MAX_VTX_SYNC_CNT__SHIFT                                                            0x10
+#define SPI_CONFIG_CNTL_1__EN_USER_ACCUM__SHIFT                                                               0x15
+#define SPI_CONFIG_CNTL_1__SA_SCREEN_MAP__SHIFT                                                               0x16
+#define SPI_CONFIG_CNTL_1__PS_GROUP_TIMEOUT__SHIFT                                                            0x17
+#define SPI_CONFIG_CNTL_1__VTX_DONE_DELAY_MASK                                                                0x0000000FL
+#define SPI_CONFIG_CNTL_1__INTERP_ONE_PRIM_PER_ROW_MASK                                                       0x00000010L
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_ENABLE_MASK                                                               0x00000060L
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_STRICT_MASK                                                               0x00000080L
+#define SPI_CONFIG_CNTL_1__PS_GROUP_TIMEOUT_MODE_MASK                                                         0x00000100L
+#define SPI_CONFIG_CNTL_1__OREO_EXPALLOC_STALL_MASK                                                           0x00000200L
+#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_CNT_MASK                                                               0x00003C00L
+#define SPI_CONFIG_CNTL_1__CSC_PWR_SAVE_DISABLE_MASK                                                          0x00004000L
+#define SPI_CONFIG_CNTL_1__CSG_PWR_SAVE_DISABLE_MASK                                                          0x00008000L
+#define SPI_CONFIG_CNTL_1__MAX_VTX_SYNC_CNT_MASK                                                              0x001F0000L
+#define SPI_CONFIG_CNTL_1__EN_USER_ACCUM_MASK                                                                 0x00200000L
+#define SPI_CONFIG_CNTL_1__SA_SCREEN_MAP_MASK                                                                 0x00400000L
+#define SPI_CONFIG_CNTL_1__PS_GROUP_TIMEOUT_MASK                                                              0xFF800000L
+//SPI_CONFIG_CNTL_2
+#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_REQUEST_CYCLE_OVHD__SHIFT                                    0x0
+#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_GRANT_CYCLE_OVHD__SHIFT                                      0x4
+#define SPI_CONFIG_CNTL_2__PWS_CSG_WAIT_DISABLE__SHIFT                                                        0x8
+#define SPI_CONFIG_CNTL_2__PWS_HS_WAIT_DISABLE__SHIFT                                                         0x9
+#define SPI_CONFIG_CNTL_2__PWS_GS_WAIT_DISABLE__SHIFT                                                         0xa
+#define SPI_CONFIG_CNTL_2__PWS_PS_WAIT_DISABLE__SHIFT                                                         0xb
+#define SPI_CONFIG_CNTL_2__CSC_HALT_ACK_DELAY__SHIFT                                                          0xc
+#define SPI_CONFIG_CNTL_2__SPP_TIMEOUT_CTR__SHIFT                                                             0x11
+#define SPI_CONFIG_CNTL_2__PC_CONTEXT_DONE_SYNC_ENABLE__SHIFT                                                 0x15
+#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_REQUEST_CYCLE_OVHD_MASK                                      0x0000000FL
+#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_GRANT_CYCLE_OVHD_MASK                                        0x000000F0L
+#define SPI_CONFIG_CNTL_2__PWS_CSG_WAIT_DISABLE_MASK                                                          0x00000100L
+#define SPI_CONFIG_CNTL_2__PWS_HS_WAIT_DISABLE_MASK                                                           0x00000200L
+#define SPI_CONFIG_CNTL_2__PWS_GS_WAIT_DISABLE_MASK                                                           0x00000400L
+#define SPI_CONFIG_CNTL_2__PWS_PS_WAIT_DISABLE_MASK                                                           0x00000800L
+#define SPI_CONFIG_CNTL_2__CSC_HALT_ACK_DELAY_MASK                                                            0x0001F000L
+#define SPI_CONFIG_CNTL_2__SPP_TIMEOUT_CTR_MASK                                                               0x001E0000L
+#define SPI_CONFIG_CNTL_2__PC_CONTEXT_DONE_SYNC_ENABLE_MASK                                                   0x00200000L
+//SPI_GS_THROTTLE_CNTL1
+#define SPI_GS_THROTTLE_CNTL1__PH_POLL_INTERVAL__SHIFT                                                        0x0
+#define SPI_GS_THROTTLE_CNTL1__PH_THROTTLE_BASE__SHIFT                                                        0x4
+#define SPI_GS_THROTTLE_CNTL1__PH_THROTTLE_STEP_SIZE__SHIFT                                                   0x8
+#define SPI_GS_THROTTLE_CNTL1__SPI_VGPR_THRESHOLD__SHIFT                                                      0xc
+#define SPI_GS_THROTTLE_CNTL1__SPI_LDS_THRESHOLD__SHIFT                                                       0x10
+#define SPI_GS_THROTTLE_CNTL1__SPI_POLL_INTERVAL__SHIFT                                                       0x14
+#define SPI_GS_THROTTLE_CNTL1__SPI_THROTTLE_BASE__SHIFT                                                       0x18
+#define SPI_GS_THROTTLE_CNTL1__SPI_THROTTLE_STEP_SIZE__SHIFT                                                  0x1c
+#define SPI_GS_THROTTLE_CNTL1__PH_POLL_INTERVAL_MASK                                                          0x0000000FL
+#define SPI_GS_THROTTLE_CNTL1__PH_THROTTLE_BASE_MASK                                                          0x000000F0L
+#define SPI_GS_THROTTLE_CNTL1__PH_THROTTLE_STEP_SIZE_MASK                                                     0x00000F00L
+#define SPI_GS_THROTTLE_CNTL1__SPI_VGPR_THRESHOLD_MASK                                                        0x0000F000L
+#define SPI_GS_THROTTLE_CNTL1__SPI_LDS_THRESHOLD_MASK                                                         0x000F0000L
+#define SPI_GS_THROTTLE_CNTL1__SPI_POLL_INTERVAL_MASK                                                         0x00F00000L
+#define SPI_GS_THROTTLE_CNTL1__SPI_THROTTLE_BASE_MASK                                                         0x0F000000L
+#define SPI_GS_THROTTLE_CNTL1__SPI_THROTTLE_STEP_SIZE_MASK                                                    0xF0000000L
+//SPI_GS_THROTTLE_CNTL2
+#define SPI_GS_THROTTLE_CNTL2__SPI_THROTTLE_MODE__SHIFT                                                       0x0
+#define SPI_GS_THROTTLE_CNTL2__GRP_LIFETIME_THRESHOLD__SHIFT                                                  0x2
+#define SPI_GS_THROTTLE_CNTL2__GRP_LIFETIME_THRESHOLD_FACTOR__SHIFT                                           0x6
+#define SPI_GS_THROTTLE_CNTL2__GRP_LIFETIME_PENALTY1__SHIFT                                                   0x8
+#define SPI_GS_THROTTLE_CNTL2__GRP_LIFETIME_PENALTY2__SHIFT                                                   0xb
+#define SPI_GS_THROTTLE_CNTL2__PS_STALL_THRESHOLD__SHIFT                                                      0xe
+#define SPI_GS_THROTTLE_CNTL2__PH_MODE__SHIFT                                                                 0x10
+#define SPI_GS_THROTTLE_CNTL2__RESERVED__SHIFT                                                                0x11
+#define SPI_GS_THROTTLE_CNTL2__SPI_THROTTLE_MODE_MASK                                                         0x00000003L
+#define SPI_GS_THROTTLE_CNTL2__GRP_LIFETIME_THRESHOLD_MASK                                                    0x0000003CL
+#define SPI_GS_THROTTLE_CNTL2__GRP_LIFETIME_THRESHOLD_FACTOR_MASK                                             0x000000C0L
+#define SPI_GS_THROTTLE_CNTL2__GRP_LIFETIME_PENALTY1_MASK                                                     0x00000700L
+#define SPI_GS_THROTTLE_CNTL2__GRP_LIFETIME_PENALTY2_MASK                                                     0x00003800L
+#define SPI_GS_THROTTLE_CNTL2__PS_STALL_THRESHOLD_MASK                                                        0x0000C000L
+#define SPI_GS_THROTTLE_CNTL2__PH_MODE_MASK                                                                   0x00010000L
+#define SPI_GS_THROTTLE_CNTL2__RESERVED_MASK                                                                  0xFFFE0000L
+//SPI_ATTRIBUTE_RING_BASE
+#define SPI_ATTRIBUTE_RING_BASE__BASE__SHIFT                                                                  0x0
+#define SPI_ATTRIBUTE_RING_BASE__BASE_MASK                                                                    0xFFFFFFFFL
+//SPI_ATTRIBUTE_RING_SIZE
+#define SPI_ATTRIBUTE_RING_SIZE__MEM_SIZE__SHIFT                                                              0x0
+#define SPI_ATTRIBUTE_RING_SIZE__BIG_PAGE__SHIFT                                                              0x10
+#define SPI_ATTRIBUTE_RING_SIZE__L1_POLICY__SHIFT                                                             0x11
+#define SPI_ATTRIBUTE_RING_SIZE__L2_POLICY__SHIFT                                                             0x13
+#define SPI_ATTRIBUTE_RING_SIZE__LLC_NOALLOC__SHIFT                                                           0x15
+#define SPI_ATTRIBUTE_RING_SIZE__GL1_PERF_COUNTER_DISABLE__SHIFT                                              0x16
+#define SPI_ATTRIBUTE_RING_SIZE__MEM_SIZE_MASK                                                                0x000000FFL
+#define SPI_ATTRIBUTE_RING_SIZE__BIG_PAGE_MASK                                                                0x00010000L
+#define SPI_ATTRIBUTE_RING_SIZE__L1_POLICY_MASK                                                               0x00060000L
+#define SPI_ATTRIBUTE_RING_SIZE__L2_POLICY_MASK                                                               0x00180000L
+#define SPI_ATTRIBUTE_RING_SIZE__LLC_NOALLOC_MASK                                                             0x00200000L
+#define SPI_ATTRIBUTE_RING_SIZE__GL1_PERF_COUNTER_DISABLE_MASK                                                0x00400000L
+//SPI_SQG_EVENT_CTL
+#define SPI_SQG_EVENT_CTL__ENABLE_SQG_TOP_EVENTS__SHIFT                                                       0x0
+#define SPI_SQG_EVENT_CTL__ENABLE_SQG_BOP_EVENTS__SHIFT                                                       0x1
+#define SPI_SQG_EVENT_CTL__ENABLE_SQG_TOP_EVENTS_MASK                                                         0x00000001L
+#define SPI_SQG_EVENT_CTL__ENABLE_SQG_BOP_EVENTS_MASK                                                         0x00000002L
+//SPI_COMPUTE_WGS_CONTROL
+#define SPI_COMPUTE_WGS_CONTROL__WGS_ENABLE__SHIFT                                                            0x0
+#define SPI_COMPUTE_WGS_CONTROL__WGS_ENABLE_MASK                                                              0x00000001L
+//SPI_GRP_LAUNCH_GUARANTEE_ENABLE
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__ENABLE__SHIFT                                                        0x0
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__HS_ASSIST_EN__SHIFT                                                  0x1
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__GS_ASSIST_EN__SHIFT                                                  0x2
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__MRT_ASSIST_EN__SHIFT                                                 0x3
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__CS_GLG_DISABLE__SHIFT                                                0x4
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__GFX_NUM_LOCK_WGP__SHIFT                                              0x5
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__CS_NUM_LOCK_WGP__SHIFT                                               0x8
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__LOCK_PERIOD__SHIFT                                                   0xb
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__LOCK_MAINT_COUNT__SHIFT                                              0xf
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__CLUSTER_LOCK_DISABLE__SHIFT                                          0x12
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__ENABLE_MASK                                                          0x00000001L
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__HS_ASSIST_EN_MASK                                                    0x00000002L
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__GS_ASSIST_EN_MASK                                                    0x00000004L
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__MRT_ASSIST_EN_MASK                                                   0x00000008L
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__CS_GLG_DISABLE_MASK                                                  0x00000010L
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__GFX_NUM_LOCK_WGP_MASK                                                0x000000E0L
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__CS_NUM_LOCK_WGP_MASK                                                 0x00000700L
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__LOCK_PERIOD_MASK                                                     0x00007800L
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__LOCK_MAINT_COUNT_MASK                                                0x00038000L
+#define SPI_GRP_LAUNCH_GUARANTEE_ENABLE__CLUSTER_LOCK_DISABLE_MASK                                            0x00040000L
+//SPI_GRP_LAUNCH_GUARANTEE_CTRL
+#define SPI_GRP_LAUNCH_GUARANTEE_CTRL__NUM_MRT_THRESHOLD__SHIFT                                               0x0
+#define SPI_GRP_LAUNCH_GUARANTEE_CTRL__GFX_PENDING_THRESHOLD__SHIFT                                           0x3
+#define SPI_GRP_LAUNCH_GUARANTEE_CTRL__PRIORITY_LOST_THRESHOLD__SHIFT                                         0x6
+#define SPI_GRP_LAUNCH_GUARANTEE_CTRL__ALLOC_SUCCESS_THRESHOLD__SHIFT                                         0xa
+#define SPI_GRP_LAUNCH_GUARANTEE_CTRL__GFX_WAVE_THRESHOLD_HIGH__SHIFT                                         0xe
+#define SPI_GRP_LAUNCH_GUARANTEE_CTRL__CS_WAVE_THRESHOLD_HIGH__SHIFT                                          0x13
+#define SPI_GRP_LAUNCH_GUARANTEE_CTRL__CU_MASK_ROTATE_PERIODS__SHIFT                                          0x18
+#define SPI_GRP_LAUNCH_GUARANTEE_CTRL__EQUAL_PRIORITY_LOST_THRESHOLD__SHIFT                                   0x1a
+#define SPI_GRP_LAUNCH_GUARANTEE_CTRL__NUM_MRT_THRESHOLD_MASK                                                 0x00000007L
+#define SPI_GRP_LAUNCH_GUARANTEE_CTRL__GFX_PENDING_THRESHOLD_MASK                                             0x00000038L
+#define SPI_GRP_LAUNCH_GUARANTEE_CTRL__PRIORITY_LOST_THRESHOLD_MASK                                           0x000003C0L
+#define SPI_GRP_LAUNCH_GUARANTEE_CTRL__ALLOC_SUCCESS_THRESHOLD_MASK                                           0x00003C00L
+#define SPI_GRP_LAUNCH_GUARANTEE_CTRL__GFX_WAVE_THRESHOLD_HIGH_MASK                                           0x0007C000L
+#define SPI_GRP_LAUNCH_GUARANTEE_CTRL__CS_WAVE_THRESHOLD_HIGH_MASK                                            0x00F80000L
+#define SPI_GRP_LAUNCH_GUARANTEE_CTRL__CU_MASK_ROTATE_PERIODS_MASK                                            0x03000000L
+#define SPI_GRP_LAUNCH_GUARANTEE_CTRL__EQUAL_PRIORITY_LOST_THRESHOLD_MASK                                     0x3C000000L
+//SPI_GOG_ALLOCATION_CTRL
+#define SPI_GOG_ALLOCATION_CTRL__GOG_ALLOCATION_TIME_OUT__SHIFT                                               0x0
+#define SPI_GOG_ALLOCATION_CTRL__GOG_RETRY_DELAY__SHIFT                                                       0x6
+#define SPI_GOG_ALLOCATION_CTRL__GOG_ALLOCATION_TIME_OUT_MASK                                                 0x0000003FL
+#define SPI_GOG_ALLOCATION_CTRL__GOG_RETRY_DELAY_MASK                                                         0x00000FC0L
+//SPI_TCP_CNTL
+#define SPI_TCP_CNTL__DEFAULT_LDS_PARTITIONS__SHIFT                                                           0x0
+#define SPI_TCP_CNTL__MIN_LDS_PARTITIONS__SHIFT                                                               0x4
+#define SPI_TCP_CNTL__MAX_LDS_PARTITIONS__SHIFT                                                               0xc
+#define SPI_TCP_CNTL__IDLE_ALLOC_OPT_DIS__SHIFT                                                               0x10
+#define SPI_TCP_CNTL__PARTIAL_DRAIN_DIS__SHIFT                                                                0x11
+#define SPI_TCP_CNTL__LDS_PINGPONG_DIS__SHIFT                                                                 0x12
+#define SPI_TCP_CNTL__DEFAULT_LDS_PARTITIONS_MASK                                                             0x00000007L
+#define SPI_TCP_CNTL__MIN_LDS_PARTITIONS_MASK                                                                 0x00000070L
+#define SPI_TCP_CNTL__MAX_LDS_PARTITIONS_MASK                                                                 0x00007000L
+#define SPI_TCP_CNTL__IDLE_ALLOC_OPT_DIS_MASK                                                                 0x00010000L
+#define SPI_TCP_CNTL__PARTIAL_DRAIN_DIS_MASK                                                                  0x00020000L
+#define SPI_TCP_CNTL__LDS_PINGPONG_DIS_MASK                                                                   0x00040000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_comp_wgsdec
+//WGS_COMPUTE_DISPATCH_INITIATOR
+#define WGS_COMPUTE_DISPATCH_INITIATOR__COMPUTE_SHADER_EN__SHIFT                                              0x0
+#define WGS_COMPUTE_DISPATCH_INITIATOR__PARTIAL_TG_EN__SHIFT                                                  0x1
+#define WGS_COMPUTE_DISPATCH_INITIATOR__FORCE_START_AT_000__SHIFT                                             0x2
+#define WGS_COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_ENBL__SHIFT                                            0x3
+#define WGS_COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_MODE__SHIFT                                            0x4
+#define WGS_COMPUTE_DISPATCH_INITIATOR__USE_THREAD_DIMENSIONS__SHIFT                                          0x5
+#define WGS_COMPUTE_DISPATCH_INITIATOR__ORDER_MODE__SHIFT                                                     0x6
+#define WGS_COMPUTE_DISPATCH_INITIATOR__SCALAR_L1_INV_VOL__SHIFT                                              0xa
+#define WGS_COMPUTE_DISPATCH_INITIATOR__VECTOR_L1_INV_VOL__SHIFT                                              0xb
+#define WGS_COMPUTE_DISPATCH_INITIATOR__PING_PONG_EN__SHIFT                                                   0xc
+#define WGS_COMPUTE_DISPATCH_INITIATOR__TUNNEL_ENABLE__SHIFT                                                  0xd
+#define WGS_COMPUTE_DISPATCH_INITIATOR__RESTORE__SHIFT                                                        0xe
+#define WGS_COMPUTE_DISPATCH_INITIATOR__CS_W32_EN__SHIFT                                                      0xf
+#define WGS_COMPUTE_DISPATCH_INITIATOR__AMP_SHADER_EN__SHIFT                                                  0x10
+#define WGS_COMPUTE_DISPATCH_INITIATOR__DISABLE_DISP_PREMPT_EN__SHIFT                                         0x11
+#define WGS_COMPUTE_DISPATCH_INITIATOR__INTERLEAVE_2D_EN__SHIFT                                               0x12
+#define WGS_COMPUTE_DISPATCH_INITIATOR__WGS_DISPATCH__SHIFT                                                   0x13
+#define WGS_COMPUTE_DISPATCH_INITIATOR__TTRACE_QUEUE_ID__SHIFT                                                0x1d
+#define WGS_COMPUTE_DISPATCH_INITIATOR__COMPUTE_SHADER_EN_MASK                                                0x00000001L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__PARTIAL_TG_EN_MASK                                                    0x00000002L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__FORCE_START_AT_000_MASK                                               0x00000004L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_ENBL_MASK                                              0x00000008L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_MODE_MASK                                              0x00000010L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__USE_THREAD_DIMENSIONS_MASK                                            0x00000020L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__ORDER_MODE_MASK                                                       0x00000040L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__SCALAR_L1_INV_VOL_MASK                                                0x00000400L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__VECTOR_L1_INV_VOL_MASK                                                0x00000800L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__PING_PONG_EN_MASK                                                     0x00001000L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__TUNNEL_ENABLE_MASK                                                    0x00002000L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__RESTORE_MASK                                                          0x00004000L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__CS_W32_EN_MASK                                                        0x00008000L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__AMP_SHADER_EN_MASK                                                    0x00010000L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__DISABLE_DISP_PREMPT_EN_MASK                                           0x00020000L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__INTERLEAVE_2D_EN_MASK                                                 0x00040000L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__WGS_DISPATCH_MASK                                                     0x00080000L
+#define WGS_COMPUTE_DISPATCH_INITIATOR__TTRACE_QUEUE_ID_MASK                                                  0xE0000000L
+//WGS_COMPUTE_DIM_X
+#define WGS_COMPUTE_DIM_X__SIZE__SHIFT                                                                        0x0
+#define WGS_COMPUTE_DIM_X__SIZE_MASK                                                                          0xFFFFFFFFL
+//WGS_COMPUTE_DIM_Y
+#define WGS_COMPUTE_DIM_Y__SIZE__SHIFT                                                                        0x0
+#define WGS_COMPUTE_DIM_Y__SIZE_MASK                                                                          0xFFFFFFFFL
+//WGS_COMPUTE_DIM_Z
+#define WGS_COMPUTE_DIM_Z__SIZE__SHIFT                                                                        0x0
+#define WGS_COMPUTE_DIM_Z__SIZE_MASK                                                                          0xFFFFFFFFL
+//WGS_COMPUTE_START_X
+#define WGS_COMPUTE_START_X__START__SHIFT                                                                     0x0
+#define WGS_COMPUTE_START_X__START_MASK                                                                       0xFFFFFFFFL
+//WGS_COMPUTE_START_Y
+#define WGS_COMPUTE_START_Y__START__SHIFT                                                                     0x0
+#define WGS_COMPUTE_START_Y__START_MASK                                                                       0xFFFFFFFFL
+//WGS_COMPUTE_START_Z
+#define WGS_COMPUTE_START_Z__START__SHIFT                                                                     0x0
+#define WGS_COMPUTE_START_Z__START_MASK                                                                       0xFFFFFFFFL
+//WGS_COMPUTE_NUM_THREAD_X
+#define WGS_COMPUTE_NUM_THREAD_X__NUM_THREAD_FULL__SHIFT                                                      0x0
+#define WGS_COMPUTE_NUM_THREAD_X__INTERLEAVE_BITS_X__SHIFT                                                    0xd
+#define WGS_COMPUTE_NUM_THREAD_X__NUM_THREAD_PARTIAL__SHIFT                                                   0x10
+#define WGS_COMPUTE_NUM_THREAD_X__NUM_THREAD_FULL_MASK                                                        0x00001FFFL
+#define WGS_COMPUTE_NUM_THREAD_X__INTERLEAVE_BITS_X_MASK                                                      0x0000E000L
+#define WGS_COMPUTE_NUM_THREAD_X__NUM_THREAD_PARTIAL_MASK                                                     0xFFFF0000L
+//WGS_COMPUTE_NUM_THREAD_Y
+#define WGS_COMPUTE_NUM_THREAD_Y__NUM_THREAD_FULL__SHIFT                                                      0x0
+#define WGS_COMPUTE_NUM_THREAD_Y__INTERLEAVE_BITS_Y__SHIFT                                                    0xd
+#define WGS_COMPUTE_NUM_THREAD_Y__NUM_THREAD_PARTIAL__SHIFT                                                   0x10
+#define WGS_COMPUTE_NUM_THREAD_Y__NUM_THREAD_FULL_MASK                                                        0x00001FFFL
+#define WGS_COMPUTE_NUM_THREAD_Y__INTERLEAVE_BITS_Y_MASK                                                      0x0000E000L
+#define WGS_COMPUTE_NUM_THREAD_Y__NUM_THREAD_PARTIAL_MASK                                                     0xFFFF0000L
+//WGS_COMPUTE_NUM_THREAD_Z
+#define WGS_COMPUTE_NUM_THREAD_Z__NUM_THREAD_FULL__SHIFT                                                      0x0
+#define WGS_COMPUTE_NUM_THREAD_Z__NUM_THREAD_PARTIAL__SHIFT                                                   0x10
+#define WGS_COMPUTE_NUM_THREAD_Z__NUM_THREAD_FULL_MASK                                                        0x0000FFFFL
+#define WGS_COMPUTE_NUM_THREAD_Z__NUM_THREAD_PARTIAL_MASK                                                     0xFFFF0000L
+//WGS_COMPUTE_PIPELINESTAT_ENABLE
+#define WGS_COMPUTE_PIPELINESTAT_ENABLE__PIPELINESTAT_ENABLE__SHIFT                                           0x0
+#define WGS_COMPUTE_PIPELINESTAT_ENABLE__PIPELINESTAT_ENABLE_MASK                                             0x00000001L
+//WGS_COMPUTE_PERFCOUNT_ENABLE
+#define WGS_COMPUTE_PERFCOUNT_ENABLE__PERFCOUNT_ENABLE__SHIFT                                                 0x0
+#define WGS_COMPUTE_PERFCOUNT_ENABLE__PERFCOUNT_ENABLE_MASK                                                   0x00000001L
+//WGS_COMPUTE_PGM_LO
+#define WGS_COMPUTE_PGM_LO__DATA__SHIFT                                                                       0x0
+#define WGS_COMPUTE_PGM_LO__DATA_MASK                                                                         0xFFFFFFFFL
+//WGS_COMPUTE_PGM_HI
+#define WGS_COMPUTE_PGM_HI__DATA__SHIFT                                                                       0x0
+#define WGS_COMPUTE_PGM_HI__DATA_MASK                                                                         0x000000FFL
+//WGS_COMPUTE_DISPATCH_PKT_ADDR_LO
+#define WGS_COMPUTE_DISPATCH_PKT_ADDR_LO__DATA__SHIFT                                                         0x0
+#define WGS_COMPUTE_DISPATCH_PKT_ADDR_LO__DATA_MASK                                                           0xFFFFFFFFL
+//WGS_COMPUTE_DISPATCH_PKT_ADDR_HI
+#define WGS_COMPUTE_DISPATCH_PKT_ADDR_HI__DATA__SHIFT                                                         0x0
+#define WGS_COMPUTE_DISPATCH_PKT_ADDR_HI__DATA_MASK                                                           0x000000FFL
+//WGS_COMPUTE_DISPATCH_SCRATCH_BASE_LO
+#define WGS_COMPUTE_DISPATCH_SCRATCH_BASE_LO__DATA__SHIFT                                                     0x0
+#define WGS_COMPUTE_DISPATCH_SCRATCH_BASE_LO__DATA_MASK                                                       0xFFFFFFFFL
+//WGS_COMPUTE_DISPATCH_SCRATCH_BASE_HI
+#define WGS_COMPUTE_DISPATCH_SCRATCH_BASE_HI__DATA__SHIFT                                                     0x0
+#define WGS_COMPUTE_DISPATCH_SCRATCH_BASE_HI__DATA_MASK                                                       0x000000FFL
+//WGS_COMPUTE_PGM_RSRC1
+#define WGS_COMPUTE_PGM_RSRC1__VGPRS__SHIFT                                                                   0x0
+#define WGS_COMPUTE_PGM_RSRC1__SGPRS__SHIFT                                                                   0x6
+#define WGS_COMPUTE_PGM_RSRC1__PRIORITY__SHIFT                                                                0xa
+#define WGS_COMPUTE_PGM_RSRC1__FLOAT_MODE__SHIFT                                                              0xc
+#define WGS_COMPUTE_PGM_RSRC1__PRIV__SHIFT                                                                    0x14
+#define WGS_COMPUTE_PGM_RSRC1__WG_RR_EN__SHIFT                                                                0x15
+#define WGS_COMPUTE_PGM_RSRC1__DEBUG_MODE__SHIFT                                                              0x16
+#define WGS_COMPUTE_PGM_RSRC1__DISABLE_PERF__SHIFT                                                            0x17
+#define WGS_COMPUTE_PGM_RSRC1__BULKY__SHIFT                                                                   0x18
+#define WGS_COMPUTE_PGM_RSRC1__CDBG_USER__SHIFT                                                               0x19
+#define WGS_COMPUTE_PGM_RSRC1__FP16_OVFL__SHIFT                                                               0x1a
+#define WGS_COMPUTE_PGM_RSRC1__WGP_MODE__SHIFT                                                                0x1d
+#define WGS_COMPUTE_PGM_RSRC1__FWD_PROGRESS__SHIFT                                                            0x1f
+#define WGS_COMPUTE_PGM_RSRC1__VGPRS_MASK                                                                     0x0000003FL
+#define WGS_COMPUTE_PGM_RSRC1__SGPRS_MASK                                                                     0x000003C0L
+#define WGS_COMPUTE_PGM_RSRC1__PRIORITY_MASK                                                                  0x00000C00L
+#define WGS_COMPUTE_PGM_RSRC1__FLOAT_MODE_MASK                                                                0x000FF000L
+#define WGS_COMPUTE_PGM_RSRC1__PRIV_MASK                                                                      0x00100000L
+#define WGS_COMPUTE_PGM_RSRC1__WG_RR_EN_MASK                                                                  0x00200000L
+#define WGS_COMPUTE_PGM_RSRC1__DEBUG_MODE_MASK                                                                0x00400000L
+#define WGS_COMPUTE_PGM_RSRC1__DISABLE_PERF_MASK                                                              0x00800000L
+#define WGS_COMPUTE_PGM_RSRC1__BULKY_MASK                                                                     0x01000000L
+#define WGS_COMPUTE_PGM_RSRC1__CDBG_USER_MASK                                                                 0x02000000L
+#define WGS_COMPUTE_PGM_RSRC1__FP16_OVFL_MASK                                                                 0x04000000L
+#define WGS_COMPUTE_PGM_RSRC1__WGP_MODE_MASK                                                                  0x20000000L
+#define WGS_COMPUTE_PGM_RSRC1__FWD_PROGRESS_MASK                                                              0x80000000L
+//WGS_COMPUTE_PGM_RSRC2
+#define WGS_COMPUTE_PGM_RSRC2__SCRATCH_EN__SHIFT                                                              0x0
+#define WGS_COMPUTE_PGM_RSRC2__USER_SGPR__SHIFT                                                               0x1
+#define WGS_COMPUTE_PGM_RSRC2__DYNAMIC_VGPR__SHIFT                                                            0x6
+#define WGS_COMPUTE_PGM_RSRC2__TGID_X_EN__SHIFT                                                               0x7
+#define WGS_COMPUTE_PGM_RSRC2__TGID_Y_EN__SHIFT                                                               0x8
+#define WGS_COMPUTE_PGM_RSRC2__TGID_Z_EN__SHIFT                                                               0x9
+#define WGS_COMPUTE_PGM_RSRC2__TG_SIZE_EN__SHIFT                                                              0xa
+#define WGS_COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT__SHIFT                                                          0xb
+#define WGS_COMPUTE_PGM_RSRC2__EXCP_EN_MSB__SHIFT                                                             0xd
+#define WGS_COMPUTE_PGM_RSRC2__LDS_SIZE__SHIFT                                                                0xf
+#define WGS_COMPUTE_PGM_RSRC2__EXCP_EN__SHIFT                                                                 0x18
+#define WGS_COMPUTE_PGM_RSRC2__WGP_TAKEOVER__SHIFT                                                            0x1f
+#define WGS_COMPUTE_PGM_RSRC2__SCRATCH_EN_MASK                                                                0x00000001L
+#define WGS_COMPUTE_PGM_RSRC2__USER_SGPR_MASK                                                                 0x0000003EL
+#define WGS_COMPUTE_PGM_RSRC2__DYNAMIC_VGPR_MASK                                                              0x00000040L
+#define WGS_COMPUTE_PGM_RSRC2__TGID_X_EN_MASK                                                                 0x00000080L
+#define WGS_COMPUTE_PGM_RSRC2__TGID_Y_EN_MASK                                                                 0x00000100L
+#define WGS_COMPUTE_PGM_RSRC2__TGID_Z_EN_MASK                                                                 0x00000200L
+#define WGS_COMPUTE_PGM_RSRC2__TG_SIZE_EN_MASK                                                                0x00000400L
+#define WGS_COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT_MASK                                                            0x00001800L
+#define WGS_COMPUTE_PGM_RSRC2__EXCP_EN_MSB_MASK                                                               0x00006000L
+#define WGS_COMPUTE_PGM_RSRC2__LDS_SIZE_MASK                                                                  0x00FF8000L
+#define WGS_COMPUTE_PGM_RSRC2__EXCP_EN_MASK                                                                   0x7F000000L
+#define WGS_COMPUTE_PGM_RSRC2__WGP_TAKEOVER_MASK                                                              0x80000000L
+//WGS_COMPUTE_VMID
+#define WGS_COMPUTE_VMID__DATA__SHIFT                                                                         0x0
+#define WGS_COMPUTE_VMID__DATA_MASK                                                                           0x0000000FL
+//WGS_COMPUTE_RESOURCE_LIMITS
+#define WGS_COMPUTE_RESOURCE_LIMITS__WAVES_PER_SH__SHIFT                                                      0x0
+#define WGS_COMPUTE_RESOURCE_LIMITS__TG_PER_CU__SHIFT                                                         0xc
+#define WGS_COMPUTE_RESOURCE_LIMITS__LOCK_THRESHOLD__SHIFT                                                    0x10
+#define WGS_COMPUTE_RESOURCE_LIMITS__SIMD_DEST_CNTL__SHIFT                                                    0x16
+#define WGS_COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST__SHIFT                                                   0x17
+#define WGS_COMPUTE_RESOURCE_LIMITS__CU_GROUP_COUNT__SHIFT                                                    0x18
+#define WGS_COMPUTE_RESOURCE_LIMITS__WAVES_PER_SH_MASK                                                        0x000003FFL
+#define WGS_COMPUTE_RESOURCE_LIMITS__TG_PER_CU_MASK                                                           0x0000F000L
+#define WGS_COMPUTE_RESOURCE_LIMITS__LOCK_THRESHOLD_MASK                                                      0x003F0000L
+#define WGS_COMPUTE_RESOURCE_LIMITS__SIMD_DEST_CNTL_MASK                                                      0x00400000L
+#define WGS_COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST_MASK                                                     0x00800000L
+#define WGS_COMPUTE_RESOURCE_LIMITS__CU_GROUP_COUNT_MASK                                                      0x07000000L
+//WGS_COMPUTE_DESTINATION_EN_SE0
+#define WGS_COMPUTE_DESTINATION_EN_SE0__CU_EN__SHIFT                                                          0x0
+#define WGS_COMPUTE_DESTINATION_EN_SE0__CU_EN_MASK                                                            0xFFFFFFFFL
+//WGS_COMPUTE_STATIC_THREAD_MGMT_SE0
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE0__SA0_CU_EN__SHIFT                                                  0x0
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE0__SA1_CU_EN__SHIFT                                                  0x10
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE0__SA0_CU_EN_MASK                                                    0x0000FFFFL
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE0__SA1_CU_EN_MASK                                                    0xFFFF0000L
+//WGS_COMPUTE_DESTINATION_EN_SE1
+#define WGS_COMPUTE_DESTINATION_EN_SE1__CU_EN__SHIFT                                                          0x0
+#define WGS_COMPUTE_DESTINATION_EN_SE1__CU_EN_MASK                                                            0xFFFFFFFFL
+//WGS_COMPUTE_STATIC_THREAD_MGMT_SE1
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE1__SA0_CU_EN__SHIFT                                                  0x0
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE1__SA1_CU_EN__SHIFT                                                  0x10
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE1__SA0_CU_EN_MASK                                                    0x0000FFFFL
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE1__SA1_CU_EN_MASK                                                    0xFFFF0000L
+//WGS_COMPUTE_TMPRING_SIZE
+#define WGS_COMPUTE_TMPRING_SIZE__WAVES__SHIFT                                                                0x0
+#define WGS_COMPUTE_TMPRING_SIZE__WAVESIZE__SHIFT                                                             0xc
+#define WGS_COMPUTE_TMPRING_SIZE__WAVES_MASK                                                                  0x00000FFFL
+#define WGS_COMPUTE_TMPRING_SIZE__WAVESIZE_MASK                                                               0x3FFFF000L
+//WGS_COMPUTE_DESTINATION_EN_SE2
+#define WGS_COMPUTE_DESTINATION_EN_SE2__CU_EN__SHIFT                                                          0x0
+#define WGS_COMPUTE_DESTINATION_EN_SE2__CU_EN_MASK                                                            0xFFFFFFFFL
+//WGS_COMPUTE_STATIC_THREAD_MGMT_SE2
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE2__SA0_CU_EN__SHIFT                                                  0x0
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE2__SA1_CU_EN__SHIFT                                                  0x10
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE2__SA0_CU_EN_MASK                                                    0x0000FFFFL
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE2__SA1_CU_EN_MASK                                                    0xFFFF0000L
+//WGS_COMPUTE_DESTINATION_EN_SE3
+#define WGS_COMPUTE_DESTINATION_EN_SE3__CU_EN__SHIFT                                                          0x0
+#define WGS_COMPUTE_DESTINATION_EN_SE3__CU_EN_MASK                                                            0xFFFFFFFFL
+//WGS_COMPUTE_STATIC_THREAD_MGMT_SE3
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE3__SA0_CU_EN__SHIFT                                                  0x0
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE3__SA1_CU_EN__SHIFT                                                  0x10
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE3__SA0_CU_EN_MASK                                                    0x0000FFFFL
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE3__SA1_CU_EN_MASK                                                    0xFFFF0000L
+//WGS_COMPUTE_RESTART_X
+#define WGS_COMPUTE_RESTART_X__RESTART__SHIFT                                                                 0x0
+#define WGS_COMPUTE_RESTART_X__RESTART_MASK                                                                   0xFFFFFFFFL
+//WGS_COMPUTE_RESTART_Y
+#define WGS_COMPUTE_RESTART_Y__RESTART__SHIFT                                                                 0x0
+#define WGS_COMPUTE_RESTART_Y__RESTART_MASK                                                                   0xFFFFFFFFL
+//WGS_COMPUTE_RESTART_Z
+#define WGS_COMPUTE_RESTART_Z__RESTART__SHIFT                                                                 0x0
+#define WGS_COMPUTE_RESTART_Z__RESTART_MASK                                                                   0xFFFFFFFFL
+//WGS_COMPUTE_THREAD_TRACE_ENABLE
+#define WGS_COMPUTE_THREAD_TRACE_ENABLE__THREAD_TRACE_ENABLE__SHIFT                                           0x0
+#define WGS_COMPUTE_THREAD_TRACE_ENABLE__THREAD_TRACE_ENABLE_MASK                                             0x00000001L
+//WGS_COMPUTE_MISC_RESERVED
+#define WGS_COMPUTE_MISC_RESERVED__SEND_SEID__SHIFT                                                           0x0
+#define WGS_COMPUTE_MISC_RESERVED__RESERVED4__SHIFT                                                           0x4
+#define WGS_COMPUTE_MISC_RESERVED__WAVE_ID_BASE__SHIFT                                                        0x5
+#define WGS_COMPUTE_MISC_RESERVED__SEND_SEID_MASK                                                             0x0000000FL
+#define WGS_COMPUTE_MISC_RESERVED__RESERVED4_MASK                                                             0x00000010L
+#define WGS_COMPUTE_MISC_RESERVED__WAVE_ID_BASE_MASK                                                          0x0001FFE0L
+//WGS_COMPUTE_DISPATCH_ID
+#define WGS_COMPUTE_DISPATCH_ID__DISPATCH_ID__SHIFT                                                           0x0
+#define WGS_COMPUTE_DISPATCH_ID__DISPATCH_ID_MASK                                                             0xFFFFFFFFL
+//WGS_COMPUTE_THREADGROUP_ID
+#define WGS_COMPUTE_THREADGROUP_ID__THREADGROUP_ID__SHIFT                                                     0x0
+#define WGS_COMPUTE_THREADGROUP_ID__THREADGROUP_ID_MASK                                                       0xFFFFFFFFL
+//WGS_COMPUTE_REQ_CTRL
+#define WGS_COMPUTE_REQ_CTRL__SOFT_GROUPING_EN__SHIFT                                                         0x0
+#define WGS_COMPUTE_REQ_CTRL__NUMBER_OF_REQUESTS_PER_CU__SHIFT                                                0x1
+#define WGS_COMPUTE_REQ_CTRL__SOFT_GROUPING_ALLOCATION_TIMEOUT__SHIFT                                         0x5
+#define WGS_COMPUTE_REQ_CTRL__HARD_LOCK_HYSTERESIS__SHIFT                                                     0x9
+#define WGS_COMPUTE_REQ_CTRL__HARD_LOCK_LOW_THRESHOLD__SHIFT                                                  0xa
+#define WGS_COMPUTE_REQ_CTRL__PRODUCER_REQUEST_LOCKOUT__SHIFT                                                 0xf
+#define WGS_COMPUTE_REQ_CTRL__GLOBAL_SCANNING_EN__SHIFT                                                       0x10
+#define WGS_COMPUTE_REQ_CTRL__ALLOCATION_RATE_THROTTLING_THRESHOLD__SHIFT                                     0x11
+#define WGS_COMPUTE_REQ_CTRL__DEDICATED_PREALLOCATION_BUFFER_LIMIT__SHIFT                                     0x14
+#define WGS_COMPUTE_REQ_CTRL__SOFT_GROUPING_EN_MASK                                                           0x00000001L
+#define WGS_COMPUTE_REQ_CTRL__NUMBER_OF_REQUESTS_PER_CU_MASK                                                  0x0000001EL
+#define WGS_COMPUTE_REQ_CTRL__SOFT_GROUPING_ALLOCATION_TIMEOUT_MASK                                           0x000001E0L
+#define WGS_COMPUTE_REQ_CTRL__HARD_LOCK_HYSTERESIS_MASK                                                       0x00000200L
+#define WGS_COMPUTE_REQ_CTRL__HARD_LOCK_LOW_THRESHOLD_MASK                                                    0x00007C00L
+#define WGS_COMPUTE_REQ_CTRL__PRODUCER_REQUEST_LOCKOUT_MASK                                                   0x00008000L
+#define WGS_COMPUTE_REQ_CTRL__GLOBAL_SCANNING_EN_MASK                                                         0x00010000L
+#define WGS_COMPUTE_REQ_CTRL__ALLOCATION_RATE_THROTTLING_THRESHOLD_MASK                                       0x000E0000L
+#define WGS_COMPUTE_REQ_CTRL__DEDICATED_PREALLOCATION_BUFFER_LIMIT_MASK                                       0x07F00000L
+//WGS_COMPUTE_STATIC_THREAD_MGMT_SE8
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE8__SA0_CU_EN__SHIFT                                                  0x0
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE8__SA1_CU_EN__SHIFT                                                  0x10
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE8__SA0_CU_EN_MASK                                                    0x0000FFFFL
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE8__SA1_CU_EN_MASK                                                    0xFFFF0000L
+//WGS_COMPUTE_USER_ACCUM_0
+#define WGS_COMPUTE_USER_ACCUM_0__CONTRIBUTION__SHIFT                                                         0x0
+#define WGS_COMPUTE_USER_ACCUM_0__CONTRIBUTION_MASK                                                           0x0000007FL
+//WGS_COMPUTE_USER_ACCUM_1
+#define WGS_COMPUTE_USER_ACCUM_1__CONTRIBUTION__SHIFT                                                         0x0
+#define WGS_COMPUTE_USER_ACCUM_1__CONTRIBUTION_MASK                                                           0x0000007FL
+//WGS_COMPUTE_USER_ACCUM_2
+#define WGS_COMPUTE_USER_ACCUM_2__CONTRIBUTION__SHIFT                                                         0x0
+#define WGS_COMPUTE_USER_ACCUM_2__CONTRIBUTION_MASK                                                           0x0000007FL
+//WGS_COMPUTE_USER_ACCUM_3
+#define WGS_COMPUTE_USER_ACCUM_3__CONTRIBUTION__SHIFT                                                         0x0
+#define WGS_COMPUTE_USER_ACCUM_3__CONTRIBUTION_MASK                                                           0x0000007FL
+//WGS_COMPUTE_PGM_RSRC3
+#define WGS_COMPUTE_PGM_RSRC3__SHARED_VGPR_CNT__SHIFT                                                         0x0
+#define WGS_COMPUTE_PGM_RSRC3__INST_PREF_SIZE__SHIFT                                                          0x4
+#define WGS_COMPUTE_PGM_RSRC3__GLG_EN__SHIFT                                                                  0xd
+#define WGS_COMPUTE_PGM_RSRC3__IMAGE_OP__SHIFT                                                                0x1f
+#define WGS_COMPUTE_PGM_RSRC3__SHARED_VGPR_CNT_MASK                                                           0x0000000FL
+#define WGS_COMPUTE_PGM_RSRC3__INST_PREF_SIZE_MASK                                                            0x00000FF0L
+#define WGS_COMPUTE_PGM_RSRC3__GLG_EN_MASK                                                                    0x00002000L
+#define WGS_COMPUTE_PGM_RSRC3__IMAGE_OP_MASK                                                                  0x80000000L
+//WGS_COMPUTE_DDID_INDEX
+#define WGS_COMPUTE_DDID_INDEX__INDEX__SHIFT                                                                  0x0
+#define WGS_COMPUTE_DDID_INDEX__INDEX_MASK                                                                    0x000007FFL
+//WGS_COMPUTE_SHADER_CHKSUM
+#define WGS_COMPUTE_SHADER_CHKSUM__CHECKSUM__SHIFT                                                            0x0
+#define WGS_COMPUTE_SHADER_CHKSUM__CHECKSUM_MASK                                                              0xFFFFFFFFL
+//WGS_COMPUTE_STATIC_THREAD_MGMT_SE4
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE4__SA0_CU_EN__SHIFT                                                  0x0
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE4__SA1_CU_EN__SHIFT                                                  0x10
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE4__SA0_CU_EN_MASK                                                    0x0000FFFFL
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE4__SA1_CU_EN_MASK                                                    0xFFFF0000L
+//WGS_COMPUTE_STATIC_THREAD_MGMT_SE5
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE5__SA0_CU_EN__SHIFT                                                  0x0
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE5__SA1_CU_EN__SHIFT                                                  0x10
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE5__SA0_CU_EN_MASK                                                    0x0000FFFFL
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE5__SA1_CU_EN_MASK                                                    0xFFFF0000L
+//WGS_COMPUTE_STATIC_THREAD_MGMT_SE6
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE6__SA0_CU_EN__SHIFT                                                  0x0
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE6__SA1_CU_EN__SHIFT                                                  0x10
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE6__SA0_CU_EN_MASK                                                    0x0000FFFFL
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE6__SA1_CU_EN_MASK                                                    0xFFFF0000L
+//WGS_COMPUTE_STATIC_THREAD_MGMT_SE7
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE7__SA0_CU_EN__SHIFT                                                  0x0
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE7__SA1_CU_EN__SHIFT                                                  0x10
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE7__SA0_CU_EN_MASK                                                    0x0000FFFFL
+#define WGS_COMPUTE_STATIC_THREAD_MGMT_SE7__SA1_CU_EN_MASK                                                    0xFFFF0000L
+//WGS_COMPUTE_DISPATCH_INTERLEAVE
+#define WGS_COMPUTE_DISPATCH_INTERLEAVE__INTERLEAVE_1D__SHIFT                                                 0x0
+#define WGS_COMPUTE_DISPATCH_INTERLEAVE__INTERLEAVE_2D_X_SIZE__SHIFT                                          0x10
+#define WGS_COMPUTE_DISPATCH_INTERLEAVE__INTERLEAVE_2D_Y_SIZE__SHIFT                                          0x18
+#define WGS_COMPUTE_DISPATCH_INTERLEAVE__INTERLEAVE_1D_MASK                                                   0x000003FFL
+#define WGS_COMPUTE_DISPATCH_INTERLEAVE__INTERLEAVE_2D_X_SIZE_MASK                                            0x000F0000L
+#define WGS_COMPUTE_DISPATCH_INTERLEAVE__INTERLEAVE_2D_Y_SIZE_MASK                                            0x0F000000L
+//WGS_COMPUTE_RELAUNCH
+#define WGS_COMPUTE_RELAUNCH__PAYLOAD__SHIFT                                                                  0x0
+#define WGS_COMPUTE_RELAUNCH__IS_EVENT__SHIFT                                                                 0x1e
+#define WGS_COMPUTE_RELAUNCH__IS_STATE__SHIFT                                                                 0x1f
+#define WGS_COMPUTE_RELAUNCH__PAYLOAD_MASK                                                                    0x3FFFFFFFL
+#define WGS_COMPUTE_RELAUNCH__IS_EVENT_MASK                                                                   0x40000000L
+#define WGS_COMPUTE_RELAUNCH__IS_STATE_MASK                                                                   0x80000000L
+//WGS_COMPUTE_WAVE_RESTORE_ADDR_LO
+#define WGS_COMPUTE_WAVE_RESTORE_ADDR_LO__ADDR__SHIFT                                                         0x0
+#define WGS_COMPUTE_WAVE_RESTORE_ADDR_LO__ADDR_MASK                                                           0xFFFFFFFFL
+//WGS_COMPUTE_WAVE_RESTORE_ADDR_HI
+#define WGS_COMPUTE_WAVE_RESTORE_ADDR_HI__ADDR__SHIFT                                                         0x0
+#define WGS_COMPUTE_WAVE_RESTORE_ADDR_HI__ADDR_MASK                                                           0x0000FFFFL
+//WGS_COMPUTE_RELAUNCH2
+#define WGS_COMPUTE_RELAUNCH2__PAYLOAD__SHIFT                                                                 0x0
+#define WGS_COMPUTE_RELAUNCH2__IS_EVENT__SHIFT                                                                0x1e
+#define WGS_COMPUTE_RELAUNCH2__IS_STATE__SHIFT                                                                0x1f
+#define WGS_COMPUTE_RELAUNCH2__PAYLOAD_MASK                                                                   0x3FFFFFFFL
+#define WGS_COMPUTE_RELAUNCH2__IS_EVENT_MASK                                                                  0x40000000L
+#define WGS_COMPUTE_RELAUNCH2__IS_STATE_MASK                                                                  0x80000000L
+//WGS_COMPUTE_PRESCALED_DIM_X
+#define WGS_COMPUTE_PRESCALED_DIM_X__SIZE__SHIFT                                                              0x0
+#define WGS_COMPUTE_PRESCALED_DIM_X__SIZE_MASK                                                                0xFFFFFFFFL
+//WGS_COMPUTE_PRESCALED_DIM_Y
+#define WGS_COMPUTE_PRESCALED_DIM_Y__SIZE__SHIFT                                                              0x0
+#define WGS_COMPUTE_PRESCALED_DIM_Y__SIZE_MASK                                                                0xFFFFFFFFL
+//WGS_COMPUTE_PRESCALED_DIM_Z
+#define WGS_COMPUTE_PRESCALED_DIM_Z__SIZE__SHIFT                                                              0x0
+#define WGS_COMPUTE_PRESCALED_DIM_Z__SIZE_MASK                                                                0xFFFFFFFFL
+//WGS_COMPUTE_USER_DATA_0
+#define WGS_COMPUTE_USER_DATA_0__DATA__SHIFT                                                                  0x0
+#define WGS_COMPUTE_USER_DATA_0__DATA_MASK                                                                    0xFFFFFFFFL
+//WGS_COMPUTE_USER_DATA_1
+#define WGS_COMPUTE_USER_DATA_1__DATA__SHIFT                                                                  0x0
+#define WGS_COMPUTE_USER_DATA_1__DATA_MASK                                                                    0xFFFFFFFFL
+//WGS_COMPUTE_USER_DATA_2
+#define WGS_COMPUTE_USER_DATA_2__DATA__SHIFT                                                                  0x0
+#define WGS_COMPUTE_USER_DATA_2__DATA_MASK                                                                    0xFFFFFFFFL
+//WGS_COMPUTE_USER_DATA_3
+#define WGS_COMPUTE_USER_DATA_3__DATA__SHIFT                                                                  0x0
+#define WGS_COMPUTE_USER_DATA_3__DATA_MASK                                                                    0xFFFFFFFFL
+//WGS_COMPUTE_USER_DATA_4
+#define WGS_COMPUTE_USER_DATA_4__DATA__SHIFT                                                                  0x0
+#define WGS_COMPUTE_USER_DATA_4__DATA_MASK                                                                    0xFFFFFFFFL
+//WGS_COMPUTE_USER_DATA_5
+#define WGS_COMPUTE_USER_DATA_5__DATA__SHIFT                                                                  0x0
+#define WGS_COMPUTE_USER_DATA_5__DATA_MASK                                                                    0xFFFFFFFFL
+//WGS_COMPUTE_USER_DATA_6
+#define WGS_COMPUTE_USER_DATA_6__DATA__SHIFT                                                                  0x0
+#define WGS_COMPUTE_USER_DATA_6__DATA_MASK                                                                    0xFFFFFFFFL
+//WGS_COMPUTE_USER_DATA_7
+#define WGS_COMPUTE_USER_DATA_7__DATA__SHIFT                                                                  0x0
+#define WGS_COMPUTE_USER_DATA_7__DATA_MASK                                                                    0xFFFFFFFFL
+//WGS_COMPUTE_USER_DATA_8
+#define WGS_COMPUTE_USER_DATA_8__DATA__SHIFT                                                                  0x0
+#define WGS_COMPUTE_USER_DATA_8__DATA_MASK                                                                    0xFFFFFFFFL
+//WGS_COMPUTE_USER_DATA_9
+#define WGS_COMPUTE_USER_DATA_9__DATA__SHIFT                                                                  0x0
+#define WGS_COMPUTE_USER_DATA_9__DATA_MASK                                                                    0xFFFFFFFFL
+//WGS_COMPUTE_USER_DATA_10
+#define WGS_COMPUTE_USER_DATA_10__DATA__SHIFT                                                                 0x0
+#define WGS_COMPUTE_USER_DATA_10__DATA_MASK                                                                   0xFFFFFFFFL
+//WGS_COMPUTE_USER_DATA_11
+#define WGS_COMPUTE_USER_DATA_11__DATA__SHIFT                                                                 0x0
+#define WGS_COMPUTE_USER_DATA_11__DATA_MASK                                                                   0xFFFFFFFFL
+//WGS_COMPUTE_USER_DATA_12
+#define WGS_COMPUTE_USER_DATA_12__DATA__SHIFT                                                                 0x0
+#define WGS_COMPUTE_USER_DATA_12__DATA_MASK                                                                   0xFFFFFFFFL
+//WGS_COMPUTE_USER_DATA_13
+#define WGS_COMPUTE_USER_DATA_13__DATA__SHIFT                                                                 0x0
+#define WGS_COMPUTE_USER_DATA_13__DATA_MASK                                                                   0xFFFFFFFFL
+//WGS_COMPUTE_USER_DATA_14
+#define WGS_COMPUTE_USER_DATA_14__DATA__SHIFT                                                                 0x0
+#define WGS_COMPUTE_USER_DATA_14__DATA_MASK                                                                   0xFFFFFFFFL
+//WGS_COMPUTE_USER_DATA_15
+#define WGS_COMPUTE_USER_DATA_15__DATA__SHIFT                                                                 0x0
+#define WGS_COMPUTE_USER_DATA_15__DATA_MASK                                                                   0xFFFFFFFFL
+//WGS_COMPUTE_DISPATCH_TUNNEL
+#define WGS_COMPUTE_DISPATCH_TUNNEL__OFF_DELAY__SHIFT                                                         0x0
+#define WGS_COMPUTE_DISPATCH_TUNNEL__OFF_DELAY_MASK                                                           0x00001FFFL
+//WGS_COMPUTE_DISPATCH_END
+#define WGS_COMPUTE_DISPATCH_END__DATA__SHIFT                                                                 0x0
+#define WGS_COMPUTE_DISPATCH_END__DATA_MASK                                                                   0xFFFFFFFFL
+//WGS_COMPUTE_NOWHERE
+#define WGS_COMPUTE_NOWHERE__DATA__SHIFT                                                                      0x0
+#define WGS_COMPUTE_NOWHERE__DATA_MASK                                                                        0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_wgsdec
+//WGS_MGCG_SYNC_CNTL
+#define WGS_MGCG_SYNC_CNTL__COOLDOWN_PERIOD__SHIFT                                                            0x0
+#define WGS_MGCG_SYNC_CNTL__WARMUP_PERIOD__SHIFT                                                              0x8
+#define WGS_MGCG_SYNC_CNTL__COOLDOWN_PERIOD_MASK                                                              0x000000FFL
+#define WGS_MGCG_SYNC_CNTL__WARMUP_PERIOD_MASK                                                                0x0000FF00L
+//WGS_INT_INFO
+#define WGS_INT_INFO__RING_ID__SHIFT                                                                          0x0
+#define WGS_INT_INFO__VMID__SHIFT                                                                             0x8
+#define WGS_INT_INFO__RING_ID_MASK                                                                            0x000000FFL
+#define WGS_INT_INFO__VMID_MASK                                                                               0x00000F00L
+//WGS_INT_CNTXID_LO
+#define WGS_INT_CNTXID_LO__DATA__SHIFT                                                                        0x0
+#define WGS_INT_CNTXID_LO__DATA_MASK                                                                          0xFFFFFFFFL
+//WGS_INT_CNTXID_HI
+#define WGS_INT_CNTXID_HI__DATA__SHIFT                                                                        0x0
+#define WGS_INT_CNTXID_HI__DATA_MASK                                                                          0x000000FFL
+//WGS_ERROR
+#define WGS_ERROR__RESERVED__SHIFT                                                                            0x0
+#define WGS_ERROR__RESERVED_MASK                                                                              0xFFFFFFFFL
+//WGS_ENDIAN_SWAP
+#define WGS_ENDIAN_SWAP__ENDIAN_SWAP__SHIFT                                                                   0x0
+#define WGS_ENDIAN_SWAP__ENDIAN_SWAP_MASK                                                                     0x00000003L
+//WGS_GEN_BASE_LO
+#define WGS_GEN_BASE_LO__BASE_LO__SHIFT                                                                       0xc
+#define WGS_GEN_BASE_LO__BASE_LO_MASK                                                                         0xFFFFF000L
+//WGS_GEN_BASE_HI
+#define WGS_GEN_BASE_HI__BASE_HI__SHIFT                                                                       0x0
+#define WGS_GEN_BASE_HI__BASE_HI_MASK                                                                         0x0000FFFFL
+//CGTT_WGS_CLK_CTRL
+#define CGTT_WGS_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_WGS_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                               0xf
+#define CGTT_WGS_CLK_CTRL__RESERVED_SOFT_STALL_OVERRIDE7__SHIFT                                               0x10
+#define CGTT_WGS_CLK_CTRL__RESERVED_SOFT_STALL_OVERRIDE6__SHIFT                                               0x11
+#define CGTT_WGS_CLK_CTRL__RESERVED_SOFT_STALL_OVERRIDE5__SHIFT                                               0x12
+#define CGTT_WGS_CLK_CTRL__RESERVED_SOFT_STALL_OVERRIDE4__SHIFT                                               0x13
+#define CGTT_WGS_CLK_CTRL__RESERVED_SOFT_STALL_OVERRIDE3__SHIFT                                               0x14
+#define CGTT_WGS_CLK_CTRL__RESERVED_SOFT_STALL_OVERRIDE2__SHIFT                                               0x15
+#define CGTT_WGS_CLK_CTRL__RESERVED_SOFT_STALL_OVERRIDE1__SHIFT                                               0x16
+#define CGTT_WGS_CLK_CTRL__RESERVED_SOFT_STALL_OVERRIDE0__SHIFT                                               0x17
+#define CGTT_WGS_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                       0x1d
+#define CGTT_WGS_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                           0x1e
+#define CGTT_WGS_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                           0x1f
+#define CGTT_WGS_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_WGS_CLK_CTRL__MGLS_OVERRIDE_MASK                                                                 0x00008000L
+#define CGTT_WGS_CLK_CTRL__RESERVED_SOFT_STALL_OVERRIDE7_MASK                                                 0x00010000L
+#define CGTT_WGS_CLK_CTRL__RESERVED_SOFT_STALL_OVERRIDE6_MASK                                                 0x00020000L
+#define CGTT_WGS_CLK_CTRL__RESERVED_SOFT_STALL_OVERRIDE5_MASK                                                 0x00040000L
+#define CGTT_WGS_CLK_CTRL__RESERVED_SOFT_STALL_OVERRIDE4_MASK                                                 0x00080000L
+#define CGTT_WGS_CLK_CTRL__RESERVED_SOFT_STALL_OVERRIDE3_MASK                                                 0x00100000L
+#define CGTT_WGS_CLK_CTRL__RESERVED_SOFT_STALL_OVERRIDE2_MASK                                                 0x00200000L
+#define CGTT_WGS_CLK_CTRL__RESERVED_SOFT_STALL_OVERRIDE1_MASK                                                 0x00400000L
+#define CGTT_WGS_CLK_CTRL__RESERVED_SOFT_STALL_OVERRIDE0_MASK                                                 0x00800000L
+#define CGTT_WGS_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                         0x20000000L
+#define CGTT_WGS_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                             0x40000000L
+#define CGTT_WGS_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                             0x80000000L
+//WGS_ME1_UCODE_ADDR
+#define WGS_ME1_UCODE_ADDR__UCODE_ADDR__SHIFT                                                                 0x0
+#define WGS_ME1_UCODE_ADDR__PIPE_SEL__SHIFT                                                                   0x1e
+#define WGS_ME1_UCODE_ADDR__UCODE_ADDR_MASK                                                                   0x007FFFFFL
+#define WGS_ME1_UCODE_ADDR__PIPE_SEL_MASK                                                                     0xC0000000L
+//WGS_ME1_UCODE_DATA
+#define WGS_ME1_UCODE_DATA__UCODE_DATA__SHIFT                                                                 0x0
+#define WGS_ME1_UCODE_DATA__UCODE_DATA_MASK                                                                   0xFFFFFFFFL
+//WGS_ME1_UCODE_CHKSUM
+#define WGS_ME1_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                             0x0
+#define WGS_ME1_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                               0xFFFFFFFFL
+//WGS_SUSPEND_CTX_SAVE_BASE_ADDR_LO
+#define WGS_SUSPEND_CTX_SAVE_BASE_ADDR_LO__ADDR__SHIFT                                                        0xc
+#define WGS_SUSPEND_CTX_SAVE_BASE_ADDR_LO__ADDR_MASK                                                          0xFFFFF000L
+//WGS_SUSPEND_CTX_SAVE_BASE_ADDR_HI
+#define WGS_SUSPEND_CTX_SAVE_BASE_ADDR_HI__ADDR_HI__SHIFT                                                     0x0
+#define WGS_SUSPEND_CTX_SAVE_BASE_ADDR_HI__ADDR_HI_MASK                                                       0x0000FFFFL
+//WGS_SUSPEND_CTX_SAVE_CONTROL
+#define WGS_SUSPEND_CTX_SAVE_CONTROL__POLICY__SHIFT                                                           0x3
+#define WGS_SUSPEND_CTX_SAVE_CONTROL__EXE_DISABLE__SHIFT                                                      0x17
+#define WGS_SUSPEND_CTX_SAVE_CONTROL__POLICY_MASK                                                             0x00000018L
+#define WGS_SUSPEND_CTX_SAVE_CONTROL__EXE_DISABLE_MASK                                                        0x00800000L
+//WGS_SUSPEND_CNTL_STACK_OFFSET
+#define WGS_SUSPEND_CNTL_STACK_OFFSET__OFFSET__SHIFT                                                          0x2
+#define WGS_SUSPEND_CNTL_STACK_OFFSET__OFFSET_MASK                                                            0x0000FFFCL
+//WGS_SUSPEND_CNTL_STACK_SIZE
+#define WGS_SUSPEND_CNTL_STACK_SIZE__SIZE__SHIFT                                                              0xc
+#define WGS_SUSPEND_CNTL_STACK_SIZE__SIZE_MASK                                                                0x0000F000L
+//WGS_SUSPEND_WG_STATE_OFFSET
+#define WGS_SUSPEND_WG_STATE_OFFSET__OFFSET__SHIFT                                                            0x2
+#define WGS_SUSPEND_WG_STATE_OFFSET__OFFSET_MASK                                                              0x03FFFFFCL
+//WGS_SUSPEND_CTX_SAVE_SIZE
+#define WGS_SUSPEND_CTX_SAVE_SIZE__SIZE__SHIFT                                                                0xc
+#define WGS_SUSPEND_CTX_SAVE_SIZE__SIZE_MASK                                                                  0x03FFF000L
+//WGS_OS_PIPES
+#define WGS_OS_PIPES__OS_PIPES__SHIFT                                                                         0x0
+#define WGS_OS_PIPES__OS_PIPES_MASK                                                                           0x000000FFL
+//WGS_SUSPEND_RESUME_REQ
+#define WGS_SUSPEND_RESUME_REQ__SUSPEND_REQ__SHIFT                                                            0x0
+#define WGS_SUSPEND_RESUME_REQ__RESUME_REQ__SHIFT                                                             0x1
+#define WGS_SUSPEND_RESUME_REQ__SUSPEND_REQ_MASK                                                              0x00000001L
+#define WGS_SUSPEND_RESUME_REQ__RESUME_REQ_MASK                                                               0x00000002L
+//WGS_IQ_WAIT_TIME3
+#define WGS_IQ_WAIT_TIME3__SUSPEND_QUE__SHIFT                                                                 0x0
+#define WGS_IQ_WAIT_TIME3__SUSPEND_QUE_MASK                                                                   0x000000FFL
+//WGS_DDID_BASE_ADDR_LO
+#define WGS_DDID_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                                            0x6
+#define WGS_DDID_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                              0xFFFFFFC0L
+//WGS_DDID_BASE_ADDR_HI
+#define WGS_DDID_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                            0x0
+#define WGS_DDID_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                              0x0000FFFFL
+//WGS_DDID_CNTL
+#define WGS_DDID_CNTL__THRESHOLD__SHIFT                                                                       0x0
+#define WGS_DDID_CNTL__SIZE__SHIFT                                                                            0x10
+#define WGS_DDID_CNTL__NO_RING_MEMORY__SHIFT                                                                  0x13
+#define WGS_DDID_CNTL__POLICY__SHIFT                                                                          0x1c
+#define WGS_DDID_CNTL__MODE__SHIFT                                                                            0x1e
+#define WGS_DDID_CNTL__ENABLE__SHIFT                                                                          0x1f
+#define WGS_DDID_CNTL__THRESHOLD_MASK                                                                         0x000000FFL
+#define WGS_DDID_CNTL__SIZE_MASK                                                                              0x00010000L
+#define WGS_DDID_CNTL__NO_RING_MEMORY_MASK                                                                    0x00080000L
+#define WGS_DDID_CNTL__POLICY_MASK                                                                            0x30000000L
+#define WGS_DDID_CNTL__MODE_MASK                                                                              0x40000000L
+#define WGS_DDID_CNTL__ENABLE_MASK                                                                            0x80000000L
+//WGS_RS64_PRGRM_CNTR_START
+#define WGS_RS64_PRGRM_CNTR_START__IP_START__SHIFT                                                            0x0
+#define WGS_RS64_PRGRM_CNTR_START__IP_START_MASK                                                              0xFFFFFFFFL
+//WGS_MTVEC_LO
+#define WGS_MTVEC_LO__ADDR_LO__SHIFT                                                                          0x0
+#define WGS_MTVEC_LO__ADDR_LO_MASK                                                                            0xFFFFFFFFL
+//WGS_MTVEC_HI
+#define WGS_MTVEC_HI__ADDR_LO__SHIFT                                                                          0x0
+#define WGS_MTVEC_HI__ADDR_LO_MASK                                                                            0xFFFFFFFFL
+//WGS_RS64_CNTL
+#define WGS_RS64_CNTL__MEC_INVALIDATE_ICACHE__SHIFT                                                           0x4
+#define WGS_RS64_CNTL__RSRV8__SHIFT                                                                           0x10
+#define WGS_RS64_CNTL__RSRV7__SHIFT                                                                           0x11
+#define WGS_RS64_CNTL__RSRV6__SHIFT                                                                           0x12
+#define WGS_RS64_CNTL__RSRV5__SHIFT                                                                           0x13
+#define WGS_RS64_CNTL__RSRV4__SHIFT                                                                           0x1a
+#define WGS_RS64_CNTL__RSRV3__SHIFT                                                                           0x1b
+#define WGS_RS64_CNTL__RSRV2__SHIFT                                                                           0x1c
+#define WGS_RS64_CNTL__RSRV1__SHIFT                                                                           0x1d
+#define WGS_RS64_CNTL__MEC_HALT__SHIFT                                                                        0x1e
+#define WGS_RS64_CNTL__MEC_STEP__SHIFT                                                                        0x1f
+#define WGS_RS64_CNTL__MEC_INVALIDATE_ICACHE_MASK                                                             0x00000010L
+#define WGS_RS64_CNTL__RSRV8_MASK                                                                             0x00010000L
+#define WGS_RS64_CNTL__RSRV7_MASK                                                                             0x00020000L
+#define WGS_RS64_CNTL__RSRV6_MASK                                                                             0x00040000L
+#define WGS_RS64_CNTL__RSRV5_MASK                                                                             0x00080000L
+#define WGS_RS64_CNTL__RSRV4_MASK                                                                             0x04000000L
+#define WGS_RS64_CNTL__RSRV3_MASK                                                                             0x08000000L
+#define WGS_RS64_CNTL__RSRV2_MASK                                                                             0x10000000L
+#define WGS_RS64_CNTL__RSRV1_MASK                                                                             0x20000000L
+#define WGS_RS64_CNTL__MEC_HALT_MASK                                                                          0x40000000L
+#define WGS_RS64_CNTL__MEC_STEP_MASK                                                                          0x80000000L
+//WGS_MIE_LO
+#define WGS_MIE_LO__MEC_INT__SHIFT                                                                            0x0
+#define WGS_MIE_LO__MEC_INT_MASK                                                                              0xFFFFFFFFL
+//WGS_MIE_HI
+#define WGS_MIE_HI__MEC_INT__SHIFT                                                                            0x0
+#define WGS_MIE_HI__MEC_INT_MASK                                                                              0xFFFFFFFFL
+//WGS_RS64_INTERRUPT
+#define WGS_RS64_INTERRUPT__MEC_INT__SHIFT                                                                    0x0
+#define WGS_RS64_INTERRUPT__MEC_INT_MASK                                                                      0xFFFFFFFFL
+//WGS_RS64_INSTR_PNTR
+#define WGS_RS64_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                0x0
+#define WGS_RS64_INSTR_PNTR__INSTR_PNTR_MASK                                                                  0x000FFFFFL
+//WGS_MIP_LO
+#define WGS_MIP_LO__MIP_LO__SHIFT                                                                             0x0
+#define WGS_MIP_LO__MIP_LO_MASK                                                                               0xFFFFFFFFL
+//WGS_MIP_HI
+#define WGS_MIP_HI__MIP_HI__SHIFT                                                                             0x0
+#define WGS_MIP_HI__MIP_HI_MASK                                                                               0xFFFFFFFFL
+//WGS_DC_BASE_CNTL
+#define WGS_DC_BASE_CNTL__VMID__SHIFT                                                                         0x0
+#define WGS_DC_BASE_CNTL__VMID_MASK                                                                           0x0000000FL
+//WGS_DC_OP_CNTL
+#define WGS_DC_OP_CNTL__INVALIDATE_DCACHE__SHIFT                                                              0x0
+#define WGS_DC_OP_CNTL__INVALIDATE_DCACHE_COMPLETE__SHIFT                                                     0x1
+#define WGS_DC_OP_CNTL__BYPASS_ALL__SHIFT                                                                     0x2
+#define WGS_DC_OP_CNTL__INVALIDATE_DCACHE_MASK                                                                0x00000001L
+#define WGS_DC_OP_CNTL__INVALIDATE_DCACHE_COMPLETE_MASK                                                       0x00000002L
+#define WGS_DC_OP_CNTL__BYPASS_ALL_MASK                                                                       0x00000004L
+//WGS_MTIMECMP_LO
+#define WGS_MTIMECMP_LO__TIME_LO__SHIFT                                                                       0x0
+#define WGS_MTIMECMP_LO__TIME_LO_MASK                                                                         0xFFFFFFFFL
+//WGS_MTIMECMP_HI
+#define WGS_MTIMECMP_HI__TIME_HI__SHIFT                                                                       0x0
+#define WGS_MTIMECMP_HI__TIME_HI_MASK                                                                         0xFFFFFFFFL
+//WGS_GP0_LO
+#define WGS_GP0_LO__PG_VIRT_HALTED__SHIFT                                                                     0x0
+#define WGS_GP0_LO__DATA__SHIFT                                                                               0x1
+#define WGS_GP0_LO__PG_VIRT_HALTED_MASK                                                                       0x00000001L
+#define WGS_GP0_LO__DATA_MASK                                                                                 0xFFFFFFFEL
+//WGS_GP0_HI
+#define WGS_GP0_HI__M_RET_ADDR__SHIFT                                                                         0x0
+#define WGS_GP0_HI__M_RET_ADDR_MASK                                                                           0xFFFFFFFFL
+//WGS_GP1_LO
+#define WGS_GP1_LO__RD_WR_SELECT_LO__SHIFT                                                                    0x0
+#define WGS_GP1_LO__RD_WR_SELECT_LO_MASK                                                                      0xFFFFFFFFL
+//WGS_GP1_HI
+#define WGS_GP1_HI__RD_WR_SELECT_HI__SHIFT                                                                    0x0
+#define WGS_GP1_HI__RD_WR_SELECT_HI_MASK                                                                      0xFFFFFFFFL
+//WGS_GP2_LO
+#define WGS_GP2_LO__STACK_PNTR_LO__SHIFT                                                                      0x0
+#define WGS_GP2_LO__STACK_PNTR_LO_MASK                                                                        0xFFFFFFFFL
+//WGS_GP2_HI
+#define WGS_GP2_HI__STACK_PNTR_HI__SHIFT                                                                      0x0
+#define WGS_GP2_HI__STACK_PNTR_HI_MASK                                                                        0xFFFFFFFFL
+//WGS_GP3_LO
+#define WGS_GP3_LO__DATA__SHIFT                                                                               0x0
+#define WGS_GP3_LO__DATA_MASK                                                                                 0xFFFFFFFFL
+//WGS_GP3_HI
+#define WGS_GP3_HI__DATA__SHIFT                                                                               0x0
+#define WGS_GP3_HI__DATA_MASK                                                                                 0xFFFFFFFFL
+//WGS_GP4_LO
+#define WGS_GP4_LO__DATA__SHIFT                                                                               0x0
+#define WGS_GP4_LO__DATA_MASK                                                                                 0xFFFFFFFFL
+//WGS_GP4_HI
+#define WGS_GP4_HI__DATA__SHIFT                                                                               0x0
+#define WGS_GP4_HI__DATA_MASK                                                                                 0xFFFFFFFFL
+//WGS_GP5_LO
+#define WGS_GP5_LO__PG_VIRT_HALTED__SHIFT                                                                     0x0
+#define WGS_GP5_LO__DATA__SHIFT                                                                               0x1
+#define WGS_GP5_LO__PG_VIRT_HALTED_MASK                                                                       0x00000001L
+#define WGS_GP5_LO__DATA_MASK                                                                                 0xFFFFFFFEL
+//WGS_GP5_HI
+#define WGS_GP5_HI__M_RET_ADDR__SHIFT                                                                         0x0
+#define WGS_GP5_HI__M_RET_ADDR_MASK                                                                           0xFFFFFFFFL
+//WGS_GP6_LO
+#define WGS_GP6_LO__RD_WR_SELECT_LO__SHIFT                                                                    0x0
+#define WGS_GP6_LO__RD_WR_SELECT_LO_MASK                                                                      0xFFFFFFFFL
+//WGS_GP6_HI
+#define WGS_GP6_HI__RD_WR_SELECT_HI__SHIFT                                                                    0x0
+#define WGS_GP6_HI__RD_WR_SELECT_HI_MASK                                                                      0xFFFFFFFFL
+//WGS_GP7_LO
+#define WGS_GP7_LO__STACK_PNTR_LO__SHIFT                                                                      0x0
+#define WGS_GP7_LO__STACK_PNTR_LO_MASK                                                                        0xFFFFFFFFL
+//WGS_GP7_HI
+#define WGS_GP7_HI__STACK_PNTR_HI__SHIFT                                                                      0x0
+#define WGS_GP7_HI__STACK_PNTR_HI_MASK                                                                        0xFFFFFFFFL
+//WGS_GP8_LO
+#define WGS_GP8_LO__DATA__SHIFT                                                                               0x0
+#define WGS_GP8_LO__DATA_MASK                                                                                 0xFFFFFFFFL
+//WGS_GP8_HI
+#define WGS_GP8_HI__DATA__SHIFT                                                                               0x0
+#define WGS_GP8_HI__DATA_MASK                                                                                 0xFFFFFFFFL
+//WGS_GP9_LO
+#define WGS_GP9_LO__DATA__SHIFT                                                                               0x0
+#define WGS_GP9_LO__DATA_MASK                                                                                 0xFFFFFFFFL
+//WGS_GP9_HI
+#define WGS_GP9_HI__DATA__SHIFT                                                                               0x0
+#define WGS_GP9_HI__DATA_MASK                                                                                 0xFFFFFFFFL
+//WGS_DM_INDEX_ADDR
+#define WGS_DM_INDEX_ADDR__ADDR__SHIFT                                                                        0x0
+#define WGS_DM_INDEX_ADDR__ADDR_MASK                                                                          0xFFFFFFFFL
+//WGS_DM_INDEX_DATA
+#define WGS_DM_INDEX_DATA__DATA__SHIFT                                                                        0x0
+#define WGS_DM_INDEX_DATA__DATA_MASK                                                                          0xFFFFFFFFL
+//WGS_LOCAL_BASE0_LO
+#define WGS_LOCAL_BASE0_LO__BASE0_LO__SHIFT                                                                   0x10
+#define WGS_LOCAL_BASE0_LO__BASE0_LO_MASK                                                                     0xFFFF0000L
+//WGS_LOCAL_BASE0_HI
+#define WGS_LOCAL_BASE0_HI__BASE0_HI__SHIFT                                                                   0x0
+#define WGS_LOCAL_BASE0_HI__BASE0_HI_MASK                                                                     0x0000FFFFL
+//WGS_LOCAL_MASK0_LO
+#define WGS_LOCAL_MASK0_LO__MASK0_LO__SHIFT                                                                   0x10
+#define WGS_LOCAL_MASK0_LO__MASK0_LO_MASK                                                                     0xFFFF0000L
+//WGS_LOCAL_MASK0_HI
+#define WGS_LOCAL_MASK0_HI__MASK0_HI__SHIFT                                                                   0x0
+#define WGS_LOCAL_MASK0_HI__MASK0_HI_MASK                                                                     0x0000FFFFL
+//WGS_LOCAL_APERTURE
+#define WGS_LOCAL_APERTURE__APERTURE__SHIFT                                                                   0x0
+#define WGS_LOCAL_APERTURE__SCOPE__SHIFT                                                                      0x6
+#define WGS_LOCAL_APERTURE__TEMPORAL__SHIFT                                                                   0x8
+#define WGS_LOCAL_APERTURE__APERTURE_MASK                                                                     0x00000007L
+#define WGS_LOCAL_APERTURE__SCOPE_MASK                                                                        0x000000C0L
+#define WGS_LOCAL_APERTURE__TEMPORAL_MASK                                                                     0x00000700L
+//WGS_LOCAL_INSTR_BASE_LO
+#define WGS_LOCAL_INSTR_BASE_LO__BASE_LO__SHIFT                                                               0x10
+#define WGS_LOCAL_INSTR_BASE_LO__BASE_LO_MASK                                                                 0xFFFF0000L
+//WGS_LOCAL_INSTR_BASE_HI
+#define WGS_LOCAL_INSTR_BASE_HI__BASE_HI__SHIFT                                                               0x0
+#define WGS_LOCAL_INSTR_BASE_HI__BASE_HI_MASK                                                                 0x0000FFFFL
+//WGS_LOCAL_INSTR_MASK_LO
+#define WGS_LOCAL_INSTR_MASK_LO__MASK_LO__SHIFT                                                               0x10
+#define WGS_LOCAL_INSTR_MASK_LO__MASK_LO_MASK                                                                 0xFFFF0000L
+//WGS_LOCAL_INSTR_MASK_HI
+#define WGS_LOCAL_INSTR_MASK_HI__MASK_HI__SHIFT                                                               0x0
+#define WGS_LOCAL_INSTR_MASK_HI__MASK_HI_MASK                                                                 0x0000FFFFL
+//WGS_LOCAL_INSTR_APERTURE
+#define WGS_LOCAL_INSTR_APERTURE__APERTURE__SHIFT                                                             0x0
+#define WGS_LOCAL_INSTR_APERTURE__SCOPE__SHIFT                                                                0x6
+#define WGS_LOCAL_INSTR_APERTURE__TEMPORAL__SHIFT                                                             0x8
+#define WGS_LOCAL_INSTR_APERTURE__APERTURE_MASK                                                               0x00000007L
+#define WGS_LOCAL_INSTR_APERTURE__SCOPE_MASK                                                                  0x000000C0L
+#define WGS_LOCAL_INSTR_APERTURE__TEMPORAL_MASK                                                               0x00000700L
+//WGS_LOCAL_SCRATCH_APERTURE
+#define WGS_LOCAL_SCRATCH_APERTURE__APERTURE__SHIFT                                                           0x0
+#define WGS_LOCAL_SCRATCH_APERTURE__APERTURE_MASK                                                             0x00000007L
+//WGS_LOCAL_SCRATCH_BASE_LO
+#define WGS_LOCAL_SCRATCH_BASE_LO__BASE_LO__SHIFT                                                             0x10
+#define WGS_LOCAL_SCRATCH_BASE_LO__BASE_LO_MASK                                                               0xFFFF0000L
+//WGS_LOCAL_SCRATCH_BASE_HI
+#define WGS_LOCAL_SCRATCH_BASE_HI__BASE_HI__SHIFT                                                             0x0
+#define WGS_LOCAL_SCRATCH_BASE_HI__BASE_HI_MASK                                                               0x0000FFFFL
+//WGS_RS64_PERFCOUNT_CNTL
+#define WGS_RS64_PERFCOUNT_CNTL__EVENT_SEL__SHIFT                                                             0x0
+#define WGS_RS64_PERFCOUNT_CNTL__EVENT_SEL_MASK                                                               0x0000001FL
+//WGS_RS64_PENDING_INTERRUPT
+#define WGS_RS64_PENDING_INTERRUPT__PENDING_INTERRUPT__SHIFT                                                  0x0
+#define WGS_RS64_PENDING_INTERRUPT__PENDING_INTERRUPT_MASK                                                    0xFFFFFFFFL
+//WGS_RS64_EXCEPTION_STATUS
+#define WGS_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_ILLEGAL_INSTRUCTION__SHIFT                                  0x0
+#define WGS_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_MISALIGNED_ADDR__SHIFT                                      0x1
+#define WGS_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_UNALIGNED_INSTRUTCION__SHIFT                                0x2
+#define WGS_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_PAGE_FAULT__SHIFT                                           0x3
+#define WGS_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_INSTRUCTION_ADDR__SHIFT                                     0x4
+#define WGS_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_ILLEGAL_INSTRUCTION_MASK                                    0x00000001L
+#define WGS_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_MISALIGNED_ADDR_MASK                                        0x00000002L
+#define WGS_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_UNALIGNED_INSTRUTCION_MASK                                  0x00000004L
+#define WGS_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_PAGE_FAULT_MASK                                             0x00000008L
+#define WGS_RS64_EXCEPTION_STATUS__RS64_EXCEPTION_INSTRUCTION_ADDR_MASK                                       0x07FFFFF0L
+//WGS_RS64_PRGRM_CNTR_START_HI
+#define WGS_RS64_PRGRM_CNTR_START_HI__IP_START__SHIFT                                                         0x0
+#define WGS_RS64_PRGRM_CNTR_START_HI__IP_START_MASK                                                           0x3FFFFFFFL
+//WGS_DC_BASE_LO
+#define WGS_DC_BASE_LO__DC_BASE_LO__SHIFT                                                                     0x10
+#define WGS_DC_BASE_LO__DC_BASE_LO_MASK                                                                       0xFFFF0000L
+//WGS_MDBASE_LO
+#define WGS_MDBASE_LO__BASE_LO__SHIFT                                                                         0x10
+#define WGS_MDBASE_LO__BASE_LO_MASK                                                                           0xFFFF0000L
+//WGS_DC_BASE_HI
+#define WGS_DC_BASE_HI__DC_BASE_HI__SHIFT                                                                     0x0
+#define WGS_DC_BASE_HI__DC_BASE_HI_MASK                                                                       0x0000FFFFL
+//WGS_MDBASE_HI
+#define WGS_MDBASE_HI__BASE_HI__SHIFT                                                                         0x0
+#define WGS_MDBASE_HI__BASE_HI_MASK                                                                           0x0000FFFFL
+//WGS_MIBOUND_LO
+#define WGS_MIBOUND_LO__BOUND_LO__SHIFT                                                                       0x0
+#define WGS_MIBOUND_LO__BOUND_LO_MASK                                                                         0xFFFFFFFFL
+//WGS_MIBOUND_HI
+#define WGS_MIBOUND_HI__BOUND_HI__SHIFT                                                                       0x0
+#define WGS_MIBOUND_HI__BOUND_HI_MASK                                                                         0xFFFFFFFFL
+//WGS_MDBOUND_LO
+#define WGS_MDBOUND_LO__BOUND_LO__SHIFT                                                                       0x0
+#define WGS_MDBOUND_LO__BOUND_LO_MASK                                                                         0xFFFFFFFFL
+//WGS_MDBOUND_HI
+#define WGS_MDBOUND_HI__BOUND_HI__SHIFT                                                                       0x0
+#define WGS_MDBOUND_HI__BOUND_HI_MASK                                                                         0xFFFFFFFFL
+//WGS_IC_OP_CNTL
+#define WGS_IC_OP_CNTL__INVALIDATE_CACHE__SHIFT                                                               0x0
+#define WGS_IC_OP_CNTL__INVALIDATE_CACHE_COMPLETE__SHIFT                                                      0x1
+#define WGS_IC_OP_CNTL__HALT_ON_PRIME__SHIFT                                                                  0x2
+#define WGS_IC_OP_CNTL__PRIME_START_PC__SHIFT                                                                 0x3
+#define WGS_IC_OP_CNTL__PRIME_ICACHE__SHIFT                                                                   0x4
+#define WGS_IC_OP_CNTL__ICACHE_PRIMED__SHIFT                                                                  0x5
+#define WGS_IC_OP_CNTL__INVALIDATE_CACHE_MASK                                                                 0x00000001L
+#define WGS_IC_OP_CNTL__INVALIDATE_CACHE_COMPLETE_MASK                                                        0x00000002L
+#define WGS_IC_OP_CNTL__HALT_ON_PRIME_MASK                                                                    0x00000004L
+#define WGS_IC_OP_CNTL__PRIME_START_PC_MASK                                                                   0x00000008L
+#define WGS_IC_OP_CNTL__PRIME_ICACHE_MASK                                                                     0x00000010L
+#define WGS_IC_OP_CNTL__ICACHE_PRIMED_MASK                                                                    0x00000020L
+//WGS_INT_STATUS
+#define WGS_INT_STATUS__SOURCE_ID__SHIFT                                                                      0x0
+#define WGS_INT_STATUS__SEND__SHIFT                                                                           0x8
+#define WGS_INT_STATUS__SOURCE_ID_MASK                                                                        0x000000FFL
+#define WGS_INT_STATUS__SEND_MASK                                                                             0x00000100L
+//WGS_IH_FREE_COUNT
+#define WGS_IH_FREE_COUNT__COUNT__SHIFT                                                                       0x0
+#define WGS_IH_FREE_COUNT__COUNT_MASK                                                                         0x000000FFL
+//WGS_ME1_PIPE_PRIORITY_CNTS
+#define WGS_ME1_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                      0x0
+#define WGS_ME1_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                     0x8
+#define WGS_ME1_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                     0x10
+#define WGS_ME1_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                      0x18
+#define WGS_ME1_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                        0x000000FFL
+#define WGS_ME1_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                       0x0000FF00L
+#define WGS_ME1_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                       0x00FF0000L
+#define WGS_ME1_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                        0xFF000000L
+//WGS_ME1_PIPE0_PRIORITY
+#define WGS_ME1_PIPE0_PRIORITY__PRIORITY__SHIFT                                                               0x0
+#define WGS_ME1_PIPE0_PRIORITY__PRIORITY_MASK                                                                 0x00000003L
+//WGS_ME1_PIPE1_PRIORITY
+#define WGS_ME1_PIPE1_PRIORITY__PRIORITY__SHIFT                                                               0x0
+#define WGS_ME1_PIPE1_PRIORITY__PRIORITY_MASK                                                                 0x00000003L
+//WGS_ME1_PIPE2_PRIORITY
+#define WGS_ME1_PIPE2_PRIORITY__PRIORITY__SHIFT                                                               0x0
+#define WGS_ME1_PIPE2_PRIORITY__PRIORITY_MASK                                                                 0x00000003L
+//WGS_ME1_PIPE3_PRIORITY
+#define WGS_ME1_PIPE3_PRIORITY__PRIORITY__SHIFT                                                               0x0
+#define WGS_ME1_PIPE3_PRIORITY__PRIORITY_MASK                                                                 0x00000003L
+//WGS_IQ_WAIT_TIME1
+#define WGS_IQ_WAIT_TIME1__IB_OFFLOAD__SHIFT                                                                  0x0
+#define WGS_IQ_WAIT_TIME1__ATOMIC_OFFLOAD__SHIFT                                                              0x8
+#define WGS_IQ_WAIT_TIME1__WRM_OFFLOAD__SHIFT                                                                 0x10
+#define WGS_IQ_WAIT_TIME1__GWS__SHIFT                                                                         0x18
+#define WGS_IQ_WAIT_TIME1__IB_OFFLOAD_MASK                                                                    0x000000FFL
+#define WGS_IQ_WAIT_TIME1__ATOMIC_OFFLOAD_MASK                                                                0x0000FF00L
+#define WGS_IQ_WAIT_TIME1__WRM_OFFLOAD_MASK                                                                   0x00FF0000L
+#define WGS_IQ_WAIT_TIME1__GWS_MASK                                                                           0xFF000000L
+//WGS_IQ_WAIT_TIME2
+#define WGS_IQ_WAIT_TIME2__QUE_SLEEP__SHIFT                                                                   0x0
+#define WGS_IQ_WAIT_TIME2__SCH_WAVE__SHIFT                                                                    0x8
+#define WGS_IQ_WAIT_TIME2__DEQ_RETRY__SHIFT                                                                   0x18
+#define WGS_IQ_WAIT_TIME2__QUE_SLEEP_MASK                                                                     0x000000FFL
+#define WGS_IQ_WAIT_TIME2__SCH_WAVE_MASK                                                                      0x0000FF00L
+#define WGS_IQ_WAIT_TIME2__DEQ_RETRY_MASK                                                                     0xFF000000L
+//WGS_IC_BASE_LO
+#define WGS_IC_BASE_LO__IC_BASE_LO__SHIFT                                                                     0xc
+#define WGS_IC_BASE_LO__IC_BASE_LO_MASK                                                                       0xFFFFF000L
+//WGS_IC_BASE_HI
+#define WGS_IC_BASE_HI__IC_BASE_HI__SHIFT                                                                     0x0
+#define WGS_IC_BASE_HI__IC_BASE_HI_MASK                                                                       0x0000FFFFL
+//WGS_IC_BASE_CNTL
+#define WGS_IC_BASE_CNTL__VMID__SHIFT                                                                         0x0
+#define WGS_IC_BASE_CNTL__PER_PIPE__SHIFT                                                                     0x5
+#define WGS_IC_BASE_CNTL__SCOPE__SHIFT                                                                        0x6
+#define WGS_IC_BASE_CNTL__TEMPORAL__SHIFT                                                                     0x8
+#define WGS_IC_BASE_CNTL__EXE_DISABLE__SHIFT                                                                  0x17
+#define WGS_IC_BASE_CNTL__CACHE_POLICY__SHIFT                                                                 0x18
+#define WGS_IC_BASE_CNTL__VMID_MASK                                                                           0x0000000FL
+#define WGS_IC_BASE_CNTL__PER_PIPE_MASK                                                                       0x00000020L
+#define WGS_IC_BASE_CNTL__SCOPE_MASK                                                                          0x000000C0L
+#define WGS_IC_BASE_CNTL__TEMPORAL_MASK                                                                       0x00000700L
+#define WGS_IC_BASE_CNTL__EXE_DISABLE_MASK                                                                    0x00800000L
+#define WGS_IC_BASE_CNTL__CACHE_POLICY_MASK                                                                   0x03000000L
+//WGS_UCODE_VERS
+#define WGS_UCODE_VERS__ENGINE__SHIFT                                                                         0x0
+#define WGS_UCODE_VERS__COMMON__SHIFT                                                                         0xa
+#define WGS_UCODE_VERS__HEADER__SHIFT                                                                         0x14
+#define WGS_UCODE_VERS__STEP__SHIFT                                                                           0x1e
+#define WGS_UCODE_VERS__ENGINE_MASK                                                                           0x000003FFL
+#define WGS_UCODE_VERS__COMMON_MASK                                                                           0x000FFC00L
+#define WGS_UCODE_VERS__HEADER_MASK                                                                           0x3FF00000L
+#define WGS_UCODE_VERS__STEP_MASK                                                                             0xC0000000L
+//WGS_BUSY_HYSTERESIS
+#define WGS_BUSY_HYSTERESIS__CAC_ACTIVE__SHIFT                                                                0x0
+#define WGS_BUSY_HYSTERESIS__CPC_BUSY__SHIFT                                                                  0x8
+#define WGS_BUSY_HYSTERESIS__CAC_ACTIVE_MASK                                                                  0x000000FFL
+#define WGS_BUSY_HYSTERESIS__CPC_BUSY_MASK                                                                    0x0000FF00L
+//WGS_STATUS
+#define WGS_STATUS__MEC1_BUSY__SHIFT                                                                          0x0
+#define WGS_STATUS__MEC2_BUSY__SHIFT                                                                          0x1
+#define WGS_STATUS__DC0_BUSY__SHIFT                                                                           0x2
+#define WGS_STATUS__DC1_BUSY__SHIFT                                                                           0x3
+#define WGS_STATUS__RCIU1_BUSY__SHIFT                                                                         0x4
+#define WGS_STATUS__RCIU2_BUSY__SHIFT                                                                         0x5
+#define WGS_STATUS__ROQ1_BUSY__SHIFT                                                                          0x6
+#define WGS_STATUS__ROQ2_BUSY__SHIFT                                                                          0x7
+#define WGS_STATUS__TMZ_STATE__SHIFT                                                                          0x8
+#define WGS_STATUS__VMID_TMZ__SHIFT                                                                           0x9
+#define WGS_STATUS__TCIU_BUSY__SHIFT                                                                          0xa
+#define WGS_STATUS__SCRATCH_RAM_BUSY__SHIFT                                                                   0xb
+#define WGS_STATUS__QU_BUSY__SHIFT                                                                            0xc
+#define WGS_STATUS__UTCL2IU_BUSY__SHIFT                                                                       0xd
+#define WGS_STATUS__SAVE_RESTORE_BUSY__SHIFT                                                                  0xe
+#define WGS_STATUS__GCRIU_BUSY__SHIFT                                                                         0xf
+#define WGS_STATUS__MES_BUSY__SHIFT                                                                           0x10
+#define WGS_STATUS__MES_SCRATCH_RAM_BUSY__SHIFT                                                               0x11
+#define WGS_STATUS__RCIU3_BUSY__SHIFT                                                                         0x12
+#define WGS_STATUS__MES_INSTRUCTION_CACHE_BUSY__SHIFT                                                         0x13
+#define WGS_STATUS__MES_DATA_CACHE_BUSY__SHIFT                                                                0x14
+#define WGS_STATUS__MEC_DATA_CACHE_BUSY__SHIFT                                                                0x15
+#define WGS_STATUS__GL1_XNACK__SHIFT                                                                          0x16
+#define WGS_STATUS__RSRV__SHIFT                                                                               0x18
+#define WGS_STATUS__WGS_BUSY__SHIFT                                                                           0x1f
+#define WGS_STATUS__MEC1_BUSY_MASK                                                                            0x00000001L
+#define WGS_STATUS__MEC2_BUSY_MASK                                                                            0x00000002L
+#define WGS_STATUS__DC0_BUSY_MASK                                                                             0x00000004L
+#define WGS_STATUS__DC1_BUSY_MASK                                                                             0x00000008L
+#define WGS_STATUS__RCIU1_BUSY_MASK                                                                           0x00000010L
+#define WGS_STATUS__RCIU2_BUSY_MASK                                                                           0x00000020L
+#define WGS_STATUS__ROQ1_BUSY_MASK                                                                            0x00000040L
+#define WGS_STATUS__ROQ2_BUSY_MASK                                                                            0x00000080L
+#define WGS_STATUS__TMZ_STATE_MASK                                                                            0x00000100L
+#define WGS_STATUS__VMID_TMZ_MASK                                                                             0x00000200L
+#define WGS_STATUS__TCIU_BUSY_MASK                                                                            0x00000400L
+#define WGS_STATUS__SCRATCH_RAM_BUSY_MASK                                                                     0x00000800L
+#define WGS_STATUS__QU_BUSY_MASK                                                                              0x00001000L
+#define WGS_STATUS__UTCL2IU_BUSY_MASK                                                                         0x00002000L
+#define WGS_STATUS__SAVE_RESTORE_BUSY_MASK                                                                    0x00004000L
+#define WGS_STATUS__GCRIU_BUSY_MASK                                                                           0x00008000L
+#define WGS_STATUS__MES_BUSY_MASK                                                                             0x00010000L
+#define WGS_STATUS__MES_SCRATCH_RAM_BUSY_MASK                                                                 0x00020000L
+#define WGS_STATUS__RCIU3_BUSY_MASK                                                                           0x00040000L
+#define WGS_STATUS__MES_INSTRUCTION_CACHE_BUSY_MASK                                                           0x00080000L
+#define WGS_STATUS__MES_DATA_CACHE_BUSY_MASK                                                                  0x00100000L
+#define WGS_STATUS__MEC_DATA_CACHE_BUSY_MASK                                                                  0x00200000L
+#define WGS_STATUS__GL1_XNACK_MASK                                                                            0x00C00000L
+#define WGS_STATUS__RSRV_MASK                                                                                 0x7F000000L
+#define WGS_STATUS__WGS_BUSY_MASK                                                                             0x80000000L
+//WGS_BUSY_STAT
+#define WGS_BUSY_STAT__MEC1_LOAD_BUSY__SHIFT                                                                  0x0
+#define WGS_BUSY_STAT__MEC1_MUTEX_BUSY__SHIFT                                                                 0x2
+#define WGS_BUSY_STAT__MEC1_MESSAGE_BUSY__SHIFT                                                               0x3
+#define WGS_BUSY_STAT__MEC1_EOP_QUEUE_BUSY__SHIFT                                                             0x4
+#define WGS_BUSY_STAT__MEC1_IQ_QUEUE_BUSY__SHIFT                                                              0x5
+#define WGS_BUSY_STAT__MEC1_IB_QUEUE_BUSY__SHIFT                                                              0x6
+#define WGS_BUSY_STAT__MEC1_TC_BUSY__SHIFT                                                                    0x7
+#define WGS_BUSY_STAT__MEC1_DMA_BUSY__SHIFT                                                                   0x8
+#define WGS_BUSY_STAT__MEC1_PARTIAL_FLUSH_BUSY__SHIFT                                                         0x9
+#define WGS_BUSY_STAT__MEC1_PIPE0_BUSY__SHIFT                                                                 0xa
+#define WGS_BUSY_STAT__MEC1_PIPE1_BUSY__SHIFT                                                                 0xb
+#define WGS_BUSY_STAT__MEC1_PIPE2_BUSY__SHIFT                                                                 0xc
+#define WGS_BUSY_STAT__MEC1_PIPE3_BUSY__SHIFT                                                                 0xd
+#define WGS_BUSY_STAT__MEC2_LOAD_BUSY__SHIFT                                                                  0x10
+#define WGS_BUSY_STAT__MEC2_MUTEX_BUSY__SHIFT                                                                 0x12
+#define WGS_BUSY_STAT__MEC2_MESSAGE_BUSY__SHIFT                                                               0x13
+#define WGS_BUSY_STAT__MEC2_EOP_QUEUE_BUSY__SHIFT                                                             0x14
+#define WGS_BUSY_STAT__MEC2_IQ_QUEUE_BUSY__SHIFT                                                              0x15
+#define WGS_BUSY_STAT__MEC2_IB_QUEUE_BUSY__SHIFT                                                              0x16
+#define WGS_BUSY_STAT__MEC2_TC_BUSY__SHIFT                                                                    0x17
+#define WGS_BUSY_STAT__MEC2_DMA_BUSY__SHIFT                                                                   0x18
+#define WGS_BUSY_STAT__MEC2_PARTIAL_FLUSH_BUSY__SHIFT                                                         0x19
+#define WGS_BUSY_STAT__MEC2_PIPE0_BUSY__SHIFT                                                                 0x1a
+#define WGS_BUSY_STAT__MEC2_PIPE1_BUSY__SHIFT                                                                 0x1b
+#define WGS_BUSY_STAT__MEC2_PIPE2_BUSY__SHIFT                                                                 0x1c
+#define WGS_BUSY_STAT__MEC2_PIPE3_BUSY__SHIFT                                                                 0x1d
+#define WGS_BUSY_STAT__MEC1_LOAD_BUSY_MASK                                                                    0x00000001L
+#define WGS_BUSY_STAT__MEC1_MUTEX_BUSY_MASK                                                                   0x00000004L
+#define WGS_BUSY_STAT__MEC1_MESSAGE_BUSY_MASK                                                                 0x00000008L
+#define WGS_BUSY_STAT__MEC1_EOP_QUEUE_BUSY_MASK                                                               0x00000010L
+#define WGS_BUSY_STAT__MEC1_IQ_QUEUE_BUSY_MASK                                                                0x00000020L
+#define WGS_BUSY_STAT__MEC1_IB_QUEUE_BUSY_MASK                                                                0x00000040L
+#define WGS_BUSY_STAT__MEC1_TC_BUSY_MASK                                                                      0x00000080L
+#define WGS_BUSY_STAT__MEC1_DMA_BUSY_MASK                                                                     0x00000100L
+#define WGS_BUSY_STAT__MEC1_PARTIAL_FLUSH_BUSY_MASK                                                           0x00000200L
+#define WGS_BUSY_STAT__MEC1_PIPE0_BUSY_MASK                                                                   0x00000400L
+#define WGS_BUSY_STAT__MEC1_PIPE1_BUSY_MASK                                                                   0x00000800L
+#define WGS_BUSY_STAT__MEC1_PIPE2_BUSY_MASK                                                                   0x00001000L
+#define WGS_BUSY_STAT__MEC1_PIPE3_BUSY_MASK                                                                   0x00002000L
+#define WGS_BUSY_STAT__MEC2_LOAD_BUSY_MASK                                                                    0x00010000L
+#define WGS_BUSY_STAT__MEC2_MUTEX_BUSY_MASK                                                                   0x00040000L
+#define WGS_BUSY_STAT__MEC2_MESSAGE_BUSY_MASK                                                                 0x00080000L
+#define WGS_BUSY_STAT__MEC2_EOP_QUEUE_BUSY_MASK                                                               0x00100000L
+#define WGS_BUSY_STAT__MEC2_IQ_QUEUE_BUSY_MASK                                                                0x00200000L
+#define WGS_BUSY_STAT__MEC2_IB_QUEUE_BUSY_MASK                                                                0x00400000L
+#define WGS_BUSY_STAT__MEC2_TC_BUSY_MASK                                                                      0x00800000L
+#define WGS_BUSY_STAT__MEC2_DMA_BUSY_MASK                                                                     0x01000000L
+#define WGS_BUSY_STAT__MEC2_PARTIAL_FLUSH_BUSY_MASK                                                           0x02000000L
+#define WGS_BUSY_STAT__MEC2_PIPE0_BUSY_MASK                                                                   0x04000000L
+#define WGS_BUSY_STAT__MEC2_PIPE1_BUSY_MASK                                                                   0x08000000L
+#define WGS_BUSY_STAT__MEC2_PIPE2_BUSY_MASK                                                                   0x10000000L
+#define WGS_BUSY_STAT__MEC2_PIPE3_BUSY_MASK                                                                   0x20000000L
+//WGS_STALLED_STAT1
+#define WGS_STALLED_STAT1__RCIU_TX_FREE_STALL__SHIFT                                                          0x3
+#define WGS_STALLED_STAT1__RCIU_PRIV_VIOLATION__SHIFT                                                         0x4
+#define WGS_STALLED_STAT1__TCIU_TX_FREE_STALL__SHIFT                                                          0x6
+#define WGS_STALLED_STAT1__TCIU_WAITING_ON_TAGS__SHIFT                                                        0x7
+#define WGS_STALLED_STAT1__MEC1_DECODING_PACKET__SHIFT                                                        0x8
+#define WGS_STALLED_STAT1__MEC1_WAIT_ON_RCIU__SHIFT                                                           0x9
+#define WGS_STALLED_STAT1__MEC1_WAIT_ON_RCIU_READ__SHIFT                                                      0xa
+#define WGS_STALLED_STAT1__MEC1_WAIT_ON_ROQ_DATA__SHIFT                                                       0xd
+#define WGS_STALLED_STAT1__MEC2_DECODING_PACKET__SHIFT                                                        0x10
+#define WGS_STALLED_STAT1__MEC2_WAIT_ON_RCIU__SHIFT                                                           0x11
+#define WGS_STALLED_STAT1__MEC2_WAIT_ON_RCIU_READ__SHIFT                                                      0x12
+#define WGS_STALLED_STAT1__MEC2_WAIT_ON_ROQ_DATA__SHIFT                                                       0x15
+#define WGS_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE__SHIFT                                                     0x16
+#define WGS_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS__SHIFT                                                     0x17
+#define WGS_STALLED_STAT1__UTCL1_WAITING_ON_TRANS__SHIFT                                                      0x18
+#define WGS_STALLED_STAT1__GCRIU_WAITING_ON_FREE__SHIFT                                                       0x19
+#define WGS_STALLED_STAT1__RCIU_TX_FREE_STALL_MASK                                                            0x00000008L
+#define WGS_STALLED_STAT1__RCIU_PRIV_VIOLATION_MASK                                                           0x00000010L
+#define WGS_STALLED_STAT1__TCIU_TX_FREE_STALL_MASK                                                            0x00000040L
+#define WGS_STALLED_STAT1__TCIU_WAITING_ON_TAGS_MASK                                                          0x00000080L
+#define WGS_STALLED_STAT1__MEC1_DECODING_PACKET_MASK                                                          0x00000100L
+#define WGS_STALLED_STAT1__MEC1_WAIT_ON_RCIU_MASK                                                             0x00000200L
+#define WGS_STALLED_STAT1__MEC1_WAIT_ON_RCIU_READ_MASK                                                        0x00000400L
+#define WGS_STALLED_STAT1__MEC1_WAIT_ON_ROQ_DATA_MASK                                                         0x00002000L
+#define WGS_STALLED_STAT1__MEC2_DECODING_PACKET_MASK                                                          0x00010000L
+#define WGS_STALLED_STAT1__MEC2_WAIT_ON_RCIU_MASK                                                             0x00020000L
+#define WGS_STALLED_STAT1__MEC2_WAIT_ON_RCIU_READ_MASK                                                        0x00040000L
+#define WGS_STALLED_STAT1__MEC2_WAIT_ON_ROQ_DATA_MASK                                                         0x00200000L
+#define WGS_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE_MASK                                                       0x00400000L
+#define WGS_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS_MASK                                                       0x00800000L
+#define WGS_STALLED_STAT1__UTCL1_WAITING_ON_TRANS_MASK                                                        0x01000000L
+#define WGS_STALLED_STAT1__GCRIU_WAITING_ON_FREE_MASK                                                         0x02000000L
+//WGS_BUSY_STAT2
+#define WGS_BUSY_STAT2__MES_LOAD_BUSY__SHIFT                                                                  0x0
+#define WGS_BUSY_STAT2__MES_MUTEX_BUSY__SHIFT                                                                 0x2
+#define WGS_BUSY_STAT2__MES_MESSAGE_BUSY__SHIFT                                                               0x3
+#define WGS_BUSY_STAT2__MES_TC_BUSY__SHIFT                                                                    0x7
+#define WGS_BUSY_STAT2__MES_DMA_BUSY__SHIFT                                                                   0x8
+#define WGS_BUSY_STAT2__MES_PIPE0_BUSY__SHIFT                                                                 0xa
+#define WGS_BUSY_STAT2__MES_PIPE1_BUSY__SHIFT                                                                 0xb
+#define WGS_BUSY_STAT2__MES_PIPE2_BUSY__SHIFT                                                                 0xc
+#define WGS_BUSY_STAT2__MES_PIPE3_BUSY__SHIFT                                                                 0xd
+#define WGS_BUSY_STAT2__MEC1_PIPE0_DC_BUSY__SHIFT                                                             0x12
+#define WGS_BUSY_STAT2__MEC1_PIPE1_DC_BUSY__SHIFT                                                             0x13
+#define WGS_BUSY_STAT2__MEC1_PIPE2_DC_BUSY__SHIFT                                                             0x14
+#define WGS_BUSY_STAT2__MEC1_PIPE3_DC_BUSY__SHIFT                                                             0x15
+#define WGS_BUSY_STAT2__MES_LOAD_BUSY_MASK                                                                    0x00000001L
+#define WGS_BUSY_STAT2__MES_MUTEX_BUSY_MASK                                                                   0x00000004L
+#define WGS_BUSY_STAT2__MES_MESSAGE_BUSY_MASK                                                                 0x00000008L
+#define WGS_BUSY_STAT2__MES_TC_BUSY_MASK                                                                      0x00000080L
+#define WGS_BUSY_STAT2__MES_DMA_BUSY_MASK                                                                     0x00000100L
+#define WGS_BUSY_STAT2__MES_PIPE0_BUSY_MASK                                                                   0x00000400L
+#define WGS_BUSY_STAT2__MES_PIPE1_BUSY_MASK                                                                   0x00000800L
+#define WGS_BUSY_STAT2__MES_PIPE2_BUSY_MASK                                                                   0x00001000L
+#define WGS_BUSY_STAT2__MES_PIPE3_BUSY_MASK                                                                   0x00002000L
+#define WGS_BUSY_STAT2__MEC1_PIPE0_DC_BUSY_MASK                                                               0x00040000L
+#define WGS_BUSY_STAT2__MEC1_PIPE1_DC_BUSY_MASK                                                               0x00080000L
+#define WGS_BUSY_STAT2__MEC1_PIPE2_DC_BUSY_MASK                                                               0x00100000L
+#define WGS_BUSY_STAT2__MEC1_PIPE3_DC_BUSY_MASK                                                               0x00200000L
+//WGS_ME1_HEADER_DUMP
+#define WGS_ME1_HEADER_DUMP__HEADER_DUMP__SHIFT                                                               0x0
+#define WGS_ME1_HEADER_DUMP__HEADER_DUMP_MASK                                                                 0xFFFFFFFFL
+//WGS_SCRATCH_INDEX
+#define WGS_SCRATCH_INDEX__SCRATCH_INDEX__SHIFT                                                               0x0
+#define WGS_SCRATCH_INDEX__SCRATCH_INDEX_64BIT_MODE__SHIFT                                                    0x1f
+#define WGS_SCRATCH_INDEX__SCRATCH_INDEX_MASK                                                                 0x000001FFL
+#define WGS_SCRATCH_INDEX__SCRATCH_INDEX_64BIT_MODE_MASK                                                      0x80000000L
+//WGS_SCRATCH_DATA
+#define WGS_SCRATCH_DATA__SCRATCH_DATA__SHIFT                                                                 0x0
+#define WGS_SCRATCH_DATA__SCRATCH_DATA_MASK                                                                   0xFFFFFFFFL
+//WGS_LATENCY_STATS_SELECT
+#define WGS_LATENCY_STATS_SELECT__INDEX__SHIFT                                                                0x0
+#define WGS_LATENCY_STATS_SELECT__CLEAR__SHIFT                                                                0x1e
+#define WGS_LATENCY_STATS_SELECT__ENABLE__SHIFT                                                               0x1f
+#define WGS_LATENCY_STATS_SELECT__INDEX_MASK                                                                  0x0000001FL
+#define WGS_LATENCY_STATS_SELECT__CLEAR_MASK                                                                  0x40000000L
+#define WGS_LATENCY_STATS_SELECT__ENABLE_MASK                                                                 0x80000000L
+//WGS_LATENCY_STATS_DATA
+#define WGS_LATENCY_STATS_DATA__DATA__SHIFT                                                                   0x0
+#define WGS_LATENCY_STATS_DATA__DATA_MASK                                                                     0xFFFFFFFFL
+//WGS_TC_PERF_COUNTER_WINDOW_SELECT
+#define WGS_TC_PERF_COUNTER_WINDOW_SELECT__INDEX__SHIFT                                                       0x0
+#define WGS_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS__SHIFT                                                      0x1e
+#define WGS_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE__SHIFT                                                      0x1f
+#define WGS_TC_PERF_COUNTER_WINDOW_SELECT__INDEX_MASK                                                         0x0000001FL
+#define WGS_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS_MASK                                                        0x40000000L
+#define WGS_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE_MASK                                                        0x80000000L
+//WGS_DATA1
+#define WGS_DATA1__DATA__SHIFT                                                                                0x0
+#define WGS_DATA1__DATA_MASK                                                                                  0xFFFFFFFFL
+//WGS_DATA2
+#define WGS_DATA2__DATA__SHIFT                                                                                0x0
+#define WGS_DATA2__DATA_MASK                                                                                  0xFFFFFFFFL
+//WGS_DATA3
+#define WGS_DATA3__DATA__SHIFT                                                                                0x0
+#define WGS_DATA3__DATA_MASK                                                                                  0xFFFFFFFFL
+//WGS_DATA4
+#define WGS_DATA4__DATA__SHIFT                                                                                0x0
+#define WGS_DATA4__DATA_MASK                                                                                  0xFFFFFFFFL
+//WGS_DATA5
+#define WGS_DATA5__DATA__SHIFT                                                                                0x0
+#define WGS_DATA5__DATA_MASK                                                                                  0xFFFFFFFFL
+//WGS_DATA6
+#define WGS_DATA6__DATA__SHIFT                                                                                0x0
+#define WGS_DATA6__DATA_MASK                                                                                  0xFFFFFFFFL
+//WGS_DATA7
+#define WGS_DATA7__DATA__SHIFT                                                                                0x0
+#define WGS_DATA7__DATA_MASK                                                                                  0xFFFFFFFFL
+//WGS_DATA8
+#define WGS_DATA8__DATA__SHIFT                                                                                0x0
+#define WGS_DATA8__DATA_MASK                                                                                  0xFFFFFFFFL
+//WGS_DATA9
+#define WGS_DATA9__DATA__SHIFT                                                                                0x0
+#define WGS_DATA9__DATA_MASK                                                                                  0xFFFFFFFFL
+//WGS_DATA10
+#define WGS_DATA10__DATA__SHIFT                                                                               0x0
+#define WGS_DATA10__DATA_MASK                                                                                 0xFFFFFFFFL
+//WGS_DATA11
+#define WGS_DATA11__DATA__SHIFT                                                                               0x0
+#define WGS_DATA11__DATA_MASK                                                                                 0xFFFFFFFFL
+//WGS_DATA12
+#define WGS_DATA12__DATA__SHIFT                                                                               0x0
+#define WGS_DATA12__DATA_MASK                                                                                 0xFFFFFFFFL
+//WGS_DATA13
+#define WGS_DATA13__DATA__SHIFT                                                                               0x0
+#define WGS_DATA13__DATA_MASK                                                                                 0xFFFFFFFFL
+//WGS_DATA14
+#define WGS_DATA14__DATA__SHIFT                                                                               0x0
+#define WGS_DATA14__DATA_MASK                                                                                 0xFFFFFFFFL
+//WGS_DATA15
+#define WGS_DATA15__DATA__SHIFT                                                                               0x0
+#define WGS_DATA15__DATA_MASK                                                                                 0xFFFFFFFFL
+//WGS_DATA16
+#define WGS_DATA16__DATA__SHIFT                                                                               0x0
+#define WGS_DATA16__DATA_MASK                                                                                 0xFFFFFFFFL
+//WGS_METADATA_BASE_ADDR_LO
+#define WGS_METADATA_BASE_ADDR_LO__MD_BASE_LO__SHIFT                                                          0x2
+#define WGS_METADATA_BASE_ADDR_LO__MD_BASE_LO_MASK                                                            0xFFFFFFFCL
+//WGS_METADATA_BASE_ADDR_HI
+#define WGS_METADATA_BASE_ADDR_HI__MD_BASE_HI__SHIFT                                                          0x0
+#define WGS_METADATA_BASE_ADDR_HI__MD_BASE_HI_MASK                                                            0x0000FFFFL
+//WGS_METADATA_CNTL
+#define WGS_METADATA_CNTL__VMID__SHIFT                                                                        0x0
+#define WGS_METADATA_CNTL__EVENT_MODE__SHIFT                                                                  0x4
+#define WGS_METADATA_CNTL__SCOPE__SHIFT                                                                       0x6
+#define WGS_METADATA_CNTL__TEMPORAL__SHIFT                                                                    0x8
+#define WGS_METADATA_CNTL__EVENT1__SHIFT                                                                      0xb
+#define WGS_METADATA_CNTL__QUEUE_ID__SHIFT                                                                    0xc
+#define WGS_METADATA_CNTL__VMID_MASK                                                                          0x0000000FL
+#define WGS_METADATA_CNTL__EVENT_MODE_MASK                                                                    0x00000030L
+#define WGS_METADATA_CNTL__SCOPE_MASK                                                                         0x000000C0L
+#define WGS_METADATA_CNTL__TEMPORAL_MASK                                                                      0x00000700L
+#define WGS_METADATA_CNTL__EVENT1_MASK                                                                        0x00000800L
+#define WGS_METADATA_CNTL__QUEUE_ID_MASK                                                                      0x00007000L
+//WGS_IQ_TIMER
+#define WGS_IQ_TIMER__WAIT_TIME__SHIFT                                                                        0x0
+#define WGS_IQ_TIMER__RETRY_TYPE__SHIFT                                                                       0x8
+#define WGS_IQ_TIMER__IMMEDIATE_EXPIRE__SHIFT                                                                 0xb
+#define WGS_IQ_TIMER__INTERRUPT_TYPE__SHIFT                                                                   0xc
+#define WGS_IQ_TIMER__CLOCK_COUNT__SHIFT                                                                      0xe
+#define WGS_IQ_TIMER__RESERVED3__SHIFT                                                                        0x10
+#define WGS_IQ_TIMER__QUANTUM_TIMER__SHIFT                                                                    0x16
+#define WGS_IQ_TIMER__RESERVED2__SHIFT                                                                        0x17
+#define WGS_IQ_TIMER__QUEUE_TYPE__SHIFT                                                                       0x1b
+#define WGS_IQ_TIMER__REARM_TIMER__SHIFT                                                                      0x1c
+#define WGS_IQ_TIMER__RESERVED1__SHIFT                                                                        0x1d
+#define WGS_IQ_TIMER__ACTIVE__SHIFT                                                                           0x1f
+#define WGS_IQ_TIMER__WAIT_TIME_MASK                                                                          0x000000FFL
+#define WGS_IQ_TIMER__RETRY_TYPE_MASK                                                                         0x00000700L
+#define WGS_IQ_TIMER__IMMEDIATE_EXPIRE_MASK                                                                   0x00000800L
+#define WGS_IQ_TIMER__INTERRUPT_TYPE_MASK                                                                     0x00003000L
+#define WGS_IQ_TIMER__CLOCK_COUNT_MASK                                                                        0x0000C000L
+#define WGS_IQ_TIMER__RESERVED3_MASK                                                                          0x003F0000L
+#define WGS_IQ_TIMER__QUANTUM_TIMER_MASK                                                                      0x00400000L
+#define WGS_IQ_TIMER__RESERVED2_MASK                                                                          0x07800000L
+#define WGS_IQ_TIMER__QUEUE_TYPE_MASK                                                                         0x08000000L
+#define WGS_IQ_TIMER__REARM_TIMER_MASK                                                                        0x10000000L
+#define WGS_IQ_TIMER__RESERVED1_MASK                                                                          0x60000000L
+#define WGS_IQ_TIMER__ACTIVE_MASK                                                                             0x80000000L
+//WGS_HALT_HYST_COUNT
+#define WGS_HALT_HYST_COUNT__COUNT__SHIFT                                                                     0x0
+#define WGS_HALT_HYST_COUNT__COUNT_MASK                                                                       0x0000000FL
+//WGS_IQ_TIMER_MESSAGE
+#define WGS_IQ_TIMER_MESSAGE__MESSAGE__SHIFT                                                                  0x0
+#define WGS_IQ_TIMER_MESSAGE__MESSAGE_MASK                                                                    0x000000FFL
+//WGS_RS64_THREAD_CNTL
+#define WGS_RS64_THREAD_CNTL__MEC_STATUS__SHIFT                                                               0x0
+#define WGS_RS64_THREAD_CNTL__MEC_INVALIDATE_ICACHE__SHIFT                                                    0x4
+#define WGS_RS64_THREAD_CNTL__MEC_PIPE_RESET__SHIFT                                                           0x10
+#define WGS_RS64_THREAD_CNTL__RSRV7__SHIFT                                                                    0x11
+#define WGS_RS64_THREAD_CNTL__RSRV6__SHIFT                                                                    0x12
+#define WGS_RS64_THREAD_CNTL__RSRV5__SHIFT                                                                    0x13
+#define WGS_RS64_THREAD_CNTL__MEC_PIPE_ACTIVE__SHIFT                                                          0x1a
+#define WGS_RS64_THREAD_CNTL__RSRV4__SHIFT                                                                    0x1b
+#define WGS_RS64_THREAD_CNTL__RSRV3__SHIFT                                                                    0x1c
+#define WGS_RS64_THREAD_CNTL__RSRV2__SHIFT                                                                    0x1d
+#define WGS_RS64_THREAD_CNTL__SPARE__SHIFT                                                                    0x1e
+#define WGS_RS64_THREAD_CNTL__RSRV1__SHIFT                                                                    0x1f
+#define WGS_RS64_THREAD_CNTL__MEC_STATUS_MASK                                                                 0x0000000FL
+#define WGS_RS64_THREAD_CNTL__MEC_INVALIDATE_ICACHE_MASK                                                      0x00000010L
+#define WGS_RS64_THREAD_CNTL__MEC_PIPE_RESET_MASK                                                             0x00010000L
+#define WGS_RS64_THREAD_CNTL__RSRV7_MASK                                                                      0x00020000L
+#define WGS_RS64_THREAD_CNTL__RSRV6_MASK                                                                      0x00040000L
+#define WGS_RS64_THREAD_CNTL__RSRV5_MASK                                                                      0x00080000L
+#define WGS_RS64_THREAD_CNTL__MEC_PIPE_ACTIVE_MASK                                                            0x04000000L
+#define WGS_RS64_THREAD_CNTL__RSRV4_MASK                                                                      0x08000000L
+#define WGS_RS64_THREAD_CNTL__RSRV3_MASK                                                                      0x10000000L
+#define WGS_RS64_THREAD_CNTL__RSRV2_MASK                                                                      0x20000000L
+#define WGS_RS64_THREAD_CNTL__SPARE_MASK                                                                      0x40000000L
+#define WGS_RS64_THREAD_CNTL__RSRV1_MASK                                                                      0x80000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_gl1dec
+//GL1_ARB_CTRL
+#define GL1_ARB_CTRL__NUM_MEM_PIPES__SHIFT                                                                    0x0
+#define GL1_ARB_CTRL__FGCG_DISABLE__SHIFT                                                                     0x4
+#define GL1_ARB_CTRL__PERF_CNTR_EN_OVERRIDE__SHIFT                                                            0x5
+#define GL1_ARB_CTRL__CHICKEN_BITS__SHIFT                                                                     0x6
+#define GL1_ARB_CTRL__RETURN_ARB_MODE_COUNTER__SHIFT                                                          0xe
+#define GL1_ARB_CTRL__RETURN_256B_ARB_MODE_COUNTER__SHIFT                                                     0x12
+#define GL1_ARB_CTRL__NUM_MEM_PIPES_MASK                                                                      0x0000000FL
+#define GL1_ARB_CTRL__FGCG_DISABLE_MASK                                                                       0x00000010L
+#define GL1_ARB_CTRL__PERF_CNTR_EN_OVERRIDE_MASK                                                              0x00000020L
+#define GL1_ARB_CTRL__CHICKEN_BITS_MASK                                                                       0x00003FC0L
+#define GL1_ARB_CTRL__RETURN_ARB_MODE_COUNTER_MASK                                                            0x0003C000L
+#define GL1_ARB_CTRL__RETURN_256B_ARB_MODE_COUNTER_MASK                                                       0x003C0000L
+//GL1_DRAM_BURST_MASK
+#define GL1_DRAM_BURST_MASK__DRAM_BURST_ADDR_MASK__SHIFT                                                      0x0
+#define GL1_DRAM_BURST_MASK__DRAM_BURST_ADDR_MASK_MASK                                                        0x000000FFL
+//GL1_ARB_STATUS
+#define GL1_ARB_STATUS__REQ_ARB_BUSY__SHIFT                                                                   0x0
+#define GL1_ARB_STATUS__RET_ARB_BUSY__SHIFT                                                                   0x1
+#define GL1_ARB_STATUS__REQ_ARB_BUSY_MASK                                                                     0x00000001L
+#define GL1_ARB_STATUS__RET_ARB_BUSY_MASK                                                                     0x00000002L
+//GL1_DRAM_BURST_CTRL
+#define GL1_DRAM_BURST_CTRL__MAX_DRAM_BURST__SHIFT                                                            0x0
+#define GL1_DRAM_BURST_CTRL__BURST_DISABLE__SHIFT                                                             0x3
+#define GL1_DRAM_BURST_CTRL__MAX_DRAM_BURST_MASK                                                              0x00000007L
+#define GL1_DRAM_BURST_CTRL__BURST_DISABLE_MASK                                                               0x00000008L
+//GL1I_GL1R_REP_FGCG_OVERRIDE
+#define GL1I_GL1R_REP_FGCG_OVERRIDE__GL1A_GL1IR_REP_FGCG_OVERRIDE__SHIFT                                      0x0
+#define GL1I_GL1R_REP_FGCG_OVERRIDE__GL1A_GL1IW_REP_FGCG_OVERRIDE__SHIFT                                      0x1
+#define GL1I_GL1R_REP_FGCG_OVERRIDE__GL1A_GL1R_SRC_REP_FGCG_OVERRIDE__SHIFT                                   0x2
+#define GL1I_GL1R_REP_FGCG_OVERRIDE__GL1A_GL1R_RET_REP_FGCG_OVERRIDE__SHIFT                                   0x3
+#define GL1I_GL1R_REP_FGCG_OVERRIDE__GL1A_GL1IR_REP_FGCG_OVERRIDE_MASK                                        0x00000001L
+#define GL1I_GL1R_REP_FGCG_OVERRIDE__GL1A_GL1IW_REP_FGCG_OVERRIDE_MASK                                        0x00000002L
+#define GL1I_GL1R_REP_FGCG_OVERRIDE__GL1A_GL1R_SRC_REP_FGCG_OVERRIDE_MASK                                     0x00000004L
+#define GL1I_GL1R_REP_FGCG_OVERRIDE__GL1A_GL1R_RET_REP_FGCG_OVERRIDE_MASK                                     0x00000008L
+//GL1A_GL1C_CREDITS
+#define GL1A_GL1C_CREDITS__GL1C_REQ_CREDITS__SHIFT                                                            0x0
+#define GL1A_GL1C_CREDITS__GL1C_DATA_CREDITS__SHIFT                                                           0x8
+#define GL1A_GL1C_CREDITS__GL1C_REQ_CREDITS_MASK                                                              0x000000FFL
+#define GL1A_GL1C_CREDITS__GL1C_DATA_CREDITS_MASK                                                             0x0000FF00L
+//GL1A_CLIENT_FREE_DELAY
+#define GL1A_CLIENT_FREE_DELAY__CLIENT_TYPE_0_FREE_DELAY__SHIFT                                               0x0
+#define GL1A_CLIENT_FREE_DELAY__CLIENT_TYPE_1_FREE_DELAY__SHIFT                                               0x3
+#define GL1A_CLIENT_FREE_DELAY__CLIENT_TYPE_2_FREE_DELAY__SHIFT                                               0x6
+#define GL1A_CLIENT_FREE_DELAY__CLIENT_TYPE_0_FREE_DELAY_MASK                                                 0x00000007L
+#define GL1A_CLIENT_FREE_DELAY__CLIENT_TYPE_1_FREE_DELAY_MASK                                                 0x00000038L
+#define GL1A_CLIENT_FREE_DELAY__CLIENT_TYPE_2_FREE_DELAY_MASK                                                 0x000001C0L
+//GL1X_ARB_CTRL
+#define GL1X_ARB_CTRL__NUM_MEM_PIPES__SHIFT                                                                   0x0
+#define GL1X_ARB_CTRL__FGCG_DISABLE__SHIFT                                                                    0x2
+#define GL1X_ARB_CTRL__PERF_CNTR_EN_OVERRIDE__SHIFT                                                           0x3
+#define GL1X_ARB_CTRL__CHICKEN_BITS__SHIFT                                                                    0x4
+#define GL1X_ARB_CTRL__NUM_MEM_PIPES_MASK                                                                     0x00000003L
+#define GL1X_ARB_CTRL__FGCG_DISABLE_MASK                                                                      0x00000004L
+#define GL1X_ARB_CTRL__PERF_CNTR_EN_OVERRIDE_MASK                                                             0x00000008L
+#define GL1X_ARB_CTRL__CHICKEN_BITS_MASK                                                                      0x00000FF0L
+//GL1X_DRAM_BURST_MASK
+#define GL1X_DRAM_BURST_MASK__DRAM_BURST_ADDR_MASK__SHIFT                                                     0x0
+#define GL1X_DRAM_BURST_MASK__DRAM_BURST_ADDR_MASK_MASK                                                       0x000000FFL
+//GL1X_ARB_STATUS
+#define GL1X_ARB_STATUS__REQ_ARB_BUSY__SHIFT                                                                  0x0
+#define GL1X_ARB_STATUS__RET_ARB_BUSY__SHIFT                                                                  0x1
+#define GL1X_ARB_STATUS__REQ_ARB_BUSY_MASK                                                                    0x00000001L
+#define GL1X_ARB_STATUS__RET_ARB_BUSY_MASK                                                                    0x00000002L
+//GL1X_DRAM_BURST_CTRL
+#define GL1X_DRAM_BURST_CTRL__MAX_DRAM_BURST__SHIFT                                                           0x0
+#define GL1X_DRAM_BURST_CTRL__BURST_DISABLE__SHIFT                                                            0x3
+#define GL1X_DRAM_BURST_CTRL__MAX_DRAM_BURST_MASK                                                             0x00000007L
+#define GL1X_DRAM_BURST_CTRL__BURST_DISABLE_MASK                                                              0x00000008L
+//GL1XI_GL1XR_REP_FGCG_OVERRIDE
+#define GL1XI_GL1XR_REP_FGCG_OVERRIDE__GL1XA_GL1XIR_REP_FGCG_OVERRIDE__SHIFT                                  0x0
+#define GL1XI_GL1XR_REP_FGCG_OVERRIDE__GL1XA_GL1XIW_REP_FGCG_OVERRIDE__SHIFT                                  0x1
+#define GL1XI_GL1XR_REP_FGCG_OVERRIDE__GL1XA_GL1XR_SRC_REP_FGCG_OVERRIDE__SHIFT                               0x2
+#define GL1XI_GL1XR_REP_FGCG_OVERRIDE__GL1XA_GL1XR_RET_REP_FGCG_OVERRIDE__SHIFT                               0x3
+#define GL1XI_GL1XR_REP_FGCG_OVERRIDE__GL1XA_GL1XIR_REP_FGCG_OVERRIDE_MASK                                    0x00000001L
+#define GL1XI_GL1XR_REP_FGCG_OVERRIDE__GL1XA_GL1XIW_REP_FGCG_OVERRIDE_MASK                                    0x00000002L
+#define GL1XI_GL1XR_REP_FGCG_OVERRIDE__GL1XA_GL1XR_SRC_REP_FGCG_OVERRIDE_MASK                                 0x00000004L
+#define GL1XI_GL1XR_REP_FGCG_OVERRIDE__GL1XA_GL1XR_RET_REP_FGCG_OVERRIDE_MASK                                 0x00000008L
+//GL1XA_GL1XC_CREDITS
+#define GL1XA_GL1XC_CREDITS__GL1XC_REQ_CREDITS__SHIFT                                                         0x0
+#define GL1XA_GL1XC_CREDITS__GL1XC_DATA_CREDITS__SHIFT                                                        0x8
+#define GL1XA_GL1XC_CREDITS__GL1XC_REQ_CREDITS_MASK                                                           0x000000FFL
+#define GL1XA_GL1XC_CREDITS__GL1XC_DATA_CREDITS_MASK                                                          0x0000FF00L
+//GL1XA_CLIENT_FREE_DELAY
+#define GL1XA_CLIENT_FREE_DELAY__CLIENT_TYPE_0_FREE_DELAY__SHIFT                                              0x0
+#define GL1XA_CLIENT_FREE_DELAY__CLIENT_TYPE_1_FREE_DELAY__SHIFT                                              0x3
+#define GL1XA_CLIENT_FREE_DELAY__CLIENT_TYPE_2_FREE_DELAY__SHIFT                                              0x6
+#define GL1XA_CLIENT_FREE_DELAY__CLIENT_TYPE_3_FREE_DELAY__SHIFT                                              0x9
+#define GL1XA_CLIENT_FREE_DELAY__CLIENT_TYPE_4_FREE_DELAY__SHIFT                                              0xc
+#define GL1XA_CLIENT_FREE_DELAY__CLIENT_TYPE_5_FREE_DELAY__SHIFT                                              0xf
+#define GL1XA_CLIENT_FREE_DELAY__CLIENT_TYPE_0_FREE_DELAY_MASK                                                0x00000007L
+#define GL1XA_CLIENT_FREE_DELAY__CLIENT_TYPE_1_FREE_DELAY_MASK                                                0x00000038L
+#define GL1XA_CLIENT_FREE_DELAY__CLIENT_TYPE_2_FREE_DELAY_MASK                                                0x000001C0L
+#define GL1XA_CLIENT_FREE_DELAY__CLIENT_TYPE_3_FREE_DELAY_MASK                                                0x00000E00L
+#define GL1XA_CLIENT_FREE_DELAY__CLIENT_TYPE_4_FREE_DELAY_MASK                                                0x00007000L
+#define GL1XA_CLIENT_FREE_DELAY__CLIENT_TYPE_5_FREE_DELAY_MASK                                                0x00038000L
+//GL1XA_COMPRESSION_MODE
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_0_OVERRIDE__SHIFT                                                 0x0
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_0_WRITE_COMPRESSION_DISABLE__SHIFT                                0x1
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_0_BYPASS_COMPRESSION__SHIFT                                       0x2
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_1_OVERRIDE__SHIFT                                                 0x3
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_1_WRITE_COMPRESSION_DISABLE__SHIFT                                0x4
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_1_BYPASS_COMPRESSION__SHIFT                                       0x5
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_2_OVERRIDE__SHIFT                                                 0x6
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_2_WRITE_COMPRESSION_DISABLE__SHIFT                                0x7
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_2_BYPASS_COMPRESSION__SHIFT                                       0x8
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_3_OVERRIDE__SHIFT                                                 0x9
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_3_WRITE_COMPRESSION_DISABLE__SHIFT                                0xa
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_3_BYPASS_COMPRESSION__SHIFT                                       0xb
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_4_OVERRIDE__SHIFT                                                 0xc
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_4_WRITE_COMPRESSION_DISABLE__SHIFT                                0xd
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_4_BYPASS_COMPRESSION__SHIFT                                       0xe
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_5_OVERRIDE__SHIFT                                                 0xf
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_5_WRITE_COMPRESSION_DISABLE__SHIFT                                0x10
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_5_BYPASS_COMPRESSION__SHIFT                                       0x11
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_6_OVERRIDE__SHIFT                                                 0x12
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_6_WRITE_COMPRESSION_DISABLE__SHIFT                                0x13
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_6_BYPASS_COMPRESSION__SHIFT                                       0x14
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_7_OVERRIDE__SHIFT                                                 0x15
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_7_WRITE_COMPRESSION_DISABLE__SHIFT                                0x16
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_7_BYPASS_COMPRESSION__SHIFT                                       0x17
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_0_OVERRIDE_MASK                                                   0x00000001L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_0_WRITE_COMPRESSION_DISABLE_MASK                                  0x00000002L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_0_BYPASS_COMPRESSION_MASK                                         0x00000004L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_1_OVERRIDE_MASK                                                   0x00000008L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_1_WRITE_COMPRESSION_DISABLE_MASK                                  0x00000010L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_1_BYPASS_COMPRESSION_MASK                                         0x00000020L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_2_OVERRIDE_MASK                                                   0x00000040L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_2_WRITE_COMPRESSION_DISABLE_MASK                                  0x00000080L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_2_BYPASS_COMPRESSION_MASK                                         0x00000100L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_3_OVERRIDE_MASK                                                   0x00000200L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_3_WRITE_COMPRESSION_DISABLE_MASK                                  0x00000400L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_3_BYPASS_COMPRESSION_MASK                                         0x00000800L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_4_OVERRIDE_MASK                                                   0x00001000L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_4_WRITE_COMPRESSION_DISABLE_MASK                                  0x00002000L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_4_BYPASS_COMPRESSION_MASK                                         0x00004000L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_5_OVERRIDE_MASK                                                   0x00008000L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_5_WRITE_COMPRESSION_DISABLE_MASK                                  0x00010000L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_5_BYPASS_COMPRESSION_MASK                                         0x00020000L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_6_OVERRIDE_MASK                                                   0x00040000L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_6_WRITE_COMPRESSION_DISABLE_MASK                                  0x00080000L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_6_BYPASS_COMPRESSION_MASK                                         0x00100000L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_7_OVERRIDE_MASK                                                   0x00200000L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_7_WRITE_COMPRESSION_DISABLE_MASK                                  0x00400000L
+#define GL1XA_COMPRESSION_MODE__CLIENT_TYPE_7_BYPASS_COMPRESSION_MASK                                         0x00800000L
+//GL1XA_COMPRESSOR_OVERRIDE
+#define GL1XA_COMPRESSOR_OVERRIDE__DATA_FORMAT__SHIFT                                                         0x0
+#define GL1XA_COMPRESSOR_OVERRIDE__MAX_COMP_BLOCK_SIZE__SHIFT                                                 0x7
+#define GL1XA_COMPRESSOR_OVERRIDE__MAX_UNCOMP_BLOCK_SIZE__SHIFT                                               0x9
+#define GL1XA_COMPRESSOR_OVERRIDE__MICRO_TILE_MODE__SHIFT                                                     0xa
+#define GL1XA_COMPRESSOR_OVERRIDE__NUM_SAMPLES_LOG2__SHIFT                                                    0xc
+#define GL1XA_COMPRESSOR_OVERRIDE__NUMBER_TYPE__SHIFT                                                         0xe
+#define GL1XA_COMPRESSOR_OVERRIDE__DATA_FORMAT_MASK                                                           0x0000003FL
+#define GL1XA_COMPRESSOR_OVERRIDE__MAX_COMP_BLOCK_SIZE_MASK                                                   0x00000180L
+#define GL1XA_COMPRESSOR_OVERRIDE__MAX_UNCOMP_BLOCK_SIZE_MASK                                                 0x00000200L
+#define GL1XA_COMPRESSOR_OVERRIDE__MICRO_TILE_MODE_MASK                                                       0x00000C00L
+#define GL1XA_COMPRESSOR_OVERRIDE__NUM_SAMPLES_LOG2_MASK                                                      0x00003000L
+#define GL1XA_COMPRESSOR_OVERRIDE__NUMBER_TYPE_MASK                                                           0x0001C000L
+//GL1C_CTRL
+#define GL1C_CTRL__BUFFER_DEPTH_MAX__SHIFT                                                                    0x0
+#define GL1C_CTRL__GL2_REQ_CREDITS__SHIFT                                                                     0x4
+#define GL1C_CTRL__GL2_DATA_CREDITS__SHIFT                                                                    0xb
+#define GL1C_CTRL__TO_L1_REPEATER_FGCG_DISABLE__SHIFT                                                         0x12
+#define GL1C_CTRL__TO_L2_REPEATER_FGCG_DISABLE__SHIFT                                                         0x13
+#define GL1C_CTRL__GL1C_LDBDCST_TIMER__SHIFT                                                                  0x14
+#define GL1C_CTRL__BUFFER_DEPTH_MAX_MASK                                                                      0x0000000FL
+#define GL1C_CTRL__GL2_REQ_CREDITS_MASK                                                                       0x000007F0L
+#define GL1C_CTRL__GL2_DATA_CREDITS_MASK                                                                      0x0003F800L
+#define GL1C_CTRL__TO_L1_REPEATER_FGCG_DISABLE_MASK                                                           0x00040000L
+#define GL1C_CTRL__TO_L2_REPEATER_FGCG_DISABLE_MASK                                                           0x00080000L
+#define GL1C_CTRL__GL1C_LDBDCST_TIMER_MASK                                                                    0x1FF00000L
+//GL1C_STATUS
+#define GL1C_STATUS__INPUT_BUFFER_VC0_FIFO_FULL__SHIFT                                                        0x0
+#define GL1C_STATUS__OUTPUT_FIFOS_BUSY__SHIFT                                                                 0x1
+#define GL1C_STATUS__GL2_REQ_VC0_STALL__SHIFT                                                                 0x3
+#define GL1C_STATUS__GL2_DATA_VC0_STALL__SHIFT                                                                0x4
+#define GL1C_STATUS__GL2_REQ_VC1_STALL__SHIFT                                                                 0x5
+#define GL1C_STATUS__GL2_DATA_VC1_STALL__SHIFT                                                                0x6
+#define GL1C_STATUS__INPUT_BUFFER_VC0_BUSY__SHIFT                                                             0x7
+#define GL1C_STATUS__SRC_DATA_FIFO_VC0_BUSY__SHIFT                                                            0x8
+#define GL1C_STATUS__GL2_RH_BUSY__SHIFT                                                                       0x9
+#define GL1C_STATUS__NUM_REQ_PENDING_FROM_L2__SHIFT                                                           0xa
+#define GL1C_STATUS__VIRTUAL_FIFO_FULL_STALL__SHIFT                                                           0x14
+#define GL1C_STATUS__REQUEST_TRACKER_BUFFER_STALL__SHIFT                                                      0x15
+#define GL1C_STATUS__REQUEST_TRACKER_BUSY__SHIFT                                                              0x16
+#define GL1C_STATUS__BUFFER_FULL__SHIFT                                                                       0x17
+#define GL1C_STATUS__INPUT_BUFFER_VC0_FIFO_FULL_MASK                                                          0x00000001L
+#define GL1C_STATUS__OUTPUT_FIFOS_BUSY_MASK                                                                   0x00000002L
+#define GL1C_STATUS__GL2_REQ_VC0_STALL_MASK                                                                   0x00000008L
+#define GL1C_STATUS__GL2_DATA_VC0_STALL_MASK                                                                  0x00000010L
+#define GL1C_STATUS__GL2_REQ_VC1_STALL_MASK                                                                   0x00000020L
+#define GL1C_STATUS__GL2_DATA_VC1_STALL_MASK                                                                  0x00000040L
+#define GL1C_STATUS__INPUT_BUFFER_VC0_BUSY_MASK                                                               0x00000080L
+#define GL1C_STATUS__SRC_DATA_FIFO_VC0_BUSY_MASK                                                              0x00000100L
+#define GL1C_STATUS__GL2_RH_BUSY_MASK                                                                         0x00000200L
+#define GL1C_STATUS__NUM_REQ_PENDING_FROM_L2_MASK                                                             0x000FFC00L
+#define GL1C_STATUS__VIRTUAL_FIFO_FULL_STALL_MASK                                                             0x00100000L
+#define GL1C_STATUS__REQUEST_TRACKER_BUFFER_STALL_MASK                                                        0x00200000L
+#define GL1C_STATUS__REQUEST_TRACKER_BUSY_MASK                                                                0x00400000L
+#define GL1C_STATUS__BUFFER_FULL_MASK                                                                         0x00800000L
+//GL1C_CTRL2
+#define GL1C_CTRL2__OVERRIDE_READ_BYPASS_TO_DECOMPRESSED__SHIFT                                               0x8
+#define GL1C_CTRL2__OVERRIDE_WRITE_BYPASS_TO_COMPRESSION_DISABLE__SHIFT                                       0x9
+#define GL1C_CTRL2__OVERRIDE_READ_BYPASS_TO_DECOMPRESSED_MASK                                                 0x00000100L
+#define GL1C_CTRL2__OVERRIDE_WRITE_BYPASS_TO_COMPRESSION_DISABLE_MASK                                         0x00000200L
+//GL1XC_CTRL
+#define GL1XC_CTRL__BUFFER_DEPTH_MAX__SHIFT                                                                   0x0
+#define GL1XC_CTRL__GL2_REQ_CREDITS__SHIFT                                                                    0x4
+#define GL1XC_CTRL__GL2_DATA_CREDITS__SHIFT                                                                   0xb
+#define GL1XC_CTRL__TO_L1_REPEATER_FGCG_DISABLE__SHIFT                                                        0x12
+#define GL1XC_CTRL__TO_L2_REPEATER_FGCG_DISABLE__SHIFT                                                        0x13
+#define GL1XC_CTRL__GCR_RSP_FGCG_DISABLE__SHIFT                                                               0x14
+#define GL1XC_CTRL__BUFFER_DEPTH_MAX_MASK                                                                     0x0000000FL
+#define GL1XC_CTRL__GL2_REQ_CREDITS_MASK                                                                      0x000007F0L
+#define GL1XC_CTRL__GL2_DATA_CREDITS_MASK                                                                     0x0003F800L
+#define GL1XC_CTRL__TO_L1_REPEATER_FGCG_DISABLE_MASK                                                          0x00040000L
+#define GL1XC_CTRL__TO_L2_REPEATER_FGCG_DISABLE_MASK                                                          0x00080000L
+#define GL1XC_CTRL__GCR_RSP_FGCG_DISABLE_MASK                                                                 0x00100000L
+//GL1XC_STATUS
+#define GL1XC_STATUS__INPUT_BUFFER_VC0_FIFO_FULL__SHIFT                                                       0x0
+#define GL1XC_STATUS__OUTPUT_FIFOS_BUSY__SHIFT                                                                0x1
+#define GL1XC_STATUS__GL2_REQ_VC0_STALL__SHIFT                                                                0x3
+#define GL1XC_STATUS__GL2_DATA_VC0_STALL__SHIFT                                                               0x4
+#define GL1XC_STATUS__GL2_REQ_VC1_STALL__SHIFT                                                                0x5
+#define GL1XC_STATUS__GL2_DATA_VC1_STALL__SHIFT                                                               0x6
+#define GL1XC_STATUS__INPUT_BUFFER_VC0_BUSY__SHIFT                                                            0x7
+#define GL1XC_STATUS__SRC_DATA_FIFO_VC0_BUSY__SHIFT                                                           0x8
+#define GL1XC_STATUS__GL2_RH_BUSY__SHIFT                                                                      0x9
+#define GL1XC_STATUS__NUM_REQ_PENDING_FROM_L2__SHIFT                                                          0xa
+#define GL1XC_STATUS__VIRTUAL_FIFO_FULL_STALL__SHIFT                                                          0x14
+#define GL1XC_STATUS__REQUEST_TRACKER_BUFFER_STALL__SHIFT                                                     0x15
+#define GL1XC_STATUS__REQUEST_TRACKER_BUSY__SHIFT                                                             0x16
+#define GL1XC_STATUS__BUFFER_FULL__SHIFT                                                                      0x17
+#define GL1XC_STATUS__INPUT_BUFFER_VC0_FIFO_FULL_MASK                                                         0x00000001L
+#define GL1XC_STATUS__OUTPUT_FIFOS_BUSY_MASK                                                                  0x00000002L
+#define GL1XC_STATUS__GL2_REQ_VC0_STALL_MASK                                                                  0x00000008L
+#define GL1XC_STATUS__GL2_DATA_VC0_STALL_MASK                                                                 0x00000010L
+#define GL1XC_STATUS__GL2_REQ_VC1_STALL_MASK                                                                  0x00000020L
+#define GL1XC_STATUS__GL2_DATA_VC1_STALL_MASK                                                                 0x00000040L
+#define GL1XC_STATUS__INPUT_BUFFER_VC0_BUSY_MASK                                                              0x00000080L
+#define GL1XC_STATUS__SRC_DATA_FIFO_VC0_BUSY_MASK                                                             0x00000100L
+#define GL1XC_STATUS__GL2_RH_BUSY_MASK                                                                        0x00000200L
+#define GL1XC_STATUS__NUM_REQ_PENDING_FROM_L2_MASK                                                            0x000FFC00L
+#define GL1XC_STATUS__VIRTUAL_FIFO_FULL_STALL_MASK                                                            0x00100000L
+#define GL1XC_STATUS__REQUEST_TRACKER_BUFFER_STALL_MASK                                                       0x00200000L
+#define GL1XC_STATUS__REQUEST_TRACKER_BUSY_MASK                                                               0x00400000L
+#define GL1XC_STATUS__BUFFER_FULL_MASK                                                                        0x00800000L
+//GL1XC_UTCL0_CNTL1
+#define GL1XC_UTCL0_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                            0x0
+#define GL1XC_UTCL0_CNTL1__GPUVM_64K_DEFAULT__SHIFT                                                           0x1
+#define GL1XC_UTCL0_CNTL1__GPUVM_PERM_MODE__SHIFT                                                             0x2
+#define GL1XC_UTCL0_CNTL1__RESP_MODE__SHIFT                                                                   0x3
+#define GL1XC_UTCL0_CNTL1__RESP_FAULT_MODE__SHIFT                                                             0x5
+#define GL1XC_UTCL0_CNTL1__CLIENTID__SHIFT                                                                    0x7
+#define GL1XC_UTCL0_CNTL1__REG_INV_VMID__SHIFT                                                                0x13
+#define GL1XC_UTCL0_CNTL1__REG_INV_TOGGLE__SHIFT                                                              0x18
+#define GL1XC_UTCL0_CNTL1__ATOMIC_REQUEST_ENABLE__SHIFT                                                       0x19
+#define GL1XC_UTCL0_CNTL1__FORCE_MISS__SHIFT                                                                  0x1a
+#define GL1XC_UTCL0_CNTL1__FORCE_IN_ORDER__SHIFT                                                              0x1b
+#define GL1XC_UTCL0_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                      0x1c
+#define GL1XC_UTCL0_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                      0x1e
+#define GL1XC_UTCL0_CNTL1__FORCE_4K_L2_RESP_MASK                                                              0x00000001L
+#define GL1XC_UTCL0_CNTL1__GPUVM_64K_DEFAULT_MASK                                                             0x00000002L
+#define GL1XC_UTCL0_CNTL1__GPUVM_PERM_MODE_MASK                                                               0x00000004L
+#define GL1XC_UTCL0_CNTL1__RESP_MODE_MASK                                                                     0x00000018L
+#define GL1XC_UTCL0_CNTL1__RESP_FAULT_MODE_MASK                                                               0x00000060L
+#define GL1XC_UTCL0_CNTL1__CLIENTID_MASK                                                                      0x0000FF80L
+#define GL1XC_UTCL0_CNTL1__REG_INV_VMID_MASK                                                                  0x00780000L
+#define GL1XC_UTCL0_CNTL1__REG_INV_TOGGLE_MASK                                                                0x01000000L
+#define GL1XC_UTCL0_CNTL1__ATOMIC_REQUEST_ENABLE_MASK                                                         0x02000000L
+#define GL1XC_UTCL0_CNTL1__FORCE_MISS_MASK                                                                    0x04000000L
+#define GL1XC_UTCL0_CNTL1__FORCE_IN_ORDER_MASK                                                                0x08000000L
+#define GL1XC_UTCL0_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                        0x30000000L
+#define GL1XC_UTCL0_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                        0xC0000000L
+//GL1XC_UTCL0_CNTL2
+#define GL1XC_UTCL0_CNTL2__SPARE__SHIFT                                                                       0x0
+#define GL1XC_UTCL0_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                              0x9
+#define GL1XC_UTCL0_CNTL2__ANY_LINE_VALID__SHIFT                                                              0xa
+#define GL1XC_UTCL0_CNTL2__FORCE_SNOOP__SHIFT                                                                 0xe
+#define GL1XC_UTCL0_CNTL2__DISABLE_BURST__SHIFT                                                               0x11
+#define GL1XC_UTCL0_CNTL2__FORCE_FRAG_2M_TO_64K__SHIFT                                                        0x1a
+#define GL1XC_UTCL0_CNTL2__FGCG_DISABLE__SHIFT                                                                0x1e
+#define GL1XC_UTCL0_CNTL2__DISABLE_ILLEGAL_PCIE_ATOMIC__SHIFT                                                 0x1f
+#define GL1XC_UTCL0_CNTL2__SPARE_MASK                                                                         0x000000FFL
+#define GL1XC_UTCL0_CNTL2__MTYPE_OVRD_DIS_MASK                                                                0x00000200L
+#define GL1XC_UTCL0_CNTL2__ANY_LINE_VALID_MASK                                                                0x00000400L
+#define GL1XC_UTCL0_CNTL2__FORCE_SNOOP_MASK                                                                   0x00004000L
+#define GL1XC_UTCL0_CNTL2__DISABLE_BURST_MASK                                                                 0x00020000L
+#define GL1XC_UTCL0_CNTL2__FORCE_FRAG_2M_TO_64K_MASK                                                          0x04000000L
+#define GL1XC_UTCL0_CNTL2__FGCG_DISABLE_MASK                                                                  0x40000000L
+#define GL1XC_UTCL0_CNTL2__DISABLE_ILLEGAL_PCIE_ATOMIC_MASK                                                   0x80000000L
+//GL1XC_UTCL0_STATUS
+#define GL1XC_UTCL0_STATUS__FAULT_DETECTED__SHIFT                                                             0x0
+#define GL1XC_UTCL0_STATUS__RETRY_DETECTED__SHIFT                                                             0x1
+#define GL1XC_UTCL0_STATUS__PRT_DETECTED__SHIFT                                                               0x2
+#define GL1XC_UTCL0_STATUS__FAULT_DETECTED_MASK                                                               0x00000001L
+#define GL1XC_UTCL0_STATUS__RETRY_DETECTED_MASK                                                               0x00000002L
+#define GL1XC_UTCL0_STATUS__PRT_DETECTED_MASK                                                                 0x00000004L
+//GL1XC_UTCL0_RETRY
+#define GL1XC_UTCL0_RETRY__INCR__SHIFT                                                                        0x0
+#define GL1XC_UTCL0_RETRY__COUNT__SHIFT                                                                       0x8
+#define GL1XC_UTCL0_RETRY__INCR_MASK                                                                          0x000000FFL
+#define GL1XC_UTCL0_RETRY__COUNT_MASK                                                                         0x00000F00L
+//GL1XC_CTRL2
+#define GL1XC_CTRL2__UTCL0_INFLIGHT_MAX__SHIFT                                                                0x0
+#define GL1XC_CTRL2__OVERRIDE_READ_BYPASS_TO_DECOMPRESSED__SHIFT                                              0x8
+#define GL1XC_CTRL2__OVERRIDE_WRITE_BYPASS_TO_COMPRESSION_DISABLE__SHIFT                                      0x9
+#define GL1XC_CTRL2__UTCL0_INFLIGHT_MAX_MASK                                                                  0x000000FFL
+#define GL1XC_CTRL2__OVERRIDE_READ_BYPASS_TO_DECOMPRESSED_MASK                                                0x00000100L
+#define GL1XC_CTRL2__OVERRIDE_WRITE_BYPASS_TO_COMPRESSION_DISABLE_MASK                                        0x00000200L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pfonly_secacdec
+//SE_CAC_CTRL_1
+#define SE_CAC_CTRL_1__CAC_WINDOW__SHIFT                                                                      0x0
+#define SE_CAC_CTRL_1__TDP_WINDOW__SHIFT                                                                      0x8
+#define SE_CAC_CTRL_1__CAC_WINDOW_MASK                                                                        0x000000FFL
+#define SE_CAC_CTRL_1__TDP_WINDOW_MASK                                                                        0xFFFFFF00L
+//SE_CAC_CTRL_2
+#define SE_CAC_CTRL_2__CAC_ENABLE__SHIFT                                                                      0x0
+#define SE_CAC_CTRL_2__SE_LCAC_ENABLE__SHIFT                                                                  0x1
+#define SE_CAC_CTRL_2__WGP_CAC_CLK_OVERRIDE__SHIFT                                                            0x2
+#define SE_CAC_CTRL_2__SE_CAC_INDEX_AUTO_INCR_EN__SHIFT                                                       0x3
+#define SE_CAC_CTRL_2__SE_LCAC_OVR_EN__SHIFT                                                                  0x4
+#define SE_CAC_CTRL_2__WGP_LCAC_OVR_EN__SHIFT                                                                 0x5
+#define SE_CAC_CTRL_2__WGP_LCAC_MODE__SHIFT                                                                   0x6
+#define SE_CAC_CTRL_2__SE_CAC_SOFT_CTRL_ENABLE__SHIFT                                                         0x7
+#define SE_CAC_CTRL_2__CAC_ENABLE_MASK                                                                        0x00000001L
+#define SE_CAC_CTRL_2__SE_LCAC_ENABLE_MASK                                                                    0x00000002L
+#define SE_CAC_CTRL_2__WGP_CAC_CLK_OVERRIDE_MASK                                                              0x00000004L
+#define SE_CAC_CTRL_2__SE_CAC_INDEX_AUTO_INCR_EN_MASK                                                         0x00000008L
+#define SE_CAC_CTRL_2__SE_LCAC_OVR_EN_MASK                                                                    0x00000010L
+#define SE_CAC_CTRL_2__WGP_LCAC_OVR_EN_MASK                                                                   0x00000020L
+#define SE_CAC_CTRL_2__WGP_LCAC_MODE_MASK                                                                     0x00000040L
+#define SE_CAC_CTRL_2__SE_CAC_SOFT_CTRL_ENABLE_MASK                                                           0x00000080L
+//SE_CAC_SOFT_CTRL
+#define SE_CAC_SOFT_CTRL__SE_CAC_SOFT_SNAP__SHIFT                                                             0x0
+#define SE_CAC_SOFT_CTRL__SE_CAC_SOFT_SNAP_MASK                                                               0x00000001L
+//SE_CAC_OVR_VAL_LOWER
+#define SE_CAC_OVR_VAL_LOWER__SE_LCAC_OVR_VAL_LOWER__SHIFT                                                    0x0
+#define SE_CAC_OVR_VAL_LOWER__SE_LCAC_OVR_VAL_LOWER_MASK                                                      0xFFFFFFFFL
+//SE_CAC_OVR_VAL_UPPER
+#define SE_CAC_OVR_VAL_UPPER__SE_LCAC_OVR_VAL_UPPER__SHIFT                                                    0x0
+#define SE_CAC_OVR_VAL_UPPER__SE_LCAC_OVR_VAL_UPPER_MASK                                                      0xFFFFFFFFL
+//SE_CAC_WINDOW_AGGR_VALUE_LO
+#define SE_CAC_WINDOW_AGGR_VALUE_LO__SE_CAC_WINDOW_AGGR_VALUE_LO__SHIFT                                       0x0
+#define SE_CAC_WINDOW_AGGR_VALUE_LO__SE_CAC_WINDOW_AGGR_VALUE_LO_MASK                                         0xFFFFFFFFL
+//SE_CAC_WINDOW_AGGR_VALUE_HI
+#define SE_CAC_WINDOW_AGGR_VALUE_HI__SE_CAC_WINDOW_AGGR_VALUE_HI__SHIFT                                       0x0
+#define SE_CAC_WINDOW_AGGR_VALUE_HI__SE_CAC_WINDOW_AGGR_VALUE_HI_MASK                                         0x000000FFL
+//SE_CAC_WINDOW_GFXCLK_CYCLE
+#define SE_CAC_WINDOW_GFXCLK_CYCLE__SE_CAC_WINDOW_GFXCLK_CYCLE__SHIFT                                         0x0
+#define SE_CAC_WINDOW_GFXCLK_CYCLE__SE_CAC_WINDOW_GFXCLK_CYCLE_MASK                                           0x0000FFFFL
+//DIDT_EDC_CTRL
+#define DIDT_EDC_CTRL__EDC_EN__SHIFT                                                                          0x0
+#define DIDT_EDC_CTRL__EDC_SW_RST__SHIFT                                                                      0x1
+#define DIDT_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT                                                             0x2
+#define DIDT_EDC_CTRL__EDC_FORCE_STALL__SHIFT                                                                 0x3
+#define DIDT_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT                                                     0x4
+#define DIDT_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS__SHIFT                                                      0xa
+#define DIDT_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT                                                        0xe
+#define DIDT_EDC_CTRL__EDC_ALGORITHM_MODE__SHIFT                                                              0xf
+#define DIDT_EDC_CTRL__EDC_AVGDIV__SHIFT                                                                      0x10
+#define DIDT_EDC_CTRL__EDC_THRESHOLD_SEL__SHIFT                                                               0x14
+#define DIDT_EDC_CTRL__EDC_PERF_COUNTER_EN__SHIFT                                                             0x15
+#define DIDT_EDC_CTRL__EDC_EN_MASK                                                                            0x00000001L
+#define DIDT_EDC_CTRL__EDC_SW_RST_MASK                                                                        0x00000002L
+#define DIDT_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK                                                               0x00000004L
+#define DIDT_EDC_CTRL__EDC_FORCE_STALL_MASK                                                                   0x00000008L
+#define DIDT_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK                                                       0x000003F0L
+#define DIDT_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS_MASK                                                        0x00003C00L
+#define DIDT_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK                                                          0x00004000L
+#define DIDT_EDC_CTRL__EDC_ALGORITHM_MODE_MASK                                                                0x00008000L
+#define DIDT_EDC_CTRL__EDC_AVGDIV_MASK                                                                        0x000F0000L
+#define DIDT_EDC_CTRL__EDC_THRESHOLD_SEL_MASK                                                                 0x00100000L
+#define DIDT_EDC_CTRL__EDC_PERF_COUNTER_EN_MASK                                                               0x00200000L
+//DIDT_EDC_THROTTLE_CTRL
+#define DIDT_EDC_THROTTLE_CTRL__SQ_STALL_EN__SHIFT                                                            0x0
+#define DIDT_EDC_THROTTLE_CTRL__DB_STALL_EN__SHIFT                                                            0x1
+#define DIDT_EDC_THROTTLE_CTRL__TCP_STALL_EN__SHIFT                                                           0x2
+#define DIDT_EDC_THROTTLE_CTRL__TD_STALL_EN__SHIFT                                                            0x3
+#define DIDT_EDC_THROTTLE_CTRL__PATTERN_EXTEND_EN__SHIFT                                                      0x4
+#define DIDT_EDC_THROTTLE_CTRL__PATTERN_EXTEND_MODE__SHIFT                                                    0x5
+#define DIDT_EDC_THROTTLE_CTRL__EDC_STRETCH_EN__SHIFT                                                         0x8
+#define DIDT_EDC_THROTTLE_CTRL__EDC_STRETCH_SRC_SEL__SHIFT                                                    0x9
+#define DIDT_EDC_THROTTLE_CTRL__EDC_MAX_HYSTERESIS__SHIFT                                                     0xa
+#define DIDT_EDC_THROTTLE_CTRL__EDC_STALL_CLAMP_EN__SHIFT                                                     0x12
+#define DIDT_EDC_THROTTLE_CTRL__SQ_STALL_EN_MASK                                                              0x00000001L
+#define DIDT_EDC_THROTTLE_CTRL__DB_STALL_EN_MASK                                                              0x00000002L
+#define DIDT_EDC_THROTTLE_CTRL__TCP_STALL_EN_MASK                                                             0x00000004L
+#define DIDT_EDC_THROTTLE_CTRL__TD_STALL_EN_MASK                                                              0x00000008L
+#define DIDT_EDC_THROTTLE_CTRL__PATTERN_EXTEND_EN_MASK                                                        0x00000010L
+#define DIDT_EDC_THROTTLE_CTRL__PATTERN_EXTEND_MODE_MASK                                                      0x000000E0L
+#define DIDT_EDC_THROTTLE_CTRL__EDC_STRETCH_EN_MASK                                                           0x00000100L
+#define DIDT_EDC_THROTTLE_CTRL__EDC_STRETCH_SRC_SEL_MASK                                                      0x00000200L
+#define DIDT_EDC_THROTTLE_CTRL__EDC_MAX_HYSTERESIS_MASK                                                       0x0003FC00L
+#define DIDT_EDC_THROTTLE_CTRL__EDC_STALL_CLAMP_EN_MASK                                                       0x00040000L
+//DIDT_EDC_THRESHOLD
+#define DIDT_EDC_THRESHOLD__EDC_THRESHOLD__SHIFT                                                              0x0
+#define DIDT_EDC_THRESHOLD__EDC_THRESHOLD_MASK                                                                0xFFFFFFFFL
+//DIDT_EDC_STRETCH_THRESHOLD
+#define DIDT_EDC_STRETCH_THRESHOLD__EDC_STRETCH_THRESHOLD__SHIFT                                              0x0
+#define DIDT_EDC_STRETCH_THRESHOLD__EDC_STRETCH_THRESHOLD_MASK                                                0xFFFFFFFFL
+//DIDT_EDC_CLK_COUNTER_THRESHOLD
+#define DIDT_EDC_CLK_COUNTER_THRESHOLD__EDC_CLK_COUNTER_THRESHOLD__SHIFT                                      0x0
+#define DIDT_EDC_CLK_COUNTER_THRESHOLD__EDC_CLK_COUNTER_THRESHOLD_MASK                                        0x0000FFFFL
+//DIDT_EDC_STALL_PATTERN_1_2
+#define DIDT_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1__SHIFT                                                0x0
+#define DIDT_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2__SHIFT                                                0x10
+#define DIDT_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1_MASK                                                  0x00007FFFL
+#define DIDT_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2_MASK                                                  0x7FFF0000L
+//DIDT_EDC_STALL_PATTERN_3_4
+#define DIDT_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3__SHIFT                                                0x0
+#define DIDT_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4__SHIFT                                                0x10
+#define DIDT_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3_MASK                                                  0x00007FFFL
+#define DIDT_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4_MASK                                                  0x7FFF0000L
+//DIDT_EDC_STALL_PATTERN_5_6
+#define DIDT_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5__SHIFT                                                0x0
+#define DIDT_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6__SHIFT                                                0x10
+#define DIDT_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5_MASK                                                  0x00007FFFL
+#define DIDT_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6_MASK                                                  0x7FFF0000L
+//DIDT_EDC_STALL_PATTERN_7
+#define DIDT_EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7__SHIFT                                                  0x0
+#define DIDT_EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7_MASK                                                    0x00007FFFL
+//DIDT_EDC_STATUS
+#define DIDT_EDC_STATUS__EDC_FSM_STATE__SHIFT                                                                 0x0
+#define DIDT_EDC_STATUS__EDC_THROTTLE_LEVEL__SHIFT                                                            0x1
+#define DIDT_EDC_STATUS__EDC_HYSTERESIS_CNT__SHIFT                                                            0x4
+#define DIDT_EDC_STATUS__EDC_THRESHOLD_STAT__SHIFT                                                            0xc
+#define DIDT_EDC_STATUS__EDC_FSM_STATE_MASK                                                                   0x00000001L
+#define DIDT_EDC_STATUS__EDC_THROTTLE_LEVEL_MASK                                                              0x0000000EL
+#define DIDT_EDC_STATUS__EDC_HYSTERESIS_CNT_MASK                                                              0x00000FF0L
+#define DIDT_EDC_STATUS__EDC_THRESHOLD_STAT_MASK                                                              0x00001000L
+//DIDT_EDC_OVERFLOW
+#define DIDT_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW__SHIFT                                            0x0
+#define DIDT_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER__SHIFT                                         0x1
+#define DIDT_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW_MASK                                              0x00000001L
+#define DIDT_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER_MASK                                           0x0001FFFEL
+//DIDT_EDC_ROLLING_POWER_DELTA
+#define DIDT_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA__SHIFT                                          0x0
+#define DIDT_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA_MASK                                            0xFFFFFFFFL
+//DIDT_EDC_AVG_POWER_DELTA_HI
+#define DIDT_EDC_AVG_POWER_DELTA_HI__EDC_AVG_POWER_DELTA_HI__SHIFT                                            0x0
+#define DIDT_EDC_AVG_POWER_DELTA_HI__EDC_AVG_POWER_DELTA_HI_MASK                                              0xFFFFFFFFL
+//DIDT_EDC_AVG_POWER_DELTA_LO
+#define DIDT_EDC_AVG_POWER_DELTA_LO__EDC_AVG_POWER_DELTA_LO__SHIFT                                            0x0
+#define DIDT_EDC_AVG_POWER_DELTA_LO__EDC_AVG_POWER_DELTA_LO_MASK                                              0xFFFFFFFFL
+//DIDT_EDC_STALL_PERF_COUNTER
+#define DIDT_EDC_STALL_PERF_COUNTER__EDC_STALL_PERF_COUNTER__SHIFT                                            0x0
+#define DIDT_EDC_STALL_PERF_COUNTER__EDC_STALL_PERF_COUNTER_MASK                                              0xFFFFFFFFL
+//DIDT_THRESHOLD
+#define DIDT_THRESHOLD__DIDT_THRESHOLD__SHIFT                                                                 0x0
+#define DIDT_THRESHOLD__DIDT_THRESHOLD_MASK                                                                   0xFFFFFFFFL
+//SE_CAC_WEIGHT_LDS1_0
+#define SE_CAC_WEIGHT_LDS1_0__WEIGHT_LDS1_SIG0__SHIFT                                                         0x0
+#define SE_CAC_WEIGHT_LDS1_0__WEIGHT_LDS1_SIG1__SHIFT                                                         0x10
+#define SE_CAC_WEIGHT_LDS1_0__WEIGHT_LDS1_SIG0_MASK                                                           0x0000FFFFL
+#define SE_CAC_WEIGHT_LDS1_0__WEIGHT_LDS1_SIG1_MASK                                                           0xFFFF0000L
+//SE_CAC_WEIGHT_LDS1_1
+#define SE_CAC_WEIGHT_LDS1_1__WEIGHT_LDS1_SIG2__SHIFT                                                         0x0
+#define SE_CAC_WEIGHT_LDS1_1__WEIGHT_LDS1_SIG3__SHIFT                                                         0x10
+#define SE_CAC_WEIGHT_LDS1_1__WEIGHT_LDS1_SIG2_MASK                                                           0x0000FFFFL
+#define SE_CAC_WEIGHT_LDS1_1__WEIGHT_LDS1_SIG3_MASK                                                           0xFFFF0000L
+//SE_CAC_WEIGHT_LDS1_2
+#define SE_CAC_WEIGHT_LDS1_2__WEIGHT_LDS1_SIG4__SHIFT                                                         0x0
+#define SE_CAC_WEIGHT_LDS1_2__WEIGHT_LDS1_SIG5__SHIFT                                                         0x10
+#define SE_CAC_WEIGHT_LDS1_2__WEIGHT_LDS1_SIG4_MASK                                                           0x0000FFFFL
+#define SE_CAC_WEIGHT_LDS1_2__WEIGHT_LDS1_SIG5_MASK                                                           0xFFFF0000L
+//SE_CAC_WEIGHT_LDS1_3
+#define SE_CAC_WEIGHT_LDS1_3__WEIGHT_LDS1_SIG6__SHIFT                                                         0x0
+#define SE_CAC_WEIGHT_LDS1_3__WEIGHT_LDS1_SIG7__SHIFT                                                         0x10
+#define SE_CAC_WEIGHT_LDS1_3__WEIGHT_LDS1_SIG6_MASK                                                           0x0000FFFFL
+#define SE_CAC_WEIGHT_LDS1_3__WEIGHT_LDS1_SIG7_MASK                                                           0xFFFF0000L
+//SE_CAC_WEIGHT_TCP_0
+#define SE_CAC_WEIGHT_TCP_0__WEIGHT_TCP_SIG0__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_TCP_0__WEIGHT_TCP_SIG1__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_TCP_0__WEIGHT_TCP_SIG0_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_TCP_0__WEIGHT_TCP_SIG1_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_TCP_1
+#define SE_CAC_WEIGHT_TCP_1__WEIGHT_TCP_SIG2__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_TCP_1__WEIGHT_TCP_SIG3__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_TCP_1__WEIGHT_TCP_SIG2_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_TCP_1__WEIGHT_TCP_SIG3_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_TCP_2
+#define SE_CAC_WEIGHT_TCP_2__WEIGHT_TCP_SIG4__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_TCP_2__WEIGHT_TCP_SIG5__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_TCP_2__WEIGHT_TCP_SIG4_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_TCP_2__WEIGHT_TCP_SIG5_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_TCP_3
+#define SE_CAC_WEIGHT_TCP_3__WEIGHT_TCP_SIG6__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_TCP_3__WEIGHT_TCP_SIG7__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_TCP_3__WEIGHT_TCP_SIG6_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_TCP_3__WEIGHT_TCP_SIG7_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_SQ_0
+#define SE_CAC_WEIGHT_SQ_0__WEIGHT_SQ_SIG0__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_SQ_0__WEIGHT_SQ_SIG1__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_SQ_0__WEIGHT_SQ_SIG0_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_SQ_0__WEIGHT_SQ_SIG1_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_SQ_1
+#define SE_CAC_WEIGHT_SQ_1__WEIGHT_SQ_SIG2__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_SQ_1__WEIGHT_SQ_SIG3__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_SQ_1__WEIGHT_SQ_SIG2_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_SQ_1__WEIGHT_SQ_SIG3_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_SQ_2
+#define SE_CAC_WEIGHT_SQ_2__WEIGHT_SQ_SIG4__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_SQ_2__WEIGHT_SQ_SIG4_MASK                                                               0x0000FFFFL
+//SE_CAC_WEIGHT_SP_0
+#define SE_CAC_WEIGHT_SP_0__WEIGHT_SP_SIG0__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_SP_0__WEIGHT_SP_SIG1__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_SP_0__WEIGHT_SP_SIG0_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_SP_0__WEIGHT_SP_SIG1_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_SP_1
+#define SE_CAC_WEIGHT_SP_1__WEIGHT_SP_SIG2__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_SP_1__WEIGHT_SP_SIG3__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_SP_1__WEIGHT_SP_SIG2_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_SP_1__WEIGHT_SP_SIG3_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_SP_2
+#define SE_CAC_WEIGHT_SP_2__WEIGHT_SP_SIG4__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_SP_2__WEIGHT_SP_SIG5__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_SP_2__WEIGHT_SP_SIG4_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_SP_2__WEIGHT_SP_SIG5_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_LDS0_0
+#define SE_CAC_WEIGHT_LDS0_0__WEIGHT_LDS0_SIG0__SHIFT                                                         0x0
+#define SE_CAC_WEIGHT_LDS0_0__WEIGHT_LDS0_SIG1__SHIFT                                                         0x10
+#define SE_CAC_WEIGHT_LDS0_0__WEIGHT_LDS0_SIG0_MASK                                                           0x0000FFFFL
+#define SE_CAC_WEIGHT_LDS0_0__WEIGHT_LDS0_SIG1_MASK                                                           0xFFFF0000L
+//SE_CAC_WEIGHT_LDS0_1
+#define SE_CAC_WEIGHT_LDS0_1__WEIGHT_LDS0_SIG2__SHIFT                                                         0x0
+#define SE_CAC_WEIGHT_LDS0_1__WEIGHT_LDS0_SIG3__SHIFT                                                         0x10
+#define SE_CAC_WEIGHT_LDS0_1__WEIGHT_LDS0_SIG2_MASK                                                           0x0000FFFFL
+#define SE_CAC_WEIGHT_LDS0_1__WEIGHT_LDS0_SIG3_MASK                                                           0xFFFF0000L
+//SE_CAC_WEIGHT_LDS0_2
+#define SE_CAC_WEIGHT_LDS0_2__WEIGHT_LDS0_SIG4__SHIFT                                                         0x0
+#define SE_CAC_WEIGHT_LDS0_2__WEIGHT_LDS0_SIG5__SHIFT                                                         0x10
+#define SE_CAC_WEIGHT_LDS0_2__WEIGHT_LDS0_SIG4_MASK                                                           0x0000FFFFL
+#define SE_CAC_WEIGHT_LDS0_2__WEIGHT_LDS0_SIG5_MASK                                                           0xFFFF0000L
+//SE_CAC_WEIGHT_LDS0_3
+#define SE_CAC_WEIGHT_LDS0_3__WEIGHT_LDS0_SIG6__SHIFT                                                         0x0
+#define SE_CAC_WEIGHT_LDS0_3__WEIGHT_LDS0_SIG7__SHIFT                                                         0x10
+#define SE_CAC_WEIGHT_LDS0_3__WEIGHT_LDS0_SIG6_MASK                                                           0x0000FFFFL
+#define SE_CAC_WEIGHT_LDS0_3__WEIGHT_LDS0_SIG7_MASK                                                           0xFFFF0000L
+//SE_CAC_WEIGHT_SQC_0
+#define SE_CAC_WEIGHT_SQC_0__WEIGHT_SQC_SIG0__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_SQC_0__WEIGHT_SQC_SIG1__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_SQC_0__WEIGHT_SQC_SIG0_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_SQC_0__WEIGHT_SQC_SIG1_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_SQC_1
+#define SE_CAC_WEIGHT_SQC_1__WEIGHT_SQC_SIG2__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_SQC_1__WEIGHT_SQC_SIG2_MASK                                                             0x0000FFFFL
+//SE_CAC_WEIGHT_CU_0
+#define SE_CAC_WEIGHT_CU_0__WEIGHT_CU_SIG0__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_CU_0__WEIGHT_CU_SIG0_MASK                                                               0x0000FFFFL
+//SE_CAC_WEIGHT_UTCL1_0
+#define SE_CAC_WEIGHT_UTCL1_0__WEIGHT_UTCL1_SIG0__SHIFT                                                       0x0
+#define SE_CAC_WEIGHT_UTCL1_0__WEIGHT_UTCL1_SIG0_MASK                                                         0x0000FFFFL
+//SE_CAC_WEIGHT_GL1C_0
+#define SE_CAC_WEIGHT_GL1C_0__WEIGHT_GL1C_SIG0__SHIFT                                                         0x0
+#define SE_CAC_WEIGHT_GL1C_0__WEIGHT_GL1C_SIG1__SHIFT                                                         0x10
+#define SE_CAC_WEIGHT_GL1C_0__WEIGHT_GL1C_SIG0_MASK                                                           0x0000FFFFL
+#define SE_CAC_WEIGHT_GL1C_0__WEIGHT_GL1C_SIG1_MASK                                                           0xFFFF0000L
+//SE_CAC_WEIGHT_GL1C_1
+#define SE_CAC_WEIGHT_GL1C_1__WEIGHT_GL1C_SIG2__SHIFT                                                         0x0
+#define SE_CAC_WEIGHT_GL1C_1__WEIGHT_GL1C_SIG3__SHIFT                                                         0x10
+#define SE_CAC_WEIGHT_GL1C_1__WEIGHT_GL1C_SIG2_MASK                                                           0x0000FFFFL
+#define SE_CAC_WEIGHT_GL1C_1__WEIGHT_GL1C_SIG3_MASK                                                           0xFFFF0000L
+//SE_CAC_WEIGHT_SPI_0
+#define SE_CAC_WEIGHT_SPI_0__WEIGHT_SPI_SIG0__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_SPI_0__WEIGHT_SPI_SIG1__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_SPI_0__WEIGHT_SPI_SIG0_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_SPI_0__WEIGHT_SPI_SIG1_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_SPI_1
+#define SE_CAC_WEIGHT_SPI_1__WEIGHT_SPI_SIG2__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_SPI_1__WEIGHT_SPI_SIG3__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_SPI_1__WEIGHT_SPI_SIG2_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_SPI_1__WEIGHT_SPI_SIG3_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_SPI_2
+#define SE_CAC_WEIGHT_SPI_2__WEIGHT_SPI_SIG4__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_SPI_2__WEIGHT_SPI_SIG4_MASK                                                             0x0000FFFFL
+//SE_CAC_WEIGHT_TCP_4
+#define SE_CAC_WEIGHT_TCP_4__WEIGHT_TCP_SIG8__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_TCP_4__WEIGHT_TCP_SIG9__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_TCP_4__WEIGHT_TCP_SIG8_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_TCP_4__WEIGHT_TCP_SIG9_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_TCP_5
+#define SE_CAC_WEIGHT_TCP_5__WEIGHT_TCP_SIG10__SHIFT                                                          0x0
+#define SE_CAC_WEIGHT_TCP_5__WEIGHT_TCP_SIG11__SHIFT                                                          0x10
+#define SE_CAC_WEIGHT_TCP_5__WEIGHT_TCP_SIG10_MASK                                                            0x0000FFFFL
+#define SE_CAC_WEIGHT_TCP_5__WEIGHT_TCP_SIG11_MASK                                                            0xFFFF0000L
+//SE_CAC_WEIGHT_TCP_6
+#define SE_CAC_WEIGHT_TCP_6__WEIGHT_TCP_SIG12__SHIFT                                                          0x0
+#define SE_CAC_WEIGHT_TCP_6__WEIGHT_TCP_SIG13__SHIFT                                                          0x10
+#define SE_CAC_WEIGHT_TCP_6__WEIGHT_TCP_SIG12_MASK                                                            0x0000FFFFL
+#define SE_CAC_WEIGHT_TCP_6__WEIGHT_TCP_SIG13_MASK                                                            0xFFFF0000L
+//SE_CAC_WEIGHT_TCP_7
+#define SE_CAC_WEIGHT_TCP_7__WEIGHT_TCP_SIG14__SHIFT                                                          0x0
+#define SE_CAC_WEIGHT_TCP_7__WEIGHT_TCP_SIG15__SHIFT                                                          0x10
+#define SE_CAC_WEIGHT_TCP_7__WEIGHT_TCP_SIG14_MASK                                                            0x0000FFFFL
+#define SE_CAC_WEIGHT_TCP_7__WEIGHT_TCP_SIG15_MASK                                                            0xFFFF0000L
+//SE_CAC_WEIGHT_TCP_8
+#define SE_CAC_WEIGHT_TCP_8__WEIGHT_TCP_SIG16__SHIFT                                                          0x0
+#define SE_CAC_WEIGHT_TCP_8__WEIGHT_TCP_SIG17__SHIFT                                                          0x10
+#define SE_CAC_WEIGHT_TCP_8__WEIGHT_TCP_SIG16_MASK                                                            0x0000FFFFL
+#define SE_CAC_WEIGHT_TCP_8__WEIGHT_TCP_SIG17_MASK                                                            0xFFFF0000L
+//SE_CAC_WEIGHT_TCP_9
+#define SE_CAC_WEIGHT_TCP_9__WEIGHT_TCP_SIG18__SHIFT                                                          0x0
+#define SE_CAC_WEIGHT_TCP_9__WEIGHT_TCP_SIG18_MASK                                                            0x0000FFFFL
+//SE_CAC_IND_INDEX
+#define SE_CAC_IND_INDEX__SE_CAC_IND_ADDR__SHIFT                                                              0x0
+#define SE_CAC_IND_INDEX__SE_CAC_IND_ADDR_MASK                                                                0xFFFFFFFFL
+//SE_CAC_IND_DATA
+#define SE_CAC_IND_DATA__SE_CAC_IND_DATA__SHIFT                                                               0x0
+#define SE_CAC_IND_DATA__SE_CAC_IND_DATA_MASK                                                                 0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_perfddec
+//GE2_SE_PERFCOUNTER0_LO
+#define GE2_SE_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                           0xFFFFFFFFL
+//GE2_SE_PERFCOUNTER0_HI
+#define GE2_SE_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                           0xFFFFFFFFL
+//GE2_SE_PERFCOUNTER1_LO
+#define GE2_SE_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                           0xFFFFFFFFL
+//GE2_SE_PERFCOUNTER1_HI
+#define GE2_SE_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                           0xFFFFFFFFL
+//GE2_SE_PERFCOUNTER2_LO
+#define GE2_SE_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                           0xFFFFFFFFL
+//GE2_SE_PERFCOUNTER2_HI
+#define GE2_SE_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                           0xFFFFFFFFL
+//GE2_SE_PERFCOUNTER3_LO
+#define GE2_SE_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                           0xFFFFFFFFL
+//GE2_SE_PERFCOUNTER3_HI
+#define GE2_SE_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                           0xFFFFFFFFL
+//GRBMH_PERFCOUNTER0_LO
+#define GRBMH_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define GRBMH_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//GRBMH_PERFCOUNTER0_HI
+#define GRBMH_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define GRBMH_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//GRBMH_PERFCOUNTER1_LO
+#define GRBMH_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define GRBMH_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//GRBMH_PERFCOUNTER1_HI
+#define GRBMH_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define GRBMH_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER0_LO
+#define PA_SU_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER0_HI
+#define PA_SU_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER1_LO
+#define PA_SU_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER1_HI
+#define PA_SU_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER2_LO
+#define PA_SU_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER2_HI
+#define PA_SU_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER3_LO
+#define PA_SU_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER3_HI
+#define PA_SU_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER0_LO
+#define PA_SC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER0_HI
+#define PA_SC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER1_LO
+#define PA_SC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER1_HI
+#define PA_SC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER2_LO
+#define PA_SC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER2_HI
+#define PA_SC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER3_LO
+#define PA_SC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER3_HI
+#define PA_SC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER4_LO
+#define PA_SC_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER4_HI
+#define PA_SC_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER5_LO
+#define PA_SC_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER5_HI
+#define PA_SC_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER6_LO
+#define PA_SC_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER6_HI
+#define PA_SC_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER7_LO
+#define PA_SC_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER7_HI
+#define PA_SC_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//SPI_PERFCOUNTER0_HI
+#define SPI_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER0_LO
+#define SPI_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER1_HI
+#define SPI_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER1_LO
+#define SPI_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER2_HI
+#define SPI_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER2_LO
+#define SPI_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER3_HI
+#define SPI_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER3_LO
+#define SPI_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER4_HI
+#define SPI_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER4_LO
+#define SPI_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER5_HI
+#define SPI_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER5_LO
+#define SPI_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//PC_PERFCOUNTER0_HI
+#define PC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define PC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//PC_PERFCOUNTER0_LO
+#define PC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define PC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//PC_PERFCOUNTER1_HI
+#define PC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define PC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//PC_PERFCOUNTER1_LO
+#define PC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define PC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//PC_PERFCOUNTER2_HI
+#define PC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define PC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//PC_PERFCOUNTER2_LO
+#define PC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define PC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//PC_PERFCOUNTER3_HI
+#define PC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define PC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//PC_PERFCOUNTER3_LO
+#define PC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define PC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER0_LO
+#define SQ_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER1_LO
+#define SQ_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER2_LO
+#define SQ_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER3_LO
+#define SQ_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER4_LO
+#define SQ_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER5_LO
+#define SQ_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER6_LO
+#define SQ_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER7_LO
+#define SQ_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQG_PERFCOUNTER0_LO
+#define SQG_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER0_HI
+#define SQG_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER1_LO
+#define SQG_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER1_HI
+#define SQG_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER2_LO
+#define SQG_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER2_HI
+#define SQG_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER3_LO
+#define SQG_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER3_HI
+#define SQG_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER4_LO
+#define SQG_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER4_HI
+#define SQG_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER5_LO
+#define SQG_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER5_HI
+#define SQG_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER6_LO
+#define SQG_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER6_HI
+#define SQG_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER7_LO
+#define SQG_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER7_HI
+#define SQG_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SX_PERFCOUNTER0_LO
+#define SX_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SX_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER0_HI
+#define SX_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SX_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER1_LO
+#define SX_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SX_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER1_HI
+#define SX_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SX_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER2_LO
+#define SX_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SX_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER2_HI
+#define SX_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SX_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER3_LO
+#define SX_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SX_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER3_HI
+#define SX_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SX_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//TA_PERFCOUNTER0_LO
+#define TA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define TA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//TA_PERFCOUNTER0_HI
+#define TA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define TA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//TA_PERFCOUNTER1_LO
+#define TA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define TA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//TA_PERFCOUNTER1_HI
+#define TA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define TA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//TD_PERFCOUNTER0_LO
+#define TD_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define TD_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//TD_PERFCOUNTER0_HI
+#define TD_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define TD_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//TD_PERFCOUNTER1_LO
+#define TD_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define TD_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//TD_PERFCOUNTER1_HI
+#define TD_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define TD_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//TCP_PERFCOUNTER0_LO
+#define TCP_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER0_HI
+#define TCP_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER1_LO
+#define TCP_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER1_HI
+#define TCP_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER2_LO
+#define TCP_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER2_HI
+#define TCP_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER3_LO
+#define TCP_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER3_HI
+#define TCP_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER4_LO
+#define TCP_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER4_HI
+#define TCP_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER5_LO
+#define TCP_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER5_HI
+#define TCP_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER6_LO
+#define TCP_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER6_HI
+#define TCP_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER7_LO
+#define TCP_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER7_HI
+#define TCP_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER_FILTER
+#define TCP_PERFCOUNTER_FILTER__BUFFER__SHIFT                                                                 0x0
+#define TCP_PERFCOUNTER_FILTER__FLAT__SHIFT                                                                   0x1
+#define TCP_PERFCOUNTER_FILTER__DIM__SHIFT                                                                    0x2
+#define TCP_PERFCOUNTER_FILTER__DATA_FORMAT__SHIFT                                                            0x5
+#define TCP_PERFCOUNTER_FILTER__COMPRESSION_ENABLE__SHIFT                                                     0xc
+#define TCP_PERFCOUNTER_FILTER__NUM_FORMAT__SHIFT                                                             0xd
+#define TCP_PERFCOUNTER_FILTER__SW_MODE__SHIFT                                                                0x11
+#define TCP_PERFCOUNTER_FILTER__NUM_SAMPLES__SHIFT                                                            0x16
+#define TCP_PERFCOUNTER_FILTER__OPCODE_TYPE__SHIFT                                                            0x18
+#define TCP_PERFCOUNTER_FILTER__TMPRL__SHIFT                                                                  0x1b
+#define TCP_PERFCOUNTER_FILTER__SCOPE__SHIFT                                                                  0x1e
+#define TCP_PERFCOUNTER_FILTER__BUFFER_MASK                                                                   0x00000001L
+#define TCP_PERFCOUNTER_FILTER__FLAT_MASK                                                                     0x00000002L
+#define TCP_PERFCOUNTER_FILTER__DIM_MASK                                                                      0x0000001CL
+#define TCP_PERFCOUNTER_FILTER__DATA_FORMAT_MASK                                                              0x00000FE0L
+#define TCP_PERFCOUNTER_FILTER__COMPRESSION_ENABLE_MASK                                                       0x00001000L
+#define TCP_PERFCOUNTER_FILTER__NUM_FORMAT_MASK                                                               0x0001E000L
+#define TCP_PERFCOUNTER_FILTER__SW_MODE_MASK                                                                  0x003E0000L
+#define TCP_PERFCOUNTER_FILTER__NUM_SAMPLES_MASK                                                              0x00C00000L
+#define TCP_PERFCOUNTER_FILTER__OPCODE_TYPE_MASK                                                              0x07000000L
+#define TCP_PERFCOUNTER_FILTER__TMPRL_MASK                                                                    0x38000000L
+#define TCP_PERFCOUNTER_FILTER__SCOPE_MASK                                                                    0xC0000000L
+//TCP_PERFCOUNTER_FILTER2
+#define TCP_PERFCOUNTER_FILTER2__REQ_MODE__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER_FILTER2__REQ_MODE_MASK                                                                0x0000000FL
+//TCP_PERFCOUNTER_FILTER_EN
+#define TCP_PERFCOUNTER_FILTER_EN__BUFFER__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER_FILTER_EN__FLAT__SHIFT                                                                0x1
+#define TCP_PERFCOUNTER_FILTER_EN__DIM__SHIFT                                                                 0x2
+#define TCP_PERFCOUNTER_FILTER_EN__DATA_FORMAT__SHIFT                                                         0x3
+#define TCP_PERFCOUNTER_FILTER_EN__NUM_FORMAT__SHIFT                                                          0x4
+#define TCP_PERFCOUNTER_FILTER_EN__SW_MODE__SHIFT                                                             0x5
+#define TCP_PERFCOUNTER_FILTER_EN__NUM_SAMPLES__SHIFT                                                         0x6
+#define TCP_PERFCOUNTER_FILTER_EN__OPCODE_TYPE__SHIFT                                                         0x7
+#define TCP_PERFCOUNTER_FILTER_EN__TMPRL__SHIFT                                                               0x8
+#define TCP_PERFCOUNTER_FILTER_EN__COMPRESSION_ENABLE__SHIFT                                                  0xb
+#define TCP_PERFCOUNTER_FILTER_EN__REQ_MODE__SHIFT                                                            0xc
+#define TCP_PERFCOUNTER_FILTER_EN__SCOPE__SHIFT                                                               0xd
+#define TCP_PERFCOUNTER_FILTER_EN__BUFFER_MASK                                                                0x00000001L
+#define TCP_PERFCOUNTER_FILTER_EN__FLAT_MASK                                                                  0x00000002L
+#define TCP_PERFCOUNTER_FILTER_EN__DIM_MASK                                                                   0x00000004L
+#define TCP_PERFCOUNTER_FILTER_EN__DATA_FORMAT_MASK                                                           0x00000008L
+#define TCP_PERFCOUNTER_FILTER_EN__NUM_FORMAT_MASK                                                            0x00000010L
+#define TCP_PERFCOUNTER_FILTER_EN__SW_MODE_MASK                                                               0x00000020L
+#define TCP_PERFCOUNTER_FILTER_EN__NUM_SAMPLES_MASK                                                           0x00000040L
+#define TCP_PERFCOUNTER_FILTER_EN__OPCODE_TYPE_MASK                                                           0x00000080L
+#define TCP_PERFCOUNTER_FILTER_EN__TMPRL_MASK                                                                 0x00000100L
+#define TCP_PERFCOUNTER_FILTER_EN__COMPRESSION_ENABLE_MASK                                                    0x00000800L
+#define TCP_PERFCOUNTER_FILTER_EN__REQ_MODE_MASK                                                              0x00001000L
+#define TCP_PERFCOUNTER_FILTER_EN__SCOPE_MASK                                                                 0x00002000L
+//TCP_PERFCOUNTER_SET_DEFINE
+#define TCP_PERFCOUNTER_SET_DEFINE__SET_A__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER_SET_DEFINE__SET_B__SHIFT                                                              0x8
+#define TCP_PERFCOUNTER_SET_DEFINE__SET_C__SHIFT                                                              0x10
+#define TCP_PERFCOUNTER_SET_DEFINE__SET_D__SHIFT                                                              0x18
+#define TCP_PERFCOUNTER_SET_DEFINE__SET_A_MASK                                                                0x000000FFL
+#define TCP_PERFCOUNTER_SET_DEFINE__SET_B_MASK                                                                0x0000FF00L
+#define TCP_PERFCOUNTER_SET_DEFINE__SET_C_MASK                                                                0x00FF0000L
+#define TCP_PERFCOUNTER_SET_DEFINE__SET_D_MASK                                                                0xFF000000L
+//GL1C_PERFCOUNTER0_LO
+#define GL1C_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1C_PERFCOUNTER0_HI
+#define GL1C_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1C_PERFCOUNTER1_LO
+#define GL1C_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1C_PERFCOUNTER1_HI
+#define GL1C_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1C_PERFCOUNTER2_LO
+#define GL1C_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1C_PERFCOUNTER2_HI
+#define GL1C_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1C_PERFCOUNTER3_LO
+#define GL1C_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1C_PERFCOUNTER3_HI
+#define GL1C_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1XC_PERFCOUNTER0_LO
+#define GL1XC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define GL1XC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//GL1XC_PERFCOUNTER0_HI
+#define GL1XC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define GL1XC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//GL1XC_PERFCOUNTER1_LO
+#define GL1XC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define GL1XC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//GL1XC_PERFCOUNTER1_HI
+#define GL1XC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define GL1XC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//GL1XC_PERFCOUNTER2_LO
+#define GL1XC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define GL1XC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//GL1XC_PERFCOUNTER2_HI
+#define GL1XC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define GL1XC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//GL1XC_PERFCOUNTER3_LO
+#define GL1XC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define GL1XC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//GL1XC_PERFCOUNTER3_HI
+#define GL1XC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define GL1XC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//CB_PERFCOUNTER0_LO
+#define CB_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define CB_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER0_HI
+#define CB_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define CB_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER1_LO
+#define CB_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define CB_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER1_HI
+#define CB_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define CB_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER2_LO
+#define CB_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define CB_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER2_HI
+#define CB_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define CB_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER3_LO
+#define CB_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define CB_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER3_HI
+#define CB_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define CB_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER0_LO
+#define DB_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define DB_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER0_HI
+#define DB_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define DB_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER1_LO
+#define DB_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define DB_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER1_HI
+#define DB_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define DB_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER2_LO
+#define DB_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define DB_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER2_HI
+#define DB_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define DB_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER3_LO
+#define DB_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define DB_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER3_HI
+#define DB_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define DB_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//RMI_PERFCOUNTER0_LO
+#define RMI_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER0_HI
+#define RMI_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER1_LO
+#define RMI_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER1_HI
+#define RMI_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER2_LO
+#define RMI_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER2_HI
+#define RMI_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER3_LO
+#define RMI_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER3_HI
+#define RMI_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//PA_PH_PERFCOUNTER0_LO
+#define PA_PH_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER0_HI
+#define PA_PH_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER1_LO
+#define PA_PH_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER1_HI
+#define PA_PH_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER2_LO
+#define PA_PH_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER2_HI
+#define PA_PH_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER3_LO
+#define PA_PH_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER3_HI
+#define PA_PH_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER4_LO
+#define PA_PH_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER4_HI
+#define PA_PH_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER5_LO
+#define PA_PH_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER5_HI
+#define PA_PH_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER6_LO
+#define PA_PH_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER6_HI
+#define PA_PH_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER7_LO
+#define PA_PH_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER7_HI
+#define PA_PH_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//UTCL1_PERFCOUNTER0_LO
+#define UTCL1_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//UTCL1_PERFCOUNTER0_HI
+#define UTCL1_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//UTCL1_PERFCOUNTER1_LO
+#define UTCL1_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//UTCL1_PERFCOUNTER1_HI
+#define UTCL1_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//UTCL1_PERFCOUNTER2_LO
+#define UTCL1_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//UTCL1_PERFCOUNTER2_HI
+#define UTCL1_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//UTCL1_PERFCOUNTER3_LO
+#define UTCL1_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//UTCL1_PERFCOUNTER3_HI
+#define UTCL1_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//WGS_PERFCOUNTER1_LO
+#define WGS_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define WGS_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//WGS_PERFCOUNTER1_HI
+#define WGS_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define WGS_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//WGS_PERFCOUNTER0_LO
+#define WGS_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define WGS_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//WGS_PERFCOUNTER0_HI
+#define WGS_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define WGS_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//WGS_PERFMON_CNTL
+#define WGS_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                                0x0
+#define WGS_PERFMON_CNTL__SPM_PERFMON_STATE__SHIFT                                                            0x4
+#define WGS_PERFMON_CNTL__PERFMON_ENABLE_MODE__SHIFT                                                          0x8
+#define WGS_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT                                                        0xa
+#define WGS_PERFMON_CNTL__PERFMON_STATE_MASK                                                                  0x0000000FL
+#define WGS_PERFMON_CNTL__SPM_PERFMON_STATE_MASK                                                              0x000000F0L
+#define WGS_PERFMON_CNTL__PERFMON_ENABLE_MODE_MASK                                                            0x00000300L
+#define WGS_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK                                                          0x00000400L
+//GL1A_PERFCOUNTER0_LO
+#define GL1A_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1A_PERFCOUNTER0_HI
+#define GL1A_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1A_PERFCOUNTER1_LO
+#define GL1A_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1A_PERFCOUNTER1_HI
+#define GL1A_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1A_PERFCOUNTER2_LO
+#define GL1A_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1A_PERFCOUNTER2_HI
+#define GL1A_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1A_PERFCOUNTER3_LO
+#define GL1A_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1A_PERFCOUNTER3_HI
+#define GL1A_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1XA_PERFCOUNTER0_LO
+#define GL1XA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define GL1XA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//GL1XA_PERFCOUNTER0_HI
+#define GL1XA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define GL1XA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//GL1XA_PERFCOUNTER1_LO
+#define GL1XA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define GL1XA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//GL1XA_PERFCOUNTER1_HI
+#define GL1XA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define GL1XA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//GL1XA_PERFCOUNTER2_LO
+#define GL1XA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define GL1XA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//GL1XA_PERFCOUNTER2_HI
+#define GL1XA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define GL1XA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//GL1XA_PERFCOUNTER3_LO
+#define GL1XA_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define GL1XA_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//GL1XA_PERFCOUNTER3_HI
+#define GL1XA_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define GL1XA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_perfsdec
+//GE2_SE_PERFCOUNTER0_SELECT
+#define GE2_SE_PERFCOUNTER0_SELECT__PERF_SEL0__SHIFT                                                          0x0
+#define GE2_SE_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                          0xa
+#define GE2_SE_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                          0x14
+#define GE2_SE_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                         0x18
+#define GE2_SE_PERFCOUNTER0_SELECT__PERF_MODE0__SHIFT                                                         0x1c
+#define GE2_SE_PERFCOUNTER0_SELECT__PERF_SEL0_MASK                                                            0x000003FFL
+#define GE2_SE_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                            0x000FFC00L
+#define GE2_SE_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                            0x00F00000L
+#define GE2_SE_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                           0x0F000000L
+#define GE2_SE_PERFCOUNTER0_SELECT__PERF_MODE0_MASK                                                           0xF0000000L
+//GE2_SE_PERFCOUNTER0_SELECT1
+#define GE2_SE_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                         0xa
+#define GE2_SE_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                        0x18
+#define GE2_SE_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                        0x1c
+#define GE2_SE_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                           0x000003FFL
+#define GE2_SE_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                           0x000FFC00L
+#define GE2_SE_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                          0x0F000000L
+#define GE2_SE_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                          0xF0000000L
+//GE2_SE_PERFCOUNTER1_SELECT
+#define GE2_SE_PERFCOUNTER1_SELECT__PERF_SEL0__SHIFT                                                          0x0
+#define GE2_SE_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                          0xa
+#define GE2_SE_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                          0x14
+#define GE2_SE_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                         0x18
+#define GE2_SE_PERFCOUNTER1_SELECT__PERF_MODE0__SHIFT                                                         0x1c
+#define GE2_SE_PERFCOUNTER1_SELECT__PERF_SEL0_MASK                                                            0x000003FFL
+#define GE2_SE_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                            0x000FFC00L
+#define GE2_SE_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                            0x00F00000L
+#define GE2_SE_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                           0x0F000000L
+#define GE2_SE_PERFCOUNTER1_SELECT__PERF_MODE0_MASK                                                           0xF0000000L
+//GE2_SE_PERFCOUNTER1_SELECT1
+#define GE2_SE_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                         0xa
+#define GE2_SE_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                        0x18
+#define GE2_SE_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                        0x1c
+#define GE2_SE_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                           0x000003FFL
+#define GE2_SE_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                           0x000FFC00L
+#define GE2_SE_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                          0x0F000000L
+#define GE2_SE_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                          0xF0000000L
+//GE2_SE_PERFCOUNTER2_SELECT
+#define GE2_SE_PERFCOUNTER2_SELECT__PERF_SEL0__SHIFT                                                          0x0
+#define GE2_SE_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                          0xa
+#define GE2_SE_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                          0x14
+#define GE2_SE_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                         0x18
+#define GE2_SE_PERFCOUNTER2_SELECT__PERF_MODE0__SHIFT                                                         0x1c
+#define GE2_SE_PERFCOUNTER2_SELECT__PERF_SEL0_MASK                                                            0x000003FFL
+#define GE2_SE_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                            0x000FFC00L
+#define GE2_SE_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                            0x00F00000L
+#define GE2_SE_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                           0x0F000000L
+#define GE2_SE_PERFCOUNTER2_SELECT__PERF_MODE0_MASK                                                           0xF0000000L
+//GE2_SE_PERFCOUNTER2_SELECT1
+#define GE2_SE_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                         0xa
+#define GE2_SE_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                        0x18
+#define GE2_SE_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                        0x1c
+#define GE2_SE_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                           0x000003FFL
+#define GE2_SE_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                           0x000FFC00L
+#define GE2_SE_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                          0x0F000000L
+#define GE2_SE_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                          0xF0000000L
+//GE2_SE_PERFCOUNTER3_SELECT
+#define GE2_SE_PERFCOUNTER3_SELECT__PERF_SEL0__SHIFT                                                          0x0
+#define GE2_SE_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                          0xa
+#define GE2_SE_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                          0x14
+#define GE2_SE_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                         0x18
+#define GE2_SE_PERFCOUNTER3_SELECT__PERF_MODE0__SHIFT                                                         0x1c
+#define GE2_SE_PERFCOUNTER3_SELECT__PERF_SEL0_MASK                                                            0x000003FFL
+#define GE2_SE_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                            0x000FFC00L
+#define GE2_SE_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                            0x00F00000L
+#define GE2_SE_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                           0x0F000000L
+#define GE2_SE_PERFCOUNTER3_SELECT__PERF_MODE0_MASK                                                           0xF0000000L
+//GE2_SE_PERFCOUNTER3_SELECT1
+#define GE2_SE_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                         0xa
+#define GE2_SE_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                        0x18
+#define GE2_SE_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                        0x1c
+#define GE2_SE_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                           0x000003FFL
+#define GE2_SE_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                           0x000FFC00L
+#define GE2_SE_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                          0x0F000000L
+#define GE2_SE_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                          0xF0000000L
+//GRBMH_PERFCOUNTER0_SELECT
+#define GRBMH_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define GRBMH_PERFCOUNTER0_SELECT__GL1CC_BUSY_USER_DEFINED_MASK__SHIFT                                        0x6
+#define GRBMH_PERFCOUNTER0_SELECT__GL1XCC_BUSY_USER_DEFINED_MASK__SHIFT                                       0x7
+#define GRBMH_PERFCOUNTER0_SELECT__SQG_BUSY_USER_DEFINED_MASK__SHIFT                                          0x8
+#define GRBMH_PERFCOUNTER0_SELECT__SC_CLEAN_USER_DEFINED_MASK__SHIFT                                          0x9
+#define GRBMH_PERFCOUNTER0_SELECT__WGS_BUSY_USER_DEFINED_MASK__SHIFT                                          0xa
+#define GRBMH_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                          0xb
+#define GRBMH_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                          0xc
+#define GRBMH_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                           0xd
+#define GRBMH_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                           0xe
+#define GRBMH_PERFCOUNTER0_SELECT__GL2C_BUSY_USER_DEFINED_MASK__SHIFT                                         0xf
+#define GRBMH_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                          0x10
+#define GRBMH_PERFCOUNTER0_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                           0x11
+#define GRBMH_PERFCOUNTER0_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                           0x12
+#define GRBMH_PERFCOUNTER0_SELECT__GL2A_BUSY_USER_DEFINED_MASK__SHIFT                                         0x13
+#define GRBMH_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                           0x14
+#define GRBMH_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                           0x15
+#define GRBMH_PERFCOUNTER0_SELECT__EA_LINK_BUSY_USER_DEFINED_MASK__SHIFT                                      0x17
+#define GRBMH_PERFCOUNTER0_SELECT__GL1A_BUSY_USER_DEFINED_MASK__SHIFT                                         0x18
+#define GRBMH_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                          0x19
+#define GRBMH_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT                                          0x1a
+#define GRBMH_PERFCOUNTER0_SELECT__TCP_BUSY_USER_DEFINED_MASK__SHIFT                                          0x1b
+#define GRBMH_PERFCOUNTER0_SELECT__GE_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1c
+#define GRBMH_PERFCOUNTER0_SELECT__UTCL1_BUSY_USER_DEFINED_MASK__SHIFT                                        0x1d
+#define GRBMH_PERFCOUNTER0_SELECT__EA_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1e
+#define GRBMH_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x0000003FL
+#define GRBMH_PERFCOUNTER0_SELECT__GL1CC_BUSY_USER_DEFINED_MASK_MASK                                          0x00000040L
+#define GRBMH_PERFCOUNTER0_SELECT__GL1XCC_BUSY_USER_DEFINED_MASK_MASK                                         0x00000080L
+#define GRBMH_PERFCOUNTER0_SELECT__SQG_BUSY_USER_DEFINED_MASK_MASK                                            0x00000100L
+#define GRBMH_PERFCOUNTER0_SELECT__SC_CLEAN_USER_DEFINED_MASK_MASK                                            0x00000200L
+#define GRBMH_PERFCOUNTER0_SELECT__WGS_BUSY_USER_DEFINED_MASK_MASK                                            0x00000400L
+#define GRBMH_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                            0x00000800L
+#define GRBMH_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                            0x00001000L
+#define GRBMH_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                             0x00002000L
+#define GRBMH_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                             0x00004000L
+#define GRBMH_PERFCOUNTER0_SELECT__GL2C_BUSY_USER_DEFINED_MASK_MASK                                           0x00008000L
+#define GRBMH_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                            0x00010000L
+#define GRBMH_PERFCOUNTER0_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                             0x00020000L
+#define GRBMH_PERFCOUNTER0_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                             0x00040000L
+#define GRBMH_PERFCOUNTER0_SELECT__GL2A_BUSY_USER_DEFINED_MASK_MASK                                           0x00080000L
+#define GRBMH_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                             0x00100000L
+#define GRBMH_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                             0x00200000L
+#define GRBMH_PERFCOUNTER0_SELECT__EA_LINK_BUSY_USER_DEFINED_MASK_MASK                                        0x00800000L
+#define GRBMH_PERFCOUNTER0_SELECT__GL1A_BUSY_USER_DEFINED_MASK_MASK                                           0x01000000L
+#define GRBMH_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                            0x02000000L
+#define GRBMH_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK                                            0x04000000L
+#define GRBMH_PERFCOUNTER0_SELECT__TCP_BUSY_USER_DEFINED_MASK_MASK                                            0x08000000L
+#define GRBMH_PERFCOUNTER0_SELECT__GE_BUSY_USER_DEFINED_MASK_MASK                                             0x10000000L
+#define GRBMH_PERFCOUNTER0_SELECT__UTCL1_BUSY_USER_DEFINED_MASK_MASK                                          0x20000000L
+#define GRBMH_PERFCOUNTER0_SELECT__EA_BUSY_USER_DEFINED_MASK_MASK                                             0x40000000L
+//GRBMH_PERFCOUNTER1_SELECT
+#define GRBMH_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define GRBMH_PERFCOUNTER1_SELECT__GL1CC_BUSY_USER_DEFINED_MASK__SHIFT                                        0x6
+#define GRBMH_PERFCOUNTER1_SELECT__GL1XCC_BUSY_USER_DEFINED_MASK__SHIFT                                       0x7
+#define GRBMH_PERFCOUNTER1_SELECT__SQG_BUSY_USER_DEFINED_MASK__SHIFT                                          0x8
+#define GRBMH_PERFCOUNTER1_SELECT__SC_CLEAN_USER_DEFINED_MASK__SHIFT                                          0x9
+#define GRBMH_PERFCOUNTER1_SELECT__WGS_BUSY_USER_DEFINED_MASK__SHIFT                                          0xa
+#define GRBMH_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                          0xb
+#define GRBMH_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                          0xc
+#define GRBMH_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                           0xd
+#define GRBMH_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                           0xe
+#define GRBMH_PERFCOUNTER1_SELECT__GL2C_BUSY_USER_DEFINED_MASK__SHIFT                                         0xf
+#define GRBMH_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                          0x10
+#define GRBMH_PERFCOUNTER1_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                           0x11
+#define GRBMH_PERFCOUNTER1_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                           0x12
+#define GRBMH_PERFCOUNTER1_SELECT__GL2A_BUSY_USER_DEFINED_MASK__SHIFT                                         0x13
+#define GRBMH_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                           0x14
+#define GRBMH_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                           0x15
+#define GRBMH_PERFCOUNTER1_SELECT__EA_LINK_BUSY_USER_DEFINED_MASK__SHIFT                                      0x17
+#define GRBMH_PERFCOUNTER1_SELECT__GL1A_BUSY_USER_DEFINED_MASK__SHIFT                                         0x18
+#define GRBMH_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                          0x19
+#define GRBMH_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT                                          0x1a
+#define GRBMH_PERFCOUNTER1_SELECT__TCP_BUSY_USER_DEFINED_MASK__SHIFT                                          0x1b
+#define GRBMH_PERFCOUNTER1_SELECT__GE_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1c
+#define GRBMH_PERFCOUNTER1_SELECT__UTCL1_BUSY_USER_DEFINED_MASK__SHIFT                                        0x1d
+#define GRBMH_PERFCOUNTER1_SELECT__EA_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1e
+#define GRBMH_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x0000003FL
+#define GRBMH_PERFCOUNTER1_SELECT__GL1CC_BUSY_USER_DEFINED_MASK_MASK                                          0x00000040L
+#define GRBMH_PERFCOUNTER1_SELECT__GL1XCC_BUSY_USER_DEFINED_MASK_MASK                                         0x00000080L
+#define GRBMH_PERFCOUNTER1_SELECT__SQG_BUSY_USER_DEFINED_MASK_MASK                                            0x00000100L
+#define GRBMH_PERFCOUNTER1_SELECT__SC_CLEAN_USER_DEFINED_MASK_MASK                                            0x00000200L
+#define GRBMH_PERFCOUNTER1_SELECT__WGS_BUSY_USER_DEFINED_MASK_MASK                                            0x00000400L
+#define GRBMH_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                            0x00000800L
+#define GRBMH_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                            0x00001000L
+#define GRBMH_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                             0x00002000L
+#define GRBMH_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                             0x00004000L
+#define GRBMH_PERFCOUNTER1_SELECT__GL2C_BUSY_USER_DEFINED_MASK_MASK                                           0x00008000L
+#define GRBMH_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                            0x00010000L
+#define GRBMH_PERFCOUNTER1_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                             0x00020000L
+#define GRBMH_PERFCOUNTER1_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                             0x00040000L
+#define GRBMH_PERFCOUNTER1_SELECT__GL2A_BUSY_USER_DEFINED_MASK_MASK                                           0x00080000L
+#define GRBMH_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                             0x00100000L
+#define GRBMH_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                             0x00200000L
+#define GRBMH_PERFCOUNTER1_SELECT__EA_LINK_BUSY_USER_DEFINED_MASK_MASK                                        0x00800000L
+#define GRBMH_PERFCOUNTER1_SELECT__GL1A_BUSY_USER_DEFINED_MASK_MASK                                           0x01000000L
+#define GRBMH_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                            0x02000000L
+#define GRBMH_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK                                            0x04000000L
+#define GRBMH_PERFCOUNTER1_SELECT__TCP_BUSY_USER_DEFINED_MASK_MASK                                            0x08000000L
+#define GRBMH_PERFCOUNTER1_SELECT__GE_BUSY_USER_DEFINED_MASK_MASK                                             0x10000000L
+#define GRBMH_PERFCOUNTER1_SELECT__UTCL1_BUSY_USER_DEFINED_MASK_MASK                                          0x20000000L
+#define GRBMH_PERFCOUNTER1_SELECT__EA_BUSY_USER_DEFINED_MASK_MASK                                             0x40000000L
+//PA_SU_PERFCOUNTER0_SELECT
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_SU_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_SU_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_SU_PERFCOUNTER0_SELECT1
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_SU_PERFCOUNTER1_SELECT
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_SU_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_SU_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_SU_PERFCOUNTER1_SELECT1
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_SU_PERFCOUNTER2_SELECT
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_SU_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_SU_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_SU_PERFCOUNTER2_SELECT1
+#define PA_SU_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_SU_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_SU_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_SU_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_SU_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_SU_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_SU_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_SU_PERFCOUNTER3_SELECT
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_SU_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_SU_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_SU_PERFCOUNTER3_SELECT1
+#define PA_SU_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_SU_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_SU_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_SU_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_SU_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_SU_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_SU_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_SC_PERFCOUNTER0_SELECT
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_SC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_SC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_SC_PERFCOUNTER0_SELECT1
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_SC_PERFCOUNTER1_SELECT
+#define PA_SC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER2_SELECT
+#define PA_SC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER3_SELECT
+#define PA_SC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER4_SELECT
+#define PA_SC_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER5_SELECT
+#define PA_SC_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER6_SELECT
+#define PA_SC_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER6_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER7_SELECT
+#define PA_SC_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER7_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//SPI_PERFCOUNTER0_SELECT
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define SPI_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define SPI_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define SPI_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define SPI_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define SPI_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define SPI_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SPI_PERFCOUNTER1_SELECT
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define SPI_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define SPI_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define SPI_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define SPI_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define SPI_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define SPI_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SPI_PERFCOUNTER2_SELECT
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define SPI_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define SPI_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define SPI_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define SPI_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define SPI_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define SPI_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SPI_PERFCOUNTER3_SELECT
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define SPI_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define SPI_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define SPI_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define SPI_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define SPI_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define SPI_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SPI_PERFCOUNTER4_SELECT
+#define SPI_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER4_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define SPI_PERFCOUNTER4_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define SPI_PERFCOUNTER4_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define SPI_PERFCOUNTER4_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SPI_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SPI_PERFCOUNTER4_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define SPI_PERFCOUNTER4_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define SPI_PERFCOUNTER4_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define SPI_PERFCOUNTER4_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SPI_PERFCOUNTER5_SELECT
+#define SPI_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER5_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define SPI_PERFCOUNTER5_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define SPI_PERFCOUNTER5_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define SPI_PERFCOUNTER5_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SPI_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SPI_PERFCOUNTER5_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define SPI_PERFCOUNTER5_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define SPI_PERFCOUNTER5_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define SPI_PERFCOUNTER5_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SPI_PERFCOUNTER0_SELECT1
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define SPI_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define SPI_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define SPI_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define SPI_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//SPI_PERFCOUNTER1_SELECT1
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define SPI_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define SPI_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define SPI_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define SPI_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//SPI_PERFCOUNTER2_SELECT1
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define SPI_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define SPI_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define SPI_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define SPI_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//SPI_PERFCOUNTER3_SELECT1
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define SPI_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define SPI_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define SPI_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define SPI_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//SPI_PERFCOUNTER4_SELECT1
+#define SPI_PERFCOUNTER4_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER4_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define SPI_PERFCOUNTER4_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define SPI_PERFCOUNTER4_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define SPI_PERFCOUNTER4_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define SPI_PERFCOUNTER4_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define SPI_PERFCOUNTER4_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define SPI_PERFCOUNTER4_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//SPI_PERFCOUNTER5_SELECT1
+#define SPI_PERFCOUNTER5_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER5_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define SPI_PERFCOUNTER5_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define SPI_PERFCOUNTER5_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define SPI_PERFCOUNTER5_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define SPI_PERFCOUNTER5_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define SPI_PERFCOUNTER5_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define SPI_PERFCOUNTER5_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//SPI_PERFCOUNTER_BINS
+#define SPI_PERFCOUNTER_BINS__BIN0_MIN__SHIFT                                                                 0x0
+#define SPI_PERFCOUNTER_BINS__BIN0_MAX__SHIFT                                                                 0x4
+#define SPI_PERFCOUNTER_BINS__BIN1_MIN__SHIFT                                                                 0x8
+#define SPI_PERFCOUNTER_BINS__BIN1_MAX__SHIFT                                                                 0xc
+#define SPI_PERFCOUNTER_BINS__BIN2_MIN__SHIFT                                                                 0x10
+#define SPI_PERFCOUNTER_BINS__BIN2_MAX__SHIFT                                                                 0x14
+#define SPI_PERFCOUNTER_BINS__BIN3_MIN__SHIFT                                                                 0x18
+#define SPI_PERFCOUNTER_BINS__BIN3_MAX__SHIFT                                                                 0x1c
+#define SPI_PERFCOUNTER_BINS__BIN0_MIN_MASK                                                                   0x0000000FL
+#define SPI_PERFCOUNTER_BINS__BIN0_MAX_MASK                                                                   0x000000F0L
+#define SPI_PERFCOUNTER_BINS__BIN1_MIN_MASK                                                                   0x00000F00L
+#define SPI_PERFCOUNTER_BINS__BIN1_MAX_MASK                                                                   0x0000F000L
+#define SPI_PERFCOUNTER_BINS__BIN2_MIN_MASK                                                                   0x000F0000L
+#define SPI_PERFCOUNTER_BINS__BIN2_MAX_MASK                                                                   0x00F00000L
+#define SPI_PERFCOUNTER_BINS__BIN3_MIN_MASK                                                                   0x0F000000L
+#define SPI_PERFCOUNTER_BINS__BIN3_MAX_MASK                                                                   0xF0000000L
+//PC_PERFCOUNTER0_SELECT
+#define PC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define PC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define PC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define PC_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define PC_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define PC_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define PC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define PC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define PC_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define PC_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//PC_PERFCOUNTER1_SELECT
+#define PC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define PC_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define PC_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define PC_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define PC_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define PC_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define PC_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define PC_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define PC_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define PC_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//PC_PERFCOUNTER2_SELECT
+#define PC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define PC_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define PC_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define PC_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define PC_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define PC_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define PC_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define PC_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define PC_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define PC_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//PC_PERFCOUNTER3_SELECT
+#define PC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define PC_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define PC_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define PC_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define PC_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define PC_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define PC_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define PC_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define PC_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define PC_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//PC_PERFCOUNTER0_SELECT1
+#define PC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define PC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define PC_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define PC_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define PC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define PC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define PC_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define PC_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//PC_PERFCOUNTER1_SELECT1
+#define PC_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define PC_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define PC_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define PC_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define PC_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define PC_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define PC_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define PC_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//PC_PERFCOUNTER2_SELECT1
+#define PC_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define PC_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define PC_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define PC_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define PC_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define PC_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define PC_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define PC_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//PC_PERFCOUNTER3_SELECT1
+#define PC_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define PC_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define PC_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define PC_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define PC_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define PC_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define PC_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define PC_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//SQ_PERFCOUNTER0_SELECT
+#define SQ_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SQ_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER1_SELECT
+#define SQ_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SQ_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER2_SELECT
+#define SQ_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER2_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SQ_PERFCOUNTER2_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER3_SELECT
+#define SQ_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER3_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SQ_PERFCOUNTER3_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER4_SELECT
+#define SQ_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER4_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER4_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SQ_PERFCOUNTER4_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER4_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER5_SELECT
+#define SQ_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER5_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER5_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SQ_PERFCOUNTER5_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER5_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER6_SELECT
+#define SQ_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER6_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER6_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER6_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SQ_PERFCOUNTER6_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER6_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER7_SELECT
+#define SQ_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER7_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER7_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER7_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SQ_PERFCOUNTER7_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER7_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER8_SELECT
+#define SQ_PERFCOUNTER8_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER8_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER8_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER8_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SQ_PERFCOUNTER8_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER8_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER9_SELECT
+#define SQ_PERFCOUNTER9_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER9_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER9_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER9_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SQ_PERFCOUNTER9_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER9_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER10_SELECT
+#define SQ_PERFCOUNTER10_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER10_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER10_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER10_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SQ_PERFCOUNTER10_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER10_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQ_PERFCOUNTER11_SELECT
+#define SQ_PERFCOUNTER11_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER11_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER11_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER11_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SQ_PERFCOUNTER11_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER11_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQ_PERFCOUNTER12_SELECT
+#define SQ_PERFCOUNTER12_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER12_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER12_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER12_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SQ_PERFCOUNTER12_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER12_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQ_PERFCOUNTER13_SELECT
+#define SQ_PERFCOUNTER13_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER13_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER13_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER13_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SQ_PERFCOUNTER13_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER13_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQ_PERFCOUNTER14_SELECT
+#define SQ_PERFCOUNTER14_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER14_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER14_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER14_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SQ_PERFCOUNTER14_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER14_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQ_PERFCOUNTER15_SELECT
+#define SQ_PERFCOUNTER15_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER15_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER15_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER15_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SQ_PERFCOUNTER15_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER15_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER0_SELECT
+#define SQG_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQG_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQG_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQG_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQG_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQG_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER1_SELECT
+#define SQG_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQG_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQG_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQG_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQG_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQG_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER2_SELECT
+#define SQG_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQG_PERFCOUNTER2_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQG_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQG_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQG_PERFCOUNTER2_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQG_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER3_SELECT
+#define SQG_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQG_PERFCOUNTER3_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQG_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQG_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQG_PERFCOUNTER3_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQG_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER4_SELECT
+#define SQG_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQG_PERFCOUNTER4_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQG_PERFCOUNTER4_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQG_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQG_PERFCOUNTER4_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQG_PERFCOUNTER4_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER5_SELECT
+#define SQG_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQG_PERFCOUNTER5_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQG_PERFCOUNTER5_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQG_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQG_PERFCOUNTER5_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQG_PERFCOUNTER5_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER6_SELECT
+#define SQG_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQG_PERFCOUNTER6_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQG_PERFCOUNTER6_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQG_PERFCOUNTER6_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQG_PERFCOUNTER6_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQG_PERFCOUNTER6_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER7_SELECT
+#define SQG_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQG_PERFCOUNTER7_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQG_PERFCOUNTER7_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQG_PERFCOUNTER7_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQG_PERFCOUNTER7_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQG_PERFCOUNTER7_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER_CTRL
+#define SQG_PERFCOUNTER_CTRL__PS_EN__SHIFT                                                                    0x0
+#define SQG_PERFCOUNTER_CTRL__GS_EN__SHIFT                                                                    0x2
+#define SQG_PERFCOUNTER_CTRL__HS_EN__SHIFT                                                                    0x4
+#define SQG_PERFCOUNTER_CTRL__CS_EN__SHIFT                                                                    0x6
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME0PIPE0_PERF__SHIFT                                                    0xe
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME0PIPE1_PERF__SHIFT                                                    0xf
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME1PIPE0_PERF__SHIFT                                                    0x10
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME1PIPE1_PERF__SHIFT                                                    0x11
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME1PIPE2_PERF__SHIFT                                                    0x12
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME1PIPE3_PERF__SHIFT                                                    0x13
+#define SQG_PERFCOUNTER_CTRL__POLL_BEFORE_PERF_READ__SHIFT                                                    0x1f
+#define SQG_PERFCOUNTER_CTRL__PS_EN_MASK                                                                      0x00000001L
+#define SQG_PERFCOUNTER_CTRL__GS_EN_MASK                                                                      0x00000004L
+#define SQG_PERFCOUNTER_CTRL__HS_EN_MASK                                                                      0x00000010L
+#define SQG_PERFCOUNTER_CTRL__CS_EN_MASK                                                                      0x00000040L
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME0PIPE0_PERF_MASK                                                      0x00004000L
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME0PIPE1_PERF_MASK                                                      0x00008000L
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME1PIPE0_PERF_MASK                                                      0x00010000L
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME1PIPE1_PERF_MASK                                                      0x00020000L
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME1PIPE2_PERF_MASK                                                      0x00040000L
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME1PIPE3_PERF_MASK                                                      0x00080000L
+#define SQG_PERFCOUNTER_CTRL__POLL_BEFORE_PERF_READ_MASK                                                      0x80000000L
+//SQG_PERFCOUNTER_CTRL2
+#define SQG_PERFCOUNTER_CTRL2__FORCE_EN__SHIFT                                                                0x0
+#define SQG_PERFCOUNTER_CTRL2__VMID_EN__SHIFT                                                                 0x1
+#define SQG_PERFCOUNTER_CTRL2__FORCE_EN_MASK                                                                  0x00000001L
+#define SQG_PERFCOUNTER_CTRL2__VMID_EN_MASK                                                                   0x0001FFFEL
+//SQG_PERF_SAMPLE_FINISH
+#define SQG_PERF_SAMPLE_FINISH__STATUS__SHIFT                                                                 0x0
+#define SQG_PERF_SAMPLE_FINISH__STATUS_MASK                                                                   0x0000007FL
+//SQ_PERFCOUNTER_CTRL
+#define SQ_PERFCOUNTER_CTRL__PS_EN__SHIFT                                                                     0x0
+#define SQ_PERFCOUNTER_CTRL__GS_EN__SHIFT                                                                     0x2
+#define SQ_PERFCOUNTER_CTRL__HS_EN__SHIFT                                                                     0x4
+#define SQ_PERFCOUNTER_CTRL__CS_EN__SHIFT                                                                     0x6
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME0PIPE0_PERF__SHIFT                                                     0xe
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME0PIPE1_PERF__SHIFT                                                     0xf
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME1PIPE0_PERF__SHIFT                                                     0x10
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME1PIPE1_PERF__SHIFT                                                     0x11
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME1PIPE2_PERF__SHIFT                                                     0x12
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME1PIPE3_PERF__SHIFT                                                     0x13
+#define SQ_PERFCOUNTER_CTRL__PS_EN_MASK                                                                       0x00000001L
+#define SQ_PERFCOUNTER_CTRL__GS_EN_MASK                                                                       0x00000004L
+#define SQ_PERFCOUNTER_CTRL__HS_EN_MASK                                                                       0x00000010L
+#define SQ_PERFCOUNTER_CTRL__CS_EN_MASK                                                                       0x00000040L
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME0PIPE0_PERF_MASK                                                       0x00004000L
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME0PIPE1_PERF_MASK                                                       0x00008000L
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME1PIPE0_PERF_MASK                                                       0x00010000L
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME1PIPE1_PERF_MASK                                                       0x00020000L
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME1PIPE2_PERF_MASK                                                       0x00040000L
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME1PIPE3_PERF_MASK                                                       0x00080000L
+//SQ_PERFCOUNTER_CTRL2
+#define SQ_PERFCOUNTER_CTRL2__FORCE_EN__SHIFT                                                                 0x0
+#define SQ_PERFCOUNTER_CTRL2__VMID_EN__SHIFT                                                                  0x1
+#define SQ_PERFCOUNTER_CTRL2__FORCE_EN_MASK                                                                   0x00000001L
+#define SQ_PERFCOUNTER_CTRL2__VMID_EN_MASK                                                                    0x0001FFFEL
+//SQ_THREAD_TRACE_BUF0_SIZE
+#define SQ_THREAD_TRACE_BUF0_SIZE__SIZE__SHIFT                                                                0x0
+#define SQ_THREAD_TRACE_BUF0_SIZE__SIZE_MASK                                                                  0x003FFFFFL
+//SQ_THREAD_TRACE_BUF0_BASE_LO
+#define SQ_THREAD_TRACE_BUF0_BASE_LO__BASE_LO__SHIFT                                                          0x0
+#define SQ_THREAD_TRACE_BUF0_BASE_LO__BASE_LO_MASK                                                            0xFFFFFFFFL
+//SQ_THREAD_TRACE_BUF0_BASE_HI
+#define SQ_THREAD_TRACE_BUF0_BASE_HI__BASE_HI__SHIFT                                                          0x0
+#define SQ_THREAD_TRACE_BUF0_BASE_HI__BASE_HI_MASK                                                            0x00001FFFL
+//SQ_THREAD_TRACE_BUF1_SIZE
+#define SQ_THREAD_TRACE_BUF1_SIZE__SIZE__SHIFT                                                                0x0
+#define SQ_THREAD_TRACE_BUF1_SIZE__SIZE_MASK                                                                  0x003FFFFFL
+//SQ_THREAD_TRACE_BUF1_BASE_LO
+#define SQ_THREAD_TRACE_BUF1_BASE_LO__BASE_LO__SHIFT                                                          0x0
+#define SQ_THREAD_TRACE_BUF1_BASE_LO__BASE_LO_MASK                                                            0xFFFFFFFFL
+//SQ_THREAD_TRACE_BUF1_BASE_HI
+#define SQ_THREAD_TRACE_BUF1_BASE_HI__BASE_HI__SHIFT                                                          0x0
+#define SQ_THREAD_TRACE_BUF1_BASE_HI__BASE_HI_MASK                                                            0x00001FFFL
+//SQ_THREAD_TRACE_CTRL
+#define SQ_THREAD_TRACE_CTRL__MODE__SHIFT                                                                     0x0
+#define SQ_THREAD_TRACE_CTRL__GL1_PERF_EN__SHIFT                                                              0x3
+#define SQ_THREAD_TRACE_CTRL__INTERRUPT_EN__SHIFT                                                             0x4
+#define SQ_THREAD_TRACE_CTRL__DOUBLE_BUFFER__SHIFT                                                            0x5
+#define SQ_THREAD_TRACE_CTRL__HIWATER__SHIFT                                                                  0x6
+#define SQ_THREAD_TRACE_CTRL__REG_AT_HWM__SHIFT                                                               0x9
+#define SQ_THREAD_TRACE_CTRL__SPI_STALL_EN__SHIFT                                                             0xb
+#define SQ_THREAD_TRACE_CTRL__SQ_STALL_EN__SHIFT                                                              0xc
+#define SQ_THREAD_TRACE_CTRL__STALL_ALL_SIMDS__SHIFT                                                          0xd
+#define SQ_THREAD_TRACE_CTRL__UTIL_TIMER__SHIFT                                                               0xe
+#define SQ_THREAD_TRACE_CTRL__WAVESTART_MODE__SHIFT                                                           0xf
+#define SQ_THREAD_TRACE_CTRL__SYNC_COUNT_MARKERS__SHIFT                                                       0x12
+#define SQ_THREAD_TRACE_CTRL__SYNC_COUNT_DRAWS__SHIFT                                                         0x13
+#define SQ_THREAD_TRACE_CTRL__LOWATER_OFFSET__SHIFT                                                           0x14
+#define SQ_THREAD_TRACE_CTRL__GL1X_PREFETCH_PAGE__SHIFT                                                       0x17
+#define SQ_THREAD_TRACE_CTRL__AUTO_FLUSH_PADDING_DIS__SHIFT                                                   0x1c
+#define SQ_THREAD_TRACE_CTRL__AUTO_FLUSH_MODE__SHIFT                                                          0x1d
+#define SQ_THREAD_TRACE_CTRL__NCP_REG_TOKEN_EN__SHIFT                                                         0x1e
+#define SQ_THREAD_TRACE_CTRL__DRAW_EVENT_EN__SHIFT                                                            0x1f
+#define SQ_THREAD_TRACE_CTRL__MODE_MASK                                                                       0x00000003L
+#define SQ_THREAD_TRACE_CTRL__GL1_PERF_EN_MASK                                                                0x00000008L
+#define SQ_THREAD_TRACE_CTRL__INTERRUPT_EN_MASK                                                               0x00000010L
+#define SQ_THREAD_TRACE_CTRL__DOUBLE_BUFFER_MASK                                                              0x00000020L
+#define SQ_THREAD_TRACE_CTRL__HIWATER_MASK                                                                    0x000001C0L
+#define SQ_THREAD_TRACE_CTRL__REG_AT_HWM_MASK                                                                 0x00000600L
+#define SQ_THREAD_TRACE_CTRL__SPI_STALL_EN_MASK                                                               0x00000800L
+#define SQ_THREAD_TRACE_CTRL__SQ_STALL_EN_MASK                                                                0x00001000L
+#define SQ_THREAD_TRACE_CTRL__STALL_ALL_SIMDS_MASK                                                            0x00002000L
+#define SQ_THREAD_TRACE_CTRL__UTIL_TIMER_MASK                                                                 0x00004000L
+#define SQ_THREAD_TRACE_CTRL__WAVESTART_MODE_MASK                                                             0x00018000L
+#define SQ_THREAD_TRACE_CTRL__SYNC_COUNT_MARKERS_MASK                                                         0x00040000L
+#define SQ_THREAD_TRACE_CTRL__SYNC_COUNT_DRAWS_MASK                                                           0x00080000L
+#define SQ_THREAD_TRACE_CTRL__LOWATER_OFFSET_MASK                                                             0x00700000L
+#define SQ_THREAD_TRACE_CTRL__GL1X_PREFETCH_PAGE_MASK                                                         0x07800000L
+#define SQ_THREAD_TRACE_CTRL__AUTO_FLUSH_PADDING_DIS_MASK                                                     0x10000000L
+#define SQ_THREAD_TRACE_CTRL__AUTO_FLUSH_MODE_MASK                                                            0x20000000L
+#define SQ_THREAD_TRACE_CTRL__NCP_REG_TOKEN_EN_MASK                                                           0x40000000L
+#define SQ_THREAD_TRACE_CTRL__DRAW_EVENT_EN_MASK                                                              0x80000000L
+//SQ_THREAD_TRACE_MASK
+#define SQ_THREAD_TRACE_MASK__SIMD_SEL__SHIFT                                                                 0x0
+#define SQ_THREAD_TRACE_MASK__WGP_SEL__SHIFT                                                                  0x4
+#define SQ_THREAD_TRACE_MASK__SA_SEL__SHIFT                                                                   0x9
+#define SQ_THREAD_TRACE_MASK__WTYPE_INCLUDE__SHIFT                                                            0xa
+#define SQ_THREAD_TRACE_MASK__EXCLUDE_NONDETAIL_SHADERDATA__SHIFT                                             0x11
+#define SQ_THREAD_TRACE_MASK__EXCLUDE_NONDETAIL_WAVESTART_EXT__SHIFT                                          0x12
+#define SQ_THREAD_TRACE_MASK__EXCLUDE_NONDETAIL_ALLOC__SHIFT                                                  0x13
+#define SQ_THREAD_TRACE_MASK__SIMD_POWER__SHIFT                                                               0x14
+#define SQ_THREAD_TRACE_MASK__SIMD_POWER_FREQ__SHIFT                                                          0x15
+#define SQ_THREAD_TRACE_MASK__SIMD_SEL_MASK                                                                   0x00000003L
+#define SQ_THREAD_TRACE_MASK__WGP_SEL_MASK                                                                    0x000000F0L
+#define SQ_THREAD_TRACE_MASK__SA_SEL_MASK                                                                     0x00000200L
+#define SQ_THREAD_TRACE_MASK__WTYPE_INCLUDE_MASK                                                              0x0001FC00L
+#define SQ_THREAD_TRACE_MASK__EXCLUDE_NONDETAIL_SHADERDATA_MASK                                               0x00020000L
+#define SQ_THREAD_TRACE_MASK__EXCLUDE_NONDETAIL_WAVESTART_EXT_MASK                                            0x00040000L
+#define SQ_THREAD_TRACE_MASK__EXCLUDE_NONDETAIL_ALLOC_MASK                                                    0x00080000L
+#define SQ_THREAD_TRACE_MASK__SIMD_POWER_MASK                                                                 0x00100000L
+#define SQ_THREAD_TRACE_MASK__SIMD_POWER_FREQ_MASK                                                            0x00200000L
+//SQ_THREAD_TRACE_TOKEN_MASK
+#define SQ_THREAD_TRACE_TOKEN_MASK__TOKEN_EXCLUDE__SHIFT                                                      0x0
+#define SQ_THREAD_TRACE_TOKEN_MASK__TTRACE_EXEC__SHIFT                                                        0xc
+#define SQ_THREAD_TRACE_TOKEN_MASK__BOP_EVENTS_TOKEN_INCLUDE__SHIFT                                           0xd
+#define SQ_THREAD_TRACE_TOKEN_MASK__EXCLUDE_BARRIER_WAIT__SHIFT                                               0xe
+#define SQ_THREAD_TRACE_TOKEN_MASK__EXCLUDE_ALUEXEC5__SHIFT                                                   0xf
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_INCLUDE__SHIFT                                                        0x10
+#define SQ_THREAD_TRACE_TOKEN_MASK__INST_EXCLUDE__SHIFT                                                       0x18
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_EXCLUDE__SHIFT                                                        0x1a
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_DETAIL_ALL__SHIFT                                                     0x1f
+#define SQ_THREAD_TRACE_TOKEN_MASK__TOKEN_EXCLUDE_MASK                                                        0x00000FFFL
+#define SQ_THREAD_TRACE_TOKEN_MASK__TTRACE_EXEC_MASK                                                          0x00001000L
+#define SQ_THREAD_TRACE_TOKEN_MASK__BOP_EVENTS_TOKEN_INCLUDE_MASK                                             0x00002000L
+#define SQ_THREAD_TRACE_TOKEN_MASK__EXCLUDE_BARRIER_WAIT_MASK                                                 0x00004000L
+#define SQ_THREAD_TRACE_TOKEN_MASK__EXCLUDE_ALUEXEC5_MASK                                                     0x00008000L
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_INCLUDE_MASK                                                          0x00FF0000L
+#define SQ_THREAD_TRACE_TOKEN_MASK__INST_EXCLUDE_MASK                                                         0x03000000L
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_EXCLUDE_MASK                                                          0x1C000000L
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_DETAIL_ALL_MASK                                                       0x80000000L
+//SQ_THREAD_TRACE_WPTR
+#define SQ_THREAD_TRACE_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SQ_THREAD_TRACE_WPTR__BUFFER_ID__SHIFT                                                                0x1f
+#define SQ_THREAD_TRACE_WPTR__OFFSET_MASK                                                                     0x1FFFFFFFL
+#define SQ_THREAD_TRACE_WPTR__BUFFER_ID_MASK                                                                  0x80000000L
+//SQ_THREAD_TRACE_HALT
+#define SQ_THREAD_TRACE_HALT__ENTER_CGCG__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_HALT__CGCG_READY__SHIFT                                                               0x1
+#define SQ_THREAD_TRACE_HALT__ENTER_POWEROFF__SHIFT                                                           0x2
+#define SQ_THREAD_TRACE_HALT__POWEROFF_READY__SHIFT                                                           0x3
+#define SQ_THREAD_TRACE_HALT__ENTER_CGCG_MASK                                                                 0x00000001L
+#define SQ_THREAD_TRACE_HALT__CGCG_READY_MASK                                                                 0x00000002L
+#define SQ_THREAD_TRACE_HALT__ENTER_POWEROFF_MASK                                                             0x00000004L
+#define SQ_THREAD_TRACE_HALT__POWEROFF_READY_MASK                                                             0x00000008L
+//SQ_THREAD_TRACE_POWEROFF_RESTORE_1
+#define SQ_THREAD_TRACE_POWEROFF_RESTORE_1__STATES__SHIFT                                                     0x0
+#define SQ_THREAD_TRACE_POWEROFF_RESTORE_1__STATES_MASK                                                       0xFFFFFFFFL
+//SQ_THREAD_TRACE_STATUS
+#define SQ_THREAD_TRACE_STATUS__FINISH_PENDING__SHIFT                                                         0x0
+#define SQ_THREAD_TRACE_STATUS__FINISH_DONE__SHIFT                                                            0xc
+#define SQ_THREAD_TRACE_STATUS__WRITE_ERROR__SHIFT                                                            0x18
+#define SQ_THREAD_TRACE_STATUS__BUSY__SHIFT                                                                   0x19
+#define SQ_THREAD_TRACE_STATUS__OWNER_VMID__SHIFT                                                             0x1c
+#define SQ_THREAD_TRACE_STATUS__FINISH_PENDING_MASK                                                           0x00000FFFL
+#define SQ_THREAD_TRACE_STATUS__FINISH_DONE_MASK                                                              0x00FFF000L
+#define SQ_THREAD_TRACE_STATUS__WRITE_ERROR_MASK                                                              0x01000000L
+#define SQ_THREAD_TRACE_STATUS__BUSY_MASK                                                                     0x02000000L
+#define SQ_THREAD_TRACE_STATUS__OWNER_VMID_MASK                                                               0xF0000000L
+//SQ_THREAD_TRACE_STATUS2
+#define SQ_THREAD_TRACE_STATUS2__BUF0_FULL__SHIFT                                                             0x0
+#define SQ_THREAD_TRACE_STATUS2__BUF1_FULL__SHIFT                                                             0x1
+#define SQ_THREAD_TRACE_STATUS2__PACKET_LOST_BUF_NO_LOCKDOWN__SHIFT                                           0x4
+#define SQ_THREAD_TRACE_STATUS2__BUF_ISSUE_STATUS__SHIFT                                                      0x8
+#define SQ_THREAD_TRACE_STATUS2__BUF_ISSUE__SHIFT                                                             0xd
+#define SQ_THREAD_TRACE_STATUS2__WRITE_BUF_FULL__SHIFT                                                        0xe
+#define SQ_THREAD_TRACE_STATUS2__BUF0_FULL_MASK                                                               0x00000001L
+#define SQ_THREAD_TRACE_STATUS2__BUF1_FULL_MASK                                                               0x00000002L
+#define SQ_THREAD_TRACE_STATUS2__PACKET_LOST_BUF_NO_LOCKDOWN_MASK                                             0x00000010L
+#define SQ_THREAD_TRACE_STATUS2__BUF_ISSUE_STATUS_MASK                                                        0x00001F00L
+#define SQ_THREAD_TRACE_STATUS2__BUF_ISSUE_MASK                                                               0x00002000L
+#define SQ_THREAD_TRACE_STATUS2__WRITE_BUF_FULL_MASK                                                          0x00004000L
+//SQ_THREAD_TRACE_GFX_DRAW_CNTR
+#define SQ_THREAD_TRACE_GFX_DRAW_CNTR__CNTR__SHIFT                                                            0x0
+#define SQ_THREAD_TRACE_GFX_DRAW_CNTR__CNTR_MASK                                                              0xFFFFFFFFL
+//SQ_THREAD_TRACE_GFX_MARKER_CNTR
+#define SQ_THREAD_TRACE_GFX_MARKER_CNTR__CNTR__SHIFT                                                          0x0
+#define SQ_THREAD_TRACE_GFX_MARKER_CNTR__CNTR_MASK                                                            0xFFFFFFFFL
+//SQ_THREAD_TRACE_HP3D_DRAW_CNTR
+#define SQ_THREAD_TRACE_HP3D_DRAW_CNTR__CNTR__SHIFT                                                           0x0
+#define SQ_THREAD_TRACE_HP3D_DRAW_CNTR__CNTR_MASK                                                             0xFFFFFFFFL
+//SQ_THREAD_TRACE_HP3D_MARKER_CNTR
+#define SQ_THREAD_TRACE_HP3D_MARKER_CNTR__CNTR__SHIFT                                                         0x0
+#define SQ_THREAD_TRACE_HP3D_MARKER_CNTR__CNTR_MASK                                                           0xFFFFFFFFL
+//SQ_THREAD_TRACE_DROPPED_CNTR
+#define SQ_THREAD_TRACE_DROPPED_CNTR__CNTR__SHIFT                                                             0x0
+#define SQ_THREAD_TRACE_DROPPED_CNTR__CNTR_MASK                                                               0xFFFFFFFFL
+//SQ_THREAD_TRACE_FINISH_DONE_DEBUG
+#define SQ_THREAD_TRACE_FINISH_DONE_DEBUG__GFX__SHIFT                                                         0x0
+#define SQ_THREAD_TRACE_FINISH_DONE_DEBUG__EXP__SHIFT                                                         0xa
+#define SQ_THREAD_TRACE_FINISH_DONE_DEBUG__GFX_MASK                                                           0x000003FFL
+#define SQ_THREAD_TRACE_FINISH_DONE_DEBUG__EXP_MASK                                                           0x0000FC00L
+//SX_PERFCOUNTER0_SELECT
+#define SX_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SX_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define SX_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define SX_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define SX_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SX_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SX_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define SX_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define SX_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define SX_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SX_PERFCOUNTER1_SELECT
+#define SX_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SX_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define SX_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define SX_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define SX_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SX_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SX_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define SX_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define SX_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define SX_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SX_PERFCOUNTER2_SELECT
+#define SX_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SX_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define SX_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define SX_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define SX_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SX_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SX_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define SX_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define SX_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define SX_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SX_PERFCOUNTER3_SELECT
+#define SX_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SX_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define SX_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define SX_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define SX_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SX_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SX_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define SX_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define SX_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define SX_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SX_PERFCOUNTER0_SELECT1
+#define SX_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define SX_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define SX_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define SX_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define SX_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define SX_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define SX_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define SX_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//SX_PERFCOUNTER1_SELECT1
+#define SX_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define SX_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define SX_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define SX_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define SX_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define SX_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define SX_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define SX_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//SX_PERFCOUNTER2_SELECT1
+#define SX_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define SX_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define SX_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define SX_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define SX_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define SX_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define SX_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define SX_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//SX_PERFCOUNTER3_SELECT1
+#define SX_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define SX_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define SX_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define SX_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define SX_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define SX_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define SX_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define SX_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//TA_PERFCOUNTER0_SELECT
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define TA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define TA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//TA_PERFCOUNTER0_SELECT1
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//TA_PERFCOUNTER1_SELECT
+#define TA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define TA_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define TA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define TA_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define TA_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define TA_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//TD_PERFCOUNTER0_SELECT
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define TD_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define TD_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//TD_PERFCOUNTER0_SELECT1
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//TD_PERFCOUNTER1_SELECT
+#define TD_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define TD_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define TD_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define TD_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define TD_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define TD_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//TCP_PERFCOUNTER0_SELECT
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define TCP_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define TCP_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCP_PERFCOUNTER0_SELECT1
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//TCP_PERFCOUNTER1_SELECT
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define TCP_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define TCP_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCP_PERFCOUNTER1_SELECT1
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//TCP_PERFCOUNTER2_SELECT
+#define TCP_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define TCP_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCP_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define TCP_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCP_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCP_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define TCP_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCP_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define TCP_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCP_PERFCOUNTER2_SELECT1
+#define TCP_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define TCP_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define TCP_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define TCP_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define TCP_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define TCP_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define TCP_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//TCP_PERFCOUNTER3_SELECT
+#define TCP_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define TCP_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCP_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define TCP_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCP_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCP_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define TCP_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCP_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define TCP_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCP_PERFCOUNTER3_SELECT1
+#define TCP_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define TCP_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define TCP_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define TCP_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define TCP_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define TCP_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define TCP_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//TCP_PERFCOUNTER4_SELECT
+#define TCP_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER4_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCP_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCP_PERFCOUNTER4_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCP_PERFCOUNTER5_SELECT
+#define TCP_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER5_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCP_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCP_PERFCOUNTER5_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCP_PERFCOUNTER6_SELECT
+#define TCP_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER6_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCP_PERFCOUNTER6_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCP_PERFCOUNTER6_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCP_PERFCOUNTER7_SELECT
+#define TCP_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER7_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCP_PERFCOUNTER7_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCP_PERFCOUNTER7_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//GL1C_PERFCOUNTER0_SELECT
+#define GL1C_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1C_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL1C_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1C_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL1C_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1C_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1C_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL1C_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1C_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL1C_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1C_PERFCOUNTER0_SELECT1
+#define GL1C_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL1C_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GL1C_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GL1C_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL1C_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL1C_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GL1C_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+//GL1C_PERFCOUNTER1_SELECT
+#define GL1C_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1C_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL1C_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1C_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL1C_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1C_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1C_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL1C_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1C_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL1C_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1C_PERFCOUNTER1_SELECT1
+#define GL1C_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL1C_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GL1C_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GL1C_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL1C_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL1C_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GL1C_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+//GL1C_PERFCOUNTER2_SELECT
+#define GL1C_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1C_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL1C_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1C_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL1C_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1C_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1C_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL1C_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1C_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL1C_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1C_PERFCOUNTER2_SELECT1
+#define GL1C_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL1C_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GL1C_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GL1C_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL1C_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL1C_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GL1C_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+//GL1C_PERFCOUNTER3_SELECT
+#define GL1C_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1C_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL1C_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1C_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL1C_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1C_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1C_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL1C_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1C_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL1C_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1C_PERFCOUNTER3_SELECT1
+#define GL1C_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL1C_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GL1C_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GL1C_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL1C_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL1C_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GL1C_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+//GL1XC_PERFCOUNTER0_SELECT
+#define GL1XC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define GL1XC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define GL1XC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define GL1XC_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define GL1XC_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define GL1XC_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define GL1XC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define GL1XC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define GL1XC_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define GL1XC_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//GL1XC_PERFCOUNTER0_SELECT1
+#define GL1XC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define GL1XC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define GL1XC_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define GL1XC_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define GL1XC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define GL1XC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define GL1XC_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define GL1XC_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//GL1XC_PERFCOUNTER1_SELECT
+#define GL1XC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define GL1XC_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define GL1XC_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define GL1XC_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define GL1XC_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define GL1XC_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define GL1XC_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define GL1XC_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define GL1XC_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define GL1XC_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//GL1XC_PERFCOUNTER1_SELECT1
+#define GL1XC_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define GL1XC_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define GL1XC_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define GL1XC_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define GL1XC_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define GL1XC_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define GL1XC_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define GL1XC_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//GL1XC_PERFCOUNTER2_SELECT
+#define GL1XC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define GL1XC_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define GL1XC_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define GL1XC_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define GL1XC_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define GL1XC_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define GL1XC_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define GL1XC_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define GL1XC_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define GL1XC_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//GL1XC_PERFCOUNTER2_SELECT1
+#define GL1XC_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define GL1XC_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define GL1XC_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define GL1XC_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define GL1XC_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define GL1XC_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define GL1XC_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define GL1XC_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//GL1XC_PERFCOUNTER3_SELECT
+#define GL1XC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define GL1XC_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define GL1XC_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define GL1XC_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define GL1XC_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define GL1XC_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define GL1XC_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define GL1XC_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define GL1XC_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define GL1XC_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//GL1XC_PERFCOUNTER3_SELECT1
+#define GL1XC_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define GL1XC_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define GL1XC_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define GL1XC_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define GL1XC_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define GL1XC_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define GL1XC_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define GL1XC_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//CB_PERFCOUNTER_FILTER
+#define CB_PERFCOUNTER_FILTER__OP_FILTER_ENABLE__SHIFT                                                        0x0
+#define CB_PERFCOUNTER_FILTER__OP_FILTER_SEL__SHIFT                                                           0x1
+#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_ENABLE__SHIFT                                                    0x4
+#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_SEL__SHIFT                                                       0x5
+#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_ENABLE__SHIFT                                                     0xa
+#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_SEL__SHIFT                                                        0xb
+#define CB_PERFCOUNTER_FILTER__MRT_FILTER_ENABLE__SHIFT                                                       0xc
+#define CB_PERFCOUNTER_FILTER__MRT_FILTER_SEL__SHIFT                                                          0xd
+#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_ENABLE__SHIFT                                               0x11
+#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_SEL__SHIFT                                                  0x12
+#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_ENABLE__SHIFT                                             0x15
+#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_SEL__SHIFT                                                0x16
+#define CB_PERFCOUNTER_FILTER__OP_FILTER_ENABLE_MASK                                                          0x00000001L
+#define CB_PERFCOUNTER_FILTER__OP_FILTER_SEL_MASK                                                             0x0000000EL
+#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_ENABLE_MASK                                                      0x00000010L
+#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_SEL_MASK                                                         0x000003E0L
+#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_ENABLE_MASK                                                       0x00000400L
+#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_SEL_MASK                                                          0x00000800L
+#define CB_PERFCOUNTER_FILTER__MRT_FILTER_ENABLE_MASK                                                         0x00001000L
+#define CB_PERFCOUNTER_FILTER__MRT_FILTER_SEL_MASK                                                            0x0000E000L
+#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_ENABLE_MASK                                                 0x00020000L
+#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_SEL_MASK                                                    0x001C0000L
+#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_ENABLE_MASK                                               0x00200000L
+#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_SEL_MASK                                                  0x00C00000L
+//CB_PERFCOUNTER0_SELECT
+#define CB_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define CB_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define CB_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define CB_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define CB_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define CB_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define CB_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define CB_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define CB_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define CB_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//CB_PERFCOUNTER0_SELECT1
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//CB_PERFCOUNTER1_SELECT
+#define CB_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define CB_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define CB_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define CB_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//CB_PERFCOUNTER2_SELECT
+#define CB_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define CB_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define CB_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define CB_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//CB_PERFCOUNTER3_SELECT
+#define CB_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define CB_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define CB_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define CB_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//DB_PERFCOUNTER0_SELECT
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define DB_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define DB_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//DB_PERFCOUNTER0_SELECT1
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//DB_PERFCOUNTER1_SELECT
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define DB_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define DB_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//DB_PERFCOUNTER1_SELECT1
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//DB_PERFCOUNTER2_SELECT
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define DB_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define DB_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//DB_PERFCOUNTER2_SELECT1
+#define DB_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define DB_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define DB_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define DB_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define DB_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define DB_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define DB_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define DB_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//DB_PERFCOUNTER3_SELECT
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define DB_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define DB_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//DB_PERFCOUNTER3_SELECT1
+#define DB_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define DB_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define DB_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define DB_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define DB_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define DB_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define DB_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define DB_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//RMI_PERFCOUNTER0_SELECT
+#define RMI_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define RMI_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define RMI_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define RMI_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define RMI_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define RMI_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define RMI_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define RMI_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define RMI_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define RMI_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//RMI_PERFCOUNTER0_SELECT1
+#define RMI_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define RMI_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define RMI_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define RMI_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define RMI_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define RMI_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define RMI_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//RMI_PERFCOUNTER1_SELECT
+#define RMI_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define RMI_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define RMI_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define RMI_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//RMI_PERFCOUNTER2_SELECT
+#define RMI_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define RMI_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define RMI_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define RMI_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define RMI_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define RMI_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define RMI_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define RMI_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define RMI_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define RMI_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//RMI_PERFCOUNTER2_SELECT1
+#define RMI_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define RMI_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define RMI_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define RMI_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define RMI_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define RMI_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define RMI_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//RMI_PERFCOUNTER3_SELECT
+#define RMI_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define RMI_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define RMI_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define RMI_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//RMI_PERF_COUNTER_CNTL
+#define RMI_PERF_COUNTER_CNTL__TRANS_BASED_PERF_EN_SEL__SHIFT                                                 0x0
+#define RMI_PERF_COUNTER_CNTL__EVENT_BASED_PERF_EN_SEL__SHIFT                                                 0x2
+#define RMI_PERF_COUNTER_CNTL__TC_PERF_EN_SEL__SHIFT                                                          0x4
+#define RMI_PERF_COUNTER_CNTL__PERF_EVENT_WINDOW_MASK0__SHIFT                                                 0x6
+#define RMI_PERF_COUNTER_CNTL__PERF_EVENT_WINDOW_MASK1__SHIFT                                                 0x8
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_CID__SHIFT                                                        0xa
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_VMID__SHIFT                                                       0xe
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_BURST_LENGTH_THRESHOLD__SHIFT                                     0x13
+#define RMI_PERF_COUNTER_CNTL__PERF_SOFT_RESET__SHIFT                                                         0x19
+#define RMI_PERF_COUNTER_CNTL__PERF_CNTR_SPM_SEL__SHIFT                                                       0x1a
+#define RMI_PERF_COUNTER_CNTL__TRANS_BASED_PERF_EN_SEL_MASK                                                   0x00000003L
+#define RMI_PERF_COUNTER_CNTL__EVENT_BASED_PERF_EN_SEL_MASK                                                   0x0000000CL
+#define RMI_PERF_COUNTER_CNTL__TC_PERF_EN_SEL_MASK                                                            0x00000030L
+#define RMI_PERF_COUNTER_CNTL__PERF_EVENT_WINDOW_MASK0_MASK                                                   0x000000C0L
+#define RMI_PERF_COUNTER_CNTL__PERF_EVENT_WINDOW_MASK1_MASK                                                   0x00000300L
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_CID_MASK                                                          0x00003C00L
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_VMID_MASK                                                         0x0007C000L
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_BURST_LENGTH_THRESHOLD_MASK                                       0x01F80000L
+#define RMI_PERF_COUNTER_CNTL__PERF_SOFT_RESET_MASK                                                           0x02000000L
+#define RMI_PERF_COUNTER_CNTL__PERF_CNTR_SPM_SEL_MASK                                                         0x04000000L
+//PA_PH_PERFCOUNTER0_SELECT
+#define PA_PH_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_PH_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_PH_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_PH_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_PH_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_PH_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_PH_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_PH_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_PH_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_PH_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_PH_PERFCOUNTER0_SELECT1
+#define PA_PH_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_PH_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_PH_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_PH_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_PH_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_PH_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_PH_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_PH_PERFCOUNTER1_SELECT
+#define PA_PH_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_PH_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_PH_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_PH_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_PH_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_PH_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_PH_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_PH_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_PH_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_PH_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_PH_PERFCOUNTER2_SELECT
+#define PA_PH_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_PH_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_PH_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_PH_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_PH_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_PH_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_PH_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_PH_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_PH_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_PH_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_PH_PERFCOUNTER3_SELECT
+#define PA_PH_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_PH_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_PH_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_PH_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_PH_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_PH_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_PH_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_PH_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_PH_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_PH_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_PH_PERFCOUNTER4_SELECT
+#define PA_PH_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_PH_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_PH_PERFCOUNTER5_SELECT
+#define PA_PH_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_PH_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_PH_PERFCOUNTER6_SELECT
+#define PA_PH_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_PH_PERFCOUNTER6_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_PH_PERFCOUNTER7_SELECT
+#define PA_PH_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_PH_PERFCOUNTER7_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_PH_PERFCOUNTER1_SELECT1
+#define PA_PH_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_PH_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_PH_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_PH_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_PH_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_PH_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_PH_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_PH_PERFCOUNTER2_SELECT1
+#define PA_PH_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_PH_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_PH_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_PH_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_PH_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_PH_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_PH_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_PH_PERFCOUNTER3_SELECT1
+#define PA_PH_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_PH_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_PH_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_PH_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_PH_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_PH_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_PH_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//UTCL1_PERFCOUNTER0_SELECT
+#define UTCL1_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define UTCL1_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define UTCL1_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                            0x14
+#define UTCL1_PERFCOUNTER0_SELECT__COUNTER_MODE1__SHIFT                                                       0x18
+#define UTCL1_PERFCOUNTER0_SELECT__COUNTER_MODE__SHIFT                                                        0x1c
+#define UTCL1_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define UTCL1_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define UTCL1_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                              0x00F00000L
+#define UTCL1_PERFCOUNTER0_SELECT__COUNTER_MODE1_MASK                                                         0x0F000000L
+#define UTCL1_PERFCOUNTER0_SELECT__COUNTER_MODE_MASK                                                          0xF0000000L
+//UTCL1_PERFCOUNTER0_SELECT1
+#define UTCL1_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define UTCL1_PERFCOUNTER0_SELECT1__COUNTER_MODE3__SHIFT                                                      0x18
+#define UTCL1_PERFCOUNTER0_SELECT1__COUNTER_MODE2__SHIFT                                                      0x1c
+#define UTCL1_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define UTCL1_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define UTCL1_PERFCOUNTER0_SELECT1__COUNTER_MODE3_MASK                                                        0x0F000000L
+#define UTCL1_PERFCOUNTER0_SELECT1__COUNTER_MODE2_MASK                                                        0xF0000000L
+//UTCL1_PERFCOUNTER1_SELECT
+#define UTCL1_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define UTCL1_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define UTCL1_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                            0x14
+#define UTCL1_PERFCOUNTER1_SELECT__COUNTER_MODE1__SHIFT                                                       0x18
+#define UTCL1_PERFCOUNTER1_SELECT__COUNTER_MODE__SHIFT                                                        0x1c
+#define UTCL1_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define UTCL1_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define UTCL1_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                              0x00F00000L
+#define UTCL1_PERFCOUNTER1_SELECT__COUNTER_MODE1_MASK                                                         0x0F000000L
+#define UTCL1_PERFCOUNTER1_SELECT__COUNTER_MODE_MASK                                                          0xF0000000L
+//UTCL1_PERFCOUNTER1_SELECT1
+#define UTCL1_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define UTCL1_PERFCOUNTER1_SELECT1__COUNTER_MODE3__SHIFT                                                      0x18
+#define UTCL1_PERFCOUNTER1_SELECT1__COUNTER_MODE2__SHIFT                                                      0x1c
+#define UTCL1_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define UTCL1_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define UTCL1_PERFCOUNTER1_SELECT1__COUNTER_MODE3_MASK                                                        0x0F000000L
+#define UTCL1_PERFCOUNTER1_SELECT1__COUNTER_MODE2_MASK                                                        0xF0000000L
+//UTCL1_PERFCOUNTER2_SELECT
+#define UTCL1_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define UTCL1_PERFCOUNTER2_SELECT__COUNTER_MODE__SHIFT                                                        0x1c
+#define UTCL1_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define UTCL1_PERFCOUNTER2_SELECT__COUNTER_MODE_MASK                                                          0xF0000000L
+//UTCL1_PERFCOUNTER3_SELECT
+#define UTCL1_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define UTCL1_PERFCOUNTER3_SELECT__COUNTER_MODE__SHIFT                                                        0x1c
+#define UTCL1_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define UTCL1_PERFCOUNTER3_SELECT__COUNTER_MODE_MASK                                                          0xF0000000L
+//WGS_PERFCOUNTER1_SELECT
+#define WGS_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define WGS_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x1c
+#define WGS_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define WGS_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0xF0000000L
+//WGS_PERFCOUNTER0_SELECT1
+#define WGS_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define WGS_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define WGS_PERFCOUNTER0_SELECT1__CNTR_MODE3__SHIFT                                                           0x18
+#define WGS_PERFCOUNTER0_SELECT1__CNTR_MODE2__SHIFT                                                           0x1c
+#define WGS_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define WGS_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define WGS_PERFCOUNTER0_SELECT1__CNTR_MODE3_MASK                                                             0x0F000000L
+#define WGS_PERFCOUNTER0_SELECT1__CNTR_MODE2_MASK                                                             0xF0000000L
+//WGS_PERFCOUNTER0_SELECT
+#define WGS_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define WGS_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define WGS_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define WGS_PERFCOUNTER0_SELECT__CNTR_MODE1__SHIFT                                                            0x18
+#define WGS_PERFCOUNTER0_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
+#define WGS_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define WGS_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define WGS_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define WGS_PERFCOUNTER0_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
+#define WGS_PERFCOUNTER0_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
+//GL1A_PERFCOUNTER0_SELECT
+#define GL1A_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1A_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL1A_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1A_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL1A_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1A_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1A_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL1A_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1A_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL1A_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1A_PERFCOUNTER0_SELECT1
+#define GL1A_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL1A_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GL1A_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GL1A_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL1A_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL1A_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GL1A_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+//GL1A_PERFCOUNTER1_SELECT
+#define GL1A_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1A_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL1A_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1A_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL1A_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1A_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1A_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL1A_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1A_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL1A_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1A_PERFCOUNTER1_SELECT1
+#define GL1A_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL1A_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GL1A_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GL1A_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL1A_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL1A_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GL1A_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+//GL1A_PERFCOUNTER2_SELECT
+#define GL1A_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1A_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL1A_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1A_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL1A_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1A_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1A_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL1A_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1A_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL1A_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1A_PERFCOUNTER2_SELECT1
+#define GL1A_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL1A_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GL1A_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GL1A_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL1A_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL1A_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GL1A_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+//GL1A_PERFCOUNTER3_SELECT
+#define GL1A_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1A_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL1A_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1A_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL1A_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1A_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1A_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL1A_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1A_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL1A_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1A_PERFCOUNTER3_SELECT1
+#define GL1A_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL1A_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GL1A_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GL1A_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL1A_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL1A_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GL1A_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+//GL1XA_PERFCOUNTER0_SELECT
+#define GL1XA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define GL1XA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define GL1XA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define GL1XA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define GL1XA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define GL1XA_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define GL1XA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define GL1XA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define GL1XA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define GL1XA_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//GL1XA_PERFCOUNTER0_SELECT1
+#define GL1XA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define GL1XA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define GL1XA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define GL1XA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define GL1XA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define GL1XA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define GL1XA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define GL1XA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//GL1XA_PERFCOUNTER1_SELECT
+#define GL1XA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define GL1XA_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define GL1XA_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define GL1XA_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define GL1XA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define GL1XA_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define GL1XA_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define GL1XA_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define GL1XA_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define GL1XA_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//GL1XA_PERFCOUNTER1_SELECT1
+#define GL1XA_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define GL1XA_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define GL1XA_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define GL1XA_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define GL1XA_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define GL1XA_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define GL1XA_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define GL1XA_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//GL1XA_PERFCOUNTER2_SELECT
+#define GL1XA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define GL1XA_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define GL1XA_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define GL1XA_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define GL1XA_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define GL1XA_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define GL1XA_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define GL1XA_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define GL1XA_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define GL1XA_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//GL1XA_PERFCOUNTER2_SELECT1
+#define GL1XA_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define GL1XA_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define GL1XA_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define GL1XA_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define GL1XA_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define GL1XA_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define GL1XA_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define GL1XA_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//GL1XA_PERFCOUNTER3_SELECT
+#define GL1XA_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define GL1XA_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define GL1XA_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define GL1XA_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define GL1XA_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define GL1XA_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define GL1XA_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define GL1XA_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define GL1XA_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define GL1XA_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//GL1XA_PERFCOUNTER3_SELECT1
+#define GL1XA_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define GL1XA_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define GL1XA_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define GL1XA_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define GL1XA_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define GL1XA_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define GL1XA_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define GL1XA_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_pwrdec
+//GFX_ICG_SPI_RA0_CLK_CTRL
+#define GFX_ICG_SPI_RA0_CLK_CTRL__GRP_OVERRIDES__SHIFT                                                        0x0
+#define GFX_ICG_SPI_RA0_CLK_CTRL__REG_OVERRIDE__SHIFT                                                         0x1f
+#define GFX_ICG_SPI_RA0_CLK_CTRL__GRP_OVERRIDES_MASK                                                          0x0000FFFFL
+#define GFX_ICG_SPI_RA0_CLK_CTRL__REG_OVERRIDE_MASK                                                           0x80000000L
+//GFX_ICG_SPI_RA1_CLK_CTRL
+#define GFX_ICG_SPI_RA1_CLK_CTRL__GRP_OVERRIDES__SHIFT                                                        0x0
+#define GFX_ICG_SPI_RA1_CLK_CTRL__GRP_OVERRIDES_MASK                                                          0x0000FFFFL
+//GFX_ICG_SPI_CS_CTRL
+#define GFX_ICG_SPI_CS_CTRL__GRP_OVERRIDES__SHIFT                                                             0x0
+#define GFX_ICG_SPI_CS_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x10
+#define GFX_ICG_SPI_CS_CTRL__GRP_OVERRIDES_MASK                                                               0x0000FFFFL
+#define GFX_ICG_SPI_CS_CTRL__OFF_HYSTERESIS_MASK                                                              0x003F0000L
+//GFX_ICG_SPI_PS_CTRL
+#define GFX_ICG_SPI_PS_CTRL__GRP_OVERRIDES__SHIFT                                                             0x0
+#define GFX_ICG_SPI_PS_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x10
+#define GFX_ICG_SPI_PS_CTRL__GRP_OVERRIDES_MASK                                                               0x0000FFFFL
+#define GFX_ICG_SPI_PS_CTRL__OFF_HYSTERESIS_MASK                                                              0x003F0000L
+//GFX_ICG_SPIS_CTRL
+#define GFX_ICG_SPIS_CTRL__GRP_OVERRIDES__SHIFT                                                               0x0
+#define GFX_ICG_SPIS_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
+#define GFX_ICG_SPIS_CTRL__GRP_OVERRIDES_MASK                                                                 0x0000FFFFL
+#define GFX_ICG_SPIS_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
+//CGTX_SPI_DEBUG_CLK_CTRL
+#define CGTX_SPI_DEBUG_CLK_CTRL__GRP5_CG_OFF_HYST__SHIFT                                                      0x0
+#define CGTX_SPI_DEBUG_CLK_CTRL__GRP5_CG_OVERRIDE__SHIFT                                                      0x6
+#define CGTX_SPI_DEBUG_CLK_CTRL__ALL_CLK_ON_OVERRIDE__SHIFT                                                   0x7
+#define CGTX_SPI_DEBUG_CLK_CTRL__SPI_REPEATER_FGCG_OVERRIDE__SHIFT                                            0x9
+#define CGTX_SPI_DEBUG_CLK_CTRL__GRP5_CG_OFF_HYST_MASK                                                        0x0000003FL
+#define CGTX_SPI_DEBUG_CLK_CTRL__GRP5_CG_OVERRIDE_MASK                                                        0x00000040L
+#define CGTX_SPI_DEBUG_CLK_CTRL__ALL_CLK_ON_OVERRIDE_MASK                                                     0x00000080L
+#define CGTX_SPI_DEBUG_CLK_CTRL__SPI_REPEATER_FGCG_OVERRIDE_MASK                                              0x00000200L
+//GFX_ICG_SPI_CTRL
+#define GFX_ICG_SPI_CTRL__GRP_OVERRIDES__SHIFT                                                                0x0
+#define GFX_ICG_SPI_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x10
+#define GFX_ICG_SPI_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
+#define GFX_ICG_SPI_CTRL__GRP_OVERRIDES_MASK                                                                  0x0000FFFFL
+#define GFX_ICG_SPI_CTRL__OFF_HYSTERESIS_MASK                                                                 0x003F0000L
+#define GFX_ICG_SPI_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
+//GFX_ICG_PC_CLK_CTRL
+#define GFX_ICG_PC_CLK_CTRL__ON_DELAY__SHIFT                                                                  0x0
+#define GFX_ICG_PC_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x4
+#define GFX_ICG_PC_CLK_CTRL__PC_GLOBAL_MGCG_OVERRIDE__SHIFT                                                   0xc
+#define GFX_ICG_PC_CLK_CTRL__PC_SC_INT_MGCG_OVERRIDE__SHIFT                                                   0xd
+#define GFX_ICG_PC_CLK_CTRL__MISS_WALKER_MGCG_OVERRIDE__SHIFT                                                 0xe
+#define GFX_ICG_PC_CLK_CTRL__PRIM_QUEUE_MGCG_OVERRIDE__SHIFT                                                  0xf
+#define GFX_ICG_PC_CLK_CTRL__GL1_IF_MGCG_OVERRIDE__SHIFT                                                      0x10
+#define GFX_ICG_PC_CLK_CTRL__GL1_READ_RETURN_MGCG_OVERRIDE__SHIFT                                             0x11
+#define GFX_ICG_PC_CLK_CTRL__PC_MEM_MGCG_OVERRIDE__SHIFT                                                      0x12
+#define GFX_ICG_PC_CLK_CTRL__LDS_WRITE_CNTL_MGCG_OVERRIDE__SHIFT                                              0x13
+#define GFX_ICG_PC_CLK_CTRL__LDS_OUT_MGCG_OVERRIDE__SHIFT                                                     0x14
+#define GFX_ICG_PC_CLK_CTRL__PC_REGS_MGCG_OVERRIDE__SHIFT                                                     0x15
+#define GFX_ICG_PC_CLK_CTRL__PC_PERFMON_MGCG_OVERRIDE__SHIFT                                                  0x16
+#define GFX_ICG_PC_CLK_CTRL__ON_DELAY_MASK                                                                    0x0000000FL
+#define GFX_ICG_PC_CLK_CTRL__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
+#define GFX_ICG_PC_CLK_CTRL__PC_GLOBAL_MGCG_OVERRIDE_MASK                                                     0x00001000L
+#define GFX_ICG_PC_CLK_CTRL__PC_SC_INT_MGCG_OVERRIDE_MASK                                                     0x00002000L
+#define GFX_ICG_PC_CLK_CTRL__MISS_WALKER_MGCG_OVERRIDE_MASK                                                   0x00004000L
+#define GFX_ICG_PC_CLK_CTRL__PRIM_QUEUE_MGCG_OVERRIDE_MASK                                                    0x00008000L
+#define GFX_ICG_PC_CLK_CTRL__GL1_IF_MGCG_OVERRIDE_MASK                                                        0x00010000L
+#define GFX_ICG_PC_CLK_CTRL__GL1_READ_RETURN_MGCG_OVERRIDE_MASK                                               0x00020000L
+#define GFX_ICG_PC_CLK_CTRL__PC_MEM_MGCG_OVERRIDE_MASK                                                        0x00040000L
+#define GFX_ICG_PC_CLK_CTRL__LDS_WRITE_CNTL_MGCG_OVERRIDE_MASK                                                0x00080000L
+#define GFX_ICG_PC_CLK_CTRL__LDS_OUT_MGCG_OVERRIDE_MASK                                                       0x00100000L
+#define GFX_ICG_PC_CLK_CTRL__PC_REGS_MGCG_OVERRIDE_MASK                                                       0x00200000L
+#define GFX_ICG_PC_CLK_CTRL__PC_PERFMON_MGCG_OVERRIDE_MASK                                                    0x00400000L
+//GFX_ICG_BCI_CTRL
+#define GFX_ICG_BCI_CTRL__GRP_OVERRIDES__SHIFT                                                                0x0
+#define GFX_ICG_BCI_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x10
+#define GFX_ICG_BCI_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
+#define GFX_ICG_BCI_CTRL__GRP_OVERRIDES_MASK                                                                  0x0000FFFFL
+#define GFX_ICG_BCI_CTRL__OFF_HYSTERESIS_MASK                                                                 0x003F0000L
+#define GFX_ICG_BCI_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
+//CGTT_VGT_CLK_CTRL
+#define CGTT_VGT_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_VGT_CLK_CTRL__PERF_ENABLE__SHIFT                                                                 0xf
+#define CGTT_VGT_CLK_CTRL__DBG_ENABLE__SHIFT                                                                  0x10
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_VGT_CLK_CTRL__PI1_OVERRIDE__SHIFT                                                                0x17
+#define CGTT_VGT_CLK_CTRL__PI0_OVERRIDE__SHIFT                                                                0x18
+#define CGTT_VGT_CLK_CTRL__HS_OVERRIDE__SHIFT                                                                 0x19
+#define CGTT_VGT_CLK_CTRL__TESS_OVERRIDE__SHIFT                                                               0x1c
+#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                               0x1d
+#define CGTT_VGT_CLK_CTRL__DEPRICATED_RBIU_INPUT_OVERRIDE__SHIFT                                              0x1e
+#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
+#define CGTT_VGT_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_VGT_CLK_CTRL__PERF_ENABLE_MASK                                                                   0x00008000L
+#define CGTT_VGT_CLK_CTRL__DBG_ENABLE_MASK                                                                    0x00010000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_VGT_CLK_CTRL__PI1_OVERRIDE_MASK                                                                  0x00800000L
+#define CGTT_VGT_CLK_CTRL__PI0_OVERRIDE_MASK                                                                  0x01000000L
+#define CGTT_VGT_CLK_CTRL__HS_OVERRIDE_MASK                                                                   0x02000000L
+#define CGTT_VGT_CLK_CTRL__TESS_OVERRIDE_MASK                                                                 0x10000000L
+#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE_MASK                                                                 0x20000000L
+#define CGTT_VGT_CLK_CTRL__DEPRICATED_RBIU_INPUT_OVERRIDE_MASK                                                0x40000000L
+#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
+//CGTT_GS_NGG_CLK_CTRL
+#define CGTT_GS_NGG_CLK_CTRL__ON_DELAY__SHIFT                                                                 0x0
+#define CGTT_GS_NGG_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                           0x4
+#define CGTT_GS_NGG_CLK_CTRL__PERF_ENABLE__SHIFT                                                              0xf
+#define CGTT_GS_NGG_CLK_CTRL__DBG_ENABLE__SHIFT                                                               0x10
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                     0x11
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                     0x12
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                     0x13
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                     0x14
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                     0x15
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                     0x16
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                     0x17
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                           0x18
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                           0x19
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                           0x1a
+#define CGTT_GS_NGG_CLK_CTRL__PERF_OVERRIDE__SHIFT                                                            0x1b
+#define CGTT_GS_NGG_CLK_CTRL__PRIMGEN_OVERRIDE__SHIFT                                                         0x1c
+#define CGTT_GS_NGG_CLK_CTRL__REG_OVERRIDE__SHIFT                                                             0x1f
+#define CGTT_GS_NGG_CLK_CTRL__ON_DELAY_MASK                                                                   0x0000000FL
+#define CGTT_GS_NGG_CLK_CTRL__OFF_HYSTERESIS_MASK                                                             0x00000FF0L
+#define CGTT_GS_NGG_CLK_CTRL__PERF_ENABLE_MASK                                                                0x00008000L
+#define CGTT_GS_NGG_CLK_CTRL__DBG_ENABLE_MASK                                                                 0x00010000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                       0x00020000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                       0x00040000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                       0x00080000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                       0x00100000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                       0x00200000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                       0x00400000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                       0x00800000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                             0x01000000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                             0x02000000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                             0x04000000L
+#define CGTT_GS_NGG_CLK_CTRL__PERF_OVERRIDE_MASK                                                              0x08000000L
+#define CGTT_GS_NGG_CLK_CTRL__PRIMGEN_OVERRIDE_MASK                                                           0x10000000L
+#define CGTT_GS_NGG_CLK_CTRL__REG_OVERRIDE_MASK                                                               0x80000000L
+//CGTT_PA_CLK_CTRL
+#define CGTT_PA_CLK_CTRL__PAB_CLIP_SU_PRIM_FIFO_CLK_OVERRIDE__SHIFT                                           0x0
+#define CGTT_PA_CLK_CTRL__PAB_SXIFCCG_CLK_OVERRIDE__SHIFT                                                     0x1
+#define CGTT_PA_CLK_CTRL__PAB_AG_CLK_OVERRIDE__SHIFT                                                          0x2
+#define CGTT_PA_CLK_CTRL__PAB_VE_VTE_REC_CLK_OVERRIDE__SHIFT                                                  0x3
+#define CGTT_PA_CLK_CTRL__PAB_ENGG_CLK_OVERRIDE__SHIFT                                                        0x4
+#define CGTT_PA_CLK_CTRL__PAB_CL_VTE_CLK_OVERRIDE__SHIFT                                                      0x5
+#define CGTT_PA_CLK_CTRL__PAB_AG_REG_CLK_OVERRIDE__SHIFT                                                      0x6
+#define CGTT_PA_CLK_CTRL__PAB_CL_VTE_REG_CLK_OVERRIDE__SHIFT                                                  0x7
+#define CGTT_PA_CLK_CTRL__PAB_VTE_REG_CLK_OVERRIDE__SHIFT                                                     0x8
+#define CGTT_PA_CLK_CTRL__PAB_NGG_INDEX_CLK_OVERRIDE__SHIFT                                                   0x9
+#define CGTT_PA_CLK_CTRL__PAB_NGG_CSB_CLK_OVERRIDE__SHIFT                                                     0xa
+#define CGTT_PA_CLK_CTRL__PAB_SU_CLK_OVERRIDE__SHIFT                                                          0xb
+#define CGTT_PA_CLK_CTRL__PAB_CL_CLK_OVERRIDE__SHIFT                                                          0xc
+#define CGTT_PA_CLK_CTRL__PAB_SU_CL_REG_CLK_OVERRIDE__SHIFT                                                   0xd
+#define CGTT_PA_CLK_CTRL__PAB_GLX_CLIENT_CLK_OVERRIDE__SHIFT                                                  0xe
+#define CGTT_PA_CLK_CTRL__CLIP_SU_PRIM_FIFO_CLK_OVERRIDE__SHIFT                                               0xf
+#define CGTT_PA_CLK_CTRL__SXIFCCG_CLK_OVERRIDE__SHIFT                                                         0x10
+#define CGTT_PA_CLK_CTRL__AG_CLK_OVERRIDE__SHIFT                                                              0x11
+#define CGTT_PA_CLK_CTRL__VE_VTE_REC_CLK_OVERRIDE__SHIFT                                                      0x12
+#define CGTT_PA_CLK_CTRL__ENGG_CLK_OVERRIDE__SHIFT                                                            0x13
+#define CGTT_PA_CLK_CTRL__CL_VTE_CLK_OVERRIDE__SHIFT                                                          0x14
+#define CGTT_PA_CLK_CTRL__AG_REG_CLK_OVERRIDE__SHIFT                                                          0x15
+#define CGTT_PA_CLK_CTRL__CL_VTE_REG_CLK_OVERRIDE__SHIFT                                                      0x16
+#define CGTT_PA_CLK_CTRL__VTE_REG_CLK_OVERRIDE__SHIFT                                                         0x17
+#define CGTT_PA_CLK_CTRL__NGG_INDEX_CLK_OVERRIDE__SHIFT                                                       0x18
+#define CGTT_PA_CLK_CTRL__NGG_CSB_CLK_OVERRIDE__SHIFT                                                         0x19
+#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE__SHIFT                                                              0x1a
+#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE__SHIFT                                                              0x1b
+#define CGTT_PA_CLK_CTRL__SU_CL_REG_CLK_OVERRIDE__SHIFT                                                       0x1c
+#define CGTT_PA_CLK_CTRL__GLX_CLIENT_CLK_OVERRIDE__SHIFT                                                      0x1d
+#define CGTT_PA_CLK_CTRL__PERFMON_CLK_OVERRIDE__SHIFT                                                         0x1e
+#define CGTT_PA_CLK_CTRL__DEBUG_BUS_EN__SHIFT                                                                 0x1f
+#define CGTT_PA_CLK_CTRL__PAB_CLIP_SU_PRIM_FIFO_CLK_OVERRIDE_MASK                                             0x00000001L
+#define CGTT_PA_CLK_CTRL__PAB_SXIFCCG_CLK_OVERRIDE_MASK                                                       0x00000002L
+#define CGTT_PA_CLK_CTRL__PAB_AG_CLK_OVERRIDE_MASK                                                            0x00000004L
+#define CGTT_PA_CLK_CTRL__PAB_VE_VTE_REC_CLK_OVERRIDE_MASK                                                    0x00000008L
+#define CGTT_PA_CLK_CTRL__PAB_ENGG_CLK_OVERRIDE_MASK                                                          0x00000010L
+#define CGTT_PA_CLK_CTRL__PAB_CL_VTE_CLK_OVERRIDE_MASK                                                        0x00000020L
+#define CGTT_PA_CLK_CTRL__PAB_AG_REG_CLK_OVERRIDE_MASK                                                        0x00000040L
+#define CGTT_PA_CLK_CTRL__PAB_CL_VTE_REG_CLK_OVERRIDE_MASK                                                    0x00000080L
+#define CGTT_PA_CLK_CTRL__PAB_VTE_REG_CLK_OVERRIDE_MASK                                                       0x00000100L
+#define CGTT_PA_CLK_CTRL__PAB_NGG_INDEX_CLK_OVERRIDE_MASK                                                     0x00000200L
+#define CGTT_PA_CLK_CTRL__PAB_NGG_CSB_CLK_OVERRIDE_MASK                                                       0x00000400L
+#define CGTT_PA_CLK_CTRL__PAB_SU_CLK_OVERRIDE_MASK                                                            0x00000800L
+#define CGTT_PA_CLK_CTRL__PAB_CL_CLK_OVERRIDE_MASK                                                            0x00001000L
+#define CGTT_PA_CLK_CTRL__PAB_SU_CL_REG_CLK_OVERRIDE_MASK                                                     0x00002000L
+#define CGTT_PA_CLK_CTRL__PAB_GLX_CLIENT_CLK_OVERRIDE_MASK                                                    0x00004000L
+#define CGTT_PA_CLK_CTRL__CLIP_SU_PRIM_FIFO_CLK_OVERRIDE_MASK                                                 0x00008000L
+#define CGTT_PA_CLK_CTRL__SXIFCCG_CLK_OVERRIDE_MASK                                                           0x00010000L
+#define CGTT_PA_CLK_CTRL__AG_CLK_OVERRIDE_MASK                                                                0x00020000L
+#define CGTT_PA_CLK_CTRL__VE_VTE_REC_CLK_OVERRIDE_MASK                                                        0x00040000L
+#define CGTT_PA_CLK_CTRL__ENGG_CLK_OVERRIDE_MASK                                                              0x00080000L
+#define CGTT_PA_CLK_CTRL__CL_VTE_CLK_OVERRIDE_MASK                                                            0x00100000L
+#define CGTT_PA_CLK_CTRL__AG_REG_CLK_OVERRIDE_MASK                                                            0x00200000L
+#define CGTT_PA_CLK_CTRL__CL_VTE_REG_CLK_OVERRIDE_MASK                                                        0x00400000L
+#define CGTT_PA_CLK_CTRL__VTE_REG_CLK_OVERRIDE_MASK                                                           0x00800000L
+#define CGTT_PA_CLK_CTRL__NGG_INDEX_CLK_OVERRIDE_MASK                                                         0x01000000L
+#define CGTT_PA_CLK_CTRL__NGG_CSB_CLK_OVERRIDE_MASK                                                           0x02000000L
+#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE_MASK                                                                0x04000000L
+#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE_MASK                                                                0x08000000L
+#define CGTT_PA_CLK_CTRL__SU_CL_REG_CLK_OVERRIDE_MASK                                                         0x10000000L
+#define CGTT_PA_CLK_CTRL__GLX_CLIENT_CLK_OVERRIDE_MASK                                                        0x20000000L
+#define CGTT_PA_CLK_CTRL__PERFMON_CLK_OVERRIDE_MASK                                                           0x40000000L
+#define CGTT_PA_CLK_CTRL__DEBUG_BUS_EN_MASK                                                                   0x80000000L
+//CGTT_SQ_CLK_CTRL
+#define CGTT_SQ_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define CGTT_SQ_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                         0x10
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
+#define CGTT_SQ_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                                0x1e
+#define CGTT_SQ_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
+#define CGTT_SQ_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define CGTT_SQ_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                           0x00010000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
+#define CGTT_SQ_CLK_CTRL__CORE_OVERRIDE_MASK                                                                  0x40000000L
+#define CGTT_SQ_CLK_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
+//CGTT_SQG_CLK_CTRL
+#define CGTT_SQG_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SQG_CLK_CTRL__CLUSTER_OVERRIDE__SHIFT                                                            0xc
+#define CGTT_SQG_CLK_CTRL__UTCL0_OVERRIDE__SHIFT                                                              0xd
+#define CGTT_SQG_CLK_CTRL__PM_OVERRIDE__SHIFT                                                                 0xe
+#define CGTT_SQG_CLK_CTRL__FORCE_GCR_RSP_FGCG__SHIFT                                                          0xf
+#define CGTT_SQG_CLK_CTRL__PM_PIT_PRIME_FGCG_OVERRIDE__SHIFT                                                  0x10
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_SQG_CLK_CTRL__FORCE_GL1X_CLKEN__SHIFT                                                            0x17
+#define CGTT_SQG_CLK_CTRL__FORCE_EXPALLOC_FGCG__SHIFT                                                         0x18
+#define CGTT_SQG_CLK_CTRL__FORCE_EXPGRANT_FGCG__SHIFT                                                         0x19
+#define CGTT_SQG_CLK_CTRL__FORCE_EXPREQ_FGCG__SHIFT                                                           0x1a
+#define CGTT_SQG_CLK_CTRL__FORCE_CMD_FGCG__SHIFT                                                              0x1b
+#define CGTT_SQG_CLK_CTRL__TTRACE_OVERRIDE__SHIFT                                                             0x1c
+#define CGTT_SQG_CLK_CTRL__PERFMON_OVERRIDE__SHIFT                                                            0x1d
+#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                               0x1e
+#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
+#define CGTT_SQG_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SQG_CLK_CTRL__CLUSTER_OVERRIDE_MASK                                                              0x00001000L
+#define CGTT_SQG_CLK_CTRL__UTCL0_OVERRIDE_MASK                                                                0x00002000L
+#define CGTT_SQG_CLK_CTRL__PM_OVERRIDE_MASK                                                                   0x00004000L
+#define CGTT_SQG_CLK_CTRL__FORCE_GCR_RSP_FGCG_MASK                                                            0x00008000L
+#define CGTT_SQG_CLK_CTRL__PM_PIT_PRIME_FGCG_OVERRIDE_MASK                                                    0x00010000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_SQG_CLK_CTRL__FORCE_GL1X_CLKEN_MASK                                                              0x00800000L
+#define CGTT_SQG_CLK_CTRL__FORCE_EXPALLOC_FGCG_MASK                                                           0x01000000L
+#define CGTT_SQG_CLK_CTRL__FORCE_EXPGRANT_FGCG_MASK                                                           0x02000000L
+#define CGTT_SQG_CLK_CTRL__FORCE_EXPREQ_FGCG_MASK                                                             0x04000000L
+#define CGTT_SQG_CLK_CTRL__FORCE_CMD_FGCG_MASK                                                                0x08000000L
+#define CGTT_SQG_CLK_CTRL__TTRACE_OVERRIDE_MASK                                                               0x10000000L
+#define CGTT_SQG_CLK_CTRL__PERFMON_OVERRIDE_MASK                                                              0x20000000L
+#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE_MASK                                                                 0x40000000L
+#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
+//SQ_ALU_CLK_CTRL
+#define SQ_ALU_CLK_CTRL__FORCE_WGP_ON_SA0__SHIFT                                                              0x0
+#define SQ_ALU_CLK_CTRL__FORCE_WGP_ON_SA1__SHIFT                                                              0x10
+#define SQ_ALU_CLK_CTRL__FORCE_WGP_ON_SA0_MASK                                                                0x0000FFFFL
+#define SQ_ALU_CLK_CTRL__FORCE_WGP_ON_SA1_MASK                                                                0xFFFF0000L
+//SQ_TEX_CLK_CTRL
+#define SQ_TEX_CLK_CTRL__FORCE_WGP_ON_SA0__SHIFT                                                              0x0
+#define SQ_TEX_CLK_CTRL__FORCE_WGP_ON_SA1__SHIFT                                                              0x10
+#define SQ_TEX_CLK_CTRL__FORCE_WGP_ON_SA0_MASK                                                                0x0000FFFFL
+#define SQ_TEX_CLK_CTRL__FORCE_WGP_ON_SA1_MASK                                                                0xFFFF0000L
+//SQ_LDS_CLK_CTRL
+#define SQ_LDS_CLK_CTRL__FORCE_WGP_ON_SA0__SHIFT                                                              0x0
+#define SQ_LDS_CLK_CTRL__FORCE_WGP_ON_SA1__SHIFT                                                              0x10
+#define SQ_LDS_CLK_CTRL__FORCE_WGP_ON_SA0_MASK                                                                0x0000FFFFL
+#define SQ_LDS_CLK_CTRL__FORCE_WGP_ON_SA1_MASK                                                                0xFFFF0000L
+//SQ_CLK_CTRL
+#define SQ_CLK_CTRL__SQ_SPI_MSG_FGCG_OVERRIDE__SHIFT                                                          0x2
+#define SQ_CLK_CTRL__SQ_SPI_EXPREQ_FGCG_OVERRIDE__SHIFT                                                       0x3
+#define SQ_CLK_CTRL__SQ_SX_EXPCMD_FGCG_OVERRIDE__SHIFT                                                        0x4
+#define SQ_CLK_CTRL__SQ_SQC_TTRACE_FGCG_OVERRIDE__SHIFT                                                       0x5
+#define SQ_CLK_CTRL__WCLK_OVERRIDE__SHIFT                                                                     0x6
+#define SQ_CLK_CTRL__PERFMON_OVERRIDE__SHIFT                                                                  0x7
+#define SQ_CLK_CTRL__OVERRIDE_LDS_IDX_BUSY__SHIFT                                                             0x8
+#define SQ_CLK_CTRL__OVERRIDE_LDS_DIRECT_BUSY__SHIFT                                                          0x9
+#define SQ_CLK_CTRL__WCLK_SLEEP_VMEM_OVERRIDE__SHIFT                                                          0xa
+#define SQ_CLK_CTRL__WCLK_SLEEP_EXPALLOC_OVERRIDE__SHIFT                                                      0xb
+#define SQ_CLK_CTRL__SQ_SP_CMD_FGCG_OVERRIDE__SHIFT                                                           0xc
+#define SQ_CLK_CTRL__SQ_SP_CONST_FGCG_OVERRIDE__SHIFT                                                         0xd
+#define SQ_CLK_CTRL__SQ_SP_EXP_FGCG_OVERRIDE__SHIFT                                                           0xe
+#define SQ_CLK_CTRL__SQ_SP_VMEM_FGCG_OVERRIDE__SHIFT                                                          0xf
+#define SQ_CLK_CTRL__SQ_LDS_DIRECT_FGCG_OVERRIDE__SHIFT                                                       0x10
+#define SQ_CLK_CTRL__IS_WAVECLK_IB_WCLK_OVERRIDE__SHIFT                                                       0x11
+#define SQ_CLK_CTRL__ISC_SET_FGCG_OVERRIDE__SHIFT                                                             0x12
+#define SQ_CLK_CTRL__ISC_CTRL_FGCG_OVERRIDE__SHIFT                                                            0x13
+#define SQ_CLK_CTRL__ISC_WAVE_CTRL_FGCG_OVERRIDE__SHIFT                                                       0x14
+#define SQ_CLK_CTRL__IB_IBUF_FGCG_OVERRIDE__SHIFT                                                             0x15
+#define SQ_CLK_CTRL__IB_WINFO_FGCG_OVERRIDE__SHIFT                                                            0x16
+#define SQ_CLK_CTRL__IB_MISC_FGCG_OVERRIDE__SHIFT                                                             0x17
+#define SQ_CLK_CTRL__EX_SALU_FGCG_OVERRIDE__SHIFT                                                             0x18
+#define SQ_CLK_CTRL__EX_VALU_FGCG_OVERRIDE__SHIFT                                                             0x19
+#define SQ_CLK_CTRL__EX_BRMSG_FGCG_OVERRIDE__SHIFT                                                            0x1a
+#define SQ_CLK_CTRL__SQ_SP_ICG_FGCG_OVERRIDE__SHIFT                                                           0x1b
+#define SQ_CLK_CTRL__SQ_MCA_FGCG_OVERRIDE__SHIFT                                                              0x1c
+#define SQ_CLK_CTRL__SQ_TA_VMEMCMD_FGCG_OVERRIDE__SHIFT                                                       0x1d
+#define SQ_CLK_CTRL__SQ_SPI_MSG_FGCG_OVERRIDE_MASK                                                            0x00000004L
+#define SQ_CLK_CTRL__SQ_SPI_EXPREQ_FGCG_OVERRIDE_MASK                                                         0x00000008L
+#define SQ_CLK_CTRL__SQ_SX_EXPCMD_FGCG_OVERRIDE_MASK                                                          0x00000010L
+#define SQ_CLK_CTRL__SQ_SQC_TTRACE_FGCG_OVERRIDE_MASK                                                         0x00000020L
+#define SQ_CLK_CTRL__WCLK_OVERRIDE_MASK                                                                       0x00000040L
+#define SQ_CLK_CTRL__PERFMON_OVERRIDE_MASK                                                                    0x00000080L
+#define SQ_CLK_CTRL__OVERRIDE_LDS_IDX_BUSY_MASK                                                               0x00000100L
+#define SQ_CLK_CTRL__OVERRIDE_LDS_DIRECT_BUSY_MASK                                                            0x00000200L
+#define SQ_CLK_CTRL__WCLK_SLEEP_VMEM_OVERRIDE_MASK                                                            0x00000400L
+#define SQ_CLK_CTRL__WCLK_SLEEP_EXPALLOC_OVERRIDE_MASK                                                        0x00000800L
+#define SQ_CLK_CTRL__SQ_SP_CMD_FGCG_OVERRIDE_MASK                                                             0x00001000L
+#define SQ_CLK_CTRL__SQ_SP_CONST_FGCG_OVERRIDE_MASK                                                           0x00002000L
+#define SQ_CLK_CTRL__SQ_SP_EXP_FGCG_OVERRIDE_MASK                                                             0x00004000L
+#define SQ_CLK_CTRL__SQ_SP_VMEM_FGCG_OVERRIDE_MASK                                                            0x00008000L
+#define SQ_CLK_CTRL__SQ_LDS_DIRECT_FGCG_OVERRIDE_MASK                                                         0x00010000L
+#define SQ_CLK_CTRL__IS_WAVECLK_IB_WCLK_OVERRIDE_MASK                                                         0x00020000L
+#define SQ_CLK_CTRL__ISC_SET_FGCG_OVERRIDE_MASK                                                               0x00040000L
+#define SQ_CLK_CTRL__ISC_CTRL_FGCG_OVERRIDE_MASK                                                              0x00080000L
+#define SQ_CLK_CTRL__ISC_WAVE_CTRL_FGCG_OVERRIDE_MASK                                                         0x00100000L
+#define SQ_CLK_CTRL__IB_IBUF_FGCG_OVERRIDE_MASK                                                               0x00200000L
+#define SQ_CLK_CTRL__IB_WINFO_FGCG_OVERRIDE_MASK                                                              0x00400000L
+#define SQ_CLK_CTRL__IB_MISC_FGCG_OVERRIDE_MASK                                                               0x00800000L
+#define SQ_CLK_CTRL__EX_SALU_FGCG_OVERRIDE_MASK                                                               0x01000000L
+#define SQ_CLK_CTRL__EX_VALU_FGCG_OVERRIDE_MASK                                                               0x02000000L
+#define SQ_CLK_CTRL__EX_BRMSG_FGCG_OVERRIDE_MASK                                                              0x04000000L
+#define SQ_CLK_CTRL__SQ_SP_ICG_FGCG_OVERRIDE_MASK                                                             0x08000000L
+#define SQ_CLK_CTRL__SQ_MCA_FGCG_OVERRIDE_MASK                                                                0x10000000L
+#define SQ_CLK_CTRL__SQ_TA_VMEMCMD_FGCG_OVERRIDE_MASK                                                         0x20000000L
+//ICG_SQ_CLK_CTRL
+#define ICG_SQ_CLK_CTRL__STATIC_OCLK_OVERRIDE__SHIFT                                                          0x0
+#define ICG_SQ_CLK_CTRL__BOUNDARY_DCLK_OVERRIDE__SHIFT                                                        0x1
+#define ICG_SQ_CLK_CTRL__BOUNDARY_CCLK_OVERRIDE__SHIFT                                                        0x2
+#define ICG_SQ_CLK_CTRL__BOUNDARY_RCLK_OVERRIDE__SHIFT                                                        0x3
+#define ICG_SQ_CLK_CTRL__DCLK_OVERRIDE__SHIFT                                                                 0x4
+#define ICG_SQ_CLK_CTRL__RCLK_OVERRIDE__SHIFT                                                                 0x5
+#define ICG_SQ_CLK_CTRL__PCLK_OVERRIDE__SHIFT                                                                 0x6
+#define ICG_SQ_CLK_CTRL__WCLK_OVERRIDE__SHIFT                                                                 0x7
+#define ICG_SQ_CLK_CTRL__SALU_CLK_OVERRIDE__SHIFT                                                             0x8
+#define ICG_SQ_CLK_CTRL__VALU_CLK_OVERRIDE__SHIFT                                                             0x9
+#define ICG_SQ_CLK_CTRL__VALU_SGPR_CLK_OVERRIDE__SHIFT                                                        0xa
+#define ICG_SQ_CLK_CTRL__VMEM_CLK_OVERRIDE__SHIFT                                                             0xb
+#define ICG_SQ_CLK_CTRL__VM_CLK_OVERRIDE__SHIFT                                                               0xc
+#define ICG_SQ_CLK_CTRL__TTRACE_CLK_OVERRIDE__SHIFT                                                           0xd
+#define ICG_SQ_CLK_CTRL__SQC_RET_CLK_OVERRIDE__SHIFT                                                          0xe
+#define ICG_SQ_CLK_CTRL__WAVEUPD_CLK_OVERRIDE__SHIFT                                                          0xf
+#define ICG_SQ_CLK_CTRL__WAVE_NEWDONE_CLK_OVERRIDE__SHIFT                                                     0x10
+#define ICG_SQ_CLK_CTRL__WAVE_STATE_CLK_OVERRIDE__SHIFT                                                       0x11
+#define ICG_SQ_CLK_CTRL__SFPU_CLK_OVERRIDE__SHIFT                                                             0x12
+#define ICG_SQ_CLK_CTRL__SQC_SPECIAL_OP_CLK_OVERRIDE__SHIFT                                                   0x13
+#define ICG_SQ_CLK_CTRL__WAVE_INSTBUF_CLK_OVERRIDE__SHIFT                                                     0x14
+#define ICG_SQ_CLK_CTRL__IS_WAVECLK_OVERRIDE__SHIFT                                                           0x15
+#define ICG_SQ_CLK_CTRL__SMEM_CLK_OVERRIDE__SHIFT                                                             0x16
+#define ICG_SQ_CLK_CTRL__SDST_FIFO_CLK_OVERRIDE__SHIFT                                                        0x17
+#define ICG_SQ_CLK_CTRL__SCALAR_BUF_CLK_OVERRIDE__SHIFT                                                       0x18
+#define ICG_SQ_CLK_CTRL__SALU_PIPE_CLK_OVERRIDE__SHIFT                                                        0x19
+#define ICG_SQ_CLK_CTRL__BRMSG_CLK_OVERRIDE__SHIFT                                                            0x1a
+#define ICG_SQ_CLK_CTRL__TAG_CLK_OVERRIDE__SHIFT                                                              0x1b
+#define ICG_SQ_CLK_CTRL__TAG_STATUS_CLK_OVERRIDE__SHIFT                                                       0x1c
+#define ICG_SQ_CLK_CTRL__EXP_CLK_OVERRIDE__SHIFT                                                              0x1d
+#define ICG_SQ_CLK_CTRL__VALU2_SGPR_CLK_OVERRIDE__SHIFT                                                       0x1e
+#define ICG_SQ_CLK_CTRL__STATIC_OCLK_OVERRIDE_MASK                                                            0x00000001L
+#define ICG_SQ_CLK_CTRL__BOUNDARY_DCLK_OVERRIDE_MASK                                                          0x00000002L
+#define ICG_SQ_CLK_CTRL__BOUNDARY_CCLK_OVERRIDE_MASK                                                          0x00000004L
+#define ICG_SQ_CLK_CTRL__BOUNDARY_RCLK_OVERRIDE_MASK                                                          0x00000008L
+#define ICG_SQ_CLK_CTRL__DCLK_OVERRIDE_MASK                                                                   0x00000010L
+#define ICG_SQ_CLK_CTRL__RCLK_OVERRIDE_MASK                                                                   0x00000020L
+#define ICG_SQ_CLK_CTRL__PCLK_OVERRIDE_MASK                                                                   0x00000040L
+#define ICG_SQ_CLK_CTRL__WCLK_OVERRIDE_MASK                                                                   0x00000080L
+#define ICG_SQ_CLK_CTRL__SALU_CLK_OVERRIDE_MASK                                                               0x00000100L
+#define ICG_SQ_CLK_CTRL__VALU_CLK_OVERRIDE_MASK                                                               0x00000200L
+#define ICG_SQ_CLK_CTRL__VALU_SGPR_CLK_OVERRIDE_MASK                                                          0x00000400L
+#define ICG_SQ_CLK_CTRL__VMEM_CLK_OVERRIDE_MASK                                                               0x00000800L
+#define ICG_SQ_CLK_CTRL__VM_CLK_OVERRIDE_MASK                                                                 0x00001000L
+#define ICG_SQ_CLK_CTRL__TTRACE_CLK_OVERRIDE_MASK                                                             0x00002000L
+#define ICG_SQ_CLK_CTRL__SQC_RET_CLK_OVERRIDE_MASK                                                            0x00004000L
+#define ICG_SQ_CLK_CTRL__WAVEUPD_CLK_OVERRIDE_MASK                                                            0x00008000L
+#define ICG_SQ_CLK_CTRL__WAVE_NEWDONE_CLK_OVERRIDE_MASK                                                       0x00010000L
+#define ICG_SQ_CLK_CTRL__WAVE_STATE_CLK_OVERRIDE_MASK                                                         0x00020000L
+#define ICG_SQ_CLK_CTRL__SFPU_CLK_OVERRIDE_MASK                                                               0x00040000L
+#define ICG_SQ_CLK_CTRL__SQC_SPECIAL_OP_CLK_OVERRIDE_MASK                                                     0x00080000L
+#define ICG_SQ_CLK_CTRL__WAVE_INSTBUF_CLK_OVERRIDE_MASK                                                       0x00100000L
+#define ICG_SQ_CLK_CTRL__IS_WAVECLK_OVERRIDE_MASK                                                             0x00200000L
+#define ICG_SQ_CLK_CTRL__SMEM_CLK_OVERRIDE_MASK                                                               0x00400000L
+#define ICG_SQ_CLK_CTRL__SDST_FIFO_CLK_OVERRIDE_MASK                                                          0x00800000L
+#define ICG_SQ_CLK_CTRL__SCALAR_BUF_CLK_OVERRIDE_MASK                                                         0x01000000L
+#define ICG_SQ_CLK_CTRL__SALU_PIPE_CLK_OVERRIDE_MASK                                                          0x02000000L
+#define ICG_SQ_CLK_CTRL__BRMSG_CLK_OVERRIDE_MASK                                                              0x04000000L
+#define ICG_SQ_CLK_CTRL__TAG_CLK_OVERRIDE_MASK                                                                0x08000000L
+#define ICG_SQ_CLK_CTRL__TAG_STATUS_CLK_OVERRIDE_MASK                                                         0x10000000L
+#define ICG_SQ_CLK_CTRL__EXP_CLK_OVERRIDE_MASK                                                                0x20000000L
+#define ICG_SQ_CLK_CTRL__VALU2_SGPR_CLK_OVERRIDE_MASK                                                         0x40000000L
+//ICG_SP_CLK_CTRL
+#define ICG_SP_CLK_CTRL__CLK_OVERRIDE__SHIFT                                                                  0x0
+#define ICG_SP_CLK_CTRL__CLK_OVERRIDE_MASK                                                                    0xFFFFFFFFL
+//GFX_ICG_SX_CLK_CTRL0
+#define GFX_ICG_SX_CLK_CTRL0__RESERVED__SHIFT                                                                 0x0
+#define GFX_ICG_SX_CLK_CTRL0__PERF_SOFT_OVERRIDE__SHIFT                                                       0x1e
+#define GFX_ICG_SX_CLK_CTRL0__REG_SOFT_OVERRIDE__SHIFT                                                        0x1f
+#define GFX_ICG_SX_CLK_CTRL0__RESERVED_MASK                                                                   0x3FFFFFFFL
+#define GFX_ICG_SX_CLK_CTRL0__PERF_SOFT_OVERRIDE_MASK                                                         0x40000000L
+#define GFX_ICG_SX_CLK_CTRL0__REG_SOFT_OVERRIDE_MASK                                                          0x80000000L
+//GFX_ICG_SX_CLK_CTRL1
+#define GFX_ICG_SX_CLK_CTRL1__RESERVED0__SHIFT                                                                0x0
+#define GFX_ICG_SX_CLK_CTRL1__DBG_EN__SHIFT                                                                   0x18
+#define GFX_ICG_SX_CLK_CTRL1__RESERVED1__SHIFT                                                                0x19
+#define GFX_ICG_SX_CLK_CTRL1__SX_SX_IO_SOFT_OVERRIDE__SHIFT                                                   0x1e
+#define GFX_ICG_SX_CLK_CTRL1__BDS_SOFT_OVERRIDE__SHIFT                                                        0x1f
+#define GFX_ICG_SX_CLK_CTRL1__RESERVED0_MASK                                                                  0x00FFFFFFL
+#define GFX_ICG_SX_CLK_CTRL1__DBG_EN_MASK                                                                     0x01000000L
+#define GFX_ICG_SX_CLK_CTRL1__RESERVED1_MASK                                                                  0x3E000000L
+#define GFX_ICG_SX_CLK_CTRL1__SX_SX_IO_SOFT_OVERRIDE_MASK                                                     0x40000000L
+#define GFX_ICG_SX_CLK_CTRL1__BDS_SOFT_OVERRIDE_MASK                                                          0x80000000L
+//GFX_ICG_SX_CLK_CTRL2
+#define GFX_ICG_SX_CLK_CTRL2__RESERVED0__SHIFT                                                                0x0
+#define GFX_ICG_SX_CLK_CTRL2__COL_WRITE_SOFT_OVERRIDE__SHIFT                                                  0x17
+#define GFX_ICG_SX_CLK_CTRL2__DBG_EN__SHIFT                                                                   0x18
+#define GFX_ICG_SX_CLK_CTRL2__COL_REQUESTER_SOFT_OVERRIDE__SHIFT                                              0x19
+#define GFX_ICG_SX_CLK_CTRL2__COL_EXPORT_SOFT_OVERRIDE__SHIFT                                                 0x1a
+#define GFX_ICG_SX_CLK_CTRL2__COL_BLEND_SOFT_OVERRIDE__SHIFT                                                  0x1b
+#define GFX_ICG_SX_CLK_CTRL2__COL_DBIF_SOFT_OVERRIDE__SHIFT                                                   0x1c
+#define GFX_ICG_SX_CLK_CTRL2__COL_BLEND_DOWNCONVERT_SOFT_OVERRIDE__SHIFT                                      0x1d
+#define GFX_ICG_SX_CLK_CTRL2__COL1_SOFT_OVERRIDE__SHIFT                                                       0x1e
+#define GFX_ICG_SX_CLK_CTRL2__COL0_SOFT_OVERRIDE__SHIFT                                                       0x1f
+#define GFX_ICG_SX_CLK_CTRL2__RESERVED0_MASK                                                                  0x007FFFFFL
+#define GFX_ICG_SX_CLK_CTRL2__COL_WRITE_SOFT_OVERRIDE_MASK                                                    0x00800000L
+#define GFX_ICG_SX_CLK_CTRL2__DBG_EN_MASK                                                                     0x01000000L
+#define GFX_ICG_SX_CLK_CTRL2__COL_REQUESTER_SOFT_OVERRIDE_MASK                                                0x02000000L
+#define GFX_ICG_SX_CLK_CTRL2__COL_EXPORT_SOFT_OVERRIDE_MASK                                                   0x04000000L
+#define GFX_ICG_SX_CLK_CTRL2__COL_BLEND_SOFT_OVERRIDE_MASK                                                    0x08000000L
+#define GFX_ICG_SX_CLK_CTRL2__COL_DBIF_SOFT_OVERRIDE_MASK                                                     0x10000000L
+#define GFX_ICG_SX_CLK_CTRL2__COL_BLEND_DOWNCONVERT_SOFT_OVERRIDE_MASK                                        0x20000000L
+#define GFX_ICG_SX_CLK_CTRL2__COL1_SOFT_OVERRIDE_MASK                                                         0x40000000L
+#define GFX_ICG_SX_CLK_CTRL2__COL0_SOFT_OVERRIDE_MASK                                                         0x80000000L
+//GFX_ICG_SX_CLK_CTRL3
+#define GFX_ICG_SX_CLK_CTRL3__RESERVED0__SHIFT                                                                0x0
+#define GFX_ICG_SX_CLK_CTRL3__DBG_EN__SHIFT                                                                   0x18
+#define GFX_ICG_SX_CLK_CTRL3__RESERVED1__SHIFT                                                                0x19
+#define GFX_ICG_SX_CLK_CTRL3__POS_WRITE_SOFT_OVERRIDE__SHIFT                                                  0x1c
+#define GFX_ICG_SX_CLK_CTRL3__POS_PAIF_SOFT_OVERRIDE__SHIFT                                                   0x1d
+#define GFX_ICG_SX_CLK_CTRL3__POS_EXPORT_SOFT_OVERRIDE__SHIFT                                                 0x1e
+#define GFX_ICG_SX_CLK_CTRL3__POS_SOFT_OVERRIDE__SHIFT                                                        0x1f
+#define GFX_ICG_SX_CLK_CTRL3__RESERVED0_MASK                                                                  0x00FFFFFFL
+#define GFX_ICG_SX_CLK_CTRL3__DBG_EN_MASK                                                                     0x01000000L
+#define GFX_ICG_SX_CLK_CTRL3__RESERVED1_MASK                                                                  0x0E000000L
+#define GFX_ICG_SX_CLK_CTRL3__POS_WRITE_SOFT_OVERRIDE_MASK                                                    0x10000000L
+#define GFX_ICG_SX_CLK_CTRL3__POS_PAIF_SOFT_OVERRIDE_MASK                                                     0x20000000L
+#define GFX_ICG_SX_CLK_CTRL3__POS_EXPORT_SOFT_OVERRIDE_MASK                                                   0x40000000L
+#define GFX_ICG_SX_CLK_CTRL3__POS_SOFT_OVERRIDE_MASK                                                          0x80000000L
+//GFX_ICG_SX_CLK_CTRL4
+#define GFX_ICG_SX_CLK_CTRL4__RESERVED0__SHIFT                                                                0x0
+#define GFX_ICG_SX_CLK_CTRL4__DBG_EN__SHIFT                                                                   0x18
+#define GFX_ICG_SX_CLK_CTRL4__RESERVED1__SHIFT                                                                0x19
+#define GFX_ICG_SX_CLK_CTRL4__IDX_WRITE_SOFT_OVERRIDE__SHIFT                                                  0x1c
+#define GFX_ICG_SX_CLK_CTRL4__IDX_PAIF_SOFT_OVERRIDE__SHIFT                                                   0x1d
+#define GFX_ICG_SX_CLK_CTRL4__IDX_EXPORT_SOFT_OVERRIDE__SHIFT                                                 0x1e
+#define GFX_ICG_SX_CLK_CTRL4__IDX_SOFT_OVERRIDE__SHIFT                                                        0x1f
+#define GFX_ICG_SX_CLK_CTRL4__RESERVED0_MASK                                                                  0x00FFFFFFL
+#define GFX_ICG_SX_CLK_CTRL4__DBG_EN_MASK                                                                     0x01000000L
+#define GFX_ICG_SX_CLK_CTRL4__RESERVED1_MASK                                                                  0x0E000000L
+#define GFX_ICG_SX_CLK_CTRL4__IDX_WRITE_SOFT_OVERRIDE_MASK                                                    0x10000000L
+#define GFX_ICG_SX_CLK_CTRL4__IDX_PAIF_SOFT_OVERRIDE_MASK                                                     0x20000000L
+#define GFX_ICG_SX_CLK_CTRL4__IDX_EXPORT_SOFT_OVERRIDE_MASK                                                   0x40000000L
+#define GFX_ICG_SX_CLK_CTRL4__IDX_SOFT_OVERRIDE_MASK                                                          0x80000000L
+//GFX_ICG_TA_CTRL
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE0__SHIFT                                                                0x0
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE1__SHIFT                                                                0x1
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE2__SHIFT                                                                0x2
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE3__SHIFT                                                                0x3
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE4__SHIFT                                                                0x4
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE5__SHIFT                                                                0x5
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE6__SHIFT                                                                0x6
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE7__SHIFT                                                                0x7
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE8__SHIFT                                                                0x8
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE9__SHIFT                                                                0x9
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE10__SHIFT                                                               0xa
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE11__SHIFT                                                               0xb
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE12__SHIFT                                                               0xc
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE13__SHIFT                                                               0xd
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE14__SHIFT                                                               0xe
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE15__SHIFT                                                               0xf
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE16__SHIFT                                                               0x10
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE17__SHIFT                                                               0x11
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE18__SHIFT                                                               0x12
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE19__SHIFT                                                               0x13
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE20__SHIFT                                                               0x14
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE21__SHIFT                                                               0x15
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE22__SHIFT                                                               0x16
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE23__SHIFT                                                               0x17
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE0_MASK                                                                  0x00000001L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE1_MASK                                                                  0x00000002L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE2_MASK                                                                  0x00000004L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE3_MASK                                                                  0x00000008L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE4_MASK                                                                  0x00000010L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE5_MASK                                                                  0x00000020L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE6_MASK                                                                  0x00000040L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE7_MASK                                                                  0x00000080L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE8_MASK                                                                  0x00000100L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE9_MASK                                                                  0x00000200L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE10_MASK                                                                 0x00000400L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE11_MASK                                                                 0x00000800L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE12_MASK                                                                 0x00001000L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE13_MASK                                                                 0x00002000L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE14_MASK                                                                 0x00004000L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE15_MASK                                                                 0x00008000L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE16_MASK                                                                 0x00010000L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE17_MASK                                                                 0x00020000L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE18_MASK                                                                 0x00040000L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE19_MASK                                                                 0x00080000L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE20_MASK                                                                 0x00100000L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE21_MASK                                                                 0x00200000L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE22_MASK                                                                 0x00400000L
+#define GFX_ICG_TA_CTRL__SOFT_OVERRIDE23_MASK                                                                 0x00800000L
+//GFX_ICG_TD_CTRL
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE0__SHIFT                                                                0x0
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE1__SHIFT                                                                0x1
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE2__SHIFT                                                                0x2
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE3__SHIFT                                                                0x3
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE4__SHIFT                                                                0x4
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE5__SHIFT                                                                0x5
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE6__SHIFT                                                                0x6
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE7__SHIFT                                                                0x7
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE8__SHIFT                                                                0x8
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE9__SHIFT                                                                0x9
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE10__SHIFT                                                               0xa
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE11__SHIFT                                                               0xb
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE12__SHIFT                                                               0xc
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE13__SHIFT                                                               0xd
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE14__SHIFT                                                               0xe
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE15__SHIFT                                                               0xf
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE16__SHIFT                                                               0x10
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE17__SHIFT                                                               0x11
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE18__SHIFT                                                               0x12
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE19__SHIFT                                                               0x13
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE20__SHIFT                                                               0x14
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE21__SHIFT                                                               0x15
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE22__SHIFT                                                               0x16
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE0_MASK                                                                  0x00000001L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE1_MASK                                                                  0x00000002L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE2_MASK                                                                  0x00000004L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE3_MASK                                                                  0x00000008L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE4_MASK                                                                  0x00000010L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE5_MASK                                                                  0x00000020L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE6_MASK                                                                  0x00000040L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE7_MASK                                                                  0x00000080L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE8_MASK                                                                  0x00000100L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE9_MASK                                                                  0x00000200L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE10_MASK                                                                 0x00000400L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE11_MASK                                                                 0x00000800L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE12_MASK                                                                 0x00001000L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE13_MASK                                                                 0x00002000L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE14_MASK                                                                 0x00004000L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE15_MASK                                                                 0x00008000L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE16_MASK                                                                 0x00010000L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE17_MASK                                                                 0x00020000L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE18_MASK                                                                 0x00040000L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE19_MASK                                                                 0x00080000L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE20_MASK                                                                 0x00100000L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE21_MASK                                                                 0x00200000L
+#define GFX_ICG_TD_CTRL__SOFT_OVERRIDE22_MASK                                                                 0x00400000L
+//DB_CGTT_CLK_CTRL_0
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0__SHIFT                                                             0x0
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1__SHIFT                                                             0x1
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2__SHIFT                                                             0x2
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3__SHIFT                                                             0x3
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4__SHIFT                                                             0x4
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5__SHIFT                                                             0x5
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6__SHIFT                                                             0x6
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE8__SHIFT                                                             0x8
+#define DB_CGTT_CLK_CTRL_0__RESERVED__SHIFT                                                                   0x9
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0_MASK                                                               0x00000001L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1_MASK                                                               0x00000002L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2_MASK                                                               0x00000004L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3_MASK                                                               0x00000008L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4_MASK                                                               0x00000010L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5_MASK                                                               0x00000020L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6_MASK                                                               0x00000040L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE8_MASK                                                               0x00000100L
+#define DB_CGTT_CLK_CTRL_0__RESERVED_MASK                                                                     0xFFFFFE00L
+//GFX_ICG_CB_CTRL
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE31__SHIFT                                                               0x0
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE30__SHIFT                                                               0x1
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE29__SHIFT                                                               0x2
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE28__SHIFT                                                               0x3
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE27__SHIFT                                                               0x4
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE26__SHIFT                                                               0x5
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE25__SHIFT                                                               0x6
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE24__SHIFT                                                               0x7
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE23__SHIFT                                                               0x8
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE22__SHIFT                                                               0x9
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE21__SHIFT                                                               0xa
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE19__SHIFT                                                               0xc
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE18__SHIFT                                                               0xd
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE17__SHIFT                                                               0xe
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE16__SHIFT                                                               0xf
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE15__SHIFT                                                               0x10
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE14__SHIFT                                                               0x11
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE11__SHIFT                                                               0x14
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE10__SHIFT                                                               0x15
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE9__SHIFT                                                                0x16
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE8__SHIFT                                                                0x17
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE7__SHIFT                                                                0x18
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE6__SHIFT                                                                0x19
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE4__SHIFT                                                                0x1b
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE2__SHIFT                                                                0x1d
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE1__SHIFT                                                                0x1e
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE0__SHIFT                                                                0x1f
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE31_MASK                                                                 0x00000001L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE30_MASK                                                                 0x00000002L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE29_MASK                                                                 0x00000004L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE28_MASK                                                                 0x00000008L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE27_MASK                                                                 0x00000010L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE26_MASK                                                                 0x00000020L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE25_MASK                                                                 0x00000040L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE24_MASK                                                                 0x00000080L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE23_MASK                                                                 0x00000100L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE22_MASK                                                                 0x00000200L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE21_MASK                                                                 0x00000400L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE19_MASK                                                                 0x00001000L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE18_MASK                                                                 0x00002000L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE17_MASK                                                                 0x00004000L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE16_MASK                                                                 0x00008000L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE15_MASK                                                                 0x00010000L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE14_MASK                                                                 0x00020000L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE11_MASK                                                                 0x00100000L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE10_MASK                                                                 0x00200000L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE9_MASK                                                                  0x00400000L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE8_MASK                                                                  0x00800000L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE7_MASK                                                                  0x01000000L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE6_MASK                                                                  0x02000000L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE4_MASK                                                                  0x08000000L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE2_MASK                                                                  0x20000000L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE1_MASK                                                                  0x40000000L
+#define GFX_ICG_CB_CTRL__SOFT_OVERRIDE0_MASK                                                                  0x80000000L
+//GFX_ICG_RMI_CTRL
+#define GFX_ICG_RMI_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define GFX_ICG_RMI_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define GFX_ICG_RMI_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                         0x10
+#define GFX_ICG_RMI_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define GFX_ICG_RMI_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define GFX_ICG_RMI_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define GFX_ICG_RMI_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define GFX_ICG_RMI_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define GFX_ICG_RMI_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define GFX_ICG_RMI_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
+#define GFX_ICG_RMI_CTRL__SOFT_OVERRIDE6__SHIFT                                                               0x19
+#define GFX_ICG_RMI_CTRL__SOFT_OVERRIDE5__SHIFT                                                               0x1a
+#define GFX_ICG_RMI_CTRL__SOFT_OVERRIDE4__SHIFT                                                               0x1b
+#define GFX_ICG_RMI_CTRL__SOFT_OVERRIDE3__SHIFT                                                               0x1c
+#define GFX_ICG_RMI_CTRL__SOFT_OVERRIDE2__SHIFT                                                               0x1d
+#define GFX_ICG_RMI_CTRL__SOFT_OVERRIDE1__SHIFT                                                               0x1e
+#define GFX_ICG_RMI_CTRL__SOFT_OVERRIDE0__SHIFT                                                               0x1f
+#define GFX_ICG_RMI_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define GFX_ICG_RMI_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define GFX_ICG_RMI_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                           0x00010000L
+#define GFX_ICG_RMI_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define GFX_ICG_RMI_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define GFX_ICG_RMI_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define GFX_ICG_RMI_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define GFX_ICG_RMI_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define GFX_ICG_RMI_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define GFX_ICG_RMI_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
+#define GFX_ICG_RMI_CTRL__SOFT_OVERRIDE6_MASK                                                                 0x02000000L
+#define GFX_ICG_RMI_CTRL__SOFT_OVERRIDE5_MASK                                                                 0x04000000L
+#define GFX_ICG_RMI_CTRL__SOFT_OVERRIDE4_MASK                                                                 0x08000000L
+#define GFX_ICG_RMI_CTRL__SOFT_OVERRIDE3_MASK                                                                 0x10000000L
+#define GFX_ICG_RMI_CTRL__SOFT_OVERRIDE2_MASK                                                                 0x20000000L
+#define GFX_ICG_RMI_CTRL__SOFT_OVERRIDE1_MASK                                                                 0x40000000L
+#define GFX_ICG_RMI_CTRL__SOFT_OVERRIDE0_MASK                                                                 0x80000000L
+//GFX_ICG_SE_CAC_CLK_CTRL
+#define GFX_ICG_SE_CAC_CLK_CTRL__SE_CAC_DYNAMIC_ICG_OVERRIDE__SHIFT                                           0x0
+#define GFX_ICG_SE_CAC_CLK_CTRL__SE_CAC_REG_ICG_OVERRIDE__SHIFT                                               0x1
+#define GFX_ICG_SE_CAC_CLK_CTRL__SE_CAC_STATIC_ICG_OVERRIDE__SHIFT                                            0x2
+#define GFX_ICG_SE_CAC_CLK_CTRL__FGCG_REP_OVERRIDE__SHIFT                                                     0x3
+#define GFX_ICG_SE_CAC_CLK_CTRL__SE_CAC_DYNAMIC_ICG_OVERRIDE_MASK                                             0x00000001L
+#define GFX_ICG_SE_CAC_CLK_CTRL__SE_CAC_REG_ICG_OVERRIDE_MASK                                                 0x00000002L
+#define GFX_ICG_SE_CAC_CLK_CTRL__SE_CAC_STATIC_ICG_OVERRIDE_MASK                                              0x00000004L
+#define GFX_ICG_SE_CAC_CLK_CTRL__FGCG_REP_OVERRIDE_MASK                                                       0x00000008L
+//CGTT_PH_CLK_CTRL0
+#define CGTT_PH_CLK_CTRL0__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_PH_CLK_CTRL0__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_PH_CLK_CTRL0__DEBUG_BUS_EN__SHIFT                                                                0x17
+#define CGTT_PH_CLK_CTRL0__DISABLE_DEBUG_BUS_FLOP_EN_ON_PERFMON__SHIFT                                        0x18
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_PH_CLK_CTRL0__PERFMON_CLK_OVERRIDE__SHIFT                                                        0x1e
+#define CGTT_PH_CLK_CTRL0__REG_CLK_OVERRIDE__SHIFT                                                            0x1f
+#define CGTT_PH_CLK_CTRL0__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_PH_CLK_CTRL0__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_PH_CLK_CTRL0__DEBUG_BUS_EN_MASK                                                                  0x00800000L
+#define CGTT_PH_CLK_CTRL0__DISABLE_DEBUG_BUS_FLOP_EN_ON_PERFMON_MASK                                          0x01000000L
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_PH_CLK_CTRL0__PERFMON_CLK_OVERRIDE_MASK                                                          0x40000000L
+#define CGTT_PH_CLK_CTRL0__REG_CLK_OVERRIDE_MASK                                                              0x80000000L
+//CGTT_PH_CLK_CTRL1
+#define CGTT_PH_CLK_CTRL1__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_PH_CLK_CTRL1__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE7__SHIFT                                                              0x18
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CGTT_PH_CLK_CTRL1__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_PH_CLK_CTRL1__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE7_MASK                                                                0x01000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+//CGTT_PH_CLK_CTRL2
+#define CGTT_PH_CLK_CTRL2__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_PH_CLK_CTRL2__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE7__SHIFT                                                              0x18
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CGTT_PH_CLK_CTRL2__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_PH_CLK_CTRL2__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE7_MASK                                                                0x01000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+//CGTT_PH_CLK_CTRL3
+#define CGTT_PH_CLK_CTRL3__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_PH_CLK_CTRL3__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE7__SHIFT                                                              0x18
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CGTT_PH_CLK_CTRL3__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_PH_CLK_CTRL3__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE7_MASK                                                                0x01000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+//GFX_ICG_TCP_CTRL
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_0__SHIFT                                                              0x0
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_1__SHIFT                                                              0x1
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_2__SHIFT                                                              0x2
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_3__SHIFT                                                              0x3
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_4__SHIFT                                                              0x4
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_5__SHIFT                                                              0x5
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_6__SHIFT                                                              0x6
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_7__SHIFT                                                              0x7
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_8__SHIFT                                                              0x8
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_9__SHIFT                                                              0x9
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_10__SHIFT                                                             0xa
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_11__SHIFT                                                             0xb
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_12__SHIFT                                                             0xc
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_13__SHIFT                                                             0xd
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_14__SHIFT                                                             0xe
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_15__SHIFT                                                             0xf
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_16__SHIFT                                                             0x10
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_17__SHIFT                                                             0x11
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_18__SHIFT                                                             0x12
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_19__SHIFT                                                             0x13
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_20__SHIFT                                                             0x14
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_21__SHIFT                                                             0x15
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_22__SHIFT                                                             0x16
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_23__SHIFT                                                             0x17
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_24__SHIFT                                                             0x18
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_25__SHIFT                                                             0x19
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_26__SHIFT                                                             0x1a
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_27__SHIFT                                                             0x1b
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_28__SHIFT                                                             0x1c
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_29__SHIFT                                                             0x1d
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_0_MASK                                                                0x00000001L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_1_MASK                                                                0x00000002L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_2_MASK                                                                0x00000004L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_3_MASK                                                                0x00000008L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_4_MASK                                                                0x00000010L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_5_MASK                                                                0x00000020L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_6_MASK                                                                0x00000040L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_7_MASK                                                                0x00000080L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_8_MASK                                                                0x00000100L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_9_MASK                                                                0x00000200L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_10_MASK                                                               0x00000400L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_11_MASK                                                               0x00000800L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_12_MASK                                                               0x00001000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_13_MASK                                                               0x00002000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_14_MASK                                                               0x00004000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_15_MASK                                                               0x00008000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_16_MASK                                                               0x00010000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_17_MASK                                                               0x00020000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_18_MASK                                                               0x00040000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_19_MASK                                                               0x00080000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_20_MASK                                                               0x00100000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_21_MASK                                                               0x00200000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_22_MASK                                                               0x00400000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_23_MASK                                                               0x00800000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_24_MASK                                                               0x01000000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_25_MASK                                                               0x02000000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_26_MASK                                                               0x04000000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_27_MASK                                                               0x08000000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_28_MASK                                                               0x10000000L
+#define GFX_ICG_TCP_CTRL__SOFT_OVERRIDE_29_MASK                                                               0x20000000L
+//GFX_ICG_TCP_CTRL2
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_0__SHIFT                                                             0x0
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_1__SHIFT                                                             0x1
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_2__SHIFT                                                             0x2
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_3__SHIFT                                                             0x3
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_4__SHIFT                                                             0x4
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_5__SHIFT                                                             0x5
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_6__SHIFT                                                             0x6
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_7__SHIFT                                                             0x7
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_8__SHIFT                                                             0x8
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_9__SHIFT                                                             0x9
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_10__SHIFT                                                            0xa
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_11__SHIFT                                                            0xb
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_12__SHIFT                                                            0xc
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_13__SHIFT                                                            0xd
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_14__SHIFT                                                            0xe
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_15__SHIFT                                                            0xf
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_16__SHIFT                                                            0x10
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_17__SHIFT                                                            0x11
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_18__SHIFT                                                            0x12
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_19__SHIFT                                                            0x13
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_20__SHIFT                                                            0x14
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_21__SHIFT                                                            0x15
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_22__SHIFT                                                            0x16
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_23__SHIFT                                                            0x17
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_24__SHIFT                                                            0x18
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_25__SHIFT                                                            0x19
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_26__SHIFT                                                            0x1a
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_27__SHIFT                                                            0x1b
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_28__SHIFT                                                            0x1c
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_29__SHIFT                                                            0x1d
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_30__SHIFT                                                            0x1e
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_0_MASK                                                               0x00000001L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_1_MASK                                                               0x00000002L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_2_MASK                                                               0x00000004L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_3_MASK                                                               0x00000008L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_4_MASK                                                               0x00000010L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_5_MASK                                                               0x00000020L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_6_MASK                                                               0x00000040L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_7_MASK                                                               0x00000080L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_8_MASK                                                               0x00000100L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_9_MASK                                                               0x00000200L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_10_MASK                                                              0x00000400L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_11_MASK                                                              0x00000800L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_12_MASK                                                              0x00001000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_13_MASK                                                              0x00002000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_14_MASK                                                              0x00004000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_15_MASK                                                              0x00008000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_16_MASK                                                              0x00010000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_17_MASK                                                              0x00020000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_18_MASK                                                              0x00040000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_19_MASK                                                              0x00080000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_20_MASK                                                              0x00100000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_21_MASK                                                              0x00200000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_22_MASK                                                              0x00400000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_23_MASK                                                              0x00800000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_24_MASK                                                              0x01000000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_25_MASK                                                              0x02000000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_26_MASK                                                              0x04000000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_27_MASK                                                              0x08000000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_28_MASK                                                              0x10000000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_29_MASK                                                              0x20000000L
+#define GFX_ICG_TCP_CTRL2__SOFT_OVERRIDE_30_MASK                                                              0x40000000L
+//ICG_LDS_CLK_CTRL
+#define ICG_LDS_CLK_CTRL__HARVEST_WGP_OVERRIDE__SHIFT                                                         0x0
+#define ICG_LDS_CLK_CTRL__CONFIG_REG_OVERRIDE__SHIFT                                                          0x1
+#define ICG_LDS_CLK_CTRL__TD_OVERRIDE__SHIFT                                                                  0x2
+#define ICG_LDS_CLK_CTRL__ATTR_WR_OVERRIDE__SHIFT                                                             0x3
+#define ICG_LDS_CLK_CTRL__DLOAD0_OVERRIDE__SHIFT                                                              0x4
+#define ICG_LDS_CLK_CTRL__DLOAD1_OVERRIDE__SHIFT                                                              0x5
+#define ICG_LDS_CLK_CTRL__SQ_LDS_VMEMCMD_OVERRIDE__SHIFT                                                      0x6
+#define ICG_LDS_CLK_CTRL__SP_LDS_VMEMREQ_OVERRIDE__SHIFT                                                      0x7
+#define ICG_LDS_CLK_CTRL__SPI_LDS_STALL_OVERRIDE__SHIFT                                                       0x8
+#define ICG_LDS_CLK_CTRL__IDX_INPUT_QUEUE_OVERRIDE__SHIFT                                                     0x9
+#define ICG_LDS_CLK_CTRL__IDX_INPUT_QUEUE_BVH_OVERRIDE__SHIFT                                                 0xa
+#define ICG_LDS_CLK_CTRL__IDX_SCHED_INPUT_OVERRIDE__SHIFT                                                     0xb
+#define ICG_LDS_CLK_CTRL__IDX_BANK_CONFLICT_OVERRIDE__SHIFT                                                   0xc
+#define ICG_LDS_CLK_CTRL__IDX_SCHEDULER_OVERRIDE__SHIFT                                                       0xd
+#define ICG_LDS_CLK_CTRL__IDX_SCHED_DATA_PIPE_OVERRIDE__SHIFT                                                 0xe
+#define ICG_LDS_CLK_CTRL__IDX_SCHED_PIPE_OVERRIDE__SHIFT                                                      0xf
+#define ICG_LDS_CLK_CTRL__IDX_SCHED_OUTPUT_OVERRIDE__SHIFT                                                    0x10
+#define ICG_LDS_CLK_CTRL__IDX_PIPE_OVERRIDE__SHIFT                                                            0x11
+#define ICG_LDS_CLK_CTRL__IDX_DIR_OVERRIDE__SHIFT                                                             0x12
+#define ICG_LDS_CLK_CTRL__IDX_WR_OVERRIDE__SHIFT                                                              0x13
+#define ICG_LDS_CLK_CTRL__WGP_ARB_OVERRIDE__SHIFT                                                             0x14
+#define ICG_LDS_CLK_CTRL__MEM_OVERRIDE__SHIFT                                                                 0x15
+#define ICG_LDS_CLK_CTRL__MEM_WR_OVERRIDE__SHIFT                                                              0x16
+#define ICG_LDS_CLK_CTRL__IDX_WR_ADDR_OVERRIDE__SHIFT                                                         0x17
+#define ICG_LDS_CLK_CTRL__IDX_RDRTN_OVERRIDE__SHIFT                                                           0x18
+#define ICG_LDS_CLK_CTRL__IDX_OUTPUT_ALIGNER_OVERRIDE__SHIFT                                                  0x19
+#define ICG_LDS_CLK_CTRL__DIR_OUTPUT_ALIGNER_OVERRIDE__SHIFT                                                  0x1a
+#define ICG_LDS_CLK_CTRL__LDS_SP_READ_OVERRIDE__SHIFT                                                         0x1b
+#define ICG_LDS_CLK_CTRL__LDS_SP_DONE_OVERRIDE__SHIFT                                                         0x1c
+#define ICG_LDS_CLK_CTRL__LDS_SQC_PERF_OVERRIDE__SHIFT                                                        0x1d
+#define ICG_LDS_CLK_CTRL__HARVEST_WGP_OVERRIDE_MASK                                                           0x00000001L
+#define ICG_LDS_CLK_CTRL__CONFIG_REG_OVERRIDE_MASK                                                            0x00000002L
+#define ICG_LDS_CLK_CTRL__TD_OVERRIDE_MASK                                                                    0x00000004L
+#define ICG_LDS_CLK_CTRL__ATTR_WR_OVERRIDE_MASK                                                               0x00000008L
+#define ICG_LDS_CLK_CTRL__DLOAD0_OVERRIDE_MASK                                                                0x00000010L
+#define ICG_LDS_CLK_CTRL__DLOAD1_OVERRIDE_MASK                                                                0x00000020L
+#define ICG_LDS_CLK_CTRL__SQ_LDS_VMEMCMD_OVERRIDE_MASK                                                        0x00000040L
+#define ICG_LDS_CLK_CTRL__SP_LDS_VMEMREQ_OVERRIDE_MASK                                                        0x00000080L
+#define ICG_LDS_CLK_CTRL__SPI_LDS_STALL_OVERRIDE_MASK                                                         0x00000100L
+#define ICG_LDS_CLK_CTRL__IDX_INPUT_QUEUE_OVERRIDE_MASK                                                       0x00000200L
+#define ICG_LDS_CLK_CTRL__IDX_INPUT_QUEUE_BVH_OVERRIDE_MASK                                                   0x00000400L
+#define ICG_LDS_CLK_CTRL__IDX_SCHED_INPUT_OVERRIDE_MASK                                                       0x00000800L
+#define ICG_LDS_CLK_CTRL__IDX_BANK_CONFLICT_OVERRIDE_MASK                                                     0x00001000L
+#define ICG_LDS_CLK_CTRL__IDX_SCHEDULER_OVERRIDE_MASK                                                         0x00002000L
+#define ICG_LDS_CLK_CTRL__IDX_SCHED_DATA_PIPE_OVERRIDE_MASK                                                   0x00004000L
+#define ICG_LDS_CLK_CTRL__IDX_SCHED_PIPE_OVERRIDE_MASK                                                        0x00008000L
+#define ICG_LDS_CLK_CTRL__IDX_SCHED_OUTPUT_OVERRIDE_MASK                                                      0x00010000L
+#define ICG_LDS_CLK_CTRL__IDX_PIPE_OVERRIDE_MASK                                                              0x00020000L
+#define ICG_LDS_CLK_CTRL__IDX_DIR_OVERRIDE_MASK                                                               0x00040000L
+#define ICG_LDS_CLK_CTRL__IDX_WR_OVERRIDE_MASK                                                                0x00080000L
+#define ICG_LDS_CLK_CTRL__WGP_ARB_OVERRIDE_MASK                                                               0x00100000L
+#define ICG_LDS_CLK_CTRL__MEM_OVERRIDE_MASK                                                                   0x00200000L
+#define ICG_LDS_CLK_CTRL__MEM_WR_OVERRIDE_MASK                                                                0x00400000L
+#define ICG_LDS_CLK_CTRL__IDX_WR_ADDR_OVERRIDE_MASK                                                           0x00800000L
+#define ICG_LDS_CLK_CTRL__IDX_RDRTN_OVERRIDE_MASK                                                             0x01000000L
+#define ICG_LDS_CLK_CTRL__IDX_OUTPUT_ALIGNER_OVERRIDE_MASK                                                    0x02000000L
+#define ICG_LDS_CLK_CTRL__DIR_OUTPUT_ALIGNER_OVERRIDE_MASK                                                    0x04000000L
+#define ICG_LDS_CLK_CTRL__LDS_SP_READ_OVERRIDE_MASK                                                           0x08000000L
+#define ICG_LDS_CLK_CTRL__LDS_SP_DONE_OVERRIDE_MASK                                                           0x10000000L
+#define ICG_LDS_CLK_CTRL__LDS_SQC_PERF_OVERRIDE_MASK                                                          0x20000000L
+//GFX_ICG_UTCL1_CTRL
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE0__SHIFT                                                             0x0
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x2
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x3
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x4
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x5
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE6__SHIFT                                                             0x6
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE7__SHIFT                                                             0x7
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE8__SHIFT                                                             0x8
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE9__SHIFT                                                             0x9
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE10__SHIFT                                                            0xa
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE11__SHIFT                                                            0xb
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE12__SHIFT                                                            0xc
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE13__SHIFT                                                            0xd
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE14__SHIFT                                                            0xe
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE15__SHIFT                                                            0xf
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE16__SHIFT                                                            0x10
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE17__SHIFT                                                            0x11
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE18__SHIFT                                                            0x12
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE19__SHIFT                                                            0x13
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE20__SHIFT                                                            0x14
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE21__SHIFT                                                            0x15
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE22__SHIFT                                                            0x16
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE23__SHIFT                                                            0x17
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE24__SHIFT                                                            0x18
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE25__SHIFT                                                            0x19
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE26__SHIFT                                                            0x1a
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE26_31__SHIFT                                                         0x1b
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE0_MASK                                                               0x00000001L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE1_MASK                                                               0x00000002L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE2_MASK                                                               0x00000004L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE3_MASK                                                               0x00000008L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE4_MASK                                                               0x00000010L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE5_MASK                                                               0x00000020L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE6_MASK                                                               0x00000040L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE7_MASK                                                               0x00000080L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE8_MASK                                                               0x00000100L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE9_MASK                                                               0x00000200L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE10_MASK                                                              0x00000400L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE11_MASK                                                              0x00000800L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE12_MASK                                                              0x00001000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE13_MASK                                                              0x00002000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE14_MASK                                                              0x00004000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE15_MASK                                                              0x00008000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE16_MASK                                                              0x00010000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE17_MASK                                                              0x00020000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE18_MASK                                                              0x00040000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE19_MASK                                                              0x00080000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE20_MASK                                                              0x00100000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE21_MASK                                                              0x00200000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE22_MASK                                                              0x00400000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE23_MASK                                                              0x00800000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE24_MASK                                                              0x01000000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE25_MASK                                                              0x02000000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE26_MASK                                                              0x04000000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE26_31_MASK                                                           0xF8000000L
+//GFX_ICG_GRBMH_CTRL
+#define GFX_ICG_GRBMH_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define GFX_ICG_GRBMH_CTRL__SOFT_OVERRIDE_SE__SHIFT                                                           0x10
+#define GFX_ICG_GRBMH_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                      0x1d
+#define GFX_ICG_GRBMH_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                          0x1e
+#define GFX_ICG_GRBMH_CTRL__OFF_HYSTERESIS_MASK                                                               0x000003F0L
+#define GFX_ICG_GRBMH_CTRL__SOFT_OVERRIDE_SE_MASK                                                             0x00FF0000L
+#define GFX_ICG_GRBMH_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                        0x20000000L
+#define GFX_ICG_GRBMH_CTRL__SOFT_OVERRIDE_DYN_MASK                                                            0x40000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_sc_pwrdec
+//CGTT_SC_CLK_CTRL0
+#define CGTT_SC_CLK_CTRL0__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SC_CLK_CTRL0__VRC_OVERRIDE__SHIFT                                                                0xc
+#define CGTT_SC_CLK_CTRL0__HZC_OVERRIDE__SHIFT                                                                0xd
+#define CGTT_SC_CLK_CTRL0__HSC_OVERRIDE__SHIFT                                                                0xe
+#define CGTT_SC_CLK_CTRL0__HPF_OVERRIDE__SHIFT                                                                0xf
+#define CGTT_SC_CLK_CTRL0__G2DYN_STALL_OVERRIDE__SHIFT                                                        0x13
+#define CGTT_SC_CLK_CTRL0__FEDYN_STALL_OVERRIDE__SHIFT                                                        0x15
+#define CGTT_SC_CLK_CTRL0__REG_CLK_STALL_OVERRIDE__SHIFT                                                      0x17
+#define CGTT_SC_CLK_CTRL0__GL1X_OVERRIDE__SHIFT                                                               0x19
+#define CGTT_SC_CLK_CTRL0__FEDYN_OVERRIDE__SHIFT                                                              0x1d
+#define CGTT_SC_CLK_CTRL0__PERFMON_OVERRIDE__SHIFT                                                            0x1e
+#define CGTT_SC_CLK_CTRL0__REG_CLK_OVERRIDE__SHIFT                                                            0x1f
+#define CGTT_SC_CLK_CTRL0__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SC_CLK_CTRL0__VRC_OVERRIDE_MASK                                                                  0x00001000L
+#define CGTT_SC_CLK_CTRL0__HZC_OVERRIDE_MASK                                                                  0x00002000L
+#define CGTT_SC_CLK_CTRL0__HSC_OVERRIDE_MASK                                                                  0x00004000L
+#define CGTT_SC_CLK_CTRL0__HPF_OVERRIDE_MASK                                                                  0x00008000L
+#define CGTT_SC_CLK_CTRL0__G2DYN_STALL_OVERRIDE_MASK                                                          0x00080000L
+#define CGTT_SC_CLK_CTRL0__FEDYN_STALL_OVERRIDE_MASK                                                          0x00200000L
+#define CGTT_SC_CLK_CTRL0__REG_CLK_STALL_OVERRIDE_MASK                                                        0x00800000L
+#define CGTT_SC_CLK_CTRL0__GL1X_OVERRIDE_MASK                                                                 0x02000000L
+#define CGTT_SC_CLK_CTRL0__FEDYN_OVERRIDE_MASK                                                                0x20000000L
+#define CGTT_SC_CLK_CTRL0__PERFMON_OVERRIDE_MASK                                                              0x40000000L
+#define CGTT_SC_CLK_CTRL0__REG_CLK_OVERRIDE_MASK                                                              0x80000000L
+//CGTT_SC_CLK_CTRL1
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE__SHIFT                                              0x11
+#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_STALL_OVERRIDE__SHIFT                                              0x12
+#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_STALL_OVERRIDE__SHIFT                                            0x15
+#define CGTT_SC_CLK_CTRL1__PBB_CLK_STALL_OVERRIDE__SHIFT                                                      0x16
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE__SHIFT                                                    0x19
+#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_OVERRIDE__SHIFT                                                    0x1a
+#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_OVERRIDE__SHIFT                                                 0x1c
+#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_OVERRIDE__SHIFT                                                  0x1d
+#define CGTT_SC_CLK_CTRL1__SC_DB_QP_SAMPLEMASK_OVERRIDE__SHIFT                                                0x1f
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE_MASK                                                0x00020000L
+#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_STALL_OVERRIDE_MASK                                                0x00040000L
+#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_STALL_OVERRIDE_MASK                                              0x00200000L
+#define CGTT_SC_CLK_CTRL1__PBB_CLK_STALL_OVERRIDE_MASK                                                        0x00400000L
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE_MASK                                                      0x02000000L
+#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_OVERRIDE_MASK                                                      0x04000000L
+#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_OVERRIDE_MASK                                                   0x10000000L
+#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_OVERRIDE_MASK                                                    0x20000000L
+#define CGTT_SC_CLK_CTRL1__SC_DB_QP_SAMPLEMASK_OVERRIDE_MASK                                                  0x80000000L
+//CGTT_SC_CLK_CTRL2
+#define CGTT_SC_CLK_CTRL2__DISABLE_DEBUG_BUS_FLOP_EN_ON_PERFMON__SHIFT                                        0xf
+#define CGTT_SC_CLK_CTRL2__SC_DB_HISZCA_OVERRIDE__SHIFT                                                       0x11
+#define CGTT_SC_CLK_CTRL2__SC_DB_STAGE_IN_TP_PFFB_WR_OVERRIDE__SHIFT                                          0x12
+#define CGTT_SC_CLK_CTRL2__SC_DB_QUADMASK_OVERRIDE__SHIFT                                                     0x13
+#define CGTT_SC_CLK_CTRL2__SC_DB_QUADMASK_Z_OVERRIDE__SHIFT                                                   0x14
+#define CGTT_SC_CLK_CTRL2__SC_DB_QUAD_PROC_OVERRIDE__SHIFT                                                    0x15
+#define CGTT_SC_CLK_CTRL2__SC_DB_QUAD_ACCUM_OVERRIDE__SHIFT                                                   0x16
+#define CGTT_SC_CLK_CTRL2__SC_DB_PKR_OVERRIDE__SHIFT                                                          0x18
+#define CGTT_SC_CLK_CTRL2__DB_SC_WAVE_INTF_CLK_OVERRIDE__SHIFT                                                0x1a
+#define CGTT_SC_CLK_CTRL2__SC_PKR_INTF_CLK_OVERRIDE__SHIFT                                                    0x1c
+#define CGTT_SC_CLK_CTRL2__SC_DB_INTF_CLK_OVERRIDE__SHIFT                                                     0x1d
+#define CGTT_SC_CLK_CTRL2__PA_SC_INTF_CLK_OVERRIDE__SHIFT                                                     0x1e
+#define CGTT_SC_CLK_CTRL2__DISABLE_DEBUG_BUS_FLOP_EN_ON_PERFMON_MASK                                          0x00008000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_HISZCA_OVERRIDE_MASK                                                         0x00020000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_STAGE_IN_TP_PFFB_WR_OVERRIDE_MASK                                            0x00040000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_QUADMASK_OVERRIDE_MASK                                                       0x00080000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_QUADMASK_Z_OVERRIDE_MASK                                                     0x00100000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_QUAD_PROC_OVERRIDE_MASK                                                      0x00200000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_QUAD_ACCUM_OVERRIDE_MASK                                                     0x00400000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_PKR_OVERRIDE_MASK                                                            0x01000000L
+#define CGTT_SC_CLK_CTRL2__DB_SC_WAVE_INTF_CLK_OVERRIDE_MASK                                                  0x04000000L
+#define CGTT_SC_CLK_CTRL2__SC_PKR_INTF_CLK_OVERRIDE_MASK                                                      0x10000000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_INTF_CLK_OVERRIDE_MASK                                                       0x20000000L
+#define CGTT_SC_CLK_CTRL2__PA_SC_INTF_CLK_OVERRIDE_MASK                                                       0x40000000L
+//CGTT_SC_CLK_CTRL3
+#define CGTT_SC_CLK_CTRL3__RESERVED_00__SHIFT                                                                 0x0
+#define CGTT_SC_CLK_CTRL3__RESERVED_01__SHIFT                                                                 0x1
+#define CGTT_SC_CLK_CTRL3__RESERVED_02__SHIFT                                                                 0x2
+#define CGTT_SC_CLK_CTRL3__RESERVED_03__SHIFT                                                                 0x3
+#define CGTT_SC_CLK_CTRL3__RESERVED_04__SHIFT                                                                 0x4
+#define CGTT_SC_CLK_CTRL3__RESERVED_05__SHIFT                                                                 0x5
+#define CGTT_SC_CLK_CTRL3__RESERVED_06__SHIFT                                                                 0x6
+#define CGTT_SC_CLK_CTRL3__RESERVED_07__SHIFT                                                                 0x7
+#define CGTT_SC_CLK_CTRL3__RESERVED_08__SHIFT                                                                 0x8
+#define CGTT_SC_CLK_CTRL3__RESERVED_09__SHIFT                                                                 0x9
+#define CGTT_SC_CLK_CTRL3__PBB_FRONT_CLK_STALL_OVERRIDE__SHIFT                                                0xa
+#define CGTT_SC_CLK_CTRL3__PBB_BATCHIN_CLK_STALL_OVERRIDE__SHIFT                                              0xb
+#define CGTT_SC_CLK_CTRL3__PBB_VRASTER_CLK_STALL_OVERRIDE__SHIFT                                              0xc
+#define CGTT_SC_CLK_CTRL3__PBB_VGATHER_CLK_STALL_OVERRIDE__SHIFT                                              0xd
+#define CGTT_SC_CLK_CTRL3__RESERVED_18__SHIFT                                                                 0x12
+#define CGTT_SC_CLK_CTRL3__RESERVED_19__SHIFT                                                                 0x13
+#define CGTT_SC_CLK_CTRL3__RESERVED_20__SHIFT                                                                 0x14
+#define CGTT_SC_CLK_CTRL3__RESERVED_21__SHIFT                                                                 0x15
+#define CGTT_SC_CLK_CTRL3__RESERVED_22__SHIFT                                                                 0x16
+#define CGTT_SC_CLK_CTRL3__RESERVED_23__SHIFT                                                                 0x17
+#define CGTT_SC_CLK_CTRL3__RESERVED_24__SHIFT                                                                 0x18
+#define CGTT_SC_CLK_CTRL3__RESERVED_25__SHIFT                                                                 0x19
+#define CGTT_SC_CLK_CTRL3__RESERVED_26__SHIFT                                                                 0x1a
+#define CGTT_SC_CLK_CTRL3__RESERVED_27__SHIFT                                                                 0x1b
+#define CGTT_SC_CLK_CTRL3__PBB_FRONT_CLK_OVERRIDE__SHIFT                                                      0x1c
+#define CGTT_SC_CLK_CTRL3__PBB_BATCHIN_CLK_OVERRIDE__SHIFT                                                    0x1d
+#define CGTT_SC_CLK_CTRL3__PBB_VRASTER_CLK_OVERRIDE__SHIFT                                                    0x1e
+#define CGTT_SC_CLK_CTRL3__PBB_VGATHER_CLK_OVERRIDE__SHIFT                                                    0x1f
+#define CGTT_SC_CLK_CTRL3__RESERVED_00_MASK                                                                   0x00000001L
+#define CGTT_SC_CLK_CTRL3__RESERVED_01_MASK                                                                   0x00000002L
+#define CGTT_SC_CLK_CTRL3__RESERVED_02_MASK                                                                   0x00000004L
+#define CGTT_SC_CLK_CTRL3__RESERVED_03_MASK                                                                   0x00000008L
+#define CGTT_SC_CLK_CTRL3__RESERVED_04_MASK                                                                   0x00000010L
+#define CGTT_SC_CLK_CTRL3__RESERVED_05_MASK                                                                   0x00000020L
+#define CGTT_SC_CLK_CTRL3__RESERVED_06_MASK                                                                   0x00000040L
+#define CGTT_SC_CLK_CTRL3__RESERVED_07_MASK                                                                   0x00000080L
+#define CGTT_SC_CLK_CTRL3__RESERVED_08_MASK                                                                   0x00000100L
+#define CGTT_SC_CLK_CTRL3__RESERVED_09_MASK                                                                   0x00000200L
+#define CGTT_SC_CLK_CTRL3__PBB_FRONT_CLK_STALL_OVERRIDE_MASK                                                  0x00000400L
+#define CGTT_SC_CLK_CTRL3__PBB_BATCHIN_CLK_STALL_OVERRIDE_MASK                                                0x00000800L
+#define CGTT_SC_CLK_CTRL3__PBB_VRASTER_CLK_STALL_OVERRIDE_MASK                                                0x00001000L
+#define CGTT_SC_CLK_CTRL3__PBB_VGATHER_CLK_STALL_OVERRIDE_MASK                                                0x00002000L
+#define CGTT_SC_CLK_CTRL3__RESERVED_18_MASK                                                                   0x00040000L
+#define CGTT_SC_CLK_CTRL3__RESERVED_19_MASK                                                                   0x00080000L
+#define CGTT_SC_CLK_CTRL3__RESERVED_20_MASK                                                                   0x00100000L
+#define CGTT_SC_CLK_CTRL3__RESERVED_21_MASK                                                                   0x00200000L
+#define CGTT_SC_CLK_CTRL3__RESERVED_22_MASK                                                                   0x00400000L
+#define CGTT_SC_CLK_CTRL3__RESERVED_23_MASK                                                                   0x00800000L
+#define CGTT_SC_CLK_CTRL3__RESERVED_24_MASK                                                                   0x01000000L
+#define CGTT_SC_CLK_CTRL3__RESERVED_25_MASK                                                                   0x02000000L
+#define CGTT_SC_CLK_CTRL3__RESERVED_26_MASK                                                                   0x04000000L
+#define CGTT_SC_CLK_CTRL3__RESERVED_27_MASK                                                                   0x08000000L
+#define CGTT_SC_CLK_CTRL3__PBB_FRONT_CLK_OVERRIDE_MASK                                                        0x10000000L
+#define CGTT_SC_CLK_CTRL3__PBB_BATCHIN_CLK_OVERRIDE_MASK                                                      0x20000000L
+#define CGTT_SC_CLK_CTRL3__PBB_VRASTER_CLK_OVERRIDE_MASK                                                      0x40000000L
+#define CGTT_SC_CLK_CTRL3__PBB_VGATHER_CLK_OVERRIDE_MASK                                                      0x80000000L
+//CGTT_SC_CLK_CTRL4
+#define CGTT_SC_CLK_CTRL4__PBB_VCOARSE_CLK_STALL_OVERRIDE__SHIFT                                              0x0
+#define CGTT_SC_CLK_CTRL4__PBB_VDETAIL_CLK_STALL_OVERRIDE__SHIFT                                              0x1
+#define CGTT_SC_CLK_CTRL4__PBB_HRASTER_CLK_STALL_OVERRIDE__SHIFT                                              0x2
+#define CGTT_SC_CLK_CTRL4__PBB_HCONFIG_CLK_STALL_OVERRIDE__SHIFT                                              0x3
+#define CGTT_SC_CLK_CTRL4__PBB_HGATHER_CLK_STALL_OVERRIDE__SHIFT                                              0x4
+#define CGTT_SC_CLK_CTRL4__PBB_HCOARSE_CLK_STALL_OVERRIDE__SHIFT                                              0x5
+#define CGTT_SC_CLK_CTRL4__PBB_HDETAIL_CLK_STALL_OVERRIDE__SHIFT                                              0x6
+#define CGTT_SC_CLK_CTRL4__PBB_BATCHOUT_CLK_STALL_OVERRIDE__SHIFT                                             0x8
+#define CGTT_SC_CLK_CTRL4__PBB_OUTPUT_CLK_STALL_OVERRIDE__SHIFT                                               0x9
+#define CGTT_SC_CLK_CTRL4__PBB_OUTMUX_CLK_STALL_OVERRIDE__SHIFT                                               0xa
+#define CGTT_SC_CLK_CTRL4__PBB_BREAKING_CLK_STALL_OVERRIDE__SHIFT                                             0xb
+#define CGTT_SC_CLK_CTRL4__PBB_PASSMEM_CLK_STALL_OVERRIDE__SHIFT                                              0xc
+#define CGTT_SC_CLK_CTRL4__PBB_VCOARSE_CLK_OVERRIDE__SHIFT                                                    0x13
+#define CGTT_SC_CLK_CTRL4__PBB_VDETAIL_CLK_OVERRIDE__SHIFT                                                    0x14
+#define CGTT_SC_CLK_CTRL4__PBB_HRASTER_CLK_OVERRIDE__SHIFT                                                    0x15
+#define CGTT_SC_CLK_CTRL4__PBB_HCONFIG_CLK_OVERRIDE__SHIFT                                                    0x16
+#define CGTT_SC_CLK_CTRL4__PBB_HGATHER_CLK_OVERRIDE__SHIFT                                                    0x17
+#define CGTT_SC_CLK_CTRL4__PBB_HCOARSE_CLK_OVERRIDE__SHIFT                                                    0x18
+#define CGTT_SC_CLK_CTRL4__PBB_HDETAIL_CLK_OVERRIDE__SHIFT                                                    0x19
+#define CGTT_SC_CLK_CTRL4__PBB_BATCHOUT_CLK_OVERRIDE__SHIFT                                                   0x1b
+#define CGTT_SC_CLK_CTRL4__PBB_OUTPUT_CLK_OVERRIDE__SHIFT                                                     0x1c
+#define CGTT_SC_CLK_CTRL4__PBB_OUTMUX_CLK_OVERRIDE__SHIFT                                                     0x1d
+#define CGTT_SC_CLK_CTRL4__PBB_BREAKING_CLK_OVERRIDE__SHIFT                                                   0x1e
+#define CGTT_SC_CLK_CTRL4__PBB_PASSMEM_CLK_OVERRIDE__SHIFT                                                    0x1f
+#define CGTT_SC_CLK_CTRL4__PBB_VCOARSE_CLK_STALL_OVERRIDE_MASK                                                0x00000001L
+#define CGTT_SC_CLK_CTRL4__PBB_VDETAIL_CLK_STALL_OVERRIDE_MASK                                                0x00000002L
+#define CGTT_SC_CLK_CTRL4__PBB_HRASTER_CLK_STALL_OVERRIDE_MASK                                                0x00000004L
+#define CGTT_SC_CLK_CTRL4__PBB_HCONFIG_CLK_STALL_OVERRIDE_MASK                                                0x00000008L
+#define CGTT_SC_CLK_CTRL4__PBB_HGATHER_CLK_STALL_OVERRIDE_MASK                                                0x00000010L
+#define CGTT_SC_CLK_CTRL4__PBB_HCOARSE_CLK_STALL_OVERRIDE_MASK                                                0x00000020L
+#define CGTT_SC_CLK_CTRL4__PBB_HDETAIL_CLK_STALL_OVERRIDE_MASK                                                0x00000040L
+#define CGTT_SC_CLK_CTRL4__PBB_BATCHOUT_CLK_STALL_OVERRIDE_MASK                                               0x00000100L
+#define CGTT_SC_CLK_CTRL4__PBB_OUTPUT_CLK_STALL_OVERRIDE_MASK                                                 0x00000200L
+#define CGTT_SC_CLK_CTRL4__PBB_OUTMUX_CLK_STALL_OVERRIDE_MASK                                                 0x00000400L
+#define CGTT_SC_CLK_CTRL4__PBB_BREAKING_CLK_STALL_OVERRIDE_MASK                                               0x00000800L
+#define CGTT_SC_CLK_CTRL4__PBB_PASSMEM_CLK_STALL_OVERRIDE_MASK                                                0x00001000L
+#define CGTT_SC_CLK_CTRL4__PBB_VCOARSE_CLK_OVERRIDE_MASK                                                      0x00080000L
+#define CGTT_SC_CLK_CTRL4__PBB_VDETAIL_CLK_OVERRIDE_MASK                                                      0x00100000L
+#define CGTT_SC_CLK_CTRL4__PBB_HRASTER_CLK_OVERRIDE_MASK                                                      0x00200000L
+#define CGTT_SC_CLK_CTRL4__PBB_HCONFIG_CLK_OVERRIDE_MASK                                                      0x00400000L
+#define CGTT_SC_CLK_CTRL4__PBB_HGATHER_CLK_OVERRIDE_MASK                                                      0x00800000L
+#define CGTT_SC_CLK_CTRL4__PBB_HCOARSE_CLK_OVERRIDE_MASK                                                      0x01000000L
+#define CGTT_SC_CLK_CTRL4__PBB_HDETAIL_CLK_OVERRIDE_MASK                                                      0x02000000L
+#define CGTT_SC_CLK_CTRL4__PBB_BATCHOUT_CLK_OVERRIDE_MASK                                                     0x08000000L
+#define CGTT_SC_CLK_CTRL4__PBB_OUTPUT_CLK_OVERRIDE_MASK                                                       0x10000000L
+#define CGTT_SC_CLK_CTRL4__PBB_OUTMUX_CLK_OVERRIDE_MASK                                                       0x20000000L
+#define CGTT_SC_CLK_CTRL4__PBB_BREAKING_CLK_OVERRIDE_MASK                                                     0x40000000L
+#define CGTT_SC_CLK_CTRL4__PBB_PASSMEM_CLK_OVERRIDE_MASK                                                      0x80000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_gl1_pwrdec
+//ICG_GL1C_CLK_CTRL
+#define ICG_GL1C_CLK_CTRL__GLOBAL_CLK_OVERRIDE__SHIFT                                                         0x0
+#define ICG_GL1C_CLK_CTRL__GLOBAL_NONHARVESTABLE_CLK_OVERRIDE__SHIFT                                          0x1
+#define ICG_GL1C_CLK_CTRL__REQUEST_CLK_OVERRIDE__SHIFT                                                        0x2
+#define ICG_GL1C_CLK_CTRL__VM_CLK_OVERRIDE__SHIFT                                                             0x3
+#define ICG_GL1C_CLK_CTRL__UTCL0_CLK_OVERRIDE__SHIFT                                                          0x4
+#define ICG_GL1C_CLK_CTRL__GCR_CLK_OVERRIDE__SHIFT                                                            0x5
+#define ICG_GL1C_CLK_CTRL__SRC_DATA_CLK_OVERRIDE__SHIFT                                                       0x6
+#define ICG_GL1C_CLK_CTRL__RETURN_CLK_OVERRIDE__SHIFT                                                         0x7
+#define ICG_GL1C_CLK_CTRL__GRBM_CLK_OVERRIDE__SHIFT                                                           0x8
+#define ICG_GL1C_CLK_CTRL__PERF_CLK_OVERRIDE__SHIFT                                                           0x9
+#define ICG_GL1C_CLK_CTRL__LDB_TRACKER0_CLK_OVERRIDE__SHIFT                                                   0xa
+#define ICG_GL1C_CLK_CTRL__LDB_TRACKER1_CLK_OVERRIDE__SHIFT                                                   0xb
+#define ICG_GL1C_CLK_CTRL__LDB_TRACKER2_CLK_OVERRIDE__SHIFT                                                   0xc
+#define ICG_GL1C_CLK_CTRL__LDB_TRACKER3_CLK_OVERRIDE__SHIFT                                                   0xd
+#define ICG_GL1C_CLK_CTRL__GLOBAL_CLK_OVERRIDE_MASK                                                           0x00000001L
+#define ICG_GL1C_CLK_CTRL__GLOBAL_NONHARVESTABLE_CLK_OVERRIDE_MASK                                            0x00000002L
+#define ICG_GL1C_CLK_CTRL__REQUEST_CLK_OVERRIDE_MASK                                                          0x00000004L
+#define ICG_GL1C_CLK_CTRL__VM_CLK_OVERRIDE_MASK                                                               0x00000008L
+#define ICG_GL1C_CLK_CTRL__UTCL0_CLK_OVERRIDE_MASK                                                            0x00000010L
+#define ICG_GL1C_CLK_CTRL__GCR_CLK_OVERRIDE_MASK                                                              0x00000020L
+#define ICG_GL1C_CLK_CTRL__SRC_DATA_CLK_OVERRIDE_MASK                                                         0x00000040L
+#define ICG_GL1C_CLK_CTRL__RETURN_CLK_OVERRIDE_MASK                                                           0x00000080L
+#define ICG_GL1C_CLK_CTRL__GRBM_CLK_OVERRIDE_MASK                                                             0x00000100L
+#define ICG_GL1C_CLK_CTRL__PERF_CLK_OVERRIDE_MASK                                                             0x00000200L
+#define ICG_GL1C_CLK_CTRL__LDB_TRACKER0_CLK_OVERRIDE_MASK                                                     0x00000400L
+#define ICG_GL1C_CLK_CTRL__LDB_TRACKER1_CLK_OVERRIDE_MASK                                                     0x00000800L
+#define ICG_GL1C_CLK_CTRL__LDB_TRACKER2_CLK_OVERRIDE_MASK                                                     0x00001000L
+#define ICG_GL1C_CLK_CTRL__LDB_TRACKER3_CLK_OVERRIDE_MASK                                                     0x00002000L
+//GL1I_GL1R_MGCG_OVERRIDE
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1IR_MGCG_RET_DCLK_OVERRIDE__SHIFT                                     0x0
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1IW_MGCG_RET_DCLK_OVERRIDE__SHIFT                                     0x1
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1IW_MGCG_SRC_DCLK_OVERRIDE__SHIFT                                     0x2
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1IR_MGCG_RET_DCLK_OVERRIDE_MASK                                       0x00000001L
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1IW_MGCG_RET_DCLK_OVERRIDE_MASK                                       0x00000002L
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1IW_MGCG_SRC_DCLK_OVERRIDE_MASK                                       0x00000004L
+//GL1XI_GL1XR_MGCG_OVERRIDE
+#define GL1XI_GL1XR_MGCG_OVERRIDE__GL1XA_GL1XIR_MGCG_RET_DCLK_OVERRIDE__SHIFT                                 0x0
+#define GL1XI_GL1XR_MGCG_OVERRIDE__GL1XA_GL1XIW_MGCG_RET_DCLK_OVERRIDE__SHIFT                                 0x1
+#define GL1XI_GL1XR_MGCG_OVERRIDE__GL1XA_GL1XIW_MGCG_SRC_DCLK_OVERRIDE__SHIFT                                 0x2
+#define GL1XI_GL1XR_MGCG_OVERRIDE__GL1XA_GL1XIR_MGCG_RET_DCLK_OVERRIDE_MASK                                   0x00000001L
+#define GL1XI_GL1XR_MGCG_OVERRIDE__GL1XA_GL1XIW_MGCG_RET_DCLK_OVERRIDE_MASK                                   0x00000002L
+#define GL1XI_GL1XR_MGCG_OVERRIDE__GL1XA_GL1XIW_MGCG_SRC_DCLK_OVERRIDE_MASK                                   0x00000004L
+//ICG_GL1XC_CLK_CTRL
+#define ICG_GL1XC_CLK_CTRL__GLOBAL_CLK_OVERRIDE__SHIFT                                                        0x0
+#define ICG_GL1XC_CLK_CTRL__GLOBAL_NONHARVESTABLE_CLK_OVERRIDE__SHIFT                                         0x1
+#define ICG_GL1XC_CLK_CTRL__REQUEST_CLK_OVERRIDE__SHIFT                                                       0x2
+#define ICG_GL1XC_CLK_CTRL__VM_CLK_OVERRIDE__SHIFT                                                            0x3
+#define ICG_GL1XC_CLK_CTRL__UTCL0_CLK_OVERRIDE__SHIFT                                                         0x4
+#define ICG_GL1XC_CLK_CTRL__GCR_CLK_OVERRIDE__SHIFT                                                           0x5
+#define ICG_GL1XC_CLK_CTRL__SRC_DATA_CLK_OVERRIDE__SHIFT                                                      0x6
+#define ICG_GL1XC_CLK_CTRL__RETURN_CLK_OVERRIDE__SHIFT                                                        0x7
+#define ICG_GL1XC_CLK_CTRL__GRBM_CLK_OVERRIDE__SHIFT                                                          0x8
+#define ICG_GL1XC_CLK_CTRL__PERF_CLK_OVERRIDE__SHIFT                                                          0x9
+#define ICG_GL1XC_CLK_CTRL__GLOBAL_CLK_OVERRIDE_MASK                                                          0x00000001L
+#define ICG_GL1XC_CLK_CTRL__GLOBAL_NONHARVESTABLE_CLK_OVERRIDE_MASK                                           0x00000002L
+#define ICG_GL1XC_CLK_CTRL__REQUEST_CLK_OVERRIDE_MASK                                                         0x00000004L
+#define ICG_GL1XC_CLK_CTRL__VM_CLK_OVERRIDE_MASK                                                              0x00000008L
+#define ICG_GL1XC_CLK_CTRL__UTCL0_CLK_OVERRIDE_MASK                                                           0x00000010L
+#define ICG_GL1XC_CLK_CTRL__GCR_CLK_OVERRIDE_MASK                                                             0x00000020L
+#define ICG_GL1XC_CLK_CTRL__SRC_DATA_CLK_OVERRIDE_MASK                                                        0x00000040L
+#define ICG_GL1XC_CLK_CTRL__RETURN_CLK_OVERRIDE_MASK                                                          0x00000080L
+#define ICG_GL1XC_CLK_CTRL__GRBM_CLK_OVERRIDE_MASK                                                            0x00000100L
+#define ICG_GL1XC_CLK_CTRL__PERF_CLK_OVERRIDE_MASK                                                            0x00000200L
+//ICG_GL1A_CTRL
+#define ICG_GL1A_CTRL__REG_CLK_OVERRIDE__SHIFT                                                                0x0
+#define ICG_GL1A_CTRL__REQ_CLI_CLK_OVERRIDE__SHIFT                                                            0x1
+#define ICG_GL1A_CTRL__REQ_ARB_CLK_OVERRIDE__SHIFT                                                            0x2
+#define ICG_GL1A_CTRL__RET_CLK_OVERRIDE__SHIFT                                                                0x3
+#define ICG_GL1A_CTRL__REQ_CREDIT_CLK_OVERRIDE__SHIFT                                                         0x4
+#define ICG_GL1A_CTRL__PERFMON_CLK_OVERRIDE__SHIFT                                                            0x5
+#define ICG_GL1A_CTRL__REG_CLK_OVERRIDE_MASK                                                                  0x00000001L
+#define ICG_GL1A_CTRL__REQ_CLI_CLK_OVERRIDE_MASK                                                              0x00000002L
+#define ICG_GL1A_CTRL__REQ_ARB_CLK_OVERRIDE_MASK                                                              0x00000004L
+#define ICG_GL1A_CTRL__RET_CLK_OVERRIDE_MASK                                                                  0x00000008L
+#define ICG_GL1A_CTRL__REQ_CREDIT_CLK_OVERRIDE_MASK                                                           0x00000010L
+#define ICG_GL1A_CTRL__PERFMON_CLK_OVERRIDE_MASK                                                              0x00000020L
+//ICG_GL1XA_CTRL
+#define ICG_GL1XA_CTRL__REG_CLK_OVERRIDE__SHIFT                                                               0x0
+#define ICG_GL1XA_CTRL__REQ_CLI_CLK_OVERRIDE__SHIFT                                                           0x1
+#define ICG_GL1XA_CTRL__REQ_ARB_CLK_OVERRIDE__SHIFT                                                           0x2
+#define ICG_GL1XA_CTRL__RET_CLK_OVERRIDE__SHIFT                                                               0x3
+#define ICG_GL1XA_CTRL__REQ_CREDIT_CLK_OVERRIDE__SHIFT                                                        0x4
+#define ICG_GL1XA_CTRL__PERFMON_CLK_OVERRIDE__SHIFT                                                           0x5
+#define ICG_GL1XA_CTRL__REG_CLK_OVERRIDE_MASK                                                                 0x00000001L
+#define ICG_GL1XA_CTRL__REQ_CLI_CLK_OVERRIDE_MASK                                                             0x00000002L
+#define ICG_GL1XA_CTRL__REQ_ARB_CLK_OVERRIDE_MASK                                                             0x00000004L
+#define ICG_GL1XA_CTRL__RET_CLK_OVERRIDE_MASK                                                                 0x00000008L
+#define ICG_GL1XA_CTRL__REQ_CREDIT_CLK_OVERRIDE_MASK                                                          0x00000010L
+#define ICG_GL1XA_CTRL__PERFMON_CLK_OVERRIDE_MASK                                                             0x00000020L
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_hypdec
+//GL1_PIPE_STEER_LSB
+#define GL1_PIPE_STEER_LSB__PIPE0__SHIFT                                                                      0x0
+#define GL1_PIPE_STEER_LSB__PIPE1__SHIFT                                                                      0x4
+#define GL1_PIPE_STEER_LSB__PIPE2__SHIFT                                                                      0x8
+#define GL1_PIPE_STEER_LSB__PIPE3__SHIFT                                                                      0xc
+#define GL1_PIPE_STEER_LSB__PIPE4__SHIFT                                                                      0x10
+#define GL1_PIPE_STEER_LSB__PIPE5__SHIFT                                                                      0x14
+#define GL1_PIPE_STEER_LSB__PIPE6__SHIFT                                                                      0x18
+#define GL1_PIPE_STEER_LSB__PIPE7__SHIFT                                                                      0x1c
+#define GL1_PIPE_STEER_LSB__PIPE0_MASK                                                                        0x0000000FL
+#define GL1_PIPE_STEER_LSB__PIPE1_MASK                                                                        0x000000F0L
+#define GL1_PIPE_STEER_LSB__PIPE2_MASK                                                                        0x00000F00L
+#define GL1_PIPE_STEER_LSB__PIPE3_MASK                                                                        0x0000F000L
+#define GL1_PIPE_STEER_LSB__PIPE4_MASK                                                                        0x000F0000L
+#define GL1_PIPE_STEER_LSB__PIPE5_MASK                                                                        0x00F00000L
+#define GL1_PIPE_STEER_LSB__PIPE6_MASK                                                                        0x0F000000L
+#define GL1_PIPE_STEER_LSB__PIPE7_MASK                                                                        0xF0000000L
+//GL1_PIPE_STEER_MSB
+#define GL1_PIPE_STEER_MSB__PIPE8__SHIFT                                                                      0x0
+#define GL1_PIPE_STEER_MSB__PIPE9__SHIFT                                                                      0x4
+#define GL1_PIPE_STEER_MSB__PIPE10__SHIFT                                                                     0x8
+#define GL1_PIPE_STEER_MSB__PIPE11__SHIFT                                                                     0xc
+#define GL1_PIPE_STEER_MSB__MODE__SHIFT                                                                       0x10
+#define GL1_PIPE_STEER_MSB__PIPE8_MASK                                                                        0x0000000FL
+#define GL1_PIPE_STEER_MSB__PIPE9_MASK                                                                        0x000000F0L
+#define GL1_PIPE_STEER_MSB__PIPE10_MASK                                                                       0x00000F00L
+#define GL1_PIPE_STEER_MSB__PIPE11_MASK                                                                       0x0000F000L
+#define GL1_PIPE_STEER_MSB__MODE_MASK                                                                         0x00010000L
+//GL1_HASH_CFG
+#define GL1_HASH_CFG__HASH_XOR_BIT0__SHIFT                                                                    0x0
+#define GL1_HASH_CFG__HASH_XOR_BIT1__SHIFT                                                                    0x6
+#define GL1_HASH_CFG__HASH_XOR_BIT2__SHIFT                                                                    0xc
+#define GL1_HASH_CFG__CHANNEL_FLIP__SHIFT                                                                     0x12
+#define GL1_HASH_CFG__CHANNEL_HASH_EN__SHIFT                                                                  0x13
+#define GL1_HASH_CFG__RESERVED__SHIFT                                                                         0x14
+#define GL1_HASH_CFG__HASH_XOR_BIT0_MASK                                                                      0x0000003FL
+#define GL1_HASH_CFG__HASH_XOR_BIT1_MASK                                                                      0x00000FC0L
+#define GL1_HASH_CFG__HASH_XOR_BIT2_MASK                                                                      0x0003F000L
+#define GL1_HASH_CFG__CHANNEL_FLIP_MASK                                                                       0x00040000L
+#define GL1_HASH_CFG__CHANNEL_HASH_EN_MASK                                                                    0x00080000L
+#define GL1_HASH_CFG__RESERVED_MASK                                                                           0xFFF00000L
+//GC_USER_SHADER_ARRAY_CONFIG
+#define GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT                                                     0x10
+#define GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK                                                       0xFFFF0000L
+//GC_USER_RB_BACKEND_DISABLE
+#define GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT                                                    0x4
+#define GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK                                                      0x000000F0L
+//GC_USER_RMI_REDUNDANCY
+#define GC_USER_RMI_REDUNDANCY__REPAIR_EN_IN_0__SHIFT                                                         0x1
+#define GC_USER_RMI_REDUNDANCY__REPAIR_EN_IN_1__SHIFT                                                         0x2
+#define GC_USER_RMI_REDUNDANCY__REPAIR_RMI_OVERRIDE__SHIFT                                                    0x3
+#define GC_USER_RMI_REDUNDANCY__REPAIR_ID_SWAP__SHIFT                                                         0x4
+#define GC_USER_RMI_REDUNDANCY__REPAIR_EN_IN_0_MASK                                                           0x00000002L
+#define GC_USER_RMI_REDUNDANCY__REPAIR_EN_IN_1_MASK                                                           0x00000004L
+#define GC_USER_RMI_REDUNDANCY__REPAIR_RMI_OVERRIDE_MASK                                                      0x00000008L
+#define GC_USER_RMI_REDUNDANCY__REPAIR_ID_SWAP_MASK                                                           0x00000010L
+//GC_USER_SHADER_RATE_CONFIG_1
+#define GC_USER_SHADER_RATE_CONFIG_1__SHADER_RATE__SHIFT                                                      0x1
+#define GC_USER_SHADER_RATE_CONFIG_1__SHADER_RATE_MASK                                                        0x000007FEL
+
+
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_grbmh_hypdec
+//GRBMH_WGP_SA0_REMAP_CNTL
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP0_SA0_REMAP_EN__SHIFT                                              0x0
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP1_SA0_REMAP_EN__SHIFT                                              0x1
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP2_SA0_REMAP_EN__SHIFT                                              0x2
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP3_SA0_REMAP_EN__SHIFT                                              0x3
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP4_SA0_REMAP_EN__SHIFT                                              0x4
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP5_SA0_REMAP_EN__SHIFT                                              0x5
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP6_SA0_REMAP_EN__SHIFT                                              0x6
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP7_SA0_REMAP_EN__SHIFT                                              0x7
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP8_SA0_REMAP_EN__SHIFT                                              0x8
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP9_SA0_REMAP_EN__SHIFT                                              0x9
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP10_SA0_REMAP_EN__SHIFT                                             0xa
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_SA0_REMAP_TO_SIDE__SHIFT                                              0xb
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_SA0_REMAP_TO_WGP__SHIFT                                               0xc
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP0_SA0_REMAP_EN__SHIFT                                              0x10
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP1_SA0_REMAP_EN__SHIFT                                              0x11
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP2_SA0_REMAP_EN__SHIFT                                              0x12
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP3_SA0_REMAP_EN__SHIFT                                              0x13
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP4_SA0_REMAP_EN__SHIFT                                              0x14
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP5_SA0_REMAP_EN__SHIFT                                              0x15
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP6_SA0_REMAP_EN__SHIFT                                              0x16
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP7_SA0_REMAP_EN__SHIFT                                              0x17
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP8_SA0_REMAP_EN__SHIFT                                              0x18
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP9_SA0_REMAP_EN__SHIFT                                              0x19
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP10_SA0_REMAP_EN__SHIFT                                             0x1a
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_SA0_REMAP_TO_SIDE__SHIFT                                              0x1b
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_SA0_REMAP_TO_WGP__SHIFT                                               0x1c
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP0_SA0_REMAP_EN_MASK                                                0x00000001L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP1_SA0_REMAP_EN_MASK                                                0x00000002L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP2_SA0_REMAP_EN_MASK                                                0x00000004L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP3_SA0_REMAP_EN_MASK                                                0x00000008L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP4_SA0_REMAP_EN_MASK                                                0x00000010L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP5_SA0_REMAP_EN_MASK                                                0x00000020L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP6_SA0_REMAP_EN_MASK                                                0x00000040L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP7_SA0_REMAP_EN_MASK                                                0x00000080L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP8_SA0_REMAP_EN_MASK                                                0x00000100L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP9_SA0_REMAP_EN_MASK                                                0x00000200L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_WGP10_SA0_REMAP_EN_MASK                                               0x00000400L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_SA0_REMAP_TO_SIDE_MASK                                                0x00000800L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE0_SA0_REMAP_TO_WGP_MASK                                                 0x0000F000L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP0_SA0_REMAP_EN_MASK                                                0x00010000L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP1_SA0_REMAP_EN_MASK                                                0x00020000L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP2_SA0_REMAP_EN_MASK                                                0x00040000L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP3_SA0_REMAP_EN_MASK                                                0x00080000L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP4_SA0_REMAP_EN_MASK                                                0x00100000L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP5_SA0_REMAP_EN_MASK                                                0x00200000L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP6_SA0_REMAP_EN_MASK                                                0x00400000L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP7_SA0_REMAP_EN_MASK                                                0x00800000L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP8_SA0_REMAP_EN_MASK                                                0x01000000L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP9_SA0_REMAP_EN_MASK                                                0x02000000L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_WGP10_SA0_REMAP_EN_MASK                                               0x04000000L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_SA0_REMAP_TO_SIDE_MASK                                                0x08000000L
+#define GRBMH_WGP_SA0_REMAP_CNTL__SIDE1_SA0_REMAP_TO_WGP_MASK                                                 0xF0000000L
+//GRBMH_WGP_SA1_REMAP_CNTL
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP0_SA1_REMAP_EN__SHIFT                                              0x0
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP1_SA1_REMAP_EN__SHIFT                                              0x1
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP2_SA1_REMAP_EN__SHIFT                                              0x2
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP3_SA1_REMAP_EN__SHIFT                                              0x3
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP4_SA1_REMAP_EN__SHIFT                                              0x4
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP5_SA1_REMAP_EN__SHIFT                                              0x5
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP6_SA1_REMAP_EN__SHIFT                                              0x6
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP7_SA1_REMAP_EN__SHIFT                                              0x7
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP8_SA1_REMAP_EN__SHIFT                                              0x8
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP9_SA1_REMAP_EN__SHIFT                                              0x9
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP10_SA1_REMAP_EN__SHIFT                                             0xa
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_SA1_REMAP_TO_SIDE__SHIFT                                              0xb
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_SA1_REMAP_TO_WGP__SHIFT                                               0xc
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP0_SA1_REMAP_EN__SHIFT                                              0x10
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP1_SA1_REMAP_EN__SHIFT                                              0x11
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP2_SA1_REMAP_EN__SHIFT                                              0x12
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP3_SA1_REMAP_EN__SHIFT                                              0x13
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP4_SA1_REMAP_EN__SHIFT                                              0x14
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP5_SA1_REMAP_EN__SHIFT                                              0x15
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP6_SA1_REMAP_EN__SHIFT                                              0x16
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP7_SA1_REMAP_EN__SHIFT                                              0x17
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP8_SA1_REMAP_EN__SHIFT                                              0x18
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP9_SA1_REMAP_EN__SHIFT                                              0x19
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP10_SA1_REMAP_EN__SHIFT                                             0x1a
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_SA1_REMAP_TO_SIDE__SHIFT                                              0x1b
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_SA1_REMAP_TO_WGP__SHIFT                                               0x1c
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP0_SA1_REMAP_EN_MASK                                                0x00000001L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP1_SA1_REMAP_EN_MASK                                                0x00000002L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP2_SA1_REMAP_EN_MASK                                                0x00000004L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP3_SA1_REMAP_EN_MASK                                                0x00000008L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP4_SA1_REMAP_EN_MASK                                                0x00000010L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP5_SA1_REMAP_EN_MASK                                                0x00000020L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP6_SA1_REMAP_EN_MASK                                                0x00000040L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP7_SA1_REMAP_EN_MASK                                                0x00000080L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP8_SA1_REMAP_EN_MASK                                                0x00000100L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP9_SA1_REMAP_EN_MASK                                                0x00000200L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_WGP10_SA1_REMAP_EN_MASK                                               0x00000400L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_SA1_REMAP_TO_SIDE_MASK                                                0x00000800L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE0_SA1_REMAP_TO_WGP_MASK                                                 0x0000F000L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP0_SA1_REMAP_EN_MASK                                                0x00010000L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP1_SA1_REMAP_EN_MASK                                                0x00020000L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP2_SA1_REMAP_EN_MASK                                                0x00040000L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP3_SA1_REMAP_EN_MASK                                                0x00080000L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP4_SA1_REMAP_EN_MASK                                                0x00100000L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP5_SA1_REMAP_EN_MASK                                                0x00200000L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP6_SA1_REMAP_EN_MASK                                                0x00400000L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP7_SA1_REMAP_EN_MASK                                                0x00800000L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP8_SA1_REMAP_EN_MASK                                                0x01000000L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP9_SA1_REMAP_EN_MASK                                                0x02000000L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_WGP10_SA1_REMAP_EN_MASK                                               0x04000000L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_SA1_REMAP_TO_SIDE_MASK                                                0x08000000L
+#define GRBMH_WGP_SA1_REMAP_CNTL__SIDE1_SA1_REMAP_TO_WGP_MASK                                                 0xF0000000L
+//GRBMH_RB_SA0_REMAP_CNTL
+#define GRBMH_RB_SA0_REMAP_CNTL__RB0_REMAP_EN__SHIFT                                                          0x0
+#define GRBMH_RB_SA0_REMAP_CNTL__RB0_REMAP__SHIFT                                                             0x1
+#define GRBMH_RB_SA0_REMAP_CNTL__RB1_REMAP_EN__SHIFT                                                          0x4
+#define GRBMH_RB_SA0_REMAP_CNTL__RB1_REMAP__SHIFT                                                             0x5
+#define GRBMH_RB_SA0_REMAP_CNTL__RB2_REMAP_EN__SHIFT                                                          0x8
+#define GRBMH_RB_SA0_REMAP_CNTL__RB2_REMAP__SHIFT                                                             0x9
+#define GRBMH_RB_SA0_REMAP_CNTL__RB3_REMAP_EN__SHIFT                                                          0xc
+#define GRBMH_RB_SA0_REMAP_CNTL__RB3_REMAP__SHIFT                                                             0xd
+#define GRBMH_RB_SA0_REMAP_CNTL__RB0_REMAP_EN_MASK                                                            0x00000001L
+#define GRBMH_RB_SA0_REMAP_CNTL__RB0_REMAP_MASK                                                               0x0000000EL
+#define GRBMH_RB_SA0_REMAP_CNTL__RB1_REMAP_EN_MASK                                                            0x00000010L
+#define GRBMH_RB_SA0_REMAP_CNTL__RB1_REMAP_MASK                                                               0x000000E0L
+#define GRBMH_RB_SA0_REMAP_CNTL__RB2_REMAP_EN_MASK                                                            0x00000100L
+#define GRBMH_RB_SA0_REMAP_CNTL__RB2_REMAP_MASK                                                               0x00000E00L
+#define GRBMH_RB_SA0_REMAP_CNTL__RB3_REMAP_EN_MASK                                                            0x00001000L
+#define GRBMH_RB_SA0_REMAP_CNTL__RB3_REMAP_MASK                                                               0x0000E000L
+//GRBMH_RB_SA1_REMAP_CNTL
+#define GRBMH_RB_SA1_REMAP_CNTL__RB0_REMAP_EN__SHIFT                                                          0x0
+#define GRBMH_RB_SA1_REMAP_CNTL__RB0_REMAP__SHIFT                                                             0x1
+#define GRBMH_RB_SA1_REMAP_CNTL__RB1_REMAP_EN__SHIFT                                                          0x4
+#define GRBMH_RB_SA1_REMAP_CNTL__RB1_REMAP__SHIFT                                                             0x5
+#define GRBMH_RB_SA1_REMAP_CNTL__RB2_REMAP_EN__SHIFT                                                          0x8
+#define GRBMH_RB_SA1_REMAP_CNTL__RB2_REMAP__SHIFT                                                             0x9
+#define GRBMH_RB_SA1_REMAP_CNTL__RB3_REMAP_EN__SHIFT                                                          0xc
+#define GRBMH_RB_SA1_REMAP_CNTL__RB3_REMAP__SHIFT                                                             0xd
+#define GRBMH_RB_SA1_REMAP_CNTL__RB0_REMAP_EN_MASK                                                            0x00000001L
+#define GRBMH_RB_SA1_REMAP_CNTL__RB0_REMAP_MASK                                                               0x0000000EL
+#define GRBMH_RB_SA1_REMAP_CNTL__RB1_REMAP_EN_MASK                                                            0x00000010L
+#define GRBMH_RB_SA1_REMAP_CNTL__RB1_REMAP_MASK                                                               0x000000E0L
+#define GRBMH_RB_SA1_REMAP_CNTL__RB2_REMAP_EN_MASK                                                            0x00000100L
+#define GRBMH_RB_SA1_REMAP_CNTL__RB2_REMAP_MASK                                                               0x00000E00L
+#define GRBMH_RB_SA1_REMAP_CNTL__RB3_REMAP_EN_MASK                                                            0x00001000L
+#define GRBMH_RB_SA1_REMAP_CNTL__RB3_REMAP_MASK                                                               0x0000E000L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvmsharedpfdec
+//GCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB
+#define GCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB__PHYSICAL_PAGE_NUMBER_LSB__SHIFT                             0x0
+#define GCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB__PHYSICAL_PAGE_NUMBER_LSB_MASK                               0xFFFFFFFFL
+//GCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB
+#define GCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB__PHYSICAL_PAGE_NUMBER_MSB__SHIFT                             0x0
+#define GCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB__PHYSICAL_PAGE_NUMBER_MSB_MASK                               0x000000FFL
+//GCUTCL2_ICG_CTRL
+#define GCUTCL2_ICG_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x0
+#define GCUTCL2_ICG_CTRL__DYNAMIC_CLOCK_OVERRIDE__SHIFT                                                       0x4
+#define GCUTCL2_ICG_CTRL__STATIC_CLOCK_OVERRIDE__SHIFT                                                        0x5
+#define GCUTCL2_ICG_CTRL__AON_CLOCK_OVERRIDE__SHIFT                                                           0x6
+#define GCUTCL2_ICG_CTRL__PERFMON_CLOCK_OVERRIDE__SHIFT                                                       0x7
+#define GCUTCL2_ICG_CTRL__OFF_HYSTERESIS_MASK                                                                 0x0000000FL
+#define GCUTCL2_ICG_CTRL__DYNAMIC_CLOCK_OVERRIDE_MASK                                                         0x00000010L
+#define GCUTCL2_ICG_CTRL__STATIC_CLOCK_OVERRIDE_MASK                                                          0x00000020L
+#define GCUTCL2_ICG_CTRL__AON_CLOCK_OVERRIDE_MASK                                                             0x00000040L
+#define GCUTCL2_ICG_CTRL__PERFMON_CLOCK_OVERRIDE_MASK                                                         0x00000080L
+//GCMC_SHARED_ACTIVE_FCN_ID
+#define GCMC_SHARED_ACTIVE_FCN_ID__VFID__SHIFT                                                                0x0
+#define GCMC_SHARED_ACTIVE_FCN_ID__VF__SHIFT                                                                  0x1f
+#define GCMC_SHARED_ACTIVE_FCN_ID__VFID_MASK                                                                  0x0000001FL
+#define GCMC_SHARED_ACTIVE_FCN_ID__VF_MASK                                                                    0x80000000L
+//GCUTCL2_CGTT_BUSY_CTRL
+#define GCUTCL2_CGTT_BUSY_CTRL__READ_DELAY__SHIFT                                                             0x0
+#define GCUTCL2_CGTT_BUSY_CTRL__ALWAYS_BUSY__SHIFT                                                            0x5
+#define GCUTCL2_CGTT_BUSY_CTRL__READ_DELAY_MASK                                                               0x0000001FL
+#define GCUTCL2_CGTT_BUSY_CTRL__ALWAYS_BUSY_MASK                                                              0x00000020L
+//GCUTCL2_GROUP_RET_FAULT_STATUS
+#define GCUTCL2_GROUP_RET_FAULT_STATUS__FAULT_GROUPS__SHIFT                                                   0x0
+#define GCUTCL2_GROUP_RET_FAULT_STATUS__FAULT_GROUPS_MASK                                                     0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvml2pfdec
+//GCVM_L2_CNTL
+#define GCVM_L2_CNTL__ENABLE_L2_CACHE__SHIFT                                                                  0x0
+#define GCVM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING__SHIFT                                                    0x1
+#define GCVM_L2_CNTL__L2_CACHE_PTE_ENDIAN_SWAP_MODE__SHIFT                                                    0x2
+#define GCVM_L2_CNTL__L2_CACHE_PDE_ENDIAN_SWAP_MODE__SHIFT                                                    0x4
+#define GCVM_L2_CNTL__L2_PDE0_CACHE_TAG_GENERATION_MODE__SHIFT                                                0x8
+#define GCVM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                          0x9
+#define GCVM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                         0xa
+#define GCVM_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY__SHIFT                                         0xb
+#define GCVM_L2_CNTL__L2_PDE0_CACHE_SPLIT_MODE__SHIFT                                                         0xc
+#define GCVM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT                                                          0xf
+#define GCVM_L2_CNTL__PDE_FAULT_CLASSIFICATION__SHIFT                                                         0x12
+#define GCVM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT                                                    0x13
+#define GCVM_L2_CNTL__IDENTITY_MODE_FRAGMENT_SIZE__SHIFT                                                      0x15
+#define GCVM_L2_CNTL__L2_PTE_CACHE_ADDR_MODE__SHIFT                                                           0x1a
+#define GCVM_L2_CNTL__ENABLE_L2_CACHE_MASK                                                                    0x00000001L
+#define GCVM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING_MASK                                                      0x00000002L
+#define GCVM_L2_CNTL__L2_CACHE_PTE_ENDIAN_SWAP_MODE_MASK                                                      0x0000000CL
+#define GCVM_L2_CNTL__L2_CACHE_PDE_ENDIAN_SWAP_MODE_MASK                                                      0x00000030L
+#define GCVM_L2_CNTL__L2_PDE0_CACHE_TAG_GENERATION_MODE_MASK                                                  0x00000100L
+#define GCVM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK                                            0x00000200L
+#define GCVM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK                                           0x00000400L
+#define GCVM_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY_MASK                                           0x00000800L
+#define GCVM_L2_CNTL__L2_PDE0_CACHE_SPLIT_MODE_MASK                                                           0x00007000L
+#define GCVM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE_MASK                                                            0x00038000L
+#define GCVM_L2_CNTL__PDE_FAULT_CLASSIFICATION_MASK                                                           0x00040000L
+#define GCVM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE_MASK                                                      0x00180000L
+#define GCVM_L2_CNTL__IDENTITY_MODE_FRAGMENT_SIZE_MASK                                                        0x03E00000L
+#define GCVM_L2_CNTL__L2_PTE_CACHE_ADDR_MODE_MASK                                                             0x0C000000L
+//GCVM_L2_CNTL2
+#define GCVM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS__SHIFT                                                          0x0
+#define GCVM_L2_CNTL2__INVALIDATE_L2_CACHE__SHIFT                                                             0x1
+#define GCVM_L2_CNTL2__DISABLE_INVALIDATE_PER_DOMAIN__SHIFT                                                   0x15
+#define GCVM_L2_CNTL2__DISABLE_BIGK_CACHE_OPTIMIZATION__SHIFT                                                 0x16
+#define GCVM_L2_CNTL2__L2_PTE_CACHE_VMID_MODE__SHIFT                                                          0x17
+#define GCVM_L2_CNTL2__INVALIDATE_CACHE_MODE__SHIFT                                                           0x1a
+#define GCVM_L2_CNTL2__PDE_CACHE_EFFECTIVE_SIZE__SHIFT                                                        0x1c
+#define GCVM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS_MASK                                                            0x00000001L
+#define GCVM_L2_CNTL2__INVALIDATE_L2_CACHE_MASK                                                               0x00000002L
+#define GCVM_L2_CNTL2__DISABLE_INVALIDATE_PER_DOMAIN_MASK                                                     0x00200000L
+#define GCVM_L2_CNTL2__DISABLE_BIGK_CACHE_OPTIMIZATION_MASK                                                   0x00400000L
+#define GCVM_L2_CNTL2__L2_PTE_CACHE_VMID_MODE_MASK                                                            0x03800000L
+#define GCVM_L2_CNTL2__INVALIDATE_CACHE_MODE_MASK                                                             0x0C000000L
+#define GCVM_L2_CNTL2__PDE_CACHE_EFFECTIVE_SIZE_MASK                                                          0x70000000L
+//GCVM_L2_CNTL3
+#define GCVM_L2_CNTL3__BANK_SELECT__SHIFT                                                                     0x0
+#define GCVM_L2_CNTL3__L2_CACHE_UPDATE_MODE__SHIFT                                                            0x6
+#define GCVM_L2_CNTL3__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE__SHIFT                                        0x8
+#define GCVM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                                                     0xf
+#define GCVM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY__SHIFT                                                     0x14
+#define GCVM_L2_CNTL3__L2_CACHE_4K_EFFECTIVE_SIZE__SHIFT                                                      0x15
+#define GCVM_L2_CNTL3__L2_CACHE_BIGK_EFFECTIVE_SIZE__SHIFT                                                    0x18
+#define GCVM_L2_CNTL3__L2_CACHE_4K_FORCE_MISS__SHIFT                                                          0x1c
+#define GCVM_L2_CNTL3__L2_CACHE_BIGK_FORCE_MISS__SHIFT                                                        0x1d
+#define GCVM_L2_CNTL3__PDE_CACHE_FORCE_MISS__SHIFT                                                            0x1e
+#define GCVM_L2_CNTL3__L2_CACHE_4K_ASSOCIATIVITY__SHIFT                                                       0x1f
+#define GCVM_L2_CNTL3__BANK_SELECT_MASK                                                                       0x0000003FL
+#define GCVM_L2_CNTL3__L2_CACHE_UPDATE_MODE_MASK                                                              0x000000C0L
+#define GCVM_L2_CNTL3__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE_MASK                                          0x00001F00L
+#define GCVM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                                                       0x000F8000L
+#define GCVM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK                                                       0x00100000L
+#define GCVM_L2_CNTL3__L2_CACHE_4K_EFFECTIVE_SIZE_MASK                                                        0x00E00000L
+#define GCVM_L2_CNTL3__L2_CACHE_BIGK_EFFECTIVE_SIZE_MASK                                                      0x0F000000L
+#define GCVM_L2_CNTL3__L2_CACHE_4K_FORCE_MISS_MASK                                                            0x10000000L
+#define GCVM_L2_CNTL3__L2_CACHE_BIGK_FORCE_MISS_MASK                                                          0x20000000L
+#define GCVM_L2_CNTL3__PDE_CACHE_FORCE_MISS_MASK                                                              0x40000000L
+#define GCVM_L2_CNTL3__L2_CACHE_4K_ASSOCIATIVITY_MASK                                                         0x80000000L
+//GCVM_L2_STATUS
+#define GCVM_L2_STATUS__L2_BUSY__SHIFT                                                                        0x0
+#define GCVM_L2_STATUS__CONTEXT_DOMAIN_BUSY__SHIFT                                                            0x1
+#define GCVM_L2_STATUS__FOUND_4K_PTE_CACHE_PARITY_ERRORS__SHIFT                                               0x11
+#define GCVM_L2_STATUS__FOUND_BIGK_PTE_CACHE_PARITY_ERRORS__SHIFT                                             0x12
+#define GCVM_L2_STATUS__FOUND_PDE0_CACHE_PARITY_ERRORS__SHIFT                                                 0x13
+#define GCVM_L2_STATUS__FOUND_PDE1_CACHE_PARITY_ERRORS__SHIFT                                                 0x14
+#define GCVM_L2_STATUS__FOUND_PDE2_CACHE_PARITY_ERRORS__SHIFT                                                 0x15
+#define GCVM_L2_STATUS__FOUND_PDE3_CACHE_PARITY_ERRORS__SHIFT                                                 0x16
+#define GCVM_L2_STATUS__L2_BUSY_MASK                                                                          0x00000001L
+#define GCVM_L2_STATUS__CONTEXT_DOMAIN_BUSY_MASK                                                              0x0001FFFEL
+#define GCVM_L2_STATUS__FOUND_4K_PTE_CACHE_PARITY_ERRORS_MASK                                                 0x00020000L
+#define GCVM_L2_STATUS__FOUND_BIGK_PTE_CACHE_PARITY_ERRORS_MASK                                               0x00040000L
+#define GCVM_L2_STATUS__FOUND_PDE0_CACHE_PARITY_ERRORS_MASK                                                   0x00080000L
+#define GCVM_L2_STATUS__FOUND_PDE1_CACHE_PARITY_ERRORS_MASK                                                   0x00100000L
+#define GCVM_L2_STATUS__FOUND_PDE2_CACHE_PARITY_ERRORS_MASK                                                   0x00200000L
+#define GCVM_L2_STATUS__FOUND_PDE3_CACHE_PARITY_ERRORS_MASK                                                   0x00400000L
+//GCVM_DUMMY_PAGE_FAULT_CNTL
+#define GCVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_FAULT_ENABLE__SHIFT                                            0x0
+#define GCVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_ADDRESS_LOGICAL__SHIFT                                         0x1
+#define GCVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_COMPARE_MSBS__SHIFT                                            0x2
+#define GCVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_FAULT_ENABLE_MASK                                              0x00000001L
+#define GCVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_ADDRESS_LOGICAL_MASK                                           0x00000002L
+#define GCVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_COMPARE_MSBS_MASK                                              0x000000FCL
+//GCVM_DUMMY_PAGE_FAULT_ADDR_LO32
+#define GCVM_DUMMY_PAGE_FAULT_ADDR_LO32__DUMMY_PAGE_ADDR_LO32__SHIFT                                          0x0
+#define GCVM_DUMMY_PAGE_FAULT_ADDR_LO32__DUMMY_PAGE_ADDR_LO32_MASK                                            0xFFFFFFFFL
+//GCVM_DUMMY_PAGE_FAULT_ADDR_HI32
+#define GCVM_DUMMY_PAGE_FAULT_ADDR_HI32__DUMMY_PAGE_ADDR_HI13__SHIFT                                          0x0
+#define GCVM_DUMMY_PAGE_FAULT_ADDR_HI32__DUMMY_PAGE_ADDR_HI13_MASK                                            0x00001FFFL
+//GCVM_INVALIDATE_CNTL
+#define GCVM_INVALIDATE_CNTL__PRI_REG_ALTERNATING__SHIFT                                                      0x0
+#define GCVM_INVALIDATE_CNTL__MAX_REG_OUTSTANDING__SHIFT                                                      0x8
+#define GCVM_INVALIDATE_CNTL__PRI_REG_ALTERNATING_MASK                                                        0x000000FFL
+#define GCVM_INVALIDATE_CNTL__MAX_REG_OUTSTANDING_MASK                                                        0x0000FF00L
+//GCVM_L2_PROTECTION_FAULT_CNTL_LO32
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                         0x0
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__ALLOW_SUBSEQUENT_PROTECTION_FAULT_STATUS_ADDR_UPDATES__SHIFT      0x1
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                      0x2
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                       0x3
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__PDE1_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                       0x4
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__PDE2_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                       0x5
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__PDE3_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                       0x6
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT          0x7
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__NACK_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                       0x8
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                 0x9
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                      0xa
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                       0xb
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                      0xc
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                    0xd
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                         0xe
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__OTHER_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                   0x1e
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__CRASH_ON_NO_RETRY_FAULT__SHIFT                                    0x1f
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                           0x00000001L
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__ALLOW_SUBSEQUENT_PROTECTION_FAULT_STATUS_ADDR_UPDATES_MASK        0x00000002L
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                        0x00000004L
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                         0x00000008L
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__PDE1_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                         0x00000010L
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__PDE2_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                         0x00000020L
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__PDE3_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                         0x00000040L
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT_MASK            0x00000080L
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__NACK_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                         0x00000100L
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                   0x00000200L
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                        0x00000400L
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                         0x00000800L
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                        0x00001000L
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                      0x00002000L
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                           0x3FFFC000L
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__OTHER_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                     0x40000000L
+#define GCVM_L2_PROTECTION_FAULT_CNTL_LO32__CRASH_ON_NO_RETRY_FAULT_MASK                                      0x80000000L
+//GCVM_L2_PROTECTION_FAULT_CNTL_HI32
+#define GCVM_L2_PROTECTION_FAULT_CNTL_HI32__CRASH_ON_RETRY_FAULT__SHIFT                                       0x0
+#define GCVM_L2_PROTECTION_FAULT_CNTL_HI32__CRASH_ON_RETRY_FAULT_MASK                                         0x00000001L
+//GCVM_L2_PROTECTION_FAULT_CNTL2
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__CLIENT_ID_PRT_FAULT_INTERRUPT__SHIFT                                  0x0
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__OTHER_CLIENT_ID_PRT_FAULT_INTERRUPT__SHIFT                            0x10
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE__SHIFT                                      0x11
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY__SHIFT                           0x12
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__ENABLE_RETRY_FAULT_INTERRUPT__SHIFT                                   0x13
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__CLIENT_ID_PRT_FAULT_INTERRUPT_MASK                                    0x0000FFFFL
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__OTHER_CLIENT_ID_PRT_FAULT_INTERRUPT_MASK                              0x00010000L
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_MASK                                        0x00020000L
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY_MASK                             0x00040000L
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__ENABLE_RETRY_FAULT_INTERRUPT_MASK                                     0x00080000L
+//GCVM_L2_PROTECTION_FAULT_MM_CNTL3
+#define GCVM_L2_PROTECTION_FAULT_MM_CNTL3__VML1_READ_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                0x0
+#define GCVM_L2_PROTECTION_FAULT_MM_CNTL3__VML1_READ_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                  0xFFFFFFFFL
+//GCVM_L2_PROTECTION_FAULT_MM_CNTL4
+#define GCVM_L2_PROTECTION_FAULT_MM_CNTL4__VML1_WRITE_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT               0x0
+#define GCVM_L2_PROTECTION_FAULT_MM_CNTL4__VML1_WRITE_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                 0xFFFFFFFFL
+//GCVM_L2_PROTECTION_FAULT_STATUS_LO32
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__MORE_FAULTS__SHIFT                                              0x0
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__WALKER_ERROR__SHIFT                                             0x1
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__PERMISSION_FAULTS__SHIFT                                        0x5
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__MAPPING_ERROR__SHIFT                                            0xa
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__CID__SHIFT                                                      0xb
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__RW__SHIFT                                                       0x14
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__ATOMIC__SHIFT                                                   0x15
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__VMID__SHIFT                                                     0x16
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__VF__SHIFT                                                       0x1a
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__VFID__SHIFT                                                     0x1b
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__MORE_FAULTS_MASK                                                0x00000001L
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__WALKER_ERROR_MASK                                               0x0000001EL
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__PERMISSION_FAULTS_MASK                                          0x000003E0L
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__MAPPING_ERROR_MASK                                              0x00000400L
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__CID_MASK                                                        0x000FF800L
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__RW_MASK                                                         0x00100000L
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__ATOMIC_MASK                                                     0x00200000L
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__VMID_MASK                                                       0x03C00000L
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__VF_MASK                                                         0x04000000L
+#define GCVM_L2_PROTECTION_FAULT_STATUS_LO32__VFID_MASK                                                       0xF8000000L
+//GCVM_L2_PROTECTION_FAULT_STATUS_HI32
+#define GCVM_L2_PROTECTION_FAULT_STATUS_HI32__PRT__SHIFT                                                      0x0
+#define GCVM_L2_PROTECTION_FAULT_STATUS_HI32__UCE__SHIFT                                                      0x1
+#define GCVM_L2_PROTECTION_FAULT_STATUS_HI32__FED__SHIFT                                                      0x2
+#define GCVM_L2_PROTECTION_FAULT_STATUS_HI32__PRT_MASK                                                        0x00000001L
+#define GCVM_L2_PROTECTION_FAULT_STATUS_HI32__UCE_MASK                                                        0x00000002L
+#define GCVM_L2_PROTECTION_FAULT_STATUS_HI32__FED_MASK                                                        0x00000004L
+//GCVM_L2_PROTECTION_FAULT_ADDR_LO32
+#define GCVM_L2_PROTECTION_FAULT_ADDR_LO32__LOGICAL_PAGE_ADDR_LO32__SHIFT                                     0x0
+#define GCVM_L2_PROTECTION_FAULT_ADDR_LO32__LOGICAL_PAGE_ADDR_LO32_MASK                                       0xFFFFFFFFL
+//GCVM_L2_PROTECTION_FAULT_ADDR_HI32
+#define GCVM_L2_PROTECTION_FAULT_ADDR_HI32__LOGICAL_PAGE_ADDR_HI13__SHIFT                                     0x0
+#define GCVM_L2_PROTECTION_FAULT_ADDR_HI32__LOGICAL_PAGE_ADDR_HI13_MASK                                       0x00001FFFL
+//GCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32
+#define GCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32__PHYSICAL_PAGE_ADDR_LO32__SHIFT                            0x0
+#define GCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32__PHYSICAL_PAGE_ADDR_LO32_MASK                              0xFFFFFFFFL
+//GCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32
+#define GCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32__PHYSICAL_PAGE_ADDR_HI8__SHIFT                             0x0
+#define GCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32__PHYSICAL_PAGE_ADDR_HI8_MASK                               0x000000FFL
+//GCVM_DEBUG
+#define GCVM_DEBUG__FLAGS__SHIFT                                                                              0x0
+#define GCVM_DEBUG__FLAGS_MASK                                                                                0xFFFFFFFFL
+//GCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32
+#define GCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                     0x0
+#define GCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                       0xFFFFFFFFL
+//GCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32
+#define GCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                     0x0
+#define GCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                       0x00001FFFL
+//GCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32
+#define GCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                    0x0
+#define GCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                      0xFFFFFFFFL
+//GCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32
+#define GCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                    0x0
+#define GCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                      0x00001FFFL
+//GCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32
+#define GCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32__PHYSICAL_PAGE_OFFSET_LO32__SHIFT                       0x0
+#define GCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32__PHYSICAL_PAGE_OFFSET_LO32_MASK                         0xFFFFFFFFL
+//GCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32
+#define GCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32__PHYSICAL_PAGE_OFFSET_HI8__SHIFT                        0x0
+#define GCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32__PHYSICAL_PAGE_OFFSET_HI8_MASK                          0x000000FFL
+//GCVM_L2_CNTL4
+#define GCVM_L2_CNTL4__L2_CACHE_4K_PARTITION_COUNT__SHIFT                                                     0x0
+#define GCVM_L2_CNTL4__VMC_TAP_PDE_REQUEST_PHYSICAL__SHIFT                                                    0x6
+#define GCVM_L2_CNTL4__VMC_TAP_PTE_REQUEST_PHYSICAL__SHIFT                                                    0x7
+#define GCVM_L2_CNTL4__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                         0x8
+#define GCVM_L2_CNTL4__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                        0x12
+#define GCVM_L2_CNTL4__BPM_CGCGLS_OVERRIDE__SHIFT                                                             0x1c
+#define GCVM_L2_CNTL4__GC_CH_FGCG_OFF__SHIFT                                                                  0x1d
+#define GCVM_L2_CNTL4__VFIFO_HEAD_OF_QUEUE__SHIFT                                                             0x1e
+#define GCVM_L2_CNTL4__VFIFO_VISIBLE_BANK_SILOS__SHIFT                                                        0x1f
+#define GCVM_L2_CNTL4__L2_CACHE_4K_PARTITION_COUNT_MASK                                                       0x0000003FL
+#define GCVM_L2_CNTL4__VMC_TAP_PDE_REQUEST_PHYSICAL_MASK                                                      0x00000040L
+#define GCVM_L2_CNTL4__VMC_TAP_PTE_REQUEST_PHYSICAL_MASK                                                      0x00000080L
+#define GCVM_L2_CNTL4__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                           0x0003FF00L
+#define GCVM_L2_CNTL4__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                          0x0FFC0000L
+#define GCVM_L2_CNTL4__BPM_CGCGLS_OVERRIDE_MASK                                                               0x10000000L
+#define GCVM_L2_CNTL4__GC_CH_FGCG_OFF_MASK                                                                    0x20000000L
+#define GCVM_L2_CNTL4__VFIFO_HEAD_OF_QUEUE_MASK                                                               0x40000000L
+#define GCVM_L2_CNTL4__VFIFO_VISIBLE_BANK_SILOS_MASK                                                          0x80000000L
+//GCVM_L2_MM_GROUP_RT_CLASSES
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_0_RT_CLASS__SHIFT                                                  0x0
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_1_RT_CLASS__SHIFT                                                  0x1
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_2_RT_CLASS__SHIFT                                                  0x2
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_3_RT_CLASS__SHIFT                                                  0x3
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_4_RT_CLASS__SHIFT                                                  0x4
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_5_RT_CLASS__SHIFT                                                  0x5
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_6_RT_CLASS__SHIFT                                                  0x6
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_7_RT_CLASS__SHIFT                                                  0x7
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_8_RT_CLASS__SHIFT                                                  0x8
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_9_RT_CLASS__SHIFT                                                  0x9
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_10_RT_CLASS__SHIFT                                                 0xa
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_11_RT_CLASS__SHIFT                                                 0xb
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_12_RT_CLASS__SHIFT                                                 0xc
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_13_RT_CLASS__SHIFT                                                 0xd
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_14_RT_CLASS__SHIFT                                                 0xe
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_15_RT_CLASS__SHIFT                                                 0xf
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_16_RT_CLASS__SHIFT                                                 0x10
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_17_RT_CLASS__SHIFT                                                 0x11
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_18_RT_CLASS__SHIFT                                                 0x12
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_19_RT_CLASS__SHIFT                                                 0x13
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_20_RT_CLASS__SHIFT                                                 0x14
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_21_RT_CLASS__SHIFT                                                 0x15
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_22_RT_CLASS__SHIFT                                                 0x16
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_23_RT_CLASS__SHIFT                                                 0x17
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_24_RT_CLASS__SHIFT                                                 0x18
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_25_RT_CLASS__SHIFT                                                 0x19
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_26_RT_CLASS__SHIFT                                                 0x1a
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_27_RT_CLASS__SHIFT                                                 0x1b
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_28_RT_CLASS__SHIFT                                                 0x1c
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_29_RT_CLASS__SHIFT                                                 0x1d
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_30_RT_CLASS__SHIFT                                                 0x1e
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_31_RT_CLASS__SHIFT                                                 0x1f
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_0_RT_CLASS_MASK                                                    0x00000001L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_1_RT_CLASS_MASK                                                    0x00000002L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_2_RT_CLASS_MASK                                                    0x00000004L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_3_RT_CLASS_MASK                                                    0x00000008L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_4_RT_CLASS_MASK                                                    0x00000010L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_5_RT_CLASS_MASK                                                    0x00000020L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_6_RT_CLASS_MASK                                                    0x00000040L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_7_RT_CLASS_MASK                                                    0x00000080L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_8_RT_CLASS_MASK                                                    0x00000100L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_9_RT_CLASS_MASK                                                    0x00000200L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_10_RT_CLASS_MASK                                                   0x00000400L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_11_RT_CLASS_MASK                                                   0x00000800L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_12_RT_CLASS_MASK                                                   0x00001000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_13_RT_CLASS_MASK                                                   0x00002000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_14_RT_CLASS_MASK                                                   0x00004000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_15_RT_CLASS_MASK                                                   0x00008000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_16_RT_CLASS_MASK                                                   0x00010000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_17_RT_CLASS_MASK                                                   0x00020000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_18_RT_CLASS_MASK                                                   0x00040000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_19_RT_CLASS_MASK                                                   0x00080000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_20_RT_CLASS_MASK                                                   0x00100000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_21_RT_CLASS_MASK                                                   0x00200000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_22_RT_CLASS_MASK                                                   0x00400000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_23_RT_CLASS_MASK                                                   0x00800000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_24_RT_CLASS_MASK                                                   0x01000000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_25_RT_CLASS_MASK                                                   0x02000000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_26_RT_CLASS_MASK                                                   0x04000000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_27_RT_CLASS_MASK                                                   0x08000000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_28_RT_CLASS_MASK                                                   0x10000000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_29_RT_CLASS_MASK                                                   0x20000000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_30_RT_CLASS_MASK                                                   0x40000000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_31_RT_CLASS_MASK                                                   0x80000000L
+//GCVM_L2_BANK_SELECT_RESERVED_CID
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_READ_CLIENT_ID__SHIFT                                      0x0
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_WRITE_CLIENT_ID__SHIFT                                     0xa
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__ENABLE__SHIFT                                                       0x14
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_INVALIDATION_MODE__SHIFT                             0x18
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_PRIVATE_INVALIDATION__SHIFT                          0x19
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_FRAGMENT_SIZE__SHIFT                                 0x1a
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_READ_CLIENT_ID_MASK                                        0x000001FFL
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_WRITE_CLIENT_ID_MASK                                       0x0007FC00L
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__ENABLE_MASK                                                         0x00100000L
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_INVALIDATION_MODE_MASK                               0x01000000L
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_PRIVATE_INVALIDATION_MASK                            0x02000000L
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_FRAGMENT_SIZE_MASK                                   0x7C000000L
+//GCVM_L2_BANK_SELECT_RESERVED_CID2
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_READ_CLIENT_ID__SHIFT                                     0x0
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_WRITE_CLIENT_ID__SHIFT                                    0xa
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__ENABLE__SHIFT                                                      0x14
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_INVALIDATION_MODE__SHIFT                            0x18
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_PRIVATE_INVALIDATION__SHIFT                         0x19
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_FRAGMENT_SIZE__SHIFT                                0x1a
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_READ_CLIENT_ID_MASK                                       0x000001FFL
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_WRITE_CLIENT_ID_MASK                                      0x0007FC00L
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__ENABLE_MASK                                                        0x00100000L
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_INVALIDATION_MODE_MASK                              0x01000000L
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_PRIVATE_INVALIDATION_MASK                           0x02000000L
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_FRAGMENT_SIZE_MASK                                  0x7C000000L
+//GCVM_L2_CACHE_PARITY_CNTL
+#define GCVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_4K_PTE_CACHES__SHIFT                               0x0
+#define GCVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_BIGK_PTE_CACHES__SHIFT                             0x1
+#define GCVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_PDE_CACHES__SHIFT                                  0x2
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_4K_PTE_CACHE__SHIFT                               0x3
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_BIGK_PTE_CACHE__SHIFT                             0x4
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_PDE_CACHE__SHIFT                                  0x5
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_BANK__SHIFT                                                    0x6
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_NUMBER__SHIFT                                                  0x9
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_ASSOC__SHIFT                                                   0xc
+#define GCVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_4K_PTE_CACHES_MASK                                 0x00000001L
+#define GCVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_BIGK_PTE_CACHES_MASK                               0x00000002L
+#define GCVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_PDE_CACHES_MASK                                    0x00000004L
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_4K_PTE_CACHE_MASK                                 0x00000008L
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_BIGK_PTE_CACHE_MASK                               0x00000010L
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_PDE_CACHE_MASK                                    0x00000020L
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_BANK_MASK                                                      0x000001C0L
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_NUMBER_MASK                                                    0x00000E00L
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_ASSOC_MASK                                                     0x0000F000L
+//GCVM_L2_ICG_CTRL
+#define GCVM_L2_ICG_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x0
+#define GCVM_L2_ICG_CTRL__DYNAMIC_CLOCK_OVERRIDE__SHIFT                                                       0x4
+#define GCVM_L2_ICG_CTRL__STATIC_CLOCK_OVERRIDE__SHIFT                                                        0x5
+#define GCVM_L2_ICG_CTRL__AON_CLOCK_OVERRIDE__SHIFT                                                           0x6
+#define GCVM_L2_ICG_CTRL__PERFMON_CLOCK_OVERRIDE__SHIFT                                                       0x7
+#define GCVM_L2_ICG_CTRL__OFF_HYSTERESIS_MASK                                                                 0x0000000FL
+#define GCVM_L2_ICG_CTRL__DYNAMIC_CLOCK_OVERRIDE_MASK                                                         0x00000010L
+#define GCVM_L2_ICG_CTRL__STATIC_CLOCK_OVERRIDE_MASK                                                          0x00000020L
+#define GCVM_L2_ICG_CTRL__AON_CLOCK_OVERRIDE_MASK                                                             0x00000040L
+#define GCVM_L2_ICG_CTRL__PERFMON_CLOCK_OVERRIDE_MASK                                                         0x00000080L
+//GCVM_L2_CNTL5
+#define GCVM_L2_CNTL5__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT                                                   0x0
+#define GCVM_L2_CNTL5__WALKER_PRIORITY_CLIENT_ID__SHIFT                                                       0x5
+#define GCVM_L2_CNTL5__WALKER_FETCH_PDE_NOALLOC_ENABLE__SHIFT                                                 0xe
+#define GCVM_L2_CNTL5__WALKER_FETCH_PDE_MTYPE_ENABLE__SHIFT                                                   0xf
+#define GCVM_L2_CNTL5__UTCL2_ATC_REQ_FGCG_OFF__SHIFT                                                          0x10
+#define GCVM_L2_CNTL5__UTCL2_ATC_INVREQ_REPEATER_FGCG_OFF__SHIFT                                              0x11
+#define GCVM_L2_CNTL5__UTCL2_ONE_OUTSTANDING_ATC_INVREQ__SHIFT                                                0x12
+#define GCVM_L2_CNTL5__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                                                     0x0000001FL
+#define GCVM_L2_CNTL5__WALKER_PRIORITY_CLIENT_ID_MASK                                                         0x00003FE0L
+#define GCVM_L2_CNTL5__WALKER_FETCH_PDE_NOALLOC_ENABLE_MASK                                                   0x00004000L
+#define GCVM_L2_CNTL5__WALKER_FETCH_PDE_MTYPE_ENABLE_MASK                                                     0x00008000L
+#define GCVM_L2_CNTL5__UTCL2_ATC_REQ_FGCG_OFF_MASK                                                            0x00010000L
+#define GCVM_L2_CNTL5__UTCL2_ATC_INVREQ_REPEATER_FGCG_OFF_MASK                                                0x00020000L
+#define GCVM_L2_CNTL5__UTCL2_ONE_OUTSTANDING_ATC_INVREQ_MASK                                                  0x00040000L
+//GCVM_L2_GCR_CNTL
+#define GCVM_L2_GCR_CNTL__GCR_ENABLE__SHIFT                                                                   0x0
+#define GCVM_L2_GCR_CNTL__GCR_CLIENT_ID__SHIFT                                                                0x1
+#define GCVM_L2_GCR_CNTL__GCR_ENABLE_MASK                                                                     0x00000001L
+#define GCVM_L2_GCR_CNTL__GCR_CLIENT_ID_MASK                                                                  0x000003FEL
+//GCVML2_WALKER_MACRO_THROTTLE_TIME
+#define GCVML2_WALKER_MACRO_THROTTLE_TIME__TIME__SHIFT                                                        0x0
+#define GCVML2_WALKER_MACRO_THROTTLE_TIME__TIME_MASK                                                          0x00FFFFFFL
+//GCVML2_WALKER_MACRO_THROTTLE_FETCH_LIMIT
+#define GCVML2_WALKER_MACRO_THROTTLE_FETCH_LIMIT__LIMIT__SHIFT                                                0x1
+#define GCVML2_WALKER_MACRO_THROTTLE_FETCH_LIMIT__LIMIT_MASK                                                  0x0000FFFEL
+//GCVML2_WALKER_MICRO_THROTTLE_TIME
+#define GCVML2_WALKER_MICRO_THROTTLE_TIME__TIME__SHIFT                                                        0x0
+#define GCVML2_WALKER_MICRO_THROTTLE_TIME__TIME_MASK                                                          0x00FFFFFFL
+//GCVML2_WALKER_MICRO_THROTTLE_FETCH_LIMIT
+#define GCVML2_WALKER_MICRO_THROTTLE_FETCH_LIMIT__LIMIT__SHIFT                                                0x1
+#define GCVML2_WALKER_MICRO_THROTTLE_FETCH_LIMIT__LIMIT_MASK                                                  0x0000FFFEL
+//GCVM_L2_CGTT_BUSY_CTRL
+#define GCVM_L2_CGTT_BUSY_CTRL__READ_DELAY__SHIFT                                                             0x0
+#define GCVM_L2_CGTT_BUSY_CTRL__ALWAYS_BUSY__SHIFT                                                            0x5
+#define GCVM_L2_CGTT_BUSY_CTRL__READ_DELAY_MASK                                                               0x0000001FL
+#define GCVM_L2_CGTT_BUSY_CTRL__ALWAYS_BUSY_MASK                                                              0x00000020L
+//GCVM_L2_PTE_CACHE_DUMP_CNTL
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__ENABLE__SHIFT                                                            0x0
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__READY__SHIFT                                                             0x1
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__BANK__SHIFT                                                              0x4
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__CACHE__SHIFT                                                             0x8
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__ASSOC__SHIFT                                                             0xc
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__INDEX__SHIFT                                                             0x10
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__ENABLE_MASK                                                              0x00000001L
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__READY_MASK                                                               0x00000002L
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__BANK_MASK                                                                0x000000F0L
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__CACHE_MASK                                                               0x00000F00L
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__ASSOC_MASK                                                               0x0000F000L
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__INDEX_MASK                                                               0xFFFF0000L
+//GCVM_L2_PTE_CACHE_DUMP_READ
+#define GCVM_L2_PTE_CACHE_DUMP_READ__DATA__SHIFT                                                              0x0
+#define GCVM_L2_PTE_CACHE_DUMP_READ__DATA_MASK                                                                0xFFFFFFFFL
+//GCVML2_MAIN_DEBUG
+#define GCVML2_MAIN_DEBUG__FLAGS__SHIFT                                                                       0x0
+#define GCVML2_MAIN_DEBUG__FLAGS_MASK                                                                         0xFFFFFFFFL
+//GCVML2_WALKER_DEBUG
+#define GCVML2_WALKER_DEBUG__FLAGS__SHIFT                                                                     0x0
+#define GCVML2_WALKER_DEBUG__FLAGS_MASK                                                                       0xFFFFFFFFL
+//GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ADDR_LO32
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ADDR_LO32__ADDR_LO32__SHIFT                               0x0
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ADDR_LO32__ADDR_LO32_MASK                                 0xFFFFFFFFL
+//GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ADDR_HI32
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ADDR_HI32__ADDR_HI13__SHIFT                               0x0
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ADDR_HI32__ADDR_HI13_MASK                                 0x00001FFFL
+//GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__VMID__SHIFT                                         0x0
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__VFID__SHIFT                                         0x4
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__VF__SHIFT                                           0x9
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__GPA__SHIFT                                          0xa
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__RD_PERM__SHIFT                                      0xc
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__WR_PERM__SHIFT                                      0xd
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__EX_PERM__SHIFT                                      0xe
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__AT_PERM__SHIFT                                      0xf
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__CLIENT_ID__SHIFT                                    0x10
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__REQ__SHIFT                                          0x1f
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__VMID_MASK                                           0x0000000FL
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__VFID_MASK                                           0x000001F0L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__VF_MASK                                             0x00000200L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__GPA_MASK                                            0x00000C00L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__RD_PERM_MASK                                        0x00001000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__WR_PERM_MASK                                        0x00002000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__EX_PERM_MASK                                        0x00004000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__AT_PERM_MASK                                        0x00008000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__CLIENT_ID_MASK                                      0x01FF0000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_ATTR__REQ_MASK                                            0x80000000L
+//GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ADDR_LO32
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ADDR_LO32__ADDR_LO32__SHIFT                              0x0
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ADDR_LO32__ADDR_LO32_MASK                                0xFFFFFFFFL
+//GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ADDR_HI32
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ADDR_HI32__ADDR_HI8__SHIFT                               0x0
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ADDR_HI32__ADDR_HI8_MASK                                 0x000000FFL
+//GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__PERMS__SHIFT                                       0x0
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__FRAGMENT_SIZE__SHIFT                               0x4
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__SNOOP__SHIFT                                       0xa
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__SPA__SHIFT                                         0xb
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__IO__SHIFT                                          0xc
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__PTE_TMZ__SHIFT                                     0xd
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__NO_PTE__SHIFT                                      0xe
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__MTYPE__SHIFT                                       0xf
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__COMP_EN__SHIFT                                     0x11
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__NACK__SHIFT                                        0x12
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__HDM__SHIFT                                         0x14
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__TMPM_RETRY__SHIFT                                  0x15
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__ACK__SHIFT                                         0x1f
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__PERMS_MASK                                         0x0000000FL
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__FRAGMENT_SIZE_MASK                                 0x000003F0L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__SNOOP_MASK                                         0x00000400L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__SPA_MASK                                           0x00000800L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__IO_MASK                                            0x00001000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__PTE_TMZ_MASK                                       0x00002000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__NO_PTE_MASK                                        0x00004000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__MTYPE_MASK                                         0x00018000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__COMP_EN_MASK                                       0x00020000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__NACK_MASK                                          0x000C0000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__HDM_MASK                                           0x00100000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__TMPM_RETRY_MASK                                    0x00200000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_ATTR__ACK_MASK                                           0x80000000L
+//GCVM_L2_BANK_SELECT_MASKS
+#define GCVM_L2_BANK_SELECT_MASKS__MASK0__SHIFT                                                               0x0
+#define GCVM_L2_BANK_SELECT_MASKS__MASK1__SHIFT                                                               0x4
+#define GCVM_L2_BANK_SELECT_MASKS__MASK2__SHIFT                                                               0x8
+#define GCVM_L2_BANK_SELECT_MASKS__MASK3__SHIFT                                                               0xc
+#define GCVM_L2_BANK_SELECT_MASKS__MASK0_MASK                                                                 0x0000000FL
+#define GCVM_L2_BANK_SELECT_MASKS__MASK1_MASK                                                                 0x000000F0L
+#define GCVM_L2_BANK_SELECT_MASKS__MASK2_MASK                                                                 0x00000F00L
+#define GCVM_L2_BANK_SELECT_MASKS__MASK3_MASK                                                                 0x0000F000L
+//GCUTCL2_CREDIT_SAFETY_GROUP_RET_CDC
+#define GCUTCL2_CREDIT_SAFETY_GROUP_RET_CDC__CREDITS__SHIFT                                                   0x0
+#define GCUTCL2_CREDIT_SAFETY_GROUP_RET_CDC__UPDATE__SHIFT                                                    0xa
+#define GCUTCL2_CREDIT_SAFETY_GROUP_RET_CDC__CREDITS_MASK                                                     0x000003FFL
+#define GCUTCL2_CREDIT_SAFETY_GROUP_RET_CDC__UPDATE_MASK                                                      0x00000400L
+//GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC
+#define GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC__CREDITS__SHIFT                                        0x0
+#define GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC__UPDATE__SHIFT                                         0xa
+#define GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC__CREDITS_MASK                                          0x000003FFL
+#define GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC__UPDATE_MASK                                           0x00000400L
+//GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC
+#define GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC__CREDITS__SHIFT                                      0x0
+#define GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC__UPDATE__SHIFT                                       0xa
+#define GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC__CREDITS_MASK                                        0x000003FFL
+#define GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC__UPDATE_MASK                                         0x00000400L
+//GCVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT
+#define GCVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT__CREDITS__SHIFT                                               0x0
+#define GCVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT__UPDATE__SHIFT                                                0xa
+#define GCVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT__CREDITS_MASK                                                 0x000003FFL
+#define GCVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT__UPDATE_MASK                                                  0x00000400L
+//GCVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ
+#define GCVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ__CREDITS__SHIFT                                               0x0
+#define GCVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ__UPDATE__SHIFT                                                0xa
+#define GCVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ__CREDITS_MASK                                                 0x000003FFL
+#define GCVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ__UPDATE_MASK                                                  0x00000400L
+//GCVM_L2_UTCL2_FED_VMID_STS
+#define GCVM_L2_UTCL2_FED_VMID_STS__VMIDS__SHIFT                                                              0x0
+#define GCVM_L2_UTCL2_FED_VMID_STS__VMIDS_MASK                                                                0x0000FFFFL
+//GCVM_L2_UTCL2_FED_ACK_REG
+#define GCVM_L2_UTCL2_FED_ACK_REG__VMIDS__SHIFT                                                               0x0
+#define GCVM_L2_UTCL2_FED_ACK_REG__VMIDS_MASK                                                                 0x0000FFFFL
+//GCVML2_IH_FAULT_INTERRUPT_CNTL
+#define GCVML2_IH_FAULT_INTERRUPT_CNTL__ENABLE_PF_POISON_INTERRUPT__SHIFT                                     0x0
+#define GCVML2_IH_FAULT_INTERRUPT_CNTL__ENABLE_VF_POISON_INTERRUPT__SHIFT                                     0x1
+#define GCVML2_IH_FAULT_INTERRUPT_CNTL__POISON_INTERRUPT_TO_PF__SHIFT                                         0x2
+#define GCVML2_IH_FAULT_INTERRUPT_CNTL__ENABLE_PF_POISON_INTERRUPT_MASK                                       0x00000001L
+#define GCVML2_IH_FAULT_INTERRUPT_CNTL__ENABLE_VF_POISON_INTERRUPT_MASK                                       0x00000002L
+#define GCVML2_IH_FAULT_INTERRUPT_CNTL__POISON_INTERRUPT_TO_PF_MASK                                           0x00000004L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcl2tlbpfdec
+//GC_L2TLB_TLB0_STATUS
+#define GC_L2TLB_TLB0_STATUS__BUSY__SHIFT                                                                     0x0
+#define GC_L2TLB_TLB0_STATUS__FOUND_PARITY_ERRORS__SHIFT                                                      0x1
+#define GC_L2TLB_TLB0_STATUS__BUSY_MASK                                                                       0x00000001L
+#define GC_L2TLB_TLB0_STATUS__FOUND_PARITY_ERRORS_MASK                                                        0x00000002L
+//GC_L2TLB_TLB0_RETRY_CNTL
+#define GC_L2TLB_TLB0_RETRY_CNTL__RETRY_COUNT__SHIFT                                                          0x0
+#define GC_L2TLB_TLB0_RETRY_CNTL__RETRY_TIMER__SHIFT                                                          0x10
+#define GC_L2TLB_TLB0_RETRY_CNTL__RETRY_ENABLE__SHIFT                                                         0x15
+#define GC_L2TLB_TLB0_RETRY_CNTL__NON_TMPM_RETRY_ENABLE__SHIFT                                                0x16
+#define GC_L2TLB_TLB0_RETRY_CNTL__TMPM_RETRY_ENABLE__SHIFT                                                    0x17
+#define GC_L2TLB_TLB0_RETRY_CNTL__RETRY_COUNT_MASK                                                            0x0000FFFFL
+#define GC_L2TLB_TLB0_RETRY_CNTL__RETRY_TIMER_MASK                                                            0x001F0000L
+#define GC_L2TLB_TLB0_RETRY_CNTL__RETRY_ENABLE_MASK                                                           0x00200000L
+#define GC_L2TLB_TLB0_RETRY_CNTL__NON_TMPM_RETRY_ENABLE_MASK                                                  0x00400000L
+#define GC_L2TLB_TLB0_RETRY_CNTL__TMPM_RETRY_ENABLE_MASK                                                      0x00800000L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvmsharedvcdec
+//GCMC_VM_FB_LOCATION_BASE_LO32
+#define GCMC_VM_FB_LOCATION_BASE_LO32__FB_BASE_LO32__SHIFT                                                    0x0
+#define GCMC_VM_FB_LOCATION_BASE_LO32__FB_BASE_LO32_MASK                                                      0xFFFFFFFFL
+//GCMC_VM_FB_LOCATION_BASE_HI32
+#define GCMC_VM_FB_LOCATION_BASE_HI32__FB_BASE_HI1__SHIFT                                                     0x0
+#define GCMC_VM_FB_LOCATION_BASE_HI32__FB_BASE_HI1_MASK                                                       0x00000001L
+//GCMC_VM_FB_LOCATION_TOP_LO32
+#define GCMC_VM_FB_LOCATION_TOP_LO32__FB_TOP_LO32__SHIFT                                                      0x0
+#define GCMC_VM_FB_LOCATION_TOP_LO32__FB_TOP_LO32_MASK                                                        0xFFFFFFFFL
+//GCMC_VM_FB_LOCATION_TOP_HI32
+#define GCMC_VM_FB_LOCATION_TOP_HI32__FB_TOP_HI1__SHIFT                                                       0x0
+#define GCMC_VM_FB_LOCATION_TOP_HI32__FB_TOP_HI1_MASK                                                         0x00000001L
+//GCMC_VM_AGP_TOP_LO32
+#define GCMC_VM_AGP_TOP_LO32__AGP_TOP_LO32__SHIFT                                                             0x0
+#define GCMC_VM_AGP_TOP_LO32__AGP_TOP_LO32_MASK                                                               0xFFFFFFFFL
+//GCMC_VM_AGP_TOP_HI32
+#define GCMC_VM_AGP_TOP_HI32__AGP_TOP_HI1__SHIFT                                                              0x0
+#define GCMC_VM_AGP_TOP_HI32__AGP_TOP_HI1_MASK                                                                0x00000001L
+//GCMC_VM_AGP_BOT_LO32
+#define GCMC_VM_AGP_BOT_LO32__AGP_BOT_LO32__SHIFT                                                             0x0
+#define GCMC_VM_AGP_BOT_LO32__AGP_BOT_LO32_MASK                                                               0xFFFFFFFFL
+//GCMC_VM_AGP_BOT_HI32
+#define GCMC_VM_AGP_BOT_HI32__AGP_BOT_HI1__SHIFT                                                              0x0
+#define GCMC_VM_AGP_BOT_HI32__AGP_BOT_HI1_MASK                                                                0x00000001L
+//GCMC_VM_AGP_BASE_LO32
+#define GCMC_VM_AGP_BASE_LO32__AGP_BASE_LO32__SHIFT                                                           0x0
+#define GCMC_VM_AGP_BASE_LO32__AGP_BASE_LO32_MASK                                                             0xFFFFFFFFL
+//GCMC_VM_AGP_BASE_HI32
+#define GCMC_VM_AGP_BASE_HI32__AGP_BASE_HI1__SHIFT                                                            0x0
+#define GCMC_VM_AGP_BASE_HI32__AGP_BASE_HI1_MASK                                                              0x00000001L
+//GCMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32
+#define GCMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32__LOGICAL_ADDR_LO32__SHIFT                                       0x0
+#define GCMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32__LOGICAL_ADDR_LO32_MASK                                         0xFFFFFFFFL
+//GCMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32
+#define GCMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32__LOGICAL_ADDR_HI7__SHIFT                                        0x0
+#define GCMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32__LOGICAL_ADDR_HI7_MASK                                          0x0000007FL
+//GCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32
+#define GCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32__LOGICAL_ADDR_LO32__SHIFT                                      0x0
+#define GCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32__LOGICAL_ADDR_LO32_MASK                                        0xFFFFFFFFL
+//GCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32
+#define GCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32__LOGICAL_ADDR_HI7__SHIFT                                       0x0
+#define GCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32__LOGICAL_ADDR_HI7_MASK                                         0x0000007FL
+//GCMC_VM_MX_L1_TLB_CNTL
+#define GCMC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB__SHIFT                                                          0x0
+#define GCMC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE__SHIFT                                                     0x3
+#define GCMC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT                                        0x5
+#define GCMC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL__SHIFT                                           0x6
+#define GCMC_VM_MX_L1_TLB_CNTL__ECO_BITS__SHIFT                                                               0x7
+#define GCMC_VM_MX_L1_TLB_CNTL__MTYPE__SHIFT                                                                  0xb
+#define GCMC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB_MASK                                                            0x00000001L
+#define GCMC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK                                                       0x00000018L
+#define GCMC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS_MASK                                          0x00000020L
+#define GCMC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL_MASK                                             0x00000040L
+#define GCMC_VM_MX_L1_TLB_CNTL__ECO_BITS_MASK                                                                 0x00000780L
+#define GCMC_VM_MX_L1_TLB_CNTL__MTYPE_MASK                                                                    0x00001800L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvml2vcdec
+//GCVM_CONTEXT0_CNTL
+#define GCVM_CONTEXT0_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT0_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define GCVM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define GCVM_CONTEXT0_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define GCVM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define GCVM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define GCVM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define GCVM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define GCVM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define GCVM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define GCVM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define GCVM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define GCVM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define GCVM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define GCVM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define GCVM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define GCVM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define GCVM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define GCVM_CONTEXT0_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define GCVM_CONTEXT0_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define GCVM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define GCVM_CONTEXT0_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define GCVM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define GCVM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define GCVM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define GCVM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define GCVM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define GCVM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define GCVM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define GCVM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define GCVM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define GCVM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define GCVM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define GCVM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define GCVM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define GCVM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+//GCVM_CONTEXT1_CNTL
+#define GCVM_CONTEXT1_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define GCVM_CONTEXT1_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define GCVM_CONTEXT1_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define GCVM_CONTEXT1_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define GCVM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define GCVM_CONTEXT1_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define GCVM_CONTEXT1_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+//GCVM_CONTEXT2_CNTL
+#define GCVM_CONTEXT2_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT2_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT2_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define GCVM_CONTEXT2_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define GCVM_CONTEXT2_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define GCVM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define GCVM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define GCVM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define GCVM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define GCVM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define GCVM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define GCVM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define GCVM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define GCVM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define GCVM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define GCVM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define GCVM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define GCVM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define GCVM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define GCVM_CONTEXT2_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT2_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define GCVM_CONTEXT2_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define GCVM_CONTEXT2_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define GCVM_CONTEXT2_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define GCVM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define GCVM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define GCVM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define GCVM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define GCVM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define GCVM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define GCVM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define GCVM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define GCVM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define GCVM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define GCVM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define GCVM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define GCVM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define GCVM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+//GCVM_CONTEXT3_CNTL
+#define GCVM_CONTEXT3_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT3_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT3_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define GCVM_CONTEXT3_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define GCVM_CONTEXT3_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define GCVM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define GCVM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define GCVM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define GCVM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define GCVM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define GCVM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define GCVM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define GCVM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define GCVM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define GCVM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define GCVM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define GCVM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define GCVM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define GCVM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define GCVM_CONTEXT3_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT3_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define GCVM_CONTEXT3_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define GCVM_CONTEXT3_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define GCVM_CONTEXT3_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define GCVM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define GCVM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define GCVM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define GCVM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define GCVM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define GCVM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define GCVM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define GCVM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define GCVM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define GCVM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define GCVM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define GCVM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define GCVM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define GCVM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+//GCVM_CONTEXT4_CNTL
+#define GCVM_CONTEXT4_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT4_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT4_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define GCVM_CONTEXT4_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define GCVM_CONTEXT4_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define GCVM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define GCVM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define GCVM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define GCVM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define GCVM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define GCVM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define GCVM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define GCVM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define GCVM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define GCVM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define GCVM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define GCVM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define GCVM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define GCVM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define GCVM_CONTEXT4_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT4_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define GCVM_CONTEXT4_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define GCVM_CONTEXT4_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define GCVM_CONTEXT4_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define GCVM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define GCVM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define GCVM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define GCVM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define GCVM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define GCVM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define GCVM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define GCVM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define GCVM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define GCVM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define GCVM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define GCVM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define GCVM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define GCVM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+//GCVM_CONTEXT5_CNTL
+#define GCVM_CONTEXT5_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT5_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT5_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define GCVM_CONTEXT5_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define GCVM_CONTEXT5_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define GCVM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define GCVM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define GCVM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define GCVM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define GCVM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define GCVM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define GCVM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define GCVM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define GCVM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define GCVM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define GCVM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define GCVM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define GCVM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define GCVM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define GCVM_CONTEXT5_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT5_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define GCVM_CONTEXT5_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define GCVM_CONTEXT5_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define GCVM_CONTEXT5_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define GCVM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define GCVM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define GCVM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define GCVM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define GCVM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define GCVM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define GCVM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define GCVM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define GCVM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define GCVM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define GCVM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define GCVM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define GCVM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define GCVM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+//GCVM_CONTEXT6_CNTL
+#define GCVM_CONTEXT6_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT6_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT6_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define GCVM_CONTEXT6_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define GCVM_CONTEXT6_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define GCVM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define GCVM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define GCVM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define GCVM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define GCVM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define GCVM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define GCVM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define GCVM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define GCVM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define GCVM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define GCVM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define GCVM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define GCVM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define GCVM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define GCVM_CONTEXT6_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT6_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define GCVM_CONTEXT6_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define GCVM_CONTEXT6_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define GCVM_CONTEXT6_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define GCVM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define GCVM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define GCVM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define GCVM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define GCVM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define GCVM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define GCVM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define GCVM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define GCVM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define GCVM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define GCVM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define GCVM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define GCVM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define GCVM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+//GCVM_CONTEXT7_CNTL
+#define GCVM_CONTEXT7_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT7_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT7_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define GCVM_CONTEXT7_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define GCVM_CONTEXT7_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define GCVM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define GCVM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define GCVM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define GCVM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define GCVM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define GCVM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define GCVM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define GCVM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define GCVM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define GCVM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define GCVM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define GCVM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define GCVM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define GCVM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define GCVM_CONTEXT7_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT7_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define GCVM_CONTEXT7_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define GCVM_CONTEXT7_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define GCVM_CONTEXT7_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define GCVM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define GCVM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define GCVM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define GCVM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define GCVM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define GCVM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define GCVM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define GCVM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define GCVM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define GCVM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define GCVM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define GCVM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define GCVM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define GCVM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+//GCVM_CONTEXT8_CNTL
+#define GCVM_CONTEXT8_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT8_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT8_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define GCVM_CONTEXT8_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define GCVM_CONTEXT8_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define GCVM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define GCVM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define GCVM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define GCVM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define GCVM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define GCVM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define GCVM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define GCVM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define GCVM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define GCVM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define GCVM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define GCVM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define GCVM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define GCVM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define GCVM_CONTEXT8_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT8_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define GCVM_CONTEXT8_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define GCVM_CONTEXT8_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define GCVM_CONTEXT8_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define GCVM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define GCVM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define GCVM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define GCVM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define GCVM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define GCVM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define GCVM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define GCVM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define GCVM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define GCVM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define GCVM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define GCVM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define GCVM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define GCVM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+//GCVM_CONTEXT9_CNTL
+#define GCVM_CONTEXT9_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT9_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT9_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define GCVM_CONTEXT9_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define GCVM_CONTEXT9_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define GCVM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define GCVM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define GCVM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define GCVM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define GCVM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define GCVM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define GCVM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define GCVM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define GCVM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define GCVM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define GCVM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define GCVM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define GCVM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define GCVM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define GCVM_CONTEXT9_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT9_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define GCVM_CONTEXT9_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define GCVM_CONTEXT9_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define GCVM_CONTEXT9_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define GCVM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define GCVM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define GCVM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define GCVM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define GCVM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define GCVM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define GCVM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define GCVM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define GCVM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define GCVM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define GCVM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define GCVM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define GCVM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define GCVM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+//GCVM_CONTEXT10_CNTL
+#define GCVM_CONTEXT10_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define GCVM_CONTEXT10_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define GCVM_CONTEXT10_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x4
+#define GCVM_CONTEXT10_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x8
+#define GCVM_CONTEXT10_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x9
+#define GCVM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xa
+#define GCVM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xb
+#define GCVM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xc
+#define GCVM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xd
+#define GCVM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xe
+#define GCVM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xf
+#define GCVM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x10
+#define GCVM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x11
+#define GCVM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x12
+#define GCVM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x13
+#define GCVM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x14
+#define GCVM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x15
+#define GCVM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x16
+#define GCVM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x17
+#define GCVM_CONTEXT10_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define GCVM_CONTEXT10_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x0000000EL
+#define GCVM_CONTEXT10_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x000000F0L
+#define GCVM_CONTEXT10_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000100L
+#define GCVM_CONTEXT10_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000200L
+#define GCVM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000400L
+#define GCVM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000800L
+#define GCVM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00001000L
+#define GCVM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00002000L
+#define GCVM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00004000L
+#define GCVM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00008000L
+#define GCVM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00010000L
+#define GCVM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00020000L
+#define GCVM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00040000L
+#define GCVM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00080000L
+#define GCVM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00100000L
+#define GCVM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00200000L
+#define GCVM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00400000L
+#define GCVM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00800000L
+//GCVM_CONTEXT11_CNTL
+#define GCVM_CONTEXT11_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define GCVM_CONTEXT11_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define GCVM_CONTEXT11_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x4
+#define GCVM_CONTEXT11_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x8
+#define GCVM_CONTEXT11_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x9
+#define GCVM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xa
+#define GCVM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xb
+#define GCVM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xc
+#define GCVM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xd
+#define GCVM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xe
+#define GCVM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xf
+#define GCVM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x10
+#define GCVM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x11
+#define GCVM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x12
+#define GCVM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x13
+#define GCVM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x14
+#define GCVM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x15
+#define GCVM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x16
+#define GCVM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x17
+#define GCVM_CONTEXT11_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define GCVM_CONTEXT11_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x0000000EL
+#define GCVM_CONTEXT11_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x000000F0L
+#define GCVM_CONTEXT11_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000100L
+#define GCVM_CONTEXT11_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000200L
+#define GCVM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000400L
+#define GCVM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000800L
+#define GCVM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00001000L
+#define GCVM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00002000L
+#define GCVM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00004000L
+#define GCVM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00008000L
+#define GCVM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00010000L
+#define GCVM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00020000L
+#define GCVM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00040000L
+#define GCVM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00080000L
+#define GCVM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00100000L
+#define GCVM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00200000L
+#define GCVM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00400000L
+#define GCVM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00800000L
+//GCVM_CONTEXT12_CNTL
+#define GCVM_CONTEXT12_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define GCVM_CONTEXT12_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define GCVM_CONTEXT12_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x4
+#define GCVM_CONTEXT12_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x8
+#define GCVM_CONTEXT12_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x9
+#define GCVM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xa
+#define GCVM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xb
+#define GCVM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xc
+#define GCVM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xd
+#define GCVM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xe
+#define GCVM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xf
+#define GCVM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x10
+#define GCVM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x11
+#define GCVM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x12
+#define GCVM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x13
+#define GCVM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x14
+#define GCVM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x15
+#define GCVM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x16
+#define GCVM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x17
+#define GCVM_CONTEXT12_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define GCVM_CONTEXT12_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x0000000EL
+#define GCVM_CONTEXT12_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x000000F0L
+#define GCVM_CONTEXT12_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000100L
+#define GCVM_CONTEXT12_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000200L
+#define GCVM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000400L
+#define GCVM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000800L
+#define GCVM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00001000L
+#define GCVM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00002000L
+#define GCVM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00004000L
+#define GCVM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00008000L
+#define GCVM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00010000L
+#define GCVM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00020000L
+#define GCVM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00040000L
+#define GCVM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00080000L
+#define GCVM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00100000L
+#define GCVM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00200000L
+#define GCVM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00400000L
+#define GCVM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00800000L
+//GCVM_CONTEXT13_CNTL
+#define GCVM_CONTEXT13_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define GCVM_CONTEXT13_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define GCVM_CONTEXT13_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x4
+#define GCVM_CONTEXT13_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x8
+#define GCVM_CONTEXT13_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x9
+#define GCVM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xa
+#define GCVM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xb
+#define GCVM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xc
+#define GCVM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xd
+#define GCVM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xe
+#define GCVM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xf
+#define GCVM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x10
+#define GCVM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x11
+#define GCVM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x12
+#define GCVM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x13
+#define GCVM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x14
+#define GCVM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x15
+#define GCVM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x16
+#define GCVM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x17
+#define GCVM_CONTEXT13_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define GCVM_CONTEXT13_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x0000000EL
+#define GCVM_CONTEXT13_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x000000F0L
+#define GCVM_CONTEXT13_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000100L
+#define GCVM_CONTEXT13_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000200L
+#define GCVM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000400L
+#define GCVM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000800L
+#define GCVM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00001000L
+#define GCVM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00002000L
+#define GCVM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00004000L
+#define GCVM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00008000L
+#define GCVM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00010000L
+#define GCVM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00020000L
+#define GCVM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00040000L
+#define GCVM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00080000L
+#define GCVM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00100000L
+#define GCVM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00200000L
+#define GCVM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00400000L
+#define GCVM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00800000L
+//GCVM_CONTEXT14_CNTL
+#define GCVM_CONTEXT14_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define GCVM_CONTEXT14_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define GCVM_CONTEXT14_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x4
+#define GCVM_CONTEXT14_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x8
+#define GCVM_CONTEXT14_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x9
+#define GCVM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xa
+#define GCVM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xb
+#define GCVM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xc
+#define GCVM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xd
+#define GCVM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xe
+#define GCVM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xf
+#define GCVM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x10
+#define GCVM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x11
+#define GCVM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x12
+#define GCVM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x13
+#define GCVM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x14
+#define GCVM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x15
+#define GCVM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x16
+#define GCVM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x17
+#define GCVM_CONTEXT14_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define GCVM_CONTEXT14_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x0000000EL
+#define GCVM_CONTEXT14_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x000000F0L
+#define GCVM_CONTEXT14_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000100L
+#define GCVM_CONTEXT14_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000200L
+#define GCVM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000400L
+#define GCVM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000800L
+#define GCVM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00001000L
+#define GCVM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00002000L
+#define GCVM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00004000L
+#define GCVM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00008000L
+#define GCVM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00010000L
+#define GCVM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00020000L
+#define GCVM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00040000L
+#define GCVM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00080000L
+#define GCVM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00100000L
+#define GCVM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00200000L
+#define GCVM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00400000L
+#define GCVM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00800000L
+//GCVM_CONTEXT15_CNTL
+#define GCVM_CONTEXT15_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define GCVM_CONTEXT15_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define GCVM_CONTEXT15_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x4
+#define GCVM_CONTEXT15_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x8
+#define GCVM_CONTEXT15_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x9
+#define GCVM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xa
+#define GCVM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xb
+#define GCVM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xc
+#define GCVM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xd
+#define GCVM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xe
+#define GCVM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xf
+#define GCVM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x10
+#define GCVM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x11
+#define GCVM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x12
+#define GCVM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x13
+#define GCVM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x14
+#define GCVM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x15
+#define GCVM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x16
+#define GCVM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x17
+#define GCVM_CONTEXT15_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define GCVM_CONTEXT15_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x0000000EL
+#define GCVM_CONTEXT15_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x000000F0L
+#define GCVM_CONTEXT15_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000100L
+#define GCVM_CONTEXT15_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000200L
+#define GCVM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000400L
+#define GCVM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000800L
+#define GCVM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00001000L
+#define GCVM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00002000L
+#define GCVM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00004000L
+#define GCVM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00008000L
+#define GCVM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00010000L
+#define GCVM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00020000L
+#define GCVM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00040000L
+#define GCVM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00080000L
+#define GCVM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00100000L
+#define GCVM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00200000L
+#define GCVM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00400000L
+#define GCVM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00800000L
+//GCVM_CONTEXTS_DISABLE
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_0__SHIFT                                                       0x0
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_1__SHIFT                                                       0x1
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_2__SHIFT                                                       0x2
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_3__SHIFT                                                       0x3
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_4__SHIFT                                                       0x4
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_5__SHIFT                                                       0x5
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_6__SHIFT                                                       0x6
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_7__SHIFT                                                       0x7
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_8__SHIFT                                                       0x8
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_9__SHIFT                                                       0x9
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_10__SHIFT                                                      0xa
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_11__SHIFT                                                      0xb
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_12__SHIFT                                                      0xc
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_13__SHIFT                                                      0xd
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_14__SHIFT                                                      0xe
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_15__SHIFT                                                      0xf
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_0_MASK                                                         0x00000001L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_1_MASK                                                         0x00000002L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_2_MASK                                                         0x00000004L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_3_MASK                                                         0x00000008L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_4_MASK                                                         0x00000010L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_5_MASK                                                         0x00000020L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_6_MASK                                                         0x00000040L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_7_MASK                                                         0x00000080L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_8_MASK                                                         0x00000100L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_9_MASK                                                         0x00000200L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_10_MASK                                                        0x00000400L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_11_MASK                                                        0x00000800L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_12_MASK                                                        0x00001000L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_13_MASK                                                        0x00002000L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_14_MASK                                                        0x00004000L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_15_MASK                                                        0x00008000L
+//GCVM_INVALIDATE_ENG0_SEM
+#define GCVM_INVALIDATE_ENG0_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG0_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG1_SEM
+#define GCVM_INVALIDATE_ENG1_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG1_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG2_SEM
+#define GCVM_INVALIDATE_ENG2_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG2_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG3_SEM
+#define GCVM_INVALIDATE_ENG3_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG3_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG4_SEM
+#define GCVM_INVALIDATE_ENG4_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG4_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG5_SEM
+#define GCVM_INVALIDATE_ENG5_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG5_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG6_SEM
+#define GCVM_INVALIDATE_ENG6_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG6_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG7_SEM
+#define GCVM_INVALIDATE_ENG7_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG7_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG8_SEM
+#define GCVM_INVALIDATE_ENG8_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG8_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG9_SEM
+#define GCVM_INVALIDATE_ENG9_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG9_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG10_SEM
+#define GCVM_INVALIDATE_ENG10_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define GCVM_INVALIDATE_ENG10_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//GCVM_INVALIDATE_ENG11_SEM
+#define GCVM_INVALIDATE_ENG11_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define GCVM_INVALIDATE_ENG11_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//GCVM_INVALIDATE_ENG12_SEM
+#define GCVM_INVALIDATE_ENG12_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define GCVM_INVALIDATE_ENG12_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//GCVM_INVALIDATE_ENG13_SEM
+#define GCVM_INVALIDATE_ENG13_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define GCVM_INVALIDATE_ENG13_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//GCVM_INVALIDATE_ENG14_SEM
+#define GCVM_INVALIDATE_ENG14_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define GCVM_INVALIDATE_ENG14_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//GCVM_INVALIDATE_ENG15_SEM
+#define GCVM_INVALIDATE_ENG15_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define GCVM_INVALIDATE_ENG15_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//GCVM_INVALIDATE_ENG16_SEM
+#define GCVM_INVALIDATE_ENG16_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define GCVM_INVALIDATE_ENG16_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//GCVM_INVALIDATE_ENG17_SEM
+#define GCVM_INVALIDATE_ENG17_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define GCVM_INVALIDATE_ENG17_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//GCVM_INVALIDATE_ENG0_REQ
+#define GCVM_INVALIDATE_ENG0_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG0_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG0_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG0_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define GCVM_INVALIDATE_ENG0_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG0_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG0_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG0_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//GCVM_INVALIDATE_ENG1_REQ
+#define GCVM_INVALIDATE_ENG1_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG1_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG1_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG1_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define GCVM_INVALIDATE_ENG1_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG1_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG1_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG1_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//GCVM_INVALIDATE_ENG2_REQ
+#define GCVM_INVALIDATE_ENG2_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG2_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG2_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG2_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define GCVM_INVALIDATE_ENG2_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG2_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG2_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG2_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//GCVM_INVALIDATE_ENG3_REQ
+#define GCVM_INVALIDATE_ENG3_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG3_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG3_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG3_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define GCVM_INVALIDATE_ENG3_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG3_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG3_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG3_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//GCVM_INVALIDATE_ENG4_REQ
+#define GCVM_INVALIDATE_ENG4_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG4_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG4_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG4_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define GCVM_INVALIDATE_ENG4_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG4_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG4_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG4_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//GCVM_INVALIDATE_ENG5_REQ
+#define GCVM_INVALIDATE_ENG5_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG5_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG5_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG5_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define GCVM_INVALIDATE_ENG5_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG5_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG5_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG5_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//GCVM_INVALIDATE_ENG6_REQ
+#define GCVM_INVALIDATE_ENG6_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG6_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG6_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG6_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define GCVM_INVALIDATE_ENG6_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG6_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG6_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG6_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//GCVM_INVALIDATE_ENG7_REQ
+#define GCVM_INVALIDATE_ENG7_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG7_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG7_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG7_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define GCVM_INVALIDATE_ENG7_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG7_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG7_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG7_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//GCVM_INVALIDATE_ENG8_REQ
+#define GCVM_INVALIDATE_ENG8_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG8_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG8_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG8_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define GCVM_INVALIDATE_ENG8_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG8_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG8_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG8_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//GCVM_INVALIDATE_ENG9_REQ
+#define GCVM_INVALIDATE_ENG9_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG9_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG9_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG9_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define GCVM_INVALIDATE_ENG9_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG9_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG9_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG9_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//GCVM_INVALIDATE_ENG10_REQ
+#define GCVM_INVALIDATE_ENG10_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG10_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define GCVM_INVALIDATE_ENG10_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define GCVM_INVALIDATE_ENG10_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE3__SHIFT                                                  0x1b
+#define GCVM_INVALIDATE_ENG10_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG10_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define GCVM_INVALIDATE_ENG10_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define GCVM_INVALIDATE_ENG10_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE3_MASK                                                    0x08000000L
+//GCVM_INVALIDATE_ENG11_REQ
+#define GCVM_INVALIDATE_ENG11_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG11_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define GCVM_INVALIDATE_ENG11_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define GCVM_INVALIDATE_ENG11_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE3__SHIFT                                                  0x1b
+#define GCVM_INVALIDATE_ENG11_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG11_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define GCVM_INVALIDATE_ENG11_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define GCVM_INVALIDATE_ENG11_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE3_MASK                                                    0x08000000L
+//GCVM_INVALIDATE_ENG12_REQ
+#define GCVM_INVALIDATE_ENG12_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG12_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define GCVM_INVALIDATE_ENG12_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define GCVM_INVALIDATE_ENG12_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE3__SHIFT                                                  0x1b
+#define GCVM_INVALIDATE_ENG12_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG12_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define GCVM_INVALIDATE_ENG12_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define GCVM_INVALIDATE_ENG12_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE3_MASK                                                    0x08000000L
+//GCVM_INVALIDATE_ENG13_REQ
+#define GCVM_INVALIDATE_ENG13_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG13_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define GCVM_INVALIDATE_ENG13_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define GCVM_INVALIDATE_ENG13_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE3__SHIFT                                                  0x1b
+#define GCVM_INVALIDATE_ENG13_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG13_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define GCVM_INVALIDATE_ENG13_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define GCVM_INVALIDATE_ENG13_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE3_MASK                                                    0x08000000L
+//GCVM_INVALIDATE_ENG14_REQ
+#define GCVM_INVALIDATE_ENG14_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG14_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define GCVM_INVALIDATE_ENG14_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define GCVM_INVALIDATE_ENG14_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE3__SHIFT                                                  0x1b
+#define GCVM_INVALIDATE_ENG14_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG14_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define GCVM_INVALIDATE_ENG14_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define GCVM_INVALIDATE_ENG14_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE3_MASK                                                    0x08000000L
+//GCVM_INVALIDATE_ENG15_REQ
+#define GCVM_INVALIDATE_ENG15_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG15_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define GCVM_INVALIDATE_ENG15_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define GCVM_INVALIDATE_ENG15_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE3__SHIFT                                                  0x1b
+#define GCVM_INVALIDATE_ENG15_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG15_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define GCVM_INVALIDATE_ENG15_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define GCVM_INVALIDATE_ENG15_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE3_MASK                                                    0x08000000L
+//GCVM_INVALIDATE_ENG16_REQ
+#define GCVM_INVALIDATE_ENG16_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG16_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define GCVM_INVALIDATE_ENG16_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define GCVM_INVALIDATE_ENG16_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE3__SHIFT                                                  0x1b
+#define GCVM_INVALIDATE_ENG16_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG16_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define GCVM_INVALIDATE_ENG16_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define GCVM_INVALIDATE_ENG16_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE3_MASK                                                    0x08000000L
+//GCVM_INVALIDATE_ENG17_REQ
+#define GCVM_INVALIDATE_ENG17_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG17_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define GCVM_INVALIDATE_ENG17_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define GCVM_INVALIDATE_ENG17_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE3__SHIFT                                                  0x1b
+#define GCVM_INVALIDATE_ENG17_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG17_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define GCVM_INVALIDATE_ENG17_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define GCVM_INVALIDATE_ENG17_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE3_MASK                                                    0x08000000L
+//GCVM_INVALIDATE_ENG0_ACK
+#define GCVM_INVALIDATE_ENG0_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG0_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG0_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG0_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG1_ACK
+#define GCVM_INVALIDATE_ENG1_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG1_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG1_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG1_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG2_ACK
+#define GCVM_INVALIDATE_ENG2_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG2_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG2_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG2_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG3_ACK
+#define GCVM_INVALIDATE_ENG3_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG3_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG3_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG3_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG4_ACK
+#define GCVM_INVALIDATE_ENG4_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG4_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG4_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG4_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG5_ACK
+#define GCVM_INVALIDATE_ENG5_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG5_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG5_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG5_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG6_ACK
+#define GCVM_INVALIDATE_ENG6_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG6_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG6_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG6_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG7_ACK
+#define GCVM_INVALIDATE_ENG7_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG7_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG7_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG7_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG8_ACK
+#define GCVM_INVALIDATE_ENG8_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG8_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG8_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG8_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG9_ACK
+#define GCVM_INVALIDATE_ENG9_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG9_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG9_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG9_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG10_ACK
+#define GCVM_INVALIDATE_ENG10_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG10_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG10_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG10_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//GCVM_INVALIDATE_ENG11_ACK
+#define GCVM_INVALIDATE_ENG11_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG11_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG11_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG11_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//GCVM_INVALIDATE_ENG12_ACK
+#define GCVM_INVALIDATE_ENG12_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG12_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG12_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG12_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//GCVM_INVALIDATE_ENG13_ACK
+#define GCVM_INVALIDATE_ENG13_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG13_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG13_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG13_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//GCVM_INVALIDATE_ENG14_ACK
+#define GCVM_INVALIDATE_ENG14_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG14_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG14_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG14_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//GCVM_INVALIDATE_ENG15_ACK
+#define GCVM_INVALIDATE_ENG15_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG15_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG15_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG15_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//GCVM_INVALIDATE_ENG16_ACK
+#define GCVM_INVALIDATE_ENG16_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG16_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG16_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG16_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//GCVM_INVALIDATE_ENG17_ACK
+#define GCVM_INVALIDATE_ENG17_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG17_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG17_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG17_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//GCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//GCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG1_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG1_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG1_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//GCVM_INVALIDATE_ENG2_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG2_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG2_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG2_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG2_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG2_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG2_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG2_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//GCVM_INVALIDATE_ENG3_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG3_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG3_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG3_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG3_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG3_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG3_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG3_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//GCVM_INVALIDATE_ENG4_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG4_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG4_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG4_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG4_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG4_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG4_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG4_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//GCVM_INVALIDATE_ENG5_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG5_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG5_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG5_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG5_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG5_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG5_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG5_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//GCVM_INVALIDATE_ENG6_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG6_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG6_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG6_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG6_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG6_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG6_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG6_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//GCVM_INVALIDATE_ENG7_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG7_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG7_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG7_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG7_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG7_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG7_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG7_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//GCVM_INVALIDATE_ENG8_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG8_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG8_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG8_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG8_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG8_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG8_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG8_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//GCVM_INVALIDATE_ENG9_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG9_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG9_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG9_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG9_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG9_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG9_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG9_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//GCVM_INVALIDATE_ENG10_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG10_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define GCVM_INVALIDATE_ENG10_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define GCVM_INVALIDATE_ENG10_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define GCVM_INVALIDATE_ENG10_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG10_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG10_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                               0x0
+#define GCVM_INVALIDATE_ENG10_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                 0x00003FFFL
+//GCVM_INVALIDATE_ENG11_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG11_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define GCVM_INVALIDATE_ENG11_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define GCVM_INVALIDATE_ENG11_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define GCVM_INVALIDATE_ENG11_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG11_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG11_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                               0x0
+#define GCVM_INVALIDATE_ENG11_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                 0x00003FFFL
+//GCVM_INVALIDATE_ENG12_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG12_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define GCVM_INVALIDATE_ENG12_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define GCVM_INVALIDATE_ENG12_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define GCVM_INVALIDATE_ENG12_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG12_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG12_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                               0x0
+#define GCVM_INVALIDATE_ENG12_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                 0x00003FFFL
+//GCVM_INVALIDATE_ENG13_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG13_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define GCVM_INVALIDATE_ENG13_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define GCVM_INVALIDATE_ENG13_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define GCVM_INVALIDATE_ENG13_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG13_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG13_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                               0x0
+#define GCVM_INVALIDATE_ENG13_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                 0x00003FFFL
+//GCVM_INVALIDATE_ENG14_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG14_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define GCVM_INVALIDATE_ENG14_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define GCVM_INVALIDATE_ENG14_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define GCVM_INVALIDATE_ENG14_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG14_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG14_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                               0x0
+#define GCVM_INVALIDATE_ENG14_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                 0x00003FFFL
+//GCVM_INVALIDATE_ENG15_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG15_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define GCVM_INVALIDATE_ENG15_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define GCVM_INVALIDATE_ENG15_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define GCVM_INVALIDATE_ENG15_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG15_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG15_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                               0x0
+#define GCVM_INVALIDATE_ENG15_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                 0x00003FFFL
+//GCVM_INVALIDATE_ENG16_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG16_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define GCVM_INVALIDATE_ENG16_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define GCVM_INVALIDATE_ENG16_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define GCVM_INVALIDATE_ENG16_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG16_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG16_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                               0x0
+#define GCVM_INVALIDATE_ENG16_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                 0x00003FFFL
+//GCVM_INVALIDATE_ENG17_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG17_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define GCVM_INVALIDATE_ENG17_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define GCVM_INVALIDATE_ENG17_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define GCVM_INVALIDATE_ENG17_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG17_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG17_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                               0x0
+#define GCVM_INVALIDATE_ENG17_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                 0x00003FFFL
+//GCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define GCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define GCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define GCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define GCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define GCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define GCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define GCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//GCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define GCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//GCVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define GCVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//GCVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define GCVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//GCVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define GCVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//GCVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define GCVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//GCVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define GCVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//GCVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define GCVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//GCVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define GCVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//GCVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define GCVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//GCVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                            0x0
+#define GCVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                              0x00001FFFL
+//GCVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                            0x0
+#define GCVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                              0x00001FFFL
+//GCVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                            0x0
+#define GCVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                              0x00001FFFL
+//GCVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                            0x0
+#define GCVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                              0x00001FFFL
+//GCVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                            0x0
+#define GCVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                              0x00001FFFL
+//GCVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                            0x0
+#define GCVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                              0x00001FFFL
+//GCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define GCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//GCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define GCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//GCVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define GCVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//GCVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define GCVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//GCVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define GCVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//GCVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define GCVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//GCVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define GCVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//GCVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define GCVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//GCVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define GCVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//GCVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define GCVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//GCVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define GCVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//GCVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                              0x0
+#define GCVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                0x00001FFFL
+//GCVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define GCVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//GCVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                              0x0
+#define GCVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                0x00001FFFL
+//GCVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define GCVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//GCVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                              0x0
+#define GCVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                0x00001FFFL
+//GCVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define GCVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//GCVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                              0x0
+#define GCVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                0x00001FFFL
+//GCVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define GCVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//GCVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                              0x0
+#define GCVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                0x00001FFFL
+//GCVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define GCVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//GCVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                              0x0
+#define GCVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                0x00001FFFL
+//GCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT                       0x0
+#define GCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                         0x5
+#define GCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                         0xa
+#define GCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                         0x0000001FL
+#define GCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                           0x000003E0L
+#define GCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                           0x0000FC00L
+//GCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define GCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define GCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define GCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define GCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define GCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+//GCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define GCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define GCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define GCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define GCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define GCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+//GCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define GCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define GCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define GCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define GCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define GCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+//GCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define GCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define GCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define GCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define GCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define GCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+//GCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define GCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define GCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define GCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define GCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define GCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+//GCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define GCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define GCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define GCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define GCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define GCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvmsharedvfdec
+//GCMC_VM_PCIE_ATOMIC_SUPPORTED
+#define GCMC_VM_PCIE_ATOMIC_SUPPORTED__PCIE_ATOMIC_SUPPORTED__SHIFT                                           0x0
+#define GCMC_VM_PCIE_ATOMIC_SUPPORTED__PCIE_ATOMIC_SUPPORTED_MASK                                             0x00000001L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcl2tlbvfdec
+//GC_L2TLB_TLB0_EN_RETRY_ON_ATOMIC
+#define GC_L2TLB_TLB0_EN_RETRY_ON_ATOMIC__EN_RETRY_ON_ATOMIC__SHIFT                                           0x0
+#define GC_L2TLB_TLB0_EN_RETRY_ON_ATOMIC__EN_RETRY_ON_ATOMIC_MASK                                             0x00000001L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvml2prdec
+//GCMC_VM_L2_PERFCOUNTER_LO
+#define GCMC_VM_L2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                            0xFFFFFFFFL
+//GCMC_VM_L2_PERFCOUNTER_HI
+#define GCMC_VM_L2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                       0x10
+#define GCMC_VM_L2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                            0x0000FFFFL
+#define GCMC_VM_L2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                         0xFFFF0000L
+//GCUTCL2_PERFCOUNTER_LO
+#define GCUTCL2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                             0x0
+#define GCUTCL2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                               0xFFFFFFFFL
+//GCUTCL2_PERFCOUNTER_HI
+#define GCUTCL2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                             0x0
+#define GCUTCL2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                          0x10
+#define GCUTCL2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                               0x0000FFFFL
+#define GCUTCL2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                            0xFFFF0000L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcatcl2prdec
+//GC_ATC_L2_PERFCOUNTER_LO
+#define GC_ATC_L2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                           0x0
+#define GC_ATC_L2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GC_ATC_L2_PERFCOUNTER_HI
+#define GC_ATC_L2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                           0x0
+#define GC_ATC_L2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                        0x10
+#define GC_ATC_L2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                             0x0000FFFFL
+#define GC_ATC_L2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                          0xFFFF0000L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcl2tlbprdec
+//GC_L2TLB_PERFCOUNTER_LO
+#define GC_L2TLB_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                            0x0
+#define GC_L2TLB_PERFCOUNTER_LO__COUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GC_L2TLB_PERFCOUNTER_HI
+#define GC_L2TLB_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                            0x0
+#define GC_L2TLB_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                         0x10
+#define GC_L2TLB_PERFCOUNTER_HI__COUNTER_HI_MASK                                                              0x0000FFFFL
+#define GC_L2TLB_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                           0xFFFF0000L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvml2pldec
+//GCMC_VM_L2_PERFCOUNTER0_CFG
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER1_CFG
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER2_CFG
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER3_CFG
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER4_CFG
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER5_CFG
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER6_CFG
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER7_CFG
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER8_CFG
+#define GCMC_VM_L2_PERFCOUNTER8_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER8_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER8_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER8_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER8_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER8_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER8_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER8_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER8_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER8_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER9_CFG
+#define GCMC_VM_L2_PERFCOUNTER9_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER9_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER9_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER9_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER9_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER9_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER9_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER9_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER9_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER9_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER10_CFG
+#define GCMC_VM_L2_PERFCOUNTER10_CFG__PERF_SEL__SHIFT                                                         0x0
+#define GCMC_VM_L2_PERFCOUNTER10_CFG__PERF_SEL_END__SHIFT                                                     0x8
+#define GCMC_VM_L2_PERFCOUNTER10_CFG__PERF_MODE__SHIFT                                                        0x18
+#define GCMC_VM_L2_PERFCOUNTER10_CFG__ENABLE__SHIFT                                                           0x1c
+#define GCMC_VM_L2_PERFCOUNTER10_CFG__CLEAR__SHIFT                                                            0x1d
+#define GCMC_VM_L2_PERFCOUNTER10_CFG__PERF_SEL_MASK                                                           0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER10_CFG__PERF_SEL_END_MASK                                                       0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER10_CFG__PERF_MODE_MASK                                                          0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER10_CFG__ENABLE_MASK                                                             0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER10_CFG__CLEAR_MASK                                                              0x20000000L
+//GCMC_VM_L2_PERFCOUNTER11_CFG
+#define GCMC_VM_L2_PERFCOUNTER11_CFG__PERF_SEL__SHIFT                                                         0x0
+#define GCMC_VM_L2_PERFCOUNTER11_CFG__PERF_SEL_END__SHIFT                                                     0x8
+#define GCMC_VM_L2_PERFCOUNTER11_CFG__PERF_MODE__SHIFT                                                        0x18
+#define GCMC_VM_L2_PERFCOUNTER11_CFG__ENABLE__SHIFT                                                           0x1c
+#define GCMC_VM_L2_PERFCOUNTER11_CFG__CLEAR__SHIFT                                                            0x1d
+#define GCMC_VM_L2_PERFCOUNTER11_CFG__PERF_SEL_MASK                                                           0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER11_CFG__PERF_SEL_END_MASK                                                       0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER11_CFG__PERF_MODE_MASK                                                          0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER11_CFG__ENABLE_MASK                                                             0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER11_CFG__CLEAR_MASK                                                              0x20000000L
+//GCMC_VM_L2_PERFCOUNTER12_CFG
+#define GCMC_VM_L2_PERFCOUNTER12_CFG__PERF_SEL__SHIFT                                                         0x0
+#define GCMC_VM_L2_PERFCOUNTER12_CFG__PERF_SEL_END__SHIFT                                                     0x8
+#define GCMC_VM_L2_PERFCOUNTER12_CFG__PERF_MODE__SHIFT                                                        0x18
+#define GCMC_VM_L2_PERFCOUNTER12_CFG__ENABLE__SHIFT                                                           0x1c
+#define GCMC_VM_L2_PERFCOUNTER12_CFG__CLEAR__SHIFT                                                            0x1d
+#define GCMC_VM_L2_PERFCOUNTER12_CFG__PERF_SEL_MASK                                                           0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER12_CFG__PERF_SEL_END_MASK                                                       0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER12_CFG__PERF_MODE_MASK                                                          0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER12_CFG__ENABLE_MASK                                                             0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER12_CFG__CLEAR_MASK                                                              0x20000000L
+//GCMC_VM_L2_PERFCOUNTER13_CFG
+#define GCMC_VM_L2_PERFCOUNTER13_CFG__PERF_SEL__SHIFT                                                         0x0
+#define GCMC_VM_L2_PERFCOUNTER13_CFG__PERF_SEL_END__SHIFT                                                     0x8
+#define GCMC_VM_L2_PERFCOUNTER13_CFG__PERF_MODE__SHIFT                                                        0x18
+#define GCMC_VM_L2_PERFCOUNTER13_CFG__ENABLE__SHIFT                                                           0x1c
+#define GCMC_VM_L2_PERFCOUNTER13_CFG__CLEAR__SHIFT                                                            0x1d
+#define GCMC_VM_L2_PERFCOUNTER13_CFG__PERF_SEL_MASK                                                           0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER13_CFG__PERF_SEL_END_MASK                                                       0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER13_CFG__PERF_MODE_MASK                                                          0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER13_CFG__ENABLE_MASK                                                             0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER13_CFG__CLEAR_MASK                                                              0x20000000L
+//GCMC_VM_L2_PERFCOUNTER14_CFG
+#define GCMC_VM_L2_PERFCOUNTER14_CFG__PERF_SEL__SHIFT                                                         0x0
+#define GCMC_VM_L2_PERFCOUNTER14_CFG__PERF_SEL_END__SHIFT                                                     0x8
+#define GCMC_VM_L2_PERFCOUNTER14_CFG__PERF_MODE__SHIFT                                                        0x18
+#define GCMC_VM_L2_PERFCOUNTER14_CFG__ENABLE__SHIFT                                                           0x1c
+#define GCMC_VM_L2_PERFCOUNTER14_CFG__CLEAR__SHIFT                                                            0x1d
+#define GCMC_VM_L2_PERFCOUNTER14_CFG__PERF_SEL_MASK                                                           0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER14_CFG__PERF_SEL_END_MASK                                                       0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER14_CFG__PERF_MODE_MASK                                                          0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER14_CFG__ENABLE_MASK                                                             0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER14_CFG__CLEAR_MASK                                                              0x20000000L
+//GCMC_VM_L2_PERFCOUNTER15_CFG
+#define GCMC_VM_L2_PERFCOUNTER15_CFG__PERF_SEL__SHIFT                                                         0x0
+#define GCMC_VM_L2_PERFCOUNTER15_CFG__PERF_SEL_END__SHIFT                                                     0x8
+#define GCMC_VM_L2_PERFCOUNTER15_CFG__PERF_MODE__SHIFT                                                        0x18
+#define GCMC_VM_L2_PERFCOUNTER15_CFG__ENABLE__SHIFT                                                           0x1c
+#define GCMC_VM_L2_PERFCOUNTER15_CFG__CLEAR__SHIFT                                                            0x1d
+#define GCMC_VM_L2_PERFCOUNTER15_CFG__PERF_SEL_MASK                                                           0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER15_CFG__PERF_SEL_END_MASK                                                       0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER15_CFG__PERF_MODE_MASK                                                          0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER15_CFG__ENABLE_MASK                                                             0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER15_CFG__CLEAR_MASK                                                              0x20000000L
+//GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                0x8
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                 0x10
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                   0x18
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                    0x19
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                         0x1a
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                            0x0000000FL
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                  0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                   0x00FF0000L
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                     0x01000000L
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                      0x02000000L
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                           0x04000000L
+//GCUTCL2_PERFCOUNTER0_CFG
+#define GCUTCL2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                             0x0
+#define GCUTCL2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                         0x8
+#define GCUTCL2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                            0x18
+#define GCUTCL2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                               0x1c
+#define GCUTCL2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                0x1d
+#define GCUTCL2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                               0x000000FFL
+#define GCUTCL2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                           0x0000FF00L
+#define GCUTCL2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                              0x0F000000L
+#define GCUTCL2_PERFCOUNTER0_CFG__ENABLE_MASK                                                                 0x10000000L
+#define GCUTCL2_PERFCOUNTER0_CFG__CLEAR_MASK                                                                  0x20000000L
+//GCUTCL2_PERFCOUNTER1_CFG
+#define GCUTCL2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                             0x0
+#define GCUTCL2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                         0x8
+#define GCUTCL2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                            0x18
+#define GCUTCL2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                               0x1c
+#define GCUTCL2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                0x1d
+#define GCUTCL2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                               0x000000FFL
+#define GCUTCL2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                           0x0000FF00L
+#define GCUTCL2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                              0x0F000000L
+#define GCUTCL2_PERFCOUNTER1_CFG__ENABLE_MASK                                                                 0x10000000L
+#define GCUTCL2_PERFCOUNTER1_CFG__CLEAR_MASK                                                                  0x20000000L
+//GCUTCL2_PERFCOUNTER2_CFG
+#define GCUTCL2_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                             0x0
+#define GCUTCL2_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                         0x8
+#define GCUTCL2_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                            0x18
+#define GCUTCL2_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                               0x1c
+#define GCUTCL2_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                                0x1d
+#define GCUTCL2_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                               0x000000FFL
+#define GCUTCL2_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                           0x0000FF00L
+#define GCUTCL2_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                              0x0F000000L
+#define GCUTCL2_PERFCOUNTER2_CFG__ENABLE_MASK                                                                 0x10000000L
+#define GCUTCL2_PERFCOUNTER2_CFG__CLEAR_MASK                                                                  0x20000000L
+//GCUTCL2_PERFCOUNTER3_CFG
+#define GCUTCL2_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                             0x0
+#define GCUTCL2_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                         0x8
+#define GCUTCL2_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                            0x18
+#define GCUTCL2_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                               0x1c
+#define GCUTCL2_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                                0x1d
+#define GCUTCL2_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                               0x000000FFL
+#define GCUTCL2_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                           0x0000FF00L
+#define GCUTCL2_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                              0x0F000000L
+#define GCUTCL2_PERFCOUNTER3_CFG__ENABLE_MASK                                                                 0x10000000L
+#define GCUTCL2_PERFCOUNTER3_CFG__CLEAR_MASK                                                                  0x20000000L
+//GCUTCL2_PERFCOUNTER_RSLT_CNTL
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                             0x0
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                   0x8
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                    0x10
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                      0x18
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                       0x19
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                            0x1a
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                               0x0000000FL
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                     0x0000FF00L
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                      0x00FF0000L
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                        0x01000000L
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                         0x02000000L
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                              0x04000000L
+//GCUTCL2_PERFCOUNTER_EVENTS_GROUP_SELECT
+#define GCUTCL2_PERFCOUNTER_EVENTS_GROUP_SELECT__SET0_GROUP__SHIFT                                            0x0
+#define GCUTCL2_PERFCOUNTER_EVENTS_GROUP_SELECT__SET1_GROUP__SHIFT                                            0x8
+#define GCUTCL2_PERFCOUNTER_EVENTS_GROUP_SELECT__SET0_GROUP_MASK                                              0x000000FFL
+#define GCUTCL2_PERFCOUNTER_EVENTS_GROUP_SELECT__SET1_GROUP_MASK                                              0x0000FF00L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcatcl2pldec
+//GC_ATC_L2_PERFCOUNTER0_CFG
+#define GC_ATC_L2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                           0x0
+#define GC_ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                       0x8
+#define GC_ATC_L2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                          0x18
+#define GC_ATC_L2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                             0x1c
+#define GC_ATC_L2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                              0x1d
+#define GC_ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                             0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                         0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                            0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER0_CFG__ENABLE_MASK                                                               0x10000000L
+#define GC_ATC_L2_PERFCOUNTER0_CFG__CLEAR_MASK                                                                0x20000000L
+//GC_ATC_L2_PERFCOUNTER1_CFG
+#define GC_ATC_L2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                           0x0
+#define GC_ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                       0x8
+#define GC_ATC_L2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                          0x18
+#define GC_ATC_L2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                             0x1c
+#define GC_ATC_L2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                              0x1d
+#define GC_ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                             0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                         0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                            0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER1_CFG__ENABLE_MASK                                                               0x10000000L
+#define GC_ATC_L2_PERFCOUNTER1_CFG__CLEAR_MASK                                                                0x20000000L
+//GC_ATC_L2_PERFCOUNTER2_CFG
+#define GC_ATC_L2_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                           0x0
+#define GC_ATC_L2_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                       0x8
+#define GC_ATC_L2_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                          0x18
+#define GC_ATC_L2_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                             0x1c
+#define GC_ATC_L2_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                              0x1d
+#define GC_ATC_L2_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                             0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                         0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                            0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER2_CFG__ENABLE_MASK                                                               0x10000000L
+#define GC_ATC_L2_PERFCOUNTER2_CFG__CLEAR_MASK                                                                0x20000000L
+//GC_ATC_L2_PERFCOUNTER3_CFG
+#define GC_ATC_L2_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                           0x0
+#define GC_ATC_L2_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                       0x8
+#define GC_ATC_L2_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                          0x18
+#define GC_ATC_L2_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                             0x1c
+#define GC_ATC_L2_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                              0x1d
+#define GC_ATC_L2_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                             0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                         0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                            0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER3_CFG__ENABLE_MASK                                                               0x10000000L
+#define GC_ATC_L2_PERFCOUNTER3_CFG__CLEAR_MASK                                                                0x20000000L
+//GC_ATC_L2_PERFCOUNTER4_CFG
+#define GC_ATC_L2_PERFCOUNTER4_CFG__PERF_SEL__SHIFT                                                           0x0
+#define GC_ATC_L2_PERFCOUNTER4_CFG__PERF_SEL_END__SHIFT                                                       0x8
+#define GC_ATC_L2_PERFCOUNTER4_CFG__PERF_MODE__SHIFT                                                          0x18
+#define GC_ATC_L2_PERFCOUNTER4_CFG__ENABLE__SHIFT                                                             0x1c
+#define GC_ATC_L2_PERFCOUNTER4_CFG__CLEAR__SHIFT                                                              0x1d
+#define GC_ATC_L2_PERFCOUNTER4_CFG__PERF_SEL_MASK                                                             0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER4_CFG__PERF_SEL_END_MASK                                                         0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER4_CFG__PERF_MODE_MASK                                                            0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER4_CFG__ENABLE_MASK                                                               0x10000000L
+#define GC_ATC_L2_PERFCOUNTER4_CFG__CLEAR_MASK                                                                0x20000000L
+//GC_ATC_L2_PERFCOUNTER5_CFG
+#define GC_ATC_L2_PERFCOUNTER5_CFG__PERF_SEL__SHIFT                                                           0x0
+#define GC_ATC_L2_PERFCOUNTER5_CFG__PERF_SEL_END__SHIFT                                                       0x8
+#define GC_ATC_L2_PERFCOUNTER5_CFG__PERF_MODE__SHIFT                                                          0x18
+#define GC_ATC_L2_PERFCOUNTER5_CFG__ENABLE__SHIFT                                                             0x1c
+#define GC_ATC_L2_PERFCOUNTER5_CFG__CLEAR__SHIFT                                                              0x1d
+#define GC_ATC_L2_PERFCOUNTER5_CFG__PERF_SEL_MASK                                                             0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER5_CFG__PERF_SEL_END_MASK                                                         0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER5_CFG__PERF_MODE_MASK                                                            0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER5_CFG__ENABLE_MASK                                                               0x10000000L
+#define GC_ATC_L2_PERFCOUNTER5_CFG__CLEAR_MASK                                                                0x20000000L
+//GC_ATC_L2_PERFCOUNTER6_CFG
+#define GC_ATC_L2_PERFCOUNTER6_CFG__PERF_SEL__SHIFT                                                           0x0
+#define GC_ATC_L2_PERFCOUNTER6_CFG__PERF_SEL_END__SHIFT                                                       0x8
+#define GC_ATC_L2_PERFCOUNTER6_CFG__PERF_MODE__SHIFT                                                          0x18
+#define GC_ATC_L2_PERFCOUNTER6_CFG__ENABLE__SHIFT                                                             0x1c
+#define GC_ATC_L2_PERFCOUNTER6_CFG__CLEAR__SHIFT                                                              0x1d
+#define GC_ATC_L2_PERFCOUNTER6_CFG__PERF_SEL_MASK                                                             0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER6_CFG__PERF_SEL_END_MASK                                                         0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER6_CFG__PERF_MODE_MASK                                                            0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER6_CFG__ENABLE_MASK                                                               0x10000000L
+#define GC_ATC_L2_PERFCOUNTER6_CFG__CLEAR_MASK                                                                0x20000000L
+//GC_ATC_L2_PERFCOUNTER7_CFG
+#define GC_ATC_L2_PERFCOUNTER7_CFG__PERF_SEL__SHIFT                                                           0x0
+#define GC_ATC_L2_PERFCOUNTER7_CFG__PERF_SEL_END__SHIFT                                                       0x8
+#define GC_ATC_L2_PERFCOUNTER7_CFG__PERF_MODE__SHIFT                                                          0x18
+#define GC_ATC_L2_PERFCOUNTER7_CFG__ENABLE__SHIFT                                                             0x1c
+#define GC_ATC_L2_PERFCOUNTER7_CFG__CLEAR__SHIFT                                                              0x1d
+#define GC_ATC_L2_PERFCOUNTER7_CFG__PERF_SEL_MASK                                                             0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER7_CFG__PERF_SEL_END_MASK                                                         0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER7_CFG__PERF_MODE_MASK                                                            0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER7_CFG__ENABLE_MASK                                                               0x10000000L
+#define GC_ATC_L2_PERFCOUNTER7_CFG__CLEAR_MASK                                                                0x20000000L
+//GC_ATC_L2_PERFCOUNTER8_CFG
+#define GC_ATC_L2_PERFCOUNTER8_CFG__PERF_SEL__SHIFT                                                           0x0
+#define GC_ATC_L2_PERFCOUNTER8_CFG__PERF_SEL_END__SHIFT                                                       0x8
+#define GC_ATC_L2_PERFCOUNTER8_CFG__PERF_MODE__SHIFT                                                          0x18
+#define GC_ATC_L2_PERFCOUNTER8_CFG__ENABLE__SHIFT                                                             0x1c
+#define GC_ATC_L2_PERFCOUNTER8_CFG__CLEAR__SHIFT                                                              0x1d
+#define GC_ATC_L2_PERFCOUNTER8_CFG__PERF_SEL_MASK                                                             0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER8_CFG__PERF_SEL_END_MASK                                                         0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER8_CFG__PERF_MODE_MASK                                                            0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER8_CFG__ENABLE_MASK                                                               0x10000000L
+#define GC_ATC_L2_PERFCOUNTER8_CFG__CLEAR_MASK                                                                0x20000000L
+//GC_ATC_L2_PERFCOUNTER9_CFG
+#define GC_ATC_L2_PERFCOUNTER9_CFG__PERF_SEL__SHIFT                                                           0x0
+#define GC_ATC_L2_PERFCOUNTER9_CFG__PERF_SEL_END__SHIFT                                                       0x8
+#define GC_ATC_L2_PERFCOUNTER9_CFG__PERF_MODE__SHIFT                                                          0x18
+#define GC_ATC_L2_PERFCOUNTER9_CFG__ENABLE__SHIFT                                                             0x1c
+#define GC_ATC_L2_PERFCOUNTER9_CFG__CLEAR__SHIFT                                                              0x1d
+#define GC_ATC_L2_PERFCOUNTER9_CFG__PERF_SEL_MASK                                                             0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER9_CFG__PERF_SEL_END_MASK                                                         0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER9_CFG__PERF_MODE_MASK                                                            0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER9_CFG__ENABLE_MASK                                                               0x10000000L
+#define GC_ATC_L2_PERFCOUNTER9_CFG__CLEAR_MASK                                                                0x20000000L
+//GC_ATC_L2_PERFCOUNTER10_CFG
+#define GC_ATC_L2_PERFCOUNTER10_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GC_ATC_L2_PERFCOUNTER10_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GC_ATC_L2_PERFCOUNTER10_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GC_ATC_L2_PERFCOUNTER10_CFG__ENABLE__SHIFT                                                            0x1c
+#define GC_ATC_L2_PERFCOUNTER10_CFG__CLEAR__SHIFT                                                             0x1d
+#define GC_ATC_L2_PERFCOUNTER10_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER10_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER10_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER10_CFG__ENABLE_MASK                                                              0x10000000L
+#define GC_ATC_L2_PERFCOUNTER10_CFG__CLEAR_MASK                                                               0x20000000L
+//GC_ATC_L2_PERFCOUNTER11_CFG
+#define GC_ATC_L2_PERFCOUNTER11_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GC_ATC_L2_PERFCOUNTER11_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GC_ATC_L2_PERFCOUNTER11_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GC_ATC_L2_PERFCOUNTER11_CFG__ENABLE__SHIFT                                                            0x1c
+#define GC_ATC_L2_PERFCOUNTER11_CFG__CLEAR__SHIFT                                                             0x1d
+#define GC_ATC_L2_PERFCOUNTER11_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER11_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER11_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER11_CFG__ENABLE_MASK                                                              0x10000000L
+#define GC_ATC_L2_PERFCOUNTER11_CFG__CLEAR_MASK                                                               0x20000000L
+//GC_ATC_L2_PERFCOUNTER12_CFG
+#define GC_ATC_L2_PERFCOUNTER12_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GC_ATC_L2_PERFCOUNTER12_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GC_ATC_L2_PERFCOUNTER12_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GC_ATC_L2_PERFCOUNTER12_CFG__ENABLE__SHIFT                                                            0x1c
+#define GC_ATC_L2_PERFCOUNTER12_CFG__CLEAR__SHIFT                                                             0x1d
+#define GC_ATC_L2_PERFCOUNTER12_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER12_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER12_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER12_CFG__ENABLE_MASK                                                              0x10000000L
+#define GC_ATC_L2_PERFCOUNTER12_CFG__CLEAR_MASK                                                               0x20000000L
+//GC_ATC_L2_PERFCOUNTER13_CFG
+#define GC_ATC_L2_PERFCOUNTER13_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GC_ATC_L2_PERFCOUNTER13_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GC_ATC_L2_PERFCOUNTER13_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GC_ATC_L2_PERFCOUNTER13_CFG__ENABLE__SHIFT                                                            0x1c
+#define GC_ATC_L2_PERFCOUNTER13_CFG__CLEAR__SHIFT                                                             0x1d
+#define GC_ATC_L2_PERFCOUNTER13_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER13_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER13_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER13_CFG__ENABLE_MASK                                                              0x10000000L
+#define GC_ATC_L2_PERFCOUNTER13_CFG__CLEAR_MASK                                                               0x20000000L
+//GC_ATC_L2_PERFCOUNTER14_CFG
+#define GC_ATC_L2_PERFCOUNTER14_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GC_ATC_L2_PERFCOUNTER14_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GC_ATC_L2_PERFCOUNTER14_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GC_ATC_L2_PERFCOUNTER14_CFG__ENABLE__SHIFT                                                            0x1c
+#define GC_ATC_L2_PERFCOUNTER14_CFG__CLEAR__SHIFT                                                             0x1d
+#define GC_ATC_L2_PERFCOUNTER14_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER14_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER14_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER14_CFG__ENABLE_MASK                                                              0x10000000L
+#define GC_ATC_L2_PERFCOUNTER14_CFG__CLEAR_MASK                                                               0x20000000L
+//GC_ATC_L2_PERFCOUNTER15_CFG
+#define GC_ATC_L2_PERFCOUNTER15_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GC_ATC_L2_PERFCOUNTER15_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GC_ATC_L2_PERFCOUNTER15_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GC_ATC_L2_PERFCOUNTER15_CFG__ENABLE__SHIFT                                                            0x1c
+#define GC_ATC_L2_PERFCOUNTER15_CFG__CLEAR__SHIFT                                                             0x1d
+#define GC_ATC_L2_PERFCOUNTER15_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER15_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER15_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER15_CFG__ENABLE_MASK                                                              0x10000000L
+#define GC_ATC_L2_PERFCOUNTER15_CFG__CLEAR_MASK                                                               0x20000000L
+//GC_ATC_L2_PERFCOUNTER_RSLT_CNTL
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                           0x0
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                 0x8
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                  0x10
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                    0x18
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                     0x19
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                          0x1a
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                             0x0000000FL
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                   0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                    0x00FF0000L
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                      0x01000000L
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                       0x02000000L
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                            0x04000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcl2tlbpldec
+//GC_L2TLB_PERFCOUNTER0_CFG
+#define GC_L2TLB_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                            0x0
+#define GC_L2TLB_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define GC_L2TLB_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                           0x18
+#define GC_L2TLB_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                              0x1c
+#define GC_L2TLB_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                               0x1d
+#define GC_L2TLB_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define GC_L2TLB_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define GC_L2TLB_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define GC_L2TLB_PERFCOUNTER0_CFG__ENABLE_MASK                                                                0x10000000L
+#define GC_L2TLB_PERFCOUNTER0_CFG__CLEAR_MASK                                                                 0x20000000L
+//GC_L2TLB_PERFCOUNTER1_CFG
+#define GC_L2TLB_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                            0x0
+#define GC_L2TLB_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define GC_L2TLB_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                           0x18
+#define GC_L2TLB_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                              0x1c
+#define GC_L2TLB_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                               0x1d
+#define GC_L2TLB_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define GC_L2TLB_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define GC_L2TLB_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define GC_L2TLB_PERFCOUNTER1_CFG__ENABLE_MASK                                                                0x10000000L
+#define GC_L2TLB_PERFCOUNTER1_CFG__CLEAR_MASK                                                                 0x20000000L
+//GC_L2TLB_PERFCOUNTER2_CFG
+#define GC_L2TLB_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                            0x0
+#define GC_L2TLB_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define GC_L2TLB_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                           0x18
+#define GC_L2TLB_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                              0x1c
+#define GC_L2TLB_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                               0x1d
+#define GC_L2TLB_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define GC_L2TLB_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define GC_L2TLB_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define GC_L2TLB_PERFCOUNTER2_CFG__ENABLE_MASK                                                                0x10000000L
+#define GC_L2TLB_PERFCOUNTER2_CFG__CLEAR_MASK                                                                 0x20000000L
+//GC_L2TLB_PERFCOUNTER3_CFG
+#define GC_L2TLB_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                            0x0
+#define GC_L2TLB_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                        0x8
+#define GC_L2TLB_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                           0x18
+#define GC_L2TLB_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                              0x1c
+#define GC_L2TLB_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                               0x1d
+#define GC_L2TLB_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                              0x000000FFL
+#define GC_L2TLB_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
+#define GC_L2TLB_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                             0x0F000000L
+#define GC_L2TLB_PERFCOUNTER3_CFG__ENABLE_MASK                                                                0x10000000L
+#define GC_L2TLB_PERFCOUNTER3_CFG__CLEAR_MASK                                                                 0x20000000L
+//GC_L2TLB_PERFCOUNTER_RSLT_CNTL
+#define GC_L2TLB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                            0x0
+#define GC_L2TLB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                  0x8
+#define GC_L2TLB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                   0x10
+#define GC_L2TLB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                     0x18
+#define GC_L2TLB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                      0x19
+#define GC_L2TLB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                           0x1a
+#define GC_L2TLB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                              0x0000000FL
+#define GC_L2TLB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                    0x0000FF00L
+#define GC_L2TLB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                     0x00FF0000L
+#define GC_L2TLB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                       0x01000000L
+#define GC_L2TLB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                        0x02000000L
+#define GC_L2TLB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                             0x04000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvmsharedhvdec
+//GCVM_PCIE_ATS_CNTL
+#define GCVM_PCIE_ATS_CNTL__STU__SHIFT                                                                        0x10
+#define GCVM_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                                 0x1f
+#define GCVM_PCIE_ATS_CNTL__STU_MASK                                                                          0x001F0000L
+#define GCVM_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                                   0x80000000L
+//GCMC_VM_NB_MMIOBASE
+#define GCMC_VM_NB_MMIOBASE__MMIOBASE__SHIFT                                                                  0x0
+#define GCMC_VM_NB_MMIOBASE__MMIOBASE_MASK                                                                    0xFFFFFFFFL
+//GCMC_VM_NB_MMIOLIMIT
+#define GCMC_VM_NB_MMIOLIMIT__MMIOLIMIT__SHIFT                                                                0x0
+#define GCMC_VM_NB_MMIOLIMIT__MMIOLIMIT_MASK                                                                  0xFFFFFFFFL
+//GCMC_VM_NB_PCI_CTRL
+#define GCMC_VM_NB_PCI_CTRL__MMIOENABLE__SHIFT                                                                0x17
+#define GCMC_VM_NB_PCI_CTRL__MMIOENABLE_MASK                                                                  0x00800000L
+//GCMC_VM_NB_PCI_ARB
+#define GCMC_VM_NB_PCI_ARB__VGA_HOLE__SHIFT                                                                   0x3
+#define GCMC_VM_NB_PCI_ARB__VGA_HOLE_MASK                                                                     0x00000008L
+//GCMC_VM_NB_TOP_OF_DRAM_SLOT1
+#define GCMC_VM_NB_TOP_OF_DRAM_SLOT1__TOP_OF_DRAM__SHIFT                                                      0x17
+#define GCMC_VM_NB_TOP_OF_DRAM_SLOT1__TOP_OF_DRAM_MASK                                                        0xFF800000L
+//GCMC_VM_NB_LOWER_TOP_OF_DRAM2
+#define GCMC_VM_NB_LOWER_TOP_OF_DRAM2__ENABLE__SHIFT                                                          0x0
+#define GCMC_VM_NB_LOWER_TOP_OF_DRAM2__LOWER_TOM2__SHIFT                                                      0x17
+#define GCMC_VM_NB_LOWER_TOP_OF_DRAM2__ENABLE_MASK                                                            0x00000001L
+#define GCMC_VM_NB_LOWER_TOP_OF_DRAM2__LOWER_TOM2_MASK                                                        0xFF800000L
+//GCMC_VM_NB_UPPER_TOP_OF_DRAM2
+#define GCMC_VM_NB_UPPER_TOP_OF_DRAM2__UPPER_TOM2__SHIFT                                                      0x0
+#define GCMC_VM_NB_UPPER_TOP_OF_DRAM2__UPPER_TOM2_MASK                                                        0x000FFFFFL
+//GCMC_VM_STEERING
+#define GCMC_VM_STEERING__DEFAULT_STEERING__SHIFT                                                             0x0
+#define GCMC_VM_STEERING__DEFAULT_STEERING_MASK                                                               0x00000003L
+//GCMC_SHARED_VIRT_RESET_REQ
+#define GCMC_SHARED_VIRT_RESET_REQ__VF__SHIFT                                                                 0x0
+#define GCMC_SHARED_VIRT_RESET_REQ__PF__SHIFT                                                                 0x8
+#define GCMC_SHARED_VIRT_RESET_REQ__VF_MASK                                                                   0x000000FFL
+#define GCMC_SHARED_VIRT_RESET_REQ__PF_MASK                                                                   0x00000100L
+//GCMC_VM_XGMI_LFB_CNTL
+#define GCMC_VM_XGMI_LFB_CNTL__PF_LFB_REGION__SHIFT                                                           0x0
+#define GCMC_VM_XGMI_LFB_CNTL__PF_MAX_REGION__SHIFT                                                           0x4
+#define GCMC_VM_XGMI_LFB_CNTL__PF_LFB_REGION_MASK                                                             0x0000000FL
+#define GCMC_VM_XGMI_LFB_CNTL__PF_MAX_REGION_MASK                                                             0x000000F0L
+//GCMC_VM_XGMI_LFB_SIZE
+#define GCMC_VM_XGMI_LFB_SIZE__PF_LFB_SIZE__SHIFT                                                             0x0
+#define GCMC_VM_XGMI_LFB_SIZE__PF_LFB_SIZE_MASK                                                               0x0001FFFFL
+//GCMC_VM_HOST_MAPPING
+#define GCMC_VM_HOST_MAPPING__MODE__SHIFT                                                                     0x0
+#define GCMC_VM_HOST_MAPPING__MODE_MASK                                                                       0x00000001L
+//GCMC_VM_XGMI_GPUIOV_ENABLE
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF0__SHIFT                                                         0x0
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF1__SHIFT                                                         0x1
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF2__SHIFT                                                         0x2
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF3__SHIFT                                                         0x3
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF4__SHIFT                                                         0x4
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF5__SHIFT                                                         0x5
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF6__SHIFT                                                         0x6
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF7__SHIFT                                                         0x7
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_PF__SHIFT                                                          0x1f
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF0_MASK                                                           0x00000001L
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF1_MASK                                                           0x00000002L
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF2_MASK                                                           0x00000004L
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF3_MASK                                                           0x00000008L
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF4_MASK                                                           0x00000010L
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF5_MASK                                                           0x00000020L
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF6_MASK                                                           0x00000040L
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF7_MASK                                                           0x00000080L
+#define GCMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_PF_MASK                                                            0x80000000L
+//GCMC_VM_CACHEABLE_DRAM_ADDRESS_START
+#define GCMC_VM_CACHEABLE_DRAM_ADDRESS_START__ADDRESS__SHIFT                                                  0x0
+#define GCMC_VM_CACHEABLE_DRAM_ADDRESS_START__ADDRESS_MASK                                                    0x0FFFFFFFL
+//GCMC_VM_CACHEABLE_DRAM_ADDRESS_END
+#define GCMC_VM_CACHEABLE_DRAM_ADDRESS_END__ADDRESS__SHIFT                                                    0x0
+#define GCMC_VM_CACHEABLE_DRAM_ADDRESS_END__ADDRESS_MASK                                                      0x0FFFFFFFL
+//GCMC_VM_LOCAL_SYSMEM_ADDRESS_START
+#define GCMC_VM_LOCAL_SYSMEM_ADDRESS_START__ADDRESS__SHIFT                                                    0x0
+#define GCMC_VM_LOCAL_SYSMEM_ADDRESS_START__ADDRESS_MASK                                                      0x0FFFFFFFL
+//GCMC_VM_LOCAL_SYSMEM_ADDRESS_END
+#define GCMC_VM_LOCAL_SYSMEM_ADDRESS_END__ADDRESS__SHIFT                                                      0x0
+#define GCMC_VM_LOCAL_SYSMEM_ADDRESS_END__ADDRESS_MASK                                                        0x0FFFFFFFL
+//GCMC_VM_LPDDR_ADDRESS_START
+#define GCMC_VM_LPDDR_ADDRESS_START__ADDRESS__SHIFT                                                           0x0
+#define GCMC_VM_LPDDR_ADDRESS_START__ADDRESS_MASK                                                             0x0FFFFFFFL
+//GCMC_VM_LPDDR_ADDRESS_END
+#define GCMC_VM_LPDDR_ADDRESS_END__ADDRESS__SHIFT                                                             0x0
+#define GCMC_VM_LPDDR_ADDRESS_END__ADDRESS_MASK                                                               0x0FFFFFFFL
+//GCMC_VM_APT_CNTL
+#define GCMC_VM_APT_CNTL__FORCE_MTYPE_UC__SHIFT                                                               0x0
+#define GCMC_VM_APT_CNTL__DIRECT_SYSTEM_EN__SHIFT                                                             0x1
+#define GCMC_VM_APT_CNTL__FRAG_APT_INTXN_MODE__SHIFT                                                          0x2
+#define GCMC_VM_APT_CNTL__CHECK_IS_LOCAL__SHIFT                                                               0x4
+#define GCMC_VM_APT_CNTL__CAP_FRAG_SIZE_2M__SHIFT                                                             0x5
+#define GCMC_VM_APT_CNTL__LOCAL_SYSMEM_APERTURE_CNTL__SHIFT                                                   0x6
+#define GCMC_VM_APT_CNTL__LPDDR_APERTURE_CNTL__SHIFT                                                          0x8
+#define GCMC_VM_APT_CNTL__FORCE_MTYPE_UC_MASK                                                                 0x00000001L
+#define GCMC_VM_APT_CNTL__DIRECT_SYSTEM_EN_MASK                                                               0x00000002L
+#define GCMC_VM_APT_CNTL__FRAG_APT_INTXN_MODE_MASK                                                            0x0000000CL
+#define GCMC_VM_APT_CNTL__CHECK_IS_LOCAL_MASK                                                                 0x00000010L
+#define GCMC_VM_APT_CNTL__CAP_FRAG_SIZE_2M_MASK                                                               0x00000020L
+#define GCMC_VM_APT_CNTL__LOCAL_SYSMEM_APERTURE_CNTL_MASK                                                     0x000000C0L
+#define GCMC_VM_APT_CNTL__LPDDR_APERTURE_CNTL_MASK                                                            0x00000300L
+//GCUTCL2_HARVEST_BYPASS_GROUPS
+#define GCUTCL2_HARVEST_BYPASS_GROUPS__BYPASS_GROUPS__SHIFT                                                   0x0
+#define GCUTCL2_HARVEST_BYPASS_GROUPS__BYPASS_GROUPS_MASK                                                     0xFFFFFFFFL
+//GCUTCL2_ROUTER_DEBUG
+#define GCUTCL2_ROUTER_DEBUG__FLAGS__SHIFT                                                                    0x0
+#define GCUTCL2_ROUTER_DEBUG__FLAGS_MASK                                                                      0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcatcl2dec
+//GC_ATC_L2_CNTL
+#define GC_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READ_REQUESTS__SHIFT                                            0x0
+#define GC_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITE_REQUESTS__SHIFT                                           0x2
+#define GC_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD__SHIFT                                0x4
+#define GC_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD__SHIFT                               0x5
+#define GC_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READ_REQUESTS__SHIFT                                       0x6
+#define GC_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITE_REQUESTS__SHIFT                                      0x8
+#define GC_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD__SHIFT                           0xa
+#define GC_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD__SHIFT                          0xb
+#define GC_ATC_L2_CNTL__CACHE_INVALIDATE_MODE__SHIFT                                                          0xc
+#define GC_ATC_L2_CNTL__FRAG_APT_INTXN_MODE__SHIFT                                                            0xf
+#define GC_ATC_L2_CNTL__CLI_GPA_REQ_FRAG_SIZE__SHIFT                                                          0x11
+#define GC_ATC_L2_CNTL__CLI_GVA_REQ_FRAG_SIZE__SHIFT                                                          0x17
+#define GC_ATC_L2_CNTL__FORCE_8T_CAP_ON_AT_RETURN__SHIFT                                                      0x1d
+#define GC_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READ_REQUESTS_MASK                                              0x00000003L
+#define GC_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITE_REQUESTS_MASK                                             0x0000000CL
+#define GC_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD_MASK                                  0x00000010L
+#define GC_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD_MASK                                 0x00000020L
+#define GC_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READ_REQUESTS_MASK                                         0x000000C0L
+#define GC_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITE_REQUESTS_MASK                                        0x00000300L
+#define GC_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD_MASK                             0x00000400L
+#define GC_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD_MASK                            0x00000800L
+#define GC_ATC_L2_CNTL__CACHE_INVALIDATE_MODE_MASK                                                            0x00007000L
+#define GC_ATC_L2_CNTL__FRAG_APT_INTXN_MODE_MASK                                                              0x00018000L
+#define GC_ATC_L2_CNTL__CLI_GPA_REQ_FRAG_SIZE_MASK                                                            0x007E0000L
+#define GC_ATC_L2_CNTL__CLI_GVA_REQ_FRAG_SIZE_MASK                                                            0x1F800000L
+#define GC_ATC_L2_CNTL__FORCE_8T_CAP_ON_AT_RETURN_MASK                                                        0x20000000L
+//GC_ATC_L2_CNTL2
+#define GC_ATC_L2_CNTL2__BANK_SELECT__SHIFT                                                                   0x0
+#define GC_ATC_L2_CNTL2__NUM_BANKS_LOG2__SHIFT                                                                0x6
+#define GC_ATC_L2_CNTL2__L2_CACHE_UPDATE_MODE__SHIFT                                                          0x9
+#define GC_ATC_L2_CNTL2__ENABLE_L2_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                           0xb
+#define GC_ATC_L2_CNTL2__L2_CACHE_SWAP_TAG_INDEX_LSBS__SHIFT                                                  0xc
+#define GC_ATC_L2_CNTL2__L2_CACHE_VMID_MODE__SHIFT                                                            0xf
+#define GC_ATC_L2_CNTL2__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE__SHIFT                                      0x12
+#define GC_ATC_L2_CNTL2__BANK_SELECT_MASK                                                                     0x0000003FL
+#define GC_ATC_L2_CNTL2__NUM_BANKS_LOG2_MASK                                                                  0x000001C0L
+#define GC_ATC_L2_CNTL2__L2_CACHE_UPDATE_MODE_MASK                                                            0x00000600L
+#define GC_ATC_L2_CNTL2__ENABLE_L2_CACHE_LRU_UPDATE_BY_WRITE_MASK                                             0x00000800L
+#define GC_ATC_L2_CNTL2__L2_CACHE_SWAP_TAG_INDEX_LSBS_MASK                                                    0x00007000L
+#define GC_ATC_L2_CNTL2__L2_CACHE_VMID_MODE_MASK                                                              0x00038000L
+#define GC_ATC_L2_CNTL2__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE_MASK                                        0x00FC0000L
+//GC_ATC_L2_CACHE_DATA0
+#define GC_ATC_L2_CACHE_DATA0__DATA_REGISTER_VALID__SHIFT                                                     0x0
+#define GC_ATC_L2_CACHE_DATA0__CACHE_ENTRY_VALID__SHIFT                                                       0x1
+#define GC_ATC_L2_CACHE_DATA0__CACHED_ATTRIBUTES__SHIFT                                                       0x2
+#define GC_ATC_L2_CACHE_DATA0__DATA_REGISTER_VALID_MASK                                                       0x00000001L
+#define GC_ATC_L2_CACHE_DATA0__CACHE_ENTRY_VALID_MASK                                                         0x00000002L
+#define GC_ATC_L2_CACHE_DATA0__CACHED_ATTRIBUTES_MASK                                                         0x00FFFFFCL
+//GC_ATC_L2_CACHE_DATA1
+#define GC_ATC_L2_CACHE_DATA1__VIRTUAL_PAGE_ADDRESS_LOW__SHIFT                                                0x0
+#define GC_ATC_L2_CACHE_DATA1__VIRTUAL_PAGE_ADDRESS_LOW_MASK                                                  0xFFFFFFFFL
+//GC_ATC_L2_CACHE_DATA2
+#define GC_ATC_L2_CACHE_DATA2__PHYSICAL_PAGE_ADDRESS__SHIFT                                                   0x0
+#define GC_ATC_L2_CACHE_DATA2__PHYSICAL_PAGE_ADDRESS_MASK                                                     0xFFFFFFFFL
+//GC_ATC_L2_CACHE_DATA3
+#define GC_ATC_L2_CACHE_DATA3__PHYSICAL_PAGE_ADDRESS__SHIFT                                                   0x0
+#define GC_ATC_L2_CACHE_DATA3__PHYSICAL_PAGE_ADDRESS_MASK                                                     0xFFFFFFFFL
+//GC_ATC_L2_CACHE_DATA4
+#define GC_ATC_L2_CACHE_DATA4__VIRTUAL_PAGE_ADDRESS_HIGH__SHIFT                                               0x0
+#define GC_ATC_L2_CACHE_DATA4__VIRTUAL_PAGE_ADDRESS_HIGH_MASK                                                 0xFFFFFFFFL
+//GC_ATC_L2_CNTL3
+#define GC_ATC_L2_CNTL3__L2_SMALLK_CACHE_FRAGMENT_SIZE__SHIFT                                                 0x0
+#define GC_ATC_L2_CNTL3__L2_MIDK_CACHE_FRAGMENT_SIZE__SHIFT                                                   0x6
+#define GC_ATC_L2_CNTL3__L2_BIGK_CACHE_FRAGMENT_SIZE__SHIFT                                                   0xc
+#define GC_ATC_L2_CNTL3__DELAY_SEND_INVALIDATION_REQUEST__SHIFT                                               0x12
+#define GC_ATC_L2_CNTL3__ATS_REQUEST_CREDIT_MINUS1__SHIFT                                                     0x15
+#define GC_ATC_L2_CNTL3__COMPCLKREQ_OFF_HYSTERESIS__SHIFT                                                     0x1b
+#define GC_ATC_L2_CNTL3__REPEATER_FGCG_OFF__SHIFT                                                             0x1e
+#define GC_ATC_L2_CNTL3__L2_SMALLK_CACHE_FRAGMENT_SIZE_MASK                                                   0x0000003FL
+#define GC_ATC_L2_CNTL3__L2_MIDK_CACHE_FRAGMENT_SIZE_MASK                                                     0x00000FC0L
+#define GC_ATC_L2_CNTL3__L2_BIGK_CACHE_FRAGMENT_SIZE_MASK                                                     0x0003F000L
+#define GC_ATC_L2_CNTL3__DELAY_SEND_INVALIDATION_REQUEST_MASK                                                 0x001C0000L
+#define GC_ATC_L2_CNTL3__ATS_REQUEST_CREDIT_MINUS1_MASK                                                       0x07E00000L
+#define GC_ATC_L2_CNTL3__COMPCLKREQ_OFF_HYSTERESIS_MASK                                                       0x38000000L
+#define GC_ATC_L2_CNTL3__REPEATER_FGCG_OFF_MASK                                                               0x40000000L
+//GC_ATC_L2_STATUS
+#define GC_ATC_L2_STATUS__BUSY__SHIFT                                                                         0x0
+#define GC_ATC_L2_STATUS__NO_OUTSTANDING_AT_REQUESTS__SHIFT                                                   0x1
+#define GC_ATC_L2_STATUS__BUSY_MASK                                                                           0x00000001L
+#define GC_ATC_L2_STATUS__NO_OUTSTANDING_AT_REQUESTS_MASK                                                     0x00000002L
+//GC_ATC_L2_MISC_CG
+#define GC_ATC_L2_MISC_CG__OFFDLY__SHIFT                                                                      0x6
+#define GC_ATC_L2_MISC_CG__ENABLE__SHIFT                                                                      0x12
+#define GC_ATC_L2_MISC_CG__MEM_LS_ENABLE__SHIFT                                                               0x13
+#define GC_ATC_L2_MISC_CG__OFFDLY_MASK                                                                        0x00000FC0L
+#define GC_ATC_L2_MISC_CG__ENABLE_MASK                                                                        0x00040000L
+#define GC_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK                                                                 0x00080000L
+//GC_ATC_L2_ICG_CTRL
+#define GC_ATC_L2_ICG_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x0
+#define GC_ATC_L2_ICG_CTRL__DYNAMIC_CLOCK_OVERRIDE__SHIFT                                                     0x4
+#define GC_ATC_L2_ICG_CTRL__STATIC_CLOCK_OVERRIDE__SHIFT                                                      0x5
+#define GC_ATC_L2_ICG_CTRL__AON_CLOCK_OVERRIDE__SHIFT                                                         0x6
+#define GC_ATC_L2_ICG_CTRL__PERFMON_CLOCK_OVERRIDE__SHIFT                                                     0x7
+#define GC_ATC_L2_ICG_CTRL__OFF_HYSTERESIS_MASK                                                               0x0000000FL
+#define GC_ATC_L2_ICG_CTRL__DYNAMIC_CLOCK_OVERRIDE_MASK                                                       0x00000010L
+#define GC_ATC_L2_ICG_CTRL__STATIC_CLOCK_OVERRIDE_MASK                                                        0x00000020L
+#define GC_ATC_L2_ICG_CTRL__AON_CLOCK_OVERRIDE_MASK                                                           0x00000040L
+#define GC_ATC_L2_ICG_CTRL__PERFMON_CLOCK_OVERRIDE_MASK                                                       0x00000080L
+//GC_ATC_L2_SDPPORT_CTRL
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPCKEN__SHIFT                                                      0x0
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPCKENRCV__SHIFT                                                   0x1
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPDATACKEN__SHIFT                                                  0x2
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPDATACKENRCV__SHIFT                                               0x3
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_WRRSPCKEN__SHIFT                                                      0x4
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_WRRSPCKENRCV__SHIFT                                                   0x5
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_REQCKEN__SHIFT                                                        0x6
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_REQCKENRCV__SHIFT                                                     0x7
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_ORIGDATACKEN__SHIFT                                                   0x8
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_ORIGDATACKENRCV__SHIFT                                                0x9
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPCKEN_MASK                                                        0x00000001L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPCKENRCV_MASK                                                     0x00000002L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPDATACKEN_MASK                                                    0x00000004L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPDATACKENRCV_MASK                                                 0x00000008L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_WRRSPCKEN_MASK                                                        0x00000010L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_WRRSPCKENRCV_MASK                                                     0x00000020L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_REQCKEN_MASK                                                          0x00000040L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_REQCKENRCV_MASK                                                       0x00000080L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_ORIGDATACKEN_MASK                                                     0x00000100L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_ORIGDATACKENRCV_MASK                                                  0x00000200L
+//GC_ATC_L2_MISC_DEBUG
+#define GC_ATC_L2_MISC_DEBUG__FLAGS__SHIFT                                                                    0x0
+#define GC_ATC_L2_MISC_DEBUG__FLAGS_MASK                                                                      0xFFFFFFFFL
+//GC_ATC_L2_BANK_SELECT_MASKS
+#define GC_ATC_L2_BANK_SELECT_MASKS__MASK0__SHIFT                                                             0x0
+#define GC_ATC_L2_BANK_SELECT_MASKS__MASK1__SHIFT                                                             0x4
+#define GC_ATC_L2_BANK_SELECT_MASKS__MASK2__SHIFT                                                             0x8
+#define GC_ATC_L2_BANK_SELECT_MASKS__MASK0_MASK                                                               0x0000000FL
+#define GC_ATC_L2_BANK_SELECT_MASKS__MASK1_MASK                                                               0x000000F0L
+#define GC_ATC_L2_BANK_SELECT_MASKS__MASK2_MASK                                                               0x00000F00L
+//GC_ATC_L2_FAULT_CNTL
+#define GC_ATC_L2_FAULT_CNTL__CLEAR_FAULT_STATUS_ADDR__SHIFT                                                  0x0
+#define GC_ATC_L2_FAULT_CNTL__ALLOW_SUBSEQUENT_FAULT_STATUS_ADDR_UPDATES__SHIFT                               0x1
+#define GC_ATC_L2_FAULT_CNTL__LOG_REQUEST_ADDRESS__SHIFT                                                      0x2
+#define GC_ATC_L2_FAULT_CNTL__CRASH_ON_NO_RETRY_FAULT__SHIFT                                                  0x3
+#define GC_ATC_L2_FAULT_CNTL__CRASH_ON_RETRY_FAULT__SHIFT                                                     0x4
+#define GC_ATC_L2_FAULT_CNTL__CLEAR_FAULT_STATUS_ADDR_MASK                                                    0x00000001L
+#define GC_ATC_L2_FAULT_CNTL__ALLOW_SUBSEQUENT_FAULT_STATUS_ADDR_UPDATES_MASK                                 0x00000002L
+#define GC_ATC_L2_FAULT_CNTL__LOG_REQUEST_ADDRESS_MASK                                                        0x00000004L
+#define GC_ATC_L2_FAULT_CNTL__CRASH_ON_NO_RETRY_FAULT_MASK                                                    0x00000008L
+#define GC_ATC_L2_FAULT_CNTL__CRASH_ON_RETRY_FAULT_MASK                                                       0x00000010L
+//GC_ATC_L2_FAULT_STATUS
+#define GC_ATC_L2_FAULT_STATUS__MORE_FAULTS__SHIFT                                                            0x0
+#define GC_ATC_L2_FAULT_STATUS__ERROR_TYPE__SHIFT                                                             0x1
+#define GC_ATC_L2_FAULT_STATUS__CID__SHIFT                                                                    0x5
+#define GC_ATC_L2_FAULT_STATUS__RWXA__SHIFT                                                                   0xe
+#define GC_ATC_L2_FAULT_STATUS__VMID__SHIFT                                                                   0x12
+#define GC_ATC_L2_FAULT_STATUS__VF__SHIFT                                                                     0x16
+#define GC_ATC_L2_FAULT_STATUS__VFID__SHIFT                                                                   0x17
+#define GC_ATC_L2_FAULT_STATUS__MORE_FAULTS_MASK                                                              0x00000001L
+#define GC_ATC_L2_FAULT_STATUS__ERROR_TYPE_MASK                                                               0x0000001EL
+#define GC_ATC_L2_FAULT_STATUS__CID_MASK                                                                      0x00003FE0L
+#define GC_ATC_L2_FAULT_STATUS__RWXA_MASK                                                                     0x0003C000L
+#define GC_ATC_L2_FAULT_STATUS__VMID_MASK                                                                     0x003C0000L
+#define GC_ATC_L2_FAULT_STATUS__VF_MASK                                                                       0x00400000L
+#define GC_ATC_L2_FAULT_STATUS__VFID_MASK                                                                     0x0F800000L
+//GC_ATC_L2_FAULT_ADDR_LO
+#define GC_ATC_L2_FAULT_ADDR_LO__LOGICAL_PAGE_ADDR__SHIFT                                                     0x0
+#define GC_ATC_L2_FAULT_ADDR_LO__LOGICAL_PAGE_ADDR_MASK                                                       0xFFFFFFFFL
+//GC_ATC_L2_FAULT_ADDR_HI
+#define GC_ATC_L2_FAULT_ADDR_HI__LOGICAL_PAGE_ADDR__SHIFT                                                     0x0
+#define GC_ATC_L2_FAULT_ADDR_HI__LOGICAL_PAGE_ADDR_MASK                                                       0xFFFFFFFFL
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcl2tlbhvdec
+//GC_L2TLB_TMZ_CNTL
+#define GC_L2TLB_TMZ_CNTL__TMZ_MODULATION__SHIFT                                                              0x0
+#define GC_L2TLB_TMZ_CNTL__TMZ_MODULATION_MASK                                                                0x00000001L
+//GC_UTCL2_L2TLB_CREDIT_SAFETY_FETCH_RDREQ
+#define GC_UTCL2_L2TLB_CREDIT_SAFETY_FETCH_RDREQ__CREDITS__SHIFT                                              0x0
+#define GC_UTCL2_L2TLB_CREDIT_SAFETY_FETCH_RDREQ__WRITE__SHIFT                                                0xa
+#define GC_UTCL2_L2TLB_CREDIT_SAFETY_FETCH_RDREQ__CREDITS_MASK                                                0x000003FFL
+#define GC_UTCL2_L2TLB_CREDIT_SAFETY_FETCH_RDREQ__WRITE_MASK                                                  0x00000400L
+//GC_L2TLB_GCC_MTYPE_ENCODING_CNTL
+#define GC_L2TLB_GCC_MTYPE_ENCODING_CNTL__ENABLE_MTYPE_ENCODING_0__SHIFT                                      0x0
+#define GC_L2TLB_GCC_MTYPE_ENCODING_CNTL__ENABLE_MTYPE_ENCODING_1__SHIFT                                      0x1
+#define GC_L2TLB_GCC_MTYPE_ENCODING_CNTL__ENABLE_MTYPE_ENCODING_2__SHIFT                                      0x2
+#define GC_L2TLB_GCC_MTYPE_ENCODING_CNTL__ENABLE_MTYPE_ENCODING_3__SHIFT                                      0x3
+#define GC_L2TLB_GCC_MTYPE_ENCODING_CNTL__ENABLE_MTYPE_ENCODING_0_MASK                                        0x00000001L
+#define GC_L2TLB_GCC_MTYPE_ENCODING_CNTL__ENABLE_MTYPE_ENCODING_1_MASK                                        0x00000002L
+#define GC_L2TLB_GCC_MTYPE_ENCODING_CNTL__ENABLE_MTYPE_ENCODING_2_MASK                                        0x00000004L
+#define GC_L2TLB_GCC_MTYPE_ENCODING_CNTL__ENABLE_MTYPE_ENCODING_3_MASK                                        0x00000008L
+//GC_L2TLB_FB_COMP_EN_CNTL
+#define GC_L2TLB_FB_COMP_EN_CNTL__COMP_EN__SHIFT                                                              0x0
+#define GC_L2TLB_FB_COMP_EN_CNTL__TLSI_COMP_EN_NO_PTE__SHIFT                                                  0x1
+#define GC_L2TLB_FB_COMP_EN_CNTL__TLSI_COMP_EN_PTE__SHIFT                                                     0x2
+#define GC_L2TLB_FB_COMP_EN_CNTL__COMP_EN_MASK                                                                0x00000001L
+#define GC_L2TLB_FB_COMP_EN_CNTL__TLSI_COMP_EN_NO_PTE_MASK                                                    0x00000002L
+#define GC_L2TLB_FB_COMP_EN_CNTL__TLSI_COMP_EN_PTE_MASK                                                       0x00000004L
+//GC_L2TLB_RETRY_TIMEOUT_STATUS_0
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_0__LOGICAL_ADDR__SHIFT                                                  0x0
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_0__LOGICAL_ADDR_MASK                                                    0xFFFFFFFFL
+//GC_L2TLB_RETRY_TIMEOUT_STATUS_1
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__LOGICAL_ADDR__SHIFT                                                  0x0
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__VMID__SHIFT                                                          0xd
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__VFID__SHIFT                                                          0x11
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__VF__SHIFT                                                            0x16
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__GPA__SHIFT                                                           0x17
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__RD_PERM__SHIFT                                                       0x19
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__WR_PERM__SHIFT                                                       0x1a
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__EXE_PERM__SHIFT                                                      0x1b
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__TMZ__SHIFT                                                           0x1c
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__CLIENT_ID__SHIFT                                                     0x1d
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__LOGICAL_ADDR_MASK                                                    0x00001FFFL
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__VMID_MASK                                                            0x0001E000L
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__VFID_MASK                                                            0x003E0000L
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__VF_MASK                                                              0x00400000L
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__GPA_MASK                                                             0x01800000L
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__RD_PERM_MASK                                                         0x02000000L
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__WR_PERM_MASK                                                         0x04000000L
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__EXE_PERM_MASK                                                        0x08000000L
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__TMZ_MASK                                                             0x10000000L
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_1__CLIENT_ID_MASK                                                       0xE0000000L
+//GC_L2TLB_RETRY_TIMEOUT_STATUS_2
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_2__CLIENT_ID__SHIFT                                                     0x0
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_2__BUS_ATOMIC_PERM__SHIFT                                               0x6
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_2__TLB_ID__SHIFT                                                        0x7
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_2__RETRY_TIMEOUT__SHIFT                                                 0xb
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_2__CLEAR_RETRY_TIMEOUT_STATUS__SHIFT                                    0x1f
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_2__CLIENT_ID_MASK                                                       0x0000003FL
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_2__BUS_ATOMIC_PERM_MASK                                                 0x00000040L
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_2__TLB_ID_MASK                                                          0x00000780L
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_2__RETRY_TIMEOUT_MASK                                                   0x00000800L
+#define GC_L2TLB_RETRY_TIMEOUT_STATUS_2__CLEAR_RETRY_TIMEOUT_STATUS_MASK                                      0x80000000L
+//GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__CLIENT_ID__SHIFT                                        0x0
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__SNOOP__SHIFT                                            0x9
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__ATOMIC__SHIFT                                           0xa
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__VF__SHIFT                                               0xb
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__HDM__SHIFT                                              0xc
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__SPACE__SHIFT                                            0xd
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__VFID__SHIFT                                             0x10
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__EXE__SHIFT                                              0x15
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__TMZ__SHIFT                                              0x16
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__VMID__SHIFT                                             0x18
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__TEE__SHIFT                                              0x1c
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__PCIE_ATOMIC__SHIFT                                      0x1d
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__RESERVED_SPACE_ENCODING_DETECTED__SHIFT                 0x1e
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__CLEAR__SHIFT                                            0x1f
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__CLIENT_ID_MASK                                          0x000001FFL
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__SNOOP_MASK                                              0x00000200L
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__ATOMIC_MASK                                             0x00000400L
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__VF_MASK                                                 0x00000800L
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__HDM_MASK                                                0x00001000L
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__SPACE_MASK                                              0x0000E000L
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__VFID_MASK                                               0x001F0000L
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__EXE_MASK                                                0x00200000L
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__TMZ_MASK                                                0x00400000L
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__VMID_MASK                                               0x0F000000L
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__TEE_MASK                                                0x10000000L
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__PCIE_ATOMIC_MASK                                        0x20000000L
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__RESERVED_SPACE_ENCODING_DETECTED_MASK                   0x40000000L
+#define GC_L2TLB_TLB0_RESERVED_SPACE_ENCODING_STATUS__CLEAR_MASK                                              0x80000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvml2pspdec
+//GCUTCL2_TRANSLATION_BYPASS_BY_VMID
+#define GCUTCL2_TRANSLATION_BYPASS_BY_VMID__TRANS_BYPASS_VMIDS__SHIFT                                         0x0
+#define GCUTCL2_TRANSLATION_BYPASS_BY_VMID__GPA_MODE_VMIDS__SHIFT                                             0x10
+#define GCUTCL2_TRANSLATION_BYPASS_BY_VMID__TRANS_BYPASS_VMIDS_MASK                                           0x0000FFFFL
+#define GCUTCL2_TRANSLATION_BYPASS_BY_VMID__GPA_MODE_VMIDS_MASK                                               0xFFFF0000L
+//GCVML2_SEC_MASTER
+#define GCVML2_SEC_MASTER__UNIT_ID__SHIFT                                                                     0x0
+#define GCVML2_SEC_MASTER__TRUST_LEVEL__SHIFT                                                                 0x6
+#define GCVML2_SEC_MASTER__UNIT_ID_MASK                                                                       0x0000003FL
+#define GCVML2_SEC_MASTER__TRUST_LEVEL_MASK                                                                   0x000003C0L
+//GCVM_IOMMU_GPU_HOST_TRANSLATION_ENABLE
+#define GCVM_IOMMU_GPU_HOST_TRANSLATION_ENABLE__GPU_HOST_TRANSLATION_ENABLE__SHIFT                            0x0
+#define GCVM_IOMMU_GPU_HOST_TRANSLATION_ENABLE__GPU_HOST_TRANSLATION_ENABLE_MASK                              0x00000001L
+//GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_CNTL
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_CNTL__ENABLE__SHIFT                                               0x0
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_CNTL__ENABLE_MASK                                                 0x00000001L
+//GCVM_IOMMU_CONTROL_REGISTER
+#define GCVM_IOMMU_CONTROL_REGISTER__IOMMUEN__SHIFT                                                           0x0
+#define GCVM_IOMMU_CONTROL_REGISTER__IOMMUEN_MASK                                                             0x00000001L
+//GCVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER
+#define GCVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER__PERFOPTEN__SHIFT                                0xd
+#define GCVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER__PERFOPTEN_MASK                                  0x00002000L
+//GCUTC_TRANSLATION_FAULT_CNTL0
+#define GCUTC_TRANSLATION_FAULT_CNTL0__DEFAULT_PHYSICAL_PAGE_ADDRESS_LSB__SHIFT                               0x0
+#define GCUTC_TRANSLATION_FAULT_CNTL0__DEFAULT_PHYSICAL_PAGE_ADDRESS_LSB_MASK                                 0xFFFFFFFFL
+//GCUTC_TRANSLATION_FAULT_CNTL1
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_PHYSICAL_PAGE_ADDRESS_MSB__SHIFT                               0x0
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_IO__SHIFT                                                      0x8
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_SPA__SHIFT                                                     0x9
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_SNOOP__SHIFT                                                   0xa
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_HDM__SHIFT                                                     0xb
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_PHYSICAL_PAGE_ADDRESS_MSB_MASK                                 0x000000FFL
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_IO_MASK                                                        0x00000100L
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_SPA_MASK                                                       0x00000200L
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_SNOOP_MASK                                                     0x00000400L
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_HDM_MASK                                                       0x00000800L
+//GCUTCL2_COMP_EN_OVERRIDES
+#define GCUTCL2_COMP_EN_OVERRIDES__GPA_MODE3__SHIFT                                                           0x0
+#define GCUTCL2_COMP_EN_OVERRIDES__LOCAL_FB_PTE__SHIFT                                                        0x1
+#define GCUTCL2_COMP_EN_OVERRIDES__REMOTE_FB_PTE__SHIFT                                                       0x2
+#define GCUTCL2_COMP_EN_OVERRIDES__ROUTER_ATCL2__SHIFT                                                        0x3
+#define GCUTCL2_COMP_EN_OVERRIDES__GPA_MODE3_MASK                                                             0x00000001L
+#define GCUTCL2_COMP_EN_OVERRIDES__LOCAL_FB_PTE_MASK                                                          0x00000002L
+#define GCUTCL2_COMP_EN_OVERRIDES__REMOTE_FB_PTE_MASK                                                         0x00000004L
+#define GCUTCL2_COMP_EN_OVERRIDES__ROUTER_ATCL2_MASK                                                          0x00000008L
+//GCUTCL2_ROUTER_CNTL
+#define GCUTCL2_ROUTER_CNTL__OVERRIDE_GPA2_EXE_AND_ATOMIC_PERMS_GRANTED__SHIFT                                0x0
+#define GCUTCL2_ROUTER_CNTL__OVERRIDE_GPA2_EXE_AND_ATOMIC_PERMS_GRANTED_MASK                                  0x00000001L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcvmsharedpspdec
+//GCMC_VM_FB_SIZE_OFFSET_VF0
+#define GCMC_VM_FB_SIZE_OFFSET_VF0__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF0__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF0__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF0__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF1
+#define GCMC_VM_FB_SIZE_OFFSET_VF1__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF1__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF1__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF1__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF2
+#define GCMC_VM_FB_SIZE_OFFSET_VF2__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF2__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF2__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF2__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF3
+#define GCMC_VM_FB_SIZE_OFFSET_VF3__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF3__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF3__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF3__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF4
+#define GCMC_VM_FB_SIZE_OFFSET_VF4__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF4__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF4__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF4__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF5
+#define GCMC_VM_FB_SIZE_OFFSET_VF5__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF5__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF5__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF5__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF6
+#define GCMC_VM_FB_SIZE_OFFSET_VF6__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF6__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF6__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF6__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF7
+#define GCMC_VM_FB_SIZE_OFFSET_VF7__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF7__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF7__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF7__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_OFFSET
+#define GCMC_VM_FB_OFFSET__FB_OFFSET__SHIFT                                                                   0x0
+#define GCMC_VM_FB_OFFSET__FB_OFFSET_MASK                                                                     0x0FFFFFFFL
+//GCMC_VM_LOCAL_FB_ADDRESS_START
+#define GCMC_VM_LOCAL_FB_ADDRESS_START__ADDRESS__SHIFT                                                        0x0
+#define GCMC_VM_LOCAL_FB_ADDRESS_START__ADDRESS_MASK                                                          0x0FFFFFFFL
+//GCMC_VM_LOCAL_FB_ADDRESS_END
+#define GCMC_VM_LOCAL_FB_ADDRESS_END__ADDRESS__SHIFT                                                          0x0
+#define GCMC_VM_LOCAL_FB_ADDRESS_END__ADDRESS_MASK                                                            0x0FFFFFFFL
+//GCMC_VM_LOCAL_FB_ADDRESS_LOCK_CNTL
+#define GCMC_VM_LOCAL_FB_ADDRESS_LOCK_CNTL__LOCK__SHIFT                                                       0x0
+#define GCMC_VM_LOCAL_FB_ADDRESS_LOCK_CNTL__LOCK_MASK                                                         0x00000001L
+//GCUTCL2_MISC_CNTL
+#define GCUTCL2_MISC_CNTL__MASK_TDISP_RUN_ATC_DIS_IOMMU_DIS_NPA__SHIFT                                        0x0
+#define GCUTCL2_MISC_CNTL__MASK_TDISP_RUN_ATC_DIS_IOMMU_DIS_NPA_MASK                                          0x00000001L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcl2tlbpspdec
+//GC_L2TLB_TLB0_CAM_ECC_CNTL
+#define GC_L2TLB_TLB0_CAM_ECC_CNTL__FORCE_PARITY_ERROR__SHIFT                                                 0x0
+#define GC_L2TLB_TLB0_CAM_ECC_CNTL__FORCE_PARITY_ERROR_MASK                                                   0x00000001L
+
+
+// addressBlock: gfx_se_sqind
+//SQ_DEBUG_STS_LOCAL
+#define SQ_DEBUG_STS_LOCAL__BUSY__SHIFT                                                                       0x0
+#define SQ_DEBUG_STS_LOCAL__WAVE_LEVEL__SHIFT                                                                 0x4
+#define SQ_DEBUG_STS_LOCAL__SQ_BUSY__SHIFT                                                                    0xc
+#define SQ_DEBUG_STS_LOCAL__IS_BUSY__SHIFT                                                                    0xd
+#define SQ_DEBUG_STS_LOCAL__IB_BUSY__SHIFT                                                                    0xe
+#define SQ_DEBUG_STS_LOCAL__ARB_BUSY__SHIFT                                                                   0xf
+#define SQ_DEBUG_STS_LOCAL__EXP_BUSY__SHIFT                                                                   0x10
+#define SQ_DEBUG_STS_LOCAL__BRMSG_BUSY__SHIFT                                                                 0x11
+#define SQ_DEBUG_STS_LOCAL__VM_BUSY__SHIFT                                                                    0x12
+#define SQ_DEBUG_STS_LOCAL__BUSY_MASK                                                                         0x00000001L
+#define SQ_DEBUG_STS_LOCAL__WAVE_LEVEL_MASK                                                                   0x000003F0L
+#define SQ_DEBUG_STS_LOCAL__SQ_BUSY_MASK                                                                      0x00001000L
+#define SQ_DEBUG_STS_LOCAL__IS_BUSY_MASK                                                                      0x00002000L
+#define SQ_DEBUG_STS_LOCAL__IB_BUSY_MASK                                                                      0x00004000L
+#define SQ_DEBUG_STS_LOCAL__ARB_BUSY_MASK                                                                     0x00008000L
+#define SQ_DEBUG_STS_LOCAL__EXP_BUSY_MASK                                                                     0x00010000L
+#define SQ_DEBUG_STS_LOCAL__BRMSG_BUSY_MASK                                                                   0x00020000L
+#define SQ_DEBUG_STS_LOCAL__VM_BUSY_MASK                                                                      0x00040000L
+//SQ_DEBUG_CTRL_LOCAL
+#define SQ_DEBUG_CTRL_LOCAL__UNUSED__SHIFT                                                                    0x0
+#define SQ_DEBUG_CTRL_LOCAL__UNUSED_MASK                                                                      0x000000FFL
+//SQ_WAVE_ACTIVE
+#define SQ_WAVE_ACTIVE__WAVE_SLOT__SHIFT                                                                      0x0
+#define SQ_WAVE_ACTIVE__WAVE_SLOT_MASK                                                                        0x000FFFFFL
+//SQ_WAVE_VALID_AND_IDLE
+#define SQ_WAVE_VALID_AND_IDLE__WAVE_SLOT__SHIFT                                                              0x0
+#define SQ_WAVE_VALID_AND_IDLE__WAVE_SLOT_MASK                                                                0x000FFFFFL
+//SQ_WAVE_MODE
+#define SQ_WAVE_MODE__FP_ROUND__SHIFT                                                                         0x0
+#define SQ_WAVE_MODE__FP_DENORM__SHIFT                                                                        0x4
+#define SQ_WAVE_MODE__DST_VGPR_MSB__SHIFT                                                                     0xc
+#define SQ_WAVE_MODE__SRC0_VGPR_MSB__SHIFT                                                                    0xe
+#define SQ_WAVE_MODE__SRC1_VGPR_MSB__SHIFT                                                                    0x10
+#define SQ_WAVE_MODE__SRC2_VGPR_MSB__SHIFT                                                                    0x12
+#define SQ_WAVE_MODE__FP16_OVFL__SHIFT                                                                        0x17
+#define SQ_WAVE_MODE__SCALAR_PREFETCH_EN__SHIFT                                                               0x18
+#define SQ_WAVE_MODE__REPLAY_MODE__SHIFT                                                                      0x19
+#define SQ_WAVE_MODE__FLAT_SCRATCH_IS_NV__SHIFT                                                               0x1a
+#define SQ_WAVE_MODE__DISABLE_PERF__SHIFT                                                                     0x1b
+#define SQ_WAVE_MODE__FP_ROUND_MASK                                                                           0x0000000FL
+#define SQ_WAVE_MODE__FP_DENORM_MASK                                                                          0x000000F0L
+#define SQ_WAVE_MODE__DST_VGPR_MSB_MASK                                                                       0x00003000L
+#define SQ_WAVE_MODE__SRC0_VGPR_MSB_MASK                                                                      0x0000C000L
+#define SQ_WAVE_MODE__SRC1_VGPR_MSB_MASK                                                                      0x00030000L
+#define SQ_WAVE_MODE__SRC2_VGPR_MSB_MASK                                                                      0x000C0000L
+#define SQ_WAVE_MODE__FP16_OVFL_MASK                                                                          0x00800000L
+#define SQ_WAVE_MODE__SCALAR_PREFETCH_EN_MASK                                                                 0x01000000L
+#define SQ_WAVE_MODE__REPLAY_MODE_MASK                                                                        0x02000000L
+#define SQ_WAVE_MODE__FLAT_SCRATCH_IS_NV_MASK                                                                 0x04000000L
+#define SQ_WAVE_MODE__DISABLE_PERF_MASK                                                                       0x08000000L
+//SQ_WAVE_STATUS
+#define SQ_WAVE_STATUS__PRIV__SHIFT                                                                           0x5
+#define SQ_WAVE_STATUS__TRAP_EN__SHIFT                                                                        0x6
+#define SQ_WAVE_STATUS__EXECZ__SHIFT                                                                          0x9
+#define SQ_WAVE_STATUS__VCCZ__SHIFT                                                                           0xa
+#define SQ_WAVE_STATUS__IN_WG__SHIFT                                                                          0xb
+#define SQ_WAVE_STATUS__TRAP__SHIFT                                                                           0xe
+#define SQ_WAVE_STATUS__TRAP_BARRIER_COMPLETE__SHIFT                                                          0xf
+#define SQ_WAVE_STATUS__VALID__SHIFT                                                                          0x10
+#define SQ_WAVE_STATUS__CLUSTER_TRAP_BARRIER_COMPLETE__SHIFT                                                  0x11
+#define SQ_WAVE_STATUS__FATAL_HALT__SHIFT                                                                     0x17
+#define SQ_WAVE_STATUS__NO_VGPRS__SHIFT                                                                       0x18
+#define SQ_WAVE_STATUS__IDLE__SHIFT                                                                           0x1c
+#define SQ_WAVE_STATUS__WAVE64__SHIFT                                                                         0x1d
+#define SQ_WAVE_STATUS__DVGPR_EN__SHIFT                                                                       0x1e
+#define SQ_WAVE_STATUS__WGP_TAKEOVER__SHIFT                                                                   0x1f
+#define SQ_WAVE_STATUS__PRIV_MASK                                                                             0x00000020L
+#define SQ_WAVE_STATUS__TRAP_EN_MASK                                                                          0x00000040L
+#define SQ_WAVE_STATUS__EXECZ_MASK                                                                            0x00000200L
+#define SQ_WAVE_STATUS__VCCZ_MASK                                                                             0x00000400L
+#define SQ_WAVE_STATUS__IN_WG_MASK                                                                            0x00000800L
+#define SQ_WAVE_STATUS__TRAP_MASK                                                                             0x00004000L
+#define SQ_WAVE_STATUS__TRAP_BARRIER_COMPLETE_MASK                                                            0x00008000L
+#define SQ_WAVE_STATUS__VALID_MASK                                                                            0x00010000L
+#define SQ_WAVE_STATUS__CLUSTER_TRAP_BARRIER_COMPLETE_MASK                                                    0x00020000L
+#define SQ_WAVE_STATUS__FATAL_HALT_MASK                                                                       0x00800000L
+#define SQ_WAVE_STATUS__NO_VGPRS_MASK                                                                         0x01000000L
+#define SQ_WAVE_STATUS__IDLE_MASK                                                                             0x10000000L
+#define SQ_WAVE_STATUS__WAVE64_MASK                                                                           0x20000000L
+#define SQ_WAVE_STATUS__DVGPR_EN_MASK                                                                         0x40000000L
+#define SQ_WAVE_STATUS__WGP_TAKEOVER_MASK                                                                     0x80000000L
+//SQ_WAVE_STATE_PRIV
+#define SQ_WAVE_STATE_PRIV__WG_RR_EN__SHIFT                                                                   0x0
+#define SQ_WAVE_STATE_PRIV__SLEEP_WAKEUP__SHIFT                                                               0x1
+#define SQ_WAVE_STATE_PRIV__BARRIER_COMPLETE__SHIFT                                                           0x2
+#define SQ_WAVE_STATE_PRIV__NAMED_BARRIER_COMPLETE__SHIFT                                                     0x3
+#define SQ_WAVE_STATE_PRIV__NAMED_BARRIER_ID__SHIFT                                                           0x4
+#define SQ_WAVE_STATE_PRIV__SCC__SHIFT                                                                        0x9
+#define SQ_WAVE_STATE_PRIV__SYS_PRIO__SHIFT                                                                   0xa
+#define SQ_WAVE_STATE_PRIV__USER_PRIO__SHIFT                                                                  0xc
+#define SQ_WAVE_STATE_PRIV__HALT__SHIFT                                                                       0xe
+#define SQ_WAVE_STATE_PRIV__POISON_ERR__SHIFT                                                                 0xf
+#define SQ_WAVE_STATE_PRIV__CLUSTER_BARRIER_COMPLETE__SHIFT                                                   0x10
+#define SQ_WAVE_STATE_PRIV__COND_DBG_SYS__SHIFT                                                               0x11
+#define SQ_WAVE_STATE_PRIV__SCRATCH_EN__SHIFT                                                                 0x12
+#define SQ_WAVE_STATE_PRIV__PERF_EN__SHIFT                                                                    0x13
+#define SQ_WAVE_STATE_PRIV__TTRACE_EN__SHIFT                                                                  0x14
+#define SQ_WAVE_STATE_PRIV__COND_DBG_USER__SHIFT                                                              0x15
+#define SQ_WAVE_STATE_PRIV__WG_RR_EN_MASK                                                                     0x00000001L
+#define SQ_WAVE_STATE_PRIV__SLEEP_WAKEUP_MASK                                                                 0x00000002L
+#define SQ_WAVE_STATE_PRIV__BARRIER_COMPLETE_MASK                                                             0x00000004L
+#define SQ_WAVE_STATE_PRIV__NAMED_BARRIER_COMPLETE_MASK                                                       0x00000008L
+#define SQ_WAVE_STATE_PRIV__NAMED_BARRIER_ID_MASK                                                             0x000001F0L
+#define SQ_WAVE_STATE_PRIV__SCC_MASK                                                                          0x00000200L
+#define SQ_WAVE_STATE_PRIV__SYS_PRIO_MASK                                                                     0x00000C00L
+#define SQ_WAVE_STATE_PRIV__USER_PRIO_MASK                                                                    0x00003000L
+#define SQ_WAVE_STATE_PRIV__HALT_MASK                                                                         0x00004000L
+#define SQ_WAVE_STATE_PRIV__POISON_ERR_MASK                                                                   0x00008000L
+#define SQ_WAVE_STATE_PRIV__CLUSTER_BARRIER_COMPLETE_MASK                                                     0x00010000L
+#define SQ_WAVE_STATE_PRIV__COND_DBG_SYS_MASK                                                                 0x00020000L
+#define SQ_WAVE_STATE_PRIV__SCRATCH_EN_MASK                                                                   0x00040000L
+#define SQ_WAVE_STATE_PRIV__PERF_EN_MASK                                                                      0x00080000L
+#define SQ_WAVE_STATE_PRIV__TTRACE_EN_MASK                                                                    0x00100000L
+#define SQ_WAVE_STATE_PRIV__COND_DBG_USER_MASK                                                                0x00200000L
+//SQ_WAVE_GPR_ALLOC
+#define SQ_WAVE_GPR_ALLOC__VGPR_BASE__SHIFT                                                                   0x0
+#define SQ_WAVE_GPR_ALLOC__VGPR_SIZE__SHIFT                                                                   0xc
+#define SQ_WAVE_GPR_ALLOC__VGPR_BASE_MASK                                                                     0x000000FFL
+#define SQ_WAVE_GPR_ALLOC__VGPR_SIZE_MASK                                                                     0x000FF000L
+//SQ_WAVE_LDS_ALLOC
+#define SQ_WAVE_LDS_ALLOC__LDS_BASE__SHIFT                                                                    0x0
+#define SQ_WAVE_LDS_ALLOC__LDS_SIZE__SHIFT                                                                    0xc
+#define SQ_WAVE_LDS_ALLOC__VGPR_SHARED_SIZE__SHIFT                                                            0x18
+#define SQ_WAVE_LDS_ALLOC__LDS_BASE_MASK                                                                      0x000001FFL
+#define SQ_WAVE_LDS_ALLOC__LDS_SIZE_MASK                                                                      0x001FF000L
+#define SQ_WAVE_LDS_ALLOC__VGPR_SHARED_SIZE_MASK                                                              0x0F000000L
+//SQ_WAVE_IB_STS
+#define SQ_WAVE_IB_STS__EXP_CNT__SHIFT                                                                        0x0
+#define SQ_WAVE_IB_STS__DS_CNT__SHIFT                                                                         0x3
+#define SQ_WAVE_IB_STS__LOAD_CNT__SHIFT                                                                       0x9
+#define SQ_WAVE_IB_STS__SAMPLE_CNT__SHIFT                                                                     0xf
+#define SQ_WAVE_IB_STS__BVH_CNT__SHIFT                                                                        0x15
+#define SQ_WAVE_IB_STS__STORE_CNT__SHIFT                                                                      0x18
+#define SQ_WAVE_IB_STS__EXP_CNT_MASK                                                                          0x00000007L
+#define SQ_WAVE_IB_STS__DS_CNT_MASK                                                                           0x000001F8L
+#define SQ_WAVE_IB_STS__LOAD_CNT_MASK                                                                         0x00007E00L
+#define SQ_WAVE_IB_STS__SAMPLE_CNT_MASK                                                                       0x001F8000L
+#define SQ_WAVE_IB_STS__BVH_CNT_MASK                                                                          0x00E00000L
+#define SQ_WAVE_IB_STS__STORE_CNT_MASK                                                                        0x3F000000L
+//SQ_PERF_SNAPSHOT_DATA
+#define SQ_PERF_SNAPSHOT_DATA__VALID__SHIFT                                                                   0x0
+#define SQ_PERF_SNAPSHOT_DATA__WAVE_ISSUE__SHIFT                                                              0x1
+#define SQ_PERF_SNAPSHOT_DATA__INST_TYPE__SHIFT                                                               0x2
+#define SQ_PERF_SNAPSHOT_DATA__NO_ISSUE_REASON__SHIFT                                                         0x6
+#define SQ_PERF_SNAPSHOT_DATA__WAVE_ID__SHIFT                                                                 0x9
+#define SQ_PERF_SNAPSHOT_DATA__SAMPLING_LOCK_ERR__SHIFT                                                       0xe
+#define SQ_PERF_SNAPSHOT_DATA__ASYNC_CNT__SHIFT                                                               0x14
+#define SQ_PERF_SNAPSHOT_DATA__TENSOR_CNT__SHIFT                                                              0x1a
+#define SQ_PERF_SNAPSHOT_DATA__VALID_MASK                                                                     0x00000001L
+#define SQ_PERF_SNAPSHOT_DATA__WAVE_ISSUE_MASK                                                                0x00000002L
+#define SQ_PERF_SNAPSHOT_DATA__INST_TYPE_MASK                                                                 0x0000003CL
+#define SQ_PERF_SNAPSHOT_DATA__NO_ISSUE_REASON_MASK                                                           0x000001C0L
+#define SQ_PERF_SNAPSHOT_DATA__WAVE_ID_MASK                                                                   0x00003E00L
+#define SQ_PERF_SNAPSHOT_DATA__SAMPLING_LOCK_ERR_MASK                                                         0x00004000L
+#define SQ_PERF_SNAPSHOT_DATA__ASYNC_CNT_MASK                                                                 0x03F00000L
+#define SQ_PERF_SNAPSHOT_DATA__TENSOR_CNT_MASK                                                                0xFC000000L
+//SQ_PERF_SNAPSHOT_PC_LO
+#define SQ_PERF_SNAPSHOT_PC_LO__PC_LO__SHIFT                                                                  0x0
+#define SQ_PERF_SNAPSHOT_PC_LO__PC_LO_MASK                                                                    0xFFFFFFFFL
+//SQ_PERF_SNAPSHOT_PC_HI
+#define SQ_PERF_SNAPSHOT_PC_HI__PC_HI__SHIFT                                                                  0x0
+#define SQ_PERF_SNAPSHOT_PC_HI__PC_HI_MASK                                                                    0x01FFFFFFL
+//SQ_WAVE_IB_DBG1
+#define SQ_WAVE_IB_DBG1__WAVE_IDLE__SHIFT                                                                     0x18
+#define SQ_WAVE_IB_DBG1__MISC_CNT__SHIFT                                                                      0x19
+#define SQ_WAVE_IB_DBG1__WAVE_IDLE_MASK                                                                       0x01000000L
+#define SQ_WAVE_IB_DBG1__MISC_CNT_MASK                                                                        0xFE000000L
+//SQ_WAVE_FLUSH_IB
+#define SQ_WAVE_FLUSH_IB__UNUSED__SHIFT                                                                       0x0
+#define SQ_WAVE_FLUSH_IB__UNUSED_MASK                                                                         0xFFFFFFFFL
+//SQ_PERF_SNAPSHOT_DATA1
+#define SQ_PERF_SNAPSHOT_DATA1__WAVE_CNT__SHIFT                                                               0x0
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_ISSUED_BRMSG__SHIFT                                                 0x9
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_ISSUED_EXPORT__SHIFT                                                0xa
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_ISSUED_LDS_DIRECT__SHIFT                                            0xb
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_ISSUED_LDS__SHIFT                                                   0xc
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_ISSUED_TEX__SHIFT                                                   0xd
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_ISSUED_SCALAR__SHIFT                                                0xe
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_ISSUED_VALU__SHIFT                                                  0xf
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_ISSUED_VALU_ARB_B__SHIFT                                            0x10
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_STALLED_BRMSG__SHIFT                                                0x11
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_STALLED_EXPORT__SHIFT                                               0x12
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_STALLED_LDS_DIRECT__SHIFT                                           0x13
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_STALLED_LDS__SHIFT                                                  0x14
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_STALLED_TEX__SHIFT                                                  0x15
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_STALLED_SCALAR__SHIFT                                               0x16
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_STALLED_VALU__SHIFT                                                 0x17
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_STALLED_VALU_ARB_B__SHIFT                                           0x18
+#define SQ_PERF_SNAPSHOT_DATA1__XCNT__SHIFT                                                                   0x1a
+#define SQ_PERF_SNAPSHOT_DATA1__WAVE_CNT_MASK                                                                 0x0000003FL
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_ISSUED_BRMSG_MASK                                                   0x00000200L
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_ISSUED_EXPORT_MASK                                                  0x00000400L
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_ISSUED_LDS_DIRECT_MASK                                              0x00000800L
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_ISSUED_LDS_MASK                                                     0x00001000L
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_ISSUED_TEX_MASK                                                     0x00002000L
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_ISSUED_SCALAR_MASK                                                  0x00004000L
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_ISSUED_VALU_MASK                                                    0x00008000L
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_ISSUED_VALU_ARB_B_MASK                                              0x00010000L
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_STALLED_BRMSG_MASK                                                  0x00020000L
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_STALLED_EXPORT_MASK                                                 0x00040000L
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_STALLED_LDS_DIRECT_MASK                                             0x00080000L
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_STALLED_LDS_MASK                                                    0x00100000L
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_STALLED_TEX_MASK                                                    0x00200000L
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_STALLED_SCALAR_MASK                                                 0x00400000L
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_STALLED_VALU_MASK                                                   0x00800000L
+#define SQ_PERF_SNAPSHOT_DATA1__ARB_STATE_STALLED_VALU_ARB_B_MASK                                             0x01000000L
+#define SQ_PERF_SNAPSHOT_DATA1__XCNT_MASK                                                                     0xFC000000L
+//SQ_PERF_SNAPSHOT_DATA2
+#define SQ_PERF_SNAPSHOT_DATA2__LOAD_CNT__SHIFT                                                               0x0
+#define SQ_PERF_SNAPSHOT_DATA2__STORE_CNT__SHIFT                                                              0x6
+#define SQ_PERF_SNAPSHOT_DATA2__BVH_CNT__SHIFT                                                                0xc
+#define SQ_PERF_SNAPSHOT_DATA2__SAMPLE_CNT__SHIFT                                                             0xf
+#define SQ_PERF_SNAPSHOT_DATA2__DS_CNT__SHIFT                                                                 0x15
+#define SQ_PERF_SNAPSHOT_DATA2__KM_CNT__SHIFT                                                                 0x1b
+#define SQ_PERF_SNAPSHOT_DATA2__LOAD_CNT_MASK                                                                 0x0000003FL
+#define SQ_PERF_SNAPSHOT_DATA2__STORE_CNT_MASK                                                                0x00000FC0L
+#define SQ_PERF_SNAPSHOT_DATA2__BVH_CNT_MASK                                                                  0x00007000L
+#define SQ_PERF_SNAPSHOT_DATA2__SAMPLE_CNT_MASK                                                               0x001F8000L
+#define SQ_PERF_SNAPSHOT_DATA2__DS_CNT_MASK                                                                   0x07E00000L
+#define SQ_PERF_SNAPSHOT_DATA2__KM_CNT_MASK                                                                   0xF8000000L
+//SQ_WAVE_EXCP_FLAG_PRIV
+#define SQ_WAVE_EXCP_FLAG_PRIV__ADDR_WATCH__SHIFT                                                             0x0
+#define SQ_WAVE_EXCP_FLAG_PRIV__MEMVIOL__SHIFT                                                                0x4
+#define SQ_WAVE_EXCP_FLAG_PRIV__SAVE_CONTEXT__SHIFT                                                           0x5
+#define SQ_WAVE_EXCP_FLAG_PRIV__ILLEGAL_INST__SHIFT                                                           0x6
+#define SQ_WAVE_EXCP_FLAG_PRIV__HOST_TRAP__SHIFT                                                              0x7
+#define SQ_WAVE_EXCP_FLAG_PRIV__WAVE_START__SHIFT                                                             0x8
+#define SQ_WAVE_EXCP_FLAG_PRIV__WAVE_END__SHIFT                                                               0x9
+#define SQ_WAVE_EXCP_FLAG_PRIV__PERF_SNAPSHOT__SHIFT                                                          0xa
+#define SQ_WAVE_EXCP_FLAG_PRIV__TRAP_AFTER_INST__SHIFT                                                        0xb
+#define SQ_WAVE_EXCP_FLAG_PRIV__XNACK_ERROR__SHIFT                                                            0xc
+#define SQ_WAVE_EXCP_FLAG_PRIV__FIRST_MEMVIOL_SOURCE__SHIFT                                                   0x1e
+#define SQ_WAVE_EXCP_FLAG_PRIV__ADDR_WATCH_MASK                                                               0x0000000FL
+#define SQ_WAVE_EXCP_FLAG_PRIV__MEMVIOL_MASK                                                                  0x00000010L
+#define SQ_WAVE_EXCP_FLAG_PRIV__SAVE_CONTEXT_MASK                                                             0x00000020L
+#define SQ_WAVE_EXCP_FLAG_PRIV__ILLEGAL_INST_MASK                                                             0x00000040L
+#define SQ_WAVE_EXCP_FLAG_PRIV__HOST_TRAP_MASK                                                                0x00000080L
+#define SQ_WAVE_EXCP_FLAG_PRIV__WAVE_START_MASK                                                               0x00000100L
+#define SQ_WAVE_EXCP_FLAG_PRIV__WAVE_END_MASK                                                                 0x00000200L
+#define SQ_WAVE_EXCP_FLAG_PRIV__PERF_SNAPSHOT_MASK                                                            0x00000400L
+#define SQ_WAVE_EXCP_FLAG_PRIV__TRAP_AFTER_INST_MASK                                                          0x00000800L
+#define SQ_WAVE_EXCP_FLAG_PRIV__XNACK_ERROR_MASK                                                              0x00001000L
+#define SQ_WAVE_EXCP_FLAG_PRIV__FIRST_MEMVIOL_SOURCE_MASK                                                     0xC0000000L
+//SQ_WAVE_EXCP_FLAG_USER
+#define SQ_WAVE_EXCP_FLAG_USER__ALU_INVALID__SHIFT                                                            0x0
+#define SQ_WAVE_EXCP_FLAG_USER__ALU_INPUT_DENORM__SHIFT                                                       0x1
+#define SQ_WAVE_EXCP_FLAG_USER__ALU_FLOAT_DIV0__SHIFT                                                         0x2
+#define SQ_WAVE_EXCP_FLAG_USER__ALU_OVERFLOW__SHIFT                                                           0x3
+#define SQ_WAVE_EXCP_FLAG_USER__ALU_UNDERFLOW__SHIFT                                                          0x4
+#define SQ_WAVE_EXCP_FLAG_USER__ALU_INEXACT__SHIFT                                                            0x5
+#define SQ_WAVE_EXCP_FLAG_USER__ALU_INT_DIV0__SHIFT                                                           0x6
+#define SQ_WAVE_EXCP_FLAG_USER__BUFFER_OOB__SHIFT                                                             0x1e
+#define SQ_WAVE_EXCP_FLAG_USER__LOD_CLAMPED__SHIFT                                                            0x1f
+#define SQ_WAVE_EXCP_FLAG_USER__ALU_INVALID_MASK                                                              0x00000001L
+#define SQ_WAVE_EXCP_FLAG_USER__ALU_INPUT_DENORM_MASK                                                         0x00000002L
+#define SQ_WAVE_EXCP_FLAG_USER__ALU_FLOAT_DIV0_MASK                                                           0x00000004L
+#define SQ_WAVE_EXCP_FLAG_USER__ALU_OVERFLOW_MASK                                                             0x00000008L
+#define SQ_WAVE_EXCP_FLAG_USER__ALU_UNDERFLOW_MASK                                                            0x00000010L
+#define SQ_WAVE_EXCP_FLAG_USER__ALU_INEXACT_MASK                                                              0x00000020L
+#define SQ_WAVE_EXCP_FLAG_USER__ALU_INT_DIV0_MASK                                                             0x00000040L
+#define SQ_WAVE_EXCP_FLAG_USER__BUFFER_OOB_MASK                                                               0x40000000L
+#define SQ_WAVE_EXCP_FLAG_USER__LOD_CLAMPED_MASK                                                              0x80000000L
+//SQ_WAVE_TRAP_CTRL
+#define SQ_WAVE_TRAP_CTRL__ALU_INVALID__SHIFT                                                                 0x0
+#define SQ_WAVE_TRAP_CTRL__ALU_INPUT_DENORM__SHIFT                                                            0x1
+#define SQ_WAVE_TRAP_CTRL__ALU_FLOAT_DIV0__SHIFT                                                              0x2
+#define SQ_WAVE_TRAP_CTRL__ALU_OVERFLOW__SHIFT                                                                0x3
+#define SQ_WAVE_TRAP_CTRL__ALU_UNDERFLOW__SHIFT                                                               0x4
+#define SQ_WAVE_TRAP_CTRL__ALU_INEXACT__SHIFT                                                                 0x5
+#define SQ_WAVE_TRAP_CTRL__ALU_INT_DIV0__SHIFT                                                                0x6
+#define SQ_WAVE_TRAP_CTRL__ADDR_WATCH__SHIFT                                                                  0x7
+#define SQ_WAVE_TRAP_CTRL__WAVE_END__SHIFT                                                                    0x8
+#define SQ_WAVE_TRAP_CTRL__TRAP_AFTER_INST__SHIFT                                                             0x9
+#define SQ_WAVE_TRAP_CTRL__ALU_INVALID_MASK                                                                   0x00000001L
+#define SQ_WAVE_TRAP_CTRL__ALU_INPUT_DENORM_MASK                                                              0x00000002L
+#define SQ_WAVE_TRAP_CTRL__ALU_FLOAT_DIV0_MASK                                                                0x00000004L
+#define SQ_WAVE_TRAP_CTRL__ALU_OVERFLOW_MASK                                                                  0x00000008L
+#define SQ_WAVE_TRAP_CTRL__ALU_UNDERFLOW_MASK                                                                 0x00000010L
+#define SQ_WAVE_TRAP_CTRL__ALU_INEXACT_MASK                                                                   0x00000020L
+#define SQ_WAVE_TRAP_CTRL__ALU_INT_DIV0_MASK                                                                  0x00000040L
+#define SQ_WAVE_TRAP_CTRL__ADDR_WATCH_MASK                                                                    0x00000080L
+#define SQ_WAVE_TRAP_CTRL__WAVE_END_MASK                                                                      0x00000100L
+#define SQ_WAVE_TRAP_CTRL__TRAP_AFTER_INST_MASK                                                               0x00000200L
+//SQ_WAVE_SCRATCH_BASE_LO
+#define SQ_WAVE_SCRATCH_BASE_LO__DATA__SHIFT                                                                  0x0
+#define SQ_WAVE_SCRATCH_BASE_LO__DATA_MASK                                                                    0xFFFFFFFFL
+//SQ_WAVE_SCRATCH_BASE_HI
+#define SQ_WAVE_SCRATCH_BASE_HI__DATA__SHIFT                                                                  0x0
+#define SQ_WAVE_SCRATCH_BASE_HI__DATA_MASK                                                                    0xFFFFFFFFL
+//SQ_WAVE_HW_ID1
+#define SQ_WAVE_HW_ID1__WAVE_ID__SHIFT                                                                        0x0
+#define SQ_WAVE_HW_ID1__SIMD_ID__SHIFT                                                                        0x8
+#define SQ_WAVE_HW_ID1__WGP_ID__SHIFT                                                                         0xa
+#define SQ_WAVE_HW_ID1__SA_ID__SHIFT                                                                          0x10
+#define SQ_WAVE_HW_ID1__DP_RATE__SHIFT                                                                        0x1d
+#define SQ_WAVE_HW_ID1__WAVE_ID_MASK                                                                          0x0000001FL
+#define SQ_WAVE_HW_ID1__SIMD_ID_MASK                                                                          0x00000300L
+#define SQ_WAVE_HW_ID1__WGP_ID_MASK                                                                           0x00003C00L
+#define SQ_WAVE_HW_ID1__SA_ID_MASK                                                                            0x00010000L
+#define SQ_WAVE_HW_ID1__DP_RATE_MASK                                                                          0xE0000000L
+//SQ_WAVE_HW_ID2
+#define SQ_WAVE_HW_ID2__QUEUE_ID__SHIFT                                                                       0x0
+#define SQ_WAVE_HW_ID2__PIPE_ID__SHIFT                                                                        0x4
+#define SQ_WAVE_HW_ID2__ME_ID__SHIFT                                                                          0x8
+#define SQ_WAVE_HW_ID2__STATE_ID__SHIFT                                                                       0xc
+#define SQ_WAVE_HW_ID2__WG_ID__SHIFT                                                                          0x10
+#define SQ_WAVE_HW_ID2__VM_ID__SHIFT                                                                          0x18
+#define SQ_WAVE_HW_ID2__QUEUE_ID_MASK                                                                         0x0000000FL
+#define SQ_WAVE_HW_ID2__PIPE_ID_MASK                                                                          0x00000030L
+#define SQ_WAVE_HW_ID2__ME_ID_MASK                                                                            0x00000300L
+#define SQ_WAVE_HW_ID2__STATE_ID_MASK                                                                         0x00007000L
+#define SQ_WAVE_HW_ID2__WG_ID_MASK                                                                            0x001F0000L
+#define SQ_WAVE_HW_ID2__VM_ID_MASK                                                                            0x0F000000L
+//SQ_WAVE_SCHED_MODE
+#define SQ_WAVE_SCHED_MODE__DEP_MODE__SHIFT                                                                   0x0
+#define SQ_WAVE_SCHED_MODE__DISABLE_VALU_ARB_STALL__SHIFT                                                     0x2
+#define SQ_WAVE_SCHED_MODE__DEP_MODE_MASK                                                                     0x00000003L
+#define SQ_WAVE_SCHED_MODE__DISABLE_VALU_ARB_STALL_MASK                                                       0x00000004L
+//SQ_WAVE_IB_STS2
+#define SQ_WAVE_IB_STS2__KM_CNT__SHIFT                                                                        0x0
+#define SQ_WAVE_IB_STS2__CLUSTER_ID__SHIFT                                                                    0x6
+#define SQ_WAVE_IB_STS2__ASYNC_CNT__SHIFT                                                                     0xc
+#define SQ_WAVE_IB_STS2__TENSOR_CNT__SHIFT                                                                    0x12
+#define SQ_WAVE_IB_STS2__WG_IN_CLUSTER__SHIFT                                                                 0x15
+#define SQ_WAVE_IB_STS2__MWAIT__SHIFT                                                                         0x19
+#define SQ_WAVE_IB_STS2__INST_PREFETCH__SHIFT                                                                 0x1c
+#define SQ_WAVE_IB_STS2__FWD_PROGRESS__SHIFT                                                                  0x1e
+#define SQ_WAVE_IB_STS2__TTRACE_EN_SPI__SHIFT                                                                 0x1f
+#define SQ_WAVE_IB_STS2__KM_CNT_MASK                                                                          0x0000001FL
+#define SQ_WAVE_IB_STS2__CLUSTER_ID_MASK                                                                      0x000003C0L
+#define SQ_WAVE_IB_STS2__ASYNC_CNT_MASK                                                                       0x0003F000L
+#define SQ_WAVE_IB_STS2__TENSOR_CNT_MASK                                                                      0x000C0000L
+#define SQ_WAVE_IB_STS2__WG_IN_CLUSTER_MASK                                                                   0x01E00000L
+#define SQ_WAVE_IB_STS2__MWAIT_MASK                                                                           0x02000000L
+#define SQ_WAVE_IB_STS2__INST_PREFETCH_MASK                                                                   0x30000000L
+#define SQ_WAVE_IB_STS2__FWD_PROGRESS_MASK                                                                    0x40000000L
+#define SQ_WAVE_IB_STS2__TTRACE_EN_SPI_MASK                                                                   0x80000000L
+//SQ_WAVE_DVGPR_ALLOC_LO
+#define SQ_WAVE_DVGPR_ALLOC_LO__SEGMENT0__SHIFT                                                               0x0
+#define SQ_WAVE_DVGPR_ALLOC_LO__SEGMENT1__SHIFT                                                               0x8
+#define SQ_WAVE_DVGPR_ALLOC_LO__SEGMENT2__SHIFT                                                               0x10
+#define SQ_WAVE_DVGPR_ALLOC_LO__SEGMENT3__SHIFT                                                               0x18
+#define SQ_WAVE_DVGPR_ALLOC_LO__SEGMENT0_MASK                                                                 0x0000007FL
+#define SQ_WAVE_DVGPR_ALLOC_LO__SEGMENT1_MASK                                                                 0x00007F00L
+#define SQ_WAVE_DVGPR_ALLOC_LO__SEGMENT2_MASK                                                                 0x007F0000L
+#define SQ_WAVE_DVGPR_ALLOC_LO__SEGMENT3_MASK                                                                 0x7F000000L
+//SQ_WAVE_DVGPR_ALLOC_HI
+#define SQ_WAVE_DVGPR_ALLOC_HI__SEGMENT4__SHIFT                                                               0x0
+#define SQ_WAVE_DVGPR_ALLOC_HI__SEGMENT5__SHIFT                                                               0x8
+#define SQ_WAVE_DVGPR_ALLOC_HI__SEGMENT6__SHIFT                                                               0x10
+#define SQ_WAVE_DVGPR_ALLOC_HI__SEGMENT7__SHIFT                                                               0x18
+#define SQ_WAVE_DVGPR_ALLOC_HI__SEGMENT4_MASK                                                                 0x0000007FL
+#define SQ_WAVE_DVGPR_ALLOC_HI__SEGMENT5_MASK                                                                 0x00007F00L
+#define SQ_WAVE_DVGPR_ALLOC_HI__SEGMENT6_MASK                                                                 0x007F0000L
+#define SQ_WAVE_DVGPR_ALLOC_HI__SEGMENT7_MASK                                                                 0x7F000000L
+//SQ_WAVE_XNACK_STATE_PRIV
+#define SQ_WAVE_XNACK_STATE_PRIV__FXPTR__SHIFT                                                                0x0
+#define SQ_WAVE_XNACK_STATE_PRIV__RPTR__SHIFT                                                                 0x8
+#define SQ_WAVE_XNACK_STATE_PRIV__REPLAY_W64H__SHIFT                                                          0x10
+#define SQ_WAVE_XNACK_STATE_PRIV__GOT_XNACK__SHIFT                                                            0x11
+#define SQ_WAVE_XNACK_STATE_PRIV__FIRST_REPLAY__SHIFT                                                         0x12
+#define SQ_WAVE_XNACK_STATE_PRIV__FXPTR_MASK                                                                  0x0000007FL
+#define SQ_WAVE_XNACK_STATE_PRIV__RPTR_MASK                                                                   0x00007F00L
+#define SQ_WAVE_XNACK_STATE_PRIV__REPLAY_W64H_MASK                                                            0x00010000L
+#define SQ_WAVE_XNACK_STATE_PRIV__GOT_XNACK_MASK                                                              0x00020000L
+#define SQ_WAVE_XNACK_STATE_PRIV__FIRST_REPLAY_MASK                                                           0x00040000L
+//SQ_WAVE_XNACK_MASK
+#define SQ_WAVE_XNACK_MASK__MASK__SHIFT                                                                       0x0
+#define SQ_WAVE_XNACK_MASK__MASK_MASK                                                                         0xFFFFFFFFL
+//SQ_WAVE_PC_LO
+#define SQ_WAVE_PC_LO__PC_LO__SHIFT                                                                           0x0
+#define SQ_WAVE_PC_LO__PC_LO_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_PC_HI
+#define SQ_WAVE_PC_HI__PC_HI__SHIFT                                                                           0x0
+#define SQ_WAVE_PC_HI__PC_HI_MASK                                                                             0x01FFFFFFL
+//SQ_WAVE_TTMP0
+#define SQ_WAVE_TTMP0__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP0__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP1
+#define SQ_WAVE_TTMP1__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP1__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP2
+#define SQ_WAVE_TTMP2__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP2__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP3
+#define SQ_WAVE_TTMP3__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP3__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP4
+#define SQ_WAVE_TTMP4__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP4__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP5
+#define SQ_WAVE_TTMP5__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP5__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP6
+#define SQ_WAVE_TTMP6__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP6__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP7
+#define SQ_WAVE_TTMP7__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP7__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP8
+#define SQ_WAVE_TTMP8__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP8__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP9
+#define SQ_WAVE_TTMP9__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP9__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP10
+#define SQ_WAVE_TTMP10__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP10__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_TTMP11
+#define SQ_WAVE_TTMP11__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP11__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_TTMP12
+#define SQ_WAVE_TTMP12__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP12__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_TTMP13
+#define SQ_WAVE_TTMP13__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP13__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_TTMP14
+#define SQ_WAVE_TTMP14__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP14__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_TTMP15
+#define SQ_WAVE_TTMP15__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP15__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_M0
+#define SQ_WAVE_M0__M0__SHIFT                                                                                 0x0
+#define SQ_WAVE_M0__M0_MASK                                                                                   0xFFFFFFFFL
+//SQ_WAVE_EXEC_LO
+#define SQ_WAVE_EXEC_LO__EXEC_LO__SHIFT                                                                       0x0
+#define SQ_WAVE_EXEC_LO__EXEC_LO_MASK                                                                         0xFFFFFFFFL
+//SQ_WAVE_EXEC_HI
+#define SQ_WAVE_EXEC_HI__EXEC_HI__SHIFT                                                                       0x0
+#define SQ_WAVE_EXEC_HI__EXEC_HI_MASK                                                                         0xFFFFFFFFL
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_4_2_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_4_2_0_offset.h
new file mode 100644
index 000000000000..45fdce3feaf5
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_4_2_0_offset.h
@@ -0,0 +1,802 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _mmhub_4_2_0_OFFSET_HEADER
+#define _mmhub_4_2_0_OFFSET_HEADER
+
+
+
+// addressBlock: mmhub_dagb_dagbdec
+// base address: 0x60000
+#define regDAGB0_CNTL_MISC2                                                                             0x00a7
+#define regDAGB0_CNTL_MISC2_BASE_IDX                                                                    1
+#define regDAGB1_CNTL_MISC2                                                                             0x01a7
+#define regDAGB1_CNTL_MISC2_BASE_IDX                                                                    1
+
+
+// addressBlock: mmhub_mm_cane_mmcanedec
+// base address: 0x60c20
+#define regMM_CANE_ICG_CTRL                                                                             0x0313
+#define regMM_CANE_ICG_CTRL_BASE_IDX                                                                    1
+
+
+// addressBlock: mmhub_mmutcl2_mmvmsharedpfdec
+// base address: 0x66000
+#define regMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB                                                     0x0000
+#define regMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_BASE_IDX                                            2
+#define regMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB                                                     0x0001
+#define regMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_BASE_IDX                                            2
+#define regMMUTCL2_CGTT_CLK_CTRL                                                                        0x0002
+#define regMMUTCL2_CGTT_CLK_CTRL_BASE_IDX                                                               2
+#define regMMMC_SHARED_ACTIVE_FCN_ID                                                                    0x0003
+#define regMMMC_SHARED_ACTIVE_FCN_ID_BASE_IDX                                                           2
+#define regMMUTCL2_CGTT_BUSY_CTRL                                                                       0x0004
+#define regMMUTCL2_CGTT_BUSY_CTRL_BASE_IDX                                                              2
+#define regMMUTCL2_GROUP_RET_FAULT_STATUS                                                               0x0005
+#define regMMUTCL2_GROUP_RET_FAULT_STATUS_BASE_IDX                                                      2
+
+
+// addressBlock: mmhub_mmutcl2_mmvml2pfdec
+// base address: 0x66090
+#define regMMVM_L2_CNTL                                                                                 0x0024
+#define regMMVM_L2_CNTL_BASE_IDX                                                                        2
+#define regMMVM_L2_CNTL2                                                                                0x0025
+#define regMMVM_L2_CNTL2_BASE_IDX                                                                       2
+#define regMMVM_L2_CNTL3                                                                                0x0026
+#define regMMVM_L2_CNTL3_BASE_IDX                                                                       2
+#define regMMVM_L2_STATUS                                                                               0x0027
+#define regMMVM_L2_STATUS_BASE_IDX                                                                      2
+#define regMMVM_DUMMY_PAGE_FAULT_CNTL                                                                   0x0028
+#define regMMVM_DUMMY_PAGE_FAULT_CNTL_BASE_IDX                                                          2
+#define regMMVM_DUMMY_PAGE_FAULT_ADDR_LO32                                                              0x0029
+#define regMMVM_DUMMY_PAGE_FAULT_ADDR_LO32_BASE_IDX                                                     2
+#define regMMVM_DUMMY_PAGE_FAULT_ADDR_HI32                                                              0x002a
+#define regMMVM_DUMMY_PAGE_FAULT_ADDR_HI32_BASE_IDX                                                     2
+#define regMMVM_INVALIDATE_CNTL                                                                         0x002b
+#define regMMVM_INVALIDATE_CNTL_BASE_IDX                                                                2
+#define regMMVM_L2_PROTECTION_FAULT_CNTL_LO32                                                           0x002c
+#define regMMVM_L2_PROTECTION_FAULT_CNTL_LO32_BASE_IDX                                                  2
+#define regMMVM_L2_PROTECTION_FAULT_CNTL_HI32                                                           0x002d
+#define regMMVM_L2_PROTECTION_FAULT_CNTL_HI32_BASE_IDX                                                  2
+#define regMMVM_L2_PROTECTION_FAULT_CNTL2                                                               0x002e
+#define regMMVM_L2_PROTECTION_FAULT_CNTL2_BASE_IDX                                                      2
+#define regMMVM_L2_PROTECTION_FAULT_MM_CNTL3                                                            0x002f
+#define regMMVM_L2_PROTECTION_FAULT_MM_CNTL3_BASE_IDX                                                   2
+#define regMMVM_L2_PROTECTION_FAULT_MM_CNTL4                                                            0x0030
+#define regMMVM_L2_PROTECTION_FAULT_MM_CNTL4_BASE_IDX                                                   2
+#define regMMVM_L2_PROTECTION_FAULT_STATUS_LO32                                                         0x0031
+#define regMMVM_L2_PROTECTION_FAULT_STATUS_LO32_BASE_IDX                                                2
+#define regMMVM_L2_PROTECTION_FAULT_STATUS_HI32                                                         0x0032
+#define regMMVM_L2_PROTECTION_FAULT_STATUS_HI32_BASE_IDX                                                2
+#define regMMVM_L2_PROTECTION_FAULT_ADDR_LO32                                                           0x0033
+#define regMMVM_L2_PROTECTION_FAULT_ADDR_LO32_BASE_IDX                                                  2
+#define regMMVM_L2_PROTECTION_FAULT_ADDR_HI32                                                           0x0034
+#define regMMVM_L2_PROTECTION_FAULT_ADDR_HI32_BASE_IDX                                                  2
+#define regMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32                                                   0x0035
+#define regMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32_BASE_IDX                                          2
+#define regMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32                                                   0x0036
+#define regMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32_BASE_IDX                                          2
+#define regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32                                             0x0038
+#define regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32_BASE_IDX                                    2
+#define regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32                                             0x0039
+#define regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32_BASE_IDX                                    2
+#define regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32                                            0x003a
+#define regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32_BASE_IDX                                   2
+#define regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32                                            0x003b
+#define regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32_BASE_IDX                                   2
+#define regMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32                                                0x003c
+#define regMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32_BASE_IDX                                       2
+#define regMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32                                                0x003d
+#define regMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32_BASE_IDX                                       2
+#define regMMVM_L2_CNTL4                                                                                0x003e
+#define regMMVM_L2_CNTL4_BASE_IDX                                                                       2
+#define regMMVM_L2_MM_GROUP_RT_CLASSES                                                                  0x003f
+#define regMMVM_L2_MM_GROUP_RT_CLASSES_BASE_IDX                                                         2
+#define regMMVM_L2_BANK_SELECT_RESERVED_CID                                                             0x0040
+#define regMMVM_L2_BANK_SELECT_RESERVED_CID_BASE_IDX                                                    2
+#define regMMVM_L2_BANK_SELECT_RESERVED_CID2                                                            0x0041
+#define regMMVM_L2_BANK_SELECT_RESERVED_CID2_BASE_IDX                                                   2
+#define regMMVM_L2_CACHE_PARITY_CNTL                                                                    0x0042
+#define regMMVM_L2_CACHE_PARITY_CNTL_BASE_IDX                                                           2
+#define regMMVM_L2_CGTT_CLK_CTRL                                                                        0x0043
+#define regMMVM_L2_CGTT_CLK_CTRL_BASE_IDX                                                               2
+#define regMMVM_L2_CNTL5                                                                                0x0044
+#define regMMVM_L2_CNTL5_BASE_IDX                                                                       2
+#define regMMVM_L2_GCR_CNTL                                                                             0x0045
+#define regMMVM_L2_GCR_CNTL_BASE_IDX                                                                    2
+#define regMMVM_L2_CGTT_BUSY_CTRL                                                                       0x0046
+#define regMMVM_L2_CGTT_BUSY_CTRL_BASE_IDX                                                              2
+#define regMMVM_L2_PTE_CACHE_DUMP_CNTL                                                                  0x0047
+#define regMMVM_L2_PTE_CACHE_DUMP_CNTL_BASE_IDX                                                         2
+#define regMMVM_L2_PTE_CACHE_DUMP_READ                                                                  0x0048
+#define regMMVM_L2_PTE_CACHE_DUMP_READ_BASE_IDX                                                         2
+#define regMMVM_L2_BANK_SELECT_MASKS                                                                    0x0051
+#define regMMVM_L2_BANK_SELECT_MASKS_BASE_IDX                                                           2
+#define regMMUTCL2_CREDIT_SAFETY_GROUP_RET_CDC                                                          0x0052
+#define regMMUTCL2_CREDIT_SAFETY_GROUP_RET_CDC_BASE_IDX                                                 2
+#define regMMUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC                                               0x0053
+#define regMMUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC_BASE_IDX                                      2
+#define regMMUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC                                             0x0054
+#define regMMUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC_BASE_IDX                                    2
+#define regMMVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT                                                      0x0055
+#define regMMVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT_BASE_IDX                                             2
+#define regMMVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ                                                      0x0056
+#define regMMVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ_BASE_IDX                                             2
+#define regMMVML2_IH_FAULT_INTERRUPT_CNTL                                                               0x0057
+#define regMMVML2_IH_FAULT_INTERRUPT_CNTL_BASE_IDX                                                      2
+
+
+// addressBlock: mmhub_mmutcl2_mmvml2prdec
+// base address: 0x66210
+#define regMMMC_VM_L2_PERFCOUNTER_LO                                                                    0x0084
+#define regMMMC_VM_L2_PERFCOUNTER_LO_BASE_IDX                                                           2
+#define regMMMC_VM_L2_PERFCOUNTER_HI                                                                    0x0085
+#define regMMMC_VM_L2_PERFCOUNTER_HI_BASE_IDX                                                           2
+#define regMMUTCL2_PERFCOUNTER_LO                                                                       0x0086
+#define regMMUTCL2_PERFCOUNTER_LO_BASE_IDX                                                              2
+#define regMMUTCL2_PERFCOUNTER_HI                                                                       0x0087
+#define regMMUTCL2_PERFCOUNTER_HI_BASE_IDX                                                              2
+
+
+// addressBlock: mmhub_mmutcl2_mmatcl2prdec
+// base address: 0x66250
+#define regMM_ATC_L2_PERFCOUNTER_LO                                                                     0x0094
+#define regMM_ATC_L2_PERFCOUNTER_LO_BASE_IDX                                                            2
+#define regMM_ATC_L2_PERFCOUNTER_HI                                                                     0x0095
+#define regMM_ATC_L2_PERFCOUNTER_HI_BASE_IDX                                                            2
+
+
+// addressBlock: mmhub_mmutcl2_mmvml2pldec
+// base address: 0x66290
+#define regMMMC_VM_L2_PERFCOUNTER0_CFG                                                                  0x00a4
+#define regMMMC_VM_L2_PERFCOUNTER0_CFG_BASE_IDX                                                         2
+#define regMMMC_VM_L2_PERFCOUNTER1_CFG                                                                  0x00a5
+#define regMMMC_VM_L2_PERFCOUNTER1_CFG_BASE_IDX                                                         2
+#define regMMMC_VM_L2_PERFCOUNTER2_CFG                                                                  0x00a6
+#define regMMMC_VM_L2_PERFCOUNTER2_CFG_BASE_IDX                                                         2
+#define regMMMC_VM_L2_PERFCOUNTER3_CFG                                                                  0x00a7
+#define regMMMC_VM_L2_PERFCOUNTER3_CFG_BASE_IDX                                                         2
+#define regMMMC_VM_L2_PERFCOUNTER4_CFG                                                                  0x00a8
+#define regMMMC_VM_L2_PERFCOUNTER4_CFG_BASE_IDX                                                         2
+#define regMMMC_VM_L2_PERFCOUNTER5_CFG                                                                  0x00a9
+#define regMMMC_VM_L2_PERFCOUNTER5_CFG_BASE_IDX                                                         2
+#define regMMMC_VM_L2_PERFCOUNTER6_CFG                                                                  0x00aa
+#define regMMMC_VM_L2_PERFCOUNTER6_CFG_BASE_IDX                                                         2
+#define regMMMC_VM_L2_PERFCOUNTER7_CFG                                                                  0x00ab
+#define regMMMC_VM_L2_PERFCOUNTER7_CFG_BASE_IDX                                                         2
+#define regMMMC_VM_L2_PERFCOUNTER8_CFG                                                                  0x00ac
+#define regMMMC_VM_L2_PERFCOUNTER8_CFG_BASE_IDX                                                         2
+#define regMMMC_VM_L2_PERFCOUNTER9_CFG                                                                  0x00ad
+#define regMMMC_VM_L2_PERFCOUNTER9_CFG_BASE_IDX                                                         2
+#define regMMMC_VM_L2_PERFCOUNTER10_CFG                                                                 0x00ae
+#define regMMMC_VM_L2_PERFCOUNTER10_CFG_BASE_IDX                                                        2
+#define regMMMC_VM_L2_PERFCOUNTER11_CFG                                                                 0x00af
+#define regMMMC_VM_L2_PERFCOUNTER11_CFG_BASE_IDX                                                        2
+#define regMMMC_VM_L2_PERFCOUNTER12_CFG                                                                 0x00b0
+#define regMMMC_VM_L2_PERFCOUNTER12_CFG_BASE_IDX                                                        2
+#define regMMMC_VM_L2_PERFCOUNTER13_CFG                                                                 0x00b1
+#define regMMMC_VM_L2_PERFCOUNTER13_CFG_BASE_IDX                                                        2
+#define regMMMC_VM_L2_PERFCOUNTER14_CFG                                                                 0x00b2
+#define regMMMC_VM_L2_PERFCOUNTER14_CFG_BASE_IDX                                                        2
+#define regMMMC_VM_L2_PERFCOUNTER15_CFG                                                                 0x00b3
+#define regMMMC_VM_L2_PERFCOUNTER15_CFG_BASE_IDX                                                        2
+#define regMMMC_VM_L2_PERFCOUNTER_RSLT_CNTL                                                             0x00b4
+#define regMMMC_VM_L2_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                    2
+#define regMMUTCL2_PERFCOUNTER0_CFG                                                                     0x00b5
+#define regMMUTCL2_PERFCOUNTER0_CFG_BASE_IDX                                                            2
+#define regMMUTCL2_PERFCOUNTER1_CFG                                                                     0x00b6
+#define regMMUTCL2_PERFCOUNTER1_CFG_BASE_IDX                                                            2
+#define regMMUTCL2_PERFCOUNTER2_CFG                                                                     0x00b7
+#define regMMUTCL2_PERFCOUNTER2_CFG_BASE_IDX                                                            2
+#define regMMUTCL2_PERFCOUNTER3_CFG                                                                     0x00b8
+#define regMMUTCL2_PERFCOUNTER3_CFG_BASE_IDX                                                            2
+#define regMMUTCL2_PERFCOUNTER_RSLT_CNTL                                                                0x00b9
+#define regMMUTCL2_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                       2
+
+
+// addressBlock: mmhub_mmutcl2_mmatcl2pldec
+// base address: 0x662f0
+#define regMM_ATC_L2_PERFCOUNTER0_CFG                                                                   0x00bc
+#define regMM_ATC_L2_PERFCOUNTER0_CFG_BASE_IDX                                                          2
+#define regMM_ATC_L2_PERFCOUNTER1_CFG                                                                   0x00bd
+#define regMM_ATC_L2_PERFCOUNTER1_CFG_BASE_IDX                                                          2
+#define regMM_ATC_L2_PERFCOUNTER_RSLT_CNTL                                                              0x00cc
+#define regMM_ATC_L2_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                     2
+
+
+// addressBlock: mmhub_mmutcl2_mmvmsharedvcdec
+// base address: 0x66400
+#define regMMMC_VM_FB_LOCATION_BASE_LO32                                                                0x0100
+#define regMMMC_VM_FB_LOCATION_BASE_LO32_BASE_IDX                                                       2
+#define regMMMC_VM_FB_LOCATION_BASE_HI32                                                                0x0101
+#define regMMMC_VM_FB_LOCATION_BASE_HI32_BASE_IDX                                                       2
+#define regMMMC_VM_FB_LOCATION_TOP_LO32                                                                 0x0102
+#define regMMMC_VM_FB_LOCATION_TOP_LO32_BASE_IDX                                                        2
+#define regMMMC_VM_FB_LOCATION_TOP_HI32                                                                 0x0103
+#define regMMMC_VM_FB_LOCATION_TOP_HI32_BASE_IDX                                                        2
+#define regMMMC_VM_AGP_TOP_LO32                                                                         0x0104
+#define regMMMC_VM_AGP_TOP_LO32_BASE_IDX                                                                2
+#define regMMMC_VM_AGP_TOP_HI32                                                                         0x0105
+#define regMMMC_VM_AGP_TOP_HI32_BASE_IDX                                                                2
+#define regMMMC_VM_AGP_BOT_LO32                                                                         0x0106
+#define regMMMC_VM_AGP_BOT_LO32_BASE_IDX                                                                2
+#define regMMMC_VM_AGP_BOT_HI32                                                                         0x0107
+#define regMMMC_VM_AGP_BOT_HI32_BASE_IDX                                                                2
+#define regMMMC_VM_AGP_BASE_LO32                                                                        0x0108
+#define regMMMC_VM_AGP_BASE_LO32_BASE_IDX                                                               2
+#define regMMMC_VM_AGP_BASE_HI32                                                                        0x0109
+#define regMMMC_VM_AGP_BASE_HI32_BASE_IDX                                                               2
+#define regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32                                                        0x010a
+#define regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32_BASE_IDX                                               2
+#define regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32                                                        0x010b
+#define regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32_BASE_IDX                                               2
+#define regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32                                                       0x010c
+#define regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32_BASE_IDX                                              2
+#define regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32                                                       0x010d
+#define regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32_BASE_IDX                                              2
+#define regMMMC_VM_MX_L1_TLB_CNTL                                                                       0x010e
+#define regMMMC_VM_MX_L1_TLB_CNTL_BASE_IDX                                                              2
+
+
+// addressBlock: mmhub_mmutcl2_mmvml2vcdec
+// base address: 0x66440
+#define regMMVM_CONTEXT0_CNTL                                                                           0x0110
+#define regMMVM_CONTEXT0_CNTL_BASE_IDX                                                                  2
+#define regMMVM_CONTEXT1_CNTL                                                                           0x0111
+#define regMMVM_CONTEXT1_CNTL_BASE_IDX                                                                  2
+#define regMMVM_CONTEXT2_CNTL                                                                           0x0112
+#define regMMVM_CONTEXT2_CNTL_BASE_IDX                                                                  2
+#define regMMVM_CONTEXT3_CNTL                                                                           0x0113
+#define regMMVM_CONTEXT3_CNTL_BASE_IDX                                                                  2
+#define regMMVM_CONTEXT4_CNTL                                                                           0x0114
+#define regMMVM_CONTEXT4_CNTL_BASE_IDX                                                                  2
+#define regMMVM_CONTEXT5_CNTL                                                                           0x0115
+#define regMMVM_CONTEXT5_CNTL_BASE_IDX                                                                  2
+#define regMMVM_CONTEXT6_CNTL                                                                           0x0116
+#define regMMVM_CONTEXT6_CNTL_BASE_IDX                                                                  2
+#define regMMVM_CONTEXT7_CNTL                                                                           0x0117
+#define regMMVM_CONTEXT7_CNTL_BASE_IDX                                                                  2
+#define regMMVM_CONTEXT8_CNTL                                                                           0x0118
+#define regMMVM_CONTEXT8_CNTL_BASE_IDX                                                                  2
+#define regMMVM_CONTEXT9_CNTL                                                                           0x0119
+#define regMMVM_CONTEXT9_CNTL_BASE_IDX                                                                  2
+#define regMMVM_CONTEXT10_CNTL                                                                          0x011a
+#define regMMVM_CONTEXT10_CNTL_BASE_IDX                                                                 2
+#define regMMVM_CONTEXT11_CNTL                                                                          0x011b
+#define regMMVM_CONTEXT11_CNTL_BASE_IDX                                                                 2
+#define regMMVM_CONTEXT12_CNTL                                                                          0x011c
+#define regMMVM_CONTEXT12_CNTL_BASE_IDX                                                                 2
+#define regMMVM_CONTEXT13_CNTL                                                                          0x011d
+#define regMMVM_CONTEXT13_CNTL_BASE_IDX                                                                 2
+#define regMMVM_CONTEXT14_CNTL                                                                          0x011e
+#define regMMVM_CONTEXT14_CNTL_BASE_IDX                                                                 2
+#define regMMVM_CONTEXT15_CNTL                                                                          0x011f
+#define regMMVM_CONTEXT15_CNTL_BASE_IDX                                                                 2
+#define regMMVM_CONTEXTS_DISABLE                                                                        0x0120
+#define regMMVM_CONTEXTS_DISABLE_BASE_IDX                                                               2
+#define regMMVM_INVALIDATE_ENG0_SEM                                                                     0x0121
+#define regMMVM_INVALIDATE_ENG0_SEM_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG1_SEM                                                                     0x0122
+#define regMMVM_INVALIDATE_ENG1_SEM_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG2_SEM                                                                     0x0123
+#define regMMVM_INVALIDATE_ENG2_SEM_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG3_SEM                                                                     0x0124
+#define regMMVM_INVALIDATE_ENG3_SEM_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG4_SEM                                                                     0x0125
+#define regMMVM_INVALIDATE_ENG4_SEM_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG5_SEM                                                                     0x0126
+#define regMMVM_INVALIDATE_ENG5_SEM_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG6_SEM                                                                     0x0127
+#define regMMVM_INVALIDATE_ENG6_SEM_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG7_SEM                                                                     0x0128
+#define regMMVM_INVALIDATE_ENG7_SEM_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG8_SEM                                                                     0x0129
+#define regMMVM_INVALIDATE_ENG8_SEM_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG9_SEM                                                                     0x012a
+#define regMMVM_INVALIDATE_ENG9_SEM_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG10_SEM                                                                    0x012b
+#define regMMVM_INVALIDATE_ENG10_SEM_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG11_SEM                                                                    0x012c
+#define regMMVM_INVALIDATE_ENG11_SEM_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG12_SEM                                                                    0x012d
+#define regMMVM_INVALIDATE_ENG12_SEM_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG13_SEM                                                                    0x012e
+#define regMMVM_INVALIDATE_ENG13_SEM_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG14_SEM                                                                    0x012f
+#define regMMVM_INVALIDATE_ENG14_SEM_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG15_SEM                                                                    0x0130
+#define regMMVM_INVALIDATE_ENG15_SEM_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG16_SEM                                                                    0x0131
+#define regMMVM_INVALIDATE_ENG16_SEM_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG17_SEM                                                                    0x0132
+#define regMMVM_INVALIDATE_ENG17_SEM_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG0_REQ                                                                     0x0133
+#define regMMVM_INVALIDATE_ENG0_REQ_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG1_REQ                                                                     0x0134
+#define regMMVM_INVALIDATE_ENG1_REQ_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG2_REQ                                                                     0x0135
+#define regMMVM_INVALIDATE_ENG2_REQ_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG3_REQ                                                                     0x0136
+#define regMMVM_INVALIDATE_ENG3_REQ_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG4_REQ                                                                     0x0137
+#define regMMVM_INVALIDATE_ENG4_REQ_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG5_REQ                                                                     0x0138
+#define regMMVM_INVALIDATE_ENG5_REQ_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG6_REQ                                                                     0x0139
+#define regMMVM_INVALIDATE_ENG6_REQ_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG7_REQ                                                                     0x013a
+#define regMMVM_INVALIDATE_ENG7_REQ_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG8_REQ                                                                     0x013b
+#define regMMVM_INVALIDATE_ENG8_REQ_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG9_REQ                                                                     0x013c
+#define regMMVM_INVALIDATE_ENG9_REQ_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG10_REQ                                                                    0x013d
+#define regMMVM_INVALIDATE_ENG10_REQ_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG11_REQ                                                                    0x013e
+#define regMMVM_INVALIDATE_ENG11_REQ_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG12_REQ                                                                    0x013f
+#define regMMVM_INVALIDATE_ENG12_REQ_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG13_REQ                                                                    0x0140
+#define regMMVM_INVALIDATE_ENG13_REQ_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG14_REQ                                                                    0x0141
+#define regMMVM_INVALIDATE_ENG14_REQ_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG15_REQ                                                                    0x0142
+#define regMMVM_INVALIDATE_ENG15_REQ_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG16_REQ                                                                    0x0143
+#define regMMVM_INVALIDATE_ENG16_REQ_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG17_REQ                                                                    0x0144
+#define regMMVM_INVALIDATE_ENG17_REQ_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG0_ACK                                                                     0x0145
+#define regMMVM_INVALIDATE_ENG0_ACK_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG1_ACK                                                                     0x0146
+#define regMMVM_INVALIDATE_ENG1_ACK_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG2_ACK                                                                     0x0147
+#define regMMVM_INVALIDATE_ENG2_ACK_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG3_ACK                                                                     0x0148
+#define regMMVM_INVALIDATE_ENG3_ACK_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG4_ACK                                                                     0x0149
+#define regMMVM_INVALIDATE_ENG4_ACK_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG5_ACK                                                                     0x014a
+#define regMMVM_INVALIDATE_ENG5_ACK_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG6_ACK                                                                     0x014b
+#define regMMVM_INVALIDATE_ENG6_ACK_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG7_ACK                                                                     0x014c
+#define regMMVM_INVALIDATE_ENG7_ACK_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG8_ACK                                                                     0x014d
+#define regMMVM_INVALIDATE_ENG8_ACK_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG9_ACK                                                                     0x014e
+#define regMMVM_INVALIDATE_ENG9_ACK_BASE_IDX                                                            2
+#define regMMVM_INVALIDATE_ENG10_ACK                                                                    0x014f
+#define regMMVM_INVALIDATE_ENG10_ACK_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG11_ACK                                                                    0x0150
+#define regMMVM_INVALIDATE_ENG11_ACK_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG12_ACK                                                                    0x0151
+#define regMMVM_INVALIDATE_ENG12_ACK_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG13_ACK                                                                    0x0152
+#define regMMVM_INVALIDATE_ENG13_ACK_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG14_ACK                                                                    0x0153
+#define regMMVM_INVALIDATE_ENG14_ACK_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG15_ACK                                                                    0x0154
+#define regMMVM_INVALIDATE_ENG15_ACK_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG16_ACK                                                                    0x0155
+#define regMMVM_INVALIDATE_ENG16_ACK_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG17_ACK                                                                    0x0156
+#define regMMVM_INVALIDATE_ENG17_ACK_BASE_IDX                                                           2
+#define regMMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32                                                         0x0157
+#define regMMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG0_ADDR_RANGE_HI32                                                         0x0158
+#define regMMVM_INVALIDATE_ENG0_ADDR_RANGE_HI32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG1_ADDR_RANGE_LO32                                                         0x0159
+#define regMMVM_INVALIDATE_ENG1_ADDR_RANGE_LO32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG1_ADDR_RANGE_HI32                                                         0x015a
+#define regMMVM_INVALIDATE_ENG1_ADDR_RANGE_HI32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG2_ADDR_RANGE_LO32                                                         0x015b
+#define regMMVM_INVALIDATE_ENG2_ADDR_RANGE_LO32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG2_ADDR_RANGE_HI32                                                         0x015c
+#define regMMVM_INVALIDATE_ENG2_ADDR_RANGE_HI32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG3_ADDR_RANGE_LO32                                                         0x015d
+#define regMMVM_INVALIDATE_ENG3_ADDR_RANGE_LO32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG3_ADDR_RANGE_HI32                                                         0x015e
+#define regMMVM_INVALIDATE_ENG3_ADDR_RANGE_HI32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG4_ADDR_RANGE_LO32                                                         0x015f
+#define regMMVM_INVALIDATE_ENG4_ADDR_RANGE_LO32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG4_ADDR_RANGE_HI32                                                         0x0160
+#define regMMVM_INVALIDATE_ENG4_ADDR_RANGE_HI32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG5_ADDR_RANGE_LO32                                                         0x0161
+#define regMMVM_INVALIDATE_ENG5_ADDR_RANGE_LO32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG5_ADDR_RANGE_HI32                                                         0x0162
+#define regMMVM_INVALIDATE_ENG5_ADDR_RANGE_HI32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG6_ADDR_RANGE_LO32                                                         0x0163
+#define regMMVM_INVALIDATE_ENG6_ADDR_RANGE_LO32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG6_ADDR_RANGE_HI32                                                         0x0164
+#define regMMVM_INVALIDATE_ENG6_ADDR_RANGE_HI32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG7_ADDR_RANGE_LO32                                                         0x0165
+#define regMMVM_INVALIDATE_ENG7_ADDR_RANGE_LO32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG7_ADDR_RANGE_HI32                                                         0x0166
+#define regMMVM_INVALIDATE_ENG7_ADDR_RANGE_HI32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG8_ADDR_RANGE_LO32                                                         0x0167
+#define regMMVM_INVALIDATE_ENG8_ADDR_RANGE_LO32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG8_ADDR_RANGE_HI32                                                         0x0168
+#define regMMVM_INVALIDATE_ENG8_ADDR_RANGE_HI32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG9_ADDR_RANGE_LO32                                                         0x0169
+#define regMMVM_INVALIDATE_ENG9_ADDR_RANGE_LO32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG9_ADDR_RANGE_HI32                                                         0x016a
+#define regMMVM_INVALIDATE_ENG9_ADDR_RANGE_HI32_BASE_IDX                                                2
+#define regMMVM_INVALIDATE_ENG10_ADDR_RANGE_LO32                                                        0x016b
+#define regMMVM_INVALIDATE_ENG10_ADDR_RANGE_LO32_BASE_IDX                                               2
+#define regMMVM_INVALIDATE_ENG10_ADDR_RANGE_HI32                                                        0x016c
+#define regMMVM_INVALIDATE_ENG10_ADDR_RANGE_HI32_BASE_IDX                                               2
+#define regMMVM_INVALIDATE_ENG11_ADDR_RANGE_LO32                                                        0x016d
+#define regMMVM_INVALIDATE_ENG11_ADDR_RANGE_LO32_BASE_IDX                                               2
+#define regMMVM_INVALIDATE_ENG11_ADDR_RANGE_HI32                                                        0x016e
+#define regMMVM_INVALIDATE_ENG11_ADDR_RANGE_HI32_BASE_IDX                                               2
+#define regMMVM_INVALIDATE_ENG12_ADDR_RANGE_LO32                                                        0x016f
+#define regMMVM_INVALIDATE_ENG12_ADDR_RANGE_LO32_BASE_IDX                                               2
+#define regMMVM_INVALIDATE_ENG12_ADDR_RANGE_HI32                                                        0x0170
+#define regMMVM_INVALIDATE_ENG12_ADDR_RANGE_HI32_BASE_IDX                                               2
+#define regMMVM_INVALIDATE_ENG13_ADDR_RANGE_LO32                                                        0x0171
+#define regMMVM_INVALIDATE_ENG13_ADDR_RANGE_LO32_BASE_IDX                                               2
+#define regMMVM_INVALIDATE_ENG13_ADDR_RANGE_HI32                                                        0x0172
+#define regMMVM_INVALIDATE_ENG13_ADDR_RANGE_HI32_BASE_IDX                                               2
+#define regMMVM_INVALIDATE_ENG14_ADDR_RANGE_LO32                                                        0x0173
+#define regMMVM_INVALIDATE_ENG14_ADDR_RANGE_LO32_BASE_IDX                                               2
+#define regMMVM_INVALIDATE_ENG14_ADDR_RANGE_HI32                                                        0x0174
+#define regMMVM_INVALIDATE_ENG14_ADDR_RANGE_HI32_BASE_IDX                                               2
+#define regMMVM_INVALIDATE_ENG15_ADDR_RANGE_LO32                                                        0x0175
+#define regMMVM_INVALIDATE_ENG15_ADDR_RANGE_LO32_BASE_IDX                                               2
+#define regMMVM_INVALIDATE_ENG15_ADDR_RANGE_HI32                                                        0x0176
+#define regMMVM_INVALIDATE_ENG15_ADDR_RANGE_HI32_BASE_IDX                                               2
+#define regMMVM_INVALIDATE_ENG16_ADDR_RANGE_LO32                                                        0x0177
+#define regMMVM_INVALIDATE_ENG16_ADDR_RANGE_LO32_BASE_IDX                                               2
+#define regMMVM_INVALIDATE_ENG16_ADDR_RANGE_HI32                                                        0x0178
+#define regMMVM_INVALIDATE_ENG16_ADDR_RANGE_HI32_BASE_IDX                                               2
+#define regMMVM_INVALIDATE_ENG17_ADDR_RANGE_LO32                                                        0x0179
+#define regMMVM_INVALIDATE_ENG17_ADDR_RANGE_LO32_BASE_IDX                                               2
+#define regMMVM_INVALIDATE_ENG17_ADDR_RANGE_HI32                                                        0x017a
+#define regMMVM_INVALIDATE_ENG17_ADDR_RANGE_HI32_BASE_IDX                                               2
+#define regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32                                                      0x017b
+#define regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             2
+#define regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32                                                      0x017c
+#define regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             2
+#define regMMVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32                                                      0x017d
+#define regMMVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             2
+#define regMMVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32                                                      0x017e
+#define regMMVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             2
+#define regMMVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32                                                      0x017f
+#define regMMVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             2
+#define regMMVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32                                                      0x0180
+#define regMMVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             2
+#define regMMVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32                                                      0x0181
+#define regMMVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             2
+#define regMMVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32                                                      0x0182
+#define regMMVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             2
+#define regMMVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32                                                      0x0183
+#define regMMVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             2
+#define regMMVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32                                                      0x0184
+#define regMMVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             2
+#define regMMVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32                                                      0x0185
+#define regMMVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             2
+#define regMMVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32                                                      0x0186
+#define regMMVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             2
+#define regMMVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32                                                      0x0187
+#define regMMVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             2
+#define regMMVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32                                                      0x0188
+#define regMMVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             2
+#define regMMVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32                                                      0x0189
+#define regMMVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             2
+#define regMMVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32                                                      0x018a
+#define regMMVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             2
+#define regMMVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32                                                      0x018b
+#define regMMVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             2
+#define regMMVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32                                                      0x018c
+#define regMMVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             2
+#define regMMVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32                                                      0x018d
+#define regMMVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             2
+#define regMMVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32                                                      0x018e
+#define regMMVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             2
+#define regMMVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32                                                     0x018f
+#define regMMVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            2
+#define regMMVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32                                                     0x0190
+#define regMMVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            2
+#define regMMVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32                                                     0x0191
+#define regMMVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            2
+#define regMMVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32                                                     0x0192
+#define regMMVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            2
+#define regMMVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32                                                     0x0193
+#define regMMVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            2
+#define regMMVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32                                                     0x0194
+#define regMMVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            2
+#define regMMVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32                                                     0x0195
+#define regMMVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            2
+#define regMMVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32                                                     0x0196
+#define regMMVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            2
+#define regMMVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32                                                     0x0197
+#define regMMVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            2
+#define regMMVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32                                                     0x0198
+#define regMMVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            2
+#define regMMVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32                                                     0x0199
+#define regMMVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            2
+#define regMMVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32                                                     0x019a
+#define regMMVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            2
+#define regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32                                                     0x019b
+#define regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            2
+#define regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32                                                     0x019c
+#define regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            2
+#define regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32                                                     0x019d
+#define regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            2
+#define regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32                                                     0x019e
+#define regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            2
+#define regMMVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32                                                     0x019f
+#define regMMVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            2
+#define regMMVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32                                                     0x01a0
+#define regMMVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            2
+#define regMMVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32                                                     0x01a1
+#define regMMVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            2
+#define regMMVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32                                                     0x01a2
+#define regMMVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            2
+#define regMMVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32                                                     0x01a3
+#define regMMVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            2
+#define regMMVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32                                                     0x01a4
+#define regMMVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            2
+#define regMMVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32                                                     0x01a5
+#define regMMVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            2
+#define regMMVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32                                                     0x01a6
+#define regMMVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            2
+#define regMMVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32                                                     0x01a7
+#define regMMVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            2
+#define regMMVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32                                                     0x01a8
+#define regMMVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            2
+#define regMMVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32                                                     0x01a9
+#define regMMVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            2
+#define regMMVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32                                                     0x01aa
+#define regMMVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            2
+#define regMMVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32                                                     0x01ab
+#define regMMVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            2
+#define regMMVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32                                                     0x01ac
+#define regMMVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            2
+#define regMMVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32                                                     0x01ad
+#define regMMVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            2
+#define regMMVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32                                                     0x01ae
+#define regMMVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            2
+#define regMMVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32                                                    0x01af
+#define regMMVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           2
+#define regMMVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32                                                    0x01b0
+#define regMMVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           2
+#define regMMVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32                                                    0x01b1
+#define regMMVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           2
+#define regMMVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32                                                    0x01b2
+#define regMMVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           2
+#define regMMVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32                                                    0x01b3
+#define regMMVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           2
+#define regMMVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32                                                    0x01b4
+#define regMMVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           2
+#define regMMVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32                                                    0x01b5
+#define regMMVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           2
+#define regMMVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32                                                    0x01b6
+#define regMMVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           2
+#define regMMVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32                                                    0x01b7
+#define regMMVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           2
+#define regMMVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32                                                    0x01b8
+#define regMMVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           2
+#define regMMVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32                                                    0x01b9
+#define regMMVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           2
+#define regMMVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32                                                    0x01ba
+#define regMMVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           2
+#define regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32                                                       0x01bb
+#define regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              2
+#define regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32                                                       0x01bc
+#define regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              2
+#define regMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32                                                       0x01bd
+#define regMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              2
+#define regMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32                                                       0x01be
+#define regMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              2
+#define regMMVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32                                                       0x01bf
+#define regMMVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              2
+#define regMMVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32                                                       0x01c0
+#define regMMVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              2
+#define regMMVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32                                                       0x01c1
+#define regMMVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              2
+#define regMMVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32                                                       0x01c2
+#define regMMVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              2
+#define regMMVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32                                                       0x01c3
+#define regMMVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              2
+#define regMMVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32                                                       0x01c4
+#define regMMVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              2
+#define regMMVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32                                                       0x01c5
+#define regMMVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              2
+#define regMMVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32                                                       0x01c6
+#define regMMVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              2
+#define regMMVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32                                                       0x01c7
+#define regMMVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              2
+#define regMMVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32                                                       0x01c8
+#define regMMVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              2
+#define regMMVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32                                                       0x01c9
+#define regMMVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              2
+#define regMMVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32                                                       0x01ca
+#define regMMVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              2
+#define regMMVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32                                                       0x01cb
+#define regMMVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              2
+#define regMMVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32                                                       0x01cc
+#define regMMVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              2
+#define regMMVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32                                                       0x01cd
+#define regMMVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              2
+#define regMMVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32                                                       0x01ce
+#define regMMVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              2
+#define regMMVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32                                                      0x01cf
+#define regMMVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             2
+#define regMMVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32                                                      0x01d0
+#define regMMVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             2
+#define regMMVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32                                                      0x01d1
+#define regMMVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             2
+#define regMMVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32                                                      0x01d2
+#define regMMVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             2
+#define regMMVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32                                                      0x01d3
+#define regMMVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             2
+#define regMMVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32                                                      0x01d4
+#define regMMVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             2
+#define regMMVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32                                                      0x01d5
+#define regMMVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             2
+#define regMMVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32                                                      0x01d6
+#define regMMVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             2
+#define regMMVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32                                                      0x01d7
+#define regMMVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             2
+#define regMMVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32                                                      0x01d8
+#define regMMVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             2
+#define regMMVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32                                                      0x01d9
+#define regMMVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             2
+#define regMMVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32                                                      0x01da
+#define regMMVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             2
+#define regMMVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                                    0x01db
+#define regMMVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                           2
+#define regMMVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x01dc
+#define regMMVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  2
+#define regMMVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x01dd
+#define regMMVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  2
+#define regMMVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x01de
+#define regMMVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  2
+#define regMMVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x01df
+#define regMMVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  2
+#define regMMVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x01e0
+#define regMMVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  2
+#define regMMVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x01e1
+#define regMMVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  2
+#define regMMVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x01e2
+#define regMMVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  2
+#define regMMVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x01e3
+#define regMMVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  2
+#define regMMVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x01e4
+#define regMMVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  2
+#define regMMVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x01e5
+#define regMMVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  2
+#define regMMVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x01e6
+#define regMMVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 2
+#define regMMVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x01e7
+#define regMMVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 2
+#define regMMVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x01e8
+#define regMMVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 2
+#define regMMVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x01e9
+#define regMMVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 2
+#define regMMVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x01ea
+#define regMMVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 2
+#define regMMVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x01eb
+#define regMMVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 2
+
+
+// addressBlock: mmhub_mmutcl2_mmvmsharedvfdec
+// base address: 0x667c0
+#define regMMMC_VM_PCIE_ATOMIC_SUPPORTED                                                                0x01f0
+#define regMMMC_VM_PCIE_ATOMIC_SUPPORTED_BASE_IDX                                                       2
+
+
+// addressBlock: mmhub_mmutcl2_mmvmsharedhvdec
+// base address: 0x667e0
+#define regMMVM_PCIE_ATS_CNTL                                                                           0x01f8
+#define regMMVM_PCIE_ATS_CNTL_BASE_IDX                                                                  2
+#define regMMMC_VM_NB_MMIOBASE                                                                          0x01f9
+#define regMMMC_VM_NB_MMIOBASE_BASE_IDX                                                                 2
+#define regMMMC_VM_NB_MMIOLIMIT                                                                         0x01fa
+#define regMMMC_VM_NB_MMIOLIMIT_BASE_IDX                                                                2
+#define regMMMC_VM_NB_PCI_CTRL                                                                          0x01fb
+#define regMMMC_VM_NB_PCI_CTRL_BASE_IDX                                                                 2
+#define regMMMC_VM_NB_PCI_ARB                                                                           0x01fc
+#define regMMMC_VM_NB_PCI_ARB_BASE_IDX                                                                  2
+#define regMMMC_VM_NB_TOP_OF_DRAM_SLOT1                                                                 0x01fd
+#define regMMMC_VM_NB_TOP_OF_DRAM_SLOT1_BASE_IDX                                                        2
+#define regMMMC_VM_NB_LOWER_TOP_OF_DRAM2                                                                0x01fe
+#define regMMMC_VM_NB_LOWER_TOP_OF_DRAM2_BASE_IDX                                                       2
+#define regMMMC_VM_NB_UPPER_TOP_OF_DRAM2                                                                0x01ff
+#define regMMMC_VM_NB_UPPER_TOP_OF_DRAM2_BASE_IDX                                                       2
+#define regMMMC_VM_STEERING                                                                             0x0200
+#define regMMMC_VM_STEERING_BASE_IDX                                                                    2
+#define regMMMC_SHARED_VIRT_RESET_REQ                                                                   0x0201
+#define regMMMC_SHARED_VIRT_RESET_REQ_BASE_IDX                                                          2
+#define regMMMC_VM_XGMI_LFB_CNTL                                                                        0x0202
+#define regMMMC_VM_XGMI_LFB_CNTL_BASE_IDX                                                               2
+#define regMMMC_VM_XGMI_LFB_SIZE                                                                        0x0203
+#define regMMMC_VM_XGMI_LFB_SIZE_BASE_IDX                                                               2
+#define regMMMC_VM_HOST_MAPPING                                                                         0x0204
+#define regMMMC_VM_HOST_MAPPING_BASE_IDX                                                                2
+#define regMMMC_VM_XGMI_GPUIOV_ENABLE                                                                   0x0205
+#define regMMMC_VM_XGMI_GPUIOV_ENABLE_BASE_IDX                                                          2
+#define regMMMC_VM_CACHEABLE_DRAM_ADDRESS_START                                                         0x0206
+#define regMMMC_VM_CACHEABLE_DRAM_ADDRESS_START_BASE_IDX                                                2
+#define regMMMC_VM_CACHEABLE_DRAM_ADDRESS_END                                                           0x0207
+#define regMMMC_VM_CACHEABLE_DRAM_ADDRESS_END_BASE_IDX                                                  2
+#define regMMMC_VM_LOCAL_SYSMEM_ADDRESS_START                                                           0x0208
+#define regMMMC_VM_LOCAL_SYSMEM_ADDRESS_START_BASE_IDX                                                  2
+#define regMMMC_VM_LOCAL_SYSMEM_ADDRESS_END                                                             0x0209
+#define regMMMC_VM_LOCAL_SYSMEM_ADDRESS_END_BASE_IDX                                                    2
+#define regMMMC_VM_LPDDR_ADDRESS_START                                                                  0x020a
+#define regMMMC_VM_LPDDR_ADDRESS_START_BASE_IDX                                                         2
+#define regMMMC_VM_LPDDR_ADDRESS_END                                                                    0x020b
+#define regMMMC_VM_LPDDR_ADDRESS_END_BASE_IDX                                                           2
+#define regMMMC_VM_APT_CNTL                                                                             0x020c
+#define regMMMC_VM_APT_CNTL_BASE_IDX                                                                    2
+#define regMMUTCL2_HARVEST_BYPASS_GROUPS                                                                0x020d
+#define regMMUTCL2_HARVEST_BYPASS_GROUPS_BASE_IDX                                                       2
+
+
+// addressBlock: mmhub_mmutcl2_mmatcl2dec
+// base address: 0x669d0
+#define regMM_ATC_L2_CNTL                                                                               0x0274
+#define regMM_ATC_L2_CNTL_BASE_IDX                                                                      2
+#define regMM_ATC_L2_CNTL2                                                                              0x0275
+#define regMM_ATC_L2_CNTL2_BASE_IDX                                                                     2
+#define regMM_ATC_L2_CACHE_DATA0                                                                        0x0278
+#define regMM_ATC_L2_CACHE_DATA0_BASE_IDX                                                               2
+#define regMM_ATC_L2_CACHE_DATA1                                                                        0x0279
+#define regMM_ATC_L2_CACHE_DATA1_BASE_IDX                                                               2
+#define regMM_ATC_L2_CACHE_DATA2                                                                        0x027a
+#define regMM_ATC_L2_CACHE_DATA2_BASE_IDX                                                               2
+#define regMM_ATC_L2_CNTL3                                                                              0x027d
+#define regMM_ATC_L2_CNTL3_BASE_IDX                                                                     2
+#define regMM_ATC_L2_CNTL4                                                                              0x027e
+#define regMM_ATC_L2_CNTL4_BASE_IDX                                                                     2
+#define regMM_ATC_L2_CNTL5                                                                              0x027f
+#define regMM_ATC_L2_CNTL5_BASE_IDX                                                                     2
+#define regMM_ATC_L2_MM_GROUP_RT_CLASSES                                                                0x0280
+#define regMM_ATC_L2_MM_GROUP_RT_CLASSES_BASE_IDX                                                       2
+#define regMM_ATC_L2_STATUS                                                                             0x0281
+#define regMM_ATC_L2_STATUS_BASE_IDX                                                                    2
+#define regMM_ATC_L2_MISC_CG                                                                            0x0282
+#define regMM_ATC_L2_MISC_CG_BASE_IDX                                                                   2
+#define regMM_ATC_L2_CGTT_CLK_CTRL                                                                      0x0283
+#define regMM_ATC_L2_CGTT_CLK_CTRL_BASE_IDX                                                             2
+#define regMM_ATC_L2_SDPPORT_CTRL                                                                       0x0287
+#define regMM_ATC_L2_SDPPORT_CTRL_BASE_IDX                                                              2
+
+
+// addressBlock: mmhub_mmutcl2_mmvml2pspdec
+// base address: 0x66b40
+#define regMMVM_IOMMU_CONTROL_REGISTER                                                                  0x02d4
+#define regMMVM_IOMMU_CONTROL_REGISTER_BASE_IDX                                                         2
+#define regMMVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER                                         0x02d5
+#define regMMVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER_BASE_IDX                                2
+
+
+// addressBlock: mmhub_mmutcl2_mmvmsharedpspdec
+// base address: 0x66b90
+#define regMMMC_VM_FB_OFFSET                                                                            0x02ec
+#define regMMMC_VM_FB_OFFSET_BASE_IDX                                                                   2
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_4_2_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_4_2_0_sh_mask.h
new file mode 100644
index 000000000000..8d8cb50a3244
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_4_2_0_sh_mask.h
@@ -0,0 +1,3013 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _mmhub_4_2_0_SH_MASK_HEADER
+#define _mmhub_4_2_0_SH_MASK_HEADER
+
+
+// addressBlock: mmhub_dagb_dagbdec
+//DAGB0_CNTL_MISC2
+#define DAGB0_CNTL_MISC2__DAGB_BUSY_OVERRIDE__SHIFT                                                           0x0
+#define DAGB0_CNTL_MISC2__SWAP_CTL__SHIFT                                                                     0x1
+#define DAGB0_CNTL_MISC2__RDRET_FIFO_PERF__SHIFT                                                              0x2
+#define DAGB0_CNTL_MISC2__DISABLE_RDRET_TAP_CHAIN_FGCG__SHIFT                                                 0x3
+#define DAGB0_CNTL_MISC2__DISABLE_WRRET_TAP_CHAIN_FGCG__SHIFT                                                 0x4
+#define DAGB0_CNTL_MISC2__DISABLE_MCA_INTR_REQ_FGCG__SHIFT                                                    0x5
+#define DAGB0_CNTL_MISC2__SWAP_CTL_MASK                                                                       0x00000002L
+#define DAGB0_CNTL_MISC2__RDRET_FIFO_PERF_MASK                                                                0x00000004L
+#define DAGB0_CNTL_MISC2__DISABLE_RDRET_TAP_CHAIN_FGCG_MASK                                                   0x00000008L
+#define DAGB0_CNTL_MISC2__DISABLE_WRRET_TAP_CHAIN_FGCG_MASK                                                   0x00000010L
+//DAGB1_CNTL_MISC2
+#define DAGB1_CNTL_MISC2__DAGB_BUSY_OVERRIDE__SHIFT                                                           0x0
+#define DAGB1_CNTL_MISC2__SWAP_CTL__SHIFT                                                                     0x1
+#define DAGB1_CNTL_MISC2__RDRET_FIFO_PERF__SHIFT                                                              0x2
+#define DAGB1_CNTL_MISC2__DISABLE_RDRET_TAP_CHAIN_FGCG__SHIFT                                                 0x3
+#define DAGB1_CNTL_MISC2__DISABLE_WRRET_TAP_CHAIN_FGCG__SHIFT                                                 0x4
+#define DAGB1_CNTL_MISC2__DISABLE_MCA_INTR_REQ_FGCG__SHIFT                                                    0x5
+#define DAGB1_CNTL_MISC2__SWAP_CTL_MASK                                                                       0x00000002L
+#define DAGB1_CNTL_MISC2__RDRET_FIFO_PERF_MASK                                                                0x00000004L
+#define DAGB1_CNTL_MISC2__DISABLE_RDRET_TAP_CHAIN_FGCG_MASK                                                   0x00000008L
+#define DAGB1_CNTL_MISC2__DISABLE_WRRET_TAP_CHAIN_FGCG_MASK                                                   0x00000010L
+
+
+// addressBlock: mmhub_mm_cane_mmcanedec
+//MM_CANE_ICG_CTRL
+#define MM_CANE_ICG_CTRL__SOFT_OVERRIDE_IREQ0__SHIFT                                                          0x0
+#define MM_CANE_ICG_CTRL__SOFT_OVERRIDE_ATRET__SHIFT                                                          0x1
+#define MM_CANE_ICG_CTRL__SOFT_OVERRIDE_OREQ__SHIFT                                                           0x2
+#define MM_CANE_ICG_CTRL__SOFT_OVERRIDE_REGISTER__SHIFT                                                       0x3
+#define MM_CANE_ICG_CTRL__SOFT_OVERRIDE_SDPM_RETURN__SHIFT                                                    0x4
+#define MM_CANE_ICG_CTRL__SOFT_OVERRIDE_IREQ0_MASK                                                            0x00000001L
+#define MM_CANE_ICG_CTRL__SOFT_OVERRIDE_ATRET_MASK                                                            0x00000002L
+#define MM_CANE_ICG_CTRL__SOFT_OVERRIDE_OREQ_MASK                                                             0x00000004L
+#define MM_CANE_ICG_CTRL__SOFT_OVERRIDE_REGISTER_MASK                                                         0x00000008L
+#define MM_CANE_ICG_CTRL__SOFT_OVERRIDE_SDPM_RETURN_MASK                                                      0x00000010L
+
+
+// addressBlock: mmhub_mmutcl2_mmvmsharedpfdec
+//MMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB
+#define MMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB__PHYSICAL_PAGE_NUMBER_LSB__SHIFT                             0x0
+#define MMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB__PHYSICAL_PAGE_NUMBER_LSB_MASK                               0xFFFFFFFFL
+//MMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB
+#define MMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB__PHYSICAL_PAGE_NUMBER_MSB__SHIFT                             0x0
+#define MMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB__PHYSICAL_PAGE_NUMBER_MSB_MASK                               0x000000FFL
+//MMUTCL2_CGTT_CLK_CTRL
+#define MMUTCL2_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                0x0
+#define MMUTCL2_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                          0x5
+#define MMUTCL2_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                                    0xd
+#define MMUTCL2_CGTT_CLK_CTRL__MIN_MGLS__SHIFT                                                                0x1a
+#define MMUTCL2_CGTT_CLK_CTRL__CGLS_DISABLE__SHIFT                                                            0x1d
+#define MMUTCL2_CGTT_CLK_CTRL__LS_DISABLE__SHIFT                                                              0x1e
+#define MMUTCL2_CGTT_CLK_CTRL__BUSY_OVERRIDE__SHIFT                                                           0x1f
+#define MMUTCL2_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                  0x0000001FL
+#define MMUTCL2_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                            0x00001FE0L
+#define MMUTCL2_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                                      0x03FFE000L
+#define MMUTCL2_CGTT_CLK_CTRL__MIN_MGLS_MASK                                                                  0x1C000000L
+#define MMUTCL2_CGTT_CLK_CTRL__CGLS_DISABLE_MASK                                                              0x20000000L
+#define MMUTCL2_CGTT_CLK_CTRL__LS_DISABLE_MASK                                                                0x40000000L
+#define MMUTCL2_CGTT_CLK_CTRL__BUSY_OVERRIDE_MASK                                                             0x80000000L
+//MMMC_SHARED_ACTIVE_FCN_ID
+#define MMMC_SHARED_ACTIVE_FCN_ID__VFID__SHIFT                                                                0x0
+#define MMMC_SHARED_ACTIVE_FCN_ID__VF__SHIFT                                                                  0x1f
+#define MMMC_SHARED_ACTIVE_FCN_ID__VFID_MASK                                                                  0x0000001FL
+#define MMMC_SHARED_ACTIVE_FCN_ID__VF_MASK                                                                    0x80000000L
+//MMUTCL2_CGTT_BUSY_CTRL
+#define MMUTCL2_CGTT_BUSY_CTRL__READ_DELAY__SHIFT                                                             0x0
+#define MMUTCL2_CGTT_BUSY_CTRL__ALWAYS_BUSY__SHIFT                                                            0x5
+#define MMUTCL2_CGTT_BUSY_CTRL__READ_DELAY_MASK                                                               0x0000001FL
+#define MMUTCL2_CGTT_BUSY_CTRL__ALWAYS_BUSY_MASK                                                              0x00000020L
+//MMUTCL2_GROUP_RET_FAULT_STATUS
+#define MMUTCL2_GROUP_RET_FAULT_STATUS__FAULT_GROUPS__SHIFT                                                   0x0
+#define MMUTCL2_GROUP_RET_FAULT_STATUS__FAULT_GROUPS_MASK                                                     0xFFFFFFFFL
+
+
+// addressBlock: mmhub_mmutcl2_mmvml2pfdec
+//MMVM_L2_CNTL
+#define MMVM_L2_CNTL__ENABLE_L2_CACHE__SHIFT                                                                  0x0
+#define MMVM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING__SHIFT                                                    0x1
+#define MMVM_L2_CNTL__L2_CACHE_PTE_ENDIAN_SWAP_MODE__SHIFT                                                    0x2
+#define MMVM_L2_CNTL__L2_CACHE_PDE_ENDIAN_SWAP_MODE__SHIFT                                                    0x4
+#define MMVM_L2_CNTL__L2_PDE0_CACHE_TAG_GENERATION_MODE__SHIFT                                                0x8
+#define MMVM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                          0x9
+#define MMVM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                         0xa
+#define MMVM_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY__SHIFT                                         0xb
+#define MMVM_L2_CNTL__L2_PDE0_CACHE_SPLIT_MODE__SHIFT                                                         0xc
+#define MMVM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT                                                          0xf
+#define MMVM_L2_CNTL__PDE_FAULT_CLASSIFICATION__SHIFT                                                         0x12
+#define MMVM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT                                                    0x13
+#define MMVM_L2_CNTL__IDENTITY_MODE_FRAGMENT_SIZE__SHIFT                                                      0x15
+#define MMVM_L2_CNTL__L2_PTE_CACHE_ADDR_MODE__SHIFT                                                           0x1a
+#define MMVM_L2_CNTL__ENABLE_L2_CACHE_MASK                                                                    0x00000001L
+#define MMVM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING_MASK                                                      0x00000002L
+#define MMVM_L2_CNTL__L2_CACHE_PTE_ENDIAN_SWAP_MODE_MASK                                                      0x0000000CL
+#define MMVM_L2_CNTL__L2_CACHE_PDE_ENDIAN_SWAP_MODE_MASK                                                      0x00000030L
+#define MMVM_L2_CNTL__L2_PDE0_CACHE_TAG_GENERATION_MODE_MASK                                                  0x00000100L
+#define MMVM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK                                            0x00000200L
+#define MMVM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK                                           0x00000400L
+#define MMVM_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY_MASK                                           0x00000800L
+#define MMVM_L2_CNTL__L2_PDE0_CACHE_SPLIT_MODE_MASK                                                           0x00007000L
+#define MMVM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE_MASK                                                            0x00038000L
+#define MMVM_L2_CNTL__PDE_FAULT_CLASSIFICATION_MASK                                                           0x00040000L
+#define MMVM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE_MASK                                                      0x00180000L
+#define MMVM_L2_CNTL__IDENTITY_MODE_FRAGMENT_SIZE_MASK                                                        0x03E00000L
+#define MMVM_L2_CNTL__L2_PTE_CACHE_ADDR_MODE_MASK                                                             0x0C000000L
+//MMVM_L2_CNTL2
+#define MMVM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS__SHIFT                                                          0x0
+#define MMVM_L2_CNTL2__INVALIDATE_L2_CACHE__SHIFT                                                             0x1
+#define MMVM_L2_CNTL2__DISABLE_INVALIDATE_PER_DOMAIN__SHIFT                                                   0x15
+#define MMVM_L2_CNTL2__DISABLE_BIGK_CACHE_OPTIMIZATION__SHIFT                                                 0x16
+#define MMVM_L2_CNTL2__L2_PTE_CACHE_VMID_MODE__SHIFT                                                          0x17
+#define MMVM_L2_CNTL2__INVALIDATE_CACHE_MODE__SHIFT                                                           0x1a
+#define MMVM_L2_CNTL2__PDE_CACHE_EFFECTIVE_SIZE__SHIFT                                                        0x1c
+#define MMVM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS_MASK                                                            0x00000001L
+#define MMVM_L2_CNTL2__INVALIDATE_L2_CACHE_MASK                                                               0x00000002L
+#define MMVM_L2_CNTL2__DISABLE_INVALIDATE_PER_DOMAIN_MASK                                                     0x00200000L
+#define MMVM_L2_CNTL2__DISABLE_BIGK_CACHE_OPTIMIZATION_MASK                                                   0x00400000L
+#define MMVM_L2_CNTL2__L2_PTE_CACHE_VMID_MODE_MASK                                                            0x03800000L
+#define MMVM_L2_CNTL2__INVALIDATE_CACHE_MODE_MASK                                                             0x0C000000L
+#define MMVM_L2_CNTL2__PDE_CACHE_EFFECTIVE_SIZE_MASK                                                          0x70000000L
+//MMVM_L2_CNTL3
+#define MMVM_L2_CNTL3__BANK_SELECT__SHIFT                                                                     0x0
+#define MMVM_L2_CNTL3__L2_CACHE_UPDATE_MODE__SHIFT                                                            0x6
+#define MMVM_L2_CNTL3__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE__SHIFT                                        0x8
+#define MMVM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                                                     0xf
+#define MMVM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY__SHIFT                                                     0x14
+#define MMVM_L2_CNTL3__L2_CACHE_4K_EFFECTIVE_SIZE__SHIFT                                                      0x15
+#define MMVM_L2_CNTL3__L2_CACHE_BIGK_EFFECTIVE_SIZE__SHIFT                                                    0x18
+#define MMVM_L2_CNTL3__L2_CACHE_4K_FORCE_MISS__SHIFT                                                          0x1c
+#define MMVM_L2_CNTL3__L2_CACHE_BIGK_FORCE_MISS__SHIFT                                                        0x1d
+#define MMVM_L2_CNTL3__PDE_CACHE_FORCE_MISS__SHIFT                                                            0x1e
+#define MMVM_L2_CNTL3__L2_CACHE_4K_ASSOCIATIVITY__SHIFT                                                       0x1f
+#define MMVM_L2_CNTL3__BANK_SELECT_MASK                                                                       0x0000003FL
+#define MMVM_L2_CNTL3__L2_CACHE_UPDATE_MODE_MASK                                                              0x000000C0L
+#define MMVM_L2_CNTL3__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE_MASK                                          0x00001F00L
+#define MMVM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                                                       0x000F8000L
+#define MMVM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK                                                       0x00100000L
+#define MMVM_L2_CNTL3__L2_CACHE_4K_EFFECTIVE_SIZE_MASK                                                        0x00E00000L
+#define MMVM_L2_CNTL3__L2_CACHE_BIGK_EFFECTIVE_SIZE_MASK                                                      0x0F000000L
+#define MMVM_L2_CNTL3__L2_CACHE_4K_FORCE_MISS_MASK                                                            0x10000000L
+#define MMVM_L2_CNTL3__L2_CACHE_BIGK_FORCE_MISS_MASK                                                          0x20000000L
+#define MMVM_L2_CNTL3__PDE_CACHE_FORCE_MISS_MASK                                                              0x40000000L
+#define MMVM_L2_CNTL3__L2_CACHE_4K_ASSOCIATIVITY_MASK                                                         0x80000000L
+//MMVM_L2_STATUS
+#define MMVM_L2_STATUS__L2_BUSY__SHIFT                                                                        0x0
+#define MMVM_L2_STATUS__CONTEXT_DOMAIN_BUSY__SHIFT                                                            0x1
+#define MMVM_L2_STATUS__FOUND_4K_PTE_CACHE_PARITY_ERRORS__SHIFT                                               0x11
+#define MMVM_L2_STATUS__FOUND_BIGK_PTE_CACHE_PARITY_ERRORS__SHIFT                                             0x12
+#define MMVM_L2_STATUS__FOUND_PDE0_CACHE_PARITY_ERRORS__SHIFT                                                 0x13
+#define MMVM_L2_STATUS__FOUND_PDE1_CACHE_PARITY_ERRORS__SHIFT                                                 0x14
+#define MMVM_L2_STATUS__FOUND_PDE2_CACHE_PARITY_ERRORS__SHIFT                                                 0x15
+#define MMVM_L2_STATUS__FOUND_PDE3_CACHE_PARITY_ERRORS__SHIFT                                                 0x16
+#define MMVM_L2_STATUS__L2_BUSY_MASK                                                                          0x00000001L
+#define MMVM_L2_STATUS__CONTEXT_DOMAIN_BUSY_MASK                                                              0x0001FFFEL
+#define MMVM_L2_STATUS__FOUND_4K_PTE_CACHE_PARITY_ERRORS_MASK                                                 0x00020000L
+#define MMVM_L2_STATUS__FOUND_BIGK_PTE_CACHE_PARITY_ERRORS_MASK                                               0x00040000L
+#define MMVM_L2_STATUS__FOUND_PDE0_CACHE_PARITY_ERRORS_MASK                                                   0x00080000L
+#define MMVM_L2_STATUS__FOUND_PDE1_CACHE_PARITY_ERRORS_MASK                                                   0x00100000L
+#define MMVM_L2_STATUS__FOUND_PDE2_CACHE_PARITY_ERRORS_MASK                                                   0x00200000L
+//MMVM_DUMMY_PAGE_FAULT_CNTL
+#define MMVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_FAULT_ENABLE__SHIFT                                            0x0
+#define MMVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_ADDRESS_LOGICAL__SHIFT                                         0x1
+#define MMVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_COMPARE_MSBS__SHIFT                                            0x2
+#define MMVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_FAULT_ENABLE_MASK                                              0x00000001L
+#define MMVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_ADDRESS_LOGICAL_MASK                                           0x00000002L
+#define MMVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_COMPARE_MSBS_MASK                                              0x000000FCL
+//MMVM_DUMMY_PAGE_FAULT_ADDR_LO32
+#define MMVM_DUMMY_PAGE_FAULT_ADDR_LO32__DUMMY_PAGE_ADDR_LO32__SHIFT                                          0x0
+#define MMVM_DUMMY_PAGE_FAULT_ADDR_LO32__DUMMY_PAGE_ADDR_LO32_MASK                                            0xFFFFFFFFL
+//MMVM_DUMMY_PAGE_FAULT_ADDR_HI32
+#define MMVM_DUMMY_PAGE_FAULT_ADDR_HI32__DUMMY_PAGE_ADDR_HI13__SHIFT                                          0x0
+#define MMVM_DUMMY_PAGE_FAULT_ADDR_HI32__DUMMY_PAGE_ADDR_HI13_MASK                                            0x00001FFFL
+//MMVM_INVALIDATE_CNTL
+#define MMVM_INVALIDATE_CNTL__PRI_REG_ALTERNATING__SHIFT                                                      0x0
+#define MMVM_INVALIDATE_CNTL__MAX_REG_OUTSTANDING__SHIFT                                                      0x8
+#define MMVM_INVALIDATE_CNTL__PRI_REG_ALTERNATING_MASK                                                        0x000000FFL
+#define MMVM_INVALIDATE_CNTL__MAX_REG_OUTSTANDING_MASK                                                        0x0000FF00L
+//MMVM_L2_PROTECTION_FAULT_CNTL_LO32
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                         0x0
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__ALLOW_SUBSEQUENT_PROTECTION_FAULT_STATUS_ADDR_UPDATES__SHIFT      0x1
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                      0x2
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                       0x3
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__PDE1_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                       0x4
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__PDE2_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                       0x5
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__PDE3_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                       0x6
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT          0x7
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__NACK_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                       0x8
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                 0x9
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                      0xa
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                       0xb
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                      0xc
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                    0xd
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                         0xe
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__OTHER_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                   0x1e
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__CRASH_ON_NO_RETRY_FAULT__SHIFT                                    0x1f
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                           0x00000001L
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__ALLOW_SUBSEQUENT_PROTECTION_FAULT_STATUS_ADDR_UPDATES_MASK        0x00000002L
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                        0x00000004L
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                         0x00000008L
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__PDE1_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                         0x00000010L
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__PDE2_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                         0x00000020L
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__PDE3_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                         0x00000040L
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT_MASK            0x00000080L
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__NACK_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                         0x00000100L
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                   0x00000200L
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                        0x00000400L
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                         0x00000800L
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                        0x00001000L
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                      0x00002000L
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                           0x3FFFC000L
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__OTHER_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                     0x40000000L
+#define MMVM_L2_PROTECTION_FAULT_CNTL_LO32__CRASH_ON_NO_RETRY_FAULT_MASK                                      0x80000000L
+//MMVM_L2_PROTECTION_FAULT_CNTL_HI32
+#define MMVM_L2_PROTECTION_FAULT_CNTL_HI32__CRASH_ON_RETRY_FAULT__SHIFT                                       0x0
+#define MMVM_L2_PROTECTION_FAULT_CNTL_HI32__CRASH_ON_RETRY_FAULT_MASK                                         0x00000001L
+//MMVM_L2_PROTECTION_FAULT_CNTL2
+#define MMVM_L2_PROTECTION_FAULT_CNTL2__CLIENT_ID_PRT_FAULT_INTERRUPT__SHIFT                                  0x0
+#define MMVM_L2_PROTECTION_FAULT_CNTL2__OTHER_CLIENT_ID_PRT_FAULT_INTERRUPT__SHIFT                            0x10
+#define MMVM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE__SHIFT                                      0x11
+#define MMVM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY__SHIFT                           0x12
+#define MMVM_L2_PROTECTION_FAULT_CNTL2__ENABLE_RETRY_FAULT_INTERRUPT__SHIFT                                   0x13
+#define MMVM_L2_PROTECTION_FAULT_CNTL2__CLIENT_ID_PRT_FAULT_INTERRUPT_MASK                                    0x0000FFFFL
+#define MMVM_L2_PROTECTION_FAULT_CNTL2__OTHER_CLIENT_ID_PRT_FAULT_INTERRUPT_MASK                              0x00010000L
+#define MMVM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_MASK                                        0x00020000L
+#define MMVM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY_MASK                             0x00040000L
+#define MMVM_L2_PROTECTION_FAULT_CNTL2__ENABLE_RETRY_FAULT_INTERRUPT_MASK                                     0x00080000L
+//MMVM_L2_PROTECTION_FAULT_MM_CNTL3
+#define MMVM_L2_PROTECTION_FAULT_MM_CNTL3__VML1_READ_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                0x0
+#define MMVM_L2_PROTECTION_FAULT_MM_CNTL3__VML1_READ_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                  0xFFFFFFFFL
+//MMVM_L2_PROTECTION_FAULT_MM_CNTL4
+#define MMVM_L2_PROTECTION_FAULT_MM_CNTL4__VML1_WRITE_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT               0x0
+#define MMVM_L2_PROTECTION_FAULT_MM_CNTL4__VML1_WRITE_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                 0xFFFFFFFFL
+//MMVM_L2_PROTECTION_FAULT_STATUS_LO32
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__MORE_FAULTS__SHIFT                                              0x0
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__WALKER_ERROR__SHIFT                                             0x1
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__PERMISSION_FAULTS__SHIFT                                        0x5
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__MAPPING_ERROR__SHIFT                                            0xa
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__CID__SHIFT                                                      0xb
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__RW__SHIFT                                                       0x14
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__ATOMIC__SHIFT                                                   0x15
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__VMID__SHIFT                                                     0x16
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__VF__SHIFT                                                       0x1a
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__VFID__SHIFT                                                     0x1b
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__MORE_FAULTS_MASK                                                0x00000001L
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__WALKER_ERROR_MASK                                               0x0000001EL
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__PERMISSION_FAULTS_MASK                                          0x000003E0L
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__MAPPING_ERROR_MASK                                              0x00000400L
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__CID_MASK                                                        0x000FF800L
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__RW_MASK                                                         0x00100000L
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__ATOMIC_MASK                                                     0x00200000L
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__VMID_MASK                                                       0x03C00000L
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__VF_MASK                                                         0x04000000L
+#define MMVM_L2_PROTECTION_FAULT_STATUS_LO32__VFID_MASK                                                       0xF8000000L
+//MMVM_L2_PROTECTION_FAULT_STATUS_HI32
+#define MMVM_L2_PROTECTION_FAULT_STATUS_HI32__PRT__SHIFT                                                      0x0
+#define MMVM_L2_PROTECTION_FAULT_STATUS_HI32__UCE__SHIFT                                                      0x1
+#define MMVM_L2_PROTECTION_FAULT_STATUS_HI32__FED__SHIFT                                                      0x2
+#define MMVM_L2_PROTECTION_FAULT_STATUS_HI32__PRT_MASK                                                        0x00000001L
+#define MMVM_L2_PROTECTION_FAULT_STATUS_HI32__UCE_MASK                                                        0x00000002L
+#define MMVM_L2_PROTECTION_FAULT_STATUS_HI32__FED_MASK                                                        0x00000004L
+//MMVM_L2_PROTECTION_FAULT_ADDR_LO32
+#define MMVM_L2_PROTECTION_FAULT_ADDR_LO32__LOGICAL_PAGE_ADDR_LO32__SHIFT                                     0x0
+#define MMVM_L2_PROTECTION_FAULT_ADDR_LO32__LOGICAL_PAGE_ADDR_LO32_MASK                                       0xFFFFFFFFL
+//MMVM_L2_PROTECTION_FAULT_ADDR_HI32
+#define MMVM_L2_PROTECTION_FAULT_ADDR_HI32__LOGICAL_PAGE_ADDR_HI13__SHIFT                                     0x0
+#define MMVM_L2_PROTECTION_FAULT_ADDR_HI32__LOGICAL_PAGE_ADDR_HI13_MASK                                       0x00001FFFL
+//MMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32
+#define MMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32__PHYSICAL_PAGE_ADDR_LO32__SHIFT                            0x0
+#define MMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32__PHYSICAL_PAGE_ADDR_LO32_MASK                              0xFFFFFFFFL
+//MMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32
+#define MMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32__PHYSICAL_PAGE_ADDR_HI8__SHIFT                             0x0
+#define MMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32__PHYSICAL_PAGE_ADDR_HI8_MASK                               0x000000FFL
+//MMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32
+#define MMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                     0x0
+#define MMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                       0xFFFFFFFFL
+//MMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32
+#define MMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                     0x0
+#define MMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                       0x00001FFFL
+//MMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32
+#define MMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                    0x0
+#define MMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                      0xFFFFFFFFL
+//MMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32
+#define MMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                    0x0
+#define MMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                      0x00001FFFL
+//MMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32
+#define MMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32__PHYSICAL_PAGE_OFFSET_LO32__SHIFT                       0x0
+#define MMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32__PHYSICAL_PAGE_OFFSET_LO32_MASK                         0xFFFFFFFFL
+//MMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32
+#define MMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32__PHYSICAL_PAGE_OFFSET_HI8__SHIFT                        0x0
+#define MMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32__PHYSICAL_PAGE_OFFSET_HI8_MASK                          0x000000FFL
+//MMVM_L2_CNTL4
+#define MMVM_L2_CNTL4__L2_CACHE_4K_PARTITION_COUNT__SHIFT                                                     0x0
+#define MMVM_L2_CNTL4__VMC_TAP_PDE_REQUEST_PHYSICAL__SHIFT                                                    0x6
+#define MMVM_L2_CNTL4__VMC_TAP_PTE_REQUEST_PHYSICAL__SHIFT                                                    0x7
+#define MMVM_L2_CNTL4__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                         0x8
+#define MMVM_L2_CNTL4__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                        0x12
+#define MMVM_L2_CNTL4__BPM_CGCGLS_OVERRIDE__SHIFT                                                             0x1c
+#define MMVM_L2_CNTL4__GC_CH_FGCG_OFF__SHIFT                                                                  0x1d
+#define MMVM_L2_CNTL4__VFIFO_HEAD_OF_QUEUE__SHIFT                                                             0x1e
+#define MMVM_L2_CNTL4__VFIFO_VISIBLE_BANK_SILOS__SHIFT                                                        0x1f
+#define MMVM_L2_CNTL4__L2_CACHE_4K_PARTITION_COUNT_MASK                                                       0x0000003FL
+#define MMVM_L2_CNTL4__VMC_TAP_PDE_REQUEST_PHYSICAL_MASK                                                      0x00000040L
+#define MMVM_L2_CNTL4__VMC_TAP_PTE_REQUEST_PHYSICAL_MASK                                                      0x00000080L
+#define MMVM_L2_CNTL4__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                           0x0003FF00L
+#define MMVM_L2_CNTL4__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                          0x0FFC0000L
+#define MMVM_L2_CNTL4__BPM_CGCGLS_OVERRIDE_MASK                                                               0x10000000L
+#define MMVM_L2_CNTL4__GC_CH_FGCG_OFF_MASK                                                                    0x20000000L
+#define MMVM_L2_CNTL4__VFIFO_HEAD_OF_QUEUE_MASK                                                               0x40000000L
+#define MMVM_L2_CNTL4__VFIFO_VISIBLE_BANK_SILOS_MASK                                                          0x80000000L
+//MMVM_L2_MM_GROUP_RT_CLASSES
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_0_RT_CLASS__SHIFT                                                  0x0
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_1_RT_CLASS__SHIFT                                                  0x1
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_2_RT_CLASS__SHIFT                                                  0x2
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_3_RT_CLASS__SHIFT                                                  0x3
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_4_RT_CLASS__SHIFT                                                  0x4
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_5_RT_CLASS__SHIFT                                                  0x5
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_6_RT_CLASS__SHIFT                                                  0x6
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_7_RT_CLASS__SHIFT                                                  0x7
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_8_RT_CLASS__SHIFT                                                  0x8
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_9_RT_CLASS__SHIFT                                                  0x9
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_10_RT_CLASS__SHIFT                                                 0xa
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_11_RT_CLASS__SHIFT                                                 0xb
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_12_RT_CLASS__SHIFT                                                 0xc
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_13_RT_CLASS__SHIFT                                                 0xd
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_14_RT_CLASS__SHIFT                                                 0xe
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_15_RT_CLASS__SHIFT                                                 0xf
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_16_RT_CLASS__SHIFT                                                 0x10
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_17_RT_CLASS__SHIFT                                                 0x11
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_18_RT_CLASS__SHIFT                                                 0x12
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_19_RT_CLASS__SHIFT                                                 0x13
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_20_RT_CLASS__SHIFT                                                 0x14
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_21_RT_CLASS__SHIFT                                                 0x15
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_22_RT_CLASS__SHIFT                                                 0x16
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_23_RT_CLASS__SHIFT                                                 0x17
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_24_RT_CLASS__SHIFT                                                 0x18
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_25_RT_CLASS__SHIFT                                                 0x19
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_26_RT_CLASS__SHIFT                                                 0x1a
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_27_RT_CLASS__SHIFT                                                 0x1b
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_28_RT_CLASS__SHIFT                                                 0x1c
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_29_RT_CLASS__SHIFT                                                 0x1d
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_30_RT_CLASS__SHIFT                                                 0x1e
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_31_RT_CLASS__SHIFT                                                 0x1f
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_0_RT_CLASS_MASK                                                    0x00000001L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_1_RT_CLASS_MASK                                                    0x00000002L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_2_RT_CLASS_MASK                                                    0x00000004L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_3_RT_CLASS_MASK                                                    0x00000008L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_4_RT_CLASS_MASK                                                    0x00000010L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_5_RT_CLASS_MASK                                                    0x00000020L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_6_RT_CLASS_MASK                                                    0x00000040L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_7_RT_CLASS_MASK                                                    0x00000080L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_8_RT_CLASS_MASK                                                    0x00000100L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_9_RT_CLASS_MASK                                                    0x00000200L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_10_RT_CLASS_MASK                                                   0x00000400L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_11_RT_CLASS_MASK                                                   0x00000800L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_12_RT_CLASS_MASK                                                   0x00001000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_13_RT_CLASS_MASK                                                   0x00002000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_14_RT_CLASS_MASK                                                   0x00004000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_15_RT_CLASS_MASK                                                   0x00008000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_16_RT_CLASS_MASK                                                   0x00010000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_17_RT_CLASS_MASK                                                   0x00020000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_18_RT_CLASS_MASK                                                   0x00040000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_19_RT_CLASS_MASK                                                   0x00080000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_20_RT_CLASS_MASK                                                   0x00100000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_21_RT_CLASS_MASK                                                   0x00200000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_22_RT_CLASS_MASK                                                   0x00400000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_23_RT_CLASS_MASK                                                   0x00800000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_24_RT_CLASS_MASK                                                   0x01000000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_25_RT_CLASS_MASK                                                   0x02000000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_26_RT_CLASS_MASK                                                   0x04000000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_27_RT_CLASS_MASK                                                   0x08000000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_28_RT_CLASS_MASK                                                   0x10000000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_29_RT_CLASS_MASK                                                   0x20000000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_30_RT_CLASS_MASK                                                   0x40000000L
+#define MMVM_L2_MM_GROUP_RT_CLASSES__GROUP_31_RT_CLASS_MASK                                                   0x80000000L
+//MMVM_L2_BANK_SELECT_RESERVED_CID
+#define MMVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_READ_CLIENT_ID__SHIFT                                      0x0
+#define MMVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_WRITE_CLIENT_ID__SHIFT                                     0xa
+#define MMVM_L2_BANK_SELECT_RESERVED_CID__ENABLE__SHIFT                                                       0x14
+#define MMVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_INVALIDATION_MODE__SHIFT                             0x18
+#define MMVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_PRIVATE_INVALIDATION__SHIFT                          0x19
+#define MMVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_FRAGMENT_SIZE__SHIFT                                 0x1a
+#define MMVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_READ_CLIENT_ID_MASK                                        0x000001FFL
+#define MMVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_WRITE_CLIENT_ID_MASK                                       0x0007FC00L
+#define MMVM_L2_BANK_SELECT_RESERVED_CID__ENABLE_MASK                                                         0x00100000L
+#define MMVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_INVALIDATION_MODE_MASK                               0x01000000L
+#define MMVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_PRIVATE_INVALIDATION_MASK                            0x02000000L
+#define MMVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_FRAGMENT_SIZE_MASK                                   0x7C000000L
+//MMVM_L2_BANK_SELECT_RESERVED_CID2
+#define MMVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_READ_CLIENT_ID__SHIFT                                     0x0
+#define MMVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_WRITE_CLIENT_ID__SHIFT                                    0xa
+#define MMVM_L2_BANK_SELECT_RESERVED_CID2__ENABLE__SHIFT                                                      0x14
+#define MMVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_INVALIDATION_MODE__SHIFT                            0x18
+#define MMVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_PRIVATE_INVALIDATION__SHIFT                         0x19
+#define MMVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_FRAGMENT_SIZE__SHIFT                                0x1a
+#define MMVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_READ_CLIENT_ID_MASK                                       0x000001FFL
+#define MMVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_WRITE_CLIENT_ID_MASK                                      0x0007FC00L
+#define MMVM_L2_BANK_SELECT_RESERVED_CID2__ENABLE_MASK                                                        0x00100000L
+#define MMVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_INVALIDATION_MODE_MASK                              0x01000000L
+#define MMVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_PRIVATE_INVALIDATION_MASK                           0x02000000L
+#define MMVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_FRAGMENT_SIZE_MASK                                  0x7C000000L
+//MMVM_L2_CACHE_PARITY_CNTL
+#define MMVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_4K_PTE_CACHES__SHIFT                               0x0
+#define MMVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_BIGK_PTE_CACHES__SHIFT                             0x1
+#define MMVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_PDE_CACHES__SHIFT                                  0x2
+#define MMVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_4K_PTE_CACHE__SHIFT                               0x3
+#define MMVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_BIGK_PTE_CACHE__SHIFT                             0x4
+#define MMVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_PDE_CACHE__SHIFT                                  0x5
+#define MMVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_BANK__SHIFT                                                    0x6
+#define MMVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_NUMBER__SHIFT                                                  0x9
+#define MMVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_ASSOC__SHIFT                                                   0xc
+#define MMVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_4K_PTE_CACHES_MASK                                 0x00000001L
+#define MMVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_BIGK_PTE_CACHES_MASK                               0x00000002L
+#define MMVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_PDE_CACHES_MASK                                    0x00000004L
+#define MMVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_4K_PTE_CACHE_MASK                                 0x00000008L
+#define MMVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_BIGK_PTE_CACHE_MASK                               0x00000010L
+#define MMVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_PDE_CACHE_MASK                                    0x00000020L
+#define MMVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_BANK_MASK                                                      0x000001C0L
+#define MMVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_NUMBER_MASK                                                    0x00000E00L
+#define MMVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_ASSOC_MASK                                                     0x0000F000L
+//MMVM_L2_CGTT_CLK_CTRL
+#define MMVM_L2_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                0x0
+#define MMVM_L2_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                          0x5
+#define MMVM_L2_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                                    0xd
+#define MMVM_L2_CGTT_CLK_CTRL__MIN_MGLS__SHIFT                                                                0x1a
+#define MMVM_L2_CGTT_CLK_CTRL__CGLS_DISABLE__SHIFT                                                            0x1d
+#define MMVM_L2_CGTT_CLK_CTRL__LS_DISABLE__SHIFT                                                              0x1e
+#define MMVM_L2_CGTT_CLK_CTRL__BUSY_OVERRIDE__SHIFT                                                           0x1f
+#define MMVM_L2_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                  0x0000001FL
+#define MMVM_L2_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                            0x00001FE0L
+#define MMVM_L2_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                                      0x03FFE000L
+#define MMVM_L2_CGTT_CLK_CTRL__MIN_MGLS_MASK                                                                  0x1C000000L
+#define MMVM_L2_CGTT_CLK_CTRL__CGLS_DISABLE_MASK                                                              0x20000000L
+#define MMVM_L2_CGTT_CLK_CTRL__LS_DISABLE_MASK                                                                0x40000000L
+#define MMVM_L2_CGTT_CLK_CTRL__BUSY_OVERRIDE_MASK                                                             0x80000000L
+//MMVM_L2_CNTL5
+#define MMVM_L2_CNTL5__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT                                                   0x0
+#define MMVM_L2_CNTL5__WALKER_PRIORITY_CLIENT_ID__SHIFT                                                       0x5
+#define MMVM_L2_CNTL5__WALKER_FETCH_PDE_NOALLOC_ENABLE__SHIFT                                                 0xe
+#define MMVM_L2_CNTL5__WALKER_FETCH_PDE_MTYPE_ENABLE__SHIFT                                                   0xf
+#define MMVM_L2_CNTL5__MM_CLIENT_RET_FGCG_OFF__SHIFT                                                          0x10
+#define MMVM_L2_CNTL5__UTCL2_ATC_REQ_FGCG_OFF__SHIFT                                                          0x11
+#define MMVM_L2_CNTL5__UTCL2_ATC_INVREQ_REPEATER_FGCG_OFF__SHIFT                                              0x12
+#define MMVM_L2_CNTL5__UTCL2_ONE_OUTSTANDING_ATC_INVREQ__SHIFT                                                0x13
+#define MMVM_L2_CNTL5__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                                                     0x0000001FL
+#define MMVM_L2_CNTL5__WALKER_PRIORITY_CLIENT_ID_MASK                                                         0x00003FE0L
+#define MMVM_L2_CNTL5__WALKER_FETCH_PDE_NOALLOC_ENABLE_MASK                                                   0x00004000L
+#define MMVM_L2_CNTL5__WALKER_FETCH_PDE_MTYPE_ENABLE_MASK                                                     0x00008000L
+#define MMVM_L2_CNTL5__MM_CLIENT_RET_FGCG_OFF_MASK                                                            0x00010000L
+#define MMVM_L2_CNTL5__UTCL2_ATC_REQ_FGCG_OFF_MASK                                                            0x00020000L
+#define MMVM_L2_CNTL5__UTCL2_ATC_INVREQ_REPEATER_FGCG_OFF_MASK                                                0x00040000L
+#define MMVM_L2_CNTL5__UTCL2_ONE_OUTSTANDING_ATC_INVREQ_MASK                                                  0x00080000L
+//MMVM_L2_GCR_CNTL
+#define MMVM_L2_GCR_CNTL__GCR_ENABLE__SHIFT                                                                   0x0
+#define MMVM_L2_GCR_CNTL__GCR_CLIENT_ID__SHIFT                                                                0x1
+#define MMVM_L2_GCR_CNTL__GCR_ENABLE_MASK                                                                     0x00000001L
+#define MMVM_L2_GCR_CNTL__GCR_CLIENT_ID_MASK                                                                  0x000003FEL
+//MMVM_L2_CGTT_BUSY_CTRL
+#define MMVM_L2_CGTT_BUSY_CTRL__READ_DELAY__SHIFT                                                             0x0
+#define MMVM_L2_CGTT_BUSY_CTRL__ALWAYS_BUSY__SHIFT                                                            0x5
+#define MMVM_L2_CGTT_BUSY_CTRL__READ_DELAY_MASK                                                               0x0000001FL
+#define MMVM_L2_CGTT_BUSY_CTRL__ALWAYS_BUSY_MASK                                                              0x00000020L
+//MMVM_L2_PTE_CACHE_DUMP_CNTL
+#define MMVM_L2_PTE_CACHE_DUMP_CNTL__ENABLE__SHIFT                                                            0x0
+#define MMVM_L2_PTE_CACHE_DUMP_CNTL__READY__SHIFT                                                             0x1
+#define MMVM_L2_PTE_CACHE_DUMP_CNTL__BANK__SHIFT                                                              0x4
+#define MMVM_L2_PTE_CACHE_DUMP_CNTL__CACHE__SHIFT                                                             0x8
+#define MMVM_L2_PTE_CACHE_DUMP_CNTL__ASSOC__SHIFT                                                             0xc
+#define MMVM_L2_PTE_CACHE_DUMP_CNTL__INDEX__SHIFT                                                             0x10
+#define MMVM_L2_PTE_CACHE_DUMP_CNTL__ENABLE_MASK                                                              0x00000001L
+#define MMVM_L2_PTE_CACHE_DUMP_CNTL__READY_MASK                                                               0x00000002L
+#define MMVM_L2_PTE_CACHE_DUMP_CNTL__BANK_MASK                                                                0x000000F0L
+#define MMVM_L2_PTE_CACHE_DUMP_CNTL__CACHE_MASK                                                               0x00000F00L
+#define MMVM_L2_PTE_CACHE_DUMP_CNTL__ASSOC_MASK                                                               0x0000F000L
+#define MMVM_L2_PTE_CACHE_DUMP_CNTL__INDEX_MASK                                                               0xFFFF0000L
+//MMVM_L2_PTE_CACHE_DUMP_READ
+#define MMVM_L2_PTE_CACHE_DUMP_READ__DATA__SHIFT                                                              0x0
+#define MMVM_L2_PTE_CACHE_DUMP_READ__DATA_MASK                                                                0xFFFFFFFFL
+//MMVM_L2_BANK_SELECT_MASKS
+#define MMVM_L2_BANK_SELECT_MASKS__MASK0__SHIFT                                                               0x0
+#define MMVM_L2_BANK_SELECT_MASKS__MASK1__SHIFT                                                               0x4
+#define MMVM_L2_BANK_SELECT_MASKS__MASK2__SHIFT                                                               0x8
+#define MMVM_L2_BANK_SELECT_MASKS__MASK3__SHIFT                                                               0xc
+#define MMVM_L2_BANK_SELECT_MASKS__MASK0_MASK                                                                 0x0000000FL
+#define MMVM_L2_BANK_SELECT_MASKS__MASK1_MASK                                                                 0x000000F0L
+#define MMVM_L2_BANK_SELECT_MASKS__MASK2_MASK                                                                 0x00000F00L
+#define MMVM_L2_BANK_SELECT_MASKS__MASK3_MASK                                                                 0x0000F000L
+//MMUTCL2_CREDIT_SAFETY_GROUP_RET_CDC
+#define MMUTCL2_CREDIT_SAFETY_GROUP_RET_CDC__CREDITS__SHIFT                                                   0x0
+#define MMUTCL2_CREDIT_SAFETY_GROUP_RET_CDC__UPDATE__SHIFT                                                    0xa
+#define MMUTCL2_CREDIT_SAFETY_GROUP_RET_CDC__CREDITS_MASK                                                     0x000003FFL
+#define MMUTCL2_CREDIT_SAFETY_GROUP_RET_CDC__UPDATE_MASK                                                      0x00000400L
+//MMUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC
+#define MMUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC__CREDITS__SHIFT                                        0x0
+#define MMUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC__UPDATE__SHIFT                                         0xa
+#define MMUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC__CREDITS_MASK                                          0x000003FFL
+#define MMUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC__UPDATE_MASK                                           0x00000400L
+//MMUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC
+#define MMUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC__CREDITS__SHIFT                                      0x0
+#define MMUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC__UPDATE__SHIFT                                       0xa
+#define MMUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC__CREDITS_MASK                                        0x000003FFL
+#define MMUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC__UPDATE_MASK                                         0x00000400L
+//MMVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT
+#define MMVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT__CREDITS__SHIFT                                               0x0
+#define MMVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT__UPDATE__SHIFT                                                0xa
+#define MMVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT__CREDITS_MASK                                                 0x000003FFL
+#define MMVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT__UPDATE_MASK                                                  0x00000400L
+//MMVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ
+#define MMVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ__CREDITS__SHIFT                                               0x0
+#define MMVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ__UPDATE__SHIFT                                                0xa
+#define MMVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ__CREDITS_MASK                                                 0x000003FFL
+#define MMVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ__UPDATE_MASK                                                  0x00000400L
+//MMVML2_IH_FAULT_INTERRUPT_CNTL
+#define MMVML2_IH_FAULT_INTERRUPT_CNTL__ENABLE_PF_POISON_INTERRUPT__SHIFT                                     0x0
+#define MMVML2_IH_FAULT_INTERRUPT_CNTL__ENABLE_VF_POISON_INTERRUPT__SHIFT                                     0x1
+#define MMVML2_IH_FAULT_INTERRUPT_CNTL__POISON_INTERRUPT_TO_PF__SHIFT                                         0x2
+#define MMVML2_IH_FAULT_INTERRUPT_CNTL__ENABLE_PF_POISON_INTERRUPT_MASK                                       0x00000001L
+#define MMVML2_IH_FAULT_INTERRUPT_CNTL__ENABLE_VF_POISON_INTERRUPT_MASK                                       0x00000002L
+#define MMVML2_IH_FAULT_INTERRUPT_CNTL__POISON_INTERRUPT_TO_PF_MASK                                           0x00000004L
+
+
+// addressBlock: mmhub_mmutcl2_mmvml2prdec
+//MMMC_VM_L2_PERFCOUNTER_LO
+#define MMMC_VM_L2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                          0x0
+#define MMMC_VM_L2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                            0xFFFFFFFFL
+//MMMC_VM_L2_PERFCOUNTER_HI
+#define MMMC_VM_L2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                          0x0
+#define MMMC_VM_L2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                       0x10
+#define MMMC_VM_L2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                            0x0000FFFFL
+#define MMMC_VM_L2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                         0xFFFF0000L
+//MMUTCL2_PERFCOUNTER_LO
+#define MMUTCL2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                             0x0
+#define MMUTCL2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                               0xFFFFFFFFL
+//MMUTCL2_PERFCOUNTER_HI
+#define MMUTCL2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                             0x0
+#define MMUTCL2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                          0x10
+#define MMUTCL2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                               0x0000FFFFL
+#define MMUTCL2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                            0xFFFF0000L
+
+
+// addressBlock: mmhub_mmutcl2_mmatcl2prdec
+//MM_ATC_L2_PERFCOUNTER_LO
+#define MM_ATC_L2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                           0x0
+#define MM_ATC_L2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                             0xFFFFFFFFL
+//MM_ATC_L2_PERFCOUNTER_HI
+#define MM_ATC_L2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                           0x0
+#define MM_ATC_L2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                        0x10
+#define MM_ATC_L2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                             0x0000FFFFL
+#define MM_ATC_L2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                          0xFFFF0000L
+
+
+// addressBlock: mmhub_mmutcl2_mmvml2pldec
+//MMMC_VM_L2_PERFCOUNTER0_CFG
+#define MMMC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                          0x0
+#define MMMC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define MMMC_VM_L2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                         0x18
+#define MMMC_VM_L2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                            0x1c
+#define MMMC_VM_L2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                             0x1d
+#define MMMC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define MMMC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define MMMC_VM_L2_PERFCOUNTER0_CFG__ENABLE_MASK                                                              0x10000000L
+#define MMMC_VM_L2_PERFCOUNTER0_CFG__CLEAR_MASK                                                               0x20000000L
+//MMMC_VM_L2_PERFCOUNTER1_CFG
+#define MMMC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                          0x0
+#define MMMC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define MMMC_VM_L2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                         0x18
+#define MMMC_VM_L2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                            0x1c
+#define MMMC_VM_L2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                             0x1d
+#define MMMC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define MMMC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define MMMC_VM_L2_PERFCOUNTER1_CFG__ENABLE_MASK                                                              0x10000000L
+#define MMMC_VM_L2_PERFCOUNTER1_CFG__CLEAR_MASK                                                               0x20000000L
+//MMMC_VM_L2_PERFCOUNTER2_CFG
+#define MMMC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                          0x0
+#define MMMC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define MMMC_VM_L2_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                         0x18
+#define MMMC_VM_L2_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                            0x1c
+#define MMMC_VM_L2_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                             0x1d
+#define MMMC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define MMMC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define MMMC_VM_L2_PERFCOUNTER2_CFG__ENABLE_MASK                                                              0x10000000L
+#define MMMC_VM_L2_PERFCOUNTER2_CFG__CLEAR_MASK                                                               0x20000000L
+//MMMC_VM_L2_PERFCOUNTER3_CFG
+#define MMMC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                          0x0
+#define MMMC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define MMMC_VM_L2_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                         0x18
+#define MMMC_VM_L2_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                            0x1c
+#define MMMC_VM_L2_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                             0x1d
+#define MMMC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define MMMC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define MMMC_VM_L2_PERFCOUNTER3_CFG__ENABLE_MASK                                                              0x10000000L
+#define MMMC_VM_L2_PERFCOUNTER3_CFG__CLEAR_MASK                                                               0x20000000L
+//MMMC_VM_L2_PERFCOUNTER4_CFG
+#define MMMC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL__SHIFT                                                          0x0
+#define MMMC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define MMMC_VM_L2_PERFCOUNTER4_CFG__PERF_MODE__SHIFT                                                         0x18
+#define MMMC_VM_L2_PERFCOUNTER4_CFG__ENABLE__SHIFT                                                            0x1c
+#define MMMC_VM_L2_PERFCOUNTER4_CFG__CLEAR__SHIFT                                                             0x1d
+#define MMMC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define MMMC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER4_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define MMMC_VM_L2_PERFCOUNTER4_CFG__ENABLE_MASK                                                              0x10000000L
+#define MMMC_VM_L2_PERFCOUNTER4_CFG__CLEAR_MASK                                                               0x20000000L
+//MMMC_VM_L2_PERFCOUNTER5_CFG
+#define MMMC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL__SHIFT                                                          0x0
+#define MMMC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define MMMC_VM_L2_PERFCOUNTER5_CFG__PERF_MODE__SHIFT                                                         0x18
+#define MMMC_VM_L2_PERFCOUNTER5_CFG__ENABLE__SHIFT                                                            0x1c
+#define MMMC_VM_L2_PERFCOUNTER5_CFG__CLEAR__SHIFT                                                             0x1d
+#define MMMC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define MMMC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER5_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define MMMC_VM_L2_PERFCOUNTER5_CFG__ENABLE_MASK                                                              0x10000000L
+#define MMMC_VM_L2_PERFCOUNTER5_CFG__CLEAR_MASK                                                               0x20000000L
+//MMMC_VM_L2_PERFCOUNTER6_CFG
+#define MMMC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL__SHIFT                                                          0x0
+#define MMMC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define MMMC_VM_L2_PERFCOUNTER6_CFG__PERF_MODE__SHIFT                                                         0x18
+#define MMMC_VM_L2_PERFCOUNTER6_CFG__ENABLE__SHIFT                                                            0x1c
+#define MMMC_VM_L2_PERFCOUNTER6_CFG__CLEAR__SHIFT                                                             0x1d
+#define MMMC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define MMMC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER6_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define MMMC_VM_L2_PERFCOUNTER6_CFG__ENABLE_MASK                                                              0x10000000L
+#define MMMC_VM_L2_PERFCOUNTER6_CFG__CLEAR_MASK                                                               0x20000000L
+//MMMC_VM_L2_PERFCOUNTER7_CFG
+#define MMMC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL__SHIFT                                                          0x0
+#define MMMC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define MMMC_VM_L2_PERFCOUNTER7_CFG__PERF_MODE__SHIFT                                                         0x18
+#define MMMC_VM_L2_PERFCOUNTER7_CFG__ENABLE__SHIFT                                                            0x1c
+#define MMMC_VM_L2_PERFCOUNTER7_CFG__CLEAR__SHIFT                                                             0x1d
+#define MMMC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define MMMC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER7_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define MMMC_VM_L2_PERFCOUNTER7_CFG__ENABLE_MASK                                                              0x10000000L
+#define MMMC_VM_L2_PERFCOUNTER7_CFG__CLEAR_MASK                                                               0x20000000L
+//MMMC_VM_L2_PERFCOUNTER8_CFG
+#define MMMC_VM_L2_PERFCOUNTER8_CFG__PERF_SEL__SHIFT                                                          0x0
+#define MMMC_VM_L2_PERFCOUNTER8_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define MMMC_VM_L2_PERFCOUNTER8_CFG__PERF_MODE__SHIFT                                                         0x18
+#define MMMC_VM_L2_PERFCOUNTER8_CFG__ENABLE__SHIFT                                                            0x1c
+#define MMMC_VM_L2_PERFCOUNTER8_CFG__CLEAR__SHIFT                                                             0x1d
+#define MMMC_VM_L2_PERFCOUNTER8_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define MMMC_VM_L2_PERFCOUNTER8_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER8_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define MMMC_VM_L2_PERFCOUNTER8_CFG__ENABLE_MASK                                                              0x10000000L
+#define MMMC_VM_L2_PERFCOUNTER8_CFG__CLEAR_MASK                                                               0x20000000L
+//MMMC_VM_L2_PERFCOUNTER9_CFG
+#define MMMC_VM_L2_PERFCOUNTER9_CFG__PERF_SEL__SHIFT                                                          0x0
+#define MMMC_VM_L2_PERFCOUNTER9_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define MMMC_VM_L2_PERFCOUNTER9_CFG__PERF_MODE__SHIFT                                                         0x18
+#define MMMC_VM_L2_PERFCOUNTER9_CFG__ENABLE__SHIFT                                                            0x1c
+#define MMMC_VM_L2_PERFCOUNTER9_CFG__CLEAR__SHIFT                                                             0x1d
+#define MMMC_VM_L2_PERFCOUNTER9_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define MMMC_VM_L2_PERFCOUNTER9_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER9_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define MMMC_VM_L2_PERFCOUNTER9_CFG__ENABLE_MASK                                                              0x10000000L
+#define MMMC_VM_L2_PERFCOUNTER9_CFG__CLEAR_MASK                                                               0x20000000L
+//MMMC_VM_L2_PERFCOUNTER10_CFG
+#define MMMC_VM_L2_PERFCOUNTER10_CFG__PERF_SEL__SHIFT                                                         0x0
+#define MMMC_VM_L2_PERFCOUNTER10_CFG__PERF_SEL_END__SHIFT                                                     0x8
+#define MMMC_VM_L2_PERFCOUNTER10_CFG__PERF_MODE__SHIFT                                                        0x18
+#define MMMC_VM_L2_PERFCOUNTER10_CFG__ENABLE__SHIFT                                                           0x1c
+#define MMMC_VM_L2_PERFCOUNTER10_CFG__CLEAR__SHIFT                                                            0x1d
+#define MMMC_VM_L2_PERFCOUNTER10_CFG__PERF_SEL_MASK                                                           0x000000FFL
+#define MMMC_VM_L2_PERFCOUNTER10_CFG__PERF_SEL_END_MASK                                                       0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER10_CFG__PERF_MODE_MASK                                                          0x0F000000L
+#define MMMC_VM_L2_PERFCOUNTER10_CFG__ENABLE_MASK                                                             0x10000000L
+#define MMMC_VM_L2_PERFCOUNTER10_CFG__CLEAR_MASK                                                              0x20000000L
+//MMMC_VM_L2_PERFCOUNTER11_CFG
+#define MMMC_VM_L2_PERFCOUNTER11_CFG__PERF_SEL__SHIFT                                                         0x0
+#define MMMC_VM_L2_PERFCOUNTER11_CFG__PERF_SEL_END__SHIFT                                                     0x8
+#define MMMC_VM_L2_PERFCOUNTER11_CFG__PERF_MODE__SHIFT                                                        0x18
+#define MMMC_VM_L2_PERFCOUNTER11_CFG__ENABLE__SHIFT                                                           0x1c
+#define MMMC_VM_L2_PERFCOUNTER11_CFG__CLEAR__SHIFT                                                            0x1d
+#define MMMC_VM_L2_PERFCOUNTER11_CFG__PERF_SEL_MASK                                                           0x000000FFL
+#define MMMC_VM_L2_PERFCOUNTER11_CFG__PERF_SEL_END_MASK                                                       0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER11_CFG__PERF_MODE_MASK                                                          0x0F000000L
+#define MMMC_VM_L2_PERFCOUNTER11_CFG__ENABLE_MASK                                                             0x10000000L
+#define MMMC_VM_L2_PERFCOUNTER11_CFG__CLEAR_MASK                                                              0x20000000L
+//MMMC_VM_L2_PERFCOUNTER12_CFG
+#define MMMC_VM_L2_PERFCOUNTER12_CFG__PERF_SEL__SHIFT                                                         0x0
+#define MMMC_VM_L2_PERFCOUNTER12_CFG__PERF_SEL_END__SHIFT                                                     0x8
+#define MMMC_VM_L2_PERFCOUNTER12_CFG__PERF_MODE__SHIFT                                                        0x18
+#define MMMC_VM_L2_PERFCOUNTER12_CFG__ENABLE__SHIFT                                                           0x1c
+#define MMMC_VM_L2_PERFCOUNTER12_CFG__CLEAR__SHIFT                                                            0x1d
+#define MMMC_VM_L2_PERFCOUNTER12_CFG__PERF_SEL_MASK                                                           0x000000FFL
+#define MMMC_VM_L2_PERFCOUNTER12_CFG__PERF_SEL_END_MASK                                                       0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER12_CFG__PERF_MODE_MASK                                                          0x0F000000L
+#define MMMC_VM_L2_PERFCOUNTER12_CFG__ENABLE_MASK                                                             0x10000000L
+#define MMMC_VM_L2_PERFCOUNTER12_CFG__CLEAR_MASK                                                              0x20000000L
+//MMMC_VM_L2_PERFCOUNTER13_CFG
+#define MMMC_VM_L2_PERFCOUNTER13_CFG__PERF_SEL__SHIFT                                                         0x0
+#define MMMC_VM_L2_PERFCOUNTER13_CFG__PERF_SEL_END__SHIFT                                                     0x8
+#define MMMC_VM_L2_PERFCOUNTER13_CFG__PERF_MODE__SHIFT                                                        0x18
+#define MMMC_VM_L2_PERFCOUNTER13_CFG__ENABLE__SHIFT                                                           0x1c
+#define MMMC_VM_L2_PERFCOUNTER13_CFG__CLEAR__SHIFT                                                            0x1d
+#define MMMC_VM_L2_PERFCOUNTER13_CFG__PERF_SEL_MASK                                                           0x000000FFL
+#define MMMC_VM_L2_PERFCOUNTER13_CFG__PERF_SEL_END_MASK                                                       0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER13_CFG__PERF_MODE_MASK                                                          0x0F000000L
+#define MMMC_VM_L2_PERFCOUNTER13_CFG__ENABLE_MASK                                                             0x10000000L
+#define MMMC_VM_L2_PERFCOUNTER13_CFG__CLEAR_MASK                                                              0x20000000L
+//MMMC_VM_L2_PERFCOUNTER14_CFG
+#define MMMC_VM_L2_PERFCOUNTER14_CFG__PERF_SEL__SHIFT                                                         0x0
+#define MMMC_VM_L2_PERFCOUNTER14_CFG__PERF_SEL_END__SHIFT                                                     0x8
+#define MMMC_VM_L2_PERFCOUNTER14_CFG__PERF_MODE__SHIFT                                                        0x18
+#define MMMC_VM_L2_PERFCOUNTER14_CFG__ENABLE__SHIFT                                                           0x1c
+#define MMMC_VM_L2_PERFCOUNTER14_CFG__CLEAR__SHIFT                                                            0x1d
+#define MMMC_VM_L2_PERFCOUNTER14_CFG__PERF_SEL_MASK                                                           0x000000FFL
+#define MMMC_VM_L2_PERFCOUNTER14_CFG__PERF_SEL_END_MASK                                                       0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER14_CFG__PERF_MODE_MASK                                                          0x0F000000L
+#define MMMC_VM_L2_PERFCOUNTER14_CFG__ENABLE_MASK                                                             0x10000000L
+#define MMMC_VM_L2_PERFCOUNTER14_CFG__CLEAR_MASK                                                              0x20000000L
+//MMMC_VM_L2_PERFCOUNTER15_CFG
+#define MMMC_VM_L2_PERFCOUNTER15_CFG__PERF_SEL__SHIFT                                                         0x0
+#define MMMC_VM_L2_PERFCOUNTER15_CFG__PERF_SEL_END__SHIFT                                                     0x8
+#define MMMC_VM_L2_PERFCOUNTER15_CFG__PERF_MODE__SHIFT                                                        0x18
+#define MMMC_VM_L2_PERFCOUNTER15_CFG__ENABLE__SHIFT                                                           0x1c
+#define MMMC_VM_L2_PERFCOUNTER15_CFG__CLEAR__SHIFT                                                            0x1d
+#define MMMC_VM_L2_PERFCOUNTER15_CFG__PERF_SEL_MASK                                                           0x000000FFL
+#define MMMC_VM_L2_PERFCOUNTER15_CFG__PERF_SEL_END_MASK                                                       0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER15_CFG__PERF_MODE_MASK                                                          0x0F000000L
+#define MMMC_VM_L2_PERFCOUNTER15_CFG__ENABLE_MASK                                                             0x10000000L
+#define MMMC_VM_L2_PERFCOUNTER15_CFG__CLEAR_MASK                                                              0x20000000L
+//MMMC_VM_L2_PERFCOUNTER_RSLT_CNTL
+#define MMMC_VM_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                          0x0
+#define MMMC_VM_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                0x8
+#define MMMC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                 0x10
+#define MMMC_VM_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                   0x18
+#define MMMC_VM_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                    0x19
+#define MMMC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                         0x1a
+#define MMMC_VM_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                            0x0000000FL
+#define MMMC_VM_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                  0x0000FF00L
+#define MMMC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                   0x00FF0000L
+#define MMMC_VM_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                     0x01000000L
+#define MMMC_VM_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                      0x02000000L
+#define MMMC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                           0x04000000L
+//MMUTCL2_PERFCOUNTER0_CFG
+#define MMUTCL2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                             0x0
+#define MMUTCL2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                         0x8
+#define MMUTCL2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                            0x18
+#define MMUTCL2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                               0x1c
+#define MMUTCL2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                0x1d
+#define MMUTCL2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                               0x000000FFL
+#define MMUTCL2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                           0x0000FF00L
+#define MMUTCL2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                              0x0F000000L
+#define MMUTCL2_PERFCOUNTER0_CFG__ENABLE_MASK                                                                 0x10000000L
+#define MMUTCL2_PERFCOUNTER0_CFG__CLEAR_MASK                                                                  0x20000000L
+//MMUTCL2_PERFCOUNTER1_CFG
+#define MMUTCL2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                             0x0
+#define MMUTCL2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                         0x8
+#define MMUTCL2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                            0x18
+#define MMUTCL2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                               0x1c
+#define MMUTCL2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                0x1d
+#define MMUTCL2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                               0x000000FFL
+#define MMUTCL2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                           0x0000FF00L
+#define MMUTCL2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                              0x0F000000L
+#define MMUTCL2_PERFCOUNTER1_CFG__ENABLE_MASK                                                                 0x10000000L
+#define MMUTCL2_PERFCOUNTER1_CFG__CLEAR_MASK                                                                  0x20000000L
+//MMUTCL2_PERFCOUNTER2_CFG
+#define MMUTCL2_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                             0x0
+#define MMUTCL2_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                         0x8
+#define MMUTCL2_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                            0x18
+#define MMUTCL2_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                               0x1c
+#define MMUTCL2_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                                0x1d
+#define MMUTCL2_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                               0x000000FFL
+#define MMUTCL2_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                           0x0000FF00L
+#define MMUTCL2_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                              0x0F000000L
+#define MMUTCL2_PERFCOUNTER2_CFG__ENABLE_MASK                                                                 0x10000000L
+#define MMUTCL2_PERFCOUNTER2_CFG__CLEAR_MASK                                                                  0x20000000L
+//MMUTCL2_PERFCOUNTER3_CFG
+#define MMUTCL2_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                             0x0
+#define MMUTCL2_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                         0x8
+#define MMUTCL2_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                            0x18
+#define MMUTCL2_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                               0x1c
+#define MMUTCL2_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                                0x1d
+#define MMUTCL2_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                               0x000000FFL
+#define MMUTCL2_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                           0x0000FF00L
+#define MMUTCL2_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                              0x0F000000L
+#define MMUTCL2_PERFCOUNTER3_CFG__ENABLE_MASK                                                                 0x10000000L
+#define MMUTCL2_PERFCOUNTER3_CFG__CLEAR_MASK                                                                  0x20000000L
+//MMUTCL2_PERFCOUNTER_RSLT_CNTL
+#define MMUTCL2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                             0x0
+#define MMUTCL2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                   0x8
+#define MMUTCL2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                    0x10
+#define MMUTCL2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                      0x18
+#define MMUTCL2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                       0x19
+#define MMUTCL2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                            0x1a
+#define MMUTCL2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                               0x0000000FL
+#define MMUTCL2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                     0x0000FF00L
+#define MMUTCL2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                      0x00FF0000L
+#define MMUTCL2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                        0x01000000L
+#define MMUTCL2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                         0x02000000L
+#define MMUTCL2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                              0x04000000L
+
+
+// addressBlock: mmhub_mmutcl2_mmatcl2pldec
+//MM_ATC_L2_PERFCOUNTER0_CFG
+#define MM_ATC_L2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                           0x0
+#define MM_ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                       0x8
+#define MM_ATC_L2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                          0x18
+#define MM_ATC_L2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                             0x1c
+#define MM_ATC_L2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                              0x1d
+#define MM_ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                             0x000000FFL
+#define MM_ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                         0x0000FF00L
+#define MM_ATC_L2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                            0x0F000000L
+#define MM_ATC_L2_PERFCOUNTER0_CFG__ENABLE_MASK                                                               0x10000000L
+#define MM_ATC_L2_PERFCOUNTER0_CFG__CLEAR_MASK                                                                0x20000000L
+//MM_ATC_L2_PERFCOUNTER1_CFG
+#define MM_ATC_L2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                           0x0
+#define MM_ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                       0x8
+#define MM_ATC_L2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                          0x18
+#define MM_ATC_L2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                             0x1c
+#define MM_ATC_L2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                              0x1d
+#define MM_ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                             0x000000FFL
+#define MM_ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                         0x0000FF00L
+#define MM_ATC_L2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                            0x0F000000L
+#define MM_ATC_L2_PERFCOUNTER1_CFG__ENABLE_MASK                                                               0x10000000L
+#define MM_ATC_L2_PERFCOUNTER1_CFG__CLEAR_MASK                                                                0x20000000L
+//MM_ATC_L2_PERFCOUNTER_RSLT_CNTL
+#define MM_ATC_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                           0x0
+#define MM_ATC_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                 0x8
+#define MM_ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                  0x10
+#define MM_ATC_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                    0x18
+#define MM_ATC_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                     0x19
+#define MM_ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                          0x1a
+#define MM_ATC_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                             0x0000000FL
+#define MM_ATC_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                   0x0000FF00L
+#define MM_ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                    0x00FF0000L
+#define MM_ATC_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                      0x01000000L
+#define MM_ATC_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                       0x02000000L
+#define MM_ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                            0x04000000L
+
+
+// addressBlock: mmhub_mmutcl2_mmvmsharedvcdec
+//MMMC_VM_FB_LOCATION_BASE_LO32
+#define MMMC_VM_FB_LOCATION_BASE_LO32__FB_BASE_LO32__SHIFT                                                    0x0
+#define MMMC_VM_FB_LOCATION_BASE_LO32__FB_BASE_LO32_MASK                                                      0xFFFFFFFFL
+//MMMC_VM_FB_LOCATION_BASE_HI32
+#define MMMC_VM_FB_LOCATION_BASE_HI32__FB_BASE_HI1__SHIFT                                                     0x0
+#define MMMC_VM_FB_LOCATION_BASE_HI32__FB_BASE_HI1_MASK                                                       0x00000001L
+//MMMC_VM_FB_LOCATION_TOP_LO32
+#define MMMC_VM_FB_LOCATION_TOP_LO32__FB_TOP_LO32__SHIFT                                                      0x0
+#define MMMC_VM_FB_LOCATION_TOP_LO32__FB_TOP_LO32_MASK                                                        0xFFFFFFFFL
+//MMMC_VM_FB_LOCATION_TOP_HI32
+#define MMMC_VM_FB_LOCATION_TOP_HI32__FB_TOP_HI1__SHIFT                                                       0x0
+#define MMMC_VM_FB_LOCATION_TOP_HI32__FB_TOP_HI1_MASK                                                         0x00000001L
+//MMMC_VM_AGP_TOP_LO32
+#define MMMC_VM_AGP_TOP_LO32__AGP_TOP_LO32__SHIFT                                                             0x0
+#define MMMC_VM_AGP_TOP_LO32__AGP_TOP_LO32_MASK                                                               0xFFFFFFFFL
+//MMMC_VM_AGP_TOP_HI32
+#define MMMC_VM_AGP_TOP_HI32__AGP_TOP_HI1__SHIFT                                                              0x0
+#define MMMC_VM_AGP_TOP_HI32__AGP_TOP_HI1_MASK                                                                0x00000001L
+//MMMC_VM_AGP_BOT_LO32
+#define MMMC_VM_AGP_BOT_LO32__AGP_BOT_LO32__SHIFT                                                             0x0
+#define MMMC_VM_AGP_BOT_LO32__AGP_BOT_LO32_MASK                                                               0xFFFFFFFFL
+//MMMC_VM_AGP_BOT_HI32
+#define MMMC_VM_AGP_BOT_HI32__AGP_BOT_HI1__SHIFT                                                              0x0
+#define MMMC_VM_AGP_BOT_HI32__AGP_BOT_HI1_MASK                                                                0x00000001L
+//MMMC_VM_AGP_BASE_LO32
+#define MMMC_VM_AGP_BASE_LO32__AGP_BASE_LO32__SHIFT                                                           0x0
+#define MMMC_VM_AGP_BASE_LO32__AGP_BASE_LO32_MASK                                                             0xFFFFFFFFL
+//MMMC_VM_AGP_BASE_HI32
+#define MMMC_VM_AGP_BASE_HI32__AGP_BASE_HI1__SHIFT                                                            0x0
+#define MMMC_VM_AGP_BASE_HI32__AGP_BASE_HI1_MASK                                                              0x00000001L
+//MMMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32
+#define MMMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32__LOGICAL_ADDR_LO32__SHIFT                                       0x0
+#define MMMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32__LOGICAL_ADDR_LO32_MASK                                         0xFFFFFFFFL
+//MMMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32
+#define MMMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32__LOGICAL_ADDR_HI7__SHIFT                                        0x0
+#define MMMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32__LOGICAL_ADDR_HI7_MASK                                          0x0000007FL
+//MMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32
+#define MMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32__LOGICAL_ADDR_LO32__SHIFT                                      0x0
+#define MMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32__LOGICAL_ADDR_LO32_MASK                                        0xFFFFFFFFL
+//MMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32
+#define MMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32__LOGICAL_ADDR_HI7__SHIFT                                       0x0
+#define MMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32__LOGICAL_ADDR_HI7_MASK                                         0x0000007FL
+//MMMC_VM_MX_L1_TLB_CNTL
+#define MMMC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB__SHIFT                                                          0x0
+#define MMMC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE__SHIFT                                                     0x3
+#define MMMC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT                                        0x5
+#define MMMC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL__SHIFT                                           0x6
+#define MMMC_VM_MX_L1_TLB_CNTL__ECO_BITS__SHIFT                                                               0x7
+#define MMMC_VM_MX_L1_TLB_CNTL__MTYPE__SHIFT                                                                  0xb
+#define MMMC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB_MASK                                                            0x00000001L
+#define MMMC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK                                                       0x00000018L
+#define MMMC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS_MASK                                          0x00000020L
+#define MMMC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL_MASK                                             0x00000040L
+#define MMMC_VM_MX_L1_TLB_CNTL__ECO_BITS_MASK                                                                 0x00000780L
+#define MMMC_VM_MX_L1_TLB_CNTL__MTYPE_MASK                                                                    0x00001800L
+
+
+// addressBlock: mmhub_mmutcl2_mmvml2vcdec
+//MMVM_CONTEXT0_CNTL
+#define MMVM_CONTEXT0_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define MMVM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define MMVM_CONTEXT0_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define MMVM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define MMVM_CONTEXT0_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define MMVM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define MMVM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define MMVM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define MMVM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define MMVM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define MMVM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define MMVM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define MMVM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define MMVM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define MMVM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define MMVM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define MMVM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define MMVM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define MMVM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define MMVM_CONTEXT0_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x18
+#define MMVM_CONTEXT0_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x19
+#define MMVM_CONTEXT0_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define MMVM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define MMVM_CONTEXT0_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define MMVM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define MMVM_CONTEXT0_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define MMVM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define MMVM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define MMVM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define MMVM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define MMVM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define MMVM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define MMVM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define MMVM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define MMVM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define MMVM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define MMVM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define MMVM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define MMVM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define MMVM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+#define MMVM_CONTEXT0_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x01000000L
+#define MMVM_CONTEXT0_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x02000000L
+//MMVM_CONTEXT1_CNTL
+#define MMVM_CONTEXT1_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define MMVM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define MMVM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define MMVM_CONTEXT1_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define MMVM_CONTEXT1_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define MMVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define MMVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define MMVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define MMVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define MMVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define MMVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define MMVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define MMVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define MMVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define MMVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define MMVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define MMVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define MMVM_CONTEXT1_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x18
+#define MMVM_CONTEXT1_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x19
+#define MMVM_CONTEXT1_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define MMVM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define MMVM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define MMVM_CONTEXT1_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define MMVM_CONTEXT1_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define MMVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define MMVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define MMVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define MMVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define MMVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define MMVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define MMVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define MMVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define MMVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define MMVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define MMVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define MMVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+#define MMVM_CONTEXT1_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x01000000L
+#define MMVM_CONTEXT1_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x02000000L
+//MMVM_CONTEXT2_CNTL
+#define MMVM_CONTEXT2_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define MMVM_CONTEXT2_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define MMVM_CONTEXT2_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define MMVM_CONTEXT2_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define MMVM_CONTEXT2_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define MMVM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define MMVM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define MMVM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define MMVM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define MMVM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define MMVM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define MMVM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define MMVM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define MMVM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define MMVM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define MMVM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define MMVM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define MMVM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define MMVM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define MMVM_CONTEXT2_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x18
+#define MMVM_CONTEXT2_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x19
+#define MMVM_CONTEXT2_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define MMVM_CONTEXT2_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define MMVM_CONTEXT2_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define MMVM_CONTEXT2_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define MMVM_CONTEXT2_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define MMVM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define MMVM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define MMVM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define MMVM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define MMVM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define MMVM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define MMVM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define MMVM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define MMVM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define MMVM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define MMVM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define MMVM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define MMVM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define MMVM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+#define MMVM_CONTEXT2_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x01000000L
+#define MMVM_CONTEXT2_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x02000000L
+//MMVM_CONTEXT3_CNTL
+#define MMVM_CONTEXT3_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define MMVM_CONTEXT3_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define MMVM_CONTEXT3_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define MMVM_CONTEXT3_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define MMVM_CONTEXT3_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define MMVM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define MMVM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define MMVM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define MMVM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define MMVM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define MMVM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define MMVM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define MMVM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define MMVM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define MMVM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define MMVM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define MMVM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define MMVM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define MMVM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define MMVM_CONTEXT3_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x18
+#define MMVM_CONTEXT3_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x19
+#define MMVM_CONTEXT3_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define MMVM_CONTEXT3_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define MMVM_CONTEXT3_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define MMVM_CONTEXT3_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define MMVM_CONTEXT3_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define MMVM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define MMVM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define MMVM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define MMVM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define MMVM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define MMVM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define MMVM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define MMVM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define MMVM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define MMVM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define MMVM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define MMVM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define MMVM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define MMVM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+#define MMVM_CONTEXT3_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x01000000L
+#define MMVM_CONTEXT3_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x02000000L
+//MMVM_CONTEXT4_CNTL
+#define MMVM_CONTEXT4_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define MMVM_CONTEXT4_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define MMVM_CONTEXT4_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define MMVM_CONTEXT4_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define MMVM_CONTEXT4_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define MMVM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define MMVM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define MMVM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define MMVM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define MMVM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define MMVM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define MMVM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define MMVM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define MMVM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define MMVM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define MMVM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define MMVM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define MMVM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define MMVM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define MMVM_CONTEXT4_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x18
+#define MMVM_CONTEXT4_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x19
+#define MMVM_CONTEXT4_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define MMVM_CONTEXT4_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define MMVM_CONTEXT4_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define MMVM_CONTEXT4_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define MMVM_CONTEXT4_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define MMVM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define MMVM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define MMVM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define MMVM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define MMVM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define MMVM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define MMVM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define MMVM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define MMVM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define MMVM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define MMVM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define MMVM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define MMVM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define MMVM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+#define MMVM_CONTEXT4_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x01000000L
+#define MMVM_CONTEXT4_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x02000000L
+//MMVM_CONTEXT5_CNTL
+#define MMVM_CONTEXT5_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define MMVM_CONTEXT5_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define MMVM_CONTEXT5_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define MMVM_CONTEXT5_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define MMVM_CONTEXT5_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define MMVM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define MMVM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define MMVM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define MMVM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define MMVM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define MMVM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define MMVM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define MMVM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define MMVM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define MMVM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define MMVM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define MMVM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define MMVM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define MMVM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define MMVM_CONTEXT5_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x18
+#define MMVM_CONTEXT5_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x19
+#define MMVM_CONTEXT5_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define MMVM_CONTEXT5_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define MMVM_CONTEXT5_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define MMVM_CONTEXT5_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define MMVM_CONTEXT5_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define MMVM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define MMVM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define MMVM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define MMVM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define MMVM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define MMVM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define MMVM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define MMVM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define MMVM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define MMVM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define MMVM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define MMVM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define MMVM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define MMVM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+#define MMVM_CONTEXT5_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x01000000L
+#define MMVM_CONTEXT5_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x02000000L
+//MMVM_CONTEXT6_CNTL
+#define MMVM_CONTEXT6_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define MMVM_CONTEXT6_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define MMVM_CONTEXT6_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define MMVM_CONTEXT6_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define MMVM_CONTEXT6_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define MMVM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define MMVM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define MMVM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define MMVM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define MMVM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define MMVM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define MMVM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define MMVM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define MMVM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define MMVM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define MMVM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define MMVM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define MMVM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define MMVM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define MMVM_CONTEXT6_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x18
+#define MMVM_CONTEXT6_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x19
+#define MMVM_CONTEXT6_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define MMVM_CONTEXT6_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define MMVM_CONTEXT6_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define MMVM_CONTEXT6_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define MMVM_CONTEXT6_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define MMVM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define MMVM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define MMVM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define MMVM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define MMVM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define MMVM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define MMVM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define MMVM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define MMVM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define MMVM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define MMVM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define MMVM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define MMVM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define MMVM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+#define MMVM_CONTEXT6_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x01000000L
+#define MMVM_CONTEXT6_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x02000000L
+//MMVM_CONTEXT7_CNTL
+#define MMVM_CONTEXT7_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define MMVM_CONTEXT7_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define MMVM_CONTEXT7_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define MMVM_CONTEXT7_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define MMVM_CONTEXT7_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define MMVM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define MMVM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define MMVM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define MMVM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define MMVM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define MMVM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define MMVM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define MMVM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define MMVM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define MMVM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define MMVM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define MMVM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define MMVM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define MMVM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define MMVM_CONTEXT7_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x18
+#define MMVM_CONTEXT7_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x19
+#define MMVM_CONTEXT7_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define MMVM_CONTEXT7_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define MMVM_CONTEXT7_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define MMVM_CONTEXT7_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define MMVM_CONTEXT7_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define MMVM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define MMVM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define MMVM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define MMVM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define MMVM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define MMVM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define MMVM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define MMVM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define MMVM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define MMVM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define MMVM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define MMVM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define MMVM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define MMVM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+#define MMVM_CONTEXT7_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x01000000L
+#define MMVM_CONTEXT7_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x02000000L
+//MMVM_CONTEXT8_CNTL
+#define MMVM_CONTEXT8_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define MMVM_CONTEXT8_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define MMVM_CONTEXT8_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define MMVM_CONTEXT8_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define MMVM_CONTEXT8_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define MMVM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define MMVM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define MMVM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define MMVM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define MMVM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define MMVM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define MMVM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define MMVM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define MMVM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define MMVM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define MMVM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define MMVM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define MMVM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define MMVM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define MMVM_CONTEXT8_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x18
+#define MMVM_CONTEXT8_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x19
+#define MMVM_CONTEXT8_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define MMVM_CONTEXT8_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define MMVM_CONTEXT8_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define MMVM_CONTEXT8_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define MMVM_CONTEXT8_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define MMVM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define MMVM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define MMVM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define MMVM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define MMVM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define MMVM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define MMVM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define MMVM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define MMVM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define MMVM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define MMVM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define MMVM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define MMVM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define MMVM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+#define MMVM_CONTEXT8_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x01000000L
+#define MMVM_CONTEXT8_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x02000000L
+//MMVM_CONTEXT9_CNTL
+#define MMVM_CONTEXT9_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define MMVM_CONTEXT9_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define MMVM_CONTEXT9_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x4
+#define MMVM_CONTEXT9_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x8
+#define MMVM_CONTEXT9_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x9
+#define MMVM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xa
+#define MMVM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xb
+#define MMVM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xc
+#define MMVM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xd
+#define MMVM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xe
+#define MMVM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xf
+#define MMVM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x10
+#define MMVM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x11
+#define MMVM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x12
+#define MMVM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x13
+#define MMVM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x14
+#define MMVM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x15
+#define MMVM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x16
+#define MMVM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x17
+#define MMVM_CONTEXT9_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x18
+#define MMVM_CONTEXT9_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x19
+#define MMVM_CONTEXT9_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define MMVM_CONTEXT9_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x0000000EL
+#define MMVM_CONTEXT9_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x000000F0L
+#define MMVM_CONTEXT9_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000100L
+#define MMVM_CONTEXT9_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000200L
+#define MMVM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000400L
+#define MMVM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000800L
+#define MMVM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00001000L
+#define MMVM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00002000L
+#define MMVM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00004000L
+#define MMVM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00008000L
+#define MMVM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00010000L
+#define MMVM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00020000L
+#define MMVM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00040000L
+#define MMVM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00080000L
+#define MMVM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00100000L
+#define MMVM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00200000L
+#define MMVM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00400000L
+#define MMVM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00800000L
+#define MMVM_CONTEXT9_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x01000000L
+#define MMVM_CONTEXT9_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x02000000L
+//MMVM_CONTEXT10_CNTL
+#define MMVM_CONTEXT10_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define MMVM_CONTEXT10_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define MMVM_CONTEXT10_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x4
+#define MMVM_CONTEXT10_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x8
+#define MMVM_CONTEXT10_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x9
+#define MMVM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xa
+#define MMVM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xb
+#define MMVM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xc
+#define MMVM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xd
+#define MMVM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xe
+#define MMVM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xf
+#define MMVM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x10
+#define MMVM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x11
+#define MMVM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x12
+#define MMVM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x13
+#define MMVM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x14
+#define MMVM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x15
+#define MMVM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x16
+#define MMVM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x17
+#define MMVM_CONTEXT10_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x18
+#define MMVM_CONTEXT10_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x19
+#define MMVM_CONTEXT10_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define MMVM_CONTEXT10_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x0000000EL
+#define MMVM_CONTEXT10_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x000000F0L
+#define MMVM_CONTEXT10_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000100L
+#define MMVM_CONTEXT10_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000200L
+#define MMVM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000400L
+#define MMVM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000800L
+#define MMVM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00001000L
+#define MMVM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00002000L
+#define MMVM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00004000L
+#define MMVM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00008000L
+#define MMVM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00010000L
+#define MMVM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00020000L
+#define MMVM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00040000L
+#define MMVM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00080000L
+#define MMVM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00100000L
+#define MMVM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00200000L
+#define MMVM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00400000L
+#define MMVM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00800000L
+#define MMVM_CONTEXT10_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x01000000L
+#define MMVM_CONTEXT10_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x02000000L
+//MMVM_CONTEXT11_CNTL
+#define MMVM_CONTEXT11_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define MMVM_CONTEXT11_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define MMVM_CONTEXT11_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x4
+#define MMVM_CONTEXT11_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x8
+#define MMVM_CONTEXT11_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x9
+#define MMVM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xa
+#define MMVM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xb
+#define MMVM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xc
+#define MMVM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xd
+#define MMVM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xe
+#define MMVM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xf
+#define MMVM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x10
+#define MMVM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x11
+#define MMVM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x12
+#define MMVM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x13
+#define MMVM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x14
+#define MMVM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x15
+#define MMVM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x16
+#define MMVM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x17
+#define MMVM_CONTEXT11_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x18
+#define MMVM_CONTEXT11_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x19
+#define MMVM_CONTEXT11_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define MMVM_CONTEXT11_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x0000000EL
+#define MMVM_CONTEXT11_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x000000F0L
+#define MMVM_CONTEXT11_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000100L
+#define MMVM_CONTEXT11_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000200L
+#define MMVM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000400L
+#define MMVM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000800L
+#define MMVM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00001000L
+#define MMVM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00002000L
+#define MMVM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00004000L
+#define MMVM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00008000L
+#define MMVM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00010000L
+#define MMVM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00020000L
+#define MMVM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00040000L
+#define MMVM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00080000L
+#define MMVM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00100000L
+#define MMVM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00200000L
+#define MMVM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00400000L
+#define MMVM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00800000L
+#define MMVM_CONTEXT11_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x01000000L
+#define MMVM_CONTEXT11_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x02000000L
+//MMVM_CONTEXT12_CNTL
+#define MMVM_CONTEXT12_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define MMVM_CONTEXT12_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define MMVM_CONTEXT12_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x4
+#define MMVM_CONTEXT12_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x8
+#define MMVM_CONTEXT12_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x9
+#define MMVM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xa
+#define MMVM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xb
+#define MMVM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xc
+#define MMVM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xd
+#define MMVM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xe
+#define MMVM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xf
+#define MMVM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x10
+#define MMVM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x11
+#define MMVM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x12
+#define MMVM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x13
+#define MMVM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x14
+#define MMVM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x15
+#define MMVM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x16
+#define MMVM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x17
+#define MMVM_CONTEXT12_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x18
+#define MMVM_CONTEXT12_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x19
+#define MMVM_CONTEXT12_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define MMVM_CONTEXT12_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x0000000EL
+#define MMVM_CONTEXT12_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x000000F0L
+#define MMVM_CONTEXT12_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000100L
+#define MMVM_CONTEXT12_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000200L
+#define MMVM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000400L
+#define MMVM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000800L
+#define MMVM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00001000L
+#define MMVM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00002000L
+#define MMVM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00004000L
+#define MMVM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00008000L
+#define MMVM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00010000L
+#define MMVM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00020000L
+#define MMVM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00040000L
+#define MMVM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00080000L
+#define MMVM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00100000L
+#define MMVM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00200000L
+#define MMVM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00400000L
+#define MMVM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00800000L
+#define MMVM_CONTEXT12_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x01000000L
+#define MMVM_CONTEXT12_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x02000000L
+//MMVM_CONTEXT13_CNTL
+#define MMVM_CONTEXT13_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define MMVM_CONTEXT13_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define MMVM_CONTEXT13_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x4
+#define MMVM_CONTEXT13_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x8
+#define MMVM_CONTEXT13_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x9
+#define MMVM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xa
+#define MMVM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xb
+#define MMVM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xc
+#define MMVM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xd
+#define MMVM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xe
+#define MMVM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xf
+#define MMVM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x10
+#define MMVM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x11
+#define MMVM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x12
+#define MMVM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x13
+#define MMVM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x14
+#define MMVM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x15
+#define MMVM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x16
+#define MMVM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x17
+#define MMVM_CONTEXT13_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x18
+#define MMVM_CONTEXT13_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x19
+#define MMVM_CONTEXT13_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define MMVM_CONTEXT13_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x0000000EL
+#define MMVM_CONTEXT13_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x000000F0L
+#define MMVM_CONTEXT13_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000100L
+#define MMVM_CONTEXT13_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000200L
+#define MMVM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000400L
+#define MMVM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000800L
+#define MMVM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00001000L
+#define MMVM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00002000L
+#define MMVM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00004000L
+#define MMVM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00008000L
+#define MMVM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00010000L
+#define MMVM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00020000L
+#define MMVM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00040000L
+#define MMVM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00080000L
+#define MMVM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00100000L
+#define MMVM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00200000L
+#define MMVM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00400000L
+#define MMVM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00800000L
+#define MMVM_CONTEXT13_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x01000000L
+#define MMVM_CONTEXT13_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x02000000L
+//MMVM_CONTEXT14_CNTL
+#define MMVM_CONTEXT14_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define MMVM_CONTEXT14_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define MMVM_CONTEXT14_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x4
+#define MMVM_CONTEXT14_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x8
+#define MMVM_CONTEXT14_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x9
+#define MMVM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xa
+#define MMVM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xb
+#define MMVM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xc
+#define MMVM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xd
+#define MMVM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xe
+#define MMVM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xf
+#define MMVM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x10
+#define MMVM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x11
+#define MMVM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x12
+#define MMVM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x13
+#define MMVM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x14
+#define MMVM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x15
+#define MMVM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x16
+#define MMVM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x17
+#define MMVM_CONTEXT14_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x18
+#define MMVM_CONTEXT14_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x19
+#define MMVM_CONTEXT14_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define MMVM_CONTEXT14_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x0000000EL
+#define MMVM_CONTEXT14_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x000000F0L
+#define MMVM_CONTEXT14_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000100L
+#define MMVM_CONTEXT14_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000200L
+#define MMVM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000400L
+#define MMVM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000800L
+#define MMVM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00001000L
+#define MMVM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00002000L
+#define MMVM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00004000L
+#define MMVM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00008000L
+#define MMVM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00010000L
+#define MMVM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00020000L
+#define MMVM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00040000L
+#define MMVM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00080000L
+#define MMVM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00100000L
+#define MMVM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00200000L
+#define MMVM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00400000L
+#define MMVM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00800000L
+#define MMVM_CONTEXT14_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x01000000L
+#define MMVM_CONTEXT14_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x02000000L
+//MMVM_CONTEXT15_CNTL
+#define MMVM_CONTEXT15_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define MMVM_CONTEXT15_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define MMVM_CONTEXT15_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x4
+#define MMVM_CONTEXT15_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x8
+#define MMVM_CONTEXT15_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x9
+#define MMVM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xa
+#define MMVM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xb
+#define MMVM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xc
+#define MMVM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xd
+#define MMVM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xe
+#define MMVM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xf
+#define MMVM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x10
+#define MMVM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x11
+#define MMVM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x12
+#define MMVM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x13
+#define MMVM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x14
+#define MMVM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x15
+#define MMVM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x16
+#define MMVM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x17
+#define MMVM_CONTEXT15_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x18
+#define MMVM_CONTEXT15_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x19
+#define MMVM_CONTEXT15_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define MMVM_CONTEXT15_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x0000000EL
+#define MMVM_CONTEXT15_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x000000F0L
+#define MMVM_CONTEXT15_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000100L
+#define MMVM_CONTEXT15_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000200L
+#define MMVM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000400L
+#define MMVM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000800L
+#define MMVM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00001000L
+#define MMVM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00002000L
+#define MMVM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00004000L
+#define MMVM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00008000L
+#define MMVM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00010000L
+#define MMVM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00020000L
+#define MMVM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00040000L
+#define MMVM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00080000L
+#define MMVM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00100000L
+#define MMVM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00200000L
+#define MMVM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00400000L
+#define MMVM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00800000L
+#define MMVM_CONTEXT15_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x01000000L
+#define MMVM_CONTEXT15_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x02000000L
+//MMVM_CONTEXTS_DISABLE
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_0__SHIFT                                                       0x0
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_1__SHIFT                                                       0x1
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_2__SHIFT                                                       0x2
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_3__SHIFT                                                       0x3
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_4__SHIFT                                                       0x4
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_5__SHIFT                                                       0x5
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_6__SHIFT                                                       0x6
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_7__SHIFT                                                       0x7
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_8__SHIFT                                                       0x8
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_9__SHIFT                                                       0x9
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_10__SHIFT                                                      0xa
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_11__SHIFT                                                      0xb
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_12__SHIFT                                                      0xc
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_13__SHIFT                                                      0xd
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_14__SHIFT                                                      0xe
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_15__SHIFT                                                      0xf
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_0_MASK                                                         0x00000001L
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_1_MASK                                                         0x00000002L
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_2_MASK                                                         0x00000004L
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_3_MASK                                                         0x00000008L
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_4_MASK                                                         0x00000010L
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_5_MASK                                                         0x00000020L
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_6_MASK                                                         0x00000040L
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_7_MASK                                                         0x00000080L
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_8_MASK                                                         0x00000100L
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_9_MASK                                                         0x00000200L
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_10_MASK                                                        0x00000400L
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_11_MASK                                                        0x00000800L
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_12_MASK                                                        0x00001000L
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_13_MASK                                                        0x00002000L
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_14_MASK                                                        0x00004000L
+#define MMVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_15_MASK                                                        0x00008000L
+//MMVM_INVALIDATE_ENG0_SEM
+#define MMVM_INVALIDATE_ENG0_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define MMVM_INVALIDATE_ENG0_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//MMVM_INVALIDATE_ENG1_SEM
+#define MMVM_INVALIDATE_ENG1_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define MMVM_INVALIDATE_ENG1_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//MMVM_INVALIDATE_ENG2_SEM
+#define MMVM_INVALIDATE_ENG2_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define MMVM_INVALIDATE_ENG2_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//MMVM_INVALIDATE_ENG3_SEM
+#define MMVM_INVALIDATE_ENG3_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define MMVM_INVALIDATE_ENG3_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//MMVM_INVALIDATE_ENG4_SEM
+#define MMVM_INVALIDATE_ENG4_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define MMVM_INVALIDATE_ENG4_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//MMVM_INVALIDATE_ENG5_SEM
+#define MMVM_INVALIDATE_ENG5_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define MMVM_INVALIDATE_ENG5_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//MMVM_INVALIDATE_ENG6_SEM
+#define MMVM_INVALIDATE_ENG6_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define MMVM_INVALIDATE_ENG6_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//MMVM_INVALIDATE_ENG7_SEM
+#define MMVM_INVALIDATE_ENG7_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define MMVM_INVALIDATE_ENG7_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//MMVM_INVALIDATE_ENG8_SEM
+#define MMVM_INVALIDATE_ENG8_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define MMVM_INVALIDATE_ENG8_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//MMVM_INVALIDATE_ENG9_SEM
+#define MMVM_INVALIDATE_ENG9_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define MMVM_INVALIDATE_ENG9_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//MMVM_INVALIDATE_ENG10_SEM
+#define MMVM_INVALIDATE_ENG10_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define MMVM_INVALIDATE_ENG10_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//MMVM_INVALIDATE_ENG11_SEM
+#define MMVM_INVALIDATE_ENG11_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define MMVM_INVALIDATE_ENG11_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//MMVM_INVALIDATE_ENG12_SEM
+#define MMVM_INVALIDATE_ENG12_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define MMVM_INVALIDATE_ENG12_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//MMVM_INVALIDATE_ENG13_SEM
+#define MMVM_INVALIDATE_ENG13_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define MMVM_INVALIDATE_ENG13_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//MMVM_INVALIDATE_ENG14_SEM
+#define MMVM_INVALIDATE_ENG14_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define MMVM_INVALIDATE_ENG14_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//MMVM_INVALIDATE_ENG15_SEM
+#define MMVM_INVALIDATE_ENG15_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define MMVM_INVALIDATE_ENG15_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//MMVM_INVALIDATE_ENG16_SEM
+#define MMVM_INVALIDATE_ENG16_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define MMVM_INVALIDATE_ENG16_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//MMVM_INVALIDATE_ENG17_SEM
+#define MMVM_INVALIDATE_ENG17_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define MMVM_INVALIDATE_ENG17_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//MMVM_INVALIDATE_ENG0_REQ
+#define MMVM_INVALIDATE_ENG0_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG0_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define MMVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define MMVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define MMVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define MMVM_INVALIDATE_ENG0_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define MMVM_INVALIDATE_ENG0_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define MMVM_INVALIDATE_ENG0_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define MMVM_INVALIDATE_ENG0_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define MMVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define MMVM_INVALIDATE_ENG0_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG0_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define MMVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define MMVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define MMVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define MMVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define MMVM_INVALIDATE_ENG0_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define MMVM_INVALIDATE_ENG0_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define MMVM_INVALIDATE_ENG0_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define MMVM_INVALIDATE_ENG0_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define MMVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//MMVM_INVALIDATE_ENG1_REQ
+#define MMVM_INVALIDATE_ENG1_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG1_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define MMVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define MMVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define MMVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define MMVM_INVALIDATE_ENG1_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define MMVM_INVALIDATE_ENG1_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define MMVM_INVALIDATE_ENG1_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define MMVM_INVALIDATE_ENG1_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define MMVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define MMVM_INVALIDATE_ENG1_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG1_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define MMVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define MMVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define MMVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define MMVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define MMVM_INVALIDATE_ENG1_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define MMVM_INVALIDATE_ENG1_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define MMVM_INVALIDATE_ENG1_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define MMVM_INVALIDATE_ENG1_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define MMVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//MMVM_INVALIDATE_ENG2_REQ
+#define MMVM_INVALIDATE_ENG2_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG2_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define MMVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define MMVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define MMVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define MMVM_INVALIDATE_ENG2_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define MMVM_INVALIDATE_ENG2_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define MMVM_INVALIDATE_ENG2_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define MMVM_INVALIDATE_ENG2_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define MMVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define MMVM_INVALIDATE_ENG2_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG2_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define MMVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define MMVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define MMVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define MMVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define MMVM_INVALIDATE_ENG2_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define MMVM_INVALIDATE_ENG2_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define MMVM_INVALIDATE_ENG2_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define MMVM_INVALIDATE_ENG2_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define MMVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//MMVM_INVALIDATE_ENG3_REQ
+#define MMVM_INVALIDATE_ENG3_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG3_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define MMVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define MMVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define MMVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define MMVM_INVALIDATE_ENG3_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define MMVM_INVALIDATE_ENG3_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define MMVM_INVALIDATE_ENG3_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define MMVM_INVALIDATE_ENG3_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define MMVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define MMVM_INVALIDATE_ENG3_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG3_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define MMVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define MMVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define MMVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define MMVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define MMVM_INVALIDATE_ENG3_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define MMVM_INVALIDATE_ENG3_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define MMVM_INVALIDATE_ENG3_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define MMVM_INVALIDATE_ENG3_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define MMVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//MMVM_INVALIDATE_ENG4_REQ
+#define MMVM_INVALIDATE_ENG4_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG4_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define MMVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define MMVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define MMVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define MMVM_INVALIDATE_ENG4_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define MMVM_INVALIDATE_ENG4_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define MMVM_INVALIDATE_ENG4_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define MMVM_INVALIDATE_ENG4_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define MMVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define MMVM_INVALIDATE_ENG4_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG4_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define MMVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define MMVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define MMVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define MMVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define MMVM_INVALIDATE_ENG4_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define MMVM_INVALIDATE_ENG4_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define MMVM_INVALIDATE_ENG4_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define MMVM_INVALIDATE_ENG4_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define MMVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//MMVM_INVALIDATE_ENG5_REQ
+#define MMVM_INVALIDATE_ENG5_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG5_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define MMVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define MMVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define MMVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define MMVM_INVALIDATE_ENG5_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define MMVM_INVALIDATE_ENG5_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define MMVM_INVALIDATE_ENG5_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define MMVM_INVALIDATE_ENG5_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define MMVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define MMVM_INVALIDATE_ENG5_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG5_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define MMVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define MMVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define MMVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define MMVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define MMVM_INVALIDATE_ENG5_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define MMVM_INVALIDATE_ENG5_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define MMVM_INVALIDATE_ENG5_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define MMVM_INVALIDATE_ENG5_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define MMVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//MMVM_INVALIDATE_ENG6_REQ
+#define MMVM_INVALIDATE_ENG6_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG6_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define MMVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define MMVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define MMVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define MMVM_INVALIDATE_ENG6_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define MMVM_INVALIDATE_ENG6_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define MMVM_INVALIDATE_ENG6_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define MMVM_INVALIDATE_ENG6_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define MMVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define MMVM_INVALIDATE_ENG6_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG6_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define MMVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define MMVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define MMVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define MMVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define MMVM_INVALIDATE_ENG6_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define MMVM_INVALIDATE_ENG6_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define MMVM_INVALIDATE_ENG6_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define MMVM_INVALIDATE_ENG6_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define MMVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//MMVM_INVALIDATE_ENG7_REQ
+#define MMVM_INVALIDATE_ENG7_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG7_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define MMVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define MMVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define MMVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define MMVM_INVALIDATE_ENG7_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define MMVM_INVALIDATE_ENG7_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define MMVM_INVALIDATE_ENG7_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define MMVM_INVALIDATE_ENG7_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define MMVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define MMVM_INVALIDATE_ENG7_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG7_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define MMVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define MMVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define MMVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define MMVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define MMVM_INVALIDATE_ENG7_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define MMVM_INVALIDATE_ENG7_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define MMVM_INVALIDATE_ENG7_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define MMVM_INVALIDATE_ENG7_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define MMVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//MMVM_INVALIDATE_ENG8_REQ
+#define MMVM_INVALIDATE_ENG8_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG8_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define MMVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define MMVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define MMVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define MMVM_INVALIDATE_ENG8_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define MMVM_INVALIDATE_ENG8_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define MMVM_INVALIDATE_ENG8_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define MMVM_INVALIDATE_ENG8_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define MMVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define MMVM_INVALIDATE_ENG8_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG8_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define MMVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define MMVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define MMVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define MMVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define MMVM_INVALIDATE_ENG8_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define MMVM_INVALIDATE_ENG8_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define MMVM_INVALIDATE_ENG8_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define MMVM_INVALIDATE_ENG8_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define MMVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//MMVM_INVALIDATE_ENG9_REQ
+#define MMVM_INVALIDATE_ENG9_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG9_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define MMVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define MMVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define MMVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define MMVM_INVALIDATE_ENG9_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define MMVM_INVALIDATE_ENG9_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define MMVM_INVALIDATE_ENG9_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define MMVM_INVALIDATE_ENG9_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define MMVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE3__SHIFT                                                   0x1b
+#define MMVM_INVALIDATE_ENG9_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG9_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define MMVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define MMVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define MMVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define MMVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define MMVM_INVALIDATE_ENG9_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define MMVM_INVALIDATE_ENG9_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define MMVM_INVALIDATE_ENG9_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define MMVM_INVALIDATE_ENG9_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+#define MMVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE3_MASK                                                     0x08000000L
+//MMVM_INVALIDATE_ENG10_REQ
+#define MMVM_INVALIDATE_ENG10_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define MMVM_INVALIDATE_ENG10_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define MMVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define MMVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define MMVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define MMVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define MMVM_INVALIDATE_ENG10_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define MMVM_INVALIDATE_ENG10_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define MMVM_INVALIDATE_ENG10_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define MMVM_INVALIDATE_ENG10_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define MMVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE3__SHIFT                                                  0x1b
+#define MMVM_INVALIDATE_ENG10_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define MMVM_INVALIDATE_ENG10_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define MMVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define MMVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define MMVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define MMVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define MMVM_INVALIDATE_ENG10_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define MMVM_INVALIDATE_ENG10_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define MMVM_INVALIDATE_ENG10_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define MMVM_INVALIDATE_ENG10_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+#define MMVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE3_MASK                                                    0x08000000L
+//MMVM_INVALIDATE_ENG11_REQ
+#define MMVM_INVALIDATE_ENG11_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define MMVM_INVALIDATE_ENG11_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define MMVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define MMVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define MMVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define MMVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define MMVM_INVALIDATE_ENG11_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define MMVM_INVALIDATE_ENG11_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define MMVM_INVALIDATE_ENG11_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define MMVM_INVALIDATE_ENG11_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define MMVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE3__SHIFT                                                  0x1b
+#define MMVM_INVALIDATE_ENG11_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define MMVM_INVALIDATE_ENG11_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define MMVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define MMVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define MMVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define MMVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define MMVM_INVALIDATE_ENG11_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define MMVM_INVALIDATE_ENG11_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define MMVM_INVALIDATE_ENG11_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define MMVM_INVALIDATE_ENG11_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+#define MMVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE3_MASK                                                    0x08000000L
+//MMVM_INVALIDATE_ENG12_REQ
+#define MMVM_INVALIDATE_ENG12_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define MMVM_INVALIDATE_ENG12_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define MMVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define MMVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define MMVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define MMVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define MMVM_INVALIDATE_ENG12_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define MMVM_INVALIDATE_ENG12_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define MMVM_INVALIDATE_ENG12_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define MMVM_INVALIDATE_ENG12_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define MMVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE3__SHIFT                                                  0x1b
+#define MMVM_INVALIDATE_ENG12_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define MMVM_INVALIDATE_ENG12_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define MMVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define MMVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define MMVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define MMVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define MMVM_INVALIDATE_ENG12_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define MMVM_INVALIDATE_ENG12_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define MMVM_INVALIDATE_ENG12_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define MMVM_INVALIDATE_ENG12_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+#define MMVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE3_MASK                                                    0x08000000L
+//MMVM_INVALIDATE_ENG13_REQ
+#define MMVM_INVALIDATE_ENG13_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define MMVM_INVALIDATE_ENG13_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define MMVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define MMVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define MMVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define MMVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define MMVM_INVALIDATE_ENG13_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define MMVM_INVALIDATE_ENG13_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define MMVM_INVALIDATE_ENG13_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define MMVM_INVALIDATE_ENG13_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define MMVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE3__SHIFT                                                  0x1b
+#define MMVM_INVALIDATE_ENG13_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define MMVM_INVALIDATE_ENG13_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define MMVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define MMVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define MMVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define MMVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define MMVM_INVALIDATE_ENG13_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define MMVM_INVALIDATE_ENG13_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define MMVM_INVALIDATE_ENG13_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define MMVM_INVALIDATE_ENG13_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+#define MMVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE3_MASK                                                    0x08000000L
+//MMVM_INVALIDATE_ENG14_REQ
+#define MMVM_INVALIDATE_ENG14_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define MMVM_INVALIDATE_ENG14_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define MMVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define MMVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define MMVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define MMVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define MMVM_INVALIDATE_ENG14_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define MMVM_INVALIDATE_ENG14_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define MMVM_INVALIDATE_ENG14_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define MMVM_INVALIDATE_ENG14_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define MMVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE3__SHIFT                                                  0x1b
+#define MMVM_INVALIDATE_ENG14_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define MMVM_INVALIDATE_ENG14_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define MMVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define MMVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define MMVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define MMVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define MMVM_INVALIDATE_ENG14_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define MMVM_INVALIDATE_ENG14_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define MMVM_INVALIDATE_ENG14_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define MMVM_INVALIDATE_ENG14_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+#define MMVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE3_MASK                                                    0x08000000L
+//MMVM_INVALIDATE_ENG15_REQ
+#define MMVM_INVALIDATE_ENG15_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define MMVM_INVALIDATE_ENG15_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define MMVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define MMVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define MMVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define MMVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define MMVM_INVALIDATE_ENG15_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define MMVM_INVALIDATE_ENG15_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define MMVM_INVALIDATE_ENG15_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define MMVM_INVALIDATE_ENG15_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define MMVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE3__SHIFT                                                  0x1b
+#define MMVM_INVALIDATE_ENG15_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define MMVM_INVALIDATE_ENG15_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define MMVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define MMVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define MMVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define MMVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define MMVM_INVALIDATE_ENG15_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define MMVM_INVALIDATE_ENG15_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define MMVM_INVALIDATE_ENG15_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define MMVM_INVALIDATE_ENG15_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+#define MMVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE3_MASK                                                    0x08000000L
+//MMVM_INVALIDATE_ENG16_REQ
+#define MMVM_INVALIDATE_ENG16_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define MMVM_INVALIDATE_ENG16_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define MMVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define MMVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define MMVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define MMVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define MMVM_INVALIDATE_ENG16_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define MMVM_INVALIDATE_ENG16_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define MMVM_INVALIDATE_ENG16_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define MMVM_INVALIDATE_ENG16_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define MMVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE3__SHIFT                                                  0x1b
+#define MMVM_INVALIDATE_ENG16_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define MMVM_INVALIDATE_ENG16_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define MMVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define MMVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define MMVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define MMVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define MMVM_INVALIDATE_ENG16_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define MMVM_INVALIDATE_ENG16_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define MMVM_INVALIDATE_ENG16_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define MMVM_INVALIDATE_ENG16_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+#define MMVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE3_MASK                                                    0x08000000L
+//MMVM_INVALIDATE_ENG17_REQ
+#define MMVM_INVALIDATE_ENG17_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define MMVM_INVALIDATE_ENG17_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define MMVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define MMVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define MMVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define MMVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define MMVM_INVALIDATE_ENG17_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define MMVM_INVALIDATE_ENG17_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define MMVM_INVALIDATE_ENG17_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define MMVM_INVALIDATE_ENG17_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define MMVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE3__SHIFT                                                  0x1b
+#define MMVM_INVALIDATE_ENG17_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define MMVM_INVALIDATE_ENG17_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define MMVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define MMVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define MMVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define MMVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define MMVM_INVALIDATE_ENG17_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define MMVM_INVALIDATE_ENG17_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define MMVM_INVALIDATE_ENG17_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define MMVM_INVALIDATE_ENG17_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+#define MMVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE3_MASK                                                    0x08000000L
+//MMVM_INVALIDATE_ENG0_ACK
+#define MMVM_INVALIDATE_ENG0_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG0_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define MMVM_INVALIDATE_ENG0_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG0_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//MMVM_INVALIDATE_ENG1_ACK
+#define MMVM_INVALIDATE_ENG1_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG1_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define MMVM_INVALIDATE_ENG1_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG1_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//MMVM_INVALIDATE_ENG2_ACK
+#define MMVM_INVALIDATE_ENG2_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG2_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define MMVM_INVALIDATE_ENG2_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG2_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//MMVM_INVALIDATE_ENG3_ACK
+#define MMVM_INVALIDATE_ENG3_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG3_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define MMVM_INVALIDATE_ENG3_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG3_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//MMVM_INVALIDATE_ENG4_ACK
+#define MMVM_INVALIDATE_ENG4_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG4_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define MMVM_INVALIDATE_ENG4_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG4_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//MMVM_INVALIDATE_ENG5_ACK
+#define MMVM_INVALIDATE_ENG5_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG5_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define MMVM_INVALIDATE_ENG5_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG5_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//MMVM_INVALIDATE_ENG6_ACK
+#define MMVM_INVALIDATE_ENG6_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG6_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define MMVM_INVALIDATE_ENG6_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG6_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//MMVM_INVALIDATE_ENG7_ACK
+#define MMVM_INVALIDATE_ENG7_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG7_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define MMVM_INVALIDATE_ENG7_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG7_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//MMVM_INVALIDATE_ENG8_ACK
+#define MMVM_INVALIDATE_ENG8_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG8_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define MMVM_INVALIDATE_ENG8_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG8_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//MMVM_INVALIDATE_ENG9_ACK
+#define MMVM_INVALIDATE_ENG9_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define MMVM_INVALIDATE_ENG9_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define MMVM_INVALIDATE_ENG9_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define MMVM_INVALIDATE_ENG9_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//MMVM_INVALIDATE_ENG10_ACK
+#define MMVM_INVALIDATE_ENG10_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define MMVM_INVALIDATE_ENG10_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG10_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define MMVM_INVALIDATE_ENG10_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//MMVM_INVALIDATE_ENG11_ACK
+#define MMVM_INVALIDATE_ENG11_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define MMVM_INVALIDATE_ENG11_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG11_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define MMVM_INVALIDATE_ENG11_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//MMVM_INVALIDATE_ENG12_ACK
+#define MMVM_INVALIDATE_ENG12_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define MMVM_INVALIDATE_ENG12_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG12_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define MMVM_INVALIDATE_ENG12_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//MMVM_INVALIDATE_ENG13_ACK
+#define MMVM_INVALIDATE_ENG13_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define MMVM_INVALIDATE_ENG13_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG13_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define MMVM_INVALIDATE_ENG13_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//MMVM_INVALIDATE_ENG14_ACK
+#define MMVM_INVALIDATE_ENG14_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define MMVM_INVALIDATE_ENG14_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG14_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define MMVM_INVALIDATE_ENG14_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//MMVM_INVALIDATE_ENG15_ACK
+#define MMVM_INVALIDATE_ENG15_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define MMVM_INVALIDATE_ENG15_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG15_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define MMVM_INVALIDATE_ENG15_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//MMVM_INVALIDATE_ENG16_ACK
+#define MMVM_INVALIDATE_ENG16_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define MMVM_INVALIDATE_ENG16_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG16_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define MMVM_INVALIDATE_ENG16_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//MMVM_INVALIDATE_ENG17_ACK
+#define MMVM_INVALIDATE_ENG17_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define MMVM_INVALIDATE_ENG17_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define MMVM_INVALIDATE_ENG17_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define MMVM_INVALIDATE_ENG17_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//MMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define MMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define MMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define MMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG0_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG0_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define MMVM_INVALIDATE_ENG0_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//MMVM_INVALIDATE_ENG1_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG1_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define MMVM_INVALIDATE_ENG1_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define MMVM_INVALIDATE_ENG1_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define MMVM_INVALIDATE_ENG1_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG1_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG1_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define MMVM_INVALIDATE_ENG1_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//MMVM_INVALIDATE_ENG2_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG2_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define MMVM_INVALIDATE_ENG2_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define MMVM_INVALIDATE_ENG2_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define MMVM_INVALIDATE_ENG2_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG2_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG2_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define MMVM_INVALIDATE_ENG2_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//MMVM_INVALIDATE_ENG3_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG3_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define MMVM_INVALIDATE_ENG3_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define MMVM_INVALIDATE_ENG3_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define MMVM_INVALIDATE_ENG3_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG3_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG3_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define MMVM_INVALIDATE_ENG3_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//MMVM_INVALIDATE_ENG4_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG4_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define MMVM_INVALIDATE_ENG4_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define MMVM_INVALIDATE_ENG4_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define MMVM_INVALIDATE_ENG4_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG4_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG4_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define MMVM_INVALIDATE_ENG4_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//MMVM_INVALIDATE_ENG5_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG5_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define MMVM_INVALIDATE_ENG5_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define MMVM_INVALIDATE_ENG5_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define MMVM_INVALIDATE_ENG5_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG5_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG5_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define MMVM_INVALIDATE_ENG5_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//MMVM_INVALIDATE_ENG6_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG6_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define MMVM_INVALIDATE_ENG6_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define MMVM_INVALIDATE_ENG6_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define MMVM_INVALIDATE_ENG6_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG6_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG6_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define MMVM_INVALIDATE_ENG6_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//MMVM_INVALIDATE_ENG7_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG7_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define MMVM_INVALIDATE_ENG7_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define MMVM_INVALIDATE_ENG7_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define MMVM_INVALIDATE_ENG7_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG7_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG7_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define MMVM_INVALIDATE_ENG7_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//MMVM_INVALIDATE_ENG8_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG8_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define MMVM_INVALIDATE_ENG8_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define MMVM_INVALIDATE_ENG8_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define MMVM_INVALIDATE_ENG8_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG8_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG8_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define MMVM_INVALIDATE_ENG8_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//MMVM_INVALIDATE_ENG9_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG9_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define MMVM_INVALIDATE_ENG9_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define MMVM_INVALIDATE_ENG9_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define MMVM_INVALIDATE_ENG9_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG9_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG9_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                                0x0
+#define MMVM_INVALIDATE_ENG9_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                  0x00003FFFL
+//MMVM_INVALIDATE_ENG10_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG10_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define MMVM_INVALIDATE_ENG10_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define MMVM_INVALIDATE_ENG10_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define MMVM_INVALIDATE_ENG10_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG10_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG10_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                               0x0
+#define MMVM_INVALIDATE_ENG10_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                 0x00003FFFL
+//MMVM_INVALIDATE_ENG11_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG11_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define MMVM_INVALIDATE_ENG11_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define MMVM_INVALIDATE_ENG11_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define MMVM_INVALIDATE_ENG11_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG11_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG11_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                               0x0
+#define MMVM_INVALIDATE_ENG11_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                 0x00003FFFL
+//MMVM_INVALIDATE_ENG12_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG12_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define MMVM_INVALIDATE_ENG12_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define MMVM_INVALIDATE_ENG12_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define MMVM_INVALIDATE_ENG12_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG12_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG12_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                               0x0
+#define MMVM_INVALIDATE_ENG12_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                 0x00003FFFL
+//MMVM_INVALIDATE_ENG13_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG13_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define MMVM_INVALIDATE_ENG13_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define MMVM_INVALIDATE_ENG13_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define MMVM_INVALIDATE_ENG13_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG13_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG13_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                               0x0
+#define MMVM_INVALIDATE_ENG13_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                 0x00003FFFL
+//MMVM_INVALIDATE_ENG14_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG14_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define MMVM_INVALIDATE_ENG14_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define MMVM_INVALIDATE_ENG14_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define MMVM_INVALIDATE_ENG14_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG14_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG14_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                               0x0
+#define MMVM_INVALIDATE_ENG14_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                 0x00003FFFL
+//MMVM_INVALIDATE_ENG15_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG15_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define MMVM_INVALIDATE_ENG15_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define MMVM_INVALIDATE_ENG15_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define MMVM_INVALIDATE_ENG15_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG15_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG15_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                               0x0
+#define MMVM_INVALIDATE_ENG15_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                 0x00003FFFL
+//MMVM_INVALIDATE_ENG16_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG16_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define MMVM_INVALIDATE_ENG16_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define MMVM_INVALIDATE_ENG16_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define MMVM_INVALIDATE_ENG16_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG16_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG16_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                               0x0
+#define MMVM_INVALIDATE_ENG16_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                 0x00003FFFL
+//MMVM_INVALIDATE_ENG17_ADDR_RANGE_LO32
+#define MMVM_INVALIDATE_ENG17_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define MMVM_INVALIDATE_ENG17_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define MMVM_INVALIDATE_ENG17_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define MMVM_INVALIDATE_ENG17_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//MMVM_INVALIDATE_ENG17_ADDR_RANGE_HI32
+#define MMVM_INVALIDATE_ENG17_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14__SHIFT                               0x0
+#define MMVM_INVALIDATE_ENG17_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI14_MASK                                 0x00003FFFL
+//MMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32
+#define MMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32
+#define MMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define MMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32
+#define MMVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32
+#define MMVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define MMVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32
+#define MMVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32
+#define MMVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define MMVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32
+#define MMVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32
+#define MMVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define MMVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32
+#define MMVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32
+#define MMVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define MMVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32
+#define MMVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32
+#define MMVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define MMVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32
+#define MMVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32
+#define MMVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define MMVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32
+#define MMVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32
+#define MMVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define MMVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32
+#define MMVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32
+#define MMVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define MMVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32
+#define MMVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32
+#define MMVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define MMVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32
+#define MMVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define MMVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32
+#define MMVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define MMVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32
+#define MMVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define MMVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32
+#define MMVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define MMVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32
+#define MMVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define MMVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32
+#define MMVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define MMVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32
+#define MMVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define MMVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32
+#define MMVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define MMVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32
+#define MMVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define MMVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32
+#define MMVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define MMVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32
+#define MMVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define MMVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32
+#define MMVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define MMVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32
+#define MMVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32
+#define MMVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define MMVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//MMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32
+#define MMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32
+#define MMVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define MMVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//MMVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32
+#define MMVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32
+#define MMVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define MMVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//MMVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32
+#define MMVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32
+#define MMVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define MMVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//MMVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32
+#define MMVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32
+#define MMVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define MMVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//MMVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32
+#define MMVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32
+#define MMVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define MMVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//MMVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32
+#define MMVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32
+#define MMVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define MMVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//MMVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32
+#define MMVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32
+#define MMVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define MMVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//MMVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32
+#define MMVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32
+#define MMVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define MMVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//MMVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32
+#define MMVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define MMVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//MMVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32
+#define MMVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                             0x0
+#define MMVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                               0x00001FFFL
+//MMVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32
+#define MMVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define MMVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32
+#define MMVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                            0x0
+#define MMVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                              0x00001FFFL
+//MMVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32
+#define MMVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define MMVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32
+#define MMVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                            0x0
+#define MMVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                              0x00001FFFL
+//MMVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32
+#define MMVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define MMVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32
+#define MMVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                            0x0
+#define MMVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                              0x00001FFFL
+//MMVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32
+#define MMVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define MMVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32
+#define MMVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                            0x0
+#define MMVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                              0x00001FFFL
+//MMVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32
+#define MMVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define MMVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32
+#define MMVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                            0x0
+#define MMVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                              0x00001FFFL
+//MMVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32
+#define MMVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define MMVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//MMVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32
+#define MMVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                            0x0
+#define MMVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                              0x00001FFFL
+//MMVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32
+#define MMVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define MMVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//MMVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32
+#define MMVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define MMVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//MMVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32
+#define MMVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define MMVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//MMVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32
+#define MMVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define MMVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//MMVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32
+#define MMVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define MMVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//MMVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32
+#define MMVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define MMVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//MMVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32
+#define MMVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define MMVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//MMVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32
+#define MMVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define MMVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//MMVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32
+#define MMVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define MMVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//MMVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32
+#define MMVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define MMVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//MMVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32
+#define MMVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define MMVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//MMVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32
+#define MMVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define MMVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//MMVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32
+#define MMVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define MMVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//MMVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32
+#define MMVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define MMVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//MMVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32
+#define MMVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define MMVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//MMVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32
+#define MMVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define MMVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//MMVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32
+#define MMVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define MMVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//MMVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32
+#define MMVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define MMVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//MMVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32
+#define MMVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define MMVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//MMVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32
+#define MMVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                               0x0
+#define MMVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                 0x00001FFFL
+//MMVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32
+#define MMVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define MMVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//MMVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32
+#define MMVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                              0x0
+#define MMVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                0x00001FFFL
+//MMVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32
+#define MMVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define MMVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//MMVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32
+#define MMVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                              0x0
+#define MMVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                0x00001FFFL
+//MMVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32
+#define MMVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define MMVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//MMVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32
+#define MMVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                              0x0
+#define MMVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                0x00001FFFL
+//MMVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32
+#define MMVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define MMVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//MMVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32
+#define MMVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                              0x0
+#define MMVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                0x00001FFFL
+//MMVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32
+#define MMVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define MMVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//MMVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32
+#define MMVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                              0x0
+#define MMVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                0x00001FFFL
+//MMVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32
+#define MMVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define MMVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//MMVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32
+#define MMVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13__SHIFT                              0x0
+#define MMVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI13_MASK                                0x00001FFFL
+//MMVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT                       0x0
+#define MMVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                         0x5
+#define MMVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                         0xa
+#define MMVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                         0x0000001FL
+#define MMVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                           0x000003E0L
+#define MMVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                           0x0000FC00L
+//MMVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define MMVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define MMVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define MMVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define MMVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define MMVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//MMVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define MMVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define MMVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define MMVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define MMVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define MMVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//MMVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define MMVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define MMVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define MMVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define MMVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define MMVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//MMVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define MMVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define MMVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define MMVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define MMVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define MMVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//MMVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define MMVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define MMVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define MMVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define MMVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define MMVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//MMVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define MMVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define MMVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define MMVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define MMVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define MMVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//MMVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define MMVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define MMVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define MMVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define MMVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define MMVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//MMVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define MMVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define MMVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define MMVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define MMVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define MMVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//MMVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define MMVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define MMVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define MMVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define MMVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define MMVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//MMVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define MMVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define MMVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define MMVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define MMVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define MMVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//MMVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define MMVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define MMVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define MMVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define MMVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define MMVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+//MMVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define MMVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define MMVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define MMVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define MMVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define MMVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+//MMVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define MMVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define MMVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define MMVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define MMVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define MMVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+//MMVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define MMVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define MMVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define MMVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define MMVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define MMVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+//MMVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define MMVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define MMVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define MMVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define MMVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define MMVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+//MMVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define MMVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define MMVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define MMVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define MMVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define MMVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define MMVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+
+
+// addressBlock: mmhub_mmutcl2_mmvmsharedvfdec
+//MMMC_VM_PCIE_ATOMIC_SUPPORTED
+#define MMMC_VM_PCIE_ATOMIC_SUPPORTED__PCIE_ATOMIC_SUPPORTED__SHIFT                                           0x0
+#define MMMC_VM_PCIE_ATOMIC_SUPPORTED__PCIE_ATOMIC_SUPPORTED_MASK                                             0x00000001L
+
+
+// addressBlock: mmhub_mmutcl2_mmvmsharedhvdec
+//MMVM_PCIE_ATS_CNTL
+#define MMVM_PCIE_ATS_CNTL__STU__SHIFT                                                                        0x10
+#define MMVM_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                                 0x1f
+#define MMVM_PCIE_ATS_CNTL__STU_MASK                                                                          0x001F0000L
+#define MMVM_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                                   0x80000000L
+//MMMC_VM_NB_MMIOBASE
+#define MMMC_VM_NB_MMIOBASE__MMIOBASE__SHIFT                                                                  0x0
+#define MMMC_VM_NB_MMIOBASE__MMIOBASE_MASK                                                                    0xFFFFFFFFL
+//MMMC_VM_NB_MMIOLIMIT
+#define MMMC_VM_NB_MMIOLIMIT__MMIOLIMIT__SHIFT                                                                0x0
+#define MMMC_VM_NB_MMIOLIMIT__MMIOLIMIT_MASK                                                                  0xFFFFFFFFL
+//MMMC_VM_NB_PCI_CTRL
+#define MMMC_VM_NB_PCI_CTRL__MMIOENABLE__SHIFT                                                                0x17
+#define MMMC_VM_NB_PCI_CTRL__MMIOENABLE_MASK                                                                  0x00800000L
+//MMMC_VM_NB_PCI_ARB
+#define MMMC_VM_NB_PCI_ARB__VGA_HOLE__SHIFT                                                                   0x3
+#define MMMC_VM_NB_PCI_ARB__VGA_HOLE_MASK                                                                     0x00000008L
+//MMMC_VM_NB_TOP_OF_DRAM_SLOT1
+#define MMMC_VM_NB_TOP_OF_DRAM_SLOT1__TOP_OF_DRAM__SHIFT                                                      0x17
+#define MMMC_VM_NB_TOP_OF_DRAM_SLOT1__TOP_OF_DRAM_MASK                                                        0xFF800000L
+//MMMC_VM_NB_LOWER_TOP_OF_DRAM2
+#define MMMC_VM_NB_LOWER_TOP_OF_DRAM2__ENABLE__SHIFT                                                          0x0
+#define MMMC_VM_NB_LOWER_TOP_OF_DRAM2__LOWER_TOM2__SHIFT                                                      0x17
+#define MMMC_VM_NB_LOWER_TOP_OF_DRAM2__ENABLE_MASK                                                            0x00000001L
+#define MMMC_VM_NB_LOWER_TOP_OF_DRAM2__LOWER_TOM2_MASK                                                        0xFF800000L
+//MMMC_VM_NB_UPPER_TOP_OF_DRAM2
+#define MMMC_VM_NB_UPPER_TOP_OF_DRAM2__UPPER_TOM2__SHIFT                                                      0x0
+#define MMMC_VM_NB_UPPER_TOP_OF_DRAM2__UPPER_TOM2_MASK                                                        0x000FFFFFL
+//MMMC_VM_STEERING
+#define MMMC_VM_STEERING__DEFAULT_STEERING__SHIFT                                                             0x0
+#define MMMC_VM_STEERING__DEFAULT_STEERING_MASK                                                               0x00000003L
+//MMMC_SHARED_VIRT_RESET_REQ
+#define MMMC_SHARED_VIRT_RESET_REQ__VF__SHIFT                                                                 0x0
+#define MMMC_SHARED_VIRT_RESET_REQ__PF__SHIFT                                                                 0x8
+#define MMMC_SHARED_VIRT_RESET_REQ__VF_MASK                                                                   0x000000FFL
+#define MMMC_SHARED_VIRT_RESET_REQ__PF_MASK                                                                   0x00000100L
+//MMMC_VM_XGMI_LFB_CNTL
+#define MMMC_VM_XGMI_LFB_CNTL__PF_LFB_REGION__SHIFT                                                           0x0
+#define MMMC_VM_XGMI_LFB_CNTL__PF_MAX_REGION__SHIFT                                                           0x4
+#define MMMC_VM_XGMI_LFB_CNTL__PF_LFB_REGION_MASK                                                             0x0000000FL
+#define MMMC_VM_XGMI_LFB_CNTL__PF_MAX_REGION_MASK                                                             0x000000F0L
+//MMMC_VM_XGMI_LFB_SIZE
+#define MMMC_VM_XGMI_LFB_SIZE__PF_LFB_SIZE__SHIFT                                                             0x0
+#define MMMC_VM_XGMI_LFB_SIZE__PF_LFB_SIZE_MASK                                                               0x0001FFFFL
+//MMMC_VM_HOST_MAPPING
+#define MMMC_VM_HOST_MAPPING__MODE__SHIFT                                                                     0x0
+#define MMMC_VM_HOST_MAPPING__MODE_MASK                                                                       0x00000001L
+//MMMC_VM_XGMI_GPUIOV_ENABLE
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF0__SHIFT                                                         0x0
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF1__SHIFT                                                         0x1
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF2__SHIFT                                                         0x2
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF3__SHIFT                                                         0x3
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF4__SHIFT                                                         0x4
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF5__SHIFT                                                         0x5
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF6__SHIFT                                                         0x6
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF7__SHIFT                                                         0x7
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_PF__SHIFT                                                          0x1f
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF0_MASK                                                           0x00000001L
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF1_MASK                                                           0x00000002L
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF2_MASK                                                           0x00000004L
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF3_MASK                                                           0x00000008L
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF4_MASK                                                           0x00000010L
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF5_MASK                                                           0x00000020L
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF6_MASK                                                           0x00000040L
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_VF7_MASK                                                           0x00000080L
+#define MMMC_VM_XGMI_GPUIOV_ENABLE__ENABLE_PF_MASK                                                            0x80000000L
+//MMMC_VM_CACHEABLE_DRAM_ADDRESS_START
+#define MMMC_VM_CACHEABLE_DRAM_ADDRESS_START__ADDRESS__SHIFT                                                  0x0
+#define MMMC_VM_CACHEABLE_DRAM_ADDRESS_START__ADDRESS_MASK                                                    0x0FFFFFFFL
+//MMMC_VM_CACHEABLE_DRAM_ADDRESS_END
+#define MMMC_VM_CACHEABLE_DRAM_ADDRESS_END__ADDRESS__SHIFT                                                    0x0
+#define MMMC_VM_CACHEABLE_DRAM_ADDRESS_END__ADDRESS_MASK                                                      0x0FFFFFFFL
+//MMMC_VM_LOCAL_SYSMEM_ADDRESS_START
+#define MMMC_VM_LOCAL_SYSMEM_ADDRESS_START__ADDRESS__SHIFT                                                    0x0
+#define MMMC_VM_LOCAL_SYSMEM_ADDRESS_START__ADDRESS_MASK                                                      0x0FFFFFFFL
+//MMMC_VM_LOCAL_SYSMEM_ADDRESS_END
+#define MMMC_VM_LOCAL_SYSMEM_ADDRESS_END__ADDRESS__SHIFT                                                      0x0
+#define MMMC_VM_LOCAL_SYSMEM_ADDRESS_END__ADDRESS_MASK                                                        0x0FFFFFFFL
+//MMMC_VM_LPDDR_ADDRESS_START
+#define MMMC_VM_LPDDR_ADDRESS_START__ADDRESS__SHIFT                                                           0x0
+#define MMMC_VM_LPDDR_ADDRESS_START__ADDRESS_MASK                                                             0x0FFFFFFFL
+//MMMC_VM_LPDDR_ADDRESS_END
+#define MMMC_VM_LPDDR_ADDRESS_END__ADDRESS__SHIFT                                                             0x0
+#define MMMC_VM_LPDDR_ADDRESS_END__ADDRESS_MASK                                                               0x0FFFFFFFL
+//MMMC_VM_APT_CNTL
+#define MMMC_VM_APT_CNTL__FORCE_MTYPE_UC__SHIFT                                                               0x0
+#define MMMC_VM_APT_CNTL__DIRECT_SYSTEM_EN__SHIFT                                                             0x1
+#define MMMC_VM_APT_CNTL__FRAG_APT_INTXN_MODE__SHIFT                                                          0x2
+#define MMMC_VM_APT_CNTL__CHECK_IS_LOCAL__SHIFT                                                               0x4
+#define MMMC_VM_APT_CNTL__CAP_FRAG_SIZE_2M__SHIFT                                                             0x5
+#define MMMC_VM_APT_CNTL__LOCAL_SYSMEM_APERTURE_CNTL__SHIFT                                                   0x6
+#define MMMC_VM_APT_CNTL__LPDDR_APERTURE_CNTL__SHIFT                                                          0x8
+#define MMMC_VM_APT_CNTL__FORCE_MTYPE_UC_MASK                                                                 0x00000001L
+#define MMMC_VM_APT_CNTL__DIRECT_SYSTEM_EN_MASK                                                               0x00000002L
+#define MMMC_VM_APT_CNTL__FRAG_APT_INTXN_MODE_MASK                                                            0x0000000CL
+#define MMMC_VM_APT_CNTL__CHECK_IS_LOCAL_MASK                                                                 0x00000010L
+#define MMMC_VM_APT_CNTL__CAP_FRAG_SIZE_2M_MASK                                                               0x00000020L
+#define MMMC_VM_APT_CNTL__LOCAL_SYSMEM_APERTURE_CNTL_MASK                                                     0x000000C0L
+#define MMMC_VM_APT_CNTL__LPDDR_APERTURE_CNTL_MASK                                                            0x00000300L
+//MMUTCL2_HARVEST_BYPASS_GROUPS
+#define MMUTCL2_HARVEST_BYPASS_GROUPS__BYPASS_GROUPS__SHIFT                                                   0x0
+#define MMUTCL2_HARVEST_BYPASS_GROUPS__BYPASS_GROUPS_MASK                                                     0xFFFFFFFFL
+
+
+// addressBlock: mmhub_mmutcl2_mmatcl2dec
+//MM_ATC_L2_CNTL
+#define MM_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READ_REQUESTS__SHIFT                                            0x0
+#define MM_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITE_REQUESTS__SHIFT                                           0x2
+#define MM_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD__SHIFT                                0x4
+#define MM_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD__SHIFT                               0x5
+#define MM_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READ_REQUESTS__SHIFT                                       0x6
+#define MM_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITE_REQUESTS__SHIFT                                      0x8
+#define MM_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD__SHIFT                           0xa
+#define MM_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD__SHIFT                          0xb
+#define MM_ATC_L2_CNTL__CACHE_INVALIDATE_MODE__SHIFT                                                          0xc
+#define MM_ATC_L2_CNTL__FRAG_APT_INTXN_MODE__SHIFT                                                            0xf
+#define MM_ATC_L2_CNTL__CLI_GPA_REQ_FRAG_SIZE__SHIFT                                                          0x11
+#define MM_ATC_L2_CNTL__CLI_GVA_REQ_FRAG_SIZE__SHIFT                                                          0x17
+#define MM_ATC_L2_CNTL__FORCE_8T_CAP_ON_AT_RETURN__SHIFT                                                      0x1d
+#define MM_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READ_REQUESTS_MASK                                              0x00000003L
+#define MM_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITE_REQUESTS_MASK                                             0x0000000CL
+#define MM_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD_MASK                                  0x00000010L
+#define MM_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD_MASK                                 0x00000020L
+#define MM_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READ_REQUESTS_MASK                                         0x000000C0L
+#define MM_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITE_REQUESTS_MASK                                        0x00000300L
+#define MM_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD_MASK                             0x00000400L
+#define MM_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD_MASK                            0x00000800L
+#define MM_ATC_L2_CNTL__CACHE_INVALIDATE_MODE_MASK                                                            0x00007000L
+#define MM_ATC_L2_CNTL__FRAG_APT_INTXN_MODE_MASK                                                              0x00018000L
+#define MM_ATC_L2_CNTL__CLI_GPA_REQ_FRAG_SIZE_MASK                                                            0x007E0000L
+//MM_ATC_L2_CNTL2
+#define MM_ATC_L2_CNTL2__BANK_SELECT__SHIFT                                                                   0x0
+#define MM_ATC_L2_CNTL2__NUM_BANKS_LOG2__SHIFT                                                                0x6
+#define MM_ATC_L2_CNTL2__L2_CACHE_UPDATE_MODE__SHIFT                                                          0x9
+#define MM_ATC_L2_CNTL2__ENABLE_L2_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                           0xb
+#define MM_ATC_L2_CNTL2__L2_CACHE_SWAP_TAG_INDEX_LSBS__SHIFT                                                  0xc
+#define MM_ATC_L2_CNTL2__L2_CACHE_VMID_MODE__SHIFT                                                            0xf
+#define MM_ATC_L2_CNTL2__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE__SHIFT                                      0x12
+#define MM_ATC_L2_CNTL2__BANK_SELECT_MASK                                                                     0x0000003FL
+#define MM_ATC_L2_CNTL2__NUM_BANKS_LOG2_MASK                                                                  0x000001C0L
+#define MM_ATC_L2_CNTL2__L2_CACHE_UPDATE_MODE_MASK                                                            0x00000600L
+#define MM_ATC_L2_CNTL2__ENABLE_L2_CACHE_LRU_UPDATE_BY_WRITE_MASK                                             0x00000800L
+#define MM_ATC_L2_CNTL2__L2_CACHE_SWAP_TAG_INDEX_LSBS_MASK                                                    0x00007000L
+#define MM_ATC_L2_CNTL2__L2_CACHE_VMID_MODE_MASK                                                              0x00038000L
+#define MM_ATC_L2_CNTL2__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE_MASK                                        0x00FC0000L
+//MM_ATC_L2_CACHE_DATA0
+#define MM_ATC_L2_CACHE_DATA0__DATA_REGISTER_VALID__SHIFT                                                     0x0
+#define MM_ATC_L2_CACHE_DATA0__CACHE_ENTRY_VALID__SHIFT                                                       0x1
+#define MM_ATC_L2_CACHE_DATA0__CACHED_ATTRIBUTES__SHIFT                                                       0x2
+#define MM_ATC_L2_CACHE_DATA0__DATA_REGISTER_VALID_MASK                                                       0x00000001L
+#define MM_ATC_L2_CACHE_DATA0__CACHE_ENTRY_VALID_MASK                                                         0x00000002L
+#define MM_ATC_L2_CACHE_DATA0__CACHED_ATTRIBUTES_MASK                                                         0x00FFFFFCL
+//MM_ATC_L2_CACHE_DATA1
+#define MM_ATC_L2_CACHE_DATA1__VIRTUAL_PAGE_ADDRESS_LOW__SHIFT                                                0x0
+#define MM_ATC_L2_CACHE_DATA1__VIRTUAL_PAGE_ADDRESS_LOW_MASK                                                  0xFFFFFFFFL
+//MM_ATC_L2_CACHE_DATA2
+#define MM_ATC_L2_CACHE_DATA2__PHYSICAL_PAGE_ADDRESS__SHIFT                                                   0x0
+#define MM_ATC_L2_CACHE_DATA2__PHYSICAL_PAGE_ADDRESS_MASK                                                     0xFFFFFFFFL
+//MM_ATC_L2_CNTL3
+#define MM_ATC_L2_CNTL3__L2_SMALLK_CACHE_FRAGMENT_SIZE__SHIFT                                                 0x0
+#define MM_ATC_L2_CNTL3__L2_BIGK_CACHE_FRAGMENT_SIZE__SHIFT                                                   0x6
+#define MM_ATC_L2_CNTL3__DELAY_SEND_INVALIDATION_REQUEST__SHIFT                                               0xc
+#define MM_ATC_L2_CNTL3__ATS_REQUEST_CREDIT_MINUS1__SHIFT                                                     0xf
+#define MM_ATC_L2_CNTL3__COMPCLKREQ_OFF_HYSTERESIS__SHIFT                                                     0x15
+#define MM_ATC_L2_CNTL3__REPEATER_FGCG_OFF__SHIFT                                                             0x18
+#define MM_ATC_L2_CNTL3__L2_SMALLK_CACHE_FRAGMENT_SIZE_MASK                                                   0x0000003FL
+#define MM_ATC_L2_CNTL3__L2_BIGK_CACHE_FRAGMENT_SIZE_MASK                                                     0x00000FC0L
+#define MM_ATC_L2_CNTL3__DELAY_SEND_INVALIDATION_REQUEST_MASK                                                 0x00007000L
+#define MM_ATC_L2_CNTL3__ATS_REQUEST_CREDIT_MINUS1_MASK                                                       0x001F8000L
+#define MM_ATC_L2_CNTL3__COMPCLKREQ_OFF_HYSTERESIS_MASK                                                       0x00E00000L
+#define MM_ATC_L2_CNTL3__REPEATER_FGCG_OFF_MASK                                                               0x01000000L
+//MM_ATC_L2_CNTL4
+#define MM_ATC_L2_CNTL4__CANE_CLKEN_OVERRIDE__SHIFT                                                           0x0
+#define MM_ATC_L2_CNTL4__L2_RT_SMALLK_CACHE_FRAGMENT_SIZE__SHIFT                                              0x1
+#define MM_ATC_L2_CNTL4__L2_RT_BIGK_CACHE_FRAGMENT_SIZE__SHIFT                                                0x7
+#define MM_ATC_L2_CNTL4__L2_RT_SMALLK_CACHE_FRAGMENT_SIZE_MASK                                                0x0000007EL
+#define MM_ATC_L2_CNTL4__L2_RT_BIGK_CACHE_FRAGMENT_SIZE_MASK                                                  0x00001F80L
+//MM_ATC_L2_CNTL5
+#define MM_ATC_L2_CNTL5__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                       0x0
+#define MM_ATC_L2_CNTL5__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                      0xa
+#define MM_ATC_L2_CNTL5__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                         0x000003FFL
+#define MM_ATC_L2_CNTL5__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                        0x000FFC00L
+//MM_ATC_L2_MM_GROUP_RT_CLASSES
+#define MM_ATC_L2_MM_GROUP_RT_CLASSES__GROUP_RT_CLASS__SHIFT                                                  0x0
+#define MM_ATC_L2_MM_GROUP_RT_CLASSES__GROUP_RT_CLASS_MASK                                                    0xFFFFFFFFL
+//MM_ATC_L2_STATUS
+#define MM_ATC_L2_STATUS__BUSY__SHIFT                                                                         0x0
+#define MM_ATC_L2_STATUS__NO_OUTSTANDING_AT_REQUESTS__SHIFT                                                   0x1
+#define MM_ATC_L2_STATUS__BUSY_MASK                                                                           0x00000001L
+#define MM_ATC_L2_STATUS__NO_OUTSTANDING_AT_REQUESTS_MASK                                                     0x00000002L
+//MM_ATC_L2_MISC_CG
+#define MM_ATC_L2_MISC_CG__OFFDLY__SHIFT                                                                      0x6
+#define MM_ATC_L2_MISC_CG__ENABLE__SHIFT                                                                      0x12
+#define MM_ATC_L2_MISC_CG__MEM_LS_ENABLE__SHIFT                                                               0x13
+#define MM_ATC_L2_MISC_CG__OFFDLY_MASK                                                                        0x00000FC0L
+#define MM_ATC_L2_MISC_CG__ENABLE_MASK                                                                        0x00040000L
+#define MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK                                                                 0x00080000L
+//MM_ATC_L2_CGTT_CLK_CTRL
+#define MM_ATC_L2_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                              0x0
+#define MM_ATC_L2_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                        0x5
+#define MM_ATC_L2_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS__SHIFT                                                  0xd
+#define MM_ATC_L2_CGTT_CLK_CTRL__MIN_MGLS__SHIFT                                                              0x1a
+#define MM_ATC_L2_CGTT_CLK_CTRL__CGLS_DISABLE__SHIFT                                                          0x1d
+#define MM_ATC_L2_CGTT_CLK_CTRL__LS_DISABLE__SHIFT                                                            0x1e
+#define MM_ATC_L2_CGTT_CLK_CTRL__BUSY_OVERRIDE__SHIFT                                                         0x1f
+#define MM_ATC_L2_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                0x0000001FL
+#define MM_ATC_L2_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                          0x00001FE0L
+#define MM_ATC_L2_CGTT_CLK_CTRL__LS_ASSERT_HYSTERESIS_MASK                                                    0x03FFE000L
+#define MM_ATC_L2_CGTT_CLK_CTRL__MIN_MGLS_MASK                                                                0x1C000000L
+#define MM_ATC_L2_CGTT_CLK_CTRL__CGLS_DISABLE_MASK                                                            0x20000000L
+#define MM_ATC_L2_CGTT_CLK_CTRL__LS_DISABLE_MASK                                                              0x40000000L
+#define MM_ATC_L2_CGTT_CLK_CTRL__BUSY_OVERRIDE_MASK                                                           0x80000000L
+//MM_ATC_L2_SDPPORT_CTRL
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPCKEN__SHIFT                                                      0x0
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPCKENRCV__SHIFT                                                   0x1
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPDATACKEN__SHIFT                                                  0x2
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPDATACKENRCV__SHIFT                                               0x3
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_WRRSPCKEN__SHIFT                                                      0x4
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_WRRSPCKENRCV__SHIFT                                                   0x5
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_REQCKEN__SHIFT                                                        0x6
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_REQCKENRCV__SHIFT                                                     0x7
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_ORIGDATACKEN__SHIFT                                                   0x8
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_ORIGDATACKENRCV__SHIFT                                                0x9
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPCKEN_MASK                                                        0x00000001L
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPCKENRCV_MASK                                                     0x00000002L
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPDATACKEN_MASK                                                    0x00000004L
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPDATACKENRCV_MASK                                                 0x00000008L
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_WRRSPCKEN_MASK                                                        0x00000010L
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_WRRSPCKENRCV_MASK                                                     0x00000020L
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_REQCKEN_MASK                                                          0x00000040L
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_REQCKENRCV_MASK                                                       0x00000080L
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_ORIGDATACKEN_MASK                                                     0x00000100L
+#define MM_ATC_L2_SDPPORT_CTRL__SDPVDCI_ORIGDATACKENRCV_MASK                                                  0x00000200L
+
+
+// addressBlock: mmhub_mmutcl2_mmvml2pspdec
+//MMVM_IOMMU_CONTROL_REGISTER
+#define MMVM_IOMMU_CONTROL_REGISTER__IOMMUEN__SHIFT                                                           0x0
+#define MMVM_IOMMU_CONTROL_REGISTER__IOMMUEN_MASK                                                             0x00000001L
+//MMVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER
+#define MMVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER__PERFOPTEN__SHIFT                                0xd
+#define MMVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER__PERFOPTEN_MASK                                  0x00002000L
+
+
+// addressBlock: mmhub_mmutcl2_mmvmsharedpspdec
+//MMMC_VM_FB_OFFSET
+#define MMMC_VM_FB_OFFSET__FB_OFFSET__SHIFT                                                                   0x0
+#define MMMC_VM_FB_OFFSET__FB_OFFSET_MASK                                                                     0x0FFFFFFFL
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/mp/mp_15_0_8_offset.h b/drivers/gpu/drm/amd/include/asic_reg/mp/mp_15_0_8_offset.h
new file mode 100644
index 000000000000..90d0d6c3b7b6
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/mp/mp_15_0_8_offset.h
@@ -0,0 +1,868 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _mp_15_0_8_OFFSET_HEADER
+#define _mp_15_0_8_OFFSET_HEADER
+
+
+
+// addressBlock: mp_SmuMp1_SmnDec
+// base address: 0x0
+#define regMP1_SMN_C2PMSG_0                                                                             0x0040
+#define regMP1_SMN_C2PMSG_0_BASE_IDX                                                                    2
+#define regMP1_SMN_C2PMSG_1                                                                             0x0041
+#define regMP1_SMN_C2PMSG_1_BASE_IDX                                                                    2
+#define regMP1_SMN_C2PMSG_2                                                                             0x0042
+#define regMP1_SMN_C2PMSG_2_BASE_IDX                                                                    2
+#define regMP1_SMN_C2PMSG_3                                                                             0x0043
+#define regMP1_SMN_C2PMSG_3_BASE_IDX                                                                    2
+#define regMP1_SMN_C2PMSG_4                                                                             0x0044
+#define regMP1_SMN_C2PMSG_4_BASE_IDX                                                                    2
+#define regMP1_SMN_C2PMSG_5                                                                             0x0045
+#define regMP1_SMN_C2PMSG_5_BASE_IDX                                                                    2
+#define regMP1_SMN_C2PMSG_6                                                                             0x0046
+#define regMP1_SMN_C2PMSG_6_BASE_IDX                                                                    2
+#define regMP1_SMN_C2PMSG_7                                                                             0x0047
+#define regMP1_SMN_C2PMSG_7_BASE_IDX                                                                    2
+#define regMP1_SMN_C2PMSG_8                                                                             0x0048
+#define regMP1_SMN_C2PMSG_8_BASE_IDX                                                                    2
+#define regMP1_SMN_C2PMSG_9                                                                             0x0049
+#define regMP1_SMN_C2PMSG_9_BASE_IDX                                                                    2
+#define regMP1_SMN_C2PMSG_10                                                                            0x004a
+#define regMP1_SMN_C2PMSG_10_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_11                                                                            0x004b
+#define regMP1_SMN_C2PMSG_11_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_12                                                                            0x004c
+#define regMP1_SMN_C2PMSG_12_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_13                                                                            0x004d
+#define regMP1_SMN_C2PMSG_13_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_14                                                                            0x004e
+#define regMP1_SMN_C2PMSG_14_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_15                                                                            0x004f
+#define regMP1_SMN_C2PMSG_15_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_16                                                                            0x0050
+#define regMP1_SMN_C2PMSG_16_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_17                                                                            0x0051
+#define regMP1_SMN_C2PMSG_17_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_18                                                                            0x0052
+#define regMP1_SMN_C2PMSG_18_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_19                                                                            0x0053
+#define regMP1_SMN_C2PMSG_19_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_20                                                                            0x0054
+#define regMP1_SMN_C2PMSG_20_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_21                                                                            0x0055
+#define regMP1_SMN_C2PMSG_21_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_22                                                                            0x0056
+#define regMP1_SMN_C2PMSG_22_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_23                                                                            0x0057
+#define regMP1_SMN_C2PMSG_23_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_24                                                                            0x0058
+#define regMP1_SMN_C2PMSG_24_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_25                                                                            0x0059
+#define regMP1_SMN_C2PMSG_25_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_26                                                                            0x005a
+#define regMP1_SMN_C2PMSG_26_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_27                                                                            0x005b
+#define regMP1_SMN_C2PMSG_27_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_28                                                                            0x005c
+#define regMP1_SMN_C2PMSG_28_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_29                                                                            0x005d
+#define regMP1_SMN_C2PMSG_29_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_30                                                                            0x005e
+#define regMP1_SMN_C2PMSG_30_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_31                                                                            0x005f
+#define regMP1_SMN_C2PMSG_31_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_32                                                                            0x0060
+#define regMP1_SMN_C2PMSG_32_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_33                                                                            0x0061
+#define regMP1_SMN_C2PMSG_33_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_34                                                                            0x0062
+#define regMP1_SMN_C2PMSG_34_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_35                                                                            0x0063
+#define regMP1_SMN_C2PMSG_35_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_36                                                                            0x0064
+#define regMP1_SMN_C2PMSG_36_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_37                                                                            0x0065
+#define regMP1_SMN_C2PMSG_37_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_38                                                                            0x0066
+#define regMP1_SMN_C2PMSG_38_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_39                                                                            0x0067
+#define regMP1_SMN_C2PMSG_39_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_40                                                                            0x0068
+#define regMP1_SMN_C2PMSG_40_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_41                                                                            0x0069
+#define regMP1_SMN_C2PMSG_41_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_42                                                                            0x006a
+#define regMP1_SMN_C2PMSG_42_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_43                                                                            0x006b
+#define regMP1_SMN_C2PMSG_43_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_44                                                                            0x006c
+#define regMP1_SMN_C2PMSG_44_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_45                                                                            0x006d
+#define regMP1_SMN_C2PMSG_45_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_46                                                                            0x006e
+#define regMP1_SMN_C2PMSG_46_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_47                                                                            0x006f
+#define regMP1_SMN_C2PMSG_47_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_48                                                                            0x0070
+#define regMP1_SMN_C2PMSG_48_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_49                                                                            0x0071
+#define regMP1_SMN_C2PMSG_49_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_50                                                                            0x0072
+#define regMP1_SMN_C2PMSG_50_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_51                                                                            0x0073
+#define regMP1_SMN_C2PMSG_51_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_52                                                                            0x0074
+#define regMP1_SMN_C2PMSG_52_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_53                                                                            0x0075
+#define regMP1_SMN_C2PMSG_53_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_54                                                                            0x0076
+#define regMP1_SMN_C2PMSG_54_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_55                                                                            0x0077
+#define regMP1_SMN_C2PMSG_55_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_56                                                                            0x0078
+#define regMP1_SMN_C2PMSG_56_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_57                                                                            0x0079
+#define regMP1_SMN_C2PMSG_57_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_58                                                                            0x007a
+#define regMP1_SMN_C2PMSG_58_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_59                                                                            0x007b
+#define regMP1_SMN_C2PMSG_59_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_60                                                                            0x007c
+#define regMP1_SMN_C2PMSG_60_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_61                                                                            0x007d
+#define regMP1_SMN_C2PMSG_61_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_62                                                                            0x007e
+#define regMP1_SMN_C2PMSG_62_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_63                                                                            0x007f
+#define regMP1_SMN_C2PMSG_63_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_64                                                                            0x0080
+#define regMP1_SMN_C2PMSG_64_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_65                                                                            0x0081
+#define regMP1_SMN_C2PMSG_65_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_66                                                                            0x0082
+#define regMP1_SMN_C2PMSG_66_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_67                                                                            0x0083
+#define regMP1_SMN_C2PMSG_67_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_68                                                                            0x0084
+#define regMP1_SMN_C2PMSG_68_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_69                                                                            0x0085
+#define regMP1_SMN_C2PMSG_69_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_70                                                                            0x0086
+#define regMP1_SMN_C2PMSG_70_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_71                                                                            0x0087
+#define regMP1_SMN_C2PMSG_71_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_72                                                                            0x0088
+#define regMP1_SMN_C2PMSG_72_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_73                                                                            0x0089
+#define regMP1_SMN_C2PMSG_73_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_74                                                                            0x008a
+#define regMP1_SMN_C2PMSG_74_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_75                                                                            0x008b
+#define regMP1_SMN_C2PMSG_75_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_76                                                                            0x008c
+#define regMP1_SMN_C2PMSG_76_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_77                                                                            0x008d
+#define regMP1_SMN_C2PMSG_77_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_78                                                                            0x008e
+#define regMP1_SMN_C2PMSG_78_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_79                                                                            0x008f
+#define regMP1_SMN_C2PMSG_79_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_80                                                                            0x0090
+#define regMP1_SMN_C2PMSG_80_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_81                                                                            0x0091
+#define regMP1_SMN_C2PMSG_81_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_82                                                                            0x0092
+#define regMP1_SMN_C2PMSG_82_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_83                                                                            0x0093
+#define regMP1_SMN_C2PMSG_83_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_84                                                                            0x0094
+#define regMP1_SMN_C2PMSG_84_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_85                                                                            0x0095
+#define regMP1_SMN_C2PMSG_85_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_86                                                                            0x0096
+#define regMP1_SMN_C2PMSG_86_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_87                                                                            0x0097
+#define regMP1_SMN_C2PMSG_87_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_88                                                                            0x0098
+#define regMP1_SMN_C2PMSG_88_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_89                                                                            0x0099
+#define regMP1_SMN_C2PMSG_89_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_90                                                                            0x009a
+#define regMP1_SMN_C2PMSG_90_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_91                                                                            0x009b
+#define regMP1_SMN_C2PMSG_91_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_92                                                                            0x009c
+#define regMP1_SMN_C2PMSG_92_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_93                                                                            0x009d
+#define regMP1_SMN_C2PMSG_93_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_94                                                                            0x009e
+#define regMP1_SMN_C2PMSG_94_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_95                                                                            0x009f
+#define regMP1_SMN_C2PMSG_95_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_96                                                                            0x00a0
+#define regMP1_SMN_C2PMSG_96_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_97                                                                            0x00a1
+#define regMP1_SMN_C2PMSG_97_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_98                                                                            0x00a2
+#define regMP1_SMN_C2PMSG_98_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_99                                                                            0x00a3
+#define regMP1_SMN_C2PMSG_99_BASE_IDX                                                                   2
+#define regMP1_SMN_C2PMSG_100                                                                           0x00a4
+#define regMP1_SMN_C2PMSG_100_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_101                                                                           0x00a5
+#define regMP1_SMN_C2PMSG_101_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_102                                                                           0x00a6
+#define regMP1_SMN_C2PMSG_102_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_103                                                                           0x00a7
+#define regMP1_SMN_C2PMSG_103_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_104                                                                           0x00a8
+#define regMP1_SMN_C2PMSG_104_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_105                                                                           0x00a9
+#define regMP1_SMN_C2PMSG_105_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_106                                                                           0x00aa
+#define regMP1_SMN_C2PMSG_106_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_107                                                                           0x00ab
+#define regMP1_SMN_C2PMSG_107_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_108                                                                           0x00ac
+#define regMP1_SMN_C2PMSG_108_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_109                                                                           0x00ad
+#define regMP1_SMN_C2PMSG_109_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_110                                                                           0x00ae
+#define regMP1_SMN_C2PMSG_110_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_111                                                                           0x00af
+#define regMP1_SMN_C2PMSG_111_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_112                                                                           0x00b0
+#define regMP1_SMN_C2PMSG_112_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_113                                                                           0x00b1
+#define regMP1_SMN_C2PMSG_113_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_114                                                                           0x00b2
+#define regMP1_SMN_C2PMSG_114_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_115                                                                           0x00b3
+#define regMP1_SMN_C2PMSG_115_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_116                                                                           0x00b4
+#define regMP1_SMN_C2PMSG_116_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_117                                                                           0x00b5
+#define regMP1_SMN_C2PMSG_117_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_118                                                                           0x00b6
+#define regMP1_SMN_C2PMSG_118_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_119                                                                           0x00b7
+#define regMP1_SMN_C2PMSG_119_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_120                                                                           0x00b8
+#define regMP1_SMN_C2PMSG_120_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_121                                                                           0x00b9
+#define regMP1_SMN_C2PMSG_121_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_122                                                                           0x00ba
+#define regMP1_SMN_C2PMSG_122_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_123                                                                           0x00bb
+#define regMP1_SMN_C2PMSG_123_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_124                                                                           0x00bc
+#define regMP1_SMN_C2PMSG_124_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_125                                                                           0x00bd
+#define regMP1_SMN_C2PMSG_125_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_126                                                                           0x00be
+#define regMP1_SMN_C2PMSG_126_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_127                                                                           0x00bf
+#define regMP1_SMN_C2PMSG_127_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_128                                                                           0x00c0
+#define regMP1_SMN_C2PMSG_128_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_129                                                                           0x00c1
+#define regMP1_SMN_C2PMSG_129_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_130                                                                           0x00c2
+#define regMP1_SMN_C2PMSG_130_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_131                                                                           0x00c3
+#define regMP1_SMN_C2PMSG_131_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_132                                                                           0x00c4
+#define regMP1_SMN_C2PMSG_132_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_133                                                                           0x00c5
+#define regMP1_SMN_C2PMSG_133_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_134                                                                           0x00c6
+#define regMP1_SMN_C2PMSG_134_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_135                                                                           0x00c7
+#define regMP1_SMN_C2PMSG_135_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_136                                                                           0x00c8
+#define regMP1_SMN_C2PMSG_136_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_137                                                                           0x00c9
+#define regMP1_SMN_C2PMSG_137_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_138                                                                           0x00ca
+#define regMP1_SMN_C2PMSG_138_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_139                                                                           0x00cb
+#define regMP1_SMN_C2PMSG_139_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_140                                                                           0x00cc
+#define regMP1_SMN_C2PMSG_140_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_141                                                                           0x00cd
+#define regMP1_SMN_C2PMSG_141_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_142                                                                           0x00ce
+#define regMP1_SMN_C2PMSG_142_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_143                                                                           0x00cf
+#define regMP1_SMN_C2PMSG_143_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_144                                                                           0x00d0
+#define regMP1_SMN_C2PMSG_144_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_145                                                                           0x00d1
+#define regMP1_SMN_C2PMSG_145_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_146                                                                           0x00d2
+#define regMP1_SMN_C2PMSG_146_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_147                                                                           0x00d3
+#define regMP1_SMN_C2PMSG_147_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_148                                                                           0x00d4
+#define regMP1_SMN_C2PMSG_148_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_149                                                                           0x00d5
+#define regMP1_SMN_C2PMSG_149_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_150                                                                           0x00d6
+#define regMP1_SMN_C2PMSG_150_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_151                                                                           0x00d7
+#define regMP1_SMN_C2PMSG_151_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_152                                                                           0x00d8
+#define regMP1_SMN_C2PMSG_152_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_153                                                                           0x00d9
+#define regMP1_SMN_C2PMSG_153_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_154                                                                           0x00da
+#define regMP1_SMN_C2PMSG_154_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_155                                                                           0x00db
+#define regMP1_SMN_C2PMSG_155_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_156                                                                           0x00dc
+#define regMP1_SMN_C2PMSG_156_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_157                                                                           0x00dd
+#define regMP1_SMN_C2PMSG_157_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_158                                                                           0x00de
+#define regMP1_SMN_C2PMSG_158_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_159                                                                           0x00df
+#define regMP1_SMN_C2PMSG_159_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_160                                                                           0x00e0
+#define regMP1_SMN_C2PMSG_160_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_161                                                                           0x00e1
+#define regMP1_SMN_C2PMSG_161_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_162                                                                           0x00e2
+#define regMP1_SMN_C2PMSG_162_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_163                                                                           0x00e3
+#define regMP1_SMN_C2PMSG_163_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_164                                                                           0x00e4
+#define regMP1_SMN_C2PMSG_164_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_165                                                                           0x00e5
+#define regMP1_SMN_C2PMSG_165_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_166                                                                           0x00e6
+#define regMP1_SMN_C2PMSG_166_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_167                                                                           0x00e7
+#define regMP1_SMN_C2PMSG_167_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_168                                                                           0x00e8
+#define regMP1_SMN_C2PMSG_168_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_169                                                                           0x00e9
+#define regMP1_SMN_C2PMSG_169_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_170                                                                           0x00ea
+#define regMP1_SMN_C2PMSG_170_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_171                                                                           0x00eb
+#define regMP1_SMN_C2PMSG_171_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_172                                                                           0x00ec
+#define regMP1_SMN_C2PMSG_172_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_173                                                                           0x00ed
+#define regMP1_SMN_C2PMSG_173_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_174                                                                           0x00ee
+#define regMP1_SMN_C2PMSG_174_BASE_IDX                                                                  2
+#define regMP1_SMN_C2PMSG_175                                                                           0x00ef
+#define regMP1_SMN_C2PMSG_175_BASE_IDX                                                                  2
+#define regMP1_SMN_IH_CREDIT                                                                            0x0140
+#define regMP1_SMN_IH_CREDIT_BASE_IDX                                                                   2
+#define regMP1_SMN_IH_SW_INT                                                                            0x0141
+#define regMP1_SMN_IH_SW_INT_BASE_IDX                                                                   2
+#define regMP1_SMN_IH_SW_INT_CTRL                                                                       0x0142
+#define regMP1_SMN_IH_SW_INT_CTRL_BASE_IDX                                                              2
+#define regMP1_SMN_FPS_CNT                                                                              0x0143
+#define regMP1_SMN_FPS_CNT_BASE_IDX                                                                     2
+#define regMP1_SMN_PUB_CTRL                                                                             0x0144
+#define regMP1_SMN_PUB_CTRL_BASE_IDX                                                                    2
+#define regMP1_SMN_EXT_SCRATCH0                                                                         0x01c0
+#define regMP1_SMN_EXT_SCRATCH0_BASE_IDX                                                                2
+#define regMP1_SMN_EXT_SCRATCH1                                                                         0x01c1
+#define regMP1_SMN_EXT_SCRATCH1_BASE_IDX                                                                2
+#define regMP1_SMN_EXT_SCRATCH2                                                                         0x01c2
+#define regMP1_SMN_EXT_SCRATCH2_BASE_IDX                                                                2
+#define regMP1_SMN_EXT_SCRATCH3                                                                         0x01c3
+#define regMP1_SMN_EXT_SCRATCH3_BASE_IDX                                                                2
+#define regMP1_SMN_EXT_SCRATCH4                                                                         0x01c4
+#define regMP1_SMN_EXT_SCRATCH4_BASE_IDX                                                                2
+#define regMP1_SMN_EXT_SCRATCH5                                                                         0x01c5
+#define regMP1_SMN_EXT_SCRATCH5_BASE_IDX                                                                2
+#define regMP1_SMN_EXT_SCRATCH6                                                                         0x01c6
+#define regMP1_SMN_EXT_SCRATCH6_BASE_IDX                                                                2
+#define regMP1_SMN_EXT_SCRATCH7                                                                         0x01c7
+#define regMP1_SMN_EXT_SCRATCH7_BASE_IDX                                                                2
+#define regMP1_SMN_EXT_SCRATCH8                                                                         0x01c8
+#define regMP1_SMN_EXT_SCRATCH8_BASE_IDX                                                                2
+#define regMP1_SMN_EXT_SCRATCH9                                                                         0x01c9
+#define regMP1_SMN_EXT_SCRATCH9_BASE_IDX                                                                2
+#define regMP1_SMN_EXT_SCRATCH10                                                                        0x01ca
+#define regMP1_SMN_EXT_SCRATCH10_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH11                                                                        0x01cb
+#define regMP1_SMN_EXT_SCRATCH11_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH12                                                                        0x01cc
+#define regMP1_SMN_EXT_SCRATCH12_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH13                                                                        0x01cd
+#define regMP1_SMN_EXT_SCRATCH13_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH14                                                                        0x01ce
+#define regMP1_SMN_EXT_SCRATCH14_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH15                                                                        0x01cf
+#define regMP1_SMN_EXT_SCRATCH15_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH16                                                                        0x01d0
+#define regMP1_SMN_EXT_SCRATCH16_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH17                                                                        0x01d1
+#define regMP1_SMN_EXT_SCRATCH17_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH18                                                                        0x01d2
+#define regMP1_SMN_EXT_SCRATCH18_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH19                                                                        0x01d3
+#define regMP1_SMN_EXT_SCRATCH19_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH20                                                                        0x01d4
+#define regMP1_SMN_EXT_SCRATCH20_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH21                                                                        0x01d5
+#define regMP1_SMN_EXT_SCRATCH21_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH22                                                                        0x01d6
+#define regMP1_SMN_EXT_SCRATCH22_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH23                                                                        0x01d7
+#define regMP1_SMN_EXT_SCRATCH23_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH24                                                                        0x01d8
+#define regMP1_SMN_EXT_SCRATCH24_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH25                                                                        0x01d9
+#define regMP1_SMN_EXT_SCRATCH25_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH26                                                                        0x01da
+#define regMP1_SMN_EXT_SCRATCH26_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH27                                                                        0x01db
+#define regMP1_SMN_EXT_SCRATCH27_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH28                                                                        0x01dc
+#define regMP1_SMN_EXT_SCRATCH28_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH29                                                                        0x01dd
+#define regMP1_SMN_EXT_SCRATCH29_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH30                                                                        0x01de
+#define regMP1_SMN_EXT_SCRATCH30_BASE_IDX                                                               2
+#define regMP1_SMN_EXT_SCRATCH31                                                                        0x01df
+#define regMP1_SMN_EXT_SCRATCH31_BASE_IDX                                                               2
+
+
+// addressBlock: mp_SmuMpASP_SmnDec
+// base address: 0x0
+#define regMPASP_SMN_C2PMSG_0                                                                           0x0040
+#define regMPASP_SMN_C2PMSG_0_BASE_IDX                                                                  1
+#define regMPASP_SMN_C2PMSG_1                                                                           0x0041
+#define regMPASP_SMN_C2PMSG_1_BASE_IDX                                                                  1
+#define regMPASP_SMN_C2PMSG_2                                                                           0x0042
+#define regMPASP_SMN_C2PMSG_2_BASE_IDX                                                                  1
+#define regMPASP_SMN_C2PMSG_3                                                                           0x0043
+#define regMPASP_SMN_C2PMSG_3_BASE_IDX                                                                  1
+#define regMPASP_SMN_C2PMSG_4                                                                           0x0044
+#define regMPASP_SMN_C2PMSG_4_BASE_IDX                                                                  1
+#define regMPASP_SMN_C2PMSG_5                                                                           0x0045
+#define regMPASP_SMN_C2PMSG_5_BASE_IDX                                                                  1
+#define regMPASP_SMN_C2PMSG_6                                                                           0x0046
+#define regMPASP_SMN_C2PMSG_6_BASE_IDX                                                                  1
+#define regMPASP_SMN_C2PMSG_7                                                                           0x0047
+#define regMPASP_SMN_C2PMSG_7_BASE_IDX                                                                  1
+#define regMPASP_SMN_C2PMSG_8                                                                           0x0048
+#define regMPASP_SMN_C2PMSG_8_BASE_IDX                                                                  1
+#define regMPASP_SMN_C2PMSG_9                                                                           0x0049
+#define regMPASP_SMN_C2PMSG_9_BASE_IDX                                                                  1
+#define regMPASP_SMN_C2PMSG_10                                                                          0x004a
+#define regMPASP_SMN_C2PMSG_10_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_11                                                                          0x004b
+#define regMPASP_SMN_C2PMSG_11_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_12                                                                          0x004c
+#define regMPASP_SMN_C2PMSG_12_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_13                                                                          0x004d
+#define regMPASP_SMN_C2PMSG_13_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_14                                                                          0x004e
+#define regMPASP_SMN_C2PMSG_14_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_15                                                                          0x004f
+#define regMPASP_SMN_C2PMSG_15_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_16                                                                          0x0050
+#define regMPASP_SMN_C2PMSG_16_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_17                                                                          0x0051
+#define regMPASP_SMN_C2PMSG_17_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_18                                                                          0x0052
+#define regMPASP_SMN_C2PMSG_18_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_19                                                                          0x0053
+#define regMPASP_SMN_C2PMSG_19_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_20                                                                          0x0054
+#define regMPASP_SMN_C2PMSG_20_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_21                                                                          0x0055
+#define regMPASP_SMN_C2PMSG_21_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_22                                                                          0x0056
+#define regMPASP_SMN_C2PMSG_22_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_23                                                                          0x0057
+#define regMPASP_SMN_C2PMSG_23_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_24                                                                          0x0058
+#define regMPASP_SMN_C2PMSG_24_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_25                                                                          0x0059
+#define regMPASP_SMN_C2PMSG_25_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_26                                                                          0x005a
+#define regMPASP_SMN_C2PMSG_26_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_27                                                                          0x005b
+#define regMPASP_SMN_C2PMSG_27_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_28                                                                          0x005c
+#define regMPASP_SMN_C2PMSG_28_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_29                                                                          0x005d
+#define regMPASP_SMN_C2PMSG_29_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_30                                                                          0x005e
+#define regMPASP_SMN_C2PMSG_30_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_31                                                                          0x005f
+#define regMPASP_SMN_C2PMSG_31_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_32                                                                          0x0060
+#define regMPASP_SMN_C2PMSG_32_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_33                                                                          0x0061
+#define regMPASP_SMN_C2PMSG_33_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_34                                                                          0x0062
+#define regMPASP_SMN_C2PMSG_34_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_35                                                                          0x0063
+#define regMPASP_SMN_C2PMSG_35_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_36                                                                          0x0064
+#define regMPASP_SMN_C2PMSG_36_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_37                                                                          0x0065
+#define regMPASP_SMN_C2PMSG_37_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_38                                                                          0x0066
+#define regMPASP_SMN_C2PMSG_38_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_39                                                                          0x0067
+#define regMPASP_SMN_C2PMSG_39_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_40                                                                          0x0068
+#define regMPASP_SMN_C2PMSG_40_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_41                                                                          0x0069
+#define regMPASP_SMN_C2PMSG_41_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_42                                                                          0x006a
+#define regMPASP_SMN_C2PMSG_42_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_43                                                                          0x006b
+#define regMPASP_SMN_C2PMSG_43_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_44                                                                          0x006c
+#define regMPASP_SMN_C2PMSG_44_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_45                                                                          0x006d
+#define regMPASP_SMN_C2PMSG_45_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_46                                                                          0x006e
+#define regMPASP_SMN_C2PMSG_46_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_47                                                                          0x006f
+#define regMPASP_SMN_C2PMSG_47_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_48                                                                          0x0070
+#define regMPASP_SMN_C2PMSG_48_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_49                                                                          0x0071
+#define regMPASP_SMN_C2PMSG_49_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_50                                                                          0x0072
+#define regMPASP_SMN_C2PMSG_50_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_51                                                                          0x0073
+#define regMPASP_SMN_C2PMSG_51_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_52                                                                          0x0074
+#define regMPASP_SMN_C2PMSG_52_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_53                                                                          0x0075
+#define regMPASP_SMN_C2PMSG_53_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_54                                                                          0x0076
+#define regMPASP_SMN_C2PMSG_54_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_55                                                                          0x0077
+#define regMPASP_SMN_C2PMSG_55_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_56                                                                          0x0078
+#define regMPASP_SMN_C2PMSG_56_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_57                                                                          0x0079
+#define regMPASP_SMN_C2PMSG_57_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_58                                                                          0x007a
+#define regMPASP_SMN_C2PMSG_58_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_59                                                                          0x007b
+#define regMPASP_SMN_C2PMSG_59_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_60                                                                          0x007c
+#define regMPASP_SMN_C2PMSG_60_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_61                                                                          0x007d
+#define regMPASP_SMN_C2PMSG_61_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_62                                                                          0x007e
+#define regMPASP_SMN_C2PMSG_62_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_63                                                                          0x007f
+#define regMPASP_SMN_C2PMSG_63_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_64                                                                          0x0080
+#define regMPASP_SMN_C2PMSG_64_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_65                                                                          0x0081
+#define regMPASP_SMN_C2PMSG_65_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_66                                                                          0x0082
+#define regMPASP_SMN_C2PMSG_66_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_67                                                                          0x0083
+#define regMPASP_SMN_C2PMSG_67_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_68                                                                          0x0084
+#define regMPASP_SMN_C2PMSG_68_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_69                                                                          0x0085
+#define regMPASP_SMN_C2PMSG_69_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_70                                                                          0x0086
+#define regMPASP_SMN_C2PMSG_70_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_71                                                                          0x0087
+#define regMPASP_SMN_C2PMSG_71_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_72                                                                          0x0088
+#define regMPASP_SMN_C2PMSG_72_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_73                                                                          0x0089
+#define regMPASP_SMN_C2PMSG_73_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_74                                                                          0x008a
+#define regMPASP_SMN_C2PMSG_74_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_75                                                                          0x008b
+#define regMPASP_SMN_C2PMSG_75_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_76                                                                          0x008c
+#define regMPASP_SMN_C2PMSG_76_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_77                                                                          0x008d
+#define regMPASP_SMN_C2PMSG_77_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_78                                                                          0x008e
+#define regMPASP_SMN_C2PMSG_78_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_79                                                                          0x008f
+#define regMPASP_SMN_C2PMSG_79_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_80                                                                          0x0090
+#define regMPASP_SMN_C2PMSG_80_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_81                                                                          0x0091
+#define regMPASP_SMN_C2PMSG_81_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_82                                                                          0x0092
+#define regMPASP_SMN_C2PMSG_82_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_83                                                                          0x0093
+#define regMPASP_SMN_C2PMSG_83_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_84                                                                          0x0094
+#define regMPASP_SMN_C2PMSG_84_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_85                                                                          0x0095
+#define regMPASP_SMN_C2PMSG_85_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_86                                                                          0x0096
+#define regMPASP_SMN_C2PMSG_86_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_87                                                                          0x0097
+#define regMPASP_SMN_C2PMSG_87_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_88                                                                          0x0098
+#define regMPASP_SMN_C2PMSG_88_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_89                                                                          0x0099
+#define regMPASP_SMN_C2PMSG_89_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_90                                                                          0x009a
+#define regMPASP_SMN_C2PMSG_90_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_91                                                                          0x009b
+#define regMPASP_SMN_C2PMSG_91_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_92                                                                          0x009c
+#define regMPASP_SMN_C2PMSG_92_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_93                                                                          0x009d
+#define regMPASP_SMN_C2PMSG_93_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_94                                                                          0x009e
+#define regMPASP_SMN_C2PMSG_94_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_95                                                                          0x009f
+#define regMPASP_SMN_C2PMSG_95_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_96                                                                          0x00a0
+#define regMPASP_SMN_C2PMSG_96_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_97                                                                          0x00a1
+#define regMPASP_SMN_C2PMSG_97_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_98                                                                          0x00a2
+#define regMPASP_SMN_C2PMSG_98_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_99                                                                          0x00a3
+#define regMPASP_SMN_C2PMSG_99_BASE_IDX                                                                 1
+#define regMPASP_SMN_C2PMSG_100                                                                         0x00a4
+#define regMPASP_SMN_C2PMSG_100_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_101                                                                         0x00a5
+#define regMPASP_SMN_C2PMSG_101_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_102                                                                         0x00a6
+#define regMPASP_SMN_C2PMSG_102_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_103                                                                         0x00a7
+#define regMPASP_SMN_C2PMSG_103_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_104                                                                         0x00a8
+#define regMPASP_SMN_C2PMSG_104_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_105                                                                         0x00a9
+#define regMPASP_SMN_C2PMSG_105_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_106                                                                         0x00aa
+#define regMPASP_SMN_C2PMSG_106_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_107                                                                         0x00ab
+#define regMPASP_SMN_C2PMSG_107_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_108                                                                         0x00ac
+#define regMPASP_SMN_C2PMSG_108_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_109                                                                         0x00ad
+#define regMPASP_SMN_C2PMSG_109_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_110                                                                         0x00ae
+#define regMPASP_SMN_C2PMSG_110_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_111                                                                         0x00af
+#define regMPASP_SMN_C2PMSG_111_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_112                                                                         0x00b0
+#define regMPASP_SMN_C2PMSG_112_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_113                                                                         0x00b1
+#define regMPASP_SMN_C2PMSG_113_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_114                                                                         0x00b2
+#define regMPASP_SMN_C2PMSG_114_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_115                                                                         0x00b3
+#define regMPASP_SMN_C2PMSG_115_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_116                                                                         0x00b4
+#define regMPASP_SMN_C2PMSG_116_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_117                                                                         0x00b5
+#define regMPASP_SMN_C2PMSG_117_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_118                                                                         0x00b6
+#define regMPASP_SMN_C2PMSG_118_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_119                                                                         0x00b7
+#define regMPASP_SMN_C2PMSG_119_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_120                                                                         0x00b8
+#define regMPASP_SMN_C2PMSG_120_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_121                                                                         0x00b9
+#define regMPASP_SMN_C2PMSG_121_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_122                                                                         0x00ba
+#define regMPASP_SMN_C2PMSG_122_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_123                                                                         0x00bb
+#define regMPASP_SMN_C2PMSG_123_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_124                                                                         0x00bc
+#define regMPASP_SMN_C2PMSG_124_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_125                                                                         0x00bd
+#define regMPASP_SMN_C2PMSG_125_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_126                                                                         0x00be
+#define regMPASP_SMN_C2PMSG_126_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_127                                                                         0x00bf
+#define regMPASP_SMN_C2PMSG_127_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_128                                                                         0x00c0
+#define regMPASP_SMN_C2PMSG_128_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_129                                                                         0x00c1
+#define regMPASP_SMN_C2PMSG_129_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_130                                                                         0x00c2
+#define regMPASP_SMN_C2PMSG_130_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_131                                                                         0x00c3
+#define regMPASP_SMN_C2PMSG_131_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_132                                                                         0x00c4
+#define regMPASP_SMN_C2PMSG_132_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_133                                                                         0x00c5
+#define regMPASP_SMN_C2PMSG_133_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_134                                                                         0x00c6
+#define regMPASP_SMN_C2PMSG_134_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_135                                                                         0x00c7
+#define regMPASP_SMN_C2PMSG_135_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_136                                                                         0x00c8
+#define regMPASP_SMN_C2PMSG_136_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_137                                                                         0x00c9
+#define regMPASP_SMN_C2PMSG_137_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_138                                                                         0x00ca
+#define regMPASP_SMN_C2PMSG_138_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_139                                                                         0x00cb
+#define regMPASP_SMN_C2PMSG_139_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_140                                                                         0x00cc
+#define regMPASP_SMN_C2PMSG_140_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_141                                                                         0x00cd
+#define regMPASP_SMN_C2PMSG_141_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_142                                                                         0x00ce
+#define regMPASP_SMN_C2PMSG_142_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_143                                                                         0x00cf
+#define regMPASP_SMN_C2PMSG_143_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_144                                                                         0x00d0
+#define regMPASP_SMN_C2PMSG_144_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_145                                                                         0x00d1
+#define regMPASP_SMN_C2PMSG_145_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_146                                                                         0x00d2
+#define regMPASP_SMN_C2PMSG_146_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_147                                                                         0x00d3
+#define regMPASP_SMN_C2PMSG_147_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_148                                                                         0x00d4
+#define regMPASP_SMN_C2PMSG_148_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_149                                                                         0x00d5
+#define regMPASP_SMN_C2PMSG_149_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_150                                                                         0x00d6
+#define regMPASP_SMN_C2PMSG_150_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_151                                                                         0x00d7
+#define regMPASP_SMN_C2PMSG_151_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_152                                                                         0x00d8
+#define regMPASP_SMN_C2PMSG_152_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_153                                                                         0x00d9
+#define regMPASP_SMN_C2PMSG_153_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_154                                                                         0x00da
+#define regMPASP_SMN_C2PMSG_154_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_155                                                                         0x00db
+#define regMPASP_SMN_C2PMSG_155_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_156                                                                         0x00dc
+#define regMPASP_SMN_C2PMSG_156_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_157                                                                         0x00dd
+#define regMPASP_SMN_C2PMSG_157_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_158                                                                         0x00de
+#define regMPASP_SMN_C2PMSG_158_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_159                                                                         0x00df
+#define regMPASP_SMN_C2PMSG_159_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_160                                                                         0x00e0
+#define regMPASP_SMN_C2PMSG_160_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_161                                                                         0x00e1
+#define regMPASP_SMN_C2PMSG_161_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_162                                                                         0x00e2
+#define regMPASP_SMN_C2PMSG_162_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_163                                                                         0x00e3
+#define regMPASP_SMN_C2PMSG_163_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_164                                                                         0x00e4
+#define regMPASP_SMN_C2PMSG_164_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_165                                                                         0x00e5
+#define regMPASP_SMN_C2PMSG_165_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_166                                                                         0x00e6
+#define regMPASP_SMN_C2PMSG_166_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_167                                                                         0x00e7
+#define regMPASP_SMN_C2PMSG_167_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_168                                                                         0x00e8
+#define regMPASP_SMN_C2PMSG_168_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_169                                                                         0x00e9
+#define regMPASP_SMN_C2PMSG_169_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_170                                                                         0x00ea
+#define regMPASP_SMN_C2PMSG_170_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_171                                                                         0x00eb
+#define regMPASP_SMN_C2PMSG_171_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_172                                                                         0x00ec
+#define regMPASP_SMN_C2PMSG_172_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_173                                                                         0x00ed
+#define regMPASP_SMN_C2PMSG_173_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_174                                                                         0x00ee
+#define regMPASP_SMN_C2PMSG_174_BASE_IDX                                                                1
+#define regMPASP_SMN_C2PMSG_175                                                                         0x00ef
+#define regMPASP_SMN_C2PMSG_175_BASE_IDX                                                                1
+#define regMPASP_SMN_IH_CREDIT                                                                          0x0140
+#define regMPASP_SMN_IH_CREDIT_BASE_IDX                                                                 1
+#define regMPASP_SMN_IH_SW_INT                                                                          0x0141
+#define regMPASP_SMN_IH_SW_INT_BASE_IDX                                                                 1
+#define regMPASP_SMN_IH_SW_INT_CTRL                                                                     0x0142
+#define regMPASP_SMN_IH_SW_INT_CTRL_BASE_IDX                                                            1
+
+
+// addressBlock: mp_SmuMpRASPub_CruDec
+// base address: 0x0
+#define regMPRAS_CRU0_MPRAS_FIRMWARE_FLAGS                                                              0xbeb009
+#define regMPRAS_CRU0_MPRAS_FIRMWARE_FLAGS_BASE_IDX                                                     3
+
+
+// addressBlock: mp_SmuMpIFOEPub_CruDec
+// base address: 0x0
+#define regMPIFOE_CRU0_MPIFOE_FIRMWARE_FLAGS                                                            0xbeb009
+#define regMPIFOE_CRU0_MPIFOE_FIRMWARE_FLAGS_BASE_IDX                                                   3
+
+
+// addressBlock: mp_SmuMp1Pub_CruDec
+// base address: 0x0
+#define regMP1_CRU0_MP1_FIRMWARE_FLAGS                                                                  0xbeb009
+#define regMP1_CRU0_MP1_FIRMWARE_FLAGS_BASE_IDX                                                         3
+
+
+// addressBlock: mp_SmuMpIOPub_CruDec
+// base address: 0x0
+#define regMPIO_CRU0_MPIO_FIRMWARE_FLAGS                                                                0xbeb009
+#define regMPIO_CRU0_MPIO_FIRMWARE_FLAGS_BASE_IDX                                                       3
+
+
+// addressBlock: MpRASMmioPublic_SmuMpRASPub_CruDec
+// base address: 0x3e00000
+#define regMPRAS_CRU1_MPRAS_FIRMWARE_FLAGS                                                              0x4009
+#define regMPRAS_CRU1_MPRAS_FIRMWARE_FLAGS_BASE_IDX                                                     9
+
+
+// addressBlock: MpIFoEMmioPublic_SmuMpIFOEPub_CruDec
+// base address: 0x49d00000
+#define regMPIFOE_CRU1_MPIFOE_FIRMWARE_FLAGS                                                            0x4009
+#define regMPIFOE_CRU1_MPIFOE_FIRMWARE_FLAGS_BASE_IDX                                                   26
+
+
+// addressBlock: Mp1MmioPublic_SmuMp1Pub_CruDec
+// base address: 0x3b00000
+#define regMP1_CRU1_MP1_FIRMWARE_FLAGS                                                                  0x4009
+#define regMP1_CRU1_MP1_FIRMWARE_FLAGS_BASE_IDX                                                         6
+
+
+// addressBlock: MpIOMmioPublic_SmuMpIOPub_CruDec
+// base address: 0xc900000
+#define regMPIO_CRU1_MPIO_FIRMWARE_FLAGS                                                                0x4009
+#define regMPIO_CRU1_MPIO_FIRMWARE_FLAGS_BASE_IDX                                                       23
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/mp/mp_15_0_8_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/mp/mp_15_0_8_sh_mask.h
new file mode 100644
index 000000000000..5f4f883ac5a2
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/mp/mp_15_0_8_sh_mask.h
@@ -0,0 +1,616 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _mp_15_0_8_SH_MASK_HEADER
+#define _mp_15_0_8_SH_MASK_HEADER
+
+
+// addressBlock: mp_SmuMp1_SmnDec
+//MP1_SMN_C2PMSG_0
+#define MP1_SMN_C2PMSG_0__CONTENT__SHIFT                                                                      0x0
+#define MP1_SMN_C2PMSG_0__CONTENT_MASK                                                                        0xFFFFFFFFL
+//MP1_SMN_C2PMSG_1
+#define MP1_SMN_C2PMSG_1__CONTENT__SHIFT                                                                      0x0
+#define MP1_SMN_C2PMSG_1__CONTENT_MASK                                                                        0xFFFFFFFFL
+//MP1_SMN_C2PMSG_2
+#define MP1_SMN_C2PMSG_2__CONTENT__SHIFT                                                                      0x0
+#define MP1_SMN_C2PMSG_2__CONTENT_MASK                                                                        0xFFFFFFFFL
+//MP1_SMN_C2PMSG_3
+#define MP1_SMN_C2PMSG_3__CONTENT__SHIFT                                                                      0x0
+#define MP1_SMN_C2PMSG_3__CONTENT_MASK                                                                        0xFFFFFFFFL
+//MP1_SMN_C2PMSG_4
+#define MP1_SMN_C2PMSG_4__CONTENT__SHIFT                                                                      0x0
+#define MP1_SMN_C2PMSG_4__CONTENT_MASK                                                                        0xFFFFFFFFL
+//MP1_SMN_C2PMSG_5
+#define MP1_SMN_C2PMSG_5__CONTENT__SHIFT                                                                      0x0
+#define MP1_SMN_C2PMSG_5__CONTENT_MASK                                                                        0xFFFFFFFFL
+//MP1_SMN_C2PMSG_6
+#define MP1_SMN_C2PMSG_6__CONTENT__SHIFT                                                                      0x0
+#define MP1_SMN_C2PMSG_6__CONTENT_MASK                                                                        0xFFFFFFFFL
+//MP1_SMN_C2PMSG_7
+#define MP1_SMN_C2PMSG_7__CONTENT__SHIFT                                                                      0x0
+#define MP1_SMN_C2PMSG_7__CONTENT_MASK                                                                        0xFFFFFFFFL
+//MP1_SMN_C2PMSG_8
+#define MP1_SMN_C2PMSG_8__CONTENT__SHIFT                                                                      0x0
+#define MP1_SMN_C2PMSG_8__CONTENT_MASK                                                                        0xFFFFFFFFL
+//MP1_SMN_C2PMSG_9
+#define MP1_SMN_C2PMSG_9__CONTENT__SHIFT                                                                      0x0
+#define MP1_SMN_C2PMSG_9__CONTENT_MASK                                                                        0xFFFFFFFFL
+//MP1_SMN_C2PMSG_10
+#define MP1_SMN_C2PMSG_10__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_10__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_11
+#define MP1_SMN_C2PMSG_11__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_11__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_12
+#define MP1_SMN_C2PMSG_12__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_12__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_13
+#define MP1_SMN_C2PMSG_13__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_13__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_14
+#define MP1_SMN_C2PMSG_14__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_14__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_15
+#define MP1_SMN_C2PMSG_15__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_15__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_16
+#define MP1_SMN_C2PMSG_16__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_16__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_17
+#define MP1_SMN_C2PMSG_17__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_17__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_18
+#define MP1_SMN_C2PMSG_18__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_18__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_19
+#define MP1_SMN_C2PMSG_19__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_19__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_20
+#define MP1_SMN_C2PMSG_20__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_20__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_21
+#define MP1_SMN_C2PMSG_21__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_21__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_22
+#define MP1_SMN_C2PMSG_22__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_22__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_23
+#define MP1_SMN_C2PMSG_23__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_23__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_24
+#define MP1_SMN_C2PMSG_24__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_24__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_25
+#define MP1_SMN_C2PMSG_25__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_25__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_26
+#define MP1_SMN_C2PMSG_26__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_26__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_27
+#define MP1_SMN_C2PMSG_27__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_27__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_28
+#define MP1_SMN_C2PMSG_28__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_28__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_29
+#define MP1_SMN_C2PMSG_29__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_29__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_30
+#define MP1_SMN_C2PMSG_30__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_30__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_31
+#define MP1_SMN_C2PMSG_31__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_31__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_32
+#define MP1_SMN_C2PMSG_32__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_32__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_33
+#define MP1_SMN_C2PMSG_33__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_33__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_34
+#define MP1_SMN_C2PMSG_34__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_34__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_35
+#define MP1_SMN_C2PMSG_35__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_35__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_36
+#define MP1_SMN_C2PMSG_36__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_36__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_37
+#define MP1_SMN_C2PMSG_37__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_37__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_38
+#define MP1_SMN_C2PMSG_38__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_38__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_39
+#define MP1_SMN_C2PMSG_39__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_39__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_40
+#define MP1_SMN_C2PMSG_40__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_40__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_41
+#define MP1_SMN_C2PMSG_41__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_41__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_42
+#define MP1_SMN_C2PMSG_42__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_42__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_43
+#define MP1_SMN_C2PMSG_43__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_43__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_44
+#define MP1_SMN_C2PMSG_44__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_44__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_45
+#define MP1_SMN_C2PMSG_45__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_45__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_46
+#define MP1_SMN_C2PMSG_46__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_46__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_47
+#define MP1_SMN_C2PMSG_47__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_47__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_48
+#define MP1_SMN_C2PMSG_48__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_48__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_49
+#define MP1_SMN_C2PMSG_49__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_49__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_50
+#define MP1_SMN_C2PMSG_50__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_50__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_51
+#define MP1_SMN_C2PMSG_51__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_51__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_52
+#define MP1_SMN_C2PMSG_52__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_52__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_53
+#define MP1_SMN_C2PMSG_53__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_53__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_54
+#define MP1_SMN_C2PMSG_54__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_54__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_55
+#define MP1_SMN_C2PMSG_55__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_55__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_56
+#define MP1_SMN_C2PMSG_56__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_56__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_57
+#define MP1_SMN_C2PMSG_57__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_57__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_58
+#define MP1_SMN_C2PMSG_58__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_58__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_59
+#define MP1_SMN_C2PMSG_59__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_59__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_60
+#define MP1_SMN_C2PMSG_60__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_60__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_61
+#define MP1_SMN_C2PMSG_61__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_61__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_62
+#define MP1_SMN_C2PMSG_62__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_62__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_63
+#define MP1_SMN_C2PMSG_63__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_63__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_64
+#define MP1_SMN_C2PMSG_64__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_64__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_65
+#define MP1_SMN_C2PMSG_65__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_65__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_66
+#define MP1_SMN_C2PMSG_66__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_66__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_67
+#define MP1_SMN_C2PMSG_67__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_67__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_68
+#define MP1_SMN_C2PMSG_68__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_68__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_69
+#define MP1_SMN_C2PMSG_69__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_69__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_70
+#define MP1_SMN_C2PMSG_70__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_70__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_71
+#define MP1_SMN_C2PMSG_71__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_71__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_72
+#define MP1_SMN_C2PMSG_72__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_72__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_73
+#define MP1_SMN_C2PMSG_73__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_73__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_74
+#define MP1_SMN_C2PMSG_74__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_74__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_75
+#define MP1_SMN_C2PMSG_75__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_75__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_76
+#define MP1_SMN_C2PMSG_76__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_76__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_77
+#define MP1_SMN_C2PMSG_77__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_77__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_78
+#define MP1_SMN_C2PMSG_78__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_78__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_79
+#define MP1_SMN_C2PMSG_79__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_79__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_80
+#define MP1_SMN_C2PMSG_80__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_80__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_81
+#define MP1_SMN_C2PMSG_81__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_81__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_82
+#define MP1_SMN_C2PMSG_82__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_82__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_83
+#define MP1_SMN_C2PMSG_83__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_83__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_84
+#define MP1_SMN_C2PMSG_84__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_84__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_85
+#define MP1_SMN_C2PMSG_85__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_85__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_86
+#define MP1_SMN_C2PMSG_86__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_86__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_87
+#define MP1_SMN_C2PMSG_87__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_87__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_88
+#define MP1_SMN_C2PMSG_88__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_88__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_89
+#define MP1_SMN_C2PMSG_89__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_89__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_90
+#define MP1_SMN_C2PMSG_90__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_90__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_91
+#define MP1_SMN_C2PMSG_91__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_91__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_92
+#define MP1_SMN_C2PMSG_92__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_92__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_93
+#define MP1_SMN_C2PMSG_93__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_93__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_94
+#define MP1_SMN_C2PMSG_94__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_94__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_95
+#define MP1_SMN_C2PMSG_95__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_95__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_96
+#define MP1_SMN_C2PMSG_96__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_96__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_97
+#define MP1_SMN_C2PMSG_97__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_97__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_98
+#define MP1_SMN_C2PMSG_98__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_98__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_99
+#define MP1_SMN_C2PMSG_99__CONTENT__SHIFT                                                                     0x0
+#define MP1_SMN_C2PMSG_99__CONTENT_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_C2PMSG_100
+#define MP1_SMN_C2PMSG_100__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_100__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_101
+#define MP1_SMN_C2PMSG_101__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_101__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_102
+#define MP1_SMN_C2PMSG_102__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_102__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_103
+#define MP1_SMN_C2PMSG_103__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_103__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_104
+#define MP1_SMN_C2PMSG_104__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_104__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_105
+#define MP1_SMN_C2PMSG_105__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_105__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_106
+#define MP1_SMN_C2PMSG_106__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_106__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_107
+#define MP1_SMN_C2PMSG_107__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_107__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_108
+#define MP1_SMN_C2PMSG_108__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_108__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_109
+#define MP1_SMN_C2PMSG_109__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_109__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_110
+#define MP1_SMN_C2PMSG_110__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_110__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_111
+#define MP1_SMN_C2PMSG_111__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_111__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_112
+#define MP1_SMN_C2PMSG_112__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_112__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_113
+#define MP1_SMN_C2PMSG_113__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_113__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_114
+#define MP1_SMN_C2PMSG_114__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_114__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_115
+#define MP1_SMN_C2PMSG_115__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_115__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_116
+#define MP1_SMN_C2PMSG_116__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_116__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_117
+#define MP1_SMN_C2PMSG_117__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_117__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_118
+#define MP1_SMN_C2PMSG_118__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_118__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_119
+#define MP1_SMN_C2PMSG_119__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_119__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_120
+#define MP1_SMN_C2PMSG_120__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_120__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_121
+#define MP1_SMN_C2PMSG_121__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_121__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_122
+#define MP1_SMN_C2PMSG_122__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_122__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_123
+#define MP1_SMN_C2PMSG_123__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_123__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_124
+#define MP1_SMN_C2PMSG_124__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_124__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_125
+#define MP1_SMN_C2PMSG_125__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_125__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_126
+#define MP1_SMN_C2PMSG_126__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_126__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_C2PMSG_127
+#define MP1_SMN_C2PMSG_127__CONTENT__SHIFT                                                                    0x0
+#define MP1_SMN_C2PMSG_127__CONTENT_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_IH_CREDIT
+#define MP1_SMN_IH_CREDIT__CREDIT_VALUE__SHIFT                                                                0x0
+#define MP1_SMN_IH_CREDIT__CLIENT_ID__SHIFT                                                                   0x10
+#define MP1_SMN_IH_CREDIT__CREDIT_VALUE_MASK                                                                  0x00000003L
+#define MP1_SMN_IH_CREDIT__CLIENT_ID_MASK                                                                     0x00FF0000L
+//MP1_SMN_IH_SW_INT
+#define MP1_SMN_IH_SW_INT__ID__SHIFT                                                                          0x0
+#define MP1_SMN_IH_SW_INT__VALID__SHIFT                                                                       0x8
+#define MP1_SMN_IH_SW_INT__ID_MASK                                                                            0x000000FFL
+#define MP1_SMN_IH_SW_INT__VALID_MASK                                                                         0x00000100L
+//MP1_SMN_IH_SW_INT_CTRL
+#define MP1_SMN_IH_SW_INT_CTRL__INT_MASK__SHIFT                                                               0x0
+#define MP1_SMN_IH_SW_INT_CTRL__INT_ACK__SHIFT                                                                0x8
+#define MP1_SMN_IH_SW_INT_CTRL__INT_MASK_MASK                                                                 0x00000001L
+#define MP1_SMN_IH_SW_INT_CTRL__INT_ACK_MASK                                                                  0x00000100L
+//MP1_SMN_FPS_CNT
+#define MP1_SMN_FPS_CNT__COUNT__SHIFT                                                                         0x0
+#define MP1_SMN_FPS_CNT__COUNT_MASK                                                                           0xFFFFFFFFL
+//MP1_SMN_PUB_CTRL
+#define MP1_SMN_PUB_CTRL__LX3_RESET__SHIFT                                                                    0x0
+#define MP1_SMN_PUB_CTRL__LX3_RESET_MASK                                                                      0x00000001L
+//MP1_SMN_EXT_SCRATCH0
+#define MP1_SMN_EXT_SCRATCH0__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH0__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH1
+#define MP1_SMN_EXT_SCRATCH1__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH1__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH2
+#define MP1_SMN_EXT_SCRATCH2__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH2__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH3
+#define MP1_SMN_EXT_SCRATCH3__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH3__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH4
+#define MP1_SMN_EXT_SCRATCH4__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH4__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH5
+#define MP1_SMN_EXT_SCRATCH5__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH5__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH6
+#define MP1_SMN_EXT_SCRATCH6__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH6__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH7
+#define MP1_SMN_EXT_SCRATCH7__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH7__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH8
+#define MP1_SMN_EXT_SCRATCH8__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH8__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH9
+#define MP1_SMN_EXT_SCRATCH9__DATA__SHIFT                                                                     0x0
+#define MP1_SMN_EXT_SCRATCH9__DATA_MASK                                                                       0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH10
+#define MP1_SMN_EXT_SCRATCH10__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH10__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH11
+#define MP1_SMN_EXT_SCRATCH11__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH11__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH12
+#define MP1_SMN_EXT_SCRATCH12__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH12__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH13
+#define MP1_SMN_EXT_SCRATCH13__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH13__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH14
+#define MP1_SMN_EXT_SCRATCH14__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH14__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH15
+#define MP1_SMN_EXT_SCRATCH15__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH15__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH16
+#define MP1_SMN_EXT_SCRATCH16__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH16__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH17
+#define MP1_SMN_EXT_SCRATCH17__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH17__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH18
+#define MP1_SMN_EXT_SCRATCH18__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH18__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH19
+#define MP1_SMN_EXT_SCRATCH19__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH19__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH20
+#define MP1_SMN_EXT_SCRATCH20__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH20__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH21
+#define MP1_SMN_EXT_SCRATCH21__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH21__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH22
+#define MP1_SMN_EXT_SCRATCH22__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH22__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH23
+#define MP1_SMN_EXT_SCRATCH23__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH23__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH24
+#define MP1_SMN_EXT_SCRATCH24__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH24__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH25
+#define MP1_SMN_EXT_SCRATCH25__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH25__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH26
+#define MP1_SMN_EXT_SCRATCH26__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH26__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH27
+#define MP1_SMN_EXT_SCRATCH27__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH27__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH28
+#define MP1_SMN_EXT_SCRATCH28__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH28__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH29
+#define MP1_SMN_EXT_SCRATCH29__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH29__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH30
+#define MP1_SMN_EXT_SCRATCH30__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH30__DATA_MASK                                                                      0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH31
+#define MP1_SMN_EXT_SCRATCH31__DATA__SHIFT                                                                    0x0
+#define MP1_SMN_EXT_SCRATCH31__DATA_MASK                                                                      0xFFFFFFFFL
+
+
+// addressBlock: mp_SmuMpASP_SmnDec
+//MPASP_SMN_C2PMSG_81
+#define MPASP_SMN_C2PMSG_81__CONTENT__SHIFT                                                                   0x0
+#define MPASP_SMN_C2PMSG_81__CONTENT_MASK                                                                     0xFFFFFFFFL
+//MPASP_SMN_IH_CREDIT
+#define MPASP_SMN_IH_CREDIT__CREDIT_VALUE__SHIFT                                                              0x0
+#define MPASP_SMN_IH_CREDIT__CLIENT_ID__SHIFT                                                                 0x10
+#define MPASP_SMN_IH_CREDIT__CREDIT_VALUE_MASK                                                                0x00000003L
+#define MPASP_SMN_IH_CREDIT__CLIENT_ID_MASK                                                                   0x00FF0000L
+//MPASP_SMN_IH_SW_INT
+#define MPASP_SMN_IH_SW_INT__ID__SHIFT                                                                        0x0
+#define MPASP_SMN_IH_SW_INT__VALID__SHIFT                                                                     0x8
+#define MPASP_SMN_IH_SW_INT__ID_MASK                                                                          0x000000FFL
+#define MPASP_SMN_IH_SW_INT__VALID_MASK                                                                       0x00000100L
+//MPASP_SMN_IH_SW_INT_CTRL
+#define MPASP_SMN_IH_SW_INT_CTRL__INT_MASK__SHIFT                                                             0x0
+#define MPASP_SMN_IH_SW_INT_CTRL__INT_ACK__SHIFT                                                              0x8
+#define MPASP_SMN_IH_SW_INT_CTRL__INT_MASK_MASK                                                               0x00000001L
+#define MPASP_SMN_IH_SW_INT_CTRL__INT_ACK_MASK                                                                0x00000100L
+
+
+// addressBlock: mp_SmuMpRASPub_CruDec
+//MPRAS_CRU0_MPRAS_FIRMWARE_FLAGS
+#define MPRAS_CRU0_MPRAS_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT                                            0x0
+#define MPRAS_CRU0_MPRAS_FIRMWARE_FLAGS__RESERVED__SHIFT                                                      0x1
+#define MPRAS_CRU0_MPRAS_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK                                              0x00000001L
+#define MPRAS_CRU0_MPRAS_FIRMWARE_FLAGS__RESERVED_MASK                                                        0xFFFFFFFEL
+
+
+// addressBlock: mp_SmuMpIFOEPub_CruDec
+//MPIFOE_CRU0_MPIFOE_FIRMWARE_FLAGS
+#define MPIFOE_CRU0_MPIFOE_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT                                          0x0
+#define MPIFOE_CRU0_MPIFOE_FIRMWARE_FLAGS__RESERVED__SHIFT                                                    0x1
+#define MPIFOE_CRU0_MPIFOE_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK                                            0x00000001L
+#define MPIFOE_CRU0_MPIFOE_FIRMWARE_FLAGS__RESERVED_MASK                                                      0xFFFFFFFEL
+
+
+// addressBlock: mp_SmuMp1Pub_CruDec
+//MP1_CRU0_MP1_FIRMWARE_FLAGS
+#define MP1_CRU0_MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT                                                0x0
+#define MP1_CRU0_MP1_FIRMWARE_FLAGS__RESERVED__SHIFT                                                          0x1
+#define MP1_CRU0_MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK                                                  0x00000001L
+#define MP1_CRU0_MP1_FIRMWARE_FLAGS__RESERVED_MASK                                                            0xFFFFFFFEL
+
+
+// addressBlock: mp_SmuMpIOPub_CruDec
+//MPIO_CRU0_MPIO_FIRMWARE_FLAGS
+#define MPIO_CRU0_MPIO_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT                                              0x0
+#define MPIO_CRU0_MPIO_FIRMWARE_FLAGS__RESERVED__SHIFT                                                        0x1
+#define MPIO_CRU0_MPIO_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK                                                0x00000001L
+#define MPIO_CRU0_MPIO_FIRMWARE_FLAGS__RESERVED_MASK                                                          0xFFFFFFFEL
+
+
+// addressBlock: MpRASMmioPublic_SmuMpRASPub_CruDec
+//MPRAS_CRU1_MPRAS_FIRMWARE_FLAGS
+#define MPRAS_CRU1_MPRAS_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT                                            0x0
+#define MPRAS_CRU1_MPRAS_FIRMWARE_FLAGS__RESERVED__SHIFT                                                      0x1
+#define MPRAS_CRU1_MPRAS_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK                                              0x00000001L
+#define MPRAS_CRU1_MPRAS_FIRMWARE_FLAGS__RESERVED_MASK                                                        0xFFFFFFFEL
+
+
+// addressBlock: MpIFoEMmioPublic_SmuMpIFOEPub_CruDec
+//MPIFOE_CRU1_MPIFOE_FIRMWARE_FLAGS
+#define MPIFOE_CRU1_MPIFOE_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT                                          0x0
+#define MPIFOE_CRU1_MPIFOE_FIRMWARE_FLAGS__RESERVED__SHIFT                                                    0x1
+#define MPIFOE_CRU1_MPIFOE_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK                                            0x00000001L
+#define MPIFOE_CRU1_MPIFOE_FIRMWARE_FLAGS__RESERVED_MASK                                                      0xFFFFFFFEL
+
+
+// addressBlock: Mp1MmioPublic_SmuMp1Pub_CruDec
+//MP1_CRU1_MP1_FIRMWARE_FLAGS
+#define MP1_CRU1_MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT                                                0x0
+#define MP1_CRU1_MP1_FIRMWARE_FLAGS__RESERVED__SHIFT                                                          0x1
+#define MP1_CRU1_MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK                                                  0x00000001L
+#define MP1_CRU1_MP1_FIRMWARE_FLAGS__RESERVED_MASK                                                            0xFFFFFFFEL
+
+
+// addressBlock: MpIOMmioPublic_SmuMpIOPub_CruDec
+//MPIO_CRU1_MPIO_FIRMWARE_FLAGS
+#define MPIO_CRU1_MPIO_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT                                              0x0
+#define MPIO_CRU1_MPIO_FIRMWARE_FLAGS__RESERVED__SHIFT                                                        0x1
+#define MPIO_CRU1_MPIO_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK                                                0x00000001L
+#define MPIO_CRU1_MPIO_FIRMWARE_FLAGS__RESERVED_MASK                                                          0xFFFFFFFEL
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_7_1_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_7_1_0_offset.h
new file mode 100644
index 000000000000..fd454d36b679
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_7_1_0_offset.h
@@ -0,0 +1,280 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _osssys_7_1_0_OFFSET_HEADER
+#define _osssys_7_1_0_OFFSET_HEADER
+
+
+
+// addressBlock: osssys_osssysdec
+// base address: 0x4280
+#define regIH_VMID_0_LUT                                                                                0x0000
+#define regIH_VMID_0_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_1_LUT                                                                                0x0001
+#define regIH_VMID_1_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_2_LUT                                                                                0x0002
+#define regIH_VMID_2_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_3_LUT                                                                                0x0003
+#define regIH_VMID_3_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_4_LUT                                                                                0x0004
+#define regIH_VMID_4_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_5_LUT                                                                                0x0005
+#define regIH_VMID_5_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_6_LUT                                                                                0x0006
+#define regIH_VMID_6_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_7_LUT                                                                                0x0007
+#define regIH_VMID_7_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_8_LUT                                                                                0x0008
+#define regIH_VMID_8_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_9_LUT                                                                                0x0009
+#define regIH_VMID_9_LUT_BASE_IDX                                                                       0
+#define regIH_VMID_10_LUT                                                                               0x000a
+#define regIH_VMID_10_LUT_BASE_IDX                                                                      0
+#define regIH_VMID_11_LUT                                                                               0x000b
+#define regIH_VMID_11_LUT_BASE_IDX                                                                      0
+#define regIH_VMID_12_LUT                                                                               0x000c
+#define regIH_VMID_12_LUT_BASE_IDX                                                                      0
+#define regIH_VMID_13_LUT                                                                               0x000d
+#define regIH_VMID_13_LUT_BASE_IDX                                                                      0
+#define regIH_VMID_14_LUT                                                                               0x000e
+#define regIH_VMID_14_LUT_BASE_IDX                                                                      0
+#define regIH_VMID_15_LUT                                                                               0x000f
+#define regIH_VMID_15_LUT_BASE_IDX                                                                      0
+#define regIH_VMID_0_LUT_MM                                                                             0x0010
+#define regIH_VMID_0_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_1_LUT_MM                                                                             0x0011
+#define regIH_VMID_1_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_2_LUT_MM                                                                             0x0012
+#define regIH_VMID_2_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_3_LUT_MM                                                                             0x0013
+#define regIH_VMID_3_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_4_LUT_MM                                                                             0x0014
+#define regIH_VMID_4_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_5_LUT_MM                                                                             0x0015
+#define regIH_VMID_5_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_6_LUT_MM                                                                             0x0016
+#define regIH_VMID_6_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_7_LUT_MM                                                                             0x0017
+#define regIH_VMID_7_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_8_LUT_MM                                                                             0x0018
+#define regIH_VMID_8_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_9_LUT_MM                                                                             0x0019
+#define regIH_VMID_9_LUT_MM_BASE_IDX                                                                    0
+#define regIH_VMID_10_LUT_MM                                                                            0x001a
+#define regIH_VMID_10_LUT_MM_BASE_IDX                                                                   0
+#define regIH_VMID_11_LUT_MM                                                                            0x001b
+#define regIH_VMID_11_LUT_MM_BASE_IDX                                                                   0
+#define regIH_VMID_12_LUT_MM                                                                            0x001c
+#define regIH_VMID_12_LUT_MM_BASE_IDX                                                                   0
+#define regIH_VMID_13_LUT_MM                                                                            0x001d
+#define regIH_VMID_13_LUT_MM_BASE_IDX                                                                   0
+#define regIH_VMID_14_LUT_MM                                                                            0x001e
+#define regIH_VMID_14_LUT_MM_BASE_IDX                                                                   0
+#define regIH_VMID_15_LUT_MM                                                                            0x001f
+#define regIH_VMID_15_LUT_MM_BASE_IDX                                                                   0
+#define regIH_COOKIE_0                                                                                  0x0020
+#define regIH_COOKIE_0_BASE_IDX                                                                         0
+#define regIH_COOKIE_1                                                                                  0x0021
+#define regIH_COOKIE_1_BASE_IDX                                                                         0
+#define regIH_COOKIE_2                                                                                  0x0022
+#define regIH_COOKIE_2_BASE_IDX                                                                         0
+#define regIH_COOKIE_3                                                                                  0x0023
+#define regIH_COOKIE_3_BASE_IDX                                                                         0
+#define regIH_COOKIE_4                                                                                  0x0024
+#define regIH_COOKIE_4_BASE_IDX                                                                         0
+#define regIH_COOKIE_5                                                                                  0x0025
+#define regIH_COOKIE_5_BASE_IDX                                                                         0
+#define regIH_COOKIE_6                                                                                  0x0026
+#define regIH_COOKIE_6_BASE_IDX                                                                         0
+#define regIH_COOKIE_7                                                                                  0x0027
+#define regIH_COOKIE_7_BASE_IDX                                                                         0
+#define regIH_VMID_LUT_INDEX                                                                            0x0028
+#define regIH_VMID_LUT_INDEX_BASE_IDX                                                                   0
+#define regIH_REGISTER_LAST_PART0                                                                       0x003f
+#define regIH_REGISTER_LAST_PART0_BASE_IDX                                                              0
+#define regIH_RB_CNTL                                                                                   0x0080
+#define regIH_RB_CNTL_BASE_IDX                                                                          0
+#define regIH_RB_RPTR                                                                                   0x0081
+#define regIH_RB_RPTR_BASE_IDX                                                                          0
+#define regIH_RB_WPTR                                                                                   0x0082
+#define regIH_RB_WPTR_BASE_IDX                                                                          0
+#define regIH_RB_BASE                                                                                   0x0083
+#define regIH_RB_BASE_BASE_IDX                                                                          0
+#define regIH_RB_BASE_HI                                                                                0x0084
+#define regIH_RB_BASE_HI_BASE_IDX                                                                       0
+#define regIH_RB_WPTR_ADDR_HI                                                                           0x0085
+#define regIH_RB_WPTR_ADDR_HI_BASE_IDX                                                                  0
+#define regIH_RB_WPTR_ADDR_LO                                                                           0x0086
+#define regIH_RB_WPTR_ADDR_LO_BASE_IDX                                                                  0
+#define regIH_DOORBELL_RPTR                                                                             0x0087
+#define regIH_DOORBELL_RPTR_BASE_IDX                                                                    0
+#define regIH_DOORBELL_RETRY_CAM                                                                        0x0088
+#define regIH_DOORBELL_RETRY_CAM_BASE_IDX                                                               0
+#define regIH_RB_CNTL_RING1                                                                             0x008c
+#define regIH_RB_CNTL_RING1_BASE_IDX                                                                    0
+#define regIH_RB_RPTR_RING1                                                                             0x008d
+#define regIH_RB_RPTR_RING1_BASE_IDX                                                                    0
+#define regIH_RB_WPTR_RING1                                                                             0x008e
+#define regIH_RB_WPTR_RING1_BASE_IDX                                                                    0
+#define regIH_RB_BASE_RING1                                                                             0x008f
+#define regIH_RB_BASE_RING1_BASE_IDX                                                                    0
+#define regIH_RB_BASE_HI_RING1                                                                          0x0090
+#define regIH_RB_BASE_HI_RING1_BASE_IDX                                                                 0
+#define regIH_DOORBELL_RPTR_RING1                                                                       0x0093
+#define regIH_DOORBELL_RPTR_RING1_BASE_IDX                                                              0
+#define regIH_RETRY_CAM_ACK                                                                             0x00a4
+#define regIH_RETRY_CAM_ACK_BASE_IDX                                                                    0
+#define regIH_VERSION                                                                                   0x00a5
+#define regIH_VERSION_BASE_IDX                                                                          0
+#define regIH_CNTL                                                                                      0x00a8
+#define regIH_CNTL_BASE_IDX                                                                             0
+#define regIH_CLK_CTRL                                                                                  0x00a9
+#define regIH_CLK_CTRL_BASE_IDX                                                                         0
+#define regIH_STORM_CLIENT_LIST_CNTL                                                                    0x00aa
+#define regIH_STORM_CLIENT_LIST_CNTL_BASE_IDX                                                           0
+#define regIH_LIMIT_INT_RATE_CNTL                                                                       0x00ab
+#define regIH_LIMIT_INT_RATE_CNTL_BASE_IDX                                                              0
+#define regIH_RETRY_INT_CAM_CNTL                                                                        0x00ac
+#define regIH_RETRY_INT_CAM_CNTL_BASE_IDX                                                               0
+#define regIH_MEM_POWER_CTRL                                                                            0x00ad
+#define regIH_MEM_POWER_CTRL_BASE_IDX                                                                   0
+#define regIH_MEM_POWER_CTRL2                                                                           0x00ae
+#define regIH_MEM_POWER_CTRL2_BASE_IDX                                                                  0
+#define regIH_CNTL2                                                                                     0x00c1
+#define regIH_CNTL2_BASE_IDX                                                                            0
+#define regIH_STATUS                                                                                    0x00c2
+#define regIH_STATUS_BASE_IDX                                                                           0
+#define regIH_PERFMON_CNTL                                                                              0x00c3
+#define regIH_PERFMON_CNTL_BASE_IDX                                                                     0
+#define regIH_PERFCOUNTER0_RESULT                                                                       0x00c4
+#define regIH_PERFCOUNTER0_RESULT_BASE_IDX                                                              0
+#define regIH_PERFCOUNTER1_RESULT                                                                       0x00c5
+#define regIH_PERFCOUNTER1_RESULT_BASE_IDX                                                              0
+#define regIH_DSM_MATCH_VALUE_BIT_31_0                                                                  0x00c7
+#define regIH_DSM_MATCH_VALUE_BIT_31_0_BASE_IDX                                                         0
+#define regIH_DSM_MATCH_VALUE_BIT_63_32                                                                 0x00c8
+#define regIH_DSM_MATCH_VALUE_BIT_63_32_BASE_IDX                                                        0
+#define regIH_DSM_MATCH_VALUE_BIT_95_64                                                                 0x00c9
+#define regIH_DSM_MATCH_VALUE_BIT_95_64_BASE_IDX                                                        0
+#define regIH_DSM_MATCH_FIELD_CONTROL                                                                   0x00ca
+#define regIH_DSM_MATCH_FIELD_CONTROL_BASE_IDX                                                          0
+#define regIH_DSM_MATCH_DATA_CONTROL                                                                    0x00cb
+#define regIH_DSM_MATCH_DATA_CONTROL_BASE_IDX                                                           0
+#define regIH_DSM_MATCH_FCN_ID                                                                          0x00cc
+#define regIH_DSM_MATCH_FCN_ID_BASE_IDX                                                                 0
+#define regIH_VF_RB_STATUS                                                                              0x00ce
+#define regIH_VF_RB_STATUS_BASE_IDX                                                                     0
+#define regIH_VF_RB_STATUS2                                                                             0x00cf
+#define regIH_VF_RB_STATUS2_BASE_IDX                                                                    0
+#define regIH_VF_RB1_STATUS                                                                             0x00d0
+#define regIH_VF_RB1_STATUS_BASE_IDX                                                                    0
+#define regIH_VF_RB1_STATUS2                                                                            0x00d1
+#define regIH_VF_RB1_STATUS2_BASE_IDX                                                                   0
+#define regIH_RB_STATUS                                                                                 0x00d4
+#define regIH_RB_STATUS_BASE_IDX                                                                        0
+#define regIH_INT_FLOOD_CNTL                                                                            0x00d5
+#define regIH_INT_FLOOD_CNTL_BASE_IDX                                                                   0
+#define regIH_RB0_INT_FLOOD_STATUS                                                                      0x00d6
+#define regIH_RB0_INT_FLOOD_STATUS_BASE_IDX                                                             0
+#define regIH_RB1_INT_FLOOD_STATUS                                                                      0x00d7
+#define regIH_RB1_INT_FLOOD_STATUS_BASE_IDX                                                             0
+#define regIH_INT_FLOOD_STATUS                                                                          0x00d9
+#define regIH_INT_FLOOD_STATUS_BASE_IDX                                                                 0
+#define regIH_INT_FLAGS                                                                                 0x00dc
+#define regIH_INT_FLAGS_BASE_IDX                                                                        0
+#define regIH_SCRATCH                                                                                   0x00e0
+#define regIH_SCRATCH_BASE_IDX                                                                          0
+#define regIH_CLIENT_CREDIT_ERROR                                                                       0x00e1
+#define regIH_CLIENT_CREDIT_ERROR_BASE_IDX                                                              0
+#define regIH_GPU_IOV_VIOLATION_LOG                                                                     0x00e2
+#define regIH_GPU_IOV_VIOLATION_LOG_BASE_IDX                                                            0
+#define regIH_GPU_IOV_VIOLATION_LOG2                                                                    0x00e3
+#define regIH_GPU_IOV_VIOLATION_LOG2_BASE_IDX                                                           0
+#define regIH_COOKIE_REC_VIOLATION_LOG                                                                  0x00e4
+#define regIH_COOKIE_REC_VIOLATION_LOG_BASE_IDX                                                         0
+#define regIH_CREDIT_STATUS                                                                             0x00e5
+#define regIH_CREDIT_STATUS_BASE_IDX                                                                    0
+#define regIH_MMHUB_ERROR                                                                               0x00e6
+#define regIH_MMHUB_ERROR_BASE_IDX                                                                      0
+#define regIH_VF_RB_STATUS3                                                                             0x00ea
+#define regIH_VF_RB_STATUS3_BASE_IDX                                                                    0
+#define regIH_VF_RB_STATUS4                                                                             0x00eb
+#define regIH_VF_RB_STATUS4_BASE_IDX                                                                    0
+#define regIH_VF_RB1_STATUS3                                                                            0x00ec
+#define regIH_VF_RB1_STATUS3_BASE_IDX                                                                   0
+#define regIH_MSI_STORM_CTRL                                                                            0x00f1
+#define regIH_MSI_STORM_CTRL_BASE_IDX                                                                   0
+#define regIH_MSI_STORM_CLIENT_INDEX                                                                    0x00f2
+#define regIH_MSI_STORM_CLIENT_INDEX_BASE_IDX                                                           0
+#define regIH_MSI_STORM_CLIENT_DATA                                                                     0x00f3
+#define regIH_MSI_STORM_CLIENT_DATA_BASE_IDX                                                            0
+#define regIH_LAST_INT_INFO0                                                                            0x00f9
+#define regIH_LAST_INT_INFO0_BASE_IDX                                                                   0
+#define regIH_LAST_INT_INFO1                                                                            0x00fa
+#define regIH_LAST_INT_INFO1_BASE_IDX                                                                   0
+#define regIH_LAST_INT_INFO2                                                                            0x00fb
+#define regIH_LAST_INT_INFO2_BASE_IDX                                                                   0
+#define regIH_REGISTER_LAST_PART2                                                                       0x00ff
+#define regIH_REGISTER_LAST_PART2_BASE_IDX                                                              0
+#define regSEM_MAILBOX                                                                                  0x010a
+#define regSEM_MAILBOX_BASE_IDX                                                                         0
+#define regSEM_MAILBOX_CLEAR                                                                            0x010b
+#define regSEM_MAILBOX_CLEAR_BASE_IDX                                                                   0
+#define regSEM_REGISTER_LAST_PART2                                                                      0x011f
+#define regSEM_REGISTER_LAST_PART2_BASE_IDX                                                             0
+#define regIH_VIRT_RESET_REQ                                                                            0x0120
+#define regIH_VIRT_RESET_REQ_BASE_IDX                                                                   0
+#define regIH_CLIENT_CFG                                                                                0x0121
+#define regIH_CLIENT_CFG_BASE_IDX                                                                       0
+#define regIH_RING1_CLIENT_CFG_INDEX                                                                    0x0122
+#define regIH_RING1_CLIENT_CFG_INDEX_BASE_IDX                                                           0
+#define regIH_RING1_CLIENT_CFG_DATA                                                                     0x0123
+#define regIH_RING1_CLIENT_CFG_DATA_BASE_IDX                                                            0
+#define regIH_CLIENT_CFG_INDEX                                                                          0x0124
+#define regIH_CLIENT_CFG_INDEX_BASE_IDX                                                                 0
+#define regIH_CLIENT_CFG_DATA                                                                           0x0125
+#define regIH_CLIENT_CFG_DATA_BASE_IDX                                                                  0
+#define regIH_CLIENT_CFG_DATA2                                                                          0x0126
+#define regIH_CLIENT_CFG_DATA2_BASE_IDX                                                                 0
+#define regIH_CID_REMAP_INDEX                                                                           0x0127
+#define regIH_CID_REMAP_INDEX_BASE_IDX                                                                  0
+#define regIH_CID_REMAP_DATA                                                                            0x0128
+#define regIH_CID_REMAP_DATA_BASE_IDX                                                                   0
+#define regIH_CHICKEN                                                                                   0x0129
+#define regIH_CHICKEN_BASE_IDX                                                                          0
+#define regIH_INT_DROP_CNTL                                                                             0x012a
+#define regIH_INT_DROP_CNTL_BASE_IDX                                                                    0
+#define regIH_INT_DROP_MATCH_VALUE0                                                                     0x012b
+#define regIH_INT_DROP_MATCH_VALUE0_BASE_IDX                                                            0
+#define regIH_INT_DROP_MATCH_VALUE1                                                                     0x012c
+#define regIH_INT_DROP_MATCH_VALUE1_BASE_IDX                                                            0
+#define regIH_INT_DROP_MATCH_MASK0                                                                      0x012d
+#define regIH_INT_DROP_MATCH_MASK0_BASE_IDX                                                             0
+#define regIH_INT_DROP_MATCH_MASK1                                                                      0x012e
+#define regIH_INT_DROP_MATCH_MASK1_BASE_IDX                                                             0
+#define regIH_MMHUB_CNTL                                                                                0x0147
+#define regIH_MMHUB_CNTL_BASE_IDX                                                                       0
+#define regIH_REGISTER_LAST_PART1                                                                       0x019f
+#define regIH_REGISTER_LAST_PART1_BASE_IDX                                                              0
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_7_1_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_7_1_0_sh_mask.h
new file mode 100644
index 000000000000..0bc2f25968ea
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_7_1_0_sh_mask.h
@@ -0,0 +1,1024 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _osssys_7_1_0_SH_MASK_HEADER
+#define _osssys_7_1_0_SH_MASK_HEADER
+
+
+// addressBlock: osssys_osssysdec
+//IH_VMID_0_LUT
+#define IH_VMID_0_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_0_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_1_LUT
+#define IH_VMID_1_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_1_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_2_LUT
+#define IH_VMID_2_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_2_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_3_LUT
+#define IH_VMID_3_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_3_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_4_LUT
+#define IH_VMID_4_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_4_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_5_LUT
+#define IH_VMID_5_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_5_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_6_LUT
+#define IH_VMID_6_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_6_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_7_LUT
+#define IH_VMID_7_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_7_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_8_LUT
+#define IH_VMID_8_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_8_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_9_LUT
+#define IH_VMID_9_LUT__PASID__SHIFT                                                                           0x0
+#define IH_VMID_9_LUT__PASID_MASK                                                                             0x0000FFFFL
+//IH_VMID_10_LUT
+#define IH_VMID_10_LUT__PASID__SHIFT                                                                          0x0
+#define IH_VMID_10_LUT__PASID_MASK                                                                            0x0000FFFFL
+//IH_VMID_11_LUT
+#define IH_VMID_11_LUT__PASID__SHIFT                                                                          0x0
+#define IH_VMID_11_LUT__PASID_MASK                                                                            0x0000FFFFL
+//IH_VMID_12_LUT
+#define IH_VMID_12_LUT__PASID__SHIFT                                                                          0x0
+#define IH_VMID_12_LUT__PASID_MASK                                                                            0x0000FFFFL
+//IH_VMID_13_LUT
+#define IH_VMID_13_LUT__PASID__SHIFT                                                                          0x0
+#define IH_VMID_13_LUT__PASID_MASK                                                                            0x0000FFFFL
+//IH_VMID_14_LUT
+#define IH_VMID_14_LUT__PASID__SHIFT                                                                          0x0
+#define IH_VMID_14_LUT__PASID_MASK                                                                            0x0000FFFFL
+//IH_VMID_15_LUT
+#define IH_VMID_15_LUT__PASID__SHIFT                                                                          0x0
+#define IH_VMID_15_LUT__PASID_MASK                                                                            0x0000FFFFL
+//IH_VMID_0_LUT_MM
+#define IH_VMID_0_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_0_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_1_LUT_MM
+#define IH_VMID_1_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_1_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_2_LUT_MM
+#define IH_VMID_2_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_2_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_3_LUT_MM
+#define IH_VMID_3_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_3_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_4_LUT_MM
+#define IH_VMID_4_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_4_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_5_LUT_MM
+#define IH_VMID_5_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_5_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_6_LUT_MM
+#define IH_VMID_6_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_6_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_7_LUT_MM
+#define IH_VMID_7_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_7_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_8_LUT_MM
+#define IH_VMID_8_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_8_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_9_LUT_MM
+#define IH_VMID_9_LUT_MM__PASID__SHIFT                                                                        0x0
+#define IH_VMID_9_LUT_MM__PASID_MASK                                                                          0x0000FFFFL
+//IH_VMID_10_LUT_MM
+#define IH_VMID_10_LUT_MM__PASID__SHIFT                                                                       0x0
+#define IH_VMID_10_LUT_MM__PASID_MASK                                                                         0x0000FFFFL
+//IH_VMID_11_LUT_MM
+#define IH_VMID_11_LUT_MM__PASID__SHIFT                                                                       0x0
+#define IH_VMID_11_LUT_MM__PASID_MASK                                                                         0x0000FFFFL
+//IH_VMID_12_LUT_MM
+#define IH_VMID_12_LUT_MM__PASID__SHIFT                                                                       0x0
+#define IH_VMID_12_LUT_MM__PASID_MASK                                                                         0x0000FFFFL
+//IH_VMID_13_LUT_MM
+#define IH_VMID_13_LUT_MM__PASID__SHIFT                                                                       0x0
+#define IH_VMID_13_LUT_MM__PASID_MASK                                                                         0x0000FFFFL
+//IH_VMID_14_LUT_MM
+#define IH_VMID_14_LUT_MM__PASID__SHIFT                                                                       0x0
+#define IH_VMID_14_LUT_MM__PASID_MASK                                                                         0x0000FFFFL
+//IH_VMID_15_LUT_MM
+#define IH_VMID_15_LUT_MM__PASID__SHIFT                                                                       0x0
+#define IH_VMID_15_LUT_MM__PASID_MASK                                                                         0x0000FFFFL
+//IH_COOKIE_0
+#define IH_COOKIE_0__CLIENT_ID__SHIFT                                                                         0x0
+#define IH_COOKIE_0__SOURCE_ID__SHIFT                                                                         0x8
+#define IH_COOKIE_0__RING_ID__SHIFT                                                                           0x10
+#define IH_COOKIE_0__VM_ID__SHIFT                                                                             0x18
+#define IH_COOKIE_0__RESERVED__SHIFT                                                                          0x1c
+#define IH_COOKIE_0__VMID_TYPE__SHIFT                                                                         0x1f
+#define IH_COOKIE_0__CLIENT_ID_MASK                                                                           0x000000FFL
+#define IH_COOKIE_0__SOURCE_ID_MASK                                                                           0x0000FF00L
+#define IH_COOKIE_0__RING_ID_MASK                                                                             0x00FF0000L
+#define IH_COOKIE_0__VM_ID_MASK                                                                               0x0F000000L
+#define IH_COOKIE_0__RESERVED_MASK                                                                            0x70000000L
+#define IH_COOKIE_0__VMID_TYPE_MASK                                                                           0x80000000L
+//IH_COOKIE_1
+#define IH_COOKIE_1__TIMESTAMP_31_0__SHIFT                                                                    0x0
+#define IH_COOKIE_1__TIMESTAMP_31_0_MASK                                                                      0xFFFFFFFFL
+//IH_COOKIE_2
+#define IH_COOKIE_2__TIMESTAMP_47_32__SHIFT                                                                   0x0
+#define IH_COOKIE_2__RESERVED__SHIFT                                                                          0x10
+#define IH_COOKIE_2__TIMESTAMP_SRC__SHIFT                                                                     0x1f
+#define IH_COOKIE_2__TIMESTAMP_47_32_MASK                                                                     0x0000FFFFL
+#define IH_COOKIE_2__RESERVED_MASK                                                                            0x7FFF0000L
+#define IH_COOKIE_2__TIMESTAMP_SRC_MASK                                                                       0x80000000L
+//IH_COOKIE_3
+#define IH_COOKIE_3__PAS_ID__SHIFT                                                                            0x0
+#define IH_COOKIE_3__RESERVED__SHIFT                                                                          0x10
+#define IH_COOKIE_3__FORCE_TO_PF__SHIFT                                                                       0x1e
+#define IH_COOKIE_3__PASID_SRC__SHIFT                                                                         0x1f
+#define IH_COOKIE_3__PAS_ID_MASK                                                                              0x0000FFFFL
+#define IH_COOKIE_3__RESERVED_MASK                                                                            0x3FFF0000L
+#define IH_COOKIE_3__PASID_SRC_MASK                                                                           0x80000000L
+//IH_COOKIE_4
+#define IH_COOKIE_4__CONTEXT_ID_31_0__SHIFT                                                                   0x0
+#define IH_COOKIE_4__CONTEXT_ID_31_0_MASK                                                                     0xFFFFFFFFL
+//IH_COOKIE_5
+#define IH_COOKIE_5__CONTEXT_ID_63_32__SHIFT                                                                  0x0
+#define IH_COOKIE_5__CONTEXT_ID_63_32_MASK                                                                    0xFFFFFFFFL
+//IH_COOKIE_6
+#define IH_COOKIE_6__CONTEXT_ID_95_64__SHIFT                                                                  0x0
+#define IH_COOKIE_6__CONTEXT_ID_95_64_MASK                                                                    0xFFFFFFFFL
+//IH_COOKIE_7
+#define IH_COOKIE_7__CONTEXT_ID_128_96__SHIFT                                                                 0x0
+#define IH_COOKIE_7__CONTEXT_ID_128_96_MASK                                                                   0xFFFFFFFFL
+//IH_VMID_LUT_INDEX
+#define IH_VMID_LUT_INDEX__INDEX__SHIFT                                                                       0x0
+#define IH_VMID_LUT_INDEX__INDEX_MASK                                                                         0x0000000FL
+//IH_REGISTER_LAST_PART0
+#define IH_REGISTER_LAST_PART0__RESERVED__SHIFT                                                               0x0
+#define IH_REGISTER_LAST_PART0__RESERVED_MASK                                                                 0xFFFFFFFFL
+//IH_RB_CNTL
+#define IH_RB_CNTL__RB_ENABLE__SHIFT                                                                          0x0
+#define IH_RB_CNTL__RB_SIZE__SHIFT                                                                            0x1
+#define IH_RB_CNTL__WPTR_WRITEBACK_ENABLE__SHIFT                                                              0x8
+#define IH_RB_CNTL__RB_FULL_DRAIN_ENABLE__SHIFT                                                               0x9
+#define IH_RB_CNTL__FULL_DRAIN_CLEAR__SHIFT                                                                   0xa
+#define IH_RB_CNTL__PAGE_RB_CLEAR__SHIFT                                                                      0xb
+#define IH_RB_CNTL__RB_USED_INT_THRESHOLD__SHIFT                                                              0xc
+#define IH_RB_CNTL__WPTR_OVERFLOW_ENABLE__SHIFT                                                               0x10
+#define IH_RB_CNTL__ENABLE_INTR__SHIFT                                                                        0x11
+#define IH_RB_CNTL__MC_SWAP__SHIFT                                                                            0x12
+#define IH_RB_CNTL__MC_SNOOP__SHIFT                                                                           0x14
+#define IH_RB_CNTL__RPTR_REARM__SHIFT                                                                         0x15
+#define IH_RB_CNTL__MC_RO__SHIFT                                                                              0x16
+#define IH_RB_CNTL__MC_VMID__SHIFT                                                                            0x18
+#define IH_RB_CNTL__MC_SPACE__SHIFT                                                                           0x1c
+#define IH_RB_CNTL__WPTR_OVERFLOW_CLEAR__SHIFT                                                                0x1f
+#define IH_RB_CNTL__RB_ENABLE_MASK                                                                            0x00000001L
+#define IH_RB_CNTL__RB_SIZE_MASK                                                                              0x0000003EL
+#define IH_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK                                                                0x00000100L
+#define IH_RB_CNTL__RB_FULL_DRAIN_ENABLE_MASK                                                                 0x00000200L
+#define IH_RB_CNTL__FULL_DRAIN_CLEAR_MASK                                                                     0x00000400L
+#define IH_RB_CNTL__PAGE_RB_CLEAR_MASK                                                                        0x00000800L
+#define IH_RB_CNTL__RB_USED_INT_THRESHOLD_MASK                                                                0x0000F000L
+#define IH_RB_CNTL__WPTR_OVERFLOW_ENABLE_MASK                                                                 0x00010000L
+#define IH_RB_CNTL__ENABLE_INTR_MASK                                                                          0x00020000L
+#define IH_RB_CNTL__MC_SWAP_MASK                                                                              0x000C0000L
+#define IH_RB_CNTL__MC_SNOOP_MASK                                                                             0x00100000L
+#define IH_RB_CNTL__RPTR_REARM_MASK                                                                           0x00200000L
+#define IH_RB_CNTL__MC_RO_MASK                                                                                0x00400000L
+#define IH_RB_CNTL__MC_VMID_MASK                                                                              0x0F000000L
+#define IH_RB_CNTL__MC_SPACE_MASK                                                                             0x70000000L
+#define IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK                                                                  0x80000000L
+//IH_RB_RPTR
+#define IH_RB_RPTR__OFFSET__SHIFT                                                                             0x2
+#define IH_RB_RPTR__OFFSET_MASK                                                                               0x0003FFFCL
+//IH_RB_WPTR
+#define IH_RB_WPTR__RB_OVERFLOW__SHIFT                                                                        0x0
+#define IH_RB_WPTR__OFFSET__SHIFT                                                                             0x2
+#define IH_RB_WPTR__RB_LEFT_NONE__SHIFT                                                                       0x12
+#define IH_RB_WPTR__RB_MAY_OVERFLOW__SHIFT                                                                    0x13
+#define IH_RB_WPTR__RB_OVERFLOW_MASK                                                                          0x00000001L
+#define IH_RB_WPTR__OFFSET_MASK                                                                               0x0003FFFCL
+#define IH_RB_WPTR__RB_LEFT_NONE_MASK                                                                         0x00040000L
+#define IH_RB_WPTR__RB_MAY_OVERFLOW_MASK                                                                      0x00080000L
+//IH_RB_BASE
+#define IH_RB_BASE__ADDR__SHIFT                                                                               0x0
+#define IH_RB_BASE__ADDR_MASK                                                                                 0xFFFFFFFFL
+//IH_RB_BASE_HI
+#define IH_RB_BASE_HI__ADDR__SHIFT                                                                            0x0
+#define IH_RB_BASE_HI__ADDR_MASK                                                                              0x0001FFFFL
+//IH_RB_WPTR_ADDR_HI
+#define IH_RB_WPTR_ADDR_HI__ADDR__SHIFT                                                                       0x0
+#define IH_RB_WPTR_ADDR_HI__ADDR_MASK                                                                         0x01FFFFFFL
+//IH_RB_WPTR_ADDR_LO
+#define IH_RB_WPTR_ADDR_LO__ADDR__SHIFT                                                                       0x2
+#define IH_RB_WPTR_ADDR_LO__ADDR_MASK                                                                         0xFFFFFFFCL
+//IH_DOORBELL_RPTR
+#define IH_DOORBELL_RPTR__OFFSET__SHIFT                                                                       0x0
+#define IH_DOORBELL_RPTR__ENABLE__SHIFT                                                                       0x1c
+#define IH_DOORBELL_RPTR__OFFSET_MASK                                                                         0x03FFFFFFL
+#define IH_DOORBELL_RPTR__ENABLE_MASK                                                                         0x10000000L
+//IH_DOORBELL_RETRY_CAM
+#define IH_DOORBELL_RETRY_CAM__OFFSET__SHIFT                                                                  0x0
+#define IH_DOORBELL_RETRY_CAM__ENABLE__SHIFT                                                                  0x1c
+#define IH_DOORBELL_RETRY_CAM__OFFSET_MASK                                                                    0x03FFFFFFL
+#define IH_DOORBELL_RETRY_CAM__ENABLE_MASK                                                                    0x10000000L
+//IH_RB_CNTL_RING1
+#define IH_RB_CNTL_RING1__RB_ENABLE__SHIFT                                                                    0x0
+#define IH_RB_CNTL_RING1__RB_SIZE__SHIFT                                                                      0x1
+#define IH_RB_CNTL_RING1__RB_FULL_DRAIN_ENABLE__SHIFT                                                         0x9
+#define IH_RB_CNTL_RING1__FULL_DRAIN_CLEAR__SHIFT                                                             0xa
+#define IH_RB_CNTL_RING1__PAGE_RB_CLEAR__SHIFT                                                                0xb
+#define IH_RB_CNTL_RING1__RB_USED_INT_THRESHOLD__SHIFT                                                        0xc
+#define IH_RB_CNTL_RING1__WPTR_OVERFLOW_ENABLE__SHIFT                                                         0x10
+#define IH_RB_CNTL_RING1__MC_SWAP__SHIFT                                                                      0x12
+#define IH_RB_CNTL_RING1__MC_SNOOP__SHIFT                                                                     0x14
+#define IH_RB_CNTL_RING1__MC_RO__SHIFT                                                                        0x16
+#define IH_RB_CNTL_RING1__MC_VMID__SHIFT                                                                      0x18
+#define IH_RB_CNTL_RING1__MC_SPACE__SHIFT                                                                     0x1c
+#define IH_RB_CNTL_RING1__WPTR_OVERFLOW_CLEAR__SHIFT                                                          0x1f
+#define IH_RB_CNTL_RING1__RB_ENABLE_MASK                                                                      0x00000001L
+#define IH_RB_CNTL_RING1__RB_SIZE_MASK                                                                        0x0000003EL
+#define IH_RB_CNTL_RING1__RB_FULL_DRAIN_ENABLE_MASK                                                           0x00000200L
+#define IH_RB_CNTL_RING1__FULL_DRAIN_CLEAR_MASK                                                               0x00000400L
+#define IH_RB_CNTL_RING1__PAGE_RB_CLEAR_MASK                                                                  0x00000800L
+#define IH_RB_CNTL_RING1__RB_USED_INT_THRESHOLD_MASK                                                          0x0000F000L
+#define IH_RB_CNTL_RING1__WPTR_OVERFLOW_ENABLE_MASK                                                           0x00010000L
+#define IH_RB_CNTL_RING1__MC_SWAP_MASK                                                                        0x000C0000L
+#define IH_RB_CNTL_RING1__MC_SNOOP_MASK                                                                       0x00100000L
+#define IH_RB_CNTL_RING1__MC_RO_MASK                                                                          0x00400000L
+#define IH_RB_CNTL_RING1__MC_VMID_MASK                                                                        0x0F000000L
+#define IH_RB_CNTL_RING1__MC_SPACE_MASK                                                                       0x70000000L
+#define IH_RB_CNTL_RING1__WPTR_OVERFLOW_CLEAR_MASK                                                            0x80000000L
+//IH_RB_RPTR_RING1
+#define IH_RB_RPTR_RING1__OFFSET__SHIFT                                                                       0x2
+#define IH_RB_RPTR_RING1__OFFSET_MASK                                                                         0x0003FFFCL
+//IH_RB_WPTR_RING1
+#define IH_RB_WPTR_RING1__RB_OVERFLOW__SHIFT                                                                  0x0
+#define IH_RB_WPTR_RING1__OFFSET__SHIFT                                                                       0x2
+#define IH_RB_WPTR_RING1__RB_LEFT_NONE__SHIFT                                                                 0x12
+#define IH_RB_WPTR_RING1__RB_MAY_OVERFLOW__SHIFT                                                              0x13
+#define IH_RB_WPTR_RING1__RB_OVERFLOW_MASK                                                                    0x00000001L
+#define IH_RB_WPTR_RING1__OFFSET_MASK                                                                         0x0003FFFCL
+#define IH_RB_WPTR_RING1__RB_LEFT_NONE_MASK                                                                   0x00040000L
+#define IH_RB_WPTR_RING1__RB_MAY_OVERFLOW_MASK                                                                0x00080000L
+//IH_RB_BASE_RING1
+#define IH_RB_BASE_RING1__ADDR__SHIFT                                                                         0x0
+#define IH_RB_BASE_RING1__ADDR_MASK                                                                           0xFFFFFFFFL
+//IH_RB_BASE_HI_RING1
+#define IH_RB_BASE_HI_RING1__ADDR__SHIFT                                                                      0x0
+#define IH_RB_BASE_HI_RING1__ADDR_MASK                                                                        0x0001FFFFL
+//IH_DOORBELL_RPTR_RING1
+#define IH_DOORBELL_RPTR_RING1__OFFSET__SHIFT                                                                 0x0
+#define IH_DOORBELL_RPTR_RING1__ENABLE__SHIFT                                                                 0x1c
+#define IH_DOORBELL_RPTR_RING1__OFFSET_MASK                                                                   0x03FFFFFFL
+#define IH_DOORBELL_RPTR_RING1__ENABLE_MASK                                                                   0x10000000L
+//IH_RETRY_CAM_ACK
+#define IH_RETRY_CAM_ACK__INDEX__SHIFT                                                                        0x0
+#define IH_RETRY_CAM_ACK__INDEX_MASK                                                                          0x000003FFL
+//IH_VERSION
+#define IH_VERSION__MINVER__SHIFT                                                                             0x0
+#define IH_VERSION__MAJVER__SHIFT                                                                             0x8
+#define IH_VERSION__REV__SHIFT                                                                                0x10
+#define IH_VERSION__MINVER_MASK                                                                               0x0000007FL
+#define IH_VERSION__MAJVER_MASK                                                                               0x00007F00L
+#define IH_VERSION__REV_MASK                                                                                  0x003F0000L
+//IH_CNTL
+#define IH_CNTL__WPTR_WRITEBACK_TIMER__SHIFT                                                                  0x0
+#define IH_CNTL__IH_IDLE_HYSTERESIS_CNTL__SHIFT                                                               0x6
+#define IH_CNTL__IH_FIFO_HIGHWATER__SHIFT                                                                     0x8
+#define IH_CNTL__MC_WR_CLEAN_CNT__SHIFT                                                                       0x14
+#define IH_CNTL__WPTR_WRITEBACK_TIMER_MASK                                                                    0x0000001FL
+#define IH_CNTL__IH_IDLE_HYSTERESIS_CNTL_MASK                                                                 0x000000C0L
+#define IH_CNTL__IH_FIFO_HIGHWATER_MASK                                                                       0x0001FF00L
+#define IH_CNTL__MC_WR_CLEAN_CNT_MASK                                                                         0x01F00000L
+//IH_CLK_CTRL
+#define IH_CLK_CTRL__IH_VF_CLK_GATING_MODE__SHIFT                                                             0x15
+#define IH_CLK_CTRL__IH_RAS_CLK_SOFT_OVERRIDE__SHIFT                                                          0x16
+#define IH_CLK_CTRL__IH_PASID_LUT_MEM_CLK_SOFT_OVERRIDE__SHIFT                                                0x17
+#define IH_CLK_CTRL__MSI_STORM_COUNTER_CLK_SOFT_OVERRIDE__SHIFT                                               0x18
+#define IH_CLK_CTRL__IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE__SHIFT                                            0x19
+#define IH_CLK_CTRL__IH_BUFFER_MEM_CLK_SOFT_OVERRIDE__SHIFT                                                   0x1a
+#define IH_CLK_CTRL__DBUS_MUX_CLK_SOFT_OVERRIDE__SHIFT                                                        0x1b
+#define IH_CLK_CTRL__OSSSYS_SHARE_CLK_SOFT_OVERRIDE__SHIFT                                                    0x1c
+#define IH_CLK_CTRL__LIMIT_SMN_CLK_SOFT_OVERRIDE__SHIFT                                                       0x1d
+#define IH_CLK_CTRL__DYN_CLK_SOFT_OVERRIDE__SHIFT                                                             0x1e
+#define IH_CLK_CTRL__REG_CLK_SOFT_OVERRIDE__SHIFT                                                             0x1f
+#define IH_CLK_CTRL__IH_RAS_CLK_SOFT_OVERRIDE_MASK                                                            0x00400000L
+#define IH_CLK_CTRL__IH_PASID_LUT_MEM_CLK_SOFT_OVERRIDE_MASK                                                  0x00800000L
+#define IH_CLK_CTRL__MSI_STORM_COUNTER_CLK_SOFT_OVERRIDE_MASK                                                 0x01000000L
+#define IH_CLK_CTRL__IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE_MASK                                              0x02000000L
+#define IH_CLK_CTRL__IH_BUFFER_MEM_CLK_SOFT_OVERRIDE_MASK                                                     0x04000000L
+#define IH_CLK_CTRL__DBUS_MUX_CLK_SOFT_OVERRIDE_MASK                                                          0x08000000L
+#define IH_CLK_CTRL__OSSSYS_SHARE_CLK_SOFT_OVERRIDE_MASK                                                      0x10000000L
+#define IH_CLK_CTRL__LIMIT_SMN_CLK_SOFT_OVERRIDE_MASK                                                         0x20000000L
+#define IH_CLK_CTRL__DYN_CLK_SOFT_OVERRIDE_MASK                                                               0x40000000L
+#define IH_CLK_CTRL__REG_CLK_SOFT_OVERRIDE_MASK                                                               0x80000000L
+//IH_STORM_CLIENT_LIST_CNTL
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT1_IS_STORM_CLIENT__SHIFT                                             0x1
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT2_IS_STORM_CLIENT__SHIFT                                             0x2
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT3_IS_STORM_CLIENT__SHIFT                                             0x3
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT4_IS_STORM_CLIENT__SHIFT                                             0x4
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT5_IS_STORM_CLIENT__SHIFT                                             0x5
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT6_IS_STORM_CLIENT__SHIFT                                             0x6
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT7_IS_STORM_CLIENT__SHIFT                                             0x7
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT8_IS_STORM_CLIENT__SHIFT                                             0x8
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT9_IS_STORM_CLIENT__SHIFT                                             0x9
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT10_IS_STORM_CLIENT__SHIFT                                            0xa
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT11_IS_STORM_CLIENT__SHIFT                                            0xb
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT12_IS_STORM_CLIENT__SHIFT                                            0xc
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT13_IS_STORM_CLIENT__SHIFT                                            0xd
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT14_IS_STORM_CLIENT__SHIFT                                            0xe
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT15_IS_STORM_CLIENT__SHIFT                                            0xf
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT16_IS_STORM_CLIENT__SHIFT                                            0x10
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT17_IS_STORM_CLIENT__SHIFT                                            0x11
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT18_IS_STORM_CLIENT__SHIFT                                            0x12
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT19_IS_STORM_CLIENT__SHIFT                                            0x13
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT20_IS_STORM_CLIENT__SHIFT                                            0x14
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT21_IS_STORM_CLIENT__SHIFT                                            0x15
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT22_IS_STORM_CLIENT__SHIFT                                            0x16
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT23_IS_STORM_CLIENT__SHIFT                                            0x17
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT24_IS_STORM_CLIENT__SHIFT                                            0x18
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT25_IS_STORM_CLIENT__SHIFT                                            0x19
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT26_IS_STORM_CLIENT__SHIFT                                            0x1a
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT27_IS_STORM_CLIENT__SHIFT                                            0x1b
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT28_IS_STORM_CLIENT__SHIFT                                            0x1c
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT29_IS_STORM_CLIENT__SHIFT                                            0x1d
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT30_IS_STORM_CLIENT__SHIFT                                            0x1e
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT31_IS_STORM_CLIENT__SHIFT                                            0x1f
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT1_IS_STORM_CLIENT_MASK                                               0x00000002L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT2_IS_STORM_CLIENT_MASK                                               0x00000004L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT3_IS_STORM_CLIENT_MASK                                               0x00000008L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT4_IS_STORM_CLIENT_MASK                                               0x00000010L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT5_IS_STORM_CLIENT_MASK                                               0x00000020L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT6_IS_STORM_CLIENT_MASK                                               0x00000040L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT7_IS_STORM_CLIENT_MASK                                               0x00000080L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT8_IS_STORM_CLIENT_MASK                                               0x00000100L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT9_IS_STORM_CLIENT_MASK                                               0x00000200L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT10_IS_STORM_CLIENT_MASK                                              0x00000400L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT11_IS_STORM_CLIENT_MASK                                              0x00000800L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT12_IS_STORM_CLIENT_MASK                                              0x00001000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT13_IS_STORM_CLIENT_MASK                                              0x00002000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT14_IS_STORM_CLIENT_MASK                                              0x00004000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT15_IS_STORM_CLIENT_MASK                                              0x00008000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT16_IS_STORM_CLIENT_MASK                                              0x00010000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT17_IS_STORM_CLIENT_MASK                                              0x00020000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT18_IS_STORM_CLIENT_MASK                                              0x00040000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT19_IS_STORM_CLIENT_MASK                                              0x00080000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT20_IS_STORM_CLIENT_MASK                                              0x00100000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT21_IS_STORM_CLIENT_MASK                                              0x00200000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT22_IS_STORM_CLIENT_MASK                                              0x00400000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT23_IS_STORM_CLIENT_MASK                                              0x00800000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT24_IS_STORM_CLIENT_MASK                                              0x01000000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT25_IS_STORM_CLIENT_MASK                                              0x02000000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT26_IS_STORM_CLIENT_MASK                                              0x04000000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT27_IS_STORM_CLIENT_MASK                                              0x08000000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT28_IS_STORM_CLIENT_MASK                                              0x10000000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT29_IS_STORM_CLIENT_MASK                                              0x20000000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT30_IS_STORM_CLIENT_MASK                                              0x40000000L
+#define IH_STORM_CLIENT_LIST_CNTL__CLIENT31_IS_STORM_CLIENT_MASK                                              0x80000000L
+//IH_LIMIT_INT_RATE_CNTL
+#define IH_LIMIT_INT_RATE_CNTL__LIMIT_ENABLE__SHIFT                                                           0x0
+#define IH_LIMIT_INT_RATE_CNTL__PERF_INTERVAL__SHIFT                                                          0x1
+#define IH_LIMIT_INT_RATE_CNTL__PERF_THRESHOLD__SHIFT                                                         0x5
+#define IH_LIMIT_INT_RATE_CNTL__RETURN_DELAY__SHIFT                                                           0x11
+#define IH_LIMIT_INT_RATE_CNTL__PERF_RESULT__SHIFT                                                            0x15
+#define IH_LIMIT_INT_RATE_CNTL__LIMIT_ENABLE_MASK                                                             0x00000001L
+#define IH_LIMIT_INT_RATE_CNTL__PERF_INTERVAL_MASK                                                            0x0000001EL
+#define IH_LIMIT_INT_RATE_CNTL__PERF_THRESHOLD_MASK                                                           0x0000FFE0L
+#define IH_LIMIT_INT_RATE_CNTL__RETURN_DELAY_MASK                                                             0x001E0000L
+#define IH_LIMIT_INT_RATE_CNTL__PERF_RESULT_MASK                                                              0xFFE00000L
+//IH_RETRY_INT_CAM_CNTL
+#define IH_RETRY_INT_CAM_CNTL__CAM_SIZE__SHIFT                                                                0x0
+#define IH_RETRY_INT_CAM_CNTL__BACK_PRESSURE_SKID_VALUE__SHIFT                                                0x8
+#define IH_RETRY_INT_CAM_CNTL__ENABLE__SHIFT                                                                  0x10
+#define IH_RETRY_INT_CAM_CNTL__MM_BACK_PRESSURE_ENABLE__SHIFT                                                 0x11
+#define IH_RETRY_INT_CAM_CNTL__GC_BACK_PRESSURE_ENABLE__SHIFT                                                 0x12
+#define IH_RETRY_INT_CAM_CNTL__PER_VF_ENTRY_SIZE__SHIFT                                                       0x14
+#define IH_RETRY_INT_CAM_CNTL__CAM_SIZE_MASK                                                                  0x0000001FL
+#define IH_RETRY_INT_CAM_CNTL__BACK_PRESSURE_SKID_VALUE_MASK                                                  0x00003F00L
+#define IH_RETRY_INT_CAM_CNTL__ENABLE_MASK                                                                    0x00010000L
+#define IH_RETRY_INT_CAM_CNTL__MM_BACK_PRESSURE_ENABLE_MASK                                                   0x00020000L
+#define IH_RETRY_INT_CAM_CNTL__GC_BACK_PRESSURE_ENABLE_MASK                                                   0x00040000L
+#define IH_RETRY_INT_CAM_CNTL__PER_VF_ENTRY_SIZE_MASK                                                         0x00300000L
+//IH_MEM_POWER_CTRL
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_CTRL_EN__SHIFT                                                 0x0
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_LS_EN__SHIFT                                                   0x1
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_DS_EN__SHIFT                                                   0x2
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_SD_EN__SHIFT                                                   0x3
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_IDLE_HYSTERESIS__SHIFT                                               0x4
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_UP_RECOVER_DELAY__SHIFT                                        0x7
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_DOWN_ENTER_DELAY__SHIFT                                        0xe
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_CTRL_EN__SHIFT                                          0x10
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_LS_EN__SHIFT                                            0x11
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_DS_EN__SHIFT                                            0x12
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_SD_EN__SHIFT                                            0x13
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_IDLE_HYSTERESIS__SHIFT                                        0x14
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_UP_RECOVER_DELAY__SHIFT                                 0x17
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_DOWN_ENTER_DELAY__SHIFT                                 0x1e
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_CTRL_EN_MASK                                                   0x00000001L
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_LS_EN_MASK                                                     0x00000002L
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_DS_EN_MASK                                                     0x00000004L
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_SD_EN_MASK                                                     0x00000008L
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_IDLE_HYSTERESIS_MASK                                                 0x00000070L
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_UP_RECOVER_DELAY_MASK                                          0x00003F80L
+#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_DOWN_ENTER_DELAY_MASK                                          0x0000C000L
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_CTRL_EN_MASK                                            0x00010000L
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_LS_EN_MASK                                              0x00020000L
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_DS_EN_MASK                                              0x00040000L
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_SD_EN_MASK                                              0x00080000L
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_IDLE_HYSTERESIS_MASK                                          0x00700000L
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_UP_RECOVER_DELAY_MASK                                   0x3F800000L
+#define IH_MEM_POWER_CTRL__IH_RETRY_INT_CAM_MEM_POWER_DOWN_ENTER_DELAY_MASK                                   0xC0000000L
+//IH_MEM_POWER_CTRL2
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_CTRL_EN__SHIFT                                             0x0
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_LS_EN__SHIFT                                               0x1
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_DS_EN__SHIFT                                               0x2
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_SD_EN__SHIFT                                               0x3
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_IDLE_HYSTERESIS__SHIFT                                           0x4
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_UP_RECOVER_DELAY__SHIFT                                    0x7
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_DOWN_ENTER_DELAY__SHIFT                                    0xe
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_CTRL_EN_MASK                                               0x00000001L
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_LS_EN_MASK                                                 0x00000002L
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_DS_EN_MASK                                                 0x00000004L
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_SD_EN_MASK                                                 0x00000008L
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_IDLE_HYSTERESIS_MASK                                             0x00000070L
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_UP_RECOVER_DELAY_MASK                                      0x00003F80L
+#define IH_MEM_POWER_CTRL2__IH_PASID_LUT_MEM_POWER_DOWN_ENTER_DELAY_MASK                                      0x0000C000L
+//IH_CNTL2
+#define IH_CNTL2__SELF_IV_FORCE_WPTR_UPDATE_TIMEOUT__SHIFT                                                    0x0
+#define IH_CNTL2__SELF_IV_FORCE_WPTR_UPDATE_ENABLE__SHIFT                                                     0x8
+#define IH_CNTL2__SELF_IV_FORCE_WPTR_UPDATE_TIMEOUT_MASK                                                      0x0000001FL
+#define IH_CNTL2__SELF_IV_FORCE_WPTR_UPDATE_ENABLE_MASK                                                       0x00000100L
+//IH_STATUS
+#define IH_STATUS__IDLE__SHIFT                                                                                0x0
+#define IH_STATUS__INPUT_IDLE__SHIFT                                                                          0x1
+#define IH_STATUS__BUFFER_IDLE__SHIFT                                                                         0x2
+#define IH_STATUS__RB_FULL__SHIFT                                                                             0x3
+#define IH_STATUS__RB_FULL_DRAIN__SHIFT                                                                       0x4
+#define IH_STATUS__RB_OVERFLOW__SHIFT                                                                         0x5
+#define IH_STATUS__MC_WR_IDLE__SHIFT                                                                          0x6
+#define IH_STATUS__MC_WR_STALL__SHIFT                                                                         0x7
+#define IH_STATUS__MC_WR_CLEAN_PENDING__SHIFT                                                                 0x8
+#define IH_STATUS__MC_WR_CLEAN_STALL__SHIFT                                                                   0x9
+#define IH_STATUS__BIF_INTERRUPT_LINE__SHIFT                                                                  0xa
+#define IH_STATUS__RB1_FULL__SHIFT                                                                            0xc
+#define IH_STATUS__RB1_FULL_DRAIN__SHIFT                                                                      0xd
+#define IH_STATUS__RB1_OVERFLOW__SHIFT                                                                        0xe
+#define IH_STATUS__SELF_INT_GEN_IDLE__SHIFT                                                                   0x12
+#define IH_STATUS__RETRY_INT_CAM_IDLE__SHIFT                                                                  0x13
+#define IH_STATUS__ZSTATES_FENCE__SHIFT                                                                       0x14
+#define IH_STATUS__IH_BUFFER_MEM_POWER_GATED__SHIFT                                                           0x15
+#define IH_STATUS__IH_RETRY_INT_CAM_MEM_POWER_GATED__SHIFT                                                    0x16
+#define IH_STATUS__IH_PASID_LUT_MEM_POWER_GATED__SHIFT                                                        0x17
+#define IH_STATUS__IDLE_MASK                                                                                  0x00000001L
+#define IH_STATUS__INPUT_IDLE_MASK                                                                            0x00000002L
+#define IH_STATUS__BUFFER_IDLE_MASK                                                                           0x00000004L
+#define IH_STATUS__RB_FULL_MASK                                                                               0x00000008L
+#define IH_STATUS__RB_FULL_DRAIN_MASK                                                                         0x00000010L
+#define IH_STATUS__RB_OVERFLOW_MASK                                                                           0x00000020L
+#define IH_STATUS__MC_WR_IDLE_MASK                                                                            0x00000040L
+#define IH_STATUS__MC_WR_STALL_MASK                                                                           0x00000080L
+#define IH_STATUS__MC_WR_CLEAN_PENDING_MASK                                                                   0x00000100L
+#define IH_STATUS__MC_WR_CLEAN_STALL_MASK                                                                     0x00000200L
+#define IH_STATUS__BIF_INTERRUPT_LINE_MASK                                                                    0x00000400L
+#define IH_STATUS__RB1_FULL_MASK                                                                              0x00001000L
+#define IH_STATUS__RB1_FULL_DRAIN_MASK                                                                        0x00002000L
+#define IH_STATUS__RB1_OVERFLOW_MASK                                                                          0x00004000L
+#define IH_STATUS__SELF_INT_GEN_IDLE_MASK                                                                     0x00040000L
+#define IH_STATUS__RETRY_INT_CAM_IDLE_MASK                                                                    0x00080000L
+#define IH_STATUS__ZSTATES_FENCE_MASK                                                                         0x00100000L
+#define IH_STATUS__IH_BUFFER_MEM_POWER_GATED_MASK                                                             0x00200000L
+#define IH_STATUS__IH_RETRY_INT_CAM_MEM_POWER_GATED_MASK                                                      0x00400000L
+#define IH_STATUS__IH_PASID_LUT_MEM_POWER_GATED_MASK                                                          0x00800000L
+//IH_PERFMON_CNTL
+#define IH_PERFMON_CNTL__ENABLE0__SHIFT                                                                       0x0
+#define IH_PERFMON_CNTL__CLEAR0__SHIFT                                                                        0x1
+#define IH_PERFMON_CNTL__PERF_SEL0__SHIFT                                                                     0x2
+#define IH_PERFMON_CNTL__ENABLE1__SHIFT                                                                       0x10
+#define IH_PERFMON_CNTL__CLEAR1__SHIFT                                                                        0x11
+#define IH_PERFMON_CNTL__PERF_SEL1__SHIFT                                                                     0x12
+#define IH_PERFMON_CNTL__ENABLE0_MASK                                                                         0x00000001L
+#define IH_PERFMON_CNTL__CLEAR0_MASK                                                                          0x00000002L
+#define IH_PERFMON_CNTL__PERF_SEL0_MASK                                                                       0x00000FFCL
+#define IH_PERFMON_CNTL__ENABLE1_MASK                                                                         0x00010000L
+#define IH_PERFMON_CNTL__CLEAR1_MASK                                                                          0x00020000L
+#define IH_PERFMON_CNTL__PERF_SEL1_MASK                                                                       0x0FFC0000L
+//IH_PERFCOUNTER0_RESULT
+#define IH_PERFCOUNTER0_RESULT__PERF_COUNT__SHIFT                                                             0x0
+#define IH_PERFCOUNTER0_RESULT__PERF_COUNT_MASK                                                               0xFFFFFFFFL
+//IH_PERFCOUNTER1_RESULT
+#define IH_PERFCOUNTER1_RESULT__PERF_COUNT__SHIFT                                                             0x0
+#define IH_PERFCOUNTER1_RESULT__PERF_COUNT_MASK                                                               0xFFFFFFFFL
+//IH_DSM_MATCH_VALUE_BIT_31_0
+#define IH_DSM_MATCH_VALUE_BIT_31_0__VALUE__SHIFT                                                             0x0
+#define IH_DSM_MATCH_VALUE_BIT_31_0__VALUE_MASK                                                               0xFFFFFFFFL
+//IH_DSM_MATCH_VALUE_BIT_63_32
+#define IH_DSM_MATCH_VALUE_BIT_63_32__VALUE__SHIFT                                                            0x0
+#define IH_DSM_MATCH_VALUE_BIT_63_32__VALUE_MASK                                                              0xFFFFFFFFL
+//IH_DSM_MATCH_VALUE_BIT_95_64
+#define IH_DSM_MATCH_VALUE_BIT_95_64__VALUE__SHIFT                                                            0x0
+#define IH_DSM_MATCH_VALUE_BIT_95_64__VALUE_MASK                                                              0xFFFFFFFFL
+//IH_DSM_MATCH_FIELD_CONTROL
+#define IH_DSM_MATCH_FIELD_CONTROL__SRC_EN__SHIFT                                                             0x0
+#define IH_DSM_MATCH_FIELD_CONTROL__FCNID_EN__SHIFT                                                           0x1
+#define IH_DSM_MATCH_FIELD_CONTROL__TIMESTAMP_EN__SHIFT                                                       0x2
+#define IH_DSM_MATCH_FIELD_CONTROL__RINGID_EN__SHIFT                                                          0x3
+#define IH_DSM_MATCH_FIELD_CONTROL__VMID_EN__SHIFT                                                            0x4
+#define IH_DSM_MATCH_FIELD_CONTROL__PASID_EN__SHIFT                                                           0x5
+#define IH_DSM_MATCH_FIELD_CONTROL__CLIENT_ID_EN__SHIFT                                                       0x6
+#define IH_DSM_MATCH_FIELD_CONTROL__DIEID_EN__SHIFT                                                           0x7
+#define IH_DSM_MATCH_FIELD_CONTROL__SRC_EN_MASK                                                               0x00000001L
+#define IH_DSM_MATCH_FIELD_CONTROL__FCNID_EN_MASK                                                             0x00000002L
+#define IH_DSM_MATCH_FIELD_CONTROL__TIMESTAMP_EN_MASK                                                         0x00000004L
+#define IH_DSM_MATCH_FIELD_CONTROL__RINGID_EN_MASK                                                            0x00000008L
+#define IH_DSM_MATCH_FIELD_CONTROL__VMID_EN_MASK                                                              0x00000010L
+#define IH_DSM_MATCH_FIELD_CONTROL__PASID_EN_MASK                                                             0x00000020L
+#define IH_DSM_MATCH_FIELD_CONTROL__CLIENT_ID_EN_MASK                                                         0x00000040L
+#define IH_DSM_MATCH_FIELD_CONTROL__DIEID_EN_MASK                                                             0x00000080L
+//IH_DSM_MATCH_DATA_CONTROL
+#define IH_DSM_MATCH_DATA_CONTROL__VALUE__SHIFT                                                               0x0
+#define IH_DSM_MATCH_DATA_CONTROL__VALUE_MASK                                                                 0x0FFFFFFFL
+//IH_DSM_MATCH_FCN_ID
+#define IH_DSM_MATCH_FCN_ID__VF_ID__SHIFT                                                                     0x0
+#define IH_DSM_MATCH_FCN_ID__PF_VF__SHIFT                                                                     0x7
+#define IH_DSM_MATCH_FCN_ID__VF_ID_MASK                                                                       0x0000001FL
+#define IH_DSM_MATCH_FCN_ID__PF_VF_MASK                                                                       0x00000080L
+//IH_VF_RB_STATUS
+#define IH_VF_RB_STATUS__RB_FULL_DRAIN_VF__SHIFT                                                              0x0
+#define IH_VF_RB_STATUS__RB_FULL_DRAIN_VF_MASK                                                                0x000000FFL
+//IH_VF_RB_STATUS2
+#define IH_VF_RB_STATUS2__RB_FULL_VF__SHIFT                                                                   0x0
+#define IH_VF_RB_STATUS2__RB_FULL_VF_MASK                                                                     0x000000FFL
+//IH_VF_RB1_STATUS
+#define IH_VF_RB1_STATUS__RB_FULL_DRAIN_VF__SHIFT                                                             0x0
+#define IH_VF_RB1_STATUS__RB_FULL_DRAIN_VF_MASK                                                               0x000000FFL
+//IH_VF_RB1_STATUS2
+#define IH_VF_RB1_STATUS2__RB_FULL_VF__SHIFT                                                                  0x0
+#define IH_VF_RB1_STATUS2__RB_FULL_VF_MASK                                                                    0x000000FFL
+//IH_RB_STATUS
+#define IH_RB_STATUS__RB_FULL__SHIFT                                                                          0x0
+#define IH_RB_STATUS__RB_FULL_DRAIN__SHIFT                                                                    0x1
+#define IH_RB_STATUS__RB_OVERFLOW__SHIFT                                                                      0x2
+#define IH_RB_STATUS__RB1_FULL__SHIFT                                                                         0x4
+#define IH_RB_STATUS__RB1_FULL_DRAIN__SHIFT                                                                   0x5
+#define IH_RB_STATUS__RB1_OVERFLOW__SHIFT                                                                     0x6
+#define IH_RB_STATUS__RB_FULL_MASK                                                                            0x00000001L
+#define IH_RB_STATUS__RB_FULL_DRAIN_MASK                                                                      0x00000002L
+#define IH_RB_STATUS__RB_OVERFLOW_MASK                                                                        0x00000004L
+#define IH_RB_STATUS__RB1_FULL_MASK                                                                           0x00000010L
+#define IH_RB_STATUS__RB1_FULL_DRAIN_MASK                                                                     0x00000020L
+#define IH_RB_STATUS__RB1_OVERFLOW_MASK                                                                       0x00000040L
+//IH_INT_FLOOD_CNTL
+#define IH_INT_FLOOD_CNTL__HIGHWATER__SHIFT                                                                   0x0
+#define IH_INT_FLOOD_CNTL__FLOOD_CNTL_ENABLE__SHIFT                                                           0x3
+#define IH_INT_FLOOD_CNTL__FLOOD_CNTL_MODE__SHIFT                                                             0x4
+#define IH_INT_FLOOD_CNTL__CLEAR_INT_FLOOD_STATUS__SHIFT                                                      0x5
+#define IH_INT_FLOOD_CNTL__HIGHWATER_MASK                                                                     0x00000007L
+#define IH_INT_FLOOD_CNTL__FLOOD_CNTL_ENABLE_MASK                                                             0x00000008L
+#define IH_INT_FLOOD_CNTL__CLEAR_INT_FLOOD_STATUS_MASK                                                        0x00000020L
+//IH_RB0_INT_FLOOD_STATUS
+#define IH_RB0_INT_FLOOD_STATUS__RB_INT_DROPPED_VF__SHIFT                                                     0x0
+#define IH_RB0_INT_FLOOD_STATUS__RB_INT_DROPPED__SHIFT                                                        0x1f
+#define IH_RB0_INT_FLOOD_STATUS__RB_INT_DROPPED_VF_MASK                                                       0x000000FFL
+#define IH_RB0_INT_FLOOD_STATUS__RB_INT_DROPPED_MASK                                                          0x80000000L
+//IH_RB1_INT_FLOOD_STATUS
+#define IH_RB1_INT_FLOOD_STATUS__RB_INT_DROPPED_VF__SHIFT                                                     0x0
+#define IH_RB1_INT_FLOOD_STATUS__RB_INT_DROPPED__SHIFT                                                        0x1f
+#define IH_RB1_INT_FLOOD_STATUS__RB_INT_DROPPED_VF_MASK                                                       0x000000FFL
+#define IH_RB1_INT_FLOOD_STATUS__RB_INT_DROPPED_MASK                                                          0x80000000L
+//IH_INT_FLOOD_STATUS
+#define IH_INT_FLOOD_STATUS__INT_DROP_CNT__SHIFT                                                              0x0
+#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_CLIENT_ID__SHIFT                                                  0x8
+#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_SOURCE_ID__SHIFT                                                  0x10
+#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_VF_ID__SHIFT                                                      0x18
+#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_VF__SHIFT                                                         0x1d
+#define IH_INT_FLOOD_STATUS__INT_DROPPED__SHIFT                                                               0x1e
+#define IH_INT_FLOOD_STATUS__INT_DROP_CNT_MASK                                                                0x000000FFL
+#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_CLIENT_ID_MASK                                                    0x0000FF00L
+#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_SOURCE_ID_MASK                                                    0x00FF0000L
+#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_VF_ID_MASK                                                        0x1F000000L
+#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_VF_MASK                                                           0x20000000L
+#define IH_INT_FLOOD_STATUS__INT_DROPPED_MASK                                                                 0x40000000L
+//IH_INT_FLAGS
+#define IH_INT_FLAGS__CLIENT_0_FLAG__SHIFT                                                                    0x0
+#define IH_INT_FLAGS__CLIENT_1_FLAG__SHIFT                                                                    0x1
+#define IH_INT_FLAGS__CLIENT_2_FLAG__SHIFT                                                                    0x2
+#define IH_INT_FLAGS__CLIENT_3_FLAG__SHIFT                                                                    0x3
+#define IH_INT_FLAGS__CLIENT_4_FLAG__SHIFT                                                                    0x4
+#define IH_INT_FLAGS__CLIENT_5_FLAG__SHIFT                                                                    0x5
+#define IH_INT_FLAGS__CLIENT_6_FLAG__SHIFT                                                                    0x6
+#define IH_INT_FLAGS__CLIENT_7_FLAG__SHIFT                                                                    0x7
+#define IH_INT_FLAGS__CLIENT_8_FLAG__SHIFT                                                                    0x8
+#define IH_INT_FLAGS__CLIENT_9_FLAG__SHIFT                                                                    0x9
+#define IH_INT_FLAGS__CLIENT_10_FLAG__SHIFT                                                                   0xa
+#define IH_INT_FLAGS__CLIENT_11_FLAG__SHIFT                                                                   0xb
+#define IH_INT_FLAGS__CLIENT_12_FLAG__SHIFT                                                                   0xc
+#define IH_INT_FLAGS__CLIENT_13_FLAG__SHIFT                                                                   0xd
+#define IH_INT_FLAGS__CLIENT_14_FLAG__SHIFT                                                                   0xe
+#define IH_INT_FLAGS__CLIENT_15_FLAG__SHIFT                                                                   0xf
+#define IH_INT_FLAGS__CLIENT_16_FLAG__SHIFT                                                                   0x10
+#define IH_INT_FLAGS__CLIENT_17_FLAG__SHIFT                                                                   0x11
+#define IH_INT_FLAGS__CLIENT_18_FLAG__SHIFT                                                                   0x12
+#define IH_INT_FLAGS__CLIENT_19_FLAG__SHIFT                                                                   0x13
+#define IH_INT_FLAGS__CLIENT_20_FLAG__SHIFT                                                                   0x14
+#define IH_INT_FLAGS__CLIENT_21_FLAG__SHIFT                                                                   0x15
+#define IH_INT_FLAGS__CLIENT_22_FLAG__SHIFT                                                                   0x16
+#define IH_INT_FLAGS__CLIENT_23_FLAG__SHIFT                                                                   0x17
+#define IH_INT_FLAGS__CLIENT_24_FLAG__SHIFT                                                                   0x18
+#define IH_INT_FLAGS__CLIENT_25_FLAG__SHIFT                                                                   0x19
+#define IH_INT_FLAGS__CLIENT_26_FLAG__SHIFT                                                                   0x1a
+#define IH_INT_FLAGS__CLIENT_27_FLAG__SHIFT                                                                   0x1b
+#define IH_INT_FLAGS__CLIENT_28_FLAG__SHIFT                                                                   0x1c
+#define IH_INT_FLAGS__CLIENT_29_FLAG__SHIFT                                                                   0x1d
+#define IH_INT_FLAGS__CLIENT_30_FLAG__SHIFT                                                                   0x1e
+#define IH_INT_FLAGS__CLIENT_31_FLAG__SHIFT                                                                   0x1f
+#define IH_INT_FLAGS__CLIENT_0_FLAG_MASK                                                                      0x00000001L
+#define IH_INT_FLAGS__CLIENT_1_FLAG_MASK                                                                      0x00000002L
+#define IH_INT_FLAGS__CLIENT_2_FLAG_MASK                                                                      0x00000004L
+#define IH_INT_FLAGS__CLIENT_3_FLAG_MASK                                                                      0x00000008L
+#define IH_INT_FLAGS__CLIENT_4_FLAG_MASK                                                                      0x00000010L
+#define IH_INT_FLAGS__CLIENT_5_FLAG_MASK                                                                      0x00000020L
+#define IH_INT_FLAGS__CLIENT_6_FLAG_MASK                                                                      0x00000040L
+#define IH_INT_FLAGS__CLIENT_7_FLAG_MASK                                                                      0x00000080L
+#define IH_INT_FLAGS__CLIENT_8_FLAG_MASK                                                                      0x00000100L
+#define IH_INT_FLAGS__CLIENT_9_FLAG_MASK                                                                      0x00000200L
+#define IH_INT_FLAGS__CLIENT_10_FLAG_MASK                                                                     0x00000400L
+#define IH_INT_FLAGS__CLIENT_11_FLAG_MASK                                                                     0x00000800L
+#define IH_INT_FLAGS__CLIENT_12_FLAG_MASK                                                                     0x00001000L
+#define IH_INT_FLAGS__CLIENT_13_FLAG_MASK                                                                     0x00002000L
+#define IH_INT_FLAGS__CLIENT_14_FLAG_MASK                                                                     0x00004000L
+#define IH_INT_FLAGS__CLIENT_15_FLAG_MASK                                                                     0x00008000L
+#define IH_INT_FLAGS__CLIENT_16_FLAG_MASK                                                                     0x00010000L
+#define IH_INT_FLAGS__CLIENT_17_FLAG_MASK                                                                     0x00020000L
+#define IH_INT_FLAGS__CLIENT_18_FLAG_MASK                                                                     0x00040000L
+#define IH_INT_FLAGS__CLIENT_19_FLAG_MASK                                                                     0x00080000L
+#define IH_INT_FLAGS__CLIENT_20_FLAG_MASK                                                                     0x00100000L
+#define IH_INT_FLAGS__CLIENT_21_FLAG_MASK                                                                     0x00200000L
+#define IH_INT_FLAGS__CLIENT_22_FLAG_MASK                                                                     0x00400000L
+#define IH_INT_FLAGS__CLIENT_23_FLAG_MASK                                                                     0x00800000L
+#define IH_INT_FLAGS__CLIENT_24_FLAG_MASK                                                                     0x01000000L
+#define IH_INT_FLAGS__CLIENT_25_FLAG_MASK                                                                     0x02000000L
+#define IH_INT_FLAGS__CLIENT_26_FLAG_MASK                                                                     0x04000000L
+#define IH_INT_FLAGS__CLIENT_27_FLAG_MASK                                                                     0x08000000L
+#define IH_INT_FLAGS__CLIENT_28_FLAG_MASK                                                                     0x10000000L
+#define IH_INT_FLAGS__CLIENT_29_FLAG_MASK                                                                     0x20000000L
+#define IH_INT_FLAGS__CLIENT_30_FLAG_MASK                                                                     0x40000000L
+#define IH_INT_FLAGS__CLIENT_31_FLAG_MASK                                                                     0x80000000L
+//IH_SCRATCH
+#define IH_SCRATCH__DATA__SHIFT                                                                               0x0
+#define IH_SCRATCH__DATA_MASK                                                                                 0xFFFFFFFFL
+//IH_CLIENT_CREDIT_ERROR
+#define IH_CLIENT_CREDIT_ERROR__CLEAR__SHIFT                                                                  0x0
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_1_ERROR__SHIFT                                                         0x1
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_2_ERROR__SHIFT                                                         0x2
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_3_ERROR__SHIFT                                                         0x3
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_4_ERROR__SHIFT                                                         0x4
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_5_ERROR__SHIFT                                                         0x5
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_6_ERROR__SHIFT                                                         0x6
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_7_ERROR__SHIFT                                                         0x7
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_8_ERROR__SHIFT                                                         0x8
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_9_ERROR__SHIFT                                                         0x9
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_10_ERROR__SHIFT                                                        0xa
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_11_ERROR__SHIFT                                                        0xb
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_12_ERROR__SHIFT                                                        0xc
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_13_ERROR__SHIFT                                                        0xd
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_14_ERROR__SHIFT                                                        0xe
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_15_ERROR__SHIFT                                                        0xf
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_16_ERROR__SHIFT                                                        0x10
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_17_ERROR__SHIFT                                                        0x11
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_18_ERROR__SHIFT                                                        0x12
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_19_ERROR__SHIFT                                                        0x13
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_20_ERROR__SHIFT                                                        0x14
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_21_ERROR__SHIFT                                                        0x15
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_22_ERROR__SHIFT                                                        0x16
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_23_ERROR__SHIFT                                                        0x17
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_24_ERROR__SHIFT                                                        0x18
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_25_ERROR__SHIFT                                                        0x19
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_26_ERROR__SHIFT                                                        0x1a
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_27_ERROR__SHIFT                                                        0x1b
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_28_ERROR__SHIFT                                                        0x1c
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_29_ERROR__SHIFT                                                        0x1d
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_30_ERROR__SHIFT                                                        0x1e
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_31_ERROR__SHIFT                                                        0x1f
+#define IH_CLIENT_CREDIT_ERROR__CLEAR_MASK                                                                    0x00000001L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_1_ERROR_MASK                                                           0x00000002L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_2_ERROR_MASK                                                           0x00000004L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_3_ERROR_MASK                                                           0x00000008L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_4_ERROR_MASK                                                           0x00000010L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_5_ERROR_MASK                                                           0x00000020L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_6_ERROR_MASK                                                           0x00000040L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_7_ERROR_MASK                                                           0x00000080L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_8_ERROR_MASK                                                           0x00000100L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_9_ERROR_MASK                                                           0x00000200L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_10_ERROR_MASK                                                          0x00000400L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_11_ERROR_MASK                                                          0x00000800L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_12_ERROR_MASK                                                          0x00001000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_13_ERROR_MASK                                                          0x00002000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_14_ERROR_MASK                                                          0x00004000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_15_ERROR_MASK                                                          0x00008000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_16_ERROR_MASK                                                          0x00010000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_17_ERROR_MASK                                                          0x00020000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_18_ERROR_MASK                                                          0x00040000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_19_ERROR_MASK                                                          0x00080000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_20_ERROR_MASK                                                          0x00100000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_21_ERROR_MASK                                                          0x00200000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_22_ERROR_MASK                                                          0x00400000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_23_ERROR_MASK                                                          0x00800000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_24_ERROR_MASK                                                          0x01000000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_25_ERROR_MASK                                                          0x02000000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_26_ERROR_MASK                                                          0x04000000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_27_ERROR_MASK                                                          0x08000000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_28_ERROR_MASK                                                          0x10000000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_29_ERROR_MASK                                                          0x20000000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_30_ERROR_MASK                                                          0x40000000L
+#define IH_CLIENT_CREDIT_ERROR__CLIENT_31_ERROR_MASK                                                          0x80000000L
+//IH_GPU_IOV_VIOLATION_LOG
+#define IH_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT                                                     0x0
+#define IH_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT                                            0x1
+#define IH_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT                                                              0x2
+#define IH_GPU_IOV_VIOLATION_LOG__OPCODE__SHIFT                                                               0x16
+#define IH_GPU_IOV_VIOLATION_LOG__VF__SHIFT                                                                   0x17
+#define IH_GPU_IOV_VIOLATION_LOG__VF_ID__SHIFT                                                                0x18
+#define IH_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK                                                       0x00000001L
+#define IH_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK                                              0x00000002L
+#define IH_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK                                                                0x000FFFFCL
+#define IH_GPU_IOV_VIOLATION_LOG__OPCODE_MASK                                                                 0x00400000L
+#define IH_GPU_IOV_VIOLATION_LOG__VF_MASK                                                                     0x00800000L
+#define IH_GPU_IOV_VIOLATION_LOG__VF_ID_MASK                                                                  0x1F000000L
+//IH_GPU_IOV_VIOLATION_LOG2
+#define IH_GPU_IOV_VIOLATION_LOG2__INITIATOR_ID__SHIFT                                                        0x0
+#define IH_GPU_IOV_VIOLATION_LOG2__INITIATOR_ID_MASK                                                          0x000003FFL
+//IH_COOKIE_REC_VIOLATION_LOG
+#define IH_COOKIE_REC_VIOLATION_LOG__VIOLATION_STATUS__SHIFT                                                  0x0
+#define IH_COOKIE_REC_VIOLATION_LOG__CLIENT_ID__SHIFT                                                         0x8
+#define IH_COOKIE_REC_VIOLATION_LOG__INITIATOR_ID__SHIFT                                                      0x10
+#define IH_COOKIE_REC_VIOLATION_LOG__DIE_ID__SHIFT                                                            0x1a
+#define IH_COOKIE_REC_VIOLATION_LOG__VIOLATION_STATUS_MASK                                                    0x00000001L
+#define IH_COOKIE_REC_VIOLATION_LOG__CLIENT_ID_MASK                                                           0x0000FF00L
+#define IH_COOKIE_REC_VIOLATION_LOG__INITIATOR_ID_MASK                                                        0x03FF0000L
+#define IH_COOKIE_REC_VIOLATION_LOG__DIE_ID_MASK                                                              0x3C000000L
+//IH_CREDIT_STATUS
+#define IH_CREDIT_STATUS__CLIENT_1_CREDIT_RETURNED__SHIFT                                                     0x1
+#define IH_CREDIT_STATUS__CLIENT_2_CREDIT_RETURNED__SHIFT                                                     0x2
+#define IH_CREDIT_STATUS__CLIENT_3_CREDIT_RETURNED__SHIFT                                                     0x3
+#define IH_CREDIT_STATUS__CLIENT_4_CREDIT_RETURNED__SHIFT                                                     0x4
+#define IH_CREDIT_STATUS__CLIENT_5_CREDIT_RETURNED__SHIFT                                                     0x5
+#define IH_CREDIT_STATUS__CLIENT_6_CREDIT_RETURNED__SHIFT                                                     0x6
+#define IH_CREDIT_STATUS__CLIENT_7_CREDIT_RETURNED__SHIFT                                                     0x7
+#define IH_CREDIT_STATUS__CLIENT_8_CREDIT_RETURNED__SHIFT                                                     0x8
+#define IH_CREDIT_STATUS__CLIENT_9_CREDIT_RETURNED__SHIFT                                                     0x9
+#define IH_CREDIT_STATUS__CLIENT_10_CREDIT_RETURNED__SHIFT                                                    0xa
+#define IH_CREDIT_STATUS__CLIENT_11_CREDIT_RETURNED__SHIFT                                                    0xb
+#define IH_CREDIT_STATUS__CLIENT_12_CREDIT_RETURNED__SHIFT                                                    0xc
+#define IH_CREDIT_STATUS__CLIENT_13_CREDIT_RETURNED__SHIFT                                                    0xd
+#define IH_CREDIT_STATUS__CLIENT_14_CREDIT_RETURNED__SHIFT                                                    0xe
+#define IH_CREDIT_STATUS__CLIENT_15_CREDIT_RETURNED__SHIFT                                                    0xf
+#define IH_CREDIT_STATUS__CLIENT_16_CREDIT_RETURNED__SHIFT                                                    0x10
+#define IH_CREDIT_STATUS__CLIENT_17_CREDIT_RETURNED__SHIFT                                                    0x11
+#define IH_CREDIT_STATUS__CLIENT_18_CREDIT_RETURNED__SHIFT                                                    0x12
+#define IH_CREDIT_STATUS__CLIENT_19_CREDIT_RETURNED__SHIFT                                                    0x13
+#define IH_CREDIT_STATUS__CLIENT_20_CREDIT_RETURNED__SHIFT                                                    0x14
+#define IH_CREDIT_STATUS__CLIENT_21_CREDIT_RETURNED__SHIFT                                                    0x15
+#define IH_CREDIT_STATUS__CLIENT_22_CREDIT_RETURNED__SHIFT                                                    0x16
+#define IH_CREDIT_STATUS__CLIENT_23_CREDIT_RETURNED__SHIFT                                                    0x17
+#define IH_CREDIT_STATUS__CLIENT_24_CREDIT_RETURNED__SHIFT                                                    0x18
+#define IH_CREDIT_STATUS__CLIENT_25_CREDIT_RETURNED__SHIFT                                                    0x19
+#define IH_CREDIT_STATUS__CLIENT_26_CREDIT_RETURNED__SHIFT                                                    0x1a
+#define IH_CREDIT_STATUS__CLIENT_27_CREDIT_RETURNED__SHIFT                                                    0x1b
+#define IH_CREDIT_STATUS__CLIENT_28_CREDIT_RETURNED__SHIFT                                                    0x1c
+#define IH_CREDIT_STATUS__CLIENT_29_CREDIT_RETURNED__SHIFT                                                    0x1d
+#define IH_CREDIT_STATUS__CLIENT_30_CREDIT_RETURNED__SHIFT                                                    0x1e
+#define IH_CREDIT_STATUS__CLIENT_31_CREDIT_RETURNED__SHIFT                                                    0x1f
+#define IH_CREDIT_STATUS__CLIENT_1_CREDIT_RETURNED_MASK                                                       0x00000002L
+#define IH_CREDIT_STATUS__CLIENT_2_CREDIT_RETURNED_MASK                                                       0x00000004L
+#define IH_CREDIT_STATUS__CLIENT_3_CREDIT_RETURNED_MASK                                                       0x00000008L
+#define IH_CREDIT_STATUS__CLIENT_4_CREDIT_RETURNED_MASK                                                       0x00000010L
+#define IH_CREDIT_STATUS__CLIENT_5_CREDIT_RETURNED_MASK                                                       0x00000020L
+#define IH_CREDIT_STATUS__CLIENT_6_CREDIT_RETURNED_MASK                                                       0x00000040L
+#define IH_CREDIT_STATUS__CLIENT_7_CREDIT_RETURNED_MASK                                                       0x00000080L
+#define IH_CREDIT_STATUS__CLIENT_8_CREDIT_RETURNED_MASK                                                       0x00000100L
+#define IH_CREDIT_STATUS__CLIENT_9_CREDIT_RETURNED_MASK                                                       0x00000200L
+#define IH_CREDIT_STATUS__CLIENT_10_CREDIT_RETURNED_MASK                                                      0x00000400L
+#define IH_CREDIT_STATUS__CLIENT_11_CREDIT_RETURNED_MASK                                                      0x00000800L
+#define IH_CREDIT_STATUS__CLIENT_12_CREDIT_RETURNED_MASK                                                      0x00001000L
+#define IH_CREDIT_STATUS__CLIENT_13_CREDIT_RETURNED_MASK                                                      0x00002000L
+#define IH_CREDIT_STATUS__CLIENT_14_CREDIT_RETURNED_MASK                                                      0x00004000L
+#define IH_CREDIT_STATUS__CLIENT_15_CREDIT_RETURNED_MASK                                                      0x00008000L
+#define IH_CREDIT_STATUS__CLIENT_16_CREDIT_RETURNED_MASK                                                      0x00010000L
+#define IH_CREDIT_STATUS__CLIENT_17_CREDIT_RETURNED_MASK                                                      0x00020000L
+#define IH_CREDIT_STATUS__CLIENT_18_CREDIT_RETURNED_MASK                                                      0x00040000L
+#define IH_CREDIT_STATUS__CLIENT_19_CREDIT_RETURNED_MASK                                                      0x00080000L
+#define IH_CREDIT_STATUS__CLIENT_20_CREDIT_RETURNED_MASK                                                      0x00100000L
+#define IH_CREDIT_STATUS__CLIENT_21_CREDIT_RETURNED_MASK                                                      0x00200000L
+#define IH_CREDIT_STATUS__CLIENT_22_CREDIT_RETURNED_MASK                                                      0x00400000L
+#define IH_CREDIT_STATUS__CLIENT_23_CREDIT_RETURNED_MASK                                                      0x00800000L
+#define IH_CREDIT_STATUS__CLIENT_24_CREDIT_RETURNED_MASK                                                      0x01000000L
+#define IH_CREDIT_STATUS__CLIENT_25_CREDIT_RETURNED_MASK                                                      0x02000000L
+#define IH_CREDIT_STATUS__CLIENT_26_CREDIT_RETURNED_MASK                                                      0x04000000L
+#define IH_CREDIT_STATUS__CLIENT_27_CREDIT_RETURNED_MASK                                                      0x08000000L
+#define IH_CREDIT_STATUS__CLIENT_28_CREDIT_RETURNED_MASK                                                      0x10000000L
+#define IH_CREDIT_STATUS__CLIENT_29_CREDIT_RETURNED_MASK                                                      0x20000000L
+#define IH_CREDIT_STATUS__CLIENT_30_CREDIT_RETURNED_MASK                                                      0x40000000L
+#define IH_CREDIT_STATUS__CLIENT_31_CREDIT_RETURNED_MASK                                                      0x80000000L
+//IH_MMHUB_ERROR
+#define IH_MMHUB_ERROR__IH_BRESP_01__SHIFT                                                                    0x1
+#define IH_MMHUB_ERROR__IH_BRESP_10__SHIFT                                                                    0x2
+#define IH_MMHUB_ERROR__IH_BRESP_11__SHIFT                                                                    0x3
+#define IH_MMHUB_ERROR__IH_BUSER_NACK_01__SHIFT                                                               0x5
+#define IH_MMHUB_ERROR__IH_BUSER_NACK_10__SHIFT                                                               0x6
+#define IH_MMHUB_ERROR__IH_BUSER_NACK_11__SHIFT                                                               0x7
+#define IH_MMHUB_ERROR__IH_BRESP_01_MASK                                                                      0x00000002L
+#define IH_MMHUB_ERROR__IH_BRESP_10_MASK                                                                      0x00000004L
+#define IH_MMHUB_ERROR__IH_BRESP_11_MASK                                                                      0x00000008L
+#define IH_MMHUB_ERROR__IH_BUSER_NACK_01_MASK                                                                 0x00000020L
+#define IH_MMHUB_ERROR__IH_BUSER_NACK_10_MASK                                                                 0x00000040L
+#define IH_MMHUB_ERROR__IH_BUSER_NACK_11_MASK                                                                 0x00000080L
+//IH_VF_RB_STATUS3
+#define IH_VF_RB_STATUS3__RB_OVERFLOW_VF__SHIFT                                                               0x0
+#define IH_VF_RB_STATUS3__RB_OVERFLOW_VF_MASK                                                                 0x000000FFL
+//IH_VF_RB_STATUS4
+#define IH_VF_RB_STATUS4__BIF_INTERRUPT_LINE_VF__SHIFT                                                        0x0
+#define IH_VF_RB_STATUS4__BIF_INTERRUPT_LINE_VF_MASK                                                          0x000000FFL
+//IH_VF_RB1_STATUS3
+#define IH_VF_RB1_STATUS3__RB_OVERFLOW_VF__SHIFT                                                              0x0
+#define IH_VF_RB1_STATUS3__RB_OVERFLOW_VF_MASK                                                                0x000000FFL
+//IH_MSI_STORM_CTRL
+#define IH_MSI_STORM_CTRL__DELAY__SHIFT                                                                       0x0
+#define IH_MSI_STORM_CTRL__DELAY_MASK                                                                         0x00000FFFL
+//IH_MSI_STORM_CLIENT_INDEX
+#define IH_MSI_STORM_CLIENT_INDEX__INDEX__SHIFT                                                               0x0
+#define IH_MSI_STORM_CLIENT_INDEX__INDEX_MASK                                                                 0x00000007L
+//IH_MSI_STORM_CLIENT_DATA
+#define IH_MSI_STORM_CLIENT_DATA__CLIENT_ID__SHIFT                                                            0x0
+#define IH_MSI_STORM_CLIENT_DATA__SOURCE_ID__SHIFT                                                            0x8
+#define IH_MSI_STORM_CLIENT_DATA__SOURCE_ID_MATCH_ENABLE__SHIFT                                               0x10
+#define IH_MSI_STORM_CLIENT_DATA__UTCL2_PAGE_FAULT_CONTEXTID_MATCH_ENABLE__SHIFT                              0x11
+#define IH_MSI_STORM_CLIENT_DATA__ENTRY_VALID__SHIFT                                                          0x1f
+#define IH_MSI_STORM_CLIENT_DATA__CLIENT_ID_MASK                                                              0x000000FFL
+#define IH_MSI_STORM_CLIENT_DATA__SOURCE_ID_MASK                                                              0x0000FF00L
+#define IH_MSI_STORM_CLIENT_DATA__SOURCE_ID_MATCH_ENABLE_MASK                                                 0x00010000L
+#define IH_MSI_STORM_CLIENT_DATA__ENTRY_VALID_MASK                                                            0x80000000L
+//IH_LAST_INT_INFO0
+#define IH_LAST_INT_INFO0__CLIENT_ID__SHIFT                                                                   0x0
+#define IH_LAST_INT_INFO0__SOURCE_ID__SHIFT                                                                   0x8
+#define IH_LAST_INT_INFO0__RING_ID__SHIFT                                                                     0x10
+#define IH_LAST_INT_INFO0__VM_ID__SHIFT                                                                       0x18
+#define IH_LAST_INT_INFO0__VMID_TYPE__SHIFT                                                                   0x1f
+#define IH_LAST_INT_INFO0__CLIENT_ID_MASK                                                                     0x000000FFL
+#define IH_LAST_INT_INFO0__SOURCE_ID_MASK                                                                     0x0000FF00L
+#define IH_LAST_INT_INFO0__RING_ID_MASK                                                                       0x00FF0000L
+#define IH_LAST_INT_INFO0__VM_ID_MASK                                                                         0x0F000000L
+#define IH_LAST_INT_INFO0__VMID_TYPE_MASK                                                                     0x80000000L
+//IH_LAST_INT_INFO1
+#define IH_LAST_INT_INFO1__CONTEXT_ID__SHIFT                                                                  0x0
+#define IH_LAST_INT_INFO1__CONTEXT_ID_MASK                                                                    0xFFFFFFFFL
+//IH_LAST_INT_INFO2
+#define IH_LAST_INT_INFO2__PAS_ID__SHIFT                                                                      0x0
+#define IH_LAST_INT_INFO2__VF_ID__SHIFT                                                                       0x10
+#define IH_LAST_INT_INFO2__VF__SHIFT                                                                          0x17
+#define IH_LAST_INT_INFO2__PAS_ID_MASK                                                                        0x0000FFFFL
+#define IH_LAST_INT_INFO2__VF_ID_MASK                                                                         0x001F0000L
+#define IH_LAST_INT_INFO2__VF_MASK                                                                            0x00800000L
+//IH_REGISTER_LAST_PART2
+#define IH_REGISTER_LAST_PART2__RESERVED__SHIFT                                                               0x0
+#define IH_REGISTER_LAST_PART2__RESERVED_MASK                                                                 0xFFFFFFFFL
+//SEM_MAILBOX
+#define SEM_MAILBOX__HOSTPORT__SHIFT                                                                          0x0
+#define SEM_MAILBOX__RESERVED__SHIFT                                                                          0x10
+#define SEM_MAILBOX__HOSTPORT_MASK                                                                            0x0000FFFFL
+#define SEM_MAILBOX__RESERVED_MASK                                                                            0xFFFF0000L
+//SEM_MAILBOX_CLEAR
+#define SEM_MAILBOX_CLEAR__CLEAR__SHIFT                                                                       0x0
+#define SEM_MAILBOX_CLEAR__RESERVED__SHIFT                                                                    0x10
+#define SEM_MAILBOX_CLEAR__CLEAR_MASK                                                                         0x0000FFFFL
+#define SEM_MAILBOX_CLEAR__RESERVED_MASK                                                                      0xFFFF0000L
+//SEM_REGISTER_LAST_PART2
+#define SEM_REGISTER_LAST_PART2__RESERVED__SHIFT                                                              0x0
+#define SEM_REGISTER_LAST_PART2__RESERVED_MASK                                                                0xFFFFFFFFL
+//IH_VIRT_RESET_REQ
+#define IH_VIRT_RESET_REQ__VF__SHIFT                                                                          0x0
+#define IH_VIRT_RESET_REQ__PF__SHIFT                                                                          0x1f
+#define IH_VIRT_RESET_REQ__VF_MASK                                                                            0x000000FFL
+#define IH_VIRT_RESET_REQ__PF_MASK                                                                            0x80000000L
+//IH_CLIENT_CFG
+#define IH_CLIENT_CFG__TOTAL_CLIENT_NUM__SHIFT                                                                0x0
+#define IH_CLIENT_CFG__TOTAL_CLIENT_NUM_MASK                                                                  0x0000007FL
+//IH_RING1_CLIENT_CFG_INDEX
+#define IH_RING1_CLIENT_CFG_INDEX__INDEX__SHIFT                                                               0x0
+#define IH_RING1_CLIENT_CFG_INDEX__INDEX_MASK                                                                 0x00000007L
+//IH_RING1_CLIENT_CFG_DATA
+#define IH_RING1_CLIENT_CFG_DATA__CLIENT_ID__SHIFT                                                            0x0
+#define IH_RING1_CLIENT_CFG_DATA__SOURCE_ID__SHIFT                                                            0x8
+#define IH_RING1_CLIENT_CFG_DATA__SOURCE_ID_MATCH_ENABLE__SHIFT                                               0x10
+#define IH_RING1_CLIENT_CFG_DATA__CLIENT_ID_MASK                                                              0x000000FFL
+#define IH_RING1_CLIENT_CFG_DATA__SOURCE_ID_MASK                                                              0x0000FF00L
+#define IH_RING1_CLIENT_CFG_DATA__SOURCE_ID_MATCH_ENABLE_MASK                                                 0x00010000L
+//IH_CLIENT_CFG_INDEX
+#define IH_CLIENT_CFG_INDEX__INDEX__SHIFT                                                                     0x0
+#define IH_CLIENT_CFG_INDEX__INDEX_MASK                                                                       0x0000001FL
+//IH_CLIENT_CFG_DATA
+#define IH_CLIENT_CFG_DATA__CLIENT_TYPE__SHIFT                                                                0x12
+#define IH_CLIENT_CFG_DATA__VF_RB_SELECT__SHIFT                                                               0x16
+#define IH_CLIENT_CFG_DATA__INTERFACE_TYPE__SHIFT                                                             0x19
+#define IH_CLIENT_CFG_DATA__CLIENT_TYPE_MASK                                                                  0x000C0000L
+#define IH_CLIENT_CFG_DATA__VF_RB_SELECT_MASK                                                                 0x00C00000L
+#define IH_CLIENT_CFG_DATA__INTERFACE_TYPE_MASK                                                               0x02000000L
+//IH_CLIENT_CFG_DATA2
+#define IH_CLIENT_CFG_DATA2__CREDIT_RETURN_ADDR__SHIFT                                                        0x0
+#define IH_CLIENT_CFG_DATA2__CREDIT_RETURN_ADDR_MASK                                                          0xFFFFFFFFL
+//IH_CID_REMAP_INDEX
+#define IH_CID_REMAP_INDEX__INDEX__SHIFT                                                                      0x0
+#define IH_CID_REMAP_INDEX__INDEX_MASK                                                                        0x00000003L
+//IH_CID_REMAP_DATA
+#define IH_CID_REMAP_DATA__CLIENT_ID__SHIFT                                                                   0x0
+#define IH_CID_REMAP_DATA__INITIATOR_ID__SHIFT                                                                0x8
+#define IH_CID_REMAP_DATA__CLIENT_ID_REMAP__SHIFT                                                             0x18
+#define IH_CID_REMAP_DATA__CLIENT_ID_MASK                                                                     0x000000FFL
+#define IH_CID_REMAP_DATA__INITIATOR_ID_MASK                                                                  0x0003FF00L
+#define IH_CID_REMAP_DATA__CLIENT_ID_REMAP_MASK                                                               0xFF000000L
+//IH_CHICKEN
+#define IH_CHICKEN__DBGU_TRIGGER_ENABLE__SHIFT                                                                0x1
+#define IH_CHICKEN__MC_SPACE_FBPA_ENABLE__SHIFT                                                               0x3
+#define IH_CHICKEN__MC_SPACE_GPA_ENABLE__SHIFT                                                                0x4
+#define IH_CHICKEN__REG_FIREWALL_ENABLE__SHIFT                                                                0x5
+#define IH_CHICKEN__DBGU_TRIGGER_ENABLE_MASK                                                                  0x00000002L
+#define IH_CHICKEN__MC_SPACE_FBPA_ENABLE_MASK                                                                 0x00000008L
+#define IH_CHICKEN__MC_SPACE_GPA_ENABLE_MASK                                                                  0x00000010L
+#define IH_CHICKEN__REG_FIREWALL_ENABLE_MASK                                                                  0x00000020L
+//IH_INT_DROP_CNTL
+#define IH_INT_DROP_CNTL__INT_DROP_EN__SHIFT                                                                  0x0
+#define IH_INT_DROP_CNTL__CLIENT_ID_MATCH_EN__SHIFT                                                           0x1
+#define IH_INT_DROP_CNTL__SOURCE_ID_MATCH_EN__SHIFT                                                           0x2
+#define IH_INT_DROP_CNTL__VF_ID_MATCH_EN__SHIFT                                                               0x3
+#define IH_INT_DROP_CNTL__VF_MATCH_EN__SHIFT                                                                  0x4
+#define IH_INT_DROP_CNTL__CONTEXT_ID_MATCH_EN__SHIFT                                                          0x5
+#define IH_INT_DROP_CNTL__INT_DROP_MODE__SHIFT                                                                0x6
+#define IH_INT_DROP_CNTL__GCUTCL2_RETRY_INT_DROP_EN__SHIFT                                                    0x8
+#define IH_INT_DROP_CNTL__MMUTCL2_RETRY_INT_DROP_EN__SHIFT                                                    0x9
+#define IH_INT_DROP_CNTL__GCUTCL2_FAULT_INT_DROP_EN__SHIFT                                                    0xa
+#define IH_INT_DROP_CNTL__MMUTCL2_FAULT_INT_DROP_EN__SHIFT                                                    0xb
+#define IH_INT_DROP_CNTL__INT_DROPPED__SHIFT                                                                  0x10
+#define IH_INT_DROP_CNTL__INT_DROP_EN_MASK                                                                    0x00000001L
+#define IH_INT_DROP_CNTL__CLIENT_ID_MATCH_EN_MASK                                                             0x00000002L
+#define IH_INT_DROP_CNTL__SOURCE_ID_MATCH_EN_MASK                                                             0x00000004L
+#define IH_INT_DROP_CNTL__VF_ID_MATCH_EN_MASK                                                                 0x00000008L
+#define IH_INT_DROP_CNTL__VF_MATCH_EN_MASK                                                                    0x00000010L
+#define IH_INT_DROP_CNTL__CONTEXT_ID_MATCH_EN_MASK                                                            0x00000020L
+#define IH_INT_DROP_CNTL__INT_DROP_MODE_MASK                                                                  0x000000C0L
+#define IH_INT_DROP_CNTL__INT_DROPPED_MASK                                                                    0x00010000L
+//IH_INT_DROP_MATCH_VALUE0
+#define IH_INT_DROP_MATCH_VALUE0__CLIENT_ID_MATCH_VALUE__SHIFT                                                0x0
+#define IH_INT_DROP_MATCH_VALUE0__SOURCE_ID_MATCH_VALUE__SHIFT                                                0x8
+#define IH_INT_DROP_MATCH_VALUE0__VF_ID_MATCH_VALUE__SHIFT                                                    0x10
+#define IH_INT_DROP_MATCH_VALUE0__VF_MATCH_VALUE__SHIFT                                                       0x17
+#define IH_INT_DROP_MATCH_VALUE0__CONTEXT_ID_39_32_MATCH_VALUE__SHIFT                                         0x18
+#define IH_INT_DROP_MATCH_VALUE0__CLIENT_ID_MATCH_VALUE_MASK                                                  0x000000FFL
+#define IH_INT_DROP_MATCH_VALUE0__SOURCE_ID_MATCH_VALUE_MASK                                                  0x0000FF00L
+#define IH_INT_DROP_MATCH_VALUE0__VF_ID_MATCH_VALUE_MASK                                                      0x001F0000L
+#define IH_INT_DROP_MATCH_VALUE0__VF_MATCH_VALUE_MASK                                                         0x00800000L
+#define IH_INT_DROP_MATCH_VALUE0__CONTEXT_ID_39_32_MATCH_VALUE_MASK                                           0xFF000000L
+//IH_INT_DROP_MATCH_VALUE1
+#define IH_INT_DROP_MATCH_VALUE1__CONTEXT_ID_31_0_MATCH_VALUE__SHIFT                                          0x0
+#define IH_INT_DROP_MATCH_VALUE1__CONTEXT_ID_31_0_MATCH_VALUE_MASK                                            0xFFFFFFFFL
+//IH_INT_DROP_MATCH_MASK0
+#define IH_INT_DROP_MATCH_MASK0__CLIENT_ID_MATCH_MASK__SHIFT                                                  0x0
+#define IH_INT_DROP_MATCH_MASK0__SOURCE_ID_MATCH_MASK__SHIFT                                                  0x8
+#define IH_INT_DROP_MATCH_MASK0__VF_ID_MATCH_MASK__SHIFT                                                      0x10
+#define IH_INT_DROP_MATCH_MASK0__VF_MATCH_MASK__SHIFT                                                         0x17
+#define IH_INT_DROP_MATCH_MASK0__CONTEXT_ID_39_32_MATCH_MASK__SHIFT                                           0x18
+#define IH_INT_DROP_MATCH_MASK0__CLIENT_ID_MATCH_MASK_MASK                                                    0x000000FFL
+#define IH_INT_DROP_MATCH_MASK0__SOURCE_ID_MATCH_MASK_MASK                                                    0x0000FF00L
+#define IH_INT_DROP_MATCH_MASK0__VF_ID_MATCH_MASK_MASK                                                        0x001F0000L
+#define IH_INT_DROP_MATCH_MASK0__VF_MATCH_MASK_MASK                                                           0x00800000L
+#define IH_INT_DROP_MATCH_MASK0__CONTEXT_ID_39_32_MATCH_MASK_MASK                                             0xFF000000L
+//IH_INT_DROP_MATCH_MASK1
+#define IH_INT_DROP_MATCH_MASK1__CONTEXT_ID_31_0_MATCH_MASK__SHIFT                                            0x0
+#define IH_INT_DROP_MATCH_MASK1__CONTEXT_ID_31_0_MATCH_MASK_MASK                                              0xFFFFFFFFL
+//IH_MMHUB_CNTL
+#define IH_MMHUB_CNTL__UNITID__SHIFT                                                                          0x0
+#define IH_MMHUB_CNTL__IV_TLVL__SHIFT                                                                         0x8
+#define IH_MMHUB_CNTL__WPTR_WB_TLVL__SHIFT                                                                    0xc
+#define IH_MMHUB_CNTL__UNITID_MASK                                                                            0x0000003FL
+#define IH_MMHUB_CNTL__IV_TLVL_MASK                                                                           0x00000F00L
+#define IH_MMHUB_CNTL__WPTR_WB_TLVL_MASK                                                                      0x0000F000L
+//IH_REGISTER_LAST_PART1
+#define IH_REGISTER_LAST_PART1__RESERVED__SHIFT                                                               0x0
+#define IH_REGISTER_LAST_PART1__RESERVED_MASK                                                                 0xFFFFFFFFL
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_15_0_8_offset.h b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_15_0_8_offset.h
new file mode 100644
index 000000000000..f17a6c2bd70b
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_15_0_8_offset.h
@@ -0,0 +1,512 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _smuio_15_0_8_OFFSET_HEADER
+#define _smuio_15_0_8_OFFSET_HEADER
+
+
+
+// addressBlock: smuio_smuio_tsc_SmuSmuioDec
+// base address: 0x5a8a0
+#define regPWROK_REFCLK_GAP_CYCLES                                                                      0x0028
+#define regPWROK_REFCLK_GAP_CYCLES_BASE_IDX                                                             1
+#define regGOLDEN_TSC_INCREMENT_UPPER                                                                   0x002b
+#define regGOLDEN_TSC_INCREMENT_UPPER_BASE_IDX                                                          1
+#define regGOLDEN_TSC_INCREMENT_LOWER                                                                   0x002c
+#define regGOLDEN_TSC_INCREMENT_LOWER_BASE_IDX                                                          1
+#define regGOLDEN_TSC_COUNT_UPPER                                                                       0x0030
+#define regGOLDEN_TSC_COUNT_UPPER_BASE_IDX                                                              1
+#define regGOLDEN_TSC_COUNT_LOWER                                                                       0x0031
+#define regGOLDEN_TSC_COUNT_LOWER_BASE_IDX                                                              1
+#define regSOC_GOLDEN_TSC_SHADOW_UPPER                                                                  0x0032
+#define regSOC_GOLDEN_TSC_SHADOW_UPPER_BASE_IDX                                                         1
+#define regSOC_GOLDEN_TSC_SHADOW_LOWER                                                                  0x0033
+#define regSOC_GOLDEN_TSC_SHADOW_LOWER_BASE_IDX                                                         1
+#define regSOC_GAP_PWROK                                                                                0x0034
+#define regSOC_GAP_PWROK_BASE_IDX                                                                       1
+
+
+// addressBlock: smuio_smuio_swtimer_SmuSmuioDec
+// base address: 0x5aca8
+#define regPWR_VIRT_RESET_REQ                                                                           0x012a
+#define regPWR_VIRT_RESET_REQ_BASE_IDX                                                                  1
+#define regPWR_DISP_TIMER_CONTROL                                                                       0x012b
+#define regPWR_DISP_TIMER_CONTROL_BASE_IDX                                                              1
+#define regPWR_DISP_TIMER_DEBUG                                                                         0x012c
+#define regPWR_DISP_TIMER_DEBUG_BASE_IDX                                                                1
+#define regPWR_DISP_TIMER2_CONTROL                                                                      0x012d
+#define regPWR_DISP_TIMER2_CONTROL_BASE_IDX                                                             1
+#define regPWR_DISP_TIMER2_DEBUG                                                                        0x012e
+#define regPWR_DISP_TIMER2_DEBUG_BASE_IDX                                                               1
+#define regPWR_DISP_TIMER_GLOBAL_CONTROL                                                                0x012f
+#define regPWR_DISP_TIMER_GLOBAL_CONTROL_BASE_IDX                                                       1
+#define regPWR_IH_CONTROL                                                                               0x0130
+#define regPWR_IH_CONTROL_BASE_IDX                                                                      1
+
+
+// addressBlock: smuio_smuio_misc_SmuSmuioDec
+// base address: 0x5a000
+#define regSMUIO_MCM_CONFIG                                                                             0x0023
+#define regSMUIO_MCM_CONFIG_BASE_IDX                                                                    0
+#define regIP_DISCOVERY_VERSION                                                                         0x0000
+#define regIP_DISCOVERY_VERSION_BASE_IDX                                                                1
+#define regSCRATCH_REGISTER0                                                                            0x01c9
+#define regSCRATCH_REGISTER0_BASE_IDX                                                                   1
+#define regSCRATCH_REGISTER1                                                                            0x01ca
+#define regSCRATCH_REGISTER1_BASE_IDX                                                                   1
+#define regSCRATCH_REGISTER2                                                                            0x01cb
+#define regSCRATCH_REGISTER2_BASE_IDX                                                                   1
+#define regSCRATCH_REGISTER3                                                                            0x01cc
+#define regSCRATCH_REGISTER3_BASE_IDX                                                                   1
+#define regSCRATCH_REGISTER4                                                                            0x01cd
+#define regSCRATCH_REGISTER4_BASE_IDX                                                                   1
+#define regSCRATCH_REGISTER5                                                                            0x01ce
+#define regSCRATCH_REGISTER5_BASE_IDX                                                                   1
+#define regSCRATCH_REGISTER6                                                                            0x01cf
+#define regSCRATCH_REGISTER6_BASE_IDX                                                                   1
+#define regSCRATCH_REGISTER7                                                                            0x01d0
+#define regSCRATCH_REGISTER7_BASE_IDX                                                                   1
+
+
+// addressBlock: smuio_smuio_i2c_SmuSmuioDec
+// base address: 0x5a100
+#define regCKSVII2C_IC_CON                                                                              0x0040
+#define regCKSVII2C_IC_CON_BASE_IDX                                                                     0
+#define regCKSVII2C_IC_TAR                                                                              0x0041
+#define regCKSVII2C_IC_TAR_BASE_IDX                                                                     0
+#define regCKSVII2C_IC_SAR                                                                              0x0042
+#define regCKSVII2C_IC_SAR_BASE_IDX                                                                     0
+#define regCKSVII2C_IC_HS_MADDR                                                                         0x0043
+#define regCKSVII2C_IC_HS_MADDR_BASE_IDX                                                                0
+#define regCKSVII2C_IC_DATA_CMD                                                                         0x0044
+#define regCKSVII2C_IC_DATA_CMD_BASE_IDX                                                                0
+#define regCKSVII2C_IC_SS_SCL_HCNT                                                                      0x0045
+#define regCKSVII2C_IC_SS_SCL_HCNT_BASE_IDX                                                             0
+#define regCKSVII2C_IC_SS_SCL_LCNT                                                                      0x0046
+#define regCKSVII2C_IC_SS_SCL_LCNT_BASE_IDX                                                             0
+#define regCKSVII2C_IC_FS_SCL_HCNT                                                                      0x0047
+#define regCKSVII2C_IC_FS_SCL_HCNT_BASE_IDX                                                             0
+#define regCKSVII2C_IC_FS_SCL_LCNT                                                                      0x0048
+#define regCKSVII2C_IC_FS_SCL_LCNT_BASE_IDX                                                             0
+#define regCKSVII2C_IC_HS_SCL_HCNT                                                                      0x0049
+#define regCKSVII2C_IC_HS_SCL_HCNT_BASE_IDX                                                             0
+#define regCKSVII2C_IC_HS_SCL_LCNT                                                                      0x004a
+#define regCKSVII2C_IC_HS_SCL_LCNT_BASE_IDX                                                             0
+#define regCKSVII2C_IC_INTR_STAT                                                                        0x004b
+#define regCKSVII2C_IC_INTR_STAT_BASE_IDX                                                               0
+#define regCKSVII2C_IC_INTR_MASK                                                                        0x004c
+#define regCKSVII2C_IC_INTR_MASK_BASE_IDX                                                               0
+#define regCKSVII2C_IC_RAW_INTR_STAT                                                                    0x004d
+#define regCKSVII2C_IC_RAW_INTR_STAT_BASE_IDX                                                           0
+#define regCKSVII2C_IC_RX_TL                                                                            0x004e
+#define regCKSVII2C_IC_RX_TL_BASE_IDX                                                                   0
+#define regCKSVII2C_IC_TX_TL                                                                            0x004f
+#define regCKSVII2C_IC_TX_TL_BASE_IDX                                                                   0
+#define regCKSVII2C_IC_CLR_INTR                                                                         0x0050
+#define regCKSVII2C_IC_CLR_INTR_BASE_IDX                                                                0
+#define regCKSVII2C_IC_CLR_RX_UNDER                                                                     0x0051
+#define regCKSVII2C_IC_CLR_RX_UNDER_BASE_IDX                                                            0
+#define regCKSVII2C_IC_CLR_RX_OVER                                                                      0x0052
+#define regCKSVII2C_IC_CLR_RX_OVER_BASE_IDX                                                             0
+#define regCKSVII2C_IC_CLR_TX_OVER                                                                      0x0053
+#define regCKSVII2C_IC_CLR_TX_OVER_BASE_IDX                                                             0
+#define regCKSVII2C_IC_CLR_RD_REQ                                                                       0x0054
+#define regCKSVII2C_IC_CLR_RD_REQ_BASE_IDX                                                              0
+#define regCKSVII2C_IC_CLR_TX_ABRT                                                                      0x0055
+#define regCKSVII2C_IC_CLR_TX_ABRT_BASE_IDX                                                             0
+#define regCKSVII2C_IC_CLR_RX_DONE                                                                      0x0056
+#define regCKSVII2C_IC_CLR_RX_DONE_BASE_IDX                                                             0
+#define regCKSVII2C_IC_CLR_ACTIVITY                                                                     0x0057
+#define regCKSVII2C_IC_CLR_ACTIVITY_BASE_IDX                                                            0
+#define regCKSVII2C_IC_CLR_STOP_DET                                                                     0x0058
+#define regCKSVII2C_IC_CLR_STOP_DET_BASE_IDX                                                            0
+#define regCKSVII2C_IC_CLR_START_DET                                                                    0x0059
+#define regCKSVII2C_IC_CLR_START_DET_BASE_IDX                                                           0
+#define regCKSVII2C_IC_CLR_GEN_CALL                                                                     0x005a
+#define regCKSVII2C_IC_CLR_GEN_CALL_BASE_IDX                                                            0
+#define regCKSVII2C_IC_ENABLE                                                                           0x005b
+#define regCKSVII2C_IC_ENABLE_BASE_IDX                                                                  0
+#define regCKSVII2C_IC_STATUS                                                                           0x005c
+#define regCKSVII2C_IC_STATUS_BASE_IDX                                                                  0
+#define regCKSVII2C_IC_TXFLR                                                                            0x005d
+#define regCKSVII2C_IC_TXFLR_BASE_IDX                                                                   0
+#define regCKSVII2C_IC_RXFLR                                                                            0x005e
+#define regCKSVII2C_IC_RXFLR_BASE_IDX                                                                   0
+#define regCKSVII2C_IC_SDA_HOLD                                                                         0x005f
+#define regCKSVII2C_IC_SDA_HOLD_BASE_IDX                                                                0
+#define regCKSVII2C_IC_TX_ABRT_SOURCE                                                                   0x0060
+#define regCKSVII2C_IC_TX_ABRT_SOURCE_BASE_IDX                                                          0
+#define regCKSVII2C_IC_SLV_DATA_NACK_ONLY                                                               0x0061
+#define regCKSVII2C_IC_SLV_DATA_NACK_ONLY_BASE_IDX                                                      0
+#define regCKSVII2C_IC_DMA_CR                                                                           0x0062
+#define regCKSVII2C_IC_DMA_CR_BASE_IDX                                                                  0
+#define regCKSVII2C_IC_DMA_TDLR                                                                         0x0063
+#define regCKSVII2C_IC_DMA_TDLR_BASE_IDX                                                                0
+#define regCKSVII2C_IC_DMA_RDLR                                                                         0x0064
+#define regCKSVII2C_IC_DMA_RDLR_BASE_IDX                                                                0
+#define regCKSVII2C_IC_SDA_SETUP                                                                        0x0065
+#define regCKSVII2C_IC_SDA_SETUP_BASE_IDX                                                               0
+#define regCKSVII2C_IC_ACK_GENERAL_CALL                                                                 0x0066
+#define regCKSVII2C_IC_ACK_GENERAL_CALL_BASE_IDX                                                        0
+#define regCKSVII2C_IC_ENABLE_STATUS                                                                    0x0067
+#define regCKSVII2C_IC_ENABLE_STATUS_BASE_IDX                                                           0
+#define regCKSVII2C_IC_FS_SPKLEN                                                                        0x0068
+#define regCKSVII2C_IC_FS_SPKLEN_BASE_IDX                                                               0
+#define regCKSVII2C_IC_HS_SPKLEN                                                                        0x0069
+#define regCKSVII2C_IC_HS_SPKLEN_BASE_IDX                                                               0
+#define regCKSVII2C_IC_CLR_RESTART_DET                                                                  0x006a
+#define regCKSVII2C_IC_CLR_RESTART_DET_BASE_IDX                                                         0
+#define regCKSVII2C_IC_COMP_PARAM_1                                                                     0x006d
+#define regCKSVII2C_IC_COMP_PARAM_1_BASE_IDX                                                            0
+#define regCKSVII2C_IC_COMP_VERSION                                                                     0x006e
+#define regCKSVII2C_IC_COMP_VERSION_BASE_IDX                                                            0
+#define regCKSVII2C_IC_COMP_TYPE                                                                        0x006f
+#define regCKSVII2C_IC_COMP_TYPE_BASE_IDX                                                               0
+#define regCKSVII2C1_IC_CON                                                                             0x0080
+#define regCKSVII2C1_IC_CON_BASE_IDX                                                                    0
+#define regCKSVII2C1_IC_TAR                                                                             0x0081
+#define regCKSVII2C1_IC_TAR_BASE_IDX                                                                    0
+#define regCKSVII2C1_IC_SAR                                                                             0x0082
+#define regCKSVII2C1_IC_SAR_BASE_IDX                                                                    0
+#define regCKSVII2C1_IC_HS_MADDR                                                                        0x0083
+#define regCKSVII2C1_IC_HS_MADDR_BASE_IDX                                                               0
+#define regCKSVII2C1_IC_DATA_CMD                                                                        0x0084
+#define regCKSVII2C1_IC_DATA_CMD_BASE_IDX                                                               0
+#define regCKSVII2C1_IC_SS_SCL_HCNT                                                                     0x0085
+#define regCKSVII2C1_IC_SS_SCL_HCNT_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_SS_SCL_LCNT                                                                     0x0086
+#define regCKSVII2C1_IC_SS_SCL_LCNT_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_FS_SCL_HCNT                                                                     0x0087
+#define regCKSVII2C1_IC_FS_SCL_HCNT_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_FS_SCL_LCNT                                                                     0x0088
+#define regCKSVII2C1_IC_FS_SCL_LCNT_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_HS_SCL_HCNT                                                                     0x0089
+#define regCKSVII2C1_IC_HS_SCL_HCNT_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_HS_SCL_LCNT                                                                     0x008a
+#define regCKSVII2C1_IC_HS_SCL_LCNT_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_INTR_STAT                                                                       0x008b
+#define regCKSVII2C1_IC_INTR_STAT_BASE_IDX                                                              0
+#define regCKSVII2C1_IC_INTR_MASK                                                                       0x008c
+#define regCKSVII2C1_IC_INTR_MASK_BASE_IDX                                                              0
+#define regCKSVII2C1_IC_RAW_INTR_STAT                                                                   0x008d
+#define regCKSVII2C1_IC_RAW_INTR_STAT_BASE_IDX                                                          0
+#define regCKSVII2C1_IC_RX_TL                                                                           0x008e
+#define regCKSVII2C1_IC_RX_TL_BASE_IDX                                                                  0
+#define regCKSVII2C1_IC_TX_TL                                                                           0x008f
+#define regCKSVII2C1_IC_TX_TL_BASE_IDX                                                                  0
+#define regCKSVII2C1_IC_CLR_INTR                                                                        0x0090
+#define regCKSVII2C1_IC_CLR_INTR_BASE_IDX                                                               0
+#define regCKSVII2C1_IC_CLR_RX_UNDER                                                                    0x0091
+#define regCKSVII2C1_IC_CLR_RX_UNDER_BASE_IDX                                                           0
+#define regCKSVII2C1_IC_CLR_RX_OVER                                                                     0x0092
+#define regCKSVII2C1_IC_CLR_RX_OVER_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_CLR_TX_OVER                                                                     0x0093
+#define regCKSVII2C1_IC_CLR_TX_OVER_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_CLR_RD_REQ                                                                      0x0094
+#define regCKSVII2C1_IC_CLR_RD_REQ_BASE_IDX                                                             0
+#define regCKSVII2C1_IC_CLR_TX_ABRT                                                                     0x0095
+#define regCKSVII2C1_IC_CLR_TX_ABRT_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_CLR_RX_DONE                                                                     0x0096
+#define regCKSVII2C1_IC_CLR_RX_DONE_BASE_IDX                                                            0
+#define regCKSVII2C1_IC_CLR_ACTIVITY                                                                    0x0097
+#define regCKSVII2C1_IC_CLR_ACTIVITY_BASE_IDX                                                           0
+#define regCKSVII2C1_IC_CLR_STOP_DET                                                                    0x0098
+#define regCKSVII2C1_IC_CLR_STOP_DET_BASE_IDX                                                           0
+#define regCKSVII2C1_IC_CLR_START_DET                                                                   0x0099
+#define regCKSVII2C1_IC_CLR_START_DET_BASE_IDX                                                          0
+#define regCKSVII2C1_IC_CLR_GEN_CALL                                                                    0x009a
+#define regCKSVII2C1_IC_CLR_GEN_CALL_BASE_IDX                                                           0
+#define regCKSVII2C1_IC_ENABLE                                                                          0x009b
+#define regCKSVII2C1_IC_ENABLE_BASE_IDX                                                                 0
+#define regCKSVII2C1_IC_STATUS                                                                          0x009c
+#define regCKSVII2C1_IC_STATUS_BASE_IDX                                                                 0
+#define regCKSVII2C1_IC_TXFLR                                                                           0x009d
+#define regCKSVII2C1_IC_TXFLR_BASE_IDX                                                                  0
+#define regCKSVII2C1_IC_RXFLR                                                                           0x009e
+#define regCKSVII2C1_IC_RXFLR_BASE_IDX                                                                  0
+#define regCKSVII2C1_IC_SDA_HOLD                                                                        0x009f
+#define regCKSVII2C1_IC_SDA_HOLD_BASE_IDX                                                               0
+#define regCKSVII2C1_IC_TX_ABRT_SOURCE                                                                  0x00a0
+#define regCKSVII2C1_IC_TX_ABRT_SOURCE_BASE_IDX                                                         0
+#define regCKSVII2C1_IC_SLV_DATA_NACK_ONLY                                                              0x00a1
+#define regCKSVII2C1_IC_SLV_DATA_NACK_ONLY_BASE_IDX                                                     0
+#define regCKSVII2C1_IC_DMA_CR                                                                          0x00a2
+#define regCKSVII2C1_IC_DMA_CR_BASE_IDX                                                                 0
+#define regCKSVII2C1_IC_DMA_TDLR                                                                        0x00a3
+#define regCKSVII2C1_IC_DMA_TDLR_BASE_IDX                                                               0
+#define regCKSVII2C1_IC_DMA_RDLR                                                                        0x00a4
+#define regCKSVII2C1_IC_DMA_RDLR_BASE_IDX                                                               0
+#define regCKSVII2C1_IC_SDA_SETUP                                                                       0x00a5
+#define regCKSVII2C1_IC_SDA_SETUP_BASE_IDX                                                              0
+#define regCKSVII2C1_IC_ACK_GENERAL_CALL                                                                0x00a6
+#define regCKSVII2C1_IC_ACK_GENERAL_CALL_BASE_IDX                                                       0
+#define regCKSVII2C1_IC_ENABLE_STATUS                                                                   0x00a7
+#define regCKSVII2C1_IC_ENABLE_STATUS_BASE_IDX                                                          0
+#define regCKSVII2C1_IC_FS_SPKLEN                                                                       0x00a8
+#define regCKSVII2C1_IC_FS_SPKLEN_BASE_IDX                                                              0
+#define regCKSVII2C1_IC_HS_SPKLEN                                                                       0x00a9
+#define regCKSVII2C1_IC_HS_SPKLEN_BASE_IDX                                                              0
+#define regCKSVII2C1_IC_CLR_RESTART_DET                                                                 0x00aa
+#define regCKSVII2C1_IC_CLR_RESTART_DET_BASE_IDX                                                        0
+#define regCKSVII2C1_IC_COMP_PARAM_1                                                                    0x00ad
+#define regCKSVII2C1_IC_COMP_PARAM_1_BASE_IDX                                                           0
+#define regCKSVII2C1_IC_COMP_VERSION                                                                    0x00ae
+#define regCKSVII2C1_IC_COMP_VERSION_BASE_IDX                                                           0
+#define regCKSVII2C1_IC_COMP_TYPE                                                                       0x00af
+#define regCKSVII2C1_IC_COMP_TYPE_BASE_IDX                                                              0
+#define regSMUIO_PWRMGT                                                                                 0x018c
+#define regSMUIO_PWRMGT_BASE_IDX                                                                        0
+
+
+// addressBlock: smuio_smuio_rom_SmuSmuioDec
+// base address: 0x5a380
+#define regROM_CNTL                                                                                     0x00e0
+#define regROM_CNTL_BASE_IDX                                                                            0
+#define regPAGE_MIRROR_CNTL                                                                             0x00e1
+#define regPAGE_MIRROR_CNTL_BASE_IDX                                                                    0
+#define regROM_STATUS                                                                                   0x00e2
+#define regROM_STATUS_BASE_IDX                                                                          0
+#define regCGTT_ROM_CLK_CTRL0                                                                           0x00e3
+#define regCGTT_ROM_CLK_CTRL0_BASE_IDX                                                                  0
+#define regROM_INDEX                                                                                    0x00e4
+#define regROM_INDEX_BASE_IDX                                                                           0
+#define regROM_DATA                                                                                     0x00e5
+#define regROM_DATA_BASE_IDX                                                                            0
+#define regROM_START                                                                                    0x00e6
+#define regROM_START_BASE_IDX                                                                           0
+#define regROM_SW_CNTL                                                                                  0x00e8
+#define regROM_SW_CNTL_BASE_IDX                                                                         0
+#define regROM_SW_STATUS                                                                                0x00e9
+#define regROM_SW_STATUS_BASE_IDX                                                                       0
+#define regROM_SW_COMMAND                                                                               0x00ea
+#define regROM_SW_COMMAND_BASE_IDX                                                                      0
+#define regROM_SW_DATA_1                                                                                0x00ec
+#define regROM_SW_DATA_1_BASE_IDX                                                                       0
+#define regROM_SW_DATA_2                                                                                0x00ed
+#define regROM_SW_DATA_2_BASE_IDX                                                                       0
+#define regROM_SW_DATA_3                                                                                0x00ee
+#define regROM_SW_DATA_3_BASE_IDX                                                                       0
+#define regROM_SW_DATA_4                                                                                0x00ef
+#define regROM_SW_DATA_4_BASE_IDX                                                                       0
+#define regROM_SW_DATA_5                                                                                0x00f0
+#define regROM_SW_DATA_5_BASE_IDX                                                                       0
+#define regROM_SW_DATA_6                                                                                0x00f1
+#define regROM_SW_DATA_6_BASE_IDX                                                                       0
+#define regROM_SW_DATA_7                                                                                0x00f2
+#define regROM_SW_DATA_7_BASE_IDX                                                                       0
+#define regROM_SW_DATA_8                                                                                0x00f3
+#define regROM_SW_DATA_8_BASE_IDX                                                                       0
+#define regROM_SW_DATA_9                                                                                0x00f4
+#define regROM_SW_DATA_9_BASE_IDX                                                                       0
+#define regROM_SW_DATA_10                                                                               0x00f5
+#define regROM_SW_DATA_10_BASE_IDX                                                                      0
+#define regROM_SW_DATA_11                                                                               0x00f6
+#define regROM_SW_DATA_11_BASE_IDX                                                                      0
+#define regROM_SW_DATA_12                                                                               0x00f7
+#define regROM_SW_DATA_12_BASE_IDX                                                                      0
+#define regROM_SW_DATA_13                                                                               0x00f8
+#define regROM_SW_DATA_13_BASE_IDX                                                                      0
+#define regROM_SW_DATA_14                                                                               0x00f9
+#define regROM_SW_DATA_14_BASE_IDX                                                                      0
+#define regROM_SW_DATA_15                                                                               0x00fa
+#define regROM_SW_DATA_15_BASE_IDX                                                                      0
+#define regROM_SW_DATA_16                                                                               0x00fb
+#define regROM_SW_DATA_16_BASE_IDX                                                                      0
+#define regROM_SW_DATA_17                                                                               0x00fc
+#define regROM_SW_DATA_17_BASE_IDX                                                                      0
+#define regROM_SW_DATA_18                                                                               0x00fd
+#define regROM_SW_DATA_18_BASE_IDX                                                                      0
+#define regROM_SW_DATA_19                                                                               0x00fe
+#define regROM_SW_DATA_19_BASE_IDX                                                                      0
+#define regROM_SW_DATA_20                                                                               0x00ff
+#define regROM_SW_DATA_20_BASE_IDX                                                                      0
+#define regROM_SW_DATA_21                                                                               0x0100
+#define regROM_SW_DATA_21_BASE_IDX                                                                      0
+#define regROM_SW_DATA_22                                                                               0x0101
+#define regROM_SW_DATA_22_BASE_IDX                                                                      0
+#define regROM_SW_DATA_23                                                                               0x0102
+#define regROM_SW_DATA_23_BASE_IDX                                                                      0
+#define regROM_SW_DATA_24                                                                               0x0103
+#define regROM_SW_DATA_24_BASE_IDX                                                                      0
+#define regROM_SW_DATA_25                                                                               0x0104
+#define regROM_SW_DATA_25_BASE_IDX                                                                      0
+#define regROM_SW_DATA_26                                                                               0x0105
+#define regROM_SW_DATA_26_BASE_IDX                                                                      0
+#define regROM_SW_DATA_27                                                                               0x0106
+#define regROM_SW_DATA_27_BASE_IDX                                                                      0
+#define regROM_SW_DATA_28                                                                               0x0107
+#define regROM_SW_DATA_28_BASE_IDX                                                                      0
+#define regROM_SW_DATA_29                                                                               0x0108
+#define regROM_SW_DATA_29_BASE_IDX                                                                      0
+#define regROM_SW_DATA_30                                                                               0x0109
+#define regROM_SW_DATA_30_BASE_IDX                                                                      0
+#define regROM_SW_DATA_31                                                                               0x010a
+#define regROM_SW_DATA_31_BASE_IDX                                                                      0
+#define regROM_SW_DATA_32                                                                               0x010b
+#define regROM_SW_DATA_32_BASE_IDX                                                                      0
+#define regROM_SW_DATA_33                                                                               0x010c
+#define regROM_SW_DATA_33_BASE_IDX                                                                      0
+#define regROM_SW_DATA_34                                                                               0x010d
+#define regROM_SW_DATA_34_BASE_IDX                                                                      0
+#define regROM_SW_DATA_35                                                                               0x010e
+#define regROM_SW_DATA_35_BASE_IDX                                                                      0
+#define regROM_SW_DATA_36                                                                               0x010f
+#define regROM_SW_DATA_36_BASE_IDX                                                                      0
+#define regROM_SW_DATA_37                                                                               0x0110
+#define regROM_SW_DATA_37_BASE_IDX                                                                      0
+#define regROM_SW_DATA_38                                                                               0x0111
+#define regROM_SW_DATA_38_BASE_IDX                                                                      0
+#define regROM_SW_DATA_39                                                                               0x0112
+#define regROM_SW_DATA_39_BASE_IDX                                                                      0
+#define regROM_SW_DATA_40                                                                               0x0113
+#define regROM_SW_DATA_40_BASE_IDX                                                                      0
+#define regROM_SW_DATA_41                                                                               0x0114
+#define regROM_SW_DATA_41_BASE_IDX                                                                      0
+#define regROM_SW_DATA_42                                                                               0x0115
+#define regROM_SW_DATA_42_BASE_IDX                                                                      0
+#define regROM_SW_DATA_43                                                                               0x0116
+#define regROM_SW_DATA_43_BASE_IDX                                                                      0
+#define regROM_SW_DATA_44                                                                               0x0117
+#define regROM_SW_DATA_44_BASE_IDX                                                                      0
+#define regROM_SW_DATA_45                                                                               0x0118
+#define regROM_SW_DATA_45_BASE_IDX                                                                      0
+#define regROM_SW_DATA_46                                                                               0x0119
+#define regROM_SW_DATA_46_BASE_IDX                                                                      0
+#define regROM_SW_DATA_47                                                                               0x011a
+#define regROM_SW_DATA_47_BASE_IDX                                                                      0
+#define regROM_SW_DATA_48                                                                               0x011b
+#define regROM_SW_DATA_48_BASE_IDX                                                                      0
+#define regROM_SW_DATA_49                                                                               0x011c
+#define regROM_SW_DATA_49_BASE_IDX                                                                      0
+#define regROM_SW_DATA_50                                                                               0x011d
+#define regROM_SW_DATA_50_BASE_IDX                                                                      0
+#define regROM_SW_DATA_51                                                                               0x011e
+#define regROM_SW_DATA_51_BASE_IDX                                                                      0
+#define regROM_SW_DATA_52                                                                               0x011f
+#define regROM_SW_DATA_52_BASE_IDX                                                                      0
+#define regROM_SW_DATA_53                                                                               0x0120
+#define regROM_SW_DATA_53_BASE_IDX                                                                      0
+#define regROM_SW_DATA_54                                                                               0x0121
+#define regROM_SW_DATA_54_BASE_IDX                                                                      0
+#define regROM_SW_DATA_55                                                                               0x0122
+#define regROM_SW_DATA_55_BASE_IDX                                                                      0
+#define regROM_SW_DATA_56                                                                               0x0123
+#define regROM_SW_DATA_56_BASE_IDX                                                                      0
+#define regROM_SW_DATA_57                                                                               0x0124
+#define regROM_SW_DATA_57_BASE_IDX                                                                      0
+#define regROM_SW_DATA_58                                                                               0x0125
+#define regROM_SW_DATA_58_BASE_IDX                                                                      0
+#define regROM_SW_DATA_59                                                                               0x0126
+#define regROM_SW_DATA_59_BASE_IDX                                                                      0
+#define regROM_SW_DATA_60                                                                               0x0127
+#define regROM_SW_DATA_60_BASE_IDX                                                                      0
+#define regROM_SW_DATA_61                                                                               0x0128
+#define regROM_SW_DATA_61_BASE_IDX                                                                      0
+#define regROM_SW_DATA_62                                                                               0x0129
+#define regROM_SW_DATA_62_BASE_IDX                                                                      0
+#define regROM_SW_DATA_63                                                                               0x012a
+#define regROM_SW_DATA_63_BASE_IDX                                                                      0
+#define regROM_SW_DATA_64                                                                               0x012b
+#define regROM_SW_DATA_64_BASE_IDX                                                                      0
+
+
+// addressBlock: smuio_smuio_gpio_SmuSmuioDec
+// base address: 0x5a500
+#define regSMU_GPIOPAD_SW_INT_STAT                                                                      0x0140
+#define regSMU_GPIOPAD_SW_INT_STAT_BASE_IDX                                                             0
+#define regSMU_GPIOPAD_MASK                                                                             0x0141
+#define regSMU_GPIOPAD_MASK_BASE_IDX                                                                    0
+#define regSMU_GPIOPAD_A                                                                                0x0142
+#define regSMU_GPIOPAD_A_BASE_IDX                                                                       0
+#define regSMU_GPIOPAD_TXIMPSEL                                                                         0x0143
+#define regSMU_GPIOPAD_TXIMPSEL_BASE_IDX                                                                0
+#define regSMU_GPIOPAD_EN                                                                               0x0144
+#define regSMU_GPIOPAD_EN_BASE_IDX                                                                      0
+#define regSMU_GPIOPAD_Y                                                                                0x0145
+#define regSMU_GPIOPAD_Y_BASE_IDX                                                                       0
+#define regSMU_GPIOPAD_RXEN                                                                             0x0146
+#define regSMU_GPIOPAD_RXEN_BASE_IDX                                                                    0
+#define regSMU_GPIOPAD_RCVR_SEL0                                                                        0x0147
+#define regSMU_GPIOPAD_RCVR_SEL0_BASE_IDX                                                               0
+#define regSMU_GPIOPAD_RCVR_SEL1                                                                        0x0148
+#define regSMU_GPIOPAD_RCVR_SEL1_BASE_IDX                                                               0
+#define regSMU_GPIOPAD_PU_EN                                                                            0x0149
+#define regSMU_GPIOPAD_PU_EN_BASE_IDX                                                                   0
+#define regSMU_GPIOPAD_PD_EN                                                                            0x014a
+#define regSMU_GPIOPAD_PD_EN_BASE_IDX                                                                   0
+#define regSMU_GPIOPAD_PINSTRAPS                                                                        0x014b
+#define regSMU_GPIOPAD_PINSTRAPS_BASE_IDX                                                               0
+#define regDFT_PINSTRAPS                                                                                0x014c
+#define regDFT_PINSTRAPS_BASE_IDX                                                                       0
+#define regSMU_GPIOPAD_INT_STAT_EN                                                                      0x014d
+#define regSMU_GPIOPAD_INT_STAT_EN_BASE_IDX                                                             0
+#define regSMU_GPIOPAD_INT_STAT                                                                         0x014e
+#define regSMU_GPIOPAD_INT_STAT_BASE_IDX                                                                0
+#define regSMU_GPIOPAD_INT_STAT_AK                                                                      0x014f
+#define regSMU_GPIOPAD_INT_STAT_AK_BASE_IDX                                                             0
+#define regSMU_GPIOPAD_INT_EN                                                                           0x0150
+#define regSMU_GPIOPAD_INT_EN_BASE_IDX                                                                  0
+#define regSMU_GPIOPAD_INT_TYPE                                                                         0x0151
+#define regSMU_GPIOPAD_INT_TYPE_BASE_IDX                                                                0
+#define regSMU_GPIOPAD_INT_POLARITY                                                                     0x0152
+#define regSMU_GPIOPAD_INT_POLARITY_BASE_IDX                                                            0
+#define regSMUIO_PCC_GPIO_SELECT                                                                        0x0155
+#define regSMUIO_PCC_GPIO_SELECT_BASE_IDX                                                               0
+#define regSMU_GPIOPAD_S0                                                                               0x0156
+#define regSMU_GPIOPAD_S0_BASE_IDX                                                                      0
+#define regSMU_GPIOPAD_S1                                                                               0x0157
+#define regSMU_GPIOPAD_S1_BASE_IDX                                                                      0
+#define regSMU_GPIOPAD_SCHMEN                                                                           0x0158
+#define regSMU_GPIOPAD_SCHMEN_BASE_IDX                                                                  0
+#define regSMU_GPIOPAD_SCL_EN                                                                           0x0159
+#define regSMU_GPIOPAD_SCL_EN_BASE_IDX                                                                  0
+#define regSMU_GPIOPAD_SDA_EN                                                                           0x015a
+#define regSMU_GPIOPAD_SDA_EN_BASE_IDX                                                                  0
+#define regSMUIO_GPIO_INT0_SELECT                                                                       0x015b
+#define regSMUIO_GPIO_INT0_SELECT_BASE_IDX                                                              0
+#define regSMUIO_GPIO_INT1_SELECT                                                                       0x015c
+#define regSMUIO_GPIO_INT1_SELECT_BASE_IDX                                                              0
+#define regSMUIO_GPIO_INT2_SELECT                                                                       0x015d
+#define regSMUIO_GPIO_INT2_SELECT_BASE_IDX                                                              0
+#define regSMUIO_GPIO_INT3_SELECT                                                                       0x015e
+#define regSMUIO_GPIO_INT3_SELECT_BASE_IDX                                                              0
+#define regSMU_GPIOPAD_MP_INT0_STAT                                                                     0x015f
+#define regSMU_GPIOPAD_MP_INT0_STAT_BASE_IDX                                                            0
+#define regSMU_GPIOPAD_MP_INT1_STAT                                                                     0x0160
+#define regSMU_GPIOPAD_MP_INT1_STAT_BASE_IDX                                                            0
+#define regSMU_GPIOPAD_MP_INT2_STAT                                                                     0x0161
+#define regSMU_GPIOPAD_MP_INT2_STAT_BASE_IDX                                                            0
+#define regSMU_GPIOPAD_MP_INT3_STAT                                                                     0x0162
+#define regSMU_GPIOPAD_MP_INT3_STAT_BASE_IDX                                                            0
+#define regSMIO_INDEX                                                                                   0x0163
+#define regSMIO_INDEX_BASE_IDX                                                                          0
+#define regS0_VID_SMIO_CNTL                                                                             0x0164
+#define regS0_VID_SMIO_CNTL_BASE_IDX                                                                    0
+#define regS1_VID_SMIO_CNTL                                                                             0x0165
+#define regS1_VID_SMIO_CNTL_BASE_IDX                                                                    0
+#define regOPEN_DRAIN_SELECT                                                                            0x0166
+#define regOPEN_DRAIN_SELECT_BASE_IDX                                                                   0
+#define regSMIO_ENABLE                                                                                  0x0167
+#define regSMIO_ENABLE_BASE_IDX                                                                         0
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_15_0_8_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_15_0_8_sh_mask.h
new file mode 100644
index 000000000000..78b1d5cf8835
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_15_0_8_sh_mask.h
@@ -0,0 +1,1113 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _smuio_15_0_8_SH_MASK_HEADER
+#define _smuio_15_0_8_SH_MASK_HEADER
+
+
+// addressBlock: smuio_smuio_tsc_SmuSmuioDec
+//PWROK_REFCLK_GAP_CYCLES
+#define PWROK_REFCLK_GAP_CYCLES__Pwrok_PreAssertion_clkgap_cycles__SHIFT                                      0x0
+#define PWROK_REFCLK_GAP_CYCLES__Pwrok_PostAssertion_clkgap_cycles__SHIFT                                     0x8
+#define PWROK_REFCLK_GAP_CYCLES__Pwrok_PreAssertion_clkgap_cycles_MASK                                        0x000000FFL
+#define PWROK_REFCLK_GAP_CYCLES__Pwrok_PostAssertion_clkgap_cycles_MASK                                       0x0000FF00L
+//GOLDEN_TSC_INCREMENT_UPPER
+#define GOLDEN_TSC_INCREMENT_UPPER__GoldenTscIncrementUpper__SHIFT                                            0x0
+#define GOLDEN_TSC_INCREMENT_UPPER__GoldenTscIncrementUpper_MASK                                              0x00FFFFFFL
+//GOLDEN_TSC_INCREMENT_LOWER
+#define GOLDEN_TSC_INCREMENT_LOWER__GoldenTscIncrementLower__SHIFT                                            0x0
+#define GOLDEN_TSC_INCREMENT_LOWER__GoldenTscIncrementLower_MASK                                              0xFFFFFFFFL
+//GOLDEN_TSC_COUNT_UPPER
+#define GOLDEN_TSC_COUNT_UPPER__GoldenTscCountUpper__SHIFT                                                    0x0
+#define GOLDEN_TSC_COUNT_UPPER__GoldenTscCountUpper_MASK                                                      0x00FFFFFFL
+//GOLDEN_TSC_COUNT_LOWER
+#define GOLDEN_TSC_COUNT_LOWER__GoldenTscCountLower__SHIFT                                                    0x0
+#define GOLDEN_TSC_COUNT_LOWER__GoldenTscCountLower_MASK                                                      0xFFFFFFFFL
+//SOC_GOLDEN_TSC_SHADOW_UPPER
+#define SOC_GOLDEN_TSC_SHADOW_UPPER__SocGoldenTscShadowUpper__SHIFT                                           0x0
+#define SOC_GOLDEN_TSC_SHADOW_UPPER__SocGoldenTscShadowUpper_MASK                                             0x00FFFFFFL
+//SOC_GOLDEN_TSC_SHADOW_LOWER
+#define SOC_GOLDEN_TSC_SHADOW_LOWER__SocGoldenTscShadowLower__SHIFT                                           0x0
+#define SOC_GOLDEN_TSC_SHADOW_LOWER__SocGoldenTscShadowLower_MASK                                             0xFFFFFFFFL
+//SOC_GAP_PWROK
+#define SOC_GAP_PWROK__soc_gap_pwrok__SHIFT                                                                   0x0
+#define SOC_GAP_PWROK__soc_gap_pwrok_MASK                                                                     0x00000001L
+
+
+// addressBlock: smuio_smuio_swtimer_SmuSmuioDec
+//PWR_VIRT_RESET_REQ
+#define PWR_VIRT_RESET_REQ__VF_FLR__SHIFT                                                                     0x0
+#define PWR_VIRT_RESET_REQ__PF_FLR__SHIFT                                                                     0x1f
+#define PWR_VIRT_RESET_REQ__VF_FLR_MASK                                                                       0x7FFFFFFFL
+#define PWR_VIRT_RESET_REQ__PF_FLR_MASK                                                                       0x80000000L
+//PWR_DISP_TIMER_CONTROL
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_COUNT__SHIFT                                                   0x0
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_ENABLE__SHIFT                                                  0x19
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_DISABLE__SHIFT                                                 0x1a
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_MASK__SHIFT                                                    0x1b
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_STAT_AK__SHIFT                                                 0x1c
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_TYPE__SHIFT                                                    0x1d
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_MODE__SHIFT                                                    0x1e
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_COUNT_MASK                                                     0x01FFFFFFL
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_ENABLE_MASK                                                    0x02000000L
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_DISABLE_MASK                                                   0x04000000L
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_MASK_MASK                                                      0x08000000L
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_STAT_AK_MASK                                                   0x10000000L
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_TYPE_MASK                                                      0x20000000L
+#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_MODE_MASK                                                      0x40000000L
+//PWR_DISP_TIMER_DEBUG
+#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT_RUNNING__SHIFT                                                   0x0
+#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT_STAT__SHIFT                                                      0x1
+#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT__SHIFT                                                           0x2
+#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_RUN_VAL__SHIFT                                                       0x7
+#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT_RUNNING_MASK                                                     0x00000001L
+#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT_STAT_MASK                                                        0x00000002L
+#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT_MASK                                                             0x00000004L
+#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_RUN_VAL_MASK                                                         0xFFFFFF80L
+//PWR_DISP_TIMER2_CONTROL
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_COUNT__SHIFT                                                  0x0
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_ENABLE__SHIFT                                                 0x19
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_DISABLE__SHIFT                                                0x1a
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_MASK__SHIFT                                                   0x1b
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_STAT_AK__SHIFT                                                0x1c
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_TYPE__SHIFT                                                   0x1d
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_MODE__SHIFT                                                   0x1e
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_COUNT_MASK                                                    0x01FFFFFFL
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_ENABLE_MASK                                                   0x02000000L
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_DISABLE_MASK                                                  0x04000000L
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_MASK_MASK                                                     0x08000000L
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_STAT_AK_MASK                                                  0x10000000L
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_TYPE_MASK                                                     0x20000000L
+#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_MODE_MASK                                                     0x40000000L
+//PWR_DISP_TIMER2_DEBUG
+#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT_RUNNING__SHIFT                                                  0x0
+#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT_STAT__SHIFT                                                     0x1
+#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT__SHIFT                                                          0x2
+#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_RUN_VAL__SHIFT                                                      0x7
+#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT_RUNNING_MASK                                                    0x00000001L
+#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT_STAT_MASK                                                       0x00000002L
+#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT_MASK                                                            0x00000004L
+#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_RUN_VAL_MASK                                                        0xFFFFFF80L
+//PWR_DISP_TIMER_GLOBAL_CONTROL
+#define PWR_DISP_TIMER_GLOBAL_CONTROL__DISP_TIMER_PULSE_WIDTH__SHIFT                                          0x0
+#define PWR_DISP_TIMER_GLOBAL_CONTROL__DISP_TIMER_PULSE_EN__SHIFT                                             0xa
+#define PWR_DISP_TIMER_GLOBAL_CONTROL__DISP_TIMER_PULSE_WIDTH_MASK                                            0x000003FFL
+#define PWR_DISP_TIMER_GLOBAL_CONTROL__DISP_TIMER_PULSE_EN_MASK                                               0x00000400L
+//PWR_IH_CONTROL
+#define PWR_IH_CONTROL__MAX_CREDIT__SHIFT                                                                     0x0
+#define PWR_IH_CONTROL__DISP_TIMER_TRIGGER_MASK__SHIFT                                                        0x5
+#define PWR_IH_CONTROL__DISP_TIMER2_TRIGGER_MASK__SHIFT                                                       0x6
+#define PWR_IH_CONTROL__PWR_IH_CLK_GATE_EN__SHIFT                                                             0x1f
+#define PWR_IH_CONTROL__MAX_CREDIT_MASK                                                                       0x0000001FL
+#define PWR_IH_CONTROL__DISP_TIMER_TRIGGER_MASK_MASK                                                          0x00000020L
+#define PWR_IH_CONTROL__DISP_TIMER2_TRIGGER_MASK_MASK                                                         0x00000040L
+#define PWR_IH_CONTROL__PWR_IH_CLK_GATE_EN_MASK                                                               0x80000000L
+
+
+// addressBlock: smuio_smuio_misc_SmuSmuioDec
+//SMUIO_MCM_CONFIG
+#define SMUIO_MCM_CONFIG__DIE_ID__SHIFT                                                                       0x0
+#define SMUIO_MCM_CONFIG__PKG_TYPE__SHIFT                                                                     0x2
+#define SMUIO_MCM_CONFIG__SOCKET_ID__SHIFT                                                                    0x8
+#define SMUIO_MCM_CONFIG__CONSOLE_K__SHIFT                                                                    0x10
+#define SMUIO_MCM_CONFIG__CONSOLE_A__SHIFT                                                                    0x11
+#define SMUIO_MCM_CONFIG__PKG_SUBTYPE__SHIFT                                                                  0x12
+#define SMUIO_MCM_CONFIG__TOPOLOGY_ID__SHIFT                                                                  0x13
+#define SMUIO_MCM_CONFIG__PKG_TYPE_DFX__SHIFT                                                                 0x18
+#define SMUIO_MCM_CONFIG__DIE_ID_MASK                                                                         0x00000003L
+#define SMUIO_MCM_CONFIG__PKG_TYPE_MASK                                                                       0x0000003CL
+#define SMUIO_MCM_CONFIG__SOCKET_ID_MASK                                                                      0x00001F00L
+#define SMUIO_MCM_CONFIG__CONSOLE_K_MASK                                                                      0x00010000L
+#define SMUIO_MCM_CONFIG__CONSOLE_A_MASK                                                                      0x00020000L
+#define SMUIO_MCM_CONFIG__PKG_SUBTYPE_MASK                                                                    0x00040000L
+#define SMUIO_MCM_CONFIG__TOPOLOGY_ID_MASK                                                                    0x00F80000L
+//IP_DISCOVERY_VERSION
+#define IP_DISCOVERY_VERSION__IP_DISCOVERY_VERSION__SHIFT                                                     0x0
+#define IP_DISCOVERY_VERSION__IP_DISCOVERY_VERSION_MASK                                                       0xFFFFFFFFL
+//SCRATCH_REGISTER0
+#define SCRATCH_REGISTER0__ScratchPad0__SHIFT                                                                 0x0
+#define SCRATCH_REGISTER0__ScratchPad0_MASK                                                                   0xFFFFFFFFL
+//SCRATCH_REGISTER1
+#define SCRATCH_REGISTER1__ScratchPad1__SHIFT                                                                 0x0
+#define SCRATCH_REGISTER1__ScratchPad1_MASK                                                                   0xFFFFFFFFL
+//SCRATCH_REGISTER2
+#define SCRATCH_REGISTER2__ScratchPad2__SHIFT                                                                 0x0
+#define SCRATCH_REGISTER2__ScratchPad2_MASK                                                                   0xFFFFFFFFL
+//SCRATCH_REGISTER3
+#define SCRATCH_REGISTER3__ScratchPad3__SHIFT                                                                 0x0
+#define SCRATCH_REGISTER3__ScratchPad3_MASK                                                                   0xFFFFFFFFL
+//SCRATCH_REGISTER4
+#define SCRATCH_REGISTER4__ScratchPad4__SHIFT                                                                 0x0
+#define SCRATCH_REGISTER4__ScratchPad4_MASK                                                                   0xFFFFFFFFL
+//SCRATCH_REGISTER5
+#define SCRATCH_REGISTER5__ScratchPad5__SHIFT                                                                 0x0
+#define SCRATCH_REGISTER5__ScratchPad5_MASK                                                                   0xFFFFFFFFL
+//SCRATCH_REGISTER6
+#define SCRATCH_REGISTER6__ScratchPad6__SHIFT                                                                 0x0
+#define SCRATCH_REGISTER6__ScratchPad6_MASK                                                                   0xFFFFFFFFL
+//SCRATCH_REGISTER7
+#define SCRATCH_REGISTER7__ScratchPad7__SHIFT                                                                 0x0
+#define SCRATCH_REGISTER7__ScratchPad7_MASK                                                                   0xFFFFFFFFL
+
+
+// addressBlock: smuio_smuio_i2c_SmuSmuioDec
+//CKSVII2C_IC_CON
+#define CKSVII2C_IC_CON__IC_MASTER_MODE__SHIFT                                                                0x0
+#define CKSVII2C_IC_CON__IC_MAX_SPEED_MODE__SHIFT                                                             0x1
+#define CKSVII2C_IC_CON__IC_10BITADDR_SLAVE__SHIFT                                                            0x3
+#define CKSVII2C_IC_CON__IC_10BITADDR_MASTER__SHIFT                                                           0x4
+#define CKSVII2C_IC_CON__IC_RESTART_EN__SHIFT                                                                 0x5
+#define CKSVII2C_IC_CON__IC_SLAVE_DISABLE__SHIFT                                                              0x6
+#define CKSVII2C_IC_CON__STOP_DET_IFADDRESSED__SHIFT                                                          0x7
+#define CKSVII2C_IC_CON__TX_EMPTY_CTRL__SHIFT                                                                 0x8
+#define CKSVII2C_IC_CON__RX_FIFO_FULL_HLD_CTRL__SHIFT                                                         0x9
+#define CKSVII2C_IC_CON__BUS_CLEAR_FEATURE_CTRL__SHIFT                                                        0xb
+#define CKSVII2C_IC_CON__IC_MASTER_MODE_MASK                                                                  0x00000001L
+#define CKSVII2C_IC_CON__IC_MAX_SPEED_MODE_MASK                                                               0x00000006L
+#define CKSVII2C_IC_CON__IC_10BITADDR_SLAVE_MASK                                                              0x00000008L
+#define CKSVII2C_IC_CON__IC_10BITADDR_MASTER_MASK                                                             0x00000010L
+#define CKSVII2C_IC_CON__IC_RESTART_EN_MASK                                                                   0x00000020L
+#define CKSVII2C_IC_CON__IC_SLAVE_DISABLE_MASK                                                                0x00000040L
+#define CKSVII2C_IC_CON__STOP_DET_IFADDRESSED_MASK                                                            0x00000080L
+#define CKSVII2C_IC_CON__TX_EMPTY_CTRL_MASK                                                                   0x00000100L
+#define CKSVII2C_IC_CON__RX_FIFO_FULL_HLD_CTRL_MASK                                                           0x00000200L
+//CKSVII2C_IC_TAR
+#define CKSVII2C_IC_TAR__IC_TAR__SHIFT                                                                        0x0
+#define CKSVII2C_IC_TAR__GC_OR_START__SHIFT                                                                   0xa
+#define CKSVII2C_IC_TAR__SPECIAL__SHIFT                                                                       0xb
+#define CKSVII2C_IC_TAR__IC_10BITADDR_MASTER__SHIFT                                                           0xc
+#define CKSVII2C_IC_TAR__IC_TAR_MASK                                                                          0x000003FFL
+#define CKSVII2C_IC_TAR__GC_OR_START_MASK                                                                     0x00000400L
+#define CKSVII2C_IC_TAR__SPECIAL_MASK                                                                         0x00000800L
+#define CKSVII2C_IC_TAR__IC_10BITADDR_MASTER_MASK                                                             0x00001000L
+//CKSVII2C_IC_SAR
+#define CKSVII2C_IC_SAR__IC_SAR__SHIFT                                                                        0x0
+#define CKSVII2C_IC_SAR__IC_SAR_MASK                                                                          0x000003FFL
+//CKSVII2C_IC_HS_MADDR
+#define CKSVII2C_IC_HS_MADDR__IC_HS_MADDR__SHIFT                                                              0x0
+#define CKSVII2C_IC_HS_MADDR__IC_HS_MADDR_MASK                                                                0x00000007L
+//CKSVII2C_IC_DATA_CMD
+#define CKSVII2C_IC_DATA_CMD__DAT__SHIFT                                                                      0x0
+#define CKSVII2C_IC_DATA_CMD__CMD__SHIFT                                                                      0x8
+#define CKSVII2C_IC_DATA_CMD__STOP__SHIFT                                                                     0x9
+#define CKSVII2C_IC_DATA_CMD__RESTART__SHIFT                                                                  0xa
+#define CKSVII2C_IC_DATA_CMD__FIRST_DATA_BYTE__SHIFT                                                          0xb
+#define CKSVII2C_IC_DATA_CMD__DAT_MASK                                                                        0x000000FFL
+#define CKSVII2C_IC_DATA_CMD__CMD_MASK                                                                        0x00000100L
+#define CKSVII2C_IC_DATA_CMD__STOP_MASK                                                                       0x00000200L
+#define CKSVII2C_IC_DATA_CMD__RESTART_MASK                                                                    0x00000400L
+//CKSVII2C_IC_SS_SCL_HCNT
+#define CKSVII2C_IC_SS_SCL_HCNT__IC_SS_SCL_HCNT__SHIFT                                                        0x0
+#define CKSVII2C_IC_SS_SCL_HCNT__IC_SS_SCL_HCNT_MASK                                                          0x0000FFFFL
+//CKSVII2C_IC_SS_SCL_LCNT
+#define CKSVII2C_IC_SS_SCL_LCNT__IC_SS_SCL_LCNT__SHIFT                                                        0x0
+#define CKSVII2C_IC_SS_SCL_LCNT__IC_SS_SCL_LCNT_MASK                                                          0x0000FFFFL
+//CKSVII2C_IC_FS_SCL_HCNT
+#define CKSVII2C_IC_FS_SCL_HCNT__IC_FS_SCL_HCNT__SHIFT                                                        0x0
+#define CKSVII2C_IC_FS_SCL_HCNT__IC_FS_SCL_HCNT_MASK                                                          0x0000FFFFL
+//CKSVII2C_IC_FS_SCL_LCNT
+#define CKSVII2C_IC_FS_SCL_LCNT__IC_FS_SCL_LCNT__SHIFT                                                        0x0
+#define CKSVII2C_IC_FS_SCL_LCNT__IC_FS_SCL_LCNT_MASK                                                          0x0000FFFFL
+//CKSVII2C_IC_HS_SCL_HCNT
+#define CKSVII2C_IC_HS_SCL_HCNT__IC_HS_SCL_HCNT__SHIFT                                                        0x0
+#define CKSVII2C_IC_HS_SCL_HCNT__IC_HS_SCL_HCNT_MASK                                                          0x0000FFFFL
+//CKSVII2C_IC_HS_SCL_LCNT
+#define CKSVII2C_IC_HS_SCL_LCNT__IC_HS_SCL_LCNT__SHIFT                                                        0x0
+#define CKSVII2C_IC_HS_SCL_LCNT__IC_HS_SCL_LCNT_MASK                                                          0x0000FFFFL
+//CKSVII2C_IC_INTR_STAT
+#define CKSVII2C_IC_INTR_STAT__R_RX_UNDER__SHIFT                                                              0x0
+#define CKSVII2C_IC_INTR_STAT__R_RX_OVER__SHIFT                                                               0x1
+#define CKSVII2C_IC_INTR_STAT__R_RX_FULL__SHIFT                                                               0x2
+#define CKSVII2C_IC_INTR_STAT__R_TX_OVER__SHIFT                                                               0x3
+#define CKSVII2C_IC_INTR_STAT__R_TX_EMPTY__SHIFT                                                              0x4
+#define CKSVII2C_IC_INTR_STAT__R_RD_REQ__SHIFT                                                                0x5
+#define CKSVII2C_IC_INTR_STAT__R_TX_ABRT__SHIFT                                                               0x6
+#define CKSVII2C_IC_INTR_STAT__R_RX_DONE__SHIFT                                                               0x7
+#define CKSVII2C_IC_INTR_STAT__R_ACTIVITY__SHIFT                                                              0x8
+#define CKSVII2C_IC_INTR_STAT__R_STOP_DET__SHIFT                                                              0x9
+#define CKSVII2C_IC_INTR_STAT__R_START_DET__SHIFT                                                             0xa
+#define CKSVII2C_IC_INTR_STAT__R_GEN_CALL__SHIFT                                                              0xb
+#define CKSVII2C_IC_INTR_STAT__R_RESTART_DET__SHIFT                                                           0xc
+#define CKSVII2C_IC_INTR_STAT__R_MST_ON_HOLD__SHIFT                                                           0xd
+#define CKSVII2C_IC_INTR_STAT__R_SCL_STUCK_AT_LOW__SHIFT                                                      0xe
+#define CKSVII2C_IC_INTR_STAT__R_RX_UNDER_MASK                                                                0x00000001L
+#define CKSVII2C_IC_INTR_STAT__R_RX_OVER_MASK                                                                 0x00000002L
+#define CKSVII2C_IC_INTR_STAT__R_RX_FULL_MASK                                                                 0x00000004L
+#define CKSVII2C_IC_INTR_STAT__R_TX_OVER_MASK                                                                 0x00000008L
+#define CKSVII2C_IC_INTR_STAT__R_TX_EMPTY_MASK                                                                0x00000010L
+#define CKSVII2C_IC_INTR_STAT__R_RD_REQ_MASK                                                                  0x00000020L
+#define CKSVII2C_IC_INTR_STAT__R_TX_ABRT_MASK                                                                 0x00000040L
+#define CKSVII2C_IC_INTR_STAT__R_RX_DONE_MASK                                                                 0x00000080L
+#define CKSVII2C_IC_INTR_STAT__R_ACTIVITY_MASK                                                                0x00000100L
+#define CKSVII2C_IC_INTR_STAT__R_STOP_DET_MASK                                                                0x00000200L
+#define CKSVII2C_IC_INTR_STAT__R_START_DET_MASK                                                               0x00000400L
+#define CKSVII2C_IC_INTR_STAT__R_GEN_CALL_MASK                                                                0x00000800L
+#define CKSVII2C_IC_INTR_STAT__R_RESTART_DET_MASK                                                             0x00001000L
+#define CKSVII2C_IC_INTR_STAT__R_MST_ON_HOLD_MASK                                                             0x00002000L
+//CKSVII2C_IC_INTR_MASK
+#define CKSVII2C_IC_INTR_MASK__M_RX_UNDER__SHIFT                                                              0x0
+#define CKSVII2C_IC_INTR_MASK__M_RX_OVER__SHIFT                                                               0x1
+#define CKSVII2C_IC_INTR_MASK__M_RX_FULL__SHIFT                                                               0x2
+#define CKSVII2C_IC_INTR_MASK__M_TX_OVER__SHIFT                                                               0x3
+#define CKSVII2C_IC_INTR_MASK__M_TX_EMPTY__SHIFT                                                              0x4
+#define CKSVII2C_IC_INTR_MASK__M_RD_REQ__SHIFT                                                                0x5
+#define CKSVII2C_IC_INTR_MASK__M_TX_ABRT__SHIFT                                                               0x6
+#define CKSVII2C_IC_INTR_MASK__M_RX_DONE__SHIFT                                                               0x7
+#define CKSVII2C_IC_INTR_MASK__M_ACTIVITY__SHIFT                                                              0x8
+#define CKSVII2C_IC_INTR_MASK__M_STOP_DET__SHIFT                                                              0x9
+#define CKSVII2C_IC_INTR_MASK__M_START_DET__SHIFT                                                             0xa
+#define CKSVII2C_IC_INTR_MASK__M_GEN_CALL__SHIFT                                                              0xb
+#define CKSVII2C_IC_INTR_MASK__M_RESTART_DET__SHIFT                                                           0xc
+#define CKSVII2C_IC_INTR_MASK__M_MST_ON_HOLD__SHIFT                                                           0xd
+#define CKSVII2C_IC_INTR_MASK__M_SCL_STUCK_AT_LOW__SHIFT                                                      0xe
+#define CKSVII2C_IC_INTR_MASK__M_RX_UNDER_MASK                                                                0x00000001L
+#define CKSVII2C_IC_INTR_MASK__M_RX_OVER_MASK                                                                 0x00000002L
+#define CKSVII2C_IC_INTR_MASK__M_RX_FULL_MASK                                                                 0x00000004L
+#define CKSVII2C_IC_INTR_MASK__M_TX_OVER_MASK                                                                 0x00000008L
+#define CKSVII2C_IC_INTR_MASK__M_TX_EMPTY_MASK                                                                0x00000010L
+#define CKSVII2C_IC_INTR_MASK__M_RD_REQ_MASK                                                                  0x00000020L
+#define CKSVII2C_IC_INTR_MASK__M_TX_ABRT_MASK                                                                 0x00000040L
+#define CKSVII2C_IC_INTR_MASK__M_RX_DONE_MASK                                                                 0x00000080L
+#define CKSVII2C_IC_INTR_MASK__M_ACTIVITY_MASK                                                                0x00000100L
+#define CKSVII2C_IC_INTR_MASK__M_STOP_DET_MASK                                                                0x00000200L
+#define CKSVII2C_IC_INTR_MASK__M_START_DET_MASK                                                               0x00000400L
+#define CKSVII2C_IC_INTR_MASK__M_GEN_CALL_MASK                                                                0x00000800L
+#define CKSVII2C_IC_INTR_MASK__M_RESTART_DET_MASK                                                             0x00001000L
+#define CKSVII2C_IC_INTR_MASK__M_MST_ON_HOLD_MASK                                                             0x00002000L
+//CKSVII2C_IC_RAW_INTR_STAT
+//CKSVII2C_IC_RX_TL
+#define CKSVII2C_IC_RX_TL__RX_TL__SHIFT                                                                       0x0
+//CKSVII2C_IC_TX_TL
+#define CKSVII2C_IC_TX_TL__TX_TL__SHIFT                                                                       0x0
+//CKSVII2C_IC_CLR_INTR
+//CKSVII2C_IC_CLR_RX_UNDER
+//CKSVII2C_IC_CLR_RX_OVER
+//CKSVII2C_IC_CLR_TX_OVER
+//CKSVII2C_IC_CLR_RD_REQ
+//CKSVII2C_IC_CLR_TX_ABRT
+//CKSVII2C_IC_CLR_RX_DONE
+//CKSVII2C_IC_CLR_ACTIVITY
+//CKSVII2C_IC_CLR_STOP_DET
+//CKSVII2C_IC_CLR_START_DET
+//CKSVII2C_IC_CLR_GEN_CALL
+//CKSVII2C_IC_ENABLE
+#define CKSVII2C_IC_ENABLE__ENABLE__SHIFT                                                                     0x0
+#define CKSVII2C_IC_ENABLE__ABORT__SHIFT                                                                      0x1
+#define CKSVII2C_IC_ENABLE__TX_CMD_BLOCK__SHIFT                                                               0x2
+#define CKSVII2C_IC_ENABLE__SDA_STUCK_RECOVERY_ENABLE__SHIFT                                                  0x3
+#define CKSVII2C_IC_ENABLE__ENABLE_MASK                                                                       0x00000001L
+#define CKSVII2C_IC_ENABLE__ABORT_MASK                                                                        0x00000002L
+//CKSVII2C_IC_STATUS
+#define CKSVII2C_IC_STATUS__ACTIVITY__SHIFT                                                                   0x0
+#define CKSVII2C_IC_STATUS__TFNF__SHIFT                                                                       0x1
+#define CKSVII2C_IC_STATUS__TFE__SHIFT                                                                        0x2
+#define CKSVII2C_IC_STATUS__RFNE__SHIFT                                                                       0x3
+#define CKSVII2C_IC_STATUS__RFF__SHIFT                                                                        0x4
+#define CKSVII2C_IC_STATUS__MST_ACTIVITY__SHIFT                                                               0x5
+#define CKSVII2C_IC_STATUS__SLV_ACTIVITY__SHIFT                                                               0x6
+#define CKSVII2C_IC_STATUS__MST_HOLD_TX_FIFO_EMPTY__SHIFT                                                     0x7
+#define CKSVII2C_IC_STATUS__MST_HOLD_RX_FIFO_FULL__SHIFT                                                      0x8
+#define CKSVII2C_IC_STATUS__SLV_HOLD_TX_FIFO_EMPTY__SHIFT                                                     0x9
+#define CKSVII2C_IC_STATUS__SLV_HOLD_RX_FIFO_FULL__SHIFT                                                      0xa
+#define CKSVII2C_IC_STATUS__SDA_STUCK_NOT_RECOVERED__SHIFT                                                    0xb
+#define CKSVII2C_IC_STATUS__ACTIVITY_MASK                                                                     0x00000001L
+#define CKSVII2C_IC_STATUS__TFNF_MASK                                                                         0x00000002L
+#define CKSVII2C_IC_STATUS__TFE_MASK                                                                          0x00000004L
+#define CKSVII2C_IC_STATUS__RFNE_MASK                                                                         0x00000008L
+#define CKSVII2C_IC_STATUS__RFF_MASK                                                                          0x00000010L
+#define CKSVII2C_IC_STATUS__MST_ACTIVITY_MASK                                                                 0x00000020L
+#define CKSVII2C_IC_STATUS__SLV_ACTIVITY_MASK                                                                 0x00000040L
+//CKSVII2C_IC_TXFLR
+#define CKSVII2C_IC_TXFLR__TXFLR__SHIFT                                                                       0x0
+//CKSVII2C_IC_RXFLR
+#define CKSVII2C_IC_RXFLR__RXFLR__SHIFT                                                                       0x0
+//CKSVII2C_IC_SDA_HOLD
+#define CKSVII2C_IC_SDA_HOLD__IC_SDA_TX_HOLD__SHIFT                                                           0x0
+#define CKSVII2C_IC_SDA_HOLD__IC_SDA_RX_HOLD__SHIFT                                                           0x10
+//CKSVII2C_IC_TX_ABRT_SOURCE
+//CKSVII2C_IC_SLV_DATA_NACK_ONLY
+//CKSVII2C_IC_DMA_CR
+//CKSVII2C_IC_DMA_TDLR
+//CKSVII2C_IC_DMA_RDLR
+//CKSVII2C_IC_SDA_SETUP
+#define CKSVII2C_IC_SDA_SETUP__SDA_SETUP__SHIFT                                                               0x0
+#define CKSVII2C_IC_SDA_SETUP__SDA_SETUP_MASK                                                                 0x000000FFL
+//CKSVII2C_IC_ACK_GENERAL_CALL
+#define CKSVII2C_IC_ACK_GENERAL_CALL__ACK_GENERAL_CALL__SHIFT                                                 0x0
+#define CKSVII2C_IC_ACK_GENERAL_CALL__ACK_GENERAL_CALL_MASK                                                   0x00000001L
+//CKSVII2C_IC_ENABLE_STATUS
+#define CKSVII2C_IC_ENABLE_STATUS__IC_EN__SHIFT                                                               0x0
+#define CKSVII2C_IC_ENABLE_STATUS__SLV_DISABLED_WHILE_BUSY__SHIFT                                             0x1
+#define CKSVII2C_IC_ENABLE_STATUS__SLV_RX_DATA_LOST__SHIFT                                                    0x2
+#define CKSVII2C_IC_ENABLE_STATUS__IC_EN_MASK                                                                 0x00000001L
+//CKSVII2C_IC_FS_SPKLEN
+#define CKSVII2C_IC_FS_SPKLEN__FS_SPKLEN__SHIFT                                                               0x0
+#define CKSVII2C_IC_FS_SPKLEN__FS_SPKLEN_MASK                                                                 0x000000FFL
+//CKSVII2C_IC_HS_SPKLEN
+#define CKSVII2C_IC_HS_SPKLEN__HS_SPKLEN__SHIFT                                                               0x0
+#define CKSVII2C_IC_HS_SPKLEN__HS_SPKLEN_MASK                                                                 0x000000FFL
+//CKSVII2C_IC_CLR_RESTART_DET
+//CKSVII2C_IC_COMP_PARAM_1
+#define CKSVII2C_IC_COMP_PARAM_1__APB_DATA_WIDTH__SHIFT                                                       0x0
+#define CKSVII2C_IC_COMP_PARAM_1__MAX_SPEED_MODE__SHIFT                                                       0x2
+#define CKSVII2C_IC_COMP_PARAM_1__HC_COUNT_VALUES__SHIFT                                                      0x4
+#define CKSVII2C_IC_COMP_PARAM_1__INTR_IO__SHIFT                                                              0x5
+#define CKSVII2C_IC_COMP_PARAM_1__HAS_DMA__SHIFT                                                              0x6
+#define CKSVII2C_IC_COMP_PARAM_1__ADD_ENCODED_PARAMS__SHIFT                                                   0x7
+#define CKSVII2C_IC_COMP_PARAM_1__RX_BUFFER_DEPTH__SHIFT                                                      0x8
+#define CKSVII2C_IC_COMP_PARAM_1__TX_BUFFER_DEPTH__SHIFT                                                      0x10
+//CKSVII2C_IC_COMP_VERSION
+#define CKSVII2C_IC_COMP_VERSION__COMP_VERSION__SHIFT                                                         0x0
+#define CKSVII2C_IC_COMP_VERSION__COMP_VERSION_MASK                                                           0xFFFFFFFFL
+//CKSVII2C_IC_COMP_TYPE
+#define CKSVII2C_IC_COMP_TYPE__COMP_TYPE__SHIFT                                                               0x0
+#define CKSVII2C_IC_COMP_TYPE__COMP_TYPE_MASK                                                                 0xFFFFFFFFL
+//CKSVII2C1_IC_CON
+#define CKSVII2C1_IC_CON__IC1_MASTER_MODE__SHIFT                                                              0x0
+#define CKSVII2C1_IC_CON__IC1_MAX_SPEED_MODE__SHIFT                                                           0x1
+#define CKSVII2C1_IC_CON__IC1_10BITADDR_SLAVE__SHIFT                                                          0x3
+#define CKSVII2C1_IC_CON__IC1_10BITADDR_MASTER__SHIFT                                                         0x4
+#define CKSVII2C1_IC_CON__IC1_RESTART_EN__SHIFT                                                               0x5
+#define CKSVII2C1_IC_CON__IC1_SLAVE_DISABLE__SHIFT                                                            0x6
+#define CKSVII2C1_IC_CON__STOP1_DET_IFADDRESSED__SHIFT                                                        0x7
+#define CKSVII2C1_IC_CON__TX1_EMPTY_CTRL__SHIFT                                                               0x8
+#define CKSVII2C1_IC_CON__RX1_FIFO_FULL_HLD_CTRL__SHIFT                                                       0x9
+#define CKSVII2C1_IC_CON__BUS_CLEAR_FEATURE_CTRL1__SHIFT                                                      0xb
+#define CKSVII2C1_IC_CON__IC1_MASTER_MODE_MASK                                                                0x00000001L
+#define CKSVII2C1_IC_CON__IC1_MAX_SPEED_MODE_MASK                                                             0x00000006L
+#define CKSVII2C1_IC_CON__IC1_10BITADDR_SLAVE_MASK                                                            0x00000008L
+#define CKSVII2C1_IC_CON__IC1_10BITADDR_MASTER_MASK                                                           0x00000010L
+#define CKSVII2C1_IC_CON__IC1_RESTART_EN_MASK                                                                 0x00000020L
+#define CKSVII2C1_IC_CON__IC1_SLAVE_DISABLE_MASK                                                              0x00000040L
+#define CKSVII2C1_IC_CON__STOP1_DET_IFADDRESSED_MASK                                                          0x00000080L
+#define CKSVII2C1_IC_CON__TX1_EMPTY_CTRL_MASK                                                                 0x00000100L
+#define CKSVII2C1_IC_CON__RX1_FIFO_FULL_HLD_CTRL_MASK                                                         0x00000200L
+//CKSVII2C1_IC_TAR
+#define CKSVII2C1_IC_TAR__IC1_TAR__SHIFT                                                                      0x0
+#define CKSVII2C1_IC_TAR__GC1_OR_START__SHIFT                                                                 0xa
+#define CKSVII2C1_IC_TAR__SPECIAL1__SHIFT                                                                     0xb
+#define CKSVII2C1_IC_TAR__IC1_10BITADDR_MASTER__SHIFT                                                         0xc
+#define CKSVII2C1_IC_TAR__IC1_TAR_MASK                                                                        0x000003FFL
+#define CKSVII2C1_IC_TAR__GC1_OR_START_MASK                                                                   0x00000400L
+#define CKSVII2C1_IC_TAR__SPECIAL1_MASK                                                                       0x00000800L
+#define CKSVII2C1_IC_TAR__IC1_10BITADDR_MASTER_MASK                                                           0x00001000L
+//CKSVII2C1_IC_SAR
+#define CKSVII2C1_IC_SAR__IC1_SAR__SHIFT                                                                      0x0
+#define CKSVII2C1_IC_SAR__IC1_SAR_MASK                                                                        0x000003FFL
+//CKSVII2C1_IC_HS_MADDR
+#define CKSVII2C1_IC_HS_MADDR__IC1_HS_MADDR__SHIFT                                                            0x0
+#define CKSVII2C1_IC_HS_MADDR__IC1_HS_MADDR_MASK                                                              0x00000007L
+//CKSVII2C1_IC_DATA_CMD
+#define CKSVII2C1_IC_DATA_CMD__DAT1__SHIFT                                                                    0x0
+#define CKSVII2C1_IC_DATA_CMD__CMD1__SHIFT                                                                    0x8
+#define CKSVII2C1_IC_DATA_CMD__STOP1__SHIFT                                                                   0x9
+#define CKSVII2C1_IC_DATA_CMD__RESTART1__SHIFT                                                                0xa
+#define CKSVII2C1_IC_DATA_CMD__FIRST1_DATA_BYTE__SHIFT                                                        0xb
+#define CKSVII2C1_IC_DATA_CMD__DAT1_MASK                                                                      0x000000FFL
+#define CKSVII2C1_IC_DATA_CMD__CMD1_MASK                                                                      0x00000100L
+#define CKSVII2C1_IC_DATA_CMD__STOP1_MASK                                                                     0x00000200L
+#define CKSVII2C1_IC_DATA_CMD__RESTART1_MASK                                                                  0x00000400L
+//CKSVII2C1_IC_SS_SCL_HCNT
+#define CKSVII2C1_IC_SS_SCL_HCNT__IC1_SS_SCL_HCNT__SHIFT                                                      0x0
+#define CKSVII2C1_IC_SS_SCL_HCNT__IC1_SS_SCL_HCNT_MASK                                                        0x0000FFFFL
+//CKSVII2C1_IC_SS_SCL_LCNT
+#define CKSVII2C1_IC_SS_SCL_LCNT__IC1_SS_SCL_LCNT__SHIFT                                                      0x0
+#define CKSVII2C1_IC_SS_SCL_LCNT__IC1_SS_SCL_LCNT_MASK                                                        0x0000FFFFL
+//CKSVII2C1_IC_FS_SCL_HCNT
+#define CKSVII2C1_IC_FS_SCL_HCNT__IC1_FS_SCL_HCNT__SHIFT                                                      0x0
+#define CKSVII2C1_IC_FS_SCL_HCNT__IC1_FS_SCL_HCNT_MASK                                                        0x0000FFFFL
+//CKSVII2C1_IC_FS_SCL_LCNT
+#define CKSVII2C1_IC_FS_SCL_LCNT__IC1_FS_SCL_LCNT__SHIFT                                                      0x0
+#define CKSVII2C1_IC_FS_SCL_LCNT__IC1_FS_SCL_LCNT_MASK                                                        0x0000FFFFL
+//CKSVII2C1_IC_HS_SCL_HCNT
+#define CKSVII2C1_IC_HS_SCL_HCNT__IC1_HS_SCL_HCNT__SHIFT                                                      0x0
+#define CKSVII2C1_IC_HS_SCL_HCNT__IC1_HS_SCL_HCNT_MASK                                                        0x0000FFFFL
+//CKSVII2C1_IC_HS_SCL_LCNT
+#define CKSVII2C1_IC_HS_SCL_LCNT__IC1_HS_SCL_LCNT__SHIFT                                                      0x0
+#define CKSVII2C1_IC_HS_SCL_LCNT__IC1_HS_SCL_LCNT_MASK                                                        0x0000FFFFL
+//CKSVII2C1_IC_INTR_STAT
+#define CKSVII2C1_IC_INTR_STAT__R1_RX_UNDER__SHIFT                                                            0x0
+#define CKSVII2C1_IC_INTR_STAT__R1_RX_OVER__SHIFT                                                             0x1
+#define CKSVII2C1_IC_INTR_STAT__R1_RX_FULL__SHIFT                                                             0x2
+#define CKSVII2C1_IC_INTR_STAT__R1_TX_OVER__SHIFT                                                             0x3
+#define CKSVII2C1_IC_INTR_STAT__R1_TX_EMPTY__SHIFT                                                            0x4
+#define CKSVII2C1_IC_INTR_STAT__R1_RD_REQ__SHIFT                                                              0x5
+#define CKSVII2C1_IC_INTR_STAT__R1_TX_ABRT__SHIFT                                                             0x6
+#define CKSVII2C1_IC_INTR_STAT__R1_RX_DONE__SHIFT                                                             0x7
+#define CKSVII2C1_IC_INTR_STAT__R1_ACTIVITY__SHIFT                                                            0x8
+#define CKSVII2C1_IC_INTR_STAT__R1_STOP_DET__SHIFT                                                            0x9
+#define CKSVII2C1_IC_INTR_STAT__R1_START_DET__SHIFT                                                           0xa
+#define CKSVII2C1_IC_INTR_STAT__R1_GEN_CALL__SHIFT                                                            0xb
+#define CKSVII2C1_IC_INTR_STAT__R1_RESTART_DET__SHIFT                                                         0xc
+#define CKSVII2C1_IC_INTR_STAT__R1_MST_ON_HOLD__SHIFT                                                         0xd
+#define CKSVII2C1_IC_INTR_STAT__R1_SCL_STUCK_AT_LOW__SHIFT                                                    0xe
+#define CKSVII2C1_IC_INTR_STAT__R1_RX_UNDER_MASK                                                              0x00000001L
+#define CKSVII2C1_IC_INTR_STAT__R1_RX_OVER_MASK                                                               0x00000002L
+#define CKSVII2C1_IC_INTR_STAT__R1_RX_FULL_MASK                                                               0x00000004L
+#define CKSVII2C1_IC_INTR_STAT__R1_TX_OVER_MASK                                                               0x00000008L
+#define CKSVII2C1_IC_INTR_STAT__R1_TX_EMPTY_MASK                                                              0x00000010L
+#define CKSVII2C1_IC_INTR_STAT__R1_RD_REQ_MASK                                                                0x00000020L
+#define CKSVII2C1_IC_INTR_STAT__R1_TX_ABRT_MASK                                                               0x00000040L
+#define CKSVII2C1_IC_INTR_STAT__R1_RX_DONE_MASK                                                               0x00000080L
+#define CKSVII2C1_IC_INTR_STAT__R1_ACTIVITY_MASK                                                              0x00000100L
+#define CKSVII2C1_IC_INTR_STAT__R1_STOP_DET_MASK                                                              0x00000200L
+#define CKSVII2C1_IC_INTR_STAT__R1_START_DET_MASK                                                             0x00000400L
+#define CKSVII2C1_IC_INTR_STAT__R1_GEN_CALL_MASK                                                              0x00000800L
+#define CKSVII2C1_IC_INTR_STAT__R1_RESTART_DET_MASK                                                           0x00001000L
+#define CKSVII2C1_IC_INTR_STAT__R1_MST_ON_HOLD_MASK                                                           0x00002000L
+//CKSVII2C1_IC_INTR_MASK
+#define CKSVII2C1_IC_INTR_MASK__M1_RX_UNDER__SHIFT                                                            0x0
+#define CKSVII2C1_IC_INTR_MASK__M1_RX_OVER__SHIFT                                                             0x1
+#define CKSVII2C1_IC_INTR_MASK__M1_RX_FULL__SHIFT                                                             0x2
+#define CKSVII2C1_IC_INTR_MASK__M1_TX_OVER__SHIFT                                                             0x3
+#define CKSVII2C1_IC_INTR_MASK__M1_TX_EMPTY__SHIFT                                                            0x4
+#define CKSVII2C1_IC_INTR_MASK__M1_RD_REQ__SHIFT                                                              0x5
+#define CKSVII2C1_IC_INTR_MASK__M1_TX_ABRT__SHIFT                                                             0x6
+#define CKSVII2C1_IC_INTR_MASK__M1_RX_DONE__SHIFT                                                             0x7
+#define CKSVII2C1_IC_INTR_MASK__M1_ACTIVITY__SHIFT                                                            0x8
+#define CKSVII2C1_IC_INTR_MASK__M1_STOP_DET__SHIFT                                                            0x9
+#define CKSVII2C1_IC_INTR_MASK__M1_START_DET__SHIFT                                                           0xa
+#define CKSVII2C1_IC_INTR_MASK__M1_GEN_CALL__SHIFT                                                            0xb
+#define CKSVII2C1_IC_INTR_MASK__M1_RESTART_DET__SHIFT                                                         0xc
+#define CKSVII2C1_IC_INTR_MASK__M1_MST_ON_HOLD__SHIFT                                                         0xd
+#define CKSVII2C1_IC_INTR_MASK__M1_SCL_STUCK_AT_LOW__SHIFT                                                    0xe
+#define CKSVII2C1_IC_INTR_MASK__M1_RX_UNDER_MASK                                                              0x00000001L
+#define CKSVII2C1_IC_INTR_MASK__M1_RX_OVER_MASK                                                               0x00000002L
+#define CKSVII2C1_IC_INTR_MASK__M1_RX_FULL_MASK                                                               0x00000004L
+#define CKSVII2C1_IC_INTR_MASK__M1_TX_OVER_MASK                                                               0x00000008L
+#define CKSVII2C1_IC_INTR_MASK__M1_TX_EMPTY_MASK                                                              0x00000010L
+#define CKSVII2C1_IC_INTR_MASK__M1_RD_REQ_MASK                                                                0x00000020L
+#define CKSVII2C1_IC_INTR_MASK__M1_TX_ABRT_MASK                                                               0x00000040L
+#define CKSVII2C1_IC_INTR_MASK__M1_RX_DONE_MASK                                                               0x00000080L
+#define CKSVII2C1_IC_INTR_MASK__M1_ACTIVITY_MASK                                                              0x00000100L
+#define CKSVII2C1_IC_INTR_MASK__M1_STOP_DET_MASK                                                              0x00000200L
+#define CKSVII2C1_IC_INTR_MASK__M1_START_DET_MASK                                                             0x00000400L
+#define CKSVII2C1_IC_INTR_MASK__M1_GEN_CALL_MASK                                                              0x00000800L
+#define CKSVII2C1_IC_INTR_MASK__M1_RESTART_DET_MASK                                                           0x00001000L
+#define CKSVII2C1_IC_INTR_MASK__M1_MST_ON_HOLD_MASK                                                           0x00002000L
+//CKSVII2C1_IC_RAW_INTR_STAT
+//CKSVII2C1_IC_RX_TL
+#define CKSVII2C1_IC_RX_TL__RX1_TL__SHIFT                                                                     0x0
+//CKSVII2C1_IC_TX_TL
+#define CKSVII2C1_IC_TX_TL__TX1_TL__SHIFT                                                                     0x0
+//CKSVII2C1_IC_CLR_INTR
+//CKSVII2C1_IC_CLR_RX_UNDER
+//CKSVII2C1_IC_CLR_RX_OVER
+//CKSVII2C1_IC_CLR_TX_OVER
+//CKSVII2C1_IC_CLR_RD_REQ
+//CKSVII2C1_IC_CLR_TX_ABRT
+//CKSVII2C1_IC_CLR_RX_DONE
+//CKSVII2C1_IC_CLR_ACTIVITY
+//CKSVII2C1_IC_CLR_STOP_DET
+//CKSVII2C1_IC_CLR_START_DET
+//CKSVII2C1_IC_CLR_GEN_CALL
+//CKSVII2C1_IC_ENABLE
+#define CKSVII2C1_IC_ENABLE__ENABLE1__SHIFT                                                                   0x0
+#define CKSVII2C1_IC_ENABLE__ABORT1__SHIFT                                                                    0x1
+#define CKSVII2C1_IC_ENABLE__TX1_CMD_BLOCK__SHIFT                                                             0x2
+#define CKSVII2C1_IC_ENABLE__SDA1_STUCK_RECOVERY_ENABLE__SHIFT                                                0x3
+#define CKSVII2C1_IC_ENABLE__ENABLE1_MASK                                                                     0x00000001L
+#define CKSVII2C1_IC_ENABLE__ABORT1_MASK                                                                      0x00000002L
+//CKSVII2C1_IC_STATUS
+#define CKSVII2C1_IC_STATUS__ACTIVITY1__SHIFT                                                                 0x0
+#define CKSVII2C1_IC_STATUS__TFNF1__SHIFT                                                                     0x1
+#define CKSVII2C1_IC_STATUS__TFE1__SHIFT                                                                      0x2
+#define CKSVII2C1_IC_STATUS__RFNE1__SHIFT                                                                     0x3
+#define CKSVII2C1_IC_STATUS__RFF1__SHIFT                                                                      0x4
+#define CKSVII2C1_IC_STATUS__MST1_ACTIVITY__SHIFT                                                             0x5
+#define CKSVII2C1_IC_STATUS__SLV1_ACTIVITY__SHIFT                                                             0x6
+#define CKSVII2C1_IC_STATUS__MST1_HOLD_TX_FIFO_EMPTY__SHIFT                                                   0x7
+#define CKSVII2C1_IC_STATUS__MST1_HOLD_RX_FIFO_FULL__SHIFT                                                    0x8
+#define CKSVII2C1_IC_STATUS__SLV1_HOLD_TX_FIFO_EMPTY__SHIFT                                                   0x9
+#define CKSVII2C1_IC_STATUS__SLV1_HOLD_RX_FIFO_FULL__SHIFT                                                    0xa
+#define CKSVII2C1_IC_STATUS__SDA1_STUCK_NOT_RECOVERED__SHIFT                                                  0xb
+#define CKSVII2C1_IC_STATUS__ACTIVITY1_MASK                                                                   0x00000001L
+#define CKSVII2C1_IC_STATUS__TFNF1_MASK                                                                       0x00000002L
+#define CKSVII2C1_IC_STATUS__TFE1_MASK                                                                        0x00000004L
+#define CKSVII2C1_IC_STATUS__RFNE1_MASK                                                                       0x00000008L
+#define CKSVII2C1_IC_STATUS__RFF1_MASK                                                                        0x00000010L
+#define CKSVII2C1_IC_STATUS__MST1_ACTIVITY_MASK                                                               0x00000020L
+#define CKSVII2C1_IC_STATUS__SLV1_ACTIVITY_MASK                                                               0x00000040L
+//CKSVII2C1_IC_TXFLR
+#define CKSVII2C1_IC_TXFLR__TXFLR1__SHIFT                                                                     0x0
+//CKSVII2C1_IC_RXFLR
+#define CKSVII2C1_IC_RXFLR__RXFLR1__SHIFT                                                                     0x0
+//CKSVII2C1_IC_SDA_HOLD
+#define CKSVII2C1_IC_SDA_HOLD__IC1_SDA_TX_HOLD__SHIFT                                                         0x0
+#define CKSVII2C1_IC_SDA_HOLD__IC1_SDA_RX_HOLD__SHIFT                                                         0x10
+//CKSVII2C1_IC_TX_ABRT_SOURCE
+//CKSVII2C1_IC_SLV_DATA_NACK_ONLY
+//CKSVII2C1_IC_DMA_CR
+//CKSVII2C1_IC_DMA_TDLR
+//CKSVII2C1_IC_DMA_RDLR
+//CKSVII2C1_IC_SDA_SETUP
+#define CKSVII2C1_IC_SDA_SETUP__SDA1_SETUP__SHIFT                                                             0x0
+#define CKSVII2C1_IC_SDA_SETUP__SDA1_SETUP_MASK                                                               0x000000FFL
+//CKSVII2C1_IC_ACK_GENERAL_CALL
+#define CKSVII2C1_IC_ACK_GENERAL_CALL__ACK1_GENERAL_CALL__SHIFT                                               0x0
+#define CKSVII2C1_IC_ACK_GENERAL_CALL__ACK1_GENERAL_CALL_MASK                                                 0x00000001L
+//CKSVII2C1_IC_ENABLE_STATUS
+#define CKSVII2C1_IC_ENABLE_STATUS__IC1_EN__SHIFT                                                             0x0
+#define CKSVII2C1_IC_ENABLE_STATUS__SLV1_DISABLED_WHILE_BUSY__SHIFT                                           0x1
+#define CKSVII2C1_IC_ENABLE_STATUS__SLV1_RX_DATA_LOST__SHIFT                                                  0x2
+#define CKSVII2C1_IC_ENABLE_STATUS__IC1_EN_MASK                                                               0x00000001L
+//CKSVII2C1_IC_FS_SPKLEN
+#define CKSVII2C1_IC_FS_SPKLEN__FS1_SPKLEN__SHIFT                                                             0x0
+//CKSVII2C1_IC_HS_SPKLEN
+#define CKSVII2C1_IC_HS_SPKLEN__HS1_SPKLEN__SHIFT                                                             0x0
+//CKSVII2C1_IC_CLR_RESTART_DET
+//CKSVII2C1_IC_COMP_PARAM_1
+#define CKSVII2C1_IC_COMP_PARAM_1__APB1_DATA_WIDTH__SHIFT                                                     0x0
+#define CKSVII2C1_IC_COMP_PARAM_1__MAX1_SPEED_MODE__SHIFT                                                     0x2
+#define CKSVII2C1_IC_COMP_PARAM_1__HC1_COUNT_VALUES__SHIFT                                                    0x4
+#define CKSVII2C1_IC_COMP_PARAM_1__INTR1_IO__SHIFT                                                            0x5
+#define CKSVII2C1_IC_COMP_PARAM_1__HAS1_DMA__SHIFT                                                            0x6
+#define CKSVII2C1_IC_COMP_PARAM_1__ADD1_ENCODED_PARAMS__SHIFT                                                 0x7
+#define CKSVII2C1_IC_COMP_PARAM_1__RX1_BUFFER_DEPTH__SHIFT                                                    0x8
+#define CKSVII2C1_IC_COMP_PARAM_1__TX1_BUFFER_DEPTH__SHIFT                                                    0x10
+//CKSVII2C1_IC_COMP_VERSION
+#define CKSVII2C1_IC_COMP_VERSION__COMP1_VERSION__SHIFT                                                       0x0
+//CKSVII2C1_IC_COMP_TYPE
+#define CKSVII2C1_IC_COMP_TYPE__COMP1_TYPE__SHIFT                                                             0x0
+//SMUIO_PWRMGT
+#define SMUIO_PWRMGT__i2c_clk_gate_en__SHIFT                                                                  0x0
+#define SMUIO_PWRMGT__i2c1_clk_gate_en__SHIFT                                                                 0x8
+#define SMUIO_PWRMGT__i2c_reg_reset__SHIFT                                                                    0x10
+#define SMUIO_PWRMGT__i2c_dwc_reset_en__SHIFT                                                                 0x11
+#define SMUIO_PWRMGT__i2c1_reg_reset__SHIFT                                                                   0x18
+#define SMUIO_PWRMGT__i2c1_dwc_reset_en__SHIFT                                                                0x19
+#define SMUIO_PWRMGT__i2c_clk_gate_en_MASK                                                                    0x00000001L
+#define SMUIO_PWRMGT__i2c1_clk_gate_en_MASK                                                                   0x00000100L
+
+
+// addressBlock: smuio_smuio_rom_SmuSmuioDec
+//ROM_CNTL
+#define ROM_CNTL__CLOCK_GATING_EN__SHIFT                                                                      0x0
+#define ROM_CNTL__READ_MODE__SHIFT                                                                            0x1
+#define ROM_CNTL__READ_MODE_OVERRIDE__SHIFT                                                                   0x3
+#define ROM_CNTL__SPI_TIMING_RELAX_SCK__SHIFT                                                                 0x4
+#define ROM_CNTL__SPI_TIMING_RELAX_SCK_OVERRIDE__SHIFT                                                        0x5
+#define ROM_CNTL__FOUR_BYTE_ADDRESS_MODE__SHIFT                                                               0x6
+#define ROM_CNTL__DUMMY_CYCLE_NUM__SHIFT                                                                      0x8
+#define ROM_CNTL__SPI_TIMING_RELAX__SHIFT                                                                     0x13
+#define ROM_CNTL__SPI_TIMING_RELAX_OVERRIDE__SHIFT                                                            0x14
+#define ROM_CNTL__SPI_FAST_MODE__SHIFT                                                                        0x15
+#define ROM_CNTL__SPI_FAST_MODE_OVERRIDE__SHIFT                                                               0x16
+#define ROM_CNTL__SCK_PRESCALE_REFCLK__SHIFT                                                                  0x17
+#define ROM_CNTL__SCK_PRESCALE_REFCLK_OVERRIDE__SHIFT                                                         0x1c
+#define ROM_CNTL__ROM_INDEX_ADDRESS_AUTO_INCREASE__SHIFT                                                      0x1d
+#define ROM_CNTL__PAD_SAMPLE_MODE__SHIFT                                                                      0x1e
+#define ROM_CNTL__PAD_SAMPLE_MODE_OVERRIDE__SHIFT                                                             0x1f
+#define ROM_CNTL__CLOCK_GATING_EN_MASK                                                                        0x00000001L
+#define ROM_CNTL__SPI_TIMING_RELAX_MASK                                                                       0x00080000L
+#define ROM_CNTL__SPI_TIMING_RELAX_OVERRIDE_MASK                                                              0x00100000L
+#define ROM_CNTL__SPI_FAST_MODE_MASK                                                                          0x00200000L
+#define ROM_CNTL__SPI_FAST_MODE_OVERRIDE_MASK                                                                 0x00400000L
+#define ROM_CNTL__SCK_PRESCALE_REFCLK_MASK                                                                    0x0F800000L
+#define ROM_CNTL__SCK_PRESCALE_REFCLK_OVERRIDE_MASK                                                           0x10000000L
+//PAGE_MIRROR_CNTL
+#define PAGE_MIRROR_CNTL__PAGE_MIRROR_BASE_ADDR__SHIFT                                                        0x0
+#define PAGE_MIRROR_CNTL__PAGE_MIRROR_ENABLE__SHIFT                                                           0x19
+#define PAGE_MIRROR_CNTL__PAGE_MIRROR_USAGE__SHIFT                                                            0x1a
+#define PAGE_MIRROR_CNTL__PAGE_MIRROR_INVALIDATE__SHIFT                                                       0x1c
+#define PAGE_MIRROR_CNTL__PAGE_MIRROR_BASE_ADDR_MASK                                                          0x01FFFFFFL
+#define PAGE_MIRROR_CNTL__PAGE_MIRROR_ENABLE_MASK                                                             0x02000000L
+#define PAGE_MIRROR_CNTL__PAGE_MIRROR_USAGE_MASK                                                              0x0C000000L
+#define PAGE_MIRROR_CNTL__PAGE_MIRROR_INVALIDATE_MASK                                                         0x10000000L
+//ROM_STATUS
+#define ROM_STATUS__ROM_BUSY__SHIFT                                                                           0x0
+#define ROM_STATUS__ROM_BUSY_MASK                                                                             0x00000001L
+//CGTT_ROM_CLK_CTRL0
+#define CGTT_ROM_CLK_CTRL0__ON_DELAY__SHIFT                                                                   0x0
+#define CGTT_ROM_CLK_CTRL0__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT                                                             0x1e
+#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT                                                             0x1f
+#define CGTT_ROM_CLK_CTRL0__ON_DELAY_MASK                                                                     0x0000000FL
+#define CGTT_ROM_CLK_CTRL0__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1_MASK                                                               0x40000000L
+#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK                                                               0x80000000L
+//ROM_INDEX
+#define ROM_INDEX__ROM_INDEX__SHIFT                                                                           0x0
+#define ROM_INDEX__ROM_INDEX_MASK                                                                             0x01FFFFFFL
+//ROM_DATA
+#define ROM_DATA__ROM_DATA__SHIFT                                                                             0x0
+#define ROM_DATA__ROM_DATA_MASK                                                                               0xFFFFFFFFL
+//ROM_START
+#define ROM_START__ROM_START__SHIFT                                                                           0x0
+#define ROM_START__ROM_START_MASK                                                                             0x01FFFFFFL
+//ROM_SW_CNTL
+#define ROM_SW_CNTL__DATA_SIZE__SHIFT                                                                         0x0
+#define ROM_SW_CNTL__COMMAND_SIZE__SHIFT                                                                      0x10
+#define ROM_SW_CNTL__ROM_SW_RETURN_DATA_ENABLE__SHIFT                                                         0x13
+#define ROM_SW_CNTL__DATA_SIZE_MASK                                                                           0x0000FFFFL
+#define ROM_SW_CNTL__COMMAND_SIZE_MASK                                                                        0x00070000L
+#define ROM_SW_CNTL__ROM_SW_RETURN_DATA_ENABLE_MASK                                                           0x00080000L
+//ROM_SW_STATUS
+#define ROM_SW_STATUS__ROM_SW_DONE__SHIFT                                                                     0x0
+#define ROM_SW_STATUS__ROM_SW_DONE_MASK                                                                       0x00000001L
+//ROM_SW_COMMAND
+#define ROM_SW_COMMAND__ROM_SW_INSTRUCTION__SHIFT                                                             0x0
+#define ROM_SW_COMMAND__ROM_SW_ADDRESS__SHIFT                                                                 0x8
+#define ROM_SW_COMMAND__ROM_SW_INSTRUCTION_MASK                                                               0x000000FFL
+#define ROM_SW_COMMAND__ROM_SW_ADDRESS_MASK                                                                   0xFFFFFF00L
+//ROM_SW_DATA_1
+#define ROM_SW_DATA_1__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_1__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_2
+#define ROM_SW_DATA_2__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_2__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_3
+#define ROM_SW_DATA_3__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_3__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_4
+#define ROM_SW_DATA_4__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_4__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_5
+#define ROM_SW_DATA_5__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_5__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_6
+#define ROM_SW_DATA_6__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_6__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_7
+#define ROM_SW_DATA_7__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_7__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_8
+#define ROM_SW_DATA_8__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_8__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_9
+#define ROM_SW_DATA_9__ROM_SW_DATA__SHIFT                                                                     0x0
+#define ROM_SW_DATA_9__ROM_SW_DATA_MASK                                                                       0xFFFFFFFFL
+//ROM_SW_DATA_10
+#define ROM_SW_DATA_10__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_10__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_11
+#define ROM_SW_DATA_11__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_11__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_12
+#define ROM_SW_DATA_12__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_12__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_13
+#define ROM_SW_DATA_13__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_13__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_14
+#define ROM_SW_DATA_14__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_14__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_15
+#define ROM_SW_DATA_15__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_15__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_16
+#define ROM_SW_DATA_16__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_16__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_17
+#define ROM_SW_DATA_17__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_17__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_18
+#define ROM_SW_DATA_18__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_18__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_19
+#define ROM_SW_DATA_19__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_19__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_20
+#define ROM_SW_DATA_20__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_20__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_21
+#define ROM_SW_DATA_21__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_21__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_22
+#define ROM_SW_DATA_22__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_22__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_23
+#define ROM_SW_DATA_23__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_23__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_24
+#define ROM_SW_DATA_24__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_24__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_25
+#define ROM_SW_DATA_25__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_25__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_26
+#define ROM_SW_DATA_26__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_26__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_27
+#define ROM_SW_DATA_27__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_27__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_28
+#define ROM_SW_DATA_28__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_28__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_29
+#define ROM_SW_DATA_29__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_29__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_30
+#define ROM_SW_DATA_30__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_30__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_31
+#define ROM_SW_DATA_31__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_31__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_32
+#define ROM_SW_DATA_32__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_32__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_33
+#define ROM_SW_DATA_33__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_33__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_34
+#define ROM_SW_DATA_34__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_34__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_35
+#define ROM_SW_DATA_35__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_35__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_36
+#define ROM_SW_DATA_36__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_36__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_37
+#define ROM_SW_DATA_37__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_37__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_38
+#define ROM_SW_DATA_38__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_38__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_39
+#define ROM_SW_DATA_39__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_39__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_40
+#define ROM_SW_DATA_40__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_40__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_41
+#define ROM_SW_DATA_41__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_41__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_42
+#define ROM_SW_DATA_42__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_42__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_43
+#define ROM_SW_DATA_43__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_43__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_44
+#define ROM_SW_DATA_44__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_44__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_45
+#define ROM_SW_DATA_45__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_45__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_46
+#define ROM_SW_DATA_46__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_46__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_47
+#define ROM_SW_DATA_47__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_47__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_48
+#define ROM_SW_DATA_48__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_48__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_49
+#define ROM_SW_DATA_49__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_49__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_50
+#define ROM_SW_DATA_50__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_50__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_51
+#define ROM_SW_DATA_51__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_51__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_52
+#define ROM_SW_DATA_52__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_52__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_53
+#define ROM_SW_DATA_53__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_53__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_54
+#define ROM_SW_DATA_54__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_54__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_55
+#define ROM_SW_DATA_55__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_55__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_56
+#define ROM_SW_DATA_56__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_56__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_57
+#define ROM_SW_DATA_57__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_57__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_58
+#define ROM_SW_DATA_58__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_58__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_59
+#define ROM_SW_DATA_59__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_59__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_60
+#define ROM_SW_DATA_60__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_60__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_61
+#define ROM_SW_DATA_61__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_61__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_62
+#define ROM_SW_DATA_62__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_62__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_63
+#define ROM_SW_DATA_63__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_63__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+//ROM_SW_DATA_64
+#define ROM_SW_DATA_64__ROM_SW_DATA__SHIFT                                                                    0x0
+#define ROM_SW_DATA_64__ROM_SW_DATA_MASK                                                                      0xFFFFFFFFL
+
+
+// addressBlock: smuio_smuio_gpio_SmuSmuioDec
+//SMU_GPIOPAD_SW_INT_STAT
+#define SMU_GPIOPAD_SW_INT_STAT__SW_INT_STAT__SHIFT                                                           0x0
+#define SMU_GPIOPAD_SW_INT_STAT__SW_INT_STAT_MASK                                                             0x00000001L
+//SMU_GPIOPAD_MASK
+#define SMU_GPIOPAD_MASK__GPIO_MASK__SHIFT                                                                    0x0
+#define SMU_GPIOPAD_MASK__GPIO_MASK_MASK                                                                      0x7FFFFFFFL
+//SMU_GPIOPAD_A
+#define SMU_GPIOPAD_A__GPIO_A__SHIFT                                                                          0x0
+#define SMU_GPIOPAD_A__GPIO_A_MASK                                                                            0x7FFFFFFFL
+//SMU_GPIOPAD_TXIMPSEL
+#define SMU_GPIOPAD_TXIMPSEL__GPIO_TXIMPSEL__SHIFT                                                            0x0
+#define SMU_GPIOPAD_TXIMPSEL__GPIO_TXIMPSEL_MASK                                                              0x7FFFFFFFL
+//SMU_GPIOPAD_EN
+#define SMU_GPIOPAD_EN__GPIO_EN__SHIFT                                                                        0x0
+#define SMU_GPIOPAD_EN__GPIO_EN_MASK                                                                          0x7FFFFFFFL
+//SMU_GPIOPAD_Y
+#define SMU_GPIOPAD_Y__GPIO_Y__SHIFT                                                                          0x0
+#define SMU_GPIOPAD_Y__GPIO_Y_MASK                                                                            0x7FFFFFFFL
+//SMU_GPIOPAD_RXEN
+#define SMU_GPIOPAD_RXEN__GPIO_RXEN__SHIFT                                                                    0x0
+#define SMU_GPIOPAD_RXEN__GPIO_RXEN_MASK                                                                      0x7FFFFFFFL
+//SMU_GPIOPAD_RCVR_SEL0
+#define SMU_GPIOPAD_RCVR_SEL0__GPIO_RCVR_SEL0__SHIFT                                                          0x0
+#define SMU_GPIOPAD_RCVR_SEL0__GPIO_RCVR_SEL0_MASK                                                            0x7FFFFFFFL
+//SMU_GPIOPAD_RCVR_SEL1
+#define SMU_GPIOPAD_RCVR_SEL1__GPIO_RCVR_SEL1__SHIFT                                                          0x0
+#define SMU_GPIOPAD_RCVR_SEL1__GPIO_RCVR_SEL1_MASK                                                            0x7FFFFFFFL
+//SMU_GPIOPAD_PU_EN
+#define SMU_GPIOPAD_PU_EN__GPIO_PU_EN__SHIFT                                                                  0x0
+#define SMU_GPIOPAD_PU_EN__GPIO_PU_EN_MASK                                                                    0x7FFFFFFFL
+//SMU_GPIOPAD_PD_EN
+#define SMU_GPIOPAD_PD_EN__GPIO_PD_EN__SHIFT                                                                  0x0
+#define SMU_GPIOPAD_PD_EN__GPIO_PD_EN_MASK                                                                    0x7FFFFFFFL
+//SMU_GPIOPAD_PINSTRAPS
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_0__SHIFT                                                         0x0
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_1__SHIFT                                                         0x1
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_2__SHIFT                                                         0x2
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_3__SHIFT                                                         0x3
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_4__SHIFT                                                         0x4
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_5__SHIFT                                                         0x5
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_6__SHIFT                                                         0x6
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_7__SHIFT                                                         0x7
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_8__SHIFT                                                         0x8
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_9__SHIFT                                                         0x9
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_10__SHIFT                                                        0xa
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_11__SHIFT                                                        0xb
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_12__SHIFT                                                        0xc
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_13__SHIFT                                                        0xd
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_14__SHIFT                                                        0xe
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_15__SHIFT                                                        0xf
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_16__SHIFT                                                        0x10
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_17__SHIFT                                                        0x11
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_18__SHIFT                                                        0x12
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_19__SHIFT                                                        0x13
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_20__SHIFT                                                        0x14
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_21__SHIFT                                                        0x15
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_22__SHIFT                                                        0x16
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_23__SHIFT                                                        0x17
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_24__SHIFT                                                        0x18
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_25__SHIFT                                                        0x19
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_26__SHIFT                                                        0x1a
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_27__SHIFT                                                        0x1b
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_28__SHIFT                                                        0x1c
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_29__SHIFT                                                        0x1d
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_30__SHIFT                                                        0x1e
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_0_MASK                                                           0x00000001L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_1_MASK                                                           0x00000002L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_2_MASK                                                           0x00000004L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_3_MASK                                                           0x00000008L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_4_MASK                                                           0x00000010L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_5_MASK                                                           0x00000020L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_6_MASK                                                           0x00000040L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_7_MASK                                                           0x00000080L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_8_MASK                                                           0x00000100L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_9_MASK                                                           0x00000200L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_10_MASK                                                          0x00000400L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_11_MASK                                                          0x00000800L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_12_MASK                                                          0x00001000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_13_MASK                                                          0x00002000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_14_MASK                                                          0x00004000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_15_MASK                                                          0x00008000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_16_MASK                                                          0x00010000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_17_MASK                                                          0x00020000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_18_MASK                                                          0x00040000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_19_MASK                                                          0x00080000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_20_MASK                                                          0x00100000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_21_MASK                                                          0x00200000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_22_MASK                                                          0x00400000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_23_MASK                                                          0x00800000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_24_MASK                                                          0x01000000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_25_MASK                                                          0x02000000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_26_MASK                                                          0x04000000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_27_MASK                                                          0x08000000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_28_MASK                                                          0x10000000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_29_MASK                                                          0x20000000L
+#define SMU_GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_30_MASK                                                          0x40000000L
+//DFT_PINSTRAPS
+#define DFT_PINSTRAPS__DFT_PINSTRAPS__SHIFT                                                                   0x0
+#define DFT_PINSTRAPS__DFT_PINSTRAPS_MASK                                                                     0x000003FFL
+//SMU_GPIOPAD_INT_STAT_EN
+#define SMU_GPIOPAD_INT_STAT_EN__GPIO_INT_STAT_EN__SHIFT                                                      0x0
+#define SMU_GPIOPAD_INT_STAT_EN__SW_INITIATED_INT_STAT_EN__SHIFT                                              0x1f
+#define SMU_GPIOPAD_INT_STAT_EN__GPIO_INT_STAT_EN_MASK                                                        0x1FFFFFFFL
+#define SMU_GPIOPAD_INT_STAT_EN__SW_INITIATED_INT_STAT_EN_MASK                                                0x80000000L
+//SMU_GPIOPAD_INT_STAT
+#define SMU_GPIOPAD_INT_STAT__GPIO_INT_STAT__SHIFT                                                            0x0
+#define SMU_GPIOPAD_INT_STAT__SW_INITIATED_INT_STAT__SHIFT                                                    0x1f
+#define SMU_GPIOPAD_INT_STAT__GPIO_INT_STAT_MASK                                                              0x1FFFFFFFL
+#define SMU_GPIOPAD_INT_STAT__SW_INITIATED_INT_STAT_MASK                                                      0x80000000L
+//SMU_GPIOPAD_INT_STAT_AK
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_0__SHIFT                                                    0x0
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_1__SHIFT                                                    0x1
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_2__SHIFT                                                    0x2
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_3__SHIFT                                                    0x3
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_4__SHIFT                                                    0x4
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_5__SHIFT                                                    0x5
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_6__SHIFT                                                    0x6
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_7__SHIFT                                                    0x7
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_8__SHIFT                                                    0x8
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_9__SHIFT                                                    0x9
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_10__SHIFT                                                   0xa
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_11__SHIFT                                                   0xb
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_12__SHIFT                                                   0xc
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_13__SHIFT                                                   0xd
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_14__SHIFT                                                   0xe
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_15__SHIFT                                                   0xf
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_16__SHIFT                                                   0x10
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_17__SHIFT                                                   0x11
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_18__SHIFT                                                   0x12
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_19__SHIFT                                                   0x13
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_20__SHIFT                                                   0x14
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_21__SHIFT                                                   0x15
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_22__SHIFT                                                   0x16
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_23__SHIFT                                                   0x17
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_24__SHIFT                                                   0x18
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_25__SHIFT                                                   0x19
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_26__SHIFT                                                   0x1a
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_27__SHIFT                                                   0x1b
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_28__SHIFT                                                   0x1c
+#define SMU_GPIOPAD_INT_STAT_AK__SW_INITIATED_INT_STAT_AK__SHIFT                                              0x1f
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_0_MASK                                                      0x00000001L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_1_MASK                                                      0x00000002L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_2_MASK                                                      0x00000004L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_3_MASK                                                      0x00000008L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_4_MASK                                                      0x00000010L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_5_MASK                                                      0x00000020L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_6_MASK                                                      0x00000040L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_7_MASK                                                      0x00000080L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_8_MASK                                                      0x00000100L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_9_MASK                                                      0x00000200L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_10_MASK                                                     0x00000400L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_11_MASK                                                     0x00000800L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_12_MASK                                                     0x00001000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_13_MASK                                                     0x00002000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_14_MASK                                                     0x00004000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_15_MASK                                                     0x00008000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_16_MASK                                                     0x00010000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_17_MASK                                                     0x00020000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_18_MASK                                                     0x00040000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_19_MASK                                                     0x00080000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_20_MASK                                                     0x00100000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_21_MASK                                                     0x00200000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_22_MASK                                                     0x00400000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_23_MASK                                                     0x00800000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_24_MASK                                                     0x01000000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_25_MASK                                                     0x02000000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_26_MASK                                                     0x04000000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_27_MASK                                                     0x08000000L
+#define SMU_GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_28_MASK                                                     0x10000000L
+#define SMU_GPIOPAD_INT_STAT_AK__SW_INITIATED_INT_STAT_AK_MASK                                                0x80000000L
+//SMU_GPIOPAD_INT_EN
+#define SMU_GPIOPAD_INT_EN__GPIO_INT_EN__SHIFT                                                                0x0
+#define SMU_GPIOPAD_INT_EN__SW_INITIATED_INT_EN__SHIFT                                                        0x1f
+#define SMU_GPIOPAD_INT_EN__GPIO_INT_EN_MASK                                                                  0x1FFFFFFFL
+#define SMU_GPIOPAD_INT_EN__SW_INITIATED_INT_EN_MASK                                                          0x80000000L
+//SMU_GPIOPAD_INT_TYPE
+#define SMU_GPIOPAD_INT_TYPE__GPIO_INT_TYPE__SHIFT                                                            0x0
+#define SMU_GPIOPAD_INT_TYPE__SW_INITIATED_INT_TYPE__SHIFT                                                    0x1f
+#define SMU_GPIOPAD_INT_TYPE__GPIO_INT_TYPE_MASK                                                              0x1FFFFFFFL
+#define SMU_GPIOPAD_INT_TYPE__SW_INITIATED_INT_TYPE_MASK                                                      0x80000000L
+//SMU_GPIOPAD_INT_POLARITY
+#define SMU_GPIOPAD_INT_POLARITY__GPIO_INT_POLARITY__SHIFT                                                    0x0
+#define SMU_GPIOPAD_INT_POLARITY__SW_INITIATED_INT_POLARITY__SHIFT                                            0x1f
+#define SMU_GPIOPAD_INT_POLARITY__GPIO_INT_POLARITY_MASK                                                      0x1FFFFFFFL
+#define SMU_GPIOPAD_INT_POLARITY__SW_INITIATED_INT_POLARITY_MASK                                              0x80000000L
+//SMUIO_PCC_GPIO_SELECT
+#define SMUIO_PCC_GPIO_SELECT__GPIO__SHIFT                                                                    0x0
+#define SMUIO_PCC_GPIO_SELECT__GPIO_MASK                                                                      0xFFFFFFFFL
+//SMU_GPIOPAD_S0
+#define SMU_GPIOPAD_S0__GPIO_S0__SHIFT                                                                        0x0
+#define SMU_GPIOPAD_S0__GPIO_S0_MASK                                                                          0x7FFFFFFFL
+//SMU_GPIOPAD_S1
+#define SMU_GPIOPAD_S1__GPIO_S1__SHIFT                                                                        0x0
+#define SMU_GPIOPAD_S1__GPIO_S1_MASK                                                                          0x7FFFFFFFL
+//SMU_GPIOPAD_SCHMEN
+#define SMU_GPIOPAD_SCHMEN__GPIO_SCHMEN__SHIFT                                                                0x0
+#define SMU_GPIOPAD_SCHMEN__GPIO_SCHMEN_MASK                                                                  0x7FFFFFFFL
+//SMU_GPIOPAD_SCL_EN
+#define SMU_GPIOPAD_SCL_EN__GPIO_SCL_EN__SHIFT                                                                0x0
+#define SMU_GPIOPAD_SCL_EN__GPIO_SCL_EN_MASK                                                                  0x7FFFFFFFL
+//SMU_GPIOPAD_SDA_EN
+#define SMU_GPIOPAD_SDA_EN__GPIO_SDA_EN__SHIFT                                                                0x0
+#define SMU_GPIOPAD_SDA_EN__GPIO_SDA_EN_MASK                                                                  0x7FFFFFFFL
+//SMUIO_GPIO_INT0_SELECT
+#define SMUIO_GPIO_INT0_SELECT__GPIO_INT0_SELECT__SHIFT                                                       0x0
+#define SMUIO_GPIO_INT0_SELECT__GPIO_INT0_SELECT_MASK                                                         0xFFFFFFFFL
+//SMUIO_GPIO_INT1_SELECT
+#define SMUIO_GPIO_INT1_SELECT__GPIO_INT1_SELECT__SHIFT                                                       0x0
+#define SMUIO_GPIO_INT1_SELECT__GPIO_INT1_SELECT_MASK                                                         0xFFFFFFFFL
+//SMUIO_GPIO_INT2_SELECT
+#define SMUIO_GPIO_INT2_SELECT__GPIO_INT2_SELECT__SHIFT                                                       0x0
+#define SMUIO_GPIO_INT2_SELECT__GPIO_INT2_SELECT_MASK                                                         0xFFFFFFFFL
+//SMUIO_GPIO_INT3_SELECT
+#define SMUIO_GPIO_INT3_SELECT__GPIO_INT3_SELECT__SHIFT                                                       0x0
+#define SMUIO_GPIO_INT3_SELECT__GPIO_INT3_SELECT_MASK                                                         0xFFFFFFFFL
+//SMU_GPIOPAD_MP_INT0_STAT
+#define SMU_GPIOPAD_MP_INT0_STAT__GPIO_MP_INT0_STAT__SHIFT                                                    0x0
+#define SMU_GPIOPAD_MP_INT0_STAT__GPIO_MP_INT0_STAT_MASK                                                      0x1FFFFFFFL
+//SMU_GPIOPAD_MP_INT1_STAT
+#define SMU_GPIOPAD_MP_INT1_STAT__GPIO_MP_INT1_STAT__SHIFT                                                    0x0
+#define SMU_GPIOPAD_MP_INT1_STAT__GPIO_MP_INT1_STAT_MASK                                                      0x1FFFFFFFL
+//SMU_GPIOPAD_MP_INT2_STAT
+#define SMU_GPIOPAD_MP_INT2_STAT__GPIO_MP_INT2_STAT__SHIFT                                                    0x0
+#define SMU_GPIOPAD_MP_INT2_STAT__GPIO_MP_INT2_STAT_MASK                                                      0x1FFFFFFFL
+//SMU_GPIOPAD_MP_INT3_STAT
+#define SMU_GPIOPAD_MP_INT3_STAT__GPIO_MP_INT3_STAT__SHIFT                                                    0x0
+#define SMU_GPIOPAD_MP_INT3_STAT__GPIO_MP_INT3_STAT_MASK                                                      0x1FFFFFFFL
+//SMIO_INDEX
+#define SMIO_INDEX__SW_SMIO_INDEX__SHIFT                                                                      0x0
+#define SMIO_INDEX__SW_SMIO_INDEX_MASK                                                                        0x00000001L
+//S0_VID_SMIO_CNTL
+#define S0_VID_SMIO_CNTL__S0_SMIO_VALUES__SHIFT                                                               0x0
+#define S0_VID_SMIO_CNTL__S0_SMIO_VALUES_MASK                                                                 0xFFFFFFFFL
+//S1_VID_SMIO_CNTL
+#define S1_VID_SMIO_CNTL__S1_SMIO_VALUES__SHIFT                                                               0x0
+#define S1_VID_SMIO_CNTL__S1_SMIO_VALUES_MASK                                                                 0xFFFFFFFFL
+//OPEN_DRAIN_SELECT
+#define OPEN_DRAIN_SELECT__OPEN_DRAIN_SELECT__SHIFT                                                           0x0
+#define OPEN_DRAIN_SELECT__RESERVED__SHIFT                                                                    0x1f
+#define OPEN_DRAIN_SELECT__OPEN_DRAIN_SELECT_MASK                                                             0x7FFFFFFFL
+#define OPEN_DRAIN_SELECT__RESERVED_MASK                                                                      0x80000000L
+//SMIO_ENABLE
+#define SMIO_ENABLE__SMIO_ENABLE__SHIFT                                                                       0x0
+#define SMIO_ENABLE__SMIO_ENABLE_MASK                                                                         0xFFFFFFFFL
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/ivsrcid/gfx/irqsrcs_gfx_12_1_0.h b/drivers/gpu/drm/amd/include/ivsrcid/gfx/irqsrcs_gfx_12_1_0.h
new file mode 100644
index 000000000000..9fe5466e9418
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/ivsrcid/gfx/irqsrcs_gfx_12_1_0.h
@@ -0,0 +1,136 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __IRQSRCS_GFX_12_1_0_H__
+#define __IRQSRCS_GFX_12_1_0_H__
+
+/* 0x0 UTCL2 has encountered a fault scenario */
+#define GFX_12_1_0__SRCID__UTCL2_FAULT				0
+/* 0x1 UTCL2 has encountered a retry scenario */
+#define GFX_12_1_0__SRCID__UTCL2_RETRY				1
+/* 0x2 UTCL2 for data poisoning */
+#define GFX_12_1_0__SRCID__UTCL2_DATA_POISONING			2
+/* 0x30 SDMA atomic*_rtn ops complete */
+#define GFX_12_1_0__SRCID__SDMA_ATOMIC_RTN_DONE			48
+/* 0x31 Trap */
+#define GFX_12_1_0__SRCID__SDMA_TRAP				49
+/* 0x32 SRBM write Protection */
+#define GFX_12_1_0__SRCID__SDMA_SRBMWRITE			50
+/* 0x33 Context Empty */
+#define GFX_12_1_0__SRCID__SDMA_CTXEMPTY			51
+/* 0x34 SDMA New Run List */
+#define GFX_12_1_0__SRCID__SDMA_PREEMPT				52
+/* 0x35 sdma mid - command buffer preempt interrupt */
+#define GFX_12_1_0__SRCID__SDMA_IB_PREEMPT			53
+/* 0x36 Doorbell BE invalid */
+#define GFX_12_1_0__SRCID__SDMA_DOORBELL_INVALID		54
+/* 0x37 Queue hang or Command timeout */
+#define GFX_12_1_0__SRCID__SDMA_QUEUE_HANG			55
+/* 0x38 SDMA atomic CMPSWAP loop timeout */
+#define GFX_12_1_0__SRCID__SDMA_ATOMIC_TIMEOUT			56
+/* 0x39 SRBM read poll timeout */
+#define GFX_12_1_0__SRCID__SDMA_POLL_TIMEOUT			57
+/* 0x3A Page retry  timeout after UTCL2 return nack = 1 */
+#define GFX_12_1_0__SRCID__SDMA_PAGE_TIMEOUT			58
+/* 0x3B Page Null from UTCL2 when nack = 2 */
+#define GFX_12_1_0__SRCID__SDMA_PAGE_NULL			59
+/* 0x3C Page Fault Error from UTCL2 when nack = 3 */
+#define GFX_12_1_0__SRCID__SDMA_PAGE_FAULT			60
+/* 0x3D MC or SEM address in VM hole */
+#define GFX_12_1_0__SRCID__SDMA_INVALID_ADDR 			61
+/* 0x3E ECC Error */
+#define GFX_12_1_0__SRCID__SDMA_ECC				62
+/* 0x3F SDMA Frozen */
+#define GFX_12_1_0__SRCID__SDMA_FROZEN				63
+/* 0x40 SRAM ECC Error */
+#define GFX_12_1_0__SRCID__SDMA_SRAM_ECC			64
+/* 0x41 GPF(Sem incomplete timeout) */
+#define GFX_12_1_0__SRCID__SDMA_SEM_INCOMPLETE_TIMEOUT		65
+/* 0x42 Semaphore wait fail timeout */
+#define GFX_12_1_0__SRCID__SDMA_SEM_WAIT_FAIL_TIMEOUT		66
+/* 0x43 Wptr less than Rptr in active queue */
+#define GFX_12_1_0__SRCID__SDMA_INVALID_RB_PTR 			67
+/* 0x44 BE command exception */
+#define GFX_12_1_0__SRCID__SDMA_BE_EXCEPTION 			68
+/* 0x46 User fence. inherit from gfx v12_0 for gfx user queue */
+#define GFX_12_1_0__SRCID__SDMA_FENCE				70
+/* 0xB0 CP_INTERRUPT pkt in RB */
+#define GFX_12_1_0__SRCID__CP_RB_INT_PKT 			176
+/* 0xB1 CP_INTERRUPT pkt in IB1 */
+#define GFX_12_1_0__SRCID__CP_IB1_INT_PKT			177
+/* 0xB2 CP_INTERRUPT pkt in IB2 */
+#define GFX_12_1_0__SRCID__CP_IB2_INT_PKT			178
+/* 0xB3 DMA Watch Interrupt */
+#define GFX_12_1_0__SRCID__CP_DMA_WATCH_INTERRUPT 		179
+/* 0xB4 PM4 Pkt Rsvd Bits Error */
+#define GFX_12_1_0__SRCID__CP_PM4_PKT_RSVD_BIT_ERROR		180
+/* 0xB5 End-of-Pipe Interrupt */
+#define GFX_12_1_0__SRCID__CP_EOP_INTERRUPT			181
+/* 0xB7 Bad Opcode Error */
+#define GFX_12_1_0__SRCID__CP_BAD_OPCODE_ERROR			183
+/* 0xB8 Privileged Register Fault */
+#define GFX_12_1_0__SRCID__CP_PRIV_REG_FAULT			184
+/* 0xB9 Privileged Instr Fault */
+#define GFX_12_1_0__SRCID__CP_PRIV_INSTR_FAULT			185
+/* 0xBA Wait Memory Semaphore Fault (Sync Object Fault) */
+#define GFX_12_1_0__SRCID__CP_WAIT_MEM_SEM_FAULT		186
+/* 0xBB Context Empty Interrupt */
+#define GFX_12_1_0__SRCID__CP_CTX_EMPTY_INTERRUPT		187
+/* 0xBC Context Busy Interrupt */
+#define GFX_12_1_0__SRCID__CP_CTX_BUSY_INTERRUPT		188
+/* 0xC0 CP.ME Wait_Reg_Mem Poll Timeout */
+#define GFX_12_1_0__SRCID__CP_ME_WAIT_REG_MEM_POLL_TIMEOUT	192
+/* 0xC1 Surface Probe Fault Signal Incomplete */
+#define GFX_12_1_0__SRCID__CP_SIG_INCOMPLETE			193
+/* 0xC2 Preemption Ack-wledge */
+#define GFX_12_1_0__SRCID__CP_PREEMPT_ACK			194
+/* 0xC3 General Protection Fault (GPF) */
+#define GFX_12_1_0__SRCID__CP_GPF				195
+/* 0xC4 GDS Alloc Error */
+#define GFX_12_1_0__SRCID__CP_GDS_ALLOC_ERROR			196
+/* 0xC5 ECC Error */
+#define GFX_12_1_0__SRCID__CP_ECC_ERROR				197
+/* 0xC8 Unattached VM Doorbell Received */
+#define GFX_12_1_0__SRCID__CP_VM_DOORBELL			200
+/* 0xC9 ECC FUE Error */
+#define GFX_12_1_0__SRCID__CP_FUE_ERROR				201
+/* 0xCA Suspend Completion Interrupt */
+#define GFX_12_1_0__SRCID__CP_SUSPEAND_REQ_INTERRUPT 		202
+/* 0xCB Resume Completion Interrupt */
+#define GFX_12_1_0__SRCID__CP_RESUME_REQ_INTERRUPT 		203
+/* 0xCA RLC Streaming Perf Monitor Interrupt
+ * ContextID[15:0] each bit indicates poison is seen on respecive indexed VMID
+ * Ex: ContextID[3] == 1 means VMID-3 encountered poison consumption
+ * ContextID[16] == 1 indicates that complete VF need to reset with FLR */
+#define GFX_12_1_0__SRCID__RLC_STRM_PERF_MONITOR_INTERRUPT 	202
+/* 0xCB RLC Poison Interrupt */
+#define GFX_12_1_0__SRCID__RLC_POISON_INTERRUPT 		203
+/* 0xE7 High on ContextID[0] - nHT Error;  ContextID[1] - illegal Opcode Error */
+#define GFX_12_1_0__SRCID__PMR_EA_ERROR_INTERRUPT 		231
+/* 0xE8 CRead timeout error */
+#define GFX_12_1_0__SRCID__GRBM_RD_TIMEOUT_ERROR		232
+/* 0xE9 Register GUI Idle */
+#define GFX_12_1_0__SRCID__GRBM_REG_GUI_IDLE			233
+/* 0xEF SQ Interrupt (ttrace wrap, errors) */
+#define GFX_12_1_0__SRCID__SQ_INTERRUPT_ID			239
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/ivsrcid/vmc/irqsrcs_vmc_1_0.h b/drivers/gpu/drm/amd/include/ivsrcid/vmc/irqsrcs_vmc_1_0.h
index d130936c9989..22075805e95c 100644
--- a/drivers/gpu/drm/amd/include/ivsrcid/vmc/irqsrcs_vmc_1_0.h
+++ b/drivers/gpu/drm/amd/include/ivsrcid/vmc/irqsrcs_vmc_1_0.h
@@ -28,10 +28,12 @@
 
 
 #define VMC_1_0__SRCID__VM_FAULT                            0
+#define VMC_1_0__SRCID__VM_RETRY                            1
 #define VMC_1_0__SRCID__VM_CONTEXT0_ALL                     256
 #define VMC_1_0__SRCID__VM_CONTEXT1_ALL                     257
 
 #define UTCL2_1_0__SRCID__FAULT                             0       /* UTC L2 has encountered a fault or retry scenario */
+#define UTCL2_1_0__SRCID__RETRY                             1       /* UTC L2 has encountered a retry scenario (GFX12.1) */
 
 
 #endif /* __IRQSRCS_VMC_1_0_H__ */
diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
index 2366e68262e6..2a454f06c022 100644
--- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
@@ -136,7 +136,6 @@ enum amd_pp_sensors {
 	AMDGPU_PP_SENSOR_CPU_CLK,
 	AMDGPU_PP_SENSOR_VDDNB,
 	AMDGPU_PP_SENSOR_VDDGFX,
-	AMDGPU_PP_SENSOR_VDDBOARD,
 	AMDGPU_PP_SENSOR_UVD_VCLK,
 	AMDGPU_PP_SENSOR_UVD_DCLK,
 	AMDGPU_PP_SENSOR_VCE_ECCLK,
@@ -162,10 +161,13 @@ enum amd_pp_sensors {
 	AMDGPU_PP_SENSOR_PEAK_PSTATE_SCLK,
 	AMDGPU_PP_SENSOR_PEAK_PSTATE_MCLK,
 	AMDGPU_PP_SENSOR_VCN_LOAD,
+	AMDGPU_PP_SENSOR_VDDBOARD,
 	AMDGPU_PP_SENSOR_NODEPOWERLIMIT,
 	AMDGPU_PP_SENSOR_NODEPOWER,
 	AMDGPU_PP_SENSOR_GPPTRESIDENCY,
 	AMDGPU_PP_SENSOR_MAXNODEPOWERLIMIT,
+	AMDGPU_PP_SENSOR_UBB_POWER,
+	AMDGPU_PP_SENSOR_UBB_POWER_LIMIT,
 };
 
 enum amd_pp_task {
diff --git a/drivers/gpu/drm/amd/include/mes_v12_api_def.h b/drivers/gpu/drm/amd/include/mes_v12_api_def.h
index 2f12cba4eb66..83e4f0142324 100644
--- a/drivers/gpu/drm/amd/include/mes_v12_api_def.h
+++ b/drivers/gpu/drm/amd/include/mes_v12_api_def.h
@@ -71,6 +71,13 @@ enum MES_SCH_API_OPCODE {
 	MES_SCH_API_MAX = 0xFF
 };
 
+enum MES_RRMT_MODE {
+	MES_RRMT_MODE_LOCAL_XCD,
+	MES_RRMT_MODE_LOCAL_REMOTE_AID,
+	MES_RRMT_MODE_REMOTE_XCD,
+	MES_RRMT_MODE_REMOTE_MID
+};
+
 union MES_API_HEADER {
 	struct {
 		uint32_t type	  : 4; /* 0 - Invalid; 1 - Scheduling; 2 - TBD */
@@ -310,7 +317,8 @@ union MESAPI_SET_HW_RESOURCES_1 {
 		union {
 			struct {
 				uint32_t enable_mes_debug_ctx : 1;
-				uint32_t reserved : 31;
+				uint32_t mes_coop_mode : 1; /* 0: non-coop; 1: coop */
+				uint32_t reserved : 30;
 			};
 			uint32_t uint32_all;
 		};
@@ -318,7 +326,8 @@ union MESAPI_SET_HW_RESOURCES_1 {
 		uint32_t                            mes_debug_ctx_size;
 		/* unit is 100ms */
 		uint32_t                            mes_kiq_unmap_timeout;
-		uint64_t                            reserved1;
+		/* shared buffer of master/slaves, valid if mes_coop_mode=1 */
+		uint64_t                            coop_sch_shared_mc_addr;
 		uint64_t                            cleaner_shader_fence_mc_addr;
 	};
 
@@ -383,6 +392,7 @@ union MESAPI__ADD_QUEUE {
 		uint32_t		pipe_id;	//used for mapping legacy kernel queue
 		uint32_t		queue_id;
 		uint32_t		alignment_mode_setting;
+		uint32_t		full_sh_mem_config_data;
 	};
 
 	uint32_t max_dwords_in_api[API_FRAME_SIZE_IN_DWORDS];
@@ -553,7 +563,26 @@ union MESAPI__RESET {
 		/* valid only if reset_queue_only = true */
 		uint32_t			doorbell_offset;
 
-		/* valid only if hang_detect_then_reset = true */
+		/*
+		 * valid only if hang_detect_then_reset or hang_detect_only = true
+		 * doorbell_offset_addr will store the structure as follows
+		 * struct
+		 * {
+		 *	uint32_t db_offset[list_size];
+		 *	uint32_t hqd_id[list_size];
+		 * }
+		 * The hqd_id has following defines :
+		 * struct
+		 * {
+		 *	uint32 queue_type : 3;  Type of the queue
+		 *	uint32 pipe_index : 4;  pipe Index
+		 *	uint32 hqd_index  : 8;  This is queue_index within the pipe
+		 *	uint32 reserved   : 17;
+		 * };
+		 * The list_size is the total queue numbers that been managed by mes.
+		 * It can be calculated from all hqd_masks(including gfX, compute and sdma)
+		 * on set_hw_resource API
+	         */
 		uint64_t			doorbell_offset_addr;
 		enum MES_QUEUE_TYPE		queue_type;
 
@@ -672,6 +701,7 @@ union MESAPI__SET_DEBUG_VMID {
 		uint32_t		process_context_array_index;
 
 		uint32_t		alignment_mode_setting;
+		uint32_t		full_sh_mem_config_data;
 	};
 
 	uint32_t max_dwords_in_api[API_FRAME_SIZE_IN_DWORDS];
@@ -696,9 +726,26 @@ enum MESAPI_MISC_OPCODE {
 
 enum {MISC_DATA_MAX_SIZE_IN_DWORDS = 20};
 
+/*
+ * RRMT(Register Remapping Table), allow the firmware to modify the upper
+ * address to correctly steer the register transaction to expected DIE
+ */
+struct RRMT_OPTION {
+	union {
+		struct {
+			uint32_t mode : 4;
+			uint32_t mid_die_id : 4;
+			uint32_t xcd_die_id : 4;
+		};
+		uint32_t all;
+	};
+};
+
+
 struct WRITE_REG {
-	uint32_t	reg_offset;
-	uint32_t	reg_value;
+	uint32_t                  reg_offset;
+	uint32_t                  reg_value;
+	struct RRMT_OPTION        rrmt_opt;
 };
 
 struct READ_REG {
@@ -711,6 +758,7 @@ struct READ_REG {
 		} bits;
 		uint32_t all;
 	} option;
+	struct RRMT_OPTION        rrmt_opt;
 };
 
 struct INV_GART {
@@ -736,6 +784,8 @@ struct WAIT_REG_MEM {
 	uint32_t mask;
 	uint32_t reg_offset1;
 	uint32_t reg_offset2;
+	struct RRMT_OPTION rrmt_opt1; /* for reg1 */
+	struct RRMT_OPTION rrmt_opt2; /* for reg2 */
 };
 
 struct SET_SHADER_DEBUGGER {
@@ -744,7 +794,8 @@ struct SET_SHADER_DEBUGGER {
 		struct {
 			uint32_t single_memop : 1; // SQ_DEBUG.single_memop
 			uint32_t single_alu_op : 1; // SQ_DEBUG.single_alu_op
-			uint32_t reserved : 30;
+			uint32_t lds_oor_reporting : 1; /* SQ_DEBUG.ADDR_OUT_OF_RANGE_REPORTING */
+			uint32_t reserved : 29;
 		};
 		uint32_t u32all;
 	} flags;
diff --git a/drivers/gpu/drm/amd/include/soc15_hw_ip.h b/drivers/gpu/drm/amd/include/soc15_hw_ip.h
index dad83037793a..a20e59584dde 100644
--- a/drivers/gpu/drm/amd/include/soc15_hw_ip.h
+++ b/drivers/gpu/drm/amd/include/soc15_hw_ip.h
@@ -100,5 +100,7 @@
 #define XGMI_HWID                                        200
 #define XGBE_HWID                                        216
 #define MP0_HWID                                         255
+#define ATU_HWID                                         294
+#define AIGC_HWID                                        295
 
 #endif
diff --git a/drivers/gpu/drm/amd/include/soc15_ih_clientid.h b/drivers/gpu/drm/amd/include/soc15_ih_clientid.h
index e1948613cc1b..d249b8407259 100644
--- a/drivers/gpu/drm/amd/include/soc15_ih_clientid.h
+++ b/drivers/gpu/drm/amd/include/soc15_ih_clientid.h
@@ -103,6 +103,7 @@ enum soc21_ih_clientid {
 	SOC21_IH_CLIENTID_VPE		= 0x18,
 	SOC21_IH_CLIENTID_PWR		= 0x19,
 	SOC21_IH_CLIENTID_LSDMA		= 0x1a,
+	SOC21_IH_CLIENTID_UTCL2		= 0x1b,
 	SOC21_IH_CLIENTID_MP0		= 0x1e,
 	SOC21_IH_CLIENTID_MP1		= 0x1f,
 	SOC21_IH_CLIENTID_MAX,
diff --git a/drivers/gpu/drm/amd/include/soc_v1_0_enum.h b/drivers/gpu/drm/amd/include/soc_v1_0_enum.h
new file mode 100644
index 000000000000..2a910785334d
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/soc_v1_0_enum.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __SOC_V1_0_ENUM_H__
+#define __SOC_V1_0_ENUM_H__
+
+typedef enum MTYPE {
+	MTYPE_NC			= 0x00000000,
+	MTYPE_RESERVED_1		= 0x00000001,
+	MTYPE_RW			= 0x00000002,
+	MTYPE_UC			= 0x00000003,
+} MTYPE;
+
+typedef enum SH_MEM_ALIGNMENT_MODE {
+	SH_MEM_ALIGNMENT_MODE_DWORD              = 0x00000000,
+	SH_MEM_ALIGNMENT_MODE_UNALIGNED          = 0x00000001,
+} SH_MEM_ALIGNMENT_MODE;
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/soc_v1_0_ih_clientid.h b/drivers/gpu/drm/amd/include/soc_v1_0_ih_clientid.h
new file mode 100644
index 000000000000..11ab4b7d61e1
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/soc_v1_0_ih_clientid.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __SOC_V1_0_IH_CLIENTID_H__
+#define __SOC_V1_0_IH_CLIENTID_H__
+
+extern const char *soc_v1_0_ih_clientid_name[];
+
+enum soc_v1_0_ih_clientid {
+	SOC_V1_0_IH_CLIENTID_IH			= 0x00,
+	SOC_V1_0_IH_CLIENTID_ATHUB		= 0x02,
+	SOC_V1_0_IH_CLIENTID_BIF		= 0x03,
+	SOC_V1_0_IH_CLIENTID_RLC		= 0x07,
+	SOC_V1_0_IH_CLIENTID_GFX		= 0x0a,
+	SOC_V1_0_IH_CLIENTID_IMU		= 0x0b,
+	SOC_V1_0_IH_CLIENTID_VCN1		= 0x0e,
+	SOC_V1_0_IH_CLIENTID_THM		= 0x0f,
+	SOC_V1_0_IH_CLIENTID_VCN		= 0x10,
+	SOC_V1_0_IH_CLIENTID_VMC		= 0x12,
+	SOC_V1_0_IH_CLIENTID_GRBM_CP		= 0x14,
+	SOC_V1_0_IH_CLIENTID_GC_AID		= 0x15,
+	SOC_V1_0_IH_CLIENTID_ROM_SMUIO		= 0x16,
+	SOC_V1_0_IH_CLIENTID_DF			= 0x17,
+	SOC_V1_0_IH_CLIENTID_PWR		= 0x19,
+	SOC_V1_0_IH_CLIENTID_LSDMA		= 0x1a,
+	SOC_V1_0_IH_CLIENTID_GC_UTCL2		= 0x1b,
+	SOC_V1_0_IH_CLIENTID_nHT		= 0X1c,
+	SOC_V1_0_IH_CLIENTID_MP0		= 0x1e,
+	SOC_V1_0_IH_CLIENTID_MP1		= 0x1f,
+	SOC_V1_0_IH_CLIENTID_MAX,
+};
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/v12_structs.h b/drivers/gpu/drm/amd/include/v12_structs.h
index 03a35f8a65b0..28f4b0132681 100644
--- a/drivers/gpu/drm/amd/include/v12_structs.h
+++ b/drivers/gpu/drm/amd/include/v12_structs.h
@@ -1186,4 +1186,2058 @@ struct v12_compute_mqd {
     uint32_t gws_63_val; // offset: 511  (0x1FF)
 };
 
+struct v12_1_compute_mqd {
+    uint32_t header; // offset: 0  (0x0)
+    uint32_t compute_dispatch_initiator; // offset: 1  (0x1)
+    uint32_t compute_dim_x; // offset: 2  (0x2)
+    uint32_t compute_dim_y; // offset: 3  (0x3)
+    uint32_t compute_dim_z; // offset: 4  (0x4)
+    uint32_t compute_start_x; // offset: 5  (0x5)
+    uint32_t compute_start_y; // offset: 6  (0x6)
+    uint32_t compute_start_z; // offset: 7  (0x7)
+    uint32_t compute_num_thread_x; // offset: 8  (0x8)
+    uint32_t compute_num_thread_y; // offset: 9  (0x9)
+    uint32_t compute_num_thread_z; // offset: 10  (0xA)
+    uint32_t compute_pipelinestat_enable; // offset: 11  (0xB)
+    uint32_t compute_perfcount_enable; // offset: 12  (0xC)
+    uint32_t compute_pgm_lo; // offset: 13  (0xD)
+    uint32_t compute_pgm_hi; // offset: 14  (0xE)
+    uint32_t compute_dispatch_pkt_addr_lo; // offset: 15  (0xF)
+    uint32_t compute_dispatch_pkt_addr_hi; // offset: 16  (0x10)
+    uint32_t compute_dispatch_scratch_base_lo; // offset: 17  (0x11)
+    uint32_t compute_dispatch_scratch_base_hi; // offset: 18  (0x12)
+    uint32_t compute_pgm_rsrc1; // offset: 19  (0x13)
+    uint32_t compute_pgm_rsrc2; // offset: 20  (0x14)
+    uint32_t compute_vmid; // offset: 21  (0x15)
+    uint32_t compute_resource_limits; // offset: 22  (0x16)
+    uint32_t compute_tmpring_size; // offset: 23  (0x17)
+    uint32_t compute_restart_x; // offset: 24  (0x18)
+    uint32_t compute_restart_y; // offset: 25  (0x19)
+    uint32_t compute_restart_z; // offset: 26  (0x1A)
+    uint32_t compute_thread_trace_enable; // offset: 27  (0x1B)
+    uint32_t compute_misc_reserved; // offset: 28  (0x1C)
+    uint32_t compute_dispatch_id; // offset: 29  (0x1D)
+    uint32_t compute_threadgroup_id; // offset: 30  (0x1E)
+    uint32_t compute_req_ctrl; // offset: 31  (0x1F)
+    uint32_t compute_user_accum_0; // offset: 32  (0x20)
+    uint32_t compute_user_accum_1; // offset: 33  (0x21)
+    uint32_t compute_user_accum_2; // offset: 34  (0x22)
+    uint32_t compute_user_accum_3; // offset: 35  (0x23)
+    uint32_t compute_pgm_rsrc3; // offset: 36  (0x24)
+    uint32_t compute_ddid_index; // offset: 37  (0x25)
+    uint32_t compute_shader_chksum; // offset: 38  (0x26)
+    uint32_t compute_dispatch_interleave; // offset: 39  (0x27)
+    uint32_t compute_current_logical_xcc_id; // offset: 40  (0x28)
+    uint32_t compute_restart_cg_tg_id; // offset: 41  (0x29)
+    uint32_t compute_tg_chunk_size; // offset: 42  (0x2A)
+    uint32_t compute_restore_tg_chunk_size; // offset: 43  (0x2B)
+    uint32_t reserved_44; // offset: 44  (0x2C)
+    uint32_t reserved_45; // offset: 45  (0x2D)
+    uint32_t compute_prescaled_dim_x; // offset: 46  (0x2E)
+    uint32_t compute_prescaled_dim_y; // offset: 47  (0x2F)
+    uint32_t compute_prescaled_dim_z; // offset: 48  (0x30)
+    uint32_t compute_static_thread_mgmt_se0; // offset: 49  (0x31)
+    uint32_t compute_static_thread_mgmt_se1; // offset: 50  (0x32)
+    uint32_t compute_static_thread_mgmt_se2; // offset: 51  (0x33)
+    uint32_t compute_static_thread_mgmt_se3; // offset: 52  (0x34)
+    uint32_t compute_static_thread_mgmt_se4; // offset: 53  (0x35)
+    uint32_t compute_static_thread_mgmt_se5; // offset: 54  (0x36)
+    uint32_t compute_static_thread_mgmt_se6; // offset: 55  (0x37)
+    uint32_t compute_static_thread_mgmt_se7; // offset: 56  (0x38)
+    uint32_t compute_static_thread_mgmt_se8; // offset: 57  (0x39)
+    uint32_t reserved_58; // offset: 58  (0x3A)
+    uint32_t reserved_59; // offset: 59  (0x3B)
+    uint32_t reserved_60; // offset: 60  (0x3C)
+    uint32_t reserved_61; // offset: 61  (0x3D)
+    uint32_t reserved_62; // offset: 62  (0x3E)
+    uint32_t reserved_63; // offset: 63  (0x3F)
+    uint32_t tg_counter_id; // offset: 64  (0x40)
+    uint32_t compute_user_data_0; // offset: 65  (0x41)
+    uint32_t compute_user_data_1; // offset: 66  (0x42)
+    uint32_t compute_user_data_2; // offset: 67  (0x43)
+    uint32_t compute_user_data_3; // offset: 68  (0x44)
+    uint32_t compute_user_data_4; // offset: 69  (0x45)
+    uint32_t compute_user_data_5; // offset: 70  (0x46)
+    uint32_t compute_user_data_6; // offset: 71  (0x47)
+    uint32_t compute_user_data_7; // offset: 72  (0x48)
+    uint32_t compute_user_data_8; // offset: 73  (0x49)
+    uint32_t compute_user_data_9; // offset: 74  (0x4A)
+    uint32_t compute_user_data_10; // offset: 75  (0x4B)
+    uint32_t compute_user_data_11; // offset: 76  (0x4C)
+    uint32_t compute_user_data_12; // offset: 77  (0x4D)
+    uint32_t compute_user_data_13; // offset: 78  (0x4E)
+    uint32_t compute_user_data_14; // offset: 79  (0x4F)
+    uint32_t compute_user_data_15; // offset: 80  (0x50)
+    uint32_t compute_user_data_16; // offset: 81  (0x51)
+    uint32_t compute_user_data_17; // offset: 82  (0x52)
+    uint32_t compute_user_data_18; // offset: 83  (0x53)
+    uint32_t compute_user_data_19; // offset: 84  (0x54)
+    uint32_t compute_user_data_20; // offset: 85  (0x55)
+    uint32_t compute_user_data_21; // offset: 86  (0x56)
+    uint32_t compute_user_data_22; // offset: 87  (0x57)
+    uint32_t compute_user_data_23; // offset: 88  (0x58)
+    uint32_t compute_user_data_24; // offset: 89  (0x59)
+    uint32_t compute_user_data_25; // offset: 90  (0x5A)
+    uint32_t compute_user_data_26; // offset: 91  (0x5B)
+    uint32_t compute_user_data_27; // offset: 92  (0x5C)
+    uint32_t compute_user_data_28; // offset: 93  (0x5D)
+    uint32_t compute_user_data_29; // offset: 94  (0x5E)
+    uint32_t compute_user_data_30; // offset: 95  (0x5F)
+    uint32_t compute_user_data_31; // offset: 96  (0x60)
+    uint32_t cp_compute_csinvoc_count_lo; // offset: 97  (0x61)
+    uint32_t cp_compute_csinvoc_count_hi; // offset: 98  (0x62)
+    uint32_t reserved_99; // offset: 99  (0x63)
+    uint32_t reserved_100; // offset: 100  (0x64)
+    uint32_t reserved_101; // offset: 101  (0x65)
+    uint32_t reserved_102; // offset: 102  (0x66)
+    uint32_t reserved_103; // offset: 103  (0x67)
+    uint32_t reserved_104; // offset: 104  (0x68)
+    uint32_t reserved_105; // offset: 105  (0x69)
+    uint32_t reserved_106; // offset: 106  (0x6A)
+    uint32_t reserved_107; // offset: 107  (0x6B)
+    uint32_t reserved_108; // offset: 108  (0x6C)
+    uint32_t reserved_109; // offset: 109  (0x6D)
+    uint32_t compute_relaunch; // offset: 110  (0x6E)
+    uint32_t compute_wave_restore_addr_lo; // offset: 111  (0x6F)
+    uint32_t compute_wave_restore_addr_hi; // offset: 112  (0x70)
+    uint32_t compute_relaunch2; // offset: 113  (0x71)
+    uint32_t cp_pq_exe_status_lo; // offset: 114  (0x72)
+    uint32_t cp_pq_exe_status_hi; // offset: 115  (0x73)
+    uint32_t cp_packet_id_lo; // offset: 116  (0x74)
+    uint32_t cp_packet_id_hi; // offset: 117  (0x75)
+    uint32_t cp_packet_exe_status_lo; // offset: 118  (0x76)
+    uint32_t cp_packet_exe_status_hi; // offset: 119  (0x77)
+    uint32_t reserved_120; // offset: 120  (0x78)
+    uint32_t reserved_121; // offset: 121  (0x79)
+    uint32_t reserved_122; // offset: 122  (0x7A)
+    uint32_t reserved_123; // offset: 123  (0x7B)
+    uint32_t ctx_save_base_addr_lo; // offset: 124  (0x7C)
+    uint32_t ctx_save_base_addr_hi; // offset: 125  (0x7D)
+    uint32_t reserved_126; // offset: 126  (0x7E)
+    uint32_t reserved_127; // offset: 127  (0x7F)
+    uint32_t cp_mqd_base_addr_lo; // offset: 128  (0x80)
+    uint32_t cp_mqd_base_addr_hi; // offset: 129  (0x81)
+    uint32_t cp_hqd_active; // offset: 130  (0x82)
+    uint32_t cp_hqd_vmid; // offset: 131  (0x83)
+    uint32_t cp_hqd_persistent_state; // offset: 132  (0x84)
+    uint32_t cp_hqd_pipe_priority; // offset: 133  (0x85)
+    uint32_t cp_hqd_queue_priority; // offset: 134  (0x86)
+    uint32_t cp_hqd_quantum; // offset: 135  (0x87)
+    uint32_t cp_hqd_pq_base_lo; // offset: 136  (0x88)
+    uint32_t cp_hqd_pq_base_hi; // offset: 137  (0x89)
+    uint32_t cp_hqd_pq_rptr; // offset: 138  (0x8A)
+    uint32_t cp_hqd_pq_rptr_report_addr_lo; // offset: 139  (0x8B)
+    uint32_t cp_hqd_pq_rptr_report_addr_hi; // offset: 140  (0x8C)
+    uint32_t cp_hqd_pq_wptr_poll_addr_lo; // offset: 141  (0x8D)
+    uint32_t cp_hqd_pq_wptr_poll_addr_hi; // offset: 142  (0x8E)
+    uint32_t cp_hqd_pq_doorbell_control; // offset: 143  (0x8F)
+    uint32_t reserved_144; // offset: 144  (0x90)
+    uint32_t cp_hqd_pq_control; // offset: 145  (0x91)
+    uint32_t cp_hqd_ib_base_addr_lo; // offset: 146  (0x92)
+    uint32_t cp_hqd_ib_base_addr_hi; // offset: 147  (0x93)
+    uint32_t cp_hqd_ib_rptr; // offset: 148  (0x94)
+    uint32_t cp_hqd_ib_control; // offset: 149  (0x95)
+    uint32_t cp_hqd_iq_timer; // offset: 150  (0x96)
+    uint32_t cp_hqd_iq_rptr; // offset: 151  (0x97)
+    uint32_t cp_hqd_dequeue_request; // offset: 152  (0x98)
+    uint32_t cp_hqd_dma_offload; // offset: 153  (0x99)
+    uint32_t cp_hqd_sema_cmd; // offset: 154  (0x9A)
+    uint32_t cp_hqd_msg_type; // offset: 155  (0x9B)
+    uint32_t cp_hqd_atomic0_preop_lo; // offset: 156  (0x9C)
+    uint32_t cp_hqd_atomic0_preop_hi; // offset: 157  (0x9D)
+    uint32_t cp_hqd_atomic1_preop_lo; // offset: 158  (0x9E)
+    uint32_t cp_hqd_atomic1_preop_hi; // offset: 159  (0x9F)
+    uint32_t cp_hqd_hq_status0; // offset: 160  (0xA0)
+    uint32_t cp_hqd_hq_control0; // offset: 161  (0xA1)
+    uint32_t cp_mqd_control; // offset: 162  (0xA2)
+    uint32_t cp_hqd_hq_status1; // offset: 163  (0xA3)
+    uint32_t cp_hqd_hq_control1; // offset: 164  (0xA4)
+    uint32_t cp_hqd_eop_base_addr_lo; // offset: 165  (0xA5)
+    uint32_t cp_hqd_eop_base_addr_hi; // offset: 166  (0xA6)
+    uint32_t cp_hqd_eop_control; // offset: 167  (0xA7)
+    uint32_t cp_hqd_eop_rptr; // offset: 168  (0xA8)
+    uint32_t cp_hqd_eop_wptr; // offset: 169  (0xA9)
+    uint32_t cp_hqd_eop_done_events; // offset: 170  (0xAA)
+    uint32_t cp_hqd_ctx_save_base_addr_lo; // offset: 171  (0xAB)
+    uint32_t cp_hqd_ctx_save_base_addr_hi; // offset: 172  (0xAC)
+    uint32_t cp_hqd_ctx_save_control; // offset: 173  (0xAD)
+    uint32_t cp_hqd_cntl_stack_offset; // offset: 174  (0xAE)
+    uint32_t cp_hqd_cntl_stack_size; // offset: 175  (0xAF)
+    uint32_t cp_hqd_wg_state_offset; // offset: 176  (0xB0)
+    uint32_t cp_hqd_ctx_save_size; // offset: 177  (0xB1)
+    uint32_t reserved_178; // offset: 178  (0xB2)
+    uint32_t cp_hqd_error; // offset: 179  (0xB3)
+    uint32_t cp_hqd_eop_wptr_mem; // offset: 180  (0xB4)
+    uint32_t cp_hqd_aql_control; // offset: 181  (0xB5)
+    uint32_t cp_hqd_pq_wptr_lo; // offset: 182  (0xB6)
+    uint32_t cp_hqd_pq_wptr_hi; // offset: 183  (0xB7)
+    uint32_t cp_hqd_suspend_cntl_stack_offset; // offset: 184  (0xB8)
+    uint32_t cp_hqd_suspend_cntl_stack_dw_cnt; // offset: 185  (0xB9)
+    uint32_t cp_hqd_suspend_wg_state_offset; // offset: 186  (0xBA)
+    uint32_t cp_hqd_ddid_rptr; // offset: 187  (0xBB)
+    uint32_t cp_hqd_ddid_wptr; // offset: 188  (0xBC)
+    uint32_t cp_hqd_ddid_inflight_count; // offset: 189  (0xBD)
+    uint32_t cp_hqd_ddid_delta_rpt_count; // offset: 190  (0xBE)
+    uint32_t cp_hqd_dequeue_status; // offset: 191  (0xBF)
+    uint32_t cp_hqd_aql_control_1; // offset: 192  (0xC0)
+    uint32_t cp_hqd_aql_dispatch_id; // offset: 193  (0xC1)
+    uint32_t cp_hqd_aql_dispatch_id_hi; // offset: 194  (0xC2)
+    uint32_t cp_hqd_kd_base; // offset: 195  (0xC3)
+    uint32_t cp_hqd_kd_base_hi; // offset: 196  (0xC4)
+    uint32_t cp_hqd_kd_cntl; // offset: 197  (0xC5)
+    uint32_t reserved_198; // offset: 198  (0xC6)
+    uint32_t reserved_199; // offset: 199  (0xC7)
+    uint32_t reserved_200; // offset: 200  (0xC8)
+    uint32_t reserved_201; // offset: 201  (0xC9)
+    uint32_t reserved_202; // offset: 202  (0xCA)
+    uint32_t reserved_203; // offset: 203  (0xCB)
+    uint32_t reserved_204; // offset: 204  (0xCC)
+    uint32_t reserved_205; // offset: 205  (0xCD)
+    uint32_t reserved_206; // offset: 206  (0xCE)
+    uint32_t reserved_207; // offset: 207  (0xCF)
+    uint32_t reserved_208; // offset: 208  (0xD0)
+    uint32_t reserved_209; // offset: 209  (0xD1)
+    uint32_t reserved_210; // offset: 210  (0xD2)
+    uint32_t reserved_211; // offset: 211  (0xD3)
+    uint32_t reserved_212; // offset: 212  (0xD4)
+    uint32_t reserved_213; // offset: 213  (0xD5)
+    uint32_t reserved_214; // offset: 214  (0xD6)
+    uint32_t reserved_215; // offset: 215  (0xD7)
+    uint32_t reserved_216; // offset: 216  (0xD8)
+    uint32_t reserved_217; // offset: 217  (0xD9)
+    uint32_t reserved_218; // offset: 218  (0xDA)
+    uint32_t reserved_219; // offset: 219  (0xDB)
+    uint32_t reserved_220; // offset: 220  (0xDC)
+    uint32_t reserved_221; // offset: 221  (0xDD)
+    uint32_t reserved_222; // offset: 222  (0xDE)
+    uint32_t reserved_223; // offset: 223  (0xDF)
+    uint32_t reserved_224; // offset: 224  (0xE0)
+    uint32_t pm4_target_xcc_in_xcp; // offset: 225  (0xE1)
+    uint32_t cp_mqd_stride_size; // offset: 226  (0xE2)
+    uint32_t xcc_sync_counter; // offset: 227  (0xE3)
+    uint32_t set_resources_header; // offset: 228  (0xE4)
+    uint32_t set_resources_dw1; // offset: 229  (0xE5)
+    uint32_t set_resources_dw2; // offset: 230  (0xE6)
+    uint32_t set_resources_dw3; // offset: 231  (0xE7)
+    uint32_t set_resources_dw4; // offset: 232  (0xE8)
+    uint32_t set_resources_dw5; // offset: 233  (0xE9)
+    uint32_t set_resources_dw6; // offset: 234  (0xEA)
+    uint32_t set_resources_dw7; // offset: 235  (0xEB)
+    uint32_t reserved_236; // offset: 236  (0xEC)
+    uint32_t reserved_237; // offset: 237  (0xED)
+    uint32_t reserved_238; // offset: 238  (0xEE)
+    uint32_t reserved_239; // offset: 239  (0xEF)
+    uint32_t reserved_240; // offset: 240  (0xF0)
+    uint32_t reserved_241; // offset: 241  (0xF1)
+    uint32_t reserved_242; // offset: 242  (0xF2)
+    uint32_t reserved_243; // offset: 243  (0xF3)
+    uint32_t reserved_244; // offset: 244  (0xF4)
+    uint32_t reserved_245; // offset: 245  (0xF5)
+    uint32_t reserved_246; // offset: 246  (0xF6)
+    uint32_t reserved_247; // offset: 247  (0xF7)
+    uint32_t reserved_248; // offset: 248  (0xF8)
+    uint32_t reserved_249; // offset: 249  (0xF9)
+    uint32_t reserved_250; // offset: 250  (0xFA)
+    uint32_t reserved_251; // offset: 251  (0xFB)
+    uint32_t reserved_252; // offset: 252  (0xFC)
+    uint32_t reserved_253; // offset: 253  (0xFD)
+    uint32_t reserved_254; // offset: 254  (0xFE)
+    uint32_t reserved_255; // offset: 255  (0xFF)
+    uint32_t control_buf_addr_lo; // offset: 256  (0x100)
+    uint32_t control_buf_addr_hi; // offset: 257  (0x101)
+    uint32_t control_buf_wptr_lo; // offset: 258  (0x102)
+    uint32_t control_buf_wptr_hi; // offset: 259  (0x103)
+    uint32_t control_buf_dptr_lo; // offset: 260  (0x104)
+    uint32_t control_buf_dptr_hi; // offset: 261  (0x105)
+    uint32_t draw_ring_addr_lo; // offset: 262  (0x106)
+    uint32_t draw_ring_addr_hi; // offset: 263  (0x107)
+    uint32_t control_buf_num_entries; // offset: 264  (0x108)
+    uint32_t reserved_265; // offset: 265  (0x109)
+    uint32_t reserved_266; // offset: 266  (0x10A)
+    uint32_t reserved_267; // offset: 267  (0x10B)
+    uint32_t reserved_268; // offset: 268  (0x10C)
+    uint32_t reserved_269; // offset: 269  (0x10D)
+    uint32_t reserved_270; // offset: 270  (0x10E)
+    uint32_t reserved_271; // offset: 271  (0x10F)
+    uint32_t dfwx_flags; // offset: 272  (0x110)
+    uint32_t dfwx_slot; // offset: 273  (0x111)
+    uint32_t dfwx_client_data_addr_lo; // offset: 274  (0x112)
+    uint32_t dfwx_client_data_addr_hi; // offset: 275  (0x113)
+    uint32_t dfwx_queue_connect_addr_lo; // offset: 276  (0x114)
+    uint32_t dfwx_queue_connect_addr_hi; // offset: 277  (0x115)
+    uint32_t reserved_278; // offset: 278  (0x116)
+    uint32_t reserved_279; // offset: 279  (0x117)
+    uint32_t reserved_280; // offset: 280  (0x118)
+    uint32_t reserved_281; // offset: 281  (0x119)
+    uint32_t reserved_282; // offset: 282  (0x11A)
+    uint32_t reserved_283; // offset: 283  (0x11B)
+    uint32_t reserved_284; // offset: 284  (0x11C)
+    uint32_t reserved_285; // offset: 285  (0x11D)
+    uint32_t reserved_286; // offset: 286  (0x11E)
+    uint32_t reserved_287; // offset: 287  (0x11F)
+    uint32_t cp_mqd_query_time_lo; // offset: 288  (0x120)
+    uint32_t cp_mqd_query_time_hi; // offset: 289  (0x121)
+    uint32_t cp_mqd_connect_start_time_lo; // offset: 290  (0x122)
+    uint32_t cp_mqd_connect_start_time_hi; // offset: 291  (0x123)
+    uint32_t cp_mqd_connect_end_time_lo; // offset: 292  (0x124)
+    uint32_t cp_mqd_connect_end_time_hi; // offset: 293  (0x125)
+    uint32_t cp_mqd_connect_end_wf_count; // offset: 294  (0x126)
+    uint32_t cp_mqd_connect_end_pq_rptr; // offset: 295  (0x127)
+    uint32_t cp_mqd_connect_end_pq_wptr; // offset: 296  (0x128)
+    uint32_t cp_mqd_connect_end_ib_rptr; // offset: 297  (0x129)
+    uint32_t cp_mqd_readindex_lo; // offset: 298  (0x12A)
+    uint32_t cp_mqd_readindex_hi; // offset: 299  (0x12B)
+    uint32_t cp_mqd_save_start_time_lo; // offset: 300  (0x12C)
+    uint32_t cp_mqd_save_start_time_hi; // offset: 301  (0x12D)
+    uint32_t cp_mqd_save_end_time_lo; // offset: 302  (0x12E)
+    uint32_t cp_mqd_save_end_time_hi; // offset: 303  (0x12F)
+    uint32_t cp_mqd_restore_start_time_lo; // offset: 304  (0x130)
+    uint32_t cp_mqd_restore_start_time_hi; // offset: 305  (0x131)
+    uint32_t cp_mqd_restore_end_time_lo; // offset: 306  (0x132)
+    uint32_t cp_mqd_restore_end_time_hi; // offset: 307  (0x133)
+    uint32_t disable_queue; // offset: 308  (0x134)
+    uint32_t reserved_309; // offset: 309  (0x135)
+    uint32_t reserved_310; // offset: 310  (0x136)
+    uint32_t reserved_311; // offset: 311  (0x137)
+    uint32_t reserved_312; // offset: 312  (0x138)
+    uint32_t reserved_313; // offset: 313  (0x139)
+    uint32_t reserved_314; // offset: 314  (0x13A)
+    uint32_t reserved_315; // offset: 315  (0x13B)
+    uint32_t reserved_316; // offset: 316  (0x13C)
+    uint32_t reserved_317; // offset: 317  (0x13D)
+    uint32_t reserved_318; // offset: 318  (0x13E)
+    uint32_t reserved_319; // offset: 319  (0x13F)
+    uint32_t reserved_320; // offset: 320  (0x140)
+    uint32_t reserved_321; // offset: 321  (0x141)
+    uint32_t reserved_322; // offset: 322  (0x142)
+    uint32_t reserved_323; // offset: 323  (0x143)
+    uint32_t reserved_324; // offset: 324  (0x144)
+    uint32_t reserved_325; // offset: 325  (0x145)
+    uint32_t reserved_326; // offset: 326  (0x146)
+    uint32_t reserved_327; // offset: 327  (0x147)
+    uint32_t reserved_328; // offset: 328  (0x148)
+    uint32_t reserved_329; // offset: 329  (0x149)
+    uint32_t reserved_330; // offset: 330  (0x14A)
+    uint32_t reserved_331; // offset: 331  (0x14B)
+    uint32_t reserved_332; // offset: 332  (0x14C)
+    uint32_t reserved_333; // offset: 333  (0x14D)
+    uint32_t reserved_334; // offset: 334  (0x14E)
+    uint32_t reserved_335; // offset: 335  (0x14F)
+    uint32_t reserved_336; // offset: 336  (0x150)
+    uint32_t reserved_337; // offset: 337  (0x151)
+    uint32_t reserved_338; // offset: 338  (0x152)
+    uint32_t reserved_339; // offset: 339  (0x153)
+    uint32_t reserved_340; // offset: 340  (0x154)
+    uint32_t reserved_341; // offset: 341  (0x155)
+    uint32_t reserved_342; // offset: 342  (0x156)
+    uint32_t reserved_343; // offset: 343  (0x157)
+    uint32_t reserved_344; // offset: 344  (0x158)
+    uint32_t reserved_345; // offset: 345  (0x159)
+    uint32_t reserved_346; // offset: 346  (0x15A)
+    uint32_t reserved_347; // offset: 347  (0x15B)
+    uint32_t reserved_348; // offset: 348  (0x15C)
+    uint32_t reserved_349; // offset: 349  (0x15D)
+    uint32_t reserved_350; // offset: 350  (0x15E)
+    uint32_t reserved_351; // offset: 351  (0x15F)
+    uint32_t reserved_352; // offset: 352  (0x160)
+    uint32_t reserved_353; // offset: 353  (0x161)
+    uint32_t reserved_354; // offset: 354  (0x162)
+    uint32_t reserved_355; // offset: 355  (0x163)
+    uint32_t reserved_356; // offset: 356  (0x164)
+    uint32_t reserved_357; // offset: 357  (0x165)
+    uint32_t reserved_358; // offset: 358  (0x166)
+    uint32_t reserved_359; // offset: 359  (0x167)
+    uint32_t reserved_360; // offset: 360  (0x168)
+    uint32_t reserved_361; // offset: 361  (0x169)
+    uint32_t reserved_362; // offset: 362  (0x16A)
+    uint32_t reserved_363; // offset: 363  (0x16B)
+    uint32_t reserved_364; // offset: 364  (0x16C)
+    uint32_t reserved_365; // offset: 365  (0x16D)
+    uint32_t reserved_366; // offset: 366  (0x16E)
+    uint32_t reserved_367; // offset: 367  (0x16F)
+    uint32_t reserved_368; // offset: 368  (0x170)
+    uint32_t reserved_369; // offset: 369  (0x171)
+    uint32_t reserved_370; // offset: 370  (0x172)
+    uint32_t reserved_371; // offset: 371  (0x173)
+    uint32_t reserved_372; // offset: 372  (0x174)
+    uint32_t reserved_373; // offset: 373  (0x175)
+    uint32_t reserved_374; // offset: 374  (0x176)
+    uint32_t reserved_375; // offset: 375  (0x177)
+    uint32_t reserved_376; // offset: 376  (0x178)
+    uint32_t reserved_377; // offset: 377  (0x179)
+    uint32_t reserved_378; // offset: 378  (0x17A)
+    uint32_t reserved_379; // offset: 379  (0x17B)
+    uint32_t reserved_380; // offset: 380  (0x17C)
+    uint32_t reserved_381; // offset: 381  (0x17D)
+    uint32_t reserved_382; // offset: 382  (0x17E)
+    uint32_t reserved_383; // offset: 383  (0x17F)
+    uint32_t iqtimer_pkt_header; // offset: 384  (0x180)
+    uint32_t iqtimer_pkt_dw0; // offset: 385  (0x181)
+    uint32_t iqtimer_pkt_dw1; // offset: 386  (0x182)
+    uint32_t iqtimer_pkt_dw2; // offset: 387  (0x183)
+    uint32_t iqtimer_pkt_dw3; // offset: 388  (0x184)
+    uint32_t iqtimer_pkt_dw4; // offset: 389  (0x185)
+    uint32_t iqtimer_pkt_dw5; // offset: 390  (0x186)
+    uint32_t iqtimer_pkt_dw6; // offset: 391  (0x187)
+    uint32_t iqtimer_pkt_dw7; // offset: 392  (0x188)
+    uint32_t iqtimer_pkt_dw8; // offset: 393  (0x189)
+    uint32_t iqtimer_pkt_dw9; // offset: 394  (0x18A)
+    uint32_t iqtimer_pkt_dw10; // offset: 395  (0x18B)
+    uint32_t iqtimer_pkt_dw11; // offset: 396  (0x18C)
+    uint32_t iqtimer_pkt_dw12; // offset: 397  (0x18D)
+    uint32_t iqtimer_pkt_dw13; // offset: 398  (0x18E)
+    uint32_t iqtimer_pkt_dw14; // offset: 399  (0x18F)
+    uint32_t iqtimer_pkt_dw15; // offset: 400  (0x190)
+    uint32_t iqtimer_pkt_dw16; // offset: 401  (0x191)
+    uint32_t iqtimer_pkt_dw17; // offset: 402  (0x192)
+    uint32_t iqtimer_pkt_dw18; // offset: 403  (0x193)
+    uint32_t iqtimer_pkt_dw19; // offset: 404  (0x194)
+    uint32_t iqtimer_pkt_dw20; // offset: 405  (0x195)
+    uint32_t iqtimer_pkt_dw21; // offset: 406  (0x196)
+    uint32_t iqtimer_pkt_dw22; // offset: 407  (0x197)
+    uint32_t iqtimer_pkt_dw23; // offset: 408  (0x198)
+    uint32_t iqtimer_pkt_dw24; // offset: 409  (0x199)
+    uint32_t iqtimer_pkt_dw25; // offset: 410  (0x19A)
+    uint32_t iqtimer_pkt_dw26; // offset: 411  (0x19B)
+    uint32_t iqtimer_pkt_dw27; // offset: 412  (0x19C)
+    uint32_t iqtimer_pkt_dw28; // offset: 413  (0x19D)
+    uint32_t iqtimer_pkt_dw29; // offset: 414  (0x19E)
+    uint32_t iqtimer_pkt_dw30; // offset: 415  (0x19F)
+    uint32_t iqtimer_pkt_dw31; // offset: 416  (0x1A0)
+    uint32_t reserved_417; // offset: 417  (0x1A1)
+    uint32_t reserved_418; // offset: 418  (0x1A2)
+    uint32_t reserved_419; // offset: 419  (0x1A3)
+    uint32_t reserved_420; // offset: 420  (0x1A4)
+    uint32_t reserved_421; // offset: 421  (0x1A5)
+    uint32_t reserved_422; // offset: 422  (0x1A6)
+    uint32_t reserved_423; // offset: 423  (0x1A7)
+    uint32_t reserved_424; // offset: 424  (0x1A8)
+    uint32_t reserved_425; // offset: 425  (0x1A9)
+    uint32_t reserved_426; // offset: 426  (0x1AA)
+    uint32_t reserved_427; // offset: 427  (0x1AB)
+    uint32_t reserved_428; // offset: 428  (0x1AC)
+    uint32_t reserved_429; // offset: 429  (0x1AD)
+    uint32_t reserved_430; // offset: 430  (0x1AE)
+    uint32_t reserved_431; // offset: 431  (0x1AF)
+    uint32_t reserved_432; // offset: 432  (0x1B0)
+    uint32_t reserved_433; // offset: 433  (0x1B1)
+    uint32_t reserved_434; // offset: 434  (0x1B2)
+    uint32_t reserved_435; // offset: 435  (0x1B3)
+    uint32_t reserved_436; // offset: 436  (0x1B4)
+    uint32_t reserved_437; // offset: 437  (0x1B5)
+    uint32_t reserved_438; // offset: 438  (0x1B6)
+    uint32_t reserved_439; // offset: 439  (0x1B7)
+    uint32_t reserved_440; // offset: 440  (0x1B8)
+    uint32_t reserved_441; // offset: 441  (0x1B9)
+    uint32_t reserved_442; // offset: 442  (0x1BA)
+    uint32_t reserved_443; // offset: 443  (0x1BB)
+    uint32_t reserved_444; // offset: 444  (0x1BC)
+    uint32_t reserved_445; // offset: 445  (0x1BD)
+    uint32_t reserved_446; // offset: 446  (0x1BE)
+    uint32_t reserved_447; // offset: 447  (0x1BF)
+    uint32_t reserved_448; // offset: 448  (0x1C0)
+    uint32_t reserved_449; // offset: 449  (0x1C1)
+    uint32_t reserved_450; // offset: 450  (0x1C2)
+    uint32_t reserved_451; // offset: 451  (0x1C3)
+    uint32_t reserved_452; // offset: 452  (0x1C4)
+    uint32_t reserved_453; // offset: 453  (0x1C5)
+    uint32_t reserved_454; // offset: 454  (0x1C6)
+    uint32_t reserved_455; // offset: 455  (0x1C7)
+    uint32_t reserved_456; // offset: 456  (0x1C8)
+    uint32_t reserved_457; // offset: 457  (0x1C9)
+    uint32_t reserved_458; // offset: 458  (0x1CA)
+    uint32_t reserved_459; // offset: 459  (0x1CB)
+    uint32_t reserved_460; // offset: 460  (0x1CC)
+    uint32_t reserved_461; // offset: 461  (0x1CD)
+    uint32_t reserved_462; // offset: 462  (0x1CE)
+    uint32_t reserved_463; // offset: 463  (0x1CF)
+    uint32_t reserved_464; // offset: 464  (0x1D0)
+    uint32_t reserved_465; // offset: 465  (0x1D1)
+    uint32_t reserved_466; // offset: 466  (0x1D2)
+    uint32_t reserved_467; // offset: 467  (0x1D3)
+    uint32_t reserved_468; // offset: 468  (0x1D4)
+    uint32_t reserved_469; // offset: 469  (0x1D5)
+    uint32_t reserved_470; // offset: 470  (0x1D6)
+    uint32_t reserved_471; // offset: 471  (0x1D7)
+    uint32_t reserved_472; // offset: 472  (0x1D8)
+    uint32_t reserved_473; // offset: 473  (0x1D9)
+    uint32_t reserved_474; // offset: 474  (0x1DA)
+    uint32_t reserved_475; // offset: 475  (0x1DB)
+    uint32_t reserved_476; // offset: 476  (0x1DC)
+    uint32_t reserved_477; // offset: 477  (0x1DD)
+    uint32_t reserved_478; // offset: 478  (0x1DE)
+    uint32_t reserved_479; // offset: 479  (0x1DF)
+    uint32_t reserved_480; // offset: 480  (0x1E0)
+    uint32_t reserved_481; // offset: 481  (0x1E1)
+    uint32_t reserved_482; // offset: 482  (0x1E2)
+    uint32_t reserved_483; // offset: 483  (0x1E3)
+    uint32_t reserved_484; // offset: 484  (0x1E4)
+    uint32_t reserved_485; // offset: 485  (0x1E5)
+    uint32_t reserved_486; // offset: 486  (0x1E6)
+    uint32_t reserved_487; // offset: 487  (0x1E7)
+    uint32_t reserved_488; // offset: 488  (0x1E8)
+    uint32_t reserved_489; // offset: 489  (0x1E9)
+    uint32_t reserved_490; // offset: 490  (0x1EA)
+    uint32_t reserved_491; // offset: 491  (0x1EB)
+    uint32_t reserved_492; // offset: 492  (0x1EC)
+    uint32_t reserved_493; // offset: 493  (0x1ED)
+    uint32_t reserved_494; // offset: 494  (0x1EE)
+    uint32_t reserved_495; // offset: 495  (0x1EF)
+    uint32_t reserved_496; // offset: 496  (0x1F0)
+    uint32_t reserved_497; // offset: 497  (0x1F1)
+    uint32_t reserved_498; // offset: 498  (0x1F2)
+    uint32_t reserved_499; // offset: 499  (0x1F3)
+    uint32_t reserved_500; // offset: 500  (0x1F4)
+    uint32_t reserved_501; // offset: 501  (0x1F5)
+    uint32_t reserved_502; // offset: 502  (0x1F6)
+    uint32_t reserved_503; // offset: 503  (0x1F7)
+    uint32_t reserved_504; // offset: 504  (0x1F8)
+    uint32_t reserved_505; // offset: 505  (0x1F9)
+    uint32_t reserved_506; // offset: 506  (0x1FA)
+    uint32_t reserved_507; // offset: 507  (0x1FB)
+    uint32_t reserved_508; // offset: 508  (0x1FC)
+    uint32_t reserved_509; // offset: 509  (0x1FD)
+    uint32_t reserved_510; // offset: 510  (0x1FE)
+    uint32_t reserved_511; // offset: 511  (0x1FF)
+    uint32_t reserved_512; // offset: 512  (0x200)
+    uint32_t reserved_513; // offset: 513  (0x201)
+    uint32_t reserved_514; // offset: 514  (0x202)
+    uint32_t reserved_515; // offset: 515  (0x203)
+    uint32_t reserved_516; // offset: 516  (0x204)
+    uint32_t reserved_517; // offset: 517  (0x205)
+    uint32_t reserved_518; // offset: 518  (0x206)
+    uint32_t reserved_519; // offset: 519  (0x207)
+    uint32_t reserved_520; // offset: 520  (0x208)
+    uint32_t reserved_521; // offset: 521  (0x209)
+    uint32_t reserved_522; // offset: 522  (0x20A)
+    uint32_t reserved_523; // offset: 523  (0x20B)
+    uint32_t reserved_524; // offset: 524  (0x20C)
+    uint32_t reserved_525; // offset: 525  (0x20D)
+    uint32_t reserved_526; // offset: 526  (0x20E)
+    uint32_t reserved_527; // offset: 527  (0x20F)
+    uint32_t reserved_528; // offset: 528  (0x210)
+    uint32_t reserved_529; // offset: 529  (0x211)
+    uint32_t reserved_530; // offset: 530  (0x212)
+    uint32_t reserved_531; // offset: 531  (0x213)
+    uint32_t reserved_532; // offset: 532  (0x214)
+    uint32_t reserved_533; // offset: 533  (0x215)
+    uint32_t reserved_534; // offset: 534  (0x216)
+    uint32_t reserved_535; // offset: 535  (0x217)
+    uint32_t reserved_536; // offset: 536  (0x218)
+    uint32_t reserved_537; // offset: 537  (0x219)
+    uint32_t reserved_538; // offset: 538  (0x21A)
+    uint32_t reserved_539; // offset: 539  (0x21B)
+    uint32_t reserved_540; // offset: 540  (0x21C)
+    uint32_t reserved_541; // offset: 541  (0x21D)
+    uint32_t reserved_542; // offset: 542  (0x21E)
+    uint32_t reserved_543; // offset: 543  (0x21F)
+    uint32_t reserved_544; // offset: 544  (0x220)
+    uint32_t reserved_545; // offset: 545  (0x221)
+    uint32_t reserved_546; // offset: 546  (0x222)
+    uint32_t reserved_547; // offset: 547  (0x223)
+    uint32_t reserved_548; // offset: 548  (0x224)
+    uint32_t reserved_549; // offset: 549  (0x225)
+    uint32_t reserved_550; // offset: 550  (0x226)
+    uint32_t reserved_551; // offset: 551  (0x227)
+    uint32_t reserved_552; // offset: 552  (0x228)
+    uint32_t reserved_553; // offset: 553  (0x229)
+    uint32_t reserved_554; // offset: 554  (0x22A)
+    uint32_t reserved_555; // offset: 555  (0x22B)
+    uint32_t reserved_556; // offset: 556  (0x22C)
+    uint32_t reserved_557; // offset: 557  (0x22D)
+    uint32_t reserved_558; // offset: 558  (0x22E)
+    uint32_t reserved_559; // offset: 559  (0x22F)
+    uint32_t reserved_560; // offset: 560  (0x230)
+    uint32_t reserved_561; // offset: 561  (0x231)
+    uint32_t reserved_562; // offset: 562  (0x232)
+    uint32_t reserved_563; // offset: 563  (0x233)
+    uint32_t reserved_564; // offset: 564  (0x234)
+    uint32_t reserved_565; // offset: 565  (0x235)
+    uint32_t reserved_566; // offset: 566  (0x236)
+    uint32_t reserved_567; // offset: 567  (0x237)
+    uint32_t reserved_568; // offset: 568  (0x238)
+    uint32_t reserved_569; // offset: 569  (0x239)
+    uint32_t reserved_570; // offset: 570  (0x23A)
+    uint32_t reserved_571; // offset: 571  (0x23B)
+    uint32_t reserved_572; // offset: 572  (0x23C)
+    uint32_t reserved_573; // offset: 573  (0x23D)
+    uint32_t reserved_574; // offset: 574  (0x23E)
+    uint32_t reserved_575; // offset: 575  (0x23F)
+    uint32_t reserved_576; // offset: 576  (0x240)
+    uint32_t reserved_577; // offset: 577  (0x241)
+    uint32_t reserved_578; // offset: 578  (0x242)
+    uint32_t reserved_579; // offset: 579  (0x243)
+    uint32_t reserved_580; // offset: 580  (0x244)
+    uint32_t reserved_581; // offset: 581  (0x245)
+    uint32_t reserved_582; // offset: 582  (0x246)
+    uint32_t reserved_583; // offset: 583  (0x247)
+    uint32_t reserved_584; // offset: 584  (0x248)
+    uint32_t reserved_585; // offset: 585  (0x249)
+    uint32_t reserved_586; // offset: 586  (0x24A)
+    uint32_t reserved_587; // offset: 587  (0x24B)
+    uint32_t reserved_588; // offset: 588  (0x24C)
+    uint32_t reserved_589; // offset: 589  (0x24D)
+    uint32_t reserved_590; // offset: 590  (0x24E)
+    uint32_t reserved_591; // offset: 591  (0x24F)
+    uint32_t reserved_592; // offset: 592  (0x250)
+    uint32_t reserved_593; // offset: 593  (0x251)
+    uint32_t reserved_594; // offset: 594  (0x252)
+    uint32_t reserved_595; // offset: 595  (0x253)
+    uint32_t reserved_596; // offset: 596  (0x254)
+    uint32_t reserved_597; // offset: 597  (0x255)
+    uint32_t reserved_598; // offset: 598  (0x256)
+    uint32_t reserved_599; // offset: 599  (0x257)
+    uint32_t reserved_600; // offset: 600  (0x258)
+    uint32_t reserved_601; // offset: 601  (0x259)
+    uint32_t reserved_602; // offset: 602  (0x25A)
+    uint32_t reserved_603; // offset: 603  (0x25B)
+    uint32_t reserved_604; // offset: 604  (0x25C)
+    uint32_t reserved_605; // offset: 605  (0x25D)
+    uint32_t reserved_606; // offset: 606  (0x25E)
+    uint32_t reserved_607; // offset: 607  (0x25F)
+    uint32_t reserved_608; // offset: 608  (0x260)
+    uint32_t reserved_609; // offset: 609  (0x261)
+    uint32_t reserved_610; // offset: 610  (0x262)
+    uint32_t reserved_611; // offset: 611  (0x263)
+    uint32_t reserved_612; // offset: 612  (0x264)
+    uint32_t reserved_613; // offset: 613  (0x265)
+    uint32_t reserved_614; // offset: 614  (0x266)
+    uint32_t reserved_615; // offset: 615  (0x267)
+    uint32_t reserved_616; // offset: 616  (0x268)
+    uint32_t reserved_617; // offset: 617  (0x269)
+    uint32_t reserved_618; // offset: 618  (0x26A)
+    uint32_t reserved_619; // offset: 619  (0x26B)
+    uint32_t reserved_620; // offset: 620  (0x26C)
+    uint32_t reserved_621; // offset: 621  (0x26D)
+    uint32_t reserved_622; // offset: 622  (0x26E)
+    uint32_t reserved_623; // offset: 623  (0x26F)
+    uint32_t reserved_624; // offset: 624  (0x270)
+    uint32_t reserved_625; // offset: 625  (0x271)
+    uint32_t reserved_626; // offset: 626  (0x272)
+    uint32_t reserved_627; // offset: 627  (0x273)
+    uint32_t reserved_628; // offset: 628  (0x274)
+    uint32_t reserved_629; // offset: 629  (0x275)
+    uint32_t reserved_630; // offset: 630  (0x276)
+    uint32_t reserved_631; // offset: 631  (0x277)
+    uint32_t reserved_632; // offset: 632  (0x278)
+    uint32_t reserved_633; // offset: 633  (0x279)
+    uint32_t reserved_634; // offset: 634  (0x27A)
+    uint32_t reserved_635; // offset: 635  (0x27B)
+    uint32_t reserved_636; // offset: 636  (0x27C)
+    uint32_t reserved_637; // offset: 637  (0x27D)
+    uint32_t reserved_638; // offset: 638  (0x27E)
+    uint32_t reserved_639; // offset: 639  (0x27F)
+    uint32_t reserved_640; // offset: 640  (0x280)
+    uint32_t reserved_641; // offset: 641  (0x281)
+    uint32_t reserved_642; // offset: 642  (0x282)
+    uint32_t reserved_643; // offset: 643  (0x283)
+    uint32_t reserved_644; // offset: 644  (0x284)
+    uint32_t reserved_645; // offset: 645  (0x285)
+    uint32_t reserved_646; // offset: 646  (0x286)
+    uint32_t reserved_647; // offset: 647  (0x287)
+    uint32_t reserved_648; // offset: 648  (0x288)
+    uint32_t reserved_649; // offset: 649  (0x289)
+    uint32_t reserved_650; // offset: 650  (0x28A)
+    uint32_t reserved_651; // offset: 651  (0x28B)
+    uint32_t reserved_652; // offset: 652  (0x28C)
+    uint32_t reserved_653; // offset: 653  (0x28D)
+    uint32_t reserved_654; // offset: 654  (0x28E)
+    uint32_t reserved_655; // offset: 655  (0x28F)
+    uint32_t reserved_656; // offset: 656  (0x290)
+    uint32_t reserved_657; // offset: 657  (0x291)
+    uint32_t reserved_658; // offset: 658  (0x292)
+    uint32_t reserved_659; // offset: 659  (0x293)
+    uint32_t reserved_660; // offset: 660  (0x294)
+    uint32_t reserved_661; // offset: 661  (0x295)
+    uint32_t reserved_662; // offset: 662  (0x296)
+    uint32_t reserved_663; // offset: 663  (0x297)
+    uint32_t reserved_664; // offset: 664  (0x298)
+    uint32_t reserved_665; // offset: 665  (0x299)
+    uint32_t reserved_666; // offset: 666  (0x29A)
+    uint32_t reserved_667; // offset: 667  (0x29B)
+    uint32_t reserved_668; // offset: 668  (0x29C)
+    uint32_t reserved_669; // offset: 669  (0x29D)
+    uint32_t reserved_670; // offset: 670  (0x29E)
+    uint32_t reserved_671; // offset: 671  (0x29F)
+    uint32_t reserved_672; // offset: 672  (0x2A0)
+    uint32_t reserved_673; // offset: 673  (0x2A1)
+    uint32_t reserved_674; // offset: 674  (0x2A2)
+    uint32_t reserved_675; // offset: 675  (0x2A3)
+    uint32_t reserved_676; // offset: 676  (0x2A4)
+    uint32_t reserved_677; // offset: 677  (0x2A5)
+    uint32_t reserved_678; // offset: 678  (0x2A6)
+    uint32_t reserved_679; // offset: 679  (0x2A7)
+    uint32_t reserved_680; // offset: 680  (0x2A8)
+    uint32_t reserved_681; // offset: 681  (0x2A9)
+    uint32_t reserved_682; // offset: 682  (0x2AA)
+    uint32_t reserved_683; // offset: 683  (0x2AB)
+    uint32_t reserved_684; // offset: 684  (0x2AC)
+    uint32_t reserved_685; // offset: 685  (0x2AD)
+    uint32_t reserved_686; // offset: 686  (0x2AE)
+    uint32_t reserved_687; // offset: 687  (0x2AF)
+    uint32_t reserved_688; // offset: 688  (0x2B0)
+    uint32_t reserved_689; // offset: 689  (0x2B1)
+    uint32_t reserved_690; // offset: 690  (0x2B2)
+    uint32_t reserved_691; // offset: 691  (0x2B3)
+    uint32_t reserved_692; // offset: 692  (0x2B4)
+    uint32_t reserved_693; // offset: 693  (0x2B5)
+    uint32_t reserved_694; // offset: 694  (0x2B6)
+    uint32_t reserved_695; // offset: 695  (0x2B7)
+    uint32_t reserved_696; // offset: 696  (0x2B8)
+    uint32_t reserved_697; // offset: 697  (0x2B9)
+    uint32_t reserved_698; // offset: 698  (0x2BA)
+    uint32_t reserved_699; // offset: 699  (0x2BB)
+    uint32_t reserved_700; // offset: 700  (0x2BC)
+    uint32_t reserved_701; // offset: 701  (0x2BD)
+    uint32_t reserved_702; // offset: 702  (0x2BE)
+    uint32_t reserved_703; // offset: 703  (0x2BF)
+    uint32_t reserved_704; // offset: 704  (0x2C0)
+    uint32_t reserved_705; // offset: 705  (0x2C1)
+    uint32_t reserved_706; // offset: 706  (0x2C2)
+    uint32_t reserved_707; // offset: 707  (0x2C3)
+    uint32_t reserved_708; // offset: 708  (0x2C4)
+    uint32_t reserved_709; // offset: 709  (0x2C5)
+    uint32_t reserved_710; // offset: 710  (0x2C6)
+    uint32_t reserved_711; // offset: 711  (0x2C7)
+    uint32_t reserved_712; // offset: 712  (0x2C8)
+    uint32_t reserved_713; // offset: 713  (0x2C9)
+    uint32_t reserved_714; // offset: 714  (0x2CA)
+    uint32_t reserved_715; // offset: 715  (0x2CB)
+    uint32_t reserved_716; // offset: 716  (0x2CC)
+    uint32_t reserved_717; // offset: 717  (0x2CD)
+    uint32_t reserved_718; // offset: 718  (0x2CE)
+    uint32_t reserved_719; // offset: 719  (0x2CF)
+    uint32_t reserved_720; // offset: 720  (0x2D0)
+    uint32_t reserved_721; // offset: 721  (0x2D1)
+    uint32_t reserved_722; // offset: 722  (0x2D2)
+    uint32_t reserved_723; // offset: 723  (0x2D3)
+    uint32_t reserved_724; // offset: 724  (0x2D4)
+    uint32_t reserved_725; // offset: 725  (0x2D5)
+    uint32_t reserved_726; // offset: 726  (0x2D6)
+    uint32_t reserved_727; // offset: 727  (0x2D7)
+    uint32_t reserved_728; // offset: 728  (0x2D8)
+    uint32_t reserved_729; // offset: 729  (0x2D9)
+    uint32_t reserved_730; // offset: 730  (0x2DA)
+    uint32_t reserved_731; // offset: 731  (0x2DB)
+    uint32_t reserved_732; // offset: 732  (0x2DC)
+    uint32_t reserved_733; // offset: 733  (0x2DD)
+    uint32_t reserved_734; // offset: 734  (0x2DE)
+    uint32_t reserved_735; // offset: 735  (0x2DF)
+    uint32_t reserved_736; // offset: 736  (0x2E0)
+    uint32_t reserved_737; // offset: 737  (0x2E1)
+    uint32_t reserved_738; // offset: 738  (0x2E2)
+    uint32_t reserved_739; // offset: 739  (0x2E3)
+    uint32_t reserved_740; // offset: 740  (0x2E4)
+    uint32_t reserved_741; // offset: 741  (0x2E5)
+    uint32_t reserved_742; // offset: 742  (0x2E6)
+    uint32_t reserved_743; // offset: 743  (0x2E7)
+    uint32_t reserved_744; // offset: 744  (0x2E8)
+    uint32_t reserved_745; // offset: 745  (0x2E9)
+    uint32_t reserved_746; // offset: 746  (0x2EA)
+    uint32_t reserved_747; // offset: 747  (0x2EB)
+    uint32_t reserved_748; // offset: 748  (0x2EC)
+    uint32_t reserved_749; // offset: 749  (0x2ED)
+    uint32_t reserved_750; // offset: 750  (0x2EE)
+    uint32_t reserved_751; // offset: 751  (0x2EF)
+    uint32_t reserved_752; // offset: 752  (0x2F0)
+    uint32_t reserved_753; // offset: 753  (0x2F1)
+    uint32_t reserved_754; // offset: 754  (0x2F2)
+    uint32_t reserved_755; // offset: 755  (0x2F3)
+    uint32_t reserved_756; // offset: 756  (0x2F4)
+    uint32_t reserved_757; // offset: 757  (0x2F5)
+    uint32_t reserved_758; // offset: 758  (0x2F6)
+    uint32_t reserved_759; // offset: 759  (0x2F7)
+    uint32_t reserved_760; // offset: 760  (0x2F8)
+    uint32_t reserved_761; // offset: 761  (0x2F9)
+    uint32_t reserved_762; // offset: 762  (0x2FA)
+    uint32_t reserved_763; // offset: 763  (0x2FB)
+    uint32_t reserved_764; // offset: 764  (0x2FC)
+    uint32_t reserved_765; // offset: 765  (0x2FD)
+    uint32_t reserved_766; // offset: 766  (0x2FE)
+    uint32_t reserved_767; // offset: 767  (0x2FF)
+    uint32_t reserved_768; // offset: 768  (0x300)
+    uint32_t reserved_769; // offset: 769  (0x301)
+    uint32_t reserved_770; // offset: 770  (0x302)
+    uint32_t reserved_771; // offset: 771  (0x303)
+    uint32_t reserved_772; // offset: 772  (0x304)
+    uint32_t reserved_773; // offset: 773  (0x305)
+    uint32_t reserved_774; // offset: 774  (0x306)
+    uint32_t reserved_775; // offset: 775  (0x307)
+    uint32_t reserved_776; // offset: 776  (0x308)
+    uint32_t reserved_777; // offset: 777  (0x309)
+    uint32_t reserved_778; // offset: 778  (0x30A)
+    uint32_t reserved_779; // offset: 779  (0x30B)
+    uint32_t reserved_780; // offset: 780  (0x30C)
+    uint32_t reserved_781; // offset: 781  (0x30D)
+    uint32_t reserved_782; // offset: 782  (0x30E)
+    uint32_t reserved_783; // offset: 783  (0x30F)
+    uint32_t reserved_784; // offset: 784  (0x310)
+    uint32_t reserved_785; // offset: 785  (0x311)
+    uint32_t reserved_786; // offset: 786  (0x312)
+    uint32_t reserved_787; // offset: 787  (0x313)
+    uint32_t reserved_788; // offset: 788  (0x314)
+    uint32_t reserved_789; // offset: 789  (0x315)
+    uint32_t reserved_790; // offset: 790  (0x316)
+    uint32_t reserved_791; // offset: 791  (0x317)
+    uint32_t reserved_792; // offset: 792  (0x318)
+    uint32_t reserved_793; // offset: 793  (0x319)
+    uint32_t reserved_794; // offset: 794  (0x31A)
+    uint32_t reserved_795; // offset: 795  (0x31B)
+    uint32_t reserved_796; // offset: 796  (0x31C)
+    uint32_t reserved_797; // offset: 797  (0x31D)
+    uint32_t reserved_798; // offset: 798  (0x31E)
+    uint32_t reserved_799; // offset: 799  (0x31F)
+    uint32_t reserved_800; // offset: 800  (0x320)
+    uint32_t reserved_801; // offset: 801  (0x321)
+    uint32_t reserved_802; // offset: 802  (0x322)
+    uint32_t reserved_803; // offset: 803  (0x323)
+    uint32_t reserved_804; // offset: 804  (0x324)
+    uint32_t reserved_805; // offset: 805  (0x325)
+    uint32_t reserved_806; // offset: 806  (0x326)
+    uint32_t reserved_807; // offset: 807  (0x327)
+    uint32_t reserved_808; // offset: 808  (0x328)
+    uint32_t reserved_809; // offset: 809  (0x329)
+    uint32_t reserved_810; // offset: 810  (0x32A)
+    uint32_t reserved_811; // offset: 811  (0x32B)
+    uint32_t reserved_812; // offset: 812  (0x32C)
+    uint32_t reserved_813; // offset: 813  (0x32D)
+    uint32_t reserved_814; // offset: 814  (0x32E)
+    uint32_t reserved_815; // offset: 815  (0x32F)
+    uint32_t reserved_816; // offset: 816  (0x330)
+    uint32_t reserved_817; // offset: 817  (0x331)
+    uint32_t reserved_818; // offset: 818  (0x332)
+    uint32_t reserved_819; // offset: 819  (0x333)
+    uint32_t reserved_820; // offset: 820  (0x334)
+    uint32_t reserved_821; // offset: 821  (0x335)
+    uint32_t reserved_822; // offset: 822  (0x336)
+    uint32_t reserved_823; // offset: 823  (0x337)
+    uint32_t reserved_824; // offset: 824  (0x338)
+    uint32_t reserved_825; // offset: 825  (0x339)
+    uint32_t reserved_826; // offset: 826  (0x33A)
+    uint32_t reserved_827; // offset: 827  (0x33B)
+    uint32_t reserved_828; // offset: 828  (0x33C)
+    uint32_t reserved_829; // offset: 829  (0x33D)
+    uint32_t reserved_830; // offset: 830  (0x33E)
+    uint32_t reserved_831; // offset: 831  (0x33F)
+    uint32_t reserved_832; // offset: 832  (0x340)
+    uint32_t reserved_833; // offset: 833  (0x341)
+    uint32_t reserved_834; // offset: 834  (0x342)
+    uint32_t reserved_835; // offset: 835  (0x343)
+    uint32_t reserved_836; // offset: 836  (0x344)
+    uint32_t reserved_837; // offset: 837  (0x345)
+    uint32_t reserved_838; // offset: 838  (0x346)
+    uint32_t reserved_839; // offset: 839  (0x347)
+    uint32_t reserved_840; // offset: 840  (0x348)
+    uint32_t reserved_841; // offset: 841  (0x349)
+    uint32_t reserved_842; // offset: 842  (0x34A)
+    uint32_t reserved_843; // offset: 843  (0x34B)
+    uint32_t reserved_844; // offset: 844  (0x34C)
+    uint32_t reserved_845; // offset: 845  (0x34D)
+    uint32_t reserved_846; // offset: 846  (0x34E)
+    uint32_t reserved_847; // offset: 847  (0x34F)
+    uint32_t reserved_848; // offset: 848  (0x350)
+    uint32_t reserved_849; // offset: 849  (0x351)
+    uint32_t reserved_850; // offset: 850  (0x352)
+    uint32_t reserved_851; // offset: 851  (0x353)
+    uint32_t reserved_852; // offset: 852  (0x354)
+    uint32_t reserved_853; // offset: 853  (0x355)
+    uint32_t reserved_854; // offset: 854  (0x356)
+    uint32_t reserved_855; // offset: 855  (0x357)
+    uint32_t reserved_856; // offset: 856  (0x358)
+    uint32_t reserved_857; // offset: 857  (0x359)
+    uint32_t reserved_858; // offset: 858  (0x35A)
+    uint32_t reserved_859; // offset: 859  (0x35B)
+    uint32_t reserved_860; // offset: 860  (0x35C)
+    uint32_t reserved_861; // offset: 861  (0x35D)
+    uint32_t reserved_862; // offset: 862  (0x35E)
+    uint32_t reserved_863; // offset: 863  (0x35F)
+    uint32_t reserved_864; // offset: 864  (0x360)
+    uint32_t reserved_865; // offset: 865  (0x361)
+    uint32_t reserved_866; // offset: 866  (0x362)
+    uint32_t reserved_867; // offset: 867  (0x363)
+    uint32_t reserved_868; // offset: 868  (0x364)
+    uint32_t reserved_869; // offset: 869  (0x365)
+    uint32_t reserved_870; // offset: 870  (0x366)
+    uint32_t reserved_871; // offset: 871  (0x367)
+    uint32_t reserved_872; // offset: 872  (0x368)
+    uint32_t reserved_873; // offset: 873  (0x369)
+    uint32_t reserved_874; // offset: 874  (0x36A)
+    uint32_t reserved_875; // offset: 875  (0x36B)
+    uint32_t reserved_876; // offset: 876  (0x36C)
+    uint32_t reserved_877; // offset: 877  (0x36D)
+    uint32_t reserved_878; // offset: 878  (0x36E)
+    uint32_t reserved_879; // offset: 879  (0x36F)
+    uint32_t reserved_880; // offset: 880  (0x370)
+    uint32_t reserved_881; // offset: 881  (0x371)
+    uint32_t reserved_882; // offset: 882  (0x372)
+    uint32_t reserved_883; // offset: 883  (0x373)
+    uint32_t reserved_884; // offset: 884  (0x374)
+    uint32_t reserved_885; // offset: 885  (0x375)
+    uint32_t reserved_886; // offset: 886  (0x376)
+    uint32_t reserved_887; // offset: 887  (0x377)
+    uint32_t reserved_888; // offset: 888  (0x378)
+    uint32_t reserved_889; // offset: 889  (0x379)
+    uint32_t reserved_890; // offset: 890  (0x37A)
+    uint32_t reserved_891; // offset: 891  (0x37B)
+    uint32_t reserved_892; // offset: 892  (0x37C)
+    uint32_t reserved_893; // offset: 893  (0x37D)
+    uint32_t reserved_894; // offset: 894  (0x37E)
+    uint32_t reserved_895; // offset: 895  (0x37F)
+    uint32_t reserved_896; // offset: 896  (0x380)
+    uint32_t reserved_897; // offset: 897  (0x381)
+    uint32_t reserved_898; // offset: 898  (0x382)
+    uint32_t reserved_899; // offset: 899  (0x383)
+    uint32_t reserved_900; // offset: 900  (0x384)
+    uint32_t reserved_901; // offset: 901  (0x385)
+    uint32_t reserved_902; // offset: 902  (0x386)
+    uint32_t reserved_903; // offset: 903  (0x387)
+    uint32_t reserved_904; // offset: 904  (0x388)
+    uint32_t reserved_905; // offset: 905  (0x389)
+    uint32_t reserved_906; // offset: 906  (0x38A)
+    uint32_t reserved_907; // offset: 907  (0x38B)
+    uint32_t reserved_908; // offset: 908  (0x38C)
+    uint32_t reserved_909; // offset: 909  (0x38D)
+    uint32_t reserved_910; // offset: 910  (0x38E)
+    uint32_t reserved_911; // offset: 911  (0x38F)
+    uint32_t reserved_912; // offset: 912  (0x390)
+    uint32_t reserved_913; // offset: 913  (0x391)
+    uint32_t reserved_914; // offset: 914  (0x392)
+    uint32_t reserved_915; // offset: 915  (0x393)
+    uint32_t reserved_916; // offset: 916  (0x394)
+    uint32_t reserved_917; // offset: 917  (0x395)
+    uint32_t reserved_918; // offset: 918  (0x396)
+    uint32_t reserved_919; // offset: 919  (0x397)
+    uint32_t reserved_920; // offset: 920  (0x398)
+    uint32_t reserved_921; // offset: 921  (0x399)
+    uint32_t reserved_922; // offset: 922  (0x39A)
+    uint32_t reserved_923; // offset: 923  (0x39B)
+    uint32_t reserved_924; // offset: 924  (0x39C)
+    uint32_t reserved_925; // offset: 925  (0x39D)
+    uint32_t reserved_926; // offset: 926  (0x39E)
+    uint32_t reserved_927; // offset: 927  (0x39F)
+    uint32_t reserved_928; // offset: 928  (0x3A0)
+    uint32_t reserved_929; // offset: 929  (0x3A1)
+    uint32_t reserved_930; // offset: 930  (0x3A2)
+    uint32_t reserved_931; // offset: 931  (0x3A3)
+    uint32_t reserved_932; // offset: 932  (0x3A4)
+    uint32_t reserved_933; // offset: 933  (0x3A5)
+    uint32_t reserved_934; // offset: 934  (0x3A6)
+    uint32_t reserved_935; // offset: 935  (0x3A7)
+    uint32_t reserved_936; // offset: 936  (0x3A8)
+    uint32_t reserved_937; // offset: 937  (0x3A9)
+    uint32_t reserved_938; // offset: 938  (0x3AA)
+    uint32_t reserved_939; // offset: 939  (0x3AB)
+    uint32_t reserved_940; // offset: 940  (0x3AC)
+    uint32_t reserved_941; // offset: 941  (0x3AD)
+    uint32_t reserved_942; // offset: 942  (0x3AE)
+    uint32_t reserved_943; // offset: 943  (0x3AF)
+    uint32_t reserved_944; // offset: 944  (0x3B0)
+    uint32_t reserved_945; // offset: 945  (0x3B1)
+    uint32_t reserved_946; // offset: 946  (0x3B2)
+    uint32_t reserved_947; // offset: 947  (0x3B3)
+    uint32_t reserved_948; // offset: 948  (0x3B4)
+    uint32_t reserved_949; // offset: 949  (0x3B5)
+    uint32_t reserved_950; // offset: 950  (0x3B6)
+    uint32_t reserved_951; // offset: 951  (0x3B7)
+    uint32_t reserved_952; // offset: 952  (0x3B8)
+    uint32_t reserved_953; // offset: 953  (0x3B9)
+    uint32_t reserved_954; // offset: 954  (0x3BA)
+    uint32_t reserved_955; // offset: 955  (0x3BB)
+    uint32_t reserved_956; // offset: 956  (0x3BC)
+    uint32_t reserved_957; // offset: 957  (0x3BD)
+    uint32_t reserved_958; // offset: 958  (0x3BE)
+    uint32_t reserved_959; // offset: 959  (0x3BF)
+    uint32_t reserved_960; // offset: 960  (0x3C0)
+    uint32_t reserved_961; // offset: 961  (0x3C1)
+    uint32_t reserved_962; // offset: 962  (0x3C2)
+    uint32_t reserved_963; // offset: 963  (0x3C3)
+    uint32_t reserved_964; // offset: 964  (0x3C4)
+    uint32_t reserved_965; // offset: 965  (0x3C5)
+    uint32_t reserved_966; // offset: 966  (0x3C6)
+    uint32_t reserved_967; // offset: 967  (0x3C7)
+    uint32_t reserved_968; // offset: 968  (0x3C8)
+    uint32_t reserved_969; // offset: 969  (0x3C9)
+    uint32_t reserved_970; // offset: 970  (0x3CA)
+    uint32_t reserved_971; // offset: 971  (0x3CB)
+    uint32_t reserved_972; // offset: 972  (0x3CC)
+    uint32_t reserved_973; // offset: 973  (0x3CD)
+    uint32_t reserved_974; // offset: 974  (0x3CE)
+    uint32_t reserved_975; // offset: 975  (0x3CF)
+    uint32_t reserved_976; // offset: 976  (0x3D0)
+    uint32_t reserved_977; // offset: 977  (0x3D1)
+    uint32_t reserved_978; // offset: 978  (0x3D2)
+    uint32_t reserved_979; // offset: 979  (0x3D3)
+    uint32_t reserved_980; // offset: 980  (0x3D4)
+    uint32_t reserved_981; // offset: 981  (0x3D5)
+    uint32_t reserved_982; // offset: 982  (0x3D6)
+    uint32_t reserved_983; // offset: 983  (0x3D7)
+    uint32_t reserved_984; // offset: 984  (0x3D8)
+    uint32_t reserved_985; // offset: 985  (0x3D9)
+    uint32_t reserved_986; // offset: 986  (0x3DA)
+    uint32_t reserved_987; // offset: 987  (0x3DB)
+    uint32_t reserved_988; // offset: 988  (0x3DC)
+    uint32_t reserved_989; // offset: 989  (0x3DD)
+    uint32_t reserved_990; // offset: 990  (0x3DE)
+    uint32_t reserved_991; // offset: 991  (0x3DF)
+    uint32_t reserved_992; // offset: 992  (0x3E0)
+    uint32_t reserved_993; // offset: 993  (0x3E1)
+    uint32_t reserved_994; // offset: 994  (0x3E2)
+    uint32_t reserved_995; // offset: 995  (0x3E3)
+    uint32_t reserved_996; // offset: 996  (0x3E4)
+    uint32_t reserved_997; // offset: 997  (0x3E5)
+    uint32_t reserved_998; // offset: 998  (0x3E6)
+    uint32_t reserved_999; // offset: 999  (0x3E7)
+    uint32_t reserved_1000; // offset: 1000  (0x3E8)
+    uint32_t reserved_1001; // offset: 1001  (0x3E9)
+    uint32_t reserved_1002; // offset: 1002  (0x3EA)
+    uint32_t reserved_1003; // offset: 1003  (0x3EB)
+    uint32_t reserved_1004; // offset: 1004  (0x3EC)
+    uint32_t reserved_1005; // offset: 1005  (0x3ED)
+    uint32_t reserved_1006; // offset: 1006  (0x3EE)
+    uint32_t reserved_1007; // offset: 1007  (0x3EF)
+    uint32_t reserved_1008; // offset: 1008  (0x3F0)
+    uint32_t reserved_1009; // offset: 1009  (0x3F1)
+    uint32_t reserved_1010; // offset: 1010  (0x3F2)
+    uint32_t reserved_1011; // offset: 1011  (0x3F3)
+    uint32_t reserved_1012; // offset: 1012  (0x3F4)
+    uint32_t reserved_1013; // offset: 1013  (0x3F5)
+    uint32_t reserved_1014; // offset: 1014  (0x3F6)
+    uint32_t reserved_1015; // offset: 1015  (0x3F7)
+    uint32_t reserved_1016; // offset: 1016  (0x3F8)
+    uint32_t reserved_1017; // offset: 1017  (0x3F9)
+    uint32_t reserved_1018; // offset: 1018  (0x3FA)
+    uint32_t reserved_1019; // offset: 1019  (0x3FB)
+    uint32_t reserved_1020; // offset: 1020  (0x3FC)
+    uint32_t reserved_1021; // offset: 1021  (0x3FD)
+    uint32_t reserved_1022; // offset: 1022  (0x3FE)
+    uint32_t reserved_1023; // offset: 1023  (0x3FF)
+};
+
+struct v12_1_mes_mqd {
+    uint32_t header; // offset: 0  (0x0)
+    uint32_t compute_dispatch_initiator; // offset: 1  (0x1)
+    uint32_t compute_dim_x; // offset: 2  (0x2)
+    uint32_t compute_dim_y; // offset: 3  (0x3)
+    uint32_t compute_dim_z; // offset: 4  (0x4)
+    uint32_t compute_start_x; // offset: 5  (0x5)
+    uint32_t compute_start_y; // offset: 6  (0x6)
+    uint32_t compute_start_z; // offset: 7  (0x7)
+    uint32_t compute_num_thread_x; // offset: 8  (0x8)
+    uint32_t compute_num_thread_y; // offset: 9  (0x9)
+    uint32_t compute_num_thread_z; // offset: 10  (0xA)
+    uint32_t compute_pipelinestat_enable; // offset: 11  (0xB)
+    uint32_t compute_perfcount_enable; // offset: 12  (0xC)
+    uint32_t compute_pgm_lo; // offset: 13  (0xD)
+    uint32_t compute_pgm_hi; // offset: 14  (0xE)
+    uint32_t compute_dispatch_pkt_addr_lo; // offset: 15  (0xF)
+    uint32_t compute_dispatch_pkt_addr_hi; // offset: 16  (0x10)
+    uint32_t compute_dispatch_scratch_base_lo; // offset: 17  (0x11)
+    uint32_t compute_dispatch_scratch_base_hi; // offset: 18  (0x12)
+    uint32_t compute_pgm_rsrc1; // offset: 19  (0x13)
+    uint32_t compute_pgm_rsrc2; // offset: 20  (0x14)
+    uint32_t compute_vmid; // offset: 21  (0x15)
+    uint32_t compute_resource_limits; // offset: 22  (0x16)
+    uint32_t compute_tmpring_size; // offset: 23  (0x17)
+    uint32_t compute_restart_x; // offset: 24  (0x18)
+    uint32_t compute_restart_y; // offset: 25  (0x19)
+    uint32_t compute_restart_z; // offset: 26  (0x1A)
+    uint32_t compute_thread_trace_enable; // offset: 27  (0x1B)
+    uint32_t compute_misc_reserved; // offset: 28  (0x1C)
+    uint32_t compute_dispatch_id; // offset: 29  (0x1D)
+    uint32_t compute_threadgroup_id; // offset: 30  (0x1E)
+    uint32_t compute_req_ctrl; // offset: 31  (0x1F)
+    uint32_t compute_user_accum_0; // offset: 32  (0x20)
+    uint32_t compute_user_accum_1; // offset: 33  (0x21)
+    uint32_t compute_user_accum_2; // offset: 34  (0x22)
+    uint32_t compute_user_accum_3; // offset: 35  (0x23)
+    uint32_t compute_pgm_rsrc3; // offset: 36  (0x24)
+    uint32_t compute_ddid_index; // offset: 37  (0x25)
+    uint32_t compute_shader_chksum; // offset: 38  (0x26)
+    uint32_t compute_dispatch_interleave; // offset: 39  (0x27)
+    uint32_t compute_current_logical_xcc_id; // offset: 40  (0x28)
+    uint32_t compute_restart_cg_tg_id; // offset: 41  (0x29)
+    uint32_t compute_tg_chunk_size; // offset: 42  (0x2A)
+    uint32_t compute_restore_tg_chunk_size; // offset: 43  (0x2B)
+    uint32_t compute_start_total_lo; // offset: 44  (0x2C)
+    uint32_t compute_start_total_hi; // offset: 45  (0x2D)
+    uint32_t compute_prescaled_dim_x; // offset: 46  (0x2E)
+    uint32_t compute_prescaled_dim_y; // offset: 47  (0x2F)
+    uint32_t compute_prescaled_dim_z; // offset: 48  (0x30)
+    uint32_t compute_static_thread_mgmt_se0; // offset: 49  (0x31)
+    uint32_t compute_static_thread_mgmt_se1; // offset: 50  (0x32)
+    uint32_t compute_static_thread_mgmt_se2; // offset: 51  (0x33)
+    uint32_t compute_static_thread_mgmt_se3; // offset: 52  (0x34)
+    uint32_t compute_static_thread_mgmt_se4; // offset: 53  (0x35)
+    uint32_t compute_static_thread_mgmt_se5; // offset: 54  (0x36)
+    uint32_t compute_static_thread_mgmt_se6; // offset: 55  (0x37)
+    uint32_t compute_static_thread_mgmt_se7; // offset: 56  (0x38)
+    uint32_t compute_static_thread_mgmt_se8; // offset: 57  (0x39)
+    uint32_t reserved_58; // offset: 58  (0x3A)
+    uint32_t reserved_59; // offset: 59  (0x3B)
+    uint32_t reserved_60; // offset: 60  (0x3C)
+    uint32_t reserved_61; // offset: 61  (0x3D)
+    uint32_t reserved_62; // offset: 62  (0x3E)
+    uint32_t reserved_63; // offset: 63  (0x3F)
+    uint32_t reserved_64; // offset: 64  (0x40)
+    uint32_t compute_user_data_0; // offset: 65  (0x41)
+    uint32_t compute_user_data_1; // offset: 66  (0x42)
+    uint32_t compute_user_data_2; // offset: 67  (0x43)
+    uint32_t compute_user_data_3; // offset: 68  (0x44)
+    uint32_t compute_user_data_4; // offset: 69  (0x45)
+    uint32_t compute_user_data_5; // offset: 70  (0x46)
+    uint32_t compute_user_data_6; // offset: 71  (0x47)
+    uint32_t compute_user_data_7; // offset: 72  (0x48)
+    uint32_t compute_user_data_8; // offset: 73  (0x49)
+    uint32_t compute_user_data_9; // offset: 74  (0x4A)
+    uint32_t compute_user_data_10; // offset: 75  (0x4B)
+    uint32_t compute_user_data_11; // offset: 76  (0x4C)
+    uint32_t compute_user_data_12; // offset: 77  (0x4D)
+    uint32_t compute_user_data_13; // offset: 78  (0x4E)
+    uint32_t compute_user_data_14; // offset: 79  (0x4F)
+    uint32_t compute_user_data_15; // offset: 80  (0x50)
+    uint32_t compute_user_data_16; // offset: 81  (0x51)
+    uint32_t compute_user_data_17; // offset: 82  (0x52)
+    uint32_t compute_user_data_18; // offset: 83  (0x53)
+    uint32_t compute_user_data_19; // offset: 84  (0x54)
+    uint32_t compute_user_data_20; // offset: 85  (0x55)
+    uint32_t compute_user_data_21; // offset: 86  (0x56)
+    uint32_t compute_user_data_22; // offset: 87  (0x57)
+    uint32_t compute_user_data_23; // offset: 88  (0x58)
+    uint32_t compute_user_data_24; // offset: 89  (0x59)
+    uint32_t compute_user_data_25; // offset: 90  (0x5A)
+    uint32_t compute_user_data_26; // offset: 91  (0x5B)
+    uint32_t compute_user_data_27; // offset: 92  (0x5C)
+    uint32_t compute_user_data_28; // offset: 93  (0x5D)
+    uint32_t compute_user_data_29; // offset: 94  (0x5E)
+    uint32_t compute_user_data_30; // offset: 95  (0x5F)
+    uint32_t compute_user_data_31; // offset: 96  (0x60)
+    uint32_t reserved_97; // offset: 97  (0x61)
+    uint32_t reserved_98; // offset: 98  (0x62)
+    uint32_t reserved_99; // offset: 99  (0x63)
+    uint32_t reserved_100; // offset: 100  (0x64)
+    uint32_t reserved_101; // offset: 101  (0x65)
+    uint32_t reserved_102; // offset: 102  (0x66)
+    uint32_t reserved_103; // offset: 103  (0x67)
+    uint32_t reserved_104; // offset: 104  (0x68)
+    uint32_t reserved_105; // offset: 105  (0x69)
+    uint32_t reserved_106; // offset: 106  (0x6A)
+    uint32_t reserved_107; // offset: 107  (0x6B)
+    uint32_t reserved_108; // offset: 108  (0x6C)
+    uint32_t reserved_109; // offset: 109  (0x6D)
+    uint32_t compute_relaunch; // offset: 110  (0x6E)
+    uint32_t compute_wave_restore_addr_lo; // offset: 111  (0x6F)
+    uint32_t compute_wave_restore_addr_hi; // offset: 112  (0x70)
+    uint32_t compute_relaunch2; // offset: 113  (0x71)
+    uint32_t cp_pq_exe_status_lo; // offset: 114  (0x72)
+    uint32_t cp_pq_exe_status_hi; // offset: 115  (0x73)
+    uint32_t cp_packet_id_lo; // offset: 116  (0x74)
+    uint32_t cp_packet_id_hi; // offset: 117  (0x75)
+    uint32_t cp_packet_exe_status_lo; // offset: 118  (0x76)
+    uint32_t cp_packet_exe_status_hi; // offset: 119  (0x77)
+    uint32_t reserved_120; // offset: 120  (0x78)
+    uint32_t reserved_121; // offset: 121  (0x79)
+    uint32_t reserved_122; // offset: 122  (0x7A)
+    uint32_t reserved_123; // offset: 123  (0x7B)
+    uint32_t ctx_save_base_addr_lo; // offset: 124  (0x7C)
+    uint32_t ctx_save_base_addr_hi; // offset: 125  (0x7D)
+    uint32_t reserved_126; // offset: 126  (0x7E)
+    uint32_t reserved_127; // offset: 127  (0x7F)
+    uint32_t cp_mqd_base_addr_lo; // offset: 128  (0x80)
+    uint32_t cp_mqd_base_addr_hi; // offset: 129  (0x81)
+    uint32_t cp_hqd_active; // offset: 130  (0x82)
+    uint32_t cp_hqd_vmid; // offset: 131  (0x83)
+    uint32_t cp_hqd_persistent_state; // offset: 132  (0x84)
+    uint32_t cp_hqd_pipe_priority; // offset: 133  (0x85)
+    uint32_t cp_hqd_queue_priority; // offset: 134  (0x86)
+    uint32_t cp_hqd_quantum; // offset: 135  (0x87)
+    uint32_t cp_hqd_pq_base_lo; // offset: 136  (0x88)
+    uint32_t cp_hqd_pq_base_hi; // offset: 137  (0x89)
+    uint32_t cp_hqd_pq_rptr; // offset: 138  (0x8A)
+    uint32_t cp_hqd_pq_rptr_report_addr_lo; // offset: 139  (0x8B)
+    uint32_t cp_hqd_pq_rptr_report_addr_hi; // offset: 140  (0x8C)
+    uint32_t cp_hqd_pq_wptr_poll_addr_lo; // offset: 141  (0x8D)
+    uint32_t cp_hqd_pq_wptr_poll_addr_hi; // offset: 142  (0x8E)
+    uint32_t cp_hqd_pq_doorbell_control; // offset: 143  (0x8F)
+    uint32_t reserved_144; // offset: 144  (0x90)
+    uint32_t cp_hqd_pq_control; // offset: 145  (0x91)
+    uint32_t cp_hqd_ib_base_addr_lo; // offset: 146  (0x92)
+    uint32_t cp_hqd_ib_base_addr_hi; // offset: 147  (0x93)
+    uint32_t cp_hqd_ib_rptr; // offset: 148  (0x94)
+    uint32_t cp_hqd_ib_control; // offset: 149  (0x95)
+    uint32_t cp_hqd_iq_timer; // offset: 150  (0x96)
+    uint32_t cp_hqd_iq_rptr; // offset: 151  (0x97)
+    uint32_t cp_hqd_dequeue_request; // offset: 152  (0x98)
+    uint32_t cp_hqd_dma_offload; // offset: 153  (0x99)
+    uint32_t cp_hqd_sema_cmd; // offset: 154  (0x9A)
+    uint32_t cp_hqd_msg_type; // offset: 155  (0x9B)
+    uint32_t cp_hqd_atomic0_preop_lo; // offset: 156  (0x9C)
+    uint32_t cp_hqd_atomic0_preop_hi; // offset: 157  (0x9D)
+    uint32_t cp_hqd_atomic1_preop_lo; // offset: 158  (0x9E)
+    uint32_t cp_hqd_atomic1_preop_hi; // offset: 159  (0x9F)
+    uint32_t cp_hqd_hq_status0; // offset: 160  (0xA0)
+    uint32_t cp_hqd_hq_control0; // offset: 161  (0xA1)
+    uint32_t cp_mqd_control; // offset: 162  (0xA2)
+    uint32_t cp_hqd_hq_status1; // offset: 163  (0xA3)
+    uint32_t cp_hqd_hq_control1; // offset: 164  (0xA4)
+    uint32_t cp_hqd_eop_base_addr_lo; // offset: 165  (0xA5)
+    uint32_t cp_hqd_eop_base_addr_hi; // offset: 166  (0xA6)
+    uint32_t cp_hqd_eop_control; // offset: 167  (0xA7)
+    uint32_t cp_hqd_eop_rptr; // offset: 168  (0xA8)
+    uint32_t cp_hqd_eop_wptr; // offset: 169  (0xA9)
+    uint32_t cp_hqd_eop_done_events; // offset: 170  (0xAA)
+    uint32_t cp_hqd_ctx_save_base_addr_lo; // offset: 171  (0xAB)
+    uint32_t cp_hqd_ctx_save_base_addr_hi; // offset: 172  (0xAC)
+    uint32_t cp_hqd_ctx_save_control; // offset: 173  (0xAD)
+    uint32_t cp_hqd_cntl_stack_offset; // offset: 174  (0xAE)
+    uint32_t cp_hqd_cntl_stack_size; // offset: 175  (0xAF)
+    uint32_t cp_hqd_wg_state_offset; // offset: 176  (0xB0)
+    uint32_t cp_hqd_ctx_save_size; // offset: 177  (0xB1)
+    uint32_t reserved_178; // offset: 178  (0xB2)
+    uint32_t cp_hqd_error; // offset: 179  (0xB3)
+    uint32_t cp_hqd_eop_wptr_mem; // offset: 180  (0xB4)
+    uint32_t cp_hqd_aql_control; // offset: 181  (0xB5)
+    uint32_t cp_hqd_pq_wptr_lo; // offset: 182  (0xB6)
+    uint32_t cp_hqd_pq_wptr_hi; // offset: 183  (0xB7)
+    uint32_t cp_hqd_gfx_control; // offset: 184  (0xB8)
+    uint32_t cp_hqd_suspend_cntl_stack_dw_cnt; // offset: 185  (0xB9)
+    uint32_t cp_hqd_suspend_wg_state_offset; // offset: 186  (0xBA)
+    uint32_t cp_hqd_ddid_rptr; // offset: 187  (0xBB)
+    uint32_t cp_hqd_ddid_wptr; // offset: 188  (0xBC)
+    uint32_t cp_hqd_ddid_inflight_count; // offset: 189  (0xBD)
+    uint32_t cp_hqd_ddid_delta_rpt_count; // offset: 190  (0xBE)
+    uint32_t cp_hqd_dequeue_status; // offset: 191  (0xBF)
+    uint32_t cp_hqd_aql_control_1; // offset: 192  (0xC0)
+    uint32_t cp_hqd_aql_dispatch_id; // offset: 193  (0xC1)
+    uint32_t cp_hqd_aql_dispatch_id_hi; // offset: 194  (0xC2)
+    uint32_t cp_hqd_kd_base; // offset: 195  (0xC3)
+    uint32_t cp_hqd_kd_base_hi; // offset: 196  (0xC4)
+    uint32_t cp_hqd_kd_cntl; // offset: 197  (0xC5)
+    uint32_t reserved_198; // offset: 198  (0xC6)
+    uint32_t reserved_199; // offset: 199  (0xC7)
+    uint32_t reserved_200; // offset: 200  (0xC8)
+    uint32_t reserved_201; // offset: 201  (0xC9)
+    uint32_t reserved_202; // offset: 202  (0xCA)
+    uint32_t reserved_203; // offset: 203  (0xCB)
+    uint32_t reserved_204; // offset: 204  (0xCC)
+    uint32_t reserved_205; // offset: 205  (0xCD)
+    uint32_t reserved_206; // offset: 206  (0xCE)
+    uint32_t reserved_207; // offset: 207  (0xCF)
+    uint32_t reserved_208; // offset: 208  (0xD0)
+    uint32_t reserved_209; // offset: 209  (0xD1)
+    uint32_t reserved_210; // offset: 210  (0xD2)
+    uint32_t reserved_211; // offset: 211  (0xD3)
+    uint32_t reserved_212; // offset: 212  (0xD4)
+    uint32_t reserved_213; // offset: 213  (0xD5)
+    uint32_t reserved_214; // offset: 214  (0xD6)
+    uint32_t reserved_215; // offset: 215  (0xD7)
+    uint32_t reserved_216; // offset: 216  (0xD8)
+    uint32_t reserved_217; // offset: 217  (0xD9)
+    uint32_t reserved_218; // offset: 218  (0xDA)
+    uint32_t reserved_219; // offset: 219  (0xDB)
+    uint32_t reserved_220; // offset: 220  (0xDC)
+    uint32_t reserved_221; // offset: 221  (0xDD)
+    uint32_t reserved_222; // offset: 222  (0xDE)
+    uint32_t reserved_223; // offset: 223  (0xDF)
+    uint32_t reserved_224; // offset: 224  (0xE0)
+    uint32_t pm4_target_xcc_in_xcp; // offset: 225  (0xE1)
+    uint32_t cp_mqd_stride_size; // offset: 226  (0xE2)
+    uint32_t xcc_sync_counter; // offset: 227  (0xE3)
+    uint32_t set_resources_header; // offset: 228  (0xE4)
+    uint32_t set_resources_dw1; // offset: 229  (0xE5)
+    uint32_t set_resources_dw2; // offset: 230  (0xE6)
+    uint32_t set_resources_dw3; // offset: 231  (0xE7)
+    uint32_t set_resources_dw4; // offset: 232  (0xE8)
+    uint32_t set_resources_dw5; // offset: 233  (0xE9)
+    uint32_t set_resources_dw6; // offset: 234  (0xEA)
+    uint32_t set_resources_dw7; // offset: 235  (0xEB)
+    uint32_t reserved_236; // offset: 236  (0xEC)
+    uint32_t reserved_237; // offset: 237  (0xED)
+    uint32_t reserved_238; // offset: 238  (0xEE)
+    uint32_t reserved_239; // offset: 239  (0xEF)
+    uint32_t reserved_240; // offset: 240  (0xF0)
+    uint32_t reserved_241; // offset: 241  (0xF1)
+    uint32_t reserved_242; // offset: 242  (0xF2)
+    uint32_t reserved_243; // offset: 243  (0xF3)
+    uint32_t reserved_244; // offset: 244  (0xF4)
+    uint32_t reserved_245; // offset: 245  (0xF5)
+    uint32_t reserved_246; // offset: 246  (0xF6)
+    uint32_t reserved_247; // offset: 247  (0xF7)
+    uint32_t reserved_248; // offset: 248  (0xF8)
+    uint32_t reserved_249; // offset: 249  (0xF9)
+    uint32_t reserved_250; // offset: 250  (0xFA)
+    uint32_t reserved_251; // offset: 251  (0xFB)
+    uint32_t reserved_252; // offset: 252  (0xFC)
+    uint32_t reserved_253; // offset: 253  (0xFD)
+    uint32_t reserved_254; // offset: 254  (0xFE)
+    uint32_t reserved_255; // offset: 255  (0xFF)
+    uint32_t control_buf_addr_lo; // offset: 256  (0x100)
+    uint32_t control_buf_addr_hi; // offset: 257  (0x101)
+    uint32_t control_buf_wptr_lo; // offset: 258  (0x102)
+    uint32_t control_buf_wptr_hi; // offset: 259  (0x103)
+    uint32_t control_buf_dptr_lo; // offset: 260  (0x104)
+    uint32_t control_buf_dptr_hi; // offset: 261  (0x105)
+    uint32_t draw_ring_addr_lo; // offset: 262  (0x106)
+    uint32_t draw_ring_addr_hi; // offset: 263  (0x107)
+    uint32_t control_buf_num_entries; // offset: 264  (0x108)
+    uint32_t reserved_265; // offset: 265  (0x109)
+    uint32_t reserved_266; // offset: 266  (0x10A)
+    uint32_t reserved_267; // offset: 267  (0x10B)
+    uint32_t reserved_268; // offset: 268  (0x10C)
+    uint32_t reserved_269; // offset: 269  (0x10D)
+    uint32_t reserved_270; // offset: 270  (0x10E)
+    uint32_t reserved_271; // offset: 271  (0x10F)
+    uint32_t dfwx_flags; // offset: 272  (0x110)
+    uint32_t dfwx_slot; // offset: 273  (0x111)
+    uint32_t dfwx_client_data_addr_lo; // offset: 274  (0x112)
+    uint32_t dfwx_client_data_addr_hi; // offset: 275  (0x113)
+    uint32_t dfwx_queue_connect_addr_lo; // offset: 276  (0x114)
+    uint32_t dfwx_queue_connect_addr_hi; // offset: 277  (0x115)
+    uint32_t reserved_278; // offset: 278  (0x116)
+    uint32_t reserved_279; // offset: 279  (0x117)
+    uint32_t reserved_280; // offset: 280  (0x118)
+    uint32_t reserved_281; // offset: 281  (0x119)
+    uint32_t reserved_282; // offset: 282  (0x11A)
+    uint32_t reserved_283; // offset: 283  (0x11B)
+    uint32_t reserved_284; // offset: 284  (0x11C)
+    uint32_t reserved_285; // offset: 285  (0x11D)
+    uint32_t reserved_286; // offset: 286  (0x11E)
+    uint32_t reserved_287; // offset: 287  (0x11F)
+    uint32_t cp_mqd_query_time_lo; // offset: 288  (0x120)
+    uint32_t cp_mqd_query_time_hi; // offset: 289  (0x121)
+    uint32_t cp_mqd_connect_start_time_lo; // offset: 290  (0x122)
+    uint32_t cp_mqd_connect_start_time_hi; // offset: 291  (0x123)
+    uint32_t cp_mqd_connect_end_time_lo; // offset: 292  (0x124)
+    uint32_t cp_mqd_connect_end_time_hi; // offset: 293  (0x125)
+    uint32_t cp_mqd_connect_end_wf_count; // offset: 294  (0x126)
+    uint32_t cp_mqd_connect_end_pq_rptr; // offset: 295  (0x127)
+    uint32_t cp_mqd_connect_end_pq_wptr; // offset: 296  (0x128)
+    uint32_t cp_mqd_connect_end_ib_rptr; // offset: 297  (0x129)
+    uint32_t cp_mqd_readindex_lo; // offset: 298  (0x12A)
+    uint32_t cp_mqd_readindex_hi; // offset: 299  (0x12B)
+    uint32_t cp_mqd_save_start_time_lo; // offset: 300  (0x12C)
+    uint32_t cp_mqd_save_start_time_hi; // offset: 301  (0x12D)
+    uint32_t cp_mqd_save_end_time_lo; // offset: 302  (0x12E)
+    uint32_t cp_mqd_save_end_time_hi; // offset: 303  (0x12F)
+    uint32_t cp_mqd_restore_start_time_lo; // offset: 304  (0x130)
+    uint32_t cp_mqd_restore_start_time_hi; // offset: 305  (0x131)
+    uint32_t cp_mqd_restore_end_time_lo; // offset: 306  (0x132)
+    uint32_t cp_mqd_restore_end_time_hi; // offset: 307  (0x133)
+    uint32_t disable_queue; // offset: 308  (0x134)
+    uint32_t reserved_309; // offset: 309  (0x135)
+    uint32_t reserved_310; // offset: 310  (0x136)
+    uint32_t reserved_311; // offset: 311  (0x137)
+    uint32_t reserved_312; // offset: 312  (0x138)
+    uint32_t reserved_313; // offset: 313  (0x139)
+    uint32_t reserved_314; // offset: 314  (0x13A)
+    uint32_t reserved_315; // offset: 315  (0x13B)
+    uint32_t reserved_316; // offset: 316  (0x13C)
+    uint32_t reserved_317; // offset: 317  (0x13D)
+    uint32_t tg_counter_id; // offset: 318  (0x13E)
+    uint32_t cp_compute_csinvoc_count_lo; // offset: 319  (0x13F)
+    uint32_t cp_compute_csinvoc_count_hi; // offset: 320  (0x140)
+    uint32_t reserved_321; // offset: 321  (0x141)
+    uint32_t reserved_322; // offset: 322  (0x142)
+    uint32_t reserved_323; // offset: 323  (0x143)
+    uint32_t reserved_324; // offset: 324  (0x144)
+    uint32_t reserved_325; // offset: 325  (0x145)
+    uint32_t reserved_326; // offset: 326  (0x146)
+    uint32_t reserved_327; // offset: 327  (0x147)
+    uint32_t reserved_328; // offset: 328  (0x148)
+    uint32_t reserved_329; // offset: 329  (0x149)
+    uint32_t reserved_330; // offset: 330  (0x14A)
+    uint32_t reserved_331; // offset: 331  (0x14B)
+    uint32_t reserved_332; // offset: 332  (0x14C)
+    uint32_t reserved_333; // offset: 333  (0x14D)
+    uint32_t reserved_334; // offset: 334  (0x14E)
+    uint32_t reserved_335; // offset: 335  (0x14F)
+    uint32_t reserved_336; // offset: 336  (0x150)
+    uint32_t reserved_337; // offset: 337  (0x151)
+    uint32_t reserved_338; // offset: 338  (0x152)
+    uint32_t reserved_339; // offset: 339  (0x153)
+    uint32_t reserved_340; // offset: 340  (0x154)
+    uint32_t reserved_341; // offset: 341  (0x155)
+    uint32_t reserved_342; // offset: 342  (0x156)
+    uint32_t reserved_343; // offset: 343  (0x157)
+    uint32_t reserved_344; // offset: 344  (0x158)
+    uint32_t reserved_345; // offset: 345  (0x159)
+    uint32_t reserved_346; // offset: 346  (0x15A)
+    uint32_t reserved_347; // offset: 347  (0x15B)
+    uint32_t reserved_348; // offset: 348  (0x15C)
+    uint32_t reserved_349; // offset: 349  (0x15D)
+    uint32_t reserved_350; // offset: 350  (0x15E)
+    uint32_t reserved_351; // offset: 351  (0x15F)
+    uint32_t reserved_352; // offset: 352  (0x160)
+    uint32_t reserved_353; // offset: 353  (0x161)
+    uint32_t reserved_354; // offset: 354  (0x162)
+    uint32_t reserved_355; // offset: 355  (0x163)
+    uint32_t reserved_356; // offset: 356  (0x164)
+    uint32_t reserved_357; // offset: 357  (0x165)
+    uint32_t reserved_358; // offset: 358  (0x166)
+    uint32_t reserved_359; // offset: 359  (0x167)
+    uint32_t reserved_360; // offset: 360  (0x168)
+    uint32_t reserved_361; // offset: 361  (0x169)
+    uint32_t reserved_362; // offset: 362  (0x16A)
+    uint32_t reserved_363; // offset: 363  (0x16B)
+    uint32_t reserved_364; // offset: 364  (0x16C)
+    uint32_t reserved_365; // offset: 365  (0x16D)
+    uint32_t reserved_366; // offset: 366  (0x16E)
+    uint32_t reserved_367; // offset: 367  (0x16F)
+    uint32_t reserved_368; // offset: 368  (0x170)
+    uint32_t reserved_369; // offset: 369  (0x171)
+    uint32_t reserved_370; // offset: 370  (0x172)
+    uint32_t reserved_371; // offset: 371  (0x173)
+    uint32_t reserved_372; // offset: 372  (0x174)
+    uint32_t reserved_373; // offset: 373  (0x175)
+    uint32_t reserved_374; // offset: 374  (0x176)
+    uint32_t reserved_375; // offset: 375  (0x177)
+    uint32_t reserved_376; // offset: 376  (0x178)
+    uint32_t reserved_377; // offset: 377  (0x179)
+    uint32_t reserved_378; // offset: 378  (0x17A)
+    uint32_t reserved_379; // offset: 379  (0x17B)
+    uint32_t reserved_380; // offset: 380  (0x17C)
+    uint32_t reserved_381; // offset: 381  (0x17D)
+    uint32_t reserved_382; // offset: 382  (0x17E)
+    uint32_t reserved_383; // offset: 383  (0x17F)
+    uint32_t iqtimer_pkt_header; // offset: 384  (0x180)
+    uint32_t iqtimer_pkt_dw0; // offset: 385  (0x181)
+    uint32_t iqtimer_pkt_dw1; // offset: 386  (0x182)
+    uint32_t iqtimer_pkt_dw2; // offset: 387  (0x183)
+    uint32_t iqtimer_pkt_dw3; // offset: 388  (0x184)
+    uint32_t iqtimer_pkt_dw4; // offset: 389  (0x185)
+    uint32_t iqtimer_pkt_dw5; // offset: 390  (0x186)
+    uint32_t iqtimer_pkt_dw6; // offset: 391  (0x187)
+    uint32_t iqtimer_pkt_dw7; // offset: 392  (0x188)
+    uint32_t iqtimer_pkt_dw8; // offset: 393  (0x189)
+    uint32_t iqtimer_pkt_dw9; // offset: 394  (0x18A)
+    uint32_t iqtimer_pkt_dw10; // offset: 395  (0x18B)
+    uint32_t iqtimer_pkt_dw11; // offset: 396  (0x18C)
+    uint32_t iqtimer_pkt_dw12; // offset: 397  (0x18D)
+    uint32_t iqtimer_pkt_dw13; // offset: 398  (0x18E)
+    uint32_t iqtimer_pkt_dw14; // offset: 399  (0x18F)
+    uint32_t iqtimer_pkt_dw15; // offset: 400  (0x190)
+    uint32_t iqtimer_pkt_dw16; // offset: 401  (0x191)
+    uint32_t iqtimer_pkt_dw17; // offset: 402  (0x192)
+    uint32_t iqtimer_pkt_dw18; // offset: 403  (0x193)
+    uint32_t iqtimer_pkt_dw19; // offset: 404  (0x194)
+    uint32_t iqtimer_pkt_dw20; // offset: 405  (0x195)
+    uint32_t iqtimer_pkt_dw21; // offset: 406  (0x196)
+    uint32_t iqtimer_pkt_dw22; // offset: 407  (0x197)
+    uint32_t iqtimer_pkt_dw23; // offset: 408  (0x198)
+    uint32_t iqtimer_pkt_dw24; // offset: 409  (0x199)
+    uint32_t iqtimer_pkt_dw25; // offset: 410  (0x19A)
+    uint32_t iqtimer_pkt_dw26; // offset: 411  (0x19B)
+    uint32_t iqtimer_pkt_dw27; // offset: 412  (0x19C)
+    uint32_t iqtimer_pkt_dw28; // offset: 413  (0x19D)
+    uint32_t iqtimer_pkt_dw29; // offset: 414  (0x19E)
+    uint32_t iqtimer_pkt_dw30; // offset: 415  (0x19F)
+    uint32_t iqtimer_pkt_dw31; // offset: 416  (0x1A0)
+    uint32_t reserved_417; // offset: 417  (0x1A1)
+    uint32_t reserved_418; // offset: 418  (0x1A2)
+    uint32_t reserved_419; // offset: 419  (0x1A3)
+    uint32_t reserved_420; // offset: 420  (0x1A4)
+    uint32_t reserved_421; // offset: 421  (0x1A5)
+    uint32_t reserved_422; // offset: 422  (0x1A6)
+    uint32_t reserved_423; // offset: 423  (0x1A7)
+    uint32_t reserved_424; // offset: 424  (0x1A8)
+    uint32_t reserved_425; // offset: 425  (0x1A9)
+    uint32_t reserved_426; // offset: 426  (0x1AA)
+    uint32_t reserved_427; // offset: 427  (0x1AB)
+    uint32_t reserved_428; // offset: 428  (0x1AC)
+    uint32_t reserved_429; // offset: 429  (0x1AD)
+    uint32_t reserved_430; // offset: 430  (0x1AE)
+    uint32_t reserved_431; // offset: 431  (0x1AF)
+    uint32_t reserved_432; // offset: 432  (0x1B0)
+    uint32_t reserved_433; // offset: 433  (0x1B1)
+    uint32_t reserved_434; // offset: 434  (0x1B2)
+    uint32_t reserved_435; // offset: 435  (0x1B3)
+    uint32_t reserved_436; // offset: 436  (0x1B4)
+    uint32_t reserved_437; // offset: 437  (0x1B5)
+    uint32_t reserved_438; // offset: 438  (0x1B6)
+    uint32_t reserved_439; // offset: 439  (0x1B7)
+    uint32_t reserved_440; // offset: 440  (0x1B8)
+    uint32_t reserved_441; // offset: 441  (0x1B9)
+    uint32_t reserved_442; // offset: 442  (0x1BA)
+    uint32_t reserved_443; // offset: 443  (0x1BB)
+    uint32_t reserved_444; // offset: 444  (0x1BC)
+    uint32_t reserved_445; // offset: 445  (0x1BD)
+    uint32_t reserved_446; // offset: 446  (0x1BE)
+    uint32_t reserved_447; // offset: 447  (0x1BF)
+    uint32_t reserved_448; // offset: 448  (0x1C0)
+    uint32_t reserved_449; // offset: 449  (0x1C1)
+    uint32_t reserved_450; // offset: 450  (0x1C2)
+    uint32_t reserved_451; // offset: 451  (0x1C3)
+    uint32_t reserved_452; // offset: 452  (0x1C4)
+    uint32_t reserved_453; // offset: 453  (0x1C5)
+    uint32_t reserved_454; // offset: 454  (0x1C6)
+    uint32_t reserved_455; // offset: 455  (0x1C7)
+    uint32_t reserved_456; // offset: 456  (0x1C8)
+    uint32_t reserved_457; // offset: 457  (0x1C9)
+    uint32_t reserved_458; // offset: 458  (0x1CA)
+    uint32_t reserved_459; // offset: 459  (0x1CB)
+    uint32_t reserved_460; // offset: 460  (0x1CC)
+    uint32_t reserved_461; // offset: 461  (0x1CD)
+    uint32_t reserved_462; // offset: 462  (0x1CE)
+    uint32_t reserved_463; // offset: 463  (0x1CF)
+    uint32_t reserved_464; // offset: 464  (0x1D0)
+    uint32_t reserved_465; // offset: 465  (0x1D1)
+    uint32_t reserved_466; // offset: 466  (0x1D2)
+    uint32_t reserved_467; // offset: 467  (0x1D3)
+    uint32_t reserved_468; // offset: 468  (0x1D4)
+    uint32_t reserved_469; // offset: 469  (0x1D5)
+    uint32_t reserved_470; // offset: 470  (0x1D6)
+    uint32_t reserved_471; // offset: 471  (0x1D7)
+    uint32_t reserved_472; // offset: 472  (0x1D8)
+    uint32_t reserved_473; // offset: 473  (0x1D9)
+    uint32_t reserved_474; // offset: 474  (0x1DA)
+    uint32_t reserved_475; // offset: 475  (0x1DB)
+    uint32_t reserved_476; // offset: 476  (0x1DC)
+    uint32_t reserved_477; // offset: 477  (0x1DD)
+    uint32_t reserved_478; // offset: 478  (0x1DE)
+    uint32_t reserved_479; // offset: 479  (0x1DF)
+    uint32_t reserved_480; // offset: 480  (0x1E0)
+    uint32_t reserved_481; // offset: 481  (0x1E1)
+    uint32_t reserved_482; // offset: 482  (0x1E2)
+    uint32_t reserved_483; // offset: 483  (0x1E3)
+    uint32_t reserved_484; // offset: 484  (0x1E4)
+    uint32_t reserved_485; // offset: 485  (0x1E5)
+    uint32_t reserved_486; // offset: 486  (0x1E6)
+    uint32_t reserved_487; // offset: 487  (0x1E7)
+    uint32_t reserved_488; // offset: 488  (0x1E8)
+    uint32_t reserved_489; // offset: 489  (0x1E9)
+    uint32_t reserved_490; // offset: 490  (0x1EA)
+    uint32_t reserved_491; // offset: 491  (0x1EB)
+    uint32_t reserved_492; // offset: 492  (0x1EC)
+    uint32_t reserved_493; // offset: 493  (0x1ED)
+    uint32_t reserved_494; // offset: 494  (0x1EE)
+    uint32_t reserved_495; // offset: 495  (0x1EF)
+    uint32_t reserved_496; // offset: 496  (0x1F0)
+    uint32_t reserved_497; // offset: 497  (0x1F1)
+    uint32_t reserved_498; // offset: 498  (0x1F2)
+    uint32_t reserved_499; // offset: 499  (0x1F3)
+    uint32_t reserved_500; // offset: 500  (0x1F4)
+    uint32_t reserved_501; // offset: 501  (0x1F5)
+    uint32_t reserved_502; // offset: 502  (0x1F6)
+    uint32_t reserved_503; // offset: 503  (0x1F7)
+    uint32_t reserved_504; // offset: 504  (0x1F8)
+    uint32_t reserved_505; // offset: 505  (0x1F9)
+    uint32_t reserved_506; // offset: 506  (0x1FA)
+    uint32_t reserved_507; // offset: 507  (0x1FB)
+    uint32_t reserved_508; // offset: 508  (0x1FC)
+    uint32_t reserved_509; // offset: 509  (0x1FD)
+    uint32_t reserved_510; // offset: 510  (0x1FE)
+    uint32_t reserved_511; // offset: 511  (0x1FF)
+    uint32_t reserved_512; // offset: 512  (0x200)
+    uint32_t reserved_513; // offset: 513  (0x201)
+    uint32_t reserved_514; // offset: 514  (0x202)
+    uint32_t reserved_515; // offset: 515  (0x203)
+    uint32_t reserved_516; // offset: 516  (0x204)
+    uint32_t reserved_517; // offset: 517  (0x205)
+    uint32_t reserved_518; // offset: 518  (0x206)
+    uint32_t reserved_519; // offset: 519  (0x207)
+    uint32_t reserved_520; // offset: 520  (0x208)
+    uint32_t reserved_521; // offset: 521  (0x209)
+    uint32_t reserved_522; // offset: 522  (0x20A)
+    uint32_t reserved_523; // offset: 523  (0x20B)
+    uint32_t reserved_524; // offset: 524  (0x20C)
+    uint32_t reserved_525; // offset: 525  (0x20D)
+    uint32_t reserved_526; // offset: 526  (0x20E)
+    uint32_t reserved_527; // offset: 527  (0x20F)
+    uint32_t reserved_528; // offset: 528  (0x210)
+    uint32_t reserved_529; // offset: 529  (0x211)
+    uint32_t reserved_530; // offset: 530  (0x212)
+    uint32_t reserved_531; // offset: 531  (0x213)
+    uint32_t reserved_532; // offset: 532  (0x214)
+    uint32_t reserved_533; // offset: 533  (0x215)
+    uint32_t reserved_534; // offset: 534  (0x216)
+    uint32_t reserved_535; // offset: 535  (0x217)
+    uint32_t reserved_536; // offset: 536  (0x218)
+    uint32_t reserved_537; // offset: 537  (0x219)
+    uint32_t reserved_538; // offset: 538  (0x21A)
+    uint32_t reserved_539; // offset: 539  (0x21B)
+    uint32_t reserved_540; // offset: 540  (0x21C)
+    uint32_t reserved_541; // offset: 541  (0x21D)
+    uint32_t reserved_542; // offset: 542  (0x21E)
+    uint32_t reserved_543; // offset: 543  (0x21F)
+    uint32_t reserved_544; // offset: 544  (0x220)
+    uint32_t reserved_545; // offset: 545  (0x221)
+    uint32_t reserved_546; // offset: 546  (0x222)
+    uint32_t reserved_547; // offset: 547  (0x223)
+    uint32_t reserved_548; // offset: 548  (0x224)
+    uint32_t reserved_549; // offset: 549  (0x225)
+    uint32_t reserved_550; // offset: 550  (0x226)
+    uint32_t reserved_551; // offset: 551  (0x227)
+    uint32_t reserved_552; // offset: 552  (0x228)
+    uint32_t reserved_553; // offset: 553  (0x229)
+    uint32_t reserved_554; // offset: 554  (0x22A)
+    uint32_t reserved_555; // offset: 555  (0x22B)
+    uint32_t reserved_556; // offset: 556  (0x22C)
+    uint32_t reserved_557; // offset: 557  (0x22D)
+    uint32_t reserved_558; // offset: 558  (0x22E)
+    uint32_t reserved_559; // offset: 559  (0x22F)
+    uint32_t reserved_560; // offset: 560  (0x230)
+    uint32_t reserved_561; // offset: 561  (0x231)
+    uint32_t reserved_562; // offset: 562  (0x232)
+    uint32_t reserved_563; // offset: 563  (0x233)
+    uint32_t reserved_564; // offset: 564  (0x234)
+    uint32_t reserved_565; // offset: 565  (0x235)
+    uint32_t reserved_566; // offset: 566  (0x236)
+    uint32_t reserved_567; // offset: 567  (0x237)
+    uint32_t reserved_568; // offset: 568  (0x238)
+    uint32_t reserved_569; // offset: 569  (0x239)
+    uint32_t reserved_570; // offset: 570  (0x23A)
+    uint32_t reserved_571; // offset: 571  (0x23B)
+    uint32_t reserved_572; // offset: 572  (0x23C)
+    uint32_t reserved_573; // offset: 573  (0x23D)
+    uint32_t reserved_574; // offset: 574  (0x23E)
+    uint32_t reserved_575; // offset: 575  (0x23F)
+    uint32_t reserved_576; // offset: 576  (0x240)
+    uint32_t reserved_577; // offset: 577  (0x241)
+    uint32_t reserved_578; // offset: 578  (0x242)
+    uint32_t reserved_579; // offset: 579  (0x243)
+    uint32_t reserved_580; // offset: 580  (0x244)
+    uint32_t reserved_581; // offset: 581  (0x245)
+    uint32_t reserved_582; // offset: 582  (0x246)
+    uint32_t reserved_583; // offset: 583  (0x247)
+    uint32_t reserved_584; // offset: 584  (0x248)
+    uint32_t reserved_585; // offset: 585  (0x249)
+    uint32_t reserved_586; // offset: 586  (0x24A)
+    uint32_t reserved_587; // offset: 587  (0x24B)
+    uint32_t reserved_588; // offset: 588  (0x24C)
+    uint32_t reserved_589; // offset: 589  (0x24D)
+    uint32_t reserved_590; // offset: 590  (0x24E)
+    uint32_t reserved_591; // offset: 591  (0x24F)
+    uint32_t reserved_592; // offset: 592  (0x250)
+    uint32_t reserved_593; // offset: 593  (0x251)
+    uint32_t reserved_594; // offset: 594  (0x252)
+    uint32_t reserved_595; // offset: 595  (0x253)
+    uint32_t reserved_596; // offset: 596  (0x254)
+    uint32_t reserved_597; // offset: 597  (0x255)
+    uint32_t reserved_598; // offset: 598  (0x256)
+    uint32_t reserved_599; // offset: 599  (0x257)
+    uint32_t reserved_600; // offset: 600  (0x258)
+    uint32_t reserved_601; // offset: 601  (0x259)
+    uint32_t reserved_602; // offset: 602  (0x25A)
+    uint32_t reserved_603; // offset: 603  (0x25B)
+    uint32_t reserved_604; // offset: 604  (0x25C)
+    uint32_t reserved_605; // offset: 605  (0x25D)
+    uint32_t reserved_606; // offset: 606  (0x25E)
+    uint32_t reserved_607; // offset: 607  (0x25F)
+    uint32_t reserved_608; // offset: 608  (0x260)
+    uint32_t reserved_609; // offset: 609  (0x261)
+    uint32_t reserved_610; // offset: 610  (0x262)
+    uint32_t reserved_611; // offset: 611  (0x263)
+    uint32_t reserved_612; // offset: 612  (0x264)
+    uint32_t reserved_613; // offset: 613  (0x265)
+    uint32_t reserved_614; // offset: 614  (0x266)
+    uint32_t reserved_615; // offset: 615  (0x267)
+    uint32_t reserved_616; // offset: 616  (0x268)
+    uint32_t reserved_617; // offset: 617  (0x269)
+    uint32_t reserved_618; // offset: 618  (0x26A)
+    uint32_t reserved_619; // offset: 619  (0x26B)
+    uint32_t reserved_620; // offset: 620  (0x26C)
+    uint32_t reserved_621; // offset: 621  (0x26D)
+    uint32_t reserved_622; // offset: 622  (0x26E)
+    uint32_t reserved_623; // offset: 623  (0x26F)
+    uint32_t reserved_624; // offset: 624  (0x270)
+    uint32_t reserved_625; // offset: 625  (0x271)
+    uint32_t reserved_626; // offset: 626  (0x272)
+    uint32_t reserved_627; // offset: 627  (0x273)
+    uint32_t reserved_628; // offset: 628  (0x274)
+    uint32_t reserved_629; // offset: 629  (0x275)
+    uint32_t reserved_630; // offset: 630  (0x276)
+    uint32_t reserved_631; // offset: 631  (0x277)
+    uint32_t reserved_632; // offset: 632  (0x278)
+    uint32_t reserved_633; // offset: 633  (0x279)
+    uint32_t reserved_634; // offset: 634  (0x27A)
+    uint32_t reserved_635; // offset: 635  (0x27B)
+    uint32_t reserved_636; // offset: 636  (0x27C)
+    uint32_t reserved_637; // offset: 637  (0x27D)
+    uint32_t reserved_638; // offset: 638  (0x27E)
+    uint32_t reserved_639; // offset: 639  (0x27F)
+    uint32_t reserved_640; // offset: 640  (0x280)
+    uint32_t reserved_641; // offset: 641  (0x281)
+    uint32_t reserved_642; // offset: 642  (0x282)
+    uint32_t reserved_643; // offset: 643  (0x283)
+    uint32_t reserved_644; // offset: 644  (0x284)
+    uint32_t reserved_645; // offset: 645  (0x285)
+    uint32_t reserved_646; // offset: 646  (0x286)
+    uint32_t reserved_647; // offset: 647  (0x287)
+    uint32_t reserved_648; // offset: 648  (0x288)
+    uint32_t reserved_649; // offset: 649  (0x289)
+    uint32_t reserved_650; // offset: 650  (0x28A)
+    uint32_t reserved_651; // offset: 651  (0x28B)
+    uint32_t reserved_652; // offset: 652  (0x28C)
+    uint32_t reserved_653; // offset: 653  (0x28D)
+    uint32_t reserved_654; // offset: 654  (0x28E)
+    uint32_t reserved_655; // offset: 655  (0x28F)
+    uint32_t reserved_656; // offset: 656  (0x290)
+    uint32_t reserved_657; // offset: 657  (0x291)
+    uint32_t reserved_658; // offset: 658  (0x292)
+    uint32_t reserved_659; // offset: 659  (0x293)
+    uint32_t reserved_660; // offset: 660  (0x294)
+    uint32_t reserved_661; // offset: 661  (0x295)
+    uint32_t reserved_662; // offset: 662  (0x296)
+    uint32_t reserved_663; // offset: 663  (0x297)
+    uint32_t reserved_664; // offset: 664  (0x298)
+    uint32_t reserved_665; // offset: 665  (0x299)
+    uint32_t reserved_666; // offset: 666  (0x29A)
+    uint32_t reserved_667; // offset: 667  (0x29B)
+    uint32_t reserved_668; // offset: 668  (0x29C)
+    uint32_t reserved_669; // offset: 669  (0x29D)
+    uint32_t reserved_670; // offset: 670  (0x29E)
+    uint32_t reserved_671; // offset: 671  (0x29F)
+    uint32_t reserved_672; // offset: 672  (0x2A0)
+    uint32_t reserved_673; // offset: 673  (0x2A1)
+    uint32_t reserved_674; // offset: 674  (0x2A2)
+    uint32_t reserved_675; // offset: 675  (0x2A3)
+    uint32_t reserved_676; // offset: 676  (0x2A4)
+    uint32_t reserved_677; // offset: 677  (0x2A5)
+    uint32_t reserved_678; // offset: 678  (0x2A6)
+    uint32_t reserved_679; // offset: 679  (0x2A7)
+    uint32_t reserved_680; // offset: 680  (0x2A8)
+    uint32_t reserved_681; // offset: 681  (0x2A9)
+    uint32_t reserved_682; // offset: 682  (0x2AA)
+    uint32_t reserved_683; // offset: 683  (0x2AB)
+    uint32_t reserved_684; // offset: 684  (0x2AC)
+    uint32_t reserved_685; // offset: 685  (0x2AD)
+    uint32_t reserved_686; // offset: 686  (0x2AE)
+    uint32_t reserved_687; // offset: 687  (0x2AF)
+    uint32_t reserved_688; // offset: 688  (0x2B0)
+    uint32_t reserved_689; // offset: 689  (0x2B1)
+    uint32_t reserved_690; // offset: 690  (0x2B2)
+    uint32_t reserved_691; // offset: 691  (0x2B3)
+    uint32_t reserved_692; // offset: 692  (0x2B4)
+    uint32_t reserved_693; // offset: 693  (0x2B5)
+    uint32_t reserved_694; // offset: 694  (0x2B6)
+    uint32_t reserved_695; // offset: 695  (0x2B7)
+    uint32_t reserved_696; // offset: 696  (0x2B8)
+    uint32_t reserved_697; // offset: 697  (0x2B9)
+    uint32_t reserved_698; // offset: 698  (0x2BA)
+    uint32_t reserved_699; // offset: 699  (0x2BB)
+    uint32_t reserved_700; // offset: 700  (0x2BC)
+    uint32_t reserved_701; // offset: 701  (0x2BD)
+    uint32_t reserved_702; // offset: 702  (0x2BE)
+    uint32_t reserved_703; // offset: 703  (0x2BF)
+    uint32_t reserved_704; // offset: 704  (0x2C0)
+    uint32_t reserved_705; // offset: 705  (0x2C1)
+    uint32_t reserved_706; // offset: 706  (0x2C2)
+    uint32_t reserved_707; // offset: 707  (0x2C3)
+    uint32_t reserved_708; // offset: 708  (0x2C4)
+    uint32_t reserved_709; // offset: 709  (0x2C5)
+    uint32_t reserved_710; // offset: 710  (0x2C6)
+    uint32_t reserved_711; // offset: 711  (0x2C7)
+    uint32_t reserved_712; // offset: 712  (0x2C8)
+    uint32_t reserved_713; // offset: 713  (0x2C9)
+    uint32_t reserved_714; // offset: 714  (0x2CA)
+    uint32_t reserved_715; // offset: 715  (0x2CB)
+    uint32_t reserved_716; // offset: 716  (0x2CC)
+    uint32_t reserved_717; // offset: 717  (0x2CD)
+    uint32_t reserved_718; // offset: 718  (0x2CE)
+    uint32_t reserved_719; // offset: 719  (0x2CF)
+    uint32_t reserved_720; // offset: 720  (0x2D0)
+    uint32_t reserved_721; // offset: 721  (0x2D1)
+    uint32_t reserved_722; // offset: 722  (0x2D2)
+    uint32_t reserved_723; // offset: 723  (0x2D3)
+    uint32_t reserved_724; // offset: 724  (0x2D4)
+    uint32_t reserved_725; // offset: 725  (0x2D5)
+    uint32_t reserved_726; // offset: 726  (0x2D6)
+    uint32_t reserved_727; // offset: 727  (0x2D7)
+    uint32_t reserved_728; // offset: 728  (0x2D8)
+    uint32_t reserved_729; // offset: 729  (0x2D9)
+    uint32_t reserved_730; // offset: 730  (0x2DA)
+    uint32_t reserved_731; // offset: 731  (0x2DB)
+    uint32_t reserved_732; // offset: 732  (0x2DC)
+    uint32_t reserved_733; // offset: 733  (0x2DD)
+    uint32_t reserved_734; // offset: 734  (0x2DE)
+    uint32_t reserved_735; // offset: 735  (0x2DF)
+    uint32_t reserved_736; // offset: 736  (0x2E0)
+    uint32_t reserved_737; // offset: 737  (0x2E1)
+    uint32_t reserved_738; // offset: 738  (0x2E2)
+    uint32_t reserved_739; // offset: 739  (0x2E3)
+    uint32_t reserved_740; // offset: 740  (0x2E4)
+    uint32_t reserved_741; // offset: 741  (0x2E5)
+    uint32_t reserved_742; // offset: 742  (0x2E6)
+    uint32_t reserved_743; // offset: 743  (0x2E7)
+    uint32_t reserved_744; // offset: 744  (0x2E8)
+    uint32_t reserved_745; // offset: 745  (0x2E9)
+    uint32_t reserved_746; // offset: 746  (0x2EA)
+    uint32_t reserved_747; // offset: 747  (0x2EB)
+    uint32_t reserved_748; // offset: 748  (0x2EC)
+    uint32_t reserved_749; // offset: 749  (0x2ED)
+    uint32_t reserved_750; // offset: 750  (0x2EE)
+    uint32_t reserved_751; // offset: 751  (0x2EF)
+    uint32_t reserved_752; // offset: 752  (0x2F0)
+    uint32_t reserved_753; // offset: 753  (0x2F1)
+    uint32_t reserved_754; // offset: 754  (0x2F2)
+    uint32_t reserved_755; // offset: 755  (0x2F3)
+    uint32_t reserved_756; // offset: 756  (0x2F4)
+    uint32_t reserved_757; // offset: 757  (0x2F5)
+    uint32_t reserved_758; // offset: 758  (0x2F6)
+    uint32_t reserved_759; // offset: 759  (0x2F7)
+    uint32_t reserved_760; // offset: 760  (0x2F8)
+    uint32_t reserved_761; // offset: 761  (0x2F9)
+    uint32_t reserved_762; // offset: 762  (0x2FA)
+    uint32_t reserved_763; // offset: 763  (0x2FB)
+    uint32_t reserved_764; // offset: 764  (0x2FC)
+    uint32_t reserved_765; // offset: 765  (0x2FD)
+    uint32_t reserved_766; // offset: 766  (0x2FE)
+    uint32_t reserved_767; // offset: 767  (0x2FF)
+    uint32_t reserved_768; // offset: 768  (0x300)
+    uint32_t reserved_769; // offset: 769  (0x301)
+    uint32_t reserved_770; // offset: 770  (0x302)
+    uint32_t reserved_771; // offset: 771  (0x303)
+    uint32_t reserved_772; // offset: 772  (0x304)
+    uint32_t reserved_773; // offset: 773  (0x305)
+    uint32_t reserved_774; // offset: 774  (0x306)
+    uint32_t reserved_775; // offset: 775  (0x307)
+    uint32_t reserved_776; // offset: 776  (0x308)
+    uint32_t reserved_777; // offset: 777  (0x309)
+    uint32_t reserved_778; // offset: 778  (0x30A)
+    uint32_t reserved_779; // offset: 779  (0x30B)
+    uint32_t reserved_780; // offset: 780  (0x30C)
+    uint32_t reserved_781; // offset: 781  (0x30D)
+    uint32_t reserved_782; // offset: 782  (0x30E)
+    uint32_t reserved_783; // offset: 783  (0x30F)
+    uint32_t reserved_784; // offset: 784  (0x310)
+    uint32_t reserved_785; // offset: 785  (0x311)
+    uint32_t reserved_786; // offset: 786  (0x312)
+    uint32_t reserved_787; // offset: 787  (0x313)
+    uint32_t reserved_788; // offset: 788  (0x314)
+    uint32_t reserved_789; // offset: 789  (0x315)
+    uint32_t reserved_790; // offset: 790  (0x316)
+    uint32_t reserved_791; // offset: 791  (0x317)
+    uint32_t reserved_792; // offset: 792  (0x318)
+    uint32_t reserved_793; // offset: 793  (0x319)
+    uint32_t reserved_794; // offset: 794  (0x31A)
+    uint32_t reserved_795; // offset: 795  (0x31B)
+    uint32_t reserved_796; // offset: 796  (0x31C)
+    uint32_t reserved_797; // offset: 797  (0x31D)
+    uint32_t reserved_798; // offset: 798  (0x31E)
+    uint32_t reserved_799; // offset: 799  (0x31F)
+    uint32_t reserved_800; // offset: 800  (0x320)
+    uint32_t reserved_801; // offset: 801  (0x321)
+    uint32_t reserved_802; // offset: 802  (0x322)
+    uint32_t reserved_803; // offset: 803  (0x323)
+    uint32_t reserved_804; // offset: 804  (0x324)
+    uint32_t reserved_805; // offset: 805  (0x325)
+    uint32_t reserved_806; // offset: 806  (0x326)
+    uint32_t reserved_807; // offset: 807  (0x327)
+    uint32_t reserved_808; // offset: 808  (0x328)
+    uint32_t reserved_809; // offset: 809  (0x329)
+    uint32_t reserved_810; // offset: 810  (0x32A)
+    uint32_t reserved_811; // offset: 811  (0x32B)
+    uint32_t reserved_812; // offset: 812  (0x32C)
+    uint32_t reserved_813; // offset: 813  (0x32D)
+    uint32_t reserved_814; // offset: 814  (0x32E)
+    uint32_t reserved_815; // offset: 815  (0x32F)
+    uint32_t reserved_816; // offset: 816  (0x330)
+    uint32_t reserved_817; // offset: 817  (0x331)
+    uint32_t reserved_818; // offset: 818  (0x332)
+    uint32_t reserved_819; // offset: 819  (0x333)
+    uint32_t reserved_820; // offset: 820  (0x334)
+    uint32_t reserved_821; // offset: 821  (0x335)
+    uint32_t reserved_822; // offset: 822  (0x336)
+    uint32_t reserved_823; // offset: 823  (0x337)
+    uint32_t reserved_824; // offset: 824  (0x338)
+    uint32_t reserved_825; // offset: 825  (0x339)
+    uint32_t reserved_826; // offset: 826  (0x33A)
+    uint32_t reserved_827; // offset: 827  (0x33B)
+    uint32_t reserved_828; // offset: 828  (0x33C)
+    uint32_t reserved_829; // offset: 829  (0x33D)
+    uint32_t reserved_830; // offset: 830  (0x33E)
+    uint32_t reserved_831; // offset: 831  (0x33F)
+    uint32_t reserved_832; // offset: 832  (0x340)
+    uint32_t reserved_833; // offset: 833  (0x341)
+    uint32_t reserved_834; // offset: 834  (0x342)
+    uint32_t reserved_835; // offset: 835  (0x343)
+    uint32_t reserved_836; // offset: 836  (0x344)
+    uint32_t reserved_837; // offset: 837  (0x345)
+    uint32_t reserved_838; // offset: 838  (0x346)
+    uint32_t reserved_839; // offset: 839  (0x347)
+    uint32_t reserved_840; // offset: 840  (0x348)
+    uint32_t reserved_841; // offset: 841  (0x349)
+    uint32_t reserved_842; // offset: 842  (0x34A)
+    uint32_t reserved_843; // offset: 843  (0x34B)
+    uint32_t reserved_844; // offset: 844  (0x34C)
+    uint32_t reserved_845; // offset: 845  (0x34D)
+    uint32_t reserved_846; // offset: 846  (0x34E)
+    uint32_t reserved_847; // offset: 847  (0x34F)
+    uint32_t reserved_848; // offset: 848  (0x350)
+    uint32_t reserved_849; // offset: 849  (0x351)
+    uint32_t reserved_850; // offset: 850  (0x352)
+    uint32_t reserved_851; // offset: 851  (0x353)
+    uint32_t reserved_852; // offset: 852  (0x354)
+    uint32_t reserved_853; // offset: 853  (0x355)
+    uint32_t reserved_854; // offset: 854  (0x356)
+    uint32_t reserved_855; // offset: 855  (0x357)
+    uint32_t reserved_856; // offset: 856  (0x358)
+    uint32_t reserved_857; // offset: 857  (0x359)
+    uint32_t reserved_858; // offset: 858  (0x35A)
+    uint32_t reserved_859; // offset: 859  (0x35B)
+    uint32_t reserved_860; // offset: 860  (0x35C)
+    uint32_t reserved_861; // offset: 861  (0x35D)
+    uint32_t reserved_862; // offset: 862  (0x35E)
+    uint32_t reserved_863; // offset: 863  (0x35F)
+    uint32_t reserved_864; // offset: 864  (0x360)
+    uint32_t reserved_865; // offset: 865  (0x361)
+    uint32_t reserved_866; // offset: 866  (0x362)
+    uint32_t reserved_867; // offset: 867  (0x363)
+    uint32_t reserved_868; // offset: 868  (0x364)
+    uint32_t reserved_869; // offset: 869  (0x365)
+    uint32_t reserved_870; // offset: 870  (0x366)
+    uint32_t reserved_871; // offset: 871  (0x367)
+    uint32_t reserved_872; // offset: 872  (0x368)
+    uint32_t reserved_873; // offset: 873  (0x369)
+    uint32_t reserved_874; // offset: 874  (0x36A)
+    uint32_t reserved_875; // offset: 875  (0x36B)
+    uint32_t reserved_876; // offset: 876  (0x36C)
+    uint32_t reserved_877; // offset: 877  (0x36D)
+    uint32_t reserved_878; // offset: 878  (0x36E)
+    uint32_t reserved_879; // offset: 879  (0x36F)
+    uint32_t reserved_880; // offset: 880  (0x370)
+    uint32_t reserved_881; // offset: 881  (0x371)
+    uint32_t reserved_882; // offset: 882  (0x372)
+    uint32_t reserved_883; // offset: 883  (0x373)
+    uint32_t reserved_884; // offset: 884  (0x374)
+    uint32_t reserved_885; // offset: 885  (0x375)
+    uint32_t reserved_886; // offset: 886  (0x376)
+    uint32_t reserved_887; // offset: 887  (0x377)
+    uint32_t reserved_888; // offset: 888  (0x378)
+    uint32_t reserved_889; // offset: 889  (0x379)
+    uint32_t reserved_890; // offset: 890  (0x37A)
+    uint32_t reserved_891; // offset: 891  (0x37B)
+    uint32_t reserved_892; // offset: 892  (0x37C)
+    uint32_t reserved_893; // offset: 893  (0x37D)
+    uint32_t reserved_894; // offset: 894  (0x37E)
+    uint32_t reserved_895; // offset: 895  (0x37F)
+    uint32_t reserved_896; // offset: 896  (0x380)
+    uint32_t reserved_897; // offset: 897  (0x381)
+    uint32_t reserved_898; // offset: 898  (0x382)
+    uint32_t reserved_899; // offset: 899  (0x383)
+    uint32_t reserved_900; // offset: 900  (0x384)
+    uint32_t reserved_901; // offset: 901  (0x385)
+    uint32_t reserved_902; // offset: 902  (0x386)
+    uint32_t reserved_903; // offset: 903  (0x387)
+    uint32_t reserved_904; // offset: 904  (0x388)
+    uint32_t reserved_905; // offset: 905  (0x389)
+    uint32_t reserved_906; // offset: 906  (0x38A)
+    uint32_t reserved_907; // offset: 907  (0x38B)
+    uint32_t reserved_908; // offset: 908  (0x38C)
+    uint32_t reserved_909; // offset: 909  (0x38D)
+    uint32_t reserved_910; // offset: 910  (0x38E)
+    uint32_t reserved_911; // offset: 911  (0x38F)
+    uint32_t reserved_912; // offset: 912  (0x390)
+    uint32_t reserved_913; // offset: 913  (0x391)
+    uint32_t reserved_914; // offset: 914  (0x392)
+    uint32_t reserved_915; // offset: 915  (0x393)
+    uint32_t reserved_916; // offset: 916  (0x394)
+    uint32_t reserved_917; // offset: 917  (0x395)
+    uint32_t reserved_918; // offset: 918  (0x396)
+    uint32_t reserved_919; // offset: 919  (0x397)
+    uint32_t reserved_920; // offset: 920  (0x398)
+    uint32_t reserved_921; // offset: 921  (0x399)
+    uint32_t reserved_922; // offset: 922  (0x39A)
+    uint32_t reserved_923; // offset: 923  (0x39B)
+    uint32_t reserved_924; // offset: 924  (0x39C)
+    uint32_t reserved_925; // offset: 925  (0x39D)
+    uint32_t reserved_926; // offset: 926  (0x39E)
+    uint32_t reserved_927; // offset: 927  (0x39F)
+    uint32_t reserved_928; // offset: 928  (0x3A0)
+    uint32_t reserved_929; // offset: 929  (0x3A1)
+    uint32_t reserved_930; // offset: 930  (0x3A2)
+    uint32_t reserved_931; // offset: 931  (0x3A3)
+    uint32_t reserved_932; // offset: 932  (0x3A4)
+    uint32_t reserved_933; // offset: 933  (0x3A5)
+    uint32_t reserved_934; // offset: 934  (0x3A6)
+    uint32_t reserved_935; // offset: 935  (0x3A7)
+    uint32_t reserved_936; // offset: 936  (0x3A8)
+    uint32_t reserved_937; // offset: 937  (0x3A9)
+    uint32_t reserved_938; // offset: 938  (0x3AA)
+    uint32_t reserved_939; // offset: 939  (0x3AB)
+    uint32_t reserved_940; // offset: 940  (0x3AC)
+    uint32_t reserved_941; // offset: 941  (0x3AD)
+    uint32_t reserved_942; // offset: 942  (0x3AE)
+    uint32_t reserved_943; // offset: 943  (0x3AF)
+    uint32_t reserved_944; // offset: 944  (0x3B0)
+    uint32_t reserved_945; // offset: 945  (0x3B1)
+    uint32_t reserved_946; // offset: 946  (0x3B2)
+    uint32_t reserved_947; // offset: 947  (0x3B3)
+    uint32_t reserved_948; // offset: 948  (0x3B4)
+    uint32_t reserved_949; // offset: 949  (0x3B5)
+    uint32_t reserved_950; // offset: 950  (0x3B6)
+    uint32_t reserved_951; // offset: 951  (0x3B7)
+    uint32_t reserved_952; // offset: 952  (0x3B8)
+    uint32_t reserved_953; // offset: 953  (0x3B9)
+    uint32_t reserved_954; // offset: 954  (0x3BA)
+    uint32_t reserved_955; // offset: 955  (0x3BB)
+    uint32_t reserved_956; // offset: 956  (0x3BC)
+    uint32_t reserved_957; // offset: 957  (0x3BD)
+    uint32_t reserved_958; // offset: 958  (0x3BE)
+    uint32_t reserved_959; // offset: 959  (0x3BF)
+    uint32_t reserved_960; // offset: 960  (0x3C0)
+    uint32_t reserved_961; // offset: 961  (0x3C1)
+    uint32_t reserved_962; // offset: 962  (0x3C2)
+    uint32_t reserved_963; // offset: 963  (0x3C3)
+    uint32_t reserved_964; // offset: 964  (0x3C4)
+    uint32_t reserved_965; // offset: 965  (0x3C5)
+    uint32_t reserved_966; // offset: 966  (0x3C6)
+    uint32_t reserved_967; // offset: 967  (0x3C7)
+    uint32_t reserved_968; // offset: 968  (0x3C8)
+    uint32_t reserved_969; // offset: 969  (0x3C9)
+    uint32_t reserved_970; // offset: 970  (0x3CA)
+    uint32_t reserved_971; // offset: 971  (0x3CB)
+    uint32_t reserved_972; // offset: 972  (0x3CC)
+    uint32_t reserved_973; // offset: 973  (0x3CD)
+    uint32_t reserved_974; // offset: 974  (0x3CE)
+    uint32_t reserved_975; // offset: 975  (0x3CF)
+    uint32_t reserved_976; // offset: 976  (0x3D0)
+    uint32_t reserved_977; // offset: 977  (0x3D1)
+    uint32_t reserved_978; // offset: 978  (0x3D2)
+    uint32_t reserved_979; // offset: 979  (0x3D3)
+    uint32_t reserved_980; // offset: 980  (0x3D4)
+    uint32_t reserved_981; // offset: 981  (0x3D5)
+    uint32_t reserved_982; // offset: 982  (0x3D6)
+    uint32_t reserved_983; // offset: 983  (0x3D7)
+    uint32_t reserved_984; // offset: 984  (0x3D8)
+    uint32_t reserved_985; // offset: 985  (0x3D9)
+    uint32_t reserved_986; // offset: 986  (0x3DA)
+    uint32_t reserved_987; // offset: 987  (0x3DB)
+    uint32_t reserved_988; // offset: 988  (0x3DC)
+    uint32_t reserved_989; // offset: 989  (0x3DD)
+    uint32_t reserved_990; // offset: 990  (0x3DE)
+    uint32_t reserved_991; // offset: 991  (0x3DF)
+    uint32_t reserved_992; // offset: 992  (0x3E0)
+    uint32_t reserved_993; // offset: 993  (0x3E1)
+    uint32_t reserved_994; // offset: 994  (0x3E2)
+    uint32_t reserved_995; // offset: 995  (0x3E3)
+    uint32_t reserved_996; // offset: 996  (0x3E4)
+    uint32_t reserved_997; // offset: 997  (0x3E5)
+    uint32_t reserved_998; // offset: 998  (0x3E6)
+    uint32_t reserved_999; // offset: 999  (0x3E7)
+    uint32_t reserved_1000; // offset: 1000  (0x3E8)
+    uint32_t reserved_1001; // offset: 1001  (0x3E9)
+    uint32_t reserved_1002; // offset: 1002  (0x3EA)
+    uint32_t reserved_1003; // offset: 1003  (0x3EB)
+    uint32_t reserved_1004; // offset: 1004  (0x3EC)
+    uint32_t reserved_1005; // offset: 1005  (0x3ED)
+    uint32_t reserved_1006; // offset: 1006  (0x3EE)
+    uint32_t reserved_1007; // offset: 1007  (0x3EF)
+    uint32_t reserved_1008; // offset: 1008  (0x3F0)
+    uint32_t reserved_1009; // offset: 1009  (0x3F1)
+    uint32_t reserved_1010; // offset: 1010  (0x3F2)
+    uint32_t reserved_1011; // offset: 1011  (0x3F3)
+    uint32_t reserved_1012; // offset: 1012  (0x3F4)
+    uint32_t reserved_1013; // offset: 1013  (0x3F5)
+    uint32_t reserved_1014; // offset: 1014  (0x3F6)
+    uint32_t reserved_1015; // offset: 1015  (0x3F7)
+    uint32_t reserved_1016; // offset: 1016  (0x3F8)
+    uint32_t reserved_1017; // offset: 1017  (0x3F9)
+    uint32_t reserved_1018; // offset: 1018  (0x3FA)
+    uint32_t reserved_1019; // offset: 1019  (0x3FB)
+    uint32_t reserved_1020; // offset: 1020  (0x3FC)
+    uint32_t reserved_1021; // offset: 1021  (0x3FD)
+    uint32_t reserved_1022; // offset: 1022  (0x3FE)
+    uint32_t reserved_1023; // offset: 1023  (0x3FF)
+};
+	
 #endif /* V11_STRUCTS_H_ */
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
index 79b174e5326d..74bb2d9f7faa 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
@@ -1243,25 +1243,6 @@ int amdgpu_dpm_odn_edit_dpm_table(struct amdgpu_device *adev,
 	return ret;
 }
 
-int amdgpu_dpm_print_clock_levels(struct amdgpu_device *adev,
-				  enum pp_clock_type type,
-				  char *buf)
-{
-	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
-	int ret = 0;
-
-	if (!pp_funcs->print_clock_levels)
-		return 0;
-
-	mutex_lock(&adev->pm.mutex);
-	ret = pp_funcs->print_clock_levels(adev->powerplay.pp_handle,
-					   type,
-					   buf);
-	mutex_unlock(&adev->pm.mutex);
-
-	return ret;
-}
-
 int amdgpu_dpm_emit_clock_levels(struct amdgpu_device *adev,
 				  enum pp_clock_type type,
 				  char *buf,
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
index 65296a819e6a..e289ed6dd1e7 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
@@ -863,14 +863,6 @@ static ssize_t amdgpu_get_pp_od_clk_voltage(struct device *dev,
 		if (ret)
 			break;
 	}
-	if (ret == -ENOENT) {
-		size = amdgpu_dpm_print_clock_levels(adev, OD_SCLK, buf);
-		size += amdgpu_dpm_print_clock_levels(adev, OD_MCLK, buf + size);
-		size += amdgpu_dpm_print_clock_levels(adev, OD_VDDC_CURVE, buf + size);
-		size += amdgpu_dpm_print_clock_levels(adev, OD_VDDGFX_OFFSET, buf + size);
-		size += amdgpu_dpm_print_clock_levels(adev, OD_RANGE, buf + size);
-		size += amdgpu_dpm_print_clock_levels(adev, OD_CCLK, buf + size);
-	}
 
 	if (size == 0)
 		size = sysfs_emit(buf, "\n");
@@ -999,8 +991,8 @@ static ssize_t amdgpu_get_pp_dpm_clock(struct device *dev,
 		return ret;
 
 	ret = amdgpu_dpm_emit_clock_levels(adev, type, buf, &size);
-	if (ret == -ENOENT)
-		size = amdgpu_dpm_print_clock_levels(adev, type, buf);
+	if (ret)
+		return ret;
 
 	if (size == 0)
 		size = sysfs_emit(buf, "\n");
@@ -2081,9 +2073,10 @@ static int pp_dpm_clk_default_attr_update(struct amdgpu_device *adev, struct amd
  * for user application to monitor various board reated attributes.
  *
  * The amdgpu driver provides a sysfs API for reporting board attributes. Presently,
- * seven types of attributes are reported. Baseboard temperature and
+ * nine types of attributes are reported. Baseboard temperature and
  * gpu board temperature are reported as binary files. Npm status, current node power limit,
- * max node power limit, node power and global ppt residency is reported as ASCII text file.
+ * max node power limit, node power, global ppt residency, baseboard_power, baseboard_power_limit
+ * is reported as ASCII text file.
  *
  * * .. code-block:: console
  *
@@ -2100,6 +2093,10 @@ static int pp_dpm_clk_default_attr_update(struct amdgpu_device *adev, struct amd
  *      hexdump /sys/bus/pci/devices/.../board/node_power
  *
  *      hexdump /sys/bus/pci/devices/.../board/global_ppt_resid
+ *
+ *      hexdump /sys/bus/pci/devices/.../board/baseboard_power
+ *
+ *      hexdump /sys/bus/pci/devices/.../board/baseboard_power_limit
  */
 
 /**
@@ -2294,6 +2291,52 @@ static ssize_t amdgpu_show_max_node_power_limit(struct device *dev,
 	return sysfs_emit(buf, "%u\n", max_nplimit);
 }
 
+/**
+ * DOC: baseboard_power
+ *
+ * The amdgpu driver provides a sysfs API for retrieving current ubb power in watts.
+ * The file baseboard_power is used for this.
+ */
+static ssize_t amdgpu_show_baseboard_power(struct device *dev,
+					   struct device_attribute *attr, char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+	u32 ubbpower;
+	int r;
+
+	/* get the ubb power */
+	r = amdgpu_pm_get_sensor_generic(adev, AMDGPU_PP_SENSOR_UBB_POWER,
+					 (void *)&ubbpower);
+	if (r)
+		return r;
+
+	return sysfs_emit(buf, "%u\n", ubbpower);
+}
+
+/**
+ * DOC: baseboard_power_limit
+ *
+ * The amdgpu driver provides a sysfs API for retrieving threshold ubb power in watts.
+ * The file baseboard_power_limit is used for this.
+ */
+static ssize_t amdgpu_show_baseboard_power_limit(struct device *dev,
+						 struct device_attribute *attr, char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+	u32 ubbpowerlimit;
+	int r;
+
+	/* get the ubb power limit */
+	r = amdgpu_pm_get_sensor_generic(adev, AMDGPU_PP_SENSOR_UBB_POWER_LIMIT,
+					 (void *)&ubbpowerlimit);
+	if (r)
+		return r;
+
+	return sysfs_emit(buf, "%u\n", ubbpowerlimit);
+}
+
 static DEVICE_ATTR(baseboard_temp, 0444, amdgpu_get_baseboard_temp_metrics, NULL);
 static DEVICE_ATTR(gpuboard_temp, 0444, amdgpu_get_gpuboard_temp_metrics, NULL);
 static DEVICE_ATTR(cur_node_power_limit, 0444, amdgpu_show_cur_node_power_limit, NULL);
@@ -2301,6 +2344,8 @@ static DEVICE_ATTR(node_power, 0444, amdgpu_show_node_power, NULL);
 static DEVICE_ATTR(global_ppt_resid, 0444, amdgpu_show_global_ppt_resid, NULL);
 static DEVICE_ATTR(max_node_power_limit, 0444, amdgpu_show_max_node_power_limit, NULL);
 static DEVICE_ATTR(npm_status, 0444, amdgpu_show_npm_status, NULL);
+static DEVICE_ATTR(baseboard_power, 0444, amdgpu_show_baseboard_power, NULL);
+static DEVICE_ATTR(baseboard_power_limit, 0444, amdgpu_show_baseboard_power_limit, NULL);
 
 static struct attribute *board_attrs[] = {
 	&dev_attr_baseboard_temp.attr,
@@ -3868,7 +3913,9 @@ static int amdgpu_retrieve_od_settings(struct amdgpu_device *adev,
 	if (ret)
 		return ret;
 
-	size = amdgpu_dpm_print_clock_levels(adev, od_type, buf);
+	ret = amdgpu_dpm_emit_clock_levels(adev, od_type, buf, &size);
+	if (ret)
+		return ret;
 	if (size == 0)
 		size = sysfs_emit(buf, "\n");
 
@@ -4756,6 +4803,14 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
 			sysfs_add_file_to_group(&adev->dev->kobj, &dev_attr_npm_status.attr,
 						amdgpu_board_attr_group.name);
 		}
+		if (amdgpu_pm_get_sensor_generic(adev, AMDGPU_PP_SENSOR_UBB_POWER_LIMIT,
+						 (void *)&tmp) != -EOPNOTSUPP) {
+			sysfs_add_file_to_group(&adev->dev->kobj,
+						&dev_attr_baseboard_power_limit.attr,
+						amdgpu_board_attr_group.name);
+			sysfs_add_file_to_group(&adev->dev->kobj, &dev_attr_baseboard_power.attr,
+						amdgpu_board_attr_group.name);
+		}
 	}
 
 	adev->pm.sysfs_initialized = true;
diff --git a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
index 3aaf3dd71868..c10cb8124349 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
@@ -724,21 +724,6 @@ static int pp_dpm_emit_clock_levels(void *handle,
 	return hwmgr->hwmgr_func->emit_clock_levels(hwmgr, type, buf, offset);
 }
 
-static int pp_dpm_print_clock_levels(void *handle,
-		enum pp_clock_type type, char *buf)
-{
-	struct pp_hwmgr *hwmgr = handle;
-
-	if (!hwmgr || !hwmgr->pm_en)
-		return -EINVAL;
-
-	if (hwmgr->hwmgr_func->print_clock_levels == NULL) {
-		pr_info_ratelimited("%s was not implemented.\n", __func__);
-		return 0;
-	}
-	return hwmgr->hwmgr_func->print_clock_levels(hwmgr, type, buf);
-}
-
 static int pp_dpm_get_sclk_od(void *handle)
 {
 	struct pp_hwmgr *hwmgr = handle;
@@ -1582,7 +1567,6 @@ static const struct amd_pm_funcs pp_dpm_funcs = {
 	.set_pp_table = pp_dpm_set_pp_table,
 	.force_clock_level = pp_dpm_force_clock_level,
 	.emit_clock_levels = pp_dpm_emit_clock_levels,
-	.print_clock_levels = pp_dpm_print_clock_levels,
 	.get_sclk_od = pp_dpm_get_sclk_od,
 	.set_sclk_od = pp_dpm_set_sclk_od,
 	.get_mclk_od = pp_dpm_get_mclk_od,
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c
index 14ccd743ca1d..8de8d66df95f 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c
@@ -1024,13 +1024,14 @@ static int smu10_force_clock_level(struct pp_hwmgr *hwmgr,
 	return 0;
 }
 
-static int smu10_print_clock_levels(struct pp_hwmgr *hwmgr,
-		enum pp_clock_type type, char *buf)
+static int smu10_emit_clock_levels(struct pp_hwmgr *hwmgr,
+				   enum pp_clock_type type, char *buf,
+				   int *offset)
 {
 	struct smu10_hwmgr *data = (struct smu10_hwmgr *)(hwmgr->backend);
 	struct smu10_voltage_dependency_table *mclk_table =
 			data->clock_vol_info.vdd_dep_on_fclk;
-	uint32_t i, now, size = 0;
+	uint32_t i, now, size = *offset;
 	uint32_t min_freq, max_freq = 0;
 	int ret = 0;
 
@@ -1048,15 +1049,15 @@ static int smu10_print_clock_levels(struct pp_hwmgr *hwmgr,
 		else
 			i = 1;
 
-		size += sprintf(buf + size, "0: %uMhz %s\n",
-					data->gfx_min_freq_limit/100,
-					i == 0 ? "*" : "");
-		size += sprintf(buf + size, "1: %uMhz %s\n",
-					i == 1 ? now : SMU10_UMD_PSTATE_GFXCLK,
-					i == 1 ? "*" : "");
-		size += sprintf(buf + size, "2: %uMhz %s\n",
-					data->gfx_max_freq_limit/100,
-					i == 2 ? "*" : "");
+		size += sysfs_emit_at(buf, size, "0: %uMhz %s\n",
+				      data->gfx_min_freq_limit / 100,
+				      i == 0 ? "*" : "");
+		size += sysfs_emit_at(buf, size, "1: %uMhz %s\n",
+				      i == 1 ? now : SMU10_UMD_PSTATE_GFXCLK,
+				      i == 1 ? "*" : "");
+		size += sysfs_emit_at(buf, size, "2: %uMhz %s\n",
+				      data->gfx_max_freq_limit / 100,
+				      i == 2 ? "*" : "");
 		break;
 	case PP_MCLK:
 		ret = smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetFclkFrequency, &now);
@@ -1064,11 +1065,12 @@ static int smu10_print_clock_levels(struct pp_hwmgr *hwmgr,
 			return ret;
 
 		for (i = 0; i < mclk_table->count; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-					i,
-					mclk_table->entries[i].clk / 100,
-					((mclk_table->entries[i].clk / 100)
-					 == now) ? "*" : "");
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n", i,
+				mclk_table->entries[i].clk / 100,
+				((mclk_table->entries[i].clk / 100) == now) ?
+					"*" :
+					"");
 		break;
 	case OD_SCLK:
 		if (hwmgr->od_enabled) {
@@ -1079,11 +1081,17 @@ static int smu10_print_clock_levels(struct pp_hwmgr *hwmgr,
 			if (ret)
 				return ret;
 
-			size += sprintf(buf + size, "%s:\n", "OD_SCLK");
-			size += sprintf(buf + size, "0: %10uMhz\n",
-			(data->gfx_actual_soft_min_freq > 0) ? data->gfx_actual_soft_min_freq : min_freq);
-			size += sprintf(buf + size, "1: %10uMhz\n",
-			(data->gfx_actual_soft_max_freq > 0) ? data->gfx_actual_soft_max_freq : max_freq);
+			size += sysfs_emit_at(buf, size, "%s:\n", "OD_SCLK");
+			size += sysfs_emit_at(
+				buf, size, "0: %10uMhz\n",
+				(data->gfx_actual_soft_min_freq > 0) ?
+					data->gfx_actual_soft_min_freq :
+					min_freq);
+			size += sysfs_emit_at(
+				buf, size, "1: %10uMhz\n",
+				(data->gfx_actual_soft_max_freq > 0) ?
+					data->gfx_actual_soft_max_freq :
+					max_freq);
 		}
 		break;
 	case OD_RANGE:
@@ -1095,16 +1103,19 @@ static int smu10_print_clock_levels(struct pp_hwmgr *hwmgr,
 			if (ret)
 				return ret;
 
-			size += sprintf(buf + size, "%s:\n", "OD_RANGE");
-			size += sprintf(buf + size, "SCLK: %7uMHz %10uMHz\n",
-				min_freq, max_freq);
+			size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE");
+			size += sysfs_emit_at(buf, size,
+					      "SCLK: %7uMHz %10uMHz\n",
+					      min_freq, max_freq);
 		}
 		break;
 	default:
 		break;
 	}
 
-	return size;
+	*offset = size;
+
+	return 0;
 }
 
 static int smu10_get_performance_level(struct pp_hwmgr *hwmgr, const struct pp_hw_power_state *state,
@@ -1652,7 +1663,7 @@ static const struct pp_hwmgr_func smu10_hwmgr_funcs = {
 	.set_cpu_power_state = smu10_set_cpu_power_state,
 	.store_cc6_data = smu10_store_cc6_data,
 	.force_clock_level = smu10_force_clock_level,
-	.print_clock_levels = smu10_print_clock_levels,
+	.emit_clock_levels = smu10_emit_clock_levels,
 	.get_dal_power_level = smu10_get_dal_power_level,
 	.get_performance_level = smu10_get_performance_level,
 	.get_current_shallow_sleep_clocks = smu10_get_current_shallow_sleep_clocks,
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c
index 9b28c0728269..2be584aefd0a 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c
@@ -4961,8 +4961,9 @@ static int smu7_force_clock_level(struct pp_hwmgr *hwmgr,
 	return 0;
 }
 
-static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr,
-		enum pp_clock_type type, char *buf)
+static int smu7_emit_clock_levels(struct pp_hwmgr *hwmgr,
+				  enum pp_clock_type type, char *buf,
+				  int *offset)
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 	struct smu7_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
@@ -4971,7 +4972,7 @@ static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr,
 	struct smu7_odn_dpm_table *odn_table = &(data->odn_dpm_table);
 	struct phm_odn_clock_levels *odn_sclk_table = &(odn_table->odn_core_clock_dpm_levels);
 	struct phm_odn_clock_levels *odn_mclk_table = &(odn_table->odn_memory_clock_dpm_levels);
-	int size = 0, ret = 0;
+	int size = *offset, ret = 0;
 	uint32_t i, now, clock, pcie_speed;
 
 	switch (type) {
@@ -4987,9 +4988,10 @@ static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr,
 		now = i;
 
 		for (i = 0; i < sclk_table->count; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-					i, sclk_table->dpm_levels[i].value / 100,
-					(i == now) ? "*" : "");
+			size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i,
+					      sclk_table->dpm_levels[i].value /
+						      100,
+					      (i == now) ? "*" : "");
 		break;
 	case PP_MCLK:
 		ret = smum_send_msg_to_smc(hwmgr, PPSMC_MSG_API_GetMclkFrequency, &clock);
@@ -5003,9 +5005,10 @@ static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr,
 		now = i;
 
 		for (i = 0; i < mclk_table->count; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-					i, mclk_table->dpm_levels[i].value / 100,
-					(i == now) ? "*" : "");
+			size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i,
+					      mclk_table->dpm_levels[i].value /
+						      100,
+					      (i == now) ? "*" : "");
 		break;
 	case PP_PCIE:
 		pcie_speed = smu7_get_current_pcie_speed(hwmgr);
@@ -5017,48 +5020,68 @@ static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr,
 		now = i;
 
 		for (i = 0; i < pcie_table->count; i++)
-			size += sprintf(buf + size, "%d: %s %s\n", i,
-					(pcie_table->dpm_levels[i].value == 0) ? "2.5GT/s, x8" :
-					(pcie_table->dpm_levels[i].value == 1) ? "5.0GT/s, x16" :
-					(pcie_table->dpm_levels[i].value == 2) ? "8.0GT/s, x16" : "",
-					(i == now) ? "*" : "");
+			size += sysfs_emit_at(
+				buf, size, "%d: %s %s\n", i,
+				(pcie_table->dpm_levels[i].value == 0) ?
+					"2.5GT/s, x8" :
+				(pcie_table->dpm_levels[i].value == 1) ?
+					"5.0GT/s, x16" :
+				(pcie_table->dpm_levels[i].value == 2) ?
+					"8.0GT/s, x16" :
+					"",
+				(i == now) ? "*" : "");
 		break;
 	case OD_SCLK:
 		if (hwmgr->od_enabled) {
-			size += sprintf(buf + size, "%s:\n", "OD_SCLK");
+			size += sysfs_emit_at(buf, size, "%s:\n", "OD_SCLK");
 			for (i = 0; i < odn_sclk_table->num_of_pl; i++)
-				size += sprintf(buf + size, "%d: %10uMHz %10umV\n",
-					i, odn_sclk_table->entries[i].clock/100,
+				size += sysfs_emit_at(
+					buf, size, "%d: %10uMHz %10umV\n", i,
+					odn_sclk_table->entries[i].clock / 100,
 					odn_sclk_table->entries[i].vddc);
 		}
 		break;
 	case OD_MCLK:
 		if (hwmgr->od_enabled) {
-			size += sprintf(buf + size, "%s:\n", "OD_MCLK");
+			size += sysfs_emit_at(buf, size, "%s:\n", "OD_MCLK");
 			for (i = 0; i < odn_mclk_table->num_of_pl; i++)
-				size += sprintf(buf + size, "%d: %10uMHz %10umV\n",
-					i, odn_mclk_table->entries[i].clock/100,
+				size += sysfs_emit_at(
+					buf, size, "%d: %10uMHz %10umV\n", i,
+					odn_mclk_table->entries[i].clock / 100,
 					odn_mclk_table->entries[i].vddc);
 		}
 		break;
 	case OD_RANGE:
 		if (hwmgr->od_enabled) {
-			size += sprintf(buf + size, "%s:\n", "OD_RANGE");
-			size += sprintf(buf + size, "SCLK: %7uMHz %10uMHz\n",
-				data->golden_dpm_table.sclk_table.dpm_levels[0].value/100,
-				hwmgr->platform_descriptor.overdriveLimit.engineClock/100);
-			size += sprintf(buf + size, "MCLK: %7uMHz %10uMHz\n",
-				data->golden_dpm_table.mclk_table.dpm_levels[0].value/100,
-				hwmgr->platform_descriptor.overdriveLimit.memoryClock/100);
-			size += sprintf(buf + size, "VDDC: %7umV %11umV\n",
-				data->odn_dpm_table.min_vddc,
-				data->odn_dpm_table.max_vddc);
+			size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE");
+			size += sysfs_emit_at(
+				buf, size, "SCLK: %7uMHz %10uMHz\n",
+				data->golden_dpm_table.sclk_table.dpm_levels[0]
+						.value /
+					100,
+				hwmgr->platform_descriptor.overdriveLimit
+						.engineClock /
+					100);
+			size += sysfs_emit_at(
+				buf, size, "MCLK: %7uMHz %10uMHz\n",
+				data->golden_dpm_table.mclk_table.dpm_levels[0]
+						.value /
+					100,
+				hwmgr->platform_descriptor.overdriveLimit
+						.memoryClock /
+					100);
+			size += sysfs_emit_at(buf, size, "VDDC: %7umV %11umV\n",
+					      data->odn_dpm_table.min_vddc,
+					      data->odn_dpm_table.max_vddc);
 		}
 		break;
 	default:
 		break;
 	}
-	return size;
+
+	*offset = size;
+
+	return 0;
 }
 
 static void smu7_set_fan_control_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
@@ -5775,7 +5798,7 @@ static const struct pp_hwmgr_func smu7_hwmgr_funcs = {
 	.set_fan_control_mode = smu7_set_fan_control_mode,
 	.get_fan_control_mode = smu7_get_fan_control_mode,
 	.force_clock_level = smu7_force_clock_level,
-	.print_clock_levels = smu7_print_clock_levels,
+	.emit_clock_levels = smu7_emit_clock_levels,
 	.powergate_gfx = smu7_powergate_gfx,
 	.get_sclk_od = smu7_get_sclk_od,
 	.set_sclk_od = smu7_set_sclk_od,
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.c
index 9b20076e26c0..736e5a8af477 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu8_hwmgr.c
@@ -1564,14 +1564,15 @@ static int smu8_force_clock_level(struct pp_hwmgr *hwmgr,
 	return 0;
 }
 
-static int smu8_print_clock_levels(struct pp_hwmgr *hwmgr,
-		enum pp_clock_type type, char *buf)
+static int smu8_emit_clock_levels(struct pp_hwmgr *hwmgr,
+				  enum pp_clock_type type, char *buf,
+				  int *offset)
 {
 	struct smu8_hwmgr *data = hwmgr->backend;
 	struct phm_clock_voltage_dependency_table *sclk_table =
 			hwmgr->dyn_state.vddc_dependency_on_sclk;
 	uint32_t i, now;
-	int size = 0;
+	int size = *offset;
 
 	switch (type) {
 	case PP_SCLK:
@@ -1582,9 +1583,9 @@ static int smu8_print_clock_levels(struct pp_hwmgr *hwmgr,
 				CURR_SCLK_INDEX);
 
 		for (i = 0; i < sclk_table->count; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-					i, sclk_table->entries[i].clk / 100,
-					(i == now) ? "*" : "");
+			size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i,
+					      sclk_table->entries[i].clk / 100,
+					      (i == now) ? "*" : "");
 		break;
 	case PP_MCLK:
 		now = PHM_GET_FIELD(cgs_read_ind_register(hwmgr->device,
@@ -1594,14 +1595,20 @@ static int smu8_print_clock_levels(struct pp_hwmgr *hwmgr,
 				CURR_MCLK_INDEX);
 
 		for (i = SMU8_NUM_NBPMEMORYCLOCK; i > 0; i--)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-					SMU8_NUM_NBPMEMORYCLOCK-i, data->sys_info.nbp_memory_clock[i-1] / 100,
-					(SMU8_NUM_NBPMEMORYCLOCK-i == now) ? "*" : "");
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n",
+				SMU8_NUM_NBPMEMORYCLOCK - i,
+				data->sys_info.nbp_memory_clock[i - 1] / 100,
+				(SMU8_NUM_NBPMEMORYCLOCK - i == now) ? "*" :
+								       "");
 		break;
 	default:
 		break;
 	}
-	return size;
+
+	*offset = size;
+
+	return 0;
 }
 
 static int smu8_get_performance_level(struct pp_hwmgr *hwmgr, const struct pp_hw_power_state *state,
@@ -2055,7 +2062,7 @@ static const struct pp_hwmgr_func smu8_hwmgr_funcs = {
 	.set_cpu_power_state = smu8_set_cpu_power_state,
 	.store_cc6_data = smu8_store_cc6_data,
 	.force_clock_level = smu8_force_clock_level,
-	.print_clock_levels = smu8_print_clock_levels,
+	.emit_clock_levels = smu8_emit_clock_levels,
 	.get_dal_power_level = smu8_get_dal_power_level,
 	.get_performance_level = smu8_get_performance_level,
 	.get_current_shallow_sleep_clocks = smu8_get_current_shallow_sleep_clocks,
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c
index 9ace863792d4..1b8a57d98759 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c
@@ -4825,146 +4825,6 @@ static int vega10_emit_clock_levels(struct pp_hwmgr *hwmgr,
 	return ret;
 }
 
-static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr,
-		enum pp_clock_type type, char *buf)
-{
-	struct vega10_hwmgr *data = hwmgr->backend;
-	struct vega10_single_dpm_table *sclk_table = &(data->dpm_table.gfx_table);
-	struct vega10_single_dpm_table *mclk_table = &(data->dpm_table.mem_table);
-	struct vega10_single_dpm_table *soc_table = &(data->dpm_table.soc_table);
-	struct vega10_single_dpm_table *dcef_table = &(data->dpm_table.dcef_table);
-	struct vega10_odn_clock_voltage_dependency_table *podn_vdd_dep = NULL;
-	uint32_t gen_speed, lane_width, current_gen_speed, current_lane_width;
-	PPTable_t *pptable = &(data->smc_state_table.pp_table);
-
-	int i, ret, now,  size = 0, count = 0;
-
-	switch (type) {
-	case PP_SCLK:
-		if (data->registry_data.sclk_dpm_key_disabled)
-			break;
-
-		ret = smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetCurrentGfxclkIndex, &now);
-		if (ret)
-			break;
-
-		if (hwmgr->pp_one_vf &&
-		    (hwmgr->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK))
-			count = 5;
-		else
-			count = sclk_table->count;
-		for (i = 0; i < count; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-					i, sclk_table->dpm_levels[i].value / 100,
-					(i == now) ? "*" : "");
-		break;
-	case PP_MCLK:
-		if (data->registry_data.mclk_dpm_key_disabled)
-			break;
-
-		ret = smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetCurrentUclkIndex, &now);
-		if (ret)
-			break;
-
-		for (i = 0; i < mclk_table->count; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-					i, mclk_table->dpm_levels[i].value / 100,
-					(i == now) ? "*" : "");
-		break;
-	case PP_SOCCLK:
-		if (data->registry_data.socclk_dpm_key_disabled)
-			break;
-
-		ret = smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetCurrentSocclkIndex, &now);
-		if (ret)
-			break;
-
-		for (i = 0; i < soc_table->count; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-					i, soc_table->dpm_levels[i].value / 100,
-					(i == now) ? "*" : "");
-		break;
-	case PP_DCEFCLK:
-		if (data->registry_data.dcefclk_dpm_key_disabled)
-			break;
-
-		ret = smum_send_msg_to_smc_with_parameter(hwmgr,
-				PPSMC_MSG_GetClockFreqMHz, CLK_DCEFCLK, &now);
-		if (ret)
-			break;
-
-		for (i = 0; i < dcef_table->count; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-					i, dcef_table->dpm_levels[i].value / 100,
-					(dcef_table->dpm_levels[i].value / 100 == now) ?
-					"*" : "");
-		break;
-	case PP_PCIE:
-		current_gen_speed =
-			vega10_get_current_pcie_link_speed_level(hwmgr);
-		current_lane_width =
-			vega10_get_current_pcie_link_width_level(hwmgr);
-		for (i = 0; i < NUM_LINK_LEVELS; i++) {
-			gen_speed = pptable->PcieGenSpeed[i];
-			lane_width = pptable->PcieLaneCount[i];
-
-			size += sprintf(buf + size, "%d: %s %s %s\n", i,
-					(gen_speed == 0) ? "2.5GT/s," :
-					(gen_speed == 1) ? "5.0GT/s," :
-					(gen_speed == 2) ? "8.0GT/s," :
-					(gen_speed == 3) ? "16.0GT/s," : "",
-					(lane_width == 1) ? "x1" :
-					(lane_width == 2) ? "x2" :
-					(lane_width == 3) ? "x4" :
-					(lane_width == 4) ? "x8" :
-					(lane_width == 5) ? "x12" :
-					(lane_width == 6) ? "x16" : "",
-					(current_gen_speed == gen_speed) &&
-					(current_lane_width == lane_width) ?
-					"*" : "");
-		}
-		break;
-
-	case OD_SCLK:
-		if (hwmgr->od_enabled) {
-			size += sprintf(buf + size, "%s:\n", "OD_SCLK");
-			podn_vdd_dep = &data->odn_dpm_table.vdd_dep_on_sclk;
-			for (i = 0; i < podn_vdd_dep->count; i++)
-				size += sprintf(buf + size, "%d: %10uMhz %10umV\n",
-					i, podn_vdd_dep->entries[i].clk / 100,
-						podn_vdd_dep->entries[i].vddc);
-		}
-		break;
-	case OD_MCLK:
-		if (hwmgr->od_enabled) {
-			size += sprintf(buf + size, "%s:\n", "OD_MCLK");
-			podn_vdd_dep = &data->odn_dpm_table.vdd_dep_on_mclk;
-			for (i = 0; i < podn_vdd_dep->count; i++)
-				size += sprintf(buf + size, "%d: %10uMhz %10umV\n",
-					i, podn_vdd_dep->entries[i].clk/100,
-						podn_vdd_dep->entries[i].vddc);
-		}
-		break;
-	case OD_RANGE:
-		if (hwmgr->od_enabled) {
-			size += sprintf(buf + size, "%s:\n", "OD_RANGE");
-			size += sprintf(buf + size, "SCLK: %7uMHz %10uMHz\n",
-				data->golden_dpm_table.gfx_table.dpm_levels[0].value/100,
-				hwmgr->platform_descriptor.overdriveLimit.engineClock/100);
-			size += sprintf(buf + size, "MCLK: %7uMHz %10uMHz\n",
-				data->golden_dpm_table.mem_table.dpm_levels[0].value/100,
-				hwmgr->platform_descriptor.overdriveLimit.memoryClock/100);
-			size += sprintf(buf + size, "VDDC: %7umV %11umV\n",
-				data->odn_dpm_table.min_vddc,
-				data->odn_dpm_table.max_vddc);
-		}
-		break;
-	default:
-		break;
-	}
-	return size;
-}
-
 static int vega10_display_configuration_changed_task(struct pp_hwmgr *hwmgr)
 {
 	struct vega10_hwmgr *data = hwmgr->backend;
@@ -5792,7 +5652,6 @@ static const struct pp_hwmgr_func vega10_hwmgr_funcs = {
 	.display_clock_voltage_request = vega10_display_clock_voltage_request,
 	.force_clock_level = vega10_force_clock_level,
 	.emit_clock_levels = vega10_emit_clock_levels,
-	.print_clock_levels = vega10_print_clock_levels,
 	.display_config_changed = vega10_display_configuration_changed_task,
 	.powergate_uvd = vega10_power_gate_uvd,
 	.powergate_vce = vega10_power_gate_vce,
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c
index 10fd4e9f016c..5a987a535e73 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c
@@ -2271,11 +2271,12 @@ static int vega12_get_current_pcie_link_speed(struct pp_hwmgr *hwmgr)
 	return link_speed[speed_level];
 }
 
-static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr,
-		enum pp_clock_type type, char *buf)
+static int vega12_emit_clock_levels(struct pp_hwmgr *hwmgr,
+				    enum pp_clock_type type, char *buf,
+				    int *offset)
 {
-	int i, now, size = 0;
 	struct pp_clock_levels_with_latency clocks;
+	int i, now, size = *offset;
 
 	switch (type) {
 	case PP_SCLK:
@@ -2289,9 +2290,13 @@ static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr,
 				"Attempt to get gfx clk levels Failed!",
 				return -1);
 		for (i = 0; i < clocks.num_levels; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-				i, clocks.data[i].clocks_in_khz / 1000,
-				(clocks.data[i].clocks_in_khz / 1000 == now / 100) ? "*" : "");
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n", i,
+				clocks.data[i].clocks_in_khz / 1000,
+				(clocks.data[i].clocks_in_khz / 1000 ==
+				 now / 100) ?
+					"*" :
+					"");
 		break;
 
 	case PP_MCLK:
@@ -2305,9 +2310,13 @@ static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr,
 				"Attempt to get memory clk levels Failed!",
 				return -1);
 		for (i = 0; i < clocks.num_levels; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-				i, clocks.data[i].clocks_in_khz / 1000,
-				(clocks.data[i].clocks_in_khz / 1000 == now / 100) ? "*" : "");
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n", i,
+				clocks.data[i].clocks_in_khz / 1000,
+				(clocks.data[i].clocks_in_khz / 1000 ==
+				 now / 100) ?
+					"*" :
+					"");
 		break;
 
 	case PP_SOCCLK:
@@ -2323,9 +2332,12 @@ static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr,
 				"Attempt to get soc clk levels Failed!",
 				return -1);
 		for (i = 0; i < clocks.num_levels; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-				i, clocks.data[i].clocks_in_khz / 1000,
-				(clocks.data[i].clocks_in_khz / 1000 == now) ? "*" : "");
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n", i,
+				clocks.data[i].clocks_in_khz / 1000,
+				(clocks.data[i].clocks_in_khz / 1000 == now) ?
+					"*" :
+					"");
 		break;
 
 	case PP_DCEFCLK:
@@ -2341,9 +2353,12 @@ static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr,
 				"Attempt to get dcef clk levels Failed!",
 				return -1);
 		for (i = 0; i < clocks.num_levels; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-				i, clocks.data[i].clocks_in_khz / 1000,
-				(clocks.data[i].clocks_in_khz / 1000 == now) ? "*" : "");
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n", i,
+				clocks.data[i].clocks_in_khz / 1000,
+				(clocks.data[i].clocks_in_khz / 1000 == now) ?
+					"*" :
+					"");
 		break;
 
 	case PP_PCIE:
@@ -2352,7 +2367,10 @@ static int vega12_print_clock_levels(struct pp_hwmgr *hwmgr,
 	default:
 		break;
 	}
-	return size;
+
+	*offset = size;
+
+	return 0;
 }
 
 static int vega12_apply_clocks_adjust_rules(struct pp_hwmgr *hwmgr)
@@ -2951,7 +2969,7 @@ static const struct pp_hwmgr_func vega12_hwmgr_funcs = {
 	.set_watermarks_for_clocks_ranges = vega12_set_watermarks_for_clocks_ranges,
 	.display_clock_voltage_request = vega12_display_clock_voltage_request,
 	.force_clock_level = vega12_force_clock_level,
-	.print_clock_levels = vega12_print_clock_levels,
+	.emit_clock_levels = vega12_emit_clock_levels,
 	.apply_clocks_adjust_rules =
 		vega12_apply_clocks_adjust_rules,
 	.pre_display_config_changed =
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c
index baf251fe5d82..5193b7d0e11b 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c
@@ -3362,8 +3362,9 @@ static int vega20_get_current_pcie_link_speed(struct pp_hwmgr *hwmgr)
 	return link_speed[speed_level];
 }
 
-static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
-		enum pp_clock_type type, char *buf)
+static int vega20_emit_clock_levels(struct pp_hwmgr *hwmgr,
+				    enum pp_clock_type type, char *buf,
+				    int *offset)
 {
 	struct vega20_hwmgr *data =
 			(struct vega20_hwmgr *)(hwmgr->backend);
@@ -3375,7 +3376,7 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
 	struct pp_clock_levels_with_latency clocks;
 	struct vega20_single_dpm_table *fclk_dpm_table =
 			&(data->dpm_table.fclk_table);
-	int i, now, size = 0;
+	int i, now, size = *offset;
 	int ret = 0;
 	uint32_t gen_speed, lane_width, current_gen_speed, current_lane_width;
 
@@ -3387,15 +3388,19 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
 				return ret);
 
 		if (vega20_get_sclks(hwmgr, &clocks)) {
-			size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n",
-				now / 100);
+			size += sysfs_emit_at(buf, size,
+					      "0: %uMhz * (DPM disabled)\n",
+					      now / 100);
 			break;
 		}
 
 		for (i = 0; i < clocks.num_levels; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-				i, clocks.data[i].clocks_in_khz / 1000,
-				(clocks.data[i].clocks_in_khz == now * 10) ? "*" : "");
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n", i,
+				clocks.data[i].clocks_in_khz / 1000,
+				(clocks.data[i].clocks_in_khz == now * 10) ?
+					"*" :
+					"");
 		break;
 
 	case PP_MCLK:
@@ -3405,15 +3410,19 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
 				return ret);
 
 		if (vega20_get_memclocks(hwmgr, &clocks)) {
-			size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n",
-				now / 100);
+			size += sysfs_emit_at(buf, size,
+					      "0: %uMhz * (DPM disabled)\n",
+					      now / 100);
 			break;
 		}
 
 		for (i = 0; i < clocks.num_levels; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-				i, clocks.data[i].clocks_in_khz / 1000,
-				(clocks.data[i].clocks_in_khz == now * 10) ? "*" : "");
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n", i,
+				clocks.data[i].clocks_in_khz / 1000,
+				(clocks.data[i].clocks_in_khz == now * 10) ?
+					"*" :
+					"");
 		break;
 
 	case PP_SOCCLK:
@@ -3423,15 +3432,19 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
 				return ret);
 
 		if (vega20_get_socclocks(hwmgr, &clocks)) {
-			size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n",
-				now / 100);
+			size += sysfs_emit_at(buf, size,
+					      "0: %uMhz * (DPM disabled)\n",
+					      now / 100);
 			break;
 		}
 
 		for (i = 0; i < clocks.num_levels; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-				i, clocks.data[i].clocks_in_khz / 1000,
-				(clocks.data[i].clocks_in_khz == now * 10) ? "*" : "");
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n", i,
+				clocks.data[i].clocks_in_khz / 1000,
+				(clocks.data[i].clocks_in_khz == now * 10) ?
+					"*" :
+					"");
 		break;
 
 	case PP_FCLK:
@@ -3441,9 +3454,13 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
 				return ret);
 
 		for (i = 0; i < fclk_dpm_table->count; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-				i, fclk_dpm_table->dpm_levels[i].value,
-				fclk_dpm_table->dpm_levels[i].value == (now / 100) ? "*" : "");
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n", i,
+				fclk_dpm_table->dpm_levels[i].value,
+				fclk_dpm_table->dpm_levels[i].value ==
+						(now / 100) ?
+					"*" :
+					"");
 		break;
 
 	case PP_DCEFCLK:
@@ -3453,15 +3470,19 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
 				return ret);
 
 		if (vega20_get_dcefclocks(hwmgr, &clocks)) {
-			size += sprintf(buf + size, "0: %uMhz * (DPM disabled)\n",
-				now / 100);
+			size += sysfs_emit_at(buf, size,
+					      "0: %uMhz * (DPM disabled)\n",
+					      now / 100);
 			break;
 		}
 
 		for (i = 0; i < clocks.num_levels; i++)
-			size += sprintf(buf + size, "%d: %uMhz %s\n",
-				i, clocks.data[i].clocks_in_khz / 1000,
-				(clocks.data[i].clocks_in_khz == now * 10) ? "*" : "");
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n", i,
+				clocks.data[i].clocks_in_khz / 1000,
+				(clocks.data[i].clocks_in_khz == now * 10) ?
+					"*" :
+					"");
 		break;
 
 	case PP_PCIE:
@@ -3473,40 +3494,45 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
 			gen_speed = pptable->PcieGenSpeed[i];
 			lane_width = pptable->PcieLaneCount[i];
 
-			size += sprintf(buf + size, "%d: %s %s %dMhz %s\n", i,
-					(gen_speed == 0) ? "2.5GT/s," :
-					(gen_speed == 1) ? "5.0GT/s," :
-					(gen_speed == 2) ? "8.0GT/s," :
-					(gen_speed == 3) ? "16.0GT/s," : "",
-					(lane_width == 1) ? "x1" :
-					(lane_width == 2) ? "x2" :
-					(lane_width == 3) ? "x4" :
-					(lane_width == 4) ? "x8" :
-					(lane_width == 5) ? "x12" :
-					(lane_width == 6) ? "x16" : "",
-					pptable->LclkFreq[i],
-					(current_gen_speed == gen_speed) &&
-					(current_lane_width == lane_width) ?
-					"*" : "");
+			size += sysfs_emit_at(
+				buf, size, "%d: %s %s %dMhz %s\n", i,
+				(gen_speed == 0) ? "2.5GT/s," :
+				(gen_speed == 1) ? "5.0GT/s," :
+				(gen_speed == 2) ? "8.0GT/s," :
+				(gen_speed == 3) ? "16.0GT/s," :
+						   "",
+				(lane_width == 1) ? "x1" :
+				(lane_width == 2) ? "x2" :
+				(lane_width == 3) ? "x4" :
+				(lane_width == 4) ? "x8" :
+				(lane_width == 5) ? "x12" :
+				(lane_width == 6) ? "x16" :
+						    "",
+				pptable->LclkFreq[i],
+				(current_gen_speed == gen_speed) &&
+						(current_lane_width ==
+						 lane_width) ?
+					"*" :
+					"");
 		}
 		break;
 
 	case OD_SCLK:
 		if (od8_settings[OD8_SETTING_GFXCLK_FMIN].feature_id &&
 		    od8_settings[OD8_SETTING_GFXCLK_FMAX].feature_id) {
-			size += sprintf(buf + size, "%s:\n", "OD_SCLK");
-			size += sprintf(buf + size, "0: %10uMhz\n",
-				od_table->GfxclkFmin);
-			size += sprintf(buf + size, "1: %10uMhz\n",
-				od_table->GfxclkFmax);
+			size += sysfs_emit_at(buf, size, "%s:\n", "OD_SCLK");
+			size += sysfs_emit_at(buf, size, "0: %10uMhz\n",
+					      od_table->GfxclkFmin);
+			size += sysfs_emit_at(buf, size, "1: %10uMhz\n",
+					      od_table->GfxclkFmax);
 		}
 		break;
 
 	case OD_MCLK:
 		if (od8_settings[OD8_SETTING_UCLK_FMAX].feature_id) {
-			size += sprintf(buf + size, "%s:\n", "OD_MCLK");
-			size += sprintf(buf + size, "1: %10uMhz\n",
-				od_table->UclkFmax);
+			size += sysfs_emit_at(buf, size, "%s:\n", "OD_MCLK");
+			size += sysfs_emit_at(buf, size, "1: %10uMhz\n",
+					      od_table->UclkFmax);
 		}
 
 		break;
@@ -3518,32 +3544,38 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
 		    od8_settings[OD8_SETTING_GFXCLK_VOLTAGE1].feature_id &&
 		    od8_settings[OD8_SETTING_GFXCLK_VOLTAGE2].feature_id &&
 		    od8_settings[OD8_SETTING_GFXCLK_VOLTAGE3].feature_id) {
-			size += sprintf(buf + size, "%s:\n", "OD_VDDC_CURVE");
-			size += sprintf(buf + size, "0: %10uMhz %10dmV\n",
-				od_table->GfxclkFreq1,
-				od_table->GfxclkVolt1 / VOLTAGE_SCALE);
-			size += sprintf(buf + size, "1: %10uMhz %10dmV\n",
-				od_table->GfxclkFreq2,
-				od_table->GfxclkVolt2 / VOLTAGE_SCALE);
-			size += sprintf(buf + size, "2: %10uMhz %10dmV\n",
-				od_table->GfxclkFreq3,
-				od_table->GfxclkVolt3 / VOLTAGE_SCALE);
+			size += sysfs_emit_at(buf, size, "%s:\n",
+					      "OD_VDDC_CURVE");
+			size += sysfs_emit_at(buf, size, "0: %10uMhz %10dmV\n",
+					      od_table->GfxclkFreq1,
+					      od_table->GfxclkVolt1 /
+						      VOLTAGE_SCALE);
+			size += sysfs_emit_at(buf, size, "1: %10uMhz %10dmV\n",
+					      od_table->GfxclkFreq2,
+					      od_table->GfxclkVolt2 /
+						      VOLTAGE_SCALE);
+			size += sysfs_emit_at(buf, size, "2: %10uMhz %10dmV\n",
+					      od_table->GfxclkFreq3,
+					      od_table->GfxclkVolt3 /
+						      VOLTAGE_SCALE);
 		}
 
 		break;
 
 	case OD_RANGE:
-		size += sprintf(buf + size, "%s:\n", "OD_RANGE");
+		size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE");
 
 		if (od8_settings[OD8_SETTING_GFXCLK_FMIN].feature_id &&
 		    od8_settings[OD8_SETTING_GFXCLK_FMAX].feature_id) {
-			size += sprintf(buf + size, "SCLK: %7uMhz %10uMhz\n",
+			size += sysfs_emit_at(
+				buf, size, "SCLK: %7uMhz %10uMhz\n",
 				od8_settings[OD8_SETTING_GFXCLK_FMIN].min_value,
 				od8_settings[OD8_SETTING_GFXCLK_FMAX].max_value);
 		}
 
 		if (od8_settings[OD8_SETTING_UCLK_FMAX].feature_id) {
-			size += sprintf(buf + size, "MCLK: %7uMhz %10uMhz\n",
+			size += sysfs_emit_at(
+				buf, size, "MCLK: %7uMhz %10uMhz\n",
 				od8_settings[OD8_SETTING_UCLK_FMAX].min_value,
 				od8_settings[OD8_SETTING_UCLK_FMAX].max_value);
 		}
@@ -3554,31 +3586,52 @@ static int vega20_print_clock_levels(struct pp_hwmgr *hwmgr,
 		    od8_settings[OD8_SETTING_GFXCLK_VOLTAGE1].feature_id &&
 		    od8_settings[OD8_SETTING_GFXCLK_VOLTAGE2].feature_id &&
 		    od8_settings[OD8_SETTING_GFXCLK_VOLTAGE3].feature_id) {
-			size += sprintf(buf + size, "VDDC_CURVE_SCLK[0]: %7uMhz %10uMhz\n",
+			size += sysfs_emit_at(
+				buf, size,
+				"VDDC_CURVE_SCLK[0]: %7uMhz %10uMhz\n",
 				od8_settings[OD8_SETTING_GFXCLK_FREQ1].min_value,
-				od8_settings[OD8_SETTING_GFXCLK_FREQ1].max_value);
-			size += sprintf(buf + size, "VDDC_CURVE_VOLT[0]: %7dmV %11dmV\n",
-				od8_settings[OD8_SETTING_GFXCLK_VOLTAGE1].min_value,
-				od8_settings[OD8_SETTING_GFXCLK_VOLTAGE1].max_value);
-			size += sprintf(buf + size, "VDDC_CURVE_SCLK[1]: %7uMhz %10uMhz\n",
+				od8_settings[OD8_SETTING_GFXCLK_FREQ1]
+					.max_value);
+			size += sysfs_emit_at(
+				buf, size, "VDDC_CURVE_VOLT[0]: %7dmV %11dmV\n",
+				od8_settings[OD8_SETTING_GFXCLK_VOLTAGE1]
+					.min_value,
+				od8_settings[OD8_SETTING_GFXCLK_VOLTAGE1]
+					.max_value);
+			size += sysfs_emit_at(
+				buf, size,
+				"VDDC_CURVE_SCLK[1]: %7uMhz %10uMhz\n",
 				od8_settings[OD8_SETTING_GFXCLK_FREQ2].min_value,
-				od8_settings[OD8_SETTING_GFXCLK_FREQ2].max_value);
-			size += sprintf(buf + size, "VDDC_CURVE_VOLT[1]: %7dmV %11dmV\n",
-				od8_settings[OD8_SETTING_GFXCLK_VOLTAGE2].min_value,
-				od8_settings[OD8_SETTING_GFXCLK_VOLTAGE2].max_value);
-			size += sprintf(buf + size, "VDDC_CURVE_SCLK[2]: %7uMhz %10uMhz\n",
+				od8_settings[OD8_SETTING_GFXCLK_FREQ2]
+					.max_value);
+			size += sysfs_emit_at(
+				buf, size, "VDDC_CURVE_VOLT[1]: %7dmV %11dmV\n",
+				od8_settings[OD8_SETTING_GFXCLK_VOLTAGE2]
+					.min_value,
+				od8_settings[OD8_SETTING_GFXCLK_VOLTAGE2]
+					.max_value);
+			size += sysfs_emit_at(
+				buf, size,
+				"VDDC_CURVE_SCLK[2]: %7uMhz %10uMhz\n",
 				od8_settings[OD8_SETTING_GFXCLK_FREQ3].min_value,
-				od8_settings[OD8_SETTING_GFXCLK_FREQ3].max_value);
-			size += sprintf(buf + size, "VDDC_CURVE_VOLT[2]: %7dmV %11dmV\n",
-				od8_settings[OD8_SETTING_GFXCLK_VOLTAGE3].min_value,
-				od8_settings[OD8_SETTING_GFXCLK_VOLTAGE3].max_value);
+				od8_settings[OD8_SETTING_GFXCLK_FREQ3]
+					.max_value);
+			size += sysfs_emit_at(
+				buf, size, "VDDC_CURVE_VOLT[2]: %7dmV %11dmV\n",
+				od8_settings[OD8_SETTING_GFXCLK_VOLTAGE3]
+					.min_value,
+				od8_settings[OD8_SETTING_GFXCLK_VOLTAGE3]
+					.max_value);
 		}
 
 		break;
 	default:
 		break;
 	}
-	return size;
+
+	*offset = size;
+
+	return 0;
 }
 
 static int vega20_set_uclk_to_highest_dpm_level(struct pp_hwmgr *hwmgr,
@@ -4412,7 +4465,7 @@ static const struct pp_hwmgr_func vega20_hwmgr_funcs = {
 	.odn_edit_dpm_table = vega20_odn_edit_dpm_table,
 	/* for sysfs to retrive/set gfxclk/memclk */
 	.force_clock_level = vega20_force_clock_level,
-	.print_clock_levels = vega20_print_clock_levels,
+	.emit_clock_levels = vega20_emit_clock_levels,
 	.read_sensor = vega20_read_sensor,
 	.get_ppfeature_status = vega20_get_ppfeature_status,
 	.set_ppfeature_status = vega20_set_ppfeature_status,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
index f51fa265230b..b7dfd98d387d 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
@@ -3014,19 +3014,6 @@ static int smu_set_power_limit(void *handle, uint32_t limit_type, uint32_t limit
 	return 0;
 }
 
-static int smu_print_smuclk_levels(struct smu_context *smu, enum smu_clk_type clk_type, char *buf)
-{
-	int ret = 0;
-
-	if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled)
-		return -EOPNOTSUPP;
-
-	if (smu->ppt_funcs->print_clk_levels)
-		ret = smu->ppt_funcs->print_clk_levels(smu, clk_type, buf);
-
-	return ret;
-}
-
 static enum smu_clk_type smu_convert_to_smuclk(enum pp_clock_type type)
 {
 	enum smu_clk_type clk_type;
@@ -3091,20 +3078,6 @@ static enum smu_clk_type smu_convert_to_smuclk(enum pp_clock_type type)
 	return clk_type;
 }
 
-static int smu_print_ppclk_levels(void *handle,
-				  enum pp_clock_type type,
-				  char *buf)
-{
-	struct smu_context *smu = handle;
-	enum smu_clk_type clk_type;
-
-	clk_type = smu_convert_to_smuclk(type);
-	if (clk_type == SMU_CLK_COUNT)
-		return -EINVAL;
-
-	return smu_print_smuclk_levels(smu, clk_type, buf);
-}
-
 static int smu_emit_ppclk_levels(void *handle, enum pp_clock_type type, char *buf, int *offset)
 {
 	struct smu_context *smu = handle;
@@ -3941,7 +3914,6 @@ static const struct amd_pm_funcs swsmu_pm_funcs = {
 	.set_fan_speed_pwm   = smu_set_fan_speed_pwm,
 	.get_fan_speed_pwm   = smu_get_fan_speed_pwm,
 	.force_clock_level       = smu_force_ppclk_levels,
-	.print_clock_levels      = smu_print_ppclk_levels,
 	.emit_clock_levels       = smu_emit_ppclk_levels,
 	.force_performance_level = smu_force_performance_level,
 	.read_sensor             = smu_read_sensor,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
index 8815fc70b63b..a4e8e488030f 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
@@ -292,6 +292,37 @@ struct smu_clock_info {
 	uint32_t max_bus_bandwidth;
 };
 
+#define SMU_MAX_DPM_LEVELS 16
+
+struct smu_dpm_clk_level {
+	bool		enabled;
+	uint32_t	value;
+};
+
+#define SMU_DPM_TABLE_FINE_GRAINED	BIT(0)
+
+struct smu_dpm_table {
+	enum smu_clk_type	clk_type;
+	uint32_t		count;
+	uint32_t		flags;
+	struct smu_dpm_clk_level dpm_levels[SMU_MAX_DPM_LEVELS];
+};
+
+#define SMU_DPM_TABLE_MIN(table) \
+	((table)->count > 0 ? (table)->dpm_levels[0].value : 0)
+
+#define SMU_DPM_TABLE_MAX(table) \
+	((table)->count > 0 ? (table)->dpm_levels[(table)->count - 1].value : 0)
+
+#define SMU_MAX_PCIE_LEVELS 3
+
+struct smu_pcie_table {
+	uint8_t pcie_gen[SMU_MAX_PCIE_LEVELS];
+	uint8_t pcie_lane[SMU_MAX_PCIE_LEVELS];
+	uint16_t lclk_freq[SMU_MAX_PCIE_LEVELS];
+	uint32_t lclk_levels;
+};
+
 struct smu_bios_boot_up_values {
 	uint32_t			revision;
 	uint32_t			gfxclk;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_12_pmfw.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_12_pmfw.h
index dd30d96e1ca2..acf7b2f43b0e 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_12_pmfw.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_12_pmfw.h
@@ -189,6 +189,13 @@ typedef enum {
   SVI_MAX_TEMP_ENTRIES,   // 13
 } SVI_TEMP_e;
 
+typedef enum{
+  SYSTEM_POWER_UBB_POWER,
+  SYSTEM_POWER_UBB_POWER_THRESHOLD,
+  SYSTEM_POWER_MAX_ENTRIES_WO_RESERVED,
+  SYSTEM_POWER_MAX_ENTRIES  = 4
+} SYSTEM_POWER_e;
+
 #define SMU_METRICS_TABLE_VERSION 0x15
 
 #define SMU_SYSTEM_METRICS_TABLE_VERSION 0x1
@@ -310,6 +317,7 @@ typedef struct {
   uint32_t NodePowerLimit;
   uint32_t NodePower;
   uint32_t GlobalPPTResidencyAcc;
+  uint16_t SystemPower[SYSTEM_POWER_MAX_ENTRIES];           // UBB Current Power and Power Threshold
 } SystemMetricsTable_t;
 #pragma pack(pop)
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v11_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v11_0.h
index 56ae555bb52a..d18934c6fbd5 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v11_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v11_0.h
@@ -53,7 +53,6 @@
 
 #define SMU11_TOOL_SIZE			0x19000
 
-#define MAX_DPM_LEVELS 16
 #define MAX_PCIE_CONF 2
 
 #define CTF_OFFSET_EDGE			5
@@ -80,39 +79,21 @@ struct smu_11_0_max_sustainable_clocks {
 	uint32_t soc_clock;
 };
 
-struct smu_11_0_dpm_clk_level {
-	bool				enabled;
-	uint32_t			value;
-};
-
-struct smu_11_0_dpm_table {
-	uint32_t			min;        /* MHz */
-	uint32_t			max;        /* MHz */
-	uint32_t			count;
-	bool				is_fine_grained;
-	struct smu_11_0_dpm_clk_level	dpm_levels[MAX_DPM_LEVELS];
-};
-
-struct smu_11_0_pcie_table {
-	uint8_t  pcie_gen[MAX_PCIE_CONF];
-	uint8_t  pcie_lane[MAX_PCIE_CONF];
-};
-
 struct smu_11_0_dpm_tables {
-	struct smu_11_0_dpm_table        soc_table;
-	struct smu_11_0_dpm_table        gfx_table;
-	struct smu_11_0_dpm_table        uclk_table;
-	struct smu_11_0_dpm_table        eclk_table;
-	struct smu_11_0_dpm_table        vclk_table;
-	struct smu_11_0_dpm_table        vclk1_table;
-	struct smu_11_0_dpm_table        dclk_table;
-	struct smu_11_0_dpm_table        dclk1_table;
-	struct smu_11_0_dpm_table        dcef_table;
-	struct smu_11_0_dpm_table        pixel_table;
-	struct smu_11_0_dpm_table        display_table;
-	struct smu_11_0_dpm_table        phy_table;
-	struct smu_11_0_dpm_table        fclk_table;
-	struct smu_11_0_pcie_table       pcie_table;
+	struct smu_dpm_table             soc_table;
+	struct smu_dpm_table             gfx_table;
+	struct smu_dpm_table             uclk_table;
+	struct smu_dpm_table             eclk_table;
+	struct smu_dpm_table             vclk_table;
+	struct smu_dpm_table             vclk1_table;
+	struct smu_dpm_table             dclk_table;
+	struct smu_dpm_table             dclk1_table;
+	struct smu_dpm_table             dcef_table;
+	struct smu_dpm_table             pixel_table;
+	struct smu_dpm_table             display_table;
+	struct smu_dpm_table             phy_table;
+	struct smu_dpm_table             fclk_table;
+	struct smu_pcie_table            pcie_table;
 };
 
 struct smu_11_0_dpm_context {
@@ -279,7 +260,7 @@ int smu_v11_0_get_dpm_level_count(struct smu_context *smu,
 
 int smu_v11_0_set_single_dpm_table(struct smu_context *smu,
 				   enum smu_clk_type clk_type,
-				   struct smu_11_0_dpm_table *single_dpm_table);
+				   struct smu_dpm_table *single_dpm_table);
 
 int smu_v11_0_get_current_pcie_link_width_level(struct smu_context *smu);
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
index 4263798d716b..c35cbb2aee93 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
@@ -44,7 +44,6 @@
 
 #define SMU13_TOOL_SIZE			0x19000
 
-#define MAX_DPM_LEVELS 16
 #define MAX_PCIE_CONF 3
 
 #define CTF_OFFSET_EDGE			5
@@ -72,39 +71,19 @@ struct smu_13_0_max_sustainable_clocks {
 	uint32_t soc_clock;
 };
 
-struct smu_13_0_dpm_clk_level {
-	bool				enabled;
-	uint32_t			value;
-};
-
-struct smu_13_0_dpm_table {
-	uint32_t			min;        /* MHz */
-	uint32_t			max;        /* MHz */
-	uint32_t			count;
-	bool				is_fine_grained;
-	struct smu_13_0_dpm_clk_level	dpm_levels[MAX_DPM_LEVELS];
-};
-
-struct smu_13_0_pcie_table {
-	uint8_t  pcie_gen[MAX_PCIE_CONF];
-	uint8_t  pcie_lane[MAX_PCIE_CONF];
-	uint16_t clk_freq[MAX_PCIE_CONF];
-	uint32_t num_of_link_levels;
-};
-
 struct smu_13_0_dpm_tables {
-	struct smu_13_0_dpm_table        soc_table;
-	struct smu_13_0_dpm_table        gfx_table;
-	struct smu_13_0_dpm_table        uclk_table;
-	struct smu_13_0_dpm_table        eclk_table;
-	struct smu_13_0_dpm_table        vclk_table;
-	struct smu_13_0_dpm_table        dclk_table;
-	struct smu_13_0_dpm_table        dcef_table;
-	struct smu_13_0_dpm_table        pixel_table;
-	struct smu_13_0_dpm_table        display_table;
-	struct smu_13_0_dpm_table        phy_table;
-	struct smu_13_0_dpm_table        fclk_table;
-	struct smu_13_0_pcie_table       pcie_table;
+	struct smu_dpm_table        soc_table;
+	struct smu_dpm_table        gfx_table;
+	struct smu_dpm_table        uclk_table;
+	struct smu_dpm_table        eclk_table;
+	struct smu_dpm_table        vclk_table;
+	struct smu_dpm_table        dclk_table;
+	struct smu_dpm_table        dcef_table;
+	struct smu_dpm_table        pixel_table;
+	struct smu_dpm_table        display_table;
+	struct smu_dpm_table        phy_table;
+	struct smu_dpm_table        fclk_table;
+	struct smu_pcie_table        pcie_table;
 };
 
 struct smu_13_0_dpm_context {
@@ -226,7 +205,7 @@ int smu_v13_0_set_power_source(struct smu_context *smu,
 
 int smu_v13_0_set_single_dpm_table(struct smu_context *smu,
 				   enum smu_clk_type clk_type,
-				   struct smu_13_0_dpm_table *single_dpm_table);
+				   struct smu_dpm_table *single_dpm_table);
 
 int smu_v13_0_get_dpm_freq_by_index(struct smu_context *smu,
 				    enum smu_clk_type clk_type, uint16_t level,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h
index 29a4583db873..4ecec85b8404 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h
@@ -43,7 +43,6 @@
 #define smnMP1_FIRMWARE_FLAGS		0x3010024
 #define smnMP1_PUB_CTRL			0x3010d10
 
-#define MAX_DPM_LEVELS 16
 #define MAX_PCIE_CONF 3
 
 #define SMU14_TOOL_SIZE			0x19000
@@ -67,39 +66,19 @@ struct smu_14_0_max_sustainable_clocks {
 	uint32_t soc_clock;
 };
 
-struct smu_14_0_dpm_clk_level {
-	bool				enabled;
-	uint32_t			value;
-};
-
-struct smu_14_0_dpm_table {
-	uint32_t			min;        /* MHz */
-	uint32_t			max;        /* MHz */
-	uint32_t			count;
-	bool				is_fine_grained;
-	struct smu_14_0_dpm_clk_level	dpm_levels[MAX_DPM_LEVELS];
-};
-
-struct smu_14_0_pcie_table {
-	uint8_t  pcie_gen[MAX_PCIE_CONF];
-	uint8_t  pcie_lane[MAX_PCIE_CONF];
-	uint16_t clk_freq[MAX_PCIE_CONF];
-	uint32_t num_of_link_levels;
-};
-
 struct smu_14_0_dpm_tables {
-	struct smu_14_0_dpm_table        soc_table;
-	struct smu_14_0_dpm_table        gfx_table;
-	struct smu_14_0_dpm_table        uclk_table;
-	struct smu_14_0_dpm_table        eclk_table;
-	struct smu_14_0_dpm_table        vclk_table;
-	struct smu_14_0_dpm_table        dclk_table;
-	struct smu_14_0_dpm_table        dcef_table;
-	struct smu_14_0_dpm_table        pixel_table;
-	struct smu_14_0_dpm_table        display_table;
-	struct smu_14_0_dpm_table        phy_table;
-	struct smu_14_0_dpm_table        fclk_table;
-	struct smu_14_0_pcie_table       pcie_table;
+	struct smu_dpm_table        soc_table;
+	struct smu_dpm_table        gfx_table;
+	struct smu_dpm_table        uclk_table;
+	struct smu_dpm_table        eclk_table;
+	struct smu_dpm_table        vclk_table;
+	struct smu_dpm_table        dclk_table;
+	struct smu_dpm_table        dcef_table;
+	struct smu_dpm_table        pixel_table;
+	struct smu_dpm_table        display_table;
+	struct smu_dpm_table        phy_table;
+	struct smu_dpm_table        fclk_table;
+	struct smu_pcie_table       pcie_table;
 };
 
 struct smu_14_0_dpm_context {
@@ -201,7 +180,7 @@ int smu_v14_0_set_power_source(struct smu_context *smu,
 
 int smu_v14_0_set_single_dpm_table(struct smu_context *smu,
 				   enum smu_clk_type clk_type,
-				   struct smu_14_0_dpm_table *single_dpm_table);
+				   struct smu_dpm_table *single_dpm_table);
 
 int smu_v14_0_gfx_ulv_control(struct smu_context *smu,
 			      bool enablement);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c
index 4fff78da81ff..0667a3f9282a 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c
@@ -74,9 +74,6 @@
 	FEATURE_DPM_FCLK_MASK | \
 	FEATURE_DPM_XGMI_MASK)
 
-/* possible frequency drift (1Mhz) */
-#define EPSILON				1
-
 #define smnPCIE_ESM_CTRL			0x111003D0
 
 #define mmCG_FDO_CTRL0_ARCT			0x8B
@@ -362,79 +359,75 @@ static int arcturus_set_default_dpm_table(struct smu_context *smu)
 {
 	struct smu_11_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
 	PPTable_t *driver_ppt = smu->smu_table.driver_pptable;
-	struct smu_11_0_dpm_table *dpm_table = NULL;
+	struct smu_dpm_table *dpm_table = NULL;
 	int ret = 0;
 
 	/* socclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.soc_table;
+	dpm_table->clk_type = SMU_SOCCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_SOCCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_SOCCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!driver_ppt->DpmDescriptor[PPCLK_SOCCLK].SnapToDiscrete;
+		if (!driver_ppt->DpmDescriptor[PPCLK_SOCCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.socclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* gfxclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.gfx_table;
+	dpm_table->clk_type = SMU_GFXCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_GFXCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!driver_ppt->DpmDescriptor[PPCLK_GFXCLK].SnapToDiscrete;
+		if (!driver_ppt->DpmDescriptor[PPCLK_GFXCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* memclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.uclk_table;
+	dpm_table->clk_type = SMU_UCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_UCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_UCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!driver_ppt->DpmDescriptor[PPCLK_UCLK].SnapToDiscrete;
+		if (!driver_ppt->DpmDescriptor[PPCLK_UCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.uclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* fclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.fclk_table;
+	dpm_table->clk_type = SMU_FCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_FCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_FCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!driver_ppt->DpmDescriptor[PPCLK_FCLK].SnapToDiscrete;
+		if (!driver_ppt->DpmDescriptor[PPCLK_FCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.fclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* XGMI PLPD is supported by 54.23.0 and onwards */
@@ -572,23 +565,20 @@ static int arcturus_populate_umd_state_clk(struct smu_context *smu)
 {
 	struct smu_11_0_dpm_context *dpm_context =
 				smu->smu_dpm.dpm_context;
-	struct smu_11_0_dpm_table *gfx_table =
-				&dpm_context->dpm_tables.gfx_table;
-	struct smu_11_0_dpm_table *mem_table =
-				&dpm_context->dpm_tables.uclk_table;
-	struct smu_11_0_dpm_table *soc_table =
-				&dpm_context->dpm_tables.soc_table;
+	struct smu_dpm_table *gfx_table = &dpm_context->dpm_tables.gfx_table;
+	struct smu_dpm_table *mem_table = &dpm_context->dpm_tables.uclk_table;
+	struct smu_dpm_table *soc_table = &dpm_context->dpm_tables.soc_table;
 	struct smu_umd_pstate_table *pstate_table =
 				&smu->pstate_table;
 
-	pstate_table->gfxclk_pstate.min = gfx_table->min;
-	pstate_table->gfxclk_pstate.peak = gfx_table->max;
+	pstate_table->gfxclk_pstate.min = SMU_DPM_TABLE_MIN(gfx_table);
+	pstate_table->gfxclk_pstate.peak = SMU_DPM_TABLE_MAX(gfx_table);
 
-	pstate_table->uclk_pstate.min = mem_table->min;
-	pstate_table->uclk_pstate.peak = mem_table->max;
+	pstate_table->uclk_pstate.min = SMU_DPM_TABLE_MIN(mem_table);
+	pstate_table->uclk_pstate.peak = SMU_DPM_TABLE_MAX(mem_table);
 
-	pstate_table->socclk_pstate.min = soc_table->min;
-	pstate_table->socclk_pstate.peak = soc_table->max;
+	pstate_table->socclk_pstate.min = SMU_DPM_TABLE_MIN(soc_table);
+	pstate_table->socclk_pstate.peak = SMU_DPM_TABLE_MAX(soc_table);
 
 	if (gfx_table->count > ARCTURUS_UMD_PSTATE_GFXCLK_LEVEL &&
 	    mem_table->count > ARCTURUS_UMD_PSTATE_MCLK_LEVEL &&
@@ -611,29 +601,6 @@ static int arcturus_populate_umd_state_clk(struct smu_context *smu)
 	return 0;
 }
 
-static void arcturus_get_clk_table(struct smu_context *smu,
-				   struct pp_clock_levels_with_latency *clocks,
-				   struct smu_11_0_dpm_table *dpm_table)
-{
-	uint32_t i;
-
-	clocks->num_levels = min_t(uint32_t,
-				   dpm_table->count,
-				   (uint32_t)PP_MAX_CLOCK_LEVELS);
-
-	for (i = 0; i < clocks->num_levels; i++) {
-		clocks->data[i].clocks_in_khz =
-			dpm_table->dpm_levels[i].value * 1000;
-		clocks->data[i].latency_in_us = 0;
-	}
-}
-
-static int arcturus_freqs_in_same_level(int32_t frequency1,
-					int32_t frequency2)
-{
-	return (abs(frequency1 - frequency2) <= EPSILON);
-}
-
 static int arcturus_get_smu_metrics_data(struct smu_context *smu,
 					 MetricsMember_t member,
 					 uint32_t *value)
@@ -800,15 +767,12 @@ static int arcturus_get_current_clk_freq_by_table(struct smu_context *smu,
 static int arcturus_emit_clk_levels(struct smu_context *smu,
 				    enum smu_clk_type type, char *buf, int *offset)
 {
-	int ret = 0;
-	struct pp_clock_levels_with_latency clocks;
-	struct smu_11_0_dpm_table *single_dpm_table;
-	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
-	struct smu_11_0_dpm_context *dpm_context = NULL;
+	struct smu_11_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
+	struct smu_dpm_table *single_dpm_table;
+	struct smu_pcie_table *pcie_table;
 	uint32_t gen_speed, lane_width;
-	uint32_t i, cur_value = 0;
-	bool freq_match;
-	unsigned int clock_mhz;
+	uint32_t cur_value = 0;
+	int ret = 0;
 	static const char attempt_string[] = "Attempt to get current";
 
 	if (amdgpu_ras_intr_triggered()) {
@@ -816,8 +780,6 @@ static int arcturus_emit_clk_levels(struct smu_context *smu,
 		return -EBUSY;
 	}
 
-	dpm_context = smu_dpm->dpm_context;
-
 	switch (type) {
 	case SMU_SCLK:
 		ret = arcturus_get_current_clk_freq_by_table(smu, SMU_GFXCLK, &cur_value);
@@ -825,10 +787,11 @@ static int arcturus_emit_clk_levels(struct smu_context *smu,
 			dev_err(smu->adev->dev, "%s gfx clk Failed!", attempt_string);
 			return ret;
 		}
-
 		single_dpm_table = &(dpm_context->dpm_tables.gfx_table);
-		arcturus_get_clk_table(smu, &clocks, single_dpm_table);
-
+		ret = smu_cmn_print_dpm_clk_levels(smu, single_dpm_table,
+						   cur_value, buf, offset);
+		if (ret < 0)
+			return ret;
 		break;
 
 	case SMU_MCLK:
@@ -837,10 +800,11 @@ static int arcturus_emit_clk_levels(struct smu_context *smu,
 			dev_err(smu->adev->dev, "%s mclk Failed!", attempt_string);
 			return ret;
 		}
-
 		single_dpm_table = &(dpm_context->dpm_tables.uclk_table);
-		arcturus_get_clk_table(smu, &clocks, single_dpm_table);
-
+		ret = smu_cmn_print_dpm_clk_levels(smu, single_dpm_table,
+						   cur_value, buf, offset);
+		if (ret < 0)
+			return ret;
 		break;
 
 	case SMU_SOCCLK:
@@ -849,10 +813,11 @@ static int arcturus_emit_clk_levels(struct smu_context *smu,
 			dev_err(smu->adev->dev, "%s socclk Failed!", attempt_string);
 			return ret;
 		}
-
 		single_dpm_table = &(dpm_context->dpm_tables.soc_table);
-		arcturus_get_clk_table(smu, &clocks, single_dpm_table);
-
+		ret = smu_cmn_print_dpm_clk_levels(smu, single_dpm_table,
+						   cur_value, buf, offset);
+		if (ret < 0)
+			return ret;
 		break;
 
 	case SMU_FCLK:
@@ -861,10 +826,11 @@ static int arcturus_emit_clk_levels(struct smu_context *smu,
 			dev_err(smu->adev->dev, "%s fclk Failed!", attempt_string);
 			return ret;
 		}
-
 		single_dpm_table = &(dpm_context->dpm_tables.fclk_table);
-		arcturus_get_clk_table(smu, &clocks, single_dpm_table);
-
+		ret = smu_cmn_print_dpm_clk_levels(smu, single_dpm_table,
+						   cur_value, buf, offset);
+		if (ret < 0)
+			return ret;
 		break;
 
 	case SMU_VCLK:
@@ -873,10 +839,11 @@ static int arcturus_emit_clk_levels(struct smu_context *smu,
 			dev_err(smu->adev->dev, "%s vclk Failed!", attempt_string);
 			return ret;
 		}
-
 		single_dpm_table = &(dpm_context->dpm_tables.vclk_table);
-		arcturus_get_clk_table(smu, &clocks, single_dpm_table);
-
+		ret = smu_cmn_print_dpm_clk_levels(smu, single_dpm_table,
+						   cur_value, buf, offset);
+		if (ret < 0)
+			return ret;
 		break;
 
 	case SMU_DCLK:
@@ -885,56 +852,27 @@ static int arcturus_emit_clk_levels(struct smu_context *smu,
 			dev_err(smu->adev->dev, "%s dclk Failed!", attempt_string);
 			return ret;
 		}
-
 		single_dpm_table = &(dpm_context->dpm_tables.dclk_table);
-		arcturus_get_clk_table(smu, &clocks, single_dpm_table);
-
+		ret = smu_cmn_print_dpm_clk_levels(smu, single_dpm_table,
+						   cur_value, buf, offset);
+		if (ret < 0)
+			return ret;
 		break;
 
 	case SMU_PCIE:
 		gen_speed = smu_v11_0_get_current_pcie_link_speed_level(smu);
 		lane_width = smu_v11_0_get_current_pcie_link_width_level(smu);
-		break;
-
-	default:
-		return -EINVAL;
-	}
-
-	switch (type) {
-	case SMU_SCLK:
-	case SMU_MCLK:
-	case SMU_SOCCLK:
-	case SMU_FCLK:
-	case SMU_VCLK:
-	case SMU_DCLK:
-		/*
-		 * For DPM disabled case, there will be only one clock level.
-		 * And it's safe to assume that is always the current clock.
-		 */
-		for (i = 0; i < clocks.num_levels; i++) {
-			clock_mhz = clocks.data[i].clocks_in_khz / 1000;
-			freq_match = arcturus_freqs_in_same_level(clock_mhz, cur_value);
-			freq_match |= (clocks.num_levels == 1);
-
-			*offset += sysfs_emit_at(buf, *offset, "%d: %uMhz %s\n",
-				i, clock_mhz,
-				freq_match ? "*" : "");
-		}
-		break;
-
-	case SMU_PCIE:
-		*offset += sysfs_emit_at(buf, *offset, "0: %s %s %dMhz *\n",
-				(gen_speed == 0) ? "2.5GT/s," :
-				(gen_speed == 1) ? "5.0GT/s," :
-				(gen_speed == 2) ? "8.0GT/s," :
-				(gen_speed == 3) ? "16.0GT/s," : "",
-				(lane_width == 1) ? "x1" :
-				(lane_width == 2) ? "x2" :
-				(lane_width == 3) ? "x4" :
-				(lane_width == 4) ? "x8" :
-				(lane_width == 5) ? "x12" :
-				(lane_width == 6) ? "x16" : "",
-				smu->smu_table.boot_values.lclk / 100);
+		pcie_table = &(dpm_context->dpm_tables.pcie_table);
+		/* Populate with current state - arcturus only has boot level lclk */
+		pcie_table->lclk_levels = 1;
+		pcie_table->pcie_gen[0] = gen_speed;
+		pcie_table->pcie_lane[0] = lane_width;
+		pcie_table->lclk_freq[0] =
+			smu->smu_table.boot_values.lclk / 100;
+		ret = smu_cmn_print_pcie_levels(smu, pcie_table, gen_speed,
+						lane_width, buf, offset);
+		if (ret < 0)
+			return ret;
 		break;
 
 	default:
@@ -1003,7 +941,7 @@ static int arcturus_force_clk_levels(struct smu_context *smu,
 			enum smu_clk_type type, uint32_t mask)
 {
 	struct smu_11_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
-	struct smu_11_0_dpm_table *single_dpm_table = NULL;
+	struct smu_dpm_table *single_dpm_table = NULL;
 	uint32_t soft_min_level, soft_max_level;
 	int ret = 0;
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/cyan_skillfish_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/cyan_skillfish_ppt.c
index 55401e6b2b0b..e0e829ea6579 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/cyan_skillfish_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/cyan_skillfish_ppt.c
@@ -287,17 +287,14 @@ static int cyan_skillfish_get_current_clk_freq(struct smu_context *smu,
 	return cyan_skillfish_get_smu_metrics_data(smu, member_type, value);
 }
 
-static int cyan_skillfish_print_clk_levels(struct smu_context *smu,
-					enum smu_clk_type clk_type,
-					char *buf)
+static int cyan_skillfish_emit_clk_levels(struct smu_context *smu,
+					  enum smu_clk_type clk_type, char *buf,
+					  int *offset)
 {
-	int ret = 0, size = 0, start_offset = 0;
+	int ret = 0, size = *offset, start_offset = *offset;
 	uint32_t cur_value = 0;
 	int i;
 
-	smu_cmn_get_sysfs_buf(&buf, &size);
-	start_offset = size;
-
 	switch (clk_type) {
 	case SMU_OD_SCLK:
 		ret  = cyan_skillfish_get_smu_metrics_data(smu, METRICS_CURR_GFXCLK, &cur_value);
@@ -354,7 +351,9 @@ static int cyan_skillfish_print_clk_levels(struct smu_context *smu,
 		return ret;
 	}
 
-	return size - start_offset;
+	*offset += size - start_offset;
+
+	return 0;
 }
 
 static bool cyan_skillfish_is_dpm_running(struct smu_context *smu)
@@ -582,7 +581,7 @@ static const struct pptable_funcs cyan_skillfish_ppt_funcs = {
 	.init_smc_tables = cyan_skillfish_init_smc_tables,
 	.fini_smc_tables = smu_v11_0_fini_smc_tables,
 	.read_sensor = cyan_skillfish_read_sensor,
-	.print_clk_levels = cyan_skillfish_print_clk_levels,
+	.emit_clk_levels = cyan_skillfish_emit_clk_levels,
 	.get_enabled_mask = cyan_skillfish_get_enabled_mask,
 	.is_dpm_running = cyan_skillfish_is_dpm_running,
 	.get_gpu_metrics = cyan_skillfish_get_gpu_metrics,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
index 7c9f77124ab2..882830770b79 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
@@ -967,169 +967,160 @@ static int navi10_set_default_dpm_table(struct smu_context *smu)
 {
 	struct smu_11_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
 	PPTable_t *driver_ppt = smu->smu_table.driver_pptable;
-	struct smu_11_0_dpm_table *dpm_table;
+	struct smu_dpm_table *dpm_table;
 	int ret = 0;
 
 	/* socclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.soc_table;
+	dpm_table->clk_type = SMU_SOCCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_SOCCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_SOCCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!driver_ppt->DpmDescriptor[PPCLK_SOCCLK].SnapToDiscrete;
+		if (!driver_ppt->DpmDescriptor[PPCLK_SOCCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.socclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* gfxclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.gfx_table;
+	dpm_table->clk_type = SMU_GFXCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_GFXCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!driver_ppt->DpmDescriptor[PPCLK_GFXCLK].SnapToDiscrete;
+		if (!driver_ppt->DpmDescriptor[PPCLK_GFXCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* uclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.uclk_table;
+	dpm_table->clk_type = SMU_UCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_UCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_UCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!driver_ppt->DpmDescriptor[PPCLK_UCLK].SnapToDiscrete;
+		if (!driver_ppt->DpmDescriptor[PPCLK_UCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.uclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* vclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.vclk_table;
+	dpm_table->clk_type = SMU_VCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_VCN_PG_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_VCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!driver_ppt->DpmDescriptor[PPCLK_VCLK].SnapToDiscrete;
+		if (!driver_ppt->DpmDescriptor[PPCLK_VCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.vclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* dclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.dclk_table;
+	dpm_table->clk_type = SMU_DCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_VCN_PG_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_DCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!driver_ppt->DpmDescriptor[PPCLK_DCLK].SnapToDiscrete;
+		if (!driver_ppt->DpmDescriptor[PPCLK_DCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* dcefclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.dcef_table;
+	dpm_table->clk_type = SMU_DCEFCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCEFCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_DCEFCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!driver_ppt->DpmDescriptor[PPCLK_DCEFCLK].SnapToDiscrete;
+		if (!driver_ppt->DpmDescriptor[PPCLK_DCEFCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dcefclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* pixelclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.pixel_table;
+	dpm_table->clk_type = SMU_PIXCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCEFCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_PIXCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!driver_ppt->DpmDescriptor[PPCLK_PIXCLK].SnapToDiscrete;
+		if (!driver_ppt->DpmDescriptor[PPCLK_PIXCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dcefclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* displayclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.display_table;
+	dpm_table->clk_type = SMU_DISPCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCEFCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_DISPCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!driver_ppt->DpmDescriptor[PPCLK_DISPCLK].SnapToDiscrete;
+		if (!driver_ppt->DpmDescriptor[PPCLK_DISPCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dcefclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* phyclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.phy_table;
+	dpm_table->clk_type = SMU_PHYCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCEFCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_PHYCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!driver_ppt->DpmDescriptor[PPCLK_PHYCLK].SnapToDiscrete;
+		if (!driver_ppt->DpmDescriptor[PPCLK_PHYCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dcefclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	return 0;
@@ -1261,104 +1252,48 @@ static int navi10_emit_clk_levels(struct smu_context *smu,
 {
 	uint16_t *curve_settings;
 	int ret = 0;
-	uint32_t cur_value = 0, value = 0;
-	uint32_t freq_values[3] = {0};
-	uint32_t i, levels, mark_index = 0, count = 0;
+	uint32_t cur_value = 0;
+	uint32_t i;
 	struct smu_table_context *table_context = &smu->smu_table;
 	uint32_t gen_speed, lane_width;
 	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
 	struct smu_11_0_dpm_context *dpm_context = smu_dpm->dpm_context;
-	PPTable_t *pptable = (PPTable_t *)table_context->driver_pptable;
 	OverDriveTable_t *od_table =
 		(OverDriveTable_t *)table_context->overdrive_table;
 	struct smu_11_0_overdrive_table *od_settings = smu->od_settings;
+	struct smu_dpm_table *single_dpm_table = NULL;
+	struct smu_pcie_table *pcie_table;
 	uint32_t min_value, max_value;
 
 	switch (clk_type) {
 	case SMU_GFXCLK:
 	case SMU_SCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.gfx_table);
+		break;
+	/* FCLK and SOC CLK mapped to SOCCLK in clock map table */
 	case SMU_SOCCLK:
+	case SMU_FCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.soc_table);
+		break;
 	case SMU_MCLK:
 	case SMU_UCLK:
-	case SMU_FCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.uclk_table);
+		break;
 	case SMU_VCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.vclk_table);
+		break;
 	case SMU_DCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.dclk_table);
+		break;
 	case SMU_DCEFCLK:
-		ret = navi10_get_current_clk_freq_by_table(smu, clk_type, &cur_value);
-		if (ret)
-			return ret;
-
-		ret = smu_v11_0_get_dpm_level_count(smu, clk_type, &count);
-		if (ret)
-			return ret;
-
-		ret = navi10_is_support_fine_grained_dpm(smu, clk_type);
-		if (ret < 0)
-			return ret;
-
-		if (!ret) {
-			for (i = 0; i < count; i++) {
-				ret = smu_v11_0_get_dpm_freq_by_index(smu,
-								      clk_type, i, &value);
-				if (ret)
-					return ret;
-
-				*offset += sysfs_emit_at(buf, *offset,
-						"%d: %uMhz %s\n",
-						i, value,
-						cur_value == value ? "*" : "");
-			}
-		} else {
-			ret = smu_v11_0_get_dpm_freq_by_index(smu,
-							      clk_type, 0, &freq_values[0]);
-			if (ret)
-				return ret;
-			ret = smu_v11_0_get_dpm_freq_by_index(smu,
-							      clk_type,
-							      count - 1,
-							      &freq_values[2]);
-			if (ret)
-				return ret;
-
-			freq_values[1] = cur_value;
-			mark_index = cur_value == freq_values[0] ? 0 :
-				     cur_value == freq_values[2] ? 2 : 1;
-
-			levels = 3;
-			if (mark_index != 1) {
-				levels = 2;
-				freq_values[1] = freq_values[2];
-			}
-
-			for (i = 0; i < levels; i++) {
-				*offset += sysfs_emit_at(buf, *offset,
-						"%d: %uMhz %s\n",
-						i, freq_values[i],
-						i == mark_index ? "*" : "");
-			}
-		}
+		single_dpm_table = &(dpm_context->dpm_tables.dcef_table);
 		break;
 	case SMU_PCIE:
 		gen_speed = smu_v11_0_get_current_pcie_link_speed_level(smu);
 		lane_width = smu_v11_0_get_current_pcie_link_width_level(smu);
-		for (i = 0; i < NUM_LINK_LEVELS; i++) {
-			*offset += sysfs_emit_at(buf, *offset, "%d: %s %s %dMhz %s\n", i,
-					(dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 0) ? "2.5GT/s," :
-					(dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 1) ? "5.0GT/s," :
-					(dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 2) ? "8.0GT/s," :
-					(dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 3) ? "16.0GT/s," : "",
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 1) ? "x1" :
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 2) ? "x2" :
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 3) ? "x4" :
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 4) ? "x8" :
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 5) ? "x12" :
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 6) ? "x16" : "",
-					pptable->LclkFreq[i],
-					(gen_speed == dpm_context->dpm_tables.pcie_table.pcie_gen[i]) &&
-					(lane_width == dpm_context->dpm_tables.pcie_table.pcie_lane[i]) ?
-					"*" : "");
-		}
-		break;
+		pcie_table = &dpm_context->dpm_tables.pcie_table;
+		return smu_cmn_print_pcie_levels(smu, pcie_table, gen_speed,
+						 lane_width, buf, offset);
 	case SMU_OD_SCLK:
 		if (!smu->od_enabled || !od_table || !od_settings)
 			return -EOPNOTSUPP;
@@ -1462,199 +1397,16 @@ static int navi10_emit_clk_levels(struct smu_context *smu,
 		break;
 	}
 
-	return 0;
-}
-
-static int navi10_print_clk_levels(struct smu_context *smu,
-			enum smu_clk_type clk_type, char *buf)
-{
-	uint16_t *curve_settings;
-	int i, levels, size = 0, ret = 0, start_offset = 0;
-	uint32_t cur_value = 0, value = 0, count = 0;
-	uint32_t freq_values[3] = {0};
-	uint32_t mark_index = 0;
-	struct smu_table_context *table_context = &smu->smu_table;
-	uint32_t gen_speed, lane_width;
-	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
-	struct smu_11_0_dpm_context *dpm_context = smu_dpm->dpm_context;
-	PPTable_t *pptable = (PPTable_t *)table_context->driver_pptable;
-	OverDriveTable_t *od_table =
-		(OverDriveTable_t *)table_context->overdrive_table;
-	struct smu_11_0_overdrive_table *od_settings = smu->od_settings;
-	uint32_t min_value, max_value;
-
-	smu_cmn_get_sysfs_buf(&buf, &size);
-	start_offset = size;
-
-	switch (clk_type) {
-	case SMU_GFXCLK:
-	case SMU_SCLK:
-	case SMU_SOCCLK:
-	case SMU_MCLK:
-	case SMU_UCLK:
-	case SMU_FCLK:
-	case SMU_VCLK:
-	case SMU_DCLK:
-	case SMU_DCEFCLK:
-		ret = navi10_get_current_clk_freq_by_table(smu, clk_type, &cur_value);
+	if (single_dpm_table) {
+		ret = navi10_get_current_clk_freq_by_table(smu, clk_type,
+							   &cur_value);
 		if (ret)
-			return size - start_offset;
-
-		ret = smu_v11_0_get_dpm_level_count(smu, clk_type, &count);
-		if (ret)
-			return size - start_offset;
-
-		ret = navi10_is_support_fine_grained_dpm(smu, clk_type);
-		if (ret < 0)
 			return ret;
-
-		if (!ret) {
-			for (i = 0; i < count; i++) {
-				ret = smu_v11_0_get_dpm_freq_by_index(smu, clk_type, i, &value);
-				if (ret)
-					return size - start_offset;
-
-				size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, value,
-						cur_value == value ? "*" : "");
-			}
-		} else {
-			ret = smu_v11_0_get_dpm_freq_by_index(smu, clk_type, 0, &freq_values[0]);
-			if (ret)
-				return size - start_offset;
-			ret = smu_v11_0_get_dpm_freq_by_index(smu, clk_type, count - 1, &freq_values[2]);
-			if (ret)
-				return size - start_offset;
-
-			freq_values[1] = cur_value;
-			mark_index = cur_value == freq_values[0] ? 0 :
-				     cur_value == freq_values[2] ? 2 : 1;
-
-			levels = 3;
-			if (mark_index != 1) {
-				levels = 2;
-				freq_values[1] = freq_values[2];
-			}
-
-			for (i = 0; i < levels; i++) {
-				size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, freq_values[i],
-						i == mark_index ? "*" : "");
-			}
-		}
-		break;
-	case SMU_PCIE:
-		gen_speed = smu_v11_0_get_current_pcie_link_speed_level(smu);
-		lane_width = smu_v11_0_get_current_pcie_link_width_level(smu);
-		for (i = 0; i < NUM_LINK_LEVELS; i++)
-			size += sysfs_emit_at(buf, size, "%d: %s %s %dMhz %s\n", i,
-					(dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 0) ? "2.5GT/s," :
-					(dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 1) ? "5.0GT/s," :
-					(dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 2) ? "8.0GT/s," :
-					(dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 3) ? "16.0GT/s," : "",
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 1) ? "x1" :
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 2) ? "x2" :
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 3) ? "x4" :
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 4) ? "x8" :
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 5) ? "x12" :
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 6) ? "x16" : "",
-					pptable->LclkFreq[i],
-					(gen_speed == dpm_context->dpm_tables.pcie_table.pcie_gen[i]) &&
-					(lane_width == dpm_context->dpm_tables.pcie_table.pcie_lane[i]) ?
-					"*" : "");
-		break;
-	case SMU_OD_SCLK:
-		if (!smu->od_enabled || !od_table || !od_settings)
-			break;
-		if (!navi10_od_feature_is_supported(od_settings, SMU_11_0_ODCAP_GFXCLK_LIMITS))
-			break;
-		size += sysfs_emit_at(buf, size, "OD_SCLK:\n");
-		size += sysfs_emit_at(buf, size, "0: %uMhz\n1: %uMhz\n",
-				      od_table->GfxclkFmin, od_table->GfxclkFmax);
-		break;
-	case SMU_OD_MCLK:
-		if (!smu->od_enabled || !od_table || !od_settings)
-			break;
-		if (!navi10_od_feature_is_supported(od_settings, SMU_11_0_ODCAP_UCLK_MAX))
-			break;
-		size += sysfs_emit_at(buf, size, "OD_MCLK:\n");
-		size += sysfs_emit_at(buf, size, "1: %uMHz\n", od_table->UclkFmax);
-		break;
-	case SMU_OD_VDDC_CURVE:
-		if (!smu->od_enabled || !od_table || !od_settings)
-			break;
-		if (!navi10_od_feature_is_supported(od_settings, SMU_11_0_ODCAP_GFXCLK_CURVE))
-			break;
-		size += sysfs_emit_at(buf, size, "OD_VDDC_CURVE:\n");
-		for (i = 0; i < 3; i++) {
-			switch (i) {
-			case 0:
-				curve_settings = &od_table->GfxclkFreq1;
-				break;
-			case 1:
-				curve_settings = &od_table->GfxclkFreq2;
-				break;
-			case 2:
-				curve_settings = &od_table->GfxclkFreq3;
-				break;
-			}
-			size += sysfs_emit_at(buf, size, "%d: %uMHz %umV\n",
-					      i, curve_settings[0],
-					curve_settings[1] / NAVI10_VOLTAGE_SCALE);
-		}
-		break;
-	case SMU_OD_RANGE:
-		if (!smu->od_enabled || !od_table || !od_settings)
-			break;
-		size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE");
-
-		if (navi10_od_feature_is_supported(od_settings, SMU_11_0_ODCAP_GFXCLK_LIMITS)) {
-			navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_GFXCLKFMIN,
-						    &min_value, NULL);
-			navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_GFXCLKFMAX,
-						    NULL, &max_value);
-			size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n",
-					min_value, max_value);
-		}
-
-		if (navi10_od_feature_is_supported(od_settings, SMU_11_0_ODCAP_UCLK_MAX)) {
-			navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_UCLKFMAX,
-						    &min_value, &max_value);
-			size += sysfs_emit_at(buf, size, "MCLK: %7uMhz %10uMhz\n",
-					min_value, max_value);
-		}
-
-		if (navi10_od_feature_is_supported(od_settings, SMU_11_0_ODCAP_GFXCLK_CURVE)) {
-			navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEFREQ_P1,
-						    &min_value, &max_value);
-			size += sysfs_emit_at(buf, size, "VDDC_CURVE_SCLK[0]: %7uMhz %10uMhz\n",
-					      min_value, max_value);
-			navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEVOLTAGE_P1,
-						    &min_value, &max_value);
-			size += sysfs_emit_at(buf, size, "VDDC_CURVE_VOLT[0]: %7dmV %11dmV\n",
-					      min_value, max_value);
-			navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEFREQ_P2,
-						    &min_value, &max_value);
-			size += sysfs_emit_at(buf, size, "VDDC_CURVE_SCLK[1]: %7uMhz %10uMhz\n",
-					      min_value, max_value);
-			navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEVOLTAGE_P2,
-						    &min_value, &max_value);
-			size += sysfs_emit_at(buf, size, "VDDC_CURVE_VOLT[1]: %7dmV %11dmV\n",
-					      min_value, max_value);
-			navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEFREQ_P3,
-						    &min_value, &max_value);
-			size += sysfs_emit_at(buf, size, "VDDC_CURVE_SCLK[2]: %7uMhz %10uMhz\n",
-					      min_value, max_value);
-			navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_VDDGFXCURVEVOLTAGE_P3,
-						    &min_value, &max_value);
-			size += sysfs_emit_at(buf, size, "VDDC_CURVE_VOLT[2]: %7dmV %11dmV\n",
-					      min_value, max_value);
-		}
-
-		break;
-	default:
-		break;
+		return smu_cmn_print_dpm_clk_levels(smu, single_dpm_table,
+						    cur_value, buf, offset);
 	}
 
-	return size - start_offset;
+	return 0;
 }
 
 static int navi10_force_clk_levels(struct smu_context *smu,
@@ -1711,18 +1463,15 @@ static int navi10_populate_umd_state_clk(struct smu_context *smu)
 {
 	struct smu_11_0_dpm_context *dpm_context =
 				smu->smu_dpm.dpm_context;
-	struct smu_11_0_dpm_table *gfx_table =
-				&dpm_context->dpm_tables.gfx_table;
-	struct smu_11_0_dpm_table *mem_table =
-				&dpm_context->dpm_tables.uclk_table;
-	struct smu_11_0_dpm_table *soc_table =
-				&dpm_context->dpm_tables.soc_table;
+	struct smu_dpm_table *gfx_table = &dpm_context->dpm_tables.gfx_table;
+	struct smu_dpm_table *mem_table = &dpm_context->dpm_tables.uclk_table;
+	struct smu_dpm_table *soc_table = &dpm_context->dpm_tables.soc_table;
 	struct smu_umd_pstate_table *pstate_table =
 				&smu->pstate_table;
 	struct amdgpu_device *adev = smu->adev;
 	uint32_t sclk_freq;
 
-	pstate_table->gfxclk_pstate.min = gfx_table->min;
+	pstate_table->gfxclk_pstate.min = SMU_DPM_TABLE_MIN(gfx_table);
 	switch (amdgpu_ip_version(adev, MP1_HWIP, 0)) {
 	case IP_VERSION(11, 0, 0):
 		switch (adev->pdev->revision) {
@@ -1771,15 +1520,15 @@ static int navi10_populate_umd_state_clk(struct smu_context *smu)
 	}
 	pstate_table->gfxclk_pstate.peak = sclk_freq;
 
-	pstate_table->uclk_pstate.min = mem_table->min;
-	pstate_table->uclk_pstate.peak = mem_table->max;
+	pstate_table->uclk_pstate.min = SMU_DPM_TABLE_MIN(mem_table);
+	pstate_table->uclk_pstate.peak = SMU_DPM_TABLE_MAX(mem_table);
 
-	pstate_table->socclk_pstate.min = soc_table->min;
-	pstate_table->socclk_pstate.peak = soc_table->max;
+	pstate_table->socclk_pstate.min = SMU_DPM_TABLE_MIN(soc_table);
+	pstate_table->socclk_pstate.peak = SMU_DPM_TABLE_MAX(soc_table);
 
-	if (gfx_table->max > NAVI10_UMD_PSTATE_PROFILING_GFXCLK &&
-	    mem_table->max > NAVI10_UMD_PSTATE_PROFILING_MEMCLK &&
-	    soc_table->max > NAVI10_UMD_PSTATE_PROFILING_SOCCLK) {
+	if (SMU_DPM_TABLE_MAX(gfx_table) > NAVI10_UMD_PSTATE_PROFILING_GFXCLK &&
+	    SMU_DPM_TABLE_MAX(mem_table) > NAVI10_UMD_PSTATE_PROFILING_MEMCLK &&
+	    SMU_DPM_TABLE_MAX(soc_table) > NAVI10_UMD_PSTATE_PROFILING_SOCCLK) {
 		pstate_table->gfxclk_pstate.standard =
 			NAVI10_UMD_PSTATE_PROFILING_GFXCLK;
 		pstate_table->uclk_pstate.standard =
@@ -2449,10 +2198,13 @@ static int navi10_update_pcie_parameters(struct smu_context *smu,
 	int i;
 
 	/* lclk dpm table setup */
-	for (i = 0; i < MAX_PCIE_CONF; i++) {
+	for (i = 0; i < NUM_LINK_LEVELS; i++) {
 		dpm_context->dpm_tables.pcie_table.pcie_gen[i] = pptable->PcieGenSpeed[i];
 		dpm_context->dpm_tables.pcie_table.pcie_lane[i] = pptable->PcieLaneCount[i];
+		dpm_context->dpm_tables.pcie_table.lclk_freq[i] =
+			pptable->LclkFreq[i];
 	}
+	dpm_context->dpm_tables.pcie_table.lclk_levels = NUM_LINK_LEVELS;
 
 	for (i = 0; i < NUM_LINK_LEVELS; i++) {
 		if (pptable->PcieGenSpeed[i] > pcie_gen_cap ||
@@ -3521,7 +3273,6 @@ static const struct pptable_funcs navi10_ppt_funcs = {
 	.dpm_set_jpeg_enable = navi10_dpm_set_jpeg_enable,
 	.i2c_init = navi10_i2c_control_init,
 	.i2c_fini = navi10_i2c_control_fini,
-	.print_clk_levels = navi10_print_clk_levels,
 	.emit_clk_levels = navi10_emit_clk_levels,
 	.force_clk_levels = navi10_force_clk_levels,
 	.populate_umd_state_clk = navi10_populate_umd_state_clk,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
index 774283ac7827..108e8b3027ea 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
@@ -954,13 +954,15 @@ static int sienna_cichlid_init_smc_tables(struct smu_context *smu)
 static int sienna_cichlid_set_default_dpm_table(struct smu_context *smu)
 {
 	struct smu_11_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
-	struct smu_11_0_dpm_table *dpm_table;
+	struct smu_dpm_table *dpm_table;
 	struct amdgpu_device *adev = smu->adev;
 	int i, ret = 0;
 	DpmDescriptor_t *table_member;
+	uint16_t *lclk_freq;
 
 	/* socclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.soc_table;
+	dpm_table->clk_type = SMU_SOCCLK;
 	GET_PPTABLE_MEMBER(DpmDescriptor, &table_member);
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_SOCCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
@@ -968,68 +970,63 @@ static int sienna_cichlid_set_default_dpm_table(struct smu_context *smu)
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!table_member[PPCLK_SOCCLK].SnapToDiscrete;
+		if (!table_member[PPCLK_SOCCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.socclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* gfxclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.gfx_table;
+	dpm_table->clk_type = SMU_GFXCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_GFXCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!table_member[PPCLK_GFXCLK].SnapToDiscrete;
+		if (!table_member[PPCLK_GFXCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* uclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.uclk_table;
+	dpm_table->clk_type = SMU_UCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_UCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_UCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!table_member[PPCLK_UCLK].SnapToDiscrete;
+		if (!table_member[PPCLK_UCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.uclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* fclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.fclk_table;
+	dpm_table->clk_type = SMU_FCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_FCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_FCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!table_member[PPCLK_FCLK].SnapToDiscrete;
+		if (!table_member[PPCLK_FCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.fclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* vclk0/1 dpm table setup */
@@ -1038,20 +1035,20 @@ static int sienna_cichlid_set_default_dpm_table(struct smu_context *smu)
 			continue;
 
 		dpm_table = &dpm_context->dpm_tables.vclk_table;
+		dpm_table->clk_type = i ? SMU_VCLK1 : SMU_VCLK;
 		if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_MM_DPM_PG_BIT)) {
 			ret = smu_v11_0_set_single_dpm_table(smu,
 							     i ? SMU_VCLK1 : SMU_VCLK,
 							     dpm_table);
 			if (ret)
 				return ret;
-			dpm_table->is_fine_grained =
-				!table_member[i ? PPCLK_VCLK_1 : PPCLK_VCLK_0].SnapToDiscrete;
+			if (!table_member[i ? PPCLK_VCLK_1 : PPCLK_VCLK_0]
+				     .SnapToDiscrete)
+				dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 		} else {
 			dpm_table->count = 1;
 			dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.vclk / 100;
 			dpm_table->dpm_levels[0].enabled = true;
-			dpm_table->min = dpm_table->dpm_levels[0].value;
-			dpm_table->max = dpm_table->dpm_levels[0].value;
 		}
 	}
 
@@ -1060,95 +1057,96 @@ static int sienna_cichlid_set_default_dpm_table(struct smu_context *smu)
 		if (adev->vcn.harvest_config & (1 << i))
 			continue;
 		dpm_table = &dpm_context->dpm_tables.dclk_table;
+		dpm_table->clk_type = i ? SMU_DCLK1 : SMU_DCLK;
 		if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_MM_DPM_PG_BIT)) {
 			ret = smu_v11_0_set_single_dpm_table(smu,
 							     i ? SMU_DCLK1 : SMU_DCLK,
 							     dpm_table);
 			if (ret)
 				return ret;
-			dpm_table->is_fine_grained =
-				!table_member[i ? PPCLK_DCLK_1 : PPCLK_DCLK_0].SnapToDiscrete;
+			if (!table_member[i ? PPCLK_DCLK_1 : PPCLK_DCLK_0]
+				     .SnapToDiscrete)
+				dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 		} else {
 			dpm_table->count = 1;
 			dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dclk / 100;
 			dpm_table->dpm_levels[0].enabled = true;
-			dpm_table->min = dpm_table->dpm_levels[0].value;
-			dpm_table->max = dpm_table->dpm_levels[0].value;
 		}
 	}
 
 	/* dcefclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.dcef_table;
+	dpm_table->clk_type = SMU_DCEFCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCEFCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_DCEFCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!table_member[PPCLK_DCEFCLK].SnapToDiscrete;
+		if (!table_member[PPCLK_DCEFCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dcefclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* pixelclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.pixel_table;
+	dpm_table->clk_type = SMU_PIXCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCEFCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_PIXCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!table_member[PPCLK_PIXCLK].SnapToDiscrete;
+		if (!table_member[PPCLK_PIXCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dcefclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* displayclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.display_table;
+	dpm_table->clk_type = SMU_DISPCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCEFCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_DISPCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!table_member[PPCLK_DISPCLK].SnapToDiscrete;
+		if (!table_member[PPCLK_DISPCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dcefclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* phyclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.phy_table;
+	dpm_table->clk_type = SMU_PHYCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCEFCLK_BIT)) {
 		ret = smu_v11_0_set_single_dpm_table(smu,
 						     SMU_PHYCLK,
 						     dpm_table);
 		if (ret)
 			return ret;
-		dpm_table->is_fine_grained =
-			!table_member[PPCLK_PHYCLK].SnapToDiscrete;
+		if (!table_member[PPCLK_PHYCLK].SnapToDiscrete)
+			dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dcefclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
+	GET_PPTABLE_MEMBER(LclkFreq, &lclk_freq);
+	for (i = 0; i < NUM_LINK_LEVELS; i++)
+		dpm_context->dpm_tables.pcie_table.lclk_freq[i] = lclk_freq[i];
+	dpm_context->dpm_tables.pcie_table.lclk_levels = NUM_LINK_LEVELS;
+
 	return 0;
 }
 
@@ -1269,103 +1267,56 @@ static void sienna_cichlid_get_od_setting_range(struct smu_11_0_7_overdrive_tabl
 		*max = od_table->max[setting];
 }
 
-static int sienna_cichlid_print_clk_levels(struct smu_context *smu,
-			enum smu_clk_type clk_type, char *buf)
+static int sienna_cichlid_emit_clk_levels(struct smu_context *smu,
+					  enum smu_clk_type clk_type, char *buf,
+					  int *offset)
 {
 	struct amdgpu_device *adev = smu->adev;
 	struct smu_table_context *table_context = &smu->smu_table;
 	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
 	struct smu_11_0_dpm_context *dpm_context = smu_dpm->dpm_context;
-	uint16_t *table_member;
-
 	struct smu_11_0_7_overdrive_table *od_settings = smu->od_settings;
 	OverDriveTable_t *od_table =
 		(OverDriveTable_t *)table_context->overdrive_table;
-	int i, size = 0, ret = 0, start_offset = 0;
-	uint32_t cur_value = 0, value = 0, count = 0;
-	uint32_t freq_values[3] = {0};
-	uint32_t mark_index = 0;
+	int size = *offset, ret = 0, start_offset = *offset;
+	uint32_t cur_value = 0;
+	struct smu_dpm_table *single_dpm_table = NULL;
+	struct smu_pcie_table *pcie_table;
 	uint32_t gen_speed, lane_width;
 	uint32_t min_value, max_value;
 
-	smu_cmn_get_sysfs_buf(&buf, &size);
-	start_offset = size;
-
 	switch (clk_type) {
 	case SMU_GFXCLK:
 	case SMU_SCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.gfx_table);
+		break;
 	case SMU_SOCCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.soc_table);
+		break;
 	case SMU_MCLK:
 	case SMU_UCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.uclk_table);
+		break;
 	case SMU_FCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.fclk_table);
+		break;
 	case SMU_VCLK:
 	case SMU_VCLK1:
+		single_dpm_table = &(dpm_context->dpm_tables.vclk_table);
+		break;
 	case SMU_DCLK:
 	case SMU_DCLK1:
+		single_dpm_table = &(dpm_context->dpm_tables.dclk_table);
+		break;
 	case SMU_DCEFCLK:
-		ret = sienna_cichlid_get_current_clk_freq_by_table(smu, clk_type, &cur_value);
-		if (ret)
-			goto print_clk_out;
-
-		ret = smu_v11_0_get_dpm_level_count(smu, clk_type, &count);
-		if (ret)
-			goto print_clk_out;
-
-		if (!sienna_cichlid_is_support_fine_grained_dpm(smu, clk_type)) {
-			for (i = 0; i < count; i++) {
-				ret = smu_v11_0_get_dpm_freq_by_index(smu, clk_type, i, &value);
-				if (ret)
-					goto print_clk_out;
-
-				size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, value,
-						cur_value == value ? "*" : "");
-			}
-		} else {
-			ret = smu_v11_0_get_dpm_freq_by_index(smu, clk_type, 0, &freq_values[0]);
-			if (ret)
-				goto print_clk_out;
-			ret = smu_v11_0_get_dpm_freq_by_index(smu, clk_type, count - 1, &freq_values[2]);
-			if (ret)
-				goto print_clk_out;
-
-			freq_values[1] = cur_value;
-			mark_index = cur_value == freq_values[0] ? 0 :
-				     cur_value == freq_values[2] ? 2 : 1;
-
-			count = 3;
-			if (mark_index != 1) {
-				count = 2;
-				freq_values[1] = freq_values[2];
-			}
-
-			for (i = 0; i < count; i++) {
-				size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, freq_values[i],
-						cur_value  == freq_values[i] ? "*" : "");
-			}
-
-		}
+		single_dpm_table = &(dpm_context->dpm_tables.dcef_table);
 		break;
 	case SMU_PCIE:
 		gen_speed = smu_v11_0_get_current_pcie_link_speed_level(smu);
 		lane_width = smu_v11_0_get_current_pcie_link_width_level(smu);
-		GET_PPTABLE_MEMBER(LclkFreq, &table_member);
-		for (i = 0; i < NUM_LINK_LEVELS; i++)
-			size += sysfs_emit_at(buf, size, "%d: %s %s %dMhz %s\n", i,
-					(dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 0) ? "2.5GT/s," :
-					(dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 1) ? "5.0GT/s," :
-					(dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 2) ? "8.0GT/s," :
-					(dpm_context->dpm_tables.pcie_table.pcie_gen[i] == 3) ? "16.0GT/s," : "",
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 1) ? "x1" :
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 2) ? "x2" :
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 3) ? "x4" :
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 4) ? "x8" :
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 5) ? "x12" :
-					(dpm_context->dpm_tables.pcie_table.pcie_lane[i] == 6) ? "x16" : "",
-					table_member[i],
-					(gen_speed == dpm_context->dpm_tables.pcie_table.pcie_gen[i]) &&
-					(lane_width == dpm_context->dpm_tables.pcie_table.pcie_lane[i]) ?
-					"*" : "");
-		break;
+		pcie_table = &dpm_context->dpm_tables.pcie_table;
+		return smu_cmn_print_pcie_levels(smu, pcie_table, gen_speed,
+						 lane_width, buf, offset);
 	case SMU_OD_SCLK:
 		if (!smu->od_enabled || !od_table || !od_settings)
 			break;
@@ -1434,8 +1385,18 @@ static int sienna_cichlid_print_clk_levels(struct smu_context *smu,
 		break;
 	}
 
-print_clk_out:
-	return size - start_offset;
+	if (single_dpm_table) {
+		ret = sienna_cichlid_get_current_clk_freq_by_table(
+			smu, clk_type, &cur_value);
+		if (ret)
+			return ret;
+		return smu_cmn_print_dpm_clk_levels(smu, single_dpm_table,
+						    cur_value, buf, offset);
+	}
+
+	*offset += size - start_offset;
+
+	return 0;
 }
 
 static int sienna_cichlid_force_clk_levels(struct smu_context *smu,
@@ -1487,24 +1448,21 @@ static int sienna_cichlid_populate_umd_state_clk(struct smu_context *smu)
 {
 	struct smu_11_0_dpm_context *dpm_context =
 				smu->smu_dpm.dpm_context;
-	struct smu_11_0_dpm_table *gfx_table =
-				&dpm_context->dpm_tables.gfx_table;
-	struct smu_11_0_dpm_table *mem_table =
-				&dpm_context->dpm_tables.uclk_table;
-	struct smu_11_0_dpm_table *soc_table =
-				&dpm_context->dpm_tables.soc_table;
+	struct smu_dpm_table *gfx_table = &dpm_context->dpm_tables.gfx_table;
+	struct smu_dpm_table *mem_table = &dpm_context->dpm_tables.uclk_table;
+	struct smu_dpm_table *soc_table = &dpm_context->dpm_tables.soc_table;
 	struct smu_umd_pstate_table *pstate_table =
 				&smu->pstate_table;
 	struct amdgpu_device *adev = smu->adev;
 
-	pstate_table->gfxclk_pstate.min = gfx_table->min;
-	pstate_table->gfxclk_pstate.peak = gfx_table->max;
+	pstate_table->gfxclk_pstate.min = SMU_DPM_TABLE_MIN(gfx_table);
+	pstate_table->gfxclk_pstate.peak = SMU_DPM_TABLE_MAX(gfx_table);
 
-	pstate_table->uclk_pstate.min = mem_table->min;
-	pstate_table->uclk_pstate.peak = mem_table->max;
+	pstate_table->uclk_pstate.min = SMU_DPM_TABLE_MIN(mem_table);
+	pstate_table->uclk_pstate.peak = SMU_DPM_TABLE_MAX(mem_table);
 
-	pstate_table->socclk_pstate.min = soc_table->min;
-	pstate_table->socclk_pstate.peak = soc_table->max;
+	pstate_table->socclk_pstate.min = SMU_DPM_TABLE_MIN(soc_table);
+	pstate_table->socclk_pstate.peak = SMU_DPM_TABLE_MAX(soc_table);
 
 	switch (amdgpu_ip_version(adev, MP1_HWIP, 0)) {
 	case IP_VERSION(11, 0, 7):
@@ -2141,7 +2099,7 @@ static int sienna_cichlid_update_pcie_parameters(struct smu_context *smu,
 						 uint8_t pcie_width_cap)
 {
 	struct smu_11_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
-	struct smu_11_0_pcie_table *pcie_table = &dpm_context->dpm_tables.pcie_table;
+	struct smu_pcie_table *pcie_table = &dpm_context->dpm_tables.pcie_table;
 	uint8_t *table_member1, *table_member2;
 	uint8_t min_gen_speed, max_gen_speed;
 	uint8_t min_lane_width, max_lane_width;
@@ -3129,7 +3087,7 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = {
 	.dpm_set_jpeg_enable = sienna_cichlid_dpm_set_jpeg_enable,
 	.i2c_init = sienna_cichlid_i2c_control_init,
 	.i2c_fini = sienna_cichlid_i2c_control_fini,
-	.print_clk_levels = sienna_cichlid_print_clk_levels,
+	.emit_clk_levels = sienna_cichlid_emit_clk_levels,
 	.force_clk_levels = sienna_cichlid_force_clk_levels,
 	.populate_umd_state_clk = sienna_cichlid_populate_umd_state_clk,
 	.pre_display_config_changed = sienna_cichlid_pre_display_config_changed,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c
index b0d6487171d7..b8d9cc5fe3d1 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c
@@ -1859,12 +1859,9 @@ int smu_v11_0_set_performance_level(struct smu_context *smu,
 {
 	struct smu_11_0_dpm_context *dpm_context =
 				smu->smu_dpm.dpm_context;
-	struct smu_11_0_dpm_table *gfx_table =
-				&dpm_context->dpm_tables.gfx_table;
-	struct smu_11_0_dpm_table *mem_table =
-				&dpm_context->dpm_tables.uclk_table;
-	struct smu_11_0_dpm_table *soc_table =
-				&dpm_context->dpm_tables.soc_table;
+	struct smu_dpm_table *gfx_table = &dpm_context->dpm_tables.gfx_table;
+	struct smu_dpm_table *mem_table = &dpm_context->dpm_tables.uclk_table;
+	struct smu_dpm_table *soc_table = &dpm_context->dpm_tables.soc_table;
 	struct smu_umd_pstate_table *pstate_table =
 				&smu->pstate_table;
 	struct amdgpu_device *adev = smu->adev;
@@ -1876,22 +1873,22 @@ int smu_v11_0_set_performance_level(struct smu_context *smu,
 
 	switch (level) {
 	case AMD_DPM_FORCED_LEVEL_HIGH:
-		sclk_min = sclk_max = gfx_table->max;
-		mclk_min = mclk_max = mem_table->max;
-		socclk_min = socclk_max = soc_table->max;
+		sclk_min = sclk_max = SMU_DPM_TABLE_MAX(gfx_table);
+		mclk_min = mclk_max = SMU_DPM_TABLE_MAX(mem_table);
+		socclk_min = socclk_max = SMU_DPM_TABLE_MAX(soc_table);
 		break;
 	case AMD_DPM_FORCED_LEVEL_LOW:
-		sclk_min = sclk_max = gfx_table->min;
-		mclk_min = mclk_max = mem_table->min;
-		socclk_min = socclk_max = soc_table->min;
+		sclk_min = sclk_max = SMU_DPM_TABLE_MIN(gfx_table);
+		mclk_min = mclk_max = SMU_DPM_TABLE_MIN(mem_table);
+		socclk_min = socclk_max = SMU_DPM_TABLE_MIN(soc_table);
 		break;
 	case AMD_DPM_FORCED_LEVEL_AUTO:
-		sclk_min = gfx_table->min;
-		sclk_max = gfx_table->max;
-		mclk_min = mem_table->min;
-		mclk_max = mem_table->max;
-		socclk_min = soc_table->min;
-		socclk_max = soc_table->max;
+		sclk_min = SMU_DPM_TABLE_MIN(gfx_table);
+		sclk_max = SMU_DPM_TABLE_MAX(gfx_table);
+		mclk_min = SMU_DPM_TABLE_MIN(mem_table);
+		mclk_max = SMU_DPM_TABLE_MAX(mem_table);
+		socclk_min = SMU_DPM_TABLE_MIN(soc_table);
+		socclk_max = SMU_DPM_TABLE_MAX(soc_table);
 		auto_level = true;
 		break;
 	case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
@@ -2028,7 +2025,7 @@ int smu_v11_0_get_dpm_level_count(struct smu_context *smu,
 
 int smu_v11_0_set_single_dpm_table(struct smu_context *smu,
 				   enum smu_clk_type clk_type,
-				   struct smu_11_0_dpm_table *single_dpm_table)
+				   struct smu_dpm_table *single_dpm_table)
 {
 	int ret = 0;
 	uint32_t clk;
@@ -2054,11 +2051,6 @@ int smu_v11_0_set_single_dpm_table(struct smu_context *smu,
 
 		single_dpm_table->dpm_levels[i].value = clk;
 		single_dpm_table->dpm_levels[i].enabled = true;
-
-		if (i == 0)
-			single_dpm_table->min = clk;
-		else if (i == single_dpm_table->count - 1)
-			single_dpm_table->max = clk;
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c
index 9626da2dba58..34b3f8680a2c 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c
@@ -559,13 +559,14 @@ static int vangogh_get_dpm_clk_limited(struct smu_context *smu, enum smu_clk_typ
 	return 0;
 }
 
-static int vangogh_print_legacy_clk_levels(struct smu_context *smu,
-			enum smu_clk_type clk_type, char *buf)
+static int vangogh_emit_legacy_clk_levels(struct smu_context *smu,
+					   enum smu_clk_type clk_type, char *buf,
+					   int *offset)
 {
+	int i, idx, size = *offset, ret = 0, start_offset = *offset;
 	DpmClocks_t *clk_table = smu->smu_table.clocks_table;
 	SmuMetrics_legacy_t metrics;
 	struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm);
-	int i, idx, size = 0, ret = 0, start_offset = 0;
 	uint32_t cur_value = 0, value = 0, count = 0;
 	bool cur_value_match_level = false;
 
@@ -575,9 +576,6 @@ static int vangogh_print_legacy_clk_levels(struct smu_context *smu,
 	if (ret)
 		return ret;
 
-	smu_cmn_get_sysfs_buf(&buf, &size);
-	start_offset = size;
-
 	switch (clk_type) {
 	case SMU_OD_SCLK:
 		if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) {
@@ -659,15 +657,18 @@ static int vangogh_print_legacy_clk_levels(struct smu_context *smu,
 		break;
 	}
 
-	return size - start_offset;
+	*offset += size - start_offset;
+
+	return 0;
 }
 
-static int vangogh_print_clk_levels(struct smu_context *smu,
-			enum smu_clk_type clk_type, char *buf)
+static int vangogh_emit_clk_levels(struct smu_context *smu,
+				    enum smu_clk_type clk_type, char *buf,
+				    int *offset)
 {
+	int i, idx, size = *offset, ret = 0, start_offset = *offset;
 	DpmClocks_t *clk_table = smu->smu_table.clocks_table;
 	SmuMetrics_t metrics;
-	int i, idx, size = 0, ret = 0, start_offset = 0;
 	uint32_t cur_value = 0, value = 0, count = 0;
 	bool cur_value_match_level = false;
 	uint32_t min, max;
@@ -678,9 +679,6 @@ static int vangogh_print_clk_levels(struct smu_context *smu,
 	if (ret)
 		return ret;
 
-	smu_cmn_get_sysfs_buf(&buf, &size);
-	start_offset = size;
-
 	switch (clk_type) {
 	case SMU_OD_SCLK:
 		size += sysfs_emit_at(buf, size, "%s:\n", "OD_SCLK");
@@ -781,18 +779,21 @@ static int vangogh_print_clk_levels(struct smu_context *smu,
 		break;
 	}
 
-	return size - start_offset;
+	*offset += size - start_offset;
+
+	return 0;
 }
 
-static int vangogh_common_print_clk_levels(struct smu_context *smu,
-			enum smu_clk_type clk_type, char *buf)
+static int vangogh_common_emit_clk_levels(struct smu_context *smu,
+					   enum smu_clk_type clk_type, char *buf,
+					   int *offset)
 {
 	int ret = 0;
 
 	if (smu->smc_fw_if_version < 0x3)
-		ret = vangogh_print_legacy_clk_levels(smu, clk_type, buf);
+		ret = vangogh_emit_legacy_clk_levels(smu, clk_type, buf, offset);
 	else
-		ret = vangogh_print_clk_levels(smu, clk_type, buf);
+		ret = vangogh_emit_clk_levels(smu, clk_type, buf, offset);
 
 	return ret;
 }
@@ -2524,7 +2525,7 @@ static const struct pptable_funcs vangogh_ppt_funcs = {
 	.interrupt_work = smu_v11_0_interrupt_work,
 	.get_gpu_metrics = vangogh_common_get_gpu_metrics,
 	.od_edit_dpm_table = vangogh_od_edit_dpm_table,
-	.print_clk_levels = vangogh_common_print_clk_levels,
+	.emit_clk_levels = vangogh_common_emit_clk_levels,
 	.set_default_dpm_table = vangogh_set_default_dpm_tables,
 	.set_fine_grain_gfx_freq_parameters = vangogh_set_fine_grain_gfx_freq_parameters,
 	.notify_rlc_state = vangogh_notify_rlc_state,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c
index eaa9ea162f16..45c0afb47c51 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c
@@ -491,11 +491,12 @@ static int renoir_set_fine_grain_gfx_freq_parameters(struct smu_context *smu)
 	return 0;
 }
 
-static int renoir_print_clk_levels(struct smu_context *smu,
-			enum smu_clk_type clk_type, char *buf)
+static int renoir_emit_clk_levels(struct smu_context *smu,
+				  enum smu_clk_type clk_type, char *buf,
+				  int *offset)
 {
-	int i, idx, size = 0, ret = 0, start_offset = 0;
 	uint32_t cur_value = 0, value = 0, count = 0, min = 0, max = 0;
+	int i, idx, size = *offset, ret = 0, start_offset = *offset;
 	SmuMetrics_t metrics;
 	bool cur_value_match_level = false;
 
@@ -505,9 +506,6 @@ static int renoir_print_clk_levels(struct smu_context *smu,
 	if (ret)
 		return ret;
 
-	smu_cmn_get_sysfs_buf(&buf, &size);
-	start_offset = size;
-
 	switch (clk_type) {
 	case SMU_OD_RANGE:
 		ret = smu_cmn_send_smc_msg_with_param(smu,
@@ -534,24 +532,24 @@ static int renoir_print_clk_levels(struct smu_context *smu,
 		/* retirve table returned paramters unit is MHz */
 		cur_value = metrics.ClockFrequency[CLOCK_GFXCLK];
 		ret = renoir_get_dpm_ultimate_freq(smu, SMU_GFXCLK, &min, &max);
-		if (!ret) {
-			/* driver only know min/max gfx_clk, Add level 1 for all other gfx clks */
-			if (cur_value  == max)
-				i = 2;
-			else if (cur_value == min)
-				i = 0;
-			else
-				i = 1;
-
-			size += sysfs_emit_at(buf, size, "0: %uMhz %s\n", min,
-					i == 0 ? "*" : "");
-			size += sysfs_emit_at(buf, size, "1: %uMhz %s\n",
-					i == 1 ? cur_value : RENOIR_UMD_PSTATE_GFXCLK,
-					i == 1 ? "*" : "");
-			size += sysfs_emit_at(buf, size, "2: %uMhz %s\n", max,
-					i == 2 ? "*" : "");
-		}
-		return size - start_offset;
+		if (ret)
+			return ret;
+		/* driver only know min/max gfx_clk, Add level 1 for all other gfx clks */
+		if (cur_value == max)
+			i = 2;
+		else if (cur_value == min)
+			i = 0;
+		else
+			i = 1;
+		size += sysfs_emit_at(buf, size, "0: %uMhz %s\n", min,
+				      i == 0 ? "*" : "");
+		size += sysfs_emit_at(buf, size, "1: %uMhz %s\n",
+				      i == 1 ? cur_value :
+					       RENOIR_UMD_PSTATE_GFXCLK,
+				      i == 1 ? "*" : "");
+		size += sysfs_emit_at(buf, size, "2: %uMhz %s\n", max,
+				      i == 2 ? "*" : "");
+		break;
 	case SMU_SOCCLK:
 		count = NUM_SOCCLK_DPM_LEVELS;
 		cur_value = metrics.ClockFrequency[CLOCK_SOCCLK];
@@ -608,7 +606,9 @@ static int renoir_print_clk_levels(struct smu_context *smu,
 		break;
 	}
 
-	return size - start_offset;
+	*offset += size - start_offset;
+
+	return 0;
 }
 
 static enum amd_pm_state_type renoir_get_current_power_state(struct smu_context *smu)
@@ -1450,7 +1450,7 @@ static int renoir_get_enabled_mask(struct smu_context *smu,
 
 static const struct pptable_funcs renoir_ppt_funcs = {
 	.set_power_state = NULL,
-	.print_clk_levels = renoir_print_clk_levels,
+	.emit_clk_levels = renoir_emit_clk_levels,
 	.get_current_power_state = renoir_get_current_power_state,
 	.dpm_set_vcn_enable = renoir_dpm_set_vcn_enable,
 	.dpm_set_jpeg_enable = renoir_dpm_set_jpeg_enable,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
index 18d5d0704509..a72aaf36c282 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
@@ -71,9 +71,6 @@
 			  FEATURE_MASK(FEATURE_DPM_XGMI_BIT)	| \
 			  FEATURE_MASK(FEATURE_DPM_VCN_BIT))
 
-/* possible frequency drift (1Mhz) */
-#define EPSILON				1
-
 #define smnPCIE_ESM_CTRL			0x111003D0
 
 /*
@@ -347,7 +344,7 @@ static int aldebaran_get_dpm_ultimate_freq(struct smu_context *smu,
 					   uint32_t *min, uint32_t *max)
 {
 	struct smu_13_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
-	struct smu_13_0_dpm_table *dpm_table;
+	struct smu_dpm_table *dpm_table;
 	uint32_t min_clk, max_clk;
 
 	if (amdgpu_sriov_vf(smu->adev)) {
@@ -376,8 +373,8 @@ static int aldebaran_get_dpm_ultimate_freq(struct smu_context *smu,
 			return -EINVAL;
 		}
 
-		min_clk = dpm_table->min;
-		max_clk = dpm_table->max;
+		min_clk = SMU_DPM_TABLE_MIN(dpm_table);
+		max_clk = SMU_DPM_TABLE_MAX(dpm_table);
 
 		if (min) {
 			if (!min_clk)
@@ -400,12 +397,13 @@ static int aldebaran_get_dpm_ultimate_freq(struct smu_context *smu,
 static int aldebaran_set_default_dpm_table(struct smu_context *smu)
 {
 	struct smu_13_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
-	struct smu_13_0_dpm_table *dpm_table = NULL;
+	struct smu_dpm_table *dpm_table = NULL;
 	PPTable_t *pptable = smu->smu_table.driver_pptable;
 	int ret = 0;
 
 	/* socclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.soc_table;
+	dpm_table->clk_type = SMU_SOCCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_SOCCLK_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_SOCCLK,
@@ -416,12 +414,11 @@ static int aldebaran_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.socclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* gfxclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.gfx_table;
+	dpm_table->clk_type = SMU_GFXCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT)) {
 		/* in the case of gfxclk, only fine-grained dpm is honored */
 		dpm_table->count = 2;
@@ -429,18 +426,15 @@ static int aldebaran_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->dpm_levels[0].enabled = true;
 		dpm_table->dpm_levels[1].value = pptable->GfxclkFmax;
 		dpm_table->dpm_levels[1].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[1].value;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* memclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.uclk_table;
+	dpm_table->clk_type = SMU_UCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_UCLK_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_UCLK,
@@ -451,12 +445,11 @@ static int aldebaran_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.uclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* fclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.fclk_table;
+	dpm_table->clk_type = SMU_FCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_FCLK_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_FCLK,
@@ -467,8 +460,6 @@ static int aldebaran_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.fclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	return 0;
@@ -593,29 +584,26 @@ static int aldebaran_populate_umd_state_clk(struct smu_context *smu)
 {
 	struct smu_13_0_dpm_context *dpm_context =
 		smu->smu_dpm.dpm_context;
-	struct smu_13_0_dpm_table *gfx_table =
-		&dpm_context->dpm_tables.gfx_table;
-	struct smu_13_0_dpm_table *mem_table =
-		&dpm_context->dpm_tables.uclk_table;
-	struct smu_13_0_dpm_table *soc_table =
-		&dpm_context->dpm_tables.soc_table;
+	struct smu_dpm_table *gfx_table = &dpm_context->dpm_tables.gfx_table;
+	struct smu_dpm_table *mem_table = &dpm_context->dpm_tables.uclk_table;
+	struct smu_dpm_table *soc_table = &dpm_context->dpm_tables.soc_table;
 	struct smu_umd_pstate_table *pstate_table =
 		&smu->pstate_table;
 
-	pstate_table->gfxclk_pstate.min = gfx_table->min;
-	pstate_table->gfxclk_pstate.peak = gfx_table->max;
-	pstate_table->gfxclk_pstate.curr.min = gfx_table->min;
-	pstate_table->gfxclk_pstate.curr.max = gfx_table->max;
+	pstate_table->gfxclk_pstate.min = SMU_DPM_TABLE_MIN(gfx_table);
+	pstate_table->gfxclk_pstate.peak = SMU_DPM_TABLE_MAX(gfx_table);
+	pstate_table->gfxclk_pstate.curr.min = SMU_DPM_TABLE_MIN(gfx_table);
+	pstate_table->gfxclk_pstate.curr.max = SMU_DPM_TABLE_MAX(gfx_table);
 
-	pstate_table->uclk_pstate.min = mem_table->min;
-	pstate_table->uclk_pstate.peak = mem_table->max;
-	pstate_table->uclk_pstate.curr.min = mem_table->min;
-	pstate_table->uclk_pstate.curr.max = mem_table->max;
+	pstate_table->uclk_pstate.min = SMU_DPM_TABLE_MIN(mem_table);
+	pstate_table->uclk_pstate.peak = SMU_DPM_TABLE_MAX(mem_table);
+	pstate_table->uclk_pstate.curr.min = SMU_DPM_TABLE_MIN(mem_table);
+	pstate_table->uclk_pstate.curr.max = SMU_DPM_TABLE_MAX(mem_table);
 
-	pstate_table->socclk_pstate.min = soc_table->min;
-	pstate_table->socclk_pstate.peak = soc_table->max;
-	pstate_table->socclk_pstate.curr.min = soc_table->min;
-	pstate_table->socclk_pstate.curr.max = soc_table->max;
+	pstate_table->socclk_pstate.min = SMU_DPM_TABLE_MIN(soc_table);
+	pstate_table->socclk_pstate.peak = SMU_DPM_TABLE_MAX(soc_table);
+	pstate_table->socclk_pstate.curr.min = SMU_DPM_TABLE_MIN(soc_table);
+	pstate_table->socclk_pstate.curr.max = SMU_DPM_TABLE_MAX(soc_table);
 
 	if (gfx_table->count > ALDEBARAN_UMD_PSTATE_GFXCLK_LEVEL &&
 	    mem_table->count > ALDEBARAN_UMD_PSTATE_MCLK_LEVEL &&
@@ -638,30 +626,6 @@ static int aldebaran_populate_umd_state_clk(struct smu_context *smu)
 	return 0;
 }
 
-static void aldebaran_get_clk_table(struct smu_context *smu,
-				    struct pp_clock_levels_with_latency *clocks,
-				    struct smu_13_0_dpm_table *dpm_table)
-{
-	uint32_t i;
-
-	clocks->num_levels = min_t(uint32_t,
-				   dpm_table->count,
-				   (uint32_t)PP_MAX_CLOCK_LEVELS);
-
-	for (i = 0; i < clocks->num_levels; i++) {
-		clocks->data[i].clocks_in_khz =
-			dpm_table->dpm_levels[i].value * 1000;
-		clocks->data[i].latency_in_us = 0;
-	}
-
-}
-
-static int aldebaran_freqs_in_same_level(int32_t frequency1,
-					 int32_t frequency2)
-{
-	return (abs(frequency1 - frequency2) <= EPSILON);
-}
-
 static int aldebaran_get_smu_metrics_data(struct smu_context *smu,
 					  MetricsMember_t member,
 					  uint32_t *value)
@@ -828,16 +792,10 @@ static int aldebaran_emit_clk_levels(struct smu_context *smu,
 {
 	int ret = 0;
 	struct smu_umd_pstate_table *pstate_table = &smu->pstate_table;
-	struct pp_clock_levels_with_latency clocks;
-	struct smu_13_0_dpm_table *single_dpm_table;
+	struct smu_dpm_table *single_dpm_table = NULL;
 	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
 	struct smu_13_0_dpm_context *dpm_context = NULL;
-	uint32_t i;
-	int display_levels;
-	uint32_t freq_values[3] = {0};
-	uint32_t min_clk, max_clk, cur_value = 0;
-	bool freq_match;
-	unsigned int clock_mhz;
+	uint32_t cur_value = 0;
 	static const char attempt_string[] = "Attempt to get current";
 
 	if (amdgpu_ras_intr_triggered()) {
@@ -848,134 +806,50 @@ static int aldebaran_emit_clk_levels(struct smu_context *smu,
 	dpm_context = smu_dpm->dpm_context;
 
 	switch (type) {
-
 	case SMU_OD_SCLK:
 		*offset += sysfs_emit_at(buf, *offset, "%s:\n", "OD_SCLK");
 		*offset += sysfs_emit_at(buf, *offset, "0: %uMhz\n1: %uMhz\n",
-				      pstate_table->gfxclk_pstate.curr.min,
-				      pstate_table->gfxclk_pstate.curr.max);
+					 pstate_table->gfxclk_pstate.curr.min,
+					 pstate_table->gfxclk_pstate.curr.max);
 		return 0;
-	case SMU_SCLK:
-		ret = aldebaran_get_current_clk_freq_by_table(smu, SMU_GFXCLK, &cur_value);
-		if (ret) {
-			dev_err(smu->adev->dev, "%s gfx clk Failed!", attempt_string);
-			return ret;
-		}
-
-		single_dpm_table = &(dpm_context->dpm_tables.gfx_table);
-		aldebaran_get_clk_table(smu, &clocks, single_dpm_table);
-
-		display_levels = (clocks.num_levels == 1) ? 1 : 2;
-
-		min_clk = pstate_table->gfxclk_pstate.curr.min;
-		max_clk = pstate_table->gfxclk_pstate.curr.max;
-
-		freq_values[0] = min_clk;
-		freq_values[1] = max_clk;
-
-		/* fine-grained dpm has only 2 levels */
-		if (cur_value > min_clk && cur_value < max_clk) {
-			display_levels++;
-			freq_values[2] = max_clk;
-			freq_values[1] = cur_value;
-		}
-		break;
-
 	case SMU_OD_MCLK:
 		*offset += sysfs_emit_at(buf, *offset, "%s:\n", "OD_MCLK");
 		*offset += sysfs_emit_at(buf, *offset, "0: %uMhz\n1: %uMhz\n",
-				      pstate_table->uclk_pstate.curr.min,
-				      pstate_table->uclk_pstate.curr.max);
+					 pstate_table->uclk_pstate.curr.min,
+					 pstate_table->uclk_pstate.curr.max);
 		return 0;
-	case SMU_MCLK:
-		ret = aldebaran_get_current_clk_freq_by_table(smu, SMU_UCLK, &cur_value);
-		if (ret) {
-			dev_err(smu->adev->dev, "%s mclk Failed!", attempt_string);
-			return ret;
-		}
 
+	case SMU_SCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.gfx_table);
+		break;
+	case SMU_MCLK:
 		single_dpm_table = &(dpm_context->dpm_tables.uclk_table);
-		aldebaran_get_clk_table(smu, &clocks, single_dpm_table);
 		break;
-
 	case SMU_SOCCLK:
-		ret = aldebaran_get_current_clk_freq_by_table(smu, SMU_SOCCLK, &cur_value);
-		if (ret) {
-			dev_err(smu->adev->dev, "%s socclk Failed!", attempt_string);
-			return ret;
-		}
-
 		single_dpm_table = &(dpm_context->dpm_tables.soc_table);
-		aldebaran_get_clk_table(smu, &clocks, single_dpm_table);
 		break;
-
 	case SMU_FCLK:
-		ret = aldebaran_get_current_clk_freq_by_table(smu, SMU_FCLK, &cur_value);
-		if (ret) {
-			dev_err(smu->adev->dev, "%s fclk Failed!", attempt_string);
-			return ret;
-		}
-
 		single_dpm_table = &(dpm_context->dpm_tables.fclk_table);
-		aldebaran_get_clk_table(smu, &clocks, single_dpm_table);
 		break;
-
 	case SMU_VCLK:
-		ret = aldebaran_get_current_clk_freq_by_table(smu, SMU_VCLK, &cur_value);
-		if (ret) {
-			dev_err(smu->adev->dev, "%s vclk Failed!", attempt_string);
-			return ret;
-		}
-
 		single_dpm_table = &(dpm_context->dpm_tables.vclk_table);
-		aldebaran_get_clk_table(smu, &clocks, single_dpm_table);
 		break;
-
 	case SMU_DCLK:
-		ret = aldebaran_get_current_clk_freq_by_table(smu, SMU_DCLK, &cur_value);
-		if (ret) {
-			dev_err(smu->adev->dev, "%s dclk Failed!", attempt_string);
-			return ret;
-		}
-
 		single_dpm_table = &(dpm_context->dpm_tables.dclk_table);
-		aldebaran_get_clk_table(smu, &clocks, single_dpm_table);
 		break;
-
 	default:
 		return -EINVAL;
 	}
 
-	switch (type) {
-	case SMU_SCLK:
-		for (i = 0; i < display_levels; i++) {
-			clock_mhz = freq_values[i];
-			freq_match = aldebaran_freqs_in_same_level(clock_mhz, cur_value);
-			freq_match |= (display_levels == 1);
-
-			*offset += sysfs_emit_at(buf, *offset, "%d: %uMhz %s\n", i,
-				clock_mhz,
-				(freq_match) ? "*" : "");
-		}
-		break;
-
-	case SMU_MCLK:
-	case SMU_SOCCLK:
-	case SMU_FCLK:
-	case SMU_VCLK:
-	case SMU_DCLK:
-		for (i = 0; i < clocks.num_levels; i++) {
-			clock_mhz = clocks.data[i].clocks_in_khz / 1000;
-			freq_match = aldebaran_freqs_in_same_level(clock_mhz, cur_value);
-			freq_match |= (clocks.num_levels == 1);
-
-			*offset += sysfs_emit_at(buf, *offset, "%d: %uMhz %s\n",
-				i, clock_mhz,
-				(freq_match) ? "*" : "");
+	if (single_dpm_table) {
+		ret = aldebaran_get_current_clk_freq_by_table(smu, type,
+							      &cur_value);
+		if (ret) {
+			dev_err(smu->adev->dev, "%s Failed!", attempt_string);
+			return ret;
 		}
-		break;
-	default:
-		return -EINVAL;
+		return smu_cmn_print_dpm_clk_levels(smu, single_dpm_table,
+						    cur_value, buf, offset);
 	}
 
 	return 0;
@@ -1040,7 +914,7 @@ static int aldebaran_force_clk_levels(struct smu_context *smu,
 				      enum smu_clk_type type, uint32_t mask)
 {
 	struct smu_13_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
-	struct smu_13_0_dpm_table *single_dpm_table = NULL;
+	struct smu_dpm_table *single_dpm_table = NULL;
 	uint32_t soft_min_level, soft_max_level;
 	int ret = 0;
 
@@ -1322,8 +1196,7 @@ static int aldebaran_set_performance_level(struct smu_context *smu,
 {
 	struct smu_dpm_context *smu_dpm = &(smu->smu_dpm);
 	struct smu_13_0_dpm_context *dpm_context = smu_dpm->dpm_context;
-	struct smu_13_0_dpm_table *gfx_table =
-		&dpm_context->dpm_tables.gfx_table;
+	struct smu_dpm_table *gfx_table = &dpm_context->dpm_tables.gfx_table;
 	struct smu_umd_pstate_table *pstate_table = &smu->pstate_table;
 	int r;
 
@@ -1331,7 +1204,8 @@ static int aldebaran_set_performance_level(struct smu_context *smu,
 	if ((smu_dpm->dpm_level == AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM) &&
 	    (level != AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM)) {
 		smu_cmn_send_smc_msg(smu, SMU_MSG_DisableDeterminism, NULL);
-		pstate_table->gfxclk_pstate.curr.max = gfx_table->max;
+		pstate_table->gfxclk_pstate.curr.max =
+			SMU_DPM_TABLE_MAX(gfx_table);
 	}
 
 	switch (level) {
@@ -1372,7 +1246,8 @@ static int aldebaran_set_soft_freq_limited_range(struct smu_context *smu,
 
 	if (clk_type != SMU_GFXCLK && clk_type != SMU_SCLK)
 		return -EINVAL;
-
+	min_clk = SMU_DPM_TABLE_MIN(&dpm_context->dpm_tables.gfx_table);
+	max_clk = SMU_DPM_TABLE_MAX(&dpm_context->dpm_tables.gfx_table);
 	if ((smu_dpm->dpm_level != AMD_DPM_FORCED_LEVEL_MANUAL)
 			&& (smu_dpm->dpm_level != AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM))
 		return -EINVAL;
@@ -1399,16 +1274,13 @@ static int aldebaran_set_soft_freq_limited_range(struct smu_context *smu,
 	}
 
 	if (smu_dpm->dpm_level == AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM) {
-		if (!max || (max < dpm_context->dpm_tables.gfx_table.min) ||
-			(max > dpm_context->dpm_tables.gfx_table.max)) {
+		if (!max || (max < min_clk) || (max > max_clk)) {
 			dev_warn(adev->dev,
 					"Invalid max frequency %d MHz specified for determinism\n", max);
 			return -EINVAL;
 		}
 
 		/* Restore default min/max clocks and enable determinism */
-		min_clk = dpm_context->dpm_tables.gfx_table.min;
-		max_clk = dpm_context->dpm_tables.gfx_table.max;
 		ret = smu_v13_0_set_soft_freq_limited_range(smu, SMU_GFXCLK, min_clk, max_clk, false);
 		if (!ret) {
 			usleep_range(500, 1000);
@@ -1449,11 +1321,14 @@ static int aldebaran_usr_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_
 			dev_err(smu->adev->dev, "Input parameter number not correct\n");
 			return -EINVAL;
 		}
-
+		min_clk = SMU_DPM_TABLE_MIN(&dpm_context->dpm_tables.gfx_table);
+		max_clk = SMU_DPM_TABLE_MAX(&dpm_context->dpm_tables.gfx_table);
 		if (input[0] == 0) {
-			if (input[1] < dpm_context->dpm_tables.gfx_table.min) {
-				dev_warn(smu->adev->dev, "Minimum GFX clk (%ld) MHz specified is less than the minimum allowed (%d) MHz\n",
-					input[1], dpm_context->dpm_tables.gfx_table.min);
+			if (input[1] < min_clk) {
+				dev_warn(
+					smu->adev->dev,
+					"Minimum GFX clk (%ld) MHz specified is less than the minimum allowed (%d) MHz\n",
+					input[1], min_clk);
 				pstate_table->gfxclk_pstate.custom.min =
 					pstate_table->gfxclk_pstate.curr.min;
 				return -EINVAL;
@@ -1461,9 +1336,11 @@ static int aldebaran_usr_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_
 
 			pstate_table->gfxclk_pstate.custom.min = input[1];
 		} else if (input[0] == 1) {
-			if (input[1] > dpm_context->dpm_tables.gfx_table.max) {
-				dev_warn(smu->adev->dev, "Maximum GFX clk (%ld) MHz specified is greater than the maximum allowed (%d) MHz\n",
-					input[1], dpm_context->dpm_tables.gfx_table.max);
+			if (input[1] > max_clk) {
+				dev_warn(
+					smu->adev->dev,
+					"Maximum GFX clk (%ld) MHz specified is greater than the maximum allowed (%d) MHz\n",
+					input[1], max_clk);
 				pstate_table->gfxclk_pstate.custom.max =
 					pstate_table->gfxclk_pstate.curr.max;
 				return -EINVAL;
@@ -1480,8 +1357,10 @@ static int aldebaran_usr_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_
 			return -EINVAL;
 		} else {
 			/* Use the default frequencies for manual and determinism mode */
-			min_clk = dpm_context->dpm_tables.gfx_table.min;
-			max_clk = dpm_context->dpm_tables.gfx_table.max;
+			min_clk = SMU_DPM_TABLE_MIN(
+				&dpm_context->dpm_tables.gfx_table);
+			max_clk = SMU_DPM_TABLE_MAX(
+				&dpm_context->dpm_tables.gfx_table);
 
 			ret = aldebaran_set_soft_freq_limited_range(
 				smu, SMU_GFXCLK, min_clk, max_clk, false);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
index a89075e25717..7340237b3c35 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
@@ -1585,18 +1585,12 @@ int smu_v13_0_set_performance_level(struct smu_context *smu,
 {
 	struct smu_13_0_dpm_context *dpm_context =
 		smu->smu_dpm.dpm_context;
-	struct smu_13_0_dpm_table *gfx_table =
-		&dpm_context->dpm_tables.gfx_table;
-	struct smu_13_0_dpm_table *mem_table =
-		&dpm_context->dpm_tables.uclk_table;
-	struct smu_13_0_dpm_table *soc_table =
-		&dpm_context->dpm_tables.soc_table;
-	struct smu_13_0_dpm_table *vclk_table =
-		&dpm_context->dpm_tables.vclk_table;
-	struct smu_13_0_dpm_table *dclk_table =
-		&dpm_context->dpm_tables.dclk_table;
-	struct smu_13_0_dpm_table *fclk_table =
-		&dpm_context->dpm_tables.fclk_table;
+	struct smu_dpm_table *gfx_table = &dpm_context->dpm_tables.gfx_table;
+	struct smu_dpm_table *mem_table = &dpm_context->dpm_tables.uclk_table;
+	struct smu_dpm_table *soc_table = &dpm_context->dpm_tables.soc_table;
+	struct smu_dpm_table *vclk_table = &dpm_context->dpm_tables.vclk_table;
+	struct smu_dpm_table *dclk_table = &dpm_context->dpm_tables.dclk_table;
+	struct smu_dpm_table *fclk_table = &dpm_context->dpm_tables.fclk_table;
 	struct smu_umd_pstate_table *pstate_table =
 		&smu->pstate_table;
 	struct amdgpu_device *adev = smu->adev;
@@ -1611,34 +1605,34 @@ int smu_v13_0_set_performance_level(struct smu_context *smu,
 
 	switch (level) {
 	case AMD_DPM_FORCED_LEVEL_HIGH:
-		sclk_min = sclk_max = gfx_table->max;
-		mclk_min = mclk_max = mem_table->max;
-		socclk_min = socclk_max = soc_table->max;
-		vclk_min = vclk_max = vclk_table->max;
-		dclk_min = dclk_max = dclk_table->max;
-		fclk_min = fclk_max = fclk_table->max;
+		sclk_min = sclk_max = SMU_DPM_TABLE_MAX(gfx_table);
+		mclk_min = mclk_max = SMU_DPM_TABLE_MAX(mem_table);
+		socclk_min = socclk_max = SMU_DPM_TABLE_MAX(soc_table);
+		vclk_min = vclk_max = SMU_DPM_TABLE_MAX(vclk_table);
+		dclk_min = dclk_max = SMU_DPM_TABLE_MAX(dclk_table);
+		fclk_min = fclk_max = SMU_DPM_TABLE_MAX(fclk_table);
 		break;
 	case AMD_DPM_FORCED_LEVEL_LOW:
-		sclk_min = sclk_max = gfx_table->min;
-		mclk_min = mclk_max = mem_table->min;
-		socclk_min = socclk_max = soc_table->min;
-		vclk_min = vclk_max = vclk_table->min;
-		dclk_min = dclk_max = dclk_table->min;
-		fclk_min = fclk_max = fclk_table->min;
+		sclk_min = sclk_max = SMU_DPM_TABLE_MIN(gfx_table);
+		mclk_min = mclk_max = SMU_DPM_TABLE_MIN(mem_table);
+		socclk_min = socclk_max = SMU_DPM_TABLE_MIN(soc_table);
+		vclk_min = vclk_max = SMU_DPM_TABLE_MIN(vclk_table);
+		dclk_min = dclk_max = SMU_DPM_TABLE_MIN(dclk_table);
+		fclk_min = fclk_max = SMU_DPM_TABLE_MIN(fclk_table);
 		break;
 	case AMD_DPM_FORCED_LEVEL_AUTO:
-		sclk_min = gfx_table->min;
-		sclk_max = gfx_table->max;
-		mclk_min = mem_table->min;
-		mclk_max = mem_table->max;
-		socclk_min = soc_table->min;
-		socclk_max = soc_table->max;
-		vclk_min = vclk_table->min;
-		vclk_max = vclk_table->max;
-		dclk_min = dclk_table->min;
-		dclk_max = dclk_table->max;
-		fclk_min = fclk_table->min;
-		fclk_max = fclk_table->max;
+		sclk_min = SMU_DPM_TABLE_MIN(gfx_table);
+		sclk_max = SMU_DPM_TABLE_MAX(gfx_table);
+		mclk_min = SMU_DPM_TABLE_MIN(mem_table);
+		mclk_max = SMU_DPM_TABLE_MAX(mem_table);
+		socclk_min = SMU_DPM_TABLE_MIN(soc_table);
+		socclk_max = SMU_DPM_TABLE_MAX(soc_table);
+		vclk_min = SMU_DPM_TABLE_MIN(vclk_table);
+		vclk_max = SMU_DPM_TABLE_MAX(vclk_table);
+		dclk_min = SMU_DPM_TABLE_MIN(dclk_table);
+		dclk_max = SMU_DPM_TABLE_MAX(dclk_table);
+		fclk_min = SMU_DPM_TABLE_MIN(fclk_table);
+		fclk_max = SMU_DPM_TABLE_MAX(fclk_table);
 		auto_level = true;
 		break;
 	case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
@@ -1909,11 +1903,12 @@ static int smu_v13_0_get_fine_grained_status(struct smu_context *smu,
 
 int smu_v13_0_set_single_dpm_table(struct smu_context *smu,
 				   enum smu_clk_type clk_type,
-				   struct smu_13_0_dpm_table *single_dpm_table)
+				   struct smu_dpm_table *single_dpm_table)
 {
 	int ret = 0;
 	uint32_t clk;
 	int i;
+	bool is_fine_grained;
 
 	ret = smu_v13_0_get_dpm_level_count(smu,
 					    clk_type,
@@ -1924,13 +1919,14 @@ int smu_v13_0_set_single_dpm_table(struct smu_context *smu,
 	}
 
 	if (amdgpu_ip_version(smu->adev, MP1_HWIP, 0) != IP_VERSION(13, 0, 2)) {
-		ret = smu_v13_0_get_fine_grained_status(smu,
-							clk_type,
-							&single_dpm_table->is_fine_grained);
+		ret = smu_v13_0_get_fine_grained_status(smu, clk_type,
+							&is_fine_grained);
 		if (ret) {
 			dev_err(smu->adev->dev, "[%s] failed to get fine grained status!\n", __func__);
 			return ret;
 		}
+		if (is_fine_grained)
+			single_dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 	}
 
 	for (i = 0; i < single_dpm_table->count; i++) {
@@ -1945,11 +1941,6 @@ int smu_v13_0_set_single_dpm_table(struct smu_context *smu,
 
 		single_dpm_table->dpm_levels[i].value = clk;
 		single_dpm_table->dpm_levels[i].enabled = true;
-
-		if (i == 0)
-			single_dpm_table->min = clk;
-		else if (i == single_dpm_table->count - 1)
-			single_dpm_table->max = clk;
 	}
 
 	return 0;
@@ -2383,9 +2374,8 @@ int smu_v13_0_update_pcie_parameters(struct smu_context *smu,
 				     uint8_t pcie_width_cap)
 {
 	struct smu_13_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
-	struct smu_13_0_pcie_table *pcie_table =
-				&dpm_context->dpm_tables.pcie_table;
-	int num_of_levels = pcie_table->num_of_link_levels;
+	struct smu_pcie_table *pcie_table = &dpm_context->dpm_tables.pcie_table;
+	int num_of_levels = pcie_table->lclk_levels;
 	uint32_t smu_pcie_arg;
 	int ret = 0;
 	int i;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
index 677781060246..aa26107b0e8c 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
@@ -571,11 +571,12 @@ static int smu_v13_0_0_set_default_dpm_table(struct smu_context *smu)
 	struct smu_table_context *table_context = &smu->smu_table;
 	PPTable_t *pptable = table_context->driver_pptable;
 	SkuTable_t *skutable = &pptable->SkuTable;
-	struct smu_13_0_dpm_table *dpm_table;
+	struct smu_dpm_table *dpm_table;
 	int ret = 0;
 
 	/* socclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.soc_table;
+	dpm_table->clk_type = SMU_SOCCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_SOCCLK_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_SOCCLK,
@@ -586,12 +587,11 @@ static int smu_v13_0_0_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.socclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* gfxclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.gfx_table;
+	dpm_table->clk_type = SMU_GFXCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_GFXCLK,
@@ -613,18 +613,16 @@ static int smu_v13_0_0_set_default_dpm_table(struct smu_context *smu)
 		    skutable->DriverReportedClocks.GameClockAc)) {
 			dpm_table->dpm_levels[dpm_table->count - 1].value =
 				skutable->DriverReportedClocks.GameClockAc;
-			dpm_table->max = skutable->DriverReportedClocks.GameClockAc;
 		}
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* uclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.uclk_table;
+	dpm_table->clk_type = SMU_UCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_UCLK_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_UCLK,
@@ -635,12 +633,11 @@ static int smu_v13_0_0_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.uclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* fclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.fclk_table;
+	dpm_table->clk_type = SMU_FCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_FCLK_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_FCLK,
@@ -651,12 +648,11 @@ static int smu_v13_0_0_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.fclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* vclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.vclk_table;
+	dpm_table->clk_type = SMU_VCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_VCLK_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_VCLK,
@@ -667,12 +663,11 @@ static int smu_v13_0_0_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.vclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* dclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.dclk_table;
+	dpm_table->clk_type = SMU_DCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCLK_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_DCLK,
@@ -683,12 +678,11 @@ static int smu_v13_0_0_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* dcefclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.dcef_table;
+	dpm_table->clk_type = SMU_DCEFCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCN_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_DCEFCLK,
@@ -699,8 +693,6 @@ static int smu_v13_0_0_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dcefclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	return 0;
@@ -876,7 +868,7 @@ static int smu_v13_0_0_get_dpm_ultimate_freq(struct smu_context *smu,
 {
 	struct smu_13_0_dpm_context *dpm_context =
 		smu->smu_dpm.dpm_context;
-	struct smu_13_0_dpm_table *dpm_table;
+	struct smu_dpm_table *dpm_table;
 
 	switch (clk_type) {
 	case SMU_MCLK:
@@ -913,9 +905,9 @@ static int smu_v13_0_0_get_dpm_ultimate_freq(struct smu_context *smu,
 	}
 
 	if (min)
-		*min = dpm_table->min;
+		*min = SMU_DPM_TABLE_MIN(dpm_table);
 	if (max)
-		*max = dpm_table->max;
+		*max = SMU_DPM_TABLE_MAX(dpm_table);
 
 	return 0;
 }
@@ -1184,27 +1176,24 @@ static int smu_v13_0_0_upload_overdrive_table(struct smu_context *smu,
 	return ret;
 }
 
-static int smu_v13_0_0_print_clk_levels(struct smu_context *smu,
-					enum smu_clk_type clk_type,
-					char *buf)
+static int smu_v13_0_0_emit_clk_levels(struct smu_context *smu,
+				       enum smu_clk_type clk_type, char *buf,
+				       int *offset)
 {
 	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
 	struct smu_13_0_dpm_context *dpm_context = smu_dpm->dpm_context;
 	OverDriveTableExternal_t *od_table =
 		(OverDriveTableExternal_t *)smu->smu_table.overdrive_table;
-	struct smu_13_0_dpm_table *single_dpm_table;
-	struct smu_13_0_pcie_table *pcie_table;
+	int i, curr_freq, size = *offset, start_offset = *offset;
+	struct smu_dpm_table *single_dpm_table = NULL;
+	struct smu_pcie_table *pcie_table;
 	uint32_t gen_speed, lane_width;
-	int i, curr_freq, size = 0, start_offset = 0;
 	int32_t min_value, max_value;
 	int ret = 0;
 
-	smu_cmn_get_sysfs_buf(&buf, &size);
-	start_offset = size;
-
 	if (amdgpu_ras_intr_triggered()) {
-		size += sysfs_emit_at(buf, size, "unavailable\n");
-		return size - start_offset;
+		sysfs_emit_at(buf, size, "unavailable\n");
+		return -EBUSY;
 	}
 
 	switch (clk_type) {
@@ -1231,61 +1220,6 @@ static int smu_v13_0_0_print_clk_levels(struct smu_context *smu,
 	case SMU_DCEFCLK:
 		single_dpm_table = &(dpm_context->dpm_tables.dcef_table);
 		break;
-	default:
-		break;
-	}
-
-	switch (clk_type) {
-	case SMU_SCLK:
-	case SMU_MCLK:
-	case SMU_SOCCLK:
-	case SMU_FCLK:
-	case SMU_VCLK:
-	case SMU_VCLK1:
-	case SMU_DCLK:
-	case SMU_DCLK1:
-	case SMU_DCEFCLK:
-		ret = smu_v13_0_0_get_current_clk_freq_by_table(smu, clk_type, &curr_freq);
-		if (ret) {
-			dev_err(smu->adev->dev, "Failed to get current clock freq!");
-			return ret;
-		}
-
-		if (single_dpm_table->is_fine_grained) {
-			/*
-			 * For fine grained dpms, there are only two dpm levels:
-			 *   - level 0 -> min clock freq
-			 *   - level 1 -> max clock freq
-			 * And the current clock frequency can be any value between them.
-			 * So, if the current clock frequency is not at level 0 or level 1,
-			 * we will fake it as three dpm levels:
-			 *   - level 0 -> min clock freq
-			 *   - level 1 -> current actual clock freq
-			 *   - level 2 -> max clock freq
-			 */
-			if ((single_dpm_table->dpm_levels[0].value != curr_freq) &&
-			     (single_dpm_table->dpm_levels[1].value != curr_freq)) {
-				size += sysfs_emit_at(buf, size, "0: %uMhz\n",
-						single_dpm_table->dpm_levels[0].value);
-				size += sysfs_emit_at(buf, size, "1: %uMhz *\n",
-						curr_freq);
-				size += sysfs_emit_at(buf, size, "2: %uMhz\n",
-						single_dpm_table->dpm_levels[1].value);
-			} else {
-				size += sysfs_emit_at(buf, size, "0: %uMhz %s\n",
-						single_dpm_table->dpm_levels[0].value,
-						single_dpm_table->dpm_levels[0].value == curr_freq ? "*" : "");
-				size += sysfs_emit_at(buf, size, "1: %uMhz %s\n",
-						single_dpm_table->dpm_levels[1].value,
-						single_dpm_table->dpm_levels[1].value == curr_freq ? "*" : "");
-			}
-		} else {
-			for (i = 0; i < single_dpm_table->count; i++)
-				size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
-						i, single_dpm_table->dpm_levels[i].value,
-						single_dpm_table->dpm_levels[i].value == curr_freq ? "*" : "");
-		}
-		break;
 	case SMU_PCIE:
 		ret = smu_v13_0_0_get_smu_metrics_data(smu,
 						       METRICS_PCIE_RATE,
@@ -1300,24 +1234,8 @@ static int smu_v13_0_0_print_clk_levels(struct smu_context *smu,
 			return ret;
 
 		pcie_table = &(dpm_context->dpm_tables.pcie_table);
-		for (i = 0; i < pcie_table->num_of_link_levels; i++)
-			size += sysfs_emit_at(buf, size, "%d: %s %s %dMhz %s\n", i,
-					(pcie_table->pcie_gen[i] == 0) ? "2.5GT/s," :
-					(pcie_table->pcie_gen[i] == 1) ? "5.0GT/s," :
-					(pcie_table->pcie_gen[i] == 2) ? "8.0GT/s," :
-					(pcie_table->pcie_gen[i] == 3) ? "16.0GT/s," : "",
-					(pcie_table->pcie_lane[i] == 1) ? "x1" :
-					(pcie_table->pcie_lane[i] == 2) ? "x2" :
-					(pcie_table->pcie_lane[i] == 3) ? "x4" :
-					(pcie_table->pcie_lane[i] == 4) ? "x8" :
-					(pcie_table->pcie_lane[i] == 5) ? "x12" :
-					(pcie_table->pcie_lane[i] == 6) ? "x16" : "",
-					pcie_table->clk_freq[i],
-					(gen_speed == DECODE_GEN_SPEED(pcie_table->pcie_gen[i])) &&
-					(lane_width == DECODE_LANE_WIDTH(pcie_table->pcie_lane[i])) ?
-					"*" : "");
-		break;
-
+		return smu_cmn_print_pcie_levels(smu, pcie_table, gen_speed,
+						 lane_width, buf, offset);
 	case SMU_OD_SCLK:
 		if (!smu_v13_0_0_is_od_feature_supported(smu,
 							 PP_OD_FEATURE_GFXCLK_BIT))
@@ -1535,7 +1453,21 @@ static int smu_v13_0_0_print_clk_levels(struct smu_context *smu,
 		break;
 	}
 
-	return size - start_offset;
+	if (single_dpm_table) {
+		ret = smu_v13_0_0_get_current_clk_freq_by_table(smu, clk_type,
+								&curr_freq);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Failed to get current clock freq!");
+			return ret;
+		}
+		return smu_cmn_print_dpm_clk_levels(smu, single_dpm_table,
+						    curr_freq, buf, offset);
+	}
+
+	*offset += size - start_offset;
+
+	return 0;
 }
 
 
@@ -1989,7 +1921,7 @@ static int smu_v13_0_0_force_clk_levels(struct smu_context *smu,
 {
 	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
 	struct smu_13_0_dpm_context *dpm_context = smu_dpm->dpm_context;
-	struct smu_13_0_dpm_table *single_dpm_table;
+	struct smu_dpm_table *single_dpm_table;
 	uint32_t soft_min_level, soft_max_level;
 	uint32_t min_freq, max_freq;
 	int ret = 0;
@@ -2035,7 +1967,7 @@ static int smu_v13_0_0_force_clk_levels(struct smu_context *smu,
 	case SMU_VCLK1:
 	case SMU_DCLK:
 	case SMU_DCLK1:
-		if (single_dpm_table->is_fine_grained) {
+		if (single_dpm_table->flags & SMU_DPM_TABLE_FINE_GRAINED) {
 			/* There is only 2 levels for fine grained DPM */
 			soft_max_level = (soft_max_level >= 1 ? 1 : 0);
 			soft_min_level = (soft_min_level >= 1 ? 1 : 0);
@@ -2303,18 +2235,12 @@ static int smu_v13_0_0_populate_umd_state_clk(struct smu_context *smu)
 {
 	struct smu_13_0_dpm_context *dpm_context =
 				smu->smu_dpm.dpm_context;
-	struct smu_13_0_dpm_table *gfx_table =
-				&dpm_context->dpm_tables.gfx_table;
-	struct smu_13_0_dpm_table *mem_table =
-				&dpm_context->dpm_tables.uclk_table;
-	struct smu_13_0_dpm_table *soc_table =
-				&dpm_context->dpm_tables.soc_table;
-	struct smu_13_0_dpm_table *vclk_table =
-				&dpm_context->dpm_tables.vclk_table;
-	struct smu_13_0_dpm_table *dclk_table =
-				&dpm_context->dpm_tables.dclk_table;
-	struct smu_13_0_dpm_table *fclk_table =
-				&dpm_context->dpm_tables.fclk_table;
+	struct smu_dpm_table *gfx_table = &dpm_context->dpm_tables.gfx_table;
+	struct smu_dpm_table *mem_table = &dpm_context->dpm_tables.uclk_table;
+	struct smu_dpm_table *soc_table = &dpm_context->dpm_tables.soc_table;
+	struct smu_dpm_table *vclk_table = &dpm_context->dpm_tables.vclk_table;
+	struct smu_dpm_table *dclk_table = &dpm_context->dpm_tables.dclk_table;
+	struct smu_dpm_table *fclk_table = &dpm_context->dpm_tables.fclk_table;
 	struct smu_umd_pstate_table *pstate_table =
 				&smu->pstate_table;
 	struct smu_table_context *table_context = &smu->smu_table;
@@ -2322,38 +2248,39 @@ static int smu_v13_0_0_populate_umd_state_clk(struct smu_context *smu)
 	DriverReportedClocks_t driver_clocks =
 			pptable->SkuTable.DriverReportedClocks;
 
-	pstate_table->gfxclk_pstate.min = gfx_table->min;
+	pstate_table->gfxclk_pstate.min = SMU_DPM_TABLE_MIN(gfx_table);
 	if (driver_clocks.GameClockAc &&
-	    (driver_clocks.GameClockAc < gfx_table->max))
+	    (driver_clocks.GameClockAc < SMU_DPM_TABLE_MAX(gfx_table)))
 		pstate_table->gfxclk_pstate.peak = driver_clocks.GameClockAc;
 	else
-		pstate_table->gfxclk_pstate.peak = gfx_table->max;
+		pstate_table->gfxclk_pstate.peak = SMU_DPM_TABLE_MAX(gfx_table);
 
-	pstate_table->uclk_pstate.min = mem_table->min;
-	pstate_table->uclk_pstate.peak = mem_table->max;
+	pstate_table->uclk_pstate.min = SMU_DPM_TABLE_MIN(mem_table);
+	pstate_table->uclk_pstate.peak = SMU_DPM_TABLE_MAX(mem_table);
 
-	pstate_table->socclk_pstate.min = soc_table->min;
-	pstate_table->socclk_pstate.peak = soc_table->max;
+	pstate_table->socclk_pstate.min = SMU_DPM_TABLE_MIN(soc_table);
+	pstate_table->socclk_pstate.peak = SMU_DPM_TABLE_MAX(soc_table);
 
-	pstate_table->vclk_pstate.min = vclk_table->min;
-	pstate_table->vclk_pstate.peak = vclk_table->max;
+	pstate_table->vclk_pstate.min = SMU_DPM_TABLE_MIN(vclk_table);
+	pstate_table->vclk_pstate.peak = SMU_DPM_TABLE_MAX(vclk_table);
 
-	pstate_table->dclk_pstate.min = dclk_table->min;
-	pstate_table->dclk_pstate.peak = dclk_table->max;
+	pstate_table->dclk_pstate.min = SMU_DPM_TABLE_MIN(dclk_table);
+	pstate_table->dclk_pstate.peak = SMU_DPM_TABLE_MAX(dclk_table);
 
-	pstate_table->fclk_pstate.min = fclk_table->min;
-	pstate_table->fclk_pstate.peak = fclk_table->max;
+	pstate_table->fclk_pstate.min = SMU_DPM_TABLE_MIN(fclk_table);
+	pstate_table->fclk_pstate.peak = SMU_DPM_TABLE_MAX(fclk_table);
 
 	if (driver_clocks.BaseClockAc &&
-	    driver_clocks.BaseClockAc < gfx_table->max)
+	    driver_clocks.BaseClockAc < SMU_DPM_TABLE_MAX(gfx_table))
 		pstate_table->gfxclk_pstate.standard = driver_clocks.BaseClockAc;
 	else
-		pstate_table->gfxclk_pstate.standard = gfx_table->max;
-	pstate_table->uclk_pstate.standard = mem_table->max;
-	pstate_table->socclk_pstate.standard = soc_table->min;
-	pstate_table->vclk_pstate.standard = vclk_table->min;
-	pstate_table->dclk_pstate.standard = dclk_table->min;
-	pstate_table->fclk_pstate.standard = fclk_table->min;
+		pstate_table->gfxclk_pstate.standard =
+			SMU_DPM_TABLE_MAX(gfx_table);
+	pstate_table->uclk_pstate.standard = SMU_DPM_TABLE_MAX(mem_table);
+	pstate_table->socclk_pstate.standard = SMU_DPM_TABLE_MIN(soc_table);
+	pstate_table->vclk_pstate.standard = SMU_DPM_TABLE_MIN(vclk_table);
+	pstate_table->dclk_pstate.standard = SMU_DPM_TABLE_MIN(dclk_table);
+	pstate_table->fclk_pstate.standard = SMU_DPM_TABLE_MIN(fclk_table);
 
 	return 0;
 }
@@ -3121,8 +3048,7 @@ static int smu_v13_0_0_update_pcie_parameters(struct smu_context *smu,
 				     uint8_t pcie_width_cap)
 {
 	struct smu_13_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
-	struct smu_13_0_pcie_table *pcie_table =
-				&dpm_context->dpm_tables.pcie_table;
+	struct smu_pcie_table *pcie_table = &dpm_context->dpm_tables.pcie_table;
 	int num_of_levels;
 	uint32_t smu_pcie_arg;
 	uint32_t link_level;
@@ -3132,7 +3058,7 @@ static int smu_v13_0_0_update_pcie_parameters(struct smu_context *smu,
 	int ret = 0;
 	int i;
 
-	pcie_table->num_of_link_levels = 0;
+	pcie_table->lclk_levels = 0;
 
 	for (link_level = 0; link_level < NUM_LINK_LEVELS; link_level++) {
 		if (!skutable->PcieGenSpeed[link_level] &&
@@ -3140,16 +3066,16 @@ static int smu_v13_0_0_update_pcie_parameters(struct smu_context *smu,
 		    !skutable->LclkFreq[link_level])
 			continue;
 
-		pcie_table->pcie_gen[pcie_table->num_of_link_levels] =
-					skutable->PcieGenSpeed[link_level];
-		pcie_table->pcie_lane[pcie_table->num_of_link_levels] =
-					skutable->PcieLaneCount[link_level];
-		pcie_table->clk_freq[pcie_table->num_of_link_levels] =
-					skutable->LclkFreq[link_level];
-		pcie_table->num_of_link_levels++;
+		pcie_table->pcie_gen[pcie_table->lclk_levels] =
+			skutable->PcieGenSpeed[link_level];
+		pcie_table->pcie_lane[pcie_table->lclk_levels] =
+			skutable->PcieLaneCount[link_level];
+		pcie_table->lclk_freq[pcie_table->lclk_levels] =
+			skutable->LclkFreq[link_level];
+		pcie_table->lclk_levels++;
 	}
 
-	num_of_levels = pcie_table->num_of_link_levels;
+	num_of_levels = pcie_table->lclk_levels;
 	if (!num_of_levels)
 		return 0;
 
@@ -3227,7 +3153,7 @@ static const struct pptable_funcs smu_v13_0_0_ppt_funcs = {
 	.get_vbios_bootup_values = smu_v13_0_get_vbios_bootup_values,
 	.read_sensor = smu_v13_0_0_read_sensor,
 	.feature_is_enabled = smu_cmn_feature_is_enabled,
-	.print_clk_levels = smu_v13_0_0_print_clk_levels,
+	.emit_clk_levels = smu_v13_0_0_emit_clk_levels,
 	.force_clk_levels = smu_v13_0_0_force_clk_levels,
 	.update_pcie_parameters = smu_v13_0_0_update_pcie_parameters,
 	.get_thermal_temperature_range = smu_v13_0_0_get_thermal_temperature_range,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c
index 9e635f733fbf..07bee1f32f54 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c
@@ -601,6 +601,40 @@ static bool smu_v13_0_12_is_temp_metrics_supported(struct smu_context *smu,
 	return false;
 }
 
+int smu_v13_0_12_get_system_power(struct smu_context *smu,
+				  enum amd_pp_sensors sensor,
+				  uint32_t *value)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct smu_table *tables = smu_table->tables;
+	SystemMetricsTable_t *metrics;
+	struct smu_table *sys_table;
+	int ret;
+
+	if (!smu_v13_0_6_cap_supported(smu, SMU_CAP(SYSTEM_POWER_METRICS)))
+		return -EOPNOTSUPP;
+
+	ret = smu_v13_0_12_get_system_metrics_table(smu);
+	if (ret)
+		return ret;
+
+	sys_table = &tables[SMU_TABLE_PMFW_SYSTEM_METRICS];
+	metrics = (SystemMetricsTable_t *)sys_table->cache.buffer;
+
+	switch (sensor) {
+	case AMDGPU_PP_SENSOR_UBB_POWER:
+		*value = metrics->SystemPower[SYSTEM_POWER_UBB_POWER];
+		break;
+	case AMDGPU_PP_SENSOR_UBB_POWER_LIMIT:
+		*value = metrics->SystemPower[SYSTEM_POWER_UBB_POWER_THRESHOLD];
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
 int smu_v13_0_12_get_npm_data(struct smu_context *smu,
 			      enum amd_pp_sensors sensor,
 			      uint32_t *value)
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c
index 6908f9930f16..b0fa726ea25d 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_4_ppt.c
@@ -494,16 +494,14 @@ static int smu_v13_0_4_get_dpm_level_count(struct smu_context *smu,
 	return 0;
 }
 
-static int smu_v13_0_4_print_clk_levels(struct smu_context *smu,
-					enum smu_clk_type clk_type, char *buf)
+static int smu_v13_0_4_emit_clk_levels(struct smu_context *smu,
+				       enum smu_clk_type clk_type, char *buf,
+				       int *offset)
 {
-	int i, idx, size = 0, ret = 0, start_offset = 0;
+	int i, idx, size = *offset, ret = 0, start_offset = *offset;
 	uint32_t cur_value = 0, value = 0, count = 0;
 	uint32_t min, max;
 
-	smu_cmn_get_sysfs_buf(&buf, &size);
-	start_offset = size;
-
 	switch (clk_type) {
 	case SMU_OD_SCLK:
 		size += sysfs_emit_at(buf, size, "%s:\n", "OD_SCLK");
@@ -525,17 +523,17 @@ static int smu_v13_0_4_print_clk_levels(struct smu_context *smu,
 	case SMU_FCLK:
 		ret = smu_v13_0_4_get_current_clk_freq(smu, clk_type, &cur_value);
 		if (ret)
-			break;
+			return ret;
 
 		ret = smu_v13_0_4_get_dpm_level_count(smu, clk_type, &count);
 		if (ret)
-			break;
+			return ret;
 
 		for (i = 0; i < count; i++) {
 			idx = (clk_type == SMU_FCLK || clk_type == SMU_MCLK) ? (count - i - 1) : i;
 			ret = smu_v13_0_4_get_dpm_freq_by_index(smu, clk_type, idx, &value);
 			if (ret)
-				break;
+				return ret;
 
 			size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, value,
 					      cur_value == value ? "*" : "");
@@ -545,7 +543,7 @@ static int smu_v13_0_4_print_clk_levels(struct smu_context *smu,
 	case SMU_SCLK:
 		ret = smu_v13_0_4_get_current_clk_freq(smu, clk_type, &cur_value);
 		if (ret)
-			break;
+			return ret;
 		min = (smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq;
 		max = (smu->gfx_actual_soft_max_freq > 0) ? smu->gfx_actual_soft_max_freq : smu->gfx_default_soft_max_freq;
 		if (cur_value  == max)
@@ -566,7 +564,9 @@ static int smu_v13_0_4_print_clk_levels(struct smu_context *smu,
 		break;
 	}
 
-	return size - start_offset;
+	*offset += size - start_offset;
+
+	return 0;
 }
 
 static int smu_v13_0_4_read_sensor(struct smu_context *smu,
@@ -1114,7 +1114,7 @@ static const struct pptable_funcs smu_v13_0_4_ppt_funcs = {
 	.mode2_reset = smu_v13_0_4_mode2_reset,
 	.get_dpm_ultimate_freq = smu_v13_0_4_get_dpm_ultimate_freq,
 	.od_edit_dpm_table = smu_v13_0_od_edit_dpm_table,
-	.print_clk_levels = smu_v13_0_4_print_clk_levels,
+	.emit_clk_levels = smu_v13_0_4_emit_clk_levels,
 	.force_clk_levels = smu_v13_0_4_force_clk_levels,
 	.set_performance_level = smu_v13_0_4_set_performance_level,
 	.set_fine_grain_gfx_freq_parameters = smu_v13_0_4_set_fine_grain_gfx_freq_parameters,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_5_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_5_ppt.c
index 4576bf008b22..796b2eb23174 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_5_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_5_ppt.c
@@ -858,16 +858,14 @@ static int smu_v13_0_5_set_soft_freq_limited_range(struct smu_context *smu,
 	return ret;
 }
 
-static int smu_v13_0_5_print_clk_levels(struct smu_context *smu,
-				enum smu_clk_type clk_type, char *buf)
+static int smu_v13_0_5_emit_clk_levels(struct smu_context *smu,
+				       enum smu_clk_type clk_type, char *buf,
+				       int *offset)
 {
-	int i, idx, size = 0, ret = 0, start_offset = 0;
+	int i, idx, size = *offset, ret = 0, start_offset = *offset;
 	uint32_t cur_value = 0, value = 0, count = 0;
 	uint32_t min = 0, max = 0;
 
-	smu_cmn_get_sysfs_buf(&buf, &size);
-	start_offset = size;
-
 	switch (clk_type) {
 	case SMU_OD_SCLK:
 		size += sysfs_emit_at(buf, size, "%s:\n", "OD_SCLK");
@@ -887,17 +885,17 @@ static int smu_v13_0_5_print_clk_levels(struct smu_context *smu,
 	case SMU_MCLK:
 		ret = smu_v13_0_5_get_current_clk_freq(smu, clk_type, &cur_value);
 		if (ret)
-			goto print_clk_out;
+			return ret;
 
 		ret = smu_v13_0_5_get_dpm_level_count(smu, clk_type, &count);
 		if (ret)
-			goto print_clk_out;
+			return ret;
 
 		for (i = 0; i < count; i++) {
 			idx = (clk_type == SMU_MCLK) ? (count - i - 1) : i;
 			ret = smu_v13_0_5_get_dpm_freq_by_index(smu, clk_type, idx, &value);
 			if (ret)
-				goto print_clk_out;
+				return ret;
 
 			size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, value,
 					cur_value == value ? "*" : "");
@@ -907,7 +905,7 @@ static int smu_v13_0_5_print_clk_levels(struct smu_context *smu,
 	case SMU_SCLK:
 		ret = smu_v13_0_5_get_current_clk_freq(smu, clk_type, &cur_value);
 		if (ret)
-			goto print_clk_out;
+			return ret;
 		min = (smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq;
 		max = (smu->gfx_actual_soft_max_freq > 0) ? smu->gfx_actual_soft_max_freq : smu->gfx_default_soft_max_freq;
 		if (cur_value  == max)
@@ -928,8 +926,9 @@ static int smu_v13_0_5_print_clk_levels(struct smu_context *smu,
 		break;
 	}
 
-print_clk_out:
-	return size - start_offset;
+	*offset += size - start_offset;
+
+	return 0;
 }
 
 
@@ -1118,7 +1117,7 @@ static const struct pptable_funcs smu_v13_0_5_ppt_funcs = {
 	.mode2_reset = smu_v13_0_5_mode2_reset,
 	.get_dpm_ultimate_freq = smu_v13_0_5_get_dpm_ultimate_freq,
 	.od_edit_dpm_table = smu_v13_0_5_od_edit_dpm_table,
-	.print_clk_levels = smu_v13_0_5_print_clk_levels,
+	.emit_clk_levels = smu_v13_0_5_emit_clk_levels,
 	.force_clk_levels = smu_v13_0_5_force_clk_levels,
 	.set_performance_level = smu_v13_0_5_set_performance_level,
 	.set_fine_grain_gfx_freq_parameters = smu_v13_0_5_set_fine_grain_gfx_freq_parameters,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
index 44e1cd821eec..57ce7a38bba8 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
@@ -83,9 +83,6 @@ MODULE_FIRMWARE("amdgpu/smu_13_0_14.bin");
 	 FEATURE_MASK(FEATURE_DPM_LCLK) | FEATURE_MASK(FEATURE_DPM_XGMI) |     \
 	 FEATURE_MASK(FEATURE_DPM_VCN))
 
-/* possible frequency drift (1Mhz) */
-#define EPSILON 1
-
 #define smnPCIE_ESM_CTRL 0x93D0
 #define smnPCIE_LC_LINK_WIDTH_CNTL 0x1a340288
 #define PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK 0x00000070L
@@ -95,9 +92,7 @@ MODULE_FIRMWARE("amdgpu/smu_13_0_14.bin");
 #define smnPCIE_LC_SPEED_CNTL                   0x1a340290
 #define PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK 0xE0
 #define PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT 0x5
-#define LINK_SPEED_MAX				4
-#define SMU_13_0_6_DSCLK_THRESHOLD 140
-
+#define LINK_SPEED_MAX 4
 #define MCA_BANK_IPID(_ip, _hwid, _type) \
 	[AMDGPU_MCA_IP_##_ip] = { .hwid = _hwid, .mcatype = _type, }
 
@@ -251,7 +246,7 @@ static const uint8_t smu_v13_0_6_throttler_map[] = {
 struct smu_v13_0_6_dpm_map {
 	enum smu_clk_type clk_type;
 	uint32_t feature_num;
-	struct smu_13_0_dpm_table *dpm_table;
+	struct smu_dpm_table *dpm_table;
 	uint32_t *freq_table;
 };
 
@@ -356,8 +351,11 @@ static void smu_v13_0_12_init_caps(struct smu_context *smu)
 	if (fw_ver > 0x04560900)
 		smu_v13_0_6_cap_set(smu, SMU_CAP(VCN_RESET));
 
-	if (fw_ver >= 0x04560D00)
+	if (fw_ver >= 0x04560D00) {
 		smu_v13_0_6_cap_set(smu, SMU_CAP(FAST_PPT));
+		if (smu->adev->gmc.xgmi.physical_node_id == 0)
+			smu_v13_0_6_cap_set(smu, SMU_CAP(SYSTEM_POWER_METRICS));
+	}
 
 	if (fw_ver >= 0x04560700) {
 		if (fw_ver >= 0x04560900) {
@@ -984,7 +982,7 @@ static int smu_v13_0_6_get_dpm_ultimate_freq(struct smu_context *smu,
 	struct smu_table_context *smu_table = &smu->smu_table;
 	struct PPTable_t *pptable =
 		(struct PPTable_t *)smu_table->driver_pptable;
-	struct smu_13_0_dpm_table *dpm_table;
+	struct smu_dpm_table *dpm_table;
 	uint32_t min_clk, max_clk, param;
 	int ret = 0, clk_id = 0;
 
@@ -1015,9 +1013,8 @@ static int smu_v13_0_6_get_dpm_ultimate_freq(struct smu_context *smu,
 			return -EINVAL;
 		}
 
-		min_clk = dpm_table->min;
-		max_clk = dpm_table->max;
-
+		min_clk = SMU_DPM_TABLE_MIN(dpm_table);
+		max_clk = SMU_DPM_TABLE_MAX(dpm_table);
 		if (min)
 			*min = min_clk;
 		if (max)
@@ -1087,7 +1084,7 @@ static int smu_v13_0_6_set_default_dpm_table(struct smu_context *smu)
 {
 	struct smu_13_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
 	struct smu_table_context *smu_table = &smu->smu_table;
-	struct smu_13_0_dpm_table *dpm_table = NULL;
+	struct smu_dpm_table *dpm_table = NULL;
 	struct PPTable_t *pptable =
 		(struct PPTable_t *)smu_table->driver_pptable;
 	uint32_t gfxclkmin, gfxclkmax, levels;
@@ -1123,6 +1120,7 @@ static int smu_v13_0_6_set_default_dpm_table(struct smu_context *smu)
 	smu_v13_0_6_pm_policy_init(smu);
 	/* gfxclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.gfx_table;
+	dpm_table->clk_type = SMU_GFXCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT)) {
 		/* In the case of gfxclk, only fine-grained dpm is honored.
 		 * Get min/max values from FW.
@@ -1131,20 +1129,15 @@ static int smu_v13_0_6_set_default_dpm_table(struct smu_context *smu)
 							&gfxclkmin, &gfxclkmax);
 		if (ret)
 			return ret;
-
 		dpm_table->count = 2;
 		dpm_table->dpm_levels[0].value = gfxclkmin;
 		dpm_table->dpm_levels[0].enabled = true;
 		dpm_table->dpm_levels[1].value = gfxclkmax;
 		dpm_table->dpm_levels[1].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[1].value;
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = pptable->MinGfxclkFrequency;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	for (j = 0; j < ARRAY_SIZE(dpm_map); j++) {
@@ -1157,15 +1150,12 @@ static int smu_v13_0_6_set_default_dpm_table(struct smu_context *smu)
 				return ret;
 		}
 		dpm_table->count = levels;
+		dpm_table->clk_type = dpm_map[j].clk_type;
 		for (i = 0; i < dpm_table->count; ++i) {
 			dpm_table->dpm_levels[i].value =
 				dpm_map[j].freq_table[i];
 			dpm_table->dpm_levels[i].enabled = true;
-
 		}
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[levels - 1].value;
-
 	}
 
 	return 0;
@@ -1202,28 +1192,25 @@ static int smu_v13_0_6_check_fw_status(struct smu_context *smu)
 static int smu_v13_0_6_populate_umd_state_clk(struct smu_context *smu)
 {
 	struct smu_13_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
-	struct smu_13_0_dpm_table *gfx_table =
-		&dpm_context->dpm_tables.gfx_table;
-	struct smu_13_0_dpm_table *mem_table =
-		&dpm_context->dpm_tables.uclk_table;
-	struct smu_13_0_dpm_table *soc_table =
-		&dpm_context->dpm_tables.soc_table;
+	struct smu_dpm_table *gfx_table = &dpm_context->dpm_tables.gfx_table;
+	struct smu_dpm_table *mem_table = &dpm_context->dpm_tables.uclk_table;
+	struct smu_dpm_table *soc_table = &dpm_context->dpm_tables.soc_table;
 	struct smu_umd_pstate_table *pstate_table = &smu->pstate_table;
 
-	pstate_table->gfxclk_pstate.min = gfx_table->min;
-	pstate_table->gfxclk_pstate.peak = gfx_table->max;
-	pstate_table->gfxclk_pstate.curr.min = gfx_table->min;
-	pstate_table->gfxclk_pstate.curr.max = gfx_table->max;
+	pstate_table->gfxclk_pstate.min = SMU_DPM_TABLE_MIN(gfx_table);
+	pstate_table->gfxclk_pstate.peak = SMU_DPM_TABLE_MAX(gfx_table);
+	pstate_table->gfxclk_pstate.curr.min = SMU_DPM_TABLE_MIN(gfx_table);
+	pstate_table->gfxclk_pstate.curr.max = SMU_DPM_TABLE_MAX(gfx_table);
 
-	pstate_table->uclk_pstate.min = mem_table->min;
-	pstate_table->uclk_pstate.peak = mem_table->max;
-	pstate_table->uclk_pstate.curr.min = mem_table->min;
-	pstate_table->uclk_pstate.curr.max = mem_table->max;
+	pstate_table->uclk_pstate.min = SMU_DPM_TABLE_MIN(mem_table);
+	pstate_table->uclk_pstate.peak = SMU_DPM_TABLE_MAX(mem_table);
+	pstate_table->uclk_pstate.curr.min = SMU_DPM_TABLE_MIN(mem_table);
+	pstate_table->uclk_pstate.curr.max = SMU_DPM_TABLE_MAX(mem_table);
 
-	pstate_table->socclk_pstate.min = soc_table->min;
-	pstate_table->socclk_pstate.peak = soc_table->max;
-	pstate_table->socclk_pstate.curr.min = soc_table->min;
-	pstate_table->socclk_pstate.curr.max = soc_table->max;
+	pstate_table->socclk_pstate.min = SMU_DPM_TABLE_MIN(soc_table);
+	pstate_table->socclk_pstate.peak = SMU_DPM_TABLE_MAX(soc_table);
+	pstate_table->socclk_pstate.curr.min = SMU_DPM_TABLE_MIN(soc_table);
+	pstate_table->socclk_pstate.curr.max = SMU_DPM_TABLE_MAX(soc_table);
 
 	if (gfx_table->count > SMU_13_0_6_UMD_PSTATE_GFXCLK_LEVEL &&
 	    mem_table->count > SMU_13_0_6_UMD_PSTATE_MCLK_LEVEL &&
@@ -1246,31 +1233,6 @@ static int smu_v13_0_6_populate_umd_state_clk(struct smu_context *smu)
 	return 0;
 }
 
-static int smu_v13_0_6_get_clk_table(struct smu_context *smu,
-				     struct pp_clock_levels_with_latency *clocks,
-				     struct smu_13_0_dpm_table *dpm_table)
-{
-	int i, count;
-
-	count = (dpm_table->count > MAX_NUM_CLOCKS) ? MAX_NUM_CLOCKS :
-						      dpm_table->count;
-	clocks->num_levels = count;
-
-	for (i = 0; i < count; i++) {
-		clocks->data[i].clocks_in_khz =
-			dpm_table->dpm_levels[i].value * 1000;
-		clocks->data[i].latency_in_us = 0;
-	}
-
-	return 0;
-}
-
-static int smu_v13_0_6_freqs_in_same_level(int32_t frequency1,
-					   int32_t frequency2)
-{
-	return (abs(frequency1 - frequency2) <= EPSILON);
-}
-
 static uint32_t smu_v13_0_6_get_throttler_status(struct smu_context *smu)
 {
 	struct smu_power_context *smu_power = &smu->smu_power;
@@ -1375,9 +1337,11 @@ static int smu_v13_0_6_get_current_clk_freq_by_table(struct smu_context *smu,
 
 	switch (clk_type) {
 	case SMU_GFXCLK:
+	case SMU_SCLK:
 		member_type = METRICS_CURR_GFXCLK;
 		break;
 	case SMU_UCLK:
+	case SMU_MCLK:
 		member_type = METRICS_CURR_UCLK;
 		break;
 	case SMU_SOCCLK:
@@ -1399,74 +1363,20 @@ static int smu_v13_0_6_get_current_clk_freq_by_table(struct smu_context *smu,
 	return smu_v13_0_6_get_smu_metrics_data(smu, member_type, value);
 }
 
-static int smu_v13_0_6_print_clks(struct smu_context *smu, char *buf, int size,
-				  struct smu_13_0_dpm_table *single_dpm_table,
-				  uint32_t curr_clk, const char *clk_name)
+static int smu_v13_0_6_emit_clk_levels(struct smu_context *smu,
+				       enum smu_clk_type type, char *buf,
+				       int *offset)
 {
-	struct pp_clock_levels_with_latency clocks;
-	int i, ret, level = -1;
-	uint32_t clk1, clk2;
-
-	ret = smu_v13_0_6_get_clk_table(smu, &clocks, single_dpm_table);
-	if (ret) {
-		dev_err(smu->adev->dev, "Attempt to get %s clk levels failed!",
-			clk_name);
-		return ret;
-	}
-
-	if (!clocks.num_levels)
-		return -EINVAL;
-
-	if (curr_clk < SMU_13_0_6_DSCLK_THRESHOLD) {
-		size += sysfs_emit_at(buf, size, "S: %uMhz *\n", curr_clk);
-		for (i = 0; i < clocks.num_levels; i++)
-			size += sysfs_emit_at(buf, size, "%d: %uMhz\n", i,
-					      clocks.data[i].clocks_in_khz /
-						      1000);
-
-	} else {
-		if ((clocks.num_levels == 1) ||
-		    (curr_clk < (clocks.data[0].clocks_in_khz / 1000)))
-			level = 0;
-		for (i = 0; i < clocks.num_levels; i++) {
-			clk1 = clocks.data[i].clocks_in_khz / 1000;
-
-			if (i < (clocks.num_levels - 1))
-				clk2 = clocks.data[i + 1].clocks_in_khz / 1000;
-
-			if (curr_clk == clk1) {
-				level = i;
-			} else if (curr_clk >= clk1 && curr_clk < clk2) {
-				level = (curr_clk - clk1) <= (clk2 - curr_clk) ?
-						i :
-						i + 1;
-			}
-
-			size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i,
-					      clk1, (level == i) ? "*" : "");
-		}
-	}
-
-	return size;
-}
-
-static int smu_v13_0_6_print_clk_levels(struct smu_context *smu,
-					enum smu_clk_type type, char *buf)
-{
-	int now, size = 0, start_offset = 0;
+	int now, size = *offset, start_offset = *offset;
 	int ret = 0;
 	struct smu_umd_pstate_table *pstate_table = &smu->pstate_table;
-	struct smu_13_0_dpm_table *single_dpm_table;
+	struct smu_dpm_table *single_dpm_table = NULL;
 	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
 	struct smu_13_0_dpm_context *dpm_context = NULL;
-	uint32_t min_clk, max_clk;
-
-	smu_cmn_get_sysfs_buf(&buf, &size);
-	start_offset = size;
 
 	if (amdgpu_ras_intr_triggered()) {
-		size += sysfs_emit_at(buf, size, "unavailable\n");
-		return size - start_offset;
+		sysfs_emit_at(buf, size, "unavailable\n");
+		return -EBUSY;
 	}
 
 	dpm_context = smu_dpm->dpm_context;
@@ -1478,46 +1388,6 @@ static int smu_v13_0_6_print_clk_levels(struct smu_context *smu,
 				      pstate_table->gfxclk_pstate.curr.min,
 				      pstate_table->gfxclk_pstate.curr.max);
 		break;
-	case SMU_SCLK:
-		ret = smu_v13_0_6_get_current_clk_freq_by_table(smu, SMU_GFXCLK,
-								&now);
-		if (ret) {
-			dev_err(smu->adev->dev,
-				"Attempt to get current gfx clk Failed!");
-			return ret;
-		}
-
-		single_dpm_table = &(dpm_context->dpm_tables.gfx_table);
-		min_clk = single_dpm_table->min;
-		max_clk = single_dpm_table->max;
-
-		if (now < SMU_13_0_6_DSCLK_THRESHOLD) {
-			size += sysfs_emit_at(buf, size, "S: %uMhz *\n",
-					      now);
-			size += sysfs_emit_at(buf, size, "0: %uMhz\n",
-					      min_clk);
-			size += sysfs_emit_at(buf, size, "1: %uMhz\n",
-					      max_clk);
-
-		} else if (!smu_v13_0_6_freqs_in_same_level(now, min_clk) &&
-		    !smu_v13_0_6_freqs_in_same_level(now, max_clk)) {
-			size += sysfs_emit_at(buf, size, "0: %uMhz\n",
-					      min_clk);
-			size += sysfs_emit_at(buf, size, "1: %uMhz *\n",
-					      now);
-			size += sysfs_emit_at(buf, size, "2: %uMhz\n",
-					      max_clk);
-		} else {
-			size += sysfs_emit_at(buf, size, "0: %uMhz %s\n",
-					      min_clk,
-					      smu_v13_0_6_freqs_in_same_level(now, min_clk) ? "*" : "");
-			size += sysfs_emit_at(buf, size, "1: %uMhz %s\n",
-					      max_clk,
-					      smu_v13_0_6_freqs_in_same_level(now, max_clk) ? "*" : "");
-		}
-
-		break;
-
 	case SMU_OD_MCLK:
 		if (!smu_v13_0_6_cap_supported(smu, SMU_CAP(SET_UCLK_MAX)))
 			return 0;
@@ -1527,101 +1397,46 @@ static int smu_v13_0_6_print_clk_levels(struct smu_context *smu,
 				      pstate_table->uclk_pstate.curr.min,
 				      pstate_table->uclk_pstate.curr.max);
 		break;
-	case SMU_MCLK:
-		ret = smu_v13_0_6_get_current_clk_freq_by_table(smu, SMU_UCLK,
-								&now);
-		if (ret) {
-			dev_err(smu->adev->dev,
-				"Attempt to get current mclk Failed!");
-			return ret;
-		}
 
+	case SMU_SCLK:
+	case SMU_GFXCLK:
+		single_dpm_table = &(dpm_context->dpm_tables.gfx_table);
+		break;
+	case SMU_MCLK:
+	case SMU_UCLK:
 		single_dpm_table = &(dpm_context->dpm_tables.uclk_table);
-
-		ret = smu_v13_0_6_print_clks(smu, buf, size, single_dpm_table,
-					     now, "mclk");
-		if (ret < 0)
-			return ret;
-
-		size += ret;
 		break;
 	case SMU_SOCCLK:
-		ret = smu_v13_0_6_get_current_clk_freq_by_table(smu, SMU_SOCCLK,
-								&now);
-		if (ret) {
-			dev_err(smu->adev->dev,
-				"Attempt to get current socclk Failed!");
-			return ret;
-		}
-
 		single_dpm_table = &(dpm_context->dpm_tables.soc_table);
-
-		ret = smu_v13_0_6_print_clks(smu, buf, size, single_dpm_table,
-					     now, "socclk");
-		if (ret < 0)
-			return ret;
-
-		size += ret;
 		break;
 	case SMU_FCLK:
-		ret = smu_v13_0_6_get_current_clk_freq_by_table(smu, SMU_FCLK,
-								&now);
-		if (ret) {
-			dev_err(smu->adev->dev,
-				"Attempt to get current fclk Failed!");
-			return ret;
-		}
-
 		single_dpm_table = &(dpm_context->dpm_tables.fclk_table);
-
-		ret = smu_v13_0_6_print_clks(smu, buf, size, single_dpm_table,
-					     now, "fclk");
-		if (ret < 0)
-			return ret;
-
-		size += ret;
 		break;
 	case SMU_VCLK:
-		ret = smu_v13_0_6_get_current_clk_freq_by_table(smu, SMU_VCLK,
-								&now);
-		if (ret) {
-			dev_err(smu->adev->dev,
-				"Attempt to get current vclk Failed!");
-			return ret;
-		}
-
 		single_dpm_table = &(dpm_context->dpm_tables.vclk_table);
-
-		ret = smu_v13_0_6_print_clks(smu, buf, size, single_dpm_table,
-					     now, "vclk");
-		if (ret < 0)
-			return ret;
-
-		size += ret;
 		break;
 	case SMU_DCLK:
-		ret = smu_v13_0_6_get_current_clk_freq_by_table(smu, SMU_DCLK,
-							       &now);
-		if (ret) {
-			dev_err(smu->adev->dev,
-				"Attempt to get current dclk Failed!");
-			return ret;
-		}
-
 		single_dpm_table = &(dpm_context->dpm_tables.dclk_table);
-
-		ret = smu_v13_0_6_print_clks(smu, buf, size, single_dpm_table,
-					     now, "dclk");
-		if (ret < 0)
-			return ret;
-
-		size += ret;
 		break;
 	default:
 		break;
 	}
 
-	return size - start_offset;
+	if (single_dpm_table) {
+		ret = smu_v13_0_6_get_current_clk_freq_by_table(smu, type,
+								&now);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Attempt to get current clk Failed!");
+			return ret;
+		}
+		return smu_cmn_print_dpm_clk_levels(smu, single_dpm_table, now,
+						    buf, offset);
+	}
+
+	*offset += size - start_offset;
+
+	return 0;
 }
 
 static int smu_v13_0_6_upload_dpm_level(struct smu_context *smu, bool max,
@@ -1687,7 +1502,7 @@ static int smu_v13_0_6_force_clk_levels(struct smu_context *smu,
 					enum smu_clk_type type, uint32_t mask)
 {
 	struct smu_13_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
-	struct smu_13_0_dpm_table *single_dpm_table = NULL;
+	struct smu_dpm_table *single_dpm_table = NULL;
 	uint32_t soft_min_level, soft_max_level;
 	int ret = 0;
 
@@ -1860,6 +1675,13 @@ static int smu_v13_0_6_read_sensor(struct smu_context *smu,
 			return ret;
 		*size = 4;
 		break;
+	case AMDGPU_PP_SENSOR_UBB_POWER:
+	case AMDGPU_PP_SENSOR_UBB_POWER_LIMIT:
+		ret = smu_v13_0_12_get_system_power(smu, sensor, (uint32_t *)data);
+		if (ret)
+			return ret;
+		*size = 4;
+		break;
 	case AMDGPU_PP_SENSOR_GPU_AVG_POWER:
 	default:
 		ret = -EOPNOTSUPP;
@@ -2146,10 +1968,8 @@ static int smu_v13_0_6_set_performance_level(struct smu_context *smu,
 {
 	struct smu_dpm_context *smu_dpm = &(smu->smu_dpm);
 	struct smu_13_0_dpm_context *dpm_context = smu_dpm->dpm_context;
-	struct smu_13_0_dpm_table *gfx_table =
-		&dpm_context->dpm_tables.gfx_table;
-	struct smu_13_0_dpm_table *uclk_table =
-		&dpm_context->dpm_tables.uclk_table;
+	struct smu_dpm_table *gfx_table = &dpm_context->dpm_tables.gfx_table;
+	struct smu_dpm_table *uclk_table = &dpm_context->dpm_tables.uclk_table;
 	struct smu_umd_pstate_table *pstate_table = &smu->pstate_table;
 	int ret;
 
@@ -2157,7 +1977,8 @@ static int smu_v13_0_6_set_performance_level(struct smu_context *smu,
 	if ((smu_dpm->dpm_level == AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM) &&
 	    (level != AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM)) {
 		smu_cmn_send_smc_msg(smu, SMU_MSG_DisableDeterminism, NULL);
-		pstate_table->gfxclk_pstate.curr.max = gfx_table->max;
+		pstate_table->gfxclk_pstate.curr.max =
+			SMU_DPM_TABLE_MAX(gfx_table);
 	}
 
 	switch (level) {
@@ -2165,24 +1986,32 @@ static int smu_v13_0_6_set_performance_level(struct smu_context *smu,
 		return 0;
 
 	case AMD_DPM_FORCED_LEVEL_AUTO:
-		if ((gfx_table->min != pstate_table->gfxclk_pstate.curr.min) ||
-		    (gfx_table->max != pstate_table->gfxclk_pstate.curr.max)) {
+		if ((SMU_DPM_TABLE_MIN(gfx_table) !=
+		     pstate_table->gfxclk_pstate.curr.min) ||
+		    (SMU_DPM_TABLE_MAX(gfx_table) !=
+		     pstate_table->gfxclk_pstate.curr.max)) {
 			ret = smu_v13_0_6_set_gfx_soft_freq_limited_range(
-				smu, gfx_table->min, gfx_table->max);
+				smu, SMU_DPM_TABLE_MIN(gfx_table),
+				SMU_DPM_TABLE_MAX(gfx_table));
 			if (ret)
 				return ret;
 
-			pstate_table->gfxclk_pstate.curr.min = gfx_table->min;
-			pstate_table->gfxclk_pstate.curr.max = gfx_table->max;
+			pstate_table->gfxclk_pstate.curr.min =
+				SMU_DPM_TABLE_MIN(gfx_table);
+			pstate_table->gfxclk_pstate.curr.max =
+				SMU_DPM_TABLE_MAX(gfx_table);
 		}
 
-		if (uclk_table->max != pstate_table->uclk_pstate.curr.max) {
+		if (SMU_DPM_TABLE_MAX(uclk_table) !=
+		    pstate_table->uclk_pstate.curr.max) {
 			/* Min UCLK is not expected to be changed */
 			ret = smu_v13_0_set_soft_freq_limited_range(
-				smu, SMU_UCLK, 0, uclk_table->max, false);
+				smu, SMU_UCLK, 0, SMU_DPM_TABLE_MAX(uclk_table),
+				false);
 			if (ret)
 				return ret;
-			pstate_table->uclk_pstate.curr.max = uclk_table->max;
+			pstate_table->uclk_pstate.curr.max =
+				SMU_DPM_TABLE_MAX(uclk_table);
 		}
 		smu_v13_0_reset_custom_level(smu);
 
@@ -2255,8 +2084,9 @@ static int smu_v13_0_6_set_soft_freq_limited_range(struct smu_context *smu,
 	}
 
 	if (smu_dpm->dpm_level == AMD_DPM_FORCED_LEVEL_PERF_DETERMINISM) {
-		if (!max || (max < dpm_context->dpm_tables.gfx_table.min) ||
-		    (max > dpm_context->dpm_tables.gfx_table.max)) {
+		min_clk = SMU_DPM_TABLE_MIN(&dpm_context->dpm_tables.gfx_table);
+		max_clk = SMU_DPM_TABLE_MAX(&dpm_context->dpm_tables.gfx_table);
+		if (!max || (max < min_clk) || (max > max_clk)) {
 			dev_warn(
 				adev->dev,
 				"Invalid max frequency %d MHz specified for determinism\n",
@@ -2265,10 +2095,8 @@ static int smu_v13_0_6_set_soft_freq_limited_range(struct smu_context *smu,
 		}
 
 		/* Restore default min/max clocks and enable determinism */
-		min_clk = dpm_context->dpm_tables.gfx_table.min;
-		max_clk = dpm_context->dpm_tables.gfx_table.max;
 		ret = smu_v13_0_6_set_gfx_soft_freq_limited_range(smu, min_clk,
-								 max_clk);
+								  max_clk);
 		if (!ret) {
 			usleep_range(500, 1000);
 			ret = smu_cmn_send_smc_msg_with_param(
@@ -2310,14 +2138,14 @@ static int smu_v13_0_6_usr_edit_dpm_table(struct smu_context *smu,
 				"Input parameter number not correct\n");
 			return -EINVAL;
 		}
-
+		min_clk = SMU_DPM_TABLE_MIN(&dpm_context->dpm_tables.gfx_table);
+		max_clk = SMU_DPM_TABLE_MAX(&dpm_context->dpm_tables.gfx_table);
 		if (input[0] == 0) {
-			if (input[1] < dpm_context->dpm_tables.gfx_table.min) {
+			if (input[1] < min_clk) {
 				dev_warn(
 					smu->adev->dev,
 					"Minimum GFX clk (%ld) MHz specified is less than the minimum allowed (%d) MHz\n",
-					input[1],
-					dpm_context->dpm_tables.gfx_table.min);
+					input[1], min_clk);
 				pstate_table->gfxclk_pstate.custom.min =
 					pstate_table->gfxclk_pstate.curr.min;
 				return -EINVAL;
@@ -2325,12 +2153,11 @@ static int smu_v13_0_6_usr_edit_dpm_table(struct smu_context *smu,
 
 			pstate_table->gfxclk_pstate.custom.min = input[1];
 		} else if (input[0] == 1) {
-			if (input[1] > dpm_context->dpm_tables.gfx_table.max) {
+			if (input[1] > max_clk) {
 				dev_warn(
 					smu->adev->dev,
 					"Maximum GFX clk (%ld) MHz specified is greater than the maximum allowed (%d) MHz\n",
-					input[1],
-					dpm_context->dpm_tables.gfx_table.max);
+					input[1], max_clk);
 				pstate_table->gfxclk_pstate.custom.max =
 					pstate_table->gfxclk_pstate.curr.max;
 				return -EINVAL;
@@ -2354,18 +2181,18 @@ static int smu_v13_0_6_usr_edit_dpm_table(struct smu_context *smu,
 				 "UCLK_LIMITS setting not supported!\n");
 			return -EOPNOTSUPP;
 		}
-
+		max_clk =
+			SMU_DPM_TABLE_MAX(&dpm_context->dpm_tables.uclk_table);
 		if (input[0] == 0) {
 			dev_info(smu->adev->dev,
 				 "Setting min UCLK level is not supported");
 			return -EINVAL;
 		} else if (input[0] == 1) {
-			if (input[1] > dpm_context->dpm_tables.uclk_table.max) {
+			if (input[1] > max_clk) {
 				dev_warn(
 					smu->adev->dev,
 					"Maximum UCLK (%ld) MHz specified is greater than the maximum allowed (%d) MHz\n",
-					input[1],
-					dpm_context->dpm_tables.uclk_table.max);
+					input[1], max_clk);
 				pstate_table->uclk_pstate.custom.max =
 					pstate_table->uclk_pstate.curr.max;
 				return -EINVAL;
@@ -2382,8 +2209,10 @@ static int smu_v13_0_6_usr_edit_dpm_table(struct smu_context *smu,
 			return -EINVAL;
 		} else {
 			/* Use the default frequencies for manual and determinism mode */
-			min_clk = dpm_context->dpm_tables.gfx_table.min;
-			max_clk = dpm_context->dpm_tables.gfx_table.max;
+			min_clk = SMU_DPM_TABLE_MIN(
+				&dpm_context->dpm_tables.gfx_table);
+			max_clk = SMU_DPM_TABLE_MAX(
+				&dpm_context->dpm_tables.gfx_table);
 
 			ret = smu_v13_0_6_set_soft_freq_limited_range(
 				smu, SMU_GFXCLK, min_clk, max_clk, false);
@@ -2391,8 +2220,10 @@ static int smu_v13_0_6_usr_edit_dpm_table(struct smu_context *smu,
 			if (ret)
 				return ret;
 
-			min_clk = dpm_context->dpm_tables.uclk_table.min;
-			max_clk = dpm_context->dpm_tables.uclk_table.max;
+			min_clk = SMU_DPM_TABLE_MIN(
+				&dpm_context->dpm_tables.uclk_table);
+			max_clk = SMU_DPM_TABLE_MAX(
+				&dpm_context->dpm_tables.uclk_table);
 			ret = smu_v13_0_6_set_soft_freq_limited_range(
 				smu, SMU_UCLK, min_clk, max_clk, false);
 			if (ret)
@@ -4010,7 +3841,7 @@ static const struct pptable_funcs smu_v13_0_6_ppt_funcs = {
 	/* dpm/clk tables */
 	.set_default_dpm_table = smu_v13_0_6_set_default_dpm_table,
 	.populate_umd_state_clk = smu_v13_0_6_populate_umd_state_clk,
-	.print_clk_levels = smu_v13_0_6_print_clk_levels,
+	.emit_clk_levels = smu_v13_0_6_emit_clk_levels,
 	.force_clk_levels = smu_v13_0_6_force_clk_levels,
 	.read_sensor = smu_v13_0_6_read_sensor,
 	.set_performance_level = smu_v13_0_6_set_performance_level,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h
index 6cbdd7c5ded9..0588a5aa952d 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h
@@ -77,6 +77,7 @@ enum smu_v13_0_6_caps {
 	SMU_CAP(NPM_METRICS),
 	SMU_CAP(RAS_EEPROM),
 	SMU_CAP(FAST_PPT),
+	SMU_CAP(SYSTEM_POWER_METRICS),
 	SMU_CAP(ALL),
 };
 
@@ -107,6 +108,9 @@ void smu_v13_0_12_tables_fini(struct smu_context *smu);
 int smu_v13_0_12_get_npm_data(struct smu_context *smu,
 			      enum amd_pp_sensors sensor,
 			      uint32_t *value);
+int smu_v13_0_12_get_system_power(struct smu_context *smu,
+				  enum amd_pp_sensors sensor,
+				  uint32_t *value);
 extern const struct cmn2asic_mapping smu_v13_0_12_feature_mask_map[];
 extern const struct cmn2asic_msg_mapping smu_v13_0_12_message_map[];
 extern const struct smu_temp_funcs smu_v13_0_12_temp_funcs;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
index a3fc35b9011e..1465e0c80175 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
@@ -578,11 +578,12 @@ static int smu_v13_0_7_set_default_dpm_table(struct smu_context *smu)
 	struct smu_13_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
 	PPTable_t *driver_ppt = smu->smu_table.driver_pptable;
 	SkuTable_t *skutable = &driver_ppt->SkuTable;
-	struct smu_13_0_dpm_table *dpm_table;
+	struct smu_dpm_table *dpm_table;
 	int ret = 0;
 
 	/* socclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.soc_table;
+	dpm_table->clk_type = SMU_SOCCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_SOCCLK_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_SOCCLK,
@@ -593,12 +594,11 @@ static int smu_v13_0_7_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.socclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* gfxclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.gfx_table;
+	dpm_table->clk_type = SMU_GFXCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_GFXCLK,
@@ -611,18 +611,16 @@ static int smu_v13_0_7_set_default_dpm_table(struct smu_context *smu)
 			skutable->DriverReportedClocks.GameClockAc)) {
 			dpm_table->dpm_levels[dpm_table->count - 1].value =
 				skutable->DriverReportedClocks.GameClockAc;
-			dpm_table->max = skutable->DriverReportedClocks.GameClockAc;
 		}
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* uclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.uclk_table;
+	dpm_table->clk_type = SMU_UCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_UCLK_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_UCLK,
@@ -633,12 +631,11 @@ static int smu_v13_0_7_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.uclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* fclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.fclk_table;
+	dpm_table->clk_type = SMU_FCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_FCLK_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_FCLK,
@@ -649,12 +646,11 @@ static int smu_v13_0_7_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.fclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* vclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.vclk_table;
+	dpm_table->clk_type = SMU_VCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_VCLK_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_VCLK,
@@ -665,12 +661,11 @@ static int smu_v13_0_7_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.vclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* dclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.dclk_table;
+	dpm_table->clk_type = SMU_DCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCLK_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_DCLK,
@@ -681,12 +676,11 @@ static int smu_v13_0_7_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* dcefclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.dcef_table;
+	dpm_table->clk_type = SMU_DCEFCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCN_BIT)) {
 		ret = smu_v13_0_set_single_dpm_table(smu,
 						     SMU_DCEFCLK,
@@ -697,8 +691,6 @@ static int smu_v13_0_7_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dcefclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	return 0;
@@ -865,7 +857,7 @@ static int smu_v13_0_7_get_dpm_ultimate_freq(struct smu_context *smu,
 {
 	struct smu_13_0_dpm_context *dpm_context =
 		smu->smu_dpm.dpm_context;
-	struct smu_13_0_dpm_table *dpm_table;
+	struct smu_dpm_table *dpm_table;
 
 	switch (clk_type) {
 	case SMU_MCLK:
@@ -902,9 +894,9 @@ static int smu_v13_0_7_get_dpm_ultimate_freq(struct smu_context *smu,
 	}
 
 	if (min)
-		*min = dpm_table->min;
+		*min = SMU_DPM_TABLE_MIN(dpm_table);
 	if (max)
-		*max = dpm_table->max;
+		*max = SMU_DPM_TABLE_MAX(dpm_table);
 
 	return 0;
 }
@@ -1173,27 +1165,24 @@ static int smu_v13_0_7_upload_overdrive_table(struct smu_context *smu,
 	return ret;
 }
 
-static int smu_v13_0_7_print_clk_levels(struct smu_context *smu,
-					enum smu_clk_type clk_type,
-					char *buf)
+static int smu_v13_0_7_emit_clk_levels(struct smu_context *smu,
+				       enum smu_clk_type clk_type, char *buf,
+				       int *offset)
 {
 	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
 	struct smu_13_0_dpm_context *dpm_context = smu_dpm->dpm_context;
 	OverDriveTableExternal_t *od_table =
 		(OverDriveTableExternal_t *)smu->smu_table.overdrive_table;
-	struct smu_13_0_dpm_table *single_dpm_table;
-	struct smu_13_0_pcie_table *pcie_table;
+	int i, curr_freq, size = *offset, start_offset = *offset;
+	struct smu_dpm_table *single_dpm_table = NULL;
+	struct smu_pcie_table *pcie_table;
 	uint32_t gen_speed, lane_width;
-	int i, curr_freq, size = 0, start_offset = 0;
 	int32_t min_value, max_value;
 	int ret = 0;
 
-	smu_cmn_get_sysfs_buf(&buf, &size);
-	start_offset = size;
-
 	if (amdgpu_ras_intr_triggered()) {
-		size += sysfs_emit_at(buf, size, "unavailable\n");
-		return size - start_offset;
+		sysfs_emit_at(buf, size, "unavailable\n");
+		return -EBUSY;
 	}
 
 	switch (clk_type) {
@@ -1220,61 +1209,6 @@ static int smu_v13_0_7_print_clk_levels(struct smu_context *smu,
 	case SMU_DCEFCLK:
 		single_dpm_table = &(dpm_context->dpm_tables.dcef_table);
 		break;
-	default:
-		break;
-	}
-
-	switch (clk_type) {
-	case SMU_SCLK:
-	case SMU_MCLK:
-	case SMU_SOCCLK:
-	case SMU_FCLK:
-	case SMU_VCLK:
-	case SMU_VCLK1:
-	case SMU_DCLK:
-	case SMU_DCLK1:
-	case SMU_DCEFCLK:
-		ret = smu_v13_0_7_get_current_clk_freq_by_table(smu, clk_type, &curr_freq);
-		if (ret) {
-			dev_err(smu->adev->dev, "Failed to get current clock freq!");
-			return ret;
-		}
-
-		if (single_dpm_table->is_fine_grained) {
-			/*
-			 * For fine grained dpms, there are only two dpm levels:
-			 *   - level 0 -> min clock freq
-			 *   - level 1 -> max clock freq
-			 * And the current clock frequency can be any value between them.
-			 * So, if the current clock frequency is not at level 0 or level 1,
-			 * we will fake it as three dpm levels:
-			 *   - level 0 -> min clock freq
-			 *   - level 1 -> current actual clock freq
-			 *   - level 2 -> max clock freq
-			 */
-			if ((single_dpm_table->dpm_levels[0].value != curr_freq) &&
-			     (single_dpm_table->dpm_levels[1].value != curr_freq)) {
-				size += sysfs_emit_at(buf, size, "0: %uMhz\n",
-						single_dpm_table->dpm_levels[0].value);
-				size += sysfs_emit_at(buf, size, "1: %uMhz *\n",
-						curr_freq);
-				size += sysfs_emit_at(buf, size, "2: %uMhz\n",
-						single_dpm_table->dpm_levels[1].value);
-			} else {
-				size += sysfs_emit_at(buf, size, "0: %uMhz %s\n",
-						single_dpm_table->dpm_levels[0].value,
-						single_dpm_table->dpm_levels[0].value == curr_freq ? "*" : "");
-				size += sysfs_emit_at(buf, size, "1: %uMhz %s\n",
-						single_dpm_table->dpm_levels[1].value,
-						single_dpm_table->dpm_levels[1].value == curr_freq ? "*" : "");
-			}
-		} else {
-			for (i = 0; i < single_dpm_table->count; i++)
-				size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
-						i, single_dpm_table->dpm_levels[i].value,
-						single_dpm_table->dpm_levels[i].value == curr_freq ? "*" : "");
-		}
-		break;
 	case SMU_PCIE:
 		ret = smu_v13_0_7_get_smu_metrics_data(smu,
 						       METRICS_PCIE_RATE,
@@ -1289,24 +1223,8 @@ static int smu_v13_0_7_print_clk_levels(struct smu_context *smu,
 			return ret;
 
 		pcie_table = &(dpm_context->dpm_tables.pcie_table);
-		for (i = 0; i < pcie_table->num_of_link_levels; i++)
-			size += sysfs_emit_at(buf, size, "%d: %s %s %dMhz %s\n", i,
-					(pcie_table->pcie_gen[i] == 0) ? "2.5GT/s," :
-					(pcie_table->pcie_gen[i] == 1) ? "5.0GT/s," :
-					(pcie_table->pcie_gen[i] == 2) ? "8.0GT/s," :
-					(pcie_table->pcie_gen[i] == 3) ? "16.0GT/s," : "",
-					(pcie_table->pcie_lane[i] == 1) ? "x1" :
-					(pcie_table->pcie_lane[i] == 2) ? "x2" :
-					(pcie_table->pcie_lane[i] == 3) ? "x4" :
-					(pcie_table->pcie_lane[i] == 4) ? "x8" :
-					(pcie_table->pcie_lane[i] == 5) ? "x12" :
-					(pcie_table->pcie_lane[i] == 6) ? "x16" : "",
-					pcie_table->clk_freq[i],
-					(gen_speed == DECODE_GEN_SPEED(pcie_table->pcie_gen[i])) &&
-					(lane_width == DECODE_LANE_WIDTH(pcie_table->pcie_lane[i])) ?
-					"*" : "");
-		break;
-
+		return smu_cmn_print_pcie_levels(smu, pcie_table, gen_speed,
+						 lane_width, buf, offset);
 	case SMU_OD_SCLK:
 		if (!smu_v13_0_7_is_od_feature_supported(smu,
 							 PP_OD_FEATURE_GFXCLK_BIT))
@@ -1524,7 +1442,21 @@ static int smu_v13_0_7_print_clk_levels(struct smu_context *smu,
 		break;
 	}
 
-	return size - start_offset;
+	if (single_dpm_table) {
+		ret = smu_v13_0_7_get_current_clk_freq_by_table(smu, clk_type,
+								&curr_freq);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Failed to get current clock freq!");
+			return ret;
+		}
+		return smu_cmn_print_dpm_clk_levels(smu, single_dpm_table,
+						    curr_freq, buf, offset);
+	}
+
+	*offset += size - start_offset;
+
+	return 0;
 }
 
 static int smu_v13_0_7_od_restore_table_single(struct smu_context *smu, long input)
@@ -1978,7 +1910,7 @@ static int smu_v13_0_7_force_clk_levels(struct smu_context *smu,
 {
 	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
 	struct smu_13_0_dpm_context *dpm_context = smu_dpm->dpm_context;
-	struct smu_13_0_dpm_table *single_dpm_table;
+	struct smu_dpm_table *single_dpm_table;
 	uint32_t soft_min_level, soft_max_level;
 	uint32_t min_freq, max_freq;
 	int ret = 0;
@@ -2024,7 +1956,7 @@ static int smu_v13_0_7_force_clk_levels(struct smu_context *smu,
 	case SMU_VCLK1:
 	case SMU_DCLK:
 	case SMU_DCLK1:
-		if (single_dpm_table->is_fine_grained) {
+		if (single_dpm_table->flags & SMU_DPM_TABLE_FINE_GRAINED) {
 			/* There is only 2 levels for fine grained DPM */
 			soft_max_level = (soft_max_level >= 1 ? 1 : 0);
 			soft_min_level = (soft_min_level >= 1 ? 1 : 0);
@@ -2289,18 +2221,12 @@ static int smu_v13_0_7_populate_umd_state_clk(struct smu_context *smu)
 {
 	struct smu_13_0_dpm_context *dpm_context =
 				smu->smu_dpm.dpm_context;
-	struct smu_13_0_dpm_table *gfx_table =
-				&dpm_context->dpm_tables.gfx_table;
-	struct smu_13_0_dpm_table *mem_table =
-				&dpm_context->dpm_tables.uclk_table;
-	struct smu_13_0_dpm_table *soc_table =
-				&dpm_context->dpm_tables.soc_table;
-	struct smu_13_0_dpm_table *vclk_table =
-				&dpm_context->dpm_tables.vclk_table;
-	struct smu_13_0_dpm_table *dclk_table =
-				&dpm_context->dpm_tables.dclk_table;
-	struct smu_13_0_dpm_table *fclk_table =
-				&dpm_context->dpm_tables.fclk_table;
+	struct smu_dpm_table *gfx_table = &dpm_context->dpm_tables.gfx_table;
+	struct smu_dpm_table *mem_table = &dpm_context->dpm_tables.uclk_table;
+	struct smu_dpm_table *soc_table = &dpm_context->dpm_tables.soc_table;
+	struct smu_dpm_table *vclk_table = &dpm_context->dpm_tables.vclk_table;
+	struct smu_dpm_table *dclk_table = &dpm_context->dpm_tables.dclk_table;
+	struct smu_dpm_table *fclk_table = &dpm_context->dpm_tables.fclk_table;
 	struct smu_umd_pstate_table *pstate_table =
 				&smu->pstate_table;
 	struct smu_table_context *table_context = &smu->smu_table;
@@ -2308,38 +2234,39 @@ static int smu_v13_0_7_populate_umd_state_clk(struct smu_context *smu)
 	DriverReportedClocks_t driver_clocks =
 		pptable->SkuTable.DriverReportedClocks;
 
-	pstate_table->gfxclk_pstate.min = gfx_table->min;
+	pstate_table->gfxclk_pstate.min = SMU_DPM_TABLE_MIN(gfx_table);
 	if (driver_clocks.GameClockAc &&
-		(driver_clocks.GameClockAc < gfx_table->max))
+	    (driver_clocks.GameClockAc < SMU_DPM_TABLE_MAX(gfx_table)))
 		pstate_table->gfxclk_pstate.peak = driver_clocks.GameClockAc;
 	else
-		pstate_table->gfxclk_pstate.peak = gfx_table->max;
+		pstate_table->gfxclk_pstate.peak = SMU_DPM_TABLE_MAX(gfx_table);
 
-	pstate_table->uclk_pstate.min = mem_table->min;
-	pstate_table->uclk_pstate.peak = mem_table->max;
+	pstate_table->uclk_pstate.min = SMU_DPM_TABLE_MIN(mem_table);
+	pstate_table->uclk_pstate.peak = SMU_DPM_TABLE_MAX(mem_table);
 
-	pstate_table->socclk_pstate.min = soc_table->min;
-	pstate_table->socclk_pstate.peak = soc_table->max;
+	pstate_table->socclk_pstate.min = SMU_DPM_TABLE_MIN(soc_table);
+	pstate_table->socclk_pstate.peak = SMU_DPM_TABLE_MAX(soc_table);
 
-	pstate_table->vclk_pstate.min = vclk_table->min;
-	pstate_table->vclk_pstate.peak = vclk_table->max;
+	pstate_table->vclk_pstate.min = SMU_DPM_TABLE_MIN(vclk_table);
+	pstate_table->vclk_pstate.peak = SMU_DPM_TABLE_MAX(vclk_table);
 
-	pstate_table->dclk_pstate.min = dclk_table->min;
-	pstate_table->dclk_pstate.peak = dclk_table->max;
+	pstate_table->dclk_pstate.min = SMU_DPM_TABLE_MIN(dclk_table);
+	pstate_table->dclk_pstate.peak = SMU_DPM_TABLE_MAX(dclk_table);
 
-	pstate_table->fclk_pstate.min = fclk_table->min;
-	pstate_table->fclk_pstate.peak = fclk_table->max;
+	pstate_table->fclk_pstate.min = SMU_DPM_TABLE_MIN(fclk_table);
+	pstate_table->fclk_pstate.peak = SMU_DPM_TABLE_MAX(fclk_table);
 
 	if (driver_clocks.BaseClockAc &&
-		driver_clocks.BaseClockAc < gfx_table->max)
+	    driver_clocks.BaseClockAc < SMU_DPM_TABLE_MAX(gfx_table))
 		pstate_table->gfxclk_pstate.standard = driver_clocks.BaseClockAc;
 	else
-		pstate_table->gfxclk_pstate.standard = gfx_table->max;
-	pstate_table->uclk_pstate.standard = mem_table->max;
-	pstate_table->socclk_pstate.standard = soc_table->min;
-	pstate_table->vclk_pstate.standard = vclk_table->min;
-	pstate_table->dclk_pstate.standard = dclk_table->min;
-	pstate_table->fclk_pstate.standard = fclk_table->min;
+		pstate_table->gfxclk_pstate.standard =
+			SMU_DPM_TABLE_MAX(gfx_table);
+	pstate_table->uclk_pstate.standard = SMU_DPM_TABLE_MAX(mem_table);
+	pstate_table->socclk_pstate.standard = SMU_DPM_TABLE_MIN(soc_table);
+	pstate_table->vclk_pstate.standard = SMU_DPM_TABLE_MIN(vclk_table);
+	pstate_table->dclk_pstate.standard = SMU_DPM_TABLE_MIN(dclk_table);
+	pstate_table->fclk_pstate.standard = SMU_DPM_TABLE_MIN(fclk_table);
 
 	return 0;
 }
@@ -2725,8 +2652,7 @@ static int smu_v13_0_7_update_pcie_parameters(struct smu_context *smu,
 				     uint8_t pcie_width_cap)
 {
 	struct smu_13_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
-	struct smu_13_0_pcie_table *pcie_table =
-				&dpm_context->dpm_tables.pcie_table;
+	struct smu_pcie_table *pcie_table = &dpm_context->dpm_tables.pcie_table;
 	int num_of_levels;
 	int link_level;
 	uint32_t smu_pcie_arg;
@@ -2736,23 +2662,23 @@ static int smu_v13_0_7_update_pcie_parameters(struct smu_context *smu,
 	int ret = 0;
 	int i;
 
-	pcie_table->num_of_link_levels = 0;
+	pcie_table->lclk_levels = 0;
 	for (link_level = 0; link_level < NUM_LINK_LEVELS; link_level++) {
 		if (!skutable->PcieGenSpeed[link_level] &&
 		    !skutable->PcieLaneCount[link_level] &&
 		    !skutable->LclkFreq[link_level])
 			continue;
 
-		pcie_table->pcie_gen[pcie_table->num_of_link_levels] =
-					skutable->PcieGenSpeed[link_level];
-		pcie_table->pcie_lane[pcie_table->num_of_link_levels] =
-					skutable->PcieLaneCount[link_level];
-		pcie_table->clk_freq[pcie_table->num_of_link_levels] =
-					skutable->LclkFreq[link_level];
-		pcie_table->num_of_link_levels++;
+		pcie_table->pcie_gen[pcie_table->lclk_levels] =
+			skutable->PcieGenSpeed[link_level];
+		pcie_table->pcie_lane[pcie_table->lclk_levels] =
+			skutable->PcieLaneCount[link_level];
+		pcie_table->lclk_freq[pcie_table->lclk_levels] =
+			skutable->LclkFreq[link_level];
+		pcie_table->lclk_levels++;
 	}
 
-	num_of_levels = pcie_table->num_of_link_levels;
+	num_of_levels = pcie_table->lclk_levels;
 	if (!num_of_levels)
 		return 0;
 
@@ -2830,7 +2756,7 @@ static const struct pptable_funcs smu_v13_0_7_ppt_funcs = {
 	.get_vbios_bootup_values = smu_v13_0_get_vbios_bootup_values,
 	.read_sensor = smu_v13_0_7_read_sensor,
 	.feature_is_enabled = smu_cmn_feature_is_enabled,
-	.print_clk_levels = smu_v13_0_7_print_clk_levels,
+	.emit_clk_levels = smu_v13_0_7_emit_clk_levels,
 	.force_clk_levels = smu_v13_0_7_force_clk_levels,
 	.update_pcie_parameters = smu_v13_0_7_update_pcie_parameters,
 	.get_thermal_temperature_range = smu_v13_0_7_get_thermal_temperature_range,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c
index 5d7e671fa3c3..e261edbf7852 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c
@@ -1038,17 +1038,15 @@ static uint32_t yellow_carp_get_umd_pstate_clk_default(struct smu_context *smu,
 	return clk_limit;
 }
 
-static int yellow_carp_print_clk_levels(struct smu_context *smu,
-				enum smu_clk_type clk_type, char *buf)
+static int yellow_carp_emit_clk_levels(struct smu_context *smu,
+				       enum smu_clk_type clk_type, char *buf,
+				       int *offset)
 {
-	int i, idx, size = 0, ret = 0, start_offset = 0;
+	int i, idx, size = *offset, ret = 0, start_offset = *offset;
 	uint32_t cur_value = 0, value = 0, count = 0;
 	uint32_t min, max;
 	uint32_t clk_limit = 0;
 
-	smu_cmn_get_sysfs_buf(&buf, &size);
-	start_offset = size;
-
 	switch (clk_type) {
 	case SMU_OD_SCLK:
 		size += sysfs_emit_at(buf, size, "%s:\n", "OD_SCLK");
@@ -1069,17 +1067,17 @@ static int yellow_carp_print_clk_levels(struct smu_context *smu,
 	case SMU_FCLK:
 		ret = yellow_carp_get_current_clk_freq(smu, clk_type, &cur_value);
 		if (ret)
-			goto print_clk_out;
+			return ret;
 
 		ret = yellow_carp_get_dpm_level_count(smu, clk_type, &count);
 		if (ret)
-			goto print_clk_out;
+			return ret;
 
 		for (i = 0; i < count; i++) {
 			idx = (clk_type == SMU_FCLK || clk_type == SMU_MCLK) ? (count - i - 1) : i;
 			ret = yellow_carp_get_dpm_freq_by_index(smu, clk_type, idx, &value);
 			if (ret)
-				goto print_clk_out;
+				return ret;
 
 			size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, value,
 					cur_value == value ? "*" : "");
@@ -1090,7 +1088,7 @@ static int yellow_carp_print_clk_levels(struct smu_context *smu,
 		clk_limit = yellow_carp_get_umd_pstate_clk_default(smu, clk_type);
 		ret = yellow_carp_get_current_clk_freq(smu, clk_type, &cur_value);
 		if (ret)
-			goto print_clk_out;
+			return ret;
 		min = (smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq;
 		max = (smu->gfx_actual_soft_max_freq > 0) ? smu->gfx_actual_soft_max_freq : smu->gfx_default_soft_max_freq;
 		if (cur_value  == max)
@@ -1111,8 +1109,9 @@ static int yellow_carp_print_clk_levels(struct smu_context *smu,
 		break;
 	}
 
-print_clk_out:
-	return size - start_offset;
+	*offset += size - start_offset;
+
+	return 0;
 }
 
 static int yellow_carp_force_clk_levels(struct smu_context *smu,
@@ -1349,7 +1348,7 @@ static const struct pptable_funcs yellow_carp_ppt_funcs = {
 	.mode2_reset = yellow_carp_mode2_reset,
 	.get_dpm_ultimate_freq = yellow_carp_get_dpm_ultimate_freq,
 	.od_edit_dpm_table = yellow_carp_od_edit_dpm_table,
-	.print_clk_levels = yellow_carp_print_clk_levels,
+	.emit_clk_levels = yellow_carp_emit_clk_levels,
 	.force_clk_levels = yellow_carp_force_clk_levels,
 	.set_performance_level = yellow_carp_set_performance_level,
 	.set_fine_grain_gfx_freq_parameters = yellow_carp_set_fine_grain_gfx_freq_parameters,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c
index f9b0938c57ea..e9b7fdb2fe22 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c
@@ -1247,18 +1247,12 @@ int smu_v14_0_set_performance_level(struct smu_context *smu,
 {
 	struct smu_14_0_dpm_context *dpm_context =
 		smu->smu_dpm.dpm_context;
-	struct smu_14_0_dpm_table *gfx_table =
-		&dpm_context->dpm_tables.gfx_table;
-	struct smu_14_0_dpm_table *mem_table =
-		&dpm_context->dpm_tables.uclk_table;
-	struct smu_14_0_dpm_table *soc_table =
-		&dpm_context->dpm_tables.soc_table;
-	struct smu_14_0_dpm_table *vclk_table =
-		&dpm_context->dpm_tables.vclk_table;
-	struct smu_14_0_dpm_table *dclk_table =
-		&dpm_context->dpm_tables.dclk_table;
-	struct smu_14_0_dpm_table *fclk_table =
-		&dpm_context->dpm_tables.fclk_table;
+	struct smu_dpm_table *gfx_table = &dpm_context->dpm_tables.gfx_table;
+	struct smu_dpm_table *mem_table = &dpm_context->dpm_tables.uclk_table;
+	struct smu_dpm_table *soc_table = &dpm_context->dpm_tables.soc_table;
+	struct smu_dpm_table *vclk_table = &dpm_context->dpm_tables.vclk_table;
+	struct smu_dpm_table *dclk_table = &dpm_context->dpm_tables.dclk_table;
+	struct smu_dpm_table *fclk_table = &dpm_context->dpm_tables.fclk_table;
 	struct smu_umd_pstate_table *pstate_table =
 		&smu->pstate_table;
 	struct amdgpu_device *adev = smu->adev;
@@ -1273,34 +1267,34 @@ int smu_v14_0_set_performance_level(struct smu_context *smu,
 
 	switch (level) {
 	case AMD_DPM_FORCED_LEVEL_HIGH:
-		sclk_min = sclk_max = gfx_table->max;
-		mclk_min = mclk_max = mem_table->max;
-		socclk_min = socclk_max = soc_table->max;
-		vclk_min = vclk_max = vclk_table->max;
-		dclk_min = dclk_max = dclk_table->max;
-		fclk_min = fclk_max = fclk_table->max;
+		sclk_min = sclk_max = SMU_DPM_TABLE_MAX(gfx_table);
+		mclk_min = mclk_max = SMU_DPM_TABLE_MAX(mem_table);
+		socclk_min = socclk_max = SMU_DPM_TABLE_MAX(soc_table);
+		vclk_min = vclk_max = SMU_DPM_TABLE_MAX(vclk_table);
+		dclk_min = dclk_max = SMU_DPM_TABLE_MAX(dclk_table);
+		fclk_min = fclk_max = SMU_DPM_TABLE_MAX(fclk_table);
 		break;
 	case AMD_DPM_FORCED_LEVEL_LOW:
-		sclk_min = sclk_max = gfx_table->min;
-		mclk_min = mclk_max = mem_table->min;
-		socclk_min = socclk_max = soc_table->min;
-		vclk_min = vclk_max = vclk_table->min;
-		dclk_min = dclk_max = dclk_table->min;
-		fclk_min = fclk_max = fclk_table->min;
+		sclk_min = sclk_max = SMU_DPM_TABLE_MIN(gfx_table);
+		mclk_min = mclk_max = SMU_DPM_TABLE_MIN(mem_table);
+		socclk_min = socclk_max = SMU_DPM_TABLE_MIN(soc_table);
+		vclk_min = vclk_max = SMU_DPM_TABLE_MIN(vclk_table);
+		dclk_min = dclk_max = SMU_DPM_TABLE_MIN(dclk_table);
+		fclk_min = fclk_max = SMU_DPM_TABLE_MIN(fclk_table);
 		break;
 	case AMD_DPM_FORCED_LEVEL_AUTO:
-		sclk_min = gfx_table->min;
-		sclk_max = gfx_table->max;
-		mclk_min = mem_table->min;
-		mclk_max = mem_table->max;
-		socclk_min = soc_table->min;
-		socclk_max = soc_table->max;
-		vclk_min = vclk_table->min;
-		vclk_max = vclk_table->max;
-		dclk_min = dclk_table->min;
-		dclk_max = dclk_table->max;
-		fclk_min = fclk_table->min;
-		fclk_max = fclk_table->max;
+		sclk_min = SMU_DPM_TABLE_MIN(gfx_table);
+		sclk_max = SMU_DPM_TABLE_MAX(gfx_table);
+		mclk_min = SMU_DPM_TABLE_MIN(mem_table);
+		mclk_max = SMU_DPM_TABLE_MAX(mem_table);
+		socclk_min = SMU_DPM_TABLE_MIN(soc_table);
+		socclk_max = SMU_DPM_TABLE_MAX(soc_table);
+		vclk_min = SMU_DPM_TABLE_MIN(vclk_table);
+		vclk_max = SMU_DPM_TABLE_MAX(vclk_table);
+		dclk_min = SMU_DPM_TABLE_MIN(dclk_table);
+		dclk_max = SMU_DPM_TABLE_MAX(dclk_table);
+		fclk_min = SMU_DPM_TABLE_MIN(fclk_table);
+		fclk_max = SMU_DPM_TABLE_MAX(fclk_table);
 		auto_level = true;
 		break;
 	case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
@@ -1522,11 +1516,12 @@ static int smu_v14_0_get_fine_grained_status(struct smu_context *smu,
 
 int smu_v14_0_set_single_dpm_table(struct smu_context *smu,
 				   enum smu_clk_type clk_type,
-				   struct smu_14_0_dpm_table *single_dpm_table)
+				   struct smu_dpm_table *single_dpm_table)
 {
 	int ret = 0;
 	uint32_t clk;
 	int i;
+	bool is_fine_grained;
 
 	ret = smu_v14_0_get_dpm_level_count(smu,
 					    clk_type,
@@ -1536,13 +1531,14 @@ int smu_v14_0_set_single_dpm_table(struct smu_context *smu,
 		return ret;
 	}
 
-	ret = smu_v14_0_get_fine_grained_status(smu,
-						clk_type,
-						&single_dpm_table->is_fine_grained);
+	ret = smu_v14_0_get_fine_grained_status(smu, clk_type,
+						&is_fine_grained);
 	if (ret) {
 		dev_err(smu->adev->dev, "[%s] failed to get fine grained status!\n", __func__);
 		return ret;
 	}
+	if (is_fine_grained)
+		single_dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
 
 	for (i = 0; i < single_dpm_table->count; i++) {
 		ret = smu_v14_0_get_dpm_freq_by_index(smu,
@@ -1556,11 +1552,6 @@ int smu_v14_0_set_single_dpm_table(struct smu_context *smu,
 
 		single_dpm_table->dpm_levels[i].value = clk;
 		single_dpm_table->dpm_levels[i].enabled = true;
-
-		if (i == 0)
-			single_dpm_table->min = clk;
-		else if (i == single_dpm_table->count - 1)
-			single_dpm_table->max = clk;
 	}
 
 	return 0;
@@ -1939,6 +1930,11 @@ int smu_v14_0_od_edit_dpm_table(struct smu_context *smu,
 			dev_err(smu->adev->dev, "Set soft max sclk failed!");
 			return ret;
 		}
+		if (smu->gfx_actual_hard_min_freq != smu->gfx_default_hard_min_freq ||
+		    smu->gfx_actual_soft_max_freq != smu->gfx_default_soft_max_freq)
+			smu->user_dpm_profile.user_od = true;
+		else
+			smu->user_dpm_profile.user_od = false;
 		break;
 	default:
 		return -ENOSYS;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c
index b1bd946d8e30..f9956f4b5112 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c
@@ -1129,16 +1129,14 @@ static int smu_v14_0_common_get_dpm_level_count(struct smu_context *smu,
 	return 0;
 }
 
-static int smu_v14_0_0_print_clk_levels(struct smu_context *smu,
-					enum smu_clk_type clk_type, char *buf)
+static int smu_v14_0_0_emit_clk_levels(struct smu_context *smu,
+				       enum smu_clk_type clk_type, char *buf,
+				       int *offset)
 {
-	int i, idx, ret = 0, size = 0, start_offset = 0;
+	int i, idx, ret = 0, size = *offset, start_offset = *offset;
 	uint32_t cur_value = 0, value = 0, count = 0;
 	uint32_t min, max;
 
-	smu_cmn_get_sysfs_buf(&buf, &size);
-	start_offset = size;
-
 	switch (clk_type) {
 	case SMU_OD_SCLK:
 		size += sysfs_emit_at(buf, size, "%s:\n", "OD_SCLK");
@@ -1162,17 +1160,17 @@ static int smu_v14_0_0_print_clk_levels(struct smu_context *smu,
 	case SMU_FCLK:
 		ret = smu_v14_0_0_get_current_clk_freq(smu, clk_type, &cur_value);
 		if (ret)
-			break;
+			return ret;
 
 		ret = smu_v14_0_common_get_dpm_level_count(smu, clk_type, &count);
 		if (ret)
-			break;
+			return ret;
 
 		for (i = 0; i < count; i++) {
 			idx = (clk_type == SMU_MCLK) ? (count - i - 1) : i;
 			ret = smu_v14_0_common_get_dpm_freq_by_index(smu, clk_type, idx, &value);
 			if (ret)
-				break;
+				return ret;
 
 			size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n", i, value,
 					      cur_value == value ? "*" : "");
@@ -1182,7 +1180,7 @@ static int smu_v14_0_0_print_clk_levels(struct smu_context *smu,
 	case SMU_SCLK:
 		ret = smu_v14_0_0_get_current_clk_freq(smu, clk_type, &cur_value);
 		if (ret)
-			break;
+			return ret;
 		min = (smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq;
 		max = (smu->gfx_actual_soft_max_freq > 0) ? smu->gfx_actual_soft_max_freq : smu->gfx_default_soft_max_freq;
 		if (cur_value  == max)
@@ -1203,7 +1201,9 @@ static int smu_v14_0_0_print_clk_levels(struct smu_context *smu,
 		break;
 	}
 
-	return size - start_offset;
+	*offset += size - start_offset;
+
+	return 0;
 }
 
 static int smu_v14_0_0_set_soft_freq_limited_range(struct smu_context *smu,
@@ -1514,9 +1514,10 @@ static int smu_v14_0_1_set_fine_grain_gfx_freq_parameters(struct smu_context *sm
 
 	smu->gfx_default_hard_min_freq = clk_table->MinGfxClk;
 	smu->gfx_default_soft_max_freq = clk_table->MaxGfxClk;
-	smu->gfx_actual_hard_min_freq = 0;
-	smu->gfx_actual_soft_max_freq = 0;
-
+	if (smu->gfx_actual_hard_min_freq == 0)
+		smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq;
+	if (smu->gfx_actual_soft_max_freq == 0)
+		smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq;
 	return 0;
 }
 
@@ -1526,8 +1527,10 @@ static int smu_v14_0_0_set_fine_grain_gfx_freq_parameters(struct smu_context *sm
 
 	smu->gfx_default_hard_min_freq = clk_table->MinGfxClk;
 	smu->gfx_default_soft_max_freq = clk_table->MaxGfxClk;
-	smu->gfx_actual_hard_min_freq = 0;
-	smu->gfx_actual_soft_max_freq = 0;
+	if (smu->gfx_actual_hard_min_freq == 0)
+		smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq;
+	if (smu->gfx_actual_soft_max_freq == 0)
+		smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq;
 
 	return 0;
 }
@@ -1665,6 +1668,29 @@ static int smu_v14_0_common_set_mall_enable(struct smu_context *smu)
 	return ret;
 }
 
+static int smu_v14_0_0_restore_user_od_settings(struct smu_context *smu)
+{
+	int ret;
+
+	ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetHardMinGfxClk,
+					      smu->gfx_actual_hard_min_freq,
+					      NULL);
+	if (ret) {
+		dev_err(smu->adev->dev, "Failed to restore hard min sclk!\n");
+		return ret;
+	}
+
+	ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMaxGfxClk,
+					      smu->gfx_actual_soft_max_freq,
+					      NULL);
+	if (ret) {
+		dev_err(smu->adev->dev, "Failed to restore soft max sclk!\n");
+		return ret;
+	}
+
+	return 0;
+}
+
 static const struct pptable_funcs smu_v14_0_0_ppt_funcs = {
 	.check_fw_status = smu_v14_0_check_fw_status,
 	.check_fw_version = smu_v14_0_check_fw_version,
@@ -1688,8 +1714,9 @@ static const struct pptable_funcs smu_v14_0_0_ppt_funcs = {
 	.mode2_reset = smu_v14_0_0_mode2_reset,
 	.get_dpm_ultimate_freq = smu_v14_0_common_get_dpm_ultimate_freq,
 	.set_soft_freq_limited_range = smu_v14_0_0_set_soft_freq_limited_range,
+	.restore_user_od_settings = smu_v14_0_0_restore_user_od_settings,
 	.od_edit_dpm_table = smu_v14_0_od_edit_dpm_table,
-	.print_clk_levels = smu_v14_0_0_print_clk_levels,
+	.emit_clk_levels = smu_v14_0_0_emit_clk_levels,
 	.force_clk_levels = smu_v14_0_0_force_clk_levels,
 	.set_performance_level = smu_v14_0_common_set_performance_level,
 	.set_fine_grain_gfx_freq_parameters = smu_v14_0_common_set_fine_grain_gfx_freq_parameters,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c
index 2cea688c604f..af1bc7b4350b 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c
@@ -501,11 +501,12 @@ static int smu_v14_0_2_set_default_dpm_table(struct smu_context *smu)
 	struct smu_table_context *table_context = &smu->smu_table;
 	PPTable_t *pptable = table_context->driver_pptable;
 	SkuTable_t *skutable = &pptable->SkuTable;
-	struct smu_14_0_dpm_table *dpm_table;
+	struct smu_dpm_table *dpm_table;
 	int ret = 0;
 
 	/* socclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.soc_table;
+	dpm_table->clk_type = SMU_SOCCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_SOCCLK_BIT)) {
 		ret = smu_v14_0_set_single_dpm_table(smu,
 						     SMU_SOCCLK,
@@ -516,12 +517,11 @@ static int smu_v14_0_2_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.socclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* gfxclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.gfx_table;
+	dpm_table->clk_type = SMU_GFXCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT)) {
 		ret = smu_v14_0_set_single_dpm_table(smu,
 						     SMU_GFXCLK,
@@ -543,18 +543,16 @@ static int smu_v14_0_2_set_default_dpm_table(struct smu_context *smu)
 		    skutable->DriverReportedClocks.GameClockAc)) {
 			dpm_table->dpm_levels[dpm_table->count - 1].value =
 				skutable->DriverReportedClocks.GameClockAc;
-			dpm_table->max = skutable->DriverReportedClocks.GameClockAc;
 		}
 	} else {
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* uclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.uclk_table;
+	dpm_table->clk_type = SMU_UCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_UCLK_BIT)) {
 		ret = smu_v14_0_set_single_dpm_table(smu,
 						     SMU_UCLK,
@@ -565,12 +563,11 @@ static int smu_v14_0_2_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.uclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* fclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.fclk_table;
+	dpm_table->clk_type = SMU_FCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_FCLK_BIT)) {
 		ret = smu_v14_0_set_single_dpm_table(smu,
 						     SMU_FCLK,
@@ -581,12 +578,11 @@ static int smu_v14_0_2_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.fclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* vclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.vclk_table;
+	dpm_table->clk_type = SMU_VCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_VCLK_BIT)) {
 		ret = smu_v14_0_set_single_dpm_table(smu,
 						     SMU_VCLK,
@@ -597,12 +593,11 @@ static int smu_v14_0_2_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.vclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* dclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.dclk_table;
+	dpm_table->clk_type = SMU_DCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCLK_BIT)) {
 		ret = smu_v14_0_set_single_dpm_table(smu,
 						     SMU_DCLK,
@@ -613,12 +608,11 @@ static int smu_v14_0_2_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	/* dcefclk dpm table setup */
 	dpm_table = &dpm_context->dpm_tables.dcef_table;
+	dpm_table->clk_type = SMU_DCEFCLK;
 	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_DCN_BIT)) {
 		ret = smu_v14_0_set_single_dpm_table(smu,
 						     SMU_DCEFCLK,
@@ -629,8 +623,6 @@ static int smu_v14_0_2_set_default_dpm_table(struct smu_context *smu)
 		dpm_table->count = 1;
 		dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dcefclk / 100;
 		dpm_table->dpm_levels[0].enabled = true;
-		dpm_table->min = dpm_table->dpm_levels[0].value;
-		dpm_table->max = dpm_table->dpm_levels[0].value;
 	}
 
 	return 0;
@@ -794,7 +786,7 @@ static int smu_v14_0_2_get_dpm_ultimate_freq(struct smu_context *smu,
 {
 	struct smu_14_0_dpm_context *dpm_context =
 		smu->smu_dpm.dpm_context;
-	struct smu_14_0_dpm_table *dpm_table;
+	struct smu_dpm_table *dpm_table;
 
 	switch (clk_type) {
 	case SMU_MCLK:
@@ -831,9 +823,9 @@ static int smu_v14_0_2_get_dpm_ultimate_freq(struct smu_context *smu,
 	}
 
 	if (min)
-		*min = dpm_table->min;
+		*min = SMU_DPM_TABLE_MIN(dpm_table);
 	if (max)
-		*max = dpm_table->max;
+		*max = SMU_DPM_TABLE_MAX(dpm_table);
 
 	return 0;
 }
@@ -1045,27 +1037,24 @@ static void smu_v14_0_2_get_od_setting_limits(struct smu_context *smu,
 		*max = od_max_setting;
 }
 
-static int smu_v14_0_2_print_clk_levels(struct smu_context *smu,
-					enum smu_clk_type clk_type,
-					char *buf)
+static int smu_v14_0_2_emit_clk_levels(struct smu_context *smu,
+				       enum smu_clk_type clk_type, char *buf,
+				       int *offset)
 {
 	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
 	struct smu_14_0_dpm_context *dpm_context = smu_dpm->dpm_context;
 	OverDriveTableExternal_t *od_table =
 		(OverDriveTableExternal_t *)smu->smu_table.overdrive_table;
-	struct smu_14_0_dpm_table *single_dpm_table;
-	struct smu_14_0_pcie_table *pcie_table;
+	struct smu_dpm_table *single_dpm_table = NULL;
+	struct smu_pcie_table *pcie_table;
 	uint32_t gen_speed, lane_width;
-	int i, curr_freq, size = 0, start_offset = 0;
+	int i, curr_freq, size = *offset, start_offset = *offset;
 	int32_t min_value, max_value;
 	int ret = 0;
 
-	smu_cmn_get_sysfs_buf(&buf, &size);
-	start_offset = size;
-
 	if (amdgpu_ras_intr_triggered()) {
-		size += sysfs_emit_at(buf, size, "unavailable\n");
-		return size - start_offset;
+		sysfs_emit_at(buf, size, "unavailable\n");
+		return -EBUSY;
 	}
 
 	switch (clk_type) {
@@ -1092,61 +1081,6 @@ static int smu_v14_0_2_print_clk_levels(struct smu_context *smu,
 	case SMU_DCEFCLK:
 		single_dpm_table = &(dpm_context->dpm_tables.dcef_table);
 		break;
-	default:
-		break;
-	}
-
-	switch (clk_type) {
-	case SMU_SCLK:
-	case SMU_MCLK:
-	case SMU_SOCCLK:
-	case SMU_FCLK:
-	case SMU_VCLK:
-	case SMU_VCLK1:
-	case SMU_DCLK:
-	case SMU_DCLK1:
-	case SMU_DCEFCLK:
-		ret = smu_v14_0_2_get_current_clk_freq_by_table(smu, clk_type, &curr_freq);
-		if (ret) {
-			dev_err(smu->adev->dev, "Failed to get current clock freq!");
-			return ret;
-		}
-
-		if (single_dpm_table->is_fine_grained) {
-			/*
-			 * For fine grained dpms, there are only two dpm levels:
-			 *   - level 0 -> min clock freq
-			 *   - level 1 -> max clock freq
-			 * And the current clock frequency can be any value between them.
-			 * So, if the current clock frequency is not at level 0 or level 1,
-			 * we will fake it as three dpm levels:
-			 *   - level 0 -> min clock freq
-			 *   - level 1 -> current actual clock freq
-			 *   - level 2 -> max clock freq
-			 */
-			if ((single_dpm_table->dpm_levels[0].value != curr_freq) &&
-			     (single_dpm_table->dpm_levels[1].value != curr_freq)) {
-				size += sysfs_emit_at(buf, size, "0: %uMhz\n",
-						single_dpm_table->dpm_levels[0].value);
-				size += sysfs_emit_at(buf, size, "1: %uMhz *\n",
-						curr_freq);
-				size += sysfs_emit_at(buf, size, "2: %uMhz\n",
-						single_dpm_table->dpm_levels[1].value);
-			} else {
-				size += sysfs_emit_at(buf, size, "0: %uMhz %s\n",
-						single_dpm_table->dpm_levels[0].value,
-						single_dpm_table->dpm_levels[0].value == curr_freq ? "*" : "");
-				size += sysfs_emit_at(buf, size, "1: %uMhz %s\n",
-						single_dpm_table->dpm_levels[1].value,
-						single_dpm_table->dpm_levels[1].value == curr_freq ? "*" : "");
-			}
-		} else {
-			for (i = 0; i < single_dpm_table->count; i++)
-				size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
-						i, single_dpm_table->dpm_levels[i].value,
-						single_dpm_table->dpm_levels[i].value == curr_freq ? "*" : "");
-		}
-		break;
 	case SMU_PCIE:
 		ret = smu_v14_0_2_get_smu_metrics_data(smu,
 						       METRICS_PCIE_RATE,
@@ -1161,25 +1095,8 @@ static int smu_v14_0_2_print_clk_levels(struct smu_context *smu,
 			return ret;
 
 		pcie_table = &(dpm_context->dpm_tables.pcie_table);
-		for (i = 0; i < pcie_table->num_of_link_levels; i++)
-			size += sysfs_emit_at(buf, size, "%d: %s %s %dMhz %s\n", i,
-					(pcie_table->pcie_gen[i] == 0) ? "2.5GT/s," :
-					(pcie_table->pcie_gen[i] == 1) ? "5.0GT/s," :
-					(pcie_table->pcie_gen[i] == 2) ? "8.0GT/s," :
-					(pcie_table->pcie_gen[i] == 3) ? "16.0GT/s," :
-					(pcie_table->pcie_gen[i] == 4) ? "32.0GT/s," : "",
-					(pcie_table->pcie_lane[i] == 1) ? "x1" :
-					(pcie_table->pcie_lane[i] == 2) ? "x2" :
-					(pcie_table->pcie_lane[i] == 3) ? "x4" :
-					(pcie_table->pcie_lane[i] == 4) ? "x8" :
-					(pcie_table->pcie_lane[i] == 5) ? "x12" :
-					(pcie_table->pcie_lane[i] == 6) ? "x16" :
-					(pcie_table->pcie_lane[i] == 7) ? "x32" : "",
-					pcie_table->clk_freq[i],
-					(gen_speed == DECODE_GEN_SPEED(pcie_table->pcie_gen[i])) &&
-					(lane_width == DECODE_LANE_WIDTH(pcie_table->pcie_lane[i])) ?
-					"*" : "");
-		break;
+		return smu_cmn_print_pcie_levels(smu, pcie_table, gen_speed,
+						 lane_width, buf, offset);
 
 	case SMU_OD_SCLK:
 		if (!smu_v14_0_2_is_od_feature_supported(smu,
@@ -1375,7 +1292,21 @@ static int smu_v14_0_2_print_clk_levels(struct smu_context *smu,
 		break;
 	}
 
-	return size - start_offset;
+	if (single_dpm_table) {
+		ret = smu_v14_0_2_get_current_clk_freq_by_table(smu, clk_type,
+								&curr_freq);
+		if (ret) {
+			dev_err(smu->adev->dev,
+				"Failed to get current clock freq!");
+			return ret;
+		}
+		return smu_cmn_print_dpm_clk_levels(smu, single_dpm_table,
+						    curr_freq, buf, offset);
+	}
+
+	*offset += size - start_offset;
+
+	return 0;
 }
 
 static int smu_v14_0_2_force_clk_levels(struct smu_context *smu,
@@ -1384,7 +1315,7 @@ static int smu_v14_0_2_force_clk_levels(struct smu_context *smu,
 {
 	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
 	struct smu_14_0_dpm_context *dpm_context = smu_dpm->dpm_context;
-	struct smu_14_0_dpm_table *single_dpm_table;
+	struct smu_dpm_table *single_dpm_table;
 	uint32_t soft_min_level, soft_max_level;
 	uint32_t min_freq, max_freq;
 	int ret = 0;
@@ -1430,7 +1361,7 @@ static int smu_v14_0_2_force_clk_levels(struct smu_context *smu,
 	case SMU_VCLK1:
 	case SMU_DCLK:
 	case SMU_DCLK1:
-		if (single_dpm_table->is_fine_grained) {
+		if (single_dpm_table->flags & SMU_DPM_TABLE_FINE_GRAINED) {
 			/* There is only 2 levels for fine grained DPM */
 			soft_max_level = (soft_max_level >= 1 ? 1 : 0);
 			soft_min_level = (soft_min_level >= 1 ? 1 : 0);
@@ -1463,9 +1394,9 @@ static int smu_v14_0_2_update_pcie_parameters(struct smu_context *smu,
 					      uint8_t pcie_width_cap)
 {
 	struct smu_14_0_dpm_context *dpm_context = smu->smu_dpm.dpm_context;
-	struct smu_14_0_pcie_table *pcie_table =
+	struct smu_pcie_table *pcie_table =
 				&dpm_context->dpm_tables.pcie_table;
-	int num_of_levels;
+	int lclk_levels;
 	uint32_t smu_pcie_arg;
 	uint32_t link_level;
 	struct smu_table_context *table_context = &smu->smu_table;
@@ -1474,34 +1405,34 @@ static int smu_v14_0_2_update_pcie_parameters(struct smu_context *smu,
 	int ret = 0;
 	int i;
 
-	pcie_table->num_of_link_levels = 0;
+	pcie_table->lclk_levels = 0;
 	for (link_level = 0; link_level < NUM_LINK_LEVELS; link_level++) {
 		if (!skutable->PcieGenSpeed[link_level] &&
 		    !skutable->PcieLaneCount[link_level] &&
 		    !skutable->LclkFreq[link_level])
 			continue;
 
-		pcie_table->pcie_gen[pcie_table->num_of_link_levels] =
+		pcie_table->pcie_gen[pcie_table->lclk_levels] =
 					skutable->PcieGenSpeed[link_level];
-		pcie_table->pcie_lane[pcie_table->num_of_link_levels] =
+		pcie_table->pcie_lane[pcie_table->lclk_levels] =
 					skutable->PcieLaneCount[link_level];
-		pcie_table->clk_freq[pcie_table->num_of_link_levels] =
+		pcie_table->lclk_freq[pcie_table->lclk_levels] =
 					skutable->LclkFreq[link_level];
-		pcie_table->num_of_link_levels++;
+		pcie_table->lclk_levels++;
 	}
-	num_of_levels = pcie_table->num_of_link_levels;
-	if (!num_of_levels)
+	lclk_levels = pcie_table->lclk_levels;
+	if (!lclk_levels)
 		return 0;
 
 	if (!(smu->adev->pm.pp_feature & PP_PCIE_DPM_MASK)) {
-		if (pcie_table->pcie_gen[num_of_levels - 1] < pcie_gen_cap)
-			pcie_gen_cap = pcie_table->pcie_gen[num_of_levels - 1];
+		if (pcie_table->pcie_gen[lclk_levels - 1] < pcie_gen_cap)
+			pcie_gen_cap = pcie_table->pcie_gen[lclk_levels - 1];
 
-		if (pcie_table->pcie_lane[num_of_levels - 1] < pcie_width_cap)
-			pcie_width_cap = pcie_table->pcie_lane[num_of_levels - 1];
+		if (pcie_table->pcie_lane[lclk_levels - 1] < pcie_width_cap)
+			pcie_width_cap = pcie_table->pcie_lane[lclk_levels - 1];
 
 		/* Force all levels to use the same settings */
-		for (i = 0; i < num_of_levels; i++) {
+		for (i = 0; i < lclk_levels; i++) {
 			pcie_table->pcie_gen[i] = pcie_gen_cap;
 			pcie_table->pcie_lane[i] = pcie_width_cap;
 			smu_pcie_arg = i << 16;
@@ -1516,7 +1447,7 @@ static int smu_v14_0_2_update_pcie_parameters(struct smu_context *smu,
 				break;
 		}
 	} else {
-		for (i = 0; i < num_of_levels; i++) {
+		for (i = 0; i < lclk_levels; i++) {
 			if (pcie_table->pcie_gen[i] > pcie_gen_cap ||
 				pcie_table->pcie_lane[i] > pcie_width_cap) {
 				pcie_table->pcie_gen[i] = pcie_table->pcie_gen[i] > pcie_gen_cap ?
@@ -1583,18 +1514,12 @@ static int smu_v14_0_2_populate_umd_state_clk(struct smu_context *smu)
 {
 	struct smu_14_0_dpm_context *dpm_context =
 		smu->smu_dpm.dpm_context;
-	struct smu_14_0_dpm_table *gfx_table =
-		&dpm_context->dpm_tables.gfx_table;
-	struct smu_14_0_dpm_table *mem_table =
-		&dpm_context->dpm_tables.uclk_table;
-	struct smu_14_0_dpm_table *soc_table =
-		&dpm_context->dpm_tables.soc_table;
-	struct smu_14_0_dpm_table *vclk_table =
-		&dpm_context->dpm_tables.vclk_table;
-	struct smu_14_0_dpm_table *dclk_table =
-		&dpm_context->dpm_tables.dclk_table;
-	struct smu_14_0_dpm_table *fclk_table =
-		&dpm_context->dpm_tables.fclk_table;
+	struct smu_dpm_table *gfx_table = &dpm_context->dpm_tables.gfx_table;
+	struct smu_dpm_table *mem_table = &dpm_context->dpm_tables.uclk_table;
+	struct smu_dpm_table *soc_table = &dpm_context->dpm_tables.soc_table;
+	struct smu_dpm_table *vclk_table = &dpm_context->dpm_tables.vclk_table;
+	struct smu_dpm_table *dclk_table = &dpm_context->dpm_tables.dclk_table;
+	struct smu_dpm_table *fclk_table = &dpm_context->dpm_tables.fclk_table;
 	struct smu_umd_pstate_table *pstate_table =
 		&smu->pstate_table;
 	struct smu_table_context *table_context = &smu->smu_table;
@@ -1602,38 +1527,39 @@ static int smu_v14_0_2_populate_umd_state_clk(struct smu_context *smu)
 	DriverReportedClocks_t driver_clocks =
 			pptable->SkuTable.DriverReportedClocks;
 
-	pstate_table->gfxclk_pstate.min = gfx_table->min;
+	pstate_table->gfxclk_pstate.min = SMU_DPM_TABLE_MIN(gfx_table);
 	if (driver_clocks.GameClockAc &&
-	    (driver_clocks.GameClockAc < gfx_table->max))
+	    (driver_clocks.GameClockAc < SMU_DPM_TABLE_MAX(gfx_table)))
 		pstate_table->gfxclk_pstate.peak = driver_clocks.GameClockAc;
 	else
-		pstate_table->gfxclk_pstate.peak = gfx_table->max;
+		pstate_table->gfxclk_pstate.peak = SMU_DPM_TABLE_MAX(gfx_table);
 
-	pstate_table->uclk_pstate.min = mem_table->min;
-	pstate_table->uclk_pstate.peak = mem_table->max;
+	pstate_table->uclk_pstate.min = SMU_DPM_TABLE_MIN(mem_table);
+	pstate_table->uclk_pstate.peak = SMU_DPM_TABLE_MAX(mem_table);
 
-	pstate_table->socclk_pstate.min = soc_table->min;
-	pstate_table->socclk_pstate.peak = soc_table->max;
+	pstate_table->socclk_pstate.min = SMU_DPM_TABLE_MIN(soc_table);
+	pstate_table->socclk_pstate.peak = SMU_DPM_TABLE_MAX(soc_table);
 
-	pstate_table->vclk_pstate.min = vclk_table->min;
-	pstate_table->vclk_pstate.peak = vclk_table->max;
+	pstate_table->vclk_pstate.min = SMU_DPM_TABLE_MIN(vclk_table);
+	pstate_table->vclk_pstate.peak = SMU_DPM_TABLE_MAX(vclk_table);
 
-	pstate_table->dclk_pstate.min = dclk_table->min;
-	pstate_table->dclk_pstate.peak = dclk_table->max;
+	pstate_table->dclk_pstate.min = SMU_DPM_TABLE_MIN(dclk_table);
+	pstate_table->dclk_pstate.peak = SMU_DPM_TABLE_MAX(dclk_table);
 
-	pstate_table->fclk_pstate.min = fclk_table->min;
-	pstate_table->fclk_pstate.peak = fclk_table->max;
+	pstate_table->fclk_pstate.min = SMU_DPM_TABLE_MIN(fclk_table);
+	pstate_table->fclk_pstate.peak = SMU_DPM_TABLE_MAX(fclk_table);
 
 	if (driver_clocks.BaseClockAc &&
-	    driver_clocks.BaseClockAc < gfx_table->max)
+	    driver_clocks.BaseClockAc < SMU_DPM_TABLE_MAX(gfx_table))
 		pstate_table->gfxclk_pstate.standard = driver_clocks.BaseClockAc;
 	else
-		pstate_table->gfxclk_pstate.standard = gfx_table->max;
-	pstate_table->uclk_pstate.standard = mem_table->max;
-	pstate_table->socclk_pstate.standard = soc_table->min;
-	pstate_table->vclk_pstate.standard = vclk_table->min;
-	pstate_table->dclk_pstate.standard = dclk_table->min;
-	pstate_table->fclk_pstate.standard = fclk_table->min;
+		pstate_table->gfxclk_pstate.standard =
+			SMU_DPM_TABLE_MAX(gfx_table);
+	pstate_table->uclk_pstate.standard = SMU_DPM_TABLE_MAX(mem_table);
+	pstate_table->socclk_pstate.standard = SMU_DPM_TABLE_MIN(soc_table);
+	pstate_table->vclk_pstate.standard = SMU_DPM_TABLE_MIN(vclk_table);
+	pstate_table->dclk_pstate.standard = SMU_DPM_TABLE_MIN(dclk_table);
+	pstate_table->fclk_pstate.standard = SMU_DPM_TABLE_MIN(fclk_table);
 
 	return 0;
 }
@@ -2873,7 +2799,7 @@ static const struct pptable_funcs smu_v14_0_2_ppt_funcs = {
 	.get_vbios_bootup_values = smu_v14_0_get_vbios_bootup_values,
 	.read_sensor = smu_v14_0_2_read_sensor,
 	.feature_is_enabled = smu_cmn_feature_is_enabled,
-	.print_clk_levels = smu_v14_0_2_print_clk_levels,
+	.emit_clk_levels = smu_v14_0_2_emit_clk_levels,
 	.force_clk_levels = smu_v14_0_2_force_clk_levels,
 	.update_pcie_parameters = smu_v14_0_2_update_pcie_parameters,
 	.get_thermal_temperature_range = smu_v14_0_2_get_thermal_temperature_range,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
index 4040ff926544..88e9699d9cc1 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
@@ -1199,3 +1199,121 @@ void smu_cmn_get_backend_workload_mask(struct smu_context *smu,
 		*backend_workload_mask |= 1 << workload_type;
 	}
 }
+
+static inline bool smu_cmn_freqs_match(uint32_t freq1, uint32_t freq2)
+{
+	/* Frequencies within 25 MHz are considered equal */
+	return (abs((int)freq1 - (int)freq2) <= 25);
+}
+
+int smu_cmn_print_dpm_clk_levels(struct smu_context *smu,
+				 struct smu_dpm_table *dpm_table,
+				 uint32_t cur_clk, char *buf, int *offset)
+{
+	uint32_t min_clk, level_index, count;
+	uint32_t freq_values[3] = { 0 };
+	bool is_fine_grained;
+	bool is_deep_sleep;
+	int size, lvl, i;
+	bool freq_match;
+
+	if (!dpm_table || !buf)
+		return -EINVAL;
+
+	level_index = 0;
+	size = *offset;
+	count = dpm_table->count;
+	is_fine_grained = dpm_table->flags & SMU_DPM_TABLE_FINE_GRAINED;
+	min_clk = SMU_DPM_TABLE_MIN(dpm_table);
+
+	/* Deep sleep - current clock < min_clock/2, TBD: cur_clk = 0 as GFXOFF */
+	is_deep_sleep = cur_clk < min_clk / 2;
+	if (is_deep_sleep) {
+		size += sysfs_emit_at(buf, size, "S: %uMhz *\n", cur_clk);
+		level_index = 1;
+	}
+
+	if (!is_fine_grained) {
+		for (i = 0; i < count; i++) {
+			freq_match = !is_deep_sleep &&
+				     smu_cmn_freqs_match(
+					     cur_clk,
+					     dpm_table->dpm_levels[i].value);
+			size += sysfs_emit_at(buf, size, "%d: %uMhz %s\n",
+					      level_index + i,
+					      dpm_table->dpm_levels[i].value,
+					      freq_match ? "*" : "");
+		}
+	} else {
+		freq_values[0] = min_clk;
+		freq_values[2] = SMU_DPM_TABLE_MAX(dpm_table);
+		freq_values[1] = cur_clk;
+
+		lvl = -1;
+		if (!is_deep_sleep) {
+			lvl = 1;
+			if (smu_cmn_freqs_match(cur_clk, freq_values[0]))
+				lvl = 0;
+			else if (smu_cmn_freqs_match(cur_clk, freq_values[2]))
+				lvl = 2;
+		}
+		count = 3;
+		if (lvl != 1) {
+			count = 2;
+			freq_values[1] = freq_values[2];
+		}
+
+		for (i = 0; i < count; i++) {
+			size += sysfs_emit_at(
+				buf, size, "%d: %uMhz %s\n", level_index + i,
+				freq_values[i],
+				(!is_deep_sleep && i == lvl) ? "*" : "");
+		}
+	}
+
+	*offset = size;
+
+	return 0;
+}
+
+int smu_cmn_print_pcie_levels(struct smu_context *smu,
+			      struct smu_pcie_table *pcie_table,
+			      uint32_t cur_gen, uint32_t cur_lane, char *buf,
+			      int *offset)
+{
+	int size, i;
+
+	if (!pcie_table || !buf)
+		return -EINVAL;
+
+	size = *offset;
+
+	for (i = 0; i < pcie_table->lclk_levels; i++) {
+		size += sysfs_emit_at(
+			buf, size, "%d: %s %s %dMhz %s\n", i,
+			(pcie_table->pcie_gen[i] == 0) ? "2.5GT/s," :
+			(pcie_table->pcie_gen[i] == 1) ? "5.0GT/s," :
+			(pcie_table->pcie_gen[i] == 2) ? "8.0GT/s," :
+			(pcie_table->pcie_gen[i] == 3) ? "16.0GT/s," :
+			(pcie_table->pcie_gen[i] == 4) ? "32.0GT/s," :
+			(pcie_table->pcie_gen[i] == 5) ? "64.0GT/s," :
+							 "",
+			(pcie_table->pcie_lane[i] == 1) ? "x1" :
+			(pcie_table->pcie_lane[i] == 2) ? "x2" :
+			(pcie_table->pcie_lane[i] == 3) ? "x4" :
+			(pcie_table->pcie_lane[i] == 4) ? "x8" :
+			(pcie_table->pcie_lane[i] == 5) ? "x12" :
+			(pcie_table->pcie_lane[i] == 6) ? "x16" :
+			(pcie_table->pcie_lane[i] == 7) ? "x32" :
+							  "",
+			pcie_table->lclk_freq[i],
+			(cur_gen == pcie_table->pcie_gen[i]) &&
+					(cur_lane == pcie_table->pcie_lane[i]) ?
+				"*" :
+				"");
+	}
+
+	*offset = size;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h
index 8d7c4814c68f..f458125e8d4e 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h
@@ -202,6 +202,16 @@ void smu_cmn_get_backend_workload_mask(struct smu_context *smu,
 				       u32 workload_mask,
 				       u32 *backend_workload_mask);
 
+int smu_cmn_print_dpm_clk_levels(struct smu_context *smu,
+				  struct smu_dpm_table *dpm_table,
+				  uint32_t cur_clk,
+				  char *buf, int *offset);
+
+int smu_cmn_print_pcie_levels(struct smu_context *smu,
+			       struct smu_pcie_table *pcie_table,
+			       uint32_t cur_gen, uint32_t cur_lane,
+			       char *buf, int *offset);
+
 /*SMU gpu metrics */
 
 /* Attribute ID mapping */
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/Makefile b/drivers/gpu/drm/amd/ras/ras_mgr/Makefile
index 5e5a2cfa4068..6759635aadc2 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/Makefile
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/Makefile
@@ -23,6 +23,7 @@ RAS_MGR_FILES = amdgpu_ras_sys.o  \
 		amdgpu_ras_eeprom_i2c.o \
 		amdgpu_ras_mp1_v13_0.o \
 		amdgpu_ras_cmd.o \
+		amdgpu_virt_ras_cmd.o \
 		amdgpu_ras_process.o \
 		amdgpu_ras_nbio_v7_9.o
 
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c
index 78419b7f7729..cb6498c30834 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c
@@ -28,6 +28,7 @@
 #include "ras_sys.h"
 #include "amdgpu_ras_cmd.h"
 #include "amdgpu_ras_mgr.h"
+#include "amdgpu_virt_ras_cmd.h"
 
 /* inject address is 52 bits */
 #define	RAS_UMC_INJECT_ADDR_LIMIT	(0x1ULL << 52)
@@ -252,6 +253,7 @@ int amdgpu_ras_handle_cmd(struct ras_core_context *ras_core, struct ras_cmd_ctx
 
 int amdgpu_ras_submit_cmd(struct ras_core_context *ras_core, struct ras_cmd_ctx *cmd)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
 	struct ras_core_context *cmd_core = ras_core;
 	int timeout = 60;
 	int res;
@@ -259,6 +261,9 @@ int amdgpu_ras_submit_cmd(struct ras_core_context *ras_core, struct ras_cmd_ctx
 	cmd->cmd_res = RAS_CMD__ERROR_INVALID_CMD;
 	cmd->output_size = 0;
 
+	if (amdgpu_sriov_vf(adev))
+		return amdgpu_virt_ras_handle_cmd(cmd_core, cmd);
+
 	if (!ras_core_is_enabled(cmd_core))
 		return RAS_CMD__ERROR_ACCESS_DENIED;
 
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
index afe8135b6258..b86638fe0f32 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
@@ -27,6 +27,7 @@
 #include "ras_sys.h"
 #include "amdgpu_ras_mgr.h"
 #include "amdgpu_ras_cmd.h"
+#include "amdgpu_virt_ras_cmd.h"
 #include "amdgpu_ras_process.h"
 #include "amdgpu_ras_eeprom_i2c.h"
 #include "amdgpu_ras_mp1_v13_0.h"
@@ -215,6 +216,7 @@ static int amdgpu_ras_mgr_get_ras_ta_init_param(struct ras_core_context *ras_cor
 	ras_ta_param->channel_dis_num = hweight32(adev->gmc.m_half_use) * 2;
 
 	ras_ta_param->active_umc_mask = adev->umc.active_mask;
+	ras_ta_param->vram_type = (uint8_t)adev->gmc.vram_type;
 
 	if (!amdgpu_ras_mgr_get_curr_nps_mode(adev, &nps_mode))
 		ras_ta_param->nps_mode = nps_mode;
@@ -316,6 +318,16 @@ static int amdgpu_ras_mgr_sw_init(struct amdgpu_ip_block *ip_block)
 	amdgpu_ras_process_init(adev);
 	ras_core_sw_init(ras_mgr->ras_core);
 	amdgpu_ras_mgr_init_event_mgr(ras_mgr->ras_core);
+
+	if (amdgpu_sriov_vf(adev)) {
+		ret = amdgpu_virt_ras_sw_init(adev);
+		if (ret) {
+			RAS_DEV_ERR(adev,
+				"Virt ras sw_init failed! ret:%d\n", ret);
+			goto err;
+		}
+	}
+
 	return 0;
 
 err:
@@ -335,6 +347,9 @@ static int amdgpu_ras_mgr_sw_fini(struct amdgpu_ip_block *ip_block)
 	if (!ras_mgr)
 		return 0;
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_ras_sw_fini(adev);
+
 	amdgpu_ras_process_fini(adev);
 	ras_core_sw_fini(ras_mgr->ras_core);
 	ras_core_destroy(ras_mgr->ras_core);
@@ -359,9 +374,13 @@ static int amdgpu_ras_mgr_hw_init(struct amdgpu_ip_block *ip_block)
 	if (!ras_mgr || !ras_mgr->ras_core)
 		return -EINVAL;
 
-	ret = ras_core_hw_init(ras_mgr->ras_core);
+	if (amdgpu_sriov_vf(adev))
+		ret = amdgpu_virt_ras_hw_init(adev);
+	else
+		ret = ras_core_hw_init(ras_mgr->ras_core);
+
 	if (ret) {
-		RAS_DEV_ERR(adev, "Failed to initialize ras core!\n");
+		RAS_DEV_ERR(adev, "Failed to initialize hw_init!, ret:%d\n", ret);
 		return ret;
 	}
 
@@ -385,7 +404,10 @@ static int amdgpu_ras_mgr_hw_fini(struct amdgpu_ip_block *ip_block)
 	if (!ras_mgr || !ras_mgr->ras_core)
 		return -EINVAL;
 
-	ras_core_hw_fini(ras_mgr->ras_core);
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_ras_hw_fini(adev);
+	else
+		ras_core_hw_fini(ras_mgr->ras_core);
 
 	ras_mgr->ras_is_ready = false;
 
@@ -423,9 +445,6 @@ int amdgpu_enable_uniras(struct amdgpu_device *adev, bool enable)
 	if (!ras_mgr || !ras_mgr->ras_core)
 		return -EPERM;
 
-	if (amdgpu_sriov_vf(adev))
-		return -EPERM;
-
 	RAS_DEV_INFO(adev, "Enable amdgpu unified ras!");
 	return ras_core_set_status(ras_mgr->ras_core, enable);
 }
@@ -434,10 +453,10 @@ bool amdgpu_uniras_enabled(struct amdgpu_device *adev)
 {
 	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
 
-	if (!ras_mgr || !ras_mgr->ras_core)
-		return false;
-
 	if (amdgpu_sriov_vf(adev))
+		return amdgpu_virt_ras_remote_uniras_enabled(adev);
+
+	if (!ras_mgr || !ras_mgr->ras_core)
 		return false;
 
 	return ras_core_is_enabled(ras_mgr->ras_core);
@@ -603,7 +622,7 @@ int amdgpu_ras_mgr_handle_ras_cmd(struct amdgpu_device *adev,
 	uint32_t ctx_buf_size = PAGE_SIZE;
 	int ret;
 
-	if (!amdgpu_ras_mgr_is_ready(adev))
+	if (!amdgpu_sriov_vf(adev) && !amdgpu_ras_mgr_is_ready(adev))
 		return -EPERM;
 
 	cmd_ctx = kzalloc(ctx_buf_size, GFP_KERNEL);
@@ -627,6 +646,9 @@ int amdgpu_ras_mgr_handle_ras_cmd(struct amdgpu_device *adev,
 
 int amdgpu_ras_mgr_pre_reset(struct amdgpu_device *adev)
 {
+	if (amdgpu_sriov_vf(adev))
+		return amdgpu_virt_ras_pre_reset(adev);
+
 	if (!amdgpu_ras_mgr_is_ready(adev)) {
 		RAS_DEV_ERR(adev, "Invalid ras suspend!\n");
 		return -EPERM;
@@ -638,6 +660,9 @@ int amdgpu_ras_mgr_pre_reset(struct amdgpu_device *adev)
 
 int amdgpu_ras_mgr_post_reset(struct amdgpu_device *adev)
 {
+	if (amdgpu_sriov_vf(adev))
+		return amdgpu_virt_ras_post_reset(adev);
+
 	if (!amdgpu_ras_mgr_is_ready(adev)) {
 		RAS_DEV_ERR(adev, "Invalid ras resume!\n");
 		return -EPERM;
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h
index 8fb7eb4b8f13..080ac84fc5a5 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h
@@ -50,6 +50,7 @@ struct amdgpu_ras_mgr {
 	struct ras_core_context *ras_core;
 	struct delayed_work retire_page_dwork;
 	struct ras_event_manager ras_event_mgr;
+	void *virt_ras_cmd;
 	uint64_t last_poison_consumption_seqno;
 	bool ras_is_ready;
 
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c
new file mode 100644
index 000000000000..a75479593864
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c
@@ -0,0 +1,469 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/pci.h>
+#include "amdgpu.h"
+#include "amdgpu_ras.h"
+#include "ras_sys.h"
+#include "amdgpu_ras_cmd.h"
+#include "amdgpu_virt_ras_cmd.h"
+#include "amdgpu_ras_mgr.h"
+
+static int amdgpu_virt_ras_remote_ioctl_cmd(struct ras_core_context *ras_core,
+			struct ras_cmd_ctx *cmd, void *output_data, uint32_t output_size)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
+	uint32_t mem_len = ALIGN(sizeof(*cmd) + output_size, AMDGPU_GPU_PAGE_SIZE);
+	struct ras_cmd_ctx *rcmd;
+	struct amdgpu_bo *rcmd_bo = NULL;
+	uint64_t mc_addr = 0;
+	void *cpu_addr = NULL;
+	int ret = 0;
+
+	ret = amdgpu_bo_create_kernel(adev, mem_len, PAGE_SIZE,
+			AMDGPU_GEM_DOMAIN_VRAM, &rcmd_bo, &mc_addr, (void **)&cpu_addr);
+	if (ret)
+		return ret;
+
+	rcmd = (struct ras_cmd_ctx *)cpu_addr;
+	memset(rcmd, 0, mem_len);
+	memcpy(rcmd, cmd, sizeof(*cmd));
+
+	ret = amdgpu_virt_send_remote_ras_cmd(ras_core->dev,
+				mc_addr - adev->gmc.vram_start, mem_len);
+	if (!ret) {
+		if (rcmd->cmd_res) {
+			ret = rcmd->cmd_res;
+			goto out;
+		}
+
+		cmd->cmd_res = rcmd->cmd_res;
+		cmd->output_size = rcmd->output_size;
+		if (rcmd->output_size && (rcmd->output_size <= output_size) && output_data)
+			memcpy(output_data, rcmd->output_buff_raw, rcmd->output_size);
+	}
+
+out:
+	amdgpu_bo_free_kernel(&rcmd_bo, &mc_addr, &cpu_addr);
+
+	return ret;
+}
+
+static int amdgpu_virt_ras_send_remote_cmd(struct ras_core_context *ras_core,
+	uint32_t cmd_id, void *input_data, uint32_t input_size,
+	void *output_data, uint32_t output_size)
+{
+	struct ras_cmd_ctx rcmd = {0};
+	int ret;
+
+	rcmd.cmd_id = cmd_id;
+	rcmd.input_size = input_size;
+	memcpy(rcmd.input_buff_raw, input_data, input_size);
+
+	ret = amdgpu_virt_ras_remote_ioctl_cmd(ras_core,
+				&rcmd, output_data, output_size);
+	if (!ret) {
+		if (rcmd.output_size != output_size)
+			return RAS_CMD__ERROR_GENERIC;
+	}
+
+	return ret;
+}
+
+static int amdgpu_virt_ras_get_batch_trace_overview(struct ras_core_context *ras_core,
+	struct ras_log_batch_overview *overview)
+{
+	struct ras_cmd_batch_trace_snapshot_req req = {0};
+	struct ras_cmd_batch_trace_snapshot_rsp rsp = {0};
+	int ret;
+
+	ret = amdgpu_virt_ras_send_remote_cmd(ras_core, RAS_CMD__GET_BATCH_TRACE_SNAPSHOT,
+				&req, sizeof(req), &rsp, sizeof(rsp));
+	if (ret)
+		return ret;
+
+	overview->first_batch_id = rsp.start_batch_id;
+	overview->last_batch_id = rsp.latest_batch_id;
+	overview->logged_batch_count = rsp.total_batch_num;
+
+	return RAS_CMD__SUCCESS;
+}
+
+static int amdgpu_virt_ras_get_cper_snapshot(struct ras_core_context *ras_core,
+			struct ras_cmd_ctx *cmd, void *data)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(ras_core->dev);
+	struct amdgpu_virt_ras_cmd *virt_ras =
+			(struct amdgpu_virt_ras_cmd *)ras_mgr->virt_ras_cmd;
+	int ret;
+
+	if (cmd->input_size != sizeof(struct ras_cmd_cper_snapshot_req))
+		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
+
+	ret = amdgpu_virt_ras_send_remote_cmd(ras_core, cmd->cmd_id,
+			cmd->input_buff_raw, cmd->input_size,
+			cmd->output_buff_raw, sizeof(struct ras_cmd_cper_snapshot_rsp));
+	if (ret)
+		return ret;
+
+	memset(&virt_ras->batch_mgr, 0, sizeof(virt_ras->batch_mgr));
+	amdgpu_virt_ras_get_batch_trace_overview(ras_core,
+					&virt_ras->batch_mgr.batch_overview);
+
+	cmd->output_size = sizeof(struct ras_cmd_cper_snapshot_rsp);
+	return RAS_CMD__SUCCESS;
+}
+
+static int amdgpu_virt_ras_get_batch_records(struct ras_core_context *ras_core, uint64_t batch_id,
+			struct ras_log_info **trace_arr, uint32_t arr_num,
+			struct ras_cmd_batch_trace_record_rsp *rsp_cache)
+{
+	struct ras_cmd_batch_trace_record_req req = {
+		.start_batch_id = batch_id,
+		.batch_num = RAS_CMD_MAX_BATCH_NUM,
+	};
+	struct ras_cmd_batch_trace_record_rsp *rsp = rsp_cache;
+	struct batch_ras_trace_info *batch;
+	int ret = 0;
+	uint8_t i;
+
+	if (!rsp->real_batch_num || (batch_id < rsp->start_batch_id) ||
+		(batch_id >=  (rsp->start_batch_id + rsp->real_batch_num))) {
+
+		memset(rsp, 0, sizeof(*rsp));
+		ret = amdgpu_virt_ras_send_remote_cmd(ras_core, RAS_CMD__GET_BATCH_TRACE_RECORD,
+			&req, sizeof(req), rsp, sizeof(*rsp));
+		if (ret)
+			return -EPIPE;
+	}
+
+	batch = &rsp->batchs[batch_id - rsp->start_batch_id];
+	if (batch_id != batch->batch_id)
+		return -ENODATA;
+
+	for (i = 0; i < batch->trace_num; i++) {
+		if (i >= arr_num)
+			break;
+		trace_arr[i] = &rsp->records[batch->offset + i];
+	}
+
+	return i;
+}
+
+static int amdgpu_virt_ras_get_cper_records(struct ras_core_context *ras_core,
+	struct ras_cmd_ctx *cmd, void *data)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(ras_core->dev);
+	struct amdgpu_virt_ras_cmd *virt_ras =
+			(struct amdgpu_virt_ras_cmd *)ras_mgr->virt_ras_cmd;
+	struct ras_cmd_cper_record_req *req =
+		(struct ras_cmd_cper_record_req *)cmd->input_buff_raw;
+	struct ras_cmd_cper_record_rsp *rsp =
+		(struct ras_cmd_cper_record_rsp *)cmd->output_buff_raw;
+	struct ras_log_batch_overview *overview = &virt_ras->batch_mgr.batch_overview;
+	struct ras_cmd_batch_trace_record_rsp *rsp_cache = &virt_ras->batch_mgr.batch_trace;
+	struct ras_log_info **trace;
+	uint32_t offset = 0, real_data_len = 0;
+	uint64_t batch_id;
+	uint8_t *out_buf;
+	int ret = 0, i, count;
+
+	if (cmd->input_size != sizeof(struct ras_cmd_cper_record_req))
+		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
+
+	if (!req->buf_size || !req->buf_ptr || !req->cper_num)
+		return RAS_CMD__ERROR_INVALID_INPUT_DATA;
+
+	trace = kcalloc(MAX_RECORD_PER_BATCH, sizeof(*trace), GFP_KERNEL);
+	if (!trace)
+		return RAS_CMD__ERROR_GENERIC;
+
+	out_buf = kzalloc(req->buf_size, GFP_KERNEL);
+	if (!out_buf) {
+		kfree(trace);
+		return RAS_CMD__ERROR_GENERIC;
+	}
+
+	memset(out_buf, 0, req->buf_size);
+
+	for (i = 0; i < req->cper_num; i++) {
+		batch_id = req->cper_start_id + i;
+		if (batch_id >= overview->last_batch_id)
+			break;
+		count = amdgpu_virt_ras_get_batch_records(ras_core, batch_id,
+							  trace, MAX_RECORD_PER_BATCH,
+							  rsp_cache);
+		if (count > 0) {
+			ret = ras_cper_generate_cper(ras_core, trace, count,
+					&out_buf[offset], req->buf_size - offset, &real_data_len);
+			if (ret)
+				break;
+
+			offset += real_data_len;
+		}
+	}
+
+	if ((ret && (ret != -ENOMEM)) ||
+	    copy_to_user(u64_to_user_ptr(req->buf_ptr), out_buf, offset)) {
+		kfree(out_buf);
+		kfree(trace);
+		return RAS_CMD__ERROR_GENERIC;
+	}
+
+	rsp->real_data_size = offset;
+	rsp->real_cper_num = i;
+	rsp->remain_num = (ret == -ENOMEM) ? (req->cper_num - i) : 0;
+	rsp->version = 0;
+
+	cmd->output_size = sizeof(struct ras_cmd_cper_record_rsp);
+
+	kfree(out_buf);
+	kfree(trace);
+
+	return RAS_CMD__SUCCESS;
+}
+
+static int __fill_get_blocks_ecc_cmd(struct amdgpu_device *adev,
+			struct vram_blocks_ecc *blks_ecc)
+{
+	struct ras_cmd_ctx *rcmd;
+
+	if (!blks_ecc || !blks_ecc->bo || !blks_ecc->cpu_addr)
+		return -EINVAL;
+
+	rcmd = (struct ras_cmd_ctx *)blks_ecc->cpu_addr;
+
+	rcmd->cmd_id = RAS_CMD__GET_ALL_BLOCK_ECC_STATUS;
+	rcmd->input_size = sizeof(struct ras_cmd_blocks_ecc_req);
+	rcmd->output_buf_size = blks_ecc->size - sizeof(*rcmd);
+
+	return 0;
+}
+
+static int __set_cmd_auto_update(struct amdgpu_device *adev,
+			enum ras_cmd_id cmd_id, uint64_t gpa_addr, uint32_t len, bool reg)
+{
+	struct ras_cmd_auto_update_req req = {0};
+	struct ras_cmd_auto_update_rsp rsp = {0};
+	int ret;
+
+	req.mode = reg ? 1 : 0;
+	req.cmd_id = cmd_id;
+	req.addr = gpa_addr;
+	req.len = len;
+	ret = amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__SET_CMD_AUTO_UPDATE,
+		&req, sizeof(req), &rsp, sizeof(rsp));
+
+	return ret;
+}
+
+static int amdgpu_virt_ras_get_block_ecc(struct ras_core_context *ras_core,
+				struct ras_cmd_ctx *cmd, void *data)
+{
+	struct amdgpu_device *adev = ras_core->dev;
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	struct amdgpu_virt_ras_cmd *virt_ras =
+			(struct amdgpu_virt_ras_cmd *)ras_mgr->virt_ras_cmd;
+	struct vram_blocks_ecc *blks_ecc = &virt_ras->blocks_ecc;
+	struct ras_cmd_ctx *blks_ecc_cmd_ctx;
+	struct ras_cmd_blocks_ecc_rsp *blks_ecc_rsp;
+	struct ras_cmd_block_ecc_info_req *input_data =
+			(struct ras_cmd_block_ecc_info_req *)cmd->input_buff_raw;
+	struct ras_cmd_block_ecc_info_rsp *output_data =
+			(struct ras_cmd_block_ecc_info_rsp *)cmd->output_buff_raw;
+	int ret = 0;
+
+	if (cmd->input_size != sizeof(struct ras_cmd_block_ecc_info_req))
+		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
+
+	if (input_data->block_id >= MAX_RAS_BLOCK_NUM)
+		return RAS_CMD__ERROR_INVALID_INPUT_DATA;
+
+	if (__fill_get_blocks_ecc_cmd(adev, blks_ecc))
+		return RAS_CMD__ERROR_GENERIC;
+
+	if (!virt_ras->blocks_ecc.auto_update_actived) {
+		ret = __set_cmd_auto_update(adev, RAS_CMD__GET_ALL_BLOCK_ECC_STATUS,
+				blks_ecc->mc_addr - adev->gmc.vram_start,
+				blks_ecc->size, true);
+		if (ret)
+			return ret;
+
+		blks_ecc->auto_update_actived = true;
+	}
+
+	blks_ecc_cmd_ctx = blks_ecc->cpu_addr;
+	blks_ecc_rsp = (struct ras_cmd_blocks_ecc_rsp *)blks_ecc_cmd_ctx->output_buff_raw;
+
+	output_data->ce_count = blks_ecc_rsp->blocks[input_data->block_id].ce_count;
+	output_data->ue_count = blks_ecc_rsp->blocks[input_data->block_id].ue_count;
+	output_data->de_count = blks_ecc_rsp->blocks[input_data->block_id].de_count;
+
+	cmd->output_size = sizeof(struct ras_cmd_block_ecc_info_rsp);
+	return RAS_CMD__SUCCESS;
+}
+
+static struct ras_cmd_func_map amdgpu_virt_ras_cmd_maps[] = {
+	{RAS_CMD__GET_CPER_SNAPSHOT, amdgpu_virt_ras_get_cper_snapshot},
+	{RAS_CMD__GET_CPER_RECORD, amdgpu_virt_ras_get_cper_records},
+	{RAS_CMD__GET_BLOCK_ECC_STATUS, amdgpu_virt_ras_get_block_ecc},
+};
+
+int amdgpu_virt_ras_handle_cmd(struct ras_core_context *ras_core,
+		struct ras_cmd_ctx *cmd)
+{
+	struct ras_cmd_func_map *ras_cmd = NULL;
+	int i, res;
+
+	for (i = 0; i < ARRAY_SIZE(amdgpu_virt_ras_cmd_maps); i++) {
+		if (cmd->cmd_id == amdgpu_virt_ras_cmd_maps[i].cmd_id) {
+			ras_cmd = &amdgpu_virt_ras_cmd_maps[i];
+			break;
+		}
+	}
+
+	if (ras_cmd)
+		res = ras_cmd->func(ras_core, cmd, NULL);
+	else
+		res = amdgpu_virt_ras_remote_ioctl_cmd(ras_core, cmd,
+					cmd->output_buff_raw, cmd->output_buf_size);
+
+	cmd->cmd_res = res;
+
+	if (cmd->output_size > cmd->output_buf_size) {
+		RAS_DEV_ERR(ras_core->dev,
+			"Output data size 0x%x exceeds buffer size 0x%x!\n",
+			cmd->output_size, cmd->output_buf_size);
+		return RAS_CMD__SUCCESS_EXEED_BUFFER;
+	}
+
+	return RAS_CMD__SUCCESS;
+}
+
+int amdgpu_virt_ras_sw_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+
+	ras_mgr->virt_ras_cmd = kzalloc(sizeof(struct amdgpu_virt_ras_cmd), GFP_KERNEL);
+	if (!ras_mgr->virt_ras_cmd)
+		return -ENOMEM;
+
+	return 0;
+}
+
+int amdgpu_virt_ras_sw_fini(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+
+	kfree(ras_mgr->virt_ras_cmd);
+	ras_mgr->virt_ras_cmd = NULL;
+
+	return 0;
+}
+
+int amdgpu_virt_ras_hw_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	struct amdgpu_virt_ras_cmd *virt_ras =
+			(struct amdgpu_virt_ras_cmd *)ras_mgr->virt_ras_cmd;
+	struct vram_blocks_ecc *blks_ecc = &virt_ras->blocks_ecc;
+
+	amdgpu_virt_get_ras_capability(adev);
+
+	memset(blks_ecc, 0, sizeof(*blks_ecc));
+	blks_ecc->size = PAGE_SIZE;
+	if (amdgpu_bo_create_kernel(adev, blks_ecc->size,
+			PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
+			&blks_ecc->bo, &blks_ecc->mc_addr,
+			(void **)&blks_ecc->cpu_addr))
+		return -ENOMEM;
+
+	return 0;
+}
+
+int amdgpu_virt_ras_hw_fini(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	struct amdgpu_virt_ras_cmd *virt_ras =
+			(struct amdgpu_virt_ras_cmd *)ras_mgr->virt_ras_cmd;
+	struct vram_blocks_ecc *blks_ecc = &virt_ras->blocks_ecc;
+
+	if (blks_ecc->bo) {
+		__set_cmd_auto_update(adev,
+			RAS_CMD__GET_ALL_BLOCK_ECC_STATUS,
+			blks_ecc->mc_addr - adev->gmc.vram_start,
+			blks_ecc->size, false);
+
+		memset(blks_ecc->cpu_addr, 0, blks_ecc->size);
+		amdgpu_bo_free_kernel(&blks_ecc->bo,
+			&blks_ecc->mc_addr, &blks_ecc->cpu_addr);
+
+		memset(blks_ecc, 0, sizeof(*blks_ecc));
+	}
+
+	return 0;
+}
+
+int amdgpu_virt_ras_pre_reset(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	struct amdgpu_virt_ras_cmd *virt_ras =
+		(struct amdgpu_virt_ras_cmd *)ras_mgr->virt_ras_cmd;
+
+	virt_ras->blocks_ecc.auto_update_actived = false;
+	return 0;
+}
+
+int amdgpu_virt_ras_post_reset(struct amdgpu_device *adev)
+{
+	return 0;
+}
+
+void amdgpu_virt_ras_set_remote_uniras(struct amdgpu_device *adev, bool en)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	struct amdgpu_virt_ras_cmd *virt_ras;
+
+	if (!ras_mgr || !ras_mgr->virt_ras_cmd)
+		return;
+
+	virt_ras = (struct amdgpu_virt_ras_cmd *)ras_mgr->virt_ras_cmd;
+	virt_ras->remote_uniras_supported = en;
+}
+
+bool amdgpu_virt_ras_remote_uniras_enabled(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	struct amdgpu_virt_ras_cmd *virt_ras;
+
+	if (amdgpu_in_reset(adev))
+		return false;
+
+	if (!ras_mgr || !ras_mgr->virt_ras_cmd)
+		return false;
+
+	virt_ras = (struct amdgpu_virt_ras_cmd *)ras_mgr->virt_ras_cmd;
+
+	return virt_ras->remote_uniras_supported;
+}
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.h b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.h
new file mode 100644
index 000000000000..53b0f3f60103
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.h
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __AMDGPU_VIRT_RAS_CMD_H__
+#define __AMDGPU_VIRT_RAS_CMD_H__
+#include "ras.h"
+
+struct remote_batch_trace_mgr {
+	struct ras_log_batch_overview  batch_overview;
+	struct ras_cmd_batch_trace_record_rsp  batch_trace;
+};
+
+struct vram_blocks_ecc {
+	struct amdgpu_bo *bo;
+	uint64_t mc_addr;
+	void *cpu_addr;
+	uint32_t size;
+	bool auto_update_actived;
+};
+
+struct amdgpu_virt_ras_cmd {
+	bool remote_uniras_supported;
+	struct remote_batch_trace_mgr batch_mgr;
+	struct vram_blocks_ecc blocks_ecc;
+};
+
+int amdgpu_virt_ras_sw_init(struct amdgpu_device *adev);
+int amdgpu_virt_ras_sw_fini(struct amdgpu_device *adev);
+int amdgpu_virt_ras_hw_init(struct amdgpu_device *adev);
+int amdgpu_virt_ras_hw_fini(struct amdgpu_device *adev);
+int amdgpu_virt_ras_handle_cmd(struct ras_core_context *ras_core,
+		struct ras_cmd_ctx *cmd);
+int amdgpu_virt_ras_pre_reset(struct amdgpu_device *adev);
+int amdgpu_virt_ras_post_reset(struct amdgpu_device *adev);
+void amdgpu_virt_ras_set_remote_uniras(struct amdgpu_device *adev, bool en);
+bool amdgpu_virt_ras_remote_uniras_enabled(struct amdgpu_device *adev);
+#endif
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_cmd.h b/drivers/gpu/drm/amd/ras/rascore/ras_cmd.h
index 48a0715eb821..b9833812c31f 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_cmd.h
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_cmd.h
@@ -75,6 +75,8 @@ enum ras_cmd_id {
 	RAS_CMD__GET_CPER_RECORD,
 	RAS_CMD__GET_BATCH_TRACE_SNAPSHOT,
 	RAS_CMD__GET_BATCH_TRACE_RECORD,
+	RAS_CMD__GET_ALL_BLOCK_ECC_STATUS,
+	RAS_CMD__SET_CMD_AUTO_UPDATE,
 	RAS_CMD__SUPPORTED_MAX = RAS_CMD_ID_COMMON_END,
 };
 
@@ -411,6 +413,37 @@ struct ras_cmd_batch_trace_record_rsp {
 	struct ras_log_info records[RAS_CMD_MAX_TRACE_NUM];
 };
 
+struct ras_cmd_auto_update_req {
+	struct ras_cmd_dev_handle dev;
+	uint32_t mode;
+	uint32_t cmd_id;
+	uint64_t addr;
+	uint32_t len;
+	uint32_t reserved[5];
+};
+
+struct ras_cmd_auto_update_rsp {
+	uint32_t version;
+	uint32_t reserved[4];
+};
+
+struct ras_cmd_blocks_ecc_req {
+	struct ras_cmd_dev_handle dev;
+};
+
+struct ras_cmd_block_ecc {
+	uint32_t ce_count;
+	uint32_t ue_count;
+	uint32_t de_count;
+};
+
+#define MAX_RAS_BLOCK_NUM  20
+struct ras_cmd_blocks_ecc_rsp {
+	uint32_t version;
+	uint32_t reserved[5];
+	struct ras_cmd_block_ecc blocks[MAX_RAS_BLOCK_NUM];
+};
+
 #pragma pack(pop)
 
 int ras_cmd_init(struct ras_core_context *ras_core);
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_psp.c b/drivers/gpu/drm/amd/ras/rascore/ras_psp.c
index ccdb42d2dd60..5d556e2a7000 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_psp.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_psp.c
@@ -507,6 +507,7 @@ static int send_load_ta_fw_cmd(struct ras_core_context *ras_core,
 	ta_init_flags->channel_dis_num = ta_ctx->init_param.channel_dis_num;
 	ta_init_flags->nps_mode = ta_ctx->init_param.nps_mode;
 	ta_init_flags->active_umc_mask = ta_ctx->init_param.active_umc_mask;
+	ta_init_flags->vram_type = ta_ctx->init_param.vram_type;
 
 	/* Setup load ras ta command */
 	memset(&psp_load_ta_cmd, 0, sizeof(psp_load_ta_cmd));
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_psp.h b/drivers/gpu/drm/amd/ras/rascore/ras_psp.h
index 71776fecfd66..347f5334c3f3 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_psp.h
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_psp.h
@@ -51,6 +51,7 @@ struct ras_ta_init_param {
 	uint8_t channel_dis_num;
 	uint8_t nps_mode;
 	uint32_t active_umc_mask;
+	uint8_t vram_type;
 };
 
 struct gpu_mem_block {
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_ta_if.h b/drivers/gpu/drm/amd/ras/rascore/ras_ta_if.h
index 0921e36d3274..e910a75b3022 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_ta_if.h
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_ta_if.h
@@ -167,6 +167,7 @@ struct ras_ta_init_flags {
 	uint8_t channel_dis_num;
 	uint8_t nps_mode;
 	uint32_t active_umc_mask;
+	uint8_t vram_type;
 };
 
 struct ras_ta_mca_addr {
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_umc.c b/drivers/gpu/drm/amd/ras/rascore/ras_umc.c
index 4dae64c424a2..a0fdc3fda761 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_umc.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_umc.c
@@ -497,27 +497,40 @@ static int ras_umc_save_bad_pages(struct ras_core_context *ras_core)
 
 int ras_umc_handle_bad_pages(struct ras_core_context *ras_core, void *data)
 {
-	struct eeprom_umc_record records[MAX_ECC_NUM_PER_RETIREMENT];
+	struct eeprom_umc_record *records;
 	int count, ret;
 
-	memset(records, 0, sizeof(records));
-	count = ras_umc_get_new_records(ras_core, records, ARRAY_SIZE(records));
-	if (count <= 0)
-		return -ENODATA;
+	records = kcalloc(MAX_ECC_NUM_PER_RETIREMENT,
+			  sizeof(*records), GFP_KERNEL);
+	if (!records)
+		return -ENOMEM;
+
+	count = ras_umc_get_new_records(ras_core, records,
+					MAX_ECC_NUM_PER_RETIREMENT);
+	if (count <= 0) {
+		ret = -ENODATA;
+		goto out;
+	}
 
 	ret = ras_umc_add_bad_pages(ras_core, records, count, false);
 	if (ret) {
 		RAS_DEV_ERR(ras_core->dev, "Failed to add ras bad page!\n");
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
 
 	ret = ras_umc_save_bad_pages(ras_core);
 	if (ret) {
 		RAS_DEV_ERR(ras_core->dev, "Failed to save ras bad page\n");
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
 
-	return 0;
+	ret = 0;
+
+out:
+	kfree(records);
+	return ret;
 }
 
 int ras_umc_sw_init(struct ras_core_context *ras_core)
diff --git a/drivers/gpu/drm/ast/ast_cursor.c b/drivers/gpu/drm/ast/ast_cursor.c
index 2d3ad7610c2e..30b62d3f0151 100644
--- a/drivers/gpu/drm/ast/ast_cursor.c
+++ b/drivers/gpu/drm/ast/ast_cursor.c
@@ -28,6 +28,7 @@
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_format_helper.h>
 #include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_print.h>
 
 #include "ast_drv.h"
@@ -181,6 +182,62 @@ static int ast_cursor_plane_helper_atomic_check(struct drm_plane *plane,
 	return 0;
 }
 
+static const u8 *ast_cursor_plane_get_argb4444(struct ast_cursor_plane *ast_cursor_plane,
+					       struct drm_shadow_plane_state *shadow_plane_state,
+					       const struct drm_rect *clip)
+{
+	struct drm_plane_state *plane_state = &shadow_plane_state->base;
+	struct drm_framebuffer *fb = plane_state->fb;
+	u8 *argb4444 = NULL;
+
+	if (drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE) == 0) {
+		switch (fb->format->format) {
+		case DRM_FORMAT_ARGB4444:
+			if (shadow_plane_state->data[0].is_iomem) {
+				struct iosys_map argb4444_dst[DRM_FORMAT_MAX_PLANES] = {
+					IOSYS_MAP_INIT_VADDR(ast_cursor_plane->argb4444),
+				};
+				unsigned int argb4444_dst_pitch[DRM_FORMAT_MAX_PLANES] = {
+					AST_HWC_PITCH,
+				};
+
+				drm_fb_memcpy(argb4444_dst, argb4444_dst_pitch,
+					      shadow_plane_state->data, fb, clip);
+				argb4444 = argb4444_dst[0].vaddr;
+			} else {
+				argb4444 = shadow_plane_state->data[0].vaddr;
+			}
+			break;
+		case DRM_FORMAT_ARGB8888:
+			{
+				struct iosys_map argb4444_dst[DRM_FORMAT_MAX_PLANES] = {
+					IOSYS_MAP_INIT_VADDR(ast_cursor_plane->argb4444),
+				};
+				unsigned int argb4444_dst_pitch[DRM_FORMAT_MAX_PLANES] = {
+					AST_HWC_PITCH,
+				};
+
+				drm_fb_argb8888_to_argb4444(argb4444_dst, argb4444_dst_pitch,
+							    shadow_plane_state->data, fb, clip,
+							    &shadow_plane_state->fmtcnv_state);
+				argb4444 = argb4444_dst[0].vaddr;
+			}
+			break;
+		}
+
+		drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE);
+	} else {
+		/*
+		 * Fall back to white square if GEM object is not ready. Gives
+		 * the user an indication where the cursor is located.
+		 */
+		memset(ast_cursor_plane->argb4444, 0xff, sizeof(ast_cursor_plane->argb4444));
+		argb4444 = ast_cursor_plane->argb4444;
+	}
+
+	return argb4444;
+}
+
 static void ast_cursor_plane_helper_atomic_update(struct drm_plane *plane,
 						  struct drm_atomic_state *state)
 {
@@ -205,29 +262,13 @@ static void ast_cursor_plane_helper_atomic_update(struct drm_plane *plane,
 	 */
 
 	if (drm_atomic_helper_damage_merged(old_plane_state, plane_state, &damage)) {
-		u8 *argb4444;
+		const u8 *argb4444 = ast_cursor_plane_get_argb4444(ast_cursor_plane,
+								   shadow_plane_state,
+								   &damage);
 
-		switch (fb->format->format) {
-		case DRM_FORMAT_ARGB4444:
-			argb4444 = shadow_plane_state->data[0].vaddr;
-			break;
-		default:
-			argb4444 = ast_cursor_plane->argb4444;
-			{
-				struct iosys_map argb4444_dst[DRM_FORMAT_MAX_PLANES] = {
-					IOSYS_MAP_INIT_VADDR(argb4444),
-				};
-				unsigned int argb4444_dst_pitch[DRM_FORMAT_MAX_PLANES] = {
-					AST_HWC_PITCH,
-				};
+		if (argb4444)
+			ast_set_cursor_image(ast, argb4444, fb->width, fb->height);
 
-				drm_fb_argb8888_to_argb4444(argb4444_dst, argb4444_dst_pitch,
-							    shadow_plane_state->data, fb, &damage,
-							    &shadow_plane_state->fmtcnv_state);
-			}
-			break;
-		}
-		ast_set_cursor_image(ast, argb4444, fb->width, fb->height);
 		ast_set_cursor_base(ast, dst_off);
 	}
 
diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi83.c b/drivers/gpu/drm/bridge/ti-sn65dsi83.c
index fffb47b62f43..f6736b4457bb 100644
--- a/drivers/gpu/drm/bridge/ti-sn65dsi83.c
+++ b/drivers/gpu/drm/bridge/ti-sn65dsi83.c
@@ -406,6 +406,10 @@ static void sn65dsi83_reset_work(struct work_struct *ws)
 {
 	struct sn65dsi83 *ctx = container_of(ws, struct sn65dsi83, reset_work);
 	int ret;
+	int idx;
+
+	if (!drm_bridge_enter(&ctx->bridge, &idx))
+		return;
 
 	/* Reset the pipe */
 	ret = sn65dsi83_reset_pipe(ctx);
@@ -415,12 +419,18 @@ static void sn65dsi83_reset_work(struct work_struct *ws)
 	}
 	if (ctx->irq)
 		enable_irq(ctx->irq);
+
+	drm_bridge_exit(idx);
 }
 
 static void sn65dsi83_handle_errors(struct sn65dsi83 *ctx)
 {
 	unsigned int irq_stat;
 	int ret;
+	int idx;
+
+	if (!drm_bridge_enter(&ctx->bridge, &idx))
+		return;
 
 	/*
 	 * Schedule a reset in case of:
@@ -448,6 +458,8 @@ static void sn65dsi83_handle_errors(struct sn65dsi83 *ctx)
 
 		schedule_work(&ctx->reset_work);
 	}
+
+	drm_bridge_exit(idx);
 }
 
 static void sn65dsi83_monitor_work(struct work_struct *work)
@@ -470,6 +482,37 @@ static void sn65dsi83_monitor_stop(struct sn65dsi83 *ctx)
 	cancel_delayed_work_sync(&ctx->monitor_work);
 }
 
+/*
+ * Release resources taken by sn65dsi83_atomic_pre_enable().
+ *
+ * Invoked by sn65dsi83_atomic_disable() normally, or by devres after
+ * sn65dsi83_remove() in case this happens befora atomic_disable.
+ */
+static void sn65dsi83_release_resources(void *data)
+{
+	struct sn65dsi83 *ctx = (struct sn65dsi83 *)data;
+	int ret;
+
+	if (ctx->irq) {
+		/* Disable irq */
+		regmap_write(ctx->regmap, REG_IRQ_EN, 0x0);
+		regmap_write(ctx->regmap, REG_IRQ_GLOBAL, 0x0);
+	} else {
+		/* Stop the polling task */
+		sn65dsi83_monitor_stop(ctx);
+	}
+
+	/* Put the chip in reset, pull EN line low, and assure 10ms reset low timing. */
+	gpiod_set_value_cansleep(ctx->enable_gpio, 0);
+	usleep_range(10000, 11000);
+
+	ret = regulator_disable(ctx->vcc);
+	if (ret)
+		dev_err(ctx->dev, "Failed to disable vcc: %d\n", ret);
+
+	regcache_mark_dirty(ctx->regmap);
+}
+
 static void sn65dsi83_atomic_pre_enable(struct drm_bridge *bridge,
 					struct drm_atomic_state *state)
 {
@@ -485,11 +528,15 @@ static void sn65dsi83_atomic_pre_enable(struct drm_bridge *bridge,
 	__le16 le16val;
 	u16 val;
 	int ret;
+	int idx;
+
+	if (!drm_bridge_enter(bridge, &idx))
+		return;
 
 	ret = regulator_enable(ctx->vcc);
 	if (ret) {
 		dev_err(ctx->dev, "Failed to enable vcc: %d\n", ret);
-		return;
+		goto err_exit;
 	}
 
 	/* Deassert reset */
@@ -632,7 +679,7 @@ static void sn65dsi83_atomic_pre_enable(struct drm_bridge *bridge,
 		dev_err(ctx->dev, "failed to lock PLL, ret=%i\n", ret);
 		/* On failure, disable PLL again and exit. */
 		regmap_write(ctx->regmap, REG_RC_PLL_EN, 0x00);
-		return;
+		goto err_add_action;
 	}
 
 	/* Trigger reset after CSR register update. */
@@ -640,6 +687,11 @@ static void sn65dsi83_atomic_pre_enable(struct drm_bridge *bridge,
 
 	/* Wait for 10ms after soft reset as specified in datasheet */
 	usleep_range(10000, 12000);
+
+err_add_action:
+	devm_add_action(ctx->dev, sn65dsi83_release_resources, ctx);
+err_exit:
+	drm_bridge_exit(idx);
 }
 
 static void sn65dsi83_atomic_enable(struct drm_bridge *bridge,
@@ -647,6 +699,10 @@ static void sn65dsi83_atomic_enable(struct drm_bridge *bridge,
 {
 	struct sn65dsi83 *ctx = bridge_to_sn65dsi83(bridge);
 	unsigned int pval;
+	int idx;
+
+	if (!drm_bridge_enter(bridge, &idx))
+		return;
 
 	/* Clear all errors that got asserted during initialization. */
 	regmap_read(ctx->regmap, REG_IRQ_STAT, &pval);
@@ -666,32 +722,22 @@ static void sn65dsi83_atomic_enable(struct drm_bridge *bridge,
 		/* Use the polling task */
 		sn65dsi83_monitor_start(ctx);
 	}
+
+	drm_bridge_exit(idx);
 }
 
 static void sn65dsi83_atomic_disable(struct drm_bridge *bridge,
 				     struct drm_atomic_state *state)
 {
 	struct sn65dsi83 *ctx = bridge_to_sn65dsi83(bridge);
-	int ret;
+	int idx;
 
-	if (ctx->irq) {
-		/* Disable irq */
-		regmap_write(ctx->regmap, REG_IRQ_EN, 0x0);
-		regmap_write(ctx->regmap, REG_IRQ_GLOBAL, 0x0);
-	} else {
-		/* Stop the polling task */
-		sn65dsi83_monitor_stop(ctx);
-	}
-
-	/* Put the chip in reset, pull EN line low, and assure 10ms reset low timing. */
-	gpiod_set_value_cansleep(ctx->enable_gpio, 0);
-	usleep_range(10000, 11000);
+	if (!drm_bridge_enter(bridge, &idx))
+		return;
 
-	ret = regulator_disable(ctx->vcc);
-	if (ret)
-		dev_err(ctx->dev, "Failed to disable vcc: %d\n", ret);
+	devm_release_action(ctx->dev, sn65dsi83_release_resources, ctx);
 
-	regcache_mark_dirty(ctx->regmap);
+	drm_bridge_exit(idx);
 }
 
 static enum drm_mode_status
@@ -1012,7 +1058,7 @@ static void sn65dsi83_remove(struct i2c_client *client)
 {
 	struct sn65dsi83 *ctx = i2c_get_clientdata(client);
 
-	drm_bridge_remove(&ctx->bridge);
+	drm_bridge_unplug(&ctx->bridge);
 }
 
 static const struct i2c_device_id sn65dsi83_id[] = {
diff --git a/drivers/gpu/drm/ci/arm.config b/drivers/gpu/drm/ci/arm.config
index 411e814819a8..d7c51670da2f 100644
--- a/drivers/gpu/drm/ci/arm.config
+++ b/drivers/gpu/drm/ci/arm.config
@@ -52,7 +52,7 @@ CONFIG_TMPFS=y
 CONFIG_PROVE_LOCKING=n
 CONFIG_DEBUG_LOCKDEP=n
 CONFIG_SOFTLOCKUP_DETECTOR=n
-CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=n
+CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=0
 
 CONFIG_FW_LOADER_COMPRESS=y
 
diff --git a/drivers/gpu/drm/ci/arm64.config b/drivers/gpu/drm/ci/arm64.config
index fddfbd4d2493..ea0e30737c4d 100644
--- a/drivers/gpu/drm/ci/arm64.config
+++ b/drivers/gpu/drm/ci/arm64.config
@@ -161,7 +161,7 @@ CONFIG_TMPFS=y
 CONFIG_PROVE_LOCKING=n
 CONFIG_DEBUG_LOCKDEP=n
 CONFIG_SOFTLOCKUP_DETECTOR=y
-CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y
+CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=1
 
 CONFIG_DETECT_HUNG_TASK=y
 
diff --git a/drivers/gpu/drm/ci/x86_64.config b/drivers/gpu/drm/ci/x86_64.config
index 8eaba388b141..7ac98a78691e 100644
--- a/drivers/gpu/drm/ci/x86_64.config
+++ b/drivers/gpu/drm/ci/x86_64.config
@@ -47,7 +47,7 @@ CONFIG_TMPFS=y
 CONFIG_PROVE_LOCKING=n
 CONFIG_DEBUG_LOCKDEP=n
 CONFIG_SOFTLOCKUP_DETECTOR=y
-CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y
+CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=1
 
 CONFIG_DETECT_HUNG_TASK=y
 
diff --git a/drivers/gpu/drm/display/drm_dp_mst_topology.c b/drivers/gpu/drm/display/drm_dp_mst_topology.c
index 64e5c176d5cc..be749dcad3b5 100644
--- a/drivers/gpu/drm/display/drm_dp_mst_topology.c
+++ b/drivers/gpu/drm/display/drm_dp_mst_topology.c
@@ -4572,7 +4572,8 @@ int drm_dp_atomic_release_time_slots(struct drm_atomic_state *state,
 	if (!payload->delete) {
 		payload->pbn = 0;
 		payload->delete = true;
-		topology_state->payload_mask &= ~BIT(payload->vcpi - 1);
+		if (payload->vcpi > 0)
+			topology_state->payload_mask &= ~BIT(payload->vcpi - 1);
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c
index 67e095e398a3..6d3ea8056b60 100644
--- a/drivers/gpu/drm/drm_atomic.c
+++ b/drivers/gpu/drm/drm_atomic.c
@@ -906,6 +906,7 @@ drm_atomic_private_obj_init(struct drm_device *dev,
 
 	drm_modeset_lock_init(&obj->lock);
 
+	obj->dev = dev;
 	obj->state = state;
 	obj->funcs = funcs;
 	list_add_tail(&obj->head, &dev->mode_config.privobj_list);
diff --git a/drivers/gpu/drm/drm_bridge.c b/drivers/gpu/drm/drm_bridge.c
index 8f355df883d8..db40c26d1cb3 100644
--- a/drivers/gpu/drm/drm_bridge.c
+++ b/drivers/gpu/drm/drm_bridge.c
@@ -27,6 +27,7 @@
 #include <linux/media-bus-format.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
+#include <linux/srcu.h>
 
 #include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
@@ -202,6 +203,67 @@ static DEFINE_MUTEX(bridge_lock);
 static LIST_HEAD(bridge_list);
 static LIST_HEAD(bridge_lingering_list);
 
+DEFINE_STATIC_SRCU(drm_bridge_unplug_srcu);
+
+/**
+ * drm_bridge_enter - Enter DRM bridge critical section
+ * @bridge: DRM bridge
+ * @idx: Pointer to index that will be passed to the matching drm_bridge_exit()
+ *
+ * This function marks and protects the beginning of a section that should not
+ * be entered after the bridge has been unplugged. The section end is marked
+ * with drm_bridge_exit(). Calls to this function can be nested.
+ *
+ * Returns:
+ * True if it is OK to enter the section, false otherwise.
+ */
+bool drm_bridge_enter(struct drm_bridge *bridge, int *idx)
+{
+	*idx = srcu_read_lock(&drm_bridge_unplug_srcu);
+
+	if (bridge->unplugged) {
+		srcu_read_unlock(&drm_bridge_unplug_srcu, *idx);
+		return false;
+	}
+
+	return true;
+}
+EXPORT_SYMBOL(drm_bridge_enter);
+
+/**
+ * drm_bridge_exit - Exit DRM bridge critical section
+ * @idx: index returned by drm_bridge_enter()
+ *
+ * This function marks the end of a section that should not be entered after
+ * the bridge has been unplugged.
+ */
+void drm_bridge_exit(int idx)
+{
+	srcu_read_unlock(&drm_bridge_unplug_srcu, idx);
+}
+EXPORT_SYMBOL(drm_bridge_exit);
+
+/**
+ * drm_bridge_unplug - declare a DRM bridge was unplugged and remove it
+ * @bridge: DRM bridge
+ *
+ * This tells the bridge has been physically unplugged and no operations on
+ * device resources must be done anymore. Entry-points can use
+ * drm_bridge_enter() and drm_bridge_exit() to protect device resources in
+ * a race free manner.
+ *
+ * Also unregisters the bridge.
+ */
+void drm_bridge_unplug(struct drm_bridge *bridge)
+{
+	bridge->unplugged = true;
+
+	synchronize_srcu(&drm_bridge_unplug_srcu);
+
+	drm_bridge_remove(bridge);
+}
+EXPORT_SYMBOL(drm_bridge_unplug);
+
 static void __drm_bridge_free(struct kref *kref)
 {
 	struct drm_bridge *bridge = container_of(kref, struct drm_bridge, refcount);
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index 4a7f72044ab8..29c3c7bb7d67 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -75,9 +75,6 @@ MODULE_PARM_DESC(drm_leak_fbdev_smem,
 		 "Allow unsafe leaking fbdev physical smem address [default=false]");
 #endif
 
-static LIST_HEAD(kernel_fb_helper_list);
-static DEFINE_MUTEX(kernel_fb_helper_lock);
-
 /**
  * DOC: fbdev helpers
  *
@@ -115,101 +112,6 @@ static DEFINE_MUTEX(kernel_fb_helper_lock);
  * mmap page writes.
  */
 
-static void drm_fb_helper_restore_lut_atomic(struct drm_crtc *crtc)
-{
-	uint16_t *r_base, *g_base, *b_base;
-
-	if (crtc->funcs->gamma_set == NULL)
-		return;
-
-	r_base = crtc->gamma_store;
-	g_base = r_base + crtc->gamma_size;
-	b_base = g_base + crtc->gamma_size;
-
-	crtc->funcs->gamma_set(crtc, r_base, g_base, b_base,
-			       crtc->gamma_size, NULL);
-}
-
-/**
- * drm_fb_helper_debug_enter - implementation for &fb_ops.fb_debug_enter
- * @info: fbdev registered by the helper
- */
-int drm_fb_helper_debug_enter(struct fb_info *info)
-{
-	struct drm_fb_helper *helper = info->par;
-	const struct drm_crtc_helper_funcs *funcs;
-	struct drm_mode_set *mode_set;
-
-	list_for_each_entry(helper, &kernel_fb_helper_list, kernel_fb_list) {
-		mutex_lock(&helper->client.modeset_mutex);
-		drm_client_for_each_modeset(mode_set, &helper->client) {
-			if (!mode_set->crtc->enabled)
-				continue;
-
-			funcs =	mode_set->crtc->helper_private;
-			if (funcs->mode_set_base_atomic == NULL)
-				continue;
-
-			if (drm_drv_uses_atomic_modeset(mode_set->crtc->dev))
-				continue;
-
-			funcs->mode_set_base_atomic(mode_set->crtc,
-						    mode_set->fb,
-						    mode_set->x,
-						    mode_set->y,
-						    ENTER_ATOMIC_MODE_SET);
-		}
-		mutex_unlock(&helper->client.modeset_mutex);
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL(drm_fb_helper_debug_enter);
-
-/**
- * drm_fb_helper_debug_leave - implementation for &fb_ops.fb_debug_leave
- * @info: fbdev registered by the helper
- */
-int drm_fb_helper_debug_leave(struct fb_info *info)
-{
-	struct drm_fb_helper *helper = info->par;
-	struct drm_client_dev *client = &helper->client;
-	struct drm_device *dev = helper->dev;
-	struct drm_crtc *crtc;
-	const struct drm_crtc_helper_funcs *funcs;
-	struct drm_mode_set *mode_set;
-	struct drm_framebuffer *fb;
-
-	mutex_lock(&client->modeset_mutex);
-	drm_client_for_each_modeset(mode_set, client) {
-		crtc = mode_set->crtc;
-		if (drm_drv_uses_atomic_modeset(crtc->dev))
-			continue;
-
-		funcs = crtc->helper_private;
-		fb = crtc->primary->fb;
-
-		if (!crtc->enabled)
-			continue;
-
-		if (!fb) {
-			drm_err(dev, "no fb to restore?\n");
-			continue;
-		}
-
-		if (funcs->mode_set_base_atomic == NULL)
-			continue;
-
-		drm_fb_helper_restore_lut_atomic(mode_set->crtc);
-		funcs->mode_set_base_atomic(mode_set->crtc, fb, crtc->x,
-					    crtc->y, LEAVE_ATOMIC_MODE_SET);
-	}
-	mutex_unlock(&client->modeset_mutex);
-
-	return 0;
-}
-EXPORT_SYMBOL(drm_fb_helper_debug_leave);
-
 static int
 __drm_fb_helper_restore_fbdev_mode_unlocked(struct drm_fb_helper *fb_helper,
 					    bool force)
@@ -397,7 +299,6 @@ void drm_fb_helper_prepare(struct drm_device *dev, struct drm_fb_helper *helper,
 	if (!preferred_bpp)
 		preferred_bpp = 32;
 
-	INIT_LIST_HEAD(&helper->kernel_fb_list);
 	spin_lock_init(&helper->damage_lock);
 	INIT_WORK(&helper->resume_work, drm_fb_helper_resume_worker);
 	INIT_WORK(&helper->damage_work, drm_fb_helper_damage_work);
@@ -534,11 +435,6 @@ void drm_fb_helper_fini(struct drm_fb_helper *fb_helper)
 
 	drm_fb_helper_release_info(fb_helper);
 
-	mutex_lock(&kernel_fb_helper_lock);
-	if (!list_empty(&fb_helper->kernel_fb_list))
-		list_del(&fb_helper->kernel_fb_list);
-	mutex_unlock(&kernel_fb_helper_lock);
-
 	if (!fb_helper->client.funcs)
 		drm_client_release(&fb_helper->client);
 }
@@ -1766,10 +1662,6 @@ __drm_fb_helper_initial_config_and_unlock(struct drm_fb_helper *fb_helper)
 	drm_info(dev, "fb%d: %s frame buffer device\n",
 		 info->node, info->fix.id);
 
-	mutex_lock(&kernel_fb_helper_lock);
-	list_add(&fb_helper->kernel_fb_list, &kernel_fb_helper_list);
-	mutex_unlock(&kernel_fb_helper_lock);
-
 	return 0;
 
 err_drm_fb_helper_release_info:
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index efc79bbf3c73..28d6792112be 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -29,6 +29,9 @@
 #include <linux/export.h>
 #include <linux/file.h>
 #include <linux/fs.h>
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+#include <linux/fs_context.h>
+#endif
 #include <linux/iosys-map.h>
 #include <linux/mem_encrypt.h>
 #include <linux/mm.h>
@@ -36,6 +39,7 @@
 #include <linux/module.h>
 #include <linux/pagemap.h>
 #include <linux/pagevec.h>
+#include <linux/sched/mm.h>
 #include <linux/shmem_fs.h>
 #include <linux/slab.h>
 #include <linux/string_helpers.h>
@@ -81,6 +85,60 @@
  * up at a later date, and as our interface with shmfs for memory allocation.
  */
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+static void drm_gem_huge_mnt_free(struct drm_device *dev, void *data)
+{
+	kern_unmount(dev->huge_mnt);
+}
+
+/**
+ * drm_gem_huge_mnt_create - Create, mount and use a huge tmpfs mountpoint
+ * @dev: DRM device that will use the huge tmpfs mountpoint
+ * @value: huge tmpfs mount option value
+ *
+ * This function creates and mounts a dedicated huge tmpfs mountpoint for the
+ * lifetime of the DRM device @dev which is used at GEM object initialization
+ * with drm_gem_object_init().
+ *
+ * The most common option for @value is "within_size" which only allocates huge
+ * pages if the page will be fully within the GEM object size. "always",
+ * "advise" and "never" are supported too but the latter would just create a
+ * mountpoint similar to the default one (`shm_mnt`). See shmemfs and
+ * Transparent Hugepage for more information.
+ *
+ * Returns:
+ * 0 on success or a negative error code on failure.
+ */
+int drm_gem_huge_mnt_create(struct drm_device *dev, const char *value)
+{
+	struct file_system_type *type;
+	struct fs_context *fc;
+	int ret;
+
+	if (unlikely(drm_gem_get_huge_mnt(dev)))
+		return 0;
+
+	type = get_fs_type("tmpfs");
+	if (unlikely(!type))
+		return -EOPNOTSUPP;
+	fc = fs_context_for_mount(type, SB_KERNMOUNT);
+	if (IS_ERR(fc))
+		return PTR_ERR(fc);
+	ret = vfs_parse_fs_string(fc, "source", "tmpfs");
+	if (unlikely(ret))
+		return -ENOPARAM;
+	ret = vfs_parse_fs_string(fc, "huge", value);
+	if (unlikely(ret))
+		return -ENOPARAM;
+
+	dev->huge_mnt = fc_mount_longterm(fc);
+	put_fs_context(fc);
+
+	return drmm_add_action_or_reset(dev, drm_gem_huge_mnt_free, NULL);
+}
+EXPORT_SYMBOL_GPL(drm_gem_huge_mnt_create);
+#endif
+
 static void
 drm_gem_init_release(struct drm_device *dev, void *ptr)
 {
@@ -113,29 +171,28 @@ drm_gem_init(struct drm_device *dev)
 }
 
 /**
- * drm_gem_object_init_with_mnt - initialize an allocated shmem-backed GEM
- * object in a given shmfs mountpoint
+ * drm_gem_object_init - initialize an allocated shmem-backed GEM object
  *
  * @dev: drm_device the object should be initialized for
  * @obj: drm_gem_object to initialize
  * @size: object size
- * @gemfs: tmpfs mount where the GEM object will be created. If NULL, use
- * the usual tmpfs mountpoint (`shm_mnt`).
  *
  * Initialize an already allocated GEM object of the specified size with
- * shmfs backing store.
+ * shmfs backing store. A huge mountpoint can be used by calling
+ * drm_gem_huge_mnt_create() beforehand.
  */
-int drm_gem_object_init_with_mnt(struct drm_device *dev,
-				 struct drm_gem_object *obj, size_t size,
-				 struct vfsmount *gemfs)
+int drm_gem_object_init(struct drm_device *dev, struct drm_gem_object *obj,
+			size_t size)
 {
+	struct vfsmount *huge_mnt;
 	struct file *filp;
 
 	drm_gem_private_object_init(dev, obj, size);
 
-	if (gemfs)
-		filp = shmem_file_setup_with_mnt(gemfs, "drm mm object", size,
-						 VM_NORESERVE);
+	huge_mnt = drm_gem_get_huge_mnt(dev);
+	if (huge_mnt)
+		filp = shmem_file_setup_with_mnt(huge_mnt, "drm mm object",
+						 size, VM_NORESERVE);
 	else
 		filp = shmem_file_setup("drm mm object", size, VM_NORESERVE);
 
@@ -146,22 +203,6 @@ int drm_gem_object_init_with_mnt(struct drm_device *dev,
 
 	return 0;
 }
-EXPORT_SYMBOL(drm_gem_object_init_with_mnt);
-
-/**
- * drm_gem_object_init - initialize an allocated shmem-backed GEM object
- * @dev: drm_device the object should be initialized for
- * @obj: drm_gem_object to initialize
- * @size: object size
- *
- * Initialize an already allocated GEM object of the specified size with
- * shmfs backing store.
- */
-int drm_gem_object_init(struct drm_device *dev, struct drm_gem_object *obj,
-			size_t size)
-{
-	return drm_gem_object_init_with_mnt(dev, obj, size, NULL);
-}
 EXPORT_SYMBOL(drm_gem_object_init);
 
 /**
@@ -969,8 +1010,10 @@ int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data,
 	if (!obj)
 		return -ENOENT;
 
-	if (args->handle == args->new_handle)
-		return 0;
+	if (args->handle == args->new_handle) {
+		ret = 0;
+		goto out;
+	}
 
 	mutex_lock(&file_priv->prime.lock);
 
@@ -1002,6 +1045,8 @@ int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data,
 
 out_unlock:
 	mutex_unlock(&file_priv->prime.lock);
+out:
+	drm_gem_object_put(obj);
 
 	return ret;
 }
@@ -1177,36 +1222,27 @@ int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size,
 }
 EXPORT_SYMBOL(drm_gem_mmap_obj);
 
-/**
- * drm_gem_mmap - memory map routine for GEM objects
- * @filp: DRM file pointer
- * @vma: VMA for the area to be mapped
- *
- * If a driver supports GEM object mapping, mmap calls on the DRM file
- * descriptor will end up here.
- *
- * Look up the GEM object based on the offset passed in (vma->vm_pgoff will
- * contain the fake offset we created when the GTT map ioctl was called on
- * the object) and map it with a call to drm_gem_mmap_obj().
- *
- * If the caller is not granted access to the buffer object, the mmap will fail
- * with EACCES. Please see the vma manager for more information.
+/*
+ * Look up a GEM object in offset space based on the exact start address. The
+ * caller must be granted access to the object. Returns a GEM object on success
+ * or a negative error code on failure. The returned GEM object needs to be
+ * released with drm_gem_object_put().
  */
-int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
+static struct drm_gem_object *
+drm_gem_object_lookup_at_offset(struct file *filp, unsigned long start,
+				unsigned long pages)
 {
 	struct drm_file *priv = filp->private_data;
 	struct drm_device *dev = priv->minor->dev;
 	struct drm_gem_object *obj = NULL;
 	struct drm_vma_offset_node *node;
-	int ret;
 
 	if (drm_dev_is_unplugged(dev))
-		return -ENODEV;
+		return ERR_PTR(-ENODEV);
 
 	drm_vma_offset_lock_lookup(dev->vma_offset_manager);
 	node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager,
-						  vma->vm_pgoff,
-						  vma_pages(vma));
+						  start, pages);
 	if (likely(node)) {
 		obj = container_of(node, struct drm_gem_object, vma_node);
 		/*
@@ -1225,14 +1261,87 @@ int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
 	drm_vma_offset_unlock_lookup(dev->vma_offset_manager);
 
 	if (!obj)
-		return -EINVAL;
+		return ERR_PTR(-EINVAL);
 
 	if (!drm_vma_node_is_allowed(node, priv)) {
 		drm_gem_object_put(obj);
-		return -EACCES;
+		return ERR_PTR(-EACCES);
 	}
 
-	ret = drm_gem_mmap_obj(obj, drm_vma_node_size(node) << PAGE_SHIFT,
+	return obj;
+}
+
+#ifdef CONFIG_MMU
+/**
+ * drm_gem_get_unmapped_area - get memory mapping region routine for GEM objects
+ * @filp: DRM file pointer
+ * @uaddr: User address hint
+ * @len: Mapping length
+ * @pgoff: Offset (in pages)
+ * @flags: Mapping flags
+ *
+ * If a driver supports GEM object mapping, before ending up in drm_gem_mmap(),
+ * mmap calls on the DRM file descriptor will first try to find a free linear
+ * address space large enough for a mapping. Since GEM objects are backed by
+ * shmem buffers, this should preferably be handled by the shmem virtual memory
+ * filesystem which can appropriately align addresses to huge page sizes when
+ * needed.
+ *
+ * Look up the GEM object based on the offset passed in (vma->vm_pgoff will
+ * contain the fake offset we created) and call shmem_get_unmapped_area() with
+ * the right file pointer.
+ *
+ * If a GEM object is not available at the given offset or if the caller is not
+ * granted access to it, fall back to mm_get_unmapped_area().
+ */
+unsigned long drm_gem_get_unmapped_area(struct file *filp, unsigned long uaddr,
+					unsigned long len, unsigned long pgoff,
+					unsigned long flags)
+{
+	struct drm_gem_object *obj;
+	unsigned long ret;
+
+	obj = drm_gem_object_lookup_at_offset(filp, pgoff, len >> PAGE_SHIFT);
+	if (IS_ERR(obj) || !obj->filp || !obj->filp->f_op->get_unmapped_area)
+		return mm_get_unmapped_area(filp, uaddr, len, 0, flags);
+
+	ret = obj->filp->f_op->get_unmapped_area(obj->filp, uaddr, len, 0,
+						 flags);
+
+	drm_gem_object_put(obj);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(drm_gem_get_unmapped_area);
+#endif
+
+/**
+ * drm_gem_mmap - memory map routine for GEM objects
+ * @filp: DRM file pointer
+ * @vma: VMA for the area to be mapped
+ *
+ * If a driver supports GEM object mapping, mmap calls on the DRM file
+ * descriptor will end up here.
+ *
+ * Look up the GEM object based on the offset passed in (vma->vm_pgoff will
+ * contain the fake offset we created) and map it with a call to
+ * drm_gem_mmap_obj().
+ *
+ * If the caller is not granted access to the buffer object, the mmap will fail
+ * with EACCES. Please see the vma manager for more information.
+ */
+int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct drm_gem_object *obj;
+	int ret;
+
+	obj = drm_gem_object_lookup_at_offset(filp, vma->vm_pgoff,
+					      vma_pages(vma));
+	if (IS_ERR(obj))
+		return PTR_ERR(obj);
+
+	ret = drm_gem_mmap_obj(obj,
+			       drm_vma_node_size(&obj->vma_node) << PAGE_SHIFT,
 			       vma);
 
 	drm_gem_object_put(obj);
diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
index 93b9cff89080..fbd1164174b0 100644
--- a/drivers/gpu/drm/drm_gem_shmem_helper.c
+++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
@@ -50,7 +50,7 @@ static const struct drm_gem_object_funcs drm_gem_shmem_funcs = {
 };
 
 static int __drm_gem_shmem_init(struct drm_device *dev, struct drm_gem_shmem_object *shmem,
-				size_t size, bool private, struct vfsmount *gemfs)
+				size_t size, bool private)
 {
 	struct drm_gem_object *obj = &shmem->base;
 	int ret = 0;
@@ -62,7 +62,7 @@ static int __drm_gem_shmem_init(struct drm_device *dev, struct drm_gem_shmem_obj
 		drm_gem_private_object_init(dev, obj, size);
 		shmem->map_wc = false; /* dma-buf mappings use always writecombine */
 	} else {
-		ret = drm_gem_object_init_with_mnt(dev, obj, size, gemfs);
+		ret = drm_gem_object_init(dev, obj, size);
 	}
 	if (ret) {
 		drm_gem_private_object_fini(obj);
@@ -94,22 +94,24 @@ static int __drm_gem_shmem_init(struct drm_device *dev, struct drm_gem_shmem_obj
 }
 
 /**
- * drm_gem_shmem_init - Initialize an allocated object.
+ * drm_gem_shmem_init - Initialize an allocated object of the given size
  * @dev: DRM device
- * @obj: The allocated shmem GEM object.
+ * @shmem: shmem GEM object to initialize
+ * @size: Size of the object to initialize
+ *
+ * This function initializes an allocated shmem GEM object.
  *
  * Returns:
  * 0 on success, or a negative error code on failure.
  */
 int drm_gem_shmem_init(struct drm_device *dev, struct drm_gem_shmem_object *shmem, size_t size)
 {
-	return __drm_gem_shmem_init(dev, shmem, size, false, NULL);
+	return __drm_gem_shmem_init(dev, shmem, size, false);
 }
 EXPORT_SYMBOL_GPL(drm_gem_shmem_init);
 
 static struct drm_gem_shmem_object *
-__drm_gem_shmem_create(struct drm_device *dev, size_t size, bool private,
-		       struct vfsmount *gemfs)
+__drm_gem_shmem_create(struct drm_device *dev, size_t size, bool private)
 {
 	struct drm_gem_shmem_object *shmem;
 	struct drm_gem_object *obj;
@@ -129,7 +131,7 @@ __drm_gem_shmem_create(struct drm_device *dev, size_t size, bool private,
 		obj = &shmem->base;
 	}
 
-	ret = __drm_gem_shmem_init(dev, shmem, size, private, gemfs);
+	ret = __drm_gem_shmem_init(dev, shmem, size, private);
 	if (ret) {
 		kfree(obj);
 		return ERR_PTR(ret);
@@ -150,31 +152,10 @@ __drm_gem_shmem_create(struct drm_device *dev, size_t size, bool private,
  */
 struct drm_gem_shmem_object *drm_gem_shmem_create(struct drm_device *dev, size_t size)
 {
-	return __drm_gem_shmem_create(dev, size, false, NULL);
+	return __drm_gem_shmem_create(dev, size, false);
 }
 EXPORT_SYMBOL_GPL(drm_gem_shmem_create);
 
-/**
- * drm_gem_shmem_create_with_mnt - Allocate an object with the given size in a
- * given mountpoint
- * @dev: DRM device
- * @size: Size of the object to allocate
- * @gemfs: tmpfs mount where the GEM object will be created
- *
- * This function creates a shmem GEM object in a given tmpfs mountpoint.
- *
- * Returns:
- * A struct drm_gem_shmem_object * on success or an ERR_PTR()-encoded negative
- * error code on failure.
- */
-struct drm_gem_shmem_object *drm_gem_shmem_create_with_mnt(struct drm_device *dev,
-							   size_t size,
-							   struct vfsmount *gemfs)
-{
-	return __drm_gem_shmem_create(dev, size, false, gemfs);
-}
-EXPORT_SYMBOL_GPL(drm_gem_shmem_create_with_mnt);
-
 /**
  * drm_gem_shmem_release - Release resources associated with a shmem GEM object.
  * @shmem: shmem GEM object
@@ -567,6 +548,26 @@ int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev,
 }
 EXPORT_SYMBOL_GPL(drm_gem_shmem_dumb_create);
 
+static bool drm_gem_shmem_try_map_pmd(struct vm_fault *vmf, unsigned long addr,
+				      struct page *page)
+{
+#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
+	unsigned long pfn = page_to_pfn(page);
+	unsigned long paddr = pfn << PAGE_SHIFT;
+	bool aligned = (addr & ~PMD_MASK) == (paddr & ~PMD_MASK);
+
+	if (aligned &&
+	    pmd_none(*vmf->pmd) &&
+	    folio_test_pmd_mappable(page_folio(page))) {
+		pfn &= PMD_MASK >> PAGE_SHIFT;
+		if (vmf_insert_pfn_pmd(vmf, pfn, false) == VM_FAULT_NOPAGE)
+			return true;
+	}
+#endif
+
+	return false;
+}
+
 static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
@@ -574,11 +575,12 @@ static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf)
 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
 	loff_t num_pages = obj->size >> PAGE_SHIFT;
 	vm_fault_t ret;
-	struct page *page;
+	struct page **pages = shmem->pages;
 	pgoff_t page_offset;
+	unsigned long pfn;
 
-	/* We don't use vmf->pgoff since that has the fake offset */
-	page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
+	/* Offset to faulty address in the VMA. */
+	page_offset = vmf->pgoff - vma->vm_pgoff;
 
 	dma_resv_lock(shmem->base.resv, NULL);
 
@@ -586,12 +588,18 @@ static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf)
 	    drm_WARN_ON_ONCE(obj->dev, !shmem->pages) ||
 	    shmem->madv < 0) {
 		ret = VM_FAULT_SIGBUS;
-	} else {
-		page = shmem->pages[page_offset];
+		goto out;
+	}
 
-		ret = vmf_insert_pfn(vma, vmf->address, page_to_pfn(page));
+	if (drm_gem_shmem_try_map_pmd(vmf, vmf->address, pages[page_offset])) {
+		ret = VM_FAULT_NOPAGE;
+		goto out;
 	}
 
+	pfn = page_to_pfn(pages[page_offset]);
+	ret = vmf_insert_pfn(vma, vmf->address, pfn);
+
+ out:
 	dma_resv_unlock(shmem->base.resv);
 
 	return ret;
@@ -824,7 +832,7 @@ drm_gem_shmem_prime_import_sg_table(struct drm_device *dev,
 	size_t size = PAGE_ALIGN(attach->dmabuf->size);
 	struct drm_gem_shmem_object *shmem;
 
-	shmem = __drm_gem_shmem_create(dev, size, true, NULL);
+	shmem = __drm_gem_shmem_create(dev, size, true);
 	if (IS_ERR(shmem))
 		return ERR_CAST(shmem);
 
@@ -872,7 +880,7 @@ struct drm_gem_object *drm_gem_shmem_prime_import_no_map(struct drm_device *dev,
 
 	size = PAGE_ALIGN(attach->dmabuf->size);
 
-	shmem = __drm_gem_shmem_create(dev, size, true, NULL);
+	shmem = __drm_gem_shmem_create(dev, size, true);
 	if (IS_ERR(shmem)) {
 		ret = PTR_ERR(shmem);
 		goto fail_detach;
diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c
index 73e550c8ff8c..39c8c50401dd 100644
--- a/drivers/gpu/drm/drm_gpusvm.c
+++ b/drivers/gpu/drm/drm_gpusvm.c
@@ -1288,6 +1288,9 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 							   DMA_BIDIRECTIONAL;
 
 retry:
+	if (time_after(jiffies, timeout))
+		return -EBUSY;
+
 	hmm_range.notifier_seq = mmu_interval_read_begin(notifier);
 	if (drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages))
 		goto set_seqno;
diff --git a/drivers/gpu/drm/drm_panic.c b/drivers/gpu/drm/drm_panic.c
index d4b6ea42db0f..f42be7f1d8c2 100644
--- a/drivers/gpu/drm/drm_panic.c
+++ b/drivers/gpu/drm/drm_panic.c
@@ -39,12 +39,6 @@ MODULE_AUTHOR("Jocelyn Falempe");
 MODULE_DESCRIPTION("DRM panic handler");
 MODULE_LICENSE("GPL");
 
-static char drm_panic_screen[16] = CONFIG_DRM_PANIC_SCREEN;
-module_param_string(panic_screen, drm_panic_screen, sizeof(drm_panic_screen), 0644);
-MODULE_PARM_DESC(panic_screen,
-		 "Choose what will be displayed by drm_panic, 'user' or 'kmsg' [default="
-		 CONFIG_DRM_PANIC_SCREEN "]");
-
 /**
  * DOC: overview
  *
@@ -813,14 +807,59 @@ static void draw_panic_static_qr_code(struct drm_scanout_buffer *sb)
 		draw_panic_static_user(sb);
 }
 #else
-static void draw_panic_static_qr_code(struct drm_scanout_buffer *sb)
+static void drm_panic_qr_init(void) {};
+static void drm_panic_qr_exit(void) {};
+#endif
+
+enum drm_panic_type {
+	DRM_PANIC_TYPE_KMSG,
+	DRM_PANIC_TYPE_USER,
+	DRM_PANIC_TYPE_QR,
+};
+
+static enum drm_panic_type drm_panic_type = -1;
+
+static const char *drm_panic_type_map[] = {
+	[DRM_PANIC_TYPE_KMSG] = "kmsg",
+	[DRM_PANIC_TYPE_USER] = "user",
+#if IS_ENABLED(CONFIG_DRM_PANIC_SCREEN_QR_CODE)
+	[DRM_PANIC_TYPE_QR] = "qr",
+#endif
+};
+
+static int drm_panic_type_set(const char *val, const struct kernel_param *kp)
 {
-	draw_panic_static_user(sb);
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(drm_panic_type_map); i++) {
+		if (!strcmp(val, drm_panic_type_map[i])) {
+			drm_panic_type = i;
+			return 0;
+		}
+	}
+
+	return -EINVAL;
 }
 
-static void drm_panic_qr_init(void) {};
-static void drm_panic_qr_exit(void) {};
+static int drm_panic_type_get(char *buffer, const struct kernel_param *kp)
+{
+	return scnprintf(buffer, PAGE_SIZE, "%s\n",
+			 drm_panic_type_map[drm_panic_type]);
+}
+
+static const struct kernel_param_ops drm_panic_ops = {
+	.set = drm_panic_type_set,
+	.get = drm_panic_type_get,
+};
+
+module_param_cb(panic_screen, &drm_panic_ops, NULL, 0644);
+MODULE_PARM_DESC(panic_screen,
+#if IS_ENABLED(CONFIG_DRM_PANIC_SCREEN_QR_CODE)
+		 "Choose what will be displayed by drm_panic, 'user', 'kmsg' or 'qr' [default="
+#else
+		 "Choose what will be displayed by drm_panic, 'user' or 'kmsg' [default="
 #endif
+		 CONFIG_DRM_PANIC_SCREEN "]");
 
 /*
  * drm_panic_is_format_supported()
@@ -838,11 +877,19 @@ static bool drm_panic_is_format_supported(const struct drm_format_info *format)
 
 static void draw_panic_dispatch(struct drm_scanout_buffer *sb)
 {
-	if (!strcmp(drm_panic_screen, "kmsg")) {
+	switch (drm_panic_type) {
+	case DRM_PANIC_TYPE_KMSG:
 		draw_panic_static_kmsg(sb);
-	} else if (!strcmp(drm_panic_screen, "qr_code")) {
+		break;
+
+#if IS_ENABLED(CONFIG_DRM_PANIC_SCREEN_QR_CODE)
+	case DRM_PANIC_TYPE_QR:
 		draw_panic_static_qr_code(sb);
-	} else {
+		break;
+#endif
+
+	case DRM_PANIC_TYPE_USER:
+	default:
 		draw_panic_static_user(sb);
 	}
 }
@@ -1025,6 +1072,8 @@ void drm_panic_unregister(struct drm_device *dev)
  */
 void __init drm_panic_init(void)
 {
+	if (drm_panic_type == -1)
+		drm_panic_type_set(CONFIG_DRM_PANIC_SCREEN, NULL);
 	drm_panic_qr_init();
 }
 
diff --git a/drivers/gpu/drm/drm_plane.c b/drivers/gpu/drm/drm_plane.c
index b143589717e6..bed2562bf911 100644
--- a/drivers/gpu/drm/drm_plane.c
+++ b/drivers/gpu/drm/drm_plane.c
@@ -1867,9 +1867,9 @@ int drm_plane_create_color_pipeline_property(struct drm_plane *plane,
 	prop = drm_property_create_enum(plane->dev, DRM_MODE_PROP_ATOMIC,
 					"COLOR_PIPELINE",
 					all_pipelines, len);
-	if (IS_ERR(prop)) {
+	if (!prop) {
 		kfree(all_pipelines);
-		return PTR_ERR(prop);
+		return -ENOMEM;
 	}
 
 	drm_object_attach_property(&plane->base, prop, 0);
diff --git a/drivers/gpu/drm/drm_syncobj.c b/drivers/gpu/drm/drm_syncobj.c
index e1b0fa4000cd..2d4ab745fdad 100644
--- a/drivers/gpu/drm/drm_syncobj.c
+++ b/drivers/gpu/drm/drm_syncobj.c
@@ -250,14 +250,14 @@ struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
 {
 	struct drm_syncobj *syncobj;
 
-	spin_lock(&file_private->syncobj_table_lock);
+	xa_lock(&file_private->syncobj_xa);
 
 	/* Check if we currently have a reference on the object */
-	syncobj = idr_find(&file_private->syncobj_idr, handle);
+	syncobj = xa_load(&file_private->syncobj_xa, handle);
 	if (syncobj)
 		drm_syncobj_get(syncobj);
 
-	spin_unlock(&file_private->syncobj_table_lock);
+	xa_unlock(&file_private->syncobj_xa);
 
 	return syncobj;
 }
@@ -598,23 +598,15 @@ int drm_syncobj_get_handle(struct drm_file *file_private,
 {
 	int ret;
 
-	/* take a reference to put in the idr */
+	/* take a reference to put in the xarray */
 	drm_syncobj_get(syncobj);
 
-	idr_preload(GFP_KERNEL);
-	spin_lock(&file_private->syncobj_table_lock);
-	ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
-	spin_unlock(&file_private->syncobj_table_lock);
-
-	idr_preload_end();
-
-	if (ret < 0) {
+	ret = xa_alloc(&file_private->syncobj_xa, handle, syncobj, xa_limit_32b,
+		       GFP_NOWAIT);
+	if (ret)
 		drm_syncobj_put(syncobj);
-		return ret;
-	}
 
-	*handle = ret;
-	return 0;
+	return ret;
 }
 EXPORT_SYMBOL(drm_syncobj_get_handle);
 
@@ -638,10 +630,7 @@ static int drm_syncobj_destroy(struct drm_file *file_private,
 {
 	struct drm_syncobj *syncobj;
 
-	spin_lock(&file_private->syncobj_table_lock);
-	syncobj = idr_remove(&file_private->syncobj_idr, handle);
-	spin_unlock(&file_private->syncobj_table_lock);
-
+	syncobj = xa_erase(&file_private->syncobj_xa, handle);
 	if (!syncobj)
 		return -EINVAL;
 
@@ -722,20 +711,13 @@ static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
 	if (fd_file(f)->f_op != &drm_syncobj_file_fops)
 		return -EINVAL;
 
-	/* take a reference to put in the idr */
+	/* take a reference to put in the xarray */
 	syncobj = fd_file(f)->private_data;
 	drm_syncobj_get(syncobj);
 
-	idr_preload(GFP_KERNEL);
-	spin_lock(&file_private->syncobj_table_lock);
-	ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
-	spin_unlock(&file_private->syncobj_table_lock);
-	idr_preload_end();
-
-	if (ret > 0) {
-		*handle = ret;
-		ret = 0;
-	} else
+	ret = xa_alloc(&file_private->syncobj_xa, handle, syncobj, xa_limit_32b,
+		       GFP_NOWAIT);
+	if (ret)
 		drm_syncobj_put(syncobj);
 
 	return ret;
@@ -814,17 +796,7 @@ static int drm_syncobj_export_sync_file(struct drm_file *file_private,
 void
 drm_syncobj_open(struct drm_file *file_private)
 {
-	idr_init_base(&file_private->syncobj_idr, 1);
-	spin_lock_init(&file_private->syncobj_table_lock);
-}
-
-static int
-drm_syncobj_release_handle(int id, void *ptr, void *data)
-{
-	struct drm_syncobj *syncobj = ptr;
-
-	drm_syncobj_put(syncobj);
-	return 0;
+	xa_init_flags(&file_private->syncobj_xa, XA_FLAGS_ALLOC1);
 }
 
 /**
@@ -838,9 +810,12 @@ drm_syncobj_release_handle(int id, void *ptr, void *data)
 void
 drm_syncobj_release(struct drm_file *file_private)
 {
-	idr_for_each(&file_private->syncobj_idr,
-		     &drm_syncobj_release_handle, file_private);
-	idr_destroy(&file_private->syncobj_idr);
+	struct drm_syncobj *syncobj;
+	unsigned long handle;
+
+	xa_for_each(&file_private->syncobj_xa, handle, syncobj)
+		drm_syncobj_put(syncobj);
+	xa_destroy(&file_private->syncobj_xa);
 }
 
 int
diff --git a/drivers/gpu/drm/drm_vblank.c b/drivers/gpu/drm/drm_vblank.c
index 5c14140cd0c2..42fe11cc139b 100644
--- a/drivers/gpu/drm/drm_vblank.c
+++ b/drivers/gpu/drm/drm_vblank.c
@@ -551,7 +551,7 @@ int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
 	dev->num_crtcs = num_crtcs;
 
 	for (i = 0; i < num_crtcs; i++) {
-		struct drm_vblank_crtc *vblank = &dev->vblank[i];
+		struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, i);
 
 		vblank->dev = dev;
 		vblank->pipe = i;
@@ -605,7 +605,9 @@ EXPORT_SYMBOL(drm_dev_has_vblank);
  */
 wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
 {
-	return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
+	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
+
+	return &vblank->queue;
 }
 EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
 
@@ -710,7 +712,6 @@ drm_crtc_vblank_helper_get_vblank_timestamp_internal(
 {
 	struct drm_device *dev = crtc->dev;
 	unsigned int pipe = crtc->index;
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 	struct timespec64 ts_etime, ts_vblank_time;
 	ktime_t stime, etime;
 	bool vbl_status;
@@ -729,10 +730,13 @@ drm_crtc_vblank_helper_get_vblank_timestamp_internal(
 		return false;
 	}
 
-	if (drm_drv_uses_atomic_modeset(dev))
+	if (drm_drv_uses_atomic_modeset(dev)) {
+		struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
+
 		mode = &vblank->hwmode;
-	else
+	} else {
 		mode = &crtc->hwmode;
+	}
 
 	/* If mode timing undefined, just return as no-op:
 	 * Happens during initial modesetting of a crtc.
@@ -1285,29 +1289,27 @@ void drm_crtc_vblank_put(struct drm_crtc *crtc)
 EXPORT_SYMBOL(drm_crtc_vblank_put);
 
 /**
- * drm_wait_one_vblank - wait for one vblank
- * @dev: DRM device
- * @pipe: CRTC index
+ * drm_crtc_wait_one_vblank - wait for one vblank
+ * @crtc: DRM crtc
  *
- * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
- * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
+ * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
+ * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
  * due to lack of driver support or because the crtc is off.
  *
- * This is the legacy version of drm_crtc_wait_one_vblank().
+ * Returns: 0 on success, negative error on failures.
  */
-void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
+int drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
 {
-	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
+	struct drm_device *dev = crtc->dev;
+	int pipe = drm_crtc_index(crtc);
+	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
 	int ret;
 	u64 last;
 
-	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
-		return;
-
 	ret = drm_vblank_get(dev, pipe);
 	if (drm_WARN(dev, ret, "vblank not available on crtc %i, ret=%i\n",
 		     pipe, ret))
-		return;
+		return ret;
 
 	last = drm_vblank_count(dev, pipe);
 
@@ -1318,20 +1320,8 @@ void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
 	drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe);
 
 	drm_vblank_put(dev, pipe);
-}
-EXPORT_SYMBOL(drm_wait_one_vblank);
 
-/**
- * drm_crtc_wait_one_vblank - wait for one vblank
- * @crtc: DRM crtc
- *
- * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
- * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
- * due to lack of driver support or because the crtc is off.
- */
-void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
-{
-	drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
+	return ret ? 0 : -ETIMEDOUT;
 }
 EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
 
@@ -1794,7 +1784,7 @@ int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
 	if (pipe >= dev->num_crtcs)
 		return -EINVAL;
 
-	vblank = &dev->vblank[pipe];
+	vblank = drm_vblank_crtc(dev, pipe);
 
 	/* If the counter is currently enabled and accurate, short-circuit
 	 * queries to return the cached timestamp of the last vblank.
diff --git a/drivers/gpu/drm/hyperv/Kconfig b/drivers/gpu/drm/hyperv/Kconfig
new file mode 100644
index 000000000000..86234f6a73f2
--- /dev/null
+++ b/drivers/gpu/drm/hyperv/Kconfig
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config DRM_HYPERV
+	tristate "DRM Support for Hyper-V synthetic video device"
+	depends on DRM && PCI && HYPERV_VMBUS
+	select DRM_CLIENT_SELECTION
+	select DRM_KMS_HELPER
+	select DRM_GEM_SHMEM_HELPER
+	help
+	 This is a KMS driver for Hyper-V synthetic video device. Choose this
+	 option if you would like to enable drm driver for Hyper-V virtual
+	 machine. Unselect Hyper-V framebuffer driver (CONFIG_FB_HYPERV) so
+	 that DRM driver is used by default.
+
+	 If M is selected the module will be called hyperv_drm.
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 4db24050edb0..f01b5d8a07c7 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -27,7 +27,10 @@ i915-y += \
 	i915_config.o \
 	i915_driver.o \
 	i915_drm_client.o \
+	i915_edram.o \
+	i915_freq.o \
 	i915_getparam.o \
+	i915_gmch.o \
 	i915_ioctl.o \
 	i915_irq.o \
 	i915_mitigations.o \
@@ -54,12 +57,6 @@ i915-y += \
 	vlv_iosf_sb.o \
 	vlv_suspend.o
 
-# core peripheral code
-i915-y += \
-	soc/intel_dram.o \
-	soc/intel_gmch.o \
-	soc/intel_rom.o
-
 # core library code
 i915-y += \
 	i915_memcpy.o \
@@ -77,6 +74,11 @@ i915-$(CONFIG_DEBUG_FS) += \
 i915-$(CONFIG_PERF_EVENTS) += \
 	i915_pmu.o
 
+# core display adaptation
+i915-y += \
+	i915_hdcp_gsc.o \
+	i915_panic.o
+
 # "Graphics Technology" (aka we talk to the gpu)
 gt-y += \
 	gt/gen2_engine_cs.o \
@@ -170,8 +172,7 @@ gem-y += \
 	gem/i915_gem_ttm_move.o \
 	gem/i915_gem_ttm_pm.o \
 	gem/i915_gem_userptr.o \
-	gem/i915_gem_wait.o \
-	gem/i915_gemfs.o
+	gem/i915_gem_wait.o
 i915-y += \
 	$(gem-y) \
 	i915_active.o \
@@ -268,6 +269,7 @@ i915-y += \
 	display/intel_dpll_mgr.o \
 	display/intel_dpt.o \
 	display/intel_dpt_common.o \
+	display/intel_dram.o \
 	display/intel_drrs.o \
 	display/intel_dsb.o \
 	display/intel_dsb_buffer.o \
@@ -281,7 +283,6 @@ i915-y += \
 	display/intel_frontbuffer.o \
 	display/intel_global_state.o \
 	display/intel_hdcp.o \
-	display/intel_hdcp_gsc.o \
 	display/intel_hdcp_gsc_message.o \
 	display/intel_hotplug.o \
 	display/intel_hotplug_irq.o \
@@ -293,7 +294,7 @@ i915-y += \
 	display/intel_modeset_setup.o \
 	display/intel_modeset_verify.o \
 	display/intel_overlay.o \
-	display/intel_panic.o \
+	display/intel_parent.o \
 	display/intel_pch.o \
 	display/intel_pch_display.o \
 	display/intel_pch_refclk.o \
@@ -302,6 +303,7 @@ i915-y += \
 	display/intel_pmdemand.o \
 	display/intel_psr.o \
 	display/intel_quirks.o \
+	display/intel_rom.o \
 	display/intel_sbi.o \
 	display/intel_sprite.o \
 	display/intel_sprite_uapi.o \
diff --git a/drivers/gpu/drm/i915/display/g4x_dp.c b/drivers/gpu/drm/i915/display/g4x_dp.c
index a3ff21b2f69f..4cb753177fd8 100644
--- a/drivers/gpu/drm/i915/display/g4x_dp.c
+++ b/drivers/gpu/drm/i915/display/g4x_dp.c
@@ -302,7 +302,7 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
 {
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret;
 
 	wakeref = intel_display_power_get_if_enabled(display,
@@ -684,12 +684,11 @@ static void intel_enable_dp(struct intel_atomic_state *state,
 	struct intel_display *display = to_intel_display(state);
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 	u32 dp_reg = intel_de_read(display, intel_dp->output_reg);
-	intel_wakeref_t wakeref;
 
 	if (drm_WARN_ON(display->drm, dp_reg & DP_PORT_EN))
 		return;
 
-	with_intel_pps_lock(intel_dp, wakeref) {
+	with_intel_pps_lock(intel_dp) {
 		if (display->platform.valleyview || display->platform.cherryview)
 			vlv_pps_port_enable_unlocked(encoder, pipe_config);
 
diff --git a/drivers/gpu/drm/i915/display/g4x_hdmi.c b/drivers/gpu/drm/i915/display/g4x_hdmi.c
index f6e2d1ed5639..8b22447e8e23 100644
--- a/drivers/gpu/drm/i915/display/g4x_hdmi.c
+++ b/drivers/gpu/drm/i915/display/g4x_hdmi.c
@@ -68,7 +68,7 @@ static bool intel_hdmi_get_hw_state(struct intel_encoder *encoder,
 {
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret;
 
 	wakeref = intel_display_power_get_if_enabled(display,
diff --git a/drivers/gpu/drm/i915/display/i9xx_plane.c b/drivers/gpu/drm/i915/display/i9xx_plane.c
index 51ccc6bd5f21..b1fecf178906 100644
--- a/drivers/gpu/drm/i915/display/i9xx_plane.c
+++ b/drivers/gpu/drm/i915/display/i9xx_plane.c
@@ -22,7 +22,6 @@
 #include "intel_fb.h"
 #include "intel_fbc.h"
 #include "intel_frontbuffer.h"
-#include "intel_panic.h"
 #include "intel_plane.h"
 #include "intel_sprite.h"
 
@@ -134,7 +133,7 @@ static struct intel_fbc *i9xx_plane_fbc(struct intel_display *display,
 					enum i9xx_plane_id i9xx_plane)
 {
 	if (i9xx_plane_has_fbc(display, i9xx_plane))
-		return display->fbc[INTEL_FBC_A];
+		return display->fbc.instances[INTEL_FBC_A];
 	else
 		return NULL;
 }
@@ -724,7 +723,7 @@ static bool i9xx_plane_get_hw_state(struct intel_plane *plane,
 	struct intel_display *display = to_intel_display(plane);
 	enum intel_display_power_domain power_domain;
 	enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret;
 	u32 val;
 
@@ -819,7 +818,7 @@ unsigned int vlv_plane_min_alignment(struct intel_plane *plane,
 {
 	struct intel_display *display = to_intel_display(plane);
 
-	if (intel_plane_can_async_flip(plane, fb->format->format, fb->modifier))
+	if (intel_plane_can_async_flip(plane, fb->format, fb->modifier))
 		return 256 * 1024;
 
 	/* FIXME undocumented so not sure what's actually needed */
@@ -843,7 +842,7 @@ static unsigned int g4x_primary_min_alignment(struct intel_plane *plane,
 {
 	struct intel_display *display = to_intel_display(plane);
 
-	if (intel_plane_can_async_flip(plane, fb->format->format, fb->modifier))
+	if (intel_plane_can_async_flip(plane, fb->format, fb->modifier))
 		return 256 * 1024;
 
 	if (intel_scanout_needs_vtd_wa(display))
diff --git a/drivers/gpu/drm/i915/display/i9xx_wm.c b/drivers/gpu/drm/i915/display/i9xx_wm.c
index 01f3803fa09f..167277cd8877 100644
--- a/drivers/gpu/drm/i915/display/i9xx_wm.c
+++ b/drivers/gpu/drm/i915/display/i9xx_wm.c
@@ -7,8 +7,6 @@
 
 #include <drm/drm_print.h>
 
-#include "soc/intel_dram.h"
-
 #include "i915_drv.h"
 #include "i915_reg.h"
 #include "i9xx_wm.h"
@@ -19,6 +17,7 @@
 #include "intel_display.h"
 #include "intel_display_regs.h"
 #include "intel_display_trace.h"
+#include "intel_dram.h"
 #include "intel_fb.h"
 #include "intel_mchbar_regs.h"
 #include "intel_wm.h"
@@ -91,7 +90,7 @@ static const struct cxsr_latency cxsr_latency_table[] = {
 
 static const struct cxsr_latency *pnv_get_cxsr_latency(struct intel_display *display)
 {
-	const struct dram_info *dram_info = intel_dram_info(display->drm);
+	const struct dram_info *dram_info = intel_dram_info(display);
 	bool is_ddr3 = dram_info->type == INTEL_DRAM_DDR3;
 	int i;
 
diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c
index 9230792960f2..dac781f54661 100644
--- a/drivers/gpu/drm/i915/display/icl_dsi.c
+++ b/drivers/gpu/drm/i915/display/icl_dsi.c
@@ -1411,7 +1411,7 @@ static void gen11_dsi_disable_io_power(struct intel_encoder *encoder)
 	enum port port;
 
 	for_each_dsi_port(port, intel_dsi->ports) {
-		intel_wakeref_t wakeref;
+		struct ref_tracker *wakeref;
 
 		wakeref = fetch_and_zero(&intel_dsi->io_wakeref[port]);
 		intel_display_power_put(display,
@@ -1722,7 +1722,7 @@ static bool gen11_dsi_get_hw_state(struct intel_encoder *encoder,
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	enum transcoder dsi_trans;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	enum port port;
 	bool ret = false;
 	u32 tmp;
diff --git a/drivers/gpu/drm/i915/display/intel_alpm.c b/drivers/gpu/drm/i915/display/intel_alpm.c
index 6372f533f65b..7ce8c674bb03 100644
--- a/drivers/gpu/drm/i915/display/intel_alpm.c
+++ b/drivers/gpu/drm/i915/display/intel_alpm.c
@@ -326,11 +326,9 @@ static void lnl_alpm_configure(struct intel_dp *intel_dp,
 		if (intel_dp->as_sdp_supported) {
 			u32 pr_alpm_ctl = PR_ALPM_CTL_ADAPTIVE_SYNC_SDP_POSITION_T1;
 
-			if (intel_dp->pr_dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_CAPABILITY)] &
-			    DP_PANEL_REPLAY_LINK_OFF_SUPPORTED_IN_PR_AFTER_ADAPTIVE_SYNC_SDP)
+			if (crtc_state->link_off_after_as_sdp_when_pr_active)
 				pr_alpm_ctl |= PR_ALPM_CTL_ALLOW_LINK_OFF_BETWEEN_AS_SDP_AND_SU;
-			if (!(intel_dp->pr_dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_CAPABILITY)] &
-						DP_PANEL_REPLAY_ASYNC_VIDEO_TIMING_NOT_SUPPORTED_IN_PR))
+			if (crtc_state->disable_as_sdp_when_pr_active)
 				pr_alpm_ctl |= PR_ALPM_CTL_AS_SDP_TRANSMISSION_IN_ACTIVE_DISABLE;
 
 			intel_de_write(display, PR_ALPM_CTL(display, cpu_transcoder),
diff --git a/drivers/gpu/drm/i915/display/intel_audio.c b/drivers/gpu/drm/i915/display/intel_audio.c
index 5bdaef38f13d..5f3c175afdd2 100644
--- a/drivers/gpu/drm/i915/display/intel_audio.c
+++ b/drivers/gpu/drm/i915/display/intel_audio.c
@@ -1042,10 +1042,10 @@ int intel_audio_min_cdclk(const struct intel_crtc_state *crtc_state)
 static unsigned long intel_audio_component_get_power(struct device *kdev)
 {
 	struct intel_display *display = to_intel_display(kdev);
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 
 	/* Catch potential impedance mismatches before they occur! */
-	BUILD_BUG_ON(sizeof(intel_wakeref_t) > sizeof(unsigned long));
+	BUILD_BUG_ON(sizeof(wakeref) > sizeof(unsigned long));
 
 	wakeref = intel_display_power_get(display, POWER_DOMAIN_AUDIO_PLAYBACK);
 
@@ -1074,7 +1074,7 @@ static void intel_audio_component_put_power(struct device *kdev,
 					    unsigned long cookie)
 {
 	struct intel_display *display = to_intel_display(kdev);
-	intel_wakeref_t wakeref = (intel_wakeref_t)cookie;
+	struct ref_tracker *wakeref = (struct ref_tracker *)cookie;
 
 	/* Stop forcing CDCLK to 2*BCLK if no need for audio to be powered. */
 	if (--display->audio.power_refcount == 0)
diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c
index 4b41068e9e35..ae0b922d5bc3 100644
--- a/drivers/gpu/drm/i915/display/intel_bios.c
+++ b/drivers/gpu/drm/i915/display/intel_bios.c
@@ -34,14 +34,13 @@
 #include <drm/drm_fixed.h>
 #include <drm/drm_print.h>
 
-#include "soc/intel_rom.h"
-
 #include "intel_display.h"
 #include "intel_display_core.h"
 #include "intel_display_rpm.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
 #include "intel_gmbus.h"
+#include "intel_rom.h"
 
 #define _INTEL_BIOS_PRIVATE
 #include "intel_vbt_defs.h"
@@ -2529,6 +2528,54 @@ intel_bios_encoder_reject_edp_rate(const struct intel_bios_encoder_data *devdata
 	return devdata->child.edp_data_rate_override & edp_rate_override_mask(rate);
 }
 
+static void sanitize_dedicated_external(struct intel_bios_encoder_data *devdata,
+					enum port port)
+{
+	struct intel_display *display = devdata->display;
+
+	if (!intel_bios_encoder_is_dedicated_external(devdata))
+		return;
+
+	/*
+	 * Since dedicated_external is for ports connected to PHYs outside of
+	 * the Type-C subsystem, clear bits that would only make sense for ports
+	 * with PHYs in the Type-C subsystem.
+	 */
+
+	/*
+	 * Bit dp_usb_type_c is marked as "don't care" in Bspec when
+	 * dedicated_external is set.
+	 */
+	if (devdata->child.dp_usb_type_c) {
+		drm_dbg_kms(display->drm,
+			    "VBT claims Port %c supports USB Type-C, but the port is dedicated external, ignoring\n",
+			    port_name(port));
+		devdata->child.dp_usb_type_c = 0;
+	}
+
+	/*
+	 * Bit tbt is marked as "don't care" in Bspec when dedicated_external is
+	 * set.
+	 */
+	if (devdata->child.tbt) {
+		drm_dbg_kms(display->drm,
+			    "VBT claims Port %c supports TBT, but the port is dedicated external, ignoring\n",
+			    port_name(port));
+		devdata->child.tbt = 0;
+	}
+
+	/*
+	 * DDI allocation for TC capable ports only make sense for PHYs in the
+	 * Type-C subsystem.
+	 */
+	if (devdata->child.dyn_port_over_tc) {
+		drm_dbg_kms(display->drm,
+			    "VBT claims Port %c supports dynamic DDI allocation in TCSS, but the port is dedicated external, ignoring\n",
+			    port_name(port));
+		devdata->child.dyn_port_over_tc = 0;
+	}
+}
+
 static void sanitize_device_type(struct intel_bios_encoder_data *devdata,
 				 enum port port)
 {
@@ -2693,6 +2740,16 @@ static void print_ddi_port(const struct intel_bios_encoder_data *devdata)
 		    supports_typec_usb, supports_tbt,
 		    devdata->dsc != NULL);
 
+	if (intel_bios_encoder_is_dedicated_external(devdata))
+		drm_dbg_kms(display->drm,
+			    "Port %c is dedicated external\n",
+			    port_name(port));
+
+	if (intel_bios_encoder_supports_dyn_port_over_tc(devdata))
+		drm_dbg_kms(display->drm,
+			    "Port %c supports dynamic DDI allocation in TCSS\n",
+			    port_name(port));
+
 	hdmi_level_shift = intel_bios_hdmi_level_shift(devdata);
 	if (hdmi_level_shift >= 0) {
 		drm_dbg_kms(display->drm,
@@ -2750,6 +2807,7 @@ static void parse_ddi_port(struct intel_bios_encoder_data *devdata)
 		return;
 	}
 
+	sanitize_dedicated_external(devdata, port);
 	sanitize_device_type(devdata, port);
 	sanitize_hdmi_level_shift(devdata, port);
 }
@@ -2777,7 +2835,7 @@ static int child_device_expected_size(u16 version)
 {
 	BUILD_BUG_ON(sizeof(struct child_device_config) < 40);
 
-	if (version > 263)
+	if (version > 264)
 		return -ENOENT;
 	else if (version >= 263)
 		return 44;
@@ -3721,6 +3779,18 @@ bool intel_bios_encoder_supports_tbt(const struct intel_bios_encoder_data *devda
 	return devdata->display->vbt.version >= 209 && devdata->child.tbt;
 }
 
+bool intel_bios_encoder_is_dedicated_external(const struct intel_bios_encoder_data *devdata)
+{
+	return devdata->display->vbt.version >= 264 &&
+		devdata->child.dedicated_external;
+}
+
+bool intel_bios_encoder_supports_dyn_port_over_tc(const struct intel_bios_encoder_data *devdata)
+{
+	return devdata->display->vbt.version >= 264 &&
+		devdata->child.dyn_port_over_tc;
+}
+
 bool intel_bios_encoder_lane_reversal(const struct intel_bios_encoder_data *devdata)
 {
 	return devdata && devdata->child.lane_reversal;
diff --git a/drivers/gpu/drm/i915/display/intel_bios.h b/drivers/gpu/drm/i915/display/intel_bios.h
index f9e438b2787b..75dff27b4228 100644
--- a/drivers/gpu/drm/i915/display/intel_bios.h
+++ b/drivers/gpu/drm/i915/display/intel_bios.h
@@ -79,6 +79,8 @@ bool intel_bios_encoder_supports_dp(const struct intel_bios_encoder_data *devdat
 bool intel_bios_encoder_supports_edp(const struct intel_bios_encoder_data *devdata);
 bool intel_bios_encoder_supports_typec_usb(const struct intel_bios_encoder_data *devdata);
 bool intel_bios_encoder_supports_tbt(const struct intel_bios_encoder_data *devdata);
+bool intel_bios_encoder_is_dedicated_external(const struct intel_bios_encoder_data *devdata);
+bool intel_bios_encoder_supports_dyn_port_over_tc(const struct intel_bios_encoder_data *devdata);
 bool intel_bios_encoder_supports_dsi(const struct intel_bios_encoder_data *devdata);
 bool intel_bios_encoder_supports_dp_dual_mode(const struct intel_bios_encoder_data *devdata);
 bool intel_bios_encoder_is_lspcon(const struct intel_bios_encoder_data *devdata);
diff --git a/drivers/gpu/drm/i915/display/intel_bo.c b/drivers/gpu/drm/i915/display/intel_bo.c
index f3687eb63467..8f372b33d48b 100644
--- a/drivers/gpu/drm/i915/display/intel_bo.c
+++ b/drivers/gpu/drm/i915/display/intel_bo.c
@@ -6,6 +6,7 @@
 #include "gem/i915_gem_mman.h"
 #include "gem/i915_gem_object.h"
 #include "gem/i915_gem_object_frontbuffer.h"
+#include "pxp/intel_pxp.h"
 #include "i915_debugfs.h"
 #include "intel_bo.h"
 
@@ -29,6 +30,11 @@ bool intel_bo_is_protected(struct drm_gem_object *obj)
 	return i915_gem_object_is_protected(to_intel_bo(obj));
 }
 
+int intel_bo_key_check(struct drm_gem_object *obj)
+{
+	return intel_pxp_key_check(obj, false);
+}
+
 int intel_bo_fb_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
 {
 	return i915_gem_fb_mmap(to_intel_bo(obj), vma);
diff --git a/drivers/gpu/drm/i915/display/intel_bo.h b/drivers/gpu/drm/i915/display/intel_bo.h
index fc05f680dc76..516a3836a6bc 100644
--- a/drivers/gpu/drm/i915/display/intel_bo.h
+++ b/drivers/gpu/drm/i915/display/intel_bo.h
@@ -16,6 +16,7 @@ bool intel_bo_is_tiled(struct drm_gem_object *obj);
 bool intel_bo_is_userptr(struct drm_gem_object *obj);
 bool intel_bo_is_shmem(struct drm_gem_object *obj);
 bool intel_bo_is_protected(struct drm_gem_object *obj);
+int intel_bo_key_check(struct drm_gem_object *obj);
 int intel_bo_fb_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
 int intel_bo_read_from_page(struct drm_gem_object *obj, u64 offset, void *dst, int size);
 
diff --git a/drivers/gpu/drm/i915/display/intel_bw.c b/drivers/gpu/drm/i915/display/intel_bw.c
index 1f6461be50ef..d27835ed49c2 100644
--- a/drivers/gpu/drm/i915/display/intel_bw.c
+++ b/drivers/gpu/drm/i915/display/intel_bw.c
@@ -6,8 +6,6 @@
 #include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_print.h>
 
-#include "soc/intel_dram.h"
-
 #include "i915_drv.h"
 #include "i915_reg.h"
 #include "intel_bw.h"
@@ -16,6 +14,7 @@
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
+#include "intel_dram.h"
 #include "intel_mchbar_regs.h"
 #include "intel_pcode.h"
 #include "intel_uncore.h"
@@ -800,7 +799,7 @@ static unsigned int icl_qgv_bw(struct intel_display *display,
 
 void intel_bw_init_hw(struct intel_display *display)
 {
-	const struct dram_info *dram_info = intel_dram_info(display->drm);
+	const struct dram_info *dram_info = intel_dram_info(display);
 
 	if (!HAS_DISPLAY(display))
 		return;
diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c
index 37801c744b05..0aa59d624095 100644
--- a/drivers/gpu/drm/i915/display/intel_cdclk.c
+++ b/drivers/gpu/drm/i915/display/intel_cdclk.c
@@ -28,10 +28,7 @@
 #include <drm/drm_fixed.h>
 #include <drm/drm_print.h>
 
-#include "soc/intel_dram.h"
-
 #include "hsw_ips.h"
-#include "i915_drv.h"
 #include "i915_reg.h"
 #include "intel_atomic.h"
 #include "intel_audio.h"
@@ -42,11 +39,13 @@
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
+#include "intel_dram.h"
 #include "intel_mchbar_regs.h"
 #include "intel_pci_config.h"
 #include "intel_pcode.h"
 #include "intel_plane.h"
 #include "intel_psr.h"
+#include "intel_step.h"
 #include "intel_vdsc.h"
 #include "skl_watermark.h"
 #include "skl_watermark_regs.h"
@@ -668,7 +667,7 @@ static void vlv_set_cdclk(struct intel_display *display,
 {
 	int cdclk = cdclk_config->cdclk;
 	u32 val, cmd = cdclk_config->voltage_level;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	int ret;
 
 	switch (cdclk) {
@@ -758,7 +757,7 @@ static void chv_set_cdclk(struct intel_display *display,
 {
 	int cdclk = cdclk_config->cdclk;
 	u32 val, cmd = cdclk_config->voltage_level;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	int ret;
 
 	switch (cdclk) {
@@ -3738,10 +3737,8 @@ static int pch_rawclk(struct intel_display *display)
 
 static int i9xx_hrawclk(struct intel_display *display)
 {
-	struct drm_i915_private *i915 = to_i915(display->drm);
-
 	/* hrawclock is 1/4 the FSB frequency */
-	return DIV_ROUND_CLOSEST(intel_fsb_freq(i915), 4);
+	return DIV_ROUND_CLOSEST(intel_fsb_freq(display), 4);
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/display/intel_cmtg.c b/drivers/gpu/drm/i915/display/intel_cmtg.c
index 165138b95cb2..e1fdc6fe9762 100644
--- a/drivers/gpu/drm/i915/display/intel_cmtg.c
+++ b/drivers/gpu/drm/i915/display/intel_cmtg.c
@@ -85,7 +85,6 @@ static bool intel_cmtg_transcoder_is_secondary(struct intel_display *display,
 					       enum transcoder trans)
 {
 	enum intel_display_power_domain power_domain;
-	intel_wakeref_t wakeref;
 	u32 val = 0;
 
 	if (!HAS_TRANSCODER(display, trans))
@@ -93,7 +92,7 @@ static bool intel_cmtg_transcoder_is_secondary(struct intel_display *display,
 
 	power_domain = POWER_DOMAIN_TRANSCODER(trans);
 
-	with_intel_display_power_if_enabled(display, power_domain, wakeref)
+	with_intel_display_power_if_enabled(display, power_domain)
 		val = intel_de_read(display, TRANS_DDI_FUNC_CTL2(display, trans));
 
 	return val & CMTG_SECONDARY_MODE;
diff --git a/drivers/gpu/drm/i915/display/intel_connector.c b/drivers/gpu/drm/i915/display/intel_connector.c
index 913d90a7a508..c128bc4215c4 100644
--- a/drivers/gpu/drm/i915/display/intel_connector.c
+++ b/drivers/gpu/drm/i915/display/intel_connector.c
@@ -156,27 +156,17 @@ void intel_connector_destroy(struct drm_connector *connector)
 int intel_connector_register(struct drm_connector *_connector)
 {
 	struct intel_connector *connector = to_intel_connector(_connector);
-	struct drm_i915_private *i915 = to_i915(_connector->dev);
 	int ret;
 
 	ret = intel_panel_register(connector);
 	if (ret)
-		goto err;
-
-	if (i915_inject_probe_failure(i915)) {
-		ret = -EFAULT;
-		goto err_panel;
-	}
+		return ret;
 
 	intel_connector_debugfs_add(connector);
 
 	return 0;
-
-err_panel:
-	intel_panel_unregister(connector);
-err:
-	return ret;
 }
+ALLOW_ERROR_INJECTION(intel_connector_register, ERRNO);
 
 void intel_connector_unregister(struct drm_connector *_connector)
 {
diff --git a/drivers/gpu/drm/i915/display/intel_crt.c b/drivers/gpu/drm/i915/display/intel_crt.c
index 82e89cdbe5a5..5f9a03877ea9 100644
--- a/drivers/gpu/drm/i915/display/intel_crt.c
+++ b/drivers/gpu/drm/i915/display/intel_crt.c
@@ -109,7 +109,7 @@ static bool intel_crt_get_hw_state(struct intel_encoder *encoder,
 {
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_crt *crt = intel_encoder_to_crt(encoder);
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret;
 
 	wakeref = intel_display_power_get_if_enabled(display,
@@ -847,7 +847,7 @@ intel_crt_detect(struct drm_connector *connector,
 	struct intel_crt *crt = intel_attached_crt(to_intel_connector(connector));
 	struct intel_encoder *encoder = &crt->base;
 	struct drm_atomic_state *state;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	int status;
 
 	drm_dbg_kms(display->drm, "[CONNECTOR:%d:%s] force=%d\n",
@@ -936,7 +936,7 @@ static int intel_crt_get_modes(struct drm_connector *connector)
 	struct intel_display *display = to_intel_display(connector->dev);
 	struct intel_crt *crt = intel_attached_crt(to_intel_connector(connector));
 	struct intel_encoder *encoder = &crt->base;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	struct i2c_adapter *ddc;
 	int ret;
 
diff --git a/drivers/gpu/drm/i915/display/intel_crtc.c b/drivers/gpu/drm/i915/display/intel_crtc.c
index 9d2a23c96c61..778ebc5095c3 100644
--- a/drivers/gpu/drm/i915/display/intel_crtc.c
+++ b/drivers/gpu/drm/i915/display/intel_crtc.c
@@ -13,8 +13,6 @@
 #include <drm/drm_vblank.h>
 #include <drm/drm_vblank_work.h>
 
-#include "i915_drv.h"
-#include "i915_vgpu.h"
 #include "i9xx_plane.h"
 #include "icl_dsi.h"
 #include "intel_atomic.h"
@@ -28,6 +26,7 @@
 #include "intel_drrs.h"
 #include "intel_dsi.h"
 #include "intel_fifo_underrun.h"
+#include "intel_parent.h"
 #include "intel_pipe_crc.h"
 #include "intel_plane.h"
 #include "intel_psr.h"
@@ -309,7 +308,7 @@ static const struct drm_crtc_funcs i8xx_crtc_funcs = {
 	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 };
 
-int intel_crtc_init(struct intel_display *display, enum pipe pipe)
+static int __intel_crtc_init(struct intel_display *display, enum pipe pipe)
 {
 	struct intel_plane *primary, *cursor;
 	const struct drm_crtc_funcs *funcs;
@@ -396,7 +395,7 @@ int intel_crtc_init(struct intel_display *display, enum pipe pipe)
 
 	drm_WARN_ON(display->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
 
-	if (HAS_CASF(display))
+	if (HAS_CASF(display) && crtc->num_scalers >= 2)
 		drm_crtc_create_sharpness_strength_property(&crtc->base);
 
 	return 0;
@@ -407,6 +406,23 @@ int intel_crtc_init(struct intel_display *display, enum pipe pipe)
 	return ret;
 }
 
+int intel_crtc_init(struct intel_display *display)
+{
+	enum pipe pipe;
+	int ret;
+
+	drm_dbg_kms(display->drm, "%d display pipe%s available.\n",
+		    INTEL_NUM_PIPES(display), str_plural(INTEL_NUM_PIPES(display)));
+
+	for_each_pipe(display, pipe) {
+		ret = __intel_crtc_init(display, pipe);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 int intel_crtc_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
 					   struct drm_file *file)
 {
@@ -553,7 +569,7 @@ void intel_pipe_update_start(struct intel_atomic_state *state,
 
 		for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
 						     new_plane_state, i) {
-			if (old_plane_state->uapi.crtc == &crtc->base)
+			if (old_plane_state->hw.crtc == &crtc->base)
 				intel_plane_init_cursor_vblank_work(old_plane_state,
 								    new_plane_state);
 		}
@@ -671,7 +687,6 @@ void intel_pipe_update_end(struct intel_atomic_state *state,
 	int scanline_end = intel_get_crtc_scanline(crtc);
 	u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
 	ktime_t end_vbl_time = ktime_get();
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
 	drm_WARN_ON(display->drm, new_crtc_state->use_dsb);
 
@@ -706,7 +721,7 @@ void intel_pipe_update_end(struct intel_atomic_state *state,
 		int i;
 
 		for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
-			if (old_plane_state->uapi.crtc == &crtc->base &&
+			if (old_plane_state->hw.crtc == &crtc->base &&
 			    old_plane_state->unpin_work.vblank) {
 				drm_vblank_work_schedule(&old_plane_state->unpin_work,
 							 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
@@ -737,7 +752,7 @@ void intel_pipe_update_end(struct intel_atomic_state *state,
 
 	local_irq_enable();
 
-	if (intel_vgpu_active(dev_priv))
+	if (intel_parent_vgpu_active(display))
 		goto out;
 
 	if (crtc->debug.start_vbl_count &&
diff --git a/drivers/gpu/drm/i915/display/intel_crtc.h b/drivers/gpu/drm/i915/display/intel_crtc.h
index 07917e8a9ae3..12507b51ee77 100644
--- a/drivers/gpu/drm/i915/display/intel_crtc.h
+++ b/drivers/gpu/drm/i915/display/intel_crtc.h
@@ -37,7 +37,7 @@ void intel_crtc_arm_vblank_event(struct intel_crtc_state *crtc_state);
 void intel_crtc_prepare_vblank_event(struct intel_crtc_state *crtc_state,
 				     struct drm_pending_vblank_event **event);
 u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state);
-int intel_crtc_init(struct intel_display *display, enum pipe pipe);
+int intel_crtc_init(struct intel_display *display);
 int intel_crtc_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
 					   struct drm_file *file_priv);
 struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc);
diff --git a/drivers/gpu/drm/i915/display/intel_cursor.c b/drivers/gpu/drm/i915/display/intel_cursor.c
index a10b2425b94d..2c5d917fbd7e 100644
--- a/drivers/gpu/drm/i915/display/intel_cursor.c
+++ b/drivers/gpu/drm/i915/display/intel_cursor.c
@@ -324,7 +324,7 @@ static bool i845_cursor_get_hw_state(struct intel_plane *plane,
 {
 	struct intel_display *display = to_intel_display(plane);
 	enum intel_display_power_domain power_domain;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret;
 
 	power_domain = POWER_DOMAIN_PIPE(PIPE_A);
@@ -727,7 +727,7 @@ static bool i9xx_cursor_get_hw_state(struct intel_plane *plane,
 {
 	struct intel_display *display = to_intel_display(plane);
 	enum intel_display_power_domain power_domain;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret;
 	u32 val;
 
@@ -974,6 +974,7 @@ static const struct drm_plane_funcs intel_cursor_plane_funcs = {
 	.atomic_duplicate_state = intel_plane_duplicate_state,
 	.atomic_destroy_state = intel_plane_destroy_state,
 	.format_mod_supported = intel_cursor_format_mod_supported,
+	.format_mod_supported_async = intel_plane_format_mod_supported_async,
 };
 
 static void intel_cursor_add_size_hints_property(struct intel_plane *plane)
diff --git a/drivers/gpu/drm/i915/display/intel_cx0_phy.c b/drivers/gpu/drm/i915/display/intel_cx0_phy.c
index d98b4cf6b60e..f6d69627154e 100644
--- a/drivers/gpu/drm/i915/display/intel_cx0_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_cx0_phy.c
@@ -11,6 +11,7 @@
 #include "intel_alpm.h"
 #include "intel_cx0_phy.h"
 #include "intel_cx0_phy_regs.h"
+#include "intel_display_regs.h"
 #include "intel_ddi.h"
 #include "intel_ddi_buf_trans.h"
 #include "intel_de.h"
@@ -104,11 +105,11 @@ static void intel_cx0_program_msgbus_timer(struct intel_encoder *encoder)
  * We also do the msgbus timer programming here to ensure that the timer
  * is already programmed before any access to the msgbus.
  */
-static intel_wakeref_t intel_cx0_phy_transaction_begin(struct intel_encoder *encoder)
+static struct ref_tracker *intel_cx0_phy_transaction_begin(struct intel_encoder *encoder)
 {
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 
 	intel_psr_pause(intel_dp);
 	wakeref = intel_display_power_get(display, POWER_DOMAIN_DC_OFF);
@@ -117,7 +118,7 @@ static intel_wakeref_t intel_cx0_phy_transaction_begin(struct intel_encoder *enc
 	return wakeref;
 }
 
-static void intel_cx0_phy_transaction_end(struct intel_encoder *encoder, intel_wakeref_t wakeref)
+static void intel_cx0_phy_transaction_end(struct intel_encoder *encoder, struct ref_tracker *wakeref)
 {
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
@@ -444,13 +445,38 @@ static u8 intel_c10_get_tx_term_ctl(const struct intel_crtc_state *crtc_state)
 	}
 }
 
+static void intel_c10_msgbus_access_begin(struct intel_encoder *encoder,
+					  u8 lane_mask)
+{
+	if (!intel_encoder_is_c10phy(encoder))
+		return;
+
+	intel_cx0_rmw(encoder, lane_mask, PHY_C10_VDR_CONTROL(1),
+		      0, C10_VDR_CTRL_MSGBUS_ACCESS, MB_WRITE_COMMITTED);
+}
+
+static void intel_c10_msgbus_access_commit(struct intel_encoder *encoder,
+					   u8 lane_mask, bool master_lane)
+{
+	u8 val = C10_VDR_CTRL_UPDATE_CFG;
+
+	if (!intel_encoder_is_c10phy(encoder))
+		return;
+
+	if (master_lane)
+		val |= C10_VDR_CTRL_MASTER_LANE;
+
+	intel_cx0_rmw(encoder, lane_mask, PHY_C10_VDR_CONTROL(1),
+		      0, val, MB_WRITE_COMMITTED);
+}
+
 void intel_cx0_phy_set_signal_levels(struct intel_encoder *encoder,
 				     const struct intel_crtc_state *crtc_state)
 {
 	struct intel_display *display = to_intel_display(encoder);
 	const struct intel_ddi_buf_trans *trans;
 	u8 owned_lane_mask;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	int n_entries, ln;
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 
@@ -467,9 +493,9 @@ void intel_cx0_phy_set_signal_levels(struct intel_encoder *encoder,
 		return;
 	}
 
+	intel_c10_msgbus_access_begin(encoder, owned_lane_mask);
+
 	if (intel_encoder_is_c10phy(encoder)) {
-		intel_cx0_rmw(encoder, owned_lane_mask, PHY_C10_VDR_CONTROL(1),
-			      0, C10_VDR_CTRL_MSGBUS_ACCESS, MB_WRITE_COMMITTED);
 		intel_cx0_rmw(encoder, owned_lane_mask, PHY_C10_VDR_CMN(3),
 			      C10_CMN3_TXVBOOST_MASK,
 			      C10_CMN3_TXVBOOST(intel_c10_get_tx_vboost_lvl(crtc_state)),
@@ -508,9 +534,7 @@ void intel_cx0_phy_set_signal_levels(struct intel_encoder *encoder,
 		      0, PHY_C10_VDR_OVRD_TX1 | PHY_C10_VDR_OVRD_TX2,
 		      MB_WRITE_COMMITTED);
 
-	if (intel_encoder_is_c10phy(encoder))
-		intel_cx0_rmw(encoder, owned_lane_mask, PHY_C10_VDR_CONTROL(1),
-			      0, C10_VDR_CTRL_UPDATE_CFG, MB_WRITE_COMMITTED);
+	intel_c10_msgbus_access_commit(encoder, owned_lane_mask, false);
 
 	intel_cx0_phy_transaction_end(encoder, wakeref);
 }
@@ -2005,7 +2029,7 @@ static const struct intel_c20pll_state * const mtl_c20_hdmi_tables[] = {
 };
 
 static const struct intel_c10pll_state * const *
-intel_c10pll_tables_get(struct intel_crtc_state *crtc_state,
+intel_c10pll_tables_get(const struct intel_crtc_state *crtc_state,
 			struct intel_encoder *encoder)
 {
 	if (intel_crtc_has_dp_encoder(crtc_state)) {
@@ -2035,6 +2059,15 @@ static void intel_cx0pll_update_ssc(struct intel_encoder *encoder,
 	}
 }
 
+#define C10_PLL_SSC_REG_START_IDX	4
+#define C10_PLL_SSC_REG_COUNT		5
+
+static bool intel_c10pll_ssc_enabled(const struct intel_c10pll_state *pll_state)
+{
+	return memchr_inv(&pll_state->pll[C10_PLL_SSC_REG_START_IDX],
+			  0, sizeof(pll_state->pll[0]) * C10_PLL_SSC_REG_COUNT);
+}
+
 static void intel_c10pll_update_pll(struct intel_encoder *encoder,
 				    struct intel_cx0pll_state *pll_state)
 {
@@ -2044,16 +2077,42 @@ static void intel_c10pll_update_pll(struct intel_encoder *encoder,
 	if (pll_state->ssc_enabled)
 		return;
 
-	drm_WARN_ON(display->drm, ARRAY_SIZE(pll_state->c10.pll) < 9);
-	for (i = 4; i < 9; i++)
+	drm_WARN_ON(display->drm, ARRAY_SIZE(pll_state->c10.pll) <
+				  C10_PLL_SSC_REG_START_IDX + C10_PLL_SSC_REG_COUNT);
+	for (i = C10_PLL_SSC_REG_START_IDX;
+	     i < C10_PLL_SSC_REG_START_IDX + C10_PLL_SSC_REG_COUNT;
+	     i++)
 		pll_state->c10.pll[i] = 0;
 }
 
+static bool c10pll_state_is_dp(const struct intel_c10pll_state *pll_state)
+{
+	return !REG_FIELD_GET8(C10_PLL15_HDMIDIV_MASK, pll_state->pll[15]);
+}
+
+static bool c20pll_state_is_dp(const struct intel_c20pll_state *pll_state)
+{
+	return pll_state->vdr.serdes_rate & PHY_C20_IS_DP;
+}
+
+static bool cx0pll_state_is_dp(const struct intel_cx0pll_state *pll_state)
+{
+	if (pll_state->use_c10)
+		return c10pll_state_is_dp(&pll_state->c10);
+
+	return c20pll_state_is_dp(&pll_state->c20);
+}
+
+/*
+ * TODO: Convert the following to align with intel_c20pll_find_table() and
+ * intel_c20pll_calc_state_from_table().
+ */
 static int intel_c10pll_calc_state_from_table(struct intel_encoder *encoder,
 					      const struct intel_c10pll_state * const *tables,
-					      bool is_dp, int port_clock,
+					      bool is_dp, int port_clock, int lane_count,
 					      struct intel_cx0pll_state *pll_state)
 {
+	struct intel_display *display = to_intel_display(encoder);
 	int i;
 
 	for (i = 0; tables[i]; i++) {
@@ -2061,7 +2120,11 @@ static int intel_c10pll_calc_state_from_table(struct intel_encoder *encoder,
 			pll_state->c10 = *tables[i];
 			intel_cx0pll_update_ssc(encoder, pll_state, is_dp);
 			intel_c10pll_update_pll(encoder, pll_state);
+
 			pll_state->use_c10 = true;
+			pll_state->lane_count = lane_count;
+
+			drm_WARN_ON(display->drm, is_dp != c10pll_state_is_dp(&pll_state->c10));
 
 			return 0;
 		}
@@ -2070,9 +2133,12 @@ static int intel_c10pll_calc_state_from_table(struct intel_encoder *encoder,
 	return -EINVAL;
 }
 
-static int intel_c10pll_calc_state(struct intel_crtc_state *crtc_state,
-				   struct intel_encoder *encoder)
+static int intel_c10pll_calc_state(const struct intel_crtc_state *crtc_state,
+				   struct intel_encoder *encoder,
+				   struct intel_dpll_hw_state *hw_state)
 {
+	struct intel_display *display = to_intel_display(encoder);
+	bool is_dp = intel_crtc_has_dp_encoder(crtc_state);
 	const struct intel_c10pll_state * const *tables;
 	int err;
 
@@ -2080,43 +2146,126 @@ static int intel_c10pll_calc_state(struct intel_crtc_state *crtc_state,
 	if (!tables)
 		return -EINVAL;
 
-	err = intel_c10pll_calc_state_from_table(encoder, tables,
-						 intel_crtc_has_dp_encoder(crtc_state),
-						 crtc_state->port_clock,
-						 &crtc_state->dpll_hw_state.cx0pll);
+	err = intel_c10pll_calc_state_from_table(encoder, tables, is_dp,
+						 crtc_state->port_clock, crtc_state->lane_count,
+						 &hw_state->cx0pll);
 
 	if (err == 0 || !intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
 		return err;
 
 	/* For HDMI PLLs try SNPS PHY algorithm, if there are no precomputed tables */
-	intel_snps_hdmi_pll_compute_c10pll(&crtc_state->dpll_hw_state.cx0pll.c10,
+	intel_snps_hdmi_pll_compute_c10pll(&hw_state->cx0pll.c10,
 					   crtc_state->port_clock);
-	intel_c10pll_update_pll(encoder,
-				&crtc_state->dpll_hw_state.cx0pll);
-	crtc_state->dpll_hw_state.cx0pll.use_c10 = true;
+	intel_c10pll_update_pll(encoder, &hw_state->cx0pll);
+
+	hw_state->cx0pll.use_c10 = true;
+	hw_state->cx0pll.lane_count = crtc_state->lane_count;
+
+	drm_WARN_ON(display->drm, is_dp != c10pll_state_is_dp(&hw_state->cx0pll.c10));
 
 	return 0;
 }
 
 static int intel_c10pll_calc_port_clock(struct intel_encoder *encoder,
-					const struct intel_c10pll_state *pll_state);
+					const struct intel_c10pll_state *pll_state)
+{
+	unsigned int frac_quot = 0, frac_rem = 0, frac_den = 1;
+	unsigned int multiplier, tx_clk_div, hdmi_div, refclk = 38400;
+	int tmpclk = 0;
+
+	if (pll_state->pll[0] & C10_PLL0_FRACEN) {
+		frac_quot = pll_state->pll[12] << 8 | pll_state->pll[11];
+		frac_rem =  pll_state->pll[14] << 8 | pll_state->pll[13];
+		frac_den =  pll_state->pll[10] << 8 | pll_state->pll[9];
+	}
+
+	multiplier = (REG_FIELD_GET8(C10_PLL3_MULTIPLIERH_MASK, pll_state->pll[3]) << 8 |
+		      pll_state->pll[2]) / 2 + 16;
+
+	tx_clk_div = REG_FIELD_GET8(C10_PLL15_TXCLKDIV_MASK, pll_state->pll[15]);
+	hdmi_div = REG_FIELD_GET8(C10_PLL15_HDMIDIV_MASK, pll_state->pll[15]);
+
+	tmpclk = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, (multiplier << 16) + frac_quot) +
+				     DIV_ROUND_CLOSEST(refclk * frac_rem, frac_den),
+				     10 << (tx_clk_div + 16));
+	tmpclk *= (hdmi_div ? 2 : 1);
+
+	return tmpclk;
+}
+
+static int readout_enabled_lane_count(struct intel_encoder *encoder)
+{
+	struct intel_display *display = to_intel_display(encoder);
+	u8 enabled_tx_lane_count = 0;
+	int max_tx_lane_count = 4;
+	bool lane_reversal;
+	int tx_lane;
+
+	lane_reversal = intel_de_read(display, XELPDP_PORT_BUF_CTL1(display, encoder->port)) &
+			XELPDP_PORT_REVERSAL;
+
+	/*
+	 * TODO: also check inactive TX lanes in all PHY lanes owned by the
+	 * display. For now checking only those PHY lane(s) which are owned
+	 * based on the active TX lane count (i.e.
+	 *   1,2 active TX lanes -> PHY lane#0
+	 *   3,4 active TX lanes -> PHY lane#0 and PHY lane#1).
+	 *
+	 * In case of lane reversal for 1, 2 active TX lanes, only PHY
+	 * lane#1 is used. This is only possible in TypeC legacy mode or if
+	 * the port is connected to a non-TC PHY. In both of these cases both
+	 * PHY lane#0 and #1 are owned by display, so check all 4 TX lanes in
+	 * both PHY lanes in those cases.
+	 */
+	if (!lane_reversal)
+		max_tx_lane_count = DDI_PORT_WIDTH_GET(intel_de_read(display,
+								     DDI_BUF_CTL(encoder->port)));
+
+	if (!drm_WARN_ON(display->drm, max_tx_lane_count == 0))
+		max_tx_lane_count = round_up(max_tx_lane_count, 2);
+
+	for (tx_lane = 0; tx_lane < max_tx_lane_count; tx_lane++) {
+		u8 phy_lane_mask = tx_lane < 2 ? INTEL_CX0_LANE0 : INTEL_CX0_LANE1;
+		int tx = tx_lane % 2 + 1;
+		u8 val;
+
+		val = intel_cx0_read(encoder, phy_lane_mask, PHY_CX0_TX_CONTROL(tx, 2));
+		if (!(val & CONTROL2_DISABLE_SINGLE_TX))
+			enabled_tx_lane_count++;
+	}
+
+	return enabled_tx_lane_count;
+}
+
+static bool readout_ssc_state(struct intel_encoder *encoder, bool is_mpll_b)
+{
+	struct intel_display *display = to_intel_display(encoder);
+
+	return intel_de_read(display, XELPDP_PORT_CLOCK_CTL(display, encoder->port)) &
+		(is_mpll_b ? XELPDP_SSC_ENABLE_PLLB : XELPDP_SSC_ENABLE_PLLA);
+}
 
 static void intel_c10pll_readout_hw_state(struct intel_encoder *encoder,
-					  struct intel_c10pll_state *pll_state)
+					  struct intel_cx0pll_state *cx0pll_state)
 {
+	struct intel_c10pll_state *pll_state = &cx0pll_state->c10;
+	struct intel_display *display = to_intel_display(encoder);
+	enum phy phy = intel_encoder_to_phy(encoder);
 	u8 lane = INTEL_CX0_LANE0;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	int i;
 
+	cx0pll_state->use_c10 = true;
+
 	wakeref = intel_cx0_phy_transaction_begin(encoder);
 
 	/*
 	 * According to C10 VDR Register programming Sequence we need
 	 * to do this to read PHY internal registers from MsgBus.
 	 */
-	intel_cx0_rmw(encoder, lane, PHY_C10_VDR_CONTROL(1),
-		      0, C10_VDR_CTRL_MSGBUS_ACCESS,
-		      MB_WRITE_COMMITTED);
+	intel_c10_msgbus_access_begin(encoder, lane);
+
+	cx0pll_state->lane_count = readout_enabled_lane_count(encoder);
 
 	for (i = 0; i < ARRAY_SIZE(pll_state->pll); i++)
 		pll_state->pll[i] = intel_cx0_read(encoder, lane, PHY_C10_VDR_PLL(i));
@@ -2127,6 +2276,15 @@ static void intel_c10pll_readout_hw_state(struct intel_encoder *encoder,
 	intel_cx0_phy_transaction_end(encoder, wakeref);
 
 	pll_state->clock = intel_c10pll_calc_port_clock(encoder, pll_state);
+
+	cx0pll_state->ssc_enabled = readout_ssc_state(encoder, true);
+
+	if (cx0pll_state->ssc_enabled != intel_c10pll_ssc_enabled(pll_state))
+		drm_dbg_kms(display->drm,
+			    "PHY %c: SSC state mismatch: port SSC is %s, PLL SSC is %s\n",
+			    phy_name(phy),
+			    str_enabled_disabled(cx0pll_state->ssc_enabled),
+			    str_enabled_disabled(intel_c10pll_ssc_enabled(pll_state)));
 }
 
 static void intel_c10_pll_program(struct intel_display *display,
@@ -2135,9 +2293,7 @@ static void intel_c10_pll_program(struct intel_display *display,
 {
 	int i;
 
-	intel_cx0_rmw(encoder, INTEL_CX0_BOTH_LANES, PHY_C10_VDR_CONTROL(1),
-		      0, C10_VDR_CTRL_MSGBUS_ACCESS,
-		      MB_WRITE_COMMITTED);
+	intel_c10_msgbus_access_begin(encoder, INTEL_CX0_BOTH_LANES);
 
 	/* Program the pll values only for the master lane */
 	for (i = 0; i < ARRAY_SIZE(pll_state->pll); i++)
@@ -2152,12 +2308,11 @@ static void intel_c10_pll_program(struct intel_display *display,
 	intel_cx0_rmw(encoder, INTEL_CX0_BOTH_LANES, PHY_C10_VDR_CUSTOM_WIDTH,
 		      C10_VDR_CUSTOM_WIDTH_MASK, C10_VDR_CUSTOM_WIDTH_8_10,
 		      MB_WRITE_COMMITTED);
-	intel_cx0_rmw(encoder, INTEL_CX0_LANE0, PHY_C10_VDR_CONTROL(1),
-		      0, C10_VDR_CTRL_MASTER_LANE | C10_VDR_CTRL_UPDATE_CFG,
-		      MB_WRITE_COMMITTED);
+
+	intel_c10_msgbus_access_commit(encoder, INTEL_CX0_LANE0, true);
 }
 
-static void intel_c10pll_dump_hw_state(struct intel_display *display,
+static void intel_c10pll_dump_hw_state(struct drm_printer *p,
 				       const struct intel_c10pll_state *hw_state)
 {
 	bool fracen;
@@ -2166,33 +2321,32 @@ static void intel_c10pll_dump_hw_state(struct intel_display *display,
 	unsigned int multiplier, tx_clk_div;
 
 	fracen = hw_state->pll[0] & C10_PLL0_FRACEN;
-	drm_dbg_kms(display->drm, "c10pll_hw_state: fracen: %s, ",
-		    str_yes_no(fracen));
+	drm_printf(p, "c10pll_hw_state: clock: %d, fracen: %s, ",
+		   hw_state->clock, str_yes_no(fracen));
 
 	if (fracen) {
 		frac_quot = hw_state->pll[12] << 8 | hw_state->pll[11];
 		frac_rem =  hw_state->pll[14] << 8 | hw_state->pll[13];
 		frac_den =  hw_state->pll[10] << 8 | hw_state->pll[9];
-		drm_dbg_kms(display->drm, "quot: %u, rem: %u, den: %u,\n",
-			    frac_quot, frac_rem, frac_den);
+		drm_printf(p, "quot: %u, rem: %u, den: %u,\n",
+			   frac_quot, frac_rem, frac_den);
 	}
 
 	multiplier = (REG_FIELD_GET8(C10_PLL3_MULTIPLIERH_MASK, hw_state->pll[3]) << 8 |
 		      hw_state->pll[2]) / 2 + 16;
 	tx_clk_div = REG_FIELD_GET8(C10_PLL15_TXCLKDIV_MASK, hw_state->pll[15]);
-	drm_dbg_kms(display->drm,
-		    "multiplier: %u, tx_clk_div: %u.\n", multiplier, tx_clk_div);
+	drm_printf(p,
+		   "multiplier: %u, tx_clk_div: %u.\n", multiplier, tx_clk_div);
 
-	drm_dbg_kms(display->drm, "c10pll_rawhw_state:");
-	drm_dbg_kms(display->drm, "tx: 0x%x, cmn: 0x%x\n", hw_state->tx,
-		    hw_state->cmn);
+	drm_printf(p, "c10pll_rawhw_state:");
+	drm_printf(p, "tx: 0x%x, cmn: 0x%x\n", hw_state->tx, hw_state->cmn);
 
 	BUILD_BUG_ON(ARRAY_SIZE(hw_state->pll) % 4);
 	for (i = 0; i < ARRAY_SIZE(hw_state->pll); i = i + 4)
-		drm_dbg_kms(display->drm,
-			    "pll[%d] = 0x%x, pll[%d] = 0x%x, pll[%d] = 0x%x, pll[%d] = 0x%x\n",
-			    i, hw_state->pll[i], i + 1, hw_state->pll[i + 1],
-			    i + 2, hw_state->pll[i + 2], i + 3, hw_state->pll[i + 3]);
+		drm_printf(p,
+			   "pll[%d] = 0x%x, pll[%d] = 0x%x, pll[%d] = 0x%x, pll[%d] = 0x%x\n",
+			   i, hw_state->pll[i], i + 1, hw_state->pll[i + 1],
+			   i + 2, hw_state->pll[i + 2], i + 3, hw_state->pll[i + 3]);
 }
 
 /*
@@ -2210,7 +2364,7 @@ static bool is_arrowlake_s_by_host_bridge(void)
 	return pdev && IS_ARROWLAKE_S_BY_HOST_BRIDGE_ID(host_bridge_pci_dev_id);
 }
 
-static u16 intel_c20_hdmi_tmds_tx_cgf_1(struct intel_crtc_state *crtc_state)
+static u16 intel_c20_hdmi_tmds_tx_cgf_1(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
 	u16 tx_misc;
@@ -2234,9 +2388,9 @@ static u16 intel_c20_hdmi_tmds_tx_cgf_1(struct intel_crtc_state *crtc_state)
 		C20_PHY_TX_DCC_BYPASS | C20_PHY_TX_TERM_CTL(tx_term_ctrl));
 }
 
-static int intel_c20_compute_hdmi_tmds_pll(struct intel_crtc_state *crtc_state)
+static int intel_c20_compute_hdmi_tmds_pll(const struct intel_crtc_state *crtc_state,
+					   struct intel_c20pll_state *pll_state)
 {
-	struct intel_c20pll_state *pll_state = &crtc_state->dpll_hw_state.cx0pll.c20;
 	u64 datarate;
 	u64 mpll_tx_clk_div;
 	u64 vco_freq_shift;
@@ -2304,7 +2458,7 @@ static int intel_c20_compute_hdmi_tmds_pll(struct intel_crtc_state *crtc_state)
 }
 
 static const struct intel_c20pll_state * const *
-intel_c20_pll_tables_get(struct intel_crtc_state *crtc_state,
+intel_c20_pll_tables_get(const struct intel_crtc_state *crtc_state,
 			 struct intel_encoder *encoder)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
@@ -2332,42 +2486,220 @@ intel_c20_pll_tables_get(struct intel_crtc_state *crtc_state,
 	return NULL;
 }
 
-static int intel_c20pll_calc_state(struct intel_crtc_state *crtc_state,
-				   struct intel_encoder *encoder)
+static u8 intel_c20_get_dp_rate(u32 clock)
 {
-	const struct intel_c20pll_state * const *tables;
-	int i;
+	switch (clock) {
+	case 162000: /* 1.62 Gbps DP1.4 */
+		return 0;
+	case 270000: /* 2.7 Gbps DP1.4 */
+		return 1;
+	case 540000: /* 5.4 Gbps DP 1.4 */
+		return 2;
+	case 810000: /* 8.1 Gbps DP1.4 */
+		return 3;
+	case 216000: /* 2.16 Gbps eDP */
+		return 4;
+	case 243000: /* 2.43 Gbps eDP */
+		return 5;
+	case 324000: /* 3.24 Gbps eDP */
+		return 6;
+	case 432000: /* 4.32 Gbps eDP */
+		return 7;
+	case 1000000: /* 10 Gbps DP2.0 */
+		return 8;
+	case 1350000: /* 13.5 Gbps DP2.0 */
+		return 9;
+	case 2000000: /* 20 Gbps DP2.0 */
+		return 10;
+	case 648000: /* 6.48 Gbps eDP*/
+		return 11;
+	case 675000: /* 6.75 Gbps eDP*/
+		return 12;
+	default:
+		MISSING_CASE(clock);
+		return 0;
+	}
+}
 
-	/* try computed C20 HDMI tables before using consolidated tables */
-	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
-		if (intel_c20_compute_hdmi_tmds_pll(crtc_state) == 0)
-			return 0;
+static u8 intel_c20_get_hdmi_rate(u32 clock)
+{
+	if (clock >= 25175 && clock <= 600000)
+		return 0;
+
+	switch (clock) {
+	case 300000: /* 3 Gbps */
+	case 600000: /* 6 Gbps */
+	case 1200000: /* 12 Gbps */
+		return 1;
+	case 800000: /* 8 Gbps */
+		return 2;
+	case 1000000: /* 10 Gbps */
+		return 3;
+	default:
+		MISSING_CASE(clock);
+		return 0;
 	}
+}
+
+static bool is_dp2(u32 clock)
+{
+	/* DP2.0 clock rates */
+	if (clock == 1000000 || clock == 1350000 || clock  == 2000000)
+		return true;
+
+	return false;
+}
+
+static int intel_get_c20_custom_width(u32 clock, bool dp)
+{
+	if (dp && is_dp2(clock))
+		return 2;
+	else if (intel_hdmi_is_frl(clock))
+		return 1;
+	else
+		return 0;
+}
+
+static void intel_c20_calc_vdr_params(struct intel_c20pll_vdr_state *vdr, bool is_dp,
+				      int port_clock)
+{
+	vdr->custom_width = intel_get_c20_custom_width(port_clock, is_dp);
+
+	vdr->serdes_rate = 0;
+	vdr->hdmi_rate = 0;
+
+	if (is_dp) {
+		vdr->serdes_rate = PHY_C20_IS_DP |
+				   PHY_C20_DP_RATE(intel_c20_get_dp_rate(port_clock));
+	} else {
+		if (intel_hdmi_is_frl(port_clock))
+			vdr->serdes_rate = PHY_C20_IS_HDMI_FRL;
+
+		vdr->hdmi_rate = intel_c20_get_hdmi_rate(port_clock);
+	}
+}
+
+#define PHY_C20_SERDES_RATE_MASK	(PHY_C20_IS_DP | PHY_C20_DP_RATE_MASK | PHY_C20_IS_HDMI_FRL)
+
+static void intel_c20_readout_vdr_params(struct intel_encoder *encoder,
+					 struct intel_c20pll_vdr_state *vdr, bool *cntx)
+{
+	u8 serdes;
+
+	serdes = intel_cx0_read(encoder, INTEL_CX0_LANE0, PHY_C20_VDR_CUSTOM_SERDES_RATE);
+	*cntx = serdes & PHY_C20_CONTEXT_TOGGLE;
+
+	vdr->custom_width = intel_cx0_read(encoder, INTEL_CX0_LANE0, PHY_C20_VDR_CUSTOM_WIDTH) &
+			    PHY_C20_CUSTOM_WIDTH_MASK;
+
+	vdr->serdes_rate = serdes & PHY_C20_SERDES_RATE_MASK;
+	if (!(vdr->serdes_rate & PHY_C20_IS_DP))
+		vdr->hdmi_rate = intel_cx0_read(encoder, INTEL_CX0_LANE0, PHY_C20_VDR_HDMI_RATE) &
+				 PHY_C20_HDMI_RATE_MASK;
+	else
+		vdr->hdmi_rate = 0;
+}
+
+static void intel_c20_program_vdr_params(struct intel_encoder *encoder,
+					 const struct intel_c20pll_vdr_state *vdr,
+					 u8 owned_lane_mask)
+{
+	struct intel_display *display = to_intel_display(encoder);
+
+	drm_WARN_ON(display->drm, vdr->custom_width & ~PHY_C20_CUSTOM_WIDTH_MASK);
+	intel_cx0_rmw(encoder, owned_lane_mask, PHY_C20_VDR_CUSTOM_WIDTH,
+		      PHY_C20_CUSTOM_WIDTH_MASK, vdr->custom_width,
+		      MB_WRITE_COMMITTED);
+
+	drm_WARN_ON(display->drm, vdr->serdes_rate & ~PHY_C20_SERDES_RATE_MASK);
+	intel_cx0_rmw(encoder, owned_lane_mask, PHY_C20_VDR_CUSTOM_SERDES_RATE,
+		      PHY_C20_SERDES_RATE_MASK, vdr->serdes_rate,
+		      MB_WRITE_COMMITTED);
+
+	if (vdr->serdes_rate & PHY_C20_IS_DP)
+		return;
+
+	drm_WARN_ON(display->drm, vdr->hdmi_rate & ~PHY_C20_HDMI_RATE_MASK);
+	intel_cx0_rmw(encoder, INTEL_CX0_BOTH_LANES, PHY_C20_VDR_HDMI_RATE,
+		      PHY_C20_HDMI_RATE_MASK, vdr->hdmi_rate,
+		      MB_WRITE_COMMITTED);
+}
+
+static const struct intel_c20pll_state *
+intel_c20_pll_find_table(const struct intel_crtc_state *crtc_state,
+			 struct intel_encoder *encoder)
+{
+	const struct intel_c20pll_state * const *tables;
+	int i;
 
 	tables = intel_c20_pll_tables_get(crtc_state, encoder);
 	if (!tables)
+		return NULL;
+
+	for (i = 0; tables[i]; i++)
+		if (crtc_state->port_clock == tables[i]->clock)
+			return tables[i];
+
+	return NULL;
+}
+
+static int intel_c20pll_calc_state_from_table(const struct intel_crtc_state *crtc_state,
+					      struct intel_encoder *encoder,
+					      struct intel_cx0pll_state *pll_state)
+{
+	const struct intel_c20pll_state *table;
+
+	table = intel_c20_pll_find_table(crtc_state, encoder);
+	if (!table)
 		return -EINVAL;
 
-	for (i = 0; tables[i]; i++) {
-		if (crtc_state->port_clock == tables[i]->clock) {
-			crtc_state->dpll_hw_state.cx0pll.c20 = *tables[i];
-			intel_cx0pll_update_ssc(encoder,
-						&crtc_state->dpll_hw_state.cx0pll,
-						intel_crtc_has_dp_encoder(crtc_state));
-			crtc_state->dpll_hw_state.cx0pll.use_c10 = false;
-			return 0;
-		}
-	}
+	pll_state->c20 = *table;
 
-	return -EINVAL;
+	intel_cx0pll_update_ssc(encoder, pll_state, intel_crtc_has_dp_encoder(crtc_state));
+
+	return 0;
 }
 
-int intel_cx0pll_calc_state(struct intel_crtc_state *crtc_state,
-			    struct intel_encoder *encoder)
+static int intel_c20pll_calc_state(const struct intel_crtc_state *crtc_state,
+				   struct intel_encoder *encoder,
+				   struct intel_dpll_hw_state *hw_state)
 {
+	struct intel_display *display = to_intel_display(encoder);
+	bool is_dp = intel_crtc_has_dp_encoder(crtc_state);
+	int err = -ENOENT;
+
+	hw_state->cx0pll.use_c10 = false;
+	hw_state->cx0pll.lane_count = crtc_state->lane_count;
+
+	/* try computed C20 HDMI tables before using consolidated tables */
+	if (!is_dp)
+		/* TODO: Update SSC state for HDMI as well */
+		err = intel_c20_compute_hdmi_tmds_pll(crtc_state, &hw_state->cx0pll.c20);
+
+	if (err)
+		err = intel_c20pll_calc_state_from_table(crtc_state, encoder,
+							 &hw_state->cx0pll);
+
+	if (err)
+		return err;
+
+	intel_c20_calc_vdr_params(&hw_state->cx0pll.c20.vdr,
+				  is_dp, crtc_state->port_clock);
+
+	drm_WARN_ON(display->drm, is_dp != c20pll_state_is_dp(&hw_state->cx0pll.c20));
+
+	return 0;
+}
+
+int intel_cx0pll_calc_state(const struct intel_crtc_state *crtc_state,
+			    struct intel_encoder *encoder,
+			    struct intel_dpll_hw_state *hw_state)
+{
+	memset(hw_state, 0, sizeof(*hw_state));
+
 	if (intel_encoder_is_c10phy(encoder))
-		return intel_c10pll_calc_state(crtc_state, encoder);
-	return intel_c20pll_calc_state(crtc_state, encoder);
+		return intel_c10pll_calc_state(crtc_state, encoder, hw_state);
+	return intel_c20pll_calc_state(crtc_state, encoder, hw_state);
 }
 
 static bool intel_c20phy_use_mpllb(const struct intel_c20pll_state *state)
@@ -2421,17 +2753,22 @@ static int intel_c20pll_calc_port_clock(struct intel_encoder *encoder,
 }
 
 static void intel_c20pll_readout_hw_state(struct intel_encoder *encoder,
-					  struct intel_c20pll_state *pll_state)
+					  struct intel_cx0pll_state *cx0pll_state)
 {
+	struct intel_c20pll_state *pll_state = &cx0pll_state->c20;
 	struct intel_display *display = to_intel_display(encoder);
 	bool cntx;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	int i;
 
+	cx0pll_state->use_c10 = false;
+
 	wakeref = intel_cx0_phy_transaction_begin(encoder);
 
-	/* 1. Read current context selection */
-	cntx = intel_cx0_read(encoder, INTEL_CX0_LANE0, PHY_C20_VDR_CUSTOM_SERDES_RATE) & PHY_C20_CONTEXT_TOGGLE;
+	cx0pll_state->lane_count = readout_enabled_lane_count(encoder);
+
+	/* 1. Read VDR params and current context selection */
+	intel_c20_readout_vdr_params(encoder, &pll_state->vdr, &cntx);
 
 	/* Read Tx configuration */
 	for (i = 0; i < ARRAY_SIZE(pll_state->tx); i++) {
@@ -2486,103 +2823,52 @@ static void intel_c20pll_readout_hw_state(struct intel_encoder *encoder,
 	pll_state->clock = intel_c20pll_calc_port_clock(encoder, pll_state);
 
 	intel_cx0_phy_transaction_end(encoder, wakeref);
+
+	cx0pll_state->ssc_enabled = readout_ssc_state(encoder, intel_c20phy_use_mpllb(pll_state));
 }
 
-static void intel_c20pll_dump_hw_state(struct intel_display *display,
+static void intel_c20pll_dump_hw_state(struct drm_printer *p,
 				       const struct intel_c20pll_state *hw_state)
 {
 	int i;
 
-	drm_dbg_kms(display->drm, "c20pll_hw_state:\n");
-	drm_dbg_kms(display->drm,
-		    "tx[0] = 0x%.4x, tx[1] = 0x%.4x, tx[2] = 0x%.4x\n",
-		    hw_state->tx[0], hw_state->tx[1], hw_state->tx[2]);
-	drm_dbg_kms(display->drm,
-		    "cmn[0] = 0x%.4x, cmn[1] = 0x%.4x, cmn[2] = 0x%.4x, cmn[3] = 0x%.4x\n",
-		    hw_state->cmn[0], hw_state->cmn[1], hw_state->cmn[2], hw_state->cmn[3]);
+	drm_printf(p, "c20pll_hw_state: clock: %d\n", hw_state->clock);
+	drm_printf(p,
+		   "tx[0] = 0x%.4x, tx[1] = 0x%.4x, tx[2] = 0x%.4x\n",
+		   hw_state->tx[0], hw_state->tx[1], hw_state->tx[2]);
+	drm_printf(p,
+		   "cmn[0] = 0x%.4x, cmn[1] = 0x%.4x, cmn[2] = 0x%.4x, cmn[3] = 0x%.4x\n",
+		   hw_state->cmn[0], hw_state->cmn[1], hw_state->cmn[2], hw_state->cmn[3]);
 
 	if (intel_c20phy_use_mpllb(hw_state)) {
 		for (i = 0; i < ARRAY_SIZE(hw_state->mpllb); i++)
-			drm_dbg_kms(display->drm, "mpllb[%d] = 0x%.4x\n", i,
-				    hw_state->mpllb[i]);
+			drm_printf(p, "mpllb[%d] = 0x%.4x\n", i, hw_state->mpllb[i]);
 	} else {
 		for (i = 0; i < ARRAY_SIZE(hw_state->mplla); i++)
-			drm_dbg_kms(display->drm, "mplla[%d] = 0x%.4x\n", i,
-				    hw_state->mplla[i]);
-	}
-}
-
-void intel_cx0pll_dump_hw_state(struct intel_display *display,
-				const struct intel_cx0pll_state *hw_state)
-{
-	if (hw_state->use_c10)
-		intel_c10pll_dump_hw_state(display, &hw_state->c10);
-	else
-		intel_c20pll_dump_hw_state(display, &hw_state->c20);
-}
+			drm_printf(p, "mplla[%d] = 0x%.4x\n", i, hw_state->mplla[i]);
 
-static u8 intel_c20_get_dp_rate(u32 clock)
-{
-	switch (clock) {
-	case 162000: /* 1.62 Gbps DP1.4 */
-		return 0;
-	case 270000: /* 2.7 Gbps DP1.4 */
-		return 1;
-	case 540000: /* 5.4 Gbps DP 1.4 */
-		return 2;
-	case 810000: /* 8.1 Gbps DP1.4 */
-		return 3;
-	case 216000: /* 2.16 Gbps eDP */
-		return 4;
-	case 243000: /* 2.43 Gbps eDP */
-		return 5;
-	case 324000: /* 3.24 Gbps eDP */
-		return 6;
-	case 432000: /* 4.32 Gbps eDP */
-		return 7;
-	case 1000000: /* 10 Gbps DP2.0 */
-		return 8;
-	case 1350000: /* 13.5 Gbps DP2.0 */
-		return 9;
-	case 2000000: /* 20 Gbps DP2.0 */
-		return 10;
-	case 648000: /* 6.48 Gbps eDP*/
-		return 11;
-	case 675000: /* 6.75 Gbps eDP*/
-		return 12;
-	default:
-		MISSING_CASE(clock);
-		return 0;
+		/* For full coverage, also print the additional PLL B entry. */
+		BUILD_BUG_ON(ARRAY_SIZE(hw_state->mplla) + 1 != ARRAY_SIZE(hw_state->mpllb));
+		drm_printf(p, "mpllb[%d] = 0x%.4x\n", i, hw_state->mpllb[i]);
 	}
-}
 
-static u8 intel_c20_get_hdmi_rate(u32 clock)
-{
-	if (clock >= 25175 && clock <= 600000)
-		return 0;
-
-	switch (clock) {
-	case 300000: /* 3 Gbps */
-	case 600000: /* 6 Gbps */
-	case 1200000: /* 12 Gbps */
-		return 1;
-	case 800000: /* 8 Gbps */
-		return 2;
-	case 1000000: /* 10 Gbps */
-		return 3;
-	default:
-		MISSING_CASE(clock);
-		return 0;
-	}
+	drm_printf(p,
+		   "vdr: custom width: 0x%02x, serdes rate: 0x%02x, hdmi rate: 0x%02x\n",
+		   hw_state->vdr.custom_width, hw_state->vdr.serdes_rate, hw_state->vdr.hdmi_rate);
 }
 
-static bool is_dp2(u32 clock)
+void intel_cx0pll_dump_hw_state(struct drm_printer *p,
+				const struct intel_cx0pll_state *hw_state)
 {
-	/* DP2.0 clock rates */
-	if (clock == 1000000 || clock == 1350000 || clock  == 2000000)
-		return true;
+	drm_printf(p,
+		   "cx0pll_hw_state: lane_count: %d, ssc_enabled: %s, use_c10: %s, tbt_mode: %s\n",
+		   hw_state->lane_count, str_yes_no(hw_state->ssc_enabled),
+		   str_yes_no(hw_state->use_c10), str_yes_no(hw_state->tbt_mode));
 
-	return false;
+	if (hw_state->use_c10)
+		intel_c10pll_dump_hw_state(p, &hw_state->c10);
+	else
+		intel_c20pll_dump_hw_state(p, &hw_state->c20);
 }
 
 static bool intel_c20_protocol_switch_valid(struct intel_encoder *encoder)
@@ -2594,23 +2880,11 @@ static bool intel_c20_protocol_switch_valid(struct intel_encoder *encoder)
 	return intel_tc_port_in_legacy_mode(intel_dig_port);
 }
 
-static int intel_get_c20_custom_width(u32 clock, bool dp)
-{
-	if (dp && is_dp2(clock))
-		return 2;
-	else if (intel_hdmi_is_frl(clock))
-		return 1;
-	else
-		return 0;
-}
-
 static void intel_c20_pll_program(struct intel_display *display,
 				  struct intel_encoder *encoder,
-				  const struct intel_c20pll_state *pll_state,
-				  bool is_dp, int port_clock)
+				  const struct intel_c20pll_state *pll_state)
 {
 	u8 owned_lane_mask = intel_cx0_get_owned_lane_mask(encoder);
-	u8 serdes;
 	bool cntx;
 	int i;
 
@@ -2679,30 +2953,11 @@ static void intel_c20_pll_program(struct intel_display *display,
 		}
 	}
 
-	/* 4. Program custom width to match the link protocol */
-	intel_cx0_rmw(encoder, owned_lane_mask, PHY_C20_VDR_CUSTOM_WIDTH,
-		      PHY_C20_CUSTOM_WIDTH_MASK,
-		      PHY_C20_CUSTOM_WIDTH(intel_get_c20_custom_width(port_clock, is_dp)),
-		      MB_WRITE_COMMITTED);
-
-	/* 5. For DP or 6. For HDMI */
-	serdes = 0;
-	if (is_dp)
-		serdes = PHY_C20_IS_DP |
-			 PHY_C20_DP_RATE(intel_c20_get_dp_rate(port_clock));
-	else if (intel_hdmi_is_frl(port_clock))
-		serdes = PHY_C20_IS_HDMI_FRL;
-
-	intel_cx0_rmw(encoder, owned_lane_mask, PHY_C20_VDR_CUSTOM_SERDES_RATE,
-		      PHY_C20_IS_DP | PHY_C20_DP_RATE_MASK | PHY_C20_IS_HDMI_FRL,
-		      serdes,
-		      MB_WRITE_COMMITTED);
-
-	if (!is_dp)
-		intel_cx0_rmw(encoder, INTEL_CX0_BOTH_LANES, PHY_C20_VDR_HDMI_RATE,
-			      PHY_C20_HDMI_RATE_MASK,
-			      intel_c20_get_hdmi_rate(port_clock),
-			      MB_WRITE_COMMITTED);
+	/*
+	 * 4. Program custom width to match the link protocol.
+	 * 5. For DP or 6. For HDMI
+	 */
+	intel_c20_program_vdr_params(encoder, &pll_state->vdr, owned_lane_mask);
 
 	/*
 	 * 7. Write Vendor specific registers to toggle context setting to load
@@ -2713,39 +2968,13 @@ static void intel_c20_pll_program(struct intel_display *display,
 		      MB_WRITE_COMMITTED);
 }
 
-static int intel_c10pll_calc_port_clock(struct intel_encoder *encoder,
-					const struct intel_c10pll_state *pll_state)
-{
-	unsigned int frac_quot = 0, frac_rem = 0, frac_den = 1;
-	unsigned int multiplier, tx_clk_div, hdmi_div, refclk = 38400;
-	int tmpclk = 0;
-
-	if (pll_state->pll[0] & C10_PLL0_FRACEN) {
-		frac_quot = pll_state->pll[12] << 8 | pll_state->pll[11];
-		frac_rem =  pll_state->pll[14] << 8 | pll_state->pll[13];
-		frac_den =  pll_state->pll[10] << 8 | pll_state->pll[9];
-	}
-
-	multiplier = (REG_FIELD_GET8(C10_PLL3_MULTIPLIERH_MASK, pll_state->pll[3]) << 8 |
-		      pll_state->pll[2]) / 2 + 16;
-
-	tx_clk_div = REG_FIELD_GET8(C10_PLL15_TXCLKDIV_MASK, pll_state->pll[15]);
-	hdmi_div = REG_FIELD_GET8(C10_PLL15_HDMIDIV_MASK, pll_state->pll[15]);
-
-	tmpclk = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, (multiplier << 16) + frac_quot) +
-				     DIV_ROUND_CLOSEST(refclk * frac_rem, frac_den),
-				     10 << (tx_clk_div + 16));
-	tmpclk *= (hdmi_div ? 2 : 1);
-
-	return tmpclk;
-}
-
 static void intel_program_port_clock_ctl(struct intel_encoder *encoder,
 					 const struct intel_cx0pll_state *pll_state,
-					 bool is_dp, int port_clock,
+					 int port_clock,
 					 bool lane_reversal)
 {
 	struct intel_display *display = to_intel_display(encoder);
+	bool is_dp = cx0pll_state_is_dp(pll_state);
 	u32 val = 0;
 
 	intel_de_rmw(display, XELPDP_PORT_BUF_CTL1(display, encoder->port),
@@ -2938,11 +3167,7 @@ static void intel_cx0_program_phy_lane(struct intel_encoder *encoder, int lane_c
 	bool dp_alt_mode = intel_tc_port_in_dp_alt_mode(enc_to_dig_port(encoder));
 	u8 owned_lane_mask = intel_cx0_get_owned_lane_mask(encoder);
 
-	if (intel_encoder_is_c10phy(encoder))
-		intel_cx0_rmw(encoder, owned_lane_mask,
-			      PHY_C10_VDR_CONTROL(1), 0,
-			      C10_VDR_CTRL_MSGBUS_ACCESS,
-			      MB_WRITE_COMMITTED);
+	intel_c10_msgbus_access_begin(encoder, owned_lane_mask);
 
 	if (lane_reversal)
 		disables = REG_GENMASK8(3, 0) >> lane_count;
@@ -2967,11 +3192,7 @@ static void intel_cx0_program_phy_lane(struct intel_encoder *encoder, int lane_c
 			      MB_WRITE_COMMITTED);
 	}
 
-	if (intel_encoder_is_c10phy(encoder))
-		intel_cx0_rmw(encoder, owned_lane_mask,
-			      PHY_C10_VDR_CONTROL(1), 0,
-			      C10_VDR_CTRL_UPDATE_CFG,
-			      MB_WRITE_COMMITTED);
+	intel_c10_msgbus_access_commit(encoder, owned_lane_mask, false);
 }
 
 static u32 intel_cx0_get_pclk_pll_request(u8 lane_mask)
@@ -2996,23 +3217,30 @@ static u32 intel_cx0_get_pclk_pll_ack(u8 lane_mask)
 	return val;
 }
 
-static void __intel_cx0pll_enable(struct intel_encoder *encoder,
-				  const struct intel_cx0pll_state *pll_state,
-				  bool is_dp, int port_clock, int lane_count)
+static void intel_cx0pll_enable(struct intel_encoder *encoder,
+				const struct intel_cx0pll_state *pll_state)
 {
+	int port_clock = pll_state->use_c10 ? pll_state->c10.clock : pll_state->c20.clock;
 	struct intel_display *display = to_intel_display(encoder);
 	enum phy phy = intel_encoder_to_phy(encoder);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 	bool lane_reversal = dig_port->lane_reversal;
 	u8 maxpclk_lane = lane_reversal ? INTEL_CX0_LANE1 :
 					  INTEL_CX0_LANE0;
-	intel_wakeref_t wakeref = intel_cx0_phy_transaction_begin(encoder);
+	struct ref_tracker *wakeref = intel_cx0_phy_transaction_begin(encoder);
+
+	/*
+	 * Lane reversal is never used in DP-alt mode, in that case the
+	 * corresponding lane swapping (based on the TypeC cable flip state
+	 * for instance) is handled automatically by the HW via a TCSS mux.
+	 */
+	drm_WARN_ON(display->drm, lane_reversal && intel_tc_port_in_dp_alt_mode(dig_port));
 
 	/*
 	 * 1. Program PORT_CLOCK_CTL REGISTER to configure
 	 * clock muxes, gating and SSC
 	 */
-	intel_program_port_clock_ctl(encoder, pll_state, is_dp, port_clock, lane_reversal);
+	intel_program_port_clock_ctl(encoder, pll_state, port_clock, lane_reversal);
 
 	/* 2. Bring PHY out of reset. */
 	intel_cx0_phy_lane_reset(encoder, lane_reversal);
@@ -3034,13 +3262,13 @@ static void __intel_cx0pll_enable(struct intel_encoder *encoder,
 	if (intel_encoder_is_c10phy(encoder))
 		intel_c10_pll_program(display, encoder, &pll_state->c10);
 	else
-		intel_c20_pll_program(display, encoder, &pll_state->c20, is_dp, port_clock);
+		intel_c20_pll_program(display, encoder, &pll_state->c20);
 
 	/*
 	 * 6. Program the enabled and disabled owned PHY lane
 	 * transmitters over message bus
 	 */
-	intel_cx0_program_phy_lane(encoder, lane_count, lane_reversal);
+	intel_cx0_program_phy_lane(encoder, pll_state->lane_count, lane_reversal);
 
 	/*
 	 * 7. Follow the Display Voltage Frequency Switching - Sequence
@@ -3074,16 +3302,25 @@ static void __intel_cx0pll_enable(struct intel_encoder *encoder,
 	 * Frequency Change. We handle this step in bxt_set_cdclk().
 	 */
 
-	/* TODO: enable TBT-ALT mode */
 	intel_cx0_phy_transaction_end(encoder, wakeref);
 }
 
-static void intel_cx0pll_enable(struct intel_encoder *encoder,
-				const struct intel_crtc_state *crtc_state)
+void intel_mtl_tbt_pll_calc_state(struct intel_dpll_hw_state *hw_state)
 {
-	__intel_cx0pll_enable(encoder, &crtc_state->dpll_hw_state.cx0pll,
-			      intel_crtc_has_dp_encoder(crtc_state),
-			      crtc_state->port_clock, crtc_state->lane_count);
+	memset(hw_state, 0, sizeof(*hw_state));
+
+	hw_state->cx0pll.tbt_mode = true;
+}
+
+bool intel_mtl_tbt_pll_readout_hw_state(struct intel_display *display,
+					struct intel_dpll *pll,
+					struct intel_dpll_hw_state *hw_state)
+{
+	memset(hw_state, 0, sizeof(*hw_state));
+
+	hw_state->cx0pll.tbt_mode = true;
+
+	return true;
 }
 
 int intel_mtl_tbt_calc_port_clock(struct intel_encoder *encoder)
@@ -3148,8 +3385,7 @@ static int intel_mtl_tbt_clock_select(struct intel_display *display,
 	}
 }
 
-void intel_mtl_tbt_pll_enable(struct intel_encoder *encoder,
-			      const struct intel_crtc_state *crtc_state)
+void intel_mtl_tbt_pll_enable_clock(struct intel_encoder *encoder, int port_clock)
 {
 	struct intel_display *display = to_intel_display(encoder);
 	enum phy phy = intel_encoder_to_phy(encoder);
@@ -3163,7 +3399,7 @@ void intel_mtl_tbt_pll_enable(struct intel_encoder *encoder,
 
 	mask = XELPDP_DDI_CLOCK_SELECT_MASK(display);
 	val |= XELPDP_DDI_CLOCK_SELECT_PREP(display,
-					    intel_mtl_tbt_clock_select(display, crtc_state->port_clock));
+					    intel_mtl_tbt_clock_select(display, port_clock));
 
 	mask |= XELPDP_FORWARD_CLOCK_UNGATE;
 	val |= XELPDP_FORWARD_CLOCK_UNGATE;
@@ -3201,18 +3437,23 @@ void intel_mtl_tbt_pll_enable(struct intel_encoder *encoder,
 	 * clock frequency.
 	 */
 	intel_de_write(display, DDI_CLK_VALFREQ(encoder->port),
-		       crtc_state->port_clock);
+		       port_clock);
 }
 
 void intel_mtl_pll_enable(struct intel_encoder *encoder,
-			  const struct intel_crtc_state *crtc_state)
+			  struct intel_dpll *pll,
+			  const struct intel_dpll_hw_state *dpll_hw_state)
+{
+	intel_cx0pll_enable(encoder, &dpll_hw_state->cx0pll);
+}
+
+void intel_mtl_pll_enable_clock(struct intel_encoder *encoder,
+				const struct intel_crtc_state *crtc_state)
 {
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 
 	if (intel_tc_port_in_tbt_alt_mode(dig_port))
-		intel_mtl_tbt_pll_enable(encoder, crtc_state);
-	else
-		intel_cx0pll_enable(encoder, crtc_state);
+		intel_mtl_tbt_pll_enable_clock(encoder, crtc_state->port_clock);
 }
 
 /*
@@ -3224,7 +3465,7 @@ void intel_lnl_mac_transmit_lfps(struct intel_encoder *encoder,
 				 const struct intel_crtc_state *crtc_state)
 {
 	struct intel_display *display = to_intel_display(encoder);
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	int i;
 	u8 owned_lane_mask;
 
@@ -3236,9 +3477,7 @@ void intel_lnl_mac_transmit_lfps(struct intel_encoder *encoder,
 
 	wakeref = intel_cx0_phy_transaction_begin(encoder);
 
-	if (intel_encoder_is_c10phy(encoder))
-		intel_cx0_rmw(encoder, owned_lane_mask, PHY_C10_VDR_CONTROL(1), 0,
-			      C10_VDR_CTRL_MSGBUS_ACCESS, MB_WRITE_COMMITTED);
+	intel_c10_msgbus_access_begin(encoder, owned_lane_mask);
 
 	for (i = 0; i < 4; i++) {
 		int tx = i % 2 + 1;
@@ -3273,7 +3512,7 @@ static void intel_cx0pll_disable(struct intel_encoder *encoder)
 {
 	struct intel_display *display = to_intel_display(encoder);
 	enum phy phy = intel_encoder_to_phy(encoder);
-	intel_wakeref_t wakeref = intel_cx0_phy_transaction_begin(encoder);
+	struct ref_tracker *wakeref = intel_cx0_phy_transaction_begin(encoder);
 
 	/* 1. Change owned PHY lane power to Disable state. */
 	intel_cx0_powerdown_change_sequence(encoder, INTEL_CX0_BOTH_LANES,
@@ -3329,7 +3568,7 @@ static bool intel_cx0_pll_is_enabled(struct intel_encoder *encoder)
 			     intel_cx0_get_pclk_pll_request(lane);
 }
 
-void intel_mtl_tbt_pll_disable(struct intel_encoder *encoder)
+void intel_mtl_tbt_pll_disable_clock(struct intel_encoder *encoder)
 {
 	struct intel_display *display = to_intel_display(encoder);
 	enum phy phy = intel_encoder_to_phy(encoder);
@@ -3368,13 +3607,16 @@ void intel_mtl_tbt_pll_disable(struct intel_encoder *encoder)
 }
 
 void intel_mtl_pll_disable(struct intel_encoder *encoder)
+{
+	intel_cx0pll_disable(encoder);
+}
+
+void intel_mtl_pll_disable_clock(struct intel_encoder *encoder)
 {
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 
 	if (intel_tc_port_in_tbt_alt_mode(dig_port))
-		intel_mtl_tbt_pll_disable(encoder);
-	else
-		intel_cx0pll_disable(encoder);
+		intel_mtl_tbt_pll_disable_clock(encoder);
 }
 
 enum icl_port_dpll_id
@@ -3398,50 +3640,20 @@ intel_mtl_port_pll_type(struct intel_encoder *encoder,
 		return ICL_PORT_DPLL_DEFAULT;
 }
 
-static void intel_c10pll_state_verify(const struct intel_crtc_state *state,
-				      struct intel_crtc *crtc,
-				      struct intel_encoder *encoder,
-				      struct intel_c10pll_state *mpllb_hw_state)
-{
-	struct intel_display *display = to_intel_display(state);
-	const struct intel_c10pll_state *mpllb_sw_state = &state->dpll_hw_state.cx0pll.c10;
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(mpllb_sw_state->pll); i++) {
-		u8 expected = mpllb_sw_state->pll[i];
-
-		INTEL_DISPLAY_STATE_WARN(display, mpllb_hw_state->pll[i] != expected,
-					 "[CRTC:%d:%s] mismatch in C10MPLLB: Register[%d] (expected 0x%02x, found 0x%02x)",
-					 crtc->base.base.id, crtc->base.name, i,
-					 expected, mpllb_hw_state->pll[i]);
-	}
-
-	INTEL_DISPLAY_STATE_WARN(display, mpllb_hw_state->tx != mpllb_sw_state->tx,
-				 "[CRTC:%d:%s] mismatch in C10MPLLB: Register TX0 (expected 0x%02x, found 0x%02x)",
-				 crtc->base.base.id, crtc->base.name,
-				 mpllb_sw_state->tx, mpllb_hw_state->tx);
-
-	INTEL_DISPLAY_STATE_WARN(display, mpllb_hw_state->cmn != mpllb_sw_state->cmn,
-				 "[CRTC:%d:%s] mismatch in C10MPLLB: Register CMN0 (expected 0x%02x, found 0x%02x)",
-				 crtc->base.base.id, crtc->base.name,
-				 mpllb_sw_state->cmn, mpllb_hw_state->cmn);
-}
-
-void intel_cx0pll_readout_hw_state(struct intel_encoder *encoder,
+bool intel_cx0pll_readout_hw_state(struct intel_encoder *encoder,
 				   struct intel_cx0pll_state *pll_state)
 {
-	pll_state->use_c10 = false;
+	memset(pll_state, 0, sizeof(*pll_state));
 
-	pll_state->tbt_mode = intel_tc_port_in_tbt_alt_mode(enc_to_dig_port(encoder));
-	if (pll_state->tbt_mode)
-		return;
+	if (!intel_cx0_pll_is_enabled(encoder))
+		return false;
 
-	if (intel_encoder_is_c10phy(encoder)) {
-		intel_c10pll_readout_hw_state(encoder, &pll_state->c10);
-		pll_state->use_c10 = true;
-	} else {
-		intel_c20pll_readout_hw_state(encoder, &pll_state->c20);
-	}
+	if (intel_encoder_is_c10phy(encoder))
+		intel_c10pll_readout_hw_state(encoder, pll_state);
+	else
+		intel_c20pll_readout_hw_state(encoder, pll_state);
+
+	return true;
 }
 
 static bool mtl_compare_hw_state_c10(const struct intel_c10pll_state *a,
@@ -3505,91 +3717,6 @@ int intel_cx0pll_calc_port_clock(struct intel_encoder *encoder,
 	return intel_c20pll_calc_port_clock(encoder, &pll_state->c20);
 }
 
-static void intel_c20pll_state_verify(const struct intel_crtc_state *state,
-				      struct intel_crtc *crtc,
-				      struct intel_encoder *encoder,
-				      struct intel_c20pll_state *mpll_hw_state)
-{
-	struct intel_display *display = to_intel_display(state);
-	const struct intel_c20pll_state *mpll_sw_state = &state->dpll_hw_state.cx0pll.c20;
-	bool sw_use_mpllb = intel_c20phy_use_mpllb(mpll_sw_state);
-	bool hw_use_mpllb = intel_c20phy_use_mpllb(mpll_hw_state);
-	int clock = intel_c20pll_calc_port_clock(encoder, mpll_sw_state);
-	int i;
-
-	INTEL_DISPLAY_STATE_WARN(display, mpll_hw_state->clock != clock,
-				 "[CRTC:%d:%s] mismatch in C20: Register CLOCK (expected %d, found %d)",
-				 crtc->base.base.id, crtc->base.name,
-				 mpll_sw_state->clock, mpll_hw_state->clock);
-
-	INTEL_DISPLAY_STATE_WARN(display, sw_use_mpllb != hw_use_mpllb,
-				 "[CRTC:%d:%s] mismatch in C20: Register MPLLB selection (expected %d, found %d)",
-				 crtc->base.base.id, crtc->base.name,
-				 sw_use_mpllb, hw_use_mpllb);
-
-	if (hw_use_mpllb) {
-		for (i = 0; i < ARRAY_SIZE(mpll_sw_state->mpllb); i++) {
-			INTEL_DISPLAY_STATE_WARN(display, mpll_hw_state->mpllb[i] != mpll_sw_state->mpllb[i],
-						 "[CRTC:%d:%s] mismatch in C20MPLLB: Register[%d] (expected 0x%04x, found 0x%04x)",
-						 crtc->base.base.id, crtc->base.name, i,
-						 mpll_sw_state->mpllb[i], mpll_hw_state->mpllb[i]);
-		}
-	} else {
-		for (i = 0; i < ARRAY_SIZE(mpll_sw_state->mplla); i++) {
-			INTEL_DISPLAY_STATE_WARN(display, mpll_hw_state->mplla[i] != mpll_sw_state->mplla[i],
-						 "[CRTC:%d:%s] mismatch in C20MPLLA: Register[%d] (expected 0x%04x, found 0x%04x)",
-						 crtc->base.base.id, crtc->base.name, i,
-						 mpll_sw_state->mplla[i], mpll_hw_state->mplla[i]);
-		}
-	}
-
-	for (i = 0; i < ARRAY_SIZE(mpll_sw_state->tx); i++) {
-		INTEL_DISPLAY_STATE_WARN(display, mpll_hw_state->tx[i] != mpll_sw_state->tx[i],
-					 "[CRTC:%d:%s] mismatch in C20: Register TX[%i] (expected 0x%04x, found 0x%04x)",
-					 crtc->base.base.id, crtc->base.name, i,
-					 mpll_sw_state->tx[i], mpll_hw_state->tx[i]);
-	}
-
-	for (i = 0; i < ARRAY_SIZE(mpll_sw_state->cmn); i++) {
-		INTEL_DISPLAY_STATE_WARN(display, mpll_hw_state->cmn[i] != mpll_sw_state->cmn[i],
-					 "[CRTC:%d:%s] mismatch in C20: Register CMN[%i] (expected 0x%04x, found 0x%04x)",
-					 crtc->base.base.id, crtc->base.name, i,
-					 mpll_sw_state->cmn[i], mpll_hw_state->cmn[i]);
-	}
-}
-
-void intel_cx0pll_state_verify(struct intel_atomic_state *state,
-			       struct intel_crtc *crtc)
-{
-	struct intel_display *display = to_intel_display(state);
-	const struct intel_crtc_state *new_crtc_state =
-		intel_atomic_get_new_crtc_state(state, crtc);
-	struct intel_encoder *encoder;
-	struct intel_cx0pll_state mpll_hw_state = {};
-
-	if (!IS_DISPLAY_VER(display, 14, 30))
-		return;
-
-	if (!new_crtc_state->hw.active)
-		return;
-
-	/* intel_get_crtc_new_encoder() only works for modeset/fastset commits */
-	if (!intel_crtc_needs_modeset(new_crtc_state) &&
-	    !intel_crtc_needs_fastset(new_crtc_state))
-		return;
-
-	encoder = intel_get_crtc_new_encoder(state, new_crtc_state);
-	intel_cx0pll_readout_hw_state(encoder, &mpll_hw_state);
-
-	if (mpll_hw_state.tbt_mode)
-		return;
-
-	if (intel_encoder_is_c10phy(encoder))
-		intel_c10pll_state_verify(new_crtc_state, crtc, encoder, &mpll_hw_state.c10);
-	else
-		intel_c20pll_state_verify(new_crtc_state, crtc, encoder, &mpll_hw_state.c20);
-}
-
 /*
  * WA 14022081154
  * The dedicated display PHYs reset to a power state that blocks S0ix, increasing idle
@@ -3613,6 +3740,7 @@ void intel_cx0_pll_power_save_wa(struct intel_display *display)
 	for_each_intel_encoder(display->drm, encoder) {
 		struct intel_cx0pll_state pll_state = {};
 		int port_clock = 162000;
+		int lane_count = 4;
 
 		if (!intel_encoder_is_dig_port(encoder))
 			continue;
@@ -3625,7 +3753,7 @@ void intel_cx0_pll_power_save_wa(struct intel_display *display)
 
 		if (intel_c10pll_calc_state_from_table(encoder,
 						       mtl_c10_edp_tables,
-						       true, port_clock,
+						       true, port_clock, lane_count,
 						       &pll_state) < 0) {
 			drm_WARN_ON(display->drm,
 				    "Unable to calc C10 state from the tables\n");
@@ -3636,7 +3764,7 @@ void intel_cx0_pll_power_save_wa(struct intel_display *display)
 			    "[ENCODER:%d:%s] Applying power saving workaround on disabled PLL\n",
 			    encoder->base.base.id, encoder->base.name);
 
-		__intel_cx0pll_enable(encoder, &pll_state, true, port_clock, 4);
+		intel_cx0pll_enable(encoder, &pll_state);
 		intel_cx0pll_disable(encoder);
 	}
 }
diff --git a/drivers/gpu/drm/i915/display/intel_cx0_phy.h b/drivers/gpu/drm/i915/display/intel_cx0_phy.h
index 84d334b865f7..9f10113e2d18 100644
--- a/drivers/gpu/drm/i915/display/intel_cx0_phy.h
+++ b/drivers/gpu/drm/i915/display/intel_cx0_phy.h
@@ -11,6 +11,7 @@
 #define MB_WRITE_COMMITTED      true
 #define MB_WRITE_UNCOMMITTED    false
 
+struct drm_printer;
 enum icl_port_dpll_id;
 struct intel_atomic_state;
 struct intel_c10pll_state;
@@ -19,6 +20,8 @@ struct intel_crtc;
 struct intel_crtc_state;
 struct intel_cx0pll_state;
 struct intel_display;
+struct intel_dpll;
+struct intel_dpll_hw_state;
 struct intel_encoder;
 struct intel_hdmi;
 
@@ -26,22 +29,30 @@ void intel_clear_response_ready_flag(struct intel_encoder *encoder,
 				     int lane);
 bool intel_encoder_is_c10phy(struct intel_encoder *encoder);
 void intel_mtl_pll_enable(struct intel_encoder *encoder,
-			  const struct intel_crtc_state *crtc_state);
+			  struct intel_dpll *pll,
+			  const struct intel_dpll_hw_state *dpll_hw_state);
 void intel_mtl_pll_disable(struct intel_encoder *encoder);
 enum icl_port_dpll_id
 intel_mtl_port_pll_type(struct intel_encoder *encoder,
 			const struct intel_crtc_state *crtc_state);
+void intel_mtl_pll_enable_clock(struct intel_encoder *encoder,
+				const struct intel_crtc_state *crtc_state);
+void intel_mtl_pll_disable_clock(struct intel_encoder *encoder);
+void intel_mtl_pll_disable_clock(struct intel_encoder *encoder);
+void intel_mtl_tbt_pll_enable_clock(struct intel_encoder *encoder,
+				    int port_clock);
+void intel_mtl_tbt_pll_disable_clock(struct intel_encoder *encoder);
 
-int intel_cx0pll_calc_state(struct intel_crtc_state *crtc_state, struct intel_encoder *encoder);
-void intel_cx0pll_readout_hw_state(struct intel_encoder *encoder,
+int intel_cx0pll_calc_state(const struct intel_crtc_state *crtc_state,
+			    struct intel_encoder *encoder,
+			    struct intel_dpll_hw_state *hw_state);
+bool intel_cx0pll_readout_hw_state(struct intel_encoder *encoder,
 				   struct intel_cx0pll_state *pll_state);
 int intel_cx0pll_calc_port_clock(struct intel_encoder *encoder,
 				 const struct intel_cx0pll_state *pll_state);
 
-void intel_cx0pll_dump_hw_state(struct intel_display *display,
+void intel_cx0pll_dump_hw_state(struct drm_printer *p,
 				const struct intel_cx0pll_state *hw_state);
-void intel_cx0pll_state_verify(struct intel_atomic_state *state,
-			       struct intel_crtc *crtc);
 bool intel_cx0pll_compare_hw_state(const struct intel_cx0pll_state *a,
 				   const struct intel_cx0pll_state *b);
 void intel_cx0_phy_set_signal_levels(struct intel_encoder *encoder,
@@ -59,7 +70,13 @@ void intel_cx0_write(struct intel_encoder *encoder,
 int intel_cx0_wait_for_ack(struct intel_encoder *encoder,
 			   int command, int lane, u32 *val);
 void intel_cx0_bus_reset(struct intel_encoder *encoder, int lane);
+
+void intel_mtl_tbt_pll_calc_state(struct intel_dpll_hw_state *hw_state);
+bool intel_mtl_tbt_pll_readout_hw_state(struct intel_display *display,
+					struct intel_dpll *pll,
+					struct intel_dpll_hw_state *hw_state);
 int intel_mtl_tbt_calc_port_clock(struct intel_encoder *encoder);
+
 void intel_cx0_pll_power_save_wa(struct intel_display *display);
 void intel_lnl_mac_transmit_lfps(struct intel_encoder *encoder,
 				 const struct intel_crtc_state *crtc_state);
diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c
index 002ccd47856d..cb91d07cdaa6 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi.c
+++ b/drivers/gpu/drm/i915/display/intel_ddi.c
@@ -89,6 +89,8 @@
 #include "skl_scaler.h"
 #include "skl_universal_plane.h"
 
+struct intel_dpll;
+
 static const u8 index_to_dp_signal_levels[] = {
 	[0] = DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_0,
 	[1] = DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_1,
@@ -726,7 +728,7 @@ int intel_ddi_toggle_hdcp_bits(struct intel_encoder *intel_encoder,
 			       bool enable, u32 hdcp_mask)
 {
 	struct intel_display *display = to_intel_display(intel_encoder);
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	int ret = 0;
 
 	wakeref = intel_display_power_get_if_enabled(display,
@@ -747,7 +749,7 @@ bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector)
 	int type = intel_connector->base.connector_type;
 	enum port port = encoder->port;
 	enum transcoder cpu_transcoder;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	enum pipe pipe = 0;
 	u32 ddi_mode;
 	bool ret;
@@ -803,7 +805,7 @@ static void intel_ddi_get_encoder_pipes(struct intel_encoder *encoder,
 {
 	struct intel_display *display = to_intel_display(encoder);
 	enum port port = encoder->port;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	enum pipe p;
 	u32 tmp;
 	u8 mst_pipe_mask = 0, dp128b132b_pipe_mask = 0;
@@ -846,7 +848,7 @@ static void intel_ddi_get_encoder_pipes(struct intel_encoder *encoder,
 	for_each_pipe(display, p) {
 		enum transcoder cpu_transcoder = (enum transcoder)p;
 		u32 port_mask, ddi_select, ddi_mode;
-		intel_wakeref_t trans_wakeref;
+		struct ref_tracker *trans_wakeref;
 
 		trans_wakeref = intel_display_power_get_if_enabled(display,
 								   POWER_DOMAIN_TRANSCODER(cpu_transcoder));
@@ -1000,7 +1002,7 @@ main_link_aux_power_domain_put(struct intel_digital_port *dig_port,
 	struct intel_display *display = to_intel_display(dig_port);
 	enum intel_display_power_domain domain =
 		intel_ddi_main_link_aux_domain(dig_port, crtc_state);
-	intel_wakeref_t wf;
+	struct ref_tracker *wf;
 
 	wf = fetch_and_zero(&dig_port->aux_wakeref);
 	if (!wf)
@@ -2446,7 +2448,7 @@ static void intel_ddi_enable_fec(struct intel_encoder *encoder,
 			return;
 	}
 
-	drm_err(display->drm, "Failed to enable FEC after retries\n");
+	drm_dbg_kms(display->drm, "Failed to enable FEC after retries\n");
 }
 
 static void intel_ddi_disable_fec(struct intel_encoder *encoder,
@@ -3128,7 +3130,7 @@ static void intel_ddi_post_disable_dp(struct intel_atomic_state *state,
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 	struct intel_dp *intel_dp = &dig_port->dp;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool is_mst = intel_crtc_has_type(old_crtc_state,
 					  INTEL_OUTPUT_DP_MST);
 
@@ -3196,7 +3198,7 @@ static void intel_ddi_post_disable_hdmi(struct intel_atomic_state *state,
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 	struct intel_hdmi *intel_hdmi = &dig_port->hdmi;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 
 	dig_port->set_infoframes(encoder, false,
 				 old_crtc_state, old_conn_state);
@@ -3667,8 +3669,8 @@ void intel_ddi_update_active_dpll(struct intel_atomic_state *state,
 		intel_atomic_get_new_crtc_state(state, crtc);
 	struct intel_crtc *pipe_crtc;
 
-	/* FIXME: Add MTL pll_mgr */
-	if (DISPLAY_VER(display) >= 14 || !intel_encoder_is_tc(encoder))
+	/* FIXME: Add NVL+ and DG2 pll_mgr */
+	if (!intel_encoder_is_tc(encoder) || !display->dpll.mgr)
 		return;
 
 	for_each_intel_crtc_in_pipe_mask(display->drm, pipe_crtc,
@@ -3963,7 +3965,7 @@ static void bdw_get_trans_port_sync_config(struct intel_crtc_state *crtc_state)
 
 	for_each_cpu_transcoder_masked(display, cpu_transcoder, transcoders) {
 		enum intel_display_power_domain power_domain;
-		intel_wakeref_t trans_wakeref;
+		struct ref_tracker *trans_wakeref;
 
 		power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
 		trans_wakeref = intel_display_power_get_if_enabled(display,
@@ -4255,19 +4257,70 @@ static void xe3plpd_ddi_get_config(struct intel_encoder *encoder,
 	intel_ddi_get_config(encoder, crtc_state);
 }
 
-static void mtl_ddi_get_config(struct intel_encoder *encoder,
-			       struct intel_crtc_state *crtc_state)
+static bool icl_ddi_tc_pll_is_tbt(const struct intel_dpll *pll)
 {
-	intel_cx0pll_readout_hw_state(encoder, &crtc_state->dpll_hw_state.cx0pll);
+	return pll->info->id == DPLL_ID_ICL_TBTPLL;
+}
 
-	if (crtc_state->dpll_hw_state.cx0pll.tbt_mode)
+static void mtl_ddi_cx0_get_config(struct intel_encoder *encoder,
+				   struct intel_crtc_state *crtc_state,
+				   enum icl_port_dpll_id port_dpll_id,
+				   enum intel_dpll_id pll_id)
+{
+	struct intel_display *display = to_intel_display(encoder);
+	struct icl_port_dpll *port_dpll;
+	struct intel_dpll *pll;
+	bool pll_active;
+
+	port_dpll = &crtc_state->icl_port_dplls[port_dpll_id];
+	pll = intel_get_dpll_by_id(display, pll_id);
+
+	if (drm_WARN_ON(display->drm, !pll))
+		return;
+
+	port_dpll->pll = pll;
+	pll_active = intel_dpll_get_hw_state(display, pll, &port_dpll->hw_state);
+	drm_WARN_ON(display->drm, !pll_active);
+
+	icl_set_active_port_dpll(crtc_state, port_dpll_id);
+
+	if (icl_ddi_tc_pll_is_tbt(crtc_state->intel_dpll))
 		crtc_state->port_clock = intel_mtl_tbt_calc_port_clock(encoder);
 	else
-		crtc_state->port_clock = intel_cx0pll_calc_port_clock(encoder, &crtc_state->dpll_hw_state.cx0pll);
+		crtc_state->port_clock = intel_dpll_get_freq(display, crtc_state->intel_dpll,
+							     &crtc_state->dpll_hw_state);
 
 	intel_ddi_get_config(encoder, crtc_state);
 }
 
+/*
+ * Get the configuration for either a port using a C10 PHY PLL, or a port using a
+ * C20 PHY PLL in the cases of:
+ * - BMG port A/B
+ * - PTL port B eDP over TypeC PHY
+ */
+static void mtl_ddi_non_tc_phy_get_config(struct intel_encoder *encoder,
+					     struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(encoder);
+
+	mtl_ddi_cx0_get_config(encoder, crtc_state, ICL_PORT_DPLL_DEFAULT,
+			       mtl_port_to_pll_id(display, encoder->port));
+}
+
+static void mtl_ddi_tc_phy_get_config(struct intel_encoder *encoder,
+				      struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(encoder);
+
+	if (intel_tc_port_in_tbt_alt_mode(enc_to_dig_port(encoder)))
+		mtl_ddi_cx0_get_config(encoder, crtc_state, ICL_PORT_DPLL_DEFAULT,
+				       DPLL_ID_ICL_TBTPLL);
+	else
+		mtl_ddi_cx0_get_config(encoder, crtc_state, ICL_PORT_DPLL_MG_PHY,
+				       mtl_port_to_pll_id(display, encoder->port));
+}
+
 static void dg2_ddi_get_config(struct intel_encoder *encoder,
 				struct intel_crtc_state *crtc_state)
 {
@@ -4305,11 +4358,6 @@ static void icl_ddi_combo_get_config(struct intel_encoder *encoder,
 	intel_ddi_get_config(encoder, crtc_state);
 }
 
-static bool icl_ddi_tc_pll_is_tbt(const struct intel_dpll *pll)
-{
-	return pll->info->id == DPLL_ID_ICL_TBTPLL;
-}
-
 static enum icl_port_dpll_id
 icl_ddi_tc_port_pll_type(struct intel_encoder *encoder,
 			 const struct intel_crtc_state *crtc_state)
@@ -5100,7 +5148,7 @@ static const char *intel_ddi_encoder_name(struct intel_display *display,
 			       port_name(port - PORT_D_XELPD + PORT_D),
 			       phy_name(phy));
 	} else if (DISPLAY_VER(display) >= 12) {
-		enum tc_port tc_port = intel_port_to_tc(display, port);
+		enum tc_port tc_port = intel_tc_phy_port_to_tc(display, port);
 
 		seq_buf_printf(s, "DDI %s%c/PHY %s%c",
 			       port >= PORT_TC1 ? "TC" : "",
@@ -5108,7 +5156,7 @@ static const char *intel_ddi_encoder_name(struct intel_display *display,
 			       tc_port != TC_PORT_NONE ? "TC" : "",
 			       tc_port != TC_PORT_NONE ? tc_port_name(tc_port) : phy_name(phy));
 	} else if (DISPLAY_VER(display) >= 11) {
-		enum tc_port tc_port = intel_port_to_tc(display, port);
+		enum tc_port tc_port = intel_tc_phy_port_to_tc(display, port);
 
 		seq_buf_printf(s, "DDI %c%s/PHY %s%c",
 			       port_name(port),
@@ -5252,10 +5300,13 @@ void intel_ddi_init(struct intel_display *display,
 		encoder->port_pll_type = intel_mtl_port_pll_type;
 		encoder->get_config = xe3plpd_ddi_get_config;
 	} else if (DISPLAY_VER(display) >= 14) {
-		encoder->enable_clock = intel_mtl_pll_enable;
-		encoder->disable_clock = intel_mtl_pll_disable;
-		encoder->port_pll_type = intel_mtl_port_pll_type;
-		encoder->get_config = mtl_ddi_get_config;
+		encoder->enable_clock = intel_mtl_pll_enable_clock;
+		encoder->disable_clock = intel_mtl_pll_disable_clock;
+		encoder->port_pll_type = icl_ddi_tc_port_pll_type;
+		if (intel_encoder_is_tc(encoder))
+			encoder->get_config = mtl_ddi_tc_phy_get_config;
+		else
+			encoder->get_config = mtl_ddi_non_tc_phy_get_config;
 	} else if (display->platform.dg2) {
 		encoder->enable_clock = intel_mpllb_enable;
 		encoder->disable_clock = intel_mpllb_disable;
@@ -5372,6 +5423,17 @@ void intel_ddi_init(struct intel_display *display,
 			goto err;
 	}
 
+	/*
+	 * FIXME: We currently need to store dedicated_external because devdata
+	 * does not live long enough for when intel_encoder_is_tc() is called on
+	 * the unbind path.  This needs to be fixed by making sure that the VBT
+	 * data is kept long enough, so that
+	 * intel_bios_encoder_is_dedicated_external() can be called directly
+	 * from intel_encoder_is_tc().
+	 */
+	if (intel_bios_encoder_is_dedicated_external(devdata))
+		dig_port->dedicated_external = true;
+
 	if (intel_encoder_is_tc(encoder)) {
 		bool is_legacy =
 			!intel_bios_encoder_supports_typec_usb(devdata) &&
diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c
index 095a319f8bc9..d5947cc9b94c 100644
--- a/drivers/gpu/drm/i915/display/intel_display.c
+++ b/drivers/gpu/drm/i915/display/intel_display.c
@@ -372,7 +372,7 @@ void assert_transcoder(struct intel_display *display,
 {
 	bool cur_state;
 	enum intel_display_power_domain power_domain;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 
 	/* we keep both pipes enabled on 830 */
 	if (display->platform.i830)
@@ -1810,7 +1810,17 @@ bool intel_phy_is_combo(struct intel_display *display, enum phy phy)
 		return false;
 }
 
-/* Prefer intel_encoder_is_tc() */
+/*
+ * This function returns true if the DDI port respective to the PHY enumeration
+ * is a Type-C capable port.
+ *
+ * Depending on the VBT, the port might be configured
+ * as a "dedicated external" port, meaning that actual physical PHY is outside
+ * of the Type-C subsystem and, as such, not really a "Type-C PHY".
+ *
+ * Prefer intel_encoder_is_tc(), especially if you really need to know if we
+ * are dealing with Type-C connections.
+ */
 bool intel_phy_is_tc(struct intel_display *display, enum phy phy)
 {
 	/*
@@ -1859,17 +1869,32 @@ enum phy intel_port_to_phy(struct intel_display *display, enum port port)
 }
 
 /* Prefer intel_encoder_to_tc() */
+/*
+ * Return TC_PORT_1..I915_MAX_TC_PORTS for any TypeC DDI port. The function
+ * can be also called for TypeC DDI ports not connected to a TypeC PHY such as
+ * the PORT_TC1..4 ports on RKL/ADLS/BMG.
+ */
 enum tc_port intel_port_to_tc(struct intel_display *display, enum port port)
 {
-	if (!intel_phy_is_tc(display, intel_port_to_phy(display, port)))
-		return TC_PORT_NONE;
-
 	if (DISPLAY_VER(display) >= 12)
 		return TC_PORT_1 + port - PORT_TC1;
 	else
 		return TC_PORT_1 + port - PORT_C;
 }
 
+/*
+ * Return TC_PORT_1..I915_MAX_TC_PORTS for TypeC DDI ports connected to a TypeC PHY.
+ * Note that on RKL, ADLS, BMG the PORT_TC1..4 ports are connected to a non-TypeC
+ * PHY, so on those platforms the function returns TC_PORT_NONE.
+ */
+enum tc_port intel_tc_phy_port_to_tc(struct intel_display *display, enum port port)
+{
+	if (!intel_phy_is_tc(display, intel_port_to_phy(display, port)))
+		return TC_PORT_NONE;
+
+	return intel_port_to_tc(display, port);
+}
+
 enum phy intel_encoder_to_phy(struct intel_encoder *encoder)
 {
 	struct intel_display *display = to_intel_display(encoder);
@@ -1894,6 +1919,10 @@ bool intel_encoder_is_snps(struct intel_encoder *encoder)
 bool intel_encoder_is_tc(struct intel_encoder *encoder)
 {
 	struct intel_display *display = to_intel_display(encoder);
+	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
+
+	if (dig_port && dig_port->dedicated_external)
+		return false;
 
 	return intel_phy_is_tc(display, intel_encoder_to_phy(encoder));
 }
@@ -1902,7 +1931,7 @@ enum tc_port intel_encoder_to_tc(struct intel_encoder *encoder)
 {
 	struct intel_display *display = to_intel_display(encoder);
 
-	return intel_port_to_tc(display, encoder->port);
+	return intel_tc_phy_port_to_tc(display, encoder->port);
 }
 
 enum intel_display_power_domain
@@ -3020,7 +3049,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 	struct intel_display *display = to_intel_display(crtc);
 	enum intel_display_power_domain power_domain;
 	enum transcoder cpu_transcoder = (enum transcoder)crtc->pipe;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret = false;
 	u32 tmp;
 
@@ -3364,7 +3393,7 @@ static bool ilk_get_pipe_config(struct intel_crtc *crtc,
 	struct intel_display *display = to_intel_display(crtc);
 	enum intel_display_power_domain power_domain;
 	enum transcoder cpu_transcoder = (enum transcoder)crtc->pipe;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret = false;
 	u32 tmp;
 
@@ -3454,12 +3483,11 @@ static bool transcoder_ddi_func_is_enabled(struct intel_display *display,
 					   enum transcoder cpu_transcoder)
 {
 	enum intel_display_power_domain power_domain;
-	intel_wakeref_t wakeref;
 	u32 tmp = 0;
 
 	power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
 
-	with_intel_display_power_if_enabled(display, power_domain, wakeref)
+	with_intel_display_power_if_enabled(display, power_domain)
 		tmp = intel_de_read(display,
 				    TRANS_DDI_FUNC_CTL(display, cpu_transcoder));
 
@@ -3481,10 +3509,9 @@ static void enabled_uncompressed_joiner_pipes(struct intel_display *display,
 					 joiner_pipes(display)) {
 		enum intel_display_power_domain power_domain;
 		enum pipe pipe = crtc->pipe;
-		intel_wakeref_t wakeref;
 
 		power_domain = POWER_DOMAIN_PIPE(pipe);
-		with_intel_display_power_if_enabled(display, power_domain, wakeref) {
+		with_intel_display_power_if_enabled(display, power_domain) {
 			u32 tmp = intel_de_read(display, ICL_PIPE_DSS_CTL1(pipe));
 
 			if (tmp & UNCOMPRESSED_JOINER_PRIMARY)
@@ -3510,10 +3537,9 @@ static void enabled_bigjoiner_pipes(struct intel_display *display,
 					 joiner_pipes(display)) {
 		enum intel_display_power_domain power_domain;
 		enum pipe pipe = crtc->pipe;
-		intel_wakeref_t wakeref;
 
 		power_domain = intel_dsc_power_domain(crtc, (enum transcoder)pipe);
-		with_intel_display_power_if_enabled(display, power_domain, wakeref) {
+		with_intel_display_power_if_enabled(display, power_domain) {
 			u32 tmp = intel_de_read(display, ICL_PIPE_DSS_CTL1(pipe));
 
 			if (!(tmp & BIG_JOINER_ENABLE))
@@ -3580,10 +3606,9 @@ static void enabled_ultrajoiner_pipes(struct intel_display *display,
 					 joiner_pipes(display)) {
 		enum intel_display_power_domain power_domain;
 		enum pipe pipe = crtc->pipe;
-		intel_wakeref_t wakeref;
 
 		power_domain = intel_dsc_power_domain(crtc, (enum transcoder)pipe);
-		with_intel_display_power_if_enabled(display, power_domain, wakeref) {
+		with_intel_display_power_if_enabled(display, power_domain) {
 			u32 tmp = intel_de_read(display, ICL_PIPE_DSS_CTL1(pipe));
 
 			if (!(tmp & ULTRA_JOINER_ENABLE))
@@ -3741,12 +3766,11 @@ static u8 hsw_enabled_transcoders(struct intel_crtc *crtc)
 	for_each_cpu_transcoder_masked(display, cpu_transcoder,
 				       panel_transcoder_mask) {
 		enum intel_display_power_domain power_domain;
-		intel_wakeref_t wakeref;
 		enum pipe trans_pipe;
 		u32 tmp = 0;
 
 		power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
-		with_intel_display_power_if_enabled(display, power_domain, wakeref)
+		with_intel_display_power_if_enabled(display, power_domain)
 			tmp = intel_de_read(display,
 					    TRANS_DDI_FUNC_CTL(display, cpu_transcoder));
 
@@ -4977,24 +5001,6 @@ pipe_config_pll_mismatch(struct drm_printer *p, bool fastset,
 	intel_dpll_dump_hw_state(display, p, b);
 }
 
-static void
-pipe_config_cx0pll_mismatch(struct drm_printer *p, bool fastset,
-			    const struct intel_crtc *crtc,
-			    const char *name,
-			    const struct intel_cx0pll_state *a,
-			    const struct intel_cx0pll_state *b)
-{
-	struct intel_display *display = to_intel_display(crtc);
-	char *chipname = a->use_c10 ? "C10" : "C20";
-
-	pipe_config_mismatch(p, fastset, crtc, name, chipname);
-
-	drm_printf(p, "expected:\n");
-	intel_cx0pll_dump_hw_state(display, a);
-	drm_printf(p, "found:\n");
-	intel_cx0pll_dump_hw_state(display, b);
-}
-
 static bool allow_vblank_delay_fastset(const struct intel_crtc_state *old_crtc_state)
 {
 	struct intel_display *display = to_intel_display(old_crtc_state);
@@ -5146,16 +5152,6 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 	} \
 } while (0)
 
-#define PIPE_CONF_CHECK_PLL_CX0(name) do { \
-	if (!intel_cx0pll_compare_hw_state(&current_config->name, \
-					   &pipe_config->name)) { \
-		pipe_config_cx0pll_mismatch(&p, fastset, crtc, __stringify(name), \
-					    &current_config->name, \
-					    &pipe_config->name); \
-		ret = false; \
-	} \
-} while (0)
-
 #define PIPE_CONF_CHECK_PLL_LT(name) do { \
 	if (!intel_lt_phy_pll_compare_hw_state(&current_config->name, \
 					       &pipe_config->name)) { \
@@ -5395,8 +5391,6 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 	/* FIXME convert MTL+ platforms over to dpll_mgr */
 	if (HAS_LT_PHY(display))
 		PIPE_CONF_CHECK_PLL_LT(dpll_hw_state.ltpll);
-	else if (DISPLAY_VER(display) >= 14)
-		PIPE_CONF_CHECK_PLL_CX0(dpll_hw_state.cx0pll);
 
 	PIPE_CONF_CHECK_X(dsi_pll.ctrl);
 	PIPE_CONF_CHECK_X(dsi_pll.div);
@@ -6032,14 +6026,6 @@ static int intel_async_flip_check_uapi(struct intel_atomic_state *state,
 		return -EINVAL;
 	}
 
-	/* FIXME: selective fetch should be disabled for async flips */
-	if (new_crtc_state->enable_psr2_sel_fetch) {
-		drm_dbg_kms(display->drm,
-			    "[CRTC:%d:%s] async flip disallowed with PSR2 selective fetch\n",
-			    crtc->base.base.id, crtc->base.name);
-		return -EINVAL;
-	}
-
 	for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
 					     new_plane_state, i) {
 		if (plane->pipe != crtc->pipe)
@@ -6130,7 +6116,7 @@ static int intel_async_flip_check_hw(struct intel_atomic_state *state, struct in
 		if (!plane->async_flip)
 			continue;
 
-		if (!intel_plane_can_async_flip(plane, new_plane_state->hw.fb->format->format,
+		if (!intel_plane_can_async_flip(plane, new_plane_state->hw.fb->format,
 						new_plane_state->hw.fb->modifier)) {
 			drm_dbg_kms(display->drm,
 				    "[PLANE:%d:%s] pixel format %p4cc / modifier 0x%llx does not support async flip\n",
@@ -7399,7 +7385,7 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state)
 	struct intel_crtc_state *new_crtc_state, *old_crtc_state;
 	struct intel_crtc *crtc;
 	struct intel_power_domain_mask put_domains[I915_MAX_PIPES] = {};
-	intel_wakeref_t wakeref = NULL;
+	struct ref_tracker *wakeref = NULL;
 	int i;
 
 	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h
index bcc6ccb69d2b..f8e6e4e82722 100644
--- a/drivers/gpu/drm/i915/display/intel_display.h
+++ b/drivers/gpu/drm/i915/display/intel_display.h
@@ -451,6 +451,7 @@ bool intel_phy_is_combo(struct intel_display *display, enum phy phy);
 bool intel_phy_is_tc(struct intel_display *display, enum phy phy);
 bool intel_phy_is_snps(struct intel_display *display, enum phy phy);
 enum tc_port intel_port_to_tc(struct intel_display *display, enum port port);
+enum tc_port intel_tc_phy_port_to_tc(struct intel_display *display, enum port port);
 
 enum phy intel_encoder_to_phy(struct intel_encoder *encoder);
 bool intel_encoder_is_combo(struct intel_encoder *encoder);
diff --git a/drivers/gpu/drm/i915/display/intel_display_core.h b/drivers/gpu/drm/i915/display/intel_display_core.h
index 9b8414b77c15..d708d322aa85 100644
--- a/drivers/gpu/drm/i915/display/intel_display_core.h
+++ b/drivers/gpu/drm/i915/display/intel_display_core.h
@@ -386,7 +386,7 @@ struct intel_display {
 
 	struct {
 		struct intel_dmc *dmc;
-		intel_wakeref_t wakeref;
+		struct ref_tracker *wakeref;
 	} dmc;
 
 	struct {
@@ -394,6 +394,21 @@ struct intel_display {
 		u32 mmio_base;
 	} dsi;
 
+	struct {
+		const struct dram_info *info;
+	} dram;
+
+	struct {
+		struct intel_fbc *instances[I915_MAX_FBCS];
+
+		/* xe3p_lpd+: FBC instance utilizing the system cache */
+		struct sys_cache_cfg {
+			/* Protect concurrecnt access to system cache configuration */
+			struct mutex lock;
+			enum intel_fbc_id id;
+		} sys_cache;
+	} fbc;
+
 	struct {
 		/* list of fbdev register on this device */
 		struct intel_fbdev *fbdev;
@@ -611,7 +626,6 @@ struct intel_display {
 	struct drm_dp_tunnel_mgr *dp_tunnel_mgr;
 	struct intel_audio audio;
 	struct intel_dpll_global dpll;
-	struct intel_fbc *fbc[I915_MAX_FBCS];
 	struct intel_frontbuffer_tracking fb_tracking;
 	struct intel_hotplug hotplug;
 	struct intel_opregion *opregion;
diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
index 9bbfdae8d024..aba13e8a9051 100644
--- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c
+++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
@@ -86,7 +86,7 @@ static int i915_frontbuffer_tracking(struct seq_file *m, void *unused)
 static int i915_sr_status(struct seq_file *m, void *unused)
 {
 	struct intel_display *display = node_to_intel_display(m->private);
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool sr_enabled = false;
 
 	wakeref = intel_display_power_get(display, POWER_DOMAIN_INIT);
diff --git a/drivers/gpu/drm/i915/display/intel_display_device.c b/drivers/gpu/drm/i915/display/intel_display_device.c
index 1170afaa8680..471f236c9ddf 100644
--- a/drivers/gpu/drm/i915/display/intel_display_device.c
+++ b/drivers/gpu/drm/i915/display/intel_display_device.c
@@ -1420,6 +1420,10 @@ static const struct platform_desc ptl_desc = {
 	}
 };
 
+static const struct platform_desc nvl_desc = {
+	PLATFORM(novalake),
+};
+
 __diag_pop();
 
 /*
@@ -1495,6 +1499,7 @@ static const struct {
 	INTEL_BMG_IDS(INTEL_DISPLAY_DEVICE, &bmg_desc),
 	INTEL_PTL_IDS(INTEL_DISPLAY_DEVICE, &ptl_desc),
 	INTEL_WCL_IDS(INTEL_DISPLAY_DEVICE, &ptl_desc),
+	INTEL_NVLS_IDS(INTEL_DISPLAY_DEVICE, &nvl_desc),
 };
 
 static const struct {
diff --git a/drivers/gpu/drm/i915/display/intel_display_device.h b/drivers/gpu/drm/i915/display/intel_display_device.h
index b559ef43d547..50b2e9ae2c18 100644
--- a/drivers/gpu/drm/i915/display/intel_display_device.h
+++ b/drivers/gpu/drm/i915/display/intel_display_device.h
@@ -103,7 +103,9 @@ struct pci_dev;
 	func(battlemage) \
 	/* Display ver 30 (based on GMD ID) */ \
 	func(pantherlake) \
-	func(pantherlake_wildcatlake)
+	func(pantherlake_wildcatlake) \
+	/* Display ver 35 (based on GMD ID) */ \
+	func(novalake)
 
 
 #define __MEMBER(name) unsigned long name:1;
@@ -147,7 +149,7 @@ struct intel_display_platforms {
 #define HAS_4TILE(__display)		((__display)->platform.dg2 || DISPLAY_VER(__display) >= 14)
 #define HAS_ASYNC_FLIPS(__display)	(DISPLAY_VER(__display) >= 5)
 #define HAS_AS_SDP(__display)		(DISPLAY_VER(__display) >= 13)
-#define HAS_AUX_CCS(__display)		(IS_DISPLAY_VER(__display, 9, 12) || (__display)->platform.alderlake_p || (__display)->platform.meteorlake)
+#define HAS_AUX_DIST(__display)		(IS_DISPLAY_VER(__display, 9, 12) || (__display)->platform.alderlake_p || (__display)->platform.meteorlake)
 #define HAS_BIGJOINER(__display)	(DISPLAY_VER(__display) >= 11 && HAS_DSC(__display))
 #define HAS_CASF(__display)		(DISPLAY_VER(__display) >= 20)
 #define HAS_CDCLK_CRAWL(__display)	(DISPLAY_INFO(__display)->has_cdclk_crawl)
@@ -173,6 +175,7 @@ struct intel_display_platforms {
 #define HAS_DSC_MST(__display)		(DISPLAY_VER(__display) >= 12 && HAS_DSC(__display))
 #define HAS_FBC(__display)		(DISPLAY_RUNTIME_INFO(__display)->fbc_mask != 0)
 #define HAS_FBC_DIRTY_RECT(__display)	(DISPLAY_VER(__display) >= 30)
+#define HAS_FBC_SYS_CACHE(__display)	(DISPLAY_VER(__display) >= 35 && !(__display)->platform.dgfx)
 #define HAS_FPGA_DBG_UNCLAIMED(__display)	(DISPLAY_INFO(__display)->has_fpga_dbg)
 #define HAS_FW_BLC(__display)		(DISPLAY_VER(__display) >= 3)
 #define HAS_GMBUS_BURST_READ(__display)	(DISPLAY_VER(__display) >= 10 || (__display)->platform.kabylake)
@@ -185,6 +188,7 @@ struct intel_display_platforms {
 #define HAS_IPS(__display)		((__display)->platform.haswell_ult || (__display)->platform.broadwell)
 #define HAS_LRR(__display)		(DISPLAY_VER(__display) >= 12)
 #define HAS_LSPCON(__display)		(IS_DISPLAY_VER(__display, 9, 10))
+#define HAS_LT_PHY(__display)		((__display)->platform.novalake)
 #define HAS_MBUS_JOINING(__display)	((__display)->platform.alderlake_p || DISPLAY_VER(__display) >= 14)
 #define HAS_MSO(__display)		(DISPLAY_VER(__display) >= 12)
 #define HAS_OVERLAY(__display)		(DISPLAY_INFO(__display)->has_overlay)
@@ -197,6 +201,7 @@ struct intel_display_platforms {
 #define HAS_TRANSCODER(__display, trans)	((DISPLAY_RUNTIME_INFO(__display)->cpu_transcoder_mask & \
 						  BIT(trans)) != 0)
 #define HAS_UNCOMPRESSED_JOINER(__display)	(DISPLAY_VER(__display) >= 13)
+#define HAS_UNDERRUN_DBG_INFO(__display)	(DISPLAY_VER(__display) >= 35)
 #define HAS_ULTRAJOINER(__display)	(((__display)->platform.dgfx && \
 					  DISPLAY_VER(__display) == 14) && HAS_DSC(__display))
 #define HAS_VRR(__display)		(DISPLAY_VER(__display) >= 11)
diff --git a/drivers/gpu/drm/i915/display/intel_display_driver.c b/drivers/gpu/drm/i915/display/intel_display_driver.c
index 7e000ba3e08b..e1d29aea0ddc 100644
--- a/drivers/gpu/drm/i915/display/intel_display_driver.c
+++ b/drivers/gpu/drm/i915/display/intel_display_driver.c
@@ -199,12 +199,8 @@ void intel_display_driver_early_probe(struct intel_display *display)
 /* part #1: call before irq install */
 int intel_display_driver_probe_noirq(struct intel_display *display)
 {
-	struct drm_i915_private *i915 = to_i915(display->drm);
 	int ret;
 
-	if (i915_inject_probe_failure(i915))
-		return -ENODEV;
-
 	if (HAS_DISPLAY(display)) {
 		ret = drm_vblank_init(display->drm,
 				      INTEL_NUM_PIPES(display));
@@ -317,6 +313,7 @@ int intel_display_driver_probe_noirq(struct intel_display *display)
 
 	return ret;
 }
+ALLOW_ERROR_INJECTION(intel_display_driver_probe_noirq, ERRNO);
 
 static void set_display_access(struct intel_display *display,
 			       bool any_task_allowed,
@@ -452,7 +449,6 @@ bool intel_display_driver_check_access(struct intel_display *display)
 /* part #2: call after irq install, but before gem init */
 int intel_display_driver_probe_nogem(struct intel_display *display)
 {
-	enum pipe pipe;
 	int ret;
 
 	if (!HAS_DISPLAY(display))
@@ -466,15 +462,9 @@ int intel_display_driver_probe_nogem(struct intel_display *display)
 
 	intel_gmbus_setup(display);
 
-	drm_dbg_kms(display->drm, "%d display pipe%s available.\n",
-		    INTEL_NUM_PIPES(display),
-		    INTEL_NUM_PIPES(display) > 1 ? "s" : "");
-
-	for_each_pipe(display, pipe) {
-		ret = intel_crtc_init(display, pipe);
-		if (ret)
-			goto err_mode_config;
-	}
+	ret = intel_crtc_init(display);
+	if (ret)
+		goto err_mode_config;
 
 	intel_plane_possible_crtcs_init(display);
 	intel_dpll_init(display);
diff --git a/drivers/gpu/drm/i915/display/intel_display_irq.c b/drivers/gpu/drm/i915/display/intel_display_irq.c
index 43b27deb4a26..9adeebb376b1 100644
--- a/drivers/gpu/drm/i915/display/intel_display_irq.c
+++ b/drivers/gpu/drm/i915/display/intel_display_irq.c
@@ -6,8 +6,6 @@
 #include <drm/drm_print.h>
 #include <drm/drm_vblank.h>
 
-#include "i915_drv.h"
-#include "i915_irq.h"
 #include "i915_reg.h"
 #include "icl_dsi_regs.h"
 #include "intel_crtc.h"
@@ -19,57 +17,83 @@
 #include "intel_display_trace.h"
 #include "intel_display_types.h"
 #include "intel_dmc.h"
-#include "intel_dmc_wl.h"
 #include "intel_dp_aux.h"
 #include "intel_dsb.h"
 #include "intel_fdi_regs.h"
 #include "intel_fifo_underrun.h"
 #include "intel_gmbus.h"
 #include "intel_hotplug_irq.h"
+#include "intel_parent.h"
 #include "intel_pipe_crc_regs.h"
 #include "intel_plane.h"
 #include "intel_pmdemand.h"
 #include "intel_psr.h"
 #include "intel_psr_regs.h"
-#include "intel_uncore.h"
 
-static void
-intel_display_irq_regs_init(struct intel_display *display, struct i915_irq_regs regs,
-			    u32 imr_val, u32 ier_val)
+static void irq_reset(struct intel_display *display, struct i915_irq_regs regs)
 {
-	intel_dmc_wl_get(display, regs.imr);
-	intel_dmc_wl_get(display, regs.ier);
-	intel_dmc_wl_get(display, regs.iir);
+	intel_de_write(display, regs.imr, 0xffffffff);
+	intel_de_posting_read(display, regs.imr);
 
-	gen2_irq_init(to_intel_uncore(display->drm), regs, imr_val, ier_val);
+	intel_de_write(display, regs.ier, 0);
 
-	intel_dmc_wl_put(display, regs.iir);
-	intel_dmc_wl_put(display, regs.ier);
-	intel_dmc_wl_put(display, regs.imr);
+	/* IIR can theoretically queue up two events. Be paranoid. */
+	intel_de_write(display, regs.iir, 0xffffffff);
+	intel_de_posting_read(display, regs.iir);
+	intel_de_write(display, regs.iir, 0xffffffff);
+	intel_de_posting_read(display, regs.iir);
 }
 
-static void
-intel_display_irq_regs_reset(struct intel_display *display, struct i915_irq_regs regs)
+/*
+ * We should clear IMR at preinstall/uninstall, and just check at postinstall.
+ */
+static void assert_iir_is_zero(struct intel_display *display, i915_reg_t reg)
 {
-	intel_dmc_wl_get(display, regs.imr);
-	intel_dmc_wl_get(display, regs.ier);
-	intel_dmc_wl_get(display, regs.iir);
+	u32 val = intel_de_read(display, reg);
 
-	gen2_irq_reset(to_intel_uncore(display->drm), regs);
+	if (val == 0)
+		return;
 
-	intel_dmc_wl_put(display, regs.iir);
-	intel_dmc_wl_put(display, regs.ier);
-	intel_dmc_wl_put(display, regs.imr);
+	drm_WARN(display->drm, 1,
+		 "Interrupt register 0x%x is not zero: 0x%08x\n",
+		 i915_mmio_reg_offset(reg), val);
+	intel_de_write(display, reg, 0xffffffff);
+	intel_de_posting_read(display, reg);
+	intel_de_write(display, reg, 0xffffffff);
+	intel_de_posting_read(display, reg);
 }
 
-static void
-intel_display_irq_regs_assert_irr_is_zero(struct intel_display *display, i915_reg_t reg)
+static void irq_init(struct intel_display *display, struct i915_irq_regs regs,
+		     u32 imr_val, u32 ier_val)
 {
-	intel_dmc_wl_get(display, reg);
+	assert_iir_is_zero(display, regs.iir);
+
+	intel_de_write(display, regs.ier, ier_val);
+	intel_de_write(display, regs.imr, imr_val);
+	intel_de_posting_read(display, regs.imr);
+}
 
-	gen2_assert_iir_is_zero(to_intel_uncore(display->drm), reg);
+static void error_reset(struct intel_display *display, struct i915_error_regs regs)
+{
+	intel_de_write(display, regs.emr, 0xffffffff);
+	intel_de_posting_read(display, regs.emr);
 
-	intel_dmc_wl_put(display, reg);
+	intel_de_write(display, regs.eir, 0xffffffff);
+	intel_de_posting_read(display, regs.eir);
+	intel_de_write(display, regs.eir, 0xffffffff);
+	intel_de_posting_read(display, regs.eir);
+}
+
+static void error_init(struct intel_display *display, struct i915_error_regs regs,
+		       u32 emr_val)
+{
+	intel_de_write(display, regs.eir, 0xffffffff);
+	intel_de_posting_read(display, regs.eir);
+	intel_de_write(display, regs.eir, 0xffffffff);
+	intel_de_posting_read(display, regs.eir);
+
+	intel_de_write(display, regs.emr, emr_val);
+	intel_de_posting_read(display, regs.emr);
 }
 
 struct pipe_fault_handler {
@@ -135,7 +159,6 @@ intel_handle_vblank(struct intel_display *display, enum pipe pipe)
 void ilk_update_display_irq(struct intel_display *display,
 			    u32 interrupt_mask, u32 enabled_irq_mask)
 {
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
 	u32 new_val;
 
 	lockdep_assert_held(&display->irq.lock);
@@ -146,7 +169,7 @@ void ilk_update_display_irq(struct intel_display *display,
 	new_val |= (~enabled_irq_mask & interrupt_mask);
 
 	if (new_val != display->irq.ilk_de_imr_mask &&
-	    !drm_WARN_ON(display->drm, !intel_irqs_enabled(dev_priv))) {
+	    !drm_WARN_ON(display->drm, !intel_parent_irq_enabled(display))) {
 		display->irq.ilk_de_imr_mask = new_val;
 		intel_de_write(display, DEIMR, display->irq.ilk_de_imr_mask);
 		intel_de_posting_read(display, DEIMR);
@@ -172,7 +195,6 @@ void ilk_disable_display_irq(struct intel_display *display, u32 bits)
 void bdw_update_port_irq(struct intel_display *display,
 			 u32 interrupt_mask, u32 enabled_irq_mask)
 {
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
 	u32 new_val;
 	u32 old_val;
 
@@ -180,7 +202,7 @@ void bdw_update_port_irq(struct intel_display *display,
 
 	drm_WARN_ON(display->drm, enabled_irq_mask & ~interrupt_mask);
 
-	if (drm_WARN_ON(display->drm, !intel_irqs_enabled(dev_priv)))
+	if (drm_WARN_ON(display->drm, !intel_parent_irq_enabled(display)))
 		return;
 
 	old_val = intel_de_read(display, GEN8_DE_PORT_IMR);
@@ -206,14 +228,13 @@ static void bdw_update_pipe_irq(struct intel_display *display,
 				enum pipe pipe, u32 interrupt_mask,
 				u32 enabled_irq_mask)
 {
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
 	u32 new_val;
 
 	lockdep_assert_held(&display->irq.lock);
 
 	drm_WARN_ON(display->drm, enabled_irq_mask & ~interrupt_mask);
 
-	if (drm_WARN_ON(display->drm, !intel_irqs_enabled(dev_priv)))
+	if (drm_WARN_ON(display->drm, !intel_parent_irq_enabled(display)))
 		return;
 
 	new_val = display->irq.de_pipe_imr_mask[pipe];
@@ -249,7 +270,6 @@ void ibx_display_interrupt_update(struct intel_display *display,
 				  u32 interrupt_mask,
 				  u32 enabled_irq_mask)
 {
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
 	u32 sdeimr = intel_de_read(display, SDEIMR);
 
 	sdeimr &= ~interrupt_mask;
@@ -259,7 +279,7 @@ void ibx_display_interrupt_update(struct intel_display *display,
 
 	lockdep_assert_held(&display->irq.lock);
 
-	if (drm_WARN_ON(display->drm, !intel_irqs_enabled(dev_priv)))
+	if (drm_WARN_ON(display->drm, !intel_parent_irq_enabled(display)))
 		return;
 
 	intel_de_write(display, SDEIMR, sdeimr);
@@ -323,7 +343,6 @@ u32 i915_pipestat_enable_mask(struct intel_display *display,
 void i915_enable_pipestat(struct intel_display *display,
 			  enum pipe pipe, u32 status_mask)
 {
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
 	i915_reg_t reg = PIPESTAT(display, pipe);
 	u32 enable_mask;
 
@@ -332,7 +351,7 @@ void i915_enable_pipestat(struct intel_display *display,
 		      pipe_name(pipe), status_mask);
 
 	lockdep_assert_held(&display->irq.lock);
-	drm_WARN_ON(display->drm, !intel_irqs_enabled(dev_priv));
+	drm_WARN_ON(display->drm, !intel_parent_irq_enabled(display));
 
 	if ((display->irq.pipestat_irq_mask[pipe] & status_mask) == status_mask)
 		return;
@@ -347,7 +366,6 @@ void i915_enable_pipestat(struct intel_display *display,
 void i915_disable_pipestat(struct intel_display *display,
 			   enum pipe pipe, u32 status_mask)
 {
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
 	i915_reg_t reg = PIPESTAT(display, pipe);
 	u32 enable_mask;
 
@@ -356,7 +374,7 @@ void i915_disable_pipestat(struct intel_display *display,
 		      pipe_name(pipe), status_mask);
 
 	lockdep_assert_held(&display->irq.lock);
-	drm_WARN_ON(display->drm, !intel_irqs_enabled(dev_priv));
+	drm_WARN_ON(display->drm, !intel_parent_irq_enabled(display));
 
 	if ((display->irq.pipestat_irq_mask[pipe] & status_mask) == 0)
 		return;
@@ -1918,15 +1936,14 @@ static void _vlv_display_irq_reset(struct intel_display *display)
 	else
 		intel_de_write(display, DPINVGTT, DPINVGTT_STATUS_MASK_VLV);
 
-	gen2_error_reset(to_intel_uncore(display->drm),
-			 VLV_ERROR_REGS);
+	error_reset(display, VLV_ERROR_REGS);
 
 	i915_hotplug_interrupt_update_locked(display, 0xffffffff, 0);
 	intel_de_rmw(display, PORT_HOTPLUG_STAT(display), 0, 0);
 
 	i9xx_pipestat_irq_reset(display);
 
-	intel_display_irq_regs_reset(display, VLV_IRQ_REGS);
+	irq_reset(display, VLV_IRQ_REGS);
 	display->irq.vlv_imr_mask = ~0u;
 }
 
@@ -2014,8 +2031,7 @@ static void _vlv_display_irq_postinstall(struct intel_display *display)
 			       DPINVGTT_STATUS_MASK_VLV |
 			       DPINVGTT_EN_MASK_VLV);
 
-	gen2_error_init(to_intel_uncore(display->drm),
-			VLV_ERROR_REGS, ~vlv_error_mask());
+	error_init(display, VLV_ERROR_REGS, ~vlv_error_mask());
 
 	pipestat_mask = PIPE_CRC_DONE_INTERRUPT_STATUS;
 
@@ -2038,7 +2054,7 @@ static void _vlv_display_irq_postinstall(struct intel_display *display)
 
 	display->irq.vlv_imr_mask = ~enable_mask;
 
-	intel_display_irq_regs_init(display, VLV_IRQ_REGS, display->irq.vlv_imr_mask, enable_mask);
+	irq_init(display, VLV_IRQ_REGS, display->irq.vlv_imr_mask, enable_mask);
 }
 
 void vlv_display_irq_postinstall(struct intel_display *display)
@@ -2054,7 +2070,7 @@ static void ibx_display_irq_reset(struct intel_display *display)
 	if (HAS_PCH_NOP(display))
 		return;
 
-	gen2_irq_reset(to_intel_uncore(display->drm), SDE_IRQ_REGS);
+	irq_reset(display, SDE_IRQ_REGS);
 
 	if (HAS_PCH_CPT(display) || HAS_PCH_LPT(display))
 		intel_de_write(display, SERR_INT, 0xffffffff);
@@ -2062,9 +2078,7 @@ static void ibx_display_irq_reset(struct intel_display *display)
 
 void ilk_display_irq_reset(struct intel_display *display)
 {
-	struct intel_uncore *uncore = to_intel_uncore(display->drm);
-
-	gen2_irq_reset(uncore, DE_IRQ_REGS);
+	irq_reset(display, DE_IRQ_REGS);
 	display->irq.ilk_de_imr_mask = ~0u;
 
 	if (DISPLAY_VER(display) == 7)
@@ -2091,10 +2105,10 @@ void gen8_display_irq_reset(struct intel_display *display)
 	for_each_pipe(display, pipe)
 		if (intel_display_power_is_enabled(display,
 						   POWER_DOMAIN_PIPE(pipe)))
-			intel_display_irq_regs_reset(display, GEN8_DE_PIPE_IRQ_REGS(pipe));
+			irq_reset(display, GEN8_DE_PIPE_IRQ_REGS(pipe));
 
-	intel_display_irq_regs_reset(display, GEN8_DE_PORT_IRQ_REGS);
-	intel_display_irq_regs_reset(display, GEN8_DE_MISC_IRQ_REGS);
+	irq_reset(display, GEN8_DE_PORT_IRQ_REGS);
+	irq_reset(display, GEN8_DE_MISC_IRQ_REGS);
 
 	if (HAS_PCH_SPLIT(display))
 		ibx_display_irq_reset(display);
@@ -2136,39 +2150,38 @@ void gen11_display_irq_reset(struct intel_display *display)
 	for_each_pipe(display, pipe)
 		if (intel_display_power_is_enabled(display,
 						   POWER_DOMAIN_PIPE(pipe)))
-			intel_display_irq_regs_reset(display, GEN8_DE_PIPE_IRQ_REGS(pipe));
+			irq_reset(display, GEN8_DE_PIPE_IRQ_REGS(pipe));
 
-	intel_display_irq_regs_reset(display, GEN8_DE_PORT_IRQ_REGS);
-	intel_display_irq_regs_reset(display, GEN8_DE_MISC_IRQ_REGS);
+	irq_reset(display, GEN8_DE_PORT_IRQ_REGS);
+	irq_reset(display, GEN8_DE_MISC_IRQ_REGS);
 
 	if (DISPLAY_VER(display) >= 14)
-		intel_display_irq_regs_reset(display, PICAINTERRUPT_IRQ_REGS);
+		irq_reset(display, PICAINTERRUPT_IRQ_REGS);
 	else
-		intel_display_irq_regs_reset(display, GEN11_DE_HPD_IRQ_REGS);
+		irq_reset(display, GEN11_DE_HPD_IRQ_REGS);
 
 	if (INTEL_PCH_TYPE(display) >= PCH_ICP)
-		intel_display_irq_regs_reset(display, SDE_IRQ_REGS);
+		irq_reset(display, SDE_IRQ_REGS);
 }
 
 void gen8_irq_power_well_post_enable(struct intel_display *display,
 				     u8 pipe_mask)
 {
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
 	u32 extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN |
 		gen8_de_pipe_flip_done_mask(display);
 	enum pipe pipe;
 
 	spin_lock_irq(&display->irq.lock);
 
-	if (!intel_irqs_enabled(dev_priv)) {
+	if (!intel_parent_irq_enabled(display)) {
 		spin_unlock_irq(&display->irq.lock);
 		return;
 	}
 
 	for_each_pipe_masked(display, pipe, pipe_mask)
-		intel_display_irq_regs_init(display, GEN8_DE_PIPE_IRQ_REGS(pipe),
-					    display->irq.de_pipe_imr_mask[pipe],
-					    ~display->irq.de_pipe_imr_mask[pipe] | extra_ier);
+		irq_init(display, GEN8_DE_PIPE_IRQ_REGS(pipe),
+			 display->irq.de_pipe_imr_mask[pipe],
+			 ~display->irq.de_pipe_imr_mask[pipe] | extra_ier);
 
 	spin_unlock_irq(&display->irq.lock);
 }
@@ -2176,23 +2189,22 @@ void gen8_irq_power_well_post_enable(struct intel_display *display,
 void gen8_irq_power_well_pre_disable(struct intel_display *display,
 				     u8 pipe_mask)
 {
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
 	enum pipe pipe;
 
 	spin_lock_irq(&display->irq.lock);
 
-	if (!intel_irqs_enabled(dev_priv)) {
+	if (!intel_parent_irq_enabled(display)) {
 		spin_unlock_irq(&display->irq.lock);
 		return;
 	}
 
 	for_each_pipe_masked(display, pipe, pipe_mask)
-		intel_display_irq_regs_reset(display, GEN8_DE_PIPE_IRQ_REGS(pipe));
+		irq_reset(display, GEN8_DE_PIPE_IRQ_REGS(pipe));
 
 	spin_unlock_irq(&display->irq.lock);
 
 	/* make sure we're done processing display irqs */
-	intel_synchronize_irq(dev_priv);
+	intel_parent_irq_synchronize(display);
 }
 
 /*
@@ -2220,13 +2232,11 @@ static void ibx_irq_postinstall(struct intel_display *display)
 	else
 		mask = SDE_GMBUS_CPT;
 
-	intel_display_irq_regs_init(display, SDE_IRQ_REGS, ~mask, 0xffffffff);
+	irq_init(display, SDE_IRQ_REGS, ~mask, 0xffffffff);
 }
 
 void valleyview_enable_display_irqs(struct intel_display *display)
 {
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
-
 	spin_lock_irq(&display->irq.lock);
 
 	if (display->irq.vlv_display_irqs_enabled)
@@ -2234,7 +2244,7 @@ void valleyview_enable_display_irqs(struct intel_display *display)
 
 	display->irq.vlv_display_irqs_enabled = true;
 
-	if (intel_irqs_enabled(dev_priv)) {
+	if (intel_parent_irq_enabled(display)) {
 		_vlv_display_irq_reset(display);
 		_vlv_display_irq_postinstall(display);
 	}
@@ -2245,8 +2255,6 @@ void valleyview_enable_display_irqs(struct intel_display *display)
 
 void valleyview_disable_display_irqs(struct intel_display *display)
 {
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
-
 	spin_lock_irq(&display->irq.lock);
 
 	if (!display->irq.vlv_display_irqs_enabled)
@@ -2254,7 +2262,7 @@ void valleyview_disable_display_irqs(struct intel_display *display)
 
 	display->irq.vlv_display_irqs_enabled = false;
 
-	if (intel_irqs_enabled(dev_priv))
+	if (intel_parent_irq_enabled(display))
 		_vlv_display_irq_reset(display);
 out:
 	spin_unlock_irq(&display->irq.lock);
@@ -2286,7 +2294,7 @@ void ilk_de_irq_postinstall(struct intel_display *display)
 	}
 
 	if (display->platform.haswell) {
-		intel_display_irq_regs_assert_irr_is_zero(display, EDP_PSR_IIR);
+		assert_iir_is_zero(display, EDP_PSR_IIR);
 		display_mask |= DE_EDP_PSR_INT_HSW;
 	}
 
@@ -2297,8 +2305,8 @@ void ilk_de_irq_postinstall(struct intel_display *display)
 
 	ibx_irq_postinstall(display);
 
-	intel_display_irq_regs_init(display, DE_IRQ_REGS, display->irq.ilk_de_imr_mask,
-				    display_mask | extra_mask);
+	irq_init(display, DE_IRQ_REGS, display->irq.ilk_de_imr_mask,
+		 display_mask | extra_mask);
 }
 
 static void mtp_irq_postinstall(struct intel_display *display);
@@ -2374,11 +2382,10 @@ void gen8_de_irq_postinstall(struct intel_display *display)
 			if (!intel_display_power_is_enabled(display, domain))
 				continue;
 
-			intel_display_irq_regs_assert_irr_is_zero(display,
-								  TRANS_PSR_IIR(display, trans));
+			assert_iir_is_zero(display, TRANS_PSR_IIR(display, trans));
 		}
 	} else {
-		intel_display_irq_regs_assert_irr_is_zero(display, EDP_PSR_IIR);
+		assert_iir_is_zero(display, EDP_PSR_IIR);
 	}
 
 	for_each_pipe(display, pipe) {
@@ -2386,44 +2393,50 @@ void gen8_de_irq_postinstall(struct intel_display *display)
 
 		if (intel_display_power_is_enabled(display,
 						   POWER_DOMAIN_PIPE(pipe)))
-			intel_display_irq_regs_init(display, GEN8_DE_PIPE_IRQ_REGS(pipe),
-						    display->irq.de_pipe_imr_mask[pipe],
-						    de_pipe_enables);
+			irq_init(display, GEN8_DE_PIPE_IRQ_REGS(pipe),
+				 display->irq.de_pipe_imr_mask[pipe],
+				 de_pipe_enables);
 	}
 
-	intel_display_irq_regs_init(display, GEN8_DE_PORT_IRQ_REGS, ~de_port_masked,
-				    de_port_enables);
-	intel_display_irq_regs_init(display, GEN8_DE_MISC_IRQ_REGS, ~de_misc_masked,
-				    de_misc_masked);
+	irq_init(display, GEN8_DE_PORT_IRQ_REGS, ~de_port_masked, de_port_enables);
+	irq_init(display, GEN8_DE_MISC_IRQ_REGS, ~de_misc_masked, de_misc_masked);
 
 	if (IS_DISPLAY_VER(display, 11, 13)) {
 		u32 de_hpd_masked = 0;
 		u32 de_hpd_enables = GEN11_DE_TC_HOTPLUG_MASK |
 				     GEN11_DE_TBT_HOTPLUG_MASK;
 
-		intel_display_irq_regs_init(display, GEN11_DE_HPD_IRQ_REGS, ~de_hpd_masked,
-					    de_hpd_enables);
+		irq_init(display, GEN11_DE_HPD_IRQ_REGS, ~de_hpd_masked, de_hpd_enables);
 	}
 }
 
+u32 xelpdp_pica_aux_mask(struct intel_display *display)
+{
+	u32 mask = XELPDP_AUX_TC_MASK;
+
+	if (DISPLAY_VER(display) >= 20)
+		mask |= XE2LPD_AUX_DDI_MASK;
+
+	return mask;
+}
+
 static void mtp_irq_postinstall(struct intel_display *display)
 {
 	u32 sde_mask = SDE_GMBUS_ICP | SDE_PICAINTERRUPT;
-	u32 de_hpd_mask = XELPDP_AUX_TC_MASK;
+	u32 de_hpd_mask = xelpdp_pica_aux_mask(display);
 	u32 de_hpd_enables = de_hpd_mask | XELPDP_DP_ALT_HOTPLUG_MASK |
 			     XELPDP_TBT_HOTPLUG_MASK;
 
-	intel_display_irq_regs_init(display, PICAINTERRUPT_IRQ_REGS, ~de_hpd_mask,
-				    de_hpd_enables);
+	irq_init(display, PICAINTERRUPT_IRQ_REGS, ~de_hpd_mask, de_hpd_enables);
 
-	intel_display_irq_regs_init(display, SDE_IRQ_REGS, ~sde_mask, 0xffffffff);
+	irq_init(display, SDE_IRQ_REGS, ~sde_mask, 0xffffffff);
 }
 
 static void icp_irq_postinstall(struct intel_display *display)
 {
 	u32 mask = SDE_GMBUS_ICP;
 
-	intel_display_irq_regs_init(display, SDE_IRQ_REGS, ~mask, 0xffffffff);
+	irq_init(display, SDE_IRQ_REGS, ~mask, 0xffffffff);
 }
 
 void gen11_de_irq_postinstall(struct intel_display *display)
diff --git a/drivers/gpu/drm/i915/display/intel_display_irq.h b/drivers/gpu/drm/i915/display/intel_display_irq.h
index 84acd31948cf..b25d180254d7 100644
--- a/drivers/gpu/drm/i915/display/intel_display_irq.h
+++ b/drivers/gpu/drm/i915/display/intel_display_irq.h
@@ -16,6 +16,8 @@ struct drm_printer;
 struct intel_display;
 struct intel_display_irq_snapshot;
 
+u32 xelpdp_pica_aux_mask(struct intel_display *display);
+
 void valleyview_enable_display_irqs(struct intel_display *display);
 void valleyview_disable_display_irqs(struct intel_display *display);
 
diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c
index 2a4cc1dcc293..9f323c39d798 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power.c
@@ -8,10 +8,7 @@
 
 #include <drm/drm_print.h>
 
-#include "soc/intel_dram.h"
-
 #include "i915_drv.h"
-#include "i915_irq.h"
 #include "i915_reg.h"
 #include "intel_backlight_regs.h"
 #include "intel_cdclk.h"
@@ -26,7 +23,9 @@
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
 #include "intel_dmc.h"
+#include "intel_dram.h"
 #include "intel_mchbar_regs.h"
+#include "intel_parent.h"
 #include "intel_pch_refclk.h"
 #include "intel_pcode.h"
 #include "intel_pmdemand.h"
@@ -545,8 +544,8 @@ __intel_display_power_get_domain(struct intel_display *display,
  * Any power domain reference obtained by this function must have a symmetric
  * call to intel_display_power_put() to release the reference again.
  */
-intel_wakeref_t intel_display_power_get(struct intel_display *display,
-					enum intel_display_power_domain domain)
+struct ref_tracker *intel_display_power_get(struct intel_display *display,
+					    enum intel_display_power_domain domain)
 {
 	struct i915_power_domains *power_domains = &display->power.domains;
 	struct ref_tracker *wakeref;
@@ -572,7 +571,7 @@ intel_wakeref_t intel_display_power_get(struct intel_display *display,
  * Any power domain reference obtained by this function must have a symmetric
  * call to intel_display_power_put() to release the reference again.
  */
-intel_wakeref_t
+struct ref_tracker *
 intel_display_power_get_if_enabled(struct intel_display *display,
 				   enum intel_display_power_domain domain)
 {
@@ -639,7 +638,7 @@ static void __intel_display_power_put(struct intel_display *display,
 
 static void
 queue_async_put_domains_work(struct i915_power_domains *power_domains,
-			     intel_wakeref_t wakeref,
+			     struct ref_tracker *wakeref,
 			     int delay_ms)
 {
 	struct intel_display *display = container_of(power_domains,
@@ -741,7 +740,7 @@ intel_display_power_put_async_work(struct work_struct *work)
  */
 void __intel_display_power_put_async(struct intel_display *display,
 				     enum intel_display_power_domain domain,
-				     intel_wakeref_t wakeref,
+				     struct ref_tracker *wakeref,
 				     int delay_ms)
 {
 	struct i915_power_domains *power_domains = &display->power.domains;
@@ -800,7 +799,7 @@ void intel_display_power_flush_work(struct intel_display *display)
 {
 	struct i915_power_domains *power_domains = &display->power.domains;
 	struct intel_power_domain_mask async_put_mask;
-	intel_wakeref_t work_wakeref;
+	struct ref_tracker *work_wakeref;
 
 	mutex_lock(&power_domains->lock);
 
@@ -854,7 +853,7 @@ intel_display_power_flush_work_sync(struct intel_display *display)
  */
 void intel_display_power_put(struct intel_display *display,
 			     enum intel_display_power_domain domain,
-			     intel_wakeref_t wakeref)
+			     struct ref_tracker *wakeref)
 {
 	__intel_display_power_put(display, domain);
 	intel_display_rpm_put(display, wakeref);
@@ -886,7 +885,7 @@ intel_display_power_get_in_set(struct intel_display *display,
 			       struct intel_display_power_domain_set *power_domain_set,
 			       enum intel_display_power_domain domain)
 {
-	intel_wakeref_t __maybe_unused wf;
+	struct ref_tracker *__maybe_unused wf;
 
 	drm_WARN_ON(display->drm, test_bit(domain, power_domain_set->mask.bits));
 
@@ -902,7 +901,7 @@ intel_display_power_get_in_set_if_enabled(struct intel_display *display,
 					  struct intel_display_power_domain_set *power_domain_set,
 					  enum intel_display_power_domain domain)
 {
-	intel_wakeref_t wf;
+	struct ref_tracker *wf;
 
 	drm_WARN_ON(display->drm, test_bit(domain, power_domain_set->mask.bits));
 
@@ -929,7 +928,7 @@ intel_display_power_put_mask_in_set(struct intel_display *display,
 		    !bitmap_subset(mask->bits, power_domain_set->mask.bits, POWER_DOMAIN_NUM));
 
 	for_each_power_domain(domain, mask) {
-		intel_wakeref_t __maybe_unused wf = INTEL_WAKEREF_DEF;
+		struct ref_tracker *__maybe_unused wf = INTEL_WAKEREF_DEF;
 
 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
 		wf = fetch_and_zero(&power_domain_set->wakerefs[domain]);
@@ -1202,7 +1201,6 @@ static void hsw_assert_cdclk(struct intel_display *display)
 
 static void assert_can_disable_lcpll(struct intel_display *display)
 {
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
 	struct intel_crtc *crtc;
 
 	for_each_intel_crtc(display->drm, crtc)
@@ -1247,7 +1245,7 @@ static void assert_can_disable_lcpll(struct intel_display *display)
 	 * gen-specific and since we only disable LCPLL after we fully disable
 	 * the interrupts, the check below should be enough.
 	 */
-	INTEL_DISPLAY_STATE_WARN(display, intel_irqs_enabled(dev_priv),
+	INTEL_DISPLAY_STATE_WARN(display, intel_parent_irq_enabled(display),
 				 "IRQs enabled\n");
 }
 
@@ -1417,8 +1415,6 @@ static void hsw_enable_pc8(struct intel_display *display)
 
 static void hsw_disable_pc8(struct intel_display *display)
 {
-	struct drm_i915_private __maybe_unused *dev_priv = to_i915(display->drm);
-
 	drm_dbg_kms(display->drm, "Disabling package C8+\n");
 
 	hsw_restore_lcpll(display);
@@ -1426,7 +1422,7 @@ static void hsw_disable_pc8(struct intel_display *display)
 
 	/* Many display registers don't survive PC8+ */
 #ifdef I915 /* FIXME */
-	intel_clock_gating_init(dev_priv);
+	intel_clock_gating_init(display->drm);
 #endif
 }
 
@@ -1618,7 +1614,7 @@ static const struct buddy_page_mask wa_1409767108_buddy_page_masks[] = {
 
 static void tgl_bw_buddy_init(struct intel_display *display)
 {
-	const struct dram_info *dram_info = intel_dram_info(display->drm);
+	const struct dram_info *dram_info = intel_dram_info(display);
 	const struct buddy_page_mask *table;
 	unsigned long abox_mask = DISPLAY_INFO(display)->abox_mask;
 	int config, i;
@@ -2006,7 +2002,7 @@ void intel_power_domains_init_hw(struct intel_display *display, bool resume)
  */
 void intel_power_domains_driver_remove(struct intel_display *display)
 {
-	intel_wakeref_t wakeref __maybe_unused =
+	struct ref_tracker *wakeref __maybe_unused =
 		fetch_and_zero(&display->power.domains.init_wakeref);
 
 	/* Remove the refcount we took to keep power well support disabled. */
@@ -2067,7 +2063,7 @@ void intel_power_domains_sanitize_state(struct intel_display *display)
  */
 void intel_power_domains_enable(struct intel_display *display)
 {
-	intel_wakeref_t wakeref __maybe_unused =
+	struct ref_tracker *wakeref __maybe_unused =
 		fetch_and_zero(&display->power.domains.init_wakeref);
 
 	intel_display_power_put(display, POWER_DOMAIN_INIT, wakeref);
@@ -2106,7 +2102,7 @@ void intel_power_domains_disable(struct intel_display *display)
 void intel_power_domains_suspend(struct intel_display *display, bool s2idle)
 {
 	struct i915_power_domains *power_domains = &display->power.domains;
-	intel_wakeref_t wakeref __maybe_unused =
+	struct ref_tracker *wakeref __maybe_unused =
 		fetch_and_zero(&power_domains->init_wakeref);
 
 	intel_display_power_put(display, POWER_DOMAIN_INIT, wakeref);
diff --git a/drivers/gpu/drm/i915/display/intel_display_power.h b/drivers/gpu/drm/i915/display/intel_display_power.h
index f8813b0e16df..d616d5d09cbe 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power.h
+++ b/drivers/gpu/drm/i915/display/intel_display_power.h
@@ -9,15 +9,17 @@
 #include <linux/mutex.h>
 #include <linux/workqueue.h>
 
-#include "intel_wakeref.h"
-
 enum aux_ch;
 enum port;
 struct i915_power_well;
 struct intel_display;
 struct intel_encoder;
+struct ref_tracker;
 struct seq_file;
 
+/* -ENOENT means we got the ref, but there's no tracking */
+#define INTEL_WAKEREF_DEF ERR_PTR(-ENOENT)
+
 /*
  * Keep the pipe, transcoder, port (DDI_LANES,DDI_IO,AUX) domain instances
  * consecutive, so that the pipe,transcoder,port -> power domain macros
@@ -142,14 +144,14 @@ struct i915_power_domains {
 	u32 target_dc_state;
 	u32 allowed_dc_mask;
 
-	intel_wakeref_t init_wakeref;
-	intel_wakeref_t disable_wakeref;
+	struct ref_tracker *init_wakeref;
+	struct ref_tracker *disable_wakeref;
 
 	struct mutex lock;
 	int domain_use_count[POWER_DOMAIN_NUM];
 
 	struct delayed_work async_put_work;
-	intel_wakeref_t async_put_wakeref;
+	struct ref_tracker *async_put_wakeref;
 	struct intel_power_domain_mask async_put_domains[2];
 	int async_put_next_delay;
 
@@ -159,7 +161,7 @@ struct i915_power_domains {
 struct intel_display_power_domain_set {
 	struct intel_power_domain_mask mask;
 #ifdef CONFIG_DRM_I915_DEBUG_RUNTIME_PM
-	intel_wakeref_t wakerefs[POWER_DOMAIN_NUM];
+	struct ref_tracker *wakerefs[POWER_DOMAIN_NUM];
 #endif
 };
 
@@ -187,24 +189,24 @@ u32 intel_display_power_get_current_dc_state(struct intel_display *display);
 
 bool intel_display_power_is_enabled(struct intel_display *display,
 				    enum intel_display_power_domain domain);
-intel_wakeref_t intel_display_power_get(struct intel_display *display,
-					enum intel_display_power_domain domain);
-intel_wakeref_t
+struct ref_tracker *intel_display_power_get(struct intel_display *display,
+					    enum intel_display_power_domain domain);
+struct ref_tracker *
 intel_display_power_get_if_enabled(struct intel_display *display,
 				   enum intel_display_power_domain domain);
 void __intel_display_power_put_async(struct intel_display *display,
 				     enum intel_display_power_domain domain,
-				     intel_wakeref_t wakeref,
+				     struct ref_tracker *wakeref,
 				     int delay_ms);
 void intel_display_power_flush_work(struct intel_display *display);
 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
 void intel_display_power_put(struct intel_display *display,
 			     enum intel_display_power_domain domain,
-			     intel_wakeref_t wakeref);
+			     struct ref_tracker *wakeref);
 static inline void
 intel_display_power_put_async(struct intel_display *display,
 			      enum intel_display_power_domain domain,
-			      intel_wakeref_t wakeref)
+			      struct ref_tracker *wakeref)
 {
 	__intel_display_power_put_async(display, domain, wakeref, -1);
 }
@@ -212,7 +214,7 @@ intel_display_power_put_async(struct intel_display *display,
 static inline void
 intel_display_power_put_async_delay(struct intel_display *display,
 				    enum intel_display_power_domain domain,
-				    intel_wakeref_t wakeref,
+				    struct ref_tracker *wakeref,
 				    int delay_ms)
 {
 	__intel_display_power_put_async(display, domain, wakeref, delay_ms);
@@ -224,7 +226,7 @@ void intel_display_power_put_unchecked(struct intel_display *display,
 static inline void
 intel_display_power_put(struct intel_display *display,
 			enum intel_display_power_domain domain,
-			intel_wakeref_t wakeref)
+			struct ref_tracker *wakeref)
 {
 	intel_display_power_put_unchecked(display, domain);
 }
@@ -232,7 +234,7 @@ intel_display_power_put(struct intel_display *display,
 static inline void
 intel_display_power_put_async(struct intel_display *display,
 			      enum intel_display_power_domain domain,
-			      intel_wakeref_t wakeref)
+			      struct ref_tracker *wakeref)
 {
 	__intel_display_power_put_async(display, domain, INTEL_WAKEREF_DEF, -1);
 }
@@ -240,7 +242,7 @@ intel_display_power_put_async(struct intel_display *display,
 static inline void
 intel_display_power_put_async_delay(struct intel_display *display,
 				    enum intel_display_power_domain domain,
-				    intel_wakeref_t wakeref,
+				    struct ref_tracker *wakeref,
 				    int delay_ms)
 {
 	__intel_display_power_put_async(display, domain, INTEL_WAKEREF_DEF, delay_ms);
@@ -297,12 +299,18 @@ enum dbuf_slice {
 void gen9_dbuf_slices_update(struct intel_display *display,
 			     u8 req_slices);
 
-#define with_intel_display_power(display, domain, wf) \
-	for ((wf) = intel_display_power_get((display), (domain)); (wf); \
+#define __with_intel_display_power(display, domain, wf) \
+	for (struct ref_tracker *(wf) = intel_display_power_get((display), (domain)); (wf); \
 	     intel_display_power_put_async((display), (domain), (wf)), (wf) = NULL)
 
-#define with_intel_display_power_if_enabled(display, domain, wf) \
-	for ((wf) = intel_display_power_get_if_enabled((display), (domain)); (wf); \
+#define with_intel_display_power(display, domain) \
+	__with_intel_display_power(display, domain, __UNIQUE_ID(wakeref))
+
+#define __with_intel_display_power_if_enabled(display, domain, wf) \
+	for (struct ref_tracker *(wf) = intel_display_power_get_if_enabled((display), (domain)); (wf); \
 	     intel_display_power_put_async((display), (domain), (wf)), (wf) = NULL)
 
+#define with_intel_display_power_if_enabled(display, domain) \
+	__with_intel_display_power_if_enabled(display, domain, __UNIQUE_ID(wakeref))
+
 #endif /* __INTEL_DISPLAY_POWER_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.c b/drivers/gpu/drm/i915/display/intel_display_power_well.c
index f4f7e73acc87..db185a859133 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power_well.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power_well.c
@@ -7,8 +7,6 @@
 
 #include <drm/drm_print.h>
 
-#include "i915_drv.h"
-#include "i915_irq.h"
 #include "i915_reg.h"
 #include "intel_backlight_regs.h"
 #include "intel_combo_phy.h"
@@ -28,6 +26,7 @@
 #include "intel_dpio_phy.h"
 #include "intel_dpll.h"
 #include "intel_hotplug.h"
+#include "intel_parent.h"
 #include "intel_pcode.h"
 #include "intel_pps.h"
 #include "intel_psr.h"
@@ -258,8 +257,9 @@ aux_ch_to_digital_port(struct intel_display *display,
 	return NULL;
 }
 
-static enum phy icl_aux_pw_to_phy(struct intel_display *display,
-				  const struct i915_power_well *power_well)
+static struct intel_encoder *
+icl_aux_pw_to_encoder(struct intel_display *display,
+		      const struct i915_power_well *power_well)
 {
 	enum aux_ch aux_ch = icl_aux_pw_to_ch(power_well);
 	struct intel_digital_port *dig_port = aux_ch_to_digital_port(display, aux_ch);
@@ -271,7 +271,23 @@ static enum phy icl_aux_pw_to_phy(struct intel_display *display,
 	 * as HDMI-only and routed to a combo PHY, the encoder either won't be
 	 * present at all or it will not have an aux_ch assigned.
 	 */
-	return dig_port ? intel_encoder_to_phy(&dig_port->base) : PHY_NONE;
+	return dig_port ? &dig_port->base : NULL;
+}
+
+static enum phy icl_aux_pw_to_phy(struct intel_display *display,
+				  const struct i915_power_well *power_well)
+{
+	struct intel_encoder *encoder = icl_aux_pw_to_encoder(display, power_well);
+
+	return encoder ? intel_encoder_to_phy(encoder) : PHY_NONE;
+}
+
+static bool icl_aux_pw_is_tc_phy(struct intel_display *display,
+				 const struct i915_power_well *power_well)
+{
+	struct intel_encoder *encoder = icl_aux_pw_to_encoder(display, power_well);
+
+	return encoder && intel_encoder_is_tc(encoder);
 }
 
 static void hsw_wait_for_power_well_enable(struct intel_display *display,
@@ -570,9 +586,7 @@ static void
 icl_aux_power_well_enable(struct intel_display *display,
 			  struct i915_power_well *power_well)
 {
-	enum phy phy = icl_aux_pw_to_phy(display, power_well);
-
-	if (intel_phy_is_tc(display, phy))
+	if (icl_aux_pw_is_tc_phy(display, power_well))
 		return icl_tc_phy_aux_power_well_enable(display, power_well);
 	else if (display->platform.icelake)
 		return icl_combo_phy_aux_power_well_enable(display,
@@ -585,9 +599,7 @@ static void
 icl_aux_power_well_disable(struct intel_display *display,
 			   struct i915_power_well *power_well)
 {
-	enum phy phy = icl_aux_pw_to_phy(display, power_well);
-
-	if (intel_phy_is_tc(display, phy))
+	if (icl_aux_pw_is_tc_phy(display, power_well))
 		return hsw_power_well_disable(display, power_well);
 	else if (display->platform.icelake)
 		return icl_combo_phy_aux_power_well_disable(display,
@@ -628,8 +640,6 @@ static bool hsw_power_well_enabled(struct intel_display *display,
 
 static void assert_can_enable_dc9(struct intel_display *display)
 {
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
-
 	drm_WARN_ONCE(display->drm,
 		      (intel_de_read(display, DC_STATE_EN) & DC_STATE_EN_DC9),
 		      "DC9 already programmed to be enabled.\n");
@@ -641,7 +651,7 @@ static void assert_can_enable_dc9(struct intel_display *display)
 		      intel_de_read(display, HSW_PWR_WELL_CTL2) &
 		      HSW_PWR_WELL_CTL_REQ(SKL_PW_CTL_IDX_PW_2),
 		      "Power well 2 on.\n");
-	drm_WARN_ONCE(display->drm, intel_irqs_enabled(dev_priv),
+	drm_WARN_ONCE(display->drm, intel_parent_irq_enabled(display),
 		      "Interrupts not disabled yet.\n");
 
 	 /*
@@ -655,9 +665,7 @@ static void assert_can_enable_dc9(struct intel_display *display)
 
 static void assert_can_disable_dc9(struct intel_display *display)
 {
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
-
-	drm_WARN_ONCE(display->drm, intel_irqs_enabled(dev_priv),
+	drm_WARN_ONCE(display->drm, intel_parent_irq_enabled(display),
 		      "Interrupts not disabled yet.\n");
 	drm_WARN_ONCE(display->drm,
 		      intel_de_read(display, DC_STATE_EN) &
@@ -1281,12 +1289,10 @@ static void vlv_display_power_well_init(struct intel_display *display)
 
 static void vlv_display_power_well_deinit(struct intel_display *display)
 {
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
-
 	valleyview_disable_display_irqs(display);
 
 	/* make sure we're done processing display irqs */
-	intel_synchronize_irq(dev_priv);
+	intel_parent_irq_synchronize(display);
 
 	vlv_pps_reset_all(display);
 
@@ -1852,7 +1858,7 @@ static void xelpdp_aux_power_well_enable(struct intel_display *display,
 	enum aux_ch aux_ch = i915_power_well_instance(power_well)->xelpdp.aux_ch;
 	enum phy phy = icl_aux_pw_to_phy(display, power_well);
 
-	if (intel_phy_is_tc(display, phy))
+	if (icl_aux_pw_is_tc_phy(display, power_well))
 		icl_tc_port_assert_ref_held(display, power_well,
 					    aux_ch_to_digital_port(display, aux_ch));
 
@@ -1860,19 +1866,19 @@ static void xelpdp_aux_power_well_enable(struct intel_display *display,
 		     XELPDP_DP_AUX_CH_CTL_POWER_REQUEST,
 		     XELPDP_DP_AUX_CH_CTL_POWER_REQUEST);
 
-	/*
-	 * The power status flag cannot be used to determine whether aux
-	 * power wells have finished powering up.  Instead we're
-	 * expected to just wait a fixed 600us after raising the request
-	 * bit.
-	 */
-	if (DISPLAY_VER(display) >= 35) {
+	if (HAS_LT_PHY(display)) {
 		if (intel_de_wait_for_set_ms(display, XELPDP_DP_AUX_CH_CTL(display, aux_ch),
 					     XELPDP_DP_AUX_CH_CTL_POWER_STATUS, 2))
 			drm_warn(display->drm,
 				 "Timeout waiting for PHY %c AUX channel power to be up\n",
 				 phy_name(phy));
 	} else {
+		/*
+		 * The power status flag cannot be used to determine whether aux
+		 * power wells have finished powering up.  Instead we're
+		 * expected to just wait a fixed 600us after raising the request
+		 * bit.
+		 */
 		usleep_range(600, 1200);
 	}
 }
@@ -1887,7 +1893,7 @@ static void xelpdp_aux_power_well_disable(struct intel_display *display,
 		     XELPDP_DP_AUX_CH_CTL_POWER_REQUEST,
 		     0);
 
-	if (DISPLAY_VER(display) >= 35) {
+	if (HAS_LT_PHY(display)) {
 		if (intel_de_wait_for_clear_ms(display, XELPDP_DP_AUX_CH_CTL(display, aux_ch),
 					       XELPDP_DP_AUX_CH_CTL_POWER_STATUS, 1))
 			drm_warn(display->drm,
diff --git a/drivers/gpu/drm/i915/display/intel_display_regs.h b/drivers/gpu/drm/i915/display/intel_display_regs.h
index 9d71e26a4fa2..9e0d853f4b61 100644
--- a/drivers/gpu/drm/i915/display/intel_display_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_display_regs.h
@@ -882,6 +882,21 @@
 #define   PIPE_MISC2_FLIP_INFO_PLANE_SEL_MASK		REG_GENMASK(2, 0) /* tgl+ */
 #define   PIPE_MISC2_FLIP_INFO_PLANE_SEL(plane_id)	REG_FIELD_PREP(PIPE_MISC2_FLIP_INFO_PLANE_SEL_MASK, (plane_id))
 
+#define _UNDERRUN_DBG1_A			0x70064
+#define _UNDERRUN_DBG1_B			0x71064
+#define UNDERRUN_DBG1(pipe)			_MMIO_PIPE(pipe, _UNDERRUN_DBG1_A, _UNDERRUN_DBG1_B)
+#define   UNDERRUN_DBUF_BLOCK_NOT_VALID_MASK	REG_GENMASK(29, 24)
+#define   UNDERRUN_DDB_EMPTY_MASK		REG_GENMASK(21, 16)
+#define   UNDERRUN_DBUF_NOT_FILLED_MASK		REG_GENMASK(13, 8)
+#define   UNDERRUN_BELOW_WM0_MASK		REG_GENMASK(5, 0)
+
+#define _UNDERRUN_DBG2_A			0x70068
+#define _UNDERRUN_DBG2_B			0x71068
+#define UNDERRUN_DBG2(pipe)			_MMIO_PIPE(pipe, _UNDERRUN_DBG2_A, _UNDERRUN_DBG2_B)
+#define   UNDERRUN_FRAME_LINE_COUNTERS_FROZEN	REG_BIT(31)
+#define   UNDERRUN_PIPE_FRAME_COUNT_MASK	REG_GENMASK(30, 20)
+#define   UNDERRUN_LINE_COUNT_MASK		REG_GENMASK(19, 0)
+
 #define DPINVGTT				_MMIO(VLV_DISPLAY_BASE + 0x7002c) /* VLV/CHV only */
 #define   DPINVGTT_EN_MASK_CHV				REG_GENMASK(27, 16)
 #define   DPINVGTT_EN_MASK_VLV				REG_GENMASK(23, 16)
@@ -1416,6 +1431,7 @@
 
 #define GEN12_DCPR_STATUS_1				_MMIO(0x46440)
 #define  XELPDP_PMDEMAND_INFLIGHT_STATUS		REG_BIT(26)
+#define  XE3P_UNDERRUN_PKGC				REG_BIT(21)
 
 #define FUSE_STRAP		_MMIO(0x42014)
 #define   ILK_INTERNAL_GRAPHICS_DISABLE	REG_BIT(31)
@@ -2349,8 +2365,13 @@ enum skl_power_gate {
 #define  DDI_BUF_CTL_TC_PHY_OWNERSHIP		REG_BIT(6)
 #define  DDI_A_4_LANES				REG_BIT(4)
 #define  DDI_PORT_WIDTH_MASK			REG_GENMASK(3, 1)
+#define  DDI_PORT_WIDTH_ENCODE(width)		((width) == 3 ? 4 : (width) - 1)
+#define  DDI_PORT_WIDTH_DECODE(regval)		((regval) == 4 ? 3 : (regval) + 1)
 #define  DDI_PORT_WIDTH(width)			REG_FIELD_PREP(DDI_PORT_WIDTH_MASK, \
-							       ((width) == 3 ? 4 : (width) - 1))
+							       DDI_PORT_WIDTH_ENCODE(width))
+#define  DDI_PORT_WIDTH_GET(regval)		DDI_PORT_WIDTH_DECODE(REG_FIELD_GET(DDI_PORT_WIDTH_MASK, \
+										    (regval)))
+
 #define  DDI_PORT_WIDTH_SHIFT			1
 #define  DDI_INIT_DISPLAY_DETECTED		REG_BIT(0)
 
diff --git a/drivers/gpu/drm/i915/display/intel_display_reset.c b/drivers/gpu/drm/i915/display/intel_display_reset.c
index 03e8c68d2913..d00ef5bdcbda 100644
--- a/drivers/gpu/drm/i915/display/intel_display_reset.c
+++ b/drivers/gpu/drm/i915/display/intel_display_reset.c
@@ -6,13 +6,13 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_print.h>
 
-#include "i915_drv.h"
 #include "intel_clock_gating.h"
 #include "intel_cx0_phy.h"
 #include "intel_display_core.h"
 #include "intel_display_driver.h"
 #include "intel_display_reset.h"
 #include "intel_display_types.h"
+#include "intel_display_utils.h"
 #include "intel_hotplug.h"
 #include "intel_pps.h"
 
@@ -79,7 +79,6 @@ bool intel_display_reset_prepare(struct intel_display *display,
 
 void intel_display_reset_finish(struct intel_display *display, bool test_only)
 {
-	struct drm_i915_private *i915 = to_i915(display->drm);
 	struct drm_modeset_acquire_ctx *ctx = &display->restore.reset_ctx;
 	struct drm_atomic_state *state;
 	int ret;
@@ -107,7 +106,7 @@ void intel_display_reset_finish(struct intel_display *display, bool test_only)
 		 */
 		intel_pps_unlock_regs_wa(display);
 		intel_display_driver_init_hw(display);
-		intel_clock_gating_init(i915);
+		intel_clock_gating_init(display->drm);
 		intel_cx0_pll_power_save_wa(display);
 		intel_hpd_init(display);
 
diff --git a/drivers/gpu/drm/i915/display/intel_display_rps.c b/drivers/gpu/drm/i915/display/intel_display_rps.c
index 82ea1ec482e4..e77811396474 100644
--- a/drivers/gpu/drm/i915/display/intel_display_rps.c
+++ b/drivers/gpu/drm/i915/display/intel_display_rps.c
@@ -3,38 +3,39 @@
  * Copyright © 2023 Intel Corporation
  */
 
+#include <linux/dma-fence.h>
+
 #include <drm/drm_crtc.h>
 #include <drm/drm_vblank.h>
 
-#include "gt/intel_rps.h"
-#include "i915_drv.h"
 #include "i915_reg.h"
 #include "intel_display_core.h"
 #include "intel_display_irq.h"
 #include "intel_display_rps.h"
 #include "intel_display_types.h"
+#include "intel_parent.h"
 
 struct wait_rps_boost {
 	struct wait_queue_entry wait;
 
 	struct drm_crtc *crtc;
-	struct i915_request *request;
+	struct dma_fence *fence;
 };
 
 static int do_rps_boost(struct wait_queue_entry *_wait,
 			unsigned mode, int sync, void *key)
 {
 	struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
-	struct i915_request *rq = wait->request;
+	struct intel_display *display = to_intel_display(wait->crtc->dev);
 
 	/*
 	 * If we missed the vblank, but the request is already running it
 	 * is reasonable to assume that it will complete before the next
-	 * vblank without our intervention, so leave RPS alone.
+	 * vblank without our intervention, so leave RPS alone if not started.
 	 */
-	if (!i915_request_started(rq))
-		intel_rps_boost(rq);
-	i915_request_put(rq);
+	intel_parent_rps_boost_if_not_started(display, wait->fence);
+
+	dma_fence_put(wait->fence);
 
 	drm_crtc_vblank_put(wait->crtc);
 
@@ -49,7 +50,7 @@ void intel_display_rps_boost_after_vblank(struct drm_crtc *crtc,
 	struct intel_display *display = to_intel_display(crtc->dev);
 	struct wait_rps_boost *wait;
 
-	if (!dma_fence_is_i915(fence))
+	if (!intel_parent_rps_available(display))
 		return;
 
 	if (DISPLAY_VER(display) < 6)
@@ -64,7 +65,7 @@ void intel_display_rps_boost_after_vblank(struct drm_crtc *crtc,
 		return;
 	}
 
-	wait->request = to_request(dma_fence_get(fence));
+	wait->fence = dma_fence_get(fence);
 	wait->crtc = crtc;
 
 	wait->wait.func = do_rps_boost;
@@ -77,12 +78,14 @@ void intel_display_rps_mark_interactive(struct intel_display *display,
 					struct intel_atomic_state *state,
 					bool interactive)
 {
-	struct drm_i915_private *i915 = to_i915(display->drm);
+	if (!intel_parent_rps_available(display))
+		return;
 
 	if (state->rps_interactive == interactive)
 		return;
 
-	intel_rps_mark_interactive(&to_gt(i915)->rps, interactive);
+	intel_parent_rps_mark_interactive(display, interactive);
+
 	state->rps_interactive = interactive;
 }
 
@@ -102,7 +105,5 @@ void ilk_display_rps_disable(struct intel_display *display)
 
 void ilk_display_rps_irq_handler(struct intel_display *display)
 {
-	struct drm_i915_private *i915 = to_i915(display->drm);
-
-	gen5_rps_irq_handler(&to_gt(i915)->rps);
+	intel_parent_rps_ilk_irq_handler(display);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_display_rps.h b/drivers/gpu/drm/i915/display/intel_display_rps.h
index 183d154f2c7c..96b1fd00ead4 100644
--- a/drivers/gpu/drm/i915/display/intel_display_rps.h
+++ b/drivers/gpu/drm/i915/display/intel_display_rps.h
@@ -13,7 +13,6 @@ struct drm_crtc;
 struct intel_atomic_state;
 struct intel_display;
 
-#ifdef I915
 void intel_display_rps_boost_after_vblank(struct drm_crtc *crtc,
 					  struct dma_fence *fence);
 void intel_display_rps_mark_interactive(struct intel_display *display,
@@ -22,25 +21,5 @@ void intel_display_rps_mark_interactive(struct intel_display *display,
 void ilk_display_rps_enable(struct intel_display *display);
 void ilk_display_rps_disable(struct intel_display *display);
 void ilk_display_rps_irq_handler(struct intel_display *display);
-#else
-static inline void intel_display_rps_boost_after_vblank(struct drm_crtc *crtc,
-							struct dma_fence *fence)
-{
-}
-static inline void intel_display_rps_mark_interactive(struct intel_display *display,
-						      struct intel_atomic_state *state,
-						      bool interactive)
-{
-}
-static inline void ilk_display_rps_enable(struct intel_display *display)
-{
-}
-static inline void ilk_display_rps_disable(struct intel_display *display)
-{
-}
-static inline void ilk_display_rps_irq_handler(struct intel_display *display)
-{
-}
-#endif
 
 #endif /* __INTEL_DISPLAY_RPS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h
index 06bf8f7c0989..6ff53cd58052 100644
--- a/drivers/gpu/drm/i915/display/intel_display_types.h
+++ b/drivers/gpu/drm/i915/display/intel_display_types.h
@@ -509,6 +509,12 @@ struct intel_hdcp {
 	bool force_hdcp14;
 };
 
+enum intel_panel_replay_dsc_support {
+	INTEL_DP_PANEL_REPLAY_DSC_NOT_SUPPORTED,
+	INTEL_DP_PANEL_REPLAY_DSC_FULL_FRAME_ONLY,
+	INTEL_DP_PANEL_REPLAY_DSC_SELECTIVE_UPDATE,
+};
+
 struct intel_connector {
 	struct drm_connector base;
 	/*
@@ -561,6 +567,30 @@ struct intel_connector {
 			} overall_throughput;
 			int max_line_width;
 		} dsc_branch_caps;
+
+		struct {
+			u8 dpcd[DP_PANEL_REPLAY_CAP_SIZE];
+#define INTEL_PR_DPCD_INDEX(pr_dpcd_register)	((pr_dpcd_register) - DP_PANEL_REPLAY_CAP_SUPPORT)
+
+			bool support;
+			bool su_support;
+			enum intel_panel_replay_dsc_support dsc_support;
+
+			u16 su_w_granularity;
+			u16 su_y_granularity;
+		} panel_replay_caps;
+
+		struct {
+			u8 dpcd[EDP_PSR_RECEIVER_CAP_SIZE];
+
+			bool support;
+			bool su_support;
+
+			u16 su_w_granularity;
+			u16 su_y_granularity;
+
+			u8 sync_latency;
+		} psr_caps;
 	} dp;
 
 	struct {
@@ -955,12 +985,6 @@ struct intel_csc_matrix {
 	u16 postoff[3];
 };
 
-enum intel_panel_replay_dsc_support {
-	INTEL_DP_PANEL_REPLAY_DSC_NOT_SUPPORTED,
-	INTEL_DP_PANEL_REPLAY_DSC_FULL_FRAME_ONLY,
-	INTEL_DP_PANEL_REPLAY_DSC_SELECTIVE_UPDATE,
-};
-
 struct scaler_filter_coeff {
 	u16 sign;
 	u16 exp;
@@ -1152,6 +1176,8 @@ struct intel_crtc_state {
 	bool enable_psr2_su_region_et;
 	bool req_psr2_sdp_prior_scanline;
 	bool has_panel_replay;
+	bool link_off_after_as_sdp_when_pr_active;
+	bool disable_as_sdp_when_pr_active;
 	bool wm_level_disabled;
 	bool pkg_c_latency_used;
 	/* Only used for state verification. */
@@ -1662,7 +1688,7 @@ struct intel_pps {
 	unsigned long last_power_on;
 	unsigned long last_backlight_off;
 	ktime_t panel_power_off_time;
-	intel_wakeref_t vdd_wakeref;
+	struct ref_tracker *vdd_wakeref;
 
 	union {
 		/*
@@ -1716,14 +1742,12 @@ struct intel_psr {
 	bool active;
 	struct work_struct work;
 	unsigned int busy_frontbuffer_bits;
-	bool sink_psr2_support;
 	bool link_standby;
 	bool sel_update_enabled;
 	bool psr2_sel_fetch_enabled;
 	bool psr2_sel_fetch_cff_enabled;
 	bool su_region_et_enabled;
 	bool req_psr2_sdp_prior_scanline;
-	u8 sink_sync_latency;
 	ktime_t last_entry_attempt;
 	ktime_t last_exit;
 	bool sink_not_reliable;
@@ -1732,8 +1756,6 @@ struct intel_psr {
 	u16 su_y_granularity;
 	bool source_panel_replay_support;
 	bool sink_panel_replay_support;
-	bool sink_panel_replay_su_support;
-	enum intel_panel_replay_dsc_support sink_panel_replay_dsc_support;
 	bool panel_replay_enabled;
 	u32 dc3co_exitline;
 	u32 dc3co_exit_delay;
@@ -1760,9 +1782,6 @@ struct intel_dp {
 	bool needs_modeset_retry;
 	bool use_max_params;
 	u8 dpcd[DP_RECEIVER_CAP_SIZE];
-	u8 psr_dpcd[EDP_PSR_RECEIVER_CAP_SIZE];
-	u8 pr_dpcd[DP_PANEL_REPLAY_CAP_SIZE];
-#define INTEL_PR_DPCD_INDEX(pr_dpcd_register)	((pr_dpcd_register) - DP_PANEL_REPLAY_CAP_SUPPORT)
 
 	u8 downstream_ports[DP_MAX_DOWNSTREAM_PORTS];
 	u8 edp_dpcd[EDP_DISPLAY_CTL_CAP_SIZE];
@@ -1940,12 +1959,13 @@ struct intel_digital_port {
 	bool lane_reversal;
 	bool ddi_a_4_lanes;
 	bool release_cl2_override;
+	bool dedicated_external;
 	u8 max_lanes;
 	/* Used for DP and ICL+ TypeC/DP and TypeC/HDMI ports. */
 	enum aux_ch aux_ch;
 	enum intel_display_power_domain ddi_io_power_domain;
-	intel_wakeref_t ddi_io_wakeref;
-	intel_wakeref_t aux_wakeref;
+	struct ref_tracker *ddi_io_wakeref;
+	struct ref_tracker *aux_wakeref;
 
 	struct intel_tc_port *tc;
 
diff --git a/drivers/gpu/drm/i915/display/intel_display_wa.c b/drivers/gpu/drm/i915/display/intel_display_wa.c
index e38e5e87877c..a00af39f7538 100644
--- a/drivers/gpu/drm/i915/display/intel_display_wa.c
+++ b/drivers/gpu/drm/i915/display/intel_display_wa.c
@@ -70,6 +70,10 @@ bool __intel_display_wa(struct intel_display *display, enum intel_display_wa wa,
 		return DISPLAY_VER(display) == 13;
 	case INTEL_DISPLAY_WA_22014263786:
 		return IS_DISPLAY_VERx100(display, 1100, 1400);
+	case INTEL_DISPLAY_WA_15018326506:
+		return display->platform.battlemage;
+	case INTEL_DISPLAY_WA_14025769978:
+		return DISPLAY_VER(display) == 35;
 	default:
 		drm_WARN(display->drm, 1, "Missing Wa number: %s\n", name);
 		break;
diff --git a/drivers/gpu/drm/i915/display/intel_display_wa.h b/drivers/gpu/drm/i915/display/intel_display_wa.h
index 3644e8e2b724..a68c0bb7e516 100644
--- a/drivers/gpu/drm/i915/display/intel_display_wa.h
+++ b/drivers/gpu/drm/i915/display/intel_display_wa.h
@@ -26,6 +26,8 @@ enum intel_display_wa {
 	INTEL_DISPLAY_WA_16025573575,
 	INTEL_DISPLAY_WA_14011503117,
 	INTEL_DISPLAY_WA_22014263786,
+	INTEL_DISPLAY_WA_15018326506,
+	INTEL_DISPLAY_WA_14025769978,
 };
 
 bool __intel_display_wa(struct intel_display *display, enum intel_display_wa wa, const char *name);
diff --git a/drivers/gpu/drm/i915/display/intel_dmc.c b/drivers/gpu/drm/i915/display/intel_dmc.c
index 6ebbd97e6351..2fb6fec6dc99 100644
--- a/drivers/gpu/drm/i915/display/intel_dmc.c
+++ b/drivers/gpu/drm/i915/display/intel_dmc.c
@@ -1322,7 +1322,7 @@ static void intel_dmc_runtime_pm_get(struct intel_display *display)
 
 static void intel_dmc_runtime_pm_put(struct intel_display *display)
 {
-	intel_wakeref_t wakeref __maybe_unused =
+	struct ref_tracker *wakeref __maybe_unused =
 		fetch_and_zero(&display->dmc.wakeref);
 
 	intel_display_power_put(display, POWER_DOMAIN_INIT, wakeref);
diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c
index 0ec82fcbcf48..441995486745 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.c
+++ b/drivers/gpu/drm/i915/display/intel_dp.c
@@ -4562,7 +4562,7 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *connector
 	 * This has to be called after intel_dp->edp_dpcd is filled, PSR checks
 	 * for SET_POWER_CAPABLE bit in intel_dp->edp_dpcd[1]
 	 */
-	intel_psr_init_dpcd(intel_dp);
+	intel_psr_init_dpcd(intel_dp, connector);
 
 	intel_edp_set_sink_rates(intel_dp);
 	intel_dp_set_max_sink_lane_count(intel_dp);
@@ -5791,9 +5791,8 @@ bool intel_digital_port_connected_locked(struct intel_encoder *encoder)
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 	bool is_glitch_free = intel_tc_port_handles_hpd_glitches(dig_port);
 	bool is_connected = false;
-	intel_wakeref_t wakeref;
 
-	with_intel_display_power(display, POWER_DOMAIN_DISPLAY_CORE, wakeref) {
+	with_intel_display_power(display, POWER_DOMAIN_DISPLAY_CORE) {
 		poll_timeout_us(is_connected = dig_port->connected(encoder),
 				is_connected || is_glitch_free,
 				30, 4000, false);
@@ -6049,10 +6048,19 @@ intel_dp_detect(struct drm_connector *_connector,
 
 	if (status == connector_status_disconnected) {
 		intel_dp_test_reset(intel_dp);
+		/*
+		 * FIXME: Resetting these caps here cause
+		 * state computation fail if the connector need to be
+		 * modeset after sink disconnect. Move resetting them
+		 * to where new sink is connected.
+		 */
 		memset(connector->dp.dsc_dpcd, 0, sizeof(connector->dp.dsc_dpcd));
+		memset(connector->dp.panel_replay_caps.dpcd, 0,
+		       sizeof(connector->dp.panel_replay_caps.dpcd));
 		intel_dp->psr.sink_panel_replay_support = false;
-		intel_dp->psr.sink_panel_replay_su_support = false;
-		intel_dp->psr.sink_panel_replay_dsc_support =
+		connector->dp.panel_replay_caps.support = false;
+		connector->dp.panel_replay_caps.su_support = false;
+		connector->dp.panel_replay_caps.dsc_support =
 			INTEL_DP_PANEL_REPLAY_DSC_NOT_SUPPORTED;
 
 		intel_dp_mst_disconnect(intel_dp);
@@ -6075,7 +6083,7 @@ intel_dp_detect(struct drm_connector *_connector,
 		connector->base.epoch_counter++;
 
 	if (!intel_dp_is_edp(intel_dp))
-		intel_psr_init_dpcd(intel_dp);
+		intel_psr_init_dpcd(intel_dp, connector);
 
 	intel_dp_detect_dsc_caps(intel_dp, connector);
 
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux.c b/drivers/gpu/drm/i915/display/intel_dp_aux.c
index 809799f63e32..b20ec3e589fa 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_aux.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux.c
@@ -6,6 +6,7 @@
 #include <drm/drm_print.h>
 
 #include "intel_de.h"
+#include "intel_display_jiffies.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
 #include "intel_dp.h"
@@ -60,16 +61,17 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp)
 	i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg(intel_dp);
 	const unsigned int timeout_ms = 10;
 	u32 status;
-	int ret;
+	bool done;
 
-	ret = intel_de_wait_ms(display, ch_ctl,
-			       DP_AUX_CH_CTL_SEND_BUSY, 0,
-			       timeout_ms, &status);
+#define C (((status = intel_de_read_notrace(display, ch_ctl)) & DP_AUX_CH_CTL_SEND_BUSY) == 0)
+	done = wait_event_timeout(display->gmbus.wait_queue, C,
+				  msecs_to_jiffies_timeout(timeout_ms));
 
-	if (ret == -ETIMEDOUT)
+	if (!done)
 		drm_err(display->drm,
 			"%s: did not complete or timeout within %ums (status 0x%08x)\n",
 			intel_dp->aux.name, timeout_ms, status);
+#undef C
 
 	return status;
 }
@@ -245,8 +247,8 @@ intel_dp_aux_xfer(struct intel_dp *intel_dp,
 	i915_reg_t ch_ctl, ch_data[5];
 	u32 aux_clock_divider;
 	enum intel_display_power_domain aux_domain;
-	intel_wakeref_t aux_wakeref;
-	intel_wakeref_t pps_wakeref = NULL;
+	struct ref_tracker *aux_wakeref;
+	struct ref_tracker *pps_wakeref = NULL;
 	int i, ret, recv_bytes;
 	int try, clock = 0;
 	u32 status;
diff --git a/drivers/gpu/drm/i915/display/intel_dpll.c b/drivers/gpu/drm/i915/display/intel_dpll.c
index 4f1db8493a2e..a4f372c9e6fc 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll.c
+++ b/drivers/gpu/drm/i915/display/intel_dpll.c
@@ -1212,27 +1212,6 @@ static int dg2_crtc_compute_clock(struct intel_atomic_state *state,
 	return 0;
 }
 
-static int mtl_crtc_compute_clock(struct intel_atomic_state *state,
-				  struct intel_crtc *crtc)
-{
-	struct intel_crtc_state *crtc_state =
-		intel_atomic_get_new_crtc_state(state, crtc);
-	struct intel_encoder *encoder =
-		intel_get_crtc_new_encoder(state, crtc_state);
-	int ret;
-
-	ret = intel_cx0pll_calc_state(crtc_state, encoder);
-	if (ret)
-		return ret;
-
-	/* TODO: Do the readback via intel_dpll_compute() */
-	crtc_state->port_clock = intel_cx0pll_calc_port_clock(encoder, &crtc_state->dpll_hw_state.cx0pll);
-
-	crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state);
-
-	return 0;
-}
-
 static int xe3plpd_crtc_compute_clock(struct intel_atomic_state *state,
 				      struct intel_crtc *crtc)
 {
@@ -1719,7 +1698,8 @@ static const struct intel_dpll_global_funcs xe3plpd_dpll_funcs = {
 };
 
 static const struct intel_dpll_global_funcs mtl_dpll_funcs = {
-	.crtc_compute_clock = mtl_crtc_compute_clock,
+	.crtc_compute_clock = hsw_crtc_compute_clock,
+	.crtc_get_dpll = hsw_crtc_get_dpll,
 };
 
 static const struct intel_dpll_global_funcs dg2_dpll_funcs = {
diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
index 9c7cf03cf022..9aa84a430f09 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
@@ -203,6 +203,22 @@ enum intel_dpll_id icl_tc_port_to_pll_id(enum tc_port tc_port)
 	return tc_port - TC_PORT_1 + DPLL_ID_ICL_MGPLL1;
 }
 
+enum intel_dpll_id mtl_port_to_pll_id(struct intel_display *display, enum port port)
+{
+	if (port >= PORT_TC1)
+		return icl_tc_port_to_pll_id(intel_port_to_tc(display, port));
+
+	switch (port) {
+	case PORT_A:
+		return DPLL_ID_ICL_DPLL0;
+	case PORT_B:
+		return DPLL_ID_ICL_DPLL1;
+	default:
+		MISSING_CASE(port);
+		return DPLL_ID_ICL_DPLL0;
+	}
+}
+
 static i915_reg_t
 intel_combo_pll_enable_reg(struct intel_display *display,
 			   struct intel_dpll *pll)
@@ -531,7 +547,7 @@ static bool ibx_pch_dpll_get_hw_state(struct intel_display *display,
 {
 	struct i9xx_dpll_hw_state *hw_state = &dpll_hw_state->i9xx;
 	const enum intel_dpll_id id = pll->info->id;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	u32 val;
 
 	wakeref = intel_display_power_get_if_enabled(display,
@@ -752,7 +768,7 @@ static bool hsw_ddi_wrpll_get_hw_state(struct intel_display *display,
 {
 	struct hsw_dpll_hw_state *hw_state = &dpll_hw_state->hsw;
 	const enum intel_dpll_id id = pll->info->id;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	u32 val;
 
 	wakeref = intel_display_power_get_if_enabled(display,
@@ -773,7 +789,7 @@ static bool hsw_ddi_spll_get_hw_state(struct intel_display *display,
 				      struct intel_dpll_hw_state *dpll_hw_state)
 {
 	struct hsw_dpll_hw_state *hw_state = &dpll_hw_state->hsw;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	u32 val;
 
 	wakeref = intel_display_power_get_if_enabled(display,
@@ -1431,7 +1447,7 @@ static bool skl_ddi_pll_get_hw_state(struct intel_display *display,
 	struct skl_dpll_hw_state *hw_state = &dpll_hw_state->skl;
 	const struct skl_dpll_regs *regs = skl_dpll_regs;
 	const enum intel_dpll_id id = pll->info->id;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret;
 	u32 val;
 
@@ -1469,7 +1485,7 @@ static bool skl_ddi_dpll0_get_hw_state(struct intel_display *display,
 	struct skl_dpll_hw_state *hw_state = &dpll_hw_state->skl;
 	const struct skl_dpll_regs *regs = skl_dpll_regs;
 	const enum intel_dpll_id id = pll->info->id;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	u32 val;
 	bool ret;
 
@@ -2172,7 +2188,7 @@ static bool bxt_ddi_pll_get_hw_state(struct intel_display *display,
 {
 	struct bxt_dpll_hw_state *hw_state = &dpll_hw_state->bxt;
 	enum port port = (enum port)pll->info->id; /* 1:1 port->PLL mapping */
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	enum dpio_phy phy;
 	enum dpio_channel ch;
 	u32 val;
@@ -3490,6 +3506,37 @@ static int icl_get_tc_phy_dplls(struct intel_atomic_state *state,
 	return ret;
 }
 
+/*
+ * Get the PLL for either a port using a C10 PHY PLL, or for a port using a
+ * C20 PHY PLL in the cases of:
+ * - BMG port A/B
+ * - PTL port B eDP over TypeC PHY
+ */
+static int mtl_get_non_tc_phy_dpll(struct intel_atomic_state *state,
+				      struct intel_crtc *crtc,
+				      struct intel_encoder *encoder)
+{
+	struct intel_display *display = to_intel_display(crtc);
+	struct intel_crtc_state *crtc_state =
+		intel_atomic_get_new_crtc_state(state, crtc);
+	struct icl_port_dpll *port_dpll =
+		&crtc_state->icl_port_dplls[ICL_PORT_DPLL_DEFAULT];
+	enum intel_dpll_id pll_id = mtl_port_to_pll_id(display, encoder->port);
+
+	port_dpll->pll = intel_find_dpll(state, crtc,
+					 &port_dpll->hw_state,
+					 BIT(pll_id));
+	if (!port_dpll->pll)
+		return -EINVAL;
+
+	intel_reference_dpll(state, crtc,
+			     port_dpll->pll, &port_dpll->hw_state);
+
+	icl_update_active_dpll(state, crtc, encoder);
+
+	return 0;
+}
+
 static int icl_compute_dplls(struct intel_atomic_state *state,
 			     struct intel_crtc *crtc,
 			     struct intel_encoder *encoder)
@@ -3551,7 +3598,7 @@ static bool mg_pll_get_hw_state(struct intel_display *display,
 	struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
 	const enum intel_dpll_id id = pll->info->id;
 	enum tc_port tc_port = icl_pll_id_to_tc_port(id);
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret = false;
 	u32 val;
 
@@ -3618,7 +3665,7 @@ static bool dkl_pll_get_hw_state(struct intel_display *display,
 	struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
 	const enum intel_dpll_id id = pll->info->id;
 	enum tc_port tc_port = icl_pll_id_to_tc_port(id);
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret = false;
 	u32 val;
 
@@ -3690,7 +3737,7 @@ static bool icl_pll_get_hw_state(struct intel_display *display,
 {
 	struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
 	const enum intel_dpll_id id = pll->info->id;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret = false;
 	u32 val;
 
@@ -3753,9 +3800,9 @@ static bool combo_pll_get_hw_state(struct intel_display *display,
 	return icl_pll_get_hw_state(display, pll, dpll_hw_state, enable_reg);
 }
 
-static bool tbt_pll_get_hw_state(struct intel_display *display,
-				 struct intel_dpll *pll,
-				 struct intel_dpll_hw_state *dpll_hw_state)
+static bool icl_tbt_pll_get_hw_state(struct intel_display *display,
+				     struct intel_dpll *pll,
+				     struct intel_dpll_hw_state *dpll_hw_state)
 {
 	return icl_pll_get_hw_state(display, pll, dpll_hw_state, TBT_PLL_ENABLE);
 }
@@ -3984,9 +4031,9 @@ static void combo_pll_enable(struct intel_display *display,
 	/* DVFS post sequence would be here. See the comment above. */
 }
 
-static void tbt_pll_enable(struct intel_display *display,
-			   struct intel_dpll *pll,
-			   const struct intel_dpll_hw_state *dpll_hw_state)
+static void icl_tbt_pll_enable(struct intel_display *display,
+			       struct intel_dpll *pll,
+			       const struct intel_dpll_hw_state *dpll_hw_state)
 {
 	const struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
 
@@ -4069,8 +4116,8 @@ static void combo_pll_disable(struct intel_display *display,
 	icl_pll_disable(display, pll, enable_reg);
 }
 
-static void tbt_pll_disable(struct intel_display *display,
-			    struct intel_dpll *pll)
+static void icl_tbt_pll_disable(struct intel_display *display,
+				struct intel_dpll *pll)
 {
 	icl_pll_disable(display, pll, TBT_PLL_ENABLE);
 }
@@ -4142,10 +4189,10 @@ static const struct intel_dpll_funcs combo_pll_funcs = {
 	.get_freq = icl_ddi_combo_pll_get_freq,
 };
 
-static const struct intel_dpll_funcs tbt_pll_funcs = {
-	.enable = tbt_pll_enable,
-	.disable = tbt_pll_disable,
-	.get_hw_state = tbt_pll_get_hw_state,
+static const struct intel_dpll_funcs icl_tbt_pll_funcs = {
+	.enable = icl_tbt_pll_enable,
+	.disable = icl_tbt_pll_disable,
+	.get_hw_state = icl_tbt_pll_get_hw_state,
 	.get_freq = icl_ddi_tbt_pll_get_freq,
 };
 
@@ -4159,7 +4206,7 @@ static const struct intel_dpll_funcs mg_pll_funcs = {
 static const struct dpll_info icl_plls[] = {
 	{ .name = "DPLL 0", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL0, },
 	{ .name = "DPLL 1", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL1, },
-	{ .name = "TBT PLL", .funcs = &tbt_pll_funcs, .id = DPLL_ID_ICL_TBTPLL,
+	{ .name = "TBT PLL", .funcs = &icl_tbt_pll_funcs, .id = DPLL_ID_ICL_TBTPLL,
 	  .is_alt_port_dpll = true, },
 	{ .name = "MG PLL 1", .funcs = &mg_pll_funcs, .id = DPLL_ID_ICL_MGPLL1, },
 	{ .name = "MG PLL 2", .funcs = &mg_pll_funcs, .id = DPLL_ID_ICL_MGPLL2, },
@@ -4207,7 +4254,7 @@ static const struct intel_dpll_funcs dkl_pll_funcs = {
 static const struct dpll_info tgl_plls[] = {
 	{ .name = "DPLL 0", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL0, },
 	{ .name = "DPLL 1", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL1, },
-	{ .name = "TBT PLL", .funcs = &tbt_pll_funcs, .id = DPLL_ID_ICL_TBTPLL,
+	{ .name = "TBT PLL", .funcs = &icl_tbt_pll_funcs, .id = DPLL_ID_ICL_TBTPLL,
 	  .is_alt_port_dpll = true, },
 	{ .name = "TC PLL 1", .funcs = &dkl_pll_funcs, .id = DPLL_ID_ICL_MGPLL1, },
 	{ .name = "TC PLL 2", .funcs = &dkl_pll_funcs, .id = DPLL_ID_ICL_MGPLL2, },
@@ -4285,7 +4332,7 @@ static const struct intel_dpll_mgr adls_pll_mgr = {
 static const struct dpll_info adlp_plls[] = {
 	{ .name = "DPLL 0", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL0, },
 	{ .name = "DPLL 1", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL1, },
-	{ .name = "TBT PLL", .funcs = &tbt_pll_funcs, .id = DPLL_ID_ICL_TBTPLL,
+	{ .name = "TBT PLL", .funcs = &icl_tbt_pll_funcs, .id = DPLL_ID_ICL_TBTPLL,
 	  .is_alt_port_dpll = true, },
 	{ .name = "TC PLL 1", .funcs = &dkl_pll_funcs, .id = DPLL_ID_ICL_MGPLL1, },
 	{ .name = "TC PLL 2", .funcs = &dkl_pll_funcs, .id = DPLL_ID_ICL_MGPLL2, },
@@ -4305,6 +4352,224 @@ static const struct intel_dpll_mgr adlp_pll_mgr = {
 	.compare_hw_state = icl_compare_hw_state,
 };
 
+static struct intel_encoder *get_intel_encoder(struct intel_display *display,
+					       const struct intel_dpll *pll)
+{
+	struct intel_encoder *encoder;
+	enum intel_dpll_id mtl_id;
+
+	for_each_intel_encoder(display->drm, encoder) {
+		mtl_id = mtl_port_to_pll_id(display, encoder->port);
+
+		if (mtl_id == pll->info->id)
+			return encoder;
+	}
+
+	return NULL;
+}
+
+static bool mtl_pll_get_hw_state(struct intel_display *display,
+				 struct intel_dpll *pll,
+				 struct intel_dpll_hw_state *dpll_hw_state)
+{
+	struct intel_encoder *encoder = get_intel_encoder(display, pll);
+
+	if (!encoder)
+		return false;
+
+	return intel_cx0pll_readout_hw_state(encoder, &dpll_hw_state->cx0pll);
+}
+
+static int mtl_pll_get_freq(struct intel_display *display,
+			    const struct intel_dpll *pll,
+			    const struct intel_dpll_hw_state *dpll_hw_state)
+{
+	struct intel_encoder *encoder = get_intel_encoder(display, pll);
+
+	if (drm_WARN_ON(display->drm, !encoder))
+		return -EINVAL;
+
+	return intel_cx0pll_calc_port_clock(encoder, &dpll_hw_state->cx0pll);
+}
+
+static void mtl_pll_enable(struct intel_display *display,
+			   struct intel_dpll *pll,
+			   const struct intel_dpll_hw_state *dpll_hw_state)
+{
+	struct intel_encoder *encoder = get_intel_encoder(display, pll);
+
+	if (drm_WARN_ON(display->drm, !encoder))
+		return;
+
+	intel_mtl_pll_enable(encoder, pll, dpll_hw_state);
+}
+
+static void mtl_pll_disable(struct intel_display *display,
+			    struct intel_dpll *pll)
+{
+	struct intel_encoder *encoder = get_intel_encoder(display, pll);
+
+	if (drm_WARN_ON(display->drm, !encoder))
+		return;
+
+	intel_mtl_pll_disable(encoder);
+}
+
+static const struct intel_dpll_funcs mtl_pll_funcs = {
+	.enable = mtl_pll_enable,
+	.disable = mtl_pll_disable,
+	.get_hw_state = mtl_pll_get_hw_state,
+	.get_freq = mtl_pll_get_freq,
+};
+
+static void mtl_tbt_pll_enable(struct intel_display *display,
+			       struct intel_dpll *pll,
+			       const struct intel_dpll_hw_state *hw_state)
+{
+}
+
+static void mtl_tbt_pll_disable(struct intel_display *display,
+				struct intel_dpll *pll)
+{
+}
+
+static int mtl_tbt_pll_get_freq(struct intel_display *display,
+				const struct intel_dpll *pll,
+				const struct intel_dpll_hw_state *dpll_hw_state)
+{
+	/*
+	 * The PLL outputs multiple frequencies at the same time, selection is
+	 * made at DDI clock mux level.
+	 */
+	drm_WARN_ON(display->drm, 1);
+
+	return 0;
+}
+
+static const struct intel_dpll_funcs mtl_tbt_pll_funcs = {
+	.enable = mtl_tbt_pll_enable,
+	.disable = mtl_tbt_pll_disable,
+	.get_hw_state = intel_mtl_tbt_pll_readout_hw_state,
+	.get_freq = mtl_tbt_pll_get_freq,
+};
+
+static const struct dpll_info mtl_plls[] = {
+	{ .name = "DPLL 0", .funcs = &mtl_pll_funcs, .id = DPLL_ID_ICL_DPLL0, },
+	{ .name = "DPLL 1", .funcs = &mtl_pll_funcs, .id = DPLL_ID_ICL_DPLL1, },
+	{ .name = "TBT PLL", .funcs = &mtl_tbt_pll_funcs, .id = DPLL_ID_ICL_TBTPLL,
+	  .is_alt_port_dpll = true, .always_on = true },
+	{ .name = "TC PLL 1", .funcs = &mtl_pll_funcs, .id = DPLL_ID_ICL_MGPLL1, },
+	{ .name = "TC PLL 2", .funcs = &mtl_pll_funcs, .id = DPLL_ID_ICL_MGPLL2, },
+	{ .name = "TC PLL 3", .funcs = &mtl_pll_funcs, .id = DPLL_ID_ICL_MGPLL3, },
+	{ .name = "TC PLL 4", .funcs = &mtl_pll_funcs, .id = DPLL_ID_ICL_MGPLL4, },
+	{}
+};
+
+/*
+ * Compute the state for either a C10 PHY PLL, or in the case of the PTL port B,
+ * eDP on TypeC PHY case for a C20 PHY PLL.
+ */
+static int mtl_compute_non_tc_phy_dpll(struct intel_atomic_state *state,
+				       struct intel_crtc *crtc,
+				       struct intel_encoder *encoder)
+{
+	struct intel_crtc_state *crtc_state =
+		intel_atomic_get_new_crtc_state(state, crtc);
+	struct icl_port_dpll *port_dpll =
+		&crtc_state->icl_port_dplls[ICL_PORT_DPLL_DEFAULT];
+	int ret;
+
+	ret = intel_cx0pll_calc_state(crtc_state, encoder, &port_dpll->hw_state);
+	if (ret)
+		return ret;
+
+	/* this is mainly for the fastset check */
+	icl_set_active_port_dpll(crtc_state, ICL_PORT_DPLL_DEFAULT);
+
+	crtc_state->port_clock = intel_cx0pll_calc_port_clock(encoder,
+							      &port_dpll->hw_state.cx0pll);
+
+	return 0;
+}
+
+static int mtl_compute_tc_phy_dplls(struct intel_atomic_state *state,
+				    struct intel_crtc *crtc,
+				    struct intel_encoder *encoder)
+{
+	struct intel_crtc_state *crtc_state =
+		intel_atomic_get_new_crtc_state(state, crtc);
+	const struct intel_crtc_state *old_crtc_state =
+		intel_atomic_get_old_crtc_state(state, crtc);
+	struct icl_port_dpll *port_dpll;
+	int ret;
+
+	port_dpll = &crtc_state->icl_port_dplls[ICL_PORT_DPLL_DEFAULT];
+	intel_mtl_tbt_pll_calc_state(&port_dpll->hw_state);
+
+	port_dpll = &crtc_state->icl_port_dplls[ICL_PORT_DPLL_MG_PHY];
+	ret = intel_cx0pll_calc_state(crtc_state, encoder, &port_dpll->hw_state);
+	if (ret)
+		return ret;
+
+	/* this is mainly for the fastset check */
+	if (old_crtc_state->intel_dpll &&
+	    old_crtc_state->intel_dpll->info->id == DPLL_ID_ICL_TBTPLL)
+		icl_set_active_port_dpll(crtc_state, ICL_PORT_DPLL_DEFAULT);
+	else
+		icl_set_active_port_dpll(crtc_state, ICL_PORT_DPLL_MG_PHY);
+
+	crtc_state->port_clock = intel_cx0pll_calc_port_clock(encoder,
+							      &port_dpll->hw_state.cx0pll);
+
+	return 0;
+}
+
+static int mtl_compute_dplls(struct intel_atomic_state *state,
+			     struct intel_crtc *crtc,
+			     struct intel_encoder *encoder)
+{
+	if (intel_encoder_is_tc(encoder))
+		return mtl_compute_tc_phy_dplls(state, crtc, encoder);
+	else
+		return mtl_compute_non_tc_phy_dpll(state, crtc, encoder);
+}
+
+static int mtl_get_dplls(struct intel_atomic_state *state,
+			 struct intel_crtc *crtc,
+			 struct intel_encoder *encoder)
+{
+	if (intel_encoder_is_tc(encoder))
+		return icl_get_tc_phy_dplls(state, crtc, encoder);
+	else
+		return mtl_get_non_tc_phy_dpll(state, crtc, encoder);
+}
+
+static void mtl_dump_hw_state(struct drm_printer *p,
+			      const struct intel_dpll_hw_state *dpll_hw_state)
+{
+	intel_cx0pll_dump_hw_state(p, &dpll_hw_state->cx0pll);
+}
+
+static bool mtl_compare_hw_state(const struct intel_dpll_hw_state *_a,
+				 const struct intel_dpll_hw_state *_b)
+{
+	const struct intel_cx0pll_state *a = &_a->cx0pll;
+	const struct intel_cx0pll_state *b = &_b->cx0pll;
+
+	return intel_cx0pll_compare_hw_state(a, b);
+}
+
+static const struct intel_dpll_mgr mtl_pll_mgr = {
+	.dpll_info = mtl_plls,
+	.compute_dplls = mtl_compute_dplls,
+	.get_dplls = mtl_get_dplls,
+	.put_dplls = icl_put_dplls,
+	.update_active_dpll = icl_update_active_dpll,
+	.update_ref_clks = icl_update_dpll_ref_clks,
+	.dump_hw_state = mtl_dump_hw_state,
+	.compare_hw_state = mtl_compare_hw_state,
+};
+
 /**
  * intel_dpll_init - Initialize DPLLs
  * @display: intel_display device
@@ -4319,9 +4584,11 @@ void intel_dpll_init(struct intel_display *display)
 
 	mutex_init(&display->dpll.lock);
 
-	if (DISPLAY_VER(display) >= 14 || display->platform.dg2)
-		/* No shared DPLLs on DG2; port PLLs are part of the PHY */
+	if (DISPLAY_VER(display) >= 35 || display->platform.dg2)
+		/* No shared DPLLs on NVL or DG2; port PLLs are part of the PHY */
 		dpll_mgr = NULL;
+	else if (DISPLAY_VER(display) >= 14)
+		dpll_mgr = &mtl_pll_mgr;
 	else if (display->platform.alderlake_p)
 		dpll_mgr = &adlp_pll_mgr;
 	else if (display->platform.alderlake_s)
@@ -4675,11 +4942,18 @@ verify_single_dpll_state(struct intel_display *display,
 				 "%s: pll enabled crtcs mismatch (expected 0x%x in 0x%x)\n",
 				 pll->info->name, pipe_mask, pll->state.pipe_mask);
 
-	INTEL_DISPLAY_STATE_WARN(display,
-				 pll->on && memcmp(&pll->state.hw_state, &dpll_hw_state,
-						   sizeof(dpll_hw_state)),
-				 "%s: pll hw state mismatch\n",
-				 pll->info->name);
+	if (INTEL_DISPLAY_STATE_WARN(display,
+				     pll->on && memcmp(&pll->state.hw_state, &dpll_hw_state,
+						       sizeof(dpll_hw_state)),
+				     "%s: pll hw state mismatch\n",
+				     pll->info->name)) {
+		struct drm_printer p = drm_dbg_printer(display->drm, DRM_UT_KMS, NULL);
+
+		drm_printf(&p, "PLL %s HW state:\n", pll->info->name);
+		intel_dpll_dump_hw_state(display, &p, &dpll_hw_state);
+		drm_printf(&p, "PLL %s SW state:\n", pll->info->name);
+		intel_dpll_dump_hw_state(display, &p, &pll->state.hw_state);
+	}
 }
 
 static bool has_alt_port_dpll(const struct intel_dpll *old_pll,
diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h
index 6183da90b28d..5b71c860515f 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h
+++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h
@@ -28,7 +28,6 @@
 #include <linux/types.h>
 
 #include "intel_display_power.h"
-#include "intel_wakeref.h"
 
 #define for_each_dpll(__display, __pll, __i) \
 	for ((__i) = 0; (__i) < (__display)->dpll.num_dpll && \
@@ -42,6 +41,7 @@ struct intel_crtc_state;
 struct intel_dpll_funcs;
 struct intel_encoder;
 struct intel_shared_dpll;
+struct ref_tracker;
 
 /**
  * enum intel_dpll_id - possible DPLL ids
@@ -255,6 +255,11 @@ struct intel_c20pll_state {
 		u16 mplla[10];
 		u16 mpllb[11];
 	};
+	struct intel_c20pll_vdr_state {
+		u8 custom_width;
+		u8 serdes_rate;
+		u8 hdmi_rate;
+	} vdr;
 };
 
 struct intel_cx0pll_state {
@@ -262,6 +267,7 @@ struct intel_cx0pll_state {
 		struct intel_c10pll_state c10;
 		struct intel_c20pll_state c20;
 	};
+	int lane_count;
 	bool ssc_enabled;
 	bool use_c10;
 	bool tbt_mode;
@@ -390,7 +396,7 @@ struct intel_dpll {
 	 * @wakeref: In some platforms a device-level runtime pm reference may
 	 * need to be grabbed to disable DC states while this DPLL is enabled
 	 */
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 };
 
 #define SKL_DPLL0 0
@@ -444,6 +450,7 @@ bool intel_dpll_compare_hw_state(struct intel_display *display,
 				 const struct intel_dpll_hw_state *a,
 				 const struct intel_dpll_hw_state *b);
 enum intel_dpll_id icl_tc_port_to_pll_id(enum tc_port tc_port);
+enum intel_dpll_id mtl_port_to_pll_id(struct intel_display *display, enum port port);
 bool intel_dpll_is_combophy(enum intel_dpll_id id);
 
 void intel_dpll_state_verify(struct intel_atomic_state *state,
diff --git a/drivers/gpu/drm/i915/display/intel_dram.c b/drivers/gpu/drm/i915/display/intel_dram.c
new file mode 100644
index 000000000000..019a722a38bf
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_dram.c
@@ -0,0 +1,868 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2020 Intel Corporation
+ */
+
+#include <linux/string_helpers.h>
+
+#include <drm/drm_managed.h>
+#include <drm/drm_print.h>
+
+#include "i915_drv.h"
+#include "i915_reg.h"
+#include "intel_display_core.h"
+#include "intel_display_utils.h"
+#include "intel_dram.h"
+#include "intel_mchbar_regs.h"
+#include "intel_pcode.h"
+#include "intel_uncore.h"
+#include "vlv_iosf_sb.h"
+
+struct dram_dimm_info {
+	u16 size;
+	u8 width, ranks;
+};
+
+struct dram_channel_info {
+	struct dram_dimm_info dimm_l, dimm_s;
+	u8 ranks;
+	bool is_16gb_dimm;
+};
+
+#define DRAM_TYPE_STR(type) [INTEL_DRAM_ ## type] = #type
+
+const char *intel_dram_type_str(enum intel_dram_type type)
+{
+	static const char * const str[] = {
+		DRAM_TYPE_STR(UNKNOWN),
+		DRAM_TYPE_STR(DDR2),
+		DRAM_TYPE_STR(DDR3),
+		DRAM_TYPE_STR(DDR4),
+		DRAM_TYPE_STR(LPDDR3),
+		DRAM_TYPE_STR(LPDDR4),
+		DRAM_TYPE_STR(DDR5),
+		DRAM_TYPE_STR(LPDDR5),
+		DRAM_TYPE_STR(GDDR),
+		DRAM_TYPE_STR(GDDR_ECC),
+	};
+
+	BUILD_BUG_ON(ARRAY_SIZE(str) != __INTEL_DRAM_TYPE_MAX);
+
+	if (type >= ARRAY_SIZE(str))
+		type = INTEL_DRAM_UNKNOWN;
+
+	return str[type];
+}
+
+#undef DRAM_TYPE_STR
+
+static enum intel_dram_type pnv_dram_type(struct intel_display *display)
+{
+	struct drm_i915_private *i915 = to_i915(display->drm);
+
+	return intel_uncore_read(&i915->uncore, CSHRDDR3CTL) & CSHRDDR3CTL_DDR3 ?
+		INTEL_DRAM_DDR3 : INTEL_DRAM_DDR2;
+}
+
+static unsigned int pnv_mem_freq(struct intel_display *display)
+{
+	struct drm_i915_private *dev_priv = to_i915(display->drm);
+	u32 tmp;
+
+	tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
+
+	switch (tmp & CLKCFG_MEM_MASK) {
+	case CLKCFG_MEM_533:
+		return 533333;
+	case CLKCFG_MEM_667:
+		return 666667;
+	case CLKCFG_MEM_800:
+		return 800000;
+	}
+
+	return 0;
+}
+
+static unsigned int ilk_mem_freq(struct intel_display *display)
+{
+	struct drm_i915_private *dev_priv = to_i915(display->drm);
+	u16 ddrpll;
+
+	ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
+	switch (ddrpll & 0xff) {
+	case 0xc:
+		return 800000;
+	case 0x10:
+		return 1066667;
+	case 0x14:
+		return 1333333;
+	case 0x18:
+		return 1600000;
+	default:
+		drm_dbg_kms(display->drm, "unknown memory frequency 0x%02x\n",
+			    ddrpll & 0xff);
+		return 0;
+	}
+}
+
+static unsigned int chv_mem_freq(struct intel_display *display)
+{
+	u32 val;
+
+	vlv_iosf_sb_get(display->drm, BIT(VLV_IOSF_SB_CCK));
+	val = vlv_iosf_sb_read(display->drm, VLV_IOSF_SB_CCK, CCK_FUSE_REG);
+	vlv_iosf_sb_put(display->drm, BIT(VLV_IOSF_SB_CCK));
+
+	switch ((val >> 2) & 0x7) {
+	case 3:
+		return 2000000;
+	default:
+		return 1600000;
+	}
+}
+
+static unsigned int vlv_mem_freq(struct intel_display *display)
+{
+	u32 val;
+
+	vlv_iosf_sb_get(display->drm, BIT(VLV_IOSF_SB_PUNIT));
+	val = vlv_iosf_sb_read(display->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS);
+	vlv_iosf_sb_put(display->drm, BIT(VLV_IOSF_SB_PUNIT));
+
+	switch ((val >> 6) & 3) {
+	case 0:
+	case 1:
+		return 800000;
+	case 2:
+		return 1066667;
+	case 3:
+		return 1333333;
+	}
+
+	return 0;
+}
+
+unsigned int intel_mem_freq(struct intel_display *display)
+{
+	if (display->platform.pineview)
+		return pnv_mem_freq(display);
+	else if (DISPLAY_VER(display) == 5)
+		return ilk_mem_freq(display);
+	else if (display->platform.cherryview)
+		return chv_mem_freq(display);
+	else if (display->platform.valleyview)
+		return vlv_mem_freq(display);
+	else
+		return 0;
+}
+
+static unsigned int i9xx_fsb_freq(struct intel_display *display)
+{
+	struct drm_i915_private *i915 = to_i915(display->drm);
+	u32 fsb;
+
+	/*
+	 * Note that this only reads the state of the FSB
+	 * straps, not the actual FSB frequency. Some BIOSen
+	 * let you configure each independently. Ideally we'd
+	 * read out the actual FSB frequency but sadly we
+	 * don't know which registers have that information,
+	 * and all the relevant docs have gone to bit heaven :(
+	 */
+	fsb = intel_uncore_read(&i915->uncore, CLKCFG) & CLKCFG_FSB_MASK;
+
+	if (display->platform.pineview || display->platform.mobile) {
+		switch (fsb) {
+		case CLKCFG_FSB_400:
+			return 400000;
+		case CLKCFG_FSB_533:
+			return 533333;
+		case CLKCFG_FSB_667:
+			return 666667;
+		case CLKCFG_FSB_800:
+			return 800000;
+		case CLKCFG_FSB_1067:
+			return 1066667;
+		case CLKCFG_FSB_1333:
+			return 1333333;
+		default:
+			MISSING_CASE(fsb);
+			return 1333333;
+		}
+	} else {
+		switch (fsb) {
+		case CLKCFG_FSB_400_ALT:
+			return 400000;
+		case CLKCFG_FSB_533:
+			return 533333;
+		case CLKCFG_FSB_667:
+			return 666667;
+		case CLKCFG_FSB_800:
+			return 800000;
+		case CLKCFG_FSB_1067_ALT:
+			return 1066667;
+		case CLKCFG_FSB_1333_ALT:
+			return 1333333;
+		case CLKCFG_FSB_1600_ALT:
+			return 1600000;
+		default:
+			MISSING_CASE(fsb);
+			return 1333333;
+		}
+	}
+}
+
+static unsigned int ilk_fsb_freq(struct intel_display *display)
+{
+	struct drm_i915_private *dev_priv = to_i915(display->drm);
+	u16 fsb;
+
+	fsb = intel_uncore_read16(&dev_priv->uncore, CSIPLL0) & 0x3ff;
+
+	switch (fsb) {
+	case 0x00c:
+		return 3200000;
+	case 0x00e:
+		return 3733333;
+	case 0x010:
+		return 4266667;
+	case 0x012:
+		return 4800000;
+	case 0x014:
+		return 5333333;
+	case 0x016:
+		return 5866667;
+	case 0x018:
+		return 6400000;
+	default:
+		drm_dbg_kms(display->drm, "unknown fsb frequency 0x%04x\n", fsb);
+		return 0;
+	}
+}
+
+unsigned int intel_fsb_freq(struct intel_display *display)
+{
+	if (DISPLAY_VER(display) == 5)
+		return ilk_fsb_freq(display);
+	else if (IS_DISPLAY_VER(display, 3, 4))
+		return i9xx_fsb_freq(display);
+	else
+		return 0;
+}
+
+static int i915_get_dram_info(struct intel_display *display, struct dram_info *dram_info)
+{
+	dram_info->fsb_freq = intel_fsb_freq(display);
+	if (dram_info->fsb_freq)
+		drm_dbg_kms(display->drm, "FSB frequency: %d kHz\n", dram_info->fsb_freq);
+
+	dram_info->mem_freq = intel_mem_freq(display);
+	if (dram_info->mem_freq)
+		drm_dbg_kms(display->drm, "DDR speed: %d kHz\n", dram_info->mem_freq);
+
+	if (display->platform.pineview)
+		dram_info->type = pnv_dram_type(display);
+
+	return 0;
+}
+
+static int intel_dimm_num_devices(const struct dram_dimm_info *dimm)
+{
+	return dimm->ranks * 64 / (dimm->width ?: 1);
+}
+
+/* Returns total Gb for the whole DIMM */
+static int skl_get_dimm_s_size(u32 val)
+{
+	return REG_FIELD_GET(SKL_DIMM_S_SIZE_MASK, val) * 8;
+}
+
+static int skl_get_dimm_l_size(u32 val)
+{
+	return REG_FIELD_GET(SKL_DIMM_L_SIZE_MASK, val) * 8;
+}
+
+static int skl_get_dimm_s_width(u32 val)
+{
+	if (skl_get_dimm_s_size(val) == 0)
+		return 0;
+
+	switch (val & SKL_DIMM_S_WIDTH_MASK) {
+	case SKL_DIMM_S_WIDTH_X8:
+	case SKL_DIMM_S_WIDTH_X16:
+	case SKL_DIMM_S_WIDTH_X32:
+		return 8 << REG_FIELD_GET(SKL_DIMM_S_WIDTH_MASK, val);
+	default:
+		MISSING_CASE(val);
+		return 0;
+	}
+}
+
+static int skl_get_dimm_l_width(u32 val)
+{
+	if (skl_get_dimm_l_size(val) == 0)
+		return 0;
+
+	switch (val & SKL_DIMM_L_WIDTH_MASK) {
+	case SKL_DIMM_L_WIDTH_X8:
+	case SKL_DIMM_L_WIDTH_X16:
+	case SKL_DIMM_L_WIDTH_X32:
+		return 8 << REG_FIELD_GET(SKL_DIMM_L_WIDTH_MASK, val);
+	default:
+		MISSING_CASE(val);
+		return 0;
+	}
+}
+
+static int skl_get_dimm_s_ranks(u32 val)
+{
+	if (skl_get_dimm_s_size(val) == 0)
+		return 0;
+
+	return REG_FIELD_GET(SKL_DIMM_S_RANK_MASK, val) + 1;
+}
+
+static int skl_get_dimm_l_ranks(u32 val)
+{
+	if (skl_get_dimm_l_size(val) == 0)
+		return 0;
+
+	return REG_FIELD_GET(SKL_DIMM_L_RANK_MASK, val) + 1;
+}
+
+/* Returns total Gb for the whole DIMM */
+static int icl_get_dimm_s_size(u32 val)
+{
+	return REG_FIELD_GET(ICL_DIMM_S_SIZE_MASK, val) * 8 / 2;
+}
+
+static int icl_get_dimm_l_size(u32 val)
+{
+	return REG_FIELD_GET(ICL_DIMM_L_SIZE_MASK, val) * 8 / 2;
+}
+
+static int icl_get_dimm_s_width(u32 val)
+{
+	if (icl_get_dimm_s_size(val) == 0)
+		return 0;
+
+	switch (val & ICL_DIMM_S_WIDTH_MASK) {
+	case ICL_DIMM_S_WIDTH_X8:
+	case ICL_DIMM_S_WIDTH_X16:
+	case ICL_DIMM_S_WIDTH_X32:
+		return 8 << REG_FIELD_GET(ICL_DIMM_S_WIDTH_MASK, val);
+	default:
+		MISSING_CASE(val);
+		return 0;
+	}
+}
+
+static int icl_get_dimm_l_width(u32 val)
+{
+	if (icl_get_dimm_l_size(val) == 0)
+		return 0;
+
+	switch (val & ICL_DIMM_L_WIDTH_MASK) {
+	case ICL_DIMM_L_WIDTH_X8:
+	case ICL_DIMM_L_WIDTH_X16:
+	case ICL_DIMM_L_WIDTH_X32:
+		return 8 << REG_FIELD_GET(ICL_DIMM_L_WIDTH_MASK, val);
+	default:
+		MISSING_CASE(val);
+		return 0;
+	}
+}
+
+static int icl_get_dimm_s_ranks(u32 val)
+{
+	if (icl_get_dimm_s_size(val) == 0)
+		return 0;
+
+	return REG_FIELD_GET(ICL_DIMM_S_RANK_MASK, val) + 1;
+}
+
+static int icl_get_dimm_l_ranks(u32 val)
+{
+	if (icl_get_dimm_l_size(val) == 0)
+		return 0;
+
+	return REG_FIELD_GET(ICL_DIMM_L_RANK_MASK, val) + 1;
+}
+
+static bool
+skl_is_16gb_dimm(const struct dram_dimm_info *dimm)
+{
+	/* Convert total Gb to Gb per DRAM device */
+	return dimm->size / (intel_dimm_num_devices(dimm) ?: 1) >= 16;
+}
+
+static void
+skl_dram_print_dimm_info(struct intel_display *display,
+			 struct dram_dimm_info *dimm,
+			 int channel, char dimm_name)
+{
+	drm_dbg_kms(display->drm,
+		    "CH%u DIMM %c size: %u Gb, width: X%u, ranks: %u, 16Gb+ DIMMs: %s\n",
+		    channel, dimm_name, dimm->size, dimm->width, dimm->ranks,
+		    str_yes_no(skl_is_16gb_dimm(dimm)));
+}
+
+static void
+skl_dram_get_dimm_l_info(struct intel_display *display,
+			 struct dram_dimm_info *dimm,
+			 int channel, u32 val)
+{
+	if (DISPLAY_VER(display) >= 11) {
+		dimm->size = icl_get_dimm_l_size(val);
+		dimm->width = icl_get_dimm_l_width(val);
+		dimm->ranks = icl_get_dimm_l_ranks(val);
+	} else {
+		dimm->size = skl_get_dimm_l_size(val);
+		dimm->width = skl_get_dimm_l_width(val);
+		dimm->ranks = skl_get_dimm_l_ranks(val);
+	}
+
+	skl_dram_print_dimm_info(display, dimm, channel, 'L');
+}
+
+static void
+skl_dram_get_dimm_s_info(struct intel_display *display,
+			 struct dram_dimm_info *dimm,
+			 int channel, u32 val)
+{
+	if (DISPLAY_VER(display) >= 11) {
+		dimm->size = icl_get_dimm_s_size(val);
+		dimm->width = icl_get_dimm_s_width(val);
+		dimm->ranks = icl_get_dimm_s_ranks(val);
+	} else {
+		dimm->size = skl_get_dimm_s_size(val);
+		dimm->width = skl_get_dimm_s_width(val);
+		dimm->ranks = skl_get_dimm_s_ranks(val);
+	}
+
+	skl_dram_print_dimm_info(display, dimm, channel, 'S');
+}
+
+static int
+skl_dram_get_channel_info(struct intel_display *display,
+			  struct dram_channel_info *ch,
+			  int channel, u32 val)
+{
+	skl_dram_get_dimm_l_info(display, &ch->dimm_l, channel, val);
+	skl_dram_get_dimm_s_info(display, &ch->dimm_s, channel, val);
+
+	if (ch->dimm_l.size == 0 && ch->dimm_s.size == 0) {
+		drm_dbg_kms(display->drm, "CH%u not populated\n", channel);
+		return -EINVAL;
+	}
+
+	if (ch->dimm_l.ranks == 2 || ch->dimm_s.ranks == 2)
+		ch->ranks = 2;
+	else if (ch->dimm_l.ranks == 1 && ch->dimm_s.ranks == 1)
+		ch->ranks = 2;
+	else
+		ch->ranks = 1;
+
+	ch->is_16gb_dimm = skl_is_16gb_dimm(&ch->dimm_l) ||
+		skl_is_16gb_dimm(&ch->dimm_s);
+
+	drm_dbg_kms(display->drm, "CH%u ranks: %u, 16Gb+ DIMMs: %s\n",
+		    channel, ch->ranks, str_yes_no(ch->is_16gb_dimm));
+
+	return 0;
+}
+
+static bool
+intel_is_dram_symmetric(const struct dram_channel_info *ch0,
+			const struct dram_channel_info *ch1)
+{
+	return !memcmp(ch0, ch1, sizeof(*ch0)) &&
+		(ch0->dimm_s.size == 0 ||
+		 !memcmp(&ch0->dimm_l, &ch0->dimm_s, sizeof(ch0->dimm_l)));
+}
+
+static int
+skl_dram_get_channels_info(struct intel_display *display, struct dram_info *dram_info)
+{
+	struct drm_i915_private *i915 = to_i915(display->drm);
+	struct dram_channel_info ch0 = {}, ch1 = {};
+	u32 val;
+	int ret;
+
+	/* Assume 16Gb+ DIMMs are present until proven otherwise */
+	dram_info->has_16gb_dimms = true;
+
+	val = intel_uncore_read(&i915->uncore,
+				SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN);
+	ret = skl_dram_get_channel_info(display, &ch0, 0, val);
+	if (ret == 0)
+		dram_info->num_channels++;
+
+	val = intel_uncore_read(&i915->uncore,
+				SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN);
+	ret = skl_dram_get_channel_info(display, &ch1, 1, val);
+	if (ret == 0)
+		dram_info->num_channels++;
+
+	if (dram_info->num_channels == 0) {
+		drm_info(display->drm, "Number of memory channels is zero\n");
+		return -EINVAL;
+	}
+
+	if (ch0.ranks == 0 && ch1.ranks == 0) {
+		drm_info(display->drm, "couldn't get memory rank information\n");
+		return -EINVAL;
+	}
+
+	dram_info->has_16gb_dimms = ch0.is_16gb_dimm || ch1.is_16gb_dimm;
+
+	dram_info->symmetric_memory = intel_is_dram_symmetric(&ch0, &ch1);
+
+	drm_dbg_kms(display->drm, "Memory configuration is symmetric? %s\n",
+		    str_yes_no(dram_info->symmetric_memory));
+
+	drm_dbg_kms(display->drm, "16Gb+ DIMMs: %s\n",
+		    str_yes_no(dram_info->has_16gb_dimms));
+
+	return 0;
+}
+
+static enum intel_dram_type
+skl_get_dram_type(struct intel_display *display)
+{
+	struct drm_i915_private *i915 = to_i915(display->drm);
+	u32 val;
+
+	val = intel_uncore_read(&i915->uncore,
+				SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN);
+
+	switch (val & SKL_DRAM_DDR_TYPE_MASK) {
+	case SKL_DRAM_DDR_TYPE_DDR3:
+		return INTEL_DRAM_DDR3;
+	case SKL_DRAM_DDR_TYPE_DDR4:
+		return INTEL_DRAM_DDR4;
+	case SKL_DRAM_DDR_TYPE_LPDDR3:
+		return INTEL_DRAM_LPDDR3;
+	case SKL_DRAM_DDR_TYPE_LPDDR4:
+		return INTEL_DRAM_LPDDR4;
+	default:
+		MISSING_CASE(val);
+		return INTEL_DRAM_UNKNOWN;
+	}
+}
+
+static int
+skl_get_dram_info(struct intel_display *display, struct dram_info *dram_info)
+{
+	int ret;
+
+	dram_info->type = skl_get_dram_type(display);
+
+	ret = skl_dram_get_channels_info(display, dram_info);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+/* Returns Gb per DRAM device */
+static int bxt_get_dimm_size(u32 val)
+{
+	switch (val & BXT_DRAM_SIZE_MASK) {
+	case BXT_DRAM_SIZE_4GBIT:
+		return 4;
+	case BXT_DRAM_SIZE_6GBIT:
+		return 6;
+	case BXT_DRAM_SIZE_8GBIT:
+		return 8;
+	case BXT_DRAM_SIZE_12GBIT:
+		return 12;
+	case BXT_DRAM_SIZE_16GBIT:
+		return 16;
+	default:
+		MISSING_CASE(val);
+		return 0;
+	}
+}
+
+static int bxt_get_dimm_width(u32 val)
+{
+	if (!bxt_get_dimm_size(val))
+		return 0;
+
+	val = (val & BXT_DRAM_WIDTH_MASK) >> BXT_DRAM_WIDTH_SHIFT;
+
+	return 8 << val;
+}
+
+static int bxt_get_dimm_ranks(u32 val)
+{
+	if (!bxt_get_dimm_size(val))
+		return 0;
+
+	switch (val & BXT_DRAM_RANK_MASK) {
+	case BXT_DRAM_RANK_SINGLE:
+		return 1;
+	case BXT_DRAM_RANK_DUAL:
+		return 2;
+	default:
+		MISSING_CASE(val);
+		return 0;
+	}
+}
+
+static enum intel_dram_type bxt_get_dimm_type(u32 val)
+{
+	if (!bxt_get_dimm_size(val))
+		return INTEL_DRAM_UNKNOWN;
+
+	switch (val & BXT_DRAM_TYPE_MASK) {
+	case BXT_DRAM_TYPE_DDR3:
+		return INTEL_DRAM_DDR3;
+	case BXT_DRAM_TYPE_LPDDR3:
+		return INTEL_DRAM_LPDDR3;
+	case BXT_DRAM_TYPE_DDR4:
+		return INTEL_DRAM_DDR4;
+	case BXT_DRAM_TYPE_LPDDR4:
+		return INTEL_DRAM_LPDDR4;
+	default:
+		MISSING_CASE(val);
+		return INTEL_DRAM_UNKNOWN;
+	}
+}
+
+static void bxt_get_dimm_info(struct dram_dimm_info *dimm, u32 val)
+{
+	dimm->width = bxt_get_dimm_width(val);
+	dimm->ranks = bxt_get_dimm_ranks(val);
+
+	/*
+	 * Size in register is Gb per DRAM device. Convert to total
+	 * Gb to match the way we report this for non-LP platforms.
+	 */
+	dimm->size = bxt_get_dimm_size(val) * intel_dimm_num_devices(dimm);
+}
+
+static int bxt_get_dram_info(struct intel_display *display, struct dram_info *dram_info)
+{
+	struct drm_i915_private *i915 = to_i915(display->drm);
+	u32 val;
+	u8 valid_ranks = 0;
+	int i;
+
+	/*
+	 * Now read each DUNIT8/9/10/11 to check the rank of each dimms.
+	 */
+	for (i = BXT_D_CR_DRP0_DUNIT_START; i <= BXT_D_CR_DRP0_DUNIT_END; i++) {
+		struct dram_dimm_info dimm;
+		enum intel_dram_type type;
+
+		val = intel_uncore_read(&i915->uncore, BXT_D_CR_DRP0_DUNIT(i));
+		if (val == 0xFFFFFFFF)
+			continue;
+
+		dram_info->num_channels++;
+
+		bxt_get_dimm_info(&dimm, val);
+		type = bxt_get_dimm_type(val);
+
+		drm_WARN_ON(display->drm, type != INTEL_DRAM_UNKNOWN &&
+			    dram_info->type != INTEL_DRAM_UNKNOWN &&
+			    dram_info->type != type);
+
+		drm_dbg_kms(display->drm,
+			    "CH%u DIMM size: %u Gb, width: X%u, ranks: %u\n",
+			    i - BXT_D_CR_DRP0_DUNIT_START,
+			    dimm.size, dimm.width, dimm.ranks);
+
+		if (valid_ranks == 0)
+			valid_ranks = dimm.ranks;
+
+		if (type != INTEL_DRAM_UNKNOWN)
+			dram_info->type = type;
+	}
+
+	if (dram_info->type == INTEL_DRAM_UNKNOWN || valid_ranks == 0) {
+		drm_info(display->drm, "couldn't get memory information\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int icl_pcode_read_mem_global_info(struct intel_display *display,
+					  struct dram_info *dram_info)
+{
+	u32 val = 0;
+	int ret;
+
+	ret = intel_pcode_read(display->drm, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
+			       ICL_PCODE_MEM_SS_READ_GLOBAL_INFO, &val, NULL);
+	if (ret)
+		return ret;
+
+	if (DISPLAY_VER(display) >= 12) {
+		switch (val & 0xf) {
+		case 0:
+			dram_info->type = INTEL_DRAM_DDR4;
+			break;
+		case 1:
+			dram_info->type = INTEL_DRAM_DDR5;
+			break;
+		case 2:
+			dram_info->type = INTEL_DRAM_LPDDR5;
+			break;
+		case 3:
+			dram_info->type = INTEL_DRAM_LPDDR4;
+			break;
+		case 4:
+			dram_info->type = INTEL_DRAM_DDR3;
+			break;
+		case 5:
+			dram_info->type = INTEL_DRAM_LPDDR3;
+			break;
+		default:
+			MISSING_CASE(val & 0xf);
+			return -EINVAL;
+		}
+	} else {
+		switch (val & 0xf) {
+		case 0:
+			dram_info->type = INTEL_DRAM_DDR4;
+			break;
+		case 1:
+			dram_info->type = INTEL_DRAM_DDR3;
+			break;
+		case 2:
+			dram_info->type = INTEL_DRAM_LPDDR3;
+			break;
+		case 3:
+			dram_info->type = INTEL_DRAM_LPDDR4;
+			break;
+		default:
+			MISSING_CASE(val & 0xf);
+			return -EINVAL;
+		}
+	}
+
+	dram_info->num_channels = (val & 0xf0) >> 4;
+	dram_info->num_qgv_points = (val & 0xf00) >> 8;
+	dram_info->num_psf_gv_points = (val & 0x3000) >> 12;
+
+	return 0;
+}
+
+static int gen11_get_dram_info(struct intel_display *display, struct dram_info *dram_info)
+{
+	int ret;
+
+	ret = skl_dram_get_channels_info(display, dram_info);
+	if (ret)
+		return ret;
+
+	return icl_pcode_read_mem_global_info(display, dram_info);
+}
+
+static int gen12_get_dram_info(struct intel_display *display, struct dram_info *dram_info)
+{
+	return icl_pcode_read_mem_global_info(display, dram_info);
+}
+
+static int xelpdp_get_dram_info(struct intel_display *display, struct dram_info *dram_info)
+{
+	struct drm_i915_private *i915 = to_i915(display->drm);
+	u32 val = intel_uncore_read(&i915->uncore, MTL_MEM_SS_INFO_GLOBAL);
+
+	switch (REG_FIELD_GET(MTL_DDR_TYPE_MASK, val)) {
+	case 0:
+		dram_info->type = INTEL_DRAM_DDR4;
+		break;
+	case 1:
+		dram_info->type = INTEL_DRAM_DDR5;
+		break;
+	case 2:
+		dram_info->type = INTEL_DRAM_LPDDR5;
+		break;
+	case 3:
+		dram_info->type = INTEL_DRAM_LPDDR4;
+		break;
+	case 4:
+		dram_info->type = INTEL_DRAM_DDR3;
+		break;
+	case 5:
+		dram_info->type = INTEL_DRAM_LPDDR3;
+		break;
+	case 8:
+		drm_WARN_ON(display->drm, !display->platform.dgfx);
+		dram_info->type = INTEL_DRAM_GDDR;
+		break;
+	case 9:
+		drm_WARN_ON(display->drm, !display->platform.dgfx);
+		dram_info->type = INTEL_DRAM_GDDR_ECC;
+		break;
+	default:
+		MISSING_CASE(val);
+		return -EINVAL;
+	}
+
+	dram_info->num_channels = REG_FIELD_GET(MTL_N_OF_POPULATED_CH_MASK, val);
+	dram_info->num_qgv_points = REG_FIELD_GET(MTL_N_OF_ENABLED_QGV_POINTS_MASK, val);
+	/* PSF GV points not supported in D14+ */
+
+	if (DISPLAY_VER(display) >= 35)
+		dram_info->ecc_impacting_de_bw = REG_FIELD_GET(XE3P_ECC_IMPACTING_DE, val);
+
+	return 0;
+}
+
+int intel_dram_detect(struct intel_display *display)
+{
+	struct dram_info *dram_info;
+	int ret;
+
+	if (display->platform.dg2 || !HAS_DISPLAY(display))
+		return 0;
+
+	dram_info = drmm_kzalloc(display->drm, sizeof(*dram_info), GFP_KERNEL);
+	if (!dram_info)
+		return -ENOMEM;
+
+	display->dram.info = dram_info;
+
+	if (DISPLAY_VER(display) >= 14)
+		ret = xelpdp_get_dram_info(display, dram_info);
+	else if (DISPLAY_VER(display) >= 12)
+		ret = gen12_get_dram_info(display, dram_info);
+	else if (DISPLAY_VER(display) >= 11)
+		ret = gen11_get_dram_info(display, dram_info);
+	else if (display->platform.broxton || display->platform.geminilake)
+		ret = bxt_get_dram_info(display, dram_info);
+	else if (DISPLAY_VER(display) >= 9)
+		ret = skl_get_dram_info(display, dram_info);
+	else
+		ret = i915_get_dram_info(display, dram_info);
+
+	drm_dbg_kms(display->drm, "DRAM type: %s\n",
+		    intel_dram_type_str(dram_info->type));
+
+	drm_dbg_kms(display->drm, "DRAM channels: %u\n", dram_info->num_channels);
+
+	drm_dbg_kms(display->drm, "Num QGV points %u\n", dram_info->num_qgv_points);
+	drm_dbg_kms(display->drm, "Num PSF GV points %u\n", dram_info->num_psf_gv_points);
+
+	/* TODO: Do we want to abort probe on dram detection failures? */
+	if (ret)
+		return 0;
+
+	return 0;
+}
+
+/*
+ * Returns NULL for platforms that don't have dram info. Avoid overzealous NULL
+ * checks, and prefer not dereferencing on platforms that shouldn't look at dram
+ * info, to catch accidental and incorrect dram info checks.
+ */
+const struct dram_info *intel_dram_info(struct intel_display *display)
+{
+	return display->dram.info;
+}
diff --git a/drivers/gpu/drm/i915/display/intel_dram.h b/drivers/gpu/drm/i915/display/intel_dram.h
new file mode 100644
index 000000000000..5800b7b4e614
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_dram.h
@@ -0,0 +1,43 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2020 Intel Corporation
+ */
+
+#ifndef __INTEL_DRAM_H__
+#define __INTEL_DRAM_H__
+
+#include <linux/types.h>
+
+struct intel_display;
+
+struct dram_info {
+	enum intel_dram_type {
+		INTEL_DRAM_UNKNOWN,
+		INTEL_DRAM_DDR2,
+		INTEL_DRAM_DDR3,
+		INTEL_DRAM_DDR4,
+		INTEL_DRAM_LPDDR3,
+		INTEL_DRAM_LPDDR4,
+		INTEL_DRAM_DDR5,
+		INTEL_DRAM_LPDDR5,
+		INTEL_DRAM_GDDR,
+		INTEL_DRAM_GDDR_ECC,
+		__INTEL_DRAM_TYPE_MAX,
+	} type;
+	unsigned int fsb_freq;
+	unsigned int mem_freq;
+	u8 num_channels;
+	u8 num_qgv_points;
+	u8 num_psf_gv_points;
+	bool ecc_impacting_de_bw; /* Only valid from Xe3p_LPD onward. */
+	bool symmetric_memory;
+	bool has_16gb_dimms;
+};
+
+int intel_dram_detect(struct intel_display *display);
+unsigned int intel_fsb_freq(struct intel_display *display);
+unsigned int intel_mem_freq(struct intel_display *display);
+const struct dram_info *intel_dram_info(struct intel_display *display);
+const char *intel_dram_type_str(enum intel_dram_type type);
+
+#endif /* __INTEL_DRAM_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dsb.c b/drivers/gpu/drm/i915/display/intel_dsb.c
index 4ad4efbf9253..ec2a3fb171ab 100644
--- a/drivers/gpu/drm/i915/display/intel_dsb.c
+++ b/drivers/gpu/drm/i915/display/intel_dsb.c
@@ -26,7 +26,7 @@
 struct intel_dsb {
 	enum intel_dsb_id id;
 
-	struct intel_dsb_buffer dsb_buf;
+	struct intel_dsb_buffer *dsb_buf;
 	struct intel_crtc *crtc;
 
 	/*
@@ -211,10 +211,10 @@ static void intel_dsb_dump(struct intel_dsb *dsb)
 	for (i = 0; i < ALIGN(dsb->free_pos, 64 / 4); i += 4)
 		drm_dbg_kms(display->drm,
 			    " 0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i * 4,
-			    intel_dsb_buffer_read(&dsb->dsb_buf, i),
-			    intel_dsb_buffer_read(&dsb->dsb_buf, i + 1),
-			    intel_dsb_buffer_read(&dsb->dsb_buf, i + 2),
-			    intel_dsb_buffer_read(&dsb->dsb_buf, i + 3));
+			    intel_dsb_buffer_read(dsb->dsb_buf, i),
+			    intel_dsb_buffer_read(dsb->dsb_buf, i + 1),
+			    intel_dsb_buffer_read(dsb->dsb_buf, i + 2),
+			    intel_dsb_buffer_read(dsb->dsb_buf, i + 3));
 	drm_dbg_kms(display->drm, "}\n");
 }
 
@@ -231,12 +231,12 @@ unsigned int intel_dsb_size(struct intel_dsb *dsb)
 
 unsigned int intel_dsb_head(struct intel_dsb *dsb)
 {
-	return intel_dsb_buffer_ggtt_offset(&dsb->dsb_buf);
+	return intel_dsb_buffer_ggtt_offset(dsb->dsb_buf);
 }
 
 static unsigned int intel_dsb_tail(struct intel_dsb *dsb)
 {
-	return intel_dsb_buffer_ggtt_offset(&dsb->dsb_buf) + intel_dsb_size(dsb);
+	return intel_dsb_buffer_ggtt_offset(dsb->dsb_buf) + intel_dsb_size(dsb);
 }
 
 static void intel_dsb_ins_align(struct intel_dsb *dsb)
@@ -263,8 +263,8 @@ static void intel_dsb_emit(struct intel_dsb *dsb, u32 ldw, u32 udw)
 	dsb->ins[0] = ldw;
 	dsb->ins[1] = udw;
 
-	intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, dsb->ins[0]);
-	intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, dsb->ins[1]);
+	intel_dsb_buffer_write(dsb->dsb_buf, dsb->free_pos++, dsb->ins[0]);
+	intel_dsb_buffer_write(dsb->dsb_buf, dsb->free_pos++, dsb->ins[1]);
 }
 
 static bool intel_dsb_prev_ins_is_write(struct intel_dsb *dsb,
@@ -335,13 +335,13 @@ void intel_dsb_reg_write_indexed(struct intel_dsb *dsb,
 
 	/* Update the count */
 	dsb->ins[0]++;
-	intel_dsb_buffer_write(&dsb->dsb_buf, dsb->ins_start_offset + 0,
+	intel_dsb_buffer_write(dsb->dsb_buf, dsb->ins_start_offset + 0,
 			       dsb->ins[0]);
 
-	intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, val);
+	intel_dsb_buffer_write(dsb->dsb_buf, dsb->free_pos++, val);
 	/* if number of data words is odd, then the last dword should be 0.*/
 	if (dsb->free_pos & 0x1)
-		intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos, 0);
+		intel_dsb_buffer_write(dsb->dsb_buf, dsb->free_pos, 0);
 }
 
 void intel_dsb_reg_write(struct intel_dsb *dsb,
@@ -521,7 +521,7 @@ static void intel_dsb_align_tail(struct intel_dsb *dsb)
 	aligned_tail = ALIGN(tail, CACHELINE_BYTES);
 
 	if (aligned_tail > tail)
-		intel_dsb_buffer_memset(&dsb->dsb_buf, dsb->free_pos, 0,
+		intel_dsb_buffer_memset(dsb->dsb_buf, dsb->free_pos, 0,
 					aligned_tail - tail);
 
 	dsb->free_pos = aligned_tail / 4;
@@ -541,7 +541,7 @@ static void intel_dsb_gosub_align(struct intel_dsb *dsb)
 	 * "Ensure GOSUB is not placed in cacheline QW slot 6 or 7 (numbered 0-7)"
 	 */
 	if (aligned_tail - tail <= 2 * 8)
-		intel_dsb_buffer_memset(&dsb->dsb_buf, dsb->free_pos, 0,
+		intel_dsb_buffer_memset(dsb->dsb_buf, dsb->free_pos, 0,
 					aligned_tail - tail);
 
 	dsb->free_pos = aligned_tail / 4;
@@ -606,14 +606,14 @@ void intel_dsb_gosub_finish(struct intel_dsb *dsb)
 	 */
 	intel_dsb_noop(dsb, 8);
 
-	intel_dsb_buffer_flush_map(&dsb->dsb_buf);
+	intel_dsb_buffer_flush_map(dsb->dsb_buf);
 }
 
 void intel_dsb_finish(struct intel_dsb *dsb)
 {
 	intel_dsb_align_tail(dsb);
 
-	intel_dsb_buffer_flush_map(&dsb->dsb_buf);
+	intel_dsb_buffer_flush_map(dsb->dsb_buf);
 }
 
 static u32 dsb_error_int_status(struct intel_display *display)
@@ -888,7 +888,7 @@ void intel_dsb_wait(struct intel_dsb *dsb)
 			      !is_busy,
 			      100, 1000, false);
 	if (ret) {
-		u32 offset = intel_dsb_buffer_ggtt_offset(&dsb->dsb_buf);
+		u32 offset = intel_dsb_buffer_ggtt_offset(dsb->dsb_buf);
 
 		intel_de_write_fw(display, DSB_CTRL(pipe, dsb->id),
 				  DSB_ENABLE | DSB_HALT);
@@ -934,6 +934,7 @@ struct intel_dsb *intel_dsb_prepare(struct intel_atomic_state *state,
 				    unsigned int max_cmds)
 {
 	struct intel_display *display = to_intel_display(state);
+	struct intel_dsb_buffer *dsb_buf;
 	struct ref_tracker *wakeref;
 	struct intel_dsb *dsb;
 	unsigned int size;
@@ -953,9 +954,12 @@ struct intel_dsb *intel_dsb_prepare(struct intel_atomic_state *state,
 	/* ~1 qword per instruction, full cachelines */
 	size = ALIGN(max_cmds * 8, CACHELINE_BYTES);
 
-	if (!intel_dsb_buffer_create(crtc, &dsb->dsb_buf, size))
+	dsb_buf = intel_dsb_buffer_create(display->drm, size);
+	if (IS_ERR(dsb_buf))
 		goto out_put_rpm;
 
+	dsb->dsb_buf = dsb_buf;
+
 	intel_display_rpm_put(display, wakeref);
 
 	dsb->id = dsb_id;
@@ -988,7 +992,7 @@ struct intel_dsb *intel_dsb_prepare(struct intel_atomic_state *state,
  */
 void intel_dsb_cleanup(struct intel_dsb *dsb)
 {
-	intel_dsb_buffer_cleanup(&dsb->dsb_buf);
+	intel_dsb_buffer_cleanup(dsb->dsb_buf);
 	kfree(dsb);
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_dsb_buffer.c b/drivers/gpu/drm/i915/display/intel_dsb_buffer.c
index c77d48bda26a..50faf3869b6c 100644
--- a/drivers/gpu/drm/i915/display/intel_dsb_buffer.c
+++ b/drivers/gpu/drm/i915/display/intel_dsb_buffer.c
@@ -7,9 +7,14 @@
 #include "gem/i915_gem_lmem.h"
 #include "i915_drv.h"
 #include "i915_vma.h"
-#include "intel_display_types.h"
 #include "intel_dsb_buffer.h"
 
+struct intel_dsb_buffer {
+	u32 *cmd_buf;
+	struct i915_vma *vma;
+	size_t buf_size;
+};
+
 u32 intel_dsb_buffer_ggtt_offset(struct intel_dsb_buffer *dsb_buf)
 {
 	return i915_ggtt_offset(dsb_buf->vma);
@@ -32,48 +37,66 @@ void intel_dsb_buffer_memset(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val,
 	memset(&dsb_buf->cmd_buf[idx], val, size);
 }
 
-bool intel_dsb_buffer_create(struct intel_crtc *crtc, struct intel_dsb_buffer *dsb_buf, size_t size)
+struct intel_dsb_buffer *intel_dsb_buffer_create(struct drm_device *drm, size_t size)
 {
-	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	struct drm_i915_private *i915 = to_i915(drm);
+	struct intel_dsb_buffer *dsb_buf;
 	struct drm_i915_gem_object *obj;
 	struct i915_vma *vma;
 	u32 *buf;
+	int ret;
+
+	dsb_buf = kzalloc(sizeof(*dsb_buf), GFP_KERNEL);
+	if (!dsb_buf)
+		return ERR_PTR(-ENOMEM);
 
 	if (HAS_LMEM(i915)) {
 		obj = i915_gem_object_create_lmem(i915, PAGE_ALIGN(size),
 						  I915_BO_ALLOC_CONTIGUOUS);
-		if (IS_ERR(obj))
-			return false;
+		if (IS_ERR(obj)) {
+			ret = PTR_ERR(obj);
+			goto err;
+		}
 	} else {
 		obj = i915_gem_object_create_internal(i915, PAGE_ALIGN(size));
-		if (IS_ERR(obj))
-			return false;
+		if (IS_ERR(obj)) {
+			ret = PTR_ERR(obj);
+			goto err;
+		}
 
 		i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE);
 	}
 
 	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
 	if (IS_ERR(vma)) {
+		ret = PTR_ERR(vma);
 		i915_gem_object_put(obj);
-		return false;
+		goto err;
 	}
 
 	buf = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WC);
 	if (IS_ERR(buf)) {
+		ret = PTR_ERR(buf);
 		i915_vma_unpin_and_release(&vma, I915_VMA_RELEASE_MAP);
-		return false;
+		goto err;
 	}
 
 	dsb_buf->vma = vma;
 	dsb_buf->cmd_buf = buf;
 	dsb_buf->buf_size = size;
 
-	return true;
+	return dsb_buf;
+
+err:
+	kfree(dsb_buf);
+
+	return ERR_PTR(ret);
 }
 
 void intel_dsb_buffer_cleanup(struct intel_dsb_buffer *dsb_buf)
 {
 	i915_vma_unpin_and_release(&dsb_buf->vma, I915_VMA_RELEASE_MAP);
+	kfree(dsb_buf);
 }
 
 void intel_dsb_buffer_flush_map(struct intel_dsb_buffer *dsb_buf)
diff --git a/drivers/gpu/drm/i915/display/intel_dsb_buffer.h b/drivers/gpu/drm/i915/display/intel_dsb_buffer.h
index 425acd393905..d746c872e0c7 100644
--- a/drivers/gpu/drm/i915/display/intel_dsb_buffer.h
+++ b/drivers/gpu/drm/i915/display/intel_dsb_buffer.h
@@ -8,21 +8,14 @@
 
 #include <linux/types.h>
 
-struct intel_crtc;
-struct i915_vma;
-
-struct intel_dsb_buffer {
-	u32 *cmd_buf;
-	struct i915_vma *vma;
-	size_t buf_size;
-};
+struct drm_device;
+struct intel_dsb_buffer;
 
 u32 intel_dsb_buffer_ggtt_offset(struct intel_dsb_buffer *dsb_buf);
 void intel_dsb_buffer_write(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val);
 u32 intel_dsb_buffer_read(struct intel_dsb_buffer *dsb_buf, u32 idx);
 void intel_dsb_buffer_memset(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val, size_t size);
-bool intel_dsb_buffer_create(struct intel_crtc *crtc, struct intel_dsb_buffer *dsb_buf,
-			     size_t size);
+struct intel_dsb_buffer *intel_dsb_buffer_create(struct drm_device *drm, size_t size);
 void intel_dsb_buffer_cleanup(struct intel_dsb_buffer *dsb_buf);
 void intel_dsb_buffer_flush_map(struct intel_dsb_buffer *dsb_buf);
 
diff --git a/drivers/gpu/drm/i915/display/intel_dsi.h b/drivers/gpu/drm/i915/display/intel_dsi.h
index 89c7166a3860..489d26ffd235 100644
--- a/drivers/gpu/drm/i915/display/intel_dsi.h
+++ b/drivers/gpu/drm/i915/display/intel_dsi.h
@@ -29,6 +29,9 @@
 
 #include "intel_display_types.h"
 
+struct intel_dsi_host;
+struct ref_tracker;
+
 #define INTEL_DSI_VIDEO_MODE	0
 #define INTEL_DSI_COMMAND_MODE	1
 
@@ -37,13 +40,11 @@
 #define DSI_DUAL_LINK_FRONT_BACK	1
 #define DSI_DUAL_LINK_PIXEL_ALT		2
 
-struct intel_dsi_host;
-
 struct intel_dsi {
 	struct intel_encoder base;
 
 	struct intel_dsi_host *dsi_hosts[I915_MAX_PORTS];
-	intel_wakeref_t io_wakeref[I915_MAX_PORTS];
+	struct ref_tracker *io_wakeref[I915_MAX_PORTS];
 
 	/* GPIO Desc for panel and backlight control */
 	struct gpio_desc *gpio_panel;
diff --git a/drivers/gpu/drm/i915/display/intel_fb.c b/drivers/gpu/drm/i915/display/intel_fb.c
index b34b4961fe1c..b9bd9b6dfe94 100644
--- a/drivers/gpu/drm/i915/display/intel_fb.c
+++ b/drivers/gpu/drm/i915/display/intel_fb.c
@@ -20,7 +20,7 @@
 #include "intel_fb.h"
 #include "intel_fb_bo.h"
 #include "intel_frontbuffer.h"
-#include "intel_panic.h"
+#include "intel_parent.h"
 #include "intel_plane.h"
 
 #define check_array_bounds(display, a, i) drm_WARN_ON((display)->drm, (i) >= ARRAY_SIZE(a))
@@ -558,7 +558,7 @@ static bool plane_has_modifier(struct intel_display *display,
 	 * where supported.
 	 */
 	if (intel_fb_is_ccs_modifier(md->modifier) &&
-	    HAS_AUX_CCS(display) != !!md->ccs.packed_aux_planes)
+	    intel_parent_has_auxccs(display) != !!md->ccs.packed_aux_planes)
 		return false;
 
 	if (md->modifier == I915_FORMAT_MOD_4_TILED_BMG_CCS &&
@@ -2216,7 +2216,7 @@ int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
 	int ret;
 	int i;
 
-	intel_fb->panic = intel_panic_alloc();
+	intel_fb->panic = intel_parent_panic_alloc(display);
 	if (!intel_fb->panic)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c
index 437d2fda20a7..fef2f35ff1e9 100644
--- a/drivers/gpu/drm/i915/display/intel_fbc.c
+++ b/drivers/gpu/drm/i915/display/intel_fbc.c
@@ -45,12 +45,6 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_print.h>
 
-#include "gem/i915_gem_stolen.h"
-
-#include "gt/intel_gt_types.h"
-
-#include "i915_drv.h"
-#include "i915_vgpu.h"
 #include "i915_vma.h"
 #include "i9xx_plane_regs.h"
 #include "intel_de.h"
@@ -64,6 +58,8 @@
 #include "intel_fbc.h"
 #include "intel_fbc_regs.h"
 #include "intel_frontbuffer.h"
+#include "intel_parent.h"
+#include "intel_step.h"
 
 #define for_each_fbc_id(__display, __fbc_id) \
 	for ((__fbc_id) = INTEL_FBC_A; (__fbc_id) < I915_MAX_FBCS; (__fbc_id)++) \
@@ -71,7 +67,9 @@
 
 #define for_each_intel_fbc(__display, __fbc, __fbc_id) \
 	for_each_fbc_id((__display), (__fbc_id)) \
-		for_each_if((__fbc) = (__display)->fbc[(__fbc_id)])
+		for_each_if((__fbc) = (__display)->fbc.instances[(__fbc_id)])
+
+#define FBC_SYS_CACHE_ID_NONE	I915_MAX_FBCS
 
 struct intel_fbc_funcs {
 	void (*activate)(struct intel_fbc *fbc);
@@ -129,6 +127,19 @@ struct intel_fbc {
 	const char *no_fbc_reason;
 };
 
+static struct intel_fbc *intel_fbc_for_pipe(struct intel_display *display, enum pipe pipe)
+{
+	struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
+	struct intel_plane *primary = NULL;
+
+	primary = to_intel_plane(crtc->base.primary);
+
+	if (drm_WARN_ON(display->drm, !primary))
+		return NULL;
+
+	return primary->fbc;
+}
+
 /* plane stride in pixels */
 static unsigned int intel_fbc_plane_stride(const struct intel_plane_state *plane_state)
 {
@@ -204,7 +215,7 @@ static unsigned int _intel_fbc_cfb_stride(struct intel_display *display,
 
 static unsigned int intel_fbc_cfb_stride(const struct intel_plane_state *plane_state)
 {
-	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
+	struct intel_display *display = to_intel_display(plane_state);
 	unsigned int stride = intel_fbc_plane_cfb_stride(plane_state);
 	unsigned int width = drm_rect_width(&plane_state->uapi.src) >> 16;
 	unsigned int cpp = intel_fbc_cfb_cpp(plane_state);
@@ -235,7 +246,7 @@ static unsigned int _intel_fbc_cfb_size(struct intel_display *display,
 
 static unsigned int intel_fbc_cfb_size(const struct intel_plane_state *plane_state)
 {
-	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
+	struct intel_display *display = to_intel_display(plane_state);
 	unsigned int height = drm_rect_height(&plane_state->uapi.src) >> 16;
 
 	return _intel_fbc_cfb_size(display, height, intel_fbc_cfb_stride(plane_state));
@@ -243,7 +254,7 @@ static unsigned int intel_fbc_cfb_size(const struct intel_plane_state *plane_sta
 
 static u16 intel_fbc_override_cfb_stride(const struct intel_plane_state *plane_state)
 {
-	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
+	struct intel_display *display = to_intel_display(plane_state);
 	unsigned int stride_aligned = intel_fbc_cfb_stride(plane_state);
 	unsigned int stride = intel_fbc_plane_cfb_stride(plane_state);
 	const struct drm_framebuffer *fb = plane_state->hw.fb;
@@ -267,9 +278,7 @@ static u16 intel_fbc_override_cfb_stride(const struct intel_plane_state *plane_s
 
 static bool intel_fbc_has_fences(struct intel_display *display)
 {
-	struct drm_i915_private __maybe_unused *i915 = to_i915(display->drm);
-
-	return intel_gt_support_legacy_fencing(to_gt(i915));
+	return intel_parent_has_fenced_regions(display);
 }
 
 static u32 i8xx_fbc_ctl(struct intel_fbc *fbc)
@@ -382,17 +391,17 @@ static void i8xx_fbc_program_cfb(struct intel_fbc *fbc)
 	struct intel_display *display = fbc->display;
 
 	drm_WARN_ON(display->drm,
-		    range_end_overflows_t(u64, i915_gem_stolen_area_address(display->drm),
-					  i915_gem_stolen_node_offset(fbc->compressed_fb),
+		    range_end_overflows_t(u64, intel_parent_stolen_area_address(display),
+					  intel_parent_stolen_node_offset(display, fbc->compressed_fb),
 					  U32_MAX));
 	drm_WARN_ON(display->drm,
-		    range_end_overflows_t(u64, i915_gem_stolen_area_address(display->drm),
-					  i915_gem_stolen_node_offset(fbc->compressed_llb),
+		    range_end_overflows_t(u64, intel_parent_stolen_area_address(display),
+					  intel_parent_stolen_node_offset(display, fbc->compressed_llb),
 					  U32_MAX));
 	intel_de_write(display, FBC_CFB_BASE,
-		       i915_gem_stolen_node_address(fbc->compressed_fb));
+		       intel_parent_stolen_node_address(display, fbc->compressed_fb));
 	intel_de_write(display, FBC_LL_BASE,
-		       i915_gem_stolen_node_address(fbc->compressed_llb));
+		       intel_parent_stolen_node_address(display, fbc->compressed_llb));
 }
 
 static const struct intel_fbc_funcs i8xx_fbc_funcs = {
@@ -500,7 +509,7 @@ static void g4x_fbc_program_cfb(struct intel_fbc *fbc)
 	struct intel_display *display = fbc->display;
 
 	intel_de_write(display, DPFC_CB_BASE,
-		       i915_gem_stolen_node_offset(fbc->compressed_fb));
+		       intel_parent_stolen_node_offset(display, fbc->compressed_fb));
 }
 
 static const struct intel_fbc_funcs g4x_fbc_funcs = {
@@ -569,7 +578,7 @@ static void ilk_fbc_program_cfb(struct intel_fbc *fbc)
 	struct intel_display *display = fbc->display;
 
 	intel_de_write(display, ILK_DPFC_CB_BASE(fbc->id),
-		       i915_gem_stolen_node_offset(fbc->compressed_fb));
+		       intel_parent_stolen_node_offset(display, fbc->compressed_fb));
 }
 
 static const struct intel_fbc_funcs ilk_fbc_funcs = {
@@ -808,7 +817,7 @@ static u64 intel_fbc_stolen_end(struct intel_display *display)
 	 * underruns, even if that range is not reserved by the BIOS. */
 	if (display->platform.broadwell ||
 	    (DISPLAY_VER(display) == 9 && !display->platform.broxton))
-		end = i915_gem_stolen_area_size(display->drm) - 8 * 1024 * 1024;
+		end = intel_parent_stolen_area_size(display) - 8 * 1024 * 1024;
 	else
 		end = U64_MAX;
 
@@ -843,14 +852,14 @@ static int find_compression_limit(struct intel_fbc *fbc,
 	size /= limit;
 
 	/* Try to over-allocate to reduce reallocations and fragmentation. */
-	ret = i915_gem_stolen_insert_node_in_range(fbc->compressed_fb,
-						   size <<= 1, 4096, 0, end);
+	ret = intel_parent_stolen_insert_node_in_range(display, fbc->compressed_fb,
+						       size <<= 1, 4096, 0, end);
 	if (ret == 0)
 		return limit;
 
 	for (; limit <= intel_fbc_max_limit(display); limit <<= 1) {
-		ret = i915_gem_stolen_insert_node_in_range(fbc->compressed_fb,
-							   size >>= 1, 4096, 0, end);
+		ret = intel_parent_stolen_insert_node_in_range(display, fbc->compressed_fb,
+							       size >>= 1, 4096, 0, end);
 		if (ret == 0)
 			return limit;
 	}
@@ -865,12 +874,12 @@ static int intel_fbc_alloc_cfb(struct intel_fbc *fbc,
 	int ret;
 
 	drm_WARN_ON(display->drm,
-		    i915_gem_stolen_node_allocated(fbc->compressed_fb));
+		    intel_parent_stolen_node_allocated(display, fbc->compressed_fb));
 	drm_WARN_ON(display->drm,
-		    i915_gem_stolen_node_allocated(fbc->compressed_llb));
+		    intel_parent_stolen_node_allocated(display, fbc->compressed_llb));
 
 	if (DISPLAY_VER(display) < 5 && !display->platform.g4x) {
-		ret = i915_gem_stolen_insert_node(fbc->compressed_llb, 4096, 4096);
+		ret = intel_parent_stolen_insert_node(display, fbc->compressed_llb, 4096, 4096);
 		if (ret)
 			goto err;
 	}
@@ -886,14 +895,14 @@ static int intel_fbc_alloc_cfb(struct intel_fbc *fbc,
 
 	drm_dbg_kms(display->drm,
 		    "reserved %llu bytes of contiguous stolen space for FBC, limit: %d\n",
-		    i915_gem_stolen_node_size(fbc->compressed_fb), fbc->limit);
+		    intel_parent_stolen_node_size(display, fbc->compressed_fb), fbc->limit);
 	return 0;
 
 err_llb:
-	if (i915_gem_stolen_node_allocated(fbc->compressed_llb))
-		i915_gem_stolen_remove_node(fbc->compressed_llb);
+	if (intel_parent_stolen_node_allocated(display, fbc->compressed_llb))
+		intel_parent_stolen_remove_node(display, fbc->compressed_llb);
 err:
-	if (i915_gem_stolen_initialized(display->drm))
+	if (intel_parent_stolen_initialized(display))
 		drm_info_once(display->drm,
 			      "not enough stolen space for compressed buffer (need %d more bytes), disabling. Hint: you may be able to increase stolen memory size in the BIOS to avoid this.\n", size);
 	return -ENOSPC;
@@ -945,15 +954,83 @@ static void intel_fbc_program_workarounds(struct intel_fbc *fbc)
 		fbc_compressor_clkgate_disable_wa(fbc, true);
 }
 
+static void fbc_sys_cache_update_config(struct intel_display *display, u32 reg,
+					enum intel_fbc_id id)
+{
+	if (!HAS_FBC_SYS_CACHE(display))
+		return;
+
+	lockdep_assert_held(&display->fbc.sys_cache.lock);
+
+	/*
+	 * Wa_14025769978:
+	 * Fixes: SoC hardware issue in read caching
+	 * Workaround: disable cache read setting which is enabled by default.
+	 */
+	if (!intel_display_wa(display, 14025769978))
+		/* Cache read enable is set by default */
+		reg |= FBC_SYS_CACHE_READ_ENABLE;
+
+	intel_de_write(display, XE3P_LPD_FBC_SYS_CACHE_USAGE_CFG, reg);
+
+	display->fbc.sys_cache.id = id;
+}
+
+static void fbc_sys_cache_disable(const struct intel_fbc *fbc)
+{
+	struct intel_display *display = fbc->display;
+	struct sys_cache_cfg *sys_cache = &display->fbc.sys_cache;
+
+	mutex_lock(&sys_cache->lock);
+	/* clear only if "fbc" reserved the cache */
+	if (sys_cache->id == fbc->id)
+		fbc_sys_cache_update_config(display, 0, FBC_SYS_CACHE_ID_NONE);
+	mutex_unlock(&sys_cache->lock);
+}
+
+static int fbc_sys_cache_limit(struct intel_display *display)
+{
+	if (DISPLAY_VER(display) == 35)
+		return 2 * 1024 * 1024;
+
+	return 0;
+}
+
+static void fbc_sys_cache_enable(const struct intel_fbc *fbc)
+{
+	struct intel_display *display = fbc->display;
+	struct sys_cache_cfg *sys_cache = &display->fbc.sys_cache;
+	int range, offset;
+	u32 cfg;
+
+	if (!HAS_FBC_SYS_CACHE(display))
+		return;
+
+	range = fbc_sys_cache_limit(display) / (64 * 1024);
+
+	offset = intel_parent_stolen_node_offset(display, fbc->compressed_fb) / (4 * 1024);
+
+	cfg = FBC_SYS_CACHE_TAG_USE_RES_SPACE | FBC_SYS_CACHEABLE_RANGE(range) |
+	      FBC_SYS_CACHE_START_BASE(offset);
+
+	mutex_lock(&sys_cache->lock);
+	/* update sys cache config only if sys cache is unassigned */
+	if (sys_cache->id == FBC_SYS_CACHE_ID_NONE)
+		fbc_sys_cache_update_config(display, cfg, fbc->id);
+	mutex_unlock(&sys_cache->lock);
+}
+
 static void __intel_fbc_cleanup_cfb(struct intel_fbc *fbc)
 {
+	struct intel_display *display = fbc->display;
+
 	if (WARN_ON(intel_fbc_hw_is_active(fbc)))
 		return;
 
-	if (i915_gem_stolen_node_allocated(fbc->compressed_llb))
-		i915_gem_stolen_remove_node(fbc->compressed_llb);
-	if (i915_gem_stolen_node_allocated(fbc->compressed_fb))
-		i915_gem_stolen_remove_node(fbc->compressed_fb);
+	if (intel_parent_stolen_node_allocated(display, fbc->compressed_llb))
+		intel_parent_stolen_remove_node(display, fbc->compressed_llb);
+	if (intel_parent_stolen_node_allocated(display, fbc->compressed_fb))
+		intel_parent_stolen_remove_node(display, fbc->compressed_fb);
 }
 
 void intel_fbc_cleanup(struct intel_display *display)
@@ -966,11 +1043,16 @@ void intel_fbc_cleanup(struct intel_display *display)
 		__intel_fbc_cleanup_cfb(fbc);
 		mutex_unlock(&fbc->lock);
 
-		i915_gem_stolen_node_free(fbc->compressed_fb);
-		i915_gem_stolen_node_free(fbc->compressed_llb);
+		intel_parent_stolen_node_free(display, fbc->compressed_fb);
+		intel_parent_stolen_node_free(display, fbc->compressed_llb);
 
 		kfree(fbc);
 	}
+
+	mutex_lock(&display->fbc.sys_cache.lock);
+	drm_WARN_ON(display->drm,
+		    display->fbc.sys_cache.id != FBC_SYS_CACHE_ID_NONE);
+	mutex_unlock(&display->fbc.sys_cache.lock);
 }
 
 static bool i8xx_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
@@ -1016,7 +1098,7 @@ static bool icl_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
 
 static bool stride_is_valid(const struct intel_plane_state *plane_state)
 {
-	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
+	struct intel_display *display = to_intel_display(plane_state);
 
 	if (DISPLAY_VER(display) >= 11)
 		return icl_fbc_stride_is_valid(plane_state);
@@ -1032,7 +1114,7 @@ static bool stride_is_valid(const struct intel_plane_state *plane_state)
 
 static bool i8xx_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
 {
-	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
+	struct intel_display *display = to_intel_display(plane_state);
 	const struct drm_framebuffer *fb = plane_state->hw.fb;
 
 	switch (fb->format->format) {
@@ -1052,7 +1134,7 @@ static bool i8xx_fbc_pixel_format_is_valid(const struct intel_plane_state *plane
 
 static bool g4x_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
 {
-	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
+	struct intel_display *display = to_intel_display(plane_state);
 	const struct drm_framebuffer *fb = plane_state->hw.fb;
 
 	switch (fb->format->format) {
@@ -1131,7 +1213,7 @@ intel_fbc_is_enable_pixel_normalizer(const struct intel_plane_state *plane_state
 
 static bool pixel_format_is_valid(const struct intel_plane_state *plane_state)
 {
-	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
+	struct intel_display *display = to_intel_display(plane_state);
 
 	if (DISPLAY_VER(display) >= 35)
 		return xe3p_lpd_fbc_pixel_format_is_valid(plane_state);
@@ -1167,7 +1249,7 @@ static bool skl_fbc_rotation_is_valid(const struct intel_plane_state *plane_stat
 
 static bool rotation_is_valid(const struct intel_plane_state *plane_state)
 {
-	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
+	struct intel_display *display = to_intel_display(plane_state);
 
 	if (DISPLAY_VER(display) >= 9)
 		return skl_fbc_rotation_is_valid(plane_state);
@@ -1206,7 +1288,7 @@ static void intel_fbc_max_surface_size(struct intel_display *display,
  */
 static bool intel_fbc_surface_size_ok(const struct intel_plane_state *plane_state)
 {
-	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
+	struct intel_display *display = to_intel_display(plane_state);
 	unsigned int effective_w, effective_h, max_w, max_h;
 
 	intel_fbc_max_surface_size(display, &max_w, &max_h);
@@ -1239,7 +1321,7 @@ static void intel_fbc_max_plane_size(struct intel_display *display,
 
 static bool intel_fbc_plane_size_valid(const struct intel_plane_state *plane_state)
 {
-	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
+	struct intel_display *display = to_intel_display(plane_state);
 	unsigned int w, h, max_w, max_h;
 
 	intel_fbc_max_plane_size(display, &max_w, &max_h);
@@ -1264,7 +1346,7 @@ static bool skl_fbc_tiling_valid(const struct intel_plane_state *plane_state)
 
 static bool tiling_is_valid(const struct intel_plane_state *plane_state)
 {
-	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
+	struct intel_display *display = to_intel_display(plane_state);
 
 	if (DISPLAY_VER(display) >= 9)
 		return skl_fbc_tiling_valid(plane_state);
@@ -1344,7 +1426,7 @@ static void intel_fbc_update_state(struct intel_atomic_state *state,
 				   struct intel_crtc *crtc,
 				   struct intel_plane *plane)
 {
-	struct intel_display *display = to_intel_display(state->base.dev);
+	struct intel_display *display = to_intel_display(state);
 	const struct intel_crtc_state *crtc_state =
 		intel_atomic_get_new_crtc_state(state, crtc);
 	const struct intel_plane_state *plane_state =
@@ -1377,7 +1459,7 @@ static void intel_fbc_update_state(struct intel_atomic_state *state,
 
 static bool intel_fbc_is_fence_ok(const struct intel_plane_state *plane_state)
 {
-	struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
+	struct intel_display *display = to_intel_display(plane_state);
 
 	/*
 	 * The use of a CPU fence is one of two ways to detect writes by the
@@ -1398,12 +1480,13 @@ static bool intel_fbc_is_fence_ok(const struct intel_plane_state *plane_state)
 
 static bool intel_fbc_is_cfb_ok(const struct intel_plane_state *plane_state)
 {
+	struct intel_display *display = to_intel_display(plane_state);
 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 	struct intel_fbc *fbc = plane->fbc;
 
 	return intel_fbc_min_limit(plane_state) <= fbc->limit &&
 		intel_fbc_cfb_size(plane_state) <= fbc->limit *
-			i915_gem_stolen_node_size(fbc->compressed_fb);
+			intel_parent_stolen_node_size(display, fbc->compressed_fb);
 }
 
 static bool intel_fbc_is_ok(const struct intel_plane_state *plane_state)
@@ -1484,8 +1567,7 @@ static int _intel_fbc_min_cdclk(const struct intel_crtc_state *crtc_state)
 static int intel_fbc_check_plane(struct intel_atomic_state *state,
 				 struct intel_plane *plane)
 {
-	struct intel_display *display = to_intel_display(state->base.dev);
-	struct drm_i915_private *i915 = to_i915(display->drm);
+	struct intel_display *display = to_intel_display(state);
 	struct intel_plane_state *plane_state =
 		intel_atomic_get_new_plane_state(state, plane);
 	const struct drm_framebuffer *fb = plane_state->hw.fb;
@@ -1496,12 +1578,12 @@ static int intel_fbc_check_plane(struct intel_atomic_state *state,
 	if (!fbc)
 		return 0;
 
-	if (!i915_gem_stolen_initialized(display->drm)) {
+	if (!intel_parent_stolen_initialized(display)) {
 		plane_state->no_fbc_reason = "stolen memory not initialised";
 		return 0;
 	}
 
-	if (intel_vgpu_active(i915)) {
+	if (intel_parent_vgpu_active(display)) {
 		plane_state->no_fbc_reason = "VGPU active";
 		return 0;
 	}
@@ -1521,6 +1603,16 @@ static int intel_fbc_check_plane(struct intel_atomic_state *state,
 		return 0;
 	}
 
+	/*
+	 * Wa_15018326506:
+	 * Fixes: Underrun during media decode
+	 * Workaround: Do not enable FBC
+	 */
+	if (intel_display_wa(display, 15018326506)) {
+		plane_state->no_fbc_reason = "Wa_15018326506";
+		return 0;
+	}
+
 	/* WaFbcTurnOffFbcWhenHyperVisorIsUsed:skl,bxt */
 	if (intel_display_vtd_active(display) &&
 	    (display->platform.skylake || display->platform.broxton)) {
@@ -1702,7 +1794,7 @@ static bool __intel_fbc_pre_update(struct intel_atomic_state *state,
 				   struct intel_crtc *crtc,
 				   struct intel_plane *plane)
 {
-	struct intel_display *display = to_intel_display(state->base.dev);
+	struct intel_display *display = to_intel_display(state);
 	struct intel_fbc *fbc = plane->fbc;
 	bool need_vblank_wait = false;
 
@@ -1775,6 +1867,8 @@ static void __intel_fbc_disable(struct intel_fbc *fbc)
 
 	__intel_fbc_cleanup_cfb(fbc);
 
+	fbc_sys_cache_disable(fbc);
+
 	/* wa_18038517565 Enable DPFC clock gating after FBC disable */
 	if (display->platform.dg2 || DISPLAY_VER(display) >= 14)
 		fbc_compressor_clkgate_disable_wa(fbc, false);
@@ -1915,7 +2009,7 @@ static void __intel_fbc_enable(struct intel_atomic_state *state,
 			       struct intel_crtc *crtc,
 			       struct intel_plane *plane)
 {
-	struct intel_display *display = to_intel_display(state->base.dev);
+	struct intel_display *display = to_intel_display(state);
 	const struct intel_plane_state *plane_state =
 		intel_atomic_get_new_plane_state(state, plane);
 	struct intel_fbc *fbc = plane->fbc;
@@ -1967,6 +2061,8 @@ static void __intel_fbc_enable(struct intel_atomic_state *state,
 
 	intel_fbc_program_workarounds(fbc);
 	intel_fbc_program_cfb(fbc);
+
+	fbc_sys_cache_enable(fbc);
 }
 
 /**
@@ -1977,7 +2073,7 @@ static void __intel_fbc_enable(struct intel_atomic_state *state,
  */
 void intel_fbc_disable(struct intel_crtc *crtc)
 {
-	struct intel_display *display = to_intel_display(crtc->base.dev);
+	struct intel_display *display = to_intel_display(crtc);
 	struct intel_plane *plane;
 
 	for_each_intel_plane(display->drm, plane) {
@@ -2119,6 +2215,37 @@ void intel_fbc_handle_fifo_underrun_irq(struct intel_display *display)
 		__intel_fbc_handle_fifo_underrun_irq(fbc);
 }
 
+/**
+ * intel_fbc_read_underrun_dbg_info - Read and log FBC-related FIFO underrun debug info
+ * @display: display device instance
+ * @pipe: the pipe possibly containing the FBC
+ * @log: log the info?
+ *
+ * If @pipe does not contain an FBC instance, this function bails early.
+ * Otherwise, FBC-related FIFO underrun is read and cleared, and then, if @log
+ * is true, printed with error level.
+ */
+void intel_fbc_read_underrun_dbg_info(struct intel_display *display,
+				      enum pipe pipe, bool log)
+{
+	struct intel_fbc *fbc = intel_fbc_for_pipe(display, pipe);
+	u32 val;
+
+	if (!fbc)
+		return;
+
+	val = intel_de_read(display, FBC_DEBUG_STATUS(fbc->id));
+	if (!(val & FBC_UNDERRUN_DECMPR))
+		return;
+
+	intel_de_write(display, FBC_DEBUG_STATUS(fbc->id), FBC_UNDERRUN_DECMPR);
+
+	if (log)
+		drm_err(display->drm,
+			"Pipe %c FIFO underrun info: FBC decompressing\n",
+			pipe_name(pipe));
+}
+
 /*
  * The DDX driver changes its behavior depending on the value it reads from
  * i915.enable_fbc, so sanitize it by translating the default value into either
@@ -2156,10 +2283,10 @@ static struct intel_fbc *intel_fbc_create(struct intel_display *display,
 	if (!fbc)
 		return NULL;
 
-	fbc->compressed_fb = i915_gem_stolen_node_alloc(display->drm);
+	fbc->compressed_fb = intel_parent_stolen_node_alloc(display);
 	if (!fbc->compressed_fb)
 		goto err;
-	fbc->compressed_llb = i915_gem_stolen_node_alloc(display->drm);
+	fbc->compressed_llb = intel_parent_stolen_node_alloc(display);
 	if (!fbc->compressed_llb)
 		goto err;
 
@@ -2184,8 +2311,8 @@ static struct intel_fbc *intel_fbc_create(struct intel_display *display,
 	return fbc;
 
 err:
-	i915_gem_stolen_node_free(fbc->compressed_llb);
-	i915_gem_stolen_node_free(fbc->compressed_fb);
+	intel_parent_stolen_node_free(display, fbc->compressed_llb);
+	intel_parent_stolen_node_free(display, fbc->compressed_fb);
 	kfree(fbc);
 
 	return NULL;
@@ -2206,7 +2333,10 @@ void intel_fbc_init(struct intel_display *display)
 		    display->params.enable_fbc);
 
 	for_each_fbc_id(display, fbc_id)
-		display->fbc[fbc_id] = intel_fbc_create(display, fbc_id);
+		display->fbc.instances[fbc_id] = intel_fbc_create(display, fbc_id);
+
+	mutex_init(&display->fbc.sys_cache.lock);
+	display->fbc.sys_cache.id = FBC_SYS_CACHE_ID_NONE;
 }
 
 /**
@@ -2226,6 +2356,11 @@ void intel_fbc_sanitize(struct intel_display *display)
 		if (intel_fbc_hw_is_active(fbc))
 			intel_fbc_hw_deactivate(fbc);
 	}
+
+	/* Ensure the sys cache usage config is clear as well */
+	mutex_lock(&display->fbc.sys_cache.lock);
+	fbc_sys_cache_update_config(display, 0, FBC_SYS_CACHE_ID_NONE);
+	mutex_unlock(&display->fbc.sys_cache.lock);
 }
 
 static int intel_fbc_debugfs_status_show(struct seq_file *m, void *unused)
@@ -2244,6 +2379,11 @@ static int intel_fbc_debugfs_status_show(struct seq_file *m, void *unused)
 		seq_puts(m, "FBC enabled\n");
 		seq_printf(m, "Compressing: %s\n",
 			   str_yes_no(intel_fbc_is_compressing(fbc)));
+
+		mutex_lock(&display->fbc.sys_cache.lock);
+		seq_printf(m, "Using system cache: %s\n",
+			   str_yes_no(display->fbc.sys_cache.id == fbc->id));
+		mutex_unlock(&display->fbc.sys_cache.lock);
 	} else {
 		seq_printf(m, "FBC disabled: %s\n", fbc->no_fbc_reason);
 	}
@@ -2325,7 +2465,7 @@ void intel_fbc_debugfs_register(struct intel_display *display)
 {
 	struct intel_fbc *fbc;
 
-	fbc = display->fbc[INTEL_FBC_A];
+	fbc = display->fbc.instances[INTEL_FBC_A];
 	if (fbc)
 		intel_fbc_debugfs_add(fbc, display->drm->debugfs_root);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_fbc.h b/drivers/gpu/drm/i915/display/intel_fbc.h
index 91424563206a..f0255ddae2b6 100644
--- a/drivers/gpu/drm/i915/display/intel_fbc.h
+++ b/drivers/gpu/drm/i915/display/intel_fbc.h
@@ -9,6 +9,7 @@
 #include <linux/types.h>
 
 enum fb_op_origin;
+enum pipe;
 struct intel_atomic_state;
 struct intel_crtc;
 struct intel_crtc_state;
@@ -46,6 +47,8 @@ void intel_fbc_flush(struct intel_display *display,
 		     unsigned int frontbuffer_bits, enum fb_op_origin origin);
 void intel_fbc_add_plane(struct intel_fbc *fbc, struct intel_plane *plane);
 void intel_fbc_handle_fifo_underrun_irq(struct intel_display *display);
+void intel_fbc_read_underrun_dbg_info(struct intel_display *display,
+				      enum pipe, bool log);
 void intel_fbc_reset_underrun(struct intel_display *display);
 void intel_fbc_crtc_debugfs_add(struct intel_crtc *crtc);
 void intel_fbc_debugfs_register(struct intel_display *display);
diff --git a/drivers/gpu/drm/i915/display/intel_fbc_regs.h b/drivers/gpu/drm/i915/display/intel_fbc_regs.h
index b1d0161a3196..4178e602d7d7 100644
--- a/drivers/gpu/drm/i915/display/intel_fbc_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_fbc_regs.h
@@ -88,6 +88,8 @@
 #define DPFC_FENCE_YOFF			_MMIO(0x3218)
 #define ILK_DPFC_FENCE_YOFF(fbc_id)	_MMIO_PIPE((fbc_id), 0x43218, 0x43258)
 #define DPFC_CHICKEN			_MMIO(0x3224)
+#define FBC_DEBUG_STATUS(fbc_id)	_MMIO_PIPE((fbc_id), 0x43220, 0x43260)
+#define   FBC_UNDERRUN_DECMPR			REG_BIT(27)
 #define ILK_DPFC_CHICKEN(fbc_id)	_MMIO_PIPE((fbc_id), 0x43224, 0x43264)
 #define   DPFC_HT_MODIFY			REG_BIT(31) /* pre-ivb */
 #define   DPFC_NUKE_ON_ANY_MODIFICATION		REG_BIT(23) /* bdw+ */
@@ -126,4 +128,14 @@
 #define   FBC_REND_NUKE			REG_BIT(2)
 #define   FBC_REND_CACHE_CLEAN		REG_BIT(1)
 
+#define XE3P_LPD_FBC_SYS_CACHE_USAGE_CFG	_MMIO(0x1344E0)
+#define   FBC_SYS_CACHE_START_BASE_MASK		REG_GENMASK(31, 16)
+#define   FBC_SYS_CACHE_START_BASE(base)	REG_FIELD_PREP(FBC_SYS_CACHE_START_BASE_MASK, (base))
+#define   FBC_SYS_CACHEABLE_RANGE_MASK		REG_GENMASK(15, 4)
+#define   FBC_SYS_CACHEABLE_RANGE(range)	REG_FIELD_PREP(FBC_SYS_CACHEABLE_RANGE_MASK, (range))
+#define   FBC_SYS_CACHE_TAG_MASK		REG_GENMASK(3, 2)
+#define   FBC_SYS_CACHE_TAG_DONT_CACHE		REG_FIELD_PREP(FBC_SYS_CACHE_TAG_MASK, 0)
+#define   FBC_SYS_CACHE_TAG_USE_RES_SPACE	REG_FIELD_PREP(FBC_SYS_CACHE_TAG_MASK, 3)
+#define   FBC_SYS_CACHE_READ_ENABLE		REG_BIT(0)
+
 #endif /* __INTEL_FBC_REGS__ */
diff --git a/drivers/gpu/drm/i915/display/intel_fifo_underrun.c b/drivers/gpu/drm/i915/display/intel_fifo_underrun.c
index c2ce8461ac9e..b413b3e871d8 100644
--- a/drivers/gpu/drm/i915/display/intel_fifo_underrun.c
+++ b/drivers/gpu/drm/i915/display/intel_fifo_underrun.c
@@ -25,6 +25,8 @@
  *
  */
 
+#include <linux/seq_buf.h>
+
 #include <drm/drm_print.h>
 
 #include "i915_reg.h"
@@ -57,6 +59,100 @@
  * The code also supports underrun detection on the PCH transcoder.
  */
 
+#define UNDERRUN_DBG1_NUM_PLANES 6
+
+static void log_underrun_dbg1(struct intel_display *display, enum pipe pipe,
+			      unsigned long plane_mask, const char *info)
+{
+	DECLARE_SEQ_BUF(planes_desc, 32);
+	unsigned int i;
+
+	if (!plane_mask)
+		return;
+
+	for_each_set_bit(i, &plane_mask, UNDERRUN_DBG1_NUM_PLANES) {
+		if (i == 0)
+			seq_buf_puts(&planes_desc, "[C]");
+		else
+			seq_buf_printf(&planes_desc, "[%d]", i);
+	}
+
+	drm_err(display->drm, "Pipe %c FIFO underrun info: %s on planes: %s\n",
+		pipe_name(pipe), info, seq_buf_str(&planes_desc));
+
+	drm_WARN_ON(display->drm, seq_buf_has_overflowed(&planes_desc));
+}
+
+static void read_underrun_dbg1(struct intel_display *display, enum pipe pipe, bool log)
+{
+	u32 val = intel_de_read(display, UNDERRUN_DBG1(pipe));
+
+	if (!val)
+		return;
+
+	intel_de_write(display, UNDERRUN_DBG1(pipe), val);
+
+	if (!log)
+		return;
+
+	log_underrun_dbg1(display, pipe, REG_FIELD_GET(UNDERRUN_DBUF_BLOCK_NOT_VALID_MASK, val),
+			  "DBUF block not valid");
+	log_underrun_dbg1(display, pipe, REG_FIELD_GET(UNDERRUN_DDB_EMPTY_MASK, val),
+			  "DDB empty");
+	log_underrun_dbg1(display, pipe, REG_FIELD_GET(UNDERRUN_DBUF_NOT_FILLED_MASK, val),
+			  "DBUF not completely filled");
+	log_underrun_dbg1(display, pipe, REG_FIELD_GET(UNDERRUN_BELOW_WM0_MASK, val),
+			  "DBUF below WM0");
+}
+
+static void read_underrun_dbg2(struct intel_display *display, enum pipe pipe, bool log)
+{
+	u32 val = intel_de_read(display, UNDERRUN_DBG2(pipe));
+
+	if (!(val & UNDERRUN_FRAME_LINE_COUNTERS_FROZEN))
+		return;
+
+	intel_de_write(display, UNDERRUN_DBG2(pipe), UNDERRUN_FRAME_LINE_COUNTERS_FROZEN);
+
+	if (log)
+		drm_err(display->drm,
+			"Pipe %c FIFO underrun info: frame count: %u, line count: %u\n",
+			pipe_name(pipe),
+			REG_FIELD_GET(UNDERRUN_PIPE_FRAME_COUNT_MASK, val),
+			REG_FIELD_GET(UNDERRUN_LINE_COUNT_MASK, val));
+}
+
+static void read_underrun_dbg_pkgc(struct intel_display *display, bool log)
+{
+	u32 val = intel_de_read(display, GEN12_DCPR_STATUS_1);
+
+	if (!(val & XE3P_UNDERRUN_PKGC))
+		return;
+
+	/*
+	 * Note: If there are multiple pipes enabled, only one of them will see
+	 * XE3P_UNDERRUN_PKGC set.
+	 */
+	intel_de_write(display, GEN12_DCPR_STATUS_1, XE3P_UNDERRUN_PKGC);
+
+	if (log)
+		drm_err(display->drm,
+			"General FIFO underrun info: Package C-state blocking memory\n");
+}
+
+static void read_underrun_dbg_info(struct intel_display *display,
+				   enum pipe pipe,
+				   bool log)
+{
+	if (!HAS_UNDERRUN_DBG_INFO(display))
+		return;
+
+	read_underrun_dbg1(display, pipe, log);
+	read_underrun_dbg2(display, pipe, log);
+	intel_fbc_read_underrun_dbg_info(display, pipe, log);
+	read_underrun_dbg_pkgc(display, log);
+}
+
 static bool ivb_can_enable_err_int(struct intel_display *display)
 {
 	struct intel_crtc *crtc;
@@ -262,6 +358,17 @@ static bool __intel_set_cpu_fifo_underrun_reporting(struct intel_display *displa
 	old = !crtc->cpu_fifo_underrun_disabled;
 	crtc->cpu_fifo_underrun_disabled = !enable;
 
+	/*
+	 * The debug bits get latched at the time of the FIFO underrun ISR bit
+	 * getting set.  That means that any non-zero debug bit that is read when
+	 * handling a FIFO underrun interrupt has the potential to belong to
+	 * another underrun event (past or future).  To alleviate this problem,
+	 * let's clear existing bits before enabling the interrupt, so that at
+	 * least we don't get information that is too out-of-date.
+	 */
+	if (enable && !old)
+		read_underrun_dbg_info(display, pipe, false);
+
 	if (HAS_GMCH(display))
 		i9xx_set_fifo_underrun_reporting(display, pipe, enable, old);
 	else if (display->platform.ironlake || display->platform.sandybridge)
@@ -379,6 +486,8 @@ void intel_cpu_fifo_underrun_irq_handler(struct intel_display *display,
 		trace_intel_cpu_fifo_underrun(display, pipe);
 
 		drm_err(display->drm, "CPU pipe %c FIFO underrun\n", pipe_name(pipe));
+
+		read_underrun_dbg_info(display, pipe, true);
 	}
 
 	intel_fbc_handle_fifo_underrun_irq(display);
diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.c b/drivers/gpu/drm/i915/display/intel_gmbus.c
index 795012d7c24c..2caff677600c 100644
--- a/drivers/gpu/drm/i915/display/intel_gmbus.c
+++ b/drivers/gpu/drm/i915/display/intel_gmbus.c
@@ -35,8 +35,6 @@
 #include <drm/drm_print.h>
 #include <drm/display/drm_hdcp_helper.h>
 
-#include "i915_drv.h"
-#include "i915_irq.h"
 #include "i915_reg.h"
 #include "intel_de.h"
 #include "intel_display_regs.h"
@@ -44,6 +42,7 @@
 #include "intel_display_wa.h"
 #include "intel_gmbus.h"
 #include "intel_gmbus_regs.h"
+#include "intel_parent.h"
 
 struct intel_gmbus {
 	struct i2c_adapter adapter;
@@ -391,12 +390,11 @@ intel_gpio_setup(struct intel_gmbus *bus, i915_reg_t gpio_reg)
 
 static bool has_gmbus_irq(struct intel_display *display)
 {
-	struct drm_i915_private *i915 = to_i915(display->drm);
 	/*
 	 * encoder->shutdown() may want to use GMBUS
 	 * after irqs have already been disabled.
 	 */
-	return HAS_GMBUS_IRQ(display) && intel_irqs_enabled(i915);
+	return HAS_GMBUS_IRQ(display) && intel_parent_irq_enabled(display);
 }
 
 static int gmbus_wait(struct intel_display *display, u32 status, u32 irq_en)
@@ -791,7 +789,7 @@ gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num)
 {
 	struct intel_gmbus *bus = to_intel_gmbus(adapter);
 	struct intel_display *display = bus->display;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	int ret;
 
 	wakeref = intel_display_power_get(display, POWER_DOMAIN_GMBUS);
@@ -831,7 +829,7 @@ int intel_gmbus_output_aksv(struct i2c_adapter *adapter)
 			.buf = buf,
 		}
 	};
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	int ret;
 
 	wakeref = intel_display_power_get(display, POWER_DOMAIN_GMBUS);
diff --git a/drivers/gpu/drm/i915/display/intel_hdcp.c b/drivers/gpu/drm/i915/display/intel_hdcp.c
index 5e1a96223a9c..7114fc405c29 100644
--- a/drivers/gpu/drm/i915/display/intel_hdcp.c
+++ b/drivers/gpu/drm/i915/display/intel_hdcp.c
@@ -29,10 +29,10 @@
 #include "intel_display_types.h"
 #include "intel_dp_mst.h"
 #include "intel_hdcp.h"
-#include "intel_hdcp_gsc.h"
 #include "intel_hdcp_gsc_message.h"
 #include "intel_hdcp_regs.h"
 #include "intel_hdcp_shim.h"
+#include "intel_parent.h"
 #include "intel_pcode.h"
 #include "intel_step.h"
 
@@ -258,7 +258,7 @@ static bool intel_hdcp2_prerequisite(struct intel_connector *connector)
 
 	/* If MTL+ make sure gsc is loaded and proxy is setup */
 	if (USE_HDCP_GSC(display)) {
-		if (!intel_hdcp_gsc_check_status(display->drm))
+		if (!intel_parent_hdcp_gsc_check_status(display))
 			return false;
 	}
 
diff --git a/drivers/gpu/drm/i915/display/intel_hdcp_gsc.c b/drivers/gpu/drm/i915/display/intel_hdcp_gsc.c
deleted file mode 100644
index 3e7b480ee9f1..000000000000
--- a/drivers/gpu/drm/i915/display/intel_hdcp_gsc.c
+++ /dev/null
@@ -1,239 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright 2023, Intel Corporation.
- */
-
-#include <drm/drm_print.h>
-#include <drm/intel/i915_hdcp_interface.h>
-
-#include "gem/i915_gem_region.h"
-#include "gt/intel_gt.h"
-#include "gt/uc/intel_gsc_uc_heci_cmd_submit.h"
-#include "i915_drv.h"
-#include "intel_hdcp_gsc.h"
-
-struct intel_hdcp_gsc_context {
-	struct drm_i915_private *i915;
-	struct i915_vma *vma;
-	void *hdcp_cmd_in;
-	void *hdcp_cmd_out;
-};
-
-bool intel_hdcp_gsc_check_status(struct drm_device *drm)
-{
-	struct drm_i915_private *i915 = to_i915(drm);
-	struct intel_gt *gt = i915->media_gt;
-	struct intel_gsc_uc *gsc = gt ? &gt->uc.gsc : NULL;
-
-	if (!gsc || !intel_uc_fw_is_running(&gsc->fw)) {
-		drm_dbg_kms(&i915->drm,
-			    "GSC components required for HDCP2.2 are not ready\n");
-		return false;
-	}
-
-	return true;
-}
-
-/*This function helps allocate memory for the command that we will send to gsc cs */
-static int intel_hdcp_gsc_initialize_message(struct drm_i915_private *i915,
-					     struct intel_hdcp_gsc_context *gsc_context)
-{
-	struct intel_gt *gt = i915->media_gt;
-	struct drm_i915_gem_object *obj = NULL;
-	struct i915_vma *vma = NULL;
-	void *cmd_in, *cmd_out;
-	int err;
-
-	/* allocate object of two page for HDCP command memory and store it */
-	obj = i915_gem_object_create_shmem(i915, 2 * PAGE_SIZE);
-
-	if (IS_ERR(obj)) {
-		drm_err(&i915->drm, "Failed to allocate HDCP streaming command!\n");
-		return PTR_ERR(obj);
-	}
-
-	cmd_in = i915_gem_object_pin_map_unlocked(obj, intel_gt_coherent_map_type(gt, obj, true));
-	if (IS_ERR(cmd_in)) {
-		drm_err(&i915->drm, "Failed to map gsc message page!\n");
-		err = PTR_ERR(cmd_in);
-		goto out_unpin;
-	}
-
-	cmd_out = cmd_in + PAGE_SIZE;
-
-	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
-	if (IS_ERR(vma)) {
-		err = PTR_ERR(vma);
-		goto out_unmap;
-	}
-
-	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
-	if (err)
-		goto out_unmap;
-
-	memset(cmd_in, 0, obj->base.size);
-
-	gsc_context->hdcp_cmd_in = cmd_in;
-	gsc_context->hdcp_cmd_out = cmd_out;
-	gsc_context->vma = vma;
-	gsc_context->i915 = i915;
-
-	return 0;
-
-out_unmap:
-	i915_gem_object_unpin_map(obj);
-out_unpin:
-	i915_gem_object_put(obj);
-	return err;
-}
-
-struct intel_hdcp_gsc_context *intel_hdcp_gsc_context_alloc(struct drm_device *drm)
-{
-	struct drm_i915_private *i915 = to_i915(drm);
-	struct intel_hdcp_gsc_context *gsc_context;
-	int ret;
-
-	gsc_context = kzalloc(sizeof(*gsc_context), GFP_KERNEL);
-	if (!gsc_context)
-		return ERR_PTR(-ENOMEM);
-
-	/*
-	 * NOTE: No need to lock the comp mutex here as it is already
-	 * going to be taken before this function called
-	 */
-	ret = intel_hdcp_gsc_initialize_message(i915, gsc_context);
-	if (ret) {
-		drm_err(&i915->drm, "Could not initialize gsc_context\n");
-		kfree(gsc_context);
-		gsc_context = ERR_PTR(ret);
-	}
-
-	return gsc_context;
-}
-
-void intel_hdcp_gsc_context_free(struct intel_hdcp_gsc_context *gsc_context)
-{
-	if (!gsc_context)
-		return;
-
-	i915_vma_unpin_and_release(&gsc_context->vma, I915_VMA_RELEASE_MAP);
-	kfree(gsc_context);
-}
-
-static int intel_gsc_send_sync(struct drm_i915_private *i915,
-			       struct intel_gsc_mtl_header *header_in,
-			       struct intel_gsc_mtl_header *header_out,
-			       u64 addr_in, u64 addr_out,
-			       size_t msg_out_len)
-{
-	struct intel_gt *gt = i915->media_gt;
-	int ret;
-
-	ret = intel_gsc_uc_heci_cmd_submit_packet(&gt->uc.gsc, addr_in,
-						  header_in->message_size,
-						  addr_out,
-						  msg_out_len + sizeof(*header_out));
-	if (ret) {
-		drm_err(&i915->drm, "failed to send gsc HDCP msg (%d)\n", ret);
-		return ret;
-	}
-
-	/*
-	 * Checking validity marker and header status to see if some error has
-	 * blocked us from sending message to gsc cs
-	 */
-	if (header_out->validity_marker != GSC_HECI_VALIDITY_MARKER) {
-		drm_err(&i915->drm, "invalid validity marker\n");
-		return -EINVAL;
-	}
-
-	if (header_out->status != 0) {
-		drm_err(&i915->drm, "header status indicates error %d\n",
-			header_out->status);
-		return -EINVAL;
-	}
-
-	if (header_out->flags & GSC_OUTFLAG_MSG_PENDING) {
-		header_in->gsc_message_handle = header_out->gsc_message_handle;
-		return -EAGAIN;
-	}
-
-	return 0;
-}
-
-/*
- * This function can now be used for sending requests and will also handle
- * receipt of reply messages hence no different function of message retrieval
- * is required. We will initialize intel_hdcp_gsc_context structure then add
- * gsc cs memory header as stated in specs after which the normal HDCP payload
- * will follow
- */
-ssize_t intel_hdcp_gsc_msg_send(struct intel_hdcp_gsc_context *gsc_context,
-				void *msg_in, size_t msg_in_len,
-				void *msg_out, size_t msg_out_len)
-{
-	struct drm_i915_private *i915 = gsc_context->i915;
-	struct intel_gt *gt = i915->media_gt;
-	struct intel_gsc_mtl_header *header_in, *header_out;
-	const size_t max_msg_size = PAGE_SIZE - sizeof(*header_in);
-	u64 addr_in, addr_out, host_session_id;
-	u32 reply_size, msg_size_in, msg_size_out;
-	int ret, tries = 0;
-
-	if (!intel_uc_uses_gsc_uc(&gt->uc))
-		return -ENODEV;
-
-	if (msg_in_len > max_msg_size || msg_out_len > max_msg_size)
-		return -ENOSPC;
-
-	msg_size_in = msg_in_len + sizeof(*header_in);
-	msg_size_out = msg_out_len + sizeof(*header_out);
-	header_in = gsc_context->hdcp_cmd_in;
-	header_out = gsc_context->hdcp_cmd_out;
-	addr_in = i915_ggtt_offset(gsc_context->vma);
-	addr_out = addr_in + PAGE_SIZE;
-
-	memset(header_in, 0, msg_size_in);
-	memset(header_out, 0, msg_size_out);
-	get_random_bytes(&host_session_id, sizeof(u64));
-	intel_gsc_uc_heci_cmd_emit_mtl_header(header_in, HECI_MEADDRESS_HDCP,
-					      msg_size_in, host_session_id);
-	memcpy(gsc_context->hdcp_cmd_in + sizeof(*header_in), msg_in, msg_in_len);
-
-	/*
-	 * Keep sending request in case the pending bit is set no need to add
-	 * message handle as we are using same address hence loc. of header is
-	 * same and it will contain the message handle. we will send the message
-	 * 20 times each message 50 ms apart
-	 */
-	do {
-		ret = intel_gsc_send_sync(i915, header_in, header_out, addr_in,
-					  addr_out, msg_out_len);
-
-		/* Only try again if gsc says so */
-		if (ret != -EAGAIN)
-			break;
-
-		msleep(50);
-
-	} while (++tries < 20);
-
-	if (ret)
-		goto err;
-
-	/* we use the same mem for the reply, so header is in the same loc */
-	reply_size = header_out->message_size - sizeof(*header_out);
-	if (reply_size > msg_out_len) {
-		drm_warn(&i915->drm, "caller with insufficient HDCP reply size %u (%d)\n",
-			 reply_size, (u32)msg_out_len);
-		reply_size = msg_out_len;
-	} else if (reply_size != msg_out_len) {
-		drm_dbg_kms(&i915->drm, "caller unexpected HCDP reply size %u (%d)\n",
-			    reply_size, (u32)msg_out_len);
-	}
-
-	memcpy(msg_out, gsc_context->hdcp_cmd_out + sizeof(*header_out), msg_out_len);
-
-err:
-	return ret;
-}
diff --git a/drivers/gpu/drm/i915/display/intel_hdcp_gsc.h b/drivers/gpu/drm/i915/display/intel_hdcp_gsc.h
deleted file mode 100644
index 9305c14aaffe..000000000000
--- a/drivers/gpu/drm/i915/display/intel_hdcp_gsc.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#ifndef __INTEL_HDCP_GSC_H__
-#define __INTEL_HDCP_GSC_H__
-
-#include <linux/types.h>
-
-struct drm_device;
-struct intel_hdcp_gsc_context;
-
-ssize_t intel_hdcp_gsc_msg_send(struct intel_hdcp_gsc_context *gsc_context,
-				void *msg_in, size_t msg_in_len,
-				void *msg_out, size_t msg_out_len);
-bool intel_hdcp_gsc_check_status(struct drm_device *drm);
-
-struct intel_hdcp_gsc_context *intel_hdcp_gsc_context_alloc(struct drm_device *drm);
-void intel_hdcp_gsc_context_free(struct intel_hdcp_gsc_context *gsc_context);
-
-#endif /* __INTEL_HDCP_GCS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_hdcp_gsc_message.c b/drivers/gpu/drm/i915/display/intel_hdcp_gsc_message.c
index 98967bb148e3..781667b710b4 100644
--- a/drivers/gpu/drm/i915/display/intel_hdcp_gsc_message.c
+++ b/drivers/gpu/drm/i915/display/intel_hdcp_gsc_message.c
@@ -10,8 +10,8 @@
 
 #include "intel_display_core.h"
 #include "intel_display_types.h"
-#include "intel_hdcp_gsc.h"
 #include "intel_hdcp_gsc_message.h"
+#include "intel_parent.h"
 
 static int
 intel_hdcp_gsc_initiate_session(struct device *dev, struct hdcp_port_data *data,
@@ -44,10 +44,9 @@ intel_hdcp_gsc_initiate_session(struct device *dev, struct hdcp_port_data *data,
 	session_init_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
 	session_init_in.protocol = data->protocol;
 
-	byte = intel_hdcp_gsc_msg_send(gsc_context, &session_init_in,
-				       sizeof(session_init_in),
-				       &session_init_out,
-				       sizeof(session_init_out));
+	byte = intel_parent_hdcp_gsc_msg_send(display, gsc_context,
+					      &session_init_in, sizeof(session_init_in),
+					      &session_init_out, sizeof(session_init_out));
 	if (byte < 0) {
 		drm_dbg_kms(display->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
 		return byte;
@@ -106,10 +105,9 @@ intel_hdcp_gsc_verify_receiver_cert_prepare_km(struct device *dev,
 	memcpy(verify_rxcert_in.r_rx, &rx_cert->r_rx, HDCP_2_2_RRX_LEN);
 	memcpy(verify_rxcert_in.rx_caps, rx_cert->rx_caps, HDCP_2_2_RXCAPS_LEN);
 
-	byte = intel_hdcp_gsc_msg_send(gsc_context, &verify_rxcert_in,
-				       sizeof(verify_rxcert_in),
-				       &verify_rxcert_out,
-				       sizeof(verify_rxcert_out));
+	byte = intel_parent_hdcp_gsc_msg_send(display, gsc_context,
+					      &verify_rxcert_in, sizeof(verify_rxcert_in),
+					      &verify_rxcert_out, sizeof(verify_rxcert_out));
 	if (byte < 0) {
 		drm_dbg_kms(display->drm, "intel_hdcp_gsc_msg_send failed: %zd\n", byte);
 		return byte;
@@ -169,10 +167,9 @@ intel_hdcp_gsc_verify_hprime(struct device *dev, struct hdcp_port_data *data,
 	memcpy(send_hprime_in.h_prime, rx_hprime->h_prime,
 	       HDCP_2_2_H_PRIME_LEN);
 
-	byte = intel_hdcp_gsc_msg_send(gsc_context, &send_hprime_in,
-				       sizeof(send_hprime_in),
-				       &send_hprime_out,
-				       sizeof(send_hprime_out));
+	byte = intel_parent_hdcp_gsc_msg_send(display, gsc_context,
+					      &send_hprime_in, sizeof(send_hprime_in),
+					      &send_hprime_out, sizeof(send_hprime_out));
 	if (byte < 0) {
 		drm_dbg_kms(display->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
 		return byte;
@@ -220,10 +217,9 @@ intel_hdcp_gsc_store_pairing_info(struct device *dev, struct hdcp_port_data *dat
 	memcpy(pairing_info_in.e_kh_km, pairing_info->e_kh_km,
 	       HDCP_2_2_E_KH_KM_LEN);
 
-	byte = intel_hdcp_gsc_msg_send(gsc_context, &pairing_info_in,
-				       sizeof(pairing_info_in),
-				       &pairing_info_out,
-				       sizeof(pairing_info_out));
+	byte = intel_parent_hdcp_gsc_msg_send(display, gsc_context,
+					      &pairing_info_in, sizeof(pairing_info_in),
+					      &pairing_info_out, sizeof(pairing_info_out));
 	if (byte < 0) {
 		drm_dbg_kms(display->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
 		return byte;
@@ -269,8 +265,9 @@ intel_hdcp_gsc_initiate_locality_check(struct device *dev,
 	lc_init_in.port.physical_port = (u8)data->hdcp_ddi;
 	lc_init_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
 
-	byte = intel_hdcp_gsc_msg_send(gsc_context, &lc_init_in, sizeof(lc_init_in),
-				       &lc_init_out, sizeof(lc_init_out));
+	byte = intel_parent_hdcp_gsc_msg_send(display, gsc_context,
+					      &lc_init_in, sizeof(lc_init_in),
+					      &lc_init_out, sizeof(lc_init_out));
 	if (byte < 0) {
 		drm_dbg_kms(display->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
 		return byte;
@@ -321,10 +318,9 @@ intel_hdcp_gsc_verify_lprime(struct device *dev, struct hdcp_port_data *data,
 	memcpy(verify_lprime_in.l_prime, rx_lprime->l_prime,
 	       HDCP_2_2_L_PRIME_LEN);
 
-	byte = intel_hdcp_gsc_msg_send(gsc_context, &verify_lprime_in,
-				       sizeof(verify_lprime_in),
-				       &verify_lprime_out,
-				       sizeof(verify_lprime_out));
+	byte = intel_parent_hdcp_gsc_msg_send(display, gsc_context,
+					      &verify_lprime_in, sizeof(verify_lprime_in),
+					      &verify_lprime_out, sizeof(verify_lprime_out));
 	if (byte < 0) {
 		drm_dbg_kms(display->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
 		return byte;
@@ -370,8 +366,9 @@ intel_hdcp_gsc_get_session_key(struct device *dev,
 	get_skey_in.port.physical_port = (u8)data->hdcp_ddi;
 	get_skey_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
 
-	byte = intel_hdcp_gsc_msg_send(gsc_context, &get_skey_in, sizeof(get_skey_in),
-				       &get_skey_out, sizeof(get_skey_out));
+	byte = intel_parent_hdcp_gsc_msg_send(display, gsc_context,
+					      &get_skey_in, sizeof(get_skey_in),
+					      &get_skey_out, sizeof(get_skey_out));
 	if (byte < 0) {
 		drm_dbg_kms(display->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
 		return byte;
@@ -434,10 +431,9 @@ intel_hdcp_gsc_repeater_check_flow_prepare_ack(struct device *dev,
 	memcpy(verify_repeater_in.receiver_ids, rep_topology->receiver_ids,
 	       HDCP_2_2_RECEIVER_IDS_MAX_LEN);
 
-	byte = intel_hdcp_gsc_msg_send(gsc_context, &verify_repeater_in,
-				       sizeof(verify_repeater_in),
-				       &verify_repeater_out,
-				       sizeof(verify_repeater_out));
+	byte = intel_parent_hdcp_gsc_msg_send(display, gsc_context,
+					      &verify_repeater_in, sizeof(verify_repeater_in),
+					      &verify_repeater_out, sizeof(verify_repeater_out));
 	if (byte < 0) {
 		drm_dbg_kms(display->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
 		return byte;
@@ -504,9 +500,9 @@ intel_hdcp_gsc_verify_mprime(struct device *dev,
 
 	verify_mprime_in->k = cpu_to_be16(data->k);
 
-	byte = intel_hdcp_gsc_msg_send(gsc_context, verify_mprime_in, cmd_size,
-				       &verify_mprime_out,
-				       sizeof(verify_mprime_out));
+	byte = intel_parent_hdcp_gsc_msg_send(display, gsc_context,
+					      verify_mprime_in, cmd_size,
+					      &verify_mprime_out, sizeof(verify_mprime_out));
 	kfree(verify_mprime_in);
 	if (byte < 0) {
 		drm_dbg_kms(display->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
@@ -552,10 +548,9 @@ static int intel_hdcp_gsc_enable_authentication(struct device *dev,
 	enable_auth_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
 	enable_auth_in.stream_type = data->streams[0].stream_type;
 
-	byte = intel_hdcp_gsc_msg_send(gsc_context, &enable_auth_in,
-				       sizeof(enable_auth_in),
-				       &enable_auth_out,
-				       sizeof(enable_auth_out));
+	byte = intel_parent_hdcp_gsc_msg_send(display, gsc_context,
+					      &enable_auth_in, sizeof(enable_auth_in),
+					      &enable_auth_out, sizeof(enable_auth_out));
 	if (byte < 0) {
 		drm_dbg_kms(display->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
 		return byte;
@@ -599,10 +594,9 @@ intel_hdcp_gsc_close_session(struct device *dev, struct hdcp_port_data *data)
 	session_close_in.port.physical_port = (u8)data->hdcp_ddi;
 	session_close_in.port.attached_transcoder = (u8)data->hdcp_transcoder;
 
-	byte = intel_hdcp_gsc_msg_send(gsc_context, &session_close_in,
-				       sizeof(session_close_in),
-				       &session_close_out,
-				       sizeof(session_close_out));
+	byte = intel_parent_hdcp_gsc_msg_send(display, gsc_context,
+					      &session_close_in, sizeof(session_close_in),
+					      &session_close_out, sizeof(session_close_out));
 	if (byte < 0) {
 		drm_dbg_kms(display->drm, "intel_hdcp_gsc_msg_send failed. %zd\n", byte);
 		return byte;
@@ -645,7 +639,7 @@ int intel_hdcp_gsc_init(struct intel_display *display)
 
 	mutex_lock(&display->hdcp.hdcp_mutex);
 
-	gsc_context = intel_hdcp_gsc_context_alloc(display->drm);
+	gsc_context = intel_parent_hdcp_gsc_context_alloc(display);
 	if (IS_ERR(gsc_context)) {
 		ret = PTR_ERR(gsc_context);
 		kfree(arbiter);
@@ -665,7 +659,7 @@ int intel_hdcp_gsc_init(struct intel_display *display)
 
 void intel_hdcp_gsc_fini(struct intel_display *display)
 {
-	intel_hdcp_gsc_context_free(display->hdcp.gsc_context);
+	intel_parent_hdcp_gsc_context_free(display, display->hdcp.gsc_context);
 	display->hdcp.gsc_context = NULL;
 	kfree(display->hdcp.arbiter);
 	display->hdcp.arbiter = NULL;
diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c
index 908faf17f93d..055e68810d0d 100644
--- a/drivers/gpu/drm/i915/display/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/display/intel_hdmi.c
@@ -2518,7 +2518,7 @@ intel_hdmi_set_edid(struct drm_connector *_connector)
 	struct intel_display *display = to_intel_display(connector);
 	struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
 	struct i2c_adapter *ddc = connector->base.ddc;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	const struct drm_edid *drm_edid;
 	bool connected = false;
 
@@ -2561,7 +2561,7 @@ intel_hdmi_detect(struct drm_connector *_connector, bool force)
 	enum drm_connector_status status = connector_status_disconnected;
 	struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
 	struct intel_encoder *encoder = &hdmi_to_dig_port(intel_hdmi)->base;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 
 	drm_dbg_kms(display->drm, "[CONNECTOR:%d:%s]\n",
 		    connector->base.base.id, connector->base.name);
diff --git a/drivers/gpu/drm/i915/display/intel_hotplug.c b/drivers/gpu/drm/i915/display/intel_hotplug.c
index 235706229ffb..970aa95ee344 100644
--- a/drivers/gpu/drm/i915/display/intel_hotplug.c
+++ b/drivers/gpu/drm/i915/display/intel_hotplug.c
@@ -27,8 +27,6 @@
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 
-#include "i915_drv.h"
-#include "i915_irq.h"
 #include "intel_connector.h"
 #include "intel_display_core.h"
 #include "intel_display_power.h"
@@ -39,6 +37,7 @@
 #include "intel_hdcp.h"
 #include "intel_hotplug.h"
 #include "intel_hotplug_irq.h"
+#include "intel_parent.h"
 
 /**
  * DOC: Hotplug
@@ -786,7 +785,7 @@ static void i915_hpd_poll_init_work(struct work_struct *work)
 		container_of(work, typeof(*display), hotplug.poll_init_work);
 	struct drm_connector_list_iter conn_iter;
 	struct intel_connector *connector;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool enabled;
 
 	mutex_lock(&display->drm->mode_config.mutex);
@@ -1177,13 +1176,12 @@ bool intel_hpd_schedule_detection(struct intel_display *display)
 static int i915_hpd_storm_ctl_show(struct seq_file *m, void *data)
 {
 	struct intel_display *display = m->private;
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
 	struct intel_hotplug *hotplug = &display->hotplug;
 
 	/* Synchronize with everything first in case there's been an HPD
 	 * storm, but we haven't finished handling it in the kernel yet
 	 */
-	intel_synchronize_irq(dev_priv);
+	intel_parent_irq_synchronize(display);
 	flush_work(&display->hotplug.dig_port_work);
 	flush_delayed_work(&display->hotplug.hotplug_work);
 
diff --git a/drivers/gpu/drm/i915/display/intel_hotplug_irq.c b/drivers/gpu/drm/i915/display/intel_hotplug_irq.c
index 46c47b3d6f42..82c39e4ffa37 100644
--- a/drivers/gpu/drm/i915/display/intel_hotplug_irq.c
+++ b/drivers/gpu/drm/i915/display/intel_hotplug_irq.c
@@ -519,12 +519,9 @@ void xelpdp_pica_irq_handler(struct intel_display *display, u32 iir)
 {
 	enum hpd_pin pin;
 	u32 hotplug_trigger = iir & (XELPDP_DP_ALT_HOTPLUG_MASK | XELPDP_TBT_HOTPLUG_MASK);
-	u32 trigger_aux = iir & XELPDP_AUX_TC_MASK;
+	u32 trigger_aux = iir & xelpdp_pica_aux_mask(display);
 	u32 pin_mask = 0, long_mask = 0;
 
-	if (DISPLAY_VER(display) >= 20)
-		trigger_aux |= iir & XE2LPD_AUX_DDI_MASK;
-
 	for (pin = HPD_PORT_TC1; pin <= HPD_PORT_TC4; pin++) {
 		u32 val;
 
diff --git a/drivers/gpu/drm/i915/display/intel_lpe_audio.c b/drivers/gpu/drm/i915/display/intel_lpe_audio.c
index 42284e9928f2..5b41abe1c64d 100644
--- a/drivers/gpu/drm/i915/display/intel_lpe_audio.c
+++ b/drivers/gpu/drm/i915/display/intel_lpe_audio.c
@@ -71,7 +71,6 @@
 #include <drm/drm_print.h>
 #include <drm/intel/intel_lpe_audio.h>
 
-#include "i915_irq.h"
 #include "intel_audio_regs.h"
 #include "intel_de.h"
 #include "intel_lpe_audio.h"
diff --git a/drivers/gpu/drm/i915/display/intel_lt_phy.c b/drivers/gpu/drm/i915/display/intel_lt_phy.c
index a67eb4f7f897..939c8975fd4c 100644
--- a/drivers/gpu/drm/i915/display/intel_lt_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_lt_phy.c
@@ -1324,11 +1324,11 @@ intel_lt_phy_config_changed(struct intel_encoder *encoder,
 	return true;
 }
 
-static intel_wakeref_t intel_lt_phy_transaction_begin(struct intel_encoder *encoder)
+static struct ref_tracker *intel_lt_phy_transaction_begin(struct intel_encoder *encoder)
 {
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 
 	intel_psr_pause(intel_dp);
 	wakeref = intel_display_power_get(display, POWER_DOMAIN_DC_OFF);
@@ -1336,7 +1336,7 @@ static intel_wakeref_t intel_lt_phy_transaction_begin(struct intel_encoder *enco
 	return wakeref;
 }
 
-static void intel_lt_phy_transaction_end(struct intel_encoder *encoder, intel_wakeref_t wakeref)
+static void intel_lt_phy_transaction_end(struct intel_encoder *encoder, struct ref_tracker *wakeref)
 {
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
@@ -1932,7 +1932,7 @@ void intel_lt_phy_pll_enable(struct intel_encoder *encoder,
 	u8 owned_lane_mask = intel_lt_phy_get_owned_lane_mask(encoder);
 	enum phy phy = intel_encoder_to_phy(encoder);
 	enum port port = encoder->port;
-	intel_wakeref_t wakeref = 0;
+	struct ref_tracker *wakeref = 0;
 	u32 lane_phy_pulse_status = owned_lane_mask == INTEL_LT_PHY_BOTH_LANES
 					? (XE3PLPDP_LANE_PHY_PULSE_STATUS(0) |
 					   XE3PLPDP_LANE_PHY_PULSE_STATUS(1))
@@ -2060,7 +2060,7 @@ void intel_lt_phy_pll_disable(struct intel_encoder *encoder)
 	struct intel_display *display = to_intel_display(encoder);
 	enum phy phy = intel_encoder_to_phy(encoder);
 	enum port port = encoder->port;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	u8 owned_lane_mask = intel_lt_phy_get_owned_lane_mask(encoder);
 	u32 lane_pipe_reset = owned_lane_mask == INTEL_LT_PHY_BOTH_LANES
 				? (XELPDP_LANE_PIPE_RESET(0) |
@@ -2137,7 +2137,7 @@ void intel_lt_phy_set_signal_levels(struct intel_encoder *encoder,
 	struct intel_display *display = to_intel_display(encoder);
 	const struct intel_ddi_buf_trans *trans;
 	u8 owned_lane_mask;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	int n_entries, ln;
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 
@@ -2222,7 +2222,7 @@ void intel_lt_phy_pll_readout_hw_state(struct intel_encoder *encoder,
 {
 	u8 owned_lane_mask;
 	u8 lane;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	int i, j, k;
 
 	pll_state->tbt_mode = intel_tc_port_in_tbt_alt_mode(enc_to_dig_port(encoder));
@@ -2310,7 +2310,7 @@ void intel_xe3plpd_pll_enable(struct intel_encoder *encoder,
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 
 	if (intel_tc_port_in_tbt_alt_mode(dig_port))
-		intel_mtl_tbt_pll_enable(encoder, crtc_state);
+		intel_mtl_tbt_pll_enable_clock(encoder, crtc_state->port_clock);
 	else
 		intel_lt_phy_pll_enable(encoder, crtc_state);
 }
@@ -2320,7 +2320,7 @@ void intel_xe3plpd_pll_disable(struct intel_encoder *encoder)
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 
 	if (intel_tc_port_in_tbt_alt_mode(dig_port))
-		intel_mtl_tbt_pll_disable(encoder);
+		intel_mtl_tbt_pll_disable_clock(encoder);
 	else
 		intel_lt_phy_pll_disable(encoder);
 
diff --git a/drivers/gpu/drm/i915/display/intel_lt_phy.h b/drivers/gpu/drm/i915/display/intel_lt_phy.h
index b7911acd7dcd..7659c92b6c3c 100644
--- a/drivers/gpu/drm/i915/display/intel_lt_phy.h
+++ b/drivers/gpu/drm/i915/display/intel_lt_phy.h
@@ -42,6 +42,4 @@ void intel_xe3plpd_pll_enable(struct intel_encoder *encoder,
 			      const struct intel_crtc_state *crtc_state);
 void intel_xe3plpd_pll_disable(struct intel_encoder *encoder);
 
-#define HAS_LT_PHY(display) (DISPLAY_VER(display) >= 35)
-
 #endif /* __INTEL_LT_PHY_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_lvds.c b/drivers/gpu/drm/i915/display/intel_lvds.c
index 89aeb4fb340e..457d60863536 100644
--- a/drivers/gpu/drm/i915/display/intel_lvds.c
+++ b/drivers/gpu/drm/i915/display/intel_lvds.c
@@ -105,7 +105,7 @@ static bool intel_lvds_get_hw_state(struct intel_encoder *encoder,
 {
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(encoder);
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret;
 
 	wakeref = intel_display_power_get_if_enabled(display, encoder->power_domain);
diff --git a/drivers/gpu/drm/i915/display/intel_modeset_setup.c b/drivers/gpu/drm/i915/display/intel_modeset_setup.c
index 0dcb0597879a..d10cbf69a5f8 100644
--- a/drivers/gpu/drm/i915/display/intel_modeset_setup.c
+++ b/drivers/gpu/drm/i915/display/intel_modeset_setup.c
@@ -940,7 +940,7 @@ void intel_modeset_setup_hw_state(struct intel_display *display,
 {
 	struct intel_encoder *encoder;
 	struct intel_crtc *crtc;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 
 	wakeref = intel_display_power_get(display, POWER_DOMAIN_INIT);
 
diff --git a/drivers/gpu/drm/i915/display/intel_modeset_verify.c b/drivers/gpu/drm/i915/display/intel_modeset_verify.c
index b361a77cd235..12a00121c274 100644
--- a/drivers/gpu/drm/i915/display/intel_modeset_verify.c
+++ b/drivers/gpu/drm/i915/display/intel_modeset_verify.c
@@ -246,7 +246,6 @@ void intel_modeset_verify_crtc(struct intel_atomic_state *state,
 	verify_crtc_state(state, crtc);
 	intel_dpll_state_verify(state, crtc);
 	intel_mpllb_state_verify(state, crtc);
-	intel_cx0pll_state_verify(state, crtc);
 	intel_lt_phy_pll_state_verify(state, crtc);
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_panic.c b/drivers/gpu/drm/i915/display/intel_panic.c
deleted file mode 100644
index 7311ce4e8b6c..000000000000
--- a/drivers/gpu/drm/i915/display/intel_panic.c
+++ /dev/null
@@ -1,27 +0,0 @@
-// SPDX-License-Identifier: MIT
-/* Copyright © 2025 Intel Corporation */
-
-#include <drm/drm_panic.h>
-
-#include "gem/i915_gem_object.h"
-#include "intel_display_types.h"
-#include "intel_fb.h"
-#include "intel_panic.h"
-
-struct intel_panic *intel_panic_alloc(void)
-{
-	return i915_gem_object_alloc_panic();
-}
-
-int intel_panic_setup(struct intel_panic *panic, struct drm_scanout_buffer *sb)
-{
-	struct intel_framebuffer *fb = sb->private;
-	struct drm_gem_object *obj = intel_fb_bo(&fb->base);
-
-	return i915_gem_object_panic_setup(panic, sb, obj, fb->panic_tiling);
-}
-
-void intel_panic_finish(struct intel_panic *panic)
-{
-	return i915_gem_object_panic_finish(panic);
-}
diff --git a/drivers/gpu/drm/i915/display/intel_panic.h b/drivers/gpu/drm/i915/display/intel_panic.h
deleted file mode 100644
index afb472e924aa..000000000000
--- a/drivers/gpu/drm/i915/display/intel_panic.h
+++ /dev/null
@@ -1,14 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/* Copyright © 2025 Intel Corporation */
-
-#ifndef __INTEL_PANIC_H__
-#define __INTEL_PANIC_H__
-
-struct drm_scanout_buffer;
-struct intel_panic;
-
-struct intel_panic *intel_panic_alloc(void);
-int intel_panic_setup(struct intel_panic *panic, struct drm_scanout_buffer *sb);
-void intel_panic_finish(struct intel_panic *panic);
-
-#endif /* __INTEL_PANIC_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_parent.c b/drivers/gpu/drm/i915/display/intel_parent.c
new file mode 100644
index 000000000000..9b1a84a439e9
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_parent.c
@@ -0,0 +1,197 @@
+// SPDX-License-Identifier: MIT
+/* Copyright © 2025 Intel Corporation */
+
+/*
+ * Convenience wrapper functions to call the parent interface functions:
+ *
+ * - display->parent->SUBSTRUCT->FUNCTION()
+ * - display->parent->FUNCTION()
+ *
+ * All functions here should be named accordingly:
+ *
+ * - intel_parent_SUBSTRUCT_FUNCTION()
+ * - intel_parent_FUNCTION()
+ *
+ * These functions may use display driver specific types for parameters and
+ * return values, translating them to and from the generic types used in the
+ * function pointer interface.
+ */
+
+#include <drm/drm_print.h>
+#include <drm/intel/display_parent_interface.h>
+
+#include "intel_display_core.h"
+#include "intel_parent.h"
+
+/* hdcp */
+ssize_t intel_parent_hdcp_gsc_msg_send(struct intel_display *display,
+				       struct intel_hdcp_gsc_context *gsc_context,
+				       void *msg_in, size_t msg_in_len,
+				       void *msg_out, size_t msg_out_len)
+{
+	return display->parent->hdcp->gsc_msg_send(gsc_context, msg_in, msg_in_len, msg_out, msg_out_len);
+}
+
+bool intel_parent_hdcp_gsc_check_status(struct intel_display *display)
+{
+	return display->parent->hdcp->gsc_check_status(display->drm);
+}
+
+struct intel_hdcp_gsc_context *intel_parent_hdcp_gsc_context_alloc(struct intel_display *display)
+{
+	return display->parent->hdcp->gsc_context_alloc(display->drm);
+}
+
+void intel_parent_hdcp_gsc_context_free(struct intel_display *display,
+					struct intel_hdcp_gsc_context *gsc_context)
+{
+	display->parent->hdcp->gsc_context_free(gsc_context);
+}
+
+/* irq */
+bool intel_parent_irq_enabled(struct intel_display *display)
+{
+	return display->parent->irq->enabled(display->drm);
+}
+
+void intel_parent_irq_synchronize(struct intel_display *display)
+{
+	display->parent->irq->synchronize(display->drm);
+}
+
+/* panic */
+struct intel_panic *intel_parent_panic_alloc(struct intel_display *display)
+{
+	return display->parent->panic->alloc();
+}
+
+int intel_parent_panic_setup(struct intel_display *display, struct intel_panic *panic, struct drm_scanout_buffer *sb)
+{
+	return display->parent->panic->setup(panic, sb);
+}
+
+void intel_parent_panic_finish(struct intel_display *display, struct intel_panic *panic)
+{
+	display->parent->panic->finish(panic);
+}
+
+/* rps */
+bool intel_parent_rps_available(struct intel_display *display)
+{
+	return display->parent->rps;
+}
+
+void intel_parent_rps_boost_if_not_started(struct intel_display *display, struct dma_fence *fence)
+{
+	if (display->parent->rps)
+		display->parent->rps->boost_if_not_started(fence);
+}
+
+void intel_parent_rps_mark_interactive(struct intel_display *display, bool interactive)
+{
+	if (display->parent->rps)
+		display->parent->rps->mark_interactive(display->drm, interactive);
+}
+
+void intel_parent_rps_ilk_irq_handler(struct intel_display *display)
+{
+	if (display->parent->rps)
+		display->parent->rps->ilk_irq_handler(display->drm);
+}
+
+/* stolen */
+int intel_parent_stolen_insert_node_in_range(struct intel_display *display,
+					     struct intel_stolen_node *node, u64 size,
+					     unsigned int align, u64 start, u64 end)
+{
+	return display->parent->stolen->insert_node_in_range(node, size, align, start, end);
+}
+
+int intel_parent_stolen_insert_node(struct intel_display *display, struct intel_stolen_node *node, u64 size,
+				    unsigned int align)
+{
+	if (drm_WARN_ON_ONCE(display->drm, !display->parent->stolen->insert_node))
+		return -ENODEV;
+
+	return display->parent->stolen->insert_node(node, size, align);
+}
+
+void intel_parent_stolen_remove_node(struct intel_display *display,
+				     struct intel_stolen_node *node)
+{
+	display->parent->stolen->remove_node(node);
+}
+
+bool intel_parent_stolen_initialized(struct intel_display *display)
+{
+	return display->parent->stolen->initialized(display->drm);
+}
+
+bool intel_parent_stolen_node_allocated(struct intel_display *display,
+					const struct intel_stolen_node *node)
+{
+	return display->parent->stolen->node_allocated(node);
+}
+
+u32 intel_parent_stolen_node_offset(struct intel_display *display, struct intel_stolen_node *node)
+{
+	return display->parent->stolen->node_offset(node);
+}
+
+u64 intel_parent_stolen_area_address(struct intel_display *display)
+{
+	if (drm_WARN_ON_ONCE(display->drm, !display->parent->stolen->area_address))
+		return 0;
+
+	return display->parent->stolen->area_address(display->drm);
+}
+
+u64 intel_parent_stolen_area_size(struct intel_display *display)
+{
+	if (drm_WARN_ON_ONCE(display->drm, !display->parent->stolen->area_size))
+		return 0;
+
+	return display->parent->stolen->area_size(display->drm);
+}
+
+u64 intel_parent_stolen_node_address(struct intel_display *display, struct intel_stolen_node *node)
+{
+	return display->parent->stolen->node_address(node);
+}
+
+u64 intel_parent_stolen_node_size(struct intel_display *display, const struct intel_stolen_node *node)
+{
+	return display->parent->stolen->node_size(node);
+}
+
+struct intel_stolen_node *intel_parent_stolen_node_alloc(struct intel_display *display)
+{
+	return display->parent->stolen->node_alloc(display->drm);
+}
+
+void intel_parent_stolen_node_free(struct intel_display *display, const struct intel_stolen_node *node)
+{
+	display->parent->stolen->node_free(node);
+}
+
+/* generic */
+void intel_parent_fence_priority_display(struct intel_display *display, struct dma_fence *fence)
+{
+	if (display->parent->fence_priority_display)
+		display->parent->fence_priority_display(fence);
+}
+
+bool intel_parent_has_auxccs(struct intel_display *display)
+{
+	return display->parent->has_auxccs && display->parent->has_auxccs(display->drm);
+}
+
+bool intel_parent_has_fenced_regions(struct intel_display *display)
+{
+	return display->parent->has_fenced_regions && display->parent->has_fenced_regions(display->drm);
+}
+
+bool intel_parent_vgpu_active(struct intel_display *display)
+{
+	return display->parent->vgpu_active && display->parent->vgpu_active(display->drm);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_parent.h b/drivers/gpu/drm/i915/display/intel_parent.h
new file mode 100644
index 000000000000..a2a631fba118
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_parent.h
@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2025 Intel Corporation */
+
+#ifndef __INTEL_PARENT_H__
+#define __INTEL_PARENT_H__
+
+#include <linux/types.h>
+
+struct dma_fence;
+struct drm_scanout_buffer;
+struct intel_display;
+struct intel_hdcp_gsc_context;
+struct intel_panic;
+struct intel_stolen_node;
+
+/* hdcp */
+ssize_t intel_parent_hdcp_gsc_msg_send(struct intel_display *display,
+				       struct intel_hdcp_gsc_context *gsc_context,
+				       void *msg_in, size_t msg_in_len,
+				       void *msg_out, size_t msg_out_len);
+bool intel_parent_hdcp_gsc_check_status(struct intel_display *display);
+struct intel_hdcp_gsc_context *intel_parent_hdcp_gsc_context_alloc(struct intel_display *display);
+void intel_parent_hdcp_gsc_context_free(struct intel_display *display,
+					struct intel_hdcp_gsc_context *gsc_context);
+
+/* irq */
+bool intel_parent_irq_enabled(struct intel_display *display);
+void intel_parent_irq_synchronize(struct intel_display *display);
+
+/* panic */
+struct intel_panic *intel_parent_panic_alloc(struct intel_display *display);
+int intel_parent_panic_setup(struct intel_display *display, struct intel_panic *panic, struct drm_scanout_buffer *sb);
+void intel_parent_panic_finish(struct intel_display *display, struct intel_panic *panic);
+
+/* rps */
+bool intel_parent_rps_available(struct intel_display *display);
+void intel_parent_rps_boost_if_not_started(struct intel_display *display, struct dma_fence *fence);
+void intel_parent_rps_mark_interactive(struct intel_display *display, bool interactive);
+void intel_parent_rps_ilk_irq_handler(struct intel_display *display);
+
+/* stolen */
+int intel_parent_stolen_insert_node_in_range(struct intel_display *display,
+					     struct intel_stolen_node *node, u64 size,
+					     unsigned int align, u64 start, u64 end);
+int intel_parent_stolen_insert_node(struct intel_display *display, struct intel_stolen_node *node, u64 size,
+				    unsigned int align);
+void intel_parent_stolen_remove_node(struct intel_display *display,
+				     struct intel_stolen_node *node);
+bool intel_parent_stolen_initialized(struct intel_display *display);
+bool intel_parent_stolen_node_allocated(struct intel_display *display,
+					const struct intel_stolen_node *node);
+u32 intel_parent_stolen_node_offset(struct intel_display *display, struct intel_stolen_node *node);
+u64 intel_parent_stolen_area_address(struct intel_display *display);
+u64 intel_parent_stolen_area_size(struct intel_display *display);
+u64 intel_parent_stolen_node_address(struct intel_display *display, struct intel_stolen_node *node);
+u64 intel_parent_stolen_node_size(struct intel_display *display, const struct intel_stolen_node *node);
+struct intel_stolen_node *intel_parent_stolen_node_alloc(struct intel_display *display);
+void intel_parent_stolen_node_free(struct intel_display *display, const struct intel_stolen_node *node);
+
+/* generic */
+bool intel_parent_has_auxccs(struct intel_display *display);
+bool intel_parent_has_fenced_regions(struct intel_display *display);
+bool intel_parent_vgpu_active(struct intel_display *display);
+void intel_parent_fence_priority_display(struct intel_display *display, struct dma_fence *fence);
+
+#endif /* __INTEL_PARENT_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_pipe_crc.c b/drivers/gpu/drm/i915/display/intel_pipe_crc.c
index 1f27643412f1..57586c78582d 100644
--- a/drivers/gpu/drm/i915/display/intel_pipe_crc.c
+++ b/drivers/gpu/drm/i915/display/intel_pipe_crc.c
@@ -30,13 +30,12 @@
 
 #include <drm/drm_print.h>
 
-#include "i915_drv.h"
-#include "i915_irq.h"
 #include "intel_atomic.h"
 #include "intel_de.h"
 #include "intel_display_irq.h"
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
+#include "intel_parent.h"
 #include "intel_pipe_crc.h"
 #include "intel_pipe_crc_regs.h"
 
@@ -589,7 +588,7 @@ int intel_crtc_set_crc_source(struct drm_crtc *_crtc, const char *source_name)
 	enum intel_display_power_domain power_domain;
 	enum intel_pipe_crc_source source;
 	enum pipe pipe = crtc->pipe;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	u32 val = 0; /* shut up gcc */
 	int ret = 0;
 	bool enable;
@@ -658,7 +657,6 @@ void intel_crtc_enable_pipe_crc(struct intel_crtc *crtc)
 void intel_crtc_disable_pipe_crc(struct intel_crtc *crtc)
 {
 	struct intel_display *display = to_intel_display(crtc);
-	struct drm_i915_private *dev_priv = to_i915(display->drm);
 	struct intel_pipe_crc *pipe_crc = &crtc->pipe_crc;
 	enum pipe pipe = crtc->pipe;
 
@@ -669,5 +667,5 @@ void intel_crtc_disable_pipe_crc(struct intel_crtc *crtc)
 
 	intel_de_write(display, PIPE_CRC_CTL(display, pipe), 0);
 	intel_de_posting_read(display, PIPE_CRC_CTL(display, pipe));
-	intel_synchronize_irq(dev_priv);
+	intel_parent_irq_synchronize(display);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_plane.c b/drivers/gpu/drm/i915/display/intel_plane.c
index ab6a58530b39..3dc2ed52147f 100644
--- a/drivers/gpu/drm/i915/display/intel_plane.c
+++ b/drivers/gpu/drm/i915/display/intel_plane.c
@@ -45,7 +45,6 @@
 #include <drm/drm_panic.h>
 #include <drm/drm_print.h>
 
-#include "gem/i915_gem_object.h"
 #include "i9xx_plane_regs.h"
 #include "intel_cdclk.h"
 #include "intel_cursor.h"
@@ -56,7 +55,7 @@
 #include "intel_fb.h"
 #include "intel_fb_pin.h"
 #include "intel_fbdev.h"
-#include "intel_panic.h"
+#include "intel_parent.h"
 #include "intel_plane.h"
 #include "intel_psr.h"
 #include "skl_scaler.h"
@@ -179,25 +178,29 @@ bool intel_plane_needs_physical(struct intel_plane *plane)
 		DISPLAY_INFO(display)->cursor_needs_physical;
 }
 
-bool intel_plane_can_async_flip(struct intel_plane *plane, u32 format,
+bool intel_plane_can_async_flip(struct intel_plane *plane,
+				const struct drm_format_info *info,
 				u64 modifier)
 {
-	if (intel_format_info_is_yuv_semiplanar(drm_format_info(format), modifier) ||
-	    format == DRM_FORMAT_C8)
+	if (intel_format_info_is_yuv_semiplanar(info, modifier) ||
+	    info->format == DRM_FORMAT_C8)
 		return false;
 
 	return plane->can_async_flip && plane->can_async_flip(modifier);
 }
 
-bool intel_plane_format_mod_supported_async(struct drm_plane *plane,
-					    u32 format,
-					    u64 modifier)
+bool intel_plane_format_mod_supported_async(struct drm_plane *_plane,
+					    u32 format, u64 modifier)
 {
-	if (!plane->funcs->format_mod_supported(plane, format, modifier))
+	struct intel_plane *plane = to_intel_plane(_plane);
+	const struct drm_format_info *info;
+
+	if (!plane->base.funcs->format_mod_supported(&plane->base, format, modifier))
 		return false;
 
-	return intel_plane_can_async_flip(to_intel_plane(plane),
-					format, modifier);
+	info = drm_get_format_info(plane->base.dev, format, modifier);
+
+	return intel_plane_can_async_flip(plane, info, modifier);
 }
 
 unsigned int intel_adjusted_rate(const struct drm_rect *src,
@@ -651,11 +654,10 @@ static int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_cr
 	    ilk_must_disable_cxsr(new_crtc_state, old_plane_state, new_plane_state))
 		new_crtc_state->disable_cxsr = true;
 
-	if (intel_plane_do_async_flip(plane, old_crtc_state, new_crtc_state)) {
+	if (intel_plane_do_async_flip(plane, old_crtc_state, new_crtc_state))
 		new_crtc_state->do_async_flip = true;
-		new_crtc_state->async_flip_planes |= BIT(plane->id);
-	} else if (plane->need_async_flip_toggle_wa &&
-		   new_crtc_state->uapi.async_flip) {
+
+	if (new_crtc_state->uapi.async_flip) {
 		/*
 		 * On platforms with double buffered async flip bit we
 		 * set the bit already one frame early during the sync
@@ -663,6 +665,9 @@ static int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_cr
 		 * hardware will therefore be ready to perform a real
 		 * async flip during the next commit, without having
 		 * to wait yet another frame for the bit to latch.
+		 *
+		 * async_flip_planes bitmask is also used by selective
+		 * fetch calculation to choose full frame update.
 		 */
 		new_crtc_state->async_flip_planes |= BIT(plane->id);
 	}
@@ -1235,8 +1240,7 @@ intel_prepare_plane_fb(struct drm_plane *_plane,
 		goto unpin_fb;
 
 	if (new_plane_state->uapi.fence) {
-		i915_gem_fence_wait_priority_display(new_plane_state->uapi.fence);
-
+		intel_parent_fence_priority_display(display, new_plane_state->uapi.fence);
 		intel_display_rps_boost_after_vblank(new_plane_state->hw.crtc,
 						     new_plane_state->uapi.fence);
 	}
@@ -1330,33 +1334,33 @@ static unsigned int intel_4tile_get_offset(unsigned int width, unsigned int x, u
 	return offset;
 }
 
-static void intel_panic_flush(struct drm_plane *plane)
+static void intel_panic_flush(struct drm_plane *_plane)
 {
-	struct intel_plane_state *plane_state = to_intel_plane_state(plane->state);
-	struct intel_crtc_state *crtc_state = to_intel_crtc_state(plane->state->crtc->state);
-	struct intel_plane *iplane = to_intel_plane(plane);
-	struct intel_display *display = to_intel_display(iplane);
-	struct drm_framebuffer *fb = plane_state->hw.fb;
-	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
+	struct intel_plane *plane = to_intel_plane(_plane);
+	struct intel_display *display = to_intel_display(plane);
+	const struct intel_plane_state *plane_state = to_intel_plane_state(plane->base.state);
+	struct intel_crtc *crtc = to_intel_crtc(plane_state->hw.crtc);
+	const struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
+	const struct intel_framebuffer *fb = to_intel_framebuffer(plane_state->hw.fb);
 
-	intel_panic_finish(intel_fb->panic);
+	intel_parent_panic_finish(display, fb->panic);
 
 	if (crtc_state->enable_psr2_sel_fetch) {
 		/* Force a full update for psr2 */
-		intel_psr2_panic_force_full_update(display, crtc_state);
+		intel_psr2_panic_force_full_update(crtc_state);
 	}
 
 	/* Flush the cache and don't disable tiling if it's the fbdev framebuffer.*/
-	if (intel_fb == intel_fbdev_framebuffer(display->fbdev.fbdev)) {
+	if (fb == intel_fbdev_framebuffer(display->fbdev.fbdev)) {
 		struct iosys_map map;
 
 		intel_fbdev_get_map(display->fbdev.fbdev, &map);
-		drm_clflush_virt_range(map.vaddr, fb->pitches[0] * fb->height);
+		drm_clflush_virt_range(map.vaddr, fb->base.pitches[0] * fb->base.height);
 		return;
 	}
 
-	if (fb->modifier && iplane->disable_tiling)
-		iplane->disable_tiling(iplane);
+	if (fb->base.modifier != DRM_FORMAT_MOD_LINEAR && plane->disable_tiling)
+		plane->disable_tiling(plane);
 }
 
 static unsigned int (*intel_get_tiling_func(u64 fb_modifier))(unsigned int width,
@@ -1394,45 +1398,43 @@ static int intel_get_scanout_buffer(struct drm_plane *plane,
 {
 	struct intel_plane_state *plane_state;
 	struct drm_gem_object *obj;
-	struct drm_framebuffer *fb;
-	struct intel_framebuffer *intel_fb;
+	struct intel_framebuffer *fb;
 	struct intel_display *display = to_intel_display(plane->dev);
 
 	if (!plane->state || !plane->state->fb || !plane->state->visible)
 		return -ENODEV;
 
 	plane_state = to_intel_plane_state(plane->state);
-	fb = plane_state->hw.fb;
-	intel_fb = to_intel_framebuffer(fb);
+	fb = to_intel_framebuffer(plane_state->hw.fb);
 
-	obj = intel_fb_bo(fb);
+	obj = intel_fb_bo(&fb->base);
 	if (!obj)
 		return -ENODEV;
 
-	if (intel_fb == intel_fbdev_framebuffer(display->fbdev.fbdev)) {
+	if (fb == intel_fbdev_framebuffer(display->fbdev.fbdev)) {
 		intel_fbdev_get_map(display->fbdev.fbdev, &sb->map[0]);
 	} else {
 		int ret;
 		/* Can't disable tiling if DPT is in use */
-		if (intel_fb_uses_dpt(fb)) {
-			if (fb->format->cpp[0] != 4)
+		if (intel_fb_uses_dpt(&fb->base)) {
+			if (fb->base.format->cpp[0] != 4)
 				return -EOPNOTSUPP;
-			intel_fb->panic_tiling = intel_get_tiling_func(fb->modifier);
-			if (!intel_fb->panic_tiling)
+			fb->panic_tiling = intel_get_tiling_func(fb->base.modifier);
+			if (!fb->panic_tiling)
 				return -EOPNOTSUPP;
 		}
-		sb->private = intel_fb;
-		ret = intel_panic_setup(intel_fb->panic, sb);
+		sb->private = fb;
+		ret = intel_parent_panic_setup(display, fb->panic, sb);
 		if (ret)
 			return ret;
 	}
-	sb->width = fb->width;
-	sb->height = fb->height;
+	sb->width = fb->base.width;
+	sb->height = fb->base.height;
 	/* Use the generic linear format, because tiling, RC, CCS, CC
 	 * will be disabled in disable_tiling()
 	 */
-	sb->format = drm_format_info(fb->format->format);
-	sb->pitch[0] = fb->pitches[0];
+	sb->format = drm_format_info(fb->base.format->format);
+	sb->pitch[0] = fb->base.pitches[0];
 
 	return 0;
 }
@@ -1464,7 +1466,7 @@ void intel_plane_init_cursor_vblank_work(struct intel_plane_state *old_plane_sta
 	    old_plane_state->ggtt_vma == new_plane_state->ggtt_vma)
 		return;
 
-	drm_vblank_work_init(&old_plane_state->unpin_work, old_plane_state->uapi.crtc,
+	drm_vblank_work_init(&old_plane_state->unpin_work, old_plane_state->hw.crtc,
 			     intel_cursor_unpin_work);
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_plane.h b/drivers/gpu/drm/i915/display/intel_plane.h
index 4e99df9de3e8..5a8f2f3baab5 100644
--- a/drivers/gpu/drm/i915/display/intel_plane.h
+++ b/drivers/gpu/drm/i915/display/intel_plane.h
@@ -8,6 +8,7 @@
 
 #include <linux/types.h>
 
+struct drm_format_info;
 struct drm_plane;
 struct drm_property;
 struct drm_rect;
@@ -21,7 +22,8 @@ enum plane_id;
 
 struct intel_plane *
 intel_crtc_get_plane(struct intel_crtc *crtc, enum plane_id plane_id);
-bool intel_plane_can_async_flip(struct intel_plane *plane, u32 format,
+bool intel_plane_can_async_flip(struct intel_plane *plane,
+				const struct drm_format_info *info,
 				u64 modifier);
 unsigned int intel_adjusted_rate(const struct drm_rect *src,
 				 const struct drm_rect *dst,
diff --git a/drivers/gpu/drm/i915/display/intel_plane_initial.c b/drivers/gpu/drm/i915/display/intel_plane_initial.c
index a1de1ec564d1..ff1afd3a8f20 100644
--- a/drivers/gpu/drm/i915/display/intel_plane_initial.c
+++ b/drivers/gpu/drm/i915/display/intel_plane_initial.c
@@ -39,7 +39,7 @@ intel_reuse_initial_plane_obj(struct intel_crtc *this,
 		const struct intel_crtc_state *crtc_state =
 			to_intel_crtc_state(crtc->base.state);
 
-		if (!crtc_state->uapi.active)
+		if (!crtc_state->hw.active)
 			continue;
 
 		if (!plane_state->ggtt_vma)
@@ -411,10 +411,12 @@ void intel_initial_plane_config(struct intel_display *display)
 	struct intel_crtc *crtc;
 
 	for_each_intel_crtc(display->drm, crtc) {
+		const struct intel_crtc_state *crtc_state =
+			to_intel_crtc_state(crtc->base.state);
 		struct intel_initial_plane_config *plane_config =
 			&plane_configs[crtc->pipe];
 
-		if (!to_intel_crtc_state(crtc->base.state)->uapi.active)
+		if (!crtc_state->hw.active)
 			continue;
 
 		/*
diff --git a/drivers/gpu/drm/i915/display/intel_pps.c b/drivers/gpu/drm/i915/display/intel_pps.c
index 25692a547764..b217ec7aa758 100644
--- a/drivers/gpu/drm/i915/display/intel_pps.c
+++ b/drivers/gpu/drm/i915/display/intel_pps.c
@@ -67,10 +67,10 @@ static const char *pps_name(struct intel_dp *intel_dp)
 	return "PPS <invalid>";
 }
 
-intel_wakeref_t intel_pps_lock(struct intel_dp *intel_dp)
+struct ref_tracker *intel_pps_lock(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 
 	/*
 	 * See vlv_pps_reset_all() why we need a power domain reference here.
@@ -81,8 +81,7 @@ intel_wakeref_t intel_pps_lock(struct intel_dp *intel_dp)
 	return wakeref;
 }
 
-intel_wakeref_t intel_pps_unlock(struct intel_dp *intel_dp,
-				 intel_wakeref_t wakeref)
+struct ref_tracker *intel_pps_unlock(struct intel_dp *intel_dp, struct ref_tracker *wakeref)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 
@@ -697,12 +696,10 @@ static void wait_panel_power_cycle(struct intel_dp *intel_dp)
 
 void intel_pps_wait_power_cycle(struct intel_dp *intel_dp)
 {
-	intel_wakeref_t wakeref;
-
 	if (!intel_dp_is_edp(intel_dp))
 		return;
 
-	with_intel_pps_lock(intel_dp, wakeref)
+	with_intel_pps_lock(intel_dp)
 		wait_panel_power_cycle(intel_dp);
 }
 
@@ -811,14 +808,13 @@ bool intel_pps_vdd_on_unlocked(struct intel_dp *intel_dp)
 void intel_pps_vdd_on(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
-	intel_wakeref_t wakeref;
 	bool vdd;
 
 	if (!intel_dp_is_edp(intel_dp))
 		return;
 
 	vdd = false;
-	with_intel_pps_lock(intel_dp, wakeref)
+	with_intel_pps_lock(intel_dp)
 		vdd = intel_pps_vdd_on_unlocked(intel_dp);
 	INTEL_DISPLAY_STATE_WARN(display, !vdd, "[ENCODER:%d:%s] %s VDD already requested on\n",
 				 dp_to_dig_port(intel_dp)->base.base.base.id,
@@ -873,8 +869,6 @@ static void intel_pps_vdd_off_sync_unlocked(struct intel_dp *intel_dp)
 
 void intel_pps_vdd_off_sync(struct intel_dp *intel_dp)
 {
-	intel_wakeref_t wakeref;
-
 	if (!intel_dp_is_edp(intel_dp))
 		return;
 
@@ -883,7 +877,7 @@ void intel_pps_vdd_off_sync(struct intel_dp *intel_dp)
 	 * vdd might still be enabled due to the delayed vdd off.
 	 * Make sure vdd is actually turned off here.
 	 */
-	with_intel_pps_lock(intel_dp, wakeref)
+	with_intel_pps_lock(intel_dp)
 		intel_pps_vdd_off_sync_unlocked(intel_dp);
 }
 
@@ -892,9 +886,8 @@ static void edp_panel_vdd_work(struct work_struct *__work)
 	struct intel_pps *pps = container_of(to_delayed_work(__work),
 					     struct intel_pps, panel_vdd_work);
 	struct intel_dp *intel_dp = container_of(pps, struct intel_dp, pps);
-	intel_wakeref_t wakeref;
 
-	with_intel_pps_lock(intel_dp, wakeref) {
+	with_intel_pps_lock(intel_dp) {
 		if (!intel_dp->pps.want_panel_vdd)
 			intel_pps_vdd_off_sync_unlocked(intel_dp);
 	}
@@ -952,12 +945,10 @@ void intel_pps_vdd_off_unlocked(struct intel_dp *intel_dp, bool sync)
 
 void intel_pps_vdd_off(struct intel_dp *intel_dp)
 {
-	intel_wakeref_t wakeref;
-
 	if (!intel_dp_is_edp(intel_dp))
 		return;
 
-	with_intel_pps_lock(intel_dp, wakeref)
+	with_intel_pps_lock(intel_dp)
 		intel_pps_vdd_off_unlocked(intel_dp, false);
 }
 
@@ -1026,12 +1017,10 @@ void intel_pps_on_unlocked(struct intel_dp *intel_dp)
 
 void intel_pps_on(struct intel_dp *intel_dp)
 {
-	intel_wakeref_t wakeref;
-
 	if (!intel_dp_is_edp(intel_dp))
 		return;
 
-	with_intel_pps_lock(intel_dp, wakeref)
+	with_intel_pps_lock(intel_dp)
 		intel_pps_on_unlocked(intel_dp);
 }
 
@@ -1082,12 +1071,10 @@ void intel_pps_off_unlocked(struct intel_dp *intel_dp)
 
 void intel_pps_off(struct intel_dp *intel_dp)
 {
-	intel_wakeref_t wakeref;
-
 	if (!intel_dp_is_edp(intel_dp))
 		return;
 
-	with_intel_pps_lock(intel_dp, wakeref)
+	with_intel_pps_lock(intel_dp)
 		intel_pps_off_unlocked(intel_dp);
 }
 
@@ -1095,7 +1082,6 @@ void intel_pps_off(struct intel_dp *intel_dp)
 void intel_pps_backlight_on(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
-	intel_wakeref_t wakeref;
 
 	/*
 	 * If we enable the backlight right away following a panel power
@@ -1105,7 +1091,7 @@ void intel_pps_backlight_on(struct intel_dp *intel_dp)
 	 */
 	wait_backlight_on(intel_dp);
 
-	with_intel_pps_lock(intel_dp, wakeref) {
+	with_intel_pps_lock(intel_dp) {
 		i915_reg_t pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
 		u32 pp;
 
@@ -1121,12 +1107,11 @@ void intel_pps_backlight_on(struct intel_dp *intel_dp)
 void intel_pps_backlight_off(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
-	intel_wakeref_t wakeref;
 
 	if (!intel_dp_is_edp(intel_dp))
 		return;
 
-	with_intel_pps_lock(intel_dp, wakeref) {
+	with_intel_pps_lock(intel_dp) {
 		i915_reg_t pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
 		u32 pp;
 
@@ -1149,11 +1134,10 @@ void intel_pps_backlight_power(struct intel_connector *connector, bool enable)
 {
 	struct intel_display *display = to_intel_display(connector);
 	struct intel_dp *intel_dp = intel_attached_dp(connector);
-	intel_wakeref_t wakeref;
 	bool is_enabled;
 
 	is_enabled = false;
-	with_intel_pps_lock(intel_dp, wakeref)
+	with_intel_pps_lock(intel_dp)
 		is_enabled = ilk_get_pp_control(intel_dp) & EDP_BLC_ENABLE;
 	if (is_enabled == enable)
 		return;
@@ -1251,9 +1235,7 @@ void vlv_pps_pipe_init(struct intel_dp *intel_dp)
 /* Call on all DP, not just eDP */
 void vlv_pps_pipe_reset(struct intel_dp *intel_dp)
 {
-	intel_wakeref_t wakeref;
-
-	with_intel_pps_lock(intel_dp, wakeref)
+	with_intel_pps_lock(intel_dp)
 		intel_dp->pps.vlv_active_pipe = vlv_active_pipe(intel_dp);
 }
 
@@ -1329,9 +1311,7 @@ void vlv_pps_port_disable(struct intel_encoder *encoder,
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 
-	intel_wakeref_t wakeref;
-
-	with_intel_pps_lock(intel_dp, wakeref)
+	with_intel_pps_lock(intel_dp)
 		intel_dp->pps.vlv_active_pipe = INVALID_PIPE;
 }
 
@@ -1362,10 +1342,9 @@ static void pps_vdd_init(struct intel_dp *intel_dp)
 
 bool intel_pps_have_panel_power_or_vdd(struct intel_dp *intel_dp)
 {
-	intel_wakeref_t wakeref;
 	bool have_power = false;
 
-	with_intel_pps_lock(intel_dp, wakeref) {
+	with_intel_pps_lock(intel_dp) {
 		have_power = edp_have_panel_power(intel_dp) ||
 			     edp_have_panel_vdd(intel_dp);
 	}
@@ -1692,12 +1671,11 @@ static void pps_init_registers(struct intel_dp *intel_dp, bool force_disable_vdd
 void intel_pps_encoder_reset(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
-	intel_wakeref_t wakeref;
 
 	if (!intel_dp_is_edp(intel_dp))
 		return;
 
-	with_intel_pps_lock(intel_dp, wakeref) {
+	with_intel_pps_lock(intel_dp) {
 		/*
 		 * Reinit the power sequencer also on the resume path, in case
 		 * BIOS did something nasty with it.
@@ -1716,7 +1694,6 @@ void intel_pps_encoder_reset(struct intel_dp *intel_dp)
 
 bool intel_pps_init(struct intel_dp *intel_dp)
 {
-	intel_wakeref_t wakeref;
 	bool ret;
 
 	intel_dp->pps.initializing = true;
@@ -1724,7 +1701,7 @@ bool intel_pps_init(struct intel_dp *intel_dp)
 
 	pps_init_timestamps(intel_dp);
 
-	with_intel_pps_lock(intel_dp, wakeref) {
+	with_intel_pps_lock(intel_dp) {
 		ret = pps_initial_setup(intel_dp);
 
 		pps_init_delays(intel_dp);
@@ -1760,9 +1737,7 @@ static void pps_init_late(struct intel_dp *intel_dp)
 
 void intel_pps_init_late(struct intel_dp *intel_dp)
 {
-	intel_wakeref_t wakeref;
-
-	with_intel_pps_lock(intel_dp, wakeref) {
+	with_intel_pps_lock(intel_dp) {
 		/* Reinit delays after per-panel info has been parsed from VBT */
 		pps_init_late(intel_dp);
 
diff --git a/drivers/gpu/drm/i915/display/intel_pps.h b/drivers/gpu/drm/i915/display/intel_pps.h
index c83007152f07..f7c96d75be45 100644
--- a/drivers/gpu/drm/i915/display/intel_pps.h
+++ b/drivers/gpu/drm/i915/display/intel_pps.h
@@ -8,20 +8,22 @@
 
 #include <linux/types.h>
 
-#include "intel_wakeref.h"
-
 enum pipe;
 struct intel_connector;
 struct intel_crtc_state;
 struct intel_display;
 struct intel_dp;
 struct intel_encoder;
+struct ref_tracker;
+
+struct ref_tracker *intel_pps_lock(struct intel_dp *intel_dp);
+struct ref_tracker *intel_pps_unlock(struct intel_dp *intel_dp, struct ref_tracker *wakeref);
 
-intel_wakeref_t intel_pps_lock(struct intel_dp *intel_dp);
-intel_wakeref_t intel_pps_unlock(struct intel_dp *intel_dp, intel_wakeref_t wakeref);
+#define __with_intel_pps_lock(dp, wf) \
+	for (struct ref_tracker *(wf) = intel_pps_lock(dp); (wf); (wf) = intel_pps_unlock((dp), (wf)))
 
-#define with_intel_pps_lock(dp, wf)						\
-	for ((wf) = intel_pps_lock(dp); (wf); (wf) = intel_pps_unlock((dp), (wf)))
+#define with_intel_pps_lock(dp) \
+	__with_intel_pps_lock((dp), __UNIQUE_ID(wakeref))
 
 void intel_pps_backlight_on(struct intel_dp *intel_dp);
 void intel_pps_backlight_off(struct intel_dp *intel_dp);
diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c
index 08bca4573974..91f4ac86c7ad 100644
--- a/drivers/gpu/drm/i915/display/intel_psr.c
+++ b/drivers/gpu/drm/i915/display/intel_psr.c
@@ -494,82 +494,37 @@ static u8 intel_dp_get_sink_sync_latency(struct intel_dp *intel_dp)
 	return val;
 }
 
-static u8 intel_dp_get_su_capability(struct intel_dp *intel_dp)
-{
-	u8 su_capability = 0;
-
-	if (intel_dp->psr.sink_panel_replay_su_support) {
-		if (drm_dp_dpcd_read_byte(&intel_dp->aux,
-					  DP_PANEL_REPLAY_CAP_CAPABILITY,
-					  &su_capability) < 0)
-			return 0;
-	} else {
-		su_capability = intel_dp->psr_dpcd[1];
-	}
-
-	return su_capability;
-}
-
-static unsigned int
-intel_dp_get_su_x_granularity_offset(struct intel_dp *intel_dp)
-{
-	return intel_dp->psr.sink_panel_replay_su_support ?
-		DP_PANEL_REPLAY_CAP_X_GRANULARITY :
-		DP_PSR2_SU_X_GRANULARITY;
-}
-
-static unsigned int
-intel_dp_get_su_y_granularity_offset(struct intel_dp *intel_dp)
-{
-	return intel_dp->psr.sink_panel_replay_su_support ?
-		DP_PANEL_REPLAY_CAP_Y_GRANULARITY :
-		DP_PSR2_SU_Y_GRANULARITY;
-}
-
-/*
- * Note: Bits related to granularity are same in panel replay and psr
- * registers. Rely on PSR definitions on these "common" bits.
- */
-static void intel_dp_get_su_granularity(struct intel_dp *intel_dp)
+static void _psr_compute_su_granularity(struct intel_dp *intel_dp,
+					struct intel_connector *connector)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	ssize_t r;
-	u16 w;
+	__le16 w;
 	u8 y;
 
-	/*
-	 * TODO: Do we need to take into account panel supporting both PSR and
-	 * Panel replay?
-	 */
-
 	/*
 	 * If sink don't have specific granularity requirements set legacy
 	 * ones.
 	 */
-	if (!(intel_dp_get_su_capability(intel_dp) &
-	      DP_PSR2_SU_GRANULARITY_REQUIRED)) {
+	if (!(connector->dp.psr_caps.dpcd[1] & DP_PSR2_SU_GRANULARITY_REQUIRED)) {
 		/* As PSR2 HW sends full lines, we do not care about x granularity */
-		w = 4;
+		w = cpu_to_le16(4);
 		y = 4;
 		goto exit;
 	}
 
-	r = drm_dp_dpcd_read(&intel_dp->aux,
-			     intel_dp_get_su_x_granularity_offset(intel_dp),
-			     &w, 2);
-	if (r != 2)
+	r = drm_dp_dpcd_read(&intel_dp->aux, DP_PSR2_SU_X_GRANULARITY, &w, sizeof(w));
+	if (r != sizeof(w))
 		drm_dbg_kms(display->drm,
 			    "Unable to read selective update x granularity\n");
 	/*
 	 * Spec says that if the value read is 0 the default granularity should
 	 * be used instead.
 	 */
-	if (r != 2 || w == 0)
-		w = 4;
+	if (r != sizeof(w) || w == 0)
+		w = cpu_to_le16(4);
 
-	r = drm_dp_dpcd_read(&intel_dp->aux,
-			     intel_dp_get_su_y_granularity_offset(intel_dp),
-			     &y, 1);
+	r = drm_dp_dpcd_read(&intel_dp->aux, DP_PSR2_SU_Y_GRANULARITY, &y, 1);
 	if (r != 1) {
 		drm_dbg_kms(display->drm,
 			    "Unable to read selective update y granularity\n");
@@ -579,17 +534,17 @@ static void intel_dp_get_su_granularity(struct intel_dp *intel_dp)
 		y = 1;
 
 exit:
-	intel_dp->psr.su_w_granularity = w;
-	intel_dp->psr.su_y_granularity = y;
+	connector->dp.psr_caps.su_w_granularity = le16_to_cpu(w);
+	connector->dp.psr_caps.su_y_granularity = y;
 }
 
 static enum intel_panel_replay_dsc_support
-compute_pr_dsc_support(struct intel_dp *intel_dp)
+compute_pr_dsc_support(struct intel_connector *connector)
 {
 	u8 pr_dsc_mode;
 	u8 val;
 
-	val = intel_dp->pr_dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_CAPABILITY)];
+	val = connector->dp.panel_replay_caps.dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_CAPABILITY)];
 	pr_dsc_mode = REG_FIELD_GET8(DP_PANEL_REPLAY_DSC_DECODE_CAPABILITY_IN_PR_MASK, val);
 
 	switch (pr_dsc_mode) {
@@ -621,7 +576,31 @@ static const char *panel_replay_dsc_support_str(enum intel_panel_replay_dsc_supp
 	};
 }
 
-static void _panel_replay_init_dpcd(struct intel_dp *intel_dp)
+static void _panel_replay_compute_su_granularity(struct intel_connector *connector)
+{
+	u16 w;
+	u8 y;
+
+	if (!(connector->dp.panel_replay_caps.dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_CAPABILITY)] &
+	       DP_PANEL_REPLAY_SU_GRANULARITY_REQUIRED)) {
+		w = 4;
+		y = 4;
+		goto exit;
+	}
+
+	/*
+	 * Spec says that if the value read is 0 the default granularity should
+	 * be used instead.
+	 */
+	w = le16_to_cpu(*(__le16 *)&connector->dp.panel_replay_caps.dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_X_GRANULARITY)]) ? : 4;
+	y = connector->dp.panel_replay_caps.dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_Y_GRANULARITY)] ? : 1;
+
+exit:
+	connector->dp.panel_replay_caps.su_w_granularity = w;
+	connector->dp.panel_replay_caps.su_y_granularity = y;
+}
+
+static void _panel_replay_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *connector)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	int ret;
@@ -631,11 +610,12 @@ static void _panel_replay_init_dpcd(struct intel_dp *intel_dp)
 		return;
 
 	ret = drm_dp_dpcd_read_data(&intel_dp->aux, DP_PANEL_REPLAY_CAP_SUPPORT,
-				    &intel_dp->pr_dpcd, sizeof(intel_dp->pr_dpcd));
+				    &connector->dp.panel_replay_caps.dpcd,
+				    sizeof(connector->dp.panel_replay_caps.dpcd));
 	if (ret < 0)
 		return;
 
-	if (!(intel_dp->pr_dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_SUPPORT)] &
+	if (!(connector->dp.panel_replay_caps.dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_SUPPORT)] &
 	      DP_PANEL_REPLAY_SUPPORT))
 		return;
 
@@ -646,7 +626,7 @@ static void _panel_replay_init_dpcd(struct intel_dp *intel_dp)
 			return;
 		}
 
-		if (!(intel_dp->pr_dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_SUPPORT)] &
+		if (!(connector->dp.panel_replay_caps.dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_SUPPORT)] &
 		      DP_PANEL_REPLAY_EARLY_TRANSPORT_SUPPORT)) {
 			drm_dbg_kms(display->drm,
 				    "Panel doesn't support early transport, eDP Panel Replay not possible\n");
@@ -654,36 +634,40 @@ static void _panel_replay_init_dpcd(struct intel_dp *intel_dp)
 		}
 	}
 
+	connector->dp.panel_replay_caps.support = true;
 	intel_dp->psr.sink_panel_replay_support = true;
 
-	if (intel_dp->pr_dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_SUPPORT)] &
-	    DP_PANEL_REPLAY_SU_SUPPORT)
-		intel_dp->psr.sink_panel_replay_su_support = true;
+	if (connector->dp.panel_replay_caps.dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_SUPPORT)] &
+	    DP_PANEL_REPLAY_SU_SUPPORT) {
+		connector->dp.panel_replay_caps.su_support = true;
+
+		_panel_replay_compute_su_granularity(connector);
+	}
 
-	intel_dp->psr.sink_panel_replay_dsc_support = compute_pr_dsc_support(intel_dp);
+	connector->dp.panel_replay_caps.dsc_support = compute_pr_dsc_support(connector);
 
 	drm_dbg_kms(display->drm,
 		    "Panel replay %sis supported by panel (in DSC mode: %s)\n",
-		    intel_dp->psr.sink_panel_replay_su_support ?
+		    connector->dp.panel_replay_caps.su_support ?
 		    "selective_update " : "",
-		    panel_replay_dsc_support_str(intel_dp->psr.sink_panel_replay_dsc_support));
+		    panel_replay_dsc_support_str(connector->dp.panel_replay_caps.dsc_support));
 }
 
-static void _psr_init_dpcd(struct intel_dp *intel_dp)
+static void _psr_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *connector)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	int ret;
 
-	ret = drm_dp_dpcd_read_data(&intel_dp->aux, DP_PSR_SUPPORT, intel_dp->psr_dpcd,
-				    sizeof(intel_dp->psr_dpcd));
+	ret = drm_dp_dpcd_read_data(&intel_dp->aux, DP_PSR_SUPPORT, connector->dp.psr_caps.dpcd,
+				    sizeof(connector->dp.psr_caps.dpcd));
 	if (ret < 0)
 		return;
 
-	if (!intel_dp->psr_dpcd[0])
+	if (!connector->dp.psr_caps.dpcd[0])
 		return;
 
 	drm_dbg_kms(display->drm, "eDP panel supports PSR version %x\n",
-		    intel_dp->psr_dpcd[0]);
+		    connector->dp.psr_caps.dpcd[0]);
 
 	if (drm_dp_has_quirk(&intel_dp->desc, DP_DPCD_QUIRK_NO_PSR)) {
 		drm_dbg_kms(display->drm,
@@ -697,13 +681,14 @@ static void _psr_init_dpcd(struct intel_dp *intel_dp)
 		return;
 	}
 
+	connector->dp.psr_caps.support = true;
 	intel_dp->psr.sink_support = true;
-	intel_dp->psr.sink_sync_latency =
-		intel_dp_get_sink_sync_latency(intel_dp);
+
+	connector->dp.psr_caps.sync_latency = intel_dp_get_sink_sync_latency(intel_dp);
 
 	if (DISPLAY_VER(display) >= 9 &&
-	    intel_dp->psr_dpcd[0] >= DP_PSR2_WITH_Y_COORD_IS_SUPPORTED) {
-		bool y_req = intel_dp->psr_dpcd[1] &
+	    connector->dp.psr_caps.dpcd[0] >= DP_PSR2_WITH_Y_COORD_IS_SUPPORTED) {
+		bool y_req = connector->dp.psr_caps.dpcd[1] &
 			     DP_PSR2_SU_Y_COORDINATE_REQUIRED;
 
 		/*
@@ -717,22 +702,21 @@ static void _psr_init_dpcd(struct intel_dp *intel_dp)
 		 * Y-coordinate requirement panels we would need to enable
 		 * GTC first.
 		 */
-		intel_dp->psr.sink_psr2_support = y_req &&
+		connector->dp.psr_caps.su_support = y_req &&
 			intel_alpm_aux_wake_supported(intel_dp);
 		drm_dbg_kms(display->drm, "PSR2 %ssupported\n",
-			    intel_dp->psr.sink_psr2_support ? "" : "not ");
+			    connector->dp.psr_caps.su_support ? "" : "not ");
 	}
+
+	if (connector->dp.psr_caps.su_support)
+		_psr_compute_su_granularity(intel_dp, connector);
 }
 
-void intel_psr_init_dpcd(struct intel_dp *intel_dp)
+void intel_psr_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *connector)
 {
-	_psr_init_dpcd(intel_dp);
+	_psr_init_dpcd(intel_dp, connector);
 
-	_panel_replay_init_dpcd(intel_dp);
-
-	if (intel_dp->psr.sink_psr2_support ||
-	    intel_dp->psr.sink_panel_replay_su_support)
-		intel_dp_get_su_granularity(intel_dp);
+	_panel_replay_init_dpcd(intel_dp, connector);
 }
 
 static void hsw_psr_setup_aux(struct intel_dp *intel_dp)
@@ -772,8 +756,9 @@ static void hsw_psr_setup_aux(struct intel_dp *intel_dp)
 		       aux_ctl);
 }
 
-static bool psr2_su_region_et_valid(struct intel_dp *intel_dp, bool panel_replay)
+static bool psr2_su_region_et_valid(struct intel_connector *connector, bool panel_replay)
 {
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
 	struct intel_display *display = to_intel_display(intel_dp);
 
 	if (DISPLAY_VER(display) < 20 || !intel_dp_is_edp(intel_dp) ||
@@ -781,9 +766,9 @@ static bool psr2_su_region_et_valid(struct intel_dp *intel_dp, bool panel_replay
 		return false;
 
 	return panel_replay ?
-		intel_dp->pr_dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_SUPPORT)] &
+		connector->dp.panel_replay_caps.dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_SUPPORT)] &
 		DP_PANEL_REPLAY_EARLY_TRANSPORT_SUPPORT :
-		intel_dp->psr_dpcd[0] == DP_PSR2_WITH_Y_COORD_ET_SUPPORTED;
+		connector->dp.psr_caps.dpcd[0] == DP_PSR2_WITH_Y_COORD_ET_SUPPORTED;
 }
 
 static void _panel_replay_enable_sink(struct intel_dp *intel_dp,
@@ -924,7 +909,7 @@ static u8 psr_compute_idle_frames(struct intel_dp *intel_dp)
 	 * off-by-one issue that HW has in some cases.
 	 */
 	idle_frames = max(6, connector->panel.vbt.psr.idle_frames);
-	idle_frames = max(idle_frames, intel_dp->psr.sink_sync_latency + 1);
+	idle_frames = max(idle_frames, connector->dp.psr_caps.sync_latency + 1);
 
 	if (drm_WARN_ON(display->drm, idle_frames > 0xf))
 		idle_frames = 0xf;
@@ -1019,10 +1004,11 @@ static int psr2_block_count(struct intel_dp *intel_dp)
 
 static u8 frames_before_su_entry(struct intel_dp *intel_dp)
 {
+	struct intel_connector *connector = intel_dp->attached_connector;
 	u8 frames_before_su_entry;
 
 	frames_before_su_entry = max_t(u8,
-				       intel_dp->psr.sink_sync_latency + 1,
+				       connector->dp.psr_caps.sync_latency + 1,
 				       2);
 
 	/* Entry setup frames must be at least 1 less than frames before SU entry */
@@ -1304,25 +1290,32 @@ static bool intel_psr2_sel_fetch_config_valid(struct intel_dp *intel_dp,
 	return crtc_state->enable_psr2_sel_fetch = true;
 }
 
-static bool psr2_granularity_check(struct intel_dp *intel_dp,
-				   struct intel_crtc_state *crtc_state)
+static bool psr2_granularity_check(struct intel_crtc_state *crtc_state,
+				   struct intel_connector *connector)
 {
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
 	struct intel_display *display = to_intel_display(intel_dp);
 	const struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config;
 	const int crtc_hdisplay = crtc_state->hw.adjusted_mode.crtc_hdisplay;
 	const int crtc_vdisplay = crtc_state->hw.adjusted_mode.crtc_vdisplay;
 	u16 y_granularity = 0;
+	u16 sink_y_granularity = crtc_state->has_panel_replay ?
+		connector->dp.panel_replay_caps.su_y_granularity :
+		connector->dp.psr_caps.su_y_granularity;
+	u16 sink_w_granularity =  crtc_state->has_panel_replay ?
+		connector->dp.panel_replay_caps.su_w_granularity :
+		connector->dp.psr_caps.su_w_granularity;
 
 	/* PSR2 HW only send full lines so we only need to validate the width */
-	if (crtc_hdisplay % intel_dp->psr.su_w_granularity)
+	if (crtc_hdisplay % sink_w_granularity)
 		return false;
 
-	if (crtc_vdisplay % intel_dp->psr.su_y_granularity)
+	if (crtc_vdisplay % sink_y_granularity)
 		return false;
 
 	/* HW tracking is only aligned to 4 lines */
 	if (!crtc_state->enable_psr2_sel_fetch)
-		return intel_dp->psr.su_y_granularity == 4;
+		return sink_y_granularity == 4;
 
 	/*
 	 * adl_p and mtl platforms have 1 line granularity.
@@ -1330,11 +1323,11 @@ static bool psr2_granularity_check(struct intel_dp *intel_dp,
 	 * to match sink requirement if multiple of 4.
 	 */
 	if (display->platform.alderlake_p || DISPLAY_VER(display) >= 14)
-		y_granularity = intel_dp->psr.su_y_granularity;
-	else if (intel_dp->psr.su_y_granularity <= 2)
+		y_granularity = sink_y_granularity;
+	else if (sink_y_granularity <= 2)
 		y_granularity = 4;
-	else if ((intel_dp->psr.su_y_granularity % 4) == 0)
-		y_granularity = intel_dp->psr.su_y_granularity;
+	else if ((sink_y_granularity % 4) == 0)
+		y_granularity = sink_y_granularity;
 
 	if (y_granularity == 0 || crtc_vdisplay % y_granularity)
 		return false;
@@ -1372,16 +1365,18 @@ static bool _compute_psr2_sdp_prior_scanline_indication(struct intel_dp *intel_d
 }
 
 static int intel_psr_entry_setup_frames(struct intel_dp *intel_dp,
+					struct drm_connector_state *conn_state,
 					const struct drm_display_mode *adjusted_mode)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
-	int psr_setup_time = drm_dp_psr_setup_time(intel_dp->psr_dpcd);
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
+	int psr_setup_time = drm_dp_psr_setup_time(connector->dp.psr_caps.dpcd);
 	int entry_setup_frames = 0;
 
 	if (psr_setup_time < 0) {
 		drm_dbg_kms(display->drm,
 			    "PSR condition failed: Invalid PSR setup time (0x%02x)\n",
-			    intel_dp->psr_dpcd[1]);
+			    connector->dp.psr_caps.dpcd[1]);
 		return -ETIME;
 	}
 
@@ -1522,14 +1517,16 @@ static bool alpm_config_valid(struct intel_dp *intel_dp,
 }
 
 static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
-				    struct intel_crtc_state *crtc_state)
+				    struct intel_crtc_state *crtc_state,
+				    struct drm_connector_state *conn_state)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	int crtc_hdisplay = crtc_state->hw.adjusted_mode.crtc_hdisplay;
 	int crtc_vdisplay = crtc_state->hw.adjusted_mode.crtc_vdisplay;
 	int psr_max_h = 0, psr_max_v = 0, max_bpp = 0;
 
-	if (!intel_dp->psr.sink_psr2_support || display->params.enable_psr == 1)
+	if (!connector->dp.psr_caps.su_support || display->params.enable_psr == 1)
 		return false;
 
 	/* JSL and EHL only supports eDP 1.3 */
@@ -1621,9 +1618,11 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
 	return true;
 }
 
-static bool intel_sel_update_config_valid(struct intel_dp *intel_dp,
-					  struct intel_crtc_state *crtc_state)
+static bool intel_sel_update_config_valid(struct intel_crtc_state *crtc_state,
+					  struct drm_connector_state *conn_state)
 {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
 	struct intel_display *display = to_intel_display(intel_dp);
 
 	if (HAS_PSR2_SEL_FETCH(display) &&
@@ -1640,7 +1639,8 @@ static bool intel_sel_update_config_valid(struct intel_dp *intel_dp,
 		goto unsupported;
 	}
 
-	if (!crtc_state->has_panel_replay && !intel_psr2_config_valid(intel_dp, crtc_state))
+	if (!crtc_state->has_panel_replay && !intel_psr2_config_valid(intel_dp, crtc_state,
+								      conn_state))
 		goto unsupported;
 
 	if (!_compute_psr2_sdp_prior_scanline_indication(intel_dp, crtc_state)) {
@@ -1653,11 +1653,11 @@ static bool intel_sel_update_config_valid(struct intel_dp *intel_dp,
 		if (DISPLAY_VER(display) < 14)
 			goto unsupported;
 
-		if (!intel_dp->psr.sink_panel_replay_su_support)
+		if (!connector->dp.panel_replay_caps.su_support)
 			goto unsupported;
 
 		if (intel_dsc_enabled_on_link(crtc_state) &&
-		    intel_dp->psr.sink_panel_replay_dsc_support !=
+		    connector->dp.panel_replay_caps.dsc_support !=
 		    INTEL_DP_PANEL_REPLAY_DSC_SELECTIVE_UPDATE) {
 			drm_dbg_kms(display->drm,
 				    "Selective update with Panel Replay not enabled because it's not supported with DSC\n");
@@ -1671,14 +1671,14 @@ static bool intel_sel_update_config_valid(struct intel_dp *intel_dp,
 		goto unsupported;
 	}
 
-	if (!psr2_granularity_check(intel_dp, crtc_state)) {
+	if (!psr2_granularity_check(crtc_state, connector)) {
 		drm_dbg_kms(display->drm,
 			    "Selective update not enabled, SU granularity not compatible\n");
 		goto unsupported;
 	}
 
-	crtc_state->enable_psr2_su_region_et =
-		psr2_su_region_et_valid(intel_dp, crtc_state->has_panel_replay);
+	crtc_state->enable_psr2_su_region_et = psr2_su_region_et_valid(connector,
+								       crtc_state->has_panel_replay);
 
 	return true;
 
@@ -1688,7 +1688,8 @@ static bool intel_sel_update_config_valid(struct intel_dp *intel_dp,
 }
 
 static bool _psr_compute_config(struct intel_dp *intel_dp,
-				struct intel_crtc_state *crtc_state)
+				struct intel_crtc_state *crtc_state,
+				struct drm_connector_state *conn_state)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
@@ -1703,7 +1704,7 @@ static bool _psr_compute_config(struct intel_dp *intel_dp,
 	if (crtc_state->vrr.enable)
 		return false;
 
-	entry_setup_frames = intel_psr_entry_setup_frames(intel_dp, adjusted_mode);
+	entry_setup_frames = intel_psr_entry_setup_frames(intel_dp, conn_state, adjusted_mode);
 
 	if (entry_setup_frames >= 0) {
 		intel_dp->psr.entry_setup_frames = entry_setup_frames;
@@ -1717,19 +1718,33 @@ static bool _psr_compute_config(struct intel_dp *intel_dp,
 	return true;
 }
 
-static bool
-_panel_replay_compute_config(struct intel_dp *intel_dp,
-			     struct intel_crtc_state *crtc_state,
-			     const struct drm_connector_state *conn_state)
+static inline bool compute_link_off_after_as_sdp_when_pr_active(struct intel_connector *connector)
+{
+	return (connector->dp.panel_replay_caps.dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_CAPABILITY)] &
+		DP_PANEL_REPLAY_LINK_OFF_SUPPORTED_IN_PR_AFTER_ADAPTIVE_SYNC_SDP);
+}
+
+static inline bool compute_disable_as_sdp_when_pr_active(struct intel_connector *connector)
+{
+	return !(connector->dp.panel_replay_caps.dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_CAPABILITY)] &
+		 DP_PANEL_REPLAY_ASYNC_VIDEO_TIMING_NOT_SUPPORTED_IN_PR);
+}
+
+static bool _panel_replay_compute_config(struct intel_crtc_state *crtc_state,
+					 const struct drm_connector_state *conn_state)
 {
-	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_connector *connector =
 		to_intel_connector(conn_state->connector);
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_hdcp *hdcp = &connector->hdcp;
 
 	if (!CAN_PANEL_REPLAY(intel_dp))
 		return false;
 
+	if (!connector->dp.panel_replay_caps.support)
+		return false;
+
 	if (!panel_replay_global_enabled(intel_dp)) {
 		drm_dbg_kms(display->drm, "Panel Replay disabled by flag\n");
 		return false;
@@ -1742,13 +1757,16 @@ _panel_replay_compute_config(struct intel_dp *intel_dp,
 	}
 
 	if (intel_dsc_enabled_on_link(crtc_state) &&
-	    intel_dp->psr.sink_panel_replay_dsc_support ==
+	    connector->dp.panel_replay_caps.dsc_support ==
 	    INTEL_DP_PANEL_REPLAY_DSC_NOT_SUPPORTED) {
 		drm_dbg_kms(display->drm,
 			    "Panel Replay not enabled because it's not supported with DSC\n");
 		return false;
 	}
 
+	crtc_state->link_off_after_as_sdp_when_pr_active = compute_link_off_after_as_sdp_when_pr_active(connector);
+	crtc_state->disable_as_sdp_when_pr_active = compute_disable_as_sdp_when_pr_active(connector);
+
 	if (!intel_dp_is_edp(intel_dp))
 		return true;
 
@@ -1824,6 +1842,7 @@ void intel_psr_compute_config(struct intel_dp *intel_dp,
 			      struct drm_connector_state *conn_state)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
 
 	if (!psr_global_enabled(intel_dp)) {
@@ -1855,18 +1874,16 @@ void intel_psr_compute_config(struct intel_dp *intel_dp,
 	}
 
 	/* Only used for state verification. */
-	crtc_state->panel_replay_dsc_support = intel_dp->psr.sink_panel_replay_dsc_support;
-	crtc_state->has_panel_replay = _panel_replay_compute_config(intel_dp,
-								    crtc_state,
-								    conn_state);
+	crtc_state->panel_replay_dsc_support = connector->dp.panel_replay_caps.dsc_support;
+	crtc_state->has_panel_replay = _panel_replay_compute_config(crtc_state, conn_state);
 
 	crtc_state->has_psr = crtc_state->has_panel_replay ? true :
-		_psr_compute_config(intel_dp, crtc_state);
+		_psr_compute_config(intel_dp, crtc_state, conn_state);
 
 	if (!crtc_state->has_psr)
 		return;
 
-	crtc_state->has_sel_update = intel_sel_update_config_valid(intel_dp, crtc_state);
+	crtc_state->has_sel_update = intel_sel_update_config_valid(crtc_state, conn_state);
 }
 
 void intel_psr_get_config(struct intel_encoder *encoder,
@@ -2701,7 +2718,7 @@ intel_psr2_sel_fetch_et_alignment(struct intel_atomic_state *state,
 	for_each_new_intel_plane_in_state(state, plane, new_plane_state, i) {
 		struct drm_rect inter;
 
-		if (new_plane_state->uapi.crtc != crtc_state->uapi.crtc)
+		if (new_plane_state->hw.crtc != crtc_state->uapi.crtc)
 			continue;
 
 		if (plane->id != PLANE_CURSOR)
@@ -2734,7 +2751,7 @@ static bool psr2_sel_fetch_plane_state_supported(const struct intel_plane_state
 	if (plane_state->uapi.dst.y1 < 0 ||
 	    plane_state->uapi.dst.x1 < 0 ||
 	    plane_state->scaler_id >= 0 ||
-	    plane_state->uapi.rotation != DRM_MODE_ROTATE_0)
+	    plane_state->hw.rotation != DRM_MODE_ROTATE_0)
 		return false;
 
 	return true;
@@ -2749,7 +2766,8 @@ static bool psr2_sel_fetch_plane_state_supported(const struct intel_plane_state
  */
 static bool psr2_sel_fetch_pipe_state_supported(const struct intel_crtc_state *crtc_state)
 {
-	if (crtc_state->scaler_state.scaler_id >= 0)
+	if (crtc_state->scaler_state.scaler_id >= 0 ||
+	    crtc_state->async_flip_planes)
 		return false;
 
 	return true;
@@ -2838,7 +2856,7 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state,
 		struct drm_rect src, damaged_area = { .x1 = 0, .y1 = -1,
 						      .x2 = INT_MAX };
 
-		if (new_plane_state->uapi.crtc != crtc_state->uapi.crtc)
+		if (new_plane_state->hw.crtc != crtc_state->uapi.crtc)
 			continue;
 
 		if (!new_plane_state->uapi.visible &&
@@ -2937,7 +2955,7 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state,
 		struct drm_rect *sel_fetch_area, inter;
 		struct intel_plane *linked = new_plane_state->planar_linked_plane;
 
-		if (new_plane_state->uapi.crtc != crtc_state->uapi.crtc ||
+		if (new_plane_state->hw.crtc != crtc_state->uapi.crtc ||
 		    !new_plane_state->uapi.visible)
 			continue;
 
@@ -2992,9 +3010,9 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state,
 	return 0;
 }
 
-void intel_psr2_panic_force_full_update(struct intel_display *display,
-					struct intel_crtc_state *crtc_state)
+void intel_psr2_panic_force_full_update(const struct intel_crtc_state *crtc_state)
 {
+	struct intel_display *display = to_intel_display(crtc_state);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 	u32 val = man_trk_ctl_enable_bit_get(display);
@@ -4109,24 +4127,22 @@ psr_source_status(struct intel_dp *intel_dp, struct seq_file *m)
 	seq_printf(m, "Source PSR/PanelReplay status: %s [0x%08x]\n", status, val);
 }
 
-static void intel_psr_sink_capability(struct intel_dp *intel_dp,
+static void intel_psr_sink_capability(struct intel_connector *connector,
 				      struct seq_file *m)
 {
-	struct intel_psr *psr = &intel_dp->psr;
-
 	seq_printf(m, "Sink support: PSR = %s",
-		   str_yes_no(psr->sink_support));
+		   str_yes_no(connector->dp.psr_caps.support));
 
-	if (psr->sink_support)
-		seq_printf(m, " [0x%02x]", intel_dp->psr_dpcd[0]);
-	if (intel_dp->psr_dpcd[0] == DP_PSR2_WITH_Y_COORD_ET_SUPPORTED)
+	if (connector->dp.psr_caps.support)
+		seq_printf(m, " [0x%02x]", connector->dp.psr_caps.dpcd[0]);
+	if (connector->dp.psr_caps.dpcd[0] == DP_PSR2_WITH_Y_COORD_ET_SUPPORTED)
 		seq_printf(m, " (Early Transport)");
-	seq_printf(m, ", Panel Replay = %s", str_yes_no(psr->sink_panel_replay_support));
+	seq_printf(m, ", Panel Replay = %s", str_yes_no(connector->dp.panel_replay_caps.support));
 	seq_printf(m, ", Panel Replay Selective Update = %s",
-		   str_yes_no(psr->sink_panel_replay_su_support));
+		   str_yes_no(connector->dp.panel_replay_caps.su_support));
 	seq_printf(m, ", Panel Replay DSC support = %s",
-		   panel_replay_dsc_support_str(psr->sink_panel_replay_dsc_support));
-	if (intel_dp->pr_dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_SUPPORT)] &
+		   panel_replay_dsc_support_str(connector->dp.panel_replay_caps.dsc_support));
+	if (connector->dp.panel_replay_caps.dpcd[INTEL_PR_DPCD_INDEX(DP_PANEL_REPLAY_CAP_SUPPORT)] &
 	    DP_PANEL_REPLAY_EARLY_TRANSPORT_SUPPORT)
 		seq_printf(m, " (Early Transport)");
 	seq_printf(m, "\n");
@@ -4164,7 +4180,8 @@ static void intel_psr_print_mode(struct intel_dp *intel_dp,
 		seq_printf(m, "  %s\n", psr->no_psr_reason);
 }
 
-static int intel_psr_status(struct seq_file *m, struct intel_dp *intel_dp)
+static int intel_psr_status(struct seq_file *m, struct intel_dp *intel_dp,
+			    struct intel_connector *connector)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
@@ -4173,9 +4190,9 @@ static int intel_psr_status(struct seq_file *m, struct intel_dp *intel_dp)
 	bool enabled;
 	u32 val, psr2_ctl;
 
-	intel_psr_sink_capability(intel_dp, m);
+	intel_psr_sink_capability(connector, m);
 
-	if (!(psr->sink_support || psr->sink_panel_replay_support))
+	if (!(connector->dp.psr_caps.support || connector->dp.panel_replay_caps.support))
 		return 0;
 
 	wakeref = intel_display_rpm_get(display);
@@ -4289,7 +4306,7 @@ static int i915_edp_psr_status_show(struct seq_file *m, void *data)
 	if (!intel_dp)
 		return -ENODEV;
 
-	return intel_psr_status(m, intel_dp);
+	return intel_psr_status(m, intel_dp, intel_dp->attached_connector);
 }
 DEFINE_SHOW_ATTRIBUTE(i915_edp_psr_status);
 
@@ -4423,7 +4440,7 @@ static int i915_psr_status_show(struct seq_file *m, void *data)
 	struct intel_connector *connector = m->private;
 	struct intel_dp *intel_dp = intel_attached_dp(connector);
 
-	return intel_psr_status(m, intel_dp);
+	return intel_psr_status(m, intel_dp, connector);
 }
 DEFINE_SHOW_ATTRIBUTE(i915_psr_status);
 
diff --git a/drivers/gpu/drm/i915/display/intel_psr.h b/drivers/gpu/drm/i915/display/intel_psr.h
index 620b35928832..b41dc4d44ff2 100644
--- a/drivers/gpu/drm/i915/display/intel_psr.h
+++ b/drivers/gpu/drm/i915/display/intel_psr.h
@@ -28,7 +28,7 @@ struct intel_plane_state;
 bool intel_encoder_can_psr(struct intel_encoder *encoder);
 bool intel_psr_needs_aux_io_power(struct intel_encoder *encoder,
 				  const struct intel_crtc_state *crtc_state);
-void intel_psr_init_dpcd(struct intel_dp *intel_dp);
+void intel_psr_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *connector);
 void intel_psr_panel_replay_enable_sink(struct intel_dp *intel_dp);
 void intel_psr_pre_plane_update(struct intel_atomic_state *state,
 				struct intel_crtc *crtc);
@@ -59,8 +59,7 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state,
 				struct intel_crtc *crtc);
 void intel_psr2_program_trans_man_trk_ctl(struct intel_dsb *dsb,
 					  const struct intel_crtc_state *crtc_state);
-void intel_psr2_panic_force_full_update(struct intel_display *display,
-					struct intel_crtc_state *crtc_state);
+void intel_psr2_panic_force_full_update(const struct intel_crtc_state *crtc_state);
 void intel_psr_pause(struct intel_dp *intel_dp);
 void intel_psr_resume(struct intel_dp *intel_dp);
 bool intel_psr_needs_vblank_notification(const struct intel_crtc_state *crtc_state);
diff --git a/drivers/gpu/drm/i915/display/intel_rom.c b/drivers/gpu/drm/i915/display/intel_rom.c
new file mode 100644
index 000000000000..2f17dc856e7f
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_rom.c
@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#include "i915_drv.h"
+#include "i915_reg.h"
+
+#include "intel_rom.h"
+#include "intel_uncore.h"
+
+struct intel_rom {
+	/* for PCI ROM */
+	struct pci_dev *pdev;
+	void __iomem *oprom;
+
+	/* for SPI */
+	struct intel_uncore *uncore;
+	loff_t offset;
+
+	size_t size;
+
+	u32 (*read32)(struct intel_rom *rom, loff_t offset);
+	u16 (*read16)(struct intel_rom *rom, loff_t offset);
+	void (*read_block)(struct intel_rom *rom, void *data, loff_t offset, size_t size);
+	void (*free)(struct intel_rom *rom);
+};
+
+static u32 spi_read32(struct intel_rom *rom, loff_t offset)
+{
+	intel_uncore_write(rom->uncore, PRIMARY_SPI_ADDRESS,
+			   rom->offset + offset);
+
+	return intel_uncore_read(rom->uncore, PRIMARY_SPI_TRIGGER);
+}
+
+static u16 spi_read16(struct intel_rom *rom, loff_t offset)
+{
+	return spi_read32(rom, offset) & 0xffff;
+}
+
+struct intel_rom *intel_rom_spi(struct drm_device *drm)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+	struct intel_rom *rom;
+	u32 static_region;
+
+	rom = kzalloc(sizeof(*rom), GFP_KERNEL);
+	if (!rom)
+		return NULL;
+
+	rom->uncore = &i915->uncore;
+
+	static_region = intel_uncore_read(rom->uncore, SPI_STATIC_REGIONS);
+	static_region &= OPTIONROM_SPI_REGIONID_MASK;
+	intel_uncore_write(rom->uncore, PRIMARY_SPI_REGIONID, static_region);
+
+	rom->offset = intel_uncore_read(rom->uncore, OROM_OFFSET) & OROM_OFFSET_MASK;
+
+	rom->size = 0x200000;
+
+	rom->read32 = spi_read32;
+	rom->read16 = spi_read16;
+
+	return rom;
+}
+
+static u32 pci_read32(struct intel_rom *rom, loff_t offset)
+{
+	return ioread32(rom->oprom + offset);
+}
+
+static u16 pci_read16(struct intel_rom *rom, loff_t offset)
+{
+	return ioread16(rom->oprom + offset);
+}
+
+static void pci_read_block(struct intel_rom *rom, void *data,
+			   loff_t offset, size_t size)
+{
+	memcpy_fromio(data, rom->oprom + offset, size);
+}
+
+static void pci_free(struct intel_rom *rom)
+{
+	pci_unmap_rom(rom->pdev, rom->oprom);
+}
+
+struct intel_rom *intel_rom_pci(struct drm_device *drm)
+{
+	struct intel_rom *rom;
+
+	rom = kzalloc(sizeof(*rom), GFP_KERNEL);
+	if (!rom)
+		return NULL;
+
+	rom->pdev = to_pci_dev(drm->dev);
+
+	rom->oprom = pci_map_rom(rom->pdev, &rom->size);
+	if (!rom->oprom) {
+		kfree(rom);
+		return NULL;
+	}
+
+	rom->read32 = pci_read32;
+	rom->read16 = pci_read16;
+	rom->read_block = pci_read_block;
+	rom->free = pci_free;
+
+	return rom;
+}
+
+u32 intel_rom_read32(struct intel_rom *rom, loff_t offset)
+{
+	return rom->read32(rom, offset);
+}
+
+u16 intel_rom_read16(struct intel_rom *rom, loff_t offset)
+{
+	return rom->read16(rom, offset);
+}
+
+void intel_rom_read_block(struct intel_rom *rom, void *data,
+			  loff_t offset, size_t size)
+{
+	u32 *ptr = data;
+	loff_t index;
+
+	if (rom->read_block) {
+		rom->read_block(rom, data, offset, size);
+		return;
+	}
+
+	for (index = 0; index < size; index += 4)
+		*ptr++ = rom->read32(rom, offset + index);
+}
+
+loff_t intel_rom_find(struct intel_rom *rom, u32 needle)
+{
+	loff_t offset;
+
+	for (offset = 0; offset < rom->size; offset += 4) {
+		if (rom->read32(rom, offset) == needle)
+			return offset;
+	}
+
+	return -ENOENT;
+}
+
+size_t intel_rom_size(struct intel_rom *rom)
+{
+	return rom->size;
+}
+
+void intel_rom_free(struct intel_rom *rom)
+{
+	if (rom && rom->free)
+		rom->free(rom);
+
+	kfree(rom);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_rom.h b/drivers/gpu/drm/i915/display/intel_rom.h
new file mode 100644
index 000000000000..4e59a375787e
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_rom.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#ifndef __INTEL_ROM_H__
+#define __INTEL_ROM_H__
+
+#include <linux/types.h>
+
+struct drm_device;
+struct intel_rom;
+
+struct intel_rom *intel_rom_spi(struct drm_device *drm);
+struct intel_rom *intel_rom_pci(struct drm_device *drm);
+
+u32 intel_rom_read32(struct intel_rom *rom, loff_t offset);
+u16 intel_rom_read16(struct intel_rom *rom, loff_t offset);
+void intel_rom_read_block(struct intel_rom *rom, void *data,
+			  loff_t offset, size_t size);
+loff_t intel_rom_find(struct intel_rom *rom, u32 needle);
+size_t intel_rom_size(struct intel_rom *rom);
+void intel_rom_free(struct intel_rom *rom);
+
+#endif /* __INTEL_ROM_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_sprite.c b/drivers/gpu/drm/i915/display/intel_sprite.c
index 69b6873a6044..6a65f92e8a03 100644
--- a/drivers/gpu/drm/i915/display/intel_sprite.c
+++ b/drivers/gpu/drm/i915/display/intel_sprite.c
@@ -462,7 +462,7 @@ vlv_sprite_get_hw_state(struct intel_plane *plane,
 	struct intel_display *display = to_intel_display(plane);
 	enum intel_display_power_domain power_domain;
 	enum plane_id plane_id = plane->id;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret;
 
 	power_domain = POWER_DOMAIN_PIPE(plane->pipe);
@@ -893,7 +893,7 @@ ivb_sprite_get_hw_state(struct intel_plane *plane,
 {
 	struct intel_display *display = to_intel_display(plane);
 	enum intel_display_power_domain power_domain;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret;
 
 	power_domain = POWER_DOMAIN_PIPE(plane->pipe);
@@ -1233,7 +1233,7 @@ g4x_sprite_get_hw_state(struct intel_plane *plane,
 {
 	struct intel_display *display = to_intel_display(plane);
 	enum intel_display_power_domain power_domain;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret;
 
 	power_domain = POWER_DOMAIN_PIPE(plane->pipe);
@@ -1567,6 +1567,7 @@ static const struct drm_plane_funcs g4x_sprite_funcs = {
 	.atomic_duplicate_state = intel_plane_duplicate_state,
 	.atomic_destroy_state = intel_plane_destroy_state,
 	.format_mod_supported = g4x_sprite_format_mod_supported,
+	.format_mod_supported_async = intel_plane_format_mod_supported_async,
 };
 
 static const struct drm_plane_funcs snb_sprite_funcs = {
@@ -1576,6 +1577,7 @@ static const struct drm_plane_funcs snb_sprite_funcs = {
 	.atomic_duplicate_state = intel_plane_duplicate_state,
 	.atomic_destroy_state = intel_plane_destroy_state,
 	.format_mod_supported = snb_sprite_format_mod_supported,
+	.format_mod_supported_async = intel_plane_format_mod_supported_async,
 };
 
 static const struct drm_plane_funcs vlv_sprite_funcs = {
@@ -1585,6 +1587,7 @@ static const struct drm_plane_funcs vlv_sprite_funcs = {
 	.atomic_duplicate_state = intel_plane_duplicate_state,
 	.atomic_destroy_state = intel_plane_destroy_state,
 	.format_mod_supported = vlv_sprite_format_mod_supported,
+	.format_mod_supported_async = intel_plane_format_mod_supported_async,
 };
 
 struct intel_plane *
diff --git a/drivers/gpu/drm/i915/display/intel_tc.c b/drivers/gpu/drm/i915/display/intel_tc.c
index 1e21fd02685d..064f572bbc85 100644
--- a/drivers/gpu/drm/i915/display/intel_tc.c
+++ b/drivers/gpu/drm/i915/display/intel_tc.c
@@ -51,7 +51,7 @@ struct intel_tc_port {
 	const struct intel_tc_phy_ops *phy_ops;
 
 	struct mutex lock;	/* protects the TypeC port mode */
-	intel_wakeref_t lock_wakeref;
+	struct ref_tracker *lock_wakeref;
 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
 	enum intel_display_power_domain lock_power_domain;
 #endif
@@ -182,7 +182,7 @@ bool intel_tc_cold_requires_aux_pw(struct intel_digital_port *dig_port)
 	       intel_display_power_legacy_aux_domain(display, dig_port->aux_ch);
 }
 
-static intel_wakeref_t
+static struct ref_tracker *
 __tc_cold_block(struct intel_tc_port *tc, enum intel_display_power_domain *domain)
 {
 	struct intel_display *display = to_intel_display(tc->dig_port);
@@ -192,11 +192,11 @@ __tc_cold_block(struct intel_tc_port *tc, enum intel_display_power_domain *domai
 	return intel_display_power_get(display, *domain);
 }
 
-static intel_wakeref_t
+static struct ref_tracker *
 tc_cold_block(struct intel_tc_port *tc)
 {
 	enum intel_display_power_domain domain;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 
 	wakeref = __tc_cold_block(tc, &domain);
 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
@@ -207,7 +207,7 @@ tc_cold_block(struct intel_tc_port *tc)
 
 static void
 __tc_cold_unblock(struct intel_tc_port *tc, enum intel_display_power_domain domain,
-		  intel_wakeref_t wakeref)
+		  struct ref_tracker *wakeref)
 {
 	struct intel_display *display = to_intel_display(tc->dig_port);
 
@@ -215,7 +215,7 @@ __tc_cold_unblock(struct intel_tc_port *tc, enum intel_display_power_domain doma
 }
 
 static void
-tc_cold_unblock(struct intel_tc_port *tc, intel_wakeref_t wakeref)
+tc_cold_unblock(struct intel_tc_port *tc, struct ref_tracker *wakeref)
 {
 	struct intel_display __maybe_unused *display = to_intel_display(tc->dig_port);
 	enum intel_display_power_domain domain = tc_phy_cold_off_domain(tc);
@@ -269,10 +269,9 @@ assert_tc_port_power_enabled(struct intel_tc_port *tc)
 static u32 get_lane_mask(struct intel_tc_port *tc)
 {
 	struct intel_display *display = to_intel_display(tc->dig_port);
-	intel_wakeref_t wakeref;
 	u32 lane_mask;
 
-	with_intel_display_power(display, POWER_DOMAIN_DISPLAY_CORE, wakeref)
+	with_intel_display_power(display, POWER_DOMAIN_DISPLAY_CORE)
 		lane_mask = intel_de_read(display, PORT_TX_DFLEXDPSP(tc->phy_fia));
 
 	drm_WARN_ON(display->drm, lane_mask == 0xffffffff);
@@ -296,7 +295,6 @@ get_pin_assignment(struct intel_tc_port *tc)
 	struct intel_display *display = to_intel_display(tc->dig_port);
 	enum tc_port tc_port = intel_encoder_to_tc(&tc->dig_port->base);
 	enum intel_tc_pin_assignment pin_assignment;
-	intel_wakeref_t wakeref;
 	i915_reg_t reg;
 	u32 mask;
 	u32 val;
@@ -312,7 +310,7 @@ get_pin_assignment(struct intel_tc_port *tc)
 		mask = DP_PIN_ASSIGNMENT_MASK(tc->phy_fia_idx);
 	}
 
-	with_intel_display_power(display, POWER_DOMAIN_DISPLAY_CORE, wakeref)
+	with_intel_display_power(display, POWER_DOMAIN_DISPLAY_CORE)
 		val = intel_de_read(display, reg);
 
 	drm_WARN_ON(display->drm, val == 0xffffffff);
@@ -527,12 +525,11 @@ static u32 icl_tc_phy_hpd_live_status(struct intel_tc_port *tc)
 	struct intel_display *display = to_intel_display(tc->dig_port);
 	struct intel_digital_port *dig_port = tc->dig_port;
 	u32 isr_bit = display->hotplug.pch_hpd[dig_port->base.hpd_pin];
-	intel_wakeref_t wakeref;
 	u32 fia_isr;
 	u32 pch_isr;
 	u32 mask = 0;
 
-	with_intel_display_power(display, tc_phy_cold_off_domain(tc), wakeref) {
+	with_intel_display_power(display, tc_phy_cold_off_domain(tc)) {
 		fia_isr = intel_de_read(display, PORT_TX_DFLEXDPSP(tc->phy_fia));
 		pch_isr = intel_de_read(display, SDEISR);
 	}
@@ -628,7 +625,7 @@ static bool icl_tc_phy_is_owned(struct intel_tc_port *tc)
 static void icl_tc_phy_get_hw_state(struct intel_tc_port *tc)
 {
 	enum intel_display_power_domain domain;
-	intel_wakeref_t tc_cold_wref;
+	struct ref_tracker *tc_cold_wref;
 
 	tc_cold_wref = __tc_cold_block(tc, &domain);
 
@@ -774,10 +771,9 @@ tgl_tc_phy_cold_off_domain(struct intel_tc_port *tc)
 static void tgl_tc_phy_init(struct intel_tc_port *tc)
 {
 	struct intel_display *display = to_intel_display(tc->dig_port);
-	intel_wakeref_t wakeref;
 	u32 val;
 
-	with_intel_display_power(display, tc_phy_cold_off_domain(tc), wakeref)
+	with_intel_display_power(display, tc_phy_cold_off_domain(tc))
 		val = intel_de_read(display, PORT_TX_DFLEXDPSP(FIA1));
 
 	drm_WARN_ON(display->drm, val == 0xffffffff);
@@ -819,12 +815,11 @@ static u32 adlp_tc_phy_hpd_live_status(struct intel_tc_port *tc)
 	enum hpd_pin hpd_pin = dig_port->base.hpd_pin;
 	u32 cpu_isr_bits = display->hotplug.hpd[hpd_pin];
 	u32 pch_isr_bit = display->hotplug.pch_hpd[hpd_pin];
-	intel_wakeref_t wakeref;
 	u32 cpu_isr;
 	u32 pch_isr;
 	u32 mask = 0;
 
-	with_intel_display_power(display, POWER_DOMAIN_DISPLAY_CORE, wakeref) {
+	with_intel_display_power(display, POWER_DOMAIN_DISPLAY_CORE) {
 		cpu_isr = intel_de_read(display, GEN11_DE_HPD_ISR);
 		pch_isr = intel_de_read(display, SDEISR);
 	}
@@ -897,7 +892,7 @@ static void adlp_tc_phy_get_hw_state(struct intel_tc_port *tc)
 	struct intel_display *display = to_intel_display(tc->dig_port);
 	enum intel_display_power_domain port_power_domain =
 		tc_port_power_domain(tc);
-	intel_wakeref_t port_wakeref;
+	struct ref_tracker *port_wakeref;
 
 	port_wakeref = intel_display_power_get(display, port_power_domain);
 
@@ -916,7 +911,7 @@ static bool adlp_tc_phy_connect(struct intel_tc_port *tc, int required_lanes)
 	struct intel_display *display = to_intel_display(tc->dig_port);
 	enum intel_display_power_domain port_power_domain =
 		tc_port_power_domain(tc);
-	intel_wakeref_t port_wakeref;
+	struct ref_tracker *port_wakeref;
 
 	if (tc->mode == TC_PORT_TBT_ALT) {
 		tc->lock_wakeref = tc_cold_block(tc);
@@ -968,7 +963,7 @@ static void adlp_tc_phy_disconnect(struct intel_tc_port *tc)
 	struct intel_display *display = to_intel_display(tc->dig_port);
 	enum intel_display_power_domain port_power_domain =
 		tc_port_power_domain(tc);
-	intel_wakeref_t port_wakeref;
+	struct ref_tracker *port_wakeref;
 
 	port_wakeref = intel_display_power_get(display, port_power_domain);
 
@@ -1015,12 +1010,11 @@ static u32 xelpdp_tc_phy_hpd_live_status(struct intel_tc_port *tc)
 	enum hpd_pin hpd_pin = dig_port->base.hpd_pin;
 	u32 pica_isr_bits = display->hotplug.hpd[hpd_pin];
 	u32 pch_isr_bit = display->hotplug.pch_hpd[hpd_pin];
-	intel_wakeref_t wakeref;
 	u32 pica_isr;
 	u32 pch_isr;
 	u32 mask = 0;
 
-	with_intel_display_power(display, POWER_DOMAIN_DISPLAY_CORE, wakeref) {
+	with_intel_display_power(display, POWER_DOMAIN_DISPLAY_CORE) {
 		pica_isr = intel_de_read(display, PICAINTERRUPT_ISR);
 		pch_isr = intel_de_read(display, SDEISR);
 	}
@@ -1175,7 +1169,7 @@ static bool xelpdp_tc_phy_is_owned(struct intel_tc_port *tc)
 static void xelpdp_tc_phy_get_hw_state(struct intel_tc_port *tc)
 {
 	struct intel_display *display = to_intel_display(tc->dig_port);
-	intel_wakeref_t tc_cold_wref;
+	struct ref_tracker *tc_cold_wref;
 	enum intel_display_power_domain domain;
 
 	tc_cold_wref = __tc_cold_block(tc, &domain);
diff --git a/drivers/gpu/drm/i915/display/intel_vbt_defs.h b/drivers/gpu/drm/i915/display/intel_vbt_defs.h
index 70e31520c560..57fda5824c9c 100644
--- a/drivers/gpu/drm/i915/display/intel_vbt_defs.h
+++ b/drivers/gpu/drm/i915/display/intel_vbt_defs.h
@@ -554,7 +554,8 @@ struct child_device_config {
 	u8 dvo_function;
 	u8 dp_usb_type_c:1;					/* 195+ */
 	u8 tbt:1;						/* 209+ */
-	u8 flags2_reserved:2;					/* 195+ */
+	u8 dedicated_external:1;				/* 264+ */
+	u8 dyn_port_over_tc:1;					/* 264+ */
 	u8 dp_port_trace_length:4;				/* 209+ */
 	u8 dp_gpio_index;					/* 195+ */
 	u16 dp_gpio_pin_num;					/* 195+ */
diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.c b/drivers/gpu/drm/i915/display/intel_vdsc.c
index 0e727fc5e80c..ad5fe841e4b3 100644
--- a/drivers/gpu/drm/i915/display/intel_vdsc.c
+++ b/drivers/gpu/drm/i915/display/intel_vdsc.c
@@ -999,7 +999,7 @@ void intel_dsc_get_config(struct intel_crtc_state *crtc_state)
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 	enum intel_display_power_domain power_domain;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	u32 dss_ctl1, dss_ctl2;
 
 	if (!intel_dsc_source_support(crtc_state))
diff --git a/drivers/gpu/drm/i915/display/intel_vga.c b/drivers/gpu/drm/i915/display/intel_vga.c
index 6e125564db34..c45c4bbc3f95 100644
--- a/drivers/gpu/drm/i915/display/intel_vga.c
+++ b/drivers/gpu/drm/i915/display/intel_vga.c
@@ -9,12 +9,12 @@
 
 #include <drm/drm_device.h>
 #include <drm/drm_print.h>
+#include <drm/intel/i915_drm.h>
 #include <video/vga.h>
 
-#include "soc/intel_gmch.h"
-
 #include "intel_de.h"
 #include "intel_display.h"
+#include "intel_display_types.h"
 #include "intel_vga.h"
 #include "intel_vga_regs.h"
 
@@ -95,6 +95,46 @@ void intel_vga_reset_io_mem(struct intel_display *display)
 	vga_put(pdev, VGA_RSRC_LEGACY_IO);
 }
 
+static int intel_gmch_vga_set_state(struct intel_display *display, bool enable_decode)
+{
+	struct pci_dev *pdev = to_pci_dev(display->drm->dev);
+	unsigned int reg = DISPLAY_VER(display) >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
+	u16 gmch_ctrl;
+
+	if (pci_bus_read_config_word(pdev->bus, PCI_DEVFN(0, 0), reg, &gmch_ctrl)) {
+		drm_err(display->drm, "failed to read control word\n");
+		return -EIO;
+	}
+
+	if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !enable_decode)
+		return 0;
+
+	if (enable_decode)
+		gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
+	else
+		gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
+
+	if (pci_bus_write_config_word(pdev->bus, PCI_DEVFN(0, 0), reg, gmch_ctrl)) {
+		drm_err(display->drm, "failed to write control word\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static unsigned int intel_gmch_vga_set_decode(struct pci_dev *pdev, bool enable_decode)
+{
+	struct intel_display *display = to_intel_display(pdev);
+
+	intel_gmch_vga_set_state(display, enable_decode);
+
+	if (enable_decode)
+		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
+		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
+	else
+		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
+}
+
 int intel_vga_register(struct intel_display *display)
 {
 
diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c
index ee8e24497d2c..b3d41705448a 100644
--- a/drivers/gpu/drm/i915/display/skl_universal_plane.c
+++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c
@@ -9,7 +9,6 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_print.h>
 
-#include "pxp/intel_pxp.h"
 #include "intel_bo.h"
 #include "intel_color.h"
 #include "intel_color_pipeline.h"
@@ -22,7 +21,7 @@
 #include "intel_fb.h"
 #include "intel_fbc.h"
 #include "intel_frontbuffer.h"
-#include "intel_panic.h"
+#include "intel_parent.h"
 #include "intel_plane.h"
 #include "intel_psr.h"
 #include "intel_psr_regs.h"
@@ -597,7 +596,7 @@ static u32 tgl_plane_min_alignment(struct intel_plane *plane,
 	 * Figure out what's going on here...
 	 */
 	if (display->platform.alderlake_p &&
-	    intel_plane_can_async_flip(plane, fb->format->format, fb->modifier))
+	    intel_plane_can_async_flip(plane, fb->format, fb->modifier))
 		return mult * 16 * 1024;
 
 	switch (fb->modifier) {
@@ -941,7 +940,7 @@ skl_plane_get_hw_state(struct intel_plane *plane,
 	struct intel_display *display = to_intel_display(plane);
 	enum intel_display_power_domain power_domain;
 	enum plane_id plane_id = plane->id;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	bool ret;
 
 	power_domain = POWER_DOMAIN_PIPE(plane->pipe);
@@ -1603,7 +1602,7 @@ icl_plane_update_noarm(struct intel_dsb *dsb,
 	}
 
 	/* FLAT CCS doesn't need to program AUX_DIST */
-	if (HAS_AUX_CCS(display))
+	if (HAS_AUX_DIST(display))
 		intel_de_write_dsb(display, dsb, PLANE_AUX_DIST(pipe, plane_id),
 				   skl_plane_aux_dist(plane_state, color_plane));
 
@@ -2308,7 +2307,7 @@ static void check_protection(struct intel_plane_state *plane_state)
 	if (DISPLAY_VER(display) < 11)
 		return;
 
-	plane_state->decrypt = intel_pxp_key_check(obj, false) == 0;
+	plane_state->decrypt = intel_bo_key_check(obj) == 0;
 	plane_state->force_black = intel_bo_is_protected(obj) &&
 		!plane_state->decrypt;
 }
@@ -2462,7 +2461,7 @@ static struct intel_fbc *skl_plane_fbc(struct intel_display *display,
 	enum intel_fbc_id fbc_id = skl_fbc_id_for_pipe(pipe);
 
 	if (skl_plane_has_fbc(display, fbc_id, plane_id))
-		return display->fbc[fbc_id];
+		return display->fbc.instances[fbc_id];
 	else
 		return NULL;
 }
@@ -2973,12 +2972,6 @@ skl_universal_plane_create(struct intel_display *display,
 	else
 		caps = skl_plane_caps(display, pipe, plane_id);
 
-	/* FIXME: xe has problems with AUX */
-	if (!IS_ENABLED(I915) && HAS_AUX_CCS(display))
-		caps &= ~(INTEL_PLANE_CAP_CCS_RC |
-			  INTEL_PLANE_CAP_CCS_RC_CC |
-			  INTEL_PLANE_CAP_CCS_MC);
-
 	modifiers = intel_fb_plane_get_modifiers(display, caps);
 
 	ret = drm_universal_plane_init(display->drm, &plane->base,
diff --git a/drivers/gpu/drm/i915/display/skl_watermark.c b/drivers/gpu/drm/i915/display/skl_watermark.c
index 54e9e0be019d..a6aab79812e5 100644
--- a/drivers/gpu/drm/i915/display/skl_watermark.c
+++ b/drivers/gpu/drm/i915/display/skl_watermark.c
@@ -8,7 +8,6 @@
 #include <drm/drm_blend.h>
 #include <drm/drm_print.h>
 
-#include "soc/intel_dram.h"
 #include "i915_reg.h"
 #include "i9xx_wm.h"
 #include "intel_atomic.h"
@@ -23,6 +22,7 @@
 #include "intel_display_rpm.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
+#include "intel_dram.h"
 #include "intel_fb.h"
 #include "intel_fixed.h"
 #include "intel_flipq.h"
@@ -718,7 +718,7 @@ static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
 	struct intel_display *display = to_intel_display(crtc);
 	enum intel_display_power_domain power_domain;
 	enum pipe pipe = crtc->pipe;
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	enum plane_id plane_id;
 
 	power_domain = POWER_DOMAIN_PIPE(pipe);
@@ -3125,7 +3125,7 @@ static bool skl_watermark_ipc_can_enable(struct intel_display *display)
 	if (display->platform.kabylake ||
 	    display->platform.coffeelake ||
 	    display->platform.cometlake) {
-		const struct dram_info *dram_info = intel_dram_info(display->drm);
+		const struct dram_info *dram_info = intel_dram_info(display);
 
 		return dram_info->symmetric_memory;
 	}
@@ -3169,7 +3169,7 @@ static void increase_wm_latency(struct intel_display *display, int inc)
 
 static bool need_16gb_dimm_wa(struct intel_display *display)
 {
-	const struct dram_info *dram_info = intel_dram_info(display->drm);
+	const struct dram_info *dram_info = intel_dram_info(display);
 
 	return (display->platform.skylake || display->platform.kabylake ||
 		display->platform.coffeelake || display->platform.cometlake ||
diff --git a/drivers/gpu/drm/i915/display/vlv_dsi.c b/drivers/gpu/drm/i915/display/vlv_dsi.c
index 19bdd8662359..d705af3bf8ba 100644
--- a/drivers/gpu/drm/i915/display/vlv_dsi.c
+++ b/drivers/gpu/drm/i915/display/vlv_dsi.c
@@ -936,7 +936,7 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
 {
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
-	intel_wakeref_t wakeref;
+	struct ref_tracker *wakeref;
 	enum port port;
 	bool active = false;
 
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
index 465ce94aee76..969f6ea2b855 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
@@ -348,12 +348,13 @@ struct drm_i915_gem_object {
  */
 #define I915_BO_ALLOC_GPU_ONLY	  BIT(6)
 #define I915_BO_ALLOC_CCS_AUX	  BIT(7)
+#define I915_BO_ALLOC_NOTHP	  BIT(8)
 /*
  * Object is allowed to retain its initial data and will not be cleared on first
  * access if used along with I915_BO_ALLOC_USER. This is mainly to keep
  * preallocated framebuffer data intact while transitioning it to i915drmfb.
  */
-#define I915_BO_PREALLOC	  BIT(8)
+#define I915_BO_PREALLOC	  BIT(9)
 #define I915_BO_ALLOC_FLAGS (I915_BO_ALLOC_CONTIGUOUS | \
 			     I915_BO_ALLOC_VOLATILE | \
 			     I915_BO_ALLOC_CPU_CLEAR | \
@@ -362,10 +363,11 @@ struct drm_i915_gem_object {
 			     I915_BO_ALLOC_PM_EARLY | \
 			     I915_BO_ALLOC_GPU_ONLY | \
 			     I915_BO_ALLOC_CCS_AUX | \
+			     I915_BO_ALLOC_NOTHP | \
 			     I915_BO_PREALLOC)
-#define I915_BO_READONLY          BIT(9)
-#define I915_TILING_QUIRK_BIT     10 /* unknown swizzling; do not release! */
-#define I915_BO_PROTECTED         BIT(11)
+#define I915_BO_READONLY          BIT(10)
+#define I915_TILING_QUIRK_BIT     11 /* unknown swizzling; do not release! */
+#define I915_BO_PROTECTED         BIT(12)
 	/**
 	 * @mem_flags - Mutable placement-related flags
 	 *
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
index 26dda55a07ff..6ad1d6f99363 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
@@ -9,14 +9,16 @@
 #include <linux/uio.h>
 
 #include <drm/drm_cache.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_print.h>
 
 #include "gem/i915_gem_region.h"
 #include "i915_drv.h"
 #include "i915_gem_object.h"
 #include "i915_gem_tiling.h"
-#include "i915_gemfs.h"
 #include "i915_scatterlist.h"
 #include "i915_trace.h"
+#include "i915_utils.h"
 
 /*
  * Move folios to appropriate lru and release the batch, decrementing the
@@ -494,9 +496,11 @@ const struct drm_i915_gem_object_ops i915_gem_shmem_ops = {
 
 static int __create_shmem(struct drm_i915_private *i915,
 			  struct drm_gem_object *obj,
-			  resource_size_t size)
+			  resource_size_t size,
+			  unsigned int flags)
 {
-	unsigned long flags = VM_NORESERVE;
+	unsigned long shmem_flags = VM_NORESERVE;
+	struct vfsmount *huge_mnt;
 	struct file *filp;
 
 	drm_gem_private_object_init(&i915->drm, obj, size);
@@ -515,11 +519,12 @@ static int __create_shmem(struct drm_i915_private *i915,
 	if (BITS_PER_LONG == 64 && size > MAX_LFS_FILESIZE)
 		return -E2BIG;
 
-	if (i915->mm.gemfs)
-		filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size,
-						 flags);
+	huge_mnt = drm_gem_get_huge_mnt(&i915->drm);
+	if (!(flags & I915_BO_ALLOC_NOTHP) && huge_mnt)
+		filp = shmem_file_setup_with_mnt(huge_mnt, "i915", size,
+						 shmem_flags);
 	else
-		filp = shmem_file_setup("i915", size, flags);
+		filp = shmem_file_setup("i915", size, shmem_flags);
 	if (IS_ERR(filp))
 		return PTR_ERR(filp);
 
@@ -548,7 +553,7 @@ static int shmem_object_init(struct intel_memory_region *mem,
 	gfp_t mask;
 	int ret;
 
-	ret = __create_shmem(i915, &obj->base, size);
+	ret = __create_shmem(i915, &obj->base, size, flags);
 	if (ret)
 		return ret;
 
@@ -644,21 +649,40 @@ i915_gem_object_create_shmem_from_data(struct drm_i915_private *i915,
 
 static int init_shmem(struct intel_memory_region *mem)
 {
-	i915_gemfs_init(mem->i915);
-	intel_memory_region_set_name(mem, "system");
+	struct drm_i915_private *i915 = mem->i915;
 
-	return 0; /* We have fallback to the kernel mnt if gemfs init failed. */
-}
+	/*
+	 * By creating our own shmemfs mountpoint, we can pass in
+	 * mount flags that better match our usecase.
+	 *
+	 * One example, although it is probably better with a per-file
+	 * control, is selecting huge page allocations ("huge=within_size").
+	 * However, we only do so on platforms which benefit from it, or to
+	 * offset the overhead of iommu lookups, where with latter it is a net
+	 * win even on platforms which would otherwise see some performance
+	 * regressions such a slow reads issue on Broadwell and Skylake.
+	 */
 
-static int release_shmem(struct intel_memory_region *mem)
-{
-	i915_gemfs_fini(mem->i915);
-	return 0;
+	if (GRAPHICS_VER(i915) < 11 && !i915_vtd_active(i915))
+		goto no_thp;
+
+	drm_gem_huge_mnt_create(&i915->drm, "within_size");
+	if (drm_gem_get_huge_mnt(&i915->drm))
+		drm_info(&i915->drm, "Using Transparent Hugepages\n");
+	else
+		drm_notice(&i915->drm,
+			   "Transparent Hugepage support is recommended for optimal performance%s\n",
+			   GRAPHICS_VER(i915) >= 11 ? " on this platform!" :
+						      " when IOMMU is enabled!");
+
+ no_thp:
+	intel_memory_region_set_name(mem, "system");
+
+	return 0; /* We have fallback to the kernel mnt if huge mnt failed. */
 }
 
 static const struct intel_memory_region_ops shmem_region_ops = {
 	.init = init_shmem,
-	.release = release_shmem,
 	.init_object = shmem_object_init,
 };
 
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
index f859c99f969b..c3e0b8da485c 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
@@ -8,6 +8,7 @@
 
 #include <drm/drm_mm.h>
 #include <drm/drm_print.h>
+#include <drm/intel/display_parent_interface.h>
 #include <drm/intel/i915_drm.h>
 
 #include "gem/i915_gem_lmem.h"
@@ -64,8 +65,8 @@ static int __i915_gem_stolen_insert_node_in_range(struct drm_i915_private *i915,
 	return ret;
 }
 
-int i915_gem_stolen_insert_node_in_range(struct intel_stolen_node *node, u64 size,
-					 unsigned int alignment, u64 start, u64 end)
+static int i915_gem_stolen_insert_node_in_range(struct intel_stolen_node *node, u64 size,
+						unsigned int alignment, u64 start, u64 end)
 {
 	return __i915_gem_stolen_insert_node_in_range(node->i915, &node->node,
 						      size, alignment,
@@ -82,8 +83,8 @@ static int __i915_gem_stolen_insert_node(struct drm_i915_private *i915,
 						      U64_MAX);
 }
 
-int i915_gem_stolen_insert_node(struct intel_stolen_node *node, u64 size,
-				unsigned int alignment)
+static int i915_gem_stolen_insert_node(struct intel_stolen_node *node, u64 size,
+				       unsigned int alignment)
 {
 	return __i915_gem_stolen_insert_node(node->i915, &node->node, size, alignment);
 }
@@ -96,7 +97,7 @@ static void __i915_gem_stolen_remove_node(struct drm_i915_private *i915,
 	mutex_unlock(&i915->mm.stolen_lock);
 }
 
-void i915_gem_stolen_remove_node(struct intel_stolen_node *node)
+static void i915_gem_stolen_remove_node(struct intel_stolen_node *node)
 {
 	__i915_gem_stolen_remove_node(node->i915, &node->node);
 }
@@ -1025,50 +1026,50 @@ bool i915_gem_object_is_stolen(const struct drm_i915_gem_object *obj)
 	return obj->ops == &i915_gem_object_stolen_ops;
 }
 
-bool i915_gem_stolen_initialized(struct drm_device *drm)
+static bool i915_gem_stolen_initialized(struct drm_device *drm)
 {
 	struct drm_i915_private *i915 = to_i915(drm);
 
 	return drm_mm_initialized(&i915->mm.stolen);
 }
 
-u64 i915_gem_stolen_area_address(struct drm_device *drm)
+static u64 i915_gem_stolen_area_address(struct drm_device *drm)
 {
 	struct drm_i915_private *i915 = to_i915(drm);
 
 	return i915->dsm.stolen.start;
 }
 
-u64 i915_gem_stolen_area_size(struct drm_device *drm)
+static u64 i915_gem_stolen_area_size(struct drm_device *drm)
 {
 	struct drm_i915_private *i915 = to_i915(drm);
 
 	return resource_size(&i915->dsm.stolen);
 }
 
-u64 i915_gem_stolen_node_address(const struct intel_stolen_node *node)
+static u64 i915_gem_stolen_node_offset(const struct intel_stolen_node *node)
+{
+	return node->node.start;
+}
+
+static u64 i915_gem_stolen_node_address(const struct intel_stolen_node *node)
 {
 	struct drm_i915_private *i915 = node->i915;
 
 	return i915->dsm.stolen.start + i915_gem_stolen_node_offset(node);
 }
 
-bool i915_gem_stolen_node_allocated(const struct intel_stolen_node *node)
+static bool i915_gem_stolen_node_allocated(const struct intel_stolen_node *node)
 {
 	return drm_mm_node_allocated(&node->node);
 }
 
-u64 i915_gem_stolen_node_offset(const struct intel_stolen_node *node)
-{
-	return node->node.start;
-}
-
-u64 i915_gem_stolen_node_size(const struct intel_stolen_node *node)
+static u64 i915_gem_stolen_node_size(const struct intel_stolen_node *node)
 {
 	return node->node.size;
 }
 
-struct intel_stolen_node *i915_gem_stolen_node_alloc(struct drm_device *drm)
+static struct intel_stolen_node *i915_gem_stolen_node_alloc(struct drm_device *drm)
 {
 	struct drm_i915_private *i915 = to_i915(drm);
 	struct intel_stolen_node *node;
@@ -1082,7 +1083,22 @@ struct intel_stolen_node *i915_gem_stolen_node_alloc(struct drm_device *drm)
 	return node;
 }
 
-void i915_gem_stolen_node_free(const struct intel_stolen_node *node)
+static void i915_gem_stolen_node_free(const struct intel_stolen_node *node)
 {
 	kfree(node);
 }
+
+const struct intel_display_stolen_interface i915_display_stolen_interface = {
+	.insert_node_in_range = i915_gem_stolen_insert_node_in_range,
+	.insert_node = i915_gem_stolen_insert_node,
+	.remove_node = i915_gem_stolen_remove_node,
+	.initialized = i915_gem_stolen_initialized,
+	.node_allocated = i915_gem_stolen_node_allocated,
+	.node_offset = i915_gem_stolen_node_offset,
+	.area_address = i915_gem_stolen_area_address,
+	.area_size = i915_gem_stolen_area_size,
+	.node_address = i915_gem_stolen_node_address,
+	.node_size = i915_gem_stolen_node_size,
+	.node_alloc = i915_gem_stolen_node_alloc,
+	.node_free = i915_gem_stolen_node_free,
+};
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.h b/drivers/gpu/drm/i915/gem/i915_gem_stolen.h
index 7b0386002ed4..6db5262046a2 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.h
@@ -8,17 +8,9 @@
 
 #include <linux/types.h>
 
-struct drm_device;
 struct drm_i915_gem_object;
 struct drm_i915_private;
-struct intel_stolen_node;
-
-int i915_gem_stolen_insert_node(struct intel_stolen_node *node, u64 size,
-				unsigned alignment);
-int i915_gem_stolen_insert_node_in_range(struct intel_stolen_node *node, u64 size,
-					 unsigned alignment, u64 start,
-					 u64 end);
-void i915_gem_stolen_remove_node(struct intel_stolen_node *node);
+
 struct intel_memory_region *
 i915_gem_stolen_smem_setup(struct drm_i915_private *i915, u16 type,
 			   u16 instance);
@@ -34,17 +26,6 @@ bool i915_gem_object_is_stolen(const struct drm_i915_gem_object *obj);
 
 #define I915_GEM_STOLEN_BIAS SZ_128K
 
-bool i915_gem_stolen_initialized(struct drm_device *drm);
-u64 i915_gem_stolen_area_address(struct drm_device *drm);
-u64 i915_gem_stolen_area_size(struct drm_device *drm);
-
-u64 i915_gem_stolen_node_address(const struct intel_stolen_node *node);
-
-bool i915_gem_stolen_node_allocated(const struct intel_stolen_node *node);
-u64 i915_gem_stolen_node_offset(const struct intel_stolen_node *node);
-u64 i915_gem_stolen_node_size(const struct intel_stolen_node *node);
-
-struct intel_stolen_node *i915_gem_stolen_node_alloc(struct drm_device *drm);
-void i915_gem_stolen_node_free(const struct intel_stolen_node *node);
+extern const struct intel_display_stolen_interface i915_display_stolen_interface;
 
 #endif /* __I915_GEM_STOLEN_H__ */
diff --git a/drivers/gpu/drm/i915/gem/i915_gemfs.c b/drivers/gpu/drm/i915/gem/i915_gemfs.c
deleted file mode 100644
index 1f1290214031..000000000000
--- a/drivers/gpu/drm/i915/gem/i915_gemfs.c
+++ /dev/null
@@ -1,71 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright © 2017 Intel Corporation
- */
-
-#include <linux/fs.h>
-#include <linux/mount.h>
-#include <linux/fs_context.h>
-
-#include <drm/drm_print.h>
-
-#include "i915_drv.h"
-#include "i915_gemfs.h"
-#include "i915_utils.h"
-
-void i915_gemfs_init(struct drm_i915_private *i915)
-{
-	struct file_system_type *type;
-	struct fs_context *fc;
-	struct vfsmount *gemfs;
-	int ret;
-
-	/*
-	 * By creating our own shmemfs mountpoint, we can pass in
-	 * mount flags that better match our usecase.
-	 *
-	 * One example, although it is probably better with a per-file
-	 * control, is selecting huge page allocations ("huge=within_size").
-	 * However, we only do so on platforms which benefit from it, or to
-	 * offset the overhead of iommu lookups, where with latter it is a net
-	 * win even on platforms which would otherwise see some performance
-	 * regressions such a slow reads issue on Broadwell and Skylake.
-	 */
-
-	if (GRAPHICS_VER(i915) < 11 && !i915_vtd_active(i915))
-		return;
-
-	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
-		goto err;
-
-	type = get_fs_type("tmpfs");
-	if (!type)
-		goto err;
-
-	fc = fs_context_for_mount(type, SB_KERNMOUNT);
-	if (IS_ERR(fc))
-		goto err;
-	ret = vfs_parse_fs_string(fc, "source", "tmpfs");
-	if (!ret)
-		ret = vfs_parse_fs_string(fc, "huge", "within_size");
-	if (!ret)
-		gemfs = fc_mount_longterm(fc);
-	put_fs_context(fc);
-	if (ret)
-		goto err;
-
-	i915->mm.gemfs = gemfs;
-	drm_info(&i915->drm, "Using Transparent Hugepages\n");
-	return;
-
-err:
-	drm_notice(&i915->drm,
-		   "Transparent Hugepage support is recommended for optimal performance%s\n",
-		   GRAPHICS_VER(i915) >= 11 ? " on this platform!" :
-					      " when IOMMU is enabled!");
-}
-
-void i915_gemfs_fini(struct drm_i915_private *i915)
-{
-	kern_unmount(i915->mm.gemfs);
-}
diff --git a/drivers/gpu/drm/i915/gem/i915_gemfs.h b/drivers/gpu/drm/i915/gem/i915_gemfs.h
deleted file mode 100644
index 16d4333c9a4e..000000000000
--- a/drivers/gpu/drm/i915/gem/i915_gemfs.h
+++ /dev/null
@@ -1,14 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2017 Intel Corporation
- */
-
-#ifndef __I915_GEMFS_H__
-#define __I915_GEMFS_H__
-
-struct drm_i915_private;
-
-void i915_gemfs_init(struct drm_i915_private *i915);
-void i915_gemfs_fini(struct drm_i915_private *i915);
-
-#endif
diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
index bd08605a1611..02e9bf87f654 100644
--- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
+++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
@@ -1316,7 +1316,7 @@ typedef struct drm_i915_gem_object *
 
 static inline bool igt_can_allocate_thp(struct drm_i915_private *i915)
 {
-	return i915->mm.gemfs && has_transparent_hugepage();
+	return !!drm_gem_get_huge_mnt(&i915->drm);
 }
 
 static struct drm_i915_gem_object *
@@ -1761,7 +1761,6 @@ static int igt_tmpfs_fallback(void *arg)
 	struct drm_i915_private *i915 = arg;
 	struct i915_address_space *vm;
 	struct i915_gem_context *ctx;
-	struct vfsmount *gemfs = i915->mm.gemfs;
 	struct drm_i915_gem_object *obj;
 	struct i915_vma *vma;
 	struct file *file;
@@ -1779,15 +1778,8 @@ static int igt_tmpfs_fallback(void *arg)
 	}
 	vm = i915_gem_context_get_eb_vm(ctx);
 
-	/*
-	 * Make sure that we don't burst into a ball of flames upon falling back
-	 * to tmpfs, which we rely on if on the off-chance we encounter a failure
-	 * when setting up gemfs.
-	 */
-
-	i915->mm.gemfs = NULL;
-
-	obj = i915_gem_object_create_shmem(i915, PAGE_SIZE);
+	obj = i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM],
+					    PAGE_SIZE, 0, I915_BO_ALLOC_NOTHP);
 	if (IS_ERR(obj)) {
 		err = PTR_ERR(obj);
 		goto out_restore;
@@ -1819,7 +1811,6 @@ static int igt_tmpfs_fallback(void *arg)
 out_put:
 	i915_gem_object_put(obj);
 out_restore:
-	i915->mm.gemfs = gemfs;
 
 	i915_vm_put(vm);
 out:
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
index b721bbd23356..98a3a7a9de50 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
@@ -963,9 +963,6 @@ int intel_engines_init_mmio(struct intel_gt *gt)
 	drm_WARN_ON(&i915->drm, engine_mask &
 		    GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES));
 
-	if (i915_inject_probe_failure(i915))
-		return -ENODEV;
-
 	for (class = 0; class < MAX_ENGINE_CLASS + 1; ++class) {
 		setup_logical_ids(gt, logical_ids, class);
 
@@ -1007,6 +1004,7 @@ int intel_engines_init_mmio(struct intel_gt *gt)
 	intel_engines_free(gt);
 	return err;
 }
+ALLOW_ERROR_INJECTION(intel_engines_init_mmio, ERRNO);
 
 void intel_engine_init_execlists(struct intel_engine_cs *engine)
 {
diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c
index 3d3b1ba76e2b..ac527d878820 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt.c
@@ -686,10 +686,6 @@ int intel_gt_init(struct intel_gt *gt)
 {
 	int err;
 
-	err = i915_inject_probe_error(gt->i915, -ENODEV);
-	if (err)
-		return err;
-
 	intel_gt_init_workarounds(gt);
 
 	/*
@@ -740,10 +736,6 @@ int intel_gt_init(struct intel_gt *gt)
 	if (err)
 		goto err_gt;
 
-	err = i915_inject_probe_error(gt->i915, -EIO);
-	if (err)
-		goto err_gt;
-
 	intel_uc_init_late(&gt->uc);
 
 	intel_migrate_init(&gt->migrate, gt);
@@ -766,6 +758,7 @@ int intel_gt_init(struct intel_gt *gt)
 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 	return err;
 }
+ALLOW_ERROR_INJECTION(intel_gt_init, ERRNO);
 
 void intel_gt_driver_remove(struct intel_gt *gt)
 {
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c b/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c
index c90b35881a26..aecd120972ea 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c
@@ -4,12 +4,12 @@
  */
 
 #include "i915_drv.h"
+#include "i915_freq.h"
 #include "i915_reg.h"
 #include "intel_gt.h"
 #include "intel_gt_clock_utils.h"
 #include "intel_gt_print.h"
 #include "intel_gt_regs.h"
-#include "soc/intel_dram.h"
 
 static u32 read_reference_ts_freq(struct intel_uncore *uncore)
 {
@@ -148,7 +148,7 @@ static u32 gen4_read_clock_frequency(struct intel_uncore *uncore)
 	 *
 	 * Testing on actual hardware has shown there is no /16.
 	 */
-	return DIV_ROUND_CLOSEST(intel_fsb_freq(uncore->i915), 4) * 1000;
+	return DIV_ROUND_CLOSEST(i9xx_fsb_freq(uncore->i915), 4) * 1000;
 }
 
 static u32 read_clock_frequency(struct intel_uncore *uncore)
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_print.h b/drivers/gpu/drm/i915/gt/intel_gt_print.h
index 7fdc78c79273..48f0afd05fd8 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_print.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt_print.h
@@ -36,10 +36,7 @@
 
 #define gt_probe_error(_gt, _fmt, ...) \
 	do { \
-		if (i915_error_injected()) \
-			gt_dbg(_gt, _fmt, ##__VA_ARGS__); \
-		else \
-			gt_err(_gt, _fmt, ##__VA_ARGS__); \
+		gt_err(_gt, _fmt, ##__VA_ARGS__); \
 	} while (0)
 
 #define gt_WARN(_gt, _condition, _fmt, ...) \
diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c
index e8927ad49142..d36e543e98df 100644
--- a/drivers/gpu/drm/i915/gt/intel_lrc.c
+++ b/drivers/gpu/drm/i915/gt/intel_lrc.c
@@ -1911,10 +1911,6 @@ void lrc_init_wa_ctx(struct intel_engine_cs *engine)
 	__i915_gem_object_flush_map(wa_ctx->vma->obj, 0, batch_ptr - batch);
 	__i915_gem_object_release_map(wa_ctx->vma->obj);
 
-	/* Verify that we can handle failure to setup the wa_ctx */
-	if (!err)
-		err = i915_inject_probe_error(engine->i915, -ENODEV);
-
 err_unpin:
 	if (err)
 		i915_vma_unpin(wa_ctx->vma);
diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c
index b01c837ab646..90b7eee78f1f 100644
--- a/drivers/gpu/drm/i915/gt/intel_rps.c
+++ b/drivers/gpu/drm/i915/gt/intel_rps.c
@@ -6,12 +6,13 @@
 #include <linux/string_helpers.h>
 
 #include <drm/intel/i915_drm.h>
+#include <drm/intel/display_parent_interface.h>
 
 #include "display/intel_display_rps.h"
 #include "display/vlv_clock.h"
-#include "soc/intel_dram.h"
 
 #include "i915_drv.h"
+#include "i915_freq.h"
 #include "i915_irq.h"
 #include "i915_reg.h"
 #include "i915_wait_util.h"
@@ -284,8 +285,8 @@ static void gen5_rps_init(struct intel_rps *rps)
 	u32 rgvmodectl;
 	int c_m, i;
 
-	fsb_freq = intel_fsb_freq(i915);
-	mem_freq = intel_mem_freq(i915);
+	fsb_freq = ilk_fsb_freq(i915);
+	mem_freq = ilk_mem_freq(i915);
 
 	if (fsb_freq <= 3200000)
 		c_m = 0;
@@ -2914,6 +2915,39 @@ bool i915_gpu_turbo_disable(void)
 }
 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
 
+static void boost_if_not_started(struct dma_fence *fence)
+{
+	struct i915_request *rq;
+
+	if (!dma_fence_is_i915(fence))
+		return;
+
+	rq = to_request(fence);
+
+	if (!i915_request_started(rq))
+		intel_rps_boost(rq);
+}
+
+static void mark_interactive(struct drm_device *drm, bool interactive)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+
+	intel_rps_mark_interactive(&to_gt(i915)->rps, interactive);
+}
+
+static void ilk_irq_handler(struct drm_device *drm)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+
+	gen5_rps_irq_handler(&to_gt(i915)->rps);
+}
+
+const struct intel_display_rps_interface i915_display_rps_interface = {
+	.boost_if_not_started = boost_if_not_started,
+	.mark_interactive = mark_interactive,
+	.ilk_irq_handler = ilk_irq_handler,
+};
+
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 #include "selftest_rps.c"
 #include "selftest_slpc.c"
diff --git a/drivers/gpu/drm/i915/gt/intel_rps.h b/drivers/gpu/drm/i915/gt/intel_rps.h
index 92fb01f5a452..5dbcebd7d4a5 100644
--- a/drivers/gpu/drm/i915/gt/intel_rps.h
+++ b/drivers/gpu/drm/i915/gt/intel_rps.h
@@ -128,4 +128,6 @@ static inline void intel_rps_clear_timer(struct intel_rps *rps)
 	clear_bit(INTEL_RPS_TIMER, &rps->flags);
 }
 
+extern const struct intel_display_rps_interface i915_display_rps_interface;
+
 #endif /* INTEL_RPS_H */
diff --git a/drivers/gpu/drm/i915/gt/intel_wopcm.c b/drivers/gpu/drm/i915/gt/intel_wopcm.c
index 1b26ff6488b3..602e088fe045 100644
--- a/drivers/gpu/drm/i915/gt/intel_wopcm.c
+++ b/drivers/gpu/drm/i915/gt/intel_wopcm.c
@@ -253,9 +253,6 @@ void intel_wopcm_init(struct intel_wopcm *wopcm)
 	GEM_BUG_ON(huc_fw_size >= wopcm_size);
 	GEM_BUG_ON(ctx_rsvd + WOPCM_RESERVED_SIZE >= wopcm_size);
 
-	if (i915_inject_probe_failure(i915))
-		return;
-
 	if (__wopcm_regs_locked(gt->uncore, &guc_wopcm_base, &guc_wopcm_size)) {
 		drm_dbg(&i915->drm, "GuC WOPCM is already locked [%uK, %uK)\n",
 			guc_wopcm_base / SZ_1K, guc_wopcm_size / SZ_1K);
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c
index 2c651ec024ef..f1e53312ed90 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c
@@ -266,10 +266,6 @@ int intel_guc_ct_init(struct intel_guc_ct *ct)
 	u32 *cmds;
 	int err;
 
-	err = i915_inject_probe_error(guc_to_i915(guc), -ENXIO);
-	if (err)
-		return err;
-
 	GEM_BUG_ON(ct->vma);
 
 	blob_size = 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + CTB_G2H_BUFFER_SIZE;
@@ -306,6 +302,7 @@ int intel_guc_ct_init(struct intel_guc_ct *ct)
 
 	return 0;
 }
+ALLOW_ERROR_INJECTION(intel_guc_ct_init, ERRNO);
 
 /**
  * intel_guc_ct_fini - Fini buffer-based communication
@@ -1394,9 +1391,6 @@ static void ct_dead_ct_worker_func(struct work_struct *w)
 	if (ct->dead_ct_reported)
 		return;
 
-	if (i915_error_injected())
-		return;
-
 	ct->dead_ct_reported = true;
 
 	guc_info(guc, "CTB is dead - reason=0x%X\n", ct->dead_ct_reason);
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc.c b/drivers/gpu/drm/i915/gt/uc/intel_huc.c
index 456d3372eef8..f3249b98c992 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_huc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_huc.c
@@ -541,10 +541,6 @@ int intel_huc_auth(struct intel_huc *huc, enum intel_huc_authentication_type typ
 	if (intel_huc_is_authenticated(huc, type))
 		return -EEXIST;
 
-	ret = i915_inject_probe_error(gt->i915, -ENXIO);
-	if (ret)
-		goto fail;
-
 	switch (type) {
 	case INTEL_HUC_AUTH_BY_GUC:
 		ret = intel_guc_auth_huc(guc, intel_guc_ggtt_offset(guc, huc->fw.rsa_data));
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc.c b/drivers/gpu/drm/i915/gt/uc/intel_uc.c
index 4a3493e8d433..bd07c72a66fc 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_uc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_uc.c
@@ -60,10 +60,6 @@ static int __intel_uc_reset_hw(struct intel_uc *uc)
 	int ret;
 	u32 guc_status;
 
-	ret = i915_inject_probe_error(gt->i915, -ENXIO);
-	if (ret)
-		return ret;
-
 	ret = intel_reset_guc(gt);
 	if (ret) {
 		gt_err(gt, "Failed to reset GuC, ret = %d\n", ret);
@@ -220,15 +216,10 @@ static void guc_handle_mmio_msg(struct intel_guc *guc)
 static int guc_enable_communication(struct intel_guc *guc)
 {
 	struct intel_gt *gt = guc_to_gt(guc);
-	struct drm_i915_private *i915 = gt->i915;
 	int ret;
 
 	GEM_BUG_ON(intel_guc_ct_enabled(&guc->ct));
 
-	ret = i915_inject_probe_error(i915, -ENXIO);
-	if (ret)
-		return ret;
-
 	ret = intel_guc_ct_enable(&guc->ct);
 	if (ret)
 		return ret;
@@ -323,9 +314,6 @@ static int __uc_init(struct intel_uc *uc)
 	if (!intel_uc_uses_guc(uc))
 		return 0;
 
-	if (i915_inject_probe_failure(uc_to_gt(uc)->i915))
-		return -ENOMEM;
-
 	ret = intel_guc_init(guc);
 	if (ret)
 		return ret;
@@ -338,6 +326,7 @@ static int __uc_init(struct intel_uc *uc)
 
 	return 0;
 }
+ALLOW_ERROR_INJECTION(__uc_init, ERRNO);
 
 static void __uc_fini(struct intel_uc *uc)
 {
@@ -381,10 +370,6 @@ static int uc_init_wopcm(struct intel_uc *uc)
 	GEM_BUG_ON(!(size & GUC_WOPCM_SIZE_MASK));
 	GEM_BUG_ON(size & ~GUC_WOPCM_SIZE_MASK);
 
-	err = i915_inject_probe_error(gt->i915, -ENXIO);
-	if (err)
-		return err;
-
 	mask = GUC_WOPCM_SIZE_MASK | GUC_WOPCM_SIZE_LOCKED;
 	err = intel_uncore_write_and_verify(uncore, GUC_WOPCM_SIZE, size, mask,
 					    size | GUC_WOPCM_SIZE_LOCKED);
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c
index e848a04a80dc..fd04fe892c28 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c
@@ -542,43 +542,6 @@ void intel_uc_fw_init_early(struct intel_uc_fw *uc_fw,
 				  INTEL_UC_FIRMWARE_NOT_SUPPORTED);
 }
 
-static void __force_fw_fetch_failures(struct intel_uc_fw *uc_fw, int e)
-{
-	struct drm_i915_private *i915 = __uc_fw_to_gt(uc_fw)->i915;
-	bool user = e == -EINVAL;
-
-	if (i915_inject_probe_error(i915, e)) {
-		/* non-existing blob */
-		uc_fw->file_selected.path = "<invalid>";
-		uc_fw->user_overridden = user;
-	} else if (i915_inject_probe_error(i915, e)) {
-		/* require next major version */
-		uc_fw->file_wanted.ver.major += 1;
-		uc_fw->file_wanted.ver.minor = 0;
-		uc_fw->user_overridden = user;
-	} else if (i915_inject_probe_error(i915, e)) {
-		/* require next minor version */
-		uc_fw->file_wanted.ver.minor += 1;
-		uc_fw->user_overridden = user;
-	} else if (uc_fw->file_wanted.ver.major &&
-		   i915_inject_probe_error(i915, e)) {
-		/* require prev major version */
-		uc_fw->file_wanted.ver.major -= 1;
-		uc_fw->file_wanted.ver.minor = 0;
-		uc_fw->user_overridden = user;
-	} else if (uc_fw->file_wanted.ver.minor &&
-		   i915_inject_probe_error(i915, e)) {
-		/* require prev minor version - hey, this should work! */
-		uc_fw->file_wanted.ver.minor -= 1;
-		uc_fw->user_overridden = user;
-	} else if (user && i915_inject_probe_error(i915, e)) {
-		/* officially unsupported platform */
-		uc_fw->file_wanted.ver.major = 0;
-		uc_fw->file_wanted.ver.minor = 0;
-		uc_fw->user_overridden = true;
-	}
-}
-
 static void uc_unpack_css_version(struct intel_uc_fw_ver *ver, u32 css_value)
 {
 	/* Get version numbers from the CSS header */
@@ -766,7 +729,7 @@ static int guc_check_version_range(struct intel_uc_fw *uc_fw)
 		return -EINVAL;
 	}
 
-	return i915_inject_probe_error(gt->i915, -EINVAL);
+	return 0;
 }
 
 static int check_fw_header(struct intel_gt *gt,
@@ -905,13 +868,6 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw)
 	GEM_BUG_ON(!gt->wopcm.size);
 	GEM_BUG_ON(!intel_uc_fw_is_enabled(uc_fw));
 
-	err = i915_inject_probe_error(i915, -ENXIO);
-	if (err)
-		goto fail;
-
-	__force_fw_fetch_failures(uc_fw, -EINVAL);
-	__force_fw_fetch_failures(uc_fw, -ESTALE);
-
 	err = try_firmware_load(uc_fw, &fw);
 	memcpy(&file_ideal, &uc_fw->file_wanted, sizeof(file_ideal));
 
@@ -1088,10 +1044,6 @@ static int uc_fw_xfer(struct intel_uc_fw *uc_fw, u32 dst_offset, u32 dma_flags)
 	u64 offset;
 	int ret;
 
-	ret = i915_inject_probe_error(gt->i915, -ETIMEDOUT);
-	if (ret)
-		return ret;
-
 	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
 
 	/* Set the source address for the uCode */
@@ -1155,16 +1107,11 @@ int intel_uc_fw_mark_load_failed(struct intel_uc_fw *uc_fw, int err)
  */
 int intel_uc_fw_upload(struct intel_uc_fw *uc_fw, u32 dst_offset, u32 dma_flags)
 {
-	struct intel_gt *gt = __uc_fw_to_gt(uc_fw);
 	int err;
 
 	/* make sure the status was cleared the last time we reset the uc */
 	GEM_BUG_ON(intel_uc_fw_is_loaded(uc_fw));
 
-	err = i915_inject_probe_error(gt->i915, -ENOEXEC);
-	if (err)
-		return err;
-
 	if (!intel_uc_fw_is_loadable(uc_fw))
 		return -ENOEXEC;
 
@@ -1198,10 +1145,6 @@ static int uc_fw_rsa_data_create(struct intel_uc_fw *uc_fw)
 	void *vaddr;
 	int err;
 
-	err = i915_inject_probe_error(gt->i915, -ENXIO);
-	if (err)
-		return err;
-
 	if (!uc_fw_need_rsa_in_memory(uc_fw))
 		return 0;
 
@@ -1243,6 +1186,7 @@ static int uc_fw_rsa_data_create(struct intel_uc_fw *uc_fw)
 	i915_vma_unpin_and_release(&vma, 0);
 	return err;
 }
+ALLOW_ERROR_INJECTION(uc_fw_rsa_data_create, ERRNO);
 
 static void uc_fw_rsa_data_destroy(struct intel_uc_fw *uc_fw)
 {
diff --git a/drivers/gpu/drm/i915/i915_driver.c b/drivers/gpu/drm/i915/i915_driver.c
index c97b76771917..75f503e52e70 100644
--- a/drivers/gpu/drm/i915/i915_driver.c
+++ b/drivers/gpu/drm/i915/i915_driver.c
@@ -59,6 +59,7 @@
 #include "display/intel_dmc.h"
 #include "display/intel_dp.h"
 #include "display/intel_dpt.h"
+#include "display/intel_dram.h"
 #include "display/intel_encoder.h"
 #include "display/intel_fbdev.h"
 #include "display/intel_gmbus.h"
@@ -81,25 +82,27 @@
 #include "gt/intel_gt_pm.h"
 #include "gt/intel_gt_print.h"
 #include "gt/intel_rc6.h"
+#include "gt/intel_rps.h"
 
 #include "pxp/intel_pxp.h"
 #include "pxp/intel_pxp_debugfs.h"
 #include "pxp/intel_pxp_pm.h"
 
-#include "soc/intel_dram.h"
-#include "soc/intel_gmch.h"
-
 #include "i915_debugfs.h"
 #include "i915_driver.h"
 #include "i915_drm_client.h"
 #include "i915_drv.h"
+#include "i915_edram.h"
 #include "i915_file_private.h"
 #include "i915_getparam.h"
+#include "i915_gmch.h"
+#include "i915_hdcp_gsc.h"
 #include "i915_hwmon.h"
 #include "i915_ioc32.h"
 #include "i915_ioctl.h"
 #include "i915_irq.h"
 #include "i915_memcpy.h"
+#include "i915_panic.h"
 #include "i915_perf.h"
 #include "i915_query.h"
 #include "i915_reg.h"
@@ -224,9 +227,6 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv)
 	struct intel_display *display = dev_priv->display;
 	int ret = 0;
 
-	if (i915_inject_probe_failure(dev_priv))
-		return -ENODEV;
-
 	intel_device_info_runtime_init_early(dev_priv);
 
 	intel_step_init(dev_priv);
@@ -262,7 +262,7 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv)
 
 	intel_irq_init(dev_priv);
 	intel_display_driver_early_probe(display);
-	intel_clock_gating_hooks_init(dev_priv);
+	intel_clock_gating_hooks_init(&dev_priv->drm);
 
 	intel_detect_preproduction_hw(dev_priv);
 
@@ -276,6 +276,7 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv)
 	i915_workqueues_cleanup(dev_priv);
 	return ret;
 }
+ALLOW_ERROR_INJECTION(i915_driver_early_probe, ERRNO);
 
 /**
  * i915_driver_late_release - cleanup the setup done in
@@ -318,10 +319,7 @@ static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv)
 	struct intel_gt *gt;
 	int ret, i;
 
-	if (i915_inject_probe_failure(dev_priv))
-		return -ENODEV;
-
-	ret = intel_gmch_bridge_setup(dev_priv);
+	ret = i915_gmch_bridge_setup(dev_priv);
 	if (ret < 0)
 		return ret;
 
@@ -338,7 +336,7 @@ static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv)
 	}
 
 	/* Try to make sure MCHBAR is enabled before poking at it */
-	intel_gmch_bar_setup(dev_priv);
+	i915_gmch_bar_setup(dev_priv);
 	intel_device_info_runtime_init(dev_priv);
 	intel_display_device_info_runtime_init(display);
 
@@ -354,10 +352,11 @@ static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv)
 	return 0;
 
 err_uncore:
-	intel_gmch_bar_teardown(dev_priv);
+	i915_gmch_bar_teardown(dev_priv);
 
 	return ret;
 }
+ALLOW_ERROR_INJECTION(i915_driver_mmio_probe, ERRNO);
 
 /**
  * i915_driver_mmio_release - cleanup the setup done in i915_driver_mmio_probe()
@@ -365,7 +364,7 @@ static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv)
  */
 static void i915_driver_mmio_release(struct drm_i915_private *dev_priv)
 {
-	intel_gmch_bar_teardown(dev_priv);
+	i915_gmch_bar_teardown(dev_priv);
 }
 
 /**
@@ -465,9 +464,6 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 	int ret;
 
-	if (i915_inject_probe_failure(dev_priv))
-		return -ENODEV;
-
 	if (HAS_PPGTT(dev_priv)) {
 		if (intel_vgpu_active(dev_priv) &&
 		    !intel_vgpu_has_full_ppgtt(dev_priv)) {
@@ -492,7 +488,7 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
 	}
 
 	/* needs to be done before ggtt probe */
-	intel_dram_edram_detect(dev_priv);
+	i915_edram_detect(dev_priv);
 
 	ret = i915_set_dma_info(dev_priv);
 	if (ret)
@@ -572,7 +568,7 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
 	 * Fill the dram structure to get the system dram info. This will be
 	 * used for memory latency calculation.
 	 */
-	ret = intel_dram_detect(dev_priv);
+	ret = intel_dram_detect(display);
 	if (ret)
 		goto err_opregion;
 
@@ -595,6 +591,7 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
 	i915_perf_fini(dev_priv);
 	return ret;
 }
+ALLOW_ERROR_INJECTION(i915_driver_hw_probe, ERRNO);
 
 /**
  * i915_driver_hw_remove - cleanup the setup done in i915_driver_hw_probe()
@@ -739,8 +736,43 @@ static void i915_welcome_messages(struct drm_i915_private *dev_priv)
 			 "DRM_I915_DEBUG_RUNTIME_PM enabled\n");
 }
 
+static void fence_priority_display(struct dma_fence *fence)
+{
+	if (dma_fence_is_i915(fence))
+		i915_gem_fence_wait_priority_display(fence);
+}
+
+static bool has_auxccs(struct drm_device *drm)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+
+	return IS_GRAPHICS_VER(i915, 9, 12) ||
+	       IS_ALDERLAKE_P(i915) ||
+	       IS_METEORLAKE(i915);
+}
+
+static bool has_fenced_regions(struct drm_device *drm)
+{
+	return intel_gt_support_legacy_fencing(to_gt(to_i915(drm)));
+}
+
+static bool vgpu_active(struct drm_device *drm)
+{
+	return intel_vgpu_active(to_i915(drm));
+}
+
 static const struct intel_display_parent_interface parent = {
+	.hdcp = &i915_display_hdcp_interface,
+	.irq = &i915_display_irq_interface,
+	.panic = &i915_display_panic_interface,
 	.rpm = &i915_display_rpm_interface,
+	.rps = &i915_display_rps_interface,
+	.stolen = &i915_display_stolen_interface,
+
+	.fence_priority_display = fence_priority_display,
+	.has_auxccs = has_auxccs,
+	.has_fenced_regions = has_fenced_regions,
+	.vgpu_active = vgpu_active,
 };
 
 const struct intel_display_parent_interface *i915_driver_parent_interface(void)
@@ -890,6 +922,7 @@ int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	i915_gem_drain_freed_objects(i915);
 	i915_ggtt_driver_late_release(i915);
 out_cleanup_mmio:
+	intel_gvt_driver_remove(i915);
 	i915_driver_mmio_release(i915);
 out_runtime_pm_put:
 	enable_rpm_wakeref_asserts(&i915->runtime_pm);
@@ -1254,7 +1287,7 @@ static int i915_drm_resume(struct drm_device *dev)
 
 	intel_display_driver_init_hw(display);
 
-	intel_clock_gating_init(dev_priv);
+	intel_clock_gating_init(&dev_priv->drm);
 
 	if (intel_display_device_present(display))
 		intel_display_driver_resume_access(display);
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 5381a934a671..44ba620325bc 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -60,7 +60,6 @@
 #include "intel_step.h"
 #include "intel_uncore.h"
 
-struct dram_info;
 struct drm_i915_clock_gating_funcs;
 struct intel_display;
 struct intel_pxp;
@@ -141,11 +140,6 @@ struct i915_gem_mm {
 	 */
 	atomic_t free_count;
 
-	/**
-	 * tmpfs instance used for shmem backed objects
-	 */
-	struct vfsmount *gemfs;
-
 	struct intel_memory_region *regions[INTEL_REGION_UNKNOWN];
 
 	struct notifier_block oom_notifier;
@@ -279,8 +273,6 @@ struct drm_i915_private {
 	u32 suspend_count;
 	struct vlv_s0ix_state *vlv_s0ix_state;
 
-	const struct dram_info *dram_info;
-
 	struct intel_runtime_pm runtime_pm;
 
 	struct i915_perf perf;
diff --git a/drivers/gpu/drm/i915/i915_edram.c b/drivers/gpu/drm/i915/i915_edram.c
new file mode 100644
index 000000000000..5818ec396d1e
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_edram.c
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: MIT
+/* Copyright © 2025 Intel Corporation */
+
+#include <drm/drm_print.h>
+
+#include "i915_drv.h"
+#include "i915_edram.h"
+#include "i915_reg.h"
+
+static u32 gen9_edram_size_mb(struct drm_i915_private *i915, u32 cap)
+{
+	static const u8 ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 };
+	static const u8 sets[4] = { 1, 1, 2, 2 };
+
+	return EDRAM_NUM_BANKS(cap) *
+		ways[EDRAM_WAYS_IDX(cap)] *
+		sets[EDRAM_SETS_IDX(cap)];
+}
+
+void i915_edram_detect(struct drm_i915_private *i915)
+{
+	u32 edram_cap = 0;
+
+	if (!(IS_HASWELL(i915) || IS_BROADWELL(i915) || GRAPHICS_VER(i915) >= 9))
+		return;
+
+	edram_cap = intel_uncore_read_fw(&i915->uncore, HSW_EDRAM_CAP);
+
+	/* NB: We can't write IDICR yet because we don't have gt funcs set up */
+
+	if (!(edram_cap & EDRAM_ENABLED))
+		return;
+
+	/*
+	 * The needed capability bits for size calculation are not there with
+	 * pre gen9 so return 128MB always.
+	 */
+	if (GRAPHICS_VER(i915) < 9)
+		i915->edram_size_mb = 128;
+	else
+		i915->edram_size_mb = gen9_edram_size_mb(i915, edram_cap);
+
+	drm_info(&i915->drm, "Found %uMB of eDRAM\n", i915->edram_size_mb);
+}
diff --git a/drivers/gpu/drm/i915/i915_edram.h b/drivers/gpu/drm/i915/i915_edram.h
new file mode 100644
index 000000000000..8319422ace9d
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_edram.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2025 Intel Corporation */
+
+#ifndef __I915_DRAM_H__
+#define __I915_DRAM_H__
+
+struct drm_i915_private;
+
+void i915_edram_detect(struct drm_i915_private *i915);
+
+#endif /* __I915_DRAM_H__ */
diff --git a/drivers/gpu/drm/i915/i915_freq.c b/drivers/gpu/drm/i915/i915_freq.c
new file mode 100644
index 000000000000..9bdaea34aef9
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_freq.c
@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: MIT
+/* Copyright © 2025 Intel Corporation */
+
+#include <drm/drm_print.h>
+
+#include "i915_drv.h"
+#include "i915_freq.h"
+#include "intel_mchbar_regs.h"
+
+unsigned int i9xx_fsb_freq(struct drm_i915_private *i915)
+{
+	u32 fsb;
+
+	/*
+	 * Note that this only reads the state of the FSB
+	 * straps, not the actual FSB frequency. Some BIOSen
+	 * let you configure each independently. Ideally we'd
+	 * read out the actual FSB frequency but sadly we
+	 * don't know which registers have that information,
+	 * and all the relevant docs have gone to bit heaven :(
+	 */
+	fsb = intel_uncore_read(&i915->uncore, CLKCFG) & CLKCFG_FSB_MASK;
+
+	if (IS_PINEVIEW(i915) || IS_MOBILE(i915)) {
+		switch (fsb) {
+		case CLKCFG_FSB_400:
+			return 400000;
+		case CLKCFG_FSB_533:
+			return 533333;
+		case CLKCFG_FSB_667:
+			return 666667;
+		case CLKCFG_FSB_800:
+			return 800000;
+		case CLKCFG_FSB_1067:
+			return 1066667;
+		case CLKCFG_FSB_1333:
+			return 1333333;
+		default:
+			MISSING_CASE(fsb);
+			return 1333333;
+		}
+	} else {
+		switch (fsb) {
+		case CLKCFG_FSB_400_ALT:
+			return 400000;
+		case CLKCFG_FSB_533:
+			return 533333;
+		case CLKCFG_FSB_667:
+			return 666667;
+		case CLKCFG_FSB_800:
+			return 800000;
+		case CLKCFG_FSB_1067_ALT:
+			return 1066667;
+		case CLKCFG_FSB_1333_ALT:
+			return 1333333;
+		case CLKCFG_FSB_1600_ALT:
+			return 1600000;
+		default:
+			MISSING_CASE(fsb);
+			return 1333333;
+		}
+	}
+}
+
+unsigned int ilk_fsb_freq(struct drm_i915_private *i915)
+{
+	u16 fsb;
+
+	fsb = intel_uncore_read16(&i915->uncore, CSIPLL0) & 0x3ff;
+
+	switch (fsb) {
+	case 0x00c:
+		return 3200000;
+	case 0x00e:
+		return 3733333;
+	case 0x010:
+		return 4266667;
+	case 0x012:
+		return 4800000;
+	case 0x014:
+		return 5333333;
+	case 0x016:
+		return 5866667;
+	case 0x018:
+		return 6400000;
+	default:
+		drm_dbg(&i915->drm, "unknown fsb frequency 0x%04x\n", fsb);
+		return 0;
+	}
+}
+
+unsigned int ilk_mem_freq(struct drm_i915_private *i915)
+{
+	u16 ddrpll;
+
+	ddrpll = intel_uncore_read16(&i915->uncore, DDRMPLL1);
+	switch (ddrpll & 0xff) {
+	case 0xc:
+		return 800000;
+	case 0x10:
+		return 1066667;
+	case 0x14:
+		return 1333333;
+	case 0x18:
+		return 1600000;
+	default:
+		drm_dbg(&i915->drm, "unknown memory frequency 0x%02x\n",
+			ddrpll & 0xff);
+		return 0;
+	}
+}
diff --git a/drivers/gpu/drm/i915/i915_freq.h b/drivers/gpu/drm/i915/i915_freq.h
new file mode 100644
index 000000000000..53b0ecb95440
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_freq.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2025 Intel Corporation */
+
+#ifndef __I915_FREQ_H__
+#define __I915_FREQ_H__
+
+struct drm_i915_private;
+
+unsigned int i9xx_fsb_freq(struct drm_i915_private *i915);
+unsigned int ilk_fsb_freq(struct drm_i915_private *i915);
+unsigned int ilk_mem_freq(struct drm_i915_private *i915);
+
+#endif /* __I915_FREQ_H__ */
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 4c82c9544b93..b40d4d88de01 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -1184,7 +1184,7 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 	 *
 	 * FIXME: break up the workarounds and apply them at the right time!
 	 */
-	intel_clock_gating_init(dev_priv);
+	intel_clock_gating_init(&dev_priv->drm);
 
 	for_each_gt(gt, dev_priv, i) {
 		ret = intel_gt_init(gt);
@@ -1235,7 +1235,7 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 		/* Minimal basic recovery for KMS */
 		ret = i915_ggtt_enable_hw(dev_priv);
 		i915_ggtt_resume(to_gt(dev_priv)->ggtt);
-		intel_clock_gating_init(dev_priv);
+		intel_clock_gating_init(&dev_priv->drm);
 	}
 
 	i915_gem_drain_freed_objects(dev_priv);
diff --git a/drivers/gpu/drm/i915/i915_gmch.c b/drivers/gpu/drm/i915/i915_gmch.c
new file mode 100644
index 000000000000..2d55831b3c58
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gmch.c
@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: MIT
+/* Copyright © 2025 Intel Corporation */
+
+#include <linux/pnp.h>
+
+#include <drm/drm_managed.h>
+#include <drm/drm_print.h>
+
+#include "i915_drv.h"
+#include "i915_gmch.h"
+#include "intel_pci_config.h"
+
+static void i915_gmch_bridge_release(struct drm_device *dev, void *bridge)
+{
+	pci_dev_put(bridge);
+}
+
+int i915_gmch_bridge_setup(struct drm_i915_private *i915)
+{
+	int domain = pci_domain_nr(to_pci_dev(i915->drm.dev)->bus);
+
+	i915->gmch.pdev = pci_get_domain_bus_and_slot(domain, 0, PCI_DEVFN(0, 0));
+	if (!i915->gmch.pdev) {
+		drm_err(&i915->drm, "bridge device not found\n");
+		return -EIO;
+	}
+
+	return drmm_add_action_or_reset(&i915->drm, i915_gmch_bridge_release,
+					i915->gmch.pdev);
+}
+
+static int mchbar_reg(struct drm_i915_private *i915)
+{
+	return GRAPHICS_VER(i915) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
+}
+
+/* Allocate space for the MCH regs if needed, return nonzero on error */
+static int
+intel_alloc_mchbar_resource(struct drm_i915_private *i915)
+{
+	u32 temp_lo, temp_hi = 0;
+	u64 mchbar_addr;
+	int ret;
+
+	if (GRAPHICS_VER(i915) >= 4)
+		pci_read_config_dword(i915->gmch.pdev, mchbar_reg(i915) + 4, &temp_hi);
+	pci_read_config_dword(i915->gmch.pdev, mchbar_reg(i915), &temp_lo);
+	mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
+
+	/* If ACPI doesn't have it, assume we need to allocate it ourselves */
+	if (IS_ENABLED(CONFIG_PNP) && mchbar_addr &&
+	    pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE))
+		return 0;
+
+	/* Get some space for it */
+	i915->gmch.mch_res.name = "i915 MCHBAR";
+	i915->gmch.mch_res.flags = IORESOURCE_MEM;
+	ret = pci_bus_alloc_resource(i915->gmch.pdev->bus,
+				     &i915->gmch.mch_res,
+				     MCHBAR_SIZE, MCHBAR_SIZE,
+				     PCIBIOS_MIN_MEM,
+				     0, pcibios_align_resource,
+				     i915->gmch.pdev);
+	if (ret) {
+		drm_dbg(&i915->drm, "failed bus alloc: %d\n", ret);
+		i915->gmch.mch_res.start = 0;
+		return ret;
+	}
+
+	if (GRAPHICS_VER(i915) >= 4)
+		pci_write_config_dword(i915->gmch.pdev, mchbar_reg(i915) + 4,
+				       upper_32_bits(i915->gmch.mch_res.start));
+
+	pci_write_config_dword(i915->gmch.pdev, mchbar_reg(i915),
+			       lower_32_bits(i915->gmch.mch_res.start));
+	return 0;
+}
+
+/* Setup MCHBAR if possible, return true if we should disable it again */
+void i915_gmch_bar_setup(struct drm_i915_private *i915)
+{
+	u32 temp;
+	bool enabled;
+
+	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
+		return;
+
+	i915->gmch.mchbar_need_disable = false;
+
+	if (IS_I915G(i915) || IS_I915GM(i915)) {
+		pci_read_config_dword(i915->gmch.pdev, DEVEN, &temp);
+		enabled = !!(temp & DEVEN_MCHBAR_EN);
+	} else {
+		pci_read_config_dword(i915->gmch.pdev, mchbar_reg(i915), &temp);
+		enabled = temp & 1;
+	}
+
+	/* If it's already enabled, don't have to do anything */
+	if (enabled)
+		return;
+
+	if (intel_alloc_mchbar_resource(i915))
+		return;
+
+	i915->gmch.mchbar_need_disable = true;
+
+	/* Space is allocated or reserved, so enable it. */
+	if (IS_I915G(i915) || IS_I915GM(i915)) {
+		pci_write_config_dword(i915->gmch.pdev, DEVEN,
+				       temp | DEVEN_MCHBAR_EN);
+	} else {
+		pci_read_config_dword(i915->gmch.pdev, mchbar_reg(i915), &temp);
+		pci_write_config_dword(i915->gmch.pdev, mchbar_reg(i915), temp | 1);
+	}
+}
+
+void i915_gmch_bar_teardown(struct drm_i915_private *i915)
+{
+	if (i915->gmch.mchbar_need_disable) {
+		if (IS_I915G(i915) || IS_I915GM(i915)) {
+			u32 deven_val;
+
+			pci_read_config_dword(i915->gmch.pdev, DEVEN,
+					      &deven_val);
+			deven_val &= ~DEVEN_MCHBAR_EN;
+			pci_write_config_dword(i915->gmch.pdev, DEVEN,
+					       deven_val);
+		} else {
+			u32 mchbar_val;
+
+			pci_read_config_dword(i915->gmch.pdev, mchbar_reg(i915),
+					      &mchbar_val);
+			mchbar_val &= ~1;
+			pci_write_config_dword(i915->gmch.pdev, mchbar_reg(i915),
+					       mchbar_val);
+		}
+	}
+
+	if (i915->gmch.mch_res.start)
+		release_resource(&i915->gmch.mch_res);
+}
diff --git a/drivers/gpu/drm/i915/i915_gmch.h b/drivers/gpu/drm/i915/i915_gmch.h
new file mode 100644
index 000000000000..3ae50bef04ea
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gmch.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2025 Intel Corporation */
+
+#ifndef __I915_GMCH_H__
+#define __I915_GMCH_H__
+
+struct drm_i915_private;
+
+int i915_gmch_bridge_setup(struct drm_i915_private *i915);
+void i915_gmch_bar_setup(struct drm_i915_private *i915);
+void i915_gmch_bar_teardown(struct drm_i915_private *i915);
+
+#endif /* __I915_GMCH_H__ */
diff --git a/drivers/gpu/drm/i915/i915_hdcp_gsc.c b/drivers/gpu/drm/i915/i915_hdcp_gsc.c
new file mode 100644
index 000000000000..9906da2aef1c
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_hdcp_gsc.c
@@ -0,0 +1,247 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2023, Intel Corporation.
+ */
+
+#include <drm/drm_print.h>
+#include <drm/intel/display_parent_interface.h>
+#include <drm/intel/i915_hdcp_interface.h>
+
+#include "gem/i915_gem_region.h"
+#include "gt/intel_gt.h"
+#include "gt/uc/intel_gsc_uc_heci_cmd_submit.h"
+#include "i915_drv.h"
+#include "i915_hdcp_gsc.h"
+
+struct intel_hdcp_gsc_context {
+	struct drm_i915_private *i915;
+	struct i915_vma *vma;
+	void *hdcp_cmd_in;
+	void *hdcp_cmd_out;
+};
+
+static bool intel_hdcp_gsc_check_status(struct drm_device *drm)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+	struct intel_gt *gt = i915->media_gt;
+	struct intel_gsc_uc *gsc = gt ? &gt->uc.gsc : NULL;
+
+	if (!gsc || !intel_uc_fw_is_running(&gsc->fw)) {
+		drm_dbg_kms(&i915->drm,
+			    "GSC components required for HDCP2.2 are not ready\n");
+		return false;
+	}
+
+	return true;
+}
+
+/*This function helps allocate memory for the command that we will send to gsc cs */
+static int intel_hdcp_gsc_initialize_message(struct drm_i915_private *i915,
+					     struct intel_hdcp_gsc_context *gsc_context)
+{
+	struct intel_gt *gt = i915->media_gt;
+	struct drm_i915_gem_object *obj = NULL;
+	struct i915_vma *vma = NULL;
+	void *cmd_in, *cmd_out;
+	int err;
+
+	/* allocate object of two page for HDCP command memory and store it */
+	obj = i915_gem_object_create_shmem(i915, 2 * PAGE_SIZE);
+
+	if (IS_ERR(obj)) {
+		drm_err(&i915->drm, "Failed to allocate HDCP streaming command!\n");
+		return PTR_ERR(obj);
+	}
+
+	cmd_in = i915_gem_object_pin_map_unlocked(obj, intel_gt_coherent_map_type(gt, obj, true));
+	if (IS_ERR(cmd_in)) {
+		drm_err(&i915->drm, "Failed to map gsc message page!\n");
+		err = PTR_ERR(cmd_in);
+		goto out_unpin;
+	}
+
+	cmd_out = cmd_in + PAGE_SIZE;
+
+	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
+	if (IS_ERR(vma)) {
+		err = PTR_ERR(vma);
+		goto out_unmap;
+	}
+
+	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
+	if (err)
+		goto out_unmap;
+
+	memset(cmd_in, 0, obj->base.size);
+
+	gsc_context->hdcp_cmd_in = cmd_in;
+	gsc_context->hdcp_cmd_out = cmd_out;
+	gsc_context->vma = vma;
+	gsc_context->i915 = i915;
+
+	return 0;
+
+out_unmap:
+	i915_gem_object_unpin_map(obj);
+out_unpin:
+	i915_gem_object_put(obj);
+	return err;
+}
+
+static struct intel_hdcp_gsc_context *intel_hdcp_gsc_context_alloc(struct drm_device *drm)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+	struct intel_hdcp_gsc_context *gsc_context;
+	int ret;
+
+	gsc_context = kzalloc(sizeof(*gsc_context), GFP_KERNEL);
+	if (!gsc_context)
+		return ERR_PTR(-ENOMEM);
+
+	/*
+	 * NOTE: No need to lock the comp mutex here as it is already
+	 * going to be taken before this function called
+	 */
+	ret = intel_hdcp_gsc_initialize_message(i915, gsc_context);
+	if (ret) {
+		drm_err(&i915->drm, "Could not initialize gsc_context\n");
+		kfree(gsc_context);
+		gsc_context = ERR_PTR(ret);
+	}
+
+	return gsc_context;
+}
+
+static void intel_hdcp_gsc_context_free(struct intel_hdcp_gsc_context *gsc_context)
+{
+	if (!gsc_context)
+		return;
+
+	i915_vma_unpin_and_release(&gsc_context->vma, I915_VMA_RELEASE_MAP);
+	kfree(gsc_context);
+}
+
+static int intel_gsc_send_sync(struct drm_i915_private *i915,
+			       struct intel_gsc_mtl_header *header_in,
+			       struct intel_gsc_mtl_header *header_out,
+			       u64 addr_in, u64 addr_out,
+			       size_t msg_out_len)
+{
+	struct intel_gt *gt = i915->media_gt;
+	int ret;
+
+	ret = intel_gsc_uc_heci_cmd_submit_packet(&gt->uc.gsc, addr_in,
+						  header_in->message_size,
+						  addr_out,
+						  msg_out_len + sizeof(*header_out));
+	if (ret) {
+		drm_err(&i915->drm, "failed to send gsc HDCP msg (%d)\n", ret);
+		return ret;
+	}
+
+	/*
+	 * Checking validity marker and header status to see if some error has
+	 * blocked us from sending message to gsc cs
+	 */
+	if (header_out->validity_marker != GSC_HECI_VALIDITY_MARKER) {
+		drm_err(&i915->drm, "invalid validity marker\n");
+		return -EINVAL;
+	}
+
+	if (header_out->status != 0) {
+		drm_err(&i915->drm, "header status indicates error %d\n",
+			header_out->status);
+		return -EINVAL;
+	}
+
+	if (header_out->flags & GSC_OUTFLAG_MSG_PENDING) {
+		header_in->gsc_message_handle = header_out->gsc_message_handle;
+		return -EAGAIN;
+	}
+
+	return 0;
+}
+
+/*
+ * This function can now be used for sending requests and will also handle
+ * receipt of reply messages hence no different function of message retrieval
+ * is required. We will initialize intel_hdcp_gsc_context structure then add
+ * gsc cs memory header as stated in specs after which the normal HDCP payload
+ * will follow
+ */
+static ssize_t intel_hdcp_gsc_msg_send(struct intel_hdcp_gsc_context *gsc_context,
+				       void *msg_in, size_t msg_in_len,
+				       void *msg_out, size_t msg_out_len)
+{
+	struct drm_i915_private *i915 = gsc_context->i915;
+	struct intel_gt *gt = i915->media_gt;
+	struct intel_gsc_mtl_header *header_in, *header_out;
+	const size_t max_msg_size = PAGE_SIZE - sizeof(*header_in);
+	u64 addr_in, addr_out, host_session_id;
+	u32 reply_size, msg_size_in, msg_size_out;
+	int ret, tries = 0;
+
+	if (!intel_uc_uses_gsc_uc(&gt->uc))
+		return -ENODEV;
+
+	if (msg_in_len > max_msg_size || msg_out_len > max_msg_size)
+		return -ENOSPC;
+
+	msg_size_in = msg_in_len + sizeof(*header_in);
+	msg_size_out = msg_out_len + sizeof(*header_out);
+	header_in = gsc_context->hdcp_cmd_in;
+	header_out = gsc_context->hdcp_cmd_out;
+	addr_in = i915_ggtt_offset(gsc_context->vma);
+	addr_out = addr_in + PAGE_SIZE;
+
+	memset(header_in, 0, msg_size_in);
+	memset(header_out, 0, msg_size_out);
+	get_random_bytes(&host_session_id, sizeof(u64));
+	intel_gsc_uc_heci_cmd_emit_mtl_header(header_in, HECI_MEADDRESS_HDCP,
+					      msg_size_in, host_session_id);
+	memcpy(gsc_context->hdcp_cmd_in + sizeof(*header_in), msg_in, msg_in_len);
+
+	/*
+	 * Keep sending request in case the pending bit is set no need to add
+	 * message handle as we are using same address hence loc. of header is
+	 * same and it will contain the message handle. we will send the message
+	 * 20 times each message 50 ms apart
+	 */
+	do {
+		ret = intel_gsc_send_sync(i915, header_in, header_out, addr_in,
+					  addr_out, msg_out_len);
+
+		/* Only try again if gsc says so */
+		if (ret != -EAGAIN)
+			break;
+
+		msleep(50);
+
+	} while (++tries < 20);
+
+	if (ret)
+		goto err;
+
+	/* we use the same mem for the reply, so header is in the same loc */
+	reply_size = header_out->message_size - sizeof(*header_out);
+	if (reply_size > msg_out_len) {
+		drm_warn(&i915->drm, "caller with insufficient HDCP reply size %u (%d)\n",
+			 reply_size, (u32)msg_out_len);
+		reply_size = msg_out_len;
+	} else if (reply_size != msg_out_len) {
+		drm_dbg_kms(&i915->drm, "caller unexpected HCDP reply size %u (%d)\n",
+			    reply_size, (u32)msg_out_len);
+	}
+
+	memcpy(msg_out, gsc_context->hdcp_cmd_out + sizeof(*header_out), msg_out_len);
+
+err:
+	return ret;
+}
+
+const struct intel_display_hdcp_interface i915_display_hdcp_interface = {
+	.gsc_msg_send = intel_hdcp_gsc_msg_send,
+	.gsc_check_status = intel_hdcp_gsc_check_status,
+	.gsc_context_alloc = intel_hdcp_gsc_context_alloc,
+	.gsc_context_free = intel_hdcp_gsc_context_free,
+};
diff --git a/drivers/gpu/drm/i915/i915_hdcp_gsc.h b/drivers/gpu/drm/i915/i915_hdcp_gsc.h
new file mode 100644
index 000000000000..e0b562cfcde3
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_hdcp_gsc.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2025 Intel Corporation */
+
+#ifndef __I915_HDCP_GSC_H__
+#define __I915_HDCP_GSC_H__
+
+extern const struct intel_display_hdcp_interface i915_display_hdcp_interface;
+
+#endif /* __I915_HDCP_GSC_H__ */
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 1898be4ddc8b..3fe978d4ea53 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -33,6 +33,7 @@
 
 #include <drm/drm_drv.h>
 #include <drm/drm_print.h>
+#include <drm/intel/display_parent_interface.h>
 
 #include "display/intel_display_irq.h"
 #include "display/intel_hotplug.h"
@@ -1252,3 +1253,18 @@ void intel_synchronize_hardirq(struct drm_i915_private *i915)
 {
 	synchronize_hardirq(to_pci_dev(i915->drm.dev)->irq);
 }
+
+static bool _intel_irq_enabled(struct drm_device *drm)
+{
+	return intel_irqs_enabled(to_i915(drm));
+}
+
+static void _intel_irq_synchronize(struct drm_device *drm)
+{
+	return intel_synchronize_irq(to_i915(drm));
+}
+
+const struct intel_display_irq_interface i915_display_irq_interface = {
+	.enabled = _intel_irq_enabled,
+	.synchronize = _intel_irq_synchronize,
+};
diff --git a/drivers/gpu/drm/i915/i915_irq.h b/drivers/gpu/drm/i915/i915_irq.h
index 58789b264575..5c87d6d41c74 100644
--- a/drivers/gpu/drm/i915/i915_irq.h
+++ b/drivers/gpu/drm/i915/i915_irq.h
@@ -51,4 +51,6 @@ void gen2_error_reset(struct intel_uncore *uncore, struct i915_error_regs regs);
 void gen2_error_init(struct intel_uncore *uncore, struct i915_error_regs regs,
 		     u32 emr_val);
 
+extern const struct intel_display_irq_interface i915_display_irq_interface;
+
 #endif /* __I915_IRQ_H__ */
diff --git a/drivers/gpu/drm/i915/i915_panic.c b/drivers/gpu/drm/i915/i915_panic.c
new file mode 100644
index 000000000000..728be077e8e8
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_panic.c
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: MIT
+/* Copyright © 2025 Intel Corporation */
+
+#include <drm/drm_panic.h>
+#include <drm/intel/display_parent_interface.h>
+
+#include "display/intel_display_types.h"
+#include "display/intel_fb.h"
+#include "gem/i915_gem_object.h"
+
+#include "i915_panic.h"
+
+static struct intel_panic *intel_panic_alloc(void)
+{
+	return i915_gem_object_alloc_panic();
+}
+
+static int intel_panic_setup(struct intel_panic *panic, struct drm_scanout_buffer *sb)
+{
+	struct intel_framebuffer *fb = sb->private;
+	struct drm_gem_object *obj = intel_fb_bo(&fb->base);
+
+	return i915_gem_object_panic_setup(panic, sb, obj, fb->panic_tiling);
+}
+
+static void intel_panic_finish(struct intel_panic *panic)
+{
+	return i915_gem_object_panic_finish(panic);
+}
+
+const struct intel_display_panic_interface i915_display_panic_interface = {
+	.alloc = intel_panic_alloc,
+	.setup = intel_panic_setup,
+	.finish = intel_panic_finish,
+};
diff --git a/drivers/gpu/drm/i915/i915_panic.h b/drivers/gpu/drm/i915/i915_panic.h
new file mode 100644
index 000000000000..743d8c861c42
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_panic.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2025 Intel Corporation */
+
+#ifndef __I915_PANIC_H__
+#define __I915_PANIC_H__
+
+extern const struct intel_display_panic_interface i915_display_panic_interface;
+
+#endif /* __I915_PANIC_H__ */
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index 37746dd619fd..34ed82bb1b1a 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -110,11 +110,6 @@ i915_param_named_unsafe(huc_firmware_path, charp, 0400,
 i915_param_named_unsafe(gsc_firmware_path, charp, 0400,
 	"GSC firmware path to use instead of the default one");
 
-#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
-i915_param_named_unsafe(inject_probe_failure, uint, 0400,
-	"Force an error after a number of failure check points (0:disabled (default), N:force failure at the Nth failure check point)");
-#endif
-
 #if IS_ENABLED(CONFIG_DRM_I915_GVT)
 i915_param_named(enable_gvt, bool, 0400,
 	"Enable support for Intel GVT-g graphics virtualization host support(default:false)");
diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h
index 0fbcb5b6d7bf..4d9d4cd3220d 100644
--- a/drivers/gpu/drm/i915/i915_params.h
+++ b/drivers/gpu/drm/i915/i915_params.h
@@ -55,7 +55,6 @@ struct drm_printer;
 	param(bool, memtest, false, 0400) \
 	param(int, mmio_debug, -IS_ENABLED(CONFIG_DRM_I915_DEBUG_MMIO), 0600) \
 	param(unsigned int, reset, 3, 0600) \
-	param(unsigned int, inject_probe_failure, 0, 0) \
 	param(char *, force_probe, CONFIG_DRM_I915_FORCE_PROBE, 0400) \
 	param(unsigned int, request_timeout_ms, CONFIG_DRM_I915_REQUEST_TIMEOUT, CONFIG_DRM_I915_REQUEST_TIMEOUT ? 0600 : 0) \
 	param(unsigned int, lmem_size, 0, 0400) \
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index b2e311f4791a..d966a00520f1 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -999,11 +999,6 @@ static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (err)
 		return err;
 
-	if (i915_inject_probe_failure(pdev_to_i915(pdev))) {
-		i915_pci_remove(pdev);
-		return -ENODEV;
-	}
-
 	err = i915_live_selftests(pdev);
 	if (err) {
 		i915_pci_remove(pdev);
@@ -1018,6 +1013,7 @@ static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	return 0;
 }
+ALLOW_ERROR_INJECTION(i915_pci_probe, ERRNO);
 
 static void i915_pci_shutdown(struct pci_dev *pdev)
 {
diff --git a/drivers/gpu/drm/i915/i915_utils.c b/drivers/gpu/drm/i915/i915_utils.c
index 89b920ccbccb..8c1aad312fd4 100644
--- a/drivers/gpu/drm/i915/i915_utils.c
+++ b/drivers/gpu/drm/i915/i915_utils.c
@@ -17,37 +17,9 @@ void add_taint_for_CI(struct drm_i915_private *i915, unsigned int taint)
 	drm_notice(&i915->drm, "CI tainted: %#x by %pS\n",
 		   taint, __builtin_return_address(0));
 
-	/* Failures that occur during fault injection testing are expected */
-	if (!i915_error_injected())
-		__add_taint_for_CI(taint);
+	__add_taint_for_CI(taint);
 }
 
-#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
-static unsigned int i915_probe_fail_count;
-
-int __i915_inject_probe_error(struct drm_i915_private *i915, int err,
-			      const char *func, int line)
-{
-	if (i915_probe_fail_count >= i915_modparams.inject_probe_failure)
-		return 0;
-
-	if (++i915_probe_fail_count < i915_modparams.inject_probe_failure)
-		return 0;
-
-	drm_info(&i915->drm, "Injecting failure %d at checkpoint %u [%s:%d]\n",
-		 err, i915_modparams.inject_probe_failure, func, line);
-
-	i915_modparams.inject_probe_failure = 0;
-	return err;
-}
-
-bool i915_error_injected(void)
-{
-	return i915_probe_fail_count && !i915_modparams.inject_probe_failure;
-}
-
-#endif
-
 bool i915_vtd_active(struct drm_i915_private *i915)
 {
 	if (device_iommu_mapped(i915->drm.dev))
diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h
index 4f75115b87d6..9e5b7fcadbe2 100644
--- a/drivers/gpu/drm/i915/i915_utils.h
+++ b/drivers/gpu/drm/i915/i915_utils.h
@@ -43,28 +43,8 @@ struct drm_i915_private;
 			     __stringify(x), (long)(x))
 #endif
 
-#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
-
-int __i915_inject_probe_error(struct drm_i915_private *i915, int err,
-			      const char *func, int line);
-#define i915_inject_probe_error(_i915, _err) \
-	__i915_inject_probe_error((_i915), (_err), __func__, __LINE__)
-bool i915_error_injected(void);
-
-#else
-
-#define i915_inject_probe_error(i915, e) ({ BUILD_BUG_ON_INVALID(i915); 0; })
-#define i915_error_injected() false
-
-#endif
-
-#define i915_inject_probe_failure(i915) i915_inject_probe_error((i915), -ENODEV)
-
 #define i915_probe_error(i915, fmt, ...) ({ \
-	if (i915_error_injected()) \
-		drm_dbg(&(i915)->drm, fmt, ##__VA_ARGS__); \
-	else \
-		drm_err(&(i915)->drm, fmt, ##__VA_ARGS__); \
+	drm_err(&(i915)->drm, fmt, ##__VA_ARGS__); \
 })
 
 #ifndef fetch_and_zero
diff --git a/drivers/gpu/drm/i915/intel_clock_gating.c b/drivers/gpu/drm/i915/intel_clock_gating.c
index 175a240ac848..7336934bb934 100644
--- a/drivers/gpu/drm/i915/intel_clock_gating.c
+++ b/drivers/gpu/drm/i915/intel_clock_gating.c
@@ -709,8 +709,10 @@ static void i830_init_clock_gating(struct drm_i915_private *i915)
 			   _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
 }
 
-void intel_clock_gating_init(struct drm_i915_private *i915)
+void intel_clock_gating_init(struct drm_device *drm)
 {
+	struct drm_i915_private *i915 = to_i915(drm);
+
 	i915->clock_gating_funcs->init_clock_gating(i915);
 }
 
@@ -749,15 +751,17 @@ CG_FUNCS(nop);
 
 /**
  * intel_clock_gating_hooks_init - setup the clock gating hooks
- * @i915: device private
+ * @drm: drm device
  *
  * Setup the hooks that configure which clocks of a given platform can be
  * gated and also apply various GT and display specific workarounds for these
  * platforms. Note that some GT specific workarounds are applied separately
  * when GPU contexts or batchbuffers start their execution.
  */
-void intel_clock_gating_hooks_init(struct drm_i915_private *i915)
+void intel_clock_gating_hooks_init(struct drm_device *drm)
 {
+	struct drm_i915_private *i915 = to_i915(drm);
+
 	if (IS_DG2(i915))
 		i915->clock_gating_funcs = &dg2_clock_gating_funcs;
 	else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
diff --git a/drivers/gpu/drm/i915/intel_clock_gating.h b/drivers/gpu/drm/i915/intel_clock_gating.h
index 5b4e4c55b2c2..3a4b443d9b8b 100644
--- a/drivers/gpu/drm/i915/intel_clock_gating.h
+++ b/drivers/gpu/drm/i915/intel_clock_gating.h
@@ -6,9 +6,9 @@
 #ifndef __INTEL_CLOCK_GATING_H__
 #define __INTEL_CLOCK_GATING_H__
 
-struct drm_i915_private;
+struct drm_device;
 
-void intel_clock_gating_init(struct drm_i915_private *i915);
-void intel_clock_gating_hooks_init(struct drm_i915_private *i915);
+void intel_clock_gating_init(struct drm_device *drm);
+void intel_clock_gating_hooks_init(struct drm_device *drm);
 
 #endif /* __INTEL_CLOCK_GATING_H__ */
diff --git a/drivers/gpu/drm/i915/intel_gvt.c b/drivers/gpu/drm/i915/intel_gvt.c
index c3efc3454ec2..5f615ec0e580 100644
--- a/drivers/gpu/drm/i915/intel_gvt.c
+++ b/drivers/gpu/drm/i915/intel_gvt.c
@@ -238,9 +238,6 @@ EXPORT_SYMBOL_NS_GPL(intel_gvt_clear_ops, "I915_GVT");
  */
 int intel_gvt_init(struct drm_i915_private *dev_priv)
 {
-	if (i915_inject_probe_failure(dev_priv))
-		return -ENODEV;
-
 	mutex_lock(&intel_gvt_mutex);
 	list_add_tail(&dev_priv->vgpu.entry, &intel_gvt_devices);
 	if (intel_gvt_ops)
diff --git a/drivers/gpu/drm/i915/intel_mchbar_regs.h b/drivers/gpu/drm/i915/intel_mchbar_regs.h
index dc2477179c3e..614d4017b57b 100644
--- a/drivers/gpu/drm/i915/intel_mchbar_regs.h
+++ b/drivers/gpu/drm/i915/intel_mchbar_regs.h
@@ -130,11 +130,11 @@
 #define   DG1_DRAM_T_RAS_MASK			REG_GENMASK(8, 1)
 
 #define SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5000)
-#define   SKL_DRAM_DDR_TYPE_MASK		(0x3 << 0)
-#define   SKL_DRAM_DDR_TYPE_DDR4		(0 << 0)
-#define   SKL_DRAM_DDR_TYPE_DDR3		(1 << 0)
-#define   SKL_DRAM_DDR_TYPE_LPDDR3		(2 << 0)
-#define   SKL_DRAM_DDR_TYPE_LPDDR4		(3 << 0)
+#define   SKL_DRAM_DDR_TYPE_MASK		REG_GENMASK(1, 0)
+#define   SKL_DRAM_DDR_TYPE_DDR4		REG_FIELD_PREP(SKL_DRAM_DDR_TYPE_MASK, 0)
+#define   SKL_DRAM_DDR_TYPE_DDR3		REG_FIELD_PREP(SKL_DRAM_DDR_TYPE_MASK, 1)
+#define   SKL_DRAM_DDR_TYPE_LPDDR3		REG_FIELD_PREP(SKL_DRAM_DDR_TYPE_MASK, 2)
+#define   SKL_DRAM_DDR_TYPE_LPDDR4		REG_FIELD_PREP(SKL_DRAM_DDR_TYPE_MASK, 3)
 
 /* snb MCH registers for reading the DRAM channel configuration */
 #define MAD_DIMM_C0				_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5004)
@@ -160,30 +160,40 @@
 
 #define SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x500C)
 #define SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5010)
-#define   SKL_DRAM_S_SHIFT			16
-#define   SKL_DRAM_SIZE_MASK			0x3F
-#define   SKL_DRAM_WIDTH_MASK			(0x3 << 8)
-#define   SKL_DRAM_WIDTH_SHIFT			8
-#define   SKL_DRAM_WIDTH_X8			(0x0 << 8)
-#define   SKL_DRAM_WIDTH_X16			(0x1 << 8)
-#define   SKL_DRAM_WIDTH_X32			(0x2 << 8)
-#define   SKL_DRAM_RANK_MASK			(0x1 << 10)
-#define   SKL_DRAM_RANK_SHIFT			10
-#define   SKL_DRAM_RANK_1			(0x0 << 10)
-#define   SKL_DRAM_RANK_2			(0x1 << 10)
-#define   SKL_DRAM_RANK_MASK			(0x1 << 10)
-#define   ICL_DRAM_SIZE_MASK			0x7F
-#define   ICL_DRAM_WIDTH_MASK			(0x3 << 7)
-#define   ICL_DRAM_WIDTH_SHIFT			7
-#define   ICL_DRAM_WIDTH_X8			(0x0 << 7)
-#define   ICL_DRAM_WIDTH_X16			(0x1 << 7)
-#define   ICL_DRAM_WIDTH_X32			(0x2 << 7)
-#define   ICL_DRAM_RANK_MASK			(0x3 << 9)
-#define   ICL_DRAM_RANK_SHIFT			9
-#define   ICL_DRAM_RANK_1			(0x0 << 9)
-#define   ICL_DRAM_RANK_2			(0x1 << 9)
-#define   ICL_DRAM_RANK_3			(0x2 << 9)
-#define   ICL_DRAM_RANK_4			(0x3 << 9)
+#define   SKL_DIMM_S_RANK_MASK			REG_GENMASK(26, 26)
+#define   SKL_DIMM_S_RANK_1			REG_FIELD_PREP(SKL_DIMM_S_RANK_MASK, 0)
+#define   SKL_DIMM_S_RANK_2			REG_FIELD_PREP(SKL_DIMM_S_RANK_MASK, 1)
+#define   SKL_DIMM_S_WIDTH_MASK			REG_GENMASK(25, 24)
+#define   SKL_DIMM_S_WIDTH_X8			REG_FIELD_PREP(SKL_DIMM_S_WIDTH_MASK, 0)
+#define   SKL_DIMM_S_WIDTH_X16			REG_FIELD_PREP(SKL_DIMM_S_WIDTH_MASK, 1)
+#define   SKL_DIMM_S_WIDTH_X32			REG_FIELD_PREP(SKL_DIMM_S_WIDTH_MASK, 2)
+#define   SKL_DIMM_S_SIZE_MASK			REG_GENMASK(21, 16)
+#define   SKL_DIMM_L_RANK_MASK			REG_GENMASK(10, 10)
+#define   SKL_DIMM_L_RANK_1			REG_FIELD_PREP(SKL_DIMM_L_RANK_MASK, 0)
+#define   SKL_DIMM_L_RANK_2			REG_FIELD_PREP(SKL_DIMM_L_RANK_MASK, 1)
+#define   SKL_DIMM_L_WIDTH_MASK			REG_GENMASK(9, 8)
+#define   SKL_DIMM_L_WIDTH_X8			REG_FIELD_PREP(SKL_DIMM_L_WIDTH_MASK, 0)
+#define   SKL_DIMM_L_WIDTH_X16			REG_FIELD_PREP(SKL_DIMM_L_WIDTH_MASK, 1)
+#define   SKL_DIMM_L_WIDTH_X32			REG_FIELD_PREP(SKL_DIMM_L_WIDTH_MASK, 2)
+#define   SKL_DIMM_L_SIZE_MASK			REG_GENMASK(5, 0)
+#define   ICL_DIMM_S_RANK_MASK			REG_GENMASK(27, 26)
+#define   ICL_DIMM_S_RANK_1			REG_FIELD_PREP(ICL_DIMM_S_RANK_MASK, 0)
+#define   ICL_DIMM_S_RANK_2			REG_FIELD_PREP(ICL_DIMM_S_RANK_MASK, 1)
+#define   ICL_DIMM_S_WIDTH_MASK			REG_GENMASK(25, 24)
+#define   ICL_DIMM_S_WIDTH_X8			REG_FIELD_PREP(ICL_DIMM_S_WIDTH_MASK, 0)
+#define   ICL_DIMM_S_WIDTH_X16			REG_FIELD_PREP(ICL_DIMM_S_WIDTH_MASK, 1)
+#define   ICL_DIMM_S_WIDTH_X32			REG_FIELD_PREP(ICL_DIMM_S_WIDTH_MASK, 2)
+#define   ICL_DIMM_S_SIZE_MASK			REG_GENMASK(22, 16)
+#define   ICL_DIMM_L_RANK_MASK			REG_GENMASK(10, 9)
+#define   ICL_DIMM_L_RANK_1			REG_FIELD_PREP(ICL_DIMM_L_RANK_MASK, 0)
+#define   ICL_DIMM_L_RANK_2			REG_FIELD_PREP(ICL_DIMM_L_RANK_MASK, 1)
+#define   ICL_DIMM_L_RANK_3			REG_FIELD_PREP(ICL_DIMM_L_RANK_MASK, 2)
+#define   ICL_DIMM_L_RANK_4			REG_FIELD_PREP(ICL_DIMM_L_RANK_MASK, 3)
+#define   ICL_DIMM_L_WIDTH_MASK			REG_GENMASK(8, 7)
+#define   ICL_DIMM_L_WIDTH_X8			REG_FIELD_PREP(ICL_DIMM_L_WIDTH_MASK, 0)
+#define   ICL_DIMM_L_WIDTH_X16			REG_FIELD_PREP(ICL_DIMM_L_WIDTH_MASK, 1)
+#define   ICL_DIMM_L_WIDTH_X32			REG_FIELD_PREP(ICL_DIMM_L_WIDTH_MASK, 2)
+#define   ICL_DIMM_L_SIZE_MASK			REG_GENMASK(6, 0)
 
 #define SA_PERF_STATUS_0_0_0_MCHBAR_PC		_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5918)
 #define  DG1_QCLK_RATIO_MASK			REG_GENMASK(9, 2)
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index 4adeb271fcbf..fdd2a940f983 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -2072,9 +2072,6 @@ static int __fw_domain_init(struct intel_uncore *uncore,
 	GEM_BUG_ON(domain_id >= FW_DOMAIN_ID_COUNT);
 	GEM_BUG_ON(uncore->fw_domain[domain_id]);
 
-	if (i915_inject_probe_failure(uncore->i915))
-		return -ENOMEM;
-
 	d = kzalloc(sizeof(*d), GFP_KERNEL);
 	if (!d)
 		return -ENOMEM;
@@ -2118,6 +2115,7 @@ static int __fw_domain_init(struct intel_uncore *uncore,
 
 	return 0;
 }
+ALLOW_ERROR_INJECTION(__fw_domain_init, ERRNO);
 
 static void fw_domain_fini(struct intel_uncore *uncore,
 			   enum forcewake_domain_id domain_id)
diff --git a/drivers/gpu/drm/i915/intel_wakeref.c b/drivers/gpu/drm/i915/intel_wakeref.c
index b1883dccc22a..98e7cee4e1dc 100644
--- a/drivers/gpu/drm/i915/intel_wakeref.c
+++ b/drivers/gpu/drm/i915/intel_wakeref.c
@@ -80,7 +80,7 @@ void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags)
 	/* Assume we are not in process context and so cannot sleep. */
 	if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) {
 		mod_delayed_work(wf->i915->unordered_wq, &wf->work,
-				 FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags));
+				 FIELD_GET(INTEL_WAKEREF_PUT_DELAY_MASK, flags));
 		return;
 	}
 
diff --git a/drivers/gpu/drm/i915/intel_wakeref.h b/drivers/gpu/drm/i915/intel_wakeref.h
index a2894a56e18f..81308bac34ba 100644
--- a/drivers/gpu/drm/i915/intel_wakeref.h
+++ b/drivers/gpu/drm/i915/intel_wakeref.h
@@ -128,17 +128,16 @@ intel_wakeref_get_if_active(struct intel_wakeref *wf)
 	return atomic_inc_not_zero(&wf->count);
 }
 
-enum {
-	INTEL_WAKEREF_PUT_ASYNC_BIT = 0,
-	__INTEL_WAKEREF_PUT_LAST_BIT__
-};
-
 static inline void
 intel_wakeref_might_get(struct intel_wakeref *wf)
 {
 	might_lock(&wf->mutex);
 }
 
+/* flags for __intel_wakeref_put() and __intel_wakeref_put_last */
+#define INTEL_WAKEREF_PUT_ASYNC		BIT(0)
+#define INTEL_WAKEREF_PUT_DELAY_MASK	GENMASK(BITS_PER_LONG - 1, 1)
+
 /**
  * __intel_wakeref_put: Release the wakeref
  * @wf: the wakeref
@@ -154,9 +153,6 @@ intel_wakeref_might_get(struct intel_wakeref *wf)
  */
 static inline void
 __intel_wakeref_put(struct intel_wakeref *wf, unsigned long flags)
-#define INTEL_WAKEREF_PUT_ASYNC BIT(INTEL_WAKEREF_PUT_ASYNC_BIT)
-#define INTEL_WAKEREF_PUT_DELAY \
-	GENMASK(BITS_PER_LONG - 1, __INTEL_WAKEREF_PUT_LAST_BIT__)
 {
 	INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
 	if (unlikely(!atomic_add_unless(&wf->count, -1, 1)))
@@ -181,7 +177,7 @@ intel_wakeref_put_delay(struct intel_wakeref *wf, unsigned long delay)
 {
 	__intel_wakeref_put(wf,
 			    INTEL_WAKEREF_PUT_ASYNC |
-			    FIELD_PREP(INTEL_WAKEREF_PUT_DELAY, delay));
+			    FIELD_PREP(INTEL_WAKEREF_PUT_DELAY_MASK, delay));
 }
 
 static inline void
diff --git a/drivers/gpu/drm/i915/soc/intel_dram.c b/drivers/gpu/drm/i915/soc/intel_dram.c
deleted file mode 100644
index 3e588762709a..000000000000
--- a/drivers/gpu/drm/i915/soc/intel_dram.c
+++ /dev/null
@@ -1,823 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright © 2020 Intel Corporation
- */
-
-#include <linux/string_helpers.h>
-
-#include <drm/drm_managed.h>
-#include <drm/drm_print.h>
-
-#include "../display/intel_display_core.h" /* FIXME */
-
-#include "i915_drv.h"
-#include "i915_reg.h"
-#include "i915_utils.h"
-#include "intel_dram.h"
-#include "intel_mchbar_regs.h"
-#include "intel_pcode.h"
-#include "intel_uncore.h"
-#include "vlv_iosf_sb.h"
-
-struct dram_dimm_info {
-	u16 size;
-	u8 width, ranks;
-};
-
-struct dram_channel_info {
-	struct dram_dimm_info dimm_l, dimm_s;
-	u8 ranks;
-	bool is_16gb_dimm;
-};
-
-#define DRAM_TYPE_STR(type) [INTEL_DRAM_ ## type] = #type
-
-const char *intel_dram_type_str(enum intel_dram_type type)
-{
-	static const char * const str[] = {
-		DRAM_TYPE_STR(UNKNOWN),
-		DRAM_TYPE_STR(DDR2),
-		DRAM_TYPE_STR(DDR3),
-		DRAM_TYPE_STR(DDR4),
-		DRAM_TYPE_STR(LPDDR3),
-		DRAM_TYPE_STR(LPDDR4),
-		DRAM_TYPE_STR(DDR5),
-		DRAM_TYPE_STR(LPDDR5),
-		DRAM_TYPE_STR(GDDR),
-		DRAM_TYPE_STR(GDDR_ECC),
-	};
-
-	BUILD_BUG_ON(ARRAY_SIZE(str) != __INTEL_DRAM_TYPE_MAX);
-
-	if (type >= ARRAY_SIZE(str))
-		type = INTEL_DRAM_UNKNOWN;
-
-	return str[type];
-}
-
-#undef DRAM_TYPE_STR
-
-static enum intel_dram_type pnv_dram_type(struct drm_i915_private *i915)
-{
-	return intel_uncore_read(&i915->uncore, CSHRDDR3CTL) & CSHRDDR3CTL_DDR3 ?
-		INTEL_DRAM_DDR3 : INTEL_DRAM_DDR2;
-}
-
-static unsigned int pnv_mem_freq(struct drm_i915_private *dev_priv)
-{
-	u32 tmp;
-
-	tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
-
-	switch (tmp & CLKCFG_MEM_MASK) {
-	case CLKCFG_MEM_533:
-		return 533333;
-	case CLKCFG_MEM_667:
-		return 666667;
-	case CLKCFG_MEM_800:
-		return 800000;
-	}
-
-	return 0;
-}
-
-static unsigned int ilk_mem_freq(struct drm_i915_private *dev_priv)
-{
-	u16 ddrpll;
-
-	ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
-	switch (ddrpll & 0xff) {
-	case 0xc:
-		return 800000;
-	case 0x10:
-		return 1066667;
-	case 0x14:
-		return 1333333;
-	case 0x18:
-		return 1600000;
-	default:
-		drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
-			ddrpll & 0xff);
-		return 0;
-	}
-}
-
-static unsigned int chv_mem_freq(struct drm_i915_private *i915)
-{
-	u32 val;
-
-	vlv_iosf_sb_get(&i915->drm, BIT(VLV_IOSF_SB_CCK));
-	val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_CCK, CCK_FUSE_REG);
-	vlv_iosf_sb_put(&i915->drm, BIT(VLV_IOSF_SB_CCK));
-
-	switch ((val >> 2) & 0x7) {
-	case 3:
-		return 2000000;
-	default:
-		return 1600000;
-	}
-}
-
-static unsigned int vlv_mem_freq(struct drm_i915_private *i915)
-{
-	u32 val;
-
-	vlv_iosf_sb_get(&i915->drm, BIT(VLV_IOSF_SB_PUNIT));
-	val = vlv_iosf_sb_read(&i915->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS);
-	vlv_iosf_sb_put(&i915->drm, BIT(VLV_IOSF_SB_PUNIT));
-
-	switch ((val >> 6) & 3) {
-	case 0:
-	case 1:
-		return 800000;
-	case 2:
-		return 1066667;
-	case 3:
-		return 1333333;
-	}
-
-	return 0;
-}
-
-unsigned int intel_mem_freq(struct drm_i915_private *i915)
-{
-	if (IS_PINEVIEW(i915))
-		return pnv_mem_freq(i915);
-	else if (GRAPHICS_VER(i915) == 5)
-		return ilk_mem_freq(i915);
-	else if (IS_CHERRYVIEW(i915))
-		return chv_mem_freq(i915);
-	else if (IS_VALLEYVIEW(i915))
-		return vlv_mem_freq(i915);
-	else
-		return 0;
-}
-
-static unsigned int i9xx_fsb_freq(struct drm_i915_private *i915)
-{
-	u32 fsb;
-
-	/*
-	 * Note that this only reads the state of the FSB
-	 * straps, not the actual FSB frequency. Some BIOSen
-	 * let you configure each independently. Ideally we'd
-	 * read out the actual FSB frequency but sadly we
-	 * don't know which registers have that information,
-	 * and all the relevant docs have gone to bit heaven :(
-	 */
-	fsb = intel_uncore_read(&i915->uncore, CLKCFG) & CLKCFG_FSB_MASK;
-
-	if (IS_PINEVIEW(i915) || IS_MOBILE(i915)) {
-		switch (fsb) {
-		case CLKCFG_FSB_400:
-			return 400000;
-		case CLKCFG_FSB_533:
-			return 533333;
-		case CLKCFG_FSB_667:
-			return 666667;
-		case CLKCFG_FSB_800:
-			return 800000;
-		case CLKCFG_FSB_1067:
-			return 1066667;
-		case CLKCFG_FSB_1333:
-			return 1333333;
-		default:
-			MISSING_CASE(fsb);
-			return 1333333;
-		}
-	} else {
-		switch (fsb) {
-		case CLKCFG_FSB_400_ALT:
-			return 400000;
-		case CLKCFG_FSB_533:
-			return 533333;
-		case CLKCFG_FSB_667:
-			return 666667;
-		case CLKCFG_FSB_800:
-			return 800000;
-		case CLKCFG_FSB_1067_ALT:
-			return 1066667;
-		case CLKCFG_FSB_1333_ALT:
-			return 1333333;
-		case CLKCFG_FSB_1600_ALT:
-			return 1600000;
-		default:
-			MISSING_CASE(fsb);
-			return 1333333;
-		}
-	}
-}
-
-static unsigned int ilk_fsb_freq(struct drm_i915_private *dev_priv)
-{
-	u16 fsb;
-
-	fsb = intel_uncore_read16(&dev_priv->uncore, CSIPLL0) & 0x3ff;
-
-	switch (fsb) {
-	case 0x00c:
-		return 3200000;
-	case 0x00e:
-		return 3733333;
-	case 0x010:
-		return 4266667;
-	case 0x012:
-		return 4800000;
-	case 0x014:
-		return 5333333;
-	case 0x016:
-		return 5866667;
-	case 0x018:
-		return 6400000;
-	default:
-		drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n", fsb);
-		return 0;
-	}
-}
-
-unsigned int intel_fsb_freq(struct drm_i915_private *i915)
-{
-	if (GRAPHICS_VER(i915) == 5)
-		return ilk_fsb_freq(i915);
-	else if (GRAPHICS_VER(i915) == 3 || GRAPHICS_VER(i915) == 4)
-		return i9xx_fsb_freq(i915);
-	else
-		return 0;
-}
-
-static int i915_get_dram_info(struct drm_i915_private *i915, struct dram_info *dram_info)
-{
-	dram_info->fsb_freq = intel_fsb_freq(i915);
-	if (dram_info->fsb_freq)
-		drm_dbg(&i915->drm, "FSB frequency: %d kHz\n", dram_info->fsb_freq);
-
-	dram_info->mem_freq = intel_mem_freq(i915);
-	if (dram_info->mem_freq)
-		drm_dbg(&i915->drm, "DDR speed: %d kHz\n", dram_info->mem_freq);
-
-	if (IS_PINEVIEW(i915))
-		dram_info->type = pnv_dram_type(i915);
-
-	return 0;
-}
-
-static int intel_dimm_num_devices(const struct dram_dimm_info *dimm)
-{
-	return dimm->ranks * 64 / (dimm->width ?: 1);
-}
-
-/* Returns total Gb for the whole DIMM */
-static int skl_get_dimm_size(u16 val)
-{
-	return (val & SKL_DRAM_SIZE_MASK) * 8;
-}
-
-static int skl_get_dimm_width(u16 val)
-{
-	if (skl_get_dimm_size(val) == 0)
-		return 0;
-
-	switch (val & SKL_DRAM_WIDTH_MASK) {
-	case SKL_DRAM_WIDTH_X8:
-	case SKL_DRAM_WIDTH_X16:
-	case SKL_DRAM_WIDTH_X32:
-		val = (val & SKL_DRAM_WIDTH_MASK) >> SKL_DRAM_WIDTH_SHIFT;
-		return 8 << val;
-	default:
-		MISSING_CASE(val);
-		return 0;
-	}
-}
-
-static int skl_get_dimm_ranks(u16 val)
-{
-	if (skl_get_dimm_size(val) == 0)
-		return 0;
-
-	val = (val & SKL_DRAM_RANK_MASK) >> SKL_DRAM_RANK_SHIFT;
-
-	return val + 1;
-}
-
-/* Returns total Gb for the whole DIMM */
-static int icl_get_dimm_size(u16 val)
-{
-	return (val & ICL_DRAM_SIZE_MASK) * 8 / 2;
-}
-
-static int icl_get_dimm_width(u16 val)
-{
-	if (icl_get_dimm_size(val) == 0)
-		return 0;
-
-	switch (val & ICL_DRAM_WIDTH_MASK) {
-	case ICL_DRAM_WIDTH_X8:
-	case ICL_DRAM_WIDTH_X16:
-	case ICL_DRAM_WIDTH_X32:
-		val = (val & ICL_DRAM_WIDTH_MASK) >> ICL_DRAM_WIDTH_SHIFT;
-		return 8 << val;
-	default:
-		MISSING_CASE(val);
-		return 0;
-	}
-}
-
-static int icl_get_dimm_ranks(u16 val)
-{
-	if (icl_get_dimm_size(val) == 0)
-		return 0;
-
-	val = (val & ICL_DRAM_RANK_MASK) >> ICL_DRAM_RANK_SHIFT;
-
-	return val + 1;
-}
-
-static bool
-skl_is_16gb_dimm(const struct dram_dimm_info *dimm)
-{
-	/* Convert total Gb to Gb per DRAM device */
-	return dimm->size / (intel_dimm_num_devices(dimm) ?: 1) >= 16;
-}
-
-static void
-skl_dram_get_dimm_info(struct drm_i915_private *i915,
-		       struct dram_dimm_info *dimm,
-		       int channel, char dimm_name, u16 val)
-{
-	if (GRAPHICS_VER(i915) >= 11) {
-		dimm->size = icl_get_dimm_size(val);
-		dimm->width = icl_get_dimm_width(val);
-		dimm->ranks = icl_get_dimm_ranks(val);
-	} else {
-		dimm->size = skl_get_dimm_size(val);
-		dimm->width = skl_get_dimm_width(val);
-		dimm->ranks = skl_get_dimm_ranks(val);
-	}
-
-	drm_dbg_kms(&i915->drm,
-		    "CH%u DIMM %c size: %u Gb, width: X%u, ranks: %u, 16Gb+ DIMMs: %s\n",
-		    channel, dimm_name, dimm->size, dimm->width, dimm->ranks,
-		    str_yes_no(skl_is_16gb_dimm(dimm)));
-}
-
-static int
-skl_dram_get_channel_info(struct drm_i915_private *i915,
-			  struct dram_channel_info *ch,
-			  int channel, u32 val)
-{
-	skl_dram_get_dimm_info(i915, &ch->dimm_l,
-			       channel, 'L', val & 0xffff);
-	skl_dram_get_dimm_info(i915, &ch->dimm_s,
-			       channel, 'S', val >> 16);
-
-	if (ch->dimm_l.size == 0 && ch->dimm_s.size == 0) {
-		drm_dbg_kms(&i915->drm, "CH%u not populated\n", channel);
-		return -EINVAL;
-	}
-
-	if (ch->dimm_l.ranks == 2 || ch->dimm_s.ranks == 2)
-		ch->ranks = 2;
-	else if (ch->dimm_l.ranks == 1 && ch->dimm_s.ranks == 1)
-		ch->ranks = 2;
-	else
-		ch->ranks = 1;
-
-	ch->is_16gb_dimm = skl_is_16gb_dimm(&ch->dimm_l) ||
-		skl_is_16gb_dimm(&ch->dimm_s);
-
-	drm_dbg_kms(&i915->drm, "CH%u ranks: %u, 16Gb+ DIMMs: %s\n",
-		    channel, ch->ranks, str_yes_no(ch->is_16gb_dimm));
-
-	return 0;
-}
-
-static bool
-intel_is_dram_symmetric(const struct dram_channel_info *ch0,
-			const struct dram_channel_info *ch1)
-{
-	return !memcmp(ch0, ch1, sizeof(*ch0)) &&
-		(ch0->dimm_s.size == 0 ||
-		 !memcmp(&ch0->dimm_l, &ch0->dimm_s, sizeof(ch0->dimm_l)));
-}
-
-static int
-skl_dram_get_channels_info(struct drm_i915_private *i915, struct dram_info *dram_info)
-{
-	struct dram_channel_info ch0 = {}, ch1 = {};
-	u32 val;
-	int ret;
-
-	/* Assume 16Gb+ DIMMs are present until proven otherwise */
-	dram_info->has_16gb_dimms = true;
-
-	val = intel_uncore_read(&i915->uncore,
-				SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN);
-	ret = skl_dram_get_channel_info(i915, &ch0, 0, val);
-	if (ret == 0)
-		dram_info->num_channels++;
-
-	val = intel_uncore_read(&i915->uncore,
-				SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN);
-	ret = skl_dram_get_channel_info(i915, &ch1, 1, val);
-	if (ret == 0)
-		dram_info->num_channels++;
-
-	if (dram_info->num_channels == 0) {
-		drm_info(&i915->drm, "Number of memory channels is zero\n");
-		return -EINVAL;
-	}
-
-	if (ch0.ranks == 0 && ch1.ranks == 0) {
-		drm_info(&i915->drm, "couldn't get memory rank information\n");
-		return -EINVAL;
-	}
-
-	dram_info->has_16gb_dimms = ch0.is_16gb_dimm || ch1.is_16gb_dimm;
-
-	dram_info->symmetric_memory = intel_is_dram_symmetric(&ch0, &ch1);
-
-	drm_dbg_kms(&i915->drm, "Memory configuration is symmetric? %s\n",
-		    str_yes_no(dram_info->symmetric_memory));
-
-	drm_dbg_kms(&i915->drm, "16Gb+ DIMMs: %s\n",
-		    str_yes_no(dram_info->has_16gb_dimms));
-
-	return 0;
-}
-
-static enum intel_dram_type
-skl_get_dram_type(struct drm_i915_private *i915)
-{
-	u32 val;
-
-	val = intel_uncore_read(&i915->uncore,
-				SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN);
-
-	switch (val & SKL_DRAM_DDR_TYPE_MASK) {
-	case SKL_DRAM_DDR_TYPE_DDR3:
-		return INTEL_DRAM_DDR3;
-	case SKL_DRAM_DDR_TYPE_DDR4:
-		return INTEL_DRAM_DDR4;
-	case SKL_DRAM_DDR_TYPE_LPDDR3:
-		return INTEL_DRAM_LPDDR3;
-	case SKL_DRAM_DDR_TYPE_LPDDR4:
-		return INTEL_DRAM_LPDDR4;
-	default:
-		MISSING_CASE(val);
-		return INTEL_DRAM_UNKNOWN;
-	}
-}
-
-static int
-skl_get_dram_info(struct drm_i915_private *i915, struct dram_info *dram_info)
-{
-	int ret;
-
-	dram_info->type = skl_get_dram_type(i915);
-
-	ret = skl_dram_get_channels_info(i915, dram_info);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-/* Returns Gb per DRAM device */
-static int bxt_get_dimm_size(u32 val)
-{
-	switch (val & BXT_DRAM_SIZE_MASK) {
-	case BXT_DRAM_SIZE_4GBIT:
-		return 4;
-	case BXT_DRAM_SIZE_6GBIT:
-		return 6;
-	case BXT_DRAM_SIZE_8GBIT:
-		return 8;
-	case BXT_DRAM_SIZE_12GBIT:
-		return 12;
-	case BXT_DRAM_SIZE_16GBIT:
-		return 16;
-	default:
-		MISSING_CASE(val);
-		return 0;
-	}
-}
-
-static int bxt_get_dimm_width(u32 val)
-{
-	if (!bxt_get_dimm_size(val))
-		return 0;
-
-	val = (val & BXT_DRAM_WIDTH_MASK) >> BXT_DRAM_WIDTH_SHIFT;
-
-	return 8 << val;
-}
-
-static int bxt_get_dimm_ranks(u32 val)
-{
-	if (!bxt_get_dimm_size(val))
-		return 0;
-
-	switch (val & BXT_DRAM_RANK_MASK) {
-	case BXT_DRAM_RANK_SINGLE:
-		return 1;
-	case BXT_DRAM_RANK_DUAL:
-		return 2;
-	default:
-		MISSING_CASE(val);
-		return 0;
-	}
-}
-
-static enum intel_dram_type bxt_get_dimm_type(u32 val)
-{
-	if (!bxt_get_dimm_size(val))
-		return INTEL_DRAM_UNKNOWN;
-
-	switch (val & BXT_DRAM_TYPE_MASK) {
-	case BXT_DRAM_TYPE_DDR3:
-		return INTEL_DRAM_DDR3;
-	case BXT_DRAM_TYPE_LPDDR3:
-		return INTEL_DRAM_LPDDR3;
-	case BXT_DRAM_TYPE_DDR4:
-		return INTEL_DRAM_DDR4;
-	case BXT_DRAM_TYPE_LPDDR4:
-		return INTEL_DRAM_LPDDR4;
-	default:
-		MISSING_CASE(val);
-		return INTEL_DRAM_UNKNOWN;
-	}
-}
-
-static void bxt_get_dimm_info(struct dram_dimm_info *dimm, u32 val)
-{
-	dimm->width = bxt_get_dimm_width(val);
-	dimm->ranks = bxt_get_dimm_ranks(val);
-
-	/*
-	 * Size in register is Gb per DRAM device. Convert to total
-	 * Gb to match the way we report this for non-LP platforms.
-	 */
-	dimm->size = bxt_get_dimm_size(val) * intel_dimm_num_devices(dimm);
-}
-
-static int bxt_get_dram_info(struct drm_i915_private *i915, struct dram_info *dram_info)
-{
-	u32 val;
-	u8 valid_ranks = 0;
-	int i;
-
-	/*
-	 * Now read each DUNIT8/9/10/11 to check the rank of each dimms.
-	 */
-	for (i = BXT_D_CR_DRP0_DUNIT_START; i <= BXT_D_CR_DRP0_DUNIT_END; i++) {
-		struct dram_dimm_info dimm;
-		enum intel_dram_type type;
-
-		val = intel_uncore_read(&i915->uncore, BXT_D_CR_DRP0_DUNIT(i));
-		if (val == 0xFFFFFFFF)
-			continue;
-
-		dram_info->num_channels++;
-
-		bxt_get_dimm_info(&dimm, val);
-		type = bxt_get_dimm_type(val);
-
-		drm_WARN_ON(&i915->drm, type != INTEL_DRAM_UNKNOWN &&
-			    dram_info->type != INTEL_DRAM_UNKNOWN &&
-			    dram_info->type != type);
-
-		drm_dbg_kms(&i915->drm,
-			    "CH%u DIMM size: %u Gb, width: X%u, ranks: %u\n",
-			    i - BXT_D_CR_DRP0_DUNIT_START,
-			    dimm.size, dimm.width, dimm.ranks);
-
-		if (valid_ranks == 0)
-			valid_ranks = dimm.ranks;
-
-		if (type != INTEL_DRAM_UNKNOWN)
-			dram_info->type = type;
-	}
-
-	if (dram_info->type == INTEL_DRAM_UNKNOWN || valid_ranks == 0) {
-		drm_info(&i915->drm, "couldn't get memory information\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int icl_pcode_read_mem_global_info(struct drm_i915_private *dev_priv,
-					  struct dram_info *dram_info)
-{
-	u32 val = 0;
-	int ret;
-
-	ret = intel_pcode_read(&dev_priv->drm, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
-			       ICL_PCODE_MEM_SS_READ_GLOBAL_INFO, &val, NULL);
-	if (ret)
-		return ret;
-
-	if (GRAPHICS_VER(dev_priv) == 12) {
-		switch (val & 0xf) {
-		case 0:
-			dram_info->type = INTEL_DRAM_DDR4;
-			break;
-		case 1:
-			dram_info->type = INTEL_DRAM_DDR5;
-			break;
-		case 2:
-			dram_info->type = INTEL_DRAM_LPDDR5;
-			break;
-		case 3:
-			dram_info->type = INTEL_DRAM_LPDDR4;
-			break;
-		case 4:
-			dram_info->type = INTEL_DRAM_DDR3;
-			break;
-		case 5:
-			dram_info->type = INTEL_DRAM_LPDDR3;
-			break;
-		default:
-			MISSING_CASE(val & 0xf);
-			return -EINVAL;
-		}
-	} else {
-		switch (val & 0xf) {
-		case 0:
-			dram_info->type = INTEL_DRAM_DDR4;
-			break;
-		case 1:
-			dram_info->type = INTEL_DRAM_DDR3;
-			break;
-		case 2:
-			dram_info->type = INTEL_DRAM_LPDDR3;
-			break;
-		case 3:
-			dram_info->type = INTEL_DRAM_LPDDR4;
-			break;
-		default:
-			MISSING_CASE(val & 0xf);
-			return -EINVAL;
-		}
-	}
-
-	dram_info->num_channels = (val & 0xf0) >> 4;
-	dram_info->num_qgv_points = (val & 0xf00) >> 8;
-	dram_info->num_psf_gv_points = (val & 0x3000) >> 12;
-
-	return 0;
-}
-
-static int gen11_get_dram_info(struct drm_i915_private *i915, struct dram_info *dram_info)
-{
-	int ret;
-
-	ret = skl_dram_get_channels_info(i915, dram_info);
-	if (ret)
-		return ret;
-
-	return icl_pcode_read_mem_global_info(i915, dram_info);
-}
-
-static int gen12_get_dram_info(struct drm_i915_private *i915, struct dram_info *dram_info)
-{
-	return icl_pcode_read_mem_global_info(i915, dram_info);
-}
-
-static int xelpdp_get_dram_info(struct drm_i915_private *i915, struct dram_info *dram_info)
-{
-	struct intel_display *display = i915->display;
-	u32 val = intel_uncore_read(&i915->uncore, MTL_MEM_SS_INFO_GLOBAL);
-
-	switch (REG_FIELD_GET(MTL_DDR_TYPE_MASK, val)) {
-	case 0:
-		dram_info->type = INTEL_DRAM_DDR4;
-		break;
-	case 1:
-		dram_info->type = INTEL_DRAM_DDR5;
-		break;
-	case 2:
-		dram_info->type = INTEL_DRAM_LPDDR5;
-		break;
-	case 3:
-		dram_info->type = INTEL_DRAM_LPDDR4;
-		break;
-	case 4:
-		dram_info->type = INTEL_DRAM_DDR3;
-		break;
-	case 5:
-		dram_info->type = INTEL_DRAM_LPDDR3;
-		break;
-	case 8:
-		drm_WARN_ON(&i915->drm, !IS_DGFX(i915));
-		dram_info->type = INTEL_DRAM_GDDR;
-		break;
-	case 9:
-		drm_WARN_ON(&i915->drm, !IS_DGFX(i915));
-		dram_info->type = INTEL_DRAM_GDDR_ECC;
-		break;
-	default:
-		MISSING_CASE(val);
-		return -EINVAL;
-	}
-
-	dram_info->num_channels = REG_FIELD_GET(MTL_N_OF_POPULATED_CH_MASK, val);
-	dram_info->num_qgv_points = REG_FIELD_GET(MTL_N_OF_ENABLED_QGV_POINTS_MASK, val);
-	/* PSF GV points not supported in D14+ */
-
-	if (DISPLAY_VER(display) >= 35)
-		dram_info->ecc_impacting_de_bw = REG_FIELD_GET(XE3P_ECC_IMPACTING_DE, val);
-
-	return 0;
-}
-
-int intel_dram_detect(struct drm_i915_private *i915)
-{
-	struct intel_display *display = i915->display;
-	struct dram_info *dram_info;
-	int ret;
-
-	if (IS_DG2(i915) || !intel_display_device_present(display))
-		return 0;
-
-	dram_info = drmm_kzalloc(&i915->drm, sizeof(*dram_info), GFP_KERNEL);
-	if (!dram_info)
-		return -ENOMEM;
-
-	i915->dram_info = dram_info;
-
-	if (DISPLAY_VER(display) >= 14)
-		ret = xelpdp_get_dram_info(i915, dram_info);
-	else if (GRAPHICS_VER(i915) >= 12)
-		ret = gen12_get_dram_info(i915, dram_info);
-	else if (GRAPHICS_VER(i915) >= 11)
-		ret = gen11_get_dram_info(i915, dram_info);
-	else if (IS_BROXTON(i915) || IS_GEMINILAKE(i915))
-		ret = bxt_get_dram_info(i915, dram_info);
-	else if (GRAPHICS_VER(i915) >= 9)
-		ret = skl_get_dram_info(i915, dram_info);
-	else
-		ret = i915_get_dram_info(i915, dram_info);
-
-	drm_dbg_kms(&i915->drm, "DRAM type: %s\n",
-		    intel_dram_type_str(dram_info->type));
-
-	drm_dbg_kms(&i915->drm, "DRAM channels: %u\n", dram_info->num_channels);
-
-	drm_dbg_kms(&i915->drm, "Num QGV points %u\n", dram_info->num_qgv_points);
-	drm_dbg_kms(&i915->drm, "Num PSF GV points %u\n", dram_info->num_psf_gv_points);
-
-	/* TODO: Do we want to abort probe on dram detection failures? */
-	if (ret)
-		return 0;
-
-	return 0;
-}
-
-/*
- * Returns NULL for platforms that don't have dram info. Avoid overzealous NULL
- * checks, and prefer not dereferencing on platforms that shouldn't look at dram
- * info, to catch accidental and incorrect dram info checks.
- */
-const struct dram_info *intel_dram_info(struct drm_device *drm)
-{
-	struct drm_i915_private *i915 = to_i915(drm);
-
-	return i915->dram_info;
-}
-
-static u32 gen9_edram_size_mb(struct drm_i915_private *i915, u32 cap)
-{
-	static const u8 ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 };
-	static const u8 sets[4] = { 1, 1, 2, 2 };
-
-	return EDRAM_NUM_BANKS(cap) *
-		ways[EDRAM_WAYS_IDX(cap)] *
-		sets[EDRAM_SETS_IDX(cap)];
-}
-
-void intel_dram_edram_detect(struct drm_i915_private *i915)
-{
-	u32 edram_cap = 0;
-
-	if (!(IS_HASWELL(i915) || IS_BROADWELL(i915) || GRAPHICS_VER(i915) >= 9))
-		return;
-
-	edram_cap = intel_uncore_read_fw(&i915->uncore, HSW_EDRAM_CAP);
-
-	/* NB: We can't write IDICR yet because we don't have gt funcs set up */
-
-	if (!(edram_cap & EDRAM_ENABLED))
-		return;
-
-	/*
-	 * The needed capability bits for size calculation are not there with
-	 * pre gen9 so return 128MB always.
-	 */
-	if (GRAPHICS_VER(i915) < 9)
-		i915->edram_size_mb = 128;
-	else
-		i915->edram_size_mb = gen9_edram_size_mb(i915, edram_cap);
-
-	drm_info(&i915->drm, "Found %uMB of eDRAM\n", i915->edram_size_mb);
-}
diff --git a/drivers/gpu/drm/i915/soc/intel_dram.h b/drivers/gpu/drm/i915/soc/intel_dram.h
deleted file mode 100644
index 8475ee379daa..000000000000
--- a/drivers/gpu/drm/i915/soc/intel_dram.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2020 Intel Corporation
- */
-
-#ifndef __INTEL_DRAM_H__
-#define __INTEL_DRAM_H__
-
-#include <linux/types.h>
-
-struct drm_i915_private;
-struct drm_device;
-
-struct dram_info {
-	enum intel_dram_type {
-		INTEL_DRAM_UNKNOWN,
-		INTEL_DRAM_DDR2,
-		INTEL_DRAM_DDR3,
-		INTEL_DRAM_DDR4,
-		INTEL_DRAM_LPDDR3,
-		INTEL_DRAM_LPDDR4,
-		INTEL_DRAM_DDR5,
-		INTEL_DRAM_LPDDR5,
-		INTEL_DRAM_GDDR,
-		INTEL_DRAM_GDDR_ECC,
-		__INTEL_DRAM_TYPE_MAX,
-	} type;
-	unsigned int fsb_freq;
-	unsigned int mem_freq;
-	u8 num_channels;
-	u8 num_qgv_points;
-	u8 num_psf_gv_points;
-	bool ecc_impacting_de_bw; /* Only valid from Xe3p_LPD onward. */
-	bool symmetric_memory;
-	bool has_16gb_dimms;
-};
-
-void intel_dram_edram_detect(struct drm_i915_private *i915);
-int intel_dram_detect(struct drm_i915_private *i915);
-unsigned int intel_fsb_freq(struct drm_i915_private *i915);
-unsigned int intel_mem_freq(struct drm_i915_private *i915);
-const struct dram_info *intel_dram_info(struct drm_device *drm);
-const char *intel_dram_type_str(enum intel_dram_type type);
-
-#endif /* __INTEL_DRAM_H__ */
diff --git a/drivers/gpu/drm/i915/soc/intel_gmch.c b/drivers/gpu/drm/i915/soc/intel_gmch.c
deleted file mode 100644
index 271da30c8290..000000000000
--- a/drivers/gpu/drm/i915/soc/intel_gmch.c
+++ /dev/null
@@ -1,188 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#include <linux/pci.h>
-#include <linux/pnp.h>
-#include <linux/vgaarb.h>
-
-#include <drm/drm_managed.h>
-#include <drm/drm_print.h>
-#include <drm/intel/i915_drm.h>
-
-#include "../display/intel_display_core.h" /* FIXME */
-
-#include "i915_drv.h"
-#include "intel_gmch.h"
-#include "intel_pci_config.h"
-
-static void intel_gmch_bridge_release(struct drm_device *dev, void *bridge)
-{
-	pci_dev_put(bridge);
-}
-
-int intel_gmch_bridge_setup(struct drm_i915_private *i915)
-{
-	int domain = pci_domain_nr(to_pci_dev(i915->drm.dev)->bus);
-
-	i915->gmch.pdev = pci_get_domain_bus_and_slot(domain, 0, PCI_DEVFN(0, 0));
-	if (!i915->gmch.pdev) {
-		drm_err(&i915->drm, "bridge device not found\n");
-		return -EIO;
-	}
-
-	return drmm_add_action_or_reset(&i915->drm, intel_gmch_bridge_release,
-					i915->gmch.pdev);
-}
-
-static int mchbar_reg(struct drm_i915_private *i915)
-{
-	return GRAPHICS_VER(i915) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
-}
-
-/* Allocate space for the MCH regs if needed, return nonzero on error */
-static int
-intel_alloc_mchbar_resource(struct drm_i915_private *i915)
-{
-	u32 temp_lo, temp_hi = 0;
-	u64 mchbar_addr;
-	int ret;
-
-	if (GRAPHICS_VER(i915) >= 4)
-		pci_read_config_dword(i915->gmch.pdev, mchbar_reg(i915) + 4, &temp_hi);
-	pci_read_config_dword(i915->gmch.pdev, mchbar_reg(i915), &temp_lo);
-	mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
-
-	/* If ACPI doesn't have it, assume we need to allocate it ourselves */
-	if (IS_ENABLED(CONFIG_PNP) && mchbar_addr &&
-	    pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE))
-		return 0;
-
-	/* Get some space for it */
-	i915->gmch.mch_res.name = "i915 MCHBAR";
-	i915->gmch.mch_res.flags = IORESOURCE_MEM;
-	ret = pci_bus_alloc_resource(i915->gmch.pdev->bus,
-				     &i915->gmch.mch_res,
-				     MCHBAR_SIZE, MCHBAR_SIZE,
-				     PCIBIOS_MIN_MEM,
-				     0, pcibios_align_resource,
-				     i915->gmch.pdev);
-	if (ret) {
-		drm_dbg(&i915->drm, "failed bus alloc: %d\n", ret);
-		i915->gmch.mch_res.start = 0;
-		return ret;
-	}
-
-	if (GRAPHICS_VER(i915) >= 4)
-		pci_write_config_dword(i915->gmch.pdev, mchbar_reg(i915) + 4,
-				       upper_32_bits(i915->gmch.mch_res.start));
-
-	pci_write_config_dword(i915->gmch.pdev, mchbar_reg(i915),
-			       lower_32_bits(i915->gmch.mch_res.start));
-	return 0;
-}
-
-/* Setup MCHBAR if possible, return true if we should disable it again */
-void intel_gmch_bar_setup(struct drm_i915_private *i915)
-{
-	u32 temp;
-	bool enabled;
-
-	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
-		return;
-
-	i915->gmch.mchbar_need_disable = false;
-
-	if (IS_I915G(i915) || IS_I915GM(i915)) {
-		pci_read_config_dword(i915->gmch.pdev, DEVEN, &temp);
-		enabled = !!(temp & DEVEN_MCHBAR_EN);
-	} else {
-		pci_read_config_dword(i915->gmch.pdev, mchbar_reg(i915), &temp);
-		enabled = temp & 1;
-	}
-
-	/* If it's already enabled, don't have to do anything */
-	if (enabled)
-		return;
-
-	if (intel_alloc_mchbar_resource(i915))
-		return;
-
-	i915->gmch.mchbar_need_disable = true;
-
-	/* Space is allocated or reserved, so enable it. */
-	if (IS_I915G(i915) || IS_I915GM(i915)) {
-		pci_write_config_dword(i915->gmch.pdev, DEVEN,
-				       temp | DEVEN_MCHBAR_EN);
-	} else {
-		pci_read_config_dword(i915->gmch.pdev, mchbar_reg(i915), &temp);
-		pci_write_config_dword(i915->gmch.pdev, mchbar_reg(i915), temp | 1);
-	}
-}
-
-void intel_gmch_bar_teardown(struct drm_i915_private *i915)
-{
-	if (i915->gmch.mchbar_need_disable) {
-		if (IS_I915G(i915) || IS_I915GM(i915)) {
-			u32 deven_val;
-
-			pci_read_config_dword(i915->gmch.pdev, DEVEN,
-					      &deven_val);
-			deven_val &= ~DEVEN_MCHBAR_EN;
-			pci_write_config_dword(i915->gmch.pdev, DEVEN,
-					       deven_val);
-		} else {
-			u32 mchbar_val;
-
-			pci_read_config_dword(i915->gmch.pdev, mchbar_reg(i915),
-					      &mchbar_val);
-			mchbar_val &= ~1;
-			pci_write_config_dword(i915->gmch.pdev, mchbar_reg(i915),
-					       mchbar_val);
-		}
-	}
-
-	if (i915->gmch.mch_res.start)
-		release_resource(&i915->gmch.mch_res);
-}
-
-int intel_gmch_vga_set_state(struct drm_i915_private *i915, bool enable_decode)
-{
-	struct intel_display *display = i915->display;
-	unsigned int reg = DISPLAY_VER(display) >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
-	u16 gmch_ctrl;
-
-	if (pci_read_config_word(i915->gmch.pdev, reg, &gmch_ctrl)) {
-		drm_err(&i915->drm, "failed to read control word\n");
-		return -EIO;
-	}
-
-	if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !enable_decode)
-		return 0;
-
-	if (enable_decode)
-		gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
-	else
-		gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
-
-	if (pci_write_config_word(i915->gmch.pdev, reg, gmch_ctrl)) {
-		drm_err(&i915->drm, "failed to write control word\n");
-		return -EIO;
-	}
-
-	return 0;
-}
-
-unsigned int intel_gmch_vga_set_decode(struct pci_dev *pdev, bool enable_decode)
-{
-	struct drm_i915_private *i915 = pdev_to_i915(pdev);
-
-	intel_gmch_vga_set_state(i915, enable_decode);
-
-	if (enable_decode)
-		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
-		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
-	else
-		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
-}
diff --git a/drivers/gpu/drm/i915/soc/intel_gmch.h b/drivers/gpu/drm/i915/soc/intel_gmch.h
deleted file mode 100644
index 23be2d113afd..000000000000
--- a/drivers/gpu/drm/i915/soc/intel_gmch.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#ifndef __INTEL_GMCH_H__
-#define __INTEL_GMCH_H__
-
-#include <linux/types.h>
-
-struct pci_dev;
-struct drm_i915_private;
-
-int intel_gmch_bridge_setup(struct drm_i915_private *i915);
-void intel_gmch_bar_setup(struct drm_i915_private *i915);
-void intel_gmch_bar_teardown(struct drm_i915_private *i915);
-int intel_gmch_vga_set_state(struct drm_i915_private *i915, bool enable_decode);
-unsigned int intel_gmch_vga_set_decode(struct pci_dev *pdev, bool enable_decode);
-
-#endif /* __INTEL_GMCH_H__ */
diff --git a/drivers/gpu/drm/i915/soc/intel_rom.c b/drivers/gpu/drm/i915/soc/intel_rom.c
deleted file mode 100644
index 2f17dc856e7f..000000000000
--- a/drivers/gpu/drm/i915/soc/intel_rom.c
+++ /dev/null
@@ -1,161 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright © 2024 Intel Corporation
- */
-
-#include "i915_drv.h"
-#include "i915_reg.h"
-
-#include "intel_rom.h"
-#include "intel_uncore.h"
-
-struct intel_rom {
-	/* for PCI ROM */
-	struct pci_dev *pdev;
-	void __iomem *oprom;
-
-	/* for SPI */
-	struct intel_uncore *uncore;
-	loff_t offset;
-
-	size_t size;
-
-	u32 (*read32)(struct intel_rom *rom, loff_t offset);
-	u16 (*read16)(struct intel_rom *rom, loff_t offset);
-	void (*read_block)(struct intel_rom *rom, void *data, loff_t offset, size_t size);
-	void (*free)(struct intel_rom *rom);
-};
-
-static u32 spi_read32(struct intel_rom *rom, loff_t offset)
-{
-	intel_uncore_write(rom->uncore, PRIMARY_SPI_ADDRESS,
-			   rom->offset + offset);
-
-	return intel_uncore_read(rom->uncore, PRIMARY_SPI_TRIGGER);
-}
-
-static u16 spi_read16(struct intel_rom *rom, loff_t offset)
-{
-	return spi_read32(rom, offset) & 0xffff;
-}
-
-struct intel_rom *intel_rom_spi(struct drm_device *drm)
-{
-	struct drm_i915_private *i915 = to_i915(drm);
-	struct intel_rom *rom;
-	u32 static_region;
-
-	rom = kzalloc(sizeof(*rom), GFP_KERNEL);
-	if (!rom)
-		return NULL;
-
-	rom->uncore = &i915->uncore;
-
-	static_region = intel_uncore_read(rom->uncore, SPI_STATIC_REGIONS);
-	static_region &= OPTIONROM_SPI_REGIONID_MASK;
-	intel_uncore_write(rom->uncore, PRIMARY_SPI_REGIONID, static_region);
-
-	rom->offset = intel_uncore_read(rom->uncore, OROM_OFFSET) & OROM_OFFSET_MASK;
-
-	rom->size = 0x200000;
-
-	rom->read32 = spi_read32;
-	rom->read16 = spi_read16;
-
-	return rom;
-}
-
-static u32 pci_read32(struct intel_rom *rom, loff_t offset)
-{
-	return ioread32(rom->oprom + offset);
-}
-
-static u16 pci_read16(struct intel_rom *rom, loff_t offset)
-{
-	return ioread16(rom->oprom + offset);
-}
-
-static void pci_read_block(struct intel_rom *rom, void *data,
-			   loff_t offset, size_t size)
-{
-	memcpy_fromio(data, rom->oprom + offset, size);
-}
-
-static void pci_free(struct intel_rom *rom)
-{
-	pci_unmap_rom(rom->pdev, rom->oprom);
-}
-
-struct intel_rom *intel_rom_pci(struct drm_device *drm)
-{
-	struct intel_rom *rom;
-
-	rom = kzalloc(sizeof(*rom), GFP_KERNEL);
-	if (!rom)
-		return NULL;
-
-	rom->pdev = to_pci_dev(drm->dev);
-
-	rom->oprom = pci_map_rom(rom->pdev, &rom->size);
-	if (!rom->oprom) {
-		kfree(rom);
-		return NULL;
-	}
-
-	rom->read32 = pci_read32;
-	rom->read16 = pci_read16;
-	rom->read_block = pci_read_block;
-	rom->free = pci_free;
-
-	return rom;
-}
-
-u32 intel_rom_read32(struct intel_rom *rom, loff_t offset)
-{
-	return rom->read32(rom, offset);
-}
-
-u16 intel_rom_read16(struct intel_rom *rom, loff_t offset)
-{
-	return rom->read16(rom, offset);
-}
-
-void intel_rom_read_block(struct intel_rom *rom, void *data,
-			  loff_t offset, size_t size)
-{
-	u32 *ptr = data;
-	loff_t index;
-
-	if (rom->read_block) {
-		rom->read_block(rom, data, offset, size);
-		return;
-	}
-
-	for (index = 0; index < size; index += 4)
-		*ptr++ = rom->read32(rom, offset + index);
-}
-
-loff_t intel_rom_find(struct intel_rom *rom, u32 needle)
-{
-	loff_t offset;
-
-	for (offset = 0; offset < rom->size; offset += 4) {
-		if (rom->read32(rom, offset) == needle)
-			return offset;
-	}
-
-	return -ENOENT;
-}
-
-size_t intel_rom_size(struct intel_rom *rom)
-{
-	return rom->size;
-}
-
-void intel_rom_free(struct intel_rom *rom)
-{
-	if (rom && rom->free)
-		rom->free(rom);
-
-	kfree(rom);
-}
diff --git a/drivers/gpu/drm/i915/soc/intel_rom.h b/drivers/gpu/drm/i915/soc/intel_rom.h
deleted file mode 100644
index 4e59a375787e..000000000000
--- a/drivers/gpu/drm/i915/soc/intel_rom.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2024 Intel Corporation
- */
-
-#ifndef __INTEL_ROM_H__
-#define __INTEL_ROM_H__
-
-#include <linux/types.h>
-
-struct drm_device;
-struct intel_rom;
-
-struct intel_rom *intel_rom_spi(struct drm_device *drm);
-struct intel_rom *intel_rom_pci(struct drm_device *drm);
-
-u32 intel_rom_read32(struct intel_rom *rom, loff_t offset);
-u16 intel_rom_read16(struct intel_rom *rom, loff_t offset);
-void intel_rom_read_block(struct intel_rom *rom, void *data,
-			  loff_t offset, size_t size);
-loff_t intel_rom_find(struct intel_rom *rom, u32 needle);
-size_t intel_rom_size(struct intel_rom *rom);
-void intel_rom_free(struct intel_rom *rom);
-
-#endif /* __INTEL_ROM_H__ */
diff --git a/drivers/gpu/drm/i915/vlv_suspend.c b/drivers/gpu/drm/i915/vlv_suspend.c
index 221e4c0b2c58..bace7b38329b 100644
--- a/drivers/gpu/drm/i915/vlv_suspend.c
+++ b/drivers/gpu/drm/i915/vlv_suspend.c
@@ -453,7 +453,7 @@ int vlv_resume_prepare(struct drm_i915_private *dev_priv, bool rpm_resume)
 	vlv_check_no_gt_access(dev_priv);
 
 	if (rpm_resume)
-		intel_clock_gating_init(dev_priv);
+		intel_clock_gating_init(&dev_priv->drm);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/imagination/pvr_gem.c b/drivers/gpu/drm/imagination/pvr_gem.c
index a66cf082af24..c07c9a915190 100644
--- a/drivers/gpu/drm/imagination/pvr_gem.c
+++ b/drivers/gpu/drm/imagination/pvr_gem.c
@@ -28,6 +28,16 @@ static void pvr_gem_object_free(struct drm_gem_object *obj)
 	drm_gem_shmem_object_free(obj);
 }
 
+static struct dma_buf *pvr_gem_export(struct drm_gem_object *obj, int flags)
+{
+	struct pvr_gem_object *pvr_obj = gem_to_pvr_gem(obj);
+
+	if (pvr_obj->flags & DRM_PVR_BO_PM_FW_PROTECT)
+		return ERR_PTR(-EPERM);
+
+	return drm_gem_prime_export(obj, flags);
+}
+
 static int pvr_gem_mmap(struct drm_gem_object *gem_obj, struct vm_area_struct *vma)
 {
 	struct pvr_gem_object *pvr_obj = gem_to_pvr_gem(gem_obj);
@@ -42,6 +52,7 @@ static int pvr_gem_mmap(struct drm_gem_object *gem_obj, struct vm_area_struct *v
 static const struct drm_gem_object_funcs pvr_gem_object_funcs = {
 	.free = pvr_gem_object_free,
 	.print_info = drm_gem_shmem_object_print_info,
+	.export = pvr_gem_export,
 	.pin = drm_gem_shmem_object_pin,
 	.unpin = drm_gem_shmem_object_unpin,
 	.get_sg_table = drm_gem_shmem_object_get_sg_table,
diff --git a/drivers/gpu/drm/nouveau/dispnv04/crtc.c b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
index c063756eaea3..80493224eb6c 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/crtc.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
@@ -837,7 +837,7 @@ nv_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
 static int
 nv04_crtc_do_mode_set_base(struct drm_crtc *crtc,
 			   struct drm_framebuffer *passed_fb,
-			   int x, int y, bool atomic)
+			   int x, int y)
 {
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
@@ -850,19 +850,12 @@ nv04_crtc_do_mode_set_base(struct drm_crtc *crtc,
 	NV_DEBUG(drm, "index %d\n", nv_crtc->index);
 
 	/* no fb bound */
-	if (!atomic && !crtc->primary->fb) {
+	if (!crtc->primary->fb) {
 		NV_DEBUG(drm, "No FB bound\n");
 		return 0;
 	}
 
-	/* If atomic, we want to switch to the fb we were passed, so
-	 * now we update pointers to do that.
-	 */
-	if (atomic) {
-		drm_fb = passed_fb;
-	} else {
-		drm_fb = crtc->primary->fb;
-	}
+	drm_fb = crtc->primary->fb;
 
 	nvbo = nouveau_gem_object(drm_fb->obj[0]);
 	nv_crtc->fb.offset = nvbo->offset;
@@ -920,15 +913,7 @@ nv04_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
 	int ret = nv_crtc_swap_fbs(crtc, old_fb);
 	if (ret)
 		return ret;
-	return nv04_crtc_do_mode_set_base(crtc, old_fb, x, y, false);
-}
-
-static int
-nv04_crtc_mode_set_base_atomic(struct drm_crtc *crtc,
-			       struct drm_framebuffer *fb,
-			       int x, int y, enum mode_set_atomic state)
-{
-	return nv04_crtc_do_mode_set_base(crtc, fb, x, y, true);
+	return nv04_crtc_do_mode_set_base(crtc, old_fb, x, y);
 }
 
 static void nv04_cursor_upload(struct drm_device *dev, struct nouveau_bo *src,
@@ -1274,7 +1259,6 @@ static const struct drm_crtc_helper_funcs nv04_crtc_helper_funcs = {
 	.commit = nv_crtc_commit,
 	.mode_set = nv_crtc_mode_set,
 	.mode_set_base = nv04_crtc_mode_set_base,
-	.mode_set_base_atomic = nv04_crtc_mode_set_base_atomic,
 	.disable = nv_crtc_disable,
 	.get_scanout_position = nouveau_display_scanoutpos,
 };
diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig
index 76f6af819037..307152ad7759 100644
--- a/drivers/gpu/drm/panel/Kconfig
+++ b/drivers/gpu/drm/panel/Kconfig
@@ -781,6 +781,19 @@ config DRM_PANEL_SAMSUNG_LD9040
 	depends on BACKLIGHT_CLASS_DEVICE
 	select VIDEOMODE_HELPERS
 
+config DRM_PANEL_SAMSUNG_LTL106HL02
+	tristate "Samsung LTL106HL02 panel"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	select VIDEOMODE_HELPERS
+	help
+	  Say Y here if you want to enable support for the Samsung LTL106HL02
+	  panel driver which is used in Microsoft Surface 2.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called panel-samsung-ltl106hl02.
+
 config DRM_PANEL_SAMSUNG_S6E3FA7
 	tristate "Samsung S6E3FA7 panel driver"
 	depends on OF
@@ -1165,6 +1178,7 @@ config DRM_PANEL_VISIONOX_RM69299
 	tristate "Visionox RM69299"
 	depends on OF
 	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
 	help
 	  Say Y here if you want to enable support for Visionox
 	  RM69299  DSI Video Mode panel.
diff --git a/drivers/gpu/drm/panel/Makefile b/drivers/gpu/drm/panel/Makefile
index b9562a6fdcb3..aeffaa95666d 100644
--- a/drivers/gpu/drm/panel/Makefile
+++ b/drivers/gpu/drm/panel/Makefile
@@ -76,6 +76,7 @@ obj-$(CONFIG_DRM_PANEL_SAMSUNG_AMS639RQ08) += panel-samsung-ams639rq08.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_ATNA33XC20) += panel-samsung-atna33xc20.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_DB7430) += panel-samsung-db7430.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_LD9040) += panel-samsung-ld9040.o
+obj-$(CONFIG_DRM_PANEL_SAMSUNG_LTL106HL02) += panel-samsung-ltl106hl02.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_S6D16D0) += panel-samsung-s6d16d0.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_S6D27A1) += panel-samsung-s6d27a1.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_S6D7AA0) += panel-samsung-s6d7aa0.o
diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c
index 415b894890ad..85dd3f4cb8e1 100644
--- a/drivers/gpu/drm/panel/panel-edp.c
+++ b/drivers/gpu/drm/panel/panel-edp.c
@@ -1880,6 +1880,7 @@ static const struct panel_delay delay_80_500_e50_d50 = {
  */
 static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x04a4, &delay_200_500_e50, "B122UAN01.0"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x0ba4, &delay_200_500_e50, "B140QAX01.H"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x105c, &delay_200_500_e50, "B116XTN01.0"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x1062, &delay_200_500_e50, "B120XAN01.0"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x125c, &delay_200_500_e50, "Unknown"),
@@ -1904,6 +1905,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x615c, &delay_200_500_e50, "B116XAN06.1"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x635c, &delay_200_500_e50, "B116XAN06.3"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x639c, &delay_200_500_e50, "B140HAK02.7"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x643d, &delay_200_500_e50, "B140HAN06.4"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x723c, &delay_200_500_e50, "B140XTN07.2"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x73aa, &delay_200_500_e50, "B116XTN02.3"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x8594, &delay_200_500_e50, "B133UAN01.0"),
@@ -2033,6 +2035,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1462, &delay_200_500_e50, "MNE007QS5-2"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1468, &delay_200_500_e50, "MNE007QB2-2"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x146e, &delay_80_500_e50_d50, "MNE007QB3-1"),
+	EDP_PANEL_ENTRY('C', 'S', 'W', 0x147c, &delay_200_500_e50_d100, "MNE007QB3-1"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1519, &delay_200_500_e80_d50, "MNF601BS1-3"),
 
 	EDP_PANEL_ENTRY('E', 'T', 'C', 0x0000, &delay_50_500_e200_d200_po2e335, "LP079QX1-SP0V"),
diff --git a/drivers/gpu/drm/panel/panel-lg-sw43408.c b/drivers/gpu/drm/panel/panel-lg-sw43408.c
index 46a56ea92ad9..293826c5006b 100644
--- a/drivers/gpu/drm/panel/panel-lg-sw43408.c
+++ b/drivers/gpu/drm/panel/panel-lg-sw43408.c
@@ -20,13 +20,18 @@
 #include <drm/display/drm_dsc.h>
 #include <drm/display/drm_dsc_helper.h>
 
-#define NUM_SUPPLIES 2
+static const struct regulator_bulk_data sw43408_supplies[] = {
+	{ .supply = "vddi", /* 1.8 V */
+	  .init_load_uA = 62000 },
+	{ .supply = "vpnl", /* 3.0 V */
+	  .init_load_uA = 857000 },
+};
 
 struct sw43408_panel {
 	struct drm_panel base;
 	struct mipi_dsi_device *link;
 
-	struct regulator_bulk_data supplies[NUM_SUPPLIES];
+	struct regulator_bulk_data *supplies;
 
 	struct gpio_desc *reset_gpio;
 
@@ -52,7 +57,7 @@ static int sw43408_unprepare(struct drm_panel *panel)
 
 	gpiod_set_value(sw43408->reset_gpio, 1);
 
-	ret = regulator_bulk_disable(ARRAY_SIZE(sw43408->supplies), sw43408->supplies);
+	ret = regulator_bulk_disable(ARRAY_SIZE(sw43408_supplies), sw43408->supplies);
 
 	return ret ? : ctx.accum_err;
 }
@@ -119,23 +124,28 @@ static int sw43408_program(struct drm_panel *panel)
 	return ctx.accum_err;
 }
 
+static void sw43408_reset(struct sw43408_panel *ctx)
+{
+	gpiod_set_value(ctx->reset_gpio, 0);
+	usleep_range(9000, 10000);
+	gpiod_set_value(ctx->reset_gpio, 1);
+	usleep_range(1000, 2000);
+	gpiod_set_value(ctx->reset_gpio, 0);
+	usleep_range(9000, 10000);
+}
+
 static int sw43408_prepare(struct drm_panel *panel)
 {
 	struct sw43408_panel *ctx = to_panel_info(panel);
 	int ret;
 
-	ret = regulator_bulk_enable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+	ret = regulator_bulk_enable(ARRAY_SIZE(sw43408_supplies), ctx->supplies);
 	if (ret < 0)
 		return ret;
 
 	usleep_range(5000, 6000);
 
-	gpiod_set_value(ctx->reset_gpio, 0);
-	usleep_range(9000, 10000);
-	gpiod_set_value(ctx->reset_gpio, 1);
-	usleep_range(1000, 2000);
-	gpiod_set_value(ctx->reset_gpio, 0);
-	usleep_range(9000, 10000);
+	sw43408_reset(ctx);
 
 	ret = sw43408_program(panel);
 	if (ret)
@@ -145,11 +155,11 @@ static int sw43408_prepare(struct drm_panel *panel)
 
 poweroff:
 	gpiod_set_value(ctx->reset_gpio, 1);
-	regulator_bulk_disable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+	regulator_bulk_disable(ARRAY_SIZE(sw43408_supplies), ctx->supplies);
 	return ret;
 }
 
-static const struct drm_display_mode sw43408_mode = {
+static const struct drm_display_mode lh546wf1_ed01_mode = {
 	.clock = (1080 + 20 + 32 + 20) * (2160 + 20 + 4 + 20) * 60 / 1000,
 
 	.hdisplay = 1080,
@@ -171,7 +181,7 @@ static const struct drm_display_mode sw43408_mode = {
 static int sw43408_get_modes(struct drm_panel *panel,
 			     struct drm_connector *connector)
 {
-	return drm_connector_helper_get_modes_fixed(connector, &sw43408_mode);
+	return drm_connector_helper_get_modes_fixed(connector, &lh546wf1_ed01_mode);
 }
 
 static int sw43408_backlight_update_status(struct backlight_device *bl)
@@ -214,7 +224,8 @@ static const struct drm_panel_funcs sw43408_funcs = {
 };
 
 static const struct of_device_id sw43408_of_match[] = {
-	{ .compatible = "lg,sw43408", },
+	{ .compatible = "lg,sw43408", }, /* legacy */
+	{ .compatible = "lg,sw43408-lh546wf1-ed01", },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, sw43408_of_match);
@@ -224,20 +235,17 @@ static int sw43408_add(struct sw43408_panel *ctx)
 	struct device *dev = &ctx->link->dev;
 	int ret;
 
-	ctx->supplies[0].supply = "vddi"; /* 1.88 V */
-	ctx->supplies[0].init_load_uA = 62000;
-	ctx->supplies[1].supply = "vpnl"; /* 3.0 V */
-	ctx->supplies[1].init_load_uA = 857000;
-
-	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(ctx->supplies),
-				      ctx->supplies);
+	ret = devm_regulator_bulk_get_const(dev,
+					    ARRAY_SIZE(sw43408_supplies),
+					    sw43408_supplies,
+					    &ctx->supplies);
 	if (ret < 0)
 		return ret;
 
 	ctx->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
 	if (IS_ERR(ctx->reset_gpio)) {
-		ret = PTR_ERR(ctx->reset_gpio);
-		return dev_err_probe(dev, ret, "cannot get reset gpio\n");
+		return dev_err_probe(dev, PTR_ERR(ctx->reset_gpio),
+				     "Failed to get reset-gpios\n");
 	}
 
 	ret = sw43408_backlight_init(ctx);
@@ -294,10 +302,6 @@ static void sw43408_remove(struct mipi_dsi_device *dsi)
 	struct sw43408_panel *ctx = mipi_dsi_get_drvdata(dsi);
 	int ret;
 
-	ret = sw43408_unprepare(&ctx->base);
-	if (ret < 0)
-		dev_err(&dsi->dev, "failed to unprepare panel: %d\n", ret);
-
 	ret = mipi_dsi_detach(dsi);
 	if (ret < 0)
 		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret);
diff --git a/drivers/gpu/drm/panel/panel-orisetech-otm8009a.c b/drivers/gpu/drm/panel/panel-orisetech-otm8009a.c
index a0f58c3b73f6..60701521c3b1 100644
--- a/drivers/gpu/drm/panel/panel-orisetech-otm8009a.c
+++ b/drivers/gpu/drm/panel/panel-orisetech-otm8009a.c
@@ -109,177 +109,140 @@ static inline struct otm8009a *panel_to_otm8009a(struct drm_panel *panel)
 	return container_of(panel, struct otm8009a, panel);
 }
 
-static void otm8009a_dcs_write_buf(struct otm8009a *ctx, const void *data,
-				   size_t len)
-{
-	struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
-
-	if (mipi_dsi_dcs_write_buffer(dsi, data, len) < 0)
-		dev_warn(ctx->dev, "mipi dsi dcs write buffer failed\n");
-}
-
-#define dcs_write_seq(ctx, seq...)			\
-({							\
-	static const u8 d[] = { seq };			\
-	otm8009a_dcs_write_buf(ctx, d, ARRAY_SIZE(d));	\
-})
-
 #define dcs_write_cmd_at(ctx, cmd, seq...)		\
 ({							\
-	dcs_write_seq(ctx, MCS_ADRSFT, (cmd) & 0xFF);	\
-	dcs_write_seq(ctx, (cmd) >> 8, seq);		\
+	mipi_dsi_dcs_write_seq_multi(ctx, MCS_ADRSFT, (cmd) & 0xFF);	\
+	mipi_dsi_dcs_write_seq_multi(ctx, (cmd) >> 8, seq);		\
 })
 
 static int otm8009a_init_sequence(struct otm8009a *ctx)
 {
 	struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
-	int ret;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
 
 	/* Enter CMD2 */
-	dcs_write_cmd_at(ctx, MCS_CMD2_ENA1, 0x80, 0x09, 0x01);
+	dcs_write_cmd_at(&dsi_ctx, MCS_CMD2_ENA1, 0x80, 0x09, 0x01);
 
 	/* Enter Orise Command2 */
-	dcs_write_cmd_at(ctx, MCS_CMD2_ENA2, 0x80, 0x09);
-
-	dcs_write_cmd_at(ctx, MCS_SD_PCH_CTRL, 0x30);
-	mdelay(10);
-
-	dcs_write_cmd_at(ctx, MCS_NO_DOC1, 0x40);
-	mdelay(10);
-
-	dcs_write_cmd_at(ctx, MCS_PWR_CTRL4 + 1, 0xA9);
-	dcs_write_cmd_at(ctx, MCS_PWR_CTRL2 + 1, 0x34);
-	dcs_write_cmd_at(ctx, MCS_P_DRV_M, 0x50);
-	dcs_write_cmd_at(ctx, MCS_VCOMDC, 0x4E);
-	dcs_write_cmd_at(ctx, MCS_OSC_ADJ, 0x66); /* 65Hz */
-	dcs_write_cmd_at(ctx, MCS_PWR_CTRL2 + 2, 0x01);
-	dcs_write_cmd_at(ctx, MCS_PWR_CTRL2 + 5, 0x34);
-	dcs_write_cmd_at(ctx, MCS_PWR_CTRL2 + 4, 0x33);
-	dcs_write_cmd_at(ctx, MCS_GVDDSET, 0x79, 0x79);
-	dcs_write_cmd_at(ctx, MCS_SD_CTRL + 1, 0x1B);
-	dcs_write_cmd_at(ctx, MCS_PWR_CTRL1 + 2, 0x83);
-	dcs_write_cmd_at(ctx, MCS_SD_PCH_CTRL + 1, 0x83);
-	dcs_write_cmd_at(ctx, MCS_RGB_VID_SET, 0x0E);
-	dcs_write_cmd_at(ctx, MCS_PANSET, 0x00, 0x01);
-
-	dcs_write_cmd_at(ctx, MCS_GOAVST, 0x85, 0x01, 0x00, 0x84, 0x01, 0x00);
-	dcs_write_cmd_at(ctx, MCS_GOACLKA1, 0x18, 0x04, 0x03, 0x39, 0x00, 0x00,
+	dcs_write_cmd_at(&dsi_ctx, MCS_CMD2_ENA2, 0x80, 0x09);
+
+	dcs_write_cmd_at(&dsi_ctx, MCS_SD_PCH_CTRL, 0x30);
+	mipi_dsi_msleep(&dsi_ctx, 10);
+
+	dcs_write_cmd_at(&dsi_ctx, MCS_NO_DOC1, 0x40);
+	mipi_dsi_msleep(&dsi_ctx, 10);
+
+	dcs_write_cmd_at(&dsi_ctx, MCS_PWR_CTRL4 + 1, 0xA9);
+	dcs_write_cmd_at(&dsi_ctx, MCS_PWR_CTRL2 + 1, 0x34);
+	dcs_write_cmd_at(&dsi_ctx, MCS_P_DRV_M, 0x50);
+	dcs_write_cmd_at(&dsi_ctx, MCS_VCOMDC, 0x4E);
+	dcs_write_cmd_at(&dsi_ctx, MCS_OSC_ADJ, 0x66); /* 65Hz */
+	dcs_write_cmd_at(&dsi_ctx, MCS_PWR_CTRL2 + 2, 0x01);
+	dcs_write_cmd_at(&dsi_ctx, MCS_PWR_CTRL2 + 5, 0x34);
+	dcs_write_cmd_at(&dsi_ctx, MCS_PWR_CTRL2 + 4, 0x33);
+	dcs_write_cmd_at(&dsi_ctx, MCS_GVDDSET, 0x79, 0x79);
+	dcs_write_cmd_at(&dsi_ctx, MCS_SD_CTRL + 1, 0x1B);
+	dcs_write_cmd_at(&dsi_ctx, MCS_PWR_CTRL1 + 2, 0x83);
+	dcs_write_cmd_at(&dsi_ctx, MCS_SD_PCH_CTRL + 1, 0x83);
+	dcs_write_cmd_at(&dsi_ctx, MCS_RGB_VID_SET, 0x0E);
+	dcs_write_cmd_at(&dsi_ctx, MCS_PANSET, 0x00, 0x01);
+
+	dcs_write_cmd_at(&dsi_ctx, MCS_GOAVST, 0x85, 0x01, 0x00, 0x84, 0x01, 0x00);
+	dcs_write_cmd_at(&dsi_ctx, MCS_GOACLKA1, 0x18, 0x04, 0x03, 0x39, 0x00, 0x00,
 			 0x00, 0x18, 0x03, 0x03, 0x3A, 0x00, 0x00, 0x00);
-	dcs_write_cmd_at(ctx, MCS_GOACLKA3, 0x18, 0x02, 0x03, 0x3B, 0x00, 0x00,
+	dcs_write_cmd_at(&dsi_ctx, MCS_GOACLKA3, 0x18, 0x02, 0x03, 0x3B, 0x00, 0x00,
 			 0x00, 0x18, 0x01, 0x03, 0x3C, 0x00, 0x00, 0x00);
-	dcs_write_cmd_at(ctx, MCS_GOAECLK, 0x01, 0x01, 0x20, 0x20, 0x00, 0x00,
+	dcs_write_cmd_at(&dsi_ctx, MCS_GOAECLK, 0x01, 0x01, 0x20, 0x20, 0x00, 0x00,
 			 0x01, 0x02, 0x00, 0x00);
 
-	dcs_write_cmd_at(ctx, MCS_NO_DOC2, 0x00);
+	dcs_write_cmd_at(&dsi_ctx, MCS_NO_DOC2, 0x00);
 
-	dcs_write_cmd_at(ctx, MCS_PANCTRLSET1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-	dcs_write_cmd_at(ctx, MCS_PANCTRLSET2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	dcs_write_cmd_at(&dsi_ctx, MCS_PANCTRLSET1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+	dcs_write_cmd_at(&dsi_ctx, MCS_PANCTRLSET2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 			 0, 0, 0, 0, 0);
-	dcs_write_cmd_at(ctx, MCS_PANCTRLSET3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	dcs_write_cmd_at(&dsi_ctx, MCS_PANCTRLSET3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 			 0, 0, 0, 0, 0);
-	dcs_write_cmd_at(ctx, MCS_PANCTRLSET4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-	dcs_write_cmd_at(ctx, MCS_PANCTRLSET5, 0, 4, 4, 4, 4, 4, 0, 0, 0, 0,
+	dcs_write_cmd_at(&dsi_ctx, MCS_PANCTRLSET4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+	dcs_write_cmd_at(&dsi_ctx, MCS_PANCTRLSET5, 0, 4, 4, 4, 4, 4, 0, 0, 0, 0,
 			 0, 0, 0, 0, 0);
-	dcs_write_cmd_at(ctx, MCS_PANCTRLSET6, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4,
+	dcs_write_cmd_at(&dsi_ctx, MCS_PANCTRLSET6, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4,
 			 4, 0, 0, 0, 0);
-	dcs_write_cmd_at(ctx, MCS_PANCTRLSET7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-	dcs_write_cmd_at(ctx, MCS_PANCTRLSET8, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+	dcs_write_cmd_at(&dsi_ctx, MCS_PANCTRLSET7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+	dcs_write_cmd_at(&dsi_ctx, MCS_PANCTRLSET8, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
 			 0xFF, 0xFF, 0xFF, 0xFF, 0xFF);
 
-	dcs_write_cmd_at(ctx, MCS_PANU2D1, 0x00, 0x26, 0x09, 0x0B, 0x01, 0x25,
+	dcs_write_cmd_at(&dsi_ctx, MCS_PANU2D1, 0x00, 0x26, 0x09, 0x0B, 0x01, 0x25,
 			 0x00, 0x00, 0x00, 0x00);
-	dcs_write_cmd_at(ctx, MCS_PANU2D2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	dcs_write_cmd_at(&dsi_ctx, MCS_PANU2D2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 			 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x0A, 0x0C, 0x02);
-	dcs_write_cmd_at(ctx, MCS_PANU2D3, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00,
+	dcs_write_cmd_at(&dsi_ctx, MCS_PANU2D3, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00,
 			 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00);
-	dcs_write_cmd_at(ctx, MCS_PAND2U1, 0x00, 0x25, 0x0C, 0x0A, 0x02, 0x26,
+	dcs_write_cmd_at(&dsi_ctx, MCS_PAND2U1, 0x00, 0x25, 0x0C, 0x0A, 0x02, 0x26,
 			 0x00, 0x00, 0x00, 0x00);
-	dcs_write_cmd_at(ctx, MCS_PAND2U2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	dcs_write_cmd_at(&dsi_ctx, MCS_PAND2U2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 			 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x0B, 0x09, 0x01);
-	dcs_write_cmd_at(ctx, MCS_PAND2U3, 0x26, 0x00, 0x00, 0x00, 0x00, 0x00,
+	dcs_write_cmd_at(&dsi_ctx, MCS_PAND2U3, 0x26, 0x00, 0x00, 0x00, 0x00, 0x00,
 			 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00);
 
-	dcs_write_cmd_at(ctx, MCS_PWR_CTRL1 + 1, 0x66);
+	dcs_write_cmd_at(&dsi_ctx, MCS_PWR_CTRL1 + 1, 0x66);
 
-	dcs_write_cmd_at(ctx, MCS_NO_DOC3, 0x06);
+	dcs_write_cmd_at(&dsi_ctx, MCS_NO_DOC3, 0x06);
 
-	dcs_write_cmd_at(ctx, MCS_GMCT2_2P, 0x00, 0x09, 0x0F, 0x0E, 0x07, 0x10,
+	dcs_write_cmd_at(&dsi_ctx, MCS_GMCT2_2P, 0x00, 0x09, 0x0F, 0x0E, 0x07, 0x10,
 			 0x0B, 0x0A, 0x04, 0x07, 0x0B, 0x08, 0x0F, 0x10, 0x0A,
 			 0x01);
-	dcs_write_cmd_at(ctx, MCS_GMCT2_2N, 0x00, 0x09, 0x0F, 0x0E, 0x07, 0x10,
+	dcs_write_cmd_at(&dsi_ctx, MCS_GMCT2_2N, 0x00, 0x09, 0x0F, 0x0E, 0x07, 0x10,
 			 0x0B, 0x0A, 0x04, 0x07, 0x0B, 0x08, 0x0F, 0x10, 0x0A,
 			 0x01);
 
 	/* Exit CMD2 */
-	dcs_write_cmd_at(ctx, MCS_CMD2_ENA1, 0xFF, 0xFF, 0xFF);
+	dcs_write_cmd_at(&dsi_ctx, MCS_CMD2_ENA1, 0xFF, 0xFF, 0xFF);
 
-	ret = mipi_dsi_dcs_nop(dsi);
-	if (ret)
-		return ret;
-
-	ret = mipi_dsi_dcs_exit_sleep_mode(dsi);
-	if (ret)
-		return ret;
+	mipi_dsi_dcs_nop_multi(&dsi_ctx);
 
-	/* Wait for sleep out exit */
-	mdelay(120);
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 120);
 
 	/* Default portrait 480x800 rgb24 */
-	dcs_write_seq(ctx, MIPI_DCS_SET_ADDRESS_MODE, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_SET_ADDRESS_MODE, 0x00);
 
-	ret = mipi_dsi_dcs_set_column_address(dsi, 0, OTM8009A_HDISPLAY - 1);
-	if (ret)
-		return ret;
+	mipi_dsi_dcs_set_column_address_multi(&dsi_ctx, 0, OTM8009A_HDISPLAY - 1);
 
-	ret = mipi_dsi_dcs_set_page_address(dsi, 0, OTM8009A_VDISPLAY - 1);
-	if (ret)
-		return ret;
+	mipi_dsi_dcs_set_page_address_multi(&dsi_ctx, 0, OTM8009A_VDISPLAY - 1);
 
 	/* See otm8009a driver documentation for pixel format descriptions */
-	ret = mipi_dsi_dcs_set_pixel_format(dsi, MIPI_DCS_PIXEL_FMT_24BIT |
+	mipi_dsi_dcs_set_pixel_format_multi(&dsi_ctx, MIPI_DCS_PIXEL_FMT_24BIT |
 					    MIPI_DCS_PIXEL_FMT_24BIT << 4);
-	if (ret)
-		return ret;
 
 	/* Disable CABC feature */
-	dcs_write_seq(ctx, MIPI_DCS_WRITE_POWER_SAVE, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_WRITE_POWER_SAVE, 0x00);
 
-	ret = mipi_dsi_dcs_set_display_on(dsi);
-	if (ret)
-		return ret;
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
 
-	ret = mipi_dsi_dcs_nop(dsi);
-	if (ret)
-		return ret;
+	mipi_dsi_dcs_nop_multi(&dsi_ctx);
 
 	/* Send Command GRAM memory write (no parameters) */
-	dcs_write_seq(ctx, MIPI_DCS_WRITE_MEMORY_START);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_WRITE_MEMORY_START);
 
 	/* Wait a short while to let the panel be ready before the 1st frame */
-	mdelay(10);
+	mipi_dsi_msleep(&dsi_ctx, 10);
 
-	return 0;
+	return dsi_ctx.accum_err;
 }
 
 static int otm8009a_disable(struct drm_panel *panel)
 {
 	struct otm8009a *ctx = panel_to_otm8009a(panel);
 	struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
-	int ret;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
 
 	backlight_disable(ctx->bl_dev);
 
-	ret = mipi_dsi_dcs_set_display_off(dsi);
-	if (ret)
-		return ret;
-
-	ret = mipi_dsi_dcs_enter_sleep_mode(dsi);
-	if (ret)
-		return ret;
+	mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
+	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 120);
 
-	msleep(120);
-
-	return 0;
+	return dsi_ctx.accum_err;
 }
 
 static int otm8009a_unprepare(struct drm_panel *panel)
@@ -383,6 +346,8 @@ static const struct drm_panel_funcs otm8009a_drm_funcs = {
 static int otm8009a_backlight_update_status(struct backlight_device *bd)
 {
 	struct otm8009a *ctx = bl_get_data(bd);
+	struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
 	u8 data[2];
 
 	if (!ctx->prepared) {
@@ -397,7 +362,7 @@ static int otm8009a_backlight_update_status(struct backlight_device *bd)
 		 */
 		data[0] = MIPI_DCS_SET_DISPLAY_BRIGHTNESS;
 		data[1] = bd->props.brightness;
-		otm8009a_dcs_write_buf(ctx, data, ARRAY_SIZE(data));
+		mipi_dsi_dcs_write_buffer_multi(&dsi_ctx, data, ARRAY_SIZE(data));
 
 		/* set Brightness Control & Backlight on */
 		data[1] = 0x24;
@@ -409,9 +374,9 @@ static int otm8009a_backlight_update_status(struct backlight_device *bd)
 
 	/* Update Brightness Control & Backlight */
 	data[0] = MIPI_DCS_WRITE_CONTROL_DISPLAY;
-	otm8009a_dcs_write_buf(ctx, data, ARRAY_SIZE(data));
+	mipi_dsi_dcs_write_buffer_multi(&dsi_ctx, data, ARRAY_SIZE(data));
 
-	return 0;
+	return dsi_ctx.accum_err;
 }
 
 static const struct backlight_ops otm8009a_backlight_ops = {
diff --git a/drivers/gpu/drm/panel/panel-samsung-ltl106hl02.c b/drivers/gpu/drm/panel/panel-samsung-ltl106hl02.c
new file mode 100644
index 000000000000..1618841b7caa
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-samsung-ltl106hl02.c
@@ -0,0 +1,179 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/array_size.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/gpio/consumer.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/regulator/consumer.h>
+
+#include <video/mipi_display.h>
+
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
+
+struct samsung_ltl106hl02 {
+	struct drm_panel panel;
+	struct mipi_dsi_device *dsi;
+
+	struct regulator *supply;
+	struct gpio_desc *reset_gpio;
+};
+
+static inline struct samsung_ltl106hl02 *to_samsung_ltl106hl02(struct drm_panel *panel)
+{
+	return container_of(panel, struct samsung_ltl106hl02, panel);
+}
+
+static void samsung_ltl106hl02_reset(struct samsung_ltl106hl02 *ctx)
+{
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	usleep_range(10000, 11000);
+	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+	usleep_range(2000, 3000);
+}
+
+static int samsung_ltl106hl02_prepare(struct drm_panel *panel)
+{
+	struct samsung_ltl106hl02 *ctx = to_samsung_ltl106hl02(panel);
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
+	struct device *dev = &ctx->dsi->dev;
+	int ret;
+
+	ret = regulator_enable(ctx->supply);
+	if (ret < 0) {
+		dev_err(dev, "failed to enable power supply %d\n", ret);
+		return ret;
+	}
+
+	if (ctx->reset_gpio)
+		samsung_ltl106hl02_reset(ctx);
+
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 70);
+
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 5);
+
+	return dsi_ctx.accum_err;
+}
+
+static int samsung_ltl106hl02_unprepare(struct drm_panel *panel)
+{
+	struct samsung_ltl106hl02 *ctx = to_samsung_ltl106hl02(panel);
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
+
+	mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 50);
+	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 150);
+
+	if (ctx->reset_gpio)
+		gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+
+	regulator_disable(ctx->supply);
+
+	return 0;
+}
+
+static const struct drm_display_mode samsung_ltl106hl02_mode = {
+	.clock = (1920 + 32 + 32 + 64) * (1080 + 6 + 3 + 22) * 60 / 1000,
+	.hdisplay = 1920,
+	.hsync_start = 1920 + 32,
+	.hsync_end = 1920 + 32 + 32,
+	.htotal = 1920 + 32 + 32 + 64,
+	.vdisplay = 1080,
+	.vsync_start = 1080 + 6,
+	.vsync_end = 1080 + 6 + 3,
+	.vtotal = 1080 + 6 + 3 + 22,
+	.width_mm = 235,
+	.height_mm = 132,
+	.type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+};
+
+static int samsung_ltl106hl02_get_modes(struct drm_panel *panel,
+					struct drm_connector *connector)
+{
+	return drm_connector_helper_get_modes_fixed(connector, &samsung_ltl106hl02_mode);
+}
+
+static const struct drm_panel_funcs samsung_ltl106hl02_panel_funcs = {
+	.prepare = samsung_ltl106hl02_prepare,
+	.unprepare = samsung_ltl106hl02_unprepare,
+	.get_modes = samsung_ltl106hl02_get_modes,
+};
+
+static int samsung_ltl106hl02_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct samsung_ltl106hl02 *ctx;
+	int ret;
+
+	ctx = devm_drm_panel_alloc(dev, struct samsung_ltl106hl02, panel,
+				   &samsung_ltl106hl02_panel_funcs,
+				   DRM_MODE_CONNECTOR_DSI);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+
+	ctx->supply = devm_regulator_get(dev, "power");
+	if (IS_ERR(ctx->supply))
+		return dev_err_probe(dev, PTR_ERR(ctx->supply),
+				     "Failed to get power regulator\n");
+
+	ctx->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(ctx->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(ctx->reset_gpio),
+				     "Failed to get reset-gpios\n");
+
+	ctx->dsi = dsi;
+	mipi_dsi_set_drvdata(dsi, ctx);
+
+	dsi->lanes = 4;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_LPM;
+
+	ret = drm_panel_of_backlight(&ctx->panel);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to get backlight\n");
+
+	drm_panel_add(&ctx->panel);
+
+	ret = devm_mipi_dsi_attach(dev, dsi);
+	if (ret < 0) {
+		drm_panel_remove(&ctx->panel);
+		return dev_err_probe(dev, ret, "Failed to attach to DSI host\n");
+	}
+
+	return 0;
+}
+
+static void samsung_ltl106hl02_remove(struct mipi_dsi_device *dsi)
+{
+	struct samsung_ltl106hl02 *ctx = mipi_dsi_get_drvdata(dsi);
+
+	drm_panel_remove(&ctx->panel);
+}
+
+static const struct of_device_id samsung_ltl106hl02_of_match[] = {
+	{ .compatible = "samsung,ltl106hl02-001" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, samsung_ltl106hl02_of_match);
+
+static struct mipi_dsi_driver samsung_ltl106hl02_driver = {
+	.driver = {
+		.name = "panel-samsung-ltl106hl02",
+		.of_match_table = samsung_ltl106hl02_of_match,
+	},
+	.probe = samsung_ltl106hl02_probe,
+	.remove = samsung_ltl106hl02_remove,
+};
+module_mipi_dsi_driver(samsung_ltl106hl02_driver);
+
+MODULE_AUTHOR("Anton Bambura <jenneron@protonmail.com>");
+MODULE_DESCRIPTION("DRM driver for Samsung LTL106HL02 video mode DSI panel");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index b26b682826bc..3acc9f3dac16 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -2509,6 +2509,31 @@ static const struct panel_desc hannstar_hsd101pww2 = {
 	.connector_type = DRM_MODE_CONNECTOR_LVDS,
 };
 
+static const struct display_timing hannstar_hsd156juw2_timing = {
+	.pixelclock = { 66000000, 72800000, 80500000 },
+	.hactive = { 1920, 1920, 1920 },
+	.hfront_porch = { 20, 30, 30 },
+	.hback_porch = { 20, 30, 30 },
+	.hsync_len = { 50, 60, 90 },
+	.vactive = { 1080, 1080, 1080 },
+	.vfront_porch = { 1, 2, 4 },
+	.vback_porch = { 1, 2, 4 },
+	.vsync_len = { 3, 40, 80 },
+	.flags = DISPLAY_FLAGS_DE_HIGH,
+};
+
+static const struct panel_desc hannstar_hsd156juw2 = {
+	.timings = &hannstar_hsd156juw2_timing,
+	.num_timings = 1,
+	.bpc = 8,
+	.size = {
+		.width = 344,
+		.height = 194,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
 static const struct drm_display_mode hitachi_tx23d38vm0caa_mode = {
 	.clock = 33333,
 	.hdisplay = 800,
@@ -5253,6 +5278,9 @@ static const struct of_device_id platform_of_match[] = {
 	}, {
 		.compatible = "hannstar,hsd101pww2",
 		.data = &hannstar_hsd101pww2,
+	}, {
+		.compatible = "hannstar,hsd156juw2",
+		.data = &hannstar_hsd156juw2,
 	}, {
 		.compatible = "hit,tx23d38vm0caa",
 		.data = &hitachi_tx23d38vm0caa
diff --git a/drivers/gpu/drm/panel/panel-sony-td4353-jdi.c b/drivers/gpu/drm/panel/panel-sony-td4353-jdi.c
index 7c989b70ab51..a14c86c60d19 100644
--- a/drivers/gpu/drm/panel/panel-sony-td4353-jdi.c
+++ b/drivers/gpu/drm/panel/panel-sony-td4353-jdi.c
@@ -212,6 +212,8 @@ static int sony_td4353_jdi_probe(struct mipi_dsi_device *dsi)
 	if (ret)
 		return dev_err_probe(dev, ret, "Failed to get backlight\n");
 
+	ctx->panel.prepare_prev_first = true;
+
 	drm_panel_add(&ctx->panel);
 
 	ret = mipi_dsi_attach(dsi);
diff --git a/drivers/gpu/drm/panfrost/panfrost_device.c b/drivers/gpu/drm/panfrost/panfrost_device.c
index c61b97af120c..dedc13e56631 100644
--- a/drivers/gpu/drm/panfrost/panfrost_device.c
+++ b/drivers/gpu/drm/panfrost/panfrost_device.c
@@ -12,6 +12,7 @@
 #include "panfrost_device.h"
 #include "panfrost_devfreq.h"
 #include "panfrost_features.h"
+#include "panfrost_gem.h"
 #include "panfrost_issues.h"
 #include "panfrost_gpu.h"
 #include "panfrost_job.h"
@@ -267,6 +268,8 @@ int panfrost_device_init(struct panfrost_device *pfdev)
 	if (err)
 		goto out_job;
 
+	panfrost_gem_init(pfdev);
+
 	return 0;
 out_job:
 	panfrost_jm_fini(pfdev);
diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/panfrost/panfrost_device.h
index e61c4329fd07..0f3992412205 100644
--- a/drivers/gpu/drm/panfrost/panfrost_device.h
+++ b/drivers/gpu/drm/panfrost/panfrost_device.h
@@ -79,6 +79,7 @@ struct panfrost_features {
 	u32 thread_max_workgroup_sz;
 	u32 thread_max_barrier_sz;
 	u32 coherency_features;
+	u32 selected_coherency;
 	u32 afbc_features;
 	u32 texture_features[4];
 	u32 js_features[16];
diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c
index 7d8c7c337606..b95120682a72 100644
--- a/drivers/gpu/drm/panfrost/panfrost_drv.c
+++ b/drivers/gpu/drm/panfrost/panfrost_drv.c
@@ -95,6 +95,7 @@ static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct
 		PANFROST_FEATURE_ARRAY(JS_FEATURES, js_features, 15);
 		PANFROST_FEATURE(NR_CORE_GROUPS, nr_core_groups);
 		PANFROST_FEATURE(THREAD_TLS_ALLOC, thread_tls_alloc);
+		PANFROST_FEATURE(SELECTED_COHERENCY, selected_coherency);
 
 	case DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP:
 		ret = panfrost_ioctl_query_timestamp(pfdev, &param->value);
@@ -125,6 +126,10 @@ static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct
 	return 0;
 }
 
+#define PANFROST_BO_FLAGS	(PANFROST_BO_NOEXEC | \
+				 PANFROST_BO_HEAP | \
+				 PANFROST_BO_WB_MMAP)
+
 static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data,
 		struct drm_file *file)
 {
@@ -134,8 +139,7 @@ static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data,
 	struct panfrost_gem_mapping *mapping;
 	int ret;
 
-	if (!args->size || args->pad ||
-	    (args->flags & ~(PANFROST_BO_NOEXEC | PANFROST_BO_HEAP)))
+	if (!args->size || args->pad || (args->flags & ~PANFROST_BO_FLAGS))
 		return -EINVAL;
 
 	/* Heaps should never be executable */
@@ -579,6 +583,91 @@ static int panfrost_ioctl_jm_ctx_destroy(struct drm_device *dev, void *data,
 	return panfrost_jm_ctx_destroy(file, args->handle);
 }
 
+static int panfrost_ioctl_sync_bo(struct drm_device *ddev, void *data,
+				  struct drm_file *file)
+{
+	struct drm_panfrost_sync_bo *args = data;
+	struct drm_panfrost_bo_sync_op *ops;
+	struct drm_gem_object *obj;
+	int ret;
+	u32 i;
+
+	if (args->pad)
+		return -EINVAL;
+
+	if (!args->op_count)
+		return 0;
+
+	ops = kvmalloc_array(args->op_count, sizeof(*ops), GFP_KERNEL);
+	if (!ops) {
+		DRM_DEBUG("Failed to allocate incoming BO sync ops array\n");
+		return -ENOMEM;
+	}
+
+	if (copy_from_user(ops, (void __user *)(uintptr_t)args->ops,
+			   args->op_count * sizeof(*ops))) {
+		DRM_DEBUG("Failed to copy in BO sync ops\n");
+		ret = -EFAULT;
+		goto err_ops;
+	}
+
+	for (i = 0; i < args->op_count; i++) {
+		obj = drm_gem_object_lookup(file, ops[i].handle);
+		if (!obj) {
+			ret = -ENOENT;
+			goto err_ops;
+		}
+
+		ret = panfrost_gem_sync(obj, ops[i].type,
+					ops[i].offset, ops[i].size);
+
+		drm_gem_object_put(obj);
+
+		if (ret)
+			goto err_ops;
+	}
+
+err_ops:
+	kvfree(ops);
+
+	return ret;
+}
+
+static int panfrost_ioctl_query_bo_info(struct drm_device *dev, void *data,
+					struct drm_file *file_priv)
+{
+	struct drm_panfrost_query_bo_info *args = data;
+	struct drm_gem_object *gem_obj;
+	struct panfrost_gem_object *bo;
+
+	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
+	if (!gem_obj) {
+		DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
+		return -ENOENT;
+	}
+
+	bo = to_panfrost_bo(gem_obj);
+	args->pad = 0;
+	args->create_flags = 0;
+	args->extra_flags = 0;
+
+	if (drm_gem_is_imported(gem_obj)) {
+		args->extra_flags |= DRM_PANFROST_BO_IS_IMPORTED;
+	} else {
+		if (bo->noexec)
+			args->create_flags |= PANFROST_BO_NOEXEC;
+
+		if (bo->is_heap)
+			args->create_flags |= PANFROST_BO_HEAP;
+
+		if (!bo->base.map_wc)
+			args->create_flags |= PANFROST_BO_WB_MMAP;
+	}
+
+	drm_gem_object_put(gem_obj);
+	return 0;
+}
+
 int panfrost_unstable_ioctl_check(void)
 {
 	if (!unstable_ioctls)
@@ -648,6 +737,8 @@ static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = {
 	PANFROST_IOCTL(SET_LABEL_BO,	set_label_bo,	DRM_RENDER_ALLOW),
 	PANFROST_IOCTL(JM_CTX_CREATE,	jm_ctx_create,	DRM_RENDER_ALLOW),
 	PANFROST_IOCTL(JM_CTX_DESTROY,	jm_ctx_destroy,	DRM_RENDER_ALLOW),
+	PANFROST_IOCTL(SYNC_BO,		sync_bo,	DRM_RENDER_ALLOW),
+	PANFROST_IOCTL(QUERY_BO_INFO,	query_bo_info,	DRM_RENDER_ALLOW),
 };
 
 static void panfrost_gpu_show_fdinfo(struct panfrost_device *pfdev,
@@ -837,6 +928,9 @@ static void panfrost_debugfs_init(struct drm_minor *minor)
  * - 1.4 - adds SET_LABEL_BO
  * - 1.5 - adds JM_CTX_{CREATE,DESTROY} ioctls and extend SUBMIT to allow
  *	   context creation with configurable priorities/affinity
+ * - 1.6 - adds PANFROST_BO_MAP_WB, PANFROST_IOCTL_SYNC_BO,
+ *	   PANFROST_IOCTL_QUERY_BO_INFO and
+ *	   DRM_PANFROST_PARAM_SELECTED_COHERENCY
  */
 static const struct drm_driver panfrost_drm_driver = {
 	.driver_features	= DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ,
@@ -849,15 +943,22 @@ static const struct drm_driver panfrost_drm_driver = {
 	.name			= "panfrost",
 	.desc			= "panfrost DRM",
 	.major			= 1,
-	.minor			= 5,
+	.minor			= 6,
 
 	.gem_create_object	= panfrost_gem_create_object,
+	.gem_prime_import	= panfrost_gem_prime_import,
 	.gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table,
 #ifdef CONFIG_DEBUG_FS
 	.debugfs_init = panfrost_debugfs_init,
 #endif
 };
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+bool panfrost_transparent_hugepage = true;
+module_param_named(transparent_hugepage, panfrost_transparent_hugepage, bool, 0400);
+MODULE_PARM_DESC(transparent_hugepage, "Use a dedicated tmpfs mount point with Transparent Hugepage enabled (true = default)");
+#endif
+
 static int panfrost_probe(struct platform_device *pdev)
 {
 	struct panfrost_device *pfdev;
diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.h b/drivers/gpu/drm/panfrost/panfrost_drv.h
new file mode 100644
index 000000000000..a2277ec61aab
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_drv.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2025 Amazon.com, Inc. or its affiliates */
+
+#ifndef __PANFROST_DRV_H__
+#define __PANFROST_DRV_H__
+
+extern bool panfrost_transparent_hugepage;
+
+#endif
diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panfrost/panfrost_gem.c
index 8041b65c6609..44985b515212 100644
--- a/drivers/gpu/drm/panfrost/panfrost_gem.c
+++ b/drivers/gpu/drm/panfrost/panfrost_gem.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+/* Copyright 2025 Amazon.com, Inc. or its affiliates */
 
 #include <linux/cleanup.h>
 #include <linux/err.h>
@@ -10,9 +11,26 @@
 #include <drm/panfrost_drm.h>
 #include <drm/drm_print.h>
 #include "panfrost_device.h"
+#include "panfrost_drv.h"
 #include "panfrost_gem.h"
 #include "panfrost_mmu.h"
 
+void panfrost_gem_init(struct panfrost_device *pfdev)
+{
+	int err;
+
+	if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
+	    !panfrost_transparent_hugepage)
+		return;
+
+	err = drm_gem_huge_mnt_create(&pfdev->base, "within_size");
+	if (drm_gem_get_huge_mnt(&pfdev->base))
+		drm_info(&pfdev->base, "Using Transparent Hugepage\n");
+	else if (err)
+		drm_warn(&pfdev->base, "Can't use Transparent Hugepage (%d)\n",
+			 err);
+}
+
 #ifdef CONFIG_DEBUG_FS
 static void panfrost_gem_debugfs_bo_add(struct panfrost_device *pfdev,
 					struct panfrost_gem_object *bo)
@@ -259,6 +277,128 @@ static size_t panfrost_gem_rss(struct drm_gem_object *obj)
 	return 0;
 }
 
+static struct sg_table *
+panfrost_gem_prime_map_dma_buf(struct dma_buf_attachment *attach,
+			       enum dma_data_direction dir)
+{
+	struct sg_table *sgt = drm_gem_map_dma_buf(attach, dir);
+
+	if (!IS_ERR(sgt))
+		attach->priv = sgt;
+
+	return sgt;
+}
+
+static void
+panfrost_gem_prime_unmap_dma_buf(struct dma_buf_attachment *attach,
+				 struct sg_table *sgt,
+				 enum dma_data_direction dir)
+{
+	attach->priv = NULL;
+	drm_gem_unmap_dma_buf(attach, sgt, dir);
+}
+
+static int
+panfrost_gem_prime_begin_cpu_access(struct dma_buf *dma_buf,
+				    enum dma_data_direction dir)
+{
+	struct drm_gem_object *obj = dma_buf->priv;
+	struct drm_device *dev = obj->dev;
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+	struct dma_buf_attachment *attach;
+
+	dma_resv_lock(obj->resv, NULL);
+	if (shmem->sgt)
+		dma_sync_sgtable_for_cpu(dev->dev, shmem->sgt, dir);
+
+	if (shmem->vaddr)
+		invalidate_kernel_vmap_range(shmem->vaddr, shmem->base.size);
+
+	list_for_each_entry(attach, &dma_buf->attachments, node) {
+		struct sg_table *sgt = attach->priv;
+
+		if (sgt)
+			dma_sync_sgtable_for_cpu(attach->dev, sgt, dir);
+	}
+	dma_resv_unlock(obj->resv);
+
+	return 0;
+}
+
+static int
+panfrost_gem_prime_end_cpu_access(struct dma_buf *dma_buf,
+				  enum dma_data_direction dir)
+{
+	struct drm_gem_object *obj = dma_buf->priv;
+	struct drm_device *dev = obj->dev;
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+	struct dma_buf_attachment *attach;
+
+	dma_resv_lock(obj->resv, NULL);
+	list_for_each_entry(attach, &dma_buf->attachments, node) {
+		struct sg_table *sgt = attach->priv;
+
+		if (sgt)
+			dma_sync_sgtable_for_device(attach->dev, sgt, dir);
+	}
+
+	if (shmem->vaddr)
+		flush_kernel_vmap_range(shmem->vaddr, shmem->base.size);
+
+	if (shmem->sgt)
+		dma_sync_sgtable_for_device(dev->dev, shmem->sgt, dir);
+
+	dma_resv_unlock(obj->resv);
+	return 0;
+}
+
+static const struct dma_buf_ops panfrost_dma_buf_ops = {
+	.attach = drm_gem_map_attach,
+	.detach = drm_gem_map_detach,
+	.map_dma_buf = panfrost_gem_prime_map_dma_buf,
+	.unmap_dma_buf = panfrost_gem_prime_unmap_dma_buf,
+	.release = drm_gem_dmabuf_release,
+	.mmap = drm_gem_dmabuf_mmap,
+	.vmap = drm_gem_dmabuf_vmap,
+	.vunmap = drm_gem_dmabuf_vunmap,
+	.begin_cpu_access = panfrost_gem_prime_begin_cpu_access,
+	.end_cpu_access = panfrost_gem_prime_end_cpu_access,
+};
+
+static struct dma_buf *
+panfrost_gem_prime_export(struct drm_gem_object *obj, int flags)
+{
+	struct drm_device *dev = obj->dev;
+	struct dma_buf_export_info exp_info = {
+		.exp_name = KBUILD_MODNAME,
+		.owner = THIS_MODULE,
+		.ops = &panfrost_dma_buf_ops,
+		.size = obj->size,
+		.flags = flags,
+		.priv = obj,
+		.resv = obj->resv,
+	};
+
+	return drm_gem_dmabuf_export(dev, &exp_info);
+}
+
+struct drm_gem_object *
+panfrost_gem_prime_import(struct drm_device *dev,
+			  struct dma_buf *dma_buf)
+{
+	struct drm_gem_object *obj = dma_buf->priv;
+
+	if (dma_buf->ops == &panfrost_dma_buf_ops && obj->dev == dev) {
+		/* Importing dmabuf exported from our own gem increases
+		 * refcount on gem itself instead of f_count of dmabuf.
+		 */
+		drm_gem_object_get(obj);
+		return obj;
+	}
+
+	return drm_gem_prime_import(dev, dma_buf);
+}
+
 static const struct drm_gem_object_funcs panfrost_gem_funcs = {
 	.free = panfrost_gem_free_object,
 	.open = panfrost_gem_open,
@@ -267,6 +407,7 @@ static const struct drm_gem_object_funcs panfrost_gem_funcs = {
 	.pin = panfrost_gem_pin,
 	.unpin = drm_gem_shmem_object_unpin,
 	.get_sg_table = drm_gem_shmem_object_get_sg_table,
+	.export = panfrost_gem_prime_export,
 	.vmap = drm_gem_shmem_object_vmap,
 	.vunmap = drm_gem_shmem_object_vunmap,
 	.mmap = drm_gem_shmem_object_mmap,
@@ -303,12 +444,42 @@ struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t
 	return &obj->base.base;
 }
 
+static bool
+should_map_wc(struct panfrost_gem_object *bo)
+{
+	struct panfrost_device *pfdev = to_panfrost_device(bo->base.base.dev);
+
+	/* We can't do uncached mappings if the device is coherent,
+	 * because the zeroing done by the shmem layer at page allocation
+	 * time happens on a cached mapping which isn't CPU-flushed (at least
+	 * not on Arm64 where the flush is deferred to PTE setup time, and
+	 * only done conditionally based on the mapping permissions). We can't
+	 * rely on dma_map_sgtable()/dma_sync_sgtable_for_xxx() either to flush
+	 * those, because they are NOPed if dma_dev_coherent() returns true.
+	 */
+	if (pfdev->coherent)
+		return false;
+
+	/* Cached mappings are explicitly requested, so no write-combine. */
+	if (bo->wb_mmap)
+		return false;
+
+	/* The default is write-combine. */
+	return true;
+}
+
 struct panfrost_gem_object *
 panfrost_gem_create(struct drm_device *dev, size_t size, u32 flags)
 {
 	struct drm_gem_shmem_object *shmem;
 	struct panfrost_gem_object *bo;
 
+	/* The heap buffer is not supposed to be CPU-visible, so don't allow
+	 * WB_MMAP on those.
+	 */
+	if ((flags & PANFROST_BO_HEAP) && (flags & PANFROST_BO_WB_MMAP))
+		return ERR_PTR(-EINVAL);
+
 	/* Round up heap allocations to 2MB to keep fault handling simple */
 	if (flags & PANFROST_BO_HEAP)
 		size = roundup(size, SZ_2M);
@@ -320,6 +491,8 @@ panfrost_gem_create(struct drm_device *dev, size_t size, u32 flags)
 	bo = to_panfrost_bo(&shmem->base);
 	bo->noexec = !!(flags & PANFROST_BO_NOEXEC);
 	bo->is_heap = !!(flags & PANFROST_BO_HEAP);
+	bo->wb_mmap = !!(flags & PANFROST_BO_WB_MMAP);
+	bo->base.map_wc = should_map_wc(bo);
 
 	return bo;
 }
@@ -366,6 +539,90 @@ panfrost_gem_set_label(struct drm_gem_object *obj, const char *label)
 	kfree_const(old_label);
 }
 
+int
+panfrost_gem_sync(struct drm_gem_object *obj, u32 type, u32 offset, u32 size)
+{
+	struct panfrost_gem_object *bo = to_panfrost_bo(obj);
+	struct drm_gem_shmem_object *shmem = &bo->base;
+	const struct drm_device *dev = shmem->base.dev;
+	struct sg_table *sgt;
+	struct scatterlist *sgl;
+	unsigned int count;
+
+	/* Make sure the range is in bounds. */
+	if (offset + size < offset || offset + size > shmem->base.size)
+		return -EINVAL;
+
+	/* Disallow CPU-cache maintenance on imported buffers. */
+	if (drm_gem_is_imported(&shmem->base))
+		return -EINVAL;
+
+	switch (type) {
+	case PANFROST_BO_SYNC_CPU_CACHE_FLUSH:
+	case PANFROST_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE:
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	/* Don't bother if it's WC-mapped */
+	if (shmem->map_wc)
+		return 0;
+
+	/* Nothing to do if the size is zero. */
+	if (size == 0)
+		return 0;
+
+	sgt = drm_gem_shmem_get_pages_sgt(shmem);
+	if (IS_ERR(sgt))
+		return PTR_ERR(sgt);
+
+	for_each_sgtable_dma_sg(sgt, sgl, count) {
+		if (size == 0)
+			break;
+
+		dma_addr_t paddr = sg_dma_address(sgl);
+		size_t len = sg_dma_len(sgl);
+
+		if (len <= offset) {
+			offset -= len;
+			continue;
+		}
+
+		paddr += offset;
+		len -= offset;
+		len = min_t(size_t, len, size);
+		size -= len;
+		offset = 0;
+
+		/* It's unclear whether dma_sync_xxx() is the right API to do CPU
+		 * cache maintenance given an IOMMU can register their own
+		 * implementation doing more than just CPU cache flushes/invalidation,
+		 * and what we really care about here is CPU caches only, but that's
+		 * the best we have that is both arch-agnostic and does at least the
+		 * CPU cache maintenance on a <page,offset,size> tuple.
+		 *
+		 * Also, I wish we could do a single
+		 *
+		 *      dma_sync_single_for_device(BIDIR)
+		 *
+		 * and get a flush+invalidate, but that's not how it's implemented
+		 * in practice (at least on arm64), so we have to make it
+		 *
+		 *      dma_sync_single_for_device(TO_DEVICE)
+		 *      dma_sync_single_for_cpu(FROM_DEVICE)
+		 *
+		 * for the flush+invalidate case.
+		 */
+		dma_sync_single_for_device(dev->dev, paddr, len, DMA_TO_DEVICE);
+		if (type == PANFROST_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE)
+			dma_sync_single_for_cpu(dev->dev, paddr, len, DMA_FROM_DEVICE);
+	}
+
+	return 0;
+}
+
 void
 panfrost_gem_internal_set_label(struct drm_gem_object *obj, const char *label)
 {
diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.h b/drivers/gpu/drm/panfrost/panfrost_gem.h
index 8de3e76f2717..79d4377019e9 100644
--- a/drivers/gpu/drm/panfrost/panfrost_gem.h
+++ b/drivers/gpu/drm/panfrost/panfrost_gem.h
@@ -98,6 +98,11 @@ struct panfrost_gem_object {
 	bool noexec		:1;
 	bool is_heap		:1;
 
+	/* On coherent devices, this reflects the creation flags, not the true
+	 * cacheability attribute of the mapping.
+	 */
+	bool wb_mmap		:1;
+
 #ifdef CONFIG_DEBUG_FS
 	struct panfrost_gem_debugfs debugfs;
 #endif
@@ -124,12 +129,17 @@ drm_mm_node_to_panfrost_mapping(struct drm_mm_node *node)
 	return container_of(node, struct panfrost_gem_mapping, mmnode);
 }
 
+void panfrost_gem_init(struct panfrost_device *pfdev);
+
 struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t size);
 
 struct drm_gem_object *
 panfrost_gem_prime_import_sg_table(struct drm_device *dev,
 				   struct dma_buf_attachment *attach,
 				   struct sg_table *sgt);
+struct drm_gem_object *
+panfrost_gem_prime_import(struct drm_device *dev,
+			  struct dma_buf *dma_buf);
 
 struct panfrost_gem_object *
 panfrost_gem_create(struct drm_device *dev, size_t size, u32 flags);
@@ -148,6 +158,8 @@ int panfrost_gem_shrinker_init(struct drm_device *dev);
 void panfrost_gem_shrinker_cleanup(struct drm_device *dev);
 
 void panfrost_gem_set_label(struct drm_gem_object *obj, const char *label);
+int panfrost_gem_sync(struct drm_gem_object *obj, u32 type,
+		      u32 offset, u32 size);
 void panfrost_gem_internal_set_label(struct drm_gem_object *obj, const char *label);
 
 #ifdef CONFIG_DEBUG_FS
diff --git a/drivers/gpu/drm/panfrost/panfrost_gpu.c b/drivers/gpu/drm/panfrost/panfrost_gpu.c
index 483d278eb154..7d555e63e21a 100644
--- a/drivers/gpu/drm/panfrost/panfrost_gpu.c
+++ b/drivers/gpu/drm/panfrost/panfrost_gpu.c
@@ -159,8 +159,8 @@ static void panfrost_gpu_init_quirks(struct panfrost_device *pfdev)
 	    pfdev->features.revision >= 0x2000)
 		quirks |= JM_MAX_JOB_THROTTLE_LIMIT << JM_JOB_THROTTLE_LIMIT_SHIFT;
 	else if (panfrost_model_eq(pfdev, 0x6000) &&
-		 pfdev->features.coherency_features == COHERENCY_ACE)
-		quirks |= (COHERENCY_ACE_LITE | COHERENCY_ACE) <<
+		 pfdev->features.coherency_features == BIT(COHERENCY_ACE))
+		quirks |= (BIT(COHERENCY_ACE_LITE) | BIT(COHERENCY_ACE)) <<
 			   JM_FORCE_COHERENCY_FEATURES_SHIFT;
 
 	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_IDVS_GROUP_SIZE))
@@ -263,7 +263,27 @@ static int panfrost_gpu_init_features(struct panfrost_device *pfdev)
 	pfdev->features.max_threads = gpu_read(pfdev, GPU_THREAD_MAX_THREADS);
 	pfdev->features.thread_max_workgroup_sz = gpu_read(pfdev, GPU_THREAD_MAX_WORKGROUP_SIZE);
 	pfdev->features.thread_max_barrier_sz = gpu_read(pfdev, GPU_THREAD_MAX_BARRIER_SIZE);
-	pfdev->features.coherency_features = gpu_read(pfdev, GPU_COHERENCY_FEATURES);
+
+	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_COHERENCY_REG))
+		pfdev->features.coherency_features = gpu_read(pfdev, GPU_COHERENCY_FEATURES);
+	else
+		pfdev->features.coherency_features = BIT(COHERENCY_ACE_LITE);
+
+	BUILD_BUG_ON(COHERENCY_ACE_LITE != DRM_PANFROST_GPU_COHERENCY_ACE_LITE);
+	BUILD_BUG_ON(COHERENCY_ACE != DRM_PANFROST_GPU_COHERENCY_ACE);
+	BUILD_BUG_ON(COHERENCY_NONE != DRM_PANFROST_GPU_COHERENCY_NONE);
+
+	if (!pfdev->coherent) {
+		pfdev->features.selected_coherency = COHERENCY_NONE;
+	} else if (pfdev->features.coherency_features & BIT(COHERENCY_ACE)) {
+		pfdev->features.selected_coherency = COHERENCY_ACE;
+	} else if (pfdev->features.coherency_features & BIT(COHERENCY_ACE_LITE)) {
+		pfdev->features.selected_coherency = COHERENCY_ACE_LITE;
+	} else {
+		drm_WARN(&pfdev->base, true, "No known coherency protocol supported");
+		pfdev->features.selected_coherency = COHERENCY_NONE;
+	}
+
 	pfdev->features.afbc_features = gpu_read(pfdev, GPU_AFBC_FEATURES);
 	for (i = 0; i < 4; i++)
 		pfdev->features.texture_features[i] = gpu_read(pfdev, GPU_TEXTURE_FEATURES(i));
diff --git a/drivers/gpu/drm/panfrost/panfrost_regs.h b/drivers/gpu/drm/panfrost/panfrost_regs.h
index 2b8f1617b836..ee15f6bf6e6f 100644
--- a/drivers/gpu/drm/panfrost/panfrost_regs.h
+++ b/drivers/gpu/drm/panfrost/panfrost_regs.h
@@ -102,9 +102,15 @@
 #define GPU_L2_PRESENT_LO		0x120	/* (RO) Level 2 cache present bitmap, low word */
 #define GPU_L2_PRESENT_HI		0x124	/* (RO) Level 2 cache present bitmap, high word */
 
+/* GPU_COHERENCY_FEATURES is a bitmask of BIT(COHERENCY_xxx) values encoding the
+ * set of supported coherency protocols. GPU_COHERENCY_ENABLE is passed a
+ * COHERENCY_xxx value.
+ */
 #define GPU_COHERENCY_FEATURES		0x300	/* (RO) Coherency features present */
-#define   COHERENCY_ACE_LITE		BIT(0)
-#define   COHERENCY_ACE			BIT(1)
+#define GPU_COHERENCY_ENABLE		0x304	/* (RW) Coherency protocol selection */
+#define   COHERENCY_ACE_LITE		0
+#define   COHERENCY_ACE			1
+#define   COHERENCY_NONE		31
 
 #define GPU_STACK_PRESENT_LO		0xE00   /* (RO) Core stack present bitmap, low word */
 #define GPU_STACK_PRESENT_HI		0xE04   /* (RO) Core stack present bitmap, high word */
diff --git a/drivers/gpu/drm/panthor/panthor_device.c b/drivers/gpu/drm/panthor/panthor_device.c
index e133b1e0ad6d..54fbb1aa07c5 100644
--- a/drivers/gpu/drm/panthor/panthor_device.c
+++ b/drivers/gpu/drm/panthor/panthor_device.c
@@ -18,6 +18,7 @@
 #include "panthor_devfreq.h"
 #include "panthor_device.h"
 #include "panthor_fw.h"
+#include "panthor_gem.h"
 #include "panthor_gpu.h"
 #include "panthor_hw.h"
 #include "panthor_mmu.h"
@@ -27,6 +28,12 @@
 
 static int panthor_gpu_coherency_init(struct panthor_device *ptdev)
 {
+	BUILD_BUG_ON(GPU_COHERENCY_NONE != DRM_PANTHOR_GPU_COHERENCY_NONE);
+	BUILD_BUG_ON(GPU_COHERENCY_ACE_LITE != DRM_PANTHOR_GPU_COHERENCY_ACE_LITE);
+	BUILD_BUG_ON(GPU_COHERENCY_ACE != DRM_PANTHOR_GPU_COHERENCY_ACE);
+
+	/* Start with no coherency, and update it if the device is flagged coherent. */
+	ptdev->gpu_info.selected_coherency = GPU_COHERENCY_NONE;
 	ptdev->coherent = device_get_dma_attr(ptdev->base.dev) == DEV_DMA_COHERENT;
 
 	if (!ptdev->coherent)
@@ -36,8 +43,10 @@ static int panthor_gpu_coherency_init(struct panthor_device *ptdev)
 	 * ACE protocol has never been supported for command stream frontend GPUs.
 	 */
 	if ((gpu_read(ptdev, GPU_COHERENCY_FEATURES) &
-		      GPU_COHERENCY_PROT_BIT(ACE_LITE)))
+		      GPU_COHERENCY_PROT_BIT(ACE_LITE))) {
+		ptdev->gpu_info.selected_coherency = GPU_COHERENCY_ACE_LITE;
 		return 0;
+	}
 
 	drm_err(&ptdev->base, "Coherency not supported by the device");
 	return -ENOTSUPP;
@@ -294,6 +303,8 @@ int panthor_device_init(struct panthor_device *ptdev)
 	if (ret)
 		goto err_unplug_fw;
 
+	panthor_gem_init(ptdev);
+
 	/* ~3 frames */
 	pm_runtime_set_autosuspend_delay(ptdev->base.dev, 50);
 	pm_runtime_use_autosuspend(ptdev->base.dev);
diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c
index d1d4c50da5bf..165dddfde6ca 100644
--- a/drivers/gpu/drm/panthor/panthor_drv.c
+++ b/drivers/gpu/drm/panthor/panthor_drv.c
@@ -177,7 +177,8 @@ panthor_get_uobj_array(const struct drm_panthor_obj_array *in, u32 min_stride,
 		 PANTHOR_UOBJ_DECL(struct drm_panthor_sync_op, timeline_value), \
 		 PANTHOR_UOBJ_DECL(struct drm_panthor_queue_submit, syncs), \
 		 PANTHOR_UOBJ_DECL(struct drm_panthor_queue_create, ringbuf_size), \
-		 PANTHOR_UOBJ_DECL(struct drm_panthor_vm_bind_op, syncs))
+		 PANTHOR_UOBJ_DECL(struct drm_panthor_vm_bind_op, syncs), \
+		 PANTHOR_UOBJ_DECL(struct drm_panthor_bo_sync_op, size))
 
 /**
  * PANTHOR_UOBJ_SET() - Copy a kernel object to a user object.
@@ -901,7 +902,8 @@ static int panthor_ioctl_vm_destroy(struct drm_device *ddev, void *data,
 	return panthor_vm_pool_destroy_vm(pfile->vms, args->id);
 }
 
-#define PANTHOR_BO_FLAGS		DRM_PANTHOR_BO_NO_MMAP
+#define PANTHOR_BO_FLAGS		(DRM_PANTHOR_BO_NO_MMAP | \
+					 DRM_PANTHOR_BO_WB_MMAP)
 
 static int panthor_ioctl_bo_create(struct drm_device *ddev, void *data,
 				   struct drm_file *file)
@@ -920,6 +922,12 @@ static int panthor_ioctl_bo_create(struct drm_device *ddev, void *data,
 		goto out_dev_exit;
 	}
 
+	if ((args->flags & DRM_PANTHOR_BO_NO_MMAP) &&
+	    (args->flags & DRM_PANTHOR_BO_WB_MMAP)) {
+		ret = -EINVAL;
+		goto out_dev_exit;
+	}
+
 	if (args->exclusive_vm_id) {
 		vm = panthor_vm_pool_get_vm(pfile->vms, args->exclusive_vm_id);
 		if (!vm) {
@@ -1396,6 +1404,66 @@ static int panthor_ioctl_set_user_mmio_offset(struct drm_device *ddev,
 	return 0;
 }
 
+static int panthor_ioctl_bo_sync(struct drm_device *ddev, void *data,
+				 struct drm_file *file)
+{
+	struct drm_panthor_bo_sync *args = data;
+	struct drm_panthor_bo_sync_op *ops;
+	struct drm_gem_object *obj;
+	int ret;
+
+	if (!args->ops.count)
+		return 0;
+
+	ret = PANTHOR_UOBJ_GET_ARRAY(ops, &args->ops);
+	if (ret)
+		return ret;
+
+	for (u32 i = 0; i < args->ops.count; i++) {
+		obj = drm_gem_object_lookup(file, ops[i].handle);
+		if (!obj) {
+			ret = -ENOENT;
+			goto err_ops;
+		}
+
+		ret = panthor_gem_sync(obj, ops[i].type, ops[i].offset,
+				       ops[i].size);
+
+		drm_gem_object_put(obj);
+
+		if (ret)
+			goto err_ops;
+	}
+
+err_ops:
+	kvfree(ops);
+
+	return ret;
+}
+
+static int panthor_ioctl_bo_query_info(struct drm_device *ddev, void *data,
+				       struct drm_file *file)
+{
+	struct drm_panthor_bo_query_info *args = data;
+	struct panthor_gem_object *bo;
+	struct drm_gem_object *obj;
+
+	obj = drm_gem_object_lookup(file, args->handle);
+	if (!obj)
+		return -ENOENT;
+
+	bo = to_panthor_bo(obj);
+	args->pad = 0;
+	args->create_flags = bo->flags;
+
+	args->extra_flags = 0;
+	if (drm_gem_is_imported(&bo->base.base))
+		args->extra_flags |= DRM_PANTHOR_BO_IS_IMPORTED;
+
+	drm_gem_object_put(obj);
+	return 0;
+}
+
 static int
 panthor_open(struct drm_device *ddev, struct drm_file *file)
 {
@@ -1470,6 +1538,8 @@ static const struct drm_ioctl_desc panthor_drm_driver_ioctls[] = {
 	PANTHOR_IOCTL(GROUP_SUBMIT, group_submit, DRM_RENDER_ALLOW),
 	PANTHOR_IOCTL(BO_SET_LABEL, bo_set_label, DRM_RENDER_ALLOW),
 	PANTHOR_IOCTL(SET_USER_MMIO_OFFSET, set_user_mmio_offset, DRM_RENDER_ALLOW),
+	PANTHOR_IOCTL(BO_SYNC, bo_sync, DRM_RENDER_ALLOW),
+	PANTHOR_IOCTL(BO_QUERY_INFO, bo_query_info, DRM_RENDER_ALLOW),
 };
 
 static int panthor_mmap(struct file *filp, struct vm_area_struct *vma)
@@ -1559,6 +1629,7 @@ static const struct file_operations panthor_drm_driver_fops = {
 	.read = drm_read,
 	.llseek = noop_llseek,
 	.mmap = panthor_mmap,
+	.get_unmapped_area = drm_gem_get_unmapped_area,
 	.show_fdinfo = drm_show_fdinfo,
 	.fop_flags = FOP_UNSIGNED_OFFSET,
 };
@@ -1604,6 +1675,11 @@ static void panthor_debugfs_init(struct drm_minor *minor)
  * - 1.3 - adds DRM_PANTHOR_GROUP_STATE_INNOCENT flag
  * - 1.4 - adds DRM_IOCTL_PANTHOR_BO_SET_LABEL ioctl
  * - 1.5 - adds DRM_PANTHOR_SET_USER_MMIO_OFFSET ioctl
+ * - 1.6 - enables GLB_COUNTER_EN
+ * - 1.7 - adds DRM_PANTHOR_BO_WB_MMAP flag
+ *       - adds DRM_IOCTL_PANTHOR_BO_SYNC ioctl
+ *       - adds DRM_IOCTL_PANTHOR_BO_QUERY_INFO ioctl
+ *       - adds drm_panthor_gpu_info::selected_coherency
  */
 static const struct drm_driver panthor_drm_driver = {
 	.driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ |
@@ -1617,15 +1693,22 @@ static const struct drm_driver panthor_drm_driver = {
 	.name = "panthor",
 	.desc = "Panthor DRM driver",
 	.major = 1,
-	.minor = 5,
+	.minor = 7,
 
 	.gem_create_object = panthor_gem_create_object,
 	.gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table,
+	.gem_prime_import = panthor_gem_prime_import,
 #ifdef CONFIG_DEBUG_FS
 	.debugfs_init = panthor_debugfs_init,
 #endif
 };
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+bool panthor_transparent_hugepage = true;
+module_param_named(transparent_hugepage, panthor_transparent_hugepage, bool, 0400);
+MODULE_PARM_DESC(transparent_hugepage, "Use a dedicated tmpfs mount point with Transparent Hugepage enabled (true = default)");
+#endif
+
 static int panthor_probe(struct platform_device *pdev)
 {
 	struct panthor_device *ptdev;
diff --git a/drivers/gpu/drm/panthor/panthor_drv.h b/drivers/gpu/drm/panthor/panthor_drv.h
new file mode 100644
index 000000000000..1bc7ddbad23e
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_drv.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2025 Amazon.com, Inc. or its affiliates */
+
+#ifndef __PANTHOR_DRV_H__
+#define __PANTHOR_DRV_H__
+
+extern bool panthor_transparent_hugepage;
+
+#endif
diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c
index 1a5e3c1a27fb..a64ec8756bed 100644
--- a/drivers/gpu/drm/panthor/panthor_fw.c
+++ b/drivers/gpu/drm/panthor/panthor_fw.c
@@ -1044,7 +1044,8 @@ static void panthor_fw_init_global_iface(struct panthor_device *ptdev)
 	if (panthor_fw_has_glb_state(ptdev))
 		glb_iface->input->ack_irq_mask |= GLB_STATE_MASK;
 
-	panthor_fw_update_reqs(glb_iface, req, GLB_IDLE_EN, GLB_IDLE_EN);
+	panthor_fw_update_reqs(glb_iface, req, GLB_IDLE_EN | GLB_COUNTER_EN,
+			       GLB_IDLE_EN | GLB_COUNTER_EN);
 	panthor_fw_toggle_reqs(glb_iface, req, ack,
 			       GLB_CFG_ALLOC_EN |
 			       GLB_CFG_POWEROFF_TIMER |
@@ -1187,7 +1188,6 @@ void panthor_fw_pre_reset(struct panthor_device *ptdev, bool on_hang)
 		else
 			ptdev->reset.fast = true;
 	}
-	panthor_fw_stop(ptdev);
 
 	panthor_job_irq_suspend(&ptdev->fw->irq);
 	panthor_fw_stop(ptdev);
@@ -1261,10 +1261,6 @@ void panthor_fw_unplug(struct panthor_device *ptdev)
 		if (ptdev->fw->irq.irq)
 			panthor_job_irq_suspend(&ptdev->fw->irq);
 
-		panthor_fw_halt_mcu(ptdev);
-		if (!panthor_fw_wait_mcu_halted(ptdev))
-			drm_warn(&ptdev->base, "Failed to halt MCU on unplug");
-
 		panthor_fw_stop(ptdev);
 	}
 
diff --git a/drivers/gpu/drm/panthor/panthor_gem.c b/drivers/gpu/drm/panthor/panthor_gem.c
index fbde78db270a..b61908fd508a 100644
--- a/drivers/gpu/drm/panthor/panthor_gem.c
+++ b/drivers/gpu/drm/panthor/panthor_gem.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 or MIT
 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
 /* Copyright 2023 Collabora ltd. */
+/* Copyright 2025 Amazon.com, Inc. or its affiliates */
 
 #include <linux/cleanup.h>
 #include <linux/dma-buf.h>
@@ -12,10 +13,27 @@
 #include <drm/panthor_drm.h>
 
 #include "panthor_device.h"
+#include "panthor_drv.h"
 #include "panthor_fw.h"
 #include "panthor_gem.h"
 #include "panthor_mmu.h"
 
+void panthor_gem_init(struct panthor_device *ptdev)
+{
+	int err;
+
+	if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
+	    !panthor_transparent_hugepage)
+		return;
+
+	err = drm_gem_huge_mnt_create(&ptdev->base, "within_size");
+	if (drm_gem_get_huge_mnt(&ptdev->base))
+		drm_info(&ptdev->base, "Using Transparent Hugepage\n");
+	else if (err)
+		drm_warn(&ptdev->base, "Can't use Transparent Hugepage (%d)\n",
+			 err);
+}
+
 #ifdef CONFIG_DEBUG_FS
 static void panthor_gem_debugfs_bo_init(struct panthor_gem_object *bo)
 {
@@ -59,6 +77,39 @@ static void panthor_gem_debugfs_set_usage_flags(struct panthor_gem_object *bo, u
 static void panthor_gem_debugfs_bo_init(struct panthor_gem_object *bo) {}
 #endif
 
+static bool
+should_map_wc(struct panthor_gem_object *bo, struct panthor_vm *exclusive_vm)
+{
+	struct panthor_device *ptdev = container_of(bo->base.base.dev, struct panthor_device, base);
+
+	/* We can't do uncached mappings if the device is coherent,
+	 * because the zeroing done by the shmem layer at page allocation
+	 * time happens on a cached mapping which isn't CPU-flushed (at least
+	 * not on Arm64 where the flush is deferred to PTE setup time, and
+	 * only done conditionally based on the mapping permissions). We can't
+	 * rely on dma_map_sgtable()/dma_sync_sgtable_for_xxx() either to flush
+	 * those, because they are NOPed if dma_dev_coherent() returns true.
+	 *
+	 * FIXME: Note that this problem is going to pop up again when we
+	 * decide to support mapping buffers with the NO_MMAP flag as
+	 * non-shareable (AKA buffers accessed only by the GPU), because we
+	 * need the same CPU flush to happen after page allocation, otherwise
+	 * there's a risk of data leak or late corruption caused by a dirty
+	 * cacheline being evicted. At this point we'll need a way to force
+	 * CPU cache maintenance regardless of whether the device is coherent
+	 * or not.
+	 */
+	if (ptdev->coherent)
+		return false;
+
+	/* Cached mappings are explicitly requested, so no write-combine. */
+	if (bo->flags & DRM_PANTHOR_BO_WB_MMAP)
+		return false;
+
+	/* The default is write-combine. */
+	return true;
+}
+
 static void panthor_gem_free_object(struct drm_gem_object *obj)
 {
 	struct panthor_gem_object *bo = to_panthor_bo(obj);
@@ -145,6 +196,7 @@ panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm,
 	bo = to_panthor_bo(&obj->base);
 	kbo->obj = &obj->base;
 	bo->flags = bo_flags;
+	bo->base.map_wc = should_map_wc(bo, vm);
 	bo->exclusive_vm_root_gem = panthor_vm_root_gem(vm);
 	drm_gem_object_get(bo->exclusive_vm_root_gem);
 	bo->base.base.resv = bo->exclusive_vm_root_gem->resv;
@@ -184,14 +236,130 @@ panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm,
 	return ERR_PTR(ret);
 }
 
+static struct sg_table *
+panthor_gem_prime_map_dma_buf(struct dma_buf_attachment *attach,
+			      enum dma_data_direction dir)
+{
+	struct sg_table *sgt = drm_gem_map_dma_buf(attach, dir);
+
+	if (!IS_ERR(sgt))
+		attach->priv = sgt;
+
+	return sgt;
+}
+
+static void
+panthor_gem_prime_unmap_dma_buf(struct dma_buf_attachment *attach,
+				struct sg_table *sgt,
+				enum dma_data_direction dir)
+{
+	attach->priv = NULL;
+	drm_gem_unmap_dma_buf(attach, sgt, dir);
+}
+
+static int
+panthor_gem_prime_begin_cpu_access(struct dma_buf *dma_buf,
+				   enum dma_data_direction dir)
+{
+	struct drm_gem_object *obj = dma_buf->priv;
+	struct drm_device *dev = obj->dev;
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+	struct dma_buf_attachment *attach;
+
+	dma_resv_lock(obj->resv, NULL);
+	if (shmem->sgt)
+		dma_sync_sgtable_for_cpu(dev->dev, shmem->sgt, dir);
+
+	if (shmem->vaddr)
+		invalidate_kernel_vmap_range(shmem->vaddr, shmem->base.size);
+
+	list_for_each_entry(attach, &dma_buf->attachments, node) {
+		struct sg_table *sgt = attach->priv;
+
+		if (sgt)
+			dma_sync_sgtable_for_cpu(attach->dev, sgt, dir);
+	}
+	dma_resv_unlock(obj->resv);
+
+	return 0;
+}
+
+static int
+panthor_gem_prime_end_cpu_access(struct dma_buf *dma_buf,
+				 enum dma_data_direction dir)
+{
+	struct drm_gem_object *obj = dma_buf->priv;
+	struct drm_device *dev = obj->dev;
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+	struct dma_buf_attachment *attach;
+
+	dma_resv_lock(obj->resv, NULL);
+	list_for_each_entry(attach, &dma_buf->attachments, node) {
+		struct sg_table *sgt = attach->priv;
+
+		if (sgt)
+			dma_sync_sgtable_for_device(attach->dev, sgt, dir);
+	}
+
+	if (shmem->vaddr)
+		flush_kernel_vmap_range(shmem->vaddr, shmem->base.size);
+
+	if (shmem->sgt)
+		dma_sync_sgtable_for_device(dev->dev, shmem->sgt, dir);
+
+	dma_resv_unlock(obj->resv);
+	return 0;
+}
+
+static const struct dma_buf_ops panthor_dma_buf_ops = {
+	.attach = drm_gem_map_attach,
+	.detach = drm_gem_map_detach,
+	.map_dma_buf = panthor_gem_prime_map_dma_buf,
+	.unmap_dma_buf = panthor_gem_prime_unmap_dma_buf,
+	.release = drm_gem_dmabuf_release,
+	.mmap = drm_gem_dmabuf_mmap,
+	.vmap = drm_gem_dmabuf_vmap,
+	.vunmap = drm_gem_dmabuf_vunmap,
+	.begin_cpu_access = panthor_gem_prime_begin_cpu_access,
+	.end_cpu_access = panthor_gem_prime_end_cpu_access,
+};
+
 static struct dma_buf *
 panthor_gem_prime_export(struct drm_gem_object *obj, int flags)
 {
+	struct drm_device *dev = obj->dev;
+	struct dma_buf_export_info exp_info = {
+		.exp_name = KBUILD_MODNAME,
+		.owner = THIS_MODULE,
+		.ops = &panthor_dma_buf_ops,
+		.size = obj->size,
+		.flags = flags,
+		.priv = obj,
+		.resv = obj->resv,
+	};
+
 	/* We can't export GEMs that have an exclusive VM. */
 	if (to_panthor_bo(obj)->exclusive_vm_root_gem)
 		return ERR_PTR(-EINVAL);
 
-	return drm_gem_prime_export(obj, flags);
+	return drm_gem_dmabuf_export(dev, &exp_info);
+}
+
+struct drm_gem_object *
+panthor_gem_prime_import(struct drm_device *dev,
+			 struct dma_buf *dma_buf)
+{
+	struct drm_gem_object *obj = dma_buf->priv;
+
+	if (dma_buf->ops == &panthor_dma_buf_ops && obj->dev == dev) {
+		/* Importing dmabuf exported from our own gem increases
+		 * refcount on gem itself instead of f_count of dmabuf.
+		 */
+		drm_gem_object_get(obj);
+		return obj;
+	}
+
+	return drm_gem_prime_import(dev, dma_buf);
 }
 
 static enum drm_gem_object_status panthor_gem_status(struct drm_gem_object *obj)
@@ -229,7 +397,6 @@ static const struct drm_gem_object_funcs panthor_gem_funcs = {
  */
 struct drm_gem_object *panthor_gem_create_object(struct drm_device *ddev, size_t size)
 {
-	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
 	struct panthor_gem_object *obj;
 
 	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
@@ -237,7 +404,6 @@ struct drm_gem_object *panthor_gem_create_object(struct drm_device *ddev, size_t
 		return ERR_PTR(-ENOMEM);
 
 	obj->base.base.funcs = &panthor_gem_funcs;
-	obj->base.map_wc = !ptdev->coherent;
 	mutex_init(&obj->label.lock);
 
 	panthor_gem_debugfs_bo_init(obj);
@@ -272,6 +438,7 @@ panthor_gem_create_with_handle(struct drm_file *file,
 
 	bo = to_panthor_bo(&shmem->base);
 	bo->flags = flags;
+	bo->base.map_wc = should_map_wc(bo, exclusive_vm);
 
 	if (exclusive_vm) {
 		bo->exclusive_vm_root_gem = panthor_vm_root_gem(exclusive_vm);
@@ -349,6 +516,91 @@ panthor_gem_kernel_bo_set_label(struct panthor_kernel_bo *bo, const char *label)
 	panthor_gem_bo_set_label(bo->obj, str);
 }
 
+int
+panthor_gem_sync(struct drm_gem_object *obj, u32 type,
+		 u64 offset, u64 size)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(obj);
+	struct drm_gem_shmem_object *shmem = &bo->base;
+	const struct drm_device *dev = shmem->base.dev;
+	struct sg_table *sgt;
+	struct scatterlist *sgl;
+	unsigned int count;
+
+	/* Make sure the range is in bounds. */
+	if (offset + size < offset || offset + size > shmem->base.size)
+		return -EINVAL;
+
+	/* Disallow CPU-cache maintenance on imported buffers. */
+	if (drm_gem_is_imported(&shmem->base))
+		return -EINVAL;
+
+	switch (type) {
+	case DRM_PANTHOR_BO_SYNC_CPU_CACHE_FLUSH:
+	case DRM_PANTHOR_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE:
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	/* Don't bother if it's WC-mapped */
+	if (shmem->map_wc)
+		return 0;
+
+	/* Nothing to do if the size is zero. */
+	if (size == 0)
+		return 0;
+
+	sgt = drm_gem_shmem_get_pages_sgt(shmem);
+	if (IS_ERR(sgt))
+		return PTR_ERR(sgt);
+
+	for_each_sgtable_dma_sg(sgt, sgl, count) {
+		if (size == 0)
+			break;
+
+		dma_addr_t paddr = sg_dma_address(sgl);
+		size_t len = sg_dma_len(sgl);
+
+		if (len <= offset) {
+			offset -= len;
+			continue;
+		}
+
+		paddr += offset;
+		len -= offset;
+		len = min_t(size_t, len, size);
+		size -= len;
+		offset = 0;
+
+		/* It's unclear whether dma_sync_xxx() is the right API to do CPU
+		 * cache maintenance given an IOMMU can register their own
+		 * implementation doing more than just CPU cache flushes/invalidation,
+		 * and what we really care about here is CPU caches only, but that's
+		 * the best we have that is both arch-agnostic and does at least the
+		 * CPU cache maintenance on a <page,offset,size> tuple.
+		 *
+		 * Also, I wish we could do a single
+		 *
+		 *      dma_sync_single_for_device(BIDIR)
+		 *
+		 * and get a flush+invalidate, but that's not how it's implemented
+		 * in practice (at least on arm64), so we have to make it
+		 *
+		 *      dma_sync_single_for_device(TO_DEVICE)
+		 *      dma_sync_single_for_cpu(FROM_DEVICE)
+		 *
+		 * for the flush+invalidate case.
+		 */
+		dma_sync_single_for_device(dev->dev, paddr, len, DMA_TO_DEVICE);
+		if (type == DRM_PANTHOR_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE)
+			dma_sync_single_for_cpu(dev->dev, paddr, len, DMA_FROM_DEVICE);
+	}
+
+	return 0;
+}
+
 #ifdef CONFIG_DEBUG_FS
 struct gem_size_totals {
 	size_t size;
diff --git a/drivers/gpu/drm/panthor/panthor_gem.h b/drivers/gpu/drm/panthor/panthor_gem.h
index 80c6e24112d0..22519c570b5a 100644
--- a/drivers/gpu/drm/panthor/panthor_gem.h
+++ b/drivers/gpu/drm/panthor/panthor_gem.h
@@ -136,6 +136,8 @@ struct panthor_gem_object *to_panthor_bo(struct drm_gem_object *obj)
 	return container_of(to_drm_gem_shmem_obj(obj), struct panthor_gem_object, base);
 }
 
+void panthor_gem_init(struct panthor_device *ptdev);
+
 struct drm_gem_object *panthor_gem_create_object(struct drm_device *ddev, size_t size);
 
 int
@@ -146,6 +148,12 @@ panthor_gem_create_with_handle(struct drm_file *file,
 
 void panthor_gem_bo_set_label(struct drm_gem_object *obj, const char *label);
 void panthor_gem_kernel_bo_set_label(struct panthor_kernel_bo *bo, const char *label);
+int panthor_gem_sync(struct drm_gem_object *obj,
+		     u32 type, u64 offset, u64 size);
+
+struct drm_gem_object *
+panthor_gem_prime_import(struct drm_device *dev,
+			 struct dma_buf *dma_buf);
 
 static inline u64
 panthor_kernel_bo_gpuva(struct panthor_kernel_bo *bo)
diff --git a/drivers/gpu/drm/panthor/panthor_gpu.c b/drivers/gpu/drm/panthor/panthor_gpu.c
index 06b231b2460a..057e167468d0 100644
--- a/drivers/gpu/drm/panthor/panthor_gpu.c
+++ b/drivers/gpu/drm/panthor/panthor_gpu.c
@@ -51,7 +51,7 @@ struct panthor_gpu {
 static void panthor_gpu_coherency_set(struct panthor_device *ptdev)
 {
 	gpu_write(ptdev, GPU_COHERENCY_PROTOCOL,
-		ptdev->coherent ? GPU_COHERENCY_PROT_BIT(ACE_LITE) : GPU_COHERENCY_NONE);
+		  ptdev->gpu_info.selected_coherency);
 }
 
 static void panthor_gpu_l2_config_set(struct panthor_device *ptdev)
@@ -289,38 +289,42 @@ int panthor_gpu_l2_power_on(struct panthor_device *ptdev)
 int panthor_gpu_flush_caches(struct panthor_device *ptdev,
 			     u32 l2, u32 lsc, u32 other)
 {
-	bool timedout = false;
 	unsigned long flags;
+	int ret = 0;
 
 	/* Serialize cache flush operations. */
 	guard(mutex)(&ptdev->gpu->cache_flush_lock);
 
 	spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
-	if (!drm_WARN_ON(&ptdev->base,
-			 ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) {
+	if (!(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) {
 		ptdev->gpu->pending_reqs |= GPU_IRQ_CLEAN_CACHES_COMPLETED;
 		gpu_write(ptdev, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other));
+	} else {
+		ret = -EIO;
 	}
 	spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
 
+	if (ret)
+		return ret;
+
 	if (!wait_event_timeout(ptdev->gpu->reqs_acked,
 				!(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED),
 				msecs_to_jiffies(100))) {
 		spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
 		if ((ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED) != 0 &&
 		    !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED))
-			timedout = true;
+			ret = -ETIMEDOUT;
 		else
 			ptdev->gpu->pending_reqs &= ~GPU_IRQ_CLEAN_CACHES_COMPLETED;
 		spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
 	}
 
-	if (timedout) {
+	if (ret) {
+		panthor_device_schedule_reset(ptdev);
 		drm_err(&ptdev->base, "Flush caches timeout");
-		return -ETIMEDOUT;
 	}
 
-	return 0;
+	return ret;
 }
 
 /**
@@ -360,6 +364,7 @@ int panthor_gpu_soft_reset(struct panthor_device *ptdev)
 		return -ETIMEDOUT;
 	}
 
+	ptdev->gpu->pending_reqs = 0;
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c
index d4839d282689..473a8bebd61e 100644
--- a/drivers/gpu/drm/panthor/panthor_mmu.c
+++ b/drivers/gpu/drm/panthor/panthor_mmu.c
@@ -375,6 +375,15 @@ struct panthor_vm {
 	 * flagged as faulty as a result.
 	 */
 	bool unhandled_fault;
+
+	/** @locked_region: Information about the currently locked region currently. */
+	struct {
+		/** @locked_region.start: Start of the locked region. */
+		u64 start;
+
+		/** @locked_region.size: Size of the locked region. */
+		u64 size;
+	} locked_region;
 };
 
 /**
@@ -510,27 +519,27 @@ static int wait_ready(struct panthor_device *ptdev, u32 as_nr)
 	return ret;
 }
 
-static int write_cmd(struct panthor_device *ptdev, u32 as_nr, u32 cmd)
+static int as_send_cmd_and_wait(struct panthor_device *ptdev, u32 as_nr, u32 cmd)
 {
 	int status;
 
 	/* write AS_COMMAND when MMU is ready to accept another command */
 	status = wait_ready(ptdev, as_nr);
-	if (!status)
+	if (!status) {
 		gpu_write(ptdev, AS_COMMAND(as_nr), cmd);
+		status = wait_ready(ptdev, as_nr);
+	}
 
 	return status;
 }
 
-static void lock_region(struct panthor_device *ptdev, u32 as_nr,
-			u64 region_start, u64 size)
+static u64 pack_region_range(struct panthor_device *ptdev, u64 region_start, u64 size)
 {
 	u8 region_width;
-	u64 region;
 	u64 region_end = region_start + size;
 
-	if (!size)
-		return;
+	if (drm_WARN_ON_ONCE(&ptdev->base, !size))
+		return 0;
 
 	/*
 	 * The locked region is a naturally aligned power of 2 block encoded as
@@ -549,106 +558,51 @@ static void lock_region(struct panthor_device *ptdev, u32 as_nr,
 	 */
 	region_start &= GENMASK_ULL(63, region_width);
 
-	region = region_width | region_start;
-
-	/* Lock the region that needs to be updated */
-	gpu_write64(ptdev, AS_LOCKADDR(as_nr), region);
-	write_cmd(ptdev, as_nr, AS_COMMAND_LOCK);
-}
-
-static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr,
-				      u64 iova, u64 size, u32 op)
-{
-	const u32 l2_flush_op = CACHE_CLEAN | CACHE_INV;
-	u32 lsc_flush_op;
-	int ret;
-
-	lockdep_assert_held(&ptdev->mmu->as.slots_lock);
-
-	switch (op) {
-	case AS_COMMAND_FLUSH_MEM:
-		lsc_flush_op = CACHE_CLEAN | CACHE_INV;
-		break;
-	case AS_COMMAND_FLUSH_PT:
-		lsc_flush_op = 0;
-		break;
-	default:
-		drm_WARN(&ptdev->base, 1, "Unexpected AS_COMMAND: %d", op);
-		return -EINVAL;
-	}
-
-	if (as_nr < 0)
-		return 0;
-
-	/*
-	 * If the AS number is greater than zero, then we can be sure
-	 * the device is up and running, so we don't need to explicitly
-	 * power it up
-	 */
-
-	lock_region(ptdev, as_nr, iova, size);
-
-	ret = wait_ready(ptdev, as_nr);
-	if (ret)
-		return ret;
-
-	ret = panthor_gpu_flush_caches(ptdev, l2_flush_op, lsc_flush_op, 0);
-	if (ret)
-		return ret;
-
-	/*
-	 * Explicitly unlock the region as the AS is not unlocked automatically
-	 * at the end of the GPU_CONTROL cache flush command, unlike
-	 * AS_COMMAND_FLUSH_MEM or AS_COMMAND_FLUSH_PT.
-	 */
-	write_cmd(ptdev, as_nr, AS_COMMAND_UNLOCK);
-
-	/* Wait for the unlock command to complete */
-	return wait_ready(ptdev, as_nr);
-}
-
-static int mmu_hw_do_operation(struct panthor_vm *vm,
-			       u64 iova, u64 size, u32 op)
-{
-	struct panthor_device *ptdev = vm->ptdev;
-	int ret;
-
-	mutex_lock(&ptdev->mmu->as.slots_lock);
-	ret = mmu_hw_do_operation_locked(ptdev, vm->as.id, iova, size, op);
-	mutex_unlock(&ptdev->mmu->as.slots_lock);
-
-	return ret;
+	return region_width | region_start;
 }
 
 static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr,
 				 u64 transtab, u64 transcfg, u64 memattr)
 {
-	int ret;
-
-	ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);
-	if (ret)
-		return ret;
-
 	gpu_write64(ptdev, AS_TRANSTAB(as_nr), transtab);
 	gpu_write64(ptdev, AS_MEMATTR(as_nr), memattr);
 	gpu_write64(ptdev, AS_TRANSCFG(as_nr), transcfg);
 
-	return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE);
+	return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UPDATE);
 }
 
-static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr)
+static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr,
+				  bool recycle_slot)
 {
+	struct panthor_vm *vm = ptdev->mmu->as.slots[as_nr].vm;
 	int ret;
 
-	ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);
+	lockdep_assert_held(&ptdev->mmu->as.slots_lock);
+
+	/* Flush+invalidate RW caches, invalidate RO ones. */
+	ret = panthor_gpu_flush_caches(ptdev, CACHE_CLEAN | CACHE_INV,
+				       CACHE_CLEAN | CACHE_INV, CACHE_INV);
 	if (ret)
 		return ret;
 
+	if (vm && vm->locked_region.size) {
+		/* Unlock the region if there's a lock pending. */
+		ret = as_send_cmd_and_wait(ptdev, vm->as.id, AS_COMMAND_UNLOCK);
+		if (ret)
+			return ret;
+	}
+
+	/* If the slot is going to be used immediately, don't bother changing
+	 * the config.
+	 */
+	if (recycle_slot)
+		return 0;
+
 	gpu_write64(ptdev, AS_TRANSTAB(as_nr), 0);
 	gpu_write64(ptdev, AS_MEMATTR(as_nr), 0);
 	gpu_write64(ptdev, AS_TRANSCFG(as_nr), AS_TRANSCFG_ADRMODE_UNMAPPED);
 
-	return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE);
+	return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UPDATE);
 }
 
 static u32 panthor_mmu_fault_mask(struct panthor_device *ptdev, u32 value)
@@ -722,6 +676,10 @@ int panthor_vm_active(struct panthor_vm *vm)
 	if (refcount_inc_not_zero(&vm->as.active_cnt))
 		goto out_dev_exit;
 
+	/* Make sure we don't race with lock/unlock_region() calls
+	 * happening around VM bind operations.
+	 */
+	mutex_lock(&vm->op_lock);
 	mutex_lock(&ptdev->mmu->as.slots_lock);
 
 	if (refcount_inc_not_zero(&vm->as.active_cnt))
@@ -759,6 +717,11 @@ int panthor_vm_active(struct panthor_vm *vm)
 
 		drm_WARN_ON(&ptdev->base, refcount_read(&lru_vm->as.active_cnt));
 		as = lru_vm->as.id;
+
+		ret = panthor_mmu_as_disable(ptdev, as, true);
+		if (ret)
+			goto out_unlock;
+
 		panthor_vm_release_as_locked(lru_vm);
 	}
 
@@ -789,6 +752,10 @@ int panthor_vm_active(struct panthor_vm *vm)
 		gpu_write(ptdev, MMU_INT_MASK, ~ptdev->mmu->as.faulty_mask);
 	}
 
+	/* The VM update is guarded by ::op_lock, which we take at the beginning
+	 * of this function, so we don't expect any locked region here.
+	 */
+	drm_WARN_ON(&vm->ptdev->base, vm->locked_region.size > 0);
 	ret = panthor_mmu_as_enable(vm->ptdev, vm->as.id, transtab, transcfg, vm->memattr);
 
 out_make_active:
@@ -799,6 +766,7 @@ int panthor_vm_active(struct panthor_vm *vm)
 
 out_unlock:
 	mutex_unlock(&ptdev->mmu->as.slots_lock);
+	mutex_unlock(&vm->op_lock);
 
 out_dev_exit:
 	drm_dev_exit(cookie);
@@ -882,44 +850,64 @@ static size_t get_pgsize(u64 addr, size_t size, size_t *count)
 	return SZ_2M;
 }
 
-static int panthor_vm_flush_range(struct panthor_vm *vm, u64 iova, u64 size)
+static void panthor_vm_declare_unusable(struct panthor_vm *vm)
 {
 	struct panthor_device *ptdev = vm->ptdev;
-	int ret = 0, cookie;
-
-	if (vm->as.id < 0)
-		return 0;
+	int cookie;
 
-	/* If the device is unplugged, we just silently skip the flush. */
-	if (!drm_dev_enter(&ptdev->base, &cookie))
-		return 0;
-
-	ret = mmu_hw_do_operation(vm, iova, size, AS_COMMAND_FLUSH_PT);
+	if (vm->unusable)
+		return;
 
-	drm_dev_exit(cookie);
-	return ret;
+	vm->unusable = true;
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+	if (vm->as.id >= 0 && drm_dev_enter(&ptdev->base, &cookie)) {
+		panthor_mmu_as_disable(ptdev, vm->as.id, false);
+		drm_dev_exit(cookie);
+	}
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
 }
 
-static int panthor_vm_unmap_pages(struct panthor_vm *vm, u64 iova, u64 size)
+static void panthor_vm_unmap_pages(struct panthor_vm *vm, u64 iova, u64 size)
 {
 	struct panthor_device *ptdev = vm->ptdev;
 	struct io_pgtable_ops *ops = vm->pgtbl_ops;
 	u64 start_iova = iova;
 	u64 offset = 0;
 
+	if (!size)
+		return;
+
+	drm_WARN_ON(&ptdev->base,
+		    (iova < vm->locked_region.start) ||
+		    (iova + size > vm->locked_region.start + vm->locked_region.size));
+
 	while (offset < size) {
 		size_t unmapped_sz = 0, pgcount;
 		size_t pgsize = get_pgsize(iova + offset, size - offset, &pgcount);
 
 		unmapped_sz = ops->unmap_pages(ops, iova + offset, pgsize, pgcount, NULL);
+		if (drm_WARN_ON_ONCE(&ptdev->base, unmapped_sz != pgsize * pgcount)) {
+			/* Gracefully handle sparsely unmapped regions to avoid leaving
+			 * page table pages behind when the drm_gpuvm and VM page table
+			 * are out-of-sync. This is not supposed to happen, hence the
+			 * above WARN_ON().
+			 */
+			while (!ops->iova_to_phys(ops, iova + unmapped_sz) &&
+			       unmapped_sz < pgsize * pgcount)
+				unmapped_sz += SZ_4K;
+
+			/* We're passed the point where we can try to fix things,
+			 * so flag the VM unusable to make sure it's not going
+			 * to be used anymore.
+			 */
+			panthor_vm_declare_unusable(vm);
 
-		if (drm_WARN_ON(&ptdev->base, unmapped_sz != pgsize * pgcount)) {
-			drm_err(&ptdev->base, "failed to unmap range %llx-%llx (requested range %llx-%llx)\n",
-				iova + offset + unmapped_sz,
-				iova + offset + pgsize * pgcount,
-				iova, iova + size);
-			panthor_vm_flush_range(vm, iova, offset + unmapped_sz);
-			return  -EINVAL;
+			/* If we don't make progress, we're screwed. That also means
+			 * something else prevents us from unmapping the region, but
+			 * there's not much we can do here: time for debugging.
+			 */
+			if (drm_WARN_ON_ONCE(&ptdev->base, !unmapped_sz))
+				return;
 		}
 
 		drm_dbg(&ptdev->base,
@@ -929,8 +917,6 @@ static int panthor_vm_unmap_pages(struct panthor_vm *vm, u64 iova, u64 size)
 
 		offset += unmapped_sz;
 	}
-
-	return panthor_vm_flush_range(vm, iova, size);
 }
 
 static int
@@ -948,6 +934,10 @@ panthor_vm_map_pages(struct panthor_vm *vm, u64 iova, int prot,
 	if (!size)
 		return 0;
 
+	drm_WARN_ON(&ptdev->base,
+		    (iova < vm->locked_region.start) ||
+		    (iova + size > vm->locked_region.start + vm->locked_region.size));
+
 	for_each_sgtable_dma_sg(sgt, sgl, count) {
 		dma_addr_t paddr = sg_dma_address(sgl);
 		size_t len = sg_dma_len(sgl);
@@ -978,16 +968,17 @@ panthor_vm_map_pages(struct panthor_vm *vm, u64 iova, int prot,
 			paddr += mapped;
 			len -= mapped;
 
-			if (drm_WARN_ON(&ptdev->base, !ret && !mapped))
+			/* If nothing was mapped, consider it an ENOMEM. */
+			if (!ret && !mapped)
 				ret = -ENOMEM;
 
-			if (ret) {
-				/* If something failed, unmap what we've already mapped before
-				 * returning. The unmap call is not supposed to fail.
+			/* If something fails, we stop there, and flag the VM unusable. */
+			if (drm_WARN_ON_ONCE(&ptdev->base, ret)) {
+				/* Unmap what we've already mapped to avoid leaving page
+				 * table pages behind.
 				 */
-				drm_WARN_ON(&ptdev->base,
-					    panthor_vm_unmap_pages(vm, start_iova,
-								   iova - start_iova));
+				panthor_vm_unmap_pages(vm, start_iova, iova - start_iova);
+				panthor_vm_declare_unusable(vm);
 				return ret;
 			}
 		}
@@ -998,7 +989,7 @@ panthor_vm_map_pages(struct panthor_vm *vm, u64 iova, int prot,
 		offset = 0;
 	}
 
-	return panthor_vm_flush_range(vm, start_iova, iova - start_iova);
+	return 0;
 }
 
 static int flags_to_prot(u32 flags)
@@ -1641,6 +1632,62 @@ static const char *access_type_name(struct panthor_device *ptdev,
 	}
 }
 
+static int panthor_vm_lock_region(struct panthor_vm *vm, u64 start, u64 size)
+{
+	struct panthor_device *ptdev = vm->ptdev;
+	int ret = 0;
+
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+	drm_WARN_ON(&ptdev->base, vm->locked_region.start || vm->locked_region.size);
+	if (vm->as.id >= 0 && size) {
+		/* Lock the region that needs to be updated */
+		gpu_write64(ptdev, AS_LOCKADDR(vm->as.id),
+			    pack_region_range(ptdev, start, size));
+
+		/* If the lock succeeded, update the locked_region info. */
+		ret = as_send_cmd_and_wait(ptdev, vm->as.id, AS_COMMAND_LOCK);
+	}
+
+	if (!ret) {
+		vm->locked_region.start = start;
+		vm->locked_region.size = size;
+	}
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+
+	return ret;
+}
+
+static void panthor_vm_unlock_region(struct panthor_vm *vm)
+{
+	struct panthor_device *ptdev = vm->ptdev;
+
+	mutex_lock(&ptdev->mmu->as.slots_lock);
+	if (vm->as.id >= 0) {
+		int ret;
+
+		/* flush+invalidate RW caches and invalidate RO ones.
+		 * TODO: See if we can use FLUSH_PA_RANGE when the physical
+		 * range is narrow enough and the HW supports it.
+		 */
+		ret = panthor_gpu_flush_caches(ptdev, CACHE_CLEAN | CACHE_INV,
+					       CACHE_CLEAN | CACHE_INV,
+					       CACHE_INV);
+
+		/* Unlock the region if the flush is effective. */
+		if (!ret)
+			ret = as_send_cmd_and_wait(ptdev, vm->as.id, AS_COMMAND_UNLOCK);
+
+		/* If we fail to flush or unlock the region, schedule a GPU reset
+		 * to unblock the situation.
+		 */
+		if (ret)
+			panthor_device_schedule_reset(ptdev);
+	}
+	vm->locked_region.start = 0;
+	vm->locked_region.size = 0;
+	mutex_unlock(&ptdev->mmu->as.slots_lock);
+}
+
 static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
 {
 	bool has_unhandled_faults = false;
@@ -1701,7 +1748,7 @@ static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
 			ptdev->mmu->as.slots[as].vm->unhandled_fault = true;
 
 		/* Disable the MMU to kill jobs on this AS. */
-		panthor_mmu_as_disable(ptdev, as);
+		panthor_mmu_as_disable(ptdev, as, false);
 		mutex_unlock(&ptdev->mmu->as.slots_lock);
 
 		status &= ~mask;
@@ -1730,7 +1777,8 @@ void panthor_mmu_suspend(struct panthor_device *ptdev)
 		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
 
 		if (vm) {
-			drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i));
+			drm_WARN_ON(&ptdev->base,
+				    panthor_mmu_as_disable(ptdev, i, false));
 			panthor_vm_release_as_locked(vm);
 		}
 	}
@@ -1845,12 +1893,13 @@ static void panthor_vm_free(struct drm_gpuvm *gpuvm)
 	drm_sched_entity_destroy(&vm->entity);
 	drm_sched_fini(&vm->sched);
 
+	mutex_lock(&vm->op_lock);
 	mutex_lock(&ptdev->mmu->as.slots_lock);
 	if (vm->as.id >= 0) {
 		int cookie;
 
 		if (drm_dev_enter(&ptdev->base, &cookie)) {
-			panthor_mmu_as_disable(ptdev, vm->as.id);
+			panthor_mmu_as_disable(ptdev, vm->as.id, false);
 			drm_dev_exit(cookie);
 		}
 
@@ -1859,6 +1908,7 @@ static void panthor_vm_free(struct drm_gpuvm *gpuvm)
 		list_del(&vm->as.lru_node);
 	}
 	mutex_unlock(&ptdev->mmu->as.slots_lock);
+	mutex_unlock(&vm->op_lock);
 
 	free_io_pgtable_ops(vm->pgtbl_ops);
 
@@ -2060,12 +2110,9 @@ static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op,
 	struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
 	struct panthor_vma *prev_vma = NULL, *next_vma = NULL;
 	u64 unmap_start, unmap_range;
-	int ret;
 
 	drm_gpuva_op_remap_to_unmap_range(&op->remap, &unmap_start, &unmap_range);
-	ret = panthor_vm_unmap_pages(vm, unmap_start, unmap_range);
-	if (ret)
-		return ret;
+	panthor_vm_unmap_pages(vm, unmap_start, unmap_range);
 
 	if (op->remap.prev) {
 		prev_vma = panthor_vm_op_ctx_get_vma(op_ctx);
@@ -2103,13 +2150,9 @@ static int panthor_gpuva_sm_step_unmap(struct drm_gpuva_op *op,
 {
 	struct panthor_vma *unmap_vma = container_of(op->unmap.va, struct panthor_vma, base);
 	struct panthor_vm *vm = priv;
-	int ret;
-
-	ret = panthor_vm_unmap_pages(vm, unmap_vma->base.va.addr,
-				     unmap_vma->base.va.range);
-	if (drm_WARN_ON(&vm->ptdev->base, ret))
-		return ret;
 
+	panthor_vm_unmap_pages(vm, unmap_vma->base.va.addr,
+			       unmap_vma->base.va.range);
 	drm_gpuva_unmap(&op->unmap);
 	panthor_vma_unlink(unmap_vma);
 	return 0;
@@ -2154,6 +2197,11 @@ panthor_vm_exec_op(struct panthor_vm *vm, struct panthor_vm_op_ctx *op,
 
 	mutex_lock(&vm->op_lock);
 	vm->op_ctx = op;
+
+	ret = panthor_vm_lock_region(vm, op->va.addr, op->va.range);
+	if (ret)
+		goto out;
+
 	switch (op_type) {
 	case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP: {
 		const struct drm_gpuvm_map_req map_req = {
@@ -2181,8 +2229,11 @@ panthor_vm_exec_op(struct panthor_vm *vm, struct panthor_vm_op_ctx *op,
 		break;
 	}
 
+	panthor_vm_unlock_region(vm);
+
+out:
 	if (ret && flag_vm_unusable_on_failure)
-		vm->unusable = true;
+		panthor_vm_declare_unusable(vm);
 
 	vm->op_ctx = NULL;
 	mutex_unlock(&vm->op_lock);
@@ -2694,7 +2745,8 @@ void panthor_mmu_unplug(struct panthor_device *ptdev)
 		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
 
 		if (vm) {
-			drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i));
+			drm_WARN_ON(&ptdev->base,
+				    panthor_mmu_as_disable(ptdev, i, false));
 			panthor_vm_release_as_locked(vm);
 		}
 	}
diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c
index a6b8024e1a3c..0f83e778d89a 100644
--- a/drivers/gpu/drm/panthor/panthor_sched.c
+++ b/drivers/gpu/drm/panthor/panthor_sched.c
@@ -23,6 +23,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/rcupdate.h>
 
 #include "panthor_devfreq.h"
 #include "panthor_device.h"
@@ -108,15 +109,6 @@ struct panthor_csg_slot {
 
 	/** @priority: Group priority. */
 	u8 priority;
-
-	/**
-	 * @idle: True if the group bound to this slot is idle.
-	 *
-	 * A group is idle when it has nothing waiting for execution on
-	 * all its queues, or when queues are blocked waiting for something
-	 * to happen (synchronization object).
-	 */
-	bool idle;
 };
 
 /**
@@ -878,8 +870,11 @@ panthor_queue_get_syncwait_obj(struct panthor_group *group, struct panthor_queue
 	struct iosys_map map;
 	int ret;
 
-	if (queue->syncwait.kmap)
-		return queue->syncwait.kmap + queue->syncwait.offset;
+	if (queue->syncwait.kmap) {
+		bo = container_of(queue->syncwait.obj,
+				  struct panthor_gem_object, base.base);
+		goto out_sync;
+	}
 
 	bo = panthor_vm_get_bo_for_va(group->vm,
 				      queue->syncwait.gpu_va,
@@ -896,6 +891,17 @@ panthor_queue_get_syncwait_obj(struct panthor_group *group, struct panthor_queue
 	if (drm_WARN_ON(&ptdev->base, !queue->syncwait.kmap))
 		goto err_put_syncwait_obj;
 
+out_sync:
+	/* Make sure the CPU caches are invalidated before the seqno is read.
+	 * drm_gem_shmem_sync() is a NOP if map_wc=true, so no need to check
+	 * it here.
+	 */
+	panthor_gem_sync(&bo->base.base, queue->syncwait.offset,
+			 queue->syncwait.sync64 ?
+			 sizeof(struct panthor_syncobj_64b) :
+			 sizeof(struct panthor_syncobj_32b),
+			 DRM_PANTHOR_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE);
+
 	return queue->syncwait.kmap + queue->syncwait.offset;
 
 err_put_syncwait_obj:
@@ -908,9 +914,8 @@ static void group_free_queue(struct panthor_group *group, struct panthor_queue *
 	if (IS_ERR_OR_NULL(queue))
 		return;
 
-	/* This should have been disabled before that point. */
-	drm_WARN_ON(&group->ptdev->base,
-		    disable_delayed_work_sync(&queue->timeout.work));
+	/* Disable the timeout before tearing down drm_sched components. */
+	disable_delayed_work_sync(&queue->timeout.work);
 
 	if (queue->entity.fence_context)
 		drm_sched_entity_destroy(&queue->entity);
@@ -939,6 +944,9 @@ static void group_release_work(struct work_struct *work)
 						   release_work);
 	u32 i;
 
+	/* dma-fences may still be accessing group->queues under rcu lock. */
+	synchronize_rcu();
+
 	for (i = 0; i < group->queue_count; i++)
 		group_free_queue(group, group->queues[i]);
 
@@ -1062,28 +1070,11 @@ group_unbind_locked(struct panthor_group *group)
 static bool
 group_is_idle(struct panthor_group *group)
 {
-	struct panthor_device *ptdev = group->ptdev;
-	u32 inactive_queues;
+	u32 inactive_queues = group->idle_queues | group->blocked_queues;
 
-	if (group->csg_id >= 0)
-		return ptdev->scheduler->csg_slots[group->csg_id].idle;
-
-	inactive_queues = group->idle_queues | group->blocked_queues;
 	return hweight32(inactive_queues) == group->queue_count;
 }
 
-static void
-queue_reset_timeout_locked(struct panthor_queue *queue)
-{
-	lockdep_assert_held(&queue->fence_ctx.lock);
-
-	if (queue->timeout.remaining != MAX_SCHEDULE_TIMEOUT) {
-		mod_delayed_work(queue->scheduler.timeout_wq,
-				 &queue->timeout.work,
-				 msecs_to_jiffies(JOB_TIMEOUT_MS));
-	}
-}
-
 static bool
 group_can_run(struct panthor_group *group)
 {
@@ -1100,6 +1091,18 @@ queue_timeout_is_suspended(struct panthor_queue *queue)
 	return queue->timeout.remaining != MAX_SCHEDULE_TIMEOUT;
 }
 
+static void
+queue_reset_timeout_locked(struct panthor_queue *queue)
+{
+	lockdep_assert_held(&queue->fence_ctx.lock);
+
+	if (!queue_timeout_is_suspended(queue)) {
+		mod_delayed_work(queue->scheduler.timeout_wq,
+				 &queue->timeout.work,
+				 msecs_to_jiffies(JOB_TIMEOUT_MS));
+	}
+}
+
 static void
 queue_suspend_timeout_locked(struct panthor_queue *queue)
 {
@@ -1200,12 +1203,10 @@ cs_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id)
 	panthor_fw_update_reqs(cs_iface, req,
 			       CS_IDLE_SYNC_WAIT |
 			       CS_IDLE_EMPTY |
-			       CS_STATE_START |
-			       CS_EXTRACT_EVENT,
+			       CS_STATE_START,
 			       CS_IDLE_SYNC_WAIT |
 			       CS_IDLE_EMPTY |
-			       CS_STATE_MASK |
-			       CS_EXTRACT_EVENT);
+			       CS_STATE_MASK);
 	if (queue->iface.input->insert != queue->iface.input->extract)
 		queue_resume_timeout(queue);
 }
@@ -1725,17 +1726,6 @@ static bool cs_slot_process_irq_locked(struct panthor_device *ptdev,
 	return (events & (CS_FAULT | CS_TILER_OOM)) != 0;
 }
 
-static void csg_slot_sync_idle_state_locked(struct panthor_device *ptdev, u32 csg_id)
-{
-	struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
-	struct panthor_fw_csg_iface *csg_iface;
-
-	lockdep_assert_held(&ptdev->scheduler->lock);
-
-	csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
-	csg_slot->idle = csg_iface->output->status_state & CSG_STATUS_STATE_IS_IDLE;
-}
-
 static void csg_slot_process_idle_event_locked(struct panthor_device *ptdev, u32 csg_id)
 {
 	struct panthor_scheduler *sched = ptdev->scheduler;
@@ -1997,10 +1987,8 @@ static int csgs_upd_ctx_apply_locked(struct panthor_device *ptdev,
 		if (acked & CSG_STATE_MASK)
 			csg_slot_sync_state_locked(ptdev, csg_id);
 
-		if (acked & CSG_STATUS_UPDATE) {
+		if (acked & CSG_STATUS_UPDATE)
 			csg_slot_sync_queues_state_locked(ptdev, csg_id);
-			csg_slot_sync_idle_state_locked(ptdev, csg_id);
-		}
 
 		if (ret && acked != req_mask &&
 		    ((csg_iface->input->req ^ csg_iface->output->ack) & req_mask) != 0) {
@@ -2020,10 +2008,10 @@ struct panthor_sched_tick_ctx {
 	struct list_head groups[PANTHOR_CSG_PRIORITY_COUNT];
 	u32 idle_group_count;
 	u32 group_count;
-	enum panthor_csg_priority min_priority;
 	struct panthor_vm *vms[MAX_CS_PER_CSG];
 	u32 as_count;
 	bool immediate_tick;
+	bool stop_tick;
 	u32 csg_upd_failed_mask;
 };
 
@@ -2066,17 +2054,21 @@ tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched,
 		if (!owned_by_tick_ctx)
 			group_get(group);
 
-		list_move_tail(&group->run_node, &ctx->groups[group->priority]);
 		ctx->group_count++;
+
+		/* If we have more than one active group with the same priority,
+		 * we need to keep ticking to rotate the CSG priority.
+		 */
 		if (group_is_idle(group))
 			ctx->idle_group_count++;
+		else if (!list_empty(&ctx->groups[group->priority]))
+			ctx->stop_tick = false;
+
+		list_move_tail(&group->run_node, &ctx->groups[group->priority]);
 
 		if (i == ctx->as_count)
 			ctx->vms[ctx->as_count++] = group->vm;
 
-		if (ctx->min_priority > group->priority)
-			ctx->min_priority = group->priority;
-
 		if (tick_ctx_is_full(sched, ctx))
 			return;
 	}
@@ -2085,31 +2077,22 @@ tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched,
 static void
 tick_ctx_insert_old_group(struct panthor_scheduler *sched,
 			  struct panthor_sched_tick_ctx *ctx,
-			  struct panthor_group *group,
-			  bool full_tick)
+			  struct panthor_group *group)
 {
 	struct panthor_csg_slot *csg_slot = &sched->csg_slots[group->csg_id];
 	struct panthor_group *other_group;
 
-	if (!full_tick) {
-		list_add_tail(&group->run_node, &ctx->old_groups[group->priority]);
-		return;
-	}
-
-	/* Rotate to make sure groups with lower CSG slot
-	 * priorities have a chance to get a higher CSG slot
-	 * priority next time they get picked. This priority
-	 * has an impact on resource request ordering, so it's
-	 * important to make sure we don't let one group starve
-	 * all other groups with the same group priority.
-	 */
+	/* Class groups in descending priority order so we can easily rotate. */
 	list_for_each_entry(other_group,
 			    &ctx->old_groups[csg_slot->group->priority],
 			    run_node) {
 		struct panthor_csg_slot *other_csg_slot = &sched->csg_slots[other_group->csg_id];
 
-		if (other_csg_slot->priority > csg_slot->priority) {
-			list_add_tail(&csg_slot->group->run_node, &other_group->run_node);
+		/* Our group has a higher prio than the one we're testing against,
+		 * place it just before.
+		 */
+		if (csg_slot->priority > other_csg_slot->priority) {
+			list_add_tail(&group->run_node, &other_group->run_node);
 			return;
 		}
 	}
@@ -2119,8 +2102,7 @@ tick_ctx_insert_old_group(struct panthor_scheduler *sched,
 
 static void
 tick_ctx_init(struct panthor_scheduler *sched,
-	      struct panthor_sched_tick_ctx *ctx,
-	      bool full_tick)
+	      struct panthor_sched_tick_ctx *ctx)
 {
 	struct panthor_device *ptdev = sched->ptdev;
 	struct panthor_csg_slots_upd_ctx upd_ctx;
@@ -2130,7 +2112,7 @@ tick_ctx_init(struct panthor_scheduler *sched,
 	memset(ctx, 0, sizeof(*ctx));
 	csgs_upd_ctx_init(&upd_ctx);
 
-	ctx->min_priority = PANTHOR_CSG_PRIORITY_COUNT;
+	ctx->stop_tick = true;
 	for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) {
 		INIT_LIST_HEAD(&ctx->groups[i]);
 		INIT_LIST_HEAD(&ctx->old_groups[i]);
@@ -2158,7 +2140,7 @@ tick_ctx_init(struct panthor_scheduler *sched,
 				group->fatal_queues |= GENMASK(group->queue_count - 1, 0);
 		}
 
-		tick_ctx_insert_old_group(sched, ctx, group, full_tick);
+		tick_ctx_insert_old_group(sched, ctx, group);
 		csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i,
 					csg_iface->output->ack ^ CSG_STATUS_UPDATE,
 					CSG_STATUS_UPDATE);
@@ -2442,32 +2424,18 @@ static u64
 tick_ctx_update_resched_target(struct panthor_scheduler *sched,
 			       const struct panthor_sched_tick_ctx *ctx)
 {
-	/* We had space left, no need to reschedule until some external event happens. */
-	if (!tick_ctx_is_full(sched, ctx))
-		goto no_tick;
-
-	/* If idle groups were scheduled, no need to wake up until some external
-	 * event happens (group unblocked, new job submitted, ...).
-	 */
-	if (ctx->idle_group_count)
-		goto no_tick;
+	u64 resched_target;
 
-	if (drm_WARN_ON(&sched->ptdev->base, ctx->min_priority >= PANTHOR_CSG_PRIORITY_COUNT))
+	if (ctx->stop_tick)
 		goto no_tick;
 
-	/* If there are groups of the same priority waiting, we need to
-	 * keep the scheduler ticking, otherwise, we'll just wait for
-	 * new groups with higher priority to be queued.
-	 */
-	if (!list_empty(&sched->groups.runnable[ctx->min_priority])) {
-		u64 resched_target = sched->last_tick + sched->tick_period;
+	resched_target = sched->last_tick + sched->tick_period;
 
-		if (time_before64(sched->resched_target, sched->last_tick) ||
-		    time_before64(resched_target, sched->resched_target))
-			sched->resched_target = resched_target;
+	if (time_before64(sched->resched_target, sched->last_tick) ||
+	    time_before64(resched_target, sched->resched_target))
+		sched->resched_target = resched_target;
 
-		return sched->resched_target - sched->last_tick;
-	}
+	return sched->resched_target - sched->last_tick;
 
 no_tick:
 	sched->resched_target = U64_MAX;
@@ -2480,9 +2448,11 @@ static void tick_work(struct work_struct *work)
 						      tick_work.work);
 	struct panthor_device *ptdev = sched->ptdev;
 	struct panthor_sched_tick_ctx ctx;
+	u64 resched_target = sched->resched_target;
 	u64 remaining_jiffies = 0, resched_delay;
 	u64 now = get_jiffies_64();
 	int prio, ret, cookie;
+	bool full_tick;
 
 	if (!drm_dev_enter(&ptdev->base, &cookie))
 		return;
@@ -2491,18 +2461,24 @@ static void tick_work(struct work_struct *work)
 	if (drm_WARN_ON(&ptdev->base, ret))
 		goto out_dev_exit;
 
-	if (time_before64(now, sched->resched_target))
-		remaining_jiffies = sched->resched_target - now;
+	/* If the tick is stopped, calculate when the next tick would be */
+	if (resched_target == U64_MAX)
+		resched_target = sched->last_tick + sched->tick_period;
+
+	if (time_before64(now, resched_target))
+		remaining_jiffies = resched_target - now;
+
+	full_tick = remaining_jiffies == 0;
 
 	mutex_lock(&sched->lock);
 	if (panthor_device_reset_is_pending(sched->ptdev))
 		goto out_unlock;
 
-	tick_ctx_init(sched, &ctx, remaining_jiffies != 0);
+	tick_ctx_init(sched, &ctx);
 	if (ctx.csg_upd_failed_mask)
 		goto out_cleanup_ctx;
 
-	if (remaining_jiffies) {
+	if (!full_tick) {
 		/* Scheduling forced in the middle of a tick. Only RT groups
 		 * can preempt non-RT ones. Currently running RT groups can't be
 		 * preempted.
@@ -2524,9 +2500,29 @@ static void tick_work(struct work_struct *work)
 	for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1;
 	     prio >= 0 && !tick_ctx_is_full(sched, &ctx);
 	     prio--) {
+		struct panthor_group *old_highest_prio_group =
+			list_first_entry_or_null(&ctx.old_groups[prio],
+						 struct panthor_group, run_node);
+
+		/* Pull out the group with the highest prio for rotation. */
+		if (old_highest_prio_group)
+			list_del(&old_highest_prio_group->run_node);
+
+		/* Re-insert old active groups so they get a chance to run with higher prio. */
+		tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true);
+
+		/* Fill the remaining slots with runnable groups. */
 		tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.runnable[prio],
 					       true, false);
-		tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true);
+
+		/* Re-insert the old group with the highest prio, and give it a chance to be
+		 * scheduled again (but with a lower prio) if there's room left.
+		 */
+		if (old_highest_prio_group) {
+			list_add_tail(&old_highest_prio_group->run_node, &ctx.old_groups[prio]);
+			tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio],
+						       true, true);
+		}
 	}
 
 	/* If we have free CSG slots left, pick idle groups */
@@ -2651,14 +2647,33 @@ static void sync_upd_work(struct work_struct *work)
 		sched_queue_delayed_work(sched, tick, 0);
 }
 
+static void sched_resume_tick(struct panthor_device *ptdev)
+{
+	struct panthor_scheduler *sched = ptdev->scheduler;
+	u64 delay_jiffies, now;
+
+	drm_WARN_ON(&ptdev->base, sched->resched_target != U64_MAX);
+
+	/* Scheduler tick was off, recalculate the resched_target based on the
+	 * last tick event, and queue the scheduler work.
+	 */
+	now = get_jiffies_64();
+	sched->resched_target = sched->last_tick + sched->tick_period;
+	if (sched->used_csg_slot_count == sched->csg_slot_count &&
+	    time_before64(now, sched->resched_target))
+		delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX);
+	else
+		delay_jiffies = 0;
+
+	sched_queue_delayed_work(sched, tick, delay_jiffies);
+}
+
 static void group_schedule_locked(struct panthor_group *group, u32 queue_mask)
 {
 	struct panthor_device *ptdev = group->ptdev;
 	struct panthor_scheduler *sched = ptdev->scheduler;
 	struct list_head *queue = &sched->groups.runnable[group->priority];
-	u64 delay_jiffies = 0;
 	bool was_idle;
-	u64 now;
 
 	if (!group_can_run(group))
 		return;
@@ -2703,13 +2718,7 @@ static void group_schedule_locked(struct panthor_group *group, u32 queue_mask)
 	/* Scheduler tick was off, recalculate the resched_target based on the
 	 * last tick event, and queue the scheduler work.
 	 */
-	now = get_jiffies_64();
-	sched->resched_target = sched->last_tick + sched->tick_period;
-	if (sched->used_csg_slot_count == sched->csg_slot_count &&
-	    time_before64(now, sched->resched_target))
-		delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX);
-
-	sched_queue_delayed_work(sched, tick, delay_jiffies);
+	sched_resume_tick(ptdev);
 }
 
 static void queue_stop(struct panthor_queue *queue,
@@ -2767,13 +2776,6 @@ static void panthor_group_start(struct panthor_group *group)
 	group_put(group);
 }
 
-static void panthor_sched_immediate_tick(struct panthor_device *ptdev)
-{
-	struct panthor_scheduler *sched = ptdev->scheduler;
-
-	sched_queue_delayed_work(sched, tick, 0);
-}
-
 /**
  * panthor_sched_report_mmu_fault() - Report MMU faults to the scheduler.
  */
@@ -2781,13 +2783,13 @@ void panthor_sched_report_mmu_fault(struct panthor_device *ptdev)
 {
 	/* Force a tick to immediately kill faulty groups. */
 	if (ptdev->scheduler)
-		panthor_sched_immediate_tick(ptdev);
+		sched_queue_delayed_work(ptdev->scheduler, tick, 0);
 }
 
 void panthor_sched_resume(struct panthor_device *ptdev)
 {
 	/* Force a tick to re-evaluate after a resume. */
-	panthor_sched_immediate_tick(ptdev);
+	sched_queue_delayed_work(ptdev->scheduler, tick, 0);
 }
 
 void panthor_sched_suspend(struct panthor_device *ptdev)
@@ -2943,8 +2945,6 @@ void panthor_sched_pre_reset(struct panthor_device *ptdev)
 	 * new jobs while we're resetting.
 	 */
 	for (i = 0; i < ARRAY_SIZE(sched->groups.runnable); i++) {
-		/* All groups should be in the idle lists. */
-		drm_WARN_ON(&ptdev->base, !list_empty(&sched->groups.runnable[i]));
 		list_for_each_entry_safe(group, group_tmp, &sched->groups.runnable[i], run_node)
 			panthor_group_stop(group);
 	}
@@ -3343,6 +3343,18 @@ queue_run_job(struct drm_sched_job *sched_job)
 	if (group->csg_id < 0) {
 		group_schedule_locked(group, BIT(job->queue_idx));
 	} else {
+		u32 queue_mask = BIT(job->queue_idx);
+		bool resume_tick = group_is_idle(group) &&
+				   (group->idle_queues & queue_mask) &&
+				   !(group->blocked_queues & queue_mask) &&
+				   sched->resched_target == U64_MAX;
+
+		/* We just added something to the queue, so it's no longer idle. */
+		group->idle_queues &= ~queue_mask;
+
+		if (resume_tick)
+			sched_resume_tick(ptdev);
+
 		gpu_write(ptdev, CSF_DOORBELL(queue->doorbell_id), 1);
 		if (!sched->pm.has_ref &&
 		    !(group->blocked_queues & BIT(job->queue_idx))) {
diff --git a/drivers/gpu/drm/pl111/pl111_display.c b/drivers/gpu/drm/pl111/pl111_display.c
index 3a9661b9b1fc..5d10bc5fdf1f 100644
--- a/drivers/gpu/drm/pl111/pl111_display.c
+++ b/drivers/gpu/drm/pl111/pl111_display.c
@@ -138,7 +138,7 @@ static void pl111_display_enable(struct drm_simple_display_pipe *pipe,
 
 	ret = clk_set_rate(priv->clk, mode->clock * 1000);
 	if (ret) {
-		dev_err(drm->dev,
+		drm_err(drm,
 			"Failed to set pixel clock rate to %d: %d\n",
 			mode->clock * 1000, ret);
 	}
@@ -553,7 +553,7 @@ pl111_init_clock_divider(struct drm_device *drm)
 	int ret;
 
 	if (IS_ERR(parent)) {
-		dev_err(drm->dev, "CLCD: unable to get clcdclk.\n");
+		drm_err(drm, "CLCD: unable to get clcdclk.\n");
 		return PTR_ERR(parent);
 	}
 
diff --git a/drivers/gpu/drm/pl111/pl111_drv.c b/drivers/gpu/drm/pl111/pl111_drv.c
index 56ff6a3fb483..ac9e4b6bd2eb 100644
--- a/drivers/gpu/drm/pl111/pl111_drv.c
+++ b/drivers/gpu/drm/pl111/pl111_drv.c
@@ -55,6 +55,7 @@
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
+#include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
 
@@ -98,7 +99,7 @@ static int pl111_modeset_init(struct drm_device *dev)
 		struct drm_panel *tmp_panel;
 		struct drm_bridge *tmp_bridge;
 
-		dev_dbg(dev->dev, "checking endpoint %d\n", i);
+		drm_dbg(dev, "checking endpoint %d\n", i);
 
 		ret = drm_of_find_panel_or_bridge(dev->dev->of_node,
 						  0, i,
@@ -114,18 +115,18 @@ static int pl111_modeset_init(struct drm_device *dev)
 				defer = true;
 			} else if (ret != -ENODEV) {
 				/* Continue, maybe something else is working */
-				dev_err(dev->dev,
+				drm_err(dev,
 					"endpoint %d returns %d\n", i, ret);
 			}
 		}
 
 		if (tmp_panel) {
-			dev_info(dev->dev,
+			drm_info(dev,
 				 "found panel on endpoint %d\n", i);
 			panel = tmp_panel;
 		}
 		if (tmp_bridge) {
-			dev_info(dev->dev,
+			drm_info(dev,
 				 "found bridge on endpoint %d\n", i);
 			bridge = tmp_bridge;
 		}
@@ -149,9 +150,9 @@ static int pl111_modeset_init(struct drm_device *dev)
 			goto finish;
 		}
 	} else if (bridge) {
-		dev_info(dev->dev, "Using non-panel bridge\n");
+		drm_info(dev, "Using non-panel bridge\n");
 	} else {
-		dev_err(dev->dev, "No bridge, exiting\n");
+		drm_err(dev, "No bridge, exiting\n");
 		return -ENODEV;
 	}
 
@@ -163,7 +164,7 @@ static int pl111_modeset_init(struct drm_device *dev)
 
 	ret = pl111_display_init(dev);
 	if (ret != 0) {
-		dev_err(dev->dev, "Failed to init display\n");
+		drm_err(dev, "Failed to init display\n");
 		goto out_bridge;
 	}
 
@@ -175,7 +176,7 @@ static int pl111_modeset_init(struct drm_device *dev)
 	if (!priv->variant->broken_vblank) {
 		ret = drm_vblank_init(dev, 1);
 		if (ret != 0) {
-			dev_err(dev->dev, "Failed to init vblank\n");
+			drm_err(dev, "Failed to init vblank\n");
 			goto out_bridge;
 		}
 	}
@@ -255,13 +256,13 @@ static int pl111_amba_probe(struct amba_device *amba_dev,
 
 	ret = of_reserved_mem_device_init(dev);
 	if (!ret) {
-		dev_info(dev, "using device-specific reserved memory\n");
+		drm_info(drm, "using device-specific reserved memory\n");
 		priv->use_device_memory = true;
 	}
 
 	if (of_property_read_u32(dev->of_node, "max-memory-bandwidth",
 				 &priv->memory_bw)) {
-		dev_info(dev, "no max memory bandwidth specified, assume unlimited\n");
+		drm_info(drm, "no max memory bandwidth specified, assume unlimited\n");
 		priv->memory_bw = 0;
 	}
 
@@ -276,17 +277,17 @@ static int pl111_amba_probe(struct amba_device *amba_dev,
 
 	priv->regs = devm_ioremap_resource(dev, &amba_dev->res);
 	if (IS_ERR(priv->regs)) {
-		dev_err(dev, "%s failed mmio\n", __func__);
+		drm_err(drm, "%s failed mmio\n", __func__);
 		ret = PTR_ERR(priv->regs);
 		goto dev_put;
 	}
 
 	/* This may override some variant settings */
-	ret = pl111_versatile_init(dev, priv);
+	ret = pl111_versatile_init(drm, priv);
 	if (ret)
 		goto dev_put;
 
-	pl111_nomadik_init(dev);
+	pl111_nomadik_init(drm);
 
 	/* turn off interrupts before requesting the irq */
 	writel(0, priv->regs + priv->ienb);
@@ -294,7 +295,7 @@ static int pl111_amba_probe(struct amba_device *amba_dev,
 	ret = devm_request_irq(dev, amba_dev->irq[0], pl111_irq, 0,
 			       variant->name, priv);
 	if (ret != 0) {
-		dev_err(dev, "%s failed irq %d\n", __func__, ret);
+		drm_err(drm, "%s failed irq %d\n", __func__, ret);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/pl111/pl111_nomadik.c b/drivers/gpu/drm/pl111/pl111_nomadik.c
index 6f385e59be22..f3218d59c5f1 100644
--- a/drivers/gpu/drm/pl111/pl111_nomadik.c
+++ b/drivers/gpu/drm/pl111/pl111_nomadik.c
@@ -9,7 +9,7 @@
 #define PMU_CTRL_OFFSET 0x0000
 #define PMU_CTRL_LCDNDIF BIT(26)
 
-void pl111_nomadik_init(struct device *dev)
+void pl111_nomadik_init(struct drm_device *dev)
 {
 	struct regmap *pmu_regmap;
 
@@ -31,6 +31,6 @@ void pl111_nomadik_init(struct device *dev)
 			   PMU_CTRL_OFFSET,
 			   PMU_CTRL_LCDNDIF,
 			   0);
-	dev_info(dev, "set Nomadik PMU mux to CLCD mode\n");
+	drm_info(dev, "set Nomadik PMU mux to CLCD mode\n");
 }
 EXPORT_SYMBOL_GPL(pl111_nomadik_init);
diff --git a/drivers/gpu/drm/pl111/pl111_nomadik.h b/drivers/gpu/drm/pl111/pl111_nomadik.h
index 47ccf5c839fc..b2c9f7cc1c8c 100644
--- a/drivers/gpu/drm/pl111/pl111_nomadik.h
+++ b/drivers/gpu/drm/pl111/pl111_nomadik.h
@@ -8,11 +8,11 @@ struct device;
 
 #ifdef CONFIG_ARCH_NOMADIK
 
-void pl111_nomadik_init(struct device *dev);
+void pl111_nomadik_init(struct drm_device *dev);
 
 #else
 
-static inline void pl111_nomadik_init(struct device *dev)
+static inline void pl111_nomadik_init(struct drm_device *dev)
 {
 }
 
diff --git a/drivers/gpu/drm/pl111/pl111_versatile.c b/drivers/gpu/drm/pl111/pl111_versatile.c
index 5f460b296c0c..0d8331a3909f 100644
--- a/drivers/gpu/drm/pl111/pl111_versatile.c
+++ b/drivers/gpu/drm/pl111/pl111_versatile.c
@@ -20,6 +20,7 @@
 #include <linux/vexpress.h>
 
 #include <drm/drm_fourcc.h>
+#include <drm/drm_print.h>
 
 #include "pl111_versatile.h"
 #include "pl111_drm.h"
@@ -116,7 +117,7 @@ static void pl111_integrator_enable(struct drm_device *drm, u32 format)
 {
 	u32 val;
 
-	dev_info(drm->dev, "enable Integrator CLCD connectors\n");
+	drm_info(drm, "enable Integrator CLCD connectors\n");
 
 	/* FIXME: really needed? */
 	val = INTEGRATOR_CLCD_LCD_STATIC1 | INTEGRATOR_CLCD_LCD_STATIC2 |
@@ -134,7 +135,7 @@ static void pl111_integrator_enable(struct drm_device *drm, u32 format)
 		val |= INTEGRATOR_CLCD_LCDMUX_VGA555;
 		break;
 	default:
-		dev_err(drm->dev, "unhandled format on Integrator 0x%08x\n",
+		drm_err(drm, "unhandled format on Integrator 0x%08x\n",
 			format);
 		break;
 	}
@@ -156,7 +157,7 @@ static void pl111_impd1_enable(struct drm_device *drm, u32 format)
 {
 	u32 val;
 
-	dev_info(drm->dev, "enable IM-PD1 CLCD connectors\n");
+	drm_info(drm, "enable IM-PD1 CLCD connectors\n");
 	val = IMPD1_CTRL_DISP_VGA | IMPD1_CTRL_DISP_ENABLE;
 
 	regmap_update_bits(versatile_syscon_map,
@@ -167,7 +168,7 @@ static void pl111_impd1_enable(struct drm_device *drm, u32 format)
 
 static void pl111_impd1_disable(struct drm_device *drm)
 {
-	dev_info(drm->dev, "disable IM-PD1 CLCD connectors\n");
+	drm_info(drm, "disable IM-PD1 CLCD connectors\n");
 
 	regmap_update_bits(versatile_syscon_map,
 			   IMPD1_CTRL_OFFSET,
@@ -194,7 +195,7 @@ static void pl111_impd1_disable(struct drm_device *drm)
 
 static void pl111_versatile_disable(struct drm_device *drm)
 {
-	dev_info(drm->dev, "disable Versatile CLCD connectors\n");
+	drm_info(drm, "disable Versatile CLCD connectors\n");
 	regmap_update_bits(versatile_syscon_map,
 			   SYS_CLCD,
 			   SYS_CLCD_CONNECTOR_MASK,
@@ -205,7 +206,7 @@ static void pl111_versatile_enable(struct drm_device *drm, u32 format)
 {
 	u32 val = 0;
 
-	dev_info(drm->dev, "enable Versatile CLCD connectors\n");
+	drm_info(drm, "enable Versatile CLCD connectors\n");
 
 	switch (format) {
 	case DRM_FORMAT_ABGR8888:
@@ -227,7 +228,7 @@ static void pl111_versatile_enable(struct drm_device *drm, u32 format)
 		val |= SYS_CLCD_MODE_5551;
 		break;
 	default:
-		dev_err(drm->dev, "unhandled format on Versatile 0x%08x\n",
+		drm_err(drm, "unhandled format on Versatile 0x%08x\n",
 			format);
 		break;
 	}
@@ -247,7 +248,7 @@ static void pl111_versatile_enable(struct drm_device *drm, u32 format)
 
 static void pl111_realview_clcd_disable(struct drm_device *drm)
 {
-	dev_info(drm->dev, "disable RealView CLCD connectors\n");
+	drm_info(drm, "disable RealView CLCD connectors\n");
 	regmap_update_bits(versatile_syscon_map,
 			   SYS_CLCD,
 			   SYS_CLCD_CONNECTOR_MASK,
@@ -256,7 +257,7 @@ static void pl111_realview_clcd_disable(struct drm_device *drm)
 
 static void pl111_realview_clcd_enable(struct drm_device *drm, u32 format)
 {
-	dev_info(drm->dev, "enable RealView CLCD connectors\n");
+	drm_info(drm, "enable RealView CLCD connectors\n");
 	regmap_update_bits(versatile_syscon_map,
 			   SYS_CLCD,
 			   SYS_CLCD_CONNECTOR_MASK,
@@ -376,7 +377,7 @@ static const struct pl111_variant_data pl111_vexpress = {
 #define VEXPRESS_FPGAMUX_DAUGHTERBOARD_1	0x01
 #define VEXPRESS_FPGAMUX_DAUGHTERBOARD_2	0x02
 
-static int pl111_vexpress_clcd_init(struct device *dev, struct device_node *np,
+static int pl111_vexpress_clcd_init(struct drm_device *dev, struct device_node *np,
 				    struct pl111_drm_dev_private *priv)
 {
 	struct platform_device *pdev;
@@ -433,22 +434,22 @@ static int pl111_vexpress_clcd_init(struct device *dev, struct device_node *np,
 		mux_motherboard = false;
 
 	if (mux_motherboard) {
-		dev_info(dev, "DVI muxed to motherboard CLCD\n");
+		drm_info(dev, "DVI muxed to motherboard CLCD\n");
 		val = VEXPRESS_FPGAMUX_MOTHERBOARD;
-	} else if (ct_clcd == dev->of_node) {
-		dev_info(dev,
+	} else if (ct_clcd == dev->dev->of_node) {
+		drm_info(dev,
 			 "DVI muxed to daughterboard 1 (core tile) CLCD\n");
 		val = VEXPRESS_FPGAMUX_DAUGHTERBOARD_1;
 	} else {
-		dev_info(dev, "core tile graphics present\n");
-		dev_info(dev, "this device will be deactivated\n");
+		drm_info(dev, "core tile graphics present\n");
+		drm_info(dev, "this device will be deactivated\n");
 		return -ENODEV;
 	}
 
 	/* Call into deep Vexpress configuration API */
 	pdev = of_find_device_by_node(np);
 	if (!pdev) {
-		dev_err(dev, "can't find the sysreg device, deferring\n");
+		drm_err(dev, "can't find the sysreg device, deferring\n");
 		return -EPROBE_DEFER;
 	}
 
@@ -461,17 +462,17 @@ static int pl111_vexpress_clcd_init(struct device *dev, struct device_node *np,
 	ret = regmap_write(map, 0, val);
 	platform_device_put(pdev);
 	if (ret) {
-		dev_err(dev, "error setting DVI muxmode\n");
+		drm_err(dev, "error setting DVI muxmode\n");
 		return -ENODEV;
 	}
 
 	priv->variant = &pl111_vexpress;
-	dev_info(dev, "initializing Versatile Express PL111\n");
+	drm_info(dev, "initializing Versatile Express PL111\n");
 
 	return 0;
 }
 
-int pl111_versatile_init(struct device *dev, struct pl111_drm_dev_private *priv)
+int pl111_versatile_init(struct drm_device *dev, struct pl111_drm_dev_private *priv)
 {
 	const struct of_device_id *clcd_id;
 	enum versatile_clcd versatile_clcd_type;
@@ -492,7 +493,7 @@ int pl111_versatile_init(struct device *dev, struct pl111_drm_dev_private *priv)
 		int ret = pl111_vexpress_clcd_init(dev, np, priv);
 		of_node_put(np);
 		if (ret)
-			dev_err(dev, "Versatile Express init failed - %d", ret);
+			drm_err(dev, "Versatile Express init failed - %d", ret);
 		return ret;
 	}
 
@@ -511,7 +512,7 @@ int pl111_versatile_init(struct device *dev, struct pl111_drm_dev_private *priv)
 	map = syscon_node_to_regmap(np);
 	of_node_put(np);
 	if (IS_ERR(map)) {
-		dev_err(dev, "no Versatile syscon regmap\n");
+		drm_err(dev, "no Versatile syscon regmap\n");
 		return PTR_ERR(map);
 	}
 
@@ -520,14 +521,14 @@ int pl111_versatile_init(struct device *dev, struct pl111_drm_dev_private *priv)
 		versatile_syscon_map = map;
 		priv->variant = &pl110_integrator;
 		priv->variant_display_enable = pl111_integrator_enable;
-		dev_info(dev, "set up callbacks for Integrator PL110\n");
+		drm_info(dev, "set up callbacks for Integrator PL110\n");
 		break;
 	case INTEGRATOR_IMPD1:
 		versatile_syscon_map = map;
 		priv->variant = &pl110_impd1;
 		priv->variant_display_enable = pl111_impd1_enable;
 		priv->variant_display_disable = pl111_impd1_disable;
-		dev_info(dev, "set up callbacks for IM-PD1 PL110\n");
+		drm_info(dev, "set up callbacks for IM-PD1 PL110\n");
 		break;
 	case VERSATILE_CLCD:
 		versatile_syscon_map = map;
@@ -542,7 +543,7 @@ int pl111_versatile_init(struct device *dev, struct pl111_drm_dev_private *priv)
 		 */
 		priv->ienb = CLCD_PL111_IENB;
 		priv->ctrl = CLCD_PL111_CNTL;
-		dev_info(dev, "set up callbacks for Versatile PL110\n");
+		drm_info(dev, "set up callbacks for Versatile PL110\n");
 		break;
 	case REALVIEW_CLCD_EB:
 	case REALVIEW_CLCD_PB1176:
@@ -553,10 +554,10 @@ int pl111_versatile_init(struct device *dev, struct pl111_drm_dev_private *priv)
 		priv->variant = &pl111_realview;
 		priv->variant_display_enable = pl111_realview_clcd_enable;
 		priv->variant_display_disable = pl111_realview_clcd_disable;
-		dev_info(dev, "set up callbacks for RealView PL111\n");
+		drm_info(dev, "set up callbacks for RealView PL111\n");
 		break;
 	default:
-		dev_info(dev, "unknown Versatile system controller\n");
+		drm_info(dev, "unknown Versatile system controller\n");
 		break;
 	}
 
diff --git a/drivers/gpu/drm/pl111/pl111_versatile.h b/drivers/gpu/drm/pl111/pl111_versatile.h
index 143877010042..7a15c5f7efe8 100644
--- a/drivers/gpu/drm/pl111/pl111_versatile.h
+++ b/drivers/gpu/drm/pl111/pl111_versatile.h
@@ -7,6 +7,6 @@
 struct device;
 struct pl111_drm_dev_private;
 
-int pl111_versatile_init(struct device *dev, struct pl111_drm_dev_private *priv);
+int pl111_versatile_init(struct drm_device *dev, struct pl111_drm_dev_private *priv);
 
 #endif
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index 9b3a3a9d60e2..2fc0334e0d6c 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -1133,7 +1133,7 @@ static void atombios_crtc_set_pll(struct drm_crtc *crtc, struct drm_display_mode
 
 static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
 				 struct drm_framebuffer *fb,
-				 int x, int y, int atomic)
+				 int x, int y)
 {
 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
@@ -1150,33 +1150,23 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
 	bool bypass_lut = false;
 
 	/* no fb bound */
-	if (!atomic && !crtc->primary->fb) {
+	if (!crtc->primary->fb) {
 		DRM_DEBUG_KMS("No FB bound\n");
 		return 0;
 	}
 
-	if (atomic)
-		target_fb = fb;
-	else
-		target_fb = crtc->primary->fb;
+	target_fb = crtc->primary->fb;
 
-	/* If atomic, assume fb object is pinned & idle & fenced and
-	 * just update base pointers
-	 */
 	obj = target_fb->obj[0];
 	rbo = gem_to_radeon_bo(obj);
 	r = radeon_bo_reserve(rbo, false);
 	if (unlikely(r != 0))
 		return r;
 
-	if (atomic)
-		fb_location = radeon_bo_gpu_offset(rbo);
-	else {
-		r = radeon_bo_pin(rbo, RADEON_GEM_DOMAIN_VRAM, &fb_location);
-		if (unlikely(r != 0)) {
-			radeon_bo_unreserve(rbo);
-			return -EINVAL;
-		}
+	r = radeon_bo_pin(rbo, RADEON_GEM_DOMAIN_VRAM, &fb_location);
+	if (unlikely(r != 0)) {
+		radeon_bo_unreserve(rbo);
+		return -EINVAL;
 	}
 
 	radeon_bo_get_tiling_flags(rbo, &tiling_flags, NULL);
@@ -1437,7 +1427,7 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
 	/* set pageflip to happen anywhere in vblank interval */
 	WREG32(EVERGREEN_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 0);
 
-	if (!atomic && fb && fb != crtc->primary->fb) {
+	if (fb && fb != crtc->primary->fb) {
 		rbo = gem_to_radeon_bo(fb->obj[0]);
 		r = radeon_bo_reserve(rbo, false);
 		if (unlikely(r != 0))
@@ -1454,7 +1444,7 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
 
 static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
 				  struct drm_framebuffer *fb,
-				  int x, int y, int atomic)
+				  int x, int y)
 {
 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
@@ -1470,15 +1460,12 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
 	bool bypass_lut = false;
 
 	/* no fb bound */
-	if (!atomic && !crtc->primary->fb) {
+	if (!crtc->primary->fb) {
 		DRM_DEBUG_KMS("No FB bound\n");
 		return 0;
 	}
 
-	if (atomic)
-		target_fb = fb;
-	else
-		target_fb = crtc->primary->fb;
+	target_fb = crtc->primary->fb;
 
 	obj = target_fb->obj[0];
 	rbo = gem_to_radeon_bo(obj);
@@ -1486,17 +1473,10 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
 	if (unlikely(r != 0))
 		return r;
 
-	/* If atomic, assume fb object is pinned & idle & fenced and
-	 * just update base pointers
-	 */
-	if (atomic)
-		fb_location = radeon_bo_gpu_offset(rbo);
-	else {
-		r = radeon_bo_pin(rbo, RADEON_GEM_DOMAIN_VRAM, &fb_location);
-		if (unlikely(r != 0)) {
-			radeon_bo_unreserve(rbo);
-			return -EINVAL;
-		}
+	r = radeon_bo_pin(rbo, RADEON_GEM_DOMAIN_VRAM, &fb_location);
+	if (unlikely(r != 0)) {
+		radeon_bo_unreserve(rbo);
+		return -EINVAL;
 	}
 	radeon_bo_get_tiling_flags(rbo, &tiling_flags, NULL);
 	radeon_bo_unreserve(rbo);
@@ -1645,7 +1625,7 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
 	/* set pageflip to happen only at start of vblank interval (front porch) */
 	WREG32(AVIVO_D1MODE_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 3);
 
-	if (!atomic && fb && fb != crtc->primary->fb) {
+	if (fb && fb != crtc->primary->fb) {
 		rbo = gem_to_radeon_bo(fb->obj[0]);
 		r = radeon_bo_reserve(rbo, false);
 		if (unlikely(r != 0))
@@ -1667,26 +1647,11 @@ int atombios_crtc_set_base(struct drm_crtc *crtc, int x, int y,
 	struct radeon_device *rdev = dev->dev_private;
 
 	if (ASIC_IS_DCE4(rdev))
-		return dce4_crtc_do_set_base(crtc, old_fb, x, y, 0);
-	else if (ASIC_IS_AVIVO(rdev))
-		return avivo_crtc_do_set_base(crtc, old_fb, x, y, 0);
-	else
-		return radeon_crtc_do_set_base(crtc, old_fb, x, y, 0);
-}
-
-int atombios_crtc_set_base_atomic(struct drm_crtc *crtc,
-				  struct drm_framebuffer *fb,
-				  int x, int y, enum mode_set_atomic state)
-{
-	struct drm_device *dev = crtc->dev;
-	struct radeon_device *rdev = dev->dev_private;
-
-	if (ASIC_IS_DCE4(rdev))
-		return dce4_crtc_do_set_base(crtc, fb, x, y, 1);
+		return dce4_crtc_do_set_base(crtc, old_fb, x, y);
 	else if (ASIC_IS_AVIVO(rdev))
-		return avivo_crtc_do_set_base(crtc, fb, x, y, 1);
+		return avivo_crtc_do_set_base(crtc, old_fb, x, y);
 	else
-		return radeon_crtc_do_set_base(crtc, fb, x, y, 1);
+		return radeon_crtc_do_set_base(crtc, old_fb, x, y);
 }
 
 /* properly set additional regs when using atombios */
@@ -2215,7 +2180,6 @@ static const struct drm_crtc_helper_funcs atombios_helper_funcs = {
 	.mode_fixup = atombios_crtc_mode_fixup,
 	.mode_set = atombios_crtc_mode_set,
 	.mode_set_base = atombios_crtc_set_base,
-	.mode_set_base_atomic = atombios_crtc_set_base_atomic,
 	.prepare = atombios_crtc_prepare,
 	.commit = atombios_crtc_commit,
 	.disable = atombios_crtc_disable,
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c
index bc4ab71613a5..3cbc6eedbf66 100644
--- a/drivers/gpu/drm/radeon/evergreen.c
+++ b/drivers/gpu/drm/radeon/evergreen.c
@@ -1630,6 +1630,7 @@ void evergreen_pm_misc(struct radeon_device *rdev)
 	int req_cm_idx = rdev->pm.requested_clock_mode_index;
 	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
 	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
+	struct drm_device *ddev = rdev_to_drm(rdev);
 
 	if (voltage->type == VOLTAGE_SW) {
 		/* 0xff0x are flags rather then an actual voltage */
@@ -1638,7 +1639,7 @@ void evergreen_pm_misc(struct radeon_device *rdev)
 		if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
 			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
 			rdev->pm.current_vddc = voltage->voltage;
-			DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
+			drm_dbg(ddev, "Setting: vddc: %d\n", voltage->voltage);
 		}
 
 		/* starting with BTC, there is one state that is used for both
@@ -1659,7 +1660,7 @@ void evergreen_pm_misc(struct radeon_device *rdev)
 		if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
 			radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
 			rdev->pm.current_vddci = voltage->vddci;
-			DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
+			drm_dbg(ddev, "Setting: vddci: %d\n", voltage->vddci);
 		}
 	}
 }
@@ -2168,6 +2169,7 @@ static void evergreen_program_watermarks(struct radeon_device *rdev,
 	u32 pipe_offset = radeon_crtc->crtc_id * 16;
 	u32 tmp, arb_control3;
 	fixed20_12 a, b, c;
+	struct drm_device *ddev = rdev_to_drm(rdev);
 
 	if (radeon_crtc->base.enabled && num_heads && mode) {
 		active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
@@ -2244,14 +2246,14 @@ static void evergreen_program_watermarks(struct radeon_device *rdev,
 		    !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
 		    !evergreen_check_latency_hiding(&wm_high) ||
 		    (rdev->disp_priority == 2)) {
-			DRM_DEBUG_KMS("force priority a to high\n");
+			drm_dbg_kms(ddev, "force priority a to high\n");
 			priority_a_cnt |= PRIORITY_ALWAYS_ON;
 		}
 		if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
 		    !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
 		    !evergreen_check_latency_hiding(&wm_low) ||
 		    (rdev->disp_priority == 2)) {
-			DRM_DEBUG_KMS("force priority b to high\n");
+			drm_dbg_kms(ddev, "force priority b to high\n");
 			priority_b_cnt |= PRIORITY_ALWAYS_ON;
 		}
 
@@ -2401,6 +2403,7 @@ static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
 {
 	u32 tmp;
 	int r;
+	struct drm_device *ddev = rdev_to_drm(rdev);
 
 	if (rdev->gart.robj == NULL) {
 		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
@@ -2448,7 +2451,7 @@ static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
 	WREG32(VM_CONTEXT1_CNTL, 0);
 
 	evergreen_pcie_gart_tlb_flush(rdev);
-	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
+	drm_info(ddev, "PCIE GART of %uM enabled (table at 0x%016llX).\n",
 		 (unsigned)(rdev->mc.gtt_size >> 20),
 		 (unsigned long long)rdev->gart.table_addr);
 	rdev->gart.ready = true;
@@ -2626,16 +2629,17 @@ static void evergreen_blank_dp_output(struct radeon_device *rdev,
 	unsigned stream_ctrl;
 	unsigned fifo_ctrl;
 	unsigned counter = 0;
+	struct drm_device *ddev = rdev_to_drm(rdev);
 
 	if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
-		DRM_ERROR("invalid dig_fe %d\n", dig_fe);
+		drm_err(ddev, "invalid dig_fe %d\n", dig_fe);
 		return;
 	}
 
 	stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
 			     evergreen_dp_offsets[dig_fe]);
 	if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
-		DRM_ERROR("dig %d , should be enable\n", dig_fe);
+		drm_err(ddev, "dig %d , should be enable\n", dig_fe);
 		return;
 	}
 
@@ -2652,7 +2656,7 @@ static void evergreen_blank_dp_output(struct radeon_device *rdev,
 				     evergreen_dp_offsets[dig_fe]);
 	}
 	if (counter >= 32)
-		DRM_ERROR("counter exceeds %d\n", counter);
+		drm_err(ddev, "counter exceeds %d\n", counter);
 
 	fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
 	fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
@@ -2998,10 +3002,11 @@ static int evergreen_cp_start(struct radeon_device *rdev)
 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
 	int r, i;
 	uint32_t cp_me;
+	struct drm_device *ddev = rdev_to_drm(rdev);
 
 	r = radeon_ring_lock(rdev, ring, 7);
 	if (r) {
-		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
+		drm_err(ddev, "radeon: cp failed to lock ring (%d).\n", r);
 		return r;
 	}
 	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
@@ -3018,7 +3023,7 @@ static int evergreen_cp_start(struct radeon_device *rdev)
 
 	r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
 	if (r) {
-		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
+		drm_err(ddev, "radeon: cp failed to lock ring (%d).\n", r);
 		return r;
 	}
 
@@ -3826,6 +3831,7 @@ u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
 {
 	u32 reset_mask = 0;
 	u32 tmp;
+	struct drm_device *ddev = rdev_to_drm(rdev);
 
 	/* GRBM_STATUS */
 	tmp = RREG32(GRBM_STATUS);
@@ -3884,7 +3890,7 @@ u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
 
 	/* Skip MC reset as it's mostly likely not hung, just busy */
 	if (reset_mask & RADEON_RESET_MC) {
-		DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
+		drm_dbg(ddev, "MC busy: 0x%08X, clearing.\n", reset_mask);
 		reset_mask &= ~RADEON_RESET_MC;
 	}
 
@@ -4495,6 +4501,7 @@ int evergreen_irq_set(struct radeon_device *rdev)
 	u32 grbm_int_cntl = 0;
 	u32 dma_cntl, dma_cntl1 = 0;
 	u32 thermal_int = 0;
+	struct drm_device *ddev = rdev_to_drm(rdev);
 
 	if (!rdev->irq.installed) {
 		WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
@@ -4520,40 +4527,40 @@ int evergreen_irq_set(struct radeon_device *rdev)
 	if (rdev->family >= CHIP_CAYMAN) {
 		/* enable CP interrupts on all rings */
 		if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
-			DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
+			drm_dbg(ddev, "%s : sw int gfx\n", __func__);
 			cp_int_cntl |= TIME_STAMP_INT_ENABLE;
 		}
 		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
-			DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
+			drm_dbg(ddev, "%s : sw int cp1\n", __func__);
 			cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
 		}
 		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
-			DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
+			drm_dbg(ddev, "%s : sw int cp2\n", __func__);
 			cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
 		}
 	} else {
 		if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
-			DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
+			drm_dbg(ddev, "%s : sw int gfx\n", __func__);
 			cp_int_cntl |= RB_INT_ENABLE;
 			cp_int_cntl |= TIME_STAMP_INT_ENABLE;
 		}
 	}
 
 	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
-		DRM_DEBUG("r600_irq_set: sw int dma\n");
+		drm_dbg(ddev, "r600_irq_set: sw int dma\n");
 		dma_cntl |= TRAP_ENABLE;
 	}
 
 	if (rdev->family >= CHIP_CAYMAN) {
 		dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
 		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
-			DRM_DEBUG("r600_irq_set: sw int dma1\n");
+			drm_dbg(ddev, "r600_irq_set: sw int dma1\n");
 			dma_cntl1 |= TRAP_ENABLE;
 		}
 	}
 
 	if (rdev->irq.dpm_thermal) {
-		DRM_DEBUG("dpm thermal\n");
+		drm_dbg(ddev, "dpm thermal\n");
 		thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
 	}
 
@@ -4713,6 +4720,7 @@ int evergreen_irq_process(struct radeon_device *rdev)
 	bool queue_thermal = false;
 	u32 status, addr;
 	const char *event_name;
+	struct drm_device *ddev = rdev_to_drm(rdev);
 
 	if (!rdev->ih.enabled || rdev->shutdown)
 		return IRQ_NONE;
@@ -4725,7 +4733,7 @@ int evergreen_irq_process(struct radeon_device *rdev)
 		return IRQ_NONE;
 
 	rptr = rdev->ih.rptr;
-	DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
+	drm_dbg(ddev, "%s start: rptr %d, wptr %d\n", __func__, rptr, wptr);
 
 	/* Order reading of wptr vs. reading of IH ring data */
 	rmb();
@@ -4766,18 +4774,18 @@ int evergreen_irq_process(struct radeon_device *rdev)
 				mask = LB_D1_VLINE_INTERRUPT;
 				event_name = "vline";
 			} else {
-				DRM_DEBUG("Unhandled interrupt: %d %d\n",
+				drm_dbg(ddev, "Unhandled interrupt: %d %d\n",
 					  src_id, src_data);
 				break;
 			}
 
 			if (!(disp_int[crtc_idx] & mask)) {
-				DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
+				drm_dbg(ddev, "IH: D%d %s - IH event w/o asserted irq bit?\n",
 					  crtc_idx + 1, event_name);
 			}
 
 			disp_int[crtc_idx] &= ~mask;
-			DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
+			drm_dbg(ddev, "IH: D%d %s\n", crtc_idx + 1, event_name);
 
 			break;
 		case 8: /* D1 page flip */
@@ -4786,7 +4794,7 @@ int evergreen_irq_process(struct radeon_device *rdev)
 		case 14: /* D4 page flip */
 		case 16: /* D5 page flip */
 		case 18: /* D6 page flip */
-			DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
+			drm_dbg(ddev, "IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
 			if (radeon_use_pflipirq > 0)
 				radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
 			break;
@@ -4804,39 +4812,39 @@ int evergreen_irq_process(struct radeon_device *rdev)
 				event_name = "HPD_RX";
 
 			} else {
-				DRM_DEBUG("Unhandled interrupt: %d %d\n",
+				drm_dbg(ddev, "Unhandled interrupt: %d %d\n",
 					  src_id, src_data);
 				break;
 			}
 
 			if (!(disp_int[hpd_idx] & mask))
-				DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
+				drm_dbg(ddev, "IH: IH event w/o asserted irq bit?\n");
 
 			disp_int[hpd_idx] &= ~mask;
-			DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
+			drm_dbg(ddev, "IH: %s%d\n", event_name, hpd_idx + 1);
 
 			break;
 		case 44: /* hdmi */
 			afmt_idx = src_data;
 			if (afmt_idx > 5) {
-				DRM_ERROR("Unhandled interrupt: %d %d\n",
+				drm_err(ddev, "Unhandled interrupt: %d %d\n",
 					  src_id, src_data);
 				break;
 			}
 
 			if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
-				DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
+				drm_dbg(ddev, "IH: IH event w/o asserted irq bit?\n");
 
 			afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
 			queue_hdmi = true;
-			DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
+			drm_dbg(ddev, "IH: HDMI%d\n", afmt_idx + 1);
 			break;
 		case 96:
-			DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
+			drm_err(ddev, "SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
 			WREG32(SRBM_INT_ACK, 0x1);
 			break;
 		case 124: /* UVD */
-			DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
+			drm_dbg(ddev, "IH: UVD int: 0x%08x\n", src_data);
 			radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
 			break;
 		case 146:
@@ -4857,11 +4865,11 @@ int evergreen_irq_process(struct radeon_device *rdev)
 		case 176: /* CP_INT in ring buffer */
 		case 177: /* CP_INT in IB1 */
 		case 178: /* CP_INT in IB2 */
-			DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
+			drm_dbg(ddev, "IH: CP int: 0x%08x\n", src_data);
 			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
 			break;
 		case 181: /* CP EOP event */
-			DRM_DEBUG("IH: CP EOP\n");
+			drm_dbg(ddev, "IH: CP EOP\n");
 			if (rdev->family >= CHIP_CAYMAN) {
 				switch (src_data) {
 				case 0:
@@ -4878,30 +4886,30 @@ int evergreen_irq_process(struct radeon_device *rdev)
 				radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
 			break;
 		case 224: /* DMA trap event */
-			DRM_DEBUG("IH: DMA trap\n");
+			drm_dbg(ddev, "IH: DMA trap\n");
 			radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
 			break;
 		case 230: /* thermal low to high */
-			DRM_DEBUG("IH: thermal low to high\n");
+			drm_dbg(ddev, "IH: thermal low to high\n");
 			rdev->pm.dpm.thermal.high_to_low = false;
 			queue_thermal = true;
 			break;
 		case 231: /* thermal high to low */
-			DRM_DEBUG("IH: thermal high to low\n");
+			drm_dbg(ddev, "IH: thermal high to low\n");
 			rdev->pm.dpm.thermal.high_to_low = true;
 			queue_thermal = true;
 			break;
 		case 233: /* GUI IDLE */
-			DRM_DEBUG("IH: GUI idle\n");
+			drm_dbg(ddev, "IH: GUI idle\n");
 			break;
 		case 244: /* DMA trap event */
 			if (rdev->family >= CHIP_CAYMAN) {
-				DRM_DEBUG("IH: DMA1 trap\n");
+				drm_dbg(ddev, "IH: DMA1 trap\n");
 				radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
 			}
 			break;
 		default:
-			DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
+			drm_dbg(ddev, "Unhandled interrupt: %d %d\n", src_id, src_data);
 			break;
 		}
 
@@ -5000,6 +5008,7 @@ static int evergreen_startup(struct radeon_device *rdev)
 {
 	struct radeon_ring *ring;
 	int r;
+	struct drm_device *ddev = rdev_to_drm(rdev);
 
 	/* enable pcie gen2 link */
 	evergreen_pcie_gen2_enable(rdev);
@@ -5016,7 +5025,7 @@ static int evergreen_startup(struct radeon_device *rdev)
 	if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
 		r = ni_mc_load_microcode(rdev);
 		if (r) {
-			DRM_ERROR("Failed to load MC firmware!\n");
+			drm_err(ddev, "Failed to load MC firmware!\n");
 			return r;
 		}
 	}
@@ -5038,7 +5047,7 @@ static int evergreen_startup(struct radeon_device *rdev)
 		rdev->rlc.cs_data = evergreen_cs_data;
 		r = sumo_rlc_init(rdev);
 		if (r) {
-			DRM_ERROR("Failed to init rlc BOs!\n");
+			drm_err(ddev, "Failed to init rlc BOs!\n");
 			return r;
 		}
 	}
@@ -5071,7 +5080,7 @@ static int evergreen_startup(struct radeon_device *rdev)
 
 	r = r600_irq_init(rdev);
 	if (r) {
-		DRM_ERROR("radeon: IH init failed (%d).\n", r);
+		drm_err(ddev, "radeon: IH init failed (%d).\n", r);
 		radeon_irq_kms_fini(rdev);
 		return r;
 	}
@@ -5109,7 +5118,7 @@ static int evergreen_startup(struct radeon_device *rdev)
 
 	r = radeon_audio_init(rdev);
 	if (r) {
-		DRM_ERROR("radeon: audio init failed\n");
+		drm_err(ddev, "radeon: audio init failed\n");
 		return r;
 	}
 
@@ -5119,6 +5128,7 @@ static int evergreen_startup(struct radeon_device *rdev)
 int evergreen_resume(struct radeon_device *rdev)
 {
 	int r;
+	struct drm_device *ddev = rdev_to_drm(rdev);
 
 	/* reset the asic, the gfx blocks are often in a bad state
 	 * after the driver is unloaded or after a resume
@@ -5141,7 +5151,7 @@ int evergreen_resume(struct radeon_device *rdev)
 	rdev->accel_working = true;
 	r = evergreen_startup(rdev);
 	if (r) {
-		DRM_ERROR("evergreen startup failed on resume\n");
+		drm_err(ddev, "evergreen startup failed on resume\n");
 		rdev->accel_working = false;
 		return r;
 	}
@@ -5176,6 +5186,7 @@ int evergreen_suspend(struct radeon_device *rdev)
 int evergreen_init(struct radeon_device *rdev)
 {
 	int r;
+	struct drm_device *ddev = rdev_to_drm(rdev);
 
 	/* Read BIOS */
 	if (!radeon_get_bios(rdev)) {
@@ -5201,7 +5212,7 @@ int evergreen_init(struct radeon_device *rdev)
 			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
 			return -EINVAL;
 		}
-		DRM_INFO("GPU not posted. posting now...\n");
+		drm_info(ddev, "GPU not posted. posting now...\n");
 		atom_asic_init(rdev->mode_info.atom_context);
 	}
 	/* init golden registers */
@@ -5233,7 +5244,7 @@ int evergreen_init(struct radeon_device *rdev)
 		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
 			r = ni_init_microcode(rdev);
 			if (r) {
-				DRM_ERROR("Failed to load firmware!\n");
+				drm_err(ddev, "Failed to load firmware!\n");
 				return r;
 			}
 		}
@@ -5241,7 +5252,7 @@ int evergreen_init(struct radeon_device *rdev)
 		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
 			r = r600_init_microcode(rdev);
 			if (r) {
-				DRM_ERROR("Failed to load firmware!\n");
+				drm_err(ddev, "Failed to load firmware!\n");
 				return r;
 			}
 		}
@@ -5287,7 +5298,7 @@ int evergreen_init(struct radeon_device *rdev)
 	 */
 	if (ASIC_IS_DCE5(rdev)) {
 		if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
-			DRM_ERROR("radeon: MC ucode required for NI+.\n");
+			drm_err(ddev, "radeon: MC ucode required for NI+.\n");
 			return -EINVAL;
 		}
 	}
@@ -5323,6 +5334,7 @@ void evergreen_fini(struct radeon_device *rdev)
 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
 {
 	u32 link_width_cntl, speed_cntl;
+	struct drm_device *ddev = rdev_to_drm(rdev);
 
 	if (radeon_pcie_gen2 == 0)
 		return;
@@ -5343,11 +5355,11 @@ void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
 
 	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
 	if (speed_cntl & LC_CURRENT_DATA_RATE) {
-		DRM_INFO("PCIE gen 2 link speeds already enabled\n");
+		drm_info(ddev, "PCIE gen 2 link speeds already enabled\n");
 		return;
 	}
 
-	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
+	drm_info(ddev, "enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
 
 	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
 	    (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index 35fb99bcd9a7..bc28117e01b4 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -41,6 +41,7 @@
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
 #include <drm/radeon_drm.h>
+#include <drm/drm_print.h>
 
 #include "atom.h"
 #include "radeon.h"
@@ -273,7 +274,7 @@ static void radeon_unpin_work_func(struct work_struct *__work)
 		radeon_bo_unpin(work->old_rbo);
 		radeon_bo_unreserve(work->old_rbo);
 	} else
-		DRM_ERROR("failed to reserve buffer after flip\n");
+		drm_err(&work->rdev->ddev, "failed to reserve buffer after flip\n");
 
 	drm_gem_object_put(&work->old_rbo->tbo.base);
 	kfree(work);
@@ -434,7 +435,7 @@ static void radeon_flip_work_func(struct work_struct *__work)
 			r = dma_fence_wait(work->fence, false);
 
 		if (r)
-			DRM_ERROR("failed to wait on page flip fence (%d)!\n", r);
+			drm_err(dev, "failed to wait on page flip fence (%d)!\n", r);
 
 		/* We continue with the page flip even if we failed to wait on
 		 * the fence, otherwise the DRM core and userspace will be
@@ -521,7 +522,7 @@ static int radeon_crtc_page_flip_target(struct drm_crtc *crtc,
 
 	r = radeon_bo_reserve(new_rbo, false);
 	if (unlikely(r != 0)) {
-		DRM_ERROR("failed to reserve new rbo buffer before flip\n");
+		drm_err(dev, "failed to reserve new rbo buffer before flip\n");
 		goto cleanup;
 	}
 	/* Only 27 bit offset for legacy CRTC */
@@ -530,14 +531,14 @@ static int radeon_crtc_page_flip_target(struct drm_crtc *crtc,
 	if (unlikely(r != 0)) {
 		radeon_bo_unreserve(new_rbo);
 		r = -EINVAL;
-		DRM_ERROR("failed to pin new rbo buffer before flip\n");
+		drm_err(dev, "failed to pin new rbo buffer before flip\n");
 		goto cleanup;
 	}
 	r = dma_resv_get_singleton(new_rbo->tbo.base.resv, DMA_RESV_USAGE_WRITE,
 				   &work->fence);
 	if (r) {
 		radeon_bo_unreserve(new_rbo);
-		DRM_ERROR("failed to get new rbo buffer fences\n");
+		drm_err(dev, "failed to get new rbo buffer fences\n");
 		goto cleanup;
 	}
 	radeon_bo_get_tiling_flags(new_rbo, &tiling_flags, NULL);
@@ -604,7 +605,7 @@ static int radeon_crtc_page_flip_target(struct drm_crtc *crtc,
 
 pflip_cleanup:
 	if (unlikely(radeon_bo_reserve(new_rbo, false) != 0)) {
-		DRM_ERROR("failed to reserve new rbo in error path\n");
+		drm_err(dev, "failed to reserve new rbo in error path\n");
 		goto cleanup;
 	}
 	radeon_bo_unpin(new_rbo);
@@ -772,15 +773,15 @@ static void radeon_print_display_setup(struct drm_device *dev)
 	uint32_t devices;
 	int i = 0;
 
-	DRM_INFO("Radeon Display Connectors\n");
+	drm_info(dev, "Radeon Display Connectors\n");
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		radeon_connector = to_radeon_connector(connector);
-		DRM_INFO("Connector %d:\n", i);
-		DRM_INFO("  %s\n", connector->name);
+		drm_info(dev, "Connector %d:\n", i);
+		drm_info(dev, "  %s\n", connector->name);
 		if (radeon_connector->hpd.hpd != RADEON_HPD_NONE)
-			DRM_INFO("  %s\n", hpd_names[radeon_connector->hpd.hpd]);
+			drm_info(dev, "  %s\n", hpd_names[radeon_connector->hpd.hpd]);
 		if (radeon_connector->ddc_bus) {
-			DRM_INFO("  DDC: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
+			drm_info(dev, "  DDC: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
 				 radeon_connector->ddc_bus->rec.mask_clk_reg,
 				 radeon_connector->ddc_bus->rec.mask_data_reg,
 				 radeon_connector->ddc_bus->rec.a_clk_reg,
@@ -790,11 +791,11 @@ static void radeon_print_display_setup(struct drm_device *dev)
 				 radeon_connector->ddc_bus->rec.y_clk_reg,
 				 radeon_connector->ddc_bus->rec.y_data_reg);
 			if (radeon_connector->router.ddc_valid)
-				DRM_INFO("  DDC Router 0x%x/0x%x\n",
+				drm_info(dev, "  DDC Router 0x%x/0x%x\n",
 					 radeon_connector->router.ddc_mux_control_pin,
 					 radeon_connector->router.ddc_mux_state);
 			if (radeon_connector->router.cd_valid)
-				DRM_INFO("  Clock/Data Router 0x%x/0x%x\n",
+				drm_info(dev, "  Clock/Data Router 0x%x/0x%x\n",
 					 radeon_connector->router.cd_mux_control_pin,
 					 radeon_connector->router.cd_mux_state);
 		} else {
@@ -804,35 +805,46 @@ static void radeon_print_display_setup(struct drm_device *dev)
 			    connector->connector_type == DRM_MODE_CONNECTOR_DVIA ||
 			    connector->connector_type == DRM_MODE_CONNECTOR_HDMIA ||
 			    connector->connector_type == DRM_MODE_CONNECTOR_HDMIB)
-				DRM_INFO("  DDC: no ddc bus - possible BIOS bug - please report to xorg-driver-ati@lists.x.org\n");
+				drm_info(dev, "  DDC: no ddc bus - possible BIOS bug - please report to xorg-driver-ati@lists.x.org\n");
 		}
-		DRM_INFO("  Encoders:\n");
+		drm_info(dev, "  Encoders:\n");
 		list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
 			radeon_encoder = to_radeon_encoder(encoder);
 			devices = radeon_encoder->devices & radeon_connector->devices;
 			if (devices) {
 				if (devices & ATOM_DEVICE_CRT1_SUPPORT)
-					DRM_INFO("    CRT1: %s\n", encoder_names[radeon_encoder->encoder_id]);
+					drm_info(dev, "    CRT1: %s\n",
+						encoder_names[radeon_encoder->encoder_id]);
 				if (devices & ATOM_DEVICE_CRT2_SUPPORT)
-					DRM_INFO("    CRT2: %s\n", encoder_names[radeon_encoder->encoder_id]);
+					drm_info(dev, "    CRT2: %s\n",
+						encoder_names[radeon_encoder->encoder_id]);
 				if (devices & ATOM_DEVICE_LCD1_SUPPORT)
-					DRM_INFO("    LCD1: %s\n", encoder_names[radeon_encoder->encoder_id]);
+					drm_info(dev, "    LCD1: %s\n",
+						encoder_names[radeon_encoder->encoder_id]);
 				if (devices & ATOM_DEVICE_DFP1_SUPPORT)
-					DRM_INFO("    DFP1: %s\n", encoder_names[radeon_encoder->encoder_id]);
+					drm_info(dev, "    DFP1: %s\n",
+						encoder_names[radeon_encoder->encoder_id]);
 				if (devices & ATOM_DEVICE_DFP2_SUPPORT)
-					DRM_INFO("    DFP2: %s\n", encoder_names[radeon_encoder->encoder_id]);
+					drm_info(dev, "    DFP2: %s\n",
+						encoder_names[radeon_encoder->encoder_id]);
 				if (devices & ATOM_DEVICE_DFP3_SUPPORT)
-					DRM_INFO("    DFP3: %s\n", encoder_names[radeon_encoder->encoder_id]);
+					drm_info(dev, "    DFP3: %s\n",
+						encoder_names[radeon_encoder->encoder_id]);
 				if (devices & ATOM_DEVICE_DFP4_SUPPORT)
-					DRM_INFO("    DFP4: %s\n", encoder_names[radeon_encoder->encoder_id]);
+					drm_info(dev, "    DFP4: %s\n",
+						encoder_names[radeon_encoder->encoder_id]);
 				if (devices & ATOM_DEVICE_DFP5_SUPPORT)
-					DRM_INFO("    DFP5: %s\n", encoder_names[radeon_encoder->encoder_id]);
+					drm_info(dev, "    DFP5: %s\n",
+						encoder_names[radeon_encoder->encoder_id]);
 				if (devices & ATOM_DEVICE_DFP6_SUPPORT)
-					DRM_INFO("    DFP6: %s\n", encoder_names[radeon_encoder->encoder_id]);
+					drm_info(dev, "    DFP6: %s\n",
+						encoder_names[radeon_encoder->encoder_id]);
 				if (devices & ATOM_DEVICE_TV1_SUPPORT)
-					DRM_INFO("    TV1: %s\n", encoder_names[radeon_encoder->encoder_id]);
+					drm_info(dev, "    TV1: %s\n",
+						encoder_names[radeon_encoder->encoder_id]);
 				if (devices & ATOM_DEVICE_CV_SUPPORT)
-					DRM_INFO("    CV: %s\n", encoder_names[radeon_encoder->encoder_id]);
+					drm_info(dev, "    CV: %s\n",
+						encoder_names[radeon_encoder->encoder_id]);
 			}
 		}
 		i++;
@@ -1747,7 +1759,7 @@ bool radeon_crtc_scaling_mode_fixup(struct drm_crtc *crtc,
 				 * (ie all encoder can work with the same
 				 *  scaling).
 				 */
-				DRM_ERROR("Scaling not consistent across encoder.\n");
+				drm_err(dev, "Scaling not consistent across encoder.\n");
 				return false;
 			}
 		}
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
index 825b351ff53c..a1054c8094d4 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
@@ -360,19 +360,12 @@ static void radeon_crtc_dpms(struct drm_crtc *crtc, int mode)
 int radeon_crtc_set_base(struct drm_crtc *crtc, int x, int y,
 			 struct drm_framebuffer *old_fb)
 {
-	return radeon_crtc_do_set_base(crtc, old_fb, x, y, 0);
-}
-
-int radeon_crtc_set_base_atomic(struct drm_crtc *crtc,
-				struct drm_framebuffer *fb,
-				int x, int y, enum mode_set_atomic state)
-{
-	return radeon_crtc_do_set_base(crtc, fb, x, y, 1);
+	return radeon_crtc_do_set_base(crtc, old_fb, x, y);
 }
 
 int radeon_crtc_do_set_base(struct drm_crtc *crtc,
 			 struct drm_framebuffer *fb,
-			 int x, int y, int atomic)
+			 int x, int y)
 {
 	struct drm_device *dev = crtc->dev;
 	struct radeon_device *rdev = dev->dev_private;
@@ -390,15 +383,12 @@ int radeon_crtc_do_set_base(struct drm_crtc *crtc,
 
 	DRM_DEBUG_KMS("\n");
 	/* no fb bound */
-	if (!atomic && !crtc->primary->fb) {
+	if (!crtc->primary->fb) {
 		DRM_DEBUG_KMS("No FB bound\n");
 		return 0;
 	}
 
-	if (atomic)
-		target_fb = fb;
-	else
-		target_fb = crtc->primary->fb;
+	target_fb = crtc->primary->fb;
 
 	switch (target_fb->format->cpp[0] * 8) {
 	case 8:
@@ -445,7 +435,7 @@ int radeon_crtc_do_set_base(struct drm_crtc *crtc,
 		 * We don't shutdown the display controller because new buffer
 		 * will end up in same spot.
 		 */
-		if (!atomic && fb && fb != crtc->primary->fb) {
+		if (fb && fb != crtc->primary->fb) {
 			struct radeon_bo *old_rbo;
 			unsigned long nsize, osize;
 
@@ -555,7 +545,7 @@ int radeon_crtc_do_set_base(struct drm_crtc *crtc,
 	WREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset, crtc_offset);
 	WREG32(RADEON_CRTC_PITCH + radeon_crtc->crtc_offset, crtc_pitch);
 
-	if (!atomic && fb && fb != crtc->primary->fb) {
+	if (fb && fb != crtc->primary->fb) {
 		rbo = gem_to_radeon_bo(fb->obj[0]);
 		r = radeon_bo_reserve(rbo, false);
 		if (unlikely(r != 0))
@@ -1108,7 +1098,6 @@ static const struct drm_crtc_helper_funcs legacy_helper_funcs = {
 	.mode_fixup = radeon_crtc_mode_fixup,
 	.mode_set = radeon_crtc_mode_set,
 	.mode_set_base = radeon_crtc_set_base,
-	.mode_set_base_atomic = radeon_crtc_set_base_atomic,
 	.prepare = radeon_crtc_prepare,
 	.commit = radeon_crtc_commit,
 	.disable = radeon_crtc_disable,
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index 9e34da2cacef..088af85902f7 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -804,10 +804,6 @@ extern bool radeon_encoder_is_digital(struct drm_encoder *encoder);
 extern void radeon_crtc_load_lut(struct drm_crtc *crtc);
 extern int atombios_crtc_set_base(struct drm_crtc *crtc, int x, int y,
 				   struct drm_framebuffer *old_fb);
-extern int atombios_crtc_set_base_atomic(struct drm_crtc *crtc,
-					 struct drm_framebuffer *fb,
-					 int x, int y,
-					 enum mode_set_atomic state);
 extern int atombios_crtc_mode_set(struct drm_crtc *crtc,
 				   struct drm_display_mode *mode,
 				   struct drm_display_mode *adjusted_mode,
@@ -817,13 +813,9 @@ extern void atombios_crtc_dpms(struct drm_crtc *crtc, int mode);
 
 extern int radeon_crtc_set_base(struct drm_crtc *crtc, int x, int y,
 				 struct drm_framebuffer *old_fb);
-extern int radeon_crtc_set_base_atomic(struct drm_crtc *crtc,
-				       struct drm_framebuffer *fb,
-				       int x, int y,
-				       enum mode_set_atomic state);
 extern int radeon_crtc_do_set_base(struct drm_crtc *crtc,
 				   struct drm_framebuffer *fb,
-				   int x, int y, int atomic);
+				   int x, int y);
 extern int radeon_crtc_cursor_set2(struct drm_crtc *crtc,
 				   struct drm_file *file_priv,
 				   uint32_t handle,
diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c
index 3b52dfc0ea1e..5edd45424562 100644
--- a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c
+++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c
@@ -7,6 +7,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/clk.h>
+#include <linux/clk/renesas.h>
 #include <linux/delay.h>
 #include <linux/dma-mapping.h>
 #include <linux/io.h>
@@ -32,6 +33,8 @@
 
 #include "rzg2l_mipi_dsi_regs.h"
 
+MODULE_IMPORT_NS("RZV2H_CPG");
+
 #define RZG2L_DCS_BUF_SIZE	128 /* Maximum DCS buffer size in external memory. */
 
 #define RZ_MIPI_DSI_FEATURE_16BPP	BIT(0)
@@ -46,6 +49,11 @@ struct rzg2l_mipi_dsi_hw_info {
 			      u64 *hsfreq_millihz);
 	unsigned int (*dphy_mode_clk_check)(struct rzg2l_mipi_dsi *dsi,
 					    unsigned long mode_freq);
+	struct {
+		const struct rzv2h_pll_limits **limits;
+		const u8 *table;
+		const u8 table_size;
+	} cpg_plldsi;
 	u32 phy_reg_offset;
 	u32 link_reg_offset;
 	unsigned long min_dclk;
@@ -53,6 +61,11 @@ struct rzg2l_mipi_dsi_hw_info {
 	u8 features;
 };
 
+struct rzv2h_dsi_mode_calc {
+	unsigned long mode_freq_khz;
+	struct rzv2h_pll_pars dsi_parameters;
+};
+
 struct rzg2l_mipi_dsi {
 	struct device *dev;
 	void __iomem *mmio;
@@ -68,17 +81,29 @@ struct rzg2l_mipi_dsi {
 	struct drm_bridge *next_bridge;
 
 	struct clk *vclk;
+	struct clk *lpclk;
 
 	enum mipi_dsi_pixel_format format;
 	unsigned int num_data_lanes;
 	unsigned int lanes;
 	unsigned long mode_flags;
 
+	struct rzv2h_dsi_mode_calc mode_calc;
+
 	/* DCS buffer pointers when using external memory. */
 	dma_addr_t dcs_buf_phys;
 	u8 *dcs_buf_virt;
 };
 
+static const struct rzv2h_pll_limits rzv2h_plldsi_div_limits = {
+	.fout = { .min = 80 * MEGA, .max = 1500 * MEGA },
+	.fvco = { .min = 1050 * MEGA, .max = 2100 * MEGA },
+	.m = { .min = 64, .max = 1023 },
+	.p = { .min = 1, .max = 4 },
+	.s = { .min = 0, .max = 5 },
+	.k = { .min = -32768, .max = 32767 },
+};
+
 static inline struct rzg2l_mipi_dsi *
 bridge_to_rzg2l_mipi_dsi(struct drm_bridge *bridge)
 {
@@ -193,6 +218,237 @@ static const struct rzg2l_mipi_dsi_timings rzg2l_mipi_dsi_global_timings[] = {
 	},
 };
 
+/**
+ * struct rzv2h_mipi_dsi_timings - Timing parameter table structure
+ *
+ * @hsfreq: Pointer to frequency threshold array
+ * @len: Number of entries in the hsfreq array
+ * @base_value: Base register value offset for this timing parameter
+ *
+ * Each timing parameter (TCLK*, THS*, etc.) has its own table with
+ * frequency thresholds and corresponding base register values.
+ */
+struct rzv2h_mipi_dsi_timings {
+	const u8 *hsfreq;
+	u8 len;
+	u8 base_value;
+};
+
+/*
+ * enum rzv2h_dsi_timing_idx - MIPI DSI timing parameter indices
+ *
+ * These enums correspond to different MIPI DSI PHY timing parameters.
+ */
+enum rzv2h_dsi_timing_idx {
+	TCLKPRPRCTL,
+	TCLKZEROCTL,
+	TCLKPOSTCTL,
+	TCLKTRAILCTL,
+	THSPRPRCTL,
+	THSZEROCTL,
+	THSTRAILCTL,
+	TLPXCTL,
+	THSEXITCTL,
+};
+
+/*
+ * RZ/V2H(P) Frequency threshold lookup tables for D-PHY timing parameters
+ *
+ * - Each array contains frequency thresholds (in units of 10 Mbps),
+ *   taken directly from the table 9.5-4 hardware manual.
+ * - These thresholds define the frequency ranges for which timing
+ *   register values must be programmed.
+ * - The actual register value is calculated in
+ *   rzv2h_dphy_find_timings_val():
+ *
+ *       register_value = timings->base_value + table_index
+ *
+ * Example (TCLKPRPRCTL, from HW manual):
+ *   0-150 Mbps   -> index 0 -> register_value = base + 0 = 0 + 0 = 0
+ *   151-260 Mbps -> index 1 -> register_value = base + 1 = 0 + 1 = 1
+ *   261-370 Mbps -> index 2 -> register_value = base + 2 = 0 + 2 = 2
+ *
+ * Each of the following arrays corresponds to a specific timing
+ * parameter (TCLKPRPRCTL, TCLKZEROCTL, TCLKPOSTCTL, etc.).
+ */
+static const u8 tclkprprctl[] = {
+	15, 26, 37, 47, 58, 69, 79, 90, 101, 111, 122, 133, 143, 150,
+};
+
+static const u8 tclkzeroctl[] = {
+	9, 11, 13, 15, 18, 21, 23, 24, 25, 27, 29, 31, 34, 36, 38,
+	41, 43, 45, 47, 50, 52, 54, 57, 59, 61, 63, 66, 68, 70, 73,
+	75, 77, 79, 82, 84, 86, 89, 91, 93, 95, 98, 100, 102, 105,
+	107, 109, 111, 114, 116, 118, 121, 123, 125, 127, 130, 132,
+	134, 137, 139, 141, 143, 146, 148, 150,
+};
+
+static const u8 tclkpostctl[] = {
+	8, 21, 34, 48, 61, 74, 88, 101, 114, 128, 141, 150,
+};
+
+static const u8 tclktrailctl[] = {
+	14, 25, 37, 48, 59, 71, 82, 94, 105, 117, 128, 139, 150,
+};
+
+static const u8 thsprprctl[] = {
+	11, 19, 29, 40, 50, 61, 72, 82, 93, 103, 114, 125, 135, 146, 150,
+};
+
+static const u8 thszeroctl[] = {
+	18, 24, 29, 35, 40, 46, 51, 57, 62, 68, 73, 79, 84, 90,
+	95, 101, 106, 112, 117, 123, 128, 134, 139, 145, 150,
+};
+
+static const u8 thstrailctl[] = {
+	10, 21, 32, 42, 53, 64, 75, 85, 96, 107, 118, 128, 139, 150,
+};
+
+static const u8 tlpxctl[] = {
+	13, 26, 39, 53, 66, 79, 93, 106, 119, 133, 146,	150,
+};
+
+static const u8 thsexitctl[] = {
+	15, 23, 31, 39, 47, 55, 63, 71, 79, 87,
+	95, 103, 111, 119, 127, 135, 143, 150,
+};
+
+/*
+ * rzv2h_dsi_timings_tables - main timing parameter lookup table
+ * Maps timing parameter enum to its frequency table, array length and
+ * base register offset value.
+ */
+static const struct rzv2h_mipi_dsi_timings rzv2h_dsi_timings_tables[] = {
+	[TCLKPRPRCTL] = {
+		.hsfreq = tclkprprctl,
+		.len = ARRAY_SIZE(tclkprprctl),
+		.base_value = 0,
+	},
+	[TCLKZEROCTL] = {
+		.hsfreq = tclkzeroctl,
+		.len = ARRAY_SIZE(tclkzeroctl),
+		.base_value = 2,
+	},
+	[TCLKPOSTCTL] = {
+		.hsfreq = tclkpostctl,
+		.len = ARRAY_SIZE(tclkpostctl),
+		.base_value = 6,
+	},
+	[TCLKTRAILCTL] = {
+		.hsfreq = tclktrailctl,
+		.len = ARRAY_SIZE(tclktrailctl),
+		.base_value = 1,
+	},
+	[THSPRPRCTL] = {
+		.hsfreq = thsprprctl,
+		.len = ARRAY_SIZE(thsprprctl),
+		.base_value = 0,
+	},
+	[THSZEROCTL] = {
+		.hsfreq = thszeroctl,
+		.len = ARRAY_SIZE(thszeroctl),
+		.base_value = 0,
+	},
+	[THSTRAILCTL] = {
+		.hsfreq = thstrailctl,
+		.len = ARRAY_SIZE(thstrailctl),
+		.base_value = 3,
+	},
+	[TLPXCTL] = {
+		.hsfreq = tlpxctl,
+		.len = ARRAY_SIZE(tlpxctl),
+		.base_value = 0,
+	},
+	[THSEXITCTL] = {
+		.hsfreq = thsexitctl,
+		.len = ARRAY_SIZE(thsexitctl),
+		.base_value = 1,
+	},
+};
+
+/**
+ * rzv2h_dphy_find_ulpsexit - Find ULP Exit timing value based on frequency
+ * The function maps frequency ranges to ULP exit timing values.
+ * Thresholds in the local hsfreq[] are expressed in Hz already.
+ *
+ * @freq: Input frequency in Hz
+ *
+ * Return: ULP exit timing value
+ */
+static u16 rzv2h_dphy_find_ulpsexit(unsigned long freq)
+{
+	/* Frequency thresholds in Hz for ULP exit timing selection */
+	static const unsigned long hsfreq[] = {
+		1953125UL,
+		3906250UL,
+		7812500UL,
+		15625000UL,
+	};
+	/* Corresponding ULP exit timing values for each frequency range */
+	static const u16 ulpsexit[] = {49, 98, 195, 391};
+	unsigned int i;
+
+	/* Find the appropriate frequency range */
+	for (i = 0; i < ARRAY_SIZE(hsfreq); i++) {
+		if (freq <= hsfreq[i])
+			break;
+	}
+
+	 /* If frequency exceeds all thresholds, use the highest range */
+	if (i == ARRAY_SIZE(hsfreq))
+		i--;
+
+	return ulpsexit[i];
+}
+
+/**
+ * rzv2h_dphy_find_timings_val - Find timing parameter value from lookup tables
+ * @freq: Input frequency in Hz
+ * @index: Index to select timing parameter table (see enum rzv2h_dsi_timing_idx)
+ *
+ * Selects the timing table for the requested parameter, finds the
+ * frequency range entry and returns the register value to program:
+ *
+ *   register_value = timings->base_value + table_index
+ *
+ * Note: frequency table entries are stored as small integers (units of 10):
+ *       threshold_in_hz = (unsigned long)table_entry * 10 * MEGA
+ *
+ * Return: timing register value to be programmed into hardware
+ */
+static u16 rzv2h_dphy_find_timings_val(unsigned long freq, u8 index)
+{
+	const struct rzv2h_mipi_dsi_timings *timings;
+	u16 i;
+
+	/* Get the timing table structure for the requested parameter */
+	timings = &rzv2h_dsi_timings_tables[index];
+
+	/*
+	 * Search through frequency table to find appropriate range
+	 * timings->hsfreq[i] contains frequency values from HW manual
+	 * Convert to Hz by multiplying by 10 * MEGA.
+	 */
+	for (i = 0; i < timings->len; i++) {
+		unsigned long hsfreq = timings->hsfreq[i] * 10 * MEGA;
+
+		if (freq <= hsfreq)
+			break;
+	}
+
+	/* If frequency exceeds table range, use the last entry */
+	if (i == timings->len)
+		i--;
+
+	/*
+	 * Calculate final register value:
+	 * - timings->base_value: base value for this timing parameter
+	 * - i: index into frequency table (0-based)
+	 * Combined they give the exact register value to program
+	 */
+	return timings->base_value + i;
+};
+
 static void rzg2l_mipi_dsi_phy_write(struct rzg2l_mipi_dsi *dsi, u32 reg, u32 data)
 {
 	iowrite32(data, dsi->mmio + dsi->info->phy_reg_offset + reg);
@@ -317,6 +573,169 @@ static int rzg2l_dphy_conf_clks(struct rzg2l_mipi_dsi *dsi, unsigned long mode_f
 	return 0;
 }
 
+static unsigned int rzv2h_dphy_mode_clk_check(struct rzg2l_mipi_dsi *dsi,
+					      unsigned long mode_freq)
+{
+	u64 hsfreq_millihz, mode_freq_hz, mode_freq_millihz;
+	struct rzv2h_pll_div_pars cpg_dsi_parameters;
+	struct rzv2h_pll_pars dsi_parameters;
+	bool parameters_found;
+	unsigned int bpp;
+
+	bpp = mipi_dsi_pixel_format_to_bpp(dsi->format);
+	mode_freq_hz = mul_u32_u32(mode_freq, KILO);
+	mode_freq_millihz = mode_freq_hz * MILLI;
+	parameters_found =
+		rzv2h_get_pll_divs_pars(dsi->info->cpg_plldsi.limits[0],
+					&cpg_dsi_parameters,
+					dsi->info->cpg_plldsi.table,
+					dsi->info->cpg_plldsi.table_size,
+					mode_freq_millihz);
+	if (!parameters_found)
+		return MODE_CLOCK_RANGE;
+
+	hsfreq_millihz = DIV_ROUND_CLOSEST_ULL(cpg_dsi_parameters.div.freq_millihz * bpp,
+					       dsi->lanes);
+	parameters_found = rzv2h_get_pll_pars(&rzv2h_plldsi_div_limits,
+					      &dsi_parameters, hsfreq_millihz);
+	if (!parameters_found)
+		return MODE_CLOCK_RANGE;
+
+	if (abs(dsi_parameters.error_millihz) >= 500)
+		return MODE_CLOCK_RANGE;
+
+	memcpy(&dsi->mode_calc.dsi_parameters, &dsi_parameters, sizeof(dsi_parameters));
+	dsi->mode_calc.mode_freq_khz = mode_freq;
+
+	return MODE_OK;
+}
+
+static int rzv2h_dphy_conf_clks(struct rzg2l_mipi_dsi *dsi, unsigned long mode_freq,
+				u64 *hsfreq_millihz)
+{
+	struct rzv2h_pll_pars *dsi_parameters = &dsi->mode_calc.dsi_parameters;
+	unsigned long status;
+
+	if (dsi->mode_calc.mode_freq_khz != mode_freq) {
+		status = rzv2h_dphy_mode_clk_check(dsi, mode_freq);
+		if (status != MODE_OK) {
+			dev_err(dsi->dev, "No PLL parameters found for mode clk %lu\n",
+				mode_freq);
+			return -EINVAL;
+		}
+	}
+
+	*hsfreq_millihz = dsi_parameters->freq_millihz;
+
+	return 0;
+}
+
+static int rzv2h_mipi_dsi_dphy_init(struct rzg2l_mipi_dsi *dsi,
+				    u64 hsfreq_millihz)
+{
+	struct rzv2h_pll_pars *dsi_parameters = &dsi->mode_calc.dsi_parameters;
+	unsigned long lpclk_rate = clk_get_rate(dsi->lpclk);
+	u32 phytclksetr, phythssetr, phytlpxsetr, phycr;
+	struct rzg2l_mipi_dsi_timings dphy_timings;
+	u16 ulpsexit;
+	u64 hsfreq;
+
+	hsfreq = DIV_ROUND_CLOSEST_ULL(hsfreq_millihz, MILLI);
+
+	if (dsi_parameters->freq_millihz != hsfreq_millihz &&
+	    !rzv2h_get_pll_pars(&rzv2h_plldsi_div_limits, dsi_parameters,
+				hsfreq_millihz)) {
+		dev_err(dsi->dev, "No PLL parameters found for HSFREQ %lluHz\n", hsfreq);
+		return -EINVAL;
+	}
+
+	dphy_timings.tclk_trail =
+		rzv2h_dphy_find_timings_val(hsfreq, TCLKTRAILCTL);
+	dphy_timings.tclk_post =
+		rzv2h_dphy_find_timings_val(hsfreq, TCLKPOSTCTL);
+	dphy_timings.tclk_zero =
+		rzv2h_dphy_find_timings_val(hsfreq, TCLKZEROCTL);
+	dphy_timings.tclk_prepare =
+		rzv2h_dphy_find_timings_val(hsfreq, TCLKPRPRCTL);
+	dphy_timings.ths_exit =
+		rzv2h_dphy_find_timings_val(hsfreq, THSEXITCTL);
+	dphy_timings.ths_trail =
+		rzv2h_dphy_find_timings_val(hsfreq, THSTRAILCTL);
+	dphy_timings.ths_zero =
+		rzv2h_dphy_find_timings_val(hsfreq, THSZEROCTL);
+	dphy_timings.ths_prepare =
+		rzv2h_dphy_find_timings_val(hsfreq, THSPRPRCTL);
+	dphy_timings.tlpx =
+		rzv2h_dphy_find_timings_val(hsfreq, TLPXCTL);
+	ulpsexit = rzv2h_dphy_find_ulpsexit(lpclk_rate);
+
+	phytclksetr = FIELD_PREP(PHYTCLKSETR_TCLKTRAILCTL, dphy_timings.tclk_trail) |
+		      FIELD_PREP(PHYTCLKSETR_TCLKPOSTCTL, dphy_timings.tclk_post) |
+		      FIELD_PREP(PHYTCLKSETR_TCLKZEROCTL, dphy_timings.tclk_zero) |
+		      FIELD_PREP(PHYTCLKSETR_TCLKPRPRCTL, dphy_timings.tclk_prepare);
+	phythssetr = FIELD_PREP(PHYTHSSETR_THSEXITCTL, dphy_timings.ths_exit) |
+		     FIELD_PREP(PHYTHSSETR_THSTRAILCTL, dphy_timings.ths_trail) |
+		     FIELD_PREP(PHYTHSSETR_THSZEROCTL, dphy_timings.ths_zero) |
+		     FIELD_PREP(PHYTHSSETR_THSPRPRCTL, dphy_timings.ths_prepare);
+	phytlpxsetr = rzg2l_mipi_dsi_phy_read(dsi, PHYTLPXSETR) & ~PHYTLPXSETR_TLPXCTL;
+	phytlpxsetr |= FIELD_PREP(PHYTLPXSETR_TLPXCTL, dphy_timings.tlpx);
+	phycr = rzg2l_mipi_dsi_phy_read(dsi, PHYCR) & ~GENMASK(9, 0);
+	phycr |= FIELD_PREP(PHYCR_ULPSEXIT, ulpsexit);
+
+	/* Setting all D-PHY Timings Registers */
+	rzg2l_mipi_dsi_phy_write(dsi, PHYTCLKSETR, phytclksetr);
+	rzg2l_mipi_dsi_phy_write(dsi, PHYTHSSETR, phythssetr);
+	rzg2l_mipi_dsi_phy_write(dsi, PHYTLPXSETR, phytlpxsetr);
+	rzg2l_mipi_dsi_phy_write(dsi, PHYCR, phycr);
+
+	rzg2l_mipi_dsi_phy_write(dsi, PLLCLKSET0R,
+				 FIELD_PREP(PLLCLKSET0R_PLL_S, dsi_parameters->s) |
+				 FIELD_PREP(PLLCLKSET0R_PLL_P, dsi_parameters->p) |
+				 FIELD_PREP(PLLCLKSET0R_PLL_M, dsi_parameters->m));
+	rzg2l_mipi_dsi_phy_write(dsi, PLLCLKSET1R,
+				 FIELD_PREP(PLLCLKSET1R_PLL_K, dsi_parameters->k));
+
+	/*
+	 * From RZ/V2H HW manual (Rev.1.20) section 9.5.3 Operation,
+	 * (C) After write to D-PHY registers we need to wait for more than 1 x tp
+	 *
+	 * tp = 1 / (PLLREFCLK / PLLCLKSET0R.PLL_P)
+	 * PLLREFCLK = 24MHz
+	 * PLLCLKSET0R.PLL_P = {1, 2, 3, 4}
+	 *
+	 * To handle all the cases lets use PLLCLKSET0R.PLL_P = 4
+	 * tp = 1 / (24MHz / 4) = 1 / 6MHz = 166.67ns
+	 */
+	ndelay(200);
+
+	rzg2l_mipi_dsi_phy_write(dsi, PLLENR, PLLENR_PLLEN);
+	/*
+	 * From RZ/V2H HW manual (Rev.1.20) section 9.5.3 Operation,
+	 * (D) After write to PLLENR.PLLEN we need to wait for more than 3000 x tp
+	 *
+	 * 3000 x tp = 3000 x 0.16667 ns = 500.01 microseconds
+	 */
+	usleep_range(510, 520);
+
+	return 0;
+}
+
+static void rzv2h_mipi_dsi_dphy_startup_late_init(struct rzg2l_mipi_dsi *dsi)
+{
+	/*
+	 * From RZ/V2H HW manual (Rev.1.20) section 9.5.3 Operation,
+	 * (E) After write to TXSETR we need to wait for more than 200 microseconds
+	 * and then write to PHYRSTR
+	 */
+	usleep_range(210, 220);
+	rzg2l_mipi_dsi_phy_write(dsi, PHYRSTR, PHYRSTR_PHYMRSTN);
+}
+
+static void rzv2h_mipi_dsi_dphy_exit(struct rzg2l_mipi_dsi *dsi)
+{
+	rzg2l_mipi_dsi_phy_write(dsi, PLLENR, 0);
+}
+
 static int rzg2l_mipi_dsi_startup(struct rzg2l_mipi_dsi *dsi,
 				  const struct drm_display_mode *mode)
 {
@@ -429,6 +848,9 @@ static void rzg2l_mipi_dsi_set_display_timing(struct rzg2l_mipi_dsi *dsi,
 	case 18:
 		vich1ppsetr = VICH1PPSETR_DT_RGB18;
 		break;
+	case 16:
+		vich1ppsetr = VICH1PPSETR_DT_RGB16;
+		break;
 	}
 
 	if ((dsi->mode_flags & MIPI_DSI_MODE_VIDEO_SYNC_PULSE) &&
@@ -979,6 +1401,10 @@ static int rzg2l_mipi_dsi_probe(struct platform_device *pdev)
 	if (IS_ERR(dsi->vclk))
 		return PTR_ERR(dsi->vclk);
 
+	dsi->lpclk = devm_clk_get(dsi->dev, "lpclk");
+	if (IS_ERR(dsi->lpclk))
+		return PTR_ERR(dsi->lpclk);
+
 	dsi->rstc = devm_reset_control_get_optional_exclusive(dsi->dev, "rst");
 	if (IS_ERR(dsi->rstc))
 		return dev_err_probe(dsi->dev, PTR_ERR(dsi->rstc),
@@ -1051,6 +1477,32 @@ static void rzg2l_mipi_dsi_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 }
 
+RZV2H_CPG_PLL_DSI_LIMITS(rzv2h_cpg_pll_dsi_limits);
+
+static const struct rzv2h_pll_limits *rzv2h_plldsi_limits[] = {
+	&rzv2h_cpg_pll_dsi_limits,
+};
+
+static const u8 rzv2h_cpg_div_table[] = {
+	2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32,
+};
+
+static const struct rzg2l_mipi_dsi_hw_info rzv2h_mipi_dsi_info = {
+	.dphy_init = rzv2h_mipi_dsi_dphy_init,
+	.dphy_startup_late_init = rzv2h_mipi_dsi_dphy_startup_late_init,
+	.dphy_exit = rzv2h_mipi_dsi_dphy_exit,
+	.dphy_mode_clk_check = rzv2h_dphy_mode_clk_check,
+	.dphy_conf_clks = rzv2h_dphy_conf_clks,
+	.cpg_plldsi.limits = rzv2h_plldsi_limits,
+	.cpg_plldsi.table = rzv2h_cpg_div_table,
+	.cpg_plldsi.table_size = ARRAY_SIZE(rzv2h_cpg_div_table),
+	.phy_reg_offset = 0x10000,
+	.link_reg_offset = 0,
+	.min_dclk = 5440,
+	.max_dclk = 187500,
+	.features = RZ_MIPI_DSI_FEATURE_16BPP,
+};
+
 static const struct rzg2l_mipi_dsi_hw_info rzg2l_mipi_dsi_info = {
 	.dphy_init = rzg2l_mipi_dsi_dphy_init,
 	.dphy_exit = rzg2l_mipi_dsi_dphy_exit,
@@ -1061,6 +1513,7 @@ static const struct rzg2l_mipi_dsi_hw_info rzg2l_mipi_dsi_info = {
 };
 
 static const struct of_device_id rzg2l_mipi_dsi_of_table[] = {
+	{ .compatible = "renesas,r9a09g057-mipi-dsi", .data = &rzv2h_mipi_dsi_info, },
 	{ .compatible = "renesas,rzg2l-mipi-dsi", .data = &rzg2l_mipi_dsi_info, },
 	{ /* sentinel */ }
 };
diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi_regs.h b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi_regs.h
index d8082a87d874..2bef20566648 100644
--- a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi_regs.h
+++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi_regs.h
@@ -40,6 +40,39 @@
 #define DSIDPHYTIM3_THS_TRAIL(x)	((x) << 8)
 #define DSIDPHYTIM3_THS_ZERO(x)		((x) << 0)
 
+/* RZ/V2H DPHY Registers */
+#define PLLENR				0x000
+#define PLLENR_PLLEN			BIT(0)
+
+#define PHYRSTR				0x004
+#define PHYRSTR_PHYMRSTN		BIT(0)
+
+#define PLLCLKSET0R			0x010
+#define PLLCLKSET0R_PLL_S		GENMASK(2, 0)
+#define PLLCLKSET0R_PLL_P		GENMASK(13, 8)
+#define PLLCLKSET0R_PLL_M		GENMASK(25, 16)
+
+#define PLLCLKSET1R			0x014
+#define PLLCLKSET1R_PLL_K		GENMASK(15, 0)
+
+#define PHYTCLKSETR			0x020
+#define PHYTCLKSETR_TCLKTRAILCTL        GENMASK(7, 0)
+#define PHYTCLKSETR_TCLKPOSTCTL         GENMASK(15, 8)
+#define PHYTCLKSETR_TCLKZEROCTL         GENMASK(23, 16)
+#define PHYTCLKSETR_TCLKPRPRCTL         GENMASK(31, 24)
+
+#define PHYTHSSETR			0x024
+#define PHYTHSSETR_THSEXITCTL           GENMASK(7, 0)
+#define PHYTHSSETR_THSTRAILCTL          GENMASK(15, 8)
+#define PHYTHSSETR_THSZEROCTL           GENMASK(23, 16)
+#define PHYTHSSETR_THSPRPRCTL           GENMASK(31, 24)
+
+#define PHYTLPXSETR			0x028
+#define PHYTLPXSETR_TLPXCTL             GENMASK(7, 0)
+
+#define PHYCR				0x030
+#define PHYCR_ULPSEXIT                  GENMASK(9, 0)
+
 /* --------------------------------------------------------*/
 
 /* Link Status Register */
@@ -130,6 +163,7 @@
 
 /* Video-Input Channel 1 Pixel Packet Set Register */
 #define VICH1PPSETR			0x420
+#define VICH1PPSETR_DT_RGB16		(0x0e << 16)
 #define VICH1PPSETR_DT_RGB18		(0x1e << 16)
 #define VICH1PPSETR_DT_RGB18_LS		(0x2e << 16)
 #define VICH1PPSETR_DT_RGB24		(0x3e << 16)
diff --git a/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c b/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
index 727cdf768161..0dc1eb5d2ae3 100644
--- a/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
@@ -467,6 +467,19 @@ static const struct dw_hdmi_plat_data rk3328_hdmi_drv_data = {
 	.use_drm_infoframe = true,
 };
 
+static struct rockchip_hdmi_chip_data rk3368_chip_data = {
+	.lcdsel_grf_reg = -1,
+};
+
+static const struct dw_hdmi_plat_data rk3368_hdmi_drv_data = {
+	.mode_valid = dw_hdmi_rockchip_mode_valid,
+	.mpll_cfg   = rockchip_mpll_cfg,
+	.cur_ctr    = rockchip_cur_ctr,
+	.phy_config = rockchip_phy_config,
+	.phy_data = &rk3368_chip_data,
+	.use_drm_infoframe = true,
+};
+
 static struct rockchip_hdmi_chip_data rk3399_chip_data = {
 	.lcdsel_grf_reg = RK3399_GRF_SOC_CON20,
 	.lcdsel_big = FIELD_PREP_WM16_CONST(RK3399_HDMI_LCDC_SEL, 0),
@@ -507,6 +520,9 @@ static const struct of_device_id dw_hdmi_rockchip_dt_ids[] = {
 	{ .compatible = "rockchip,rk3328-dw-hdmi",
 	  .data = &rk3328_hdmi_drv_data
 	},
+	{ .compatible = "rockchip,rk3368-dw-hdmi",
+	 .data = &rk3368_hdmi_drv_data
+	},
 	{ .compatible = "rockchip,rk3399-dw-hdmi",
 	  .data = &rk3399_hdmi_drv_data
 	},
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
index 1d4f1b822e7b..bd7936c03da2 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -344,7 +344,7 @@ drm_sched_rq_select_entity_fifo(struct drm_gpu_scheduler *sched,
  */
 static void drm_sched_run_job_queue(struct drm_gpu_scheduler *sched)
 {
-	if (!READ_ONCE(sched->pause_submit))
+	if (!drm_sched_is_stopped(sched))
 		queue_work(sched->submit_wq, &sched->work_run_job);
 }
 
@@ -354,7 +354,7 @@ static void drm_sched_run_job_queue(struct drm_gpu_scheduler *sched)
  */
 static void drm_sched_run_free_queue(struct drm_gpu_scheduler *sched)
 {
-	if (!READ_ONCE(sched->pause_submit))
+	if (!drm_sched_is_stopped(sched))
 		queue_work(sched->submit_wq, &sched->work_free_job);
 }
 
@@ -729,7 +729,9 @@ EXPORT_SYMBOL(drm_sched_start);
  *
  * Drivers can still save and restore their state for recovery operations, but
  * we shouldn't make this a general scheduler feature around the dma_fence
- * interface.
+ * interface. The suggested driver-side replacement is to use
+ * drm_sched_for_each_pending_job() after stopping the scheduler and implement
+ * their own recovery operations.
  */
 void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
 {
@@ -1567,3 +1569,35 @@ void drm_sched_wqueue_start(struct drm_gpu_scheduler *sched)
 	queue_work(sched->submit_wq, &sched->work_free_job);
 }
 EXPORT_SYMBOL(drm_sched_wqueue_start);
+
+/**
+ * drm_sched_is_stopped() - Checks whether drm_sched is stopped
+ * @sched: DRM scheduler
+ *
+ * Return: true if sched is stopped, false otherwise
+ */
+bool drm_sched_is_stopped(struct drm_gpu_scheduler *sched)
+{
+	return READ_ONCE(sched->pause_submit);
+}
+EXPORT_SYMBOL(drm_sched_is_stopped);
+
+/**
+ * drm_sched_job_is_signaled() - DRM scheduler job is signaled
+ * @job: DRM scheduler job
+ *
+ * Determine if DRM scheduler job is signaled. DRM scheduler should be stopped
+ * to obtain a stable snapshot of state. Both parent fence (hardware fence) and
+ * finished fence (software fence) are checked to determine signaling state.
+ *
+ * Return: true if job is signaled, false otherwise
+ */
+bool drm_sched_job_is_signaled(struct drm_sched_job *job)
+{
+	struct drm_sched_fence *s_fence = job->s_fence;
+
+	WARN_ON(!drm_sched_is_stopped(job->sched));
+	return (s_fence->parent && dma_fence_is_signaled(s_fence->parent)) ||
+		dma_fence_is_signaled(&s_fence->finished);
+}
+EXPORT_SYMBOL(drm_sched_job_is_signaled);
diff --git a/drivers/gpu/drm/sitronix/Kconfig b/drivers/gpu/drm/sitronix/Kconfig
index 6de7d92d9b74..41a428ef8295 100644
--- a/drivers/gpu/drm/sitronix/Kconfig
+++ b/drivers/gpu/drm/sitronix/Kconfig
@@ -1,16 +1,44 @@
-config DRM_ST7571_I2C
-	tristate "DRM support for Sitronix ST7571 display panels (I2C)"
-	depends on DRM && I2C && MMU
+config DRM_ST7571
+	tristate "DRM support for Sitronix ST7567/ST7571 display panels"
+	depends on DRM && MMU
 	select DRM_CLIENT_SELECTION
 	select DRM_GEM_SHMEM_HELPER
 	select DRM_KMS_HELPER
-	select REGMAP_I2C
 	select VIDEOMODE_HELPERS
 	help
-	  DRM driver for Sitronix ST7571 panels controlled over I2C.
+	  Sitronix ST7571 is a driver and controller for 4-level gray
+	  scale and monochrome dot matrix LCD panels.
+
+	  DRM driver for Sitronix ST7567/ST7571 panels.
+	  This is only the core driver, a driver for the appropriate bus
+	  transport in your chip also must be selected.
+
+	  if M is selected the module will be called st7571.
+
+config DRM_ST7571_I2C
+	tristate "DRM support for Sitronix ST7567/ST7571 display panels (I2C)"
+	depends on DRM_ST7571 && I2C
+	select REGMAP
+	help
+	  Sitronix ST7571 is a driver and controller for 4-level gray
+	  scale and monochrome dot matrix LCD panels.
+
+	  DRM driver for Sitronix ST7565/ST7571 panels connected via I2C bus.
 
 	  if M is selected the module will be called st7571-i2c.
 
+config DRM_ST7571_SPI
+	tristate "DRM support for Sitronix ST7567/ST7571 display panels (SPI)"
+	depends on DRM_ST7571 && SPI
+	select REGMAP_SPI
+	help
+	  Sitronix ST7571 is a driver and controller for 4-level gray
+	  scale and monochrome dot matrix LCD panels.
+
+	  DRM driver for Sitronix ST7565/ST7571 panels connected via SPI bus.
+
+	  if M is selected the module will be called st7571-spi.
+
 config DRM_ST7586
 	tristate "DRM support for Sitronix ST7586 display panels"
 	depends on DRM && SPI
@@ -40,3 +68,13 @@ config DRM_ST7735R
 
 	  If M is selected the module will be called st7735r.
 
+config DRM_ST7920
+	tristate "DRM support for Sitronix ST7920 LCD displays"
+	depends on DRM && SPI && MMU
+	select DRM_GEM_SHMEM_HELPER
+	select DRM_KMS_HELPER
+	select REGMAP_SPI
+	help
+	  DRM driver for the ST7920 Sitronix LCD controllers.
+
+	  If M is selected the module will be called st7920.
diff --git a/drivers/gpu/drm/sitronix/Makefile b/drivers/gpu/drm/sitronix/Makefile
index bd139e5a6995..d03beff37628 100644
--- a/drivers/gpu/drm/sitronix/Makefile
+++ b/drivers/gpu/drm/sitronix/Makefile
@@ -1,3 +1,6 @@
+obj-$(CONFIG_DRM_ST7571)		+= st7571.o
 obj-$(CONFIG_DRM_ST7571_I2C)		+= st7571-i2c.o
+obj-$(CONFIG_DRM_ST7571_SPI)		+= st7571-spi.o
 obj-$(CONFIG_DRM_ST7586)		+= st7586.o
 obj-$(CONFIG_DRM_ST7735R)		+= st7735r.o
+obj-$(CONFIG_DRM_ST7920)		+= st7920.o
diff --git a/drivers/gpu/drm/sitronix/st7571-i2c.c b/drivers/gpu/drm/sitronix/st7571-i2c.c
index 4e73c8b415d6..44bc94be33d6 100644
--- a/drivers/gpu/drm/sitronix/st7571-i2c.c
+++ b/drivers/gpu/drm/sitronix/st7571-i2c.c
@@ -1,131 +1,18 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
- * Driver for Sitronix ST7571, a 4 level gray scale dot matrix LCD controller
+ * Driver for Sitronix ST7571 connected via I2C bus.
  *
  * Copyright (C) 2025 Marcus Folkesson <marcus.folkesson@gmail.com>
  */
 
-#include <linux/bitfield.h>
-#include <linux/delay.h>
-#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/module.h>
 #include <linux/regmap.h>
 
-#include <drm/clients/drm_client_setup.h>
-#include <drm/drm_atomic.h>
-#include <drm/drm_atomic_helper.h>
-#include <drm/drm_connector.h>
-#include <drm/drm_crtc_helper.h>
-#include <drm/drm_damage_helper.h>
-#include <drm/drm_drv.h>
-#include <drm/drm_encoder.h>
-#include <drm/drm_fb_helper.h>
-#include <drm/drm_fbdev_shmem.h>
-#include <drm/drm_fourcc.h>
-#include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_framebuffer_helper.h>
-#include <drm/drm_gem_shmem_helper.h>
-#include <drm/drm_modeset_helper_vtables.h>
-#include <drm/drm_module.h>
-#include <drm/drm_plane.h>
-#include <drm/drm_probe_helper.h>
+#include "st7571.h"
 
-#include <video/display_timing.h>
-#include <video/of_display_timing.h>
-
-#define ST7571_COMMAND_MODE			(0x00)
-#define ST7571_DATA_MODE			(0x40)
-
-/* Normal mode command set */
-#define ST7571_DISPLAY_OFF			(0xae)
-#define ST7571_DISPLAY_ON			(0xaf)
-#define ST7571_OSC_ON				(0xab)
-#define ST7571_SET_COLUMN_LSB(c)		(0x00 | FIELD_PREP(GENMASK(3, 0), (c)))
-#define ST7571_SET_COLUMN_MSB(c)		(0x10 | FIELD_PREP(GENMASK(2, 0), (c) >> 4))
-#define ST7571_SET_COM0_LSB(x)			(FIELD_PREP(GENMASK(6, 0), (x)))
-#define ST7571_SET_COM0_MSB			(0x44)
-#define ST7571_SET_COM_SCAN_DIR(d)		(0xc0 | FIELD_PREP(GENMASK(3, 3), (d)))
-#define ST7571_SET_CONTRAST_LSB(c)		(FIELD_PREP(GENMASK(5, 0), (c)))
-#define ST7571_SET_CONTRAST_MSB			(0x81)
-#define ST7571_SET_DISPLAY_DUTY_LSB(d)		(FIELD_PREP(GENMASK(7, 0), (d)))
-#define ST7571_SET_DISPLAY_DUTY_MSB		(0x48)
-#define ST7571_SET_ENTIRE_DISPLAY_ON(p)		(0xa4 | FIELD_PREP(GENMASK(0, 0), (p)))
-#define ST7571_SET_LCD_BIAS(b)			(0x50 | FIELD_PREP(GENMASK(2, 0), (b)))
-#define ST7571_SET_MODE_LSB(m)			(FIELD_PREP(GENMASK(7, 2), (m)))
-#define ST7571_SET_MODE_MSB			(0x38)
-#define ST7571_SET_PAGE(p)			(0xb0 | FIELD_PREP(GENMASK(3, 0), (p)))
-#define ST7571_SET_POWER(p)			(0x28 | FIELD_PREP(GENMASK(2, 0), (p)))
-#define ST7571_SET_REGULATOR_REG(r)		(0x20 | FIELD_PREP(GENMASK(2, 0), (r)))
-#define ST7571_SET_REVERSE(r)			(0xa6 | FIELD_PREP(GENMASK(0, 0), (r)))
-#define ST7571_SET_SEG_SCAN_DIR(d)		(0xa0 | FIELD_PREP(GENMASK(0, 0), (d)))
-#define ST7571_SET_START_LINE_LSB(l)		(FIELD_PREP(GENMASK(6, 0), (l)))
-#define ST7571_SET_START_LINE_MSB		(0x40)
-
-/* Extension command set 3 */
-#define ST7571_COMMAND_SET_3			(0x7b)
-#define ST7571_SET_COLOR_MODE(c)		(0x10 | FIELD_PREP(GENMASK(0, 0), (c)))
-#define ST7571_COMMAND_SET_NORMAL		(0x00)
-
-/* ST7567 commands */
-#define ST7567_SET_LCD_BIAS(m) (0xa2 | FIELD_PREP(GENMASK(0, 0), (m)))
-
-#define ST7571_PAGE_HEIGHT 8
-
-#define DRIVER_NAME "st7571"
-#define DRIVER_DESC "ST7571 DRM driver"
-#define DRIVER_MAJOR 1
-#define DRIVER_MINOR 0
-
-enum st7571_color_mode {
-	ST7571_COLOR_MODE_GRAY = 0,
-	ST7571_COLOR_MODE_BLACKWHITE = 1,
-};
-
-struct st7571_device;
-
-struct st7571_panel_constraints {
-	u32 min_nlines;
-	u32 max_nlines;
-	u32 min_ncols;
-	u32 max_ncols;
-	bool support_grayscale;
-};
-
-struct st7571_panel_data {
-	int (*init)(struct st7571_device *st7571);
-	int (*parse_dt)(struct st7571_device *st7571);
-	struct st7571_panel_constraints constraints;
-};
-
-struct st7571_panel_format {
-	void (*prepare_buffer)(struct st7571_device *st7571,
-			       const struct iosys_map *vmap,
-			       struct drm_framebuffer *fb,
-			       struct drm_rect *rect,
-			       struct drm_format_conv_state *fmtcnv_state);
-	int (*update_rect)(struct drm_framebuffer *fb, struct drm_rect *rect);
-	enum st7571_color_mode mode;
-	const u8 nformats;
-	const u32 formats[];
-};
-
-struct st7571_device {
-	struct drm_device dev;
-
-	struct drm_plane primary_plane;
-	struct drm_crtc crtc;
-	struct drm_encoder encoder;
-	struct drm_connector connector;
-
-	struct drm_display_mode mode;
-
-	const struct st7571_panel_format *pformat;
-	const struct st7571_panel_data *pdata;
+struct st7571_i2c_transport {
 	struct i2c_client *client;
-	struct gpio_desc *reset;
-	struct regmap *regmap;
 
 	/*
 	 * Depending on the hardware design, the acknowledge signal may be hard to
@@ -149,42 +36,21 @@ struct st7571_device {
 	 *
 	 */
 	bool ignore_nak;
-
-	bool grayscale;
-	bool inverted;
-	u32 height_mm;
-	u32 width_mm;
-	u32 startline;
-	u32 nlines;
-	u32 ncols;
-	u32 bpp;
-
-	/* Intermediate buffer in LCD friendly format */
-	u8 *hwbuf;
-
-	/* Row of (transformed) pixels ready to be written to the display */
-	u8 *row;
 };
 
-static inline struct st7571_device *drm_to_st7571(struct drm_device *dev)
-{
-	return container_of(dev, struct st7571_device, dev);
-}
-
-static int st7571_regmap_write(void *context, const void *data, size_t count)
+static int st7571_i2c_regmap_write(void *context, const void *data, size_t count)
 {
-	struct i2c_client *client = context;
-	struct st7571_device *st7571 = i2c_get_clientdata(client);
+	struct st7571_i2c_transport *t = context;
 	int ret;
 
 	struct i2c_msg msg = {
-		.addr = st7571->client->addr,
-		.flags = st7571->ignore_nak ? I2C_M_IGNORE_NAK : 0,
+		.addr = t->client->addr,
+		.flags = t->ignore_nak ? I2C_M_IGNORE_NAK : 0,
 		.len = count,
 		.buf = (u8 *)data
 	};
 
-	ret = i2c_transfer(st7571->client->adapter, &msg, 1);
+	ret = i2c_transfer(t->client->adapter, &msg, 1);
 
 	/*
 	 * Unfortunately, there is no way to check if the transfer failed because of
@@ -192,770 +58,41 @@ static int st7571_regmap_write(void *context, const void *data, size_t count)
 	 *
 	 * However, if the transfer fails and ignore_nak is set, we know it is an error.
 	 */
-	if (ret < 0 && st7571->ignore_nak)
+	if (ret < 0 && t->ignore_nak)
 		return ret;
 
 	return 0;
 }
 
 /* The st7571 driver does not read registers but regmap expects a .read */
-static int st7571_regmap_read(void *context, const void *reg_buf,
-			      size_t reg_size, void *val_buf, size_t val_size)
+static int st7571_i2c_regmap_read(void *context, const void *reg_buf,
+				  size_t reg_size, void *val_buf, size_t val_size)
 {
 	return -EOPNOTSUPP;
 }
 
-static int st7571_send_command_list(struct st7571_device *st7571,
-				    const u8 *cmd_list, size_t len)
-{
-	int ret;
-
-	for (int i = 0; i < len; i++) {
-		ret = regmap_write(st7571->regmap, ST7571_COMMAND_MODE, cmd_list[i]);
-		if (ret < 0)
-			return ret;
-	}
-
-	return ret;
-}
-
-static inline u8 st7571_transform_xy(const char *p, int x, int y, u8 bpp)
-{
-	int xrest = x % 8;
-	u8 result = 0;
-	u8 row_len = 16 * bpp;
-
-	/*
-	 * Transforms an (x, y) pixel coordinate into a vertical 8-bit
-	 * column from the framebuffer. It calculates the corresponding byte in the
-	 * framebuffer, extracts the bit at the given x position across 8 consecutive
-	 * rows, and packs those bits into a single byte.
-	 *
-	 * Return an 8-bit value representing a vertical column of pixels.
-	 */
-	x = x / 8;
-	y = (y / 8) * 8;
-
-	for (int i = 0; i < 8; i++) {
-		int row_idx = y + i;
-		u8 byte = p[row_idx * row_len + x];
-		u8 bit = (byte >> xrest) & 1;
-
-		result |= (bit << i);
-	}
-
-	return result;
-}
-
-static int st7571_set_position(struct st7571_device *st7571, int x, int y)
-{
-	u8 cmd_list[] = {
-		ST7571_SET_COLUMN_LSB(x),
-		ST7571_SET_COLUMN_MSB(x),
-		ST7571_SET_PAGE(y / ST7571_PAGE_HEIGHT),
-	};
-
-	return st7571_send_command_list(st7571, cmd_list, ARRAY_SIZE(cmd_list));
-}
-
-static int st7571_fb_clear_screen(struct st7571_device *st7571)
-{
-	u32 npixels = st7571->ncols * round_up(st7571->nlines, ST7571_PAGE_HEIGHT) * st7571->bpp;
-	char pixelvalue = 0x00;
-
-	st7571_set_position(st7571, 0, 0);
-	for (int i = 0; i < npixels; i++)
-		regmap_bulk_write(st7571->regmap, ST7571_DATA_MODE, &pixelvalue, 1);
-
-	return 0;
-}
-
-static void st7571_prepare_buffer_monochrome(struct st7571_device *st7571,
-					     const struct iosys_map *vmap,
-					     struct drm_framebuffer *fb,
-					     struct drm_rect *rect,
-					     struct drm_format_conv_state *fmtcnv_state)
-{
-	unsigned int dst_pitch;
-	struct iosys_map dst;
-	u32 size;
-
-	switch (fb->format->format) {
-	case DRM_FORMAT_XRGB8888:
-		dst_pitch = DIV_ROUND_UP(drm_rect_width(rect), 8);
-		iosys_map_set_vaddr(&dst, st7571->hwbuf);
-
-		drm_fb_xrgb8888_to_mono(&dst, &dst_pitch, vmap, fb, rect, fmtcnv_state);
-		break;
-
-	case DRM_FORMAT_R1:
-		size = (rect->x2 - rect->x1) * (rect->y2 - rect->y1) / 8;
-		memcpy(st7571->hwbuf, vmap->vaddr, size);
-		break;
-	}
-}
-
-static void st7571_prepare_buffer_grayscale(struct st7571_device *st7571,
-					    const struct iosys_map *vmap,
-					    struct drm_framebuffer *fb,
-					    struct drm_rect *rect,
-					    struct drm_format_conv_state *fmtcnv_state)
-{
-	u32 size = (rect->x2 - rect->x1) * (rect->y2 - rect->y1) / 8;
-	unsigned int dst_pitch;
-	struct iosys_map dst;
-
-	switch (fb->format->format) {
-	case DRM_FORMAT_XRGB8888:
-		dst_pitch = DIV_ROUND_UP(drm_rect_width(rect), 4);
-		iosys_map_set_vaddr(&dst, st7571->hwbuf);
-
-		drm_fb_xrgb8888_to_gray2(&dst, &dst_pitch, vmap, fb, rect, fmtcnv_state);
-		break;
-
-	case DRM_FORMAT_R1:
-		size = (rect->x2 - rect->x1) * (rect->y2 - rect->y1) / 8;
-		memcpy(st7571->hwbuf, vmap->vaddr, size);
-		break;
-
-	case DRM_FORMAT_R2:
-		size = (rect->x2 - rect->x1) * (rect->y2 - rect->y1) / 4;
-		memcpy(st7571->hwbuf, vmap->vaddr, size);
-		break;
-	}
-}
-
-static int st7571_fb_update_rect_monochrome(struct drm_framebuffer *fb, struct drm_rect *rect)
-{
-	struct st7571_device *st7571 = drm_to_st7571(fb->dev);
-	char *row = st7571->row;
-
-	/* Align y to display page boundaries */
-	rect->y1 = round_down(rect->y1, ST7571_PAGE_HEIGHT);
-	rect->y2 = min_t(unsigned int, round_up(rect->y2, ST7571_PAGE_HEIGHT), st7571->nlines);
-
-	for (int y = rect->y1; y < rect->y2; y += ST7571_PAGE_HEIGHT) {
-		for (int x = rect->x1; x < rect->x2; x++)
-			row[x] = st7571_transform_xy(st7571->hwbuf, x, y, 1);
-
-		st7571_set_position(st7571, rect->x1, y);
-
-		/* TODO: Investige why we can't write multiple bytes at once */
-		for (int x = rect->x1; x < rect->x2; x++)
-			regmap_bulk_write(st7571->regmap, ST7571_DATA_MODE, row + x, 1);
-	}
-
-	return 0;
-}
-
-static int st7571_fb_update_rect_grayscale(struct drm_framebuffer *fb, struct drm_rect *rect)
-{
-	struct st7571_device *st7571 = drm_to_st7571(fb->dev);
-	u32 format = fb->format->format;
-	char *row = st7571->row;
-	int x1;
-	int x2;
-
-	/* Align y to display page boundaries */
-	rect->y1 = round_down(rect->y1, ST7571_PAGE_HEIGHT);
-	rect->y2 = min_t(unsigned int, round_up(rect->y2, ST7571_PAGE_HEIGHT), st7571->nlines);
-
-	switch (format) {
-	case DRM_FORMAT_R1:
-		x1 = rect->x1 * 1;
-		x2 = rect->x2 * 1;
-		break;
-	case DRM_FORMAT_R2:
-		fallthrough;
-	case DRM_FORMAT_XRGB8888:
-		x1 = rect->x1 * 2;
-		x2 = rect->x2 * 2;
-		break;
-	}
-
-	for (int y = rect->y1; y < rect->y2; y += ST7571_PAGE_HEIGHT) {
-		for (int x = x1; x < x2; x++)
-			row[x] = st7571_transform_xy(st7571->hwbuf, x, y, 2);
-
-		st7571_set_position(st7571, rect->x1, y);
-
-		/* TODO: Investige why we can't write multiple bytes at once */
-		for (int x = x1; x < x2; x++) {
-			regmap_bulk_write(st7571->regmap, ST7571_DATA_MODE, row + x, 1);
-
-			/*
-			 * As the display supports grayscale, all pixels must be written as two bits
-			 * even if the format is monochrome.
-			 *
-			 * The bit values maps to the following grayscale:
-			 * 0 0 = Black
-			 * 0 1 = Dark gray
-			 * 1 0 = Light gray
-			 * 1 1 = White
-			 *
-			 * For monochrome formats, write the same value twice to get
-			 * either a black or white pixel.
-			 */
-			if (format == DRM_FORMAT_R1)
-				regmap_bulk_write(st7571->regmap, ST7571_DATA_MODE, row + x, 1);
-		}
-	}
-
-	return 0;
-}
-
-static int st7571_connector_get_modes(struct drm_connector *conn)
-{
-	struct st7571_device *st7571 = drm_to_st7571(conn->dev);
-
-	return drm_connector_helper_get_modes_fixed(conn, &st7571->mode);
-}
-
-static const struct drm_connector_helper_funcs st7571_connector_helper_funcs = {
-	.get_modes = st7571_connector_get_modes,
-};
-
-static const struct st7571_panel_format st7571_monochrome = {
-	.prepare_buffer = st7571_prepare_buffer_monochrome,
-	.update_rect = st7571_fb_update_rect_monochrome,
-	.mode = ST7571_COLOR_MODE_BLACKWHITE,
-	.formats = {
-		DRM_FORMAT_XRGB8888,
-		DRM_FORMAT_R1,
-	},
-	.nformats = 2,
-};
-
-static const struct st7571_panel_format st7571_grayscale = {
-	.prepare_buffer = st7571_prepare_buffer_grayscale,
-	.update_rect = st7571_fb_update_rect_grayscale,
-	.mode = ST7571_COLOR_MODE_GRAY,
-	.formats = {
-		DRM_FORMAT_XRGB8888,
-		DRM_FORMAT_R1,
-		DRM_FORMAT_R2,
-	},
-	.nformats = 3,
-};
-
-static const u64 st7571_primary_plane_fmtmods[] = {
-	DRM_FORMAT_MOD_LINEAR,
-	DRM_FORMAT_MOD_INVALID
-};
-
-static int st7571_primary_plane_helper_atomic_check(struct drm_plane *plane,
-						    struct drm_atomic_state *state)
-{
-	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
-	struct drm_crtc *new_crtc = new_plane_state->crtc;
-	struct drm_crtc_state *new_crtc_state = NULL;
-
-	if (new_crtc)
-		new_crtc_state = drm_atomic_get_new_crtc_state(state, new_crtc);
-
-	return drm_atomic_helper_check_plane_state(new_plane_state, new_crtc_state,
-						   DRM_PLANE_NO_SCALING,
-						   DRM_PLANE_NO_SCALING,
-						   false, false);
-}
-
-static void st7571_primary_plane_helper_atomic_update(struct drm_plane *plane,
-						      struct drm_atomic_state *state)
-{
-	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
-	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
-	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
-	struct drm_framebuffer *fb = plane_state->fb;
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_device *dev = plane->dev;
-	struct drm_rect damage;
-	struct st7571_device *st7571 = drm_to_st7571(plane->dev);
-	int ret, idx;
-
-	if (!fb)
-		return; /* no framebuffer; plane is disabled */
-
-	ret = drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE);
-	if (ret)
-		return;
-
-	if (!drm_dev_enter(dev, &idx))
-		goto out_drm_gem_fb_end_cpu_access;
-
-	drm_atomic_helper_damage_iter_init(&iter, old_plane_state, plane_state);
-	drm_atomic_for_each_plane_damage(&iter, &damage) {
-		st7571->pformat->prepare_buffer(st7571,
-						&shadow_plane_state->data[0],
-						fb, &damage,
-						&shadow_plane_state->fmtcnv_state);
-
-		st7571->pformat->update_rect(fb, &damage);
-	}
-
-	drm_dev_exit(idx);
-
-out_drm_gem_fb_end_cpu_access:
-	drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE);
-}
-
-static void st7571_primary_plane_helper_atomic_disable(struct drm_plane *plane,
-						       struct drm_atomic_state *state)
-{
-	struct drm_device *dev = plane->dev;
-	struct st7571_device *st7571 = drm_to_st7571(plane->dev);
-	int idx;
-
-	if (!drm_dev_enter(dev, &idx))
-		return;
-
-	st7571_fb_clear_screen(st7571);
-	drm_dev_exit(idx);
-}
-
-static const struct drm_plane_helper_funcs st7571_primary_plane_helper_funcs = {
-	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
-	.atomic_check = st7571_primary_plane_helper_atomic_check,
-	.atomic_update = st7571_primary_plane_helper_atomic_update,
-	.atomic_disable = st7571_primary_plane_helper_atomic_disable,
-};
-
-static const struct drm_plane_funcs st7571_primary_plane_funcs = {
-	.update_plane = drm_atomic_helper_update_plane,
-	.disable_plane = drm_atomic_helper_disable_plane,
-	.destroy = drm_plane_cleanup,
-	DRM_GEM_SHADOW_PLANE_FUNCS,
-};
-
-/*
- * CRTC
- */
-
-static enum drm_mode_status st7571_crtc_mode_valid(struct drm_crtc *crtc,
-						   const struct drm_display_mode *mode)
-{
-	struct st7571_device *st7571 = drm_to_st7571(crtc->dev);
-
-	return drm_crtc_helper_mode_valid_fixed(crtc, mode, &st7571->mode);
-}
-
-static const struct drm_crtc_helper_funcs st7571_crtc_helper_funcs = {
-	.atomic_check = drm_crtc_helper_atomic_check,
-	.mode_valid = st7571_crtc_mode_valid,
-};
-
-static const struct drm_crtc_funcs st7571_crtc_funcs = {
-	.reset = drm_atomic_helper_crtc_reset,
-	.destroy = drm_crtc_cleanup,
-	.set_config = drm_atomic_helper_set_config,
-	.page_flip = drm_atomic_helper_page_flip,
-	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
-};
-
-/*
- * Encoder
- */
-
-static void st7571_encoder_atomic_enable(struct drm_encoder *encoder,
-					 struct drm_atomic_state *state)
-{
-	struct drm_device *drm = encoder->dev;
-	struct st7571_device *st7571 = drm_to_st7571(drm);
-	u8 command = ST7571_DISPLAY_ON;
-	int ret;
-
-	ret = st7571->pdata->init(st7571);
-	if (ret)
-		return;
-
-	st7571_send_command_list(st7571, &command, 1);
-}
-
-static void st7571_encoder_atomic_disable(struct drm_encoder *encoder,
-					  struct drm_atomic_state *state)
-{
-	struct drm_device *drm = encoder->dev;
-	struct st7571_device *st7571 = drm_to_st7571(drm);
-	u8 command = ST7571_DISPLAY_OFF;
-
-	st7571_send_command_list(st7571, &command, 1);
-}
-
-static const struct drm_encoder_funcs st7571_encoder_funcs = {
-	.destroy = drm_encoder_cleanup,
-
-};
-
-static const struct drm_encoder_helper_funcs st7571_encoder_helper_funcs = {
-	.atomic_enable = st7571_encoder_atomic_enable,
-	.atomic_disable = st7571_encoder_atomic_disable,
-};
-
-/*
- * Connector
- */
-
-static const struct drm_connector_funcs st7571_connector_funcs = {
-	.reset = drm_atomic_helper_connector_reset,
-	.fill_modes = drm_helper_probe_single_connector_modes,
-	.destroy = drm_connector_cleanup,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+static const struct regmap_bus st7571_i2c_regmap_bus = {
+	.read = st7571_i2c_regmap_read,
+	.write = st7571_i2c_regmap_write,
 };
 
-static const struct drm_mode_config_funcs st7571_mode_config_funcs = {
-	.fb_create = drm_gem_fb_create_with_dirty,
-	.atomic_check = drm_atomic_helper_check,
-	.atomic_commit = drm_atomic_helper_commit,
-};
-
-static struct drm_display_mode st7571_mode(struct st7571_device *st7571)
-{
-	struct drm_display_mode mode = {
-		DRM_SIMPLE_MODE(st7571->ncols, st7571->nlines,
-				st7571->width_mm, st7571->height_mm),
-	};
-
-	return mode;
-}
-
-static int st7571_mode_config_init(struct st7571_device *st7571)
-{
-	struct drm_device *dev = &st7571->dev;
-	const struct st7571_panel_constraints *constraints = &st7571->pdata->constraints;
-	int ret;
-
-	ret = drmm_mode_config_init(dev);
-	if (ret)
-		return ret;
-
-	dev->mode_config.min_width = constraints->min_ncols;
-	dev->mode_config.min_height = constraints->min_nlines;
-	dev->mode_config.max_width = constraints->max_ncols;
-	dev->mode_config.max_height = constraints->max_nlines;
-	dev->mode_config.preferred_depth = 24;
-	dev->mode_config.funcs = &st7571_mode_config_funcs;
-
-	return 0;
-}
-
-static int st7571_plane_init(struct st7571_device *st7571,
-			     const struct st7571_panel_format *pformat)
-{
-	struct drm_plane *primary_plane = &st7571->primary_plane;
-	struct drm_device *dev = &st7571->dev;
-	int ret;
-
-	ret = drm_universal_plane_init(dev, primary_plane, 0,
-				       &st7571_primary_plane_funcs,
-				       pformat->formats,
-				       pformat->nformats,
-				       st7571_primary_plane_fmtmods,
-				       DRM_PLANE_TYPE_PRIMARY, NULL);
-	if (ret)
-		return ret;
-
-	drm_plane_helper_add(primary_plane, &st7571_primary_plane_helper_funcs);
-	drm_plane_enable_fb_damage_clips(primary_plane);
-
-	return 0;
-}
-
-static int st7571_crtc_init(struct st7571_device *st7571)
-{
-	struct drm_plane *primary_plane = &st7571->primary_plane;
-	struct drm_crtc *crtc = &st7571->crtc;
-	struct drm_device *dev = &st7571->dev;
-	int ret;
-
-	ret = drm_crtc_init_with_planes(dev, crtc, primary_plane, NULL,
-					&st7571_crtc_funcs, NULL);
-	if (ret)
-		return ret;
-
-	drm_crtc_helper_add(crtc, &st7571_crtc_helper_funcs);
-
-	return 0;
-}
-
-static int st7571_encoder_init(struct st7571_device *st7571)
-{
-	struct drm_encoder *encoder = &st7571->encoder;
-	struct drm_crtc *crtc = &st7571->crtc;
-	struct drm_device *dev = &st7571->dev;
-	int ret;
-
-	ret = drm_encoder_init(dev, encoder, &st7571_encoder_funcs, DRM_MODE_ENCODER_NONE, NULL);
-	if (ret)
-		return ret;
-
-	drm_encoder_helper_add(encoder, &st7571_encoder_helper_funcs);
-
-	encoder->possible_crtcs = drm_crtc_mask(crtc);
-
-	return 0;
-}
-
-static int st7571_connector_init(struct st7571_device *st7571)
-{
-	struct drm_connector *connector = &st7571->connector;
-	struct drm_encoder *encoder = &st7571->encoder;
-	struct drm_device *dev = &st7571->dev;
-	int ret;
-
-	ret = drm_connector_init(dev, connector, &st7571_connector_funcs,
-				 DRM_MODE_CONNECTOR_Unknown);
-	if (ret)
-		return ret;
-
-	drm_connector_helper_add(connector, &st7571_connector_helper_funcs);
-
-	return drm_connector_attach_encoder(connector, encoder);
-}
-
-DEFINE_DRM_GEM_FOPS(st7571_fops);
-
-static const struct drm_driver st7571_driver = {
-	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC,
-
-	.name		 = DRIVER_NAME,
-	.desc		 = DRIVER_DESC,
-	.major		 = DRIVER_MAJOR,
-	.minor		 = DRIVER_MINOR,
-
-	.fops		 = &st7571_fops,
-	DRM_GEM_SHMEM_DRIVER_OPS,
-	DRM_FBDEV_SHMEM_DRIVER_OPS,
-};
-
-static const struct regmap_bus st7571_regmap_bus = {
-	.read = st7571_regmap_read,
-	.write = st7571_regmap_write,
-};
-
-static const struct regmap_config st7571_regmap_config = {
+static const struct regmap_config st7571_i2c_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
 	.use_single_write = true,
 };
 
-static int st7571_validate_parameters(struct st7571_device *st7571)
-{
-	struct device *dev = st7571->dev.dev;
-	const struct st7571_panel_constraints *constraints = &st7571->pdata->constraints;
-
-	if (st7571->width_mm  == 0) {
-		dev_err(dev, "Invalid panel width\n");
-		return -EINVAL;
-	}
-
-	if (st7571->height_mm == 0) {
-		dev_err(dev, "Invalid panel height\n");
-		return -EINVAL;
-	}
-
-	if (st7571->nlines < constraints->min_nlines ||
-	    st7571->nlines > constraints->max_nlines) {
-		dev_err(dev, "Invalid timing configuration.\n");
-		return -EINVAL;
-	}
-
-	if (st7571->startline + st7571->nlines > constraints->max_nlines) {
-		dev_err(dev, "Invalid timing configuration.\n");
-		return -EINVAL;
-	}
-
-	if (st7571->ncols < constraints->min_ncols ||
-	    st7571->ncols > constraints->max_ncols) {
-		dev_err(dev, "Invalid timing configuration.\n");
-		return -EINVAL;
-	}
-
-	if (st7571->grayscale && !constraints->support_grayscale) {
-		dev_err(dev, "Grayscale not supported\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int st7567_parse_dt(struct st7571_device *st7567)
-{
-	struct device *dev = &st7567->client->dev;
-	struct device_node *np = dev->of_node;
-	struct display_timing dt;
-	int ret;
-
-	ret = of_get_display_timing(np, "panel-timing", &dt);
-	if (ret) {
-		dev_err(dev, "Failed to get display timing from DT\n");
-		return ret;
-	}
-
-	of_property_read_u32(np, "width-mm", &st7567->width_mm);
-	of_property_read_u32(np, "height-mm", &st7567->height_mm);
-	st7567->inverted = of_property_read_bool(np, "sitronix,inverted");
-
-	st7567->pformat = &st7571_monochrome;
-	st7567->bpp = 1;
-
-	st7567->startline = dt.vfront_porch.typ;
-	st7567->nlines = dt.vactive.typ;
-	st7567->ncols = dt.hactive.typ;
-
-	return 0;
-}
-
-static int st7571_parse_dt(struct st7571_device *st7571)
-{
-	struct device *dev = &st7571->client->dev;
-	struct device_node *np = dev->of_node;
-	struct display_timing dt;
-	int ret;
-
-	ret = of_get_display_timing(np, "panel-timing", &dt);
-	if (ret) {
-		dev_err(dev, "Failed to get display timing from DT\n");
-		return ret;
-	}
-
-	of_property_read_u32(np, "width-mm", &st7571->width_mm);
-	of_property_read_u32(np, "height-mm", &st7571->height_mm);
-	st7571->grayscale = of_property_read_bool(np, "sitronix,grayscale");
-	st7571->inverted = of_property_read_bool(np, "sitronix,inverted");
-
-	if (st7571->grayscale) {
-		st7571->pformat = &st7571_grayscale;
-		st7571->bpp = 2;
-	} else {
-		st7571->pformat = &st7571_monochrome;
-		st7571->bpp = 1;
-	}
-
-	st7571->startline = dt.vfront_porch.typ;
-	st7571->nlines = dt.vactive.typ;
-	st7571->ncols = dt.hactive.typ;
-
-	st7571->reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
-	if (IS_ERR(st7571->reset))
-		return dev_err_probe(dev, PTR_ERR(st7571->reset),
-				     "Failed to get reset gpio\n");
-
-
-	return 0;
-}
-
-static void st7571_reset(struct st7571_device *st7571)
-{
-	gpiod_set_value_cansleep(st7571->reset, 1);
-	fsleep(20);
-	gpiod_set_value_cansleep(st7571->reset, 0);
-}
-
-static int st7567_lcd_init(struct st7571_device *st7567)
-{
-	/*
-	 * Most of the initialization sequence is taken directly from the
-	 * referential initial code in the ST7567 datasheet.
-	 */
-	u8 commands[] = {
-		ST7571_DISPLAY_OFF,
-
-		ST7567_SET_LCD_BIAS(1),
-
-		ST7571_SET_SEG_SCAN_DIR(0),
-		ST7571_SET_COM_SCAN_DIR(1),
-
-		ST7571_SET_REGULATOR_REG(4),
-		ST7571_SET_CONTRAST_MSB,
-		ST7571_SET_CONTRAST_LSB(0x20),
-
-		ST7571_SET_START_LINE_MSB,
-		ST7571_SET_START_LINE_LSB(st7567->startline),
-
-		ST7571_SET_POWER(0x4),	/* Power Control, VC: ON, VR: OFF, VF: OFF */
-		ST7571_SET_POWER(0x6),	/* Power Control, VC: ON, VR: ON, VF: OFF */
-		ST7571_SET_POWER(0x7),	/* Power Control, VC: ON, VR: ON, VF: ON */
-
-		ST7571_SET_REVERSE(st7567->inverted ? 1 : 0),
-		ST7571_SET_ENTIRE_DISPLAY_ON(0),
-	};
-
-	return st7571_send_command_list(st7567, commands, ARRAY_SIZE(commands));
-}
-
-static int st7571_lcd_init(struct st7571_device *st7571)
-{
-	/*
-	 * Most of the initialization sequence is taken directly from the
-	 * referential initial code in the ST7571 datasheet.
-	 */
-	u8 commands[] = {
-		ST7571_DISPLAY_OFF,
-
-		ST7571_SET_MODE_MSB,
-		ST7571_SET_MODE_LSB(0x2e),
-
-		ST7571_SET_SEG_SCAN_DIR(0),
-		ST7571_SET_COM_SCAN_DIR(1),
-
-		ST7571_SET_COM0_MSB,
-		ST7571_SET_COM0_LSB(0x00),
-
-		ST7571_SET_START_LINE_MSB,
-		ST7571_SET_START_LINE_LSB(st7571->startline),
-
-		ST7571_OSC_ON,
-		ST7571_SET_REGULATOR_REG(5),
-		ST7571_SET_CONTRAST_MSB,
-		ST7571_SET_CONTRAST_LSB(0x33),
-		ST7571_SET_LCD_BIAS(0x04),
-		ST7571_SET_DISPLAY_DUTY_MSB,
-		ST7571_SET_DISPLAY_DUTY_LSB(st7571->nlines),
-
-		ST7571_SET_POWER(0x4),	/* Power Control, VC: ON, VR: OFF, VF: OFF */
-		ST7571_SET_POWER(0x6),	/* Power Control, VC: ON, VR: ON, VF: OFF */
-		ST7571_SET_POWER(0x7),	/* Power Control, VC: ON, VR: ON, VF: ON */
-
-		ST7571_COMMAND_SET_3,
-		ST7571_SET_COLOR_MODE(st7571->pformat->mode),
-		ST7571_COMMAND_SET_NORMAL,
-
-		ST7571_SET_REVERSE(st7571->inverted ? 1 : 0),
-		ST7571_SET_ENTIRE_DISPLAY_ON(0),
-	};
-
-	/* Perform a reset before initializing the controller */
-	st7571_reset(st7571);
-
-	return st7571_send_command_list(st7571, commands, ARRAY_SIZE(commands));
-}
-
-static int st7571_probe(struct i2c_client *client)
+static int st7571_i2c_probe(struct i2c_client *client)
 {
 	struct st7571_device *st7571;
-	struct drm_device *dev;
-	int ret;
-
-	st7571 = devm_drm_dev_alloc(&client->dev, &st7571_driver,
-				    struct st7571_device, dev);
-	if (IS_ERR(st7571))
-		return PTR_ERR(st7571);
-
-	dev = &st7571->dev;
-	st7571->client = client;
-	i2c_set_clientdata(client, st7571);
-	st7571->pdata = device_get_match_data(&client->dev);
-
-	ret = st7571->pdata->parse_dt(st7571);
-	if (ret)
-		return ret;
+	struct st7571_i2c_transport *t;
+	struct regmap *regmap;
 
-	ret = st7571_validate_parameters(st7571);
-	if (ret)
-		return ret;
+	t = devm_kzalloc(&client->dev, sizeof(*t), GFP_KERNEL);
+	if (!t)
+		return -ENOMEM;
 
-	st7571->mode = st7571_mode(st7571);
+	t->client = client;
 
 	/*
 	 * The hardware design could make it hard to detect a NAK on the I2C bus.
@@ -964,94 +101,31 @@ static int st7571_probe(struct i2c_client *client)
 	 * cruft in the logs.
 	 */
 	if (i2c_check_functionality(client->adapter, I2C_FUNC_PROTOCOL_MANGLING))
-		st7571->ignore_nak = true;
+		t->ignore_nak = true;
 
-	st7571->regmap = devm_regmap_init(&client->dev, &st7571_regmap_bus,
-					  client, &st7571_regmap_config);
-	if (IS_ERR(st7571->regmap)) {
-		return dev_err_probe(&client->dev, PTR_ERR(st7571->regmap),
+	regmap = devm_regmap_init(&client->dev, &st7571_i2c_regmap_bus,
+				  t, &st7571_i2c_regmap_config);
+	if (IS_ERR(regmap)) {
+		return dev_err_probe(&client->dev, PTR_ERR(regmap),
 				     "Failed to initialize regmap\n");
 	}
 
-	st7571->hwbuf = devm_kzalloc(&client->dev,
-				     (st7571->nlines * st7571->ncols * st7571->bpp) / 8,
-				     GFP_KERNEL);
-	if (!st7571->hwbuf)
-		return -ENOMEM;
-
-	st7571->row = devm_kzalloc(&client->dev,
-				   (st7571->ncols * st7571->bpp),
-				   GFP_KERNEL);
-	if (!st7571->row)
-		return -ENOMEM;
-
-	ret = st7571_mode_config_init(st7571);
-	if (ret)
-		return dev_err_probe(&client->dev, ret,
-				     "Failed to initialize mode config\n");
-
-	ret = st7571_plane_init(st7571, st7571->pformat);
-	if (ret)
-		return dev_err_probe(&client->dev, ret,
-				     "Failed to initialize primary plane\n");
-
-	ret = st7571_crtc_init(st7571);
-	if (ret < 0)
-		return dev_err_probe(&client->dev, ret,
-				     "Failed to initialize CRTC\n");
-
-	ret = st7571_encoder_init(st7571);
-	if (ret < 0)
-		return dev_err_probe(&client->dev, ret,
-				     "Failed to initialize encoder\n");
-
-	ret = st7571_connector_init(st7571);
-	if (ret < 0)
-		return dev_err_probe(&client->dev, ret,
-				     "Failed to initialize connector\n");
-
-	drm_mode_config_reset(dev);
-
-	ret = drm_dev_register(dev, 0);
-	if (ret)
-		return dev_err_probe(&client->dev, ret,
-				     "Failed to register DRM device\n");
+	st7571 = st7571_probe(&client->dev, regmap);
+	if (IS_ERR(st7571))
+		return dev_err_probe(&client->dev, PTR_ERR(st7571),
+				     "Failed to initialize regmap\n");
 
-	drm_client_setup(dev, NULL);
+	i2c_set_clientdata(client, st7571);
 	return 0;
 }
 
-static void st7571_remove(struct i2c_client *client)
+static void st7571_i2c_remove(struct i2c_client *client)
 {
 	struct st7571_device *st7571 = i2c_get_clientdata(client);
 
-	drm_dev_unplug(&st7571->dev);
+	st7571_remove(st7571);
 }
 
-static const struct st7571_panel_data st7567_config = {
-	.init = st7567_lcd_init,
-	.parse_dt = st7567_parse_dt,
-	.constraints = {
-		.min_nlines = 1,
-		.max_nlines = 64,
-		.min_ncols = 128,
-		.max_ncols = 128,
-		.support_grayscale = false,
-	},
-};
-
-static const struct st7571_panel_data st7571_config = {
-	.init = st7571_lcd_init,
-	.parse_dt = st7571_parse_dt,
-	.constraints = {
-		.min_nlines = 1,
-		.max_nlines = 128,
-		.min_ncols = 128,
-		.max_ncols = 128,
-		.support_grayscale = true,
-	},
-};
-
 static const struct of_device_id st7571_of_match[] = {
 	{ .compatible = "sitronix,st7567", .data = &st7567_config },
 	{ .compatible = "sitronix,st7571", .data = &st7571_config },
@@ -1068,16 +142,17 @@ MODULE_DEVICE_TABLE(i2c, st7571_id);
 
 static struct i2c_driver st7571_i2c_driver = {
 	.driver = {
-		.name = "st7571",
+		.name = "st7571-i2c",
 		.of_match_table = st7571_of_match,
 	},
-	.probe = st7571_probe,
-	.remove = st7571_remove,
+	.probe = st7571_i2c_probe,
+	.remove = st7571_i2c_remove,
 	.id_table = st7571_id,
 };
 
 module_i2c_driver(st7571_i2c_driver);
 
 MODULE_AUTHOR("Marcus Folkesson <marcus.folkesson@gmail.com>");
-MODULE_DESCRIPTION("DRM Driver for Sitronix ST7571 LCD controller");
+MODULE_DESCRIPTION("DRM Driver for Sitronix ST7571 LCD controller (I2C)");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("DRM_ST7571");
diff --git a/drivers/gpu/drm/sitronix/st7571-spi.c b/drivers/gpu/drm/sitronix/st7571-spi.c
new file mode 100644
index 000000000000..0206e9162f1c
--- /dev/null
+++ b/drivers/gpu/drm/sitronix/st7571-spi.c
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for Sitronix ST7571 connected via SPI bus.
+ *
+ * Copyright (C) 2025 Marcus Folkesson <marcus.folkesson@gmail.com>
+ */
+
+#include <linux/spi/spi.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+
+#include "st7571.h"
+
+static const struct regmap_config st7571_spi_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.can_multi_write = true,
+};
+
+static int st7571_spi_probe(struct spi_device *spi)
+{
+	struct st7571_device *st7571;
+	struct regmap *regmap;
+
+	regmap = devm_regmap_init_spi(spi, &st7571_spi_regmap_config);
+	if (IS_ERR(regmap)) {
+		return dev_err_probe(&spi->dev, PTR_ERR(regmap),
+				     "Failed to initialize regmap\n");
+	}
+
+	st7571 = st7571_probe(&spi->dev, regmap);
+	if (IS_ERR(st7571))
+		return dev_err_probe(&spi->dev, PTR_ERR(st7571),
+				     "Failed to initialize regmap\n");
+
+	spi_set_drvdata(spi, st7571);
+	return 0;
+}
+
+static void st7571_spi_remove(struct spi_device *spi)
+{
+	struct st7571_device *st7571 = spi_get_drvdata(spi);
+
+	st7571_remove(st7571);
+}
+
+static const struct of_device_id st7571_of_match[] = {
+	{ .compatible = "sitronix,st7567", .data = &st7567_config },
+	{ .compatible = "sitronix,st7571", .data = &st7571_config },
+	{},
+};
+MODULE_DEVICE_TABLE(of, st7571_of_match);
+
+static const struct spi_device_id st7571_spi_id[] = {
+	{ "st7567", 0 },
+	{ "st7571", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(spi, st7571_spi_id);
+
+static struct spi_driver st7571_spi_driver = {
+	.driver = {
+		.name = "st7571-spi",
+		.of_match_table = st7571_of_match,
+	},
+	.probe = st7571_spi_probe,
+	.remove = st7571_spi_remove,
+	.id_table = st7571_spi_id,
+};
+
+module_spi_driver(st7571_spi_driver);
+
+MODULE_AUTHOR("Marcus Folkesson <marcus.folkesson@gmail.com>");
+MODULE_DESCRIPTION("DRM Driver for Sitronix ST7571 LCD controller (SPI)");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("DRM_ST7571");
diff --git a/drivers/gpu/drm/sitronix/st7571.c b/drivers/gpu/drm/sitronix/st7571.c
new file mode 100644
index 000000000000..5fd575d972a2
--- /dev/null
+++ b/drivers/gpu/drm/sitronix/st7571.c
@@ -0,0 +1,918 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for Sitronix ST7571, a 4 level gray scale dot matrix LCD controller
+ *
+ * Copyright (C) 2025 Marcus Folkesson <marcus.folkesson@gmail.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+
+#include <drm/clients/drm_client_setup.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_damage_helper.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_encoder.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_fbdev_shmem.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_gem_shmem_helper.h>
+#include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_module.h>
+#include <drm/drm_plane.h>
+#include <drm/drm_probe_helper.h>
+
+#include <video/display_timing.h>
+#include <video/of_display_timing.h>
+
+#include "st7571.h"
+
+#define ST7571_COMMAND_MODE			(0x00)
+#define ST7571_DATA_MODE			(0x40)
+
+/* Normal mode command set */
+#define ST7571_DISPLAY_OFF			(0xae)
+#define ST7571_DISPLAY_ON			(0xaf)
+#define ST7571_OSC_ON				(0xab)
+#define ST7571_SET_COLUMN_LSB(c)		(0x00 | FIELD_PREP(GENMASK(3, 0), (c)))
+#define ST7571_SET_COLUMN_MSB(c)		(0x10 | FIELD_PREP(GENMASK(2, 0), (c) >> 4))
+#define ST7571_SET_COM0_LSB(x)			(FIELD_PREP(GENMASK(6, 0), (x)))
+#define ST7571_SET_COM0_MSB			(0x44)
+#define ST7571_SET_COM_SCAN_DIR(d)		(0xc0 | FIELD_PREP(GENMASK(3, 3), (d)))
+#define ST7571_SET_CONTRAST_LSB(c)		(FIELD_PREP(GENMASK(5, 0), (c)))
+#define ST7571_SET_CONTRAST_MSB			(0x81)
+#define ST7571_SET_DISPLAY_DUTY_LSB(d)		(FIELD_PREP(GENMASK(7, 0), (d)))
+#define ST7571_SET_DISPLAY_DUTY_MSB		(0x48)
+#define ST7571_SET_ENTIRE_DISPLAY_ON(p)		(0xa4 | FIELD_PREP(GENMASK(0, 0), (p)))
+#define ST7571_SET_LCD_BIAS(b)			(0x50 | FIELD_PREP(GENMASK(2, 0), (b)))
+#define ST7571_SET_MODE_LSB(m)			(FIELD_PREP(GENMASK(7, 2), (m)))
+#define ST7571_SET_MODE_MSB			(0x38)
+#define ST7571_SET_PAGE(p)			(0xb0 | FIELD_PREP(GENMASK(3, 0), (p)))
+#define ST7571_SET_POWER(p)			(0x28 | FIELD_PREP(GENMASK(2, 0), (p)))
+#define ST7571_SET_REGULATOR_REG(r)		(0x20 | FIELD_PREP(GENMASK(2, 0), (r)))
+#define ST7571_SET_REVERSE(r)			(0xa6 | FIELD_PREP(GENMASK(0, 0), (r)))
+#define ST7571_SET_SEG_SCAN_DIR(d)		(0xa0 | FIELD_PREP(GENMASK(0, 0), (d)))
+#define ST7571_SET_START_LINE_LSB(l)		(FIELD_PREP(GENMASK(6, 0), (l)))
+#define ST7571_SET_START_LINE_MSB		(0x40)
+
+/* Extension command set 3 */
+#define ST7571_COMMAND_SET_3			(0x7b)
+#define ST7571_SET_COLOR_MODE(c)		(0x10 | FIELD_PREP(GENMASK(0, 0), (c)))
+#define ST7571_COMMAND_SET_NORMAL		(0x00)
+
+/* ST7567 commands */
+#define ST7567_SET_LCD_BIAS(m) (0xa2 | FIELD_PREP(GENMASK(0, 0), (m)))
+
+#define ST7571_PAGE_HEIGHT 8
+
+#define DRIVER_NAME "st7571"
+#define DRIVER_DESC "ST7571 DRM driver"
+#define DRIVER_MAJOR 1
+#define DRIVER_MINOR 0
+
+static inline struct st7571_device *drm_to_st7571(struct drm_device *drm)
+{
+	return container_of(drm, struct st7571_device, drm);
+}
+
+static int st7571_send_command_list(struct st7571_device *st7571,
+				    const u8 *cmd_list, size_t len)
+{
+	int ret;
+
+	for (int i = 0; i < len; i++) {
+		ret = regmap_write(st7571->regmap, ST7571_COMMAND_MODE, cmd_list[i]);
+		if (ret < 0)
+			return ret;
+	}
+
+	return ret;
+}
+
+static inline u8 st7571_transform_xy(const char *p, int x, int y, u8 bpp)
+{
+	int xrest = x % 8;
+	u8 result = 0;
+	u8 row_len = 16 * bpp;
+
+	/*
+	 * Transforms an (x, y) pixel coordinate into a vertical 8-bit
+	 * column from the framebuffer. It calculates the corresponding byte in the
+	 * framebuffer, extracts the bit at the given x position across 8 consecutive
+	 * rows, and packs those bits into a single byte.
+	 *
+	 * Return an 8-bit value representing a vertical column of pixels.
+	 */
+	x = x / 8;
+	y = (y / 8) * 8;
+
+	for (int i = 0; i < 8; i++) {
+		int row_idx = y + i;
+		u8 byte = p[row_idx * row_len + x];
+		u8 bit = (byte >> xrest) & 1;
+
+		result |= (bit << i);
+	}
+
+	return result;
+}
+
+static int st7571_set_position(struct st7571_device *st7571, int x, int y)
+{
+	u8 cmd_list[] = {
+		ST7571_SET_COLUMN_LSB(x),
+		ST7571_SET_COLUMN_MSB(x),
+		ST7571_SET_PAGE(y / ST7571_PAGE_HEIGHT),
+	};
+
+	return st7571_send_command_list(st7571, cmd_list, ARRAY_SIZE(cmd_list));
+}
+
+static int st7571_fb_clear_screen(struct st7571_device *st7571)
+{
+	u32 npixels = st7571->ncols * round_up(st7571->nlines, ST7571_PAGE_HEIGHT) * st7571->bpp;
+	char pixelvalue = 0x00;
+
+	st7571_set_position(st7571, 0, 0);
+	for (int i = 0; i < npixels; i++)
+		regmap_bulk_write(st7571->regmap, ST7571_DATA_MODE, &pixelvalue, 1);
+
+	return 0;
+}
+
+static void st7571_prepare_buffer_monochrome(struct st7571_device *st7571,
+					     const struct iosys_map *vmap,
+					     struct drm_framebuffer *fb,
+					     struct drm_rect *rect,
+					     struct drm_format_conv_state *fmtcnv_state)
+{
+	unsigned int dst_pitch;
+	struct iosys_map dst;
+	u32 size;
+
+	switch (fb->format->format) {
+	case DRM_FORMAT_XRGB8888:
+		dst_pitch = DIV_ROUND_UP(drm_rect_width(rect), 8);
+		iosys_map_set_vaddr(&dst, st7571->hwbuf);
+
+		drm_fb_xrgb8888_to_mono(&dst, &dst_pitch, vmap, fb, rect, fmtcnv_state);
+		break;
+
+	case DRM_FORMAT_R1:
+		size = (rect->x2 - rect->x1) * (rect->y2 - rect->y1) / 8;
+		memcpy(st7571->hwbuf, vmap->vaddr, size);
+		break;
+	}
+}
+
+static void st7571_prepare_buffer_grayscale(struct st7571_device *st7571,
+					    const struct iosys_map *vmap,
+					    struct drm_framebuffer *fb,
+					    struct drm_rect *rect,
+					    struct drm_format_conv_state *fmtcnv_state)
+{
+	u32 size = (rect->x2 - rect->x1) * (rect->y2 - rect->y1) / 8;
+	unsigned int dst_pitch;
+	struct iosys_map dst;
+
+	switch (fb->format->format) {
+	case DRM_FORMAT_XRGB8888:
+		dst_pitch = DIV_ROUND_UP(drm_rect_width(rect), 4);
+		iosys_map_set_vaddr(&dst, st7571->hwbuf);
+
+		drm_fb_xrgb8888_to_gray2(&dst, &dst_pitch, vmap, fb, rect, fmtcnv_state);
+		break;
+
+	case DRM_FORMAT_R1:
+		size = (rect->x2 - rect->x1) * (rect->y2 - rect->y1) / 8;
+		memcpy(st7571->hwbuf, vmap->vaddr, size);
+		break;
+
+	case DRM_FORMAT_R2:
+		size = (rect->x2 - rect->x1) * (rect->y2 - rect->y1) / 4;
+		memcpy(st7571->hwbuf, vmap->vaddr, size);
+		break;
+	}
+}
+
+static int st7571_fb_update_rect_monochrome(struct drm_framebuffer *fb, struct drm_rect *rect)
+{
+	struct st7571_device *st7571 = drm_to_st7571(fb->dev);
+	char *row = st7571->row;
+
+	/* Align y to display page boundaries */
+	rect->y1 = round_down(rect->y1, ST7571_PAGE_HEIGHT);
+	rect->y2 = min_t(unsigned int, round_up(rect->y2, ST7571_PAGE_HEIGHT), st7571->nlines);
+
+	for (int y = rect->y1; y < rect->y2; y += ST7571_PAGE_HEIGHT) {
+		for (int x = rect->x1; x < rect->x2; x++)
+			row[x] = st7571_transform_xy(st7571->hwbuf, x, y, 1);
+
+		st7571_set_position(st7571, rect->x1, y);
+
+		/* TODO: Investige why we can't write multiple bytes at once */
+		for (int x = rect->x1; x < rect->x2; x++)
+			regmap_bulk_write(st7571->regmap, ST7571_DATA_MODE, row + x, 1);
+	}
+
+	return 0;
+}
+
+static int st7571_fb_update_rect_grayscale(struct drm_framebuffer *fb, struct drm_rect *rect)
+{
+	struct st7571_device *st7571 = drm_to_st7571(fb->dev);
+	u32 format = fb->format->format;
+	char *row = st7571->row;
+	int x1;
+	int x2;
+
+	/* Align y to display page boundaries */
+	rect->y1 = round_down(rect->y1, ST7571_PAGE_HEIGHT);
+	rect->y2 = min_t(unsigned int, round_up(rect->y2, ST7571_PAGE_HEIGHT), st7571->nlines);
+
+	switch (format) {
+	case DRM_FORMAT_R1:
+		x1 = rect->x1 * 1;
+		x2 = rect->x2 * 1;
+		break;
+	case DRM_FORMAT_R2:
+		fallthrough;
+	case DRM_FORMAT_XRGB8888:
+		x1 = rect->x1 * 2;
+		x2 = rect->x2 * 2;
+		break;
+	}
+
+	for (int y = rect->y1; y < rect->y2; y += ST7571_PAGE_HEIGHT) {
+		for (int x = x1; x < x2; x++)
+			row[x] = st7571_transform_xy(st7571->hwbuf, x, y, 2);
+
+		st7571_set_position(st7571, rect->x1, y);
+
+		/* TODO: Investige why we can't write multiple bytes at once */
+		for (int x = x1; x < x2; x++) {
+			regmap_bulk_write(st7571->regmap, ST7571_DATA_MODE, row + x, 1);
+
+			/*
+			 * As the display supports grayscale, all pixels must be written as two bits
+			 * even if the format is monochrome.
+			 *
+			 * The bit values maps to the following grayscale:
+			 * 0 0 = Black
+			 * 0 1 = Dark gray
+			 * 1 0 = Light gray
+			 * 1 1 = White
+			 *
+			 * For monochrome formats, write the same value twice to get
+			 * either a black or white pixel.
+			 */
+			if (format == DRM_FORMAT_R1)
+				regmap_bulk_write(st7571->regmap, ST7571_DATA_MODE, row + x, 1);
+		}
+	}
+
+	return 0;
+}
+
+static int st7571_connector_get_modes(struct drm_connector *conn)
+{
+	struct st7571_device *st7571 = drm_to_st7571(conn->dev);
+
+	return drm_connector_helper_get_modes_fixed(conn, &st7571->mode);
+}
+
+static const struct drm_connector_helper_funcs st7571_connector_helper_funcs = {
+	.get_modes = st7571_connector_get_modes,
+};
+
+static const struct st7571_panel_format st7571_monochrome = {
+	.prepare_buffer = st7571_prepare_buffer_monochrome,
+	.update_rect = st7571_fb_update_rect_monochrome,
+	.mode = ST7571_COLOR_MODE_BLACKWHITE,
+	.formats = {
+		DRM_FORMAT_XRGB8888,
+		DRM_FORMAT_R1,
+	},
+	.nformats = 2,
+};
+
+static const struct st7571_panel_format st7571_grayscale = {
+	.prepare_buffer = st7571_prepare_buffer_grayscale,
+	.update_rect = st7571_fb_update_rect_grayscale,
+	.mode = ST7571_COLOR_MODE_GRAY,
+	.formats = {
+		DRM_FORMAT_XRGB8888,
+		DRM_FORMAT_R1,
+		DRM_FORMAT_R2,
+	},
+	.nformats = 3,
+};
+
+static const u64 st7571_primary_plane_fmtmods[] = {
+	DRM_FORMAT_MOD_LINEAR,
+	DRM_FORMAT_MOD_INVALID
+};
+
+static int st7571_primary_plane_helper_atomic_check(struct drm_plane *plane,
+						    struct drm_atomic_state *state)
+{
+	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
+	struct drm_crtc *new_crtc = new_plane_state->crtc;
+	struct drm_crtc_state *new_crtc_state = NULL;
+
+	if (new_crtc)
+		new_crtc_state = drm_atomic_get_new_crtc_state(state, new_crtc);
+
+	return drm_atomic_helper_check_plane_state(new_plane_state, new_crtc_state,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   false, false);
+}
+
+static void st7571_primary_plane_helper_atomic_update(struct drm_plane *plane,
+						      struct drm_atomic_state *state)
+{
+	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
+	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
+	struct drm_framebuffer *fb = plane_state->fb;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_device *drm = plane->dev;
+	struct drm_rect damage;
+	struct st7571_device *st7571 = drm_to_st7571(plane->dev);
+	int ret, idx;
+
+	if (!fb)
+		return; /* no framebuffer; plane is disabled */
+
+	ret = drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE);
+	if (ret)
+		return;
+
+	if (!drm_dev_enter(drm, &idx))
+		goto out_drm_gem_fb_end_cpu_access;
+
+	drm_atomic_helper_damage_iter_init(&iter, old_plane_state, plane_state);
+	drm_atomic_for_each_plane_damage(&iter, &damage) {
+		st7571->pformat->prepare_buffer(st7571,
+						&shadow_plane_state->data[0],
+						fb, &damage,
+						&shadow_plane_state->fmtcnv_state);
+
+		st7571->pformat->update_rect(fb, &damage);
+	}
+
+	drm_dev_exit(idx);
+
+out_drm_gem_fb_end_cpu_access:
+	drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE);
+}
+
+static void st7571_primary_plane_helper_atomic_disable(struct drm_plane *plane,
+						       struct drm_atomic_state *state)
+{
+	struct drm_device *drm = plane->dev;
+	struct st7571_device *st7571 = drm_to_st7571(plane->dev);
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
+
+	st7571_fb_clear_screen(st7571);
+	drm_dev_exit(idx);
+}
+
+static const struct drm_plane_helper_funcs st7571_primary_plane_helper_funcs = {
+	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
+	.atomic_check = st7571_primary_plane_helper_atomic_check,
+	.atomic_update = st7571_primary_plane_helper_atomic_update,
+	.atomic_disable = st7571_primary_plane_helper_atomic_disable,
+};
+
+static const struct drm_plane_funcs st7571_primary_plane_funcs = {
+	.update_plane = drm_atomic_helper_update_plane,
+	.disable_plane = drm_atomic_helper_disable_plane,
+	.destroy = drm_plane_cleanup,
+	DRM_GEM_SHADOW_PLANE_FUNCS,
+};
+
+/*
+ * CRTC
+ */
+
+static enum drm_mode_status st7571_crtc_mode_valid(struct drm_crtc *crtc,
+						   const struct drm_display_mode *mode)
+{
+	struct st7571_device *st7571 = drm_to_st7571(crtc->dev);
+
+	return drm_crtc_helper_mode_valid_fixed(crtc, mode, &st7571->mode);
+}
+
+static const struct drm_crtc_helper_funcs st7571_crtc_helper_funcs = {
+	.atomic_check = drm_crtc_helper_atomic_check,
+	.mode_valid = st7571_crtc_mode_valid,
+};
+
+static const struct drm_crtc_funcs st7571_crtc_funcs = {
+	.reset = drm_atomic_helper_crtc_reset,
+	.destroy = drm_crtc_cleanup,
+	.set_config = drm_atomic_helper_set_config,
+	.page_flip = drm_atomic_helper_page_flip,
+	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
+};
+
+/*
+ * Encoder
+ */
+
+static void st7571_encoder_atomic_enable(struct drm_encoder *encoder,
+					 struct drm_atomic_state *state)
+{
+	struct drm_device *drm = encoder->dev;
+	struct st7571_device *st7571 = drm_to_st7571(drm);
+	u8 command = ST7571_DISPLAY_ON;
+	int ret;
+
+	ret = st7571->pdata->init(st7571);
+	if (ret)
+		return;
+
+	st7571_send_command_list(st7571, &command, 1);
+}
+
+static void st7571_encoder_atomic_disable(struct drm_encoder *encoder,
+					  struct drm_atomic_state *state)
+{
+	struct drm_device *drm = encoder->dev;
+	struct st7571_device *st7571 = drm_to_st7571(drm);
+	u8 command = ST7571_DISPLAY_OFF;
+
+	st7571_send_command_list(st7571, &command, 1);
+}
+
+static const struct drm_encoder_funcs st7571_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+
+};
+
+static const struct drm_encoder_helper_funcs st7571_encoder_helper_funcs = {
+	.atomic_enable = st7571_encoder_atomic_enable,
+	.atomic_disable = st7571_encoder_atomic_disable,
+};
+
+/*
+ * Connector
+ */
+
+static const struct drm_connector_funcs st7571_connector_funcs = {
+	.reset = drm_atomic_helper_connector_reset,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = drm_connector_cleanup,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static const struct drm_mode_config_funcs st7571_mode_config_funcs = {
+	.fb_create = drm_gem_fb_create_with_dirty,
+	.atomic_check = drm_atomic_helper_check,
+	.atomic_commit = drm_atomic_helper_commit,
+};
+
+static struct drm_display_mode st7571_mode(struct st7571_device *st7571)
+{
+	struct drm_display_mode mode = {
+		DRM_SIMPLE_MODE(st7571->ncols, st7571->nlines,
+				st7571->width_mm, st7571->height_mm),
+	};
+
+	return mode;
+}
+
+static int st7571_mode_config_init(struct st7571_device *st7571)
+{
+	struct drm_device *drm = &st7571->drm;
+	const struct st7571_panel_constraints *constraints = &st7571->pdata->constraints;
+	int ret;
+
+	ret = drmm_mode_config_init(drm);
+	if (ret)
+		return ret;
+
+	drm->mode_config.min_width = constraints->min_ncols;
+	drm->mode_config.min_height = constraints->min_nlines;
+	drm->mode_config.max_width = constraints->max_ncols;
+	drm->mode_config.max_height = constraints->max_nlines;
+	drm->mode_config.preferred_depth = 24;
+	drm->mode_config.funcs = &st7571_mode_config_funcs;
+
+	return 0;
+}
+
+static int st7571_plane_init(struct st7571_device *st7571,
+			     const struct st7571_panel_format *pformat)
+{
+	struct drm_plane *primary_plane = &st7571->primary_plane;
+	struct drm_device *drm = &st7571->drm;
+	int ret;
+
+	ret = drm_universal_plane_init(drm, primary_plane, 0,
+				       &st7571_primary_plane_funcs,
+				       pformat->formats,
+				       pformat->nformats,
+				       st7571_primary_plane_fmtmods,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret)
+		return ret;
+
+	drm_plane_helper_add(primary_plane, &st7571_primary_plane_helper_funcs);
+	drm_plane_enable_fb_damage_clips(primary_plane);
+
+	return 0;
+}
+
+static int st7571_crtc_init(struct st7571_device *st7571)
+{
+	struct drm_plane *primary_plane = &st7571->primary_plane;
+	struct drm_crtc *crtc = &st7571->crtc;
+	struct drm_device *drm = &st7571->drm;
+	int ret;
+
+	ret = drm_crtc_init_with_planes(drm, crtc, primary_plane, NULL,
+					&st7571_crtc_funcs, NULL);
+	if (ret)
+		return ret;
+
+	drm_crtc_helper_add(crtc, &st7571_crtc_helper_funcs);
+
+	return 0;
+}
+
+static int st7571_encoder_init(struct st7571_device *st7571)
+{
+	struct drm_encoder *encoder = &st7571->encoder;
+	struct drm_crtc *crtc = &st7571->crtc;
+	struct drm_device *drm = &st7571->drm;
+	int ret;
+
+	ret = drm_encoder_init(drm, encoder, &st7571_encoder_funcs, DRM_MODE_ENCODER_NONE, NULL);
+	if (ret)
+		return ret;
+
+	drm_encoder_helper_add(encoder, &st7571_encoder_helper_funcs);
+
+	encoder->possible_crtcs = drm_crtc_mask(crtc);
+
+	return 0;
+}
+
+static int st7571_connector_init(struct st7571_device *st7571)
+{
+	struct drm_connector *connector = &st7571->connector;
+	struct drm_encoder *encoder = &st7571->encoder;
+	struct drm_device *drm = &st7571->drm;
+	int ret;
+
+	ret = drm_connector_init(drm, connector, &st7571_connector_funcs,
+				 DRM_MODE_CONNECTOR_Unknown);
+	if (ret)
+		return ret;
+
+	drm_connector_helper_add(connector, &st7571_connector_helper_funcs);
+
+	return drm_connector_attach_encoder(connector, encoder);
+}
+
+DEFINE_DRM_GEM_FOPS(st7571_fops);
+
+static const struct drm_driver st7571_driver = {
+	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC,
+
+	.name		 = DRIVER_NAME,
+	.desc		 = DRIVER_DESC,
+	.major		 = DRIVER_MAJOR,
+	.minor		 = DRIVER_MINOR,
+
+	.fops		 = &st7571_fops,
+	DRM_GEM_SHMEM_DRIVER_OPS,
+	DRM_FBDEV_SHMEM_DRIVER_OPS,
+};
+
+static int st7571_validate_parameters(struct st7571_device *st7571)
+{
+	struct device *dev = st7571->dev;
+	const struct st7571_panel_constraints *constraints = &st7571->pdata->constraints;
+
+	if (st7571->width_mm  == 0) {
+		dev_err(dev, "Invalid panel width\n");
+		return -EINVAL;
+	}
+
+	if (st7571->height_mm == 0) {
+		dev_err(dev, "Invalid panel height\n");
+		return -EINVAL;
+	}
+
+	if (st7571->nlines < constraints->min_nlines ||
+	    st7571->nlines > constraints->max_nlines) {
+		dev_err(dev, "Invalid timing configuration.\n");
+		return -EINVAL;
+	}
+
+	if (st7571->startline + st7571->nlines > constraints->max_nlines) {
+		dev_err(dev, "Invalid timing configuration.\n");
+		return -EINVAL;
+	}
+
+	if (st7571->ncols < constraints->min_ncols ||
+	    st7571->ncols > constraints->max_ncols) {
+		dev_err(dev, "Invalid timing configuration.\n");
+		return -EINVAL;
+	}
+
+	if (st7571->grayscale && !constraints->support_grayscale) {
+		dev_err(dev, "Grayscale not supported\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int st7567_parse_dt(struct st7571_device *st7567)
+{
+	struct device *dev = st7567->dev;
+	struct device_node *np = dev->of_node;
+	struct display_timing dt;
+	int ret;
+
+	ret = of_get_display_timing(np, "panel-timing", &dt);
+	if (ret) {
+		dev_err(dev, "Failed to get display timing from DT\n");
+		return ret;
+	}
+
+	of_property_read_u32(np, "width-mm", &st7567->width_mm);
+	of_property_read_u32(np, "height-mm", &st7567->height_mm);
+	st7567->inverted = of_property_read_bool(np, "sitronix,inverted");
+
+	st7567->pformat = &st7571_monochrome;
+	st7567->bpp = 1;
+
+	st7567->startline = dt.vfront_porch.typ;
+	st7567->nlines = dt.vactive.typ;
+	st7567->ncols = dt.hactive.typ;
+
+	return 0;
+}
+
+static int st7571_parse_dt(struct st7571_device *st7571)
+{
+	struct device *dev = st7571->dev;
+	struct device_node *np = dev->of_node;
+	struct display_timing dt;
+	int ret;
+
+	ret = of_get_display_timing(np, "panel-timing", &dt);
+	if (ret) {
+		dev_err(dev, "Failed to get display timing from DT\n");
+		return ret;
+	}
+
+	of_property_read_u32(np, "width-mm", &st7571->width_mm);
+	of_property_read_u32(np, "height-mm", &st7571->height_mm);
+	st7571->grayscale = of_property_read_bool(np, "sitronix,grayscale");
+	st7571->inverted = of_property_read_bool(np, "sitronix,inverted");
+
+	if (st7571->grayscale) {
+		st7571->pformat = &st7571_grayscale;
+		st7571->bpp = 2;
+	} else {
+		st7571->pformat = &st7571_monochrome;
+		st7571->bpp = 1;
+	}
+
+	st7571->startline = dt.vfront_porch.typ;
+	st7571->nlines = dt.vactive.typ;
+	st7571->ncols = dt.hactive.typ;
+
+	st7571->reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(st7571->reset))
+		return dev_err_probe(dev, PTR_ERR(st7571->reset),
+				     "Failed to get reset gpio\n");
+
+	return 0;
+}
+
+static void st7571_reset(struct st7571_device *st7571)
+{
+	gpiod_set_value_cansleep(st7571->reset, 1);
+	fsleep(20);
+	gpiod_set_value_cansleep(st7571->reset, 0);
+}
+
+static int st7567_lcd_init(struct st7571_device *st7567)
+{
+	/*
+	 * Most of the initialization sequence is taken directly from the
+	 * referential initial code in the ST7567 datasheet.
+	 */
+	u8 commands[] = {
+		ST7571_DISPLAY_OFF,
+
+		ST7567_SET_LCD_BIAS(1),
+
+		ST7571_SET_SEG_SCAN_DIR(0),
+		ST7571_SET_COM_SCAN_DIR(1),
+
+		ST7571_SET_REGULATOR_REG(4),
+		ST7571_SET_CONTRAST_MSB,
+		ST7571_SET_CONTRAST_LSB(0x20),
+
+		ST7571_SET_START_LINE_MSB,
+		ST7571_SET_START_LINE_LSB(st7567->startline),
+
+		ST7571_SET_POWER(0x4),	/* Power Control, VC: ON, VR: OFF, VF: OFF */
+		ST7571_SET_POWER(0x6),	/* Power Control, VC: ON, VR: ON, VF: OFF */
+		ST7571_SET_POWER(0x7),	/* Power Control, VC: ON, VR: ON, VF: ON */
+
+		ST7571_SET_REVERSE(st7567->inverted ? 1 : 0),
+		ST7571_SET_ENTIRE_DISPLAY_ON(0),
+	};
+
+	return st7571_send_command_list(st7567, commands, ARRAY_SIZE(commands));
+}
+
+static int st7571_lcd_init(struct st7571_device *st7571)
+{
+	/*
+	 * Most of the initialization sequence is taken directly from the
+	 * referential initial code in the ST7571 datasheet.
+	 */
+	u8 commands[] = {
+		ST7571_DISPLAY_OFF,
+
+		ST7571_SET_MODE_MSB,
+		ST7571_SET_MODE_LSB(0x2e),
+
+		ST7571_SET_SEG_SCAN_DIR(0),
+		ST7571_SET_COM_SCAN_DIR(1),
+
+		ST7571_SET_COM0_MSB,
+		ST7571_SET_COM0_LSB(0x00),
+
+		ST7571_SET_START_LINE_MSB,
+		ST7571_SET_START_LINE_LSB(st7571->startline),
+
+		ST7571_OSC_ON,
+		ST7571_SET_REGULATOR_REG(5),
+		ST7571_SET_CONTRAST_MSB,
+		ST7571_SET_CONTRAST_LSB(0x33),
+		ST7571_SET_LCD_BIAS(0x04),
+		ST7571_SET_DISPLAY_DUTY_MSB,
+		ST7571_SET_DISPLAY_DUTY_LSB(st7571->nlines),
+
+		ST7571_SET_POWER(0x4),	/* Power Control, VC: ON, VR: OFF, VF: OFF */
+		ST7571_SET_POWER(0x6),	/* Power Control, VC: ON, VR: ON, VF: OFF */
+		ST7571_SET_POWER(0x7),	/* Power Control, VC: ON, VR: ON, VF: ON */
+
+		ST7571_COMMAND_SET_3,
+		ST7571_SET_COLOR_MODE(st7571->pformat->mode),
+		ST7571_COMMAND_SET_NORMAL,
+
+		ST7571_SET_REVERSE(st7571->inverted ? 1 : 0),
+		ST7571_SET_ENTIRE_DISPLAY_ON(0),
+	};
+
+	/* Perform a reset before initializing the controller */
+	st7571_reset(st7571);
+
+	return st7571_send_command_list(st7571, commands, ARRAY_SIZE(commands));
+}
+
+struct st7571_device *st7571_probe(struct device *dev,
+				   struct regmap *regmap)
+{
+	struct st7571_device *st7571;
+	struct drm_device *drm;
+	int ret;
+
+	st7571 = devm_drm_dev_alloc(dev, &st7571_driver,
+				    struct st7571_device, drm);
+	if (IS_ERR(st7571))
+		return st7571;
+
+	drm = &st7571->drm;
+	st7571->dev = dev;
+	st7571->pdata = device_get_match_data(st7571->dev);
+
+	ret = st7571->pdata->parse_dt(st7571);
+	if (ret)
+		return ERR_PTR(ret);
+
+	ret = st7571_validate_parameters(st7571);
+	if (ret)
+		return ERR_PTR(ret);
+
+	st7571->mode = st7571_mode(st7571);
+	st7571->regmap = regmap;
+
+	st7571->hwbuf = devm_kzalloc(st7571->dev,
+				     (st7571->nlines * st7571->ncols * st7571->bpp) / 8,
+				     GFP_KERNEL);
+	if (!st7571->hwbuf)
+		return ERR_PTR(-ENOMEM);
+
+	st7571->row = devm_kzalloc(st7571->dev,
+				   (st7571->ncols * st7571->bpp),
+				   GFP_KERNEL);
+	if (!st7571->row)
+		return ERR_PTR(-ENOMEM);
+
+	ret = st7571_mode_config_init(st7571);
+	if (ret) {
+		dev_err(st7571->dev, "Failed to initialize mode config\n");
+		return ERR_PTR(ret);
+	}
+
+	ret = st7571_plane_init(st7571, st7571->pformat);
+	if (ret) {
+		dev_err(st7571->dev, "Failed to initialize primary plane\n");
+		return ERR_PTR(ret);
+	}
+
+	ret = st7571_crtc_init(st7571);
+	if (ret < 0) {
+		dev_err(st7571->dev, "Failed to initialize CRTC\n");
+		return ERR_PTR(ret);
+	}
+
+	ret = st7571_encoder_init(st7571);
+	if (ret < 0) {
+		dev_err(st7571->dev, "Failed to initialize encoder\n");
+		return ERR_PTR(ret);
+	}
+
+	ret = st7571_connector_init(st7571);
+	if (ret < 0) {
+		dev_err(st7571->dev, "Failed to initialize connector\n");
+		return ERR_PTR(ret);
+	}
+
+	drm_mode_config_reset(drm);
+
+	ret = drm_dev_register(drm, 0);
+	if (ret) {
+		dev_err(st7571->dev, "Failed to register DRM device\n");
+		return ERR_PTR(ret);
+	}
+
+	drm_client_setup(drm, NULL);
+	return st7571;
+}
+EXPORT_SYMBOL_GPL(st7571_probe);
+
+void st7571_remove(struct st7571_device *st7571)
+{
+	drm_dev_unplug(&st7571->drm);
+}
+EXPORT_SYMBOL_GPL(st7571_remove);
+
+const struct st7571_panel_data st7567_config = {
+	.init = st7567_lcd_init,
+	.parse_dt = st7567_parse_dt,
+	.constraints = {
+		.min_nlines = 1,
+		.max_nlines = 64,
+		.min_ncols = 128,
+		.max_ncols = 128,
+		.support_grayscale = false,
+	},
+};
+EXPORT_SYMBOL_NS_GPL(st7567_config, "DRM_ST7571");
+
+const struct st7571_panel_data st7571_config = {
+	.init = st7571_lcd_init,
+	.parse_dt = st7571_parse_dt,
+	.constraints = {
+		.min_nlines = 1,
+		.max_nlines = 128,
+		.min_ncols = 128,
+		.max_ncols = 128,
+		.support_grayscale = true,
+	},
+};
+EXPORT_SYMBOL_NS_GPL(st7571_config, "DRM_ST7571");
+
+MODULE_AUTHOR("Marcus Folkesson <marcus.folkesson@gmail.com>");
+MODULE_DESCRIPTION("DRM Driver for Sitronix ST7571 LCD controller");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/sitronix/st7571.h b/drivers/gpu/drm/sitronix/st7571.h
new file mode 100644
index 000000000000..af264f2e2ea4
--- /dev/null
+++ b/drivers/gpu/drm/sitronix/st7571.h
@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Header file for:
+ * Driver for Sitronix ST7571, a 4 level gray scale dot matrix LCD controller
+ *
+ * Copyright (C) 2025 Marcus Folkesson <marcus.folkesson@gmail.com>
+ */
+
+#ifndef __ST7571_H__
+#define __ST7571_H__
+
+#include <drm/drm_connector.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_encoder.h>
+#include <drm/drm_format_helper.h>
+
+#include <linux/regmap.h>
+
+enum st7571_color_mode {
+	ST7571_COLOR_MODE_GRAY = 0,
+	ST7571_COLOR_MODE_BLACKWHITE = 1,
+};
+
+struct st7571_device;
+
+struct st7571_panel_constraints {
+	u32 min_nlines;
+	u32 max_nlines;
+	u32 min_ncols;
+	u32 max_ncols;
+	bool support_grayscale;
+};
+
+struct st7571_panel_data {
+	int (*init)(struct st7571_device *st7571);
+	int (*parse_dt)(struct st7571_device *st7571);
+	struct st7571_panel_constraints constraints;
+};
+
+struct st7571_panel_format {
+	void (*prepare_buffer)(struct st7571_device *st7571,
+			       const struct iosys_map *vmap,
+			       struct drm_framebuffer *fb,
+			       struct drm_rect *rect,
+			       struct drm_format_conv_state *fmtcnv_state);
+	int (*update_rect)(struct drm_framebuffer *fb, struct drm_rect *rect);
+	enum st7571_color_mode mode;
+	const u8 nformats;
+	const u32 formats[];
+};
+
+struct st7571_device {
+	struct drm_device drm;
+	struct device *dev;
+
+	struct drm_plane primary_plane;
+	struct drm_crtc crtc;
+	struct drm_encoder encoder;
+	struct drm_connector connector;
+
+	struct drm_display_mode mode;
+
+	const struct st7571_panel_format *pformat;
+	const struct st7571_panel_data *pdata;
+	struct gpio_desc *reset;
+	struct regmap *regmap;
+
+	bool grayscale;
+	bool inverted;
+	u32 height_mm;
+	u32 width_mm;
+	u32 startline;
+	u32 nlines;
+	u32 ncols;
+	u32 bpp;
+
+	/* Intermediate buffer in LCD friendly format */
+	u8 *hwbuf;
+
+	/* Row of (transformed) pixels ready to be written to the display */
+	u8 *row;
+};
+
+extern const struct st7571_panel_data st7567_config;
+extern const struct st7571_panel_data st7571_config;
+
+struct st7571_device *st7571_probe(struct device *dev, struct regmap *regmap);
+void st7571_remove(struct st7571_device *st7571);
+
+#endif /* __ST7571_H__ */
diff --git a/drivers/gpu/drm/sitronix/st7920.c b/drivers/gpu/drm/sitronix/st7920.c
new file mode 100644
index 000000000000..f35a157fdad8
--- /dev/null
+++ b/drivers/gpu/drm/sitronix/st7920.c
@@ -0,0 +1,867 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * DRM driver for Sitronix ST7920 LCD displays
+ *
+ * Copyright 2025 Iker Pedrosa <ikerpedrosam@gmail.com>
+ *
+ */
+
+#include <linux/bitrev.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/spi/spi.h>
+
+#include <drm/clients/drm_client_setup.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_damage_helper.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_fbdev_shmem.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_gem_shmem_helper.h>
+#include <drm/drm_plane.h>
+#include <drm/drm_print.h>
+#include <drm/drm_probe_helper.h>
+
+#define DRIVER_NAME	"sitronix_st7920"
+#define DRIVER_DESC	"DRM driver for Sitronix ST7920 LCD displays"
+#define DRIVER_MAJOR	1
+#define DRIVER_MINOR	0
+
+/* Display organization */
+#define ST7920_PITCH		16
+#define ST7920_SCANLINES	64
+#define BYTES_IN_DISPLAY	(ST7920_PITCH * ST7920_SCANLINES)
+#define BYTES_IN_SEGMENT	2
+#define PIXELS_PER_SEGMENT	(BYTES_IN_SEGMENT * 8)
+#define ST7920_DEFAULT_WIDTH	128
+#define ST7920_DEFAULT_HEIGHT	64
+
+/* Sync sequence */
+#define SYNC_BITS			0xF8
+#define RW_HIGH				0x04
+#define RS_HIGH				0x02
+
+/* Commands */
+#define SET_DISPLAY_ON			0x0C
+#define SET_DISPLAY_OFF			0x08
+#define SET_DISPLAY_CLEAR		0x01
+#define SET_BASIC_INSTRUCTION_SET	0x30
+#define SET_EXT_INSTRUCTION_SET		0x34
+#define SET_GRAPHICS_DISPLAY		0x36
+#define SET_GDRAM_ADDRESS		0x80
+#define SET_GDRAM_DATA			0xFF /* Driver internal command */
+
+/* Masks */
+#define HIGH_DATA_MASK			0xF0
+#define LOW_DATA_MASK			0x0F
+#define TOP_VERTICAL_ADDRESS		0x80
+#define BOTTOM_VERTICAL_ADDRESS		0x60
+#define TOP_HORIZONTAL_ADDRESS		0x00
+#define BOTTOM_HORIZONTAL_ADDRESS	0x80
+
+#define CMD_SIZE			35
+
+struct spi7920_error {
+	int errno;
+};
+
+struct st7920_device {
+	struct drm_device drm;
+	struct drm_display_mode mode;
+	struct drm_plane primary_plane;
+	struct drm_crtc crtc;
+	struct drm_encoder encoder;
+	struct drm_connector connector;
+	struct spi_device *spi;
+
+	struct regmap *regmap;
+
+	struct gpio_desc *reset_gpio;
+
+	u32 height;
+	u32 width;
+};
+
+struct st7920_plane_state {
+	struct drm_shadow_plane_state base;
+	/* Intermediate buffer to convert pixels from XRGB8888 to HW format */
+	u8 *buffer;
+};
+
+struct st7920_crtc_state {
+	struct drm_crtc_state base;
+	/* Buffer to store pixels in HW format and written to the panel */
+	u8 *data_array;
+};
+
+static inline struct st7920_plane_state *to_st7920_plane_state(struct drm_plane_state *state)
+{
+	return container_of(state, struct st7920_plane_state, base.base);
+}
+
+static inline struct st7920_crtc_state *to_st7920_crtc_state(struct drm_crtc_state *state)
+{
+	return container_of(state, struct st7920_crtc_state, base);
+}
+
+static inline struct st7920_device *drm_to_st7920(struct drm_device *drm)
+{
+	return container_of(drm, struct st7920_device, drm);
+}
+
+static int st7920_store_gdram_address(const void *data, u8 *reg)
+{
+	const u8 y_addr = *(const u8 *)data;
+	bool bottom_screen = (y_addr >= 32);
+	int i = 0;
+
+	reg[i++] = SYNC_BITS;
+	/* Set vertical address */
+	if (!bottom_screen)
+		reg[i++] = TOP_VERTICAL_ADDRESS + (*(uint8_t *)data & HIGH_DATA_MASK);
+	else
+		reg[i++] = BOTTOM_VERTICAL_ADDRESS + (*(uint8_t *)data & HIGH_DATA_MASK);
+
+	reg[i++] = *(uint8_t *)data << 4;
+	/* Set horizontal address */
+	reg[i++] = SET_GDRAM_ADDRESS;
+	if (!bottom_screen)
+		reg[i++] = TOP_HORIZONTAL_ADDRESS;
+	else
+		reg[i++] = BOTTOM_HORIZONTAL_ADDRESS;
+
+	return i;
+}
+
+static int st7920_store_gdram_data(const void *data, u8 *reg)
+{
+	const u8 *line_data = data;
+	int i = 0, j = 0;
+
+	reg[i++] = SYNC_BITS | RS_HIGH;
+
+	for (j = 0; j < 16; j++) {
+		reg[i++] = line_data[j] & 0xF0;
+		reg[i++] = (line_data[j] << 4) & 0xF0;
+	}
+
+	return i;
+}
+
+static int st7920_store_others(int cmd, const void *data, u8 *reg)
+{
+	int i = 0;
+
+	reg[i++] = SYNC_BITS;
+	reg[i++] = cmd & HIGH_DATA_MASK;
+	reg[i++] = (cmd & LOW_DATA_MASK) << 4;
+
+	return i;
+}
+
+static void st7920_spi_write(struct spi_device *spi, int cmd, const void *data,
+			     int delay_us, struct spi7920_error *err)
+{
+	u8 reg[CMD_SIZE] = {0};
+	int size = 0;
+	int ret;
+
+	if (err->errno)
+		return;
+
+	/*
+	 * First the sync bits are sent: 11111WS0.
+	 * Where W is the read/write (RW) bit and S is the register/data (RS) bit.
+	 * Then, every 8 bits instruction/data will be separated into 2 groups.
+	 * Higher 4 bits (DB7~DB4) will be placed in the first section followed by
+	 * 4 '0's. And lower 4 bits (DB3~DB0) will be placed in the second section
+	 * followed by 4 '0's.
+	 */
+	if (cmd == SET_GDRAM_ADDRESS)
+		size = st7920_store_gdram_address(data, reg);
+	else if (cmd == SET_GDRAM_DATA)
+		size = st7920_store_gdram_data(data, reg);
+	else
+		size = st7920_store_others(cmd, data, reg);
+
+	ret = spi_write(spi, reg, size);
+	if (ret) {
+		err->errno = ret;
+		return;
+	}
+
+	if (delay_us)
+		udelay(delay_us);
+}
+
+static const struct regmap_config st7920_spi_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+};
+
+static const struct of_device_id st7920_of_match[] = {
+	/* st7920 family */
+	{
+		.compatible = "sitronix,st7920",
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, st7920_of_match);
+
+/*
+ * The SPI core always reports a MODALIAS uevent of the form "spi:<dev>", even
+ * if the device was registered via OF. This means that the module will not be
+ * auto loaded, unless it contains an alias that matches the MODALIAS reported.
+ *
+ * To workaround this issue, add a SPI device ID table. Even when this should
+ * not be needed for this driver to match the registered SPI devices.
+ */
+static const struct spi_device_id st7920_spi_id[] = {
+	/* st7920 family */
+	{ "st7920",  0 },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(spi, st7920_spi_id);
+
+static void st7920_power_on(struct st7920_device *st7920,
+			    struct spi7920_error *err)
+{
+	st7920_spi_write(st7920->spi, SET_DISPLAY_ON, NULL, 72, err);
+}
+
+static void st7920_power_off(struct st7920_device *st7920,
+			     struct spi7920_error *err)
+{
+	st7920_spi_write(st7920->spi, SET_DISPLAY_CLEAR, NULL, 1600, err);
+	st7920_spi_write(st7920->spi, SET_DISPLAY_OFF, NULL, 72, err);
+}
+
+static void st7920_hw_reset(struct st7920_device *st7920)
+{
+	if (!st7920->reset_gpio)
+		return;
+
+	gpiod_set_value_cansleep(st7920->reset_gpio, 1);
+	usleep_range(15, 20);
+	gpiod_set_value_cansleep(st7920->reset_gpio, 0);
+	msleep(40);
+}
+
+static int st7920_init(struct st7920_device *st7920)
+{
+	struct spi7920_error err = {0};
+
+	st7920_spi_write(st7920->spi, SET_BASIC_INSTRUCTION_SET, NULL, 72, &err);
+	st7920_power_on(st7920, &err);
+	st7920_spi_write(st7920->spi, SET_GRAPHICS_DISPLAY, NULL, 72, &err);
+	st7920_spi_write(st7920->spi, SET_DISPLAY_CLEAR, NULL, 1600, &err);
+
+	return err.errno;
+}
+
+static int st7920_update_rect(struct st7920_device *st7920,
+			      struct drm_rect *rect, u8 *buf,
+			      u8 *data_array)
+{
+	struct spi7920_error err = {0};
+	u32 array_idx = 0;
+	int i, j;
+
+	/*
+	 * The screen is divided in 64(Y)x8(X) segments and each segment is
+	 * further divided in 2 bytes (D15~D0).
+	 *
+	 * Segment 0x0 is in the top-right corner, while segment 63x15 is in the
+	 * bottom-left. They would be displayed in the screen in the following way:
+	 * 0x0  0x1  0x2  ... 0x15
+	 * 1x0  1x1  1x2  ... 1x15
+	 * ...
+	 * 63x0 63x1 63x2 ... 63x15
+	 *
+	 * The data in each byte is big endian.
+	 */
+
+	for (i = 0; i < ST7920_SCANLINES; i++) {
+		u8 *line_start = buf + (i * ST7920_PITCH);
+		u8 line_buffer[ST7920_PITCH];
+
+		for (j = 0; j < ST7920_PITCH; j++) {
+			line_buffer[j] = bitrev8(line_start[j]);
+			data_array[array_idx++] = line_buffer[j];
+		}
+
+		st7920_spi_write(st7920->spi, SET_GDRAM_ADDRESS, &i, 72, &err);
+		st7920_spi_write(st7920->spi, SET_GDRAM_DATA, line_buffer, 72, &err);
+	}
+
+	return err.errno;
+}
+
+static void st7920_clear_screen(struct st7920_device *st7920, u8 *data_array)
+{
+	struct spi7920_error err = {0};
+
+	memset(data_array, 0, BYTES_IN_DISPLAY);
+
+	st7920_spi_write(st7920->spi, SET_DISPLAY_CLEAR, NULL, 1600, &err);
+}
+
+static int st7920_fb_blit_rect(struct drm_framebuffer *fb,
+			       const struct iosys_map *vmap,
+			       struct drm_rect *rect,
+			       u8 *buf, u8 *data_array,
+			       struct drm_format_conv_state *fmtcnv_state)
+{
+	struct st7920_device *st7920 = drm_to_st7920(fb->dev);
+	struct iosys_map dst;
+	unsigned int dst_pitch;
+	int ret;
+
+	/* Align y to display page boundaries */
+	rect->y1 = round_down(rect->y1, PIXELS_PER_SEGMENT);
+	rect->y2 = min_t(unsigned int, round_up(rect->y2, PIXELS_PER_SEGMENT), st7920->height);
+
+	dst_pitch = DIV_ROUND_UP(drm_rect_width(rect), 8);
+
+	iosys_map_set_vaddr(&dst, buf);
+	drm_fb_xrgb8888_to_mono(&dst, &dst_pitch, vmap, fb, rect, fmtcnv_state);
+
+	ret = st7920_update_rect(st7920, rect, buf, data_array);
+
+	return ret;
+}
+
+static int st7920_primary_plane_atomic_check(struct drm_plane *plane,
+					     struct drm_atomic_state *state)
+{
+	struct drm_device *drm = plane->dev;
+	struct st7920_device *st7920 = drm_to_st7920(drm);
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
+	struct st7920_plane_state *st7920_state = to_st7920_plane_state(plane_state);
+	struct drm_shadow_plane_state *shadow_plane_state = &st7920_state->base;
+	struct drm_crtc *crtc = plane_state->crtc;
+	struct drm_crtc_state *crtc_state = NULL;
+	const struct drm_format_info *fi;
+	unsigned int pitch;
+	int ret;
+
+	if (crtc)
+		crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
+
+	ret = drm_atomic_helper_check_plane_state(plane_state, crtc_state,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  false, false);
+	if (ret)
+		return ret;
+	else if (!plane_state->visible)
+		return 0;
+
+	fi = drm_format_info(DRM_FORMAT_R1);
+	if (!fi)
+		return -EINVAL;
+
+	pitch = drm_format_info_min_pitch(fi, 0, st7920->width);
+
+	if (plane_state->fb->format != fi) {
+		void *buf;
+
+		/* format conversion necessary; reserve buffer */
+		buf = drm_format_conv_state_reserve(&shadow_plane_state->fmtcnv_state,
+						    pitch, GFP_KERNEL);
+		if (!buf)
+			return -ENOMEM;
+	}
+
+	st7920_state->buffer = kcalloc(pitch, st7920->height, GFP_KERNEL);
+	if (!st7920_state->buffer)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void st7920_primary_plane_atomic_update(struct drm_plane *plane,
+					       struct drm_atomic_state *state)
+{
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
+	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
+	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, plane_state->crtc);
+	struct st7920_crtc_state *st7920_crtc_state =  to_st7920_crtc_state(crtc_state);
+	struct st7920_plane_state *st7920_plane_state = to_st7920_plane_state(plane_state);
+	struct drm_framebuffer *fb = plane_state->fb;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_device *drm = plane->dev;
+	struct drm_rect dst_clip;
+	struct drm_rect damage;
+	int idx;
+	int ret;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
+
+	if (drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE) == 0) {
+		drm_atomic_helper_damage_iter_init(&iter, old_plane_state, plane_state);
+		drm_atomic_for_each_plane_damage(&iter, &damage) {
+			dst_clip = plane_state->dst;
+
+			if (!drm_rect_intersect(&dst_clip, &damage))
+				continue;
+
+			ret = st7920_fb_blit_rect(fb, &shadow_plane_state->data[0], &dst_clip,
+						  st7920_plane_state->buffer,
+						  st7920_crtc_state->data_array,
+						  &shadow_plane_state->fmtcnv_state);
+			if (ret)
+				drm_err_once(plane->dev, "Failed to write to device: %d.\n", ret);
+		}
+
+		drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE);
+	}
+
+	drm_dev_exit(idx);
+}
+
+static void st7920_primary_plane_atomic_disable(struct drm_plane *plane,
+						struct drm_atomic_state *state)
+{
+	struct drm_device *drm = plane->dev;
+	struct st7920_device *st7920 = drm_to_st7920(drm);
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
+	struct drm_crtc_state *crtc_state;
+	struct st7920_crtc_state *st7920_crtc_state;
+	int idx;
+
+	if (!plane_state->crtc)
+		return;
+
+	crtc_state = drm_atomic_get_new_crtc_state(state, plane_state->crtc);
+	st7920_crtc_state = to_st7920_crtc_state(crtc_state);
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
+
+	st7920_clear_screen(st7920, st7920_crtc_state->data_array);
+
+	drm_dev_exit(idx);
+}
+
+/* Called during init to allocate the plane's atomic state. */
+static void st7920_primary_plane_reset(struct drm_plane *plane)
+{
+	struct st7920_plane_state *st7920_state;
+
+	drm_WARN_ON_ONCE(plane->dev, plane->state);
+
+	st7920_state = kzalloc(sizeof(*st7920_state), GFP_KERNEL);
+	if (!st7920_state)
+		return;
+
+	__drm_gem_reset_shadow_plane(plane, &st7920_state->base);
+}
+
+static struct drm_plane_state *st7920_primary_plane_duplicate_state(struct drm_plane *plane)
+{
+	struct drm_shadow_plane_state *new_shadow_plane_state;
+	struct st7920_plane_state *st7920_state;
+
+	if (drm_WARN_ON_ONCE(plane->dev, !plane->state))
+		return NULL;
+
+	st7920_state = kzalloc(sizeof(*st7920_state), GFP_KERNEL);
+	if (!st7920_state)
+		return NULL;
+
+	new_shadow_plane_state = &st7920_state->base;
+
+	__drm_gem_duplicate_shadow_plane_state(plane, new_shadow_plane_state);
+
+	return &new_shadow_plane_state->base;
+}
+
+static void st7920_primary_plane_destroy_state(struct drm_plane *plane,
+					       struct drm_plane_state *state)
+{
+	struct st7920_plane_state *st7920_state = to_st7920_plane_state(state);
+
+	kfree(st7920_state->buffer);
+
+	__drm_gem_destroy_shadow_plane_state(&st7920_state->base);
+
+	kfree(st7920_state);
+}
+
+static const struct drm_plane_helper_funcs st7920_primary_plane_helper_funcs = {
+	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
+	.atomic_check = st7920_primary_plane_atomic_check,
+	.atomic_update = st7920_primary_plane_atomic_update,
+	.atomic_disable = st7920_primary_plane_atomic_disable,
+};
+
+static const struct drm_plane_funcs st7920_primary_plane_funcs = {
+	.update_plane = drm_atomic_helper_update_plane,
+	.disable_plane = drm_atomic_helper_disable_plane,
+	.reset = st7920_primary_plane_reset,
+	.atomic_duplicate_state = st7920_primary_plane_duplicate_state,
+	.atomic_destroy_state = st7920_primary_plane_destroy_state,
+	.destroy = drm_plane_cleanup,
+};
+
+static enum drm_mode_status st7920_crtc_mode_valid(struct drm_crtc *crtc,
+						   const struct drm_display_mode *mode)
+{
+	struct st7920_device *st7920 = drm_to_st7920(crtc->dev);
+
+	return drm_crtc_helper_mode_valid_fixed(crtc, mode, &st7920->mode);
+}
+
+static int st7920_crtc_atomic_check(struct drm_crtc *crtc,
+				    struct drm_atomic_state *state)
+{
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
+	struct st7920_crtc_state *st7920_state = to_st7920_crtc_state(crtc_state);
+	int ret;
+
+	ret = drm_crtc_helper_atomic_check(crtc, state);
+	if (ret)
+		return ret;
+
+	st7920_state->data_array = kmalloc(BYTES_IN_DISPLAY, GFP_KERNEL);
+	if (!st7920_state->data_array)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void st7920_crtc_atomic_enable(struct drm_crtc *crtc,
+				      struct drm_atomic_state *state)
+{
+	struct drm_device *drm = crtc->dev;
+	struct st7920_device *st7920 = drm_to_st7920(drm);
+	int idx;
+	int ret;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
+
+	st7920_hw_reset(st7920);
+
+	ret = st7920_init(st7920);
+	if (ret)
+		drm_err(drm, "Failed to init hardware: %d\n", ret);
+
+	drm_dev_exit(idx);
+}
+
+static void st7920_crtc_atomic_disable(struct drm_crtc *crtc,
+				       struct drm_atomic_state *state)
+{
+	struct spi7920_error err = {0};
+	struct drm_device *drm = crtc->dev;
+	struct st7920_device *st7920 = drm_to_st7920(drm);
+	int idx;
+
+	drm_dev_enter(drm, &idx);
+
+	st7920_power_off(st7920, &err);
+
+	drm_dev_exit(idx);
+}
+
+/* Called during init to allocate the CRTC's atomic state. */
+static void st7920_crtc_reset(struct drm_crtc *crtc)
+{
+	struct st7920_crtc_state *st7920_state;
+
+	drm_WARN_ON_ONCE(crtc->dev, crtc->state);
+
+	st7920_state = kzalloc(sizeof(*st7920_state), GFP_KERNEL);
+	if (!st7920_state)
+		return;
+
+	__drm_atomic_helper_crtc_reset(crtc, &st7920_state->base);
+}
+
+static struct drm_crtc_state *st7920_crtc_duplicate_state(struct drm_crtc *crtc)
+{
+	struct st7920_crtc_state *st7920_state;
+
+	if (drm_WARN_ON_ONCE(crtc->dev, !crtc->state))
+		return NULL;
+
+	st7920_state = kzalloc(sizeof(*st7920_state), GFP_KERNEL);
+	if (!st7920_state)
+		return NULL;
+
+	__drm_atomic_helper_crtc_duplicate_state(crtc, &st7920_state->base);
+
+	return &st7920_state->base;
+}
+
+static void st7920_crtc_destroy_state(struct drm_crtc *crtc,
+				      struct drm_crtc_state *state)
+{
+	struct st7920_crtc_state *st7920_state = to_st7920_crtc_state(state);
+
+	kfree(st7920_state->data_array);
+
+	__drm_atomic_helper_crtc_destroy_state(state);
+
+	kfree(st7920_state);
+}
+
+/*
+ * The CRTC is always enabled. Screen updates are performed by
+ * the primary plane's atomic_update function. Disabling clears
+ * the screen in the primary plane's atomic_disable function.
+ */
+static const struct drm_crtc_helper_funcs st7920_crtc_helper_funcs = {
+	.mode_valid = st7920_crtc_mode_valid,
+	.atomic_check = st7920_crtc_atomic_check,
+	.atomic_enable = st7920_crtc_atomic_enable,
+	.atomic_disable = st7920_crtc_atomic_disable,
+};
+
+static const struct drm_crtc_funcs st7920_crtc_funcs = {
+	.reset = st7920_crtc_reset,
+	.destroy = drm_crtc_cleanup,
+	.set_config = drm_atomic_helper_set_config,
+	.page_flip = drm_atomic_helper_page_flip,
+	.atomic_duplicate_state = st7920_crtc_duplicate_state,
+	.atomic_destroy_state = st7920_crtc_destroy_state,
+};
+
+static const struct drm_encoder_funcs st7920_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
+static int st7920_connector_get_modes(struct drm_connector *connector)
+{
+	struct st7920_device *st7920 = drm_to_st7920(connector->dev);
+
+	return drm_connector_helper_get_modes_fixed(connector, &st7920->mode);
+}
+
+static const struct drm_connector_helper_funcs st7920_connector_helper_funcs = {
+	.get_modes = st7920_connector_get_modes,
+};
+
+static const struct drm_connector_funcs st7920_connector_funcs = {
+	.reset = drm_atomic_helper_connector_reset,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = drm_connector_cleanup,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static const struct drm_mode_config_funcs st7920_mode_config_funcs = {
+	.fb_create = drm_gem_fb_create_with_dirty,
+	.atomic_check = drm_atomic_helper_check,
+	.atomic_commit = drm_atomic_helper_commit,
+};
+
+static const uint32_t st7920_formats[] = {
+	DRM_FORMAT_XRGB8888,
+};
+
+DEFINE_DRM_GEM_FOPS(st7920_fops);
+
+static const struct drm_driver st7920_drm_driver = {
+	DRM_GEM_SHMEM_DRIVER_OPS,
+	DRM_FBDEV_SHMEM_DRIVER_OPS,
+	.name			= DRIVER_NAME,
+	.desc			= DRIVER_DESC,
+	.major			= DRIVER_MAJOR,
+	.minor			= DRIVER_MINOR,
+	.driver_features = DRIVER_ATOMIC | DRIVER_GEM | DRIVER_MODESET,
+	.fops			= &st7920_fops,
+};
+
+static int st7920_init_modeset(struct st7920_device *st7920)
+{
+	struct drm_display_mode *mode = &st7920->mode;
+	struct drm_device *drm = &st7920->drm;
+	unsigned long max_width, max_height;
+	struct drm_plane *primary_plane;
+	struct drm_crtc *crtc;
+	struct drm_encoder *encoder;
+	struct drm_connector *connector;
+	int ret;
+
+	/*
+	 * Modesetting
+	 */
+
+	ret = drmm_mode_config_init(drm);
+	if (ret) {
+		drm_err(drm, "DRM mode config init failed: %d\n", ret);
+		return ret;
+	}
+
+	mode->type = DRM_MODE_TYPE_DRIVER;
+	mode->clock = 30;
+	mode->hdisplay = st7920->width;
+	mode->htotal = st7920->width;
+	mode->hsync_start = st7920->width;
+	mode->hsync_end = st7920->width;
+	mode->vdisplay = st7920->height;
+	mode->vtotal = st7920->height;
+	mode->vsync_start = st7920->height;
+	mode->vsync_end = st7920->height;
+	mode->width_mm = 27;
+	mode->height_mm = 27;
+
+	max_width = max_t(unsigned long, mode->hdisplay, DRM_SHADOW_PLANE_MAX_WIDTH);
+	max_height = max_t(unsigned long, mode->vdisplay, DRM_SHADOW_PLANE_MAX_HEIGHT);
+
+	drm->mode_config.min_width = mode->hdisplay;
+	drm->mode_config.max_width = max_width;
+	drm->mode_config.min_height = mode->vdisplay;
+	drm->mode_config.max_height = max_height;
+	drm->mode_config.preferred_depth = 24;
+	drm->mode_config.funcs = &st7920_mode_config_funcs;
+
+	/* Primary plane */
+
+	primary_plane = &st7920->primary_plane;
+	ret = drm_universal_plane_init(drm, primary_plane, 0, &st7920_primary_plane_funcs,
+				       st7920_formats, ARRAY_SIZE(st7920_formats),
+				       NULL, DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret) {
+		drm_err(drm, "DRM primary plane init failed: %d\n", ret);
+		return ret;
+	}
+
+	drm_plane_helper_add(primary_plane, &st7920_primary_plane_helper_funcs);
+
+	drm_plane_enable_fb_damage_clips(primary_plane);
+
+	/* CRTC */
+
+	crtc = &st7920->crtc;
+	ret = drm_crtc_init_with_planes(drm, crtc, primary_plane, NULL,
+					&st7920_crtc_funcs, NULL);
+	if (ret) {
+		drm_err(drm, "DRM crtc init failed: %d\n", ret);
+		return ret;
+	}
+
+	drm_crtc_helper_add(crtc, &st7920_crtc_helper_funcs);
+
+	/* Encoder */
+
+	encoder = &st7920->encoder;
+	ret = drm_encoder_init(drm, encoder, &st7920_encoder_funcs,
+			       DRM_MODE_ENCODER_NONE, NULL);
+	if (ret) {
+		drm_err(drm, "DRM encoder init failed: %d\n", ret);
+		return ret;
+	}
+
+	encoder->possible_crtcs = drm_crtc_mask(crtc);
+
+	/* Connector */
+
+	connector = &st7920->connector;
+	ret = drm_connector_init(drm, connector, &st7920_connector_funcs,
+				 DRM_MODE_CONNECTOR_Unknown);
+	if (ret) {
+		drm_err(drm, "DRM connector init failed: %d\n", ret);
+		return ret;
+	}
+
+	drm_connector_helper_add(connector, &st7920_connector_helper_funcs);
+
+	ret = drm_connector_attach_encoder(connector, encoder);
+	if (ret) {
+		drm_err(drm, "DRM attach connector to encoder failed: %d\n", ret);
+		return ret;
+	}
+
+	drm_mode_config_reset(drm);
+
+	return 0;
+}
+
+static int st7920_probe(struct spi_device *spi)
+{
+	struct st7920_device *st7920;
+	struct regmap *regmap;
+	struct device *dev = &spi->dev;
+	struct drm_device *drm;
+	int ret;
+
+	regmap = devm_regmap_init_spi(spi, &st7920_spi_regmap_config);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	st7920 = devm_drm_dev_alloc(dev, &st7920_drm_driver,
+				    struct st7920_device, drm);
+	if (IS_ERR(st7920))
+		return PTR_ERR(st7920);
+
+	drm = &st7920->drm;
+
+	st7920->drm.dev = dev;
+	st7920->regmap = regmap;
+	st7920->spi = spi;
+	st7920->width = ST7920_DEFAULT_WIDTH;
+	st7920->height = ST7920_DEFAULT_HEIGHT;
+
+	st7920->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(st7920->reset_gpio)) {
+		ret = PTR_ERR(st7920->reset_gpio);
+		return dev_err_probe(dev, ret, "Unable to retrieve reset GPIO\n");
+	}
+
+	spi_set_drvdata(spi, st7920);
+
+	ret = st7920_init_modeset(st7920);
+	if (ret)
+		return ret;
+
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		return dev_err_probe(dev, ret, "DRM device register failed\n");
+
+	drm_client_setup(drm, NULL);
+
+	return 0;
+}
+
+static void st7920_remove(struct spi_device *spi)
+{
+	struct st7920_device *st7920 = spi_get_drvdata(spi);
+
+	drm_dev_unplug(&st7920->drm);
+	drm_atomic_helper_shutdown(&st7920->drm);
+}
+
+static void st7920_shutdown(struct spi_device *spi)
+{
+	struct st7920_device *st7920 = spi_get_drvdata(spi);
+
+	drm_atomic_helper_shutdown(&st7920->drm);
+}
+
+static struct spi_driver st7920_spi_driver = {
+	.driver = {
+		.name = DRIVER_NAME,
+		.of_match_table = st7920_of_match,
+	},
+	.id_table = st7920_spi_id,
+	.probe = st7920_probe,
+	.remove = st7920_remove,
+	.shutdown = st7920_shutdown,
+};
+module_spi_driver(st7920_spi_driver);
+
+MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_AUTHOR("Iker Pedrosa <ipedrosam@gmail.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/sysfb/efidrm.c b/drivers/gpu/drm/sysfb/efidrm.c
index 1b683d55d6ea..50e0aeef709c 100644
--- a/drivers/gpu/drm/sysfb/efidrm.c
+++ b/drivers/gpu/drm/sysfb/efidrm.c
@@ -4,7 +4,7 @@
 #include <linux/efi.h>
 #include <linux/limits.h>
 #include <linux/platform_device.h>
-#include <linux/screen_info.h>
+#include <linux/sysfb.h>
 
 #include <drm/clients/drm_client_setup.h>
 #include <drm/drm_atomic.h>
@@ -24,7 +24,6 @@
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 
-#include <video/edid.h>
 #include <video/pixel_format.h>
 
 #include "drm_sysfb_helper.h"
@@ -141,6 +140,7 @@ static const struct drm_mode_config_funcs efidrm_mode_config_funcs = {
 static struct efidrm_device *efidrm_device_create(struct drm_driver *drv,
 						  struct platform_device *pdev)
 {
+	const struct sysfb_display_info *dpy;
 	const struct screen_info *si;
 	const struct drm_format_info *format;
 	int width, height, stride;
@@ -160,9 +160,11 @@ static struct efidrm_device *efidrm_device_create(struct drm_driver *drv,
 	size_t nformats;
 	int ret;
 
-	si = dev_get_platdata(&pdev->dev);
-	if (!si)
+	dpy = dev_get_platdata(&pdev->dev);
+	if (!dpy)
 		return ERR_PTR(-ENODEV);
+	si = &dpy->screen;
+
 	if (screen_info_video_type(si) != VIDEO_TYPE_EFI)
 		return ERR_PTR(-ENODEV);
 
@@ -204,8 +206,8 @@ static struct efidrm_device *efidrm_device_create(struct drm_driver *drv,
 		&format->format, width, height, stride);
 
 #if defined(CONFIG_FIRMWARE_EDID)
-	if (drm_edid_header_is_valid(edid_info.dummy) == 8)
-		sysfb->edid = edid_info.dummy;
+	if (drm_edid_header_is_valid(dpy->edid.dummy) == 8)
+		sysfb->edid = dpy->edid.dummy;
 #endif
 	sysfb->fb_mode = drm_sysfb_mode(width, height, 0, 0);
 	sysfb->fb_format = format;
diff --git a/drivers/gpu/drm/sysfb/vesadrm.c b/drivers/gpu/drm/sysfb/vesadrm.c
index 7b7b5ba26317..0680638b8131 100644
--- a/drivers/gpu/drm/sysfb/vesadrm.c
+++ b/drivers/gpu/drm/sysfb/vesadrm.c
@@ -4,7 +4,7 @@
 #include <linux/ioport.h>
 #include <linux/limits.h>
 #include <linux/platform_device.h>
-#include <linux/screen_info.h>
+#include <linux/sysfb.h>
 
 #include <drm/clients/drm_client_setup.h>
 #include <drm/drm_atomic.h>
@@ -25,7 +25,6 @@
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 
-#include <video/edid.h>
 #include <video/pixel_format.h>
 #include <video/vga.h>
 
@@ -391,6 +390,7 @@ static const struct drm_mode_config_funcs vesadrm_mode_config_funcs = {
 static struct vesadrm_device *vesadrm_device_create(struct drm_driver *drv,
 						    struct platform_device *pdev)
 {
+	const struct sysfb_display_info *dpy;
 	const struct screen_info *si;
 	const struct drm_format_info *format;
 	int width, height, stride;
@@ -410,9 +410,11 @@ static struct vesadrm_device *vesadrm_device_create(struct drm_driver *drv,
 	size_t nformats;
 	int ret;
 
-	si = dev_get_platdata(&pdev->dev);
-	if (!si)
+	dpy = dev_get_platdata(&pdev->dev);
+	if (!dpy)
 		return ERR_PTR(-ENODEV);
+	si = &dpy->screen;
+
 	if (screen_info_video_type(si) != VIDEO_TYPE_VLFB)
 		return ERR_PTR(-ENODEV);
 
@@ -471,8 +473,8 @@ static struct vesadrm_device *vesadrm_device_create(struct drm_driver *drv,
 	}
 
 #if defined(CONFIG_FIRMWARE_EDID)
-	if (drm_edid_header_is_valid(edid_info.dummy) == 8)
-		sysfb->edid = edid_info.dummy;
+	if (drm_edid_header_is_valid(dpy->edid.dummy) == 8)
+		sysfb->edid = dpy->edid.dummy;
 #endif
 	sysfb->fb_mode = drm_sysfb_mode(width, height, 0, 0);
 	sysfb->fb_format = format;
diff --git a/drivers/gpu/drm/tests/drm_atomic_state_test.c b/drivers/gpu/drm/tests/drm_atomic_state_test.c
index 2f6ac7a09f44..bc27f65b2823 100644
--- a/drivers/gpu/drm/tests/drm_atomic_state_test.c
+++ b/drivers/gpu/drm/tests/drm_atomic_state_test.c
@@ -156,24 +156,29 @@ static int set_up_atomic_state(struct kunit *test,
 
 	if (connector) {
 		conn_state = drm_atomic_get_connector_state(state, connector);
-		KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+		if (IS_ERR(conn_state))
+			return PTR_ERR(conn_state);
 
 		ret = drm_atomic_set_crtc_for_connector(conn_state, crtc);
-		KUNIT_EXPECT_EQ(test, ret, 0);
+		if (ret)
+			return ret;
 	}
 
 	crtc_state = drm_atomic_get_crtc_state(state, crtc);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+	if (IS_ERR(crtc_state))
+		return PTR_ERR(crtc_state);
 
 	ret = drm_atomic_set_mode_for_crtc(crtc_state, &drm_atomic_test_mode);
-	KUNIT_EXPECT_EQ(test, ret, 0);
+	if (ret)
+		return ret;
 
 	crtc_state->enable = true;
 	crtc_state->active = true;
 
 	if (connector) {
 		ret = drm_atomic_commit(state);
-		KUNIT_ASSERT_EQ(test, ret, 0);
+		if (ret)
+			return ret;
 	} else {
 		// dummy connector mask
 		crtc_state->connector_mask = DRM_TEST_CONN_0;
@@ -206,7 +211,13 @@ static void drm_test_check_connector_changed_modeset(struct kunit *test)
 	drm_modeset_acquire_init(&ctx, 0);
 
 	// first modeset to enable
+retry_set_up:
 	ret = set_up_atomic_state(test, priv, old_conn, &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_set_up;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
@@ -277,13 +288,26 @@ static void drm_test_check_valid_clones(struct kunit *test)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_set_up:
 	ret = set_up_atomic_state(test, priv, NULL, &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_set_up;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
 
+retry:
 	crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+	if (PTR_ERR(crtc_state) == -EDEADLK) {
+		drm_atomic_state_clear(state);
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry;
+	}
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
 
 	crtc_state->encoder_mask = param->encoder_mask;
@@ -292,6 +316,12 @@ static void drm_test_check_valid_clones(struct kunit *test)
 	crtc_state->mode_changed = true;
 
 	ret = drm_atomic_helper_check_modeset(drm, state);
+	if (ret == -EDEADLK) {
+		drm_atomic_state_clear(state);
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry;
+	}
 	KUNIT_ASSERT_EQ(test, ret, param->expected_result);
 
 	drm_modeset_drop_locks(&ctx);
diff --git a/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c b/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
index 8bd412735000..70f9aa702143 100644
--- a/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
+++ b/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
@@ -257,10 +257,16 @@ static void drm_test_check_broadcast_rgb_crtc_mode_changed(struct kunit *test)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
@@ -326,10 +332,16 @@ static void drm_test_check_broadcast_rgb_crtc_mode_not_changed(struct kunit *tes
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
@@ -397,10 +409,16 @@ static void drm_test_check_broadcast_rgb_auto_cea_mode(struct kunit *test)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
@@ -457,10 +475,17 @@ static void drm_test_check_broadcast_rgb_auto_cea_mode_vic_1(struct kunit *test)
 	KUNIT_ASSERT_NOT_NULL(test, mode);
 
 	crtc = priv->crtc;
+
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     mode,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
@@ -518,10 +543,16 @@ static void drm_test_check_broadcast_rgb_full_cea_mode(struct kunit *test)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
@@ -580,10 +611,17 @@ static void drm_test_check_broadcast_rgb_full_cea_mode_vic_1(struct kunit *test)
 	KUNIT_ASSERT_NOT_NULL(test, mode);
 
 	crtc = priv->crtc;
+
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     mode,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
@@ -643,10 +681,16 @@ static void drm_test_check_broadcast_rgb_limited_cea_mode(struct kunit *test)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
@@ -705,10 +749,17 @@ static void drm_test_check_broadcast_rgb_limited_cea_mode_vic_1(struct kunit *te
 	KUNIT_ASSERT_NOT_NULL(test, mode);
 
 	crtc = priv->crtc;
+
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     mode,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
@@ -870,10 +921,16 @@ static void drm_test_check_output_bpc_crtc_mode_changed(struct kunit *test)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
@@ -946,10 +1003,16 @@ static void drm_test_check_output_bpc_crtc_mode_not_changed(struct kunit *test)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
@@ -1022,10 +1085,16 @@ static void drm_test_check_output_bpc_dvi(struct kunit *test)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	conn_state = conn->state;
@@ -1069,10 +1138,16 @@ static void drm_test_check_tmds_char_rate_rgb_8bpc(struct kunit *test)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	conn_state = conn->state;
@@ -1118,10 +1193,16 @@ static void drm_test_check_tmds_char_rate_rgb_10bpc(struct kunit *test)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	conn_state = conn->state;
@@ -1167,10 +1248,16 @@ static void drm_test_check_tmds_char_rate_rgb_12bpc(struct kunit *test)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	conn_state = conn->state;
@@ -1218,10 +1305,16 @@ static void drm_test_check_hdmi_funcs_reject_rate(struct kunit *test)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	/* You shouldn't be doing that at home. */
@@ -1292,10 +1385,16 @@ static void drm_test_check_max_tmds_rate_bpc_fallback_rgb(struct kunit *test)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_EXPECT_EQ(test, ret, 0);
 
 	conn_state = conn->state;
@@ -1440,10 +1539,16 @@ static void drm_test_check_max_tmds_rate_bpc_fallback_ignore_yuv422(struct kunit
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_EXPECT_EQ(test, ret, 0);
 
 	conn_state = conn->state;
@@ -1669,10 +1774,17 @@ static void drm_test_check_output_bpc_format_vic_1(struct kunit *test)
 	drm_modeset_acquire_init(&ctx, 0);
 
 	crtc = priv->crtc;
+
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     mode,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_EXPECT_EQ(test, ret, 0);
 
 	conn_state = conn->state;
@@ -1736,10 +1848,16 @@ static void drm_test_check_output_bpc_format_driver_rgb_only(struct kunit *test)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_EXPECT_EQ(test, ret, 0);
 
 	conn_state = conn->state;
@@ -1805,10 +1923,16 @@ static void drm_test_check_output_bpc_format_display_rgb_only(struct kunit *test
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_EXPECT_EQ(test, ret, 0);
 
 	conn_state = conn->state;
@@ -1865,10 +1989,16 @@ static void drm_test_check_output_bpc_format_driver_8bpc_only(struct kunit *test
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_EXPECT_EQ(test, ret, 0);
 
 	conn_state = conn->state;
@@ -1927,10 +2057,16 @@ static void drm_test_check_output_bpc_format_display_8bpc_only(struct kunit *tes
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_EXPECT_EQ(test, ret, 0);
 
 	conn_state = conn->state;
@@ -1970,10 +2106,17 @@ static void drm_test_check_disable_connector(struct kunit *test)
 
 	drm = &priv->drm;
 	crtc = priv->crtc;
+
+retry_conn_enable:
 	ret = drm_kunit_helper_enable_crtc_connector(test, drm,
 						     crtc, conn,
 						     preferred,
 						     &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_conn_enable;
+	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
 	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c
index 2eda87882e65..6d95447a989d 100644
--- a/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c
+++ b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c
@@ -692,7 +692,7 @@ static int threaded_fence_signal(void *arg)
 
 	msleep(20);
 
-	return dma_fence_signal(fence);
+	return dma_fence_check_and_signal(fence) ? -EINVAL : 0;
 }
 
 static void ttm_bo_validate_move_fence_not_signaled(struct kunit *test)
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index bd27607f8076..acb9197db879 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -268,8 +268,8 @@ static void ttm_bo_release(struct kref *kref)
 					      30 * HZ);
 		}
 
-		if (bo->bdev->funcs->release_notify)
-			bo->bdev->funcs->release_notify(bo);
+		if (bdev->funcs->release_notify)
+			bdev->funcs->release_notify(bo);
 
 		drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
 		ttm_mem_io_free(bdev, bo->resource);
@@ -283,7 +283,7 @@ static void ttm_bo_release(struct kref *kref)
 			ttm_bo_flush_all_fences(bo);
 			bo->deleted = true;
 
-			spin_lock(&bo->bdev->lru_lock);
+			spin_lock(&bdev->lru_lock);
 
 			/*
 			 * Make pinned bos immediately available to
@@ -299,7 +299,7 @@ static void ttm_bo_release(struct kref *kref)
 			}
 
 			kref_init(&bo->kref);
-			spin_unlock(&bo->bdev->lru_lock);
+			spin_unlock(&bdev->lru_lock);
 
 			INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete);
 
@@ -359,7 +359,6 @@ static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo,
 static int ttm_bo_evict(struct ttm_buffer_object *bo,
 			struct ttm_operation_ctx *ctx)
 {
-	struct ttm_device *bdev = bo->bdev;
 	struct ttm_resource *evict_mem;
 	struct ttm_placement placement;
 	struct ttm_place hop;
@@ -370,7 +369,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo,
 	dma_resv_assert_held(bo->base.resv);
 
 	placement.num_placement = 0;
-	bdev->funcs->evict_flags(bo, &placement);
+	bo->bdev->funcs->evict_flags(bo, &placement);
 
 	if (!placement.num_placement) {
 		ret = ttm_bo_wait_ctx(bo, ctx);
@@ -423,16 +422,16 @@ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 			      const struct ttm_place *place)
 {
 	struct ttm_resource *res = bo->resource;
-	struct ttm_device *bdev = bo->bdev;
 
 	dma_resv_assert_held(bo->base.resv);
-	if (bo->resource->mem_type == TTM_PL_SYSTEM)
+
+	if (res->mem_type == TTM_PL_SYSTEM)
 		return true;
 
 	/* Don't evict this BO if it's outside of the
 	 * requested placement range
 	 */
-	return ttm_resource_intersects(bdev, res, place, bo->base.size);
+	return ttm_resource_intersects(bo->bdev, res, place, bo->base.size);
 }
 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 
@@ -1027,7 +1026,7 @@ int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo,
 			 struct sg_table *sg, struct dma_resv *resv,
 			 void (*destroy) (struct ttm_buffer_object *))
 {
-	struct ttm_operation_ctx ctx = { interruptible, false };
+	struct ttm_operation_ctx ctx = { .interruptible = interruptible };
 	int ret;
 
 	ret = ttm_bo_init_reserved(bdev, bo, type, placement, alignment, &ctx,
@@ -1108,10 +1107,13 @@ struct ttm_bo_swapout_walk {
 static s64
 ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo)
 {
-	struct ttm_place place = {.mem_type = bo->resource->mem_type};
+	struct ttm_resource *res = bo->resource;
+	struct ttm_place place = { .mem_type = res->mem_type };
 	struct ttm_bo_swapout_walk *swapout_walk =
 		container_of(walk, typeof(*swapout_walk), walk);
 	struct ttm_operation_ctx *ctx = walk->arg.ctx;
+	struct ttm_device *bdev = bo->bdev;
+	struct ttm_tt *tt = bo->ttm;
 	s64 ret;
 
 	/*
@@ -1120,20 +1122,19 @@ ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo)
 	 * The driver may use the fact that we're moving from SYSTEM
 	 * as an indication that we're about to swap out.
 	 */
-	if (bo->pin_count || !bo->bdev->funcs->eviction_valuable(bo, &place)) {
+	if (bo->pin_count || !bdev->funcs->eviction_valuable(bo, &place)) {
 		ret = -EBUSY;
 		goto out;
 	}
 
-	if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) ||
-	    bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL ||
-	    bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED) {
+	if (!tt || !ttm_tt_is_populated(tt) ||
+	    tt->page_flags & (TTM_TT_FLAG_EXTERNAL | TTM_TT_FLAG_SWAPPED)) {
 		ret = -EBUSY;
 		goto out;
 	}
 
 	if (bo->deleted) {
-		pgoff_t num_pages = bo->ttm->num_pages;
+		pgoff_t num_pages = tt->num_pages;
 
 		ret = ttm_bo_wait_ctx(bo, ctx);
 		if (ret)
@@ -1147,7 +1148,7 @@ ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo)
 	/*
 	 * Move to system cached
 	 */
-	if (bo->resource->mem_type != TTM_PL_SYSTEM) {
+	if (res->mem_type != TTM_PL_SYSTEM) {
 		struct ttm_resource *evict_mem;
 		struct ttm_place hop;
 
@@ -1174,21 +1175,21 @@ ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo)
 		goto out;
 
 	ttm_bo_unmap_virtual(bo);
-	if (bo->bdev->funcs->swap_notify)
-		bo->bdev->funcs->swap_notify(bo);
+	if (bdev->funcs->swap_notify)
+		bdev->funcs->swap_notify(bo);
 
-	if (ttm_tt_is_populated(bo->ttm)) {
-		spin_lock(&bo->bdev->lru_lock);
-		ttm_resource_del_bulk_move(bo->resource, bo);
-		spin_unlock(&bo->bdev->lru_lock);
+	if (ttm_tt_is_populated(tt)) {
+		spin_lock(&bdev->lru_lock);
+		ttm_resource_del_bulk_move(res, bo);
+		spin_unlock(&bdev->lru_lock);
 
-		ret = ttm_tt_swapout(bo->bdev, bo->ttm, swapout_walk->gfp_flags);
+		ret = ttm_tt_swapout(bdev, tt, swapout_walk->gfp_flags);
 
-		spin_lock(&bo->bdev->lru_lock);
+		spin_lock(&bdev->lru_lock);
 		if (ret)
-			ttm_resource_add_bulk_move(bo->resource, bo);
-		ttm_resource_move_to_lru_tail(bo->resource);
-		spin_unlock(&bo->bdev->lru_lock);
+			ttm_resource_add_bulk_move(res, bo);
+		ttm_resource_move_to_lru_tail(res);
+		spin_unlock(&bdev->lru_lock);
 	}
 
 out:
@@ -1261,6 +1262,7 @@ void ttm_bo_tt_destroy(struct ttm_buffer_object *bo)
 int ttm_bo_populate(struct ttm_buffer_object *bo,
 		    struct ttm_operation_ctx *ctx)
 {
+	struct ttm_device *bdev = bo->bdev;
 	struct ttm_tt *tt = bo->ttm;
 	bool swapped;
 	int ret;
@@ -1271,16 +1273,16 @@ int ttm_bo_populate(struct ttm_buffer_object *bo,
 		return 0;
 
 	swapped = ttm_tt_is_swapped(tt);
-	ret = ttm_tt_populate(bo->bdev, tt, ctx);
+	ret = ttm_tt_populate(bdev, tt, ctx);
 	if (ret)
 		return ret;
 
 	if (swapped && !ttm_tt_is_swapped(tt) && !bo->pin_count &&
 	    bo->resource) {
-		spin_lock(&bo->bdev->lru_lock);
+		spin_lock(&bdev->lru_lock);
 		ttm_resource_add_bulk_move(bo->resource, bo);
 		ttm_resource_move_to_lru_tail(bo->resource);
-		spin_unlock(&bo->bdev->lru_lock);
+		spin_unlock(&bdev->lru_lock);
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index 2ff35d55e462..cabcfeaa70dc 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -174,13 +174,13 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 
 	dst_iter = ttm_kmap_iter_linear_io_init(&_dst_iter.io, bdev, dst_mem);
 	if (PTR_ERR(dst_iter) == -EINVAL && dst_man->use_tt)
-		dst_iter = ttm_kmap_iter_tt_init(&_dst_iter.tt, bo->ttm);
+		dst_iter = ttm_kmap_iter_tt_init(&_dst_iter.tt, ttm);
 	if (IS_ERR(dst_iter))
 		return PTR_ERR(dst_iter);
 
 	src_iter = ttm_kmap_iter_linear_io_init(&_src_iter.io, bdev, src_mem);
 	if (PTR_ERR(src_iter) == -EINVAL && src_man->use_tt)
-		src_iter = ttm_kmap_iter_tt_init(&_src_iter.tt, bo->ttm);
+		src_iter = ttm_kmap_iter_tt_init(&_src_iter.tt, ttm);
 	if (IS_ERR(src_iter)) {
 		ret = PTR_ERR(src_iter);
 		goto out_src_iter;
@@ -318,11 +318,11 @@ static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
 {
 	struct ttm_resource *mem = bo->resource;
 
-	if (bo->resource->bus.addr) {
+	if (mem->bus.addr) {
 		map->bo_kmap_type = ttm_bo_map_premapped;
-		map->virtual = ((u8 *)bo->resource->bus.addr) + offset;
+		map->virtual = ((u8 *)mem->bus.addr) + offset;
 	} else {
-		resource_size_t res = bo->resource->bus.offset + offset;
+		resource_size_t res = mem->bus.offset + offset;
 
 		map->bo_kmap_type = ttm_bo_map_iomap;
 		if (mem->bus.caching == ttm_write_combined)
@@ -343,13 +343,10 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 			   struct ttm_bo_kmap_obj *map)
 {
 	struct ttm_resource *mem = bo->resource;
-	struct ttm_operation_ctx ctx = {
-		.interruptible = false,
-		.no_wait_gpu = false
-	};
+	struct ttm_operation_ctx ctx = { };
 	struct ttm_tt *ttm = bo->ttm;
 	struct ttm_resource_manager *man =
-			ttm_manager_type(bo->bdev, bo->resource->mem_type);
+			ttm_manager_type(bo->bdev, mem->mem_type);
 	pgprot_t prot;
 	int ret;
 
@@ -428,20 +425,21 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo,
 		unsigned long start_page, unsigned long num_pages,
 		struct ttm_bo_kmap_obj *map)
 {
+	struct ttm_resource *res = bo->resource;
 	unsigned long offset, size;
 	int ret;
 
 	map->virtual = NULL;
 	map->bo = bo;
-	if (num_pages > PFN_UP(bo->resource->size))
+	if (num_pages > PFN_UP(res->size))
 		return -EINVAL;
-	if ((start_page + num_pages) > PFN_UP(bo->resource->size))
+	if ((start_page + num_pages) > PFN_UP(res->size))
 		return -EINVAL;
 
-	ret = ttm_mem_io_reserve(bo->bdev, bo->resource);
+	ret = ttm_mem_io_reserve(bo->bdev, res);
 	if (ret)
 		return ret;
-	if (!bo->resource->bus.is_iomem) {
+	if (!res->bus.is_iomem) {
 		return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
 	} else {
 		offset = start_page << PAGE_SHIFT;
@@ -530,10 +528,7 @@ int ttm_bo_vmap(struct ttm_buffer_object *bo, struct iosys_map *map)
 		iosys_map_set_vaddr_iomem(map, vaddr_iomem);
 
 	} else {
-		struct ttm_operation_ctx ctx = {
-			.interruptible = false,
-			.no_wait_gpu = false
-		};
+		struct ttm_operation_ctx ctx = { };
 		struct ttm_tt *ttm = bo->ttm;
 		pgprot_t prot;
 		void *vaddr;
@@ -581,7 +576,7 @@ void ttm_bo_vunmap(struct ttm_buffer_object *bo, struct iosys_map *map)
 		iounmap(map->vaddr_iomem);
 	iosys_map_clear(map);
 
-	ttm_mem_io_free(bo->bdev, bo->resource);
+	ttm_mem_io_free(bo->bdev, mem);
 }
 EXPORT_SYMBOL(ttm_bo_vunmap);
 
@@ -644,12 +639,11 @@ static int ttm_bo_move_to_ghost(struct ttm_buffer_object *bo,
 static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo,
 				       struct dma_fence *fence)
 {
-	struct ttm_device *bdev = bo->bdev;
 	struct ttm_resource_manager *from;
 	struct dma_fence *tmp;
 	int i;
 
-	from = ttm_manager_type(bdev, bo->resource->mem_type);
+	from = ttm_manager_type(bo->bdev, bo->resource->mem_type);
 
 	/**
 	 * BO doesn't have a TTM we need to bind/unbind. Just remember
@@ -743,8 +737,8 @@ EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);
 void ttm_bo_move_sync_cleanup(struct ttm_buffer_object *bo,
 			      struct ttm_resource *new_mem)
 {
-	struct ttm_device *bdev = bo->bdev;
-	struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type);
+	struct ttm_resource_manager *man =
+		ttm_manager_type(bo->bdev, new_mem->mem_type);
 	int ret;
 
 	ret = ttm_bo_wait_free_node(bo, man->use_tt);
@@ -848,13 +842,12 @@ static int ttm_lru_walk_ticketlock(struct ttm_bo_lru_cursor *curs,
 				   struct ttm_buffer_object *bo)
 {
 	struct ttm_lru_walk_arg *arg = curs->arg;
-	struct dma_resv *resv = bo->base.resv;
 	int ret;
 
 	if (arg->ctx->interruptible)
-		ret = dma_resv_lock_interruptible(resv, arg->ticket);
+		ret = dma_resv_lock_interruptible(bo->base.resv, arg->ticket);
 	else
-		ret = dma_resv_lock(resv, arg->ticket);
+		ret = dma_resv_lock(bo->base.resv, arg->ticket);
 
 	if (!ret) {
 		curs->needs_unlock = true;
@@ -1098,7 +1091,7 @@ long ttm_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo,
 		.num_placement = 1,
 		.placement = &sys_placement_flags,
 	};
-	struct ttm_tt *tt = bo->ttm;
+	struct ttm_device *bdev = bo->bdev;
 	long lret;
 
 	dma_resv_assert_held(bo->base.resv);
@@ -1120,19 +1113,19 @@ long ttm_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo,
 		return lret;
 
 	if (bo->bulk_move) {
-		spin_lock(&bo->bdev->lru_lock);
+		spin_lock(&bdev->lru_lock);
 		ttm_resource_del_bulk_move(bo->resource, bo);
-		spin_unlock(&bo->bdev->lru_lock);
+		spin_unlock(&bdev->lru_lock);
 	}
 
-	lret = ttm_tt_backup(bo->bdev, tt, (struct ttm_backup_flags)
+	lret = ttm_tt_backup(bdev, bo->ttm, (struct ttm_backup_flags)
 			     {.purge = flags.purge,
 			      .writeback = flags.writeback});
 
 	if (lret <= 0 && bo->bulk_move) {
-		spin_lock(&bo->bdev->lru_lock);
+		spin_lock(&bdev->lru_lock);
 		ttm_resource_add_bulk_move(bo->resource, bo);
-		spin_unlock(&bo->bdev->lru_lock);
+		spin_unlock(&bdev->lru_lock);
 	}
 
 	if (lret < 0 && lret != -EINTR)
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index e6abc7b40b18..a80510489c45 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -186,7 +186,6 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,
 {
 	struct vm_area_struct *vma = vmf->vma;
 	struct ttm_buffer_object *bo = vma->vm_private_data;
-	struct ttm_device *bdev = bo->bdev;
 	unsigned long page_offset;
 	unsigned long page_last;
 	unsigned long pfn;
@@ -205,7 +204,7 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,
 	if (unlikely(ret != 0))
 		return ret;
 
-	err = ttm_mem_io_reserve(bdev, bo->resource);
+	err = ttm_mem_io_reserve(bo->bdev, bo->resource);
 	if (unlikely(err != 0))
 		return VM_FAULT_SIGBUS;
 
@@ -293,7 +292,6 @@ vm_fault_t ttm_bo_vm_dummy_page(struct vm_fault *vmf, pgprot_t prot)
 {
 	struct vm_area_struct *vma = vmf->vma;
 	struct ttm_buffer_object *bo = vma->vm_private_data;
-	struct drm_device *ddev = bo->base.dev;
 	vm_fault_t ret = VM_FAULT_NOPAGE;
 	unsigned long address;
 	unsigned long pfn;
@@ -305,7 +303,8 @@ vm_fault_t ttm_bo_vm_dummy_page(struct vm_fault *vmf, pgprot_t prot)
 		return VM_FAULT_OOM;
 
 	/* Set the page to be freed using drmm release action */
-	if (drmm_add_action_or_reset(ddev, ttm_bo_release_dummy_page, page))
+	if (drmm_add_action_or_reset(bo->base.dev, ttm_bo_release_dummy_page,
+				     page))
 		return VM_FAULT_OOM;
 
 	pfn = page_to_pfn(page);
@@ -322,10 +321,9 @@ EXPORT_SYMBOL(ttm_bo_vm_dummy_page);
 vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
-	pgprot_t prot;
 	struct ttm_buffer_object *bo = vma->vm_private_data;
-	struct drm_device *ddev = bo->base.dev;
 	vm_fault_t ret;
+	pgprot_t prot;
 	int idx;
 
 	ret = ttm_bo_vm_reserve(bo, vmf);
@@ -333,7 +331,7 @@ vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf)
 		return ret;
 
 	prot = vma->vm_page_prot;
-	if (drm_dev_enter(ddev, &idx)) {
+	if (drm_dev_enter(bo->base.dev, &idx)) {
 		ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT);
 		drm_dev_exit(idx);
 	} else {
diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
index 9a51afaf0749..d3bfb9a696a7 100644
--- a/drivers/gpu/drm/ttm/ttm_device.c
+++ b/drivers/gpu/drm/ttm/ttm_device.c
@@ -135,10 +135,7 @@ static int ttm_global_init(void)
  */
 int ttm_device_prepare_hibernation(struct ttm_device *bdev)
 {
-	struct ttm_operation_ctx ctx = {
-		.interruptible = false,
-		.no_wait_gpu = false,
-	};
+	struct ttm_operation_ctx ctx = { };
 	int ret;
 
 	do {
diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c
index 18b6db015619..217e45958099 100644
--- a/drivers/gpu/drm/ttm/ttm_pool.c
+++ b/drivers/gpu/drm/ttm/ttm_pool.c
@@ -845,32 +845,34 @@ EXPORT_SYMBOL(ttm_pool_alloc);
 int ttm_pool_restore_and_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
 			       const struct ttm_operation_ctx *ctx)
 {
+	struct ttm_pool_tt_restore *restore = tt->restore;
 	struct ttm_pool_alloc_state alloc;
 
 	if (WARN_ON(!ttm_tt_is_backed_up(tt)))
 		return -EINVAL;
 
-	if (!tt->restore) {
+	if (!restore) {
 		gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
 
 		ttm_pool_alloc_state_init(tt, &alloc);
 		if (ctx->gfp_retry_mayfail)
 			gfp |= __GFP_RETRY_MAYFAIL;
 
-		tt->restore = kzalloc(sizeof(*tt->restore), gfp);
-		if (!tt->restore)
+		restore = kzalloc(sizeof(*restore), gfp);
+		if (!restore)
 			return -ENOMEM;
 
-		tt->restore->snapshot_alloc = alloc;
-		tt->restore->pool = pool;
-		tt->restore->restored_pages = 1;
-	} else {
-		struct ttm_pool_tt_restore *restore = tt->restore;
-		int ret;
+		restore->snapshot_alloc = alloc;
+		restore->pool = pool;
+		restore->restored_pages = 1;
 
+		tt->restore = restore;
+	} else {
 		alloc = restore->snapshot_alloc;
-		if (ttm_pool_restore_valid(tt->restore)) {
-			ret = ttm_pool_restore_commit(restore, tt->backup, ctx, &alloc);
+		if (ttm_pool_restore_valid(restore)) {
+			int ret = ttm_pool_restore_commit(restore, tt->backup,
+							  ctx, &alloc);
+
 			if (ret)
 				return ret;
 		}
@@ -878,7 +880,7 @@ int ttm_pool_restore_and_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
 			return 0;
 	}
 
-	return __ttm_pool_alloc(pool, tt, ctx, &alloc, tt->restore);
+	return __ttm_pool_alloc(pool, tt, ctx, &alloc, restore);
 }
 
 /**
diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c
index f5aa29dc6ec0..192fca24f37e 100644
--- a/drivers/gpu/drm/ttm/ttm_resource.c
+++ b/drivers/gpu/drm/ttm/ttm_resource.c
@@ -446,9 +446,6 @@ bool ttm_resource_intersects(struct ttm_device *bdev,
 {
 	struct ttm_resource_manager *man;
 
-	if (!res)
-		return false;
-
 	man = ttm_manager_type(bdev, res->mem_type);
 	if (!place || !man->func->intersects)
 		return true;
@@ -548,10 +545,7 @@ EXPORT_SYMBOL(ttm_resource_manager_init);
 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
 				   struct ttm_resource_manager *man)
 {
-	struct ttm_operation_ctx ctx = {
-		.interruptible = false,
-		.no_wait_gpu = false,
-	};
+	struct ttm_operation_ctx ctx = { };
 	struct dma_fence *fence;
 	int ret, i;
 
@@ -628,11 +622,11 @@ ttm_resource_cursor_check_bulk(struct ttm_resource_cursor *cursor,
 			       struct ttm_lru_item *next_lru)
 {
 	struct ttm_resource *next = ttm_lru_item_to_res(next_lru);
-	struct ttm_lru_bulk_move *bulk = NULL;
-	struct ttm_buffer_object *bo = next->bo;
+	struct ttm_lru_bulk_move *bulk;
 
 	lockdep_assert_held(&cursor->man->bdev->lru_lock);
-	bulk = bo->bulk_move;
+
+	bulk = next->bo->bulk_move;
 
 	if (cursor->bulk != bulk) {
 		if (bulk) {
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index 611d20ab966d..af33fa020249 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -456,7 +456,7 @@ EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_tt_unpopulate);
 /* Test the shrinker functions and dump the result */
 static int ttm_tt_debugfs_shrink_show(struct seq_file *m, void *data)
 {
-	struct ttm_operation_ctx ctx = { false, false };
+	struct ttm_operation_ctx ctx = { };
 
 	seq_printf(m, "%d\n", ttm_global_swapout(&ctx, GFP_KERNEL));
 	return 0;
diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs
index 0389c558c036..264c2362237a 100644
--- a/drivers/gpu/drm/tyr/driver.rs
+++ b/drivers/gpu/drm/tyr/driver.rs
@@ -16,10 +16,10 @@
 use kernel::regulator;
 use kernel::regulator::Regulator;
 use kernel::sizes::SZ_2M;
+use kernel::sync::aref::ARef;
 use kernel::sync::Arc;
 use kernel::sync::Mutex;
 use kernel::time;
-use kernel::types::ARef;
 
 use crate::file::File;
 use crate::gem::TyrObject;
diff --git a/drivers/gpu/drm/v3d/Makefile b/drivers/gpu/drm/v3d/Makefile
index fcf710926057..b7d673f1153b 100644
--- a/drivers/gpu/drm/v3d/Makefile
+++ b/drivers/gpu/drm/v3d/Makefile
@@ -13,8 +13,7 @@ v3d-y := \
 	v3d_trace_points.o \
 	v3d_sched.o \
 	v3d_sysfs.o \
-	v3d_submit.o \
-	v3d_gemfs.o
+	v3d_submit.o
 
 v3d-$(CONFIG_DEBUG_FS) += v3d_debugfs.o
 
diff --git a/drivers/gpu/drm/v3d/v3d_bo.c b/drivers/gpu/drm/v3d/v3d_bo.c
index d9547f5117b9..c4316b768b3d 100644
--- a/drivers/gpu/drm/v3d/v3d_bo.c
+++ b/drivers/gpu/drm/v3d/v3d_bo.c
@@ -114,7 +114,7 @@ v3d_bo_create_finish(struct drm_gem_object *obj)
 	if (IS_ERR(sgt))
 		return PTR_ERR(sgt);
 
-	if (!v3d->gemfs)
+	if (!drm_gem_get_huge_mnt(obj->dev))
 		align = SZ_4K;
 	else if (obj->size >= SZ_1M)
 		align = SZ_1M;
@@ -150,12 +150,10 @@ struct v3d_bo *v3d_bo_create(struct drm_device *dev, struct drm_file *file_priv,
 			     size_t unaligned_size)
 {
 	struct drm_gem_shmem_object *shmem_obj;
-	struct v3d_dev *v3d = to_v3d_dev(dev);
 	struct v3d_bo *bo;
 	int ret;
 
-	shmem_obj = drm_gem_shmem_create_with_mnt(dev, unaligned_size,
-						  v3d->gemfs);
+	shmem_obj = drm_gem_shmem_create(dev, unaligned_size);
 	if (IS_ERR(shmem_obj))
 		return ERR_CAST(shmem_obj);
 	bo = to_v3d_bo(&shmem_obj->base);
diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c
index e8a46c8bad8a..8faa9382846f 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.c
+++ b/drivers/gpu/drm/v3d/v3d_drv.c
@@ -107,7 +107,7 @@ static int v3d_get_param_ioctl(struct drm_device *dev, void *data,
 		args->value = v3d->perfmon_info.max_counters;
 		return 0;
 	case DRM_V3D_PARAM_SUPPORTS_SUPER_PAGES:
-		args->value = !!v3d->gemfs;
+		args->value = !!drm_gem_get_huge_mnt(dev);
 		return 0;
 	case DRM_V3D_PARAM_GLOBAL_RESET_COUNTER:
 		mutex_lock(&v3d->reset_lock);
diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h
index 1884686985b8..99a39329bb85 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.h
+++ b/drivers/gpu/drm/v3d/v3d_drv.h
@@ -158,11 +158,6 @@ struct v3d_dev {
 	struct drm_mm mm;
 	spinlock_t mm_lock;
 
-	/*
-	 * tmpfs instance used for shmem backed objects
-	 */
-	struct vfsmount *gemfs;
-
 	struct work_struct overflow_mem_work;
 
 	struct v3d_queue_state queue[V3D_MAX_QUEUES];
@@ -569,6 +564,7 @@ extern const struct dma_fence_ops v3d_fence_ops;
 struct dma_fence *v3d_fence_create(struct v3d_dev *v3d, enum v3d_queue q);
 
 /* v3d_gem.c */
+extern bool super_pages;
 int v3d_gem_init(struct drm_device *dev);
 void v3d_gem_destroy(struct drm_device *dev);
 void v3d_reset_sms(struct v3d_dev *v3d);
@@ -576,11 +572,6 @@ void v3d_reset(struct v3d_dev *v3d);
 void v3d_invalidate_caches(struct v3d_dev *v3d);
 void v3d_clean_caches(struct v3d_dev *v3d);
 
-/* v3d_gemfs.c */
-extern bool super_pages;
-void v3d_gemfs_init(struct v3d_dev *v3d);
-void v3d_gemfs_fini(struct v3d_dev *v3d);
-
 /* v3d_submit.c */
 void v3d_job_cleanup(struct v3d_job *job);
 void v3d_job_put(struct v3d_job *job);
diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
index 5a180dc6c452..697b0b3ca92c 100644
--- a/drivers/gpu/drm/v3d/v3d_gem.c
+++ b/drivers/gpu/drm/v3d/v3d_gem.c
@@ -259,6 +259,24 @@ v3d_invalidate_caches(struct v3d_dev *v3d)
 	v3d_invalidate_slices(v3d, 0);
 }
 
+static void
+v3d_huge_mnt_init(struct v3d_dev *v3d)
+{
+	int err = 0;
+
+	if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && super_pages)
+		err = drm_gem_huge_mnt_create(&v3d->drm, "within_size");
+
+	if (drm_gem_get_huge_mnt(&v3d->drm))
+		drm_info(&v3d->drm, "Using Transparent Hugepages\n");
+	else if (err)
+		drm_warn(&v3d->drm, "Can't use Transparent Hugepages (%d)\n",
+			 err);
+	else
+		drm_notice(&v3d->drm,
+			   "Transparent Hugepage support is recommended for optimal performance on this platform!\n");
+}
+
 int
 v3d_gem_init(struct drm_device *dev)
 {
@@ -310,7 +328,7 @@ v3d_gem_init(struct drm_device *dev)
 	v3d_init_hw_state(v3d);
 	v3d_mmu_set_page_table(v3d);
 
-	v3d_gemfs_init(v3d);
+	v3d_huge_mnt_init(v3d);
 
 	ret = v3d_sched_init(v3d);
 	if (ret) {
@@ -330,7 +348,6 @@ v3d_gem_destroy(struct drm_device *dev)
 	enum v3d_queue q;
 
 	v3d_sched_fini(v3d);
-	v3d_gemfs_fini(v3d);
 
 	/* Waiting for jobs to finish would need to be done before
 	 * unregistering V3D.
diff --git a/drivers/gpu/drm/v3d/v3d_gemfs.c b/drivers/gpu/drm/v3d/v3d_gemfs.c
deleted file mode 100644
index bf351fc0d488..000000000000
--- a/drivers/gpu/drm/v3d/v3d_gemfs.c
+++ /dev/null
@@ -1,62 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/* Copyright (C) 2024 Raspberry Pi */
-
-#include <linux/fs.h>
-#include <linux/mount.h>
-#include <linux/fs_context.h>
-
-#include <drm/drm_print.h>
-
-#include "v3d_drv.h"
-
-void v3d_gemfs_init(struct v3d_dev *v3d)
-{
-	struct file_system_type *type;
-	struct fs_context *fc;
-	struct vfsmount *gemfs;
-	int ret;
-
-	/*
-	 * By creating our own shmemfs mountpoint, we can pass in
-	 * mount flags that better match our usecase. However, we
-	 * only do so on platforms which benefit from it.
-	 */
-	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
-		goto err;
-
-	/* The user doesn't want to enable Super Pages */
-	if (!super_pages)
-		goto err;
-
-	type = get_fs_type("tmpfs");
-	if (!type)
-		goto err;
-
-	fc = fs_context_for_mount(type, SB_KERNMOUNT);
-	if (IS_ERR(fc))
-		goto err;
-	ret = vfs_parse_fs_string(fc, "source", "tmpfs");
-	if (!ret)
-		ret = vfs_parse_fs_string(fc, "huge", "within_size");
-	if (!ret)
-		gemfs = fc_mount_longterm(fc);
-	put_fs_context(fc);
-	if (ret)
-		goto err;
-
-	v3d->gemfs = gemfs;
-	drm_info(&v3d->drm, "Using Transparent Hugepages\n");
-
-	return;
-
-err:
-	v3d->gemfs = NULL;
-	drm_notice(&v3d->drm,
-		   "Transparent Hugepage support is recommended for optimal performance on this platform!\n");
-}
-
-void v3d_gemfs_fini(struct v3d_dev *v3d)
-{
-	if (v3d->gemfs)
-		kern_unmount(v3d->gemfs);
-}
diff --git a/drivers/gpu/drm/vgem/Kconfig b/drivers/gpu/drm/vgem/Kconfig
new file mode 100644
index 000000000000..c419cdadd54c
--- /dev/null
+++ b/drivers/gpu/drm/vgem/Kconfig
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config DRM_VGEM
+	tristate "Virtual GEM provider"
+	depends on DRM && MMU
+	select DRM_GEM_SHMEM_HELPER
+	help
+	  Choose this option to get a virtual graphics memory manager,
+	  as used by Mesa's software renderer for enhanced performance.
+	  If M is selected the module will be called vgem.
diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile
index 62be4a5227e4..7f08b4cd91d6 100644
--- a/drivers/gpu/drm/xe/Makefile
+++ b/drivers/gpu/drm/xe/Makefile
@@ -95,6 +95,7 @@ xe-y += xe_bb.o \
 	xe_oa.o \
 	xe_observation.o \
 	xe_pagefault.o \
+	xe_page_reclaim.o \
 	xe_pat.o \
 	xe_pci.o \
 	xe_pcode.o \
@@ -173,6 +174,7 @@ xe-$(CONFIG_PCI_IOV) += \
 	xe_lmtt.o \
 	xe_lmtt_2l.o \
 	xe_lmtt_ml.o \
+	xe_mert.o \
 	xe_pci_sriov.o \
 	xe_sriov_packet.o \
 	xe_sriov_pf.o \
@@ -195,16 +197,10 @@ endif
 
 # i915 Display compat #defines and #includes
 subdir-ccflags-$(CONFIG_DRM_XE_DISPLAY) += \
-	-I$(src)/display/ext \
 	-I$(src)/compat-i915-headers \
 	-I$(srctree)/drivers/gpu/drm/i915/display/ \
 	-Ddrm_i915_private=xe_device
 
-# Rule to build SOC code shared with i915
-$(obj)/i915-soc/%.o: $(srctree)/drivers/gpu/drm/i915/soc/%.c FORCE
-	$(call cmd,force_checksrc)
-	$(call if_changed_rule,cc_o_c)
-
 # Rule to build display code shared with i915
 $(obj)/i915-display/%.o: $(srctree)/drivers/gpu/drm/i915/display/%.c FORCE
 	$(call cmd,force_checksrc)
@@ -212,12 +208,10 @@ $(obj)/i915-display/%.o: $(srctree)/drivers/gpu/drm/i915/display/%.c FORCE
 
 # Display code specific to xe
 xe-$(CONFIG_DRM_XE_DISPLAY) += \
-	display/ext/i915_irq.o \
 	display/intel_bo.o \
 	display/intel_fb_bo.o \
 	display/intel_fbdev_fb.o \
 	display/xe_display.o \
-	display/xe_display_misc.o \
 	display/xe_display_rpm.o \
 	display/xe_display_wa.o \
 	display/xe_dsb_buffer.o \
@@ -228,11 +222,6 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \
 	display/xe_stolen.o \
 	display/xe_tdf.o
 
-# SOC code shared with i915
-xe-$(CONFIG_DRM_XE_DISPLAY) += \
-	i915-soc/intel_dram.o \
-	i915-soc/intel_rom.o
-
 # Display code shared with i915
 xe-$(CONFIG_DRM_XE_DISPLAY) += \
 	i915-display/icl_dsi.o \
@@ -267,6 +256,7 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \
 	i915-display/intel_display_power_map.o \
 	i915-display/intel_display_power_well.o \
 	i915-display/intel_display_rpm.o \
+	i915-display/intel_display_rps.o \
 	i915-display/intel_display_trace.o \
 	i915-display/intel_display_utils.o \
 	i915-display/intel_display_wa.o \
@@ -283,6 +273,7 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \
 	i915-display/intel_dpll.o \
 	i915-display/intel_dpll_mgr.o \
 	i915-display/intel_dpt_common.o \
+	i915-display/intel_dram.o \
 	i915-display/intel_drrs.o \
 	i915-display/intel_dsb.o \
 	i915-display/intel_dsi.o \
@@ -310,14 +301,16 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \
 	i915-display/intel_modeset_setup.o \
 	i915-display/intel_modeset_verify.o \
 	i915-display/intel_panel.o \
+	i915-display/intel_parent.o \
+	i915-display/intel_pch.o \
 	i915-display/intel_pfit.o \
 	i915-display/intel_plane.o \
 	i915-display/intel_pmdemand.o \
-	i915-display/intel_pch.o \
 	i915-display/intel_pps.o \
 	i915-display/intel_psr.o \
 	i915-display/intel_qp_tables.o \
 	i915-display/intel_quirks.o \
+	i915-display/intel_rom.o \
 	i915-display/intel_snps_hdmi_pll.o \
 	i915-display/intel_snps_phy.o \
 	i915-display/intel_tc.o \
diff --git a/drivers/gpu/drm/xe/abi/guc_actions_abi.h b/drivers/gpu/drm/xe/abi/guc_actions_abi.h
index 47756e4674a1..83a6e7794982 100644
--- a/drivers/gpu/drm/xe/abi/guc_actions_abi.h
+++ b/drivers/gpu/drm/xe/abi/guc_actions_abi.h
@@ -139,6 +139,10 @@ enum xe_guc_action {
 	XE_GUC_ACTION_DEREGISTER_G2G = 0x4508,
 	XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE = 0x4600,
 	XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC = 0x4601,
+	XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE = 0x4602,
+	XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC = 0x4603,
+	XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE = 0x4604,
+	XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR = 0x4605,
 	XE_GUC_ACTION_CLIENT_SOFT_RESET = 0x5507,
 	XE_GUC_ACTION_SET_ENG_UTIL_BUFF = 0x550A,
 	XE_GUC_ACTION_SET_DEVICE_ENGINE_ACTIVITY_BUFFER = 0x550C,
@@ -151,6 +155,8 @@ enum xe_guc_action {
 	XE_GUC_ACTION_TLB_INVALIDATION = 0x7000,
 	XE_GUC_ACTION_TLB_INVALIDATION_DONE = 0x7001,
 	XE_GUC_ACTION_TLB_INVALIDATION_ALL = 0x7002,
+	XE_GUC_ACTION_PAGE_RECLAMATION = 0x7003,
+	XE_GUC_ACTION_PAGE_RECLAMATION_DONE = 0x7004,
 	XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION = 0x8002,
 	XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE = 0x8003,
 	XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED = 0x8004,
diff --git a/drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h b/drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h
index 0b28659d94e9..d9f21202e1a9 100644
--- a/drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h
+++ b/drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h
@@ -502,13 +502,17 @@
 #define VF2GUC_VF_RESET_RESPONSE_MSG_0_MBZ		GUC_HXG_RESPONSE_MSG_0_DATA0
 
 /**
- * DOC: VF2GUC_NOTIFY_RESFIX_DONE
+ * DOC: VF2GUC_RESFIX_DONE
  *
- * This action is used by VF to notify the GuC that the VF KMD has completed
- * post-migration recovery steps.
+ * This action is used by VF to inform the GuC that the VF KMD has completed
+ * post-migration recovery steps. From GuC VF compatibility 1.27.0 onwards, it
+ * shall only be sent after posting RESFIX_START and that both @MARKER fields
+ * must match.
  *
  * This message must be sent as `MMIO HXG Message`_.
  *
+ * Updated since GuC VF compatibility 1.27.0.
+ *
  *  +---+-------+--------------------------------------------------------------+
  *  |   | Bits  | Description                                                  |
  *  +===+=======+==============================================================+
@@ -516,9 +520,11 @@
  *  |   +-------+--------------------------------------------------------------+
  *  |   | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_                                 |
  *  |   +-------+--------------------------------------------------------------+
- *  |   | 27:16 | DATA0 = MBZ                                                  |
+ *  |   | 27:16 | DATA0 = MARKER = MBZ (only prior 1.27.0)                     |
  *  |   +-------+--------------------------------------------------------------+
- *  |   |  15:0 | ACTION = _`GUC_ACTION_VF2GUC_NOTIFY_RESFIX_DONE` = 0x5508    |
+ *  |   | 27:16 | DATA0 = MARKER - can't be zero (1.27.0+)                     |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   |  15:0 | ACTION = _`GUC_ACTION_VF2GUC_RESFIX_DONE` = 0x5508           |
  *  +---+-------+--------------------------------------------------------------+
  *
  *  +---+-------+--------------------------------------------------------------+
@@ -531,13 +537,13 @@
  *  |   |  27:0 | DATA0 = MBZ                                                  |
  *  +---+-------+--------------------------------------------------------------+
  */
-#define GUC_ACTION_VF2GUC_NOTIFY_RESFIX_DONE		0x5508u
+#define GUC_ACTION_VF2GUC_RESFIX_DONE			0x5508u
 
-#define VF2GUC_NOTIFY_RESFIX_DONE_REQUEST_MSG_LEN	GUC_HXG_REQUEST_MSG_MIN_LEN
-#define VF2GUC_NOTIFY_RESFIX_DONE_REQUEST_MSG_0_MBZ	GUC_HXG_REQUEST_MSG_0_DATA0
+#define VF2GUC_RESFIX_DONE_REQUEST_MSG_LEN		GUC_HXG_REQUEST_MSG_MIN_LEN
+#define VF2GUC_RESFIX_DONE_REQUEST_MSG_0_MARKER		GUC_HXG_REQUEST_MSG_0_DATA0
 
-#define VF2GUC_NOTIFY_RESFIX_DONE_RESPONSE_MSG_LEN	GUC_HXG_RESPONSE_MSG_MIN_LEN
-#define VF2GUC_NOTIFY_RESFIX_DONE_RESPONSE_MSG_0_MBZ	GUC_HXG_RESPONSE_MSG_0_DATA0
+#define VF2GUC_RESFIX_DONE_RESPONSE_MSG_LEN		GUC_HXG_RESPONSE_MSG_MIN_LEN
+#define VF2GUC_RESFIX_DONE_RESPONSE_MSG_0_MBZ		GUC_HXG_RESPONSE_MSG_0_DATA0
 
 /**
  * DOC: VF2GUC_QUERY_SINGLE_KLV
@@ -656,4 +662,45 @@
 #define PF2GUC_SAVE_RESTORE_VF_RESPONSE_MSG_LEN		GUC_HXG_RESPONSE_MSG_MIN_LEN
 #define PF2GUC_SAVE_RESTORE_VF_RESPONSE_MSG_0_USED	GUC_HXG_RESPONSE_MSG_0_DATA0
 
+/**
+ * DOC: VF2GUC_RESFIX_START
+ *
+ * This action is used by VF to inform the GuC that the VF KMD will be starting
+ * post-migration recovery fixups. The @MARKER sent with this action must match
+ * with the MARKER posted in the VF2GUC_RESFIX_DONE message.
+ *
+ * This message must be sent as `MMIO HXG Message`_.
+ *
+ * Available since GuC VF compatibility 1.27.0.
+ *
+ *  +---+-------+--------------------------------------------------------------+
+ *  |   | Bits  | Description                                                  |
+ *  +===+=======+==============================================================+
+ *  | 0 |    31 | ORIGIN = GUC_HXG_ORIGIN_HOST_                                |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_                                 |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   | 27:16 | DATA0 = MARKER - can't be zero                               |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   |  15:0 | ACTION = _`GUC_ACTION_VF2GUC_RESFIX_START` = 0x550F          |
+ *  +---+-------+--------------------------------------------------------------+
+ *
+ *  +---+-------+--------------------------------------------------------------+
+ *  |   | Bits  | Description                                                  |
+ *  +===+=======+==============================================================+
+ *  | 0 |    31 | ORIGIN = GUC_HXG_ORIGIN_GUC_                                 |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_                        |
+ *  |   +-------+--------------------------------------------------------------+
+ *  |   |  27:0 | DATA0 = MBZ                                                  |
+ *  +---+-------+--------------------------------------------------------------+
+ */
+#define GUC_ACTION_VF2GUC_RESFIX_START			0x550Fu
+
+#define VF2GUC_RESFIX_START_REQUEST_MSG_LEN		GUC_HXG_REQUEST_MSG_MIN_LEN
+#define VF2GUC_RESFIX_START_REQUEST_MSG_0_MARKER	GUC_HXG_REQUEST_MSG_0_DATA0
+
+#define VF2GUC_RESFIX_START_RESPONSE_MSG_LEN		GUC_HXG_RESPONSE_MSG_MIN_LEN
+#define VF2GUC_RESFIX_START_RESPONSE_MSG_0_MBZ		GUC_HXG_RESPONSE_MSG_0_DATA0
+
 #endif
diff --git a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h
index 265a135e7061..89a4f8c504e6 100644
--- a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h
+++ b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h
@@ -352,6 +352,12 @@ enum  {
  *      :1: NORMAL = schedule VF always, irrespective of whether it has work or not
  *      :2: HIGH = schedule VF in the next time-slice after current active
  *          time-slice completes if it has active work
+ *
+ * _`GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT` : 0x8A0D
+ *      Given that multi-LRC contexts are incompatible with SRIOV scheduler
+ *      groups and cause the latter to be turned off when registered with the
+ *      GuC, this config allows the PF to set a threshold for multi-LRC context
+ *      registrations by VFs to monitor their behavior.
  */
 
 #define GUC_KLV_VF_CFG_GGTT_START_KEY		0x0001
@@ -410,6 +416,9 @@ enum  {
 #define   GUC_SCHED_PRIORITY_NORMAL		1u
 #define   GUC_SCHED_PRIORITY_HIGH		2u
 
+#define GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT_KEY	0x8a0d
+#define GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT_LEN	1u
+
 /*
  * Workaround keys:
  */
diff --git a/drivers/gpu/drm/xe/abi/guc_lfd_abi.h b/drivers/gpu/drm/xe/abi/guc_lfd_abi.h
new file mode 100644
index 000000000000..b6ed20d5b508
--- /dev/null
+++ b/drivers/gpu/drm/xe/abi/guc_lfd_abi.h
@@ -0,0 +1,171 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2025 Intel Corporation
+ */
+
+#ifndef _ABI_GUC_LFD_ABI_H_
+#define _ABI_GUC_LFD_ABI_H_
+
+#include <linux/types.h>
+
+#include "guc_lic_abi.h"
+
+/* The current major version of GuC-Log-File format. */
+#define GUC_LFD_FORMAT_VERSION_MAJOR		0x0001
+/* The current minor version of GuC-Log-File format. */
+#define GUC_LFD_FORMAT_VERSION_MINOR		0x0000
+
+/** enum guc_lfd_type - Log format descriptor type */
+enum guc_lfd_type {
+	/**
+	 * @GUC_LFD_TYPE_FW_REQUIRED_RANGE_START: Start of range for
+	 * required LFDs from GuC
+	 * @GUC_LFD_TYPE_FW_VERSION: GuC Firmware Version structure.
+	 * @GUC_LFD_TYPE_GUC_DEVICE_ID: GuC microcontroller device ID.
+	 * @GUC_LFD_TYPE_TSC_FREQUENCY: Frequency of GuC timestamps.
+	 * @GUC_LFD_TYPE_GMD_ID: HW GMD ID.
+	 * @GUC_LFD_TYPE_BUILD_PLATFORM_ID: GuC build platform ID.
+	 * @GUC_LFD_TYPE_FW_REQUIRED_RANGE_END: End of range for
+	 * required LFDs from GuC
+	 */
+	GUC_LFD_TYPE_FW_REQUIRED_RANGE_START	= 0x1,
+	GUC_LFD_TYPE_FW_VERSION			= 0x1,
+	GUC_LFD_TYPE_GUC_DEVICE_ID		= 0x2,
+	GUC_LFD_TYPE_TSC_FREQUENCY		= 0x3,
+	GUC_LFD_TYPE_GMD_ID			= 0x4,
+	GUC_LFD_TYPE_BUILD_PLATFORM_ID		= 0x5,
+	GUC_LFD_TYPE_FW_REQUIRED_RANGE_END	= 0x1FFF,
+
+	/**
+	 * @GUC_LFD_TYPE_FW_OPTIONAL_RANGE_START: Start of range for
+	 * optional LFDs from GuC
+	 * @GUC_LFD_TYPE_LOG_EVENTS_BUFFER: Log-event-entries buffer.
+	 * @GUC_LFD_TYPE_FW_CRASH_DUMP: GuC generated crash-dump blob.
+	 * @GUC_LFD_TYPE_FW_OPTIONAL_RANGE_END: End of range for
+	 * optional LFDs from GuC
+	 */
+	GUC_LFD_TYPE_FW_OPTIONAL_RANGE_START	= 0x2000,
+	GUC_LFD_TYPE_LOG_EVENTS_BUFFER		= 0x2000,
+	GUC_LFD_TYPE_FW_CRASH_DUMP		= 0x2001,
+	GUC_LFD_TYPE_FW_OPTIONAL_RANGE_END	= 0x3FFF,
+
+	/**
+	 * @GUC_LFD_TYPE_KMD_REQUIRED_RANGE_START: Start of range for
+	 * required KMD LFDs
+	 * @GUC_LFD_TYPE_OS_ID: An identifier for the OS.
+	 * @GUC_LFD_TYPE_KMD_REQUIRED_RANGE_END: End of this range for
+	 * required KMD LFDs
+	 */
+	GUC_LFD_TYPE_KMD_REQUIRED_RANGE_START	= 0x4000,
+	GUC_LFD_TYPE_OS_ID			= 0x4000,
+	GUC_LFD_TYPE_KMD_REQUIRED_RANGE_END	= 0x5FFF,
+
+	/**
+	 * @GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_START: Start of range for
+	 * optional KMD LFDs
+	 * @GUC_LFD_TYPE_BINARY_SCHEMA_FORMAT: Binary representation of
+	 * GuC log-events schema.
+	 * @GUC_LFD_TYPE_HOST_COMMENT: ASCII string containing comments
+	 * from the host/KMD.
+	 * @GUC_LFD_TYPE_TIMESTAMP_ANCHOR: A timestamp anchor, to convert
+	 * between host and GuC timestamp.
+	 * @GUC_LFD_TYPE_TIMESTAMP_ANCHOR_CONFIG: Timestamp anchor
+	 * configuration, definition of timestamp frequency and bit width.
+	 * @GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_END: End of this range for
+	 * optional KMD LFDs
+	 */
+	GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_START	= 0x6000,
+	GUC_LFD_TYPE_BINARY_SCHEMA_FORMAT	= 0x6000,
+	GUC_LFD_TYPE_HOST_COMMENT		= 0x6001,
+	GUC_LFD_TYPE_TIMESTAMP_ANCHOR		= 0x6002,
+	GUC_LFD_TYPE_TIMESTAMP_ANCHOR_CONFIG	= 0x6003,
+	GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_END	= 0x7FFF,
+
+	/*
+	 * @GUC_LFD_TYPE_RESERVED_RANGE_START: Start of reserved range
+	 * @GUC_LFD_TYPE_RESERVED_RANGE_END: End of reserved range
+	 */
+	GUC_LFD_TYPE_RESERVED_RANGE_START	= 0x8000,
+	GUC_LFD_TYPE_RESERVED_RANGE_END		= 0xFFFF,
+};
+
+/** enum guc_lfd_os_type - OS Type LFD-ID */
+enum guc_lfd_os_type {
+	/** @GUC_LFD_OS_TYPE_OSID_WIN: Windows OS */
+	GUC_LFD_OS_TYPE_OSID_WIN = 0x1,
+	/** @GUC_LFD_OS_TYPE_OSID_LIN: Linux OS */
+	GUC_LFD_OS_TYPE_OSID_LIN = 0x2,
+	/** @GUC_LFD_OS_TYPE_OSID_VMW: VMWare OS */
+	GUC_LFD_OS_TYPE_OSID_VMW = 0x3,
+	/** @GUC_LFD_OS_TYPE_OSID_OTHER: Other */
+	GUC_LFD_OS_TYPE_OSID_OTHER = 0x4,
+};
+
+/** struct guc_lfd_data - A generic header structure for all LFD blocks */
+struct guc_lfd_data {
+	/** @header: A 32 bits dword, contains multiple bit fields */
+	u32 header;
+	/* LFD type. See guc_lfd_type */
+#define GUC_LFD_DATA_HEADER_MASK_TYPE		GENMASK(31, 16)
+#define GUC_LFD_DATA_HEADER_MASK_MAGIC		GENMASK(15, 0)
+
+	/** @data_count: Number of dwords the `data` field contains. */
+	u32 data_count;
+	/** @data: Data defined by GUC_LFD_DATA_HEADER_MASK_TYPE */
+	u32 data[] __counted_by(data_count);
+} __packed;
+
+/**
+ * struct guc_lfd_data_log_events_buf - GuC Log Events Buffer.
+ * This is optional fw LFD data
+ */
+struct guc_lfd_data_log_events_buf {
+	/**
+	 * @log_events_format_version: version of GuC log format of buffer
+	 */
+	u32 log_events_format_version;
+	/**
+	 * @log_event: The log event data.
+	 * Size in dwords is LFD block size - 1.
+	 */
+	u32 log_event[];
+} __packed;
+
+/** struct guc_lfd_data_os_info - OS Version Information. */
+struct guc_lfd_data_os_info {
+	/**
+	 * @os_id: enum values to identify the OS brand.
+	 * See guc_lfd_os_type for the range of types
+	 */
+	u32 os_id;
+	/**
+	 * @build_version: ASCII string containing OS build version
+	 * information based on os_id. String is padded with null
+	 * characters to ensure its DWORD aligned.
+	 * Size in dwords is LFD block size - 1.
+	 */
+	char build_version[];
+} __packed;
+
+/**
+ * struct guc_logfile_header - Header of GuC Log Streaming-LFD-File Format.
+ * This structure encapsulates the layout of the guc-log-file format
+ */
+struct guc_lfd_file_header {
+	/**
+	 * @magic: A magic number set by producer of a GuC log file to
+	 * identify that file is a valid guc-log-file containing a stream
+	 * of LFDs.
+	 */
+	u64 magic;
+	/** @version: Version of this file format layout */
+	u32 version;
+#define GUC_LFD_FILE_HEADER_VERSION_MASK_MAJOR	GENMASK(31, 16)
+#define GUC_LFD_FILE_HEADER_VERSION_MASK_MINOR	GENMASK(15, 0)
+
+	/** @stream: A stream of one or more guc_lfd_data LFD blocks
+	 */
+	u32 stream[];
+} __packed;
+
+#endif
diff --git a/drivers/gpu/drm/xe/abi/guc_lic_abi.h b/drivers/gpu/drm/xe/abi/guc_lic_abi.h
new file mode 100644
index 000000000000..9169644093a2
--- /dev/null
+++ b/drivers/gpu/drm/xe/abi/guc_lic_abi.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2025 Intel Corporation
+ */
+
+#ifndef _ABI_GUC_LIC_ABI_H_
+#define _ABI_GUC_LIC_ABI_H_
+
+#include <linux/types.h>
+
+/**
+ * enum guc_lic_type - Log Init Config KLV IDs.
+ */
+enum guc_lic_type {
+	/**
+	 * @GUC_LIC_TYPE_GUC_SW_VERSION: GuC firmware version. Value
+	 * is a 32 bit number represented by guc_sw_version.
+	 */
+	GUC_LIC_TYPE_GUC_SW_VERSION = 0x1,
+	/**
+	 * @GUC_LIC_TYPE_GUC_DEVICE_ID: GuC device id. Value is a 32
+	 * bit.
+	 */
+	GUC_LIC_TYPE_GUC_DEVICE_ID = 0x2,
+	/**
+	 * @GUC_LIC_TYPE_TSC_FREQUENCY: GuC timestamp counter
+	 * frequency. Value is a 32 bit number representing frequency in
+	 * kHz. This timestamp is utilized in log entries, timer and
+	 * for engine utilization tracking.
+	 */
+	GUC_LIC_TYPE_TSC_FREQUENCY = 0x3,
+	/**
+	 * @GUC_LIC_TYPE_GMD_ID: HW GMD ID. Value is a 32 bit number
+	 * representing graphics, media and display HW architecture IDs.
+	 */
+	GUC_LIC_TYPE_GMD_ID = 0x4,
+	/**
+	 * @GUC_LIC_TYPE_BUILD_PLATFORM_ID: GuC build platform ID.
+	 * Value is 32 bits.
+	 */
+	GUC_LIC_TYPE_BUILD_PLATFORM_ID = 0x5,
+};
+
+/**
+ * struct guc_lic - GuC LIC (Log-Init-Config) structure.
+ *
+ * This is populated by the GUC at log init time and is located in the log
+ * buffer memory allocation.
+ */
+struct guc_lic {
+	/**
+	 * @magic: A magic number set by GuC to identify that this
+	 * structure contains valid information: magic = GUC_LIC_MAGIC.
+	 */
+	u32 magic;
+#define GUC_LIC_MAGIC			0x8086900D
+	/**
+	 * @version: The version of the this structure.
+	 * Major and minor version number are represented as bit fields.
+	 */
+	u32 version;
+#define GUC_LIC_VERSION_MASK_MAJOR		GENMASK(31, 16)
+#define GUC_LIC_VERSION_MASK_MINOR		GENMASK(15, 0)
+
+#define GUC_LIC_VERSION_MAJOR	1u
+#define GUC_LIC_VERSION_MINOR	0u
+
+	/** @data_count: Number of dwords the `data` array contains. */
+	u32 data_count;
+	/**
+	 * @data: Array of dwords representing a list of LIC KLVs of
+	 * type guc_klv_generic with keys represented by guc_lic_type
+	 */
+	u32 data[] __counted_by(data_count);
+} __packed;
+
+#endif
diff --git a/drivers/gpu/drm/xe/abi/guc_log_abi.h b/drivers/gpu/drm/xe/abi/guc_log_abi.h
index 554630b7ccd9..fbf212d59a40 100644
--- a/drivers/gpu/drm/xe/abi/guc_log_abi.h
+++ b/drivers/gpu/drm/xe/abi/guc_log_abi.h
@@ -8,11 +8,45 @@
 
 #include <linux/types.h>
 
+/**
+ * DOC: GuC Log buffer Layout
+ *
+ * The in-memory log buffer layout is as follows::
+ *
+ *  +===============================+	      0000h
+ *  |    Crash dump state header    |		^
+ *  +-------------------------------+ 32B	|
+ *  |      Debug state header       |		|
+ *  +-------------------------------+ 64B	4KB
+ *  |     Capture state header      |		|
+ *  +-------------------------------+ 96B	|
+ *  |                               |		v
+ *  +===============================+ <--- EVENT_DATA_OFFSET
+ *  |  Event logs(raw data)         |		^
+ *  |                               |		|
+ *  |                               | EVENT_DATA_BUFFER_SIZE
+ *  |                               |		|
+ *  |                               |		v
+ *  +===============================+ <--- CRASH_DUMP_OFFSET
+ *  | Crash Dump(raw data)          |		^
+ *  |                               |		|
+ *  |                               | CRASH_DUMP_BUFFER_SIZE
+ *  |                               |		|
+ *  |                               |		v
+ *  +===============================+ <--- STATE_CAPTURE_OFFSET
+ *  | Error state capture(raw data) |		^
+ *  |                               |		|
+ *  |                               | STATE_CAPTURE_BUFFER_SIZE
+ *  |                               |		|
+ *  |                               |		v
+ *  +===============================+ Total: GUC_LOG_SIZE
+ */
+
 /* GuC logging buffer types */
-enum guc_log_buffer_type {
-	GUC_LOG_BUFFER_CRASH_DUMP,
-	GUC_LOG_BUFFER_DEBUG,
-	GUC_LOG_BUFFER_CAPTURE,
+enum guc_log_type {
+	GUC_LOG_TYPE_EVENT_DATA,
+	GUC_LOG_TYPE_CRASH_DUMP,
+	GUC_LOG_TYPE_STATE_CAPTURE,
 };
 
 #define GUC_LOG_BUFFER_TYPE_MAX		3
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/gem/i915_gem_object.h b/drivers/gpu/drm/xe/compat-i915-headers/gem/i915_gem_object.h
deleted file mode 100644
index 0548b2e0316f..000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/gem/i915_gem_object.h
+++ /dev/null
@@ -1,13 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/* Copyright © 2025 Intel Corporation */
-
-#ifndef __I915_GEM_OBJECT_H__
-#define __I915_GEM_OBJECT_H__
-
-struct dma_fence;
-
-static inline void i915_gem_fence_wait_priority_display(struct dma_fence *fence)
-{
-}
-
-#endif
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/gem/i915_gem_stolen.h b/drivers/gpu/drm/xe/compat-i915-headers/gem/i915_gem_stolen.h
deleted file mode 100644
index 48e3256ba37e..000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/gem/i915_gem_stolen.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2024 Intel Corporation
- */
-
-#ifndef _I915_GEM_STOLEN_H_
-#define _I915_GEM_STOLEN_H_
-
-#include <linux/types.h>
-
-struct drm_device;
-struct intel_stolen_node;
-
-int i915_gem_stolen_insert_node_in_range(struct intel_stolen_node *node, u64 size,
-					 unsigned int align, u64 start, u64 end);
-
-int i915_gem_stolen_insert_node(struct intel_stolen_node *node, u64 size,
-				unsigned int align);
-
-void i915_gem_stolen_remove_node(struct intel_stolen_node *node);
-
-bool i915_gem_stolen_initialized(struct drm_device *drm);
-
-bool i915_gem_stolen_node_allocated(const struct intel_stolen_node *node);
-
-u32 i915_gem_stolen_node_offset(struct intel_stolen_node *node);
-
-u64 i915_gem_stolen_area_address(struct drm_device *drm);
-
-u64 i915_gem_stolen_area_size(struct drm_device *drm);
-
-u64 i915_gem_stolen_node_address(struct intel_stolen_node *node);
-
-u64 i915_gem_stolen_node_size(const struct intel_stolen_node *node);
-
-struct intel_stolen_node *i915_gem_stolen_node_alloc(struct drm_device *drm);
-
-void i915_gem_stolen_node_free(const struct intel_stolen_node *node);
-
-#endif
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/gt/intel_gt_types.h b/drivers/gpu/drm/xe/compat-i915-headers/gt/intel_gt_types.h
deleted file mode 100644
index c15806d6c4f7..000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/gt/intel_gt_types.h
+++ /dev/null
@@ -1,11 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#ifndef __INTEL_GT_TYPES__
-#define __INTEL_GT_TYPES__
-
-#define intel_gt_support_legacy_fencing(gt) 0
-
-#endif
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_active.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_active.h
deleted file mode 100644
index 6f0ab3753563..000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/i915_active.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2022 Intel Corporation
- */
-
-#ifndef _I915_ACTIVE_H_
-#define _I915_ACTIVE_H_
-
-#include "i915_active_types.h"
-
-static inline void i915_active_init(struct i915_active *ref,
-				    int (*active)(struct i915_active *ref),
-				    void (*retire)(struct i915_active *ref),
-				    unsigned long flags)
-{
-	(void) active;
-	(void) retire;
-}
-
-#define i915_active_fini(active) do { } while (0)
-
-#endif
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_active_types.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_active_types.h
deleted file mode 100644
index 8c31f9a8b168..000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/i915_active_types.h
+++ /dev/null
@@ -1,13 +0,0 @@
-/*
- * SPDX-License-Identifier: MIT
- *
- * Copyright © 2019 Intel Corporation
- */
-
-#ifndef _I915_ACTIVE_TYPES_H_
-#define _I915_ACTIVE_TYPES_H_
-
-struct i915_active {};
-#define I915_ACTIVE_RETIRE_SLEEPS 0
-
-#endif /* _I915_ACTIVE_TYPES_H_ */
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_drv.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_drv.h
index 3e79a74ff7de..04d1925f9a19 100644
--- a/drivers/gpu/drm/xe/compat-i915-headers/i915_drv.h
+++ b/drivers/gpu/drm/xe/compat-i915-headers/i915_drv.h
@@ -19,19 +19,4 @@ static inline struct drm_i915_private *to_i915(const struct drm_device *dev)
 	return container_of(dev, struct drm_i915_private, drm);
 }
 
-/* compat platform checks only for soc/ usage */
-#define IS_PLATFORM(xe, x) ((xe)->info.platform == x)
-#define IS_I915G(dev_priv)	(dev_priv && 0)
-#define IS_I915GM(dev_priv)	(dev_priv && 0)
-#define IS_PINEVIEW(dev_priv)	(dev_priv && 0)
-#define IS_VALLEYVIEW(dev_priv)	(dev_priv && 0)
-#define IS_CHERRYVIEW(dev_priv)	(dev_priv && 0)
-#define IS_HASWELL(dev_priv)	(dev_priv && 0)
-#define IS_BROADWELL(dev_priv)	(dev_priv && 0)
-#define IS_BROXTON(dev_priv)	(dev_priv && 0)
-#define IS_GEMINILAKE(dev_priv)	(dev_priv && 0)
-#define IS_DG2(dev_priv)	IS_PLATFORM(dev_priv, XE_DG2)
-
-#define IS_MOBILE(xe) (xe && 0)
-
 #endif
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_irq.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_irq.h
deleted file mode 100644
index 61707a07f91f..000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/i915_irq.h
+++ /dev/null
@@ -1,6 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#include "../../i915/i915_irq.h"
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_utils.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_utils.h
index bcd441dc0fce..3639721f0bf8 100644
--- a/drivers/gpu/drm/xe/compat-i915-headers/i915_utils.h
+++ b/drivers/gpu/drm/xe/compat-i915-headers/i915_utils.h
@@ -3,11 +3,5 @@
  * Copyright © 2023 Intel Corporation
  */
 
-/* for soc/ */
-#ifndef MISSING_CASE
-#define MISSING_CASE(x) WARN(1, "Missing case (%s == %ld)\n", \
-			     __stringify(x), (long)(x))
-#endif
-
 /* for a couple of users under i915/display */
 #define i915_inject_probe_failure(unused) ((unused) && 0)
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_vgpu.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_vgpu.h
deleted file mode 100644
index 4931c7198f13..000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/i915_vgpu.h
+++ /dev/null
@@ -1,18 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#ifndef _I915_VGPU_H_
-#define _I915_VGPU_H_
-
-#include <linux/types.h>
-
-struct drm_i915_private;
-
-static inline bool intel_vgpu_active(struct drm_i915_private *i915)
-{
-	return false;
-}
-
-#endif /* _I915_VGPU_H_ */
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_wakeref.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_wakeref.h
deleted file mode 100644
index 2a32faea9db5..000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/intel_wakeref.h
+++ /dev/null
@@ -1,10 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#include <linux/types.h>
-
-typedef struct ref_tracker *intel_wakeref_t;
-
-#define INTEL_WAKEREF_DEF ERR_PTR(-ENOENT)
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/pxp/intel_pxp.h b/drivers/gpu/drm/xe/compat-i915-headers/pxp/intel_pxp.h
deleted file mode 100644
index 97fd0ddf0b3a..000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/pxp/intel_pxp.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#ifndef __INTEL_PXP_H__
-#define __INTEL_PXP_H__
-
-#include <linux/errno.h>
-#include <linux/types.h>
-
-#include "xe_pxp.h"
-
-struct drm_gem_object;
-
-static inline int intel_pxp_key_check(struct drm_gem_object *obj, bool assign)
-{
-	/*
-	 * The assign variable is used in i915 to assign the key to the BO at
-	 * first submission time. In Xe the key is instead assigned at BO
-	 * creation time, so the assign variable must always be false.
-	 */
-	if (assign)
-		return -EINVAL;
-
-	return xe_pxp_obj_key_check(obj);
-}
-
-#endif
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/soc/intel_dram.h b/drivers/gpu/drm/xe/compat-i915-headers/soc/intel_dram.h
deleted file mode 100644
index 65707e20c557..000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/soc/intel_dram.h
+++ /dev/null
@@ -1,6 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#include "../../../i915/soc/intel_dram.h"
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/soc/intel_gmch.h b/drivers/gpu/drm/xe/compat-i915-headers/soc/intel_gmch.h
deleted file mode 100644
index 33c5257b3a71..000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/soc/intel_gmch.h
+++ /dev/null
@@ -1,6 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#include "../../../i915/soc/intel_gmch.h"
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/soc/intel_rom.h b/drivers/gpu/drm/xe/compat-i915-headers/soc/intel_rom.h
deleted file mode 100644
index 05cbfb697b2b..000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/soc/intel_rom.h
+++ /dev/null
@@ -1,6 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2024 Intel Corporation
- */
-
-#include "../../../i915/soc/intel_rom.h"
diff --git a/drivers/gpu/drm/xe/display/ext/i915_irq.c b/drivers/gpu/drm/xe/display/ext/i915_irq.c
deleted file mode 100644
index 3c6bca66ddab..000000000000
--- a/drivers/gpu/drm/xe/display/ext/i915_irq.c
+++ /dev/null
@@ -1,85 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#include "i915_irq.h"
-#include "i915_reg.h"
-#include "intel_uncore.h"
-
-void gen2_irq_reset(struct intel_uncore *uncore, struct i915_irq_regs regs)
-{
-	intel_uncore_write(uncore, regs.imr, 0xffffffff);
-	intel_uncore_posting_read(uncore, regs.imr);
-
-	intel_uncore_write(uncore, regs.ier, 0);
-
-	/* IIR can theoretically queue up two events. Be paranoid. */
-	intel_uncore_write(uncore, regs.iir, 0xffffffff);
-	intel_uncore_posting_read(uncore, regs.iir);
-	intel_uncore_write(uncore, regs.iir, 0xffffffff);
-	intel_uncore_posting_read(uncore, regs.iir);
-}
-
-/*
- * We should clear IMR at preinstall/uninstall, and just check at postinstall.
- */
-void gen2_assert_iir_is_zero(struct intel_uncore *uncore, i915_reg_t reg)
-{
-	struct xe_device *xe = container_of(uncore, struct xe_device, uncore);
-	u32 val = intel_uncore_read(uncore, reg);
-
-	if (val == 0)
-		return;
-
-	drm_WARN(&xe->drm, 1,
-		 "Interrupt register 0x%x is not zero: 0x%08x\n",
-		 i915_mmio_reg_offset(reg), val);
-	intel_uncore_write(uncore, reg, 0xffffffff);
-	intel_uncore_posting_read(uncore, reg);
-	intel_uncore_write(uncore, reg, 0xffffffff);
-	intel_uncore_posting_read(uncore, reg);
-}
-
-void gen2_irq_init(struct intel_uncore *uncore, struct i915_irq_regs regs,
-		   u32 imr_val, u32 ier_val)
-{
-	gen2_assert_iir_is_zero(uncore, regs.iir);
-
-	intel_uncore_write(uncore, regs.ier, ier_val);
-	intel_uncore_write(uncore, regs.imr, imr_val);
-	intel_uncore_posting_read(uncore, regs.imr);
-}
-
-void gen2_error_reset(struct intel_uncore *uncore, struct i915_error_regs regs)
-{
-	intel_uncore_write(uncore, regs.emr, 0xffffffff);
-	intel_uncore_posting_read(uncore, regs.emr);
-
-	intel_uncore_write(uncore, regs.eir, 0xffffffff);
-	intel_uncore_posting_read(uncore, regs.eir);
-	intel_uncore_write(uncore, regs.eir, 0xffffffff);
-	intel_uncore_posting_read(uncore, regs.eir);
-}
-
-void gen2_error_init(struct intel_uncore *uncore, struct i915_error_regs regs,
-		     u32 emr_val)
-{
-	intel_uncore_write(uncore, regs.eir, 0xffffffff);
-	intel_uncore_posting_read(uncore, regs.eir);
-	intel_uncore_write(uncore, regs.eir, 0xffffffff);
-	intel_uncore_posting_read(uncore, regs.eir);
-
-	intel_uncore_write(uncore, regs.emr, emr_val);
-	intel_uncore_posting_read(uncore, regs.emr);
-}
-
-bool intel_irqs_enabled(struct xe_device *xe)
-{
-	return atomic_read(&xe->irq.enabled);
-}
-
-void intel_synchronize_irq(struct xe_device *xe)
-{
-	synchronize_irq(to_pci_dev(xe->drm.dev)->irq);
-}
diff --git a/drivers/gpu/drm/xe/display/intel_bo.c b/drivers/gpu/drm/xe/display/intel_bo.c
index bad2243b9114..e8049a255d21 100644
--- a/drivers/gpu/drm/xe/display/intel_bo.c
+++ b/drivers/gpu/drm/xe/display/intel_bo.c
@@ -3,9 +3,10 @@
 
 #include <drm/drm_gem.h>
 
-#include "xe_bo.h"
 #include "intel_bo.h"
 #include "intel_frontbuffer.h"
+#include "xe_bo.h"
+#include "xe_pxp.h"
 
 bool intel_bo_is_tiled(struct drm_gem_object *obj)
 {
@@ -29,6 +30,11 @@ bool intel_bo_is_protected(struct drm_gem_object *obj)
 	return xe_bo_is_protected(gem_to_xe_bo(obj));
 }
 
+int intel_bo_key_check(struct drm_gem_object *obj)
+{
+	return xe_pxp_obj_key_check(obj);
+}
+
 int intel_bo_fb_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
 {
 	return drm_gem_prime_mmap(obj, vma);
diff --git a/drivers/gpu/drm/xe/display/xe_display.c b/drivers/gpu/drm/xe/display/xe_display.c
index 8b0afa270216..eda65a05f601 100644
--- a/drivers/gpu/drm/xe/display/xe_display.c
+++ b/drivers/gpu/drm/xe/display/xe_display.c
@@ -17,7 +17,6 @@
 #include <drm/intel/display_parent_interface.h>
 #include <uapi/drm/xe_drm.h>
 
-#include "soc/intel_dram.h"
 #include "intel_acpi.h"
 #include "intel_audio.h"
 #include "intel_bw.h"
@@ -29,6 +28,7 @@
 #include "intel_dmc.h"
 #include "intel_dmc_wl.h"
 #include "intel_dp.h"
+#include "intel_dram.h"
 #include "intel_encoder.h"
 #include "intel_fbdev.h"
 #include "intel_hdcp.h"
@@ -36,7 +36,10 @@
 #include "intel_opregion.h"
 #include "skl_watermark.h"
 #include "xe_display_rpm.h"
+#include "xe_hdcp_gsc.h"
 #include "xe_module.h"
+#include "xe_panic.h"
+#include "xe_stolen.h"
 
 /* Ensure drm and display members are placed properly. */
 INTEL_DISPLAY_MEMBER_STATIC_ASSERT(struct xe_device, drm, display);
@@ -122,7 +125,7 @@ int xe_display_init_early(struct xe_device *xe)
 	 * Fill the dram structure to get the system dram info. This will be
 	 * used for memory latency calculation.
 	 */
-	err = intel_dram_detect(xe);
+	err = intel_dram_detect(display);
 	if (err)
 		goto err_opregion;
 
@@ -516,8 +519,29 @@ static void display_device_remove(struct drm_device *dev, void *arg)
 	intel_display_device_remove(display);
 }
 
+static bool irq_enabled(struct drm_device *drm)
+{
+	struct xe_device *xe = to_xe_device(drm);
+
+	return atomic_read(&xe->irq.enabled);
+}
+
+static void irq_synchronize(struct drm_device *drm)
+{
+	synchronize_irq(to_pci_dev(drm->dev)->irq);
+}
+
+static const struct intel_display_irq_interface xe_display_irq_interface = {
+	.enabled = irq_enabled,
+	.synchronize = irq_synchronize,
+};
+
 static const struct intel_display_parent_interface parent = {
+	.hdcp = &xe_display_hdcp_interface,
+	.irq = &xe_display_irq_interface,
+	.panic = &xe_display_panic_interface,
 	.rpm = &xe_display_rpm_interface,
+	.stolen = &xe_display_stolen_interface,
 };
 
 /**
diff --git a/drivers/gpu/drm/xe/display/xe_display_misc.c b/drivers/gpu/drm/xe/display/xe_display_misc.c
deleted file mode 100644
index 242c2ef4ca93..000000000000
--- a/drivers/gpu/drm/xe/display/xe_display_misc.c
+++ /dev/null
@@ -1,16 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#include "intel_display_types.h"
-
-struct pci_dev;
-
-unsigned int intel_gmch_vga_set_decode(struct pci_dev *pdev, bool enable_decode);
-
-unsigned int intel_gmch_vga_set_decode(struct pci_dev *pdev, bool enable_decode)
-{
-	/* ToDo: Implement the actual handling of vga decode */
-	return 0;
-}
diff --git a/drivers/gpu/drm/xe/display/xe_display_rpm.c b/drivers/gpu/drm/xe/display/xe_display_rpm.c
index 340f65884812..b3db40035499 100644
--- a/drivers/gpu/drm/xe/display/xe_display_rpm.c
+++ b/drivers/gpu/drm/xe/display/xe_display_rpm.c
@@ -9,6 +9,9 @@
 #include "xe_device_types.h"
 #include "xe_pm.h"
 
+/* -ENOENT means we got the ref, but there's no tracking */
+#define INTEL_WAKEREF_DEF ERR_PTR(-ENOENT)
+
 static struct ref_tracker *xe_display_rpm_get(const struct drm_device *drm)
 {
 	return xe_pm_runtime_resume_and_get(to_xe_device(drm)) ? INTEL_WAKEREF_DEF : NULL;
diff --git a/drivers/gpu/drm/xe/display/xe_dsb_buffer.c b/drivers/gpu/drm/xe/display/xe_dsb_buffer.c
index 58581d7aaae6..fa0acb11eaad 100644
--- a/drivers/gpu/drm/xe/display/xe_dsb_buffer.c
+++ b/drivers/gpu/drm/xe/display/xe_dsb_buffer.c
@@ -3,44 +3,49 @@
  * Copyright 2023, Intel Corporation.
  */
 
-#include "i915_vma.h"
-#include "intel_display_types.h"
 #include "intel_dsb_buffer.h"
 #include "xe_bo.h"
 #include "xe_device.h"
 #include "xe_device_types.h"
 
+struct intel_dsb_buffer {
+	u32 *cmd_buf;
+	struct xe_bo *bo;
+	size_t buf_size;
+};
+
 u32 intel_dsb_buffer_ggtt_offset(struct intel_dsb_buffer *dsb_buf)
 {
-	return xe_bo_ggtt_addr(dsb_buf->vma->bo);
+	return xe_bo_ggtt_addr(dsb_buf->bo);
 }
 
 void intel_dsb_buffer_write(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val)
 {
-	iosys_map_wr(&dsb_buf->vma->bo->vmap, idx * 4, u32, val);
+	iosys_map_wr(&dsb_buf->bo->vmap, idx * 4, u32, val);
 }
 
 u32 intel_dsb_buffer_read(struct intel_dsb_buffer *dsb_buf, u32 idx)
 {
-	return iosys_map_rd(&dsb_buf->vma->bo->vmap, idx * 4, u32);
+	return iosys_map_rd(&dsb_buf->bo->vmap, idx * 4, u32);
 }
 
 void intel_dsb_buffer_memset(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val, size_t size)
 {
 	WARN_ON(idx > (dsb_buf->buf_size - size) / sizeof(*dsb_buf->cmd_buf));
 
-	iosys_map_memset(&dsb_buf->vma->bo->vmap, idx * 4, val, size);
+	iosys_map_memset(&dsb_buf->bo->vmap, idx * 4, val, size);
 }
 
-bool intel_dsb_buffer_create(struct intel_crtc *crtc, struct intel_dsb_buffer *dsb_buf, size_t size)
+struct intel_dsb_buffer *intel_dsb_buffer_create(struct drm_device *drm, size_t size)
 {
-	struct xe_device *xe = to_xe_device(crtc->base.dev);
+	struct xe_device *xe = to_xe_device(drm);
+	struct intel_dsb_buffer *dsb_buf;
 	struct xe_bo *obj;
-	struct i915_vma *vma;
+	int ret;
 
-	vma = kzalloc(sizeof(*vma), GFP_KERNEL);
-	if (!vma)
-		return false;
+	dsb_buf = kzalloc(sizeof(*dsb_buf), GFP_KERNEL);
+	if (!dsb_buf)
+		return ERR_PTR(-ENOMEM);
 
 	/* Set scanout flag for WC mapping */
 	obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe),
@@ -49,26 +54,30 @@ bool intel_dsb_buffer_create(struct intel_crtc *crtc, struct intel_dsb_buffer *d
 					XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) |
 					XE_BO_FLAG_SCANOUT | XE_BO_FLAG_GGTT, false);
 	if (IS_ERR(obj)) {
-		kfree(vma);
-		return false;
+		ret = PTR_ERR(obj);
+		goto err_pin_map;
 	}
 
-	vma->bo = obj;
-	dsb_buf->vma = vma;
+	dsb_buf->bo = obj;
 	dsb_buf->buf_size = size;
 
-	return true;
+	return dsb_buf;
+
+err_pin_map:
+	kfree(dsb_buf);
+
+	return ERR_PTR(ret);
 }
 
 void intel_dsb_buffer_cleanup(struct intel_dsb_buffer *dsb_buf)
 {
-	xe_bo_unpin_map_no_vm(dsb_buf->vma->bo);
-	kfree(dsb_buf->vma);
+	xe_bo_unpin_map_no_vm(dsb_buf->bo);
+	kfree(dsb_buf);
 }
 
 void intel_dsb_buffer_flush_map(struct intel_dsb_buffer *dsb_buf)
 {
-	struct xe_device *xe = dsb_buf->vma->bo->tile->xe;
+	struct xe_device *xe = dsb_buf->bo->tile->xe;
 
 	/*
 	 * The memory barrier here is to ensure coherency of DSB vs MMIO,
diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c
index 1fd4a815e784..6a935a75f2a4 100644
--- a/drivers/gpu/drm/xe/display/xe_fb_pin.c
+++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c
@@ -210,10 +210,11 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb,
 	/* TODO: Consider sharing framebuffer mapping?
 	 * embed i915_vma inside intel_framebuffer
 	 */
-	xe_pm_runtime_get_noresume(xe);
-	ret = mutex_lock_interruptible(&ggtt->lock);
+	guard(xe_pm_runtime_noresume)(xe);
+	ACQUIRE(mutex_intr, lock)(&ggtt->lock);
+	ret = ACQUIRE_ERR(mutex_intr, &lock);
 	if (ret)
-		goto out;
+		return ret;
 
 	align = XE_PAGE_SIZE;
 	if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
@@ -223,15 +224,13 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb,
 		vma->node = bo->ggtt_node[tile0->id];
 	} else if (view->type == I915_GTT_VIEW_NORMAL) {
 		vma->node = xe_ggtt_node_init(ggtt);
-		if (IS_ERR(vma->node)) {
-			ret = PTR_ERR(vma->node);
-			goto out_unlock;
-		}
+		if (IS_ERR(vma->node))
+			return PTR_ERR(vma->node);
 
 		ret = xe_ggtt_node_insert_locked(vma->node, xe_bo_size(bo), align, 0);
 		if (ret) {
 			xe_ggtt_node_fini(vma->node);
-			goto out_unlock;
+			return ret;
 		}
 
 		xe_ggtt_map_bo(ggtt, vma->node, bo, xe->pat.idx[XE_CACHE_NONE]);
@@ -245,13 +244,13 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb,
 		vma->node = xe_ggtt_node_init(ggtt);
 		if (IS_ERR(vma->node)) {
 			ret = PTR_ERR(vma->node);
-			goto out_unlock;
+			return ret;
 		}
 
 		ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0);
 		if (ret) {
 			xe_ggtt_node_fini(vma->node);
-			goto out_unlock;
+			return ret;
 		}
 
 		ggtt_ofs = vma->node->base.start;
@@ -265,10 +264,6 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb,
 					   rot_info->plane[i].dst_stride);
 	}
 
-out_unlock:
-	mutex_unlock(&ggtt->lock);
-out:
-	xe_pm_runtime_put(xe);
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c
index 4ae847b628e2..07acae121aa7 100644
--- a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c
+++ b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c
@@ -3,18 +3,20 @@
  * Copyright 2023, Intel Corporation.
  */
 
+#include <linux/delay.h>
+
 #include <drm/drm_print.h>
+#include <drm/intel/display_parent_interface.h>
 #include <drm/intel/i915_hdcp_interface.h>
-#include <linux/delay.h>
 
 #include "abi/gsc_command_header_abi.h"
-#include "intel_hdcp_gsc.h"
 #include "xe_bo.h"
 #include "xe_device.h"
 #include "xe_device_types.h"
 #include "xe_force_wake.h"
 #include "xe_gsc_proxy.h"
 #include "xe_gsc_submit.h"
+#include "xe_hdcp_gsc.h"
 #include "xe_map.h"
 #include "xe_pm.h"
 #include "xe_uc_fw.h"
@@ -30,14 +32,12 @@ struct intel_hdcp_gsc_context {
 
 #define HDCP_GSC_HEADER_SIZE sizeof(struct intel_gsc_mtl_header)
 
-bool intel_hdcp_gsc_check_status(struct drm_device *drm)
+static bool intel_hdcp_gsc_check_status(struct drm_device *drm)
 {
 	struct xe_device *xe = to_xe_device(drm);
 	struct xe_tile *tile = xe_device_get_root_tile(xe);
 	struct xe_gt *gt = tile->media_gt;
 	struct xe_gsc *gsc = &gt->uc.gsc;
-	bool ret = true;
-	unsigned int fw_ref;
 
 	if (!gsc || !xe_uc_fw_is_enabled(&gsc->fw)) {
 		drm_dbg_kms(&xe->drm,
@@ -45,22 +45,15 @@ bool intel_hdcp_gsc_check_status(struct drm_device *drm)
 		return false;
 	}
 
-	xe_pm_runtime_get(xe);
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC);
-	if (!fw_ref) {
+	guard(xe_pm_runtime)(xe);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC);
+	if (!fw_ref.domains) {
 		drm_dbg_kms(&xe->drm,
 			    "failed to get forcewake to check proxy status\n");
-		ret = false;
-		goto out;
+		return false;
 	}
 
-	if (!xe_gsc_proxy_init_done(gsc))
-		ret = false;
-
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-out:
-	xe_pm_runtime_put(xe);
-	return ret;
+	return xe_gsc_proxy_init_done(gsc);
 }
 
 /*This function helps allocate memory for the command that we will send to gsc cs */
@@ -96,7 +89,7 @@ static int intel_hdcp_gsc_initialize_message(struct xe_device *xe,
 	return ret;
 }
 
-struct intel_hdcp_gsc_context *intel_hdcp_gsc_context_alloc(struct drm_device *drm)
+static struct intel_hdcp_gsc_context *intel_hdcp_gsc_context_alloc(struct drm_device *drm)
 {
 	struct xe_device *xe = to_xe_device(drm);
 	struct intel_hdcp_gsc_context *gsc_context;
@@ -120,7 +113,7 @@ struct intel_hdcp_gsc_context *intel_hdcp_gsc_context_alloc(struct drm_device *d
 	return gsc_context;
 }
 
-void intel_hdcp_gsc_context_free(struct intel_hdcp_gsc_context *gsc_context)
+static void intel_hdcp_gsc_context_free(struct intel_hdcp_gsc_context *gsc_context)
 {
 	if (!gsc_context)
 		return;
@@ -155,9 +148,9 @@ static int xe_gsc_send_sync(struct xe_device *xe,
 	return ret;
 }
 
-ssize_t intel_hdcp_gsc_msg_send(struct intel_hdcp_gsc_context *gsc_context,
-				void *msg_in, size_t msg_in_len,
-				void *msg_out, size_t msg_out_len)
+static ssize_t intel_hdcp_gsc_msg_send(struct intel_hdcp_gsc_context *gsc_context,
+				       void *msg_in, size_t msg_in_len,
+				       void *msg_out, size_t msg_out_len)
 {
 	struct xe_device *xe = gsc_context->xe;
 	const size_t max_msg_size = PAGE_SIZE - HDCP_GSC_HEADER_SIZE;
@@ -166,17 +159,15 @@ ssize_t intel_hdcp_gsc_msg_send(struct intel_hdcp_gsc_context *gsc_context,
 	u32 addr_out_off, addr_in_wr_off = 0;
 	int ret, tries = 0;
 
-	if (msg_in_len > max_msg_size || msg_out_len > max_msg_size) {
-		ret = -ENOSPC;
-		goto out;
-	}
+	if (msg_in_len > max_msg_size || msg_out_len > max_msg_size)
+		return -ENOSPC;
 
 	msg_size_in = msg_in_len + HDCP_GSC_HEADER_SIZE;
 	msg_size_out = msg_out_len + HDCP_GSC_HEADER_SIZE;
 	addr_out_off = PAGE_SIZE;
 
 	host_session_id = xe_gsc_create_host_session_id();
-	xe_pm_runtime_get_noresume(xe);
+	guard(xe_pm_runtime_noresume)(xe);
 	addr_in_wr_off = xe_gsc_emit_header(xe, &gsc_context->hdcp_bo->vmap,
 					    addr_in_wr_off, HECI_MEADDRESS_HDCP,
 					    host_session_id, msg_in_len);
@@ -201,13 +192,18 @@ ssize_t intel_hdcp_gsc_msg_send(struct intel_hdcp_gsc_context *gsc_context,
 	} while (++tries < 20);
 
 	if (ret)
-		goto out;
+		return ret;
 
 	xe_map_memcpy_from(xe, msg_out, &gsc_context->hdcp_bo->vmap,
 			   addr_out_off + HDCP_GSC_HEADER_SIZE,
 			   msg_out_len);
 
-out:
-	xe_pm_runtime_put(xe);
 	return ret;
 }
+
+const struct intel_display_hdcp_interface xe_display_hdcp_interface = {
+	.gsc_msg_send = intel_hdcp_gsc_msg_send,
+	.gsc_check_status = intel_hdcp_gsc_check_status,
+	.gsc_context_alloc = intel_hdcp_gsc_context_alloc,
+	.gsc_context_free = intel_hdcp_gsc_context_free,
+};
diff --git a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.h b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.h
new file mode 100644
index 000000000000..c1062e4b62f7
--- /dev/null
+++ b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2025 Intel Corporation */
+
+#ifndef __XE_HDCP_GSC_H__
+#define __XE_HDCP_GSC_H__
+
+extern const struct intel_display_hdcp_interface xe_display_hdcp_interface;
+
+#endif
diff --git a/drivers/gpu/drm/xe/display/xe_panic.c b/drivers/gpu/drm/xe/display/xe_panic.c
index df663286092a..e078494dc8ba 100644
--- a/drivers/gpu/drm/xe/display/xe_panic.c
+++ b/drivers/gpu/drm/xe/display/xe_panic.c
@@ -3,11 +3,12 @@
 
 #include <drm/drm_cache.h>
 #include <drm/drm_panic.h>
+#include <drm/intel/display_parent_interface.h>
 
 #include "intel_display_types.h"
 #include "intel_fb.h"
-#include "intel_panic.h"
 #include "xe_bo.h"
+#include "xe_panic.h"
 #include "xe_res_cursor.h"
 
 struct intel_panic {
@@ -74,7 +75,7 @@ static void xe_panic_page_set_pixel(struct drm_scanout_buffer *sb, unsigned int
 		iosys_map_wr(&panic->vmap, offset, u32, color);
 }
 
-struct intel_panic *intel_panic_alloc(void)
+static struct intel_panic *xe_panic_alloc(void)
 {
 	struct intel_panic *panic;
 
@@ -83,7 +84,7 @@ struct intel_panic *intel_panic_alloc(void)
 	return panic;
 }
 
-int intel_panic_setup(struct intel_panic *panic, struct drm_scanout_buffer *sb)
+static int xe_panic_setup(struct intel_panic *panic, struct drm_scanout_buffer *sb)
 {
 	struct intel_framebuffer *fb = (struct intel_framebuffer *)sb->private;
 	struct xe_bo *bo = gem_to_xe_bo(intel_fb_bo(&fb->base));
@@ -96,7 +97,8 @@ int intel_panic_setup(struct intel_panic *panic, struct drm_scanout_buffer *sb)
 	return 0;
 }
 
-void intel_panic_finish(struct intel_panic *panic)
-{
-	xe_panic_kunmap(panic);
-}
+const struct intel_display_panic_interface xe_display_panic_interface = {
+	.alloc = xe_panic_alloc,
+	.setup = xe_panic_setup,
+	.finish = xe_panic_kunmap,
+};
diff --git a/drivers/gpu/drm/xe/display/xe_panic.h b/drivers/gpu/drm/xe/display/xe_panic.h
new file mode 100644
index 000000000000..3054b511011b
--- /dev/null
+++ b/drivers/gpu/drm/xe/display/xe_panic.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2025 Intel Corporation */
+
+#ifndef __XE_PANIC_H__
+#define __XE_PANIC_H__
+
+extern const struct intel_display_panic_interface xe_display_panic_interface;
+
+#endif
diff --git a/drivers/gpu/drm/xe/display/xe_plane_initial.c b/drivers/gpu/drm/xe/display/xe_plane_initial.c
index 12d25c5290fd..01c105a93bb9 100644
--- a/drivers/gpu/drm/xe/display/xe_plane_initial.c
+++ b/drivers/gpu/drm/xe/display/xe_plane_initial.c
@@ -58,7 +58,7 @@ intel_reuse_initial_plane_obj(struct intel_crtc *this,
 		const struct intel_crtc_state *crtc_state =
 			to_intel_crtc_state(crtc->base.state);
 
-		if (!crtc_state->uapi.active)
+		if (!crtc_state->hw.active)
 			continue;
 
 		if (!plane_state->ggtt_vma)
@@ -290,10 +290,12 @@ void intel_initial_plane_config(struct intel_display *display)
 	struct intel_crtc *crtc;
 
 	for_each_intel_crtc(display->drm, crtc) {
+		const struct intel_crtc_state *crtc_state =
+			to_intel_crtc_state(crtc->base.state);
 		struct intel_initial_plane_config *plane_config =
 			&plane_configs[crtc->pipe];
 
-		if (!to_intel_crtc_state(crtc->base.state)->uapi.active)
+		if (!crtc_state->hw.active)
 			continue;
 
 		/*
diff --git a/drivers/gpu/drm/xe/display/xe_stolen.c b/drivers/gpu/drm/xe/display/xe_stolen.c
index 9f04ba36e930..12771709183a 100644
--- a/drivers/gpu/drm/xe/display/xe_stolen.c
+++ b/drivers/gpu/drm/xe/display/xe_stolen.c
@@ -1,8 +1,10 @@
 // SPDX-License-Identifier: MIT
 /* Copyright © 2025 Intel Corporation */
 
-#include "gem/i915_gem_stolen.h"
+#include <drm/intel/display_parent_interface.h>
+
 #include "xe_res_cursor.h"
+#include "xe_stolen.h"
 #include "xe_ttm_stolen_mgr.h"
 #include "xe_validation.h"
 
@@ -11,8 +13,8 @@ struct intel_stolen_node {
 	struct xe_bo *bo;
 };
 
-int i915_gem_stolen_insert_node_in_range(struct intel_stolen_node *node, u64 size,
-					 unsigned int align, u64 start, u64 end)
+static int xe_stolen_insert_node_in_range(struct intel_stolen_node *node, u64 size,
+					  unsigned int align, u64 start, u64 end)
 {
 	struct xe_device *xe = node->xe;
 
@@ -41,33 +43,25 @@ int i915_gem_stolen_insert_node_in_range(struct intel_stolen_node *node, u64 siz
 	return err;
 }
 
-int i915_gem_stolen_insert_node(struct intel_stolen_node *node, u64 size, unsigned int align)
-{
-	/* Not used on xe */
-	WARN_ON(1);
-
-	return -ENODEV;
-}
-
-void i915_gem_stolen_remove_node(struct intel_stolen_node *node)
+static void xe_stolen_remove_node(struct intel_stolen_node *node)
 {
 	xe_bo_unpin_map_no_vm(node->bo);
 	node->bo = NULL;
 }
 
-bool i915_gem_stolen_initialized(struct drm_device *drm)
+static bool xe_stolen_initialized(struct drm_device *drm)
 {
 	struct xe_device *xe = to_xe_device(drm);
 
 	return ttm_manager_type(&xe->ttm, XE_PL_STOLEN);
 }
 
-bool i915_gem_stolen_node_allocated(const struct intel_stolen_node *node)
+static bool xe_stolen_node_allocated(const struct intel_stolen_node *node)
 {
 	return node->bo;
 }
 
-u32 i915_gem_stolen_node_offset(struct intel_stolen_node *node)
+static u64 xe_stolen_node_offset(const struct intel_stolen_node *node)
 {
 	struct xe_res_cursor res;
 
@@ -75,35 +69,19 @@ u32 i915_gem_stolen_node_offset(struct intel_stolen_node *node)
 	return res.start;
 }
 
-/* Used for < gen4. These are not supported by Xe */
-u64 i915_gem_stolen_area_address(struct drm_device *drm)
-{
-	WARN_ON(1);
-
-	return 0;
-}
-
-/* Used for gen9 specific WA. Gen9 is not supported by Xe */
-u64 i915_gem_stolen_area_size(struct drm_device *drm)
-{
-	WARN_ON(1);
-
-	return 0;
-}
-
-u64 i915_gem_stolen_node_address(struct intel_stolen_node *node)
+static u64 xe_stolen_node_address(const struct intel_stolen_node *node)
 {
 	struct xe_device *xe = node->xe;
 
-	return xe_ttm_stolen_gpu_offset(xe) + i915_gem_stolen_node_offset(node);
+	return xe_ttm_stolen_gpu_offset(xe) + xe_stolen_node_offset(node);
 }
 
-u64 i915_gem_stolen_node_size(const struct intel_stolen_node *node)
+static u64 xe_stolen_node_size(const struct intel_stolen_node *node)
 {
 	return node->bo->ttm.base.size;
 }
 
-struct intel_stolen_node *i915_gem_stolen_node_alloc(struct drm_device *drm)
+static struct intel_stolen_node *xe_stolen_node_alloc(struct drm_device *drm)
 {
 	struct xe_device *xe = to_xe_device(drm);
 	struct intel_stolen_node *node;
@@ -117,7 +95,19 @@ struct intel_stolen_node *i915_gem_stolen_node_alloc(struct drm_device *drm)
 	return node;
 }
 
-void i915_gem_stolen_node_free(const struct intel_stolen_node *node)
+static void xe_stolen_node_free(const struct intel_stolen_node *node)
 {
 	kfree(node);
 }
+
+const struct intel_display_stolen_interface xe_display_stolen_interface = {
+	.insert_node_in_range = xe_stolen_insert_node_in_range,
+	.remove_node = xe_stolen_remove_node,
+	.initialized = xe_stolen_initialized,
+	.node_allocated = xe_stolen_node_allocated,
+	.node_offset = xe_stolen_node_offset,
+	.node_address = xe_stolen_node_address,
+	.node_size = xe_stolen_node_size,
+	.node_alloc = xe_stolen_node_alloc,
+	.node_free = xe_stolen_node_free,
+};
diff --git a/drivers/gpu/drm/xe/display/xe_stolen.h b/drivers/gpu/drm/xe/display/xe_stolen.h
new file mode 100644
index 000000000000..db86b9e01242
--- /dev/null
+++ b/drivers/gpu/drm/xe/display/xe_stolen.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2025 Intel Corporation */
+
+#ifndef __XE_STOLEN_H__
+#define __XE_STOLEN_H__
+
+extern const struct intel_display_stolen_interface xe_display_stolen_interface;
+
+#endif
diff --git a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h
index 5d41ca297447..885fcf211e6d 100644
--- a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h
+++ b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h
@@ -47,6 +47,7 @@
 
 #define GFX_OP_PIPE_CONTROL(len)	((0x3<<29)|(0x3<<27)|(0x2<<24)|((len)-2))
 
+#define   PIPE_CONTROL0_QUEUE_DRAIN_MODE		BIT(12)
 #define	  PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE	BIT(10)	/* gen12 */
 #define	  PIPE_CONTROL0_HDC_PIPELINE_FLUSH		BIT(9)	/* gen12 */
 
diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h
index 917a088c28f2..93643da57428 100644
--- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h
@@ -227,6 +227,9 @@
 
 #define MIRROR_FUSE1				XE_REG(0x911c)
 
+#define FUSE2					XE_REG(0x9120)
+#define   PRODUCTION_HW				REG_BIT(2)
+
 #define MIRROR_L3BANK_ENABLE			XE_REG(0x9130)
 #define   XE3_L3BANK_ENABLE			REG_GENMASK(31, 0)
 
diff --git a/drivers/gpu/drm/xe/regs/xe_gtt_defs.h b/drivers/gpu/drm/xe/regs/xe_gtt_defs.h
index 4389e5a76f89..4d83461e538b 100644
--- a/drivers/gpu/drm/xe/regs/xe_gtt_defs.h
+++ b/drivers/gpu/drm/xe/regs/xe_gtt_defs.h
@@ -9,6 +9,7 @@
 #define XELPG_GGTT_PTE_PAT0	BIT_ULL(52)
 #define XELPG_GGTT_PTE_PAT1	BIT_ULL(53)
 
+#define XE_PTE_ADDR_MASK	GENMASK_ULL(51, 12)
 #define GGTT_PTE_VFID		GENMASK_ULL(11, 2)
 
 #define GUC_GGTT_TOP		0xFEE00000
diff --git a/drivers/gpu/drm/xe/regs/xe_guc_regs.h b/drivers/gpu/drm/xe/regs/xe_guc_regs.h
index 2118f7dec287..87984713dd12 100644
--- a/drivers/gpu/drm/xe/regs/xe_guc_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_guc_regs.h
@@ -90,6 +90,9 @@
 #define GUC_SEND_INTERRUPT			XE_REG(0xc4c8)
 #define   GUC_SEND_TRIGGER			REG_BIT(0)
 
+#define GUC_INTR_CHICKEN			XE_REG(0xc50c)
+#define   DISABLE_SIGNALING_ENGINES		REG_BIT(1)
+
 #define GUC_BCS_RCS_IER				XE_REG(0xc550)
 #define GUC_VCS2_VCS1_IER			XE_REG(0xc554)
 #define GUC_WD_VECS_IER				XE_REG(0xc558)
diff --git a/drivers/gpu/drm/xe/regs/xe_irq_regs.h b/drivers/gpu/drm/xe/regs/xe_irq_regs.h
index 2f97662d958d..9d74f454d3ff 100644
--- a/drivers/gpu/drm/xe/regs/xe_irq_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_irq_regs.h
@@ -20,6 +20,7 @@
 #define   GU_MISC_IRQ				REG_BIT(29)
 #define   ERROR_IRQ(x)				REG_BIT(26 + (x))
 #define   DISPLAY_IRQ				REG_BIT(16)
+#define   SOC_H2DMEMINT_IRQ			REG_BIT(13)
 #define   I2C_IRQ				REG_BIT(12)
 #define   GT_DW_IRQ(x)				REG_BIT(x)
 
diff --git a/drivers/gpu/drm/xe/regs/xe_mert_regs.h b/drivers/gpu/drm/xe/regs/xe_mert_regs.h
new file mode 100644
index 000000000000..c345e11ceea8
--- /dev/null
+++ b/drivers/gpu/drm/xe/regs/xe_mert_regs.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2025 Intel Corporation
+ */
+
+#ifndef _XE_MERT_REGS_H_
+#define _XE_MERT_REGS_H_
+
+#include "regs/xe_reg_defs.h"
+
+#define MERT_LMEM_CFG				XE_REG(0x1448b0)
+
+#define MERT_TLB_CT_INTR_ERR_ID_PORT		XE_REG(0x145190)
+#define   MERT_TLB_CT_VFID_MASK			REG_GENMASK(16, 9)
+#define   MERT_TLB_CT_ERROR_MASK		REG_GENMASK(5, 0)
+#define     MERT_TLB_CT_LMTT_FAULT		0x05
+
+#define MERT_TLB_INV_DESC_A			XE_REG(0x14cf7c)
+#define   MERT_TLB_INV_DESC_A_VALID		REG_BIT(0)
+
+#endif /* _XE_MERT_REGS_H_ */
diff --git a/drivers/gpu/drm/xe/regs/xe_oa_regs.h b/drivers/gpu/drm/xe/regs/xe_oa_regs.h
index e693a50706f8..04a729e610aa 100644
--- a/drivers/gpu/drm/xe/regs/xe_oa_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_oa_regs.h
@@ -100,4 +100,21 @@
 #define OAM_COMPRESSION_T3_CONTROL		XE_REG(0x1c2e00)
 #define  OAM_LAT_MEASURE_ENABLE			REG_BIT(4)
 
+/* Actual address is MEDIA_GT_GSI_OFFSET + the base addr below */
+#define XE_OAM_SAG_BASE				0x13000
+#define XE_OAM_SCMI_0_BASE			0x14000
+#define XE_OAM_SCMI_1_BASE			0x14800
+#define XE_OAM_SAG_BASE_ADJ			(MEDIA_GT_GSI_OFFSET + XE_OAM_SAG_BASE)
+#define XE_OAM_SCMI_0_BASE_ADJ			(MEDIA_GT_GSI_OFFSET + XE_OAM_SCMI_0_BASE)
+#define XE_OAM_SCMI_1_BASE_ADJ			(MEDIA_GT_GSI_OFFSET + XE_OAM_SCMI_1_BASE)
+
+#define OAMERT_CONTROL				XE_REG(0x1453a0)
+#define OAMERT_DEBUG				XE_REG(0x1453a4)
+#define OAMERT_STATUS				XE_REG(0x1453a8)
+#define OAMERT_HEAD_POINTER			XE_REG(0x1453ac)
+#define OAMERT_TAIL_POINTER			XE_REG(0x1453b0)
+#define OAMERT_BUFFER				XE_REG(0x1453b4)
+#define OAMERT_CONTEXT_CONTROL			XE_REG(0x1453c8)
+#define OAMERT_MMIO_TRG				XE_REG(0x1453cc)
+
 #endif
diff --git a/drivers/gpu/drm/xe/tests/xe_args_test.c b/drivers/gpu/drm/xe/tests/xe_args_test.c
index f3fb23aa5d2e..2687a1b054dd 100644
--- a/drivers/gpu/drm/xe/tests/xe_args_test.c
+++ b/drivers/gpu/drm/xe/tests/xe_args_test.c
@@ -78,6 +78,24 @@ static void pick_arg_example(struct kunit *test)
 #undef buz
 }
 
+static void if_args_example(struct kunit *test)
+{
+	enum { Z = 1, Q };
+
+#define foo	X, Y
+#define bar	IF_ARGS(Z, Q, foo)
+#define buz	IF_ARGS(Z, Q, DROP_FIRST_ARG(FIRST_ARG(foo)))
+
+	KUNIT_EXPECT_EQ(test, bar, Z);
+	KUNIT_EXPECT_EQ(test, buz, Q);
+	KUNIT_EXPECT_STREQ(test, __stringify(bar), "Z");
+	KUNIT_EXPECT_STREQ(test, __stringify(buz), "Q");
+
+#undef foo
+#undef bar
+#undef buz
+}
+
 static void sep_comma_example(struct kunit *test)
 {
 #define foo(f)	f(X) f(Y) f(Z) f(Q)
@@ -198,6 +216,40 @@ static void last_arg_test(struct kunit *test)
 	KUNIT_EXPECT_STREQ(test, __stringify(LAST_ARG(MAX_ARGS)), "-12");
 }
 
+static void if_args_test(struct kunit *test)
+{
+	bool with_args = true;
+	bool no_args = false;
+	enum { X = 100 };
+
+	KUNIT_EXPECT_TRUE(test, IF_ARGS(true, false, FOO_ARGS));
+	KUNIT_EXPECT_FALSE(test, IF_ARGS(true, false, NO_ARGS));
+
+	KUNIT_EXPECT_TRUE(test, CONCATENATE(IF_ARGS(with, no, FOO_ARGS), _args));
+	KUNIT_EXPECT_FALSE(test, CONCATENATE(IF_ARGS(with, no, NO_ARGS), _args));
+
+	KUNIT_EXPECT_STREQ(test, __stringify(IF_ARGS(yes, no, FOO_ARGS)), "yes");
+	KUNIT_EXPECT_STREQ(test, __stringify(IF_ARGS(yes, no, NO_ARGS)), "no");
+
+	KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, FOO_ARGS), -1, FOO_ARGS), 4);
+	KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, FOO_ARGS), -1, NO_ARGS), -1);
+	KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, NO_ARGS), -1, FOO_ARGS), 0);
+	KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, NO_ARGS), -1, NO_ARGS), -1);
+
+	KUNIT_EXPECT_EQ(test,
+			CALL_ARGS(FIRST_ARG,
+				  CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, FOO_ARGS), _ARGS)), X);
+	KUNIT_EXPECT_EQ(test,
+			CALL_ARGS(FIRST_ARG,
+				  CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, NO_ARGS), _ARGS)), -1);
+	KUNIT_EXPECT_EQ(test,
+			CALL_ARGS(COUNT_ARGS,
+				  CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, FOO_ARGS), _ARGS)), 4);
+	KUNIT_EXPECT_EQ(test,
+			CALL_ARGS(COUNT_ARGS,
+				  CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, NO_ARGS), _ARGS)), 12);
+}
+
 static struct kunit_case args_tests[] = {
 	KUNIT_CASE(count_args_test),
 	KUNIT_CASE(call_args_example),
@@ -209,6 +261,8 @@ static struct kunit_case args_tests[] = {
 	KUNIT_CASE(last_arg_example),
 	KUNIT_CASE(last_arg_test),
 	KUNIT_CASE(pick_arg_example),
+	KUNIT_CASE(if_args_example),
+	KUNIT_CASE(if_args_test),
 	KUNIT_CASE(sep_comma_example),
 	{}
 };
diff --git a/drivers/gpu/drm/xe/tests/xe_bo.c b/drivers/gpu/drm/xe/tests/xe_bo.c
index 2294cf89f3e1..2278e589a493 100644
--- a/drivers/gpu/drm/xe/tests/xe_bo.c
+++ b/drivers/gpu/drm/xe/tests/xe_bo.c
@@ -185,8 +185,7 @@ static int ccs_test_run_device(struct xe_device *xe)
 		return 0;
 	}
 
-	xe_pm_runtime_get(xe);
-
+	guard(xe_pm_runtime)(xe);
 	for_each_tile(tile, xe, id) {
 		/* For igfx run only for primary tile */
 		if (!IS_DGFX(xe) && id > 0)
@@ -194,8 +193,6 @@ static int ccs_test_run_device(struct xe_device *xe)
 		ccs_test_run_tile(xe, tile, test);
 	}
 
-	xe_pm_runtime_put(xe);
-
 	return 0;
 }
 
@@ -356,13 +353,10 @@ static int evict_test_run_device(struct xe_device *xe)
 		return 0;
 	}
 
-	xe_pm_runtime_get(xe);
-
+	guard(xe_pm_runtime)(xe);
 	for_each_tile(tile, xe, id)
 		evict_test_run_tile(xe, tile, test);
 
-	xe_pm_runtime_put(xe);
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/xe/tests/xe_dma_buf.c b/drivers/gpu/drm/xe/tests/xe_dma_buf.c
index 5df98de5ba3c..954b6b911ea0 100644
--- a/drivers/gpu/drm/xe/tests/xe_dma_buf.c
+++ b/drivers/gpu/drm/xe/tests/xe_dma_buf.c
@@ -266,7 +266,7 @@ static int dma_buf_run_device(struct xe_device *xe)
 	const struct dma_buf_test_params *params;
 	struct kunit *test = kunit_get_current_test();
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	for (params = test_params; params->mem_mask; ++params) {
 		struct dma_buf_test_params p = *params;
 
@@ -274,7 +274,6 @@ static int dma_buf_run_device(struct xe_device *xe)
 		test->priv = &p;
 		xe_test_dmabuf_import_same_driver(xe);
 	}
-	xe_pm_runtime_put(xe);
 
 	/* A non-zero return would halt iteration over driver devices */
 	return 0;
diff --git a/drivers/gpu/drm/xe/tests/xe_migrate.c b/drivers/gpu/drm/xe/tests/xe_migrate.c
index 5904d658d1f2..34e2f0f4631f 100644
--- a/drivers/gpu/drm/xe/tests/xe_migrate.c
+++ b/drivers/gpu/drm/xe/tests/xe_migrate.c
@@ -344,8 +344,7 @@ static int migrate_test_run_device(struct xe_device *xe)
 	struct xe_tile *tile;
 	int id;
 
-	xe_pm_runtime_get(xe);
-
+	guard(xe_pm_runtime)(xe);
 	for_each_tile(tile, xe, id) {
 		struct xe_migrate *m = tile->migrate;
 		struct drm_exec *exec = XE_VALIDATION_OPT_OUT;
@@ -356,8 +355,6 @@ static int migrate_test_run_device(struct xe_device *xe)
 		xe_vm_unlock(m->q->vm);
 	}
 
-	xe_pm_runtime_put(xe);
-
 	return 0;
 }
 
@@ -759,13 +756,10 @@ static int validate_ccs_test_run_device(struct xe_device *xe)
 		return 0;
 	}
 
-	xe_pm_runtime_get(xe);
-
+	guard(xe_pm_runtime)(xe);
 	for_each_tile(tile, xe, id)
 		validate_ccs_test_run_tile(xe, tile, test);
 
-	xe_pm_runtime_put(xe);
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/xe/tests/xe_mocs.c b/drivers/gpu/drm/xe/tests/xe_mocs.c
index 6bb278167aaf..daf3c6836c75 100644
--- a/drivers/gpu/drm/xe/tests/xe_mocs.c
+++ b/drivers/gpu/drm/xe/tests/xe_mocs.c
@@ -43,14 +43,12 @@ static void read_l3cc_table(struct xe_gt *gt,
 {
 	struct kunit *test = kunit_get_current_test();
 	u32 l3cc, l3cc_expected;
-	unsigned int fw_ref, i;
+	unsigned int i;
 	u32 reg_val;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
-		xe_force_wake_put(gt_to_fw(gt), fw_ref);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
 		KUNIT_FAIL_AND_ABORT(test, "Forcewake Failed.\n");
-	}
 
 	for (i = 0; i < info->num_mocs_regs; i++) {
 		if (!(i & 1)) {
@@ -74,7 +72,6 @@ static void read_l3cc_table(struct xe_gt *gt,
 		KUNIT_EXPECT_EQ_MSG(test, l3cc_expected, l3cc,
 				    "l3cc idx=%u has incorrect val.\n", i);
 	}
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 
 static void read_mocs_table(struct xe_gt *gt,
@@ -82,14 +79,14 @@ static void read_mocs_table(struct xe_gt *gt,
 {
 	struct kunit *test = kunit_get_current_test();
 	u32 mocs, mocs_expected;
-	unsigned int fw_ref, i;
+	unsigned int i;
 	u32 reg_val;
 
 	KUNIT_EXPECT_TRUE_MSG(test, info->unused_entries_index,
 			      "Unused entries index should have been defined\n");
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	KUNIT_ASSERT_NE_MSG(test, fw_ref, 0, "Forcewake Failed.\n");
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	KUNIT_ASSERT_NE_MSG(test, fw_ref.domains, 0, "Forcewake Failed.\n");
 
 	for (i = 0; i < info->num_mocs_regs; i++) {
 		if (regs_are_mcr(gt))
@@ -106,8 +103,6 @@ static void read_mocs_table(struct xe_gt *gt,
 		KUNIT_EXPECT_EQ_MSG(test, mocs_expected, mocs,
 				    "mocs reg 0x%x has incorrect val.\n", i);
 	}
-
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 
 static int mocs_kernel_test_run_device(struct xe_device *xe)
@@ -120,8 +115,7 @@ static int mocs_kernel_test_run_device(struct xe_device *xe)
 	unsigned int flags;
 	int id;
 
-	xe_pm_runtime_get(xe);
-
+	guard(xe_pm_runtime)(xe);
 	for_each_gt(gt, xe, id) {
 		flags = live_mocs_init(&mocs, gt);
 		if (flags & HAS_GLOBAL_MOCS)
@@ -130,8 +124,6 @@ static int mocs_kernel_test_run_device(struct xe_device *xe)
 			read_l3cc_table(gt, &mocs.table);
 	}
 
-	xe_pm_runtime_put(xe);
-
 	return 0;
 }
 
@@ -155,8 +147,7 @@ static int mocs_reset_test_run_device(struct xe_device *xe)
 	int id;
 	struct kunit *test = kunit_get_current_test();
 
-	xe_pm_runtime_get(xe);
-
+	guard(xe_pm_runtime)(xe);
 	for_each_gt(gt, xe, id) {
 		flags = live_mocs_init(&mocs, gt);
 		kunit_info(test, "mocs_reset_test before reset\n");
@@ -174,8 +165,6 @@ static int mocs_reset_test_run_device(struct xe_device *xe)
 			read_l3cc_table(gt, &mocs.table);
 	}
 
-	xe_pm_runtime_put(xe);
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/xe/xe_args.h b/drivers/gpu/drm/xe/xe_args.h
index 4dbc7e53c624..f550b5e3b993 100644
--- a/drivers/gpu/drm/xe/xe_args.h
+++ b/drivers/gpu/drm/xe/xe_args.h
@@ -121,6 +121,33 @@
 #define PICK_ARG11(args...)		PICK_ARG10(DROP_FIRST_ARG(args))
 #define PICK_ARG12(args...)		PICK_ARG11(DROP_FIRST_ARG(args))
 
+/**
+ * IF_ARGS() - Make selection based on optional argument list.
+ * @then: token to return if arguments are present
+ * @else: token to return if arguments are empty
+ * @...: arguments to check (optional)
+ *
+ * This macro allows to select a token based on the presence of the argument list.
+ *
+ * Example:
+ *
+ *	#define foo	X, Y
+ *	#define bar	IF_ARGS(Z, Q, foo)
+ *	#define buz	IF_ARGS(Z, Q, DROP_FIRST_ARG(FIRST_ARG(foo)))
+ *
+ *	With above definitions bar expands to Z while buz expands to Q.
+ */
+#if defined(CONFIG_CC_IS_CLANG) || GCC_VERSION >= 100100
+#define IF_ARGS(then, else, ...)	FIRST_ARG(__VA_OPT__(then,) else)
+#else
+#define IF_ARGS(then, else, ...)	_IF_ARGS(then, else, CALL_ARGS(FIRST_ARG, __VA_ARGS__))
+#define _IF_ARGS(then, else, ...)	__IF_ARGS(then, else, CALL_ARGS(COUNT_ARGS, __VA_ARGS__))
+#define __IF_ARGS(then, else, n)	___IF_ARGS(then, else, CALL_ARGS(CONCATENATE, ___IF_ARG, n))
+#define ___IF_ARGS(then, else, if)	CALL_ARGS(if, then, else)
+#define ___IF_ARG1(then, else)		then
+#define ___IF_ARG0(then, else)		else
+#endif
+
 /**
  * ARGS_SEP_COMMA - Definition of a comma character.
  *
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index b0bd31d14bb9..8b6474cd3eaf 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -516,8 +516,7 @@ static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo,
 		 * non-coherent and require a CPU:WC mapping.
 		 */
 		if ((!bo->cpu_caching && bo->flags & XE_BO_FLAG_SCANOUT) ||
-		    (xe->info.graphics_verx100 >= 1270 &&
-		     bo->flags & XE_BO_FLAG_PAGETABLE))
+		     (!xe->info.has_cached_pt && bo->flags & XE_BO_FLAG_PAGETABLE))
 			caching = ttm_write_combined;
 	}
 
@@ -1527,7 +1526,7 @@ static bool xe_ttm_bo_lock_in_destructor(struct ttm_buffer_object *ttm_bo)
 	 * always succeed here, as long as we hold the lru lock.
 	 */
 	spin_lock(&ttm_bo->bdev->lru_lock);
-	locked = dma_resv_trylock(ttm_bo->base.resv);
+	locked = dma_resv_trylock(&ttm_bo->base._resv);
 	spin_unlock(&ttm_bo->bdev->lru_lock);
 	xe_assert(xe, locked);
 
@@ -1547,13 +1546,6 @@ static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo)
 	bo = ttm_to_xe_bo(ttm_bo);
 	xe_assert(xe_bo_device(bo), !(bo->created && kref_read(&ttm_bo->base.refcount)));
 
-	/*
-	 * Corner case where TTM fails to allocate memory and this BOs resv
-	 * still points the VMs resv
-	 */
-	if (ttm_bo->base.resv != &ttm_bo->base._resv)
-		return;
-
 	if (!xe_ttm_bo_lock_in_destructor(ttm_bo))
 		return;
 
@@ -1563,14 +1555,14 @@ static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo)
 	 * TODO: Don't do this for external bos once we scrub them after
 	 * unbind.
 	 */
-	dma_resv_for_each_fence(&cursor, ttm_bo->base.resv,
+	dma_resv_for_each_fence(&cursor, &ttm_bo->base._resv,
 				DMA_RESV_USAGE_BOOKKEEP, fence) {
 		if (xe_fence_is_xe_preempt(fence) &&
 		    !dma_fence_is_signaled(fence)) {
 			if (!replacement)
 				replacement = dma_fence_get_stub();
 
-			dma_resv_replace_fences(ttm_bo->base.resv,
+			dma_resv_replace_fences(&ttm_bo->base._resv,
 						fence->context,
 						replacement,
 						DMA_RESV_USAGE_BOOKKEEP);
@@ -1578,7 +1570,7 @@ static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo)
 	}
 	dma_fence_put(replacement);
 
-	dma_resv_unlock(ttm_bo->base.resv);
+	dma_resv_unlock(&ttm_bo->base._resv);
 }
 
 static void xe_ttm_bo_delete_mem_notify(struct ttm_buffer_object *ttm_bo)
@@ -2033,13 +2025,9 @@ static int xe_bo_vm_access(struct vm_area_struct *vma, unsigned long addr,
 	struct ttm_buffer_object *ttm_bo = vma->vm_private_data;
 	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
 	struct xe_device *xe = xe_bo_device(bo);
-	int ret;
 
-	xe_pm_runtime_get(xe);
-	ret = ttm_bo_vm_access(vma, addr, buf, len, write);
-	xe_pm_runtime_put(xe);
-
-	return ret;
+	guard(xe_pm_runtime)(xe);
+	return ttm_bo_vm_access(vma, addr, buf, len, write);
 }
 
 /**
@@ -3183,7 +3171,8 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data,
 	if (XE_IOCTL_DBG(xe, args->flags &
 			 ~(DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING |
 			   DRM_XE_GEM_CREATE_FLAG_SCANOUT |
-			   DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM)))
+			   DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM |
+			   DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION)))
 		return -EINVAL;
 
 	if (XE_IOCTL_DBG(xe, args->handle))
@@ -3205,6 +3194,12 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data,
 	if (args->flags & DRM_XE_GEM_CREATE_FLAG_SCANOUT)
 		bo_flags |= XE_BO_FLAG_SCANOUT;
 
+	if (args->flags & DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION) {
+		if (XE_IOCTL_DBG(xe, GRAPHICS_VER(xe) < 20))
+			return -EOPNOTSUPP;
+		bo_flags |= XE_BO_FLAG_NO_COMPRESSION;
+	}
+
 	bo_flags |= args->placement << (ffs(XE_BO_FLAG_SYSTEM) - 1);
 
 	/* CCS formats need physical placement at a 64K alignment in VRAM. */
@@ -3526,8 +3521,12 @@ bool xe_bo_needs_ccs_pages(struct xe_bo *bo)
 	 * Compression implies coh_none, therefore we know for sure that WB
 	 * memory can't currently use compression, which is likely one of the
 	 * common cases.
+	 * Additionally, userspace may explicitly request no compression via the
+	 * DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION flag, which should also disable
+	 * CCS usage.
 	 */
-	if (bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)
+	if (bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB ||
+	    bo->flags & XE_BO_FLAG_NO_COMPRESSION)
 		return false;
 
 	return true;
diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
index 911d5b90461a..8ab4474129c3 100644
--- a/drivers/gpu/drm/xe/xe_bo.h
+++ b/drivers/gpu/drm/xe/xe_bo.h
@@ -50,6 +50,7 @@
 #define XE_BO_FLAG_GGTT3		BIT(23)
 #define XE_BO_FLAG_CPU_ADDR_MIRROR	BIT(24)
 #define XE_BO_FLAG_FORCE_USER_VRAM	BIT(25)
+#define XE_BO_FLAG_NO_COMPRESSION	BIT(26)
 
 /* this one is trigger internally only */
 #define XE_BO_FLAG_INTERNAL_TEST	BIT(30)
diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c
index e91da9589c5f..0907868b32d6 100644
--- a/drivers/gpu/drm/xe/xe_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_debugfs.c
@@ -68,7 +68,7 @@ static int info(struct seq_file *m, void *data)
 	struct xe_gt *gt;
 	u8 id;
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 
 	drm_printf(&p, "graphics_verx100 %d\n", xe->info.graphics_verx100);
 	drm_printf(&p, "media_verx100 %d\n", xe->info.media_verx100);
@@ -93,9 +93,10 @@ static int info(struct seq_file *m, void *data)
 			   xe_force_wake_ref(gt_to_fw(gt), XE_FW_GT));
 		drm_printf(&p, "gt%d engine_mask 0x%llx\n", id,
 			   gt->info.engine_mask);
+		drm_printf(&p, "gt%d multi_queue_engine_class_mask 0x%x\n", id,
+			   gt->info.multi_queue_engine_class_mask);
 	}
 
-	xe_pm_runtime_put(xe);
 	return 0;
 }
 
@@ -110,9 +111,8 @@ static int sriov_info(struct seq_file *m, void *data)
 
 static int workarounds(struct xe_device *xe, struct drm_printer *p)
 {
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	xe_wa_device_dump(xe, p);
-	xe_pm_runtime_put(xe);
 
 	return 0;
 }
@@ -134,7 +134,7 @@ static int dgfx_pkg_residencies_show(struct seq_file *m, void *data)
 
 	xe = node_to_xe(m->private);
 	p = drm_seq_file_printer(m);
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	mmio = xe_root_tile_mmio(xe);
 	static const struct {
 		u32 offset;
@@ -151,7 +151,6 @@ static int dgfx_pkg_residencies_show(struct seq_file *m, void *data)
 	for (int i = 0; i < ARRAY_SIZE(residencies); i++)
 		read_residency_counter(xe, mmio, residencies[i].offset, residencies[i].name, &p);
 
-	xe_pm_runtime_put(xe);
 	return 0;
 }
 
@@ -163,7 +162,7 @@ static int dgfx_pcie_link_residencies_show(struct seq_file *m, void *data)
 
 	xe = node_to_xe(m->private);
 	p = drm_seq_file_printer(m);
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	mmio = xe_root_tile_mmio(xe);
 
 	static const struct {
@@ -178,7 +177,6 @@ static int dgfx_pcie_link_residencies_show(struct seq_file *m, void *data)
 	for (int i = 0; i < ARRAY_SIZE(residencies); i++)
 		read_residency_counter(xe, mmio, residencies[i].offset, residencies[i].name, &p);
 
-	xe_pm_runtime_put(xe);
 	return 0;
 }
 
@@ -277,16 +275,14 @@ static ssize_t wedged_mode_set(struct file *f, const char __user *ubuf,
 
 	xe->wedged.mode = wedged_mode;
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	for_each_gt(gt, xe, id) {
 		ret = xe_guc_ads_scheduler_policy_toggle_reset(&gt->uc.guc.ads);
 		if (ret) {
 			xe_gt_err(gt, "Failed to update GuC ADS scheduler policy. GuC may still cause engine reset even with wedged_mode=2\n");
-			xe_pm_runtime_put(xe);
 			return -EIO;
 		}
 	}
-	xe_pm_runtime_put(xe);
 
 	return size;
 }
@@ -297,6 +293,39 @@ static const struct file_operations wedged_mode_fops = {
 	.write = wedged_mode_set,
 };
 
+static ssize_t page_reclaim_hw_assist_show(struct file *f, char __user *ubuf,
+					   size_t size, loff_t *pos)
+{
+	struct xe_device *xe = file_inode(f)->i_private;
+	char buf[8];
+	int len;
+
+	len = scnprintf(buf, sizeof(buf), "%d\n", xe->info.has_page_reclaim_hw_assist);
+	return simple_read_from_buffer(ubuf, size, pos, buf, len);
+}
+
+static ssize_t page_reclaim_hw_assist_set(struct file *f, const char __user *ubuf,
+					  size_t size, loff_t *pos)
+{
+	struct xe_device *xe = file_inode(f)->i_private;
+	bool val;
+	ssize_t ret;
+
+	ret = kstrtobool_from_user(ubuf, size, &val);
+	if (ret)
+		return ret;
+
+	xe->info.has_page_reclaim_hw_assist = val;
+
+	return size;
+}
+
+static const struct file_operations page_reclaim_hw_assist_fops = {
+	.owner = THIS_MODULE,
+	.read = page_reclaim_hw_assist_show,
+	.write = page_reclaim_hw_assist_set,
+};
+
 static ssize_t atomic_svm_timeslice_ms_show(struct file *f, char __user *ubuf,
 					    size_t size, loff_t *pos)
 {
@@ -332,6 +361,74 @@ static const struct file_operations atomic_svm_timeslice_ms_fops = {
 	.write = atomic_svm_timeslice_ms_set,
 };
 
+static ssize_t min_run_period_lr_ms_show(struct file *f, char __user *ubuf,
+					 size_t size, loff_t *pos)
+{
+	struct xe_device *xe = file_inode(f)->i_private;
+	char buf[32];
+	int len = 0;
+
+	len = scnprintf(buf, sizeof(buf), "%d\n", xe->min_run_period_lr_ms);
+
+	return simple_read_from_buffer(ubuf, size, pos, buf, len);
+}
+
+static ssize_t min_run_period_lr_ms_set(struct file *f, const char __user *ubuf,
+					size_t size, loff_t *pos)
+{
+	struct xe_device *xe = file_inode(f)->i_private;
+	u32 min_run_period_lr_ms;
+	ssize_t ret;
+
+	ret = kstrtouint_from_user(ubuf, size, 0, &min_run_period_lr_ms);
+	if (ret)
+		return ret;
+
+	xe->min_run_period_lr_ms = min_run_period_lr_ms;
+
+	return size;
+}
+
+static const struct file_operations min_run_period_lr_ms_fops = {
+	.owner = THIS_MODULE,
+	.read = min_run_period_lr_ms_show,
+	.write = min_run_period_lr_ms_set,
+};
+
+static ssize_t min_run_period_pf_ms_show(struct file *f, char __user *ubuf,
+					 size_t size, loff_t *pos)
+{
+	struct xe_device *xe = file_inode(f)->i_private;
+	char buf[32];
+	int len = 0;
+
+	len = scnprintf(buf, sizeof(buf), "%d\n", xe->min_run_period_pf_ms);
+
+	return simple_read_from_buffer(ubuf, size, pos, buf, len);
+}
+
+static ssize_t min_run_period_pf_ms_set(struct file *f, const char __user *ubuf,
+					size_t size, loff_t *pos)
+{
+	struct xe_device *xe = file_inode(f)->i_private;
+	u32 min_run_period_pf_ms;
+	ssize_t ret;
+
+	ret = kstrtouint_from_user(ubuf, size, 0, &min_run_period_pf_ms);
+	if (ret)
+		return ret;
+
+	xe->min_run_period_pf_ms = min_run_period_pf_ms;
+
+	return size;
+}
+
+static const struct file_operations min_run_period_pf_ms_fops = {
+	.owner = THIS_MODULE,
+	.read = min_run_period_pf_ms_show,
+	.write = min_run_period_pf_ms_set,
+};
+
 static ssize_t disable_late_binding_show(struct file *f, char __user *ubuf,
 					 size_t size, loff_t *pos)
 {
@@ -375,7 +472,6 @@ void xe_debugfs_register(struct xe_device *xe)
 	struct ttm_resource_manager *man;
 	struct xe_tile *tile;
 	struct xe_gt *gt;
-	u32 mem_type;
 	u8 tile_id;
 	u8 id;
 
@@ -400,19 +496,22 @@ void xe_debugfs_register(struct xe_device *xe)
 	debugfs_create_file("atomic_svm_timeslice_ms", 0600, root, xe,
 			    &atomic_svm_timeslice_ms_fops);
 
-	debugfs_create_file("disable_late_binding", 0600, root, xe,
-			    &disable_late_binding_fops);
+	debugfs_create_file("min_run_period_lr_ms", 0600, root, xe,
+			    &min_run_period_lr_ms_fops);
 
-	for (mem_type = XE_PL_VRAM0; mem_type <= XE_PL_VRAM1; ++mem_type) {
-		man = ttm_manager_type(bdev, mem_type);
+	debugfs_create_file("min_run_period_pf_ms", 0600, root, xe,
+			    &min_run_period_pf_ms_fops);
 
-		if (man) {
-			char name[16];
+	debugfs_create_file("disable_late_binding", 0600, root, xe,
+			    &disable_late_binding_fops);
 
-			snprintf(name, sizeof(name), "vram%d_mm", mem_type - XE_PL_VRAM0);
-			ttm_resource_manager_create_debugfs(man, root, name);
-		}
-	}
+	/*
+	 * Don't expose page reclaim configuration file if not supported by the
+	 * hardware initially.
+	 */
+	if (xe->info.has_page_reclaim_hw_assist)
+		debugfs_create_file("page_reclaim_hw_assist", 0600, root, xe,
+				    &page_reclaim_hw_assist_fops);
 
 	man = ttm_manager_type(bdev, XE_PL_TT);
 	ttm_resource_manager_create_debugfs(man, root, "gtt_mm");
diff --git a/drivers/gpu/drm/xe/xe_devcoredump.c b/drivers/gpu/drm/xe/xe_devcoredump.c
index d444eda65ca6..7263c2a5f3a8 100644
--- a/drivers/gpu/drm/xe/xe_devcoredump.c
+++ b/drivers/gpu/drm/xe/xe_devcoredump.c
@@ -276,7 +276,6 @@ static void xe_devcoredump_deferred_snap_work(struct work_struct *work)
 	struct xe_devcoredump_snapshot *ss = container_of(work, typeof(*ss), work);
 	struct xe_devcoredump *coredump = container_of(ss, typeof(*coredump), snapshot);
 	struct xe_device *xe = coredump_to_xe(coredump);
-	unsigned int fw_ref;
 
 	/*
 	 * NB: Despite passing a GFP_ flags parameter here, more allocations are done
@@ -287,15 +286,15 @@ static void xe_devcoredump_deferred_snap_work(struct work_struct *work)
 			      xe_devcoredump_read, xe_devcoredump_free,
 			      XE_COREDUMP_TIMEOUT_JIFFIES);
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 
 	/* keep going if fw fails as we still want to save the memory and SW data */
-	fw_ref = xe_force_wake_get(gt_to_fw(ss->gt), XE_FORCEWAKE_ALL);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
-		xe_gt_info(ss->gt, "failed to get forcewake for coredump capture\n");
-	xe_vm_snapshot_capture_delayed(ss->vm);
-	xe_guc_exec_queue_snapshot_capture_delayed(ss->ge);
-	xe_force_wake_put(gt_to_fw(ss->gt), fw_ref);
+	xe_with_force_wake(fw_ref, gt_to_fw(ss->gt), XE_FORCEWAKE_ALL) {
+		if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
+			xe_gt_info(ss->gt, "failed to get forcewake for coredump capture\n");
+		xe_vm_snapshot_capture_delayed(ss->vm);
+		xe_guc_exec_queue_snapshot_capture_delayed(ss->ge);
+	}
 
 	ss->read.chunk_position = 0;
 
@@ -306,7 +305,7 @@ static void xe_devcoredump_deferred_snap_work(struct work_struct *work)
 		ss->read.buffer = kvmalloc(XE_DEVCOREDUMP_CHUNK_MAX,
 					   GFP_USER);
 		if (!ss->read.buffer)
-			goto put_pm;
+			return;
 
 		__xe_devcoredump_read(ss->read.buffer,
 				      XE_DEVCOREDUMP_CHUNK_MAX,
@@ -314,15 +313,12 @@ static void xe_devcoredump_deferred_snap_work(struct work_struct *work)
 	} else {
 		ss->read.buffer = kvmalloc(ss->read.size, GFP_USER);
 		if (!ss->read.buffer)
-			goto put_pm;
+			return;
 
 		__xe_devcoredump_read(ss->read.buffer, ss->read.size, 0,
 				      coredump);
 		xe_devcoredump_snapshot_free(ss);
 	}
-
-put_pm:
-	xe_pm_runtime_put(xe);
 }
 
 static void devcoredump_snapshot(struct xe_devcoredump *coredump,
@@ -332,7 +328,6 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump,
 	struct xe_devcoredump_snapshot *ss = &coredump->snapshot;
 	struct xe_guc *guc = exec_queue_to_guc(q);
 	const char *process_name = "no process";
-	unsigned int fw_ref;
 	bool cookie;
 
 	ss->snapshot_time = ktime_get_real();
@@ -348,10 +343,10 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump,
 	ss->gt = q->gt;
 	INIT_WORK(&ss->work, xe_devcoredump_deferred_snap_work);
 
-	cookie = dma_fence_begin_signalling();
-
 	/* keep going if fw fails as we still want to save the memory and SW data */
-	fw_ref = xe_force_wake_get(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
+
+	cookie = dma_fence_begin_signalling();
 
 	ss->guc.log = xe_guc_log_snapshot_capture(&guc->log, true);
 	ss->guc.ct = xe_guc_ct_snapshot_capture(&guc->ct);
@@ -364,7 +359,6 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump,
 
 	queue_work(system_unbound_wq, &ss->work);
 
-	xe_force_wake_put(gt_to_fw(q->gt), fw_ref);
 	dma_fence_end_signalling(cookie);
 }
 
diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
index c7d373c70f0f..00afc84a8683 100644
--- a/drivers/gpu/drm/xe/xe_device.c
+++ b/drivers/gpu/drm/xe/xe_device.c
@@ -166,7 +166,7 @@ static void xe_file_close(struct drm_device *dev, struct drm_file *file)
 	struct xe_exec_queue *q;
 	unsigned long idx;
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 
 	/*
 	 * No need for exec_queue.lock here as there is no contention for it
@@ -177,15 +177,18 @@ static void xe_file_close(struct drm_device *dev, struct drm_file *file)
 	xa_for_each(&xef->exec_queue.xa, idx, q) {
 		if (q->vm && q->hwe->hw_engine_group)
 			xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
-		xe_exec_queue_kill(q);
+
+		if (xe_exec_queue_is_multi_queue_primary(q))
+			xe_exec_queue_group_kill_put(q->multi_queue.group);
+		else
+			xe_exec_queue_kill(q);
+
 		xe_exec_queue_put(q);
 	}
 	xa_for_each(&xef->vm.xa, idx, vm)
 		xe_vm_close_and_put(vm);
 
 	xe_file_put(xef);
-
-	xe_pm_runtime_put(xe);
 }
 
 static const struct drm_ioctl_desc xe_ioctls[] = {
@@ -209,6 +212,8 @@ static const struct drm_ioctl_desc xe_ioctls[] = {
 	DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl,
 			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl,
+			  DRM_RENDER_ALLOW),
 };
 
 static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
@@ -220,10 +225,10 @@ static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 	if (xe_device_wedged(xe))
 		return -ECANCELED;
 
-	ret = xe_pm_runtime_get_ioctl(xe);
+	ACQUIRE(xe_pm_runtime_ioctl, pm)(xe);
+	ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm);
 	if (ret >= 0)
 		ret = drm_ioctl(file, cmd, arg);
-	xe_pm_runtime_put(xe);
 
 	return ret;
 }
@@ -238,10 +243,10 @@ static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned lo
 	if (xe_device_wedged(xe))
 		return -ECANCELED;
 
-	ret = xe_pm_runtime_get_ioctl(xe);
+	ACQUIRE(xe_pm_runtime_ioctl, pm)(xe);
+	ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm);
 	if (ret >= 0)
 		ret = drm_compat_ioctl(file, cmd, arg);
-	xe_pm_runtime_put(xe);
 
 	return ret;
 }
@@ -455,6 +460,7 @@ struct xe_device *xe_device_create(struct pci_dev *pdev,
 	xe->info.revid = pdev->revision;
 	xe->info.force_execlist = xe_modparam.force_execlist;
 	xe->atomic_svm_timeslice_ms = 5;
+	xe->min_run_period_lr_ms = 5;
 
 	err = xe_irq_init(xe);
 	if (err)
@@ -775,7 +781,6 @@ ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */
 static int probe_has_flat_ccs(struct xe_device *xe)
 {
 	struct xe_gt *gt;
-	unsigned int fw_ref;
 	u32 reg;
 
 	/* Always enabled/disabled, no runtime check to do */
@@ -786,8 +791,8 @@ static int probe_has_flat_ccs(struct xe_device *xe)
 	if (!gt)
 		return 0;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 
 	reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER);
@@ -797,11 +802,64 @@ static int probe_has_flat_ccs(struct xe_device *xe)
 		drm_dbg(&xe->drm,
 			"Flat CCS has been disabled in bios, May lead to performance impact");
 
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-
 	return 0;
 }
 
+/*
+ * Detect if the driver is being run on pre-production hardware.  We don't
+ * keep workarounds for pre-production hardware long term, so print an
+ * error and add taint if we're being loaded on a pre-production platform
+ * for which the pre-prod workarounds have already been removed.
+ *
+ * The general policy is that we'll remove any workarounds that only apply to
+ * pre-production hardware around the time force_probe restrictions are lifted
+ * for a platform of the next major IP generation (for example, Xe2 pre-prod
+ * workarounds should be removed around the time the first Xe3 platforms have
+ * force_probe lifted).
+ */
+static void detect_preproduction_hw(struct xe_device *xe)
+{
+	struct xe_gt *gt;
+	int id;
+
+	/*
+	 * SR-IOV VFs don't have access to the FUSE2 register, so we can't
+	 * check pre-production status there.  But the host OS will notice
+	 * and report the pre-production status, which should be enough to
+	 * help us catch mistaken use of pre-production hardware.
+	 */
+	if (IS_SRIOV_VF(xe))
+		return;
+
+	/*
+	 * The "SW_CAP" fuse contains a bit indicating whether the device is a
+	 * production or pre-production device.  This fuse is reflected through
+	 * the GT "FUSE2" register, even though the contents of the fuse are
+	 * not GT-specific.  Every GT's reflection of this fuse should show the
+	 * same value, so we'll just use the first available GT for lookup.
+	 */
+	for_each_gt(gt, xe, id)
+		break;
+
+	if (!gt)
+		return;
+
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) {
+		xe_gt_err(gt, "Forcewake failure; cannot determine production/pre-production hw status.\n");
+		return;
+	}
+
+	if (xe_mmio_read32(&gt->mmio, FUSE2) & PRODUCTION_HW)
+		return;
+
+	xe_info(xe, "Pre-production hardware detected.\n");
+	if (!xe->info.has_pre_prod_wa) {
+		xe_err(xe, "Pre-production workarounds for this platform have already been removed.\n");
+		add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK);
+	}
+}
+
 int xe_device_probe(struct xe_device *xe)
 {
 	struct xe_tile *tile;
@@ -972,6 +1030,8 @@ int xe_device_probe(struct xe_device *xe)
 	if (err)
 		goto err_unregister_display;
 
+	detect_preproduction_hw(xe);
+
 	return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe);
 
 err_unregister_display:
@@ -1034,7 +1094,6 @@ void xe_device_wmb(struct xe_device *xe)
  */
 static void tdf_request_sync(struct xe_device *xe)
 {
-	unsigned int fw_ref;
 	struct xe_gt *gt;
 	u8 id;
 
@@ -1042,8 +1101,8 @@ static void tdf_request_sync(struct xe_device *xe)
 		if (xe_gt_is_media_type(gt))
 			continue;
 
-		fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-		if (!fw_ref)
+		CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+		if (!fw_ref.domains)
 			return;
 
 		xe_mmio_write32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST);
@@ -1056,17 +1115,14 @@ static void tdf_request_sync(struct xe_device *xe)
 		 * transient and need to be flushed..
 		 */
 		if (xe_mmio_wait32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0,
-				   150, NULL, false))
+				   300, NULL, false))
 			xe_gt_err_once(gt, "TD flush timeout\n");
-
-		xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	}
 }
 
 void xe_device_l2_flush(struct xe_device *xe)
 {
 	struct xe_gt *gt;
-	unsigned int fw_ref;
 
 	gt = xe_root_mmio_gt(xe);
 	if (!gt)
@@ -1075,8 +1131,8 @@ void xe_device_l2_flush(struct xe_device *xe)
 	if (!XE_GT_WA(gt, 16023588340))
 		return;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!fw_ref.domains)
 		return;
 
 	spin_lock(&gt->global_invl_lock);
@@ -1086,8 +1142,6 @@ void xe_device_l2_flush(struct xe_device *xe)
 		xe_gt_err_once(gt, "Global invalidation timeout\n");
 
 	spin_unlock(&gt->global_invl_lock);
-
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h
index 32cc6323b7f6..6604b89330d5 100644
--- a/drivers/gpu/drm/xe/xe_device.h
+++ b/drivers/gpu/drm/xe/xe_device.h
@@ -172,6 +172,11 @@ static inline bool xe_device_has_lmtt(struct xe_device *xe)
 	return IS_DGFX(xe);
 }
 
+static inline bool xe_device_has_mert(struct xe_device *xe)
+{
+	return xe->info.has_mert;
+}
+
 u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size);
 
 void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p);
diff --git a/drivers/gpu/drm/xe/xe_device_sysfs.c b/drivers/gpu/drm/xe/xe_device_sysfs.c
index ec9c06b06fb5..a73e0e957cb0 100644
--- a/drivers/gpu/drm/xe/xe_device_sysfs.c
+++ b/drivers/gpu/drm/xe/xe_device_sysfs.c
@@ -57,9 +57,8 @@ vram_d3cold_threshold_store(struct device *dev, struct device_attribute *attr,
 
 	drm_dbg(&xe->drm, "vram_d3cold_threshold: %u\n", vram_d3cold_threshold);
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	ret = xe_pm_set_vram_threshold(xe, vram_d3cold_threshold);
-	xe_pm_runtime_put(xe);
 
 	return ret ?: count;
 }
@@ -84,33 +83,31 @@ lb_fan_control_version_show(struct device *dev, struct device_attribute *attr, c
 	u16 major = 0, minor = 0, hotfix = 0, build = 0;
 	int ret;
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 
 	ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_CAPABILITY_STATUS, 0),
 			    &cap, NULL);
 	if (ret)
-		goto out;
+		return ret;
 
 	if (REG_FIELD_GET(V1_FAN_PROVISIONED, cap)) {
 		ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_LOW, 0),
 				    &ver_low, NULL);
 		if (ret)
-			goto out;
+			return ret;
 
 		ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_HIGH, 0),
 				    &ver_high, NULL);
 		if (ret)
-			goto out;
+			return ret;
 
 		major = REG_FIELD_GET(MAJOR_VERSION_MASK, ver_low);
 		minor = REG_FIELD_GET(MINOR_VERSION_MASK, ver_low);
 		hotfix = REG_FIELD_GET(HOTFIX_VERSION_MASK, ver_high);
 		build = REG_FIELD_GET(BUILD_VERSION_MASK, ver_high);
 	}
-out:
-	xe_pm_runtime_put(xe);
 
-	return ret ?: sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build);
+	return sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build);
 }
 static DEVICE_ATTR_ADMIN_RO(lb_fan_control_version);
 
@@ -123,33 +120,31 @@ lb_voltage_regulator_version_show(struct device *dev, struct device_attribute *a
 	u16 major = 0, minor = 0, hotfix = 0, build = 0;
 	int ret;
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 
 	ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_CAPABILITY_STATUS, 0),
 			    &cap, NULL);
 	if (ret)
-		goto out;
+		return ret;
 
 	if (REG_FIELD_GET(VR_PARAMS_PROVISIONED, cap)) {
 		ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_LOW, 0),
 				    &ver_low, NULL);
 		if (ret)
-			goto out;
+			return ret;
 
 		ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_HIGH, 0),
 				    &ver_high, NULL);
 		if (ret)
-			goto out;
+			return ret;
 
 		major = REG_FIELD_GET(MAJOR_VERSION_MASK, ver_low);
 		minor = REG_FIELD_GET(MINOR_VERSION_MASK, ver_low);
 		hotfix = REG_FIELD_GET(HOTFIX_VERSION_MASK, ver_high);
 		build = REG_FIELD_GET(BUILD_VERSION_MASK, ver_high);
 	}
-out:
-	xe_pm_runtime_put(xe);
 
-	return ret ?: sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build);
+	return sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build);
 }
 static DEVICE_ATTR_ADMIN_RO(lb_voltage_regulator_version);
 
@@ -233,9 +228,8 @@ auto_link_downgrade_capable_show(struct device *dev, struct device_attribute *at
 	struct xe_device *xe = pdev_to_xe_device(pdev);
 	u32 cap, val;
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	val = xe_mmio_read32(xe_root_tile_mmio(xe), BMG_PCIE_CAP);
-	xe_pm_runtime_put(xe);
 
 	cap = REG_FIELD_GET(LINK_DOWNGRADE, val);
 	return sysfs_emit(buf, "%u\n", cap == DOWNGRADE_CAPABLE);
@@ -251,11 +245,10 @@ auto_link_downgrade_status_show(struct device *dev, struct device_attribute *att
 	u32 val = 0;
 	int ret;
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	ret = xe_pcode_read(xe_device_get_root_tile(xe),
 			    PCODE_MBOX(DGFX_PCODE_STATUS, DGFX_GET_INIT_STATUS, 0),
 			    &val, NULL);
-	xe_pm_runtime_put(xe);
 
 	return ret ?: sysfs_emit(buf, "%u\n", REG_FIELD_GET(DGFX_LINK_DOWNGRADE_STATUS, val));
 }
diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h
index 0b2fa7c56d38..dad355fec50c 100644
--- a/drivers/gpu/drm/xe/xe_device_types.h
+++ b/drivers/gpu/drm/xe/xe_device_types.h
@@ -17,6 +17,7 @@
 #include "xe_late_bind_fw_types.h"
 #include "xe_lmtt_types.h"
 #include "xe_memirq_types.h"
+#include "xe_mert.h"
 #include "xe_oa_types.h"
 #include "xe_pagefault_types.h"
 #include "xe_platform_types.h"
@@ -35,7 +36,6 @@
 #define TEST_VM_OPS_ERROR
 #endif
 
-struct dram_info;
 struct intel_display;
 struct intel_dg_nvm_dev;
 struct xe_ggtt;
@@ -184,6 +184,13 @@ struct xe_tile {
 		 * Media GT shares a pool with its primary GT.
 		 */
 		struct xe_sa_manager *kernel_bb_pool;
+
+		/**
+		 * @mem.reclaim_pool: Pool for PRLs allocated.
+		 *
+		 * Only main GT has page reclaim list allocations.
+		 */
+		struct xe_sa_manager *reclaim_pool;
 	} mem;
 
 	/** @sriov: tile level virtualization data */
@@ -220,6 +227,9 @@ struct xe_tile {
 
 	/** @debugfs: debugfs directory associated with this tile */
 	struct dentry *debugfs;
+
+	/** @mert: MERT-related data */
+	struct xe_mert mert;
 };
 
 /**
@@ -286,6 +296,8 @@ struct xe_device {
 		u8 has_asid:1;
 		/** @info.has_atomic_enable_pte_bit: Device has atomic enable PTE bit */
 		u8 has_atomic_enable_pte_bit:1;
+		/** @info.has_cached_pt: Supports caching pagetable */
+		u8 has_cached_pt:1;
 		/** @info.has_device_atomics_on_smem: Supports device atomics on SMEM */
 		u8 has_device_atomics_on_smem:1;
 		/** @info.has_fan_control: Device supports fan control */
@@ -298,6 +310,8 @@ struct xe_device {
 		u8 has_heci_cscfi:1;
 		/** @info.has_heci_gscfi: device has heci gscfi */
 		u8 has_heci_gscfi:1;
+		/** @info.has_i2c: Device has I2C controller */
+		u8 has_i2c:1;
 		/** @info.has_late_bind: Device has firmware late binding support */
 		u8 has_late_bind:1;
 		/** @info.has_llc: Device has a shared CPU+GPU last level cache */
@@ -308,6 +322,12 @@ struct xe_device {
 		u8 has_mbx_power_limits:1;
 		/** @info.has_mem_copy_instr: Device supports MEM_COPY instruction */
 		u8 has_mem_copy_instr:1;
+		/** @info.has_mert: Device has standalone MERT */
+		u8 has_mert:1;
+		/** @info.has_page_reclaim_hw_assist: Device supports page reclamation feature */
+		u8 has_page_reclaim_hw_assist:1;
+		/** @info.has_pre_prod_wa: Pre-production workarounds still present in driver */
+		u8 has_pre_prod_wa:1;
 		/** @info.has_pxp: Device has PXP support */
 		u8 has_pxp:1;
 		/** @info.has_range_tlb_inval: Has range based TLB invalidations */
@@ -606,6 +626,12 @@ struct xe_device {
 	/** @atomic_svm_timeslice_ms: Atomic SVM fault timeslice MS */
 	u32 atomic_svm_timeslice_ms;
 
+	/** @min_run_period_lr_ms: LR VM (preempt fence mode) timeslice */
+	u32 min_run_period_lr_ms;
+
+	/** @min_run_period_pf_ms: LR VM (page fault mode) timeslice */
+	u32 min_run_period_pf_ms;
+
 #ifdef TEST_VM_OPS_ERROR
 	/**
 	 * @vm_inject_error_position: inject errors at different places in VM
@@ -648,13 +674,6 @@ struct xe_device {
 	 * drm_i915_private during build. After cleanup these should go away,
 	 * migrating to the right sub-structs
 	 */
-	const struct dram_info *dram_info;
-
-	/*
-	 * edram size in MB.
-	 * Cannot be determined by PCIID. You must always read a register.
-	 */
-	u32 edram_size_mb;
 
 	struct intel_uncore {
 		spinlock_t lock;
diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c
index 54e42960daad..7c74a31d4486 100644
--- a/drivers/gpu/drm/xe/xe_dma_buf.c
+++ b/drivers/gpu/drm/xe/xe_dma_buf.c
@@ -124,7 +124,7 @@ static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach,
 	case XE_PL_TT:
 		sgt = drm_prime_pages_to_sg(obj->dev,
 					    bo->ttm.ttm->pages,
-					    bo->ttm.ttm->num_pages);
+					    obj->size >> PAGE_SHIFT);
 		if (IS_ERR(sgt))
 			return sgt;
 
diff --git a/drivers/gpu/drm/xe/xe_drm_client.c b/drivers/gpu/drm/xe/xe_drm_client.c
index f931ff9b1ec0..2787bbb36141 100644
--- a/drivers/gpu/drm/xe/xe_drm_client.c
+++ b/drivers/gpu/drm/xe/xe_drm_client.c
@@ -285,32 +285,31 @@ static struct xe_hw_engine *any_engine(struct xe_device *xe)
 	return NULL;
 }
 
-static bool force_wake_get_any_engine(struct xe_device *xe,
-				      struct xe_hw_engine **phwe,
-				      unsigned int *pfw_ref)
+/*
+ * Pick any engine and grab its forcewake.  On error phwe will be NULL and
+ * the returned forcewake reference will be invalid.  Callers should check
+ * phwe against NULL.
+ */
+static struct xe_force_wake_ref force_wake_get_any_engine(struct xe_device *xe,
+							  struct xe_hw_engine **phwe)
 {
 	enum xe_force_wake_domains domain;
-	unsigned int fw_ref;
+	struct xe_force_wake_ref fw_ref = {};
 	struct xe_hw_engine *hwe;
-	struct xe_force_wake *fw;
+
+	*phwe = NULL;
 
 	hwe = any_engine(xe);
 	if (!hwe)
-		return false;
+		return fw_ref;	/* will be invalid */
 
 	domain = xe_hw_engine_to_fw_domain(hwe);
-	fw = gt_to_fw(hwe->gt);
-
-	fw_ref = xe_force_wake_get(fw, domain);
-	if (!xe_force_wake_ref_has_domain(fw_ref, domain)) {
-		xe_force_wake_put(fw, fw_ref);
-		return false;
-	}
 
-	*phwe = hwe;
-	*pfw_ref = fw_ref;
+	fw_ref = xe_force_wake_constructor(gt_to_fw(hwe->gt), domain);
+	if (xe_force_wake_ref_has_domain(fw_ref.domains, domain))
+		*phwe = hwe;	/* valid forcewake */
 
-	return true;
+	return fw_ref;
 }
 
 static void show_run_ticks(struct drm_printer *p, struct drm_file *file)
@@ -322,7 +321,6 @@ static void show_run_ticks(struct drm_printer *p, struct drm_file *file)
 	struct xe_hw_engine *hwe;
 	struct xe_exec_queue *q;
 	u64 gpu_timestamp;
-	unsigned int fw_ref;
 
 	/*
 	 * RING_TIMESTAMP registers are inaccessible in VF mode.
@@ -339,29 +337,26 @@ static void show_run_ticks(struct drm_printer *p, struct drm_file *file)
 	wait_var_event(&xef->exec_queue.pending_removal,
 		       !atomic_read(&xef->exec_queue.pending_removal));
 
-	xe_pm_runtime_get(xe);
-	if (!force_wake_get_any_engine(xe, &hwe, &fw_ref)) {
-		xe_pm_runtime_put(xe);
-		return;
-	}
-
-	/* Accumulate all the exec queues from this client */
-	mutex_lock(&xef->exec_queue.lock);
-	xa_for_each(&xef->exec_queue.xa, i, q) {
-		xe_exec_queue_get(q);
-		mutex_unlock(&xef->exec_queue.lock);
-
-		xe_exec_queue_update_run_ticks(q);
+	scoped_guard(xe_pm_runtime, xe) {
+		CLASS(xe_force_wake_release_only, fw_ref)(force_wake_get_any_engine(xe, &hwe));
+		if (!hwe)
+			return;
 
+		/* Accumulate all the exec queues from this client */
 		mutex_lock(&xef->exec_queue.lock);
-		xe_exec_queue_put(q);
-	}
-	mutex_unlock(&xef->exec_queue.lock);
+		xa_for_each(&xef->exec_queue.xa, i, q) {
+			xe_exec_queue_get(q);
+			mutex_unlock(&xef->exec_queue.lock);
 
-	gpu_timestamp = xe_hw_engine_read_timestamp(hwe);
+			xe_exec_queue_update_run_ticks(q);
 
-	xe_force_wake_put(gt_to_fw(hwe->gt), fw_ref);
-	xe_pm_runtime_put(xe);
+			mutex_lock(&xef->exec_queue.lock);
+			xe_exec_queue_put(q);
+		}
+		mutex_unlock(&xef->exec_queue.lock);
+
+		gpu_timestamp = xe_hw_engine_read_timestamp(hwe);
+	}
 
 	for (class = 0; class < XE_ENGINE_CLASS_MAX; class++) {
 		const char *class_name;
diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c
index 97dfb7945b7a..a5c36a317a70 100644
--- a/drivers/gpu/drm/xe/xe_eu_stall.c
+++ b/drivers/gpu/drm/xe/xe_eu_stall.c
@@ -315,7 +315,7 @@ static int xe_eu_stall_user_ext_set_property(struct xe_device *xe, u64 extension
 		return -EFAULT;
 
 	if (XE_IOCTL_DBG(xe, ext.property >= ARRAY_SIZE(xe_set_eu_stall_property_funcs)) ||
-	    XE_IOCTL_DBG(xe, ext.pad))
+	    XE_IOCTL_DBG(xe, !ext.property) || XE_IOCTL_DBG(xe, ext.pad))
 		return -EINVAL;
 
 	idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_set_eu_stall_property_funcs));
diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c
index 4d81210e41f5..730a5c9c2637 100644
--- a/drivers/gpu/drm/xe/xe_exec.c
+++ b/drivers/gpu/drm/xe/xe_exec.c
@@ -121,7 +121,7 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 	u64 addresses[XE_HW_ENGINE_MAX_INSTANCE];
 	struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn};
 	struct drm_exec *exec = &vm_exec.exec;
-	u32 i, num_syncs, num_ufence = 0;
+	u32 i, num_syncs, num_in_sync = 0, num_ufence = 0;
 	struct xe_validation_ctx ctx;
 	struct xe_sched_job *job;
 	struct xe_vm *vm;
@@ -132,7 +132,8 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 
 	if (XE_IOCTL_DBG(xe, args->extensions) ||
 	    XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) ||
-	    XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
+	    XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]) ||
+	    XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS))
 		return -EINVAL;
 
 	q = xe_exec_queue_lookup(xef, args->exec_queue_id);
@@ -182,6 +183,9 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 
 		if (xe_sync_is_ufence(&syncs[num_syncs]))
 			num_ufence++;
+
+		if (!num_in_sync && xe_sync_needs_wait(&syncs[num_syncs]))
+			num_in_sync++;
 	}
 
 	if (XE_IOCTL_DBG(xe, num_ufence > 1)) {
@@ -202,7 +206,9 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 	mode = xe_hw_engine_group_find_exec_mode(q);
 
 	if (mode == EXEC_MODE_DMA_FENCE) {
-		err = xe_hw_engine_group_get_mode(group, mode, &previous_mode);
+		err = xe_hw_engine_group_get_mode(group, mode, &previous_mode,
+						  syncs, num_in_sync ?
+						  num_syncs : 0);
 		if (err)
 			goto err_syncs;
 	}
diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c
index 8724f8de67e2..41023a464480 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue.c
+++ b/drivers/gpu/drm/xe/xe_exec_queue.c
@@ -13,6 +13,7 @@
 #include <drm/drm_syncobj.h>
 #include <uapi/drm/xe_drm.h>
 
+#include "xe_bo.h"
 #include "xe_dep_scheduler.h"
 #include "xe_device.h"
 #include "xe_gt.h"
@@ -53,6 +54,54 @@
  * the ring operations the different engine classes support.
  */
 
+/**
+ * DOC: Multi Queue Group
+ *
+ * Multi Queue Group is another mode of execution supported by the compute
+ * and blitter copy command streamers (CCS and BCS, respectively). It is
+ * an enhancement of the existing hardware architecture and leverages the
+ * same submission model. It enables support for efficient, parallel
+ * execution of multiple queues within a single shared context. The multi
+ * queue group functionality is only supported with GuC submission backend.
+ * All the queues of a group must use the same address space (VM).
+ *
+ * The DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE execution queue property
+ * supports creating a multi queue group and adding queues to a queue group.
+ *
+ * The XE_EXEC_QUEUE_CREATE ioctl call with above property with value field
+ * set to DRM_XE_MULTI_GROUP_CREATE, will create a new multi queue group with
+ * the queue being created as the primary queue (aka q0) of the group. To add
+ * secondary queues to the group, they need to be created with the above
+ * property with id of the primary queue as the value. The properties of
+ * the primary queue (like priority, time slice) applies to the whole group.
+ * So, these properties can't be set for secondary queues of a group.
+ *
+ * The hardware does not support removing a queue from a multi-queue group.
+ * However, queues can be dynamically added to the group. A group can have
+ * up to 64 queues. To support this, XeKMD holds references to LRCs of the
+ * queues even after the queues are destroyed by the user until the whole
+ * group is destroyed. The secondary queues hold a reference to the primary
+ * queue thus preventing the group from being destroyed when user destroys
+ * the primary queue. Once the primary queue is destroyed, secondary queues
+ * can't be added to the queue group, but they can continue to submit the
+ * jobs if the DRM_XE_MULTI_GROUP_KEEP_ACTIVE flag is set during the multi
+ * queue group creation.
+ *
+ * The queues of a multi queue group can set their priority within the group
+ * through the DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY property.
+ * This multi queue priority can also be set dynamically through the
+ * XE_EXEC_QUEUE_SET_PROPERTY ioctl. This is the only other property
+ * supported by the secondary queues of a multi queue group, other than
+ * DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE.
+ *
+ * When GuC reports an error on any of the queues of a multi queue group,
+ * the queue cleanup mechanism is invoked for all the queues of the group
+ * as hardware cannot make progress on the multi queue context.
+ *
+ * Refer :ref:`multi-queue-group-guc-interface` for multi queue group GuC
+ * interface.
+ */
+
 enum xe_exec_queue_sched_prop {
 	XE_EXEC_QUEUE_JOB_TIMEOUT = 0,
 	XE_EXEC_QUEUE_TIMESLICE = 1,
@@ -61,7 +110,35 @@ enum xe_exec_queue_sched_prop {
 };
 
 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
-				      u64 extensions, int ext_number);
+				      u64 extensions);
+
+static void xe_exec_queue_group_cleanup(struct xe_exec_queue *q)
+{
+	struct xe_exec_queue_group *group = q->multi_queue.group;
+	struct xe_lrc *lrc;
+	unsigned long idx;
+
+	if (xe_exec_queue_is_multi_queue_secondary(q)) {
+		/*
+		 * Put pairs with get from xe_exec_queue_lookup() call
+		 * in xe_exec_queue_group_validate().
+		 */
+		xe_exec_queue_put(xe_exec_queue_multi_queue_primary(q));
+		return;
+	}
+
+	if (!group)
+		return;
+
+	/* Primary queue cleanup */
+	xa_for_each(&group->xa, idx, lrc)
+		xe_lrc_put(lrc);
+
+	xa_destroy(&group->xa);
+	mutex_destroy(&group->list_lock);
+	xe_bo_unpin_map_no_vm(group->cgp_bo);
+	kfree(group);
+}
 
 static void __xe_exec_queue_free(struct xe_exec_queue *q)
 {
@@ -73,12 +150,17 @@ static void __xe_exec_queue_free(struct xe_exec_queue *q)
 
 	if (xe_exec_queue_uses_pxp(q))
 		xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q);
+
+	if (xe_exec_queue_is_multi_queue(q))
+		xe_exec_queue_group_cleanup(q);
+
 	if (q->vm)
 		xe_vm_put(q->vm);
 
 	if (q->xef)
 		xe_file_put(q->xef);
 
+	kvfree(q->replay_state);
 	kfree(q);
 }
 
@@ -147,6 +229,7 @@ static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,
 	INIT_LIST_HEAD(&q->multi_gt_link);
 	INIT_LIST_HEAD(&q->hw_engine_group_link);
 	INIT_LIST_HEAD(&q->pxp.link);
+	q->multi_queue.priority = XE_MULTI_QUEUE_PRIORITY_NORMAL;
 
 	q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us;
 	q->sched_props.preempt_timeout_us =
@@ -175,7 +258,7 @@ static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,
 		 * may set q->usm, must come before xe_lrc_create(),
 		 * may overwrite q->sched_props, must come before q->ops->init()
 		 */
-		err = exec_queue_user_extensions(xe, q, extensions, 0);
+		err = exec_queue_user_extensions(xe, q, extensions);
 		if (err) {
 			__xe_exec_queue_free(q);
 			return ERR_PTR(err);
@@ -225,8 +308,8 @@ static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags)
 		struct xe_lrc *lrc;
 
 		xe_gt_sriov_vf_wait_valid_ggtt(q->gt);
-		lrc = xe_lrc_create(q->hwe, q->vm, xe_lrc_ring_size(),
-				    q->msix_vec, flags);
+		lrc = xe_lrc_create(q->hwe, q->vm, q->replay_state,
+				    xe_lrc_ring_size(), q->msix_vec, flags);
 		if (IS_ERR(lrc)) {
 			err = PTR_ERR(lrc);
 			goto err_lrc;
@@ -383,6 +466,26 @@ struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe,
 }
 ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO);
 
+static void xe_exec_queue_group_kill(struct kref *ref)
+{
+	struct xe_exec_queue_group *group = container_of(ref, struct xe_exec_queue_group,
+							 kill_refcount);
+	xe_exec_queue_kill(group->primary);
+}
+
+static inline void xe_exec_queue_group_kill_get(struct xe_exec_queue_group *group)
+{
+	kref_get(&group->kill_refcount);
+}
+
+void xe_exec_queue_group_kill_put(struct xe_exec_queue_group *group)
+{
+	if (!group)
+		return;
+
+	kref_put(&group->kill_refcount, xe_exec_queue_group_kill);
+}
+
 void xe_exec_queue_destroy(struct kref *ref)
 {
 	struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount);
@@ -567,6 +670,217 @@ exec_queue_set_pxp_type(struct xe_device *xe, struct xe_exec_queue *q, u64 value
 	return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM);
 }
 
+static int exec_queue_set_hang_replay_state(struct xe_device *xe,
+					    struct xe_exec_queue *q,
+					    u64 value)
+{
+	size_t size = xe_gt_lrc_hang_replay_size(q->gt, q->class);
+	u64 __user *address = u64_to_user_ptr(value);
+	void *ptr;
+
+	ptr = vmemdup_user(address, size);
+	if (XE_IOCTL_DBG(xe, IS_ERR(ptr)))
+		return PTR_ERR(ptr);
+
+	q->replay_state = ptr;
+
+	return 0;
+}
+
+static int xe_exec_queue_group_init(struct xe_device *xe, struct xe_exec_queue *q)
+{
+	struct xe_tile *tile = gt_to_tile(q->gt);
+	struct xe_exec_queue_group *group;
+	struct xe_bo *bo;
+
+	group = kzalloc(sizeof(*group), GFP_KERNEL);
+	if (!group)
+		return -ENOMEM;
+
+	bo = xe_bo_create_pin_map_novm(xe, tile, SZ_4K, ttm_bo_type_kernel,
+				       XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+				       XE_BO_FLAG_PINNED_LATE_RESTORE |
+				       XE_BO_FLAG_FORCE_USER_VRAM |
+				       XE_BO_FLAG_GGTT_INVALIDATE |
+				       XE_BO_FLAG_GGTT, false);
+	if (IS_ERR(bo)) {
+		drm_err(&xe->drm, "CGP bo allocation for queue group failed: %ld\n",
+			PTR_ERR(bo));
+		kfree(group);
+		return PTR_ERR(bo);
+	}
+
+	xe_map_memset(xe, &bo->vmap, 0, 0, SZ_4K);
+
+	group->primary = q;
+	group->cgp_bo = bo;
+	INIT_LIST_HEAD(&group->list);
+	kref_init(&group->kill_refcount);
+	xa_init_flags(&group->xa, XA_FLAGS_ALLOC1);
+	mutex_init(&group->list_lock);
+	q->multi_queue.group = group;
+
+	/* group->list_lock is used in submission backend */
+	if (IS_ENABLED(CONFIG_LOCKDEP)) {
+		fs_reclaim_acquire(GFP_KERNEL);
+		might_lock(&group->list_lock);
+		fs_reclaim_release(GFP_KERNEL);
+	}
+
+	return 0;
+}
+
+static inline bool xe_exec_queue_supports_multi_queue(struct xe_exec_queue *q)
+{
+	return q->gt->info.multi_queue_engine_class_mask & BIT(q->class);
+}
+
+static int xe_exec_queue_group_validate(struct xe_device *xe, struct xe_exec_queue *q,
+					u32 primary_id)
+{
+	struct xe_exec_queue_group *group;
+	struct xe_exec_queue *primary;
+	int ret;
+
+	/*
+	 * Get from below xe_exec_queue_lookup() pairs with put
+	 * in xe_exec_queue_group_cleanup().
+	 */
+	primary = xe_exec_queue_lookup(q->vm->xef, primary_id);
+	if (XE_IOCTL_DBG(xe, !primary))
+		return -ENOENT;
+
+	if (XE_IOCTL_DBG(xe, !xe_exec_queue_is_multi_queue_primary(primary)) ||
+	    XE_IOCTL_DBG(xe, q->vm != primary->vm) ||
+	    XE_IOCTL_DBG(xe, q->logical_mask != primary->logical_mask)) {
+		ret = -EINVAL;
+		goto put_primary;
+	}
+
+	group = primary->multi_queue.group;
+	q->multi_queue.valid = true;
+	q->multi_queue.group = group;
+
+	return 0;
+put_primary:
+	xe_exec_queue_put(primary);
+	return ret;
+}
+
+#define XE_MAX_GROUP_SIZE	64
+static int xe_exec_queue_group_add(struct xe_device *xe, struct xe_exec_queue *q)
+{
+	struct xe_exec_queue_group *group = q->multi_queue.group;
+	u32 pos;
+	int err;
+
+	xe_assert(xe, xe_exec_queue_is_multi_queue_secondary(q));
+
+	/* Primary queue holds a reference to LRCs of all secondary queues */
+	err = xa_alloc(&group->xa, &pos, xe_lrc_get(q->lrc[0]),
+		       XA_LIMIT(1, XE_MAX_GROUP_SIZE - 1), GFP_KERNEL);
+	if (XE_IOCTL_DBG(xe, err)) {
+		xe_lrc_put(q->lrc[0]);
+
+		/* It is invalid if queue group limit is exceeded */
+		if (err == -EBUSY)
+			err = -EINVAL;
+
+		return err;
+	}
+
+	q->multi_queue.pos = pos;
+
+	if (group->primary->multi_queue.keep_active) {
+		xe_exec_queue_group_kill_get(group);
+		q->multi_queue.keep_active = true;
+	}
+
+	return 0;
+}
+
+static void xe_exec_queue_group_delete(struct xe_device *xe, struct xe_exec_queue *q)
+{
+	struct xe_exec_queue_group *group = q->multi_queue.group;
+	struct xe_lrc *lrc;
+
+	xe_assert(xe, xe_exec_queue_is_multi_queue_secondary(q));
+
+	lrc = xa_erase(&group->xa, q->multi_queue.pos);
+	xe_assert(xe, lrc);
+	xe_lrc_put(lrc);
+
+	if (q->multi_queue.keep_active) {
+		xe_exec_queue_group_kill_put(group);
+		q->multi_queue.keep_active = false;
+	}
+}
+
+static int exec_queue_set_multi_group(struct xe_device *xe, struct xe_exec_queue *q,
+				      u64 value)
+{
+	if (XE_IOCTL_DBG(xe, !xe_exec_queue_supports_multi_queue(q)))
+		return -ENODEV;
+
+	if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe)))
+		return -EOPNOTSUPP;
+
+	if (XE_IOCTL_DBG(xe, !q->vm->xef))
+		return -EINVAL;
+
+	if (XE_IOCTL_DBG(xe, xe_exec_queue_is_parallel(q)))
+		return -EINVAL;
+
+	if (XE_IOCTL_DBG(xe, xe_exec_queue_is_multi_queue(q)))
+		return -EINVAL;
+
+	if (value & DRM_XE_MULTI_GROUP_CREATE) {
+		if (XE_IOCTL_DBG(xe, value & ~(DRM_XE_MULTI_GROUP_CREATE |
+					       DRM_XE_MULTI_GROUP_KEEP_ACTIVE)))
+			return -EINVAL;
+
+		/*
+		 * KEEP_ACTIVE is not supported in preempt fence mode as in that mode,
+		 * VM_DESTROY ioctl expects all exec queues of that VM are already killed.
+		 */
+		if (XE_IOCTL_DBG(xe, (value & DRM_XE_MULTI_GROUP_KEEP_ACTIVE) &&
+				 xe_vm_in_preempt_fence_mode(q->vm)))
+			return -EINVAL;
+
+		q->multi_queue.valid = true;
+		q->multi_queue.is_primary = true;
+		q->multi_queue.pos = 0;
+		if (value & DRM_XE_MULTI_GROUP_KEEP_ACTIVE)
+			q->multi_queue.keep_active = true;
+
+		return 0;
+	}
+
+	/* While adding secondary queues, the upper 32 bits must be 0 */
+	if (XE_IOCTL_DBG(xe, value & (~0ull << 32)))
+		return -EINVAL;
+
+	return xe_exec_queue_group_validate(xe, q, value);
+}
+
+static int exec_queue_set_multi_queue_priority(struct xe_device *xe, struct xe_exec_queue *q,
+					       u64 value)
+{
+	if (XE_IOCTL_DBG(xe, value > XE_MULTI_QUEUE_PRIORITY_HIGH))
+		return -EINVAL;
+
+	/* For queue creation time (!q->xef) setting, just store the priority value */
+	if (!q->xef) {
+		q->multi_queue.priority = value;
+		return 0;
+	}
+
+	if (!xe_exec_queue_is_multi_queue(q))
+		return -EINVAL;
+
+	return q->ops->set_multi_queue_priority(q, value);
+}
+
 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe,
 					     struct xe_exec_queue *q,
 					     u64 value);
@@ -575,11 +889,76 @@ static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = {
 	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority,
 	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice,
 	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type,
+	[DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE] = exec_queue_set_hang_replay_state,
+	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP] = exec_queue_set_multi_group,
+	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY] =
+							exec_queue_set_multi_queue_priority,
 };
 
+int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data,
+				     struct drm_file *file)
+{
+	struct xe_device *xe = to_xe_device(dev);
+	struct xe_file *xef = to_xe_file(file);
+	struct drm_xe_exec_queue_set_property *args = data;
+	struct xe_exec_queue *q;
+	int ret;
+	u32 idx;
+
+	if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
+		return -EINVAL;
+
+	if (XE_IOCTL_DBG(xe, args->property !=
+			 DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY))
+		return -EINVAL;
+
+	q = xe_exec_queue_lookup(xef, args->exec_queue_id);
+	if (XE_IOCTL_DBG(xe, !q))
+		return -ENOENT;
+
+	idx = array_index_nospec(args->property,
+				 ARRAY_SIZE(exec_queue_set_property_funcs));
+	ret = exec_queue_set_property_funcs[idx](xe, q, args->value);
+	if (XE_IOCTL_DBG(xe, ret))
+		goto err_post_lookup;
+
+	xe_exec_queue_put(q);
+	return 0;
+
+ err_post_lookup:
+	xe_exec_queue_put(q);
+	return ret;
+}
+
+static int exec_queue_user_ext_check(struct xe_exec_queue *q, u64 properties)
+{
+	u64 secondary_queue_valid_props = BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP) |
+				  BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY);
+
+	/*
+	 * Only MULTI_QUEUE_PRIORITY property is valid for secondary queues of a
+	 * multi-queue group.
+	 */
+	if (xe_exec_queue_is_multi_queue_secondary(q) &&
+	    properties & ~secondary_queue_valid_props)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int exec_queue_user_ext_check_final(struct xe_exec_queue *q, u64 properties)
+{
+	/* MULTI_QUEUE_PRIORITY only applies to multi-queue group queues */
+	if ((properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY)) &&
+	    !(properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP)))
+		return -EINVAL;
+
+	return 0;
+}
+
 static int exec_queue_user_ext_set_property(struct xe_device *xe,
 					    struct xe_exec_queue *q,
-					    u64 extension)
+					    u64 extension, u64 *properties)
 {
 	u64 __user *address = u64_to_user_ptr(extension);
 	struct drm_xe_ext_set_property ext;
@@ -595,27 +974,35 @@ static int exec_queue_user_ext_set_property(struct xe_device *xe,
 	    XE_IOCTL_DBG(xe, ext.pad) ||
 	    XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY &&
 			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE &&
-			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE))
+			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE &&
+			 ext.property != DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE &&
+			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP &&
+			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY))
 		return -EINVAL;
 
 	idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs));
 	if (!exec_queue_set_property_funcs[idx])
 		return -EINVAL;
 
+	*properties |= BIT_ULL(idx);
+	err = exec_queue_user_ext_check(q, *properties);
+	if (XE_IOCTL_DBG(xe, err))
+		return err;
+
 	return exec_queue_set_property_funcs[idx](xe, q, ext.value);
 }
 
 typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe,
 					       struct xe_exec_queue *q,
-					       u64 extension);
+					       u64 extension, u64 *properties);
 
 static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = {
 	[DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property,
 };
 
 #define MAX_USER_EXTENSIONS	16
-static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
-				      u64 extensions, int ext_number)
+static int __exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
+					u64 extensions, int ext_number, u64 *properties)
 {
 	u64 __user *address = u64_to_user_ptr(extensions);
 	struct drm_xe_user_extension ext;
@@ -636,13 +1023,36 @@ static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue
 
 	idx = array_index_nospec(ext.name,
 				 ARRAY_SIZE(exec_queue_user_extension_funcs));
-	err = exec_queue_user_extension_funcs[idx](xe, q, extensions);
+	err = exec_queue_user_extension_funcs[idx](xe, q, extensions, properties);
 	if (XE_IOCTL_DBG(xe, err))
 		return err;
 
 	if (ext.next_extension)
-		return exec_queue_user_extensions(xe, q, ext.next_extension,
-						  ++ext_number);
+		return __exec_queue_user_extensions(xe, q, ext.next_extension,
+						    ++ext_number, properties);
+
+	return 0;
+}
+
+static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
+				      u64 extensions)
+{
+	u64 properties = 0;
+	int err;
+
+	err = __exec_queue_user_extensions(xe, q, extensions, 0, &properties);
+	if (XE_IOCTL_DBG(xe, err))
+		return err;
+
+	err = exec_queue_user_ext_check_final(q, properties);
+	if (XE_IOCTL_DBG(xe, err))
+		return err;
+
+	if (xe_exec_queue_is_multi_queue_primary(q)) {
+		err = xe_exec_queue_group_init(xe, q);
+		if (XE_IOCTL_DBG(xe, err))
+			return err;
+	}
 
 	return 0;
 }
@@ -798,12 +1208,18 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
 		if (IS_ERR(q))
 			return PTR_ERR(q);
 
+		if (xe_exec_queue_is_multi_queue_secondary(q)) {
+			err = xe_exec_queue_group_add(xe, q);
+			if (XE_IOCTL_DBG(xe, err))
+				goto put_exec_queue;
+		}
+
 		if (xe_vm_in_preempt_fence_mode(vm)) {
 			q->lr.context = dma_fence_context_alloc(1);
 
 			err = xe_vm_add_compute_exec_queue(vm, q);
 			if (XE_IOCTL_DBG(xe, err))
-				goto put_exec_queue;
+				goto delete_queue_group;
 		}
 
 		if (q->vm && q->hwe->hw_engine_group) {
@@ -826,6 +1242,9 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
 
 kill_exec_queue:
 	xe_exec_queue_kill(q);
+delete_queue_group:
+	if (xe_exec_queue_is_multi_queue_secondary(q))
+		xe_exec_queue_group_delete(xe, q);
 put_exec_queue:
 	xe_exec_queue_put(q);
 	return err;
@@ -981,6 +1400,11 @@ void xe_exec_queue_kill(struct xe_exec_queue *q)
 
 	q->ops->kill(q);
 	xe_vm_remove_compute_exec_queue(q->vm, q);
+
+	if (!xe_exec_queue_is_multi_queue_primary(q) && q->multi_queue.keep_active) {
+		xe_exec_queue_group_kill_put(q->multi_queue.group);
+		q->multi_queue.keep_active = false;
+	}
 }
 
 int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data,
@@ -1007,7 +1431,10 @@ int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data,
 	if (q->vm && q->hwe->hw_engine_group)
 		xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
 
-	xe_exec_queue_kill(q);
+	if (xe_exec_queue_is_multi_queue_primary(q))
+		xe_exec_queue_group_kill_put(q->multi_queue.group);
+	else
+		xe_exec_queue_kill(q);
 
 	trace_xe_exec_queue_close(q);
 	xe_exec_queue_put(q);
diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h
index fda4d4f9bda8..b5ad975d7e97 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue.h
+++ b/drivers/gpu/drm/xe/xe_exec_queue.h
@@ -66,6 +66,55 @@ static inline bool xe_exec_queue_uses_pxp(struct xe_exec_queue *q)
 	return q->pxp.type;
 }
 
+/**
+ * xe_exec_queue_is_multi_queue() - Whether an exec_queue is part of a queue group.
+ * @q: The exec_queue
+ *
+ * Return: True if the exec_queue is part of a queue group, false otherwise.
+ */
+static inline bool xe_exec_queue_is_multi_queue(struct xe_exec_queue *q)
+{
+	return q->multi_queue.valid;
+}
+
+/**
+ * xe_exec_queue_is_multi_queue_primary() - Whether an exec_queue is primary queue
+ * of a multi queue group.
+ * @q: The exec_queue
+ *
+ * Return: True if @q is primary queue of a queue group, false otherwise.
+ */
+static inline bool xe_exec_queue_is_multi_queue_primary(struct xe_exec_queue *q)
+{
+	return q->multi_queue.is_primary;
+}
+
+/**
+ * xe_exec_queue_is_multi_queue_secondary() - Whether an exec_queue is secondary queue
+ * of a multi queue group.
+ * @q: The exec_queue
+ *
+ * Return: True if @q is secondary queue of a queue group, false otherwise.
+ */
+static inline bool xe_exec_queue_is_multi_queue_secondary(struct xe_exec_queue *q)
+{
+	return xe_exec_queue_is_multi_queue(q) && !xe_exec_queue_is_multi_queue_primary(q);
+}
+
+/**
+ * xe_exec_queue_multi_queue_primary() - Get multi queue group's primary queue
+ * @q: The exec_queue
+ *
+ * If @q belongs to a multi queue group, then the primary queue of the group will
+ * be returned. Otherwise, @q will be returned.
+ */
+static inline struct xe_exec_queue *xe_exec_queue_multi_queue_primary(struct xe_exec_queue *q)
+{
+	return xe_exec_queue_is_multi_queue(q) ? q->multi_queue.group->primary : q;
+}
+
+void xe_exec_queue_group_kill_put(struct xe_exec_queue_group *group);
+
 bool xe_exec_queue_is_lr(struct xe_exec_queue *q);
 
 bool xe_exec_queue_is_idle(struct xe_exec_queue *q);
@@ -78,6 +127,8 @@ int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *file);
 int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data,
 				     struct drm_file *file);
+int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data,
+				     struct drm_file *file);
 enum xe_exec_queue_priority xe_exec_queue_device_get_max_priority(struct xe_device *xe);
 
 void xe_exec_queue_last_fence_put(struct xe_exec_queue *e, struct xe_vm *vm);
@@ -111,4 +162,21 @@ int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch);
 
 struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q);
 
+/**
+ * xe_exec_queue_idle_skip_suspend() - Can exec queue skip suspend
+ * @q: The exec_queue
+ *
+ * If an exec queue is not parallel and is idle, the suspend steps can be
+ * skipped in the submission backend immediatley signaling the suspend fence.
+ * Parallel queues cannot skip this step due to limitations in the submission
+ * backend.
+ *
+ * Return: True if exec queue is idle and can skip suspend steps, False
+ * otherwise
+ */
+static inline bool xe_exec_queue_idle_skip_suspend(struct xe_exec_queue *q)
+{
+	return !xe_exec_queue_is_parallel(q) && xe_exec_queue_is_idle(q);
+}
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h
index 771ffe35cd0c..67ea5eebf70b 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue_types.h
+++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h
@@ -32,6 +32,44 @@ enum xe_exec_queue_priority {
 	XE_EXEC_QUEUE_PRIORITY_COUNT
 };
 
+/**
+ * enum xe_multi_queue_priority - Multi Queue priority values
+ *
+ * The priority values of the queues within the multi queue group.
+ */
+enum xe_multi_queue_priority {
+	/** @XE_MULTI_QUEUE_PRIORITY_LOW: Priority low */
+	XE_MULTI_QUEUE_PRIORITY_LOW = 0,
+	/** @XE_MULTI_QUEUE_PRIORITY_NORMAL: Priority normal */
+	XE_MULTI_QUEUE_PRIORITY_NORMAL,
+	/** @XE_MULTI_QUEUE_PRIORITY_HIGH: Priority high */
+	XE_MULTI_QUEUE_PRIORITY_HIGH,
+};
+
+/**
+ * struct xe_exec_queue_group - Execution multi queue group
+ *
+ * Contains multi queue group information.
+ */
+struct xe_exec_queue_group {
+	/** @primary: Primary queue of this group */
+	struct xe_exec_queue *primary;
+	/** @cgp_bo: BO for the Context Group Page */
+	struct xe_bo *cgp_bo;
+	/** @xa: xarray to store LRCs */
+	struct xarray xa;
+	/** @list: List of all secondary queues in the group */
+	struct list_head list;
+	/** @list_lock: Secondary queue list lock */
+	struct mutex list_lock;
+	/** @kill_refcount: ref count to kill primary queue */
+	struct kref kill_refcount;
+	/** @sync_pending: CGP_SYNC_DONE g2h response pending */
+	bool sync_pending;
+	/** @banned: Group banned */
+	bool banned;
+};
+
 /**
  * struct xe_exec_queue - Execution queue
  *
@@ -111,6 +149,24 @@ struct xe_exec_queue {
 		struct xe_guc_exec_queue *guc;
 	};
 
+	/** @multi_queue: Multi queue information */
+	struct {
+		/** @multi_queue.group: Queue group information */
+		struct xe_exec_queue_group *group;
+		/** @multi_queue.link: Link into group's secondary queues list */
+		struct list_head link;
+		/** @multi_queue.priority: Queue priority within the multi-queue group */
+		enum xe_multi_queue_priority priority;
+		/** @multi_queue.pos: Position of queue within the multi-queue group */
+		u8 pos;
+		/** @multi_queue.valid: Queue belongs to a multi queue group */
+		u8 valid:1;
+		/** @multi_queue.is_primary: Is primary queue (Q0) of the group */
+		u8 is_primary:1;
+		/** @multi_queue.keep_active: Keep the group active after primary is destroyed */
+		u8 keep_active:1;
+	} multi_queue;
+
 	/** @sched_props: scheduling properties */
 	struct {
 		/** @sched_props.timeslice_us: timeslice period in micro-seconds */
@@ -167,6 +223,9 @@ struct xe_exec_queue {
 	/** @ufence_timeline_value: User fence timeline value */
 	u64 ufence_timeline_value;
 
+	/** @replay_state: GPU hang replay state */
+	void *replay_state;
+
 	/** @ops: submission backend exec queue operations */
 	const struct xe_exec_queue_ops *ops;
 
@@ -213,6 +272,9 @@ struct xe_exec_queue_ops {
 	int (*set_timeslice)(struct xe_exec_queue *q, u32 timeslice_us);
 	/** @set_preempt_timeout: Set preemption timeout for exec queue */
 	int (*set_preempt_timeout)(struct xe_exec_queue *q, u32 preempt_timeout_us);
+	/** @set_multi_queue_priority: Set multi queue priority */
+	int (*set_multi_queue_priority)(struct xe_exec_queue *q,
+					enum xe_multi_queue_priority priority);
 	/**
 	 * @suspend: Suspend exec queue from executing, allowed to be called
 	 * multiple times in a row before resume with the caveat that
diff --git a/drivers/gpu/drm/xe/xe_execlist.c b/drivers/gpu/drm/xe/xe_execlist.c
index 769d05517f93..46c17a18a3f4 100644
--- a/drivers/gpu/drm/xe/xe_execlist.c
+++ b/drivers/gpu/drm/xe/xe_execlist.c
@@ -269,7 +269,7 @@ struct xe_execlist_port *xe_execlist_port_create(struct xe_device *xe,
 
 	port->hwe = hwe;
 
-	port->lrc = xe_lrc_create(hwe, NULL, SZ_16K, XE_IRQ_DEFAULT_MSIX, 0);
+	port->lrc = xe_lrc_create(hwe, NULL, NULL, SZ_16K, XE_IRQ_DEFAULT_MSIX, 0);
 	if (IS_ERR(port->lrc)) {
 		err = PTR_ERR(port->lrc);
 		goto err;
diff --git a/drivers/gpu/drm/xe/xe_force_wake.c b/drivers/gpu/drm/xe/xe_force_wake.c
index c59a9b330697..76e054f314ee 100644
--- a/drivers/gpu/drm/xe/xe_force_wake.c
+++ b/drivers/gpu/drm/xe/xe_force_wake.c
@@ -166,6 +166,13 @@ static int domain_sleep_wait(struct xe_gt *gt,
  * xe_force_wake_ref_has_domain() function. Caller must call
  * xe_force_wake_put() function to decrease incremented refcounts.
  *
+ * When possible, scope-based forcewake (through CLASS(xe_force_wake, ...) or
+ * xe_with_force_wake()) should be used instead of direct calls to this
+ * function.  Direct usage of get/put should only be used when the function
+ * has goto-based flows that can interfere with scope-based cleanup, or when
+ * the lifetime of the forcewake reference does not match a specific scope
+ * (e.g., forcewake obtained in one function and released in a different one).
+ *
  * Return: opaque reference to woken domains or zero if none of requested
  * domains were awake.
  */
diff --git a/drivers/gpu/drm/xe/xe_force_wake.h b/drivers/gpu/drm/xe/xe_force_wake.h
index 0e3e84bfa51c..1e2198f6a007 100644
--- a/drivers/gpu/drm/xe/xe_force_wake.h
+++ b/drivers/gpu/drm/xe/xe_force_wake.h
@@ -61,4 +61,44 @@ xe_force_wake_ref_has_domain(unsigned int fw_ref, enum xe_force_wake_domains dom
 	return fw_ref & domain;
 }
 
+struct xe_force_wake_ref {
+	struct xe_force_wake *fw;
+	unsigned int domains;
+};
+
+static struct xe_force_wake_ref
+xe_force_wake_constructor(struct xe_force_wake *fw, unsigned int domains)
+{
+	struct xe_force_wake_ref fw_ref = { .fw = fw };
+
+	fw_ref.domains = xe_force_wake_get(fw, domains);
+
+	return fw_ref;
+}
+
+DEFINE_CLASS(xe_force_wake, struct xe_force_wake_ref,
+	     xe_force_wake_put(_T.fw, _T.domains),
+	     xe_force_wake_constructor(fw, domains),
+	     struct xe_force_wake *fw, unsigned int domains);
+
+/*
+ * Scoped helper for the forcewake class, using the same trick as scoped_guard()
+ * to bind the lifetime to the next statement/block.
+ */
+#define __xe_with_force_wake(ref, fw, domains, done) \
+	for (CLASS(xe_force_wake, ref)(fw, domains), *(done) = NULL; \
+	     !(done); (done) = (void *)1)
+
+#define xe_with_force_wake(ref, fw, domains) \
+	__xe_with_force_wake(ref, fw, domains, __UNIQUE_ID(done))
+
+/*
+ * Used when xe_force_wake_constructor() has already been called by another
+ * function and the current function is responsible for releasing the forcewake
+ * reference in all possible cases and error paths.
+ */
+DEFINE_CLASS(xe_force_wake_release_only, struct xe_force_wake_ref,
+	     if (_T.fw) xe_force_wake_put(_T.fw, _T.domains), fw_ref,
+	     struct xe_force_wake_ref fw_ref);
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c
index ef481b334af4..48ab8b43fcd0 100644
--- a/drivers/gpu/drm/xe/xe_ggtt.c
+++ b/drivers/gpu/drm/xe/xe_ggtt.c
@@ -396,9 +396,8 @@ static void ggtt_node_remove_work_func(struct work_struct *work)
 						 delayed_removal_work);
 	struct xe_device *xe = tile_to_xe(node->ggtt->tile);
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	ggtt_node_remove(node);
-	xe_pm_runtime_put(xe);
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_gsc.c b/drivers/gpu/drm/xe/xe_gsc.c
index dd69cb834f8e..a3157b0fe791 100644
--- a/drivers/gpu/drm/xe/xe_gsc.c
+++ b/drivers/gpu/drm/xe/xe_gsc.c
@@ -352,7 +352,6 @@ static void gsc_work(struct work_struct *work)
 	struct xe_gsc *gsc = container_of(work, typeof(*gsc), work);
 	struct xe_gt *gt = gsc_to_gt(gsc);
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	u32 actions;
 	int ret;
 
@@ -361,13 +360,12 @@ static void gsc_work(struct work_struct *work)
 	gsc->work_actions = 0;
 	spin_unlock_irq(&gsc->lock);
 
-	xe_pm_runtime_get(xe);
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC);
+	guard(xe_pm_runtime)(xe);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC);
 
 	if (actions & GSC_ACTION_ER_COMPLETE) {
-		ret = gsc_er_complete(gt);
-		if (ret)
-			goto out;
+		if (gsc_er_complete(gt))
+			return;
 	}
 
 	if (actions & GSC_ACTION_FW_LOAD) {
@@ -380,10 +378,6 @@ static void gsc_work(struct work_struct *work)
 
 	if (actions & GSC_ACTION_SW_PROXY)
 		xe_gsc_proxy_request_handler(gsc);
-
-out:
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-	xe_pm_runtime_put(xe);
 }
 
 void xe_gsc_hwe_irq_handler(struct xe_hw_engine *hwe, u16 intr_vec)
@@ -615,7 +609,6 @@ void xe_gsc_print_info(struct xe_gsc *gsc, struct drm_printer *p)
 {
 	struct xe_gt *gt = gsc_to_gt(gsc);
 	struct xe_mmio *mmio = &gt->mmio;
-	unsigned int fw_ref;
 
 	xe_uc_fw_print(&gsc->fw, p);
 
@@ -624,8 +617,8 @@ void xe_gsc_print_info(struct xe_gsc *gsc, struct drm_printer *p)
 	if (!xe_uc_fw_is_enabled(&gsc->fw))
 		return;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC);
-	if (!fw_ref)
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC);
+	if (!fw_ref.domains)
 		return;
 
 	drm_printf(p, "\nHECI1 FWSTS: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
@@ -635,6 +628,4 @@ void xe_gsc_print_info(struct xe_gsc *gsc, struct drm_printer *p)
 			xe_mmio_read32(mmio, HECI_FWSTS4(MTL_GSC_HECI1_BASE)),
 			xe_mmio_read32(mmio, HECI_FWSTS5(MTL_GSC_HECI1_BASE)),
 			xe_mmio_read32(mmio, HECI_FWSTS6(MTL_GSC_HECI1_BASE)));
-
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
diff --git a/drivers/gpu/drm/xe/xe_gsc_debugfs.c b/drivers/gpu/drm/xe/xe_gsc_debugfs.c
index 461d7e99c2b3..b13928b50eb9 100644
--- a/drivers/gpu/drm/xe/xe_gsc_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_gsc_debugfs.c
@@ -37,9 +37,8 @@ static int gsc_info(struct seq_file *m, void *data)
 	struct xe_device *xe = gsc_to_xe(gsc);
 	struct drm_printer p = drm_seq_file_printer(m);
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	xe_gsc_print_info(gsc, &p);
-	xe_pm_runtime_put(xe);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/xe/xe_gsc_proxy.c b/drivers/gpu/drm/xe/xe_gsc_proxy.c
index 464282a89eef..e7573a0c5e5d 100644
--- a/drivers/gpu/drm/xe/xe_gsc_proxy.c
+++ b/drivers/gpu/drm/xe/xe_gsc_proxy.c
@@ -440,22 +440,19 @@ static void xe_gsc_proxy_remove(void *arg)
 	struct xe_gsc *gsc = arg;
 	struct xe_gt *gt = gsc_to_gt(gsc);
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref = 0;
 
 	if (!gsc->proxy.component_added)
 		return;
 
 	/* disable HECI2 IRQs */
-	xe_pm_runtime_get(xe);
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC);
-	if (!fw_ref)
-		xe_gt_err(gt, "failed to get forcewake to disable GSC interrupts\n");
+	scoped_guard(xe_pm_runtime, xe) {
+		CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC);
+		if (!fw_ref.domains)
+			xe_gt_err(gt, "failed to get forcewake to disable GSC interrupts\n");
 
-	/* try do disable irq even if forcewake failed */
-	gsc_proxy_irq_toggle(gsc, false);
-
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-	xe_pm_runtime_put(xe);
+		/* try do disable irq even if forcewake failed */
+		gsc_proxy_irq_toggle(gsc, false);
+	}
 
 	xe_gsc_wait_for_worker_completion(gsc);
 
diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c
index dbb5e7a9bc6a..313ce83ab0e5 100644
--- a/drivers/gpu/drm/xe/xe_gt.c
+++ b/drivers/gpu/drm/xe/xe_gt.c
@@ -103,14 +103,13 @@ void xe_gt_sanitize(struct xe_gt *gt)
 
 static void xe_gt_enable_host_l2_vram(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
 	u32 reg;
 
 	if (!XE_GT_WA(gt, 16023588340))
 		return;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!fw_ref.domains)
 		return;
 
 	if (xe_gt_is_main_type(gt)) {
@@ -120,12 +119,10 @@ static void xe_gt_enable_host_l2_vram(struct xe_gt *gt)
 	}
 
 	xe_gt_mcr_multicast_write(gt, XEHPC_L3CLOS_MASK(3), 0xF);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 
 static void xe_gt_disable_host_l2_vram(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
 	u32 reg;
 
 	if (!XE_GT_WA(gt, 16023588340))
@@ -134,15 +131,13 @@ static void xe_gt_disable_host_l2_vram(struct xe_gt *gt)
 	if (xe_gt_is_media_type(gt))
 		return;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!fw_ref.domains)
 		return;
 
 	reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL);
 	reg &= ~CG_DIS_CNTLBUS;
 	xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg);
-
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 
 static void gt_reset_worker(struct work_struct *w);
@@ -389,7 +384,6 @@ int xe_gt_record_default_lrcs(struct xe_gt *gt)
 
 int xe_gt_init_early(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
 	int err;
 
 	if (IS_SRIOV_PF(gt_to_xe(gt))) {
@@ -436,13 +430,12 @@ int xe_gt_init_early(struct xe_gt *gt)
 	if (err)
 		return err;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 
 	xe_gt_mcr_init_early(gt);
 	xe_pat_init(gt);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 
 	return 0;
 }
@@ -460,16 +453,15 @@ static void dump_pat_on_error(struct xe_gt *gt)
 
 static int gt_init_with_gt_forcewake(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
 	int err;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 
 	err = xe_uc_init(&gt->uc);
 	if (err)
-		goto err_force_wake;
+		return err;
 
 	xe_gt_topology_init(gt);
 	xe_gt_mcr_init(gt);
@@ -478,7 +470,7 @@ static int gt_init_with_gt_forcewake(struct xe_gt *gt)
 	if (xe_gt_is_main_type(gt)) {
 		err = xe_ggtt_init(gt_to_tile(gt)->mem.ggtt);
 		if (err)
-			goto err_force_wake;
+			return err;
 		if (IS_SRIOV_PF(gt_to_xe(gt)))
 			xe_lmtt_init(&gt_to_tile(gt)->sriov.pf.lmtt);
 	}
@@ -492,17 +484,17 @@ static int gt_init_with_gt_forcewake(struct xe_gt *gt)
 	err = xe_hw_engines_init_early(gt);
 	if (err) {
 		dump_pat_on_error(gt);
-		goto err_force_wake;
+		return err;
 	}
 
 	err = xe_hw_engine_class_sysfs_init(gt);
 	if (err)
-		goto err_force_wake;
+		return err;
 
 	/* Initialize CCS mode sysfs after early initialization of HW engines */
 	err = xe_gt_ccs_mode_sysfs_init(gt);
 	if (err)
-		goto err_force_wake;
+		return err;
 
 	/*
 	 * Stash hardware-reported version.  Since this register does not exist
@@ -510,25 +502,16 @@ static int gt_init_with_gt_forcewake(struct xe_gt *gt)
 	 */
 	gt->info.gmdid = xe_mmio_read32(&gt->mmio, GMD_ID);
 
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
-
-err_force_wake:
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-
-	return err;
 }
 
 static int gt_init_with_all_forcewake(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
 	int err;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
-		err = -ETIMEDOUT;
-		goto err_force_wake;
-	}
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
+		return -ETIMEDOUT;
 
 	xe_gt_mcr_set_implicit_defaults(gt);
 	xe_wa_process_gt(gt);
@@ -537,20 +520,20 @@ static int gt_init_with_all_forcewake(struct xe_gt *gt)
 
 	err = xe_gt_clock_init(gt);
 	if (err)
-		goto err_force_wake;
+		return err;
 
 	xe_mocs_init(gt);
 	err = xe_execlist_init(gt);
 	if (err)
-		goto err_force_wake;
+		return err;
 
 	err = xe_hw_engines_init(gt);
 	if (err)
-		goto err_force_wake;
+		return err;
 
 	err = xe_uc_init_post_hwconfig(&gt->uc);
 	if (err)
-		goto err_force_wake;
+		return err;
 
 	if (xe_gt_is_main_type(gt)) {
 		/*
@@ -561,10 +544,8 @@ static int gt_init_with_all_forcewake(struct xe_gt *gt)
 
 			gt->usm.bb_pool = xe_sa_bo_manager_init(gt_to_tile(gt),
 								IS_DGFX(xe) ? SZ_1M : SZ_512K, 16);
-			if (IS_ERR(gt->usm.bb_pool)) {
-				err = PTR_ERR(gt->usm.bb_pool);
-				goto err_force_wake;
-			}
+			if (IS_ERR(gt->usm.bb_pool))
+				return PTR_ERR(gt->usm.bb_pool);
 		}
 	}
 
@@ -573,12 +554,12 @@ static int gt_init_with_all_forcewake(struct xe_gt *gt)
 
 		err = xe_migrate_init(tile->migrate);
 		if (err)
-			goto err_force_wake;
+			return err;
 	}
 
 	err = xe_uc_load_hw(&gt->uc);
 	if (err)
-		goto err_force_wake;
+		return err;
 
 	/* Configure default CCS mode of 1 engine with all resources */
 	if (xe_gt_ccs_mode_enabled(gt)) {
@@ -592,14 +573,7 @@ static int gt_init_with_all_forcewake(struct xe_gt *gt)
 	if (IS_SRIOV_PF(gt_to_xe(gt)))
 		xe_gt_sriov_pf_init_hw(gt);
 
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-
 	return 0;
-
-err_force_wake:
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-
-	return err;
 }
 
 static void xe_gt_fini(void *arg)
@@ -797,9 +771,6 @@ static int do_gt_restart(struct xe_gt *gt)
 		xe_gt_sriov_pf_init_hw(gt);
 
 	xe_mocs_init(gt);
-	err = xe_uc_start(&gt->uc);
-	if (err)
-		return err;
 
 	for_each_hw_engine(hwe, gt, id)
 		xe_reg_sr_apply_mmio(&hwe->reg_sr, gt);
@@ -807,6 +778,10 @@ static int do_gt_restart(struct xe_gt *gt)
 	/* Get CCS mode in sync between sw/hw */
 	xe_gt_apply_ccs_mode(gt);
 
+	err = xe_uc_start(&gt->uc);
+	if (err)
+		return err;
+
 	/* Restore GT freq to expected values */
 	xe_gt_sanitize_freq(gt);
 
@@ -901,56 +876,42 @@ void xe_gt_reset_async(struct xe_gt *gt)
 
 void xe_gt_suspend_prepare(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
-
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 	xe_uc_suspend_prepare(&gt->uc);
-
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 
 int xe_gt_suspend(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
 	int err;
 
 	xe_gt_dbg(gt, "suspending\n");
 	xe_gt_sanitize(gt);
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
-		goto err_msg;
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) {
+		xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(-ETIMEDOUT));
+		return -ETIMEDOUT;
+	}
 
 	err = xe_uc_suspend(&gt->uc);
-	if (err)
-		goto err_force_wake;
+	if (err) {
+		xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err));
+		return err;
+	}
 
 	xe_gt_idle_disable_pg(gt);
 
 	xe_gt_disable_host_l2_vram(gt);
 
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	xe_gt_dbg(gt, "suspended\n");
 
 	return 0;
-
-err_msg:
-	err = -ETIMEDOUT;
-err_force_wake:
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-	xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err));
-
-	return err;
 }
 
 void xe_gt_shutdown(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
-
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 	do_gt_reset(gt);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 
 /**
@@ -975,32 +936,72 @@ int xe_gt_sanitize_freq(struct xe_gt *gt)
 
 int xe_gt_resume(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
 	int err;
 
 	xe_gt_dbg(gt, "resuming\n");
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
-		goto err_msg;
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) {
+		xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(-ETIMEDOUT));
+		return -ETIMEDOUT;
+	}
 
 	err = do_gt_restart(gt);
 	if (err)
-		goto err_force_wake;
+		return err;
 
 	xe_gt_idle_enable_pg(gt);
 
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	xe_gt_dbg(gt, "resumed\n");
 
 	return 0;
+}
 
-err_msg:
-	err = -ETIMEDOUT;
-err_force_wake:
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-	xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(err));
+/**
+ * xe_gt_runtime_suspend() - GT runtime suspend
+ * @gt: the GT object
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+int xe_gt_runtime_suspend(struct xe_gt *gt)
+{
+	xe_gt_dbg(gt, "runtime suspending\n");
 
-	return err;
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) {
+		xe_gt_err(gt, "runtime suspend failed (%pe)\n", ERR_PTR(-ETIMEDOUT));
+		return -ETIMEDOUT;
+	}
+
+	xe_uc_runtime_suspend(&gt->uc);
+	xe_gt_disable_host_l2_vram(gt);
+
+	xe_gt_dbg(gt, "runtime suspended\n");
+
+	return 0;
+}
+
+/**
+ * xe_gt_runtime_resume() - GT runtime resume
+ * @gt: the GT object
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+int xe_gt_runtime_resume(struct xe_gt *gt)
+{
+	xe_gt_dbg(gt, "runtime resuming\n");
+
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) {
+		xe_gt_err(gt, "runtime resume failed (%pe)\n", ERR_PTR(-ETIMEDOUT));
+		return -ETIMEDOUT;
+	}
+
+	xe_gt_enable_host_l2_vram(gt);
+	xe_uc_runtime_resume(&gt->uc);
+
+	xe_gt_dbg(gt, "runtime resumed\n");
+
+	return 0;
 }
 
 struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt,
diff --git a/drivers/gpu/drm/xe/xe_gt.h b/drivers/gpu/drm/xe/xe_gt.h
index 9d710049da45..94969ddd9d88 100644
--- a/drivers/gpu/drm/xe/xe_gt.h
+++ b/drivers/gpu/drm/xe/xe_gt.h
@@ -58,6 +58,8 @@ int xe_gt_suspend(struct xe_gt *gt);
 void xe_gt_shutdown(struct xe_gt *gt);
 int xe_gt_resume(struct xe_gt *gt);
 void xe_gt_reset_async(struct xe_gt *gt);
+int xe_gt_runtime_resume(struct xe_gt *gt);
+int xe_gt_runtime_suspend(struct xe_gt *gt);
 void xe_gt_sanitize(struct xe_gt *gt);
 int xe_gt_sanitize_freq(struct xe_gt *gt);
 
diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.c b/drivers/gpu/drm/xe/xe_gt_debugfs.c
index e4fd632f43cf..e4f38b5150fc 100644
--- a/drivers/gpu/drm/xe/xe_gt_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_gt_debugfs.c
@@ -105,35 +105,24 @@ int xe_gt_debugfs_show_with_rpm(struct seq_file *m, void *data)
 	struct drm_info_node *node = m->private;
 	struct xe_gt *gt = node_to_gt(node);
 	struct xe_device *xe = gt_to_xe(gt);
-	int ret;
-
-	xe_pm_runtime_get(xe);
-	ret = xe_gt_debugfs_simple_show(m, data);
-	xe_pm_runtime_put(xe);
 
-	return ret;
+	guard(xe_pm_runtime)(xe);
+	return xe_gt_debugfs_simple_show(m, data);
 }
 
 static int hw_engines(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_hw_engine *hwe;
 	enum xe_hw_engine_id id;
-	unsigned int fw_ref;
-	int ret = 0;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
-		ret = -ETIMEDOUT;
-		goto fw_put;
-	}
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
+		return -ETIMEDOUT;
 
 	for_each_hw_engine(hwe, gt, id)
 		xe_hw_engine_print(hwe, p);
 
-fw_put:
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-
-	return ret;
+	return 0;
 }
 
 static int steering(struct xe_gt *gt, struct drm_printer *p)
@@ -220,6 +209,7 @@ static const struct drm_info_list vf_safe_debugfs_list[] = {
 	{ "default_lrc_vcs", .show = xe_gt_debugfs_show_with_rpm, .data = vcs_default_lrc },
 	{ "default_lrc_vecs", .show = xe_gt_debugfs_show_with_rpm, .data = vecs_default_lrc },
 	{ "hwconfig", .show = xe_gt_debugfs_show_with_rpm, .data = hwconfig },
+	{ "pat_sw_config", .show = xe_gt_debugfs_simple_show, .data = xe_pat_dump_sw_config },
 };
 
 /* everything else should be added here */
@@ -269,9 +259,8 @@ static void force_reset(struct xe_gt *gt)
 {
 	struct xe_device *xe = gt_to_xe(gt);
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	xe_gt_reset_async(gt);
-	xe_pm_runtime_put(xe);
 }
 
 static ssize_t force_reset_write(struct file *file,
@@ -297,9 +286,8 @@ static void force_reset_sync(struct xe_gt *gt)
 {
 	struct xe_device *xe = gt_to_xe(gt);
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	xe_gt_reset(gt);
-	xe_pm_runtime_put(xe);
 }
 
 static ssize_t force_reset_sync_write(struct file *file,
diff --git a/drivers/gpu/drm/xe/xe_gt_freq.c b/drivers/gpu/drm/xe/xe_gt_freq.c
index 849ea6c86e8e..a40dd074106f 100644
--- a/drivers/gpu/drm/xe/xe_gt_freq.c
+++ b/drivers/gpu/drm/xe/xe_gt_freq.c
@@ -70,9 +70,8 @@ static ssize_t act_freq_show(struct kobject *kobj,
 	struct xe_guc_pc *pc = dev_to_pc(dev);
 	u32 freq;
 
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	freq = xe_guc_pc_get_act_freq(pc);
-	xe_pm_runtime_put(dev_to_xe(dev));
 
 	return sysfs_emit(buf, "%d\n", freq);
 }
@@ -86,9 +85,8 @@ static ssize_t cur_freq_show(struct kobject *kobj,
 	u32 freq;
 	ssize_t ret;
 
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	ret = xe_guc_pc_get_cur_freq(pc, &freq);
-	xe_pm_runtime_put(dev_to_xe(dev));
 	if (ret)
 		return ret;
 
@@ -113,9 +111,8 @@ static ssize_t rpe_freq_show(struct kobject *kobj,
 	struct xe_guc_pc *pc = dev_to_pc(dev);
 	u32 freq;
 
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	freq = xe_guc_pc_get_rpe_freq(pc);
-	xe_pm_runtime_put(dev_to_xe(dev));
 
 	return sysfs_emit(buf, "%d\n", freq);
 }
@@ -128,9 +125,8 @@ static ssize_t rpa_freq_show(struct kobject *kobj,
 	struct xe_guc_pc *pc = dev_to_pc(dev);
 	u32 freq;
 
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	freq = xe_guc_pc_get_rpa_freq(pc);
-	xe_pm_runtime_put(dev_to_xe(dev));
 
 	return sysfs_emit(buf, "%d\n", freq);
 }
@@ -154,9 +150,8 @@ static ssize_t min_freq_show(struct kobject *kobj,
 	u32 freq;
 	ssize_t ret;
 
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	ret = xe_guc_pc_get_min_freq(pc, &freq);
-	xe_pm_runtime_put(dev_to_xe(dev));
 	if (ret)
 		return ret;
 
@@ -175,9 +170,8 @@ static ssize_t min_freq_store(struct kobject *kobj,
 	if (ret)
 		return ret;
 
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	ret = xe_guc_pc_set_min_freq(pc, freq);
-	xe_pm_runtime_put(dev_to_xe(dev));
 	if (ret)
 		return ret;
 
@@ -193,9 +187,8 @@ static ssize_t max_freq_show(struct kobject *kobj,
 	u32 freq;
 	ssize_t ret;
 
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	ret = xe_guc_pc_get_max_freq(pc, &freq);
-	xe_pm_runtime_put(dev_to_xe(dev));
 	if (ret)
 		return ret;
 
@@ -214,9 +207,8 @@ static ssize_t max_freq_store(struct kobject *kobj,
 	if (ret)
 		return ret;
 
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	ret = xe_guc_pc_set_max_freq(pc, freq);
-	xe_pm_runtime_put(dev_to_xe(dev));
 	if (ret)
 		return ret;
 
@@ -243,9 +235,8 @@ static ssize_t power_profile_store(struct kobject *kobj,
 	struct xe_guc_pc *pc = dev_to_pc(dev);
 	int err;
 
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	err = xe_guc_pc_set_power_profile(pc, buff);
-	xe_pm_runtime_put(dev_to_xe(dev));
 
 	return err ?: count;
 }
@@ -293,8 +284,10 @@ int xe_gt_freq_init(struct xe_gt *gt)
 		return -ENOMEM;
 
 	err = sysfs_create_files(gt->freq, freq_attrs);
-	if (err)
+	if (err) {
+		kobject_put(gt->freq);
 		return err;
+	}
 
 	err = devm_add_action_or_reset(xe->drm.dev, freq_fini, gt->freq);
 	if (err)
diff --git a/drivers/gpu/drm/xe/xe_gt_idle.c b/drivers/gpu/drm/xe/xe_gt_idle.c
index bdc9d9877ec4..c1c9bec3c487 100644
--- a/drivers/gpu/drm/xe/xe_gt_idle.c
+++ b/drivers/gpu/drm/xe/xe_gt_idle.c
@@ -5,6 +5,7 @@
 
 #include <drm/drm_managed.h>
 
+#include <generated/xe_wa_oob.h>
 #include "xe_force_wake.h"
 #include "xe_device.h"
 #include "xe_gt.h"
@@ -16,6 +17,7 @@
 #include "xe_mmio.h"
 #include "xe_pm.h"
 #include "xe_sriov.h"
+#include "xe_wa.h"
 
 /**
  * DOC: Xe GT Idle
@@ -103,7 +105,6 @@ void xe_gt_idle_enable_pg(struct xe_gt *gt)
 	struct xe_gt_idle *gtidle = &gt->gtidle;
 	struct xe_mmio *mmio = &gt->mmio;
 	u32 vcs_mask, vecs_mask;
-	unsigned int fw_ref;
 	int i, j;
 
 	if (IS_SRIOV_VF(xe))
@@ -135,7 +136,7 @@ void xe_gt_idle_enable_pg(struct xe_gt *gt)
 		}
 	}
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
 	if (xe->info.skip_guc_pc) {
 		/*
 		 * GuC sets the hysteresis value when GuC PC is enabled
@@ -145,14 +146,18 @@ void xe_gt_idle_enable_pg(struct xe_gt *gt)
 		xe_mmio_write32(mmio, RENDER_POWERGATE_IDLE_HYSTERESIS, 25);
 	}
 
+	if (XE_GT_WA(gt, 14020316580))
+		gtidle->powergate_enable &= ~(VDN_HCP_POWERGATE_ENABLE(0) |
+					      VDN_MFXVDENC_POWERGATE_ENABLE(0) |
+					      VDN_HCP_POWERGATE_ENABLE(2) |
+					      VDN_MFXVDENC_POWERGATE_ENABLE(2));
+
 	xe_mmio_write32(mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 
 void xe_gt_idle_disable_pg(struct xe_gt *gt)
 {
 	struct xe_gt_idle *gtidle = &gt->gtidle;
-	unsigned int fw_ref;
 
 	if (IS_SRIOV_VF(gt_to_xe(gt)))
 		return;
@@ -160,9 +165,8 @@ void xe_gt_idle_disable_pg(struct xe_gt *gt)
 	xe_device_assert_mem_access(gt_to_xe(gt));
 	gtidle->powergate_enable = 0;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
 	xe_mmio_write32(&gt->mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 
 /**
@@ -181,7 +185,6 @@ int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p)
 	enum xe_gt_idle_state state;
 	u32 pg_enabled, pg_status = 0;
 	u32 vcs_mask, vecs_mask;
-	unsigned int fw_ref;
 	int n;
 	/*
 	 * Media Slices
@@ -218,14 +221,12 @@ int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p)
 
 	/* Do not wake the GT to read powergating status */
 	if (state != GT_IDLE_C6) {
-		fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-		if (!fw_ref)
+		CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+		if (!fw_ref.domains)
 			return -ETIMEDOUT;
 
 		pg_enabled = xe_mmio_read32(&gt->mmio, POWERGATE_ENABLE);
 		pg_status = xe_mmio_read32(&gt->mmio, POWERGATE_DOMAIN_STATUS);
-
-		xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	}
 
 	if (gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK) {
@@ -263,13 +264,9 @@ static ssize_t name_show(struct kobject *kobj,
 	struct device *dev = kobj_to_dev(kobj);
 	struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
 	struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
-	ssize_t ret;
-
-	xe_pm_runtime_get(pc_to_xe(pc));
-	ret = sysfs_emit(buff, "%s\n", gtidle->name);
-	xe_pm_runtime_put(pc_to_xe(pc));
 
-	return ret;
+	guard(xe_pm_runtime)(pc_to_xe(pc));
+	return sysfs_emit(buff, "%s\n", gtidle->name);
 }
 static struct kobj_attribute name_attr = __ATTR_RO(name);
 
@@ -281,9 +278,8 @@ static ssize_t idle_status_show(struct kobject *kobj,
 	struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
 	enum xe_gt_idle_state state;
 
-	xe_pm_runtime_get(pc_to_xe(pc));
-	state = gtidle->idle_status(pc);
-	xe_pm_runtime_put(pc_to_xe(pc));
+	scoped_guard(xe_pm_runtime, pc_to_xe(pc))
+		state = gtidle->idle_status(pc);
 
 	return sysfs_emit(buff, "%s\n", gt_idle_state_to_string(state));
 }
@@ -311,9 +307,8 @@ static ssize_t idle_residency_ms_show(struct kobject *kobj,
 	struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
 	u64 residency;
 
-	xe_pm_runtime_get(pc_to_xe(pc));
-	residency = xe_gt_idle_residency_msec(gtidle);
-	xe_pm_runtime_put(pc_to_xe(pc));
+	scoped_guard(xe_pm_runtime, pc_to_xe(pc))
+		residency = xe_gt_idle_residency_msec(gtidle);
 
 	return sysfs_emit(buff, "%llu\n", residency);
 }
@@ -396,21 +391,17 @@ void xe_gt_idle_enable_c6(struct xe_gt *gt)
 
 int xe_gt_idle_disable_c6(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
-
 	xe_device_assert_mem_access(gt_to_xe(gt));
 
 	if (IS_SRIOV_VF(gt_to_xe(gt)))
 		return 0;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 
 	xe_mmio_write32(&gt->mmio, RC_CONTROL, 0);
 	xe_mmio_write32(&gt->mmio, RC_STATE, 0);
 
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-
 	return 0;
 }
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c
index 59c5c6b4d994..6e8507c24986 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c
@@ -269,7 +269,8 @@ static u32 encode_config_ggtt(u32 *cfg, const struct xe_gt_sriov_config *config,
 }
 
 /* Return: number of configuration dwords written */
-static u32 encode_config(u32 *cfg, const struct xe_gt_sriov_config *config, bool details)
+static u32 encode_config(struct xe_gt *gt, u32 *cfg, const struct xe_gt_sriov_config *config,
+			 bool details)
 {
 	u32 n = 0;
 
@@ -303,9 +304,11 @@ static u32 encode_config(u32 *cfg, const struct xe_gt_sriov_config *config, bool
 	cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_PREEMPT_TIMEOUT);
 	cfg[n++] = config->preempt_timeout;
 
-#define encode_threshold_config(TAG, ...) ({					\
-	cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_THRESHOLD_##TAG);			\
-	cfg[n++] = config->thresholds[MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG)];	\
+#define encode_threshold_config(TAG, NAME, VER...) ({					\
+	if (IF_ARGS(GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, VER), true, VER)) {		\
+		cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_THRESHOLD_##TAG);			\
+		cfg[n++] = config->thresholds[MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG)];	\
+	}										\
 });
 
 	MAKE_XE_GUC_KLV_THRESHOLDS_SET(encode_threshold_config);
@@ -328,7 +331,7 @@ static int pf_push_full_vf_config(struct xe_gt *gt, unsigned int vfid)
 		return -ENOBUFS;
 
 	cfg = xe_guc_buf_cpu_ptr(buf);
-	num_dwords = encode_config(cfg, config, true);
+	num_dwords = encode_config(gt, cfg, config, true);
 	xe_gt_assert(gt, num_dwords <= max_cfg_dwords);
 
 	if (xe_gt_is_media_type(gt)) {
@@ -2518,7 +2521,7 @@ ssize_t xe_gt_sriov_pf_config_save(struct xe_gt *gt, unsigned int vfid, void *bu
 			ret = -ENOBUFS;
 		} else {
 			config = pf_pick_vf_config(gt, vfid);
-			ret = encode_config(buf, config, false) * sizeof(u32);
+			ret = encode_config(gt, buf, config, false) * sizeof(u32);
 		}
 	}
 	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
@@ -2551,11 +2554,13 @@ static int pf_restore_vf_config_klv(struct xe_gt *gt, unsigned int vfid,
 		return pf_provision_preempt_timeout(gt, vfid, value[0]);
 
 	/* auto-generate case statements */
-#define define_threshold_key_to_provision_case(TAG, ...)				\
+#define define_threshold_key_to_provision_case(TAG, NAME, VER...)			\
 	case MAKE_GUC_KLV_VF_CFG_THRESHOLD_KEY(TAG):					\
 		BUILD_BUG_ON(MAKE_GUC_KLV_VF_CFG_THRESHOLD_LEN(TAG) != 1u);		\
 		if (len != MAKE_GUC_KLV_VF_CFG_THRESHOLD_LEN(TAG))			\
 			return -EBADMSG;						\
+		if (IF_ARGS(!GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, VER), false, VER))	\
+			return -EKEYREJECTED;						\
 		return pf_provision_threshold(gt, vfid,					\
 					      MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG),	\
 					      value[0]);
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c
index 5278ea4fd655..ece9eed5d7c5 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c
@@ -21,6 +21,7 @@
 #include "xe_gt_sriov_pf_monitor.h"
 #include "xe_gt_sriov_pf_policy.h"
 #include "xe_gt_sriov_pf_service.h"
+#include "xe_guc.h"
 #include "xe_pm.h"
 #include "xe_sriov_pf.h"
 #include "xe_sriov_pf_provision.h"
@@ -123,11 +124,10 @@ static int POLICY##_set(void *data, u64 val)					\
 	if (val > (TYPE)~0ull)							\
 		return -EOVERFLOW;						\
 										\
-	xe_pm_runtime_get(xe);							\
+	guard(xe_pm_runtime)(xe);							\
 	err = xe_gt_sriov_pf_policy_set_##POLICY(gt, val);			\
 	if (!err)								\
 		xe_sriov_pf_provision_set_custom_mode(xe);			\
-	xe_pm_runtime_put(xe);							\
 										\
 	return err;								\
 }										\
@@ -189,12 +189,11 @@ static int CONFIG##_set(void *data, u64 val)					\
 	if (val > (TYPE)~0ull)							\
 		return -EOVERFLOW;						\
 										\
-	xe_pm_runtime_get(xe);							\
+	guard(xe_pm_runtime)(xe);							\
 	err = xe_sriov_pf_wait_ready(xe) ?:					\
 	      xe_gt_sriov_pf_config_set_##CONFIG(gt, vfid, val);		\
 	if (!err)								\
 		xe_sriov_pf_provision_set_custom_mode(xe);			\
-	xe_pm_runtime_put(xe);							\
 										\
 	return err;								\
 }										\
@@ -249,11 +248,10 @@ static int set_threshold(void *data, u64 val, enum xe_guc_klv_threshold_index in
 	if (val > (u32)~0ull)
 		return -EOVERFLOW;
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	err = xe_gt_sriov_pf_config_set_threshold(gt, vfid, index, val);
 	if (!err)
 		xe_sriov_pf_provision_set_custom_mode(xe);
-	xe_pm_runtime_put(xe);
 
 	return err;
 }
@@ -304,9 +302,11 @@ static void pf_add_config_attrs(struct xe_gt *gt, struct dentry *parent, unsigne
 				   &sched_priority_fops);
 
 	/* register all threshold attributes */
-#define register_threshold_attribute(TAG, NAME, ...) \
-	debugfs_create_file_unsafe("threshold_" #NAME, 0644, parent, parent, \
-				   &NAME##_fops);
+#define register_threshold_attribute(TAG, NAME, VER...) ({				\
+	if (IF_ARGS(GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, VER), true, VER))		\
+		debugfs_create_file_unsafe("threshold_" #NAME, 0644, parent, parent,	\
+					   &NAME##_fops);				\
+});
 	MAKE_XE_GUC_KLV_THRESHOLDS_SET(register_threshold_attribute)
 #undef register_threshold_attribute
 }
@@ -358,9 +358,8 @@ static ssize_t control_write(struct file *file, const char __user *buf, size_t c
 		xe_gt_assert(gt, sizeof(cmd) > strlen(control_cmds[n].cmd));
 
 		if (sysfs_streq(cmd, control_cmds[n].cmd)) {
-			xe_pm_runtime_get(xe);
+			guard(xe_pm_runtime)(xe);
 			ret = control_cmds[n].fn ? (*control_cmds[n].fn)(gt, vfid) : 0;
-			xe_pm_runtime_put(xe);
 			break;
 		}
 	}
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c
index 3174a8dee779..7410e7b93256 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c
@@ -1026,7 +1026,7 @@ static void action_ring_cleanup(void *arg)
 
 static void pf_gt_migration_check_support(struct xe_gt *gt)
 {
-	if (GUC_FIRMWARE_VER(&gt->uc.guc) < MAKE_GUC_VER(70, 54, 0))
+	if (!GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 54))
 		xe_sriov_pf_migration_disable(gt_to_xe(gt), "requires GuC version >= 70.54.0");
 }
 
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
index 4c73a077d314..b8b391cfc8eb 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
@@ -5,6 +5,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/bsearch.h>
+#include <linux/delay.h>
 
 #include <drm/drm_managed.h>
 #include <drm/drm_print.h>
@@ -41,6 +42,37 @@
 
 #define make_u64_from_u32(hi, lo) ((u64)((u64)(u32)(hi) << 32 | (u32)(lo)))
 
+#ifdef CONFIG_DRM_XE_DEBUG
+enum VF_MIGRATION_WAIT_POINTS {
+	VF_MIGRATION_WAIT_RESFIX_START	= BIT(0),
+	VF_MIGRATION_WAIT_FIXUPS	= BIT(1),
+	VF_MIGRATION_WAIT_RESTART_JOBS	= BIT(2),
+	VF_MIGRATION_WAIT_RESFIX_DONE	= BIT(3),
+};
+
+#define VF_MIGRATION_WAIT_DELAY_IN_MS	1000
+static void vf_post_migration_inject_wait(struct xe_gt *gt,
+					  enum VF_MIGRATION_WAIT_POINTS wait)
+{
+	while (gt->sriov.vf.migration.debug.resfix_stoppers & wait) {
+		xe_gt_dbg(gt,
+			  "*TESTING* injecting %u ms delay due to resfix_stoppers=%#x, to continue clear %#x\n",
+			  VF_MIGRATION_WAIT_DELAY_IN_MS,
+			  gt->sriov.vf.migration.debug.resfix_stoppers, wait);
+
+		msleep(VF_MIGRATION_WAIT_DELAY_IN_MS);
+	}
+}
+
+#define VF_MIGRATION_INJECT_WAIT(gt, _POS) ({					\
+	struct xe_gt *__gt = (gt);						\
+	vf_post_migration_inject_wait(__gt, VF_MIGRATION_WAIT_##_POS);		\
+	})
+
+#else
+#define VF_MIGRATION_INJECT_WAIT(_gt, ...)	typecheck(struct xe_gt *, (_gt))
+#endif
+
 static int guc_action_vf_reset(struct xe_guc *guc)
 {
 	u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = {
@@ -299,12 +331,13 @@ void xe_gt_sriov_vf_guc_versions(struct xe_gt *gt,
 		*found = gt->sriov.vf.guc_version;
 }
 
-static int guc_action_vf_notify_resfix_done(struct xe_guc *guc)
+static int guc_action_vf_resfix_start(struct xe_guc *guc, u16 marker)
 {
 	u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = {
 		FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
 		FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
-		FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_VF2GUC_NOTIFY_RESFIX_DONE),
+		FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_VF2GUC_RESFIX_START) |
+		FIELD_PREP(VF2GUC_RESFIX_START_REQUEST_MSG_0_MARKER, marker),
 	};
 	int ret;
 
@@ -313,28 +346,43 @@ static int guc_action_vf_notify_resfix_done(struct xe_guc *guc)
 	return ret > 0 ? -EPROTO : ret;
 }
 
-/**
- * vf_notify_resfix_done - Notify GuC about resource fixups apply completed.
- * @gt: the &xe_gt struct instance linked to target GuC
- *
- * Returns: 0 if the operation completed successfully, or a negative error
- * code otherwise.
- */
-static int vf_notify_resfix_done(struct xe_gt *gt)
+static int vf_resfix_start(struct xe_gt *gt, u16 marker)
 {
 	struct xe_guc *guc = &gt->uc.guc;
-	int err;
 
 	xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt)));
 
-	err = guc_action_vf_notify_resfix_done(guc);
-	if (unlikely(err))
-		xe_gt_sriov_err(gt, "Failed to notify GuC about resource fixup done (%pe)\n",
-				ERR_PTR(err));
-	else
-		xe_gt_sriov_dbg_verbose(gt, "sent GuC resource fixup done\n");
+	VF_MIGRATION_INJECT_WAIT(gt, RESFIX_START);
 
-	return err;
+	xe_gt_sriov_dbg_verbose(gt, "Sending resfix start marker %u\n", marker);
+
+	return guc_action_vf_resfix_start(guc, marker);
+}
+
+static int guc_action_vf_resfix_done(struct xe_guc *guc, u16 marker)
+{
+	u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = {
+		FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
+		FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
+		FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_VF2GUC_RESFIX_DONE) |
+		FIELD_PREP(VF2GUC_RESFIX_DONE_REQUEST_MSG_0_MARKER, marker),
+	};
+	int ret;
+
+	ret = xe_guc_mmio_send(guc, request, ARRAY_SIZE(request));
+
+	return ret > 0 ? -EPROTO : ret;
+}
+
+static int vf_resfix_done(struct xe_gt *gt, u16 marker)
+{
+	struct xe_guc *guc = &gt->uc.guc;
+
+	xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt)));
+
+	xe_gt_sriov_dbg_verbose(gt, "Sending resfix done marker %u\n", marker);
+
+	return guc_action_vf_resfix_done(guc, marker);
 }
 
 static int guc_action_query_single_klv(struct xe_guc *guc, u32 key,
@@ -733,7 +781,7 @@ static void vf_start_migration_recovery(struct xe_gt *gt)
 
 	spin_lock(&gt->sriov.vf.migration.lock);
 
-	if (!gt->sriov.vf.migration.recovery_queued ||
+	if (!gt->sriov.vf.migration.recovery_queued &&
 	    !gt->sriov.vf.migration.recovery_teardown) {
 		gt->sriov.vf.migration.recovery_queued = true;
 		WRITE_ONCE(gt->sriov.vf.migration.recovery_inprogress, true);
@@ -1123,12 +1171,8 @@ static bool vf_post_migration_shutdown(struct xe_gt *gt)
 			return true;
 	}
 
-	spin_lock_irq(&gt->sriov.vf.migration.lock);
-	gt->sriov.vf.migration.recovery_queued = false;
-	spin_unlock_irq(&gt->sriov.vf.migration.lock);
-
 	xe_guc_ct_flush_and_stop(&gt->uc.guc.ct);
-	xe_guc_submit_pause(&gt->uc.guc);
+	xe_guc_submit_pause_vf(&gt->uc.guc);
 	xe_tlb_inval_reset(&gt->tlb_inval);
 
 	return false;
@@ -1144,6 +1188,8 @@ static int vf_post_migration_fixups(struct xe_gt *gt)
 	void *buf = gt->sriov.vf.migration.scratch;
 	int err;
 
+	VF_MIGRATION_INJECT_WAIT(gt, FIXUPS);
+
 	/* xe_gt_sriov_vf_query_config will fixup the GGTT addresses */
 	err = xe_gt_sriov_vf_query_config(gt);
 	if (err)
@@ -1162,13 +1208,22 @@ static int vf_post_migration_fixups(struct xe_gt *gt)
 
 static void vf_post_migration_rearm(struct xe_gt *gt)
 {
+	VF_MIGRATION_INJECT_WAIT(gt, RESTART_JOBS);
+
+	/*
+	 * Make sure interrupts on the new HW are properly set. The GuC IRQ
+	 * must be working at this point, since the recovery did started,
+	 * but the rest was not enabled using the procedure from spec.
+	 */
+	xe_irq_resume(gt_to_xe(gt));
+
 	xe_guc_ct_restart(&gt->uc.guc.ct);
-	xe_guc_submit_unpause_prepare(&gt->uc.guc);
+	xe_guc_submit_unpause_prepare_vf(&gt->uc.guc);
 }
 
 static void vf_post_migration_kickstart(struct xe_gt *gt)
 {
-	xe_guc_submit_unpause(&gt->uc.guc);
+	xe_guc_submit_unpause_vf(&gt->uc.guc);
 }
 
 static void vf_post_migration_abort(struct xe_gt *gt)
@@ -1183,37 +1238,49 @@ static void vf_post_migration_abort(struct xe_gt *gt)
 	xe_guc_submit_pause_abort(&gt->uc.guc);
 }
 
-static int vf_post_migration_notify_resfix_done(struct xe_gt *gt)
+static int vf_post_migration_resfix_done(struct xe_gt *gt, u16 marker)
 {
-	bool skip_resfix = false;
+	VF_MIGRATION_INJECT_WAIT(gt, RESFIX_DONE);
 
 	spin_lock_irq(&gt->sriov.vf.migration.lock);
-	if (gt->sriov.vf.migration.recovery_queued) {
-		skip_resfix = true;
-		xe_gt_sriov_dbg(gt, "another recovery imminent, resfix skipped\n");
-	} else {
+	if (gt->sriov.vf.migration.recovery_queued)
+		xe_gt_sriov_dbg(gt, "another recovery imminent\n");
+	else
 		WRITE_ONCE(gt->sriov.vf.migration.recovery_inprogress, false);
-	}
 	spin_unlock_irq(&gt->sriov.vf.migration.lock);
 
-	if (skip_resfix)
-		return -EAGAIN;
+	return vf_resfix_done(gt, marker);
+}
 
-	/*
-	 * Make sure interrupts on the new HW are properly set. The GuC IRQ
-	 * must be working at this point, since the recovery did started,
-	 * but the rest was not enabled using the procedure from spec.
-	 */
-	xe_irq_resume(gt_to_xe(gt));
+static int vf_post_migration_resfix_start(struct xe_gt *gt, u16 marker)
+{
+	int err;
+
+	err = vf_resfix_start(gt, marker);
 
-	return vf_notify_resfix_done(gt);
+	guard(spinlock_irq) (&gt->sriov.vf.migration.lock);
+	gt->sriov.vf.migration.recovery_queued = false;
+
+	return err;
+}
+
+static u16 vf_post_migration_next_resfix_marker(struct xe_gt *gt)
+{
+	xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt)));
+
+	BUILD_BUG_ON(1 + ((typeof(gt->sriov.vf.migration.resfix_marker))~0) >
+		     FIELD_MAX(VF2GUC_RESFIX_START_REQUEST_MSG_0_MARKER));
+
+	/* add 1 to avoid zero-marker */
+	return 1 + gt->sriov.vf.migration.resfix_marker++;
 }
 
 static void vf_post_migration_recovery(struct xe_gt *gt)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	int err;
+	u16 marker;
 	bool retry;
+	int err;
 
 	xe_gt_sriov_dbg(gt, "migration recovery in progress\n");
 
@@ -1227,15 +1294,30 @@ static void vf_post_migration_recovery(struct xe_gt *gt)
 		goto fail;
 	}
 
+	marker = vf_post_migration_next_resfix_marker(gt);
+
+	err = vf_post_migration_resfix_start(gt, marker);
+	if (unlikely(err)) {
+		xe_gt_sriov_err(gt, "Recovery failed at GuC RESFIX_START step (%pe)\n",
+				ERR_PTR(err));
+		goto fail;
+	}
+
 	err = vf_post_migration_fixups(gt);
 	if (err)
 		goto fail;
 
 	vf_post_migration_rearm(gt);
 
-	err = vf_post_migration_notify_resfix_done(gt);
-	if (err && err != -EAGAIN)
+	err = vf_post_migration_resfix_done(gt, marker);
+	if (err) {
+		if (err == -EREMCHG)
+			goto queue;
+
+		xe_gt_sriov_err(gt, "Recovery failed at GuC RESFIX_DONE step (%pe)\n",
+				ERR_PTR(err));
 		goto fail;
+	}
 
 	vf_post_migration_kickstart(gt);
 
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c
index 2ed5b6780d30..507718326e1f 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c
@@ -69,4 +69,16 @@ void xe_gt_sriov_vf_debugfs_register(struct xe_gt *gt, struct dentry *root)
 	vfdentry->d_inode->i_private = gt;
 
 	drm_debugfs_create_files(vf_info, ARRAY_SIZE(vf_info), vfdentry, minor);
+
+	/*
+	 *      /sys/kernel/debug/dri/BDF/
+	 *      ├── tile0
+	 *          ├── gt0
+	 *              ├── vf
+	 *                  ├── resfix_stoppers
+	 */
+	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
+		debugfs_create_x8("resfix_stoppers", 0600, vfdentry,
+				  &gt->sriov.vf.migration.debug.resfix_stoppers);
+	}
 }
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h
index 420b0e6089de..510c33116fbd 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h
@@ -52,6 +52,19 @@ struct xe_gt_sriov_vf_migration {
 	wait_queue_head_t wq;
 	/** @scratch: Scratch memory for VF recovery */
 	void *scratch;
+	/** @debug: Debug hooks for delaying migration */
+	struct {
+		/**
+		 * @debug.resfix_stoppers: Stop and wait at different stages
+		 * during post migration recovery
+		 */
+		u8 resfix_stoppers;
+	} debug;
+	/**
+	 * @resfix_marker: Marker sent on start and on end of post-migration
+	 * steps.
+	 */
+	u8 resfix_marker;
 	/** @recovery_teardown: VF post migration recovery is being torn down */
 	bool recovery_teardown;
 	/** @recovery_queued: VF post migration recovery in queued */
diff --git a/drivers/gpu/drm/xe/xe_gt_stats.c b/drivers/gpu/drm/xe/xe_gt_stats.c
index 5f74706bab81..fb2904bd0abd 100644
--- a/drivers/gpu/drm/xe/xe_gt_stats.c
+++ b/drivers/gpu/drm/xe/xe_gt_stats.c
@@ -66,6 +66,16 @@ static const char *const stat_description[__XE_GT_STATS_NUM_IDS] = {
 	DEF_STAT_STR(SVM_4K_BIND_US, "svm_4K_bind_us"),
 	DEF_STAT_STR(SVM_64K_BIND_US, "svm_64K_bind_us"),
 	DEF_STAT_STR(SVM_2M_BIND_US, "svm_2M_bind_us"),
+	DEF_STAT_STR(HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT,
+		     "hw_engine_group_suspend_lr_queue_count"),
+	DEF_STAT_STR(HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT,
+		     "hw_engine_group_skip_lr_queue_count"),
+	DEF_STAT_STR(HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT,
+		     "hw_engine_group_wait_dma_queue_count"),
+	DEF_STAT_STR(HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US,
+		     "hw_engine_group_suspend_lr_queue_us"),
+	DEF_STAT_STR(HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US,
+		     "hw_engine_group_wait_dma_queue_us"),
 };
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_gt_stats.h b/drivers/gpu/drm/xe/xe_gt_stats.h
index e8aea32bc971..59a7bf60e242 100644
--- a/drivers/gpu/drm/xe/xe_gt_stats.h
+++ b/drivers/gpu/drm/xe/xe_gt_stats.h
@@ -6,6 +6,8 @@
 #ifndef _XE_GT_STATS_H_
 #define _XE_GT_STATS_H_
 
+#include <linux/ktime.h>
+
 #include "xe_gt_stats_types.h"
 
 struct xe_gt;
@@ -23,4 +25,34 @@ xe_gt_stats_incr(struct xe_gt *gt, const enum xe_gt_stats_id id,
 }
 
 #endif
+
+/**
+ * xe_gt_stats_ktime_us_delta() - Get delta in microseconds between now and a
+ * start time
+ * @start: Start time
+ *
+ * Helper for GT stats to get delta in microseconds between now and a start
+ * time, compiles out if GT stats are disabled.
+ *
+ * Return: Delta in microseconds between now and a start time
+ */
+static inline s64 xe_gt_stats_ktime_us_delta(ktime_t start)
+{
+	return IS_ENABLED(CONFIG_DEBUG_FS) ?
+		ktime_us_delta(ktime_get(), start) : 0;
+}
+
+/**
+ * xe_gt_stats_ktime_get() - Get current ktime
+ *
+ * Helper for GT stats to get current ktime, compiles out if GT stats are
+ * disabled.
+ *
+ * Return: Get current ktime
+ */
+static inline ktime_t xe_gt_stats_ktime_get(void)
+{
+	return IS_ENABLED(CONFIG_DEBUG_FS) ? ktime_get() : 0;
+}
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_gt_stats_types.h b/drivers/gpu/drm/xe/xe_gt_stats_types.h
index d8348a8de2e1..b92d013091d5 100644
--- a/drivers/gpu/drm/xe/xe_gt_stats_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_stats_types.h
@@ -44,6 +44,11 @@ enum xe_gt_stats_id {
 	XE_GT_STATS_ID_SVM_4K_BIND_US,
 	XE_GT_STATS_ID_SVM_64K_BIND_US,
 	XE_GT_STATS_ID_SVM_2M_BIND_US,
+	XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT,
+	XE_GT_STATS_ID_HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT,
+	XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT,
+	XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US,
+	XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US,
 	/* must be the last entry */
 	__XE_GT_STATS_NUM_IDS,
 };
diff --git a/drivers/gpu/drm/xe/xe_gt_throttle.c b/drivers/gpu/drm/xe/xe_gt_throttle.c
index 82c5fbcdfbe3..570358310e97 100644
--- a/drivers/gpu/drm/xe/xe_gt_throttle.c
+++ b/drivers/gpu/drm/xe/xe_gt_throttle.c
@@ -85,7 +85,7 @@ u32 xe_gt_throttle_get_limit_reasons(struct xe_gt *gt)
 {
 	struct xe_device *xe = gt_to_xe(gt);
 	struct xe_reg reg;
-	u32 val, mask;
+	u32 mask;
 
 	if (xe_gt_is_media_type(gt))
 		reg = MTL_MEDIA_PERF_LIMIT_REASONS;
@@ -97,11 +97,8 @@ u32 xe_gt_throttle_get_limit_reasons(struct xe_gt *gt)
 	else
 		mask = GT0_PERF_LIMIT_REASONS_MASK;
 
-	xe_pm_runtime_get(xe);
-	val = xe_mmio_read32(&gt->mmio, reg) & mask;
-	xe_pm_runtime_put(xe);
-
-	return val;
+	guard(xe_pm_runtime)(xe);
+	return xe_mmio_read32(&gt->mmio, reg) & mask;
 }
 
 static bool is_throttled_by(struct xe_gt *gt, u32 mask)
@@ -140,7 +137,7 @@ static ssize_t reasons_show(struct kobject *kobj,
 		struct throttle_attribute *other_ta = kobj_attribute_to_throttle(kattr);
 
 		if (other_ta->mask != U32_MAX && reasons & other_ta->mask)
-			ret += sysfs_emit_at(buff, ret, "%s ", (*pother)->name);
+			ret += sysfs_emit_at(buff, ret, "%s ", (*pother)->name + strlen("reason_"));
 	}
 
 	if (drm_WARN_ONCE(&xe->drm, !ret, "Unknown reason: %#x\n", reasons))
diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h
index 0a728180b6fe..5318d92fd473 100644
--- a/drivers/gpu/drm/xe/xe_gt_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_types.h
@@ -140,6 +140,11 @@ struct xe_gt {
 		u64 engine_mask;
 		/** @info.gmdid: raw GMD_ID value from hardware */
 		u32 gmdid;
+		/**
+		 * @multi_queue_engine_class_mask: Bitmask of engine classes with
+		 * multi queue support enabled.
+		 */
+		u16 multi_queue_engine_class_mask;
 		/** @info.id: Unique ID of this GT within the PCI Device */
 		u8 id;
 		/** @info.has_indirect_ring_state: GT has indirect ring state support */
diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c
index a686b04879d6..09ac092c3687 100644
--- a/drivers/gpu/drm/xe/xe_guc.c
+++ b/drivers/gpu/drm/xe/xe_guc.c
@@ -104,7 +104,7 @@ static u32 guc_ctl_log_params_flags(struct xe_guc *guc)
 	u32 offset = guc_bo_ggtt_addr(guc, guc->log.bo) >> PAGE_SHIFT;
 	u32 flags;
 
-	#if (((CRASH_BUFFER_SIZE) % SZ_1M) == 0)
+	#if (((XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE) % SZ_1M) == 0)
 	#define LOG_UNIT SZ_1M
 	#define LOG_FLAG GUC_LOG_LOG_ALLOC_UNITS
 	#else
@@ -112,7 +112,7 @@ static u32 guc_ctl_log_params_flags(struct xe_guc *guc)
 	#define LOG_FLAG 0
 	#endif
 
-	#if (((CAPTURE_BUFFER_SIZE) % SZ_1M) == 0)
+	#if (((XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE) % SZ_1M) == 0)
 	#define CAPTURE_UNIT SZ_1M
 	#define CAPTURE_FLAG GUC_LOG_CAPTURE_ALLOC_UNITS
 	#else
@@ -120,20 +120,21 @@ static u32 guc_ctl_log_params_flags(struct xe_guc *guc)
 	#define CAPTURE_FLAG 0
 	#endif
 
-	BUILD_BUG_ON(!CRASH_BUFFER_SIZE);
-	BUILD_BUG_ON(!IS_ALIGNED(CRASH_BUFFER_SIZE, LOG_UNIT));
-	BUILD_BUG_ON(!DEBUG_BUFFER_SIZE);
-	BUILD_BUG_ON(!IS_ALIGNED(DEBUG_BUFFER_SIZE, LOG_UNIT));
-	BUILD_BUG_ON(!CAPTURE_BUFFER_SIZE);
-	BUILD_BUG_ON(!IS_ALIGNED(CAPTURE_BUFFER_SIZE, CAPTURE_UNIT));
+	BUILD_BUG_ON(!XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE);
+	BUILD_BUG_ON(!IS_ALIGNED(XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE, LOG_UNIT));
+	BUILD_BUG_ON(!XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE);
+	BUILD_BUG_ON(!IS_ALIGNED(XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE, LOG_UNIT));
+	BUILD_BUG_ON(!XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE);
+	BUILD_BUG_ON(!IS_ALIGNED(XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE, CAPTURE_UNIT));
 
 	flags = GUC_LOG_VALID |
 		GUC_LOG_NOTIFY_ON_HALF_FULL |
 		CAPTURE_FLAG |
 		LOG_FLAG |
-		FIELD_PREP(GUC_LOG_CRASH, CRASH_BUFFER_SIZE / LOG_UNIT - 1) |
-		FIELD_PREP(GUC_LOG_DEBUG, DEBUG_BUFFER_SIZE / LOG_UNIT - 1) |
-		FIELD_PREP(GUC_LOG_CAPTURE, CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) |
+		FIELD_PREP(GUC_LOG_CRASH_DUMP, XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE / LOG_UNIT - 1) |
+		FIELD_PREP(GUC_LOG_EVENT_DATA, XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE / LOG_UNIT - 1) |
+		FIELD_PREP(GUC_LOG_STATE_CAPTURE, XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE /
+			   CAPTURE_UNIT - 1) |
 		FIELD_PREP(GUC_LOG_BUF_ADDR, offset);
 
 	#undef LOG_UNIT
@@ -660,11 +661,9 @@ static void guc_fini_hw(void *arg)
 {
 	struct xe_guc *guc = arg;
 	struct xe_gt *gt = guc_to_gt(guc);
-	unsigned int fw_ref;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	xe_uc_sanitize_reset(&guc_to_gt(guc)->uc);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
+	xe_with_force_wake(fw_ref, gt_to_fw(gt), XE_FORCEWAKE_ALL)
+		xe_uc_sanitize_reset(&guc_to_gt(guc)->uc);
 
 	guc_g2g_fini(guc);
 }
@@ -768,6 +767,10 @@ int xe_guc_init(struct xe_guc *guc)
 	if (!xe_uc_fw_is_enabled(&guc->fw))
 		return 0;
 
+	/* Disable page reclaim if GuC FW does not support */
+	if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 14, 0))
+		xe->info.has_page_reclaim_hw_assist = false;
+
 	if (IS_SRIOV_VF(xe)) {
 		ret = xe_guc_ct_init(&guc->ct);
 		if (ret)
@@ -1485,6 +1488,12 @@ int xe_guc_mmio_send_recv(struct xe_guc *guc, const u32 *request,
 		u32 hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, header);
 		u32 error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, header);
 
+		if (unlikely(error == XE_GUC_RESPONSE_VF_MIGRATED)) {
+			xe_gt_dbg(gt, "GuC mmio request %#x rejected due to MIGRATION (hint %#x)\n",
+				  request[0], hint);
+			return -EREMCHG;
+		}
+
 		xe_gt_err(gt, "GuC mmio request %#x: failure %#x hint %#x\n",
 			  request[0], error, hint);
 		return -ENXIO;
@@ -1618,18 +1627,51 @@ int xe_guc_start(struct xe_guc *guc)
 	return xe_guc_submit_start(guc);
 }
 
+/**
+ * xe_guc_runtime_suspend() - GuC runtime suspend
+ * @guc: The GuC object
+ *
+ * Stop further runs of submission tasks on given GuC and runtime suspend
+ * GuC CT.
+ */
+void xe_guc_runtime_suspend(struct xe_guc *guc)
+{
+	xe_guc_submit_pause(guc);
+	xe_guc_submit_disable(guc);
+	xe_guc_ct_runtime_suspend(&guc->ct);
+}
+
+/**
+ * xe_guc_runtime_resume() - GuC runtime resume
+ * @guc: The GuC object
+ *
+ * Runtime resume GuC CT and allow further runs of submission tasks on
+ * given GuC.
+ */
+void xe_guc_runtime_resume(struct xe_guc *guc)
+{
+	/*
+	 * Runtime PM flows are not applicable for VFs, so it's safe to
+	 * directly enable IRQ.
+	 */
+	guc_enable_irq(guc);
+
+	xe_guc_ct_runtime_resume(&guc->ct);
+	xe_guc_submit_enable(guc);
+	xe_guc_submit_unpause(guc);
+}
+
 void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p)
 {
 	struct xe_gt *gt = guc_to_gt(guc);
-	unsigned int fw_ref;
 	u32 status;
 	int i;
 
 	xe_uc_fw_print(&guc->fw, p);
 
 	if (!IS_SRIOV_VF(gt_to_xe(gt))) {
-		fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-		if (!fw_ref)
+		CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+		if (!fw_ref.domains)
 			return;
 
 		status = xe_mmio_read32(&gt->mmio, GUC_STATUS);
@@ -1649,8 +1691,6 @@ void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p)
 			drm_printf(p, "\t%2d: \t0x%x\n",
 				   i, xe_mmio_read32(&gt->mmio, SOFT_SCRATCH(i)));
 		}
-
-		xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	}
 
 	drm_puts(p, "\n");
diff --git a/drivers/gpu/drm/xe/xe_guc.h b/drivers/gpu/drm/xe/xe_guc.h
index e2d4c5f44ae3..a169f231cbd8 100644
--- a/drivers/gpu/drm/xe/xe_guc.h
+++ b/drivers/gpu/drm/xe/xe_guc.h
@@ -18,10 +18,16 @@
  */
 #define MAKE_GUC_VER(maj, min, pat)	(((maj) << 16) | ((min) << 8) | (pat))
 #define MAKE_GUC_VER_STRUCT(ver)	MAKE_GUC_VER((ver).major, (ver).minor, (ver).patch)
+#define MAKE_GUC_VER_ARGS(ver...) \
+	(BUILD_BUG_ON_ZERO(COUNT_ARGS(ver) < 2 || COUNT_ARGS(ver) > 3) + \
+	 MAKE_GUC_VER(PICK_ARG1(ver), PICK_ARG2(ver), IF_ARGS(PICK_ARG3(ver), 0, PICK_ARG3(ver))))
+
 #define GUC_SUBMIT_VER(guc) \
 	MAKE_GUC_VER_STRUCT((guc)->fw.versions.found[XE_UC_FW_VER_COMPATIBILITY])
 #define GUC_FIRMWARE_VER(guc) \
 	MAKE_GUC_VER_STRUCT((guc)->fw.versions.found[XE_UC_FW_VER_RELEASE])
+#define GUC_FIRMWARE_VER_AT_LEAST(guc, ver...) \
+	xe_guc_fw_version_at_least((guc), MAKE_GUC_VER_ARGS(ver))
 
 struct drm_printer;
 
@@ -35,6 +41,8 @@ int xe_guc_upload(struct xe_guc *guc);
 int xe_guc_min_load_for_hwconfig(struct xe_guc *guc);
 int xe_guc_enable_communication(struct xe_guc *guc);
 int xe_guc_opt_in_features_enable(struct xe_guc *guc);
+void xe_guc_runtime_suspend(struct xe_guc *guc);
+void xe_guc_runtime_resume(struct xe_guc *guc);
 int xe_guc_suspend(struct xe_guc *guc);
 void xe_guc_notify(struct xe_guc *guc);
 int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr);
@@ -94,4 +102,19 @@ static inline struct drm_device *guc_to_drm(struct xe_guc *guc)
 	return &guc_to_xe(guc)->drm;
 }
 
+/**
+ * xe_guc_fw_version_at_least() - Check if GuC is at least of given version.
+ * @guc: the &xe_guc
+ * @ver: the version to check
+ *
+ * The @ver should be prepared using MAKE_GUC_VER(major, minor, patch).
+ *
+ * Return: true if loaded GuC firmware is at least of given version,
+ *         false otherwise.
+ */
+static inline bool xe_guc_fw_version_at_least(const struct xe_guc *guc, u32 ver)
+{
+	return GUC_FIRMWARE_VER(guc) >= ver;
+}
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c
index bcb85a1bf26d..5feeb91426ee 100644
--- a/drivers/gpu/drm/xe/xe_guc_ads.c
+++ b/drivers/gpu/drm/xe/xe_guc_ads.c
@@ -317,7 +317,7 @@ static void guc_waklv_init(struct xe_guc_ads *ads)
 	offset = guc_ads_waklv_offset(ads);
 	remain = guc_ads_waklv_size(ads);
 
-	if (XE_GT_WA(gt, 14019882105) || XE_GT_WA(gt, 16021333562))
+	if (XE_GT_WA(gt, 16021333562))
 		guc_waklv_enable(ads, NULL, 0, &offset, &remain,
 				 GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED);
 	if (XE_GT_WA(gt, 18024947630))
@@ -347,10 +347,10 @@ static void guc_waklv_init(struct xe_guc_ads *ads)
 		guc_waklv_enable(ads, NULL, 0, &offset, &remain,
 				 GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET);
 
-	if (GUC_FIRMWARE_VER(&gt->uc.guc) >= MAKE_GUC_VER(70, 44, 0) && XE_GT_WA(gt, 16026508708))
+	if (GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 44) && XE_GT_WA(gt, 16026508708))
 		guc_waklv_enable(ads, NULL, 0, &offset, &remain,
 				 GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH);
-	if (GUC_FIRMWARE_VER(&gt->uc.guc) >= MAKE_GUC_VER(70, 47, 0) && XE_GT_WA(gt, 16026007364)) {
+	if (GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 47) && XE_GT_WA(gt, 16026007364)) {
 		u32 data[] = {
 			0x0,
 			0xF,
diff --git a/drivers/gpu/drm/xe/xe_guc_buf.c b/drivers/gpu/drm/xe/xe_guc_buf.c
index 3ce442500130..c36fc31e0438 100644
--- a/drivers/gpu/drm/xe/xe_guc_buf.c
+++ b/drivers/gpu/drm/xe/xe_guc_buf.c
@@ -30,7 +30,7 @@ static int guc_buf_cache_init(struct xe_guc_buf_cache *cache, u32 size)
 	struct xe_gt *gt = cache_to_gt(cache);
 	struct xe_sa_manager *sam;
 
-	sam = __xe_sa_bo_manager_init(gt_to_tile(gt), size, 0, sizeof(u32));
+	sam = __xe_sa_bo_manager_init(gt_to_tile(gt), size, 0, sizeof(u32), 0);
 	if (IS_ERR(sam))
 		return PTR_ERR(sam);
 	cache->sam = sam;
diff --git a/drivers/gpu/drm/xe/xe_guc_capture.c b/drivers/gpu/drm/xe/xe_guc_capture.c
index 0c1fbe97b8bf..2cda92f7b323 100644
--- a/drivers/gpu/drm/xe/xe_guc_capture.c
+++ b/drivers/gpu/drm/xe/xe_guc_capture.c
@@ -843,7 +843,7 @@ static void check_guc_capture_size(struct xe_guc *guc)
 {
 	int capture_size = guc_capture_output_size_est(guc);
 	int spare_size = capture_size * GUC_CAPTURE_OVERBUFFER_MULTIPLIER;
-	u32 buffer_size = xe_guc_log_section_size_capture(&guc->log);
+	u32 buffer_size = XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE;
 
 	/*
 	 * NOTE: capture_size is much smaller than the capture region
@@ -949,7 +949,7 @@ guc_capture_init_node(struct xe_guc *guc, struct __guc_capture_parsed_output *no
  *                  ADS module also calls separately for PF vs VF.
  *
  *     --> alloc B: GuC output capture buf (registered via guc_init_params(log_param))
- *                  Size = #define CAPTURE_BUFFER_SIZE (warns if on too-small)
+ *                  Size = XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE (warns if on too-small)
  *                  Note2: 'x 3' to hold multiple capture groups
  *
  * GUC Runtime notify capture:
@@ -1367,7 +1367,7 @@ static int __guc_capture_flushlog_complete(struct xe_guc *guc)
 {
 	u32 action[] = {
 		XE_GUC_ACTION_LOG_BUFFER_FILE_FLUSH_COMPLETE,
-		GUC_LOG_BUFFER_CAPTURE
+		GUC_LOG_TYPE_STATE_CAPTURE
 	};
 
 	return xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action));
@@ -1384,8 +1384,8 @@ static void __guc_capture_process_output(struct xe_guc *guc)
 	u32 log_buf_state_offset;
 	u32 src_data_offset;
 
-	log_buf_state_offset = sizeof(struct guc_log_buffer_state) * GUC_LOG_BUFFER_CAPTURE;
-	src_data_offset = xe_guc_get_log_buffer_offset(&guc->log, GUC_LOG_BUFFER_CAPTURE);
+	log_buf_state_offset = sizeof(struct guc_log_buffer_state) * GUC_LOG_TYPE_STATE_CAPTURE;
+	src_data_offset = XE_GUC_LOG_STATE_CAPTURE_OFFSET;
 
 	/*
 	 * Make a copy of the state structure, inside GuC log buffer
@@ -1395,15 +1395,15 @@ static void __guc_capture_process_output(struct xe_guc *guc)
 	xe_map_memcpy_from(guc_to_xe(guc), &log_buf_state_local, &guc->log.bo->vmap,
 			   log_buf_state_offset, sizeof(struct guc_log_buffer_state));
 
-	buffer_size = xe_guc_get_log_buffer_size(&guc->log, GUC_LOG_BUFFER_CAPTURE);
+	buffer_size = XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE;
 	read_offset = log_buf_state_local.read_ptr;
 	write_offset = log_buf_state_local.sampled_write_ptr;
 	full_count = FIELD_GET(GUC_LOG_BUFFER_STATE_BUFFER_FULL_CNT, log_buf_state_local.flags);
 
 	/* Bookkeeping stuff */
 	tmp = FIELD_GET(GUC_LOG_BUFFER_STATE_FLUSH_TO_FILE, log_buf_state_local.flags);
-	guc->log.stats[GUC_LOG_BUFFER_CAPTURE].flush += tmp;
-	new_overflow = xe_guc_check_log_buf_overflow(&guc->log, GUC_LOG_BUFFER_CAPTURE,
+	guc->log.stats[GUC_LOG_TYPE_STATE_CAPTURE].flush += tmp;
+	new_overflow = xe_guc_check_log_buf_overflow(&guc->log, GUC_LOG_TYPE_STATE_CAPTURE,
 						     full_count);
 
 	/* Now copy the actual logs. */
diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c
index 4ac434ad216f..c3df9b3f1b4d 100644
--- a/drivers/gpu/drm/xe/xe_guc_ct.c
+++ b/drivers/gpu/drm/xe/xe_guc_ct.c
@@ -42,6 +42,21 @@ static void ct_exit_safe_mode(struct xe_guc_ct *ct);
 static void guc_ct_change_state(struct xe_guc_ct *ct,
 				enum xe_guc_ct_state state);
 
+static struct xe_guc *ct_to_guc(struct xe_guc_ct *ct)
+{
+	return container_of(ct, struct xe_guc, ct);
+}
+
+static struct xe_gt *ct_to_gt(struct xe_guc_ct *ct)
+{
+	return container_of(ct, struct xe_gt, uc.guc.ct);
+}
+
+static struct xe_device *ct_to_xe(struct xe_guc_ct *ct)
+{
+	return gt_to_xe(ct_to_gt(ct));
+}
+
 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
 enum {
 	/* Internal states, not error conditions */
@@ -68,14 +83,101 @@ enum {
 static void ct_dead_worker_func(struct work_struct *w);
 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code);
 
-#define CT_DEAD(ct, ctb, reason_code)		ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code)
+static void ct_dead_fini(struct xe_guc_ct *ct)
+{
+	cancel_work_sync(&ct->dead.worker);
+}
+
+static void ct_dead_init(struct xe_guc_ct *ct)
+{
+	spin_lock_init(&ct->dead.lock);
+	INIT_WORK(&ct->dead.worker, ct_dead_worker_func);
+
+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
+	stack_depot_init();
+#endif
+}
+
+static void fast_req_stack_save(struct xe_guc_ct *ct, unsigned int slot)
+{
+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
+	unsigned long entries[SZ_32];
+	unsigned int n;
+
+	n = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
+	/* May be called under spinlock, so avoid sleeping */
+	ct->fast_req[slot].stack = stack_depot_save(entries, n, GFP_NOWAIT);
+#endif
+}
+
+static void fast_req_dump(struct xe_guc_ct *ct, u16 fence, unsigned int slot)
+{
+	struct xe_gt *gt = ct_to_gt(ct);
+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
+	char *buf __cleanup(kfree) = kmalloc(SZ_4K, GFP_NOWAIT);
+
+	if (buf && stack_depot_snprint(ct->fast_req[slot].stack, buf, SZ_4K, 0))
+		xe_gt_err(gt, "Fence 0x%x was used by action %#04x sent at:\n%s\n",
+			  fence, ct->fast_req[slot].action, buf);
+	else
+		xe_gt_err(gt, "Fence 0x%x was used by action %#04x [failed to retrieve stack]\n",
+			  fence, ct->fast_req[slot].action);
+#else
+	xe_gt_err(gt, "Fence 0x%x was used by action %#04x\n",
+		  fence, ct->fast_req[slot].action);
+#endif
+}
+
+static void fast_req_report(struct xe_guc_ct *ct, u16 fence)
+{
+	u16 fence_min = U16_MAX, fence_max = 0;
+	struct xe_gt *gt = ct_to_gt(ct);
+	unsigned int n;
+
+	lockdep_assert_held(&ct->lock);
+
+	for (n = 0; n < ARRAY_SIZE(ct->fast_req); n++) {
+		if (ct->fast_req[n].fence < fence_min)
+			fence_min = ct->fast_req[n].fence;
+		if (ct->fast_req[n].fence > fence_max)
+			fence_max = ct->fast_req[n].fence;
+
+		if (ct->fast_req[n].fence != fence)
+			continue;
+
+		return fast_req_dump(ct, fence, n);
+	}
+
+	xe_gt_warn(gt, "Fence 0x%x not found - tracking buffer wrapped? [range = 0x%x -> 0x%x, next = 0x%X]\n",
+		   fence, fence_min, fence_max, ct->fence_seqno);
+}
+
+static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action)
+{
+	unsigned int slot = fence % ARRAY_SIZE(ct->fast_req);
+
+	fast_req_stack_save(ct, slot);
+	ct->fast_req[slot].fence = fence;
+	ct->fast_req[slot].action = action;
+}
+
+#define CT_DEAD(ct, ctb, reason_code)	ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code)
+
 #else
+
+static void ct_dead_fini(struct xe_guc_ct *ct) { }
+static void ct_dead_init(struct xe_guc_ct *ct) { }
+
+static void fast_req_report(struct xe_guc_ct *ct, u16 fence) { }
+static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) { }
+
 #define CT_DEAD(ct, ctb, reason)			\
 	do {						\
 		struct guc_ctb *_ctb = (ctb);		\
 		if (_ctb)				\
 			_ctb->info.broken = true;	\
 	} while (0)
+
 #endif
 
 /* Used when a CT send wants to block and / or receive data */
@@ -112,24 +214,6 @@ static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence)
 	return g2h_fence->seqno == ~0x0;
 }
 
-static struct xe_guc *
-ct_to_guc(struct xe_guc_ct *ct)
-{
-	return container_of(ct, struct xe_guc, ct);
-}
-
-static struct xe_gt *
-ct_to_gt(struct xe_guc_ct *ct)
-{
-	return container_of(ct, struct xe_gt, uc.guc.ct);
-}
-
-static struct xe_device *
-ct_to_xe(struct xe_guc_ct *ct)
-{
-	return gt_to_xe(ct_to_gt(ct));
-}
-
 /**
  * DOC: GuC CTB Blob
  *
@@ -169,8 +253,11 @@ ct_to_xe(struct xe_guc_ct *ct)
 #define CTB_DESC_SIZE		ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
 #define CTB_H2G_BUFFER_OFFSET	(CTB_DESC_SIZE * 2)
 #define CTB_H2G_BUFFER_SIZE	(SZ_4K)
+#define CTB_H2G_BUFFER_DWORDS	(CTB_H2G_BUFFER_SIZE / sizeof(u32))
 #define CTB_G2H_BUFFER_SIZE	(SZ_128K)
+#define CTB_G2H_BUFFER_DWORDS	(CTB_G2H_BUFFER_SIZE / sizeof(u32))
 #define G2H_ROOM_BUFFER_SIZE	(CTB_G2H_BUFFER_SIZE / 2)
+#define G2H_ROOM_BUFFER_DWORDS	(CTB_G2H_BUFFER_DWORDS / 2)
 
 /**
  * xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full
@@ -199,9 +286,7 @@ static void guc_ct_fini(struct drm_device *drm, void *arg)
 {
 	struct xe_guc_ct *ct = arg;
 
-#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
-	cancel_work_sync(&ct->dead.worker);
-#endif
+	ct_dead_fini(ct);
 	ct_exit_safe_mode(ct);
 	destroy_workqueue(ct->g2h_wq);
 	xa_destroy(&ct->fence_lookup);
@@ -239,13 +324,8 @@ int xe_guc_ct_init_noalloc(struct xe_guc_ct *ct)
 	xa_init(&ct->fence_lookup);
 	INIT_WORK(&ct->g2h_worker, g2h_worker_func);
 	INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func);
-#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
-	spin_lock_init(&ct->dead.lock);
-	INIT_WORK(&ct->dead.worker, ct_dead_worker_func);
-#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
-	stack_depot_init();
-#endif
-#endif
+
+	ct_dead_init(ct);
 	init_waitqueue_head(&ct->wq);
 	init_waitqueue_head(&ct->g2h_fence_wq);
 
@@ -326,7 +406,7 @@ int xe_guc_ct_init_post_hwconfig(struct xe_guc_ct *ct)
 static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g,
 				struct iosys_map *map)
 {
-	h2g->info.size = CTB_H2G_BUFFER_SIZE / sizeof(u32);
+	h2g->info.size = CTB_H2G_BUFFER_DWORDS;
 	h2g->info.resv_space = 0;
 	h2g->info.tail = 0;
 	h2g->info.head = 0;
@@ -344,8 +424,8 @@ static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g,
 static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h,
 				struct iosys_map *map)
 {
-	g2h->info.size = CTB_G2H_BUFFER_SIZE / sizeof(u32);
-	g2h->info.resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32);
+	g2h->info.size = CTB_G2H_BUFFER_DWORDS;
+	g2h->info.resv_space = G2H_ROOM_BUFFER_DWORDS;
 	g2h->info.head = 0;
 	g2h->info.tail = 0;
 	g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head,
@@ -640,6 +720,39 @@ void xe_guc_ct_stop(struct xe_guc_ct *ct)
 	stop_g2h_handler(ct);
 }
 
+/**
+ * xe_guc_ct_runtime_suspend() - GuC CT runtime suspend
+ * @ct: the &xe_guc_ct
+ *
+ * Set GuC CT to disabled state.
+ */
+void xe_guc_ct_runtime_suspend(struct xe_guc_ct *ct)
+{
+	struct guc_ctb *g2h = &ct->ctbs.g2h;
+	u32 credits = CIRC_SPACE(0, 0, CTB_G2H_BUFFER_DWORDS) - G2H_ROOM_BUFFER_DWORDS;
+
+	/* We should be back to guc_ct_ctb_g2h_init() values */
+	xe_gt_assert(ct_to_gt(ct), g2h->info.space == credits);
+
+	/*
+	 * Since we're already in runtime suspend path, we shouldn't have pending
+	 * messages. But if there happen to be any, we'd probably want them to be
+	 * thrown as errors for further investigation.
+	 */
+	xe_guc_ct_disable(ct);
+}
+
+/**
+ * xe_guc_ct_runtime_resume() - GuC CT runtime resume
+ * @ct: the &xe_guc_ct
+ *
+ * Restart GuC CT and set it to enabled state.
+ */
+void xe_guc_ct_runtime_resume(struct xe_guc_ct *ct)
+{
+	xe_guc_ct_restart(ct);
+}
+
 static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len)
 {
 	struct guc_ctb *h2g = &ct->ctbs.h2g;
@@ -747,28 +860,6 @@ static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)
 	spin_unlock_irq(&ct->fast_lock);
 }
 
-#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
-static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action)
-{
-	unsigned int slot = fence % ARRAY_SIZE(ct->fast_req);
-#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
-	unsigned long entries[SZ_32];
-	unsigned int n;
-
-	n = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
-
-	/* May be called under spinlock, so avoid sleeping */
-	ct->fast_req[slot].stack = stack_depot_save(entries, n, GFP_NOWAIT);
-#endif
-	ct->fast_req[slot].fence = fence;
-	ct->fast_req[slot].action = action;
-}
-#else
-static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action)
-{
-}
-#endif
-
 /*
  * The CT protocol accepts a 16 bits fence. This field is fully owned by the
  * driver, the GuC will just copy it to the reply message. Since we need to
@@ -1310,10 +1401,12 @@ static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len)
 	lockdep_assert_held(&ct->lock);
 
 	switch (action) {
+	case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE:
 	case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
 	case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
 	case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:
 	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
+	case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
 		g2h_release_space(ct, len);
 	}
 
@@ -1338,55 +1431,6 @@ static int guc_crash_process_msg(struct xe_guc_ct *ct, u32 action)
 	return 0;
 }
 
-#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
-static void fast_req_report(struct xe_guc_ct *ct, u16 fence)
-{
-	u16 fence_min = U16_MAX, fence_max = 0;
-	struct xe_gt *gt = ct_to_gt(ct);
-	bool found = false;
-	unsigned int n;
-#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
-	char *buf;
-#endif
-
-	lockdep_assert_held(&ct->lock);
-
-	for (n = 0; n < ARRAY_SIZE(ct->fast_req); n++) {
-		if (ct->fast_req[n].fence < fence_min)
-			fence_min = ct->fast_req[n].fence;
-		if (ct->fast_req[n].fence > fence_max)
-			fence_max = ct->fast_req[n].fence;
-
-		if (ct->fast_req[n].fence != fence)
-			continue;
-		found = true;
-
-#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
-		buf = kmalloc(SZ_4K, GFP_NOWAIT);
-		if (buf && stack_depot_snprint(ct->fast_req[n].stack, buf, SZ_4K, 0))
-			xe_gt_err(gt, "Fence 0x%x was used by action %#04x sent at:\n%s",
-				  fence, ct->fast_req[n].action, buf);
-		else
-			xe_gt_err(gt, "Fence 0x%x was used by action %#04x [failed to retrieve stack]\n",
-				  fence, ct->fast_req[n].action);
-		kfree(buf);
-#else
-		xe_gt_err(gt, "Fence 0x%x was used by action %#04x\n",
-			  fence, ct->fast_req[n].action);
-#endif
-		break;
-	}
-
-	if (!found)
-		xe_gt_warn(gt, "Fence 0x%x not found - tracking buffer wrapped? [range = 0x%x -> 0x%x, next = 0x%X]\n",
-			   fence, fence_min, fence_max, ct->fence_seqno);
-}
-#else
-static void fast_req_report(struct xe_guc_ct *ct, u16 fence)
-{
-}
-#endif
-
 static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len)
 {
 	struct xe_gt *gt =  ct_to_gt(ct);
@@ -1549,6 +1593,15 @@ static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
 		ret = xe_guc_pagefault_handler(guc, payload, adj_len);
 		break;
 	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
+	case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
+		/*
+		 * Page reclamation is an extension of TLB invalidation. Both
+		 * operations share the same seqno and fence. When either
+		 * action completes, we need to signal the corresponding
+		 * fence. Since the handling logic (lookup fence by seqno,
+		 * fence signalling) is identical, we use the same handler
+		 * for both G2H events.
+		 */
 		ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len);
 		break;
 	case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF:
@@ -1572,6 +1625,13 @@ static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
 		ret = xe_guc_g2g_test_notification(guc, payload, adj_len);
 		break;
 #endif
+	case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE:
+		ret = xe_guc_exec_queue_cgp_sync_done_handler(guc, payload, adj_len);
+		break;
+	case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR:
+		ret = xe_guc_exec_queue_cgp_context_error_handler(guc, payload,
+								  adj_len);
+		break;
 	default:
 		xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action);
 	}
@@ -1714,6 +1774,7 @@ static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path)
 		switch (action) {
 		case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
 		case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
+		case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
 			break;	/* Process these in fast-path */
 		default:
 			return 0;
@@ -1750,6 +1811,12 @@ static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len)
 		ret = xe_guc_pagefault_handler(guc, payload, adj_len);
 		break;
 	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
+	case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
+		/*
+		 * Seqno and fence handling of page reclamation and TLB
+		 * invalidation is identical, so we can use the same handler
+		 * for both actions.
+		 */
 		__g2h_release_space(ct, len);
 		ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len);
 		break;
diff --git a/drivers/gpu/drm/xe/xe_guc_ct.h b/drivers/gpu/drm/xe/xe_guc_ct.h
index ca1ce2b3c354..5599939f8fe1 100644
--- a/drivers/gpu/drm/xe/xe_guc_ct.h
+++ b/drivers/gpu/drm/xe/xe_guc_ct.h
@@ -17,6 +17,8 @@ int xe_guc_ct_init_post_hwconfig(struct xe_guc_ct *ct);
 int xe_guc_ct_enable(struct xe_guc_ct *ct);
 int xe_guc_ct_restart(struct xe_guc_ct *ct);
 void xe_guc_ct_disable(struct xe_guc_ct *ct);
+void xe_guc_ct_runtime_resume(struct xe_guc_ct *ct);
+void xe_guc_ct_runtime_suspend(struct xe_guc_ct *ct);
 void xe_guc_ct_stop(struct xe_guc_ct *ct);
 void xe_guc_ct_flush_and_stop(struct xe_guc_ct *ct);
 void xe_guc_ct_fast_path(struct xe_guc_ct *ct);
diff --git a/drivers/gpu/drm/xe/xe_guc_debugfs.c b/drivers/gpu/drm/xe/xe_guc_debugfs.c
index 0b102ab46c4d..23827e87450f 100644
--- a/drivers/gpu/drm/xe/xe_guc_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_guc_debugfs.c
@@ -70,13 +70,9 @@ static int guc_debugfs_show(struct seq_file *m, void *data)
 	struct xe_gt *gt = grandparent->d_inode->i_private;
 	struct xe_device *xe = gt_to_xe(gt);
 	int (*print)(struct xe_guc *, struct drm_printer *) = node->info_ent->data;
-	int ret;
 
-	xe_pm_runtime_get(xe);
-	ret = print(&gt->uc.guc, &p);
-	xe_pm_runtime_put(xe);
-
-	return ret;
+	guard(xe_pm_runtime)(xe);
+	return print(&gt->uc.guc, &p);
 }
 
 static int guc_log(struct xe_guc *guc, struct drm_printer *p)
@@ -85,6 +81,12 @@ static int guc_log(struct xe_guc *guc, struct drm_printer *p)
 	return 0;
 }
 
+static int guc_log_lfd(struct xe_guc *guc, struct drm_printer *p)
+{
+	xe_guc_log_print_lfd(&guc->log, p);
+	return 0;
+}
+
 static int guc_log_dmesg(struct xe_guc *guc, struct drm_printer *p)
 {
 	xe_guc_log_print_dmesg(&guc->log);
@@ -121,6 +123,7 @@ static const struct drm_info_list slpc_debugfs_list[] = {
 /* everything else should be added here */
 static const struct drm_info_list pf_only_debugfs_list[] = {
 	{ "guc_log", .show = guc_debugfs_show, .data = guc_log },
+	{ "guc_log_lfd", .show = guc_debugfs_show, .data = guc_log_lfd },
 	{ "guc_log_dmesg", .show = guc_debugfs_show, .data = guc_log_dmesg },
 };
 
diff --git a/drivers/gpu/drm/xe/xe_guc_fwif.h b/drivers/gpu/drm/xe/xe_guc_fwif.h
index c90dd266e9cf..a04faec477ae 100644
--- a/drivers/gpu/drm/xe/xe_guc_fwif.h
+++ b/drivers/gpu/drm/xe/xe_guc_fwif.h
@@ -16,6 +16,8 @@
 #define G2H_LEN_DW_DEREGISTER_CONTEXT		3
 #define G2H_LEN_DW_TLB_INVALIDATE		3
 #define G2H_LEN_DW_G2G_NOTIFY_MIN		3
+#define G2H_LEN_DW_MULTI_QUEUE_CONTEXT		3
+#define G2H_LEN_DW_PAGE_RECLAMATION		3
 
 #define GUC_ID_MAX			65535
 #define GUC_ID_UNKNOWN			0xffffffff
@@ -62,6 +64,8 @@ struct guc_ctxt_registration_info {
 	u32 wq_base_lo;
 	u32 wq_base_hi;
 	u32 wq_size;
+	u32 cgp_lo;
+	u32 cgp_hi;
 	u32 hwlrca_lo;
 	u32 hwlrca_hi;
 };
@@ -91,9 +95,9 @@ struct guc_update_exec_queue_policy {
 #define   GUC_LOG_NOTIFY_ON_HALF_FULL	BIT(1)
 #define   GUC_LOG_CAPTURE_ALLOC_UNITS	BIT(2)
 #define   GUC_LOG_LOG_ALLOC_UNITS	BIT(3)
-#define   GUC_LOG_CRASH			REG_GENMASK(5, 4)
-#define   GUC_LOG_DEBUG			REG_GENMASK(9, 6)
-#define   GUC_LOG_CAPTURE		REG_GENMASK(11, 10)
+#define   GUC_LOG_CRASH_DUMP		REG_GENMASK(5, 4)
+#define   GUC_LOG_EVENT_DATA		REG_GENMASK(9, 6)
+#define   GUC_LOG_STATE_CAPTURE		REG_GENMASK(11, 10)
 #define   GUC_LOG_BUF_ADDR		REG_GENMASK(31, 12)
 
 #define GUC_CTL_WA			1
diff --git a/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h b/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h
index 0a028c94756d..45ab5a3b5218 100644
--- a/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h
@@ -24,6 +24,11 @@
  * ABI and the associated &NAME, that may be used in code or debugfs/sysfs::
  *
  *	define(TAG, NAME)
+ *
+ * If required, KLVs can be labeled with GuC firmware version that added them::
+ *
+ *	define(TAG, NAME, MAJOR, MINOR)
+ *	define(TAG, NAME, MAJOR, MINOR, PATCH)
  */
 #define MAKE_XE_GUC_KLV_THRESHOLDS_SET(define)		\
 	define(CAT_ERR, cat_error_count)		\
@@ -32,6 +37,7 @@
 	define(H2G_STORM, guc_time_us)			\
 	define(IRQ_STORM, irq_time_us)			\
 	define(DOORBELL_STORM, doorbell_time_us)	\
+	define(MULTI_LRC_COUNT, multi_lrc_count, 70, 53)\
 	/* end */
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_guc_log.c b/drivers/gpu/drm/xe/xe_guc_log.c
index c01ccb35dc75..d7473b9673bb 100644
--- a/drivers/gpu/drm/xe/xe_guc_log.c
+++ b/drivers/gpu/drm/xe/xe_guc_log.c
@@ -7,8 +7,10 @@
 
 #include <linux/fault-inject.h>
 
+#include <linux/utsname.h>
 #include <drm/drm_managed.h>
 
+#include "abi/guc_lfd_abi.h"
 #include "regs/xe_guc_regs.h"
 #include "xe_bo.h"
 #include "xe_devcoredump.h"
@@ -19,6 +21,77 @@
 #include "xe_mmio.h"
 #include "xe_module.h"
 
+#define GUC_LOG_CHUNK_SIZE			SZ_2M
+
+/* Magic keys define */
+#define GUC_LFD_DRIVER_KEY_STREAMING		0x8086AAAA474C5346
+#define GUC_LFD_LOG_BUFFER_MARKER_2		0xDEADFEED
+#define GUC_LFD_CRASH_DUMP_BUFFER_MARKER_2	0x8086DEAD
+#define GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_2	0xBEEFFEED
+#define GUC_LFD_LOG_BUFFER_MARKER_1V2		0xCABBA9E6
+#define GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_1V2	0xCABBA9F7
+#define GUC_LFD_DATA_HEADER_MAGIC		0x8086
+
+/* LFD supported LIC type range */
+#define GUC_LIC_TYPE_FIRST			GUC_LIC_TYPE_GUC_SW_VERSION
+#define GUC_LIC_TYPE_LAST			GUC_LIC_TYPE_BUILD_PLATFORM_ID
+#define GUC_LFD_TYPE_FW_RANGE_FIRST		GUC_LFD_TYPE_FW_VERSION
+#define GUC_LFD_TYPE_FW_RANGE_LAST		GUC_LFD_TYPE_BUILD_PLATFORM_ID
+
+#define GUC_LOG_BUFFER_STATE_HEADER_LENGTH	4096
+#define GUC_LOG_BUFFER_INIT_CONFIG		3
+
+struct guc_log_buffer_entry_list {
+	u32 offset;
+	u32 rd_ptr;
+	u32 wr_ptr;
+	u32 wrap_offset;
+	u32 buf_size;
+};
+
+struct guc_lic_save {
+	u32 version;
+	/*
+	 * Array of init config KLV values.
+	 * Range from GUC_LOG_LIC_TYPE_FIRST to GUC_LOG_LIC_TYPE_LAST
+	 */
+	u32 values[GUC_LIC_TYPE_LAST - GUC_LIC_TYPE_FIRST + 1];
+	struct guc_log_buffer_entry_list entry[GUC_LOG_BUFFER_INIT_CONFIG];
+};
+
+static struct guc_log_buffer_entry_markers {
+	u32 key[2];
+} const entry_markers[GUC_LOG_BUFFER_INIT_CONFIG + 1] = {
+	{{
+		GUC_LFD_LOG_BUFFER_MARKER_1V2,
+		GUC_LFD_LOG_BUFFER_MARKER_2
+	}},
+	{{
+		GUC_LFD_LOG_BUFFER_MARKER_1V2,
+		GUC_LFD_CRASH_DUMP_BUFFER_MARKER_2
+	}},
+	{{
+		GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_1V2,
+		GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_2
+	}},
+	{{
+		GUC_LIC_MAGIC,
+		(FIELD_PREP_CONST(GUC_LIC_VERSION_MASK_MAJOR, GUC_LIC_VERSION_MAJOR) |
+		 FIELD_PREP_CONST(GUC_LIC_VERSION_MASK_MINOR, GUC_LIC_VERSION_MINOR))
+	}}
+};
+
+static struct guc_log_lic_lfd_map {
+	u32 lic;
+	u32 lfd;
+} const lic_lfd_type_map[] = {
+	{GUC_LIC_TYPE_GUC_SW_VERSION,		GUC_LFD_TYPE_FW_VERSION},
+	{GUC_LIC_TYPE_GUC_DEVICE_ID,		GUC_LFD_TYPE_GUC_DEVICE_ID},
+	{GUC_LIC_TYPE_TSC_FREQUENCY,		GUC_LFD_TYPE_TSC_FREQUENCY},
+	{GUC_LIC_TYPE_GMD_ID,			GUC_LFD_TYPE_GMD_ID},
+	{GUC_LIC_TYPE_BUILD_PLATFORM_ID,	GUC_LFD_TYPE_BUILD_PLATFORM_ID}
+};
+
 static struct xe_guc *
 log_to_guc(struct xe_guc_log *log)
 {
@@ -37,33 +110,6 @@ log_to_xe(struct xe_guc_log *log)
 	return gt_to_xe(log_to_gt(log));
 }
 
-static size_t guc_log_size(void)
-{
-	/*
-	 *  GuC Log buffer Layout
-	 *
-	 *  +===============================+ 00B
-	 *  |    Crash dump state header    |
-	 *  +-------------------------------+ 32B
-	 *  |      Debug state header       |
-	 *  +-------------------------------+ 64B
-	 *  |     Capture state header      |
-	 *  +-------------------------------+ 96B
-	 *  |                               |
-	 *  +===============================+ PAGE_SIZE (4KB)
-	 *  |        Crash Dump logs        |
-	 *  +===============================+ + CRASH_SIZE
-	 *  |          Debug logs           |
-	 *  +===============================+ + DEBUG_SIZE
-	 *  |         Capture logs          |
-	 *  +===============================+ + CAPTURE_SIZE
-	 */
-	return PAGE_SIZE + CRASH_BUFFER_SIZE + DEBUG_BUFFER_SIZE +
-		CAPTURE_BUFFER_SIZE;
-}
-
-#define GUC_LOG_CHUNK_SIZE	SZ_2M
-
 static struct xe_guc_log_snapshot *xe_guc_log_snapshot_alloc(struct xe_guc_log *log, bool atomic)
 {
 	struct xe_guc_log_snapshot *snapshot;
@@ -145,7 +191,6 @@ struct xe_guc_log_snapshot *xe_guc_log_snapshot_capture(struct xe_guc_log *log,
 	struct xe_device *xe = log_to_xe(log);
 	struct xe_guc *guc = log_to_guc(log);
 	struct xe_gt *gt = log_to_gt(log);
-	unsigned int fw_ref;
 	size_t remain;
 	int i;
 
@@ -165,13 +210,12 @@ struct xe_guc_log_snapshot *xe_guc_log_snapshot_capture(struct xe_guc_log *log,
 		remain -= size;
 	}
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref) {
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!fw_ref.domains)
 		snapshot->stamp = ~0ULL;
-	} else {
+	else
 		snapshot->stamp = xe_mmio_read64_2x32(&gt->mmio, GUC_PMTIMESTAMP_LO);
-		xe_force_wake_put(gt_to_fw(gt), fw_ref);
-	}
+
 	snapshot->ktime = ktime_get_boottime_ns();
 	snapshot->level = log->level;
 	snapshot->ver_found = guc->fw.versions.found[XE_UC_FW_VER_RELEASE];
@@ -216,6 +260,318 @@ void xe_guc_log_snapshot_print(struct xe_guc_log_snapshot *snapshot, struct drm_
 	}
 }
 
+static inline void lfd_output_binary(struct drm_printer *p, char *buf, int buf_size)
+{
+	seq_write(p->arg, buf, buf_size);
+}
+
+static inline int xe_guc_log_add_lfd_header(struct guc_lfd_data *lfd)
+{
+	lfd->header = FIELD_PREP_CONST(GUC_LFD_DATA_HEADER_MASK_MAGIC, GUC_LFD_DATA_HEADER_MAGIC);
+	return offsetof(struct guc_lfd_data, data);
+}
+
+static int xe_guc_log_add_typed_payload(struct drm_printer *p, u32 type,
+					u32 data_len, void *data)
+{
+	struct guc_lfd_data lfd;
+	int len;
+
+	len = xe_guc_log_add_lfd_header(&lfd);
+	lfd.header |= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, type);
+	/* make length DW aligned */
+	lfd.data_count = DIV_ROUND_UP(data_len, sizeof(u32));
+	lfd_output_binary(p, (char *)&lfd, len);
+
+	lfd_output_binary(p, data, data_len);
+	len += lfd.data_count * sizeof(u32);
+
+	return len;
+}
+
+static inline int lic_type_to_index(u32 lic_type)
+{
+	XE_WARN_ON(lic_type < GUC_LIC_TYPE_FIRST || lic_type > GUC_LIC_TYPE_LAST);
+
+	return lic_type - GUC_LIC_TYPE_FIRST;
+}
+
+static inline int lfd_type_to_index(u32 lfd_type)
+{
+	int i, lic_type = 0;
+
+	XE_WARN_ON(lfd_type < GUC_LFD_TYPE_FW_RANGE_FIRST || lfd_type > GUC_LFD_TYPE_FW_RANGE_LAST);
+
+	for (i = 0; i < ARRAY_SIZE(lic_lfd_type_map); i++)
+		if (lic_lfd_type_map[i].lfd == lfd_type)
+			lic_type = lic_lfd_type_map[i].lic;
+
+	/* If not found, lic_type_to_index will warning invalid type */
+	return lic_type_to_index(lic_type);
+}
+
+static int xe_guc_log_add_klv(struct drm_printer *p, u32 lfd_type,
+			      struct guc_lic_save *config)
+{
+	int klv_index = lfd_type_to_index(lfd_type);
+
+	return xe_guc_log_add_typed_payload(p, lfd_type, sizeof(u32), &config->values[klv_index]);
+}
+
+static int xe_guc_log_add_os_id(struct drm_printer *p, u32 id)
+{
+	struct guc_lfd_data_os_info os_id;
+	struct guc_lfd_data lfd;
+	int len, info_len, section_len;
+	char *version;
+	u32 blank = 0;
+
+	len = xe_guc_log_add_lfd_header(&lfd);
+	lfd.header |= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, GUC_LFD_TYPE_OS_ID);
+
+	os_id.os_id = id;
+	section_len = offsetof(struct guc_lfd_data_os_info, build_version);
+
+	version = init_utsname()->release;
+	info_len = strlen(version);
+
+	/* make length DW aligned */
+	lfd.data_count = DIV_ROUND_UP(section_len + info_len, sizeof(u32));
+	lfd_output_binary(p, (char *)&lfd, len);
+	lfd_output_binary(p, (char *)&os_id, section_len);
+	lfd_output_binary(p, version, info_len);
+
+	/* Padding with 0 */
+	section_len = lfd.data_count * sizeof(u32) - section_len - info_len;
+	if (section_len)
+		lfd_output_binary(p, (char *)&blank, section_len);
+
+	len +=  lfd.data_count * sizeof(u32);
+	return len;
+}
+
+static void xe_guc_log_loop_log_init(struct guc_lic *init, struct guc_lic_save *config)
+{
+	struct guc_klv_generic_dw_t *p = (void *)init->data;
+	int i;
+
+	for (i = 0; i < init->data_count;) {
+		int klv_len = FIELD_GET(GUC_KLV_0_LEN, p->kl) + 1;
+		int key = FIELD_GET(GUC_KLV_0_KEY, p->kl);
+
+		if (key < GUC_LIC_TYPE_FIRST || key > GUC_LIC_TYPE_LAST) {
+			XE_WARN_ON(key < GUC_LIC_TYPE_FIRST || key > GUC_LIC_TYPE_LAST);
+			break;
+		}
+		config->values[lic_type_to_index(key)] = p->value;
+		i += klv_len + 1; /* Whole KLV structure length in dwords */
+		p = (void *)((u32 *)p + klv_len);
+	}
+}
+
+static int find_marker(u32 mark0, u32 mark1)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(entry_markers); i++)
+		if (mark0 == entry_markers[i].key[0] && mark1 == entry_markers[i].key[1])
+			return i;
+
+	return ARRAY_SIZE(entry_markers);
+}
+
+static void xe_guc_log_load_lic(void *guc_log, struct guc_lic_save *config)
+{
+	u32 offset = GUC_LOG_BUFFER_STATE_HEADER_LENGTH;
+	struct guc_log_buffer_state *p = guc_log;
+
+	config->version = p->version;
+	while (p->marker[0]) {
+		int index;
+
+		index = find_marker(p->marker[0], p->marker[1]);
+
+		if (index < ARRAY_SIZE(entry_markers)) {
+			if (index == GUC_LOG_BUFFER_INIT_CONFIG) {
+				/* Load log init config */
+				xe_guc_log_loop_log_init((void *)p, config);
+
+				/* LIC structure is the last */
+				return;
+			}
+			config->entry[index].offset = offset;
+			config->entry[index].rd_ptr = p->read_ptr;
+			config->entry[index].wr_ptr = p->write_ptr;
+			config->entry[index].wrap_offset = p->wrap_offset;
+			config->entry[index].buf_size = p->size;
+		}
+		offset += p->size;
+		p++;
+	}
+}
+
+static int
+xe_guc_log_output_lfd_init(struct drm_printer *p, struct xe_guc_log_snapshot *snapshot,
+			   struct guc_lic_save *config)
+{
+	int type, len;
+	size_t size = 0;
+
+	/* FW required types */
+	for (type = GUC_LFD_TYPE_FW_RANGE_FIRST; type <= GUC_LFD_TYPE_FW_RANGE_LAST; type++)
+		size += xe_guc_log_add_klv(p, type, config);
+
+	/* KMD required type(s) */
+	len = xe_guc_log_add_os_id(p, GUC_LFD_OS_TYPE_OSID_LIN);
+	size += len;
+
+	return size;
+}
+
+static void
+xe_guc_log_print_chunks(struct drm_printer *p, struct xe_guc_log_snapshot *snapshot,
+			u32 from, u32 to)
+{
+	int chunk_from = from % GUC_LOG_CHUNK_SIZE;
+	int chunk_id = from / GUC_LOG_CHUNK_SIZE;
+	int to_chunk_id = to / GUC_LOG_CHUNK_SIZE;
+	int chunk_to = to % GUC_LOG_CHUNK_SIZE;
+	int pos = from;
+
+	do {
+		size_t size = (to_chunk_id == chunk_id ? chunk_to : GUC_LOG_CHUNK_SIZE) -
+			      chunk_from;
+
+		lfd_output_binary(p, snapshot->copy[chunk_id] + chunk_from, size);
+		pos += size;
+		chunk_id++;
+		chunk_from = 0;
+	} while (pos < to);
+}
+
+static inline int
+xe_guc_log_add_log_event(struct drm_printer *p, struct xe_guc_log_snapshot *snapshot,
+			 struct guc_lic_save *config)
+{
+	size_t size;
+	u32 data_len, section_len;
+	struct guc_lfd_data lfd;
+	struct guc_log_buffer_entry_list *entry;
+	struct guc_lfd_data_log_events_buf events_buf;
+
+	entry = &config->entry[GUC_LOG_TYPE_EVENT_DATA];
+
+	/* Skip empty log */
+	if (entry->rd_ptr == entry->wr_ptr)
+		return 0;
+
+	size = xe_guc_log_add_lfd_header(&lfd);
+	lfd.header |= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, GUC_LFD_TYPE_LOG_EVENTS_BUFFER);
+	events_buf.log_events_format_version = config->version;
+
+	/* Adjust to log_format_buf */
+	section_len = offsetof(struct guc_lfd_data_log_events_buf, log_event);
+	data_len = section_len;
+
+	/* Calculate data length */
+	data_len += entry->rd_ptr < entry->wr_ptr ? (entry->wr_ptr - entry->rd_ptr) :
+		(entry->wr_ptr + entry->wrap_offset - entry->rd_ptr);
+	/* make length u32 aligned */
+	lfd.data_count = DIV_ROUND_UP(data_len, sizeof(u32));
+
+	/* Output GUC_LFD_TYPE_LOG_EVENTS_BUFFER header */
+	lfd_output_binary(p, (char *)&lfd, size);
+	lfd_output_binary(p, (char *)&events_buf, section_len);
+
+	/* Output data from guc log chunks directly */
+	if (entry->rd_ptr < entry->wr_ptr) {
+		xe_guc_log_print_chunks(p, snapshot, entry->offset + entry->rd_ptr,
+					entry->offset + entry->wr_ptr);
+	} else {
+		/* 1st, print from rd to wrap offset */
+		xe_guc_log_print_chunks(p, snapshot, entry->offset + entry->rd_ptr,
+					entry->offset + entry->wrap_offset);
+
+		/* 2nd, print from buf start to wr */
+		xe_guc_log_print_chunks(p, snapshot, entry->offset, entry->offset + entry->wr_ptr);
+	}
+	return size;
+}
+
+static int
+xe_guc_log_add_crash_dump(struct drm_printer *p, struct xe_guc_log_snapshot *snapshot,
+			  struct guc_lic_save *config)
+{
+	struct guc_log_buffer_entry_list *entry;
+	int chunk_from, chunk_id;
+	int from, to, i;
+	size_t size = 0;
+	u32 *buf32;
+
+	entry = &config->entry[GUC_LOG_TYPE_CRASH_DUMP];
+
+	/* Skip zero sized crash dump */
+	if (!entry->buf_size)
+		return 0;
+
+	/* Check if crash dump section are all zero */
+	from = entry->offset;
+	to = entry->offset + entry->buf_size;
+	chunk_from = from % GUC_LOG_CHUNK_SIZE;
+	chunk_id = from / GUC_LOG_CHUNK_SIZE;
+	buf32 = snapshot->copy[chunk_id] + chunk_from;
+
+	for (i = 0; i < entry->buf_size / sizeof(u32); i++)
+		if (buf32[i])
+			break;
+
+	/* Buffer has non-zero data? */
+	if (i < entry->buf_size / sizeof(u32)) {
+		struct guc_lfd_data lfd;
+
+		size = xe_guc_log_add_lfd_header(&lfd);
+		lfd.header |= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, GUC_LFD_TYPE_FW_CRASH_DUMP);
+		/* Calculate data length */
+		lfd.data_count = DIV_ROUND_UP(entry->buf_size, sizeof(u32));
+		/* Output GUC_LFD_TYPE_FW_CRASH_DUMP header */
+		lfd_output_binary(p, (char *)&lfd, size);
+
+		/* rd/wr ptr is not used for crash dump */
+		xe_guc_log_print_chunks(p, snapshot, from, to);
+	}
+	return size;
+}
+
+static void
+xe_guc_log_snapshot_print_lfd(struct xe_guc_log_snapshot *snapshot, struct drm_printer *p)
+{
+	struct guc_lfd_file_header header;
+	struct guc_lic_save config;
+	size_t size;
+
+	if (!snapshot || !snapshot->size)
+		return;
+
+	header.magic = GUC_LFD_DRIVER_KEY_STREAMING;
+	header.version = FIELD_PREP_CONST(GUC_LFD_FILE_HEADER_VERSION_MASK_MINOR,
+					  GUC_LFD_FORMAT_VERSION_MINOR) |
+			 FIELD_PREP_CONST(GUC_LFD_FILE_HEADER_VERSION_MASK_MAJOR,
+					  GUC_LFD_FORMAT_VERSION_MAJOR);
+
+	/* Output LFD file header */
+	lfd_output_binary(p, (char *)&header,
+			  offsetof(struct guc_lfd_file_header, stream));
+
+	/* Output LFD stream */
+	xe_guc_log_load_lic(snapshot->copy[0], &config);
+	size = xe_guc_log_output_lfd_init(p, snapshot, &config);
+	if (!size)
+		return;
+
+	xe_guc_log_add_log_event(p, snapshot, &config);
+	xe_guc_log_add_crash_dump(p, snapshot, &config);
+}
+
 /**
  * xe_guc_log_print_dmesg - dump a copy of the GuC log to dmesg
  * @log: GuC log structure
@@ -251,13 +607,27 @@ void xe_guc_log_print(struct xe_guc_log *log, struct drm_printer *p)
 	xe_guc_log_snapshot_free(snapshot);
 }
 
+/**
+ * xe_guc_log_print_lfd - dump a copy of the GuC log in LFD format
+ * @log: GuC log structure
+ * @p: the printer object to output to
+ */
+void xe_guc_log_print_lfd(struct xe_guc_log *log, struct drm_printer *p)
+{
+	struct xe_guc_log_snapshot *snapshot;
+
+	snapshot = xe_guc_log_snapshot_capture(log, false);
+	xe_guc_log_snapshot_print_lfd(snapshot, p);
+	xe_guc_log_snapshot_free(snapshot);
+}
+
 int xe_guc_log_init(struct xe_guc_log *log)
 {
 	struct xe_device *xe = log_to_xe(log);
 	struct xe_tile *tile = gt_to_tile(log_to_gt(log));
 	struct xe_bo *bo;
 
-	bo = xe_managed_bo_create_pin_map(xe, tile, guc_log_size(),
+	bo = xe_managed_bo_create_pin_map(xe, tile, GUC_LOG_SIZE,
 					  XE_BO_FLAG_SYSTEM |
 					  XE_BO_FLAG_GGTT |
 					  XE_BO_FLAG_GGTT_INVALIDATE |
@@ -265,7 +635,7 @@ int xe_guc_log_init(struct xe_guc_log *log)
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
-	xe_map_memset(xe, &bo->vmap, 0, 0, guc_log_size());
+	xe_map_memset(xe, &bo->vmap, 0, 0, xe_bo_size(bo));
 	log->bo = bo;
 	log->level = xe_modparam.guc_log_level;
 
@@ -274,71 +644,6 @@ int xe_guc_log_init(struct xe_guc_log *log)
 
 ALLOW_ERROR_INJECTION(xe_guc_log_init, ERRNO); /* See xe_pci_probe() */
 
-static u32 xe_guc_log_section_size_crash(struct xe_guc_log *log)
-{
-	return CRASH_BUFFER_SIZE;
-}
-
-static u32 xe_guc_log_section_size_debug(struct xe_guc_log *log)
-{
-	return DEBUG_BUFFER_SIZE;
-}
-
-/**
- * xe_guc_log_section_size_capture - Get capture buffer size within log sections.
- * @log: The log object.
- *
- * This function will return the capture buffer size within log sections.
- *
- * Return: capture buffer size.
- */
-u32 xe_guc_log_section_size_capture(struct xe_guc_log *log)
-{
-	return CAPTURE_BUFFER_SIZE;
-}
-
-/**
- * xe_guc_get_log_buffer_size - Get log buffer size for a type.
- * @log: The log object.
- * @type: The log buffer type
- *
- * Return: buffer size.
- */
-u32 xe_guc_get_log_buffer_size(struct xe_guc_log *log, enum guc_log_buffer_type type)
-{
-	switch (type) {
-	case GUC_LOG_BUFFER_CRASH_DUMP:
-		return xe_guc_log_section_size_crash(log);
-	case GUC_LOG_BUFFER_DEBUG:
-		return xe_guc_log_section_size_debug(log);
-	case GUC_LOG_BUFFER_CAPTURE:
-		return xe_guc_log_section_size_capture(log);
-	}
-	return 0;
-}
-
-/**
- * xe_guc_get_log_buffer_offset - Get offset in log buffer for a type.
- * @log: The log object.
- * @type: The log buffer type
- *
- * This function will return the offset in the log buffer for a type.
- * Return: buffer offset.
- */
-u32 xe_guc_get_log_buffer_offset(struct xe_guc_log *log, enum guc_log_buffer_type type)
-{
-	enum guc_log_buffer_type i;
-	u32 offset = PAGE_SIZE;/* for the log_buffer_states */
-
-	for (i = GUC_LOG_BUFFER_CRASH_DUMP; i < GUC_LOG_BUFFER_TYPE_MAX; ++i) {
-		if (i == type)
-			break;
-		offset += xe_guc_get_log_buffer_size(log, i);
-	}
-
-	return offset;
-}
-
 /**
  * xe_guc_check_log_buf_overflow - Check if log buffer overflowed
  * @log: The log object.
@@ -352,7 +657,7 @@ u32 xe_guc_get_log_buffer_offset(struct xe_guc_log *log, enum guc_log_buffer_typ
  *
  * Return: True if overflowed.
  */
-bool xe_guc_check_log_buf_overflow(struct xe_guc_log *log, enum guc_log_buffer_type type,
+bool xe_guc_check_log_buf_overflow(struct xe_guc_log *log, enum guc_log_type type,
 				   unsigned int full_cnt)
 {
 	unsigned int prev_full_cnt = log->stats[type].sampled_overflow;
diff --git a/drivers/gpu/drm/xe/xe_guc_log.h b/drivers/gpu/drm/xe/xe_guc_log.h
index 98a47ac42b08..1b05bb60c1c7 100644
--- a/drivers/gpu/drm/xe/xe_guc_log.h
+++ b/drivers/gpu/drm/xe/xe_guc_log.h
@@ -13,14 +13,26 @@ struct drm_printer;
 struct xe_device;
 
 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
-#define CRASH_BUFFER_SIZE       SZ_1M
-#define DEBUG_BUFFER_SIZE       SZ_8M
-#define CAPTURE_BUFFER_SIZE     SZ_2M
+#define XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE	SZ_8M
+#define XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE	SZ_1M
+#define XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE	SZ_2M
 #else
-#define CRASH_BUFFER_SIZE	SZ_16K
-#define DEBUG_BUFFER_SIZE	SZ_64K
-#define CAPTURE_BUFFER_SIZE	SZ_1M
+#define XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE	SZ_64K
+#define XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE	SZ_16K
+#define XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE	SZ_1M
 #endif
+
+#define GUC_LOG_SIZE (SZ_4K + \
+		      XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE + \
+		      XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE + \
+		      XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE)
+
+#define XE_GUC_LOG_EVENT_DATA_OFFSET	SZ_4K
+#define XE_GUC_LOG_CRASH_DUMP_OFFSET	(XE_GUC_LOG_EVENT_DATA_OFFSET + \
+					 XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE)
+#define XE_GUC_LOG_STATE_CAPTURE_OFFSET	(XE_GUC_LOG_CRASH_DUMP_OFFSET + \
+					 XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE)
+
 /*
  * While we're using plain log level in i915, GuC controls are much more...
  * "elaborate"? We have a couple of bits for verbosity, separate bit for actual
@@ -40,6 +52,7 @@ struct xe_device;
 
 int xe_guc_log_init(struct xe_guc_log *log);
 void xe_guc_log_print(struct xe_guc_log *log, struct drm_printer *p);
+void xe_guc_log_print_lfd(struct xe_guc_log *log, struct drm_printer *p);
 void xe_guc_log_print_dmesg(struct xe_guc_log *log);
 struct xe_guc_log_snapshot *xe_guc_log_snapshot_capture(struct xe_guc_log *log, bool atomic);
 void xe_guc_log_snapshot_print(struct xe_guc_log_snapshot *snapshot, struct drm_printer *p);
@@ -51,11 +64,8 @@ xe_guc_log_get_level(struct xe_guc_log *log)
 	return log->level;
 }
 
-u32 xe_guc_log_section_size_capture(struct xe_guc_log *log);
-u32 xe_guc_get_log_buffer_size(struct xe_guc_log *log, enum guc_log_buffer_type type);
-u32 xe_guc_get_log_buffer_offset(struct xe_guc_log *log, enum guc_log_buffer_type type);
 bool xe_guc_check_log_buf_overflow(struct xe_guc_log *log,
-				   enum guc_log_buffer_type type,
+				   enum guc_log_type type,
 				   unsigned int full_cnt);
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c
index 951a49fb1d3e..54702a0fd05b 100644
--- a/drivers/gpu/drm/xe/xe_guc_pc.c
+++ b/drivers/gpu/drm/xe/xe_guc_pc.c
@@ -76,7 +76,7 @@
  * exposes a programming interface to the host for the control of SLPC.
  *
  * Frequency management:
- * =====================
+ * ---------------------
  *
  * Xe driver enables SLPC with all of its defaults features and frequency
  * selection, which varies per platform.
@@ -87,7 +87,7 @@
  * for any workload.
  *
  * Render-C States:
- * ================
+ * ----------------
  *
  * Render-C states is also a GuC PC feature that is now enabled in Xe for
  * all platforms.
@@ -499,21 +499,17 @@ u32 xe_guc_pc_get_cur_freq_fw(struct xe_guc_pc *pc)
 int xe_guc_pc_get_cur_freq(struct xe_guc_pc *pc, u32 *freq)
 {
 	struct xe_gt *gt = pc_to_gt(pc);
-	unsigned int fw_ref;
 
 	/*
 	 * GuC SLPC plays with cur freq request when GuCRC is enabled
 	 * Block RC6 for a more reliable read.
 	 */
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT)) {
-		xe_force_wake_put(gt_to_fw(gt), fw_ref);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
 		return -ETIMEDOUT;
-	}
 
 	*freq = get_cur_freq(gt);
 
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
 }
 
@@ -1087,13 +1083,8 @@ int xe_guc_pc_gucrc_disable(struct xe_guc_pc *pc)
  */
 int xe_guc_pc_override_gucrc_mode(struct xe_guc_pc *pc, enum slpc_gucrc_mode mode)
 {
-	int ret;
-
-	xe_pm_runtime_get(pc_to_xe(pc));
-	ret = pc_action_set_param(pc, SLPC_PARAM_PWRGATE_RC_MODE, mode);
-	xe_pm_runtime_put(pc_to_xe(pc));
-
-	return ret;
+	guard(xe_pm_runtime)(pc_to_xe(pc));
+	return pc_action_set_param(pc, SLPC_PARAM_PWRGATE_RC_MODE, mode);
 }
 
 /**
@@ -1104,13 +1095,8 @@ int xe_guc_pc_override_gucrc_mode(struct xe_guc_pc *pc, enum slpc_gucrc_mode mod
  */
 int xe_guc_pc_unset_gucrc_mode(struct xe_guc_pc *pc)
 {
-	int ret;
-
-	xe_pm_runtime_get(pc_to_xe(pc));
-	ret = pc_action_unset_param(pc, SLPC_PARAM_PWRGATE_RC_MODE);
-	xe_pm_runtime_put(pc_to_xe(pc));
-
-	return ret;
+	guard(xe_pm_runtime)(pc_to_xe(pc));
+	return pc_action_unset_param(pc, SLPC_PARAM_PWRGATE_RC_MODE);
 }
 
 static void pc_init_pcode_freq(struct xe_guc_pc *pc)
@@ -1198,7 +1184,7 @@ int xe_guc_pc_set_power_profile(struct xe_guc_pc *pc, const char *buf)
 		return -EINVAL;
 
 	guard(mutex)(&pc->freq_lock);
-	xe_pm_runtime_get_noresume(pc_to_xe(pc));
+	guard(xe_pm_runtime_noresume)(pc_to_xe(pc));
 
 	ret = pc_action_set_param(pc,
 				  SLPC_PARAM_POWER_PROFILE,
@@ -1209,8 +1195,6 @@ int xe_guc_pc_set_power_profile(struct xe_guc_pc *pc, const char *buf)
 	else
 		pc->power_profile = val;
 
-	xe_pm_runtime_put(pc_to_xe(pc));
-
 	return ret;
 }
 
@@ -1223,17 +1207,14 @@ int xe_guc_pc_start(struct xe_guc_pc *pc)
 	struct xe_device *xe = pc_to_xe(pc);
 	struct xe_gt *gt = pc_to_gt(pc);
 	u32 size = PAGE_ALIGN(sizeof(struct slpc_shared_data));
-	unsigned int fw_ref;
 	ktime_t earlier;
 	int ret;
 
 	xe_gt_assert(gt, xe_device_uc_enabled(xe));
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT)) {
-		xe_force_wake_put(gt_to_fw(gt), fw_ref);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
 		return -ETIMEDOUT;
-	}
 
 	if (xe->info.skip_guc_pc) {
 		if (xe->info.platform != XE_PVC)
@@ -1241,9 +1222,7 @@ int xe_guc_pc_start(struct xe_guc_pc *pc)
 
 		/* Request max possible since dynamic freq mgmt is not enabled */
 		pc_set_cur_freq(pc, UINT_MAX);
-
-		ret = 0;
-		goto out;
+		return 0;
 	}
 
 	xe_map_memset(xe, &pc->bo->vmap, 0, 0, size);
@@ -1252,7 +1231,7 @@ int xe_guc_pc_start(struct xe_guc_pc *pc)
 	earlier = ktime_get();
 	ret = pc_action_reset(pc);
 	if (ret)
-		goto out;
+		return ret;
 
 	if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING,
 			      SLPC_RESET_TIMEOUT_MS)) {
@@ -1263,8 +1242,7 @@ int xe_guc_pc_start(struct xe_guc_pc *pc)
 		if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING,
 				      SLPC_RESET_EXTENDED_TIMEOUT_MS)) {
 			xe_gt_err(gt, "GuC PC Start failed: Dynamic GT frequency control and GT sleep states are now disabled.\n");
-			ret = -EIO;
-			goto out;
+			return -EIO;
 		}
 
 		xe_gt_warn(gt, "GuC PC excessive start time: %lldms",
@@ -1273,21 +1251,20 @@ int xe_guc_pc_start(struct xe_guc_pc *pc)
 
 	ret = pc_init_freqs(pc);
 	if (ret)
-		goto out;
+		return ret;
 
 	ret = pc_set_mert_freq_cap(pc);
 	if (ret)
-		goto out;
+		return ret;
 
 	if (xe->info.platform == XE_PVC) {
 		xe_guc_pc_gucrc_disable(pc);
-		ret = 0;
-		goto out;
+		return 0;
 	}
 
 	ret = pc_action_setup_gucrc(pc, GUCRC_FIRMWARE_CONTROL);
 	if (ret)
-		goto out;
+		return ret;
 
 	/* Enable SLPC Optimized Strategy for compute */
 	ret = pc_action_set_strategy(pc, SLPC_OPTIMIZED_STRATEGY_COMPUTE);
@@ -1297,8 +1274,6 @@ int xe_guc_pc_start(struct xe_guc_pc *pc)
 	if (unlikely(ret))
 		xe_gt_err(gt, "Failed to set SLPC power profile: %pe\n", ERR_PTR(ret));
 
-out:
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return ret;
 }
 
@@ -1330,19 +1305,16 @@ static void xe_guc_pc_fini_hw(void *arg)
 {
 	struct xe_guc_pc *pc = arg;
 	struct xe_device *xe = pc_to_xe(pc);
-	unsigned int fw_ref;
 
 	if (xe_device_wedged(xe))
 		return;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(pc_to_gt(pc)), XE_FORCEWAKE_ALL);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(pc_to_gt(pc)), XE_FORCEWAKE_ALL);
 	xe_guc_pc_gucrc_disable(pc);
 	XE_WARN_ON(xe_guc_pc_stop(pc));
 
 	/* Bind requested freq to mert_freq_cap before unload */
 	pc_set_cur_freq(pc, min(pc_max_freq_cap(pc), xe_guc_pc_get_rpe_freq(pc)));
-
-	xe_force_wake_put(gt_to_fw(pc_to_gt(pc)), fw_ref);
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c
index ed7be50b2f72..43fd2069f9b2 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit.c
+++ b/drivers/gpu/drm/xe/xe_guc_submit.c
@@ -19,6 +19,7 @@
 #include "abi/guc_klvs_abi.h"
 #include "regs/xe_lrc_layout.h"
 #include "xe_assert.h"
+#include "xe_bo.h"
 #include "xe_devcoredump.h"
 #include "xe_device.h"
 #include "xe_exec_queue.h"
@@ -47,6 +48,8 @@
 #include "xe_uc_fw.h"
 #include "xe_vm.h"
 
+#define XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN		6
+
 static struct xe_guc *
 exec_queue_to_guc(struct xe_exec_queue *q)
 {
@@ -72,6 +75,7 @@ exec_queue_to_guc(struct xe_exec_queue *q)
 #define EXEC_QUEUE_STATE_EXTRA_REF		(1 << 11)
 #define EXEC_QUEUE_STATE_PENDING_RESUME		(1 << 12)
 #define EXEC_QUEUE_STATE_PENDING_TDR_EXIT	(1 << 13)
+#define EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND	(1 << 14)
 
 static bool exec_queue_registered(struct xe_exec_queue *q)
 {
@@ -263,6 +267,21 @@ static void clear_exec_queue_pending_tdr_exit(struct xe_exec_queue *q)
 	atomic_and(~EXEC_QUEUE_STATE_PENDING_TDR_EXIT, &q->guc->state);
 }
 
+static bool exec_queue_idle_skip_suspend(struct xe_exec_queue *q)
+{
+	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND;
+}
+
+static void set_exec_queue_idle_skip_suspend(struct xe_exec_queue *q)
+{
+	atomic_or(EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND, &q->guc->state);
+}
+
+static void clear_exec_queue_idle_skip_suspend(struct xe_exec_queue *q)
+{
+	atomic_and(~EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND, &q->guc->state);
+}
+
 static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q)
 {
 	return (atomic_read(&q->guc->state) &
@@ -541,7 +560,8 @@ static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q)
 	u32 slpc_exec_queue_freq_req = 0;
 	u32 preempt_timeout_us = q->sched_props.preempt_timeout_us;
 
-	xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
+	xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q) &&
+		     !xe_exec_queue_is_multi_queue_secondary(q));
 
 	if (q->flags & EXEC_QUEUE_FLAG_LOW_LATENCY)
 		slpc_exec_queue_freq_req |= SLPC_CTX_FREQ_REQ_IS_COMPUTE;
@@ -561,6 +581,8 @@ static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue
 {
 	struct exec_queue_policy policy;
 
+	xe_assert(guc_to_xe(guc), !xe_exec_queue_is_multi_queue_secondary(q));
+
 	__guc_exec_queue_policy_start_klv(&policy, q->guc->id);
 	__guc_exec_queue_policy_add_preemption_timeout(&policy, 1);
 
@@ -568,6 +590,89 @@ static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue
 		       __guc_exec_queue_policy_action_size(&policy), 0, 0);
 }
 
+static bool vf_recovery(struct xe_guc *guc)
+{
+	return xe_gt_recovery_pending(guc_to_gt(guc));
+}
+
+static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q)
+{
+	struct xe_guc *guc = exec_queue_to_guc(q);
+	struct xe_device *xe = guc_to_xe(guc);
+
+	/** to wakeup xe_wait_user_fence ioctl if exec queue is reset */
+	wake_up_all(&xe->ufence_wq);
+
+	if (xe_exec_queue_is_lr(q))
+		queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr);
+	else
+		xe_sched_tdr_queue_imm(&q->guc->sched);
+}
+
+static void xe_guc_exec_queue_group_trigger_cleanup(struct xe_exec_queue *q)
+{
+	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
+	struct xe_exec_queue_group *group = q->multi_queue.group;
+	struct xe_exec_queue *eq;
+
+	xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)),
+		     xe_exec_queue_is_multi_queue(q));
+
+	/* Group banned, skip timeout check in TDR */
+	WRITE_ONCE(group->banned, true);
+	xe_guc_exec_queue_trigger_cleanup(primary);
+
+	mutex_lock(&group->list_lock);
+	list_for_each_entry(eq, &group->list, multi_queue.link)
+		xe_guc_exec_queue_trigger_cleanup(eq);
+	mutex_unlock(&group->list_lock);
+}
+
+static void xe_guc_exec_queue_reset_trigger_cleanup(struct xe_exec_queue *q)
+{
+	if (xe_exec_queue_is_multi_queue(q)) {
+		struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
+		struct xe_exec_queue_group *group = q->multi_queue.group;
+		struct xe_exec_queue *eq;
+
+		/* Group banned, skip timeout check in TDR */
+		WRITE_ONCE(group->banned, true);
+
+		set_exec_queue_reset(primary);
+		if (!exec_queue_banned(primary) && !exec_queue_check_timeout(primary))
+			xe_guc_exec_queue_trigger_cleanup(primary);
+
+		mutex_lock(&group->list_lock);
+		list_for_each_entry(eq, &group->list, multi_queue.link) {
+			set_exec_queue_reset(eq);
+			if (!exec_queue_banned(eq) && !exec_queue_check_timeout(eq))
+				xe_guc_exec_queue_trigger_cleanup(eq);
+		}
+		mutex_unlock(&group->list_lock);
+	} else {
+		set_exec_queue_reset(q);
+		if (!exec_queue_banned(q) && !exec_queue_check_timeout(q))
+			xe_guc_exec_queue_trigger_cleanup(q);
+	}
+}
+
+static void set_exec_queue_group_banned(struct xe_exec_queue *q)
+{
+	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
+	struct xe_exec_queue_group *group = q->multi_queue.group;
+	struct xe_exec_queue *eq;
+
+	/* Ban all queues of the multi-queue group */
+	xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)),
+		     xe_exec_queue_is_multi_queue(q));
+	set_exec_queue_banned(primary);
+
+	mutex_lock(&group->list_lock);
+	list_for_each_entry(eq, &group->list, multi_queue.link)
+		set_exec_queue_banned(eq);
+	mutex_unlock(&group->list_lock);
+}
+
 #define parallel_read(xe_, map_, field_) \
 	xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \
 			field_)
@@ -575,6 +680,181 @@ static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue
 	xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \
 			field_, val_)
 
+/**
+ * DOC: Multi Queue Group GuC interface
+ *
+ * The multi queue group coordination between KMD and GuC is through a software
+ * construct called Context Group Page (CGP). The CGP is a KMD managed 4KB page
+ * allocated in the global GTT.
+ *
+ * CGP format:
+ *
+ * +-----------+---------------------------+---------------------------------------------+
+ * | DWORD     | Name                      | Description                                 |
+ * +-----------+---------------------------+---------------------------------------------+
+ * | 0         | Version                   | Bits [15:8]=Major ver, [7:0]=Minor ver      |
+ * +-----------+---------------------------+---------------------------------------------+
+ * | 1..15     | RESERVED                  | MBZ                                         |
+ * +-----------+---------------------------+---------------------------------------------+
+ * | 16        | KMD_QUEUE_UPDATE_MASK_DW0 | KMD queue mask for queues 31..0             |
+ * +-----------+---------------------------+---------------------------------------------+
+ * | 17        | KMD_QUEUE_UPDATE_MASK_DW1 | KMD queue mask for queues 63..32            |
+ * +-----------+---------------------------+---------------------------------------------+
+ * | 18..31    | RESERVED                  | MBZ                                         |
+ * +-----------+---------------------------+---------------------------------------------+
+ * | 32        | Q0CD_DW0                  | Queue 0 context LRC descriptor lower DWORD  |
+ * +-----------+---------------------------+---------------------------------------------+
+ * | 33        | Q0ContextIndex            | Context ID for Queue 0                      |
+ * +-----------+---------------------------+---------------------------------------------+
+ * | 34        | Q1CD_DW0                  | Queue 1 context LRC descriptor lower DWORD  |
+ * +-----------+---------------------------+---------------------------------------------+
+ * | 35        | Q1ContextIndex            | Context ID for Queue 1                      |
+ * +-----------+---------------------------+---------------------------------------------+
+ * | ...       |...                        | ...                                         |
+ * +-----------+---------------------------+---------------------------------------------+
+ * | 158       | Q63CD_DW0                 | Queue 63 context LRC descriptor lower DWORD |
+ * +-----------+---------------------------+---------------------------------------------+
+ * | 159       | Q63ContextIndex           | Context ID for Queue 63                     |
+ * +-----------+---------------------------+---------------------------------------------+
+ * | 160..1024 | RESERVED                  | MBZ                                         |
+ * +-----------+---------------------------+---------------------------------------------+
+ *
+ * While registering Q0 with GuC, CGP is updated with Q0 entry and GuC is notified
+ * through XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE H2G message which specifies
+ * the CGP address. When the secondary queues are added to the group, the CGP is
+ * updated with entry for that queue and GuC is notified through the H2G interface
+ * XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC. GuC responds to these H2G messages
+ * with a XE_GUC_ACTION_NOTIFY_MULTIQ_CONTEXT_CGP_SYNC_DONE G2H message. GuC also
+ * sends a XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR notification for any
+ * error in the CGP. Only one of these CGP update messages can be outstanding
+ * (waiting for GuC response) at any time. The bits in KMD_QUEUE_UPDATE_MASK_DW*
+ * fields indicate which queue entry is being updated in the CGP.
+ *
+ * The primary queue (Q0) represents the multi queue group context in GuC and
+ * submission on any queue of the group must be through Q0 GuC interface only.
+ *
+ * As it is not required to register secondary queues with GuC, the secondary queue
+ * context ids in the CGP are populated with Q0 context id.
+ */
+
+#define CGP_VERSION_MAJOR_SHIFT	8
+
+static void xe_guc_exec_queue_group_cgp_update(struct xe_device *xe,
+					       struct xe_exec_queue *q)
+{
+	struct xe_exec_queue_group *group = q->multi_queue.group;
+	u32 guc_id = group->primary->guc->id;
+
+	/* Currently implementing CGP version 1.0 */
+	xe_map_wr(xe, &group->cgp_bo->vmap, 0, u32,
+		  1 << CGP_VERSION_MAJOR_SHIFT);
+
+	xe_map_wr(xe, &group->cgp_bo->vmap,
+		  (32 + q->multi_queue.pos * 2) * sizeof(u32),
+		  u32, lower_32_bits(xe_lrc_descriptor(q->lrc[0])));
+
+	xe_map_wr(xe, &group->cgp_bo->vmap,
+		  (33 + q->multi_queue.pos * 2) * sizeof(u32),
+		  u32, guc_id);
+
+	if (q->multi_queue.pos / 32) {
+		xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32),
+			  u32, BIT(q->multi_queue.pos % 32));
+		xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32), u32, 0);
+	} else {
+		xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32),
+			  u32, BIT(q->multi_queue.pos));
+		xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32), u32, 0);
+	}
+}
+
+static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc,
+					     struct xe_exec_queue *q,
+					     const u32 *action, u32 len)
+{
+	struct xe_exec_queue_group *group = q->multi_queue.group;
+	struct xe_device *xe = guc_to_xe(guc);
+	long ret;
+
+	/*
+	 * As all queues of a multi queue group use single drm scheduler
+	 * submit workqueue, CGP synchronization with GuC are serialized.
+	 * Hence, no locking is required here.
+	 * Wait for any pending CGP_SYNC_DONE response before updating the
+	 * CGP page and sending CGP_SYNC message.
+	 *
+	 * FIXME: Support VF migration
+	 */
+	ret = wait_event_timeout(guc->ct.wq,
+				 !READ_ONCE(group->sync_pending) ||
+				 xe_guc_read_stopped(guc), HZ);
+	if (!ret || xe_guc_read_stopped(guc)) {
+		/* CGP_SYNC failed. Reset gt, cleanup the group */
+		xe_gt_warn(guc_to_gt(guc), "Wait for CGP_SYNC_DONE response failed!\n");
+		set_exec_queue_group_banned(q);
+		xe_gt_reset_async(q->gt);
+		xe_guc_exec_queue_group_trigger_cleanup(q);
+		return;
+	}
+
+	xe_lrc_set_multi_queue_priority(q->lrc[0], q->multi_queue.priority);
+	xe_guc_exec_queue_group_cgp_update(xe, q);
+
+	WRITE_ONCE(group->sync_pending, true);
+	xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_MULTI_QUEUE_CONTEXT, 1);
+}
+
+static void __register_exec_queue_group(struct xe_guc *guc,
+					struct xe_exec_queue *q,
+					struct guc_ctxt_registration_info *info)
+{
+#define MAX_MULTI_QUEUE_REG_SIZE	(8)
+	u32 action[MAX_MULTI_QUEUE_REG_SIZE];
+	int len = 0;
+
+	action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE;
+	action[len++] = info->flags;
+	action[len++] = info->context_idx;
+	action[len++] = info->engine_class;
+	action[len++] = info->engine_submit_mask;
+	action[len++] = 0; /* Reserved */
+	action[len++] = info->cgp_lo;
+	action[len++] = info->cgp_hi;
+
+	xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_REG_SIZE);
+#undef MAX_MULTI_QUEUE_REG_SIZE
+
+	/*
+	 * The above XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE do expect a
+	 * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response
+	 * from guc.
+	 */
+	xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
+}
+
+static void xe_guc_exec_queue_group_add(struct xe_guc *guc,
+					struct xe_exec_queue *q)
+{
+#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE  (2)
+	u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE];
+	int len = 0;
+
+	xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_multi_queue_secondary(q));
+
+	action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC;
+	action[len++] = q->multi_queue.group->primary->guc->id;
+
+	xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE);
+#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE
+
+	/*
+	 * The above XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC do expect a
+	 * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response
+	 * from guc.
+	 */
+	xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
+}
+
 static void __register_mlrc_exec_queue(struct xe_guc *guc,
 				       struct xe_exec_queue *q,
 				       struct guc_ctxt_registration_info *info)
@@ -670,6 +950,13 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type)
 	info.flags = CONTEXT_REGISTRATION_FLAG_KMD |
 		FIELD_PREP(CONTEXT_REGISTRATION_FLAG_TYPE, ctx_type);
 
+	if (xe_exec_queue_is_multi_queue(q)) {
+		struct xe_exec_queue_group *group = q->multi_queue.group;
+
+		info.cgp_lo = xe_bo_ggtt_addr(group->cgp_bo);
+		info.cgp_hi = 0;
+	}
+
 	if (xe_exec_queue_is_parallel(q)) {
 		u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc);
 		struct iosys_map map = xe_lrc_parallel_map(lrc);
@@ -700,11 +987,18 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type)
 
 	set_exec_queue_registered(q);
 	trace_xe_exec_queue_register(q);
-	if (xe_exec_queue_is_parallel(q))
+	if (xe_exec_queue_is_multi_queue_primary(q))
+		__register_exec_queue_group(guc, q, &info);
+	else if (xe_exec_queue_is_parallel(q))
 		__register_mlrc_exec_queue(guc, q, &info);
-	else
+	else if (!xe_exec_queue_is_multi_queue_secondary(q))
 		__register_exec_queue(guc, &info);
-	init_policies(guc, q);
+
+	if (!xe_exec_queue_is_multi_queue_secondary(q))
+		init_policies(guc, q);
+
+	if (xe_exec_queue_is_multi_queue_secondary(q))
+		xe_guc_exec_queue_group_add(guc, q);
 }
 
 static u32 wq_space_until_wrap(struct xe_exec_queue *q)
@@ -712,9 +1006,22 @@ static u32 wq_space_until_wrap(struct xe_exec_queue *q)
 	return (WQ_SIZE - q->guc->wqi_tail);
 }
 
-static bool vf_recovery(struct xe_guc *guc)
+static inline void relaxed_ms_sleep(unsigned int delay_ms)
 {
-	return xe_gt_recovery_pending(guc_to_gt(guc));
+	unsigned long min_us, max_us;
+
+	if (!delay_ms)
+		return;
+
+	if (delay_ms > 20) {
+		msleep(delay_ms);
+		return;
+	}
+
+	min_us = mul_u32_u32(delay_ms, 1000);
+	max_us = min_us + 500;
+
+	usleep_range(min_us, max_us);
 }
 
 static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size)
@@ -722,21 +1029,23 @@ static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size)
 	struct xe_guc *guc = exec_queue_to_guc(q);
 	struct xe_device *xe = guc_to_xe(guc);
 	struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]);
-	unsigned int sleep_period_ms = 1;
+	unsigned int sleep_period_ms = 1, sleep_total_ms = 0;
 
 #define AVAILABLE_SPACE \
 	CIRC_SPACE(q->guc->wqi_tail, q->guc->wqi_head, WQ_SIZE)
 	if (wqi_size > AVAILABLE_SPACE && !vf_recovery(guc)) {
 try_again:
 		q->guc->wqi_head = parallel_read(xe, map, wq_desc.head);
-		if (wqi_size > AVAILABLE_SPACE) {
-			if (sleep_period_ms == 1024) {
+		if (wqi_size > AVAILABLE_SPACE && !vf_recovery(guc)) {
+			if (sleep_total_ms > 2000) {
 				xe_gt_reset_async(q->gt);
 				return -ENODEV;
 			}
 
 			msleep(sleep_period_ms);
-			sleep_period_ms <<= 1;
+			sleep_total_ms += sleep_period_ms;
+			if (sleep_period_ms < 64)
+				sleep_period_ms <<= 1;
 			goto try_again;
 		}
 	}
@@ -825,7 +1134,7 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job)
 	if (!job->restore_replay || job->last_replay) {
 		if (xe_exec_queue_is_parallel(q))
 			wq_item_append(q);
-		else
+		else if (!exec_queue_idle_skip_suspend(q))
 			xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
 		job->last_replay = false;
 	}
@@ -833,6 +1142,12 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job)
 	if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q))
 		return;
 
+	/*
+	 * All queues in a multi-queue group will use the primary queue
+	 * of the group to interface with GuC.
+	 */
+	q = xe_exec_queue_multi_queue_primary(q);
+
 	if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) {
 		action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET;
 		action[len++] = q->guc->id;
@@ -879,6 +1194,18 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job)
 	trace_xe_sched_job_run(job);
 
 	if (!killed_or_banned_or_wedged && !xe_sched_job_is_error(job)) {
+		if (xe_exec_queue_is_multi_queue_secondary(q)) {
+			struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
+
+			if (exec_queue_killed_or_banned_or_wedged(primary)) {
+				killed_or_banned_or_wedged = true;
+				goto run_job_out;
+			}
+
+			if (!exec_queue_registered(primary))
+				register_exec_queue(primary, GUC_CONTEXT_NORMAL);
+		}
+
 		if (!exec_queue_registered(q))
 			register_exec_queue(q, GUC_CONTEXT_NORMAL);
 		if (!job->restore_replay)
@@ -887,6 +1214,7 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job)
 		job->restore_replay = false;
 	}
 
+run_job_out:
 	/*
 	 * We don't care about job-fence ordering in LR VMs because these fences
 	 * are never exported; they are used solely to keep jobs on the pending
@@ -912,6 +1240,11 @@ int xe_guc_read_stopped(struct xe_guc *guc)
 	return atomic_read(&guc->submission_state.stopped);
 }
 
+static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc,
+						    struct xe_exec_queue *q,
+						    u32 runnable_state);
+static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q);
+
 #define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable)			\
 	u32 action[] = {						\
 		XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET,			\
@@ -925,7 +1258,9 @@ static void disable_scheduling_deregister(struct xe_guc *guc,
 	MAKE_SCHED_CONTEXT_ACTION(q, DISABLE);
 	int ret;
 
-	set_min_preemption_timeout(guc, q);
+	if (!xe_exec_queue_is_multi_queue_secondary(q))
+		set_min_preemption_timeout(guc, q);
+
 	smp_rmb();
 	ret = wait_event_timeout(guc->ct.wq,
 				 (!exec_queue_pending_enable(q) &&
@@ -953,23 +1288,12 @@ static void disable_scheduling_deregister(struct xe_guc *guc,
 	 * Reserve space for both G2H here as the 2nd G2H is sent from a G2H
 	 * handler and we are not allowed to reserved G2H space in handlers.
 	 */
-	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
-		       G2H_LEN_DW_SCHED_CONTEXT_MODE_SET +
-		       G2H_LEN_DW_DEREGISTER_CONTEXT, 2);
-}
-
-static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q)
-{
-	struct xe_guc *guc = exec_queue_to_guc(q);
-	struct xe_device *xe = guc_to_xe(guc);
-
-	/** to wakeup xe_wait_user_fence ioctl if exec queue is reset */
-	wake_up_all(&xe->ufence_wq);
-
-	if (xe_exec_queue_is_lr(q))
-		queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr);
+	if (xe_exec_queue_is_multi_queue_secondary(q))
+		handle_multi_queue_secondary_sched_done(guc, q, 0);
 	else
-		xe_sched_tdr_queue_imm(&q->guc->sched);
+		xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
+			       G2H_LEN_DW_SCHED_CONTEXT_MODE_SET +
+			       G2H_LEN_DW_DEREGISTER_CONTEXT, 2);
 }
 
 /**
@@ -1161,8 +1485,11 @@ static void enable_scheduling(struct xe_exec_queue *q)
 	set_exec_queue_enabled(q);
 	trace_xe_exec_queue_scheduling_enable(q);
 
-	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
-		       G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
+	if (xe_exec_queue_is_multi_queue_secondary(q))
+		handle_multi_queue_secondary_sched_done(guc, q, 1);
+	else
+		xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
+			       G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
 
 	ret = wait_event_timeout(guc->ct.wq,
 				 !exec_queue_pending_enable(q) ||
@@ -1186,14 +1513,17 @@ static void disable_scheduling(struct xe_exec_queue *q, bool immediate)
 	xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
 	xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));
 
-	if (immediate)
+	if (immediate && !xe_exec_queue_is_multi_queue_secondary(q))
 		set_min_preemption_timeout(guc, q);
 	clear_exec_queue_enabled(q);
 	set_exec_queue_pending_disable(q);
 	trace_xe_exec_queue_scheduling_disable(q);
 
-	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
-		       G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
+	if (xe_exec_queue_is_multi_queue_secondary(q))
+		handle_multi_queue_secondary_sched_done(guc, q, 0);
+	else
+		xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
+			       G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
 }
 
 static void __deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q)
@@ -1211,8 +1541,11 @@ static void __deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q)
 	set_exec_queue_destroyed(q);
 	trace_xe_exec_queue_deregister(q);
 
-	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
-		       G2H_LEN_DW_DEREGISTER_CONTEXT, 1);
+	if (xe_exec_queue_is_multi_queue_secondary(q))
+		handle_deregister_done(guc, q);
+	else
+		xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
+			       G2H_LEN_DW_DEREGISTER_CONTEXT, 1);
 }
 
 static enum drm_gpu_sched_stat
@@ -1225,7 +1558,6 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 	struct xe_guc *guc = exec_queue_to_guc(q);
 	const char *process_name = "no process";
 	struct xe_device *xe = guc_to_xe(guc);
-	unsigned int fw_ref;
 	int err = -ETIME;
 	pid_t pid = -1;
 	int i = 0;
@@ -1251,6 +1583,19 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 		exec_queue_killed_or_banned_or_wedged(q) ||
 		exec_queue_destroyed(q);
 
+	/* Skip timeout check if multi-queue group is banned */
+	if (xe_exec_queue_is_multi_queue(q) &&
+	    READ_ONCE(q->multi_queue.group->banned))
+		skip_timeout_check = true;
+
+	/*
+	 * FIXME: In multi-queue scenario, the TDR must ensure that the whole
+	 * multi-queue group is off the HW before signaling the fences to avoid
+	 * possible memory corruptions. This means disabling scheduling on the
+	 * primary queue before or during the secondary queue's TDR. Need to
+	 * implement this in least obtrusive way.
+	 */
+
 	/*
 	 * If devcoredump not captured and GuC capture for the job is not ready
 	 * do manual capture first and decide later if we need to use it
@@ -1258,13 +1603,11 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 	if (!exec_queue_killed(q) && !xe->devcoredump.captured &&
 	    !xe_guc_capture_get_matching_and_lock(q)) {
 		/* take force wake before engine register manual capture */
-		fw_ref = xe_force_wake_get(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
-		if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
+		CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
+		if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
 			xe_gt_info(q->gt, "failed to get forcewake for coredump capture\n");
 
 		xe_engine_snapshot_capture_for_queue(q);
-
-		xe_force_wake_put(gt_to_fw(q->gt), fw_ref);
 	}
 
 	/*
@@ -1405,7 +1748,10 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 	xe_sched_add_pending_job(sched, job);
 	xe_sched_submission_start(sched);
 
-	xe_guc_exec_queue_trigger_cleanup(q);
+	if (xe_exec_queue_is_multi_queue(q))
+		xe_guc_exec_queue_group_trigger_cleanup(q);
+	else
+		xe_guc_exec_queue_trigger_cleanup(q);
 
 	/* Mark all outstanding jobs as bad, thus completing them */
 	spin_lock(&sched->base.job_list_lock);
@@ -1455,17 +1801,23 @@ static void __guc_exec_queue_destroy_async(struct work_struct *w)
 	struct xe_exec_queue *q = ge->q;
 	struct xe_guc *guc = exec_queue_to_guc(q);
 
-	xe_pm_runtime_get(guc_to_xe(guc));
+	guard(xe_pm_runtime)(guc_to_xe(guc));
 	trace_xe_exec_queue_destroy(q);
 
+	if (xe_exec_queue_is_multi_queue_secondary(q)) {
+		struct xe_exec_queue_group *group = q->multi_queue.group;
+
+		mutex_lock(&group->list_lock);
+		list_del(&q->multi_queue.link);
+		mutex_unlock(&group->list_lock);
+	}
+
 	if (xe_exec_queue_is_lr(q))
 		cancel_work_sync(&ge->lr_tdr);
 	/* Confirm no work left behind accessing device structures */
 	cancel_delayed_work_sync(&ge->sched.base.work_tdr);
 
 	xe_exec_queue_fini(q);
-
-	xe_pm_runtime_put(guc_to_xe(guc));
 }
 
 static void guc_exec_queue_destroy_async(struct xe_exec_queue *q)
@@ -1570,9 +1922,10 @@ static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg)
 {
 	struct xe_exec_queue *q = msg->private_data;
 	struct xe_guc *guc = exec_queue_to_guc(q);
+	bool idle_skip_suspend = xe_exec_queue_idle_skip_suspend(q);
 
-	if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) &&
-	    exec_queue_enabled(q)) {
+	if (!idle_skip_suspend && guc_exec_queue_allowed_to_change_state(q) &&
+	    !exec_queue_suspended(q) && exec_queue_enabled(q)) {
 		wait_event(guc->ct.wq, vf_recovery(guc) ||
 			   ((q->guc->resume_time != RESUME_PENDING ||
 			   xe_guc_read_stopped(guc)) && !exec_queue_pending_disable(q)));
@@ -1585,17 +1938,39 @@ static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg)
 				since_resume_ms;
 
 			if (wait_ms > 0 && q->guc->resume_time)
-				msleep(wait_ms);
+				relaxed_ms_sleep(wait_ms);
 
 			set_exec_queue_suspended(q);
 			disable_scheduling(q, false);
 		}
 	} else if (q->guc->suspend_pending) {
+		if (idle_skip_suspend)
+			set_exec_queue_idle_skip_suspend(q);
 		set_exec_queue_suspended(q);
 		suspend_fence_signal(q);
 	}
 }
 
+static void sched_context(struct xe_exec_queue *q)
+{
+	struct xe_guc *guc = exec_queue_to_guc(q);
+	struct xe_lrc *lrc = q->lrc[0];
+	u32 action[] = {
+		XE_GUC_ACTION_SCHED_CONTEXT,
+		q->guc->id,
+	};
+
+	xe_gt_assert(guc_to_gt(guc), !xe_exec_queue_is_parallel(q));
+	xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q));
+	xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
+	xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));
+
+	trace_xe_exec_queue_submit(q);
+
+	xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
+	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);
+}
+
 static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg)
 {
 	struct xe_exec_queue *q = msg->private_data;
@@ -1603,19 +1978,53 @@ static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg)
 	if (guc_exec_queue_allowed_to_change_state(q)) {
 		clear_exec_queue_suspended(q);
 		if (!exec_queue_enabled(q)) {
+			if (exec_queue_idle_skip_suspend(q)) {
+				struct xe_lrc *lrc = q->lrc[0];
+
+				clear_exec_queue_idle_skip_suspend(q);
+				xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
+			}
 			q->guc->resume_time = RESUME_PENDING;
 			set_exec_queue_pending_resume(q);
 			enable_scheduling(q);
+		} else if (exec_queue_idle_skip_suspend(q)) {
+			clear_exec_queue_idle_skip_suspend(q);
+			sched_context(q);
 		}
 	} else {
 		clear_exec_queue_suspended(q);
+		clear_exec_queue_idle_skip_suspend(q);
 	}
 }
 
-#define CLEANUP		1	/* Non-zero values to catch uninitialized msg */
-#define SET_SCHED_PROPS	2
-#define SUSPEND		3
-#define RESUME		4
+static void __guc_exec_queue_process_msg_set_multi_queue_priority(struct xe_sched_msg *msg)
+{
+	struct xe_exec_queue *q = msg->private_data;
+
+	if (guc_exec_queue_allowed_to_change_state(q)) {
+#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE        (2)
+		struct xe_guc *guc = exec_queue_to_guc(q);
+		struct xe_exec_queue_group *group = q->multi_queue.group;
+		u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE];
+		int len = 0;
+
+		action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC;
+		action[len++] = group->primary->guc->id;
+
+		xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE);
+#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE
+
+		xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
+	}
+
+	kfree(msg);
+}
+
+#define CLEANUP				1	/* Non-zero values to catch uninitialized msg */
+#define SET_SCHED_PROPS			2
+#define SUSPEND				3
+#define RESUME				4
+#define SET_MULTI_QUEUE_PRIORITY	5
 #define OPCODE_MASK	0xf
 #define MSG_LOCKED	BIT(8)
 #define MSG_HEAD	BIT(9)
@@ -1639,6 +2048,9 @@ static void guc_exec_queue_process_msg(struct xe_sched_msg *msg)
 	case RESUME:
 		__guc_exec_queue_process_msg_resume(msg);
 		break;
+	case SET_MULTI_QUEUE_PRIORITY:
+		__guc_exec_queue_process_msg_set_multi_queue_priority(msg);
+		break;
 	default:
 		XE_WARN_ON("Unknown message type");
 	}
@@ -1660,6 +2072,7 @@ static int guc_exec_queue_init(struct xe_exec_queue *q)
 {
 	struct xe_gpu_scheduler *sched;
 	struct xe_guc *guc = exec_queue_to_guc(q);
+	struct workqueue_struct *submit_wq = NULL;
 	struct xe_guc_exec_queue *ge;
 	long timeout;
 	int err, i;
@@ -1680,8 +2093,20 @@ static int guc_exec_queue_init(struct xe_exec_queue *q)
 
 	timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT :
 		  msecs_to_jiffies(q->sched_props.job_timeout_ms);
+
+	/*
+	 * Use primary queue's submit_wq for all secondary queues of a
+	 * multi queue group. This serialization avoids any locking around
+	 * CGP synchronization with GuC.
+	 */
+	if (xe_exec_queue_is_multi_queue_secondary(q)) {
+		struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
+
+		submit_wq = primary->guc->sched.base.submit_wq;
+	}
+
 	err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops,
-			    NULL, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64,
+			    submit_wq, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64,
 			    timeout, guc_to_gt(guc)->ordered_wq, NULL,
 			    q->name, gt_to_xe(q->gt)->drm.dev);
 	if (err)
@@ -1710,7 +2135,23 @@ static int guc_exec_queue_init(struct xe_exec_queue *q)
 
 	xe_exec_queue_assign_name(q, q->guc->id);
 
-	trace_xe_exec_queue_create(q);
+	/*
+	 * Maintain secondary queues of the multi queue group in a list
+	 * for handling dependencies across the queues in the group.
+	 */
+	if (xe_exec_queue_is_multi_queue_secondary(q)) {
+		struct xe_exec_queue_group *group = q->multi_queue.group;
+
+		INIT_LIST_HEAD(&q->multi_queue.link);
+		mutex_lock(&group->list_lock);
+		list_add_tail(&q->multi_queue.link, &group->list);
+		mutex_unlock(&group->list_lock);
+	}
+
+	if (xe_exec_queue_is_multi_queue(q))
+		trace_xe_exec_queue_create_multi_queue(q);
+	else
+		trace_xe_exec_queue_create(q);
 
 	return 0;
 
@@ -1842,6 +2283,27 @@ static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q,
 	return 0;
 }
 
+static int guc_exec_queue_set_multi_queue_priority(struct xe_exec_queue *q,
+						   enum xe_multi_queue_priority priority)
+{
+	struct xe_sched_msg *msg;
+
+	xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), xe_exec_queue_is_multi_queue(q));
+
+	if (q->multi_queue.priority == priority ||
+	    exec_queue_killed_or_banned_or_wedged(q))
+		return 0;
+
+	msg = kmalloc(sizeof(*msg), GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;
+
+	q->multi_queue.priority = priority;
+	guc_exec_queue_add_msg(q, msg, SET_MULTI_QUEUE_PRIORITY);
+
+	return 0;
+}
+
 static int guc_exec_queue_suspend(struct xe_exec_queue *q)
 {
 	struct xe_gpu_scheduler *sched = &q->guc->sched;
@@ -1916,6 +2378,10 @@ static void guc_exec_queue_resume(struct xe_exec_queue *q)
 
 static bool guc_exec_queue_reset_status(struct xe_exec_queue *q)
 {
+	if (xe_exec_queue_is_multi_queue_secondary(q) &&
+	    guc_exec_queue_reset_status(xe_exec_queue_multi_queue_primary(q)))
+		return true;
+
 	return exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q);
 }
 
@@ -1933,6 +2399,7 @@ static const struct xe_exec_queue_ops guc_exec_queue_ops = {
 	.set_priority = guc_exec_queue_set_priority,
 	.set_timeslice = guc_exec_queue_set_timeslice,
 	.set_preempt_timeout = guc_exec_queue_set_preempt_timeout,
+	.set_multi_queue_priority = guc_exec_queue_set_multi_queue_priority,
 	.suspend = guc_exec_queue_suspend,
 	.suspend_wait = guc_exec_queue_suspend_wait,
 	.resume = guc_exec_queue_resume,
@@ -2182,6 +2649,22 @@ void xe_guc_submit_pause(struct xe_guc *guc)
 	struct xe_exec_queue *q;
 	unsigned long index;
 
+	mutex_lock(&guc->submission_state.lock);
+	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
+		xe_sched_submission_stop(&q->guc->sched);
+	mutex_unlock(&guc->submission_state.lock);
+}
+
+/**
+ * xe_guc_submit_pause_vf - Stop further runs of submission tasks for VF.
+ * @guc: the &xe_guc struct instance whose scheduler is to be disabled
+ */
+void xe_guc_submit_pause_vf(struct xe_guc *guc)
+{
+	struct xe_exec_queue *q;
+	unsigned long index;
+
+	xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc)));
 	xe_gt_assert(guc_to_gt(guc), vf_recovery(guc));
 
 	mutex_lock(&guc->submission_state.lock);
@@ -2253,10 +2736,11 @@ static void guc_exec_queue_unpause_prepare(struct xe_guc *guc,
 					   struct xe_exec_queue *q)
 {
 	struct xe_gpu_scheduler *sched = &q->guc->sched;
-	struct xe_sched_job *job = NULL;
+	struct xe_sched_job *job = NULL, *__job;
 	bool restore_replay = false;
 
-	list_for_each_entry(job, &sched->base.pending_list, drm.list) {
+	list_for_each_entry(__job, &sched->base.pending_list, drm.list) {
+		job = __job;
 		restore_replay |= job->restore_replay;
 		if (restore_replay) {
 			xe_gt_dbg(guc_to_gt(guc), "Replay JOB - guc_id=%d, seqno=%d",
@@ -2272,14 +2756,15 @@ static void guc_exec_queue_unpause_prepare(struct xe_guc *guc,
 }
 
 /**
- * xe_guc_submit_unpause_prepare - Prepare unpause submission tasks on given GuC.
+ * xe_guc_submit_unpause_prepare_vf - Prepare unpause submission tasks for VF.
  * @guc: the &xe_guc struct instance whose scheduler is to be prepared for unpause
  */
-void xe_guc_submit_unpause_prepare(struct xe_guc *guc)
+void xe_guc_submit_unpause_prepare_vf(struct xe_guc *guc)
 {
 	struct xe_exec_queue *q;
 	unsigned long index;
 
+	xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc)));
 	xe_gt_assert(guc_to_gt(guc), vf_recovery(guc));
 
 	mutex_lock(&guc->submission_state.lock);
@@ -2355,6 +2840,23 @@ void xe_guc_submit_unpause(struct xe_guc *guc)
 	struct xe_exec_queue *q;
 	unsigned long index;
 
+	mutex_lock(&guc->submission_state.lock);
+	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
+		xe_sched_submission_start(&q->guc->sched);
+	mutex_unlock(&guc->submission_state.lock);
+}
+
+/**
+ * xe_guc_submit_unpause_vf - Allow further runs of submission tasks for VF.
+ * @guc: the &xe_guc struct instance whose scheduler is to be enabled
+ */
+void xe_guc_submit_unpause_vf(struct xe_guc *guc)
+{
+	struct xe_exec_queue *q;
+	unsigned long index;
+
+	xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc)));
+
 	mutex_lock(&guc->submission_state.lock);
 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
 		/*
@@ -2431,7 +2933,11 @@ static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q)
 
 	trace_xe_exec_queue_deregister(q);
 
-	xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action));
+	if (xe_exec_queue_is_multi_queue_secondary(q))
+		handle_deregister_done(guc, q);
+	else
+		xe_guc_ct_send_g2h_handler(&guc->ct, action,
+					   ARRAY_SIZE(action));
 }
 
 static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q,
@@ -2481,6 +2987,16 @@ static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q,
 	}
 }
 
+static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc,
+						    struct xe_exec_queue *q,
+						    u32 runnable_state)
+{
+	/* Take CT lock here as handle_sched_done() do send a h2g message */
+	mutex_lock(&guc->ct.lock);
+	handle_sched_done(guc, q, runnable_state);
+	mutex_unlock(&guc->ct.lock);
+}
+
 int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
 {
 	struct xe_exec_queue *q;
@@ -2575,9 +3091,7 @@ int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len)
 	 * jobs by setting timeout of the job to the minimum value kicking
 	 * guc_exec_queue_timedout_job.
 	 */
-	set_exec_queue_reset(q);
-	if (!exec_queue_banned(q) && !exec_queue_check_timeout(q))
-		xe_guc_exec_queue_trigger_cleanup(q);
+	xe_guc_exec_queue_reset_trigger_cleanup(q);
 
 	return 0;
 }
@@ -2656,9 +3170,7 @@ int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg,
 	trace_xe_exec_queue_memory_cat_error(q);
 
 	/* Treat the same as engine reset */
-	set_exec_queue_reset(q);
-	if (!exec_queue_banned(q) && !exec_queue_check_timeout(q))
-		xe_guc_exec_queue_trigger_cleanup(q);
+	xe_guc_exec_queue_reset_trigger_cleanup(q);
 
 	return 0;
 }
@@ -2685,6 +3197,73 @@ int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 le
 	return 0;
 }
 
+int xe_guc_exec_queue_cgp_context_error_handler(struct xe_guc *guc, u32 *msg,
+						u32 len)
+{
+	struct xe_gt *gt = guc_to_gt(guc);
+	struct xe_device *xe = guc_to_xe(guc);
+	struct xe_exec_queue *q;
+	u32 guc_id = msg[2];
+
+	if (unlikely(len != XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN)) {
+		drm_err(&xe->drm, "Invalid length %u", len);
+		return -EPROTO;
+	}
+
+	q = g2h_exec_queue_lookup(guc, guc_id);
+	if (unlikely(!q))
+		return -EPROTO;
+
+	xe_gt_dbg(gt,
+		  "CGP context error: [%s] err=0x%x, q0_id=0x%x LRCA=0x%x guc_id=0x%x",
+		  msg[0] & 1 ? "uc" : "kmd", msg[1], msg[2], msg[3], msg[4]);
+
+	trace_xe_exec_queue_cgp_context_error(q);
+
+	/* Treat the same as engine reset */
+	xe_guc_exec_queue_reset_trigger_cleanup(q);
+
+	return 0;
+}
+
+/**
+ * xe_guc_exec_queue_cgp_sync_done_handler - CGP synchronization done handler
+ * @guc: guc
+ * @msg: message indicating CGP sync done
+ * @len: length of message
+ *
+ * Set multi queue group's sync_pending flag to false and wakeup anyone waiting
+ * for CGP synchronization to complete.
+ *
+ * Return: 0 on success, -EPROTO for malformed messages.
+ */
+int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
+{
+	struct xe_device *xe = guc_to_xe(guc);
+	struct xe_exec_queue *q;
+	u32 guc_id = msg[0];
+
+	if (unlikely(len < 1)) {
+		drm_err(&xe->drm, "Invalid CGP_SYNC_DONE length %u", len);
+		return -EPROTO;
+	}
+
+	q = g2h_exec_queue_lookup(guc, guc_id);
+	if (unlikely(!q))
+		return -EPROTO;
+
+	if (!xe_exec_queue_is_multi_queue_primary(q)) {
+		drm_err(&xe->drm, "Unexpected CGP_SYNC_DONE response");
+		return -EPROTO;
+	}
+
+	/* Wakeup the serialized cgp update wait */
+	WRITE_ONCE(q->multi_queue.group->sync_pending, false);
+	xe_guc_ct_wake_waiters(&guc->ct);
+
+	return 0;
+}
+
 static void
 guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q,
 				   struct xe_guc_submit_exec_queue_snapshot *snapshot)
@@ -2784,6 +3363,11 @@ xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q)
 	if (snapshot->parallel_execution)
 		guc_exec_queue_wq_snapshot_capture(q, snapshot);
 
+	if (xe_exec_queue_is_multi_queue(q)) {
+		snapshot->multi_queue.valid = true;
+		snapshot->multi_queue.primary = xe_exec_queue_multi_queue_primary(q)->guc->id;
+		snapshot->multi_queue.pos = q->multi_queue.pos;
+	}
 	spin_lock(&sched->base.job_list_lock);
 	snapshot->pending_list_size = list_count_nodes(&sched->base.pending_list);
 	snapshot->pending_list = kmalloc_array(snapshot->pending_list_size,
@@ -2866,6 +3450,11 @@ xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snaps
 	if (snapshot->parallel_execution)
 		guc_exec_queue_wq_snapshot_print(snapshot, p);
 
+	if (snapshot->multi_queue.valid) {
+		drm_printf(p, "\tMulti queue primary GuC ID: %d\n", snapshot->multi_queue.primary);
+		drm_printf(p, "\tMulti queue position: %d\n", snapshot->multi_queue.pos);
+	}
+
 	for (i = 0; snapshot->pending_list && i < snapshot->pending_list_size;
 	     i++)
 		drm_printf(p, "\tJob: seqno=%d, fence=%d, finished=%d\n",
diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h
index b49a2748ec46..4d89b2975fe9 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit.h
+++ b/drivers/gpu/drm/xe/xe_guc_submit.h
@@ -21,9 +21,11 @@ void xe_guc_submit_reset_wait(struct xe_guc *guc);
 void xe_guc_submit_stop(struct xe_guc *guc);
 int xe_guc_submit_start(struct xe_guc *guc);
 void xe_guc_submit_pause(struct xe_guc *guc);
-void xe_guc_submit_unpause(struct xe_guc *guc);
-void xe_guc_submit_unpause_prepare(struct xe_guc *guc);
 void xe_guc_submit_pause_abort(struct xe_guc *guc);
+void xe_guc_submit_pause_vf(struct xe_guc *guc);
+void xe_guc_submit_unpause(struct xe_guc *guc);
+void xe_guc_submit_unpause_vf(struct xe_guc *guc);
+void xe_guc_submit_unpause_prepare_vf(struct xe_guc *guc);
 void xe_guc_submit_wedge(struct xe_guc *guc);
 
 int xe_guc_read_stopped(struct xe_guc *guc);
@@ -34,6 +36,9 @@ int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg,
 					       u32 len);
 int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len);
 int xe_guc_error_capture_handler(struct xe_guc *guc, u32 *msg, u32 len);
+int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc *guc, u32 *msg, u32 len);
+int xe_guc_exec_queue_cgp_context_error_handler(struct xe_guc *guc, u32 *msg,
+						u32 len);
 
 struct xe_guc_submit_exec_queue_snapshot *
 xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q);
diff --git a/drivers/gpu/drm/xe/xe_guc_submit_types.h b/drivers/gpu/drm/xe/xe_guc_submit_types.h
index dc7456c34583..25e29e85502c 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_submit_types.h
@@ -135,6 +135,19 @@ struct xe_guc_submit_exec_queue_snapshot {
 		u32 wq[WQ_SIZE / sizeof(u32)];
 	} parallel;
 
+	/** @multi_queue: snapshot of the multi queue information */
+	struct {
+		/**
+		 * @multi_queue.primary: GuC id of the primary exec queue
+		 * of the multi queue group.
+		 */
+		u32 primary;
+		/** @multi_queue.pos: Position of the exec queue within the multi queue group */
+		u8 pos;
+		/** @valid: The exec queue is part of a multi queue group */
+		bool valid;
+	} multi_queue;
+
 	/** @pending_list_size: Size of the pending list snapshot array */
 	int pending_list_size;
 	/** @pending_list: snapshot of the pending list info */
diff --git a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
index a80175c7c478..6532a88d51e2 100644
--- a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
+++ b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
@@ -13,6 +13,7 @@
 #include "xe_guc_tlb_inval.h"
 #include "xe_force_wake.h"
 #include "xe_mmio.h"
+#include "xe_sa.h"
 #include "xe_tlb_inval.h"
 
 #include "regs/xe_guc_regs.h"
@@ -34,9 +35,12 @@ static int send_tlb_inval(struct xe_guc *guc, const u32 *action, int len)
 			      G2H_LEN_DW_TLB_INVALIDATE, 1);
 }
 
-#define MAKE_INVAL_OP(type)	((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \
+#define MAKE_INVAL_OP_FLUSH(type, flush_cache)	((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \
 		XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT | \
-		XE_GUC_TLB_INVAL_FLUSH_CACHE)
+		(flush_cache ? \
+		XE_GUC_TLB_INVAL_FLUSH_CACHE : 0))
+
+#define MAKE_INVAL_OP(type)	MAKE_INVAL_OP_FLUSH(type, true)
 
 static int send_tlb_inval_all(struct xe_tlb_inval *tlb_inval, u32 seqno)
 {
@@ -71,12 +75,11 @@ static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno)
 		return send_tlb_inval(guc, action, ARRAY_SIZE(action));
 	} else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) {
 		struct xe_mmio *mmio = &gt->mmio;
-		unsigned int fw_ref;
 
 		if (IS_SRIOV_VF(xe))
 			return -ECANCELED;
 
-		fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+		CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
 		if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) {
 			xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC1,
 					PVC_GUC_TLB_INV_DESC1_INVALIDATE);
@@ -86,12 +89,25 @@ static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno)
 			xe_mmio_write32(mmio, GUC_TLB_INV_CR,
 					GUC_TLB_INV_CR_INVALIDATE);
 		}
-		xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	}
 
 	return -ECANCELED;
 }
 
+static int send_page_reclaim(struct xe_guc *guc, u32 seqno,
+			     u64 gpu_addr)
+{
+	u32 action[] = {
+		XE_GUC_ACTION_PAGE_RECLAMATION,
+		seqno,
+		lower_32_bits(gpu_addr),
+		upper_32_bits(gpu_addr),
+	};
+
+	return xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
+			      G2H_LEN_DW_PAGE_RECLAMATION, 1);
+}
+
 /*
  * Ensure that roundup_pow_of_two(length) doesn't overflow.
  * Note that roundup_pow_of_two() operates on unsigned long,
@@ -100,20 +116,21 @@ static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno)
 #define MAX_RANGE_TLB_INVALIDATION_LENGTH (rounddown_pow_of_two(ULONG_MAX))
 
 static int send_tlb_inval_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno,
-				u64 start, u64 end, u32 asid)
+				u64 start, u64 end, u32 asid,
+				struct drm_suballoc *prl_sa)
 {
 #define MAX_TLB_INVALIDATION_LEN	7
 	struct xe_guc *guc = tlb_inval->private;
 	struct xe_gt *gt = guc_to_gt(guc);
 	u32 action[MAX_TLB_INVALIDATION_LEN];
 	u64 length = end - start;
-	int len = 0;
+	int len = 0, err;
 
 	if (guc_to_xe(guc)->info.force_execlist)
 		return -ECANCELED;
 
 	action[len++] = XE_GUC_ACTION_TLB_INVALIDATION;
-	action[len++] = seqno;
+	action[len++] = !prl_sa ? seqno : TLB_INVALIDATION_SEQNO_INVALID;
 	if (!gt_to_xe(gt)->info.has_range_tlb_inval ||
 	    length > MAX_RANGE_TLB_INVALIDATION_LENGTH) {
 		action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL);
@@ -154,7 +171,8 @@ static int send_tlb_inval_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno,
 						    ilog2(SZ_2M) + 1)));
 		xe_gt_assert(gt, IS_ALIGNED(start, length));
 
-		action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_PAGE_SELECTIVE);
+		/* Flush on NULL case, Media is not required to modify flush due to no PPC so NOP */
+		action[len++] = MAKE_INVAL_OP_FLUSH(XE_GUC_TLB_INVAL_PAGE_SELECTIVE, !prl_sa);
 		action[len++] = asid;
 		action[len++] = lower_32_bits(start);
 		action[len++] = upper_32_bits(start);
@@ -163,7 +181,10 @@ static int send_tlb_inval_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno,
 
 	xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN);
 
-	return send_tlb_inval(guc, action, len);
+	err = send_tlb_inval(guc, action, len);
+	if (!err && prl_sa)
+		err = send_page_reclaim(guc, seqno, xe_sa_bo_gpu_addr(prl_sa));
+	return err;
 }
 
 static bool tlb_inval_initialized(struct xe_tlb_inval *tlb_inval)
diff --git a/drivers/gpu/drm/xe/xe_heci_gsc.c b/drivers/gpu/drm/xe/xe_heci_gsc.c
index 2b3d49dd394c..495cdd4f948d 100644
--- a/drivers/gpu/drm/xe/xe_heci_gsc.c
+++ b/drivers/gpu/drm/xe/xe_heci_gsc.c
@@ -223,7 +223,7 @@ void xe_heci_gsc_irq_handler(struct xe_device *xe, u32 iir)
 	if (xe->heci_gsc.irq < 0)
 		return;
 
-	ret = generic_handle_irq(xe->heci_gsc.irq);
+	ret = generic_handle_irq_safe(xe->heci_gsc.irq);
 	if (ret)
 		drm_err_ratelimited(&xe->drm, "error handling GSC irq: %d\n", ret);
 }
@@ -243,7 +243,7 @@ void xe_heci_csc_irq_handler(struct xe_device *xe, u32 iir)
 	if (xe->heci_gsc.irq < 0)
 		return;
 
-	ret = generic_handle_irq(xe->heci_gsc.irq);
+	ret = generic_handle_irq_safe(xe->heci_gsc.irq);
 	if (ret)
 		drm_err_ratelimited(&xe->drm, "error handling GSC irq: %d\n", ret);
 }
diff --git a/drivers/gpu/drm/xe/xe_huc.c b/drivers/gpu/drm/xe/xe_huc.c
index 0a70c8924582..4212162913af 100644
--- a/drivers/gpu/drm/xe/xe_huc.c
+++ b/drivers/gpu/drm/xe/xe_huc.c
@@ -300,19 +300,16 @@ void xe_huc_sanitize(struct xe_huc *huc)
 void xe_huc_print_info(struct xe_huc *huc, struct drm_printer *p)
 {
 	struct xe_gt *gt = huc_to_gt(huc);
-	unsigned int fw_ref;
 
 	xe_uc_fw_print(&huc->fw, p);
 
 	if (!xe_uc_fw_is_enabled(&huc->fw))
 		return;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!fw_ref.domains)
 		return;
 
 	drm_printf(p, "\nHuC status: 0x%08x\n",
 		   xe_mmio_read32(&gt->mmio, HUC_KERNEL_LOAD_INFO));
-
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
diff --git a/drivers/gpu/drm/xe/xe_huc_debugfs.c b/drivers/gpu/drm/xe/xe_huc_debugfs.c
index 3a888a40188b..df9c4d79b710 100644
--- a/drivers/gpu/drm/xe/xe_huc_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_huc_debugfs.c
@@ -37,9 +37,8 @@ static int huc_info(struct seq_file *m, void *data)
 	struct xe_device *xe = huc_to_xe(huc);
 	struct drm_printer p = drm_seq_file_printer(m);
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	xe_huc_print_info(huc, &p);
-	xe_pm_runtime_put(xe);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c b/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c
index 640950172088..cb45cdceef67 100644
--- a/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c
+++ b/drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c
@@ -43,16 +43,14 @@ static ssize_t xe_hw_engine_class_sysfs_attr_show(struct kobject *kobj,
 {
 	struct xe_device *xe = kobj_to_xe(kobj);
 	struct kobj_attribute *kattr;
-	ssize_t ret = -EIO;
 
 	kattr = container_of(attr, struct kobj_attribute, attr);
 	if (kattr->show) {
-		xe_pm_runtime_get(xe);
-		ret = kattr->show(kobj, kattr, buf);
-		xe_pm_runtime_put(xe);
+		guard(xe_pm_runtime)(xe);
+		return kattr->show(kobj, kattr, buf);
 	}
 
-	return ret;
+	return -EIO;
 }
 
 static ssize_t xe_hw_engine_class_sysfs_attr_store(struct kobject *kobj,
@@ -62,16 +60,14 @@ static ssize_t xe_hw_engine_class_sysfs_attr_store(struct kobject *kobj,
 {
 	struct xe_device *xe = kobj_to_xe(kobj);
 	struct kobj_attribute *kattr;
-	ssize_t ret = -EIO;
 
 	kattr = container_of(attr, struct kobj_attribute, attr);
 	if (kattr->store) {
-		xe_pm_runtime_get(xe);
-		ret = kattr->store(kobj, kattr, buf, count);
-		xe_pm_runtime_put(xe);
+		guard(xe_pm_runtime)(xe);
+		return kattr->store(kobj, kattr, buf, count);
 	}
 
-	return ret;
+	return -EIO;
 }
 
 static const struct sysfs_ops xe_hw_engine_class_sysfs_ops = {
diff --git a/drivers/gpu/drm/xe/xe_hw_engine_group.c b/drivers/gpu/drm/xe/xe_hw_engine_group.c
index fa4db5f23342..f69a32c27458 100644
--- a/drivers/gpu/drm/xe/xe_hw_engine_group.c
+++ b/drivers/gpu/drm/xe/xe_hw_engine_group.c
@@ -9,7 +9,9 @@
 #include "xe_device.h"
 #include "xe_exec_queue.h"
 #include "xe_gt.h"
+#include "xe_gt_stats.h"
 #include "xe_hw_engine_group.h"
+#include "xe_sync.h"
 #include "xe_vm.h"
 
 static void
@@ -20,7 +22,8 @@ hw_engine_group_resume_lr_jobs_func(struct work_struct *w)
 	int err;
 	enum xe_hw_engine_group_execution_mode previous_mode;
 
-	err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode);
+	err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode,
+					  NULL, 0);
 	if (err)
 		return;
 
@@ -188,23 +191,39 @@ void xe_hw_engine_group_resume_faulting_lr_jobs(struct xe_hw_engine_group *group
 /**
  * xe_hw_engine_group_suspend_faulting_lr_jobs() - Suspend the faulting LR jobs of this group
  * @group: The hw engine group
+ * @has_deps: dma-fence job triggering suspend has dependencies
  *
  * Return: 0 on success, negative error code on error.
  */
-static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group)
+static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group,
+						       bool has_deps)
 {
 	int err;
 	struct xe_exec_queue *q;
+	struct xe_gt *gt = NULL;
 	bool need_resume = false;
+	ktime_t start = xe_gt_stats_ktime_get();
 
 	lockdep_assert_held_write(&group->mode_sem);
 
 	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
+		bool idle_skip_suspend;
+
 		if (!xe_vm_in_fault_mode(q->vm))
 			continue;
 
-		need_resume = true;
+		idle_skip_suspend = xe_exec_queue_idle_skip_suspend(q);
+		if (!idle_skip_suspend && has_deps)
+			return -EAGAIN;
+
+		xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, 1);
+		if (idle_skip_suspend)
+			xe_gt_stats_incr(q->gt,
+					 XE_GT_STATS_ID_HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT, 1);
+
+		need_resume |= !idle_skip_suspend;
 		q->ops->suspend(q);
+		gt = q->gt;
 	}
 
 	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
@@ -216,6 +235,12 @@ static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group
 			return err;
 	}
 
+	if (gt) {
+		xe_gt_stats_incr(gt,
+				 XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US,
+				 xe_gt_stats_ktime_us_delta(start));
+	}
+
 	if (need_resume)
 		xe_hw_engine_group_resume_faulting_lr_jobs(group);
 
@@ -236,7 +261,9 @@ static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group
 {
 	long timeout;
 	struct xe_exec_queue *q;
+	struct xe_gt *gt = NULL;
 	struct dma_fence *fence;
+	ktime_t start = xe_gt_stats_ktime_get();
 
 	lockdep_assert_held_write(&group->mode_sem);
 
@@ -244,18 +271,26 @@ static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group
 		if (xe_vm_in_lr_mode(q->vm))
 			continue;
 
+		xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, 1);
 		fence = xe_exec_queue_last_fence_get_for_resume(q, q->vm);
 		timeout = dma_fence_wait(fence, false);
 		dma_fence_put(fence);
+		gt = q->gt;
 
 		if (timeout < 0)
 			return -ETIME;
 	}
 
+	if (gt) {
+		xe_gt_stats_incr(gt,
+				 XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US,
+				 xe_gt_stats_ktime_us_delta(start));
+	}
+
 	return 0;
 }
 
-static int switch_mode(struct xe_hw_engine_group *group)
+static int switch_mode(struct xe_hw_engine_group *group, bool has_deps)
 {
 	int err = 0;
 	enum xe_hw_engine_group_execution_mode new_mode;
@@ -265,7 +300,8 @@ static int switch_mode(struct xe_hw_engine_group *group)
 	switch (group->cur_mode) {
 	case EXEC_MODE_LR:
 		new_mode = EXEC_MODE_DMA_FENCE;
-		err = xe_hw_engine_group_suspend_faulting_lr_jobs(group);
+		err = xe_hw_engine_group_suspend_faulting_lr_jobs(group,
+								  has_deps);
 		break;
 	case EXEC_MODE_DMA_FENCE:
 		new_mode = EXEC_MODE_LR;
@@ -281,19 +317,36 @@ static int switch_mode(struct xe_hw_engine_group *group)
 	return 0;
 }
 
+static int wait_syncs(struct xe_sync_entry *syncs, int num_syncs)
+{
+	int err, i;
+
+	for (i = 0; i < num_syncs; ++i) {
+		err = xe_sync_entry_wait(syncs + i);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
 /**
  * xe_hw_engine_group_get_mode() - Get the group to execute in the new mode
  * @group: The hw engine group
  * @new_mode: The new execution mode
  * @previous_mode: Pointer to the previous mode provided for use by caller
+ * @syncs: Syncs from exec IOCTL
+ * @num_syncs: Number of syncs from exec IOCTL
  *
  * Return: 0 if successful, -EINTR if locking failed.
  */
 int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group,
 				enum xe_hw_engine_group_execution_mode new_mode,
-				enum xe_hw_engine_group_execution_mode *previous_mode)
+				enum xe_hw_engine_group_execution_mode *previous_mode,
+				struct xe_sync_entry *syncs, int num_syncs)
 __acquires(&group->mode_sem)
 {
+	bool has_deps = !!num_syncs;
 	int err = down_read_interruptible(&group->mode_sem);
 
 	if (err)
@@ -303,15 +356,25 @@ __acquires(&group->mode_sem)
 
 	if (new_mode != group->cur_mode) {
 		up_read(&group->mode_sem);
+retry:
 		err = down_write_killable(&group->mode_sem);
 		if (err)
 			return err;
 
 		if (new_mode != group->cur_mode) {
-			err = switch_mode(group);
+			err = switch_mode(group, has_deps);
 			if (err) {
 				up_write(&group->mode_sem);
-				return err;
+
+				if (err != -EAGAIN)
+					return err;
+
+				err = wait_syncs(syncs, num_syncs);
+				if (err)
+					return err;
+
+				has_deps = false;
+				goto retry;
 			}
 		}
 		downgrade_write(&group->mode_sem);
diff --git a/drivers/gpu/drm/xe/xe_hw_engine_group.h b/drivers/gpu/drm/xe/xe_hw_engine_group.h
index 797ee81acbf2..8b17ccd30b70 100644
--- a/drivers/gpu/drm/xe/xe_hw_engine_group.h
+++ b/drivers/gpu/drm/xe/xe_hw_engine_group.h
@@ -11,6 +11,7 @@
 struct drm_device;
 struct xe_exec_queue;
 struct xe_gt;
+struct xe_sync_entry;
 
 int xe_hw_engine_setup_groups(struct xe_gt *gt);
 
@@ -19,7 +20,8 @@ void xe_hw_engine_group_del_exec_queue(struct xe_hw_engine_group *group, struct
 
 int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group,
 				enum xe_hw_engine_group_execution_mode new_mode,
-				enum xe_hw_engine_group_execution_mode *previous_mode);
+				enum xe_hw_engine_group_execution_mode *previous_mode,
+				struct xe_sync_entry *syncs, int num_syncs);
 void xe_hw_engine_group_put(struct xe_hw_engine_group *group);
 
 enum xe_hw_engine_group_execution_mode
diff --git a/drivers/gpu/drm/xe/xe_hw_fence.c b/drivers/gpu/drm/xe/xe_hw_fence.c
index b2a0c46dfcd4..f6057456e460 100644
--- a/drivers/gpu/drm/xe/xe_hw_fence.c
+++ b/drivers/gpu/drm/xe/xe_hw_fence.c
@@ -85,7 +85,6 @@ void xe_hw_fence_irq_finish(struct xe_hw_fence_irq *irq)
 {
 	struct xe_hw_fence *fence, *next;
 	unsigned long flags;
-	int err;
 	bool tmp;
 
 	if (XE_WARN_ON(!list_empty(&irq->pending))) {
@@ -93,9 +92,8 @@ void xe_hw_fence_irq_finish(struct xe_hw_fence_irq *irq)
 		spin_lock_irqsave(&irq->lock, flags);
 		list_for_each_entry_safe(fence, next, &irq->pending, irq_link) {
 			list_del_init(&fence->irq_link);
-			err = dma_fence_signal_locked(&fence->dma);
+			XE_WARN_ON(dma_fence_check_and_signal_locked(&fence->dma));
 			dma_fence_put(&fence->dma);
-			XE_WARN_ON(err);
 		}
 		spin_unlock_irqrestore(&irq->lock, flags);
 		dma_fence_end_signalling(tmp);
diff --git a/drivers/gpu/drm/xe/xe_hwmon.c b/drivers/gpu/drm/xe/xe_hwmon.c
index 97879daeefc1..ff2aea52ef75 100644
--- a/drivers/gpu/drm/xe/xe_hwmon.c
+++ b/drivers/gpu/drm/xe/xe_hwmon.c
@@ -502,7 +502,7 @@ xe_hwmon_power_max_interval_show(struct device *dev, struct device_attribute *at
 
 	int ret = 0;
 
-	xe_pm_runtime_get(hwmon->xe);
+	guard(xe_pm_runtime)(hwmon->xe);
 
 	mutex_lock(&hwmon->hwmon_lock);
 
@@ -521,8 +521,6 @@ xe_hwmon_power_max_interval_show(struct device *dev, struct device_attribute *at
 
 	mutex_unlock(&hwmon->hwmon_lock);
 
-	xe_pm_runtime_put(hwmon->xe);
-
 	x = REG_FIELD_GET(PWR_LIM_TIME_X, reg_val);
 	y = REG_FIELD_GET(PWR_LIM_TIME_Y, reg_val);
 
@@ -604,7 +602,7 @@ xe_hwmon_power_max_interval_store(struct device *dev, struct device_attribute *a
 	rxy = REG_FIELD_PREP(PWR_LIM_TIME_X, x) |
 			       REG_FIELD_PREP(PWR_LIM_TIME_Y, y);
 
-	xe_pm_runtime_get(hwmon->xe);
+	guard(xe_pm_runtime)(hwmon->xe);
 
 	mutex_lock(&hwmon->hwmon_lock);
 
@@ -616,8 +614,6 @@ xe_hwmon_power_max_interval_store(struct device *dev, struct device_attribute *a
 
 	mutex_unlock(&hwmon->hwmon_lock);
 
-	xe_pm_runtime_put(hwmon->xe);
-
 	return count;
 }
 
@@ -1124,37 +1120,25 @@ xe_hwmon_read(struct device *dev, enum hwmon_sensor_types type, u32 attr,
 	      int channel, long *val)
 {
 	struct xe_hwmon *hwmon = dev_get_drvdata(dev);
-	int ret;
 
-	xe_pm_runtime_get(hwmon->xe);
+	guard(xe_pm_runtime)(hwmon->xe);
 
 	switch (type) {
 	case hwmon_temp:
-		ret = xe_hwmon_temp_read(hwmon, attr, channel, val);
-		break;
+		return xe_hwmon_temp_read(hwmon, attr, channel, val);
 	case hwmon_power:
-		ret = xe_hwmon_power_read(hwmon, attr, channel, val);
-		break;
+		return xe_hwmon_power_read(hwmon, attr, channel, val);
 	case hwmon_curr:
-		ret = xe_hwmon_curr_read(hwmon, attr, channel, val);
-		break;
+		return xe_hwmon_curr_read(hwmon, attr, channel, val);
 	case hwmon_in:
-		ret = xe_hwmon_in_read(hwmon, attr, channel, val);
-		break;
+		return xe_hwmon_in_read(hwmon, attr, channel, val);
 	case hwmon_energy:
-		ret = xe_hwmon_energy_read(hwmon, attr, channel, val);
-		break;
+		return xe_hwmon_energy_read(hwmon, attr, channel, val);
 	case hwmon_fan:
-		ret = xe_hwmon_fan_read(hwmon, attr, channel, val);
-		break;
+		return xe_hwmon_fan_read(hwmon, attr, channel, val);
 	default:
-		ret = -EOPNOTSUPP;
-		break;
+		return -EOPNOTSUPP;
 	}
-
-	xe_pm_runtime_put(hwmon->xe);
-
-	return ret;
 }
 
 static int
@@ -1162,25 +1146,17 @@ xe_hwmon_write(struct device *dev, enum hwmon_sensor_types type, u32 attr,
 	       int channel, long val)
 {
 	struct xe_hwmon *hwmon = dev_get_drvdata(dev);
-	int ret;
 
-	xe_pm_runtime_get(hwmon->xe);
+	guard(xe_pm_runtime)(hwmon->xe);
 
 	switch (type) {
 	case hwmon_power:
-		ret = xe_hwmon_power_write(hwmon, attr, channel, val);
-		break;
+		return xe_hwmon_power_write(hwmon, attr, channel, val);
 	case hwmon_curr:
-		ret = xe_hwmon_curr_write(hwmon, attr, channel, val);
-		break;
+		return xe_hwmon_curr_write(hwmon, attr, channel, val);
 	default:
-		ret = -EOPNOTSUPP;
-		break;
+		return -EOPNOTSUPP;
 	}
-
-	xe_pm_runtime_put(hwmon->xe);
-
-	return ret;
 }
 
 static int xe_hwmon_read_label(struct device *dev,
diff --git a/drivers/gpu/drm/xe/xe_i2c.c b/drivers/gpu/drm/xe/xe_i2c.c
index 0b5452be0c87..8eccbae05705 100644
--- a/drivers/gpu/drm/xe/xe_i2c.c
+++ b/drivers/gpu/drm/xe/xe_i2c.c
@@ -319,7 +319,7 @@ int xe_i2c_probe(struct xe_device *xe)
 	struct xe_i2c *i2c;
 	int ret;
 
-	if (xe->info.platform != XE_BATTLEMAGE)
+	if (!xe->info.has_i2c)
 		return 0;
 
 	if (IS_SRIOV_VF(xe))
diff --git a/drivers/gpu/drm/xe/xe_irq.c b/drivers/gpu/drm/xe/xe_irq.c
index 024e13e606ec..baf5d2c6e802 100644
--- a/drivers/gpu/drm/xe/xe_irq.c
+++ b/drivers/gpu/drm/xe/xe_irq.c
@@ -21,6 +21,7 @@
 #include "xe_hw_error.h"
 #include "xe_i2c.h"
 #include "xe_memirq.h"
+#include "xe_mert.h"
 #include "xe_mmio.h"
 #include "xe_pxp.h"
 #include "xe_sriov.h"
@@ -525,6 +526,7 @@ static irqreturn_t dg1_irq_handler(int irq, void *arg)
 				xe_heci_csc_irq_handler(xe, master_ctl);
 			xe_display_irq_handler(xe, master_ctl);
 			xe_i2c_irq_handler(xe, master_ctl);
+			xe_mert_irq_handler(xe, master_ctl);
 			gu_misc_iir = gu_misc_irq_ack(xe, master_ctl);
 		}
 	}
diff --git a/drivers/gpu/drm/xe/xe_lmtt.c b/drivers/gpu/drm/xe/xe_lmtt.c
index 4dc1de482eee..3059ea6525bc 100644
--- a/drivers/gpu/drm/xe/xe_lmtt.c
+++ b/drivers/gpu/drm/xe/xe_lmtt.c
@@ -8,15 +8,18 @@
 #include <drm/drm_managed.h>
 
 #include "regs/xe_gt_regs.h"
+#include "regs/xe_mert_regs.h"
 
 #include "xe_assert.h"
 #include "xe_bo.h"
 #include "xe_tlb_inval.h"
 #include "xe_lmtt.h"
 #include "xe_map.h"
+#include "xe_mert.h"
 #include "xe_mmio.h"
 #include "xe_res_cursor.h"
 #include "xe_sriov.h"
+#include "xe_tile.h"
 #include "xe_tile_sriov_printk.h"
 
 /**
@@ -196,16 +199,22 @@ static void lmtt_setup_dir_ptr(struct xe_lmtt *lmtt)
 	struct xe_device *xe = tile_to_xe(tile);
 	dma_addr_t offset = xe_bo_main_addr(lmtt->pd->bo, XE_PAGE_SIZE);
 	struct xe_gt *gt;
+	u32 config;
 	u8 id;
 
 	lmtt_debug(lmtt, "DIR offset %pad\n", &offset);
 	lmtt_assert(lmtt, xe_bo_is_vram(lmtt->pd->bo));
 	lmtt_assert(lmtt, IS_ALIGNED(offset, SZ_64K));
 
+	config = LMEM_EN | REG_FIELD_PREP(LMTT_DIR_PTR, offset / SZ_64K);
+
 	for_each_gt_on_tile(gt, tile, id)
 		xe_mmio_write32(&gt->mmio,
 				GRAPHICS_VER(xe) >= 20 ? XE2_LMEM_CFG : LMEM_CFG,
-				LMEM_EN | REG_FIELD_PREP(LMTT_DIR_PTR, offset / SZ_64K));
+				config);
+
+	if (xe_device_has_mert(xe) && xe_tile_is_root(tile))
+		xe_mmio_write32(&tile->mmio, MERT_LMEM_CFG, config);
 }
 
 /**
@@ -262,19 +271,29 @@ static int lmtt_invalidate_hw(struct xe_lmtt *lmtt)
  * @lmtt: the &xe_lmtt to invalidate
  *
  * Send requests to all GuCs on this tile to invalidate all TLBs.
+ * If the platform has a standalone MERT, also invalidate MERT's TLB.
  *
  * This function should be called only when running as a PF driver.
  */
 void xe_lmtt_invalidate_hw(struct xe_lmtt *lmtt)
 {
+	struct xe_tile *tile = lmtt_to_tile(lmtt);
+	struct xe_device *xe = lmtt_to_xe(lmtt);
 	int err;
 
-	lmtt_assert(lmtt, IS_SRIOV_PF(lmtt_to_xe(lmtt)));
+	lmtt_assert(lmtt, IS_SRIOV_PF(xe));
 
 	err = lmtt_invalidate_hw(lmtt);
 	if (err)
-		xe_tile_sriov_err(lmtt_to_tile(lmtt), "LMTT invalidation failed (%pe)",
+		xe_tile_sriov_err(tile, "LMTT invalidation failed (%pe)",
 				  ERR_PTR(err));
+
+	if (xe_device_has_mert(xe) && xe_tile_is_root(tile)) {
+		err = xe_mert_invalidate_lmtt(tile);
+		if (err)
+			xe_tile_sriov_err(tile, "MERT LMTT invalidation failed (%pe)",
+					  ERR_PTR(err));
+	}
 }
 
 static void lmtt_write_pte(struct xe_lmtt *lmtt, struct xe_lmtt_pt *pt,
diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c
index b5083c99dd50..70eae7d03a27 100644
--- a/drivers/gpu/drm/xe/xe_lrc.c
+++ b/drivers/gpu/drm/xe/xe_lrc.c
@@ -44,6 +44,11 @@
 #define LRC_INDIRECT_CTX_BO_SIZE		SZ_4K
 #define LRC_INDIRECT_RING_STATE_SIZE		SZ_4K
 
+#define LRC_PRIORITY				GENMASK_ULL(10, 9)
+#define LRC_PRIORITY_LOW			0
+#define LRC_PRIORITY_NORMAL			1
+#define LRC_PRIORITY_HIGH			2
+
 /*
  * Layout of the LRC and associated data allocated as
  * lrc->bo:
@@ -91,13 +96,19 @@ gt_engine_needs_indirect_ctx(struct xe_gt *gt, enum xe_engine_class class)
 	return false;
 }
 
-size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class)
+/**
+ * xe_gt_lrc_hang_replay_size() - Hang replay size
+ * @gt: The GT
+ * @class: Hardware engine class
+ *
+ * Determine size of GPU hang replay state for a GT and hardware engine class.
+ *
+ * Return: Size of GPU hang replay size
+ */
+size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	size_t size;
-
-	/* Per-process HW status page (PPHWSP) */
-	size = LRC_PPHWSP_SIZE;
+	size_t size = 0;
 
 	/* Engine context image */
 	switch (class) {
@@ -123,11 +134,18 @@ size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class)
 		size += 1 * SZ_4K;
 	}
 
+	return size;
+}
+
+size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class)
+{
+	size_t size = xe_gt_lrc_hang_replay_size(gt, class);
+
 	/* Add indirect ring state page */
 	if (xe_gt_has_indirect_ring_state(gt))
 		size += LRC_INDIRECT_RING_STATE_SIZE;
 
-	return size;
+	return size + LRC_PPHWSP_SIZE;
 }
 
 /*
@@ -1386,8 +1404,33 @@ setup_indirect_ctx(struct xe_lrc *lrc, struct xe_hw_engine *hwe)
 	return 0;
 }
 
+static u8 xe_multi_queue_prio_to_lrc(struct xe_lrc *lrc, enum xe_multi_queue_priority priority)
+{
+	struct xe_device *xe = gt_to_xe(lrc->gt);
+
+	xe_assert(xe, (priority >= XE_MULTI_QUEUE_PRIORITY_LOW &&
+		       priority <= XE_MULTI_QUEUE_PRIORITY_HIGH));
+
+	/* xe_multi_queue_priority is directly mapped to LRC priority values */
+	return priority;
+}
+
+/**
+ * xe_lrc_set_multi_queue_priority() - Set multi queue priority in LRC
+ * @lrc: Logical Ring Context
+ * @priority: Multi queue priority of the exec queue
+ *
+ * Convert @priority to LRC multi queue priority and update the @lrc descriptor
+ */
+void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_priority priority)
+{
+	lrc->desc &= ~LRC_PRIORITY;
+	lrc->desc |= FIELD_PREP(LRC_PRIORITY, xe_multi_queue_prio_to_lrc(lrc, priority));
+}
+
 static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
-		       struct xe_vm *vm, u32 ring_size, u16 msix_vec,
+		       struct xe_vm *vm, void *replay_state, u32 ring_size,
+		       u16 msix_vec,
 		       u32 init_flags)
 {
 	struct xe_gt *gt = hwe->gt;
@@ -1402,6 +1445,7 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
 
 	kref_init(&lrc->refcount);
 	lrc->gt = gt;
+	lrc->replay_size = xe_gt_lrc_hang_replay_size(gt, hwe->class);
 	lrc->size = lrc_size;
 	lrc->flags = 0;
 	lrc->ring.size = ring_size;
@@ -1438,11 +1482,14 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
 	 * scratch.
 	 */
 	map = __xe_lrc_pphwsp_map(lrc);
-	if (gt->default_lrc[hwe->class]) {
+	if (gt->default_lrc[hwe->class] || replay_state) {
 		xe_map_memset(xe, &map, 0, 0, LRC_PPHWSP_SIZE);	/* PPHWSP */
 		xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE,
 				 gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE,
 				 lrc_size - LRC_PPHWSP_SIZE);
+		if (replay_state)
+			xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE,
+					 replay_state, lrc->replay_size);
 	} else {
 		void *init_data = empty_lrc_data(hwe);
 
@@ -1550,6 +1597,7 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
  * xe_lrc_create - Create a LRC
  * @hwe: Hardware Engine
  * @vm: The VM (address space)
+ * @replay_state: GPU hang replay state
  * @ring_size: LRC ring size
  * @msix_vec: MSI-X interrupt vector (for platforms that support it)
  * @flags: LRC initialization flags
@@ -1560,7 +1608,7 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
  * upon failure.
  */
 struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm,
-			     u32 ring_size, u16 msix_vec, u32 flags)
+			     void *replay_state, u32 ring_size, u16 msix_vec, u32 flags)
 {
 	struct xe_lrc *lrc;
 	int err;
@@ -1569,7 +1617,7 @@ struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm,
 	if (!lrc)
 		return ERR_PTR(-ENOMEM);
 
-	err = xe_lrc_init(lrc, hwe, vm, ring_size, msix_vec, flags);
+	err = xe_lrc_init(lrc, hwe, vm, replay_state, ring_size, msix_vec, flags);
 	if (err) {
 		kfree(lrc);
 		return ERR_PTR(err);
@@ -2235,6 +2283,8 @@ struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc)
 	snapshot->lrc_bo = xe_bo_get(lrc->bo);
 	snapshot->lrc_offset = xe_lrc_pphwsp_offset(lrc);
 	snapshot->lrc_size = lrc->size;
+	snapshot->replay_offset = 0;
+	snapshot->replay_size = lrc->replay_size;
 	snapshot->lrc_snapshot = NULL;
 	snapshot->ctx_timestamp = lower_32_bits(xe_lrc_ctx_timestamp(lrc));
 	snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc);
@@ -2305,6 +2355,9 @@ void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer
 	}
 
 	drm_printf(p, "\n\t[HWCTX].length: 0x%lx\n", snapshot->lrc_size - LRC_PPHWSP_SIZE);
+	drm_printf(p, "\n\t[HWCTX].replay_offset: 0x%lx\n", snapshot->replay_offset);
+	drm_printf(p, "\n\t[HWCTX].replay_length: 0x%lx\n", snapshot->replay_size);
+
 	drm_puts(p, "\t[HWCTX].data: ");
 	for (; i < snapshot->lrc_size; i += sizeof(u32)) {
 		u32 *val = snapshot->lrc_snapshot + i;
diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h
index 2fb628da5c43..8acf85273c1a 100644
--- a/drivers/gpu/drm/xe/xe_lrc.h
+++ b/drivers/gpu/drm/xe/xe_lrc.h
@@ -13,6 +13,7 @@ struct drm_printer;
 struct xe_bb;
 struct xe_device;
 struct xe_exec_queue;
+enum xe_multi_queue_priority;
 enum xe_engine_class;
 struct xe_gt;
 struct xe_hw_engine;
@@ -23,6 +24,7 @@ struct xe_lrc_snapshot {
 	struct xe_bo *lrc_bo;
 	void *lrc_snapshot;
 	unsigned long lrc_size, lrc_offset;
+	unsigned long replay_size, replay_offset;
 
 	u32 context_desc;
 	u32 ring_addr;
@@ -49,7 +51,7 @@ struct xe_lrc_snapshot {
 #define XE_LRC_CREATE_USER_CTX		BIT(2)
 
 struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm,
-			     u32 ring_size, u16 msix_vec, u32 flags);
+			     void *replay_state, u32 ring_size, u16 msix_vec, u32 flags);
 void xe_lrc_destroy(struct kref *ref);
 
 /**
@@ -86,6 +88,7 @@ static inline size_t xe_lrc_ring_size(void)
 	return SZ_16K;
 }
 
+size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class);
 size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class);
 u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc);
 u32 xe_lrc_regs_offset(struct xe_lrc *lrc);
@@ -133,6 +136,8 @@ void xe_lrc_dump_default(struct drm_printer *p,
 
 u32 *xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue *q, u32 *cs);
 
+void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_priority priority);
+
 struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc);
 void xe_lrc_snapshot_capture_delayed(struct xe_lrc_snapshot *snapshot);
 void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer *p);
diff --git a/drivers/gpu/drm/xe/xe_lrc_types.h b/drivers/gpu/drm/xe/xe_lrc_types.h
index e9883706e004..a4373d280c39 100644
--- a/drivers/gpu/drm/xe/xe_lrc_types.h
+++ b/drivers/gpu/drm/xe/xe_lrc_types.h
@@ -25,6 +25,9 @@ struct xe_lrc {
 	/** @size: size of the lrc and optional indirect ring state */
 	u32 size;
 
+	/** @replay_size: Size LRC needed for replaying a hang */
+	u32 replay_size;
+
 	/** @gt: gt which this LRC belongs to */
 	struct xe_gt *gt;
 
diff --git a/drivers/gpu/drm/xe/xe_mert.c b/drivers/gpu/drm/xe/xe_mert.c
new file mode 100644
index 000000000000..f7689e922953
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_mert.c
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright(c) 2025, Intel Corporation. All rights reserved.
+ */
+
+#include "regs/xe_irq_regs.h"
+#include "regs/xe_mert_regs.h"
+
+#include "xe_device.h"
+#include "xe_mert.h"
+#include "xe_mmio.h"
+#include "xe_tile.h"
+
+/**
+ * xe_mert_invalidate_lmtt - Invalidate MERT LMTT
+ * @tile: the &xe_tile
+ *
+ * Trigger invalidation of the MERT LMTT and wait for completion.
+ *
+ * Return: 0 on success or -ETIMEDOUT in case of a timeout.
+ */
+int xe_mert_invalidate_lmtt(struct xe_tile *tile)
+{
+	struct xe_device *xe = tile_to_xe(tile);
+	struct xe_mert *mert = &tile->mert;
+	const long timeout = HZ / 4;
+	unsigned long flags;
+
+	xe_assert(xe, xe_device_has_mert(xe));
+	xe_assert(xe, xe_tile_is_root(tile));
+
+	spin_lock_irqsave(&mert->lock, flags);
+	if (!mert->tlb_inv_triggered) {
+		mert->tlb_inv_triggered = true;
+		reinit_completion(&mert->tlb_inv_done);
+		xe_mmio_write32(&tile->mmio, MERT_TLB_INV_DESC_A, MERT_TLB_INV_DESC_A_VALID);
+	}
+	spin_unlock_irqrestore(&mert->lock, flags);
+
+	if (!wait_for_completion_timeout(&mert->tlb_inv_done, timeout))
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+/**
+ * xe_mert_irq_handler - Handler for MERT interrupts
+ * @xe: the &xe_device
+ * @master_ctl: interrupt register
+ *
+ * Handle interrupts generated by MERT.
+ */
+void xe_mert_irq_handler(struct xe_device *xe, u32 master_ctl)
+{
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	unsigned long flags;
+	u32 reg_val;
+	u8 err;
+
+	if (!(master_ctl & SOC_H2DMEMINT_IRQ))
+		return;
+
+	reg_val = xe_mmio_read32(&tile->mmio, MERT_TLB_CT_INTR_ERR_ID_PORT);
+	xe_mmio_write32(&tile->mmio, MERT_TLB_CT_INTR_ERR_ID_PORT, 0);
+
+	err = FIELD_GET(MERT_TLB_CT_ERROR_MASK, reg_val);
+	if (err == MERT_TLB_CT_LMTT_FAULT)
+		drm_dbg(&xe->drm, "MERT catastrophic error: LMTT fault (VF%u)\n",
+			FIELD_GET(MERT_TLB_CT_VFID_MASK, reg_val));
+	else if (err)
+		drm_dbg(&xe->drm, "MERT catastrophic error: Unexpected fault (0x%x)\n", err);
+
+	spin_lock_irqsave(&tile->mert.lock, flags);
+	if (tile->mert.tlb_inv_triggered) {
+		reg_val = xe_mmio_read32(&tile->mmio, MERT_TLB_INV_DESC_A);
+		if (!(reg_val & MERT_TLB_INV_DESC_A_VALID)) {
+			tile->mert.tlb_inv_triggered = false;
+			complete_all(&tile->mert.tlb_inv_done);
+		}
+	}
+	spin_unlock_irqrestore(&tile->mert.lock, flags);
+}
diff --git a/drivers/gpu/drm/xe/xe_mert.h b/drivers/gpu/drm/xe/xe_mert.h
new file mode 100644
index 000000000000..2e14c5dec008
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_mert.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright(c) 2025, Intel Corporation. All rights reserved.
+ */
+
+#ifndef __XE_MERT_H__
+#define __XE_MERT_H__
+
+#include <linux/completion.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+
+struct xe_device;
+struct xe_tile;
+
+struct xe_mert {
+	/** @lock: protects the TLB invalidation status */
+	spinlock_t lock;
+	/** @tlb_inv_triggered: indicates if TLB invalidation was triggered */
+	bool tlb_inv_triggered;
+	/** @mert.tlb_inv_done: completion of TLB invalidation */
+	struct completion tlb_inv_done;
+};
+
+#ifdef CONFIG_PCI_IOV
+int xe_mert_invalidate_lmtt(struct xe_tile *tile);
+void xe_mert_irq_handler(struct xe_device *xe, u32 master_ctl);
+#else
+static inline void xe_mert_irq_handler(struct xe_device *xe, u32 master_ctl) { }
+#endif
+
+#endif /* __XE_MERT_H__ */
diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c
index 2184af413b91..f3b66b55acfb 100644
--- a/drivers/gpu/drm/xe/xe_migrate.c
+++ b/drivers/gpu/drm/xe/xe_migrate.c
@@ -34,6 +34,7 @@
 #include "xe_res_cursor.h"
 #include "xe_sa.h"
 #include "xe_sched_job.h"
+#include "xe_sriov_vf_ccs.h"
 #include "xe_sync.h"
 #include "xe_trace_bo.h"
 #include "xe_validation.h"
@@ -1103,12 +1104,16 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q,
 	u32 batch_size, batch_size_allocated;
 	struct xe_device *xe = gt_to_xe(gt);
 	struct xe_res_cursor src_it, ccs_it;
+	struct xe_sriov_vf_ccs_ctx *ctx;
+	struct xe_sa_manager *bb_pool;
 	u64 size = xe_bo_size(src_bo);
 	struct xe_bb *bb = NULL;
 	u64 src_L0, src_L0_ofs;
 	u32 src_L0_pt;
 	int err;
 
+	ctx = &xe->sriov.vf.ccs.contexts[read_write];
+
 	xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it);
 
 	xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo),
@@ -1141,11 +1146,15 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q,
 		size -= src_L0;
 	}
 
+	bb_pool = ctx->mem.ccs_bb_pool;
+	guard(mutex) (xe_sa_bo_swap_guard(bb_pool));
+	xe_sa_bo_swap_shadow(bb_pool);
+
 	bb = xe_bb_ccs_new(gt, batch_size, read_write);
 	if (IS_ERR(bb)) {
 		drm_err(&xe->drm, "BB allocation failed.\n");
 		err = PTR_ERR(bb);
-		goto err_ret;
+		return err;
 	}
 
 	batch_size_allocated = batch_size;
@@ -1194,10 +1203,52 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q,
 	xe_assert(xe, (batch_size_allocated == bb->len));
 	src_bo->bb_ccs[read_write] = bb;
 
+	xe_sriov_vf_ccs_rw_update_bb_addr(ctx);
+	xe_sa_bo_sync_shadow(bb->bo);
 	return 0;
+}
 
-err_ret:
-	return err;
+/**
+ * xe_migrate_ccs_rw_copy_clear() - Clear the CCS read/write batch buffer
+ * content.
+ * @src_bo: The buffer object @src is currently bound to.
+ * @read_write : Creates BB commands for CCS read/write.
+ *
+ * Directly clearing the BB lacks atomicity and can lead to undefined
+ * behavior if the vCPU is halted mid-operation during the clearing
+ * process. To avoid this issue, we use a shadow buffer object approach.
+ *
+ * First swap the SA BO address with the shadow BO, perform the clearing
+ * operation on the BB, update the shadow BO in the ring buffer, then
+ * sync the shadow and the actual buffer to maintain consistency.
+ *
+ * Returns: None.
+ */
+void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo,
+				  enum xe_sriov_vf_ccs_rw_ctxs read_write)
+{
+	struct xe_bb *bb = src_bo->bb_ccs[read_write];
+	struct xe_device *xe = xe_bo_device(src_bo);
+	struct xe_sriov_vf_ccs_ctx *ctx;
+	struct xe_sa_manager *bb_pool;
+	u32 *cs;
+
+	xe_assert(xe, IS_SRIOV_VF(xe));
+
+	ctx = &xe->sriov.vf.ccs.contexts[read_write];
+	bb_pool = ctx->mem.ccs_bb_pool;
+
+	guard(mutex) (xe_sa_bo_swap_guard(bb_pool));
+	xe_sa_bo_swap_shadow(bb_pool);
+
+	cs = xe_sa_bo_cpu_addr(bb->bo);
+	memset(cs, MI_NOOP, bb->len * sizeof(u32));
+	xe_sriov_vf_ccs_rw_update_bb_addr(ctx);
+
+	xe_sa_bo_sync_shadow(bb->bo);
+
+	xe_bb_free(bb, NULL);
+	src_bo->bb_ccs[read_write] = NULL;
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h
index 260e298e5dd7..464c05dde1ba 100644
--- a/drivers/gpu/drm/xe/xe_migrate.h
+++ b/drivers/gpu/drm/xe/xe_migrate.h
@@ -134,6 +134,9 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q,
 			   struct xe_bo *src_bo,
 			   enum xe_sriov_vf_ccs_rw_ctxs read_write);
 
+void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo,
+				  enum xe_sriov_vf_ccs_rw_ctxs read_write);
+
 struct xe_lrc *xe_migrate_lrc(struct xe_migrate *migrate);
 struct xe_exec_queue *xe_migrate_exec_queue(struct xe_migrate *migrate);
 struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_offset,
diff --git a/drivers/gpu/drm/xe/xe_mocs.c b/drivers/gpu/drm/xe/xe_mocs.c
index 6613d3b48a84..0b7225bd77e0 100644
--- a/drivers/gpu/drm/xe/xe_mocs.c
+++ b/drivers/gpu/drm/xe/xe_mocs.c
@@ -811,26 +811,20 @@ int xe_mocs_dump(struct xe_gt *gt, struct drm_printer *p)
 	struct xe_device *xe = gt_to_xe(gt);
 	enum xe_force_wake_domains domain;
 	struct xe_mocs_info table;
-	unsigned int fw_ref, flags;
-	int err = 0;
+	unsigned int flags;
 
 	flags = get_mocs_settings(xe, &table);
 
 	domain = flags & HAS_LNCF_MOCS ? XE_FORCEWAKE_ALL : XE_FW_GT;
-	xe_pm_runtime_get_noresume(xe);
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), domain);
 
-	if (!xe_force_wake_ref_has_domain(fw_ref, domain)) {
-		err = -ETIMEDOUT;
-		goto err_fw;
-	}
+	guard(xe_pm_runtime_noresume)(xe);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), domain);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, domain))
+		return -ETIMEDOUT;
 
 	table.ops->dump(&table, flags, gt, p);
 
-err_fw:
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-	xe_pm_runtime_put(xe);
-	return err;
+	return 0;
 }
 
 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
diff --git a/drivers/gpu/drm/xe/xe_nvm.c b/drivers/gpu/drm/xe/xe_nvm.c
index 33f4ac82fc80..01510061d4d4 100644
--- a/drivers/gpu/drm/xe/xe_nvm.c
+++ b/drivers/gpu/drm/xe/xe_nvm.c
@@ -10,6 +10,7 @@
 #include "xe_device_types.h"
 #include "xe_mmio.h"
 #include "xe_nvm.h"
+#include "xe_pcode_api.h"
 #include "regs/xe_gsc_regs.h"
 #include "xe_sriov.h"
 
@@ -45,39 +46,50 @@ static bool xe_nvm_non_posted_erase(struct xe_device *xe)
 {
 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);
 
-	if (xe->info.platform != XE_BATTLEMAGE)
+	switch (xe->info.platform) {
+	case XE_CRESCENTISLAND:
+	case XE_BATTLEMAGE:
+		return !(xe_mmio_read32(mmio, XE_REG(GEN12_CNTL_PROTECTED_NVM_REG)) &
+			 NVM_NON_POSTED_ERASE_CHICKEN_BIT);
+	default:
 		return false;
-	return !(xe_mmio_read32(mmio, XE_REG(GEN12_CNTL_PROTECTED_NVM_REG)) &
-		 NVM_NON_POSTED_ERASE_CHICKEN_BIT);
+	}
 }
 
 static bool xe_nvm_writable_override(struct xe_device *xe)
 {
 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);
 	bool writable_override;
-	resource_size_t base;
+	struct xe_reg reg;
+	u32 test_bit;
 
 	switch (xe->info.platform) {
+	case XE_CRESCENTISLAND:
+		reg = PCODE_SCRATCH(0);
+		test_bit = FDO_MODE;
+		break;
 	case XE_BATTLEMAGE:
-		base = DG2_GSC_HECI2_BASE;
+		reg = HECI_FWSTS2(DG2_GSC_HECI2_BASE);
+		test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE;
 		break;
 	case XE_PVC:
-		base = PVC_GSC_HECI2_BASE;
+		reg = HECI_FWSTS2(PVC_GSC_HECI2_BASE);
+		test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE;
 		break;
 	case XE_DG2:
-		base = DG2_GSC_HECI2_BASE;
+		reg = HECI_FWSTS2(DG2_GSC_HECI2_BASE);
+		test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE;
 		break;
 	case XE_DG1:
-		base = DG1_GSC_HECI2_BASE;
+		reg = HECI_FWSTS2(DG1_GSC_HECI2_BASE);
+		test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE;
 		break;
 	default:
 		drm_err(&xe->drm, "Unknown platform\n");
 		return true;
 	}
 
-	writable_override =
-		!(xe_mmio_read32(mmio, HECI_FWSTS2(base)) &
-		  HECI_FW_STATUS_2_NVM_ACCESS_MODE);
+	writable_override = !(xe_mmio_read32(mmio, reg) & test_bit);
 	if (writable_override)
 		drm_info(&xe->drm, "NVM access overridden by jumper\n");
 	return writable_override;
diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c
index 890c363282ae..abf87fe0b345 100644
--- a/drivers/gpu/drm/xe/xe_oa.c
+++ b/drivers/gpu/drm/xe/xe_oa.c
@@ -1105,11 +1105,12 @@ static int xe_oa_enable_metric_set(struct xe_oa_stream *stream)
 			oag_buf_size_select(stream) |
 			oag_configure_mmio_trigger(stream, true));
 
-	xe_mmio_write32(mmio, __oa_regs(stream)->oa_ctx_ctrl, stream->periodic ?
-			(OAG_OAGLBCTXCTRL_COUNTER_RESUME |
+	xe_mmio_write32(mmio, __oa_regs(stream)->oa_ctx_ctrl,
+			OAG_OAGLBCTXCTRL_COUNTER_RESUME |
+			(stream->periodic ?
 			 OAG_OAGLBCTXCTRL_TIMER_ENABLE |
 			 REG_FIELD_PREP(OAG_OAGLBCTXCTRL_TIMER_PERIOD_MASK,
-					stream->period_exponent)) : 0);
+					 stream->period_exponent) : 0));
 
 	/*
 	 * Initialize Super Queue Internal Cnt Register
@@ -1254,6 +1255,9 @@ static int xe_oa_set_no_preempt(struct xe_oa *oa, u64 value,
 static int xe_oa_set_prop_num_syncs(struct xe_oa *oa, u64 value,
 				    struct xe_oa_open_param *param)
 {
+	if (XE_IOCTL_DBG(oa->xe, value > DRM_XE_MAX_SYNCS))
+		return -EINVAL;
+
 	param->num_syncs = value;
 	return 0;
 }
@@ -1343,7 +1347,7 @@ static int xe_oa_user_ext_set_property(struct xe_oa *oa, enum xe_oa_user_extn_fr
 		     ARRAY_SIZE(xe_oa_set_property_funcs_config));
 
 	if (XE_IOCTL_DBG(oa->xe, ext.property >= ARRAY_SIZE(xe_oa_set_property_funcs_open)) ||
-	    XE_IOCTL_DBG(oa->xe, ext.pad))
+	    XE_IOCTL_DBG(oa->xe, !ext.property) || XE_IOCTL_DBG(oa->xe, ext.pad))
 		return -EINVAL;
 
 	idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_oa_set_property_funcs_open));
@@ -1937,6 +1941,7 @@ static bool oa_unit_supports_oa_format(struct xe_oa_open_param *param, int type)
 			type == DRM_XE_OA_FMT_TYPE_OAC || type == DRM_XE_OA_FMT_TYPE_PEC;
 	case DRM_XE_OA_UNIT_TYPE_OAM:
 	case DRM_XE_OA_UNIT_TYPE_OAM_SAG:
+	case DRM_XE_OA_UNIT_TYPE_MERT:
 		return type == DRM_XE_OA_FMT_TYPE_OAM || type == DRM_XE_OA_FMT_TYPE_OAM_MPEC;
 	default:
 		return false;
@@ -1962,10 +1967,6 @@ static int xe_oa_assign_hwe(struct xe_oa *oa, struct xe_oa_open_param *param)
 	enum xe_hw_engine_id id;
 	int ret = 0;
 
-	/* If not provided, OA unit defaults to OA unit 0 as per uapi */
-	if (!param->oa_unit)
-		param->oa_unit = &xe_root_mmio_gt(oa->xe)->oa.oa_unit[0];
-
 	/* When we have an exec_q, get hwe from the exec_q */
 	if (param->exec_q) {
 		param->hwe = xe_gt_hw_engine(param->exec_q->gt, param->exec_q->class,
@@ -2031,7 +2032,15 @@ int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *f
 	if (ret)
 		return ret;
 
+	/* If not provided, OA unit defaults to OA unit 0 as per uapi */
+	if (!param.oa_unit)
+		param.oa_unit = &xe_root_mmio_gt(oa->xe)->oa.oa_unit[0];
+
 	if (param.exec_queue_id > 0) {
+		/* An exec_queue is only needed for OAR/OAC functionality on OAG */
+		if (XE_IOCTL_DBG(oa->xe, param.oa_unit->type != DRM_XE_OA_UNIT_TYPE_OAG))
+			return -EINVAL;
+
 		param.exec_q = xe_exec_queue_lookup(xef, param.exec_queue_id);
 		if (XE_IOCTL_DBG(oa->xe, !param.exec_q))
 			return -ENOENT;
@@ -2220,6 +2229,8 @@ static const struct xe_mmio_range xe2_oa_mux_regs[] = {
 	{ .start = 0xE18C, .end = 0xE18C },	/* SAMPLER_MODE */
 	{ .start = 0xE590, .end = 0xE590 },	/* TDL_LSC_LAT_MEASURE_TDL_GFX */
 	{ .start = 0x13000, .end = 0x137FC },	/* PES_0_PESL0 - PES_63_UPPER_PESL3 */
+	{ .start = 0x145194, .end = 0x145194 },	/* SYS_MEM_LAT_MEASURE */
+	{ .start = 0x145340, .end = 0x14537C },	/* MERTSS_PES_0 - MERTSS_PES_7 */
 	{},
 };
 
@@ -2511,7 +2522,12 @@ int xe_oa_register(struct xe_device *xe)
 static u32 num_oa_units_per_gt(struct xe_gt *gt)
 {
 	if (xe_gt_is_main_type(gt) || GRAPHICS_VER(gt_to_xe(gt)) < 20)
-		return 1;
+		/*
+		 * Mert OA unit belongs to the SoC, not a gt, so should be accessed using
+		 * xe_root_tile_mmio(). However, for all known platforms this is the same as
+		 * accessing via xe_root_mmio_gt()->mmio.
+		 */
+		return xe_device_has_mert(gt_to_xe(gt)) ? 2 : 1;
 	else if (!IS_DGFX(gt_to_xe(gt)))
 		return XE_OAM_UNIT_SCMI_0 + 1; /* SAG + SCMI_0 */
 	else
@@ -2566,40 +2582,57 @@ static u32 __hwe_oa_unit(struct xe_hw_engine *hwe)
 static struct xe_oa_regs __oam_regs(u32 base)
 {
 	return (struct xe_oa_regs) {
-		base,
-		OAM_HEAD_POINTER(base),
-		OAM_TAIL_POINTER(base),
-		OAM_BUFFER(base),
-		OAM_CONTEXT_CONTROL(base),
-		OAM_CONTROL(base),
-		OAM_DEBUG(base),
-		OAM_STATUS(base),
-		OAM_CONTROL_COUNTER_SEL_MASK,
+		.base		= base,
+		.oa_head_ptr	= OAM_HEAD_POINTER(base),
+		.oa_tail_ptr	= OAM_TAIL_POINTER(base),
+		.oa_buffer	= OAM_BUFFER(base),
+		.oa_ctx_ctrl	= OAM_CONTEXT_CONTROL(base),
+		.oa_ctrl	= OAM_CONTROL(base),
+		.oa_debug	= OAM_DEBUG(base),
+		.oa_status	= OAM_STATUS(base),
+		.oa_mmio_trg	= OAM_MMIO_TRG(base),
+		.oa_ctrl_counter_select_mask = OAM_CONTROL_COUNTER_SEL_MASK,
 	};
 }
 
 static struct xe_oa_regs __oag_regs(void)
 {
 	return (struct xe_oa_regs) {
-		0,
-		OAG_OAHEADPTR,
-		OAG_OATAILPTR,
-		OAG_OABUFFER,
-		OAG_OAGLBCTXCTRL,
-		OAG_OACONTROL,
-		OAG_OA_DEBUG,
-		OAG_OASTATUS,
-		OAG_OACONTROL_OA_COUNTER_SEL_MASK,
+		.base		= 0,
+		.oa_head_ptr	= OAG_OAHEADPTR,
+		.oa_tail_ptr	= OAG_OATAILPTR,
+		.oa_buffer	= OAG_OABUFFER,
+		.oa_ctx_ctrl	= OAG_OAGLBCTXCTRL,
+		.oa_ctrl	= OAG_OACONTROL,
+		.oa_debug	= OAG_OA_DEBUG,
+		.oa_status	= OAG_OASTATUS,
+		.oa_mmio_trg	= OAG_MMIOTRIGGER,
+		.oa_ctrl_counter_select_mask = OAG_OACONTROL_OA_COUNTER_SEL_MASK,
+	};
+}
+
+static struct xe_oa_regs __oamert_regs(void)
+{
+	return (struct xe_oa_regs) {
+		.base		= 0,
+		.oa_head_ptr	= OAMERT_HEAD_POINTER,
+		.oa_tail_ptr	= OAMERT_TAIL_POINTER,
+		.oa_buffer	= OAMERT_BUFFER,
+		.oa_ctx_ctrl	= OAMERT_CONTEXT_CONTROL,
+		.oa_ctrl	= OAMERT_CONTROL,
+		.oa_debug	= OAMERT_DEBUG,
+		.oa_status	= OAMERT_STATUS,
+		.oa_mmio_trg	= OAMERT_MMIO_TRG,
+		.oa_ctrl_counter_select_mask = OAM_CONTROL_COUNTER_SEL_MASK,
 	};
 }
 
 static void __xe_oa_init_oa_units(struct xe_gt *gt)
 {
-	/* Actual address is MEDIA_GT_GSI_OFFSET + oam_base_addr[i] */
 	const u32 oam_base_addr[] = {
-		[XE_OAM_UNIT_SAG]    = 0x13000,
-		[XE_OAM_UNIT_SCMI_0] = 0x14000,
-		[XE_OAM_UNIT_SCMI_1] = 0x14800,
+		[XE_OAM_UNIT_SAG]    = XE_OAM_SAG_BASE,
+		[XE_OAM_UNIT_SCMI_0] = XE_OAM_SCMI_0_BASE,
+		[XE_OAM_UNIT_SCMI_1] = XE_OAM_SCMI_1_BASE,
 	};
 	int i, num_units = gt->oa.num_oa_units;
 
@@ -2607,8 +2640,15 @@ static void __xe_oa_init_oa_units(struct xe_gt *gt)
 		struct xe_oa_unit *u = &gt->oa.oa_unit[i];
 
 		if (xe_gt_is_main_type(gt)) {
-			u->regs = __oag_regs();
-			u->type = DRM_XE_OA_UNIT_TYPE_OAG;
+			if (!i) {
+				u->regs = __oag_regs();
+				u->type = DRM_XE_OA_UNIT_TYPE_OAG;
+			} else {
+				xe_gt_assert(gt, xe_device_has_mert(gt_to_xe(gt)));
+				xe_gt_assert(gt, gt == xe_root_mmio_gt(gt_to_xe(gt)));
+				u->regs = __oamert_regs();
+				u->type = DRM_XE_OA_UNIT_TYPE_MERT;
+			}
 		} else {
 			xe_gt_assert(gt, GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270);
 			u->regs = __oam_regs(oam_base_addr[i]);
diff --git a/drivers/gpu/drm/xe/xe_oa_types.h b/drivers/gpu/drm/xe/xe_oa_types.h
index cf080f412189..08cc8d7c2215 100644
--- a/drivers/gpu/drm/xe/xe_oa_types.h
+++ b/drivers/gpu/drm/xe/xe_oa_types.h
@@ -87,6 +87,7 @@ struct xe_oa_regs {
 	struct xe_reg oa_ctrl;
 	struct xe_reg oa_debug;
 	struct xe_reg oa_status;
+	struct xe_reg oa_mmio_trg;
 	u32 oa_ctrl_counter_select_mask;
 };
 
diff --git a/drivers/gpu/drm/xe/xe_page_reclaim.c b/drivers/gpu/drm/xe/xe_page_reclaim.c
new file mode 100644
index 000000000000..fd8c33761127
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_page_reclaim.c
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2025 Intel Corporation
+ */
+
+#include <linux/bitfield.h>
+#include <linux/kref.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+
+#include "xe_page_reclaim.h"
+
+#include "regs/xe_gt_regs.h"
+#include "xe_assert.h"
+#include "xe_macros.h"
+#include "xe_mmio.h"
+#include "xe_pat.h"
+#include "xe_sa.h"
+#include "xe_tlb_inval_types.h"
+#include "xe_vm.h"
+
+/**
+ * xe_page_reclaim_skip() - Decide whether PRL should be skipped for a VMA
+ * @tile: Tile owning the VMA
+ * @vma: VMA under consideration
+ *
+ * PPC flushing may be handled by HW for specific PAT encodings.
+ * Skip PPC flushing/Page Reclaim for scenarios below due to redundant
+ * flushes.
+ * - pat_index is transient display (1)
+ *
+ * Return: true when page reclamation is unnecessary, false otherwise.
+ */
+bool xe_page_reclaim_skip(struct xe_tile *tile, struct xe_vma *vma)
+{
+	u8 l3_policy;
+
+	l3_policy = xe_pat_index_get_l3_policy(tile->xe, vma->attr.pat_index);
+
+	/*
+	 *   - l3_policy:   0=WB, 1=XD ("WB - Transient Display"), 3=UC
+	 * Transient display flushes is taken care by HW, l3_policy = 1.
+	 *
+	 * HW will sequence these transient flushes at various sync points so
+	 * any event of page reclamation will hit these sync points before
+	 * page reclamation could execute.
+	 */
+	return (l3_policy == XE_L3_POLICY_XD);
+}
+
+/**
+ * xe_page_reclaim_create_prl_bo() - Back a PRL with a suballocated GGTT BO
+ * @tlb_inval: TLB invalidation frontend associated with the request
+ * @prl: page reclaim list data that bo will copy from
+ * @fence: tlb invalidation fence that page reclaim action is paired to
+ *
+ * Suballocates a 4K BO out of the tile reclaim pool, copies the PRL CPU
+ * copy into the BO and queues the buffer for release when @fence signals.
+ *
+ * Return: struct drm_suballoc pointer on success or ERR_PTR on failure.
+ */
+struct drm_suballoc *xe_page_reclaim_create_prl_bo(struct xe_tlb_inval *tlb_inval,
+						   struct xe_page_reclaim_list *prl,
+						   struct xe_tlb_inval_fence *fence)
+{
+	struct xe_gt *gt = container_of(tlb_inval, struct xe_gt, tlb_inval);
+	struct xe_tile *tile = gt_to_tile(gt);
+	/* (+1) for NULL page_reclaim_entry to indicate end of list */
+	int prl_size = min(prl->num_entries + 1, XE_PAGE_RECLAIM_MAX_ENTRIES) *
+		sizeof(struct xe_guc_page_reclaim_entry);
+	struct drm_suballoc *prl_sa;
+
+	/* Maximum size of PRL is 1 4K-page */
+	prl_sa = __xe_sa_bo_new(tile->mem.reclaim_pool,
+				prl_size, GFP_ATOMIC);
+	if (IS_ERR(prl_sa))
+		return prl_sa;
+
+	memcpy(xe_sa_bo_cpu_addr(prl_sa), prl->entries,
+	       prl_size);
+	xe_sa_bo_flush_write(prl_sa);
+	/* Queue up sa_bo_free on tlb invalidation fence signal */
+	xe_sa_bo_free(prl_sa, &fence->base);
+
+	return prl_sa;
+}
+
+/**
+ * xe_page_reclaim_list_invalidate() - Mark a PRL as invalid
+ * @prl: Page reclaim list to reset
+ *
+ * Clears the entries pointer and marks the list as invalid so
+ * future use knows PRL is unusable. It is expected that the entries
+ * have already been released.
+ */
+void xe_page_reclaim_list_invalidate(struct xe_page_reclaim_list *prl)
+{
+	xe_page_reclaim_entries_put(prl->entries);
+	prl->entries = NULL;
+	prl->num_entries = XE_PAGE_RECLAIM_INVALID_LIST;
+}
+
+/**
+ * xe_page_reclaim_list_init() - Initialize a page reclaim list
+ * @prl: Page reclaim list to initialize
+ *
+ * NULLs both values in list to prepare on initalization.
+ */
+void xe_page_reclaim_list_init(struct xe_page_reclaim_list *prl)
+{
+	// xe_page_reclaim_list_invalidate(prl);
+	prl->entries = NULL;
+	prl->num_entries = 0;
+}
+
+/**
+ * xe_page_reclaim_list_alloc_entries() - Allocate page reclaim list entries
+ * @prl: Page reclaim list to allocate entries for
+ *
+ * Allocate one 4K page for the PRL entries, otherwise assign prl->entries to NULL.
+ */
+int xe_page_reclaim_list_alloc_entries(struct xe_page_reclaim_list *prl)
+{
+	struct page *page;
+
+	if (XE_WARN_ON(prl->entries))
+		return 0;
+
+	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+	if (page) {
+		prl->entries = page_address(page);
+		prl->num_entries = 0;
+	}
+
+	return page ? 0 : -ENOMEM;
+}
diff --git a/drivers/gpu/drm/xe/xe_page_reclaim.h b/drivers/gpu/drm/xe/xe_page_reclaim.h
new file mode 100644
index 000000000000..a4f58e0ce9b4
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_page_reclaim.h
@@ -0,0 +1,105 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2025 Intel Corporation
+ */
+
+#ifndef _XE_PAGE_RECLAIM_H_
+#define _XE_PAGE_RECLAIM_H_
+
+#include <linux/kref.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/workqueue.h>
+#include <linux/bits.h>
+
+#define XE_PAGE_RECLAIM_MAX_ENTRIES	512
+#define XE_PAGE_RECLAIM_LIST_MAX_SIZE	SZ_4K
+
+struct xe_tlb_inval;
+struct xe_tlb_inval_fence;
+struct xe_tile;
+struct xe_vma;
+
+struct xe_guc_page_reclaim_entry {
+	u64 qw;
+/* valid reclaim entry bit */
+#define XE_PAGE_RECLAIM_VALID		BIT_ULL(0)
+/*
+ * offset order of page size to be reclaimed
+ * page_size = 1 << (XE_PTE_SHIFT + reclamation_size)
+ */
+#define XE_PAGE_RECLAIM_SIZE		GENMASK_ULL(6, 1)
+#define XE_PAGE_RECLAIM_RSVD_0		GENMASK_ULL(11, 7)
+/* lower 20 bits of the physical address */
+#define XE_PAGE_RECLAIM_ADDR_LO		GENMASK_ULL(31, 12)
+/* upper 20 bits of the physical address */
+#define XE_PAGE_RECLAIM_ADDR_HI		GENMASK_ULL(51, 32)
+#define XE_PAGE_RECLAIM_RSVD_1		GENMASK_ULL(63, 52)
+} __packed;
+
+struct xe_page_reclaim_list {
+	/** @entries: array of page reclaim entries, page allocated */
+	struct xe_guc_page_reclaim_entry *entries;
+	/** @num_entries: number of entries */
+	int num_entries;
+#define XE_PAGE_RECLAIM_INVALID_LIST	-1
+};
+
+/**
+ * xe_page_reclaim_list_is_new() - Check if PRL is new allocation
+ * @prl: Pointer to page reclaim list
+ *
+ * PRL indicates it hasn't been allocated through both values being NULL
+ */
+static inline bool xe_page_reclaim_list_is_new(struct xe_page_reclaim_list *prl)
+{
+	return !prl->entries && prl->num_entries == 0;
+}
+
+/**
+ * xe_page_reclaim_list_valid() - Check if the page reclaim list is valid
+ * @prl: Pointer to page reclaim list
+ *
+ * PRL uses the XE_PAGE_RECLAIM_INVALID_LIST to indicate that a PRL
+ * is unusable.
+ */
+static inline bool xe_page_reclaim_list_valid(struct xe_page_reclaim_list *prl)
+{
+	return !xe_page_reclaim_list_is_new(prl) &&
+	       prl->num_entries != XE_PAGE_RECLAIM_INVALID_LIST;
+}
+
+bool xe_page_reclaim_skip(struct xe_tile *tile, struct xe_vma *vma);
+struct drm_suballoc *xe_page_reclaim_create_prl_bo(struct xe_tlb_inval *tlb_inval,
+						   struct xe_page_reclaim_list *prl,
+						   struct xe_tlb_inval_fence *fence);
+void xe_page_reclaim_list_invalidate(struct xe_page_reclaim_list *prl);
+void xe_page_reclaim_list_init(struct xe_page_reclaim_list *prl);
+int xe_page_reclaim_list_alloc_entries(struct xe_page_reclaim_list *prl);
+/**
+ * xe_page_reclaim_entries_get() - Increment the reference count of page reclaim entries.
+ * @entries: Pointer to the array of page reclaim entries.
+ *
+ * This function increments the reference count of the backing page.
+ */
+static inline void xe_page_reclaim_entries_get(struct xe_guc_page_reclaim_entry *entries)
+{
+	if (entries)
+		get_page(virt_to_page(entries));
+}
+
+/**
+ * xe_page_reclaim_entries_put() - Decrement the reference count of page reclaim entries.
+ * @entries: Pointer to the array of page reclaim entries.
+ *
+ * This function decrements the reference count of the backing page
+ * and frees it if the count reaches zero.
+ */
+static inline void xe_page_reclaim_entries_put(struct xe_guc_page_reclaim_entry *entries)
+{
+	if (entries)
+		put_page(virt_to_page(entries));
+}
+
+#endif	/* _XE_PAGE_RECLAIM_H_ */
diff --git a/drivers/gpu/drm/xe/xe_pat.c b/drivers/gpu/drm/xe/xe_pat.c
index 68171cceea18..2c3375e0250b 100644
--- a/drivers/gpu/drm/xe/xe_pat.c
+++ b/drivers/gpu/drm/xe/xe_pat.c
@@ -9,6 +9,7 @@
 
 #include <generated/xe_wa_oob.h>
 
+#include "regs/xe_gt_regs.h"
 #include "regs/xe_reg_defs.h"
 #include "xe_assert.h"
 #include "xe_device.h"
@@ -50,8 +51,37 @@
 #define XELP_PAT_WC				REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 1)
 #define XELP_PAT_UC				REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 0)
 
+#define PAT_LABEL_LEN 20
+
 static const char *XELP_MEM_TYPE_STR_MAP[] = { "UC", "WC", "WT", "WB" };
 
+static void xe_pat_index_label(char *label, size_t len, int index)
+{
+	snprintf(label, len, "PAT[%2d] ", index);
+}
+
+static void xelp_pat_entry_dump(struct drm_printer *p, int index, u32 pat)
+{
+	u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);
+
+	drm_printf(p, "PAT[%2d] = %s (%#8x)\n", index,
+		   XELP_MEM_TYPE_STR_MAP[mem_type], pat);
+}
+
+static void xehpc_pat_entry_dump(struct drm_printer *p, int index, u32 pat)
+{
+	drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index,
+		   REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat),
+		   REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat);
+}
+
+static void xelpg_pat_entry_dump(struct drm_printer *p, int index, u32 pat)
+{
+	drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index,
+		   REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat),
+		   REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat);
+}
+
 struct xe_pat_ops {
 	void (*program_graphics)(struct xe_gt *gt, const struct xe_pat_table_entry table[],
 				 int n_entries);
@@ -196,6 +226,19 @@ u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index)
 	return xe->pat.table[pat_index].coh_mode;
 }
 
+bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index)
+{
+	WARN_ON(pat_index >= xe->pat.n_entries);
+	return !!(xe->pat.table[pat_index].value & XE2_COMP_EN);
+}
+
+u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index)
+{
+	WARN_ON(pat_index >= xe->pat.n_entries);
+
+	return REG_FIELD_GET(XE2_L3_POLICY, xe->pat.table[pat_index].value);
+}
+
 static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],
 			int n_entries)
 {
@@ -233,24 +276,20 @@ static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry ta
 static int xelp_dump(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	int i;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 
 	drm_printf(p, "PAT table:\n");
 
 	for (i = 0; i < xe->pat.n_entries; i++) {
 		u32 pat = xe_mmio_read32(&gt->mmio, XE_REG(_PAT_INDEX(i)));
-		u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);
 
-		drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i,
-			   XELP_MEM_TYPE_STR_MAP[mem_type], pat);
+		xelp_pat_entry_dump(p, i, pat);
 	}
 
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
 }
 
@@ -262,26 +301,20 @@ static const struct xe_pat_ops xelp_pat_ops = {
 static int xehp_dump(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	int i;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 
 	drm_printf(p, "PAT table:\n");
 
 	for (i = 0; i < xe->pat.n_entries; i++) {
 		u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
-		u8 mem_type;
-
-		mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);
 
-		drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i,
-			   XELP_MEM_TYPE_STR_MAP[mem_type], pat);
+		xelp_pat_entry_dump(p, i, pat);
 	}
 
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
 }
 
@@ -293,11 +326,10 @@ static const struct xe_pat_ops xehp_pat_ops = {
 static int xehpc_dump(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	int i;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 
 	drm_printf(p, "PAT table:\n");
@@ -305,12 +337,9 @@ static int xehpc_dump(struct xe_gt *gt, struct drm_printer *p)
 	for (i = 0; i < xe->pat.n_entries; i++) {
 		u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
 
-		drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i,
-			   REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat),
-			   REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat);
+		xehpc_pat_entry_dump(p, i, pat);
 	}
 
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
 }
 
@@ -322,11 +351,10 @@ static const struct xe_pat_ops xehpc_pat_ops = {
 static int xelpg_dump(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	int i;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 
 	drm_printf(p, "PAT table:\n");
@@ -339,12 +367,9 @@ static int xelpg_dump(struct xe_gt *gt, struct drm_printer *p)
 		else
 			pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
 
-		drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i,
-			   REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat),
-			   REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat);
+		xelpg_pat_entry_dump(p, i, pat);
 	}
 
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
 }
 
@@ -358,15 +383,38 @@ static const struct xe_pat_ops xelpg_pat_ops = {
 	.dump = xelpg_dump,
 };
 
+static void xe2_pat_entry_dump(struct drm_printer *p, const char *label, u32 pat, bool rsvd)
+{
+	drm_printf(p, "%s= [ %u, %u, %u, %u, %u, %u ]  (%#8x)%s\n", label,
+		   !!(pat & XE2_NO_PROMOTE),
+		   !!(pat & XE2_COMP_EN),
+		   REG_FIELD_GET(XE2_L3_CLOS, pat),
+		   REG_FIELD_GET(XE2_L3_POLICY, pat),
+		   REG_FIELD_GET(XE2_L4_POLICY, pat),
+		   REG_FIELD_GET(XE2_COH_MODE, pat),
+		   pat, rsvd ? " *" : "");
+}
+
+static void xe3p_xpc_pat_entry_dump(struct drm_printer *p, const char *label, u32 pat, bool rsvd)
+{
+	drm_printf(p, "%s= [ %u, %u, %u, %u, %u ]  (%#8x)%s\n", label,
+		   !!(pat & XE2_NO_PROMOTE),
+		   REG_FIELD_GET(XE2_L3_CLOS, pat),
+		   REG_FIELD_GET(XE2_L3_POLICY, pat),
+		   REG_FIELD_GET(XE2_L4_POLICY, pat),
+		   REG_FIELD_GET(XE2_COH_MODE, pat),
+		   pat, rsvd ? " *" : "");
+}
+
 static int xe2_dump(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	u32 pat;
 	int i;
+	char label[PAT_LABEL_LEN];
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 
 	drm_printf(p, "PAT table: (* = reserved entry)\n");
@@ -377,14 +425,8 @@ static int xe2_dump(struct xe_gt *gt, struct drm_printer *p)
 		else
 			pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
 
-		drm_printf(p, "PAT[%2d] = [ %u, %u, %u, %u, %u, %u ]  (%#8x)%s\n", i,
-			   !!(pat & XE2_NO_PROMOTE),
-			   !!(pat & XE2_COMP_EN),
-			   REG_FIELD_GET(XE2_L3_CLOS, pat),
-			   REG_FIELD_GET(XE2_L3_POLICY, pat),
-			   REG_FIELD_GET(XE2_L4_POLICY, pat),
-			   REG_FIELD_GET(XE2_COH_MODE, pat),
-			   pat, xe->pat.table[i].valid ? "" : " *");
+		xe_pat_index_label(label, sizeof(label), i);
+		xe2_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid);
 	}
 
 	/*
@@ -397,16 +439,8 @@ static int xe2_dump(struct xe_gt *gt, struct drm_printer *p)
 		pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA));
 
 	drm_printf(p, "Page Table Access:\n");
-	drm_printf(p, "PTA_MODE= [ %u, %u, %u, %u, %u, %u ]  (%#8x)\n",
-		   !!(pat & XE2_NO_PROMOTE),
-		   !!(pat & XE2_COMP_EN),
-		   REG_FIELD_GET(XE2_L3_CLOS, pat),
-		   REG_FIELD_GET(XE2_L3_POLICY, pat),
-		   REG_FIELD_GET(XE2_L4_POLICY, pat),
-		   REG_FIELD_GET(XE2_COH_MODE, pat),
-		   pat);
+	xe2_pat_entry_dump(p, "PTA_MODE", pat, false);
 
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
 }
 
@@ -419,12 +453,12 @@ static const struct xe_pat_ops xe2_pat_ops = {
 static int xe3p_xpc_dump(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	u32 pat;
 	int i;
+	char label[PAT_LABEL_LEN];
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 
 	drm_printf(p, "PAT table: (* = reserved entry)\n");
@@ -432,13 +466,8 @@ static int xe3p_xpc_dump(struct xe_gt *gt, struct drm_printer *p)
 	for (i = 0; i < xe->pat.n_entries; i++) {
 		pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
 
-		drm_printf(p, "PAT[%2d] = [ %u, %u, %u, %u, %u ]  (%#8x)%s\n", i,
-			   !!(pat & XE2_NO_PROMOTE),
-			   REG_FIELD_GET(XE2_L3_CLOS, pat),
-			   REG_FIELD_GET(XE2_L3_POLICY, pat),
-			   REG_FIELD_GET(XE2_L4_POLICY, pat),
-			   REG_FIELD_GET(XE2_COH_MODE, pat),
-			   pat, xe->pat.table[i].valid ? "" : " *");
+		xe_pat_index_label(label, sizeof(label), i);
+		xe3p_xpc_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid);
 	}
 
 	/*
@@ -448,15 +477,8 @@ static int xe3p_xpc_dump(struct xe_gt *gt, struct drm_printer *p)
 	pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA));
 
 	drm_printf(p, "Page Table Access:\n");
-	drm_printf(p, "PTA_MODE= [ %u, %u, %u, %u, %u ]  (%#8x)\n",
-		   !!(pat & XE2_NO_PROMOTE),
-		   REG_FIELD_GET(XE2_L3_CLOS, pat),
-		   REG_FIELD_GET(XE2_L3_POLICY, pat),
-		   REG_FIELD_GET(XE2_L4_POLICY, pat),
-		   REG_FIELD_GET(XE2_COH_MODE, pat),
-		   pat);
+	xe3p_xpc_pat_entry_dump(p, "PTA_MODE", pat, false);
 
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
 }
 
@@ -578,3 +600,65 @@ int xe_pat_dump(struct xe_gt *gt, struct drm_printer *p)
 
 	return xe->pat.ops->dump(gt, p);
 }
+
+/**
+ * xe_pat_dump_sw_config() - Dump the software-configured GT PAT table into a drm printer.
+ * @gt: the &xe_gt
+ * @p: the &drm_printer
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p)
+{
+	struct xe_device *xe = gt_to_xe(gt);
+	char label[PAT_LABEL_LEN];
+
+	if (!xe->pat.table || !xe->pat.n_entries)
+		return -EOPNOTSUPP;
+
+	drm_printf(p, "PAT table:%s\n", GRAPHICS_VER(xe) >= 20 ? " (* = reserved entry)" : "");
+	for (u32 i = 0; i < xe->pat.n_entries; i++) {
+		u32 pat = xe->pat.table[i].value;
+
+		if (GRAPHICS_VERx100(xe) == 3511) {
+			xe_pat_index_label(label, sizeof(label), i);
+			xe3p_xpc_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid);
+		} else if (GRAPHICS_VER(xe) == 30 || GRAPHICS_VER(xe) == 20) {
+			xe_pat_index_label(label, sizeof(label), i);
+			xe2_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid);
+		} else if (xe->info.platform == XE_METEORLAKE) {
+			xelpg_pat_entry_dump(p, i, pat);
+		} else if (xe->info.platform == XE_PVC) {
+			xehpc_pat_entry_dump(p, i, pat);
+		} else if (xe->info.platform == XE_DG2 || GRAPHICS_VERx100(xe) <= 1210) {
+			xelp_pat_entry_dump(p, i, pat);
+		} else {
+			return -EOPNOTSUPP;
+		}
+	}
+
+	if (xe->pat.pat_pta) {
+		u32 pat = xe->pat.pat_pta->value;
+
+		drm_printf(p, "Page Table Access:\n");
+		xe2_pat_entry_dump(p, "PTA_MODE", pat, false);
+	}
+
+	if (xe->pat.pat_ats) {
+		u32 pat = xe->pat.pat_ats->value;
+
+		drm_printf(p, "PCIe ATS/PASID:\n");
+		xe2_pat_entry_dump(p, "PAT_ATS ", pat, false);
+	}
+
+	drm_printf(p, "Cache Level:\n");
+	drm_printf(p, "IDX[XE_CACHE_NONE] = %d\n", xe->pat.idx[XE_CACHE_NONE]);
+	drm_printf(p, "IDX[XE_CACHE_WT] = %d\n", xe->pat.idx[XE_CACHE_WT]);
+	drm_printf(p, "IDX[XE_CACHE_WB] = %d\n", xe->pat.idx[XE_CACHE_WB]);
+	if (GRAPHICS_VER(xe) >= 20) {
+		drm_printf(p, "IDX[XE_CACHE_NONE_COMPRESSION] = %d\n",
+			   xe->pat.idx[XE_CACHE_NONE_COMPRESSION]);
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/xe/xe_pat.h b/drivers/gpu/drm/xe/xe_pat.h
index 05dae03a5f54..d5dadfb7f924 100644
--- a/drivers/gpu/drm/xe/xe_pat.h
+++ b/drivers/gpu/drm/xe/xe_pat.h
@@ -49,6 +49,7 @@ void xe_pat_init_early(struct xe_device *xe);
 void xe_pat_init(struct xe_gt *gt);
 
 int xe_pat_dump(struct xe_gt *gt, struct drm_printer *p);
+int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p);
 
 /**
  * xe_pat_index_get_coh_mode - Extract the coherency mode for the given
@@ -58,4 +59,24 @@ int xe_pat_dump(struct xe_gt *gt, struct drm_printer *p);
  */
 u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index);
 
+/**
+ * xe_pat_index_get_comp_en - Extract the compression enable flag for
+ * the given pat_index.
+ * @xe: xe device
+ * @pat_index: The pat_index to query
+ *
+ * Return: true if compression is enabled for this pat_index, false otherwise.
+ */
+bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index);
+
+#define XE_L3_POLICY_WB		0 /* Write-back */
+#define XE_L3_POLICY_XD		1 /* WB - Transient Display */
+#define XE_L3_POLICY_UC		3 /* Uncached */
+/**
+ * xe_pat_index_get_l3_policy - Extract the L3 policy for the given pat_index.
+ * @xe: xe device
+ * @pat_index: The pat_index to query
+ */
+u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index);
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c
index 9c9ea10d994c..18d4e6b5c319 100644
--- a/drivers/gpu/drm/xe/xe_pci.c
+++ b/drivers/gpu/drm/xe/xe_pci.c
@@ -108,6 +108,7 @@ static const struct xe_graphics_desc graphics_xe2 = {
 
 static const struct xe_graphics_desc graphics_xe3p_xpc = {
 	XE2_GFX_FEATURES,
+	.has_indirect_ring_state = 1,
 	.hw_engine_mask =
 		GENMASK(XE_HW_ENGINE_BCS8, XE_HW_ENGINE_BCS1) |
 		GENMASK(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0),
@@ -168,6 +169,7 @@ static const struct xe_device_desc tgl_desc = {
 	.pre_gmdid_media_ip = &media_ip_xem,
 	PLATFORM(TIGERLAKE),
 	.dma_mask_size = 39,
+	.has_cached_pt = true,
 	.has_display = true,
 	.has_llc = true,
 	.has_sriov = true,
@@ -182,6 +184,7 @@ static const struct xe_device_desc rkl_desc = {
 	.pre_gmdid_media_ip = &media_ip_xem,
 	PLATFORM(ROCKETLAKE),
 	.dma_mask_size = 39,
+	.has_cached_pt = true,
 	.has_display = true,
 	.has_llc = true,
 	.max_gt_per_tile = 1,
@@ -197,6 +200,7 @@ static const struct xe_device_desc adl_s_desc = {
 	.pre_gmdid_media_ip = &media_ip_xem,
 	PLATFORM(ALDERLAKE_S),
 	.dma_mask_size = 39,
+	.has_cached_pt = true,
 	.has_display = true,
 	.has_llc = true,
 	.has_sriov = true,
@@ -217,6 +221,7 @@ static const struct xe_device_desc adl_p_desc = {
 	.pre_gmdid_media_ip = &media_ip_xem,
 	PLATFORM(ALDERLAKE_P),
 	.dma_mask_size = 39,
+	.has_cached_pt = true,
 	.has_display = true,
 	.has_llc = true,
 	.has_sriov = true,
@@ -235,6 +240,7 @@ static const struct xe_device_desc adl_n_desc = {
 	.pre_gmdid_media_ip = &media_ip_xem,
 	PLATFORM(ALDERLAKE_N),
 	.dma_mask_size = 39,
+	.has_cached_pt = true,
 	.has_display = true,
 	.has_llc = true,
 	.has_sriov = true,
@@ -361,7 +367,9 @@ static const struct xe_device_desc bmg_desc = {
 	.has_mbx_power_limits = true,
 	.has_gsc_nvm = 1,
 	.has_heci_cscfi = 1,
+	.has_i2c = true,
 	.has_late_bind = true,
+	.has_pre_prod_wa = 1,
 	.has_sriov = true,
 	.has_mem_copy_instr = true,
 	.max_gt_per_tile = 2,
@@ -381,6 +389,7 @@ static const struct xe_device_desc ptl_desc = {
 	.has_flat_ccs = 1,
 	.has_sriov = true,
 	.has_mem_copy_instr = true,
+	.has_pre_prod_wa = 1,
 	.max_gt_per_tile = 2,
 	.needs_scratch = true,
 	.needs_shared_vf_gt_wq = true,
@@ -394,6 +403,7 @@ static const struct xe_device_desc nvls_desc = {
 	.has_display = true,
 	.has_flat_ccs = 1,
 	.has_mem_copy_instr = true,
+	.has_pre_prod_wa = 1,
 	.max_gt_per_tile = 2,
 	.require_force_probe = true,
 	.va_bits = 48,
@@ -406,7 +416,11 @@ static const struct xe_device_desc cri_desc = {
 	.dma_mask_size = 52,
 	.has_display = false,
 	.has_flat_ccs = false,
+	.has_gsc_nvm = 1,
+	.has_i2c = true,
 	.has_mbx_power_limits = true,
+	.has_mert = true,
+	.has_pre_prod_wa = 1,
 	.has_sriov = true,
 	.max_gt_per_tile = 2,
 	.require_force_probe = true,
@@ -663,6 +677,7 @@ static int xe_info_init_early(struct xe_device *xe,
 	xe->info.vram_flags = desc->vram_flags;
 
 	xe->info.is_dgfx = desc->is_dgfx;
+	xe->info.has_cached_pt = desc->has_cached_pt;
 	xe->info.has_fan_control = desc->has_fan_control;
 	/* runtime fusing may force flat_ccs to disabled later */
 	xe->info.has_flat_ccs = desc->has_flat_ccs;
@@ -670,8 +685,12 @@ static int xe_info_init_early(struct xe_device *xe,
 	xe->info.has_gsc_nvm = desc->has_gsc_nvm;
 	xe->info.has_heci_gscfi = desc->has_heci_gscfi;
 	xe->info.has_heci_cscfi = desc->has_heci_cscfi;
+	xe->info.has_i2c = desc->has_i2c;
 	xe->info.has_late_bind = desc->has_late_bind;
 	xe->info.has_llc = desc->has_llc;
+	xe->info.has_mert = desc->has_mert;
+	xe->info.has_page_reclaim_hw_assist = desc->has_page_reclaim_hw_assist;
+	xe->info.has_pre_prod_wa = desc->has_pre_prod_wa;
 	xe->info.has_pxp = desc->has_pxp;
 	xe->info.has_sriov = xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev)) &&
 		desc->has_sriov;
@@ -755,6 +774,7 @@ static struct xe_gt *alloc_primary_gt(struct xe_tile *tile,
 	gt->info.type = XE_GT_TYPE_MAIN;
 	gt->info.id = tile->id * xe->info.max_gt_per_tile;
 	gt->info.has_indirect_ring_state = graphics_desc->has_indirect_ring_state;
+	gt->info.multi_queue_engine_class_mask = graphics_desc->multi_queue_engine_class_mask;
 	gt->info.engine_mask = graphics_desc->hw_engine_mask;
 
 	/*
@@ -1153,6 +1173,15 @@ static int xe_pci_runtime_suspend(struct device *dev)
 	struct xe_device *xe = pdev_to_xe_device(pdev);
 	int err;
 
+	/*
+	 * We hold an additional reference to the runtime PM to keep PF in D0
+	 * during VFs lifetime, as our VFs do not implement the PM capability.
+	 * This means we should never be runtime suspending as long as VFs are
+	 * enabled.
+	 */
+	xe_assert(xe, !IS_SRIOV_VF(xe));
+	xe_assert(xe, !pci_num_vf(pdev));
+
 	err = xe_pm_runtime_suspend(xe);
 	if (err)
 		return err;
diff --git a/drivers/gpu/drm/xe/xe_pci_sriov.c b/drivers/gpu/drm/xe/xe_pci_sriov.c
index 9ff69c4843b0..3fd22034f03e 100644
--- a/drivers/gpu/drm/xe/xe_pci_sriov.c
+++ b/drivers/gpu/drm/xe/xe_pci_sriov.c
@@ -219,7 +219,6 @@ static int pf_disable_vfs(struct xe_device *xe)
 int xe_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
 {
 	struct xe_device *xe = pdev_to_xe_device(pdev);
-	int ret;
 
 	if (!IS_SRIOV_PF(xe))
 		return -ENODEV;
@@ -233,14 +232,11 @@ int xe_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
 	if (num_vfs && pci_num_vf(pdev))
 		return -EBUSY;
 
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	if (num_vfs > 0)
-		ret = pf_enable_vfs(xe, num_vfs);
+		return pf_enable_vfs(xe, num_vfs);
 	else
-		ret = pf_disable_vfs(xe);
-	xe_pm_runtime_put(xe);
-
-	return ret;
+		return pf_disable_vfs(xe);
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h
index 9892c063a9c5..3bb51d155951 100644
--- a/drivers/gpu/drm/xe/xe_pci_types.h
+++ b/drivers/gpu/drm/xe/xe_pci_types.h
@@ -37,16 +37,21 @@ struct xe_device_desc {
 	u8 require_force_probe:1;
 	u8 is_dgfx:1;
 
+	u8 has_cached_pt:1;
 	u8 has_display:1;
 	u8 has_fan_control:1;
 	u8 has_flat_ccs:1;
 	u8 has_gsc_nvm:1;
 	u8 has_heci_gscfi:1;
 	u8 has_heci_cscfi:1;
+	u8 has_i2c:1;
 	u8 has_late_bind:1;
 	u8 has_llc:1;
 	u8 has_mbx_power_limits:1;
 	u8 has_mem_copy_instr:1;
+	u8 has_mert:1;
+	u8 has_pre_prod_wa:1;
+	u8 has_page_reclaim_hw_assist:1;
 	u8 has_pxp:1;
 	u8 has_sriov:1;
 	u8 needs_scratch:1;
@@ -58,6 +63,7 @@ struct xe_device_desc {
 
 struct xe_graphics_desc {
 	u64 hw_engine_mask;	/* hardware engines provided by graphics IP */
+	u16 multi_queue_engine_class_mask; /* bitmask of engine classes which support multi queue */
 
 	u8 has_asid:1;
 	u8 has_atomic_enable_pte_bit:1;
diff --git a/drivers/gpu/drm/xe/xe_pcode_api.h b/drivers/gpu/drm/xe/xe_pcode_api.h
index 70dcd6625680..975892d6b230 100644
--- a/drivers/gpu/drm/xe/xe_pcode_api.h
+++ b/drivers/gpu/drm/xe/xe_pcode_api.h
@@ -77,11 +77,13 @@
 
 #define PCODE_SCRATCH(x)		XE_REG(0x138320 + ((x) * 4))
 /* PCODE_SCRATCH0 */
+#define   BREADCRUMB_VERSION		REG_GENMASK(31, 29)
 #define   AUXINFO_REG_OFFSET		REG_GENMASK(17, 15)
 #define   OVERFLOW_REG_OFFSET		REG_GENMASK(14, 12)
 #define   HISTORY_TRACKING		REG_BIT(11)
 #define   OVERFLOW_SUPPORT		REG_BIT(10)
 #define   AUXINFO_SUPPORT		REG_BIT(9)
+#define   FDO_MODE			REG_BIT(4)
 #define   BOOT_STATUS			REG_GENMASK(3, 1)
 #define      CRITICAL_FAILURE		4
 #define      NON_CRITICAL_FAILURE	7
diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c
index 766922530265..4390ba69610d 100644
--- a/drivers/gpu/drm/xe/xe_pm.c
+++ b/drivers/gpu/drm/xe/xe_pm.c
@@ -591,7 +591,7 @@ int xe_pm_runtime_suspend(struct xe_device *xe)
 	}
 
 	for_each_gt(gt, xe, id) {
-		err = xe_gt_suspend(gt);
+		err = xe->d3cold.allowed ? xe_gt_suspend(gt) : xe_gt_runtime_suspend(gt);
 		if (err)
 			goto out_resume;
 	}
@@ -633,10 +633,10 @@ int xe_pm_runtime_resume(struct xe_device *xe)
 
 	xe_rpm_lockmap_acquire(xe);
 
-	for_each_gt(gt, xe, id)
-		xe_gt_idle_disable_c6(gt);
-
 	if (xe->d3cold.allowed) {
+		for_each_gt(gt, xe, id)
+			xe_gt_idle_disable_c6(gt);
+
 		err = xe_pcode_ready(xe, true);
 		if (err)
 			goto out;
@@ -657,7 +657,7 @@ int xe_pm_runtime_resume(struct xe_device *xe)
 	xe_irq_resume(xe);
 
 	for_each_gt(gt, xe, id)
-		xe_gt_resume(gt);
+		xe->d3cold.allowed ? xe_gt_resume(gt) : xe_gt_runtime_resume(gt);
 
 	xe_display_pm_runtime_resume(xe);
 
diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index 884127b4d97d..6cd78bb2b652 100644
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -12,6 +12,7 @@
 #include "xe_exec_queue.h"
 #include "xe_gt.h"
 #include "xe_migrate.h"
+#include "xe_page_reclaim.h"
 #include "xe_pt_types.h"
 #include "xe_pt_walk.h"
 #include "xe_res_cursor.h"
@@ -1535,6 +1536,9 @@ struct xe_pt_stage_unbind_walk {
 	/** @modified_end: Walk range start, modified like @modified_start. */
 	u64 modified_end;
 
+	/** @prl: Backing pointer to page reclaim list in pt_update_ops */
+	struct xe_page_reclaim_list *prl;
+
 	/* Output */
 	/* @wupd: Structure to track the page-table updates we're building */
 	struct xe_walk_update wupd;
@@ -1572,6 +1576,68 @@ static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level,
 	return false;
 }
 
+/* Huge 2MB leaf lives directly in a level-1 table and has no children */
+static bool is_2m_pte(struct xe_pt *pte)
+{
+	return pte->level == 1 && !pte->base.children;
+}
+
+/* page_size = 2^(reclamation_size + XE_PTE_SHIFT) */
+#define COMPUTE_RECLAIM_ADDRESS_MASK(page_size)				\
+({									\
+	BUILD_BUG_ON(!__builtin_constant_p(page_size));			\
+	ilog2(page_size) - XE_PTE_SHIFT;				\
+})
+
+static int generate_reclaim_entry(struct xe_tile *tile,
+				  struct xe_page_reclaim_list *prl,
+				  u64 pte, struct xe_pt *xe_child)
+{
+	struct xe_guc_page_reclaim_entry *reclaim_entries = prl->entries;
+	u64 phys_page = (pte & XE_PTE_ADDR_MASK) >> XE_PTE_SHIFT;
+	int num_entries = prl->num_entries;
+	u32 reclamation_size;
+
+	xe_tile_assert(tile, xe_child->level <= MAX_HUGEPTE_LEVEL);
+	xe_tile_assert(tile, reclaim_entries);
+	xe_tile_assert(tile, num_entries < XE_PAGE_RECLAIM_MAX_ENTRIES - 1);
+
+	if (!xe_page_reclaim_list_valid(prl))
+		return -EINVAL;
+
+	/**
+	 * reclamation_size indicates the size of the page to be
+	 * invalidated and flushed from non-coherent cache.
+	 * Page size is computed as 2^(reclamation_size + XE_PTE_SHIFT) bytes.
+	 * Only 4K, 64K (level 0), and 2M pages are supported by hardware for page reclaim
+	 */
+	if (xe_child->level == 0 && !(pte & XE_PTE_PS64)) {
+		reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_4K);  /* reclamation_size = 0 */
+	} else if (xe_child->level == 0) {
+		reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_64K); /* reclamation_size = 4 */
+	} else if (is_2m_pte(xe_child)) {
+		reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_2M);  /* reclamation_size = 9 */
+	} else {
+		xe_page_reclaim_list_invalidate(prl);
+		vm_dbg(&tile_to_xe(tile)->drm,
+		       "PRL invalidate: unsupported PTE level=%u pte=%#llx\n",
+		       xe_child->level, pte);
+		return -EINVAL;
+	}
+
+	reclaim_entries[num_entries].qw =
+		FIELD_PREP(XE_PAGE_RECLAIM_VALID, 1) |
+		FIELD_PREP(XE_PAGE_RECLAIM_SIZE, reclamation_size) |
+		FIELD_PREP(XE_PAGE_RECLAIM_ADDR_LO, phys_page) |
+		FIELD_PREP(XE_PAGE_RECLAIM_ADDR_HI, phys_page >> 20);
+	prl->num_entries++;
+	vm_dbg(&tile_to_xe(tile)->drm,
+	       "PRL add entry: level=%u pte=%#llx reclamation_size=%u prl_idx=%d\n",
+	       xe_child->level, pte, reclamation_size, num_entries);
+
+	return 0;
+}
+
 static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset,
 				    unsigned int level, u64 addr, u64 next,
 				    struct xe_ptw **child,
@@ -1579,11 +1645,48 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset,
 				    struct xe_pt_walk *walk)
 {
 	struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
+	struct xe_pt_stage_unbind_walk *xe_walk =
+		container_of(walk, typeof(*xe_walk), base);
+	struct xe_device *xe = tile_to_xe(xe_walk->tile);
 
 	XE_WARN_ON(!*child);
 	XE_WARN_ON(!level);
+	/* Check for leaf node */
+	if (xe_walk->prl && xe_page_reclaim_list_valid(xe_walk->prl) &&
+	    !xe_child->base.children) {
+		struct iosys_map *leaf_map = &xe_child->bo->vmap;
+		pgoff_t first = xe_pt_offset(addr, 0, walk);
+		pgoff_t count = xe_pt_num_entries(addr, next, 0, walk);
+
+		for (pgoff_t i = 0; i < count; i++) {
+			u64 pte = xe_map_rd(xe, leaf_map, (first + i) * sizeof(u64), u64);
+			int ret;
+
+			/* Account for NULL terminated entry on end (-1) */
+			if (xe_walk->prl->num_entries < XE_PAGE_RECLAIM_MAX_ENTRIES - 1) {
+				ret = generate_reclaim_entry(xe_walk->tile, xe_walk->prl,
+							     pte, xe_child);
+				if (ret)
+					break;
+			} else {
+				/* overflow, mark as invalid */
+				xe_page_reclaim_list_invalidate(xe_walk->prl);
+				vm_dbg(&xe->drm,
+				       "PRL invalidate: overflow while adding pte=%#llx",
+				       pte);
+				break;
+			}
+		}
+	}
 
-	xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk);
+	/* If aborting page walk early, invalidate PRL since PTE may be dropped from this abort */
+	if (xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk) &&
+	    xe_walk->prl && level > 1 && xe_child->base.children && xe_child->num_live != 0) {
+		xe_page_reclaim_list_invalidate(xe_walk->prl);
+		vm_dbg(&xe->drm,
+		       "PRL invalidate: kill at level=%u addr=%#llx next=%#llx num_live=%u\n",
+		       level, addr, next, xe_child->num_live);
+	}
 
 	return 0;
 }
@@ -1654,6 +1757,8 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile,
 {
 	u64 start = range ? xe_svm_range_start(range) : xe_vma_start(vma);
 	u64 end = range ? xe_svm_range_end(range) : xe_vma_end(vma);
+	struct xe_vm_pgtable_update_op *pt_update_op =
+		container_of(entries, struct xe_vm_pgtable_update_op, entries[0]);
 	struct xe_pt_stage_unbind_walk xe_walk = {
 		.base = {
 			.ops = &xe_pt_stage_unbind_ops,
@@ -1665,6 +1770,7 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile,
 		.modified_start = start,
 		.modified_end = end,
 		.wupd.entries = entries,
+		.prl = pt_update_op->prl,
 	};
 	struct xe_pt *pt = vm->pt_root[tile->id];
 
@@ -1897,6 +2003,7 @@ static int unbind_op_prepare(struct xe_tile *tile,
 			     struct xe_vm_pgtable_update_ops *pt_update_ops,
 			     struct xe_vma *vma)
 {
+	struct xe_device *xe = tile_to_xe(tile);
 	u32 current_op = pt_update_ops->current_op;
 	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
 	int err;
@@ -1914,6 +2021,17 @@ static int unbind_op_prepare(struct xe_tile *tile,
 	pt_op->vma = vma;
 	pt_op->bind = false;
 	pt_op->rebind = false;
+	/*
+	 * Maintain one PRL located in pt_update_ops that all others in unbind op reference.
+	 * Ensure that PRL is allocated only once, and if invalidated, remains an invalidated PRL.
+	 */
+	if (xe->info.has_page_reclaim_hw_assist &&
+	    xe_page_reclaim_list_is_new(&pt_update_ops->prl))
+		xe_page_reclaim_list_alloc_entries(&pt_update_ops->prl);
+
+	/* Page reclaim may not be needed due to other features, so skip the corresponding VMA */
+	pt_op->prl = (xe_page_reclaim_list_valid(&pt_update_ops->prl) &&
+		     !xe_page_reclaim_skip(tile, vma)) ? &pt_update_ops->prl : NULL;
 
 	err = vma_reserve_fences(tile_to_xe(tile), vma);
 	if (err)
@@ -1979,6 +2097,7 @@ static int unbind_range_prepare(struct xe_vm *vm,
 	pt_op->vma = XE_INVALID_VMA;
 	pt_op->bind = false;
 	pt_op->rebind = false;
+	pt_op->prl = NULL;
 
 	pt_op->num_entries = xe_pt_stage_unbind(tile, vm, NULL, range,
 						pt_op->entries);
@@ -2096,6 +2215,7 @@ xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops)
 	init_llist_head(&pt_update_ops->deferred);
 	pt_update_ops->start = ~0x0ull;
 	pt_update_ops->last = 0x0ull;
+	xe_page_reclaim_list_init(&pt_update_ops->prl);
 }
 
 /**
@@ -2393,6 +2513,17 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
 			goto kill_vm_tile1;
 		}
 		update.ijob = ijob;
+		/*
+		 * Only add page reclaim for the primary GT. Media GT does not have
+		 * any PPC to flush, so enabling the PPC flush bit for media is
+		 * effectively a NOP and provides no performance benefit nor
+		 * interfere with primary GT.
+		 */
+		if (xe_page_reclaim_list_valid(&pt_update_ops->prl)) {
+			xe_tlb_inval_job_add_page_reclaim(ijob, &pt_update_ops->prl);
+			/* Release ref from alloc, job will now handle it */
+			xe_page_reclaim_list_invalidate(&pt_update_ops->prl);
+		}
 
 		if (tile->media_gt) {
 			dep_scheduler = to_dep_scheduler(q, tile->media_gt);
@@ -2518,6 +2649,8 @@ void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops)
 		&vops->pt_update_ops[tile->id];
 	int i;
 
+	xe_page_reclaim_entries_put(pt_update_ops->prl.entries);
+
 	lockdep_assert_held(&vops->vm->lock);
 	xe_vm_assert_held(vops->vm);
 
diff --git a/drivers/gpu/drm/xe/xe_pt_types.h b/drivers/gpu/drm/xe/xe_pt_types.h
index 881f01e14db8..88fabf8e2655 100644
--- a/drivers/gpu/drm/xe/xe_pt_types.h
+++ b/drivers/gpu/drm/xe/xe_pt_types.h
@@ -8,6 +8,7 @@
 
 #include <linux/types.h>
 
+#include "xe_page_reclaim.h"
 #include "xe_pt_walk.h"
 
 struct xe_bo;
@@ -79,6 +80,8 @@ struct xe_vm_pgtable_update_op {
 	struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1];
 	/** @vma: VMA for operation, operation not valid if NULL */
 	struct xe_vma *vma;
+	/** @prl: Backing pointer to page reclaim list of pt_update_ops */
+	struct xe_page_reclaim_list *prl;
 	/** @num_entries: number of entries for this update operation */
 	u32 num_entries;
 	/** @bind: is a bind */
@@ -95,6 +98,8 @@ struct xe_vm_pgtable_update_ops {
 	struct llist_head deferred;
 	/** @q: exec queue for PT operations */
 	struct xe_exec_queue *q;
+	/** @prl: embedded page reclaim list */
+	struct xe_page_reclaim_list prl;
 	/** @start: start address of ops */
 	u64 start;
 	/** @last: last address of ops */
diff --git a/drivers/gpu/drm/xe/xe_pxp.c b/drivers/gpu/drm/xe/xe_pxp.c
index bdbdbbf6a678..508f4c128a48 100644
--- a/drivers/gpu/drm/xe/xe_pxp.c
+++ b/drivers/gpu/drm/xe/xe_pxp.c
@@ -58,10 +58,9 @@ bool xe_pxp_is_enabled(const struct xe_pxp *pxp)
 static bool pxp_prerequisites_done(const struct xe_pxp *pxp)
 {
 	struct xe_gt *gt = pxp->gt;
-	unsigned int fw_ref;
 	bool ready;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 
 	/*
 	 * If force_wake fails we could falsely report the prerequisites as not
@@ -71,14 +70,12 @@ static bool pxp_prerequisites_done(const struct xe_pxp *pxp)
 	 * PXP. Therefore, we can just log the force_wake error and not escalate
 	 * it.
 	 */
-	XE_WARN_ON(!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL));
+	XE_WARN_ON(!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL));
 
 	/* PXP requires both HuC authentication via GSC and GSC proxy initialized */
 	ready = xe_huc_is_authenticated(&gt->uc.huc, XE_HUC_AUTH_VIA_GSC) &&
 		xe_gsc_proxy_init_done(&gt->uc.gsc);
 
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-
 	return ready;
 }
 
@@ -104,13 +101,12 @@ int xe_pxp_get_readiness_status(struct xe_pxp *pxp)
 	    xe_uc_fw_status_to_error(pxp->gt->uc.gsc.fw.status))
 		return -EIO;
 
-	xe_pm_runtime_get(pxp->xe);
+	guard(xe_pm_runtime)(pxp->xe);
 
 	/* PXP requires both HuC loaded and GSC proxy initialized */
 	if (pxp_prerequisites_done(pxp))
 		ret = 1;
 
-	xe_pm_runtime_put(pxp->xe);
 	return ret;
 }
 
@@ -135,35 +131,28 @@ static void pxp_invalidate_queues(struct xe_pxp *pxp);
 static int pxp_terminate_hw(struct xe_pxp *pxp)
 {
 	struct xe_gt *gt = pxp->gt;
-	unsigned int fw_ref;
 	int ret = 0;
 
 	drm_dbg(&pxp->xe->drm, "Terminating PXP\n");
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT)) {
-		ret = -EIO;
-		goto out;
-	}
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
+		return -EIO;
 
 	/* terminate the hw session */
 	ret = xe_pxp_submit_session_termination(pxp, ARB_SESSION);
 	if (ret)
-		goto out;
+		return ret;
 
 	ret = pxp_wait_for_session_state(pxp, ARB_SESSION, false);
 	if (ret)
-		goto out;
+		return ret;
 
 	/* Trigger full HW cleanup */
 	xe_mmio_write32(&gt->mmio, KCR_GLOBAL_TERMINATE, 1);
 
 	/* now we can tell the GSC to clean up its own state */
-	ret = xe_pxp_submit_session_invalidation(&pxp->gsc_res, ARB_SESSION);
-
-out:
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-	return ret;
+	return xe_pxp_submit_session_invalidation(&pxp->gsc_res, ARB_SESSION);
 }
 
 static void mark_termination_in_progress(struct xe_pxp *pxp)
@@ -326,14 +315,12 @@ static int kcr_pxp_set_status(const struct xe_pxp *pxp, bool enable)
 {
 	u32 val = enable ? _MASKED_BIT_ENABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES) :
 		  _MASKED_BIT_DISABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES);
-	unsigned int fw_ref;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(pxp->gt), XE_FW_GT);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT))
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(pxp->gt), XE_FW_GT);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
 		return -EIO;
 
 	xe_mmio_write32(&pxp->gt->mmio, KCR_INIT, val);
-	xe_force_wake_put(gt_to_fw(pxp->gt), fw_ref);
 
 	return 0;
 }
@@ -453,34 +440,28 @@ int xe_pxp_init(struct xe_device *xe)
 static int __pxp_start_arb_session(struct xe_pxp *pxp)
 {
 	int ret;
-	unsigned int fw_ref;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(pxp->gt), XE_FW_GT);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT))
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(pxp->gt), XE_FW_GT);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
 		return -EIO;
 
-	if (pxp_session_is_in_play(pxp, ARB_SESSION)) {
-		ret = -EEXIST;
-		goto out_force_wake;
-	}
+	if (pxp_session_is_in_play(pxp, ARB_SESSION))
+		return -EEXIST;
 
 	ret = xe_pxp_submit_session_init(&pxp->gsc_res, ARB_SESSION);
 	if (ret) {
 		drm_err(&pxp->xe->drm, "Failed to init PXP arb session: %pe\n", ERR_PTR(ret));
-		goto out_force_wake;
+		return ret;
 	}
 
 	ret = pxp_wait_for_session_state(pxp, ARB_SESSION, true);
 	if (ret) {
 		drm_err(&pxp->xe->drm, "PXP ARB session failed to go in play%pe\n", ERR_PTR(ret));
-		goto out_force_wake;
+		return ret;
 	}
 
 	drm_dbg(&pxp->xe->drm, "PXP ARB session is active\n");
-
-out_force_wake:
-	xe_force_wake_put(gt_to_fw(pxp->gt), fw_ref);
-	return ret;
+	return 0;
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c
index 1c0915e2cc16..75490683bad2 100644
--- a/drivers/gpu/drm/xe/xe_query.c
+++ b/drivers/gpu/drm/xe/xe_query.c
@@ -122,7 +122,6 @@ query_engine_cycles(struct xe_device *xe,
 	__ktime_func_t cpu_clock;
 	struct xe_hw_engine *hwe;
 	struct xe_gt *gt;
-	unsigned int fw_ref;
 
 	if (IS_SRIOV_VF(xe))
 		return -EOPNOTSUPP;
@@ -158,16 +157,13 @@ query_engine_cycles(struct xe_device *xe,
 	if (!hwe)
 		return -EINVAL;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))  {
-		xe_force_wake_put(gt_to_fw(gt), fw_ref);
-		return -EIO;
-	}
-
-	hwe_read_timestamp(hwe, &resp.engine_cycles, &resp.cpu_timestamp,
-			   &resp.cpu_delta, cpu_clock);
+	xe_with_force_wake(fw_ref, gt_to_fw(gt), XE_FORCEWAKE_ALL) {
+		if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
+			return -EIO;
 
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
+		hwe_read_timestamp(hwe, &resp.engine_cycles, &resp.cpu_timestamp,
+				   &resp.cpu_delta, cpu_clock);
+	}
 
 	if (GRAPHICS_VER(xe) >= 20)
 		resp.width = 64;
@@ -342,6 +338,9 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query)
 	if (xe->info.has_usm && IS_ENABLED(CONFIG_DRM_XE_GPUSVM))
 		config->info[DRM_XE_QUERY_CONFIG_FLAGS] |=
 			DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR;
+	if (GRAPHICS_VER(xe) >= 20)
+		config->info[DRM_XE_QUERY_CONFIG_FLAGS] |=
+			DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT;
 	config->info[DRM_XE_QUERY_CONFIG_FLAGS] |=
 			DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY;
 	config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT] =
@@ -686,7 +685,9 @@ static int query_oa_units(struct xe_device *xe,
 			du->capabilities = DRM_XE_OA_CAPS_BASE | DRM_XE_OA_CAPS_SYNCS |
 					   DRM_XE_OA_CAPS_OA_BUFFER_SIZE |
 					   DRM_XE_OA_CAPS_WAIT_NUM_REPORTS |
-					   DRM_XE_OA_CAPS_OAM;
+					   DRM_XE_OA_CAPS_OAM |
+					   DRM_XE_OA_CAPS_OA_UNIT_GT_ID;
+			du->gt_id = u->gt->info.id;
 			j = 0;
 			for_each_hw_engine(hwe, gt, hwe_id) {
 				if (!xe_hw_engine_is_reserved(hwe) &&
diff --git a/drivers/gpu/drm/xe/xe_reg_sr.c b/drivers/gpu/drm/xe/xe_reg_sr.c
index fc8447a838c4..1a465385f909 100644
--- a/drivers/gpu/drm/xe/xe_reg_sr.c
+++ b/drivers/gpu/drm/xe/xe_reg_sr.c
@@ -168,7 +168,6 @@ void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt)
 {
 	struct xe_reg_sr_entry *entry;
 	unsigned long reg;
-	unsigned int fw_ref;
 
 	if (xa_empty(&sr->xa))
 		return;
@@ -178,20 +177,14 @@ void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt)
 
 	xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name);
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
-		goto err_force_wake;
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) {
+		xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n");
+		return;
+	}
 
 	xa_for_each(&sr->xa, reg, entry)
 		apply_one_mmio(gt, entry);
-
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-
-	return;
-
-err_force_wake:
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-	xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n");
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.c b/drivers/gpu/drm/xe/xe_reg_whitelist.c
index 7ca360b2c20d..1391cb6ec9c6 100644
--- a/drivers/gpu/drm/xe/xe_reg_whitelist.c
+++ b/drivers/gpu/drm/xe/xe_reg_whitelist.c
@@ -9,6 +9,7 @@
 #include "regs/xe_gt_regs.h"
 #include "regs/xe_oa_regs.h"
 #include "regs/xe_regs.h"
+#include "xe_device.h"
 #include "xe_gt_types.h"
 #include "xe_gt_printk.h"
 #include "xe_platform_types.h"
@@ -26,6 +27,13 @@ static bool match_not_render(const struct xe_device *xe,
 	return hwe->class != XE_ENGINE_CLASS_RENDER;
 }
 
+static bool match_has_mert(const struct xe_device *xe,
+			   const struct xe_gt *gt,
+			   const struct xe_hw_engine *hwe)
+{
+	return xe_device_has_mert((struct xe_device *)xe);
+}
+
 static const struct xe_rtp_entry_sr register_whitelist[] = {
 	{ XE_RTP_NAME("WaAllowPMDepthAndInvocationCountAccessFromUMD, 1408556865"),
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)),
@@ -67,28 +75,6 @@ static const struct xe_rtp_entry_sr register_whitelist[] = {
 		       ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(WHITELIST(CSBE_DEBUG_STATUS(RENDER_RING_BASE), 0))
 	},
-	{ XE_RTP_NAME("oa_reg_render"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED),
-		       ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(WHITELIST(OAG_MMIOTRIGGER,
-				   RING_FORCE_TO_NONPRIV_ACCESS_RW),
-			 WHITELIST(OAG_OASTATUS,
-				   RING_FORCE_TO_NONPRIV_ACCESS_RD),
-			 WHITELIST(OAG_OAHEADPTR,
-				   RING_FORCE_TO_NONPRIV_ACCESS_RD |
-				   RING_FORCE_TO_NONPRIV_RANGE_4))
-	},
-	{ XE_RTP_NAME("oa_reg_compute"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED),
-		       ENGINE_CLASS(COMPUTE)),
-	  XE_RTP_ACTIONS(WHITELIST(OAG_MMIOTRIGGER,
-				   RING_FORCE_TO_NONPRIV_ACCESS_RW),
-			 WHITELIST(OAG_OASTATUS,
-				   RING_FORCE_TO_NONPRIV_ACCESS_RD),
-			 WHITELIST(OAG_OAHEADPTR,
-				   RING_FORCE_TO_NONPRIV_ACCESS_RD |
-				   RING_FORCE_TO_NONPRIV_RANGE_4))
-	},
 	{ XE_RTP_NAME("14024997852"),
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(WHITELIST(FF_MODE,
@@ -96,6 +82,57 @@ static const struct xe_rtp_entry_sr register_whitelist[] = {
 			 WHITELIST(VFLSKPD,
 				   RING_FORCE_TO_NONPRIV_ACCESS_RW))
 	},
+
+#define WHITELIST_OA_MMIO_TRG(trg, status, head) \
+	WHITELIST(trg, RING_FORCE_TO_NONPRIV_ACCESS_RW), \
+	WHITELIST(status, RING_FORCE_TO_NONPRIV_ACCESS_RD), \
+	WHITELIST(head, RING_FORCE_TO_NONPRIV_ACCESS_RD | RING_FORCE_TO_NONPRIV_RANGE_4)
+
+#define WHITELIST_OAG_MMIO_TRG \
+	WHITELIST_OA_MMIO_TRG(OAG_MMIOTRIGGER, OAG_OASTATUS, OAG_OAHEADPTR)
+
+#define WHITELIST_OAM_MMIO_TRG \
+	WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SAG_BASE_ADJ), \
+			      OAM_STATUS(XE_OAM_SAG_BASE_ADJ), \
+			      OAM_HEAD_POINTER(XE_OAM_SAG_BASE_ADJ)), \
+	WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SCMI_0_BASE_ADJ), \
+			      OAM_STATUS(XE_OAM_SCMI_0_BASE_ADJ), \
+			      OAM_HEAD_POINTER(XE_OAM_SCMI_0_BASE_ADJ)), \
+	WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SCMI_1_BASE_ADJ), \
+			      OAM_STATUS(XE_OAM_SCMI_1_BASE_ADJ), \
+			      OAM_HEAD_POINTER(XE_OAM_SCMI_1_BASE_ADJ))
+
+#define WHITELIST_OA_MERT_MMIO_TRG \
+	WHITELIST_OA_MMIO_TRG(OAMERT_MMIO_TRG, OAMERT_STATUS, OAMERT_HEAD_POINTER)
+
+	{ XE_RTP_NAME("oag_mmio_trg_rcs"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED),
+		       ENGINE_CLASS(RENDER)),
+	  XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG)
+	},
+	{ XE_RTP_NAME("oag_mmio_trg_ccs"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED),
+		       ENGINE_CLASS(COMPUTE)),
+	  XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG)
+	},
+	{ XE_RTP_NAME("oam_mmio_trg_vcs"),
+	  XE_RTP_RULES(MEDIA_VERSION_RANGE(1300, XE_RTP_END_VERSION_UNDEFINED),
+		       ENGINE_CLASS(VIDEO_DECODE)),
+	  XE_RTP_ACTIONS(WHITELIST_OAM_MMIO_TRG)
+	},
+	{ XE_RTP_NAME("oam_mmio_trg_vecs"),
+	  XE_RTP_RULES(MEDIA_VERSION_RANGE(1300, XE_RTP_END_VERSION_UNDEFINED),
+		       ENGINE_CLASS(VIDEO_ENHANCE)),
+	  XE_RTP_ACTIONS(WHITELIST_OAM_MMIO_TRG)
+	},
+	{ XE_RTP_NAME("oa_mert_mmio_trg_ccs"),
+	  XE_RTP_RULES(FUNC(match_has_mert), ENGINE_CLASS(COMPUTE)),
+	  XE_RTP_ACTIONS(WHITELIST_OA_MERT_MMIO_TRG)
+	},
+	{ XE_RTP_NAME("oa_mert_mmio_trg_bcs"),
+	  XE_RTP_RULES(FUNC(match_has_mert), ENGINE_CLASS(COPY)),
+	  XE_RTP_ACTIONS(WHITELIST_OA_MERT_MMIO_TRG)
+	},
 };
 
 static void whitelist_apply_to_hwe(struct xe_hw_engine *hwe)
diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c
index ac0c6dcffe15..957b9e2fd138 100644
--- a/drivers/gpu/drm/xe/xe_ring_ops.c
+++ b/drivers/gpu/drm/xe/xe_ring_ops.c
@@ -12,7 +12,7 @@
 #include "regs/xe_engine_regs.h"
 #include "regs/xe_gt_regs.h"
 #include "regs/xe_lrc_layout.h"
-#include "xe_exec_queue_types.h"
+#include "xe_exec_queue.h"
 #include "xe_gt.h"
 #include "xe_lrc.h"
 #include "xe_macros.h"
@@ -135,12 +135,11 @@ emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset,
 	return i;
 }
 
-static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw,
-				int i)
+static int emit_pipe_invalidate(struct xe_exec_queue *q, u32 mask_flags,
+				bool invalidate_tlb, u32 *dw, int i)
 {
 	u32 flags0 = 0;
-	u32 flags1 = PIPE_CONTROL_CS_STALL |
-		PIPE_CONTROL_COMMAND_CACHE_INVALIDATE |
+	u32 flags1 = PIPE_CONTROL_COMMAND_CACHE_INVALIDATE |
 		PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE |
 		PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
 		PIPE_CONTROL_VF_CACHE_INVALIDATE |
@@ -152,6 +151,11 @@ static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw,
 	if (invalidate_tlb)
 		flags1 |= PIPE_CONTROL_TLB_INVALIDATE;
 
+	if (xe_exec_queue_is_multi_queue(q))
+		flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE;
+	else
+		flags1 |= PIPE_CONTROL_CS_STALL;
+
 	flags1 &= ~mask_flags;
 
 	if (flags1 & PIPE_CONTROL_VF_CACHE_INVALIDATE)
@@ -175,54 +179,52 @@ static int emit_store_imm_ppgtt_posted(u64 addr, u64 value,
 
 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i)
 {
-	struct xe_gt *gt = job->q->gt;
+	struct xe_exec_queue *q = job->q;
+	struct xe_gt *gt = q->gt;
 	bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
-	u32 flags;
+	u32 flags0, flags1;
 
 	if (XE_GT_WA(gt, 14016712196))
 		i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH,
 				      LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0);
 
-	flags = (PIPE_CONTROL_CS_STALL |
-		 PIPE_CONTROL_TILE_CACHE_FLUSH |
+	flags0 = PIPE_CONTROL0_HDC_PIPELINE_FLUSH;
+	flags1 = (PIPE_CONTROL_TILE_CACHE_FLUSH |
 		 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
 		 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
 		 PIPE_CONTROL_DC_FLUSH_ENABLE |
 		 PIPE_CONTROL_FLUSH_ENABLE);
 
 	if (XE_GT_WA(gt, 1409600907))
-		flags |= PIPE_CONTROL_DEPTH_STALL;
+		flags1 |= PIPE_CONTROL_DEPTH_STALL;
 
 	if (lacks_render)
-		flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
+		flags1 &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
 	else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
-		flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
+		flags1 &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
 
-	return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0);
-}
-
-static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i)
-{
-	if (hwe->class != XE_ENGINE_CLASS_RENDER)
-		return i;
+	if (xe_exec_queue_is_multi_queue(q))
+		flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE;
+	else
+		flags1 |= PIPE_CONTROL_CS_STALL;
 
-	if (XE_GT_WA(hwe->gt, 16020292621))
-		i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC,
-				      RING_NOPID(hwe->mmio_base).addr, 0);
-
-	return i;
+	return emit_pipe_control(dw, i, flags0, flags1, 0, 0);
 }
 
-static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw,
-			      int i)
+static int emit_pipe_imm_ggtt(struct xe_exec_queue *q, u32 addr, u32 value,
+			      bool stall_only, u32 *dw, int i)
 {
-	u32 flags = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_GLOBAL_GTT_IVB |
-		    PIPE_CONTROL_QW_WRITE;
+	u32 flags0 = 0, flags1 = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_QW_WRITE;
 
 	if (!stall_only)
-		flags |= PIPE_CONTROL_FLUSH_ENABLE;
+		flags1 |= PIPE_CONTROL_FLUSH_ENABLE;
 
-	return emit_pipe_control(dw, i, 0, flags, addr, value);
+	if (xe_exec_queue_is_multi_queue(q))
+		flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE;
+	else
+		flags1 |= PIPE_CONTROL_CS_STALL;
+
+	return emit_pipe_control(dw, i, flags0, flags1, addr, value);
 }
 
 static u32 get_ppgtt_flag(struct xe_sched_job *job)
@@ -371,7 +373,7 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job,
 		mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS;
 
 	/* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */
-	i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i);
+	i = emit_pipe_invalidate(job->q, mask_flags, job->ring_ops_flush_tlb, dw, i);
 
 	/* hsdes: 1809175790 */
 	if (has_aux_ccs(xe))
@@ -391,12 +393,10 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job,
 						job->user_fence.value,
 						dw, i);
 
-	i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);
+	i = emit_pipe_imm_ggtt(job->q, xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);
 
 	i = emit_user_interrupt(dw, i);
 
-	i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i);
-
 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
 
 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
diff --git a/drivers/gpu/drm/xe/xe_sa.c b/drivers/gpu/drm/xe/xe_sa.c
index 63a5263dcf1b..a87c1436c7c1 100644
--- a/drivers/gpu/drm/xe/xe_sa.c
+++ b/drivers/gpu/drm/xe/xe_sa.c
@@ -29,6 +29,7 @@ static void xe_sa_bo_manager_fini(struct drm_device *drm, void *arg)
 		kvfree(sa_manager->cpu_ptr);
 
 	sa_manager->bo = NULL;
+	sa_manager->shadow = NULL;
 }
 
 /**
@@ -37,12 +38,14 @@ static void xe_sa_bo_manager_fini(struct drm_device *drm, void *arg)
  * @size: number of bytes to allocate
  * @guard: number of bytes to exclude from suballocations
  * @align: alignment for each suballocated chunk
+ * @flags: flags for suballocator
  *
  * Prepares the suballocation manager for suballocations.
  *
  * Return: a pointer to the &xe_sa_manager or an ERR_PTR on failure.
  */
-struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u32 guard, u32 align)
+struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size,
+					      u32 guard, u32 align, u32 flags)
 {
 	struct xe_device *xe = tile_to_xe(tile);
 	struct xe_sa_manager *sa_manager;
@@ -79,6 +82,26 @@ struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u3
 		memset(sa_manager->cpu_ptr, 0, bo->ttm.base.size);
 	}
 
+	if (flags & XE_SA_BO_MANAGER_FLAG_SHADOW) {
+		struct xe_bo *shadow;
+
+		ret = drmm_mutex_init(&xe->drm, &sa_manager->swap_guard);
+		if (ret)
+			return ERR_PTR(ret);
+
+		shadow = xe_managed_bo_create_pin_map(xe, tile, size,
+						      XE_BO_FLAG_VRAM_IF_DGFX(tile) |
+						      XE_BO_FLAG_GGTT |
+						      XE_BO_FLAG_GGTT_INVALIDATE |
+						      XE_BO_FLAG_PINNED_NORESTORE);
+		if (IS_ERR(shadow)) {
+			drm_err(&xe->drm, "Failed to prepare %uKiB BO for SA manager (%pe)\n",
+				size / SZ_1K, shadow);
+			return ERR_CAST(shadow);
+		}
+		sa_manager->shadow = shadow;
+	}
+
 	drm_suballoc_manager_init(&sa_manager->base, managed_size, align);
 	ret = drmm_add_action_or_reset(&xe->drm, xe_sa_bo_manager_fini,
 				       sa_manager);
@@ -88,6 +111,48 @@ struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u3
 	return sa_manager;
 }
 
+/**
+ * xe_sa_bo_swap_shadow() - Swap the SA BO with shadow BO.
+ * @sa_manager: the XE sub allocator manager
+ *
+ * Swaps the sub-allocator primary buffer object with shadow buffer object.
+ *
+ * Return: None.
+ */
+void xe_sa_bo_swap_shadow(struct xe_sa_manager *sa_manager)
+{
+	struct xe_device *xe = tile_to_xe(sa_manager->bo->tile);
+
+	xe_assert(xe, sa_manager->shadow);
+	lockdep_assert_held(&sa_manager->swap_guard);
+
+	swap(sa_manager->bo, sa_manager->shadow);
+	if (!sa_manager->bo->vmap.is_iomem)
+		sa_manager->cpu_ptr = sa_manager->bo->vmap.vaddr;
+}
+
+/**
+ * xe_sa_bo_sync_shadow() - Sync the SA Shadow BO with primary BO.
+ * @sa_bo: the sub-allocator buffer object.
+ *
+ * Synchronize sub-allocator shadow buffer object with primary buffer object.
+ *
+ * Return: None.
+ */
+void xe_sa_bo_sync_shadow(struct drm_suballoc *sa_bo)
+{
+	struct xe_sa_manager *sa_manager = to_xe_sa_manager(sa_bo->manager);
+	struct xe_device *xe = tile_to_xe(sa_manager->bo->tile);
+
+	xe_assert(xe, sa_manager->shadow);
+	lockdep_assert_held(&sa_manager->swap_guard);
+
+	xe_map_memcpy_to(xe, &sa_manager->shadow->vmap,
+			 drm_suballoc_soffset(sa_bo),
+			 xe_sa_bo_cpu_addr(sa_bo),
+			 drm_suballoc_size(sa_bo));
+}
+
 /**
  * __xe_sa_bo_new() - Make a suballocation but use custom gfp flags.
  * @sa_manager: the &xe_sa_manager
diff --git a/drivers/gpu/drm/xe/xe_sa.h b/drivers/gpu/drm/xe/xe_sa.h
index 1be744350836..05e9a4e00e78 100644
--- a/drivers/gpu/drm/xe/xe_sa.h
+++ b/drivers/gpu/drm/xe/xe_sa.h
@@ -14,12 +14,14 @@
 struct dma_fence;
 struct xe_tile;
 
-struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u32 guard, u32 align);
+#define XE_SA_BO_MANAGER_FLAG_SHADOW	BIT(0)
+struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size,
+					      u32 guard, u32 align, u32 flags);
 struct drm_suballoc *__xe_sa_bo_new(struct xe_sa_manager *sa_manager, u32 size, gfp_t gfp);
 
 static inline struct xe_sa_manager *xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u32 align)
 {
-	return __xe_sa_bo_manager_init(tile, size, SZ_4K, align);
+	return __xe_sa_bo_manager_init(tile, size, SZ_4K, align, 0);
 }
 
 /**
@@ -69,4 +71,18 @@ static inline void *xe_sa_bo_cpu_addr(struct drm_suballoc *sa)
 		drm_suballoc_soffset(sa);
 }
 
+void xe_sa_bo_swap_shadow(struct xe_sa_manager *sa_manager);
+void xe_sa_bo_sync_shadow(struct drm_suballoc *sa_bo);
+
+/**
+ * xe_sa_bo_swap_guard() - Retrieve the SA BO swap guard within sub-allocator.
+ * @sa_manager: the &xe_sa_manager
+ *
+ * Return: Sub alloctor swap guard mutex.
+ */
+static inline struct mutex *xe_sa_bo_swap_guard(struct xe_sa_manager *sa_manager)
+{
+	return &sa_manager->swap_guard;
+}
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_sa_types.h b/drivers/gpu/drm/xe/xe_sa_types.h
index cb7238799dcb..1085c9c37d6b 100644
--- a/drivers/gpu/drm/xe/xe_sa_types.h
+++ b/drivers/gpu/drm/xe/xe_sa_types.h
@@ -12,6 +12,9 @@ struct xe_bo;
 struct xe_sa_manager {
 	struct drm_suballoc_manager base;
 	struct xe_bo *bo;
+	struct xe_bo *shadow;
+	/** @swap_guard: Timeline guard updating @bo and @shadow */
+	struct mutex swap_guard;
 	void *cpu_ptr;
 	bool is_iomem;
 };
diff --git a/drivers/gpu/drm/xe/xe_sriov_packet.c b/drivers/gpu/drm/xe/xe_sriov_packet.c
index bab994696896..2cefefaed9ba 100644
--- a/drivers/gpu/drm/xe/xe_sriov_packet.c
+++ b/drivers/gpu/drm/xe/xe_sriov_packet.c
@@ -358,7 +358,7 @@ ssize_t xe_sriov_packet_write_single(struct xe_device *xe, unsigned int vfid,
 
 #define MIGRATION_DESCRIPTOR_DWORDS	(GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_DEVID_LEN + \
 					 GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_REVID_LEN)
-static size_t pf_descriptor_init(struct xe_device *xe, unsigned int vfid)
+static int pf_descriptor_init(struct xe_device *xe, unsigned int vfid)
 {
 	struct xe_sriov_packet **desc = pf_pick_descriptor(xe, vfid);
 	struct xe_sriov_packet *data;
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf.c b/drivers/gpu/drm/xe/xe_sriov_pf.c
index 7c779d63179f..72423bb17e6f 100644
--- a/drivers/gpu/drm/xe/xe_sriov_pf.c
+++ b/drivers/gpu/drm/xe/xe_sriov_pf.c
@@ -90,6 +90,7 @@ bool xe_sriov_pf_readiness(struct xe_device *xe)
  */
 int xe_sriov_pf_init_early(struct xe_device *xe)
 {
+	struct xe_mert *mert = &xe_device_get_root_tile(xe)->mert;
 	int err;
 
 	xe_assert(xe, IS_SRIOV_PF(xe));
@@ -111,6 +112,9 @@ int xe_sriov_pf_init_early(struct xe_device *xe)
 
 	xe_sriov_pf_service_init(xe);
 
+	spin_lock_init(&mert->lock);
+	init_completion(&mert->tlb_inv_done);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c b/drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c
index bad751217e1e..e84bdde9bc80 100644
--- a/drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c
@@ -70,9 +70,8 @@ static ssize_t from_file_write_to_xe_call(struct file *file, const char __user *
 	if (ret < 0)
 		return ret;
 	if (yes) {
-		xe_pm_runtime_get(xe);
+		guard(xe_pm_runtime)(xe);
 		ret = call(xe);
-		xe_pm_runtime_put(xe);
 	}
 	if (ret < 0)
 		return ret;
@@ -209,9 +208,8 @@ static ssize_t from_file_write_to_vf_call(struct file *file, const char __user *
 	if (ret < 0)
 		return ret;
 	if (yes) {
-		xe_pm_runtime_get(xe);
+		guard(xe_pm_runtime)(xe);
 		ret = call(xe, vfid);
-		xe_pm_runtime_put(xe);
 	}
 	if (ret < 0)
 		return ret;
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c b/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c
index c0b767ac735c..3d140506ba36 100644
--- a/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c
+++ b/drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c
@@ -389,16 +389,12 @@ static ssize_t xe_sriov_dev_attr_store(struct kobject *kobj, struct attribute *a
 	struct xe_sriov_dev_attr *vattr = to_xe_sriov_dev_attr(attr);
 	struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj);
 	struct xe_device *xe = vkobj->xe;
-	ssize_t ret;
 
 	if (!vattr->store)
 		return -EPERM;
 
-	xe_pm_runtime_get(xe);
-	ret = xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, buf, count);
-	xe_pm_runtime_put(xe);
-
-	return ret;
+	guard(xe_pm_runtime)(xe);
+	return xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, buf, count);
 }
 
 static ssize_t xe_sriov_vf_attr_show(struct kobject *kobj, struct attribute *attr, char *buf)
@@ -423,18 +419,14 @@ static ssize_t xe_sriov_vf_attr_store(struct kobject *kobj, struct attribute *at
 	struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj);
 	struct xe_device *xe = vkobj->xe;
 	unsigned int vfid = vkobj->vfid;
-	ssize_t ret;
 
 	xe_sriov_pf_assert_vfid(xe, vfid);
 
 	if (!vattr->store)
 		return -EPERM;
 
-	xe_pm_runtime_get(xe);
-	ret = xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, vfid, buf, count);
-	xe_pm_runtime_get(xe);
-
-	return ret;
+	guard(xe_pm_runtime)(xe);
+	return xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, vfid, buf, count);
 }
 
 static const struct sysfs_ops xe_sriov_dev_sysfs_ops = {
diff --git a/drivers/gpu/drm/xe/xe_sriov_vf.c b/drivers/gpu/drm/xe/xe_sriov_vf.c
index 284ce37ca92d..1b75405b8d02 100644
--- a/drivers/gpu/drm/xe/xe_sriov_vf.c
+++ b/drivers/gpu/drm/xe/xe_sriov_vf.c
@@ -49,11 +49,13 @@
  *
  * As soon as Virtual GPU of the VM starts, the VF driver within receives
  * the MIGRATED interrupt and schedules post-migration recovery worker.
- * That worker queries GuC for new provisioning (using MMIO communication),
+ * That worker sends `VF2GUC_RESFIX_START` action along with non-zero
+ * marker, queries GuC for new provisioning (using MMIO communication),
  * and applies fixups to any non-virtualized resources used by the VF.
  *
  * When the VF driver is ready to continue operation on the newly connected
- * hardware, it sends `VF2GUC_NOTIFY_RESFIX_DONE` which causes it to
+ * hardware, it sends `VF2GUC_RESFIX_DONE` action along with the same
+ * marker which was sent with `VF2GUC_RESFIX_START` which causes it to
  * enter the long awaited `VF_RUNNING` state, and therefore start handling
  * CTB messages and scheduling workloads from the VF::
  *
@@ -102,12 +104,17 @@
  *      |                              [ ]        new VF provisioning  [ ]
  *      |                              [ ]---------------------------> [ ]
  *      |                               |                              [ ]
+ *      |                               |   VF2GUC_RESFIX_START        [ ]
+ *      |                              [ ] <---------------------------[ ]
+ *      |                              [ ]                             [ ]
+ *      |                              [ ]                     success [ ]
+ *      |                              [ ]---------------------------> [ ]
  *      |                               |       VF driver applies post [ ]
  *      |                               |      migration fixups -------[ ]
  *      |                               |                       |      [ ]
  *      |                               |                       -----> [ ]
  *      |                               |                              [ ]
- *      |                               |    VF2GUC_NOTIFY_RESFIX_DONE [ ]
+ *      |                               |    VF2GUC_RESFIX_DONE        [ ]
  *      |                              [ ] <---------------------------[ ]
  *      |                              [ ]                             [ ]
  *      |                              [ ]  GuC sets new VF state to   [ ]
@@ -118,6 +125,55 @@
  *      |                              [ ]---------------------------> [ ]
  *      |                               |                               |
  *      |                               |                               |
+ *
+ * Handling of VF double migration flow is shown below::
+ *
+ *     GuC1                                             VF
+ *      |                                               |
+ *      |                                              [ ]<--- start fixups
+ *      |                  VF2GUC_RESFIX_START(marker) [ ]
+ *     [ ] <-------------------------------------------[ ]
+ *     [ ]                                             [ ]
+ *     [ ]---\                                         [ ]
+ *     [ ]   store marker                              [ ]
+ *     [ ]<--/                                         [ ]
+ *     [ ]                                             [ ]
+ *     [ ] success                                     [ ]
+ *     [ ] ------------------------------------------> [ ]
+ *      |                                              [ ]
+ *      |                                              [ ]---\
+ *      |                                              [ ]   do fixups
+ *      |                                              [ ]<--/
+ *      |                                              [ ]
+ *      -------------- VF paused / saved ----------------
+ *      :
+ *
+ *     GuC2
+ *      |
+ *      ----------------- VF restored  ------------------
+ *      |
+ *     [ ]
+ *     [ ]---\
+ *     [ ]   reset marker
+ *     [ ]<--/
+ *     [ ]
+ *      ----------------- VF resumed  ------------------
+ *      |                                              [ ]
+ *      |                                              [ ]
+ *      |                   VF2GUC_RESFIX_DONE(marker) [ ]
+ *     [ ] <-------------------------------------------[ ]
+ *     [ ]                                             [ ]
+ *     [ ]---\                                         [ ]
+ *     [ ]   check marker                              [ ]
+ *     [ ]   (mismatch)                                [ ]
+ *     [ ]<--/                                         [ ]
+ *     [ ]                                             [ ]
+ *     [ ] RESPONSE_VF_MIGRATED                        [ ]
+ *     [ ] ------------------------------------------> [ ]
+ *      |                                              [ ]---\
+ *      |                                              [ ]  reschedule fixups
+ *      |                                              [ ]<--/
+ *      |                                               |
  */
 
 /**
@@ -170,6 +226,26 @@ void xe_sriov_vf_init_early(struct xe_device *xe)
 	vf_migration_init_early(xe);
 }
 
+static int vf_migration_init_late(struct xe_device *xe)
+{
+	struct xe_gt *gt = xe_root_mmio_gt(xe);
+	struct xe_uc_fw_version guc_version;
+
+	if (!xe_sriov_vf_migration_supported(xe))
+		return 0;
+
+	xe_gt_sriov_vf_guc_versions(gt, NULL, &guc_version);
+	if (MAKE_GUC_VER_STRUCT(guc_version) < MAKE_GUC_VER(1, 27, 0)) {
+		xe_sriov_vf_migration_disable(xe,
+					      "requires GuC ABI >= 1.27.0, but only %u.%u.%u found",
+					      guc_version.major, guc_version.minor,
+					      guc_version.patch);
+		return 0;
+	}
+
+	return xe_sriov_vf_ccs_init(xe);
+}
+
 /**
  * xe_sriov_vf_init_late() - SR-IOV VF late initialization functions.
  * @xe: the &xe_device to initialize
@@ -180,7 +256,7 @@ void xe_sriov_vf_init_early(struct xe_device *xe)
  */
 int xe_sriov_vf_init_late(struct xe_device *xe)
 {
-	return xe_sriov_vf_ccs_init(xe);
+	return vf_migration_init_late(xe);
 }
 
 static int sa_info_vf_ccs(struct seq_file *m, void *data)
diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c
index 797a4b866226..052a5071e69f 100644
--- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c
+++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c
@@ -150,7 +150,8 @@ static int alloc_bb_pool(struct xe_tile *tile, struct xe_sriov_vf_ccs_ctx *ctx)
 	xe_sriov_info(xe, "Allocating %s CCS BB pool size = %lldMB\n",
 		      ctx->ctx_id ? "Restore" : "Save", bb_pool_size / SZ_1M);
 
-	sa_manager = xe_sa_bo_manager_init(tile, bb_pool_size, SZ_16);
+	sa_manager = __xe_sa_bo_manager_init(tile, bb_pool_size, SZ_4K, SZ_16,
+					     XE_SA_BO_MANAGER_FLAG_SHADOW);
 
 	if (IS_ERR(sa_manager)) {
 		xe_sriov_err(xe, "Suballocator init failed with error: %pe\n",
@@ -162,9 +163,12 @@ static int alloc_bb_pool(struct xe_tile *tile, struct xe_sriov_vf_ccs_ctx *ctx)
 	offset = 0;
 	xe_map_memset(xe, &sa_manager->bo->vmap, offset, MI_NOOP,
 		      bb_pool_size);
+	xe_map_memset(xe, &sa_manager->shadow->vmap, offset, MI_NOOP,
+		      bb_pool_size);
 
 	offset = bb_pool_size - sizeof(u32);
 	xe_map_wr(xe, &sa_manager->bo->vmap, offset, u32, MI_BATCH_BUFFER_END);
+	xe_map_wr(xe, &sa_manager->shadow->vmap, offset, u32, MI_BATCH_BUFFER_END);
 
 	ctx->mem.ccs_bb_pool = sa_manager;
 
@@ -381,6 +385,18 @@ int xe_sriov_vf_ccs_init(struct xe_device *xe)
 	return err;
 }
 
+#define XE_SRIOV_VF_CCS_RW_BB_ADDR_OFFSET	(2 * sizeof(u32))
+void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx)
+{
+	u64 addr = xe_sa_manager_gpu_addr(ctx->mem.ccs_bb_pool);
+	struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q);
+	struct xe_device *xe = gt_to_xe(ctx->mig_q->gt);
+
+	xe_device_wmb(xe);
+	xe_map_wr(xe, &lrc->bo->vmap, XE_SRIOV_VF_CCS_RW_BB_ADDR_OFFSET, u32, addr);
+	xe_device_wmb(xe);
+}
+
 /**
  * xe_sriov_vf_ccs_attach_bo - Insert CCS read write commands in the BO.
  * @bo: the &buffer object to which batch buffer commands will be added.
@@ -441,9 +457,7 @@ int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo)
 		if (!bb)
 			continue;
 
-		memset(bb->cs, MI_NOOP, bb->len * sizeof(u32));
-		xe_bb_free(bb, NULL);
-		bo->bb_ccs[ctx_id] = NULL;
+		xe_migrate_ccs_rw_copy_clear(bo, ctx_id);
 	}
 	return 0;
 }
@@ -463,8 +477,7 @@ void xe_sriov_vf_ccs_print(struct xe_device *xe, struct drm_printer *p)
 	if (!IS_VF_CCS_READY(xe))
 		return;
 
-	xe_pm_runtime_get(xe);
-
+	guard(xe_pm_runtime)(xe);
 	for_each_ccs_rw_ctx(ctx_id) {
 		bb_pool = xe->sriov.vf.ccs.contexts[ctx_id].mem.ccs_bb_pool;
 		if (!bb_pool)
@@ -475,6 +488,4 @@ void xe_sriov_vf_ccs_print(struct xe_device *xe, struct drm_printer *p)
 		drm_suballoc_dump_debug_info(&bb_pool->base, p, xe_sa_manager_gpu_addr(bb_pool));
 		drm_puts(p, "\n");
 	}
-
-	xe_pm_runtime_put(xe);
 }
diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h
index f8ca6efce9ee..00e58b36c510 100644
--- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h
+++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h
@@ -20,6 +20,7 @@ int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo);
 int xe_sriov_vf_ccs_register_context(struct xe_device *xe);
 void xe_sriov_vf_ccs_rebase(struct xe_device *xe);
 void xe_sriov_vf_ccs_print(struct xe_device *xe, struct drm_printer *p);
+void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx);
 
 static inline bool xe_sriov_vf_ccs_ready(struct xe_device *xe)
 {
diff --git a/drivers/gpu/drm/xe/xe_sriov_vfio.c b/drivers/gpu/drm/xe/xe_sriov_vfio.c
index e9a7615bb5c5..3da81af97b8b 100644
--- a/drivers/gpu/drm/xe/xe_sriov_vfio.c
+++ b/drivers/gpu/drm/xe/xe_sriov_vfio.c
@@ -21,7 +21,7 @@ EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_get_pf, "xe-vfio-pci");
 bool xe_sriov_vfio_migration_supported(struct xe_device *xe)
 {
 	if (!IS_SRIOV_PF(xe))
-		return -EPERM;
+		return false;
 
 	return xe_sriov_pf_migration_supported(xe);
 }
diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c
index 1662bfddd4bc..2869866537ad 100644
--- a/drivers/gpu/drm/xe/xe_survivability_mode.c
+++ b/drivers/gpu/drm/xe/xe_survivability_mode.c
@@ -16,11 +16,10 @@
 #include "xe_heci_gsc.h"
 #include "xe_i2c.h"
 #include "xe_mmio.h"
+#include "xe_nvm.h"
 #include "xe_pcode_api.h"
 #include "xe_vsec.h"
 
-#define MAX_SCRATCH_MMIO 8
-
 /**
  * DOC: Survivability Mode
  *
@@ -48,19 +47,43 @@
  *
  * Refer :ref:`xe_configfs` for more details on how to use configfs
  *
- * Survivability mode is indicated by the below admin-only readable sysfs which provides additional
- * debug information::
+ * Survivability mode is indicated by the below admin-only readable sysfs entry. It
+ * provides information about the type of survivability mode (Boot/Runtime).
+ *
+ * .. code-block:: shell
+ *
+ *	# cat /sys/bus/pci/devices/<device>/survivability_mode
+ *	  Boot
+ *
+ *
+ * Any additional debug information if present will be visible under the directory
+ * ``survivability_info``::
+ *
+ *	/sys/bus/pci/devices/<device>/survivability_info/
+ *	├── aux_info0
+ *	├── aux_info1
+ *	├── aux_info2
+ *	├── aux_info3
+ *	├── aux_info4
+ *	├── capability_info
+ *	├── fdo_mode
+ *	├── postcode_trace
+ *	└── postcode_trace_overflow
+ *
+ * This directory has the following attributes
+ *
+ * - ``capability_info`` : Indicates Boot status and support for additional information
  *
- *	/sys/bus/pci/devices/<device>/survivability_mode
+ * - ``postcode_trace``, ``postcode_trace_overflow`` : Each postcode is a 8bit value and
+ *   represents a boot failure event. When a new failure event is logged by PCODE the
+ *   existing postcodes are shifted left. These entries provide a history of 8 postcodes.
  *
- * Capability Information:
- *	Provides boot status
- * Postcode Information:
- *	Provides information about the failure
- * Overflow Information
- *	Provides history of previous failures
- * Auxiliary Information
- *	Certain failures may have information in addition to postcode information
+ * - ``aux_info<n>`` : Some failures have additional debug information
+ *
+ * - ``fdo_mode`` : To allow recovery in scenarios where MEI itself fails, a new SPI Flash
+ *   Descriptor Override (FDO) mode is added in v2 survivability breadcrumbs. This mode is enabled
+ *   by PCODE and provides the ability to directly update the firmware via SPI Driver without
+ *   any dependency on MEI. Xe KMD initializes the nvm aux driver if FDO mode is enabled.
  *
  * Runtime Survivability
  * =====================
@@ -68,61 +91,77 @@
  * Certain runtime firmware errors can cause the device to enter a wedged state
  * (:ref:`xe-device-wedging`) requiring a firmware flash to restore normal operation.
  * Runtime Survivability Mode indicates that a firmware flash is necessary to recover the device and
- * is indicated by the presence of survivability mode sysfs::
+ * is indicated by the presence of survivability mode sysfs.
+ * Survivability mode sysfs provides information about the type of survivability mode.
  *
- *	/sys/bus/pci/devices/<device>/survivability_mode
+ * .. code-block:: shell
  *
- * Survivability mode sysfs provides information about the type of survivability mode.
+ *	# cat /sys/bus/pci/devices/<device>/survivability_mode
+ *	  Runtime
  *
  * When such errors occur, userspace is notified with the drm device wedged uevent and runtime
  * survivability mode. User can then initiate a firmware flash using userspace tools like fwupd
  * to restore device to normal operation.
  */
 
-static u32 aux_history_offset(u32 reg_value)
+static const char * const reg_map[] = {
+	[CAPABILITY_INFO]         = "Capability Info",
+	[POSTCODE_TRACE]          = "Postcode trace",
+	[POSTCODE_TRACE_OVERFLOW] = "Postcode trace overflow",
+	[AUX_INFO0]               = "Auxiliary Info 0",
+	[AUX_INFO1]               = "Auxiliary Info 1",
+	[AUX_INFO2]               = "Auxiliary Info 2",
+	[AUX_INFO3]               = "Auxiliary Info 3",
+	[AUX_INFO4]               = "Auxiliary Info 4",
+};
+
+#define FDO_INFO	(MAX_SCRATCH_REG + 1)
+
+struct xe_survivability_attribute {
+	struct device_attribute attr;
+	u8 index;
+};
+
+static struct
+xe_survivability_attribute *dev_attr_to_survivability_attr(struct device_attribute *attr)
 {
-	return REG_FIELD_GET(AUXINFO_HISTORY_OFFSET, reg_value);
+	return container_of(attr, struct xe_survivability_attribute, attr);
 }
 
-static void set_survivability_info(struct xe_mmio *mmio, struct xe_survivability_info *info,
-				   int id, char *name)
+static void set_survivability_info(struct xe_mmio *mmio, u32  *info, int id)
 {
-	strscpy(info[id].name, name, sizeof(info[id].name));
-	info[id].reg = PCODE_SCRATCH(id).raw;
-	info[id].value = xe_mmio_read32(mmio, PCODE_SCRATCH(id));
+	info[id] = xe_mmio_read32(mmio, PCODE_SCRATCH(id));
 }
 
 static void populate_survivability_info(struct xe_device *xe)
 {
 	struct xe_survivability *survivability = &xe->survivability;
-	struct xe_survivability_info *info = survivability->info;
+	u32 *info = survivability->info;
 	struct xe_mmio *mmio;
 	u32 id = 0, reg_value;
-	char name[NAME_MAX];
-	int index;
 
 	mmio = xe_root_tile_mmio(xe);
-	set_survivability_info(mmio, info, id, "Capability Info");
-	reg_value = info[id].value;
+	set_survivability_info(mmio, info, CAPABILITY_INFO);
+	reg_value = info[CAPABILITY_INFO];
+
+	survivability->version = REG_FIELD_GET(BREADCRUMB_VERSION, reg_value);
+	/* FDO mode is exposed only from version 2 */
+	if (survivability->version >= 2)
+		survivability->fdo_mode = REG_FIELD_GET(FDO_MODE, reg_value);
 
 	if (reg_value & HISTORY_TRACKING) {
-		id++;
-		set_survivability_info(mmio, info, id, "Postcode Info");
+		set_survivability_info(mmio, info, POSTCODE_TRACE);
 
-		if (reg_value & OVERFLOW_SUPPORT) {
-			id = REG_FIELD_GET(OVERFLOW_REG_OFFSET, reg_value);
-			set_survivability_info(mmio, info, id, "Overflow Info");
-		}
+		if (reg_value & OVERFLOW_SUPPORT)
+			set_survivability_info(mmio, info, POSTCODE_TRACE_OVERFLOW);
 	}
 
+	/* Traverse the linked list of aux info registers */
 	if (reg_value & AUXINFO_SUPPORT) {
-		id = REG_FIELD_GET(AUXINFO_REG_OFFSET, reg_value);
-
-		for (index = 0; id && reg_value; index++, reg_value = info[id].value,
-		     id = aux_history_offset(reg_value)) {
-			snprintf(name, NAME_MAX, "Auxiliary Info %d", index);
-			set_survivability_info(mmio, info, id, name);
-		}
+		for (id = REG_FIELD_GET(AUXINFO_REG_OFFSET, reg_value);
+		     id >= AUX_INFO0 && id < MAX_SCRATCH_REG;
+		     id =  REG_FIELD_GET(AUXINFO_HISTORY_OFFSET, info[id]))
+			set_survivability_info(mmio, info, id);
 	}
 }
 
@@ -130,15 +169,14 @@ static void log_survivability_info(struct pci_dev *pdev)
 {
 	struct xe_device *xe = pdev_to_xe_device(pdev);
 	struct xe_survivability *survivability = &xe->survivability;
-	struct xe_survivability_info *info = survivability->info;
+	u32 *info = survivability->info;
 	int id;
 
 	dev_info(&pdev->dev, "Survivability Boot Status : Critical Failure (%d)\n",
 		 survivability->boot_status);
-	for (id = 0; id < MAX_SCRATCH_MMIO; id++) {
-		if (info[id].reg)
-			dev_info(&pdev->dev, "%s: 0x%x - 0x%x\n", info[id].name,
-				 info[id].reg, info[id].value);
+	for (id = 0; id < MAX_SCRATCH_REG; id++) {
+		if (info[id])
+			dev_info(&pdev->dev, "%s: 0x%x\n", reg_map[id], info[id]);
 	}
 }
 
@@ -156,43 +194,99 @@ static ssize_t survivability_mode_show(struct device *dev,
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct xe_device *xe = pdev_to_xe_device(pdev);
 	struct xe_survivability *survivability = &xe->survivability;
-	struct xe_survivability_info *info = survivability->info;
-	int index = 0, count = 0;
 
-	count += sysfs_emit_at(buff, count, "Survivability mode type: %s\n",
-			       survivability->type ? "Runtime" : "Boot");
+	return sysfs_emit(buff, "%s\n", survivability->type ? "Runtime" : "Boot");
+}
+
+static DEVICE_ATTR_ADMIN_RO(survivability_mode);
 
-	if (!check_boot_failure(xe))
-		return count;
+static ssize_t survivability_info_show(struct device *dev,
+				       struct device_attribute *attr, char *buff)
+{
+	struct xe_survivability_attribute *sa = dev_attr_to_survivability_attr(attr);
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct xe_device *xe = pdev_to_xe_device(pdev);
+	struct xe_survivability *survivability = &xe->survivability;
+	u32 *info = survivability->info;
 
-	for (index = 0; index < MAX_SCRATCH_MMIO; index++) {
-		if (info[index].reg)
-			count += sysfs_emit_at(buff, count, "%s: 0x%x - 0x%x\n", info[index].name,
-					       info[index].reg, info[index].value);
-	}
+	if (sa->index == FDO_INFO)
+		return sysfs_emit(buff, "%s\n", str_enabled_disabled(survivability->fdo_mode));
 
-	return count;
+	return sysfs_emit(buff, "0x%x\n", info[sa->index]);
 }
 
-static DEVICE_ATTR_ADMIN_RO(survivability_mode);
+#define SURVIVABILITY_ATTR_RO(name, _index)					\
+	struct xe_survivability_attribute attr_##name =	{			\
+		.attr =  __ATTR(name, 0400, survivability_info_show, NULL),	\
+		.index = _index,						\
+	}
+
+SURVIVABILITY_ATTR_RO(capability_info, CAPABILITY_INFO);
+SURVIVABILITY_ATTR_RO(postcode_trace, POSTCODE_TRACE);
+SURVIVABILITY_ATTR_RO(postcode_trace_overflow, POSTCODE_TRACE_OVERFLOW);
+SURVIVABILITY_ATTR_RO(aux_info0, AUX_INFO0);
+SURVIVABILITY_ATTR_RO(aux_info1, AUX_INFO1);
+SURVIVABILITY_ATTR_RO(aux_info2, AUX_INFO2);
+SURVIVABILITY_ATTR_RO(aux_info3, AUX_INFO3);
+SURVIVABILITY_ATTR_RO(aux_info4, AUX_INFO4);
+SURVIVABILITY_ATTR_RO(fdo_mode, FDO_INFO);
 
 static void xe_survivability_mode_fini(void *arg)
 {
 	struct xe_device *xe = arg;
+	struct xe_survivability *survivability = &xe->survivability;
 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
 	struct device *dev = &pdev->dev;
 
-	sysfs_remove_file(&dev->kobj, &dev_attr_survivability_mode.attr);
+	if (survivability->fdo_mode)
+		xe_nvm_fini(xe);
+
+	device_remove_file(dev, &dev_attr_survivability_mode);
 }
 
+static umode_t survivability_info_attrs_visible(struct kobject *kobj, struct attribute *attr,
+						int idx)
+{
+	struct xe_device *xe = kdev_to_xe_device(kobj_to_dev(kobj));
+	struct xe_survivability *survivability = &xe->survivability;
+	u32 *info = survivability->info;
+
+	/* FDO mode is visible only when supported */
+	if (idx >= MAX_SCRATCH_REG && survivability->version >= 2)
+		return 0400;
+	else if (info[idx])
+		return 0400;
+
+	return 0;
+}
+
+/* Attributes are ordered according to enum scratch_reg */
+static struct attribute *survivability_info_attrs[] = {
+	&attr_capability_info.attr.attr,
+	&attr_postcode_trace.attr.attr,
+	&attr_postcode_trace_overflow.attr.attr,
+	&attr_aux_info0.attr.attr,
+	&attr_aux_info1.attr.attr,
+	&attr_aux_info2.attr.attr,
+	&attr_aux_info3.attr.attr,
+	&attr_aux_info4.attr.attr,
+	&attr_fdo_mode.attr.attr,
+	NULL,
+};
+
+static const struct attribute_group survivability_info_group = {
+	.name = "survivability_info",
+	.attrs = survivability_info_attrs,
+	.is_visible = survivability_info_attrs_visible,
+};
+
 static int create_survivability_sysfs(struct pci_dev *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct xe_device *xe = pdev_to_xe_device(pdev);
 	int ret;
 
-	/* create survivability mode sysfs */
-	ret = sysfs_create_file(&dev->kobj, &dev_attr_survivability_mode.attr);
+	ret = device_create_file(dev, &dev_attr_survivability_mode);
 	if (ret) {
 		dev_warn(dev, "Failed to create survivability sysfs files\n");
 		return ret;
@@ -203,6 +297,12 @@ static int create_survivability_sysfs(struct pci_dev *pdev)
 	if (ret)
 		return ret;
 
+	if (check_boot_failure(xe)) {
+		ret = devm_device_add_group(dev, &survivability_info_group);
+		if (ret)
+			return ret;
+	}
+
 	return 0;
 }
 
@@ -220,12 +320,16 @@ static int enable_boot_survivability_mode(struct pci_dev *pdev)
 	/* Make sure xe_heci_gsc_init() knows about survivability mode */
 	survivability->mode = true;
 
-	ret = xe_heci_gsc_init(xe);
-	if (ret)
-		goto err;
+	xe_heci_gsc_init(xe);
 
 	xe_vsec_init(xe);
 
+	if (survivability->fdo_mode) {
+		ret = xe_nvm_init(xe);
+		if (ret)
+			goto err;
+	}
+
 	ret = xe_i2c_probe(xe);
 	if (ret)
 		goto err;
@@ -235,29 +339,11 @@ static int enable_boot_survivability_mode(struct pci_dev *pdev)
 	return 0;
 
 err:
+	dev_err(dev, "Failed to enable Survivability Mode\n");
 	survivability->mode = false;
 	return ret;
 }
 
-static int init_survivability_mode(struct xe_device *xe)
-{
-	struct xe_survivability *survivability = &xe->survivability;
-	struct xe_survivability_info *info;
-
-	survivability->size = MAX_SCRATCH_MMIO;
-
-	info = devm_kcalloc(xe->drm.dev, survivability->size, sizeof(*info),
-			    GFP_KERNEL);
-	if (!info)
-		return -ENOMEM;
-
-	survivability->info = info;
-
-	populate_survivability_info(xe);
-
-	return 0;
-}
-
 /**
  * xe_survivability_mode_is_boot_enabled- check if boot survivability mode is enabled
  * @xe: xe device instance
@@ -325,9 +411,7 @@ int xe_survivability_mode_runtime_enable(struct xe_device *xe)
 		return -EINVAL;
 	}
 
-	ret = init_survivability_mode(xe);
-	if (ret)
-		return ret;
+	populate_survivability_info(xe);
 
 	ret = create_survivability_sysfs(pdev);
 	if (ret)
@@ -356,17 +440,16 @@ int xe_survivability_mode_boot_enable(struct xe_device *xe)
 {
 	struct xe_survivability *survivability = &xe->survivability;
 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
-	int ret;
 
 	if (!xe_survivability_mode_is_requested(xe))
 		return 0;
 
-	ret = init_survivability_mode(xe);
-	if (ret)
-		return ret;
+	populate_survivability_info(xe);
 
-	/* Log breadcrumbs but do not enter survivability mode for Critical boot errors */
-	if (survivability->boot_status == CRITICAL_FAILURE) {
+	/*
+	 * v2 supports survivability mode for critical errors
+	 */
+	if (survivability->version < 2  && survivability->boot_status == CRITICAL_FAILURE) {
 		log_survivability_info(pdev);
 		return -ENXIO;
 	}
diff --git a/drivers/gpu/drm/xe/xe_survivability_mode_types.h b/drivers/gpu/drm/xe/xe_survivability_mode_types.h
index cd65a5d167c9..bd5dc1c955ff 100644
--- a/drivers/gpu/drm/xe/xe_survivability_mode_types.h
+++ b/drivers/gpu/drm/xe/xe_survivability_mode_types.h
@@ -9,23 +9,29 @@
 #include <linux/limits.h>
 #include <linux/types.h>
 
+enum scratch_reg {
+	CAPABILITY_INFO,
+	POSTCODE_TRACE,
+	POSTCODE_TRACE_OVERFLOW,
+	AUX_INFO0,
+	AUX_INFO1,
+	AUX_INFO2,
+	AUX_INFO3,
+	AUX_INFO4,
+	MAX_SCRATCH_REG,
+};
+
 enum xe_survivability_type {
 	XE_SURVIVABILITY_TYPE_BOOT,
 	XE_SURVIVABILITY_TYPE_RUNTIME,
 };
 
-struct xe_survivability_info {
-	char name[NAME_MAX];
-	u32 reg;
-	u32 value;
-};
-
 /**
  * struct xe_survivability: Contains survivability mode information
  */
 struct xe_survivability {
-	/** @info: struct that holds survivability info from scratch registers */
-	struct xe_survivability_info *info;
+	/** @info: survivability debug info */
+	u32 info[MAX_SCRATCH_REG];
 
 	/** @size: number of scratch registers */
 	u32 size;
@@ -38,6 +44,12 @@ struct xe_survivability {
 
 	/** @type: survivability type */
 	enum xe_survivability_type type;
+
+	/** @fdo_mode: indicates if FDO mode is enabled */
+	bool fdo_mode;
+
+	/** @version: breadcrumb version of survivability mode  */
+	u8 version;
 };
 
 #endif /* _XE_SURVIVABILITY_MODE_TYPES_H_ */
diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c
index 55c5a0eb82e1..93550c7c84ac 100644
--- a/drivers/gpu/drm/xe/xe_svm.c
+++ b/drivers/gpu/drm/xe/xe_svm.c
@@ -176,24 +176,13 @@ xe_svm_range_notifier_event_end(struct xe_vm *vm, struct drm_gpusvm_range *r,
 						   mmu_range);
 }
 
-static s64 xe_svm_stats_ktime_us_delta(ktime_t start)
-{
-	return IS_ENABLED(CONFIG_DEBUG_FS) ?
-		ktime_us_delta(ktime_get(), start) : 0;
-}
-
 static void xe_svm_tlb_inval_us_stats_incr(struct xe_gt *gt, ktime_t start)
 {
-	s64 us_delta = xe_svm_stats_ktime_us_delta(start);
+	s64 us_delta = xe_gt_stats_ktime_us_delta(start);
 
 	xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_TLB_INVAL_US, us_delta);
 }
 
-static ktime_t xe_svm_stats_ktime_get(void)
-{
-	return IS_ENABLED(CONFIG_DEBUG_FS) ? ktime_get() : 0;
-}
-
 static void xe_svm_invalidate(struct drm_gpusvm *gpusvm,
 			      struct drm_gpusvm_notifier *notifier,
 			      const struct mmu_notifier_range *mmu_range)
@@ -202,7 +191,7 @@ static void xe_svm_invalidate(struct drm_gpusvm *gpusvm,
 	struct xe_device *xe = vm->xe;
 	struct drm_gpusvm_range *r, *first;
 	struct xe_tile *tile;
-	ktime_t start = xe_svm_stats_ktime_get();
+	ktime_t start = xe_gt_stats_ktime_get();
 	u64 adj_start = mmu_range->start, adj_end = mmu_range->end;
 	u8 tile_mask = 0, id;
 	long err;
@@ -285,19 +274,21 @@ static int __xe_svm_garbage_collector(struct xe_vm *vm,
 	return 0;
 }
 
-static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 range_start, u64 range_end)
+static void xe_vma_set_default_attributes(struct xe_vma *vma)
+{
+	vma->attr.preferred_loc.devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE;
+	vma->attr.preferred_loc.migration_policy = DRM_XE_MIGRATE_ALL_PAGES;
+	vma->attr.pat_index = vma->attr.default_pat_index;
+	vma->attr.atomic_access = DRM_XE_ATOMIC_UNDEFINED;
+}
+
+static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 start, u64 end)
 {
 	struct xe_vma *vma;
-	struct xe_vma_mem_attr default_attr = {
-		.preferred_loc = {
-			.devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE,
-			.migration_policy = DRM_XE_MIGRATE_ALL_PAGES,
-		},
-		.atomic_access = DRM_XE_ATOMIC_UNDEFINED,
-	};
-	int err = 0;
+	bool has_default_attr;
+	int err;
 
-	vma = xe_vm_find_vma_by_addr(vm, range_start);
+	vma = xe_vm_find_vma_by_addr(vm, start);
 	if (!vma)
 		return -EINVAL;
 
@@ -306,25 +297,30 @@ static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 range_start, u64
 		return 0;
 	}
 
-	if (xe_vma_has_default_mem_attrs(vma))
-		return 0;
-
 	vm_dbg(&vm->xe->drm, "Existing VMA start=0x%016llx, vma_end=0x%016llx",
 	       xe_vma_start(vma), xe_vma_end(vma));
 
-	if (xe_vma_start(vma) == range_start && xe_vma_end(vma) == range_end) {
-		default_attr.pat_index = vma->attr.default_pat_index;
-		default_attr.default_pat_index  = vma->attr.default_pat_index;
-		vma->attr = default_attr;
-	} else {
-		vm_dbg(&vm->xe->drm, "Split VMA start=0x%016llx, vma_end=0x%016llx",
-		       range_start, range_end);
-		err = xe_vm_alloc_cpu_addr_mirror_vma(vm, range_start, range_end - range_start);
-		if (err) {
-			drm_warn(&vm->xe->drm, "VMA SPLIT failed: %pe\n", ERR_PTR(err));
-			xe_vm_kill(vm, true);
-			return err;
-		}
+	has_default_attr = xe_vma_has_default_mem_attrs(vma);
+
+	if (has_default_attr) {
+		start = xe_vma_start(vma);
+		end = xe_vma_end(vma);
+	} else if (xe_vma_start(vma) == start && xe_vma_end(vma) == end) {
+		xe_vma_set_default_attributes(vma);
+	}
+
+	xe_vm_find_cpu_addr_mirror_vma_range(vm, &start, &end);
+
+	if (xe_vma_start(vma) == start && xe_vma_end(vma) == end && has_default_attr)
+		return 0;
+
+	vm_dbg(&vm->xe->drm, "New VMA start=0x%016llx, vma_end=0x%016llx",  start, end);
+
+	err = xe_vm_alloc_cpu_addr_mirror_vma(vm, start, end - start);
+	if (err) {
+		drm_warn(&vm->xe->drm, "New VMA MAP failed: %pe\n", ERR_PTR(err));
+		xe_vm_kill(vm, true);
+		return err;
 	}
 
 	/*
@@ -435,7 +431,7 @@ static void xe_svm_copy_us_stats_incr(struct xe_gt *gt,
 				      unsigned long npages,
 				      ktime_t start)
 {
-	s64 us_delta = xe_svm_stats_ktime_us_delta(start);
+	s64 us_delta = xe_gt_stats_ktime_us_delta(start);
 
 	if (dir == XE_SVM_COPY_TO_VRAM) {
 		switch (npages) {
@@ -487,7 +483,7 @@ static int xe_svm_copy(struct page **pages,
 	u64 vram_addr = XE_VRAM_ADDR_INVALID;
 	int err = 0, pos = 0;
 	bool sram = dir == XE_SVM_COPY_TO_SRAM;
-	ktime_t start = xe_svm_stats_ktime_get();
+	ktime_t start = xe_gt_stats_ktime_get();
 
 	/*
 	 * This flow is complex: it locates physically contiguous device pages,
@@ -979,7 +975,7 @@ static void xe_svm_range_##elem##_us_stats_incr(struct xe_gt *gt, \
 						struct xe_svm_range *range, \
 						ktime_t start) \
 { \
-	s64 us_delta = xe_svm_stats_ktime_us_delta(start); \
+	s64 us_delta = xe_gt_stats_ktime_us_delta(start); \
 \
 	switch (xe_svm_range_size(range)) { \
 	case SZ_4K: \
@@ -1024,7 +1020,7 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
 	struct drm_pagemap *dpagemap;
 	struct xe_tile *tile = gt_to_tile(gt);
 	int migrate_try_count = ctx.devmem_only ? 3 : 1;
-	ktime_t start = xe_svm_stats_ktime_get(), bind_start, get_pages_start;
+	ktime_t start = xe_gt_stats_ktime_get(), bind_start, get_pages_start;
 	int err;
 
 	lockdep_assert_held_write(&vm->lock);
@@ -1063,7 +1059,7 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
 
 	if (--migrate_try_count >= 0 &&
 	    xe_svm_range_needs_migrate_to_vram(range, vma, !!dpagemap || ctx.devmem_only)) {
-		ktime_t migrate_start = xe_svm_stats_ktime_get();
+		ktime_t migrate_start = xe_gt_stats_ktime_get();
 
 		/* TODO : For multi-device dpagemap will be used to find the
 		 * remote tile and remote device. Will need to modify
@@ -1100,7 +1096,7 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
 	}
 
 get_pages:
-	get_pages_start = xe_svm_stats_ktime_get();
+	get_pages_start = xe_gt_stats_ktime_get();
 
 	range_debug(range, "GET PAGES");
 	err = xe_svm_range_get_pages(vm, range, &ctx);
@@ -1127,7 +1123,7 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
 	xe_svm_range_get_pages_us_stats_incr(gt, range, get_pages_start);
 	range_debug(range, "PAGE FAULT - BIND");
 
-	bind_start = xe_svm_stats_ktime_get();
+	bind_start = xe_gt_stats_ktime_get();
 	xe_validation_guard(&vctx, &vm->xe->val, &exec, (struct xe_val_flags) {}, err) {
 		err = xe_vm_drm_exec_lock(vm, &exec);
 		drm_exec_retry_on_contention(&exec);
diff --git a/drivers/gpu/drm/xe/xe_svm.h b/drivers/gpu/drm/xe/xe_svm.h
index 0955d2ac8d74..fa757dd07954 100644
--- a/drivers/gpu/drm/xe/xe_svm.h
+++ b/drivers/gpu/drm/xe/xe_svm.h
@@ -214,7 +214,7 @@ int xe_svm_init(struct xe_vm *vm)
 {
 #if IS_ENABLED(CONFIG_DRM_GPUSVM)
 	return drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM (simple)", &vm->xe->drm,
-			       NULL, NULL, 0, 0, 0, NULL, NULL, 0);
+			       NULL, 0, 0, 0, NULL, NULL, 0);
 #else
 	return 0;
 #endif
diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c
index ff74528ca0c6..c8fdcdbd6ae7 100644
--- a/drivers/gpu/drm/xe/xe_sync.c
+++ b/drivers/gpu/drm/xe/xe_sync.c
@@ -228,6 +228,32 @@ int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job)
 	return 0;
 }
 
+/**
+ * xe_sync_entry_wait() - Wait on in-sync
+ * @sync: Sync object
+ *
+ * If the sync is in an in-sync, wait on the sync to signal.
+ *
+ * Return: 0 on success, -ERESTARTSYS on failure (interruption)
+ */
+int xe_sync_entry_wait(struct xe_sync_entry *sync)
+{
+	return xe_sync_needs_wait(sync) ?
+		dma_fence_wait(sync->fence, true) : 0;
+}
+
+/**
+ * xe_sync_needs_wait() - Sync needs a wait (input dma-fence not signaled)
+ * @sync: Sync object
+ *
+ * Return: True if sync needs a wait, False otherwise
+ */
+bool xe_sync_needs_wait(struct xe_sync_entry *sync)
+{
+	return sync->fence &&
+	       !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &sync->fence->flags);
+}
+
 void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence)
 {
 	if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL))
@@ -311,8 +337,11 @@ xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
 		struct xe_tile *tile;
 		u8 id;
 
-		for_each_tile(tile, vm->xe, id)
-			num_fence += (1 + XE_MAX_GT_PER_TILE);
+		for_each_tile(tile, vm->xe, id) {
+			num_fence++;
+			for_each_tlb_inval(i)
+				num_fence++;
+		}
 
 		fences = kmalloc_array(num_fence, sizeof(*fences),
 				       GFP_KERNEL);
diff --git a/drivers/gpu/drm/xe/xe_sync.h b/drivers/gpu/drm/xe/xe_sync.h
index 51f2d803e977..6b949194acff 100644
--- a/drivers/gpu/drm/xe/xe_sync.h
+++ b/drivers/gpu/drm/xe/xe_sync.h
@@ -29,6 +29,8 @@ int xe_sync_entry_add_deps(struct xe_sync_entry *sync,
 			   struct xe_sched_job *job);
 void xe_sync_entry_signal(struct xe_sync_entry *sync,
 			  struct dma_fence *fence);
+int xe_sync_entry_wait(struct xe_sync_entry *sync);
+bool xe_sync_needs_wait(struct xe_sync_entry *sync);
 void xe_sync_entry_cleanup(struct xe_sync_entry *sync);
 struct dma_fence *
 xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
diff --git a/drivers/gpu/drm/xe/xe_tile.c b/drivers/gpu/drm/xe/xe_tile.c
index 4f4f9a5c43af..63c060c2ea5c 100644
--- a/drivers/gpu/drm/xe/xe_tile.c
+++ b/drivers/gpu/drm/xe/xe_tile.c
@@ -209,6 +209,11 @@ int xe_tile_init(struct xe_tile *tile)
 	if (IS_ERR(tile->mem.kernel_bb_pool))
 		return PTR_ERR(tile->mem.kernel_bb_pool);
 
+	/* Optimistically anticipate at most 256 TLB fences with PRL */
+	tile->mem.reclaim_pool = xe_sa_bo_manager_init(tile, SZ_1M, XE_PAGE_RECLAIM_LIST_MAX_SIZE);
+	if (IS_ERR(tile->mem.reclaim_pool))
+		return PTR_ERR(tile->mem.reclaim_pool);
+
 	return 0;
 }
 void xe_tile_migrate_wait(struct xe_tile *tile)
diff --git a/drivers/gpu/drm/xe/xe_tile_debugfs.c b/drivers/gpu/drm/xe/xe_tile_debugfs.c
index fff242a5ae56..5df2f461b7b7 100644
--- a/drivers/gpu/drm/xe/xe_tile_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_tile_debugfs.c
@@ -82,13 +82,9 @@ int xe_tile_debugfs_show_with_rpm(struct seq_file *m, void *data)
 	struct drm_info_node *node = m->private;
 	struct xe_tile *tile = node_to_tile(node);
 	struct xe_device *xe = tile_to_xe(tile);
-	int ret;
 
-	xe_pm_runtime_get(xe);
-	ret = xe_tile_debugfs_simple_show(m, data);
-	xe_pm_runtime_put(xe);
-
-	return ret;
+	guard(xe_pm_runtime)(xe);
+	return xe_tile_debugfs_simple_show(m, data);
 }
 
 static int ggtt(struct xe_tile *tile, struct drm_printer *p)
@@ -110,6 +106,13 @@ static const struct drm_info_list vf_safe_debugfs_list[] = {
 	{ "sa_info", .show = xe_tile_debugfs_show_with_rpm, .data = sa_info },
 };
 
+static void tile_debugfs_create_vram_mm(struct xe_tile *tile)
+{
+	if (tile->mem.vram)
+		ttm_resource_manager_create_debugfs(&tile->mem.vram->ttm.manager, tile->debugfs,
+						    "vram_mm");
+}
+
 /**
  * xe_tile_debugfs_register - Register tile's debugfs attributes
  * @tile: the &xe_tile to register
@@ -139,4 +142,6 @@ void xe_tile_debugfs_register(struct xe_tile *tile)
 	drm_debugfs_create_files(vf_safe_debugfs_list,
 				 ARRAY_SIZE(vf_safe_debugfs_list),
 				 tile->debugfs, minor);
+
+	tile_debugfs_create_vram_mm(tile);
 }
diff --git a/drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c b/drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c
index f3f478f14ff5..7f97db2f89bb 100644
--- a/drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c
@@ -141,12 +141,11 @@ static int NAME##_set(void *data, u64 val)					\
 	if (val > (TYPE)~0ull)							\
 		return -EOVERFLOW;						\
 										\
-	xe_pm_runtime_get(xe);							\
+	guard(xe_pm_runtime)(xe);						\
 	err = xe_sriov_pf_wait_ready(xe) ?:					\
 	      xe_gt_sriov_pf_config_set_##CONFIG(gt, vfid, val);		\
 	if (!err)								\
 		xe_sriov_pf_provision_set_custom_mode(xe);			\
-	xe_pm_runtime_put(xe);							\
 										\
 	return err;								\
 }										\
diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c
index 918a59e686ea..dec042248164 100644
--- a/drivers/gpu/drm/xe/xe_tlb_inval.c
+++ b/drivers/gpu/drm/xe/xe_tlb_inval.c
@@ -199,6 +199,20 @@ void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval)
 	mutex_unlock(&tlb_inval->seqno_lock);
 }
 
+/**
+ * xe_tlb_inval_reset_timeout() - Reset TLB inval fence timeout
+ * @tlb_inval: TLB invalidation client
+ *
+ * Reset the TLB invalidation timeout timer.
+ */
+static void xe_tlb_inval_reset_timeout(struct xe_tlb_inval *tlb_inval)
+{
+	lockdep_assert_held(&tlb_inval->pending_lock);
+
+	mod_delayed_work(system_wq, &tlb_inval->fence_tdr,
+			 tlb_inval->ops->timeout_delay(tlb_inval));
+}
+
 static bool xe_tlb_inval_seqno_past(struct xe_tlb_inval *tlb_inval, int seqno)
 {
 	int seqno_recv = READ_ONCE(tlb_inval->seqno_recv);
@@ -299,6 +313,7 @@ int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval)
  * @start: start address
  * @end: end address
  * @asid: address space id
+ * @prl_sa: suballocation of page reclaim list if used, NULL indicates PPC flush
  *
  * Issue a range based TLB invalidation if supported, if not fallback to a full
  * TLB invalidation. Completion of TLB is asynchronous and caller can use
@@ -308,10 +323,10 @@ int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval)
  */
 int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,
 		       struct xe_tlb_inval_fence *fence, u64 start, u64 end,
-		       u32 asid)
+		       u32 asid, struct drm_suballoc *prl_sa)
 {
 	return xe_tlb_inval_issue(tlb_inval, fence, tlb_inval->ops->ppgtt,
-				  start, end, asid);
+				  start, end, asid, prl_sa);
 }
 
 /**
@@ -327,7 +342,7 @@ void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm)
 	u64 range = 1ull << vm->xe->info.va_bits;
 
 	xe_tlb_inval_fence_init(tlb_inval, &fence, true);
-	xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid);
+	xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid, NULL);
 	xe_tlb_inval_fence_wait(&fence);
 }
 
@@ -360,6 +375,12 @@ void xe_tlb_inval_done_handler(struct xe_tlb_inval *tlb_inval, int seqno)
 	 * process_g2h_msg().
 	 */
 	spin_lock_irqsave(&tlb_inval->pending_lock, flags);
+	if (seqno == TLB_INVALIDATION_SEQNO_INVALID) {
+		xe_tlb_inval_reset_timeout(tlb_inval);
+		spin_unlock_irqrestore(&tlb_inval->pending_lock, flags);
+		return;
+	}
+
 	if (xe_tlb_inval_seqno_past(tlb_inval, seqno)) {
 		spin_unlock_irqrestore(&tlb_inval->pending_lock, flags);
 		return;
diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.h b/drivers/gpu/drm/xe/xe_tlb_inval.h
index 05614915463a..858d0690f995 100644
--- a/drivers/gpu/drm/xe/xe_tlb_inval.h
+++ b/drivers/gpu/drm/xe/xe_tlb_inval.h
@@ -23,7 +23,7 @@ int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval);
 void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm);
 int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,
 		       struct xe_tlb_inval_fence *fence,
-		       u64 start, u64 end, u32 asid);
+		       u64 start, u64 end, u32 asid, struct drm_suballoc *prl_sa);
 
 void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval,
 			     struct xe_tlb_inval_fence *fence,
diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_job.c b/drivers/gpu/drm/xe/xe_tlb_inval_job.c
index 1ae0dec2cf31..6a7bd6315797 100644
--- a/drivers/gpu/drm/xe/xe_tlb_inval_job.c
+++ b/drivers/gpu/drm/xe/xe_tlb_inval_job.c
@@ -7,7 +7,9 @@
 #include "xe_dep_job_types.h"
 #include "xe_dep_scheduler.h"
 #include "xe_exec_queue.h"
+#include "xe_gt_printk.h"
 #include "xe_gt_types.h"
+#include "xe_page_reclaim.h"
 #include "xe_tlb_inval.h"
 #include "xe_tlb_inval_job.h"
 #include "xe_migrate.h"
@@ -24,6 +26,8 @@ struct xe_tlb_inval_job {
 	struct xe_exec_queue *q;
 	/** @vm: VM which TLB invalidation is being issued for */
 	struct xe_vm *vm;
+	/** @prl: Embedded copy of page reclaim list */
+	struct xe_page_reclaim_list prl;
 	/** @refcount: ref count of this job */
 	struct kref refcount;
 	/**
@@ -47,9 +51,16 @@ static struct dma_fence *xe_tlb_inval_job_run(struct xe_dep_job *dep_job)
 		container_of(dep_job, typeof(*job), dep);
 	struct xe_tlb_inval_fence *ifence =
 		container_of(job->fence, typeof(*ifence), base);
+	struct drm_suballoc *prl_sa = NULL;
+
+	if (xe_page_reclaim_list_valid(&job->prl)) {
+		prl_sa = xe_page_reclaim_create_prl_bo(job->tlb_inval, &job->prl, ifence);
+		if (IS_ERR(prl_sa))
+			prl_sa = NULL; /* Indicate fall back PPC flush with NULL */
+	}
 
 	xe_tlb_inval_range(job->tlb_inval, ifence, job->start,
-			   job->end, job->vm->usm.asid);
+			   job->end, job->vm->usm.asid, prl_sa);
 
 	return job->fence;
 }
@@ -107,6 +118,7 @@ xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval,
 	job->start = start;
 	job->end = end;
 	job->fence_armed = false;
+	xe_page_reclaim_list_init(&job->prl);
 	job->dep.ops = &dep_job_ops;
 	job->type = type;
 	kref_init(&job->refcount);
@@ -140,6 +152,25 @@ xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval,
 	return ERR_PTR(err);
 }
 
+/**
+ * xe_tlb_inval_job_add_page_reclaim() - Embed PRL into a TLB job
+ * @job: TLB invalidation job that may trigger reclamation
+ * @prl: Page reclaim list populated during unbind
+ *
+ * Copies @prl into the job and takes an extra reference to the entry page so
+ * ownership can transfer to the TLB fence when the job is pushed.
+ */
+void xe_tlb_inval_job_add_page_reclaim(struct xe_tlb_inval_job *job,
+				       struct xe_page_reclaim_list *prl)
+{
+	struct xe_device *xe = gt_to_xe(job->q->gt);
+
+	xe_gt_WARN_ON(job->q->gt, !xe->info.has_page_reclaim_hw_assist);
+	job->prl = *prl;
+	/* Pair with put in job_destroy */
+	xe_page_reclaim_entries_get(job->prl.entries);
+}
+
 static void xe_tlb_inval_job_destroy(struct kref *ref)
 {
 	struct xe_tlb_inval_job *job = container_of(ref, typeof(*job),
@@ -150,6 +181,9 @@ static void xe_tlb_inval_job_destroy(struct kref *ref)
 	struct xe_device *xe = gt_to_xe(q->gt);
 	struct xe_vm *vm = job->vm;
 
+	/* BO creation retains a copy (if used), so no longer needed */
+	xe_page_reclaim_entries_put(job->prl.entries);
+
 	if (!job->fence_armed)
 		kfree(ifence);
 	else
diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_job.h b/drivers/gpu/drm/xe/xe_tlb_inval_job.h
index 4d6df1a6c6ca..03d6e21cd611 100644
--- a/drivers/gpu/drm/xe/xe_tlb_inval_job.h
+++ b/drivers/gpu/drm/xe/xe_tlb_inval_job.h
@@ -12,6 +12,7 @@ struct dma_fence;
 struct xe_dep_scheduler;
 struct xe_exec_queue;
 struct xe_migrate;
+struct xe_page_reclaim_list;
 struct xe_tlb_inval;
 struct xe_tlb_inval_job;
 struct xe_vm;
@@ -21,6 +22,9 @@ xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval,
 			struct xe_dep_scheduler *dep_scheduler,
 			struct xe_vm *vm, u64 start, u64 end, int type);
 
+void xe_tlb_inval_job_add_page_reclaim(struct xe_tlb_inval_job *job,
+				       struct xe_page_reclaim_list *prl);
+
 int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job);
 
 struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job,
diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_types.h b/drivers/gpu/drm/xe/xe_tlb_inval_types.h
index 8f8b060e9005..48d1503e8460 100644
--- a/drivers/gpu/drm/xe/xe_tlb_inval_types.h
+++ b/drivers/gpu/drm/xe/xe_tlb_inval_types.h
@@ -9,6 +9,7 @@
 #include <linux/workqueue.h>
 #include <linux/dma-fence.h>
 
+struct drm_suballoc;
 struct xe_tlb_inval;
 
 /** struct xe_tlb_inval_ops - TLB invalidation ops (backend) */
@@ -40,12 +41,13 @@ struct xe_tlb_inval_ops {
 	 * @start: Start address
 	 * @end: End address
 	 * @asid: Address space ID
+	 * @prl_sa: Suballocation for page reclaim list
 	 *
 	 * Return 0 on success, -ECANCELED if backend is mid-reset, error on
 	 * failure
 	 */
 	int (*ppgtt)(struct xe_tlb_inval *tlb_inval, u32 seqno, u64 start,
-		     u64 end, u32 asid);
+		     u64 end, u32 asid, struct drm_suballoc *prl_sa);
 
 	/**
 	 * @initialized: Backend is initialized
@@ -80,6 +82,7 @@ struct xe_tlb_inval {
 	const struct xe_tlb_inval_ops *ops;
 	/** @tlb_inval.seqno: TLB invalidation seqno, protected by CT lock */
 #define TLB_INVALIDATION_SEQNO_MAX	0x100000
+#define TLB_INVALIDATION_SEQNO_INVALID	TLB_INVALIDATION_SEQNO_MAX
 	int seqno;
 	/** @tlb_invalidation.seqno_lock: protects @tlb_invalidation.seqno */
 	struct mutex seqno_lock;
diff --git a/drivers/gpu/drm/xe/xe_trace.h b/drivers/gpu/drm/xe/xe_trace.h
index 79a97b086cb2..6d12fcc13f43 100644
--- a/drivers/gpu/drm/xe/xe_trace.h
+++ b/drivers/gpu/drm/xe/xe_trace.h
@@ -13,6 +13,7 @@
 #include <linux/types.h>
 
 #include "xe_exec_queue_types.h"
+#include "xe_exec_queue.h"
 #include "xe_gpu_scheduler_types.h"
 #include "xe_gt_types.h"
 #include "xe_guc_exec_queue_types.h"
@@ -97,11 +98,51 @@ DECLARE_EVENT_CLASS(xe_exec_queue,
 			      __entry->guc_state, __entry->flags)
 );
 
+DECLARE_EVENT_CLASS(xe_exec_queue_multi_queue,
+		    TP_PROTO(struct xe_exec_queue *q),
+		    TP_ARGS(q),
+
+		    TP_STRUCT__entry(
+			     __string(dev, __dev_name_eq(q))
+			     __field(enum xe_engine_class, class)
+			     __field(u32, logical_mask)
+			     __field(u8, gt_id)
+			     __field(u16, width)
+			     __field(u32, guc_id)
+			     __field(u32, guc_state)
+			     __field(u32, flags)
+			     __field(u32, primary)
+			     ),
+
+		    TP_fast_assign(
+			   __assign_str(dev);
+			   __entry->class = q->class;
+			   __entry->logical_mask = q->logical_mask;
+			   __entry->gt_id = q->gt->info.id;
+			   __entry->width = q->width;
+			   __entry->guc_id = q->guc->id;
+			   __entry->guc_state = atomic_read(&q->guc->state);
+			   __entry->flags = q->flags;
+			   __entry->primary = xe_exec_queue_multi_queue_primary(q)->guc->id;
+			   ),
+
+		    TP_printk("dev=%s, %d:0x%x, gt=%d, width=%d guc_id=%d, guc_state=0x%x, flags=0x%x, primary=%d",
+			      __get_str(dev), __entry->class, __entry->logical_mask,
+			      __entry->gt_id, __entry->width, __entry->guc_id,
+			      __entry->guc_state, __entry->flags,
+			      __entry->primary)
+);
+
 DEFINE_EVENT(xe_exec_queue, xe_exec_queue_create,
 	     TP_PROTO(struct xe_exec_queue *q),
 	     TP_ARGS(q)
 );
 
+DEFINE_EVENT(xe_exec_queue_multi_queue, xe_exec_queue_create_multi_queue,
+	     TP_PROTO(struct xe_exec_queue *q),
+	     TP_ARGS(q)
+);
+
 DEFINE_EVENT(xe_exec_queue, xe_exec_queue_supress_resume,
 	     TP_PROTO(struct xe_exec_queue *q),
 	     TP_ARGS(q)
@@ -172,6 +213,11 @@ DEFINE_EVENT(xe_exec_queue, xe_exec_queue_memory_cat_error,
 	     TP_ARGS(q)
 );
 
+DEFINE_EVENT(xe_exec_queue, xe_exec_queue_cgp_context_error,
+	     TP_PROTO(struct xe_exec_queue *q),
+	     TP_ARGS(q)
+);
+
 DEFINE_EVENT(xe_exec_queue, xe_exec_queue_stop,
 	     TP_PROTO(struct xe_exec_queue *q),
 	     TP_ARGS(q)
diff --git a/drivers/gpu/drm/xe/xe_uc.c b/drivers/gpu/drm/xe/xe_uc.c
index 465bda355443..157520ea1783 100644
--- a/drivers/gpu/drm/xe/xe_uc.c
+++ b/drivers/gpu/drm/xe/xe_uc.c
@@ -218,9 +218,12 @@ int xe_uc_load_hw(struct xe_uc *uc)
 
 	xe_guc_engine_activity_enable_stats(&uc->guc);
 
-	/* We don't fail the driver load if HuC fails to auth, but let's warn */
+	/* We don't fail the driver load if HuC fails to auth */
 	ret = xe_huc_auth(&uc->huc, XE_HUC_AUTH_VIA_GUC);
-	xe_gt_assert(uc_to_gt(uc), !ret);
+	if (ret)
+		xe_gt_err(uc_to_gt(uc),
+			  "HuC authentication failed (%pe), continuing with no HuC\n",
+			  ERR_PTR(ret));
 
 	/* GSC load is async */
 	xe_gsc_load_start(&uc->gsc);
@@ -301,6 +304,34 @@ int xe_uc_suspend(struct xe_uc *uc)
 	return xe_guc_suspend(&uc->guc);
 }
 
+/**
+ * xe_uc_runtime_suspend() - UC runtime suspend
+ * @uc: the UC object
+ *
+ * Runtime suspend all UCs.
+ */
+void xe_uc_runtime_suspend(struct xe_uc *uc)
+{
+	if (!xe_device_uc_enabled(uc_to_xe(uc)))
+		return;
+
+	xe_guc_runtime_suspend(&uc->guc);
+}
+
+/**
+ * xe_uc_runtime_resume() - UC runtime resume
+ * @uc: the UC object
+ *
+ * Runtime resume all UCs.
+ */
+void xe_uc_runtime_resume(struct xe_uc *uc)
+{
+	if (!xe_device_uc_enabled(uc_to_xe(uc)))
+		return;
+
+	xe_guc_runtime_resume(&uc->guc);
+}
+
 /**
  * xe_uc_declare_wedged() - Declare UC wedged
  * @uc: the UC object
diff --git a/drivers/gpu/drm/xe/xe_uc.h b/drivers/gpu/drm/xe/xe_uc.h
index 21c9306098cf..5398da1a8097 100644
--- a/drivers/gpu/drm/xe/xe_uc.h
+++ b/drivers/gpu/drm/xe/xe_uc.h
@@ -14,6 +14,8 @@ int xe_uc_init_post_hwconfig(struct xe_uc *uc);
 int xe_uc_load_hw(struct xe_uc *uc);
 void xe_uc_gucrc_disable(struct xe_uc *uc);
 int xe_uc_reset_prepare(struct xe_uc *uc);
+void xe_uc_runtime_resume(struct xe_uc *uc);
+void xe_uc_runtime_suspend(struct xe_uc *uc);
 void xe_uc_stop_prepare(struct xe_uc *uc);
 void xe_uc_stop(struct xe_uc *uc);
 int xe_uc_start(struct xe_uc *uc);
diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c
index 622b76078567..dcb4a32e7a64 100644
--- a/drivers/gpu/drm/xe/xe_uc_fw.c
+++ b/drivers/gpu/drm/xe/xe_uc_fw.c
@@ -115,11 +115,11 @@ struct fw_blobs_by_type {
 #define XE_GT_TYPE_ANY XE_GT_TYPE_UNINITIALIZED
 
 #define XE_GUC_FIRMWARE_DEFS(fw_def, mmp_ver, major_ver)					\
-	fw_def(PANTHERLAKE,	GT_TYPE_ANY,	major_ver(xe,	guc,	ptl,	70, 49, 4))	\
-	fw_def(BATTLEMAGE,	GT_TYPE_ANY,	major_ver(xe,	guc,	bmg,	70, 49, 4))	\
-	fw_def(LUNARLAKE,	GT_TYPE_ANY,	major_ver(xe,	guc,	lnl,	70, 45, 2))	\
-	fw_def(METEORLAKE,	GT_TYPE_ANY,	major_ver(i915,	guc,	mtl,	70, 44, 1))	\
-	fw_def(DG2,		GT_TYPE_ANY,	major_ver(i915,	guc,	dg2,	70, 45, 2))	\
+	fw_def(PANTHERLAKE,	GT_TYPE_ANY,	major_ver(xe,	guc,	ptl,	70, 54, 0))	\
+	fw_def(BATTLEMAGE,	GT_TYPE_ANY,	major_ver(xe,	guc,	bmg,	70, 54, 0))	\
+	fw_def(LUNARLAKE,	GT_TYPE_ANY,	major_ver(xe,	guc,	lnl,	70, 53, 0))	\
+	fw_def(METEORLAKE,	GT_TYPE_ANY,	major_ver(i915,	guc,	mtl,	70, 53, 0))	\
+	fw_def(DG2,		GT_TYPE_ANY,	major_ver(i915,	guc,	dg2,	70, 53, 0))	\
 	fw_def(DG1,		GT_TYPE_ANY,	major_ver(i915,	guc,	dg1,	70, 44, 1))	\
 	fw_def(ALDERLAKE_N,	GT_TYPE_ANY,	major_ver(i915,	guc,	tgl,	70, 44, 1))	\
 	fw_def(ALDERLAKE_P,	GT_TYPE_ANY,	major_ver(i915,	guc,	adlp,	70, 44, 1))	\
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 7cac646bdf1c..95e22ff95ea8 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -1508,7 +1508,10 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef)
 	INIT_WORK(&vm->destroy_work, vm_destroy_work_func);
 
 	INIT_LIST_HEAD(&vm->preempt.exec_queues);
-	vm->preempt.min_run_period_ms = 10;	/* FIXME: Wire up to uAPI */
+	if (flags & XE_VM_FLAG_FAULT_MODE)
+		vm->preempt.min_run_period_ms = xe->min_run_period_pf_ms;
+	else
+		vm->preempt.min_run_period_ms = xe->min_run_period_lr_ms;
 
 	for_each_tile(tile, xe, id)
 		xe_range_fence_tree_init(&vm->rftree[id]);
@@ -2233,6 +2236,7 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops,
 	struct drm_gpuva_ops *ops;
 	struct drm_gpuva_op *__op;
 	struct drm_gpuvm_bo *vm_bo;
+	u64 range_start = addr;
 	u64 range_end = addr + range;
 	int err;
 
@@ -2245,10 +2249,16 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops,
 
 	switch (operation) {
 	case DRM_XE_VM_BIND_OP_MAP:
+		if (flags & DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR) {
+			xe_vm_find_cpu_addr_mirror_vma_range(vm, &range_start, &range_end);
+			vops->flags |= XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP;
+		}
+
+		fallthrough;
 	case DRM_XE_VM_BIND_OP_MAP_USERPTR: {
 		struct drm_gpuvm_map_req map_req = {
-			.map.va.addr = addr,
-			.map.va.range = range,
+			.map.va.addr = range_start,
+			.map.va.range = range_end - range_start,
 			.map.gem.obj = obj,
 			.map.gem.offset = bo_offset_or_userptr,
 		};
@@ -2448,8 +2458,17 @@ static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op,
 		if (IS_ERR(vma))
 			return vma;
 
-		if (xe_vma_is_userptr(vma))
+		if (xe_vma_is_userptr(vma)) {
 			err = xe_vma_userptr_pin_pages(to_userptr_vma(vma));
+			/*
+			 * -EBUSY has dedicated meaning that a user fence
+			 * attached to the VMA is busy, in practice
+			 * xe_vma_userptr_pin_pages can only fail with -EBUSY if
+			 * we are low on memory so convert this to -ENOMEM.
+			 */
+			if (err == -EBUSY)
+				err = -ENOMEM;
+		}
 	}
 	if (err) {
 		prep_vma_destroy(vm, vma, false);
@@ -2724,7 +2743,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops,
 
 			if (xe_vma_is_cpu_addr_mirror(vma) &&
 			    xe_svm_has_mapping(vm, xe_vma_start(vma),
-					       xe_vma_end(vma)))
+					       xe_vma_end(vma)) &&
+			    !(vops->flags & XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP))
 				return -EBUSY;
 
 			if (!xe_vma_is_cpu_addr_mirror(vma))
@@ -3104,19 +3124,19 @@ static struct dma_fence *ops_execute(struct xe_vm *vm,
 	struct dma_fence *fence = NULL;
 	struct dma_fence **fences = NULL;
 	struct dma_fence_array *cf = NULL;
-	int number_tiles = 0, current_fence = 0, n_fence = 0, err;
+	int number_tiles = 0, current_fence = 0, n_fence = 0, err, i;
 	u8 id;
 
 	number_tiles = vm_ops_setup_tile_args(vm, vops);
 	if (number_tiles == 0)
 		return ERR_PTR(-ENODATA);
 
-	if (vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT) {
-		for_each_tile(tile, vm->xe, id)
-			++n_fence;
-	} else {
-		for_each_tile(tile, vm->xe, id)
-			n_fence += (1 + XE_MAX_GT_PER_TILE);
+	for_each_tile(tile, vm->xe, id) {
+		++n_fence;
+
+		if (!(vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT))
+			for_each_tlb_inval(i)
+				++n_fence;
 	}
 
 	fences = kmalloc_array(n_fence, sizeof(*fences), GFP_KERNEL);
@@ -3146,7 +3166,6 @@ static struct dma_fence *ops_execute(struct xe_vm *vm,
 
 	for_each_tile(tile, vm->xe, id) {
 		struct xe_exec_queue *q = vops->pt_update_ops[tile->id].q;
-		int i;
 
 		fence = NULL;
 		if (!vops->pt_update_ops[id].num_ops)
@@ -3211,7 +3230,8 @@ static void op_add_ufence(struct xe_vm *vm, struct xe_vma_op *op,
 {
 	switch (op->base.op) {
 	case DRM_GPUVA_OP_MAP:
-		vma_add_ufence(op->map.vma, ufence);
+		if (!xe_vma_is_cpu_addr_mirror(op->map.vma))
+			vma_add_ufence(op->map.vma, ufence);
 		break;
 	case DRM_GPUVA_OP_REMAP:
 		if (op->remap.prev)
@@ -3324,6 +3344,9 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm,
 	if (XE_IOCTL_DBG(xe, args->extensions))
 		return -EINVAL;
 
+	if (XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS))
+		return -EINVAL;
+
 	if (args->num_binds > 1) {
 		u64 __user *bind_user =
 			u64_to_user_ptr(args->vector_of_binds);
@@ -3484,6 +3507,10 @@ static int xe_vm_bind_ioctl_validate_bo(struct xe_device *xe, struct xe_bo *bo,
 {
 	u16 coh_mode;
 
+	if (XE_IOCTL_DBG(xe, (bo->flags & XE_BO_FLAG_NO_COMPRESSION) &&
+			 xe_pat_index_get_comp_en(xe, pat_index)))
+		return -EINVAL;
+
 	if (XE_IOCTL_DBG(xe, range > xe_bo_size(bo)) ||
 	    XE_IOCTL_DBG(xe, obj_offset >
 			 xe_bo_size(bo) - range)) {
@@ -3907,7 +3934,7 @@ int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start,
 
 		err = xe_tlb_inval_range(&tile->primary_gt->tlb_inval,
 					 &fence[fence_id], start, end,
-					 vm->usm.asid);
+					 vm->usm.asid, NULL);
 		if (err)
 			goto wait;
 		++fence_id;
@@ -3920,7 +3947,7 @@ int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start,
 
 		err = xe_tlb_inval_range(&tile->media_gt->tlb_inval,
 					 &fence[fence_id], start, end,
-					 vm->usm.asid);
+					 vm->usm.asid, NULL);
 		if (err)
 			goto wait;
 		++fence_id;
@@ -4026,10 +4053,18 @@ int xe_vm_validate_protected(struct xe_vm *vm)
 }
 
 struct xe_vm_snapshot {
+	int uapi_flags;
 	unsigned long num_snaps;
 	struct {
 		u64 ofs, bo_ofs;
 		unsigned long len;
+#define XE_VM_SNAP_FLAG_USERPTR		BIT(0)
+#define XE_VM_SNAP_FLAG_READ_ONLY	BIT(1)
+#define XE_VM_SNAP_FLAG_IS_NULL		BIT(2)
+		unsigned long flags;
+		int uapi_mem_region;
+		int pat_index;
+		int cpu_caching;
 		struct xe_bo *bo;
 		void *data;
 		struct mm_struct *mm;
@@ -4058,6 +4093,13 @@ struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm)
 		goto out_unlock;
 	}
 
+	if (vm->flags & XE_VM_FLAG_FAULT_MODE)
+		snap->uapi_flags |= DRM_XE_VM_CREATE_FLAG_FAULT_MODE;
+	if (vm->flags & XE_VM_FLAG_LR_MODE)
+		snap->uapi_flags |= DRM_XE_VM_CREATE_FLAG_LR_MODE;
+	if (vm->flags & XE_VM_FLAG_SCRATCH_PAGE)
+		snap->uapi_flags |= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE;
+
 	snap->num_snaps = num_snaps;
 	i = 0;
 	drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) {
@@ -4070,9 +4112,25 @@ struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm)
 
 		snap->snap[i].ofs = xe_vma_start(vma);
 		snap->snap[i].len = xe_vma_size(vma);
+		snap->snap[i].flags = xe_vma_read_only(vma) ?
+			XE_VM_SNAP_FLAG_READ_ONLY : 0;
+		snap->snap[i].pat_index = vma->attr.pat_index;
 		if (bo) {
+			snap->snap[i].cpu_caching = bo->cpu_caching;
 			snap->snap[i].bo = xe_bo_get(bo);
 			snap->snap[i].bo_ofs = xe_vma_bo_offset(vma);
+			switch (bo->ttm.resource->mem_type) {
+			case XE_PL_SYSTEM:
+			case XE_PL_TT:
+				snap->snap[i].uapi_mem_region = 0;
+				break;
+			case XE_PL_VRAM0:
+				snap->snap[i].uapi_mem_region = 1;
+				break;
+			case XE_PL_VRAM1:
+				snap->snap[i].uapi_mem_region = 2;
+				break;
+			}
 		} else if (xe_vma_is_userptr(vma)) {
 			struct mm_struct *mm =
 				to_userptr_vma(vma)->userptr.notifier.mm;
@@ -4083,8 +4141,14 @@ struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm)
 				snap->snap[i].data = ERR_PTR(-EFAULT);
 
 			snap->snap[i].bo_ofs = xe_vma_userptr(vma);
+			snap->snap[i].flags |= XE_VM_SNAP_FLAG_USERPTR;
+			snap->snap[i].uapi_mem_region = 0;
+		} else if (xe_vma_is_null(vma)) {
+			snap->snap[i].flags |= XE_VM_SNAP_FLAG_IS_NULL;
+			snap->snap[i].uapi_mem_region = -1;
 		} else {
 			snap->snap[i].data = ERR_PTR(-ENOENT);
+			snap->snap[i].uapi_mem_region = -1;
 		}
 		i++;
 	}
@@ -4103,7 +4167,8 @@ void xe_vm_snapshot_capture_delayed(struct xe_vm_snapshot *snap)
 		struct xe_bo *bo = snap->snap[i].bo;
 		int err;
 
-		if (IS_ERR(snap->snap[i].data))
+		if (IS_ERR(snap->snap[i].data) ||
+		    snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL)
 			continue;
 
 		snap->snap[i].data = kvmalloc(snap->snap[i].len, GFP_USER);
@@ -4149,15 +4214,32 @@ void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p)
 		return;
 	}
 
+	drm_printf(p, "VM.uapi_flags: 0x%x\n", snap->uapi_flags);
 	for (i = 0; i < snap->num_snaps; i++) {
 		drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len);
 
+		drm_printf(p, "[%llx].properties: %s|%s|mem_region=0x%lx|pat_index=%d|cpu_caching=%d\n",
+			   snap->snap[i].ofs,
+			   snap->snap[i].flags & XE_VM_SNAP_FLAG_READ_ONLY ?
+			   "read_only" : "read_write",
+			   snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL ?
+			   "null_sparse" :
+			   snap->snap[i].flags & XE_VM_SNAP_FLAG_USERPTR ?
+			   "userptr" : "bo",
+			   snap->snap[i].uapi_mem_region == -1 ? 0 :
+			   BIT(snap->snap[i].uapi_mem_region),
+			   snap->snap[i].pat_index,
+			   snap->snap[i].cpu_caching);
+
 		if (IS_ERR(snap->snap[i].data)) {
 			drm_printf(p, "[%llx].error: %li\n", snap->snap[i].ofs,
 				   PTR_ERR(snap->snap[i].data));
 			continue;
 		}
 
+		if (snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL)
+			continue;
+
 		drm_printf(p, "[%llx].data: ", snap->snap[i].ofs);
 
 		for (j = 0; j < snap->snap[i].len; j += sizeof(u32)) {
@@ -4311,6 +4393,8 @@ static int xe_vm_alloc_vma(struct xe_vm *vm,
 
 	if (is_madvise)
 		vops.flags |= XE_VMA_OPS_FLAG_MADVISE;
+	else
+		vops.flags |= XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP;
 
 	err = vm_bind_ioctl_ops_parse(vm, ops, &vops);
 	if (err)
@@ -4384,6 +4468,46 @@ int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range)
 	return xe_vm_alloc_vma(vm, &map_req, true);
 }
 
+static bool is_cpu_addr_vma_with_default_attr(struct xe_vma *vma)
+{
+	return vma && xe_vma_is_cpu_addr_mirror(vma) &&
+	       xe_vma_has_default_mem_attrs(vma);
+}
+
+/**
+ * xe_vm_find_cpu_addr_mirror_vma_range - Extend a VMA range to include adjacent CPU-mirrored VMAs
+ * @vm: VM to search within
+ * @start: Input/output pointer to the starting address of the range
+ * @end: Input/output pointer to the end address of the range
+ *
+ * Given a range defined by @start and @range, this function checks the VMAs
+ * immediately before and after the range. If those neighboring VMAs are
+ * CPU-address-mirrored and have default memory attributes, the function
+ * updates @start and @range to include them. This extended range can then
+ * be used for merging or other operations that require a unified VMA.
+ *
+ * The function does not perform the merge itself; it only computes the
+ * mergeable boundaries.
+ */
+void xe_vm_find_cpu_addr_mirror_vma_range(struct xe_vm *vm, u64 *start, u64 *end)
+{
+	struct xe_vma *prev, *next;
+
+	lockdep_assert_held(&vm->lock);
+
+	if (*start >= SZ_4K) {
+		prev = xe_vm_find_vma_by_addr(vm, *start - SZ_4K);
+		if (is_cpu_addr_vma_with_default_attr(prev))
+			*start = xe_vma_start(prev);
+	}
+
+	if (*end < vm->size) {
+		next = xe_vm_find_vma_by_addr(vm, *end + 1);
+		if (is_cpu_addr_vma_with_default_attr(next))
+			*end = xe_vma_end(next);
+	}
+}
+
 /**
  * xe_vm_alloc_cpu_addr_mirror_vma - Allocate CPU addr mirror vma
  * @vm: Pointer to the xe_vm structure
diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h
index ef8a5019574e..361f10b3c453 100644
--- a/drivers/gpu/drm/xe/xe_vm.h
+++ b/drivers/gpu/drm/xe/xe_vm.h
@@ -68,6 +68,9 @@ xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range);
 
 bool xe_vma_has_default_mem_attrs(struct xe_vma *vma);
 
+void xe_vm_find_cpu_addr_mirror_vma_range(struct xe_vm *vm,
+					  u64 *start,
+					  u64 *end);
 /**
  * xe_vm_has_scratch() - Whether the vm is configured for scratch PTEs
  * @vm: The vm
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index ccd6cc090309..18bad1dd08e6 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -263,7 +263,7 @@ struct xe_vm {
 		 * @min_run_period_ms: The minimum run period before preempting
 		 * an engine again
 		 */
-		s64 min_run_period_ms;
+		unsigned int min_run_period_ms;
 		/** @exec_queues: list of exec queues attached to this VM */
 		struct list_head exec_queues;
 		/** @num_exec_queues: number exec queues attached to this VM */
@@ -467,6 +467,7 @@ struct xe_vma_ops {
 #define XE_VMA_OPS_FLAG_MADVISE          BIT(1)
 #define XE_VMA_OPS_ARRAY_OF_BINDS	 BIT(2)
 #define XE_VMA_OPS_FLAG_SKIP_TLB_WAIT	 BIT(3)
+#define XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP  BIT(4)
 	u32 flags;
 #ifdef TEST_VM_OPS_ERROR
 	/** @inject_error: inject error to test error handling */
diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c
index d50baefcd124..1b9e9b028975 100644
--- a/drivers/gpu/drm/xe/xe_vram.c
+++ b/drivers/gpu/drm/xe/xe_vram.c
@@ -156,12 +156,11 @@ static int determine_lmem_bar_size(struct xe_device *xe, struct xe_vram_region *
 static int get_flat_ccs_offset(struct xe_gt *gt, u64 tile_size, u64 *poffset)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	u64 offset;
 	u32 reg;
 
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 
 	if (GRAPHICS_VER(xe) >= 20) {
@@ -193,7 +192,6 @@ static int get_flat_ccs_offset(struct xe_gt *gt, u64 tile_size, u64 *poffset)
 		offset = (u64)REG_FIELD_GET(XEHP_FLAT_CCS_PTR, reg) * SZ_64K;
 	}
 
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	*poffset = offset;
 
 	return 0;
diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c
index 3764abca3d4f..a93717e77da0 100644
--- a/drivers/gpu/drm/xe/xe_wa.c
+++ b/drivers/gpu/drm/xe/xe_wa.c
@@ -15,6 +15,7 @@
 
 #include "regs/xe_engine_regs.h"
 #include "regs/xe_gt_regs.h"
+#include "regs/xe_guc_regs.h"
 #include "regs/xe_regs.h"
 #include "xe_device_types.h"
 #include "xe_force_wake.h"
@@ -216,20 +217,6 @@ static const struct xe_rtp_entry_sr gt_was[] = {
 	  XE_RTP_ACTIONS(SET(XELPMP_SQCNT1, ENFORCE_RAR))
 	},
 
-	/* Xe2_LPG */
-
-	{ XE_RTP_NAME("16020975621"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)),
-	  XE_RTP_ACTIONS(SET(XEHP_SLICE_UNIT_LEVEL_CLKGATE, SBEUNIT_CLKGATE_DIS))
-	},
-	{ XE_RTP_NAME("14018157293"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)),
-	  XE_RTP_ACTIONS(SET(XEHPC_L3CLOS_MASK(0), ~0),
-			 SET(XEHPC_L3CLOS_MASK(1), ~0),
-			 SET(XEHPC_L3CLOS_MASK(2), ~0),
-			 SET(XEHPC_L3CLOS_MASK(3), ~0))
-	},
-
 	/* Xe2_LPM */
 
 	{ XE_RTP_NAME("14017421178"),
@@ -270,14 +257,6 @@ static const struct xe_rtp_entry_sr gt_was[] = {
 	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F1C(0), MFXPIPE_CLKGATE_DIS)),
 	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
 	},
-	{ XE_RTP_NAME("14020316580"),
-	  XE_RTP_RULES(MEDIA_VERSION(1301)),
-	  XE_RTP_ACTIONS(CLR(POWERGATE_ENABLE,
-			     VDN_HCP_POWERGATE_ENABLE(0) |
-			     VDN_MFXVDENC_POWERGATE_ENABLE(0) |
-			     VDN_HCP_POWERGATE_ENABLE(2) |
-			     VDN_MFXVDENC_POWERGATE_ENABLE(2))),
-	},
 	{ XE_RTP_NAME("14019449301"),
 	  XE_RTP_RULES(MEDIA_VERSION(1301), ENGINE_CLASS(VIDEO_DECODE)),
 	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F08(0), CG3DDISHRS_CLKGATE_DIS)),
@@ -323,6 +302,10 @@ static const struct xe_rtp_entry_sr gt_was[] = {
 	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), RAMDFTUNIT_CLKGATE_DIS)),
 	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
 	},
+	{ XE_RTP_NAME("16028005424"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005)),
+	  XE_RTP_ACTIONS(SET(GUC_INTR_CHICKEN, DISABLE_SIGNALING_ENGINES))
+	},
 };
 
 static const struct xe_rtp_entry_sr engine_was[] = {
@@ -512,11 +495,6 @@ static const struct xe_rtp_entry_sr engine_was[] = {
 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, XE2_ALLOC_DPA_STARVE_FIX_DIS))
 	},
-	{ XE_RTP_NAME("14018957109"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
-		       FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN5, DISABLE_SAMPLE_G_PERFORMANCE))
-	},
 	{ XE_RTP_NAME("14020338487"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(ROW_CHICKEN3, XE2_EUPEND_CHK_FLUSH_DIS))
@@ -526,11 +504,6 @@ static const struct xe_rtp_entry_sr engine_was[] = {
 		       FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(ROW_CHICKEN4, DISABLE_TDL_PUSH))
 	},
-	{ XE_RTP_NAME("14019322943"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
-		       FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, TGM_WRITE_EOM_FORCE))
-	},
 	{ XE_RTP_NAME("14018471104"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, ENABLE_SMP_LD_RENDER_SURFACE_CONTROL))
@@ -701,7 +674,7 @@ static const struct xe_rtp_entry_sr engine_was[] = {
 	  XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN7, CLEAR_OPTIMIZATION_DISABLE))
 	},
 	{ XE_RTP_NAME("18041344222"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3001),
+	  XE_RTP_RULES(GRAPHICS_VERSION(3000),
 		       FUNC(xe_rtp_match_first_render_or_compute),
 		       FUNC(xe_rtp_match_not_sriov_vf),
 		       FUNC(xe_rtp_match_gt_has_discontiguous_dss_groups)),
@@ -807,17 +780,6 @@ static const struct xe_rtp_entry_sr lrc_was[] = {
 
 	/* Xe2_LPG */
 
-	{ XE_RTP_NAME("16020518922"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
-		       ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(FF_MODE,
-			     DIS_TE_AUTOSTRIP |
-			     DIS_MESH_PARTIAL_AUTOSTRIP |
-			     DIS_MESH_AUTOSTRIP),
-			 SET(VFLSKPD,
-			     DIS_PARTIAL_AUTOSTRIP |
-			     DIS_AUTOSTRIP))
-	},
 	{ XE_RTP_NAME("14019386621"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(SET(VF_SCRATCHPAD, XE2_VFG_TED_CREDIT_INTERFACE_DISABLE))
@@ -826,20 +788,10 @@ static const struct xe_rtp_entry_sr lrc_was[] = {
 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT))
 	},
-	{ XE_RTP_NAME("14020013138"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
-		       ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS))
-	},
 	{ XE_RTP_NAME("14019988906"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD))
 	},
-	{ XE_RTP_NAME("16020183090"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
-		       ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(INSTPM(RENDER_RING_BASE), ENABLE_SEMAPHORE_POLL_BIT))
-	},
 	{ XE_RTP_NAME("18033852989"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN1, DISABLE_BOTTOM_CLIP_RECTANGLE_TEST))
diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules
index fb38eb3d6e9a..5cd7fa6d2a5c 100644
--- a/drivers/gpu/drm/xe/xe_wa_oob.rules
+++ b/drivers/gpu/drm/xe/xe_wa_oob.rules
@@ -16,15 +16,11 @@
 16017236439	PLATFORM(PVC)
 14019821291	MEDIA_VERSION_RANGE(1300, 2000)
 14015076503	MEDIA_VERSION(1300)
-16020292621	GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)
-14018913170	GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)
-		MEDIA_VERSION(2000), GRAPHICS_STEP(A0, A1)
-		GRAPHICS_VERSION_RANGE(1270, 1274)
+14018913170	GRAPHICS_VERSION_RANGE(1270, 1274)
 		MEDIA_VERSION(1300)
 		PLATFORM(DG2)
 14018094691	GRAPHICS_VERSION_RANGE(2001, 2002)
 		GRAPHICS_VERSION(2004)
-14019882105	GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)
 18024947630	GRAPHICS_VERSION(2001)
 		GRAPHICS_VERSION(2004)
 		MEDIA_VERSION(2000)
@@ -76,3 +72,4 @@
 
 15015404425_disable	PLATFORM(PANTHERLAKE), MEDIA_STEP(B0, FOREVER)
 16026007364    MEDIA_VERSION(3000)
+14020316580    MEDIA_VERSION(1301)
diff --git a/drivers/gpu/nova-core/Kconfig b/drivers/gpu/nova-core/Kconfig
index 20d3e6d0d796..527920f9c4d3 100644
--- a/drivers/gpu/nova-core/Kconfig
+++ b/drivers/gpu/nova-core/Kconfig
@@ -3,7 +3,7 @@ config NOVA_CORE
 	depends on 64BIT
 	depends on PCI
 	depends on RUST
-	depends on RUST_FW_LOADER_ABSTRACTIONS
+	select RUST_FW_LOADER_ABSTRACTIONS
 	select AUXILIARY_BUS
 	default n
 	help
diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs
index 6f946d14868a..3991ccc0c10f 100644
--- a/drivers/gpu/nova-core/gsp/cmdq.rs
+++ b/drivers/gpu/nova-core/gsp/cmdq.rs
@@ -588,21 +588,23 @@ fn wait_for_msg(&self, timeout: Delta) -> Result<GspMessage<'_>> {
             header.length(),
         );
 
+        let payload_length = header.payload_length();
+
         // Check that the driver read area is large enough for the message.
-        if slice_1.len() + slice_2.len() < header.length() {
+        if slice_1.len() + slice_2.len() < payload_length {
             return Err(EIO);
         }
 
         // Cut the message slices down to the actual length of the message.
-        let (slice_1, slice_2) = if slice_1.len() > header.length() {
-            // PANIC: we checked above that `slice_1` is at least as long as `msg_header.length()`.
-            (slice_1.split_at(header.length()).0, &slice_2[0..0])
+        let (slice_1, slice_2) = if slice_1.len() > payload_length {
+            // PANIC: we checked above that `slice_1` is at least as long as `payload_length`.
+            (slice_1.split_at(payload_length).0, &slice_2[0..0])
         } else {
             (
                 slice_1,
                 // PANIC: we checked above that `slice_1.len() + slice_2.len()` is at least as
-                // large as `msg_header.length()`.
-                slice_2.split_at(header.length() - slice_1.len()).0,
+                // large as `payload_length`.
+                slice_2.split_at(payload_length - slice_1.len()).0,
             )
         };
 
diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs
index abffd6beec65..caeb0d251fe5 100644
--- a/drivers/gpu/nova-core/gsp/fw.rs
+++ b/drivers/gpu/nova-core/gsp/fw.rs
@@ -141,8 +141,8 @@ unsafe impl AsBytes for GspFwWprMeta {}
 // are valid.
 unsafe impl FromBytes for GspFwWprMeta {}
 
-type GspFwWprMetaBootResumeInfo = r570_144::GspFwWprMeta__bindgen_ty_1;
-type GspFwWprMetaBootInfo = r570_144::GspFwWprMeta__bindgen_ty_1__bindgen_ty_1;
+type GspFwWprMetaBootResumeInfo = bindings::GspFwWprMeta__bindgen_ty_1;
+type GspFwWprMetaBootInfo = bindings::GspFwWprMeta__bindgen_ty_1__bindgen_ty_1;
 
 impl GspFwWprMeta {
     /// Fill in and return a `GspFwWprMeta` suitable for booting `gsp_firmware` using the
@@ -150,8 +150,8 @@ impl GspFwWprMeta {
     pub(crate) fn new(gsp_firmware: &GspFirmware, fb_layout: &FbLayout) -> Self {
         Self(bindings::GspFwWprMeta {
             // CAST: we want to store the bits of `GSP_FW_WPR_META_MAGIC` unmodified.
-            magic: r570_144::GSP_FW_WPR_META_MAGIC as u64,
-            revision: u64::from(r570_144::GSP_FW_WPR_META_REVISION),
+            magic: bindings::GSP_FW_WPR_META_MAGIC as u64,
+            revision: u64::from(bindings::GSP_FW_WPR_META_REVISION),
             sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_handle(),
             sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size),
             sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_handle(),
@@ -315,19 +315,19 @@ fn from(value: MsgFunction) -> Self {
 #[repr(u32)]
 pub(crate) enum SeqBufOpcode {
     // Core operation opcodes
-    CoreReset = r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESET,
-    CoreResume = r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME,
-    CoreStart = r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_START,
-    CoreWaitForHalt = r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT,
+    CoreReset = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESET,
+    CoreResume = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME,
+    CoreStart = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_START,
+    CoreWaitForHalt = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT,
 
     // Delay opcode
-    DelayUs = r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_DELAY_US,
+    DelayUs = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_DELAY_US,
 
     // Register operation opcodes
-    RegModify = r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_MODIFY,
-    RegPoll = r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_POLL,
-    RegStore = r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_STORE,
-    RegWrite = r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE,
+    RegModify = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_MODIFY,
+    RegPoll = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_POLL,
+    RegStore = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_STORE,
+    RegWrite = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE,
 }
 
 impl fmt::Display for SeqBufOpcode {
@@ -351,25 +351,25 @@ impl TryFrom<u32> for SeqBufOpcode {
 
     fn try_from(value: u32) -> Result<SeqBufOpcode> {
         match value {
-            r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESET => {
+            bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESET => {
                 Ok(SeqBufOpcode::CoreReset)
             }
-            r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME => {
+            bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME => {
                 Ok(SeqBufOpcode::CoreResume)
             }
-            r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_START => {
+            bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_START => {
                 Ok(SeqBufOpcode::CoreStart)
             }
-            r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT => {
+            bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT => {
                 Ok(SeqBufOpcode::CoreWaitForHalt)
             }
-            r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_DELAY_US => Ok(SeqBufOpcode::DelayUs),
-            r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_MODIFY => {
+            bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_DELAY_US => Ok(SeqBufOpcode::DelayUs),
+            bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_MODIFY => {
                 Ok(SeqBufOpcode::RegModify)
             }
-            r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_POLL => Ok(SeqBufOpcode::RegPoll),
-            r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_STORE => Ok(SeqBufOpcode::RegStore),
-            r570_144::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE => Ok(SeqBufOpcode::RegWrite),
+            bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_POLL => Ok(SeqBufOpcode::RegPoll),
+            bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_STORE => Ok(SeqBufOpcode::RegStore),
+            bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE => Ok(SeqBufOpcode::RegWrite),
             _ => Err(EINVAL),
         }
     }
@@ -385,7 +385,7 @@ fn from(value: SeqBufOpcode) -> Self {
 /// Wrapper for GSP sequencer register write payload.
 #[repr(transparent)]
 #[derive(Copy, Clone)]
-pub(crate) struct RegWritePayload(r570_144::GSP_SEQ_BUF_PAYLOAD_REG_WRITE);
+pub(crate) struct RegWritePayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_WRITE);
 
 impl RegWritePayload {
     /// Returns the register address.
@@ -408,7 +408,7 @@ unsafe impl AsBytes for RegWritePayload {}
 /// Wrapper for GSP sequencer register modify payload.
 #[repr(transparent)]
 #[derive(Copy, Clone)]
-pub(crate) struct RegModifyPayload(r570_144::GSP_SEQ_BUF_PAYLOAD_REG_MODIFY);
+pub(crate) struct RegModifyPayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_MODIFY);
 
 impl RegModifyPayload {
     /// Returns the register address.
@@ -436,7 +436,7 @@ unsafe impl AsBytes for RegModifyPayload {}
 /// Wrapper for GSP sequencer register poll payload.
 #[repr(transparent)]
 #[derive(Copy, Clone)]
-pub(crate) struct RegPollPayload(r570_144::GSP_SEQ_BUF_PAYLOAD_REG_POLL);
+pub(crate) struct RegPollPayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_POLL);
 
 impl RegPollPayload {
     /// Returns the register address.
@@ -469,7 +469,7 @@ unsafe impl AsBytes for RegPollPayload {}
 /// Wrapper for GSP sequencer delay payload.
 #[repr(transparent)]
 #[derive(Copy, Clone)]
-pub(crate) struct DelayUsPayload(r570_144::GSP_SEQ_BUF_PAYLOAD_DELAY_US);
+pub(crate) struct DelayUsPayload(bindings::GSP_SEQ_BUF_PAYLOAD_DELAY_US);
 
 impl DelayUsPayload {
     /// Returns the delay value in microseconds.
@@ -487,7 +487,7 @@ unsafe impl AsBytes for DelayUsPayload {}
 /// Wrapper for GSP sequencer register store payload.
 #[repr(transparent)]
 #[derive(Copy, Clone)]
-pub(crate) struct RegStorePayload(r570_144::GSP_SEQ_BUF_PAYLOAD_REG_STORE);
+pub(crate) struct RegStorePayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_STORE);
 
 impl RegStorePayload {
     /// Returns the register address.
@@ -510,7 +510,7 @@ unsafe impl AsBytes for RegStorePayload {}
 
 /// Wrapper for GSP sequencer buffer command.
 #[repr(transparent)]
-pub(crate) struct SequencerBufferCmd(r570_144::GSP_SEQUENCER_BUFFER_CMD);
+pub(crate) struct SequencerBufferCmd(bindings::GSP_SEQUENCER_BUFFER_CMD);
 
 impl SequencerBufferCmd {
     /// Returns the opcode as a `SeqBufOpcode` enum, or error if invalid.
@@ -612,7 +612,7 @@ unsafe impl AsBytes for SequencerBufferCmd {}
 
 /// Wrapper for GSP run CPU sequencer RPC.
 #[repr(transparent)]
-pub(crate) struct RunCpuSequencer(r570_144::rpc_run_cpu_sequencer_v17_00);
+pub(crate) struct RunCpuSequencer(bindings::rpc_run_cpu_sequencer_v17_00);
 
 impl RunCpuSequencer {
     /// Returns the command index.
@@ -797,13 +797,6 @@ fn init(cmd_size: usize, function: MsgFunction) -> impl Init<Self, Error> {
     }
 }
 
-// SAFETY: We can't derive the Zeroable trait for this binding because the
-// procedural macro doesn't support the syntax used by bindgen to create the
-// __IncompleteArrayField types. So instead we implement it here, which is safe
-// because these are explicitly padded structures only containing types for
-// which any bit pattern, including all zeros, is valid.
-unsafe impl Zeroable for bindings::rpc_message_header_v {}
-
 /// GSP Message Element.
 ///
 /// This is essentially a message header expected to be followed by the message data.
@@ -853,11 +846,16 @@ pub(crate) fn set_checksum(&mut self, checksum: u32) {
         self.inner.checkSum = checksum;
     }
 
-    /// Returns the total length of the message.
+    /// Returns the length of the message's payload.
+    pub(crate) fn payload_length(&self) -> usize {
+        // `rpc.length` includes the length of the RPC message header.
+        num::u32_as_usize(self.inner.rpc.length)
+            .saturating_sub(size_of::<bindings::rpc_message_header_v>())
+    }
+
+    /// Returns the total length of the message, message and RPC headers included.
     pub(crate) fn length(&self) -> usize {
-        // `rpc.length` includes the length of the GspRpcHeader but not the message header.
-        size_of::<Self>() - size_of::<bindings::rpc_message_header_v>()
-            + num::u32_as_usize(self.inner.rpc.length)
+        size_of::<Self>() + self.payload_length()
     }
 
     // Returns the sequence number of the message.
diff --git a/drivers/gpu/nova-core/gsp/fw/r570_144.rs b/drivers/gpu/nova-core/gsp/fw/r570_144.rs
index 048234d1a9d1..e99d315ae74c 100644
--- a/drivers/gpu/nova-core/gsp/fw/r570_144.rs
+++ b/drivers/gpu/nova-core/gsp/fw/r570_144.rs
@@ -24,8 +24,11 @@
     unreachable_pub,
     unsafe_op_in_unsafe_fn
 )]
-use kernel::{
-    ffi,
-    prelude::Zeroable, //
-};
+use kernel::ffi;
+use pin_init::MaybeZeroable;
+
 include!("r570_144/bindings.rs");
+
+// SAFETY: This type has a size of zero, so its inclusion into another type should not affect their
+// ability to implement `Zeroable`.
+unsafe impl<T> kernel::prelude::Zeroable for __IncompleteArrayField<T> {}
diff --git a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs
index 5bcfbcd1ad22..6d25fe0bffa9 100644
--- a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs
+++ b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs
@@ -320,11 +320,12 @@ fn fmt(&self, fmt: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
 pub const NV_VGPU_MSG_EVENT_NUM_EVENTS: _bindgen_ty_3 = 4131;
 pub type _bindgen_ty_3 = ffi::c_uint;
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct NV0080_CTRL_GPU_GET_SRIOV_CAPS_PARAMS {
     pub totalVFs: u32_,
     pub firstVfOffset: u32_,
     pub vfFeatureMask: u32_,
+    pub __bindgen_padding_0: [u8; 4usize],
     pub FirstVFBar0Address: u64_,
     pub FirstVFBar1Address: u64_,
     pub FirstVFBar2Address: u64_,
@@ -340,23 +341,26 @@ pub struct NV0080_CTRL_GPU_GET_SRIOV_CAPS_PARAMS {
     pub bClientRmAllocatedCtxBuffer: u8_,
     pub bNonPowerOf2ChannelCountSupported: u8_,
     pub bVfResizableBAR1Supported: u8_,
+    pub __bindgen_padding_1: [u8; 7usize],
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct NV2080_CTRL_BIOS_GET_SKU_INFO_PARAMS {
     pub BoardID: u32_,
     pub chipSKU: [ffi::c_char; 9usize],
     pub chipSKUMod: [ffi::c_char; 5usize],
+    pub __bindgen_padding_0: [u8; 2usize],
     pub skuConfigVersion: u32_,
     pub project: [ffi::c_char; 5usize],
     pub projectSKU: [ffi::c_char; 5usize],
     pub CDP: [ffi::c_char; 6usize],
     pub projectSKUMod: [ffi::c_char; 2usize],
+    pub __bindgen_padding_1: [u8; 2usize],
     pub businessCycle: u32_,
 }
 pub type NV2080_CTRL_CMD_FB_GET_FB_REGION_SURFACE_MEM_TYPE_FLAG = [u8_; 17usize];
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO {
     pub base: u64_,
     pub limit: u64_,
@@ -368,13 +372,14 @@ pub struct NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO {
     pub blackList: NV2080_CTRL_CMD_FB_GET_FB_REGION_SURFACE_MEM_TYPE_FLAG,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO_PARAMS {
     pub numFBRegions: u32_,
+    pub __bindgen_padding_0: [u8; 4usize],
     pub fbRegion: [NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO; 16usize],
 }
 #[repr(C)]
-#[derive(Debug, Copy, Clone)]
+#[derive(Debug, Copy, Clone, MaybeZeroable)]
 pub struct NV2080_CTRL_GPU_GET_GID_INFO_PARAMS {
     pub index: u32_,
     pub flags: u32_,
@@ -391,14 +396,14 @@ fn default() -> Self {
     }
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct DOD_METHOD_DATA {
     pub status: u32_,
     pub acpiIdListLen: u32_,
     pub acpiIdList: [u32_; 16usize],
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct JT_METHOD_DATA {
     pub status: u32_,
     pub jtCaps: u32_,
@@ -407,14 +412,14 @@ pub struct JT_METHOD_DATA {
     pub __bindgen_padding_0: u8,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct MUX_METHOD_DATA_ELEMENT {
     pub acpiId: u32_,
     pub mode: u32_,
     pub status: u32_,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct MUX_METHOD_DATA {
     pub tableLen: u32_,
     pub acpiIdMuxModeTable: [MUX_METHOD_DATA_ELEMENT; 16usize],
@@ -422,13 +427,13 @@ pub struct MUX_METHOD_DATA {
     pub acpiIdMuxStateTable: [MUX_METHOD_DATA_ELEMENT; 16usize],
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct CAPS_METHOD_DATA {
     pub status: u32_,
     pub optimusCaps: u32_,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct ACPI_METHOD_DATA {
     pub bValid: u8_,
     pub __bindgen_padding_0: [u8; 3usize],
@@ -438,20 +443,20 @@ pub struct ACPI_METHOD_DATA {
     pub capsMethodData: CAPS_METHOD_DATA,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct VIRTUAL_DISPLAY_GET_MAX_RESOLUTION_PARAMS {
     pub headIndex: u32_,
     pub maxHResolution: u32_,
     pub maxVResolution: u32_,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct VIRTUAL_DISPLAY_GET_NUM_HEADS_PARAMS {
     pub numHeads: u32_,
     pub maxNumHeads: u32_,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct BUSINFO {
     pub deviceID: u16_,
     pub vendorID: u16_,
@@ -461,7 +466,7 @@ pub struct BUSINFO {
     pub __bindgen_padding_0: u8,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct GSP_VF_INFO {
     pub totalVFs: u32_,
     pub firstVFOffset: u32_,
@@ -474,34 +479,37 @@ pub struct GSP_VF_INFO {
     pub __bindgen_padding_0: [u8; 5usize],
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct GSP_PCIE_CONFIG_REG {
     pub linkCap: u32_,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct EcidManufacturingInfo {
     pub ecidLow: u32_,
     pub ecidHigh: u32_,
     pub ecidExtended: u32_,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct FW_WPR_LAYOUT_OFFSET {
     pub nonWprHeapOffset: u64_,
     pub frtsOffset: u64_,
 }
 #[repr(C)]
-#[derive(Debug, Copy, Clone)]
+#[derive(Debug, Copy, Clone, MaybeZeroable)]
 pub struct GspStaticConfigInfo_t {
     pub grCapsBits: [u8_; 23usize],
+    pub __bindgen_padding_0: u8,
     pub gidInfo: NV2080_CTRL_GPU_GET_GID_INFO_PARAMS,
     pub SKUInfo: NV2080_CTRL_BIOS_GET_SKU_INFO_PARAMS,
+    pub __bindgen_padding_1: [u8; 4usize],
     pub fbRegionInfoParams: NV2080_CTRL_CMD_FB_GET_FB_REGION_INFO_PARAMS,
     pub sriovCaps: NV0080_CTRL_GPU_GET_SRIOV_CAPS_PARAMS,
     pub sriovMaxGfid: u32_,
     pub engineCaps: [u32_; 3usize],
     pub poisonFuseEnabled: u8_,
+    pub __bindgen_padding_2: [u8; 7usize],
     pub fb_length: u64_,
     pub fbio_mask: u64_,
     pub fb_bus_width: u32_,
@@ -527,16 +535,20 @@ pub struct GspStaticConfigInfo_t {
     pub bIsMigSupported: u8_,
     pub RTD3GC6TotalBoardPower: u16_,
     pub RTD3GC6PerstDelay: u16_,
+    pub __bindgen_padding_3: [u8; 2usize],
     pub bar1PdeBase: u64_,
     pub bar2PdeBase: u64_,
     pub bVbiosValid: u8_,
+    pub __bindgen_padding_4: [u8; 3usize],
     pub vbiosSubVendor: u32_,
     pub vbiosSubDevice: u32_,
     pub bPageRetirementSupported: u8_,
     pub bSplitVasBetweenServerClientRm: u8_,
     pub bClRootportNeedsNosnoopWAR: u8_,
+    pub __bindgen_padding_5: u8,
     pub displaylessMaxHeads: VIRTUAL_DISPLAY_GET_NUM_HEADS_PARAMS,
     pub displaylessMaxResolution: VIRTUAL_DISPLAY_GET_MAX_RESOLUTION_PARAMS,
+    pub __bindgen_padding_6: [u8; 4usize],
     pub displaylessMaxPixels: u64_,
     pub hInternalClient: u32_,
     pub hInternalDevice: u32_,
@@ -558,7 +570,7 @@ fn default() -> Self {
     }
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct GspSystemInfo {
     pub gpuPhysAddr: u64_,
     pub gpuPhysFbAddr: u64_,
@@ -615,7 +627,7 @@ pub struct GspSystemInfo {
     pub hostPageSize: u64_,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct MESSAGE_QUEUE_INIT_ARGUMENTS {
     pub sharedMemPhysAddr: u64_,
     pub pageTableEntryCount: u32_,
@@ -624,7 +636,7 @@ pub struct MESSAGE_QUEUE_INIT_ARGUMENTS {
     pub statQueueOffset: u64_,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct GSP_SR_INIT_ARGUMENTS {
     pub oldLevel: u32_,
     pub flags: u32_,
@@ -632,7 +644,7 @@ pub struct GSP_SR_INIT_ARGUMENTS {
     pub __bindgen_padding_0: [u8; 3usize],
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct GSP_ARGUMENTS_CACHED {
     pub messageQueueInitArguments: MESSAGE_QUEUE_INIT_ARGUMENTS,
     pub srInitArguments: GSP_SR_INIT_ARGUMENTS,
@@ -642,13 +654,13 @@ pub struct GSP_ARGUMENTS_CACHED {
     pub profilerArgs: GSP_ARGUMENTS_CACHED__bindgen_ty_1,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct GSP_ARGUMENTS_CACHED__bindgen_ty_1 {
     pub pa: u64_,
     pub size: u64_,
 }
 #[repr(C)]
-#[derive(Copy, Clone, Zeroable)]
+#[derive(Copy, Clone, MaybeZeroable)]
 pub union rpc_message_rpc_union_field_v03_00 {
     pub spare: u32_,
     pub cpuRmGfid: u32_,
@@ -664,6 +676,7 @@ fn default() -> Self {
 }
 pub type rpc_message_rpc_union_field_v = rpc_message_rpc_union_field_v03_00;
 #[repr(C)]
+#[derive(MaybeZeroable)]
 pub struct rpc_message_header_v03_00 {
     pub header_version: u32_,
     pub signature: u32_,
@@ -686,7 +699,7 @@ fn default() -> Self {
 }
 pub type rpc_message_header_v = rpc_message_header_v03_00;
 #[repr(C)]
-#[derive(Copy, Clone, Zeroable)]
+#[derive(Copy, Clone, MaybeZeroable)]
 pub struct GspFwWprMeta {
     pub magic: u64_,
     pub revision: u64_,
@@ -721,19 +734,19 @@ pub struct GspFwWprMeta {
     pub verified: u64_,
 }
 #[repr(C)]
-#[derive(Copy, Clone, Zeroable)]
+#[derive(Copy, Clone, MaybeZeroable)]
 pub union GspFwWprMeta__bindgen_ty_1 {
     pub __bindgen_anon_1: GspFwWprMeta__bindgen_ty_1__bindgen_ty_1,
     pub __bindgen_anon_2: GspFwWprMeta__bindgen_ty_1__bindgen_ty_2,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct GspFwWprMeta__bindgen_ty_1__bindgen_ty_1 {
     pub sysmemAddrOfSignature: u64_,
     pub sizeOfSignature: u64_,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct GspFwWprMeta__bindgen_ty_1__bindgen_ty_2 {
     pub gspFwHeapFreeListWprOffset: u32_,
     pub unused0: u32_,
@@ -749,13 +762,13 @@ fn default() -> Self {
     }
 }
 #[repr(C)]
-#[derive(Copy, Clone, Zeroable)]
+#[derive(Copy, Clone, MaybeZeroable)]
 pub union GspFwWprMeta__bindgen_ty_2 {
     pub __bindgen_anon_1: GspFwWprMeta__bindgen_ty_2__bindgen_ty_1,
     pub __bindgen_anon_2: GspFwWprMeta__bindgen_ty_2__bindgen_ty_2,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct GspFwWprMeta__bindgen_ty_2__bindgen_ty_1 {
     pub partitionRpcAddr: u64_,
     pub partitionRpcRequestOffset: u16_,
@@ -767,7 +780,7 @@ pub struct GspFwWprMeta__bindgen_ty_2__bindgen_ty_1 {
     pub lsUcodeVersion: u32_,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct GspFwWprMeta__bindgen_ty_2__bindgen_ty_2 {
     pub partitionRpcPadding: [u32_; 4usize],
     pub sysmemAddrOfCrashReportQueue: u64_,
@@ -802,7 +815,7 @@ fn default() -> Self {
 pub const LibosMemoryRegionLoc_LIBOS_MEMORY_REGION_LOC_FB: LibosMemoryRegionLoc = 2;
 pub type LibosMemoryRegionLoc = ffi::c_uint;
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct LibosMemoryRegionInitArgument {
     pub id8: LibosAddress,
     pub pa: LibosAddress,
@@ -812,7 +825,7 @@ pub struct LibosMemoryRegionInitArgument {
     pub __bindgen_padding_0: [u8; 6usize],
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct PACKED_REGISTRY_ENTRY {
     pub nameOffset: u32_,
     pub type_: u8_,
@@ -821,14 +834,14 @@ pub struct PACKED_REGISTRY_ENTRY {
     pub length: u32_,
 }
 #[repr(C)]
-#[derive(Debug, Default)]
+#[derive(Debug, Default, MaybeZeroable)]
 pub struct PACKED_REGISTRY_TABLE {
     pub size: u32_,
     pub numEntries: u32_,
     pub entries: __IncompleteArrayField<PACKED_REGISTRY_ENTRY>,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct msgqTxHeader {
     pub version: u32_,
     pub size: u32_,
@@ -840,13 +853,13 @@ pub struct msgqTxHeader {
     pub entryOff: u32_,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone, Zeroable)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct msgqRxHeader {
     pub readPtr: u32_,
 }
 #[repr(C)]
 #[repr(align(8))]
-#[derive(Zeroable)]
+#[derive(MaybeZeroable)]
 pub struct GSP_MSG_QUEUE_ELEMENT {
     pub authTagBuffer: [u8_; 16usize],
     pub aadBuffer: [u8_; 16usize],
@@ -866,7 +879,7 @@ fn default() -> Self {
     }
 }
 #[repr(C)]
-#[derive(Debug, Default)]
+#[derive(Debug, Default, MaybeZeroable)]
 pub struct rpc_run_cpu_sequencer_v17_00 {
     pub bufferSizeDWord: u32_,
     pub cmdIndex: u32_,
@@ -884,20 +897,20 @@ pub struct rpc_run_cpu_sequencer_v17_00 {
 pub const GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME: GSP_SEQ_BUF_OPCODE = 8;
 pub type GSP_SEQ_BUF_OPCODE = ffi::c_uint;
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct GSP_SEQ_BUF_PAYLOAD_REG_WRITE {
     pub addr: u32_,
     pub val: u32_,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct GSP_SEQ_BUF_PAYLOAD_REG_MODIFY {
     pub addr: u32_,
     pub mask: u32_,
     pub val: u32_,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct GSP_SEQ_BUF_PAYLOAD_REG_POLL {
     pub addr: u32_,
     pub mask: u32_,
@@ -906,24 +919,24 @@ pub struct GSP_SEQ_BUF_PAYLOAD_REG_POLL {
     pub error: u32_,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct GSP_SEQ_BUF_PAYLOAD_DELAY_US {
     pub val: u32_,
 }
 #[repr(C)]
-#[derive(Debug, Default, Copy, Clone)]
+#[derive(Debug, Default, Copy, Clone, MaybeZeroable)]
 pub struct GSP_SEQ_BUF_PAYLOAD_REG_STORE {
     pub addr: u32_,
     pub index: u32_,
 }
 #[repr(C)]
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, MaybeZeroable)]
 pub struct GSP_SEQUENCER_BUFFER_CMD {
     pub opCode: GSP_SEQ_BUF_OPCODE,
     pub payload: GSP_SEQUENCER_BUFFER_CMD__bindgen_ty_1,
 }
 #[repr(C)]
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, MaybeZeroable)]
 pub union GSP_SEQUENCER_BUFFER_CMD__bindgen_ty_1 {
     pub regWrite: GSP_SEQ_BUF_PAYLOAD_REG_WRITE,
     pub regModify: GSP_SEQ_BUF_PAYLOAD_REG_MODIFY,
diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs
index 2d0369c49092..d78a30fbb70f 100644
--- a/drivers/gpu/nova-core/gsp/sequencer.rs
+++ b/drivers/gpu/nova-core/gsp/sequencer.rs
@@ -14,12 +14,12 @@
     device,
     io::poll::read_poll_timeout,
     prelude::*,
+    sync::aref::ARef,
     time::{
         delay::fsleep,
         Delta, //
     },
-    transmute::FromBytes,
-    types::ARef, //
+    transmute::FromBytes, //
 };
 
 use crate::{
diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs
index abf423560ff4..7c26e4a2d61c 100644
--- a/drivers/gpu/nova-core/vbios.rs
+++ b/drivers/gpu/nova-core/vbios.rs
@@ -11,8 +11,8 @@
         Alignable,
         Alignment, //
     },
+    sync::aref::ARef,
     transmute::FromBytes,
-    types::ARef,
 };
 
 use crate::{
diff --git a/drivers/hid/hid-picolcd_debugfs.c b/drivers/hid/hid-picolcd_debugfs.c
index d01176da8896..085847a92e07 100644
--- a/drivers/hid/hid-picolcd_debugfs.c
+++ b/drivers/hid/hid-picolcd_debugfs.c
@@ -11,6 +11,7 @@
 #include <linux/hid-debug.h>
 
 #include <linux/fb.h>
+#include <linux/hex.h>
 #include <linux/seq_file.h>
 #include <linux/debugfs.h>
 
diff --git a/drivers/hv/mshv_common.c b/drivers/hv/mshv_common.c
index 58027b23c206..63f09bb5136e 100644
--- a/drivers/hv/mshv_common.c
+++ b/drivers/hv/mshv_common.c
@@ -142,6 +142,7 @@ int hv_call_get_partition_property(u64 partition_id,
 }
 EXPORT_SYMBOL_GPL(hv_call_get_partition_property);
 
+#ifdef CONFIG_X86
 /*
  * Corresponding sleep states have to be initialized in order for a subsequent
  * HVCALL_ENTER_SLEEP_STATE call to succeed. Currently only S5 state as per
@@ -237,3 +238,4 @@ void hv_machine_power_off(void)
 	BUG();
 
 }
+#endif
diff --git a/drivers/hv/mshv_regions.c b/drivers/hv/mshv_regions.c
index 202b9d551e39..30bacba6aec3 100644
--- a/drivers/hv/mshv_regions.c
+++ b/drivers/hv/mshv_regions.c
@@ -58,7 +58,7 @@ static long mshv_region_process_chunk(struct mshv_mem_region *region,
 
 	page_order = folio_order(page_folio(page));
 	/* The hypervisor only supports 4K and 2M page sizes */
-	if (page_order && page_order != HPAGE_PMD_ORDER)
+	if (page_order && page_order != PMD_ORDER)
 		return -EINVAL;
 
 	stride = 1 << page_order;
@@ -494,13 +494,6 @@ static bool mshv_region_interval_invalidate(struct mmu_interval_notifier *mni,
 	unsigned long mstart, mend;
 	int ret = -EPERM;
 
-	if (mmu_notifier_range_blockable(range))
-		mutex_lock(&region->mutex);
-	else if (!mutex_trylock(&region->mutex))
-		goto out_fail;
-
-	mmu_interval_set_seq(mni, cur_seq);
-
 	mstart = max(range->start, region->start_uaddr);
 	mend = min(range->end, region->start_uaddr +
 		   (region->nr_pages << HV_HYP_PAGE_SHIFT));
@@ -508,10 +501,17 @@ static bool mshv_region_interval_invalidate(struct mmu_interval_notifier *mni,
 	page_offset = HVPFN_DOWN(mstart - region->start_uaddr);
 	page_count = HVPFN_DOWN(mend - mstart);
 
+	if (mmu_notifier_range_blockable(range))
+		mutex_lock(&region->mutex);
+	else if (!mutex_trylock(&region->mutex))
+		goto out_fail;
+
+	mmu_interval_set_seq(mni, cur_seq);
+
 	ret = mshv_region_remap_pages(region, HV_MAP_GPA_NO_ACCESS,
 				      page_offset, page_count);
 	if (ret)
-		goto out_fail;
+		goto out_unlock;
 
 	mshv_region_invalidate_pages(region, page_offset, page_count);
 
@@ -519,6 +519,8 @@ static bool mshv_region_interval_invalidate(struct mmu_interval_notifier *mni,
 
 	return true;
 
+out_unlock:
+	mutex_unlock(&region->mutex);
 out_fail:
 	WARN_ONCE(ret,
 		  "Failed to invalidate region %#llx-%#llx (range %#lx-%#lx, event: %u, pages %#llx-%#llx, mm: %#llx): %d\n",
diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c
index a53af6fe81a6..9c937190be81 100644
--- a/drivers/hv/vmbus_drv.c
+++ b/drivers/hv/vmbus_drv.c
@@ -29,7 +29,7 @@
 #include <linux/delay.h>
 #include <linux/panic_notifier.h>
 #include <linux/ptrace.h>
-#include <linux/screen_info.h>
+#include <linux/sysfb.h>
 #include <linux/efi.h>
 #include <linux/random.h>
 #include <linux/kernel.h>
@@ -2340,8 +2340,8 @@ static void __maybe_unused vmbus_reserve_fb(void)
 	if (efi_enabled(EFI_BOOT)) {
 		/* Gen2 VM: get FB base from EFI framebuffer */
 		if (IS_ENABLED(CONFIG_SYSFB)) {
-			start = screen_info.lfb_base;
-			size = max_t(__u32, screen_info.lfb_size, 0x800000);
+			start = sysfb_primary_display.screen.lfb_base;
+			size = max_t(__u32, sysfb_primary_display.screen.lfb_size, 0x800000);
 		}
 	} else {
 		/* Gen1 VM: get FB base from PCI */
diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
index 157678b821fc..d9bac1e3057b 100644
--- a/drivers/hwmon/Kconfig
+++ b/drivers/hwmon/Kconfig
@@ -1983,8 +1983,8 @@ config SENSORS_SHT3x
 	depends on I2C
 	select CRC8
 	help
-	  If you say yes here you get support for the Sensiron SHT30 and SHT31
-	  humidity and temperature sensors.
+	  If you say yes here you get support for the Sensiron SHT30, SHT31 and
+	  SHT85 humidity and temperature sensors.
 
 	  This driver can also be built as a module. If so, the module
 	  will be called sht3x.
diff --git a/drivers/hwmon/asus-ec-sensors.c b/drivers/hwmon/asus-ec-sensors.c
index 61b18b88ee8f..8435be30c3f7 100644
--- a/drivers/hwmon/asus-ec-sensors.c
+++ b/drivers/hwmon/asus-ec-sensors.c
@@ -474,6 +474,14 @@ static const struct ec_board_info board_info_maximus_vi_hero = {
 	.family = family_intel_300_series,
 };
 
+static const struct ec_board_info board_info_maximus_x_hero = {
+	.sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB |
+		SENSOR_TEMP_T_SENSOR |
+		SENSOR_TEMP_VRM | SENSOR_FAN_CPU_OPT,
+	.mutex_path = ASUS_HW_ACCESS_MUTEX_SB_PCI0_LPCB_SIO1_MUT0,
+	.family = family_intel_300_series,
+};
+
 static const struct ec_board_info board_info_maximus_xi_hero = {
 	.sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB |
 		SENSOR_TEMP_T_SENSOR |
@@ -793,6 +801,8 @@ static const struct dmi_system_id dmi_table[] = {
 					&board_info_pro_art_x870E_creator_wifi),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("Pro WS TRX50-SAGE WIFI",
 					&board_info_pro_ws_trx50_sage_wifi),
+	DMI_EXACT_MATCH_ASUS_BOARD_NAME("Pro WS TRX50-SAGE WIFI A",
+					&board_info_pro_ws_trx50_sage_wifi),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("Pro WS WRX90E-SAGE SE",
 					&board_info_pro_ws_wrx90e_sage_se),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("Pro WS X570-ACE",
@@ -815,6 +825,8 @@ static const struct dmi_system_id dmi_table[] = {
 					&board_info_maximus_xi_hero),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG MAXIMUS XI HERO (WI-FI)",
 					&board_info_maximus_xi_hero),
+	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG MAXIMUS X HERO",
+					&board_info_maximus_x_hero),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG MAXIMUS Z690 FORMULA",
 					&board_info_maximus_z690_formula),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX B550-E GAMING",
diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c
index a34753fc2973..6040a8940674 100644
--- a/drivers/hwmon/dell-smm-hwmon.c
+++ b/drivers/hwmon/dell-smm-hwmon.c
@@ -76,6 +76,9 @@
 #define DELL_SMM_NO_TEMP	10
 #define DELL_SMM_NO_FANS	4
 
+/* limit fan multiplier to avoid overflow */
+#define DELL_SMM_MAX_FAN_MULT (INT_MAX / U16_MAX)
+
 struct smm_regs {
 	unsigned int eax;
 	unsigned int ebx;
@@ -1253,6 +1256,12 @@ static int dell_smm_init_data(struct device *dev, const struct dell_smm_ops *ops
 	data->ops = ops;
 	/* All options must not be 0 */
 	data->i8k_fan_mult = fan_mult ? : I8K_FAN_MULT;
+	if (data->i8k_fan_mult > DELL_SMM_MAX_FAN_MULT) {
+		dev_err(dev,
+			"fan multiplier %u is too large (max %u)\n",
+			data->i8k_fan_mult, DELL_SMM_MAX_FAN_MULT);
+		return -EINVAL;
+	}
 	data->i8k_fan_max = fan_max ? : I8K_FAN_HIGH;
 	data->i8k_pwm_mult = DIV_ROUND_UP(255, data->i8k_fan_max);
 
diff --git a/drivers/hwmon/fam15h_power.c b/drivers/hwmon/fam15h_power.c
index 8ecebea53651..efcbea2d070e 100644
--- a/drivers/hwmon/fam15h_power.c
+++ b/drivers/hwmon/fam15h_power.c
@@ -372,15 +372,14 @@ static void tweak_runavg_range(struct pci_dev *pdev)
 		REG_TDP_RUNNING_AVERAGE, val);
 }
 
-#ifdef CONFIG_PM
-static int fam15h_power_resume(struct pci_dev *pdev)
+static int fam15h_power_resume(struct device *dev)
 {
+	struct pci_dev *pdev = to_pci_dev(dev);
 	tweak_runavg_range(pdev);
 	return 0;
 }
-#else
-#define fam15h_power_resume NULL
-#endif
+
+static DEFINE_SIMPLE_DEV_PM_OPS(fam15h_power_ops, NULL, fam15h_power_resume);
 
 static int fam15h_power_init_data(struct pci_dev *f4,
 				  struct fam15h_power_data *data)
@@ -493,7 +492,7 @@ static struct pci_driver fam15h_power_driver = {
 	.name = "fam15h_power",
 	.id_table = fam15h_power_id_table,
 	.probe = fam15h_power_probe,
-	.resume = fam15h_power_resume,
+	.driver.pm = &fam15h_power_ops,
 };
 
 module_pci_driver(fam15h_power_driver);
diff --git a/drivers/hwmon/ibmpex.c b/drivers/hwmon/ibmpex.c
index 228c5f6c6f38..129f3a9e8fe9 100644
--- a/drivers/hwmon/ibmpex.c
+++ b/drivers/hwmon/ibmpex.c
@@ -277,6 +277,9 @@ static ssize_t ibmpex_high_low_store(struct device *dev,
 {
 	struct ibmpex_bmc_data *data = dev_get_drvdata(dev);
 
+	if (!data)
+		return -ENODEV;
+
 	ibmpex_reset_high_low_data(data);
 
 	return count;
@@ -508,6 +511,9 @@ static void ibmpex_bmc_delete(struct ibmpex_bmc_data *data)
 {
 	int i, j;
 
+	hwmon_device_unregister(data->hwmon_dev);
+	dev_set_drvdata(data->bmc_device, NULL);
+
 	device_remove_file(data->bmc_device,
 			   &sensor_dev_attr_reset_high_low.dev_attr);
 	device_remove_file(data->bmc_device, &dev_attr_name.attr);
@@ -521,8 +527,7 @@ static void ibmpex_bmc_delete(struct ibmpex_bmc_data *data)
 		}
 
 	list_del(&data->list);
-	dev_set_drvdata(data->bmc_device, NULL);
-	hwmon_device_unregister(data->hwmon_dev);
+
 	ipmi_destroy_user(data->user);
 	kfree(data->sensors);
 	kfree(data);
diff --git a/drivers/hwmon/pmbus/q54sj108a2.c b/drivers/hwmon/pmbus/q54sj108a2.c
index 4d7086d83aa3..fc030ca34480 100644
--- a/drivers/hwmon/pmbus/q54sj108a2.c
+++ b/drivers/hwmon/pmbus/q54sj108a2.c
@@ -7,6 +7,7 @@
  */
 
 #include <linux/debugfs.h>
+#include <linux/hex.h>
 #include <linux/i2c.h>
 #include <linux/kstrtox.h>
 #include <linux/module.h>
diff --git a/drivers/hwmon/pmbus/ucd9000.c b/drivers/hwmon/pmbus/ucd9000.c
index 55e7af3a5f98..9b5d34a110ba 100644
--- a/drivers/hwmon/pmbus/ucd9000.c
+++ b/drivers/hwmon/pmbus/ucd9000.c
@@ -8,6 +8,7 @@
 
 #include <linux/debugfs.h>
 #include <linux/delay.h>
+#include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/of.h>
diff --git a/drivers/hwmon/sht3x.c b/drivers/hwmon/sht3x.c
index f36c0229328f..08306ccb6d0b 100644
--- a/drivers/hwmon/sht3x.c
+++ b/drivers/hwmon/sht3x.c
@@ -933,6 +933,7 @@ static int sht3x_probe(struct i2c_client *client)
 static const struct i2c_device_id sht3x_ids[] = {
 	{"sht3x", sht3x},
 	{"sts3x", sts3x},
+	{"sht85", sht3x},
 	{}
 };
 
diff --git a/drivers/hwmon/tmp401.c b/drivers/hwmon/tmp401.c
index fbaa34973694..07f596581c6e 100644
--- a/drivers/hwmon/tmp401.c
+++ b/drivers/hwmon/tmp401.c
@@ -397,7 +397,7 @@ static int tmp401_chip_read(struct device *dev, u32 attr, int channel, long *val
 		ret = regmap_read(data->regmap, TMP401_CONVERSION_RATE, &regval);
 		if (ret < 0)
 			return ret;
-		*val = (1 << (7 - regval)) * 125;
+		*val = (1 << (7 - min(regval, 7))) * 125;
 		break;
 	case hwmon_chip_temp_reset_history:
 		*val = 0;
diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
index cea87fcb4a1a..09ba55bae1fa 100644
--- a/drivers/i2c/busses/Kconfig
+++ b/drivers/i2c/busses/Kconfig
@@ -167,6 +167,7 @@ config I2C_I801
 	    Panther Lake (SOC)
 	    Wildcat Lake (SOC)
 	    Diamond Rapids (SOC)
+	    Nova Lake (PCH)
 
 	  This driver can also be built as a module.  If so, the module
 	  will be called i2c-i801.
diff --git a/drivers/i2c/busses/i2c-bcm-iproc.c b/drivers/i2c/busses/i2c-bcm-iproc.c
index e418a4f23f15..b5629cffe99b 100644
--- a/drivers/i2c/busses/i2c-bcm-iproc.c
+++ b/drivers/i2c/busses/i2c-bcm-iproc.c
@@ -1098,8 +1098,7 @@ static int bcm_iproc_i2c_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, iproc_i2c);
 	iproc_i2c->device = &pdev->dev;
-	iproc_i2c->type =
-		(enum bcm_iproc_i2c_type)of_device_get_match_data(&pdev->dev);
+	iproc_i2c->type = (kernel_ulong_t)of_device_get_match_data(&pdev->dev);
 	init_completion(&iproc_i2c->done);
 
 	iproc_i2c->base = devm_platform_ioremap_resource(pdev, 0);
diff --git a/drivers/i2c/busses/i2c-designware-master.c b/drivers/i2c/busses/i2c-designware-master.c
index 45bfca05bb30..f247cf323207 100644
--- a/drivers/i2c/busses/i2c-designware-master.c
+++ b/drivers/i2c/busses/i2c-designware-master.c
@@ -361,6 +361,10 @@ static int amd_i2c_dw_xfer_quirk(struct i2c_adapter *adap, struct i2c_msg *msgs,
 	u8 *tx_buf;
 	unsigned int val;
 
+	ACQUIRE(pm_runtime_active_auto_try, pm)(dev->dev);
+	if (ACQUIRE_ERR(pm_runtime_active_auto_try, &pm))
+		return -ENXIO;
+
 	/*
 	 * In order to enable the interrupt for UCSI i.e. AMD NAVI GPU card,
 	 * it is mandatory to set the right value in specific register
@@ -593,11 +597,12 @@ i2c_dw_read(struct dw_i2c_dev *dev)
 	unsigned int rx_valid;
 
 	for (; dev->msg_read_idx < dev->msgs_num; dev->msg_read_idx++) {
+		u32 flags = msgs[dev->msg_read_idx].flags;
 		unsigned int tmp;
 		u32 len;
 		u8 *buf;
 
-		if (!(msgs[dev->msg_read_idx].flags & I2C_M_RD))
+		if (!(flags & I2C_M_RD))
 			continue;
 
 		if (!(dev->status & STATUS_READ_IN_PROGRESS)) {
@@ -611,8 +616,6 @@ i2c_dw_read(struct dw_i2c_dev *dev)
 		regmap_read(dev->map, DW_IC_RXFLR, &rx_valid);
 
 		for (; len > 0 && rx_valid > 0; len--, rx_valid--) {
-			u32 flags = msgs[dev->msg_read_idx].flags;
-
 			regmap_read(dev->map, DW_IC_DATA_CMD, &tmp);
 			tmp &= DW_IC_DATA_CMD_DAT;
 			/* Ensure length byte is a valid value */
@@ -821,14 +824,6 @@ i2c_dw_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
 
 	pm_runtime_get_sync(dev->dev);
 
-	switch (dev->flags & MODEL_MASK) {
-	case MODEL_AMD_NAVI_GPU:
-		ret = amd_i2c_dw_xfer_quirk(adap, msgs, num);
-		goto done_nolock;
-	default:
-		break;
-	}
-
 	reinit_completion(&dev->cmd_complete);
 	dev->msgs = msgs;
 	dev->msgs_num = num;
@@ -918,6 +913,11 @@ static const struct i2c_algorithm i2c_dw_algo = {
 	.functionality = i2c_dw_func,
 };
 
+static const struct i2c_algorithm amd_i2c_dw_algo = {
+	.xfer = amd_i2c_dw_xfer_quirk,
+	.functionality = i2c_dw_func,
+};
+
 static const struct i2c_adapter_quirks i2c_dw_quirks = {
 	.flags = I2C_AQ_NO_ZERO_LEN,
 };
@@ -1053,7 +1053,10 @@ int i2c_dw_probe_master(struct dw_i2c_dev *dev)
 		scnprintf(adap->name, sizeof(adap->name),
 			  "Synopsys DesignWare I2C adapter");
 	adap->retries = 3;
-	adap->algo = &i2c_dw_algo;
+	if ((dev->flags & MODEL_MASK) == MODEL_AMD_NAVI_GPU)
+		adap->algo = &amd_i2c_dw_algo;
+	else
+		adap->algo = &i2c_dw_algo;
 	adap->quirks = &i2c_dw_quirks;
 	adap->dev.parent = dev->dev;
 	i2c_set_adapdata(adap, dev);
diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c
index 7be99656a67d..077b34535ec7 100644
--- a/drivers/i2c/busses/i2c-designware-platdrv.c
+++ b/drivers/i2c/busses/i2c-designware-platdrv.c
@@ -334,9 +334,9 @@ static void dw_i2c_plat_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id dw_i2c_of_match[] = {
-	{ .compatible = "snps,designware-i2c", },
-	{ .compatible = "mscc,ocelot-i2c", .data = (void *)MODEL_MSCC_OCELOT },
 	{ .compatible = "baikal,bt1-sys-i2c", .data = (void *)MODEL_BAIKAL_BT1 },
+	{ .compatible = "mscc,ocelot-i2c", .data = (void *)MODEL_MSCC_OCELOT },
+	{ .compatible = "snps,designware-i2c" },
 	{}
 };
 MODULE_DEVICE_TABLE(of, dw_i2c_of_match);
diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c
index 81e6e2d7ad3d..9e1789725edf 100644
--- a/drivers/i2c/busses/i2c-i801.c
+++ b/drivers/i2c/busses/i2c-i801.c
@@ -85,6 +85,7 @@
  * Panther Lake-P (SOC)		0xe422	32	hard	yes	yes	yes
  * Wildcat Lake-U (SOC)		0x4d22	32	hard	yes	yes	yes
  * Diamond Rapids (SOC)		0x5827	32	hard	yes	yes	yes
+ * Nova Lake-S (PCH)		0x6e23	32	hard	yes	yes	yes
  *
  * Features supported by this driver:
  * Software PEC				no
@@ -245,6 +246,7 @@
 #define PCI_DEVICE_ID_INTEL_BIRCH_STREAM_SMBUS		0x5796
 #define PCI_DEVICE_ID_INTEL_DIAMOND_RAPIDS_SMBUS	0x5827
 #define PCI_DEVICE_ID_INTEL_BROXTON_SMBUS		0x5ad4
+#define PCI_DEVICE_ID_INTEL_NOVA_LAKE_S_SMBUS		0x6e23
 #define PCI_DEVICE_ID_INTEL_ARROW_LAKE_H_SMBUS		0x7722
 #define PCI_DEVICE_ID_INTEL_RAPTOR_LAKE_S_SMBUS		0x7a23
 #define PCI_DEVICE_ID_INTEL_ALDER_LAKE_S_SMBUS		0x7aa3
@@ -1061,6 +1063,7 @@ static const struct pci_device_id i801_ids[] = {
 	{ PCI_DEVICE_DATA(INTEL, PANTHER_LAKE_H_SMBUS,		FEATURES_ICH5 | FEATURE_TCO_CNL) },
 	{ PCI_DEVICE_DATA(INTEL, PANTHER_LAKE_P_SMBUS,		FEATURES_ICH5 | FEATURE_TCO_CNL) },
 	{ PCI_DEVICE_DATA(INTEL, WILDCAT_LAKE_U_SMBUS,		FEATURES_ICH5 | FEATURE_TCO_CNL) },
+	{ PCI_DEVICE_DATA(INTEL, NOVA_LAKE_S_SMBUS,		FEATURES_ICH5 | FEATURE_TCO_CNL) },
 	{ 0, }
 };
 
diff --git a/drivers/i2c/busses/i2c-pxa.c b/drivers/i2c/busses/i2c-pxa.c
index 968a8b8794da..09af3b3625f1 100644
--- a/drivers/i2c/busses/i2c-pxa.c
+++ b/drivers/i2c/busses/i2c-pxa.c
@@ -1266,7 +1266,7 @@ static int i2c_pxa_probe_dt(struct platform_device *pdev, struct pxa_i2c *i2c,
 	i2c->use_pio = of_property_read_bool(np, "mrvl,i2c-polling");
 	i2c->fast_mode = of_property_read_bool(np, "mrvl,i2c-fast-mode");
 
-	*i2c_types = (enum pxa_i2c_types)device_get_match_data(&pdev->dev);
+	*i2c_types = (kernel_ulong_t)device_get_match_data(&pdev->dev);
 
 	return 0;
 }
diff --git a/drivers/i2c/busses/i2c-rcar.c b/drivers/i2c/busses/i2c-rcar.c
index d51884ab99f4..5ce8f8e4856f 100644
--- a/drivers/i2c/busses/i2c-rcar.c
+++ b/drivers/i2c/busses/i2c-rcar.c
@@ -1141,7 +1141,7 @@ static int rcar_i2c_probe(struct platform_device *pdev)
 	if (IS_ERR(priv->io))
 		return PTR_ERR(priv->io);
 
-	priv->devtype = (enum rcar_i2c_type)of_device_get_match_data(dev);
+	priv->devtype = (kernel_ulong_t)of_device_get_match_data(dev);
 	init_waitqueue_head(&priv->wait);
 
 	adap = &priv->adap;
diff --git a/drivers/iio/adc/at91-sama5d2_adc.c b/drivers/iio/adc/at91-sama5d2_adc.c
index b4c36e6a7490..aa4ba3f5a506 100644
--- a/drivers/iio/adc/at91-sama5d2_adc.c
+++ b/drivers/iio/adc/at91-sama5d2_adc.c
@@ -2481,6 +2481,7 @@ static void at91_adc_remove(struct platform_device *pdev)
 	struct at91_adc_state *st = iio_priv(indio_dev);
 
 	iio_device_unregister(indio_dev);
+	cancel_work_sync(&st->touch_st.workq);
 
 	at91_adc_dma_disable(st);
 
diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c
index 61596cda2b65..35ba852a172a 100644
--- a/drivers/infiniband/core/addr.c
+++ b/drivers/infiniband/core/addr.c
@@ -80,37 +80,25 @@ static const struct nla_policy ib_nl_addr_policy[LS_NLA_TYPE_MAX] = {
 		.min = sizeof(struct rdma_nla_ls_gid)},
 };
 
-static inline bool ib_nl_is_good_ip_resp(const struct nlmsghdr *nlh)
+static void ib_nl_process_ip_rsep(const struct nlmsghdr *nlh)
 {
 	struct nlattr *tb[LS_NLA_TYPE_MAX] = {};
+	union ib_gid gid;
+	struct addr_req *req;
+	int found = 0;
 	int ret;
 
 	if (nlh->nlmsg_flags & RDMA_NL_LS_F_ERR)
-		return false;
+		return;
 
 	ret = nla_parse_deprecated(tb, LS_NLA_TYPE_MAX - 1, nlmsg_data(nlh),
 				   nlmsg_len(nlh), ib_nl_addr_policy, NULL);
 	if (ret)
-		return false;
-
-	return true;
-}
-
-static void ib_nl_process_good_ip_rsep(const struct nlmsghdr *nlh)
-{
-	const struct nlattr *head, *curr;
-	union ib_gid gid;
-	struct addr_req *req;
-	int len, rem;
-	int found = 0;
-
-	head = (const struct nlattr *)nlmsg_data(nlh);
-	len = nlmsg_len(nlh);
+		return;
 
-	nla_for_each_attr(curr, head, len, rem) {
-		if (curr->nla_type == LS_NLA_TYPE_DGID)
-			memcpy(&gid, nla_data(curr), nla_len(curr));
-	}
+	if (!tb[LS_NLA_TYPE_DGID])
+		return;
+	memcpy(&gid, nla_data(tb[LS_NLA_TYPE_DGID]), sizeof(gid));
 
 	spin_lock_bh(&lock);
 	list_for_each_entry(req, &req_list, list) {
@@ -137,8 +125,7 @@ int ib_nl_handle_ip_res_resp(struct sk_buff *skb,
 	    !(NETLINK_CB(skb).sk))
 		return -EPERM;
 
-	if (ib_nl_is_good_ip_resp(nlh))
-		ib_nl_process_good_ip_rsep(nlh);
+	ib_nl_process_ip_rsep(nlh);
 
 	return 0;
 }
diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c
index 81cf3c902e81..a91cb61d0d80 100644
--- a/drivers/infiniband/core/cache.c
+++ b/drivers/infiniband/core/cache.c
@@ -121,6 +121,104 @@ struct ib_gid_table {
 	u32				default_gid_indices;
 };
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+#define IB_GID_TABLE_ENTRY_TRACE_BUFFER_SIZE 1024
+static struct ib_gid_table_entry_trace_buffer {
+	struct ib_gid_table_entry *entry; // no-ref
+	struct net_device *ndev; // no-ref
+	bool registered;
+	atomic_t count;
+	int nr_entries;
+	unsigned long entries[20];
+} ib_gid_table_entry_trace_buffer[IB_GID_TABLE_ENTRY_TRACE_BUFFER_SIZE];
+static bool ib_gid_table_entry_trace_buffer_exhausted;
+
+void dump_ib_gid_table_entry_trace_buffer(const struct net_device *ndev)
+{
+	struct ib_gid_table_entry_trace_buffer *ptr;
+	int count, balance = 0;
+	int i;
+
+	for (i = 0; i < IB_GID_TABLE_ENTRY_TRACE_BUFFER_SIZE; i++) {
+		ptr = &ib_gid_table_entry_trace_buffer[i];
+		if (!ptr->entry || ptr->ndev != ndev)
+			continue;
+		count = atomic_read(&ptr->count);
+		balance += count;
+		pr_info("Call trace for %s@%p %+d %sat\n", ndev->name, ptr->entry, count,
+			ptr->registered ? "" : "not_registered");
+		stack_trace_print(ptr->entries, ptr->nr_entries, 4);
+	}
+	if (!ib_gid_table_entry_trace_buffer_exhausted)
+		pr_info("balance for %s@ib_gid_table_entry is %d\n", ndev->name, balance);
+	else
+		pr_info("balance for %s@ib_gid_table_entry is unknown\n", ndev->name);
+}
+
+static void erase_ib_gid_table_entry_trace_buffer(struct ib_gid_table_entry *entry)
+{
+	int i, balance = 0;
+
+	for (i = 0; i < IB_GID_TABLE_ENTRY_TRACE_BUFFER_SIZE; i++)
+		if (ib_gid_table_entry_trace_buffer[i].entry == entry)
+			balance += atomic_read(&ib_gid_table_entry_trace_buffer[i].count);
+	if (balance)
+		return;
+	for (i = 0; i < IB_GID_TABLE_ENTRY_TRACE_BUFFER_SIZE; i++)
+		if (ib_gid_table_entry_trace_buffer[i].entry == entry)
+			ib_gid_table_entry_trace_buffer[i].entry = NULL;
+}
+
+static void save_ib_gid_table_entry_trace_buffer(struct ib_gid_table_entry *entry,
+						 struct net_device *ndev, int delta)
+{
+	struct ib_gid_table_entry_trace_buffer *ptr;
+	unsigned long entries[ARRAY_SIZE(ptr->entries)];
+	unsigned long nr_entries;
+	int i;
+	bool registered;
+
+	if (!delta) {
+		for (i = 0; i < IB_GID_TABLE_ENTRY_TRACE_BUFFER_SIZE; i++)
+			if (ib_gid_table_entry_trace_buffer[i].entry == entry)
+				ib_gid_table_entry_trace_buffer[i].entry = NULL;
+		delta = 1;
+	}
+	if (!ndev)
+		return;
+	if (in_nmi())
+		return;
+	registered = ndev->reg_state == NETREG_REGISTERED;
+	nr_entries = stack_trace_save(entries, ARRAY_SIZE(ptr->entries), 1);
+	nr_entries = trim_netdev_trace(entries, nr_entries);
+	for (i = 0; i < IB_GID_TABLE_ENTRY_TRACE_BUFFER_SIZE; i++) {
+		ptr = &ib_gid_table_entry_trace_buffer[i];
+		if (ptr->entry == entry && ptr->nr_entries == nr_entries &&
+		    ptr->registered == registered &&
+		    !memcmp(ptr->entries, entries, nr_entries * sizeof(unsigned long))) {
+			atomic_add(delta, &ptr->count);
+			return;
+		}
+	}
+	for (i = 0; i < IB_GID_TABLE_ENTRY_TRACE_BUFFER_SIZE; i++) {
+		ptr = &ib_gid_table_entry_trace_buffer[i];
+		if (!ptr->entry && !cmpxchg(&ptr->entry, NULL, entry)) {
+			ptr->ndev = ndev;
+			ptr->registered = registered;
+			atomic_set(&ptr->count, delta);
+			ptr->nr_entries = nr_entries;
+			memmove(ptr->entries, entries, nr_entries * sizeof(unsigned long));
+			return;
+		}
+	}
+	ib_gid_table_entry_trace_buffer_exhausted = true;
+}
+#else
+static inline void erase_ib_gid_table_entry_trace_buffer(struct ib_gid_table_entry *entry) { };
+static inline void save_ib_gid_table_entry_trace_buffer(struct ib_gid_table_entry *entry,
+							struct net_device *ndev, int delta) { };
+#endif
+
 static void dispatch_gid_change_event(struct ib_device *ib_dev, u32 port)
 {
 	struct ib_event event;
@@ -256,8 +354,11 @@ static void free_gid_entry_locked(struct ib_gid_table_entry *entry)
 	/* Now this index is ready to be allocated */
 	write_unlock_irq(&table->rwlock);
 
-	if (entry->ndev_storage)
+	if (entry->ndev_storage) {
+		save_ib_gid_table_entry_trace_buffer(entry, entry->ndev_storage->ndev, -1);
 		call_rcu(&entry->ndev_storage->rcu_head, put_gid_ndev);
+	}
+	erase_ib_gid_table_entry_trace_buffer(entry);
 	kfree(entry);
 }
 
@@ -312,6 +413,7 @@ alloc_gid_entry(const struct ib_gid_attr *attr)
 		entry->ndev_storage->ndev = ndev;
 	}
 	kref_init(&entry->kref);
+	save_ib_gid_table_entry_trace_buffer(entry, ndev, 0);
 	memcpy(&entry->attr, attr, sizeof(*attr));
 	INIT_WORK(&entry->del_work, free_gid_work);
 	entry->state = GID_TABLE_ENTRY_INVALID;
@@ -336,15 +438,21 @@ static void store_gid_entry(struct ib_gid_table *table,
 static void get_gid_entry(struct ib_gid_table_entry *entry)
 {
 	kref_get(&entry->kref);
+	save_ib_gid_table_entry_trace_buffer(entry, entry->ndev_storage ?
+					     entry->ndev_storage->ndev : NULL, 1);
 }
 
 static void put_gid_entry(struct ib_gid_table_entry *entry)
 {
+	save_ib_gid_table_entry_trace_buffer(entry, entry->ndev_storage ?
+					     entry->ndev_storage->ndev : NULL, -1);
 	kref_put(&entry->kref, schedule_free_gid);
 }
 
 static void put_gid_entry_locked(struct ib_gid_table_entry *entry)
 {
+	save_ib_gid_table_entry_trace_buffer(entry, entry->ndev_storage ?
+					     entry->ndev_storage->ndev : NULL, -1);
 	kref_put(&entry->kref, free_gid_entry);
 }
 
@@ -407,6 +515,7 @@ static void del_gid(struct ib_device *ib_dev, u32 port,
 	if (ndev_storage) {
 		entry->ndev_storage = NULL;
 		rcu_assign_pointer(entry->attr.ndev, NULL);
+		save_ib_gid_table_entry_trace_buffer(entry, ndev_storage->ndev, -1);
 		call_rcu(&ndev_storage->rcu_head, put_gid_ndev);
 	}
 
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 95e89f5c147c..f00f1d3fbd9c 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -2009,6 +2009,7 @@ static void destroy_mc(struct rdma_id_private *id_priv,
 		ib_sa_free_multicast(mc->sa_mc);
 
 	if (rdma_protocol_roce(id_priv->id.device, id_priv->id.port_num)) {
+		struct rdma_cm_event *event = &mc->iboe_join.event;
 		struct rdma_dev_addr *dev_addr =
 			&id_priv->id.route.addr.dev_addr;
 		struct net_device *ndev = NULL;
@@ -2031,6 +2032,8 @@ static void destroy_mc(struct rdma_id_private *id_priv,
 		dev_put(ndev);
 
 		cancel_work_sync(&mc->iboe_join.work);
+		if (event->event == RDMA_CM_EVENT_MULTICAST_JOIN)
+			rdma_destroy_ah_attr(&event->param.ud.ah_attr);
 	}
 	kfree(mc);
 }
diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h
index 05102769a918..c851cc40045e 100644
--- a/drivers/infiniband/core/core_priv.h
+++ b/drivers/infiniband/core/core_priv.h
@@ -46,6 +46,12 @@
 #include "mad_priv.h"
 #include "restrack.h"
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+void dump_ib_gid_table_entry_trace_buffer(const struct net_device *ndev);
+#else
+static inline void dump_ib_gid_table_entry_trace_buffer(const struct net_device *ndev) { };
+#endif
+
 /* Total number of ports combined across all struct ib_devices's */
 #define RDMA_MAX_PORTS 8192
 
@@ -142,6 +148,7 @@ int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u32 port,
 
 int roce_gid_mgmt_init(void);
 void roce_gid_mgmt_cleanup(void);
+void roce_flush_gid_cache_wq(void);
 
 unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u32 port);
 
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index 13e8a1714bbd..55dc8d7a52e8 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -1300,6 +1300,7 @@ static void disable_device(struct ib_device *device)
 
 	WARN_ON(!refcount_read(&device->refcount));
 
+	roce_flush_gid_cache_wq();
 	down_write(&devices_rwsem);
 	xa_clear_mark(&devices, device->index, DEVICE_REGISTERED);
 	up_write(&devices_rwsem);
@@ -1412,6 +1413,29 @@ static void ib_device_notify_register(struct ib_device *device)
 	up_read(&devices_rwsem);
 }
 
+static int ib_check_netdev_state(struct ib_device *device, const char *id)
+{
+	struct ib_port_data *pdata;
+	u32 port;
+	struct net_device *ndev = NULL;
+	int ret = 0;
+
+	rcu_read_lock();
+	pdata = device->port_data;
+	if (pdata) {
+		rdma_for_each_port(device, port) {
+			ndev = rcu_dereference(pdata[port].netdev);
+			if (ndev && ndev->reg_state != NETREG_REGISTERED) {
+				pr_info("%s: netdev %s is no longer registered.\n", id, ndev->name);
+				ret = -ENODEV;
+				break;
+			}
+		}
+	}
+	rcu_read_unlock();
+	return ret;
+}
+
 /**
  * ib_register_device - Register an IB device with IB core
  * @device: Device to register
@@ -1484,7 +1508,9 @@ int ib_register_device(struct ib_device *device, const char *name,
 		goto dev_cleanup;
 	}
 
+	ib_check_netdev_state(device, "before enable_device_and_get()");
 	ret = enable_device_and_get(device);
+	ib_check_netdev_state(device, "after enable_device_and_get()");
 	if (ret) {
 		void (*dealloc_fn)(struct ib_device *);
 
@@ -1664,7 +1690,11 @@ static void ib_unregister_work(struct work_struct *work)
 	struct ib_device *ib_dev =
 		container_of(work, struct ib_device, unregistration_work);
 
+	if (IS_ENABLED(CONFIG_NET_DEV_REFCNT_TRACKER))
+		pr_info("netdevice_event(NETDEV_UNREGISTER) on %p start\n", ib_dev);
 	__ib_unregister_device(ib_dev);
+	if (IS_ENABLED(CONFIG_NET_DEV_REFCNT_TRACKER))
+		pr_info("netdevice_event(NETDEV_UNREGISTER) on %p end\n", ib_dev);
 	put_device(&ib_dev->dev);
 }
 
@@ -1712,6 +1742,7 @@ static int rdma_dev_change_netns(struct ib_device *device, struct net *cur_net,
 		ret = -ENODEV;
 		goto out;
 	}
+	ib_check_netdev_state(device, "inside rdma_dev_change_netns()");
 
 	kobject_uevent(&device->dev.kobj, KOBJ_REMOVE);
 	disable_device(device);
@@ -2254,6 +2285,8 @@ int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev,
 	spin_unlock_irqrestore(&pdata->netdev_lock, flags);
 
 	add_ndev_hash(pdata);
+	if (IS_ENABLED(CONFIG_NET_DEV_REFCNT_TRACKER))
+		pr_info("add_ndev_hash(%s) on %p done\n", ndev->name, ib_dev);
 
 	/* Make sure that the device is registered before we send events */
 	if (xa_load(&devices, ib_dev->index) != ib_dev)
diff --git a/drivers/infiniband/core/roce_gid_mgmt.c b/drivers/infiniband/core/roce_gid_mgmt.c
index a9f2c6b1b29e..c398b3b5d6d8 100644
--- a/drivers/infiniband/core/roce_gid_mgmt.c
+++ b/drivers/infiniband/core/roce_gid_mgmt.c
@@ -438,6 +438,18 @@ static void del_netdev_ips(struct ib_device *ib_dev, u32 port,
 	ib_cache_gid_del_all_netdev_gids(ib_dev, port, cookie);
 }
 
+static void del_netdev_ips0(struct ib_device *ib_dev, u32 port,
+			    struct net_device *rdma_ndev, void *cookie)
+{
+	if (IS_ENABLED(CONFIG_NET_DEV_REFCNT_TRACKER))
+		pr_info("netdevice_event(NETDEV_UNREGISTER) for %p on %p start\n",
+			rdma_ndev, ib_dev);
+	ib_cache_gid_del_all_netdev_gids(ib_dev, port, cookie);
+	if (IS_ENABLED(CONFIG_NET_DEV_REFCNT_TRACKER))
+		pr_info("netdevice_event(NETDEV_UNREGISTER) for %p on %p end\n",
+			rdma_ndev, ib_dev);
+}
+
 /**
  * del_default_gids - Delete default GIDs of the event/cookie netdevice
  * @ib_dev:	RDMA device pointer
@@ -661,10 +673,7 @@ static int netdevice_queue_work(struct netdev_event_work_cmd *cmds,
 {
 	unsigned int i;
 	struct netdev_event_work *ndev_work =
-		kmalloc(sizeof(*ndev_work), GFP_KERNEL);
-
-	if (!ndev_work)
-		return NOTIFY_DONE;
+		kmalloc(sizeof(*ndev_work), GFP_KERNEL | __GFP_NOFAIL);
 
 	memcpy(ndev_work->cmds, cmds, sizeof(ndev_work->cmds));
 	for (i = 0; i < ARRAY_SIZE(ndev_work->cmds) && ndev_work->cmds[i].cb; i++) {
@@ -760,7 +769,7 @@ static int netdevice_event(struct notifier_block *this, unsigned long event,
 			   void *ptr)
 {
 	static const struct netdev_event_work_cmd del_cmd = {
-		.cb = del_netdev_ips, .filter = pass_all_filter};
+		.cb = del_netdev_ips0, .filter = pass_all_filter};
 	static const struct netdev_event_work_cmd
 			bonding_default_del_cmd_join = {
 				.cb	= del_netdev_default_ips_join,
@@ -776,6 +785,9 @@ static int netdevice_event(struct notifier_block *this, unsigned long event,
 	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
 	struct netdev_event_work_cmd cmds[ROCE_NETDEV_CALLBACK_SZ] = { {NULL} };
 
+	if (event == NETDEV_DEBUG_UNREGISTER)
+		dump_ib_gid_table_entry_trace_buffer(ndev);
+
 	if (ndev->type != ARPHRD_ETHER)
 		return NOTIFY_DONE;
 
@@ -948,3 +960,8 @@ void __exit roce_gid_mgmt_cleanup(void)
 	 */
 	destroy_workqueue(gid_cache_wq);
 }
+
+void roce_flush_gid_cache_wq(void)
+{
+	flush_workqueue(gid_cache_wq);
+}
diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c
index 22d3e25c3b9d..755bba8d58bb 100644
--- a/drivers/infiniband/hw/efa/efa_verbs.c
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -1320,13 +1320,9 @@ static int umem_to_page_list(struct efa_dev *dev,
 			     u32 hp_cnt,
 			     u8 hp_shift)
 {
-	u32 pages_in_hp = BIT(hp_shift - PAGE_SHIFT);
 	struct ib_block_iter biter;
 	unsigned int hp_idx = 0;
 
-	ibdev_dbg(&dev->ibdev, "hp_cnt[%u], pages_in_hp[%u]\n",
-		  hp_cnt, pages_in_hp);
-
 	rdma_umem_for_each_dma_block(umem, &biter, BIT(hp_shift))
 		page_list[hp_idx++] = rdma_block_iter_dma_address(&biter);
 
diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c
index cc2a12f735d3..13d7499131d4 100644
--- a/drivers/infiniband/hw/irdma/utils.c
+++ b/drivers/infiniband/hw/irdma/utils.c
@@ -251,7 +251,7 @@ int irdma_net_event(struct notifier_block *notifier, unsigned long event,
 		    void *ptr)
 {
 	struct neighbour *neigh = ptr;
-	struct net_device *real_dev, *netdev = (struct net_device *)neigh->dev;
+	struct net_device *real_dev, *netdev;
 	struct irdma_device *iwdev;
 	struct ib_device *ibdev;
 	__be32 *p;
@@ -260,6 +260,7 @@ int irdma_net_event(struct notifier_block *notifier, unsigned long event,
 
 	switch (event) {
 	case NETEVENT_NEIGH_UPDATE:
+		netdev = neigh->dev;
 		real_dev = rdma_vlan_dev_real_dev(netdev);
 		if (!real_dev)
 			real_dev = netdev;
diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c
index 1becc8779123..7600412b0739 100644
--- a/drivers/infiniband/hw/mana/cq.c
+++ b/drivers/infiniband/hw/mana/cq.c
@@ -56,6 +56,10 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		doorbell = mana_ucontext->doorbell;
 	} else {
 		is_rnic_cq = true;
+		if (attr->cqe > U32_MAX / COMP_ENTRY_SIZE / 2 + 1) {
+			ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe);
+			return -EINVAL;
+		}
 		buf_size = MANA_PAGE_ALIGN(roundup_pow_of_two(attr->cqe * COMP_ENTRY_SIZE));
 		cq->cqe = buf_size / COMP_ENTRY_SIZE;
 		err = mana_ib_create_kernel_queue(mdev, buf_size, GDMA_CQ, &cq->queue);
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-pri.h b/drivers/infiniband/ulp/rtrs/rtrs-pri.h
index ef29bd483b5a..59529d593869 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-pri.h
+++ b/drivers/infiniband/ulp/rtrs/rtrs-pri.h
@@ -150,7 +150,7 @@ enum rtrs_msg_types {
 
 /**
  * enum rtrs_msg_flags - RTRS message flags.
- * @RTRS_NEED_INVAL:	Send invalidation in response.
+ * @RTRS_MSG_NEED_INVAL_F: Send invalidation in response.
  * @RTRS_MSG_NEW_RKEY_F: Send refreshed rkey in response.
  */
 enum rtrs_msg_flags {
@@ -179,16 +179,19 @@ struct rtrs_sg_desc {
  * @recon_cnt:	   Reconnections counter
  * @sess_uuid:	   UUID of a session (path)
  * @paths_uuid:	   UUID of a group of sessions (paths)
- *
+ * @first_conn:    %1 if the connection request is the first for that session,
+ *			otherwise %0
  * NOTE: max size 56 bytes, see man rdma_connect().
  */
 struct rtrs_msg_conn_req {
-	/* Is set to 0 by cma.c in case of AF_IB, do not touch that.
-	 * see https://www.spinics.net/lists/linux-rdma/msg22397.html
+	/**
+	 * @__cma_version: Is set to 0 by cma.c in case of AF_IB, do not touch
+	 * that. See https://www.spinics.net/lists/linux-rdma/msg22397.html
 	 */
 	u8		__cma_version;
-	/* On sender side that should be set to 0, or cma_save_ip_info()
-	 * extract garbage and will fail.
+	/**
+	 * @__ip_version: On sender side that should be set to 0, or
+	 * cma_save_ip_info() extract garbage and will fail.
 	 */
 	u8		__ip_version;
 	__le16		magic;
@@ -199,6 +202,7 @@ struct rtrs_msg_conn_req {
 	uuid_t		sess_uuid;
 	uuid_t		paths_uuid;
 	u8		first_conn : 1;
+	/* private: */
 	u8		reserved_bits : 7;
 	u8		reserved[11];
 };
@@ -211,6 +215,7 @@ struct rtrs_msg_conn_req {
  * @queue_depth:   max inflight messages (queue-depth) in this session
  * @max_io_size:   max io size server supports
  * @max_hdr_size:  max msg header size server supports
+ * @flags:	   RTRS message flags for this message
  *
  * NOTE: size is 56 bytes, max possible is 136 bytes, see man rdma_accept().
  */
@@ -222,22 +227,24 @@ struct rtrs_msg_conn_rsp {
 	__le32		max_io_size;
 	__le32		max_hdr_size;
 	__le32		flags;
+	/* private: */
 	u8		reserved[36];
 };
 
 /**
- * struct rtrs_msg_info_req
+ * struct rtrs_msg_info_req - client additional info request
  * @type:		@RTRS_MSG_INFO_REQ
  * @pathname:		Path name chosen by client
  */
 struct rtrs_msg_info_req {
 	__le16		type;
 	u8		pathname[NAME_MAX];
+	/* private: */
 	u8		reserved[15];
 };
 
 /**
- * struct rtrs_msg_info_rsp
+ * struct rtrs_msg_info_rsp - server additional info response
  * @type:		@RTRS_MSG_INFO_RSP
  * @sg_cnt:		Number of @desc entries
  * @desc:		RDMA buffers where the client can write to server
@@ -245,12 +252,14 @@ struct rtrs_msg_info_req {
 struct rtrs_msg_info_rsp {
 	__le16		type;
 	__le16          sg_cnt;
+	/* private: */
 	u8              reserved[4];
+	/* public: */
 	struct rtrs_sg_desc desc[];
 };
 
 /**
- * struct rtrs_msg_rkey_rsp
+ * struct rtrs_msg_rkey_rsp - server refreshed rkey response
  * @type:		@RTRS_MSG_RKEY_RSP
  * @buf_id:		RDMA buf_id of the new rkey
  * @rkey:		new remote key for RDMA buffers id from server
@@ -264,6 +273,7 @@ struct rtrs_msg_rkey_rsp {
 /**
  * struct rtrs_msg_rdma_read - RDMA data transfer request from client
  * @type:		always @RTRS_MSG_READ
+ * @flags:		RTRS message flags (enum rtrs_msg_flags)
  * @usr_len:		length of user payload
  * @sg_cnt:		number of @desc entries
  * @desc:		RDMA buffers where the server can write the result to
@@ -277,7 +287,7 @@ struct rtrs_msg_rdma_read {
 };
 
 /**
- * struct_msg_rdma_write - Message transferred to server with RDMA-Write
+ * struct rtrs_msg_rdma_write - Message transferred to server with RDMA-Write
  * @type:		always @RTRS_MSG_WRITE
  * @usr_len:		length of user payload
  */
@@ -287,7 +297,7 @@ struct rtrs_msg_rdma_write {
 };
 
 /**
- * struct_msg_rdma_hdr - header for read or write request
+ * struct rtrs_msg_rdma_hdr - header for read or write request
  * @type:		@RTRS_MSG_WRITE | @RTRS_MSG_READ
  */
 struct rtrs_msg_rdma_hdr {
diff --git a/drivers/infiniband/ulp/rtrs/rtrs.h b/drivers/infiniband/ulp/rtrs/rtrs.h
index b48b53a7c143..b5bd35712de0 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs.h
+++ b/drivers/infiniband/ulp/rtrs/rtrs.h
@@ -24,8 +24,8 @@ struct rtrs_srv_op;
 
 /**
  * enum rtrs_clt_link_ev - Events about connectivity state of a client
- * @RTRS_CLT_LINK_EV_RECONNECTED	Client was reconnected.
- * @RTRS_CLT_LINK_EV_DISCONNECTED	Client was disconnected.
+ * @RTRS_CLT_LINK_EV_RECONNECTED:	Client was reconnected.
+ * @RTRS_CLT_LINK_EV_DISCONNECTED:	Client was disconnected.
  */
 enum rtrs_clt_link_ev {
 	RTRS_CLT_LINK_EV_RECONNECTED,
@@ -33,7 +33,9 @@ enum rtrs_clt_link_ev {
 };
 
 /**
- * Source and destination address of a path to be established
+ * struct rtrs_addr - Source and destination address of a path to be established
+ * @src:	source address
+ * @dst:	destination address
  */
 struct rtrs_addr {
 	struct sockaddr_storage *src;
@@ -41,7 +43,7 @@ struct rtrs_addr {
 };
 
 /**
- * rtrs_clt_ops - it holds the link event callback and private pointer.
+ * struct rtrs_clt_ops - it holds the link event callback and private pointer.
  * @priv: User supplied private data.
  * @link_ev: Event notification callback function for connection state changes
  *	@priv: User supplied data that was passed to rtrs_clt_open()
@@ -67,10 +69,10 @@ enum wait_type {
 };
 
 /**
- * enum rtrs_clt_con_type() type of ib connection to use with a given
+ * enum rtrs_clt_con_type - type of ib connection to use with a given
  * rtrs_permit
- * @ADMIN_CON - use connection reserved for "service" messages
- * @IO_CON - use a connection reserved for IO
+ * @RTRS_ADMIN_CON: use connection reserved for "service" messages
+ * @RTRS_IO_CON: use a connection reserved for IO
  */
 enum rtrs_clt_con_type {
 	RTRS_ADMIN_CON,
@@ -85,7 +87,7 @@ void rtrs_clt_put_permit(struct rtrs_clt_sess *sess,
 			 struct rtrs_permit *permit);
 
 /**
- * rtrs_clt_req_ops - it holds the request confirmation callback
+ * struct rtrs_clt_req_ops - it holds the request confirmation callback
  * and a private pointer.
  * @priv: User supplied private data.
  * @conf_fn:	callback function to be called as confirmation
@@ -105,7 +107,11 @@ int rtrs_clt_request(int dir, struct rtrs_clt_req_ops *ops,
 int rtrs_clt_rdma_cq_direct(struct rtrs_clt_sess *clt, unsigned int index);
 
 /**
- * rtrs_attrs - RTRS session attributes
+ * struct rtrs_attrs - RTRS session attributes
+ * @queue_depth:	queue_depth saved from rtrs_clt_sess message
+ * @max_io_size:	max_io_size from rtrs_clt_sess message, capped to
+ *			  @max_segments * %SZ_4K
+ * @max_segments:	max_segments saved from rtrs_clt_sess message
  */
 struct rtrs_attrs {
 	u32		queue_depth;
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index 23ed2fc688f0..2012ba22a7af 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -33,6 +33,7 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/module.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/err.h>
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index 71269446353d..e314e6a84d96 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -33,6 +33,7 @@
  */
 
 #include <linux/module.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/err.h>
diff --git a/drivers/input/keyboard/atkbd.c b/drivers/input/keyboard/atkbd.c
index 6c999d89ee4b..422e28ad1e8e 100644
--- a/drivers/input/keyboard/atkbd.c
+++ b/drivers/input/keyboard/atkbd.c
@@ -1937,6 +1937,13 @@ static const struct dmi_system_id atkbd_dmi_quirk_table[] __initconst = {
 		},
 		.callback = atkbd_deactivate_fixup,
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "HONOR"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "FMB-P"),
+		},
+		.callback = atkbd_deactivate_fixup,
+	},
 	{ }
 };
 
diff --git a/drivers/input/keyboard/cros_ec_keyb.c b/drivers/input/keyboard/cros_ec_keyb.c
index 1c6b0461dc35..2822c592880b 100644
--- a/drivers/input/keyboard/cros_ec_keyb.c
+++ b/drivers/input/keyboard/cros_ec_keyb.c
@@ -269,7 +269,8 @@ static int cros_ec_keyb_work(struct notifier_block *nb,
 
 		if (ckdev->ec->event_size != ckdev->cols) {
 			dev_err(ckdev->dev,
-				"Discarded incomplete key matrix event.\n");
+				"Discarded key matrix event, unexpected length: %d != %d\n",
+				ckdev->ec->event_size, ckdev->cols);
 			return NOTIFY_OK;
 		}
 
diff --git a/drivers/input/keyboard/lkkbd.c b/drivers/input/keyboard/lkkbd.c
index c035216dd27c..2f130f819363 100644
--- a/drivers/input/keyboard/lkkbd.c
+++ b/drivers/input/keyboard/lkkbd.c
@@ -670,7 +670,8 @@ static int lkkbd_connect(struct serio *serio, struct serio_driver *drv)
 
 	return 0;
 
- fail3:	serio_close(serio);
+ fail3:	disable_work_sync(&lk->tq);
+	serio_close(serio);
  fail2:	serio_set_drvdata(serio, NULL);
  fail1:	input_free_device(input_dev);
 	kfree(lk);
@@ -684,6 +685,8 @@ static void lkkbd_disconnect(struct serio *serio)
 {
 	struct lkkbd *lk = serio_get_drvdata(serio);
 
+	disable_work_sync(&lk->tq);
+
 	input_get_device(lk->dev);
 	input_unregister_device(lk->dev);
 	serio_close(serio);
diff --git a/drivers/input/keyboard/omap4-keypad.c b/drivers/input/keyboard/omap4-keypad.c
index bffe89c0717a..e783244d0c91 100644
--- a/drivers/input/keyboard/omap4-keypad.c
+++ b/drivers/input/keyboard/omap4-keypad.c
@@ -193,7 +193,6 @@ static irqreturn_t omap4_keypad_irq_thread_fn(int irq, void *dev_id)
 	kbd_write_irqreg(keypad_data, OMAP4_KBD_IRQSTATUS,
 			 kbd_read_irqreg(keypad_data, OMAP4_KBD_IRQSTATUS));
 
-	pm_runtime_mark_last_busy(dev);
 	pm_runtime_put_autosuspend(dev);
 
 	return IRQ_HANDLED;
@@ -231,7 +230,6 @@ static int omap4_keypad_open(struct input_dev *input)
 	enable_irq(keypad_data->irq);
 
 out:
-	pm_runtime_mark_last_busy(dev);
 	pm_runtime_put_autosuspend(dev);
 
 	return error;
@@ -265,7 +263,6 @@ static void omap4_keypad_close(struct input_dev *input)
 	enable_irq(keypad_data->irq);
 	clk_disable_unprepare(keypad_data->fck);
 
-	pm_runtime_mark_last_busy(dev);
 	pm_runtime_put_autosuspend(dev);
 }
 
@@ -404,7 +401,6 @@ static int omap4_keypad_probe(struct platform_device *pdev)
 		omap4_keypad_stop(keypad_data);
 	}
 
-	pm_runtime_mark_last_busy(dev);
 	pm_runtime_put_autosuspend(dev);
 	if (error)
 		return error;
diff --git a/drivers/input/misc/cs40l50-vibra.c b/drivers/input/misc/cs40l50-vibra.c
index 7aa7d577e01b..90410025bbae 100644
--- a/drivers/input/misc/cs40l50-vibra.c
+++ b/drivers/input/misc/cs40l50-vibra.c
@@ -308,7 +308,6 @@ static void cs40l50_add_worker(struct work_struct *work)
 			list_add(&effect->list, &vib->effect_head);
 	}
 err_pm:
-	pm_runtime_mark_last_busy(vib->dev);
 	pm_runtime_put_autosuspend(vib->dev);
 err_exit:
 	work_data->error = error;
@@ -368,7 +367,6 @@ static void cs40l50_start_worker(struct work_struct *work)
 		dev_err(vib->dev, "Effect to play not found\n");
 	}
 
-	pm_runtime_mark_last_busy(vib->dev);
 	pm_runtime_put_autosuspend(vib->dev);
 err_free:
 	kfree(work_data);
@@ -384,7 +382,6 @@ static void cs40l50_stop_worker(struct work_struct *work)
 
 	vib->dsp.write(vib->dev, vib->regmap, vib->dsp.stop_cmd);
 
-	pm_runtime_mark_last_busy(vib->dev);
 	pm_runtime_put_autosuspend(vib->dev);
 
 	kfree(work_data);
@@ -456,7 +453,6 @@ static void cs40l50_erase_worker(struct work_struct *work)
 	list_del(&erase_effect->list);
 	kfree(erase_effect);
 err_pm:
-	pm_runtime_mark_last_busy(vib->dev);
 	pm_runtime_put_autosuspend(vib->dev);
 err_exit:
 	work_data->error = error;
diff --git a/drivers/input/misc/pf1550-onkey.c b/drivers/input/misc/pf1550-onkey.c
index 9be6377151cb..0d1b570bbe47 100644
--- a/drivers/input/misc/pf1550-onkey.c
+++ b/drivers/input/misc/pf1550-onkey.c
@@ -173,7 +173,7 @@ static int pf1550_onkey_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(pf1550_onkey_pm_ops, pf1550_onkey_suspend,
+static DEFINE_SIMPLE_DEV_PM_OPS(pf1550_onkey_pm_ops, pf1550_onkey_suspend,
 			 pf1550_onkey_resume);
 
 static const struct platform_device_id pf1550_onkey_id[] = {
diff --git a/drivers/input/mouse/byd.c b/drivers/input/mouse/byd.c
index 71aa23dd7d8d..7f85c7ab68c5 100644
--- a/drivers/input/mouse/byd.c
+++ b/drivers/input/mouse/byd.c
@@ -314,10 +314,8 @@ static psmouse_ret_t byd_process_byte(struct psmouse *psmouse)
 		break;
 	}
 	default:
-		psmouse_warn(psmouse,
-			     "Unrecognized Z: pkt = %02x %02x %02x %02x\n",
-			     psmouse->packet[0], psmouse->packet[1],
-			     psmouse->packet[2], psmouse->packet[3]);
+		psmouse_warn(psmouse, "Unrecognized Z: pkt = %*ph\n",
+			     4, psmouse->packet);
 		return PSMOUSE_BAD_DATA;
 	}
 
diff --git a/drivers/input/mouse/cyapa.c b/drivers/input/mouse/cyapa.c
index 00c87c0532a6..6e0d956617a1 100644
--- a/drivers/input/mouse/cyapa.c
+++ b/drivers/input/mouse/cyapa.c
@@ -403,7 +403,6 @@ static int cyapa_open(struct input_dev *input)
 	}
 
 	pm_runtime_get_sync(dev);
-	pm_runtime_mark_last_busy(dev);
 	pm_runtime_put_sync_autosuspend(dev);
 out:
 	mutex_unlock(&cyapa->state_sync_lock);
@@ -666,7 +665,6 @@ static int cyapa_reinitialize(struct cyapa *cyapa)
 		pm_runtime_enable(dev);
 
 		pm_runtime_get_sync(dev);
-		pm_runtime_mark_last_busy(dev);
 		pm_runtime_put_sync_autosuspend(dev);
 	}
 
@@ -710,7 +708,6 @@ static irqreturn_t cyapa_irq(int irq, void *dev_id)
 			 * process.
 			 */
 			pm_runtime_get_sync(dev);
-			pm_runtime_mark_last_busy(dev);
 			pm_runtime_put_sync_autosuspend(dev);
 		}
 
diff --git a/drivers/input/mouse/cyapa_gen5.c b/drivers/input/mouse/cyapa_gen5.c
index 3b4439f10635..59f6e97d5482 100644
--- a/drivers/input/mouse/cyapa_gen5.c
+++ b/drivers/input/mouse/cyapa_gen5.c
@@ -2833,7 +2833,6 @@ static int cyapa_pip_event_process(struct cyapa *cyapa,
 		 * process.
 		 */
 		pm_runtime_get_sync(dev);
-		pm_runtime_mark_last_busy(dev);
 		pm_runtime_put_sync_autosuspend(dev);
 		return 0;
 	} else if (report_id != PIP_TOUCH_REPORT_ID &&
diff --git a/drivers/input/serio/i8042-acpipnpio.h b/drivers/input/serio/i8042-acpipnpio.h
index 1caa6c4ca435..654771275ce8 100644
--- a/drivers/input/serio/i8042-acpipnpio.h
+++ b/drivers/input/serio/i8042-acpipnpio.h
@@ -1169,6 +1169,13 @@ static const struct dmi_system_id i8042_dmi_quirk_table[] __initconst = {
 		.driver_data = (void *)(SERIO_QUIRK_NOMUX | SERIO_QUIRK_RESET_ALWAYS |
 					SERIO_QUIRK_NOLOOP | SERIO_QUIRK_NOPNP)
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_NAME, "X5KK45xS_X5SP45xS"),
+		},
+		.driver_data = (void *)(SERIO_QUIRK_NOMUX | SERIO_QUIRK_RESET_ALWAYS |
+					SERIO_QUIRK_NOLOOP | SERIO_QUIRK_NOPNP)
+	},
 	/*
 	 * A lot of modern Clevo barebones have touchpad and/or keyboard issues
 	 * after suspend fixable with the forcenorestore quirk.
diff --git a/drivers/input/touchscreen/iqs5xx.c b/drivers/input/touchscreen/iqs5xx.c
index 4ebd7565ae6e..c63819abaf9b 100644
--- a/drivers/input/touchscreen/iqs5xx.c
+++ b/drivers/input/touchscreen/iqs5xx.c
@@ -17,6 +17,7 @@
 #include <linux/err.h>
 #include <linux/firmware.h>
 #include <linux/gpio/consumer.h>
+#include <linux/hex.h>
 #include <linux/i2c.h>
 #include <linux/input.h>
 #include <linux/input/mt.h>
diff --git a/drivers/input/touchscreen/stmfts.c b/drivers/input/touchscreen/stmfts.c
index 119cd26851cf..4b166b0a9a5a 100644
--- a/drivers/input/touchscreen/stmfts.c
+++ b/drivers/input/touchscreen/stmfts.c
@@ -120,7 +120,7 @@ static int stmfts_brightness_set(struct led_classdev *led_cdev,
 			err = regulator_enable(sdata->ledvdd);
 			if (err) {
 				dev_warn(&sdata->client->dev,
-					 "failed to disable ledvdd regulator: %d\n",
+					 "failed to enable ledvdd regulator: %d\n",
 					 err);
 				return err;
 			}
@@ -141,7 +141,7 @@ static enum led_brightness stmfts_brightness_get(struct led_classdev *led_cdev)
 
 /*
  * We can't simply use i2c_smbus_read_i2c_block_data because we
- * need to read more than 255 bytes (
+ * need to read 256 bytes, which exceeds the 255-byte SMBus block limit.
  */
 static int stmfts_read_events(struct stmfts_data *sdata)
 {
@@ -410,7 +410,7 @@ static ssize_t stmfts_sysfs_chip_id(struct device *dev,
 {
 	struct stmfts_data *sdata = dev_get_drvdata(dev);
 
-	return sprintf(buf, "%#x\n", sdata->chip_id);
+	return sysfs_emit(buf, "%#x\n", sdata->chip_id);
 }
 
 static ssize_t stmfts_sysfs_chip_version(struct device *dev,
@@ -418,7 +418,7 @@ static ssize_t stmfts_sysfs_chip_version(struct device *dev,
 {
 	struct stmfts_data *sdata = dev_get_drvdata(dev);
 
-	return sprintf(buf, "%u\n", sdata->chip_ver);
+	return sysfs_emit(buf, "%u\n", sdata->chip_ver);
 }
 
 static ssize_t stmfts_sysfs_fw_ver(struct device *dev,
@@ -426,7 +426,7 @@ static ssize_t stmfts_sysfs_fw_ver(struct device *dev,
 {
 	struct stmfts_data *sdata = dev_get_drvdata(dev);
 
-	return sprintf(buf, "%u\n", sdata->fw_ver);
+	return sysfs_emit(buf, "%u\n", sdata->fw_ver);
 }
 
 static ssize_t stmfts_sysfs_config_id(struct device *dev,
@@ -434,7 +434,7 @@ static ssize_t stmfts_sysfs_config_id(struct device *dev,
 {
 	struct stmfts_data *sdata = dev_get_drvdata(dev);
 
-	return sprintf(buf, "%#x\n", sdata->config_id);
+	return sysfs_emit(buf, "%#x\n", sdata->config_id);
 }
 
 static ssize_t stmfts_sysfs_config_version(struct device *dev,
@@ -442,7 +442,7 @@ static ssize_t stmfts_sysfs_config_version(struct device *dev,
 {
 	struct stmfts_data *sdata = dev_get_drvdata(dev);
 
-	return sprintf(buf, "%u\n", sdata->config_ver);
+	return sysfs_emit(buf, "%u\n", sdata->config_ver);
 }
 
 static ssize_t stmfts_sysfs_read_status(struct device *dev,
@@ -457,7 +457,7 @@ static ssize_t stmfts_sysfs_read_status(struct device *dev,
 	if (err)
 		return err;
 
-	return sprintf(buf, "%#02x\n", status[0]);
+	return sysfs_emit(buf, "%#02x\n", status[0]);
 }
 
 static ssize_t stmfts_sysfs_hover_enable_read(struct device *dev,
@@ -465,7 +465,7 @@ static ssize_t stmfts_sysfs_hover_enable_read(struct device *dev,
 {
 	struct stmfts_data *sdata = dev_get_drvdata(dev);
 
-	return sprintf(buf, "%u\n", sdata->hover_enabled);
+	return sysfs_emit(buf, "%u\n", sdata->hover_enabled);
 }
 
 static ssize_t stmfts_sysfs_hover_enable_write(struct device *dev,
@@ -594,9 +594,6 @@ static void stmfts_power_off(void *data)
 						sdata->regulators);
 }
 
-/* This function is void because I don't want to prevent using the touch key
- * only because the LEDs don't get registered
- */
 static int stmfts_enable_led(struct stmfts_data *sdata)
 {
 	int err;
diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h
index 97aeda1ad01c..3327116a441c 100644
--- a/drivers/iommu/generic_pt/iommu_pt.h
+++ b/drivers/iommu/generic_pt/iommu_pt.h
@@ -372,6 +372,9 @@ static inline struct pt_table_p *_table_alloc(struct pt_common *common,
 
 	table_mem = iommu_alloc_pages_node_sz(iommu_table->nid, gfp,
 					      log2_to_int(lg2sz));
+	if (!table_mem)
+		return ERR_PTR(-ENOMEM);
+
 	if (pt_feature(common, PT_FEAT_DMA_INCOHERENT) &&
 	    mode == ALLOC_NORMAL) {
 		int ret = iommu_pages_start_incoherent(
diff --git a/drivers/iommu/iommufd/io_pagetable.c b/drivers/iommu/iommufd/io_pagetable.c
index 54cf4d856179..436992331111 100644
--- a/drivers/iommu/iommufd/io_pagetable.c
+++ b/drivers/iommu/iommufd/io_pagetable.c
@@ -495,7 +495,11 @@ int iopt_map_file_pages(struct iommufd_ctx *ictx, struct io_pagetable *iopt,
 		return -EOVERFLOW;
 
 	start_byte = start - ALIGN_DOWN(start, PAGE_SIZE);
-	dmabuf = dma_buf_get(fd);
+	if (IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
+		dmabuf = dma_buf_get(fd);
+	else
+		dmabuf = ERR_PTR(-ENXIO);
+
 	if (!IS_ERR(dmabuf)) {
 		pages = iopt_alloc_dmabuf_pages(ictx, dmabuf, start_byte, start,
 						length,
diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c
index c4322fd26f93..550ff36dec3a 100644
--- a/drivers/iommu/iommufd/selftest.c
+++ b/drivers/iommu/iommufd/selftest.c
@@ -1184,14 +1184,20 @@ static int iommufd_test_add_reserved(struct iommufd_ucmd *ucmd,
 				     unsigned int mockpt_id,
 				     unsigned long start, size_t length)
 {
+	unsigned long last;
 	struct iommufd_ioas *ioas;
 	int rc;
 
+	if (!length)
+		return -EINVAL;
+	if (check_add_overflow(start, length - 1, &last))
+		return -EOVERFLOW;
+
 	ioas = iommufd_get_ioas(ucmd->ictx, mockpt_id);
 	if (IS_ERR(ioas))
 		return PTR_ERR(ioas);
 	down_write(&ioas->iopt.iova_rwsem);
-	rc = iopt_reserve_iova(&ioas->iopt, start, start + length - 1, NULL);
+	rc = iopt_reserve_iova(&ioas->iopt, start, last, NULL);
 	up_write(&ioas->iopt.iova_rwsem);
 	iommufd_put_object(ucmd->ictx, &ioas->obj);
 	return rc;
@@ -1215,8 +1221,10 @@ static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd,
 	page_size = 1 << __ffs(mock->domain.pgsize_bitmap);
 	if (iova % page_size || length % page_size ||
 	    (uintptr_t)uptr % page_size ||
-	    check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end))
-		return -EINVAL;
+	    check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) {
+		rc = -EINVAL;
+		goto out_put;
+	}
 
 	for (; length; length -= page_size) {
 		struct page *pages[1];
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index f334f49c9791..118d0c16e633 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -297,6 +297,14 @@ config RENESAS_RZG2L_IRQC
 	  Enable support for the Renesas RZ/G2L (and alike SoC) Interrupt Controller
 	  for external devices.
 
+config RENESAS_RZT2H_ICU
+	bool "Renesas RZ/{T2H,N2H} ICU support" if COMPILE_TEST
+	select GENERIC_IRQ_CHIP
+	select IRQ_DOMAIN_HIERARCHY
+	help
+	  Enable support for the Renesas RZ/{T2H,N2H} Interrupt Controller
+	  (ICU).
+
 config RENESAS_RZV2H_ICU
 	bool "Renesas RZ/V2H(P) ICU support" if COMPILE_TEST
 	select GENERIC_IRQ_CHIP
diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index 6a229443efe0..26aa3b6ec99f 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -54,6 +54,7 @@ obj-$(CONFIG_RENESAS_INTC_IRQPIN)	+= irq-renesas-intc-irqpin.o
 obj-$(CONFIG_RENESAS_IRQC)		+= irq-renesas-irqc.o
 obj-$(CONFIG_RENESAS_RZA1_IRQC)		+= irq-renesas-rza1.o
 obj-$(CONFIG_RENESAS_RZG2L_IRQC)	+= irq-renesas-rzg2l.o
+obj-$(CONFIG_RENESAS_RZT2H_ICU)		+= irq-renesas-rzt2h.o
 obj-$(CONFIG_RENESAS_RZV2H_ICU)		+= irq-renesas-rzv2h.o
 obj-$(CONFIG_VERSATILE_FPGA_IRQ)	+= irq-versatile-fpga.o
 obj-$(CONFIG_ARCH_NSPIRE)		+= irq-zevio.o
diff --git a/drivers/irqchip/irq-aclint-sswi.c b/drivers/irqchip/irq-aclint-sswi.c
index fee30f3bc5ac..325501f34a9b 100644
--- a/drivers/irqchip/irq-aclint-sswi.c
+++ b/drivers/irqchip/irq-aclint-sswi.c
@@ -109,7 +109,7 @@ static int __init aclint_sswi_probe(struct fwnode_handle *fwnode)
 	if (!is_of_node(fwnode))
 		return -EINVAL;
 
-	reg = of_iomap(to_of_node(fwnode), 0);
+	reg = of_io_request_and_map(to_of_node(fwnode), 0, NULL);
 	if (!reg)
 		return -ENOMEM;
 
diff --git a/drivers/irqchip/irq-renesas-rzt2h.c b/drivers/irqchip/irq-renesas-rzt2h.c
new file mode 100644
index 000000000000..53cf80e1155a
--- /dev/null
+++ b/drivers/irqchip/irq-renesas-rzt2h.c
@@ -0,0 +1,280 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/bitfield.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/irq-renesas-rzt2h.h>
+#include <linux/irqdomain.h>
+#include <linux/of_platform.h>
+#include <linux/pm_runtime.h>
+#include <linux/reset.h>
+#include <linux/spinlock.h>
+
+#define RZT2H_ICU_INTCPU_NS_START		0
+#define RZT2H_ICU_INTCPU_NS_COUNT		14
+
+#define RZT2H_ICU_INTCPU_S_START		(RZT2H_ICU_INTCPU_NS_START +	\
+						 RZT2H_ICU_INTCPU_NS_COUNT)
+#define RZT2H_ICU_INTCPU_S_COUNT		2
+
+#define RZT2H_ICU_IRQ_NS_START			(RZT2H_ICU_INTCPU_S_START +	\
+						 RZT2H_ICU_INTCPU_S_COUNT)
+#define RZT2H_ICU_IRQ_NS_COUNT			14
+
+#define RZT2H_ICU_IRQ_S_START			(RZT2H_ICU_IRQ_NS_START +	\
+						 RZT2H_ICU_IRQ_NS_COUNT)
+#define RZT2H_ICU_IRQ_S_COUNT			2
+
+#define RZT2H_ICU_SEI_START			(RZT2H_ICU_IRQ_S_START +	\
+						 RZT2H_ICU_IRQ_S_COUNT)
+#define RZT2H_ICU_SEI_COUNT			1
+
+#define RZT2H_ICU_NUM_IRQ			(RZT2H_ICU_INTCPU_NS_COUNT +	\
+						 RZT2H_ICU_INTCPU_S_COUNT +	\
+						 RZT2H_ICU_IRQ_NS_COUNT +	\
+						 RZT2H_ICU_IRQ_S_COUNT +	\
+						 RZT2H_ICU_SEI_COUNT)
+
+#define RZT2H_ICU_IRQ_IN_RANGE(n, type)						\
+	((n) >= RZT2H_ICU_##type##_START &&					\
+	 (n) <  RZT2H_ICU_##type##_START + RZT2H_ICU_##type##_COUNT)
+
+#define RZT2H_ICU_PORTNF_MD			0xc
+#define RZT2H_ICU_PORTNF_MDi_MASK(i)		(GENMASK(1, 0) << ((i) * 2))
+#define RZT2H_ICU_PORTNF_MDi_PREP(i, val)	(FIELD_PREP(GENMASK(1, 0), val) << ((i) * 2))
+
+#define RZT2H_ICU_MD_LOW_LEVEL			0b00
+#define RZT2H_ICU_MD_FALLING_EDGE		0b01
+#define RZT2H_ICU_MD_RISING_EDGE		0b10
+#define RZT2H_ICU_MD_BOTH_EDGES			0b11
+
+#define RZT2H_ICU_DMACn_RSSELi(n, i)		(0x7d0 + 0x18 * (n) + 0x4 * (i))
+#define RZT2H_ICU_DMAC_REQ_SELx_MASK(x)		(GENMASK(9, 0) << ((x) * 10))
+#define RZT2H_ICU_DMAC_REQ_SELx_PREP(x, val)	(FIELD_PREP(GENMASK(9, 0), val) << ((x) * 10))
+
+struct rzt2h_icu_priv {
+	void __iomem		*base_ns;
+	void __iomem		*base_s;
+	struct irq_fwspec	fwspec[RZT2H_ICU_NUM_IRQ];
+	raw_spinlock_t		lock;
+};
+
+void rzt2h_icu_register_dma_req(struct platform_device *icu_dev, u8 dmac_index, u8 dmac_channel,
+				u16 req_no)
+{
+	struct rzt2h_icu_priv *priv = platform_get_drvdata(icu_dev);
+	u8 y, upper;
+	u32 val;
+
+	y = dmac_channel / 3;
+	upper = dmac_channel % 3;
+
+	guard(raw_spinlock_irqsave)(&priv->lock);
+	val = readl(priv->base_ns + RZT2H_ICU_DMACn_RSSELi(dmac_index, y));
+	val &= ~RZT2H_ICU_DMAC_REQ_SELx_MASK(upper);
+	val |= RZT2H_ICU_DMAC_REQ_SELx_PREP(upper, req_no);
+	writel(val, priv->base_ns + RZT2H_ICU_DMACn_RSSELi(dmac_index, y));
+}
+EXPORT_SYMBOL_GPL(rzt2h_icu_register_dma_req);
+
+static inline struct rzt2h_icu_priv *irq_data_to_priv(struct irq_data *data)
+{
+	return data->domain->host_data;
+}
+
+static inline int rzt2h_icu_irq_to_offset(struct irq_data *d, void __iomem **base,
+					  unsigned int *offset)
+{
+	struct rzt2h_icu_priv *priv = irq_data_to_priv(d);
+	unsigned int hwirq = irqd_to_hwirq(d);
+
+	/*
+	 * Safety IRQs and SEI use a separate register space from the non-safety IRQs.
+	 * SEI interrupt number follows immediately after the safety IRQs.
+	 */
+	if (RZT2H_ICU_IRQ_IN_RANGE(hwirq, IRQ_NS)) {
+		*offset = hwirq - RZT2H_ICU_IRQ_NS_START;
+		*base = priv->base_ns;
+	} else if (RZT2H_ICU_IRQ_IN_RANGE(hwirq, IRQ_S) || RZT2H_ICU_IRQ_IN_RANGE(hwirq, SEI)) {
+		*offset = hwirq - RZT2H_ICU_IRQ_S_START;
+		*base = priv->base_s;
+	} else {
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int rzt2h_icu_irq_set_type(struct irq_data *d, unsigned int type)
+{
+	struct rzt2h_icu_priv *priv = irq_data_to_priv(d);
+	unsigned int offset, parent_type;
+	void __iomem *base;
+	u32 val, md;
+	int ret;
+
+	ret = rzt2h_icu_irq_to_offset(d, &base, &offset);
+	if (ret)
+		return ret;
+
+	switch (type & IRQ_TYPE_SENSE_MASK) {
+	case IRQ_TYPE_LEVEL_LOW:
+		md = RZT2H_ICU_MD_LOW_LEVEL;
+		parent_type = IRQ_TYPE_LEVEL_HIGH;
+		break;
+	case IRQ_TYPE_EDGE_FALLING:
+		md = RZT2H_ICU_MD_FALLING_EDGE;
+		parent_type = IRQ_TYPE_EDGE_RISING;
+		break;
+	case IRQ_TYPE_EDGE_RISING:
+		md = RZT2H_ICU_MD_RISING_EDGE;
+		parent_type = IRQ_TYPE_EDGE_RISING;
+		break;
+	case IRQ_TYPE_EDGE_BOTH:
+		md = RZT2H_ICU_MD_BOTH_EDGES;
+		parent_type = IRQ_TYPE_EDGE_RISING;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	scoped_guard(raw_spinlock, &priv->lock) {
+		val = readl_relaxed(base + RZT2H_ICU_PORTNF_MD);
+		val &= ~RZT2H_ICU_PORTNF_MDi_MASK(offset);
+		val |= RZT2H_ICU_PORTNF_MDi_PREP(offset, md);
+		writel_relaxed(val, base + RZT2H_ICU_PORTNF_MD);
+	}
+
+	return irq_chip_set_type_parent(d, parent_type);
+}
+
+static int rzt2h_icu_set_type(struct irq_data *d, unsigned int type)
+{
+	unsigned int hw_irq = irqd_to_hwirq(d);
+
+	/* IRQn and SEI are selectable, others are edge-only. */
+	if (RZT2H_ICU_IRQ_IN_RANGE(hw_irq, IRQ_NS) ||
+	    RZT2H_ICU_IRQ_IN_RANGE(hw_irq, IRQ_S) ||
+	    RZT2H_ICU_IRQ_IN_RANGE(hw_irq, SEI))
+		return rzt2h_icu_irq_set_type(d, type);
+
+	if ((type & IRQ_TYPE_SENSE_MASK) != IRQ_TYPE_EDGE_RISING)
+		return -EINVAL;
+
+	return irq_chip_set_type_parent(d, IRQ_TYPE_EDGE_RISING);
+}
+
+static const struct irq_chip rzt2h_icu_chip = {
+	.name			= "rzt2h-icu",
+	.irq_mask		= irq_chip_mask_parent,
+	.irq_unmask		= irq_chip_unmask_parent,
+	.irq_eoi		= irq_chip_eoi_parent,
+	.irq_set_type		= rzt2h_icu_set_type,
+	.irq_set_wake		= irq_chip_set_wake_parent,
+	.irq_set_affinity	= irq_chip_set_affinity_parent,
+	.irq_retrigger		= irq_chip_retrigger_hierarchy,
+	.irq_get_irqchip_state	= irq_chip_get_parent_state,
+	.irq_set_irqchip_state	= irq_chip_set_parent_state,
+	.flags			= IRQCHIP_MASK_ON_SUSPEND |
+				  IRQCHIP_SET_TYPE_MASKED |
+				  IRQCHIP_SKIP_SET_WAKE,
+};
+
+static int rzt2h_icu_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs,
+			   void *arg)
+{
+	struct rzt2h_icu_priv *priv = domain->host_data;
+	irq_hw_number_t hwirq;
+	unsigned int type;
+	int ret;
+
+	ret = irq_domain_translate_twocell(domain, arg, &hwirq, &type);
+	if (ret)
+		return ret;
+
+	ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq, &rzt2h_icu_chip, NULL);
+	if (ret)
+		return ret;
+
+	return irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &priv->fwspec[hwirq]);
+}
+
+static const struct irq_domain_ops rzt2h_icu_domain_ops = {
+	.alloc		= rzt2h_icu_alloc,
+	.free		= irq_domain_free_irqs_common,
+	.translate	= irq_domain_translate_twocell,
+};
+
+static int rzt2h_icu_parse_interrupts(struct rzt2h_icu_priv *priv, struct device_node *np)
+{
+	struct of_phandle_args map;
+	unsigned int i;
+	int ret;
+
+	for (i = 0; i < RZT2H_ICU_NUM_IRQ; i++) {
+		ret = of_irq_parse_one(np, i, &map);
+		if (ret)
+			return ret;
+
+		of_phandle_args_to_fwspec(np, map.args, map.args_count, &priv->fwspec[i]);
+	}
+
+	return 0;
+}
+
+static int rzt2h_icu_init(struct platform_device *pdev, struct device_node *parent)
+{
+	struct irq_domain *irq_domain, *parent_domain;
+	struct device_node *node = pdev->dev.of_node;
+	struct device *dev = &pdev->dev;
+	struct rzt2h_icu_priv *priv;
+	int ret;
+
+	parent_domain = irq_find_host(parent);
+	if (!parent_domain)
+		return dev_err_probe(dev, -ENODEV, "cannot find parent domain\n");
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	raw_spin_lock_init(&priv->lock);
+
+	platform_set_drvdata(pdev, priv);
+
+	priv->base_ns = devm_of_iomap(dev, dev->of_node, 0, NULL);
+	if (IS_ERR(priv->base_ns))
+		return PTR_ERR(priv->base_ns);
+
+	priv->base_s = devm_of_iomap(dev, dev->of_node, 1, NULL);
+	if (IS_ERR(priv->base_s))
+		return PTR_ERR(priv->base_s);
+
+	ret = rzt2h_icu_parse_interrupts(priv, node);
+	if (ret)
+		return dev_err_probe(dev, ret, "cannot parse interrupts: %d\n", ret);
+
+	ret = devm_pm_runtime_enable(dev);
+	if (ret)
+		return dev_err_probe(dev, ret, "devm_pm_runtime_enable failed: %d\n", ret);
+
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret)
+		return dev_err_probe(dev, ret, "pm_runtime_resume_and_get failed: %d\n", ret);
+
+	irq_domain = irq_domain_create_hierarchy(parent_domain, 0, RZT2H_ICU_NUM_IRQ,
+						 dev_fwnode(dev), &rzt2h_icu_domain_ops, priv);
+	if (!irq_domain) {
+		pm_runtime_put(dev);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+IRQCHIP_PLATFORM_DRIVER_BEGIN(rzt2h_icu)
+IRQCHIP_MATCH("renesas,r9a09g077-icu", rzt2h_icu_init)
+IRQCHIP_PLATFORM_DRIVER_END(rzt2h_icu)
+MODULE_AUTHOR("Cosmin Tanislav <cosmin-gabriel.tanislav.xa@renesas.com>");
+MODULE_DESCRIPTION("Renesas RZ/T2H ICU Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/irqchip/irq-renesas-rzv2h.c b/drivers/irqchip/irq-renesas-rzv2h.c
index 899a423b5da8..0c44b6109842 100644
--- a/drivers/irqchip/irq-renesas-rzv2h.c
+++ b/drivers/irqchip/irq-renesas-rzv2h.c
@@ -616,6 +616,7 @@ static int rzv2h_icu_probe(struct platform_device *pdev, struct device_node *par
 
 IRQCHIP_PLATFORM_DRIVER_BEGIN(rzv2h_icu)
 IRQCHIP_MATCH("renesas,r9a09g047-icu", rzg3e_icu_probe)
+IRQCHIP_MATCH("renesas,r9a09g056-icu", rzv2h_icu_probe)
 IRQCHIP_MATCH("renesas,r9a09g057-icu", rzv2h_icu_probe)
 IRQCHIP_PLATFORM_DRIVER_END(rzv2h_icu)
 MODULE_AUTHOR("Fabrizio Castro <fabrizio.castro.jz@renesas.com>");
diff --git a/drivers/irqchip/irq-riscv-aplic-direct.c b/drivers/irqchip/irq-riscv-aplic-direct.c
index c2a75bf3d20c..5a9650225dd8 100644
--- a/drivers/irqchip/irq-riscv-aplic-direct.c
+++ b/drivers/irqchip/irq-riscv-aplic-direct.c
@@ -8,6 +8,7 @@
 #include <linux/bitfield.h>
 #include <linux/bitops.h>
 #include <linux/cpu.h>
+#include <linux/cpumask.h>
 #include <linux/interrupt.h>
 #include <linux/irqchip.h>
 #include <linux/irqchip/chained_irq.h>
@@ -171,6 +172,15 @@ static void aplic_idc_set_delivery(struct aplic_idc *idc, bool en)
 	writel(de, idc->regs + APLIC_IDC_IDELIVERY);
 }
 
+void aplic_direct_restore_states(struct aplic_priv *priv)
+{
+	struct aplic_direct *direct = container_of(priv, struct aplic_direct, priv);
+	int cpu;
+
+	for_each_cpu(cpu, &direct->lmask)
+		aplic_idc_set_delivery(per_cpu_ptr(&aplic_idcs, cpu), true);
+}
+
 static int aplic_direct_dying_cpu(unsigned int cpu)
 {
 	if (aplic_direct_parent_irq)
diff --git a/drivers/irqchip/irq-riscv-aplic-main.c b/drivers/irqchip/irq-riscv-aplic-main.c
index 93e7c51f944a..b760942e57f9 100644
--- a/drivers/irqchip/irq-riscv-aplic-main.c
+++ b/drivers/irqchip/irq-riscv-aplic-main.c
@@ -12,10 +12,165 @@
 #include <linux/of.h>
 #include <linux/of_irq.h>
 #include <linux/platform_device.h>
+#include <linux/pm_domain.h>
+#include <linux/pm_runtime.h>
 #include <linux/printk.h>
+#include <linux/syscore_ops.h>
 
 #include "irq-riscv-aplic-main.h"
 
+static LIST_HEAD(aplics);
+
+static void aplic_restore_states(struct aplic_priv *priv)
+{
+	struct aplic_saved_regs *saved_regs = &priv->saved_hw_regs;
+	struct aplic_src_ctrl *srcs;
+	void __iomem *regs;
+	u32 nr_irqs, i;
+
+	regs = priv->regs;
+	writel(saved_regs->domaincfg, regs + APLIC_DOMAINCFG);
+#ifdef CONFIG_RISCV_M_MODE
+	writel(saved_regs->msiaddr, regs + APLIC_xMSICFGADDR);
+	writel(saved_regs->msiaddrh, regs + APLIC_xMSICFGADDRH);
+#endif
+	/*
+	 * The sourcecfg[i] has to be restored prior to the target[i], interrupt-pending and
+	 * interrupt-enable bits. The AIA specification states that "Whenever interrupt source i is
+	 * inactive in an interrupt domain, the corresponding interrupt-pending and interrupt-enable
+	 * bits within the domain are read-only zeros, and register target[i] is also read-only
+	 * zero."
+	 */
+	nr_irqs = priv->nr_irqs;
+	for (i = 0; i < nr_irqs; i++) {
+		srcs = &priv->saved_hw_regs.srcs[i];
+		writel(srcs->sourcecfg, regs + APLIC_SOURCECFG_BASE + i * sizeof(u32));
+		writel(srcs->target, regs + APLIC_TARGET_BASE + i * sizeof(u32));
+	}
+
+	for (i = 0; i <= nr_irqs; i += 32) {
+		srcs = &priv->saved_hw_regs.srcs[i];
+		writel(-1U, regs + APLIC_CLRIE_BASE + (i / 32) * sizeof(u32));
+		writel(srcs->ie, regs + APLIC_SETIE_BASE + (i / 32) * sizeof(u32));
+
+		/* Re-trigger the interrupts if it forwards interrupts to target harts by MSIs */
+		if (!priv->nr_idcs)
+			writel(readl(regs + APLIC_CLRIP_BASE + (i / 32) * sizeof(u32)),
+			       regs + APLIC_SETIP_BASE + (i / 32) * sizeof(u32));
+	}
+
+	if (priv->nr_idcs)
+		aplic_direct_restore_states(priv);
+}
+
+static void aplic_save_states(struct aplic_priv *priv)
+{
+	struct aplic_src_ctrl *srcs;
+	void __iomem *regs;
+	u32 i, nr_irqs;
+
+	regs = priv->regs;
+	nr_irqs = priv->nr_irqs;
+	/* The valid interrupt source IDs range from 1 to N, where N is priv->nr_irqs */
+	for (i = 0; i < nr_irqs; i++) {
+		srcs = &priv->saved_hw_regs.srcs[i];
+		srcs->target = readl(regs + APLIC_TARGET_BASE + i * sizeof(u32));
+
+		if (i % 32)
+			continue;
+
+		srcs->ie = readl(regs + APLIC_SETIE_BASE + (i / 32) * sizeof(u32));
+	}
+
+	/* Save the nr_irqs bit if needed */
+	if (!(nr_irqs % 32)) {
+		srcs = &priv->saved_hw_regs.srcs[nr_irqs];
+		srcs->ie = readl(regs + APLIC_SETIE_BASE + (nr_irqs / 32) * sizeof(u32));
+	}
+}
+
+static int aplic_syscore_suspend(void)
+{
+	struct aplic_priv *priv;
+
+	list_for_each_entry(priv, &aplics, head)
+		aplic_save_states(priv);
+
+	return 0;
+}
+
+static void aplic_syscore_resume(void)
+{
+	struct aplic_priv *priv;
+
+	list_for_each_entry(priv, &aplics, head)
+		aplic_restore_states(priv);
+}
+
+static struct syscore_ops aplic_syscore_ops = {
+	.suspend = aplic_syscore_suspend,
+	.resume = aplic_syscore_resume,
+};
+
+static int aplic_pm_notifier(struct notifier_block *nb, unsigned long action, void *data)
+{
+	struct aplic_priv *priv = container_of(nb, struct aplic_priv, genpd_nb);
+
+	switch (action) {
+	case GENPD_NOTIFY_PRE_OFF:
+		aplic_save_states(priv);
+		break;
+	case GENPD_NOTIFY_ON:
+		aplic_restore_states(priv);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static void aplic_pm_remove(void *data)
+{
+	struct aplic_priv *priv = data;
+	struct device *dev = priv->dev;
+
+	list_del(&priv->head);
+	if (dev->pm_domain)
+		dev_pm_genpd_remove_notifier(dev);
+}
+
+static int aplic_pm_add(struct device *dev, struct aplic_priv *priv)
+{
+	struct aplic_src_ctrl *srcs;
+	int ret;
+
+	srcs = devm_kzalloc(dev, (priv->nr_irqs + 1) * sizeof(*srcs), GFP_KERNEL);
+	if (!srcs)
+		return -ENOMEM;
+
+	priv->saved_hw_regs.srcs = srcs;
+	list_add(&priv->head, &aplics);
+	if (dev->pm_domain) {
+		priv->genpd_nb.notifier_call = aplic_pm_notifier;
+		ret = dev_pm_genpd_add_notifier(dev, &priv->genpd_nb);
+		if (ret)
+			goto remove_head;
+
+		ret = devm_pm_runtime_enable(dev);
+		if (ret)
+			goto remove_notifier;
+	}
+
+	return devm_add_action_or_reset(dev, aplic_pm_remove, priv);
+
+remove_notifier:
+	dev_pm_genpd_remove_notifier(dev);
+remove_head:
+	list_del(&priv->head);
+	return ret;
+}
+
 void aplic_irq_unmask(struct irq_data *d)
 {
 	struct aplic_priv *priv = irq_data_get_irq_chip_data(d);
@@ -60,6 +215,8 @@ int aplic_irq_set_type(struct irq_data *d, unsigned int type)
 	sourcecfg += (d->hwirq - 1) * sizeof(u32);
 	writel(val, sourcecfg);
 
+	priv->saved_hw_regs.srcs[d->hwirq - 1].sourcecfg = val;
+
 	return 0;
 }
 
@@ -82,6 +239,7 @@ int aplic_irqdomain_translate(struct irq_fwspec *fwspec, u32 gsi_base,
 
 void aplic_init_hw_global(struct aplic_priv *priv, bool msi_mode)
 {
+	struct aplic_saved_regs *saved_regs = &priv->saved_hw_regs;
 	u32 val;
 #ifdef CONFIG_RISCV_M_MODE
 	u32 valh;
@@ -95,6 +253,8 @@ void aplic_init_hw_global(struct aplic_priv *priv, bool msi_mode)
 		valh |= FIELD_PREP(APLIC_xMSICFGADDRH_HHXS, priv->msicfg.hhxs);
 		writel(val, priv->regs + APLIC_xMSICFGADDR);
 		writel(valh, priv->regs + APLIC_xMSICFGADDRH);
+		saved_regs->msiaddr = val;
+		saved_regs->msiaddrh = valh;
 	}
 #endif
 
@@ -106,6 +266,8 @@ void aplic_init_hw_global(struct aplic_priv *priv, bool msi_mode)
 	writel(val, priv->regs + APLIC_DOMAINCFG);
 	if (readl(priv->regs + APLIC_DOMAINCFG) != val)
 		dev_warn(priv->dev, "unable to write 0x%x in domaincfg\n", val);
+
+	saved_regs->domaincfg = val;
 }
 
 static void aplic_init_hw_irqs(struct aplic_priv *priv)
@@ -176,7 +338,7 @@ int aplic_setup_priv(struct aplic_priv *priv, struct device *dev, void __iomem *
 	/* Setup initial state APLIC interrupts */
 	aplic_init_hw_irqs(priv);
 
-	return 0;
+	return aplic_pm_add(dev, priv);
 }
 
 static int aplic_probe(struct platform_device *pdev)
@@ -209,6 +371,8 @@ static int aplic_probe(struct platform_device *pdev)
 	if (rc)
 		dev_err_probe(dev, rc, "failed to setup APLIC in %s mode\n",
 			      msi_mode ? "MSI" : "direct");
+	else
+		register_syscore_ops(&aplic_syscore_ops);
 
 #ifdef CONFIG_ACPI
 	if (!acpi_disabled)
diff --git a/drivers/irqchip/irq-riscv-aplic-main.h b/drivers/irqchip/irq-riscv-aplic-main.h
index b0ad8cde69b1..2d8ad7138541 100644
--- a/drivers/irqchip/irq-riscv-aplic-main.h
+++ b/drivers/irqchip/irq-riscv-aplic-main.h
@@ -23,7 +23,25 @@ struct aplic_msicfg {
 	u32			lhxw;
 };
 
+struct aplic_src_ctrl {
+	u32			sourcecfg;
+	u32			target;
+	u32			ie;
+};
+
+struct aplic_saved_regs {
+	u32			domaincfg;
+#ifdef CONFIG_RISCV_M_MODE
+	u32			msiaddr;
+	u32			msiaddrh;
+#endif
+	struct aplic_src_ctrl	*srcs;
+};
+
 struct aplic_priv {
+	struct list_head	head;
+	struct notifier_block	genpd_nb;
+	struct aplic_saved_regs	saved_hw_regs;
 	struct device		*dev;
 	u32			gsi_base;
 	u32			nr_irqs;
@@ -40,6 +58,7 @@ int aplic_irqdomain_translate(struct irq_fwspec *fwspec, u32 gsi_base,
 			      unsigned long *hwirq, unsigned int *type);
 void aplic_init_hw_global(struct aplic_priv *priv, bool msi_mode);
 int aplic_setup_priv(struct aplic_priv *priv, struct device *dev, void __iomem *regs);
+void aplic_direct_restore_states(struct aplic_priv *priv);
 int aplic_direct_setup(struct device *dev, void __iomem *regs);
 #ifdef CONFIG_RISCV_APLIC_MSI
 int aplic_msi_setup(struct device *dev, void __iomem *regs);
diff --git a/drivers/irqchip/irq-riscv-imsic-early.c b/drivers/irqchip/irq-riscv-imsic-early.c
index 6bac67cc0b6d..ba903fa689bd 100644
--- a/drivers/irqchip/irq-riscv-imsic-early.c
+++ b/drivers/irqchip/irq-riscv-imsic-early.c
@@ -7,6 +7,7 @@
 #define pr_fmt(fmt) "riscv-imsic: " fmt
 #include <linux/acpi.h>
 #include <linux/cpu.h>
+#include <linux/cpu_pm.h>
 #include <linux/export.h>
 #include <linux/interrupt.h>
 #include <linux/init.h>
@@ -123,14 +124,8 @@ static void imsic_handle_irq(struct irq_desc *desc)
 	chained_irq_exit(chip, desc);
 }
 
-static int imsic_starting_cpu(unsigned int cpu)
+static void imsic_hw_states_init(void)
 {
-	/* Mark per-CPU IMSIC state as online */
-	imsic_state_online();
-
-	/* Enable per-CPU parent interrupt */
-	enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq));
-
 	/* Setup IPIs */
 	imsic_ipi_starting_cpu();
 
@@ -142,6 +137,18 @@ static int imsic_starting_cpu(unsigned int cpu)
 
 	/* Enable local interrupt delivery */
 	imsic_local_delivery(true);
+}
+
+static int imsic_starting_cpu(unsigned int cpu)
+{
+	/* Mark per-CPU IMSIC state as online */
+	imsic_state_online();
+
+	/* Enable per-CPU parent interrupt */
+	enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq));
+
+	/* Initialize the IMSIC registers to enable the interrupt delivery */
+	imsic_hw_states_init();
 
 	return 0;
 }
@@ -157,6 +164,22 @@ static int imsic_dying_cpu(unsigned int cpu)
 	return 0;
 }
 
+static int imsic_pm_notifier(struct notifier_block *self, unsigned long cmd, void *v)
+{
+	switch (cmd) {
+	case CPU_PM_EXIT:
+		/* Initialize the IMSIC registers to enable the interrupt delivery */
+		imsic_hw_states_init();
+		break;
+	}
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block imsic_pm_notifier_block = {
+	.notifier_call = imsic_pm_notifier,
+};
+
 static int __init imsic_early_probe(struct fwnode_handle *fwnode)
 {
 	struct irq_domain *domain;
@@ -194,7 +217,7 @@ static int __init imsic_early_probe(struct fwnode_handle *fwnode)
 	cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_IMSIC_STARTING, "irqchip/riscv/imsic:starting",
 			  imsic_starting_cpu, imsic_dying_cpu);
 
-	return 0;
+	return cpu_pm_register_notifier(&imsic_pm_notifier_block);
 }
 
 static int __init imsic_early_dt_init(struct device_node *node, struct device_node *parent)
diff --git a/drivers/mailbox/Kconfig b/drivers/mailbox/Kconfig
index 29f16f220384..60fbea596a39 100644
--- a/drivers/mailbox/Kconfig
+++ b/drivers/mailbox/Kconfig
@@ -199,7 +199,7 @@ config POLARFIRE_SOC_MAILBOX
 	tristate "PolarFire SoC (MPFS) Mailbox"
 	depends on HAS_IOMEM
 	depends on MFD_SYSCON
-	depends on ARCH_MICROCHIP_POLARFIRE || COMPILE_TEST
+	depends on ARCH_MICROCHIP || COMPILE_TEST
 	help
 	  This driver adds support for the PolarFire SoC (MPFS) mailbox controller.
 
@@ -279,7 +279,7 @@ config MTK_ADSP_MBOX
 	tristate "MediaTek ADSP Mailbox Controller"
 	depends on ARCH_MEDIATEK || COMPILE_TEST
 	help
-          Say yes here to add support for "MediaTek ADSP Mailbox Controller.
+          Say yes here to add support for MediaTek ADSP Mailbox Controller.
           This mailbox driver is used to send notification or short message
           between processors with ADSP. It will place the message to share
 	  buffer and will access the ipc control.
@@ -304,6 +304,15 @@ config MTK_GPUEB_MBOX
 	  Say Y or m here if you want to support the MT8196 SoC in your kernel
 	  build.
 
+config MTK_VCP_MBOX
+	tristate "MediaTek VCP Mailbox Support"
+	depends on ARCH_MEDIATEK || COMPILE_TEST
+	help
+	  Say yes here to add support for the MediaTek VCP mailbox driver.
+	  The mailbox implementation provides access from the application
+	  processor to Video Companion Processor Unit.
+	  If unsure say N.
+
 config ZYNQMP_IPI_MBOX
 	tristate "Xilinx ZynqMP IPI Mailbox"
 	depends on ARCH_ZYNQMP && OF
@@ -379,6 +388,15 @@ config BCM74110_MAILBOX
 	  processor and coprocessor that handles various power management task
 	  and more.
 
+config RCAR_MFIS_MBOX
+	bool "Renesas R-Car Multifunctional Interface Mailbox Support"
+	depends on (ARM_SCMI_PROTOCOL && ARCH_RENESAS) || COMPILE_TEST
+	help
+	  This driver provides support for mailboxes of the MFIS
+	  (Multifunctional Interface) via the SCMI interface.
+	  It is used to send short message between different domains
+	  like AP, RT, and SCP.
+
 config RISCV_SBI_MPXY_MBOX
 	tristate "RISC-V SBI Message Proxy (MPXY) Mailbox"
 	depends on RISCV_SBI
diff --git a/drivers/mailbox/Makefile b/drivers/mailbox/Makefile
index 81820a4f5528..d41016af86c9 100644
--- a/drivers/mailbox/Makefile
+++ b/drivers/mailbox/Makefile
@@ -27,6 +27,8 @@ obj-$(CONFIG_PL320_MBOX)	+= pl320-ipc.o
 
 obj-$(CONFIG_OMAP2PLUS_MBOX)	+= omap-mailbox.o
 
+obj-$(CONFIG_RCAR_MFIS_MBOX)	+= rcar-mfis-mailbox.o
+
 obj-$(CONFIG_ROCKCHIP_MBOX)	+= rockchip-mailbox.o
 
 obj-$(CONFIG_PCC)		+= pcc.o
@@ -65,6 +67,8 @@ obj-$(CONFIG_MTK_CMDQ_MBOX)	+= mtk-cmdq-mailbox.o
 
 obj-$(CONFIG_MTK_GPUEB_MBOX)	+= mtk-gpueb-mailbox.o
 
+obj-$(CONFIG_MTK_VCP_MBOX)	+= mtk-vcp-mailbox.o
+
 obj-$(CONFIG_ZYNQMP_IPI_MBOX)	+= zynqmp-ipi-mailbox.o
 
 obj-$(CONFIG_SUN6I_MSGBOX)	+= sun6i-msgbox.o
diff --git a/drivers/mailbox/mailbox-mchp-ipc-sbi.c b/drivers/mailbox/mailbox-mchp-ipc-sbi.c
index a6e52009a424..d444491a584e 100644
--- a/drivers/mailbox/mailbox-mchp-ipc-sbi.c
+++ b/drivers/mailbox/mailbox-mchp-ipc-sbi.c
@@ -180,20 +180,20 @@ static irqreturn_t mchp_ipc_cluster_aggr_isr(int irq, void *data)
 	/* Find out the hart that originated the irq */
 	for_each_online_cpu(i) {
 		hartid = cpuid_to_hartid_map(i);
-		if (irq == ipc->cluster_cfg[hartid].irq)
+		if (irq == ipc->cluster_cfg[i].irq)
 			break;
 	}
 
 	status_msg.cluster = hartid;
-	memcpy(ipc->cluster_cfg[hartid].buf_base, &status_msg, sizeof(struct mchp_ipc_status));
+	memcpy(ipc->cluster_cfg[i].buf_base, &status_msg, sizeof(struct mchp_ipc_status));
 
-	ret = mchp_ipc_sbi_send(SBI_EXT_IPC_STATUS, ipc->cluster_cfg[hartid].buf_base_addr);
+	ret = mchp_ipc_sbi_send(SBI_EXT_IPC_STATUS, ipc->cluster_cfg[i].buf_base_addr);
 	if (ret < 0) {
 		dev_err_ratelimited(ipc->dev, "could not get IHC irq status ret=%d\n", ret);
 		return IRQ_HANDLED;
 	}
 
-	memcpy(&status_msg, ipc->cluster_cfg[hartid].buf_base, sizeof(struct mchp_ipc_status));
+	memcpy(&status_msg, ipc->cluster_cfg[i].buf_base, sizeof(struct mchp_ipc_status));
 
 	/*
 	 * Iterate over each bit set in the IHC interrupt status register (IRQ_STATUS) to identify
@@ -385,21 +385,21 @@ static int mchp_ipc_get_cluster_aggr_irq(struct mchp_ipc_sbi_mbox *ipc)
 		if (ret <= 0)
 			continue;
 
-		ipc->cluster_cfg[hartid].irq = ret;
-		ret = devm_request_irq(ipc->dev, ipc->cluster_cfg[hartid].irq,
+		ipc->cluster_cfg[cpuid].irq = ret;
+		ret = devm_request_irq(ipc->dev, ipc->cluster_cfg[cpuid].irq,
 				       mchp_ipc_cluster_aggr_isr, IRQF_SHARED,
 				       "miv-ihc-irq", ipc);
 		if (ret)
 			return ret;
 
-		ipc->cluster_cfg[hartid].buf_base = devm_kmalloc(ipc->dev,
-								 sizeof(struct mchp_ipc_status),
-								 GFP_KERNEL);
+		ipc->cluster_cfg[cpuid].buf_base = devm_kmalloc(ipc->dev,
+								sizeof(struct mchp_ipc_status),
+								GFP_KERNEL);
 
-		if (!ipc->cluster_cfg[hartid].buf_base)
+		if (!ipc->cluster_cfg[cpuid].buf_base)
 			return -ENOMEM;
 
-		ipc->cluster_cfg[hartid].buf_base_addr = __pa(ipc->cluster_cfg[hartid].buf_base);
+		ipc->cluster_cfg[cpuid].buf_base_addr = __pa(ipc->cluster_cfg[cpuid].buf_base);
 
 		irq_found = true;
 	}
diff --git a/drivers/mailbox/mailbox.c b/drivers/mailbox/mailbox.c
index 2acc6ec229a4..617ba505691d 100644
--- a/drivers/mailbox/mailbox.c
+++ b/drivers/mailbox/mailbox.c
@@ -489,12 +489,10 @@ EXPORT_SYMBOL_GPL(mbox_free_channel);
 static struct mbox_chan *fw_mbox_index_xlate(struct mbox_controller *mbox,
 					     const struct fwnode_reference_args *sp)
 {
-	int ind = sp->args[0];
-
-	if (ind >= mbox->num_chans)
+	if (sp->nargs < 1 || sp->args[0] >= mbox->num_chans)
 		return ERR_PTR(-EINVAL);
 
-	return &mbox->chans[ind];
+	return &mbox->chans[sp->args[0]];
 }
 
 /**
diff --git a/drivers/mailbox/mtk-cmdq-mailbox.c b/drivers/mailbox/mtk-cmdq-mailbox.c
index 5791f80f995a..1bf6984948ef 100644
--- a/drivers/mailbox/mtk-cmdq-mailbox.c
+++ b/drivers/mailbox/mtk-cmdq-mailbox.c
@@ -14,6 +14,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/sizes.h>
 #include <linux/mailbox_controller.h>
 #include <linux/mailbox/mtk-cmdq-mailbox.h>
 #include <linux/of.h>
@@ -43,6 +44,13 @@
 #define GCE_CTRL_BY_SW				GENMASK(2, 0)
 #define GCE_DDR_EN				GENMASK(18, 16)
 
+#define GCE_VM_ID_MAP(n)		(0x5018 + (n) / 10 * 4)
+#define GCE_VM_ID_MAP_THR_FLD_SHIFT(n)		((n) % 10 * 3)
+#define GCE_VM_ID_MAP_HOST_VM			GENMASK(2, 0)
+#define GCE_VM_CPR_GSIZE		0x50c4
+#define GCE_VM_CPR_GSIZE_FLD_SHIFT(vm_id)	((vm_id) * 4)
+#define GCE_VM_CPR_GSIZE_MAX			GENMASK(3, 0)
+
 #define CMDQ_THR_ACTIVE_SLOT_CYCLES	0x3200
 #define CMDQ_THR_ENABLED		0x1
 #define CMDQ_THR_DISABLED		0x0
@@ -87,22 +95,33 @@ struct cmdq {
 struct gce_plat {
 	u32 thread_nr;
 	u8 shift;
+	dma_addr_t mminfra_offset;
 	bool control_by_sw;
 	bool sw_ddr_en;
+	bool gce_vm;
 	u32 gce_num;
 };
 
 static inline u32 cmdq_convert_gce_addr(dma_addr_t addr, const struct gce_plat *pdata)
 {
 	/* Convert DMA addr (PA or IOVA) to GCE readable addr */
-	return addr >> pdata->shift;
+	return (addr + pdata->mminfra_offset) >> pdata->shift;
 }
 
 static inline dma_addr_t cmdq_revert_gce_addr(u32 addr, const struct gce_plat *pdata)
 {
 	/* Revert GCE readable addr to DMA addr (PA or IOVA) */
-	return (dma_addr_t)addr << pdata->shift;
+	return ((dma_addr_t)addr << pdata->shift) - pdata->mminfra_offset;
+}
+
+void cmdq_get_mbox_priv(struct mbox_chan *chan, struct cmdq_mbox_priv *priv)
+{
+	struct cmdq *cmdq = container_of(chan->mbox, struct cmdq, mbox);
+
+	priv->shift_pa = cmdq->pdata->shift;
+	priv->mminfra_offset = cmdq->pdata->mminfra_offset;
 }
+EXPORT_SYMBOL(cmdq_get_mbox_priv);
 
 u8 cmdq_get_shift_pa(struct mbox_chan *chan)
 {
@@ -112,6 +131,45 @@ u8 cmdq_get_shift_pa(struct mbox_chan *chan)
 }
 EXPORT_SYMBOL(cmdq_get_shift_pa);
 
+static void cmdq_vm_init(struct cmdq *cmdq)
+{
+	int i;
+	u32 vm_cpr_gsize = 0, vm_id_map = 0;
+	u32 *vm_map = NULL;
+
+	if (!cmdq->pdata->gce_vm)
+		return;
+
+	vm_map = kcalloc(cmdq->pdata->thread_nr, sizeof(*vm_map), GFP_KERNEL);
+	if (!vm_map)
+		return;
+
+	/* only configure the max CPR SRAM size to host vm (vm_id = 0) currently */
+	vm_cpr_gsize = GCE_VM_CPR_GSIZE_MAX << GCE_VM_CPR_GSIZE_FLD_SHIFT(0);
+
+	/* set all thread mapping to host vm currently */
+	for (i = 0; i < cmdq->pdata->thread_nr; i++)
+		vm_map[i] = GCE_VM_ID_MAP_HOST_VM << GCE_VM_ID_MAP_THR_FLD_SHIFT(i);
+
+	/* set the amount of CPR SRAM to allocate to each VM */
+	writel(vm_cpr_gsize, cmdq->base + GCE_VM_CPR_GSIZE);
+
+	/* config CPR_GSIZE before setting VM_ID_MAP to avoid data leakage */
+	for (i = 0; i < cmdq->pdata->thread_nr; i++) {
+		vm_id_map |= vm_map[i];
+		/* config every 10 threads, e.g., thread id=0~9, 10~19, ..., into one register */
+		if ((i + 1) % 10 == 0) {
+			writel(vm_id_map, cmdq->base + GCE_VM_ID_MAP(i));
+			vm_id_map = 0;
+		}
+	}
+	/* config remaining threads settings */
+	if (cmdq->pdata->thread_nr % 10 != 0)
+		writel(vm_id_map, cmdq->base + GCE_VM_ID_MAP(cmdq->pdata->thread_nr - 1));
+
+	kfree(vm_map);
+}
+
 static void cmdq_gctl_value_toggle(struct cmdq *cmdq, bool ddr_enable)
 {
 	u32 val = cmdq->pdata->control_by_sw ? GCE_CTRL_BY_SW : 0;
@@ -156,6 +214,7 @@ static void cmdq_init(struct cmdq *cmdq)
 
 	WARN_ON(clk_bulk_enable(cmdq->pdata->gce_num, cmdq->clocks));
 
+	cmdq_vm_init(cmdq);
 	cmdq_gctl_value_toggle(cmdq, true);
 
 	writel(CMDQ_THR_ACTIVE_SLOT_CYCLES, cmdq->base + CMDQ_THR_SLOT_CYCLES);
@@ -782,6 +841,16 @@ static const struct gce_plat gce_plat_mt8195 = {
 	.gce_num = 2
 };
 
+static const struct gce_plat gce_plat_mt8196 = {
+	.thread_nr = 32,
+	.shift = 3,
+	.mminfra_offset = SZ_2G,
+	.control_by_sw = true,
+	.sw_ddr_en = true,
+	.gce_vm = true,
+	.gce_num = 2
+};
+
 static const struct of_device_id cmdq_of_ids[] = {
 	{.compatible = "mediatek,mt6779-gce", .data = (void *)&gce_plat_mt6779},
 	{.compatible = "mediatek,mt8173-gce", .data = (void *)&gce_plat_mt8173},
@@ -790,6 +859,7 @@ static const struct of_device_id cmdq_of_ids[] = {
 	{.compatible = "mediatek,mt8188-gce", .data = (void *)&gce_plat_mt8188},
 	{.compatible = "mediatek,mt8192-gce", .data = (void *)&gce_plat_mt8192},
 	{.compatible = "mediatek,mt8195-gce", .data = (void *)&gce_plat_mt8195},
+	{.compatible = "mediatek,mt8196-gce", .data = (void *)&gce_plat_mt8196},
 	{}
 };
 MODULE_DEVICE_TABLE(of, cmdq_of_ids);
diff --git a/drivers/mailbox/mtk-vcp-mailbox.c b/drivers/mailbox/mtk-vcp-mailbox.c
new file mode 100644
index 000000000000..cedad575528f
--- /dev/null
+++ b/drivers/mailbox/mtk-vcp-mailbox.c
@@ -0,0 +1,170 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2025 MediaTek Corporation. All rights reserved.
+ * Author: Jjian Zhou <jjian.zhou.@mediatek.com>
+ */
+
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/mailbox_controller.h>
+#include <linux/mailbox/mtk-vcp-mailbox.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+struct mtk_vcp_mbox {
+	struct mbox_controller mbox;
+	void __iomem *base;
+	struct device *dev;
+	const struct mtk_vcp_mbox_cfg *cfg;
+	struct mtk_ipi_info ipi_recv;
+	struct mbox_chan chans;
+};
+
+struct mtk_vcp_mbox_cfg {
+	u16 set_in;
+	u16 clr_out;
+};
+
+static irqreturn_t mtk_vcp_mbox_irq_thread(int irq, void *data)
+{
+	struct mtk_vcp_mbox *priv = data;
+
+	/* get irq status */
+	priv->ipi_recv.irq_status = readl(priv->base + priv->cfg->clr_out);
+
+	__ioread32_copy(priv->ipi_recv.msg, priv->base,
+			MTK_VCP_MBOX_SLOT_MAX_SIZE / 4);
+
+	mbox_chan_received_data(&priv->chans, &priv->ipi_recv);
+
+	/* clear irq status */
+	writel(priv->ipi_recv.irq_status, priv->base + priv->cfg->clr_out);
+
+	return IRQ_HANDLED;
+}
+
+static struct mbox_chan *mtk_vcp_mbox_xlate(struct mbox_controller *mbox,
+					    const struct of_phandle_args *sp)
+{
+	if (sp->args_count)
+		return NULL;
+
+	return &mbox->chans[0];
+}
+
+static int mtk_vcp_mbox_send_data(struct mbox_chan *chan, void *data)
+{
+	struct mtk_vcp_mbox *priv = chan->con_priv;
+	struct mtk_ipi_info *ipi_info = data;
+	u32 status;
+
+	if (!ipi_info->msg) {
+		dev_err(priv->dev, "msg buffer is NULL.\n");
+		return -EINVAL;
+	}
+
+	status = readl(priv->base + priv->cfg->set_in);
+	if (status & BIT(ipi_info->index)) {
+		dev_warn(priv->dev, "mailbox IPI %d is busy.\n", ipi_info->id);
+		return -EBUSY;
+	}
+
+	if (ipi_info->slot_ofs + ipi_info->len > MTK_VCP_MBOX_SLOT_MAX_SIZE)
+		return -EINVAL;
+	__iowrite32_copy(priv->base + ipi_info->slot_ofs, ipi_info->msg,
+			 ipi_info->len);
+
+	writel(BIT(ipi_info->index), priv->base + priv->cfg->set_in);
+
+	return 0;
+}
+
+static bool mtk_vcp_mbox_last_tx_done(struct mbox_chan *chan)
+{
+	struct mtk_ipi_info *ipi_info = chan->active_req;
+	struct mtk_vcp_mbox *priv = chan->con_priv;
+
+	return !(readl(priv->base + priv->cfg->set_in) & BIT(ipi_info->index));
+}
+
+static const struct mbox_chan_ops mtk_vcp_mbox_chan_ops = {
+	.send_data	= mtk_vcp_mbox_send_data,
+	.last_tx_done	= mtk_vcp_mbox_last_tx_done,
+};
+
+static int mtk_vcp_mbox_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct mtk_vcp_mbox *priv;
+	struct mbox_controller *mbox;
+	int ret, irq;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = dev;
+	priv->chans.con_priv = priv;
+	mbox = &priv->mbox;
+	mbox->dev = dev;
+	mbox->ops = &mtk_vcp_mbox_chan_ops;
+	mbox->txdone_irq = false;
+	mbox->txdone_poll = true;
+	mbox->of_xlate = mtk_vcp_mbox_xlate;
+	mbox->num_chans = 1;
+	mbox->chans = &priv->chans;
+
+	priv->ipi_recv.msg = devm_kzalloc(dev, MTK_VCP_MBOX_SLOT_MAX_SIZE,
+					  GFP_KERNEL);
+	if (!priv->ipi_recv.msg)
+		return -ENOMEM;
+
+	priv->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(priv->base))
+		return PTR_ERR(priv->base);
+
+	priv->cfg = of_device_get_match_data(dev);
+	if (!priv->cfg)
+		return -EINVAL;
+
+	platform_set_drvdata(pdev, priv);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	ret = devm_request_threaded_irq(dev, irq, NULL,
+					mtk_vcp_mbox_irq_thread, IRQF_ONESHOT,
+					dev_name(dev), priv);
+	if (ret < 0)
+		return ret;
+
+	return devm_mbox_controller_register(dev, &priv->mbox);
+}
+
+static const struct mtk_vcp_mbox_cfg mt8196_cfg = {
+	.set_in		= 0x100,
+	.clr_out	= 0x10c,
+};
+
+static const struct of_device_id mtk_vcp_mbox_of_match[] = {
+	{ .compatible = "mediatek,mt8196-vcp-mbox", .data = &mt8196_cfg },
+	{},
+};
+MODULE_DEVICE_TABLE(of, mtk_vcp_mbox_of_match);
+
+static struct platform_driver mtk_vcp_mbox_driver = {
+	.probe		= mtk_vcp_mbox_probe,
+	.driver = {
+		.name	= "mtk_vcp_mbox",
+		.of_match_table = mtk_vcp_mbox_of_match,
+	},
+};
+module_platform_driver(mtk_vcp_mbox_driver);
+
+MODULE_AUTHOR("Jjian Zhou <jjian.zhou@mediatek.com>");
+MODULE_DESCRIPTION("MTK VCP Mailbox Controller");
+MODULE_LICENSE("GPL");
diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c
index ff292b9e0be9..967eb249f40d 100644
--- a/drivers/mailbox/pcc.c
+++ b/drivers/mailbox/pcc.c
@@ -305,22 +305,6 @@ static void pcc_chan_acknowledge(struct pcc_chan_info *pchan)
 		pcc_chan_reg_read_modify_write(&pchan->db);
 }
 
-static void *write_response(struct pcc_chan_info *pchan)
-{
-	struct pcc_header pcc_header;
-	void *buffer;
-	int data_len;
-
-	memcpy_fromio(&pcc_header, pchan->chan.shmem,
-		      sizeof(pcc_header));
-	data_len = pcc_header.length - sizeof(u32) + sizeof(struct pcc_header);
-
-	buffer = pchan->chan.rx_alloc(pchan->chan.mchan->cl, data_len);
-	if (buffer != NULL)
-		memcpy_fromio(buffer, pchan->chan.shmem, data_len);
-	return buffer;
-}
-
 /**
  * pcc_mbox_irq - PCC mailbox interrupt handler
  * @irq:	interrupt number
@@ -332,8 +316,6 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p)
 {
 	struct pcc_chan_info *pchan;
 	struct mbox_chan *chan = p;
-	struct pcc_header *pcc_header = chan->active_req;
-	void *handle = NULL;
 
 	pchan = chan->con_priv;
 
@@ -357,17 +339,8 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p)
 	 * required to avoid any possible race in updatation of this flag.
 	 */
 	pchan->chan_in_use = false;
-
-	if (pchan->chan.rx_alloc)
-		handle = write_response(pchan);
-
-	if (chan->active_req) {
-		pcc_header = chan->active_req;
-		if (pcc_header->flags & PCC_CMD_COMPLETION_NOTIFY)
-			mbox_chan_txdone(chan, 0);
-	}
-
-	mbox_chan_received_data(chan, handle);
+	mbox_chan_received_data(chan, NULL);
+	mbox_chan_txdone(chan, 0);
 
 	pcc_chan_acknowledge(pchan);
 
@@ -404,33 +377,20 @@ pcc_mbox_request_channel(struct mbox_client *cl, int subspace_id)
 		return ERR_PTR(-EBUSY);
 	}
 
-	rc = mbox_bind_client(chan, cl);
-	if (rc)
-		return ERR_PTR(rc);
-
 	pcc_mchan = &pchan->chan;
 	pcc_mchan->shmem = acpi_os_ioremap(pcc_mchan->shmem_base_addr,
 					   pcc_mchan->shmem_size);
 	if (!pcc_mchan->shmem)
-		goto err;
-
-	pcc_mchan->manage_writes = false;
+		return ERR_PTR(-ENXIO);
 
-	/* This indicates that the channel is ready to accept messages.
-	 * This needs to happen after the channel has registered
-	 * its callback. There is no access point to do that in
-	 * the mailbox API. That implies that the mailbox client must
-	 * have set the allocate callback function prior to
-	 * sending any messages.
-	 */
-	if (pchan->type == ACPI_PCCT_TYPE_EXT_PCC_SLAVE_SUBSPACE)
-		pcc_chan_reg_read_modify_write(&pchan->cmd_update);
+	rc = mbox_bind_client(chan, cl);
+	if (rc) {
+		iounmap(pcc_mchan->shmem);
+		pcc_mchan->shmem = NULL;
+		return ERR_PTR(rc);
+	}
 
 	return pcc_mchan;
-
-err:
-	mbox_free_channel(chan);
-	return ERR_PTR(-ENXIO);
 }
 EXPORT_SYMBOL_GPL(pcc_mbox_request_channel);
 
@@ -459,38 +419,8 @@ void pcc_mbox_free_channel(struct pcc_mbox_chan *pchan)
 }
 EXPORT_SYMBOL_GPL(pcc_mbox_free_channel);
 
-static int pcc_write_to_buffer(struct mbox_chan *chan, void *data)
-{
-	struct pcc_chan_info *pchan = chan->con_priv;
-	struct pcc_mbox_chan *pcc_mbox_chan = &pchan->chan;
-	struct pcc_header *pcc_header = data;
-
-	if (!pchan->chan.manage_writes)
-		return 0;
-
-	/* The PCC header length includes the command field
-	 * but not the other values from the header.
-	 */
-	int len = pcc_header->length - sizeof(u32) + sizeof(struct pcc_header);
-	u64 val;
-
-	pcc_chan_reg_read(&pchan->cmd_complete, &val);
-	if (!val) {
-		pr_info("%s pchan->cmd_complete not set", __func__);
-		return -1;
-	}
-	memcpy_toio(pcc_mbox_chan->shmem,  data, len);
-	return 0;
-}
-
-
 /**
- * pcc_send_data - Called from Mailbox Controller code. If
- *		pchan->chan.rx_alloc is set, then the command complete
- *		flag is checked and the data is written to the shared
- *		buffer io memory.
- *
- *		If pchan->chan.rx_alloc is not set, then it is used
+ * pcc_send_data - Called from Mailbox Controller code. Used
  *		here only to ring the channel doorbell. The PCC client
  *		specific read/write is done in the client driver in
  *		order to maintain atomicity over PCC channel once
@@ -506,37 +436,24 @@ static int pcc_send_data(struct mbox_chan *chan, void *data)
 	int ret;
 	struct pcc_chan_info *pchan = chan->con_priv;
 
-	ret = pcc_write_to_buffer(chan, data);
-	if (ret)
-		return ret;
-
 	ret = pcc_chan_reg_read_modify_write(&pchan->cmd_update);
 	if (ret)
 		return ret;
 
 	ret = pcc_chan_reg_read_modify_write(&pchan->db);
-
 	if (!ret && pchan->plat_irq > 0)
 		pchan->chan_in_use = true;
 
 	return ret;
 }
 
-
 static bool pcc_last_tx_done(struct mbox_chan *chan)
 {
 	struct pcc_chan_info *pchan = chan->con_priv;
-	u64 val;
 
-	pcc_chan_reg_read(&pchan->cmd_complete, &val);
-	if (!val)
-		return false;
-	else
-		return true;
+	return pcc_mbox_cmd_complete_check(pchan);
 }
 
-
-
 /**
  * pcc_startup - Called from Mailbox Controller code. Used here
  *		to request the interrupt.
@@ -550,6 +467,12 @@ static int pcc_startup(struct mbox_chan *chan)
 	unsigned long irqflags;
 	int rc;
 
+	/*
+	 * Clear and acknowledge any pending interrupts on responder channel
+	 * before enabling the interrupt
+	 */
+	pcc_chan_acknowledge(pchan);
+
 	if (pchan->plat_irq > 0) {
 		irqflags = pcc_chan_plat_irq_can_be_shared(pchan) ?
 						IRQF_SHARED | IRQF_ONESHOT : 0;
@@ -877,8 +800,13 @@ static int pcc_mbox_probe(struct platform_device *pdev)
 		(unsigned long) pcct_tbl + sizeof(struct acpi_table_pcct));
 
 	acpi_pcct_tbl = (struct acpi_table_pcct *) pcct_tbl;
-	if (acpi_pcct_tbl->flags & ACPI_PCCT_DOORBELL)
+	if (acpi_pcct_tbl->flags & ACPI_PCCT_DOORBELL) {
 		pcc_mbox_ctrl->txdone_irq = true;
+		pcc_mbox_ctrl->txdone_poll = false;
+	} else {
+		pcc_mbox_ctrl->txdone_irq = false;
+		pcc_mbox_ctrl->txdone_poll = true;
+	}
 
 	for (i = 0; i < count; i++) {
 		struct pcc_chan_info *pchan = chan_info + i;
diff --git a/drivers/mailbox/rcar-mfis-mailbox.c b/drivers/mailbox/rcar-mfis-mailbox.c
new file mode 100644
index 000000000000..e4be8e5fb8a7
--- /dev/null
+++ b/drivers/mailbox/rcar-mfis-mailbox.c
@@ -0,0 +1,168 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Renesas MFIS (Multifunctional Inferface) Mailbox Driver
+//
+// Copyright (C) 2025, Renesas Electronics Corporation.
+//
+
+#include <linux/device.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/interrupt.h>
+#include <linux/mailbox_controller.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+enum direction {
+	TX,
+	RX,
+	NUM_DIRECTION,
+};
+
+struct mfis_chan {
+	u32 __iomem *reg;
+	bool active;
+};
+
+struct mfis_priv {
+	struct mbox_controller mbox;
+	spinlock_t lock;
+	struct mbox_chan  chan[NUM_DIRECTION];
+	struct mfis_chan mchan[NUM_DIRECTION];
+};
+
+#define mfis_chan_to_priv(ch) chan->con_priv
+#define mfis_chan_to_mchan(priv, ch) (priv->mchan + (ch - priv->chan))
+
+static int mfis_send_data(struct mbox_chan *chan, void *data)
+{
+	struct mfis_priv *priv  = mfis_chan_to_priv(chan);
+	struct mfis_chan *mchan = mfis_chan_to_mchan(priv, chan);
+
+	iowrite32(0x1, mchan->reg);
+
+	return 0;
+}
+
+static irqreturn_t mfis_interrupt(int irq, void *data)
+{
+	struct mfis_priv *priv = data;
+
+	guard(spinlock)(&priv->lock);
+
+	for (int i = 0; i < NUM_DIRECTION; i++) {
+		struct mbox_chan *chan  = priv->chan  + i;
+		struct mfis_chan *mchan = priv->mchan + i;
+
+		if (mchan->active)
+			mbox_chan_received_data(chan, 0);
+
+		iowrite32(0x0, mchan->reg);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int mfis_chan_set_active(struct mbox_chan *chan, bool active)
+{
+	struct mfis_priv *priv  = mfis_chan_to_priv(chan);
+	struct mfis_chan *mchan = mfis_chan_to_mchan(priv, chan);
+
+	guard(spinlock_irqsave)(&priv->lock);
+
+	mchan->active = active;
+
+	return 0;
+}
+
+static int mfis_startup(struct mbox_chan *chan)
+{
+	return mfis_chan_set_active(chan, true);
+}
+
+static void mfis_shutdown(struct mbox_chan *chan)
+{
+	mfis_chan_set_active(chan, false);
+}
+
+static bool mfis_last_tx_done(struct mbox_chan *chan)
+{
+	return true;
+}
+
+static const struct mbox_chan_ops mfis_chan_ops = {
+	.send_data	= mfis_send_data,
+	.startup	= mfis_startup,
+	.shutdown	= mfis_shutdown,
+	.last_tx_done	= mfis_last_tx_done
+};
+
+static int mfis_mbox_probe(struct platform_device *pdev)
+{
+	struct mfis_priv *priv;
+	struct device *dev = &pdev->dev;
+	struct mbox_controller *mbox;
+	void __iomem *reg;
+	int ret, irq;
+
+	irq = of_irq_get(dev->of_node, 0);
+	if (irq < 0)
+		return irq;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	ret = devm_request_irq(dev, irq, mfis_interrupt, 0, dev_name(dev), priv);
+	if (ret < 0)
+		return ret;
+
+	reg = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(reg))
+		return PTR_ERR(reg);
+
+	spin_lock_init(&priv->lock);
+
+	priv->mchan[TX].reg	= reg + 0x4;
+	priv->mchan[RX].reg	= reg;
+
+	mbox = &priv->mbox;
+
+	mbox->chans	= priv->chan;
+	mbox->chans[TX].mbox = mbox;
+	mbox->chans[RX].mbox = mbox;
+	mbox->chans[TX].con_priv = priv;
+	mbox->chans[RX].con_priv = priv;
+	mbox->txdone_poll = true;
+	mbox->txdone_irq = false;
+	mbox->txpoll_period = 1;
+	mbox->num_chans = NUM_DIRECTION;
+	mbox->ops = &mfis_chan_ops;
+	mbox->dev = dev;
+
+	ret = mbox_controller_register(mbox);
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, mbox);
+
+	return 0;
+}
+
+static const struct of_device_id mfis_mbox_of_match[] = {
+	{ .compatible = "rcar,mfis-mailbox-gen5" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, mfis_mbox_of_match);
+
+static struct platform_driver mfis_mbox_driver = {
+	.driver = {
+		.name = "rcar-mfis-mailbox",
+		.of_match_table = mfis_mbox_of_match,
+	},
+	.probe	= mfis_mbox_probe,
+};
+module_platform_driver(mfis_mbox_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("R-Car MFIS mailbox driver");
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 79704fbc523b..cbeb5f918d09 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -11,6 +11,7 @@
 #include <linux/completion.h>
 #include <linux/err.h>
 #include <linux/module.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/key.h>
diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c
index 170bf67a2edd..b41424a4c139 100644
--- a/drivers/md/dm-integrity.c
+++ b/drivers/md/dm-integrity.c
@@ -17,6 +17,7 @@
 #include <linux/sort.h>
 #include <linux/rbtree.h>
 #include <linux/delay.h>
+#include <linux/hex.h>
 #include <linux/random.h>
 #include <linux/reboot.h>
 #include <crypto/hash.h>
diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c
index 5c17472d7896..ca094f14a287 100644
--- a/drivers/md/dm-verity-target.c
+++ b/drivers/md/dm-verity-target.c
@@ -17,6 +17,7 @@
 #include "dm-verity-fec.h"
 #include "dm-verity-verify-sig.h"
 #include "dm-audit.h"
+#include <linux/hex.h>
 #include <linux/module.h>
 #include <linux/reboot.h>
 #include <linux/string.h>
diff --git a/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c b/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c
index e2eff17952ab..bf92576bb2fc 100644
--- a/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c
+++ b/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c
@@ -19,6 +19,7 @@
 #include <linux/completion.h>
 #include <linux/ctype.h>
 #include <linux/delay.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
diff --git a/drivers/media/cec/usb/rainshadow/rainshadow-cec.c b/drivers/media/cec/usb/rainshadow/rainshadow-cec.c
index 08f58456d682..6c0cee4b066f 100644
--- a/drivers/media/cec/usb/rainshadow/rainshadow-cec.c
+++ b/drivers/media/cec/usb/rainshadow/rainshadow-cec.c
@@ -19,6 +19,7 @@
 #include <linux/completion.h>
 #include <linux/ctype.h>
 #include <linux/delay.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
diff --git a/drivers/media/common/videobuf2/videobuf2-core.c b/drivers/media/common/videobuf2/videobuf2-core.c
index 2df566f409b6..2d1f253b4929 100644
--- a/drivers/media/common/videobuf2/videobuf2-core.c
+++ b/drivers/media/common/videobuf2/videobuf2-core.c
@@ -605,8 +605,7 @@ static void __vb2_queue_free(struct vb2_queue *q, unsigned int start, unsigned i
 	 */
 	if (vb2_get_num_buffers(q)) {
 		bool unbalanced = q->cnt_start_streaming != q->cnt_stop_streaming ||
-				  q->cnt_prepare_streaming != q->cnt_unprepare_streaming ||
-				  q->cnt_wait_prepare != q->cnt_wait_finish;
+				  q->cnt_prepare_streaming != q->cnt_unprepare_streaming;
 
 		if (unbalanced) {
 			pr_info("unbalanced counters for queue %p:\n", q);
@@ -617,13 +616,8 @@ static void __vb2_queue_free(struct vb2_queue *q, unsigned int start, unsigned i
 			if (q->cnt_prepare_streaming != q->cnt_unprepare_streaming)
 				pr_info("     prepare_streaming: %u unprepare_streaming: %u\n",
 					q->cnt_prepare_streaming, q->cnt_unprepare_streaming);
-			if (q->cnt_wait_prepare != q->cnt_wait_finish)
-				pr_info("     wait_prepare: %u wait_finish: %u\n",
-					q->cnt_wait_prepare, q->cnt_wait_finish);
 		}
 		q->cnt_queue_setup = 0;
-		q->cnt_wait_prepare = 0;
-		q->cnt_wait_finish = 0;
 		q->cnt_prepare_streaming = 0;
 		q->cnt_start_streaming = 0;
 		q->cnt_stop_streaming = 0;
@@ -2037,10 +2031,7 @@ static int __vb2_wait_for_done_vb(struct vb2_queue *q, int nonblocking)
 		 * become ready or for streamoff. Driver's lock is released to
 		 * allow streamoff or qbuf to be called while waiting.
 		 */
-		if (q->ops->wait_prepare)
-			call_void_qop(q, wait_prepare, q);
-		else if (q->lock)
-			mutex_unlock(q->lock);
+		mutex_unlock(q->lock);
 
 		/*
 		 * All locks have been released, it is safe to sleep now.
@@ -2050,10 +2041,7 @@ static int __vb2_wait_for_done_vb(struct vb2_queue *q, int nonblocking)
 				!list_empty(&q->done_list) || !q->streaming ||
 				q->error);
 
-		if (q->ops->wait_finish)
-			call_void_qop(q, wait_finish, q);
-		else if (q->lock)
-			mutex_lock(q->lock);
+		mutex_lock(q->lock);
 
 		q->waiting_in_dqbuf = 0;
 		/*
@@ -2653,12 +2641,8 @@ int vb2_core_queue_init(struct vb2_queue *q)
 	if (WARN_ON(q->min_reqbufs_allocation > q->max_num_buffers))
 		return -EINVAL;
 
-	/* Either both or none are set */
-	if (WARN_ON(!q->ops->wait_prepare ^ !q->ops->wait_finish))
-		return -EINVAL;
-
-	/* Warn if q->lock is NULL and no custom wait_prepare is provided */
-	if (WARN_ON(!q->lock && !q->ops->wait_prepare))
+	/* Warn if q->lock is NULL */
+	if (WARN_ON(!q->lock))
 		return -EINVAL;
 
 	INIT_LIST_HEAD(&q->queued_list);
@@ -3220,17 +3204,10 @@ static int vb2_thread(void *data)
 				continue;
 			prequeue--;
 		} else {
-			if (!threadio->stop) {
-				if (q->ops->wait_finish)
-					call_void_qop(q, wait_finish, q);
-				else if (q->lock)
-					mutex_lock(q->lock);
+			mutex_lock(q->lock);
+			if (!threadio->stop)
 				ret = vb2_core_dqbuf(q, &index, NULL, 0);
-				if (q->ops->wait_prepare)
-					call_void_qop(q, wait_prepare, q);
-				else if (q->lock)
-					mutex_unlock(q->lock);
-			}
+			mutex_unlock(q->lock);
 			dprintk(q, 5, "file io: vb2_dqbuf result: %d\n", ret);
 			if (!ret)
 				vb = vb2_get_buffer(q, index);
@@ -3245,15 +3222,9 @@ static int vb2_thread(void *data)
 		if (copy_timestamp)
 			vb->timestamp = ktime_get_ns();
 		if (!threadio->stop) {
-			if (q->ops->wait_finish)
-				call_void_qop(q, wait_finish, q);
-			else if (q->lock)
-				mutex_lock(q->lock);
+			mutex_lock(q->lock);
 			ret = vb2_core_qbuf(q, vb, NULL, NULL);
-			if (q->ops->wait_prepare)
-				call_void_qop(q, wait_prepare, q);
-			else if (q->lock)
-				mutex_unlock(q->lock);
+			mutex_unlock(q->lock);
 		}
 		if (ret || threadio->stop)
 			break;
diff --git a/drivers/media/common/videobuf2/videobuf2-v4l2.c b/drivers/media/common/videobuf2/videobuf2-v4l2.c
index 83862d57b126..4baded4fd3b8 100644
--- a/drivers/media/common/videobuf2/videobuf2-v4l2.c
+++ b/drivers/media/common/videobuf2/videobuf2-v4l2.c
@@ -1302,20 +1302,6 @@ void vb2_video_unregister_device(struct video_device *vdev)
 }
 EXPORT_SYMBOL_GPL(vb2_video_unregister_device);
 
-/* vb2_ops helpers. Only use if vq->lock is non-NULL. */
-
-void vb2_ops_wait_prepare(struct vb2_queue *vq)
-{
-	mutex_unlock(vq->lock);
-}
-EXPORT_SYMBOL_GPL(vb2_ops_wait_prepare);
-
-void vb2_ops_wait_finish(struct vb2_queue *vq)
-{
-	mutex_lock(vq->lock);
-}
-EXPORT_SYMBOL_GPL(vb2_ops_wait_finish);
-
 /*
  * Note that this function is called during validation time and
  * thus the req_queue_mutex is held to ensure no request objects
diff --git a/drivers/media/dvb-core/dmxdev.c b/drivers/media/dvb-core/dmxdev.c
index 8c6f5aafda1d..c946c8ea6e39 100644
--- a/drivers/media/dvb-core/dmxdev.c
+++ b/drivers/media/dvb-core/dmxdev.c
@@ -171,6 +171,7 @@ static int dvb_dvr_open(struct inode *inode, struct file *file)
 		dvb_ringbuffer_init(&dmxdev->dvr_buffer, mem, DVR_BUFFER_SIZE);
 		if (dmxdev->may_do_mmap)
 			dvb_vb2_init(&dmxdev->dvr_vb2_ctx, "dvr",
+				     &dmxdev->mutex,
 				     file->f_flags & O_NONBLOCK);
 		dvbdev->readers--;
 	}
@@ -397,11 +398,11 @@ static int dvb_dmxdev_section_callback(const u8 *buffer1, size_t buffer1_len,
 	if (dvb_vb2_is_streaming(&dmxdevfilter->vb2_ctx)) {
 		ret = dvb_vb2_fill_buffer(&dmxdevfilter->vb2_ctx,
 					  buffer1, buffer1_len,
-					  buffer_flags);
+					  buffer_flags, true);
 		if (ret == buffer1_len)
 			ret = dvb_vb2_fill_buffer(&dmxdevfilter->vb2_ctx,
 						  buffer2, buffer2_len,
-						  buffer_flags);
+						  buffer_flags, true);
 	} else {
 		ret = dvb_dmxdev_buffer_write(&dmxdevfilter->buffer,
 					      buffer1, buffer1_len);
@@ -452,10 +453,10 @@ static int dvb_dmxdev_ts_callback(const u8 *buffer1, size_t buffer1_len,
 
 	if (dvb_vb2_is_streaming(ctx)) {
 		ret = dvb_vb2_fill_buffer(ctx, buffer1, buffer1_len,
-					  buffer_flags);
+					  buffer_flags, false);
 		if (ret == buffer1_len)
 			ret = dvb_vb2_fill_buffer(ctx, buffer2, buffer2_len,
-						  buffer_flags);
+						  buffer_flags, false);
 	} else {
 		if (buffer->error) {
 			spin_unlock(&dmxdevfilter->dev->lock);
@@ -815,9 +816,21 @@ static int dvb_demux_open(struct inode *inode, struct file *file)
 	dmxdev->may_do_mmap = 0;
 #endif
 
+	/*
+	 * The mutex passed to dvb_vb2_init is unlocked when a buffer
+	 * is in a blocking wait. However, dmxdevfilter has two mutexes:
+	 * dmxdevfilter->mutex and dmxdev->mutex. So this will not work.
+	 * The solution would be to support unlocking two mutexes in vb2,
+	 * but since this problem has been here since the beginning and
+	 * nobody ever complained, we leave it as-is rather than adding
+	 * that second mutex pointer to vb2.
+	 *
+	 * In the unlikely event that someone complains about this, then
+	 * this comment will hopefully help.
+	 */
 	dvb_ringbuffer_init(&dmxdevfilter->buffer, NULL, 8192);
 	dvb_vb2_init(&dmxdevfilter->vb2_ctx, "demux_filter",
-		     file->f_flags & O_NONBLOCK);
+		     &dmxdevfilter->mutex, file->f_flags & O_NONBLOCK);
 	dmxdevfilter->type = DMXDEV_TYPE_NONE;
 	dvb_dmxdev_filter_state_set(dmxdevfilter, DMXDEV_STATE_ALLOCATED);
 	timer_setup(&dmxdevfilter->timer, dvb_dmxdev_filter_timeout, 0);
@@ -1217,24 +1230,11 @@ static int dvb_demux_mmap(struct file *file, struct vm_area_struct *vma)
 {
 	struct dmxdev_filter *dmxdevfilter = file->private_data;
 	struct dmxdev *dmxdev = dmxdevfilter->dev;
-	int ret;
 
 	if (!dmxdev->may_do_mmap)
 		return -ENOTTY;
 
-	if (mutex_lock_interruptible(&dmxdev->mutex))
-		return -ERESTARTSYS;
-
-	if (mutex_lock_interruptible(&dmxdevfilter->mutex)) {
-		mutex_unlock(&dmxdev->mutex);
-		return -ERESTARTSYS;
-	}
-	ret = dvb_vb2_mmap(&dmxdevfilter->vb2_ctx, vma);
-
-	mutex_unlock(&dmxdevfilter->mutex);
-	mutex_unlock(&dmxdev->mutex);
-
-	return ret;
+	return dvb_vb2_mmap(&dmxdevfilter->vb2_ctx, vma);
 }
 #endif
 
@@ -1367,7 +1367,6 @@ static int dvb_dvr_mmap(struct file *file, struct vm_area_struct *vma)
 {
 	struct dvb_device *dvbdev = file->private_data;
 	struct dmxdev *dmxdev = dvbdev->priv;
-	int ret;
 
 	if (!dmxdev->may_do_mmap)
 		return -ENOTTY;
@@ -1375,12 +1374,7 @@ static int dvb_dvr_mmap(struct file *file, struct vm_area_struct *vma)
 	if (dmxdev->exit)
 		return -ENODEV;
 
-	if (mutex_lock_interruptible(&dmxdev->mutex))
-		return -ERESTARTSYS;
-
-	ret = dvb_vb2_mmap(&dmxdev->dvr_vb2_ctx, vma);
-	mutex_unlock(&dmxdev->mutex);
-	return ret;
+	return dvb_vb2_mmap(&dmxdev->dvr_vb2_ctx, vma);
 }
 #endif
 
diff --git a/drivers/media/dvb-core/dvb_vb2.c b/drivers/media/dvb-core/dvb_vb2.c
index 29edaaff7a5c..672b0efdca21 100644
--- a/drivers/media/dvb-core/dvb_vb2.c
+++ b/drivers/media/dvb-core/dvb_vb2.c
@@ -103,31 +103,12 @@ static void _stop_streaming(struct vb2_queue *vq)
 	spin_unlock_irqrestore(&ctx->slock, flags);
 }
 
-static void _dmxdev_lock(struct vb2_queue *vq)
-{
-	struct dvb_vb2_ctx *ctx = vb2_get_drv_priv(vq);
-
-	mutex_lock(&ctx->mutex);
-	dprintk(3, "[%s]\n", ctx->name);
-}
-
-static void _dmxdev_unlock(struct vb2_queue *vq)
-{
-	struct dvb_vb2_ctx *ctx = vb2_get_drv_priv(vq);
-
-	if (mutex_is_locked(&ctx->mutex))
-		mutex_unlock(&ctx->mutex);
-	dprintk(3, "[%s]\n", ctx->name);
-}
-
 static const struct vb2_ops dvb_vb2_qops = {
 	.queue_setup		= _queue_setup,
 	.buf_prepare		= _buffer_prepare,
 	.buf_queue		= _buffer_queue,
 	.start_streaming	= _start_streaming,
 	.stop_streaming		= _stop_streaming,
-	.wait_prepare		= _dmxdev_unlock,
-	.wait_finish		= _dmxdev_lock,
 };
 
 static void _fill_dmx_buffer(struct vb2_buffer *vb, void *pb)
@@ -158,9 +139,10 @@ static const struct vb2_buf_ops dvb_vb2_buf_ops = {
 };
 
 /*
- * Videobuf operations
+ * vb2 operations
  */
-int dvb_vb2_init(struct dvb_vb2_ctx *ctx, const char *name, int nonblocking)
+int dvb_vb2_init(struct dvb_vb2_ctx *ctx, const char *name,
+		 struct mutex *mutex, int nonblocking)
 {
 	struct vb2_queue *q = &ctx->vb_q;
 	int ret;
@@ -175,14 +157,7 @@ int dvb_vb2_init(struct dvb_vb2_ctx *ctx, const char *name, int nonblocking)
 	q->ops = &dvb_vb2_qops;
 	q->mem_ops = &vb2_vmalloc_memops;
 	q->buf_ops = &dvb_vb2_buf_ops;
-	ret = vb2_core_queue_init(q);
-	if (ret) {
-		ctx->state = DVB_VB2_STATE_NONE;
-		dprintk(1, "[%s] errno=%d\n", ctx->name, ret);
-		return ret;
-	}
-
-	mutex_init(&ctx->mutex);
+	q->lock = mutex;
 	spin_lock_init(&ctx->slock);
 	INIT_LIST_HEAD(&ctx->dvb_q);
 
@@ -190,6 +165,13 @@ int dvb_vb2_init(struct dvb_vb2_ctx *ctx, const char *name, int nonblocking)
 	ctx->nonblocking = nonblocking;
 	ctx->state = DVB_VB2_STATE_INIT;
 
+	ret = vb2_core_queue_init(q);
+	if (ret) {
+		ctx->state = DVB_VB2_STATE_NONE;
+		dprintk(1, "[%s] errno=%d\n", ctx->name, ret);
+		return ret;
+	}
+
 	dprintk(3, "[%s]\n", ctx->name);
 
 	return 0;
@@ -249,7 +231,8 @@ int dvb_vb2_is_streaming(struct dvb_vb2_ctx *ctx)
 
 int dvb_vb2_fill_buffer(struct dvb_vb2_ctx *ctx,
 			const unsigned char *src, int len,
-			enum dmx_buffer_flags *buffer_flags)
+			enum dmx_buffer_flags *buffer_flags,
+			bool flush)
 {
 	unsigned long flags = 0;
 	void *vbuf = NULL;
@@ -306,7 +289,7 @@ int dvb_vb2_fill_buffer(struct dvb_vb2_ctx *ctx,
 		}
 	}
 
-	if (ctx->nonblocking && ctx->buf) {
+	if (flush && ctx->buf) {
 		vb2_set_plane_payload(&ctx->buf->vb, 0, ll);
 		vb2_buffer_done(&ctx->buf->vb, VB2_BUF_STATE_DONE);
 		list_del(&ctx->buf->list);
diff --git a/drivers/media/i2c/ccs/ccs-reg-access.c b/drivers/media/i2c/ccs/ccs-reg-access.c
index fd36889ccc1d..a0181a5d2f34 100644
--- a/drivers/media/i2c/ccs/ccs-reg-access.c
+++ b/drivers/media/i2c/ccs/ccs-reg-access.c
@@ -12,6 +12,7 @@
 #include <linux/unaligned.h>
 
 #include <linux/delay.h>
+#include <linux/hex.h>
 #include <linux/i2c.h>
 
 #include "ccs.h"
diff --git a/drivers/media/mc/mc-request.c b/drivers/media/mc/mc-request.c
index 2ac9ac0a740b..3cca9a0c7c97 100644
--- a/drivers/media/mc/mc-request.c
+++ b/drivers/media/mc/mc-request.c
@@ -315,12 +315,12 @@ int media_request_alloc(struct media_device *mdev, int *alloc_fd)
 
 	fd_prepare_file(fdf)->private_data = req;
 
-	*alloc_fd = fd_publish(fdf);
-
 	snprintf(req->debug_str, sizeof(req->debug_str), "%u:%d",
-		 atomic_inc_return(&mdev->request_id), *alloc_fd);
+		 atomic_inc_return(&mdev->request_id), fd_prepare_fd(fdf));
 	dev_dbg(mdev->dev, "request: allocated %s\n", req->debug_str);
 
+	*alloc_fd = fd_publish(fdf);
+
 	return 0;
 
 err_free_req:
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-debugifc.c b/drivers/media/usb/pvrusb2/pvrusb2-debugifc.c
index 81d711269ab5..9f936085acbb 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-debugifc.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-debugifc.c
@@ -4,6 +4,7 @@
  *  Copyright (C) 2005 Mike Isely <isely@pobox.com>
  */
 
+#include <linux/hex.h>
 #include <linux/string.h>
 #include "pvrusb2-debugifc.h"
 #include "pvrusb2-hdw.h"
diff --git a/drivers/memory/mtk-smi.c b/drivers/memory/mtk-smi.c
index 733e22f695ab..aaeba8ab211e 100644
--- a/drivers/memory/mtk-smi.c
+++ b/drivers/memory/mtk-smi.c
@@ -595,25 +595,28 @@ static int mtk_smi_device_link_common(struct device *dev, struct device **com_de
 
 	smi_com_pdev = of_find_device_by_node(smi_com_node);
 	of_node_put(smi_com_node);
-	if (smi_com_pdev) {
-		/* smi common is the supplier, Make sure it is ready before */
-		if (!platform_get_drvdata(smi_com_pdev)) {
-			put_device(&smi_com_pdev->dev);
-			return -EPROBE_DEFER;
-		}
-		smi_com_dev = &smi_com_pdev->dev;
-		link = device_link_add(dev, smi_com_dev,
-				       DL_FLAG_PM_RUNTIME | DL_FLAG_STATELESS);
-		if (!link) {
-			dev_err(dev, "Unable to link smi-common dev\n");
-			put_device(&smi_com_pdev->dev);
-			return -ENODEV;
-		}
-		*com_dev = smi_com_dev;
-	} else {
+	if (!smi_com_pdev) {
 		dev_err(dev, "Failed to get the smi_common device\n");
 		return -EINVAL;
 	}
+
+	/* smi common is the supplier, Make sure it is ready before */
+	if (!platform_get_drvdata(smi_com_pdev)) {
+		put_device(&smi_com_pdev->dev);
+		return -EPROBE_DEFER;
+	}
+
+	smi_com_dev = &smi_com_pdev->dev;
+	link = device_link_add(dev, smi_com_dev,
+			       DL_FLAG_PM_RUNTIME | DL_FLAG_STATELESS);
+	if (!link) {
+		dev_err(dev, "Unable to link smi-common dev\n");
+		put_device(&smi_com_pdev->dev);
+		return -ENODEV;
+	}
+
+	*com_dev = smi_com_dev;
+
 	return 0;
 }
 
@@ -674,6 +677,7 @@ static int mtk_smi_larb_probe(struct platform_device *pdev)
 err_pm_disable:
 	pm_runtime_disable(dev);
 	device_link_remove(dev, larb->smi_common_dev);
+	put_device(larb->smi_common_dev);
 	return ret;
 }
 
@@ -684,6 +688,7 @@ static void mtk_smi_larb_remove(struct platform_device *pdev)
 	device_link_remove(&pdev->dev, larb->smi_common_dev);
 	pm_runtime_disable(&pdev->dev);
 	component_del(&pdev->dev, &mtk_smi_larb_component_ops);
+	put_device(larb->smi_common_dev);
 }
 
 static int __maybe_unused mtk_smi_larb_resume(struct device *dev)
@@ -917,6 +922,7 @@ static void mtk_smi_common_remove(struct platform_device *pdev)
 	if (common->plat->type == MTK_SMI_GEN2_SUB_COMM)
 		device_link_remove(&pdev->dev, common->smi_common_dev);
 	pm_runtime_disable(&pdev->dev);
+	put_device(common->smi_common_dev);
 }
 
 static int __maybe_unused mtk_smi_common_resume(struct device *dev)
diff --git a/drivers/mfd/ls2k-bmc-core.c b/drivers/mfd/ls2k-bmc-core.c
index e42f1de9e641..408056bfb2fe 100644
--- a/drivers/mfd/ls2k-bmc-core.c
+++ b/drivers/mfd/ls2k-bmc-core.c
@@ -464,53 +464,36 @@ static int ls2k_bmc_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	resource_size_t base;
 	int ret;
 
-	ret = pci_enable_device(dev);
+	ret = pcim_enable_device(dev);
 	if (ret)
 		return ret;
 
 	ddata = devm_kzalloc(&dev->dev, sizeof(*ddata), GFP_KERNEL);
-	if (!ddata) {
-		ret = -ENOMEM;
-		goto disable_pci;
-	}
+	if (!ddata)
+		return -ENOMEM;
 
 	ddata->dev = &dev->dev;
 
 	ret = ls2k_bmc_init(ddata);
 	if (ret)
-		goto disable_pci;
+		return ret;
 
 	ret = ls2k_bmc_parse_mode(dev, &pd);
 	if (ret)
-		goto disable_pci;
+		return ret;
 
 	ls2k_bmc_cells[LS2K_BMC_DISPLAY].platform_data = &pd;
 	ls2k_bmc_cells[LS2K_BMC_DISPLAY].pdata_size = sizeof(pd);
-	base = dev->resource[0].start + LS2K_DISPLAY_RES_START;
+	base = pci_resource_start(dev, 0) + LS2K_DISPLAY_RES_START;
 
 	/* Remove conflicting efifb device */
 	ret = aperture_remove_conflicting_devices(base, SZ_4M, "simple-framebuffer");
-	if (ret) {
-		dev_err(&dev->dev, "Failed to removed firmware framebuffers: %d\n", ret);
-		goto disable_pci;
-	}
-
-	ret = devm_mfd_add_devices(&dev->dev, PLATFORM_DEVID_AUTO,
-				   ls2k_bmc_cells, ARRAY_SIZE(ls2k_bmc_cells),
-				   &dev->resource[0], 0, NULL);
 	if (ret)
-		goto disable_pci;
+		return dev_err_probe(&dev->dev, ret, "Failed to removed firmware framebuffers\n");
 
-	return 0;
-
-disable_pci:
-	pci_disable_device(dev);
-	return ret;
-}
-
-static void ls2k_bmc_remove(struct pci_dev *dev)
-{
-	pci_disable_device(dev);
+	return devm_mfd_add_devices(&dev->dev, PLATFORM_DEVID_AUTO,
+				    ls2k_bmc_cells, ARRAY_SIZE(ls2k_bmc_cells),
+				    pci_resource_n(dev, 0), 0, NULL);
 }
 
 static struct pci_device_id ls2k_bmc_devices[] = {
@@ -523,7 +506,6 @@ static struct pci_driver ls2k_bmc_driver = {
 	.name = "ls2k-bmc",
 	.id_table = ls2k_bmc_devices,
 	.probe = ls2k_bmc_probe,
-	.remove = ls2k_bmc_remove,
 };
 module_pci_driver(ls2k_bmc_driver);
 
diff --git a/drivers/misc/eeprom/at24.c b/drivers/misc/eeprom/at24.c
index f721825199ce..0200288d3a7a 100644
--- a/drivers/misc/eeprom/at24.c
+++ b/drivers/misc/eeprom/at24.c
@@ -657,10 +657,8 @@ static int at24_probe(struct i2c_client *client)
 	if (!i2c_fn_i2c && !i2c_fn_block)
 		page_size = 1;
 
-	if (!page_size) {
-		dev_err(dev, "page_size must not be 0!\n");
-		return -EINVAL;
-	}
+	if (!page_size)
+		return dev_err_probe(dev, -EINVAL, "page_size must not be 0!\n");
 
 	if (!is_power_of_2(page_size))
 		dev_warn(dev, "page_size looks suspicious (no power of 2)!\n");
@@ -674,11 +672,9 @@ static int at24_probe(struct i2c_client *client)
 				(flags & AT24_FLAG_ADDR16) ? 65536 : 256);
 	}
 
-	if ((flags & AT24_FLAG_SERIAL) && (flags & AT24_FLAG_MAC)) {
-		dev_err(dev,
-			"invalid device data - cannot have both AT24_FLAG_SERIAL & AT24_FLAG_MAC.");
-		return -EINVAL;
-	}
+	if ((flags & AT24_FLAG_SERIAL) && (flags & AT24_FLAG_MAC))
+		return dev_err_probe(dev, -EINVAL,
+				     "invalid device data - cannot have both AT24_FLAG_SERIAL & AT24_FLAG_MAC.");
 
 	regmap_config.val_bits = 8;
 	regmap_config.reg_bits = (flags & AT24_FLAG_ADDR16) ? 16 : 8;
@@ -759,10 +755,8 @@ static int at24_probe(struct i2c_client *client)
 	full_power = acpi_dev_state_d0(&client->dev);
 	if (full_power) {
 		err = regulator_enable(at24->vcc_reg);
-		if (err) {
-			dev_err(dev, "Failed to enable vcc regulator\n");
-			return err;
-		}
+		if (err)
+			return dev_err_probe(dev, err, "Failed to enable vcc regulator\n");
 
 		pm_runtime_set_active(dev);
 	}
diff --git a/drivers/misc/kgdbts.c b/drivers/misc/kgdbts.c
index 0cf31164b470..3b7a041ea351 100644
--- a/drivers/misc/kgdbts.c
+++ b/drivers/misc/kgdbts.c
@@ -89,6 +89,7 @@
 #include <linux/syscalls.h>
 #include <linux/nmi.h>
 #include <linux/delay.h>
+#include <linux/hex.h>
 #include <linux/kthread.h>
 #include <linux/module.h>
 #include <linux/sched/task.h>
diff --git a/drivers/misc/lkdtm/bugs.c b/drivers/misc/lkdtm/bugs.c
index 376047beea3d..fa05d77acb55 100644
--- a/drivers/misc/lkdtm/bugs.c
+++ b/drivers/misc/lkdtm/bugs.c
@@ -8,6 +8,7 @@
 #include "lkdtm.h"
 #include <linux/cpu.h>
 #include <linux/list.h>
+#include <linux/hrtimer.h>
 #include <linux/sched.h>
 #include <linux/sched/signal.h>
 #include <linux/sched/task_stack.h>
@@ -100,11 +101,61 @@ static void lkdtm_PANIC_STOP_IRQOFF(void)
 	stop_machine(panic_stop_irqoff_fn, &v, cpu_online_mask);
 }
 
+static bool wait_for_panic;
+
+static enum hrtimer_restart panic_in_hardirq(struct hrtimer *timer)
+{
+	panic("from hard IRQ context");
+
+	wait_for_panic = false;
+	return HRTIMER_NORESTART;
+}
+
+static void lkdtm_PANIC_IN_HARDIRQ(void)
+{
+	struct hrtimer timer;
+
+	wait_for_panic = true;
+	hrtimer_setup_on_stack(&timer, panic_in_hardirq,
+			       CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
+	hrtimer_start(&timer, us_to_ktime(100), HRTIMER_MODE_REL_HARD);
+
+	while (wait_for_panic)
+		;
+
+	hrtimer_cancel(&timer);
+}
+
 static void lkdtm_BUG(void)
 {
 	BUG();
 }
 
+static bool wait_for_bug;
+
+static enum hrtimer_restart bug_in_hardirq(struct hrtimer *timer)
+{
+	BUG();
+
+	wait_for_bug = false;
+	return HRTIMER_NORESTART;
+}
+
+static void lkdtm_BUG_IN_HARDIRQ(void)
+{
+	struct hrtimer timer;
+
+	wait_for_bug = true;
+	hrtimer_setup_on_stack(&timer, bug_in_hardirq,
+			       CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
+	hrtimer_start(&timer, us_to_ktime(100), HRTIMER_MODE_REL_HARD);
+
+	while (wait_for_bug)
+		;
+
+	hrtimer_cancel(&timer);
+}
+
 static int warn_counter;
 
 static void lkdtm_WARNING(void)
@@ -696,7 +747,9 @@ static noinline void lkdtm_CORRUPT_PAC(void)
 static struct crashtype crashtypes[] = {
 	CRASHTYPE(PANIC),
 	CRASHTYPE(PANIC_STOP_IRQOFF),
+	CRASHTYPE(PANIC_IN_HARDIRQ),
 	CRASHTYPE(BUG),
+	CRASHTYPE(BUG_IN_HARDIRQ),
 	CRASHTYPE(WARNING),
 	CRASHTYPE(WARNING_MESSAGE),
 	CRASHTYPE(EXCEPTION),
diff --git a/drivers/misc/pch_phub.c b/drivers/misc/pch_phub.c
index 7bee179841bc..0d63e834dbe7 100644
--- a/drivers/misc/pch_phub.c
+++ b/drivers/misc/pch_phub.c
@@ -7,6 +7,7 @@
 #include <linux/kernel.h>
 #include <linux/types.h>
 #include <linux/fs.h>
+#include <linux/hex.h>
 #include <linux/uaccess.h>
 #include <linux/string.h>
 #include <linux/pci.h>
diff --git a/drivers/mmc/core/card.h b/drivers/mmc/core/card.h
index 1200951bab08..a9619dd45270 100644
--- a/drivers/mmc/core/card.h
+++ b/drivers/mmc/core/card.h
@@ -89,6 +89,7 @@ struct mmc_fixup {
 #define CID_MANFID_MICRON       0x13
 #define CID_MANFID_SAMSUNG      0x15
 #define CID_MANFID_APACER       0x27
+#define CID_MANFID_SANDISK_MMC  0x45
 #define CID_MANFID_SWISSBIT     0x5D
 #define CID_MANFID_KINGSTON     0x70
 #define CID_MANFID_HYNIX	0x90
@@ -305,4 +306,9 @@ static inline int mmc_card_no_uhs_ddr50_tuning(const struct mmc_card *c)
 	return c->quirks & MMC_QUIRK_NO_UHS_DDR50_TUNING;
 }
 
+static inline int mmc_card_broken_mdt(const struct mmc_card *c)
+{
+	return c->quirks & MMC_QUIRK_BROKEN_MDT;
+}
+
 #endif
diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c
index 7c86efb1044a..8846550a8892 100644
--- a/drivers/mmc/core/mmc.c
+++ b/drivers/mmc/core/mmc.c
@@ -671,7 +671,19 @@ static int mmc_decode_ext_csd(struct mmc_card *card, u8 *ext_csd)
 		card->ext_csd.enhanced_rpmb_supported =
 					(card->ext_csd.rel_param &
 					 EXT_CSD_WR_REL_PARAM_EN_RPMB_REL_WR);
+
+		if (card->ext_csd.rev >= 9) {
+			/* Adjust production date as per JEDEC JESD84-B51B September 2025 */
+			if (card->cid.year < 2023)
+				card->cid.year += 16;
+		} else {
+			/* Handle vendors with broken MDT reporting */
+			if (mmc_card_broken_mdt(card) && card->cid.year >= 2010 &&
+			    card->cid.year <= 2012)
+				card->cid.year += 16;
+		}
 	}
+
 out:
 	return err;
 }
diff --git a/drivers/mmc/core/quirks.h b/drivers/mmc/core/quirks.h
index c417ed34c057..f5e8a0f6d11b 100644
--- a/drivers/mmc/core/quirks.h
+++ b/drivers/mmc/core/quirks.h
@@ -170,6 +170,9 @@ static const struct mmc_fixup __maybe_unused mmc_ext_csd_fixups[] = {
 	MMC_FIXUP_EXT_CSD_REV(CID_NAME_ANY, CID_MANFID_NUMONYX,
 			      0x014e, add_quirk, MMC_QUIRK_BROKEN_HPI, 6),
 
+	MMC_FIXUP(CID_NAME_ANY, CID_MANFID_SANDISK_MMC, CID_OEMID_ANY, add_quirk_mmc,
+		  MMC_QUIRK_BROKEN_MDT),
+
 	END_FIXUP
 };
 
diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig
index 24f07df32a1a..6d79cc9a79e2 100644
--- a/drivers/mmc/host/Kconfig
+++ b/drivers/mmc/host/Kconfig
@@ -315,14 +315,14 @@ config MMC_SDHCI_ESDHC_MCF
 
 config MMC_SDHCI_ESDHC_IMX
 	tristate "SDHCI support for the Freescale eSDHC/uSDHC i.MX controller"
-	depends on ARCH_MXC || COMPILE_TEST
+	depends on ARCH_MXC || ARCH_S32 || COMPILE_TEST
 	depends on MMC_SDHCI_PLTFM
 	depends on OF
 	select MMC_SDHCI_IO_ACCESSORS
 	select MMC_CQHCI
 	help
 	  This selects the Freescale eSDHC/uSDHC controller support
-	  found on i.MX25, i.MX35 i.MX5x and i.MX6x.
+	  found on i.MX25, i.MX35, i.MX5x, i.MX6x, and S32G.
 
 	  If you have a controller with this interface, say Y or M here.
 
diff --git a/drivers/mmc/host/dw_mmc-exynos.c b/drivers/mmc/host/dw_mmc-exynos.c
index 384609671a9a..261344d3a8cf 100644
--- a/drivers/mmc/host/dw_mmc-exynos.c
+++ b/drivers/mmc/host/dw_mmc-exynos.c
@@ -185,8 +185,8 @@ static void dw_mci_exynos_set_clksel_timing(struct dw_mci *host, u32 timing)
 	 * HOLD register should be bypassed in case there is no phase shift
 	 * applied on CMD/DATA that is sent to the card.
 	 */
-	if (!SDMMC_CLKSEL_GET_DRV_WD3(clksel) && host->slot)
-		set_bit(DW_MMC_CARD_NO_USE_HOLD, &host->slot->flags);
+	if (!SDMMC_CLKSEL_GET_DRV_WD3(clksel))
+		set_bit(DW_MMC_CARD_NO_USE_HOLD, &host->flags);
 }
 
 static int dw_mci_exynos_runtime_resume(struct device *dev)
@@ -530,11 +530,10 @@ static s8 dw_mci_exynos_get_best_clksmpl(u8 candidates)
 	return loc;
 }
 
-static int dw_mci_exynos_execute_tuning(struct dw_mci_slot *slot, u32 opcode)
+static int dw_mci_exynos_execute_tuning(struct dw_mci *host, u32 opcode)
 {
-	struct dw_mci *host = slot->host;
 	struct dw_mci_exynos_priv_data *priv = host->priv;
-	struct mmc_host *mmc = slot->mmc;
+	struct mmc_host *mmc = host->mmc;
 	u8 start_smpl, smpl, candidates = 0;
 	s8 found;
 	int ret = 0;
diff --git a/drivers/mmc/host/dw_mmc-hi3798cv200.c b/drivers/mmc/host/dw_mmc-hi3798cv200.c
index 0ccfae1b2dbe..4b723ed3259c 100644
--- a/drivers/mmc/host/dw_mmc-hi3798cv200.c
+++ b/drivers/mmc/host/dw_mmc-hi3798cv200.c
@@ -57,11 +57,9 @@ static void dw_mci_hi3798cv200_set_ios(struct dw_mci *host, struct mmc_ios *ios)
 		clk_set_phase(priv->drive_clk, 135);
 }
 
-static int dw_mci_hi3798cv200_execute_tuning(struct dw_mci_slot *slot,
-					     u32 opcode)
+static int dw_mci_hi3798cv200_execute_tuning(struct dw_mci *host, u32 opcode)
 {
 	static const int degrees[] = { 0, 45, 90, 135, 180, 225, 270, 315 };
-	struct dw_mci *host = slot->host;
 	struct hi3798cv200_priv *priv = host->priv;
 	int raise_point = -1, fall_point = -1;
 	int err, prev_err = -1;
@@ -72,7 +70,7 @@ static int dw_mci_hi3798cv200_execute_tuning(struct dw_mci_slot *slot,
 		clk_set_phase(priv->sample_clk, degrees[i]);
 		mci_writel(host, RINTSTS, ALL_INT_CLR);
 
-		err = mmc_send_tuning(slot->mmc, opcode, NULL);
+		err = mmc_send_tuning(host->mmc, opcode, NULL);
 		if (!err)
 			found = 1;
 
diff --git a/drivers/mmc/host/dw_mmc-hi3798mv200.c b/drivers/mmc/host/dw_mmc-hi3798mv200.c
index 5791a975a944..a64907e2e84c 100644
--- a/drivers/mmc/host/dw_mmc-hi3798mv200.c
+++ b/drivers/mmc/host/dw_mmc-hi3798mv200.c
@@ -74,25 +74,24 @@ static void dw_mci_hi3798mv200_set_ios(struct dw_mci *host, struct mmc_ios *ios)
 	}
 }
 
-static inline int dw_mci_hi3798mv200_enable_tuning(struct dw_mci_slot *slot)
+static inline int dw_mci_hi3798mv200_enable_tuning(struct dw_mci *host)
 {
-	struct dw_mci_hi3798mv200_priv *priv = slot->host->priv;
+	struct dw_mci_hi3798mv200_priv *priv = host->priv;
 
 	return regmap_clear_bits(priv->crg_reg, priv->sap_dll_offset, SAP_DLL_CTRL_DLLMODE);
 }
 
-static inline int dw_mci_hi3798mv200_disable_tuning(struct dw_mci_slot *slot)
+static inline int dw_mci_hi3798mv200_disable_tuning(struct dw_mci *host)
 {
-	struct dw_mci_hi3798mv200_priv *priv = slot->host->priv;
+	struct dw_mci_hi3798mv200_priv *priv = host->priv;
 
 	return regmap_set_bits(priv->crg_reg, priv->sap_dll_offset, SAP_DLL_CTRL_DLLMODE);
 }
 
-static int dw_mci_hi3798mv200_execute_tuning_mix_mode(struct dw_mci_slot *slot,
+static int dw_mci_hi3798mv200_execute_tuning_mix_mode(struct dw_mci *host,
 					     u32 opcode)
 {
 	static const int degrees[] = { 0, 45, 90, 135, 180, 225, 270, 315 };
-	struct dw_mci *host = slot->host;
 	struct dw_mci_hi3798mv200_priv *priv = host->priv;
 	int raise_point = -1, fall_point = -1, mid;
 	int err, prev_err = -1;
@@ -101,7 +100,7 @@ static int dw_mci_hi3798mv200_execute_tuning_mix_mode(struct dw_mci_slot *slot,
 	int i;
 	int ret;
 
-	ret = dw_mci_hi3798mv200_enable_tuning(slot);
+	ret = dw_mci_hi3798mv200_enable_tuning(host);
 	if (ret < 0)
 		return ret;
 
@@ -115,7 +114,7 @@ static int dw_mci_hi3798mv200_execute_tuning_mix_mode(struct dw_mci_slot *slot,
 		 *
 		 * Treat edge(flip) found as an error too.
 		 */
-		err = mmc_send_tuning(slot->mmc, opcode, NULL);
+		err = mmc_send_tuning(host->mmc, opcode, NULL);
 		regval = mci_readl(host, TUNING_CTRL);
 		if (err || (regval & SDMMC_TUNING_FIND_EDGE))
 			err = 1;
@@ -136,7 +135,7 @@ static int dw_mci_hi3798mv200_execute_tuning_mix_mode(struct dw_mci_slot *slot,
 	}
 
 tuning_out:
-	ret = dw_mci_hi3798mv200_disable_tuning(slot);
+	ret = dw_mci_hi3798mv200_disable_tuning(host);
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/mmc/host/dw_mmc-k3.c b/drivers/mmc/host/dw_mmc-k3.c
index ad6aa1aea549..23c303106dca 100644
--- a/drivers/mmc/host/dw_mmc-k3.c
+++ b/drivers/mmc/host/dw_mmc-k3.c
@@ -138,15 +138,13 @@ static int dw_mci_hi6220_parse_dt(struct dw_mci *host)
 	return 0;
 }
 
-static int dw_mci_hi6220_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios)
+static int dw_mci_hi6220_switch_voltage(struct dw_mci *host, struct mmc_ios *ios)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
 	struct k3_priv *priv;
-	struct dw_mci *host;
+	struct mmc_host *mmc = host->mmc;
 	int min_uv, max_uv;
 	int ret;
 
-	host = slot->host;
 	priv = host->priv;
 
 	if (!priv || !priv->reg)
@@ -199,7 +197,7 @@ static void dw_mci_hi6220_set_ios(struct dw_mci *host, struct mmc_ios *ios)
 	host->bus_hz = clk_get_rate(host->biu_clk);
 }
 
-static int dw_mci_hi6220_execute_tuning(struct dw_mci_slot *slot, u32 opcode)
+static int dw_mci_hi6220_execute_tuning(struct dw_mci *host, u32 opcode)
 {
 	return 0;
 }
@@ -364,11 +362,10 @@ static int dw_mci_get_best_clksmpl(unsigned int sample_flag)
 	return middle_range;
 }
 
-static int dw_mci_hi3660_execute_tuning(struct dw_mci_slot *slot, u32 opcode)
+static int dw_mci_hi3660_execute_tuning(struct dw_mci *host, u32 opcode)
 {
 	int i = 0;
-	struct dw_mci *host = slot->host;
-	struct mmc_host *mmc = slot->mmc;
+	struct mmc_host *mmc = host->mmc;
 	int smpl_phase = 0;
 	u32 tuning_sample_flag = 0;
 	int best_clksmpl = 0;
@@ -398,15 +395,13 @@ static int dw_mci_hi3660_execute_tuning(struct dw_mci_slot *slot, u32 opcode)
 	return 0;
 }
 
-static int dw_mci_hi3660_switch_voltage(struct mmc_host *mmc,
+static int dw_mci_hi3660_switch_voltage(struct dw_mci *host,
 					struct mmc_ios *ios)
 {
-	int ret = 0;
-	struct dw_mci_slot *slot = mmc_priv(mmc);
 	struct k3_priv *priv;
-	struct dw_mci *host;
+	struct mmc_host *mmc = host->mmc;
+	int ret = 0;
 
-	host = slot->host;
 	priv = host->priv;
 
 	if (!priv || !priv->reg)
diff --git a/drivers/mmc/host/dw_mmc-pltfm.c b/drivers/mmc/host/dw_mmc-pltfm.c
index de820ffd2133..af947e109d50 100644
--- a/drivers/mmc/host/dw_mmc-pltfm.c
+++ b/drivers/mmc/host/dw_mmc-pltfm.c
@@ -45,6 +45,7 @@ int dw_mci_pltfm_register(struct platform_device *pdev,
 	host->dev = &pdev->dev;
 	host->irq_flags = 0;
 	host->pdata = pdev->dev.platform_data;
+	host->pdev = pdev;
 
 	host->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &regs);
 	if (IS_ERR(host->regs))
@@ -53,7 +54,6 @@ int dw_mci_pltfm_register(struct platform_device *pdev,
 	/* Get registers' physical base address */
 	host->phy_regs = regs->start;
 
-	platform_set_drvdata(pdev, host);
 	return dw_mci_probe(host);
 }
 EXPORT_SYMBOL_GPL(dw_mci_pltfm_register);
diff --git a/drivers/mmc/host/dw_mmc-rockchip.c b/drivers/mmc/host/dw_mmc-rockchip.c
index 62c68cda1e21..879188fa6c1b 100644
--- a/drivers/mmc/host/dw_mmc-rockchip.c
+++ b/drivers/mmc/host/dw_mmc-rockchip.c
@@ -282,11 +282,10 @@ static void dw_mci_rk3288_set_ios(struct dw_mci *host, struct mmc_ios *ios)
 #define TUNING_ITERATION_TO_PHASE(i, num_phases) \
 		(DIV_ROUND_UP((i) * 360, num_phases))
 
-static int dw_mci_rk3288_execute_tuning(struct dw_mci_slot *slot, u32 opcode)
+static int dw_mci_rk3288_execute_tuning(struct dw_mci *host, u32 opcode)
 {
-	struct dw_mci *host = slot->host;
 	struct dw_mci_rockchip_priv_data *priv = host->priv;
-	struct mmc_host *mmc = slot->mmc;
+	struct mmc_host *mmc = host->mmc;
 	int ret = 0;
 	int i;
 	bool v, prev_v = 0, first_v;
@@ -475,8 +474,8 @@ static int dw_mci_rockchip_init(struct dw_mci *host)
 	struct dw_mci_rockchip_priv_data *priv = host->priv;
 	int ret, i;
 
-	/* It is slot 8 on Rockchip SoCs */
-	host->sdio_id0 = 8;
+	/* SDIO irq is the 8th on Rockchip SoCs */
+	host->sdio_irq = 8;
 
 	if (of_device_is_compatible(host->dev->of_node, "rockchip,rk3288-dw-mshc")) {
 		host->bus_hz /= RK3288_CLKGEN_DIV;
diff --git a/drivers/mmc/host/dw_mmc-starfive.c b/drivers/mmc/host/dw_mmc-starfive.c
index 34964b0dab21..11472a641cbe 100644
--- a/drivers/mmc/host/dw_mmc-starfive.c
+++ b/drivers/mmc/host/dw_mmc-starfive.c
@@ -53,11 +53,10 @@ static void dw_mci_starfive_set_sample_phase(struct dw_mci *host, u32 smpl_phase
 	mdelay(1);
 }
 
-static int dw_mci_starfive_execute_tuning(struct dw_mci_slot *slot,
+static int dw_mci_starfive_execute_tuning(struct dw_mci *host,
 					     u32 opcode)
 {
 	static const int grade  = MAX_DELAY_CHAIN;
-	struct dw_mci *host = slot->host;
 	int smpl_phase, smpl_raise = -1, smpl_fall = -1;
 	int ret;
 
@@ -65,7 +64,7 @@ static int dw_mci_starfive_execute_tuning(struct dw_mci_slot *slot,
 		dw_mci_starfive_set_sample_phase(host, smpl_phase);
 		mci_writel(host, RINTSTS, ALL_INT_CLR);
 
-		ret = mmc_send_tuning(slot->mmc, opcode, NULL);
+		ret = mmc_send_tuning(host->mmc, opcode, NULL);
 
 		if (!ret && smpl_raise < 0) {
 			smpl_raise = smpl_phase;
diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c
index 9e74b675e92d..8a0362257a2c 100644
--- a/drivers/mmc/host/dw_mmc.c
+++ b/drivers/mmc/host/dw_mmc.c
@@ -7,35 +7,28 @@
  * Copyright (C) 2009, 2010 Imagination Technologies Ltd.
  */
 
-#include <linux/blkdev.h>
+#include <linux/bitops.h>
 #include <linux/clk.h>
 #include <linux/debugfs.h>
+#include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/dma-mapping.h>
 #include <linux/err.h>
-#include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/iopoll.h>
-#include <linux/ioport.h>
-#include <linux/ktime.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
-#include <linux/prandom.h>
-#include <linux/seq_file.h>
-#include <linux/slab.h>
-#include <linux/stat.h>
-#include <linux/delay.h>
 #include <linux/irq.h>
+#include <linux/ktime.h>
 #include <linux/mmc/card.h>
 #include <linux/mmc/host.h>
 #include <linux/mmc/mmc.h>
 #include <linux/mmc/sd.h>
 #include <linux/mmc/sdio.h>
-#include <linux/bitops.h>
-#include <linux/regulator/consumer.h>
-#include <linux/of.h>
 #include <linux/mmc/slot-gpio.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regulator/consumer.h>
 
 #include "dw_mmc.h"
 
@@ -107,15 +100,15 @@ struct idmac_desc {
 #if defined(CONFIG_DEBUG_FS)
 static int dw_mci_req_show(struct seq_file *s, void *v)
 {
-	struct dw_mci_slot *slot = s->private;
+	struct dw_mci *host = s->private;
 	struct mmc_request *mrq;
 	struct mmc_command *cmd;
 	struct mmc_command *stop;
 	struct mmc_data	*data;
 
 	/* Make sure we get a consistent snapshot */
-	spin_lock_bh(&slot->host->lock);
-	mrq = slot->mrq;
+	spin_lock_bh(&host->lock);
+	mrq = host->mrq;
 
 	if (mrq) {
 		cmd = mrq->cmd;
@@ -140,7 +133,7 @@ static int dw_mci_req_show(struct seq_file *s, void *v)
 				   stop->resp[2], stop->error);
 	}
 
-	spin_unlock_bh(&slot->host->lock);
+	spin_unlock_bh(&host->lock);
 
 	return 0;
 }
@@ -165,10 +158,9 @@ static int dw_mci_regs_show(struct seq_file *s, void *v)
 }
 DEFINE_SHOW_ATTRIBUTE(dw_mci_regs);
 
-static void dw_mci_init_debugfs(struct dw_mci_slot *slot)
+static void dw_mci_init_debugfs(struct dw_mci *host)
 {
-	struct mmc_host	*mmc = slot->mmc;
-	struct dw_mci *host = slot->host;
+	struct mmc_host *mmc = host->mmc;
 	struct dentry *root;
 
 	root = mmc->debugfs_root;
@@ -176,7 +168,7 @@ static void dw_mci_init_debugfs(struct dw_mci_slot *slot)
 		return;
 
 	debugfs_create_file("regs", 0400, root, host, &dw_mci_regs_fops);
-	debugfs_create_file("req", 0400, root, slot, &dw_mci_req_fops);
+	debugfs_create_file("req", 0400, root, host, &dw_mci_req_fops);
 	debugfs_create_u32("state", 0400, root, &host->state);
 	debugfs_create_xul("pending_events", 0400, root,
 			   &host->pending_events);
@@ -231,9 +223,8 @@ static void dw_mci_wait_while_busy(struct dw_mci *host, u32 cmd_flags)
 	}
 }
 
-static void mci_send_cmd(struct dw_mci_slot *slot, u32 cmd, u32 arg)
+static void mci_send_cmd(struct dw_mci *host, u32 cmd, u32 arg)
 {
-	struct dw_mci *host = slot->host;
 	unsigned int cmd_status = 0;
 
 	mci_writel(host, CMDARG, arg);
@@ -244,15 +235,14 @@ static void mci_send_cmd(struct dw_mci_slot *slot, u32 cmd, u32 arg)
 	if (readl_poll_timeout_atomic(host->regs + SDMMC_CMD, cmd_status,
 				      !(cmd_status & SDMMC_CMD_START),
 				      1, 500 * USEC_PER_MSEC))
-		dev_err(&slot->mmc->class_dev,
+		dev_err(&host->mmc->class_dev,
 			"Timeout sending command (cmd %#x arg %#x status %#x)\n",
 			cmd, arg, cmd_status);
 }
 
 static u32 dw_mci_prepare_command(struct mmc_host *mmc, struct mmc_command *cmd)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	struct dw_mci *host = slot->host;
+	struct dw_mci *host = mmc_priv(mmc);
 	u32 cmdr;
 
 	cmd->error = -EINPROGRESS;
@@ -274,8 +264,8 @@ static u32 dw_mci_prepare_command(struct mmc_host *mmc, struct mmc_command *cmd)
 		cmdr |= SDMMC_CMD_VOLT_SWITCH;
 
 		/* Change state to continue to handle CMD11 weirdness */
-		WARN_ON(slot->host->state != STATE_SENDING_CMD);
-		slot->host->state = STATE_SENDING_CMD11;
+		WARN_ON(host->state != STATE_SENDING_CMD);
+		host->state = STATE_SENDING_CMD11;
 
 		/*
 		 * We need to disable low power mode (automatic clock stop)
@@ -289,9 +279,9 @@ static u32 dw_mci_prepare_command(struct mmc_host *mmc, struct mmc_command *cmd)
 		 * until the voltage change is all done.
 		 */
 		clk_en_a = mci_readl(host, CLKENA);
-		clk_en_a &= ~(SDMMC_CLKEN_LOW_PWR << slot->id);
+		clk_en_a &= ~SDMMC_CLKEN_LOW_PWR;
 		mci_writel(host, CLKENA, clk_en_a);
-		mci_send_cmd(slot, SDMMC_CMD_UPD_CLK |
+		mci_send_cmd(host, SDMMC_CMD_UPD_CLK |
 			     SDMMC_CMD_PRV_DAT_WAIT, 0);
 	}
 
@@ -311,7 +301,7 @@ static u32 dw_mci_prepare_command(struct mmc_host *mmc, struct mmc_command *cmd)
 			cmdr |= SDMMC_CMD_DAT_WR;
 	}
 
-	if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &slot->flags))
+	if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &host->flags))
 		cmdr |= SDMMC_CMD_USE_HOLD_REG;
 
 	return cmdr;
@@ -350,7 +340,7 @@ static u32 dw_mci_prep_stop_abort(struct dw_mci *host, struct mmc_command *cmd)
 	cmdr = stop->opcode | SDMMC_CMD_STOP |
 		SDMMC_CMD_RESP_CRC | SDMMC_CMD_RESP_EXP;
 
-	if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &host->slot->flags))
+	if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &host->flags))
 		cmdr |= SDMMC_CMD_USE_HOLD_REG;
 
 	return cmdr;
@@ -480,7 +470,7 @@ static void dw_mci_dmac_complete_dma(void *arg)
 	if ((host->use_dma == TRANS_MODE_EDMAC) &&
 	    data && (data->flags & MMC_DATA_READ))
 		/* Invalidate cache after read */
-		dma_sync_sg_for_cpu(mmc_dev(host->slot->mmc),
+		dma_sync_sg_for_cpu(mmc_dev(host->mmc),
 				    data->sg,
 				    data->sg_len,
 				    DMA_FROM_DEVICE);
@@ -575,23 +565,26 @@ static int dw_mci_idmac_init(struct dw_mci *host)
 	return 0;
 }
 
-static inline int dw_mci_prepare_desc64(struct dw_mci *host,
-					 struct mmc_data *data,
-					 unsigned int sg_len)
+static inline int dw_mci_prepare_desc(struct dw_mci *host, struct mmc_data *data,
+				      unsigned int sg_len, bool is_64bit)
 {
 	unsigned int desc_len;
-	struct idmac_desc_64addr *desc_first, *desc_last, *desc;
-	u32 val;
-	int i;
+	struct idmac_desc *desc_first, *desc_last, *desc;
+	struct idmac_desc_64addr *desc64_first, *desc64_last, *desc64;
+	u32 val, des0;
+	int i, err;
 
-	desc_first = desc_last = desc = host->sg_cpu;
+	if (is_64bit)
+		desc64_first = desc64_last = desc64 = host->sg_cpu;
+	else
+		desc_first = desc_last = desc = host->sg_cpu;
 
 	for (i = 0; i < sg_len; i++) {
 		unsigned int length = sg_dma_len(&data->sg[i]);
 
 		u64 mem_addr = sg_dma_address(&data->sg[i]);
 
-		for ( ; length ; desc++) {
+		while (length > 0) {
 			desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ?
 				   length : DW_MCI_DESC_DATA_LENGTH;
 
@@ -603,113 +596,60 @@ static inline int dw_mci_prepare_desc64(struct dw_mci *host,
 			 * isn't still owned by IDMAC as IDMAC's write
 			 * ops and CPU's read ops are asynchronous.
 			 */
-			if (readl_poll_timeout_atomic(&desc->des0, val,
-						!(val & IDMAC_DES0_OWN),
-						10, 100 * USEC_PER_MSEC))
+			if (is_64bit)
+				err = readl_poll_timeout_atomic(&desc64->des0, val,
+					IDMAC_OWN_CLR64(val), 10, 100 * USEC_PER_MSEC);
+			else
+				err = readl_poll_timeout_atomic(&desc->des0, val,
+					IDMAC_OWN_CLR64(val), 10, 100 * USEC_PER_MSEC);
+			if (err)
 				goto err_own_bit;
 
-			/*
-			 * Set the OWN bit and disable interrupts
-			 * for this descriptor
-			 */
-			desc->des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC |
-						IDMAC_DES0_CH;
-
-			/* Buffer length */
-			IDMAC_64ADDR_SET_BUFFER1_SIZE(desc, desc_len);
-
-			/* Physical address to DMA to/from */
-			desc->des4 = mem_addr & 0xffffffff;
-			desc->des5 = mem_addr >> 32;
-
-			/* Update physical address for the next desc */
-			mem_addr += desc_len;
-
-			/* Save pointer to the last descriptor */
-			desc_last = desc;
-		}
-	}
-
-	/* Set first descriptor */
-	desc_first->des0 |= IDMAC_DES0_FD;
-
-	/* Set last descriptor */
-	desc_last->des0 &= ~(IDMAC_DES0_CH | IDMAC_DES0_DIC);
-	desc_last->des0 |= IDMAC_DES0_LD;
-
-	return 0;
-err_own_bit:
-	/* restore the descriptor chain as it's polluted */
-	dev_dbg(host->dev, "descriptor is still owned by IDMAC.\n");
-	memset(host->sg_cpu, 0, DESC_RING_BUF_SZ);
-	dw_mci_idmac_init(host);
-	return -EINVAL;
-}
-
-
-static inline int dw_mci_prepare_desc32(struct dw_mci *host,
-					 struct mmc_data *data,
-					 unsigned int sg_len)
-{
-	unsigned int desc_len;
-	struct idmac_desc *desc_first, *desc_last, *desc;
-	u32 val;
-	int i;
-
-	desc_first = desc_last = desc = host->sg_cpu;
-
-	for (i = 0; i < sg_len; i++) {
-		unsigned int length = sg_dma_len(&data->sg[i]);
-
-		u32 mem_addr = sg_dma_address(&data->sg[i]);
-
-		for ( ; length ; desc++) {
-			desc_len = (length <= DW_MCI_DESC_DATA_LENGTH) ?
-				   length : DW_MCI_DESC_DATA_LENGTH;
-
-			length -= desc_len;
-
-			/*
-			 * Wait for the former clear OWN bit operation
-			 * of IDMAC to make sure that this descriptor
-			 * isn't still owned by IDMAC as IDMAC's write
-			 * ops and CPU's read ops are asynchronous.
-			 */
-			if (readl_poll_timeout_atomic(&desc->des0, val,
-						      IDMAC_OWN_CLR64(val),
-						      10,
-						      100 * USEC_PER_MSEC))
-				goto err_own_bit;
+			des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC | IDMAC_DES0_CH;
+			if (is_64bit)
+				desc64->des0 = des0;
+			else
+				desc->des0 = cpu_to_le32(des0);
 
 			/*
-			 * Set the OWN bit and disable interrupts
-			 * for this descriptor
+			 * 1. Set OWN bit and disable interrupts for this descriptor
+			 * 2. Set Buffer length
+			 * Set physical address to DMA to/from
 			 */
-			desc->des0 = cpu_to_le32(IDMAC_DES0_OWN |
-						 IDMAC_DES0_DIC |
-						 IDMAC_DES0_CH);
-
-			/* Buffer length */
-			IDMAC_SET_BUFFER1_SIZE(desc, desc_len);
-
-			/* Physical address to DMA to/from */
-			desc->des2 = cpu_to_le32(mem_addr);
+			if (is_64bit) {
+				desc64->des0 = IDMAC_DES0_OWN | IDMAC_DES0_DIC | IDMAC_DES0_CH;
+				IDMAC_64ADDR_SET_BUFFER1_SIZE(desc64, desc_len);
+				desc64->des4 = mem_addr & 0xffffffff;
+				desc64->des5 = mem_addr >> 32;
+			} else {
+				IDMAC_SET_BUFFER1_SIZE(desc, desc_len);
+				desc->des2 = cpu_to_le32(mem_addr);
+			}
 
 			/* Update physical address for the next desc */
 			mem_addr += desc_len;
 
 			/* Save pointer to the last descriptor */
-			desc_last = desc;
+			if (is_64bit) {
+				desc64_last = desc64;
+				desc64++;
+			} else {
+				desc_last = desc;
+				desc++;
+			}
 		}
 	}
 
-	/* Set first descriptor */
-	desc_first->des0 |= cpu_to_le32(IDMAC_DES0_FD);
-
-	/* Set last descriptor */
-	desc_last->des0 &= cpu_to_le32(~(IDMAC_DES0_CH |
-				       IDMAC_DES0_DIC));
-	desc_last->des0 |= cpu_to_le32(IDMAC_DES0_LD);
+	/* Set the first descriptor and the last descriptor */
+	if (is_64bit) {
+		desc64_first->des0 |= IDMAC_DES0_FD;
+		desc64_last->des0 &= ~(IDMAC_DES0_CH | IDMAC_DES0_DIC);
+		desc64_last->des0 |= IDMAC_DES0_LD;
+	} else {
+		desc_first->des0 |= cpu_to_le32(IDMAC_DES0_FD);
+		desc_last->des0 &= cpu_to_le32(~(IDMAC_DES0_CH | IDMAC_DES0_DIC));
+		desc_last->des0 |= cpu_to_le32(IDMAC_DES0_LD);
+	}
 
 	return 0;
 err_own_bit:
@@ -725,11 +665,7 @@ static int dw_mci_idmac_start_dma(struct dw_mci *host, unsigned int sg_len)
 	u32 temp;
 	int ret;
 
-	if (host->dma_64bit_address == 1)
-		ret = dw_mci_prepare_desc64(host, host->data, sg_len);
-	else
-		ret = dw_mci_prepare_desc32(host, host->data, sg_len);
-
+	ret = dw_mci_prepare_desc(host, host->data, sg_len, host->dma_64bit_address);
 	if (ret)
 		goto out;
 
@@ -823,7 +759,7 @@ static int dw_mci_edmac_start_dma(struct dw_mci *host,
 
 	/* Flush cache before write */
 	if (host->data->flags & MMC_DATA_WRITE)
-		dma_sync_sg_for_device(mmc_dev(host->slot->mmc), sgl,
+		dma_sync_sg_for_device(mmc_dev(host->mmc), sgl,
 				       sg_elems, DMA_TO_DEVICE);
 
 	dma_async_issue_pending(host->dms->ch);
@@ -913,16 +849,16 @@ static int dw_mci_pre_dma_transfer(struct dw_mci *host,
 static void dw_mci_pre_req(struct mmc_host *mmc,
 			   struct mmc_request *mrq)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
+	struct dw_mci *host = mmc_priv(mmc);
 	struct mmc_data *data = mrq->data;
 
-	if (!slot->host->use_dma || !data)
+	if (!host->use_dma || !data)
 		return;
 
 	/* This data might be unmapped at this time */
 	data->host_cookie = COOKIE_UNMAPPED;
 
-	if (dw_mci_pre_dma_transfer(slot->host, mrq->data,
+	if (dw_mci_pre_dma_transfer(host, mrq->data,
 				COOKIE_PRE_MAPPED) < 0)
 		data->host_cookie = COOKIE_UNMAPPED;
 }
@@ -931,14 +867,14 @@ static void dw_mci_post_req(struct mmc_host *mmc,
 			    struct mmc_request *mrq,
 			    int err)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
+	struct dw_mci *host = mmc_priv(mmc);
 	struct mmc_data *data = mrq->data;
 
-	if (!slot->host->use_dma || !data)
+	if (!host->use_dma || !data)
 		return;
 
 	if (data->host_cookie != COOKIE_UNMAPPED)
-		dma_unmap_sg(slot->host->dev,
+		dma_unmap_sg(host->dev,
 			     data->sg,
 			     data->sg_len,
 			     mmc_get_dma_dir(data));
@@ -948,8 +884,7 @@ static void dw_mci_post_req(struct mmc_host *mmc,
 static int dw_mci_get_cd(struct mmc_host *mmc)
 {
 	int present;
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	struct dw_mci *host = slot->host;
+	struct dw_mci *host = mmc_priv(mmc);
 	int gpio_cd = mmc_gpio_get_cd(mmc);
 
 	/* Use platform get_cd function, else try onboard card detect */
@@ -957,7 +892,7 @@ static int dw_mci_get_cd(struct mmc_host *mmc)
 				|| !mmc_card_is_removable(mmc))) {
 		present = 1;
 
-		if (!test_bit(DW_MMC_CARD_PRESENT, &slot->flags)) {
+		if (!test_bit(DW_MMC_CARD_PRESENT, &host->flags)) {
 			if (mmc->caps & MMC_CAP_NEEDS_POLL) {
 				dev_info(&mmc->class_dev,
 					"card is polling.\n");
@@ -965,21 +900,21 @@ static int dw_mci_get_cd(struct mmc_host *mmc)
 				dev_info(&mmc->class_dev,
 					"card is non-removable.\n");
 			}
-			set_bit(DW_MMC_CARD_PRESENT, &slot->flags);
+			set_bit(DW_MMC_CARD_PRESENT, &host->flags);
 		}
 
 		return present;
 	} else if (gpio_cd >= 0)
 		present = gpio_cd;
 	else
-		present = (mci_readl(slot->host, CDETECT) & (1 << slot->id))
+		present = (mci_readl(host, CDETECT) & BIT(0))
 			== 0 ? 1 : 0;
 
 	spin_lock_bh(&host->lock);
-	if (present && !test_and_set_bit(DW_MMC_CARD_PRESENT, &slot->flags))
+	if (present && !test_and_set_bit(DW_MMC_CARD_PRESENT, &host->flags))
 		dev_dbg(&mmc->class_dev, "card is present\n");
 	else if (!present &&
-			!test_and_clear_bit(DW_MMC_CARD_PRESENT, &slot->flags))
+			!test_and_clear_bit(DW_MMC_CARD_PRESENT, &host->flags))
 		dev_dbg(&mmc->class_dev, "card is not present\n");
 	spin_unlock_bh(&host->lock);
 
@@ -1200,10 +1135,9 @@ static void dw_mci_submit_data(struct dw_mci *host, struct mmc_data *data)
 	}
 }
 
-static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)
+static void dw_mci_setup_bus(struct dw_mci *host, bool force_clkinit)
 {
-	struct dw_mci *host = slot->host;
-	unsigned int clock = slot->clock;
+	unsigned int clock = host->clock;
 	u32 div;
 	u32 clk_en_a;
 	u32 sdmmc_cmd_bits = SDMMC_CMD_UPD_CLK | SDMMC_CMD_PRV_DAT_WAIT;
@@ -1212,11 +1146,11 @@ static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)
 	if (host->state == STATE_WAITING_CMD11_DONE)
 		sdmmc_cmd_bits |= SDMMC_CMD_VOLT_SWITCH;
 
-	slot->mmc->actual_clock = 0;
+	host->mmc->actual_clock = 0;
 
 	if (!clock) {
 		mci_writel(host, CLKENA, 0);
-		mci_send_cmd(slot, sdmmc_cmd_bits, 0);
+		mci_send_cmd(host, sdmmc_cmd_bits, 0);
 	} else if (clock != host->current_speed || force_clkinit) {
 		div = host->bus_hz / clock;
 		if (host->bus_hz % clock && host->bus_hz > clock)
@@ -1228,14 +1162,14 @@ static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)
 
 		div = (host->bus_hz != clock) ? DIV_ROUND_UP(div, 2) : 0;
 
-		if ((clock != slot->__clk_old &&
-			!test_bit(DW_MMC_CARD_NEEDS_POLL, &slot->flags)) ||
+		if ((clock != host->clk_old &&
+			!test_bit(DW_MMC_CARD_NEEDS_POLL, &host->flags)) ||
 			force_clkinit) {
 			/* Silent the verbose log if calling from PM context */
 			if (!force_clkinit)
-				dev_info(&slot->mmc->class_dev,
-					 "Bus speed (slot %d) = %dHz (slot req %dHz, actual %dHZ div = %d)\n",
-					 slot->id, host->bus_hz, clock,
+				dev_info(&host->mmc->class_dev,
+					 "Bus speed = %dHz (req %dHz, actual %dHZ div = %d)\n",
+					 host->bus_hz, clock,
 					 div ? ((host->bus_hz / div) >> 1) :
 					 host->bus_hz, div);
 
@@ -1243,9 +1177,9 @@ static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)
 			 * If card is polling, display the message only
 			 * one time at boot time.
 			 */
-			if (slot->mmc->caps & MMC_CAP_NEEDS_POLL &&
-					slot->mmc->f_min == clock)
-				set_bit(DW_MMC_CARD_NEEDS_POLL, &slot->flags);
+			if (host->mmc->caps & MMC_CAP_NEEDS_POLL &&
+					host->mmc->f_min == clock)
+				set_bit(DW_MMC_CARD_NEEDS_POLL, &host->flags);
 		}
 
 		/* disable clock */
@@ -1253,33 +1187,33 @@ static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)
 		mci_writel(host, CLKSRC, 0);
 
 		/* inform CIU */
-		mci_send_cmd(slot, sdmmc_cmd_bits, 0);
+		mci_send_cmd(host, sdmmc_cmd_bits, 0);
 
 		/* set clock to desired speed */
 		mci_writel(host, CLKDIV, div);
 
 		/* inform CIU */
-		mci_send_cmd(slot, sdmmc_cmd_bits, 0);
+		mci_send_cmd(host, sdmmc_cmd_bits, 0);
 
 		/* enable clock; only low power if no SDIO */
-		clk_en_a = SDMMC_CLKEN_ENABLE << slot->id;
-		if (!test_bit(DW_MMC_CARD_NO_LOW_PWR, &slot->flags))
-			clk_en_a |= SDMMC_CLKEN_LOW_PWR << slot->id;
+		clk_en_a = SDMMC_CLKEN_ENABLE;
+		if (!test_bit(DW_MMC_CARD_NO_LOW_PWR, &host->flags))
+			clk_en_a |= SDMMC_CLKEN_LOW_PWR;
 		mci_writel(host, CLKENA, clk_en_a);
 
 		/* inform CIU */
-		mci_send_cmd(slot, sdmmc_cmd_bits, 0);
+		mci_send_cmd(host, sdmmc_cmd_bits, 0);
 
 		/* keep the last clock value that was requested from core */
-		slot->__clk_old = clock;
-		slot->mmc->actual_clock = div ? ((host->bus_hz / div) >> 1) :
+		host->clk_old = clock;
+		host->mmc->actual_clock = div ? ((host->bus_hz / div) >> 1) :
 					  host->bus_hz;
 	}
 
 	host->current_speed = clock;
 
-	/* Set the current slot bus width */
-	mci_writel(host, CTYPE, (slot->ctype << slot->id));
+	/* Set bus width */
+	mci_writel(host, CTYPE, host->ctype);
 }
 
 static void dw_mci_set_data_timeout(struct dw_mci *host,
@@ -1314,14 +1248,13 @@ static void dw_mci_set_data_timeout(struct dw_mci *host,
 }
 
 static void __dw_mci_start_request(struct dw_mci *host,
-				   struct dw_mci_slot *slot,
 				   struct mmc_command *cmd)
 {
 	struct mmc_request *mrq;
 	struct mmc_data	*data;
 	u32 cmdflags;
 
-	mrq = slot->mrq;
+	mrq = host->mrq;
 
 	host->mrq = mrq;
 
@@ -1338,10 +1271,10 @@ static void __dw_mci_start_request(struct dw_mci *host,
 		mci_writel(host, BLKSIZ, data->blksz);
 	}
 
-	cmdflags = dw_mci_prepare_command(slot->mmc, cmd);
+	cmdflags = dw_mci_prepare_command(host->mmc, cmd);
 
 	/* this is the first command, send the initialization clock */
-	if (test_and_clear_bit(DW_MMC_CARD_NEED_INIT, &slot->flags))
+	if (test_and_clear_bit(DW_MMC_CARD_NEED_INIT, &host->flags))
 		cmdflags |= SDMMC_CMD_INIT;
 
 	if (data) {
@@ -1374,27 +1307,25 @@ static void __dw_mci_start_request(struct dw_mci *host,
 	host->stop_cmdr = dw_mci_prep_stop_abort(host, cmd);
 }
 
-static void dw_mci_start_request(struct dw_mci *host,
-				 struct dw_mci_slot *slot)
+static void dw_mci_start_request(struct dw_mci *host)
 {
-	struct mmc_request *mrq = slot->mrq;
+	struct mmc_request *mrq = host->mrq;
 	struct mmc_command *cmd;
 
 	cmd = mrq->sbc ? mrq->sbc : mrq->cmd;
-	__dw_mci_start_request(host, slot, cmd);
+	__dw_mci_start_request(host, cmd);
 }
 
 /* must be called with host->lock held */
-static void dw_mci_queue_request(struct dw_mci *host, struct dw_mci_slot *slot,
-				 struct mmc_request *mrq)
+static void dw_mci_queue_request(struct dw_mci *host, struct mmc_request *mrq)
 {
-	dev_vdbg(&slot->mmc->class_dev, "queue request: state=%d\n",
+	dev_vdbg(&host->mmc->class_dev, "queue request: state=%d\n",
 		 host->state);
 
-	slot->mrq = mrq;
+	host->mrq = mrq;
 
 	if (host->state == STATE_WAITING_CMD11_DONE) {
-		dev_warn(&slot->mmc->class_dev,
+		dev_warn(&host->mmc->class_dev,
 			 "Voltage change didn't complete\n");
 		/*
 		 * this case isn't expected to happen, so we can
@@ -1406,18 +1337,15 @@ static void dw_mci_queue_request(struct dw_mci *host, struct dw_mci_slot *slot,
 
 	if (host->state == STATE_IDLE) {
 		host->state = STATE_SENDING_CMD;
-		dw_mci_start_request(host, slot);
-	} else {
-		list_add_tail(&slot->queue_node, &host->queue);
+		dw_mci_start_request(host);
 	}
 }
 
 static void dw_mci_request(struct mmc_host *mmc, struct mmc_request *mrq)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	struct dw_mci *host = slot->host;
+	struct dw_mci *host = mmc_priv(mmc);
 
-	WARN_ON(slot->mrq);
+	WARN_ON(host->mrq);
 
 	/*
 	 * The check for card presence and queueing of the request must be
@@ -1433,141 +1361,117 @@ static void dw_mci_request(struct mmc_host *mmc, struct mmc_request *mrq)
 
 	spin_lock_bh(&host->lock);
 
-	dw_mci_queue_request(host, slot, mrq);
+	dw_mci_queue_request(host, mrq);
 
 	spin_unlock_bh(&host->lock);
 }
 
 static void dw_mci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	const struct dw_mci_drv_data *drv_data = slot->host->drv_data;
+	struct dw_mci *host = mmc_priv(mmc);
+	const struct dw_mci_drv_data *drv_data = host->drv_data;
 	u32 regs;
 	int ret;
 
 	switch (ios->bus_width) {
 	case MMC_BUS_WIDTH_4:
-		slot->ctype = SDMMC_CTYPE_4BIT;
+		host->ctype = SDMMC_CTYPE_4BIT;
 		break;
 	case MMC_BUS_WIDTH_8:
-		slot->ctype = SDMMC_CTYPE_8BIT;
+		host->ctype = SDMMC_CTYPE_8BIT;
 		break;
 	default:
 		/* set default 1 bit mode */
-		slot->ctype = SDMMC_CTYPE_1BIT;
+		host->ctype = SDMMC_CTYPE_1BIT;
 	}
 
-	regs = mci_readl(slot->host, UHS_REG);
+	regs = mci_readl(host, UHS_REG);
 
 	/* DDR mode set */
 	if (ios->timing == MMC_TIMING_MMC_DDR52 ||
 	    ios->timing == MMC_TIMING_UHS_DDR50 ||
 	    ios->timing == MMC_TIMING_MMC_HS400)
-		regs |= ((0x1 << slot->id) << 16);
+		regs |= BIT(16);
 	else
-		regs &= ~((0x1 << slot->id) << 16);
+		regs &= ~BIT(16);
 
-	mci_writel(slot->host, UHS_REG, regs);
-	slot->host->timing = ios->timing;
+	mci_writel(host, UHS_REG, regs);
+	host->timing = ios->timing;
 
 	/*
 	 * Use mirror of ios->clock to prevent race with mmc
 	 * core ios update when finding the minimum.
 	 */
-	slot->clock = ios->clock;
+	host->clock = ios->clock;
 
 	if (drv_data && drv_data->set_ios)
-		drv_data->set_ios(slot->host, ios);
+		drv_data->set_ios(host, ios);
 
 	switch (ios->power_mode) {
 	case MMC_POWER_UP:
-		if (!IS_ERR(mmc->supply.vmmc)) {
-			ret = mmc_regulator_set_ocr(mmc, mmc->supply.vmmc,
-					ios->vdd);
-			if (ret) {
-				dev_err(slot->host->dev,
-					"failed to enable vmmc regulator\n");
-				/*return, if failed turn on vmmc*/
-				return;
-			}
+		ret = mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, ios->vdd);
+		if (ret) {
+			dev_err(host->dev, "failed to enable vmmc regulator\n");
+			return;
 		}
-		set_bit(DW_MMC_CARD_NEED_INIT, &slot->flags);
-		regs = mci_readl(slot->host, PWREN);
-		regs |= (1 << slot->id);
-		mci_writel(slot->host, PWREN, regs);
+		set_bit(DW_MMC_CARD_NEED_INIT, &host->flags);
+		regs = mci_readl(host, PWREN);
+		regs |= BIT(0);
+		mci_writel(host, PWREN, regs);
 		break;
 	case MMC_POWER_ON:
-		if (!slot->host->vqmmc_enabled) {
-			if (!IS_ERR(mmc->supply.vqmmc)) {
-				ret = regulator_enable(mmc->supply.vqmmc);
-				if (ret < 0)
-					dev_err(slot->host->dev,
-						"failed to enable vqmmc\n");
-				else
-					slot->host->vqmmc_enabled = true;
-
-			} else {
-				/* Keep track so we don't reset again */
-				slot->host->vqmmc_enabled = true;
-			}
-
-			/* Reset our state machine after powering on */
-			dw_mci_ctrl_reset(slot->host,
-					  SDMMC_CTRL_ALL_RESET_FLAGS);
-		}
-
+		mmc_regulator_enable_vqmmc(mmc);
 		/* Adjust clock / bus width after power is up */
-		dw_mci_setup_bus(slot, false);
+		dw_mci_setup_bus(host, false);
 
 		break;
 	case MMC_POWER_OFF:
 		/* Turn clock off before power goes down */
-		dw_mci_setup_bus(slot, false);
+		dw_mci_setup_bus(host, false);
 
 		if (!IS_ERR(mmc->supply.vmmc))
 			mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
 
-		if (!IS_ERR(mmc->supply.vqmmc) && slot->host->vqmmc_enabled)
-			regulator_disable(mmc->supply.vqmmc);
-		slot->host->vqmmc_enabled = false;
+		mmc_regulator_disable_vqmmc(mmc);
 
-		regs = mci_readl(slot->host, PWREN);
-		regs &= ~(1 << slot->id);
-		mci_writel(slot->host, PWREN, regs);
+		regs = mci_readl(host, PWREN);
+		regs &= ~BIT(0);
+		mci_writel(host, PWREN, regs);
+		/* Reset our state machine after powering off */
+		dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS);
 		break;
 	default:
 		break;
 	}
 
-	if (slot->host->state == STATE_WAITING_CMD11_DONE && ios->clock != 0)
-		slot->host->state = STATE_IDLE;
+	if (host->state == STATE_WAITING_CMD11_DONE && ios->clock != 0)
+		host->state = STATE_IDLE;
 }
 
 static int dw_mci_card_busy(struct mmc_host *mmc)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
+	struct dw_mci *host = mmc_priv(mmc);
 	u32 status;
 
 	/*
 	 * Check the busy bit which is low when DAT[3:0]
 	 * (the data lines) are 0000
 	 */
-	status = mci_readl(slot->host, STATUS);
+	status = mci_readl(host, STATUS);
 
 	return !!(status & SDMMC_STATUS_BUSY);
 }
 
 static int dw_mci_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	struct dw_mci *host = slot->host;
+	struct dw_mci *host = mmc_priv(mmc);
 	const struct dw_mci_drv_data *drv_data = host->drv_data;
 	u32 uhs;
-	u32 v18 = SDMMC_UHS_18V << slot->id;
+	u32 v18 = SDMMC_UHS_18V;
 	int ret;
 
 	if (drv_data && drv_data->switch_voltage)
-		return drv_data->switch_voltage(mmc, ios);
+		return drv_data->switch_voltage(host, ios);
 
 	/*
 	 * Program the voltage.  Note that some instances of dw_mmc may use
@@ -1597,7 +1501,7 @@ static int dw_mci_switch_voltage(struct mmc_host *mmc, struct mmc_ios *ios)
 static int dw_mci_get_ro(struct mmc_host *mmc)
 {
 	int read_only;
-	struct dw_mci_slot *slot = mmc_priv(mmc);
+	struct dw_mci *host = mmc_priv(mmc);
 	int gpio_ro = mmc_gpio_get_ro(mmc);
 
 	/* Use platform get_ro function, else try on board write protect */
@@ -1605,7 +1509,7 @@ static int dw_mci_get_ro(struct mmc_host *mmc)
 		read_only = gpio_ro;
 	else
 		read_only =
-			mci_readl(slot->host, WRTPRT) & (1 << slot->id) ? 1 : 0;
+			mci_readl(host, WRTPRT) & BIT(0) ? 1 : 0;
 
 	dev_dbg(&mmc->class_dev, "card is %s\n",
 		read_only ? "read-only" : "read-write");
@@ -1615,8 +1519,7 @@ static int dw_mci_get_ro(struct mmc_host *mmc)
 
 static void dw_mci_hw_reset(struct mmc_host *mmc)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	struct dw_mci *host = slot->host;
+	struct dw_mci *host = mmc_priv(mmc);
 	const struct dw_mci_drv_data *drv_data = host->drv_data;
 	int reset;
 
@@ -1639,18 +1542,17 @@ static void dw_mci_hw_reset(struct mmc_host *mmc)
 	 * tRSTH >= 1us:   RST_n high period
 	 */
 	reset = mci_readl(host, RST_N);
-	reset &= ~(SDMMC_RST_HWACTIVE << slot->id);
+	reset &= ~SDMMC_RST_HWACTIVE;
 	mci_writel(host, RST_N, reset);
 	usleep_range(1, 2);
-	reset |= SDMMC_RST_HWACTIVE << slot->id;
+	reset |= SDMMC_RST_HWACTIVE;
 	mci_writel(host, RST_N, reset);
 	usleep_range(200, 300);
 }
 
-static void dw_mci_prepare_sdio_irq(struct dw_mci_slot *slot, bool prepare)
+static void dw_mci_prepare_sdio_irq(struct dw_mci *host, bool prepare)
 {
-	struct dw_mci *host = slot->host;
-	const u32 clken_low_pwr = SDMMC_CLKEN_LOW_PWR << slot->id;
+	const u32 clken_low_pwr = SDMMC_CLKEN_LOW_PWR;
 	u32 clk_en_a_old;
 	u32 clk_en_a;
 
@@ -1662,23 +1564,22 @@ static void dw_mci_prepare_sdio_irq(struct dw_mci_slot *slot, bool prepare)
 
 	clk_en_a_old = mci_readl(host, CLKENA);
 	if (prepare) {
-		set_bit(DW_MMC_CARD_NO_LOW_PWR, &slot->flags);
+		set_bit(DW_MMC_CARD_NO_LOW_PWR, &host->flags);
 		clk_en_a = clk_en_a_old & ~clken_low_pwr;
 	} else {
-		clear_bit(DW_MMC_CARD_NO_LOW_PWR, &slot->flags);
+		clear_bit(DW_MMC_CARD_NO_LOW_PWR, &host->flags);
 		clk_en_a = clk_en_a_old | clken_low_pwr;
 	}
 
 	if (clk_en_a != clk_en_a_old) {
 		mci_writel(host, CLKENA, clk_en_a);
-		mci_send_cmd(slot, SDMMC_CMD_UPD_CLK | SDMMC_CMD_PRV_DAT_WAIT,
+		mci_send_cmd(host, SDMMC_CMD_UPD_CLK | SDMMC_CMD_PRV_DAT_WAIT,
 			     0);
 	}
 }
 
-static void __dw_mci_enable_sdio_irq(struct dw_mci_slot *slot, int enb)
+static void __dw_mci_enable_sdio_irq(struct dw_mci *host, int enb)
 {
-	struct dw_mci *host = slot->host;
 	unsigned long irqflags;
 	u32 int_mask;
 
@@ -1687,9 +1588,9 @@ static void __dw_mci_enable_sdio_irq(struct dw_mci_slot *slot, int enb)
 	/* Enable/disable Slot Specific SDIO interrupt */
 	int_mask = mci_readl(host, INTMASK);
 	if (enb)
-		int_mask |= SDMMC_INT_SDIO(slot->sdio_id);
+		int_mask |= SDMMC_INT_SDIO(host->sdio_irq);
 	else
-		int_mask &= ~SDMMC_INT_SDIO(slot->sdio_id);
+		int_mask &= ~SDMMC_INT_SDIO(host->sdio_irq);
 	mci_writel(host, INTMASK, int_mask);
 
 	spin_unlock_irqrestore(&host->irq_lock, irqflags);
@@ -1697,11 +1598,10 @@ static void __dw_mci_enable_sdio_irq(struct dw_mci_slot *slot, int enb)
 
 static void dw_mci_enable_sdio_irq(struct mmc_host *mmc, int enb)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	struct dw_mci *host = slot->host;
+	struct dw_mci *host = mmc_priv(mmc);
 
-	dw_mci_prepare_sdio_irq(slot, enb);
-	__dw_mci_enable_sdio_irq(slot, enb);
+	dw_mci_prepare_sdio_irq(host, enb);
+	__dw_mci_enable_sdio_irq(host, enb);
 
 	/* Avoid runtime suspending the device when SDIO IRQ is enabled */
 	if (enb)
@@ -1712,28 +1612,26 @@ static void dw_mci_enable_sdio_irq(struct mmc_host *mmc, int enb)
 
 static void dw_mci_ack_sdio_irq(struct mmc_host *mmc)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
+	struct dw_mci *host = mmc_priv(mmc);
 
-	__dw_mci_enable_sdio_irq(slot, 1);
+	__dw_mci_enable_sdio_irq(host, 1);
 }
 
 static int dw_mci_execute_tuning(struct mmc_host *mmc, u32 opcode)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	struct dw_mci *host = slot->host;
+	struct dw_mci *host = mmc_priv(mmc);
 	const struct dw_mci_drv_data *drv_data = host->drv_data;
 	int err = -EINVAL;
 
 	if (drv_data && drv_data->execute_tuning)
-		err = drv_data->execute_tuning(slot, opcode);
+		err = drv_data->execute_tuning(host, opcode);
 	return err;
 }
 
 static int dw_mci_prepare_hs400_tuning(struct mmc_host *mmc,
 				       struct mmc_ios *ios)
 {
-	struct dw_mci_slot *slot = mmc_priv(mmc);
-	struct dw_mci *host = slot->host;
+	struct dw_mci *host = mmc_priv(mmc);
 	const struct dw_mci_drv_data *drv_data = host->drv_data;
 
 	if (drv_data && drv_data->prepare_hs400_tuning)
@@ -1804,7 +1702,7 @@ static bool dw_mci_reset(struct dw_mci *host)
 
 ciu_out:
 	/* After a CTRL reset we need to have CIU set clock registers  */
-	mci_send_cmd(host->slot, SDMMC_CMD_UPD_CLK, 0);
+	mci_send_cmd(host, SDMMC_CMD_UPD_CLK, 0);
 
 	return ret;
 }
@@ -1895,29 +1793,16 @@ static void dw_mci_request_end(struct dw_mci *host, struct mmc_request *mrq)
 	__releases(&host->lock)
 	__acquires(&host->lock)
 {
-	struct dw_mci_slot *slot;
-	struct mmc_host	*prev_mmc = host->slot->mmc;
+	struct mmc_host	*prev_mmc = host->mmc;
 
 	WARN_ON(host->cmd || host->data);
 
-	host->slot->mrq = NULL;
 	host->mrq = NULL;
-	if (!list_empty(&host->queue)) {
-		slot = list_entry(host->queue.next,
-				  struct dw_mci_slot, queue_node);
-		list_del(&slot->queue_node);
-		dev_vdbg(host->dev, "list not empty: %s is next\n",
-			 mmc_hostname(slot->mmc));
-		host->state = STATE_SENDING_CMD;
-		dw_mci_start_request(host, slot);
-	} else {
-		dev_vdbg(host->dev, "list empty\n");
 
-		if (host->state == STATE_SENDING_CMD11)
-			host->state = STATE_WAITING_CMD11_DONE;
-		else
-			host->state = STATE_IDLE;
-	}
+	if (host->state == STATE_SENDING_CMD11)
+		host->state = STATE_WAITING_CMD11_DONE;
+	else
+		host->state = STATE_IDLE;
 
 	spin_unlock(&host->lock);
 	mmc_request_done(prev_mmc, mrq);
@@ -2095,8 +1980,7 @@ static void dw_mci_work_func(struct work_struct *t)
 			set_bit(EVENT_CMD_COMPLETE, &host->completed_events);
 			err = dw_mci_command_complete(host, cmd);
 			if (cmd == mrq->sbc && !err) {
-				__dw_mci_start_request(host, host->slot,
-						       mrq->cmd);
+				__dw_mci_start_request(host, mrq->cmd);
 				goto unlock;
 			}
 
@@ -2803,9 +2687,7 @@ static void dw_mci_cmd_interrupt(struct dw_mci *host, u32 status)
 
 static void dw_mci_handle_cd(struct dw_mci *host)
 {
-	struct dw_mci_slot *slot = host->slot;
-
-	mmc_detect_change(slot->mmc,
+	mmc_detect_change(host->mmc,
 		msecs_to_jiffies(host->pdata->detect_delay_ms));
 }
 
@@ -2813,7 +2695,6 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)
 {
 	struct dw_mci *host = dev_id;
 	u32 pending;
-	struct dw_mci_slot *slot = host->slot;
 
 	pending = mci_readl(host, MINTSTS); /* read-only mask reg */
 
@@ -2914,11 +2795,11 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)
 			dw_mci_handle_cd(host);
 		}
 
-		if (pending & SDMMC_INT_SDIO(slot->sdio_id)) {
+		if (pending & SDMMC_INT_SDIO(host->sdio_irq)) {
 			mci_writel(host, RINTSTS,
-				   SDMMC_INT_SDIO(slot->sdio_id));
-			__dw_mci_enable_sdio_irq(slot, 0);
-			sdio_signal_irq(slot->mmc);
+				   SDMMC_INT_SDIO(host->sdio_irq));
+			__dw_mci_enable_sdio_irq(host, 0);
+			sdio_signal_irq(host->mmc);
 		}
 
 	}
@@ -2950,11 +2831,10 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static int dw_mci_init_slot_caps(struct dw_mci_slot *slot)
+static int dw_mci_init_host_caps(struct dw_mci *host)
 {
-	struct dw_mci *host = slot->host;
 	const struct dw_mci_drv_data *drv_data = host->drv_data;
-	struct mmc_host *mmc = slot->mmc;
+	struct mmc_host *mmc = host->mmc;
 	int ctrl_id;
 
 	if (host->pdata->caps)
@@ -3002,40 +2882,40 @@ static int dw_mci_init_slot_caps(struct dw_mci_slot *slot)
 	return 0;
 }
 
-static int dw_mci_init_slot(struct dw_mci *host)
+static struct dw_mci *dw_mci_init_host(struct dw_mci *host)
 {
 	struct mmc_host *mmc;
-	struct dw_mci_slot *slot;
+	struct dw_mci *local_host;
 	int ret;
 
-	mmc = devm_mmc_alloc_host(host->dev, sizeof(*slot));
+	mmc = devm_mmc_alloc_host(host->dev, sizeof(*host));
 	if (!mmc)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
+
+	local_host = mmc_priv(mmc);
+	memcpy(local_host, host, sizeof(*host));
+	local_host->mmc = mmc;
 
-	slot = mmc_priv(mmc);
-	slot->id = 0;
-	slot->sdio_id = host->sdio_id0 + slot->id;
-	slot->mmc = mmc;
-	slot->host = host;
-	host->slot = slot;
+	if (host->pdev)
+		platform_set_drvdata(host->pdev, local_host);
 
 	mmc->ops = &dw_mci_ops;
 
 	/*if there are external regulators, get them*/
 	ret = mmc_regulator_get_supply(mmc);
 	if (ret)
-		return ret;
+		return ERR_PTR(ret);
 
 	if (!mmc->ocr_avail)
 		mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34;
 
 	ret = mmc_of_parse(mmc);
 	if (ret)
-		return ret;
+		return ERR_PTR(ret);
 
-	ret = dw_mci_init_slot_caps(slot);
+	ret = dw_mci_init_host_caps(local_host);
 	if (ret)
-		return ret;
+		return ERR_PTR(ret);
 
 	/* Useful defaults if platform data is unset. */
 	if (host->use_dma == TRANS_MODE_IDMAC) {
@@ -3065,20 +2945,19 @@ static int dw_mci_init_slot(struct dw_mci *host)
 
 	ret = mmc_add_host(mmc);
 	if (ret)
-		return ret;
+		return ERR_PTR(ret);
 
 #if defined(CONFIG_DEBUG_FS)
-	dw_mci_init_debugfs(slot);
+	dw_mci_init_debugfs(local_host);
 #endif
 
-	return 0;
+	return local_host;
 }
 
-static void dw_mci_cleanup_slot(struct dw_mci_slot *slot)
+static void dw_mci_cleanup_host(struct dw_mci *host)
 {
 	/* Debugfs stuff is cleaned up by mmc core */
-	mmc_remove_host(slot->mmc);
-	slot->host->slot = NULL;
+	mmc_remove_host(host->mmc);
 }
 
 static void dw_mci_init_dma(struct dw_mci *host)
@@ -3346,13 +3225,13 @@ static void dw_mci_enable_cd(struct dw_mci *host)
 	u32 temp;
 
 	/*
-	 * No need for CD if all slots have a non-error GPIO
+	 * No need for CD if host has a non-error GPIO
 	 * as well as broken card detection is found.
 	 */
-	if (host->slot->mmc->caps & MMC_CAP_NEEDS_POLL)
+	if (host->mmc->caps & MMC_CAP_NEEDS_POLL)
 		return;
 
-	if (mmc_gpio_get_cd(host->slot->mmc) < 0) {
+	if (mmc_gpio_get_cd(host->mmc) < 0) {
 		spin_lock_irqsave(&host->irq_lock, irqflags);
 		temp = mci_readl(host, INTMASK);
 		temp  |= SDMMC_INT_CD;
@@ -3364,6 +3243,7 @@ static void dw_mci_enable_cd(struct dw_mci *host)
 int dw_mci_probe(struct dw_mci *host)
 {
 	const struct dw_mci_drv_data *drv_data = host->drv_data;
+	struct dw_mci *local_host;
 	int width, i, ret = 0;
 	u32 fifo_size;
 
@@ -3442,7 +3322,6 @@ int dw_mci_probe(struct dw_mci *host)
 
 	spin_lock_init(&host->lock);
 	spin_lock_init(&host->irq_lock);
-	INIT_LIST_HEAD(&host->queue);
 
 	dw_mci_init_fault(host);
 
@@ -3532,12 +3411,6 @@ int dw_mci_probe(struct dw_mci *host)
 	else
 		host->fifo_reg = host->regs + DATA_240A_OFFSET;
 
-	INIT_WORK(&host->bh_work, dw_mci_work_func);
-	ret = devm_request_irq(host->dev, host->irq, dw_mci_interrupt,
-			       host->irq_flags, "dw-mci", host);
-	if (ret)
-		goto err_dmaunmap;
-
 	/*
 	 * Enable interrupts for command done, data over, data empty,
 	 * receive ready and error such as transmit, receive timeout, crc error
@@ -3552,15 +3425,20 @@ int dw_mci_probe(struct dw_mci *host)
 		 "DW MMC controller at irq %d,%d bit host data width,%u deep fifo\n",
 		 host->irq, width, fifo_size);
 
-	/* We need at least one slot to succeed */
-	ret = dw_mci_init_slot(host);
-	if (ret) {
-		dev_dbg(host->dev, "slot %d init failed\n", i);
+	local_host = dw_mci_init_host(host);
+	if (IS_ERR(local_host)) {
+		dev_dbg(host->dev, "host init failed\n");
 		goto err_dmaunmap;
 	}
 
-	/* Now that slots are all setup, we can enable card detect */
-	dw_mci_enable_cd(host);
+	INIT_WORK(&local_host->bh_work, dw_mci_work_func);
+	ret = devm_request_irq(local_host->dev, local_host->irq, dw_mci_interrupt,
+			       local_host->irq_flags, "dw-mci", local_host);
+	if (ret)
+		goto err_dmaunmap;
+
+	/* Now that host is all setup, we can enable card detect */
+	dw_mci_enable_cd(local_host);
 
 	return 0;
 
@@ -3582,9 +3460,9 @@ EXPORT_SYMBOL(dw_mci_probe);
 
 void dw_mci_remove(struct dw_mci *host)
 {
-	dev_dbg(host->dev, "remove slot\n");
-	if (host->slot)
-		dw_mci_cleanup_slot(host->slot);
+	dev_dbg(host->dev, "remove host\n");
+	if (host)
+		dw_mci_cleanup_host(host);
 
 	mci_writel(host, RINTSTS, 0xFFFFFFFF);
 	mci_writel(host, INTMASK, 0); /* disable all mmc interrupt first */
@@ -3603,8 +3481,6 @@ void dw_mci_remove(struct dw_mci *host)
 }
 EXPORT_SYMBOL(dw_mci_remove);
 
-
-
 #ifdef CONFIG_PM
 int dw_mci_runtime_suspend(struct device *dev)
 {
@@ -3615,9 +3491,9 @@ int dw_mci_runtime_suspend(struct device *dev)
 
 	clk_disable_unprepare(host->ciu_clk);
 
-	if (host->slot &&
-	    (mmc_host_can_gpio_cd(host->slot->mmc) ||
-	     !mmc_card_is_removable(host->slot->mmc)))
+	if (host &&
+	    (mmc_host_can_gpio_cd(host->mmc) ||
+	     !mmc_card_is_removable(host->mmc)))
 		clk_disable_unprepare(host->biu_clk);
 
 	return 0;
@@ -3629,9 +3505,9 @@ int dw_mci_runtime_resume(struct device *dev)
 	int ret = 0;
 	struct dw_mci *host = dev_get_drvdata(dev);
 
-	if (host->slot &&
-	    (mmc_host_can_gpio_cd(host->slot->mmc) ||
-	     !mmc_card_is_removable(host->slot->mmc))) {
+	if (host &&
+	    (mmc_host_can_gpio_cd(host->mmc) ||
+	     !mmc_card_is_removable(host->mmc))) {
 		ret = clk_prepare_enable(host->biu_clk);
 		if (ret)
 			return ret;
@@ -3667,25 +3543,25 @@ int dw_mci_runtime_resume(struct device *dev)
 	mci_writel(host, CTRL, SDMMC_CTRL_INT_ENABLE);
 
 
-	if (host->slot && host->slot->mmc->pm_flags & MMC_PM_KEEP_POWER)
-		dw_mci_set_ios(host->slot->mmc, &host->slot->mmc->ios);
+	if (host && host->mmc->pm_flags & MMC_PM_KEEP_POWER)
+		dw_mci_set_ios(host->mmc, &host->mmc->ios);
 
 	/* Force setup bus to guarantee available clock output */
-	dw_mci_setup_bus(host->slot, true);
+	dw_mci_setup_bus(host, true);
 
 	/* Re-enable SDIO interrupts. */
-	if (sdio_irq_claimed(host->slot->mmc))
-		__dw_mci_enable_sdio_irq(host->slot, 1);
+	if (sdio_irq_claimed(host->mmc))
+		__dw_mci_enable_sdio_irq(host, 1);
 
-	/* Now that slots are all setup, we can enable card detect */
+	/* Now that host is setup, we can enable card detect */
 	dw_mci_enable_cd(host);
 
 	return 0;
 
 err:
-	if (host->slot &&
-	    (mmc_host_can_gpio_cd(host->slot->mmc) ||
-	     !mmc_card_is_removable(host->slot->mmc)))
+	if (host &&
+	    (mmc_host_can_gpio_cd(host->mmc) ||
+	     !mmc_card_is_removable(host->mmc)))
 		clk_disable_unprepare(host->biu_clk);
 
 	return ret;
diff --git a/drivers/mmc/host/dw_mmc.h b/drivers/mmc/host/dw_mmc.h
index 648b4a5641bf..66f64026b5a7 100644
--- a/drivers/mmc/host/dw_mmc.h
+++ b/drivers/mmc/host/dw_mmc.h
@@ -57,14 +57,14 @@ struct dw_mci_dma_slave {
 };
 
 /**
- * struct dw_mci - MMC controller state shared between all slots
+ * struct dw_mci - MMC controller state
  * @lock: Spinlock protecting the queue and associated data.
  * @irq_lock: Spinlock protecting the INTMASK setting.
  * @regs: Pointer to MMIO registers.
  * @fifo_reg: Pointer to MMIO registers for data FIFO
  * @sg: Scatterlist entry currently being processed by PIO code, if any.
  * @sg_miter: PIO mapping scatterlist iterator.
- * @mrq: The request currently being processed on @slot,
+ * @mrq: The request currently being processed on @host,
  *	or NULL if the controller is idle.
  * @cmd: The command currently being sent to the card, or NULL.
  * @data: The data currently being transferred, or NULL if no data
@@ -96,7 +96,6 @@ struct dw_mci_dma_slave {
  * @completed_events: Bitmask of events which the state machine has
  *	processed.
  * @state: BH work state.
- * @queue: List of slots waiting for access to the controller.
  * @bus_hz: The rate of @mck in Hz. This forms the basis for MMC bus
  *	rate and timeout calculations.
  * @current_speed: Configured rate of the controller.
@@ -109,7 +108,6 @@ struct dw_mci_dma_slave {
  * @priv: Implementation defined private data.
  * @biu_clk: Pointer to bus interface unit clock instance.
  * @ciu_clk: Pointer to card interface unit clock instance.
- * @slot: Slots sharing this MMC controller.
  * @fifo_depth: depth of FIFO.
  * @data_addr_override: override fifo reg offset with this value.
  * @wm_aligned: force fifo watermark equal with data length in PIO mode.
@@ -121,23 +119,25 @@ struct dw_mci_dma_slave {
  * @push_data: Pointer to FIFO push function.
  * @pull_data: Pointer to FIFO pull function.
  * @quirks: Set of quirks that apply to specific versions of the IP.
- * @vqmmc_enabled: Status of vqmmc, should be true or false.
  * @irq_flags: The flags to be passed to request_irq.
  * @irq: The irq value to be passed to request_irq.
- * @sdio_id0: Number of slot0 in the SDIO interrupt registers.
+ * @sdio_irq: SDIO interrupt bit in interrupt registers.
  * @cmd11_timer: Timer for SD3.0 voltage switch over scheme.
  * @cto_timer: Timer for broken command transfer over scheme.
  * @dto_timer: Timer for broken data transfer over scheme.
+ * @mmc: The mmc_host representing this dw_mci.
+ * @flags: Random state bits associated with the host.
+ * @ctype: Card type for this host.
+ * @clock: Clock rate configured by set_ios(). Protected by host->lock.
+ * @clk_old: The last clock value that was requested from core.
+ * @pdev: platform_device registered
  *
  * Locking
  * =======
  *
- * @lock is a softirq-safe spinlock protecting @queue as well as
- * @slot, @mrq and @state. These must always be updated
+ * @lock is a softirq-safe spinlock protecting as well as
+ * @mrq and @state. These must always be updated
  * at the same time while holding @lock.
- * The @mrq field of struct dw_mci_slot is also protected by @lock,
- * and must always be written at the same time as the slot is added to
- * @queue.
  *
  * @irq_lock is an irq-safe spinlock protecting the INTMASK register
  * to allow the interrupt handler to modify it directly.  Held for only long
@@ -199,7 +199,6 @@ struct dw_mci {
 	unsigned long		pending_events;
 	unsigned long		completed_events;
 	enum dw_mci_state	state;
-	struct list_head	queue;
 
 	u32			bus_hz;
 	u32			current_speed;
@@ -212,7 +211,6 @@ struct dw_mci {
 	void			*priv;
 	struct clk		*biu_clk;
 	struct clk		*ciu_clk;
-	struct dw_mci_slot	*slot;
 
 	/* FIFO push and pull */
 	int			fifo_depth;
@@ -228,11 +226,10 @@ struct dw_mci {
 	void (*pull_data)(struct dw_mci *host, void *buf, int cnt);
 
 	u32			quirks;
-	bool			vqmmc_enabled;
 	unsigned long		irq_flags; /* IRQ flags */
 	int			irq;
 
-	int			sdio_id0;
+	int			sdio_irq;
 
 	struct timer_list       cmd11_timer;
 	struct timer_list       cto_timer;
@@ -242,6 +239,17 @@ struct dw_mci {
 	struct fault_attr	fail_data_crc;
 	struct hrtimer		fault_timer;
 #endif
+	struct mmc_host		*mmc;
+	unsigned long		flags;
+#define DW_MMC_CARD_PRESENT	0
+#define DW_MMC_CARD_NEED_INIT	1
+#define DW_MMC_CARD_NO_LOW_PWR	2
+#define DW_MMC_CARD_NO_USE_HOLD 3
+#define DW_MMC_CARD_NEEDS_POLL	4
+	u32			ctype;
+	unsigned int		clock;
+	unsigned int		clk_old;
+	struct platform_device	*pdev;
 };
 
 /* DMA ops for Internal/External DMAC interface */
@@ -255,8 +263,6 @@ struct dw_mci_dma_ops {
 	void (*exit)(struct dw_mci *host);
 };
 
-struct dma_pdata;
-
 /* Board platform data */
 struct dw_mci_board {
 	unsigned int bus_hz; /* Clock speed at the cclk_in pad */
@@ -276,7 +282,6 @@ struct dw_mci_board {
 
 	struct reset_control *rstc;
 	struct dw_mci_dma_ops *dma_ops;
-	struct dma_pdata *data;
 };
 
 /* Support for longer data read timeout */
@@ -546,44 +551,6 @@ static inline int dw_mci_runtime_suspend(struct device *device) { return -EOPNOT
 static inline int dw_mci_runtime_resume(struct device *device) { return -EOPNOTSUPP; }
 #endif
 
-/**
- * struct dw_mci_slot - MMC slot state
- * @mmc: The mmc_host representing this slot.
- * @host: The MMC controller this slot is using.
- * @ctype: Card type for this slot.
- * @mrq: mmc_request currently being processed or waiting to be
- *	processed, or NULL when the slot is idle.
- * @queue_node: List node for placing this node in the @queue list of
- *	&struct dw_mci.
- * @clock: Clock rate configured by set_ios(). Protected by host->lock.
- * @__clk_old: The last clock value that was requested from core.
- *	Keeping track of this helps us to avoid spamming the console.
- * @flags: Random state bits associated with the slot.
- * @id: Number of this slot.
- * @sdio_id: Number of this slot in the SDIO interrupt registers.
- */
-struct dw_mci_slot {
-	struct mmc_host		*mmc;
-	struct dw_mci		*host;
-
-	u32			ctype;
-
-	struct mmc_request	*mrq;
-	struct list_head	queue_node;
-
-	unsigned int		clock;
-	unsigned int		__clk_old;
-
-	unsigned long		flags;
-#define DW_MMC_CARD_PRESENT	0
-#define DW_MMC_CARD_NEED_INIT	1
-#define DW_MMC_CARD_NO_LOW_PWR	2
-#define DW_MMC_CARD_NO_USE_HOLD 3
-#define DW_MMC_CARD_NEEDS_POLL	4
-	int			id;
-	int			sdio_id;
-};
-
 /**
  * dw_mci driver data - dw-mshc implementation specific driver data.
  * @caps: mmc subsystem specified capabilities of the controller(s).
@@ -609,10 +576,10 @@ struct dw_mci_drv_data {
 	int		(*init)(struct dw_mci *host);
 	void		(*set_ios)(struct dw_mci *host, struct mmc_ios *ios);
 	int		(*parse_dt)(struct dw_mci *host);
-	int		(*execute_tuning)(struct dw_mci_slot *slot, u32 opcode);
+	int		(*execute_tuning)(struct dw_mci *host, u32 opcode);
 	int		(*prepare_hs400_tuning)(struct dw_mci *host,
 						struct mmc_ios *ios);
-	int		(*switch_voltage)(struct mmc_host *mmc,
+	int		(*switch_voltage)(struct dw_mci *host,
 					  struct mmc_ios *ios);
 	void		(*set_data_timeout)(struct dw_mci *host,
 					  unsigned int timeout_ns);
diff --git a/drivers/mmc/host/mtk-sd.c b/drivers/mmc/host/mtk-sd.c
index daed659f63f6..302ac8529c4f 100644
--- a/drivers/mmc/host/mtk-sd.c
+++ b/drivers/mmc/host/mtk-sd.c
@@ -203,9 +203,10 @@
 #define SDC_CFG_DTOC            GENMASK(31, 24)	/* RW */
 
 /* SDC_STS mask */
-#define SDC_STS_SDCBUSY         BIT(0)	/* RW */
-#define SDC_STS_CMDBUSY         BIT(1)	/* RW */
-#define SDC_STS_SWR_COMPL       BIT(31)	/* RW */
+#define SDC_STS_SDCBUSY			BIT(0)  /* RW */
+#define SDC_STS_CMDBUSY			BIT(1)  /* RW */
+#define SDC_STS_SPM_RESOURCE_RELEASE	BIT(3)  /* RW */
+#define SDC_STS_SWR_COMPL		BIT(31) /* RW */
 
 /* SDC_ADV_CFG0 mask */
 #define SDC_DAT1_IRQ_TRIGGER	BIT(19)	/* RW */
@@ -448,6 +449,7 @@ struct mtk_mmc_compatible {
 	bool use_internal_cd;
 	bool support_new_tx;
 	bool support_new_rx;
+	bool support_spm_res_release;
 };
 
 struct msdc_tune_para {
@@ -673,6 +675,25 @@ static const struct mtk_mmc_compatible mt8516_compat = {
 	.stop_dly_sel = 3,
 };
 
+static const struct mtk_mmc_compatible mt8189_compat = {
+	.clk_div_bits = 12,
+	.recheck_sdio_irq = false,
+	.hs400_tune = false,
+	.needs_top_base = true,
+	.pad_tune_reg = MSDC_PAD_TUNE0,
+	.async_fifo = true,
+	.data_tune = true,
+	.busy_check = true,
+	.stop_clk_fix = true,
+	.stop_dly_sel = 1,
+	.pop_en_cnt = 2,
+	.enhance_rx = true,
+	.support_64g = true,
+	.support_new_tx = true,
+	.support_new_rx = true,
+	.support_spm_res_release = true,
+};
+
 static const struct mtk_mmc_compatible mt8196_compat = {
 	.clk_div_bits = 12,
 	.recheck_sdio_irq = false,
@@ -703,6 +724,7 @@ static const struct of_device_id msdc_of_ids[] = {
 	{ .compatible = "mediatek,mt8135-mmc", .data = &mt8135_compat},
 	{ .compatible = "mediatek,mt8173-mmc", .data = &mt8173_compat},
 	{ .compatible = "mediatek,mt8183-mmc", .data = &mt8183_compat},
+	{ .compatible = "mediatek,mt8189-mmc", .data = &mt8189_compat},
 	{ .compatible = "mediatek,mt8196-mmc", .data = &mt8196_compat},
 	{ .compatible = "mediatek,mt8516-mmc", .data = &mt8516_compat},
 
@@ -3296,6 +3318,10 @@ static int msdc_runtime_suspend(struct device *dev)
 
 		__msdc_enable_sdio_irq(host, 0);
 	}
+
+	if (host->dev_comp->support_spm_res_release)
+		sdr_set_bits(host->base + SDC_STS, SDC_STS_SPM_RESOURCE_RELEASE);
+
 	msdc_gate_clock(host);
 	return 0;
 }
diff --git a/drivers/mmc/host/sdhci-esdhc-imx.c b/drivers/mmc/host/sdhci-esdhc-imx.c
index a7a5df673b0f..97461e20425d 100644
--- a/drivers/mmc/host/sdhci-esdhc-imx.c
+++ b/drivers/mmc/host/sdhci-esdhc-imx.c
@@ -216,6 +216,8 @@
 #define ESDHC_FLAG_DUMMY_PAD		BIT(19)
 
 #define ESDHC_AUTO_TUNING_WINDOW	3
+/* 100ms timeout for data inhibit */
+#define ESDHC_DATA_INHIBIT_WAIT_US	100000
 
 enum wp_types {
 	ESDHC_WP_NONE,		/* no WP, neither controller nor gpio */
@@ -1453,6 +1455,22 @@ static void esdhc_set_uhs_signaling(struct sdhci_host *host, unsigned timing)
 
 static void esdhc_reset(struct sdhci_host *host, u8 mask)
 {
+	u32 present_state;
+	int ret;
+
+	/*
+	 * For data or full reset, ensure any active data transfer completes
+	 * before resetting to avoid system hang.
+	 */
+	if (mask & (SDHCI_RESET_DATA | SDHCI_RESET_ALL)) {
+		ret = readl_poll_timeout_atomic(host->ioaddr + ESDHC_PRSSTAT, present_state,
+						!(present_state & SDHCI_DATA_INHIBIT), 2,
+						ESDHC_DATA_INHIBIT_WAIT_US);
+		if (ret == -ETIMEDOUT)
+			dev_warn(mmc_dev(host->mmc),
+				 "timeout waiting for data transfer completion\n");
+	}
+
 	sdhci_and_cqhci_reset(host, mask);
 
 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c
index 3b85233131b3..da356627d9de 100644
--- a/drivers/mmc/host/sdhci-msm.c
+++ b/drivers/mmc/host/sdhci-msm.c
@@ -157,6 +157,17 @@
 #define CQHCI_VENDOR_CFG1	0xA00
 #define CQHCI_VENDOR_DIS_RST_ON_CQ_EN	(0x3 << 13)
 
+/* non command queue crypto enable register*/
+#define NONCQ_CRYPTO_PARM		0x70
+#define NONCQ_CRYPTO_DUN		0x74
+
+#define DISABLE_CRYPTO			BIT(15)
+#define CRYPTO_GENERAL_ENABLE		BIT(1)
+#define HC_VENDOR_SPECIFIC_FUNC4	0x260
+
+#define ICE_HCI_PARAM_CCI	GENMASK(7, 0)
+#define ICE_HCI_PARAM_CE	GENMASK(8, 8)
+
 struct sdhci_msm_offset {
 	u32 core_hc_mode;
 	u32 core_mci_data_cnt;
@@ -300,6 +311,7 @@ struct sdhci_msm_host {
 	u32 dll_config;
 	u32 ddr_config;
 	bool vqmmc_enabled;
+	bool non_cqe_ice_init_done;
 };
 
 static const struct sdhci_msm_offset *sdhci_priv_msm_offset(struct sdhci_host *host)
@@ -2012,6 +2024,68 @@ static int sdhci_msm_ice_keyslot_evict(struct blk_crypto_profile *profile,
 	return qcom_ice_evict_key(msm_host->ice, slot);
 }
 
+static void sdhci_msm_non_cqe_ice_init(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host);
+	struct mmc_host *mmc = msm_host->mmc;
+	struct cqhci_host *cq_host = mmc->cqe_private;
+	u32 config;
+
+	config = sdhci_readl(host, HC_VENDOR_SPECIFIC_FUNC4);
+	config &= ~DISABLE_CRYPTO;
+	sdhci_writel(host, config, HC_VENDOR_SPECIFIC_FUNC4);
+	config = cqhci_readl(cq_host, CQHCI_CFG);
+	config |= CRYPTO_GENERAL_ENABLE;
+	cqhci_writel(cq_host, config, CQHCI_CFG);
+}
+
+static void sdhci_msm_ice_cfg(struct sdhci_host *host, struct mmc_request *mrq)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host);
+	struct mmc_host *mmc = msm_host->mmc;
+	struct cqhci_host *cq_host = mmc->cqe_private;
+	unsigned int crypto_params = 0;
+	int key_index;
+
+	if (mrq->crypto_ctx) {
+		if (!msm_host->non_cqe_ice_init_done) {
+			sdhci_msm_non_cqe_ice_init(host);
+			msm_host->non_cqe_ice_init_done = true;
+		}
+
+		key_index = mrq->crypto_key_slot;
+		crypto_params = FIELD_PREP(ICE_HCI_PARAM_CE, 1) |
+				FIELD_PREP(ICE_HCI_PARAM_CCI, key_index);
+
+		cqhci_writel(cq_host, crypto_params, NONCQ_CRYPTO_PARM);
+		cqhci_writel(cq_host, lower_32_bits(mrq->crypto_ctx->bc_dun[0]),
+			     NONCQ_CRYPTO_DUN);
+	} else {
+		cqhci_writel(cq_host, crypto_params, NONCQ_CRYPTO_PARM);
+	}
+
+	/* Ensure crypto configuration is written before proceeding */
+	wmb();
+}
+
+/*
+ * Handle non-CQE MMC requests with ICE crypto support.
+ * Configures ICE registers before passing the request to
+ * the standard SDHCI handler.
+ */
+static void sdhci_msm_request(struct mmc_host *mmc, struct mmc_request *mrq)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+
+	/* Only need to handle non-CQE crypto requests in this path */
+	if (mmc->caps2 & MMC_CAP2_CRYPTO)
+		sdhci_msm_ice_cfg(host, mrq);
+
+	sdhci_request(mmc, mrq);
+}
+
 static const struct blk_crypto_ll_ops sdhci_msm_crypto_ops = {
 	.keyslot_program	= sdhci_msm_ice_keyslot_program,
 	.keyslot_evict		= sdhci_msm_ice_keyslot_evict,
@@ -2762,6 +2836,9 @@ static int sdhci_msm_probe(struct platform_device *pdev)
 
 	msm_host->mmc->caps |= MMC_CAP_WAIT_WHILE_BUSY | MMC_CAP_NEED_RSP_BUSY;
 
+#ifdef CONFIG_MMC_CRYPTO
+	host->mmc_host_ops.request = sdhci_msm_request;
+#endif
 	/* Set the timeout value to max possible */
 	host->max_timeout_count = 0xF;
 
diff --git a/drivers/mmc/host/sdhci-of-arasan.c b/drivers/mmc/host/sdhci-of-arasan.c
index b97d042897ad..ab7f0ffe7b4f 100644
--- a/drivers/mmc/host/sdhci-of-arasan.c
+++ b/drivers/mmc/host/sdhci-of-arasan.c
@@ -99,7 +99,7 @@
 #define HIWORD_UPDATE(val, mask, shift) \
 		((val) << (shift) | (mask) << ((shift) + 16))
 
-#define CD_STABLE_TIMEOUT_US		1000000
+#define CD_STABLE_TIMEOUT_US		2000000
 #define CD_STABLE_MAX_SLEEP_US		10
 
 /**
diff --git a/drivers/mtd/nand/spi/Makefile b/drivers/mtd/nand/spi/Makefile
index 6d3d203df048..a47bd22cd309 100644
--- a/drivers/mtd/nand/spi/Makefile
+++ b/drivers/mtd/nand/spi/Makefile
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
 spinand-objs := core.o otp.o
-spinand-objs += alliancememory.o ato.o esmt.o fmsh.o foresee.o gigadevice.o macronix.o
-spinand-objs += micron.o paragon.o skyhigh.o toshiba.o winbond.o xtx.o
+spinand-objs += alliancememory.o ato.o dosilicon.o esmt.o fmsh.o foresee.o gigadevice.o
+spinand-objs += macronix.o micron.o paragon.o skyhigh.o toshiba.o winbond.o xtx.o
 obj-$(CONFIG_MTD_SPI_NAND) += spinand.o
diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c
index d207286572d8..0346916b032b 100644
--- a/drivers/mtd/nand/spi/core.c
+++ b/drivers/mtd/nand/spi/core.c
@@ -1227,6 +1227,7 @@ static const struct nand_ops spinand_ops = {
 static const struct spinand_manufacturer *spinand_manufacturers[] = {
 	&alliancememory_spinand_manufacturer,
 	&ato_spinand_manufacturer,
+	&dosilicon_spinand_manufacturer,
 	&esmt_8c_spinand_manufacturer,
 	&esmt_c8_spinand_manufacturer,
 	&fmsh_spinand_manufacturer,
diff --git a/drivers/mtd/nand/spi/dosilicon.c b/drivers/mtd/nand/spi/dosilicon.c
new file mode 100644
index 000000000000..f99899866ceb
--- /dev/null
+++ b/drivers/mtd/nand/spi/dosilicon.c
@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Author: Ahmed Naseef <naseefkm@gmail.com>
+ */
+
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/mtd/spinand.h>
+
+#define SPINAND_MFR_DOSILICON        0xE5
+
+static SPINAND_OP_VARIANTS(read_cache_variants,
+		SPINAND_PAGE_READ_FROM_CACHE_1S_1S_4S_OP(0, 1, NULL, 0, 0),
+		SPINAND_PAGE_READ_FROM_CACHE_1S_1S_2S_OP(0, 1, NULL, 0, 0),
+		SPINAND_PAGE_READ_FROM_CACHE_FAST_1S_1S_1S_OP(0, 1, NULL, 0, 0),
+		SPINAND_PAGE_READ_FROM_CACHE_1S_1S_1S_OP(0, 1, NULL, 0, 0));
+
+static SPINAND_OP_VARIANTS(write_cache_variants,
+		SPINAND_PROG_LOAD_1S_1S_4S_OP(true, 0, NULL, 0),
+		SPINAND_PROG_LOAD_1S_1S_1S_OP(true, 0, NULL, 0));
+
+static SPINAND_OP_VARIANTS(update_cache_variants,
+		SPINAND_PROG_LOAD_1S_1S_4S_OP(false, 0, NULL, 0),
+		SPINAND_PROG_LOAD_1S_1S_1S_OP(false, 0, NULL, 0));
+
+static int ds35xx_ooblayout_ecc(struct mtd_info *mtd, int section,
+				struct mtd_oob_region *region)
+{
+	if (section > 3)
+		return -ERANGE;
+
+	region->offset = 8 + (section * 16);
+	region->length = 8;
+
+	return 0;
+}
+
+static int ds35xx_ooblayout_free(struct mtd_info *mtd, int section,
+				 struct mtd_oob_region *region)
+{
+	if (section > 3)
+		return -ERANGE;
+
+	if (section == 0) {
+		/* reserve 2 bytes for the BBM */
+		region->offset = 2;
+		region->length = 6;
+	} else {
+		region->offset = section * 16;
+		region->length = 8;
+	}
+
+	return 0;
+}
+
+static const struct mtd_ooblayout_ops ds35xx_ooblayout = {
+	.ecc = ds35xx_ooblayout_ecc,
+	.free = ds35xx_ooblayout_free,
+};
+
+static const struct spinand_info dosilicon_spinand_table[] = {
+	SPINAND_INFO("DS35Q1GA",
+		SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x71),
+		NAND_MEMORG(1, 2048, 64, 64, 1024, 20, 1, 1, 1),
+		NAND_ECCREQ(4, 512),
+		SPINAND_INFO_OP_VARIANTS(&read_cache_variants,
+					 &write_cache_variants,
+					 &update_cache_variants),
+		SPINAND_HAS_QE_BIT,
+		SPINAND_ECCINFO(&ds35xx_ooblayout, NULL)),
+	SPINAND_INFO("DS35M1GA",
+		SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x21),
+		NAND_MEMORG(1, 2048, 64, 64, 1024, 20, 1, 1, 1),
+		NAND_ECCREQ(4, 512),
+		SPINAND_INFO_OP_VARIANTS(&read_cache_variants,
+					 &write_cache_variants,
+					 &update_cache_variants),
+		SPINAND_HAS_QE_BIT,
+		SPINAND_ECCINFO(&ds35xx_ooblayout, NULL)),
+};
+
+static const struct spinand_manufacturer_ops dosilicon_spinand_manuf_ops = {
+};
+
+const struct spinand_manufacturer dosilicon_spinand_manufacturer = {
+	.id = SPINAND_MFR_DOSILICON,
+	.name = "Dosilicon",
+	.chips = dosilicon_spinand_table,
+	.nchips = ARRAY_SIZE(dosilicon_spinand_table),
+	.ops = &dosilicon_spinand_manuf_ops,
+};
diff --git a/drivers/mtd/nand/spi/esmt.c b/drivers/mtd/nand/spi/esmt.c
index e60e4ac1fd6f..3e86f346f751 100644
--- a/drivers/mtd/nand/spi/esmt.c
+++ b/drivers/mtd/nand/spi/esmt.c
@@ -215,7 +215,7 @@ static const struct spinand_info esmt_c8_spinand_table[] = {
 		     SPINAND_FACT_OTP_INFO(2, 0, &f50l1g41lb_fact_otp_ops)),
 	SPINAND_INFO("F50D1G41LB",
 		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_ADDR, 0x11, 0x7f,
-				0x7f),
+				0x7f, 0x7f),
 		     NAND_MEMORG(1, 2048, 64, 64, 1024, 20, 1, 1, 1),
 		     NAND_ECCREQ(1, 512),
 		     SPINAND_INFO_OP_VARIANTS(&read_cache_variants,
diff --git a/drivers/net/bonding/bond_options.c b/drivers/net/bonding/bond_options.c
index 384499c869b8..fa65a0e92b8e 100644
--- a/drivers/net/bonding/bond_options.c
+++ b/drivers/net/bonding/bond_options.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/errno.h>
+#include <linux/hex.h>
 #include <linux/if.h>
 #include <linux/netdevice.h>
 #include <linux/spinlock.h>
diff --git a/drivers/net/can/Kconfig b/drivers/net/can/Kconfig
index e15e320db476..cfaea6178a71 100644
--- a/drivers/net/can/Kconfig
+++ b/drivers/net/can/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 
 menuconfig CAN_DEV
-	tristate "CAN Device Drivers"
+	bool "CAN Device Drivers"
 	default y
 	depends on CAN
 	help
@@ -17,10 +17,7 @@ menuconfig CAN_DEV
 	  virtual ones. If you own such devices or plan to use the virtual CAN
 	  interfaces to develop applications, say Y here.
 
-	  To compile as a module, choose M here: the module will be called
-	  can-dev.
-
-if CAN_DEV
+if CAN_DEV && CAN
 
 config CAN_VCAN
 	tristate "Virtual Local CAN Interface (vcan)"
diff --git a/drivers/net/can/Makefile b/drivers/net/can/Makefile
index d7bc10a6b8ea..37e2f1a2faec 100644
--- a/drivers/net/can/Makefile
+++ b/drivers/net/can/Makefile
@@ -7,7 +7,7 @@ obj-$(CONFIG_CAN_VCAN)		+= vcan.o
 obj-$(CONFIG_CAN_VXCAN)		+= vxcan.o
 obj-$(CONFIG_CAN_SLCAN)		+= slcan/
 
-obj-y				+= dev/
+obj-$(CONFIG_CAN_DEV)		+= dev/
 obj-y				+= esd/
 obj-y				+= rcar/
 obj-y				+= rockchip/
diff --git a/drivers/net/can/can327.c b/drivers/net/can/can327.c
index b66fc16aedd2..90f5e35f3c8f 100644
--- a/drivers/net/can/can327.c
+++ b/drivers/net/can/can327.c
@@ -18,6 +18,7 @@
 #include <linux/bitops.h>
 #include <linux/ctype.h>
 #include <linux/errno.h>
+#include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/list.h>
 #include <linux/lockdep.h>
diff --git a/drivers/net/can/dev/Makefile b/drivers/net/can/dev/Makefile
index 633687d6b6c0..64226acf0f3d 100644
--- a/drivers/net/can/dev/Makefile
+++ b/drivers/net/can/dev/Makefile
@@ -1,9 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
 
-obj-$(CONFIG_CAN_DEV) += can-dev.o
-
-can-dev-y += skb.o
+obj-$(CONFIG_CAN) += can-dev.o
 
+can-dev-$(CONFIG_CAN_DEV) += skb.o
 can-dev-$(CONFIG_CAN_CALC_BITTIMING) += calc_bittiming.o
 can-dev-$(CONFIG_CAN_NETLINK) += bittiming.o
 can-dev-$(CONFIG_CAN_NETLINK) += dev.o
diff --git a/drivers/net/can/slcan/slcan-core.c b/drivers/net/can/slcan/slcan-core.c
index cd789e178d34..7439849d5c84 100644
--- a/drivers/net/can/slcan/slcan-core.c
+++ b/drivers/net/can/slcan/slcan-core.c
@@ -50,6 +50,7 @@
 #include <linux/netdevice.h>
 #include <linux/skbuff.h>
 #include <linux/rtnetlink.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/workqueue.h>
diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c
index e29e85b67fd4..a0233e550a5a 100644
--- a/drivers/net/can/usb/gs_usb.c
+++ b/drivers/net/can/usb/gs_usb.c
@@ -1074,7 +1074,7 @@ static int gs_can_open(struct net_device *netdev)
 	usb_free_urb(urb);
 out_usb_kill_anchored_urbs:
 	if (!parent->active_channels) {
-		usb_kill_anchored_urbs(&dev->tx_submitted);
+		usb_kill_anchored_urbs(&parent->rx_submitted);
 
 		if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP)
 			gs_usb_timestamp_stop(parent);
diff --git a/drivers/net/dsa/lantiq/lantiq_gswip.c b/drivers/net/dsa/lantiq/lantiq_gswip.c
index 57dd063c0740..b094001a7c80 100644
--- a/drivers/net/dsa/lantiq/lantiq_gswip.c
+++ b/drivers/net/dsa/lantiq/lantiq_gswip.c
@@ -444,9 +444,6 @@ static void gswip_remove(struct platform_device *pdev)
 	if (!priv)
 		return;
 
-	/* disable the switch */
-	gswip_disable_switch(priv);
-
 	dsa_unregister_switch(priv->ds);
 
 	for (i = 0; i < priv->num_gphy_fw; i++)
diff --git a/drivers/net/dsa/lantiq/lantiq_gswip.h b/drivers/net/dsa/lantiq/lantiq_gswip.h
index 9c38e51a75e8..2e0f2afbadbb 100644
--- a/drivers/net/dsa/lantiq/lantiq_gswip.h
+++ b/drivers/net/dsa/lantiq/lantiq_gswip.h
@@ -294,8 +294,6 @@ struct gswip_priv {
 	u16 version;
 };
 
-void gswip_disable_switch(struct gswip_priv *priv);
-
 int gswip_probe_common(struct gswip_priv *priv, u32 version);
 
 #endif /* __LANTIQ_GSWIP_H */
diff --git a/drivers/net/dsa/lantiq/lantiq_gswip_common.c b/drivers/net/dsa/lantiq/lantiq_gswip_common.c
index 9da39edf8f57..e790f2ef7588 100644
--- a/drivers/net/dsa/lantiq/lantiq_gswip_common.c
+++ b/drivers/net/dsa/lantiq/lantiq_gswip_common.c
@@ -752,6 +752,13 @@ static int gswip_setup(struct dsa_switch *ds)
 	return 0;
 }
 
+static void gswip_teardown(struct dsa_switch *ds)
+{
+	struct gswip_priv *priv = ds->priv;
+
+	regmap_clear_bits(priv->mdio, GSWIP_MDIO_GLOB, GSWIP_MDIO_GLOB_ENABLE);
+}
+
 static enum dsa_tag_protocol gswip_get_tag_protocol(struct dsa_switch *ds,
 						    int port,
 						    enum dsa_tag_protocol mp)
@@ -1629,6 +1636,7 @@ static const struct phylink_mac_ops gswip_phylink_mac_ops = {
 static const struct dsa_switch_ops gswip_switch_ops = {
 	.get_tag_protocol	= gswip_get_tag_protocol,
 	.setup			= gswip_setup,
+	.teardown		= gswip_teardown,
 	.port_setup		= gswip_port_setup,
 	.port_enable		= gswip_port_enable,
 	.port_disable		= gswip_port_disable,
@@ -1656,12 +1664,6 @@ static const struct dsa_switch_ops gswip_switch_ops = {
 	.port_hsr_leave		= dsa_port_simple_hsr_leave,
 };
 
-void gswip_disable_switch(struct gswip_priv *priv)
-{
-	regmap_clear_bits(priv->mdio, GSWIP_MDIO_GLOB, GSWIP_MDIO_GLOB_ENABLE);
-}
-EXPORT_SYMBOL_GPL(gswip_disable_switch);
-
 static int gswip_validate_cpu_port(struct dsa_switch *ds)
 {
 	struct gswip_priv *priv = ds->priv;
@@ -1718,15 +1720,14 @@ int gswip_probe_common(struct gswip_priv *priv, u32 version)
 
 	err = gswip_validate_cpu_port(priv->ds);
 	if (err)
-		goto disable_switch;
+		goto unregister_switch;
 
 	dev_info(priv->dev, "probed GSWIP version %lx mod %lx\n",
 		 GSWIP_VERSION_REV(version), GSWIP_VERSION_MOD(version));
 
 	return 0;
 
-disable_switch:
-	gswip_disable_switch(priv);
+unregister_switch:
 	dsa_unregister_switch(priv->ds);
 
 	return err;
diff --git a/drivers/net/dsa/lantiq/mxl-gsw1xx.c b/drivers/net/dsa/lantiq/mxl-gsw1xx.c
index 0816c61a47f1..f8ff8a604bf5 100644
--- a/drivers/net/dsa/lantiq/mxl-gsw1xx.c
+++ b/drivers/net/dsa/lantiq/mxl-gsw1xx.c
@@ -11,10 +11,12 @@
 
 #include <linux/bits.h>
 #include <linux/delay.h>
+#include <linux/jiffies.h>
 #include <linux/module.h>
 #include <linux/of_device.h>
 #include <linux/of_mdio.h>
 #include <linux/regmap.h>
+#include <linux/workqueue.h>
 #include <net/dsa.h>
 
 #include "lantiq_gswip.h"
@@ -29,6 +31,7 @@ struct gsw1xx_priv {
 	struct			regmap *clk;
 	struct			regmap *shell;
 	struct			phylink_pcs pcs;
+	struct delayed_work	clear_raneg;
 	phy_interface_t		tbi_interface;
 	struct gswip_priv	gswip;
 };
@@ -145,7 +148,9 @@ static void gsw1xx_pcs_disable(struct phylink_pcs *pcs)
 {
 	struct gsw1xx_priv *priv = pcs_to_gsw1xx(pcs);
 
-	/* Assert SGMII shell reset */
+	cancel_delayed_work_sync(&priv->clear_raneg);
+
+	/* Assert SGMII shell reset (will also clear RANEG bit) */
 	regmap_set_bits(priv->shell, GSW1XX_SHELL_RST_REQ,
 			GSW1XX_RST_REQ_SGMII_SHELL);
 
@@ -255,10 +260,16 @@ static int gsw1xx_pcs_reset(struct gsw1xx_priv *priv)
 	      FIELD_PREP(GSW1XX_SGMII_PHY_RX0_CFG2_FILT_CNT,
 			 GSW1XX_SGMII_PHY_RX0_CFG2_FILT_CNT_DEF);
 
-	/* TODO: Take care of inverted RX pair once generic property is
+	/* RX lane seems to be inverted internally, so bit
+	 * GSW1XX_SGMII_PHY_RX0_CFG2_INVERT needs to be set for normal
+	 * (ie. non-inverted) operation.
+	 *
+	 * TODO: Take care of inverted RX pair once generic property is
 	 *       available
 	 */
 
+	val |= GSW1XX_SGMII_PHY_RX0_CFG2_INVERT;
+
 	ret = regmap_write(priv->sgmii, GSW1XX_SGMII_PHY_RX0_CFG2, val);
 	if (ret < 0)
 		return ret;
@@ -422,12 +433,29 @@ static int gsw1xx_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode,
 	return 0;
 }
 
+static void gsw1xx_pcs_clear_raneg(struct work_struct *work)
+{
+	struct gsw1xx_priv *priv =
+		container_of(work, struct gsw1xx_priv, clear_raneg.work);
+
+	regmap_clear_bits(priv->sgmii, GSW1XX_SGMII_TBI_ANEGCTL,
+			  GSW1XX_SGMII_TBI_ANEGCTL_RANEG);
+}
+
 static void gsw1xx_pcs_an_restart(struct phylink_pcs *pcs)
 {
 	struct gsw1xx_priv *priv = pcs_to_gsw1xx(pcs);
 
+	cancel_delayed_work_sync(&priv->clear_raneg);
+
 	regmap_set_bits(priv->sgmii, GSW1XX_SGMII_TBI_ANEGCTL,
 			GSW1XX_SGMII_TBI_ANEGCTL_RANEG);
+
+	/* despite being documented as self-clearing, the RANEG bit
+	 * sometimes remains set, preventing auto-negotiation from happening.
+	 * MaxLinear advises to manually clear the bit after 10ms.
+	 */
+	schedule_delayed_work(&priv->clear_raneg, msecs_to_jiffies(10));
 }
 
 static void gsw1xx_pcs_link_up(struct phylink_pcs *pcs,
@@ -630,6 +658,8 @@ static int gsw1xx_probe(struct mdio_device *mdiodev)
 	if (ret)
 		return ret;
 
+	INIT_DELAYED_WORK(&priv->clear_raneg, gsw1xx_pcs_clear_raneg);
+
 	ret = gswip_probe_common(&priv->gswip, version);
 	if (ret)
 		return ret;
@@ -642,25 +672,31 @@ static int gsw1xx_probe(struct mdio_device *mdiodev)
 static void gsw1xx_remove(struct mdio_device *mdiodev)
 {
 	struct gswip_priv *priv = dev_get_drvdata(&mdiodev->dev);
+	struct gsw1xx_priv *gsw1xx_priv;
 
 	if (!priv)
 		return;
 
-	gswip_disable_switch(priv);
-
 	dsa_unregister_switch(priv->ds);
+
+	gsw1xx_priv = container_of(priv, struct gsw1xx_priv, gswip);
+	cancel_delayed_work_sync(&gsw1xx_priv->clear_raneg);
 }
 
 static void gsw1xx_shutdown(struct mdio_device *mdiodev)
 {
 	struct gswip_priv *priv = dev_get_drvdata(&mdiodev->dev);
+	struct gsw1xx_priv *gsw1xx_priv;
 
 	if (!priv)
 		return;
 
+	dsa_switch_shutdown(priv->ds);
+
 	dev_set_drvdata(&mdiodev->dev, NULL);
 
-	gswip_disable_switch(priv);
+	gsw1xx_priv = container_of(priv, struct gsw1xx_priv, gswip);
+	cancel_delayed_work_sync(&gsw1xx_priv->clear_raneg);
 }
 
 static const struct gswip_hw_info gsw12x_data = {
diff --git a/drivers/net/ethernet/broadcom/b44.c b/drivers/net/ethernet/broadcom/b44.c
index 888f28f11406..90df02e0039c 100644
--- a/drivers/net/ethernet/broadcom/b44.c
+++ b/drivers/net/ethernet/broadcom/b44.c
@@ -1790,6 +1790,9 @@ static int b44_nway_reset(struct net_device *dev)
 	u32 bmcr;
 	int r;
 
+	if (bp->flags & B44_FLAG_EXTERNAL_PHY)
+		return phy_ethtool_nway_reset(dev);
+
 	spin_lock_irq(&bp->lock);
 	b44_readphy(bp, MII_BMCR, &bmcr);
 	b44_readphy(bp, MII_BMCR, &bmcr);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
index 3e77a96e5a3e..c94a391b1ba5 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
@@ -268,13 +268,11 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
 	case XDP_TX:
 		rx_buf = &rxr->rx_buf_ring[cons];
 		mapping = rx_buf->mapping - bp->rx_dma_offset;
-		*event &= BNXT_TX_CMP_EVENT;
 
 		if (unlikely(xdp_buff_has_frags(xdp))) {
 			struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
 
 			tx_needed += sinfo->nr_frags;
-			*event = BNXT_AGG_EVENT;
 		}
 
 		if (tx_avail < tx_needed) {
@@ -287,6 +285,7 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
 		dma_sync_single_for_device(&pdev->dev, mapping + offset, *len,
 					   bp->rx_dir);
 
+		*event &= ~BNXT_RX_EVENT;
 		*event |= BNXT_TX_EVENT;
 		__bnxt_xmit_xdp(bp, txr, mapping + offset, *len,
 				NEXT_RX(rxr->rx_prod), xdp);
diff --git a/drivers/net/ethernet/chelsio/cxgb3/common.h b/drivers/net/ethernet/chelsio/cxgb3/common.h
index ecd025dda8d6..14000977730c 100644
--- a/drivers/net/ethernet/chelsio/cxgb3/common.h
+++ b/drivers/net/ethernet/chelsio/cxgb3/common.h
@@ -36,6 +36,7 @@
 #include <linux/types.h>
 #include <linux/ctype.h>
 #include <linux/delay.h>
+#include <linux/hex.h>
 #include <linux/netdevice.h>
 #include <linux/ethtool.h>
 #include <linux/mdio.h>
diff --git a/drivers/net/ethernet/freescale/enetc/enetc.c b/drivers/net/ethernet/freescale/enetc/enetc.c
index d5e5800b84ef..53b26cece16a 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc.c
@@ -1787,7 +1787,8 @@ int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
 	int xdp_tx_bd_cnt, i, k;
 	int xdp_tx_frm_cnt = 0;
 
-	if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags)))
+	if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags) ||
+		     !netif_carrier_ok(ndev)))
 		return -ENETDOWN;
 
 	enetc_lock_mdio();
diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c
index c685a5c0cc51..a753265961af 100644
--- a/drivers/net/ethernet/freescale/fec_main.c
+++ b/drivers/net/ethernet/freescale/fec_main.c
@@ -3933,7 +3933,12 @@ static int fec_enet_txq_xmit_frame(struct fec_enet_private *fep,
 	txq->bd.cur = bdp;
 
 	/* Trigger transmission start */
-	writel(0, txq->bd.reg_desc_active);
+	if (!(fep->quirks & FEC_QUIRK_ERR007885) ||
+	    !readl(txq->bd.reg_desc_active) ||
+	    !readl(txq->bd.reg_desc_active) ||
+	    !readl(txq->bd.reg_desc_active) ||
+	    !readl(txq->bd.reg_desc_active))
+		writel(0, txq->bd.reg_desc_active);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index a5a2b18d309b..a7a088a77f37 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -647,12 +647,9 @@ static int gve_setup_device_resources(struct gve_priv *priv)
 	err = gve_alloc_counter_array(priv);
 	if (err)
 		goto abort_with_rss_config_cache;
-	err = gve_init_clock(priv);
-	if (err)
-		goto abort_with_counter;
 	err = gve_alloc_notify_blocks(priv);
 	if (err)
-		goto abort_with_clock;
+		goto abort_with_counter;
 	err = gve_alloc_stats_report(priv);
 	if (err)
 		goto abort_with_ntfy_blocks;
@@ -683,10 +680,16 @@ static int gve_setup_device_resources(struct gve_priv *priv)
 		}
 	}
 
+	err = gve_init_clock(priv);
+	if (err) {
+		dev_err(&priv->pdev->dev, "Failed to init clock");
+		goto abort_with_ptype_lut;
+	}
+
 	err = gve_init_rss_config(priv, priv->rx_cfg.num_queues);
 	if (err) {
 		dev_err(&priv->pdev->dev, "Failed to init RSS config");
-		goto abort_with_ptype_lut;
+		goto abort_with_clock;
 	}
 
 	err = gve_adminq_report_stats(priv, priv->stats_report_len,
@@ -698,6 +701,8 @@ static int gve_setup_device_resources(struct gve_priv *priv)
 	gve_set_device_resources_ok(priv);
 	return 0;
 
+abort_with_clock:
+	gve_teardown_clock(priv);
 abort_with_ptype_lut:
 	kvfree(priv->ptype_lut_dqo);
 	priv->ptype_lut_dqo = NULL;
@@ -705,8 +710,6 @@ static int gve_setup_device_resources(struct gve_priv *priv)
 	gve_free_stats_report(priv);
 abort_with_ntfy_blocks:
 	gve_free_notify_blocks(priv);
-abort_with_clock:
-	gve_teardown_clock(priv);
 abort_with_counter:
 	gve_free_counter_array(priv);
 abort_with_rss_config_cache:
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
index cf8abbe01840..c589baea7c77 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
@@ -10555,6 +10555,9 @@ int hclge_set_vlan_filter(struct hnae3_handle *handle, __be16 proto,
 	bool writen_to_tbl = false;
 	int ret = 0;
 
+	if (vlan_id >= VLAN_N_VID)
+		return -EINVAL;
+
 	/* When device is resetting or reset failed, firmware is unable to
 	 * handle mailbox. Just record the vlan id, and remove it after
 	 * reset finished.
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
index c7ff12a6c076..b7d4e06a55d4 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
@@ -193,10 +193,10 @@ static int hclge_get_ring_chain_from_mbx(
 		return -EINVAL;
 
 	for (i = 0; i < ring_num; i++) {
-		if (req->msg.param[i].tqp_index >= vport->nic.kinfo.rss_size) {
+		if (req->msg.param[i].tqp_index >= vport->nic.kinfo.num_tqps) {
 			dev_err(&hdev->pdev->dev, "tqp index(%u) is out of range(0-%u)\n",
 				req->msg.param[i].tqp_index,
-				vport->nic.kinfo.rss_size - 1U);
+				vport->nic.kinfo.num_tqps - 1U);
 			return -EINVAL;
 		}
 	}
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
index 8fcf220a120d..70327a73dee3 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
@@ -368,12 +368,12 @@ static int hclgevf_knic_setup(struct hclgevf_dev *hdev)
 	new_tqps = kinfo->rss_size * num_tc;
 	kinfo->num_tqps = min(new_tqps, hdev->num_tqps);
 
-	kinfo->tqp = devm_kcalloc(&hdev->pdev->dev, kinfo->num_tqps,
+	kinfo->tqp = devm_kcalloc(&hdev->pdev->dev, hdev->num_tqps,
 				  sizeof(struct hnae3_queue *), GFP_KERNEL);
 	if (!kinfo->tqp)
 		return -ENOMEM;
 
-	for (i = 0; i < kinfo->num_tqps; i++) {
+	for (i = 0; i < hdev->num_tqps; i++) {
 		hdev->htqp[i].q.handle = &hdev->nic;
 		hdev->htqp[i].q.tqp_index = i;
 		kinfo->tqp[i] = &hdev->htqp[i].q;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
index 887adf4807d1..ea77fbd98396 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
@@ -197,6 +197,11 @@ static int mlx5_devlink_reload_down(struct devlink *devlink, bool netns_change,
 	struct pci_dev *pdev = dev->pdev;
 	int ret = 0;
 
+	if (mlx5_fw_reset_in_progress(dev)) {
+		NL_SET_ERR_MSG_MOD(extack, "Can't reload during firmware reset");
+		return -EBUSY;
+	}
+
 	if (mlx5_dev_is_lightweight(dev)) {
 		if (action != DEVLINK_RELOAD_ACTION_DRIVER_REINIT)
 			return -EOPNOTSUPP;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c
index 7bcf822a89f9..6b4ec457ce22 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c
@@ -33,6 +33,7 @@
 #include "lib/eq.h"
 #include "fw_tracer.h"
 #include "fw_tracer_tracepoint.h"
+#include <linux/ctype.h>
 
 static int mlx5_query_mtrc_caps(struct mlx5_fw_tracer *tracer)
 {
@@ -358,6 +359,47 @@ static const char *VAL_PARM		= "%llx";
 static const char *REPLACE_64_VAL_PARM	= "%x%x";
 static const char *PARAM_CHAR		= "%";
 
+static bool mlx5_is_valid_spec(const char *str)
+{
+	/* Parse format specifiers to find the actual type.
+	 * Structure: %[flags][width][.precision][length]type
+	 * Skip flags, width, precision & length.
+	 */
+	while (isdigit(*str) || *str == '#' || *str == '.' || *str == 'l')
+		str++;
+
+	/* Check if it's a valid integer/hex specifier or %%:
+	 * Valid formats: %x, %d, %i, %u, etc.
+	 */
+	if (*str != 'x' && *str != 'X' && *str != 'd' && *str != 'i' &&
+	    *str != 'u' && *str != 'c' && *str != '%')
+		return false;
+
+	return true;
+}
+
+static bool mlx5_tracer_validate_params(const char *str)
+{
+	const char *substr = str;
+
+	if (!str)
+		return false;
+
+	substr = strstr(substr, PARAM_CHAR);
+	while (substr) {
+		if (!mlx5_is_valid_spec(substr + 1))
+			return false;
+
+		if (*(substr + 1) == '%')
+			substr = strstr(substr + 2, PARAM_CHAR);
+		else
+			substr = strstr(substr + 1, PARAM_CHAR);
+
+	}
+
+	return true;
+}
+
 static int mlx5_tracer_message_hash(u32 message_id)
 {
 	return jhash_1word(message_id, 0) & (MESSAGE_HASH_SIZE - 1);
@@ -419,6 +461,10 @@ static int mlx5_tracer_get_num_of_params(char *str)
 	char *substr, *pstr = str;
 	int num_of_params = 0;
 
+	/* Validate that all parameters are valid before processing */
+	if (!mlx5_tracer_validate_params(str))
+		return -EINVAL;
+
 	/* replace %llx with %x%x */
 	substr = strstr(pstr, VAL_PARM);
 	while (substr) {
@@ -427,11 +473,15 @@ static int mlx5_tracer_get_num_of_params(char *str)
 		substr = strstr(pstr, VAL_PARM);
 	}
 
-	/* count all the % characters */
+	/* count all the % characters, but skip %% (escaped percent) */
 	substr = strstr(str, PARAM_CHAR);
 	while (substr) {
-		num_of_params += 1;
-		str = substr + 1;
+		if (*(substr + 1) != '%') {
+			num_of_params += 1;
+			str = substr + 1;
+		} else {
+			str = substr + 2;
+		}
 		substr = strstr(str, PARAM_CHAR);
 	}
 
@@ -570,14 +620,17 @@ void mlx5_tracer_print_trace(struct tracer_string_format *str_frmt,
 {
 	char	tmp[512];
 
-	snprintf(tmp, sizeof(tmp), str_frmt->string,
-		 str_frmt->params[0],
-		 str_frmt->params[1],
-		 str_frmt->params[2],
-		 str_frmt->params[3],
-		 str_frmt->params[4],
-		 str_frmt->params[5],
-		 str_frmt->params[6]);
+	if (str_frmt->invalid_string)
+		snprintf(tmp, sizeof(tmp), "BAD_FORMAT: %s", str_frmt->string);
+	else
+		snprintf(tmp, sizeof(tmp), str_frmt->string,
+			 str_frmt->params[0],
+			 str_frmt->params[1],
+			 str_frmt->params[2],
+			 str_frmt->params[3],
+			 str_frmt->params[4],
+			 str_frmt->params[5],
+			 str_frmt->params[6]);
 
 	trace_mlx5_fw(dev->tracer, trace_timestamp, str_frmt->lost,
 		      str_frmt->event_id, tmp);
@@ -609,6 +662,13 @@ static int mlx5_tracer_handle_raw_string(struct mlx5_fw_tracer *tracer,
 	return 0;
 }
 
+static void mlx5_tracer_handle_bad_format_string(struct mlx5_fw_tracer *tracer,
+						 struct tracer_string_format *cur_string)
+{
+	cur_string->invalid_string = true;
+	list_add_tail(&cur_string->list, &tracer->ready_strings_list);
+}
+
 static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer,
 					   struct tracer_event *tracer_event)
 {
@@ -619,12 +679,18 @@ static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer,
 		if (!cur_string)
 			return mlx5_tracer_handle_raw_string(tracer, tracer_event);
 
-		cur_string->num_of_params = mlx5_tracer_get_num_of_params(cur_string->string);
-		cur_string->last_param_num = 0;
 		cur_string->event_id = tracer_event->event_id;
 		cur_string->tmsn = tracer_event->string_event.tmsn;
 		cur_string->timestamp = tracer_event->string_event.timestamp;
 		cur_string->lost = tracer_event->lost_event;
+		cur_string->last_param_num = 0;
+		cur_string->num_of_params = mlx5_tracer_get_num_of_params(cur_string->string);
+		if (cur_string->num_of_params < 0) {
+			pr_debug("%s Invalid format string parameters\n",
+				 __func__);
+			mlx5_tracer_handle_bad_format_string(tracer, cur_string);
+			return 0;
+		}
 		if (cur_string->num_of_params == 0) /* trace with no params */
 			list_add_tail(&cur_string->list, &tracer->ready_strings_list);
 	} else {
@@ -634,6 +700,11 @@ static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer,
 				 __func__, tracer_event->string_event.tmsn);
 			return mlx5_tracer_handle_raw_string(tracer, tracer_event);
 		}
+		if (cur_string->num_of_params < 0) {
+			pr_debug("%s string parameter of invalid string, dumping\n",
+				 __func__);
+			return 0;
+		}
 		cur_string->last_param_num += 1;
 		if (cur_string->last_param_num > TRACER_MAX_PARAMS) {
 			pr_debug("%s Number of params exceeds the max (%d)\n",
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.h b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.h
index 5c548bb74f07..30d0bcba8847 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.h
@@ -125,6 +125,7 @@ struct tracer_string_format {
 	struct list_head list;
 	u32 timestamp;
 	bool lost;
+	bool invalid_string;
 };
 
 enum mlx5_fw_tracer_ownership_state {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 811178d8976c..262dc032e276 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -69,7 +69,7 @@ struct page_pool;
 #define MLX5E_METADATA_ETHER_TYPE (0x8CE4)
 #define MLX5E_METADATA_ETHER_LEN 8
 
-#define MLX5E_ETH_HARD_MTU (ETH_HLEN + PSP_ENCAP_HLEN + PSP_TRL_SIZE + VLAN_HLEN + ETH_FCS_LEN)
+#define MLX5E_ETH_HARD_MTU (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN)
 
 #define MLX5E_HW2SW_MTU(params, hwmtu) ((hwmtu) - ((params)->hard_mtu))
 #define MLX5E_SW2HW_MTU(params, swmtu) ((swmtu) + ((params)->hard_mtu))
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
index 35d9530037a6..a8fb4bec369c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
@@ -342,9 +342,8 @@ static void mlx5e_ipsec_init_macs(struct mlx5e_ipsec_sa_entry *sa_entry,
 		rt_dst_entry = &rt->dst;
 		break;
 	case AF_INET6:
-		rt_dst_entry = ipv6_stub->ipv6_dst_lookup_flow(
-			dev_net(netdev), NULL, &fl6, NULL);
-		if (IS_ERR(rt_dst_entry))
+		if (!IS_ENABLED(CONFIG_IPV6) ||
+		    ip6_dst_lookup(dev_net(netdev), NULL, &rt_dst_entry, &fl6))
 			goto neigh;
 		break;
 	default:
@@ -359,6 +358,9 @@ static void mlx5e_ipsec_init_macs(struct mlx5e_ipsec_sa_entry *sa_entry,
 
 	neigh_ha_snapshot(addr, n, netdev);
 	ether_addr_copy(dst, addr);
+	if (attrs->dir == XFRM_DEV_OFFLOAD_OUT &&
+	    is_zero_ether_addr(addr))
+		neigh_event_send(n, NULL);
 	dst_release(rt_dst_entry);
 	neigh_release(n);
 	return;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 6168f0814414..07fc4d2c8fad 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -6825,7 +6825,6 @@ static void _mlx5e_remove(struct auxiliary_device *adev)
 	 * is already unregistered before changing to NIC profile.
 	 */
 	if (priv->netdev->reg_state == NETREG_REGISTERED) {
-		mlx5e_psp_unregister(priv);
 		unregister_netdev(priv->netdev);
 		_mlx5e_suspend(adev, false);
 	} else {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
index 14884b9ea7f3..a01ee656a1e7 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
@@ -939,7 +939,11 @@ void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq)
 	sq->dma_fifo_cc = dma_fifo_cc;
 	sq->cc = sqcc;
 
-	netdev_tx_completed_queue(sq->txq, npkts, nbytes);
+	/* Do not update BQL for TXQs that got replaced by new active ones, as
+	 * netdev_tx_reset_queue() is called for them in mlx5e_activate_txqsq().
+	 */
+	if (sq == sq->priv->txq2sq[sq->txq_ix])
+		netdev_tx_completed_queue(sq->txq, npkts, nbytes);
 }
 
 #ifdef CONFIG_MLX5_CORE_IPOIB
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
index 8de6c7f6c294..ea94a727633f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
@@ -52,6 +52,7 @@
 #include "devlink.h"
 #include "lag/lag.h"
 #include "en/tc/post_meter.h"
+#include "fw_reset.h"
 
 /* There are two match-all miss flows, one for unicast dst mac and
  * one for multicast.
@@ -3991,6 +3992,11 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
 	if (IS_ERR(esw))
 		return PTR_ERR(esw);
 
+	if (mlx5_fw_reset_in_progress(esw->dev)) {
+		NL_SET_ERR_MSG_MOD(extack, "Can't change eswitch mode during firmware reset");
+		return -EBUSY;
+	}
+
 	if (esw_mode_from_devlink(mode, &mlx5_mode))
 		return -EINVAL;
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
index 2bceb42c98cc..ae10665c53f3 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
@@ -15,6 +15,7 @@ enum {
 	MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS,
 	MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED,
 	MLX5_FW_RESET_FLAGS_UNLOAD_EVENT,
+	MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS,
 };
 
 struct mlx5_fw_reset {
@@ -128,6 +129,16 @@ int mlx5_fw_reset_query(struct mlx5_core_dev *dev, u8 *reset_level, u8 *reset_ty
 	return mlx5_reg_mfrl_query(dev, reset_level, reset_type, NULL, NULL);
 }
 
+bool mlx5_fw_reset_in_progress(struct mlx5_core_dev *dev)
+{
+	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
+
+	if (!fw_reset)
+		return false;
+
+	return test_bit(MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS, &fw_reset->reset_flags);
+}
+
 static int mlx5_fw_reset_get_reset_method(struct mlx5_core_dev *dev,
 					  u8 *reset_method)
 {
@@ -243,6 +254,8 @@ static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev)
 							BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE));
 		devl_unlock(devlink);
 	}
+
+	clear_bit(MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS, &fw_reset->reset_flags);
 }
 
 static void mlx5_stop_sync_reset_poll(struct mlx5_core_dev *dev)
@@ -462,27 +475,48 @@ static void mlx5_sync_reset_request_event(struct work_struct *work)
 	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
 						      reset_request_work);
 	struct mlx5_core_dev *dev = fw_reset->dev;
+	bool nack_request = false;
+	struct devlink *devlink;
 	int err;
 
 	err = mlx5_fw_reset_get_reset_method(dev, &fw_reset->reset_method);
-	if (err)
+	if (err) {
+		nack_request = true;
 		mlx5_core_warn(dev, "Failed reading MFRL, err %d\n", err);
+	} else if (!mlx5_is_reset_now_capable(dev, fw_reset->reset_method) ||
+		   test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST,
+			    &fw_reset->reset_flags)) {
+		nack_request = true;
+	}
 
-	if (err || test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags) ||
-	    !mlx5_is_reset_now_capable(dev, fw_reset->reset_method)) {
+	devlink = priv_to_devlink(dev);
+	/* For external resets, try to acquire devl_lock. Skip if devlink reset is
+	 * pending (lock already held)
+	 */
+	if (nack_request ||
+	    (!test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP,
+		       &fw_reset->reset_flags) &&
+	     !devl_trylock(devlink))) {
 		err = mlx5_fw_reset_set_reset_sync_nack(dev);
 		mlx5_core_warn(dev, "PCI Sync FW Update Reset Nack %s",
 			       err ? "Failed" : "Sent");
 		return;
 	}
+
 	if (mlx5_sync_reset_set_reset_requested(dev))
-		return;
+		goto unlock;
+
+	set_bit(MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS, &fw_reset->reset_flags);
 
 	err = mlx5_fw_reset_set_reset_sync_ack(dev);
 	if (err)
 		mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack Failed. Error code: %d\n", err);
 	else
 		mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack. Device reset is expected.\n");
+
+unlock:
+	if (!test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags))
+		devl_unlock(devlink);
 }
 
 static int mlx5_pci_link_toggle(struct mlx5_core_dev *dev, u16 dev_id)
@@ -722,6 +756,8 @@ static void mlx5_sync_reset_abort_event(struct work_struct *work)
 
 	if (mlx5_sync_reset_clear_reset_requested(dev, true))
 		return;
+
+	clear_bit(MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS, &fw_reset->reset_flags);
 	mlx5_core_warn(dev, "PCI Sync FW Update Reset Aborted.\n");
 }
 
@@ -758,6 +794,7 @@ static void mlx5_sync_reset_timeout_work(struct work_struct *work)
 
 	if (mlx5_sync_reset_clear_reset_requested(dev, true))
 		return;
+	clear_bit(MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS, &fw_reset->reset_flags);
 	mlx5_core_warn(dev, "PCI Sync FW Update Reset Timeout.\n");
 }
 
@@ -844,7 +881,8 @@ void mlx5_drain_fw_reset(struct mlx5_core_dev *dev)
 	cancel_work_sync(&fw_reset->reset_reload_work);
 	cancel_work_sync(&fw_reset->reset_now_work);
 	cancel_work_sync(&fw_reset->reset_abort_work);
-	cancel_delayed_work(&fw_reset->reset_timeout_work);
+	if (test_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags))
+		mlx5_sync_reset_clear_reset_requested(dev, true);
 }
 
 static const struct devlink_param mlx5_fw_reset_devlink_params[] = {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h
index d5b28525c960..2d96b2adc1cd 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.h
@@ -10,6 +10,7 @@ int mlx5_fw_reset_query(struct mlx5_core_dev *dev, u8 *reset_level, u8 *reset_ty
 int mlx5_fw_reset_set_reset_sync(struct mlx5_core_dev *dev, u8 reset_type_sel,
 				 struct netlink_ext_ack *extack);
 int mlx5_fw_reset_set_live_patch(struct mlx5_core_dev *dev);
+bool mlx5_fw_reset_in_progress(struct mlx5_core_dev *dev);
 
 int mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev *dev);
 void mlx5_sync_reset_unload_flow(struct mlx5_core_dev *dev, bool locked);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
index 1ac933cd8f02..a459a30f36ca 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
@@ -1413,6 +1413,7 @@ static int __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev)
 static void mlx5_lag_unregister_hca_devcom_comp(struct mlx5_core_dev *dev)
 {
 	mlx5_devcom_unregister_component(dev->priv.hca_devcom_comp);
+	dev->priv.hca_devcom_comp = NULL;
 }
 
 static int mlx5_lag_register_hca_devcom_comp(struct mlx5_core_dev *dev)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c
index aad52d3a90e6..2d86af8f0d9b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c
@@ -67,12 +67,19 @@ static int mlx5_mpesw_metadata_set(struct mlx5_lag *ldev)
 
 static int enable_mpesw(struct mlx5_lag *ldev)
 {
-	int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
 	struct mlx5_core_dev *dev0;
 	int err;
+	int idx;
 	int i;
 
-	if (idx < 0 || ldev->mode != MLX5_LAG_MODE_NONE)
+	if (ldev->mode == MLX5_LAG_MODE_MPESW)
+		return 0;
+
+	if (ldev->mode != MLX5_LAG_MODE_NONE)
+		return -EINVAL;
+
+	idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
+	if (idx < 0)
 		return -EINVAL;
 
 	dev0 = ldev->pf[idx].dev;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 1ab569ce3fcf..4209da722f9a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -2231,6 +2231,7 @@ static void shutdown(struct pci_dev *pdev)
 
 	mlx5_core_info(dev, "Shutdown was called\n");
 	set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state);
+	mlx5_drain_fw_reset(dev);
 	mlx5_drain_health_wq(dev);
 	err = mlx5_try_fast_unload(dev);
 	if (err)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_dbg.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_dbg.c
index 030a5776c937..8803fa071c50 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_dbg.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_dbg.c
@@ -2,6 +2,7 @@
 // Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
 #include <linux/debugfs.h>
+#include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/seq_file.h>
 #include <linux/version.h>
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr.c
index 5afe6b155ef0..81935f87bfcd 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr.c
@@ -440,7 +440,9 @@ int mlxsw_sp_mr_route_add(struct mlxsw_sp_mr_table *mr_table,
 		rhashtable_remove_fast(&mr_table->route_ht,
 				       &mr_orig_route->ht_node,
 				       mlxsw_sp_mr_route_ht_params);
+		mutex_lock(&mr_table->route_list_lock);
 		list_del(&mr_orig_route->node);
+		mutex_unlock(&mr_table->route_list_lock);
 		mlxsw_sp_mr_route_destroy(mr_table, mr_orig_route);
 	}
 
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
index a2033837182e..2d0e89bd2fb9 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
@@ -2265,6 +2265,7 @@ mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
 	if (!neigh_entry)
 		return NULL;
 
+	neigh_hold(n);
 	neigh_entry->key.n = n;
 	neigh_entry->rif = rif;
 	INIT_LIST_HEAD(&neigh_entry->nexthop_list);
@@ -2274,6 +2275,7 @@ mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
 
 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
 {
+	neigh_release(neigh_entry->key.n);
 	kfree(neigh_entry);
 }
 
@@ -2858,6 +2860,11 @@ static int mlxsw_sp_router_schedule_work(struct net *net,
 	if (!net_work)
 		return NOTIFY_BAD;
 
+	/* Take a reference to ensure the neighbour won't be destructed until
+	 * we drop the reference in the work item.
+	 */
+	neigh_clone(n);
+
 	INIT_WORK(&net_work->work, cb);
 	net_work->mlxsw_sp = router->mlxsw_sp;
 	net_work->n = n;
@@ -2881,11 +2888,6 @@ static int mlxsw_sp_router_schedule_neigh_work(struct mlxsw_sp_router *router,
 	struct net *net;
 
 	net = neigh_parms_net(n->parms);
-
-	/* Take a reference to ensure the neighbour won't be destructed until we
-	 * drop the reference in delayed work.
-	 */
-	neigh_clone(n);
 	return mlxsw_sp_router_schedule_work(net, router, n,
 					     mlxsw_sp_router_neigh_event_work);
 }
@@ -4320,6 +4322,8 @@ mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp,
 	if (err)
 		goto err_neigh_entry_insert;
 
+	neigh_release(old_n);
+
 	read_lock_bh(&n->lock);
 	nud_state = n->nud_state;
 	dead = n->dead;
@@ -4328,14 +4332,10 @@ mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp,
 
 	list_for_each_entry(nh, &neigh_entry->nexthop_list,
 			    neigh_list_node) {
-		neigh_release(old_n);
-		neigh_clone(n);
 		__mlxsw_sp_nexthop_neigh_update(nh, !entry_connected);
 		mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
 	}
 
-	neigh_release(n);
-
 	return 0;
 
 err_neigh_entry_insert:
@@ -4428,6 +4428,11 @@ static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
 		}
 	}
 
+	/* Release the reference taken by neigh_lookup() / neigh_create() since
+	 * neigh_entry already holds one.
+	 */
+	neigh_release(n);
+
 	/* If that is the first nexthop connected to that neigh, add to
 	 * nexthop_neighs_list
 	 */
@@ -4454,11 +4459,9 @@ static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
 					struct mlxsw_sp_nexthop *nh)
 {
 	struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
-	struct neighbour *n;
 
 	if (!neigh_entry)
 		return;
-	n = neigh_entry->key.n;
 
 	__mlxsw_sp_nexthop_neigh_update(nh, true);
 	list_del(&nh->neigh_list_node);
@@ -4472,8 +4475,6 @@ static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
 
 	if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
 		mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
-
-	neigh_release(n);
 }
 
 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev)
diff --git a/drivers/net/ethernet/micrel/ksz884x.c b/drivers/net/ethernet/micrel/ksz884x.c
index cdde19b8edc4..4980a6e44607 100644
--- a/drivers/net/ethernet/micrel/ksz884x.c
+++ b/drivers/net/ethernet/micrel/ksz884x.c
@@ -12,6 +12,7 @@
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/hex.h>
 #include <linux/ioport.h>
 #include <linux/pci.h>
 #include <linux/proc_fs.h>
diff --git a/drivers/net/ethernet/pasemi/pasemi_mac.c b/drivers/net/ethernet/pasemi/pasemi_mac.c
index fe58024b5901..00909372ea61 100644
--- a/drivers/net/ethernet/pasemi/pasemi_mac.c
+++ b/drivers/net/ethernet/pasemi/pasemi_mac.c
@@ -11,6 +11,7 @@
 #include <linux/interrupt.h>
 #include <linux/dmaengine.h>
 #include <linux/delay.h>
+#include <linux/hex.h>
 #include <linux/netdevice.h>
 #include <linux/of_mdio.h>
 #include <linux/etherdevice.h>
diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c
index 405e91eb3141..755083852eef 100644
--- a/drivers/net/ethernet/realtek/r8169_main.c
+++ b/drivers/net/ethernet/realtek/r8169_main.c
@@ -2655,9 +2655,6 @@ static void rtl_wol_enable_rx(struct rtl8169_private *tp)
 
 static void rtl_prepare_power_down(struct rtl8169_private *tp)
 {
-	if (tp->dash_enabled)
-		return;
-
 	if (tp->mac_version == RTL_GIGA_MAC_VER_32 ||
 	    tp->mac_version == RTL_GIGA_MAC_VER_33)
 		rtl_ephy_write(tp, 0x19, 0xff64);
@@ -4812,7 +4809,7 @@ static void rtl8169_down(struct rtl8169_private *tp)
 	rtl_disable_exit_l1(tp);
 	rtl_prepare_power_down(tp);
 
-	if (tp->dash_type != RTL_DASH_NONE)
+	if (tp->dash_type != RTL_DASH_NONE && !tp->saved_wolopts)
 		rtl8168_driver_stop(tp);
 }
 
diff --git a/drivers/net/ethernet/ti/Kconfig b/drivers/net/ethernet/ti/Kconfig
index a54d71155263..fe5b2926d8ab 100644
--- a/drivers/net/ethernet/ti/Kconfig
+++ b/drivers/net/ethernet/ti/Kconfig
@@ -209,6 +209,7 @@ config TI_ICSSG_PRUETH_SR1
 	depends on PRU_REMOTEPROC
 	depends on NET_SWITCHDEV
 	depends on ARCH_K3 && OF && TI_K3_UDMA_GLUE_LAYER
+	depends on PTP_1588_CLOCK_OPTIONAL
 	help
 	  Support dual Gigabit Ethernet ports over the ICSSG PRU Subsystem.
 	  This subsystem is available on the AM65 SR1.0 platform.
@@ -234,7 +235,7 @@ config TI_PRUETH
 	depends on PRU_REMOTEPROC
 	depends on NET_SWITCHDEV
 	select TI_ICSS_IEP
-	imply PTP_1588_CLOCK
+	depends on PTP_1588_CLOCK_OPTIONAL
 	help
 	  Some TI SoCs has Programmable Realtime Unit (PRU) cores which can
 	  support Single or Dual Ethernet ports with the help of firmware code
diff --git a/drivers/net/ipvlan/ipvlan_core.c b/drivers/net/ipvlan/ipvlan_core.c
index dea411e132db..2efa3ba148aa 100644
--- a/drivers/net/ipvlan/ipvlan_core.c
+++ b/drivers/net/ipvlan/ipvlan_core.c
@@ -737,6 +737,9 @@ static rx_handler_result_t ipvlan_handle_mode_l2(struct sk_buff **pskb,
 	struct ethhdr *eth = eth_hdr(skb);
 	rx_handler_result_t ret = RX_HANDLER_PASS;
 
+	if (unlikely(skb->pkt_type == PACKET_LOOPBACK))
+		return RX_HANDLER_PASS;
+
 	if (is_multicast_ether_addr(eth->h_dest)) {
 		if (ipvlan_external_frame(skb, port)) {
 			struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c
index 9cb4dfc242f5..bbf9c02e09d4 100644
--- a/drivers/net/netconsole.c
+++ b/drivers/net/netconsole.c
@@ -36,6 +36,7 @@
 #include <linux/inet.h>
 #include <linux/configfs.h>
 #include <linux/etherdevice.h>
+#include <linux/hex.h>
 #include <linux/u64_stats_sync.h>
 #include <linux/utsname.h>
 #include <linux/rtnetlink.h>
diff --git a/drivers/net/netdevsim/dev.c b/drivers/net/netdevsim/dev.c
index 2683a989873e..351ff4ed3eac 100644
--- a/drivers/net/netdevsim/dev.c
+++ b/drivers/net/netdevsim/dev.c
@@ -18,6 +18,7 @@
 #include <linux/debugfs.h>
 #include <linux/device.h>
 #include <linux/etherdevice.h>
+#include <linux/hex.h>
 #include <linux/inet.h>
 #include <linux/jiffies.h>
 #include <linux/kernel.h>
diff --git a/drivers/net/phy/marvell-88q2xxx.c b/drivers/net/phy/marvell-88q2xxx.c
index f3d83b04c953..201dee1a1698 100644
--- a/drivers/net/phy/marvell-88q2xxx.c
+++ b/drivers/net/phy/marvell-88q2xxx.c
@@ -698,7 +698,7 @@ static int mv88q2xxx_hwmon_write(struct device *dev,
 
 	switch (attr) {
 	case hwmon_temp_max:
-		clamp_val(val, -75000, 180000);
+		val = clamp(val, -75000, 180000);
 		val = (val / 1000) + 75;
 		val = FIELD_PREP(MDIO_MMD_PCS_MV_TEMP_SENSOR3_INT_THRESH_MASK,
 				 val);
diff --git a/drivers/net/phy/realtek/realtek_main.c b/drivers/net/phy/realtek/realtek_main.c
index 67ecf3d4af2b..6ff0385201a5 100644
--- a/drivers/net/phy/realtek/realtek_main.c
+++ b/drivers/net/phy/realtek/realtek_main.c
@@ -691,10 +691,6 @@ static int rtl8211f_config_aldps(struct phy_device *phydev)
 
 static int rtl8211f_config_phy_eee(struct phy_device *phydev)
 {
-	/* RTL8211FVD has no PHYCR2 register */
-	if (phydev->drv->phy_id == RTL_8211FVD_PHYID)
-		return 0;
-
 	/* Disable PHY-mode EEE so LPI is passed to the MAC */
 	return phy_modify_paged(phydev, RTL8211F_PHYCR_PAGE, RTL8211F_PHYCR2,
 				RTL8211F_PHYCR2_PHY_EEE_ENABLE, 0);
diff --git a/drivers/net/phy/sfp.c b/drivers/net/phy/sfp.c
index 0401fa6b24d2..6166e9196364 100644
--- a/drivers/net/phy/sfp.c
+++ b/drivers/net/phy/sfp.c
@@ -497,6 +497,8 @@ static const struct sfp_quirk sfp_quirks[] = {
 	SFP_QUIRK("ALCATELLUCENT", "3FE46541AA", sfp_quirk_2500basex,
 		  sfp_fixup_nokia),
 
+	SFP_QUIRK_F("BIDB", "X-ONU-SFPP", sfp_fixup_potron),
+
 	// FLYPRO SFP-10GT-CS-30M uses Rollball protocol to talk to the PHY.
 	SFP_QUIRK_F("FLYPRO", "SFP-10GT-CS-30M", sfp_fixup_rollball),
 
diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c
index fa5192583860..29179e582067 100644
--- a/drivers/net/usb/r8152.c
+++ b/drivers/net/usb/r8152.c
@@ -10,6 +10,7 @@
 #include <linux/etherdevice.h>
 #include <linux/mii.h>
 #include <linux/ethtool.h>
+#include <linux/hex.h>
 #include <linux/phy.h>
 #include <linux/usb.h>
 #include <linux/crc32.h>
diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c
index 1d9faa70ba3b..b54e07ef84c4 100644
--- a/drivers/net/usb/usbnet.c
+++ b/drivers/net/usb/usbnet.c
@@ -18,6 +18,7 @@
  */
 
 #include <linux/module.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
diff --git a/drivers/net/wireless/ath/ath12k/Makefile b/drivers/net/wireless/ath/ath12k/Makefile
index d95ee525a6cd..3ba1236956cc 100644
--- a/drivers/net/wireless/ath/ath12k/Makefile
+++ b/drivers/net/wireless/ath/ath12k/Makefile
@@ -2,8 +2,6 @@
 obj-$(CONFIG_ATH12K) += ath12k.o
 ath12k-y += core.o \
 	    hal.o \
-	    hal_tx.o \
-	    hal_rx.o \
 	    wmi.o \
 	    mac.o \
 	    reg.o \
@@ -12,11 +10,12 @@ ath12k-y += core.o \
 	    dp.o  \
 	    dp_tx.o \
 	    dp_rx.o \
+	    dp_htt.o \
+	    dp_peer.o \
 	    debug.o \
 	    ce.o \
 	    peer.o \
 	    dbring.o \
-	    hw.o \
 	    mhi.o \
 	    pci.o \
 	    dp_mon.o \
@@ -24,6 +23,9 @@ ath12k-y += core.o \
 	    p2p.o
 
 ath12k-$(CONFIG_ATH12K_AHB) += ahb.o
+
+obj-$(CONFIG_ATH12K) += wifi7/
+
 ath12k-$(CONFIG_ATH12K_DEBUGFS) += debugfs.o debugfs_htt_stats.o debugfs_sta.o
 ath12k-$(CONFIG_ACPI) += acpi.o
 ath12k-$(CONFIG_ATH12K_TRACING) += trace.o
diff --git a/drivers/net/wireless/ath/ath12k/ahb.c b/drivers/net/wireless/ath/ath12k/ahb.c
index b30527c402f6..9a4d34e49104 100644
--- a/drivers/net/wireless/ath/ath12k/ahb.c
+++ b/drivers/net/wireless/ath/ath12k/ahb.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved.
- * Copyright (c) 2022-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #include <linux/dma-mapping.h>
@@ -16,18 +16,11 @@
 #include "debug.h"
 #include "hif.h"
 
-static const struct of_device_id ath12k_ahb_of_match[] = {
-	{ .compatible = "qcom,ipq5332-wifi",
-	  .data = (void *)ATH12K_HW_IPQ5332_HW10,
-	},
-	{ }
-};
-
-MODULE_DEVICE_TABLE(of, ath12k_ahb_of_match);
-
 #define ATH12K_IRQ_CE0_OFFSET 4
 #define ATH12K_MAX_UPDS 1
 #define ATH12K_UPD_IRQ_WRD_LEN  18
+
+static struct ath12k_ahb_driver *ath12k_ahb_family_drivers[ATH12K_DEVICE_FAMILY_MAX];
 static const char ath12k_userpd_irq[][9] = {"spawn",
 				     "ready",
 				     "stop-ack"};
@@ -130,7 +123,7 @@ enum ext_irq_num {
 
 static u32 ath12k_ahb_read32(struct ath12k_base *ab, u32 offset)
 {
-	if (ab->ce_remap && offset < HAL_SEQ_WCSS_CMEM_OFFSET)
+	if (ab->ce_remap && offset < ab->cmem_offset)
 		return ioread32(ab->mem_ce + offset);
 	return ioread32(ab->mem + offset);
 }
@@ -138,7 +131,7 @@ static u32 ath12k_ahb_read32(struct ath12k_base *ab, u32 offset)
 static void ath12k_ahb_write32(struct ath12k_base *ab, u32 offset,
 			       u32 value)
 {
-	if (ab->ce_remap && offset < HAL_SEQ_WCSS_CMEM_OFFSET)
+	if (ab->ce_remap && offset < ab->cmem_offset)
 		iowrite32(value, ab->mem_ce + offset);
 	else
 		iowrite32(value, ab->mem + offset);
@@ -531,9 +524,10 @@ static int ath12k_ahb_ext_grp_napi_poll(struct napi_struct *napi, int budget)
 						struct ath12k_ext_irq_grp,
 						napi);
 	struct ath12k_base *ab = irq_grp->ab;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	int work_done;
 
-	work_done = ath12k_dp_service_srng(ab, irq_grp, budget);
+	work_done = ath12k_dp_service_srng(dp, irq_grp, budget);
 	if (work_done < budget) {
 		napi_complete_done(napi, work_done);
 		ath12k_ahb_ext_grp_enable(irq_grp);
@@ -563,12 +557,10 @@ static int ath12k_ahb_config_ext_irq(struct ath12k_base *ab)
 {
 	const struct ath12k_hw_ring_mask *ring_mask;
 	struct ath12k_ext_irq_grp *irq_grp;
-	const struct hal_ops *hal_ops;
 	int i, j, irq, irq_idx, ret;
 	u32 num_irq;
 
 	ring_mask = ab->hw_params->ring_mask;
-	hal_ops = ab->hw_params->hal_ops;
 	for (i = 0; i < ATH12K_EXT_IRQ_GRP_NUM_MAX; i++) {
 		irq_grp = &ab->ext_irq_grp[i];
 		num_irq = 0;
@@ -588,7 +580,7 @@ static int ath12k_ahb_config_ext_irq(struct ath12k_base *ab)
 			 * tcl_to_wbm_rbm_map point to the same ring number.
 			 */
 			if (ring_mask->tx[i] &
-			    BIT(hal_ops->tcl_to_wbm_rbm_map[j].wbm_ring_num)) {
+			    BIT(ab->hal.tcl_to_wbm_rbm_map[j].wbm_ring_num)) {
 				irq_grp->irqs[num_irq++] =
 					wbm2host_tx_completions_ring1 - j;
 			}
@@ -698,7 +690,7 @@ static int ath12k_ahb_map_service_to_pipe(struct ath12k_base *ab, u16 service_id
 	return 0;
 }
 
-static const struct ath12k_hif_ops ath12k_ahb_hif_ops_ipq5332 = {
+static const struct ath12k_hif_ops ath12k_ahb_hif_ops = {
 	.start = ath12k_ahb_start,
 	.stop = ath12k_ahb_stop,
 	.read32 = ath12k_ahb_read32,
@@ -935,7 +927,8 @@ static int ath12k_ahb_resource_init(struct ath12k_base *ab)
 			goto err_mem_unmap;
 		}
 		ab->ce_remap = true;
-		ab->ce_remap_base_addr = HAL_IPQ5332_CE_WFSS_REG_BASE;
+		ab->cmem_offset = ce_remap->cmem_offset;
+		ab->ce_remap_base_addr = ce_remap->base;
 	}
 
 	ab_ahb->xo_clk = devm_clk_get(ab->dev, "xo");
@@ -988,13 +981,34 @@ static void ath12k_ahb_resource_deinit(struct ath12k_base *ab)
 	ab_ahb->xo_clk = NULL;
 }
 
+static enum ath12k_device_family
+ath12k_ahb_get_device_family(const struct platform_device *pdev)
+{
+	enum ath12k_device_family device_family_id;
+	struct ath12k_ahb_driver *driver;
+	const struct of_device_id *of_id;
+
+	for (device_family_id = ATH12K_DEVICE_FAMILY_START;
+	     device_family_id < ATH12K_DEVICE_FAMILY_MAX; device_family_id++) {
+		driver = ath12k_ahb_family_drivers[device_family_id];
+		if (driver) {
+			of_id = of_match_device(driver->id_table, &pdev->dev);
+			if (of_id) {
+				/* Found the driver */
+				return device_family_id;
+			}
+		}
+	}
+
+	return ATH12K_DEVICE_FAMILY_MAX;
+}
+
 static int ath12k_ahb_probe(struct platform_device *pdev)
 {
-	struct ath12k_base *ab;
-	const struct ath12k_hif_ops *hif_ops;
+	enum ath12k_device_family device_id;
 	struct ath12k_ahb *ab_ahb;
-	enum ath12k_hw_rev hw_rev;
-	u32 addr, userpd_id;
+	struct ath12k_base *ab;
+	u32 addr;
 	int ret;
 
 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
@@ -1008,25 +1022,32 @@ static int ath12k_ahb_probe(struct platform_device *pdev)
 	if (!ab)
 		return -ENOMEM;
 
-	hw_rev = (enum ath12k_hw_rev)(kernel_ulong_t)of_device_get_match_data(&pdev->dev);
-	switch (hw_rev) {
-	case ATH12K_HW_IPQ5332_HW10:
-		hif_ops = &ath12k_ahb_hif_ops_ipq5332;
-		userpd_id = ATH12K_IPQ5332_USERPD_ID;
-		break;
-	default:
-		ret = -EOPNOTSUPP;
+	ab_ahb = ath12k_ab_to_ahb(ab);
+	ab_ahb->ab = ab;
+	ab->hif.ops = &ath12k_ahb_hif_ops;
+	ab->pdev = pdev;
+	platform_set_drvdata(pdev, ab);
+
+	device_id = ath12k_ahb_get_device_family(pdev);
+	if (device_id >= ATH12K_DEVICE_FAMILY_MAX) {
+		ath12k_err(ab, "failed to get device family: %d\n", device_id);
+		ret = -EINVAL;
 		goto err_core_free;
 	}
 
-	ab->hif.ops = hif_ops;
-	ab->pdev = pdev;
-	ab->hw_rev = hw_rev;
-	ab->target_mem_mode = ATH12K_QMI_MEMORY_MODE_DEFAULT;
-	platform_set_drvdata(pdev, ab);
-	ab_ahb = ath12k_ab_to_ahb(ab);
-	ab_ahb->ab = ab;
-	ab_ahb->userpd_id = userpd_id;
+	ath12k_dbg(ab, ATH12K_DBG_AHB, "AHB device family id: %d\n", device_id);
+
+	ab_ahb->device_family_ops = &ath12k_ahb_family_drivers[device_id]->ops;
+
+	/* Call device specific probe. This is the callback that can
+	 * be used to override any ops in future
+	 * probe is validated for NULL during registration.
+	 */
+	ret = ab_ahb->device_family_ops->probe(pdev);
+	if (ret) {
+		ath12k_err(ab, "failed to probe device: %d\n", ret);
+		goto err_core_free;
+	}
 
 	/* Set fixed_mem_region to true for platforms that support fixed memory
 	 * reservation from DT. If memory is reserved from DT for FW, ath12k driver
@@ -1065,14 +1086,26 @@ static int ath12k_ahb_probe(struct platform_device *pdev)
 		goto err_rproc_deconfigure;
 	}
 
+	/* Invoke arch_init here so that arch-specific init operations
+	 * can utilize already initialized ab fields, such as HAL SRNGs.
+	 */
+	ret = ab_ahb->device_family_ops->arch_init(ab);
+	if (ret) {
+		ath12k_err(ab, "AHB arch_init failed %d\n", ret);
+		goto err_rproc_deconfigure;
+	}
+
 	ret = ath12k_core_init(ab);
 	if (ret) {
 		ath12k_err(ab, "failed to init core: %d\n", ret);
-		goto err_rproc_deconfigure;
+		goto err_deinit_arch;
 	}
 
 	return 0;
 
+err_deinit_arch:
+	ab_ahb->device_family_ops->arch_deinit(ab);
+
 err_rproc_deconfigure:
 	ath12k_ahb_deconfigure_rproc(ab);
 
@@ -1111,11 +1144,13 @@ static void ath12k_ahb_remove_prepare(struct ath12k_base *ab)
 static void ath12k_ahb_free_resources(struct ath12k_base *ab)
 {
 	struct platform_device *pdev = ab->pdev;
+	struct ath12k_ahb *ab_ahb = ath12k_ab_to_ahb(ab);
 
 	ath12k_hal_srng_deinit(ab);
 	ath12k_ce_free_pipes(ab);
 	ath12k_ahb_resource_deinit(ab);
 	ath12k_ahb_deconfigure_rproc(ab);
+	ab_ahb->device_family_ops->arch_deinit(ab);
 	ath12k_core_free(ab);
 	platform_set_drvdata(pdev, NULL);
 }
@@ -1136,21 +1171,47 @@ static void ath12k_ahb_remove(struct platform_device *pdev)
 	ath12k_ahb_free_resources(ab);
 }
 
-static struct platform_driver ath12k_ahb_driver = {
-	.driver         = {
-		.name   = "ath12k_ahb",
-		.of_match_table = ath12k_ahb_of_match,
-	},
-	.probe  = ath12k_ahb_probe,
-	.remove = ath12k_ahb_remove,
-};
-
-int ath12k_ahb_init(void)
+int ath12k_ahb_register_driver(const enum ath12k_device_family device_id,
+			       struct ath12k_ahb_driver *driver)
 {
-	return platform_driver_register(&ath12k_ahb_driver);
+	struct platform_driver *ahb_driver;
+
+	if (device_id >= ATH12K_DEVICE_FAMILY_MAX)
+		return -EINVAL;
+
+	if (!driver || !driver->ops.probe ||
+	    !driver->ops.arch_init || !driver->ops.arch_deinit)
+		return -EINVAL;
+
+	if (ath12k_ahb_family_drivers[device_id]) {
+		pr_err("Driver already registered for id %d\n", device_id);
+		return -EALREADY;
+	}
+
+	ath12k_ahb_family_drivers[device_id] = driver;
+
+	ahb_driver = &ath12k_ahb_family_drivers[device_id]->driver;
+	ahb_driver->driver.name = driver->name;
+	ahb_driver->driver.of_match_table = driver->id_table;
+	ahb_driver->probe  = ath12k_ahb_probe;
+	ahb_driver->remove = ath12k_ahb_remove;
+
+	return platform_driver_register(ahb_driver);
 }
+EXPORT_SYMBOL(ath12k_ahb_register_driver);
 
-void ath12k_ahb_exit(void)
+void ath12k_ahb_unregister_driver(const enum ath12k_device_family device_id)
 {
-	platform_driver_unregister(&ath12k_ahb_driver);
+	struct platform_driver *ahb_driver;
+
+	if (device_id >= ATH12K_DEVICE_FAMILY_MAX)
+		return;
+
+	if (!ath12k_ahb_family_drivers[device_id])
+		return;
+
+	ahb_driver = &ath12k_ahb_family_drivers[device_id]->driver;
+	platform_driver_unregister(ahb_driver);
+	ath12k_ahb_family_drivers[device_id] = NULL;
 }
+EXPORT_SYMBOL(ath12k_ahb_unregister_driver);
diff --git a/drivers/net/wireless/ath/ath12k/ahb.h b/drivers/net/wireless/ath/ath12k/ahb.h
index d56244b20a6a..8a040d03d27a 100644
--- a/drivers/net/wireless/ath/ath12k/ahb.h
+++ b/drivers/net/wireless/ath/ath12k/ahb.h
@@ -1,13 +1,14 @@
 /* SPDX-License-Identifier: BSD-3-Clause-Clear */
 /*
  * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved.
- * Copyright (c) 2022-2025, Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 #ifndef ATH12K_AHB_H
 #define ATH12K_AHB_H
 
 #include <linux/clk.h>
 #include <linux/remoteproc/qcom_rproc.h>
+#include <linux/platform_device.h>
 #include "core.h"
 
 #define ATH12K_AHB_RECOVERY_TIMEOUT (3 * HZ)
@@ -43,6 +44,12 @@ enum ath12k_ahb_userpd_irq {
 
 struct ath12k_base;
 
+struct ath12k_ahb_device_family_ops {
+	int (*probe)(struct platform_device *pdev);
+	int (*arch_init)(struct ath12k_base *ab);
+	void (*arch_deinit)(struct ath12k_base *ab);
+};
+
 struct ath12k_ahb {
 	struct ath12k_base *ab;
 	struct rproc *tgt_rproc;
@@ -59,6 +66,15 @@ struct ath12k_ahb {
 	u32 spawn_bit;
 	u32 stop_bit;
 	int userpd_irq_num[ATH12K_USERPD_MAX_IRQ];
+	const struct ath12k_ahb_ops *ahb_ops;
+	const struct ath12k_ahb_device_family_ops *device_family_ops;
+};
+
+struct ath12k_ahb_driver {
+	const char *name;
+	const struct of_device_id *id_table;
+	struct ath12k_ahb_device_family_ops ops;
+	struct platform_driver driver;
 };
 
 static inline struct ath12k_ahb *ath12k_ab_to_ahb(struct ath12k_base *ab)
@@ -66,15 +82,8 @@ static inline struct ath12k_ahb *ath12k_ab_to_ahb(struct ath12k_base *ab)
 	return (struct ath12k_ahb *)ab->drv_priv;
 }
 
-#ifdef CONFIG_ATH12K_AHB
-int ath12k_ahb_init(void);
-void ath12k_ahb_exit(void);
-#else
-static inline int ath12k_ahb_init(void)
-{
-	return 0;
-}
+int ath12k_ahb_register_driver(const enum ath12k_device_family device_id,
+			       struct ath12k_ahb_driver *driver);
+void ath12k_ahb_unregister_driver(const enum ath12k_device_family device_id);
 
-static inline void ath12k_ahb_exit(void) {};
-#endif
 #endif
diff --git a/drivers/net/wireless/ath/ath12k/ce.c b/drivers/net/wireless/ath/ath12k/ce.c
index 9a63608838ac..9f9d2f2477c7 100644
--- a/drivers/net/wireless/ath/ath12k/ce.c
+++ b/drivers/net/wireless/ath/ath12k/ce.c
@@ -1,314 +1,13 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2022, 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #include "dp_rx.h"
 #include "debug.h"
 #include "hif.h"
 
-const struct ce_attr ath12k_host_ce_config_qcn9274[] = {
-	/* CE0: host->target HTC control and raw streams */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 16,
-		.src_sz_max = 2048,
-		.dest_nentries = 0,
-	},
-
-	/* CE1: target->host HTT + HTC control */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 0,
-		.src_sz_max = 2048,
-		.dest_nentries = 512,
-		.recv_cb = ath12k_htc_rx_completion_handler,
-	},
-
-	/* CE2: target->host WMI */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 0,
-		.src_sz_max = 2048,
-		.dest_nentries = 128,
-		.recv_cb = ath12k_htc_rx_completion_handler,
-	},
-
-	/* CE3: host->target WMI (mac0) */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 32,
-		.src_sz_max = 2048,
-		.dest_nentries = 0,
-	},
-
-	/* CE4: host->target HTT */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 2048,
-		.src_sz_max = 256,
-		.dest_nentries = 0,
-	},
-
-	/* CE5: target->host pktlog */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 0,
-		.src_sz_max = 2048,
-		.dest_nentries = 512,
-		.recv_cb = ath12k_dp_htt_htc_t2h_msg_handler,
-	},
-
-	/* CE6: target autonomous hif_memcpy */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 0,
-		.src_sz_max = 0,
-		.dest_nentries = 0,
-	},
-
-	/* CE7: host->target WMI (mac1) */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 32,
-		.src_sz_max = 2048,
-		.dest_nentries = 0,
-	},
-
-	/* CE8: target autonomous hif_memcpy */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 0,
-		.src_sz_max = 0,
-		.dest_nentries = 0,
-	},
-
-	/* CE9: MHI */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 0,
-		.src_sz_max = 0,
-		.dest_nentries = 0,
-	},
-
-	/* CE10: MHI */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 0,
-		.src_sz_max = 0,
-		.dest_nentries = 0,
-	},
-
-	/* CE11: MHI */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 0,
-		.src_sz_max = 0,
-		.dest_nentries = 0,
-	},
-
-	/* CE12: CV Prefetch */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 0,
-		.src_sz_max = 0,
-		.dest_nentries = 0,
-	},
-
-	/* CE13: CV Prefetch */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 0,
-		.src_sz_max = 0,
-		.dest_nentries = 0,
-	},
-
-	/* CE14: target->host dbg log */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 0,
-		.src_sz_max = 2048,
-		.dest_nentries = 512,
-		.recv_cb = ath12k_htc_rx_completion_handler,
-	},
-
-	/* CE15: reserved for future use */
-	{
-		.flags = (CE_ATTR_FLAGS | CE_ATTR_DIS_INTR),
-		.src_nentries = 0,
-		.src_sz_max = 0,
-		.dest_nentries = 0,
-	},
-};
-
-const struct ce_attr ath12k_host_ce_config_wcn7850[] = {
-	/* CE0: host->target HTC control and raw streams */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 16,
-		.src_sz_max = 2048,
-		.dest_nentries = 0,
-	},
-
-	/* CE1: target->host HTT + HTC control */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 0,
-		.src_sz_max = 2048,
-		.dest_nentries = 512,
-		.recv_cb = ath12k_htc_rx_completion_handler,
-	},
-
-	/* CE2: target->host WMI */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 0,
-		.src_sz_max = 2048,
-		.dest_nentries = 64,
-		.recv_cb = ath12k_htc_rx_completion_handler,
-	},
-
-	/* CE3: host->target WMI (mac0) */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 32,
-		.src_sz_max = 2048,
-		.dest_nentries = 0,
-	},
-
-	/* CE4: host->target HTT */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 2048,
-		.src_sz_max = 256,
-		.dest_nentries = 0,
-	},
-
-	/* CE5: target->host pktlog */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 0,
-		.src_sz_max = 0,
-		.dest_nentries = 0,
-	},
-
-	/* CE6: target autonomous hif_memcpy */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 0,
-		.src_sz_max = 0,
-		.dest_nentries = 0,
-	},
-
-	/* CE7: host->target WMI (mac1) */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 0,
-		.src_sz_max = 2048,
-		.dest_nentries = 0,
-	},
-
-	/* CE8: target autonomous hif_memcpy */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 0,
-		.src_sz_max = 0,
-		.dest_nentries = 0,
-	},
-
-};
-
-const struct ce_attr ath12k_host_ce_config_ipq5332[] = {
-	/* CE0: host->target HTC control and raw streams */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 16,
-		.src_sz_max = 2048,
-		.dest_nentries = 0,
-	},
-	/* CE1: target->host HTT + HTC control */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 0,
-		.src_sz_max = 2048,
-		.dest_nentries = 512,
-		.recv_cb = ath12k_htc_rx_completion_handler,
-	},
-	/* CE2: target->host WMI */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 0,
-		.src_sz_max = 2048,
-		.dest_nentries = 128,
-		.recv_cb = ath12k_htc_rx_completion_handler,
-	},
-	/* CE3: host->target WMI */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 32,
-		.src_sz_max = 2048,
-		.dest_nentries = 0,
-	},
-	/* CE4: host->target HTT */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 2048,
-		.src_sz_max = 256,
-		.dest_nentries = 0,
-	},
-	/* CE5: target -> host PKTLOG */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 0,
-		.src_sz_max = 2048,
-		.dest_nentries = 512,
-		.recv_cb = ath12k_dp_htt_htc_t2h_msg_handler,
-	},
-	/* CE6: Target autonomous HIF_memcpy */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 0,
-		.src_sz_max = 0,
-		.dest_nentries = 0,
-	},
-	/* CE7: CV Prefetch */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 0,
-		.src_sz_max = 0,
-		.dest_nentries = 0,
-	},
-	/* CE8: Target HIF memcpy (Generic HIF memcypy) */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 0,
-		.src_sz_max = 0,
-		.dest_nentries = 0,
-	},
-	/* CE9: WMI logging/CFR/Spectral/Radar */
-	{
-		.flags = CE_ATTR_FLAGS,
-		.src_nentries = 0,
-		.src_sz_max = 2048,
-		.dest_nentries = 128,
-	},
-	/* CE10: Unused */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 0,
-		.src_sz_max = 0,
-		.dest_nentries = 0,
-	},
-	/* CE11: Unused */
-	{
-		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
-		.src_nentries = 0,
-		.src_sz_max = 0,
-		.dest_nentries = 0,
-	},
-};
-
 static int ath12k_ce_rx_buf_enqueue_pipe(struct ath12k_ce_pipe *pipe,
 					 struct sk_buff *skb, dma_addr_t paddr)
 {
@@ -341,7 +40,7 @@ static int ath12k_ce_rx_buf_enqueue_pipe(struct ath12k_ce_pipe *pipe,
 		goto exit;
 	}
 
-	ath12k_hal_ce_dst_set_desc(desc, paddr);
+	ath12k_hal_ce_dst_set_desc(&ab->hal, desc, paddr);
 
 	ring->skb[write_index] = skb;
 	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
@@ -434,7 +133,7 @@ static int ath12k_ce_completed_recv_next(struct ath12k_ce_pipe *pipe,
 		goto err;
 	}
 
-	*nbytes = ath12k_hal_ce_dst_status_get_length(desc);
+	*nbytes = ath12k_hal_ce_dst_status_get_length(&ab->hal, desc);
 
 	*skb = pipe->dest_ring->skb[sw_index];
 	pipe->dest_ring->skb[sw_index] = NULL;
@@ -666,6 +365,7 @@ ath12k_ce_alloc_ring(struct ath12k_base *ab, int nentries, int desc_sz)
 
 static int ath12k_ce_alloc_pipe(struct ath12k_base *ab, int ce_id)
 {
+	struct ath12k_hal *hal = &ab->hal;
 	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id];
 	const struct ce_attr *attr = &ab->hw_params->host_ce_config[ce_id];
 	struct ath12k_ce_ring *ring;
@@ -677,7 +377,7 @@ static int ath12k_ce_alloc_pipe(struct ath12k_base *ab, int ce_id)
 	if (attr->src_nentries) {
 		pipe->send_cb = ath12k_ce_send_done_cb;
 		nentries = roundup_pow_of_two(attr->src_nentries);
-		desc_sz = ath12k_hal_ce_get_desc_size(HAL_CE_DESC_SRC);
+		desc_sz = ath12k_hal_ce_get_desc_size(hal, HAL_CE_DESC_SRC);
 		ring = ath12k_ce_alloc_ring(ab, nentries, desc_sz);
 		if (IS_ERR(ring))
 			return PTR_ERR(ring);
@@ -687,13 +387,13 @@ static int ath12k_ce_alloc_pipe(struct ath12k_base *ab, int ce_id)
 	if (attr->dest_nentries) {
 		pipe->recv_cb = attr->recv_cb;
 		nentries = roundup_pow_of_two(attr->dest_nentries);
-		desc_sz = ath12k_hal_ce_get_desc_size(HAL_CE_DESC_DST);
+		desc_sz = ath12k_hal_ce_get_desc_size(hal, HAL_CE_DESC_DST);
 		ring = ath12k_ce_alloc_ring(ab, nentries, desc_sz);
 		if (IS_ERR(ring))
 			return PTR_ERR(ring);
 		pipe->dest_ring = ring;
 
-		desc_sz = ath12k_hal_ce_get_desc_size(HAL_CE_DESC_DST_STATUS);
+		desc_sz = ath12k_hal_ce_get_desc_size(hal, HAL_CE_DESC_DST_STATUS);
 		ring = ath12k_ce_alloc_ring(ab, nentries, desc_sz);
 		if (IS_ERR(ring))
 			return PTR_ERR(ring);
@@ -786,7 +486,7 @@ int ath12k_ce_send(struct ath12k_base *ab, struct sk_buff *skb, u8 pipe_id,
 	if (pipe->attr_flags & CE_ATTR_BYTE_SWAP_DATA)
 		byte_swap_data = 1;
 
-	ath12k_hal_ce_src_set_desc(desc, ATH12K_SKB_CB(skb)->paddr,
+	ath12k_hal_ce_src_set_desc(&ab->hal, desc, ATH12K_SKB_CB(skb)->paddr,
 				   skb->len, transfer_id, byte_swap_data);
 
 	pipe->src_ring->skb[write_index] = skb;
@@ -972,6 +672,7 @@ int ath12k_ce_init_pipes(struct ath12k_base *ab)
 
 void ath12k_ce_free_pipes(struct ath12k_base *ab)
 {
+	struct ath12k_hal *hal = &ab->hal;
 	struct ath12k_ce_pipe *pipe;
 	int desc_sz;
 	int i;
@@ -980,7 +681,8 @@ void ath12k_ce_free_pipes(struct ath12k_base *ab)
 		pipe = &ab->ce.ce_pipe[i];
 
 		if (pipe->src_ring) {
-			desc_sz = ath12k_hal_ce_get_desc_size(HAL_CE_DESC_SRC);
+			desc_sz = ath12k_hal_ce_get_desc_size(hal,
+							      HAL_CE_DESC_SRC);
 			dma_free_coherent(ab->dev,
 					  pipe->src_ring->nentries * desc_sz +
 					  CE_DESC_RING_ALIGN,
@@ -991,7 +693,8 @@ void ath12k_ce_free_pipes(struct ath12k_base *ab)
 		}
 
 		if (pipe->dest_ring) {
-			desc_sz = ath12k_hal_ce_get_desc_size(HAL_CE_DESC_DST);
+			desc_sz = ath12k_hal_ce_get_desc_size(hal,
+							      HAL_CE_DESC_DST);
 			dma_free_coherent(ab->dev,
 					  pipe->dest_ring->nentries * desc_sz +
 					  CE_DESC_RING_ALIGN,
@@ -1003,7 +706,8 @@ void ath12k_ce_free_pipes(struct ath12k_base *ab)
 
 		if (pipe->status_ring) {
 			desc_sz =
-			  ath12k_hal_ce_get_desc_size(HAL_CE_DESC_DST_STATUS);
+			  ath12k_hal_ce_get_desc_size(hal,
+						      HAL_CE_DESC_DST_STATUS);
 			dma_free_coherent(ab->dev,
 					  pipe->status_ring->nentries * desc_sz +
 					  CE_DESC_RING_ALIGN,
diff --git a/drivers/net/wireless/ath/ath12k/ce.h b/drivers/net/wireless/ath/ath12k/ce.h
index 57f75899ee03..df4f2a4f8480 100644
--- a/drivers/net/wireless/ath/ath12k/ce.h
+++ b/drivers/net/wireless/ath/ath12k/ce.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause-Clear */
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2022, 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #ifndef ATH12K_CE_H
@@ -85,6 +85,7 @@ struct ce_ie_addr {
 struct ce_remap {
 	u32 base;
 	u32 size;
+	u32 cmem_offset;
 };
 
 struct ce_attr {
@@ -173,10 +174,6 @@ struct ath12k_ce {
 	struct ath12k_hp_update_timer hp_timer[CE_COUNT_MAX];
 };
 
-extern const struct ce_attr ath12k_host_ce_config_qcn9274[];
-extern const struct ce_attr ath12k_host_ce_config_wcn7850[];
-extern const struct ce_attr ath12k_host_ce_config_ipq5332[];
-
 void ath12k_ce_cleanup_pipes(struct ath12k_base *ab);
 void ath12k_ce_rx_replenish_retry(struct timer_list *t);
 void ath12k_ce_per_engine_service(struct ath12k_base *ab, u16 ce_id);
diff --git a/drivers/net/wireless/ath/ath12k/cmn_defs.h b/drivers/net/wireless/ath/ath12k/cmn_defs.h
new file mode 100644
index 000000000000..20208ffea1c9
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/cmn_defs.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef ATH12K_CMN_DEFS_H
+#define ATH12K_CMN_DEFS_H
+
+#include <net/mac80211.h>
+
+#define MAX_RADIOS 2
+#define ATH12K_MAX_DEVICES 3
+#define ATH12K_GROUP_MAX_RADIO (ATH12K_MAX_DEVICES * MAX_RADIOS)
+
+#define ATH12K_SCAN_MAX_LINKS	ATH12K_GROUP_MAX_RADIO
+/* Define 1 scan link for each radio for parallel scan purposes */
+#define ATH12K_NUM_MAX_LINKS (IEEE80211_MLD_MAX_NUM_LINKS + ATH12K_SCAN_MAX_LINKS)
+
+#define MAX_MU_GROUP_ID 64
+#endif
diff --git a/drivers/net/wireless/ath/ath12k/core.c b/drivers/net/wireless/ath/ath12k/core.c
index cc352eef1939..9d6c50a94e64 100644
--- a/drivers/net/wireless/ath/ath12k/core.c
+++ b/drivers/net/wireless/ath/ath12k/core.c
@@ -21,15 +21,18 @@
 #include "hif.h"
 #include "pci.h"
 #include "wow.h"
+#include "dp_cmn.h"
+#include "peer.h"
 
-static int ahb_err, pci_err;
 unsigned int ath12k_debug_mask;
 module_param_named(debug_mask, ath12k_debug_mask, uint, 0644);
 MODULE_PARM_DESC(debug_mask, "Debugging mask");
+EXPORT_SYMBOL(ath12k_debug_mask);
 
 bool ath12k_ftm_mode;
 module_param_named(ftm_mode, ath12k_ftm_mode, bool, 0444);
 MODULE_PARM_DESC(ftm_mode, "Boots up in factory test mode");
+EXPORT_SYMBOL(ath12k_ftm_mode);
 
 /* protected with ath12k_hw_group_mutex */
 static struct list_head ath12k_hw_group_list = LIST_HEAD_INIT(ath12k_hw_group_list);
@@ -632,6 +635,7 @@ u32 ath12k_core_get_max_peers_per_radio(struct ath12k_base *ab)
 {
 	return ath12k_core_get_max_station_per_radio(ab) + TARGET_NUM_VDEVS(ab);
 }
+EXPORT_SYMBOL(ath12k_core_get_max_peers_per_radio);
 
 struct reserved_mem *ath12k_core_get_reserved_mem(struct ath12k_base *ab,
 						  int index)
@@ -700,6 +704,8 @@ void ath12k_core_to_group_ref_put(struct ath12k_base *ab)
 
 static void ath12k_core_stop(struct ath12k_base *ab)
 {
+	ath12k_link_sta_rhash_tbl_destroy(ab);
+
 	ath12k_core_to_group_ref_put(ab);
 
 	if (!test_bit(ATH12K_FLAG_CRASH_FLUSH, &ab->dev_flags))
@@ -710,7 +716,7 @@ static void ath12k_core_stop(struct ath12k_base *ab)
 	ath12k_dp_rx_pdev_reo_cleanup(ab);
 	ath12k_hif_stop(ab);
 	ath12k_wmi_detach(ab);
-	ath12k_dp_free(ab);
+	ath12k_dp_cmn_device_deinit(ath12k_ab_to_dp(ab));
 
 	/* De-Init of components as needed */
 }
@@ -895,7 +901,7 @@ static int ath12k_core_start(struct ath12k_base *ab)
 		goto err_hif_stop;
 	}
 
-	ret = ath12k_dp_htt_connect(&ab->dp);
+	ret = ath12k_dp_htt_connect(ath12k_ab_to_dp(ab));
 	if (ret) {
 		ath12k_err(ab, "failed to connect to HTT: %d\n", ret);
 		goto err_hif_stop;
@@ -920,7 +926,7 @@ static int ath12k_core_start(struct ath12k_base *ab)
 		goto err_hif_stop;
 	}
 
-	ath12k_dp_cc_config(ab);
+	ath12k_hal_cc_config(ab);
 
 	ret = ath12k_dp_rx_pdev_reo_setup(ab);
 	if (ret) {
@@ -928,8 +934,6 @@ static int ath12k_core_start(struct ath12k_base *ab)
 		goto err_hif_stop;
 	}
 
-	ath12k_dp_hal_rx_desc_init(ab);
-
 	ret = ath12k_wmi_cmd_init(ab);
 	if (ret) {
 		ath12k_err(ab, "failed to send wmi init cmd: %d\n", ret);
@@ -964,6 +968,12 @@ static int ath12k_core_start(struct ath12k_base *ab)
 	/* Indicate the core start in the appropriate group */
 	ath12k_core_to_group_ref_get(ab);
 
+	ret = ath12k_link_sta_rhash_tbl_init(ab);
+	if (ret) {
+		ath12k_warn(ab, "failed to init peer addr rhash table %d\n", ret);
+		goto err_reo_cleanup;
+	}
+
 	return 0;
 
 err_reo_cleanup:
@@ -1288,7 +1298,7 @@ int ath12k_core_qmi_firmware_ready(struct ath12k_base *ab)
 		goto err_firmware_stop;
 	}
 
-	ret = ath12k_dp_alloc(ab);
+	ret = ath12k_dp_cmn_device_init(ath12k_ab_to_dp(ab));
 	if (ret) {
 		ath12k_err(ab, "failed to init DP: %d\n", ret);
 		goto err_firmware_stop;
@@ -1300,7 +1310,7 @@ int ath12k_core_qmi_firmware_ready(struct ath12k_base *ab)
 	ret = ath12k_core_start(ab);
 	if (ret) {
 		ath12k_err(ab, "failed to start core: %d\n", ret);
-		goto err_dp_free;
+		goto err_deinit;
 	}
 
 	mutex_unlock(&ab->core_lock);
@@ -1333,8 +1343,8 @@ int ath12k_core_qmi_firmware_ready(struct ath12k_base *ab)
 	mutex_unlock(&ag->mutex);
 	goto exit;
 
-err_dp_free:
-	ath12k_dp_free(ab);
+err_deinit:
+	ath12k_dp_cmn_device_deinit(ath12k_ab_to_dp(ab));
 	mutex_unlock(&ab->core_lock);
 	mutex_unlock(&ag->mutex);
 
@@ -1350,13 +1360,14 @@ static int ath12k_core_reconfigure_on_crash(struct ath12k_base *ab)
 	int ret, total_vdev;
 
 	mutex_lock(&ab->core_lock);
+	ath12k_link_sta_rhash_tbl_destroy(ab);
 	ath12k_dp_pdev_free(ab);
 	ath12k_ce_cleanup_pipes(ab);
 	ath12k_wmi_detach(ab);
 	ath12k_dp_rx_pdev_reo_cleanup(ab);
 	mutex_unlock(&ab->core_lock);
 
-	ath12k_dp_free(ab);
+	ath12k_dp_cmn_device_deinit(ath12k_ab_to_dp(ab));
 	ath12k_hal_srng_deinit(ab);
 	total_vdev = ab->num_radios * TARGET_NUM_VDEVS(ab);
 	ab->free_vdev_map = (1LL << total_vdev) - 1;
@@ -1565,6 +1576,7 @@ static void ath12k_core_post_reconfigure_recovery(struct ath12k_base *ab)
 				ath12k_core_halt(ar);
 			}
 
+			ath12k_mac_dp_peer_cleanup(ah);
 			break;
 		case ATH12K_HW_STATE_OFF:
 			ath12k_warn(ab,
@@ -1739,17 +1751,11 @@ enum ath12k_qmi_mem_mode ath12k_core_get_memory_mode(struct ath12k_base *ab)
 
 	return ATH12K_QMI_MEMORY_MODE_DEFAULT;
 }
+EXPORT_SYMBOL(ath12k_core_get_memory_mode);
 
 int ath12k_core_pre_init(struct ath12k_base *ab)
 {
 	const struct ath12k_mem_profile_based_param *param;
-	int ret;
-
-	ret = ath12k_hw_init(ab);
-	if (ret) {
-		ath12k_err(ab, "failed to init hw params: %d\n", ret);
-		return ret;
-	}
 
 	param = &ath12k_mem_profile_based_param[ab->target_mem_mode];
 	ab->profile_param = param;
@@ -1996,6 +2002,8 @@ static struct ath12k_hw_group *ath12k_core_hw_group_assign(struct ath12k_base *a
 	ag->ab[ab->device_id] = ab;
 	ab->ag = ag;
 
+	ath12k_dp_cmn_hw_group_assign(ath12k_ab_to_dp(ab), ag);
+
 	ath12k_dbg(ab, ATH12K_DBG_BOOT, "wsi group-id %d num-devices %d index %d",
 		   ag->id, ag->num_devices, wsi->index);
 
@@ -2023,6 +2031,8 @@ void ath12k_core_hw_group_unassign(struct ath12k_base *ab)
 		return;
 	}
 
+	ath12k_dp_cmn_hw_group_unassign(ath12k_ab_to_dp(ab), ag);
+
 	ag->ab[device_id] = NULL;
 	ab->ag = NULL;
 	ab->device_id = ATH12K_INVALID_DEVICE_ID;
@@ -2253,7 +2263,6 @@ struct ath12k_base *ath12k_core_alloc(struct device *dev, size_t priv_size,
 	spin_lock_init(&ab->base_lock);
 	init_completion(&ab->reset_complete);
 
-	INIT_LIST_HEAD(&ab->peers);
 	init_waitqueue_head(&ab->peer_mapping_wq);
 	init_waitqueue_head(&ab->wmi_ab.tx_credits_wq);
 	INIT_WORK(&ab->restart_work, ath12k_core_restart);
@@ -2291,31 +2300,5 @@ struct ath12k_base *ath12k_core_alloc(struct device *dev, size_t priv_size,
 	return NULL;
 }
 
-static int ath12k_init(void)
-{
-	ahb_err = ath12k_ahb_init();
-	if (ahb_err)
-		pr_warn("Failed to initialize ath12k AHB device: %d\n", ahb_err);
-
-	pci_err = ath12k_pci_init();
-	if (pci_err)
-		pr_warn("Failed to initialize ath12k PCI device: %d\n", pci_err);
-
-	/* If both failed, return one of the failures (arbitrary) */
-	return ahb_err && pci_err ? ahb_err : 0;
-}
-
-static void ath12k_exit(void)
-{
-	if (!pci_err)
-		ath12k_pci_exit();
-
-	if (!ahb_err)
-		ath12k_ahb_exit();
-}
-
-module_init(ath12k_init);
-module_exit(ath12k_exit);
-
-MODULE_DESCRIPTION("Driver support for Qualcomm Technologies 802.11be WLAN devices");
+MODULE_DESCRIPTION("Driver support for Qualcomm Technologies WLAN devices");
 MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/ath/ath12k/core.h b/drivers/net/wireless/ath/ath12k/core.h
index 3c1e0069be1e..31d5d10beb85 100644
--- a/drivers/net/wireless/ath/ath12k/core.h
+++ b/drivers/net/wireless/ath/ath12k/core.h
@@ -18,6 +18,7 @@
 #include <linux/panic_notifier.h>
 #include <linux/average.h>
 #include <linux/of.h>
+#include <linux/rhashtable.h>
 #include "qmi.h"
 #include "htc.h"
 #include "wmi.h"
@@ -26,7 +27,6 @@
 #include "ce.h"
 #include "mac.h"
 #include "hw.h"
-#include "hal_rx.h"
 #include "reg.h"
 #include "dbring.h"
 #include "fw.h"
@@ -34,6 +34,8 @@
 #include "wow.h"
 #include "debugfs_htt_stats.h"
 #include "coredump.h"
+#include "cmn_defs.h"
+#include "dp_cmn.h"
 
 #define SM(_v, _f) (((_v) << _f##_LSB) & _f##_MASK)
 
@@ -64,8 +66,6 @@
 #define ATH12K_RECONFIGURE_TIMEOUT_HZ		(10 * HZ)
 #define ATH12K_RECOVER_START_TIMEOUT_HZ		(20 * HZ)
 
-#define ATH12K_MAX_DEVICES 3
-#define ATH12K_GROUP_MAX_RADIO (ATH12K_MAX_DEVICES * MAX_RADIOS)
 #define ATH12K_INVALID_GROUP_ID  0xFF
 #define ATH12K_INVALID_DEVICE_ID 0xFF
 
@@ -310,16 +310,9 @@ struct ath12k_link_vif {
 	u32 vdev_id;
 	u32 beacon_interval;
 	u32 dtim_period;
-	u16 ast_hash;
-	u16 ast_idx;
-	u16 tcl_metadata;
-	u8 hal_addr_search_flags;
-	u8 search_type;
 
 	struct ath12k *ar;
 
-	int bank_id;
-	u8 vdev_id_check_en;
 	bool beacon_prot;
 
 	struct wmi_wmm_params_all_arg wmm_params;
@@ -360,6 +353,8 @@ struct ath12k_link_vif {
 };
 
 struct ath12k_vif {
+	struct ath12k_dp_vif dp_vif;
+
 	enum wmi_vdev_type vdev_type;
 	enum wmi_vdev_subtype vdev_subtype;
 	struct ieee80211_vif *vif;
@@ -383,10 +378,7 @@ struct ath12k_vif {
 	} u;
 
 	u32 aid;
-	u32 key_cipher;
-	u8 tx_encap_type;
 	bool ps;
-	atomic_t mcbc_gsn;
 
 	struct ath12k_link_vif deflink;
 	struct ath12k_link_vif __rcu *link[ATH12K_NUM_MAX_LINKS];
@@ -407,51 +399,8 @@ struct ath12k_vif_iter {
 	struct ath12k_link_vif *arvif;
 };
 
-#define HAL_AST_IDX_INVALID	0xFFFF
-#define HAL_RX_MAX_MCS		12
-#define HAL_RX_MAX_MCS_HT	31
-#define HAL_RX_MAX_MCS_VHT	9
-#define HAL_RX_MAX_MCS_HE	11
-#define HAL_RX_MAX_MCS_BE	15
-#define HAL_RX_MAX_NSS		8
-#define HAL_RX_MAX_NUM_LEGACY_RATES 12
-
 #define ATH12K_SCAN_TIMEOUT_HZ (20 * HZ)
 
-struct ath12k_rx_peer_rate_stats {
-	u64 ht_mcs_count[HAL_RX_MAX_MCS_HT + 1];
-	u64 vht_mcs_count[HAL_RX_MAX_MCS_VHT + 1];
-	u64 he_mcs_count[HAL_RX_MAX_MCS_HE + 1];
-	u64 be_mcs_count[HAL_RX_MAX_MCS_BE + 1];
-	u64 nss_count[HAL_RX_MAX_NSS];
-	u64 bw_count[HAL_RX_BW_MAX];
-	u64 gi_count[HAL_RX_GI_MAX];
-	u64 legacy_count[HAL_RX_MAX_NUM_LEGACY_RATES];
-	u64 rx_rate[HAL_RX_BW_MAX][HAL_RX_GI_MAX][HAL_RX_MAX_NSS][HAL_RX_MAX_MCS_HT + 1];
-};
-
-struct ath12k_rx_peer_stats {
-	u64 num_msdu;
-	u64 num_mpdu_fcs_ok;
-	u64 num_mpdu_fcs_err;
-	u64 tcp_msdu_count;
-	u64 udp_msdu_count;
-	u64 other_msdu_count;
-	u64 ampdu_msdu_count;
-	u64 non_ampdu_msdu_count;
-	u64 stbc_count;
-	u64 beamformed_count;
-	u64 coding_count[HAL_RX_SU_MU_CODING_MAX];
-	u64 tid_count[IEEE80211_NUM_TIDS + 1];
-	u64 pream_cnt[HAL_RX_PREAMBLE_MAX];
-	u64 reception_type[HAL_RX_RECEPTION_TYPE_MAX];
-	u64 rx_duration;
-	u64 dcm_count;
-	u64 ru_alloc_cnt[HAL_RX_RU_ALLOC_TYPE_MAX];
-	struct ath12k_rx_peer_rate_stats pkt_stats;
-	struct ath12k_rx_peer_rate_stats byte_stats;
-};
-
 #define ATH12K_HE_MCS_NUM       12
 #define ATH12K_VHT_MCS_NUM      10
 #define ATH12K_BW_NUM           5
@@ -533,12 +482,6 @@ struct ath12k_per_ppdu_tx_stats {
 	u32 retry_bytes;
 };
 
-struct ath12k_wbm_tx_stats {
-	u64 wbm_tx_comp_stats[HAL_WBM_REL_HTT_TX_COMP_STATUS_MAX];
-};
-
-DECLARE_EWMA(avg_rssi, 10, 8)
-
 struct ath12k_link_sta {
 	struct ath12k_link_vif *arvif;
 	struct ath12k_sta *ahsta;
@@ -553,15 +496,7 @@ struct ath12k_link_sta {
 	u32 smps;
 
 	struct wiphy_work update_wk;
-	struct rate_info txrate;
-	struct rate_info last_txrate;
-	u64 rx_duration;
-	u64 tx_duration;
-	u8 rssi_comb;
-	struct ewma_avg_rssi avg_rssi;
 	u8 link_id;
-	struct ath12k_rx_peer_stats *rx_stats;
-	struct ath12k_wbm_tx_stats *wbm_tx_stats;
 	u32 bw_prev;
 	u32 peer_nss;
 	s8 rssi_beacon;
@@ -572,14 +507,9 @@ struct ath12k_link_sta {
 
 	 /* for firmware use only */
 	u8 link_idx;
-	u32 tx_retry_failed;
-	u32 tx_retry_count;
-};
 
-struct ath12k_reoq_buf {
-	void *vaddr;
-	dma_addr_t paddr_aligned;
-	u32 size;
+	/* peer addr based rhashtable list pointer */
+	struct rhash_head rhash_addr;
 };
 
 struct ath12k_sta {
@@ -594,8 +524,6 @@ struct ath12k_sta {
 	u8 num_peer;
 
 	enum ieee80211_sta_state state;
-
-	struct ath12k_reoq_buf reoq_bufs[IEEE80211_NUM_TIDS + 1];
 };
 
 #define ATH12K_HALF_20MHZ_BW	10
@@ -667,23 +595,6 @@ struct ath12k_debug {
 	bool extd_rx_stats;
 };
 
-struct ath12k_per_peer_tx_stats {
-	u32 succ_bytes;
-	u32 retry_bytes;
-	u32 failed_bytes;
-	u32 duration;
-	u16 succ_pkts;
-	u16 retry_pkts;
-	u16 failed_pkts;
-	u16 ru_start;
-	u16 ru_tones;
-	u8 ba_fails;
-	u8 ppdu_type;
-	u32 mu_grpid;
-	u32 mu_pos;
-	bool is_ampdu;
-};
-
 struct ath12k_pdev_rssi_offsets {
 	s32 temp_offset;
 	s8 min_nf_dbm;
@@ -809,9 +720,6 @@ struct ath12k {
 	struct ath12k_wow wow;
 	struct completion target_suspend;
 	bool target_suspend_ack;
-	struct ath12k_per_peer_tx_stats peer_tx_stats;
-	struct list_head ppdu_stats_info;
-	u32 ppdu_stat_list_depth;
 
 	struct ath12k_per_peer_tx_stats cached_stats;
 	u32 last_ppdu_id;
@@ -866,8 +774,7 @@ struct ath12k_hw {
 
 	DECLARE_BITMAP(free_ml_peer_id_map, ATH12K_MAX_MLO_PEERS);
 
-	/* protected by wiphy_lock() */
-	struct list_head ml_peers;
+	struct ath12k_dp_hw dp_hw;
 
 	/* Keep last */
 	struct ath12k radio[] __aligned(sizeof(void *));
@@ -941,32 +848,6 @@ struct ath12k_board_data {
 	size_t len;
 };
 
-struct ath12k_device_dp_tx_err_stats {
-	/* TCL Ring Descriptor unavailable */
-	u32 desc_na[DP_TCL_NUM_RING_MAX];
-	/* Other failures during dp_tx due to mem allocation failure
-	 * idr unavailable etc.
-	 */
-	atomic_t misc_fail;
-};
-
-struct ath12k_device_dp_stats {
-	u32 err_ring_pkts;
-	u32 invalid_rbm;
-	u32 rxdma_error[HAL_REO_ENTR_RING_RXDMA_ECODE_MAX];
-	u32 reo_error[HAL_REO_DEST_RING_ERROR_CODE_MAX];
-	u32 hal_reo_error[DP_REO_DST_RING_MAX];
-	struct ath12k_device_dp_tx_err_stats tx_err;
-	u32 reo_rx[DP_REO_DST_RING_MAX][ATH12K_MAX_DEVICES];
-	u32 rx_wbm_rel_source[HAL_WBM_REL_SRC_MODULE_MAX][ATH12K_MAX_DEVICES];
-	u32 tqm_rel_reason[MAX_TQM_RELEASE_REASON];
-	u32 fw_tx_status[MAX_FW_TX_STATUS];
-	u32 tx_wbm_rel_source[HAL_WBM_REL_SRC_MODULE_MAX];
-	u32 tx_enqueued[DP_TCL_NUM_RING_MAX];
-	u32 tx_completed[DP_TCL_NUM_RING_MAX];
-	u32 reo_excep_msdu_buf_type;
-};
-
 struct ath12k_reg_freq {
 	u32 start_freq;
 	u32 end_freq;
@@ -987,6 +868,11 @@ struct ath12k_hw_link {
  * wiphy, protected with struct ath12k_hw_group::mutex.
  */
 struct ath12k_hw_group {
+	/* Keep dp_hw_grp as the first member to allow efficient
+	 * usage of cache lines for DP fields
+	 */
+	struct ath12k_dp_hw_group dp_hw_grp;
+	struct ath12k_hw_link hw_links[ATH12K_GROUP_MAX_RADIO];
 	struct list_head list;
 	u8 id;
 	u8 num_devices;
@@ -1009,7 +895,6 @@ struct ath12k_hw_group {
 	bool mlo_capable;
 	struct device_node *wsi_node[ATH12K_MAX_DEVICES];
 	struct ath12k_mlo_memory mlo_mem;
-	struct ath12k_hw_link hw_links[ATH12K_GROUP_MAX_RADIO];
 	bool hw_link_id_init_done;
 };
 
@@ -1035,6 +920,12 @@ struct ath12k_mem_profile_based_param {
 	struct ath12k_dp_profile_params dp_params;
 };
 
+enum ath12k_device_family {
+	ATH12K_DEVICE_FAMILY_START,
+	ATH12K_DEVICE_FAMILY_WIFI7 = ATH12K_DEVICE_FAMILY_START,
+	ATH12K_DEVICE_FAMILY_MAX,
+};
+
 /* Master structure to hold the hw data which may be used in core module */
 struct ath12k_base {
 	enum ath12k_hw_rev hw_rev;
@@ -1054,13 +945,14 @@ struct ath12k_base {
 
 	struct ath12k_htc htc;
 
-	struct ath12k_dp dp;
+	struct ath12k_dp *dp;
 
 	void __iomem *mem;
 	unsigned long mem_len;
 
 	void __iomem *mem_ce;
 	u32 ce_remap_base_addr;
+	u32 cmem_offset;
 	bool ce_remap;
 
 	struct {
@@ -1105,7 +997,6 @@ struct ath12k_base {
 	struct ath12k_wmi_hal_reg_capabilities_ext_arg hal_reg_cap[MAX_RADIOS];
 	unsigned long long free_vdev_map;
 	unsigned long long free_vdev_stats_id_map;
-	struct list_head peers;
 	wait_queue_head_t peer_mapping_wq;
 	u8 mac_addr[ETH_ALEN];
 	bool wmi_ready;
@@ -1135,7 +1026,6 @@ struct ath12k_base {
 
 	/* Current DFS Regulatory */
 	enum ath12k_dfs_region dfs_region;
-	struct ath12k_device_dp_stats device_stats;
 #ifdef CONFIG_ATH12K_DEBUGFS
 	struct dentry *debugfs_soc;
 #endif
@@ -1196,8 +1086,6 @@ struct ath12k_base {
 		bool fw_features_valid;
 	} fw;
 
-	const struct hal_rx_ops *hal_rx_ops;
-
 	struct completion restart_completed;
 
 #ifdef CONFIG_ACPI
@@ -1241,6 +1129,14 @@ struct ath12k_base {
 	const struct ath12k_mem_profile_based_param *profile_param;
 	enum ath12k_qmi_mem_mode target_mem_mode;
 
+	/* FIXME: Define this field in a ag equivalent object available
+	 * during the initial phase of probe later.
+	 */
+	const struct ieee80211_ops *ath12k_ops;
+
+	struct rhashtable *rhead_sta_addr;
+	struct rhashtable_params rhash_sta_addr_param;
+
 	/* must be last */
 	u8 drv_priv[] __aligned(sizeof(void *));
 };
@@ -1516,4 +1412,18 @@ static inline s32 ath12k_pdev_get_noise_floor(struct ath12k *ar)
 	return ar->rssi_info.noise_floor;
 }
 
+/* The @ab->dp NULL check or assertion is intentionally omitted because
+ * @ab->dp is guaranteed to be non-NULL after a successful probe and
+ * remains valid until teardown. Invoking this before allocation or
+ * after teardown is considered invalid usage.
+ */
+static inline struct ath12k_dp *ath12k_ab_to_dp(struct ath12k_base *ab)
+{
+	return ab->dp;
+}
+
+static inline struct ath12k *ath12k_pdev_dp_to_ar(struct ath12k_pdev_dp *dp)
+{
+	return container_of(dp, struct ath12k, dp);
+}
 #endif /* _CORE_H_ */
diff --git a/drivers/net/wireless/ath/ath12k/dbring.c b/drivers/net/wireless/ath/ath12k/dbring.c
index 6604dacea2ae..f71ec2a58469 100644
--- a/drivers/net/wireless/ath/ath12k/dbring.c
+++ b/drivers/net/wireless/ath/ath12k/dbring.c
@@ -1,12 +1,12 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #include "core.h"
 #include "debug.h"
+#include "hal.h"
 
 static int ath12k_dbring_bufs_replenish(struct ath12k *ar,
 					struct ath12k_dbring *ring,
@@ -55,7 +55,7 @@ static int ath12k_dbring_bufs_replenish(struct ath12k *ar,
 	cookie = u32_encode_bits(ar->pdev_idx, DP_RXDMA_BUF_COOKIE_PDEV_ID) |
 		 u32_encode_bits(buf_id, DP_RXDMA_BUF_COOKIE_BUF_ID);
 
-	ath12k_hal_rx_buf_addr_info_set(desc, paddr, cookie, 0);
+	ath12k_hal_rx_buf_addr_info_set(&ab->hal, desc, paddr, cookie, 0);
 
 	ath12k_hal_srng_access_end(ab, srng);
 
@@ -298,7 +298,7 @@ int ath12k_dbring_buffer_release_event(struct ath12k_base *ab,
 
 		num_buff_reaped++;
 
-		ath12k_hal_rx_buf_addr_info_get(&desc, &paddr, &cookie, &rbm);
+		ath12k_hal_rx_buf_addr_info_get(&ab->hal, &desc, &paddr, &cookie, &rbm);
 
 		buf_id = u32_get_bits(cookie, DP_RXDMA_BUF_COOKIE_BUF_ID);
 
diff --git a/drivers/net/wireless/ath/ath12k/debug.c b/drivers/net/wireless/ath/ath12k/debug.c
index 5ce100cd9a9d..34b3b2c920dc 100644
--- a/drivers/net/wireless/ath/ath12k/debug.c
+++ b/drivers/net/wireless/ath/ath12k/debug.c
@@ -1,7 +1,8 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ *
  */
 
 #include <linux/vmalloc.h>
@@ -21,6 +22,7 @@ void ath12k_info(struct ath12k_base *ab, const char *fmt, ...)
 	/* TODO: Trace the log */
 	va_end(args);
 }
+EXPORT_SYMBOL(ath12k_info);
 
 void ath12k_err(struct ath12k_base *ab, const char *fmt, ...)
 {
@@ -35,6 +37,7 @@ void ath12k_err(struct ath12k_base *ab, const char *fmt, ...)
 	/* TODO: Trace the log */
 	va_end(args);
 }
+EXPORT_SYMBOL(ath12k_err);
 
 void __ath12k_warn(struct device *dev, const char *fmt, ...)
 {
@@ -49,6 +52,7 @@ void __ath12k_warn(struct device *dev, const char *fmt, ...)
 	/* TODO: Trace the log */
 	va_end(args);
 }
+EXPORT_SYMBOL(__ath12k_warn);
 
 #ifdef CONFIG_ATH12K_DEBUG
 
@@ -72,6 +76,7 @@ void __ath12k_dbg(struct ath12k_base *ab, enum ath12k_debug_mask mask,
 
 	va_end(args);
 }
+EXPORT_SYMBOL(__ath12k_dbg);
 
 void ath12k_dbg_dump(struct ath12k_base *ab,
 		     enum ath12k_debug_mask mask,
@@ -100,5 +105,6 @@ void ath12k_dbg_dump(struct ath12k_base *ab,
 		}
 	}
 }
+EXPORT_SYMBOL(ath12k_dbg_dump);
 
 #endif /* CONFIG_ATH12K_DEBUG */
diff --git a/drivers/net/wireless/ath/ath12k/debugfs.c b/drivers/net/wireless/ath/ath12k/debugfs.c
index d6a86f075d73..358031fa14eb 100644
--- a/drivers/net/wireless/ath/ath12k/debugfs.c
+++ b/drivers/net/wireless/ath/ath12k/debugfs.c
@@ -967,7 +967,7 @@ static int ath12k_open_link_stats(struct inode *inode, struct file *file)
 				 "\nlink[%d] Tx Frame descriptor Encrypt Type = ",
 				 link_id);
 
-		for (i = 0; i < HAL_ENCRYPT_TYPE_MAX; i++) {
+		for (i = 0; i < DP_ENCRYPT_TYPE_MAX; i++) {
 			len += scnprintf(buf + len, buf_len - len,
 					 " %d:%d", i,
 					 linkstat.tx_encrypt_type[i]);
@@ -1020,13 +1020,15 @@ void ath12k_debugfs_op_vif_add(struct ieee80211_hw *hw,
 	debugfs_create_file("link_stats", 0400, vif->debugfs_dir, ahvif,
 			    &ath12k_fops_link_stats);
 }
+EXPORT_SYMBOL(ath12k_debugfs_op_vif_add);
 
 static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file,
 						   char __user *user_buf,
 						   size_t count, loff_t *ppos)
 {
 	struct ath12k_base *ab = file->private_data;
-	struct ath12k_device_dp_stats *device_stats = &ab->device_stats;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct ath12k_device_dp_stats *device_stats = &dp->device_stats;
 	int len = 0, i, j, ret;
 	struct ath12k *ar;
 	const int size = 4096;
@@ -1155,6 +1157,7 @@ static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file,
 
 	len += scnprintf(buf + len, size - len, "\n");
 
+	rcu_read_lock();
 	for (i = 0; i < ab->num_radios; i++) {
 		ar = ath12k_mac_get_ar_by_pdev_id(ab, DP_SW2HW_MACID(i));
 		if (ar) {
@@ -1163,6 +1166,7 @@ static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file,
 					atomic_read(&ar->dp.num_tx_pending));
 		}
 	}
+	rcu_read_unlock();
 
 	len += scnprintf(buf + len, size - len, "\nREO Rx Received:\n");
 
diff --git a/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.h b/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.h
index 9bd3a632b002..8008658371aa 100644
--- a/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.h
+++ b/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.h
@@ -1,12 +1,14 @@
 /* SPDX-License-Identifier: BSD-3-Clause-Clear */
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #ifndef DEBUG_HTT_STATS_H
 #define DEBUG_HTT_STATS_H
 
+#include "dp_htt.h"
+
 #define ATH12K_HTT_STATS_BUF_SIZE		(1024 * 512)
 #define ATH12K_HTT_STATS_COOKIE_LSB		GENMASK_ULL(31, 0)
 #define ATH12K_HTT_STATS_COOKIE_MSB		GENMASK_ULL(63, 32)
diff --git a/drivers/net/wireless/ath/ath12k/debugfs_sta.c b/drivers/net/wireless/ath/ath12k/debugfs_sta.c
index 5bd2bf4c9dac..585c40bd2951 100644
--- a/drivers/net/wireless/ath/ath12k/debugfs_sta.c
+++ b/drivers/net/wireless/ath/ath12k/debugfs_sta.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
- * Copyright (c) 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #include <linux/vmalloc.h>
@@ -11,6 +11,7 @@
 #include "debug.h"
 #include "debugfs_htt_stats.h"
 #include "debugfs.h"
+#include "dp_cmn.h"
 
 static
 u32 ath12k_dbg_sta_dump_rate_stats(u8 *buf, u32 offset, const int size,
@@ -144,40 +145,40 @@ static ssize_t ath12k_dbg_sta_dump_rx_stats(struct file *file,
 	const int size = ATH12K_STA_RX_STATS_BUF_SIZE;
 	struct ath12k_hw *ah = ahsta->ahvif->ah;
 	struct ath12k_rx_peer_stats *rx_stats;
+	struct ath12k_dp_link_peer *link_peer;
 	struct ath12k_link_sta *arsta;
 	u8 link_id = link_sta->link_id;
 	int len = 0, i, ret = 0;
+	struct ath12k_dp *dp;
 	bool he_rates_avail;
 	struct ath12k *ar;
 
-	wiphy_lock(ah->hw->wiphy);
+	guard(wiphy)(ah->hw->wiphy);
 
-	if (!(BIT(link_id) & ahsta->links_map)) {
-		wiphy_unlock(ah->hw->wiphy);
+	if (!(BIT(link_id) & ahsta->links_map))
 		return -ENOENT;
-	}
 
 	arsta = wiphy_dereference(ah->hw->wiphy, ahsta->link[link_id]);
-	if (!arsta || !arsta->arvif->ar) {
-		wiphy_unlock(ah->hw->wiphy);
+	if (!arsta || !arsta->arvif->ar)
 		return -ENOENT;
-	}
 
 	ar = arsta->arvif->ar;
 
 	u8 *buf __free(kfree) = kzalloc(size, GFP_KERNEL);
-	if (!buf) {
-		ret = -ENOENT;
-		goto out;
-	}
+	if (!buf)
+		return -ENOMEM;
 
-	spin_lock_bh(&ar->ab->base_lock);
+	dp = ath12k_ab_to_dp(ar->ab);
 
-	rx_stats = arsta->rx_stats;
-	if (!rx_stats) {
-		ret = -ENOENT;
-		goto unlock;
-	}
+	guard(spinlock_bh)(&dp->dp_lock);
+
+	link_peer = ath12k_dp_link_peer_find_by_addr(dp, arsta->addr);
+	if (!link_peer)
+		return -ENOENT;
+
+	rx_stats = link_peer->peer_stats.rx_stats;
+	if (!rx_stats)
+		return -ENOENT;
 
 	len += scnprintf(buf + len, size - len, "RX peer stats:\n\n");
 	len += scnprintf(buf + len, size - len, "Num of MSDUs: %llu\n",
@@ -237,13 +238,8 @@ static ssize_t ath12k_dbg_sta_dump_rx_stats(struct file *file,
 	len += ath12k_dbg_sta_dump_rate_stats(buf, len, size, he_rates_avail,
 					      &rx_stats->byte_stats);
 
-unlock:
-	spin_unlock_bh(&ar->ab->base_lock);
-
 	if (len)
 		ret = simple_read_from_buffer(user_buf, count, ppos, buf, len);
-out:
-	wiphy_unlock(ah->hw->wiphy);
 	return ret;
 }
 
@@ -261,10 +257,9 @@ static ssize_t ath12k_dbg_sta_reset_rx_stats(struct file *file,
 	struct ieee80211_link_sta *link_sta = file->private_data;
 	struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(link_sta->sta);
 	struct ath12k_hw *ah = ahsta->ahvif->ah;
-	struct ath12k_rx_peer_stats *rx_stats;
-	struct ath12k_link_sta *arsta;
 	u8 link_id = link_sta->link_id;
-	struct ath12k *ar;
+	struct ath12k_link_sta *arsta;
+	struct ath12k_dp *dp;
 	bool reset;
 	int ret;
 
@@ -288,19 +283,9 @@ static ssize_t ath12k_dbg_sta_reset_rx_stats(struct file *file,
 		goto out;
 	}
 
-	ar = arsta->arvif->ar;
-
-	spin_lock_bh(&ar->ab->base_lock);
-
-	rx_stats = arsta->rx_stats;
-	if (!rx_stats) {
-		spin_unlock_bh(&ar->ab->base_lock);
-		ret = -ENOENT;
-		goto out;
-	}
+	dp = ath12k_ab_to_dp(arsta->arvif->ar->ab);
 
-	memset(rx_stats, 0, sizeof(*rx_stats));
-	spin_unlock_bh(&ar->ab->base_lock);
+	ath12k_dp_link_peer_reset_rx_stats(dp, arsta->addr);
 
 	ret = count;
 out:
@@ -335,3 +320,4 @@ void ath12k_debugfs_link_sta_op_add(struct ieee80211_hw *hw,
 				    &fops_reset_rx_stats);
 	}
 }
+EXPORT_SYMBOL(ath12k_debugfs_link_sta_op_add);
diff --git a/drivers/net/wireless/ath/ath12k/dp.c b/drivers/net/wireless/ath/ath12k/dp.c
index 4a54b8c35311..9f05eea6695a 100644
--- a/drivers/net/wireless/ath/ath12k/dp.c
+++ b/drivers/net/wireless/ath/ath12k/dp.c
@@ -1,63 +1,58 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #include <crypto/hash.h>
 #include "core.h"
 #include "dp_tx.h"
-#include "hal_tx.h"
 #include "hif.h"
+#include "hal.h"
 #include "debug.h"
-#include "dp_rx.h"
 #include "peer.h"
-#include "dp_mon.h"
+#include "dp_cmn.h"
 
 enum ath12k_dp_desc_type {
 	ATH12K_DP_TX_DESC,
 	ATH12K_DP_RX_DESC,
 };
 
-static void ath12k_dp_htt_htc_tx_complete(struct ath12k_base *ab,
-					  struct sk_buff *skb)
-{
-	dev_kfree_skb_any(skb);
-}
-
 void ath12k_dp_peer_cleanup(struct ath12k *ar, int vdev_id, const u8 *addr)
 {
 	struct ath12k_base *ab = ar->ab;
-	struct ath12k_peer *peer;
+	struct ath12k_dp_link_peer *peer;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 
 	/* TODO: Any other peer specific DP cleanup */
 
-	spin_lock_bh(&ab->base_lock);
-	peer = ath12k_peer_find(ab, vdev_id, addr);
-	if (!peer) {
+	spin_lock_bh(&dp->dp_lock);
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, addr);
+	if (!peer || !peer->dp_peer) {
 		ath12k_warn(ab, "failed to lookup peer %pM on vdev %d\n",
 			    addr, vdev_id);
-		spin_unlock_bh(&ab->base_lock);
+		spin_unlock_bh(&dp->dp_lock);
 		return;
 	}
 
 	if (!peer->primary_link) {
-		spin_unlock_bh(&ab->base_lock);
+		spin_unlock_bh(&dp->dp_lock);
 		return;
 	}
 
 	ath12k_dp_rx_peer_tid_cleanup(ar, peer);
-	crypto_free_shash(peer->tfm_mmic);
-	peer->dp_setup_done = false;
-	spin_unlock_bh(&ab->base_lock);
+	crypto_free_shash(peer->dp_peer->tfm_mmic);
+	peer->dp_peer->dp_setup_done = false;
+	spin_unlock_bh(&dp->dp_lock);
 }
 
 int ath12k_dp_peer_setup(struct ath12k *ar, int vdev_id, const u8 *addr)
 {
 	struct ath12k_base *ab = ar->ab;
-	struct ath12k_peer *peer;
+	struct ath12k_dp_link_peer *peer;
 	u32 reo_dest;
 	int ret = 0, tid;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 
 	/* NOTE: reo_dest ring id starts from 1 unlike mac_id which starts from 0 */
 	reo_dest = ar->dp.mac_id + 1;
@@ -92,19 +87,19 @@ int ath12k_dp_peer_setup(struct ath12k *ar, int vdev_id, const u8 *addr)
 	return 0;
 
 peer_clean:
-	spin_lock_bh(&ab->base_lock);
+	spin_lock_bh(&dp->dp_lock);
 
-	peer = ath12k_peer_find(ab, vdev_id, addr);
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, addr);
 	if (!peer) {
 		ath12k_warn(ab, "failed to find the peer to del rx tid\n");
-		spin_unlock_bh(&ab->base_lock);
+		spin_unlock_bh(&dp->dp_lock);
 		return -ENOENT;
 	}
 
 	for (tid--; tid >= 0; tid--)
-		ath12k_dp_rx_peer_tid_delete(ar, peer, tid);
+		ath12k_dp_arch_rx_peer_tid_delete(dp, peer, tid);
 
-	spin_unlock_bh(&ab->base_lock);
+	spin_unlock_bh(&dp->dp_lock);
 
 	return ret;
 }
@@ -147,7 +142,7 @@ static int ath12k_dp_srng_calculate_msi_group(struct ath12k_base *ab,
 			grp_mask = &ab->hw_params->ring_mask->rx_wbm_rel[0];
 			ring_num = 0;
 		} else {
-			map = ab->hw_params->hal_ops->tcl_to_wbm_rbm_map;
+			map = ab->hal.tcl_to_wbm_rbm_map;
 			for (i = 0; i < ab->hw_params->max_tx_ring; i++) {
 				if (ring_num == map[i].wbm_ring_num) {
 					ring_num = i;
@@ -338,50 +333,6 @@ int ath12k_dp_srng_setup(struct ath12k_base *ab, struct dp_srng *ring,
 	return 0;
 }
 
-static
-u32 ath12k_dp_tx_get_vdev_bank_config(struct ath12k_base *ab,
-				      struct ath12k_link_vif *arvif)
-{
-	u32 bank_config = 0;
-	struct ath12k_vif *ahvif = arvif->ahvif;
-
-	/* Only valid for raw frames with HW crypto enabled.
-	 * With SW crypto, mac80211 sets key per packet
-	 */
-	if (ahvif->tx_encap_type == HAL_TCL_ENCAP_TYPE_RAW &&
-	    test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ab->dev_flags))
-		bank_config |=
-			u32_encode_bits(ath12k_dp_tx_get_encrypt_type(ahvif->key_cipher),
-					HAL_TX_BANK_CONFIG_ENCRYPT_TYPE);
-
-	bank_config |= u32_encode_bits(ahvif->tx_encap_type,
-					HAL_TX_BANK_CONFIG_ENCAP_TYPE);
-	bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_SRC_BUFFER_SWAP) |
-			u32_encode_bits(0, HAL_TX_BANK_CONFIG_LINK_META_SWAP) |
-			u32_encode_bits(0, HAL_TX_BANK_CONFIG_EPD);
-
-	/* only valid if idx_lookup_override is not set in tcl_data_cmd */
-	if (ahvif->vdev_type == WMI_VDEV_TYPE_STA)
-		bank_config |= u32_encode_bits(1, HAL_TX_BANK_CONFIG_INDEX_LOOKUP_EN);
-	else
-		bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_INDEX_LOOKUP_EN);
-
-	bank_config |= u32_encode_bits(arvif->hal_addr_search_flags & HAL_TX_ADDRX_EN,
-					HAL_TX_BANK_CONFIG_ADDRX_EN) |
-			u32_encode_bits(!!(arvif->hal_addr_search_flags &
-					HAL_TX_ADDRY_EN),
-					HAL_TX_BANK_CONFIG_ADDRY_EN);
-
-	bank_config |= u32_encode_bits(ieee80211_vif_is_mesh(ahvif->vif) ? 3 : 0,
-					HAL_TX_BANK_CONFIG_MESH_EN) |
-			u32_encode_bits(arvif->vdev_id_check_en,
-					HAL_TX_BANK_CONFIG_VDEV_ID_CHECK_EN);
-
-	bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_DSCP_TIP_MAP_ID);
-
-	return bank_config;
-}
-
 static int ath12k_dp_tx_get_bank_profile(struct ath12k_base *ab,
 					 struct ath12k_link_vif *arvif,
 					 struct ath12k_dp *dp)
@@ -392,7 +343,7 @@ static int ath12k_dp_tx_get_bank_profile(struct ath12k_base *ab,
 	bool configure_register = false;
 
 	/* convert vdev params into hal_tx_bank_config */
-	bank_config = ath12k_dp_tx_get_vdev_bank_config(ab, arvif);
+	bank_config = ath12k_dp_arch_tx_get_vdev_bank_config(dp, arvif);
 
 	spin_lock_bh(&dp->tx_bank_lock);
 	/* TODO: implement using idr kernel framework*/
@@ -424,7 +375,8 @@ static int ath12k_dp_tx_get_bank_profile(struct ath12k_base *ab,
 	spin_unlock_bh(&dp->tx_bank_lock);
 
 	if (configure_register)
-		ath12k_hal_tx_configure_bank_register(ab, bank_config, bank_id);
+		ath12k_hal_tx_configure_bank_register(ab,
+						      bank_config, bank_id);
 
 	ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt tcl bank_id %d input 0x%x match 0x%x num_users %u",
 		   bank_id, bank_config, dp->bank_profiles[bank_id].bank_config,
@@ -442,7 +394,7 @@ void ath12k_dp_tx_put_bank_profile(struct ath12k_dp *dp, u8 bank_id)
 
 static void ath12k_dp_deinit_bank_profiles(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 
 	kfree(dp->bank_profiles);
 	dp->bank_profiles = NULL;
@@ -450,7 +402,7 @@ static void ath12k_dp_deinit_bank_profiles(struct ath12k_base *ab)
 
 static int ath12k_dp_init_bank_profiles(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	u32 num_tcl_banks = ab->hw_params->num_tcl_banks;
 	int i;
 
@@ -473,7 +425,7 @@ static int ath12k_dp_init_bank_profiles(struct ath12k_base *ab)
 
 static void ath12k_dp_srng_common_cleanup(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	int i;
 
 	ath12k_dp_srng_cleanup(ab, &dp->reo_status_ring);
@@ -490,7 +442,7 @@ static void ath12k_dp_srng_common_cleanup(struct ath12k_base *ab)
 
 static int ath12k_dp_srng_common_setup(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	const struct ath12k_hal_tcl_to_wbm_rbm_map *map;
 	struct hal_srng *srng;
 	int i, ret, tx_comp_ring_num;
@@ -506,7 +458,7 @@ static int ath12k_dp_srng_common_setup(struct ath12k_base *ab)
 	}
 
 	for (i = 0; i < ab->hw_params->max_tx_ring; i++) {
-		map = ab->hw_params->hal_ops->tcl_to_wbm_rbm_map;
+		map = ab->hal.tcl_to_wbm_rbm_map;
 		tx_comp_ring_num = map[i].wbm_ring_num;
 
 		ret = ath12k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_data_ring,
@@ -596,7 +548,7 @@ static int ath12k_dp_srng_common_setup(struct ath12k_base *ab)
 
 static void ath12k_dp_scatter_idle_link_desc_cleanup(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	struct hal_wbm_idle_scatter_list *slist = dp->scatter_list;
 	int i;
 
@@ -616,7 +568,7 @@ static int ath12k_dp_scatter_idle_link_desc_setup(struct ath12k_base *ab,
 						  u32 n_link_desc,
 						  u32 last_bank_sz)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	struct dp_link_desc_bank *link_desc_banks = dp->link_desc_banks;
 	struct hal_wbm_idle_scatter_list *slist = dp->scatter_list;
 	u32 n_entries_per_buf;
@@ -659,7 +611,7 @@ static int ath12k_dp_scatter_idle_link_desc_setup(struct ath12k_base *ab,
 		paddr = link_desc_banks[i].paddr;
 		while (n_entries) {
 			cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i);
-			ath12k_hal_set_link_desc_addr(scatter_buf, cookie,
+			ath12k_hal_set_link_desc_addr(dp->hal, scatter_buf, cookie,
 						      paddr, rbm);
 			n_entries--;
 			paddr += HAL_LINK_DESC_SIZE;
@@ -710,7 +662,7 @@ static int ath12k_dp_link_desc_bank_alloc(struct ath12k_base *ab,
 					  int n_link_desc_bank,
 					  int last_bank_sz)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	int i;
 	int ret = 0;
 	int desc_sz = DP_LINK_DESC_ALLOC_SIZE_THRESH;
@@ -758,7 +710,7 @@ void ath12k_dp_link_desc_cleanup(struct ath12k_base *ab,
 
 static int ath12k_wbm_idle_ring_setup(struct ath12k_base *ab, u32 *n_link_desc)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	u32 n_mpdu_link_desc, n_mpdu_queue_desc;
 	u32 n_tx_msdu_link_desc, n_rx_msdu_link_desc;
 	int ret = 0;
@@ -797,6 +749,7 @@ int ath12k_dp_link_desc_setup(struct ath12k_base *ab,
 			      u32 ring_type, struct hal_srng *srng,
 			      u32 n_link_desc)
 {
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	u32 tot_mem_sz;
 	u32 n_link_desc_bank, last_bank_sz;
 	u32 entry_sz, align_bytes, n_entries;
@@ -804,7 +757,7 @@ int ath12k_dp_link_desc_setup(struct ath12k_base *ab,
 	u32 paddr;
 	int i, ret;
 	u32 cookie;
-	enum hal_rx_buf_return_buf_manager rbm = ab->dp.idle_link_rbm;
+	enum hal_rx_buf_return_buf_manager rbm = dp->idle_link_rbm;
 
 	tot_mem_sz = n_link_desc * HAL_LINK_DESC_SIZE;
 	tot_mem_sz += HAL_LINK_DESC_ALIGN;
@@ -865,7 +818,8 @@ int ath12k_dp_link_desc_setup(struct ath12k_base *ab,
 		while (n_entries &&
 		       (desc = ath12k_hal_srng_src_get_next_entry(ab, srng))) {
 			cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i);
-			ath12k_hal_set_link_desc_addr(desc, cookie, paddr, rbm);
+			ath12k_hal_set_link_desc_addr(dp->hal, desc, cookie, paddr,
+						      rbm);
 			n_entries--;
 			paddr += HAL_LINK_DESC_SIZE;
 		}
@@ -883,133 +837,18 @@ int ath12k_dp_link_desc_setup(struct ath12k_base *ab,
 	return ret;
 }
 
-int ath12k_dp_service_srng(struct ath12k_base *ab,
-			   struct ath12k_ext_irq_grp *irq_grp,
-			   int budget)
+void ath12k_dp_pdev_free(struct ath12k_base *ab)
 {
-	struct napi_struct *napi = &irq_grp->napi;
-	int grp_id = irq_grp->grp_id;
-	int work_done = 0;
-	int i = 0, j;
-	int tot_work_done = 0;
-	enum dp_monitor_mode monitor_mode;
-	u8 ring_mask;
-
-	if (ab->hw_params->ring_mask->tx[grp_id]) {
-		i = fls(ab->hw_params->ring_mask->tx[grp_id]) - 1;
-		ath12k_dp_tx_completion_handler(ab, i);
-	}
-
-	if (ab->hw_params->ring_mask->rx_err[grp_id]) {
-		work_done = ath12k_dp_rx_process_err(ab, napi, budget);
-		budget -= work_done;
-		tot_work_done += work_done;
-		if (budget <= 0)
-			goto done;
-	}
-
-	if (ab->hw_params->ring_mask->rx_wbm_rel[grp_id]) {
-		work_done = ath12k_dp_rx_process_wbm_err(ab,
-							 napi,
-							 budget);
-		budget -= work_done;
-		tot_work_done += work_done;
-
-		if (budget <= 0)
-			goto done;
-	}
-
-	if (ab->hw_params->ring_mask->rx[grp_id]) {
-		i = fls(ab->hw_params->ring_mask->rx[grp_id]) - 1;
-		work_done = ath12k_dp_rx_process(ab, i, napi,
-						 budget);
-		budget -= work_done;
-		tot_work_done += work_done;
-		if (budget <= 0)
-			goto done;
-	}
-
-	if (ab->hw_params->ring_mask->rx_mon_status[grp_id]) {
-		ring_mask = ab->hw_params->ring_mask->rx_mon_status[grp_id];
-		for (i = 0; i < ab->num_radios; i++) {
-			for (j = 0; j < ab->hw_params->num_rxdma_per_pdev; j++) {
-				int id = i * ab->hw_params->num_rxdma_per_pdev + j;
-
-				if (ring_mask & BIT(id)) {
-					work_done =
-					ath12k_dp_mon_process_ring(ab, id, napi, budget,
-								   0);
-					budget -= work_done;
-					tot_work_done += work_done;
-					if (budget <= 0)
-						goto done;
-				}
-			}
-		}
-	}
-
-	if (ab->hw_params->ring_mask->rx_mon_dest[grp_id]) {
-		monitor_mode = ATH12K_DP_RX_MONITOR_MODE;
-		ring_mask = ab->hw_params->ring_mask->rx_mon_dest[grp_id];
-		for (i = 0; i < ab->num_radios; i++) {
-			for (j = 0; j < ab->hw_params->num_rxdma_per_pdev; j++) {
-				int id = i * ab->hw_params->num_rxdma_per_pdev + j;
-
-				if (ring_mask & BIT(id)) {
-					work_done =
-					ath12k_dp_mon_process_ring(ab, id, napi, budget,
-								   monitor_mode);
-					budget -= work_done;
-					tot_work_done += work_done;
-
-					if (budget <= 0)
-						goto done;
-				}
-			}
-		}
-	}
-
-	if (ab->hw_params->ring_mask->tx_mon_dest[grp_id]) {
-		monitor_mode = ATH12K_DP_TX_MONITOR_MODE;
-		ring_mask = ab->hw_params->ring_mask->tx_mon_dest[grp_id];
-		for (i = 0; i < ab->num_radios; i++) {
-			for (j = 0; j < ab->hw_params->num_rxdma_per_pdev; j++) {
-				int id = i * ab->hw_params->num_rxdma_per_pdev + j;
-
-				if (ring_mask & BIT(id)) {
-					work_done =
-					ath12k_dp_mon_process_ring(ab, id, napi, budget,
-								   monitor_mode);
-					budget -= work_done;
-					tot_work_done += work_done;
-
-					if (budget <= 0)
-						goto done;
-				}
-			}
-		}
-	}
-
-	if (ab->hw_params->ring_mask->reo_status[grp_id])
-		ath12k_dp_rx_process_reo_status(ab);
-
-	if (ab->hw_params->ring_mask->host2rxdma[grp_id]) {
-		struct ath12k_dp *dp = &ab->dp;
-		struct dp_rxdma_ring *rx_ring = &dp->rx_refill_buf_ring;
-		LIST_HEAD(list);
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct ath12k *ar;
+	int i;
 
-		ath12k_dp_rx_bufs_replenish(ab, rx_ring, &list, 0);
+	for (i = 0; i < ab->num_radios; i++) {
+		ar = ab->pdevs[i].ar;
+		rcu_assign_pointer(dp->dp_pdevs[ar->pdev_idx], NULL);
 	}
 
-	/* TODO: Implement handler for other interrupts */
-
-done:
-	return tot_work_done;
-}
-
-void ath12k_dp_pdev_free(struct ath12k_base *ab)
-{
-	int i;
+	synchronize_rcu();
 
 	for (i = 0; i < ab->num_radios; i++)
 		ath12k_dp_rx_pdev_free(ab, i);
@@ -1025,31 +864,10 @@ void ath12k_dp_pdev_pre_alloc(struct ath12k *ar)
 	/* TODO: Add any RXDMA setup required per pdev */
 }
 
-bool ath12k_dp_wmask_compaction_rx_tlv_supported(struct ath12k_base *ab)
-{
-	if (test_bit(WMI_TLV_SERVICE_WMSK_COMPACTION_RX_TLVS, ab->wmi_ab.svc_map) &&
-	    ab->hw_params->hal_ops->rxdma_ring_wmask_rx_mpdu_start &&
-	    ab->hw_params->hal_ops->rxdma_ring_wmask_rx_msdu_end &&
-	    ab->hw_params->hal_ops->get_hal_rx_compact_ops) {
-		return true;
-	}
-	return false;
-}
-
-void ath12k_dp_hal_rx_desc_init(struct ath12k_base *ab)
-{
-	if (ath12k_dp_wmask_compaction_rx_tlv_supported(ab)) {
-		/* RX TLVS compaction is supported, hence change the hal_rx_ops
-		 * to compact hal_rx_ops.
-		 */
-		ab->hal_rx_ops = ab->hw_params->hal_ops->get_hal_rx_compact_ops();
-	}
-	ab->hal.hal_desc_sz =
-		ab->hal_rx_ops->rx_desc_get_desc_size();
-}
-
 int ath12k_dp_pdev_alloc(struct ath12k_base *ab)
 {
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct ath12k_pdev_dp *dp_pdev;
 	struct ath12k *ar;
 	int ret;
 	int i;
@@ -1061,6 +879,14 @@ int ath12k_dp_pdev_alloc(struct ath12k_base *ab)
 	/* TODO: Per-pdev rx ring unlike tx ring which is mapped to different AC's */
 	for (i = 0; i < ab->num_radios; i++) {
 		ar = ab->pdevs[i].ar;
+
+		dp_pdev = &ar->dp;
+
+		dp_pdev->hw = ar->ah->hw;
+		dp_pdev->dp = dp;
+		dp_pdev->hw_link_id = ar->hw_link_id;
+		dp_pdev->dp_hw = &ar->ah->dp_hw;
+
 		ret = ath12k_dp_rx_pdev_alloc(ab, i);
 		if (ret) {
 			ath12k_warn(ab, "failed to allocate pdev rx for pdev_id :%d\n",
@@ -1074,6 +900,11 @@ int ath12k_dp_pdev_alloc(struct ath12k_base *ab)
 		}
 	}
 
+	for (i = 0; i < ab->num_radios; i++) {
+		ar = ab->pdevs[i].ar;
+		rcu_assign_pointer(dp->dp_pdevs[ar->pdev_idx], &ar->dp);
+	}
+
 	return 0;
 err:
 	ath12k_dp_pdev_free(ab);
@@ -1081,40 +912,23 @@ int ath12k_dp_pdev_alloc(struct ath12k_base *ab)
 	return ret;
 }
 
-int ath12k_dp_htt_connect(struct ath12k_dp *dp)
+static void ath12k_dp_update_vdev_search(struct ath12k_link_vif *arvif)
 {
-	struct ath12k_htc_svc_conn_req conn_req = {};
-	struct ath12k_htc_svc_conn_resp conn_resp = {};
-	int status;
-
-	conn_req.ep_ops.ep_tx_complete = ath12k_dp_htt_htc_tx_complete;
-	conn_req.ep_ops.ep_rx_complete = ath12k_dp_htt_htc_t2h_msg_handler;
-
-	/* connect to control service */
-	conn_req.service_id = ATH12K_HTC_SVC_ID_HTT_DATA_MSG;
-
-	status = ath12k_htc_connect_service(&dp->ab->htc, &conn_req,
-					    &conn_resp);
-
-	if (status)
-		return status;
+	u8 link_id = arvif->link_id;
+	struct ath12k_vif *ahvif = arvif->ahvif;
+	struct ath12k_dp_link_vif *dp_link_vif;
 
-	dp->eid = conn_resp.eid;
+	dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, link_id);
 
-	return 0;
-}
-
-static void ath12k_dp_update_vdev_search(struct ath12k_link_vif *arvif)
-{
 	switch (arvif->ahvif->vdev_type) {
 	case WMI_VDEV_TYPE_STA:
-		arvif->hal_addr_search_flags = HAL_TX_ADDRY_EN;
-		arvif->search_type = HAL_TX_ADDR_SEARCH_INDEX;
+		dp_link_vif->hal_addr_search_flags = HAL_TX_ADDRY_EN;
+		dp_link_vif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT;
 		break;
 	case WMI_VDEV_TYPE_AP:
 	case WMI_VDEV_TYPE_IBSS:
-		arvif->hal_addr_search_flags = HAL_TX_ADDRX_EN;
-		arvif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT;
+		dp_link_vif->hal_addr_search_flags = HAL_TX_ADDRX_EN;
+		dp_link_vif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT;
 		break;
 	case WMI_VDEV_TYPE_MONITOR:
 	default:
@@ -1125,22 +939,29 @@ static void ath12k_dp_update_vdev_search(struct ath12k_link_vif *arvif)
 void ath12k_dp_vdev_tx_attach(struct ath12k *ar, struct ath12k_link_vif *arvif)
 {
 	struct ath12k_base *ab = ar->ab;
+	struct ath12k_vif *ahvif = arvif->ahvif;
+	u8 link_id = arvif->link_id;
+	int bank_id;
+	struct ath12k_dp_link_vif *dp_link_vif;
+
+	dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, link_id);
 
-	arvif->tcl_metadata |= u32_encode_bits(1, HTT_TCL_META_DATA_TYPE) |
-			       u32_encode_bits(arvif->vdev_id,
-					       HTT_TCL_META_DATA_VDEV_ID) |
-			       u32_encode_bits(ar->pdev->pdev_id,
-					       HTT_TCL_META_DATA_PDEV_ID);
+	dp_link_vif->tcl_metadata |= u32_encode_bits(1, HTT_TCL_META_DATA_TYPE) |
+				     u32_encode_bits(arvif->vdev_id,
+						     HTT_TCL_META_DATA_VDEV_ID) |
+				     u32_encode_bits(ar->pdev->pdev_id,
+						     HTT_TCL_META_DATA_PDEV_ID);
 
 	/* set HTT extension valid bit to 0 by default */
-	arvif->tcl_metadata &= ~HTT_TCL_META_DATA_VALID_HTT;
+	dp_link_vif->tcl_metadata &= ~HTT_TCL_META_DATA_VALID_HTT;
 
 	ath12k_dp_update_vdev_search(arvif);
-	arvif->vdev_id_check_en = true;
-	arvif->bank_id = ath12k_dp_tx_get_bank_profile(ab, arvif, &ab->dp);
+	dp_link_vif->vdev_id_check_en = true;
+	bank_id = ath12k_dp_tx_get_bank_profile(ab, arvif, ath12k_ab_to_dp(ab));
+	dp_link_vif->bank_id = bank_id;
 
 	/* TODO: error path for bank id failure */
-	if (arvif->bank_id == DP_INVALID_BANK_ID) {
+	if (bank_id == DP_INVALID_BANK_ID) {
 		ath12k_err(ar->ab, "Failed to initialize DP TX Banks");
 		return;
 	}
@@ -1150,7 +971,7 @@ static void ath12k_dp_cc_cleanup(struct ath12k_base *ab)
 {
 	struct ath12k_rx_desc_info *desc_info;
 	struct ath12k_tx_desc_info *tx_desc_info, *tmp1;
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	struct ath12k_skb_cb *skb_cb;
 	struct sk_buff *skb;
 	struct ath12k *ar;
@@ -1273,15 +1094,13 @@ static void ath12k_dp_cc_cleanup(struct ath12k_base *ab)
 
 static void ath12k_dp_reoq_lut_cleanup(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 
 	if (!ab->hw_params->reoq_lut_support)
 		return;
 
 	if (dp->reoq_lut.vaddr_unaligned) {
-		ath12k_hif_write32(ab,
-				   HAL_SEQ_WCSS_UMAC_REO_REG +
-				   HAL_REO1_QDESC_LUT_BASE0(ab), 0);
+		ath12k_hal_write_reoq_lut_addr(ab, 0);
 		dma_free_coherent(ab->dev, dp->reoq_lut.size,
 				  dp->reoq_lut.vaddr_unaligned,
 				  dp->reoq_lut.paddr_unaligned);
@@ -1289,9 +1108,7 @@ static void ath12k_dp_reoq_lut_cleanup(struct ath12k_base *ab)
 	}
 
 	if (dp->ml_reoq_lut.vaddr_unaligned) {
-		ath12k_hif_write32(ab,
-				   HAL_SEQ_WCSS_UMAC_REO_REG +
-				   HAL_REO1_QDESC_LUT_BASE1(ab), 0);
+		ath12k_hal_write_ml_reoq_lut_addr(ab, 0);
 		dma_free_coherent(ab->dev, dp->ml_reoq_lut.size,
 				  dp->ml_reoq_lut.vaddr_unaligned,
 				  dp->ml_reoq_lut.paddr_unaligned);
@@ -1299,11 +1116,13 @@ static void ath12k_dp_reoq_lut_cleanup(struct ath12k_base *ab)
 	}
 }
 
-void ath12k_dp_free(struct ath12k_base *ab)
+static void ath12k_dp_cleanup(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	int i;
 
+	ath12k_dp_link_peer_rhash_tbl_destroy(dp);
+
 	if (!dp->ab)
 		return;
 
@@ -1324,60 +1143,6 @@ void ath12k_dp_free(struct ath12k_base *ab)
 
 	ath12k_dp_rx_free(ab);
 	/* Deinit any SOC level resource */
-	dp->ab = NULL;
-}
-
-void ath12k_dp_cc_config(struct ath12k_base *ab)
-{
-	u32 cmem_base = ab->qmi.dev_mem[ATH12K_QMI_DEVMEM_CMEM_INDEX].start;
-	u32 reo_base = HAL_SEQ_WCSS_UMAC_REO_REG;
-	u32 wbm_base = HAL_SEQ_WCSS_UMAC_WBM_REG;
-	u32 val = 0;
-
-	if (ath12k_ftm_mode)
-		return;
-
-	ath12k_hif_write32(ab, reo_base + HAL_REO1_SW_COOKIE_CFG0(ab), cmem_base);
-
-	val |= u32_encode_bits(ATH12K_CMEM_ADDR_MSB,
-			       HAL_REO1_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB) |
-		u32_encode_bits(ATH12K_CC_PPT_MSB,
-				HAL_REO1_SW_COOKIE_CFG_COOKIE_PPT_MSB) |
-		u32_encode_bits(ATH12K_CC_SPT_MSB,
-				HAL_REO1_SW_COOKIE_CFG_COOKIE_SPT_MSB) |
-		u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_ALIGN) |
-		u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_ENABLE) |
-		u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_GLOBAL_ENABLE);
-
-	ath12k_hif_write32(ab, reo_base + HAL_REO1_SW_COOKIE_CFG1(ab), val);
-
-	/* Enable HW CC for WBM */
-	ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG0, cmem_base);
-
-	val = u32_encode_bits(ATH12K_CMEM_ADDR_MSB,
-			      HAL_WBM_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB) |
-		u32_encode_bits(ATH12K_CC_PPT_MSB,
-				HAL_WBM_SW_COOKIE_CFG_COOKIE_PPT_MSB) |
-		u32_encode_bits(ATH12K_CC_SPT_MSB,
-				HAL_WBM_SW_COOKIE_CFG_COOKIE_SPT_MSB) |
-		u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_ALIGN);
-
-	ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG1, val);
-
-	/* Enable conversion complete indication */
-	val = ath12k_hif_read32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG2);
-	val |= u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_RELEASE_PATH_EN) |
-		u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_ERR_PATH_EN) |
-		u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_CONV_IND_EN);
-
-	ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG2, val);
-
-	/* Enable Cookie conversion for WBM2SW Rings */
-	val = ath12k_hif_read32(ab, wbm_base + HAL_WBM_SW_COOKIE_CONVERT_CFG);
-	val |= u32_encode_bits(1, HAL_WBM_SW_COOKIE_CONV_CFG_GLOBAL_EN) |
-	       ab->hw_params->hal_params->wbm2sw_cc_enable;
-
-	ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CONVERT_CFG, val);
 }
 
 static u32 ath12k_dp_cc_cookie_gen(u16 ppt_idx, u16 spt_idx)
@@ -1385,26 +1150,23 @@ static u32 ath12k_dp_cc_cookie_gen(u16 ppt_idx, u16 spt_idx)
 	return (u32)ppt_idx << ATH12K_CC_PPT_SHIFT | spt_idx;
 }
 
-static inline void *ath12k_dp_cc_get_desc_addr_ptr(struct ath12k_base *ab,
-						   u16 ppt_idx, u16 spt_idx)
+static void *ath12k_dp_cc_get_desc_addr_ptr(struct ath12k_dp *dp,
+					    u16 ppt_idx, u16 spt_idx)
 {
-	struct ath12k_dp *dp = &ab->dp;
-
 	return dp->spt_info[ppt_idx].vaddr + spt_idx;
 }
 
-struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_base *ab,
+struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_dp *dp,
 						  u32 cookie)
 {
-	struct ath12k_dp *dp = &ab->dp;
 	struct ath12k_rx_desc_info **desc_addr_ptr;
 	u16 start_ppt_idx, end_ppt_idx, ppt_idx, spt_idx;
 
 	ppt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_PPT);
 	spt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_SPT);
 
-	start_ppt_idx = dp->rx_ppt_base + ATH12K_RX_SPT_PAGE_OFFSET(ab);
-	end_ppt_idx = start_ppt_idx + ATH12K_NUM_RX_SPT_PAGES(ab);
+	start_ppt_idx = dp->rx_ppt_base + ATH12K_RX_SPT_PAGE_OFFSET(dp->ab);
+	end_ppt_idx = start_ppt_idx + ATH12K_NUM_RX_SPT_PAGES(dp->ab);
 
 	if (ppt_idx < start_ppt_idx ||
 	    ppt_idx >= end_ppt_idx ||
@@ -1412,12 +1174,13 @@ struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_base *ab,
 		return NULL;
 
 	ppt_idx = ppt_idx - dp->rx_ppt_base;
-	desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, spt_idx);
+	desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, spt_idx);
 
 	return *desc_addr_ptr;
 }
+EXPORT_SYMBOL(ath12k_dp_get_rx_desc);
 
-struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_base *ab,
+struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_dp *dp,
 						  u32 cookie)
 {
 	struct ath12k_tx_desc_info **desc_addr_ptr;
@@ -1428,21 +1191,22 @@ struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_base *ab,
 
 	start_ppt_idx = ATH12K_TX_SPT_PAGE_OFFSET;
 	end_ppt_idx = start_ppt_idx +
-		      (ATH12K_TX_SPT_PAGES_PER_POOL(ab) * ATH12K_HW_MAX_QUEUES);
+		      (ATH12K_TX_SPT_PAGES_PER_POOL(dp->ab) * ATH12K_HW_MAX_QUEUES);
 
 	if (ppt_idx < start_ppt_idx ||
 	    ppt_idx >= end_ppt_idx ||
 	    spt_idx > ATH12K_MAX_SPT_ENTRIES)
 		return NULL;
 
-	desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, spt_idx);
+	desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, spt_idx);
 
 	return *desc_addr_ptr;
 }
+EXPORT_SYMBOL(ath12k_dp_get_tx_desc);
 
 static int ath12k_dp_cc_desc_init(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	struct ath12k_rx_desc_info *rx_descs, **rx_desc_addr;
 	struct ath12k_tx_desc_info *tx_descs, **tx_desc_addr;
 	u32 num_rx_spt_pages = ATH12K_NUM_RX_SPT_PAGES(ab);
@@ -1482,7 +1246,7 @@ static int ath12k_dp_cc_desc_init(struct ath12k_base *ab)
 			list_add_tail(&rx_descs[j].list, &dp->rx_desc_free_list);
 
 			/* Update descriptor VA in SPT */
-			rx_desc_addr = ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, j);
+			rx_desc_addr = ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, j);
 			*rx_desc_addr = &rx_descs[j];
 		}
 	}
@@ -1521,7 +1285,7 @@ static int ath12k_dp_cc_desc_init(struct ath12k_base *ab)
 
 				/* Update descriptor VA in SPT */
 				tx_desc_addr =
-					ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, j);
+					ath12k_dp_cc_get_desc_addr_ptr(dp, ppt_idx, j);
 				*tx_desc_addr = &tx_descs[j];
 			}
 		}
@@ -1571,7 +1335,7 @@ void ath12k_dp_partner_cc_init(struct ath12k_base *ab)
 		if (ag->ab[i] == ab)
 			continue;
 
-		ath12k_dp_cmem_init(ab, &ag->ab[i]->dp, ATH12K_DP_RX_DESC);
+		ath12k_dp_cmem_init(ab, ath12k_ab_to_dp(ag->ab[i]), ATH12K_DP_RX_DESC);
 	}
 }
 
@@ -1582,7 +1346,7 @@ static u32 ath12k_dp_get_num_spt_pages(struct ath12k_base *ab)
 
 static int ath12k_dp_cc_init(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	int i, ret = 0;
 
 	INIT_LIST_HEAD(&dp->rx_desc_free_list);
@@ -1668,8 +1432,7 @@ static int ath12k_dp_alloc_reoq_lut(struct ath12k_base *ab,
 
 static int ath12k_dp_reoq_lut_setup(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
-	u32 val;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	int ret;
 
 	if (!ab->hw_params->reoq_lut_support)
@@ -1697,50 +1460,24 @@ static int ath12k_dp_reoq_lut_setup(struct ath12k_base *ab)
 	 * register only
 	 */
 
-	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_LUT_BASE0(ab),
-			   dp->reoq_lut.paddr >> 8);
-
-	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_LUT_BASE1(ab),
-			   dp->ml_reoq_lut.paddr >> 8);
-
-	val = ath12k_hif_read32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_ADDR(ab));
-
-	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_ADDR(ab),
-			   val | HAL_REO_QDESC_ADDR_READ_LUT_ENABLE);
-
-	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_MAX_PEERID(ab),
-			   HAL_REO_QDESC_MAX_PEERID);
+	ath12k_hal_write_reoq_lut_addr(ab, dp->reoq_lut.paddr >> 8);
+	ath12k_hal_write_ml_reoq_lut_addr(ab, dp->ml_reoq_lut.paddr >> 8);
+	ath12k_hal_reoq_lut_addr_read_enable(ab);
+	ath12k_hal_reoq_lut_set_max_peerid(ab);
 
 	return 0;
 }
 
-static enum hal_rx_buf_return_buf_manager
-ath12k_dp_get_idle_link_rbm(struct ath12k_base *ab)
-{
-	switch (ab->device_id) {
-	case 0:
-		return HAL_RX_BUF_RBM_WBM_DEV0_IDLE_DESC_LIST;
-	case 1:
-		return HAL_RX_BUF_RBM_WBM_DEV1_IDLE_DESC_LIST;
-	case 2:
-		return HAL_RX_BUF_RBM_WBM_DEV2_IDLE_DESC_LIST;
-	default:
-		ath12k_warn(ab, "invalid %d device id, so choose default rbm\n",
-			    ab->device_id);
-		WARN_ON(1);
-		return HAL_RX_BUF_RBM_WBM_DEV0_IDLE_DESC_LIST;
-	}
-}
-
-int ath12k_dp_alloc(struct ath12k_base *ab)
+static int ath12k_dp_setup(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp;
 	struct hal_srng *srng = NULL;
 	size_t size = 0;
 	u32 n_link_desc = 0;
 	int ret;
 	int i;
 
+	dp = ath12k_ab_to_dp(ab);
 	dp->ab = ab;
 
 	INIT_LIST_HEAD(&dp->reo_cmd_list);
@@ -1749,13 +1486,25 @@ int ath12k_dp_alloc(struct ath12k_base *ab)
 	spin_lock_init(&dp->reo_cmd_lock);
 	spin_lock_init(&dp->reo_rxq_flush_lock);
 
+	spin_lock_init(&dp->dp_lock);
+	INIT_LIST_HEAD(&dp->peers);
+
+	mutex_init(&dp->link_peer_rhash_tbl_lock);
+
 	dp->reo_cmd_cache_flush_count = 0;
-	dp->idle_link_rbm = ath12k_dp_get_idle_link_rbm(ab);
+	dp->idle_link_rbm =
+			ath12k_hal_get_idle_link_rbm(&ab->hal, ab->device_id);
+
+	ret = ath12k_dp_link_peer_rhash_tbl_init(dp);
+	if (ret) {
+		ath12k_warn(ab, "failed to init link_peer rhash table: %d\n", ret);
+		return ret;
+	}
 
 	ret = ath12k_wbm_idle_ring_setup(ab, &n_link_desc);
 	if (ret) {
 		ath12k_warn(ab, "failed to setup wbm_idle_ring: %d\n", ret);
-		return ret;
+		goto rhash_destroy;
 	}
 
 	srng = &ab->hal.srng_list[dp->wbm_idle_ring.ring_id];
@@ -1783,7 +1532,7 @@ int ath12k_dp_alloc(struct ath12k_base *ab)
 	if (ret)
 		goto fail_dp_bank_profiles_cleanup;
 
-	size = sizeof(struct hal_wbm_release_ring_tx) *
+	size = ab->hal.hal_wbm_release_ring_tx_size *
 	       DP_TX_COMP_RING_SIZE(ab);
 
 	ret = ath12k_dp_reoq_lut_setup(ab);
@@ -1836,6 +1585,48 @@ int ath12k_dp_alloc(struct ath12k_base *ab)
 fail_link_desc_cleanup:
 	ath12k_dp_link_desc_cleanup(ab, dp->link_desc_banks,
 				    HAL_WBM_IDLE_LINK, &dp->wbm_idle_ring);
+rhash_destroy:
+	ath12k_dp_link_peer_rhash_tbl_destroy(dp);
 
 	return ret;
 }
+
+void ath12k_dp_cmn_device_deinit(struct ath12k_dp *dp)
+{
+	ath12k_dp_cleanup(dp->ab);
+}
+
+int ath12k_dp_cmn_device_init(struct ath12k_dp *dp)
+{
+	int ret;
+
+	ret = ath12k_dp_setup(dp->ab);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void ath12k_dp_cmn_hw_group_unassign(struct ath12k_dp *dp,
+				     struct ath12k_hw_group *ag)
+{
+	struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp;
+
+	lockdep_assert_held(&ag->mutex);
+
+	dp_hw_grp->dp[dp->device_id] = NULL;
+
+	dp->ag = NULL;
+	dp->device_id = ATH12K_INVALID_DEVICE_ID;
+}
+
+void ath12k_dp_cmn_hw_group_assign(struct ath12k_dp *dp,
+				   struct ath12k_hw_group *ag)
+{
+	struct ath12k_base *ab = dp->ab;
+	struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp;
+
+	dp->ag = ag;
+	dp->device_id = ab->device_id;
+	dp_hw_grp->dp[dp->device_id] = dp;
+}
diff --git a/drivers/net/wireless/ath/ath12k/dp.h b/drivers/net/wireless/ath/ath12k/dp.h
index 4ffec6ad7d8d..f8cfc7bb29dd 100644
--- a/drivers/net/wireless/ath/ath12k/dp.h
+++ b/drivers/net/wireless/ath/ath12k/dp.h
@@ -1,29 +1,35 @@
 /* SPDX-License-Identifier: BSD-3-Clause-Clear */
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #ifndef ATH12K_DP_H
 #define ATH12K_DP_H
 
-#include "hal_desc.h"
-#include "hal_rx.h"
 #include "hw.h"
+#include "dp_htt.h"
+#include "dp_cmn.h"
+#include <linux/rhashtable.h>
 
 #define MAX_RXDMA_PER_PDEV     2
 
 struct ath12k_base;
-struct ath12k_peer;
+struct ath12k_dp_link_peer;
 struct ath12k_dp;
 struct ath12k_vif;
 struct ath12k_link_vif;
-struct hal_tcl_status_ring;
 struct ath12k_ext_irq_grp;
+struct ath12k_dp_rx_tid;
+struct ath12k_dp_rx_tid_rxq;
 
 #define DP_MON_PURGE_TIMEOUT_MS     100
 #define DP_MON_SERVICE_BUDGET       128
 
+#define DP_ENCAP_TYPE_MAX	4
+#define DP_ENCRYPT_TYPE_MAX	12
+#define DP_DESC_TYPE_MAX	2
+
 struct dp_srng {
 	u32 *vaddr_unaligned;
 	u32 *vaddr;
@@ -149,6 +155,18 @@ struct ath12k_pdev_dp {
 	u32 mac_id;
 	atomic_t num_tx_pending;
 	wait_queue_head_t tx_empty_waitq;
+
+	struct ath12k_dp *dp;
+	struct ieee80211_hw *hw;
+	u8 hw_link_id;
+	struct ath12k_dp_hw *dp_hw;
+
+	/* Protects ppdu stats */
+	spinlock_t ppdu_list_lock;
+	struct ath12k_per_peer_tx_stats peer_tx_stats;
+	struct list_head ppdu_stats_info;
+	u32 ppdu_stat_list_depth;
+
 	struct dp_srng rxdma_mon_dst_ring[MAX_RXDMA_PER_PDEV];
 	struct dp_srng tx_mon_dst_ring[MAX_RXDMA_PER_PDEV];
 
@@ -360,9 +378,77 @@ struct ath12k_link_stats {
 	u32 tx_completed;
 	u32 tx_bcast_mcast;
 	u32 tx_dropped;
-	u32 tx_encap_type[HAL_TCL_ENCAP_TYPE_MAX];
-	u32 tx_encrypt_type[HAL_ENCRYPT_TYPE_MAX];
-	u32 tx_desc_type[HAL_TCL_DESC_TYPE_MAX];
+	u32 tx_encap_type[DP_ENCAP_TYPE_MAX];
+	u32 tx_encrypt_type[DP_ENCRYPT_TYPE_MAX];
+	u32 tx_desc_type[DP_DESC_TYPE_MAX];
+};
+
+/* DP arch ops to communicate from common module
+ * to arch specific module
+ */
+struct ath12k_dp_arch_ops {
+	int (*service_srng)(struct ath12k_dp *dp,
+			    struct ath12k_ext_irq_grp *irq_grp,
+			    int budget);
+	u32 (*tx_get_vdev_bank_config)(struct ath12k_base *ab,
+				       struct ath12k_link_vif *arvif);
+	int (*reo_cmd_send)(struct ath12k_base *ab,
+			    struct ath12k_dp_rx_tid_rxq *rx_tid,
+			    enum hal_reo_cmd_type type,
+			    struct ath12k_hal_reo_cmd *cmd,
+			    void (*cb)(struct ath12k_dp *dp, void *ctx,
+				       enum hal_reo_cmd_status status));
+	void (*setup_pn_check_reo_cmd)(struct ath12k_hal_reo_cmd *cmd,
+				       struct ath12k_dp_rx_tid *rx_tid,
+				       u32 cipher, enum set_key_cmd key_cmd);
+	void (*rx_peer_tid_delete)(struct ath12k_base *ab,
+				   struct ath12k_dp_link_peer *peer, u8 tid);
+	int (*reo_cache_flush)(struct ath12k_base *ab,
+			       struct ath12k_dp_rx_tid_rxq *rx_tid);
+	int (*rx_link_desc_return)(struct ath12k_dp *dp,
+				   struct ath12k_buffer_addr *buf_addr_info,
+				   enum hal_wbm_rel_bm_act action);
+	void (*rx_frags_cleanup)(struct ath12k_dp_rx_tid *rx_tid,
+				 bool rel_link_desc);
+	int (*peer_rx_tid_reo_update)(struct ath12k_dp *dp,
+				      struct ath12k_dp_link_peer *peer,
+				      struct ath12k_dp_rx_tid *rx_tid,
+				      u32 ba_win_sz, u16 ssn,
+				      bool update_ssn);
+	int (*rx_assign_reoq)(struct ath12k_base *ab, struct ath12k_dp_peer *dp_peer,
+			      struct ath12k_dp_rx_tid *rx_tid,
+			      u16 ssn, enum hal_pn_type pn_type);
+	void (*peer_rx_tid_qref_setup)(struct ath12k_base *ab, u16 peer_id, u16 tid,
+				       dma_addr_t paddr);
+	void (*peer_rx_tid_qref_reset)(struct ath12k_base *ab, u16 peer_id, u16 tid);
+	int (*rx_tid_delete_handler)(struct ath12k_base *ab,
+				     struct ath12k_dp_rx_tid_rxq *rx_tid);
+};
+
+struct ath12k_device_dp_tx_err_stats {
+	/* TCL Ring Descriptor unavailable */
+	u32 desc_na[DP_TCL_NUM_RING_MAX];
+	/* Other failures during dp_tx due to mem allocation failure
+	 * idr unavailable etc.
+	 */
+	atomic_t misc_fail;
+};
+
+struct ath12k_device_dp_stats {
+	u32 err_ring_pkts;
+	u32 invalid_rbm;
+	u32 rxdma_error[HAL_REO_ENTR_RING_RXDMA_ECODE_MAX];
+	u32 reo_error[HAL_REO_DEST_RING_ERROR_CODE_MAX];
+	u32 hal_reo_error[DP_REO_DST_RING_MAX];
+	struct ath12k_device_dp_tx_err_stats tx_err;
+	u32 reo_rx[DP_REO_DST_RING_MAX][ATH12K_MAX_DEVICES];
+	u32 rx_wbm_rel_source[HAL_WBM_REL_SRC_MODULE_MAX][ATH12K_MAX_DEVICES];
+	u32 tqm_rel_reason[MAX_TQM_RELEASE_REASON];
+	u32 fw_tx_status[MAX_FW_TX_STATUS];
+	u32 tx_wbm_rel_source[HAL_WBM_REL_SRC_MODULE_MAX];
+	u32 tx_enqueued[DP_TCL_NUM_RING_MAX];
+	u32 tx_completed[DP_TCL_NUM_RING_MAX];
+	u32 reo_excep_msdu_buf_type;
 };
 
 struct ath12k_dp {
@@ -427,1512 +513,137 @@ struct ath12k_dp {
 	struct dp_rxdma_mon_ring rx_mon_status_refill_ring[MAX_RXDMA_PER_PDEV];
 	struct ath12k_reo_q_addr_lut reoq_lut;
 	struct ath12k_reo_q_addr_lut ml_reoq_lut;
-};
-
-/* HTT definitions */
-#define HTT_TAG_TCL_METADATA_VERSION		5
-
-#define HTT_TCL_META_DATA_TYPE			GENMASK(1, 0)
-#define HTT_TCL_META_DATA_VALID_HTT		BIT(2)
-
-/* vdev meta data */
-#define HTT_TCL_META_DATA_VDEV_ID		 GENMASK(10, 3)
-#define HTT_TCL_META_DATA_PDEV_ID		 GENMASK(12, 11)
-#define HTT_TCL_META_DATA_HOST_INSPECTED_MISSION BIT(13)
-
-/* peer meta data */
-#define HTT_TCL_META_DATA_PEER_ID		GENMASK(15, 3)
-
-/* Global sequence number */
-#define HTT_TCL_META_DATA_TYPE_GLOBAL_SEQ_NUM		3
-#define HTT_TCL_META_DATA_GLOBAL_SEQ_HOST_INSPECTED	BIT(2)
-#define HTT_TCL_META_DATA_GLOBAL_SEQ_NUM		GENMASK(14, 3)
-#define HTT_TX_MLO_MCAST_HOST_REINJECT_BASE_VDEV_ID	128
-
-/* HTT tx completion is overlaid in wbm_release_ring */
-#define HTT_TX_WBM_COMP_INFO0_STATUS		GENMASK(16, 13)
-#define HTT_TX_WBM_COMP_INFO1_REINJECT_REASON	GENMASK(3, 0)
-#define HTT_TX_WBM_COMP_INFO1_EXCEPTION_FRAME	BIT(4)
-
-#define HTT_TX_WBM_COMP_INFO2_ACK_RSSI		GENMASK(31, 24)
-
-struct htt_tx_wbm_completion {
-	__le32 rsvd0[2];
-	__le32 info0;
-	__le32 info1;
-	__le32 info2;
-	__le32 info3;
-	__le32 info4;
-	__le32 rsvd1;
-
-} __packed;
-
-enum htt_h2t_msg_type {
-	HTT_H2T_MSG_TYPE_VERSION_REQ		= 0,
-	HTT_H2T_MSG_TYPE_SRING_SETUP		= 0xb,
-	HTT_H2T_MSG_TYPE_RX_RING_SELECTION_CFG	= 0xc,
-	HTT_H2T_MSG_TYPE_EXT_STATS_CFG		= 0x10,
-	HTT_H2T_MSG_TYPE_PPDU_STATS_CFG		= 0x11,
-	HTT_H2T_MSG_TYPE_VDEV_TXRX_STATS_CFG	= 0x1a,
-	HTT_H2T_MSG_TYPE_TX_MONITOR_CFG		= 0x1b,
-};
-
-#define HTT_VER_REQ_INFO_MSG_ID		GENMASK(7, 0)
-#define HTT_OPTION_TCL_METADATA_VER_V1	1
-#define HTT_OPTION_TCL_METADATA_VER_V2	2
-#define HTT_OPTION_TAG			GENMASK(7, 0)
-#define HTT_OPTION_LEN			GENMASK(15, 8)
-#define HTT_OPTION_VALUE		GENMASK(31, 16)
-#define HTT_TCL_METADATA_VER_SZ		4
-
-struct htt_ver_req_cmd {
-	__le32 ver_reg_info;
-	__le32 tcl_metadata_version;
-} __packed;
-
-enum htt_srng_ring_type {
-	HTT_HW_TO_SW_RING,
-	HTT_SW_TO_HW_RING,
-	HTT_SW_TO_SW_RING,
-};
-
-enum htt_srng_ring_id {
-	HTT_RXDMA_HOST_BUF_RING,
-	HTT_RXDMA_MONITOR_STATUS_RING,
-	HTT_RXDMA_MONITOR_BUF_RING,
-	HTT_RXDMA_MONITOR_DESC_RING,
-	HTT_RXDMA_MONITOR_DEST_RING,
-	HTT_HOST1_TO_FW_RXBUF_RING,
-	HTT_HOST2_TO_FW_RXBUF_RING,
-	HTT_RXDMA_NON_MONITOR_DEST_RING,
-	HTT_RXDMA_HOST_BUF_RING2,
-	HTT_TX_MON_HOST2MON_BUF_RING,
-	HTT_TX_MON_MON2HOST_DEST_RING,
-	HTT_RX_MON_HOST2MON_BUF_RING,
-	HTT_RX_MON_MON2HOST_DEST_RING,
-};
-
-/* host -> target  HTT_SRING_SETUP message
- *
- * After target is booted up, Host can send SRING setup message for
- * each host facing LMAC SRING. Target setups up HW registers based
- * on setup message and confirms back to Host if response_required is set.
- * Host should wait for confirmation message before sending new SRING
- * setup message
- *
- * The message would appear as follows:
- *
- * |31            24|23    20|19|18 16|15|14          8|7                0|
- * |--------------- +-----------------+----------------+------------------|
- * |    ring_type   |      ring_id    |    pdev_id     |     msg_type     |
- * |----------------------------------------------------------------------|
- * |                          ring_base_addr_lo                           |
- * |----------------------------------------------------------------------|
- * |                         ring_base_addr_hi                            |
- * |----------------------------------------------------------------------|
- * |ring_misc_cfg_flag|ring_entry_size|            ring_size              |
- * |----------------------------------------------------------------------|
- * |                         ring_head_offset32_remote_addr_lo            |
- * |----------------------------------------------------------------------|
- * |                         ring_head_offset32_remote_addr_hi            |
- * |----------------------------------------------------------------------|
- * |                         ring_tail_offset32_remote_addr_lo            |
- * |----------------------------------------------------------------------|
- * |                         ring_tail_offset32_remote_addr_hi            |
- * |----------------------------------------------------------------------|
- * |                          ring_msi_addr_lo                            |
- * |----------------------------------------------------------------------|
- * |                          ring_msi_addr_hi                            |
- * |----------------------------------------------------------------------|
- * |                          ring_msi_data                               |
- * |----------------------------------------------------------------------|
- * |         intr_timer_th            |IM|      intr_batch_counter_th     |
- * |----------------------------------------------------------------------|
- * |          reserved        |RR|PTCF|        intr_low_threshold         |
- * |----------------------------------------------------------------------|
- * Where
- *     IM = sw_intr_mode
- *     RR = response_required
- *     PTCF = prefetch_timer_cfg
- *
- * The message is interpreted as follows:
- * dword0  - b'0:7   - msg_type: This will be set to
- *                     HTT_H2T_MSG_TYPE_SRING_SETUP
- *           b'8:15  - pdev_id:
- *                     0 (for rings at SOC/UMAC level),
- *                     1/2/3 mac id (for rings at LMAC level)
- *           b'16:23 - ring_id: identify which ring is to setup,
- *                     more details can be got from enum htt_srng_ring_id
- *           b'24:31 - ring_type: identify type of host rings,
- *                     more details can be got from enum htt_srng_ring_type
- * dword1  - b'0:31  - ring_base_addr_lo: Lower 32bits of ring base address
- * dword2  - b'0:31  - ring_base_addr_hi: Upper 32bits of ring base address
- * dword3  - b'0:15  - ring_size: size of the ring in unit of 4-bytes words
- *           b'16:23 - ring_entry_size: Size of each entry in 4-byte word units
- *           b'24:31 - ring_misc_cfg_flag: Valid only for HW_TO_SW_RING and
- *                     SW_TO_HW_RING.
- *                     Refer to HTT_SRING_SETUP_RING_MISC_CFG_RING defs.
- * dword4  - b'0:31  - ring_head_off32_remote_addr_lo:
- *                     Lower 32 bits of memory address of the remote variable
- *                     storing the 4-byte word offset that identifies the head
- *                     element within the ring.
- *                     (The head offset variable has type u32.)
- *                     Valid for HW_TO_SW and SW_TO_SW rings.
- * dword5  - b'0:31  - ring_head_off32_remote_addr_hi:
- *                     Upper 32 bits of memory address of the remote variable
- *                     storing the 4-byte word offset that identifies the head
- *                     element within the ring.
- *                     (The head offset variable has type u32.)
- *                     Valid for HW_TO_SW and SW_TO_SW rings.
- * dword6  - b'0:31  - ring_tail_off32_remote_addr_lo:
- *                     Lower 32 bits of memory address of the remote variable
- *                     storing the 4-byte word offset that identifies the tail
- *                     element within the ring.
- *                     (The tail offset variable has type u32.)
- *                     Valid for HW_TO_SW and SW_TO_SW rings.
- * dword7  - b'0:31  - ring_tail_off32_remote_addr_hi:
- *                     Upper 32 bits of memory address of the remote variable
- *                     storing the 4-byte word offset that identifies the tail
- *                     element within the ring.
- *                     (The tail offset variable has type u32.)
- *                     Valid for HW_TO_SW and SW_TO_SW rings.
- * dword8  - b'0:31  - ring_msi_addr_lo: Lower 32bits of MSI cfg address
- *                     valid only for HW_TO_SW_RING and SW_TO_HW_RING
- * dword9  - b'0:31  - ring_msi_addr_hi: Upper 32bits of MSI cfg address
- *                     valid only for HW_TO_SW_RING and SW_TO_HW_RING
- * dword10 - b'0:31  - ring_msi_data: MSI data
- *                     Refer to HTT_SRING_SETUP_RING_MSC_CFG_xxx defs
- *                     valid only for HW_TO_SW_RING and SW_TO_HW_RING
- * dword11 - b'0:14  - intr_batch_counter_th:
- *                     batch counter threshold is in units of 4-byte words.
- *                     HW internally maintains and increments batch count.
- *                     (see SRING spec for detail description).
- *                     When batch count reaches threshold value, an interrupt
- *                     is generated by HW.
- *           b'15    - sw_intr_mode:
- *                     This configuration shall be static.
- *                     Only programmed at power up.
- *                     0: generate pulse style sw interrupts
- *                     1: generate level style sw interrupts
- *           b'16:31 - intr_timer_th:
- *                     The timer init value when timer is idle or is
- *                     initialized to start downcounting.
- *                     In 8us units (to cover a range of 0 to 524 ms)
- * dword12 - b'0:15  - intr_low_threshold:
- *                     Used only by Consumer ring to generate ring_sw_int_p.
- *                     Ring entries low threshold water mark, that is used
- *                     in combination with the interrupt timer as well as
- *                     the clearing of the level interrupt.
- *           b'16:18 - prefetch_timer_cfg:
- *                     Used only by Consumer ring to set timer mode to
- *                     support Application prefetch handling.
- *                     The external tail offset/pointer will be updated
- *                     at following intervals:
- *                     3'b000: (Prefetch feature disabled; used only for debug)
- *                     3'b001: 1 usec
- *                     3'b010: 4 usec
- *                     3'b011: 8 usec (default)
- *                     3'b100: 16 usec
- *                     Others: Reserved
- *           b'19    - response_required:
- *                     Host needs HTT_T2H_MSG_TYPE_SRING_SETUP_DONE as response
- *           b'20:31 - reserved:  reserved for future use
- */
-
-#define HTT_SRNG_SETUP_CMD_INFO0_MSG_TYPE	GENMASK(7, 0)
-#define HTT_SRNG_SETUP_CMD_INFO0_PDEV_ID	GENMASK(15, 8)
-#define HTT_SRNG_SETUP_CMD_INFO0_RING_ID	GENMASK(23, 16)
-#define HTT_SRNG_SETUP_CMD_INFO0_RING_TYPE	GENMASK(31, 24)
-
-#define HTT_SRNG_SETUP_CMD_INFO1_RING_SIZE			GENMASK(15, 0)
-#define HTT_SRNG_SETUP_CMD_INFO1_RING_ENTRY_SIZE		GENMASK(23, 16)
-#define HTT_SRNG_SETUP_CMD_INFO1_RING_LOOP_CNT_DIS		BIT(25)
-#define HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_MSI_SWAP		BIT(27)
-#define HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_HOST_FW_SWAP	BIT(28)
-#define HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_TLV_SWAP		BIT(29)
-
-#define HTT_SRNG_SETUP_CMD_INTR_INFO_BATCH_COUNTER_THRESH	GENMASK(14, 0)
-#define HTT_SRNG_SETUP_CMD_INTR_INFO_SW_INTR_MODE		BIT(15)
-#define HTT_SRNG_SETUP_CMD_INTR_INFO_INTR_TIMER_THRESH		GENMASK(31, 16)
-
-#define HTT_SRNG_SETUP_CMD_INFO2_INTR_LOW_THRESH	GENMASK(15, 0)
-#define HTT_SRNG_SETUP_CMD_INFO2_PRE_FETCH_TIMER_CFG	GENMASK(18, 16)
-#define HTT_SRNG_SETUP_CMD_INFO2_RESPONSE_REQUIRED	BIT(19)
-
-struct htt_srng_setup_cmd {
-	__le32 info0;
-	__le32 ring_base_addr_lo;
-	__le32 ring_base_addr_hi;
-	__le32 info1;
-	__le32 ring_head_off32_remote_addr_lo;
-	__le32 ring_head_off32_remote_addr_hi;
-	__le32 ring_tail_off32_remote_addr_lo;
-	__le32 ring_tail_off32_remote_addr_hi;
-	__le32 ring_msi_addr_lo;
-	__le32 ring_msi_addr_hi;
-	__le32 msi_data;
-	__le32 intr_info;
-	__le32 info2;
-} __packed;
-
-/* host -> target FW  PPDU_STATS config message
- *
- * @details
- * The following field definitions describe the format of the HTT host
- * to target FW for PPDU_STATS_CFG msg.
- * The message allows the host to configure the PPDU_STATS_IND messages
- * produced by the target.
- *
- * |31          24|23          16|15           8|7            0|
- * |-----------------------------------------------------------|
- * |    REQ bit mask             |   pdev_mask  |   msg type   |
- * |-----------------------------------------------------------|
- * Header fields:
- *  - MSG_TYPE
- *    Bits 7:0
- *    Purpose: identifies this is a req to configure ppdu_stats_ind from target
- *    Value: 0x11
- *  - PDEV_MASK
- *    Bits 8:15
- *    Purpose: identifies which pdevs this PPDU stats configuration applies to
- *    Value: This is a overloaded field, refer to usage and interpretation of
- *           PDEV in interface document.
- *           Bit   8    :  Reserved for SOC stats
- *           Bit 9 - 15 :  Indicates PDEV_MASK in DBDC
- *                         Indicates MACID_MASK in DBS
- *  - REQ_TLV_BIT_MASK
- *    Bits 16:31
- *    Purpose: each set bit indicates the corresponding PPDU stats TLV type
- *        needs to be included in the target's PPDU_STATS_IND messages.
- *    Value: refer htt_ppdu_stats_tlv_tag_t <<<???
- *
- */
-
-struct htt_ppdu_stats_cfg_cmd {
-	__le32 msg;
-} __packed;
-
-#define HTT_PPDU_STATS_CFG_MSG_TYPE		GENMASK(7, 0)
-#define HTT_PPDU_STATS_CFG_SOC_STATS		BIT(8)
-#define HTT_PPDU_STATS_CFG_PDEV_ID		GENMASK(15, 9)
-#define HTT_PPDU_STATS_CFG_TLV_TYPE_BITMASK	GENMASK(31, 16)
-
-enum htt_ppdu_stats_tag_type {
-	HTT_PPDU_STATS_TAG_COMMON,
-	HTT_PPDU_STATS_TAG_USR_COMMON,
-	HTT_PPDU_STATS_TAG_USR_RATE,
-	HTT_PPDU_STATS_TAG_USR_MPDU_ENQ_BITMAP_64,
-	HTT_PPDU_STATS_TAG_USR_MPDU_ENQ_BITMAP_256,
-	HTT_PPDU_STATS_TAG_SCH_CMD_STATUS,
-	HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON,
-	HTT_PPDU_STATS_TAG_USR_COMPLTN_BA_BITMAP_64,
-	HTT_PPDU_STATS_TAG_USR_COMPLTN_BA_BITMAP_256,
-	HTT_PPDU_STATS_TAG_USR_COMPLTN_ACK_BA_STATUS,
-	HTT_PPDU_STATS_TAG_USR_COMPLTN_FLUSH,
-	HTT_PPDU_STATS_TAG_USR_COMMON_ARRAY,
-	HTT_PPDU_STATS_TAG_INFO,
-	HTT_PPDU_STATS_TAG_TX_MGMTCTRL_PAYLOAD,
-
-	/* New TLV's are added above to this line */
-	HTT_PPDU_STATS_TAG_MAX,
-};
-
-#define HTT_PPDU_STATS_TAG_DEFAULT (BIT(HTT_PPDU_STATS_TAG_COMMON) \
-				   | BIT(HTT_PPDU_STATS_TAG_USR_COMMON) \
-				   | BIT(HTT_PPDU_STATS_TAG_USR_RATE) \
-				   | BIT(HTT_PPDU_STATS_TAG_SCH_CMD_STATUS) \
-				   | BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON) \
-				   | BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_ACK_BA_STATUS) \
-				   | BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_FLUSH) \
-				   | BIT(HTT_PPDU_STATS_TAG_USR_COMMON_ARRAY))
-
-#define HTT_PPDU_STATS_TAG_PKTLOG  (BIT(HTT_PPDU_STATS_TAG_USR_MPDU_ENQ_BITMAP_64) | \
-				    BIT(HTT_PPDU_STATS_TAG_USR_MPDU_ENQ_BITMAP_256) | \
-				    BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_BA_BITMAP_64) | \
-				    BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_BA_BITMAP_256) | \
-				    BIT(HTT_PPDU_STATS_TAG_INFO) | \
-				    BIT(HTT_PPDU_STATS_TAG_TX_MGMTCTRL_PAYLOAD) | \
-				    HTT_PPDU_STATS_TAG_DEFAULT)
-
-enum htt_stats_internal_ppdu_frametype {
-	HTT_STATS_PPDU_FTYPE_CTRL,
-	HTT_STATS_PPDU_FTYPE_DATA,
-	HTT_STATS_PPDU_FTYPE_BAR,
-	HTT_STATS_PPDU_FTYPE_MAX
-};
-
-/* HTT_H2T_MSG_TYPE_RX_RING_SELECTION_CFG Message
- *
- * details:
- *    HTT_H2T_MSG_TYPE_RX_RING_SELECTION_CFG message is sent by host to
- *    configure RXDMA rings.
- *    The configuration is per ring based and includes both packet subtypes
- *    and PPDU/MPDU TLVs.
- *
- *    The message would appear as follows:
- *
- *    |31   29|28|27|26|25|24|23       16|15             8|7             0|
- *    |-------+--+--+--+--+--+-----------+----------------+---------------|
- *    | rsvd1 |ED|DT|OV|PS|SS|  ring_id  |     pdev_id    |    msg_type   |
- *    |-------------------------------------------------------------------|
- *    |              rsvd2               |           ring_buffer_size     |
- *    |-------------------------------------------------------------------|
- *    |                        packet_type_enable_flags_0                 |
- *    |-------------------------------------------------------------------|
- *    |                        packet_type_enable_flags_1                 |
- *    |-------------------------------------------------------------------|
- *    |                        packet_type_enable_flags_2                 |
- *    |-------------------------------------------------------------------|
- *    |                        packet_type_enable_flags_3                 |
- *    |-------------------------------------------------------------------|
- *    |                         tlv_filter_in_flags                       |
- *    |-------------------------------------------------------------------|
- * Where:
- *     PS = pkt_swap
- *     SS = status_swap
- * The message is interpreted as follows:
- * dword0 - b'0:7   - msg_type: This will be set to
- *                    HTT_H2T_MSG_TYPE_RX_RING_SELECTION_CFG
- *          b'8:15  - pdev_id:
- *                    0 (for rings at SOC/UMAC level),
- *                    1/2/3 mac id (for rings at LMAC level)
- *          b'16:23 - ring_id : Identify the ring to configure.
- *                    More details can be got from enum htt_srng_ring_id
- *          b'24    - status_swap: 1 is to swap status TLV
- *          b'25    - pkt_swap:  1 is to swap packet TLV
- *          b'26    - rx_offset_valid (OV): flag to indicate rx offsets
- *		      configuration fields are valid
- *          b'27    - drop_thresh_valid (DT): flag to indicate if the
- *		      rx_drop_threshold field is valid
- *          b'28    - rx_mon_global_en: Enable/Disable global register
- *		      configuration in Rx monitor module.
- *          b'29:31 - rsvd1:  reserved for future use
- * dword1 - b'0:16  - ring_buffer_size: size of buffers referenced by rx ring,
- *                    in byte units.
- *                    Valid only for HW_TO_SW_RING and SW_TO_HW_RING
- *        - b'16:31 - rsvd2: Reserved for future use
- * dword2 - b'0:31  - packet_type_enable_flags_0:
- *                    Enable MGMT packet from 0b0000 to 0b1001
- *                    bits from low to high: FP, MD, MO - 3 bits
- *                        FP: Filter_Pass
- *                        MD: Monitor_Direct
- *                        MO: Monitor_Other
- *                    10 mgmt subtypes * 3 bits -> 30 bits
- *                    Refer to PKT_TYPE_ENABLE_FLAG0_xxx_MGMT_xxx defs
- * dword3 - b'0:31  - packet_type_enable_flags_1:
- *                    Enable MGMT packet from 0b1010 to 0b1111
- *                    bits from low to high: FP, MD, MO - 3 bits
- *                    Refer to PKT_TYPE_ENABLE_FLAG1_xxx_MGMT_xxx defs
- * dword4 - b'0:31 -  packet_type_enable_flags_2:
- *                    Enable CTRL packet from 0b0000 to 0b1001
- *                    bits from low to high: FP, MD, MO - 3 bits
- *                    Refer to PKT_TYPE_ENABLE_FLAG2_xxx_CTRL_xxx defs
- * dword5 - b'0:31  - packet_type_enable_flags_3:
- *                    Enable CTRL packet from 0b1010 to 0b1111,
- *                    MCAST_DATA, UCAST_DATA, NULL_DATA
- *                    bits from low to high: FP, MD, MO - 3 bits
- *                    Refer to PKT_TYPE_ENABLE_FLAG3_xxx_CTRL_xxx defs
- * dword6 - b'0:31 -  tlv_filter_in_flags:
- *                    Filter in Attention/MPDU/PPDU/Header/User tlvs
- *                    Refer to CFG_TLV_FILTER_IN_FLAG defs
- */
-
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE	GENMASK(7, 0)
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID	GENMASK(15, 8)
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_RING_ID	GENMASK(23, 16)
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_SS		BIT(24)
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PS		BIT(25)
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_OFFSET_VALID	BIT(26)
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_DROP_THRES_VAL	BIT(27)
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_EN_RXMON		BIT(28)
-
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO1_BUF_SIZE		GENMASK(15, 0)
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_MGMT	GENMASK(18, 16)
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_CTRL	GENMASK(21, 19)
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_DATA	GENMASK(24, 22)
-
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO2_DROP_THRESHOLD	GENMASK(9, 0)
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_LOG_MGMT_TYPE	BIT(17)
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_CTRL_TYPE	BIT(18)
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_LOG_DATA_TYPE	BIT(19)
-
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO3_EN_TLV_PKT_OFFSET	BIT(0)
-#define HTT_RX_RING_SELECTION_CFG_CMD_INFO3_PKT_TLV_OFFSET	GENMASK(14, 1)
-
-#define HTT_RX_RING_SELECTION_CFG_RX_PACKET_OFFSET      GENMASK(15, 0)
-#define HTT_RX_RING_SELECTION_CFG_RX_HEADER_OFFSET      GENMASK(31, 16)
-#define HTT_RX_RING_SELECTION_CFG_RX_MPDU_END_OFFSET    GENMASK(15, 0)
-#define HTT_RX_RING_SELECTION_CFG_RX_MPDU_START_OFFSET  GENMASK(31, 16)
-#define HTT_RX_RING_SELECTION_CFG_RX_MSDU_END_OFFSET    GENMASK(15, 0)
-#define HTT_RX_RING_SELECTION_CFG_RX_MSDU_START_OFFSET  GENMASK(31, 16)
-#define HTT_RX_RING_SELECTION_CFG_RX_ATTENTION_OFFSET   GENMASK(15, 0)
-
-#define HTT_RX_RING_SELECTION_CFG_WORD_MASK_COMPACT_SET	BIT(23)
-#define HTT_RX_RING_SELECTION_CFG_RX_MPDU_START_MASK	GENMASK(15, 0)
-#define HTT_RX_RING_SELECTION_CFG_RX_MPDU_END_MASK	GENMASK(18, 16)
-#define HTT_RX_RING_SELECTION_CFG_RX_MSDU_END_MASK	GENMASK(16, 0)
-
-enum htt_rx_filter_tlv_flags {
-	HTT_RX_FILTER_TLV_FLAGS_MPDU_START		= BIT(0),
-	HTT_RX_FILTER_TLV_FLAGS_MSDU_START		= BIT(1),
-	HTT_RX_FILTER_TLV_FLAGS_RX_PACKET		= BIT(2),
-	HTT_RX_FILTER_TLV_FLAGS_MSDU_END		= BIT(3),
-	HTT_RX_FILTER_TLV_FLAGS_MPDU_END		= BIT(4),
-	HTT_RX_FILTER_TLV_FLAGS_PACKET_HEADER		= BIT(5),
-	HTT_RX_FILTER_TLV_FLAGS_PER_MSDU_HEADER		= BIT(6),
-	HTT_RX_FILTER_TLV_FLAGS_ATTENTION		= BIT(7),
-	HTT_RX_FILTER_TLV_FLAGS_PPDU_START		= BIT(8),
-	HTT_RX_FILTER_TLV_FLAGS_PPDU_END		= BIT(9),
-	HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS	= BIT(10),
-	HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS_EXT	= BIT(11),
-	HTT_RX_FILTER_TLV_FLAGS_PPDU_END_STATUS_DONE	= BIT(12),
-	HTT_RX_FILTER_TLV_FLAGS_PPDU_START_USER_INFO	= BIT(13),
-};
-
-enum htt_rx_mgmt_pkt_filter_tlv_flags0 {
-	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ		= BIT(0),
-	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ		= BIT(1),
-	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ		= BIT(2),
-	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP		= BIT(3),
-	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP		= BIT(4),
-	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP		= BIT(5),
-	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ	= BIT(6),
-	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ	= BIT(7),
-	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ	= BIT(8),
-	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP	= BIT(9),
-	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP	= BIT(10),
-	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP	= BIT(11),
-	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ		= BIT(12),
-	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ		= BIT(13),
-	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ		= BIT(14),
-	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP		= BIT(15),
-	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP		= BIT(16),
-	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP		= BIT(17),
-	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV	= BIT(18),
-	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV	= BIT(19),
-	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV	= BIT(20),
-	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7		= BIT(21),
-	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7		= BIT(22),
-	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7		= BIT(23),
-	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON		= BIT(24),
-	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON		= BIT(25),
-	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON		= BIT(26),
-	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM		= BIT(27),
-	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM		= BIT(28),
-	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM		= BIT(29),
-};
-
-enum htt_rx_mgmt_pkt_filter_tlv_flags1 {
-	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC		= BIT(0),
-	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC		= BIT(1),
-	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC		= BIT(2),
-	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH		= BIT(3),
-	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH		= BIT(4),
-	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH		= BIT(5),
-	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH		= BIT(6),
-	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH		= BIT(7),
-	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH		= BIT(8),
-	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION		= BIT(9),
-	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION		= BIT(10),
-	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION		= BIT(11),
-	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK	= BIT(12),
-	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK	= BIT(13),
-	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK	= BIT(14),
-	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15	= BIT(15),
-	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15	= BIT(16),
-	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15	= BIT(17),
-};
-
-enum htt_rx_ctrl_pkt_filter_tlv_flags2 {
-	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1	= BIT(0),
-	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1	= BIT(1),
-	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1	= BIT(2),
-	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2	= BIT(3),
-	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2	= BIT(4),
-	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2	= BIT(5),
-	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER	= BIT(6),
-	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER	= BIT(7),
-	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER	= BIT(8),
-	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4	= BIT(9),
-	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4	= BIT(10),
-	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4	= BIT(11),
-	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL	= BIT(12),
-	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL	= BIT(13),
-	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL	= BIT(14),
-	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP	= BIT(15),
-	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP	= BIT(16),
-	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP	= BIT(17),
-	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT	= BIT(18),
-	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT	= BIT(19),
-	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT	= BIT(20),
-	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER	= BIT(21),
-	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER	= BIT(22),
-	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER	= BIT(23),
-	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR		= BIT(24),
-	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_BAR		= BIT(25),
-	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_BAR		= BIT(26),
-	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BA			= BIT(27),
-	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_BA			= BIT(28),
-	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_BA			= BIT(29),
-};
-
-enum htt_rx_ctrl_pkt_filter_tlv_flags3 {
-	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL		= BIT(0),
-	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL		= BIT(1),
-	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL		= BIT(2),
-	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_RTS		= BIT(3),
-	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_RTS		= BIT(4),
-	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_RTS		= BIT(5),
-	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CTS		= BIT(6),
-	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CTS		= BIT(7),
-	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CTS		= BIT(8),
-	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_ACK		= BIT(9),
-	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_ACK		= BIT(10),
-	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_ACK		= BIT(11),
-	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND		= BIT(12),
-	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND		= BIT(13),
-	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND		= BIT(14),
-	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK		= BIT(15),
-	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK		= BIT(16),
-	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK		= BIT(17),
-};
-
-enum htt_rx_data_pkt_filter_tlv_flasg3 {
-	HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST	= BIT(18),
-	HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_MCAST	= BIT(19),
-	HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_MCAST	= BIT(20),
-	HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST	= BIT(21),
-	HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_UCAST	= BIT(22),
-	HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_UCAST	= BIT(23),
-	HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA	= BIT(24),
-	HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA	= BIT(25),
-	HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA	= BIT(26),
-};
-
-#define HTT_RX_FP_MGMT_FILTER_FLAGS0 \
-	(HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ \
-	| HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP \
-	| HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ \
-	| HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP \
-	| HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ \
-	| HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP \
-	| HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV \
-	| HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON \
-	| HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM)
-
-#define HTT_RX_MD_MGMT_FILTER_FLAGS0 \
-	(HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ \
-	| HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP \
-	| HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ \
-	| HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP \
-	| HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ \
-	| HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP \
-	| HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV \
-	| HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON \
-	| HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM)
-
-#define HTT_RX_MO_MGMT_FILTER_FLAGS0 \
-	(HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ \
-	| HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP \
-	| HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ \
-	| HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP \
-	| HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ \
-	| HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP \
-	| HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV \
-	| HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON \
-	| HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM)
-
-#define HTT_RX_FP_MGMT_FILTER_FLAGS1 (HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC \
-				     | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH \
-				     | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH \
-				     | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION \
-				     | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK)
-
-#define HTT_RX_MD_MGMT_FILTER_FLAGS1 (HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC \
-				     | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH \
-				     | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH \
-				     | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION \
-				     | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK)
-
-#define HTT_RX_MO_MGMT_FILTER_FLAGS1 (HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC \
-				     | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH \
-				     | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH \
-				     | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION \
-				     | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK)
-
-#define HTT_RX_FP_CTRL_FILTER_FLASG2 (HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER \
-				     | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR \
-				     | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BA)
-
-#define HTT_RX_MD_CTRL_FILTER_FLASG2 (HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER \
-				     | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_BAR \
-				     | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_BA)
-
-#define HTT_RX_MO_CTRL_FILTER_FLASG2 (HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER \
-				     | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_BAR \
-				     | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_BA)
-
-#define HTT_RX_FP_CTRL_FILTER_FLASG3 (HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL \
-				     | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_RTS \
-				     | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CTS \
-				     | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_ACK \
-				     | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND \
-				     | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK)
-
-#define HTT_RX_MD_CTRL_FILTER_FLASG3 (HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL \
-				     | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_RTS \
-				     | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CTS \
-				     | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_ACK \
-				     | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND \
-				     | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK)
-
-#define HTT_RX_MO_CTRL_FILTER_FLASG3 (HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL \
-				     | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_RTS \
-				     | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CTS \
-				     | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_ACK \
-				     | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND \
-				     | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK)
-
-#define HTT_RX_FP_DATA_FILTER_FLASG3 (HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST \
-				     | HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST \
-				     | HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA)
-
-#define HTT_RX_MD_DATA_FILTER_FLASG3 (HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_MCAST \
-				     | HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_UCAST \
-				     | HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA)
-
-#define HTT_RX_MO_DATA_FILTER_FLASG3 (HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_MCAST \
-				     | HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_UCAST \
-				     | HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA)
-
-#define HTT_RX_MON_FP_MGMT_FILTER_FLAGS0 \
-		(HTT_RX_FP_MGMT_FILTER_FLAGS0 | \
-		HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7)
-
-#define HTT_RX_MON_MO_MGMT_FILTER_FLAGS0 \
-		(HTT_RX_MO_MGMT_FILTER_FLAGS0 | \
-		HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7)
-
-#define HTT_RX_MON_FP_MGMT_FILTER_FLAGS1 \
-		(HTT_RX_FP_MGMT_FILTER_FLAGS1 | \
-		HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15)
-
-#define HTT_RX_MON_MO_MGMT_FILTER_FLAGS1 \
-		(HTT_RX_MO_MGMT_FILTER_FLAGS1 | \
-		HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15)
-
-#define HTT_RX_MON_FP_CTRL_FILTER_FLASG2 \
-		(HTT_RX_FP_CTRL_FILTER_FLASG2 | \
-		HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1 | \
-		HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2 | \
-		HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER | \
-		HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4 | \
-		HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL | \
-		HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP | \
-		HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT)
-
-#define HTT_RX_MON_MO_CTRL_FILTER_FLASG2 \
-		(HTT_RX_MO_CTRL_FILTER_FLASG2 | \
-		HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1 | \
-		HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2 | \
-		HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER | \
-		HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4 | \
-		HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL | \
-		HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP | \
-		HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT)
-
-#define HTT_RX_MON_FP_CTRL_FILTER_FLASG3 HTT_RX_FP_CTRL_FILTER_FLASG3
-
-#define HTT_RX_MON_MO_CTRL_FILTER_FLASG3 HTT_RX_MO_CTRL_FILTER_FLASG3
-
-#define HTT_RX_MON_FP_DATA_FILTER_FLASG3 HTT_RX_FP_DATA_FILTER_FLASG3
-
-#define HTT_RX_MON_MO_DATA_FILTER_FLASG3 HTT_RX_MO_DATA_FILTER_FLASG3
-
-#define HTT_RX_MON_FILTER_TLV_FLAGS \
-		(HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \
-		HTT_RX_FILTER_TLV_FLAGS_PPDU_START | \
-		HTT_RX_FILTER_TLV_FLAGS_PPDU_END | \
-		HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS | \
-		HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS_EXT | \
-		HTT_RX_FILTER_TLV_FLAGS_PPDU_END_STATUS_DONE)
-
-#define HTT_RX_MON_FILTER_TLV_FLAGS_MON_STATUS_RING \
-		(HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \
-		HTT_RX_FILTER_TLV_FLAGS_PPDU_START | \
-		HTT_RX_FILTER_TLV_FLAGS_PPDU_END | \
-		HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS | \
-		HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS_EXT | \
-		HTT_RX_FILTER_TLV_FLAGS_PPDU_END_STATUS_DONE)
-
-#define HTT_RX_MON_FILTER_TLV_FLAGS_MON_BUF_RING \
-		(HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \
-		HTT_RX_FILTER_TLV_FLAGS_MSDU_START | \
-		HTT_RX_FILTER_TLV_FLAGS_RX_PACKET | \
-		HTT_RX_FILTER_TLV_FLAGS_MSDU_END | \
-		HTT_RX_FILTER_TLV_FLAGS_MPDU_END | \
-		HTT_RX_FILTER_TLV_FLAGS_PACKET_HEADER | \
-		HTT_RX_FILTER_TLV_FLAGS_PER_MSDU_HEADER | \
-		HTT_RX_FILTER_TLV_FLAGS_ATTENTION)
-
-#define HTT_RX_MON_FILTER_TLV_FLAGS_MON_DEST_RING \
-	(HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \
-	HTT_RX_FILTER_TLV_FLAGS_MSDU_START | \
-	HTT_RX_FILTER_TLV_FLAGS_RX_PACKET | \
-	HTT_RX_FILTER_TLV_FLAGS_MSDU_END | \
-	HTT_RX_FILTER_TLV_FLAGS_MPDU_END | \
-	HTT_RX_FILTER_TLV_FLAGS_PACKET_HEADER | \
-	HTT_RX_FILTER_TLV_FLAGS_PER_MSDU_HEADER | \
-	HTT_RX_FILTER_TLV_FLAGS_PPDU_START | \
-	HTT_RX_FILTER_TLV_FLAGS_PPDU_END | \
-	HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS | \
-	HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS_EXT | \
-	HTT_RX_FILTER_TLV_FLAGS_PPDU_END_STATUS_DONE | \
-	HTT_RX_FILTER_TLV_FLAGS_PPDU_START_USER_INFO)
-
-/* msdu start. mpdu end, attention, rx hdr tlv's are not subscribed */
-#define HTT_RX_TLV_FLAGS_RXDMA_RING \
-		(HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \
-		HTT_RX_FILTER_TLV_FLAGS_RX_PACKET | \
-		HTT_RX_FILTER_TLV_FLAGS_MSDU_END)
-
-#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE	GENMASK(7, 0)
-#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID	GENMASK(15, 8)
-
-struct htt_rx_ring_selection_cfg_cmd {
-	__le32 info0;
-	__le32 info1;
-	__le32 pkt_type_en_flags0;
-	__le32 pkt_type_en_flags1;
-	__le32 pkt_type_en_flags2;
-	__le32 pkt_type_en_flags3;
-	__le32 rx_filter_tlv;
-	__le32 rx_packet_offset;
-	__le32 rx_mpdu_offset;
-	__le32 rx_msdu_offset;
-	__le32 rx_attn_offset;
-	__le32 info2;
-	__le32 reserved[2];
-	__le32 rx_mpdu_start_end_mask;
-	__le32 rx_msdu_end_word_mask;
-	__le32 info3;
-} __packed;
-
-#define HTT_RX_RING_TLV_DROP_THRESHOLD_VALUE	32
-#define HTT_RX_RING_DEFAULT_DMA_LENGTH		0x7
-#define HTT_RX_RING_PKT_TLV_OFFSET		0x1
-
-struct htt_rx_ring_tlv_filter {
-	u32 rx_filter; /* see htt_rx_filter_tlv_flags */
-	u32 pkt_filter_flags0; /* MGMT */
-	u32 pkt_filter_flags1; /* MGMT */
-	u32 pkt_filter_flags2; /* CTRL */
-	u32 pkt_filter_flags3; /* DATA */
-	bool offset_valid;
-	u16 rx_packet_offset;
-	u16 rx_header_offset;
-	u16 rx_mpdu_end_offset;
-	u16 rx_mpdu_start_offset;
-	u16 rx_msdu_end_offset;
-	u16 rx_msdu_start_offset;
-	u16 rx_attn_offset;
-	u16 rx_mpdu_start_wmask;
-	u16 rx_mpdu_end_wmask;
-	u32 rx_msdu_end_wmask;
-	u32 conf_len_ctrl;
-	u32 conf_len_mgmt;
-	u32 conf_len_data;
-	u16 rx_drop_threshold;
-	bool enable_log_mgmt_type;
-	bool enable_log_ctrl_type;
-	bool enable_log_data_type;
-	bool enable_rx_tlv_offset;
-	u16 rx_tlv_offset;
-	bool drop_threshold_valid;
-	bool rxmon_disable;
-};
-
-#define HTT_STATS_FRAME_CTRL_TYPE_MGMT  0x0
-#define HTT_STATS_FRAME_CTRL_TYPE_CTRL  0x1
-#define HTT_STATS_FRAME_CTRL_TYPE_DATA  0x2
-#define HTT_STATS_FRAME_CTRL_TYPE_RESV  0x3
-
-#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE	GENMASK(7, 0)
-#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID	GENMASK(15, 8)
-#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_RING_ID	GENMASK(23, 16)
-#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_SS		BIT(24)
-#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PS		BIT(25)
-
-#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_RING_BUFF_SIZE	GENMASK(15, 0)
-#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_PKT_TYPE		GENMASK(18, 16)
-#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_MGMT	GENMASK(21, 19)
-#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_CTRL	GENMASK(24, 22)
-#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_DATA	GENMASK(27, 25)
-
-#define HTT_TX_RING_SELECTION_CFG_CMD_INFO2_PKT_TYPE_EN_FLAG	GENMASK(2, 0)
-
-struct htt_tx_ring_selection_cfg_cmd {
-	__le32 info0;
-	__le32 info1;
-	__le32 info2;
-	__le32 tlv_filter_mask_in0;
-	__le32 tlv_filter_mask_in1;
-	__le32 tlv_filter_mask_in2;
-	__le32 tlv_filter_mask_in3;
-	__le32 reserved[3];
-} __packed;
-
-#define HTT_TX_RING_TLV_FILTER_MGMT_DMA_LEN	GENMASK(3, 0)
-#define HTT_TX_RING_TLV_FILTER_CTRL_DMA_LEN	GENMASK(7, 4)
-#define HTT_TX_RING_TLV_FILTER_DATA_DMA_LEN	GENMASK(11, 8)
-
-#define HTT_TX_MON_FILTER_HYBRID_MODE \
-		(HTT_TX_FILTER_TLV_FLAGS0_RESPONSE_START_STATUS | \
-		HTT_TX_FILTER_TLV_FLAGS0_RESPONSE_END_STATUS | \
-		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START | \
-		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_END | \
-		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START_PPDU | \
-		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_USER_PPDU | \
-		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_ACK_OR_BA | \
-		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_1K_BA | \
-		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START_PROT | \
-		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_PROT | \
-		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_USER_RESPONSE | \
-		HTT_TX_FILTER_TLV_FLAGS0_RECEIVED_RESPONSE_INFO | \
-		HTT_TX_FILTER_TLV_FLAGS0_RECEIVED_RESPONSE_INFO_PART2)
-
-struct htt_tx_ring_tlv_filter {
-	u32 tx_mon_downstream_tlv_flags;
-	u32 tx_mon_upstream_tlv_flags0;
-	u32 tx_mon_upstream_tlv_flags1;
-	u32 tx_mon_upstream_tlv_flags2;
-	bool tx_mon_mgmt_filter;
-	bool tx_mon_data_filter;
-	bool tx_mon_ctrl_filter;
-	u16 tx_mon_pkt_dma_len;
-} __packed;
-
-enum htt_tx_mon_upstream_tlv_flags0 {
-	HTT_TX_FILTER_TLV_FLAGS0_RESPONSE_START_STATUS		= BIT(1),
-	HTT_TX_FILTER_TLV_FLAGS0_RESPONSE_END_STATUS		= BIT(2),
-	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START		= BIT(3),
-	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_END		= BIT(4),
-	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START_PPDU	= BIT(5),
-	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_USER_PPDU	= BIT(6),
-	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_ACK_OR_BA	= BIT(7),
-	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_1K_BA		= BIT(8),
-	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START_PROT	= BIT(9),
-	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_PROT		= BIT(10),
-	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_USER_RESPONSE	= BIT(11),
-	HTT_TX_FILTER_TLV_FLAGS0_RX_FRAME_BITMAP_ACK		= BIT(12),
-	HTT_TX_FILTER_TLV_FLAGS0_RX_FRAME_1K_BITMAP_ACK		= BIT(13),
-	HTT_TX_FILTER_TLV_FLAGS0_COEX_TX_STATUS			= BIT(14),
-	HTT_TX_FILTER_TLV_FLAGS0_RECEIVED_RESPONSE_INFO		= BIT(15),
-	HTT_TX_FILTER_TLV_FLAGS0_RECEIVED_RESPONSE_INFO_PART2	= BIT(16),
-};
-
-#define HTT_TX_FILTER_TLV_FLAGS2_TXPCU_PHYTX_OTHER_TRANSMIT_INFO32	BIT(11)
-
-/* HTT message target->host */
-
-enum htt_t2h_msg_type {
-	HTT_T2H_MSG_TYPE_VERSION_CONF,
-	HTT_T2H_MSG_TYPE_PEER_MAP	= 0x3,
-	HTT_T2H_MSG_TYPE_PEER_UNMAP	= 0x4,
-	HTT_T2H_MSG_TYPE_RX_ADDBA	= 0x5,
-	HTT_T2H_MSG_TYPE_PKTLOG		= 0x8,
-	HTT_T2H_MSG_TYPE_SEC_IND	= 0xb,
-	HTT_T2H_MSG_TYPE_PEER_MAP2	= 0x1e,
-	HTT_T2H_MSG_TYPE_PEER_UNMAP2	= 0x1f,
-	HTT_T2H_MSG_TYPE_PPDU_STATS_IND = 0x1d,
-	HTT_T2H_MSG_TYPE_EXT_STATS_CONF = 0x1c,
-	HTT_T2H_MSG_TYPE_BKPRESSURE_EVENT_IND = 0x24,
-	HTT_T2H_MSG_TYPE_MLO_TIMESTAMP_OFFSET_IND = 0x28,
-	HTT_T2H_MSG_TYPE_PEER_MAP3	= 0x2b,
-	HTT_T2H_MSG_TYPE_VDEV_TXRX_STATS_PERIODIC_IND = 0x2c,
-};
-
-#define HTT_TARGET_VERSION_MAJOR 3
-
-#define HTT_T2H_MSG_TYPE		GENMASK(7, 0)
-#define HTT_T2H_VERSION_CONF_MINOR	GENMASK(15, 8)
-#define HTT_T2H_VERSION_CONF_MAJOR	GENMASK(23, 16)
-
-struct htt_t2h_version_conf_msg {
-	__le32 version;
-} __packed;
-
-#define HTT_T2H_PEER_MAP_INFO_VDEV_ID	GENMASK(15, 8)
-#define HTT_T2H_PEER_MAP_INFO_PEER_ID	GENMASK(31, 16)
-#define HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16	GENMASK(15, 0)
-#define HTT_T2H_PEER_MAP_INFO1_HW_PEER_ID	GENMASK(31, 16)
-#define HTT_T2H_PEER_MAP_INFO2_AST_HASH_VAL	GENMASK(15, 0)
-#define HTT_T2H_PEER_MAP3_INFO2_HW_PEER_ID	GENMASK(15, 0)
-#define HTT_T2H_PEER_MAP3_INFO2_AST_HASH_VAL	GENMASK(31, 16)
-#define HTT_T2H_PEER_MAP_INFO2_NEXT_HOP_M	BIT(16)
-#define HTT_T2H_PEER_MAP_INFO2_NEXT_HOP_S	16
-
-struct htt_t2h_peer_map_event {
-	__le32 info;
-	__le32 mac_addr_l32;
-	__le32 info1;
-	__le32 info2;
-} __packed;
-
-#define HTT_T2H_PEER_UNMAP_INFO_VDEV_ID	HTT_T2H_PEER_MAP_INFO_VDEV_ID
-#define HTT_T2H_PEER_UNMAP_INFO_PEER_ID	HTT_T2H_PEER_MAP_INFO_PEER_ID
-#define HTT_T2H_PEER_UNMAP_INFO1_MAC_ADDR_H16 \
-					HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16
-#define HTT_T2H_PEER_MAP_INFO1_NEXT_HOP_M HTT_T2H_PEER_MAP_INFO2_NEXT_HOP_M
-#define HTT_T2H_PEER_MAP_INFO1_NEXT_HOP_S HTT_T2H_PEER_MAP_INFO2_NEXT_HOP_S
-
-struct htt_t2h_peer_unmap_event {
-	__le32 info;
-	__le32 mac_addr_l32;
-	__le32 info1;
-} __packed;
-
-struct htt_resp_msg {
-	union {
-		struct htt_t2h_version_conf_msg version_msg;
-		struct htt_t2h_peer_map_event peer_map_ev;
-		struct htt_t2h_peer_unmap_event peer_unmap_ev;
-	};
-} __packed;
-
-#define HTT_VDEV_GET_STATS_U64(msg_l32, msg_u32)\
-	(((u64)__le32_to_cpu(msg_u32) << 32) | (__le32_to_cpu(msg_l32)))
-#define HTT_T2H_VDEV_STATS_PERIODIC_MSG_TYPE		GENMASK(7, 0)
-#define HTT_T2H_VDEV_STATS_PERIODIC_PDEV_ID		GENMASK(15, 8)
-#define HTT_T2H_VDEV_STATS_PERIODIC_NUM_VDEV		GENMASK(23, 16)
-#define HTT_T2H_VDEV_STATS_PERIODIC_PAYLOAD_BYTES	GENMASK(15, 0)
-#define HTT_VDEV_TXRX_STATS_COMMON_TLV		0
-#define HTT_VDEV_TXRX_STATS_HW_STATS_TLV	1
-
-struct htt_t2h_vdev_txrx_stats_ind {
-	__le32 vdev_id;
-	__le32 rx_msdu_byte_cnt_lo;
-	__le32 rx_msdu_byte_cnt_hi;
-	__le32 rx_msdu_cnt_lo;
-	__le32 rx_msdu_cnt_hi;
-	__le32 tx_msdu_byte_cnt_lo;
-	__le32 tx_msdu_byte_cnt_hi;
-	__le32 tx_msdu_cnt_lo;
-	__le32 tx_msdu_cnt_hi;
-	__le32 tx_retry_cnt_lo;
-	__le32 tx_retry_cnt_hi;
-	__le32 tx_retry_byte_cnt_lo;
-	__le32 tx_retry_byte_cnt_hi;
-	__le32 tx_drop_cnt_lo;
-	__le32 tx_drop_cnt_hi;
-	__le32 tx_drop_byte_cnt_lo;
-	__le32 tx_drop_byte_cnt_hi;
-	__le32 msdu_ttl_cnt_lo;
-	__le32 msdu_ttl_cnt_hi;
-	__le32 msdu_ttl_byte_cnt_lo;
-	__le32 msdu_ttl_byte_cnt_hi;
-} __packed;
-
-struct htt_t2h_vdev_common_stats_tlv {
-	__le32 soc_drop_count_lo;
-	__le32 soc_drop_count_hi;
-} __packed;
-
-/* ppdu stats
- *
- * @details
- * The following field definitions describe the format of the HTT target
- * to host ppdu stats indication message.
- *
- *
- * |31                         16|15   12|11   10|9      8|7            0 |
- * |----------------------------------------------------------------------|
- * |    payload_size             | rsvd  |pdev_id|mac_id  |    msg type   |
- * |----------------------------------------------------------------------|
- * |                          ppdu_id                                     |
- * |----------------------------------------------------------------------|
- * |                        Timestamp in us                               |
- * |----------------------------------------------------------------------|
- * |                          reserved                                    |
- * |----------------------------------------------------------------------|
- * |                    type-specific stats info                          |
- * |                     (see htt_ppdu_stats.h)                           |
- * |----------------------------------------------------------------------|
- * Header fields:
- *  - MSG_TYPE
- *    Bits 7:0
- *    Purpose: Identifies this is a PPDU STATS indication
- *             message.
- *    Value: 0x1d
- *  - mac_id
- *    Bits 9:8
- *    Purpose: mac_id of this ppdu_id
- *    Value: 0-3
- *  - pdev_id
- *    Bits 11:10
- *    Purpose: pdev_id of this ppdu_id
- *    Value: 0-3
- *     0 (for rings at SOC level),
- *     1/2/3 PDEV -> 0/1/2
- *  - payload_size
- *    Bits 31:16
- *    Purpose: total tlv size
- *    Value: payload_size in bytes
- */
-
-#define HTT_T2H_PPDU_STATS_INFO_PDEV_ID GENMASK(11, 10)
-#define HTT_T2H_PPDU_STATS_INFO_PAYLOAD_SIZE GENMASK(31, 16)
-
-struct ath12k_htt_ppdu_stats_msg {
-	__le32 info;
-	__le32 ppdu_id;
-	__le32 timestamp;
-	__le32 rsvd;
-	u8 data[];
-} __packed;
-
-struct htt_tlv {
-	__le32 header;
-	u8 value[];
-} __packed;
-
-#define HTT_TLV_TAG			GENMASK(11, 0)
-#define HTT_TLV_LEN			GENMASK(23, 12)
-
-enum HTT_PPDU_STATS_BW {
-	HTT_PPDU_STATS_BANDWIDTH_5MHZ   = 0,
-	HTT_PPDU_STATS_BANDWIDTH_10MHZ  = 1,
-	HTT_PPDU_STATS_BANDWIDTH_20MHZ  = 2,
-	HTT_PPDU_STATS_BANDWIDTH_40MHZ  = 3,
-	HTT_PPDU_STATS_BANDWIDTH_80MHZ  = 4,
-	HTT_PPDU_STATS_BANDWIDTH_160MHZ = 5, /* includes 80+80 */
-	HTT_PPDU_STATS_BANDWIDTH_DYN    = 6,
-};
-
-#define HTT_PPDU_STATS_CMN_FLAGS_FRAME_TYPE_M	GENMASK(7, 0)
-#define HTT_PPDU_STATS_CMN_FLAGS_QUEUE_TYPE_M	GENMASK(15, 8)
-/* bw - HTT_PPDU_STATS_BW */
-#define HTT_PPDU_STATS_CMN_FLAGS_BW_M		GENMASK(19, 16)
-
-struct htt_ppdu_stats_common {
-	__le32 ppdu_id;
-	__le16 sched_cmdid;
-	u8 ring_id;
-	u8 num_users;
-	__le32 flags; /* %HTT_PPDU_STATS_COMMON_FLAGS_*/
-	__le32 chain_mask;
-	__le32 fes_duration_us; /* frame exchange sequence */
-	__le32 ppdu_sch_eval_start_tstmp_us;
-	__le32 ppdu_sch_end_tstmp_us;
-	__le32 ppdu_start_tstmp_us;
-	/* BIT [15 :  0] - phy mode (WLAN_PHY_MODE) with which ppdu was transmitted
-	 * BIT [31 : 16] - bandwidth (in MHz) with which ppdu was transmitted
+	const struct ath12k_hw_params *hw_params;
+	struct device *dev;
+	struct ath12k_hal *hal;
+
+	/* RCU on dp_pdevs[] provides a teardown synchronization mechanism,
+	 * ensuring in-flight data path readers complete before reclaim. Writers
+	 * update internal fields under their own synchronization, while readers of
+	 * internal fields may perform lockless read if occasional inconsistency
+	 * is acceptable or use additional synchronization for a coherent view.
+	 *
+	 * RCU is used for dp_pdevs[] at this stage to align with
+	 * ab->pdevs_active[]. However, if the teardown paths ensure quiescence,
+	 * both dp_pdevs[] and pdevs_active[] can be converted to plain pointers,
+	 * removing RCU synchronize overhead.
+	 *
+	 * TODO: evaluate removal of RCU from dp_pdevs in the future
 	 */
-	__le16 phy_mode;
-	__le16 bw_mhz;
-} __packed;
+	struct ath12k_pdev_dp __rcu *dp_pdevs[MAX_RADIOS];
 
-enum htt_ppdu_stats_gi {
-	HTT_PPDU_STATS_SGI_0_8_US,
-	HTT_PPDU_STATS_SGI_0_4_US,
-	HTT_PPDU_STATS_SGI_1_6_US,
-	HTT_PPDU_STATS_SGI_3_2_US,
-};
+	struct ath12k_hw_group *ag;
+	u8 device_id;
 
-#define HTT_PPDU_STATS_USER_RATE_INFO0_USER_POS_M	GENMASK(3, 0)
-#define HTT_PPDU_STATS_USER_RATE_INFO0_MU_GROUP_ID_M	GENMASK(11, 4)
-
-enum HTT_PPDU_STATS_PPDU_TYPE {
-	HTT_PPDU_STATS_PPDU_TYPE_SU,
-	HTT_PPDU_STATS_PPDU_TYPE_MU_MIMO,
-	HTT_PPDU_STATS_PPDU_TYPE_MU_OFDMA,
-	HTT_PPDU_STATS_PPDU_TYPE_MU_MIMO_OFDMA,
-	HTT_PPDU_STATS_PPDU_TYPE_UL_TRIG,
-	HTT_PPDU_STATS_PPDU_TYPE_BURST_BCN,
-	HTT_PPDU_STATS_PPDU_TYPE_UL_BSR_RESP,
-	HTT_PPDU_STATS_PPDU_TYPE_UL_BSR_TRIG,
-	HTT_PPDU_STATS_PPDU_TYPE_UL_RESP,
-	HTT_PPDU_STATS_PPDU_TYPE_MAX
-};
+	/* Lock for protection of peers and rhead_peer_addr */
+	spinlock_t dp_lock;
 
-#define HTT_PPDU_STATS_USER_RATE_INFO1_RESP_TYPE_VALD_M	BIT(0)
-#define HTT_PPDU_STATS_USER_RATE_INFO1_PPDU_TYPE_M	GENMASK(5, 1)
-
-#define HTT_PPDU_STATS_USER_RATE_FLAGS_LTF_SIZE_M	GENMASK(1, 0)
-#define HTT_PPDU_STATS_USER_RATE_FLAGS_STBC_M		BIT(2)
-#define HTT_PPDU_STATS_USER_RATE_FLAGS_HE_RE_M		BIT(3)
-#define HTT_PPDU_STATS_USER_RATE_FLAGS_TXBF_M		GENMASK(7, 4)
-#define HTT_PPDU_STATS_USER_RATE_FLAGS_BW_M		GENMASK(11, 8)
-#define HTT_PPDU_STATS_USER_RATE_FLAGS_NSS_M		GENMASK(15, 12)
-#define HTT_PPDU_STATS_USER_RATE_FLAGS_MCS_M		GENMASK(19, 16)
-#define HTT_PPDU_STATS_USER_RATE_FLAGS_PREAMBLE_M	GENMASK(23, 20)
-#define HTT_PPDU_STATS_USER_RATE_FLAGS_GI_M		GENMASK(27, 24)
-#define HTT_PPDU_STATS_USER_RATE_FLAGS_DCM_M		BIT(28)
-#define HTT_PPDU_STATS_USER_RATE_FLAGS_LDPC_M		BIT(29)
-
-#define HTT_USR_RATE_PPDU_TYPE(_val) \
-		le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_INFO1_PPDU_TYPE_M)
-#define HTT_USR_RATE_PREAMBLE(_val) \
-		le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_PREAMBLE_M)
-#define HTT_USR_RATE_BW(_val) \
-		le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_BW_M)
-#define HTT_USR_RATE_NSS(_val) \
-		le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_NSS_M)
-#define HTT_USR_RATE_MCS(_val) \
-		le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_MCS_M)
-#define HTT_USR_RATE_GI(_val) \
-		le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_GI_M)
-#define HTT_USR_RATE_DCM(_val) \
-		le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_DCM_M)
-
-#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_LTF_SIZE_M		GENMASK(1, 0)
-#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_STBC_M		BIT(2)
-#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_HE_RE_M		BIT(3)
-#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_TXBF_M		GENMASK(7, 4)
-#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_BW_M		GENMASK(11, 8)
-#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_NSS_M		GENMASK(15, 12)
-#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_MCS_M		GENMASK(19, 16)
-#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_PREAMBLE_M		GENMASK(23, 20)
-#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_GI_M		GENMASK(27, 24)
-#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_DCM_M		BIT(28)
-#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_LDPC_M		BIT(29)
-
-struct htt_ppdu_stats_user_rate {
-	u8 tid_num;
-	u8 reserved0;
-	__le16 sw_peer_id;
-	__le32 info0; /* %HTT_PPDU_STATS_USER_RATE_INFO0_*/
-	__le16 ru_end;
-	__le16 ru_start;
-	__le16 resp_ru_end;
-	__le16 resp_ru_start;
-	__le32 info1; /* %HTT_PPDU_STATS_USER_RATE_INFO1_ */
-	__le32 rate_flags; /* %HTT_PPDU_STATS_USER_RATE_FLAGS_ */
-	/* Note: resp_rate_info is only valid for if resp_type is UL */
-	__le32 resp_rate_flags; /* %HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_ */
-} __packed;
+	struct ath12k_dp_arch_ops *ops;
 
-#define HTT_PPDU_STATS_TX_INFO_FLAGS_RATECODE_M		GENMASK(7, 0)
-#define HTT_PPDU_STATS_TX_INFO_FLAGS_IS_AMPDU_M		BIT(8)
-#define HTT_PPDU_STATS_TX_INFO_FLAGS_BA_ACK_FAILED_M	GENMASK(10, 9)
-#define HTT_PPDU_STATS_TX_INFO_FLAGS_BW_M		GENMASK(13, 11)
-#define HTT_PPDU_STATS_TX_INFO_FLAGS_SGI_M		BIT(14)
-#define HTT_PPDU_STATS_TX_INFO_FLAGS_PEERID_M		GENMASK(31, 16)
-
-#define HTT_TX_INFO_IS_AMSDU(_flags) \
-			u32_get_bits(_flags, HTT_PPDU_STATS_TX_INFO_FLAGS_IS_AMPDU_M)
-#define HTT_TX_INFO_BA_ACK_FAILED(_flags) \
-			u32_get_bits(_flags, HTT_PPDU_STATS_TX_INFO_FLAGS_BA_ACK_FAILED_M)
-#define HTT_TX_INFO_RATECODE(_flags) \
-			u32_get_bits(_flags, HTT_PPDU_STATS_TX_INFO_FLAGS_RATECODE_M)
-#define HTT_TX_INFO_PEERID(_flags) \
-			u32_get_bits(_flags, HTT_PPDU_STATS_TX_INFO_FLAGS_PEERID_M)
-
-enum  htt_ppdu_stats_usr_compln_status {
-	HTT_PPDU_STATS_USER_STATUS_OK,
-	HTT_PPDU_STATS_USER_STATUS_FILTERED,
-	HTT_PPDU_STATS_USER_STATUS_RESP_TIMEOUT,
-	HTT_PPDU_STATS_USER_STATUS_RESP_MISMATCH,
-	HTT_PPDU_STATS_USER_STATUS_ABORT,
-};
-
-#define HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_LONG_RETRY_M	GENMASK(3, 0)
-#define HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_SHORT_RETRY_M	GENMASK(7, 4)
-#define HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_IS_AMPDU_M		BIT(8)
-#define HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_RESP_TYPE_M		GENMASK(12, 9)
-
-#define HTT_USR_CMPLTN_IS_AMPDU(_val) \
-	    le32_get_bits(_val, HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_IS_AMPDU_M)
-#define HTT_USR_CMPLTN_LONG_RETRY(_val) \
-	    le32_get_bits(_val, HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_LONG_RETRY_M)
-#define HTT_USR_CMPLTN_SHORT_RETRY(_val) \
-	    le32_get_bits(_val, HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_SHORT_RETRY_M)
-
-struct htt_ppdu_stats_usr_cmpltn_cmn {
-	u8 status;
-	u8 tid_num;
-	__le16 sw_peer_id;
-	/* RSSI value of last ack packet (units = dB above noise floor) */
-	__le32 ack_rssi;
-	__le16 mpdu_tried;
-	__le16 mpdu_success;
-	__le32 flags; /* %HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_LONG_RETRIES*/
-} __packed;
+	/* Linked list of struct ath12k_dp_link_peer */
+	struct list_head peers;
 
-#define HTT_PPDU_STATS_ACK_BA_INFO_NUM_MPDU_M	GENMASK(8, 0)
-#define HTT_PPDU_STATS_ACK_BA_INFO_NUM_MSDU_M	GENMASK(24, 9)
-#define HTT_PPDU_STATS_ACK_BA_INFO_TID_NUM	GENMASK(31, 25)
+	/* For rhash table init and deinit protection */
+	struct mutex link_peer_rhash_tbl_lock;
 
-#define HTT_PPDU_STATS_NON_QOS_TID	16
-
-struct htt_ppdu_stats_usr_cmpltn_ack_ba_status {
-	__le32 ppdu_id;
-	__le16 sw_peer_id;
-	__le16 reserved0;
-	__le32 info; /* %HTT_PPDU_STATS_USR_CMPLTN_CMN_INFO_ */
-	__le16 current_seq;
-	__le16 start_seq;
-	__le32 success_bytes;
-} __packed;
-
-struct htt_ppdu_user_stats {
-	u16 peer_id;
-	u16 delay_ba;
-	u32 tlv_flags;
-	bool is_valid_peer_id;
-	struct htt_ppdu_stats_user_rate rate;
-	struct htt_ppdu_stats_usr_cmpltn_cmn cmpltn_cmn;
-	struct htt_ppdu_stats_usr_cmpltn_ack_ba_status ack_ba;
+	/* The rhashtable containing struct ath12k_link_peer keyed by mac addr */
+	struct rhashtable *rhead_peer_addr;
+	struct rhashtable_params rhash_peer_addr_param;
+	struct ath12k_device_dp_stats device_stats;
 };
 
-#define HTT_PPDU_STATS_MAX_USERS	8
-#define HTT_PPDU_DESC_MAX_DEPTH	16
+static inline u32 ath12k_dp_arch_tx_get_vdev_bank_config(struct ath12k_dp *dp,
+							 struct ath12k_link_vif *arvif)
+{
+	return dp->ops->tx_get_vdev_bank_config(dp->ab, arvif);
+}
 
-struct htt_ppdu_stats {
-	struct htt_ppdu_stats_common common;
-	struct htt_ppdu_user_stats user_stats[HTT_PPDU_STATS_MAX_USERS];
-};
+static inline int ath12k_dp_arch_reo_cmd_send(struct ath12k_dp *dp,
+					      struct ath12k_dp_rx_tid_rxq *rx_tid,
+					      enum hal_reo_cmd_type type,
+					      struct ath12k_hal_reo_cmd *cmd,
+					      void (*cb)(struct ath12k_dp *dp, void *ctx,
+							 enum hal_reo_cmd_status status))
+{
+	return dp->ops->reo_cmd_send(dp->ab, rx_tid, type, cmd, cb);
+}
 
-struct htt_ppdu_stats_info {
-	u32 tlv_bitmap;
-	u32 ppdu_id;
-	u32 frame_type;
-	u32 frame_ctrl;
-	u32 delay_ba;
-	u32 bar_num_users;
-	struct htt_ppdu_stats ppdu_stats;
-	struct list_head list;
-};
+static inline
+void ath12k_dp_arch_setup_pn_check_reo_cmd(struct ath12k_dp *dp,
+					   struct ath12k_hal_reo_cmd *cmd,
+					   struct ath12k_dp_rx_tid *rx_tid,
+					   u32 cipher,
+					   enum set_key_cmd key_cmd)
+{
+	dp->ops->setup_pn_check_reo_cmd(cmd, rx_tid, cipher, key_cmd);
+}
 
-/* @brief target -> host MLO offset indiciation message
- *
- * @details
- * The following field definitions describe the format of the HTT target
- * to host mlo offset indication message.
- *
- *
- * |31        29|28    |26|25  22|21 16|15  13|12     10 |9     8|7     0|
- * |---------------------------------------------------------------------|
- * |   rsvd1    | mac_freq                    |chip_id   |pdev_id|msgtype|
- * |---------------------------------------------------------------------|
- * |                           sync_timestamp_lo_us                      |
- * |---------------------------------------------------------------------|
- * |                           sync_timestamp_hi_us                      |
- * |---------------------------------------------------------------------|
- * |                           mlo_offset_lo                             |
- * |---------------------------------------------------------------------|
- * |                           mlo_offset_hi                             |
- * |---------------------------------------------------------------------|
- * |                           mlo_offset_clcks                          |
- * |---------------------------------------------------------------------|
- * |   rsvd2           | mlo_comp_clks |mlo_comp_us                      |
- * |---------------------------------------------------------------------|
- * |   rsvd3                   |mlo_comp_timer                           |
- * |---------------------------------------------------------------------|
- * Header fields
- *  - MSG_TYPE
- *    Bits 7:0
- *    Purpose: Identifies this is a MLO offset indication msg
- *  - PDEV_ID
- *    Bits 9:8
- *    Purpose: Pdev of this MLO offset
- *  - CHIP_ID
- *    Bits 12:10
- *    Purpose: chip_id of this MLO offset
- *  - MAC_FREQ
- *    Bits 28:13
- *  - SYNC_TIMESTAMP_LO_US
- *    Purpose: clock frequency of the mac HW block in MHz
- *    Bits: 31:0
- *    Purpose: lower 32 bits of the WLAN global time stamp at which
- *             last sync interrupt was received
- *  - SYNC_TIMESTAMP_HI_US
- *    Bits: 31:0
- *    Purpose: upper 32 bits of WLAN global time stamp at which
- *             last sync interrupt was received
- *  - MLO_OFFSET_LO
- *    Bits: 31:0
- *    Purpose: lower 32 bits of the MLO offset in us
- *  - MLO_OFFSET_HI
- *    Bits: 31:0
- *    Purpose: upper 32 bits of the MLO offset in us
- *  - MLO_COMP_US
- *    Bits: 15:0
- *    Purpose: MLO time stamp compensation applied in us
- *  - MLO_COMP_CLCKS
- *    Bits: 25:16
- *    Purpose: MLO time stamp compensation applied in clock ticks
- *  - MLO_COMP_TIMER
- *    Bits: 21:0
- *    Purpose: Periodic timer at which compensation is applied
- */
+static inline void ath12k_dp_arch_rx_peer_tid_delete(struct ath12k_dp *dp,
+						     struct ath12k_dp_link_peer *peer,
+						     u8 tid)
+{
+	dp->ops->rx_peer_tid_delete(dp->ab, peer, tid);
+}
 
-#define HTT_T2H_MLO_OFFSET_INFO_MSG_TYPE        GENMASK(7, 0)
-#define HTT_T2H_MLO_OFFSET_INFO_PDEV_ID         GENMASK(9, 8)
-
-struct ath12k_htt_mlo_offset_msg {
-	__le32 info;
-	__le32 sync_timestamp_lo_us;
-	__le32 sync_timestamp_hi_us;
-	__le32 mlo_offset_hi;
-	__le32 mlo_offset_lo;
-	__le32 mlo_offset_clks;
-	__le32 mlo_comp_clks;
-	__le32 mlo_comp_timer;
-} __packed;
+static inline int ath12k_dp_arch_reo_cache_flush(struct ath12k_dp *dp,
+						 struct ath12k_dp_rx_tid_rxq *rx_tid)
+{
+	return dp->ops->reo_cache_flush(dp->ab, rx_tid);
+}
 
-/* @brief host -> target FW extended statistics retrieve
- *
- * @details
- * The following field definitions describe the format of the HTT host
- * to target FW extended stats retrieve message.
- * The message specifies the type of stats the host wants to retrieve.
- *
- * |31          24|23          16|15           8|7            0|
- * |-----------------------------------------------------------|
- * |   reserved   | stats type   |   pdev_mask  |   msg type   |
- * |-----------------------------------------------------------|
- * |                   config param [0]                        |
- * |-----------------------------------------------------------|
- * |                   config param [1]                        |
- * |-----------------------------------------------------------|
- * |                   config param [2]                        |
- * |-----------------------------------------------------------|
- * |                   config param [3]                        |
- * |-----------------------------------------------------------|
- * |                         reserved                          |
- * |-----------------------------------------------------------|
- * |                        cookie LSBs                        |
- * |-----------------------------------------------------------|
- * |                        cookie MSBs                        |
- * |-----------------------------------------------------------|
- * Header fields:
- *  - MSG_TYPE
- *    Bits 7:0
- *    Purpose: identifies this is a extended stats upload request message
- *    Value: 0x10
- *  - PDEV_MASK
- *    Bits 8:15
- *    Purpose: identifies the mask of PDEVs to retrieve stats from
- *    Value: This is a overloaded field, refer to usage and interpretation of
- *           PDEV in interface document.
- *           Bit   8    :  Reserved for SOC stats
- *           Bit 9 - 15 :  Indicates PDEV_MASK in DBDC
- *                         Indicates MACID_MASK in DBS
- *  - STATS_TYPE
- *    Bits 23:16
- *    Purpose: identifies which FW statistics to upload
- *    Value: Defined by htt_dbg_ext_stats_type (see htt_stats.h)
- *  - Reserved
- *    Bits 31:24
- *  - CONFIG_PARAM [0]
- *    Bits 31:0
- *    Purpose: give an opaque configuration value to the specified stats type
- *    Value: stats-type specific configuration value
- *           Refer to htt_stats.h for interpretation for each stats sub_type
- *  - CONFIG_PARAM [1]
- *    Bits 31:0
- *    Purpose: give an opaque configuration value to the specified stats type
- *    Value: stats-type specific configuration value
- *           Refer to htt_stats.h for interpretation for each stats sub_type
- *  - CONFIG_PARAM [2]
- *    Bits 31:0
- *    Purpose: give an opaque configuration value to the specified stats type
- *    Value: stats-type specific configuration value
- *           Refer to htt_stats.h for interpretation for each stats sub_type
- *  - CONFIG_PARAM [3]
- *    Bits 31:0
- *    Purpose: give an opaque configuration value to the specified stats type
- *    Value: stats-type specific configuration value
- *           Refer to htt_stats.h for interpretation for each stats sub_type
- *  - Reserved [31:0] for future use.
- *  - COOKIE_LSBS
- *    Bits 31:0
- *    Purpose: Provide a mechanism to match a target->host stats confirmation
- *        message with its preceding host->target stats request message.
- *    Value: LSBs of the opaque cookie specified by the host-side requestor
- *  - COOKIE_MSBS
- *    Bits 31:0
- *    Purpose: Provide a mechanism to match a target->host stats confirmation
- *        message with its preceding host->target stats request message.
- *    Value: MSBs of the opaque cookie specified by the host-side requestor
- */
+static inline
+int ath12k_dp_arch_rx_link_desc_return(struct ath12k_dp *dp,
+				       struct ath12k_buffer_addr *buf_addr_info,
+				       enum hal_wbm_rel_bm_act action)
+{
+	return dp->ops->rx_link_desc_return(dp, buf_addr_info, action);
+}
 
-struct htt_ext_stats_cfg_hdr {
-	u8 msg_type;
-	u8 pdev_mask;
-	u8 stats_type;
-	u8 reserved;
-} __packed;
+static inline
+void ath12k_dp_arch_rx_frags_cleanup(struct ath12k_dp *dp,
+				     struct ath12k_dp_rx_tid *rx_tid,
+				     bool rel_link_desc)
+{
+	dp->ops->rx_frags_cleanup(rx_tid, rel_link_desc);
+}
 
-struct htt_ext_stats_cfg_cmd {
-	struct htt_ext_stats_cfg_hdr hdr;
-	__le32 cfg_param0;
-	__le32 cfg_param1;
-	__le32 cfg_param2;
-	__le32 cfg_param3;
-	__le32 reserved;
-	__le32 cookie_lsb;
-	__le32 cookie_msb;
-} __packed;
+static inline int ath12k_dp_arch_peer_rx_tid_reo_update(struct ath12k_dp *dp,
+							struct ath12k_dp_link_peer *peer,
+							struct ath12k_dp_rx_tid *rx_tid,
+							u32 ba_win_sz, u16 ssn,
+							bool update_ssn)
+{
+	return dp->ops->peer_rx_tid_reo_update(dp, peer, rx_tid,
+					       ba_win_sz, ssn, update_ssn);
+}
 
-/* htt stats config default params */
-#define HTT_STAT_DEFAULT_RESET_START_OFFSET 0
-#define HTT_STAT_DEFAULT_CFG0_ALL_HWQS 0xffffffff
-#define HTT_STAT_DEFAULT_CFG0_ALL_TXQS 0xffffffff
-#define HTT_STAT_DEFAULT_CFG0_ALL_CMDQS 0xffff
-#define HTT_STAT_DEFAULT_CFG0_ALL_RINGS 0xffff
-#define HTT_STAT_DEFAULT_CFG0_ACTIVE_PEERS 0xff
-#define HTT_STAT_DEFAULT_CFG0_CCA_CUMULATIVE 0x00
-#define HTT_STAT_DEFAULT_CFG0_ACTIVE_VDEVS 0x00
-
-/* HTT_DBG_EXT_STATS_PEER_INFO
- * PARAMS:
- * @config_param0:
- *  [Bit0] - [0] for sw_peer_id, [1] for mac_addr based request
- *  [Bit15 : Bit 1] htt_peer_stats_req_mode_t
- *  [Bit31 : Bit16] sw_peer_id
- * @config_param1:
- *  peer_stats_req_type_mask:32 (enum htt_peer_stats_tlv_enum)
- *   0 bit htt_peer_stats_cmn_tlv
- *   1 bit htt_peer_details_tlv
- *   2 bit htt_tx_peer_rate_stats_tlv
- *   3 bit htt_rx_peer_rate_stats_tlv
- *   4 bit htt_tx_tid_stats_tlv/htt_tx_tid_stats_v1_tlv
- *   5 bit htt_rx_tid_stats_tlv
- *   6 bit htt_msdu_flow_stats_tlv
- * @config_param2: [Bit31 : Bit0] mac_addr31to0
- * @config_param3: [Bit15 : Bit0] mac_addr47to32
- *                [Bit31 : Bit16] reserved
- */
-#define HTT_STAT_PEER_INFO_MAC_ADDR BIT(0)
-#define HTT_STAT_DEFAULT_PEER_REQ_TYPE 0x7f
-
-/* Used to set different configs to the specified stats type.*/
-struct htt_ext_stats_cfg_params {
-	u32 cfg0;
-	u32 cfg1;
-	u32 cfg2;
-	u32 cfg3;
-};
+static inline int ath12k_dp_arch_rx_assign_reoq(struct ath12k_dp *dp,
+						struct ath12k_dp_peer *dp_peer,
+						struct ath12k_dp_rx_tid *rx_tid,
+						u16 ssn, enum hal_pn_type pn_type)
+{
+	return dp->ops->rx_assign_reoq(dp->ab, dp_peer, rx_tid, ssn, pn_type);
+}
 
-enum vdev_stats_offload_timer_duration {
-	ATH12K_STATS_TIMER_DUR_500MS = 1,
-	ATH12K_STATS_TIMER_DUR_1SEC = 2,
-	ATH12K_STATS_TIMER_DUR_2SEC = 3,
-};
+static inline void ath12k_dp_arch_peer_rx_tid_qref_setup(struct ath12k_dp *dp,
+							 u16 peer_id, u16 tid,
+							 dma_addr_t paddr)
+{
+	dp->ops->peer_rx_tid_qref_setup(dp->ab, peer_id, tid, paddr);
+}
 
-#define ATH12K_HTT_MAC_ADDR_L32_0	GENMASK(7, 0)
-#define ATH12K_HTT_MAC_ADDR_L32_1	GENMASK(15, 8)
-#define ATH12K_HTT_MAC_ADDR_L32_2	GENMASK(23, 16)
-#define ATH12K_HTT_MAC_ADDR_L32_3	GENMASK(31, 24)
-#define ATH12K_HTT_MAC_ADDR_H16_0	GENMASK(7, 0)
-#define ATH12K_HTT_MAC_ADDR_H16_1	GENMASK(15, 8)
+static inline void ath12k_dp_arch_peer_rx_tid_qref_reset(struct ath12k_dp *dp,
+							 u16 peer_id, u16 tid)
+{
+	dp->ops->peer_rx_tid_qref_reset(dp->ab, peer_id, tid);
+}
 
-struct htt_mac_addr {
-	__le32 mac_addr_l32;
-	__le32 mac_addr_h16;
-} __packed;
+static inline
+int ath12k_dp_arch_rx_tid_delete_handler(struct ath12k_dp *dp,
+					 struct ath12k_dp_rx_tid_rxq *rx_tid)
+{
+	return dp->ops->rx_tid_delete_handler(dp->ab, rx_tid);
+}
 
 static inline void ath12k_dp_get_mac_addr(u32 addr_l32, u16 addr_h16, u8 *addr)
 {
@@ -1940,20 +651,39 @@ static inline void ath12k_dp_get_mac_addr(u32 addr_l32, u16 addr_h16, u8 *addr)
 	memcpy(addr + 4, &addr_h16, ETH_ALEN - 4);
 }
 
-int ath12k_dp_service_srng(struct ath12k_base *ab,
-			   struct ath12k_ext_irq_grp *irq_grp,
-			   int budget);
-int ath12k_dp_htt_connect(struct ath12k_dp *dp);
+static inline struct ath12k_dp *
+ath12k_dp_hw_grp_to_dp(struct ath12k_dp_hw_group *dp_hw_grp, u8 device_id)
+{
+	return dp_hw_grp->dp[device_id];
+}
+
+static inline int
+ath12k_dp_service_srng(struct ath12k_dp *dp, struct ath12k_ext_irq_grp *irq_grp,
+		       int budget)
+{
+	return dp->ops->service_srng(dp, irq_grp, budget);
+}
+
+static inline struct ieee80211_hw *
+ath12k_pdev_dp_to_hw(struct ath12k_pdev_dp *pdev)
+{
+	return pdev->hw;
+}
+
+static inline struct ath12k_pdev_dp *
+ath12k_dp_to_pdev_dp(struct ath12k_dp *dp, u8 pdev_idx)
+{
+	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
+			 "ath12k dp to dp pdev called without rcu lock");
+
+	return rcu_dereference(dp->dp_pdevs[pdev_idx]);
+}
+
 void ath12k_dp_vdev_tx_attach(struct ath12k *ar, struct ath12k_link_vif *arvif);
-void ath12k_dp_free(struct ath12k_base *ab);
-int ath12k_dp_alloc(struct ath12k_base *ab);
-void ath12k_dp_cc_config(struct ath12k_base *ab);
 void ath12k_dp_partner_cc_init(struct ath12k_base *ab);
 int ath12k_dp_pdev_alloc(struct ath12k_base *ab);
 void ath12k_dp_pdev_pre_alloc(struct ath12k *ar);
 void ath12k_dp_pdev_free(struct ath12k_base *ab);
-int ath12k_dp_tx_htt_srng_setup(struct ath12k_base *ab, u32 ring_id,
-				int mac_id, enum hal_ring_type ring_type);
 int ath12k_dp_peer_setup(struct ath12k *ar, int vdev_id, const u8 *addr);
 void ath12k_dp_peer_cleanup(struct ath12k *ar, int vdev_id, const u8 *addr);
 void ath12k_dp_srng_cleanup(struct ath12k_base *ab, struct dp_srng *ring);
@@ -1967,10 +697,8 @@ int ath12k_dp_link_desc_setup(struct ath12k_base *ab,
 			      struct dp_link_desc_bank *link_desc_banks,
 			      u32 ring_type, struct hal_srng *srng,
 			      u32 n_link_desc);
-struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_base *ab,
+struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_dp *dp,
 						  u32 cookie);
-struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_base *ab,
+struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_dp *dp,
 						  u32 desc_id);
-bool ath12k_dp_wmask_compaction_rx_tlv_supported(struct ath12k_base *ab);
-void ath12k_dp_hal_rx_desc_init(struct ath12k_base *ab);
 #endif
diff --git a/drivers/net/wireless/ath/ath12k/dp_cmn.h b/drivers/net/wireless/ath/ath12k/dp_cmn.h
new file mode 100644
index 000000000000..e17f044ff812
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/dp_cmn.h
@@ -0,0 +1,106 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef ATH12K_DP_CMN_H
+#define ATH12K_DP_CMN_H
+
+#include "cmn_defs.h"
+
+struct ath12k_hw_group;
+
+/*
+ * ML Peer IDs start from 8192, assuming max SLO clients count 1536,
+ * then max peer id shall be 9728, therefore rounding the peer table size
+ * to the nearest next power of 2 i.e 16384.
+ */
+#define MAX_DP_PEER_LIST_SIZE  16384
+
+struct ath12k_dp_hw {
+	struct ath12k_dp_peer __rcu *dp_peers[MAX_DP_PEER_LIST_SIZE];
+
+	/* Lock for protection of dp_peer_list and peers */
+	spinlock_t peer_lock;
+	struct list_head dp_peers_list;
+};
+
+struct ath12k_dp_hw_group {
+	struct ath12k_dp *dp[ATH12K_MAX_DEVICES];
+};
+
+struct ath12k_dp_link_vif {
+	u32 vdev_id;
+	u8 search_type;
+	u8 hal_addr_search_flags;
+	u8 pdev_idx;
+	u8 lmac_id;
+	u16 ast_idx;
+	u16 ast_hash;
+	u16 tcl_metadata;
+	u8 vdev_id_check_en;
+	int bank_id;
+};
+
+struct ath12k_dp_vif {
+	u8 tx_encap_type;
+	u32 key_cipher;
+	atomic_t mcbc_gsn;
+	struct ath12k_dp_link_vif dp_link_vif[ATH12K_NUM_MAX_LINKS];
+};
+
+/* TODO: Move this to a separate dp_stats file */
+struct ath12k_per_peer_tx_stats {
+	u32 succ_bytes;
+	u32 retry_bytes;
+	u32 failed_bytes;
+	u32 duration;
+	u16 succ_pkts;
+	u16 retry_pkts;
+	u16 failed_pkts;
+	u16 ru_start;
+	u16 ru_tones;
+	u8 ba_fails;
+	u8 ppdu_type;
+	u32 mu_grpid;
+	u32 mu_pos;
+	bool is_ampdu;
+};
+
+struct ath12k_dp_peer_create_params {
+	struct ieee80211_sta *sta;
+	bool is_mlo;
+	u16 peer_id;
+	bool ucast_ra_only;
+};
+
+struct ath12k_dp_link_peer_rate_info {
+	struct rate_info txrate;
+	u64 rx_duration;
+	u64 tx_duration;
+	u8 rssi_comb;
+	s8 signal_avg;
+};
+
+static inline struct ath12k_dp_link_vif *
+ath12k_dp_vif_to_dp_link_vif(struct ath12k_dp_vif *dp_vif, u8 link_id)
+{
+	return &dp_vif->dp_link_vif[link_id];
+}
+
+void ath12k_dp_cmn_device_deinit(struct ath12k_dp *dp);
+int ath12k_dp_cmn_device_init(struct ath12k_dp *dp);
+void ath12k_dp_cmn_hw_group_unassign(struct ath12k_dp *dp,
+				     struct ath12k_hw_group *ag);
+void ath12k_dp_cmn_hw_group_assign(struct ath12k_dp *dp,
+				   struct ath12k_hw_group *ag);
+int ath12k_dp_link_peer_assign(struct ath12k_dp *dp, struct ath12k_dp_hw *dp_hw,
+			       u8 vdev_id, struct ieee80211_sta *sta, u8 *addr,
+			       u8 link_id, u32 hw_link_id);
+void ath12k_dp_link_peer_unassign(struct ath12k_dp *dp, struct ath12k_dp_hw *dp_hw,
+				  u8 vdev_id, u8 *addr, u32 hw_link_id);
+void
+ath12k_dp_link_peer_get_sta_rate_info_stats(struct ath12k_dp *dp, const u8 *addr,
+					    struct ath12k_dp_link_peer_rate_info *info);
+void ath12k_dp_link_peer_reset_rx_stats(struct ath12k_dp *dp, const u8 *addr);
+#endif
diff --git a/drivers/net/wireless/ath/ath12k/dp_htt.c b/drivers/net/wireless/ath/ath12k/dp_htt.c
new file mode 100644
index 000000000000..cc71c5c5de5a
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/dp_htt.c
@@ -0,0 +1,1353 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include "core.h"
+#include "peer.h"
+#include "htc.h"
+#include "dp_htt.h"
+#include "debugfs_htt_stats.h"
+#include "debugfs.h"
+
+static void ath12k_dp_htt_htc_tx_complete(struct ath12k_base *ab,
+					  struct sk_buff *skb)
+{
+	dev_kfree_skb_any(skb);
+}
+
+int ath12k_dp_htt_connect(struct ath12k_dp *dp)
+{
+	struct ath12k_htc_svc_conn_req conn_req = {};
+	struct ath12k_htc_svc_conn_resp conn_resp = {};
+	int status;
+
+	conn_req.ep_ops.ep_tx_complete = ath12k_dp_htt_htc_tx_complete;
+	conn_req.ep_ops.ep_rx_complete = ath12k_dp_htt_htc_t2h_msg_handler;
+
+	/* connect to control service */
+	conn_req.service_id = ATH12K_HTC_SVC_ID_HTT_DATA_MSG;
+
+	status = ath12k_htc_connect_service(&dp->ab->htc, &conn_req,
+					    &conn_resp);
+
+	if (status)
+		return status;
+
+	dp->eid = conn_resp.eid;
+
+	return 0;
+}
+
+static int ath12k_get_ppdu_user_index(struct htt_ppdu_stats *ppdu_stats,
+				      u16 peer_id)
+{
+	int i;
+
+	for (i = 0; i < HTT_PPDU_STATS_MAX_USERS - 1; i++) {
+		if (ppdu_stats->user_stats[i].is_valid_peer_id) {
+			if (peer_id == ppdu_stats->user_stats[i].peer_id)
+				return i;
+		} else {
+			return i;
+		}
+	}
+
+	return -EINVAL;
+}
+
+static int ath12k_htt_tlv_ppdu_stats_parse(struct ath12k_base *ab,
+					   u16 tag, u16 len, const void *ptr,
+					   void *data)
+{
+	const struct htt_ppdu_stats_usr_cmpltn_ack_ba_status *ba_status;
+	const struct htt_ppdu_stats_usr_cmpltn_cmn *cmplt_cmn;
+	const struct htt_ppdu_stats_user_rate *user_rate;
+	struct htt_ppdu_stats_info *ppdu_info;
+	struct htt_ppdu_user_stats *user_stats;
+	int cur_user;
+	u16 peer_id;
+
+	ppdu_info = data;
+
+	switch (tag) {
+	case HTT_PPDU_STATS_TAG_COMMON:
+		if (len < sizeof(struct htt_ppdu_stats_common)) {
+			ath12k_warn(ab, "Invalid len %d for the tag 0x%x\n",
+				    len, tag);
+			return -EINVAL;
+		}
+		memcpy(&ppdu_info->ppdu_stats.common, ptr,
+		       sizeof(struct htt_ppdu_stats_common));
+		break;
+	case HTT_PPDU_STATS_TAG_USR_RATE:
+		if (len < sizeof(struct htt_ppdu_stats_user_rate)) {
+			ath12k_warn(ab, "Invalid len %d for the tag 0x%x\n",
+				    len, tag);
+			return -EINVAL;
+		}
+		user_rate = ptr;
+		peer_id = le16_to_cpu(user_rate->sw_peer_id);
+		cur_user = ath12k_get_ppdu_user_index(&ppdu_info->ppdu_stats,
+						      peer_id);
+		if (cur_user < 0)
+			return -EINVAL;
+		user_stats = &ppdu_info->ppdu_stats.user_stats[cur_user];
+		user_stats->peer_id = peer_id;
+		user_stats->is_valid_peer_id = true;
+		memcpy(&user_stats->rate, ptr,
+		       sizeof(struct htt_ppdu_stats_user_rate));
+		user_stats->tlv_flags |= BIT(tag);
+		break;
+	case HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON:
+		if (len < sizeof(struct htt_ppdu_stats_usr_cmpltn_cmn)) {
+			ath12k_warn(ab, "Invalid len %d for the tag 0x%x\n",
+				    len, tag);
+			return -EINVAL;
+		}
+
+		cmplt_cmn = ptr;
+		peer_id = le16_to_cpu(cmplt_cmn->sw_peer_id);
+		cur_user = ath12k_get_ppdu_user_index(&ppdu_info->ppdu_stats,
+						      peer_id);
+		if (cur_user < 0)
+			return -EINVAL;
+		user_stats = &ppdu_info->ppdu_stats.user_stats[cur_user];
+		user_stats->peer_id = peer_id;
+		user_stats->is_valid_peer_id = true;
+		memcpy(&user_stats->cmpltn_cmn, ptr,
+		       sizeof(struct htt_ppdu_stats_usr_cmpltn_cmn));
+		user_stats->tlv_flags |= BIT(tag);
+		break;
+	case HTT_PPDU_STATS_TAG_USR_COMPLTN_ACK_BA_STATUS:
+		if (len <
+		    sizeof(struct htt_ppdu_stats_usr_cmpltn_ack_ba_status)) {
+			ath12k_warn(ab, "Invalid len %d for the tag 0x%x\n",
+				    len, tag);
+			return -EINVAL;
+		}
+
+		ba_status = ptr;
+		peer_id = le16_to_cpu(ba_status->sw_peer_id);
+		cur_user = ath12k_get_ppdu_user_index(&ppdu_info->ppdu_stats,
+						      peer_id);
+		if (cur_user < 0)
+			return -EINVAL;
+		user_stats = &ppdu_info->ppdu_stats.user_stats[cur_user];
+		user_stats->peer_id = peer_id;
+		user_stats->is_valid_peer_id = true;
+		memcpy(&user_stats->ack_ba, ptr,
+		       sizeof(struct htt_ppdu_stats_usr_cmpltn_ack_ba_status));
+		user_stats->tlv_flags |= BIT(tag);
+		break;
+	}
+	return 0;
+}
+
+int ath12k_dp_htt_tlv_iter(struct ath12k_base *ab, const void *ptr, size_t len,
+			   int (*iter)(struct ath12k_base *ar, u16 tag, u16 len,
+				       const void *ptr, void *data),
+			   void *data)
+{
+	const struct htt_tlv *tlv;
+	const void *begin = ptr;
+	u16 tlv_tag, tlv_len;
+	int ret = -EINVAL;
+
+	while (len > 0) {
+		if (len < sizeof(*tlv)) {
+			ath12k_err(ab, "htt tlv parse failure at byte %zd (%zu bytes left, %zu expected)\n",
+				   ptr - begin, len, sizeof(*tlv));
+			return -EINVAL;
+		}
+		tlv = (struct htt_tlv *)ptr;
+		tlv_tag = le32_get_bits(tlv->header, HTT_TLV_TAG);
+		tlv_len = le32_get_bits(tlv->header, HTT_TLV_LEN);
+		ptr += sizeof(*tlv);
+		len -= sizeof(*tlv);
+
+		if (tlv_len > len) {
+			ath12k_err(ab, "htt tlv parse failure of tag %u at byte %zd (%zu bytes left, %u expected)\n",
+				   tlv_tag, ptr - begin, len, tlv_len);
+			return -EINVAL;
+		}
+		ret = iter(ab, tlv_tag, tlv_len, ptr, data);
+		if (ret == -ENOMEM)
+			return ret;
+
+		ptr += tlv_len;
+		len -= tlv_len;
+	}
+	return 0;
+}
+
+static void
+ath12k_update_per_peer_tx_stats(struct ath12k_pdev_dp *dp_pdev,
+				struct htt_ppdu_stats *ppdu_stats, u8 user)
+{
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_base *ab = dp->ab;
+	struct ath12k_dp_link_peer *peer;
+	struct htt_ppdu_stats_user_rate *user_rate;
+	struct ath12k_per_peer_tx_stats *peer_stats = &dp_pdev->peer_tx_stats;
+	struct htt_ppdu_user_stats *usr_stats = &ppdu_stats->user_stats[user];
+	struct htt_ppdu_stats_common *common = &ppdu_stats->common;
+	int ret;
+	u8 flags, mcs, nss, bw, sgi, dcm, ppdu_type, rate_idx = 0;
+	u32 v, succ_bytes = 0;
+	u16 tones, rate = 0, succ_pkts = 0;
+	u32 tx_duration = 0;
+	u8 tid = HTT_PPDU_STATS_NON_QOS_TID;
+	u16 tx_retry_failed = 0, tx_retry_count = 0;
+	bool is_ampdu = false, is_ofdma;
+
+	if (!(usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_RATE)))
+		return;
+
+	if (usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON)) {
+		is_ampdu =
+			HTT_USR_CMPLTN_IS_AMPDU(usr_stats->cmpltn_cmn.flags);
+		tx_retry_failed =
+			__le16_to_cpu(usr_stats->cmpltn_cmn.mpdu_tried) -
+			__le16_to_cpu(usr_stats->cmpltn_cmn.mpdu_success);
+		tx_retry_count =
+			HTT_USR_CMPLTN_LONG_RETRY(usr_stats->cmpltn_cmn.flags) +
+			HTT_USR_CMPLTN_SHORT_RETRY(usr_stats->cmpltn_cmn.flags);
+	}
+
+	if (usr_stats->tlv_flags &
+	    BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_ACK_BA_STATUS)) {
+		succ_bytes = le32_to_cpu(usr_stats->ack_ba.success_bytes);
+		succ_pkts = le32_get_bits(usr_stats->ack_ba.info,
+					  HTT_PPDU_STATS_ACK_BA_INFO_NUM_MSDU_M);
+		tid = le32_get_bits(usr_stats->ack_ba.info,
+				    HTT_PPDU_STATS_ACK_BA_INFO_TID_NUM);
+	}
+
+	if (common->fes_duration_us)
+		tx_duration = le32_to_cpu(common->fes_duration_us);
+
+	user_rate = &usr_stats->rate;
+	flags = HTT_USR_RATE_PREAMBLE(user_rate->rate_flags);
+	bw = HTT_USR_RATE_BW(user_rate->rate_flags) - 2;
+	nss = HTT_USR_RATE_NSS(user_rate->rate_flags) + 1;
+	mcs = HTT_USR_RATE_MCS(user_rate->rate_flags);
+	sgi = HTT_USR_RATE_GI(user_rate->rate_flags);
+	dcm = HTT_USR_RATE_DCM(user_rate->rate_flags);
+
+	ppdu_type = HTT_USR_RATE_PPDU_TYPE(user_rate->info1);
+	is_ofdma = (ppdu_type == HTT_PPDU_STATS_PPDU_TYPE_MU_OFDMA) ||
+		   (ppdu_type == HTT_PPDU_STATS_PPDU_TYPE_MU_MIMO_OFDMA);
+
+	/* Note: If host configured fixed rates and in some other special
+	 * cases, the broadcast/management frames are sent in different rates.
+	 * Firmware rate's control to be skipped for this?
+	 */
+
+	if (flags == WMI_RATE_PREAMBLE_HE && mcs > ATH12K_HE_MCS_MAX) {
+		ath12k_warn(ab, "Invalid HE mcs %d peer stats",  mcs);
+		return;
+	}
+
+	if (flags == WMI_RATE_PREAMBLE_VHT && mcs > ATH12K_VHT_MCS_MAX) {
+		ath12k_warn(ab, "Invalid VHT mcs %d peer stats",  mcs);
+		return;
+	}
+
+	if (flags == WMI_RATE_PREAMBLE_HT && (mcs > ATH12K_HT_MCS_MAX || nss < 1)) {
+		ath12k_warn(ab, "Invalid HT mcs %d nss %d peer stats",
+			    mcs, nss);
+		return;
+	}
+
+	if (flags == WMI_RATE_PREAMBLE_CCK || flags == WMI_RATE_PREAMBLE_OFDM) {
+		ret = ath12k_mac_hw_ratecode_to_legacy_rate(mcs,
+							    flags,
+							    &rate_idx,
+							    &rate);
+		if (ret < 0)
+			return;
+	}
+
+	rcu_read_lock();
+	peer = ath12k_dp_link_peer_find_by_peerid(dp_pdev, usr_stats->peer_id);
+
+	if (!peer || !peer->sta) {
+		rcu_read_unlock();
+		return;
+	}
+
+	spin_lock_bh(&dp->dp_lock);
+
+	memset(&peer->txrate, 0, sizeof(peer->txrate));
+
+	peer->txrate.bw = ath12k_mac_bw_to_mac80211_bw(bw);
+
+	switch (flags) {
+	case WMI_RATE_PREAMBLE_OFDM:
+		peer->txrate.legacy = rate;
+		break;
+	case WMI_RATE_PREAMBLE_CCK:
+		peer->txrate.legacy = rate;
+		break;
+	case WMI_RATE_PREAMBLE_HT:
+		peer->txrate.mcs = mcs + 8 * (nss - 1);
+		peer->txrate.flags = RATE_INFO_FLAGS_MCS;
+		if (sgi)
+			peer->txrate.flags |= RATE_INFO_FLAGS_SHORT_GI;
+		break;
+	case WMI_RATE_PREAMBLE_VHT:
+		peer->txrate.mcs = mcs;
+		peer->txrate.flags = RATE_INFO_FLAGS_VHT_MCS;
+		if (sgi)
+			peer->txrate.flags |= RATE_INFO_FLAGS_SHORT_GI;
+		break;
+	case WMI_RATE_PREAMBLE_HE:
+		peer->txrate.mcs = mcs;
+		peer->txrate.flags = RATE_INFO_FLAGS_HE_MCS;
+		peer->txrate.he_dcm = dcm;
+		peer->txrate.he_gi = ath12k_he_gi_to_nl80211_he_gi(sgi);
+		tones = le16_to_cpu(user_rate->ru_end) -
+			le16_to_cpu(user_rate->ru_start) + 1;
+		v = ath12k_he_ru_tones_to_nl80211_he_ru_alloc(tones);
+		peer->txrate.he_ru_alloc = v;
+		if (is_ofdma)
+			peer->txrate.bw = RATE_INFO_BW_HE_RU;
+		break;
+	case WMI_RATE_PREAMBLE_EHT:
+		peer->txrate.mcs = mcs;
+		peer->txrate.flags = RATE_INFO_FLAGS_EHT_MCS;
+		peer->txrate.he_dcm = dcm;
+		peer->txrate.eht_gi = ath12k_mac_eht_gi_to_nl80211_eht_gi(sgi);
+		tones = le16_to_cpu(user_rate->ru_end) -
+			le16_to_cpu(user_rate->ru_start) + 1;
+		v = ath12k_mac_eht_ru_tones_to_nl80211_eht_ru_alloc(tones);
+		peer->txrate.eht_ru_alloc = v;
+		if (is_ofdma)
+			peer->txrate.bw = RATE_INFO_BW_EHT_RU;
+		break;
+	}
+
+	peer->tx_retry_failed += tx_retry_failed;
+	peer->tx_retry_count += tx_retry_count;
+	peer->txrate.nss = nss;
+	peer->tx_duration += tx_duration;
+	memcpy(&peer->last_txrate, &peer->txrate, sizeof(struct rate_info));
+
+	spin_unlock_bh(&dp->dp_lock);
+
+	/* PPDU stats reported for mgmt packet doesn't have valid tx bytes.
+	 * So skip peer stats update for mgmt packets.
+	 */
+	if (tid < HTT_PPDU_STATS_NON_QOS_TID) {
+		memset(peer_stats, 0, sizeof(*peer_stats));
+		peer_stats->succ_pkts = succ_pkts;
+		peer_stats->succ_bytes = succ_bytes;
+		peer_stats->is_ampdu = is_ampdu;
+		peer_stats->duration = tx_duration;
+		peer_stats->ba_fails =
+			HTT_USR_CMPLTN_LONG_RETRY(usr_stats->cmpltn_cmn.flags) +
+			HTT_USR_CMPLTN_SHORT_RETRY(usr_stats->cmpltn_cmn.flags);
+	}
+
+	rcu_read_unlock();
+}
+
+static void ath12k_htt_update_ppdu_stats(struct ath12k_pdev_dp *dp_pdev,
+					 struct htt_ppdu_stats *ppdu_stats)
+{
+	u8 user;
+
+	for (user = 0; user < HTT_PPDU_STATS_MAX_USERS - 1; user++)
+		ath12k_update_per_peer_tx_stats(dp_pdev, ppdu_stats, user);
+}
+
+static
+struct htt_ppdu_stats_info *ath12k_dp_htt_get_ppdu_desc(struct ath12k_pdev_dp *dp_pdev,
+							u32 ppdu_id)
+{
+	struct htt_ppdu_stats_info *ppdu_info;
+
+	lockdep_assert_held(&dp_pdev->ppdu_list_lock);
+	if (!list_empty(&dp_pdev->ppdu_stats_info)) {
+		list_for_each_entry(ppdu_info, &dp_pdev->ppdu_stats_info, list) {
+			if (ppdu_info->ppdu_id == ppdu_id)
+				return ppdu_info;
+		}
+
+		if (dp_pdev->ppdu_stat_list_depth > HTT_PPDU_DESC_MAX_DEPTH) {
+			ppdu_info = list_first_entry(&dp_pdev->ppdu_stats_info,
+						     typeof(*ppdu_info), list);
+			list_del(&ppdu_info->list);
+			dp_pdev->ppdu_stat_list_depth--;
+			ath12k_htt_update_ppdu_stats(dp_pdev, &ppdu_info->ppdu_stats);
+			kfree(ppdu_info);
+		}
+	}
+
+	ppdu_info = kzalloc(sizeof(*ppdu_info), GFP_ATOMIC);
+	if (!ppdu_info)
+		return NULL;
+
+	list_add_tail(&ppdu_info->list, &dp_pdev->ppdu_stats_info);
+	dp_pdev->ppdu_stat_list_depth++;
+
+	return ppdu_info;
+}
+
+static void ath12k_copy_to_delay_stats(struct ath12k_dp_link_peer *peer,
+				       struct htt_ppdu_user_stats *usr_stats)
+{
+	peer->ppdu_stats_delayba.sw_peer_id = le16_to_cpu(usr_stats->rate.sw_peer_id);
+	peer->ppdu_stats_delayba.info0 = le32_to_cpu(usr_stats->rate.info0);
+	peer->ppdu_stats_delayba.ru_end = le16_to_cpu(usr_stats->rate.ru_end);
+	peer->ppdu_stats_delayba.ru_start = le16_to_cpu(usr_stats->rate.ru_start);
+	peer->ppdu_stats_delayba.info1 = le32_to_cpu(usr_stats->rate.info1);
+	peer->ppdu_stats_delayba.rate_flags = le32_to_cpu(usr_stats->rate.rate_flags);
+	peer->ppdu_stats_delayba.resp_rate_flags =
+		le32_to_cpu(usr_stats->rate.resp_rate_flags);
+
+	peer->delayba_flag = true;
+}
+
+static void ath12k_copy_to_bar(struct ath12k_dp_link_peer *peer,
+			       struct htt_ppdu_user_stats *usr_stats)
+{
+	usr_stats->rate.sw_peer_id = cpu_to_le16(peer->ppdu_stats_delayba.sw_peer_id);
+	usr_stats->rate.info0 = cpu_to_le32(peer->ppdu_stats_delayba.info0);
+	usr_stats->rate.ru_end = cpu_to_le16(peer->ppdu_stats_delayba.ru_end);
+	usr_stats->rate.ru_start = cpu_to_le16(peer->ppdu_stats_delayba.ru_start);
+	usr_stats->rate.info1 = cpu_to_le32(peer->ppdu_stats_delayba.info1);
+	usr_stats->rate.rate_flags = cpu_to_le32(peer->ppdu_stats_delayba.rate_flags);
+	usr_stats->rate.resp_rate_flags =
+		cpu_to_le32(peer->ppdu_stats_delayba.resp_rate_flags);
+
+	peer->delayba_flag = false;
+}
+
+static int ath12k_htt_pull_ppdu_stats(struct ath12k_base *ab,
+				      struct sk_buff *skb)
+{
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct ath12k_htt_ppdu_stats_msg *msg;
+	struct htt_ppdu_stats_info *ppdu_info;
+	struct ath12k_dp_link_peer *peer = NULL;
+	struct htt_ppdu_user_stats *usr_stats = NULL;
+	u32 peer_id = 0;
+	struct ath12k_pdev_dp *dp_pdev;
+	int ret, i;
+	u8 pdev_id, pdev_idx;
+	u32 ppdu_id, len;
+
+	msg = (struct ath12k_htt_ppdu_stats_msg *)skb->data;
+	len = le32_get_bits(msg->info, HTT_T2H_PPDU_STATS_INFO_PAYLOAD_SIZE);
+	if (len > (skb->len - struct_size(msg, data, 0))) {
+		ath12k_warn(ab,
+			    "HTT PPDU STATS event has unexpected payload size %u, should be smaller than %u\n",
+			    len, skb->len);
+		return -EINVAL;
+	}
+
+	pdev_id = le32_get_bits(msg->info, HTT_T2H_PPDU_STATS_INFO_PDEV_ID);
+	ppdu_id = le32_to_cpu(msg->ppdu_id);
+
+	pdev_idx = DP_HW2SW_MACID(pdev_id);
+	if (pdev_idx >= MAX_RADIOS) {
+		ath12k_warn(ab, "HTT PPDU STATS invalid pdev id %u", pdev_id);
+		return -EINVAL;
+	}
+
+	rcu_read_lock();
+
+	dp_pdev = ath12k_dp_to_pdev_dp(dp, pdev_idx);
+	if (!dp_pdev) {
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	spin_lock_bh(&dp_pdev->ppdu_list_lock);
+	ppdu_info = ath12k_dp_htt_get_ppdu_desc(dp_pdev, ppdu_id);
+	if (!ppdu_info) {
+		spin_unlock_bh(&dp_pdev->ppdu_list_lock);
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	ppdu_info->ppdu_id = ppdu_id;
+	ret = ath12k_dp_htt_tlv_iter(ab, msg->data, len,
+				     ath12k_htt_tlv_ppdu_stats_parse,
+				     (void *)ppdu_info);
+	if (ret) {
+		spin_unlock_bh(&dp_pdev->ppdu_list_lock);
+		ath12k_warn(ab, "Failed to parse tlv %d\n", ret);
+		goto exit;
+	}
+
+	if (ppdu_info->ppdu_stats.common.num_users >= HTT_PPDU_STATS_MAX_USERS) {
+		spin_unlock_bh(&dp_pdev->ppdu_list_lock);
+		ath12k_warn(ab,
+			    "HTT PPDU STATS event has unexpected num_users %u, should be smaller than %u\n",
+			    ppdu_info->ppdu_stats.common.num_users,
+			    HTT_PPDU_STATS_MAX_USERS);
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	/* back up data rate tlv for all peers */
+	if (ppdu_info->frame_type == HTT_STATS_PPDU_FTYPE_DATA &&
+	    (ppdu_info->tlv_bitmap & (1 << HTT_PPDU_STATS_TAG_USR_COMMON)) &&
+	    ppdu_info->delay_ba) {
+		for (i = 0; i < ppdu_info->ppdu_stats.common.num_users; i++) {
+			peer_id = ppdu_info->ppdu_stats.user_stats[i].peer_id;
+			peer = ath12k_dp_link_peer_find_by_peerid(dp_pdev, peer_id);
+			if (!peer)
+				continue;
+
+			usr_stats = &ppdu_info->ppdu_stats.user_stats[i];
+			if (usr_stats->delay_ba)
+				ath12k_copy_to_delay_stats(peer, usr_stats);
+		}
+	}
+
+	/* restore all peers' data rate tlv to mu-bar tlv */
+	if (ppdu_info->frame_type == HTT_STATS_PPDU_FTYPE_BAR &&
+	    (ppdu_info->tlv_bitmap & (1 << HTT_PPDU_STATS_TAG_USR_COMMON))) {
+		for (i = 0; i < ppdu_info->bar_num_users; i++) {
+			peer_id = ppdu_info->ppdu_stats.user_stats[i].peer_id;
+			peer = ath12k_dp_link_peer_find_by_peerid(dp_pdev, peer_id);
+			if (!peer)
+				continue;
+
+			usr_stats = &ppdu_info->ppdu_stats.user_stats[i];
+			if (peer->delayba_flag)
+				ath12k_copy_to_bar(peer, usr_stats);
+		}
+	}
+
+	spin_unlock_bh(&dp_pdev->ppdu_list_lock);
+
+exit:
+	rcu_read_unlock();
+
+	return ret;
+}
+
+static void ath12k_htt_mlo_offset_event_handler(struct ath12k_base *ab,
+						struct sk_buff *skb)
+{
+	struct ath12k_htt_mlo_offset_msg *msg;
+	struct ath12k_pdev *pdev;
+	struct ath12k *ar;
+	u8 pdev_id;
+
+	msg = (struct ath12k_htt_mlo_offset_msg *)skb->data;
+	pdev_id = u32_get_bits(__le32_to_cpu(msg->info),
+			       HTT_T2H_MLO_OFFSET_INFO_PDEV_ID);
+
+	rcu_read_lock();
+	ar = ath12k_mac_get_ar_by_pdev_id(ab, pdev_id);
+	if (!ar) {
+		/* It is possible that the ar is not yet active (started).
+		 * The above function will only look for the active pdev
+		 * and hence %NULL return is possible. Just silently
+		 * discard this message
+		 */
+		goto exit;
+	}
+
+	spin_lock_bh(&ar->data_lock);
+	pdev = ar->pdev;
+
+	pdev->timestamp.info = __le32_to_cpu(msg->info);
+	pdev->timestamp.sync_timestamp_lo_us = __le32_to_cpu(msg->sync_timestamp_lo_us);
+	pdev->timestamp.sync_timestamp_hi_us = __le32_to_cpu(msg->sync_timestamp_hi_us);
+	pdev->timestamp.mlo_offset_lo = __le32_to_cpu(msg->mlo_offset_lo);
+	pdev->timestamp.mlo_offset_hi = __le32_to_cpu(msg->mlo_offset_hi);
+	pdev->timestamp.mlo_offset_clks = __le32_to_cpu(msg->mlo_offset_clks);
+	pdev->timestamp.mlo_comp_clks = __le32_to_cpu(msg->mlo_comp_clks);
+	pdev->timestamp.mlo_comp_timer = __le32_to_cpu(msg->mlo_comp_timer);
+
+	spin_unlock_bh(&ar->data_lock);
+exit:
+	rcu_read_unlock();
+}
+
+void ath12k_dp_htt_htc_t2h_msg_handler(struct ath12k_base *ab,
+				       struct sk_buff *skb)
+{
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct htt_resp_msg *resp = (struct htt_resp_msg *)skb->data;
+	enum htt_t2h_msg_type type;
+	u16 peer_id;
+	u8 vdev_id;
+	u8 mac_addr[ETH_ALEN];
+	u16 peer_mac_h16;
+	u16 ast_hash = 0;
+	u16 hw_peer_id;
+
+	type = le32_get_bits(resp->version_msg.version, HTT_T2H_MSG_TYPE);
+
+	ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt rx msg type :0x%0x\n", type);
+
+	switch (type) {
+	case HTT_T2H_MSG_TYPE_VERSION_CONF:
+		dp->htt_tgt_ver_major = le32_get_bits(resp->version_msg.version,
+						      HTT_T2H_VERSION_CONF_MAJOR);
+		dp->htt_tgt_ver_minor = le32_get_bits(resp->version_msg.version,
+						      HTT_T2H_VERSION_CONF_MINOR);
+		complete(&dp->htt_tgt_version_received);
+		break;
+	/* TODO: remove unused peer map versions after testing */
+	case HTT_T2H_MSG_TYPE_PEER_MAP:
+		vdev_id = le32_get_bits(resp->peer_map_ev.info,
+					HTT_T2H_PEER_MAP_INFO_VDEV_ID);
+		peer_id = le32_get_bits(resp->peer_map_ev.info,
+					HTT_T2H_PEER_MAP_INFO_PEER_ID);
+		peer_mac_h16 = le32_get_bits(resp->peer_map_ev.info1,
+					     HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16);
+		ath12k_dp_get_mac_addr(le32_to_cpu(resp->peer_map_ev.mac_addr_l32),
+				       peer_mac_h16, mac_addr);
+		ath12k_dp_link_peer_map_event(ab, vdev_id, peer_id, mac_addr, 0, 0);
+		break;
+	case HTT_T2H_MSG_TYPE_PEER_MAP2:
+		vdev_id = le32_get_bits(resp->peer_map_ev.info,
+					HTT_T2H_PEER_MAP_INFO_VDEV_ID);
+		peer_id = le32_get_bits(resp->peer_map_ev.info,
+					HTT_T2H_PEER_MAP_INFO_PEER_ID);
+		peer_mac_h16 = le32_get_bits(resp->peer_map_ev.info1,
+					     HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16);
+		ath12k_dp_get_mac_addr(le32_to_cpu(resp->peer_map_ev.mac_addr_l32),
+				       peer_mac_h16, mac_addr);
+		ast_hash = le32_get_bits(resp->peer_map_ev.info2,
+					 HTT_T2H_PEER_MAP_INFO2_AST_HASH_VAL);
+		hw_peer_id = le32_get_bits(resp->peer_map_ev.info1,
+					   HTT_T2H_PEER_MAP_INFO1_HW_PEER_ID);
+		ath12k_dp_link_peer_map_event(ab, vdev_id, peer_id, mac_addr, ast_hash,
+					      hw_peer_id);
+		break;
+	case HTT_T2H_MSG_TYPE_PEER_MAP3:
+		vdev_id = le32_get_bits(resp->peer_map_ev.info,
+					HTT_T2H_PEER_MAP_INFO_VDEV_ID);
+		peer_id = le32_get_bits(resp->peer_map_ev.info,
+					HTT_T2H_PEER_MAP_INFO_PEER_ID);
+		peer_mac_h16 = le32_get_bits(resp->peer_map_ev.info1,
+					     HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16);
+		ath12k_dp_get_mac_addr(le32_to_cpu(resp->peer_map_ev.mac_addr_l32),
+				       peer_mac_h16, mac_addr);
+		ast_hash = le32_get_bits(resp->peer_map_ev.info2,
+					 HTT_T2H_PEER_MAP3_INFO2_AST_HASH_VAL);
+		hw_peer_id = le32_get_bits(resp->peer_map_ev.info2,
+					   HTT_T2H_PEER_MAP3_INFO2_HW_PEER_ID);
+		ath12k_dp_link_peer_map_event(ab, vdev_id, peer_id, mac_addr, ast_hash,
+					      hw_peer_id);
+		break;
+	case HTT_T2H_MSG_TYPE_PEER_UNMAP:
+	case HTT_T2H_MSG_TYPE_PEER_UNMAP2:
+		peer_id = le32_get_bits(resp->peer_unmap_ev.info,
+					HTT_T2H_PEER_UNMAP_INFO_PEER_ID);
+		ath12k_dp_link_peer_unmap_event(ab, peer_id);
+		break;
+	case HTT_T2H_MSG_TYPE_PPDU_STATS_IND:
+		ath12k_htt_pull_ppdu_stats(ab, skb);
+		break;
+	case HTT_T2H_MSG_TYPE_EXT_STATS_CONF:
+		ath12k_debugfs_htt_ext_stats_handler(ab, skb);
+		break;
+	case HTT_T2H_MSG_TYPE_MLO_TIMESTAMP_OFFSET_IND:
+		ath12k_htt_mlo_offset_event_handler(ab, skb);
+		break;
+	default:
+		ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt event %d not handled\n",
+			   type);
+		break;
+	}
+
+	dev_kfree_skb_any(skb);
+}
+EXPORT_SYMBOL(ath12k_dp_htt_htc_t2h_msg_handler);
+
+static int
+ath12k_dp_tx_get_ring_id_type(struct ath12k_base *ab,
+			      int mac_id, u32 ring_id,
+			      enum hal_ring_type ring_type,
+			      enum htt_srng_ring_type *htt_ring_type,
+			      enum htt_srng_ring_id *htt_ring_id)
+{
+	int ret = 0;
+
+	switch (ring_type) {
+	case HAL_RXDMA_BUF:
+		/* for some targets, host fills rx buffer to fw and fw fills to
+		 * rxbuf ring for each rxdma
+		 */
+		if (!ab->hw_params->rx_mac_buf_ring) {
+			if (!(ring_id == HAL_SRNG_SW2RXDMA_BUF0 ||
+			      ring_id == HAL_SRNG_SW2RXDMA_BUF1)) {
+				ret = -EINVAL;
+			}
+			*htt_ring_id = HTT_RXDMA_HOST_BUF_RING;
+			*htt_ring_type = HTT_SW_TO_HW_RING;
+		} else {
+			if (ring_id == HAL_SRNG_SW2RXDMA_BUF0) {
+				*htt_ring_id = HTT_HOST1_TO_FW_RXBUF_RING;
+				*htt_ring_type = HTT_SW_TO_SW_RING;
+			} else {
+				*htt_ring_id = HTT_RXDMA_HOST_BUF_RING;
+				*htt_ring_type = HTT_SW_TO_HW_RING;
+			}
+		}
+		break;
+	case HAL_RXDMA_DST:
+		*htt_ring_id = HTT_RXDMA_NON_MONITOR_DEST_RING;
+		*htt_ring_type = HTT_HW_TO_SW_RING;
+		break;
+	case HAL_RXDMA_MONITOR_BUF:
+		*htt_ring_id = HTT_RX_MON_HOST2MON_BUF_RING;
+		*htt_ring_type = HTT_SW_TO_HW_RING;
+		break;
+	case HAL_RXDMA_MONITOR_STATUS:
+		*htt_ring_id = HTT_RXDMA_MONITOR_STATUS_RING;
+		*htt_ring_type = HTT_SW_TO_HW_RING;
+		break;
+	case HAL_RXDMA_MONITOR_DST:
+		*htt_ring_id = HTT_RX_MON_MON2HOST_DEST_RING;
+		*htt_ring_type = HTT_HW_TO_SW_RING;
+		break;
+	case HAL_RXDMA_MONITOR_DESC:
+		*htt_ring_id = HTT_RXDMA_MONITOR_DESC_RING;
+		*htt_ring_type = HTT_SW_TO_HW_RING;
+		break;
+	default:
+		ath12k_warn(ab, "Unsupported ring type in DP :%d\n", ring_type);
+		ret = -EINVAL;
+	}
+	return ret;
+}
+
+int ath12k_dp_tx_htt_srng_setup(struct ath12k_base *ab, u32 ring_id,
+				int mac_id, enum hal_ring_type ring_type)
+{
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct htt_srng_setup_cmd *cmd;
+	struct hal_srng *srng = &ab->hal.srng_list[ring_id];
+	struct hal_srng_params params;
+	struct sk_buff *skb;
+	u32 ring_entry_sz;
+	int len = sizeof(*cmd);
+	dma_addr_t hp_addr, tp_addr;
+	enum htt_srng_ring_type htt_ring_type;
+	enum htt_srng_ring_id htt_ring_id;
+	int ret;
+
+	skb = ath12k_htc_alloc_skb(ab, len);
+	if (!skb)
+		return -ENOMEM;
+
+	memset(&params, 0, sizeof(params));
+	ath12k_hal_srng_get_params(ab, srng, &params);
+
+	hp_addr = ath12k_hal_srng_get_hp_addr(ab, srng);
+	tp_addr = ath12k_hal_srng_get_tp_addr(ab, srng);
+
+	ret = ath12k_dp_tx_get_ring_id_type(ab, mac_id, ring_id,
+					    ring_type, &htt_ring_type,
+					    &htt_ring_id);
+	if (ret)
+		goto err_free;
+
+	skb_put(skb, len);
+	cmd = (struct htt_srng_setup_cmd *)skb->data;
+	cmd->info0 = le32_encode_bits(HTT_H2T_MSG_TYPE_SRING_SETUP,
+				      HTT_SRNG_SETUP_CMD_INFO0_MSG_TYPE);
+	if (htt_ring_type == HTT_SW_TO_HW_RING ||
+	    htt_ring_type == HTT_HW_TO_SW_RING)
+		cmd->info0 |= le32_encode_bits(DP_SW2HW_MACID(mac_id),
+					       HTT_SRNG_SETUP_CMD_INFO0_PDEV_ID);
+	else
+		cmd->info0 |= le32_encode_bits(mac_id,
+					       HTT_SRNG_SETUP_CMD_INFO0_PDEV_ID);
+	cmd->info0 |= le32_encode_bits(htt_ring_type,
+				       HTT_SRNG_SETUP_CMD_INFO0_RING_TYPE);
+	cmd->info0 |= le32_encode_bits(htt_ring_id,
+				       HTT_SRNG_SETUP_CMD_INFO0_RING_ID);
+
+	cmd->ring_base_addr_lo = cpu_to_le32(params.ring_base_paddr &
+					     HAL_ADDR_LSB_REG_MASK);
+
+	cmd->ring_base_addr_hi = cpu_to_le32((u64)params.ring_base_paddr >>
+					     HAL_ADDR_MSB_REG_SHIFT);
+
+	ret = ath12k_hal_srng_get_entrysize(ab, ring_type);
+	if (ret < 0)
+		goto err_free;
+
+	ring_entry_sz = ret;
+
+	ring_entry_sz >>= 2;
+	cmd->info1 = le32_encode_bits(ring_entry_sz,
+				      HTT_SRNG_SETUP_CMD_INFO1_RING_ENTRY_SIZE);
+	cmd->info1 |= le32_encode_bits(params.num_entries * ring_entry_sz,
+				       HTT_SRNG_SETUP_CMD_INFO1_RING_SIZE);
+	cmd->info1 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_MSI_SWAP),
+				       HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_MSI_SWAP);
+	cmd->info1 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP),
+				       HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_TLV_SWAP);
+	cmd->info1 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_RING_PTR_SWAP),
+				       HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_HOST_FW_SWAP);
+	if (htt_ring_type == HTT_SW_TO_HW_RING)
+		cmd->info1 |= cpu_to_le32(HTT_SRNG_SETUP_CMD_INFO1_RING_LOOP_CNT_DIS);
+
+	cmd->ring_head_off32_remote_addr_lo = cpu_to_le32(lower_32_bits(hp_addr));
+	cmd->ring_head_off32_remote_addr_hi = cpu_to_le32(upper_32_bits(hp_addr));
+
+	cmd->ring_tail_off32_remote_addr_lo = cpu_to_le32(lower_32_bits(tp_addr));
+	cmd->ring_tail_off32_remote_addr_hi = cpu_to_le32(upper_32_bits(tp_addr));
+
+	cmd->ring_msi_addr_lo = cpu_to_le32(lower_32_bits(params.msi_addr));
+	cmd->ring_msi_addr_hi = cpu_to_le32(upper_32_bits(params.msi_addr));
+	cmd->msi_data = cpu_to_le32(params.msi_data);
+
+	cmd->intr_info =
+		le32_encode_bits(params.intr_batch_cntr_thres_entries * ring_entry_sz,
+				 HTT_SRNG_SETUP_CMD_INTR_INFO_BATCH_COUNTER_THRESH);
+	cmd->intr_info |=
+		le32_encode_bits(params.intr_timer_thres_us >> 3,
+				 HTT_SRNG_SETUP_CMD_INTR_INFO_INTR_TIMER_THRESH);
+
+	cmd->info2 = 0;
+	if (params.flags & HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN) {
+		cmd->info2 = le32_encode_bits(params.low_threshold,
+					      HTT_SRNG_SETUP_CMD_INFO2_INTR_LOW_THRESH);
+	}
+
+	ath12k_dbg(ab, ATH12K_DBG_HAL,
+		   "%s msi_addr_lo:0x%x, msi_addr_hi:0x%x, msi_data:0x%x\n",
+		   __func__, cmd->ring_msi_addr_lo, cmd->ring_msi_addr_hi,
+		   cmd->msi_data);
+
+	ath12k_dbg(ab, ATH12K_DBG_HAL,
+		   "ring_id:%d, ring_type:%d, intr_info:0x%x, flags:0x%x\n",
+		   ring_id, ring_type, cmd->intr_info, cmd->info2);
+
+	ret = ath12k_htc_send(&ab->htc, dp->eid, skb);
+	if (ret)
+		goto err_free;
+
+	return 0;
+
+err_free:
+	dev_kfree_skb_any(skb);
+
+	return ret;
+}
+
+int ath12k_dp_tx_htt_h2t_ver_req_msg(struct ath12k_base *ab)
+{
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct sk_buff *skb;
+	struct htt_ver_req_cmd *cmd;
+	int len = sizeof(*cmd);
+	u32 metadata_version;
+	int ret;
+
+	init_completion(&dp->htt_tgt_version_received);
+
+	skb = ath12k_htc_alloc_skb(ab, len);
+	if (!skb)
+		return -ENOMEM;
+
+	skb_put(skb, len);
+	cmd = (struct htt_ver_req_cmd *)skb->data;
+	cmd->ver_reg_info = le32_encode_bits(HTT_H2T_MSG_TYPE_VERSION_REQ,
+					     HTT_OPTION_TAG);
+	metadata_version = ath12k_ftm_mode ? HTT_OPTION_TCL_METADATA_VER_V1 :
+			   HTT_OPTION_TCL_METADATA_VER_V2;
+
+	cmd->tcl_metadata_version = le32_encode_bits(HTT_TAG_TCL_METADATA_VERSION,
+						     HTT_OPTION_TAG) |
+				    le32_encode_bits(HTT_TCL_METADATA_VER_SZ,
+						     HTT_OPTION_LEN) |
+				    le32_encode_bits(metadata_version,
+						     HTT_OPTION_VALUE);
+
+	ret = ath12k_htc_send(&ab->htc, dp->eid, skb);
+	if (ret) {
+		dev_kfree_skb_any(skb);
+		return ret;
+	}
+
+	ret = wait_for_completion_timeout(&dp->htt_tgt_version_received,
+					  HTT_TARGET_VERSION_TIMEOUT_HZ);
+	if (ret == 0) {
+		ath12k_warn(ab, "htt target version request timed out\n");
+		return -ETIMEDOUT;
+	}
+
+	if (dp->htt_tgt_ver_major != HTT_TARGET_VERSION_MAJOR) {
+		ath12k_err(ab, "unsupported htt major version %d supported version is %d\n",
+			   dp->htt_tgt_ver_major, HTT_TARGET_VERSION_MAJOR);
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+int ath12k_dp_tx_htt_h2t_ppdu_stats_req(struct ath12k *ar, u32 mask)
+{
+	struct ath12k_base *ab = ar->ab;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct sk_buff *skb;
+	struct htt_ppdu_stats_cfg_cmd *cmd;
+	int len = sizeof(*cmd);
+	u8 pdev_mask;
+	int ret;
+	int i;
+
+	for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) {
+		skb = ath12k_htc_alloc_skb(ab, len);
+		if (!skb)
+			return -ENOMEM;
+
+		skb_put(skb, len);
+		cmd = (struct htt_ppdu_stats_cfg_cmd *)skb->data;
+		cmd->msg = le32_encode_bits(HTT_H2T_MSG_TYPE_PPDU_STATS_CFG,
+					    HTT_PPDU_STATS_CFG_MSG_TYPE);
+
+		pdev_mask = 1 << (i + ar->pdev_idx);
+		cmd->msg |= le32_encode_bits(pdev_mask, HTT_PPDU_STATS_CFG_PDEV_ID);
+		cmd->msg |= le32_encode_bits(mask, HTT_PPDU_STATS_CFG_TLV_TYPE_BITMASK);
+
+		ret = ath12k_htc_send(&ab->htc, dp->eid, skb);
+		if (ret) {
+			dev_kfree_skb_any(skb);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+int ath12k_dp_tx_htt_rx_filter_setup(struct ath12k_base *ab, u32 ring_id,
+				     int mac_id, enum hal_ring_type ring_type,
+				     int rx_buf_size,
+				     struct htt_rx_ring_tlv_filter *tlv_filter)
+{
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct htt_rx_ring_selection_cfg_cmd *cmd;
+	struct hal_srng *srng = &ab->hal.srng_list[ring_id];
+	struct hal_srng_params params;
+	struct sk_buff *skb;
+	int len = sizeof(*cmd);
+	enum htt_srng_ring_type htt_ring_type;
+	enum htt_srng_ring_id htt_ring_id;
+	int ret;
+
+	skb = ath12k_htc_alloc_skb(ab, len);
+	if (!skb)
+		return -ENOMEM;
+
+	memset(&params, 0, sizeof(params));
+	ath12k_hal_srng_get_params(ab, srng, &params);
+
+	ret = ath12k_dp_tx_get_ring_id_type(ab, mac_id, ring_id,
+					    ring_type, &htt_ring_type,
+					    &htt_ring_id);
+	if (ret)
+		goto err_free;
+
+	skb_put(skb, len);
+	cmd = (struct htt_rx_ring_selection_cfg_cmd *)skb->data;
+	cmd->info0 = le32_encode_bits(HTT_H2T_MSG_TYPE_RX_RING_SELECTION_CFG,
+				      HTT_RX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE);
+	if (htt_ring_type == HTT_SW_TO_HW_RING ||
+	    htt_ring_type == HTT_HW_TO_SW_RING)
+		cmd->info0 |=
+			le32_encode_bits(DP_SW2HW_MACID(mac_id),
+					 HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID);
+	else
+		cmd->info0 |=
+			le32_encode_bits(mac_id,
+					 HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID);
+	cmd->info0 |= le32_encode_bits(htt_ring_id,
+				       HTT_RX_RING_SELECTION_CFG_CMD_INFO0_RING_ID);
+	cmd->info0 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_MSI_SWAP),
+				       HTT_RX_RING_SELECTION_CFG_CMD_INFO0_SS);
+	cmd->info0 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP),
+				       HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PS);
+	cmd->info0 |= le32_encode_bits(tlv_filter->offset_valid,
+				       HTT_RX_RING_SELECTION_CFG_CMD_INFO0_OFFSET_VALID);
+	cmd->info0 |=
+		le32_encode_bits(tlv_filter->drop_threshold_valid,
+				 HTT_RX_RING_SELECTION_CFG_CMD_INFO0_DROP_THRES_VAL);
+	cmd->info0 |= le32_encode_bits(!tlv_filter->rxmon_disable,
+				       HTT_RX_RING_SELECTION_CFG_CMD_INFO0_EN_RXMON);
+
+	cmd->info1 = le32_encode_bits(rx_buf_size,
+				      HTT_RX_RING_SELECTION_CFG_CMD_INFO1_BUF_SIZE);
+	cmd->info1 |= le32_encode_bits(tlv_filter->conf_len_mgmt,
+				       HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_MGMT);
+	cmd->info1 |= le32_encode_bits(tlv_filter->conf_len_ctrl,
+				       HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_CTRL);
+	cmd->info1 |= le32_encode_bits(tlv_filter->conf_len_data,
+				       HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_DATA);
+	cmd->pkt_type_en_flags0 = cpu_to_le32(tlv_filter->pkt_filter_flags0);
+	cmd->pkt_type_en_flags1 = cpu_to_le32(tlv_filter->pkt_filter_flags1);
+	cmd->pkt_type_en_flags2 = cpu_to_le32(tlv_filter->pkt_filter_flags2);
+	cmd->pkt_type_en_flags3 = cpu_to_le32(tlv_filter->pkt_filter_flags3);
+	cmd->rx_filter_tlv = cpu_to_le32(tlv_filter->rx_filter);
+
+	cmd->info2 = le32_encode_bits(tlv_filter->rx_drop_threshold,
+				      HTT_RX_RING_SELECTION_CFG_CMD_INFO2_DROP_THRESHOLD);
+	cmd->info2 |=
+		le32_encode_bits(tlv_filter->enable_log_mgmt_type,
+				 HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_LOG_MGMT_TYPE);
+	cmd->info2 |=
+		le32_encode_bits(tlv_filter->enable_log_ctrl_type,
+				 HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_CTRL_TYPE);
+	cmd->info2 |=
+		le32_encode_bits(tlv_filter->enable_log_data_type,
+				 HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_LOG_DATA_TYPE);
+
+	cmd->info3 =
+		le32_encode_bits(tlv_filter->enable_rx_tlv_offset,
+				 HTT_RX_RING_SELECTION_CFG_CMD_INFO3_EN_TLV_PKT_OFFSET);
+	cmd->info3 |=
+		le32_encode_bits(tlv_filter->rx_tlv_offset,
+				 HTT_RX_RING_SELECTION_CFG_CMD_INFO3_PKT_TLV_OFFSET);
+
+	if (tlv_filter->offset_valid) {
+		cmd->rx_packet_offset =
+			le32_encode_bits(tlv_filter->rx_packet_offset,
+					 HTT_RX_RING_SELECTION_CFG_RX_PACKET_OFFSET);
+
+		cmd->rx_packet_offset |=
+			le32_encode_bits(tlv_filter->rx_header_offset,
+					 HTT_RX_RING_SELECTION_CFG_RX_HEADER_OFFSET);
+
+		cmd->rx_mpdu_offset =
+			le32_encode_bits(tlv_filter->rx_mpdu_end_offset,
+					 HTT_RX_RING_SELECTION_CFG_RX_MPDU_END_OFFSET);
+
+		cmd->rx_mpdu_offset |=
+			le32_encode_bits(tlv_filter->rx_mpdu_start_offset,
+					 HTT_RX_RING_SELECTION_CFG_RX_MPDU_START_OFFSET);
+
+		cmd->rx_msdu_offset =
+			le32_encode_bits(tlv_filter->rx_msdu_end_offset,
+					 HTT_RX_RING_SELECTION_CFG_RX_MSDU_END_OFFSET);
+
+		cmd->rx_msdu_offset |=
+			le32_encode_bits(tlv_filter->rx_msdu_start_offset,
+					 HTT_RX_RING_SELECTION_CFG_RX_MSDU_START_OFFSET);
+
+		cmd->rx_attn_offset =
+			le32_encode_bits(tlv_filter->rx_attn_offset,
+					 HTT_RX_RING_SELECTION_CFG_RX_ATTENTION_OFFSET);
+	}
+
+	if (tlv_filter->rx_mpdu_start_wmask > 0 &&
+	    tlv_filter->rx_msdu_end_wmask > 0) {
+		cmd->info2 |=
+			le32_encode_bits(true,
+					 HTT_RX_RING_SELECTION_CFG_WORD_MASK_COMPACT_SET);
+		cmd->rx_mpdu_start_end_mask =
+			le32_encode_bits(tlv_filter->rx_mpdu_start_wmask,
+					 HTT_RX_RING_SELECTION_CFG_RX_MPDU_START_MASK);
+		/* mpdu_end is not used for any hardwares so far
+		 * please assign it in future if any chip is
+		 * using through hal ops
+		 */
+		cmd->rx_mpdu_start_end_mask |=
+			le32_encode_bits(tlv_filter->rx_mpdu_end_wmask,
+					 HTT_RX_RING_SELECTION_CFG_RX_MPDU_END_MASK);
+		cmd->rx_msdu_end_word_mask =
+			le32_encode_bits(tlv_filter->rx_msdu_end_wmask,
+					 HTT_RX_RING_SELECTION_CFG_RX_MSDU_END_MASK);
+	}
+
+	ret = ath12k_htc_send(&ab->htc, dp->eid, skb);
+	if (ret)
+		goto err_free;
+
+	return 0;
+
+err_free:
+	dev_kfree_skb_any(skb);
+
+	return ret;
+}
+EXPORT_SYMBOL(ath12k_dp_tx_htt_rx_filter_setup);
+
+int
+ath12k_dp_tx_htt_h2t_ext_stats_req(struct ath12k *ar, u8 type,
+				   struct htt_ext_stats_cfg_params *cfg_params,
+				   u64 cookie)
+{
+	struct ath12k_base *ab = ar->ab;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct sk_buff *skb;
+	struct htt_ext_stats_cfg_cmd *cmd;
+	int len = sizeof(*cmd);
+	int ret;
+	u32 pdev_id;
+
+	skb = ath12k_htc_alloc_skb(ab, len);
+	if (!skb)
+		return -ENOMEM;
+
+	skb_put(skb, len);
+
+	cmd = (struct htt_ext_stats_cfg_cmd *)skb->data;
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->hdr.msg_type = HTT_H2T_MSG_TYPE_EXT_STATS_CFG;
+
+	pdev_id = ath12k_mac_get_target_pdev_id(ar);
+	cmd->hdr.pdev_mask = 1 << pdev_id;
+
+	cmd->hdr.stats_type = type;
+	cmd->cfg_param0 = cpu_to_le32(cfg_params->cfg0);
+	cmd->cfg_param1 = cpu_to_le32(cfg_params->cfg1);
+	cmd->cfg_param2 = cpu_to_le32(cfg_params->cfg2);
+	cmd->cfg_param3 = cpu_to_le32(cfg_params->cfg3);
+	cmd->cookie_lsb = cpu_to_le32(lower_32_bits(cookie));
+	cmd->cookie_msb = cpu_to_le32(upper_32_bits(cookie));
+
+	ret = ath12k_htc_send(&ab->htc, dp->eid, skb);
+	if (ret) {
+		ath12k_warn(ab, "failed to send htt type stats request: %d",
+			    ret);
+		dev_kfree_skb_any(skb);
+		return ret;
+	}
+
+	return 0;
+}
+
+int ath12k_dp_tx_htt_monitor_mode_ring_config(struct ath12k *ar, bool reset)
+{
+	struct ath12k_base *ab = ar->ab;
+	int ret;
+
+	ret = ath12k_dp_tx_htt_rx_monitor_mode_ring_config(ar, reset);
+	if (ret) {
+		ath12k_err(ab, "failed to setup rx monitor filter %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+int ath12k_dp_tx_htt_rx_monitor_mode_ring_config(struct ath12k *ar, bool reset)
+{
+	struct ath12k_base *ab = ar->ab;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct htt_rx_ring_tlv_filter tlv_filter = {};
+	int ret, ring_id, i;
+
+	tlv_filter.offset_valid = false;
+
+	if (!reset) {
+		tlv_filter.rx_filter = HTT_RX_MON_FILTER_TLV_FLAGS_MON_DEST_RING;
+
+		tlv_filter.drop_threshold_valid = true;
+		tlv_filter.rx_drop_threshold = HTT_RX_RING_TLV_DROP_THRESHOLD_VALUE;
+
+		tlv_filter.enable_log_mgmt_type = true;
+		tlv_filter.enable_log_ctrl_type = true;
+		tlv_filter.enable_log_data_type = true;
+
+		tlv_filter.conf_len_ctrl = HTT_RX_RING_DEFAULT_DMA_LENGTH;
+		tlv_filter.conf_len_mgmt = HTT_RX_RING_DEFAULT_DMA_LENGTH;
+		tlv_filter.conf_len_data = HTT_RX_RING_DEFAULT_DMA_LENGTH;
+
+		tlv_filter.enable_rx_tlv_offset = true;
+		tlv_filter.rx_tlv_offset = HTT_RX_RING_PKT_TLV_OFFSET;
+
+		tlv_filter.pkt_filter_flags0 =
+					HTT_RX_MON_FP_MGMT_FILTER_FLAGS0 |
+					HTT_RX_MON_MO_MGMT_FILTER_FLAGS0;
+		tlv_filter.pkt_filter_flags1 =
+					HTT_RX_MON_FP_MGMT_FILTER_FLAGS1 |
+					HTT_RX_MON_MO_MGMT_FILTER_FLAGS1;
+		tlv_filter.pkt_filter_flags2 =
+					HTT_RX_MON_FP_CTRL_FILTER_FLASG2 |
+					HTT_RX_MON_MO_CTRL_FILTER_FLASG2;
+		tlv_filter.pkt_filter_flags3 =
+					HTT_RX_MON_FP_CTRL_FILTER_FLASG3 |
+					HTT_RX_MON_MO_CTRL_FILTER_FLASG3 |
+					HTT_RX_MON_FP_DATA_FILTER_FLASG3 |
+					HTT_RX_MON_MO_DATA_FILTER_FLASG3;
+	} else {
+		tlv_filter = ath12k_mac_mon_status_filter_default;
+
+		if (ath12k_debugfs_is_extd_rx_stats_enabled(ar))
+			tlv_filter.rx_filter = ath12k_debugfs_rx_filter(ar);
+	}
+
+	if (ab->hw_params->rxdma1_enable) {
+		for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) {
+			ring_id = ar->dp.rxdma_mon_dst_ring[i].ring_id;
+			ret = ath12k_dp_tx_htt_rx_filter_setup(ar->ab, ring_id,
+							       ar->dp.mac_id + i,
+							       HAL_RXDMA_MONITOR_DST,
+							       DP_RXDMA_REFILL_RING_SIZE,
+							       &tlv_filter);
+			if (ret) {
+				ath12k_err(ab,
+					   "failed to setup filter for monitor buf %d\n",
+					   ret);
+				return ret;
+			}
+		}
+		return 0;
+	}
+
+	if (!reset) {
+		for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) {
+			ring_id = dp->rx_mac_buf_ring[i].ring_id;
+			ret = ath12k_dp_tx_htt_rx_filter_setup(ar->ab, ring_id,
+							       i,
+							       HAL_RXDMA_BUF,
+							       DP_RXDMA_REFILL_RING_SIZE,
+							       &tlv_filter);
+			if (ret) {
+				ath12k_err(ab,
+					   "failed to setup filter for mon rx buf %d\n",
+					   ret);
+				return ret;
+			}
+		}
+	}
+
+	for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) {
+		ring_id = dp->rx_mon_status_refill_ring[i].refill_buf_ring.ring_id;
+		if (!reset) {
+			tlv_filter.rx_filter =
+				HTT_RX_MON_FILTER_TLV_FLAGS_MON_STATUS_RING;
+		}
+
+		ret = ath12k_dp_tx_htt_rx_filter_setup(ab, ring_id,
+						       i,
+						       HAL_RXDMA_MONITOR_STATUS,
+						       RX_MON_STATUS_BUF_SIZE,
+						       &tlv_filter);
+		if (ret) {
+			ath12k_err(ab,
+				   "failed to setup filter for mon status buf %d\n",
+				   ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+int ath12k_dp_tx_htt_tx_filter_setup(struct ath12k_base *ab, u32 ring_id,
+				     int mac_id, enum hal_ring_type ring_type,
+				     int tx_buf_size,
+				     struct htt_tx_ring_tlv_filter *htt_tlv_filter)
+{
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct htt_tx_ring_selection_cfg_cmd *cmd;
+	struct hal_srng *srng = &ab->hal.srng_list[ring_id];
+	struct hal_srng_params params;
+	struct sk_buff *skb;
+	int len = sizeof(*cmd);
+	enum htt_srng_ring_type htt_ring_type;
+	enum htt_srng_ring_id htt_ring_id;
+	int ret;
+
+	skb = ath12k_htc_alloc_skb(ab, len);
+	if (!skb)
+		return -ENOMEM;
+
+	memset(&params, 0, sizeof(params));
+	ath12k_hal_srng_get_params(ab, srng, &params);
+
+	ret = ath12k_dp_tx_get_ring_id_type(ab, mac_id, ring_id,
+					    ring_type, &htt_ring_type,
+					    &htt_ring_id);
+
+	if (ret)
+		goto err_free;
+
+	skb_put(skb, len);
+	cmd = (struct htt_tx_ring_selection_cfg_cmd *)skb->data;
+	cmd->info0 = le32_encode_bits(HTT_H2T_MSG_TYPE_TX_MONITOR_CFG,
+				      HTT_TX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE);
+	if (htt_ring_type == HTT_SW_TO_HW_RING ||
+	    htt_ring_type == HTT_HW_TO_SW_RING)
+		cmd->info0 |=
+			le32_encode_bits(DP_SW2HW_MACID(mac_id),
+					 HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID);
+	else
+		cmd->info0 |=
+			le32_encode_bits(mac_id,
+					 HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID);
+	cmd->info0 |= le32_encode_bits(htt_ring_id,
+				       HTT_TX_RING_SELECTION_CFG_CMD_INFO0_RING_ID);
+	cmd->info0 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_MSI_SWAP),
+				       HTT_TX_RING_SELECTION_CFG_CMD_INFO0_SS);
+	cmd->info0 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP),
+				       HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PS);
+
+	cmd->info1 |=
+		le32_encode_bits(tx_buf_size,
+				 HTT_TX_RING_SELECTION_CFG_CMD_INFO1_RING_BUFF_SIZE);
+
+	if (htt_tlv_filter->tx_mon_mgmt_filter) {
+		cmd->info1 |=
+			le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_MGMT,
+					 HTT_TX_RING_SELECTION_CFG_CMD_INFO1_PKT_TYPE);
+		cmd->info1 |=
+		le32_encode_bits(htt_tlv_filter->tx_mon_pkt_dma_len,
+				 HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_MGMT);
+		cmd->info2 |=
+		le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_MGMT,
+				 HTT_TX_RING_SELECTION_CFG_CMD_INFO2_PKT_TYPE_EN_FLAG);
+	}
+
+	if (htt_tlv_filter->tx_mon_data_filter) {
+		cmd->info1 |=
+			le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_CTRL,
+					 HTT_TX_RING_SELECTION_CFG_CMD_INFO1_PKT_TYPE);
+		cmd->info1 |=
+		le32_encode_bits(htt_tlv_filter->tx_mon_pkt_dma_len,
+				 HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_CTRL);
+		cmd->info2 |=
+		le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_CTRL,
+				 HTT_TX_RING_SELECTION_CFG_CMD_INFO2_PKT_TYPE_EN_FLAG);
+	}
+
+	if (htt_tlv_filter->tx_mon_ctrl_filter) {
+		cmd->info1 |=
+			le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_DATA,
+					 HTT_TX_RING_SELECTION_CFG_CMD_INFO1_PKT_TYPE);
+		cmd->info1 |=
+		le32_encode_bits(htt_tlv_filter->tx_mon_pkt_dma_len,
+				 HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_DATA);
+		cmd->info2 |=
+		le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_DATA,
+				 HTT_TX_RING_SELECTION_CFG_CMD_INFO2_PKT_TYPE_EN_FLAG);
+	}
+
+	cmd->tlv_filter_mask_in0 =
+		cpu_to_le32(htt_tlv_filter->tx_mon_downstream_tlv_flags);
+	cmd->tlv_filter_mask_in1 =
+		cpu_to_le32(htt_tlv_filter->tx_mon_upstream_tlv_flags0);
+	cmd->tlv_filter_mask_in2 =
+		cpu_to_le32(htt_tlv_filter->tx_mon_upstream_tlv_flags1);
+	cmd->tlv_filter_mask_in3 =
+		cpu_to_le32(htt_tlv_filter->tx_mon_upstream_tlv_flags2);
+
+	ret = ath12k_htc_send(&ab->htc, dp->eid, skb);
+	if (ret)
+		goto err_free;
+
+	return 0;
+
+err_free:
+	dev_kfree_skb_any(skb);
+	return ret;
+}
diff --git a/drivers/net/wireless/ath/ath12k/dp_htt.h b/drivers/net/wireless/ath/ath12k/dp_htt.h
new file mode 100644
index 000000000000..6020e632f74e
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/dp_htt.h
@@ -0,0 +1,1546 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef ATH12K_DP_HTT_H
+#define ATH12K_DP_HTT_H
+
+struct ath12k_dp;
+
+/* HTT definitions */
+#define HTT_TAG_TCL_METADATA_VERSION		5
+
+#define HTT_TCL_META_DATA_TYPE			GENMASK(1, 0)
+#define HTT_TCL_META_DATA_VALID_HTT		BIT(2)
+
+/* vdev meta data */
+#define HTT_TCL_META_DATA_VDEV_ID		 GENMASK(10, 3)
+#define HTT_TCL_META_DATA_PDEV_ID		 GENMASK(12, 11)
+#define HTT_TCL_META_DATA_HOST_INSPECTED_MISSION BIT(13)
+
+/* peer meta data */
+#define HTT_TCL_META_DATA_PEER_ID		GENMASK(15, 3)
+
+/* Global sequence number */
+#define HTT_TCL_META_DATA_TYPE_GLOBAL_SEQ_NUM		3
+#define HTT_TCL_META_DATA_GLOBAL_SEQ_HOST_INSPECTED	BIT(2)
+#define HTT_TCL_META_DATA_GLOBAL_SEQ_NUM		GENMASK(14, 3)
+#define HTT_TX_MLO_MCAST_HOST_REINJECT_BASE_VDEV_ID	128
+
+/* HTT tx completion is overlaid in wbm_release_ring */
+#define HTT_TX_WBM_COMP_INFO0_STATUS		GENMASK(16, 13)
+#define HTT_TX_WBM_COMP_INFO1_REINJECT_REASON	GENMASK(3, 0)
+#define HTT_TX_WBM_COMP_INFO1_EXCEPTION_FRAME	BIT(4)
+
+#define HTT_TX_WBM_COMP_INFO2_ACK_RSSI		GENMASK(31, 24)
+
+#define HTT_TARGET_VERSION_TIMEOUT_HZ (3 * HZ)
+
+struct htt_tx_wbm_completion {
+	__le32 rsvd0[2];
+	__le32 info0;
+	__le32 info1;
+	__le32 info2;
+	__le32 info3;
+	__le32 info4;
+	__le32 rsvd1;
+
+} __packed;
+
+enum htt_h2t_msg_type {
+	HTT_H2T_MSG_TYPE_VERSION_REQ		= 0,
+	HTT_H2T_MSG_TYPE_SRING_SETUP		= 0xb,
+	HTT_H2T_MSG_TYPE_RX_RING_SELECTION_CFG	= 0xc,
+	HTT_H2T_MSG_TYPE_EXT_STATS_CFG		= 0x10,
+	HTT_H2T_MSG_TYPE_PPDU_STATS_CFG		= 0x11,
+	HTT_H2T_MSG_TYPE_VDEV_TXRX_STATS_CFG	= 0x1a,
+	HTT_H2T_MSG_TYPE_TX_MONITOR_CFG		= 0x1b,
+};
+
+#define HTT_VER_REQ_INFO_MSG_ID		GENMASK(7, 0)
+#define HTT_OPTION_TCL_METADATA_VER_V1	1
+#define HTT_OPTION_TCL_METADATA_VER_V2	2
+#define HTT_OPTION_TAG			GENMASK(7, 0)
+#define HTT_OPTION_LEN			GENMASK(15, 8)
+#define HTT_OPTION_VALUE		GENMASK(31, 16)
+#define HTT_TCL_METADATA_VER_SZ		4
+
+struct htt_ver_req_cmd {
+	__le32 ver_reg_info;
+	__le32 tcl_metadata_version;
+} __packed;
+
+enum htt_srng_ring_type {
+	HTT_HW_TO_SW_RING,
+	HTT_SW_TO_HW_RING,
+	HTT_SW_TO_SW_RING,
+};
+
+enum htt_srng_ring_id {
+	HTT_RXDMA_HOST_BUF_RING,
+	HTT_RXDMA_MONITOR_STATUS_RING,
+	HTT_RXDMA_MONITOR_BUF_RING,
+	HTT_RXDMA_MONITOR_DESC_RING,
+	HTT_RXDMA_MONITOR_DEST_RING,
+	HTT_HOST1_TO_FW_RXBUF_RING,
+	HTT_HOST2_TO_FW_RXBUF_RING,
+	HTT_RXDMA_NON_MONITOR_DEST_RING,
+	HTT_RXDMA_HOST_BUF_RING2,
+	HTT_TX_MON_HOST2MON_BUF_RING,
+	HTT_TX_MON_MON2HOST_DEST_RING,
+	HTT_RX_MON_HOST2MON_BUF_RING,
+	HTT_RX_MON_MON2HOST_DEST_RING,
+};
+
+/* host -> target  HTT_SRING_SETUP message
+ *
+ * After target is booted up, Host can send SRING setup message for
+ * each host facing LMAC SRING. Target setups up HW registers based
+ * on setup message and confirms back to Host if response_required is set.
+ * Host should wait for confirmation message before sending new SRING
+ * setup message
+ *
+ * The message would appear as follows:
+ *
+ * |31            24|23    20|19|18 16|15|14          8|7                0|
+ * |--------------- +-----------------+----------------+------------------|
+ * |    ring_type   |      ring_id    |    pdev_id     |     msg_type     |
+ * |----------------------------------------------------------------------|
+ * |                          ring_base_addr_lo                           |
+ * |----------------------------------------------------------------------|
+ * |                         ring_base_addr_hi                            |
+ * |----------------------------------------------------------------------|
+ * |ring_misc_cfg_flag|ring_entry_size|            ring_size              |
+ * |----------------------------------------------------------------------|
+ * |                         ring_head_offset32_remote_addr_lo            |
+ * |----------------------------------------------------------------------|
+ * |                         ring_head_offset32_remote_addr_hi            |
+ * |----------------------------------------------------------------------|
+ * |                         ring_tail_offset32_remote_addr_lo            |
+ * |----------------------------------------------------------------------|
+ * |                         ring_tail_offset32_remote_addr_hi            |
+ * |----------------------------------------------------------------------|
+ * |                          ring_msi_addr_lo                            |
+ * |----------------------------------------------------------------------|
+ * |                          ring_msi_addr_hi                            |
+ * |----------------------------------------------------------------------|
+ * |                          ring_msi_data                               |
+ * |----------------------------------------------------------------------|
+ * |         intr_timer_th            |IM|      intr_batch_counter_th     |
+ * |----------------------------------------------------------------------|
+ * |          reserved        |RR|PTCF|        intr_low_threshold         |
+ * |----------------------------------------------------------------------|
+ * Where
+ *     IM = sw_intr_mode
+ *     RR = response_required
+ *     PTCF = prefetch_timer_cfg
+ *
+ * The message is interpreted as follows:
+ * dword0  - b'0:7   - msg_type: This will be set to
+ *                     HTT_H2T_MSG_TYPE_SRING_SETUP
+ *           b'8:15  - pdev_id:
+ *                     0 (for rings at SOC/UMAC level),
+ *                     1/2/3 mac id (for rings at LMAC level)
+ *           b'16:23 - ring_id: identify which ring is to setup,
+ *                     more details can be got from enum htt_srng_ring_id
+ *           b'24:31 - ring_type: identify type of host rings,
+ *                     more details can be got from enum htt_srng_ring_type
+ * dword1  - b'0:31  - ring_base_addr_lo: Lower 32bits of ring base address
+ * dword2  - b'0:31  - ring_base_addr_hi: Upper 32bits of ring base address
+ * dword3  - b'0:15  - ring_size: size of the ring in unit of 4-bytes words
+ *           b'16:23 - ring_entry_size: Size of each entry in 4-byte word units
+ *           b'24:31 - ring_misc_cfg_flag: Valid only for HW_TO_SW_RING and
+ *                     SW_TO_HW_RING.
+ *                     Refer to HTT_SRING_SETUP_RING_MISC_CFG_RING defs.
+ * dword4  - b'0:31  - ring_head_off32_remote_addr_lo:
+ *                     Lower 32 bits of memory address of the remote variable
+ *                     storing the 4-byte word offset that identifies the head
+ *                     element within the ring.
+ *                     (The head offset variable has type u32.)
+ *                     Valid for HW_TO_SW and SW_TO_SW rings.
+ * dword5  - b'0:31  - ring_head_off32_remote_addr_hi:
+ *                     Upper 32 bits of memory address of the remote variable
+ *                     storing the 4-byte word offset that identifies the head
+ *                     element within the ring.
+ *                     (The head offset variable has type u32.)
+ *                     Valid for HW_TO_SW and SW_TO_SW rings.
+ * dword6  - b'0:31  - ring_tail_off32_remote_addr_lo:
+ *                     Lower 32 bits of memory address of the remote variable
+ *                     storing the 4-byte word offset that identifies the tail
+ *                     element within the ring.
+ *                     (The tail offset variable has type u32.)
+ *                     Valid for HW_TO_SW and SW_TO_SW rings.
+ * dword7  - b'0:31  - ring_tail_off32_remote_addr_hi:
+ *                     Upper 32 bits of memory address of the remote variable
+ *                     storing the 4-byte word offset that identifies the tail
+ *                     element within the ring.
+ *                     (The tail offset variable has type u32.)
+ *                     Valid for HW_TO_SW and SW_TO_SW rings.
+ * dword8  - b'0:31  - ring_msi_addr_lo: Lower 32bits of MSI cfg address
+ *                     valid only for HW_TO_SW_RING and SW_TO_HW_RING
+ * dword9  - b'0:31  - ring_msi_addr_hi: Upper 32bits of MSI cfg address
+ *                     valid only for HW_TO_SW_RING and SW_TO_HW_RING
+ * dword10 - b'0:31  - ring_msi_data: MSI data
+ *                     Refer to HTT_SRING_SETUP_RING_MSC_CFG_xxx defs
+ *                     valid only for HW_TO_SW_RING and SW_TO_HW_RING
+ * dword11 - b'0:14  - intr_batch_counter_th:
+ *                     batch counter threshold is in units of 4-byte words.
+ *                     HW internally maintains and increments batch count.
+ *                     (see SRING spec for detail description).
+ *                     When batch count reaches threshold value, an interrupt
+ *                     is generated by HW.
+ *           b'15    - sw_intr_mode:
+ *                     This configuration shall be static.
+ *                     Only programmed at power up.
+ *                     0: generate pulse style sw interrupts
+ *                     1: generate level style sw interrupts
+ *           b'16:31 - intr_timer_th:
+ *                     The timer init value when timer is idle or is
+ *                     initialized to start downcounting.
+ *                     In 8us units (to cover a range of 0 to 524 ms)
+ * dword12 - b'0:15  - intr_low_threshold:
+ *                     Used only by Consumer ring to generate ring_sw_int_p.
+ *                     Ring entries low threshold water mark, that is used
+ *                     in combination with the interrupt timer as well as
+ *                     the clearing of the level interrupt.
+ *           b'16:18 - prefetch_timer_cfg:
+ *                     Used only by Consumer ring to set timer mode to
+ *                     support Application prefetch handling.
+ *                     The external tail offset/pointer will be updated
+ *                     at following intervals:
+ *                     3'b000: (Prefetch feature disabled; used only for debug)
+ *                     3'b001: 1 usec
+ *                     3'b010: 4 usec
+ *                     3'b011: 8 usec (default)
+ *                     3'b100: 16 usec
+ *                     Others: Reserved
+ *           b'19    - response_required:
+ *                     Host needs HTT_T2H_MSG_TYPE_SRING_SETUP_DONE as response
+ *           b'20:31 - reserved:  reserved for future use
+ */
+
+#define HTT_SRNG_SETUP_CMD_INFO0_MSG_TYPE	GENMASK(7, 0)
+#define HTT_SRNG_SETUP_CMD_INFO0_PDEV_ID	GENMASK(15, 8)
+#define HTT_SRNG_SETUP_CMD_INFO0_RING_ID	GENMASK(23, 16)
+#define HTT_SRNG_SETUP_CMD_INFO0_RING_TYPE	GENMASK(31, 24)
+
+#define HTT_SRNG_SETUP_CMD_INFO1_RING_SIZE			GENMASK(15, 0)
+#define HTT_SRNG_SETUP_CMD_INFO1_RING_ENTRY_SIZE		GENMASK(23, 16)
+#define HTT_SRNG_SETUP_CMD_INFO1_RING_LOOP_CNT_DIS		BIT(25)
+#define HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_MSI_SWAP		BIT(27)
+#define HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_HOST_FW_SWAP	BIT(28)
+#define HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_TLV_SWAP		BIT(29)
+
+#define HTT_SRNG_SETUP_CMD_INTR_INFO_BATCH_COUNTER_THRESH	GENMASK(14, 0)
+#define HTT_SRNG_SETUP_CMD_INTR_INFO_SW_INTR_MODE		BIT(15)
+#define HTT_SRNG_SETUP_CMD_INTR_INFO_INTR_TIMER_THRESH		GENMASK(31, 16)
+
+#define HTT_SRNG_SETUP_CMD_INFO2_INTR_LOW_THRESH	GENMASK(15, 0)
+#define HTT_SRNG_SETUP_CMD_INFO2_PRE_FETCH_TIMER_CFG	GENMASK(18, 16)
+#define HTT_SRNG_SETUP_CMD_INFO2_RESPONSE_REQUIRED	BIT(19)
+
+struct htt_srng_setup_cmd {
+	__le32 info0;
+	__le32 ring_base_addr_lo;
+	__le32 ring_base_addr_hi;
+	__le32 info1;
+	__le32 ring_head_off32_remote_addr_lo;
+	__le32 ring_head_off32_remote_addr_hi;
+	__le32 ring_tail_off32_remote_addr_lo;
+	__le32 ring_tail_off32_remote_addr_hi;
+	__le32 ring_msi_addr_lo;
+	__le32 ring_msi_addr_hi;
+	__le32 msi_data;
+	__le32 intr_info;
+	__le32 info2;
+} __packed;
+
+/* host -> target FW  PPDU_STATS config message
+ *
+ * @details
+ * The following field definitions describe the format of the HTT host
+ * to target FW for PPDU_STATS_CFG msg.
+ * The message allows the host to configure the PPDU_STATS_IND messages
+ * produced by the target.
+ *
+ * |31          24|23          16|15           8|7            0|
+ * |-----------------------------------------------------------|
+ * |    REQ bit mask             |   pdev_mask  |   msg type   |
+ * |-----------------------------------------------------------|
+ * Header fields:
+ *  - MSG_TYPE
+ *    Bits 7:0
+ *    Purpose: identifies this is a req to configure ppdu_stats_ind from target
+ *    Value: 0x11
+ *  - PDEV_MASK
+ *    Bits 8:15
+ *    Purpose: identifies which pdevs this PPDU stats configuration applies to
+ *    Value: This is a overloaded field, refer to usage and interpretation of
+ *           PDEV in interface document.
+ *           Bit   8    :  Reserved for SOC stats
+ *           Bit 9 - 15 :  Indicates PDEV_MASK in DBDC
+ *                         Indicates MACID_MASK in DBS
+ *  - REQ_TLV_BIT_MASK
+ *    Bits 16:31
+ *    Purpose: each set bit indicates the corresponding PPDU stats TLV type
+ *        needs to be included in the target's PPDU_STATS_IND messages.
+ *    Value: refer htt_ppdu_stats_tlv_tag_t <<<???
+ *
+ */
+
+struct htt_ppdu_stats_cfg_cmd {
+	__le32 msg;
+} __packed;
+
+#define HTT_PPDU_STATS_CFG_MSG_TYPE		GENMASK(7, 0)
+#define HTT_PPDU_STATS_CFG_SOC_STATS		BIT(8)
+#define HTT_PPDU_STATS_CFG_PDEV_ID		GENMASK(15, 9)
+#define HTT_PPDU_STATS_CFG_TLV_TYPE_BITMASK	GENMASK(31, 16)
+
+enum htt_ppdu_stats_tag_type {
+	HTT_PPDU_STATS_TAG_COMMON,
+	HTT_PPDU_STATS_TAG_USR_COMMON,
+	HTT_PPDU_STATS_TAG_USR_RATE,
+	HTT_PPDU_STATS_TAG_USR_MPDU_ENQ_BITMAP_64,
+	HTT_PPDU_STATS_TAG_USR_MPDU_ENQ_BITMAP_256,
+	HTT_PPDU_STATS_TAG_SCH_CMD_STATUS,
+	HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON,
+	HTT_PPDU_STATS_TAG_USR_COMPLTN_BA_BITMAP_64,
+	HTT_PPDU_STATS_TAG_USR_COMPLTN_BA_BITMAP_256,
+	HTT_PPDU_STATS_TAG_USR_COMPLTN_ACK_BA_STATUS,
+	HTT_PPDU_STATS_TAG_USR_COMPLTN_FLUSH,
+	HTT_PPDU_STATS_TAG_USR_COMMON_ARRAY,
+	HTT_PPDU_STATS_TAG_INFO,
+	HTT_PPDU_STATS_TAG_TX_MGMTCTRL_PAYLOAD,
+
+	/* New TLV's are added above to this line */
+	HTT_PPDU_STATS_TAG_MAX,
+};
+
+#define HTT_PPDU_STATS_TAG_DEFAULT (BIT(HTT_PPDU_STATS_TAG_COMMON) \
+				   | BIT(HTT_PPDU_STATS_TAG_USR_COMMON) \
+				   | BIT(HTT_PPDU_STATS_TAG_USR_RATE) \
+				   | BIT(HTT_PPDU_STATS_TAG_SCH_CMD_STATUS) \
+				   | BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON) \
+				   | BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_ACK_BA_STATUS) \
+				   | BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_FLUSH) \
+				   | BIT(HTT_PPDU_STATS_TAG_USR_COMMON_ARRAY))
+
+#define HTT_PPDU_STATS_TAG_PKTLOG  (BIT(HTT_PPDU_STATS_TAG_USR_MPDU_ENQ_BITMAP_64) | \
+				    BIT(HTT_PPDU_STATS_TAG_USR_MPDU_ENQ_BITMAP_256) | \
+				    BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_BA_BITMAP_64) | \
+				    BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_BA_BITMAP_256) | \
+				    BIT(HTT_PPDU_STATS_TAG_INFO) | \
+				    BIT(HTT_PPDU_STATS_TAG_TX_MGMTCTRL_PAYLOAD) | \
+				    HTT_PPDU_STATS_TAG_DEFAULT)
+
+enum htt_stats_internal_ppdu_frametype {
+	HTT_STATS_PPDU_FTYPE_CTRL,
+	HTT_STATS_PPDU_FTYPE_DATA,
+	HTT_STATS_PPDU_FTYPE_BAR,
+	HTT_STATS_PPDU_FTYPE_MAX
+};
+
+/* HTT_H2T_MSG_TYPE_RX_RING_SELECTION_CFG Message
+ *
+ * details:
+ *    HTT_H2T_MSG_TYPE_RX_RING_SELECTION_CFG message is sent by host to
+ *    configure RXDMA rings.
+ *    The configuration is per ring based and includes both packet subtypes
+ *    and PPDU/MPDU TLVs.
+ *
+ *    The message would appear as follows:
+ *
+ *    |31   29|28|27|26|25|24|23       16|15             8|7             0|
+ *    |-------+--+--+--+--+--+-----------+----------------+---------------|
+ *    | rsvd1 |ED|DT|OV|PS|SS|  ring_id  |     pdev_id    |    msg_type   |
+ *    |-------------------------------------------------------------------|
+ *    |              rsvd2               |           ring_buffer_size     |
+ *    |-------------------------------------------------------------------|
+ *    |                        packet_type_enable_flags_0                 |
+ *    |-------------------------------------------------------------------|
+ *    |                        packet_type_enable_flags_1                 |
+ *    |-------------------------------------------------------------------|
+ *    |                        packet_type_enable_flags_2                 |
+ *    |-------------------------------------------------------------------|
+ *    |                        packet_type_enable_flags_3                 |
+ *    |-------------------------------------------------------------------|
+ *    |                         tlv_filter_in_flags                       |
+ *    |-------------------------------------------------------------------|
+ * Where:
+ *     PS = pkt_swap
+ *     SS = status_swap
+ * The message is interpreted as follows:
+ * dword0 - b'0:7   - msg_type: This will be set to
+ *                    HTT_H2T_MSG_TYPE_RX_RING_SELECTION_CFG
+ *          b'8:15  - pdev_id:
+ *                    0 (for rings at SOC/UMAC level),
+ *                    1/2/3 mac id (for rings at LMAC level)
+ *          b'16:23 - ring_id : Identify the ring to configure.
+ *                    More details can be got from enum htt_srng_ring_id
+ *          b'24    - status_swap: 1 is to swap status TLV
+ *          b'25    - pkt_swap:  1 is to swap packet TLV
+ *          b'26    - rx_offset_valid (OV): flag to indicate rx offsets
+ *		      configuration fields are valid
+ *          b'27    - drop_thresh_valid (DT): flag to indicate if the
+ *		      rx_drop_threshold field is valid
+ *          b'28    - rx_mon_global_en: Enable/Disable global register
+ *		      configuration in Rx monitor module.
+ *          b'29:31 - rsvd1:  reserved for future use
+ * dword1 - b'0:16  - ring_buffer_size: size of buffers referenced by rx ring,
+ *                    in byte units.
+ *                    Valid only for HW_TO_SW_RING and SW_TO_HW_RING
+ *        - b'16:31 - rsvd2: Reserved for future use
+ * dword2 - b'0:31  - packet_type_enable_flags_0:
+ *                    Enable MGMT packet from 0b0000 to 0b1001
+ *                    bits from low to high: FP, MD, MO - 3 bits
+ *                        FP: Filter_Pass
+ *                        MD: Monitor_Direct
+ *                        MO: Monitor_Other
+ *                    10 mgmt subtypes * 3 bits -> 30 bits
+ *                    Refer to PKT_TYPE_ENABLE_FLAG0_xxx_MGMT_xxx defs
+ * dword3 - b'0:31  - packet_type_enable_flags_1:
+ *                    Enable MGMT packet from 0b1010 to 0b1111
+ *                    bits from low to high: FP, MD, MO - 3 bits
+ *                    Refer to PKT_TYPE_ENABLE_FLAG1_xxx_MGMT_xxx defs
+ * dword4 - b'0:31 -  packet_type_enable_flags_2:
+ *                    Enable CTRL packet from 0b0000 to 0b1001
+ *                    bits from low to high: FP, MD, MO - 3 bits
+ *                    Refer to PKT_TYPE_ENABLE_FLAG2_xxx_CTRL_xxx defs
+ * dword5 - b'0:31  - packet_type_enable_flags_3:
+ *                    Enable CTRL packet from 0b1010 to 0b1111,
+ *                    MCAST_DATA, UCAST_DATA, NULL_DATA
+ *                    bits from low to high: FP, MD, MO - 3 bits
+ *                    Refer to PKT_TYPE_ENABLE_FLAG3_xxx_CTRL_xxx defs
+ * dword6 - b'0:31 -  tlv_filter_in_flags:
+ *                    Filter in Attention/MPDU/PPDU/Header/User tlvs
+ *                    Refer to CFG_TLV_FILTER_IN_FLAG defs
+ */
+
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE	GENMASK(7, 0)
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID	GENMASK(15, 8)
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_RING_ID	GENMASK(23, 16)
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_SS		BIT(24)
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PS		BIT(25)
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_OFFSET_VALID	BIT(26)
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_DROP_THRES_VAL	BIT(27)
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO0_EN_RXMON		BIT(28)
+
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO1_BUF_SIZE		GENMASK(15, 0)
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_MGMT	GENMASK(18, 16)
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_CTRL	GENMASK(21, 19)
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_DATA	GENMASK(24, 22)
+
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO2_DROP_THRESHOLD	GENMASK(9, 0)
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_LOG_MGMT_TYPE	BIT(17)
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_CTRL_TYPE	BIT(18)
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_LOG_DATA_TYPE	BIT(19)
+
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO3_EN_TLV_PKT_OFFSET	BIT(0)
+#define HTT_RX_RING_SELECTION_CFG_CMD_INFO3_PKT_TLV_OFFSET	GENMASK(14, 1)
+
+#define HTT_RX_RING_SELECTION_CFG_RX_PACKET_OFFSET      GENMASK(15, 0)
+#define HTT_RX_RING_SELECTION_CFG_RX_HEADER_OFFSET      GENMASK(31, 16)
+#define HTT_RX_RING_SELECTION_CFG_RX_MPDU_END_OFFSET    GENMASK(15, 0)
+#define HTT_RX_RING_SELECTION_CFG_RX_MPDU_START_OFFSET  GENMASK(31, 16)
+#define HTT_RX_RING_SELECTION_CFG_RX_MSDU_END_OFFSET    GENMASK(15, 0)
+#define HTT_RX_RING_SELECTION_CFG_RX_MSDU_START_OFFSET  GENMASK(31, 16)
+#define HTT_RX_RING_SELECTION_CFG_RX_ATTENTION_OFFSET   GENMASK(15, 0)
+
+#define HTT_RX_RING_SELECTION_CFG_WORD_MASK_COMPACT_SET	BIT(23)
+#define HTT_RX_RING_SELECTION_CFG_RX_MPDU_START_MASK	GENMASK(15, 0)
+#define HTT_RX_RING_SELECTION_CFG_RX_MPDU_END_MASK	GENMASK(18, 16)
+#define HTT_RX_RING_SELECTION_CFG_RX_MSDU_END_MASK	GENMASK(16, 0)
+
+enum htt_rx_filter_tlv_flags {
+	HTT_RX_FILTER_TLV_FLAGS_MPDU_START		= BIT(0),
+	HTT_RX_FILTER_TLV_FLAGS_MSDU_START		= BIT(1),
+	HTT_RX_FILTER_TLV_FLAGS_RX_PACKET		= BIT(2),
+	HTT_RX_FILTER_TLV_FLAGS_MSDU_END		= BIT(3),
+	HTT_RX_FILTER_TLV_FLAGS_MPDU_END		= BIT(4),
+	HTT_RX_FILTER_TLV_FLAGS_PACKET_HEADER		= BIT(5),
+	HTT_RX_FILTER_TLV_FLAGS_PER_MSDU_HEADER		= BIT(6),
+	HTT_RX_FILTER_TLV_FLAGS_ATTENTION		= BIT(7),
+	HTT_RX_FILTER_TLV_FLAGS_PPDU_START		= BIT(8),
+	HTT_RX_FILTER_TLV_FLAGS_PPDU_END		= BIT(9),
+	HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS	= BIT(10),
+	HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS_EXT	= BIT(11),
+	HTT_RX_FILTER_TLV_FLAGS_PPDU_END_STATUS_DONE	= BIT(12),
+	HTT_RX_FILTER_TLV_FLAGS_PPDU_START_USER_INFO	= BIT(13),
+};
+
+enum htt_rx_mgmt_pkt_filter_tlv_flags0 {
+	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ		= BIT(0),
+	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ		= BIT(1),
+	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ		= BIT(2),
+	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP		= BIT(3),
+	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP		= BIT(4),
+	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP		= BIT(5),
+	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ	= BIT(6),
+	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ	= BIT(7),
+	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ	= BIT(8),
+	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP	= BIT(9),
+	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP	= BIT(10),
+	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP	= BIT(11),
+	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ		= BIT(12),
+	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ		= BIT(13),
+	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ		= BIT(14),
+	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP		= BIT(15),
+	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP		= BIT(16),
+	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP		= BIT(17),
+	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV	= BIT(18),
+	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV	= BIT(19),
+	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV	= BIT(20),
+	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7		= BIT(21),
+	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7		= BIT(22),
+	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7		= BIT(23),
+	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON		= BIT(24),
+	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON		= BIT(25),
+	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON		= BIT(26),
+	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM		= BIT(27),
+	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM		= BIT(28),
+	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM		= BIT(29),
+};
+
+enum htt_rx_mgmt_pkt_filter_tlv_flags1 {
+	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC		= BIT(0),
+	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC		= BIT(1),
+	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC		= BIT(2),
+	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH		= BIT(3),
+	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH		= BIT(4),
+	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH		= BIT(5),
+	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH		= BIT(6),
+	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH		= BIT(7),
+	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH		= BIT(8),
+	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION		= BIT(9),
+	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION		= BIT(10),
+	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION		= BIT(11),
+	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK	= BIT(12),
+	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK	= BIT(13),
+	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK	= BIT(14),
+	HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15	= BIT(15),
+	HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15	= BIT(16),
+	HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15	= BIT(17),
+};
+
+enum htt_rx_ctrl_pkt_filter_tlv_flags2 {
+	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1	= BIT(0),
+	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1	= BIT(1),
+	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1	= BIT(2),
+	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2	= BIT(3),
+	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2	= BIT(4),
+	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2	= BIT(5),
+	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER	= BIT(6),
+	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER	= BIT(7),
+	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER	= BIT(8),
+	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4	= BIT(9),
+	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4	= BIT(10),
+	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4	= BIT(11),
+	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL	= BIT(12),
+	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL	= BIT(13),
+	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL	= BIT(14),
+	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP	= BIT(15),
+	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP	= BIT(16),
+	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP	= BIT(17),
+	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT	= BIT(18),
+	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT	= BIT(19),
+	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT	= BIT(20),
+	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER	= BIT(21),
+	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER	= BIT(22),
+	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER	= BIT(23),
+	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR		= BIT(24),
+	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_BAR		= BIT(25),
+	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_BAR		= BIT(26),
+	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BA			= BIT(27),
+	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_BA			= BIT(28),
+	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_BA			= BIT(29),
+};
+
+enum htt_rx_ctrl_pkt_filter_tlv_flags3 {
+	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL		= BIT(0),
+	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL		= BIT(1),
+	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL		= BIT(2),
+	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_RTS		= BIT(3),
+	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_RTS		= BIT(4),
+	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_RTS		= BIT(5),
+	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CTS		= BIT(6),
+	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CTS		= BIT(7),
+	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CTS		= BIT(8),
+	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_ACK		= BIT(9),
+	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_ACK		= BIT(10),
+	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_ACK		= BIT(11),
+	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND		= BIT(12),
+	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND		= BIT(13),
+	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND		= BIT(14),
+	HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK		= BIT(15),
+	HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK		= BIT(16),
+	HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK		= BIT(17),
+};
+
+enum htt_rx_data_pkt_filter_tlv_flasg3 {
+	HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST	= BIT(18),
+	HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_MCAST	= BIT(19),
+	HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_MCAST	= BIT(20),
+	HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST	= BIT(21),
+	HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_UCAST	= BIT(22),
+	HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_UCAST	= BIT(23),
+	HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA	= BIT(24),
+	HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA	= BIT(25),
+	HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA	= BIT(26),
+};
+
+#define HTT_RX_FP_MGMT_FILTER_FLAGS0 \
+	(HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ \
+	| HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP \
+	| HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ \
+	| HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP \
+	| HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ \
+	| HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP \
+	| HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV \
+	| HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON \
+	| HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM)
+
+#define HTT_RX_MD_MGMT_FILTER_FLAGS0 \
+	(HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ \
+	| HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP \
+	| HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ \
+	| HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP \
+	| HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ \
+	| HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP \
+	| HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV \
+	| HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON \
+	| HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM)
+
+#define HTT_RX_MO_MGMT_FILTER_FLAGS0 \
+	(HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_REQ \
+	| HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ASSOC_RESP \
+	| HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_REQ \
+	| HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_REASSOC_RESP \
+	| HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_REQ \
+	| HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_RESP \
+	| HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_PROBE_TIMING_ADV \
+	| HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_BEACON \
+	| HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_ATIM)
+
+#define HTT_RX_FP_MGMT_FILTER_FLAGS1 (HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC \
+				     | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH \
+				     | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH \
+				     | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION \
+				     | HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK)
+
+#define HTT_RX_MD_MGMT_FILTER_FLAGS1 (HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC \
+				     | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH \
+				     | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH \
+				     | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION \
+				     | HTT_RX_MD_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK)
+
+#define HTT_RX_MO_MGMT_FILTER_FLAGS1 (HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_DISASSOC \
+				     | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_AUTH \
+				     | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_DEAUTH \
+				     | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION \
+				     | HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_ACTION_NOACK)
+
+#define HTT_RX_FP_CTRL_FILTER_FLASG2 (HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER \
+				     | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR \
+				     | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BA)
+
+#define HTT_RX_MD_CTRL_FILTER_FLASG2 (HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER \
+				     | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_BAR \
+				     | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS2_BA)
+
+#define HTT_RX_MO_CTRL_FILTER_FLASG2 (HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_WRAPPER \
+				     | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_BAR \
+				     | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_BA)
+
+#define HTT_RX_FP_CTRL_FILTER_FLASG3 (HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL \
+				     | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_RTS \
+				     | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CTS \
+				     | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_ACK \
+				     | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND \
+				     | HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK)
+
+#define HTT_RX_MD_CTRL_FILTER_FLASG3 (HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL \
+				     | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_RTS \
+				     | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CTS \
+				     | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_ACK \
+				     | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND \
+				     | HTT_RX_MD_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK)
+
+#define HTT_RX_MO_CTRL_FILTER_FLASG3 (HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_PSPOLL \
+				     | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_RTS \
+				     | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CTS \
+				     | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_ACK \
+				     | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND \
+				     | HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS3_CFEND_ACK)
+
+#define HTT_RX_FP_DATA_FILTER_FLASG3 (HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST \
+				     | HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST \
+				     | HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA)
+
+#define HTT_RX_MD_DATA_FILTER_FLASG3 (HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_MCAST \
+				     | HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_UCAST \
+				     | HTT_RX_MD_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA)
+
+#define HTT_RX_MO_DATA_FILTER_FLASG3 (HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_MCAST \
+				     | HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_UCAST \
+				     | HTT_RX_MO_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA)
+
+#define HTT_RX_MON_FP_MGMT_FILTER_FLAGS0 \
+		(HTT_RX_FP_MGMT_FILTER_FLAGS0 | \
+		HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7)
+
+#define HTT_RX_MON_MO_MGMT_FILTER_FLAGS0 \
+		(HTT_RX_MO_MGMT_FILTER_FLAGS0 | \
+		HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS0_RESERVED_7)
+
+#define HTT_RX_MON_FP_MGMT_FILTER_FLAGS1 \
+		(HTT_RX_FP_MGMT_FILTER_FLAGS1 | \
+		HTT_RX_FP_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15)
+
+#define HTT_RX_MON_MO_MGMT_FILTER_FLAGS1 \
+		(HTT_RX_MO_MGMT_FILTER_FLAGS1 | \
+		HTT_RX_MO_MGMT_PKT_FILTER_TLV_FLAGS1_RESERVED_15)
+
+#define HTT_RX_MON_FP_CTRL_FILTER_FLASG2 \
+		(HTT_RX_FP_CTRL_FILTER_FLASG2 | \
+		HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1 | \
+		HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2 | \
+		HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER | \
+		HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4 | \
+		HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL | \
+		HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP | \
+		HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT)
+
+#define HTT_RX_MON_MO_CTRL_FILTER_FLASG2 \
+		(HTT_RX_MO_CTRL_FILTER_FLASG2 | \
+		HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_1 | \
+		HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_2 | \
+		HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_TRIGGER | \
+		HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_RESERVED_4 | \
+		HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_BF_REP_POLL | \
+		HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_VHT_NDP | \
+		HTT_RX_MO_CTRL_PKT_FILTER_TLV_FLAGS2_CTRL_FRAME_EXT)
+
+#define HTT_RX_MON_FP_CTRL_FILTER_FLASG3 HTT_RX_FP_CTRL_FILTER_FLASG3
+
+#define HTT_RX_MON_MO_CTRL_FILTER_FLASG3 HTT_RX_MO_CTRL_FILTER_FLASG3
+
+#define HTT_RX_MON_FP_DATA_FILTER_FLASG3 HTT_RX_FP_DATA_FILTER_FLASG3
+
+#define HTT_RX_MON_MO_DATA_FILTER_FLASG3 HTT_RX_MO_DATA_FILTER_FLASG3
+
+#define HTT_RX_MON_FILTER_TLV_FLAGS \
+		(HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \
+		HTT_RX_FILTER_TLV_FLAGS_PPDU_START | \
+		HTT_RX_FILTER_TLV_FLAGS_PPDU_END | \
+		HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS | \
+		HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS_EXT | \
+		HTT_RX_FILTER_TLV_FLAGS_PPDU_END_STATUS_DONE)
+
+#define HTT_RX_MON_FILTER_TLV_FLAGS_MON_STATUS_RING \
+		(HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \
+		HTT_RX_FILTER_TLV_FLAGS_PPDU_START | \
+		HTT_RX_FILTER_TLV_FLAGS_PPDU_END | \
+		HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS | \
+		HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS_EXT | \
+		HTT_RX_FILTER_TLV_FLAGS_PPDU_END_STATUS_DONE)
+
+#define HTT_RX_MON_FILTER_TLV_FLAGS_MON_BUF_RING \
+		(HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \
+		HTT_RX_FILTER_TLV_FLAGS_MSDU_START | \
+		HTT_RX_FILTER_TLV_FLAGS_RX_PACKET | \
+		HTT_RX_FILTER_TLV_FLAGS_MSDU_END | \
+		HTT_RX_FILTER_TLV_FLAGS_MPDU_END | \
+		HTT_RX_FILTER_TLV_FLAGS_PACKET_HEADER | \
+		HTT_RX_FILTER_TLV_FLAGS_PER_MSDU_HEADER | \
+		HTT_RX_FILTER_TLV_FLAGS_ATTENTION)
+
+#define HTT_RX_MON_FILTER_TLV_FLAGS_MON_DEST_RING \
+	(HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \
+	HTT_RX_FILTER_TLV_FLAGS_MSDU_START | \
+	HTT_RX_FILTER_TLV_FLAGS_RX_PACKET | \
+	HTT_RX_FILTER_TLV_FLAGS_MSDU_END | \
+	HTT_RX_FILTER_TLV_FLAGS_MPDU_END | \
+	HTT_RX_FILTER_TLV_FLAGS_PACKET_HEADER | \
+	HTT_RX_FILTER_TLV_FLAGS_PER_MSDU_HEADER | \
+	HTT_RX_FILTER_TLV_FLAGS_PPDU_START | \
+	HTT_RX_FILTER_TLV_FLAGS_PPDU_END | \
+	HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS | \
+	HTT_RX_FILTER_TLV_FLAGS_PPDU_END_USER_STATS_EXT | \
+	HTT_RX_FILTER_TLV_FLAGS_PPDU_END_STATUS_DONE | \
+	HTT_RX_FILTER_TLV_FLAGS_PPDU_START_USER_INFO)
+
+/* msdu start. mpdu end, attention, rx hdr tlv's are not subscribed */
+#define HTT_RX_TLV_FLAGS_RXDMA_RING \
+		(HTT_RX_FILTER_TLV_FLAGS_MPDU_START | \
+		HTT_RX_FILTER_TLV_FLAGS_RX_PACKET | \
+		HTT_RX_FILTER_TLV_FLAGS_MSDU_END)
+
+#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE	GENMASK(7, 0)
+#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID	GENMASK(15, 8)
+
+struct htt_rx_ring_selection_cfg_cmd {
+	__le32 info0;
+	__le32 info1;
+	__le32 pkt_type_en_flags0;
+	__le32 pkt_type_en_flags1;
+	__le32 pkt_type_en_flags2;
+	__le32 pkt_type_en_flags3;
+	__le32 rx_filter_tlv;
+	__le32 rx_packet_offset;
+	__le32 rx_mpdu_offset;
+	__le32 rx_msdu_offset;
+	__le32 rx_attn_offset;
+	__le32 info2;
+	__le32 reserved[2];
+	__le32 rx_mpdu_start_end_mask;
+	__le32 rx_msdu_end_word_mask;
+	__le32 info3;
+} __packed;
+
+#define HTT_RX_RING_TLV_DROP_THRESHOLD_VALUE	32
+#define HTT_RX_RING_DEFAULT_DMA_LENGTH		0x7
+#define HTT_RX_RING_PKT_TLV_OFFSET		0x1
+
+struct htt_rx_ring_tlv_filter {
+	u32 rx_filter; /* see htt_rx_filter_tlv_flags */
+	u32 pkt_filter_flags0; /* MGMT */
+	u32 pkt_filter_flags1; /* MGMT */
+	u32 pkt_filter_flags2; /* CTRL */
+	u32 pkt_filter_flags3; /* DATA */
+	bool offset_valid;
+	u16 rx_packet_offset;
+	u16 rx_header_offset;
+	u16 rx_mpdu_end_offset;
+	u16 rx_mpdu_start_offset;
+	u16 rx_msdu_end_offset;
+	u16 rx_msdu_start_offset;
+	u16 rx_attn_offset;
+	u16 rx_mpdu_start_wmask;
+	u16 rx_mpdu_end_wmask;
+	u32 rx_msdu_end_wmask;
+	u32 conf_len_ctrl;
+	u32 conf_len_mgmt;
+	u32 conf_len_data;
+	u16 rx_drop_threshold;
+	bool enable_log_mgmt_type;
+	bool enable_log_ctrl_type;
+	bool enable_log_data_type;
+	bool enable_rx_tlv_offset;
+	u16 rx_tlv_offset;
+	bool drop_threshold_valid;
+	bool rxmon_disable;
+};
+
+#define HTT_STATS_FRAME_CTRL_TYPE_MGMT  0x0
+#define HTT_STATS_FRAME_CTRL_TYPE_CTRL  0x1
+#define HTT_STATS_FRAME_CTRL_TYPE_DATA  0x2
+#define HTT_STATS_FRAME_CTRL_TYPE_RESV  0x3
+
+#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE	GENMASK(7, 0)
+#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID	GENMASK(15, 8)
+#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_RING_ID	GENMASK(23, 16)
+#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_SS		BIT(24)
+#define HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PS		BIT(25)
+
+#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_RING_BUFF_SIZE	GENMASK(15, 0)
+#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_PKT_TYPE		GENMASK(18, 16)
+#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_MGMT	GENMASK(21, 19)
+#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_CTRL	GENMASK(24, 22)
+#define HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_DATA	GENMASK(27, 25)
+
+#define HTT_TX_RING_SELECTION_CFG_CMD_INFO2_PKT_TYPE_EN_FLAG	GENMASK(2, 0)
+
+struct htt_tx_ring_selection_cfg_cmd {
+	__le32 info0;
+	__le32 info1;
+	__le32 info2;
+	__le32 tlv_filter_mask_in0;
+	__le32 tlv_filter_mask_in1;
+	__le32 tlv_filter_mask_in2;
+	__le32 tlv_filter_mask_in3;
+	__le32 reserved[3];
+} __packed;
+
+#define HTT_TX_RING_TLV_FILTER_MGMT_DMA_LEN	GENMASK(3, 0)
+#define HTT_TX_RING_TLV_FILTER_CTRL_DMA_LEN	GENMASK(7, 4)
+#define HTT_TX_RING_TLV_FILTER_DATA_DMA_LEN	GENMASK(11, 8)
+
+#define HTT_TX_MON_FILTER_HYBRID_MODE \
+		(HTT_TX_FILTER_TLV_FLAGS0_RESPONSE_START_STATUS | \
+		HTT_TX_FILTER_TLV_FLAGS0_RESPONSE_END_STATUS | \
+		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START | \
+		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_END | \
+		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START_PPDU | \
+		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_USER_PPDU | \
+		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_ACK_OR_BA | \
+		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_1K_BA | \
+		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START_PROT | \
+		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_PROT | \
+		HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_USER_RESPONSE | \
+		HTT_TX_FILTER_TLV_FLAGS0_RECEIVED_RESPONSE_INFO | \
+		HTT_TX_FILTER_TLV_FLAGS0_RECEIVED_RESPONSE_INFO_PART2)
+
+struct htt_tx_ring_tlv_filter {
+	u32 tx_mon_downstream_tlv_flags;
+	u32 tx_mon_upstream_tlv_flags0;
+	u32 tx_mon_upstream_tlv_flags1;
+	u32 tx_mon_upstream_tlv_flags2;
+	bool tx_mon_mgmt_filter;
+	bool tx_mon_data_filter;
+	bool tx_mon_ctrl_filter;
+	u16 tx_mon_pkt_dma_len;
+} __packed;
+
+enum htt_tx_mon_upstream_tlv_flags0 {
+	HTT_TX_FILTER_TLV_FLAGS0_RESPONSE_START_STATUS		= BIT(1),
+	HTT_TX_FILTER_TLV_FLAGS0_RESPONSE_END_STATUS		= BIT(2),
+	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START		= BIT(3),
+	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_END		= BIT(4),
+	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START_PPDU	= BIT(5),
+	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_USER_PPDU	= BIT(6),
+	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_ACK_OR_BA	= BIT(7),
+	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_1K_BA		= BIT(8),
+	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_START_PROT	= BIT(9),
+	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_PROT		= BIT(10),
+	HTT_TX_FILTER_TLV_FLAGS0_TX_FES_STATUS_USER_RESPONSE	= BIT(11),
+	HTT_TX_FILTER_TLV_FLAGS0_RX_FRAME_BITMAP_ACK		= BIT(12),
+	HTT_TX_FILTER_TLV_FLAGS0_RX_FRAME_1K_BITMAP_ACK		= BIT(13),
+	HTT_TX_FILTER_TLV_FLAGS0_COEX_TX_STATUS			= BIT(14),
+	HTT_TX_FILTER_TLV_FLAGS0_RECEIVED_RESPONSE_INFO		= BIT(15),
+	HTT_TX_FILTER_TLV_FLAGS0_RECEIVED_RESPONSE_INFO_PART2	= BIT(16),
+};
+
+#define HTT_TX_FILTER_TLV_FLAGS2_TXPCU_PHYTX_OTHER_TRANSMIT_INFO32	BIT(11)
+
+/* HTT message target->host */
+
+enum htt_t2h_msg_type {
+	HTT_T2H_MSG_TYPE_VERSION_CONF,
+	HTT_T2H_MSG_TYPE_PEER_MAP	= 0x3,
+	HTT_T2H_MSG_TYPE_PEER_UNMAP	= 0x4,
+	HTT_T2H_MSG_TYPE_RX_ADDBA	= 0x5,
+	HTT_T2H_MSG_TYPE_PKTLOG		= 0x8,
+	HTT_T2H_MSG_TYPE_SEC_IND	= 0xb,
+	HTT_T2H_MSG_TYPE_PEER_MAP2	= 0x1e,
+	HTT_T2H_MSG_TYPE_PEER_UNMAP2	= 0x1f,
+	HTT_T2H_MSG_TYPE_PPDU_STATS_IND = 0x1d,
+	HTT_T2H_MSG_TYPE_EXT_STATS_CONF = 0x1c,
+	HTT_T2H_MSG_TYPE_BKPRESSURE_EVENT_IND = 0x24,
+	HTT_T2H_MSG_TYPE_MLO_TIMESTAMP_OFFSET_IND = 0x28,
+	HTT_T2H_MSG_TYPE_PEER_MAP3	= 0x2b,
+	HTT_T2H_MSG_TYPE_VDEV_TXRX_STATS_PERIODIC_IND = 0x2c,
+};
+
+#define HTT_TARGET_VERSION_MAJOR 3
+
+#define HTT_T2H_MSG_TYPE		GENMASK(7, 0)
+#define HTT_T2H_VERSION_CONF_MINOR	GENMASK(15, 8)
+#define HTT_T2H_VERSION_CONF_MAJOR	GENMASK(23, 16)
+
+struct htt_t2h_version_conf_msg {
+	__le32 version;
+} __packed;
+
+#define HTT_T2H_PEER_MAP_INFO_VDEV_ID	GENMASK(15, 8)
+#define HTT_T2H_PEER_MAP_INFO_PEER_ID	GENMASK(31, 16)
+#define HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16	GENMASK(15, 0)
+#define HTT_T2H_PEER_MAP_INFO1_HW_PEER_ID	GENMASK(31, 16)
+#define HTT_T2H_PEER_MAP_INFO2_AST_HASH_VAL	GENMASK(15, 0)
+#define HTT_T2H_PEER_MAP3_INFO2_HW_PEER_ID	GENMASK(15, 0)
+#define HTT_T2H_PEER_MAP3_INFO2_AST_HASH_VAL	GENMASK(31, 16)
+#define HTT_T2H_PEER_MAP_INFO2_NEXT_HOP_M	BIT(16)
+#define HTT_T2H_PEER_MAP_INFO2_NEXT_HOP_S	16
+
+struct htt_t2h_peer_map_event {
+	__le32 info;
+	__le32 mac_addr_l32;
+	__le32 info1;
+	__le32 info2;
+} __packed;
+
+#define HTT_T2H_PEER_UNMAP_INFO_VDEV_ID	HTT_T2H_PEER_MAP_INFO_VDEV_ID
+#define HTT_T2H_PEER_UNMAP_INFO_PEER_ID	HTT_T2H_PEER_MAP_INFO_PEER_ID
+#define HTT_T2H_PEER_UNMAP_INFO1_MAC_ADDR_H16 \
+					HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16
+#define HTT_T2H_PEER_MAP_INFO1_NEXT_HOP_M HTT_T2H_PEER_MAP_INFO2_NEXT_HOP_M
+#define HTT_T2H_PEER_MAP_INFO1_NEXT_HOP_S HTT_T2H_PEER_MAP_INFO2_NEXT_HOP_S
+
+struct htt_t2h_peer_unmap_event {
+	__le32 info;
+	__le32 mac_addr_l32;
+	__le32 info1;
+} __packed;
+
+struct htt_resp_msg {
+	union {
+		struct htt_t2h_version_conf_msg version_msg;
+		struct htt_t2h_peer_map_event peer_map_ev;
+		struct htt_t2h_peer_unmap_event peer_unmap_ev;
+	};
+} __packed;
+
+#define HTT_VDEV_GET_STATS_U64(msg_l32, msg_u32)\
+	(((u64)__le32_to_cpu(msg_u32) << 32) | (__le32_to_cpu(msg_l32)))
+#define HTT_T2H_VDEV_STATS_PERIODIC_MSG_TYPE		GENMASK(7, 0)
+#define HTT_T2H_VDEV_STATS_PERIODIC_PDEV_ID		GENMASK(15, 8)
+#define HTT_T2H_VDEV_STATS_PERIODIC_NUM_VDEV		GENMASK(23, 16)
+#define HTT_T2H_VDEV_STATS_PERIODIC_PAYLOAD_BYTES	GENMASK(15, 0)
+#define HTT_VDEV_TXRX_STATS_COMMON_TLV		0
+#define HTT_VDEV_TXRX_STATS_HW_STATS_TLV	1
+
+struct htt_t2h_vdev_txrx_stats_ind {
+	__le32 vdev_id;
+	__le32 rx_msdu_byte_cnt_lo;
+	__le32 rx_msdu_byte_cnt_hi;
+	__le32 rx_msdu_cnt_lo;
+	__le32 rx_msdu_cnt_hi;
+	__le32 tx_msdu_byte_cnt_lo;
+	__le32 tx_msdu_byte_cnt_hi;
+	__le32 tx_msdu_cnt_lo;
+	__le32 tx_msdu_cnt_hi;
+	__le32 tx_retry_cnt_lo;
+	__le32 tx_retry_cnt_hi;
+	__le32 tx_retry_byte_cnt_lo;
+	__le32 tx_retry_byte_cnt_hi;
+	__le32 tx_drop_cnt_lo;
+	__le32 tx_drop_cnt_hi;
+	__le32 tx_drop_byte_cnt_lo;
+	__le32 tx_drop_byte_cnt_hi;
+	__le32 msdu_ttl_cnt_lo;
+	__le32 msdu_ttl_cnt_hi;
+	__le32 msdu_ttl_byte_cnt_lo;
+	__le32 msdu_ttl_byte_cnt_hi;
+} __packed;
+
+struct htt_t2h_vdev_common_stats_tlv {
+	__le32 soc_drop_count_lo;
+	__le32 soc_drop_count_hi;
+} __packed;
+
+/* ppdu stats
+ *
+ * @details
+ * The following field definitions describe the format of the HTT target
+ * to host ppdu stats indication message.
+ *
+ *
+ * |31                         16|15   12|11   10|9      8|7            0 |
+ * |----------------------------------------------------------------------|
+ * |    payload_size             | rsvd  |pdev_id|mac_id  |    msg type   |
+ * |----------------------------------------------------------------------|
+ * |                          ppdu_id                                     |
+ * |----------------------------------------------------------------------|
+ * |                        Timestamp in us                               |
+ * |----------------------------------------------------------------------|
+ * |                          reserved                                    |
+ * |----------------------------------------------------------------------|
+ * |                    type-specific stats info                          |
+ * |                     (see htt_ppdu_stats.h)                           |
+ * |----------------------------------------------------------------------|
+ * Header fields:
+ *  - MSG_TYPE
+ *    Bits 7:0
+ *    Purpose: Identifies this is a PPDU STATS indication
+ *             message.
+ *    Value: 0x1d
+ *  - mac_id
+ *    Bits 9:8
+ *    Purpose: mac_id of this ppdu_id
+ *    Value: 0-3
+ *  - pdev_id
+ *    Bits 11:10
+ *    Purpose: pdev_id of this ppdu_id
+ *    Value: 0-3
+ *     0 (for rings at SOC level),
+ *     1/2/3 PDEV -> 0/1/2
+ *  - payload_size
+ *    Bits 31:16
+ *    Purpose: total tlv size
+ *    Value: payload_size in bytes
+ */
+
+#define HTT_T2H_PPDU_STATS_INFO_PDEV_ID GENMASK(11, 10)
+#define HTT_T2H_PPDU_STATS_INFO_PAYLOAD_SIZE GENMASK(31, 16)
+
+struct ath12k_htt_ppdu_stats_msg {
+	__le32 info;
+	__le32 ppdu_id;
+	__le32 timestamp;
+	__le32 rsvd;
+	u8 data[];
+} __packed;
+
+struct htt_tlv {
+	__le32 header;
+	u8 value[];
+} __packed;
+
+#define HTT_TLV_TAG			GENMASK(11, 0)
+#define HTT_TLV_LEN			GENMASK(23, 12)
+
+enum HTT_PPDU_STATS_BW {
+	HTT_PPDU_STATS_BANDWIDTH_5MHZ   = 0,
+	HTT_PPDU_STATS_BANDWIDTH_10MHZ  = 1,
+	HTT_PPDU_STATS_BANDWIDTH_20MHZ  = 2,
+	HTT_PPDU_STATS_BANDWIDTH_40MHZ  = 3,
+	HTT_PPDU_STATS_BANDWIDTH_80MHZ  = 4,
+	HTT_PPDU_STATS_BANDWIDTH_160MHZ = 5, /* includes 80+80 */
+	HTT_PPDU_STATS_BANDWIDTH_DYN    = 6,
+};
+
+#define HTT_PPDU_STATS_CMN_FLAGS_FRAME_TYPE_M	GENMASK(7, 0)
+#define HTT_PPDU_STATS_CMN_FLAGS_QUEUE_TYPE_M	GENMASK(15, 8)
+/* bw - HTT_PPDU_STATS_BW */
+#define HTT_PPDU_STATS_CMN_FLAGS_BW_M		GENMASK(19, 16)
+
+struct htt_ppdu_stats_common {
+	__le32 ppdu_id;
+	__le16 sched_cmdid;
+	u8 ring_id;
+	u8 num_users;
+	__le32 flags; /* %HTT_PPDU_STATS_COMMON_FLAGS_*/
+	__le32 chain_mask;
+	__le32 fes_duration_us; /* frame exchange sequence */
+	__le32 ppdu_sch_eval_start_tstmp_us;
+	__le32 ppdu_sch_end_tstmp_us;
+	__le32 ppdu_start_tstmp_us;
+	/* BIT [15 :  0] - phy mode (WLAN_PHY_MODE) with which ppdu was transmitted
+	 * BIT [31 : 16] - bandwidth (in MHz) with which ppdu was transmitted
+	 */
+	__le16 phy_mode;
+	__le16 bw_mhz;
+} __packed;
+
+enum htt_ppdu_stats_gi {
+	HTT_PPDU_STATS_SGI_0_8_US,
+	HTT_PPDU_STATS_SGI_0_4_US,
+	HTT_PPDU_STATS_SGI_1_6_US,
+	HTT_PPDU_STATS_SGI_3_2_US,
+};
+
+#define HTT_PPDU_STATS_USER_RATE_INFO0_USER_POS_M	GENMASK(3, 0)
+#define HTT_PPDU_STATS_USER_RATE_INFO0_MU_GROUP_ID_M	GENMASK(11, 4)
+
+enum HTT_PPDU_STATS_PPDU_TYPE {
+	HTT_PPDU_STATS_PPDU_TYPE_SU,
+	HTT_PPDU_STATS_PPDU_TYPE_MU_MIMO,
+	HTT_PPDU_STATS_PPDU_TYPE_MU_OFDMA,
+	HTT_PPDU_STATS_PPDU_TYPE_MU_MIMO_OFDMA,
+	HTT_PPDU_STATS_PPDU_TYPE_UL_TRIG,
+	HTT_PPDU_STATS_PPDU_TYPE_BURST_BCN,
+	HTT_PPDU_STATS_PPDU_TYPE_UL_BSR_RESP,
+	HTT_PPDU_STATS_PPDU_TYPE_UL_BSR_TRIG,
+	HTT_PPDU_STATS_PPDU_TYPE_UL_RESP,
+	HTT_PPDU_STATS_PPDU_TYPE_MAX
+};
+
+#define HTT_PPDU_STATS_USER_RATE_INFO1_RESP_TYPE_VALD_M	BIT(0)
+#define HTT_PPDU_STATS_USER_RATE_INFO1_PPDU_TYPE_M	GENMASK(5, 1)
+
+#define HTT_PPDU_STATS_USER_RATE_FLAGS_LTF_SIZE_M	GENMASK(1, 0)
+#define HTT_PPDU_STATS_USER_RATE_FLAGS_STBC_M		BIT(2)
+#define HTT_PPDU_STATS_USER_RATE_FLAGS_HE_RE_M		BIT(3)
+#define HTT_PPDU_STATS_USER_RATE_FLAGS_TXBF_M		GENMASK(7, 4)
+#define HTT_PPDU_STATS_USER_RATE_FLAGS_BW_M		GENMASK(11, 8)
+#define HTT_PPDU_STATS_USER_RATE_FLAGS_NSS_M		GENMASK(15, 12)
+#define HTT_PPDU_STATS_USER_RATE_FLAGS_MCS_M		GENMASK(19, 16)
+#define HTT_PPDU_STATS_USER_RATE_FLAGS_PREAMBLE_M	GENMASK(23, 20)
+#define HTT_PPDU_STATS_USER_RATE_FLAGS_GI_M		GENMASK(27, 24)
+#define HTT_PPDU_STATS_USER_RATE_FLAGS_DCM_M		BIT(28)
+#define HTT_PPDU_STATS_USER_RATE_FLAGS_LDPC_M		BIT(29)
+
+#define HTT_USR_RATE_PPDU_TYPE(_val) \
+		le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_INFO1_PPDU_TYPE_M)
+#define HTT_USR_RATE_PREAMBLE(_val) \
+		le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_PREAMBLE_M)
+#define HTT_USR_RATE_BW(_val) \
+		le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_BW_M)
+#define HTT_USR_RATE_NSS(_val) \
+		le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_NSS_M)
+#define HTT_USR_RATE_MCS(_val) \
+		le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_MCS_M)
+#define HTT_USR_RATE_GI(_val) \
+		le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_GI_M)
+#define HTT_USR_RATE_DCM(_val) \
+		le32_get_bits(_val, HTT_PPDU_STATS_USER_RATE_FLAGS_DCM_M)
+
+#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_LTF_SIZE_M		GENMASK(1, 0)
+#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_STBC_M		BIT(2)
+#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_HE_RE_M		BIT(3)
+#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_TXBF_M		GENMASK(7, 4)
+#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_BW_M		GENMASK(11, 8)
+#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_NSS_M		GENMASK(15, 12)
+#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_MCS_M		GENMASK(19, 16)
+#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_PREAMBLE_M		GENMASK(23, 20)
+#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_GI_M		GENMASK(27, 24)
+#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_DCM_M		BIT(28)
+#define HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_LDPC_M		BIT(29)
+
+struct htt_ppdu_stats_user_rate {
+	u8 tid_num;
+	u8 reserved0;
+	__le16 sw_peer_id;
+	__le32 info0; /* %HTT_PPDU_STATS_USER_RATE_INFO0_*/
+	__le16 ru_end;
+	__le16 ru_start;
+	__le16 resp_ru_end;
+	__le16 resp_ru_start;
+	__le32 info1; /* %HTT_PPDU_STATS_USER_RATE_INFO1_ */
+	__le32 rate_flags; /* %HTT_PPDU_STATS_USER_RATE_FLAGS_ */
+	/* Note: resp_rate_info is only valid for if resp_type is UL */
+	__le32 resp_rate_flags; /* %HTT_PPDU_STATS_USER_RATE_RESP_FLAGS_ */
+} __packed;
+
+#define HTT_PPDU_STATS_TX_INFO_FLAGS_RATECODE_M		GENMASK(7, 0)
+#define HTT_PPDU_STATS_TX_INFO_FLAGS_IS_AMPDU_M		BIT(8)
+#define HTT_PPDU_STATS_TX_INFO_FLAGS_BA_ACK_FAILED_M	GENMASK(10, 9)
+#define HTT_PPDU_STATS_TX_INFO_FLAGS_BW_M		GENMASK(13, 11)
+#define HTT_PPDU_STATS_TX_INFO_FLAGS_SGI_M		BIT(14)
+#define HTT_PPDU_STATS_TX_INFO_FLAGS_PEERID_M		GENMASK(31, 16)
+
+#define HTT_TX_INFO_IS_AMSDU(_flags) \
+			u32_get_bits(_flags, HTT_PPDU_STATS_TX_INFO_FLAGS_IS_AMPDU_M)
+#define HTT_TX_INFO_BA_ACK_FAILED(_flags) \
+			u32_get_bits(_flags, HTT_PPDU_STATS_TX_INFO_FLAGS_BA_ACK_FAILED_M)
+#define HTT_TX_INFO_RATECODE(_flags) \
+			u32_get_bits(_flags, HTT_PPDU_STATS_TX_INFO_FLAGS_RATECODE_M)
+#define HTT_TX_INFO_PEERID(_flags) \
+			u32_get_bits(_flags, HTT_PPDU_STATS_TX_INFO_FLAGS_PEERID_M)
+
+enum  htt_ppdu_stats_usr_compln_status {
+	HTT_PPDU_STATS_USER_STATUS_OK,
+	HTT_PPDU_STATS_USER_STATUS_FILTERED,
+	HTT_PPDU_STATS_USER_STATUS_RESP_TIMEOUT,
+	HTT_PPDU_STATS_USER_STATUS_RESP_MISMATCH,
+	HTT_PPDU_STATS_USER_STATUS_ABORT,
+};
+
+#define HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_LONG_RETRY_M	GENMASK(3, 0)
+#define HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_SHORT_RETRY_M	GENMASK(7, 4)
+#define HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_IS_AMPDU_M		BIT(8)
+#define HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_RESP_TYPE_M		GENMASK(12, 9)
+
+#define HTT_USR_CMPLTN_IS_AMPDU(_val) \
+	    le32_get_bits(_val, HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_IS_AMPDU_M)
+#define HTT_USR_CMPLTN_LONG_RETRY(_val) \
+	    le32_get_bits(_val, HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_LONG_RETRY_M)
+#define HTT_USR_CMPLTN_SHORT_RETRY(_val) \
+	    le32_get_bits(_val, HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_SHORT_RETRY_M)
+
+struct htt_ppdu_stats_usr_cmpltn_cmn {
+	u8 status;
+	u8 tid_num;
+	__le16 sw_peer_id;
+	/* RSSI value of last ack packet (units = dB above noise floor) */
+	__le32 ack_rssi;
+	__le16 mpdu_tried;
+	__le16 mpdu_success;
+	__le32 flags; /* %HTT_PPDU_STATS_USR_CMPLTN_CMN_FLAGS_LONG_RETRIES*/
+} __packed;
+
+#define HTT_PPDU_STATS_ACK_BA_INFO_NUM_MPDU_M	GENMASK(8, 0)
+#define HTT_PPDU_STATS_ACK_BA_INFO_NUM_MSDU_M	GENMASK(24, 9)
+#define HTT_PPDU_STATS_ACK_BA_INFO_TID_NUM	GENMASK(31, 25)
+
+#define HTT_PPDU_STATS_NON_QOS_TID	16
+
+struct htt_ppdu_stats_usr_cmpltn_ack_ba_status {
+	__le32 ppdu_id;
+	__le16 sw_peer_id;
+	__le16 reserved0;
+	__le32 info; /* %HTT_PPDU_STATS_USR_CMPLTN_CMN_INFO_ */
+	__le16 current_seq;
+	__le16 start_seq;
+	__le32 success_bytes;
+} __packed;
+
+struct htt_ppdu_user_stats {
+	u16 peer_id;
+	u16 delay_ba;
+	u32 tlv_flags;
+	bool is_valid_peer_id;
+	struct htt_ppdu_stats_user_rate rate;
+	struct htt_ppdu_stats_usr_cmpltn_cmn cmpltn_cmn;
+	struct htt_ppdu_stats_usr_cmpltn_ack_ba_status ack_ba;
+};
+
+#define HTT_PPDU_STATS_MAX_USERS	8
+#define HTT_PPDU_DESC_MAX_DEPTH	16
+
+struct htt_ppdu_stats {
+	struct htt_ppdu_stats_common common;
+	struct htt_ppdu_user_stats user_stats[HTT_PPDU_STATS_MAX_USERS];
+};
+
+struct htt_ppdu_stats_info {
+	u32 tlv_bitmap;
+	u32 ppdu_id;
+	u32 frame_type;
+	u32 frame_ctrl;
+	u32 delay_ba;
+	u32 bar_num_users;
+	struct htt_ppdu_stats ppdu_stats;
+	struct list_head list;
+};
+
+/* @brief target -> host MLO offset indiciation message
+ *
+ * @details
+ * The following field definitions describe the format of the HTT target
+ * to host mlo offset indication message.
+ *
+ *
+ * |31        29|28    |26|25  22|21 16|15  13|12     10 |9     8|7     0|
+ * |---------------------------------------------------------------------|
+ * |   rsvd1    | mac_freq                    |chip_id   |pdev_id|msgtype|
+ * |---------------------------------------------------------------------|
+ * |                           sync_timestamp_lo_us                      |
+ * |---------------------------------------------------------------------|
+ * |                           sync_timestamp_hi_us                      |
+ * |---------------------------------------------------------------------|
+ * |                           mlo_offset_lo                             |
+ * |---------------------------------------------------------------------|
+ * |                           mlo_offset_hi                             |
+ * |---------------------------------------------------------------------|
+ * |                           mlo_offset_clcks                          |
+ * |---------------------------------------------------------------------|
+ * |   rsvd2           | mlo_comp_clks |mlo_comp_us                      |
+ * |---------------------------------------------------------------------|
+ * |   rsvd3                   |mlo_comp_timer                           |
+ * |---------------------------------------------------------------------|
+ * Header fields
+ *  - MSG_TYPE
+ *    Bits 7:0
+ *    Purpose: Identifies this is a MLO offset indication msg
+ *  - PDEV_ID
+ *    Bits 9:8
+ *    Purpose: Pdev of this MLO offset
+ *  - CHIP_ID
+ *    Bits 12:10
+ *    Purpose: chip_id of this MLO offset
+ *  - MAC_FREQ
+ *    Bits 28:13
+ *  - SYNC_TIMESTAMP_LO_US
+ *    Purpose: clock frequency of the mac HW block in MHz
+ *    Bits: 31:0
+ *    Purpose: lower 32 bits of the WLAN global time stamp at which
+ *             last sync interrupt was received
+ *  - SYNC_TIMESTAMP_HI_US
+ *    Bits: 31:0
+ *    Purpose: upper 32 bits of WLAN global time stamp at which
+ *             last sync interrupt was received
+ *  - MLO_OFFSET_LO
+ *    Bits: 31:0
+ *    Purpose: lower 32 bits of the MLO offset in us
+ *  - MLO_OFFSET_HI
+ *    Bits: 31:0
+ *    Purpose: upper 32 bits of the MLO offset in us
+ *  - MLO_COMP_US
+ *    Bits: 15:0
+ *    Purpose: MLO time stamp compensation applied in us
+ *  - MLO_COMP_CLCKS
+ *    Bits: 25:16
+ *    Purpose: MLO time stamp compensation applied in clock ticks
+ *  - MLO_COMP_TIMER
+ *    Bits: 21:0
+ *    Purpose: Periodic timer at which compensation is applied
+ */
+
+#define HTT_T2H_MLO_OFFSET_INFO_MSG_TYPE        GENMASK(7, 0)
+#define HTT_T2H_MLO_OFFSET_INFO_PDEV_ID         GENMASK(9, 8)
+
+struct ath12k_htt_mlo_offset_msg {
+	__le32 info;
+	__le32 sync_timestamp_lo_us;
+	__le32 sync_timestamp_hi_us;
+	__le32 mlo_offset_hi;
+	__le32 mlo_offset_lo;
+	__le32 mlo_offset_clks;
+	__le32 mlo_comp_clks;
+	__le32 mlo_comp_timer;
+} __packed;
+
+/* @brief host -> target FW extended statistics retrieve
+ *
+ * @details
+ * The following field definitions describe the format of the HTT host
+ * to target FW extended stats retrieve message.
+ * The message specifies the type of stats the host wants to retrieve.
+ *
+ * |31          24|23          16|15           8|7            0|
+ * |-----------------------------------------------------------|
+ * |   reserved   | stats type   |   pdev_mask  |   msg type   |
+ * |-----------------------------------------------------------|
+ * |                   config param [0]                        |
+ * |-----------------------------------------------------------|
+ * |                   config param [1]                        |
+ * |-----------------------------------------------------------|
+ * |                   config param [2]                        |
+ * |-----------------------------------------------------------|
+ * |                   config param [3]                        |
+ * |-----------------------------------------------------------|
+ * |                         reserved                          |
+ * |-----------------------------------------------------------|
+ * |                        cookie LSBs                        |
+ * |-----------------------------------------------------------|
+ * |                        cookie MSBs                        |
+ * |-----------------------------------------------------------|
+ * Header fields:
+ *  - MSG_TYPE
+ *    Bits 7:0
+ *    Purpose: identifies this is a extended stats upload request message
+ *    Value: 0x10
+ *  - PDEV_MASK
+ *    Bits 8:15
+ *    Purpose: identifies the mask of PDEVs to retrieve stats from
+ *    Value: This is a overloaded field, refer to usage and interpretation of
+ *           PDEV in interface document.
+ *           Bit   8    :  Reserved for SOC stats
+ *           Bit 9 - 15 :  Indicates PDEV_MASK in DBDC
+ *                         Indicates MACID_MASK in DBS
+ *  - STATS_TYPE
+ *    Bits 23:16
+ *    Purpose: identifies which FW statistics to upload
+ *    Value: Defined by htt_dbg_ext_stats_type (see htt_stats.h)
+ *  - Reserved
+ *    Bits 31:24
+ *  - CONFIG_PARAM [0]
+ *    Bits 31:0
+ *    Purpose: give an opaque configuration value to the specified stats type
+ *    Value: stats-type specific configuration value
+ *           Refer to htt_stats.h for interpretation for each stats sub_type
+ *  - CONFIG_PARAM [1]
+ *    Bits 31:0
+ *    Purpose: give an opaque configuration value to the specified stats type
+ *    Value: stats-type specific configuration value
+ *           Refer to htt_stats.h for interpretation for each stats sub_type
+ *  - CONFIG_PARAM [2]
+ *    Bits 31:0
+ *    Purpose: give an opaque configuration value to the specified stats type
+ *    Value: stats-type specific configuration value
+ *           Refer to htt_stats.h for interpretation for each stats sub_type
+ *  - CONFIG_PARAM [3]
+ *    Bits 31:0
+ *    Purpose: give an opaque configuration value to the specified stats type
+ *    Value: stats-type specific configuration value
+ *           Refer to htt_stats.h for interpretation for each stats sub_type
+ *  - Reserved [31:0] for future use.
+ *  - COOKIE_LSBS
+ *    Bits 31:0
+ *    Purpose: Provide a mechanism to match a target->host stats confirmation
+ *        message with its preceding host->target stats request message.
+ *    Value: LSBs of the opaque cookie specified by the host-side requestor
+ *  - COOKIE_MSBS
+ *    Bits 31:0
+ *    Purpose: Provide a mechanism to match a target->host stats confirmation
+ *        message with its preceding host->target stats request message.
+ *    Value: MSBs of the opaque cookie specified by the host-side requestor
+ */
+
+struct htt_ext_stats_cfg_hdr {
+	u8 msg_type;
+	u8 pdev_mask;
+	u8 stats_type;
+	u8 reserved;
+} __packed;
+
+struct htt_ext_stats_cfg_cmd {
+	struct htt_ext_stats_cfg_hdr hdr;
+	__le32 cfg_param0;
+	__le32 cfg_param1;
+	__le32 cfg_param2;
+	__le32 cfg_param3;
+	__le32 reserved;
+	__le32 cookie_lsb;
+	__le32 cookie_msb;
+} __packed;
+
+/* htt stats config default params */
+#define HTT_STAT_DEFAULT_RESET_START_OFFSET 0
+#define HTT_STAT_DEFAULT_CFG0_ALL_HWQS 0xffffffff
+#define HTT_STAT_DEFAULT_CFG0_ALL_TXQS 0xffffffff
+#define HTT_STAT_DEFAULT_CFG0_ALL_CMDQS 0xffff
+#define HTT_STAT_DEFAULT_CFG0_ALL_RINGS 0xffff
+#define HTT_STAT_DEFAULT_CFG0_ACTIVE_PEERS 0xff
+#define HTT_STAT_DEFAULT_CFG0_CCA_CUMULATIVE 0x00
+#define HTT_STAT_DEFAULT_CFG0_ACTIVE_VDEVS 0x00
+
+/* HTT_DBG_EXT_STATS_PEER_INFO
+ * PARAMS:
+ * @config_param0:
+ *  [Bit0] - [0] for sw_peer_id, [1] for mac_addr based request
+ *  [Bit15 : Bit 1] htt_peer_stats_req_mode_t
+ *  [Bit31 : Bit16] sw_peer_id
+ * @config_param1:
+ *  peer_stats_req_type_mask:32 (enum htt_peer_stats_tlv_enum)
+ *   0 bit htt_peer_stats_cmn_tlv
+ *   1 bit htt_peer_details_tlv
+ *   2 bit htt_tx_peer_rate_stats_tlv
+ *   3 bit htt_rx_peer_rate_stats_tlv
+ *   4 bit htt_tx_tid_stats_tlv/htt_tx_tid_stats_v1_tlv
+ *   5 bit htt_rx_tid_stats_tlv
+ *   6 bit htt_msdu_flow_stats_tlv
+ * @config_param2: [Bit31 : Bit0] mac_addr31to0
+ * @config_param3: [Bit15 : Bit0] mac_addr47to32
+ *                [Bit31 : Bit16] reserved
+ */
+#define HTT_STAT_PEER_INFO_MAC_ADDR BIT(0)
+#define HTT_STAT_DEFAULT_PEER_REQ_TYPE 0x7f
+
+/* Used to set different configs to the specified stats type.*/
+struct htt_ext_stats_cfg_params {
+	u32 cfg0;
+	u32 cfg1;
+	u32 cfg2;
+	u32 cfg3;
+};
+
+enum vdev_stats_offload_timer_duration {
+	ATH12K_STATS_TIMER_DUR_500MS = 1,
+	ATH12K_STATS_TIMER_DUR_1SEC = 2,
+	ATH12K_STATS_TIMER_DUR_2SEC = 3,
+};
+
+#define ATH12K_HTT_MAC_ADDR_L32_0	GENMASK(7, 0)
+#define ATH12K_HTT_MAC_ADDR_L32_1	GENMASK(15, 8)
+#define ATH12K_HTT_MAC_ADDR_L32_2	GENMASK(23, 16)
+#define ATH12K_HTT_MAC_ADDR_L32_3	GENMASK(31, 24)
+#define ATH12K_HTT_MAC_ADDR_H16_0	GENMASK(7, 0)
+#define ATH12K_HTT_MAC_ADDR_H16_1	GENMASK(15, 8)
+
+struct htt_mac_addr {
+	__le32 mac_addr_l32;
+	__le32 mac_addr_h16;
+} __packed;
+
+int ath12k_dp_htt_connect(struct ath12k_dp *dp);
+int ath12k_dp_tx_htt_srng_setup(struct ath12k_base *ab, u32 ring_id,
+				int mac_id, enum hal_ring_type ring_type);
+
+void ath12k_dp_htt_htc_t2h_msg_handler(struct ath12k_base *ab,
+				       struct sk_buff *skb);
+int ath12k_dp_htt_tlv_iter(struct ath12k_base *ab, const void *ptr, size_t len,
+			   int (*iter)(struct ath12k_base *ar, u16 tag, u16 len,
+				       const void *ptr, void *data),
+				       void *data);
+int ath12k_dp_tx_htt_h2t_ver_req_msg(struct ath12k_base *ab);
+int ath12k_dp_tx_htt_h2t_ppdu_stats_req(struct ath12k *ar, u32 mask);
+int
+ath12k_dp_tx_htt_h2t_ext_stats_req(struct ath12k *ar, u8 type,
+				   struct htt_ext_stats_cfg_params *cfg_params,
+				   u64 cookie);
+int ath12k_dp_tx_htt_rx_monitor_mode_ring_config(struct ath12k *ar, bool reset);
+
+int ath12k_dp_tx_htt_rx_filter_setup(struct ath12k_base *ab, u32 ring_id,
+				     int mac_id, enum hal_ring_type ring_type,
+				     int rx_buf_size,
+				     struct htt_rx_ring_tlv_filter *tlv_filter);
+int ath12k_dp_tx_htt_tx_filter_setup(struct ath12k_base *ab, u32 ring_id,
+				     int mac_id, enum hal_ring_type ring_type,
+				     int tx_buf_size,
+				     struct htt_tx_ring_tlv_filter *htt_tlv_filter);
+int ath12k_dp_tx_htt_monitor_mode_ring_config(struct ath12k *ar, bool reset);
+#endif
diff --git a/drivers/net/wireless/ath/ath12k/dp_mon.c b/drivers/net/wireless/ath/ath12k/dp_mon.c
index 39d1967584db..737287a9aa46 100644
--- a/drivers/net/wireless/ath/ath12k/dp_mon.c
+++ b/drivers/net/wireless/ath/ath12k/dp_mon.c
@@ -6,1756 +6,11 @@
 
 #include "dp_mon.h"
 #include "debug.h"
-#include "dp_rx.h"
 #include "dp_tx.h"
 #include "peer.h"
 
-#define ATH12K_LE32_DEC_ENC(value, dec_bits, enc_bits)	\
-		u32_encode_bits(le32_get_bits(value, dec_bits), enc_bits)
-
-#define ATH12K_LE64_DEC_ENC(value, dec_bits, enc_bits) \
-		u32_encode_bits(le64_get_bits(value, dec_bits), enc_bits)
-
-static void
-ath12k_dp_mon_rx_handle_ofdma_info(const struct hal_rx_ppdu_end_user_stats *ppdu_end_user,
-				   struct hal_rx_user_status *rx_user_status)
-{
-	rx_user_status->ul_ofdma_user_v0_word0 =
-		__le32_to_cpu(ppdu_end_user->usr_resp_ref);
-	rx_user_status->ul_ofdma_user_v0_word1 =
-		__le32_to_cpu(ppdu_end_user->usr_resp_ref_ext);
-}
-
-static void
-ath12k_dp_mon_rx_populate_byte_count(const struct hal_rx_ppdu_end_user_stats *stats,
-				     void *ppduinfo,
-				     struct hal_rx_user_status *rx_user_status)
-{
-	rx_user_status->mpdu_ok_byte_count =
-		le32_get_bits(stats->info7,
-			      HAL_RX_PPDU_END_USER_STATS_INFO7_MPDU_OK_BYTE_COUNT);
-	rx_user_status->mpdu_err_byte_count =
-		le32_get_bits(stats->info8,
-			      HAL_RX_PPDU_END_USER_STATS_INFO8_MPDU_ERR_BYTE_COUNT);
-}
-
-static void
-ath12k_dp_mon_rx_populate_mu_user_info(const struct hal_rx_ppdu_end_user_stats *rx_tlv,
-				       struct hal_rx_mon_ppdu_info *ppdu_info,
-				       struct hal_rx_user_status *rx_user_status)
-{
-	rx_user_status->ast_index = ppdu_info->ast_index;
-	rx_user_status->tid = ppdu_info->tid;
-	rx_user_status->tcp_ack_msdu_count =
-		ppdu_info->tcp_ack_msdu_count;
-	rx_user_status->tcp_msdu_count =
-		ppdu_info->tcp_msdu_count;
-	rx_user_status->udp_msdu_count =
-		ppdu_info->udp_msdu_count;
-	rx_user_status->other_msdu_count =
-		ppdu_info->other_msdu_count;
-	rx_user_status->frame_control = ppdu_info->frame_control;
-	rx_user_status->frame_control_info_valid =
-		ppdu_info->frame_control_info_valid;
-	rx_user_status->data_sequence_control_info_valid =
-		ppdu_info->data_sequence_control_info_valid;
-	rx_user_status->first_data_seq_ctrl =
-		ppdu_info->first_data_seq_ctrl;
-	rx_user_status->preamble_type = ppdu_info->preamble_type;
-	rx_user_status->ht_flags = ppdu_info->ht_flags;
-	rx_user_status->vht_flags = ppdu_info->vht_flags;
-	rx_user_status->he_flags = ppdu_info->he_flags;
-	rx_user_status->rs_flags = ppdu_info->rs_flags;
-
-	rx_user_status->mpdu_cnt_fcs_ok =
-		ppdu_info->num_mpdu_fcs_ok;
-	rx_user_status->mpdu_cnt_fcs_err =
-		ppdu_info->num_mpdu_fcs_err;
-	memcpy(&rx_user_status->mpdu_fcs_ok_bitmap[0], &ppdu_info->mpdu_fcs_ok_bitmap[0],
-	       HAL_RX_NUM_WORDS_PER_PPDU_BITMAP *
-	       sizeof(ppdu_info->mpdu_fcs_ok_bitmap[0]));
-
-	ath12k_dp_mon_rx_populate_byte_count(rx_tlv, ppdu_info, rx_user_status);
-}
-
-static void ath12k_dp_mon_parse_vht_sig_a(const struct hal_rx_vht_sig_a_info *vht_sig,
-					  struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	u32 nsts, info0, info1;
-	u8 gi_setting;
-
-	info0 = __le32_to_cpu(vht_sig->info0);
-	info1 = __le32_to_cpu(vht_sig->info1);
-
-	ppdu_info->ldpc = u32_get_bits(info1, HAL_RX_VHT_SIG_A_INFO_INFO1_SU_MU_CODING);
-	ppdu_info->mcs = u32_get_bits(info1, HAL_RX_VHT_SIG_A_INFO_INFO1_MCS);
-	gi_setting = u32_get_bits(info1, HAL_RX_VHT_SIG_A_INFO_INFO1_GI_SETTING);
-	switch (gi_setting) {
-	case HAL_RX_VHT_SIG_A_NORMAL_GI:
-		ppdu_info->gi = HAL_RX_GI_0_8_US;
-		break;
-	case HAL_RX_VHT_SIG_A_SHORT_GI:
-	case HAL_RX_VHT_SIG_A_SHORT_GI_AMBIGUITY:
-		ppdu_info->gi = HAL_RX_GI_0_4_US;
-		break;
-	}
-
-	ppdu_info->is_stbc = u32_get_bits(info0, HAL_RX_VHT_SIG_A_INFO_INFO0_STBC);
-	nsts = u32_get_bits(info0, HAL_RX_VHT_SIG_A_INFO_INFO0_NSTS);
-	if (ppdu_info->is_stbc && nsts > 0)
-		nsts = ((nsts + 1) >> 1) - 1;
-
-	ppdu_info->nss = u32_get_bits(nsts, VHT_SIG_SU_NSS_MASK) + 1;
-	ppdu_info->bw = u32_get_bits(info0, HAL_RX_VHT_SIG_A_INFO_INFO0_BW);
-	ppdu_info->beamformed = u32_get_bits(info1,
-					     HAL_RX_VHT_SIG_A_INFO_INFO1_BEAMFORMED);
-	ppdu_info->vht_flag_values5 = u32_get_bits(info0,
-						   HAL_RX_VHT_SIG_A_INFO_INFO0_GROUP_ID);
-	ppdu_info->vht_flag_values3[0] = (((ppdu_info->mcs) << 4) |
-					    ppdu_info->nss);
-	ppdu_info->vht_flag_values2 = ppdu_info->bw;
-	ppdu_info->vht_flag_values4 =
-		u32_get_bits(info1, HAL_RX_VHT_SIG_A_INFO_INFO1_SU_MU_CODING);
-}
-
-static void ath12k_dp_mon_parse_ht_sig(const struct hal_rx_ht_sig_info *ht_sig,
-				       struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	u32 info0 = __le32_to_cpu(ht_sig->info0);
-	u32 info1 = __le32_to_cpu(ht_sig->info1);
-
-	ppdu_info->mcs = u32_get_bits(info0, HAL_RX_HT_SIG_INFO_INFO0_MCS);
-	ppdu_info->bw = u32_get_bits(info0, HAL_RX_HT_SIG_INFO_INFO0_BW);
-	ppdu_info->is_stbc = u32_get_bits(info1, HAL_RX_HT_SIG_INFO_INFO1_STBC);
-	ppdu_info->ldpc = u32_get_bits(info1, HAL_RX_HT_SIG_INFO_INFO1_FEC_CODING);
-	ppdu_info->gi = u32_get_bits(info1, HAL_RX_HT_SIG_INFO_INFO1_GI);
-	ppdu_info->nss = (ppdu_info->mcs >> 3) + 1;
-}
-
-static void ath12k_dp_mon_parse_l_sig_b(const struct hal_rx_lsig_b_info *lsigb,
-					struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	u32 info0 = __le32_to_cpu(lsigb->info0);
-	u8 rate;
-
-	rate = u32_get_bits(info0, HAL_RX_LSIG_B_INFO_INFO0_RATE);
-	switch (rate) {
-	case 1:
-		rate = HAL_RX_LEGACY_RATE_1_MBPS;
-		break;
-	case 2:
-	case 5:
-		rate = HAL_RX_LEGACY_RATE_2_MBPS;
-		break;
-	case 3:
-	case 6:
-		rate = HAL_RX_LEGACY_RATE_5_5_MBPS;
-		break;
-	case 4:
-	case 7:
-		rate = HAL_RX_LEGACY_RATE_11_MBPS;
-		break;
-	default:
-		rate = HAL_RX_LEGACY_RATE_INVALID;
-	}
-
-	ppdu_info->rate = rate;
-	ppdu_info->cck_flag = 1;
-}
-
-static void ath12k_dp_mon_parse_l_sig_a(const struct hal_rx_lsig_a_info *lsiga,
-					struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	u32 info0 = __le32_to_cpu(lsiga->info0);
-	u8 rate;
-
-	rate = u32_get_bits(info0, HAL_RX_LSIG_A_INFO_INFO0_RATE);
-	switch (rate) {
-	case 8:
-		rate = HAL_RX_LEGACY_RATE_48_MBPS;
-		break;
-	case 9:
-		rate = HAL_RX_LEGACY_RATE_24_MBPS;
-		break;
-	case 10:
-		rate = HAL_RX_LEGACY_RATE_12_MBPS;
-		break;
-	case 11:
-		rate = HAL_RX_LEGACY_RATE_6_MBPS;
-		break;
-	case 12:
-		rate = HAL_RX_LEGACY_RATE_54_MBPS;
-		break;
-	case 13:
-		rate = HAL_RX_LEGACY_RATE_36_MBPS;
-		break;
-	case 14:
-		rate = HAL_RX_LEGACY_RATE_18_MBPS;
-		break;
-	case 15:
-		rate = HAL_RX_LEGACY_RATE_9_MBPS;
-		break;
-	default:
-		rate = HAL_RX_LEGACY_RATE_INVALID;
-	}
-
-	ppdu_info->rate = rate;
-}
-
-static void
-ath12k_dp_mon_parse_he_sig_b2_ofdma(const struct hal_rx_he_sig_b2_ofdma_info *ofdma,
-				    struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	u32 info0, value;
-
-	info0 = __le32_to_cpu(ofdma->info0);
-
-	ppdu_info->he_data1 |= HE_MCS_KNOWN | HE_DCM_KNOWN | HE_CODING_KNOWN;
-
-	/* HE-data2 */
-	ppdu_info->he_data2 |= HE_TXBF_KNOWN;
-
-	ppdu_info->mcs = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_MCS);
-	value = ppdu_info->mcs << HE_TRANSMIT_MCS_SHIFT;
-	ppdu_info->he_data3 |= value;
-
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_DCM);
-	value = value << HE_DCM_SHIFT;
-	ppdu_info->he_data3 |= value;
-
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_CODING);
-	ppdu_info->ldpc = value;
-	value = value << HE_CODING_SHIFT;
-	ppdu_info->he_data3 |= value;
-
-	/* HE-data4 */
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_ID);
-	value = value << HE_STA_ID_SHIFT;
-	ppdu_info->he_data4 |= value;
-
-	ppdu_info->nss =
-		u32_get_bits(info0,
-			     HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_NSTS) + 1;
-	ppdu_info->beamformed = u32_get_bits(info0,
-					     HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_TXBF);
-}
-
-static void
-ath12k_dp_mon_parse_he_sig_b2_mu(const struct hal_rx_he_sig_b2_mu_info *he_sig_b2_mu,
-				 struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	u32 info0, value;
-
-	info0 = __le32_to_cpu(he_sig_b2_mu->info0);
-
-	ppdu_info->he_data1 |= HE_MCS_KNOWN | HE_CODING_KNOWN;
-
-	ppdu_info->mcs = u32_get_bits(info0, HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_MCS);
-	value = ppdu_info->mcs << HE_TRANSMIT_MCS_SHIFT;
-	ppdu_info->he_data3 |= value;
-
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_CODING);
-	ppdu_info->ldpc = value;
-	value = value << HE_CODING_SHIFT;
-	ppdu_info->he_data3 |= value;
-
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_ID);
-	value = value << HE_STA_ID_SHIFT;
-	ppdu_info->he_data4 |= value;
-
-	ppdu_info->nss =
-		u32_get_bits(info0,
-			     HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_NSTS) + 1;
-}
-
-static void
-ath12k_dp_mon_parse_he_sig_b1_mu(const struct hal_rx_he_sig_b1_mu_info *he_sig_b1_mu,
-				 struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	u32 info0 = __le32_to_cpu(he_sig_b1_mu->info0);
-	u16 ru_tones;
-
-	ru_tones = u32_get_bits(info0,
-				HAL_RX_HE_SIG_B1_MU_INFO_INFO0_RU_ALLOCATION);
-	ppdu_info->ru_alloc = ath12k_he_ru_tones_to_nl80211_he_ru_alloc(ru_tones);
-	ppdu_info->he_RU[0] = ru_tones;
-}
-
-static void
-ath12k_dp_mon_parse_he_sig_mu(const struct hal_rx_he_sig_a_mu_dl_info *he_sig_a_mu_dl,
-			      struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	u32 info0, info1, value;
-	u16 he_gi = 0, he_ltf = 0;
-
-	info0 = __le32_to_cpu(he_sig_a_mu_dl->info0);
-	info1 = __le32_to_cpu(he_sig_a_mu_dl->info1);
-
-	ppdu_info->he_mu_flags = 1;
-
-	ppdu_info->he_data1 = HE_MU_FORMAT_TYPE;
-	ppdu_info->he_data1 |=
-			HE_BSS_COLOR_KNOWN |
-			HE_DL_UL_KNOWN |
-			HE_LDPC_EXTRA_SYMBOL_KNOWN |
-			HE_STBC_KNOWN |
-			HE_DATA_BW_RU_KNOWN |
-			HE_DOPPLER_KNOWN;
-
-	ppdu_info->he_data2 =
-			HE_GI_KNOWN |
-			HE_LTF_SYMBOLS_KNOWN |
-			HE_PRE_FEC_PADDING_KNOWN |
-			HE_PE_DISAMBIGUITY_KNOWN |
-			HE_TXOP_KNOWN |
-			HE_MIDABLE_PERIODICITY_KNOWN;
-
-	/* data3 */
-	ppdu_info->he_data3 = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_BSS_COLOR);
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_UL_FLAG);
-	value = value << HE_DL_UL_SHIFT;
-	ppdu_info->he_data3 |= value;
-
-	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_LDPC_EXTRA);
-	value = value << HE_LDPC_EXTRA_SYMBOL_SHIFT;
-	ppdu_info->he_data3 |= value;
-
-	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_STBC);
-	value = value << HE_STBC_SHIFT;
-	ppdu_info->he_data3 |= value;
-
-	/* data4 */
-	ppdu_info->he_data4 = u32_get_bits(info0,
-					   HAL_RX_HE_SIG_A_MU_DL_INFO0_SPATIAL_REUSE);
-	ppdu_info->he_data4 = value;
-
-	/* data5 */
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_TRANSMIT_BW);
-	ppdu_info->he_data5 = value;
-	ppdu_info->bw = value;
-
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_CP_LTF_SIZE);
-	switch (value) {
-	case 0:
-		he_gi = HE_GI_0_8;
-		he_ltf = HE_LTF_4_X;
-		break;
-	case 1:
-		he_gi = HE_GI_0_8;
-		he_ltf = HE_LTF_2_X;
-		break;
-	case 2:
-		he_gi = HE_GI_1_6;
-		he_ltf = HE_LTF_2_X;
-		break;
-	case 3:
-		he_gi = HE_GI_3_2;
-		he_ltf = HE_LTF_4_X;
-		break;
-	}
-
-	ppdu_info->gi = he_gi;
-	value = he_gi << HE_GI_SHIFT;
-	ppdu_info->he_data5 |= value;
-
-	value = he_ltf << HE_LTF_SIZE_SHIFT;
-	ppdu_info->he_data5 |= value;
-
-	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_NUM_LTF_SYMB);
-	value = (value << HE_LTF_SYM_SHIFT);
-	ppdu_info->he_data5 |= value;
-
-	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_PKT_EXT_FACTOR);
-	value = value << HE_PRE_FEC_PAD_SHIFT;
-	ppdu_info->he_data5 |= value;
-
-	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_PKT_EXT_PE_DISAM);
-	value = value << HE_PE_DISAMBIGUITY_SHIFT;
-	ppdu_info->he_data5 |= value;
-
-	/*data6*/
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_DOPPLER_INDICATION);
-	value = value << HE_DOPPLER_SHIFT;
-	ppdu_info->he_data6 |= value;
-
-	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_TXOP_DURATION);
-	value = value << HE_TXOP_SHIFT;
-	ppdu_info->he_data6 |= value;
-
-	/* HE-MU Flags */
-	/* HE-MU-flags1 */
-	ppdu_info->he_flags1 =
-		HE_SIG_B_MCS_KNOWN |
-		HE_SIG_B_DCM_KNOWN |
-		HE_SIG_B_COMPRESSION_FLAG_1_KNOWN |
-		HE_SIG_B_SYM_NUM_KNOWN |
-		HE_RU_0_KNOWN;
-
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_MCS_OF_SIGB);
-	ppdu_info->he_flags1 |= value;
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_DCM_OF_SIGB);
-	value = value << HE_DCM_FLAG_1_SHIFT;
-	ppdu_info->he_flags1 |= value;
-
-	/* HE-MU-flags2 */
-	ppdu_info->he_flags2 = HE_BW_KNOWN;
-
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_TRANSMIT_BW);
-	ppdu_info->he_flags2 |= value;
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_COMP_MODE_SIGB);
-	value = value << HE_SIG_B_COMPRESSION_FLAG_2_SHIFT;
-	ppdu_info->he_flags2 |= value;
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_NUM_SIGB_SYMB);
-	value = value - 1;
-	value = value << HE_NUM_SIG_B_SYMBOLS_SHIFT;
-	ppdu_info->he_flags2 |= value;
-
-	ppdu_info->is_stbc = info1 &
-			     HAL_RX_HE_SIG_A_MU_DL_INFO1_STBC;
-}
-
-static void ath12k_dp_mon_parse_he_sig_su(const struct hal_rx_he_sig_a_su_info *he_sig_a,
-					  struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	u32 info0, info1, value;
-	u32 dcm;
-	u8 he_dcm = 0, he_stbc = 0;
-	u16 he_gi = 0, he_ltf = 0;
-
-	ppdu_info->he_flags = 1;
-
-	info0 = __le32_to_cpu(he_sig_a->info0);
-	info1 = __le32_to_cpu(he_sig_a->info1);
-
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_FORMAT_IND);
-	if (value == 0)
-		ppdu_info->he_data1 = HE_TRIG_FORMAT_TYPE;
-	else
-		ppdu_info->he_data1 = HE_SU_FORMAT_TYPE;
-
-	ppdu_info->he_data1 |=
-			HE_BSS_COLOR_KNOWN |
-			HE_BEAM_CHANGE_KNOWN |
-			HE_DL_UL_KNOWN |
-			HE_MCS_KNOWN |
-			HE_DCM_KNOWN |
-			HE_CODING_KNOWN |
-			HE_LDPC_EXTRA_SYMBOL_KNOWN |
-			HE_STBC_KNOWN |
-			HE_DATA_BW_RU_KNOWN |
-			HE_DOPPLER_KNOWN;
-
-	ppdu_info->he_data2 |=
-			HE_GI_KNOWN |
-			HE_TXBF_KNOWN |
-			HE_PE_DISAMBIGUITY_KNOWN |
-			HE_TXOP_KNOWN |
-			HE_LTF_SYMBOLS_KNOWN |
-			HE_PRE_FEC_PADDING_KNOWN |
-			HE_MIDABLE_PERIODICITY_KNOWN;
-
-	ppdu_info->he_data3 = u32_get_bits(info0,
-					   HAL_RX_HE_SIG_A_SU_INFO_INFO0_BSS_COLOR);
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_BEAM_CHANGE);
-	value = value << HE_BEAM_CHANGE_SHIFT;
-	ppdu_info->he_data3 |= value;
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_DL_UL_FLAG);
-	value = value << HE_DL_UL_SHIFT;
-	ppdu_info->he_data3 |= value;
-
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_MCS);
-	ppdu_info->mcs = value;
-	value = value << HE_TRANSMIT_MCS_SHIFT;
-	ppdu_info->he_data3 |= value;
-
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_DCM);
-	he_dcm = value;
-	value = value << HE_DCM_SHIFT;
-	ppdu_info->he_data3 |= value;
-	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_CODING);
-	value = value << HE_CODING_SHIFT;
-	ppdu_info->he_data3 |= value;
-	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_LDPC_EXTRA);
-	value = value << HE_LDPC_EXTRA_SYMBOL_SHIFT;
-	ppdu_info->he_data3 |= value;
-	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_STBC);
-	he_stbc = value;
-	value = value << HE_STBC_SHIFT;
-	ppdu_info->he_data3 |= value;
-
-	/* data4 */
-	ppdu_info->he_data4 = u32_get_bits(info0,
-					   HAL_RX_HE_SIG_A_SU_INFO_INFO0_SPATIAL_REUSE);
-
-	/* data5 */
-	value = u32_get_bits(info0,
-			     HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_BW);
-	ppdu_info->he_data5 = value;
-	ppdu_info->bw = value;
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_CP_LTF_SIZE);
-	switch (value) {
-	case 0:
-		he_gi = HE_GI_0_8;
-		he_ltf = HE_LTF_1_X;
-		break;
-	case 1:
-		he_gi = HE_GI_0_8;
-		he_ltf = HE_LTF_2_X;
-		break;
-	case 2:
-		he_gi = HE_GI_1_6;
-		he_ltf = HE_LTF_2_X;
-		break;
-	case 3:
-		if (he_dcm && he_stbc) {
-			he_gi = HE_GI_0_8;
-			he_ltf = HE_LTF_4_X;
-		} else {
-			he_gi = HE_GI_3_2;
-			he_ltf = HE_LTF_4_X;
-		}
-		break;
-	}
-	ppdu_info->gi = he_gi;
-	value = he_gi << HE_GI_SHIFT;
-	ppdu_info->he_data5 |= value;
-	value = he_ltf << HE_LTF_SIZE_SHIFT;
-	ppdu_info->ltf_size = he_ltf;
-	ppdu_info->he_data5 |= value;
-
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_NSTS);
-	value = (value << HE_LTF_SYM_SHIFT);
-	ppdu_info->he_data5 |= value;
-
-	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_PKT_EXT_FACTOR);
-	value = value << HE_PRE_FEC_PAD_SHIFT;
-	ppdu_info->he_data5 |= value;
-
-	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_TXBF);
-	value = value << HE_TXBF_SHIFT;
-	ppdu_info->he_data5 |= value;
-	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_PKT_EXT_PE_DISAM);
-	value = value << HE_PE_DISAMBIGUITY_SHIFT;
-	ppdu_info->he_data5 |= value;
-
-	/* data6 */
-	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_NSTS);
-	value++;
-	ppdu_info->he_data6 = value;
-	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_DOPPLER_IND);
-	value = value << HE_DOPPLER_SHIFT;
-	ppdu_info->he_data6 |= value;
-	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_TXOP_DURATION);
-	value = value << HE_TXOP_SHIFT;
-	ppdu_info->he_data6 |= value;
-
-	ppdu_info->mcs =
-		u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_MCS);
-	ppdu_info->bw =
-		u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_BW);
-	ppdu_info->ldpc = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_CODING);
-	ppdu_info->is_stbc = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_STBC);
-	ppdu_info->beamformed = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_TXBF);
-	dcm = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_DCM);
-	ppdu_info->nss = u32_get_bits(info0,
-				      HAL_RX_HE_SIG_A_SU_INFO_INFO0_NSTS) + 1;
-	ppdu_info->dcm = dcm;
-}
-
-static void
-ath12k_dp_mon_hal_rx_parse_u_sig_cmn(const struct hal_mon_usig_cmn *cmn,
-				     struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	u32 common;
-
-	ppdu_info->u_sig_info.bw = le32_get_bits(cmn->info0,
-						 HAL_RX_USIG_CMN_INFO0_BW);
-	ppdu_info->u_sig_info.ul_dl = le32_get_bits(cmn->info0,
-						    HAL_RX_USIG_CMN_INFO0_UL_DL);
-
-	common = __le32_to_cpu(ppdu_info->u_sig_info.usig.common);
-	common |= IEEE80211_RADIOTAP_EHT_USIG_COMMON_PHY_VER_KNOWN |
-		  IEEE80211_RADIOTAP_EHT_USIG_COMMON_BW_KNOWN |
-		  IEEE80211_RADIOTAP_EHT_USIG_COMMON_UL_DL_KNOWN |
-		  IEEE80211_RADIOTAP_EHT_USIG_COMMON_BSS_COLOR_KNOWN |
-		  IEEE80211_RADIOTAP_EHT_USIG_COMMON_TXOP_KNOWN |
-		  ATH12K_LE32_DEC_ENC(cmn->info0,
-				      HAL_RX_USIG_CMN_INFO0_PHY_VERSION,
-				      IEEE80211_RADIOTAP_EHT_USIG_COMMON_PHY_VER) |
-		  u32_encode_bits(ppdu_info->u_sig_info.bw,
-				  IEEE80211_RADIOTAP_EHT_USIG_COMMON_BW) |
-		  u32_encode_bits(ppdu_info->u_sig_info.ul_dl,
-				  IEEE80211_RADIOTAP_EHT_USIG_COMMON_UL_DL) |
-		  ATH12K_LE32_DEC_ENC(cmn->info0,
-				      HAL_RX_USIG_CMN_INFO0_BSS_COLOR,
-				      IEEE80211_RADIOTAP_EHT_USIG_COMMON_BSS_COLOR) |
-		  ATH12K_LE32_DEC_ENC(cmn->info0,
-				      HAL_RX_USIG_CMN_INFO0_TXOP,
-				      IEEE80211_RADIOTAP_EHT_USIG_COMMON_TXOP);
-	ppdu_info->u_sig_info.usig.common = cpu_to_le32(common);
-
-	switch (ppdu_info->u_sig_info.bw) {
-	default:
-		fallthrough;
-	case HAL_EHT_BW_20:
-		ppdu_info->bw = HAL_RX_BW_20MHZ;
-		break;
-	case HAL_EHT_BW_40:
-		ppdu_info->bw = HAL_RX_BW_40MHZ;
-		break;
-	case HAL_EHT_BW_80:
-		ppdu_info->bw = HAL_RX_BW_80MHZ;
-		break;
-	case HAL_EHT_BW_160:
-		ppdu_info->bw = HAL_RX_BW_160MHZ;
-		break;
-	case HAL_EHT_BW_320_1:
-	case HAL_EHT_BW_320_2:
-		ppdu_info->bw = HAL_RX_BW_320MHZ;
-		break;
-	}
-}
-
-static void
-ath12k_dp_mon_hal_rx_parse_u_sig_tb(const struct hal_mon_usig_tb *usig_tb,
-				    struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	struct ieee80211_radiotap_eht_usig *usig = &ppdu_info->u_sig_info.usig;
-	enum ieee80211_radiotap_eht_usig_tb spatial_reuse1, spatial_reuse2;
-	u32 common, value, mask;
-
-	spatial_reuse1 = IEEE80211_RADIOTAP_EHT_USIG2_TB_B3_B6_SPATIAL_REUSE_1;
-	spatial_reuse2 = IEEE80211_RADIOTAP_EHT_USIG2_TB_B7_B10_SPATIAL_REUSE_2;
-
-	common = __le32_to_cpu(usig->common);
-	value = __le32_to_cpu(usig->value);
-	mask = __le32_to_cpu(usig->mask);
-
-	ppdu_info->u_sig_info.ppdu_type_comp_mode =
-				le32_get_bits(usig_tb->info0,
-					      HAL_RX_USIG_TB_INFO0_PPDU_TYPE_COMP_MODE);
-
-	common |= ATH12K_LE32_DEC_ENC(usig_tb->info0,
-				      HAL_RX_USIG_TB_INFO0_RX_INTEG_CHECK_PASS,
-				      IEEE80211_RADIOTAP_EHT_USIG_COMMON_BAD_USIG_CRC);
-
-	value |= IEEE80211_RADIOTAP_EHT_USIG1_TB_B20_B25_DISREGARD |
-		 u32_encode_bits(ppdu_info->u_sig_info.ppdu_type_comp_mode,
-				 IEEE80211_RADIOTAP_EHT_USIG2_TB_B0_B1_PPDU_TYPE) |
-		 IEEE80211_RADIOTAP_EHT_USIG2_TB_B2_VALIDATE |
-		 ATH12K_LE32_DEC_ENC(usig_tb->info0,
-				     HAL_RX_USIG_TB_INFO0_SPATIAL_REUSE_1,
-				     spatial_reuse1) |
-		 ATH12K_LE32_DEC_ENC(usig_tb->info0,
-				     HAL_RX_USIG_TB_INFO0_SPATIAL_REUSE_2,
-				     spatial_reuse2) |
-		 IEEE80211_RADIOTAP_EHT_USIG2_TB_B11_B15_DISREGARD |
-		 ATH12K_LE32_DEC_ENC(usig_tb->info0,
-				     HAL_RX_USIG_TB_INFO0_CRC,
-				     IEEE80211_RADIOTAP_EHT_USIG2_TB_B16_B19_CRC) |
-		 ATH12K_LE32_DEC_ENC(usig_tb->info0,
-				     HAL_RX_USIG_TB_INFO0_TAIL,
-				     IEEE80211_RADIOTAP_EHT_USIG2_TB_B20_B25_TAIL);
-
-	mask |= IEEE80211_RADIOTAP_EHT_USIG1_TB_B20_B25_DISREGARD |
-		IEEE80211_RADIOTAP_EHT_USIG2_TB_B0_B1_PPDU_TYPE |
-		IEEE80211_RADIOTAP_EHT_USIG2_TB_B2_VALIDATE |
-		spatial_reuse1 | spatial_reuse2 |
-		IEEE80211_RADIOTAP_EHT_USIG2_TB_B11_B15_DISREGARD |
-		IEEE80211_RADIOTAP_EHT_USIG2_TB_B16_B19_CRC |
-		IEEE80211_RADIOTAP_EHT_USIG2_TB_B20_B25_TAIL;
-
-	usig->common = cpu_to_le32(common);
-	usig->value = cpu_to_le32(value);
-	usig->mask = cpu_to_le32(mask);
-}
-
-static void
-ath12k_dp_mon_hal_rx_parse_u_sig_mu(const struct hal_mon_usig_mu *usig_mu,
-				    struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	struct ieee80211_radiotap_eht_usig *usig = &ppdu_info->u_sig_info.usig;
-	enum ieee80211_radiotap_eht_usig_mu sig_symb, punc;
-	u32 common, value, mask;
-
-	sig_symb = IEEE80211_RADIOTAP_EHT_USIG2_MU_B11_B15_EHT_SIG_SYMBOLS;
-	punc = IEEE80211_RADIOTAP_EHT_USIG2_MU_B3_B7_PUNCTURED_INFO;
-
-	common = __le32_to_cpu(usig->common);
-	value = __le32_to_cpu(usig->value);
-	mask = __le32_to_cpu(usig->mask);
-
-	ppdu_info->u_sig_info.ppdu_type_comp_mode =
-				le32_get_bits(usig_mu->info0,
-					      HAL_RX_USIG_MU_INFO0_PPDU_TYPE_COMP_MODE);
-	ppdu_info->u_sig_info.eht_sig_mcs =
-				le32_get_bits(usig_mu->info0,
-					      HAL_RX_USIG_MU_INFO0_EHT_SIG_MCS);
-	ppdu_info->u_sig_info.num_eht_sig_sym =
-				le32_get_bits(usig_mu->info0,
-					      HAL_RX_USIG_MU_INFO0_NUM_EHT_SIG_SYM);
-
-	common |= ATH12K_LE32_DEC_ENC(usig_mu->info0,
-				      HAL_RX_USIG_MU_INFO0_RX_INTEG_CHECK_PASS,
-				      IEEE80211_RADIOTAP_EHT_USIG_COMMON_BAD_USIG_CRC);
-
-	value |= IEEE80211_RADIOTAP_EHT_USIG1_MU_B20_B24_DISREGARD |
-		 IEEE80211_RADIOTAP_EHT_USIG1_MU_B25_VALIDATE |
-		 u32_encode_bits(ppdu_info->u_sig_info.ppdu_type_comp_mode,
-				 IEEE80211_RADIOTAP_EHT_USIG2_MU_B0_B1_PPDU_TYPE) |
-		 IEEE80211_RADIOTAP_EHT_USIG2_MU_B2_VALIDATE |
-		 ATH12K_LE32_DEC_ENC(usig_mu->info0,
-				     HAL_RX_USIG_MU_INFO0_PUNC_CH_INFO,
-				     punc) |
-		 IEEE80211_RADIOTAP_EHT_USIG2_MU_B8_VALIDATE |
-		 u32_encode_bits(ppdu_info->u_sig_info.eht_sig_mcs,
-				 IEEE80211_RADIOTAP_EHT_USIG2_MU_B9_B10_SIG_MCS) |
-		 u32_encode_bits(ppdu_info->u_sig_info.num_eht_sig_sym,
-				 sig_symb) |
-		 ATH12K_LE32_DEC_ENC(usig_mu->info0,
-				     HAL_RX_USIG_MU_INFO0_CRC,
-				     IEEE80211_RADIOTAP_EHT_USIG2_MU_B16_B19_CRC) |
-		 ATH12K_LE32_DEC_ENC(usig_mu->info0,
-				     HAL_RX_USIG_MU_INFO0_TAIL,
-				     IEEE80211_RADIOTAP_EHT_USIG2_MU_B20_B25_TAIL);
-
-	mask |= IEEE80211_RADIOTAP_EHT_USIG1_MU_B20_B24_DISREGARD |
-		IEEE80211_RADIOTAP_EHT_USIG1_MU_B25_VALIDATE |
-		IEEE80211_RADIOTAP_EHT_USIG2_MU_B0_B1_PPDU_TYPE |
-		IEEE80211_RADIOTAP_EHT_USIG2_MU_B2_VALIDATE |
-		punc |
-		IEEE80211_RADIOTAP_EHT_USIG2_MU_B8_VALIDATE |
-		IEEE80211_RADIOTAP_EHT_USIG2_MU_B9_B10_SIG_MCS |
-		sig_symb |
-		IEEE80211_RADIOTAP_EHT_USIG2_MU_B16_B19_CRC |
-		IEEE80211_RADIOTAP_EHT_USIG2_MU_B20_B25_TAIL;
-
-	usig->common = cpu_to_le32(common);
-	usig->value = cpu_to_le32(value);
-	usig->mask = cpu_to_le32(mask);
-}
-
 static void
-ath12k_dp_mon_hal_rx_parse_u_sig_hdr(const struct hal_mon_usig_hdr *usig,
-				     struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	u8 comp_mode;
-
-	ppdu_info->eht_usig = true;
-
-	ath12k_dp_mon_hal_rx_parse_u_sig_cmn(&usig->cmn, ppdu_info);
-
-	comp_mode = le32_get_bits(usig->non_cmn.mu.info0,
-				  HAL_RX_USIG_MU_INFO0_PPDU_TYPE_COMP_MODE);
-
-	if (comp_mode == 0 && ppdu_info->u_sig_info.ul_dl)
-		ath12k_dp_mon_hal_rx_parse_u_sig_tb(&usig->non_cmn.tb, ppdu_info);
-	else
-		ath12k_dp_mon_hal_rx_parse_u_sig_mu(&usig->non_cmn.mu, ppdu_info);
-}
-
-static void
-ath12k_dp_mon_hal_aggr_tlv(struct hal_rx_mon_ppdu_info *ppdu_info,
-			   u16 tlv_len, const void *tlv_data)
-{
-	if (tlv_len <= HAL_RX_MON_MAX_AGGR_SIZE - ppdu_info->tlv_aggr.cur_len) {
-		memcpy(ppdu_info->tlv_aggr.buf + ppdu_info->tlv_aggr.cur_len,
-		       tlv_data, tlv_len);
-		ppdu_info->tlv_aggr.cur_len += tlv_len;
-	}
-}
-
-static inline bool
-ath12k_dp_mon_hal_rx_is_frame_type_ndp(const struct hal_rx_u_sig_info *usig_info)
-{
-	if (usig_info->ppdu_type_comp_mode == 1 &&
-	    usig_info->eht_sig_mcs == 0 &&
-	    usig_info->num_eht_sig_sym == 0)
-		return true;
-
-	return false;
-}
-
-static inline bool
-ath12k_dp_mon_hal_rx_is_non_ofdma(const struct hal_rx_u_sig_info *usig_info)
-{
-	u32 ppdu_type_comp_mode = usig_info->ppdu_type_comp_mode;
-	u32 ul_dl = usig_info->ul_dl;
-
-	if ((ppdu_type_comp_mode == HAL_RX_RECEPTION_TYPE_MU_MIMO && ul_dl == 0) ||
-	    (ppdu_type_comp_mode == HAL_RX_RECEPTION_TYPE_MU_OFDMA && ul_dl == 0) ||
-	    (ppdu_type_comp_mode == HAL_RX_RECEPTION_TYPE_MU_MIMO  && ul_dl == 1))
-		return true;
-
-	return false;
-}
-
-static inline bool
-ath12k_dp_mon_hal_rx_is_ofdma(const struct hal_rx_u_sig_info *usig_info)
-{
-	if (usig_info->ppdu_type_comp_mode == 0 && usig_info->ul_dl == 0)
-		return true;
-
-	return false;
-}
-
-static void
-ath12k_dp_mon_hal_rx_parse_eht_sig_ndp(const struct hal_eht_sig_ndp_cmn_eb *eht_sig_ndp,
-				       struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht;
-	u32 known, data;
-
-	known = __le32_to_cpu(eht->known);
-	known |= IEEE80211_RADIOTAP_EHT_KNOWN_SPATIAL_REUSE |
-		 IEEE80211_RADIOTAP_EHT_KNOWN_EHT_LTF |
-		 IEEE80211_RADIOTAP_EHT_KNOWN_NSS_S |
-		 IEEE80211_RADIOTAP_EHT_KNOWN_BEAMFORMED_S |
-		 IEEE80211_RADIOTAP_EHT_KNOWN_DISREGARD_S |
-		 IEEE80211_RADIOTAP_EHT_KNOWN_CRC1 |
-		 IEEE80211_RADIOTAP_EHT_KNOWN_TAIL1;
-	eht->known = cpu_to_le32(known);
-
-	data = __le32_to_cpu(eht->data[0]);
-	data |= ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0,
-				    HAL_RX_EHT_SIG_NDP_CMN_INFO0_SPATIAL_REUSE,
-				    IEEE80211_RADIOTAP_EHT_DATA0_SPATIAL_REUSE);
-	/* GI and LTF size are separately indicated in radiotap header
-	 * and hence will be parsed from other TLV
-	 */
-	data |=	ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0,
-				    HAL_RX_EHT_SIG_NDP_CMN_INFO0_NUM_LTF_SYM,
-				    IEEE80211_RADIOTAP_EHT_DATA0_EHT_LTF);
-
-	data |=	ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0,
-				    HAL_RX_EHT_SIG_NDP_CMN_INFO0_CRC,
-				    IEEE80211_RADIOTAP_EHT_DATA0_CRC1_O);
-
-	data |= ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0,
-				    HAL_RX_EHT_SIG_NDP_CMN_INFO0_DISREGARD,
-				    IEEE80211_RADIOTAP_EHT_DATA0_DISREGARD_S);
-	eht->data[0] = cpu_to_le32(data);
-
-	data = __le32_to_cpu(eht->data[7]);
-	data |=	ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0,
-				    HAL_RX_EHT_SIG_NDP_CMN_INFO0_NSS,
-				    IEEE80211_RADIOTAP_EHT_DATA7_NSS_S);
-
-	data |=	ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0,
-				    HAL_RX_EHT_SIG_NDP_CMN_INFO0_BEAMFORMED,
-				    IEEE80211_RADIOTAP_EHT_DATA7_BEAMFORMED_S);
-	eht->data[7] = cpu_to_le32(data);
-}
-
-static void
-ath12k_dp_mon_hal_rx_parse_usig_overflow(const struct hal_eht_sig_usig_overflow *ovflow,
-					 struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht;
-	u32 known, data;
-
-	known = __le32_to_cpu(eht->known);
-	known |= IEEE80211_RADIOTAP_EHT_KNOWN_SPATIAL_REUSE |
-		 IEEE80211_RADIOTAP_EHT_KNOWN_EHT_LTF |
-		 IEEE80211_RADIOTAP_EHT_KNOWN_LDPC_EXTRA_SYM_OM |
-		 IEEE80211_RADIOTAP_EHT_KNOWN_PRE_PADD_FACOR_OM |
-		 IEEE80211_RADIOTAP_EHT_KNOWN_PE_DISAMBIGUITY_OM |
-		 IEEE80211_RADIOTAP_EHT_KNOWN_DISREGARD_O;
-	eht->known = cpu_to_le32(known);
-
-	data = __le32_to_cpu(eht->data[0]);
-	data |=	ATH12K_LE32_DEC_ENC(ovflow->info0,
-				    HAL_RX_EHT_SIG_OVERFLOW_INFO0_SPATIAL_REUSE,
-				    IEEE80211_RADIOTAP_EHT_DATA0_SPATIAL_REUSE);
-
-	/* GI and LTF size are separately indicated in radiotap header
-	 * and hence will be parsed from other TLV
-	 */
-	data |=	ATH12K_LE32_DEC_ENC(ovflow->info0,
-				    HAL_RX_EHT_SIG_OVERFLOW_INFO0_NUM_LTF_SYM,
-				    IEEE80211_RADIOTAP_EHT_DATA0_EHT_LTF);
-
-	data |=	ATH12K_LE32_DEC_ENC(ovflow->info0,
-				    HAL_RX_EHT_SIG_OVERFLOW_INFO0_LDPC_EXTA_SYM,
-				    IEEE80211_RADIOTAP_EHT_DATA0_LDPC_EXTRA_SYM_OM);
-
-	data |=	ATH12K_LE32_DEC_ENC(ovflow->info0,
-				    HAL_RX_EHT_SIG_OVERFLOW_INFO0_PRE_FEC_PAD_FACTOR,
-				    IEEE80211_RADIOTAP_EHT_DATA0_PRE_PADD_FACOR_OM);
-
-	data |=	ATH12K_LE32_DEC_ENC(ovflow->info0,
-				    HAL_RX_EHT_SIG_OVERFLOW_INFO0_DISAMBIGUITY,
-				    IEEE80211_RADIOTAP_EHT_DATA0_PE_DISAMBIGUITY_OM);
-
-	data |=	ATH12K_LE32_DEC_ENC(ovflow->info0,
-				    HAL_RX_EHT_SIG_OVERFLOW_INFO0_DISREGARD,
-				    IEEE80211_RADIOTAP_EHT_DATA0_DISREGARD_O);
-	eht->data[0] = cpu_to_le32(data);
-}
-
-static void
-ath12k_dp_mon_hal_rx_parse_non_ofdma_users(const struct hal_eht_sig_non_ofdma_cmn_eb *eb,
-					   struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht;
-	u32 known, data;
-
-	known = __le32_to_cpu(eht->known);
-	known |= IEEE80211_RADIOTAP_EHT_KNOWN_NR_NON_OFDMA_USERS_M;
-	eht->known = cpu_to_le32(known);
-
-	data = __le32_to_cpu(eht->data[7]);
-	data |=	ATH12K_LE32_DEC_ENC(eb->info0,
-				    HAL_RX_EHT_SIG_NON_OFDMA_INFO0_NUM_USERS,
-				    IEEE80211_RADIOTAP_EHT_DATA7_NUM_OF_NON_OFDMA_USERS);
-	eht->data[7] = cpu_to_le32(data);
-}
-
-static void
-ath12k_dp_mon_hal_rx_parse_eht_mumimo_user(const struct hal_eht_sig_mu_mimo *user,
-					   struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	struct hal_rx_eht_info *eht_info = &ppdu_info->eht_info;
-	u32 user_idx;
-
-	if (eht_info->num_user_info >= ARRAY_SIZE(eht_info->user_info))
-		return;
-
-	user_idx = eht_info->num_user_info++;
-
-	eht_info->user_info[user_idx] |=
-		IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID_KNOWN |
-		IEEE80211_RADIOTAP_EHT_USER_INFO_MCS_KNOWN |
-		IEEE80211_RADIOTAP_EHT_USER_INFO_CODING_KNOWN |
-		IEEE80211_RADIOTAP_EHT_USER_INFO_SPATIAL_CONFIG_KNOWN_M |
-		ATH12K_LE32_DEC_ENC(user->info0,
-				    HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_STA_ID,
-				    IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID) |
-		ATH12K_LE32_DEC_ENC(user->info0,
-				    HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_CODING,
-				    IEEE80211_RADIOTAP_EHT_USER_INFO_CODING) |
-		ATH12K_LE32_DEC_ENC(user->info0,
-				    HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_MCS,
-				    IEEE80211_RADIOTAP_EHT_USER_INFO_MCS) |
-		ATH12K_LE32_DEC_ENC(user->info0,
-				    HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_SPATIAL_CODING,
-				    IEEE80211_RADIOTAP_EHT_USER_INFO_SPATIAL_CONFIG_M);
-
-	ppdu_info->mcs = le32_get_bits(user->info0,
-				       HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_MCS);
-}
-
-static void
-ath12k_dp_mon_hal_rx_parse_eht_non_mumimo_user(const struct hal_eht_sig_non_mu_mimo *user,
-					       struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	struct hal_rx_eht_info *eht_info = &ppdu_info->eht_info;
-	u32 user_idx;
-
-	if (eht_info->num_user_info >= ARRAY_SIZE(eht_info->user_info))
-		return;
-
-	user_idx = eht_info->num_user_info++;
-
-	eht_info->user_info[user_idx] |=
-		IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID_KNOWN |
-		IEEE80211_RADIOTAP_EHT_USER_INFO_MCS_KNOWN |
-		IEEE80211_RADIOTAP_EHT_USER_INFO_CODING_KNOWN |
-		IEEE80211_RADIOTAP_EHT_USER_INFO_NSS_KNOWN_O |
-		IEEE80211_RADIOTAP_EHT_USER_INFO_BEAMFORMING_KNOWN_O |
-		ATH12K_LE32_DEC_ENC(user->info0,
-				    HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_STA_ID,
-				    IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID) |
-		ATH12K_LE32_DEC_ENC(user->info0,
-				    HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_CODING,
-				    IEEE80211_RADIOTAP_EHT_USER_INFO_CODING) |
-		ATH12K_LE32_DEC_ENC(user->info0,
-				    HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_MCS,
-				    IEEE80211_RADIOTAP_EHT_USER_INFO_MCS) |
-		ATH12K_LE32_DEC_ENC(user->info0,
-				    HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_NSS,
-				    IEEE80211_RADIOTAP_EHT_USER_INFO_NSS_O) |
-		ATH12K_LE32_DEC_ENC(user->info0,
-				    HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_BEAMFORMED,
-				    IEEE80211_RADIOTAP_EHT_USER_INFO_BEAMFORMING_O);
-
-	ppdu_info->mcs = le32_get_bits(user->info0,
-				       HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_MCS);
-
-	ppdu_info->nss = le32_get_bits(user->info0,
-				       HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_NSS) + 1;
-}
-
-static inline bool
-ath12k_dp_mon_hal_rx_is_mu_mimo_user(const struct hal_rx_u_sig_info *usig_info)
-{
-	if (usig_info->ppdu_type_comp_mode == HAL_RX_RECEPTION_TYPE_SU &&
-	    usig_info->ul_dl == 1)
-		return true;
-
-	return false;
-}
-
-static void
-ath12k_dp_mon_hal_rx_parse_eht_sig_non_ofdma(const void *tlv,
-					     struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	const struct hal_eht_sig_non_ofdma_cmn_eb *eb = tlv;
-
-	ath12k_dp_mon_hal_rx_parse_usig_overflow(tlv, ppdu_info);
-	ath12k_dp_mon_hal_rx_parse_non_ofdma_users(eb, ppdu_info);
-
-	if (ath12k_dp_mon_hal_rx_is_mu_mimo_user(&ppdu_info->u_sig_info))
-		ath12k_dp_mon_hal_rx_parse_eht_mumimo_user(&eb->user_field.mu_mimo,
-							   ppdu_info);
-	else
-		ath12k_dp_mon_hal_rx_parse_eht_non_mumimo_user(&eb->user_field.n_mu_mimo,
-							       ppdu_info);
-}
-
-static void
-ath12k_dp_mon_hal_rx_parse_ru_allocation(const struct hal_eht_sig_ofdma_cmn_eb *eb,
-					 struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	const struct hal_eht_sig_ofdma_cmn_eb1 *ofdma_cmn_eb1 = &eb->eb1;
-	const struct hal_eht_sig_ofdma_cmn_eb2 *ofdma_cmn_eb2 = &eb->eb2;
-	struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht;
-	enum ieee80211_radiotap_eht_data ru_123, ru_124, ru_125, ru_126;
-	enum ieee80211_radiotap_eht_data ru_121, ru_122, ru_112, ru_111;
-	u32 data;
-
-	ru_123 = IEEE80211_RADIOTAP_EHT_DATA4_RU_ALLOC_CC_1_2_3;
-	ru_124 = IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_4;
-	ru_125 = IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_5;
-	ru_126 = IEEE80211_RADIOTAP_EHT_DATA6_RU_ALLOC_CC_1_2_6;
-	ru_121 = IEEE80211_RADIOTAP_EHT_DATA3_RU_ALLOC_CC_1_2_1;
-	ru_122 = IEEE80211_RADIOTAP_EHT_DATA3_RU_ALLOC_CC_1_2_2;
-	ru_112 = IEEE80211_RADIOTAP_EHT_DATA2_RU_ALLOC_CC_1_1_2;
-	ru_111 = IEEE80211_RADIOTAP_EHT_DATA1_RU_ALLOC_CC_1_1_1;
-
-	switch (ppdu_info->u_sig_info.bw) {
-	case HAL_EHT_BW_320_2:
-	case HAL_EHT_BW_320_1:
-		data = __le32_to_cpu(eht->data[4]);
-		/* CC1 2::3 */
-		data |=	IEEE80211_RADIOTAP_EHT_DATA4_RU_ALLOC_CC_1_2_3_KNOWN |
-			ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0,
-					    HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_3,
-					    ru_123);
-		eht->data[4] = cpu_to_le32(data);
-
-		data = __le32_to_cpu(eht->data[5]);
-		/* CC1 2::4 */
-		data |=	IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_4_KNOWN |
-			ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0,
-					    HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_4,
-					    ru_124);
-
-		/* CC1 2::5 */
-		data |=	IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_5_KNOWN |
-			ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0,
-					    HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_5,
-					    ru_125);
-		eht->data[5] = cpu_to_le32(data);
-
-		data = __le32_to_cpu(eht->data[6]);
-		/* CC1 2::6 */
-		data |=	IEEE80211_RADIOTAP_EHT_DATA6_RU_ALLOC_CC_1_2_6_KNOWN |
-			ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0,
-					    HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_6,
-					    ru_126);
-		eht->data[6] = cpu_to_le32(data);
-
-		fallthrough;
-	case HAL_EHT_BW_160:
-		data = __le32_to_cpu(eht->data[3]);
-		/* CC1 2::1 */
-		data |=	IEEE80211_RADIOTAP_EHT_DATA3_RU_ALLOC_CC_1_2_1_KNOWN |
-			ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0,
-					    HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_1,
-					    ru_121);
-		/* CC1 2::2 */
-		data |=	IEEE80211_RADIOTAP_EHT_DATA3_RU_ALLOC_CC_1_2_2_KNOWN |
-			ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0,
-					    HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_2,
-					    ru_122);
-		eht->data[3] = cpu_to_le32(data);
-
-		fallthrough;
-	case HAL_EHT_BW_80:
-		data = __le32_to_cpu(eht->data[2]);
-		/* CC1 1::2 */
-		data |=	IEEE80211_RADIOTAP_EHT_DATA2_RU_ALLOC_CC_1_1_2_KNOWN |
-			ATH12K_LE64_DEC_ENC(ofdma_cmn_eb1->info0,
-					    HAL_RX_EHT_SIG_OFDMA_EB1_RU_ALLOC_1_2,
-					    ru_112);
-		eht->data[2] = cpu_to_le32(data);
-
-		fallthrough;
-	case HAL_EHT_BW_40:
-		fallthrough;
-	case HAL_EHT_BW_20:
-		data = __le32_to_cpu(eht->data[1]);
-		/* CC1 1::1 */
-		data |=	IEEE80211_RADIOTAP_EHT_DATA1_RU_ALLOC_CC_1_1_1_KNOWN |
-			ATH12K_LE64_DEC_ENC(ofdma_cmn_eb1->info0,
-					    HAL_RX_EHT_SIG_OFDMA_EB1_RU_ALLOC_1_1,
-					    ru_111);
-		eht->data[1] = cpu_to_le32(data);
-		break;
-	default:
-		break;
-	}
-}
-
-static void
-ath12k_dp_mon_hal_rx_parse_eht_sig_ofdma(const void *tlv,
-					 struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	const struct hal_eht_sig_ofdma_cmn_eb *ofdma = tlv;
-
-	ath12k_dp_mon_hal_rx_parse_usig_overflow(tlv, ppdu_info);
-	ath12k_dp_mon_hal_rx_parse_ru_allocation(ofdma, ppdu_info);
-
-	ath12k_dp_mon_hal_rx_parse_eht_non_mumimo_user(&ofdma->user_field.n_mu_mimo,
-						       ppdu_info);
-}
-
-static void
-ath12k_dp_mon_parse_eht_sig_hdr(struct hal_rx_mon_ppdu_info *ppdu_info,
-				const void *tlv_data)
-{
-	ppdu_info->is_eht = true;
-
-	if (ath12k_dp_mon_hal_rx_is_frame_type_ndp(&ppdu_info->u_sig_info))
-		ath12k_dp_mon_hal_rx_parse_eht_sig_ndp(tlv_data, ppdu_info);
-	else if (ath12k_dp_mon_hal_rx_is_non_ofdma(&ppdu_info->u_sig_info))
-		ath12k_dp_mon_hal_rx_parse_eht_sig_non_ofdma(tlv_data, ppdu_info);
-	else if (ath12k_dp_mon_hal_rx_is_ofdma(&ppdu_info->u_sig_info))
-		ath12k_dp_mon_hal_rx_parse_eht_sig_ofdma(tlv_data, ppdu_info);
-}
-
-static inline enum ath12k_eht_ru_size
-hal_rx_mon_hal_ru_size_to_ath12k_ru_size(u32 hal_ru_size)
-{
-	switch (hal_ru_size) {
-	case HAL_EHT_RU_26:
-		return ATH12K_EHT_RU_26;
-	case HAL_EHT_RU_52:
-		return ATH12K_EHT_RU_52;
-	case HAL_EHT_RU_78:
-		return ATH12K_EHT_RU_52_26;
-	case HAL_EHT_RU_106:
-		return ATH12K_EHT_RU_106;
-	case HAL_EHT_RU_132:
-		return ATH12K_EHT_RU_106_26;
-	case HAL_EHT_RU_242:
-		return ATH12K_EHT_RU_242;
-	case HAL_EHT_RU_484:
-		return ATH12K_EHT_RU_484;
-	case HAL_EHT_RU_726:
-		return ATH12K_EHT_RU_484_242;
-	case HAL_EHT_RU_996:
-		return ATH12K_EHT_RU_996;
-	case HAL_EHT_RU_996x2:
-		return ATH12K_EHT_RU_996x2;
-	case HAL_EHT_RU_996x3:
-		return ATH12K_EHT_RU_996x3;
-	case HAL_EHT_RU_996x4:
-		return ATH12K_EHT_RU_996x4;
-	case HAL_EHT_RU_NONE:
-		return ATH12K_EHT_RU_INVALID;
-	case HAL_EHT_RU_996_484:
-		return ATH12K_EHT_RU_996_484;
-	case HAL_EHT_RU_996x2_484:
-		return ATH12K_EHT_RU_996x2_484;
-	case HAL_EHT_RU_996x3_484:
-		return ATH12K_EHT_RU_996x3_484;
-	case HAL_EHT_RU_996_484_242:
-		return ATH12K_EHT_RU_996_484_242;
-	default:
-		return ATH12K_EHT_RU_INVALID;
-	}
-}
-
-static inline u32
-hal_rx_ul_ofdma_ru_size_to_width(enum ath12k_eht_ru_size ru_size)
-{
-	switch (ru_size) {
-	case ATH12K_EHT_RU_26:
-		return RU_26;
-	case ATH12K_EHT_RU_52:
-		return RU_52;
-	case ATH12K_EHT_RU_52_26:
-		return RU_52_26;
-	case ATH12K_EHT_RU_106:
-		return RU_106;
-	case ATH12K_EHT_RU_106_26:
-		return RU_106_26;
-	case ATH12K_EHT_RU_242:
-		return RU_242;
-	case ATH12K_EHT_RU_484:
-		return RU_484;
-	case ATH12K_EHT_RU_484_242:
-		return RU_484_242;
-	case ATH12K_EHT_RU_996:
-		return RU_996;
-	case ATH12K_EHT_RU_996_484:
-		return RU_996_484;
-	case ATH12K_EHT_RU_996_484_242:
-		return RU_996_484_242;
-	case ATH12K_EHT_RU_996x2:
-		return RU_2X996;
-	case ATH12K_EHT_RU_996x2_484:
-		return RU_2X996_484;
-	case ATH12K_EHT_RU_996x3:
-		return RU_3X996;
-	case ATH12K_EHT_RU_996x3_484:
-		return RU_3X996_484;
-	case ATH12K_EHT_RU_996x4:
-		return RU_4X996;
-	default:
-		return RU_INVALID;
-	}
-}
-
-static void
-ath12k_dp_mon_hal_rx_parse_user_info(const struct hal_receive_user_info *rx_usr_info,
-				     u16 user_id,
-				     struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	struct hal_rx_user_status *mon_rx_user_status = NULL;
-	struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht;
-	enum ath12k_eht_ru_size rtap_ru_size = ATH12K_EHT_RU_INVALID;
-	u32 ru_width, reception_type, ru_index = HAL_EHT_RU_INVALID;
-	u32 ru_type_80_0, ru_start_index_80_0;
-	u32 ru_type_80_1, ru_start_index_80_1;
-	u32 ru_type_80_2, ru_start_index_80_2;
-	u32 ru_type_80_3, ru_start_index_80_3;
-	u32 ru_size = 0, num_80mhz_with_ru = 0;
-	u64 ru_index_320mhz = 0;
-	u32 ru_index_per80mhz;
-
-	reception_type = le32_get_bits(rx_usr_info->info0,
-				       HAL_RX_USR_INFO0_RECEPTION_TYPE);
-
-	switch (reception_type) {
-	case HAL_RECEPTION_TYPE_SU:
-		ppdu_info->reception_type = HAL_RX_RECEPTION_TYPE_SU;
-		break;
-	case HAL_RECEPTION_TYPE_DL_MU_MIMO:
-	case HAL_RECEPTION_TYPE_UL_MU_MIMO:
-		ppdu_info->reception_type = HAL_RX_RECEPTION_TYPE_MU_MIMO;
-		break;
-	case HAL_RECEPTION_TYPE_DL_MU_OFMA:
-	case HAL_RECEPTION_TYPE_UL_MU_OFDMA:
-		ppdu_info->reception_type = HAL_RX_RECEPTION_TYPE_MU_OFDMA;
-		break;
-	case HAL_RECEPTION_TYPE_DL_MU_OFDMA_MIMO:
-	case HAL_RECEPTION_TYPE_UL_MU_OFDMA_MIMO:
-		ppdu_info->reception_type = HAL_RX_RECEPTION_TYPE_MU_OFDMA_MIMO;
-	}
-
-	ppdu_info->is_stbc = le32_get_bits(rx_usr_info->info0, HAL_RX_USR_INFO0_STBC);
-	ppdu_info->ldpc = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_LDPC);
-	ppdu_info->dcm = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_STA_DCM);
-	ppdu_info->bw = le32_get_bits(rx_usr_info->info1, HAL_RX_USR_INFO1_RX_BW);
-	ppdu_info->mcs = le32_get_bits(rx_usr_info->info1, HAL_RX_USR_INFO1_MCS);
-	ppdu_info->nss = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_NSS) + 1;
-
-	if (user_id < HAL_MAX_UL_MU_USERS) {
-		mon_rx_user_status = &ppdu_info->userstats[user_id];
-		mon_rx_user_status->mcs = ppdu_info->mcs;
-		mon_rx_user_status->nss = ppdu_info->nss;
-	}
-
-	if (!(ppdu_info->reception_type == HAL_RX_RECEPTION_TYPE_MU_MIMO ||
-	      ppdu_info->reception_type == HAL_RX_RECEPTION_TYPE_MU_OFDMA ||
-	      ppdu_info->reception_type == HAL_RX_RECEPTION_TYPE_MU_OFDMA_MIMO))
-		return;
-
-	/* RU allocation present only for OFDMA reception */
-	ru_type_80_0 = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_RU_TYPE_80_0);
-	ru_start_index_80_0 = le32_get_bits(rx_usr_info->info3,
-					    HAL_RX_USR_INFO3_RU_START_IDX_80_0);
-	if (ru_type_80_0 != HAL_EHT_RU_NONE) {
-		ru_size += ru_type_80_0;
-		ru_index_per80mhz = ru_start_index_80_0;
-		ru_index = ru_index_per80mhz;
-		ru_index_320mhz |= HAL_RU_PER80(ru_type_80_0, 0, ru_index_per80mhz);
-		num_80mhz_with_ru++;
-	}
-
-	ru_type_80_1 = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_RU_TYPE_80_1);
-	ru_start_index_80_1 = le32_get_bits(rx_usr_info->info3,
-					    HAL_RX_USR_INFO3_RU_START_IDX_80_1);
-	if (ru_type_80_1 != HAL_EHT_RU_NONE) {
-		ru_size += ru_type_80_1;
-		ru_index_per80mhz = ru_start_index_80_1;
-		ru_index = ru_index_per80mhz;
-		ru_index_320mhz |= HAL_RU_PER80(ru_type_80_1, 1, ru_index_per80mhz);
-		num_80mhz_with_ru++;
-	}
-
-	ru_type_80_2 = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_RU_TYPE_80_2);
-	ru_start_index_80_2 = le32_get_bits(rx_usr_info->info3,
-					    HAL_RX_USR_INFO3_RU_START_IDX_80_2);
-	if (ru_type_80_2 != HAL_EHT_RU_NONE) {
-		ru_size += ru_type_80_2;
-		ru_index_per80mhz = ru_start_index_80_2;
-		ru_index = ru_index_per80mhz;
-		ru_index_320mhz |= HAL_RU_PER80(ru_type_80_2, 2, ru_index_per80mhz);
-		num_80mhz_with_ru++;
-	}
-
-	ru_type_80_3 = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_RU_TYPE_80_3);
-	ru_start_index_80_3 = le32_get_bits(rx_usr_info->info2,
-					    HAL_RX_USR_INFO3_RU_START_IDX_80_3);
-	if (ru_type_80_3 != HAL_EHT_RU_NONE) {
-		ru_size += ru_type_80_3;
-		ru_index_per80mhz = ru_start_index_80_3;
-		ru_index = ru_index_per80mhz;
-		ru_index_320mhz |= HAL_RU_PER80(ru_type_80_3, 3, ru_index_per80mhz);
-		num_80mhz_with_ru++;
-	}
-
-	if (num_80mhz_with_ru > 1) {
-		/* Calculate the MRU index */
-		switch (ru_index_320mhz) {
-		case HAL_EHT_RU_996_484_0:
-		case HAL_EHT_RU_996x2_484_0:
-		case HAL_EHT_RU_996x3_484_0:
-			ru_index = 0;
-			break;
-		case HAL_EHT_RU_996_484_1:
-		case HAL_EHT_RU_996x2_484_1:
-		case HAL_EHT_RU_996x3_484_1:
-			ru_index = 1;
-			break;
-		case HAL_EHT_RU_996_484_2:
-		case HAL_EHT_RU_996x2_484_2:
-		case HAL_EHT_RU_996x3_484_2:
-			ru_index = 2;
-			break;
-		case HAL_EHT_RU_996_484_3:
-		case HAL_EHT_RU_996x2_484_3:
-		case HAL_EHT_RU_996x3_484_3:
-			ru_index = 3;
-			break;
-		case HAL_EHT_RU_996_484_4:
-		case HAL_EHT_RU_996x2_484_4:
-		case HAL_EHT_RU_996x3_484_4:
-			ru_index = 4;
-			break;
-		case HAL_EHT_RU_996_484_5:
-		case HAL_EHT_RU_996x2_484_5:
-		case HAL_EHT_RU_996x3_484_5:
-			ru_index = 5;
-			break;
-		case HAL_EHT_RU_996_484_6:
-		case HAL_EHT_RU_996x2_484_6:
-		case HAL_EHT_RU_996x3_484_6:
-			ru_index = 6;
-			break;
-		case HAL_EHT_RU_996_484_7:
-		case HAL_EHT_RU_996x2_484_7:
-		case HAL_EHT_RU_996x3_484_7:
-			ru_index = 7;
-			break;
-		case HAL_EHT_RU_996x2_484_8:
-			ru_index = 8;
-			break;
-		case HAL_EHT_RU_996x2_484_9:
-			ru_index = 9;
-			break;
-		case HAL_EHT_RU_996x2_484_10:
-			ru_index = 10;
-			break;
-		case HAL_EHT_RU_996x2_484_11:
-			ru_index = 11;
-			break;
-		default:
-			ru_index = HAL_EHT_RU_INVALID;
-			break;
-		}
-
-		ru_size += 4;
-	}
-
-	rtap_ru_size = hal_rx_mon_hal_ru_size_to_ath12k_ru_size(ru_size);
-	if (rtap_ru_size != ATH12K_EHT_RU_INVALID) {
-		u32 known, data;
-
-		known = __le32_to_cpu(eht->known);
-		known |= IEEE80211_RADIOTAP_EHT_KNOWN_RU_MRU_SIZE_OM;
-		eht->known = cpu_to_le32(known);
-
-		data = __le32_to_cpu(eht->data[1]);
-		data |=	u32_encode_bits(rtap_ru_size,
-					IEEE80211_RADIOTAP_EHT_DATA1_RU_SIZE);
-		eht->data[1] = cpu_to_le32(data);
-	}
-
-	if (ru_index != HAL_EHT_RU_INVALID) {
-		u32 known, data;
-
-		known = __le32_to_cpu(eht->known);
-		known |= IEEE80211_RADIOTAP_EHT_KNOWN_RU_MRU_INDEX_OM;
-		eht->known = cpu_to_le32(known);
-
-		data = __le32_to_cpu(eht->data[1]);
-		data |=	u32_encode_bits(rtap_ru_size,
-					IEEE80211_RADIOTAP_EHT_DATA1_RU_INDEX);
-		eht->data[1] = cpu_to_le32(data);
-	}
-
-	if (mon_rx_user_status && ru_index != HAL_EHT_RU_INVALID &&
-	    rtap_ru_size != ATH12K_EHT_RU_INVALID) {
-		mon_rx_user_status->ul_ofdma_ru_start_index = ru_index;
-		mon_rx_user_status->ul_ofdma_ru_size = rtap_ru_size;
-
-		ru_width = hal_rx_ul_ofdma_ru_size_to_width(rtap_ru_size);
-
-		mon_rx_user_status->ul_ofdma_ru_width = ru_width;
-		mon_rx_user_status->ofdma_info_valid = 1;
-	}
-}
-
-static void ath12k_dp_mon_parse_rx_msdu_end_err(u32 info, u32 *errmap)
-{
-	if (info & RX_MSDU_END_INFO13_FCS_ERR)
-		*errmap |= HAL_RX_MPDU_ERR_FCS;
-
-	if (info & RX_MSDU_END_INFO13_DECRYPT_ERR)
-		*errmap |= HAL_RX_MPDU_ERR_DECRYPT;
-
-	if (info & RX_MSDU_END_INFO13_TKIP_MIC_ERR)
-		*errmap |= HAL_RX_MPDU_ERR_TKIP_MIC;
-
-	if (info & RX_MSDU_END_INFO13_A_MSDU_ERROR)
-		*errmap |= HAL_RX_MPDU_ERR_AMSDU_ERR;
-
-	if (info & RX_MSDU_END_INFO13_OVERFLOW_ERR)
-		*errmap |= HAL_RX_MPDU_ERR_OVERFLOW;
-
-	if (info & RX_MSDU_END_INFO13_MSDU_LEN_ERR)
-		*errmap |= HAL_RX_MPDU_ERR_MSDU_LEN;
-
-	if (info & RX_MSDU_END_INFO13_MPDU_LEN_ERR)
-		*errmap |= HAL_RX_MPDU_ERR_MPDU_LEN;
-}
-
-static void
-ath12k_parse_cmn_usr_info(const struct hal_phyrx_common_user_info *cmn_usr_info,
-			  struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht;
-	u32 known, data, cp_setting, ltf_size;
-
-	known = __le32_to_cpu(eht->known);
-	known |= IEEE80211_RADIOTAP_EHT_KNOWN_GI |
-		IEEE80211_RADIOTAP_EHT_KNOWN_EHT_LTF;
-	eht->known = cpu_to_le32(known);
-
-	cp_setting = le32_get_bits(cmn_usr_info->info0,
-				   HAL_RX_CMN_USR_INFO0_CP_SETTING);
-	ltf_size = le32_get_bits(cmn_usr_info->info0,
-				 HAL_RX_CMN_USR_INFO0_LTF_SIZE);
-
-	data = __le32_to_cpu(eht->data[0]);
-	data |= u32_encode_bits(cp_setting, IEEE80211_RADIOTAP_EHT_DATA0_GI);
-	data |= u32_encode_bits(ltf_size, IEEE80211_RADIOTAP_EHT_DATA0_LTF);
-	eht->data[0] = cpu_to_le32(data);
-
-	if (!ppdu_info->ltf_size)
-		ppdu_info->ltf_size = ltf_size;
-	if (!ppdu_info->gi)
-		ppdu_info->gi = cp_setting;
-}
-
-static void
-ath12k_dp_mon_parse_status_msdu_end(struct ath12k_mon_data *pmon,
-				    const struct hal_rx_msdu_end *msdu_end)
-{
-	ath12k_dp_mon_parse_rx_msdu_end_err(__le32_to_cpu(msdu_end->info2),
-					    &pmon->err_bitmap);
-	pmon->decap_format = le32_get_bits(msdu_end->info1,
-					   RX_MSDU_END_INFO11_DECAP_FORMAT);
-}
-
-static enum hal_rx_mon_status
-ath12k_dp_mon_rx_parse_status_tlv(struct ath12k *ar,
-				  struct ath12k_mon_data *pmon,
-				  const struct hal_tlv_64_hdr *tlv)
-{
-	struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info;
-	const void *tlv_data = tlv->value;
-	u32 info[7], userid;
-	u16 tlv_tag, tlv_len;
-
-	tlv_tag = le64_get_bits(tlv->tl, HAL_TLV_64_HDR_TAG);
-	tlv_len = le64_get_bits(tlv->tl, HAL_TLV_64_HDR_LEN);
-	userid = le64_get_bits(tlv->tl, HAL_TLV_64_USR_ID);
-
-	if (ppdu_info->tlv_aggr.in_progress && ppdu_info->tlv_aggr.tlv_tag != tlv_tag) {
-		ath12k_dp_mon_parse_eht_sig_hdr(ppdu_info, ppdu_info->tlv_aggr.buf);
-
-		ppdu_info->tlv_aggr.in_progress = false;
-		ppdu_info->tlv_aggr.cur_len = 0;
-	}
-
-	switch (tlv_tag) {
-	case HAL_RX_PPDU_START: {
-		const struct hal_rx_ppdu_start *ppdu_start = tlv_data;
-
-		u64 ppdu_ts = ath12k_le32hilo_to_u64(ppdu_start->ppdu_start_ts_63_32,
-						     ppdu_start->ppdu_start_ts_31_0);
-
-		info[0] = __le32_to_cpu(ppdu_start->info0);
-
-		ppdu_info->ppdu_id = u32_get_bits(info[0],
-						  HAL_RX_PPDU_START_INFO0_PPDU_ID);
-
-		info[1] = __le32_to_cpu(ppdu_start->info1);
-		ppdu_info->chan_num = u32_get_bits(info[1],
-						   HAL_RX_PPDU_START_INFO1_CHAN_NUM);
-		ppdu_info->freq = u32_get_bits(info[1],
-					       HAL_RX_PPDU_START_INFO1_CHAN_FREQ);
-		ppdu_info->ppdu_ts = ppdu_ts;
-
-		if (ppdu_info->ppdu_id != ppdu_info->last_ppdu_id) {
-			ppdu_info->last_ppdu_id = ppdu_info->ppdu_id;
-			ppdu_info->num_users = 0;
-			memset(&ppdu_info->mpdu_fcs_ok_bitmap, 0,
-			       HAL_RX_NUM_WORDS_PER_PPDU_BITMAP *
-			       sizeof(ppdu_info->mpdu_fcs_ok_bitmap[0]));
-		}
-		break;
-	}
-	case HAL_RX_PPDU_END_USER_STATS: {
-		const struct hal_rx_ppdu_end_user_stats *eu_stats = tlv_data;
-		u32 tid_bitmap;
-
-		info[0] = __le32_to_cpu(eu_stats->info0);
-		info[1] = __le32_to_cpu(eu_stats->info1);
-		info[2] = __le32_to_cpu(eu_stats->info2);
-		info[4] = __le32_to_cpu(eu_stats->info4);
-		info[5] = __le32_to_cpu(eu_stats->info5);
-		info[6] = __le32_to_cpu(eu_stats->info6);
-
-		ppdu_info->ast_index =
-			u32_get_bits(info[2], HAL_RX_PPDU_END_USER_STATS_INFO2_AST_INDEX);
-		ppdu_info->fc_valid =
-			u32_get_bits(info[1], HAL_RX_PPDU_END_USER_STATS_INFO1_FC_VALID);
-		tid_bitmap = u32_get_bits(info[6],
-					  HAL_RX_PPDU_END_USER_STATS_INFO6_TID_BITMAP);
-		ppdu_info->tid = ffs(tid_bitmap) - 1;
-		ppdu_info->tcp_msdu_count =
-			u32_get_bits(info[4],
-				     HAL_RX_PPDU_END_USER_STATS_INFO4_TCP_MSDU_CNT);
-		ppdu_info->udp_msdu_count =
-			u32_get_bits(info[4],
-				     HAL_RX_PPDU_END_USER_STATS_INFO4_UDP_MSDU_CNT);
-		ppdu_info->other_msdu_count =
-			u32_get_bits(info[5],
-				     HAL_RX_PPDU_END_USER_STATS_INFO5_OTHER_MSDU_CNT);
-		ppdu_info->tcp_ack_msdu_count =
-			u32_get_bits(info[5],
-				     HAL_RX_PPDU_END_USER_STATS_INFO5_TCP_ACK_MSDU_CNT);
-		ppdu_info->preamble_type =
-			u32_get_bits(info[1],
-				     HAL_RX_PPDU_END_USER_STATS_INFO1_PKT_TYPE);
-		ppdu_info->num_mpdu_fcs_ok =
-			u32_get_bits(info[1],
-				     HAL_RX_PPDU_END_USER_STATS_INFO1_MPDU_CNT_FCS_OK);
-		ppdu_info->num_mpdu_fcs_err =
-			u32_get_bits(info[0],
-				     HAL_RX_PPDU_END_USER_STATS_INFO0_MPDU_CNT_FCS_ERR);
-		ppdu_info->peer_id =
-			u32_get_bits(info[0], HAL_RX_PPDU_END_USER_STATS_INFO0_PEER_ID);
-
-		switch (ppdu_info->preamble_type) {
-		case HAL_RX_PREAMBLE_11N:
-			ppdu_info->ht_flags = 1;
-			break;
-		case HAL_RX_PREAMBLE_11AC:
-			ppdu_info->vht_flags = 1;
-			break;
-		case HAL_RX_PREAMBLE_11AX:
-			ppdu_info->he_flags = 1;
-			break;
-		case HAL_RX_PREAMBLE_11BE:
-			ppdu_info->is_eht = true;
-			break;
-		default:
-			break;
-		}
-
-		if (userid < HAL_MAX_UL_MU_USERS) {
-			struct hal_rx_user_status *rxuser_stats =
-				&ppdu_info->userstats[userid];
-
-			if (ppdu_info->num_mpdu_fcs_ok > 1 ||
-			    ppdu_info->num_mpdu_fcs_err > 1)
-				ppdu_info->userstats[userid].ampdu_present = true;
-
-			ppdu_info->num_users += 1;
-
-			ath12k_dp_mon_rx_handle_ofdma_info(eu_stats, rxuser_stats);
-			ath12k_dp_mon_rx_populate_mu_user_info(eu_stats, ppdu_info,
-							       rxuser_stats);
-		}
-		ppdu_info->mpdu_fcs_ok_bitmap[0] = __le32_to_cpu(eu_stats->rsvd1[0]);
-		ppdu_info->mpdu_fcs_ok_bitmap[1] = __le32_to_cpu(eu_stats->rsvd1[1]);
-		break;
-	}
-	case HAL_RX_PPDU_END_USER_STATS_EXT: {
-		const struct hal_rx_ppdu_end_user_stats_ext *eu_stats = tlv_data;
-
-		ppdu_info->mpdu_fcs_ok_bitmap[2] = __le32_to_cpu(eu_stats->info1);
-		ppdu_info->mpdu_fcs_ok_bitmap[3] = __le32_to_cpu(eu_stats->info2);
-		ppdu_info->mpdu_fcs_ok_bitmap[4] = __le32_to_cpu(eu_stats->info3);
-		ppdu_info->mpdu_fcs_ok_bitmap[5] = __le32_to_cpu(eu_stats->info4);
-		ppdu_info->mpdu_fcs_ok_bitmap[6] = __le32_to_cpu(eu_stats->info5);
-		ppdu_info->mpdu_fcs_ok_bitmap[7] = __le32_to_cpu(eu_stats->info6);
-		break;
-	}
-	case HAL_PHYRX_HT_SIG:
-		ath12k_dp_mon_parse_ht_sig(tlv_data, ppdu_info);
-		break;
-
-	case HAL_PHYRX_L_SIG_B:
-		ath12k_dp_mon_parse_l_sig_b(tlv_data, ppdu_info);
-		break;
-
-	case HAL_PHYRX_L_SIG_A:
-		ath12k_dp_mon_parse_l_sig_a(tlv_data, ppdu_info);
-		break;
-
-	case HAL_PHYRX_VHT_SIG_A:
-		ath12k_dp_mon_parse_vht_sig_a(tlv_data, ppdu_info);
-		break;
-
-	case HAL_PHYRX_HE_SIG_A_SU:
-		ath12k_dp_mon_parse_he_sig_su(tlv_data, ppdu_info);
-		break;
-
-	case HAL_PHYRX_HE_SIG_A_MU_DL:
-		ath12k_dp_mon_parse_he_sig_mu(tlv_data, ppdu_info);
-		break;
-
-	case HAL_PHYRX_HE_SIG_B1_MU:
-		ath12k_dp_mon_parse_he_sig_b1_mu(tlv_data, ppdu_info);
-		break;
-
-	case HAL_PHYRX_HE_SIG_B2_MU:
-		ath12k_dp_mon_parse_he_sig_b2_mu(tlv_data, ppdu_info);
-		break;
-
-	case HAL_PHYRX_HE_SIG_B2_OFDMA:
-		ath12k_dp_mon_parse_he_sig_b2_ofdma(tlv_data, ppdu_info);
-		break;
-
-	case HAL_PHYRX_RSSI_LEGACY: {
-		const struct hal_rx_phyrx_rssi_legacy_info *rssi = tlv_data;
-
-		info[0] = __le32_to_cpu(rssi->info0);
-		info[2] = __le32_to_cpu(rssi->info2);
-
-		/* TODO: Please note that the combined rssi will not be accurate
-		 * in MU case. Rssi in MU needs to be retrieved from
-		 * PHYRX_OTHER_RECEIVE_INFO TLV.
-		 */
-		ppdu_info->rssi_comb =
-			u32_get_bits(info[2],
-				     HAL_RX_RSSI_LEGACY_INFO_INFO2_RSSI_COMB_PPDU);
-
-		ppdu_info->bw = u32_get_bits(info[0],
-					     HAL_RX_RSSI_LEGACY_INFO_INFO0_RX_BW);
-		break;
-	}
-	case HAL_PHYRX_COMMON_USER_INFO: {
-		ath12k_parse_cmn_usr_info(tlv_data, ppdu_info);
-		break;
-	}
-	case HAL_RX_PPDU_START_USER_INFO:
-		ath12k_dp_mon_hal_rx_parse_user_info(tlv_data, userid, ppdu_info);
-		break;
-
-	case HAL_RXPCU_PPDU_END_INFO: {
-		const struct hal_rx_ppdu_end_duration *ppdu_rx_duration = tlv_data;
-
-		info[0] = __le32_to_cpu(ppdu_rx_duration->info0);
-		ppdu_info->rx_duration =
-			u32_get_bits(info[0], HAL_RX_PPDU_END_DURATION);
-		ppdu_info->tsft = __le32_to_cpu(ppdu_rx_duration->rsvd0[1]);
-		ppdu_info->tsft = (ppdu_info->tsft << 32) |
-				   __le32_to_cpu(ppdu_rx_duration->rsvd0[0]);
-		break;
-	}
-	case HAL_RX_MPDU_START: {
-		const struct hal_rx_mpdu_start *mpdu_start = tlv_data;
-		u16 peer_id;
-
-		info[1] = __le32_to_cpu(mpdu_start->info1);
-		peer_id = u32_get_bits(info[1], HAL_RX_MPDU_START_INFO1_PEERID);
-		if (peer_id)
-			ppdu_info->peer_id = peer_id;
-
-		ppdu_info->mpdu_len += u32_get_bits(info[1],
-						    HAL_RX_MPDU_START_INFO2_MPDU_LEN);
-		if (userid < HAL_MAX_UL_MU_USERS) {
-			info[0] = __le32_to_cpu(mpdu_start->info0);
-			ppdu_info->userid = userid;
-			ppdu_info->userstats[userid].ampdu_id =
-				u32_get_bits(info[0], HAL_RX_MPDU_START_INFO0_PPDU_ID);
-		}
-
-		return HAL_RX_MON_STATUS_MPDU_START;
-	}
-	case HAL_RX_MSDU_START:
-		/* TODO: add msdu start parsing logic */
-		break;
-	case HAL_MON_BUF_ADDR:
-		return HAL_RX_MON_STATUS_BUF_ADDR;
-	case HAL_RX_MSDU_END:
-		ath12k_dp_mon_parse_status_msdu_end(pmon, tlv_data);
-		return HAL_RX_MON_STATUS_MSDU_END;
-	case HAL_RX_MPDU_END:
-		return HAL_RX_MON_STATUS_MPDU_END;
-	case HAL_PHYRX_GENERIC_U_SIG:
-		ath12k_dp_mon_hal_rx_parse_u_sig_hdr(tlv_data, ppdu_info);
-		break;
-	case HAL_PHYRX_GENERIC_EHT_SIG:
-		/* Handle the case where aggregation is in progress
-		 * or the current TLV is one of the TLVs which should be
-		 * aggregated
-		 */
-		if (!ppdu_info->tlv_aggr.in_progress) {
-			ppdu_info->tlv_aggr.in_progress = true;
-			ppdu_info->tlv_aggr.tlv_tag = tlv_tag;
-			ppdu_info->tlv_aggr.cur_len = 0;
-		}
-
-		ppdu_info->is_eht = true;
-
-		ath12k_dp_mon_hal_aggr_tlv(ppdu_info, tlv_len, tlv_data);
-		break;
-	case HAL_DUMMY:
-		return HAL_RX_MON_STATUS_BUF_DONE;
-	case HAL_RX_PPDU_END_STATUS_DONE:
-	case 0:
-		return HAL_RX_MON_STATUS_PPDU_DONE;
-	default:
-		break;
-	}
-
-	return HAL_RX_MON_STATUS_PPDU_NOT_DONE;
-}
-
-static void
-ath12k_dp_mon_fill_rx_stats_info(struct ath12k *ar,
-				 struct hal_rx_mon_ppdu_info *ppdu_info,
+ath12k_dp_mon_fill_rx_stats_info(struct hal_rx_mon_ppdu_info *ppdu_info,
 				 struct ieee80211_rx_status *rx_status)
 {
 	u32 center_freq = ppdu_info->freq;
@@ -1781,7 +36,7 @@ ath12k_dp_mon_fill_rx_stats_info(struct ath12k *ar,
 	}
 }
 
-static struct sk_buff
+struct sk_buff
 *ath12k_dp_rx_alloc_mon_status_buf(struct ath12k_base *ab,
 				   struct dp_rxdma_mon_ring *rx_ring,
 				   int *buf_id)
@@ -1824,48 +79,9 @@ static struct sk_buff
 fail_alloc_skb:
 	return NULL;
 }
+EXPORT_SYMBOL(ath12k_dp_rx_alloc_mon_status_buf);
 
-static enum dp_mon_status_buf_state
-ath12k_dp_rx_mon_buf_done(struct ath12k_base *ab, struct hal_srng *srng,
-			  struct dp_rxdma_mon_ring *rx_ring)
-{
-	struct ath12k_skb_rxcb *rxcb;
-	struct hal_tlv_64_hdr *tlv;
-	struct sk_buff *skb;
-	void *status_desc;
-	dma_addr_t paddr;
-	u32 cookie;
-	int buf_id;
-	u8 rbm;
-
-	status_desc = ath12k_hal_srng_src_next_peek(ab, srng);
-	if (!status_desc)
-		return DP_MON_STATUS_NO_DMA;
-
-	ath12k_hal_rx_buf_addr_info_get(status_desc, &paddr, &cookie, &rbm);
-
-	buf_id = u32_get_bits(cookie, DP_RXDMA_BUF_COOKIE_BUF_ID);
-
-	spin_lock_bh(&rx_ring->idr_lock);
-	skb = idr_find(&rx_ring->bufs_idr, buf_id);
-	spin_unlock_bh(&rx_ring->idr_lock);
-
-	if (!skb)
-		return DP_MON_STATUS_NO_DMA;
-
-	rxcb = ATH12K_SKB_RXCB(skb);
-	dma_sync_single_for_cpu(ab->dev, rxcb->paddr,
-				skb->len + skb_tailroom(skb),
-				DMA_FROM_DEVICE);
-
-	tlv = (struct hal_tlv_64_hdr *)skb->data;
-	if (le64_get_bits(tlv->tl, HAL_TLV_HDR_TAG) != HAL_RX_STATUS_BUFFER_DONE)
-		return DP_MON_STATUS_NO_DMA;
-
-	return DP_MON_STATUS_REPLINISH;
-}
-
-static u32 ath12k_dp_mon_comp_ppduid(u32 msdu_ppdu_id, u32 *ppdu_id)
+u32 ath12k_dp_mon_comp_ppduid(u32 msdu_ppdu_id, u32 *ppdu_id)
 {
 	u32 ret = 0;
 
@@ -1884,26 +100,15 @@ static u32 ath12k_dp_mon_comp_ppduid(u32 msdu_ppdu_id, u32 *ppdu_id)
 	}
 	return ret;
 }
-
-static
-void ath12k_dp_mon_next_link_desc_get(struct hal_rx_msdu_link *msdu_link,
-				      dma_addr_t *paddr, u32 *sw_cookie, u8 *rbm,
-				      struct ath12k_buffer_addr **pp_buf_addr_info)
-{
-	struct ath12k_buffer_addr *buf_addr_info;
-
-	buf_addr_info = &msdu_link->buf_addr_info;
-
-	ath12k_hal_rx_buf_addr_info_get(buf_addr_info, paddr, sw_cookie, rbm);
-
-	*pp_buf_addr_info = buf_addr_info;
-}
+EXPORT_SYMBOL(ath12k_dp_mon_comp_ppduid);
 
 static void
-ath12k_dp_mon_fill_rx_rate(struct ath12k *ar,
+ath12k_dp_mon_fill_rx_rate(struct ath12k_pdev_dp *dp_pdev,
 			   struct hal_rx_mon_ppdu_info *ppdu_info,
 			   struct ieee80211_rx_status *rx_status)
 {
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_base *ab = dp->ab;
 	struct ieee80211_supported_band *sband;
 	enum rx_msdu_start_pkt_type pkt_type;
 	u8 rate_mcs, nss, sgi;
@@ -1919,6 +124,8 @@ ath12k_dp_mon_fill_rx_rate(struct ath12k *ar,
 	case RX_MSDU_START_PKT_TYPE_11B:
 		is_cck = (pkt_type == RX_MSDU_START_PKT_TYPE_11B);
 		if (rx_status->band < NUM_NL80211_BANDS) {
+			struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev);
+
 			sband = &ar->mac.sbands[rx_status->band];
 			rx_status->rate_idx = ath12k_mac_hw_rate_to_idx(sband, rate_mcs,
 									is_cck);
@@ -1927,7 +134,7 @@ ath12k_dp_mon_fill_rx_rate(struct ath12k *ar,
 	case RX_MSDU_START_PKT_TYPE_11N:
 		rx_status->encoding = RX_ENC_HT;
 		if (rate_mcs > ATH12K_HT_MCS_MAX) {
-			ath12k_warn(ar->ab,
+			ath12k_warn(ab,
 				    "Received with invalid mcs in HT mode %d\n",
 				     rate_mcs);
 			break;
@@ -1940,7 +147,7 @@ ath12k_dp_mon_fill_rx_rate(struct ath12k *ar,
 		rx_status->encoding = RX_ENC_VHT;
 		rx_status->rate_idx = rate_mcs;
 		if (rate_mcs > ATH12K_VHT_MCS_MAX) {
-			ath12k_warn(ar->ab,
+			ath12k_warn(ab,
 				    "Received with invalid mcs in VHT mode %d\n",
 				     rate_mcs);
 			break;
@@ -1951,7 +158,7 @@ ath12k_dp_mon_fill_rx_rate(struct ath12k *ar,
 	case RX_MSDU_START_PKT_TYPE_11AX:
 		rx_status->rate_idx = rate_mcs;
 		if (rate_mcs > ATH12K_HE_MCS_MAX) {
-			ath12k_warn(ar->ab,
+			ath12k_warn(ab,
 				    "Received with invalid mcs in HE mode %d\n",
 				    rate_mcs);
 			break;
@@ -1962,7 +169,7 @@ ath12k_dp_mon_fill_rx_rate(struct ath12k *ar,
 	case RX_MSDU_START_PKT_TYPE_11BE:
 		rx_status->rate_idx = rate_mcs;
 		if (rate_mcs > ATH12K_EHT_MCS_MAX) {
-			ath12k_warn(ar->ab,
+			ath12k_warn(ab,
 				    "Received with invalid mcs in EHT mode %d\n",
 				    rate_mcs);
 			break;
@@ -1971,24 +178,24 @@ ath12k_dp_mon_fill_rx_rate(struct ath12k *ar,
 		rx_status->he_gi = ath12k_he_gi_to_nl80211_he_gi(sgi);
 		break;
 	default:
-		ath12k_dbg(ar->ab, ATH12K_DBG_DATA,
+		ath12k_dbg(ab, ATH12K_DBG_DATA,
 			   "monitor receives invalid preamble type %d",
 			    pkt_type);
 		break;
 	}
 }
 
-static void ath12k_dp_mon_rx_msdus_set_payload(struct ath12k *ar,
+static void ath12k_dp_mon_rx_msdus_set_payload(struct ath12k_base *ab,
 					       struct sk_buff *head_msdu,
 					       struct sk_buff *tail_msdu)
 {
 	u32 rx_pkt_offset, l2_hdr_offset, total_offset;
 
-	rx_pkt_offset = ar->ab->hal.hal_desc_sz;
+	rx_pkt_offset = ab->hal.hal_desc_sz;
 	l2_hdr_offset =
-		ath12k_dp_rx_h_l3pad(ar->ab, (struct hal_rx_desc *)tail_msdu->data);
+		ath12k_dp_rx_h_l3pad(ab, (struct hal_rx_desc *)tail_msdu->data);
 
-	if (ar->ab->hw_params->rxdma1_enable)
+	if (ab->hw_params->rxdma1_enable)
 		total_offset = ATH12K_MON_RX_PKT_OFFSET;
 	else
 		total_offset = rx_pkt_offset + l2_hdr_offset;
@@ -1996,13 +203,14 @@ static void ath12k_dp_mon_rx_msdus_set_payload(struct ath12k *ar,
 	skb_pull(head_msdu, total_offset);
 }
 
-static struct sk_buff *
-ath12k_dp_mon_rx_merg_msdus(struct ath12k *ar,
+struct sk_buff *
+ath12k_dp_mon_rx_merg_msdus(struct ath12k_pdev_dp *dp_pdev,
 			    struct dp_mon_mpdu *mon_mpdu,
 			    struct hal_rx_mon_ppdu_info *ppdu_info,
 			    struct ieee80211_rx_status *rxs)
 {
-	struct ath12k_base *ab = ar->ab;
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_base *ab = dp->ab;
 	struct sk_buff *msdu, *mpdu_buf, *prev_buf, *head_frag_list;
 	struct sk_buff *head_msdu, *tail_msdu;
 	struct hal_rx_desc *rx_desc;
@@ -2019,11 +227,13 @@ ath12k_dp_mon_rx_merg_msdus(struct ath12k *ar,
 	if (!head_msdu || !tail_msdu)
 		goto err_merge_fail;
 
-	ath12k_dp_mon_fill_rx_stats_info(ar, ppdu_info, rxs);
+	ath12k_dp_mon_fill_rx_stats_info(ppdu_info, rxs);
 
 	if (unlikely(rxs->band == NUM_NL80211_BANDS ||
-		     !ath12k_ar_to_hw(ar)->wiphy->bands[rxs->band])) {
-		ath12k_dbg(ar->ab, ATH12K_DBG_DATA,
+		     !ath12k_pdev_dp_to_hw(dp_pdev)->wiphy->bands[rxs->band])) {
+		struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev);
+
+		ath12k_dbg(ab, ATH12K_DBG_DATA,
 			   "sband is NULL for status band %d channel_num %d center_freq %d pdev_id %d\n",
 			   rxs->band, channel_num, ppdu_info->freq, ar->pdev_idx);
 
@@ -2041,17 +251,17 @@ ath12k_dp_mon_rx_merg_msdus(struct ath12k *ar,
 		rxs->freq = ieee80211_channel_to_frequency(channel_num,
 							   rxs->band);
 
-	ath12k_dp_mon_fill_rx_rate(ar, ppdu_info, rxs);
+	ath12k_dp_mon_fill_rx_rate(dp_pdev, ppdu_info, rxs);
 
 	if (decap_format == DP_RX_DECAP_TYPE_RAW) {
-		ath12k_dp_mon_rx_msdus_set_payload(ar, head_msdu, tail_msdu);
+		ath12k_dp_mon_rx_msdus_set_payload(ab, head_msdu, tail_msdu);
 
 		prev_buf = head_msdu;
 		msdu = head_msdu->next;
 		head_frag_list = NULL;
 
 		while (msdu) {
-			ath12k_dp_mon_rx_msdus_set_payload(ar, head_msdu, tail_msdu);
+			ath12k_dp_mon_rx_msdus_set_payload(ab, head_msdu, tail_msdu);
 
 			if (!head_frag_list)
 				head_frag_list = msdu;
@@ -2075,7 +285,7 @@ ath12k_dp_mon_rx_merg_msdus(struct ath12k *ar,
 
 		rx_desc = (struct hal_rx_desc *)head_msdu->data;
 		hdr_desc =
-			ab->hal_rx_ops->rx_desc_get_msdu_payload(rx_desc);
+			ab->hal.ops->rx_desc_get_msdu_payload(rx_desc);
 
 		/* Base size */
 		wh = (struct ieee80211_hdr_3addr *)hdr_desc;
@@ -2086,7 +296,7 @@ ath12k_dp_mon_rx_merg_msdus(struct ath12k *ar,
 		msdu = head_msdu;
 
 		while (msdu) {
-			ath12k_dp_mon_rx_msdus_set_payload(ar, head_msdu, tail_msdu);
+			ath12k_dp_mon_rx_msdus_set_payload(ab, head_msdu, tail_msdu);
 			if (qos_pkt) {
 				dest = skb_push(msdu, sizeof(__le16));
 				if (!dest)
@@ -2121,6 +331,7 @@ ath12k_dp_mon_rx_merg_msdus(struct ath12k *ar,
 	}
 	return NULL;
 }
+EXPORT_SYMBOL(ath12k_dp_mon_rx_merg_msdus);
 
 static void
 ath12k_dp_mon_rx_update_radiotap_he(struct hal_rx_mon_ppdu_info *rx_status,
@@ -2170,11 +381,12 @@ ath12k_dp_mon_rx_update_radiotap_he_mu(struct hal_rx_mon_ppdu_info *rx_status,
 	rtap_buf[rtap_len] = rx_status->he_RU[3];
 }
 
-static void ath12k_dp_mon_update_radiotap(struct ath12k *ar,
-					  struct hal_rx_mon_ppdu_info *ppduinfo,
-					  struct sk_buff *mon_skb,
-					  struct ieee80211_rx_status *rxs)
+void ath12k_dp_mon_update_radiotap(struct ath12k_pdev_dp *dp_pdev,
+				   struct hal_rx_mon_ppdu_info *ppduinfo,
+				   struct sk_buff *mon_skb,
+				   struct ieee80211_rx_status *rxs)
 {
+	struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev);
 	struct ieee80211_supported_band *sband;
 	s32 noise_floor;
 	u8 *ptr = NULL;
@@ -2267,6 +479,9 @@ static void ath12k_dp_mon_update_radiotap(struct ath12k *ar,
 		rxs->encoding = RX_ENC_HT;
 		rxs->rate_idx = ppduinfo->rate;
 	} else {
+		struct ath12k *ar;
+
+		ar = ath12k_pdev_dp_to_ar(dp_pdev);
 		rxs->encoding = RX_ENC_LEGACY;
 		sband = &ar->mac.sbands[rxs->band];
 		rxs->rate_idx = ath12k_mac_hw_rate_to_idx(sband, ppduinfo->rate,
@@ -2275,13 +490,17 @@ static void ath12k_dp_mon_update_radiotap(struct ath12k *ar,
 
 	rxs->mactime = ppduinfo->tsft;
 }
+EXPORT_SYMBOL(ath12k_dp_mon_update_radiotap);
 
-static void ath12k_dp_mon_rx_deliver_msdu(struct ath12k *ar, struct napi_struct *napi,
-					  struct sk_buff *msdu,
-					  const struct hal_rx_mon_ppdu_info *ppduinfo,
-					  struct ieee80211_rx_status *status,
-					  u8 decap)
+void ath12k_dp_mon_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev,
+				   struct napi_struct *napi,
+				   struct sk_buff *msdu,
+				   const struct hal_rx_mon_ppdu_info *ppduinfo,
+				   struct ieee80211_rx_status *status,
+				   u8 decap)
 {
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_base *ab = dp->ab;
 	static const struct ieee80211_radiotap_he known = {
 		.data1 = cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_DATA_MCS_KNOWN |
 				     IEEE80211_RADIOTAP_HE_DATA1_BW_RU_ALLOC_KNOWN),
@@ -2290,10 +509,13 @@ static void ath12k_dp_mon_rx_deliver_msdu(struct ath12k *ar, struct napi_struct
 	struct ieee80211_rx_status *rx_status;
 	struct ieee80211_radiotap_he *he = NULL;
 	struct ieee80211_sta *pubsta = NULL;
-	struct ath12k_peer *peer;
+	struct ath12k_dp_link_peer *peer;
 	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
+	struct hal_rx_desc_data rx_info;
 	bool is_mcbc = rxcb->is_mcbc;
 	bool is_eapol_tkip = rxcb->is_eapol;
+	struct hal_rx_desc *rx_desc = (struct hal_rx_desc *)msdu->data;
+	u8 addr[ETH_ALEN] = {};
 
 	status->link_valid = 0;
 
@@ -2304,23 +526,28 @@ static void ath12k_dp_mon_rx_deliver_msdu(struct ath12k *ar, struct napi_struct
 		status->flag |= RX_FLAG_RADIOTAP_HE;
 	}
 
-	spin_lock_bh(&ar->ab->base_lock);
-	peer = ath12k_peer_find_by_id(ar->ab, ppduinfo->peer_id);
+	ath12k_dp_extract_rx_desc_data(dp->hal, &rx_info, rx_desc, rx_desc);
+
+	rcu_read_lock();
+	spin_lock_bh(&dp->dp_lock);
+	peer = ath12k_dp_rx_h_find_link_peer(dp_pdev, msdu, &rx_info);
 	if (peer && peer->sta) {
 		pubsta = peer->sta;
+		memcpy(addr, peer->addr, ETH_ALEN);
 		if (pubsta->valid_links) {
 			status->link_valid = 1;
 			status->link_id = peer->link_id;
 		}
 	}
 
-	spin_unlock_bh(&ar->ab->base_lock);
+	spin_unlock_bh(&dp->dp_lock);
+	rcu_read_unlock();
 
-	ath12k_dbg(ar->ab, ATH12K_DBG_DATA,
+	ath12k_dbg(ab, ATH12K_DBG_DATA,
 		   "rx skb %p len %u peer %pM %u %s %s%s%s%s%s%s%s%s %srate_idx %u vht_nss %u freq %u band %u flag 0x%x fcs-err %i mic-err %i amsdu-more %i\n",
 		   msdu,
 		   msdu->len,
-		   peer ? peer->addr : NULL,
+		   addr,
 		   rxcb->tid,
 		   (is_mcbc) ? "mcast" : "ucast",
 		   (status->encoding == RX_ENC_LEGACY) ? "legacy" : "",
@@ -2340,7 +567,7 @@ static void ath12k_dp_mon_rx_deliver_msdu(struct ath12k *ar, struct napi_struct
 		   !!(status->flag & RX_FLAG_MMIC_ERROR),
 		   !!(status->flag & RX_FLAG_AMSDU_MORE));
 
-	ath12k_dbg_dump(ar->ab, ATH12K_DBG_DP_RX, NULL, "dp rx msdu: ",
+	ath12k_dbg_dump(ab, ATH12K_DBG_DP_RX, NULL, "dp rx msdu: ",
 			msdu->data, msdu->len);
 	rx_status = IEEE80211_SKB_RXCB(msdu);
 	*rx_status = *status;
@@ -2356,66 +583,11 @@ static void ath12k_dp_mon_rx_deliver_msdu(struct ath12k *ar, struct napi_struct
 	    !(is_mcbc && rx_status->flag & RX_FLAG_DECRYPTED))
 		rx_status->flag |= RX_FLAG_8023;
 
-	ieee80211_rx_napi(ath12k_ar_to_hw(ar), pubsta, msdu, napi);
+	ieee80211_rx_napi(ath12k_pdev_dp_to_hw(dp_pdev), pubsta, msdu, napi);
 }
+EXPORT_SYMBOL(ath12k_dp_mon_rx_deliver_msdu);
 
-static int ath12k_dp_mon_rx_deliver(struct ath12k *ar,
-				    struct dp_mon_mpdu *mon_mpdu,
-				    struct hal_rx_mon_ppdu_info *ppduinfo,
-				    struct napi_struct *napi)
-{
-	struct ath12k_pdev_dp *dp = &ar->dp;
-	struct sk_buff *mon_skb, *skb_next, *header;
-	struct ieee80211_rx_status *rxs = &dp->rx_status;
-	u8 decap = DP_RX_DECAP_TYPE_RAW;
-
-	mon_skb = ath12k_dp_mon_rx_merg_msdus(ar, mon_mpdu, ppduinfo, rxs);
-	if (!mon_skb)
-		goto mon_deliver_fail;
-
-	header = mon_skb;
-	rxs->flag = 0;
-
-	if (mon_mpdu->err_bitmap & HAL_RX_MPDU_ERR_FCS)
-		rxs->flag = RX_FLAG_FAILED_FCS_CRC;
-
-	do {
-		skb_next = mon_skb->next;
-		if (!skb_next)
-			rxs->flag &= ~RX_FLAG_AMSDU_MORE;
-		else
-			rxs->flag |= RX_FLAG_AMSDU_MORE;
-
-		if (mon_skb == header) {
-			header = NULL;
-			rxs->flag &= ~RX_FLAG_ALLOW_SAME_PN;
-		} else {
-			rxs->flag |= RX_FLAG_ALLOW_SAME_PN;
-		}
-		rxs->flag |= RX_FLAG_ONLY_MONITOR;
-
-		if (!(rxs->flag & RX_FLAG_ONLY_MONITOR))
-			decap = mon_mpdu->decap_format;
-
-		ath12k_dp_mon_update_radiotap(ar, ppduinfo, mon_skb, rxs);
-		ath12k_dp_mon_rx_deliver_msdu(ar, napi, mon_skb, ppduinfo, rxs, decap);
-		mon_skb = skb_next;
-	} while (mon_skb);
-	rxs->flag = 0;
-
-	return 0;
-
-mon_deliver_fail:
-	mon_skb = mon_mpdu->head;
-	while (mon_skb) {
-		skb_next = mon_skb->next;
-		dev_kfree_skb_any(mon_skb);
-		mon_skb = skb_next;
-	}
-	return -EINVAL;
-}
-
-static int ath12k_dp_pkt_set_pktlen(struct sk_buff *skb, u32 len)
+int ath12k_dp_pkt_set_pktlen(struct sk_buff *skb, u32 len)
 {
 	if (skb->len > len) {
 		skb_trim(skb, len);
@@ -2432,192 +604,54 @@ static int ath12k_dp_pkt_set_pktlen(struct sk_buff *skb, u32 len)
 
 	return 0;
 }
+EXPORT_SYMBOL(ath12k_dp_pkt_set_pktlen);
 
-/* Hardware fill buffer with 128 bytes aligned. So need to reap it
- * with 128 bytes aligned.
- */
-#define RXDMA_DATA_DMA_BLOCK_SIZE 128
-
-static void
-ath12k_dp_mon_get_buf_len(struct hal_rx_msdu_desc_info *info,
-			  bool *is_frag, u32 *total_len,
-			  u32 *frag_len, u32 *msdu_cnt)
-{
-	if (info->msdu_flags & RX_MSDU_DESC_INFO0_MSDU_CONTINUATION) {
-		*is_frag = true;
-		*frag_len = (RX_MON_STATUS_BASE_BUF_SIZE -
-			     sizeof(struct hal_rx_desc)) &
-			     ~(RXDMA_DATA_DMA_BLOCK_SIZE - 1);
-		*total_len += *frag_len;
-	} else {
-		if (*is_frag)
-			*frag_len = info->msdu_len - *total_len;
-		else
-			*frag_len = info->msdu_len;
-
-		*msdu_cnt -= 1;
-	}
-}
-
-static int
-ath12k_dp_mon_parse_status_buf(struct ath12k *ar,
+int
+ath12k_dp_mon_parse_status_buf(struct ath12k_pdev_dp *dp_pdev,
 			       struct ath12k_mon_data *pmon,
 			       const struct dp_mon_packet_info *packet_info)
 {
-	struct ath12k_base *ab = ar->ab;
-	struct dp_rxdma_mon_ring *buf_ring = &ab->dp.rxdma_mon_buf_ring;
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_base *ab = dp->ab;
+	struct dp_rxdma_mon_ring *buf_ring = &dp->rxdma_mon_buf_ring;
 	struct sk_buff *msdu;
-	int buf_id;
-	u32 offset;
-
-	buf_id = u32_get_bits(packet_info->cookie, DP_RXDMA_BUF_COOKIE_BUF_ID);
-
-	spin_lock_bh(&buf_ring->idr_lock);
-	msdu = idr_remove(&buf_ring->bufs_idr, buf_id);
-	spin_unlock_bh(&buf_ring->idr_lock);
-
-	if (unlikely(!msdu)) {
-		ath12k_warn(ab, "mon dest desc with inval buf_id %d\n", buf_id);
-		return 0;
-	}
-
-	dma_unmap_single(ab->dev, ATH12K_SKB_RXCB(msdu)->paddr,
-			 msdu->len + skb_tailroom(msdu),
-			 DMA_FROM_DEVICE);
-
-	offset = packet_info->dma_length + ATH12K_MON_RX_DOT11_OFFSET;
-	if (ath12k_dp_pkt_set_pktlen(msdu, offset)) {
-		dev_kfree_skb_any(msdu);
-		goto dest_replenish;
-	}
-
-	if (!pmon->mon_mpdu->head)
-		pmon->mon_mpdu->head = msdu;
-	else
-		pmon->mon_mpdu->tail->next = msdu;
-
-	pmon->mon_mpdu->tail = msdu;
-
-dest_replenish:
-	ath12k_dp_mon_buf_replenish(ab, buf_ring, 1);
-
-	return 0;
-}
-
-static int
-ath12k_dp_mon_parse_rx_dest_tlv(struct ath12k *ar,
-				struct ath12k_mon_data *pmon,
-				enum hal_rx_mon_status hal_status,
-				const void *tlv_data)
-{
-	switch (hal_status) {
-	case HAL_RX_MON_STATUS_MPDU_START:
-		if (WARN_ON_ONCE(pmon->mon_mpdu))
-			break;
-
-		pmon->mon_mpdu = kzalloc(sizeof(*pmon->mon_mpdu), GFP_ATOMIC);
-		if (!pmon->mon_mpdu)
-			return -ENOMEM;
-		break;
-	case HAL_RX_MON_STATUS_BUF_ADDR:
-		return ath12k_dp_mon_parse_status_buf(ar, pmon, tlv_data);
-	case HAL_RX_MON_STATUS_MPDU_END:
-		/* If no MSDU then free empty MPDU */
-		if (pmon->mon_mpdu->tail) {
-			pmon->mon_mpdu->tail->next = NULL;
-			list_add_tail(&pmon->mon_mpdu->list, &pmon->dp_rx_mon_mpdu_list);
-		} else {
-			kfree(pmon->mon_mpdu);
-		}
-		pmon->mon_mpdu = NULL;
-		break;
-	case HAL_RX_MON_STATUS_MSDU_END:
-		pmon->mon_mpdu->decap_format = pmon->decap_format;
-		pmon->mon_mpdu->err_bitmap = pmon->err_bitmap;
-		break;
-	default:
-		break;
-	}
-
-	return 0;
-}
-
-static enum hal_rx_mon_status
-ath12k_dp_mon_parse_rx_dest(struct ath12k *ar, struct ath12k_mon_data *pmon,
-			    struct sk_buff *skb)
-{
-	struct hal_tlv_64_hdr *tlv;
-	struct ath12k_skb_rxcb *rxcb;
-	enum hal_rx_mon_status hal_status;
-	u16 tlv_tag, tlv_len;
-	u8 *ptr = skb->data;
-
-	do {
-		tlv = (struct hal_tlv_64_hdr *)ptr;
-		tlv_tag = le64_get_bits(tlv->tl, HAL_TLV_64_HDR_TAG);
-
-		/* The actual length of PPDU_END is the combined length of many PHY
-		 * TLVs that follow. Skip the TLV header and
-		 * rx_rxpcu_classification_overview that follows the header to get to
-		 * next TLV.
-		 */
-
-		if (tlv_tag == HAL_RX_PPDU_END)
-			tlv_len = sizeof(struct hal_rx_rxpcu_classification_overview);
-		else
-			tlv_len = le64_get_bits(tlv->tl, HAL_TLV_64_HDR_LEN);
-
-		hal_status = ath12k_dp_mon_rx_parse_status_tlv(ar, pmon, tlv);
-
-		if (ar->monitor_started && ar->ab->hw_params->rxdma1_enable &&
-		    ath12k_dp_mon_parse_rx_dest_tlv(ar, pmon, hal_status, tlv->value))
-			return HAL_RX_MON_STATUS_PPDU_DONE;
-
-		ptr += sizeof(*tlv) + tlv_len;
-		ptr = PTR_ALIGN(ptr, HAL_TLV_64_ALIGN);
-
-		if ((ptr - skb->data) > skb->len)
-			break;
+	int buf_id;
+	u32 offset;
 
-	} while ((hal_status == HAL_RX_MON_STATUS_PPDU_NOT_DONE) ||
-		 (hal_status == HAL_RX_MON_STATUS_BUF_ADDR) ||
-		 (hal_status == HAL_RX_MON_STATUS_MPDU_START) ||
-		 (hal_status == HAL_RX_MON_STATUS_MPDU_END) ||
-		 (hal_status == HAL_RX_MON_STATUS_MSDU_END));
+	buf_id = u32_get_bits(packet_info->cookie, DP_RXDMA_BUF_COOKIE_BUF_ID);
 
-	rxcb = ATH12K_SKB_RXCB(skb);
-	if (rxcb->is_end_of_ppdu)
-		hal_status = HAL_RX_MON_STATUS_PPDU_DONE;
+	spin_lock_bh(&buf_ring->idr_lock);
+	msdu = idr_remove(&buf_ring->bufs_idr, buf_id);
+	spin_unlock_bh(&buf_ring->idr_lock);
 
-	return hal_status;
-}
+	if (unlikely(!msdu)) {
+		ath12k_warn(ab, "mon dest desc with inval buf_id %d\n", buf_id);
+		return 0;
+	}
 
-enum hal_rx_mon_status
-ath12k_dp_mon_rx_parse_mon_status(struct ath12k *ar,
-				  struct ath12k_mon_data *pmon,
-				  struct sk_buff *skb,
-				  struct napi_struct *napi)
-{
-	struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info;
-	struct dp_mon_mpdu *tmp;
-	struct dp_mon_mpdu *mon_mpdu = pmon->mon_mpdu;
-	enum hal_rx_mon_status hal_status;
+	dma_unmap_single(ab->dev, ATH12K_SKB_RXCB(msdu)->paddr,
+			 msdu->len + skb_tailroom(msdu),
+			 DMA_FROM_DEVICE);
 
-	hal_status = ath12k_dp_mon_parse_rx_dest(ar, pmon, skb);
-	if (hal_status != HAL_RX_MON_STATUS_PPDU_DONE)
-		return hal_status;
+	offset = packet_info->dma_length + ATH12K_MON_RX_DOT11_OFFSET;
+	if (ath12k_dp_pkt_set_pktlen(msdu, offset)) {
+		dev_kfree_skb_any(msdu);
+		goto dest_replenish;
+	}
 
-	list_for_each_entry_safe(mon_mpdu, tmp, &pmon->dp_rx_mon_mpdu_list, list) {
-		list_del(&mon_mpdu->list);
+	if (!pmon->mon_mpdu->head)
+		pmon->mon_mpdu->head = msdu;
+	else
+		pmon->mon_mpdu->tail->next = msdu;
 
-		if (mon_mpdu->head && mon_mpdu->tail)
-			ath12k_dp_mon_rx_deliver(ar, mon_mpdu, ppdu_info, napi);
+	pmon->mon_mpdu->tail = msdu;
 
-		kfree(mon_mpdu);
-	}
+dest_replenish:
+	ath12k_dp_mon_buf_replenish(ab, buf_ring, 1);
 
-	return hal_status;
+	return 0;
 }
+EXPORT_SYMBOL(ath12k_dp_mon_parse_status_buf);
 
 int ath12k_dp_mon_buf_replenish(struct ath12k_base *ab,
 				struct dp_rxdma_mon_ring *buf_ring,
@@ -2693,763 +727,94 @@ int ath12k_dp_mon_buf_replenish(struct ath12k_base *ab,
 	spin_unlock_bh(&srng->lock);
 	return -ENOMEM;
 }
+EXPORT_SYMBOL(ath12k_dp_mon_buf_replenish);
 
 int ath12k_dp_mon_status_bufs_replenish(struct ath12k_base *ab,
 					struct dp_rxdma_mon_ring *rx_ring,
 					int req_entries)
 {
-	enum hal_rx_buf_return_buf_manager mgr =
-		ab->hw_params->hal_params->rx_buf_rbm;
-	int num_free, num_remain, buf_id;
-	struct ath12k_buffer_addr *desc;
-	struct hal_srng *srng;
-	struct sk_buff *skb;
-	dma_addr_t paddr;
-	u32 cookie;
-
-	req_entries = min(req_entries, rx_ring->bufs_max);
-
-	srng = &ab->hal.srng_list[rx_ring->refill_buf_ring.ring_id];
-
-	spin_lock_bh(&srng->lock);
-
-	ath12k_hal_srng_access_begin(ab, srng);
-
-	num_free = ath12k_hal_srng_src_num_free(ab, srng, true);
-	if (!req_entries && (num_free > (rx_ring->bufs_max * 3) / 4))
-		req_entries = num_free;
-
-	req_entries = min(num_free, req_entries);
-	num_remain = req_entries;
-
-	while (num_remain > 0) {
-		skb = dev_alloc_skb(RX_MON_STATUS_BUF_SIZE);
-		if (!skb)
-			break;
-
-		if (!IS_ALIGNED((unsigned long)skb->data,
-				RX_MON_STATUS_BUF_ALIGN)) {
-			skb_pull(skb,
-				 PTR_ALIGN(skb->data, RX_MON_STATUS_BUF_ALIGN) -
-				 skb->data);
-		}
-
-		paddr = dma_map_single(ab->dev, skb->data,
-				       skb->len + skb_tailroom(skb),
-				       DMA_FROM_DEVICE);
-		if (dma_mapping_error(ab->dev, paddr))
-			goto fail_free_skb;
-
-		spin_lock_bh(&rx_ring->idr_lock);
-		buf_id = idr_alloc(&rx_ring->bufs_idr, skb, 0,
-				   rx_ring->bufs_max * 3, GFP_ATOMIC);
-		spin_unlock_bh(&rx_ring->idr_lock);
-		if (buf_id < 0)
-			goto fail_dma_unmap;
-		cookie = u32_encode_bits(buf_id, DP_RXDMA_BUF_COOKIE_BUF_ID);
-
-		desc = ath12k_hal_srng_src_get_next_entry(ab, srng);
-		if (!desc)
-			goto fail_buf_unassign;
-
-		ATH12K_SKB_RXCB(skb)->paddr = paddr;
-
-		num_remain--;
-
-		ath12k_hal_rx_buf_addr_info_set(desc, paddr, cookie, mgr);
-	}
-
-	ath12k_hal_srng_access_end(ab, srng);
-
-	spin_unlock_bh(&srng->lock);
-
-	return req_entries - num_remain;
-
-fail_buf_unassign:
-	spin_lock_bh(&rx_ring->idr_lock);
-	idr_remove(&rx_ring->bufs_idr, buf_id);
-	spin_unlock_bh(&rx_ring->idr_lock);
-fail_dma_unmap:
-	dma_unmap_single(ab->dev, paddr, skb->len + skb_tailroom(skb),
-			 DMA_FROM_DEVICE);
-fail_free_skb:
-	dev_kfree_skb_any(skb);
-
-	ath12k_hal_srng_access_end(ab, srng);
-
-	spin_unlock_bh(&srng->lock);
-
-	return req_entries - num_remain;
-}
-
-static struct dp_mon_tx_ppdu_info *
-ath12k_dp_mon_tx_get_ppdu_info(struct ath12k_mon_data *pmon,
-			       unsigned int ppdu_id,
-			       enum dp_mon_tx_ppdu_info_type type)
-{
-	struct dp_mon_tx_ppdu_info *tx_ppdu_info;
-
-	if (type == DP_MON_TX_PROT_PPDU_INFO) {
-		tx_ppdu_info = pmon->tx_prot_ppdu_info;
-
-		if (tx_ppdu_info && !tx_ppdu_info->is_used)
-			return tx_ppdu_info;
-		kfree(tx_ppdu_info);
-	} else {
-		tx_ppdu_info = pmon->tx_data_ppdu_info;
-
-		if (tx_ppdu_info && !tx_ppdu_info->is_used)
-			return tx_ppdu_info;
-		kfree(tx_ppdu_info);
-	}
-
-	/* allocate new tx_ppdu_info */
-	tx_ppdu_info = kzalloc(sizeof(*tx_ppdu_info), GFP_ATOMIC);
-	if (!tx_ppdu_info)
-		return NULL;
-
-	tx_ppdu_info->is_used = 0;
-	tx_ppdu_info->ppdu_id = ppdu_id;
-
-	if (type == DP_MON_TX_PROT_PPDU_INFO)
-		pmon->tx_prot_ppdu_info = tx_ppdu_info;
-	else
-		pmon->tx_data_ppdu_info = tx_ppdu_info;
-
-	return tx_ppdu_info;
-}
-
-static struct dp_mon_tx_ppdu_info *
-ath12k_dp_mon_hal_tx_ppdu_info(struct ath12k_mon_data *pmon,
-			       u16 tlv_tag)
-{
-	switch (tlv_tag) {
-	case HAL_TX_FES_SETUP:
-	case HAL_TX_FLUSH:
-	case HAL_PCU_PPDU_SETUP_INIT:
-	case HAL_TX_PEER_ENTRY:
-	case HAL_TX_QUEUE_EXTENSION:
-	case HAL_TX_MPDU_START:
-	case HAL_TX_MSDU_START:
-	case HAL_TX_DATA:
-	case HAL_MON_BUF_ADDR:
-	case HAL_TX_MPDU_END:
-	case HAL_TX_LAST_MPDU_FETCHED:
-	case HAL_TX_LAST_MPDU_END:
-	case HAL_COEX_TX_REQ:
-	case HAL_TX_RAW_OR_NATIVE_FRAME_SETUP:
-	case HAL_SCH_CRITICAL_TLV_REFERENCE:
-	case HAL_TX_FES_SETUP_COMPLETE:
-	case HAL_TQM_MPDU_GLOBAL_START:
-	case HAL_SCHEDULER_END:
-	case HAL_TX_FES_STATUS_USER_PPDU:
-		break;
-	case HAL_TX_FES_STATUS_PROT: {
-		if (!pmon->tx_prot_ppdu_info->is_used)
-			pmon->tx_prot_ppdu_info->is_used = true;
-
-		return pmon->tx_prot_ppdu_info;
-	}
-	}
-
-	if (!pmon->tx_data_ppdu_info->is_used)
-		pmon->tx_data_ppdu_info->is_used = true;
-
-	return pmon->tx_data_ppdu_info;
-}
-
-#define MAX_MONITOR_HEADER 512
-#define MAX_DUMMY_FRM_BODY 128
-
-struct sk_buff *ath12k_dp_mon_tx_alloc_skb(void)
-{
-	struct sk_buff *skb;
-
-	skb = dev_alloc_skb(MAX_MONITOR_HEADER + MAX_DUMMY_FRM_BODY);
-	if (!skb)
-		return NULL;
-
-	skb_reserve(skb, MAX_MONITOR_HEADER);
-
-	if (!IS_ALIGNED((unsigned long)skb->data, 4))
-		skb_pull(skb, PTR_ALIGN(skb->data, 4) - skb->data);
-
-	return skb;
-}
-
-static int
-ath12k_dp_mon_tx_gen_cts2self_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info)
-{
-	struct sk_buff *skb;
-	struct ieee80211_cts *cts;
-
-	skb = ath12k_dp_mon_tx_alloc_skb();
-	if (!skb)
-		return -ENOMEM;
-
-	cts = (struct ieee80211_cts *)skb->data;
-	memset(cts, 0, MAX_DUMMY_FRM_BODY);
-	cts->frame_control =
-		cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_CTS);
-	cts->duration = cpu_to_le16(tx_ppdu_info->rx_status.rx_duration);
-	memcpy(cts->ra, tx_ppdu_info->rx_status.addr1, sizeof(cts->ra));
-
-	skb_put(skb, sizeof(*cts));
-	tx_ppdu_info->tx_mon_mpdu->head = skb;
-	tx_ppdu_info->tx_mon_mpdu->tail = NULL;
-	list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list,
-		      &tx_ppdu_info->dp_tx_mon_mpdu_list);
-
-	return 0;
-}
-
-static int
-ath12k_dp_mon_tx_gen_rts_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info)
-{
-	struct sk_buff *skb;
-	struct ieee80211_rts *rts;
-
-	skb = ath12k_dp_mon_tx_alloc_skb();
-	if (!skb)
-		return -ENOMEM;
-
-	rts = (struct ieee80211_rts *)skb->data;
-	memset(rts, 0, MAX_DUMMY_FRM_BODY);
-	rts->frame_control =
-		cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_RTS);
-	rts->duration = cpu_to_le16(tx_ppdu_info->rx_status.rx_duration);
-	memcpy(rts->ra, tx_ppdu_info->rx_status.addr1, sizeof(rts->ra));
-	memcpy(rts->ta, tx_ppdu_info->rx_status.addr2, sizeof(rts->ta));
-
-	skb_put(skb, sizeof(*rts));
-	tx_ppdu_info->tx_mon_mpdu->head = skb;
-	tx_ppdu_info->tx_mon_mpdu->tail = NULL;
-	list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list,
-		      &tx_ppdu_info->dp_tx_mon_mpdu_list);
-
-	return 0;
-}
-
-static int
-ath12k_dp_mon_tx_gen_3addr_qos_null_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info)
-{
-	struct sk_buff *skb;
-	struct ieee80211_qos_hdr *qhdr;
-
-	skb = ath12k_dp_mon_tx_alloc_skb();
-	if (!skb)
-		return -ENOMEM;
-
-	qhdr = (struct ieee80211_qos_hdr *)skb->data;
-	memset(qhdr, 0, MAX_DUMMY_FRM_BODY);
-	qhdr->frame_control =
-		cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_NULLFUNC);
-	qhdr->duration_id = cpu_to_le16(tx_ppdu_info->rx_status.rx_duration);
-	memcpy(qhdr->addr1, tx_ppdu_info->rx_status.addr1, ETH_ALEN);
-	memcpy(qhdr->addr2, tx_ppdu_info->rx_status.addr2, ETH_ALEN);
-	memcpy(qhdr->addr3, tx_ppdu_info->rx_status.addr3, ETH_ALEN);
-
-	skb_put(skb, sizeof(*qhdr));
-	tx_ppdu_info->tx_mon_mpdu->head = skb;
-	tx_ppdu_info->tx_mon_mpdu->tail = NULL;
-	list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list,
-		      &tx_ppdu_info->dp_tx_mon_mpdu_list);
-
-	return 0;
-}
-
-static int
-ath12k_dp_mon_tx_gen_4addr_qos_null_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info)
-{
-	struct sk_buff *skb;
-	struct dp_mon_qosframe_addr4 *qhdr;
-
-	skb = ath12k_dp_mon_tx_alloc_skb();
-	if (!skb)
-		return -ENOMEM;
-
-	qhdr = (struct dp_mon_qosframe_addr4 *)skb->data;
-	memset(qhdr, 0, MAX_DUMMY_FRM_BODY);
-	qhdr->frame_control =
-		cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_NULLFUNC);
-	qhdr->duration = cpu_to_le16(tx_ppdu_info->rx_status.rx_duration);
-	memcpy(qhdr->addr1, tx_ppdu_info->rx_status.addr1, ETH_ALEN);
-	memcpy(qhdr->addr2, tx_ppdu_info->rx_status.addr2, ETH_ALEN);
-	memcpy(qhdr->addr3, tx_ppdu_info->rx_status.addr3, ETH_ALEN);
-	memcpy(qhdr->addr4, tx_ppdu_info->rx_status.addr4, ETH_ALEN);
-
-	skb_put(skb, sizeof(*qhdr));
-	tx_ppdu_info->tx_mon_mpdu->head = skb;
-	tx_ppdu_info->tx_mon_mpdu->tail = NULL;
-	list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list,
-		      &tx_ppdu_info->dp_tx_mon_mpdu_list);
-
-	return 0;
-}
-
-static int
-ath12k_dp_mon_tx_gen_ack_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info)
-{
-	struct sk_buff *skb;
-	struct dp_mon_frame_min_one *fbmhdr;
-
-	skb = ath12k_dp_mon_tx_alloc_skb();
-	if (!skb)
-		return -ENOMEM;
-
-	fbmhdr = (struct dp_mon_frame_min_one *)skb->data;
-	memset(fbmhdr, 0, MAX_DUMMY_FRM_BODY);
-	fbmhdr->frame_control =
-		cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_CFACK);
-	memcpy(fbmhdr->addr1, tx_ppdu_info->rx_status.addr1, ETH_ALEN);
-
-	/* set duration zero for ack frame */
-	fbmhdr->duration = 0;
-
-	skb_put(skb, sizeof(*fbmhdr));
-	tx_ppdu_info->tx_mon_mpdu->head = skb;
-	tx_ppdu_info->tx_mon_mpdu->tail = NULL;
-	list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list,
-		      &tx_ppdu_info->dp_tx_mon_mpdu_list);
-
-	return 0;
-}
-
-static int
-ath12k_dp_mon_tx_gen_prot_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info)
-{
-	int ret = 0;
-
-	switch (tx_ppdu_info->rx_status.medium_prot_type) {
-	case DP_MON_TX_MEDIUM_RTS_LEGACY:
-	case DP_MON_TX_MEDIUM_RTS_11AC_STATIC_BW:
-	case DP_MON_TX_MEDIUM_RTS_11AC_DYNAMIC_BW:
-		ret = ath12k_dp_mon_tx_gen_rts_frame(tx_ppdu_info);
-		break;
-	case DP_MON_TX_MEDIUM_CTS2SELF:
-		ret = ath12k_dp_mon_tx_gen_cts2self_frame(tx_ppdu_info);
-		break;
-	case DP_MON_TX_MEDIUM_QOS_NULL_NO_ACK_3ADDR:
-		ret = ath12k_dp_mon_tx_gen_3addr_qos_null_frame(tx_ppdu_info);
-		break;
-	case DP_MON_TX_MEDIUM_QOS_NULL_NO_ACK_4ADDR:
-		ret = ath12k_dp_mon_tx_gen_4addr_qos_null_frame(tx_ppdu_info);
-		break;
-	}
-
-	return ret;
-}
-
-static enum dp_mon_tx_tlv_status
-ath12k_dp_mon_tx_parse_status_tlv(struct ath12k_base *ab,
-				  struct ath12k_mon_data *pmon,
-				  u16 tlv_tag, const void *tlv_data, u32 userid)
-{
-	struct dp_mon_tx_ppdu_info *tx_ppdu_info;
-	enum dp_mon_tx_tlv_status status = DP_MON_TX_STATUS_PPDU_NOT_DONE;
-	u32 info[7];
-
-	tx_ppdu_info = ath12k_dp_mon_hal_tx_ppdu_info(pmon, tlv_tag);
-
-	switch (tlv_tag) {
-	case HAL_TX_FES_SETUP: {
-		const struct hal_tx_fes_setup *tx_fes_setup = tlv_data;
-
-		info[0] = __le32_to_cpu(tx_fes_setup->info0);
-		tx_ppdu_info->ppdu_id = __le32_to_cpu(tx_fes_setup->schedule_id);
-		tx_ppdu_info->num_users =
-			u32_get_bits(info[0], HAL_TX_FES_SETUP_INFO0_NUM_OF_USERS);
-		status = DP_MON_TX_FES_SETUP;
-		break;
-	}
-
-	case HAL_TX_FES_STATUS_END: {
-		const struct hal_tx_fes_status_end *tx_fes_status_end = tlv_data;
-		u32 tst_15_0, tst_31_16;
-
-		info[0] = __le32_to_cpu(tx_fes_status_end->info0);
-		tst_15_0 =
-			u32_get_bits(info[0],
-				     HAL_TX_FES_STATUS_END_INFO0_START_TIMESTAMP_15_0);
-		tst_31_16 =
-			u32_get_bits(info[0],
-				     HAL_TX_FES_STATUS_END_INFO0_START_TIMESTAMP_31_16);
-
-		tx_ppdu_info->rx_status.ppdu_ts = (tst_15_0 | (tst_31_16 << 16));
-		status = DP_MON_TX_FES_STATUS_END;
-		break;
-	}
-
-	case HAL_RX_RESPONSE_REQUIRED_INFO: {
-		const struct hal_rx_resp_req_info *rx_resp_req_info = tlv_data;
-		u32 addr_32;
-		u16 addr_16;
-
-		info[0] = __le32_to_cpu(rx_resp_req_info->info0);
-		info[1] = __le32_to_cpu(rx_resp_req_info->info1);
-		info[2] = __le32_to_cpu(rx_resp_req_info->info2);
-		info[3] = __le32_to_cpu(rx_resp_req_info->info3);
-		info[4] = __le32_to_cpu(rx_resp_req_info->info4);
-		info[5] = __le32_to_cpu(rx_resp_req_info->info5);
-
-		tx_ppdu_info->rx_status.ppdu_id =
-			u32_get_bits(info[0], HAL_RX_RESP_REQ_INFO0_PPDU_ID);
-		tx_ppdu_info->rx_status.reception_type =
-			u32_get_bits(info[0], HAL_RX_RESP_REQ_INFO0_RECEPTION_TYPE);
-		tx_ppdu_info->rx_status.rx_duration =
-			u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_DURATION);
-		tx_ppdu_info->rx_status.mcs =
-			u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_RATE_MCS);
-		tx_ppdu_info->rx_status.sgi =
-			u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_SGI);
-		tx_ppdu_info->rx_status.is_stbc =
-			u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_STBC);
-		tx_ppdu_info->rx_status.ldpc =
-			u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_LDPC);
-		tx_ppdu_info->rx_status.is_ampdu =
-			u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_IS_AMPDU);
-		tx_ppdu_info->rx_status.num_users =
-			u32_get_bits(info[2], HAL_RX_RESP_REQ_INFO2_NUM_USER);
-
-		addr_32 = u32_get_bits(info[3], HAL_RX_RESP_REQ_INFO3_ADDR1_31_0);
-		addr_16 = u32_get_bits(info[3], HAL_RX_RESP_REQ_INFO4_ADDR1_47_32);
-		ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr1);
-
-		addr_16 = u32_get_bits(info[4], HAL_RX_RESP_REQ_INFO4_ADDR1_15_0);
-		addr_32 = u32_get_bits(info[5], HAL_RX_RESP_REQ_INFO5_ADDR1_47_16);
-		ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr2);
-
-		if (tx_ppdu_info->rx_status.reception_type == 0)
-			ath12k_dp_mon_tx_gen_cts2self_frame(tx_ppdu_info);
-		status = DP_MON_RX_RESPONSE_REQUIRED_INFO;
-		break;
-	}
-
-	case HAL_PCU_PPDU_SETUP_INIT: {
-		const struct hal_tx_pcu_ppdu_setup_init *ppdu_setup = tlv_data;
-		u32 addr_32;
-		u16 addr_16;
-
-		info[0] = __le32_to_cpu(ppdu_setup->info0);
-		info[1] = __le32_to_cpu(ppdu_setup->info1);
-		info[2] = __le32_to_cpu(ppdu_setup->info2);
-		info[3] = __le32_to_cpu(ppdu_setup->info3);
-		info[4] = __le32_to_cpu(ppdu_setup->info4);
-		info[5] = __le32_to_cpu(ppdu_setup->info5);
-		info[6] = __le32_to_cpu(ppdu_setup->info6);
-
-		/* protection frame address 1 */
-		addr_32 = u32_get_bits(info[1],
-				       HAL_TX_PPDU_SETUP_INFO1_PROT_FRAME_ADDR1_31_0);
-		addr_16 = u32_get_bits(info[2],
-				       HAL_TX_PPDU_SETUP_INFO2_PROT_FRAME_ADDR1_47_32);
-		ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr1);
-
-		/* protection frame address 2 */
-		addr_16 = u32_get_bits(info[2],
-				       HAL_TX_PPDU_SETUP_INFO2_PROT_FRAME_ADDR2_15_0);
-		addr_32 = u32_get_bits(info[3],
-				       HAL_TX_PPDU_SETUP_INFO3_PROT_FRAME_ADDR2_47_16);
-		ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr2);
-
-		/* protection frame address 3 */
-		addr_32 = u32_get_bits(info[4],
-				       HAL_TX_PPDU_SETUP_INFO4_PROT_FRAME_ADDR3_31_0);
-		addr_16 = u32_get_bits(info[5],
-				       HAL_TX_PPDU_SETUP_INFO5_PROT_FRAME_ADDR3_47_32);
-		ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr3);
-
-		/* protection frame address 4 */
-		addr_16 = u32_get_bits(info[5],
-				       HAL_TX_PPDU_SETUP_INFO5_PROT_FRAME_ADDR4_15_0);
-		addr_32 = u32_get_bits(info[6],
-				       HAL_TX_PPDU_SETUP_INFO6_PROT_FRAME_ADDR4_47_16);
-		ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr4);
-
-		status = u32_get_bits(info[0],
-				      HAL_TX_PPDU_SETUP_INFO0_MEDIUM_PROT_TYPE);
-		break;
-	}
-
-	case HAL_TX_QUEUE_EXTENSION: {
-		const struct hal_tx_queue_exten *tx_q_exten = tlv_data;
-
-		info[0] = __le32_to_cpu(tx_q_exten->info0);
-
-		tx_ppdu_info->rx_status.frame_control =
-			u32_get_bits(info[0],
-				     HAL_TX_Q_EXT_INFO0_FRAME_CTRL);
-		tx_ppdu_info->rx_status.fc_valid = true;
-		break;
-	}
-
-	case HAL_TX_FES_STATUS_START: {
-		const struct hal_tx_fes_status_start *tx_fes_start = tlv_data;
-
-		info[0] = __le32_to_cpu(tx_fes_start->info0);
-
-		tx_ppdu_info->rx_status.medium_prot_type =
-			u32_get_bits(info[0],
-				     HAL_TX_FES_STATUS_START_INFO0_MEDIUM_PROT_TYPE);
-		break;
-	}
-
-	case HAL_TX_FES_STATUS_PROT: {
-		const struct hal_tx_fes_status_prot *tx_fes_status = tlv_data;
-		u32 start_timestamp;
-		u32 end_timestamp;
-
-		info[0] = __le32_to_cpu(tx_fes_status->info0);
-		info[1] = __le32_to_cpu(tx_fes_status->info1);
-
-		start_timestamp =
-			u32_get_bits(info[0],
-				     HAL_TX_FES_STAT_PROT_INFO0_STRT_FRM_TS_15_0);
-		start_timestamp |=
-			u32_get_bits(info[0],
-				     HAL_TX_FES_STAT_PROT_INFO0_STRT_FRM_TS_31_16) << 15;
-		end_timestamp =
-			u32_get_bits(info[1],
-				     HAL_TX_FES_STAT_PROT_INFO1_END_FRM_TS_15_0);
-		end_timestamp |=
-			u32_get_bits(info[1],
-				     HAL_TX_FES_STAT_PROT_INFO1_END_FRM_TS_31_16) << 15;
-		tx_ppdu_info->rx_status.rx_duration = end_timestamp - start_timestamp;
-
-		ath12k_dp_mon_tx_gen_prot_frame(tx_ppdu_info);
-		break;
-	}
-
-	case HAL_TX_FES_STATUS_START_PPDU:
-	case HAL_TX_FES_STATUS_START_PROT: {
-		const struct hal_tx_fes_status_start_prot *tx_fes_stat_start = tlv_data;
-		u64 ppdu_ts;
-
-		info[0] = __le32_to_cpu(tx_fes_stat_start->info0);
-
-		tx_ppdu_info->rx_status.ppdu_ts =
-			u32_get_bits(info[0],
-				     HAL_TX_FES_STAT_STRT_INFO0_PROT_TS_LOWER_32);
-		ppdu_ts = (u32_get_bits(info[1],
-					HAL_TX_FES_STAT_STRT_INFO1_PROT_TS_UPPER_32));
-		tx_ppdu_info->rx_status.ppdu_ts |= ppdu_ts << 32;
-		break;
-	}
-
-	case HAL_TX_FES_STATUS_USER_PPDU: {
-		const struct hal_tx_fes_status_user_ppdu *tx_fes_usr_ppdu = tlv_data;
-
-		info[0] = __le32_to_cpu(tx_fes_usr_ppdu->info0);
-
-		tx_ppdu_info->rx_status.rx_duration =
-			u32_get_bits(info[0],
-				     HAL_TX_FES_STAT_USR_PPDU_INFO0_DURATION);
-		break;
-	}
-
-	case HAL_MACTX_HE_SIG_A_SU:
-		ath12k_dp_mon_parse_he_sig_su(tlv_data, &tx_ppdu_info->rx_status);
-		break;
-
-	case HAL_MACTX_HE_SIG_A_MU_DL:
-		ath12k_dp_mon_parse_he_sig_mu(tlv_data, &tx_ppdu_info->rx_status);
-		break;
-
-	case HAL_MACTX_HE_SIG_B1_MU:
-		ath12k_dp_mon_parse_he_sig_b1_mu(tlv_data, &tx_ppdu_info->rx_status);
-		break;
-
-	case HAL_MACTX_HE_SIG_B2_MU:
-		ath12k_dp_mon_parse_he_sig_b2_mu(tlv_data, &tx_ppdu_info->rx_status);
-		break;
-
-	case HAL_MACTX_HE_SIG_B2_OFDMA:
-		ath12k_dp_mon_parse_he_sig_b2_ofdma(tlv_data, &tx_ppdu_info->rx_status);
-		break;
-
-	case HAL_MACTX_VHT_SIG_A:
-		ath12k_dp_mon_parse_vht_sig_a(tlv_data, &tx_ppdu_info->rx_status);
-		break;
-
-	case HAL_MACTX_L_SIG_A:
-		ath12k_dp_mon_parse_l_sig_a(tlv_data, &tx_ppdu_info->rx_status);
-		break;
-
-	case HAL_MACTX_L_SIG_B:
-		ath12k_dp_mon_parse_l_sig_b(tlv_data, &tx_ppdu_info->rx_status);
-		break;
+	enum hal_rx_buf_return_buf_manager mgr =
+		ab->hal.hal_params->rx_buf_rbm;
+	int num_free, num_remain, buf_id;
+	struct ath12k_buffer_addr *desc;
+	struct hal_srng *srng;
+	struct sk_buff *skb;
+	dma_addr_t paddr;
+	u32 cookie;
 
-	case HAL_RX_FRAME_BITMAP_ACK: {
-		const struct hal_rx_frame_bitmap_ack *fbm_ack = tlv_data;
-		u32 addr_32;
-		u16 addr_16;
+	req_entries = min(req_entries, rx_ring->bufs_max);
 
-		info[0] = __le32_to_cpu(fbm_ack->info0);
-		info[1] = __le32_to_cpu(fbm_ack->info1);
+	srng = &ab->hal.srng_list[rx_ring->refill_buf_ring.ring_id];
 
-		addr_32 = u32_get_bits(info[0],
-				       HAL_RX_FBM_ACK_INFO0_ADDR1_31_0);
-		addr_16 = u32_get_bits(info[1],
-				       HAL_RX_FBM_ACK_INFO1_ADDR1_47_32);
-		ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr1);
+	spin_lock_bh(&srng->lock);
 
-		ath12k_dp_mon_tx_gen_ack_frame(tx_ppdu_info);
-		break;
-	}
+	ath12k_hal_srng_access_begin(ab, srng);
 
-	case HAL_MACTX_PHY_DESC: {
-		const struct hal_tx_phy_desc *tx_phy_desc = tlv_data;
-
-		info[0] = __le32_to_cpu(tx_phy_desc->info0);
-		info[1] = __le32_to_cpu(tx_phy_desc->info1);
-		info[2] = __le32_to_cpu(tx_phy_desc->info2);
-		info[3] = __le32_to_cpu(tx_phy_desc->info3);
-
-		tx_ppdu_info->rx_status.beamformed =
-			u32_get_bits(info[0],
-				     HAL_TX_PHY_DESC_INFO0_BF_TYPE);
-		tx_ppdu_info->rx_status.preamble_type =
-			u32_get_bits(info[0],
-				     HAL_TX_PHY_DESC_INFO0_PREAMBLE_11B);
-		tx_ppdu_info->rx_status.mcs =
-			u32_get_bits(info[1],
-				     HAL_TX_PHY_DESC_INFO1_MCS);
-		tx_ppdu_info->rx_status.ltf_size =
-			u32_get_bits(info[3],
-				     HAL_TX_PHY_DESC_INFO3_LTF_SIZE);
-		tx_ppdu_info->rx_status.nss =
-			u32_get_bits(info[2],
-				     HAL_TX_PHY_DESC_INFO2_NSS);
-		tx_ppdu_info->rx_status.chan_num =
-			u32_get_bits(info[3],
-				     HAL_TX_PHY_DESC_INFO3_ACTIVE_CHANNEL);
-		tx_ppdu_info->rx_status.bw =
-			u32_get_bits(info[0],
-				     HAL_TX_PHY_DESC_INFO0_BANDWIDTH);
-		break;
-	}
+	num_free = ath12k_hal_srng_src_num_free(ab, srng, true);
+	if (!req_entries && (num_free > (rx_ring->bufs_max * 3) / 4))
+		req_entries = num_free;
 
-	case HAL_TX_MPDU_START: {
-		struct dp_mon_mpdu *mon_mpdu = tx_ppdu_info->tx_mon_mpdu;
+	req_entries = min(num_free, req_entries);
+	num_remain = req_entries;
 
-		mon_mpdu = kzalloc(sizeof(*mon_mpdu), GFP_ATOMIC);
-		if (!mon_mpdu)
-			return DP_MON_TX_STATUS_PPDU_NOT_DONE;
-		status = DP_MON_TX_MPDU_START;
-		break;
-	}
+	while (num_remain > 0) {
+		skb = dev_alloc_skb(RX_MON_STATUS_BUF_SIZE);
+		if (!skb)
+			break;
 
-	case HAL_TX_MPDU_END:
-		list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list,
-			      &tx_ppdu_info->dp_tx_mon_mpdu_list);
-		break;
-	}
+		if (!IS_ALIGNED((unsigned long)skb->data,
+				RX_MON_STATUS_BUF_ALIGN)) {
+			skb_pull(skb,
+				 PTR_ALIGN(skb->data, RX_MON_STATUS_BUF_ALIGN) -
+				 skb->data);
+		}
 
-	return status;
-}
+		paddr = dma_map_single(ab->dev, skb->data,
+				       skb->len + skb_tailroom(skb),
+				       DMA_FROM_DEVICE);
+		if (dma_mapping_error(ab->dev, paddr))
+			goto fail_free_skb;
 
-enum dp_mon_tx_tlv_status
-ath12k_dp_mon_tx_status_get_num_user(u16 tlv_tag,
-				     struct hal_tlv_hdr *tx_tlv,
-				     u8 *num_users)
-{
-	u32 tlv_status = DP_MON_TX_STATUS_PPDU_NOT_DONE;
-	u32 info0;
+		spin_lock_bh(&rx_ring->idr_lock);
+		buf_id = idr_alloc(&rx_ring->bufs_idr, skb, 0,
+				   rx_ring->bufs_max * 3, GFP_ATOMIC);
+		spin_unlock_bh(&rx_ring->idr_lock);
+		if (buf_id < 0)
+			goto fail_dma_unmap;
+		cookie = u32_encode_bits(buf_id, DP_RXDMA_BUF_COOKIE_BUF_ID);
 
-	switch (tlv_tag) {
-	case HAL_TX_FES_SETUP: {
-		struct hal_tx_fes_setup *tx_fes_setup =
-				(struct hal_tx_fes_setup *)tx_tlv;
+		desc = ath12k_hal_srng_src_get_next_entry(ab, srng);
+		if (!desc)
+			goto fail_buf_unassign;
 
-		info0 = __le32_to_cpu(tx_fes_setup->info0);
+		ATH12K_SKB_RXCB(skb)->paddr = paddr;
 
-		*num_users = u32_get_bits(info0, HAL_TX_FES_SETUP_INFO0_NUM_OF_USERS);
-		tlv_status = DP_MON_TX_FES_SETUP;
-		break;
-	}
+		num_remain--;
 
-	case HAL_RX_RESPONSE_REQUIRED_INFO: {
-		/* TODO: need to update *num_users */
-		tlv_status = DP_MON_RX_RESPONSE_REQUIRED_INFO;
-		break;
+		ath12k_hal_rx_buf_addr_info_set(&ab->hal, desc, paddr, cookie, mgr);
 	}
-	}
-
-	return tlv_status;
-}
 
-static void
-ath12k_dp_mon_tx_process_ppdu_info(struct ath12k *ar,
-				   struct napi_struct *napi,
-				   struct dp_mon_tx_ppdu_info *tx_ppdu_info)
-{
-	struct dp_mon_mpdu *tmp, *mon_mpdu;
+	ath12k_hal_srng_access_end(ab, srng);
 
-	list_for_each_entry_safe(mon_mpdu, tmp,
-				 &tx_ppdu_info->dp_tx_mon_mpdu_list, list) {
-		list_del(&mon_mpdu->list);
+	spin_unlock_bh(&srng->lock);
 
-		if (mon_mpdu->head)
-			ath12k_dp_mon_rx_deliver(ar, mon_mpdu,
-						 &tx_ppdu_info->rx_status, napi);
+	return req_entries - num_remain;
 
-		kfree(mon_mpdu);
-	}
-}
+fail_buf_unassign:
+	spin_lock_bh(&rx_ring->idr_lock);
+	idr_remove(&rx_ring->bufs_idr, buf_id);
+	spin_unlock_bh(&rx_ring->idr_lock);
+fail_dma_unmap:
+	dma_unmap_single(ab->dev, paddr, skb->len + skb_tailroom(skb),
+			 DMA_FROM_DEVICE);
+fail_free_skb:
+	dev_kfree_skb_any(skb);
 
-enum hal_rx_mon_status
-ath12k_dp_mon_tx_parse_mon_status(struct ath12k *ar,
-				  struct ath12k_mon_data *pmon,
-				  struct sk_buff *skb,
-				  struct napi_struct *napi,
-				  u32 ppdu_id)
-{
-	struct ath12k_base *ab = ar->ab;
-	struct dp_mon_tx_ppdu_info *tx_prot_ppdu_info, *tx_data_ppdu_info;
-	struct hal_tlv_hdr *tlv;
-	u8 *ptr = skb->data;
-	u16 tlv_tag;
-	u16 tlv_len;
-	u32 tlv_userid = 0;
-	u8 num_user;
-	u32 tlv_status = DP_MON_TX_STATUS_PPDU_NOT_DONE;
-
-	tx_prot_ppdu_info = ath12k_dp_mon_tx_get_ppdu_info(pmon, ppdu_id,
-							   DP_MON_TX_PROT_PPDU_INFO);
-	if (!tx_prot_ppdu_info)
-		return -ENOMEM;
-
-	tlv = (struct hal_tlv_hdr *)ptr;
-	tlv_tag = le32_get_bits(tlv->tl, HAL_TLV_HDR_TAG);
-
-	tlv_status = ath12k_dp_mon_tx_status_get_num_user(tlv_tag, tlv, &num_user);
-	if (tlv_status == DP_MON_TX_STATUS_PPDU_NOT_DONE || !num_user)
-		return -EINVAL;
-
-	tx_data_ppdu_info = ath12k_dp_mon_tx_get_ppdu_info(pmon, ppdu_id,
-							   DP_MON_TX_DATA_PPDU_INFO);
-	if (!tx_data_ppdu_info)
-		return -ENOMEM;
-
-	do {
-		tlv = (struct hal_tlv_hdr *)ptr;
-		tlv_tag = le32_get_bits(tlv->tl, HAL_TLV_HDR_TAG);
-		tlv_len = le32_get_bits(tlv->tl, HAL_TLV_HDR_LEN);
-		tlv_userid = le32_get_bits(tlv->tl, HAL_TLV_USR_ID);
-
-		tlv_status = ath12k_dp_mon_tx_parse_status_tlv(ab, pmon,
-							       tlv_tag, ptr,
-							       tlv_userid);
-		ptr += tlv_len;
-		ptr = PTR_ALIGN(ptr, HAL_TLV_ALIGN);
-		if ((ptr - skb->data) >= DP_TX_MONITOR_BUF_SIZE)
-			break;
-	} while (tlv_status != DP_MON_TX_FES_STATUS_END);
+	ath12k_hal_srng_access_end(ab, srng);
 
-	ath12k_dp_mon_tx_process_ppdu_info(ar, napi, tx_data_ppdu_info);
-	ath12k_dp_mon_tx_process_ppdu_info(ar, napi, tx_prot_ppdu_info);
+	spin_unlock_bh(&srng->lock);
 
-	return tlv_status;
+	return req_entries - num_remain;
 }
 
 static void
@@ -3465,6 +830,9 @@ ath12k_dp_mon_rx_update_peer_rate_table_stats(struct ath12k_rx_peer_stats *rx_st
 	u32 gi_idx = ppdu_info->gi;
 	u32 len;
 
+	if (!rx_stats)
+		return;
+
 	if (mcs_idx > HAL_RX_MAX_MCS_HT || nss_idx >= HAL_RX_MAX_NSS ||
 	    bw_idx >= HAL_RX_BW_MAX || gi_idx >= HAL_RX_GI_MAX) {
 		return;
@@ -3485,15 +853,14 @@ ath12k_dp_mon_rx_update_peer_rate_table_stats(struct ath12k_rx_peer_stats *rx_st
 	stats->rx_rate[bw_idx][gi_idx][nss_idx][mcs_idx] += len;
 }
 
-static void ath12k_dp_mon_rx_update_peer_su_stats(struct ath12k *ar,
-						  struct ath12k_link_sta *arsta,
-						  struct hal_rx_mon_ppdu_info *ppdu_info)
+void ath12k_dp_mon_rx_update_peer_su_stats(struct ath12k_dp_link_peer *peer,
+					   struct hal_rx_mon_ppdu_info *ppdu_info)
 {
-	struct ath12k_rx_peer_stats *rx_stats = arsta->rx_stats;
+	struct ath12k_rx_peer_stats *rx_stats = peer->peer_stats.rx_stats;
 	u32 num_msdu;
 
-	arsta->rssi_comb = ppdu_info->rssi_comb;
-	ewma_avg_rssi_add(&arsta->avg_rssi, ppdu_info->rssi_comb);
+	peer->rssi_comb = ppdu_info->rssi_comb;
+	ewma_avg_rssi_add(&peer->avg_rssi, ppdu_info->rssi_comb);
 	if (!rx_stats)
 		return;
 
@@ -3541,7 +908,7 @@ static void ath12k_dp_mon_rx_update_peer_su_stats(struct ath12k *ar,
 	rx_stats->dcm_count += ppdu_info->dcm;
 
 	rx_stats->rx_duration += ppdu_info->rx_duration;
-	arsta->rx_duration = rx_stats->rx_duration;
+	peer->rx_duration = rx_stats->rx_duration;
 
 	if (ppdu_info->nss > 0 && ppdu_info->nss <= HAL_RX_MAX_NSS) {
 		rx_stats->pkt_stats.nss_count[ppdu_info->nss - 1] += num_msdu;
@@ -3594,6 +961,7 @@ static void ath12k_dp_mon_rx_update_peer_su_stats(struct ath12k *ar,
 	ath12k_dp_mon_rx_update_peer_rate_table_stats(rx_stats, ppdu_info,
 						      NULL, num_msdu);
 }
+EXPORT_SYMBOL(ath12k_dp_mon_rx_update_peer_su_stats);
 
 void ath12k_dp_mon_rx_process_ulofdma(struct hal_rx_mon_ppdu_info *ppdu_info)
 {
@@ -3642,39 +1010,33 @@ void ath12k_dp_mon_rx_process_ulofdma(struct hal_rx_mon_ppdu_info *ppdu_info)
 	}
 	ppdu_info->ldpc = 1;
 }
+EXPORT_SYMBOL(ath12k_dp_mon_rx_process_ulofdma);
 
 static void
-ath12k_dp_mon_rx_update_user_stats(struct ath12k *ar,
+ath12k_dp_mon_rx_update_user_stats(struct ath12k_base *ab,
 				   struct hal_rx_mon_ppdu_info *ppdu_info,
 				   u32 uid)
 {
-	struct ath12k_link_sta *arsta;
 	struct ath12k_rx_peer_stats *rx_stats = NULL;
 	struct hal_rx_user_status *user_stats = &ppdu_info->userstats[uid];
-	struct ath12k_peer *peer;
+	struct ath12k_dp_link_peer *peer;
 	u32 num_msdu;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 
 	if (user_stats->ast_index == 0 || user_stats->ast_index == 0xFFFF)
 		return;
 
-	peer = ath12k_peer_find_by_ast(ar->ab, user_stats->ast_index);
+	peer = ath12k_dp_link_peer_find_by_ast(dp, user_stats->ast_index);
 
 	if (!peer) {
-		ath12k_warn(ar->ab, "peer ast idx %d can't be found\n",
+		ath12k_warn(ab, "peer ast idx %d can't be found\n",
 			    user_stats->ast_index);
 		return;
 	}
 
-	arsta = ath12k_peer_get_link_sta(ar->ab, peer);
-	if (!arsta) {
-		ath12k_warn(ar->ab, "link sta not found on peer %pM id %d\n",
-			    peer->addr, peer->peer_id);
-		return;
-	}
-
-	arsta->rssi_comb = ppdu_info->rssi_comb;
-	ewma_avg_rssi_add(&arsta->avg_rssi, ppdu_info->rssi_comb);
-	rx_stats = arsta->rx_stats;
+	peer->rssi_comb = ppdu_info->rssi_comb;
+	ewma_avg_rssi_add(&peer->avg_rssi, ppdu_info->rssi_comb);
+	rx_stats = peer->peer_stats.rx_stats;
 	if (!rx_stats)
 		return;
 
@@ -3718,7 +1080,7 @@ ath12k_dp_mon_rx_update_user_stats(struct ath12k *ar,
 		rx_stats->ru_alloc_cnt[user_stats->ul_ofdma_ru_size] += num_msdu;
 
 	rx_stats->rx_duration += ppdu_info->rx_duration;
-	arsta->rx_duration = rx_stats->rx_duration;
+	peer->rx_duration = rx_stats->rx_duration;
 
 	if (user_stats->nss > 0 && user_stats->nss <= HAL_RX_MAX_NSS) {
 		rx_stats->pkt_stats.nss_count[user_stats->nss - 1] += num_msdu;
@@ -3749,8 +1111,8 @@ ath12k_dp_mon_rx_update_user_stats(struct ath12k *ar,
 						      user_stats, num_msdu);
 }
 
-static void
-ath12k_dp_mon_rx_update_peer_mu_stats(struct ath12k *ar,
+void
+ath12k_dp_mon_rx_update_peer_mu_stats(struct ath12k_base *ab,
 				      struct hal_rx_mon_ppdu_info *ppdu_info)
 {
 	u32 num_users, i;
@@ -3760,679 +1122,6 @@ ath12k_dp_mon_rx_update_peer_mu_stats(struct ath12k *ar,
 		num_users = HAL_MAX_UL_MU_USERS;
 
 	for (i = 0; i < num_users; i++)
-		ath12k_dp_mon_rx_update_user_stats(ar, ppdu_info, i);
-}
-
-static void
-ath12k_dp_mon_rx_memset_ppdu_info(struct hal_rx_mon_ppdu_info *ppdu_info)
-{
-	memset(ppdu_info, 0, sizeof(*ppdu_info));
-	ppdu_info->peer_id = HAL_INVALID_PEERID;
-}
-
-int ath12k_dp_mon_srng_process(struct ath12k *ar, int *budget,
-			       struct napi_struct *napi)
-{
-	struct ath12k_base *ab = ar->ab;
-	struct ath12k_pdev_dp *pdev_dp = &ar->dp;
-	struct ath12k_mon_data *pmon = (struct ath12k_mon_data *)&pdev_dp->mon_data;
-	struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info;
-	struct ath12k_dp *dp = &ab->dp;
-	struct hal_mon_dest_desc *mon_dst_desc;
-	struct sk_buff *skb;
-	struct ath12k_skb_rxcb *rxcb;
-	struct dp_srng *mon_dst_ring;
-	struct hal_srng *srng;
-	struct dp_rxdma_mon_ring *buf_ring;
-	struct ath12k_link_sta *arsta;
-	struct ath12k_peer *peer;
-	struct sk_buff_head skb_list;
-	u64 cookie;
-	int num_buffs_reaped = 0, srng_id, buf_id;
-	u32 hal_status, end_offset, info0, end_reason;
-	u8 pdev_idx = ath12k_hw_mac_id_to_pdev_id(ab->hw_params, ar->pdev_idx);
-
-	__skb_queue_head_init(&skb_list);
-	srng_id = ath12k_hw_mac_id_to_srng_id(ab->hw_params, pdev_idx);
-	mon_dst_ring = &pdev_dp->rxdma_mon_dst_ring[srng_id];
-	buf_ring = &dp->rxdma_mon_buf_ring;
-
-	srng = &ab->hal.srng_list[mon_dst_ring->ring_id];
-	spin_lock_bh(&srng->lock);
-	ath12k_hal_srng_access_begin(ab, srng);
-
-	while (likely(*budget)) {
-		mon_dst_desc = ath12k_hal_srng_dst_peek(ab, srng);
-		if (unlikely(!mon_dst_desc))
-			break;
-
-		/* In case of empty descriptor, the cookie in the ring descriptor
-		 * is invalid. Therefore, this entry is skipped, and ring processing
-		 * continues.
-		 */
-		info0 = le32_to_cpu(mon_dst_desc->info0);
-		if (u32_get_bits(info0, HAL_MON_DEST_INFO0_EMPTY_DESC))
-			goto move_next;
-
-		cookie = le32_to_cpu(mon_dst_desc->cookie);
-		buf_id = u32_get_bits(cookie, DP_RXDMA_BUF_COOKIE_BUF_ID);
-
-		spin_lock_bh(&buf_ring->idr_lock);
-		skb = idr_remove(&buf_ring->bufs_idr, buf_id);
-		spin_unlock_bh(&buf_ring->idr_lock);
-
-		if (unlikely(!skb)) {
-			ath12k_warn(ab, "monitor destination with invalid buf_id %d\n",
-				    buf_id);
-			goto move_next;
-		}
-
-		rxcb = ATH12K_SKB_RXCB(skb);
-		dma_unmap_single(ab->dev, rxcb->paddr,
-				 skb->len + skb_tailroom(skb),
-				 DMA_FROM_DEVICE);
-
-		end_reason = u32_get_bits(info0, HAL_MON_DEST_INFO0_END_REASON);
-
-		/* HAL_MON_FLUSH_DETECTED implies that an rx flush received at the end of
-		 * rx PPDU and HAL_MON_PPDU_TRUNCATED implies that the PPDU got
-		 * truncated due to a system level error. In both the cases, buffer data
-		 * can be discarded
-		 */
-		if ((end_reason == HAL_MON_FLUSH_DETECTED) ||
-		    (end_reason == HAL_MON_PPDU_TRUNCATED)) {
-			ath12k_dbg(ab, ATH12K_DBG_DATA,
-				   "Monitor dest descriptor end reason %d", end_reason);
-			dev_kfree_skb_any(skb);
-			goto move_next;
-		}
-
-		/* Calculate the budget when the ring descriptor with the
-		 * HAL_MON_END_OF_PPDU to ensure that one PPDU worth of data is always
-		 * reaped. This helps to efficiently utilize the NAPI budget.
-		 */
-		if (end_reason == HAL_MON_END_OF_PPDU) {
-			*budget -= 1;
-			rxcb->is_end_of_ppdu = true;
-		}
-
-		end_offset = u32_get_bits(info0, HAL_MON_DEST_INFO0_END_OFFSET);
-		if (likely(end_offset <= DP_RX_BUFFER_SIZE)) {
-			skb_put(skb, end_offset);
-		} else {
-			ath12k_warn(ab,
-				    "invalid offset on mon stats destination %u\n",
-				    end_offset);
-			skb_put(skb, DP_RX_BUFFER_SIZE);
-		}
-
-		__skb_queue_tail(&skb_list, skb);
-
-move_next:
-		ath12k_dp_mon_buf_replenish(ab, buf_ring, 1);
-		ath12k_hal_srng_dst_get_next_entry(ab, srng);
-		num_buffs_reaped++;
-	}
-
-	ath12k_hal_srng_access_end(ab, srng);
-	spin_unlock_bh(&srng->lock);
-
-	if (!num_buffs_reaped)
-		return 0;
-
-	/* In some cases, one PPDU worth of data can be spread across multiple NAPI
-	 * schedules, To avoid losing existing parsed ppdu_info information, skip
-	 * the memset of the ppdu_info structure and continue processing it.
-	 */
-	if (!ppdu_info->ppdu_continuation)
-		ath12k_dp_mon_rx_memset_ppdu_info(ppdu_info);
-
-	while ((skb = __skb_dequeue(&skb_list))) {
-		hal_status = ath12k_dp_mon_rx_parse_mon_status(ar, pmon, skb, napi);
-		if (hal_status != HAL_RX_MON_STATUS_PPDU_DONE) {
-			ppdu_info->ppdu_continuation = true;
-			dev_kfree_skb_any(skb);
-			continue;
-		}
-
-		if (ppdu_info->peer_id == HAL_INVALID_PEERID)
-			goto free_skb;
-
-		rcu_read_lock();
-		spin_lock_bh(&ab->base_lock);
-		peer = ath12k_peer_find_by_id(ab, ppdu_info->peer_id);
-		if (!peer || !peer->sta) {
-			ath12k_dbg(ab, ATH12K_DBG_DATA,
-				   "failed to find the peer with monitor peer_id %d\n",
-				   ppdu_info->peer_id);
-			goto next_skb;
-		}
-
-		if (ppdu_info->reception_type == HAL_RX_RECEPTION_TYPE_SU) {
-			arsta = ath12k_peer_get_link_sta(ar->ab, peer);
-			if (!arsta) {
-				ath12k_warn(ar->ab, "link sta not found on peer %pM id %d\n",
-					    peer->addr, peer->peer_id);
-				spin_unlock_bh(&ab->base_lock);
-				rcu_read_unlock();
-				dev_kfree_skb_any(skb);
-				continue;
-			}
-			ath12k_dp_mon_rx_update_peer_su_stats(ar, arsta,
-							      ppdu_info);
-		} else if ((ppdu_info->fc_valid) &&
-			   (ppdu_info->ast_index != HAL_AST_IDX_INVALID)) {
-			ath12k_dp_mon_rx_process_ulofdma(ppdu_info);
-			ath12k_dp_mon_rx_update_peer_mu_stats(ar, ppdu_info);
-		}
-
-next_skb:
-		spin_unlock_bh(&ab->base_lock);
-		rcu_read_unlock();
-free_skb:
-		dev_kfree_skb_any(skb);
-		ath12k_dp_mon_rx_memset_ppdu_info(ppdu_info);
-	}
-
-	return num_buffs_reaped;
-}
-
-static int ath12k_dp_rx_reap_mon_status_ring(struct ath12k_base *ab, int mac_id,
-					     int *budget, struct sk_buff_head *skb_list)
-{
-	const struct ath12k_hw_hal_params *hal_params;
-	int buf_id, srng_id, num_buffs_reaped = 0;
-	enum dp_mon_status_buf_state reap_status;
-	struct dp_rxdma_mon_ring *rx_ring;
-	struct ath12k_mon_data *pmon;
-	struct ath12k_skb_rxcb *rxcb;
-	struct hal_tlv_64_hdr *tlv;
-	void *rx_mon_status_desc;
-	struct hal_srng *srng;
-	struct ath12k_dp *dp;
-	struct sk_buff *skb;
-	struct ath12k *ar;
-	dma_addr_t paddr;
-	u32 cookie;
-	u8 rbm;
-
-	ar = ab->pdevs[ath12k_hw_mac_id_to_pdev_id(ab->hw_params, mac_id)].ar;
-	dp = &ab->dp;
-	pmon = &ar->dp.mon_data;
-	srng_id = ath12k_hw_mac_id_to_srng_id(ab->hw_params, mac_id);
-	rx_ring = &dp->rx_mon_status_refill_ring[srng_id];
-
-	srng = &ab->hal.srng_list[rx_ring->refill_buf_ring.ring_id];
-
-	spin_lock_bh(&srng->lock);
-
-	ath12k_hal_srng_access_begin(ab, srng);
-
-	while (*budget) {
-		*budget -= 1;
-		rx_mon_status_desc = ath12k_hal_srng_src_peek(ab, srng);
-		if (!rx_mon_status_desc) {
-			pmon->buf_state = DP_MON_STATUS_REPLINISH;
-			break;
-		}
-		ath12k_hal_rx_buf_addr_info_get(rx_mon_status_desc, &paddr,
-						&cookie, &rbm);
-		if (paddr) {
-			buf_id = u32_get_bits(cookie, DP_RXDMA_BUF_COOKIE_BUF_ID);
-
-			spin_lock_bh(&rx_ring->idr_lock);
-			skb = idr_find(&rx_ring->bufs_idr, buf_id);
-			spin_unlock_bh(&rx_ring->idr_lock);
-
-			if (!skb) {
-				ath12k_warn(ab, "rx monitor status with invalid buf_id %d\n",
-					    buf_id);
-				pmon->buf_state = DP_MON_STATUS_REPLINISH;
-				goto move_next;
-			}
-
-			rxcb = ATH12K_SKB_RXCB(skb);
-
-			dma_sync_single_for_cpu(ab->dev, rxcb->paddr,
-						skb->len + skb_tailroom(skb),
-						DMA_FROM_DEVICE);
-
-			tlv = (struct hal_tlv_64_hdr *)skb->data;
-			if (le64_get_bits(tlv->tl, HAL_TLV_HDR_TAG) !=
-					HAL_RX_STATUS_BUFFER_DONE) {
-				pmon->buf_state = DP_MON_STATUS_NO_DMA;
-				ath12k_warn(ab,
-					    "mon status DONE not set %llx, buf_id %d\n",
-					    le64_get_bits(tlv->tl, HAL_TLV_HDR_TAG),
-					    buf_id);
-				/* RxDMA status done bit might not be set even
-				 * though tp is moved by HW.
-				 */
-
-				/* If done status is missing:
-				 * 1. As per MAC team's suggestion,
-				 *    when HP + 1 entry is peeked and if DMA
-				 *    is not done and if HP + 2 entry's DMA done
-				 *    is set. skip HP + 1 entry and
-				 *    start processing in next interrupt.
-				 * 2. If HP + 2 entry's DMA done is not set,
-				 *    poll onto HP + 1 entry DMA done to be set.
-				 *    Check status for same buffer for next time
-				 *    dp_rx_mon_status_srng_process
-				 */
-				reap_status = ath12k_dp_rx_mon_buf_done(ab, srng,
-									rx_ring);
-				if (reap_status == DP_MON_STATUS_NO_DMA)
-					continue;
-
-				spin_lock_bh(&rx_ring->idr_lock);
-				idr_remove(&rx_ring->bufs_idr, buf_id);
-				spin_unlock_bh(&rx_ring->idr_lock);
-
-				dma_unmap_single(ab->dev, rxcb->paddr,
-						 skb->len + skb_tailroom(skb),
-						 DMA_FROM_DEVICE);
-
-				dev_kfree_skb_any(skb);
-				pmon->buf_state = DP_MON_STATUS_REPLINISH;
-				goto move_next;
-			}
-
-			spin_lock_bh(&rx_ring->idr_lock);
-			idr_remove(&rx_ring->bufs_idr, buf_id);
-			spin_unlock_bh(&rx_ring->idr_lock);
-
-			dma_unmap_single(ab->dev, rxcb->paddr,
-					 skb->len + skb_tailroom(skb),
-					 DMA_FROM_DEVICE);
-
-			if (ath12k_dp_pkt_set_pktlen(skb, RX_MON_STATUS_BUF_SIZE)) {
-				dev_kfree_skb_any(skb);
-				goto move_next;
-			}
-			__skb_queue_tail(skb_list, skb);
-		} else {
-			pmon->buf_state = DP_MON_STATUS_REPLINISH;
-		}
-move_next:
-		skb = ath12k_dp_rx_alloc_mon_status_buf(ab, rx_ring,
-							&buf_id);
-
-		if (!skb) {
-			ath12k_warn(ab, "failed to alloc buffer for status ring\n");
-			hal_params = ab->hw_params->hal_params;
-			ath12k_hal_rx_buf_addr_info_set(rx_mon_status_desc, 0, 0,
-							hal_params->rx_buf_rbm);
-			num_buffs_reaped++;
-			break;
-		}
-		rxcb = ATH12K_SKB_RXCB(skb);
-
-		cookie = u32_encode_bits(mac_id, DP_RXDMA_BUF_COOKIE_PDEV_ID) |
-			 u32_encode_bits(buf_id, DP_RXDMA_BUF_COOKIE_BUF_ID);
-
-		ath12k_hal_rx_buf_addr_info_set(rx_mon_status_desc, rxcb->paddr,
-						cookie,
-						ab->hw_params->hal_params->rx_buf_rbm);
-		ath12k_hal_srng_src_get_next_entry(ab, srng);
-		num_buffs_reaped++;
-	}
-	ath12k_hal_srng_access_end(ab, srng);
-	spin_unlock_bh(&srng->lock);
-
-	return num_buffs_reaped;
-}
-
-static u32
-ath12k_dp_rx_mon_mpdu_pop(struct ath12k *ar, int mac_id,
-			  void *ring_entry, struct sk_buff **head_msdu,
-			  struct sk_buff **tail_msdu,
-			  struct list_head *used_list,
-			  u32 *npackets, u32 *ppdu_id)
-{
-	struct ath12k_mon_data *pmon = (struct ath12k_mon_data *)&ar->dp.mon_data;
-	struct ath12k_buffer_addr *p_buf_addr_info, *p_last_buf_addr_info;
-	u32 msdu_ppdu_id = 0, msdu_cnt = 0, total_len = 0, frag_len = 0;
-	u32 rx_buf_size, rx_pkt_offset, sw_cookie;
-	bool is_frag, is_first_msdu, drop_mpdu = false;
-	struct hal_reo_entrance_ring *ent_desc =
-		(struct hal_reo_entrance_ring *)ring_entry;
-	u32 rx_bufs_used = 0, i = 0, desc_bank = 0;
-	struct hal_rx_desc *rx_desc, *tail_rx_desc;
-	struct hal_rx_msdu_link *msdu_link_desc;
-	struct sk_buff *msdu = NULL, *last = NULL;
-	struct ath12k_rx_desc_info *desc_info;
-	struct ath12k_buffer_addr buf_info;
-	struct hal_rx_msdu_list msdu_list;
-	struct ath12k_skb_rxcb *rxcb;
-	u16 num_msdus = 0;
-	dma_addr_t paddr;
-	u8 rbm;
-
-	ath12k_hal_rx_reo_ent_buf_paddr_get(ring_entry, &paddr,
-					    &sw_cookie,
-					    &p_last_buf_addr_info, &rbm,
-					    &msdu_cnt);
-
-	spin_lock_bh(&pmon->mon_lock);
-
-	if (le32_get_bits(ent_desc->info1,
-			  HAL_REO_ENTR_RING_INFO1_RXDMA_PUSH_REASON) ==
-			  HAL_REO_DEST_RING_PUSH_REASON_ERR_DETECTED) {
-		u8 rxdma_err = le32_get_bits(ent_desc->info1,
-					     HAL_REO_ENTR_RING_INFO1_RXDMA_ERROR_CODE);
-		if (rxdma_err == HAL_REO_ENTR_RING_RXDMA_ECODE_FLUSH_REQUEST_ERR ||
-		    rxdma_err == HAL_REO_ENTR_RING_RXDMA_ECODE_MPDU_LEN_ERR ||
-		    rxdma_err == HAL_REO_ENTR_RING_RXDMA_ECODE_OVERFLOW_ERR) {
-			drop_mpdu = true;
-			pmon->rx_mon_stats.dest_mpdu_drop++;
-		}
-	}
-
-	is_frag = false;
-	is_first_msdu = true;
-	rx_pkt_offset = sizeof(struct hal_rx_desc);
-
-	do {
-		if (pmon->mon_last_linkdesc_paddr == paddr) {
-			pmon->rx_mon_stats.dup_mon_linkdesc_cnt++;
-			spin_unlock_bh(&pmon->mon_lock);
-			return rx_bufs_used;
-		}
-
-		desc_bank = u32_get_bits(sw_cookie, DP_LINK_DESC_BANK_MASK);
-		msdu_link_desc =
-			ar->ab->dp.link_desc_banks[desc_bank].vaddr +
-			(paddr - ar->ab->dp.link_desc_banks[desc_bank].paddr);
-
-		ath12k_hal_rx_msdu_list_get(ar, msdu_link_desc, &msdu_list,
-					    &num_msdus);
-		desc_info = ath12k_dp_get_rx_desc(ar->ab,
-						  msdu_list.sw_cookie[num_msdus - 1]);
-		tail_rx_desc = (struct hal_rx_desc *)(desc_info->skb)->data;
-
-		for (i = 0; i < num_msdus; i++) {
-			u32 l2_hdr_offset;
-
-			if (pmon->mon_last_buf_cookie == msdu_list.sw_cookie[i]) {
-				ath12k_dbg(ar->ab, ATH12K_DBG_DATA,
-					   "i %d last_cookie %d is same\n",
-					   i, pmon->mon_last_buf_cookie);
-				drop_mpdu = true;
-				pmon->rx_mon_stats.dup_mon_buf_cnt++;
-				continue;
-			}
-
-			desc_info =
-				ath12k_dp_get_rx_desc(ar->ab, msdu_list.sw_cookie[i]);
-			msdu = desc_info->skb;
-
-			if (!msdu) {
-				ath12k_dbg(ar->ab, ATH12K_DBG_DATA,
-					   "msdu_pop: invalid msdu (%d/%d)\n",
-					   i + 1, num_msdus);
-				goto next_msdu;
-			}
-			rxcb = ATH12K_SKB_RXCB(msdu);
-			if (rxcb->paddr != msdu_list.paddr[i]) {
-				ath12k_dbg(ar->ab, ATH12K_DBG_DATA,
-					   "i %d paddr %lx != %lx\n",
-					   i, (unsigned long)rxcb->paddr,
-					   (unsigned long)msdu_list.paddr[i]);
-				drop_mpdu = true;
-				continue;
-			}
-			if (!rxcb->unmapped) {
-				dma_unmap_single(ar->ab->dev, rxcb->paddr,
-						 msdu->len +
-						 skb_tailroom(msdu),
-						 DMA_FROM_DEVICE);
-				rxcb->unmapped = 1;
-			}
-			if (drop_mpdu) {
-				ath12k_dbg(ar->ab, ATH12K_DBG_DATA,
-					   "i %d drop msdu %p *ppdu_id %x\n",
-					   i, msdu, *ppdu_id);
-				dev_kfree_skb_any(msdu);
-				msdu = NULL;
-				goto next_msdu;
-			}
-
-			rx_desc = (struct hal_rx_desc *)msdu->data;
-			l2_hdr_offset = ath12k_dp_rx_h_l3pad(ar->ab, tail_rx_desc);
-			if (is_first_msdu) {
-				if (!ath12k_dp_rxdesc_mpdu_valid(ar->ab, rx_desc)) {
-					drop_mpdu = true;
-					dev_kfree_skb_any(msdu);
-					msdu = NULL;
-					pmon->mon_last_linkdesc_paddr = paddr;
-					goto next_msdu;
-				}
-				msdu_ppdu_id =
-					ath12k_dp_rxdesc_get_ppduid(ar->ab, rx_desc);
-
-				if (ath12k_dp_mon_comp_ppduid(msdu_ppdu_id,
-							      ppdu_id)) {
-					spin_unlock_bh(&pmon->mon_lock);
-					return rx_bufs_used;
-				}
-				pmon->mon_last_linkdesc_paddr = paddr;
-				is_first_msdu = false;
-			}
-			ath12k_dp_mon_get_buf_len(&msdu_list.msdu_info[i],
-						  &is_frag, &total_len,
-						  &frag_len, &msdu_cnt);
-			rx_buf_size = rx_pkt_offset + l2_hdr_offset + frag_len;
-
-			if (ath12k_dp_pkt_set_pktlen(msdu, rx_buf_size)) {
-				dev_kfree_skb_any(msdu);
-				goto next_msdu;
-			}
-
-			if (!(*head_msdu))
-				*head_msdu = msdu;
-			else if (last)
-				last->next = msdu;
-
-			last = msdu;
-next_msdu:
-			pmon->mon_last_buf_cookie = msdu_list.sw_cookie[i];
-			rx_bufs_used++;
-			desc_info->skb = NULL;
-			list_add_tail(&desc_info->list, used_list);
-		}
-
-		ath12k_hal_rx_buf_addr_info_set(&buf_info, paddr, sw_cookie, rbm);
-
-		ath12k_dp_mon_next_link_desc_get(msdu_link_desc, &paddr,
-						 &sw_cookie, &rbm,
-						 &p_buf_addr_info);
-
-		ath12k_dp_rx_link_desc_return(ar->ab, &buf_info,
-					      HAL_WBM_REL_BM_ACT_PUT_IN_IDLE);
-
-		p_last_buf_addr_info = p_buf_addr_info;
-
-	} while (paddr && msdu_cnt);
-
-	spin_unlock_bh(&pmon->mon_lock);
-
-	if (last)
-		last->next = NULL;
-
-	*tail_msdu = msdu;
-
-	if (msdu_cnt == 0)
-		*npackets = 1;
-
-	return rx_bufs_used;
-}
-
-/* The destination ring processing is stuck if the destination is not
- * moving while status ring moves 16 PPDU. The destination ring processing
- * skips this destination ring PPDU as a workaround.
- */
-#define MON_DEST_RING_STUCK_MAX_CNT 16
-
-static void ath12k_dp_rx_mon_dest_process(struct ath12k *ar, int mac_id,
-					  u32 quota, struct napi_struct *napi)
-{
-	struct ath12k_mon_data *pmon = (struct ath12k_mon_data *)&ar->dp.mon_data;
-	struct ath12k_pdev_mon_stats *rx_mon_stats;
-	u32 ppdu_id, rx_bufs_used = 0, ring_id;
-	u32 mpdu_rx_bufs_used, npackets = 0;
-	struct ath12k_dp *dp = &ar->ab->dp;
-	struct ath12k_base *ab = ar->ab;
-	void *ring_entry, *mon_dst_srng;
-	struct dp_mon_mpdu *tmp_mpdu;
-	LIST_HEAD(rx_desc_used_list);
-	struct hal_srng *srng;
-
-	ring_id = dp->rxdma_err_dst_ring[mac_id].ring_id;
-	srng = &ab->hal.srng_list[ring_id];
-
-	mon_dst_srng = &ab->hal.srng_list[ring_id];
-
-	spin_lock_bh(&srng->lock);
-
-	ath12k_hal_srng_access_begin(ab, mon_dst_srng);
-
-	ppdu_id = pmon->mon_ppdu_info.ppdu_id;
-	rx_mon_stats = &pmon->rx_mon_stats;
-
-	while ((ring_entry = ath12k_hal_srng_dst_peek(ar->ab, mon_dst_srng))) {
-		struct sk_buff *head_msdu, *tail_msdu;
-
-		head_msdu = NULL;
-		tail_msdu = NULL;
-
-		mpdu_rx_bufs_used = ath12k_dp_rx_mon_mpdu_pop(ar, mac_id, ring_entry,
-							      &head_msdu, &tail_msdu,
-							      &rx_desc_used_list,
-							      &npackets, &ppdu_id);
-
-		rx_bufs_used += mpdu_rx_bufs_used;
-
-		if (mpdu_rx_bufs_used) {
-			dp->mon_dest_ring_stuck_cnt = 0;
-		} else {
-			dp->mon_dest_ring_stuck_cnt++;
-			rx_mon_stats->dest_mon_not_reaped++;
-		}
-
-		if (dp->mon_dest_ring_stuck_cnt > MON_DEST_RING_STUCK_MAX_CNT) {
-			rx_mon_stats->dest_mon_stuck++;
-			ath12k_dbg(ar->ab, ATH12K_DBG_DATA,
-				   "status ring ppdu_id=%d dest ring ppdu_id=%d mon_dest_ring_stuck_cnt=%d dest_mon_not_reaped=%u dest_mon_stuck=%u\n",
-				   pmon->mon_ppdu_info.ppdu_id, ppdu_id,
-				   dp->mon_dest_ring_stuck_cnt,
-				   rx_mon_stats->dest_mon_not_reaped,
-				   rx_mon_stats->dest_mon_stuck);
-			spin_lock_bh(&pmon->mon_lock);
-			pmon->mon_ppdu_info.ppdu_id = ppdu_id;
-			spin_unlock_bh(&pmon->mon_lock);
-			continue;
-		}
-
-		if (ppdu_id != pmon->mon_ppdu_info.ppdu_id) {
-			spin_lock_bh(&pmon->mon_lock);
-			pmon->mon_ppdu_status = DP_PPDU_STATUS_START;
-			spin_unlock_bh(&pmon->mon_lock);
-			ath12k_dbg(ar->ab, ATH12K_DBG_DATA,
-				   "dest_rx: new ppdu_id %x != status ppdu_id %x dest_mon_not_reaped = %u dest_mon_stuck = %u\n",
-				   ppdu_id, pmon->mon_ppdu_info.ppdu_id,
-				   rx_mon_stats->dest_mon_not_reaped,
-				   rx_mon_stats->dest_mon_stuck);
-			break;
-		}
-
-		if (head_msdu && tail_msdu) {
-			tmp_mpdu = kzalloc(sizeof(*tmp_mpdu), GFP_ATOMIC);
-			if (!tmp_mpdu)
-				break;
-
-			tmp_mpdu->head = head_msdu;
-			tmp_mpdu->tail = tail_msdu;
-			tmp_mpdu->err_bitmap = pmon->err_bitmap;
-			tmp_mpdu->decap_format = pmon->decap_format;
-			ath12k_dp_mon_rx_deliver(ar, tmp_mpdu,
-						 &pmon->mon_ppdu_info, napi);
-			rx_mon_stats->dest_mpdu_done++;
-			kfree(tmp_mpdu);
-		}
-
-		ring_entry = ath12k_hal_srng_dst_get_next_entry(ar->ab,
-								mon_dst_srng);
-	}
-	ath12k_hal_srng_access_end(ar->ab, mon_dst_srng);
-
-	spin_unlock_bh(&srng->lock);
-
-	if (rx_bufs_used) {
-		rx_mon_stats->dest_ppdu_done++;
-		ath12k_dp_rx_bufs_replenish(ar->ab,
-					    &dp->rx_refill_buf_ring,
-					    &rx_desc_used_list,
-					    rx_bufs_used);
-	}
-}
-
-static int
-__ath12k_dp_mon_process_ring(struct ath12k *ar, int mac_id,
-			     struct napi_struct *napi, int *budget)
-{
-	struct ath12k_mon_data *pmon = (struct ath12k_mon_data *)&ar->dp.mon_data;
-	struct ath12k_pdev_mon_stats *rx_mon_stats = &pmon->rx_mon_stats;
-	struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info;
-	enum hal_rx_mon_status hal_status;
-	struct sk_buff_head skb_list;
-	int num_buffs_reaped;
-	struct sk_buff *skb;
-
-	__skb_queue_head_init(&skb_list);
-
-	num_buffs_reaped = ath12k_dp_rx_reap_mon_status_ring(ar->ab, mac_id,
-							     budget, &skb_list);
-	if (!num_buffs_reaped)
-		goto exit;
-
-	while ((skb = __skb_dequeue(&skb_list))) {
-		memset(ppdu_info, 0, sizeof(*ppdu_info));
-		ppdu_info->peer_id = HAL_INVALID_PEERID;
-
-		hal_status = ath12k_dp_mon_parse_rx_dest(ar, pmon, skb);
-
-		if (ar->monitor_started &&
-		    pmon->mon_ppdu_status == DP_PPDU_STATUS_START &&
-		    hal_status == HAL_TLV_STATUS_PPDU_DONE) {
-			rx_mon_stats->status_ppdu_done++;
-			pmon->mon_ppdu_status = DP_PPDU_STATUS_DONE;
-			ath12k_dp_rx_mon_dest_process(ar, mac_id, *budget, napi);
-			pmon->mon_ppdu_status = DP_PPDU_STATUS_START;
-		}
-
-		dev_kfree_skb_any(skb);
-	}
-
-exit:
-	return num_buffs_reaped;
-}
-
-int ath12k_dp_mon_process_ring(struct ath12k_base *ab, int mac_id,
-			       struct napi_struct *napi, int budget,
-			       enum dp_monitor_mode monitor_mode)
-{
-	struct ath12k *ar = ath12k_ab_to_ar(ab, mac_id);
-	int num_buffs_reaped = 0;
-
-	if (ab->hw_params->rxdma1_enable) {
-		if (monitor_mode == ATH12K_DP_RX_MONITOR_MODE)
-			num_buffs_reaped = ath12k_dp_mon_srng_process(ar, &budget, napi);
-	} else {
-		if (ar->monitor_started)
-			num_buffs_reaped =
-				__ath12k_dp_mon_process_ring(ar, mac_id, napi, &budget);
-	}
-
-	return num_buffs_reaped;
+		ath12k_dp_mon_rx_update_user_stats(ab, ppdu_info, i);
 }
+EXPORT_SYMBOL(ath12k_dp_mon_rx_update_peer_mu_stats);
diff --git a/drivers/net/wireless/ath/ath12k/dp_mon.h b/drivers/net/wireless/ath/ath12k/dp_mon.h
index e25595cbdcf3..167028d27513 100644
--- a/drivers/net/wireless/ath/ath12k/dp_mon.h
+++ b/drivers/net/wireless/ath/ath12k/dp_mon.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause-Clear */
 /*
  * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #ifndef ATH12K_DP_MON_H
@@ -12,6 +12,12 @@
 #define ATH12K_MON_RX_DOT11_OFFSET	5
 #define ATH12K_MON_RX_PKT_OFFSET	8
 
+#define ATH12K_LE32_DEC_ENC(value, dec_bits, enc_bits)	\
+		u32_encode_bits(le32_get_bits(value, dec_bits), enc_bits)
+
+#define ATH12K_LE64_DEC_ENC(value, dec_bits, enc_bits) \
+		u32_encode_bits(le64_get_bits(value, dec_bits), enc_bits)
+
 enum dp_monitor_mode {
 	ATH12K_DP_TX_MONITOR_MODE,
 	ATH12K_DP_RX_MONITOR_MODE
@@ -77,31 +83,41 @@ struct dp_mon_tx_ppdu_info {
 	struct dp_mon_mpdu *tx_mon_mpdu;
 };
 
-enum hal_rx_mon_status
-ath12k_dp_mon_rx_parse_mon_status(struct ath12k *ar,
-				  struct ath12k_mon_data *pmon,
-				  struct sk_buff *skb,
-				  struct napi_struct *napi);
 int ath12k_dp_mon_buf_replenish(struct ath12k_base *ab,
 				struct dp_rxdma_mon_ring *buf_ring,
 				int req_entries);
 int ath12k_dp_mon_status_bufs_replenish(struct ath12k_base *ab,
 					struct dp_rxdma_mon_ring *rx_ring,
 					int req_entries);
-int ath12k_dp_mon_process_ring(struct ath12k_base *ab, int mac_id,
-			       struct napi_struct *napi, int budget,
-			       enum dp_monitor_mode monitor_mode);
-struct sk_buff *ath12k_dp_mon_tx_alloc_skb(void);
-enum dp_mon_tx_tlv_status
-ath12k_dp_mon_tx_status_get_num_user(u16 tlv_tag,
-				     struct hal_tlv_hdr *tx_tlv,
-				     u8 *num_users);
-enum hal_rx_mon_status
-ath12k_dp_mon_tx_parse_mon_status(struct ath12k *ar,
-				  struct ath12k_mon_data *pmon,
-				  struct sk_buff *skb,
-				  struct napi_struct *napi,
-				  u32 ppdu_id);
 void ath12k_dp_mon_rx_process_ulofdma(struct hal_rx_mon_ppdu_info *ppdu_info);
-int ath12k_dp_mon_srng_process(struct ath12k *ar, int *budget, struct napi_struct *napi);
+void
+ath12k_dp_mon_rx_update_peer_mu_stats(struct ath12k_base *ab,
+				      struct hal_rx_mon_ppdu_info *ppdu_info);
+void ath12k_dp_mon_rx_update_peer_su_stats(struct ath12k_dp_link_peer *peer,
+					   struct hal_rx_mon_ppdu_info *ppdu_info);
+int ath12k_dp_pkt_set_pktlen(struct sk_buff *skb, u32 len);
+struct sk_buff
+*ath12k_dp_rx_alloc_mon_status_buf(struct ath12k_base *ab,
+				   struct dp_rxdma_mon_ring *rx_ring,
+				   int *buf_id);
+u32 ath12k_dp_mon_comp_ppduid(u32 msdu_ppdu_id, u32 *ppdu_id);
+int
+ath12k_dp_mon_parse_status_buf(struct ath12k_pdev_dp *dp_pdev,
+			       struct ath12k_mon_data *pmon,
+			       const struct dp_mon_packet_info *packet_info);
+void ath12k_dp_mon_update_radiotap(struct ath12k_pdev_dp *dp_pdev,
+				   struct hal_rx_mon_ppdu_info *ppduinfo,
+				   struct sk_buff *mon_skb,
+				   struct ieee80211_rx_status *rxs);
+void ath12k_dp_mon_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev,
+				   struct napi_struct *napi,
+				   struct sk_buff *msdu,
+				   const struct hal_rx_mon_ppdu_info *ppduinfo,
+				   struct ieee80211_rx_status *status,
+				   u8 decap);
+struct sk_buff *
+ath12k_dp_mon_rx_merg_msdus(struct ath12k_pdev_dp *dp_pdev,
+			    struct dp_mon_mpdu *mon_mpdu,
+			    struct hal_rx_mon_ppdu_info *ppdu_info,
+			    struct ieee80211_rx_status *rxs);
 #endif
diff --git a/drivers/net/wireless/ath/ath12k/dp_peer.c b/drivers/net/wireless/ath/ath12k/dp_peer.c
new file mode 100644
index 000000000000..2e66872b5572
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/dp_peer.c
@@ -0,0 +1,690 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include "core.h"
+#include "dp_peer.h"
+#include "debug.h"
+#include "debugfs.h"
+
+void ath12k_dp_link_peer_free(struct ath12k_dp_link_peer *peer)
+{
+	list_del(&peer->list);
+
+	kfree(peer->peer_stats.rx_stats);
+	kfree(peer);
+}
+
+struct ath12k_dp_link_peer *
+ath12k_dp_link_peer_find_by_vdev_and_addr(struct ath12k_dp *dp,
+					  int vdev_id, const u8 *addr)
+{
+	struct ath12k_dp_link_peer *peer;
+
+	lockdep_assert_held(&dp->dp_lock);
+
+	list_for_each_entry(peer, &dp->peers, list) {
+		if (peer->vdev_id != vdev_id)
+			continue;
+		if (!ether_addr_equal(peer->addr, addr))
+			continue;
+
+		return peer;
+	}
+
+	return NULL;
+}
+
+struct ath12k_dp_link_peer *
+ath12k_dp_link_peer_find_by_pdev_and_addr(struct ath12k_dp *dp, u8 pdev_idx,
+					  const u8 *addr)
+{
+	struct ath12k_dp_link_peer *peer;
+
+	lockdep_assert_held(&dp->dp_lock);
+
+	list_for_each_entry(peer, &dp->peers, list) {
+		if (peer->pdev_idx != pdev_idx)
+			continue;
+		if (!ether_addr_equal(peer->addr, addr))
+			continue;
+
+		return peer;
+	}
+
+	return NULL;
+}
+
+struct ath12k_dp_link_peer *
+ath12k_dp_link_peer_find_by_addr(struct ath12k_dp *dp, const u8 *addr)
+{
+	lockdep_assert_held(&dp->dp_lock);
+
+	return rhashtable_lookup_fast(dp->rhead_peer_addr, addr,
+				      dp->rhash_peer_addr_param);
+}
+EXPORT_SYMBOL(ath12k_dp_link_peer_find_by_addr);
+
+static struct ath12k_dp_link_peer *
+ath12k_dp_link_peer_find_by_ml_id(struct ath12k_dp *dp, int ml_peer_id)
+{
+	struct ath12k_dp_link_peer *peer;
+
+	lockdep_assert_held(&dp->dp_lock);
+
+	list_for_each_entry(peer, &dp->peers, list)
+		if (ml_peer_id == peer->ml_id)
+			return peer;
+
+	return NULL;
+}
+
+static struct ath12k_dp_link_peer *
+ath12k_dp_link_peer_search_by_id(struct ath12k_dp *dp, int peer_id)
+{
+	struct ath12k_dp_link_peer *peer;
+
+	lockdep_assert_held(&dp->dp_lock);
+
+	if (peer_id == HAL_INVALID_PEERID)
+		return NULL;
+
+	if (peer_id & ATH12K_PEER_ML_ID_VALID)
+		return ath12k_dp_link_peer_find_by_ml_id(dp, peer_id);
+
+	list_for_each_entry(peer, &dp->peers, list)
+		if (peer_id == peer->peer_id)
+			return peer;
+
+	return NULL;
+}
+
+bool ath12k_dp_link_peer_exist_by_vdev_id(struct ath12k_dp *dp, int vdev_id)
+{
+	struct ath12k_dp_link_peer *peer;
+
+	spin_lock_bh(&dp->dp_lock);
+
+	list_for_each_entry(peer, &dp->peers, list) {
+		if (vdev_id == peer->vdev_id) {
+			spin_unlock_bh(&dp->dp_lock);
+			return true;
+		}
+	}
+	spin_unlock_bh(&dp->dp_lock);
+	return false;
+}
+
+struct ath12k_dp_link_peer *
+ath12k_dp_link_peer_find_by_ast(struct ath12k_dp *dp, int ast_hash)
+{
+	struct ath12k_dp_link_peer *peer;
+
+	lockdep_assert_held(&dp->dp_lock);
+
+	list_for_each_entry(peer, &dp->peers, list)
+		if (ast_hash == peer->ast_hash)
+			return peer;
+
+	return NULL;
+}
+
+void ath12k_dp_link_peer_unmap_event(struct ath12k_base *ab, u16 peer_id)
+{
+	struct ath12k_dp_link_peer *peer;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+
+	spin_lock_bh(&dp->dp_lock);
+
+	peer = ath12k_dp_link_peer_search_by_id(dp, peer_id);
+	if (!peer) {
+		ath12k_warn(ab, "peer-unmap-event: unknown peer id %d\n",
+			    peer_id);
+		goto exit;
+	}
+
+	ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "htt peer unmap vdev %d peer %pM id %d\n",
+		   peer->vdev_id, peer->addr, peer_id);
+
+	ath12k_dp_link_peer_free(peer);
+	wake_up(&ab->peer_mapping_wq);
+
+exit:
+	spin_unlock_bh(&dp->dp_lock);
+}
+
+void ath12k_dp_link_peer_map_event(struct ath12k_base *ab, u8 vdev_id, u16 peer_id,
+				   u8 *mac_addr, u16 ast_hash, u16 hw_peer_id)
+{
+	struct ath12k_dp_link_peer *peer;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct ath12k *ar;
+
+	spin_lock_bh(&dp->dp_lock);
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, mac_addr);
+	if (!peer) {
+		peer = kzalloc(sizeof(*peer), GFP_ATOMIC);
+		if (!peer)
+			goto exit;
+
+		peer->vdev_id = vdev_id;
+		peer->peer_id = peer_id;
+		peer->ast_hash = ast_hash;
+		peer->hw_peer_id = hw_peer_id;
+		ether_addr_copy(peer->addr, mac_addr);
+
+		rcu_read_lock();
+		ar = ath12k_mac_get_ar_by_vdev_id(ab, vdev_id);
+		if (ar && ath12k_debugfs_is_extd_rx_stats_enabled(ar) &&
+		    !peer->peer_stats.rx_stats) {
+			peer->peer_stats.rx_stats =
+				kzalloc(sizeof(*peer->peer_stats.rx_stats), GFP_ATOMIC);
+		}
+		rcu_read_unlock();
+
+		list_add(&peer->list, &dp->peers);
+		wake_up(&ab->peer_mapping_wq);
+		ewma_avg_rssi_init(&peer->avg_rssi);
+	}
+	ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "htt peer map vdev %d peer %pM id %d\n",
+		   vdev_id, mac_addr, peer_id);
+
+exit:
+	spin_unlock_bh(&dp->dp_lock);
+}
+
+struct ath12k_link_sta *ath12k_dp_link_peer_to_link_sta(struct ath12k_base *ab,
+							struct ath12k_dp_link_peer *peer)
+{
+	struct ath12k_sta *ahsta;
+	struct ath12k_link_sta *arsta;
+
+	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
+			 "ath12k_dp_link_peer to ath12k_link_sta called without rcu lock");
+
+	if (!peer->sta)
+		return NULL;
+
+	ahsta = ath12k_sta_to_ahsta(peer->sta);
+	if (peer->ml_id & ATH12K_PEER_ML_ID_VALID) {
+		if (!(ahsta->links_map & BIT(peer->link_id))) {
+			ath12k_warn(ab, "peer %pM id %d link_id %d can't found in STA link_map 0x%x\n",
+				    peer->addr, peer->peer_id, peer->link_id,
+				    ahsta->links_map);
+			return NULL;
+		}
+		arsta = rcu_dereference(ahsta->link[peer->link_id]);
+		if (!arsta)
+			return NULL;
+	} else {
+		arsta =  &ahsta->deflink;
+	}
+	return arsta;
+}
+
+static int ath12k_dp_link_peer_rhash_addr_tbl_init(struct ath12k_dp *dp)
+{
+	struct ath12k_base *ab = dp->ab;
+	struct rhashtable_params *param;
+	struct rhashtable *rhash_addr_tbl;
+	int ret;
+
+	lockdep_assert_held(&dp->link_peer_rhash_tbl_lock);
+
+	rhash_addr_tbl = kzalloc(sizeof(*dp->rhead_peer_addr), GFP_KERNEL);
+	if (!rhash_addr_tbl)
+		return -ENOMEM;
+
+	param = &dp->rhash_peer_addr_param;
+
+	param->key_offset = offsetof(struct ath12k_dp_link_peer, addr);
+	param->head_offset = offsetof(struct ath12k_dp_link_peer, rhash_addr);
+	param->key_len = sizeof_field(struct ath12k_dp_link_peer, addr);
+	param->automatic_shrinking = true;
+	param->nelem_hint = ab->num_radios * ath12k_core_get_max_peers_per_radio(ab);
+
+	ret = rhashtable_init(rhash_addr_tbl, param);
+	if (ret) {
+		ath12k_warn(ab, "failed to init peer addr rhash table %d\n", ret);
+		goto err_free;
+	}
+
+	dp->rhead_peer_addr = rhash_addr_tbl;
+
+	return 0;
+
+err_free:
+	kfree(rhash_addr_tbl);
+
+	return ret;
+}
+
+int ath12k_dp_link_peer_rhash_tbl_init(struct ath12k_dp *dp)
+{
+	int ret;
+
+	mutex_lock(&dp->link_peer_rhash_tbl_lock);
+	ret = ath12k_dp_link_peer_rhash_addr_tbl_init(dp);
+	mutex_unlock(&dp->link_peer_rhash_tbl_lock);
+
+	return ret;
+}
+
+void ath12k_dp_link_peer_rhash_tbl_destroy(struct ath12k_dp *dp)
+{
+	mutex_lock(&dp->link_peer_rhash_tbl_lock);
+	rhashtable_destroy(dp->rhead_peer_addr);
+	kfree(dp->rhead_peer_addr);
+	dp->rhead_peer_addr = NULL;
+	mutex_unlock(&dp->link_peer_rhash_tbl_lock);
+}
+
+static int ath12k_dp_link_peer_rhash_insert(struct ath12k_dp *dp,
+					    struct ath12k_dp_link_peer *peer)
+{
+	struct ath12k_dp_link_peer *tmp;
+
+	lockdep_assert_held(&dp->dp_lock);
+
+	tmp = rhashtable_lookup_get_insert_fast(dp->rhead_peer_addr, &peer->rhash_addr,
+						dp->rhash_peer_addr_param);
+	if (!tmp)
+		return 0;
+	else if (IS_ERR(tmp))
+		return PTR_ERR(tmp);
+	else
+		return -EEXIST;
+}
+
+static int ath12k_dp_link_peer_rhash_remove(struct ath12k_dp *dp,
+					    struct ath12k_dp_link_peer *peer)
+{
+	int ret;
+
+	lockdep_assert_held(&dp->dp_lock);
+
+	ret = rhashtable_remove_fast(dp->rhead_peer_addr, &peer->rhash_addr,
+				     dp->rhash_peer_addr_param);
+	if (ret && ret != -ENOENT)
+		return ret;
+
+	return 0;
+}
+
+int ath12k_dp_link_peer_rhash_add(struct ath12k_dp *dp,
+				  struct ath12k_dp_link_peer *peer)
+{
+	int ret;
+
+	lockdep_assert_held(&dp->dp_lock);
+
+	ret = ath12k_dp_link_peer_rhash_insert(dp, peer);
+	if (ret)
+		ath12k_warn(dp, "failed to add peer %pM with id %d in rhash_addr ret %d\n",
+			    peer->addr, peer->peer_id, ret);
+
+	return ret;
+}
+
+void ath12k_dp_link_peer_rhash_delete(struct ath12k_dp *dp,
+				      struct ath12k_dp_link_peer *peer)
+{
+	/* No failure handling and hence return type is void */
+	int ret;
+
+	lockdep_assert_held(&dp->dp_lock);
+
+	ret = ath12k_dp_link_peer_rhash_remove(dp, peer);
+	if (ret)
+		ath12k_warn(dp, "failed to remove peer %pM with id %d in rhash_addr ret %d\n",
+			    peer->addr, peer->peer_id, ret);
+}
+
+struct ath12k_dp_peer *ath12k_dp_peer_find_by_addr(struct ath12k_dp_hw *dp_hw, u8 *addr)
+{
+	struct ath12k_dp_peer *peer;
+
+	lockdep_assert_held(&dp_hw->peer_lock);
+
+	list_for_each_entry(peer, &dp_hw->dp_peers_list, list) {
+		if (ether_addr_equal(peer->addr, addr))
+			return peer;
+	}
+
+	return NULL;
+}
+EXPORT_SYMBOL(ath12k_dp_peer_find_by_addr);
+
+struct ath12k_dp_peer *ath12k_dp_peer_find_by_addr_and_sta(struct ath12k_dp_hw *dp_hw,
+							   u8 *addr,
+							   struct ieee80211_sta *sta)
+{
+	struct ath12k_dp_peer *dp_peer;
+
+	lockdep_assert_held(&dp_hw->peer_lock);
+
+	list_for_each_entry(dp_peer, &dp_hw->dp_peers_list, list) {
+		if (ether_addr_equal(dp_peer->addr, addr) && (dp_peer->sta == sta))
+			return dp_peer;
+	}
+
+	return NULL;
+}
+
+static struct ath12k_dp_peer *ath12k_dp_peer_create_find(struct ath12k_dp_hw *dp_hw,
+							 u8 *addr,
+							 struct ieee80211_sta *sta,
+							 bool mlo_peer)
+{
+	struct ath12k_dp_peer *dp_peer;
+
+	lockdep_assert_held(&dp_hw->peer_lock);
+
+	list_for_each_entry(dp_peer, &dp_hw->dp_peers_list, list) {
+		if (ether_addr_equal(dp_peer->addr, addr)) {
+			if (!sta || mlo_peer || dp_peer->is_mlo ||
+			    dp_peer->sta == sta)
+				return dp_peer;
+		}
+	}
+
+	return NULL;
+}
+
+/*
+ * Index of ath12k_dp_peer for MLO client is same as peer id of ath12k_dp_peer,
+ * while for ath12k_dp_link_peer(mlo and non-mlo) and ath12k_dp_peer for
+ * Non-MLO client it is derived as ((DEVICE_ID << 10) | (10 bits of peer id)).
+ *
+ * This is done because ml_peer_id and peer_id_table are at hw granularity,
+ * while link_peer_id is at device granularity, hence in order to avoid
+ * conflict this approach is followed.
+ */
+#define ATH12K_DP_PEER_TABLE_DEVICE_ID_SHIFT        10
+
+u16 ath12k_dp_peer_get_peerid_index(struct ath12k_dp *dp, u16 peer_id)
+{
+	return (peer_id & ATH12K_PEER_ML_ID_VALID) ? peer_id :
+		((dp->device_id << ATH12K_DP_PEER_TABLE_DEVICE_ID_SHIFT) | peer_id);
+}
+
+struct ath12k_dp_peer *ath12k_dp_peer_find_by_peerid(struct ath12k_pdev_dp *dp_pdev,
+						     u16 peer_id)
+{
+	u16 index;
+	struct ath12k_dp *dp = dp_pdev->dp;
+
+	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
+			 "ath12k dp peer find by peerid index called without rcu lock");
+
+	if (!peer_id || peer_id >= ATH12K_DP_PEER_ID_INVALID)
+		return NULL;
+
+	index = ath12k_dp_peer_get_peerid_index(dp, peer_id);
+
+	return rcu_dereference(dp_pdev->dp_hw->dp_peers[index]);
+}
+EXPORT_SYMBOL(ath12k_dp_peer_find_by_peerid);
+
+struct ath12k_dp_link_peer *
+ath12k_dp_link_peer_find_by_peerid(struct ath12k_pdev_dp *dp_pdev, u16 peer_id)
+{
+	struct ath12k_dp_peer *dp_peer = NULL;
+	u8 link_id;
+
+	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
+			 "ath12k dp link peer find by peerid index called without rcu lock");
+
+	if (dp_pdev->hw_link_id >= ATH12K_GROUP_MAX_RADIO)
+		return NULL;
+
+	dp_peer = ath12k_dp_peer_find_by_peerid(dp_pdev, peer_id);
+	if (!dp_peer)
+		return NULL;
+
+	link_id = dp_peer->hw_links[dp_pdev->hw_link_id];
+
+	return rcu_dereference(dp_peer->link_peers[link_id]);
+}
+EXPORT_SYMBOL(ath12k_dp_link_peer_find_by_peerid);
+
+int ath12k_dp_peer_create(struct ath12k_dp_hw *dp_hw, u8 *addr,
+			  struct ath12k_dp_peer_create_params *params)
+{
+	struct ath12k_dp_peer *dp_peer;
+
+	spin_lock_bh(&dp_hw->peer_lock);
+	dp_peer = ath12k_dp_peer_create_find(dp_hw, addr, params->sta, params->is_mlo);
+	if (dp_peer) {
+		spin_unlock_bh(&dp_hw->peer_lock);
+		return -EEXIST;
+	}
+	spin_unlock_bh(&dp_hw->peer_lock);
+
+	dp_peer = kzalloc(sizeof(*dp_peer), GFP_ATOMIC);
+	if (!dp_peer)
+		return -ENOMEM;
+
+	ether_addr_copy(dp_peer->addr, addr);
+	dp_peer->sta = params->sta;
+	dp_peer->is_mlo = params->is_mlo;
+
+	/*
+	 * For MLO client, the host assigns the ML peer ID, so set peer_id in dp_peer
+	 * For non-MLO client, host gets link peer ID from firmware and will be
+	 * assigned at the time of link peer creation
+	 */
+	dp_peer->peer_id = params->is_mlo ? params->peer_id : ATH12K_DP_PEER_ID_INVALID;
+	dp_peer->ucast_ra_only = params->ucast_ra_only;
+
+	dp_peer->sec_type = HAL_ENCRYPT_TYPE_OPEN;
+	dp_peer->sec_type_grp = HAL_ENCRYPT_TYPE_OPEN;
+	dp_peer->ucast_ra_only = params->ucast_ra_only;
+
+	spin_lock_bh(&dp_hw->peer_lock);
+
+	list_add(&dp_peer->list, &dp_hw->dp_peers_list);
+
+	/*
+	 * For MLO client, the peer_id for ath12k_dp_peer is allocated by host
+	 * and that peer_id is known at this point, and hence this ath12k_dp_peer
+	 * can be added to the RCU table using the peer_id.
+	 * For non-MLO client, this addition to RCU table shall be done at the
+	 * time of assignment of ath12k_dp_link_peer to ath12k_dp_peer.
+	 */
+	if (dp_peer->is_mlo)
+		rcu_assign_pointer(dp_hw->dp_peers[dp_peer->peer_id], dp_peer);
+
+	spin_unlock_bh(&dp_hw->peer_lock);
+
+	return 0;
+}
+
+void ath12k_dp_peer_delete(struct ath12k_dp_hw *dp_hw, u8 *addr,
+			   struct ieee80211_sta *sta)
+{
+	struct ath12k_dp_peer *dp_peer;
+
+	spin_lock_bh(&dp_hw->peer_lock);
+
+	dp_peer = ath12k_dp_peer_find_by_addr_and_sta(dp_hw, addr, sta);
+	if (!dp_peer) {
+		spin_unlock_bh(&dp_hw->peer_lock);
+		return;
+	}
+
+	if (dp_peer->is_mlo)
+		rcu_assign_pointer(dp_hw->dp_peers[dp_peer->peer_id], NULL);
+
+	list_del(&dp_peer->list);
+
+	spin_unlock_bh(&dp_hw->peer_lock);
+
+	synchronize_rcu();
+	kfree(dp_peer);
+}
+
+int ath12k_dp_link_peer_assign(struct ath12k_dp *dp, struct ath12k_dp_hw *dp_hw,
+			       u8 vdev_id, struct ieee80211_sta *sta, u8 *addr,
+			       u8 link_id, u32 hw_link_id)
+{
+	struct ath12k_dp_peer *dp_peer;
+	struct ath12k_dp_link_peer *peer, *temp_peer;
+	u16 peerid_index;
+	int ret = -EINVAL;
+	u8 *dp_peer_mac = !sta ? addr : sta->addr;
+
+	spin_lock_bh(&dp->dp_lock);
+
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, addr);
+	if (!peer) {
+		ath12k_warn(dp, "failed to find dp_link_peer with mac %pM on vdev %u\n",
+			    addr, vdev_id);
+		ret = -ENOENT;
+		goto err_peer;
+	}
+
+	spin_lock_bh(&dp_hw->peer_lock);
+
+	dp_peer = ath12k_dp_peer_find_by_addr_and_sta(dp_hw, dp_peer_mac, sta);
+	if (!dp_peer) {
+		ath12k_warn(dp, "failed to find dp_peer with mac %pM\n", dp_peer_mac);
+		ret = -ENOENT;
+		goto err_dp_peer;
+	}
+
+	/*
+	 * Set peer_id in dp_peer for non-mlo client, peer_id for mlo client is
+	 * set during dp_peer create
+	 */
+	if (!dp_peer->is_mlo)
+		dp_peer->peer_id = peer->peer_id;
+
+	peer->dp_peer = dp_peer;
+	peer->hw_link_id = hw_link_id;
+
+	dp_peer->hw_links[peer->hw_link_id] = link_id;
+
+	peerid_index = ath12k_dp_peer_get_peerid_index(dp, peer->peer_id);
+
+	rcu_assign_pointer(dp_peer->link_peers[peer->link_id], peer);
+
+	rcu_assign_pointer(dp_hw->dp_peers[peerid_index], dp_peer);
+
+	spin_unlock_bh(&dp_hw->peer_lock);
+
+	/*
+	 * In case of Split PHY and roaming scenario, pdev idx
+	 * might differ but both the pdev will share same rhash
+	 * table. In that case update the rhash table if link_peer is
+	 * already present
+	 */
+	temp_peer = ath12k_dp_link_peer_find_by_addr(dp, addr);
+	if (temp_peer && temp_peer->hw_link_id != hw_link_id)
+		ath12k_dp_link_peer_rhash_delete(dp, temp_peer);
+
+	ret = ath12k_dp_link_peer_rhash_add(dp, peer);
+	if (ret) {
+		/*
+		 * If new entry addition failed, add back old entry
+		 * If old entry addition also fails, then nothing
+		 * can be done, simply proceed
+		 */
+		if (temp_peer)
+			ath12k_dp_link_peer_rhash_add(dp, temp_peer);
+	}
+
+	spin_unlock_bh(&dp->dp_lock);
+
+	return ret;
+
+err_dp_peer:
+	spin_unlock_bh(&dp_hw->peer_lock);
+
+err_peer:
+	spin_unlock_bh(&dp->dp_lock);
+
+	return ret;
+}
+
+void ath12k_dp_link_peer_unassign(struct ath12k_dp *dp, struct ath12k_dp_hw *dp_hw,
+				  u8 vdev_id, u8 *addr, u32 hw_link_id)
+{
+	struct ath12k_dp_peer *dp_peer;
+	struct ath12k_dp_link_peer *peer, *temp_peer;
+	u16 peerid_index;
+
+	spin_lock_bh(&dp->dp_lock);
+
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, addr);
+	if (!peer || !peer->dp_peer) {
+		spin_unlock_bh(&dp->dp_lock);
+		return;
+	}
+
+	spin_lock_bh(&dp_hw->peer_lock);
+
+	dp_peer = peer->dp_peer;
+	dp_peer->hw_links[peer->hw_link_id] = 0;
+
+	peerid_index = ath12k_dp_peer_get_peerid_index(dp, peer->peer_id);
+
+	rcu_assign_pointer(dp_peer->link_peers[peer->link_id], NULL);
+
+	rcu_assign_pointer(dp_hw->dp_peers[peerid_index], NULL);
+
+	spin_unlock_bh(&dp_hw->peer_lock);
+
+	/* To handle roaming and split phy scenario */
+	temp_peer = ath12k_dp_link_peer_find_by_addr(dp, addr);
+	if (temp_peer && temp_peer->hw_link_id == hw_link_id)
+		ath12k_dp_link_peer_rhash_delete(dp, peer);
+
+	spin_unlock_bh(&dp->dp_lock);
+
+	synchronize_rcu();
+}
+
+void
+ath12k_dp_link_peer_get_sta_rate_info_stats(struct ath12k_dp *dp, const u8 *addr,
+					    struct ath12k_dp_link_peer_rate_info *info)
+{
+	struct ath12k_dp_link_peer *link_peer;
+
+	guard(spinlock_bh)(&dp->dp_lock);
+
+	link_peer = ath12k_dp_link_peer_find_by_addr(dp, addr);
+	if (!link_peer)
+		return;
+
+	info->rx_duration = link_peer->rx_duration;
+	info->tx_duration = link_peer->tx_duration;
+	info->txrate.legacy = link_peer->txrate.legacy;
+	info->txrate.mcs = link_peer->txrate.mcs;
+	info->txrate.nss = link_peer->txrate.nss;
+	info->txrate.bw = link_peer->txrate.bw;
+	info->txrate.he_gi = link_peer->txrate.he_gi;
+	info->txrate.he_dcm = link_peer->txrate.he_dcm;
+	info->txrate.he_ru_alloc = link_peer->txrate.he_ru_alloc;
+	info->txrate.flags = link_peer->txrate.flags;
+	info->rssi_comb = link_peer->rssi_comb;
+	info->signal_avg = ewma_avg_rssi_read(&link_peer->avg_rssi);
+}
+
+void ath12k_dp_link_peer_reset_rx_stats(struct ath12k_dp *dp, const u8 *addr)
+{
+	struct ath12k_rx_peer_stats *rx_stats;
+	struct ath12k_dp_link_peer *link_peer;
+
+	guard(spinlock_bh)(&dp->dp_lock);
+
+	link_peer = ath12k_dp_link_peer_find_by_addr(dp, addr);
+	if (!link_peer || !link_peer->peer_stats.rx_stats)
+		return;
+
+	rx_stats = link_peer->peer_stats.rx_stats;
+	if (rx_stats)
+		memset(rx_stats, 0, sizeof(*rx_stats));
+}
diff --git a/drivers/net/wireless/ath/ath12k/dp_peer.h b/drivers/net/wireless/ath/ath12k/dp_peer.h
new file mode 100644
index 000000000000..20294ff09513
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/dp_peer.h
@@ -0,0 +1,182 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef ATH12K_DP_PEER_H
+#define ATH12K_DP_PEER_H
+
+#include "dp_rx.h"
+
+#define ATH12K_DP_PEER_ID_INVALID              0x3FFF
+
+struct ppdu_user_delayba {
+	u16 sw_peer_id;
+	u32 info0;
+	u16 ru_end;
+	u16 ru_start;
+	u32 info1;
+	u32 rate_flags;
+	u32 resp_rate_flags;
+};
+
+#define ATH12K_PEER_ML_ID_VALID         BIT(13)
+
+struct ath12k_rx_peer_rate_stats {
+	u64 ht_mcs_count[HAL_RX_MAX_MCS_HT + 1];
+	u64 vht_mcs_count[HAL_RX_MAX_MCS_VHT + 1];
+	u64 he_mcs_count[HAL_RX_MAX_MCS_HE + 1];
+	u64 be_mcs_count[HAL_RX_MAX_MCS_BE + 1];
+	u64 nss_count[HAL_RX_MAX_NSS];
+	u64 bw_count[HAL_RX_BW_MAX];
+	u64 gi_count[HAL_RX_GI_MAX];
+	u64 legacy_count[HAL_RX_MAX_NUM_LEGACY_RATES];
+	u64 rx_rate[HAL_RX_BW_MAX][HAL_RX_GI_MAX][HAL_RX_MAX_NSS][HAL_RX_MAX_MCS_HT + 1];
+};
+
+struct ath12k_rx_peer_stats {
+	u64 num_msdu;
+	u64 num_mpdu_fcs_ok;
+	u64 num_mpdu_fcs_err;
+	u64 tcp_msdu_count;
+	u64 udp_msdu_count;
+	u64 other_msdu_count;
+	u64 ampdu_msdu_count;
+	u64 non_ampdu_msdu_count;
+	u64 stbc_count;
+	u64 beamformed_count;
+	u64 coding_count[HAL_RX_SU_MU_CODING_MAX];
+	u64 tid_count[IEEE80211_NUM_TIDS + 1];
+	u64 pream_cnt[HAL_RX_PREAMBLE_MAX];
+	u64 reception_type[HAL_RX_RECEPTION_TYPE_MAX];
+	u64 rx_duration;
+	u64 dcm_count;
+	u64 ru_alloc_cnt[HAL_RX_RU_ALLOC_TYPE_MAX];
+	struct ath12k_rx_peer_rate_stats pkt_stats;
+	struct ath12k_rx_peer_rate_stats byte_stats;
+};
+
+struct ath12k_wbm_tx_stats {
+	u64 wbm_tx_comp_stats[HAL_WBM_REL_HTT_TX_COMP_STATUS_MAX];
+};
+
+struct ath12k_dp_peer_stats {
+	struct ath12k_rx_peer_stats *rx_stats;
+	struct ath12k_wbm_tx_stats *wbm_tx_stats;
+};
+
+DECLARE_EWMA(avg_rssi, 10, 8)
+
+struct ath12k_dp_link_peer {
+	struct list_head list;
+	struct ieee80211_sta *sta;
+	struct ath12k_dp_peer *dp_peer;
+	int vdev_id;
+	u8 addr[ETH_ALEN];
+	int peer_id;
+	u16 ast_hash;
+	u8 pdev_idx;
+	u16 hw_peer_id;
+
+	struct ppdu_user_delayba ppdu_stats_delayba;
+	bool delayba_flag;
+	bool is_authorized;
+	bool mlo;
+	/* protected by ab->data_lock */
+
+	u16 ml_id;
+
+	/* any other ML info common for all partners can be added
+	 * here and would be same for all partner peers.
+	 */
+	u8 ml_addr[ETH_ALEN];
+
+	/* To ensure only certain work related to dp is done once */
+	bool primary_link;
+
+	/* for reference to ath12k_link_sta */
+	u8 link_id;
+
+	/* peer addr based rhashtable list pointer */
+	struct rhash_head rhash_addr;
+
+	u8 hw_link_id;
+	u32 rx_tid_active_bitmask;
+
+	/* link stats */
+	struct rate_info txrate;
+	struct rate_info last_txrate;
+	u64 rx_duration;
+	u64 tx_duration;
+	u8 rssi_comb;
+	struct ewma_avg_rssi avg_rssi;
+	struct ath12k_dp_peer_stats peer_stats;
+	u32 tx_retry_failed;
+	u32 tx_retry_count;
+};
+
+void ath12k_dp_link_peer_unmap_event(struct ath12k_base *ab, u16 peer_id);
+void ath12k_dp_link_peer_map_event(struct ath12k_base *ab, u8 vdev_id, u16 peer_id,
+				   u8 *mac_addr, u16 ast_hash, u16 hw_peer_id);
+
+struct ath12k_dp_peer {
+	struct list_head list;
+	bool is_mlo;
+	bool dp_setup_done;
+
+	u8 ucast_keyidx;
+	u8 addr[ETH_ALEN];
+
+	u8 mcast_keyidx;
+	bool ucast_ra_only;
+	int peer_id;
+	struct ieee80211_sta *sta;
+
+	u8 hw_links[ATH12K_GROUP_MAX_RADIO];
+
+	u16 sec_type_grp;
+	u16 sec_type;
+
+	/* Info used in MMIC verification of * RX fragments */
+	struct crypto_shash *tfm_mmic;
+	struct ieee80211_key_conf *keys[WMI_MAX_KEY_INDEX + 1];
+	struct ath12k_dp_link_peer __rcu *link_peers[ATH12K_NUM_MAX_LINKS];
+	struct ath12k_reoq_buf reoq_bufs[IEEE80211_NUM_TIDS + 1];
+	struct ath12k_dp_rx_tid rx_tid[IEEE80211_NUM_TIDS + 1];
+};
+
+struct ath12k_dp_link_peer *
+ath12k_dp_link_peer_find_by_vdev_and_addr(struct ath12k_dp *dp,
+					  int vdev_id, const u8 *addr);
+struct ath12k_dp_link_peer *
+ath12k_dp_link_peer_find_by_addr(struct ath12k_dp *dp, const u8 *addr);
+bool ath12k_dp_link_peer_exist_by_vdev_id(struct ath12k_dp *dp, int vdev_id);
+struct ath12k_dp_link_peer *
+ath12k_dp_link_peer_find_by_ast(struct ath12k_dp *dp, int ast_hash);
+struct ath12k_dp_link_peer *
+ath12k_dp_link_peer_find_by_pdev_and_addr(struct ath12k_dp *dp, u8 pdev_idx,
+					  const u8 *addr);
+struct ath12k_link_sta *ath12k_dp_link_peer_to_link_sta(struct ath12k_base *ab,
+							struct ath12k_dp_link_peer *peer);
+int ath12k_dp_link_peer_rhash_tbl_init(struct ath12k_dp *dp);
+void ath12k_dp_link_peer_rhash_tbl_destroy(struct ath12k_dp *dp);
+int ath12k_dp_link_peer_rhash_add(struct ath12k_dp *dp,
+				  struct ath12k_dp_link_peer *peer);
+void ath12k_dp_link_peer_rhash_delete(struct ath12k_dp *dp,
+				      struct ath12k_dp_link_peer *peer);
+int ath12k_dp_peer_create(struct ath12k_dp_hw *dp_hw, u8 *addr,
+			  struct ath12k_dp_peer_create_params *params);
+void ath12k_dp_peer_delete(struct ath12k_dp_hw *dp_hw, u8 *addr,
+			   struct ieee80211_sta *sta);
+struct ath12k_dp_peer *ath12k_dp_peer_find_by_addr(struct ath12k_dp_hw *dp_hw, u8 *addr);
+struct ath12k_dp_peer *ath12k_dp_peer_find_by_addr_and_sta(struct ath12k_dp_hw *dp_hw,
+							   u8 *addr,
+							   struct ieee80211_sta *sta);
+u16 ath12k_dp_peer_get_peerid_index(struct ath12k_dp *dp, u16 peer_id);
+struct ath12k_dp_peer *ath12k_dp_peer_find_by_peerid(struct ath12k_pdev_dp *dp_pdev,
+						     u16 peer_id);
+struct ath12k_dp_link_peer *
+ath12k_dp_link_peer_find_by_peerid(struct ath12k_pdev_dp *dp_pdev, u16 peer_id);
+void ath12k_dp_link_peer_free(struct ath12k_dp_link_peer *peer);
+#endif
diff --git a/drivers/net/wireless/ath/ath12k/dp_rx.c b/drivers/net/wireless/ath/ath12k/dp_rx.c
index d28d8ffec0f8..a32ee9f8061a 100644
--- a/drivers/net/wireless/ath/ath12k/dp_rx.c
+++ b/drivers/net/wireless/ath/ath12k/dp_rx.c
@@ -10,257 +10,16 @@
 #include <crypto/hash.h>
 #include "core.h"
 #include "debug.h"
-#include "hal_desc.h"
 #include "hw.h"
 #include "dp_rx.h"
-#include "hal_rx.h"
 #include "dp_tx.h"
 #include "peer.h"
 #include "dp_mon.h"
 #include "debugfs_htt_stats.h"
 
-#define ATH12K_DP_RX_FRAGMENT_TIMEOUT_MS (2 * HZ)
-
 static int ath12k_dp_rx_tid_delete_handler(struct ath12k_base *ab,
 					   struct ath12k_dp_rx_tid_rxq *rx_tid);
 
-static enum hal_encrypt_type ath12k_dp_rx_h_enctype(struct ath12k_base *ab,
-						    struct hal_rx_desc *desc)
-{
-	if (!ab->hal_rx_ops->rx_desc_encrypt_valid(desc))
-		return HAL_ENCRYPT_TYPE_OPEN;
-
-	return ab->hal_rx_ops->rx_desc_get_encrypt_type(desc);
-}
-
-u8 ath12k_dp_rx_h_decap_type(struct ath12k_base *ab,
-			     struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_decap_type(desc);
-}
-
-static u8 ath12k_dp_rx_h_mesh_ctl_present(struct ath12k_base *ab,
-					  struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_mesh_ctl(desc);
-}
-
-static bool ath12k_dp_rx_h_seq_ctrl_valid(struct ath12k_base *ab,
-					  struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_mpdu_seq_ctl_vld(desc);
-}
-
-static bool ath12k_dp_rx_h_fc_valid(struct ath12k_base *ab,
-				    struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_mpdu_fc_valid(desc);
-}
-
-static bool ath12k_dp_rx_h_more_frags(struct ath12k_base *ab,
-				      struct sk_buff *skb)
-{
-	struct ieee80211_hdr *hdr;
-
-	hdr = (struct ieee80211_hdr *)(skb->data + ab->hal.hal_desc_sz);
-	return ieee80211_has_morefrags(hdr->frame_control);
-}
-
-static u16 ath12k_dp_rx_h_frag_no(struct ath12k_base *ab,
-				  struct sk_buff *skb)
-{
-	struct ieee80211_hdr *hdr;
-
-	hdr = (struct ieee80211_hdr *)(skb->data + ab->hal.hal_desc_sz);
-	return le16_to_cpu(hdr->seq_ctrl) & IEEE80211_SCTL_FRAG;
-}
-
-static u16 ath12k_dp_rx_h_seq_no(struct ath12k_base *ab,
-				 struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_mpdu_start_seq_no(desc);
-}
-
-static bool ath12k_dp_rx_h_msdu_done(struct ath12k_base *ab,
-				     struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->dp_rx_h_msdu_done(desc);
-}
-
-static bool ath12k_dp_rx_h_l4_cksum_fail(struct ath12k_base *ab,
-					 struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->dp_rx_h_l4_cksum_fail(desc);
-}
-
-static bool ath12k_dp_rx_h_ip_cksum_fail(struct ath12k_base *ab,
-					 struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->dp_rx_h_ip_cksum_fail(desc);
-}
-
-static bool ath12k_dp_rx_h_is_decrypted(struct ath12k_base *ab,
-					struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->dp_rx_h_is_decrypted(desc);
-}
-
-u32 ath12k_dp_rx_h_mpdu_err(struct ath12k_base *ab,
-			    struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->dp_rx_h_mpdu_err(desc);
-}
-
-static u16 ath12k_dp_rx_h_msdu_len(struct ath12k_base *ab,
-				   struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_msdu_len(desc);
-}
-
-static u8 ath12k_dp_rx_h_sgi(struct ath12k_base *ab,
-			     struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_msdu_sgi(desc);
-}
-
-static u8 ath12k_dp_rx_h_rate_mcs(struct ath12k_base *ab,
-				  struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_msdu_rate_mcs(desc);
-}
-
-static u8 ath12k_dp_rx_h_rx_bw(struct ath12k_base *ab,
-			       struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_msdu_rx_bw(desc);
-}
-
-static u32 ath12k_dp_rx_h_freq(struct ath12k_base *ab,
-			       struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_msdu_freq(desc);
-}
-
-static u8 ath12k_dp_rx_h_pkt_type(struct ath12k_base *ab,
-				  struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_msdu_pkt_type(desc);
-}
-
-static u8 ath12k_dp_rx_h_nss(struct ath12k_base *ab,
-			     struct hal_rx_desc *desc)
-{
-	return hweight8(ab->hal_rx_ops->rx_desc_get_msdu_nss(desc));
-}
-
-static u8 ath12k_dp_rx_h_tid(struct ath12k_base *ab,
-			     struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_mpdu_tid(desc);
-}
-
-static u16 ath12k_dp_rx_h_peer_id(struct ath12k_base *ab,
-				  struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_mpdu_peer_id(desc);
-}
-
-u8 ath12k_dp_rx_h_l3pad(struct ath12k_base *ab,
-			struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_l3_pad_bytes(desc);
-}
-
-static bool ath12k_dp_rx_h_first_msdu(struct ath12k_base *ab,
-				      struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_first_msdu(desc);
-}
-
-static bool ath12k_dp_rx_h_last_msdu(struct ath12k_base *ab,
-				     struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_last_msdu(desc);
-}
-
-static void ath12k_dp_rx_desc_end_tlv_copy(struct ath12k_base *ab,
-					   struct hal_rx_desc *fdesc,
-					   struct hal_rx_desc *ldesc)
-{
-	ab->hal_rx_ops->rx_desc_copy_end_tlv(fdesc, ldesc);
-}
-
-static void ath12k_dp_rxdesc_set_msdu_len(struct ath12k_base *ab,
-					  struct hal_rx_desc *desc,
-					  u16 len)
-{
-	ab->hal_rx_ops->rx_desc_set_msdu_len(desc, len);
-}
-
-u32 ath12k_dp_rxdesc_get_ppduid(struct ath12k_base *ab,
-				struct hal_rx_desc *rx_desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_mpdu_ppdu_id(rx_desc);
-}
-
-bool ath12k_dp_rxdesc_mpdu_valid(struct ath12k_base *ab,
-				 struct hal_rx_desc *rx_desc)
-{
-	u32 tlv_tag;
-
-	tlv_tag = ab->hal_rx_ops->rx_desc_get_mpdu_start_tag(rx_desc);
-
-	return tlv_tag == HAL_RX_MPDU_START;
-}
-
-static bool ath12k_dp_rx_h_is_da_mcbc(struct ath12k_base *ab,
-				      struct hal_rx_desc *desc)
-{
-	return (ath12k_dp_rx_h_first_msdu(ab, desc) &&
-		ab->hal_rx_ops->rx_desc_is_da_mcbc(desc));
-}
-
-static bool ath12k_dp_rxdesc_mac_addr2_valid(struct ath12k_base *ab,
-					     struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_mac_addr2_valid(desc);
-}
-
-static u8 *ath12k_dp_rxdesc_get_mpdu_start_addr2(struct ath12k_base *ab,
-						 struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_mpdu_start_addr2(desc);
-}
-
-static void ath12k_dp_rx_desc_get_dot11_hdr(struct ath12k_base *ab,
-					    struct hal_rx_desc *desc,
-					    struct ieee80211_hdr *hdr)
-{
-	ab->hal_rx_ops->rx_desc_get_dot11_hdr(desc, hdr);
-}
-
-static void ath12k_dp_rx_desc_get_crypto_header(struct ath12k_base *ab,
-						struct hal_rx_desc *desc,
-						u8 *crypto_hdr,
-						enum hal_encrypt_type enctype)
-{
-	ab->hal_rx_ops->rx_desc_get_crypto_header(desc, crypto_hdr, enctype);
-}
-
-static inline u8 ath12k_dp_rx_get_msdu_src_link(struct ath12k_base *ab,
-						struct hal_rx_desc *desc)
-{
-	return ab->hal_rx_ops->rx_desc_get_msdu_src_link_id(desc);
-}
-
-static void ath12k_dp_clean_up_skb_list(struct sk_buff_head *skb_list)
-{
-	struct sk_buff *skb;
-
-	while ((skb = __skb_dequeue(skb_list)))
-		dev_kfree_skb_any(skb);
-}
-
 static size_t ath12k_dp_list_cut_nodes(struct list_head *list,
 				       struct list_head *head,
 				       size_t count)
@@ -305,11 +64,12 @@ static void ath12k_dp_rx_enqueue_free(struct ath12k_dp *dp,
 }
 
 /* Returns number of Rx buffers replenished */
-int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab,
+int ath12k_dp_rx_bufs_replenish(struct ath12k_dp *dp,
 				struct dp_rxdma_ring *rx_ring,
 				struct list_head *used_list,
 				int req_entries)
 {
+	struct ath12k_base *ab = dp->ab;
 	struct ath12k_buffer_addr *desc;
 	struct hal_srng *srng;
 	struct sk_buff *skb;
@@ -317,13 +77,12 @@ int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab,
 	int num_remain;
 	u32 cookie;
 	dma_addr_t paddr;
-	struct ath12k_dp *dp = &ab->dp;
 	struct ath12k_rx_desc_info *rx_desc;
-	enum hal_rx_buf_return_buf_manager mgr = ab->hw_params->hal_params->rx_buf_rbm;
+	enum hal_rx_buf_return_buf_manager mgr = dp->hal->hal_params->rx_buf_rbm;
 
 	req_entries = min(req_entries, rx_ring->bufs_max);
 
-	srng = &ab->hal.srng_list[rx_ring->refill_buf_ring.ring_id];
+	srng = &dp->hal->srng_list[rx_ring->refill_buf_ring.ring_id];
 
 	spin_lock_bh(&srng->lock);
 
@@ -362,10 +121,10 @@ int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab,
 				 skb->data);
 		}
 
-		paddr = dma_map_single(ab->dev, skb->data,
+		paddr = dma_map_single(dp->dev, skb->data,
 				       skb->len + skb_tailroom(skb),
 				       DMA_FROM_DEVICE);
-		if (dma_mapping_error(ab->dev, paddr))
+		if (dma_mapping_error(dp->dev, paddr))
 			goto fail_free_skb;
 
 		rx_desc = list_first_entry_or_null(used_list,
@@ -386,13 +145,14 @@ int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab,
 
 		num_remain--;
 
-		ath12k_hal_rx_buf_addr_info_set(desc, paddr, cookie, mgr);
+		ath12k_hal_rx_buf_addr_info_set(dp->hal, desc, paddr, cookie,
+						mgr);
 	}
 
 	goto out;
 
 fail_dma_unmap:
-	dma_unmap_single(ab->dev, paddr, skb->len + skb_tailroom(skb),
+	dma_unmap_single(dp->dev, paddr, skb->len + skb_tailroom(skb),
 			 DMA_FROM_DEVICE);
 fail_free_skb:
 	dev_kfree_skb_any(skb);
@@ -406,6 +166,7 @@ int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab,
 
 	return req_entries - num_remain;
 }
+EXPORT_SYMBOL(ath12k_dp_rx_bufs_replenish);
 
 static int ath12k_dp_rxdma_mon_buf_ring_free(struct ath12k_base *ab,
 					     struct dp_rxdma_mon_ring *rx_ring)
@@ -432,7 +193,7 @@ static int ath12k_dp_rxdma_mon_buf_ring_free(struct ath12k_base *ab,
 
 static int ath12k_dp_rxdma_buf_free(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	int i;
 
 	ath12k_dp_rxdma_mon_buf_ring_free(ab, &dp->rxdma_mon_buf_ring);
@@ -475,14 +236,14 @@ static int ath12k_dp_rxdma_ring_buf_setup(struct ath12k_base *ab,
 	rx_ring->bufs_max = rx_ring->refill_buf_ring.size /
 			ath12k_hal_srng_get_entrysize(ab, HAL_RXDMA_BUF);
 
-	ath12k_dp_rx_bufs_replenish(ab, rx_ring, &list, 0);
+	ath12k_dp_rx_bufs_replenish(ath12k_ab_to_dp(ab), rx_ring, &list, 0);
 
 	return 0;
 }
 
 static int ath12k_dp_rxdma_buf_setup(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	struct dp_rxdma_mon_ring *mon_ring;
 	int ret, i;
 
@@ -529,7 +290,7 @@ static void ath12k_dp_rx_pdev_srng_free(struct ath12k *ar)
 
 void ath12k_dp_rx_pdev_reo_cleanup(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	int i;
 
 	for (i = 0; i < DP_REO_DST_RING_MAX; i++)
@@ -538,7 +299,7 @@ void ath12k_dp_rx_pdev_reo_cleanup(struct ath12k_base *ab)
 
 int ath12k_dp_rx_pdev_reo_setup(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	int ret;
 	int i;
 
@@ -584,13 +345,15 @@ static int ath12k_dp_rx_pdev_srng_alloc(struct ath12k *ar)
 	return 0;
 }
 
-static void ath12k_dp_init_rx_tid_rxq(struct ath12k_dp_rx_tid_rxq *rx_tid_rxq,
-				      struct ath12k_dp_rx_tid *rx_tid)
+void ath12k_dp_init_rx_tid_rxq(struct ath12k_dp_rx_tid_rxq *rx_tid_rxq,
+			       struct ath12k_dp_rx_tid *rx_tid,
+			       bool active)
 {
 	rx_tid_rxq->tid = rx_tid->tid;
-	rx_tid_rxq->active = rx_tid->active;
+	rx_tid_rxq->active = active;
 	rx_tid_rxq->qbuf = rx_tid->qbuf;
 }
+EXPORT_SYMBOL(ath12k_dp_init_rx_tid_rxq);
 
 static void ath12k_dp_rx_tid_cleanup(struct ath12k_base *ab,
 				     struct ath12k_reoq_buf *tid_qbuf)
@@ -605,7 +368,7 @@ static void ath12k_dp_rx_tid_cleanup(struct ath12k_base *ab,
 
 void ath12k_dp_rx_reo_cmd_list_cleanup(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	struct ath12k_dp_rx_reo_cmd *cmd, *tmp;
 	struct ath12k_dp_rx_reo_cache_flush_elem *cmd_cache, *tmp_cache;
 	struct dp_reo_update_rx_queue_elem *cmd_queue, *tmp_queue;
@@ -635,8 +398,8 @@ void ath12k_dp_rx_reo_cmd_list_cleanup(struct ath12k_base *ab)
 	spin_unlock_bh(&dp->reo_cmd_lock);
 }
 
-static void ath12k_dp_reo_cmd_free(struct ath12k_dp *dp, void *ctx,
-				   enum hal_reo_cmd_status status)
+void ath12k_dp_reo_cmd_free(struct ath12k_dp *dp, void *ctx,
+			    enum hal_reo_cmd_status status)
 {
 	struct ath12k_dp_rx_tid_rxq *rx_tid = ctx;
 
@@ -646,109 +409,9 @@ static void ath12k_dp_reo_cmd_free(struct ath12k_dp *dp, void *ctx,
 
 	ath12k_dp_rx_tid_cleanup(dp->ab, &rx_tid->qbuf);
 }
+EXPORT_SYMBOL(ath12k_dp_reo_cmd_free);
 
-static int ath12k_dp_reo_cmd_send(struct ath12k_base *ab,
-				  struct ath12k_dp_rx_tid_rxq *rx_tid,
-				  enum hal_reo_cmd_type type,
-				  struct ath12k_hal_reo_cmd *cmd,
-				  void (*cb)(struct ath12k_dp *dp, void *ctx,
-					     enum hal_reo_cmd_status status))
-{
-	struct ath12k_dp *dp = &ab->dp;
-	struct ath12k_dp_rx_reo_cmd *dp_cmd;
-	struct hal_srng *cmd_ring;
-	int cmd_num;
-
-	cmd_ring = &ab->hal.srng_list[dp->reo_cmd_ring.ring_id];
-	cmd_num = ath12k_hal_reo_cmd_send(ab, cmd_ring, type, cmd);
-
-	/* cmd_num should start from 1, during failure return the error code */
-	if (cmd_num < 0)
-		return cmd_num;
-
-	/* reo cmd ring descriptors has cmd_num starting from 1 */
-	if (cmd_num == 0)
-		return -EINVAL;
-
-	if (!cb)
-		return 0;
-
-	/* Can this be optimized so that we keep the pending command list only
-	 * for tid delete command to free up the resource on the command status
-	 * indication?
-	 */
-	dp_cmd = kzalloc(sizeof(*dp_cmd), GFP_ATOMIC);
-
-	if (!dp_cmd)
-		return -ENOMEM;
-
-	memcpy(&dp_cmd->data, rx_tid, sizeof(*rx_tid));
-	dp_cmd->cmd_num = cmd_num;
-	dp_cmd->handler = cb;
-
-	spin_lock_bh(&dp->reo_cmd_lock);
-	list_add_tail(&dp_cmd->list, &dp->reo_cmd_list);
-	spin_unlock_bh(&dp->reo_cmd_lock);
-
-	return 0;
-}
-
-static int ath12k_dp_reo_cache_flush(struct ath12k_base *ab,
-				     struct ath12k_dp_rx_tid_rxq *rx_tid)
-{
-	struct ath12k_hal_reo_cmd cmd = {};
-	int ret;
-
-	cmd.addr_lo = lower_32_bits(rx_tid->qbuf.paddr_aligned);
-	cmd.addr_hi = upper_32_bits(rx_tid->qbuf.paddr_aligned);
-	/* HAL_REO_CMD_FLG_FLUSH_FWD_ALL_MPDUS - all pending MPDUs
-	 *in the bitmap will be forwarded/flushed to REO output rings
-	 */
-	cmd.flag = HAL_REO_CMD_FLG_NEED_STATUS |
-		   HAL_REO_CMD_FLG_FLUSH_FWD_ALL_MPDUS;
-
-	/* For all QoS TIDs (except NON_QOS), the driver allocates a maximum
-	 * window size of 1024. In such cases, the driver can issue a single
-	 * 1KB descriptor flush command instead of sending multiple 128-byte
-	 * flush commands for each QoS TID, improving efficiency.
-	 */
-
-	if (rx_tid->tid != HAL_DESC_REO_NON_QOS_TID)
-		cmd.flag |= HAL_REO_CMD_FLG_FLUSH_QUEUE_1K_DESC;
-
-	ret = ath12k_dp_reo_cmd_send(ab, rx_tid,
-				     HAL_REO_CMD_FLUSH_CACHE,
-				     &cmd, ath12k_dp_reo_cmd_free);
-	return ret;
-}
-
-static void ath12k_peer_rx_tid_qref_reset(struct ath12k_base *ab, u16 peer_id, u16 tid)
-{
-	struct ath12k_reo_queue_ref *qref;
-	struct ath12k_dp *dp = &ab->dp;
-	bool ml_peer = false;
-
-	if (!ab->hw_params->reoq_lut_support)
-		return;
-
-	if (peer_id & ATH12K_PEER_ML_ID_VALID) {
-		peer_id &= ~ATH12K_PEER_ML_ID_VALID;
-		ml_peer = true;
-	}
-
-	if (ml_peer)
-		qref = (struct ath12k_reo_queue_ref *)dp->ml_reoq_lut.vaddr +
-				(peer_id * (IEEE80211_NUM_TIDS + 1) + tid);
-	else
-		qref = (struct ath12k_reo_queue_ref *)dp->reoq_lut.vaddr +
-				(peer_id * (IEEE80211_NUM_TIDS + 1) + tid);
-
-	qref->info0 = u32_encode_bits(0, BUFFER_ADDR_INFO0_ADDR);
-	qref->info1 = u32_encode_bits(0, BUFFER_ADDR_INFO1_ADDR) |
-		      u32_encode_bits(tid, DP_REO_QREF_NUM);
-}
-
-static void ath12k_dp_rx_process_reo_cmd_update_rx_queue_list(struct ath12k_dp *dp)
+void ath12k_dp_rx_process_reo_cmd_update_rx_queue_list(struct ath12k_dp *dp)
 {
 	struct ath12k_base *ab = dp->ab;
 	struct dp_reo_update_rx_queue_elem *elem, *tmp;
@@ -762,10 +425,10 @@ static void ath12k_dp_rx_process_reo_cmd_update_rx_queue_list(struct ath12k_dp *
 		if (ath12k_dp_rx_tid_delete_handler(ab, &elem->rx_tid))
 			break;
 
-		ath12k_peer_rx_tid_qref_reset(ab,
-					      elem->is_ml_peer ? elem->ml_peer_id :
-					      elem->peer_id,
-					      elem->rx_tid.tid);
+		ath12k_dp_arch_peer_rx_tid_qref_reset(dp,
+						      elem->is_ml_peer ?
+						      elem->ml_peer_id : elem->peer_id,
+						      elem->rx_tid.tid);
 
 		if (ab->hw_params->reoq_lut_support)
 			ath12k_hal_reo_shared_qaddr_cache_clear(ab);
@@ -776,9 +439,10 @@ static void ath12k_dp_rx_process_reo_cmd_update_rx_queue_list(struct ath12k_dp *
 
 	spin_unlock_bh(&dp->reo_rxq_flush_lock);
 }
+EXPORT_SYMBOL(ath12k_dp_rx_process_reo_cmd_update_rx_queue_list);
 
-static void ath12k_dp_rx_tid_del_func(struct ath12k_dp *dp, void *ctx,
-				      enum hal_reo_cmd_status status)
+void ath12k_dp_rx_tid_del_func(struct ath12k_dp *dp, void *ctx,
+			       enum hal_reo_cmd_status status)
 {
 	struct ath12k_base *ab = dp->ab;
 	struct ath12k_dp_rx_tid_rxq *rx_tid = ctx;
@@ -796,9 +460,9 @@ static void ath12k_dp_rx_tid_del_func(struct ath12k_dp *dp, void *ctx,
 	/* Retry the HAL_REO_CMD_UPDATE_RX_QUEUE command for entries
 	 * in the pending queue list marked TID as inactive
 	 */
-	spin_lock_bh(&dp->ab->base_lock);
+	spin_lock_bh(&dp->dp_lock);
 	ath12k_dp_rx_process_reo_cmd_update_rx_queue_list(dp);
-	spin_unlock_bh(&dp->ab->base_lock);
+	spin_unlock_bh(&dp->dp_lock);
 
 	elem = kzalloc(sizeof(*elem), GFP_ATOMIC);
 	if (!elem)
@@ -825,11 +489,12 @@ static void ath12k_dp_rx_tid_del_func(struct ath12k_dp *dp, void *ctx,
 			 * will be called during core destroy.
 			 */
 
-			if (ath12k_dp_reo_cache_flush(ab, &elem->data))
+			if (ath12k_dp_arch_reo_cache_flush(dp, &elem->data))
 				break;
 
 			list_del(&elem->list);
 			dp->reo_cmd_cache_flush_count--;
+
 			kfree(elem);
 		}
 	}
@@ -839,55 +504,17 @@ static void ath12k_dp_rx_tid_del_func(struct ath12k_dp *dp, void *ctx,
 free_desc:
 	ath12k_dp_rx_tid_cleanup(ab, &rx_tid->qbuf);
 }
+EXPORT_SYMBOL(ath12k_dp_rx_tid_del_func);
 
 static int ath12k_dp_rx_tid_delete_handler(struct ath12k_base *ab,
 					   struct ath12k_dp_rx_tid_rxq *rx_tid)
 {
-	struct ath12k_hal_reo_cmd cmd = {};
-
-	cmd.flag = HAL_REO_CMD_FLG_NEED_STATUS;
-	cmd.addr_lo = lower_32_bits(rx_tid->qbuf.paddr_aligned);
-	cmd.addr_hi = upper_32_bits(rx_tid->qbuf.paddr_aligned);
-	cmd.upd0 |= HAL_REO_CMD_UPD0_VLD;
-	/* Observed flush cache failure, to avoid that set vld bit during delete */
-	cmd.upd1 |= HAL_REO_CMD_UPD1_VLD;
-
-	return ath12k_dp_reo_cmd_send(ab, rx_tid,
-				      HAL_REO_CMD_UPDATE_RX_QUEUE, &cmd,
-				      ath12k_dp_rx_tid_del_func);
-}
-
-static void ath12k_peer_rx_tid_qref_setup(struct ath12k_base *ab, u16 peer_id, u16 tid,
-					  dma_addr_t paddr)
-{
-	struct ath12k_reo_queue_ref *qref;
-	struct ath12k_dp *dp = &ab->dp;
-	bool ml_peer = false;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 
-	if (!ab->hw_params->reoq_lut_support)
-		return;
-
-	if (peer_id & ATH12K_PEER_ML_ID_VALID) {
-		peer_id &= ~ATH12K_PEER_ML_ID_VALID;
-		ml_peer = true;
-	}
-
-	if (ml_peer)
-		qref = (struct ath12k_reo_queue_ref *)dp->ml_reoq_lut.vaddr +
-				(peer_id * (IEEE80211_NUM_TIDS + 1) + tid);
-	else
-		qref = (struct ath12k_reo_queue_ref *)dp->reoq_lut.vaddr +
-				(peer_id * (IEEE80211_NUM_TIDS + 1) + tid);
-
-	qref->info0 = u32_encode_bits(lower_32_bits(paddr),
-				      BUFFER_ADDR_INFO0_ADDR);
-	qref->info1 = u32_encode_bits(upper_32_bits(paddr),
-				      BUFFER_ADDR_INFO1_ADDR) |
-		      u32_encode_bits(tid, DP_REO_QREF_NUM);
-	ath12k_hal_reo_shared_qaddr_cache_clear(ab);
+	return ath12k_dp_arch_rx_tid_delete_handler(dp, rx_tid);
 }
 
-static void ath12k_dp_mark_tid_as_inactive(struct ath12k_dp *dp, int peer_id, u8 tid)
+void ath12k_dp_mark_tid_as_inactive(struct ath12k_dp *dp, int peer_id, u8 tid)
 {
 	struct dp_reo_update_rx_queue_elem *elem;
 	struct ath12k_dp_rx_tid_rxq *rx_tid;
@@ -904,192 +531,39 @@ static void ath12k_dp_mark_tid_as_inactive(struct ath12k_dp *dp, int peer_id, u8
 	}
 	spin_unlock_bh(&dp->reo_rxq_flush_lock);
 }
+EXPORT_SYMBOL(ath12k_dp_mark_tid_as_inactive);
 
-void ath12k_dp_rx_peer_tid_delete(struct ath12k *ar,
-				  struct ath12k_peer *peer, u8 tid)
-{
-	struct ath12k_dp_rx_tid *rx_tid = &peer->rx_tid[tid];
-	struct ath12k_base *ab = ar->ab;
-	struct ath12k_dp *dp = &ab->dp;
-
-	if (!rx_tid->active)
-		return;
-
-	rx_tid->active = false;
-
-	ath12k_dp_mark_tid_as_inactive(dp, peer->peer_id, tid);
-	ath12k_dp_rx_process_reo_cmd_update_rx_queue_list(dp);
-}
-
-int ath12k_dp_rx_link_desc_return(struct ath12k_base *ab,
-				  struct ath12k_buffer_addr *buf_addr_info,
-				  enum hal_wbm_rel_bm_act action)
-{
-	struct hal_wbm_release_ring *desc;
-	struct ath12k_dp *dp = &ab->dp;
-	struct hal_srng *srng;
-	int ret = 0;
-
-	srng = &ab->hal.srng_list[dp->wbm_desc_rel_ring.ring_id];
-
-	spin_lock_bh(&srng->lock);
-
-	ath12k_hal_srng_access_begin(ab, srng);
-
-	desc = ath12k_hal_srng_src_get_next_entry(ab, srng);
-	if (!desc) {
-		ret = -ENOBUFS;
-		goto exit;
-	}
-
-	ath12k_hal_rx_msdu_link_desc_set(ab, desc, buf_addr_info, action);
-
-exit:
-	ath12k_hal_srng_access_end(ab, srng);
-
-	spin_unlock_bh(&srng->lock);
-
-	return ret;
-}
-
-static void ath12k_dp_rx_frags_cleanup(struct ath12k_dp_rx_tid *rx_tid,
-				       bool rel_link_desc)
-{
-	struct ath12k_buffer_addr *buf_addr_info;
-	struct ath12k_base *ab = rx_tid->ab;
-
-	lockdep_assert_held(&ab->base_lock);
-
-	if (rx_tid->dst_ring_desc) {
-		if (rel_link_desc) {
-			buf_addr_info = &rx_tid->dst_ring_desc->buf_addr_info;
-			ath12k_dp_rx_link_desc_return(ab, buf_addr_info,
-						      HAL_WBM_REL_BM_ACT_PUT_IN_IDLE);
-		}
-		kfree(rx_tid->dst_ring_desc);
-		rx_tid->dst_ring_desc = NULL;
-	}
-
-	rx_tid->cur_sn = 0;
-	rx_tid->last_frag_no = 0;
-	rx_tid->rx_frag_bitmap = 0;
-	__skb_queue_purge(&rx_tid->rx_frags);
-}
-
-void ath12k_dp_rx_peer_tid_cleanup(struct ath12k *ar, struct ath12k_peer *peer)
+void ath12k_dp_rx_peer_tid_cleanup(struct ath12k *ar, struct ath12k_dp_link_peer *peer)
 {
 	struct ath12k_dp_rx_tid *rx_tid;
 	int i;
+	struct ath12k_base *ab = ar->ab;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 
-	lockdep_assert_held(&ar->ab->base_lock);
+	lockdep_assert_held(&dp->dp_lock);
+
+	if (!peer->primary_link)
+		return;
 
 	for (i = 0; i <= IEEE80211_NUM_TIDS; i++) {
-		rx_tid = &peer->rx_tid[i];
+		rx_tid = &peer->dp_peer->rx_tid[i];
 
-		ath12k_dp_rx_peer_tid_delete(ar, peer, i);
-		ath12k_dp_rx_frags_cleanup(rx_tid, true);
+		ath12k_dp_arch_rx_peer_tid_delete(dp, peer, i);
+		ath12k_dp_arch_rx_frags_cleanup(dp, rx_tid, true);
 
-		spin_unlock_bh(&ar->ab->base_lock);
+		spin_unlock_bh(&dp->dp_lock);
 		timer_delete_sync(&rx_tid->frag_timer);
-		spin_lock_bh(&ar->ab->base_lock);
-	}
-}
-
-static int ath12k_peer_rx_tid_reo_update(struct ath12k *ar,
-					 struct ath12k_peer *peer,
-					 struct ath12k_dp_rx_tid *rx_tid,
-					 u32 ba_win_sz, u16 ssn,
-					 bool update_ssn)
-{
-	struct ath12k_hal_reo_cmd cmd = {};
-	int ret;
-	struct ath12k_dp_rx_tid_rxq rx_tid_rxq;
-
-	ath12k_dp_init_rx_tid_rxq(&rx_tid_rxq, rx_tid);
-
-	cmd.addr_lo = lower_32_bits(rx_tid_rxq.qbuf.paddr_aligned);
-	cmd.addr_hi = upper_32_bits(rx_tid_rxq.qbuf.paddr_aligned);
-	cmd.flag = HAL_REO_CMD_FLG_NEED_STATUS;
-	cmd.upd0 = HAL_REO_CMD_UPD0_BA_WINDOW_SIZE;
-	cmd.ba_window_size = ba_win_sz;
-
-	if (update_ssn) {
-		cmd.upd0 |= HAL_REO_CMD_UPD0_SSN;
-		cmd.upd2 = u32_encode_bits(ssn, HAL_REO_CMD_UPD2_SSN);
-	}
-
-	ret = ath12k_dp_reo_cmd_send(ar->ab, &rx_tid_rxq,
-				     HAL_REO_CMD_UPDATE_RX_QUEUE, &cmd,
-				     NULL);
-	if (ret) {
-		ath12k_warn(ar->ab, "failed to update rx tid queue, tid %d (%d)\n",
-			    rx_tid_rxq.tid, ret);
-		return ret;
-	}
-
-	rx_tid->ba_win_sz = ba_win_sz;
-
-	return 0;
-}
-
-static int ath12k_dp_rx_assign_reoq(struct ath12k_base *ab,
-				    struct ath12k_sta *ahsta,
-				    struct ath12k_dp_rx_tid *rx_tid,
-				    u16 ssn, enum hal_pn_type pn_type)
-{
-	u32 ba_win_sz = rx_tid->ba_win_sz;
-	struct ath12k_reoq_buf *buf;
-	void *vaddr, *vaddr_aligned;
-	dma_addr_t paddr_aligned;
-	u8 tid = rx_tid->tid;
-	u32 hw_desc_sz;
-	int ret;
-
-	buf = &ahsta->reoq_bufs[tid];
-	if (!buf->vaddr) {
-		/* TODO: Optimize the memory allocation for qos tid based on
-		 * the actual BA window size in REO tid update path.
-		 */
-		if (tid == HAL_DESC_REO_NON_QOS_TID)
-			hw_desc_sz = ath12k_hal_reo_qdesc_size(ba_win_sz, tid);
-		else
-			hw_desc_sz = ath12k_hal_reo_qdesc_size(DP_BA_WIN_SZ_MAX, tid);
-
-		vaddr = kzalloc(hw_desc_sz + HAL_LINK_DESC_ALIGN - 1, GFP_ATOMIC);
-		if (!vaddr)
-			return -ENOMEM;
-
-		vaddr_aligned = PTR_ALIGN(vaddr, HAL_LINK_DESC_ALIGN);
-
-		ath12k_hal_reo_qdesc_setup(vaddr_aligned, tid, ba_win_sz,
-					   ssn, pn_type);
-
-		paddr_aligned = dma_map_single(ab->dev, vaddr_aligned, hw_desc_sz,
-					       DMA_BIDIRECTIONAL);
-		ret = dma_mapping_error(ab->dev, paddr_aligned);
-		if (ret) {
-			kfree(vaddr);
-			return ret;
-		}
-
-		buf->vaddr = vaddr;
-		buf->paddr_aligned = paddr_aligned;
-		buf->size = hw_desc_sz;
+		spin_lock_bh(&dp->dp_lock);
 	}
-
-	rx_tid->qbuf = *buf;
-	rx_tid->active = true;
-
-	return 0;
 }
 
 static int ath12k_dp_prepare_reo_update_elem(struct ath12k_dp *dp,
-					     struct ath12k_peer *peer,
+					     struct ath12k_dp_link_peer *peer,
 					     struct ath12k_dp_rx_tid *rx_tid)
 {
 	struct dp_reo_update_rx_queue_elem *elem;
 
-	lockdep_assert_held(&dp->ab->base_lock);
+	lockdep_assert_held(&dp->dp_lock);
 
 	elem = kzalloc(sizeof(*elem), GFP_ATOMIC);
 	if (!elem)
@@ -1099,7 +573,8 @@ static int ath12k_dp_prepare_reo_update_elem(struct ath12k_dp *dp,
 	elem->is_ml_peer = peer->mlo;
 	elem->ml_peer_id = peer->ml_id;
 
-	ath12k_dp_init_rx_tid_rxq(&elem->rx_tid, rx_tid);
+	ath12k_dp_init_rx_tid_rxq(&elem->rx_tid, rx_tid,
+				  (peer->rx_tid_active_bitmask & (1 << rx_tid->tid)));
 
 	spin_lock_bh(&dp->reo_rxq_flush_lock);
 	list_add_tail(&elem->list, &dp->reo_cmd_update_rx_queue_list);
@@ -1113,31 +588,30 @@ int ath12k_dp_rx_peer_tid_setup(struct ath12k *ar, const u8 *peer_mac, int vdev_
 				enum hal_pn_type pn_type)
 {
 	struct ath12k_base *ab = ar->ab;
-	struct ath12k_dp *dp = &ab->dp;
-	struct ath12k_peer *peer;
-	struct ath12k_sta *ahsta;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct ath12k_dp_link_peer *peer;
 	struct ath12k_dp_rx_tid *rx_tid;
 	dma_addr_t paddr_aligned;
 	int ret;
 
-	spin_lock_bh(&ab->base_lock);
+	spin_lock_bh(&dp->dp_lock);
 
-	peer = ath12k_peer_find(ab, vdev_id, peer_mac);
-	if (!peer) {
-		spin_unlock_bh(&ab->base_lock);
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, peer_mac);
+	if (!peer || !peer->dp_peer) {
+		spin_unlock_bh(&dp->dp_lock);
 		ath12k_warn(ab, "failed to find the peer to set up rx tid\n");
 		return -ENOENT;
 	}
 
 	if (ab->hw_params->dp_primary_link_only &&
 	    !peer->primary_link) {
-		spin_unlock_bh(&ab->base_lock);
+		spin_unlock_bh(&dp->dp_lock);
 		return 0;
 	}
 
 	if (ab->hw_params->reoq_lut_support &&
 	    (!dp->reoq_lut.vaddr || !dp->ml_reoq_lut.vaddr)) {
-		spin_unlock_bh(&ab->base_lock);
+		spin_unlock_bh(&dp->dp_lock);
 		ath12k_warn(ab, "reo qref table is not setup\n");
 		return -EINVAL;
 	}
@@ -1145,16 +619,16 @@ int ath12k_dp_rx_peer_tid_setup(struct ath12k *ar, const u8 *peer_mac, int vdev_
 	if (peer->peer_id > DP_MAX_PEER_ID || tid > IEEE80211_NUM_TIDS) {
 		ath12k_warn(ab, "peer id of peer %d or tid %d doesn't allow reoq setup\n",
 			    peer->peer_id, tid);
-		spin_unlock_bh(&ab->base_lock);
+		spin_unlock_bh(&dp->dp_lock);
 		return -EINVAL;
 	}
 
-	rx_tid = &peer->rx_tid[tid];
+	rx_tid = &peer->dp_peer->rx_tid[tid];
 	/* Update the tid queue if it is already setup */
-	if (rx_tid->active) {
-		ret = ath12k_peer_rx_tid_reo_update(ar, peer, rx_tid,
-						    ba_win_sz, ssn, true);
-		spin_unlock_bh(&ab->base_lock);
+	if (peer->rx_tid_active_bitmask & (1 << tid)) {
+		ret = ath12k_dp_arch_peer_rx_tid_reo_update(dp, peer, rx_tid,
+							    ba_win_sz, ssn, true);
+		spin_unlock_bh(&dp->dp_lock);
 		if (ret) {
 			ath12k_warn(ab, "failed to update reo for rx tid %d\n", tid);
 			return ret;
@@ -1180,14 +654,15 @@ int ath12k_dp_rx_peer_tid_setup(struct ath12k *ar, const u8 *peer_mac, int vdev_
 
 	rx_tid->ba_win_sz = ba_win_sz;
 
-	ahsta = ath12k_sta_to_ahsta(peer->sta);
-	ret = ath12k_dp_rx_assign_reoq(ab, ahsta, rx_tid, ssn, pn_type);
+	ret = ath12k_dp_arch_rx_assign_reoq(dp, peer->dp_peer, rx_tid, ssn, pn_type);
 	if (ret) {
-		spin_unlock_bh(&ab->base_lock);
+		spin_unlock_bh(&dp->dp_lock);
 		ath12k_warn(ab, "failed to assign reoq buf for rx tid %u\n", tid);
 		return ret;
 	}
 
+	peer->rx_tid_active_bitmask |= (1 << tid);
+
 	/* Pre-allocate the update_rxq_list for the corresponding tid
 	 * This will be used during the tid delete. The reason we are not
 	 * allocating during tid delete is that, if any alloc fail in update_rxq_list
@@ -1197,7 +672,7 @@ int ath12k_dp_rx_peer_tid_setup(struct ath12k *ar, const u8 *peer_mac, int vdev_
 	if (ret) {
 		ath12k_warn(ab, "failed to alloc update_rxq_list for rx tid %u\n", tid);
 		ath12k_dp_rx_tid_cleanup(ab, &rx_tid->qbuf);
-		spin_unlock_bh(&ab->base_lock);
+		spin_unlock_bh(&dp->dp_lock);
 		return ret;
 	}
 
@@ -1207,15 +682,15 @@ int ath12k_dp_rx_peer_tid_setup(struct ath12k *ar, const u8 *peer_mac, int vdev_
 		 * and tid with qaddr.
 		 */
 		if (peer->mlo)
-			ath12k_peer_rx_tid_qref_setup(ab, peer->ml_id, tid,
-						      paddr_aligned);
+			ath12k_dp_arch_peer_rx_tid_qref_setup(dp, peer->ml_id, tid,
+							      paddr_aligned);
 		else
-			ath12k_peer_rx_tid_qref_setup(ab, peer->peer_id, tid,
-						      paddr_aligned);
+			ath12k_dp_arch_peer_rx_tid_qref_setup(dp, peer->peer_id, tid,
+							      paddr_aligned);
 
-		spin_unlock_bh(&ab->base_lock);
+		spin_unlock_bh(&dp->dp_lock);
 	} else {
-		spin_unlock_bh(&ab->base_lock);
+		spin_unlock_bh(&dp->dp_lock);
 		ret = ath12k_wmi_peer_rx_reorder_queue_setup(ar, vdev_id, peer_mac,
 							     paddr_aligned, tid, 1,
 							     ba_win_sz);
@@ -1257,7 +732,8 @@ int ath12k_dp_rx_ampdu_stop(struct ath12k *ar,
 			    u8 link_id)
 {
 	struct ath12k_base *ab = ar->ab;
-	struct ath12k_peer *peer;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct ath12k_dp_link_peer *peer;
 	struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(params->sta);
 	struct ath12k_link_sta *arsta;
 	int vdev_id;
@@ -1273,24 +749,30 @@ int ath12k_dp_rx_ampdu_stop(struct ath12k *ar,
 
 	vdev_id = arsta->arvif->vdev_id;
 
-	spin_lock_bh(&ab->base_lock);
+	spin_lock_bh(&dp->dp_lock);
 
-	peer = ath12k_peer_find(ab, vdev_id, arsta->addr);
-	if (!peer) {
-		spin_unlock_bh(&ab->base_lock);
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, arsta->addr);
+	if (!peer || !peer->dp_peer) {
+		spin_unlock_bh(&dp->dp_lock);
 		ath12k_warn(ab, "failed to find the peer to stop rx aggregation\n");
 		return -ENOENT;
 	}
 
-	active = peer->rx_tid[params->tid].active;
+	if (ab->hw_params->dp_primary_link_only &&
+	    !peer->primary_link) {
+		spin_unlock_bh(&dp->dp_lock);
+		return 0;
+	}
 
+	active = peer->rx_tid_active_bitmask & (1 << params->tid);
 	if (!active) {
-		spin_unlock_bh(&ab->base_lock);
+		spin_unlock_bh(&dp->dp_lock);
 		return 0;
 	}
 
-	ret = ath12k_peer_rx_tid_reo_update(ar, peer, peer->rx_tid, 1, 0, false);
-	spin_unlock_bh(&ab->base_lock);
+	ret = ath12k_dp_arch_peer_rx_tid_reo_update(dp, peer, peer->dp_peer->rx_tid,
+						    1, 0, false);
+	spin_unlock_bh(&dp->dp_lock);
 	if (ret) {
 		ath12k_warn(ab, "failed to update reo for rx tid %d: %d\n",
 			    params->tid, ret);
@@ -1307,8 +789,9 @@ int ath12k_dp_rx_peer_pn_replay_config(struct ath12k_link_vif *arvif,
 {
 	struct ath12k *ar = arvif->ar;
 	struct ath12k_base *ab = ar->ab;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	struct ath12k_hal_reo_cmd cmd = {};
-	struct ath12k_peer *peer;
+	struct ath12k_dp_link_peer *peer;
 	struct ath12k_dp_rx_tid *rx_tid;
 	struct ath12k_dp_rx_tid_rxq rx_tid_rxq;
 	u8 tid;
@@ -1321,49 +804,29 @@ int ath12k_dp_rx_peer_pn_replay_config(struct ath12k_link_vif *arvif,
 	if (!(key->flags & IEEE80211_KEY_FLAG_PAIRWISE))
 		return 0;
 
-	cmd.flag = HAL_REO_CMD_FLG_NEED_STATUS;
-	cmd.upd0 = HAL_REO_CMD_UPD0_PN |
-		    HAL_REO_CMD_UPD0_PN_SIZE |
-		    HAL_REO_CMD_UPD0_PN_VALID |
-		    HAL_REO_CMD_UPD0_PN_CHECK |
-		    HAL_REO_CMD_UPD0_SVLD;
-
-	switch (key->cipher) {
-	case WLAN_CIPHER_SUITE_TKIP:
-	case WLAN_CIPHER_SUITE_CCMP:
-	case WLAN_CIPHER_SUITE_CCMP_256:
-	case WLAN_CIPHER_SUITE_GCMP:
-	case WLAN_CIPHER_SUITE_GCMP_256:
-		if (key_cmd == SET_KEY) {
-			cmd.upd1 |= HAL_REO_CMD_UPD1_PN_CHECK;
-			cmd.pn_size = 48;
-		}
-		break;
-	default:
-		break;
-	}
+	spin_lock_bh(&dp->dp_lock);
 
-	spin_lock_bh(&ab->base_lock);
-
-	peer = ath12k_peer_find(ab, arvif->vdev_id, peer_addr);
-	if (!peer) {
-		spin_unlock_bh(&ab->base_lock);
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id,
+							 peer_addr);
+	if (!peer || !peer->dp_peer) {
+		spin_unlock_bh(&dp->dp_lock);
 		ath12k_warn(ab, "failed to find the peer %pM to configure pn replay detection\n",
 			    peer_addr);
 		return -ENOENT;
 	}
 
 	for (tid = 0; tid <= IEEE80211_NUM_TIDS; tid++) {
-		rx_tid = &peer->rx_tid[tid];
-		if (!rx_tid->active)
+		if (!(peer->rx_tid_active_bitmask & (1 << tid)))
 			continue;
 
-		ath12k_dp_init_rx_tid_rxq(&rx_tid_rxq, rx_tid);
-		cmd.addr_lo = lower_32_bits(rx_tid_rxq.qbuf.paddr_aligned);
-		cmd.addr_hi = upper_32_bits(rx_tid_rxq.qbuf.paddr_aligned);
-		ret = ath12k_dp_reo_cmd_send(ab, &rx_tid_rxq,
-					     HAL_REO_CMD_UPDATE_RX_QUEUE,
-					     &cmd, NULL);
+		rx_tid = &peer->dp_peer->rx_tid[tid];
+		ath12k_dp_init_rx_tid_rxq(&rx_tid_rxq, rx_tid,
+					  (peer->rx_tid_active_bitmask & (1 << tid)));
+		ath12k_dp_arch_setup_pn_check_reo_cmd(dp, &cmd, rx_tid, key->cipher,
+						      key_cmd);
+		ret = ath12k_dp_arch_reo_cmd_send(dp, &rx_tid_rxq,
+						  HAL_REO_CMD_UPDATE_RX_QUEUE,
+						  &cmd, NULL);
 		if (ret) {
 			ath12k_warn(ab, "failed to configure rx tid %d queue of peer %pM for pn replay detection %d\n",
 				    tid, peer_addr, ret);
@@ -1371,732 +834,14 @@ int ath12k_dp_rx_peer_pn_replay_config(struct ath12k_link_vif *arvif,
 		}
 	}
 
-	spin_unlock_bh(&ab->base_lock);
-
-	return ret;
-}
-
-static int ath12k_get_ppdu_user_index(struct htt_ppdu_stats *ppdu_stats,
-				      u16 peer_id)
-{
-	int i;
-
-	for (i = 0; i < HTT_PPDU_STATS_MAX_USERS - 1; i++) {
-		if (ppdu_stats->user_stats[i].is_valid_peer_id) {
-			if (peer_id == ppdu_stats->user_stats[i].peer_id)
-				return i;
-		} else {
-			return i;
-		}
-	}
-
-	return -EINVAL;
-}
-
-static int ath12k_htt_tlv_ppdu_stats_parse(struct ath12k_base *ab,
-					   u16 tag, u16 len, const void *ptr,
-					   void *data)
-{
-	const struct htt_ppdu_stats_usr_cmpltn_ack_ba_status *ba_status;
-	const struct htt_ppdu_stats_usr_cmpltn_cmn *cmplt_cmn;
-	const struct htt_ppdu_stats_user_rate *user_rate;
-	struct htt_ppdu_stats_info *ppdu_info;
-	struct htt_ppdu_user_stats *user_stats;
-	int cur_user;
-	u16 peer_id;
-
-	ppdu_info = data;
-
-	switch (tag) {
-	case HTT_PPDU_STATS_TAG_COMMON:
-		if (len < sizeof(struct htt_ppdu_stats_common)) {
-			ath12k_warn(ab, "Invalid len %d for the tag 0x%x\n",
-				    len, tag);
-			return -EINVAL;
-		}
-		memcpy(&ppdu_info->ppdu_stats.common, ptr,
-		       sizeof(struct htt_ppdu_stats_common));
-		break;
-	case HTT_PPDU_STATS_TAG_USR_RATE:
-		if (len < sizeof(struct htt_ppdu_stats_user_rate)) {
-			ath12k_warn(ab, "Invalid len %d for the tag 0x%x\n",
-				    len, tag);
-			return -EINVAL;
-		}
-		user_rate = ptr;
-		peer_id = le16_to_cpu(user_rate->sw_peer_id);
-		cur_user = ath12k_get_ppdu_user_index(&ppdu_info->ppdu_stats,
-						      peer_id);
-		if (cur_user < 0)
-			return -EINVAL;
-		user_stats = &ppdu_info->ppdu_stats.user_stats[cur_user];
-		user_stats->peer_id = peer_id;
-		user_stats->is_valid_peer_id = true;
-		memcpy(&user_stats->rate, ptr,
-		       sizeof(struct htt_ppdu_stats_user_rate));
-		user_stats->tlv_flags |= BIT(tag);
-		break;
-	case HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON:
-		if (len < sizeof(struct htt_ppdu_stats_usr_cmpltn_cmn)) {
-			ath12k_warn(ab, "Invalid len %d for the tag 0x%x\n",
-				    len, tag);
-			return -EINVAL;
-		}
-
-		cmplt_cmn = ptr;
-		peer_id = le16_to_cpu(cmplt_cmn->sw_peer_id);
-		cur_user = ath12k_get_ppdu_user_index(&ppdu_info->ppdu_stats,
-						      peer_id);
-		if (cur_user < 0)
-			return -EINVAL;
-		user_stats = &ppdu_info->ppdu_stats.user_stats[cur_user];
-		user_stats->peer_id = peer_id;
-		user_stats->is_valid_peer_id = true;
-		memcpy(&user_stats->cmpltn_cmn, ptr,
-		       sizeof(struct htt_ppdu_stats_usr_cmpltn_cmn));
-		user_stats->tlv_flags |= BIT(tag);
-		break;
-	case HTT_PPDU_STATS_TAG_USR_COMPLTN_ACK_BA_STATUS:
-		if (len <
-		    sizeof(struct htt_ppdu_stats_usr_cmpltn_ack_ba_status)) {
-			ath12k_warn(ab, "Invalid len %d for the tag 0x%x\n",
-				    len, tag);
-			return -EINVAL;
-		}
-
-		ba_status = ptr;
-		peer_id = le16_to_cpu(ba_status->sw_peer_id);
-		cur_user = ath12k_get_ppdu_user_index(&ppdu_info->ppdu_stats,
-						      peer_id);
-		if (cur_user < 0)
-			return -EINVAL;
-		user_stats = &ppdu_info->ppdu_stats.user_stats[cur_user];
-		user_stats->peer_id = peer_id;
-		user_stats->is_valid_peer_id = true;
-		memcpy(&user_stats->ack_ba, ptr,
-		       sizeof(struct htt_ppdu_stats_usr_cmpltn_ack_ba_status));
-		user_stats->tlv_flags |= BIT(tag);
-		break;
-	}
-	return 0;
-}
-
-int ath12k_dp_htt_tlv_iter(struct ath12k_base *ab, const void *ptr, size_t len,
-			   int (*iter)(struct ath12k_base *ar, u16 tag, u16 len,
-				       const void *ptr, void *data),
-			   void *data)
-{
-	const struct htt_tlv *tlv;
-	const void *begin = ptr;
-	u16 tlv_tag, tlv_len;
-	int ret = -EINVAL;
-
-	while (len > 0) {
-		if (len < sizeof(*tlv)) {
-			ath12k_err(ab, "htt tlv parse failure at byte %zd (%zu bytes left, %zu expected)\n",
-				   ptr - begin, len, sizeof(*tlv));
-			return -EINVAL;
-		}
-		tlv = (struct htt_tlv *)ptr;
-		tlv_tag = le32_get_bits(tlv->header, HTT_TLV_TAG);
-		tlv_len = le32_get_bits(tlv->header, HTT_TLV_LEN);
-		ptr += sizeof(*tlv);
-		len -= sizeof(*tlv);
-
-		if (tlv_len > len) {
-			ath12k_err(ab, "htt tlv parse failure of tag %u at byte %zd (%zu bytes left, %u expected)\n",
-				   tlv_tag, ptr - begin, len, tlv_len);
-			return -EINVAL;
-		}
-		ret = iter(ab, tlv_tag, tlv_len, ptr, data);
-		if (ret == -ENOMEM)
-			return ret;
-
-		ptr += tlv_len;
-		len -= tlv_len;
-	}
-	return 0;
-}
-
-static void
-ath12k_update_per_peer_tx_stats(struct ath12k *ar,
-				struct htt_ppdu_stats *ppdu_stats, u8 user)
-{
-	struct ath12k_base *ab = ar->ab;
-	struct ath12k_peer *peer;
-	struct ath12k_link_sta *arsta;
-	struct htt_ppdu_stats_user_rate *user_rate;
-	struct ath12k_per_peer_tx_stats *peer_stats = &ar->peer_tx_stats;
-	struct htt_ppdu_user_stats *usr_stats = &ppdu_stats->user_stats[user];
-	struct htt_ppdu_stats_common *common = &ppdu_stats->common;
-	int ret;
-	u8 flags, mcs, nss, bw, sgi, dcm, ppdu_type, rate_idx = 0;
-	u32 v, succ_bytes = 0;
-	u16 tones, rate = 0, succ_pkts = 0;
-	u32 tx_duration = 0;
-	u8 tid = HTT_PPDU_STATS_NON_QOS_TID;
-	u16 tx_retry_failed = 0, tx_retry_count = 0;
-	bool is_ampdu = false, is_ofdma;
-
-	if (!(usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_RATE)))
-		return;
-
-	if (usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON)) {
-		is_ampdu =
-			HTT_USR_CMPLTN_IS_AMPDU(usr_stats->cmpltn_cmn.flags);
-		tx_retry_failed =
-			__le16_to_cpu(usr_stats->cmpltn_cmn.mpdu_tried) -
-			__le16_to_cpu(usr_stats->cmpltn_cmn.mpdu_success);
-		tx_retry_count =
-			HTT_USR_CMPLTN_LONG_RETRY(usr_stats->cmpltn_cmn.flags) +
-			HTT_USR_CMPLTN_SHORT_RETRY(usr_stats->cmpltn_cmn.flags);
-	}
-
-	if (usr_stats->tlv_flags &
-	    BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_ACK_BA_STATUS)) {
-		succ_bytes = le32_to_cpu(usr_stats->ack_ba.success_bytes);
-		succ_pkts = le32_get_bits(usr_stats->ack_ba.info,
-					  HTT_PPDU_STATS_ACK_BA_INFO_NUM_MSDU_M);
-		tid = le32_get_bits(usr_stats->ack_ba.info,
-				    HTT_PPDU_STATS_ACK_BA_INFO_TID_NUM);
-	}
-
-	if (common->fes_duration_us)
-		tx_duration = le32_to_cpu(common->fes_duration_us);
-
-	user_rate = &usr_stats->rate;
-	flags = HTT_USR_RATE_PREAMBLE(user_rate->rate_flags);
-	bw = HTT_USR_RATE_BW(user_rate->rate_flags) - 2;
-	nss = HTT_USR_RATE_NSS(user_rate->rate_flags) + 1;
-	mcs = HTT_USR_RATE_MCS(user_rate->rate_flags);
-	sgi = HTT_USR_RATE_GI(user_rate->rate_flags);
-	dcm = HTT_USR_RATE_DCM(user_rate->rate_flags);
-
-	ppdu_type = HTT_USR_RATE_PPDU_TYPE(user_rate->info1);
-	is_ofdma = (ppdu_type == HTT_PPDU_STATS_PPDU_TYPE_MU_OFDMA) ||
-		   (ppdu_type == HTT_PPDU_STATS_PPDU_TYPE_MU_MIMO_OFDMA);
-
-	/* Note: If host configured fixed rates and in some other special
-	 * cases, the broadcast/management frames are sent in different rates.
-	 * Firmware rate's control to be skipped for this?
-	 */
-
-	if (flags == WMI_RATE_PREAMBLE_HE && mcs > ATH12K_HE_MCS_MAX) {
-		ath12k_warn(ab, "Invalid HE mcs %d peer stats",  mcs);
-		return;
-	}
-
-	if (flags == WMI_RATE_PREAMBLE_VHT && mcs > ATH12K_VHT_MCS_MAX) {
-		ath12k_warn(ab, "Invalid VHT mcs %d peer stats",  mcs);
-		return;
-	}
-
-	if (flags == WMI_RATE_PREAMBLE_HT && (mcs > ATH12K_HT_MCS_MAX || nss < 1)) {
-		ath12k_warn(ab, "Invalid HT mcs %d nss %d peer stats",
-			    mcs, nss);
-		return;
-	}
-
-	if (flags == WMI_RATE_PREAMBLE_CCK || flags == WMI_RATE_PREAMBLE_OFDM) {
-		ret = ath12k_mac_hw_ratecode_to_legacy_rate(mcs,
-							    flags,
-							    &rate_idx,
-							    &rate);
-		if (ret < 0)
-			return;
-	}
-
-	rcu_read_lock();
-	spin_lock_bh(&ab->base_lock);
-	peer = ath12k_peer_find_by_id(ab, usr_stats->peer_id);
-
-	if (!peer || !peer->sta) {
-		spin_unlock_bh(&ab->base_lock);
-		rcu_read_unlock();
-		return;
-	}
-
-	arsta = ath12k_peer_get_link_sta(ab, peer);
-	if (!arsta) {
-		spin_unlock_bh(&ab->base_lock);
-		rcu_read_unlock();
-		return;
-	}
-
-	memset(&arsta->txrate, 0, sizeof(arsta->txrate));
-
-	arsta->txrate.bw = ath12k_mac_bw_to_mac80211_bw(bw);
-
-	switch (flags) {
-	case WMI_RATE_PREAMBLE_OFDM:
-		arsta->txrate.legacy = rate;
-		break;
-	case WMI_RATE_PREAMBLE_CCK:
-		arsta->txrate.legacy = rate;
-		break;
-	case WMI_RATE_PREAMBLE_HT:
-		arsta->txrate.mcs = mcs + 8 * (nss - 1);
-		arsta->txrate.flags = RATE_INFO_FLAGS_MCS;
-		if (sgi)
-			arsta->txrate.flags |= RATE_INFO_FLAGS_SHORT_GI;
-		break;
-	case WMI_RATE_PREAMBLE_VHT:
-		arsta->txrate.mcs = mcs;
-		arsta->txrate.flags = RATE_INFO_FLAGS_VHT_MCS;
-		if (sgi)
-			arsta->txrate.flags |= RATE_INFO_FLAGS_SHORT_GI;
-		break;
-	case WMI_RATE_PREAMBLE_HE:
-		arsta->txrate.mcs = mcs;
-		arsta->txrate.flags = RATE_INFO_FLAGS_HE_MCS;
-		arsta->txrate.he_dcm = dcm;
-		arsta->txrate.he_gi = ath12k_he_gi_to_nl80211_he_gi(sgi);
-		tones = le16_to_cpu(user_rate->ru_end) -
-			le16_to_cpu(user_rate->ru_start) + 1;
-		v = ath12k_he_ru_tones_to_nl80211_he_ru_alloc(tones);
-		arsta->txrate.he_ru_alloc = v;
-		if (is_ofdma)
-			arsta->txrate.bw = RATE_INFO_BW_HE_RU;
-		break;
-	case WMI_RATE_PREAMBLE_EHT:
-		arsta->txrate.mcs = mcs;
-		arsta->txrate.flags = RATE_INFO_FLAGS_EHT_MCS;
-		arsta->txrate.he_dcm = dcm;
-		arsta->txrate.eht_gi = ath12k_mac_eht_gi_to_nl80211_eht_gi(sgi);
-		tones = le16_to_cpu(user_rate->ru_end) -
-			le16_to_cpu(user_rate->ru_start) + 1;
-		v = ath12k_mac_eht_ru_tones_to_nl80211_eht_ru_alloc(tones);
-		arsta->txrate.eht_ru_alloc = v;
-		if (is_ofdma)
-			arsta->txrate.bw = RATE_INFO_BW_EHT_RU;
-		break;
-	}
-
-	arsta->tx_retry_failed += tx_retry_failed;
-	arsta->tx_retry_count += tx_retry_count;
-	arsta->txrate.nss = nss;
-	arsta->tx_duration += tx_duration;
-	memcpy(&arsta->last_txrate, &arsta->txrate, sizeof(struct rate_info));
-
-	/* PPDU stats reported for mgmt packet doesn't have valid tx bytes.
-	 * So skip peer stats update for mgmt packets.
-	 */
-	if (tid < HTT_PPDU_STATS_NON_QOS_TID) {
-		memset(peer_stats, 0, sizeof(*peer_stats));
-		peer_stats->succ_pkts = succ_pkts;
-		peer_stats->succ_bytes = succ_bytes;
-		peer_stats->is_ampdu = is_ampdu;
-		peer_stats->duration = tx_duration;
-		peer_stats->ba_fails =
-			HTT_USR_CMPLTN_LONG_RETRY(usr_stats->cmpltn_cmn.flags) +
-			HTT_USR_CMPLTN_SHORT_RETRY(usr_stats->cmpltn_cmn.flags);
-	}
-
-	spin_unlock_bh(&ab->base_lock);
-	rcu_read_unlock();
-}
-
-static void ath12k_htt_update_ppdu_stats(struct ath12k *ar,
-					 struct htt_ppdu_stats *ppdu_stats)
-{
-	u8 user;
-
-	for (user = 0; user < HTT_PPDU_STATS_MAX_USERS - 1; user++)
-		ath12k_update_per_peer_tx_stats(ar, ppdu_stats, user);
-}
-
-static
-struct htt_ppdu_stats_info *ath12k_dp_htt_get_ppdu_desc(struct ath12k *ar,
-							u32 ppdu_id)
-{
-	struct htt_ppdu_stats_info *ppdu_info;
-
-	lockdep_assert_held(&ar->data_lock);
-	if (!list_empty(&ar->ppdu_stats_info)) {
-		list_for_each_entry(ppdu_info, &ar->ppdu_stats_info, list) {
-			if (ppdu_info->ppdu_id == ppdu_id)
-				return ppdu_info;
-		}
-
-		if (ar->ppdu_stat_list_depth > HTT_PPDU_DESC_MAX_DEPTH) {
-			ppdu_info = list_first_entry(&ar->ppdu_stats_info,
-						     typeof(*ppdu_info), list);
-			list_del(&ppdu_info->list);
-			ar->ppdu_stat_list_depth--;
-			ath12k_htt_update_ppdu_stats(ar, &ppdu_info->ppdu_stats);
-			kfree(ppdu_info);
-		}
-	}
-
-	ppdu_info = kzalloc(sizeof(*ppdu_info), GFP_ATOMIC);
-	if (!ppdu_info)
-		return NULL;
-
-	list_add_tail(&ppdu_info->list, &ar->ppdu_stats_info);
-	ar->ppdu_stat_list_depth++;
-
-	return ppdu_info;
-}
-
-static void ath12k_copy_to_delay_stats(struct ath12k_peer *peer,
-				       struct htt_ppdu_user_stats *usr_stats)
-{
-	peer->ppdu_stats_delayba.sw_peer_id = le16_to_cpu(usr_stats->rate.sw_peer_id);
-	peer->ppdu_stats_delayba.info0 = le32_to_cpu(usr_stats->rate.info0);
-	peer->ppdu_stats_delayba.ru_end = le16_to_cpu(usr_stats->rate.ru_end);
-	peer->ppdu_stats_delayba.ru_start = le16_to_cpu(usr_stats->rate.ru_start);
-	peer->ppdu_stats_delayba.info1 = le32_to_cpu(usr_stats->rate.info1);
-	peer->ppdu_stats_delayba.rate_flags = le32_to_cpu(usr_stats->rate.rate_flags);
-	peer->ppdu_stats_delayba.resp_rate_flags =
-		le32_to_cpu(usr_stats->rate.resp_rate_flags);
-
-	peer->delayba_flag = true;
-}
-
-static void ath12k_copy_to_bar(struct ath12k_peer *peer,
-			       struct htt_ppdu_user_stats *usr_stats)
-{
-	usr_stats->rate.sw_peer_id = cpu_to_le16(peer->ppdu_stats_delayba.sw_peer_id);
-	usr_stats->rate.info0 = cpu_to_le32(peer->ppdu_stats_delayba.info0);
-	usr_stats->rate.ru_end = cpu_to_le16(peer->ppdu_stats_delayba.ru_end);
-	usr_stats->rate.ru_start = cpu_to_le16(peer->ppdu_stats_delayba.ru_start);
-	usr_stats->rate.info1 = cpu_to_le32(peer->ppdu_stats_delayba.info1);
-	usr_stats->rate.rate_flags = cpu_to_le32(peer->ppdu_stats_delayba.rate_flags);
-	usr_stats->rate.resp_rate_flags =
-		cpu_to_le32(peer->ppdu_stats_delayba.resp_rate_flags);
-
-	peer->delayba_flag = false;
-}
-
-static int ath12k_htt_pull_ppdu_stats(struct ath12k_base *ab,
-				      struct sk_buff *skb)
-{
-	struct ath12k_htt_ppdu_stats_msg *msg;
-	struct htt_ppdu_stats_info *ppdu_info;
-	struct ath12k_peer *peer = NULL;
-	struct htt_ppdu_user_stats *usr_stats = NULL;
-	u32 peer_id = 0;
-	struct ath12k *ar;
-	int ret, i;
-	u8 pdev_id;
-	u32 ppdu_id, len;
-
-	msg = (struct ath12k_htt_ppdu_stats_msg *)skb->data;
-	len = le32_get_bits(msg->info, HTT_T2H_PPDU_STATS_INFO_PAYLOAD_SIZE);
-	if (len > (skb->len - struct_size(msg, data, 0))) {
-		ath12k_warn(ab,
-			    "HTT PPDU STATS event has unexpected payload size %u, should be smaller than %u\n",
-			    len, skb->len);
-		return -EINVAL;
-	}
-
-	pdev_id = le32_get_bits(msg->info, HTT_T2H_PPDU_STATS_INFO_PDEV_ID);
-	ppdu_id = le32_to_cpu(msg->ppdu_id);
-
-	rcu_read_lock();
-	ar = ath12k_mac_get_ar_by_pdev_id(ab, pdev_id);
-	if (!ar) {
-		ret = -EINVAL;
-		goto exit;
-	}
-
-	spin_lock_bh(&ar->data_lock);
-	ppdu_info = ath12k_dp_htt_get_ppdu_desc(ar, ppdu_id);
-	if (!ppdu_info) {
-		spin_unlock_bh(&ar->data_lock);
-		ret = -EINVAL;
-		goto exit;
-	}
-
-	ppdu_info->ppdu_id = ppdu_id;
-	ret = ath12k_dp_htt_tlv_iter(ab, msg->data, len,
-				     ath12k_htt_tlv_ppdu_stats_parse,
-				     (void *)ppdu_info);
-	if (ret) {
-		spin_unlock_bh(&ar->data_lock);
-		ath12k_warn(ab, "Failed to parse tlv %d\n", ret);
-		goto exit;
-	}
-
-	if (ppdu_info->ppdu_stats.common.num_users >= HTT_PPDU_STATS_MAX_USERS) {
-		spin_unlock_bh(&ar->data_lock);
-		ath12k_warn(ab,
-			    "HTT PPDU STATS event has unexpected num_users %u, should be smaller than %u\n",
-			    ppdu_info->ppdu_stats.common.num_users,
-			    HTT_PPDU_STATS_MAX_USERS);
-		ret = -EINVAL;
-		goto exit;
-	}
-
-	/* back up data rate tlv for all peers */
-	if (ppdu_info->frame_type == HTT_STATS_PPDU_FTYPE_DATA &&
-	    (ppdu_info->tlv_bitmap & (1 << HTT_PPDU_STATS_TAG_USR_COMMON)) &&
-	    ppdu_info->delay_ba) {
-		for (i = 0; i < ppdu_info->ppdu_stats.common.num_users; i++) {
-			peer_id = ppdu_info->ppdu_stats.user_stats[i].peer_id;
-			spin_lock_bh(&ab->base_lock);
-			peer = ath12k_peer_find_by_id(ab, peer_id);
-			if (!peer) {
-				spin_unlock_bh(&ab->base_lock);
-				continue;
-			}
-
-			usr_stats = &ppdu_info->ppdu_stats.user_stats[i];
-			if (usr_stats->delay_ba)
-				ath12k_copy_to_delay_stats(peer, usr_stats);
-			spin_unlock_bh(&ab->base_lock);
-		}
-	}
-
-	/* restore all peers' data rate tlv to mu-bar tlv */
-	if (ppdu_info->frame_type == HTT_STATS_PPDU_FTYPE_BAR &&
-	    (ppdu_info->tlv_bitmap & (1 << HTT_PPDU_STATS_TAG_USR_COMMON))) {
-		for (i = 0; i < ppdu_info->bar_num_users; i++) {
-			peer_id = ppdu_info->ppdu_stats.user_stats[i].peer_id;
-			spin_lock_bh(&ab->base_lock);
-			peer = ath12k_peer_find_by_id(ab, peer_id);
-			if (!peer) {
-				spin_unlock_bh(&ab->base_lock);
-				continue;
-			}
-
-			usr_stats = &ppdu_info->ppdu_stats.user_stats[i];
-			if (peer->delayba_flag)
-				ath12k_copy_to_bar(peer, usr_stats);
-			spin_unlock_bh(&ab->base_lock);
-		}
-	}
-
-	spin_unlock_bh(&ar->data_lock);
-
-exit:
-	rcu_read_unlock();
+	spin_unlock_bh(&dp->dp_lock);
 
 	return ret;
 }
+EXPORT_SYMBOL(ath12k_dp_rx_get_msdu_last_buf);
 
-static void ath12k_htt_mlo_offset_event_handler(struct ath12k_base *ab,
-						struct sk_buff *skb)
-{
-	struct ath12k_htt_mlo_offset_msg *msg;
-	struct ath12k_pdev *pdev;
-	struct ath12k *ar;
-	u8 pdev_id;
-
-	msg = (struct ath12k_htt_mlo_offset_msg *)skb->data;
-	pdev_id = u32_get_bits(__le32_to_cpu(msg->info),
-			       HTT_T2H_MLO_OFFSET_INFO_PDEV_ID);
-
-	rcu_read_lock();
-	ar = ath12k_mac_get_ar_by_pdev_id(ab, pdev_id);
-	if (!ar) {
-		/* It is possible that the ar is not yet active (started).
-		 * The above function will only look for the active pdev
-		 * and hence %NULL return is possible. Just silently
-		 * discard this message
-		 */
-		goto exit;
-	}
-
-	spin_lock_bh(&ar->data_lock);
-	pdev = ar->pdev;
-
-	pdev->timestamp.info = __le32_to_cpu(msg->info);
-	pdev->timestamp.sync_timestamp_lo_us = __le32_to_cpu(msg->sync_timestamp_lo_us);
-	pdev->timestamp.sync_timestamp_hi_us = __le32_to_cpu(msg->sync_timestamp_hi_us);
-	pdev->timestamp.mlo_offset_lo = __le32_to_cpu(msg->mlo_offset_lo);
-	pdev->timestamp.mlo_offset_hi = __le32_to_cpu(msg->mlo_offset_hi);
-	pdev->timestamp.mlo_offset_clks = __le32_to_cpu(msg->mlo_offset_clks);
-	pdev->timestamp.mlo_comp_clks = __le32_to_cpu(msg->mlo_comp_clks);
-	pdev->timestamp.mlo_comp_timer = __le32_to_cpu(msg->mlo_comp_timer);
-
-	spin_unlock_bh(&ar->data_lock);
-exit:
-	rcu_read_unlock();
-}
-
-void ath12k_dp_htt_htc_t2h_msg_handler(struct ath12k_base *ab,
-				       struct sk_buff *skb)
-{
-	struct ath12k_dp *dp = &ab->dp;
-	struct htt_resp_msg *resp = (struct htt_resp_msg *)skb->data;
-	enum htt_t2h_msg_type type;
-	u16 peer_id;
-	u8 vdev_id;
-	u8 mac_addr[ETH_ALEN];
-	u16 peer_mac_h16;
-	u16 ast_hash = 0;
-	u16 hw_peer_id;
-
-	type = le32_get_bits(resp->version_msg.version, HTT_T2H_MSG_TYPE);
-
-	ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt rx msg type :0x%0x\n", type);
-
-	switch (type) {
-	case HTT_T2H_MSG_TYPE_VERSION_CONF:
-		dp->htt_tgt_ver_major = le32_get_bits(resp->version_msg.version,
-						      HTT_T2H_VERSION_CONF_MAJOR);
-		dp->htt_tgt_ver_minor = le32_get_bits(resp->version_msg.version,
-						      HTT_T2H_VERSION_CONF_MINOR);
-		complete(&dp->htt_tgt_version_received);
-		break;
-	/* TODO: remove unused peer map versions after testing */
-	case HTT_T2H_MSG_TYPE_PEER_MAP:
-		vdev_id = le32_get_bits(resp->peer_map_ev.info,
-					HTT_T2H_PEER_MAP_INFO_VDEV_ID);
-		peer_id = le32_get_bits(resp->peer_map_ev.info,
-					HTT_T2H_PEER_MAP_INFO_PEER_ID);
-		peer_mac_h16 = le32_get_bits(resp->peer_map_ev.info1,
-					     HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16);
-		ath12k_dp_get_mac_addr(le32_to_cpu(resp->peer_map_ev.mac_addr_l32),
-				       peer_mac_h16, mac_addr);
-		ath12k_peer_map_event(ab, vdev_id, peer_id, mac_addr, 0, 0);
-		break;
-	case HTT_T2H_MSG_TYPE_PEER_MAP2:
-		vdev_id = le32_get_bits(resp->peer_map_ev.info,
-					HTT_T2H_PEER_MAP_INFO_VDEV_ID);
-		peer_id = le32_get_bits(resp->peer_map_ev.info,
-					HTT_T2H_PEER_MAP_INFO_PEER_ID);
-		peer_mac_h16 = le32_get_bits(resp->peer_map_ev.info1,
-					     HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16);
-		ath12k_dp_get_mac_addr(le32_to_cpu(resp->peer_map_ev.mac_addr_l32),
-				       peer_mac_h16, mac_addr);
-		ast_hash = le32_get_bits(resp->peer_map_ev.info2,
-					 HTT_T2H_PEER_MAP_INFO2_AST_HASH_VAL);
-		hw_peer_id = le32_get_bits(resp->peer_map_ev.info1,
-					   HTT_T2H_PEER_MAP_INFO1_HW_PEER_ID);
-		ath12k_peer_map_event(ab, vdev_id, peer_id, mac_addr, ast_hash,
-				      hw_peer_id);
-		break;
-	case HTT_T2H_MSG_TYPE_PEER_MAP3:
-		vdev_id = le32_get_bits(resp->peer_map_ev.info,
-					HTT_T2H_PEER_MAP_INFO_VDEV_ID);
-		peer_id = le32_get_bits(resp->peer_map_ev.info,
-					HTT_T2H_PEER_MAP_INFO_PEER_ID);
-		peer_mac_h16 = le32_get_bits(resp->peer_map_ev.info1,
-					     HTT_T2H_PEER_MAP_INFO1_MAC_ADDR_H16);
-		ath12k_dp_get_mac_addr(le32_to_cpu(resp->peer_map_ev.mac_addr_l32),
-				       peer_mac_h16, mac_addr);
-		ast_hash = le32_get_bits(resp->peer_map_ev.info2,
-					 HTT_T2H_PEER_MAP3_INFO2_AST_HASH_VAL);
-		hw_peer_id = le32_get_bits(resp->peer_map_ev.info2,
-					   HTT_T2H_PEER_MAP3_INFO2_HW_PEER_ID);
-		ath12k_peer_map_event(ab, vdev_id, peer_id, mac_addr, ast_hash,
-				      hw_peer_id);
-		break;
-	case HTT_T2H_MSG_TYPE_PEER_UNMAP:
-	case HTT_T2H_MSG_TYPE_PEER_UNMAP2:
-		peer_id = le32_get_bits(resp->peer_unmap_ev.info,
-					HTT_T2H_PEER_UNMAP_INFO_PEER_ID);
-		ath12k_peer_unmap_event(ab, peer_id);
-		break;
-	case HTT_T2H_MSG_TYPE_PPDU_STATS_IND:
-		ath12k_htt_pull_ppdu_stats(ab, skb);
-		break;
-	case HTT_T2H_MSG_TYPE_EXT_STATS_CONF:
-		ath12k_debugfs_htt_ext_stats_handler(ab, skb);
-		break;
-	case HTT_T2H_MSG_TYPE_MLO_TIMESTAMP_OFFSET_IND:
-		ath12k_htt_mlo_offset_event_handler(ab, skb);
-		break;
-	default:
-		ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt event %d not handled\n",
-			   type);
-		break;
-	}
-
-	dev_kfree_skb_any(skb);
-}
-
-static int ath12k_dp_rx_msdu_coalesce(struct ath12k *ar,
-				      struct sk_buff_head *msdu_list,
-				      struct sk_buff *first, struct sk_buff *last,
-				      u8 l3pad_bytes, int msdu_len)
-{
-	struct ath12k_base *ab = ar->ab;
-	struct sk_buff *skb;
-	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(first);
-	int buf_first_hdr_len, buf_first_len;
-	struct hal_rx_desc *ldesc;
-	int space_extra, rem_len, buf_len;
-	u32 hal_rx_desc_sz = ar->ab->hal.hal_desc_sz;
-	bool is_continuation;
-
-	/* As the msdu is spread across multiple rx buffers,
-	 * find the offset to the start of msdu for computing
-	 * the length of the msdu in the first buffer.
-	 */
-	buf_first_hdr_len = hal_rx_desc_sz + l3pad_bytes;
-	buf_first_len = DP_RX_BUFFER_SIZE - buf_first_hdr_len;
-
-	if (WARN_ON_ONCE(msdu_len <= buf_first_len)) {
-		skb_put(first, buf_first_hdr_len + msdu_len);
-		skb_pull(first, buf_first_hdr_len);
-		return 0;
-	}
-
-	ldesc = (struct hal_rx_desc *)last->data;
-	rxcb->is_first_msdu = ath12k_dp_rx_h_first_msdu(ab, ldesc);
-	rxcb->is_last_msdu = ath12k_dp_rx_h_last_msdu(ab, ldesc);
-
-	/* MSDU spans over multiple buffers because the length of the MSDU
-	 * exceeds DP_RX_BUFFER_SIZE - HAL_RX_DESC_SIZE. So assume the data
-	 * in the first buf is of length DP_RX_BUFFER_SIZE - HAL_RX_DESC_SIZE.
-	 */
-	skb_put(first, DP_RX_BUFFER_SIZE);
-	skb_pull(first, buf_first_hdr_len);
-
-	/* When an MSDU spread over multiple buffers MSDU_END
-	 * tlvs are valid only in the last buffer. Copy those tlvs.
-	 */
-	ath12k_dp_rx_desc_end_tlv_copy(ab, rxcb->rx_desc, ldesc);
-
-	space_extra = msdu_len - (buf_first_len + skb_tailroom(first));
-	if (space_extra > 0 &&
-	    (pskb_expand_head(first, 0, space_extra, GFP_ATOMIC) < 0)) {
-		/* Free up all buffers of the MSDU */
-		while ((skb = __skb_dequeue(msdu_list)) != NULL) {
-			rxcb = ATH12K_SKB_RXCB(skb);
-			if (!rxcb->is_continuation) {
-				dev_kfree_skb_any(skb);
-				break;
-			}
-			dev_kfree_skb_any(skb);
-		}
-		return -ENOMEM;
-	}
-
-	rem_len = msdu_len - buf_first_len;
-	while ((skb = __skb_dequeue(msdu_list)) != NULL && rem_len > 0) {
-		rxcb = ATH12K_SKB_RXCB(skb);
-		is_continuation = rxcb->is_continuation;
-		if (is_continuation)
-			buf_len = DP_RX_BUFFER_SIZE - hal_rx_desc_sz;
-		else
-			buf_len = rem_len;
-
-		if (buf_len > (DP_RX_BUFFER_SIZE - hal_rx_desc_sz)) {
-			WARN_ON_ONCE(1);
-			dev_kfree_skb_any(skb);
-			return -EINVAL;
-		}
-
-		skb_put(skb, buf_len + hal_rx_desc_sz);
-		skb_pull(skb, hal_rx_desc_sz);
-		skb_copy_from_linear_data(skb, skb_put(first, buf_len),
-					  buf_len);
-		dev_kfree_skb_any(skb);
-
-		rem_len -= buf_len;
-		if (!is_continuation)
-			break;
-	}
-
-	return 0;
-}
-
-static struct sk_buff *ath12k_dp_rx_get_msdu_last_buf(struct sk_buff_head *msdu_list,
-						      struct sk_buff *first)
+struct sk_buff *ath12k_dp_rx_get_msdu_last_buf(struct sk_buff_head *msdu_list,
+					       struct sk_buff *first)
 {
 	struct sk_buff *skb;
 	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(first);
@@ -2113,14 +858,7 @@ static struct sk_buff *ath12k_dp_rx_get_msdu_last_buf(struct sk_buff_head *msdu_
 	return NULL;
 }
 
-static void ath12k_dp_rx_h_csum_offload(struct sk_buff *msdu,
-					struct ath12k_dp_rx_info *rx_info)
-{
-	msdu->ip_summed = (rx_info->ip_csum_fail || rx_info->l4_csum_fail) ?
-			   CHECKSUM_NONE : CHECKSUM_UNNECESSARY;
-}
-
-int ath12k_dp_rx_crypto_mic_len(struct ath12k *ar, enum hal_encrypt_type enctype)
+int ath12k_dp_rx_crypto_mic_len(struct ath12k_dp *dp, enum hal_encrypt_type enctype)
 {
 	switch (enctype) {
 	case HAL_ENCRYPT_TYPE_OPEN:
@@ -2142,11 +880,11 @@ int ath12k_dp_rx_crypto_mic_len(struct ath12k *ar, enum hal_encrypt_type enctype
 		break;
 	}
 
-	ath12k_warn(ar->ab, "unsupported encryption type %d for mic len\n", enctype);
+	ath12k_warn(dp->ab, "unsupported encryption type %d for mic len\n", enctype);
 	return 0;
 }
 
-static int ath12k_dp_rx_crypto_param_len(struct ath12k *ar,
+static int ath12k_dp_rx_crypto_param_len(struct ath12k_pdev_dp *dp_pdev,
 					 enum hal_encrypt_type enctype)
 {
 	switch (enctype) {
@@ -2170,11 +908,11 @@ static int ath12k_dp_rx_crypto_param_len(struct ath12k *ar,
 		break;
 	}
 
-	ath12k_warn(ar->ab, "unsupported encryption type %d\n", enctype);
+	ath12k_warn(dp_pdev->dp->ab, "unsupported encryption type %d\n", enctype);
 	return 0;
 }
 
-static int ath12k_dp_rx_crypto_icv_len(struct ath12k *ar,
+static int ath12k_dp_rx_crypto_icv_len(struct ath12k_pdev_dp *dp_pdev,
 				       enum hal_encrypt_type enctype)
 {
 	switch (enctype) {
@@ -2195,16 +933,15 @@ static int ath12k_dp_rx_crypto_icv_len(struct ath12k *ar,
 		break;
 	}
 
-	ath12k_warn(ar->ab, "unsupported encryption type %d\n", enctype);
+	ath12k_warn(dp_pdev->dp->ab, "unsupported encryption type %d\n", enctype);
 	return 0;
 }
 
-static void ath12k_dp_rx_h_undecap_nwifi(struct ath12k *ar,
+static void ath12k_dp_rx_h_undecap_nwifi(struct ath12k_pdev_dp *dp_pdev,
 					 struct sk_buff *msdu,
 					 enum hal_encrypt_type enctype,
-					 struct ieee80211_rx_status *status)
+					 struct hal_rx_desc_data *rx_info)
 {
-	struct ath12k_base *ab = ar->ab;
 	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
 	u8 decap_hdr[DP_MAX_NWIFI_HDR_LEN];
 	struct ieee80211_hdr *hdr;
@@ -2225,7 +962,7 @@ static void ath12k_dp_rx_h_undecap_nwifi(struct ath12k *ar,
 
 	qos_ctl = rxcb->tid;
 
-	if (ath12k_dp_rx_h_mesh_ctl_present(ab, rxcb->rx_desc))
+	if (rx_info->mesh_ctrl_present)
 		qos_ctl |= IEEE80211_QOS_CTL_MESH_CONTROL_PRESENT;
 
 	/* TODO: Add other QoS ctl fields when required */
@@ -2234,9 +971,10 @@ static void ath12k_dp_rx_h_undecap_nwifi(struct ath12k *ar,
 	memcpy(decap_hdr, hdr, hdr_len);
 
 	/* Rebuild crypto header for mac80211 use */
-	if (!(status->flag & RX_FLAG_IV_STRIPPED)) {
-		crypto_hdr = skb_push(msdu, ath12k_dp_rx_crypto_param_len(ar, enctype));
-		ath12k_dp_rx_desc_get_crypto_header(ar->ab,
+	if (!(rx_info->rx_status->flag & RX_FLAG_IV_STRIPPED)) {
+		crypto_hdr = skb_push(msdu,
+				      ath12k_dp_rx_crypto_param_len(dp_pdev, enctype));
+		ath12k_dp_rx_desc_get_crypto_header(dp_pdev->dp->hal,
 						    rxcb->rx_desc, crypto_hdr,
 						    enctype);
 	}
@@ -2247,11 +985,13 @@ static void ath12k_dp_rx_h_undecap_nwifi(struct ath12k *ar,
 	memcpy(skb_push(msdu, hdr_len), decap_hdr, hdr_len);
 }
 
-static void ath12k_dp_rx_h_undecap_raw(struct ath12k *ar, struct sk_buff *msdu,
+static void ath12k_dp_rx_h_undecap_raw(struct ath12k_pdev_dp *dp_pdev,
+				       struct sk_buff *msdu,
 				       enum hal_encrypt_type enctype,
 				       struct ieee80211_rx_status *status,
 				       bool decrypted)
 {
+	struct ath12k_dp *dp = dp_pdev->dp;
 	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
 	struct ieee80211_hdr *hdr;
 	size_t hdr_len;
@@ -2273,20 +1013,20 @@ static void ath12k_dp_rx_h_undecap_raw(struct ath12k *ar, struct sk_buff *msdu,
 	/* Tail */
 	if (status->flag & RX_FLAG_IV_STRIPPED) {
 		skb_trim(msdu, msdu->len -
-			 ath12k_dp_rx_crypto_mic_len(ar, enctype));
+			 ath12k_dp_rx_crypto_mic_len(dp, enctype));
 
 		skb_trim(msdu, msdu->len -
-			 ath12k_dp_rx_crypto_icv_len(ar, enctype));
+			 ath12k_dp_rx_crypto_icv_len(dp_pdev, enctype));
 	} else {
 		/* MIC */
 		if (status->flag & RX_FLAG_MIC_STRIPPED)
 			skb_trim(msdu, msdu->len -
-				 ath12k_dp_rx_crypto_mic_len(ar, enctype));
+				 ath12k_dp_rx_crypto_mic_len(dp, enctype));
 
 		/* ICV */
 		if (status->flag & RX_FLAG_ICV_STRIPPED)
 			skb_trim(msdu, msdu->len -
-				 ath12k_dp_rx_crypto_icv_len(ar, enctype));
+				 ath12k_dp_rx_crypto_icv_len(dp_pdev, enctype));
 	}
 
 	/* MMIC */
@@ -2298,58 +1038,59 @@ static void ath12k_dp_rx_h_undecap_raw(struct ath12k *ar, struct sk_buff *msdu,
 	/* Head */
 	if (status->flag & RX_FLAG_IV_STRIPPED) {
 		hdr_len = ieee80211_hdrlen(hdr->frame_control);
-		crypto_len = ath12k_dp_rx_crypto_param_len(ar, enctype);
+		crypto_len = ath12k_dp_rx_crypto_param_len(dp_pdev, enctype);
 
 		memmove(msdu->data + crypto_len, msdu->data, hdr_len);
 		skb_pull(msdu, crypto_len);
 	}
 }
 
-static void ath12k_get_dot11_hdr_from_rx_desc(struct ath12k *ar,
+static void ath12k_get_dot11_hdr_from_rx_desc(struct ath12k_pdev_dp *dp_pdev,
 					      struct sk_buff *msdu,
 					      struct ath12k_skb_rxcb *rxcb,
-					      struct ieee80211_rx_status *status,
-					      enum hal_encrypt_type enctype)
+					      enum hal_encrypt_type enctype,
+					      struct hal_rx_desc_data *rx_info)
 {
 	struct hal_rx_desc *rx_desc = rxcb->rx_desc;
-	struct ath12k_base *ab = ar->ab;
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_hal *hal = dp->hal;
 	size_t hdr_len, crypto_len;
 	struct ieee80211_hdr hdr;
 	__le16 qos_ctl;
-	u8 *crypto_hdr, mesh_ctrl;
+	u8 *crypto_hdr;
 
-	ath12k_dp_rx_desc_get_dot11_hdr(ab, rx_desc, &hdr);
+	ath12k_dp_rx_desc_get_dot11_hdr(hal, rx_desc, &hdr);
 	hdr_len = ieee80211_hdrlen(hdr.frame_control);
-	mesh_ctrl = ath12k_dp_rx_h_mesh_ctl_present(ab, rx_desc);
 
-	if (!(status->flag & RX_FLAG_IV_STRIPPED)) {
-		crypto_len = ath12k_dp_rx_crypto_param_len(ar, enctype);
+	if (!(rx_info->rx_status->flag & RX_FLAG_IV_STRIPPED)) {
+		crypto_len = ath12k_dp_rx_crypto_param_len(dp_pdev, enctype);
 		crypto_hdr = skb_push(msdu, crypto_len);
-		ath12k_dp_rx_desc_get_crypto_header(ab, rx_desc, crypto_hdr, enctype);
+		ath12k_dp_rx_desc_get_crypto_header(dp->hal, rx_desc, crypto_hdr,
+						    enctype);
 	}
 
 	skb_push(msdu, hdr_len);
 	memcpy(msdu->data, &hdr, min(hdr_len, sizeof(hdr)));
 
 	if (rxcb->is_mcbc)
-		status->flag &= ~RX_FLAG_PN_VALIDATED;
+		rx_info->rx_status->flag &= ~RX_FLAG_PN_VALIDATED;
 
 	/* Add QOS header */
 	if (ieee80211_is_data_qos(hdr.frame_control)) {
 		struct ieee80211_hdr *qos_ptr = (struct ieee80211_hdr *)msdu->data;
 
 		qos_ctl = cpu_to_le16(rxcb->tid & IEEE80211_QOS_CTL_TID_MASK);
-		if (mesh_ctrl)
+		if (rx_info->mesh_ctrl_present)
 			qos_ctl |= cpu_to_le16(IEEE80211_QOS_CTL_MESH_CONTROL_PRESENT);
 
 		memcpy(ieee80211_get_qos_ctl(qos_ptr), &qos_ctl, IEEE80211_QOS_CTL_LEN);
 	}
 }
 
-static void ath12k_dp_rx_h_undecap_eth(struct ath12k *ar,
+static void ath12k_dp_rx_h_undecap_eth(struct ath12k_pdev_dp *dp_pdev,
 				       struct sk_buff *msdu,
 				       enum hal_encrypt_type enctype,
-				       struct ieee80211_rx_status *status)
+				       struct hal_rx_desc_data *rx_info)
 {
 	struct ieee80211_hdr *hdr;
 	struct ethhdr *eth;
@@ -2365,7 +1106,7 @@ static void ath12k_dp_rx_h_undecap_eth(struct ath12k *ar,
 	skb_pull(msdu, sizeof(*eth));
 	memcpy(skb_push(msdu, sizeof(rfc)), &rfc,
 	       sizeof(rfc));
-	ath12k_get_dot11_hdr_from_rx_desc(ar, msdu, rxcb, status, enctype);
+	ath12k_get_dot11_hdr_from_rx_desc(dp_pdev, msdu, rxcb, enctype, rx_info);
 
 	/* original 802.11 header has a different DA and in
 	 * case of 4addr it may also have different SA
@@ -2375,24 +1116,20 @@ static void ath12k_dp_rx_h_undecap_eth(struct ath12k *ar,
 	ether_addr_copy(ieee80211_get_SA(hdr), sa);
 }
 
-static void ath12k_dp_rx_h_undecap(struct ath12k *ar, struct sk_buff *msdu,
-				   struct hal_rx_desc *rx_desc,
-				   enum hal_encrypt_type enctype,
-				   struct ieee80211_rx_status *status,
-				   bool decrypted)
+void ath12k_dp_rx_h_undecap(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu,
+			    struct hal_rx_desc *rx_desc,
+			    enum hal_encrypt_type enctype,
+			    bool decrypted,
+			    struct hal_rx_desc_data *rx_info)
 {
-	struct ath12k_base *ab = ar->ab;
-	u8 decap;
 	struct ethhdr *ehdr;
 
-	decap = ath12k_dp_rx_h_decap_type(ab, rx_desc);
-
-	switch (decap) {
+	switch (rx_info->decap_type) {
 	case DP_RX_DECAP_TYPE_NATIVE_WIFI:
-		ath12k_dp_rx_h_undecap_nwifi(ar, msdu, enctype, status);
+		ath12k_dp_rx_h_undecap_nwifi(dp_pdev, msdu, enctype, rx_info);
 		break;
 	case DP_RX_DECAP_TYPE_RAW:
-		ath12k_dp_rx_h_undecap_raw(ar, msdu, enctype, status,
+		ath12k_dp_rx_h_undecap_raw(dp_pdev, msdu, enctype, rx_info->rx_status,
 					   decrypted);
 		break;
 	case DP_RX_DECAP_TYPE_ETHERNET2_DIX:
@@ -2401,7 +1138,7 @@ static void ath12k_dp_rx_h_undecap(struct ath12k *ar, struct sk_buff *msdu,
 		/* mac80211 allows fast path only for authorized STA */
 		if (ehdr->h_proto == cpu_to_be16(ETH_P_PAE)) {
 			ATH12K_SKB_RXCB(msdu)->is_eapol = true;
-			ath12k_dp_rx_h_undecap_eth(ar, msdu, enctype, status);
+			ath12k_dp_rx_h_undecap_eth(dp_pdev, msdu, enctype, rx_info);
 			break;
 		}
 
@@ -2409,125 +1146,53 @@ static void ath12k_dp_rx_h_undecap(struct ath12k *ar, struct sk_buff *msdu,
 		 * remove eth header and add 802.11 header.
 		 */
 		if (ATH12K_SKB_RXCB(msdu)->is_mcbc && decrypted)
-			ath12k_dp_rx_h_undecap_eth(ar, msdu, enctype, status);
+			ath12k_dp_rx_h_undecap_eth(dp_pdev, msdu, enctype, rx_info);
 		break;
 	case DP_RX_DECAP_TYPE_8023:
 		/* TODO: Handle undecap for these formats */
 		break;
 	}
 }
+EXPORT_SYMBOL(ath12k_dp_rx_h_undecap);
 
-struct ath12k_peer *
-ath12k_dp_rx_h_find_peer(struct ath12k_base *ab, struct sk_buff *msdu,
-			 struct ath12k_dp_rx_info *rx_info)
+struct ath12k_dp_link_peer *
+ath12k_dp_rx_h_find_link_peer(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu,
+			      struct hal_rx_desc_data *rx_info)
 {
 	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
-	struct ath12k_peer *peer = NULL;
+	struct ath12k_dp_link_peer *peer = NULL;
+	struct ath12k_dp *dp = dp_pdev->dp;
 
-	lockdep_assert_held(&ab->base_lock);
+	lockdep_assert_held(&dp->dp_lock);
 
 	if (rxcb->peer_id)
-		peer = ath12k_peer_find_by_id(ab, rxcb->peer_id);
+		peer = ath12k_dp_link_peer_find_by_peerid(dp_pdev, rxcb->peer_id);
 
 	if (peer)
 		return peer;
 
 	if (rx_info->addr2_present)
-		peer = ath12k_peer_find_by_addr(ab, rx_info->addr2);
+		peer = ath12k_dp_link_peer_find_by_addr(dp, rx_info->addr2);
 
 	return peer;
 }
 
-static void ath12k_dp_rx_h_mpdu(struct ath12k *ar,
-				struct sk_buff *msdu,
-				struct hal_rx_desc *rx_desc,
-				struct ath12k_dp_rx_info *rx_info)
-{
-	struct ath12k_base *ab = ar->ab;
-	struct ath12k_skb_rxcb *rxcb;
-	enum hal_encrypt_type enctype;
-	bool is_decrypted = false;
-	struct ieee80211_hdr *hdr;
-	struct ath12k_peer *peer;
-	struct ieee80211_rx_status *rx_status = rx_info->rx_status;
-	u32 err_bitmap;
-
-	/* PN for multicast packets will be checked in mac80211 */
-	rxcb = ATH12K_SKB_RXCB(msdu);
-	rxcb->is_mcbc = rx_info->is_mcbc;
-
-	if (rxcb->is_mcbc)
-		rxcb->peer_id = rx_info->peer_id;
-
-	spin_lock_bh(&ar->ab->base_lock);
-	peer = ath12k_dp_rx_h_find_peer(ar->ab, msdu, rx_info);
-	if (peer) {
-		/* resetting mcbc bit because mcbc packets are unicast
-		 * packets only for AP as STA sends unicast packets.
-		 */
-		rxcb->is_mcbc = rxcb->is_mcbc && !peer->ucast_ra_only;
-
-		if (rxcb->is_mcbc)
-			enctype = peer->sec_type_grp;
-		else
-			enctype = peer->sec_type;
-	} else {
-		enctype = HAL_ENCRYPT_TYPE_OPEN;
-	}
-	spin_unlock_bh(&ar->ab->base_lock);
-
-	err_bitmap = ath12k_dp_rx_h_mpdu_err(ab, rx_desc);
-	if (enctype != HAL_ENCRYPT_TYPE_OPEN && !err_bitmap)
-		is_decrypted = ath12k_dp_rx_h_is_decrypted(ab, rx_desc);
-
-	/* Clear per-MPDU flags while leaving per-PPDU flags intact */
-	rx_status->flag &= ~(RX_FLAG_FAILED_FCS_CRC |
-			     RX_FLAG_MMIC_ERROR |
-			     RX_FLAG_DECRYPTED |
-			     RX_FLAG_IV_STRIPPED |
-			     RX_FLAG_MMIC_STRIPPED);
-
-	if (err_bitmap & HAL_RX_MPDU_ERR_FCS)
-		rx_status->flag |= RX_FLAG_FAILED_FCS_CRC;
-	if (err_bitmap & HAL_RX_MPDU_ERR_TKIP_MIC)
-		rx_status->flag |= RX_FLAG_MMIC_ERROR;
-
-	if (is_decrypted) {
-		rx_status->flag |= RX_FLAG_DECRYPTED | RX_FLAG_MMIC_STRIPPED;
-
-		if (rx_info->is_mcbc)
-			rx_status->flag |= RX_FLAG_MIC_STRIPPED |
-					RX_FLAG_ICV_STRIPPED;
-		else
-			rx_status->flag |= RX_FLAG_IV_STRIPPED |
-					   RX_FLAG_PN_VALIDATED;
-	}
-
-	ath12k_dp_rx_h_csum_offload(msdu, rx_info);
-	ath12k_dp_rx_h_undecap(ar, msdu, rx_desc,
-			       enctype, rx_status, is_decrypted);
-
-	if (!is_decrypted || rx_info->is_mcbc)
-		return;
-
-	if (rx_info->decap_type != DP_RX_DECAP_TYPE_ETHERNET2_DIX) {
-		hdr = (void *)msdu->data;
-		hdr->frame_control &= ~__cpu_to_le16(IEEE80211_FCTL_PROTECTED);
-	}
-}
-
-static void ath12k_dp_rx_h_rate(struct ath12k *ar, struct ath12k_dp_rx_info *rx_info)
+static void ath12k_dp_rx_h_rate(struct ath12k_pdev_dp *dp_pdev,
+				struct hal_rx_desc_data *rx_info)
 {
+	struct ath12k_dp *dp = dp_pdev->dp;
 	struct ieee80211_supported_band *sband;
 	struct ieee80211_rx_status *rx_status = rx_info->rx_status;
 	enum rx_msdu_start_pkt_type pkt_type = rx_info->pkt_type;
 	u8 bw = rx_info->bw, sgi = rx_info->sgi;
 	u8 rate_mcs = rx_info->rate_mcs, nss = rx_info->nss;
 	bool is_cck;
+	struct ath12k *ar;
 
 	switch (pkt_type) {
 	case RX_MSDU_START_PKT_TYPE_11A:
 	case RX_MSDU_START_PKT_TYPE_11B:
+		ar = ath12k_pdev_dp_to_ar(dp_pdev);
 		is_cck = (pkt_type == RX_MSDU_START_PKT_TYPE_11B);
 		sband = &ar->mac.sbands[rx_status->band];
 		rx_status->rate_idx = ath12k_mac_hw_rate_to_idx(sband, rate_mcs,
@@ -2536,7 +1201,7 @@ static void ath12k_dp_rx_h_rate(struct ath12k *ar, struct ath12k_dp_rx_info *rx_
 	case RX_MSDU_START_PKT_TYPE_11N:
 		rx_status->encoding = RX_ENC_HT;
 		if (rate_mcs > ATH12K_HT_MCS_MAX) {
-			ath12k_warn(ar->ab,
+			ath12k_warn(dp->ab,
 				    "Received with invalid mcs in HT mode %d\n",
 				     rate_mcs);
 			break;
@@ -2550,7 +1215,7 @@ static void ath12k_dp_rx_h_rate(struct ath12k *ar, struct ath12k_dp_rx_info *rx_
 		rx_status->encoding = RX_ENC_VHT;
 		rx_status->rate_idx = rate_mcs;
 		if (rate_mcs > ATH12K_VHT_MCS_MAX) {
-			ath12k_warn(ar->ab,
+			ath12k_warn(dp->ab,
 				    "Received with invalid mcs in VHT mode %d\n",
 				     rate_mcs);
 			break;
@@ -2563,7 +1228,7 @@ static void ath12k_dp_rx_h_rate(struct ath12k *ar, struct ath12k_dp_rx_info *rx_
 	case RX_MSDU_START_PKT_TYPE_11AX:
 		rx_status->rate_idx = rate_mcs;
 		if (rate_mcs > ATH12K_HE_MCS_MAX) {
-			ath12k_warn(ar->ab,
+			ath12k_warn(dp->ab,
 				    "Received with invalid mcs in HE mode %d\n",
 				    rate_mcs);
 			break;
@@ -2577,7 +1242,7 @@ static void ath12k_dp_rx_h_rate(struct ath12k *ar, struct ath12k_dp_rx_info *rx_
 		rx_status->rate_idx = rate_mcs;
 
 		if (rate_mcs > ATH12K_EHT_MCS_MAX) {
-			ath12k_warn(ar->ab,
+			ath12k_warn(dp->ab,
 				    "Received with invalid mcs in EHT mode %d\n",
 				    rate_mcs);
 			break;
@@ -2593,33 +1258,8 @@ static void ath12k_dp_rx_h_rate(struct ath12k *ar, struct ath12k_dp_rx_info *rx_
 	}
 }
 
-void ath12k_dp_rx_h_fetch_info(struct ath12k_base *ab, struct hal_rx_desc *rx_desc,
-			       struct ath12k_dp_rx_info *rx_info)
-{
-	rx_info->ip_csum_fail = ath12k_dp_rx_h_ip_cksum_fail(ab, rx_desc);
-	rx_info->l4_csum_fail = ath12k_dp_rx_h_l4_cksum_fail(ab, rx_desc);
-	rx_info->is_mcbc = ath12k_dp_rx_h_is_da_mcbc(ab, rx_desc);
-	rx_info->decap_type = ath12k_dp_rx_h_decap_type(ab, rx_desc);
-	rx_info->pkt_type = ath12k_dp_rx_h_pkt_type(ab, rx_desc);
-	rx_info->sgi = ath12k_dp_rx_h_sgi(ab, rx_desc);
-	rx_info->rate_mcs = ath12k_dp_rx_h_rate_mcs(ab, rx_desc);
-	rx_info->bw = ath12k_dp_rx_h_rx_bw(ab, rx_desc);
-	rx_info->nss = ath12k_dp_rx_h_nss(ab, rx_desc);
-	rx_info->tid = ath12k_dp_rx_h_tid(ab, rx_desc);
-	rx_info->peer_id = ath12k_dp_rx_h_peer_id(ab, rx_desc);
-	rx_info->phy_meta_data = ath12k_dp_rx_h_freq(ab, rx_desc);
-
-	if (ath12k_dp_rxdesc_mac_addr2_valid(ab, rx_desc)) {
-		ether_addr_copy(rx_info->addr2,
-				ath12k_dp_rxdesc_get_mpdu_start_addr2(ab, rx_desc));
-		rx_info->addr2_present = true;
-	}
-
-	ath12k_dbg_dump(ab, ATH12K_DBG_DATA, NULL, "rx_desc: ",
-			rx_desc, sizeof(*rx_desc));
-}
-
-void ath12k_dp_rx_h_ppdu(struct ath12k *ar, struct ath12k_dp_rx_info *rx_info)
+void ath12k_dp_rx_h_ppdu(struct ath12k_pdev_dp *dp_pdev,
+			 struct hal_rx_desc_data *rx_info)
 {
 	struct ieee80211_rx_status *rx_status = rx_info->rx_status;
 	u8 channel_num;
@@ -2652,7 +1292,9 @@ void ath12k_dp_rx_h_ppdu(struct ath12k *ar, struct ath12k_dp_rx_info *rx_info)
 	}
 
 	if (unlikely(rx_status->band == NUM_NL80211_BANDS ||
-		     !ath12k_ar_to_hw(ar)->wiphy->bands[rx_status->band])) {
+		     !ath12k_pdev_dp_to_hw(dp_pdev)->wiphy->bands[rx_status->band])) {
+		struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev);
+
 		ath12k_warn(ar->ab, "sband is NULL for status band %d channel_num %d center_freq %d pdev_id %d\n",
 			    rx_status->band, channel_num, center_freq, ar->pdev_idx);
 
@@ -2676,43 +1318,41 @@ void ath12k_dp_rx_h_ppdu(struct ath12k *ar, struct ath12k_dp_rx_info *rx_info)
 								 rx_status->band);
 
 h_rate:
-	ath12k_dp_rx_h_rate(ar, rx_info);
+	ath12k_dp_rx_h_rate(dp_pdev, rx_info);
 }
+EXPORT_SYMBOL(ath12k_dp_rx_h_ppdu);
 
-static void ath12k_dp_rx_deliver_msdu(struct ath12k *ar, struct napi_struct *napi,
-				      struct sk_buff *msdu,
-				      struct ath12k_dp_rx_info *rx_info)
+void ath12k_dp_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev, struct napi_struct *napi,
+			       struct sk_buff *msdu,
+			       struct hal_rx_desc_data *rx_info)
 {
-	struct ath12k_base *ab = ar->ab;
+	struct ath12k_dp *dp = dp_pdev->dp;
 	struct ieee80211_rx_status *rx_status;
 	struct ieee80211_sta *pubsta;
-	struct ath12k_peer *peer;
+	struct ath12k_dp_peer *peer;
 	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
 	struct ieee80211_rx_status *status = rx_info->rx_status;
 	u8 decap = rx_info->decap_type;
 	bool is_mcbc = rxcb->is_mcbc;
 	bool is_eapol = rxcb->is_eapol;
 
-	spin_lock_bh(&ab->base_lock);
-	peer = ath12k_dp_rx_h_find_peer(ab, msdu, rx_info);
+	peer = ath12k_dp_peer_find_by_peerid(dp_pdev, rx_info->peer_id);
 
 	pubsta = peer ? peer->sta : NULL;
 
 	if (pubsta && pubsta->valid_links) {
 		status->link_valid = 1;
-		status->link_id = peer->link_id;
+		status->link_id = peer->hw_links[rxcb->hw_link_id];
 	}
 
-	spin_unlock_bh(&ab->base_lock);
-
-	ath12k_dbg(ab, ATH12K_DBG_DATA,
+	ath12k_dbg(dp->ab, ATH12K_DBG_DATA,
 		   "rx skb %p len %u peer %pM %d %s sn %u %s%s%s%s%s%s%s%s%s%s rate_idx %u vht_nss %u freq %u band %u flag 0x%x fcs-err %i mic-err %i amsdu-more %i\n",
 		   msdu,
 		   msdu->len,
 		   peer ? peer->addr : NULL,
 		   rxcb->tid,
 		   is_mcbc ? "mcast" : "ucast",
-		   ath12k_dp_rx_h_seq_no(ab, rxcb->rx_desc),
+		   rx_info->seq_no,
 		   (status->encoding == RX_ENC_LEGACY) ? "legacy" : "",
 		   (status->encoding == RX_ENC_HT) ? "ht" : "",
 		   (status->encoding == RX_ENC_VHT) ? "vht" : "",
@@ -2731,7 +1371,7 @@ static void ath12k_dp_rx_deliver_msdu(struct ath12k *ar, struct napi_struct *nap
 		   !!(status->flag & RX_FLAG_MMIC_ERROR),
 		   !!(status->flag & RX_FLAG_AMSDU_MORE));
 
-	ath12k_dbg_dump(ab, ATH12K_DBG_DP_RX, NULL, "dp rx msdu: ",
+	ath12k_dbg_dump(dp->ab, ATH12K_DBG_DP_RX, NULL, "dp rx msdu: ",
 			msdu->data, msdu->len);
 
 	rx_status = IEEE80211_SKB_RXCB(msdu);
@@ -2748,19 +1388,19 @@ static void ath12k_dp_rx_deliver_msdu(struct ath12k *ar, struct napi_struct *nap
 	    !(is_mcbc && rx_status->flag & RX_FLAG_DECRYPTED))
 		rx_status->flag |= RX_FLAG_8023;
 
-	ieee80211_rx_napi(ath12k_ar_to_hw(ar), pubsta, msdu, napi);
+	ieee80211_rx_napi(ath12k_pdev_dp_to_hw(dp_pdev), pubsta, msdu, napi);
 }
+EXPORT_SYMBOL(ath12k_dp_rx_deliver_msdu);
 
-static bool ath12k_dp_rx_check_nwifi_hdr_len_valid(struct ath12k_base *ab,
-						   struct hal_rx_desc *rx_desc,
-						   struct sk_buff *msdu)
+bool ath12k_dp_rx_check_nwifi_hdr_len_valid(struct ath12k_dp *dp,
+					    struct hal_rx_desc *rx_desc,
+					    struct sk_buff *msdu,
+					    struct hal_rx_desc_data *rx_info)
 {
 	struct ieee80211_hdr *hdr;
-	u8 decap_type;
 	u32 hdr_len;
 
-	decap_type = ath12k_dp_rx_h_decap_type(ab, rx_desc);
-	if (decap_type != DP_RX_DECAP_TYPE_NATIVE_WIFI)
+	if (rx_info->decap_type != DP_RX_DECAP_TYPE_NATIVE_WIFI)
 		return true;
 
 	hdr = (struct ieee80211_hdr *)msdu->data;
@@ -2769,385 +1409,73 @@ static bool ath12k_dp_rx_check_nwifi_hdr_len_valid(struct ath12k_base *ab,
 	if ((likely(hdr_len <= DP_MAX_NWIFI_HDR_LEN)))
 		return true;
 
-	ab->device_stats.invalid_rbm++;
+	dp->device_stats.invalid_rbm++;
 	WARN_ON_ONCE(1);
 	return false;
 }
+EXPORT_SYMBOL(ath12k_dp_rx_check_nwifi_hdr_len_valid);
 
-static int ath12k_dp_rx_process_msdu(struct ath12k *ar,
-				     struct sk_buff *msdu,
-				     struct sk_buff_head *msdu_list,
-				     struct ath12k_dp_rx_info *rx_info)
+static void ath12k_dp_rx_frag_timer(struct timer_list *timer)
 {
-	struct ath12k_base *ab = ar->ab;
-	struct hal_rx_desc *rx_desc, *lrx_desc;
-	struct ath12k_skb_rxcb *rxcb;
-	struct sk_buff *last_buf;
-	u8 l3_pad_bytes;
-	u16 msdu_len;
-	int ret;
-	u32 hal_rx_desc_sz = ar->ab->hal.hal_desc_sz;
+	struct ath12k_dp_rx_tid *rx_tid = timer_container_of(rx_tid, timer,
+							     frag_timer);
 
-	last_buf = ath12k_dp_rx_get_msdu_last_buf(msdu_list, msdu);
-	if (!last_buf) {
-		ath12k_warn(ab,
-			    "No valid Rx buffer to access MSDU_END tlv\n");
-		ret = -EIO;
-		goto free_out;
+	spin_lock_bh(&rx_tid->dp->dp_lock);
+	if (rx_tid->last_frag_no &&
+	    rx_tid->rx_frag_bitmap == GENMASK(rx_tid->last_frag_no, 0)) {
+		spin_unlock_bh(&rx_tid->dp->dp_lock);
+		return;
 	}
-
-	rx_desc = (struct hal_rx_desc *)msdu->data;
-	lrx_desc = (struct hal_rx_desc *)last_buf->data;
-	if (!ath12k_dp_rx_h_msdu_done(ab, lrx_desc)) {
-		ath12k_warn(ab, "msdu_done bit in msdu_end is not set\n");
-		ret = -EIO;
-		goto free_out;
-	}
-
-	rxcb = ATH12K_SKB_RXCB(msdu);
-	rxcb->rx_desc = rx_desc;
-	msdu_len = ath12k_dp_rx_h_msdu_len(ab, lrx_desc);
-	l3_pad_bytes = ath12k_dp_rx_h_l3pad(ab, lrx_desc);
-
-	if (rxcb->is_frag) {
-		skb_pull(msdu, hal_rx_desc_sz);
-	} else if (!rxcb->is_continuation) {
-		if ((msdu_len + hal_rx_desc_sz) > DP_RX_BUFFER_SIZE) {
-			ret = -EINVAL;
-			ath12k_warn(ab, "invalid msdu len %u\n", msdu_len);
-			ath12k_dbg_dump(ab, ATH12K_DBG_DATA, NULL, "", rx_desc,
-					sizeof(*rx_desc));
-			goto free_out;
-		}
-		skb_put(msdu, hal_rx_desc_sz + l3_pad_bytes + msdu_len);
-		skb_pull(msdu, hal_rx_desc_sz + l3_pad_bytes);
-	} else {
-		ret = ath12k_dp_rx_msdu_coalesce(ar, msdu_list,
-						 msdu, last_buf,
-						 l3_pad_bytes, msdu_len);
-		if (ret) {
-			ath12k_warn(ab,
-				    "failed to coalesce msdu rx buffer%d\n", ret);
-			goto free_out;
-		}
-	}
-
-	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(ab, rx_desc, msdu))) {
-		ret = -EINVAL;
-		goto free_out;
-	}
-
-	ath12k_dp_rx_h_fetch_info(ab, rx_desc, rx_info);
-	ath12k_dp_rx_h_ppdu(ar, rx_info);
-	ath12k_dp_rx_h_mpdu(ar, msdu, rx_desc, rx_info);
-
-	rx_info->rx_status->flag |= RX_FLAG_SKIP_MONITOR | RX_FLAG_DUP_VALIDATED;
-
-	return 0;
-
-free_out:
-	return ret;
-}
-
-static void ath12k_dp_rx_process_received_packets(struct ath12k_base *ab,
-						  struct napi_struct *napi,
-						  struct sk_buff_head *msdu_list,
-						  int ring_id)
-{
-	struct ath12k_hw_group *ag = ab->ag;
-	struct ieee80211_rx_status rx_status = {};
-	struct ath12k_skb_rxcb *rxcb;
-	struct sk_buff *msdu;
-	struct ath12k *ar;
-	struct ath12k_hw_link *hw_links = ag->hw_links;
-	struct ath12k_base *partner_ab;
-	struct ath12k_dp_rx_info rx_info;
-	u8 hw_link_id, pdev_id;
-	int ret;
-
-	if (skb_queue_empty(msdu_list))
-		return;
-
-	rx_info.addr2_present = false;
-	rx_info.rx_status = &rx_status;
-
-	rcu_read_lock();
-
-	while ((msdu = __skb_dequeue(msdu_list))) {
-		rxcb = ATH12K_SKB_RXCB(msdu);
-		hw_link_id = rxcb->hw_link_id;
-		partner_ab = ath12k_ag_to_ab(ag,
-					     hw_links[hw_link_id].device_id);
-		pdev_id = ath12k_hw_mac_id_to_pdev_id(partner_ab->hw_params,
-						      hw_links[hw_link_id].pdev_idx);
-		ar = partner_ab->pdevs[pdev_id].ar;
-		if (!rcu_dereference(partner_ab->pdevs_active[pdev_id])) {
-			dev_kfree_skb_any(msdu);
-			continue;
-		}
-
-		if (test_bit(ATH12K_FLAG_CAC_RUNNING, &ar->dev_flags)) {
-			dev_kfree_skb_any(msdu);
-			continue;
-		}
-
-		ret = ath12k_dp_rx_process_msdu(ar, msdu, msdu_list, &rx_info);
-		if (ret) {
-			ath12k_dbg(ab, ATH12K_DBG_DATA,
-				   "Unable to process msdu %d", ret);
-			dev_kfree_skb_any(msdu);
-			continue;
-		}
-
-		ath12k_dp_rx_deliver_msdu(ar, napi, msdu, &rx_info);
-	}
-
-	rcu_read_unlock();
-}
-
-static u16 ath12k_dp_rx_get_peer_id(struct ath12k_base *ab,
-				    enum ath12k_peer_metadata_version ver,
-				    __le32 peer_metadata)
-{
-	switch (ver) {
-	default:
-		ath12k_warn(ab, "Unknown peer metadata version: %d", ver);
-		fallthrough;
-	case ATH12K_PEER_METADATA_V0:
-		return le32_get_bits(peer_metadata,
-				     RX_MPDU_DESC_META_DATA_V0_PEER_ID);
-	case ATH12K_PEER_METADATA_V1:
-		return le32_get_bits(peer_metadata,
-				     RX_MPDU_DESC_META_DATA_V1_PEER_ID);
-	case ATH12K_PEER_METADATA_V1A:
-		return le32_get_bits(peer_metadata,
-				     RX_MPDU_DESC_META_DATA_V1A_PEER_ID);
-	case ATH12K_PEER_METADATA_V1B:
-		return le32_get_bits(peer_metadata,
-				     RX_MPDU_DESC_META_DATA_V1B_PEER_ID);
-	}
-}
-
-int ath12k_dp_rx_process(struct ath12k_base *ab, int ring_id,
-			 struct napi_struct *napi, int budget)
-{
-	struct ath12k_hw_group *ag = ab->ag;
-	struct list_head rx_desc_used_list[ATH12K_MAX_DEVICES];
-	struct ath12k_hw_link *hw_links = ag->hw_links;
-	int num_buffs_reaped[ATH12K_MAX_DEVICES] = {};
-	struct ath12k_rx_desc_info *desc_info;
-	struct ath12k_dp *dp = &ab->dp;
-	struct dp_rxdma_ring *rx_ring = &dp->rx_refill_buf_ring;
-	struct hal_reo_dest_ring *desc;
-	struct ath12k_base *partner_ab;
-	struct sk_buff_head msdu_list;
-	struct ath12k_skb_rxcb *rxcb;
-	int total_msdu_reaped = 0;
-	u8 hw_link_id, device_id;
-	struct hal_srng *srng;
-	struct sk_buff *msdu;
-	bool done = false;
-	u64 desc_va;
-
-	__skb_queue_head_init(&msdu_list);
-
-	for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++)
-		INIT_LIST_HEAD(&rx_desc_used_list[device_id]);
-
-	srng = &ab->hal.srng_list[dp->reo_dst_ring[ring_id].ring_id];
-
-	spin_lock_bh(&srng->lock);
-
-try_again:
-	ath12k_hal_srng_access_begin(ab, srng);
-
-	while ((desc = ath12k_hal_srng_dst_get_next_entry(ab, srng))) {
-		struct rx_mpdu_desc *mpdu_info;
-		struct rx_msdu_desc *msdu_info;
-		enum hal_reo_dest_ring_push_reason push_reason;
-		u32 cookie;
-
-		cookie = le32_get_bits(desc->buf_addr_info.info1,
-				       BUFFER_ADDR_INFO1_SW_COOKIE);
-
-		hw_link_id = le32_get_bits(desc->info0,
-					   HAL_REO_DEST_RING_INFO0_SRC_LINK_ID);
-
-		desc_va = ((u64)le32_to_cpu(desc->buf_va_hi) << 32 |
-			   le32_to_cpu(desc->buf_va_lo));
-		desc_info = (struct ath12k_rx_desc_info *)((unsigned long)desc_va);
-
-		device_id = hw_links[hw_link_id].device_id;
-		partner_ab = ath12k_ag_to_ab(ag, device_id);
-		if (unlikely(!partner_ab)) {
-			if (desc_info->skb) {
-				dev_kfree_skb_any(desc_info->skb);
-				desc_info->skb = NULL;
-			}
-
-			continue;
-		}
-
-		/* retry manual desc retrieval */
-		if (!desc_info) {
-			desc_info = ath12k_dp_get_rx_desc(partner_ab, cookie);
-			if (!desc_info) {
-				ath12k_warn(partner_ab, "Invalid cookie in manual descriptor retrieval: 0x%x\n",
-					    cookie);
-				continue;
-			}
-		}
-
-		if (desc_info->magic != ATH12K_DP_RX_DESC_MAGIC)
-			ath12k_warn(ab, "Check HW CC implementation");
-
-		msdu = desc_info->skb;
-		desc_info->skb = NULL;
-
-		list_add_tail(&desc_info->list, &rx_desc_used_list[device_id]);
-
-		rxcb = ATH12K_SKB_RXCB(msdu);
-		dma_unmap_single(partner_ab->dev, rxcb->paddr,
-				 msdu->len + skb_tailroom(msdu),
-				 DMA_FROM_DEVICE);
-
-		num_buffs_reaped[device_id]++;
-		ab->device_stats.reo_rx[ring_id][ab->device_id]++;
-
-		push_reason = le32_get_bits(desc->info0,
-					    HAL_REO_DEST_RING_INFO0_PUSH_REASON);
-		if (push_reason !=
-		    HAL_REO_DEST_RING_PUSH_REASON_ROUTING_INSTRUCTION) {
-			dev_kfree_skb_any(msdu);
-			ab->device_stats.hal_reo_error[ring_id]++;
-			continue;
-		}
-
-		msdu_info = &desc->rx_msdu_info;
-		mpdu_info = &desc->rx_mpdu_info;
-
-		rxcb->is_first_msdu = !!(le32_to_cpu(msdu_info->info0) &
-					 RX_MSDU_DESC_INFO0_FIRST_MSDU_IN_MPDU);
-		rxcb->is_last_msdu = !!(le32_to_cpu(msdu_info->info0) &
-					RX_MSDU_DESC_INFO0_LAST_MSDU_IN_MPDU);
-		rxcb->is_continuation = !!(le32_to_cpu(msdu_info->info0) &
-					   RX_MSDU_DESC_INFO0_MSDU_CONTINUATION);
-		rxcb->hw_link_id = hw_link_id;
-		rxcb->peer_id = ath12k_dp_rx_get_peer_id(ab, dp->peer_metadata_ver,
-							 mpdu_info->peer_meta_data);
-		rxcb->tid = le32_get_bits(mpdu_info->info0,
-					  RX_MPDU_DESC_INFO0_TID);
-
-		__skb_queue_tail(&msdu_list, msdu);
-
-		if (!rxcb->is_continuation) {
-			total_msdu_reaped++;
-			done = true;
-		} else {
-			done = false;
-		}
-
-		if (total_msdu_reaped >= budget)
-			break;
-	}
-
-	/* Hw might have updated the head pointer after we cached it.
-	 * In this case, even though there are entries in the ring we'll
-	 * get rx_desc NULL. Give the read another try with updated cached
-	 * head pointer so that we can reap complete MPDU in the current
-	 * rx processing.
-	 */
-	if (!done && ath12k_hal_srng_dst_num_free(ab, srng, true)) {
-		ath12k_hal_srng_access_end(ab, srng);
-		goto try_again;
-	}
-
-	ath12k_hal_srng_access_end(ab, srng);
-
-	spin_unlock_bh(&srng->lock);
-
-	if (!total_msdu_reaped)
-		goto exit;
-
-	for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) {
-		if (!num_buffs_reaped[device_id])
-			continue;
-
-		partner_ab = ath12k_ag_to_ab(ag, device_id);
-		rx_ring = &partner_ab->dp.rx_refill_buf_ring;
-
-		ath12k_dp_rx_bufs_replenish(partner_ab, rx_ring,
-					    &rx_desc_used_list[device_id],
-					    num_buffs_reaped[device_id]);
-	}
-
-	ath12k_dp_rx_process_received_packets(ab, napi, &msdu_list,
-					      ring_id);
-
-exit:
-	return total_msdu_reaped;
-}
-
-static void ath12k_dp_rx_frag_timer(struct timer_list *timer)
-{
-	struct ath12k_dp_rx_tid *rx_tid = timer_container_of(rx_tid, timer,
-							     frag_timer);
-
-	spin_lock_bh(&rx_tid->ab->base_lock);
-	if (rx_tid->last_frag_no &&
-	    rx_tid->rx_frag_bitmap == GENMASK(rx_tid->last_frag_no, 0)) {
-		spin_unlock_bh(&rx_tid->ab->base_lock);
-		return;
-	}
-	ath12k_dp_rx_frags_cleanup(rx_tid, true);
-	spin_unlock_bh(&rx_tid->ab->base_lock);
+	ath12k_dp_arch_rx_frags_cleanup(rx_tid->dp, rx_tid, true);
+	spin_unlock_bh(&rx_tid->dp->dp_lock);
 }
 
 int ath12k_dp_rx_peer_frag_setup(struct ath12k *ar, const u8 *peer_mac, int vdev_id)
 {
 	struct ath12k_base *ab = ar->ab;
 	struct crypto_shash *tfm;
-	struct ath12k_peer *peer;
+	struct ath12k_dp_link_peer *peer;
 	struct ath12k_dp_rx_tid *rx_tid;
 	int i;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 
 	tfm = crypto_alloc_shash("michael_mic", 0, 0);
 	if (IS_ERR(tfm))
 		return PTR_ERR(tfm);
 
-	spin_lock_bh(&ab->base_lock);
+	spin_lock_bh(&dp->dp_lock);
 
-	peer = ath12k_peer_find(ab, vdev_id, peer_mac);
-	if (!peer) {
-		spin_unlock_bh(&ab->base_lock);
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, vdev_id, peer_mac);
+	if (!peer || !peer->dp_peer) {
+		spin_unlock_bh(&dp->dp_lock);
 		crypto_free_shash(tfm);
 		ath12k_warn(ab, "failed to find the peer to set up fragment info\n");
 		return -ENOENT;
 	}
 
 	if (!peer->primary_link) {
-		spin_unlock_bh(&ab->base_lock);
+		spin_unlock_bh(&dp->dp_lock);
 		crypto_free_shash(tfm);
 		return 0;
 	}
 
 	for (i = 0; i <= IEEE80211_NUM_TIDS; i++) {
-		rx_tid = &peer->rx_tid[i];
-		rx_tid->ab = ab;
+		rx_tid = &peer->dp_peer->rx_tid[i];
+		rx_tid->dp = dp;
 		timer_setup(&rx_tid->frag_timer, ath12k_dp_rx_frag_timer, 0);
 		skb_queue_head_init(&rx_tid->rx_frags);
 	}
 
-	peer->tfm_mmic = tfm;
-	peer->dp_setup_done = true;
-	spin_unlock_bh(&ab->base_lock);
+	peer->dp_peer->tfm_mmic = tfm;
+	peer->dp_peer->dp_setup_done = true;
+	spin_unlock_bh(&dp->dp_lock);
 
 	return 0;
 }
 
-static int ath12k_dp_rx_h_michael_mic(struct crypto_shash *tfm, u8 *key,
-				      struct ieee80211_hdr *hdr, u8 *data,
-				      size_t data_len, u8 *mic)
+int ath12k_dp_rx_h_michael_mic(struct crypto_shash *tfm, u8 *key,
+			       struct ieee80211_hdr *hdr, u8 *data,
+			       size_t data_len, u8 *mic)
 {
 	SHASH_DESC_ON_STACK(desc, tfm);
 	u8 mic_hdr[16] = {};
@@ -3185,78 +1513,16 @@ static int ath12k_dp_rx_h_michael_mic(struct crypto_shash *tfm, u8 *key,
 	shash_desc_zero(desc);
 	return ret;
 }
+EXPORT_SYMBOL(ath12k_dp_rx_h_michael_mic);
 
-static int ath12k_dp_rx_h_verify_tkip_mic(struct ath12k *ar, struct ath12k_peer *peer,
-					  struct sk_buff *msdu)
-{
-	struct ath12k_base *ab = ar->ab;
-	struct hal_rx_desc *rx_desc = (struct hal_rx_desc *)msdu->data;
-	struct ieee80211_rx_status *rxs = IEEE80211_SKB_RXCB(msdu);
-	struct ieee80211_key_conf *key_conf;
-	struct ieee80211_hdr *hdr;
-	struct ath12k_dp_rx_info rx_info;
-	u8 mic[IEEE80211_CCMP_MIC_LEN];
-	int head_len, tail_len, ret;
-	size_t data_len;
-	u32 hdr_len, hal_rx_desc_sz = ar->ab->hal.hal_desc_sz;
-	u8 *key, *data;
-	u8 key_idx;
-
-	if (ath12k_dp_rx_h_enctype(ab, rx_desc) != HAL_ENCRYPT_TYPE_TKIP_MIC)
-		return 0;
-
-	rx_info.addr2_present = false;
-	rx_info.rx_status = rxs;
-
-	hdr = (struct ieee80211_hdr *)(msdu->data + hal_rx_desc_sz);
-	hdr_len = ieee80211_hdrlen(hdr->frame_control);
-	head_len = hdr_len + hal_rx_desc_sz + IEEE80211_TKIP_IV_LEN;
-	tail_len = IEEE80211_CCMP_MIC_LEN + IEEE80211_TKIP_ICV_LEN + FCS_LEN;
-
-	if (!is_multicast_ether_addr(hdr->addr1))
-		key_idx = peer->ucast_keyidx;
-	else
-		key_idx = peer->mcast_keyidx;
-
-	key_conf = peer->keys[key_idx];
-
-	data = msdu->data + head_len;
-	data_len = msdu->len - head_len - tail_len;
-	key = &key_conf->key[NL80211_TKIP_DATA_OFFSET_RX_MIC_KEY];
-
-	ret = ath12k_dp_rx_h_michael_mic(peer->tfm_mmic, key, hdr, data, data_len, mic);
-	if (ret || memcmp(mic, data + data_len, IEEE80211_CCMP_MIC_LEN))
-		goto mic_fail;
-
-	return 0;
-
-mic_fail:
-	(ATH12K_SKB_RXCB(msdu))->is_first_msdu = true;
-	(ATH12K_SKB_RXCB(msdu))->is_last_msdu = true;
-
-	ath12k_dp_rx_h_fetch_info(ab, rx_desc, &rx_info);
-
-	rxs->flag |= RX_FLAG_MMIC_ERROR | RX_FLAG_MMIC_STRIPPED |
-		    RX_FLAG_IV_STRIPPED | RX_FLAG_DECRYPTED;
-	skb_pull(msdu, hal_rx_desc_sz);
-
-	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(ab, rx_desc, msdu)))
-		return -EINVAL;
-
-	ath12k_dp_rx_h_ppdu(ar, &rx_info);
-	ath12k_dp_rx_h_undecap(ar, msdu, rx_desc,
-			       HAL_ENCRYPT_TYPE_TKIP_MIC, rxs, true);
-	ieee80211_rx(ath12k_ar_to_hw(ar), msdu);
-	return -EINVAL;
-}
-
-static void ath12k_dp_rx_h_undecap_frag(struct ath12k *ar, struct sk_buff *msdu,
-					enum hal_encrypt_type enctype, u32 flags)
+void ath12k_dp_rx_h_undecap_frag(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu,
+				 enum hal_encrypt_type enctype, u32 flags)
 {
+	struct ath12k_dp *dp = dp_pdev->dp;
 	struct ieee80211_hdr *hdr;
 	size_t hdr_len;
 	size_t crypto_len;
-	u32 hal_rx_desc_sz = ar->ab->hal.hal_desc_sz;
+	u32 hal_rx_desc_sz = dp->ab->hal.hal_desc_sz;
 
 	if (!flags)
 		return;
@@ -3265,258 +1531,43 @@ static void ath12k_dp_rx_h_undecap_frag(struct ath12k *ar, struct sk_buff *msdu,
 
 	if (flags & RX_FLAG_MIC_STRIPPED)
 		skb_trim(msdu, msdu->len -
-			 ath12k_dp_rx_crypto_mic_len(ar, enctype));
+			 ath12k_dp_rx_crypto_mic_len(dp, enctype));
 
 	if (flags & RX_FLAG_ICV_STRIPPED)
 		skb_trim(msdu, msdu->len -
-			 ath12k_dp_rx_crypto_icv_len(ar, enctype));
+			 ath12k_dp_rx_crypto_icv_len(dp_pdev, enctype));
 
 	if (flags & RX_FLAG_IV_STRIPPED) {
 		hdr_len = ieee80211_hdrlen(hdr->frame_control);
-		crypto_len = ath12k_dp_rx_crypto_param_len(ar, enctype);
+		crypto_len = ath12k_dp_rx_crypto_param_len(dp_pdev, enctype);
 
 		memmove(msdu->data + hal_rx_desc_sz + crypto_len,
 			msdu->data + hal_rx_desc_sz, hdr_len);
 		skb_pull(msdu, crypto_len);
 	}
 }
+EXPORT_SYMBOL(ath12k_dp_rx_h_undecap_frag);
 
-static int ath12k_dp_rx_h_defrag(struct ath12k *ar,
-				 struct ath12k_peer *peer,
-				 struct ath12k_dp_rx_tid *rx_tid,
-				 struct sk_buff **defrag_skb)
-{
-	struct ath12k_base *ab = ar->ab;
-	struct hal_rx_desc *rx_desc;
-	struct sk_buff *skb, *first_frag, *last_frag;
-	struct ieee80211_hdr *hdr;
-	enum hal_encrypt_type enctype;
-	bool is_decrypted = false;
-	int msdu_len = 0;
-	int extra_space;
-	u32 flags, hal_rx_desc_sz = ar->ab->hal.hal_desc_sz;
-
-	first_frag = skb_peek(&rx_tid->rx_frags);
-	last_frag = skb_peek_tail(&rx_tid->rx_frags);
-
-	skb_queue_walk(&rx_tid->rx_frags, skb) {
-		flags = 0;
-		rx_desc = (struct hal_rx_desc *)skb->data;
-		hdr = (struct ieee80211_hdr *)(skb->data + hal_rx_desc_sz);
-
-		enctype = ath12k_dp_rx_h_enctype(ab, rx_desc);
-		if (enctype != HAL_ENCRYPT_TYPE_OPEN)
-			is_decrypted = ath12k_dp_rx_h_is_decrypted(ab,
-								   rx_desc);
-
-		if (is_decrypted) {
-			if (skb != first_frag)
-				flags |= RX_FLAG_IV_STRIPPED;
-			if (skb != last_frag)
-				flags |= RX_FLAG_ICV_STRIPPED |
-					 RX_FLAG_MIC_STRIPPED;
-		}
-
-		/* RX fragments are always raw packets */
-		if (skb != last_frag)
-			skb_trim(skb, skb->len - FCS_LEN);
-		ath12k_dp_rx_h_undecap_frag(ar, skb, enctype, flags);
-
-		if (skb != first_frag)
-			skb_pull(skb, hal_rx_desc_sz +
-				      ieee80211_hdrlen(hdr->frame_control));
-		msdu_len += skb->len;
-	}
-
-	extra_space = msdu_len - (DP_RX_BUFFER_SIZE + skb_tailroom(first_frag));
-	if (extra_space > 0 &&
-	    (pskb_expand_head(first_frag, 0, extra_space, GFP_ATOMIC) < 0))
-		return -ENOMEM;
-
-	__skb_unlink(first_frag, &rx_tid->rx_frags);
-	while ((skb = __skb_dequeue(&rx_tid->rx_frags))) {
-		skb_put_data(first_frag, skb->data, skb->len);
-		dev_kfree_skb_any(skb);
-	}
-
-	hdr = (struct ieee80211_hdr *)(first_frag->data + hal_rx_desc_sz);
-	hdr->frame_control &= ~__cpu_to_le16(IEEE80211_FCTL_MOREFRAGS);
-	ATH12K_SKB_RXCB(first_frag)->is_frag = 1;
-
-	if (ath12k_dp_rx_h_verify_tkip_mic(ar, peer, first_frag))
-		first_frag = NULL;
-
-	*defrag_skb = first_frag;
-	return 0;
-}
-
-static int ath12k_dp_rx_h_defrag_reo_reinject(struct ath12k *ar,
-					      struct ath12k_dp_rx_tid *rx_tid,
-					      struct sk_buff *defrag_skb)
-{
-	struct ath12k_base *ab = ar->ab;
-	struct ath12k_dp *dp = &ab->dp;
-	struct hal_rx_desc *rx_desc = (struct hal_rx_desc *)defrag_skb->data;
-	struct hal_reo_entrance_ring *reo_ent_ring;
-	struct hal_reo_dest_ring *reo_dest_ring;
-	struct dp_link_desc_bank *link_desc_banks;
-	struct hal_rx_msdu_link *msdu_link;
-	struct hal_rx_msdu_details *msdu0;
-	struct hal_srng *srng;
-	dma_addr_t link_paddr, buf_paddr;
-	u32 desc_bank, msdu_info, msdu_ext_info, mpdu_info;
-	u32 cookie, hal_rx_desc_sz, dest_ring_info0, queue_addr_hi;
-	int ret;
-	struct ath12k_rx_desc_info *desc_info;
-	enum hal_rx_buf_return_buf_manager idle_link_rbm = dp->idle_link_rbm;
-	u8 dst_ind;
-
-	hal_rx_desc_sz = ab->hal.hal_desc_sz;
-	link_desc_banks = dp->link_desc_banks;
-	reo_dest_ring = rx_tid->dst_ring_desc;
-
-	ath12k_hal_rx_reo_ent_paddr_get(ab, &reo_dest_ring->buf_addr_info,
-					&link_paddr, &cookie);
-	desc_bank = u32_get_bits(cookie, DP_LINK_DESC_BANK_MASK);
-
-	msdu_link = (struct hal_rx_msdu_link *)(link_desc_banks[desc_bank].vaddr +
-			(link_paddr - link_desc_banks[desc_bank].paddr));
-	msdu0 = &msdu_link->msdu_link[0];
-	msdu_ext_info = le32_to_cpu(msdu0->rx_msdu_ext_info.info0);
-	dst_ind = u32_get_bits(msdu_ext_info, RX_MSDU_EXT_DESC_INFO0_REO_DEST_IND);
-
-	memset(msdu0, 0, sizeof(*msdu0));
-
-	msdu_info = u32_encode_bits(1, RX_MSDU_DESC_INFO0_FIRST_MSDU_IN_MPDU) |
-		    u32_encode_bits(1, RX_MSDU_DESC_INFO0_LAST_MSDU_IN_MPDU) |
-		    u32_encode_bits(0, RX_MSDU_DESC_INFO0_MSDU_CONTINUATION) |
-		    u32_encode_bits(defrag_skb->len - hal_rx_desc_sz,
-				    RX_MSDU_DESC_INFO0_MSDU_LENGTH) |
-		    u32_encode_bits(1, RX_MSDU_DESC_INFO0_VALID_SA) |
-		    u32_encode_bits(1, RX_MSDU_DESC_INFO0_VALID_DA);
-	msdu0->rx_msdu_info.info0 = cpu_to_le32(msdu_info);
-	msdu0->rx_msdu_ext_info.info0 = cpu_to_le32(msdu_ext_info);
-
-	/* change msdu len in hal rx desc */
-	ath12k_dp_rxdesc_set_msdu_len(ab, rx_desc, defrag_skb->len - hal_rx_desc_sz);
-
-	buf_paddr = dma_map_single(ab->dev, defrag_skb->data,
-				   defrag_skb->len + skb_tailroom(defrag_skb),
-				   DMA_TO_DEVICE);
-	if (dma_mapping_error(ab->dev, buf_paddr))
-		return -ENOMEM;
-
-	spin_lock_bh(&dp->rx_desc_lock);
-	desc_info = list_first_entry_or_null(&dp->rx_desc_free_list,
-					     struct ath12k_rx_desc_info,
-					     list);
-	if (!desc_info) {
-		spin_unlock_bh(&dp->rx_desc_lock);
-		ath12k_warn(ab, "failed to find rx desc for reinject\n");
-		ret = -ENOMEM;
-		goto err_unmap_dma;
-	}
-
-	desc_info->skb = defrag_skb;
-	desc_info->in_use = true;
-
-	list_del(&desc_info->list);
-	spin_unlock_bh(&dp->rx_desc_lock);
-
-	ATH12K_SKB_RXCB(defrag_skb)->paddr = buf_paddr;
-
-	ath12k_hal_rx_buf_addr_info_set(&msdu0->buf_addr_info, buf_paddr,
-					desc_info->cookie,
-					HAL_RX_BUF_RBM_SW3_BM);
-
-	/* Fill mpdu details into reo entrance ring */
-	srng = &ab->hal.srng_list[dp->reo_reinject_ring.ring_id];
-
-	spin_lock_bh(&srng->lock);
-	ath12k_hal_srng_access_begin(ab, srng);
-
-	reo_ent_ring = ath12k_hal_srng_src_get_next_entry(ab, srng);
-	if (!reo_ent_ring) {
-		ath12k_hal_srng_access_end(ab, srng);
-		spin_unlock_bh(&srng->lock);
-		ret = -ENOSPC;
-		goto err_free_desc;
-	}
-	memset(reo_ent_ring, 0, sizeof(*reo_ent_ring));
-
-	ath12k_hal_rx_buf_addr_info_set(&reo_ent_ring->buf_addr_info, link_paddr,
-					cookie,
-					idle_link_rbm);
-
-	mpdu_info = u32_encode_bits(1, RX_MPDU_DESC_INFO0_MSDU_COUNT) |
-		    u32_encode_bits(0, RX_MPDU_DESC_INFO0_FRAG_FLAG) |
-		    u32_encode_bits(1, RX_MPDU_DESC_INFO0_RAW_MPDU) |
-		    u32_encode_bits(1, RX_MPDU_DESC_INFO0_VALID_PN) |
-		    u32_encode_bits(rx_tid->tid, RX_MPDU_DESC_INFO0_TID);
-
-	reo_ent_ring->rx_mpdu_info.info0 = cpu_to_le32(mpdu_info);
-	reo_ent_ring->rx_mpdu_info.peer_meta_data =
-		reo_dest_ring->rx_mpdu_info.peer_meta_data;
-
-	if (ab->hw_params->reoq_lut_support) {
-		reo_ent_ring->queue_addr_lo = reo_dest_ring->rx_mpdu_info.peer_meta_data;
-		queue_addr_hi = 0;
-	} else {
-		reo_ent_ring->queue_addr_lo =
-				cpu_to_le32(lower_32_bits(rx_tid->qbuf.paddr_aligned));
-		queue_addr_hi = upper_32_bits(rx_tid->qbuf.paddr_aligned);
-	}
-
-	reo_ent_ring->info0 = le32_encode_bits(queue_addr_hi,
-					       HAL_REO_ENTR_RING_INFO0_QUEUE_ADDR_HI) |
-			      le32_encode_bits(dst_ind,
-					       HAL_REO_ENTR_RING_INFO0_DEST_IND);
-
-	reo_ent_ring->info1 = le32_encode_bits(rx_tid->cur_sn,
-					       HAL_REO_ENTR_RING_INFO1_MPDU_SEQ_NUM);
-	dest_ring_info0 = le32_get_bits(reo_dest_ring->info0,
-					HAL_REO_DEST_RING_INFO0_SRC_LINK_ID);
-	reo_ent_ring->info2 =
-		cpu_to_le32(u32_get_bits(dest_ring_info0,
-					 HAL_REO_ENTR_RING_INFO2_SRC_LINK_ID));
-
-	ath12k_hal_srng_access_end(ab, srng);
-	spin_unlock_bh(&srng->lock);
-
-	return 0;
-
-err_free_desc:
-	spin_lock_bh(&dp->rx_desc_lock);
-	desc_info->in_use = false;
-	desc_info->skb = NULL;
-	list_add_tail(&desc_info->list, &dp->rx_desc_free_list);
-	spin_unlock_bh(&dp->rx_desc_lock);
-err_unmap_dma:
-	dma_unmap_single(ab->dev, buf_paddr, defrag_skb->len + skb_tailroom(defrag_skb),
-			 DMA_TO_DEVICE);
-	return ret;
-}
-
-static int ath12k_dp_rx_h_cmp_frags(struct ath12k_base *ab,
+static int ath12k_dp_rx_h_cmp_frags(struct ath12k_hal *hal,
 				    struct sk_buff *a, struct sk_buff *b)
 {
 	int frag1, frag2;
 
-	frag1 = ath12k_dp_rx_h_frag_no(ab, a);
-	frag2 = ath12k_dp_rx_h_frag_no(ab, b);
+	frag1 = ath12k_dp_rx_h_frag_no(hal, a);
+	frag2 = ath12k_dp_rx_h_frag_no(hal, b);
 
 	return frag1 - frag2;
 }
 
-static void ath12k_dp_rx_h_sort_frags(struct ath12k_base *ab,
-				      struct sk_buff_head *frag_list,
-				      struct sk_buff *cur_frag)
+void ath12k_dp_rx_h_sort_frags(struct ath12k_hal *hal,
+			       struct sk_buff_head *frag_list,
+			       struct sk_buff *cur_frag)
 {
 	struct sk_buff *skb;
 	int cmp;
 
 	skb_queue_walk(frag_list, skb) {
-		cmp = ath12k_dp_rx_h_cmp_frags(ab, skb, cur_frag);
+		cmp = ath12k_dp_rx_h_cmp_frags(hal, skb, cur_frag);
 		if (cmp < 0)
 			continue;
 		__skb_queue_before(frag_list, skb, cur_frag);
@@ -3524,13 +1575,14 @@ static void ath12k_dp_rx_h_sort_frags(struct ath12k_base *ab,
 	}
 	__skb_queue_tail(frag_list, cur_frag);
 }
+EXPORT_SYMBOL(ath12k_dp_rx_h_sort_frags);
 
-static u64 ath12k_dp_rx_h_get_pn(struct ath12k *ar, struct sk_buff *skb)
+u64 ath12k_dp_rx_h_get_pn(struct ath12k_dp *dp, struct sk_buff *skb)
 {
 	struct ieee80211_hdr *hdr;
 	u64 pn = 0;
 	u8 *ehdr;
-	u32 hal_rx_desc_sz = ar->ab->hal.hal_desc_sz;
+	u32 hal_rx_desc_sz = dp->ab->hal.hal_desc_sz;
 
 	hdr = (struct ieee80211_hdr *)(skb->data + hal_rx_desc_sz);
 	ehdr = skb->data + hal_rx_desc_sz + ieee80211_hdrlen(hdr->frame_control);
@@ -3544,1071 +1596,43 @@ static u64 ath12k_dp_rx_h_get_pn(struct ath12k *ar, struct sk_buff *skb)
 
 	return pn;
 }
+EXPORT_SYMBOL(ath12k_dp_rx_h_get_pn);
 
-static bool
-ath12k_dp_rx_h_defrag_validate_incr_pn(struct ath12k *ar, struct ath12k_dp_rx_tid *rx_tid)
-{
-	struct ath12k_base *ab = ar->ab;
-	enum hal_encrypt_type encrypt_type;
-	struct sk_buff *first_frag, *skb;
-	struct hal_rx_desc *desc;
-	u64 last_pn;
-	u64 cur_pn;
-
-	first_frag = skb_peek(&rx_tid->rx_frags);
-	desc = (struct hal_rx_desc *)first_frag->data;
-
-	encrypt_type = ath12k_dp_rx_h_enctype(ab, desc);
-	if (encrypt_type != HAL_ENCRYPT_TYPE_CCMP_128 &&
-	    encrypt_type != HAL_ENCRYPT_TYPE_CCMP_256 &&
-	    encrypt_type != HAL_ENCRYPT_TYPE_GCMP_128 &&
-	    encrypt_type != HAL_ENCRYPT_TYPE_AES_GCMP_256)
-		return true;
-
-	last_pn = ath12k_dp_rx_h_get_pn(ar, first_frag);
-	skb_queue_walk(&rx_tid->rx_frags, skb) {
-		if (skb == first_frag)
-			continue;
-
-		cur_pn = ath12k_dp_rx_h_get_pn(ar, skb);
-		if (cur_pn != last_pn + 1)
-			return false;
-		last_pn = cur_pn;
-	}
-	return true;
-}
-
-static int ath12k_dp_rx_frag_h_mpdu(struct ath12k *ar,
-				    struct sk_buff *msdu,
-				    struct hal_reo_dest_ring *ring_desc)
+void ath12k_dp_rx_free(struct ath12k_base *ab)
 {
-	struct ath12k_base *ab = ar->ab;
-	struct hal_rx_desc *rx_desc;
-	struct ath12k_peer *peer;
-	struct ath12k_dp_rx_tid *rx_tid;
-	struct sk_buff *defrag_skb = NULL;
-	u32 peer_id;
-	u16 seqno, frag_no;
-	u8 tid;
-	int ret = 0;
-	bool more_frags;
-
-	rx_desc = (struct hal_rx_desc *)msdu->data;
-	peer_id = ath12k_dp_rx_h_peer_id(ab, rx_desc);
-	tid = ath12k_dp_rx_h_tid(ab, rx_desc);
-	seqno = ath12k_dp_rx_h_seq_no(ab, rx_desc);
-	frag_no = ath12k_dp_rx_h_frag_no(ab, msdu);
-	more_frags = ath12k_dp_rx_h_more_frags(ab, msdu);
-
-	if (!ath12k_dp_rx_h_seq_ctrl_valid(ab, rx_desc) ||
-	    !ath12k_dp_rx_h_fc_valid(ab, rx_desc) ||
-	    tid > IEEE80211_NUM_TIDS)
-		return -EINVAL;
-
-	/* received unfragmented packet in reo
-	 * exception ring, this shouldn't happen
-	 * as these packets typically come from
-	 * reo2sw srngs.
-	 */
-	if (WARN_ON_ONCE(!frag_no && !more_frags))
-		return -EINVAL;
-
-	spin_lock_bh(&ab->base_lock);
-	peer = ath12k_peer_find_by_id(ab, peer_id);
-	if (!peer) {
-		ath12k_warn(ab, "failed to find the peer to de-fragment received fragment peer_id %d\n",
-			    peer_id);
-		ret = -ENOENT;
-		goto out_unlock;
-	}
-
-	if (!peer->dp_setup_done) {
-		ath12k_warn(ab, "The peer %pM [%d] has uninitialized datapath\n",
-			    peer->addr, peer_id);
-		ret = -ENOENT;
-		goto out_unlock;
-	}
-
-	rx_tid = &peer->rx_tid[tid];
-
-	if ((!skb_queue_empty(&rx_tid->rx_frags) && seqno != rx_tid->cur_sn) ||
-	    skb_queue_empty(&rx_tid->rx_frags)) {
-		/* Flush stored fragments and start a new sequence */
-		ath12k_dp_rx_frags_cleanup(rx_tid, true);
-		rx_tid->cur_sn = seqno;
-	}
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct dp_srng *srng;
+	int i;
 
-	if (rx_tid->rx_frag_bitmap & BIT(frag_no)) {
-		/* Fragment already present */
-		ret = -EINVAL;
-		goto out_unlock;
-	}
+	ath12k_dp_srng_cleanup(ab, &dp->rx_refill_buf_ring.refill_buf_ring);
 
-	if ((!rx_tid->rx_frag_bitmap || frag_no > __fls(rx_tid->rx_frag_bitmap)))
-		__skb_queue_tail(&rx_tid->rx_frags, msdu);
-	else
-		ath12k_dp_rx_h_sort_frags(ab, &rx_tid->rx_frags, msdu);
-
-	rx_tid->rx_frag_bitmap |= BIT(frag_no);
-	if (!more_frags)
-		rx_tid->last_frag_no = frag_no;
-
-	if (frag_no == 0) {
-		rx_tid->dst_ring_desc = kmemdup(ring_desc,
-						sizeof(*rx_tid->dst_ring_desc),
-						GFP_ATOMIC);
-		if (!rx_tid->dst_ring_desc) {
-			ret = -ENOMEM;
-			goto out_unlock;
+	for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) {
+		if (ab->hw_params->rx_mac_buf_ring)
+			ath12k_dp_srng_cleanup(ab, &dp->rx_mac_buf_ring[i]);
+		if (!ab->hw_params->rxdma1_enable) {
+			srng = &dp->rx_mon_status_refill_ring[i].refill_buf_ring;
+			ath12k_dp_srng_cleanup(ab, srng);
 		}
-	} else {
-		ath12k_dp_rx_link_desc_return(ab, &ring_desc->buf_addr_info,
-					      HAL_WBM_REL_BM_ACT_PUT_IN_IDLE);
-	}
-
-	if (!rx_tid->last_frag_no ||
-	    rx_tid->rx_frag_bitmap != GENMASK(rx_tid->last_frag_no, 0)) {
-		mod_timer(&rx_tid->frag_timer, jiffies +
-					       ATH12K_DP_RX_FRAGMENT_TIMEOUT_MS);
-		goto out_unlock;
 	}
 
-	spin_unlock_bh(&ab->base_lock);
-	timer_delete_sync(&rx_tid->frag_timer);
-	spin_lock_bh(&ab->base_lock);
-
-	peer = ath12k_peer_find_by_id(ab, peer_id);
-	if (!peer)
-		goto err_frags_cleanup;
-
-	if (!ath12k_dp_rx_h_defrag_validate_incr_pn(ar, rx_tid))
-		goto err_frags_cleanup;
-
-	if (ath12k_dp_rx_h_defrag(ar, peer, rx_tid, &defrag_skb))
-		goto err_frags_cleanup;
-
-	if (!defrag_skb)
-		goto err_frags_cleanup;
-
-	if (ath12k_dp_rx_h_defrag_reo_reinject(ar, rx_tid, defrag_skb))
-		goto err_frags_cleanup;
+	for (i = 0; i < ab->hw_params->num_rxdma_dst_ring; i++)
+		ath12k_dp_srng_cleanup(ab, &dp->rxdma_err_dst_ring[i]);
 
-	ath12k_dp_rx_frags_cleanup(rx_tid, false);
-	goto out_unlock;
+	ath12k_dp_srng_cleanup(ab, &dp->rxdma_mon_buf_ring.refill_buf_ring);
 
-err_frags_cleanup:
-	dev_kfree_skb_any(defrag_skb);
-	ath12k_dp_rx_frags_cleanup(rx_tid, true);
-out_unlock:
-	spin_unlock_bh(&ab->base_lock);
-	return ret;
+	ath12k_dp_rxdma_buf_free(ab);
 }
 
-static int
-ath12k_dp_process_rx_err_buf(struct ath12k *ar, struct hal_reo_dest_ring *desc,
-			     struct list_head *used_list,
-			     bool drop, u32 cookie)
+void ath12k_dp_rx_pdev_free(struct ath12k_base *ab, int mac_id)
 {
-	struct ath12k_base *ab = ar->ab;
-	struct sk_buff *msdu;
-	struct ath12k_skb_rxcb *rxcb;
-	struct hal_rx_desc *rx_desc;
-	u16 msdu_len;
-	u32 hal_rx_desc_sz = ab->hal.hal_desc_sz;
-	struct ath12k_rx_desc_info *desc_info;
-	u64 desc_va;
-
-	desc_va = ((u64)le32_to_cpu(desc->buf_va_hi) << 32 |
-		   le32_to_cpu(desc->buf_va_lo));
-	desc_info = (struct ath12k_rx_desc_info *)((unsigned long)desc_va);
-
-	/* retry manual desc retrieval */
-	if (!desc_info) {
-		desc_info = ath12k_dp_get_rx_desc(ab, cookie);
-		if (!desc_info) {
-			ath12k_warn(ab, "Invalid cookie in DP rx error descriptor retrieval: 0x%x\n",
-				    cookie);
-			return -EINVAL;
-		}
-	}
-
-	if (desc_info->magic != ATH12K_DP_RX_DESC_MAGIC)
-		ath12k_warn(ab, " RX Exception, Check HW CC implementation");
-
-	msdu = desc_info->skb;
-	desc_info->skb = NULL;
-
-	list_add_tail(&desc_info->list, used_list);
-
-	rxcb = ATH12K_SKB_RXCB(msdu);
-	dma_unmap_single(ar->ab->dev, rxcb->paddr,
-			 msdu->len + skb_tailroom(msdu),
-			 DMA_FROM_DEVICE);
-
-	if (drop) {
-		dev_kfree_skb_any(msdu);
-		return 0;
-	}
-
-	rcu_read_lock();
-	if (!rcu_dereference(ar->ab->pdevs_active[ar->pdev_idx])) {
-		dev_kfree_skb_any(msdu);
-		goto exit;
-	}
-
-	if (test_bit(ATH12K_FLAG_CAC_RUNNING, &ar->dev_flags)) {
-		dev_kfree_skb_any(msdu);
-		goto exit;
-	}
-
-	rx_desc = (struct hal_rx_desc *)msdu->data;
-	msdu_len = ath12k_dp_rx_h_msdu_len(ar->ab, rx_desc);
-	if ((msdu_len + hal_rx_desc_sz) > DP_RX_BUFFER_SIZE) {
-		ath12k_warn(ar->ab, "invalid msdu leng %u", msdu_len);
-		ath12k_dbg_dump(ar->ab, ATH12K_DBG_DATA, NULL, "", rx_desc,
-				sizeof(*rx_desc));
-		dev_kfree_skb_any(msdu);
-		goto exit;
-	}
-
-	skb_put(msdu, hal_rx_desc_sz + msdu_len);
-
-	if (ath12k_dp_rx_frag_h_mpdu(ar, msdu, desc)) {
-		dev_kfree_skb_any(msdu);
-		ath12k_dp_rx_link_desc_return(ar->ab, &desc->buf_addr_info,
-					      HAL_WBM_REL_BM_ACT_PUT_IN_IDLE);
-	}
-exit:
-	rcu_read_unlock();
-	return 0;
-}
-
-static int ath12k_dp_h_msdu_buffer_type(struct ath12k_base *ab,
-					struct list_head *list,
-					struct hal_reo_dest_ring *desc)
-{
-	struct ath12k_rx_desc_info *desc_info;
-	struct ath12k_skb_rxcb *rxcb;
-	struct sk_buff *msdu;
-	u64 desc_va;
-
-	ab->device_stats.reo_excep_msdu_buf_type++;
-
-	desc_va = (u64)le32_to_cpu(desc->buf_va_hi) << 32 |
-		  le32_to_cpu(desc->buf_va_lo);
-	desc_info = (struct ath12k_rx_desc_info *)(uintptr_t)desc_va;
-	if (!desc_info) {
-		u32 cookie;
-
-		cookie = le32_get_bits(desc->buf_addr_info.info1,
-				       BUFFER_ADDR_INFO1_SW_COOKIE);
-		desc_info = ath12k_dp_get_rx_desc(ab, cookie);
-		if (!desc_info) {
-			ath12k_warn(ab, "Invalid cookie in manual descriptor retrieval: 0x%x\n",
-				    cookie);
-			return -EINVAL;
-		}
-	}
-
-	if (desc_info->magic != ATH12K_DP_RX_DESC_MAGIC) {
-		ath12k_warn(ab, "rx exception, magic check failed with value: %u\n",
-			    desc_info->magic);
-		return -EINVAL;
-	}
-
-	msdu = desc_info->skb;
-	desc_info->skb = NULL;
-	list_add_tail(&desc_info->list, list);
-	rxcb = ATH12K_SKB_RXCB(msdu);
-	dma_unmap_single(ab->dev, rxcb->paddr, msdu->len + skb_tailroom(msdu),
-			 DMA_FROM_DEVICE);
-	dev_kfree_skb_any(msdu);
-
-	return 0;
-}
-
-int ath12k_dp_rx_process_err(struct ath12k_base *ab, struct napi_struct *napi,
-			     int budget)
-{
-	struct ath12k_hw_group *ag = ab->ag;
-	struct list_head rx_desc_used_list[ATH12K_MAX_DEVICES];
-	u32 msdu_cookies[HAL_NUM_RX_MSDUS_PER_LINK_DESC];
-	int num_buffs_reaped[ATH12K_MAX_DEVICES] = {};
-	struct dp_link_desc_bank *link_desc_banks;
-	enum hal_rx_buf_return_buf_manager rbm;
-	struct hal_rx_msdu_link *link_desc_va;
-	int tot_n_bufs_reaped, quota, ret, i;
-	struct hal_reo_dest_ring *reo_desc;
-	struct dp_rxdma_ring *rx_ring;
-	struct dp_srng *reo_except;
-	struct ath12k_hw_link *hw_links = ag->hw_links;
-	struct ath12k_base *partner_ab;
-	u8 hw_link_id, device_id;
-	u32 desc_bank, num_msdus;
-	struct hal_srng *srng;
-	struct ath12k *ar;
-	dma_addr_t paddr;
-	bool is_frag;
-	bool drop;
-	int pdev_id;
-
-	tot_n_bufs_reaped = 0;
-	quota = budget;
-
-	for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++)
-		INIT_LIST_HEAD(&rx_desc_used_list[device_id]);
-
-	reo_except = &ab->dp.reo_except_ring;
-
-	srng = &ab->hal.srng_list[reo_except->ring_id];
-
-	spin_lock_bh(&srng->lock);
-
-	ath12k_hal_srng_access_begin(ab, srng);
-
-	while (budget &&
-	       (reo_desc = ath12k_hal_srng_dst_get_next_entry(ab, srng))) {
-		drop = false;
-		ab->device_stats.err_ring_pkts++;
-
-		hw_link_id = le32_get_bits(reo_desc->info0,
-					   HAL_REO_DEST_RING_INFO0_SRC_LINK_ID);
-		device_id = hw_links[hw_link_id].device_id;
-		partner_ab = ath12k_ag_to_ab(ag, device_id);
-
-		/* Below case is added to handle data packet from un-associated clients.
-		 * As it is expected that AST lookup will fail for
-		 * un-associated station's data packets.
-		 */
-		if (le32_get_bits(reo_desc->info0, HAL_REO_DEST_RING_INFO0_BUFFER_TYPE) ==
-		    HAL_REO_DEST_RING_BUFFER_TYPE_MSDU) {
-			if (!ath12k_dp_h_msdu_buffer_type(partner_ab,
-							  &rx_desc_used_list[device_id],
-							  reo_desc)) {
-				num_buffs_reaped[device_id]++;
-				tot_n_bufs_reaped++;
-			}
-			goto next_desc;
-		}
-
-		ret = ath12k_hal_desc_reo_parse_err(ab, reo_desc, &paddr,
-						    &desc_bank);
-		if (ret) {
-			ath12k_warn(ab, "failed to parse error reo desc %d\n",
-				    ret);
-			continue;
-		}
-
-		pdev_id = ath12k_hw_mac_id_to_pdev_id(partner_ab->hw_params,
-						      hw_links[hw_link_id].pdev_idx);
-		ar = partner_ab->pdevs[pdev_id].ar;
-
-		link_desc_banks = partner_ab->dp.link_desc_banks;
-		link_desc_va = link_desc_banks[desc_bank].vaddr +
-			       (paddr - link_desc_banks[desc_bank].paddr);
-		ath12k_hal_rx_msdu_link_info_get(link_desc_va, &num_msdus, msdu_cookies,
-						 &rbm);
-		if (rbm != partner_ab->dp.idle_link_rbm &&
-		    rbm != HAL_RX_BUF_RBM_SW3_BM &&
-		    rbm != partner_ab->hw_params->hal_params->rx_buf_rbm) {
-			ab->device_stats.invalid_rbm++;
-			ath12k_warn(ab, "invalid return buffer manager %d\n", rbm);
-			ath12k_dp_rx_link_desc_return(partner_ab,
-						      &reo_desc->buf_addr_info,
-						      HAL_WBM_REL_BM_ACT_REL_MSDU);
-			continue;
-		}
-
-		is_frag = !!(le32_to_cpu(reo_desc->rx_mpdu_info.info0) &
-			     RX_MPDU_DESC_INFO0_FRAG_FLAG);
-
-		/* Process only rx fragments with one msdu per link desc below, and drop
-		 * msdu's indicated due to error reasons.
-		 * Dynamic fragmentation not supported in Multi-link client, so drop the
-		 * partner device buffers.
-		 */
-		if (!is_frag || num_msdus > 1 ||
-		    partner_ab->device_id != ab->device_id) {
-			drop = true;
-
-			/* Return the link desc back to wbm idle list */
-			ath12k_dp_rx_link_desc_return(partner_ab,
-						      &reo_desc->buf_addr_info,
-						      HAL_WBM_REL_BM_ACT_PUT_IN_IDLE);
-		}
-
-		for (i = 0; i < num_msdus; i++) {
-			if (!ath12k_dp_process_rx_err_buf(ar, reo_desc,
-							  &rx_desc_used_list[device_id],
-							  drop,
-							  msdu_cookies[i])) {
-				num_buffs_reaped[device_id]++;
-				tot_n_bufs_reaped++;
-			}
-		}
-
-next_desc:
-		if (tot_n_bufs_reaped >= quota) {
-			tot_n_bufs_reaped = quota;
-			goto exit;
-		}
-
-		budget = quota - tot_n_bufs_reaped;
-	}
-
-exit:
-	ath12k_hal_srng_access_end(ab, srng);
-
-	spin_unlock_bh(&srng->lock);
-
-	for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) {
-		if (!num_buffs_reaped[device_id])
-			continue;
-
-		partner_ab = ath12k_ag_to_ab(ag, device_id);
-		rx_ring = &partner_ab->dp.rx_refill_buf_ring;
-
-		ath12k_dp_rx_bufs_replenish(partner_ab, rx_ring,
-					    &rx_desc_used_list[device_id],
-					    num_buffs_reaped[device_id]);
-	}
-
-	return tot_n_bufs_reaped;
-}
-
-static void ath12k_dp_rx_null_q_desc_sg_drop(struct ath12k *ar,
-					     int msdu_len,
-					     struct sk_buff_head *msdu_list)
-{
-	struct sk_buff *skb, *tmp;
-	struct ath12k_skb_rxcb *rxcb;
-	int n_buffs;
-
-	n_buffs = DIV_ROUND_UP(msdu_len,
-			       (DP_RX_BUFFER_SIZE - ar->ab->hal.hal_desc_sz));
-
-	skb_queue_walk_safe(msdu_list, skb, tmp) {
-		rxcb = ATH12K_SKB_RXCB(skb);
-		if (rxcb->err_rel_src == HAL_WBM_REL_SRC_MODULE_REO &&
-		    rxcb->err_code == HAL_REO_DEST_RING_ERROR_CODE_DESC_ADDR_ZERO) {
-			if (!n_buffs)
-				break;
-			__skb_unlink(skb, msdu_list);
-			dev_kfree_skb_any(skb);
-			n_buffs--;
-		}
-	}
-}
-
-static int ath12k_dp_rx_h_null_q_desc(struct ath12k *ar, struct sk_buff *msdu,
-				      struct ath12k_dp_rx_info *rx_info,
-				      struct sk_buff_head *msdu_list)
-{
-	struct ath12k_base *ab = ar->ab;
-	u16 msdu_len;
-	struct hal_rx_desc *desc = (struct hal_rx_desc *)msdu->data;
-	u8 l3pad_bytes;
-	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
-	u32 hal_rx_desc_sz = ar->ab->hal.hal_desc_sz;
-
-	msdu_len = ath12k_dp_rx_h_msdu_len(ab, desc);
-
-	if (!rxcb->is_frag && ((msdu_len + hal_rx_desc_sz) > DP_RX_BUFFER_SIZE)) {
-		/* First buffer will be freed by the caller, so deduct it's length */
-		msdu_len = msdu_len - (DP_RX_BUFFER_SIZE - hal_rx_desc_sz);
-		ath12k_dp_rx_null_q_desc_sg_drop(ar, msdu_len, msdu_list);
-		return -EINVAL;
-	}
-
-	/* Even after cleaning up the sg buffers in the msdu list with above check
-	 * any msdu received with continuation flag needs to be dropped as invalid.
-	 * This protects against some random err frame with continuation flag.
-	 */
-	if (rxcb->is_continuation)
-		return -EINVAL;
-
-	if (!ath12k_dp_rx_h_msdu_done(ab, desc)) {
-		ath12k_warn(ar->ab,
-			    "msdu_done bit not set in null_q_des processing\n");
-		__skb_queue_purge(msdu_list);
-		return -EIO;
-	}
-
-	/* Handle NULL queue descriptor violations arising out a missing
-	 * REO queue for a given peer or a given TID. This typically
-	 * may happen if a packet is received on a QOS enabled TID before the
-	 * ADDBA negotiation for that TID, when the TID queue is setup. Or
-	 * it may also happen for MC/BC frames if they are not routed to the
-	 * non-QOS TID queue, in the absence of any other default TID queue.
-	 * This error can show up both in a REO destination or WBM release ring.
-	 */
-
-	if (rxcb->is_frag) {
-		skb_pull(msdu, hal_rx_desc_sz);
-	} else {
-		l3pad_bytes = ath12k_dp_rx_h_l3pad(ab, desc);
-
-		if ((hal_rx_desc_sz + l3pad_bytes + msdu_len) > DP_RX_BUFFER_SIZE)
-			return -EINVAL;
-
-		skb_put(msdu, hal_rx_desc_sz + l3pad_bytes + msdu_len);
-		skb_pull(msdu, hal_rx_desc_sz + l3pad_bytes);
-	}
-	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(ab, desc, msdu)))
-		return -EINVAL;
-
-	ath12k_dp_rx_h_fetch_info(ab, desc, rx_info);
-	ath12k_dp_rx_h_ppdu(ar, rx_info);
-	ath12k_dp_rx_h_mpdu(ar, msdu, desc, rx_info);
-
-	rxcb->tid = rx_info->tid;
-
-	/* Please note that caller will having the access to msdu and completing
-	 * rx with mac80211. Need not worry about cleaning up amsdu_list.
-	 */
-
-	return 0;
-}
-
-static bool ath12k_dp_rx_h_reo_err(struct ath12k *ar, struct sk_buff *msdu,
-				   struct ath12k_dp_rx_info *rx_info,
-				   struct sk_buff_head *msdu_list)
-{
-	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
-	bool drop = false;
-
-	ar->ab->device_stats.reo_error[rxcb->err_code]++;
-
-	switch (rxcb->err_code) {
-	case HAL_REO_DEST_RING_ERROR_CODE_DESC_ADDR_ZERO:
-		if (ath12k_dp_rx_h_null_q_desc(ar, msdu, rx_info, msdu_list))
-			drop = true;
-		break;
-	case HAL_REO_DEST_RING_ERROR_CODE_PN_CHECK_FAILED:
-		/* TODO: Do not drop PN failed packets in the driver;
-		 * instead, it is good to drop such packets in mac80211
-		 * after incrementing the replay counters.
-		 */
-		fallthrough;
-	default:
-		/* TODO: Review other errors and process them to mac80211
-		 * as appropriate.
-		 */
-		drop = true;
-		break;
-	}
-
-	return drop;
-}
-
-static bool ath12k_dp_rx_h_tkip_mic_err(struct ath12k *ar, struct sk_buff *msdu,
-					struct ath12k_dp_rx_info *rx_info)
-{
-	struct ath12k_base *ab = ar->ab;
-	u16 msdu_len;
-	struct hal_rx_desc *desc = (struct hal_rx_desc *)msdu->data;
-	u8 l3pad_bytes;
-	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
-	u32 hal_rx_desc_sz = ar->ab->hal.hal_desc_sz;
-
-	rxcb->is_first_msdu = ath12k_dp_rx_h_first_msdu(ab, desc);
-	rxcb->is_last_msdu = ath12k_dp_rx_h_last_msdu(ab, desc);
-
-	l3pad_bytes = ath12k_dp_rx_h_l3pad(ab, desc);
-	msdu_len = ath12k_dp_rx_h_msdu_len(ab, desc);
-
-	if ((hal_rx_desc_sz + l3pad_bytes + msdu_len) > DP_RX_BUFFER_SIZE) {
-		ath12k_dbg(ab, ATH12K_DBG_DATA,
-			   "invalid msdu len in tkip mic err %u\n", msdu_len);
-		ath12k_dbg_dump(ab, ATH12K_DBG_DATA, NULL, "", desc,
-				sizeof(*desc));
-		return true;
-	}
-
-	skb_put(msdu, hal_rx_desc_sz + l3pad_bytes + msdu_len);
-	skb_pull(msdu, hal_rx_desc_sz + l3pad_bytes);
-
-	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(ab, desc, msdu)))
-		return true;
-
-	ath12k_dp_rx_h_ppdu(ar, rx_info);
-
-	rx_info->rx_status->flag |= (RX_FLAG_MMIC_STRIPPED | RX_FLAG_MMIC_ERROR |
-				     RX_FLAG_DECRYPTED);
-
-	ath12k_dp_rx_h_undecap(ar, msdu, desc,
-			       HAL_ENCRYPT_TYPE_TKIP_MIC, rx_info->rx_status, false);
-	return false;
-}
-
-static bool ath12k_dp_rx_h_rxdma_err(struct ath12k *ar,  struct sk_buff *msdu,
-				     struct ath12k_dp_rx_info *rx_info)
-{
-	struct ath12k_base *ab = ar->ab;
-	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
-	struct hal_rx_desc *rx_desc = (struct hal_rx_desc *)msdu->data;
-	bool drop = false;
-	u32 err_bitmap;
-
-	ar->ab->device_stats.rxdma_error[rxcb->err_code]++;
-
-	switch (rxcb->err_code) {
-	case HAL_REO_ENTR_RING_RXDMA_ECODE_DECRYPT_ERR:
-	case HAL_REO_ENTR_RING_RXDMA_ECODE_TKIP_MIC_ERR:
-		err_bitmap = ath12k_dp_rx_h_mpdu_err(ab, rx_desc);
-		if (err_bitmap & HAL_RX_MPDU_ERR_TKIP_MIC) {
-			ath12k_dp_rx_h_fetch_info(ab, rx_desc, rx_info);
-			drop = ath12k_dp_rx_h_tkip_mic_err(ar, msdu, rx_info);
-			break;
-		}
-		fallthrough;
-	default:
-		/* TODO: Review other rxdma error code to check if anything is
-		 * worth reporting to mac80211
-		 */
-		drop = true;
-		break;
-	}
-
-	return drop;
-}
-
-static void ath12k_dp_rx_wbm_err(struct ath12k *ar,
-				 struct napi_struct *napi,
-				 struct sk_buff *msdu,
-				 struct sk_buff_head *msdu_list)
-{
-	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
-	struct ieee80211_rx_status rxs = {};
-	struct ath12k_dp_rx_info rx_info;
-	bool drop = true;
-
-	rx_info.addr2_present = false;
-	rx_info.rx_status = &rxs;
-
-	switch (rxcb->err_rel_src) {
-	case HAL_WBM_REL_SRC_MODULE_REO:
-		drop = ath12k_dp_rx_h_reo_err(ar, msdu, &rx_info, msdu_list);
-		break;
-	case HAL_WBM_REL_SRC_MODULE_RXDMA:
-		drop = ath12k_dp_rx_h_rxdma_err(ar, msdu, &rx_info);
-		break;
-	default:
-		/* msdu will get freed */
-		break;
-	}
-
-	if (drop) {
-		dev_kfree_skb_any(msdu);
-		return;
-	}
-
-	rx_info.rx_status->flag |= RX_FLAG_SKIP_MONITOR;
-
-	ath12k_dp_rx_deliver_msdu(ar, napi, msdu, &rx_info);
-}
-
-int ath12k_dp_rx_process_wbm_err(struct ath12k_base *ab,
-				 struct napi_struct *napi, int budget)
-{
-	struct list_head rx_desc_used_list[ATH12K_MAX_DEVICES];
-	struct ath12k_hw_group *ag = ab->ag;
-	struct ath12k *ar;
-	struct ath12k_dp *dp = &ab->dp;
-	struct dp_rxdma_ring *rx_ring;
-	struct hal_rx_wbm_rel_info err_info;
-	struct hal_srng *srng;
-	struct sk_buff *msdu;
-	struct sk_buff_head msdu_list, scatter_msdu_list;
-	struct ath12k_skb_rxcb *rxcb;
-	void *rx_desc;
-	int num_buffs_reaped[ATH12K_MAX_DEVICES] = {};
-	int total_num_buffs_reaped = 0;
-	struct ath12k_rx_desc_info *desc_info;
-	struct ath12k_device_dp_stats *device_stats = &ab->device_stats;
-	struct ath12k_hw_link *hw_links = ag->hw_links;
-	struct ath12k_base *partner_ab;
-	u8 hw_link_id, device_id;
-	int ret, pdev_id;
-	struct hal_rx_desc *msdu_data;
-
-	__skb_queue_head_init(&msdu_list);
-	__skb_queue_head_init(&scatter_msdu_list);
-
-	for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++)
-		INIT_LIST_HEAD(&rx_desc_used_list[device_id]);
-
-	srng = &ab->hal.srng_list[dp->rx_rel_ring.ring_id];
-	spin_lock_bh(&srng->lock);
-
-	ath12k_hal_srng_access_begin(ab, srng);
-
-	while (budget) {
-		rx_desc = ath12k_hal_srng_dst_get_next_entry(ab, srng);
-		if (!rx_desc)
-			break;
-
-		ret = ath12k_hal_wbm_desc_parse_err(ab, rx_desc, &err_info);
-		if (ret) {
-			ath12k_warn(ab,
-				    "failed to parse rx error in wbm_rel ring desc %d\n",
-				    ret);
-			continue;
-		}
-
-		desc_info = err_info.rx_desc;
-
-		/* retry manual desc retrieval if hw cc is not done */
-		if (!desc_info) {
-			desc_info = ath12k_dp_get_rx_desc(ab, err_info.cookie);
-			if (!desc_info) {
-				ath12k_warn(ab, "Invalid cookie in DP WBM rx error descriptor retrieval: 0x%x\n",
-					    err_info.cookie);
-				continue;
-			}
-		}
-
-		if (desc_info->magic != ATH12K_DP_RX_DESC_MAGIC)
-			ath12k_warn(ab, "WBM RX err, Check HW CC implementation");
-
-		msdu = desc_info->skb;
-		desc_info->skb = NULL;
-
-		device_id = desc_info->device_id;
-		partner_ab = ath12k_ag_to_ab(ag, device_id);
-		if (unlikely(!partner_ab)) {
-			dev_kfree_skb_any(msdu);
-
-			/* In any case continuation bit is set
-			 * in the previous record, cleanup scatter_msdu_list
-			 */
-			ath12k_dp_clean_up_skb_list(&scatter_msdu_list);
-			continue;
-		}
-
-		list_add_tail(&desc_info->list, &rx_desc_used_list[device_id]);
-
-		rxcb = ATH12K_SKB_RXCB(msdu);
-		dma_unmap_single(partner_ab->dev, rxcb->paddr,
-				 msdu->len + skb_tailroom(msdu),
-				 DMA_FROM_DEVICE);
-
-		num_buffs_reaped[device_id]++;
-		total_num_buffs_reaped++;
-
-		if (!err_info.continuation)
-			budget--;
-
-		if (err_info.push_reason !=
-		    HAL_REO_DEST_RING_PUSH_REASON_ERR_DETECTED) {
-			dev_kfree_skb_any(msdu);
-			continue;
-		}
-
-		msdu_data = (struct hal_rx_desc *)msdu->data;
-		rxcb->err_rel_src = err_info.err_rel_src;
-		rxcb->err_code = err_info.err_code;
-		rxcb->is_first_msdu = err_info.first_msdu;
-		rxcb->is_last_msdu = err_info.last_msdu;
-		rxcb->is_continuation = err_info.continuation;
-		rxcb->rx_desc = msdu_data;
-
-		if (err_info.continuation) {
-			__skb_queue_tail(&scatter_msdu_list, msdu);
-			continue;
-		}
-
-		hw_link_id = ath12k_dp_rx_get_msdu_src_link(partner_ab,
-							    msdu_data);
-		if (hw_link_id >= ATH12K_GROUP_MAX_RADIO) {
-			dev_kfree_skb_any(msdu);
-
-			/* In any case continuation bit is set
-			 * in the previous record, cleanup scatter_msdu_list
-			 */
-			ath12k_dp_clean_up_skb_list(&scatter_msdu_list);
-			continue;
-		}
-
-		if (!skb_queue_empty(&scatter_msdu_list)) {
-			struct sk_buff *msdu;
-
-			skb_queue_walk(&scatter_msdu_list, msdu) {
-				rxcb = ATH12K_SKB_RXCB(msdu);
-				rxcb->hw_link_id = hw_link_id;
-			}
-
-			skb_queue_splice_tail_init(&scatter_msdu_list,
-						   &msdu_list);
-		}
-
-		rxcb = ATH12K_SKB_RXCB(msdu);
-		rxcb->hw_link_id = hw_link_id;
-		__skb_queue_tail(&msdu_list, msdu);
-	}
-
-	/* In any case continuation bit is set in the
-	 * last record, cleanup scatter_msdu_list
-	 */
-	ath12k_dp_clean_up_skb_list(&scatter_msdu_list);
-
-	ath12k_hal_srng_access_end(ab, srng);
-
-	spin_unlock_bh(&srng->lock);
-
-	if (!total_num_buffs_reaped)
-		goto done;
-
-	for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) {
-		if (!num_buffs_reaped[device_id])
-			continue;
-
-		partner_ab = ath12k_ag_to_ab(ag, device_id);
-		rx_ring = &partner_ab->dp.rx_refill_buf_ring;
-
-		ath12k_dp_rx_bufs_replenish(ab, rx_ring,
-					    &rx_desc_used_list[device_id],
-					    num_buffs_reaped[device_id]);
-	}
-
-	rcu_read_lock();
-	while ((msdu = __skb_dequeue(&msdu_list))) {
-		rxcb = ATH12K_SKB_RXCB(msdu);
-		hw_link_id = rxcb->hw_link_id;
-
-		device_id = hw_links[hw_link_id].device_id;
-		partner_ab = ath12k_ag_to_ab(ag, device_id);
-		if (unlikely(!partner_ab)) {
-			ath12k_dbg(ab, ATH12K_DBG_DATA,
-				   "Unable to process WBM error msdu due to invalid hw link id %d device id %d\n",
-				   hw_link_id, device_id);
-			dev_kfree_skb_any(msdu);
-			continue;
-		}
-
-		pdev_id = ath12k_hw_mac_id_to_pdev_id(partner_ab->hw_params,
-						      hw_links[hw_link_id].pdev_idx);
-		ar = partner_ab->pdevs[pdev_id].ar;
-
-		if (!ar || !rcu_dereference(ar->ab->pdevs_active[pdev_id])) {
-			dev_kfree_skb_any(msdu);
-			continue;
-		}
-
-		if (test_bit(ATH12K_FLAG_CAC_RUNNING, &ar->dev_flags)) {
-			dev_kfree_skb_any(msdu);
-			continue;
-		}
-
-		if (rxcb->err_rel_src < HAL_WBM_REL_SRC_MODULE_MAX) {
-			device_id = ar->ab->device_id;
-			device_stats->rx_wbm_rel_source[rxcb->err_rel_src][device_id]++;
-		}
-
-		ath12k_dp_rx_wbm_err(ar, napi, msdu, &msdu_list);
-	}
-	rcu_read_unlock();
-done:
-	return total_num_buffs_reaped;
-}
-
-void ath12k_dp_rx_process_reo_status(struct ath12k_base *ab)
-{
-	struct ath12k_dp *dp = &ab->dp;
-	struct hal_tlv_64_hdr *hdr;
-	struct hal_srng *srng;
-	struct ath12k_dp_rx_reo_cmd *cmd, *tmp;
-	bool found = false;
-	u16 tag;
-	struct hal_reo_status reo_status;
-
-	srng = &ab->hal.srng_list[dp->reo_status_ring.ring_id];
-
-	memset(&reo_status, 0, sizeof(reo_status));
-
-	spin_lock_bh(&srng->lock);
-
-	ath12k_hal_srng_access_begin(ab, srng);
-
-	while ((hdr = ath12k_hal_srng_dst_get_next_entry(ab, srng))) {
-		tag = le64_get_bits(hdr->tl, HAL_SRNG_TLV_HDR_TAG);
-
-		switch (tag) {
-		case HAL_REO_GET_QUEUE_STATS_STATUS:
-			ath12k_hal_reo_status_queue_stats(ab, hdr,
-							  &reo_status);
-			break;
-		case HAL_REO_FLUSH_QUEUE_STATUS:
-			ath12k_hal_reo_flush_queue_status(ab, hdr,
-							  &reo_status);
-			break;
-		case HAL_REO_FLUSH_CACHE_STATUS:
-			ath12k_hal_reo_flush_cache_status(ab, hdr,
-							  &reo_status);
-			break;
-		case HAL_REO_UNBLOCK_CACHE_STATUS:
-			ath12k_hal_reo_unblk_cache_status(ab, hdr,
-							  &reo_status);
-			break;
-		case HAL_REO_FLUSH_TIMEOUT_LIST_STATUS:
-			ath12k_hal_reo_flush_timeout_list_status(ab, hdr,
-								 &reo_status);
-			break;
-		case HAL_REO_DESCRIPTOR_THRESHOLD_REACHED_STATUS:
-			ath12k_hal_reo_desc_thresh_reached_status(ab, hdr,
-								  &reo_status);
-			break;
-		case HAL_REO_UPDATE_RX_REO_QUEUE_STATUS:
-			ath12k_hal_reo_update_rx_reo_queue_status(ab, hdr,
-								  &reo_status);
-			break;
-		default:
-			ath12k_warn(ab, "Unknown reo status type %d\n", tag);
-			continue;
-		}
-
-		spin_lock_bh(&dp->reo_cmd_lock);
-		list_for_each_entry_safe(cmd, tmp, &dp->reo_cmd_list, list) {
-			if (reo_status.uniform_hdr.cmd_num == cmd->cmd_num) {
-				found = true;
-				list_del(&cmd->list);
-				break;
-			}
-		}
-		spin_unlock_bh(&dp->reo_cmd_lock);
-
-		if (found) {
-			cmd->handler(dp, (void *)&cmd->data,
-				     reo_status.uniform_hdr.cmd_status);
-			kfree(cmd);
-		}
-
-		found = false;
-	}
-
-	ath12k_hal_srng_access_end(ab, srng);
-
-	spin_unlock_bh(&srng->lock);
-}
-
-void ath12k_dp_rx_free(struct ath12k_base *ab)
-{
-	struct ath12k_dp *dp = &ab->dp;
-	struct dp_srng *srng;
-	int i;
-
-	ath12k_dp_srng_cleanup(ab, &dp->rx_refill_buf_ring.refill_buf_ring);
-
-	for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) {
-		if (ab->hw_params->rx_mac_buf_ring)
-			ath12k_dp_srng_cleanup(ab, &dp->rx_mac_buf_ring[i]);
-		if (!ab->hw_params->rxdma1_enable) {
-			srng = &dp->rx_mon_status_refill_ring[i].refill_buf_ring;
-			ath12k_dp_srng_cleanup(ab, srng);
-		}
-	}
-
-	for (i = 0; i < ab->hw_params->num_rxdma_dst_ring; i++)
-		ath12k_dp_srng_cleanup(ab, &dp->rxdma_err_dst_ring[i]);
-
-	ath12k_dp_srng_cleanup(ab, &dp->rxdma_mon_buf_ring.refill_buf_ring);
-
-	ath12k_dp_rxdma_buf_free(ab);
-}
-
-void ath12k_dp_rx_pdev_free(struct ath12k_base *ab, int mac_id)
-{
-	struct ath12k *ar = ab->pdevs[mac_id].ar;
+	struct ath12k *ar = ab->pdevs[mac_id].ar;
 
 	ath12k_dp_rx_pdev_srng_free(ar);
 }
 
-int ath12k_dp_rxdma_ring_sel_config_qcn9274(struct ath12k_base *ab)
-{
-	struct ath12k_dp *dp = &ab->dp;
-	struct htt_rx_ring_tlv_filter tlv_filter = {};
-	u32 ring_id;
-	int ret;
-	u32 hal_rx_desc_sz = ab->hal.hal_desc_sz;
-
-	ring_id = dp->rx_refill_buf_ring.refill_buf_ring.ring_id;
-
-	tlv_filter.rx_filter = HTT_RX_TLV_FLAGS_RXDMA_RING;
-	tlv_filter.pkt_filter_flags2 = HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR;
-	tlv_filter.pkt_filter_flags3 = HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST |
-					HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST |
-					HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA;
-	tlv_filter.offset_valid = true;
-	tlv_filter.rx_packet_offset = hal_rx_desc_sz;
-
-	tlv_filter.rx_mpdu_start_offset =
-		ab->hal_rx_ops->rx_desc_get_mpdu_start_offset();
-	tlv_filter.rx_msdu_end_offset =
-		ab->hal_rx_ops->rx_desc_get_msdu_end_offset();
-
-	if (ath12k_dp_wmask_compaction_rx_tlv_supported(ab)) {
-		tlv_filter.rx_mpdu_start_wmask =
-			ab->hw_params->hal_ops->rxdma_ring_wmask_rx_mpdu_start();
-		tlv_filter.rx_msdu_end_wmask =
-			ab->hw_params->hal_ops->rxdma_ring_wmask_rx_msdu_end();
-		ath12k_dbg(ab, ATH12K_DBG_DATA,
-			   "Configuring compact tlv masks rx_mpdu_start_wmask 0x%x rx_msdu_end_wmask 0x%x\n",
-			   tlv_filter.rx_mpdu_start_wmask, tlv_filter.rx_msdu_end_wmask);
-	}
-
-	ret = ath12k_dp_tx_htt_rx_filter_setup(ab, ring_id, 0,
-					       HAL_RXDMA_BUF,
-					       DP_RXDMA_REFILL_RING_SIZE,
-					       &tlv_filter);
-
-	return ret;
-}
-
-int ath12k_dp_rxdma_ring_sel_config_wcn7850(struct ath12k_base *ab)
-{
-	struct ath12k_dp *dp = &ab->dp;
-	struct htt_rx_ring_tlv_filter tlv_filter = {};
-	u32 ring_id;
-	int ret = 0;
-	u32 hal_rx_desc_sz = ab->hal.hal_desc_sz;
-	int i;
-
-	ring_id = dp->rx_refill_buf_ring.refill_buf_ring.ring_id;
-
-	tlv_filter.rx_filter = HTT_RX_TLV_FLAGS_RXDMA_RING;
-	tlv_filter.pkt_filter_flags2 = HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR;
-	tlv_filter.pkt_filter_flags3 = HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST |
-					HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST |
-					HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA;
-	tlv_filter.offset_valid = true;
-	tlv_filter.rx_packet_offset = hal_rx_desc_sz;
-
-	tlv_filter.rx_header_offset = offsetof(struct hal_rx_desc_wcn7850, pkt_hdr_tlv);
-
-	tlv_filter.rx_mpdu_start_offset =
-		ab->hal_rx_ops->rx_desc_get_mpdu_start_offset();
-	tlv_filter.rx_msdu_end_offset =
-		ab->hal_rx_ops->rx_desc_get_msdu_end_offset();
-
-	/* TODO: Selectively subscribe to required qwords within msdu_end
-	 * and mpdu_start and setup the mask in below msg
-	 * and modify the rx_desc struct
-	 */
-
-	for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) {
-		ring_id = dp->rx_mac_buf_ring[i].ring_id;
-		ret = ath12k_dp_tx_htt_rx_filter_setup(ab, ring_id, i,
-						       HAL_RXDMA_BUF,
-						       DP_RXDMA_REFILL_RING_SIZE,
-						       &tlv_filter);
-	}
-
-	return ret;
-}
-
 int ath12k_dp_rx_htt_setup(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	u32 ring_id;
 	int i, ret;
 
@@ -4680,7 +1704,7 @@ int ath12k_dp_rx_htt_setup(struct ath12k_base *ab)
 
 int ath12k_dp_rx_alloc(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = &ab->dp;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	struct dp_srng *srng;
 	int i, ret;
 
diff --git a/drivers/net/wireless/ath/ath12k/dp_rx.h b/drivers/net/wireless/ath/ath12k/dp_rx.h
index 69d0a36a91d8..1ec5382f5995 100644
--- a/drivers/net/wireless/ath/ath12k/dp_rx.h
+++ b/drivers/net/wireless/ath/ath12k/dp_rx.h
@@ -1,21 +1,26 @@
 /* SPDX-License-Identifier: BSD-3-Clause-Clear */
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 #ifndef ATH12K_DP_RX_H
 #define ATH12K_DP_RX_H
 
+#include <crypto/hash.h>
 #include "core.h"
-#include "rx_desc.h"
 #include "debug.h"
 
 #define DP_MAX_NWIFI_HDR_LEN	30
 
+struct ath12k_reoq_buf {
+	void *vaddr;
+	dma_addr_t paddr_aligned;
+	u32 size;
+};
+
 struct ath12k_dp_rx_tid {
 	u8 tid;
 	u32 ba_win_sz;
-	bool active;
 	struct ath12k_reoq_buf qbuf;
 
 	/* Info related to rx fragments */
@@ -28,7 +33,7 @@ struct ath12k_dp_rx_tid {
 
 	/* Timer info related to fragments */
 	struct timer_list frag_timer;
-	struct ath12k_base *ab;
+	struct ath12k_dp *dp;
 };
 
 struct ath12k_dp_rx_tid_rxq {
@@ -59,6 +64,8 @@ struct ath12k_dp_rx_reo_cmd {
 			enum hal_reo_cmd_status status);
 };
 
+#define ATH12K_DP_RX_FRAGMENT_TIMEOUT_MS (2 * HZ)
+
 #define ATH12K_DP_RX_REO_DESC_FREE_THRES  64
 #define ATH12K_DP_RX_REO_DESC_FREE_TIMEOUT_MS 1000
 
@@ -77,24 +84,6 @@ struct ath12k_dp_rx_rfc1042_hdr {
 	__be16 snap_type;
 } __packed;
 
-struct ath12k_dp_rx_info {
-	struct ieee80211_rx_status *rx_status;
-	u32 phy_meta_data;
-	u16 peer_id;
-	u8 decap_type;
-	u8 pkt_type;
-	u8 sgi;
-	u8 rate_mcs;
-	u8 bw;
-	u8 nss;
-	u8 addr2[ETH_ALEN];
-	u8 tid;
-	bool ip_csum_fail;
-	bool l4_csum_fail;
-	bool is_mcbc;
-	bool addr2_present;
-};
-
 static inline u32 ath12k_he_gi_to_nl80211_he_gi(u8 sgi)
 {
 	u32 ret = 0;
@@ -117,6 +106,109 @@ static inline u32 ath12k_he_gi_to_nl80211_he_gi(u8 sgi)
 	return ret;
 }
 
+static inline bool ath12k_dp_rx_h_more_frags(struct ath12k_hal *hal,
+					     struct sk_buff *skb)
+{
+	struct ieee80211_hdr *hdr;
+
+	hdr = (struct ieee80211_hdr *)(skb->data + hal->hal_desc_sz);
+	return ieee80211_has_morefrags(hdr->frame_control);
+}
+
+static inline u16 ath12k_dp_rx_h_frag_no(struct ath12k_hal *hal,
+					 struct sk_buff *skb)
+{
+	struct ieee80211_hdr *hdr;
+
+	hdr = (struct ieee80211_hdr *)(skb->data + hal->hal_desc_sz);
+	return le16_to_cpu(hdr->seq_ctrl) & IEEE80211_SCTL_FRAG;
+}
+
+static inline u8 ath12k_dp_rx_h_l3pad(struct ath12k_base *ab,
+				      struct hal_rx_desc *desc)
+{
+	return ab->hal.ops->rx_desc_get_l3_pad_bytes(desc);
+}
+
+static inline void ath12k_dp_rx_desc_end_tlv_copy(struct ath12k_hal *hal,
+						  struct hal_rx_desc *fdesc,
+						  struct hal_rx_desc *ldesc)
+{
+	hal->ops->rx_desc_copy_end_tlv(fdesc, ldesc);
+}
+
+static inline void ath12k_dp_rxdesc_set_msdu_len(struct ath12k_hal *hal,
+						 struct hal_rx_desc *desc,
+						 u16 len)
+{
+	hal->ops->rx_desc_set_msdu_len(desc, len);
+}
+
+static inline u32 ath12k_dp_rxdesc_get_ppduid(struct ath12k_base *ab,
+					      struct hal_rx_desc *rx_desc)
+{
+	return ab->hal.ops->rx_desc_get_mpdu_ppdu_id(rx_desc);
+}
+
+static inline void ath12k_dp_rx_desc_get_dot11_hdr(struct ath12k_hal *hal,
+						   struct hal_rx_desc *desc,
+						   struct ieee80211_hdr *hdr)
+{
+	hal->ops->rx_desc_get_dot11_hdr(desc, hdr);
+}
+
+static inline void ath12k_dp_rx_desc_get_crypto_header(struct ath12k_hal *hal,
+						       struct hal_rx_desc *desc,
+						       u8 *crypto_hdr,
+						       enum hal_encrypt_type enctype)
+{
+	hal->ops->rx_desc_get_crypto_header(desc, crypto_hdr, enctype);
+}
+
+static inline u8 ath12k_dp_rx_get_msdu_src_link(struct ath12k_hal *hal,
+						struct hal_rx_desc *desc)
+{
+	return hal->ops->rx_desc_get_msdu_src_link_id(desc);
+}
+
+static inline void ath12k_dp_clean_up_skb_list(struct sk_buff_head *skb_list)
+{
+	struct sk_buff *skb;
+
+	while ((skb = __skb_dequeue(skb_list)))
+		dev_kfree_skb_any(skb);
+}
+
+static inline
+void ath12k_dp_extract_rx_desc_data(struct ath12k_hal *hal,
+				    struct hal_rx_desc_data *rx_info,
+				    struct hal_rx_desc *rx_desc,
+				    struct hal_rx_desc *ldesc)
+{
+	hal->ops->extract_rx_desc_data(rx_info, rx_desc, ldesc);
+}
+
+void ath12k_dp_rx_h_undecap(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu,
+			    struct hal_rx_desc *rx_desc,
+			    enum hal_encrypt_type enctype,
+			    bool decrypted,
+			    struct hal_rx_desc_data *rx_info);
+void ath12k_dp_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev, struct napi_struct *napi,
+			       struct sk_buff *msdu,
+			       struct hal_rx_desc_data *rx_info);
+bool ath12k_dp_rx_check_nwifi_hdr_len_valid(struct ath12k_dp *dp,
+					    struct hal_rx_desc *rx_desc,
+					    struct sk_buff *msdu,
+					    struct hal_rx_desc_data *rx_info);
+u64 ath12k_dp_rx_h_get_pn(struct ath12k_dp *dp, struct sk_buff *skb);
+void ath12k_dp_rx_h_sort_frags(struct ath12k_hal *hal,
+			       struct sk_buff_head *frag_list,
+			       struct sk_buff *cur_frag);
+void ath12k_dp_rx_h_undecap_frag(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu,
+				 enum hal_encrypt_type enctype, u32 flags);
+int ath12k_dp_rx_h_michael_mic(struct crypto_shash *tfm, u8 *key,
+			       struct ieee80211_hdr *hdr, u8 *data,
+			       size_t data_len, u8 *mic);
 int ath12k_dp_rx_ampdu_start(struct ath12k *ar,
 			     struct ieee80211_ampdu_params *params,
 			     u8 link_id);
@@ -127,14 +219,12 @@ int ath12k_dp_rx_peer_pn_replay_config(struct ath12k_link_vif *arvif,
 				       const u8 *peer_addr,
 				       enum set_key_cmd key_cmd,
 				       struct ieee80211_key_conf *key);
-void ath12k_dp_rx_peer_tid_cleanup(struct ath12k *ar, struct ath12k_peer *peer);
+void ath12k_dp_rx_peer_tid_cleanup(struct ath12k *ar, struct ath12k_dp_link_peer *peer);
 void ath12k_dp_rx_peer_tid_delete(struct ath12k *ar,
-				  struct ath12k_peer *peer, u8 tid);
+				  struct ath12k_dp_link_peer *peer, u8 tid);
 int ath12k_dp_rx_peer_tid_setup(struct ath12k *ar, const u8 *peer_mac, int vdev_id,
 				u8 tid, u32 ba_win_sz, u16 ssn,
 				enum hal_pn_type pn_type);
-void ath12k_dp_htt_htc_t2h_msg_handler(struct ath12k_base *ab,
-				       struct sk_buff *skb);
 int ath12k_dp_rx_pdev_reo_setup(struct ath12k_base *ab);
 void ath12k_dp_rx_pdev_reo_cleanup(struct ath12k_base *ab);
 int ath12k_dp_rx_htt_setup(struct ath12k_base *ab);
@@ -143,15 +233,7 @@ void ath12k_dp_rx_free(struct ath12k_base *ab);
 int ath12k_dp_rx_pdev_alloc(struct ath12k_base *ab, int pdev_idx);
 void ath12k_dp_rx_pdev_free(struct ath12k_base *ab, int pdev_idx);
 void ath12k_dp_rx_reo_cmd_list_cleanup(struct ath12k_base *ab);
-void ath12k_dp_rx_process_reo_status(struct ath12k_base *ab);
-int ath12k_dp_rx_process_wbm_err(struct ath12k_base *ab,
-				 struct napi_struct *napi, int budget);
-int ath12k_dp_rx_process_err(struct ath12k_base *ab, struct napi_struct *napi,
-			     int budget);
-int ath12k_dp_rx_process(struct ath12k_base *ab, int mac_id,
-			 struct napi_struct *napi,
-			 int budget);
-int ath12k_dp_rx_bufs_replenish(struct ath12k_base *ab,
+int ath12k_dp_rx_bufs_replenish(struct ath12k_dp *dp,
 				struct dp_rxdma_ring *rx_ring,
 				struct list_head *used_list,
 				int req_entries);
@@ -160,32 +242,27 @@ int ath12k_dp_rx_peer_frag_setup(struct ath12k *ar, const u8 *peer_mac, int vdev
 
 u8 ath12k_dp_rx_h_l3pad(struct ath12k_base *ab,
 			struct hal_rx_desc *desc);
-struct ath12k_peer *
-ath12k_dp_rx_h_find_peer(struct ath12k_base *ab, struct sk_buff *msdu,
-			 struct ath12k_dp_rx_info *rx_info);
+struct ath12k_dp_link_peer *
+ath12k_dp_rx_h_find_link_peer(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu,
+			      struct hal_rx_desc_data *rx_info);
 u8 ath12k_dp_rx_h_decap_type(struct ath12k_base *ab,
 			     struct hal_rx_desc *desc);
 u32 ath12k_dp_rx_h_mpdu_err(struct ath12k_base *ab,
 			    struct hal_rx_desc *desc);
-void ath12k_dp_rx_h_ppdu(struct ath12k *ar, struct ath12k_dp_rx_info *rx_info);
-int ath12k_dp_rxdma_ring_sel_config_qcn9274(struct ath12k_base *ab);
-int ath12k_dp_rxdma_ring_sel_config_wcn7850(struct ath12k_base *ab);
-
-int ath12k_dp_htt_tlv_iter(struct ath12k_base *ab, const void *ptr, size_t len,
-			   int (*iter)(struct ath12k_base *ar, u16 tag, u16 len,
-				       const void *ptr, void *data),
-			   void *data);
-void ath12k_dp_rx_h_fetch_info(struct ath12k_base *ab,  struct hal_rx_desc *rx_desc,
-			       struct ath12k_dp_rx_info *rx_info);
-
-int ath12k_dp_rx_crypto_mic_len(struct ath12k *ar, enum hal_encrypt_type enctype);
+int ath12k_dp_rx_crypto_mic_len(struct ath12k_dp *dp, enum hal_encrypt_type enctype);
 u32 ath12k_dp_rxdesc_get_ppduid(struct ath12k_base *ab,
 				struct hal_rx_desc *rx_desc);
-bool ath12k_dp_rxdesc_mpdu_valid(struct ath12k_base *ab,
-				 struct hal_rx_desc *rx_desc);
-int ath12k_dp_rx_link_desc_return(struct ath12k_base *ab,
-				  struct ath12k_buffer_addr *buf_addr_info,
-				  enum hal_wbm_rel_bm_act action);
-bool ath12k_dp_rxdesc_mpdu_valid(struct ath12k_base *ab,
-				 struct hal_rx_desc *rx_desc);
+void ath12k_dp_rx_h_ppdu(struct ath12k_pdev_dp *dp_pdev,
+			 struct hal_rx_desc_data *rx_info);
+struct sk_buff *ath12k_dp_rx_get_msdu_last_buf(struct sk_buff_head *msdu_list,
+					       struct sk_buff *first);
+void ath12k_dp_reo_cmd_free(struct ath12k_dp *dp, void *ctx,
+			    enum hal_reo_cmd_status status);
+void ath12k_dp_rx_tid_del_func(struct ath12k_dp *dp, void *ctx,
+			       enum hal_reo_cmd_status status);
+void ath12k_dp_rx_process_reo_cmd_update_rx_queue_list(struct ath12k_dp *dp);
+void ath12k_dp_init_rx_tid_rxq(struct ath12k_dp_rx_tid_rxq *rx_tid_rxq,
+			       struct ath12k_dp_rx_tid *rx_tid,
+			       bool active);
+void ath12k_dp_mark_tid_as_inactive(struct ath12k_dp *dp, int peer_id, u8 tid);
 #endif /* ATH12K_DP_RX_H */
diff --git a/drivers/net/wireless/ath/ath12k/dp_tx.c b/drivers/net/wireless/ath/ath12k/dp_tx.c
index abc84ca8467a..c10da6195c9c 100644
--- a/drivers/net/wireless/ath/ath12k/dp_tx.c
+++ b/drivers/net/wireless/ath/ath12k/dp_tx.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #include "core.h"
@@ -12,7 +12,7 @@
 #include "peer.h"
 #include "mac.h"
 
-static enum hal_tcl_encap_type
+enum hal_tcl_encap_type
 ath12k_dp_tx_get_encap_type(struct ath12k_base *ab, struct sk_buff *skb)
 {
 	struct ieee80211_tx_info *tx_info = IEEE80211_SKB_CB(skb);
@@ -25,8 +25,9 @@ ath12k_dp_tx_get_encap_type(struct ath12k_base *ab, struct sk_buff *skb)
 
 	return HAL_TCL_ENCAP_TYPE_NATIVE_WIFI;
 }
+EXPORT_SYMBOL(ath12k_dp_tx_get_encap_type);
 
-static void ath12k_dp_tx_encap_nwifi(struct sk_buff *skb)
+void ath12k_dp_tx_encap_nwifi(struct sk_buff *skb)
 {
 	struct ieee80211_hdr *hdr = (void *)skb->data;
 	u8 *qos_ctl;
@@ -42,8 +43,9 @@ static void ath12k_dp_tx_encap_nwifi(struct sk_buff *skb)
 	hdr = (void *)skb->data;
 	hdr->frame_control &= ~__cpu_to_le16(IEEE80211_STYPE_QOS_DATA);
 }
+EXPORT_SYMBOL(ath12k_dp_tx_encap_nwifi);
 
-static u8 ath12k_dp_tx_get_tid(struct sk_buff *skb)
+u8 ath12k_dp_tx_get_tid(struct sk_buff *skb)
 {
 	struct ieee80211_hdr *hdr = (void *)skb->data;
 	struct ath12k_skb_cb *cb = ATH12K_SKB_CB(skb);
@@ -55,6 +57,7 @@ static u8 ath12k_dp_tx_get_tid(struct sk_buff *skb)
 	else
 		return skb->priority & IEEE80211_QOS_CTL_TID_MASK;
 }
+EXPORT_SYMBOL(ath12k_dp_tx_get_tid);
 
 enum hal_encrypt_type ath12k_dp_tx_get_encrypt_type(u32 cipher)
 {
@@ -77,19 +80,21 @@ enum hal_encrypt_type ath12k_dp_tx_get_encrypt_type(u32 cipher)
 		return HAL_ENCRYPT_TYPE_OPEN;
 	}
 }
+EXPORT_SYMBOL(ath12k_dp_tx_get_encrypt_type);
 
-static void ath12k_dp_tx_release_txbuf(struct ath12k_dp *dp,
-				       struct ath12k_tx_desc_info *tx_desc,
-				       u8 pool_id)
+void ath12k_dp_tx_release_txbuf(struct ath12k_dp *dp,
+				struct ath12k_tx_desc_info *tx_desc,
+				u8 pool_id)
 {
 	spin_lock_bh(&dp->tx_desc_lock[pool_id]);
 	tx_desc->skb_ext_desc = NULL;
 	list_move_tail(&tx_desc->list, &dp->tx_desc_free_list[pool_id]);
 	spin_unlock_bh(&dp->tx_desc_lock[pool_id]);
 }
+EXPORT_SYMBOL(ath12k_dp_tx_release_txbuf);
 
-static struct ath12k_tx_desc_info *ath12k_dp_tx_assign_buffer(struct ath12k_dp *dp,
-							      u8 pool_id)
+struct ath12k_tx_desc_info *ath12k_dp_tx_assign_buffer(struct ath12k_dp *dp,
+						       u8 pool_id)
 {
 	struct ath12k_tx_desc_info *desc;
 
@@ -108,28 +113,9 @@ static struct ath12k_tx_desc_info *ath12k_dp_tx_assign_buffer(struct ath12k_dp *
 
 	return desc;
 }
+EXPORT_SYMBOL(ath12k_dp_tx_assign_buffer);
 
-static void ath12k_hal_tx_cmd_ext_desc_setup(struct ath12k_base *ab,
-					     struct hal_tx_msdu_ext_desc *tcl_ext_cmd,
-					     struct hal_tx_info *ti)
-{
-	tcl_ext_cmd->info0 = le32_encode_bits(ti->paddr,
-					      HAL_TX_MSDU_EXT_INFO0_BUF_PTR_LO);
-	tcl_ext_cmd->info1 = le32_encode_bits(0x0,
-					      HAL_TX_MSDU_EXT_INFO1_BUF_PTR_HI) |
-			       le32_encode_bits(ti->data_len,
-						HAL_TX_MSDU_EXT_INFO1_BUF_LEN);
-
-	tcl_ext_cmd->info1 |= le32_encode_bits(1, HAL_TX_MSDU_EXT_INFO1_EXTN_OVERRIDE) |
-				le32_encode_bits(ti->encap_type,
-						 HAL_TX_MSDU_EXT_INFO1_ENCAP_TYPE) |
-				le32_encode_bits(ti->encrypt_type,
-						 HAL_TX_MSDU_EXT_INFO1_ENCRYPT_TYPE);
-}
-
-#define HTT_META_DATA_ALIGNMENT 0x8
-
-static void *ath12k_dp_metadata_align_skb(struct sk_buff *skb, u8 tail_len)
+void *ath12k_dp_metadata_align_skb(struct sk_buff *skb, u8 tail_len)
 {
 	struct sk_buff *tail;
 	void *metadata;
@@ -141,29 +127,7 @@ static void *ath12k_dp_metadata_align_skb(struct sk_buff *skb, u8 tail_len)
 	memset(metadata, 0, tail_len);
 	return metadata;
 }
-
-/* Preparing HTT Metadata when utilized with ext MSDU */
-static int ath12k_dp_prepare_htt_metadata(struct sk_buff *skb)
-{
-	struct hal_tx_msdu_metadata *desc_ext;
-	u8 htt_desc_size;
-	/* Size rounded of multiple of 8 bytes */
-	u8 htt_desc_size_aligned;
-
-	htt_desc_size = sizeof(struct hal_tx_msdu_metadata);
-	htt_desc_size_aligned = ALIGN(htt_desc_size, HTT_META_DATA_ALIGNMENT);
-
-	desc_ext = ath12k_dp_metadata_align_skb(skb, htt_desc_size_aligned);
-	if (!desc_ext)
-		return -ENOMEM;
-
-	desc_ext->info0 = le32_encode_bits(1, HAL_TX_MSDU_METADATA_INFO0_ENCRYPT_FLAG) |
-			  le32_encode_bits(0, HAL_TX_MSDU_METADATA_INFO0_ENCRYPT_TYPE) |
-			  le32_encode_bits(1,
-					   HAL_TX_MSDU_METADATA_INFO0_HOST_TX_DESC_POOL);
-
-	return 0;
-}
+EXPORT_SYMBOL(ath12k_dp_metadata_align_skb);
 
 static void ath12k_dp_tx_move_payload(struct sk_buff *skb,
 				      unsigned long delta,
@@ -182,10 +146,9 @@ static void ath12k_dp_tx_move_payload(struct sk_buff *skb,
 	}
 }
 
-static int ath12k_dp_tx_align_payload(struct ath12k_base *ab,
-				      struct sk_buff **pskb)
+int ath12k_dp_tx_align_payload(struct ath12k_dp *dp, struct sk_buff **pskb)
 {
-	u32 iova_mask = ab->hw_params->iova_mask;
+	u32 iova_mask = dp->hw_params->iova_mask;
 	unsigned long offset, delta1, delta2;
 	struct sk_buff *skb2, *skb = *pskb;
 	unsigned int headroom = skb_headroom(skb);
@@ -218,1564 +181,33 @@ static int ath12k_dp_tx_align_payload(struct ath12k_base *ab,
 out:
 	return ret;
 }
+EXPORT_SYMBOL(ath12k_dp_tx_align_payload);
 
-int ath12k_dp_tx(struct ath12k *ar, struct ath12k_link_vif *arvif,
-		 struct sk_buff *skb, bool gsn_valid, int mcbc_gsn,
-		 bool is_mcast)
-{
-	struct ath12k_base *ab = ar->ab;
-	struct ath12k_dp *dp = &ab->dp;
-	struct hal_tx_info ti = {};
-	struct ath12k_tx_desc_info *tx_desc;
-	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
-	struct ath12k_skb_cb *skb_cb = ATH12K_SKB_CB(skb);
-	struct hal_tcl_data_cmd *hal_tcl_desc;
-	struct hal_tx_msdu_ext_desc *msg;
-	struct sk_buff *skb_ext_desc = NULL;
-	struct hal_srng *tcl_ring;
-	struct ieee80211_hdr *hdr = (void *)skb->data;
-	struct ath12k_vif *ahvif = arvif->ahvif;
-	struct dp_tx_ring *tx_ring;
-	u8 pool_id;
-	u8 hal_ring_id;
-	int ret;
-	u8 ring_selector, ring_map = 0;
-	bool tcl_ring_retry;
-	bool msdu_ext_desc = false;
-	bool add_htt_metadata = false;
-	u32 iova_mask = ab->hw_params->iova_mask;
-	bool is_diff_encap = false;
-	bool is_null_frame = false;
-
-	if (test_bit(ATH12K_FLAG_CRASH_FLUSH, &ar->ab->dev_flags))
-		return -ESHUTDOWN;
-
-	if (!(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) &&
-	    !ieee80211_is_data(hdr->frame_control))
-		return -EOPNOTSUPP;
-
-	pool_id = skb_get_queue_mapping(skb) & (ATH12K_HW_MAX_QUEUES - 1);
-
-	/* Let the default ring selection be based on current processor
-	 * number, where one of the 3 tcl rings are selected based on
-	 * the smp_processor_id(). In case that ring
-	 * is full/busy, we resort to other available rings.
-	 * If all rings are full, we drop the packet.
-	 * TODO: Add throttling logic when all rings are full
-	 */
-	ring_selector = ab->hw_params->hw_ops->get_ring_selector(skb);
-
-tcl_ring_sel:
-	tcl_ring_retry = false;
-	ti.ring_id = ring_selector % ab->hw_params->max_tx_ring;
-
-	ring_map |= BIT(ti.ring_id);
-	ti.rbm_id = ab->hw_params->hal_ops->tcl_to_wbm_rbm_map[ti.ring_id].rbm_id;
-
-	tx_ring = &dp->tx_ring[ti.ring_id];
-
-	tx_desc = ath12k_dp_tx_assign_buffer(dp, pool_id);
-	if (!tx_desc)
-		return -ENOMEM;
-
-	ti.bank_id = arvif->bank_id;
-	ti.meta_data_flags = arvif->tcl_metadata;
-
-	if (ahvif->tx_encap_type == HAL_TCL_ENCAP_TYPE_RAW &&
-	    test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ar->ab->dev_flags)) {
-		if (skb_cb->flags & ATH12K_SKB_CIPHER_SET) {
-			ti.encrypt_type =
-				ath12k_dp_tx_get_encrypt_type(skb_cb->cipher);
-
-			if (ieee80211_has_protected(hdr->frame_control))
-				skb_put(skb, IEEE80211_CCMP_MIC_LEN);
-		} else {
-			ti.encrypt_type = HAL_ENCRYPT_TYPE_OPEN;
-		}
-
-		msdu_ext_desc = true;
-	}
-
-	if (gsn_valid) {
-		/* Reset and Initialize meta_data_flags with Global Sequence
-		 * Number (GSN) info.
-		 */
-		ti.meta_data_flags =
-			u32_encode_bits(HTT_TCL_META_DATA_TYPE_GLOBAL_SEQ_NUM,
-					HTT_TCL_META_DATA_TYPE) |
-			u32_encode_bits(mcbc_gsn, HTT_TCL_META_DATA_GLOBAL_SEQ_NUM);
-	}
-
-	ti.encap_type = ath12k_dp_tx_get_encap_type(ab, skb);
-	ti.addr_search_flags = arvif->hal_addr_search_flags;
-	ti.search_type = arvif->search_type;
-	ti.type = HAL_TCL_DESC_TYPE_BUFFER;
-	ti.pkt_offset = 0;
-	ti.lmac_id = ar->lmac_id;
-
-	ti.vdev_id = arvif->vdev_id;
-	if (gsn_valid)
-		ti.vdev_id += HTT_TX_MLO_MCAST_HOST_REINJECT_BASE_VDEV_ID;
-
-	ti.bss_ast_hash = arvif->ast_hash;
-	ti.bss_ast_idx = arvif->ast_idx;
-	ti.dscp_tid_tbl_idx = 0;
-
-	if (skb->ip_summed == CHECKSUM_PARTIAL &&
-	    ti.encap_type != HAL_TCL_ENCAP_TYPE_RAW) {
-		ti.flags0 |= u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_IP4_CKSUM_EN) |
-			     u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_UDP4_CKSUM_EN) |
-			     u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_UDP6_CKSUM_EN) |
-			     u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_TCP4_CKSUM_EN) |
-			     u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_TCP6_CKSUM_EN);
-	}
-
-	ti.flags1 |= u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO3_TID_OVERWRITE);
-
-	ti.tid = ath12k_dp_tx_get_tid(skb);
-
-	switch (ti.encap_type) {
-	case HAL_TCL_ENCAP_TYPE_NATIVE_WIFI:
-		is_null_frame = ieee80211_is_nullfunc(hdr->frame_control);
-		if (ahvif->vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED) {
-			if (skb->protocol == cpu_to_be16(ETH_P_PAE) || is_null_frame)
-				is_diff_encap = true;
-
-			/* Firmware expects msdu ext descriptor for nwifi/raw packets
-			 * received in ETH mode. Without this, observed tx fail for
-			 * Multicast packets in ETH mode.
-			 */
-			msdu_ext_desc = true;
-		} else {
-			ath12k_dp_tx_encap_nwifi(skb);
-		}
-		break;
-	case HAL_TCL_ENCAP_TYPE_RAW:
-		if (!test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags)) {
-			ret = -EINVAL;
-			goto fail_remove_tx_buf;
-		}
-		break;
-	case HAL_TCL_ENCAP_TYPE_ETHERNET:
-		/* no need to encap */
-		break;
-	case HAL_TCL_ENCAP_TYPE_802_3:
-	default:
-		/* TODO: Take care of other encap modes as well */
-		ret = -EINVAL;
-		atomic_inc(&ab->device_stats.tx_err.misc_fail);
-		goto fail_remove_tx_buf;
-	}
-
-	if (iova_mask &&
-	    (unsigned long)skb->data & iova_mask) {
-		ret = ath12k_dp_tx_align_payload(ab, &skb);
-		if (ret) {
-			ath12k_warn(ab, "failed to align TX buffer %d\n", ret);
-			/* don't bail out, give original buffer
-			 * a chance even unaligned.
-			 */
-			goto map;
-		}
-
-		/* hdr is pointing to a wrong place after alignment,
-		 * so refresh it for later use.
-		 */
-		hdr = (void *)skb->data;
-	}
-map:
-	ti.paddr = dma_map_single(ab->dev, skb->data, skb->len, DMA_TO_DEVICE);
-	if (dma_mapping_error(ab->dev, ti.paddr)) {
-		atomic_inc(&ab->device_stats.tx_err.misc_fail);
-		ath12k_warn(ab, "failed to DMA map data Tx buffer\n");
-		ret = -ENOMEM;
-		goto fail_remove_tx_buf;
-	}
-
-	if ((!test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ar->ab->dev_flags) &&
-	     !(skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) &&
-	     !(skb_cb->flags & ATH12K_SKB_CIPHER_SET) &&
-	     ieee80211_has_protected(hdr->frame_control)) ||
-	    is_diff_encap) {
-		/* Firmware is not expecting meta data for qos null
-		 * nwifi packet received in ETH encap mode.
-		 */
-		if (is_null_frame && msdu_ext_desc)
-			goto skip_htt_meta;
-
-		/* Add metadata for sw encrypted vlan group traffic
-		 * and EAPOL nwifi packet received in ETH encap mode.
-		 */
-		add_htt_metadata = true;
-		msdu_ext_desc = true;
-		ti.meta_data_flags |= HTT_TCL_META_DATA_VALID_HTT;
-skip_htt_meta:
-		ti.flags0 |= u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_TO_FW);
-		ti.encap_type = HAL_TCL_ENCAP_TYPE_RAW;
-		ti.encrypt_type = HAL_ENCRYPT_TYPE_OPEN;
-	}
-
-	tx_desc->skb = skb;
-	tx_desc->mac_id = ar->pdev_idx;
-	ti.desc_id = tx_desc->desc_id;
-	ti.data_len = skb->len;
-	skb_cb->paddr = ti.paddr;
-	skb_cb->vif = ahvif->vif;
-	skb_cb->ar = ar;
-
-	if (msdu_ext_desc) {
-		skb_ext_desc = dev_alloc_skb(sizeof(struct hal_tx_msdu_ext_desc));
-		if (!skb_ext_desc) {
-			ret = -ENOMEM;
-			goto fail_unmap_dma;
-		}
-
-		skb_put(skb_ext_desc, sizeof(struct hal_tx_msdu_ext_desc));
-		memset(skb_ext_desc->data, 0, skb_ext_desc->len);
-
-		msg = (struct hal_tx_msdu_ext_desc *)skb_ext_desc->data;
-		ath12k_hal_tx_cmd_ext_desc_setup(ab, msg, &ti);
-
-		if (add_htt_metadata) {
-			ret = ath12k_dp_prepare_htt_metadata(skb_ext_desc);
-			if (ret < 0) {
-				ath12k_dbg(ab, ATH12K_DBG_DP_TX,
-					   "Failed to add HTT meta data, dropping packet\n");
-				goto fail_free_ext_skb;
-			}
-		}
-
-		ti.paddr = dma_map_single(ab->dev, skb_ext_desc->data,
-					  skb_ext_desc->len, DMA_TO_DEVICE);
-		ret = dma_mapping_error(ab->dev, ti.paddr);
-		if (ret)
-			goto fail_free_ext_skb;
-
-		ti.data_len = skb_ext_desc->len;
-		ti.type = HAL_TCL_DESC_TYPE_EXT_DESC;
-
-		skb_cb->paddr_ext_desc = ti.paddr;
-		tx_desc->skb_ext_desc = skb_ext_desc;
-	}
-
-	hal_ring_id = tx_ring->tcl_data_ring.ring_id;
-	tcl_ring = &ab->hal.srng_list[hal_ring_id];
-
-	spin_lock_bh(&tcl_ring->lock);
-
-	ath12k_hal_srng_access_begin(ab, tcl_ring);
-
-	hal_tcl_desc = ath12k_hal_srng_src_get_next_entry(ab, tcl_ring);
-	if (!hal_tcl_desc) {
-		/* NOTE: It is highly unlikely we'll be running out of tcl_ring
-		 * desc because the desc is directly enqueued onto hw queue.
-		 */
-		ath12k_hal_srng_access_end(ab, tcl_ring);
-		ab->device_stats.tx_err.desc_na[ti.ring_id]++;
-		spin_unlock_bh(&tcl_ring->lock);
-		ret = -ENOMEM;
-
-		/* Checking for available tcl descriptors in another ring in
-		 * case of failure due to full tcl ring now, is better than
-		 * checking this ring earlier for each pkt tx.
-		 * Restart ring selection if some rings are not checked yet.
-		 */
-		if (ring_map != (BIT(ab->hw_params->max_tx_ring) - 1) &&
-		    ab->hw_params->tcl_ring_retry) {
-			tcl_ring_retry = true;
-			ring_selector++;
-		}
-
-		goto fail_unmap_dma_ext;
-	}
-
-	spin_lock_bh(&arvif->link_stats_lock);
-	arvif->link_stats.tx_encap_type[ti.encap_type]++;
-	arvif->link_stats.tx_encrypt_type[ti.encrypt_type]++;
-	arvif->link_stats.tx_desc_type[ti.type]++;
-
-	if (is_mcast)
-		arvif->link_stats.tx_bcast_mcast++;
-	else
-		arvif->link_stats.tx_enqueued++;
-	spin_unlock_bh(&arvif->link_stats_lock);
-
-	ab->device_stats.tx_enqueued[ti.ring_id]++;
-
-	ath12k_hal_tx_cmd_desc_setup(ab, hal_tcl_desc, &ti);
-
-	ath12k_hal_srng_access_end(ab, tcl_ring);
-
-	spin_unlock_bh(&tcl_ring->lock);
-
-	ath12k_dbg_dump(ab, ATH12K_DBG_DP_TX, NULL, "dp tx msdu: ",
-			skb->data, skb->len);
-
-	atomic_inc(&ar->dp.num_tx_pending);
-
-	return 0;
-
-fail_unmap_dma_ext:
-	if (skb_cb->paddr_ext_desc)
-		dma_unmap_single(ab->dev, skb_cb->paddr_ext_desc,
-				 skb_ext_desc->len,
-				 DMA_TO_DEVICE);
-fail_free_ext_skb:
-	kfree_skb(skb_ext_desc);
-
-fail_unmap_dma:
-	dma_unmap_single(ab->dev, ti.paddr, ti.data_len, DMA_TO_DEVICE);
-
-fail_remove_tx_buf:
-	ath12k_dp_tx_release_txbuf(dp, tx_desc, pool_id);
-
-	spin_lock_bh(&arvif->link_stats_lock);
-	arvif->link_stats.tx_dropped++;
-	spin_unlock_bh(&arvif->link_stats_lock);
-
-	if (tcl_ring_retry)
-		goto tcl_ring_sel;
-
-	return ret;
-}
-
-static void ath12k_dp_tx_free_txbuf(struct ath12k_base *ab,
-				    struct dp_tx_ring *tx_ring,
-				    struct ath12k_tx_desc_params *desc_params)
+void ath12k_dp_tx_free_txbuf(struct ath12k_dp *dp,
+			     struct dp_tx_ring *tx_ring,
+			     struct ath12k_tx_desc_params *desc_params)
 {
-	struct ath12k *ar;
+	struct ath12k_pdev_dp *dp_pdev;
 	struct sk_buff *msdu = desc_params->skb;
 	struct ath12k_skb_cb *skb_cb;
-	u8 pdev_id = ath12k_hw_mac_id_to_pdev_id(ab->hw_params, desc_params->mac_id);
+	u8 pdev_idx = ath12k_hw_mac_id_to_pdev_id(dp->hw_params, desc_params->mac_id);
 
 	skb_cb = ATH12K_SKB_CB(msdu);
-	ar = ab->pdevs[pdev_id].ar;
 
-	dma_unmap_single(ab->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE);
+	dma_unmap_single(dp->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE);
 	if (skb_cb->paddr_ext_desc) {
-		dma_unmap_single(ab->dev, skb_cb->paddr_ext_desc,
+		dma_unmap_single(dp->dev, skb_cb->paddr_ext_desc,
 				 desc_params->skb_ext_desc->len, DMA_TO_DEVICE);
 		dev_kfree_skb_any(desc_params->skb_ext_desc);
 	}
 
-	ieee80211_free_txskb(ar->ah->hw, msdu);
-
-	if (atomic_dec_and_test(&ar->dp.num_tx_pending))
-		wake_up(&ar->dp.tx_empty_waitq);
-}
-
-static void
-ath12k_dp_tx_htt_tx_complete_buf(struct ath12k_base *ab,
-				 struct ath12k_tx_desc_params *desc_params,
-				 struct dp_tx_ring *tx_ring,
-				 struct ath12k_dp_htt_wbm_tx_status *ts,
-				 u16 peer_id)
-{
-	struct ieee80211_tx_info *info;
-	struct ath12k_link_vif *arvif;
-	struct ath12k_skb_cb *skb_cb;
-	struct ieee80211_vif *vif;
-	struct ath12k_vif *ahvif;
-	struct ath12k *ar;
-	struct sk_buff *msdu = desc_params->skb;
-	s32 noise_floor;
-	struct ieee80211_tx_status status = {};
-	struct ath12k_peer *peer;
-
-	skb_cb = ATH12K_SKB_CB(msdu);
-	info = IEEE80211_SKB_CB(msdu);
-
-	ar = skb_cb->ar;
-	ab->device_stats.tx_completed[tx_ring->tcl_data_ring_id]++;
-
-	if (atomic_dec_and_test(&ar->dp.num_tx_pending))
-		wake_up(&ar->dp.tx_empty_waitq);
-
-	dma_unmap_single(ab->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE);
-	if (skb_cb->paddr_ext_desc) {
-		dma_unmap_single(ab->dev, skb_cb->paddr_ext_desc,
-				 desc_params->skb_ext_desc->len, DMA_TO_DEVICE);
-		dev_kfree_skb_any(desc_params->skb_ext_desc);
-	}
-
-	vif = skb_cb->vif;
-	if (vif) {
-		ahvif = ath12k_vif_to_ahvif(vif);
-		rcu_read_lock();
-		arvif = rcu_dereference(ahvif->link[skb_cb->link_id]);
-		if (arvif) {
-			spin_lock_bh(&arvif->link_stats_lock);
-			arvif->link_stats.tx_completed++;
-			spin_unlock_bh(&arvif->link_stats_lock);
-		}
-		rcu_read_unlock();
-	}
-
-	memset(&info->status, 0, sizeof(info->status));
-
-	if (ts->acked) {
-		if (!(info->flags & IEEE80211_TX_CTL_NO_ACK)) {
-			info->flags |= IEEE80211_TX_STAT_ACK;
-			info->status.ack_signal = ts->ack_rssi;
-
-			if (!test_bit(WMI_TLV_SERVICE_HW_DB2DBM_CONVERSION_SUPPORT,
-				      ab->wmi_ab.svc_map)) {
-				spin_lock_bh(&ar->data_lock);
-				noise_floor = ath12k_pdev_get_noise_floor(ar);
-				spin_unlock_bh(&ar->data_lock);
-
-				info->status.ack_signal += noise_floor;
-			}
-
-			info->status.flags = IEEE80211_TX_STATUS_ACK_SIGNAL_VALID;
-		} else {
-			info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
-		}
-	}
-	rcu_read_lock();
-	spin_lock_bh(&ab->base_lock);
-	peer = ath12k_peer_find_by_id(ab, peer_id);
-	if (!peer || !peer->sta) {
-		ath12k_dbg(ab, ATH12K_DBG_DATA,
-			   "dp_tx: failed to find the peer with peer_id %d\n", peer_id);
-		spin_unlock_bh(&ab->base_lock);
-		ieee80211_free_txskb(ath12k_ar_to_hw(ar), msdu);
-		goto exit;
-	} else {
-		status.sta = peer->sta;
-	}
-	spin_unlock_bh(&ab->base_lock);
-
-	status.info = info;
-	status.skb = msdu;
-	ieee80211_tx_status_ext(ath12k_ar_to_hw(ar), &status);
-exit:
-	rcu_read_unlock();
-}
-
-static void
-ath12k_dp_tx_process_htt_tx_complete(struct ath12k_base *ab, void *desc,
-				     struct dp_tx_ring *tx_ring,
-				     struct ath12k_tx_desc_params *desc_params)
-{
-	struct htt_tx_wbm_completion *status_desc;
-	struct ath12k_dp_htt_wbm_tx_status ts = {};
-	enum hal_wbm_htt_tx_comp_status wbm_status;
-	u16 peer_id;
-
-	status_desc = desc;
-
-	wbm_status = le32_get_bits(status_desc->info0,
-				   HTT_TX_WBM_COMP_INFO0_STATUS);
-	ab->device_stats.fw_tx_status[wbm_status]++;
-
-	switch (wbm_status) {
-	case HAL_WBM_REL_HTT_TX_COMP_STATUS_OK:
-		ts.acked = (wbm_status == HAL_WBM_REL_HTT_TX_COMP_STATUS_OK);
-		ts.ack_rssi = le32_get_bits(status_desc->info2,
-					    HTT_TX_WBM_COMP_INFO2_ACK_RSSI);
-
-		peer_id = le32_get_bits(((struct hal_wbm_completion_ring_tx *)desc)->
-				info3, HAL_WBM_COMPL_TX_INFO3_PEER_ID);
-
-		ath12k_dp_tx_htt_tx_complete_buf(ab, desc_params, tx_ring, &ts, peer_id);
-		break;
-	case HAL_WBM_REL_HTT_TX_COMP_STATUS_DROP:
-	case HAL_WBM_REL_HTT_TX_COMP_STATUS_TTL:
-	case HAL_WBM_REL_HTT_TX_COMP_STATUS_REINJ:
-	case HAL_WBM_REL_HTT_TX_COMP_STATUS_INSPECT:
-	case HAL_WBM_REL_HTT_TX_COMP_STATUS_VDEVID_MISMATCH:
-		ath12k_dp_tx_free_txbuf(ab, tx_ring, desc_params);
-		break;
-	case HAL_WBM_REL_HTT_TX_COMP_STATUS_MEC_NOTIFY:
-		/* This event is to be handled only when the driver decides to
-		 * use WDS offload functionality.
-		 */
-		break;
-	default:
-		ath12k_warn(ab, "Unknown htt wbm tx status %d\n", wbm_status);
-		break;
-	}
-}
-
-static void ath12k_dp_tx_update_txcompl(struct ath12k *ar, struct hal_tx_status *ts)
-{
-	struct ath12k_base *ab = ar->ab;
-	struct ath12k_peer *peer;
-	struct ieee80211_sta *sta;
-	struct ath12k_sta *ahsta;
-	struct ath12k_link_sta *arsta;
-	struct rate_info txrate = {};
-	u16 rate, ru_tones;
-	u8 rate_idx = 0;
-	int ret;
-
-	spin_lock_bh(&ab->base_lock);
-	peer = ath12k_peer_find_by_id(ab, ts->peer_id);
-	if (!peer || !peer->sta) {
-		ath12k_dbg(ab, ATH12K_DBG_DP_TX,
-			   "failed to find the peer by id %u\n", ts->peer_id);
-		spin_unlock_bh(&ab->base_lock);
-		return;
-	}
-	sta = peer->sta;
-	ahsta = ath12k_sta_to_ahsta(sta);
-	arsta = &ahsta->deflink;
-
-	/* This is to prefer choose the real NSS value arsta->last_txrate.nss,
-	 * if it is invalid, then choose the NSS value while assoc.
-	 */
-	if (arsta->last_txrate.nss)
-		txrate.nss = arsta->last_txrate.nss;
-	else
-		txrate.nss = arsta->peer_nss;
-	spin_unlock_bh(&ab->base_lock);
-
-	switch (ts->pkt_type) {
-	case HAL_TX_RATE_STATS_PKT_TYPE_11A:
-	case HAL_TX_RATE_STATS_PKT_TYPE_11B:
-		ret = ath12k_mac_hw_ratecode_to_legacy_rate(ts->mcs,
-							    ts->pkt_type,
-							    &rate_idx,
-							    &rate);
-		if (ret < 0) {
-			ath12k_warn(ab, "Invalid tx legacy rate %d\n", ret);
-			return;
-		}
-
-		txrate.legacy = rate;
-		break;
-	case HAL_TX_RATE_STATS_PKT_TYPE_11N:
-		if (ts->mcs > ATH12K_HT_MCS_MAX) {
-			ath12k_warn(ab, "Invalid HT mcs index %d\n", ts->mcs);
-			return;
-		}
-
-		if (txrate.nss != 0)
-			txrate.mcs = ts->mcs + 8 * (txrate.nss - 1);
-
-		txrate.flags = RATE_INFO_FLAGS_MCS;
-
-		if (ts->sgi)
-			txrate.flags |= RATE_INFO_FLAGS_SHORT_GI;
-		break;
-	case HAL_TX_RATE_STATS_PKT_TYPE_11AC:
-		if (ts->mcs > ATH12K_VHT_MCS_MAX) {
-			ath12k_warn(ab, "Invalid VHT mcs index %d\n", ts->mcs);
-			return;
-		}
-
-		txrate.mcs = ts->mcs;
-		txrate.flags = RATE_INFO_FLAGS_VHT_MCS;
-
-		if (ts->sgi)
-			txrate.flags |= RATE_INFO_FLAGS_SHORT_GI;
-		break;
-	case HAL_TX_RATE_STATS_PKT_TYPE_11AX:
-		if (ts->mcs > ATH12K_HE_MCS_MAX) {
-			ath12k_warn(ab, "Invalid HE mcs index %d\n", ts->mcs);
-			return;
-		}
-
-		txrate.mcs = ts->mcs;
-		txrate.flags = RATE_INFO_FLAGS_HE_MCS;
-		txrate.he_gi = ath12k_he_gi_to_nl80211_he_gi(ts->sgi);
-		break;
-	case HAL_TX_RATE_STATS_PKT_TYPE_11BE:
-		if (ts->mcs > ATH12K_EHT_MCS_MAX) {
-			ath12k_warn(ab, "Invalid EHT mcs index %d\n", ts->mcs);
-			return;
-		}
-
-		txrate.mcs = ts->mcs;
-		txrate.flags = RATE_INFO_FLAGS_EHT_MCS;
-		txrate.eht_gi = ath12k_mac_eht_gi_to_nl80211_eht_gi(ts->sgi);
-		break;
-	default:
-		ath12k_warn(ab, "Invalid tx pkt type: %d\n", ts->pkt_type);
-		return;
-	}
-
-	txrate.bw = ath12k_mac_bw_to_mac80211_bw(ts->bw);
-
-	if (ts->ofdma && ts->pkt_type == HAL_TX_RATE_STATS_PKT_TYPE_11AX) {
-		txrate.bw = RATE_INFO_BW_HE_RU;
-		ru_tones = ath12k_mac_he_convert_tones_to_ru_tones(ts->tones);
-		txrate.he_ru_alloc =
-			ath12k_he_ru_tones_to_nl80211_he_ru_alloc(ru_tones);
-	}
-
-	if (ts->ofdma && ts->pkt_type == HAL_TX_RATE_STATS_PKT_TYPE_11BE) {
-		txrate.bw = RATE_INFO_BW_EHT_RU;
-		txrate.eht_ru_alloc =
-			ath12k_mac_eht_ru_tones_to_nl80211_eht_ru_alloc(ts->tones);
-	}
-
-	spin_lock_bh(&ab->base_lock);
-	arsta->txrate = txrate;
-	spin_unlock_bh(&ab->base_lock);
-}
-
-static void ath12k_dp_tx_complete_msdu(struct ath12k *ar,
-				       struct ath12k_tx_desc_params *desc_params,
-				       struct hal_tx_status *ts,
-				       int ring)
-{
-	struct ath12k_base *ab = ar->ab;
-	struct ath12k_hw *ah = ar->ah;
-	struct ieee80211_tx_info *info;
-	struct ath12k_link_vif *arvif;
-	struct ath12k_skb_cb *skb_cb;
-	struct ieee80211_vif *vif;
-	struct ath12k_vif *ahvif;
-	struct sk_buff *msdu = desc_params->skb;
-	s32 noise_floor;
-	struct ieee80211_tx_status status = {};
-	struct ieee80211_rate_status status_rate = {};
-	struct ath12k_peer *peer;
-	struct ath12k_link_sta *arsta;
-	struct ath12k_sta *ahsta;
-	struct rate_info rate;
-
-	if (WARN_ON_ONCE(ts->buf_rel_source != HAL_WBM_REL_SRC_MODULE_TQM)) {
-		/* Must not happen */
-		return;
-	}
-
-	skb_cb = ATH12K_SKB_CB(msdu);
-	ab->device_stats.tx_completed[ring]++;
-
-	dma_unmap_single(ab->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE);
-	if (skb_cb->paddr_ext_desc) {
-		dma_unmap_single(ab->dev, skb_cb->paddr_ext_desc,
-				 desc_params->skb_ext_desc->len, DMA_TO_DEVICE);
-		dev_kfree_skb_any(desc_params->skb_ext_desc);
-	}
-
-	rcu_read_lock();
-
-	if (!rcu_dereference(ab->pdevs_active[ar->pdev_idx])) {
-		ieee80211_free_txskb(ah->hw, msdu);
-		goto exit;
-	}
-
-	if (!skb_cb->vif) {
-		ieee80211_free_txskb(ah->hw, msdu);
-		goto exit;
-	}
-
-	vif = skb_cb->vif;
-	if (vif) {
-		ahvif = ath12k_vif_to_ahvif(vif);
-		arvif = rcu_dereference(ahvif->link[skb_cb->link_id]);
-		if (arvif) {
-			spin_lock_bh(&arvif->link_stats_lock);
-			arvif->link_stats.tx_completed++;
-			spin_unlock_bh(&arvif->link_stats_lock);
-		}
-	}
-
-	info = IEEE80211_SKB_CB(msdu);
-	memset(&info->status, 0, sizeof(info->status));
-
-	/* skip tx rate update from ieee80211_status*/
-	info->status.rates[0].idx = -1;
-
-	switch (ts->status) {
-	case HAL_WBM_TQM_REL_REASON_FRAME_ACKED:
-		if (!(info->flags & IEEE80211_TX_CTL_NO_ACK)) {
-			info->flags |= IEEE80211_TX_STAT_ACK;
-			info->status.ack_signal = ts->ack_rssi;
-
-			if (!test_bit(WMI_TLV_SERVICE_HW_DB2DBM_CONVERSION_SUPPORT,
-				      ab->wmi_ab.svc_map)) {
-				spin_lock_bh(&ar->data_lock);
-				noise_floor = ath12k_pdev_get_noise_floor(ar);
-				spin_unlock_bh(&ar->data_lock);
-
-				info->status.ack_signal += noise_floor;
-			}
-
-			info->status.flags = IEEE80211_TX_STATUS_ACK_SIGNAL_VALID;
-		}
-		break;
-	case HAL_WBM_TQM_REL_REASON_CMD_REMOVE_TX:
-		if (info->flags & IEEE80211_TX_CTL_NO_ACK) {
-			info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
-			break;
-		}
-		fallthrough;
-	case HAL_WBM_TQM_REL_REASON_CMD_REMOVE_MPDU:
-	case HAL_WBM_TQM_REL_REASON_DROP_THRESHOLD:
-	case HAL_WBM_TQM_REL_REASON_CMD_REMOVE_AGED_FRAMES:
-		/* The failure status is due to internal firmware tx failure
-		 * hence drop the frame; do not update the status of frame to
-		 * the upper layer
-		 */
-		ieee80211_free_txskb(ah->hw, msdu);
-		goto exit;
-	default:
-		ath12k_dbg(ab, ATH12K_DBG_DP_TX, "tx frame is not acked status %d\n",
-			   ts->status);
-		break;
-	}
-
-	/* NOTE: Tx rate status reporting. Tx completion status does not have
-	 * necessary information (for example nss) to build the tx rate.
-	 * Might end up reporting it out-of-band from HTT stats.
-	 */
-
-	ath12k_dp_tx_update_txcompl(ar, ts);
-
-	spin_lock_bh(&ab->base_lock);
-	peer = ath12k_peer_find_by_id(ab, ts->peer_id);
-	if (!peer || !peer->sta) {
-		ath12k_err(ab,
-			   "dp_tx: failed to find the peer with peer_id %d\n",
-			   ts->peer_id);
-		spin_unlock_bh(&ab->base_lock);
-		ieee80211_free_txskb(ath12k_ar_to_hw(ar), msdu);
-		goto exit;
-	}
-	ahsta = ath12k_sta_to_ahsta(peer->sta);
-	arsta = &ahsta->deflink;
-
-	spin_unlock_bh(&ab->base_lock);
-
-	status.sta = peer->sta;
-	status.info = info;
-	status.skb = msdu;
-	rate = arsta->last_txrate;
-
-	status_rate.rate_idx = rate;
-	status_rate.try_count = 1;
-
-	status.rates = &status_rate;
-	status.n_rates = 1;
-	ieee80211_tx_status_ext(ath12k_ar_to_hw(ar), &status);
-
-exit:
-	rcu_read_unlock();
-}
-
-static void ath12k_dp_tx_status_parse(struct ath12k_base *ab,
-				      struct hal_wbm_completion_ring_tx *desc,
-				      struct hal_tx_status *ts)
-{
-	u32 info0 = le32_to_cpu(desc->rate_stats.info0);
-
-	ts->buf_rel_source =
-		le32_get_bits(desc->info0, HAL_WBM_COMPL_TX_INFO0_REL_SRC_MODULE);
-	if (ts->buf_rel_source != HAL_WBM_REL_SRC_MODULE_FW &&
-	    ts->buf_rel_source != HAL_WBM_REL_SRC_MODULE_TQM)
-		return;
-
-	if (ts->buf_rel_source == HAL_WBM_REL_SRC_MODULE_FW)
-		return;
-
-	ts->status = le32_get_bits(desc->info0,
-				   HAL_WBM_COMPL_TX_INFO0_TQM_RELEASE_REASON);
-
-	ts->ppdu_id = le32_get_bits(desc->info1,
-				    HAL_WBM_COMPL_TX_INFO1_TQM_STATUS_NUMBER);
-
-	ts->peer_id = le32_get_bits(desc->info3, HAL_WBM_COMPL_TX_INFO3_PEER_ID);
-
-	ts->ack_rssi = le32_get_bits(desc->info2,
-				     HAL_WBM_COMPL_TX_INFO2_ACK_FRAME_RSSI);
-
-	if (info0 & HAL_TX_RATE_STATS_INFO0_VALID) {
-		ts->pkt_type = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_PKT_TYPE);
-		ts->mcs = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_MCS);
-		ts->sgi = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_SGI);
-		ts->bw = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_BW);
-		ts->tones = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_TONES_IN_RU);
-		ts->ofdma = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_OFDMA_TX);
-	}
-}
-
-void ath12k_dp_tx_completion_handler(struct ath12k_base *ab, int ring_id)
-{
-	struct ath12k *ar;
-	struct ath12k_dp *dp = &ab->dp;
-	int hal_ring_id = dp->tx_ring[ring_id].tcl_comp_ring.ring_id;
-	struct hal_srng *status_ring = &ab->hal.srng_list[hal_ring_id];
-	struct ath12k_tx_desc_info *tx_desc = NULL;
-	struct hal_tx_status ts = {};
-	struct ath12k_tx_desc_params desc_params;
-	struct dp_tx_ring *tx_ring = &dp->tx_ring[ring_id];
-	struct hal_wbm_release_ring *desc;
-	u8 pdev_id;
-	u64 desc_va;
-	enum hal_wbm_rel_src_module buf_rel_source;
-	enum hal_wbm_tqm_rel_reason rel_status;
-
-	spin_lock_bh(&status_ring->lock);
-
-	ath12k_hal_srng_access_begin(ab, status_ring);
-
-	while (ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_head) !=
-	       tx_ring->tx_status_tail) {
-		desc = ath12k_hal_srng_dst_get_next_entry(ab, status_ring);
-		if (!desc)
-			break;
-
-		memcpy(&tx_ring->tx_status[tx_ring->tx_status_head],
-		       desc, sizeof(*desc));
-		tx_ring->tx_status_head =
-			ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_head);
-	}
-
-	if (ath12k_hal_srng_dst_peek(ab, status_ring) &&
-	    (ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_head) ==
-	     tx_ring->tx_status_tail)) {
-		/* TODO: Process pending tx_status messages when kfifo_is_full() */
-		ath12k_warn(ab, "Unable to process some of the tx_status ring desc because status_fifo is full\n");
-	}
-
-	ath12k_hal_srng_access_end(ab, status_ring);
-
-	spin_unlock_bh(&status_ring->lock);
-
-	while (ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_tail) !=
-	       tx_ring->tx_status_head) {
-		struct hal_wbm_completion_ring_tx *tx_status;
-		u32 desc_id;
-
-		tx_ring->tx_status_tail =
-			ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_tail);
-		tx_status = &tx_ring->tx_status[tx_ring->tx_status_tail];
-		ath12k_dp_tx_status_parse(ab, tx_status, &ts);
-
-		if (le32_get_bits(tx_status->info0, HAL_WBM_COMPL_TX_INFO0_CC_DONE)) {
-			/* HW done cookie conversion */
-			desc_va = ((u64)le32_to_cpu(tx_status->buf_va_hi) << 32 |
-				   le32_to_cpu(tx_status->buf_va_lo));
-			tx_desc = (struct ath12k_tx_desc_info *)((unsigned long)desc_va);
-		} else {
-			/* SW does cookie conversion to VA */
-			desc_id = le32_get_bits(tx_status->buf_va_hi,
-						BUFFER_ADDR_INFO1_SW_COOKIE);
-
-			tx_desc = ath12k_dp_get_tx_desc(ab, desc_id);
-		}
-		if (!tx_desc) {
-			ath12k_warn(ab, "unable to retrieve tx_desc!");
-			continue;
-		}
-
-		desc_params.mac_id = tx_desc->mac_id;
-		desc_params.skb = tx_desc->skb;
-		desc_params.skb_ext_desc = tx_desc->skb_ext_desc;
-
-		/* Find the HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE value */
-		buf_rel_source = le32_get_bits(tx_status->info0,
-					       HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE);
-		ab->device_stats.tx_wbm_rel_source[buf_rel_source]++;
-
-		rel_status = le32_get_bits(tx_status->info0,
-					   HAL_WBM_COMPL_TX_INFO0_TQM_RELEASE_REASON);
-		ab->device_stats.tqm_rel_reason[rel_status]++;
-
-		/* Release descriptor as soon as extracting necessary info
-		 * to reduce contention
-		 */
-		ath12k_dp_tx_release_txbuf(dp, tx_desc, tx_desc->pool_id);
-		if (ts.buf_rel_source == HAL_WBM_REL_SRC_MODULE_FW) {
-			ath12k_dp_tx_process_htt_tx_complete(ab, (void *)tx_status,
-							     tx_ring, &desc_params);
-			continue;
-		}
-
-		pdev_id = ath12k_hw_mac_id_to_pdev_id(ab->hw_params, desc_params.mac_id);
-		ar = ab->pdevs[pdev_id].ar;
-
-		if (atomic_dec_and_test(&ar->dp.num_tx_pending))
-			wake_up(&ar->dp.tx_empty_waitq);
-
-		ath12k_dp_tx_complete_msdu(ar, &desc_params, &ts,
-					   tx_ring->tcl_data_ring_id);
-	}
-}
-
-static int
-ath12k_dp_tx_get_ring_id_type(struct ath12k_base *ab,
-			      int mac_id, u32 ring_id,
-			      enum hal_ring_type ring_type,
-			      enum htt_srng_ring_type *htt_ring_type,
-			      enum htt_srng_ring_id *htt_ring_id)
-{
-	int ret = 0;
-
-	switch (ring_type) {
-	case HAL_RXDMA_BUF:
-		/* for some targets, host fills rx buffer to fw and fw fills to
-		 * rxbuf ring for each rxdma
-		 */
-		if (!ab->hw_params->rx_mac_buf_ring) {
-			if (!(ring_id == HAL_SRNG_SW2RXDMA_BUF0 ||
-			      ring_id == HAL_SRNG_SW2RXDMA_BUF1)) {
-				ret = -EINVAL;
-			}
-			*htt_ring_id = HTT_RXDMA_HOST_BUF_RING;
-			*htt_ring_type = HTT_SW_TO_HW_RING;
-		} else {
-			if (ring_id == HAL_SRNG_SW2RXDMA_BUF0) {
-				*htt_ring_id = HTT_HOST1_TO_FW_RXBUF_RING;
-				*htt_ring_type = HTT_SW_TO_SW_RING;
-			} else {
-				*htt_ring_id = HTT_RXDMA_HOST_BUF_RING;
-				*htt_ring_type = HTT_SW_TO_HW_RING;
-			}
-		}
-		break;
-	case HAL_RXDMA_DST:
-		*htt_ring_id = HTT_RXDMA_NON_MONITOR_DEST_RING;
-		*htt_ring_type = HTT_HW_TO_SW_RING;
-		break;
-	case HAL_RXDMA_MONITOR_BUF:
-		*htt_ring_id = HTT_RX_MON_HOST2MON_BUF_RING;
-		*htt_ring_type = HTT_SW_TO_HW_RING;
-		break;
-	case HAL_RXDMA_MONITOR_STATUS:
-		*htt_ring_id = HTT_RXDMA_MONITOR_STATUS_RING;
-		*htt_ring_type = HTT_SW_TO_HW_RING;
-		break;
-	case HAL_RXDMA_MONITOR_DST:
-		*htt_ring_id = HTT_RX_MON_MON2HOST_DEST_RING;
-		*htt_ring_type = HTT_HW_TO_SW_RING;
-		break;
-	case HAL_RXDMA_MONITOR_DESC:
-		*htt_ring_id = HTT_RXDMA_MONITOR_DESC_RING;
-		*htt_ring_type = HTT_SW_TO_HW_RING;
-		break;
-	default:
-		ath12k_warn(ab, "Unsupported ring type in DP :%d\n", ring_type);
-		ret = -EINVAL;
-	}
-	return ret;
-}
-
-int ath12k_dp_tx_htt_srng_setup(struct ath12k_base *ab, u32 ring_id,
-				int mac_id, enum hal_ring_type ring_type)
-{
-	struct htt_srng_setup_cmd *cmd;
-	struct hal_srng *srng = &ab->hal.srng_list[ring_id];
-	struct hal_srng_params params;
-	struct sk_buff *skb;
-	u32 ring_entry_sz;
-	int len = sizeof(*cmd);
-	dma_addr_t hp_addr, tp_addr;
-	enum htt_srng_ring_type htt_ring_type;
-	enum htt_srng_ring_id htt_ring_id;
-	int ret;
-
-	skb = ath12k_htc_alloc_skb(ab, len);
-	if (!skb)
-		return -ENOMEM;
-
-	memset(&params, 0, sizeof(params));
-	ath12k_hal_srng_get_params(ab, srng, &params);
-
-	hp_addr = ath12k_hal_srng_get_hp_addr(ab, srng);
-	tp_addr = ath12k_hal_srng_get_tp_addr(ab, srng);
-
-	ret = ath12k_dp_tx_get_ring_id_type(ab, mac_id, ring_id,
-					    ring_type, &htt_ring_type,
-					    &htt_ring_id);
-	if (ret)
-		goto err_free;
+	guard(rcu)();
 
-	skb_put(skb, len);
-	cmd = (struct htt_srng_setup_cmd *)skb->data;
-	cmd->info0 = le32_encode_bits(HTT_H2T_MSG_TYPE_SRING_SETUP,
-				      HTT_SRNG_SETUP_CMD_INFO0_MSG_TYPE);
-	if (htt_ring_type == HTT_SW_TO_HW_RING ||
-	    htt_ring_type == HTT_HW_TO_SW_RING)
-		cmd->info0 |= le32_encode_bits(DP_SW2HW_MACID(mac_id),
-					       HTT_SRNG_SETUP_CMD_INFO0_PDEV_ID);
-	else
-		cmd->info0 |= le32_encode_bits(mac_id,
-					       HTT_SRNG_SETUP_CMD_INFO0_PDEV_ID);
-	cmd->info0 |= le32_encode_bits(htt_ring_type,
-				       HTT_SRNG_SETUP_CMD_INFO0_RING_TYPE);
-	cmd->info0 |= le32_encode_bits(htt_ring_id,
-				       HTT_SRNG_SETUP_CMD_INFO0_RING_ID);
-
-	cmd->ring_base_addr_lo = cpu_to_le32(params.ring_base_paddr &
-					     HAL_ADDR_LSB_REG_MASK);
-
-	cmd->ring_base_addr_hi = cpu_to_le32((u64)params.ring_base_paddr >>
-					     HAL_ADDR_MSB_REG_SHIFT);
-
-	ret = ath12k_hal_srng_get_entrysize(ab, ring_type);
-	if (ret < 0)
-		goto err_free;
-
-	ring_entry_sz = ret;
-
-	ring_entry_sz >>= 2;
-	cmd->info1 = le32_encode_bits(ring_entry_sz,
-				      HTT_SRNG_SETUP_CMD_INFO1_RING_ENTRY_SIZE);
-	cmd->info1 |= le32_encode_bits(params.num_entries * ring_entry_sz,
-				       HTT_SRNG_SETUP_CMD_INFO1_RING_SIZE);
-	cmd->info1 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_MSI_SWAP),
-				       HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_MSI_SWAP);
-	cmd->info1 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP),
-				       HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_TLV_SWAP);
-	cmd->info1 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_RING_PTR_SWAP),
-				       HTT_SRNG_SETUP_CMD_INFO1_RING_FLAGS_HOST_FW_SWAP);
-	if (htt_ring_type == HTT_SW_TO_HW_RING)
-		cmd->info1 |= cpu_to_le32(HTT_SRNG_SETUP_CMD_INFO1_RING_LOOP_CNT_DIS);
-
-	cmd->ring_head_off32_remote_addr_lo = cpu_to_le32(lower_32_bits(hp_addr));
-	cmd->ring_head_off32_remote_addr_hi = cpu_to_le32(upper_32_bits(hp_addr));
-
-	cmd->ring_tail_off32_remote_addr_lo = cpu_to_le32(lower_32_bits(tp_addr));
-	cmd->ring_tail_off32_remote_addr_hi = cpu_to_le32(upper_32_bits(tp_addr));
-
-	cmd->ring_msi_addr_lo = cpu_to_le32(lower_32_bits(params.msi_addr));
-	cmd->ring_msi_addr_hi = cpu_to_le32(upper_32_bits(params.msi_addr));
-	cmd->msi_data = cpu_to_le32(params.msi_data);
-
-	cmd->intr_info =
-		le32_encode_bits(params.intr_batch_cntr_thres_entries * ring_entry_sz,
-				 HTT_SRNG_SETUP_CMD_INTR_INFO_BATCH_COUNTER_THRESH);
-	cmd->intr_info |=
-		le32_encode_bits(params.intr_timer_thres_us >> 3,
-				 HTT_SRNG_SETUP_CMD_INTR_INFO_INTR_TIMER_THRESH);
-
-	cmd->info2 = 0;
-	if (params.flags & HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN) {
-		cmd->info2 = le32_encode_bits(params.low_threshold,
-					      HTT_SRNG_SETUP_CMD_INFO2_INTR_LOW_THRESH);
-	}
+	dp_pdev = ath12k_dp_to_pdev_dp(dp, pdev_idx);
 
-	ath12k_dbg(ab, ATH12K_DBG_HAL,
-		   "%s msi_addr_lo:0x%x, msi_addr_hi:0x%x, msi_data:0x%x\n",
-		   __func__, cmd->ring_msi_addr_lo, cmd->ring_msi_addr_hi,
-		   cmd->msi_data);
+	ieee80211_free_txskb(ath12k_pdev_dp_to_hw(dp_pdev), msdu);
 
-	ath12k_dbg(ab, ATH12K_DBG_HAL,
-		   "ring_id:%d, ring_type:%d, intr_info:0x%x, flags:0x%x\n",
-		   ring_id, ring_type, cmd->intr_info, cmd->info2);
-
-	ret = ath12k_htc_send(&ab->htc, ab->dp.eid, skb);
-	if (ret)
-		goto err_free;
-
-	return 0;
-
-err_free:
-	dev_kfree_skb_any(skb);
-
-	return ret;
-}
-
-#define HTT_TARGET_VERSION_TIMEOUT_HZ (3 * HZ)
-
-int ath12k_dp_tx_htt_h2t_ver_req_msg(struct ath12k_base *ab)
-{
-	struct ath12k_dp *dp = &ab->dp;
-	struct sk_buff *skb;
-	struct htt_ver_req_cmd *cmd;
-	int len = sizeof(*cmd);
-	u32 metadata_version;
-	int ret;
-
-	init_completion(&dp->htt_tgt_version_received);
-
-	skb = ath12k_htc_alloc_skb(ab, len);
-	if (!skb)
-		return -ENOMEM;
-
-	skb_put(skb, len);
-	cmd = (struct htt_ver_req_cmd *)skb->data;
-	cmd->ver_reg_info = le32_encode_bits(HTT_H2T_MSG_TYPE_VERSION_REQ,
-					     HTT_OPTION_TAG);
-	metadata_version = ath12k_ftm_mode ? HTT_OPTION_TCL_METADATA_VER_V1 :
-			   HTT_OPTION_TCL_METADATA_VER_V2;
-
-	cmd->tcl_metadata_version = le32_encode_bits(HTT_TAG_TCL_METADATA_VERSION,
-						     HTT_OPTION_TAG) |
-				    le32_encode_bits(HTT_TCL_METADATA_VER_SZ,
-						     HTT_OPTION_LEN) |
-				    le32_encode_bits(metadata_version,
-						     HTT_OPTION_VALUE);
-
-	ret = ath12k_htc_send(&ab->htc, dp->eid, skb);
-	if (ret) {
-		dev_kfree_skb_any(skb);
-		return ret;
-	}
-
-	ret = wait_for_completion_timeout(&dp->htt_tgt_version_received,
-					  HTT_TARGET_VERSION_TIMEOUT_HZ);
-	if (ret == 0) {
-		ath12k_warn(ab, "htt target version request timed out\n");
-		return -ETIMEDOUT;
-	}
-
-	if (dp->htt_tgt_ver_major != HTT_TARGET_VERSION_MAJOR) {
-		ath12k_err(ab, "unsupported htt major version %d supported version is %d\n",
-			   dp->htt_tgt_ver_major, HTT_TARGET_VERSION_MAJOR);
-		return -EOPNOTSUPP;
-	}
-
-	return 0;
-}
-
-int ath12k_dp_tx_htt_h2t_ppdu_stats_req(struct ath12k *ar, u32 mask)
-{
-	struct ath12k_base *ab = ar->ab;
-	struct ath12k_dp *dp = &ab->dp;
-	struct sk_buff *skb;
-	struct htt_ppdu_stats_cfg_cmd *cmd;
-	int len = sizeof(*cmd);
-	u8 pdev_mask;
-	int ret;
-	int i;
-
-	for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) {
-		skb = ath12k_htc_alloc_skb(ab, len);
-		if (!skb)
-			return -ENOMEM;
-
-		skb_put(skb, len);
-		cmd = (struct htt_ppdu_stats_cfg_cmd *)skb->data;
-		cmd->msg = le32_encode_bits(HTT_H2T_MSG_TYPE_PPDU_STATS_CFG,
-					    HTT_PPDU_STATS_CFG_MSG_TYPE);
-
-		pdev_mask = 1 << (i + ar->pdev_idx);
-		cmd->msg |= le32_encode_bits(pdev_mask, HTT_PPDU_STATS_CFG_PDEV_ID);
-		cmd->msg |= le32_encode_bits(mask, HTT_PPDU_STATS_CFG_TLV_TYPE_BITMASK);
-
-		ret = ath12k_htc_send(&ab->htc, dp->eid, skb);
-		if (ret) {
-			dev_kfree_skb_any(skb);
-			return ret;
-		}
-	}
-
-	return 0;
-}
-
-int ath12k_dp_tx_htt_rx_filter_setup(struct ath12k_base *ab, u32 ring_id,
-				     int mac_id, enum hal_ring_type ring_type,
-				     int rx_buf_size,
-				     struct htt_rx_ring_tlv_filter *tlv_filter)
-{
-	struct htt_rx_ring_selection_cfg_cmd *cmd;
-	struct hal_srng *srng = &ab->hal.srng_list[ring_id];
-	struct hal_srng_params params;
-	struct sk_buff *skb;
-	int len = sizeof(*cmd);
-	enum htt_srng_ring_type htt_ring_type;
-	enum htt_srng_ring_id htt_ring_id;
-	int ret;
-
-	skb = ath12k_htc_alloc_skb(ab, len);
-	if (!skb)
-		return -ENOMEM;
-
-	memset(&params, 0, sizeof(params));
-	ath12k_hal_srng_get_params(ab, srng, &params);
-
-	ret = ath12k_dp_tx_get_ring_id_type(ab, mac_id, ring_id,
-					    ring_type, &htt_ring_type,
-					    &htt_ring_id);
-	if (ret)
-		goto err_free;
-
-	skb_put(skb, len);
-	cmd = (struct htt_rx_ring_selection_cfg_cmd *)skb->data;
-	cmd->info0 = le32_encode_bits(HTT_H2T_MSG_TYPE_RX_RING_SELECTION_CFG,
-				      HTT_RX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE);
-	if (htt_ring_type == HTT_SW_TO_HW_RING ||
-	    htt_ring_type == HTT_HW_TO_SW_RING)
-		cmd->info0 |=
-			le32_encode_bits(DP_SW2HW_MACID(mac_id),
-					 HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID);
-	else
-		cmd->info0 |=
-			le32_encode_bits(mac_id,
-					 HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID);
-	cmd->info0 |= le32_encode_bits(htt_ring_id,
-				       HTT_RX_RING_SELECTION_CFG_CMD_INFO0_RING_ID);
-	cmd->info0 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_MSI_SWAP),
-				       HTT_RX_RING_SELECTION_CFG_CMD_INFO0_SS);
-	cmd->info0 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP),
-				       HTT_RX_RING_SELECTION_CFG_CMD_INFO0_PS);
-	cmd->info0 |= le32_encode_bits(tlv_filter->offset_valid,
-				       HTT_RX_RING_SELECTION_CFG_CMD_INFO0_OFFSET_VALID);
-	cmd->info0 |=
-		le32_encode_bits(tlv_filter->drop_threshold_valid,
-				 HTT_RX_RING_SELECTION_CFG_CMD_INFO0_DROP_THRES_VAL);
-	cmd->info0 |= le32_encode_bits(!tlv_filter->rxmon_disable,
-				       HTT_RX_RING_SELECTION_CFG_CMD_INFO0_EN_RXMON);
-
-	cmd->info1 = le32_encode_bits(rx_buf_size,
-				      HTT_RX_RING_SELECTION_CFG_CMD_INFO1_BUF_SIZE);
-	cmd->info1 |= le32_encode_bits(tlv_filter->conf_len_mgmt,
-				       HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_MGMT);
-	cmd->info1 |= le32_encode_bits(tlv_filter->conf_len_ctrl,
-				       HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_CTRL);
-	cmd->info1 |= le32_encode_bits(tlv_filter->conf_len_data,
-				       HTT_RX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_DATA);
-	cmd->pkt_type_en_flags0 = cpu_to_le32(tlv_filter->pkt_filter_flags0);
-	cmd->pkt_type_en_flags1 = cpu_to_le32(tlv_filter->pkt_filter_flags1);
-	cmd->pkt_type_en_flags2 = cpu_to_le32(tlv_filter->pkt_filter_flags2);
-	cmd->pkt_type_en_flags3 = cpu_to_le32(tlv_filter->pkt_filter_flags3);
-	cmd->rx_filter_tlv = cpu_to_le32(tlv_filter->rx_filter);
-
-	cmd->info2 = le32_encode_bits(tlv_filter->rx_drop_threshold,
-				      HTT_RX_RING_SELECTION_CFG_CMD_INFO2_DROP_THRESHOLD);
-	cmd->info2 |=
-		le32_encode_bits(tlv_filter->enable_log_mgmt_type,
-				 HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_LOG_MGMT_TYPE);
-	cmd->info2 |=
-		le32_encode_bits(tlv_filter->enable_log_ctrl_type,
-				 HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_CTRL_TYPE);
-	cmd->info2 |=
-		le32_encode_bits(tlv_filter->enable_log_data_type,
-				 HTT_RX_RING_SELECTION_CFG_CMD_INFO2_EN_LOG_DATA_TYPE);
-
-	cmd->info3 =
-		le32_encode_bits(tlv_filter->enable_rx_tlv_offset,
-				 HTT_RX_RING_SELECTION_CFG_CMD_INFO3_EN_TLV_PKT_OFFSET);
-	cmd->info3 |=
-		le32_encode_bits(tlv_filter->rx_tlv_offset,
-				 HTT_RX_RING_SELECTION_CFG_CMD_INFO3_PKT_TLV_OFFSET);
-
-	if (tlv_filter->offset_valid) {
-		cmd->rx_packet_offset =
-			le32_encode_bits(tlv_filter->rx_packet_offset,
-					 HTT_RX_RING_SELECTION_CFG_RX_PACKET_OFFSET);
-
-		cmd->rx_packet_offset |=
-			le32_encode_bits(tlv_filter->rx_header_offset,
-					 HTT_RX_RING_SELECTION_CFG_RX_HEADER_OFFSET);
-
-		cmd->rx_mpdu_offset =
-			le32_encode_bits(tlv_filter->rx_mpdu_end_offset,
-					 HTT_RX_RING_SELECTION_CFG_RX_MPDU_END_OFFSET);
-
-		cmd->rx_mpdu_offset |=
-			le32_encode_bits(tlv_filter->rx_mpdu_start_offset,
-					 HTT_RX_RING_SELECTION_CFG_RX_MPDU_START_OFFSET);
-
-		cmd->rx_msdu_offset =
-			le32_encode_bits(tlv_filter->rx_msdu_end_offset,
-					 HTT_RX_RING_SELECTION_CFG_RX_MSDU_END_OFFSET);
-
-		cmd->rx_msdu_offset |=
-			le32_encode_bits(tlv_filter->rx_msdu_start_offset,
-					 HTT_RX_RING_SELECTION_CFG_RX_MSDU_START_OFFSET);
-
-		cmd->rx_attn_offset =
-			le32_encode_bits(tlv_filter->rx_attn_offset,
-					 HTT_RX_RING_SELECTION_CFG_RX_ATTENTION_OFFSET);
-	}
-
-	if (tlv_filter->rx_mpdu_start_wmask > 0 &&
-	    tlv_filter->rx_msdu_end_wmask > 0) {
-		cmd->info2 |=
-			le32_encode_bits(true,
-					 HTT_RX_RING_SELECTION_CFG_WORD_MASK_COMPACT_SET);
-		cmd->rx_mpdu_start_end_mask =
-			le32_encode_bits(tlv_filter->rx_mpdu_start_wmask,
-					 HTT_RX_RING_SELECTION_CFG_RX_MPDU_START_MASK);
-		/* mpdu_end is not used for any hardwares so far
-		 * please assign it in future if any chip is
-		 * using through hal ops
-		 */
-		cmd->rx_mpdu_start_end_mask |=
-			le32_encode_bits(tlv_filter->rx_mpdu_end_wmask,
-					 HTT_RX_RING_SELECTION_CFG_RX_MPDU_END_MASK);
-		cmd->rx_msdu_end_word_mask =
-			le32_encode_bits(tlv_filter->rx_msdu_end_wmask,
-					 HTT_RX_RING_SELECTION_CFG_RX_MSDU_END_MASK);
-	}
-
-	ret = ath12k_htc_send(&ab->htc, ab->dp.eid, skb);
-	if (ret)
-		goto err_free;
-
-	return 0;
-
-err_free:
-	dev_kfree_skb_any(skb);
-
-	return ret;
-}
-
-int
-ath12k_dp_tx_htt_h2t_ext_stats_req(struct ath12k *ar, u8 type,
-				   struct htt_ext_stats_cfg_params *cfg_params,
-				   u64 cookie)
-{
-	struct ath12k_base *ab = ar->ab;
-	struct ath12k_dp *dp = &ab->dp;
-	struct sk_buff *skb;
-	struct htt_ext_stats_cfg_cmd *cmd;
-	int len = sizeof(*cmd);
-	int ret;
-	u32 pdev_id;
-
-	skb = ath12k_htc_alloc_skb(ab, len);
-	if (!skb)
-		return -ENOMEM;
-
-	skb_put(skb, len);
-
-	cmd = (struct htt_ext_stats_cfg_cmd *)skb->data;
-	memset(cmd, 0, sizeof(*cmd));
-	cmd->hdr.msg_type = HTT_H2T_MSG_TYPE_EXT_STATS_CFG;
-
-	pdev_id = ath12k_mac_get_target_pdev_id(ar);
-	cmd->hdr.pdev_mask = 1 << pdev_id;
-
-	cmd->hdr.stats_type = type;
-	cmd->cfg_param0 = cpu_to_le32(cfg_params->cfg0);
-	cmd->cfg_param1 = cpu_to_le32(cfg_params->cfg1);
-	cmd->cfg_param2 = cpu_to_le32(cfg_params->cfg2);
-	cmd->cfg_param3 = cpu_to_le32(cfg_params->cfg3);
-	cmd->cookie_lsb = cpu_to_le32(lower_32_bits(cookie));
-	cmd->cookie_msb = cpu_to_le32(upper_32_bits(cookie));
-
-	ret = ath12k_htc_send(&ab->htc, dp->eid, skb);
-	if (ret) {
-		ath12k_warn(ab, "failed to send htt type stats request: %d",
-			    ret);
-		dev_kfree_skb_any(skb);
-		return ret;
-	}
-
-	return 0;
-}
-
-int ath12k_dp_tx_htt_monitor_mode_ring_config(struct ath12k *ar, bool reset)
-{
-	struct ath12k_base *ab = ar->ab;
-	int ret;
-
-	ret = ath12k_dp_tx_htt_rx_monitor_mode_ring_config(ar, reset);
-	if (ret) {
-		ath12k_err(ab, "failed to setup rx monitor filter %d\n", ret);
-		return ret;
-	}
-
-	return 0;
-}
-
-int ath12k_dp_tx_htt_rx_monitor_mode_ring_config(struct ath12k *ar, bool reset)
-{
-	struct ath12k_base *ab = ar->ab;
-	struct htt_rx_ring_tlv_filter tlv_filter = {};
-	int ret, ring_id, i;
-
-	tlv_filter.offset_valid = false;
-
-	if (!reset) {
-		tlv_filter.rx_filter = HTT_RX_MON_FILTER_TLV_FLAGS_MON_DEST_RING;
-
-		tlv_filter.drop_threshold_valid = true;
-		tlv_filter.rx_drop_threshold = HTT_RX_RING_TLV_DROP_THRESHOLD_VALUE;
-
-		tlv_filter.enable_log_mgmt_type = true;
-		tlv_filter.enable_log_ctrl_type = true;
-		tlv_filter.enable_log_data_type = true;
-
-		tlv_filter.conf_len_ctrl = HTT_RX_RING_DEFAULT_DMA_LENGTH;
-		tlv_filter.conf_len_mgmt = HTT_RX_RING_DEFAULT_DMA_LENGTH;
-		tlv_filter.conf_len_data = HTT_RX_RING_DEFAULT_DMA_LENGTH;
-
-		tlv_filter.enable_rx_tlv_offset = true;
-		tlv_filter.rx_tlv_offset = HTT_RX_RING_PKT_TLV_OFFSET;
-
-		tlv_filter.pkt_filter_flags0 =
-					HTT_RX_MON_FP_MGMT_FILTER_FLAGS0 |
-					HTT_RX_MON_MO_MGMT_FILTER_FLAGS0;
-		tlv_filter.pkt_filter_flags1 =
-					HTT_RX_MON_FP_MGMT_FILTER_FLAGS1 |
-					HTT_RX_MON_MO_MGMT_FILTER_FLAGS1;
-		tlv_filter.pkt_filter_flags2 =
-					HTT_RX_MON_FP_CTRL_FILTER_FLASG2 |
-					HTT_RX_MON_MO_CTRL_FILTER_FLASG2;
-		tlv_filter.pkt_filter_flags3 =
-					HTT_RX_MON_FP_CTRL_FILTER_FLASG3 |
-					HTT_RX_MON_MO_CTRL_FILTER_FLASG3 |
-					HTT_RX_MON_FP_DATA_FILTER_FLASG3 |
-					HTT_RX_MON_MO_DATA_FILTER_FLASG3;
-	} else {
-		tlv_filter = ath12k_mac_mon_status_filter_default;
-
-		if (ath12k_debugfs_is_extd_rx_stats_enabled(ar))
-			tlv_filter.rx_filter = ath12k_debugfs_rx_filter(ar);
-	}
-
-	if (ab->hw_params->rxdma1_enable) {
-		for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) {
-			ring_id = ar->dp.rxdma_mon_dst_ring[i].ring_id;
-			ret = ath12k_dp_tx_htt_rx_filter_setup(ar->ab, ring_id,
-							       ar->dp.mac_id + i,
-							       HAL_RXDMA_MONITOR_DST,
-							       DP_RXDMA_REFILL_RING_SIZE,
-							       &tlv_filter);
-			if (ret) {
-				ath12k_err(ab,
-					   "failed to setup filter for monitor buf %d\n",
-					   ret);
-				return ret;
-			}
-		}
-		return 0;
-	}
-
-	if (!reset) {
-		for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) {
-			ring_id = ab->dp.rx_mac_buf_ring[i].ring_id;
-			ret = ath12k_dp_tx_htt_rx_filter_setup(ar->ab, ring_id,
-							       i,
-							       HAL_RXDMA_BUF,
-							       DP_RXDMA_REFILL_RING_SIZE,
-							       &tlv_filter);
-			if (ret) {
-				ath12k_err(ab,
-					   "failed to setup filter for mon rx buf %d\n",
-					   ret);
-				return ret;
-			}
-		}
-	}
-
-	for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) {
-		ring_id = ab->dp.rx_mon_status_refill_ring[i].refill_buf_ring.ring_id;
-		if (!reset) {
-			tlv_filter.rx_filter =
-				HTT_RX_MON_FILTER_TLV_FLAGS_MON_STATUS_RING;
-		}
-
-		ret = ath12k_dp_tx_htt_rx_filter_setup(ab, ring_id,
-						       i,
-						       HAL_RXDMA_MONITOR_STATUS,
-						       RX_MON_STATUS_BUF_SIZE,
-						       &tlv_filter);
-		if (ret) {
-			ath12k_err(ab,
-				   "failed to setup filter for mon status buf %d\n",
-				   ret);
-			return ret;
-		}
-	}
-
-	return 0;
-}
-
-int ath12k_dp_tx_htt_tx_filter_setup(struct ath12k_base *ab, u32 ring_id,
-				     int mac_id, enum hal_ring_type ring_type,
-				     int tx_buf_size,
-				     struct htt_tx_ring_tlv_filter *htt_tlv_filter)
-{
-	struct htt_tx_ring_selection_cfg_cmd *cmd;
-	struct hal_srng *srng = &ab->hal.srng_list[ring_id];
-	struct hal_srng_params params;
-	struct sk_buff *skb;
-	int len = sizeof(*cmd);
-	enum htt_srng_ring_type htt_ring_type;
-	enum htt_srng_ring_id htt_ring_id;
-	int ret;
-
-	skb = ath12k_htc_alloc_skb(ab, len);
-	if (!skb)
-		return -ENOMEM;
-
-	memset(&params, 0, sizeof(params));
-	ath12k_hal_srng_get_params(ab, srng, &params);
-
-	ret = ath12k_dp_tx_get_ring_id_type(ab, mac_id, ring_id,
-					    ring_type, &htt_ring_type,
-					    &htt_ring_id);
-
-	if (ret)
-		goto err_free;
-
-	skb_put(skb, len);
-	cmd = (struct htt_tx_ring_selection_cfg_cmd *)skb->data;
-	cmd->info0 = le32_encode_bits(HTT_H2T_MSG_TYPE_TX_MONITOR_CFG,
-				      HTT_TX_RING_SELECTION_CFG_CMD_INFO0_MSG_TYPE);
-	if (htt_ring_type == HTT_SW_TO_HW_RING ||
-	    htt_ring_type == HTT_HW_TO_SW_RING)
-		cmd->info0 |=
-			le32_encode_bits(DP_SW2HW_MACID(mac_id),
-					 HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID);
-	else
-		cmd->info0 |=
-			le32_encode_bits(mac_id,
-					 HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PDEV_ID);
-	cmd->info0 |= le32_encode_bits(htt_ring_id,
-				       HTT_TX_RING_SELECTION_CFG_CMD_INFO0_RING_ID);
-	cmd->info0 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_MSI_SWAP),
-				       HTT_TX_RING_SELECTION_CFG_CMD_INFO0_SS);
-	cmd->info0 |= le32_encode_bits(!!(params.flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP),
-				       HTT_TX_RING_SELECTION_CFG_CMD_INFO0_PS);
-
-	cmd->info1 |=
-		le32_encode_bits(tx_buf_size,
-				 HTT_TX_RING_SELECTION_CFG_CMD_INFO1_RING_BUFF_SIZE);
-
-	if (htt_tlv_filter->tx_mon_mgmt_filter) {
-		cmd->info1 |=
-			le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_MGMT,
-					 HTT_TX_RING_SELECTION_CFG_CMD_INFO1_PKT_TYPE);
-		cmd->info1 |=
-		le32_encode_bits(htt_tlv_filter->tx_mon_pkt_dma_len,
-				 HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_MGMT);
-		cmd->info2 |=
-		le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_MGMT,
-				 HTT_TX_RING_SELECTION_CFG_CMD_INFO2_PKT_TYPE_EN_FLAG);
-	}
-
-	if (htt_tlv_filter->tx_mon_data_filter) {
-		cmd->info1 |=
-			le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_CTRL,
-					 HTT_TX_RING_SELECTION_CFG_CMD_INFO1_PKT_TYPE);
-		cmd->info1 |=
-		le32_encode_bits(htt_tlv_filter->tx_mon_pkt_dma_len,
-				 HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_CTRL);
-		cmd->info2 |=
-		le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_CTRL,
-				 HTT_TX_RING_SELECTION_CFG_CMD_INFO2_PKT_TYPE_EN_FLAG);
-	}
-
-	if (htt_tlv_filter->tx_mon_ctrl_filter) {
-		cmd->info1 |=
-			le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_DATA,
-					 HTT_TX_RING_SELECTION_CFG_CMD_INFO1_PKT_TYPE);
-		cmd->info1 |=
-		le32_encode_bits(htt_tlv_filter->tx_mon_pkt_dma_len,
-				 HTT_TX_RING_SELECTION_CFG_CMD_INFO1_CONF_LEN_DATA);
-		cmd->info2 |=
-		le32_encode_bits(HTT_STATS_FRAME_CTRL_TYPE_DATA,
-				 HTT_TX_RING_SELECTION_CFG_CMD_INFO2_PKT_TYPE_EN_FLAG);
-	}
-
-	cmd->tlv_filter_mask_in0 =
-		cpu_to_le32(htt_tlv_filter->tx_mon_downstream_tlv_flags);
-	cmd->tlv_filter_mask_in1 =
-		cpu_to_le32(htt_tlv_filter->tx_mon_upstream_tlv_flags0);
-	cmd->tlv_filter_mask_in2 =
-		cpu_to_le32(htt_tlv_filter->tx_mon_upstream_tlv_flags1);
-	cmd->tlv_filter_mask_in3 =
-		cpu_to_le32(htt_tlv_filter->tx_mon_upstream_tlv_flags2);
-
-	ret = ath12k_htc_send(&ab->htc, ab->dp.eid, skb);
-	if (ret)
-		goto err_free;
-
-	return 0;
-
-err_free:
-	dev_kfree_skb_any(skb);
-	return ret;
+	if (atomic_dec_and_test(&dp_pdev->num_tx_pending))
+		wake_up(&dp_pdev->tx_empty_waitq);
 }
+EXPORT_SYMBOL(ath12k_dp_tx_free_txbuf);
diff --git a/drivers/net/wireless/ath/ath12k/dp_tx.h b/drivers/net/wireless/ath/ath12k/dp_tx.h
index 10acdcf1fa8f..7cef20540179 100644
--- a/drivers/net/wireless/ath/ath12k/dp_tx.h
+++ b/drivers/net/wireless/ath/ath12k/dp_tx.h
@@ -1,41 +1,32 @@
 /* SPDX-License-Identifier: BSD-3-Clause-Clear */
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2022, 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #ifndef ATH12K_DP_TX_H
 #define ATH12K_DP_TX_H
 
 #include "core.h"
-#include "hal_tx.h"
 
 struct ath12k_dp_htt_wbm_tx_status {
 	bool acked;
 	s8 ack_rssi;
 };
 
-int ath12k_dp_tx_htt_h2t_ver_req_msg(struct ath12k_base *ab);
-int ath12k_dp_tx(struct ath12k *ar, struct ath12k_link_vif *arvif,
-		 struct sk_buff *skb, bool gsn_valid, int mcbc_gsn,
-		 bool is_mcast);
-void ath12k_dp_tx_completion_handler(struct ath12k_base *ab, int ring_id);
-
-int ath12k_dp_tx_htt_h2t_ppdu_stats_req(struct ath12k *ar, u32 mask);
-int
-ath12k_dp_tx_htt_h2t_ext_stats_req(struct ath12k *ar, u8 type,
-				   struct htt_ext_stats_cfg_params *cfg_params,
-				   u64 cookie);
-int ath12k_dp_tx_htt_rx_monitor_mode_ring_config(struct ath12k *ar, bool reset);
-
-int ath12k_dp_tx_htt_rx_filter_setup(struct ath12k_base *ab, u32 ring_id,
-				     int mac_id, enum hal_ring_type ring_type,
-				     int rx_buf_size,
-				     struct htt_rx_ring_tlv_filter *tlv_filter);
 void ath12k_dp_tx_put_bank_profile(struct ath12k_dp *dp, u8 bank_id);
-int ath12k_dp_tx_htt_tx_filter_setup(struct ath12k_base *ab, u32 ring_id,
-				     int mac_id, enum hal_ring_type ring_type,
-				     int tx_buf_size,
-				     struct htt_tx_ring_tlv_filter *htt_tlv_filter);
-int ath12k_dp_tx_htt_monitor_mode_ring_config(struct ath12k *ar, bool reset);
+enum hal_tcl_encap_type
+ath12k_dp_tx_get_encap_type(struct ath12k_base *ab, struct sk_buff *skb);
+void ath12k_dp_tx_encap_nwifi(struct sk_buff *skb);
+u8 ath12k_dp_tx_get_tid(struct sk_buff *skb);
+void *ath12k_dp_metadata_align_skb(struct sk_buff *skb, u8 tail_len);
+int ath12k_dp_tx_align_payload(struct ath12k_dp *dp, struct sk_buff **pskb);
+void ath12k_dp_tx_release_txbuf(struct ath12k_dp *dp,
+				struct ath12k_tx_desc_info *tx_desc,
+				u8 pool_id);
+struct ath12k_tx_desc_info *ath12k_dp_tx_assign_buffer(struct ath12k_dp *dp,
+						       u8 pool_id);
+void ath12k_dp_tx_free_txbuf(struct ath12k_dp *dp,
+			     struct dp_tx_ring *tx_ring,
+			     struct ath12k_tx_desc_params *desc_params);
 #endif
diff --git a/drivers/net/wireless/ath/ath12k/hal.c b/drivers/net/wireless/ath/ath12k/hal.c
index 6406fcf5d69f..c7a152490fa0 100644
--- a/drivers/net/wireless/ath/ath12k/hal.c
+++ b/drivers/net/wireless/ath/ath12k/hal.c
@@ -1,1573 +1,162 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 #include <linux/dma-mapping.h>
-#include "hal_tx.h"
-#include "hal_rx.h"
 #include "debug.h"
-#include "hal_desc.h"
 #include "hif.h"
 
-static const struct hal_srng_config hw_srng_config_template[] = {
-	/* TODO: max_rings can populated by querying HW capabilities */
-	[HAL_REO_DST] = {
-		.start_ring_id = HAL_SRNG_RING_ID_REO2SW1,
-		.max_rings = 8,
-		.entry_size = sizeof(struct hal_reo_dest_ring) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_UMAC,
-		.ring_dir = HAL_SRNG_DIR_DST,
-		.max_size = HAL_REO_REO2SW1_RING_BASE_MSB_RING_SIZE,
-	},
-	[HAL_REO_EXCEPTION] = {
-		/* Designating REO2SW0 ring as exception ring.
-		 * Any of theREO2SW rings can be used as exception ring.
-		 */
-		.start_ring_id = HAL_SRNG_RING_ID_REO2SW0,
-		.max_rings = 1,
-		.entry_size = sizeof(struct hal_reo_dest_ring) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_UMAC,
-		.ring_dir = HAL_SRNG_DIR_DST,
-		.max_size = HAL_REO_REO2SW0_RING_BASE_MSB_RING_SIZE,
-	},
-	[HAL_REO_REINJECT] = {
-		.start_ring_id = HAL_SRNG_RING_ID_SW2REO,
-		.max_rings = 4,
-		.entry_size = sizeof(struct hal_reo_entrance_ring) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_UMAC,
-		.ring_dir = HAL_SRNG_DIR_SRC,
-		.max_size = HAL_REO_SW2REO_RING_BASE_MSB_RING_SIZE,
-	},
-	[HAL_REO_CMD] = {
-		.start_ring_id = HAL_SRNG_RING_ID_REO_CMD,
-		.max_rings = 1,
-		.entry_size = (sizeof(struct hal_tlv_64_hdr) +
-			sizeof(struct hal_reo_get_queue_stats)) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_UMAC,
-		.ring_dir = HAL_SRNG_DIR_SRC,
-		.max_size = HAL_REO_CMD_RING_BASE_MSB_RING_SIZE,
-	},
-	[HAL_REO_STATUS] = {
-		.start_ring_id = HAL_SRNG_RING_ID_REO_STATUS,
-		.max_rings = 1,
-		.entry_size = (sizeof(struct hal_tlv_64_hdr) +
-			sizeof(struct hal_reo_get_queue_stats_status)) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_UMAC,
-		.ring_dir = HAL_SRNG_DIR_DST,
-		.max_size = HAL_REO_STATUS_RING_BASE_MSB_RING_SIZE,
-	},
-	[HAL_TCL_DATA] = {
-		.start_ring_id = HAL_SRNG_RING_ID_SW2TCL1,
-		.max_rings = 6,
-		.entry_size = sizeof(struct hal_tcl_data_cmd) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_UMAC,
-		.ring_dir = HAL_SRNG_DIR_SRC,
-		.max_size = HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE,
-	},
-	[HAL_TCL_CMD] = {
-		.start_ring_id = HAL_SRNG_RING_ID_SW2TCL_CMD,
-		.max_rings = 1,
-		.entry_size = sizeof(struct hal_tcl_gse_cmd) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_UMAC,
-		.ring_dir = HAL_SRNG_DIR_SRC,
-		.max_size = HAL_SW2TCL1_CMD_RING_BASE_MSB_RING_SIZE,
-	},
-	[HAL_TCL_STATUS] = {
-		.start_ring_id = HAL_SRNG_RING_ID_TCL_STATUS,
-		.max_rings = 1,
-		.entry_size = (sizeof(struct hal_tlv_hdr) +
-			     sizeof(struct hal_tcl_status_ring)) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_UMAC,
-		.ring_dir = HAL_SRNG_DIR_DST,
-		.max_size = HAL_TCL_STATUS_RING_BASE_MSB_RING_SIZE,
-	},
-	[HAL_CE_SRC] = {
-		.start_ring_id = HAL_SRNG_RING_ID_CE0_SRC,
-		.max_rings = 16,
-		.entry_size = sizeof(struct hal_ce_srng_src_desc) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_UMAC,
-		.ring_dir = HAL_SRNG_DIR_SRC,
-		.max_size = HAL_CE_SRC_RING_BASE_MSB_RING_SIZE,
-	},
-	[HAL_CE_DST] = {
-		.start_ring_id = HAL_SRNG_RING_ID_CE0_DST,
-		.max_rings = 16,
-		.entry_size = sizeof(struct hal_ce_srng_dest_desc) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_UMAC,
-		.ring_dir = HAL_SRNG_DIR_SRC,
-		.max_size = HAL_CE_DST_RING_BASE_MSB_RING_SIZE,
-	},
-	[HAL_CE_DST_STATUS] = {
-		.start_ring_id = HAL_SRNG_RING_ID_CE0_DST_STATUS,
-		.max_rings = 16,
-		.entry_size = sizeof(struct hal_ce_srng_dst_status_desc) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_UMAC,
-		.ring_dir = HAL_SRNG_DIR_DST,
-		.max_size = HAL_CE_DST_STATUS_RING_BASE_MSB_RING_SIZE,
-	},
-	[HAL_WBM_IDLE_LINK] = {
-		.start_ring_id = HAL_SRNG_RING_ID_WBM_IDLE_LINK,
-		.max_rings = 1,
-		.entry_size = sizeof(struct hal_wbm_link_desc) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_UMAC,
-		.ring_dir = HAL_SRNG_DIR_SRC,
-		.max_size = HAL_WBM_IDLE_LINK_RING_BASE_MSB_RING_SIZE,
-	},
-	[HAL_SW2WBM_RELEASE] = {
-		.start_ring_id = HAL_SRNG_RING_ID_WBM_SW0_RELEASE,
-		.max_rings = 2,
-		.entry_size = sizeof(struct hal_wbm_release_ring) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_UMAC,
-		.ring_dir = HAL_SRNG_DIR_SRC,
-		.max_size = HAL_SW2WBM_RELEASE_RING_BASE_MSB_RING_SIZE,
-	},
-	[HAL_WBM2SW_RELEASE] = {
-		.start_ring_id = HAL_SRNG_RING_ID_WBM2SW0_RELEASE,
-		.max_rings = 8,
-		.entry_size = sizeof(struct hal_wbm_release_ring) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_UMAC,
-		.ring_dir = HAL_SRNG_DIR_DST,
-		.max_size = HAL_WBM2SW_RELEASE_RING_BASE_MSB_RING_SIZE,
-	},
-	[HAL_RXDMA_BUF] = {
-		.start_ring_id = HAL_SRNG_SW2RXDMA_BUF0,
-		.max_rings = 1,
-		.entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_DMAC,
-		.ring_dir = HAL_SRNG_DIR_SRC,
-		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
-	},
-	[HAL_RXDMA_DST] = {
-		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_RXDMA2SW0,
-		.max_rings = 0,
-		.entry_size = 0,
-		.mac_type = ATH12K_HAL_SRNG_PMAC,
-		.ring_dir = HAL_SRNG_DIR_DST,
-		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
-	},
-	[HAL_RXDMA_MONITOR_BUF] = {
-		.start_ring_id = HAL_SRNG_SW2RXMON_BUF0,
-		.max_rings = 1,
-		.entry_size = sizeof(struct hal_mon_buf_ring) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_PMAC,
-		.ring_dir = HAL_SRNG_DIR_SRC,
-		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
-	},
-	[HAL_RXDMA_MONITOR_STATUS] = {
-		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA1_STATBUF,
-		.max_rings = 1,
-		.entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_PMAC,
-		.ring_dir = HAL_SRNG_DIR_SRC,
-		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
-	},
-	[HAL_RXDMA_MONITOR_DESC] = { 0, },
-	[HAL_RXDMA_DIR_BUF] = {
-		.start_ring_id = HAL_SRNG_RING_ID_RXDMA_DIR_BUF,
-		.max_rings = 2,
-		.entry_size = 8 >> 2, /* TODO: Define the struct */
-		.mac_type = ATH12K_HAL_SRNG_PMAC,
-		.ring_dir = HAL_SRNG_DIR_SRC,
-		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
-	},
-	[HAL_PPE2TCL] = {
-		.start_ring_id = HAL_SRNG_RING_ID_PPE2TCL1,
-		.max_rings = 1,
-		.entry_size = sizeof(struct hal_tcl_entrance_from_ppe_ring) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_PMAC,
-		.ring_dir = HAL_SRNG_DIR_SRC,
-		.max_size = HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE,
-	},
-	[HAL_PPE_RELEASE] = {
-		.start_ring_id = HAL_SRNG_RING_ID_WBM_PPE_RELEASE,
-		.max_rings = 1,
-		.entry_size = sizeof(struct hal_wbm_release_ring) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_PMAC,
-		.ring_dir = HAL_SRNG_DIR_SRC,
-		.max_size = HAL_WBM2PPE_RELEASE_RING_BASE_MSB_RING_SIZE,
-	},
-	[HAL_TX_MONITOR_BUF] = {
-		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2TXMON_BUF0,
-		.max_rings = 1,
-		.entry_size = sizeof(struct hal_mon_buf_ring) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_PMAC,
-		.ring_dir = HAL_SRNG_DIR_SRC,
-		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
-	},
-	[HAL_RXDMA_MONITOR_DST] = {
-		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXMON_BUF0,
-		.max_rings = 1,
-		.entry_size = sizeof(struct hal_mon_dest_desc) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_PMAC,
-		.ring_dir = HAL_SRNG_DIR_DST,
-		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
-	},
-	[HAL_TX_MONITOR_DST] = {
-		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_TXMON2SW0_BUF0,
-		.max_rings = 1,
-		.entry_size = sizeof(struct hal_mon_dest_desc) >> 2,
-		.mac_type = ATH12K_HAL_SRNG_PMAC,
-		.ring_dir = HAL_SRNG_DIR_DST,
-		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
-	}
-};
-
-static const struct ath12k_hal_tcl_to_wbm_rbm_map
-ath12k_hal_qcn9274_tcl_to_wbm_rbm_map[DP_TCL_NUM_RING_MAX] = {
-	{
-		.wbm_ring_num = 0,
-		.rbm_id = HAL_RX_BUF_RBM_SW0_BM,
-	},
-	{
-		.wbm_ring_num = 1,
-		.rbm_id = HAL_RX_BUF_RBM_SW1_BM,
-	},
-	{
-		.wbm_ring_num = 2,
-		.rbm_id = HAL_RX_BUF_RBM_SW2_BM,
-	},
-	{
-		.wbm_ring_num = 4,
-		.rbm_id = HAL_RX_BUF_RBM_SW4_BM,
-	}
-};
-
-static const struct ath12k_hal_tcl_to_wbm_rbm_map
-ath12k_hal_wcn7850_tcl_to_wbm_rbm_map[DP_TCL_NUM_RING_MAX] = {
-	{
-		.wbm_ring_num = 0,
-		.rbm_id = HAL_RX_BUF_RBM_SW0_BM,
-	},
-	{
-		.wbm_ring_num = 2,
-		.rbm_id = HAL_RX_BUF_RBM_SW2_BM,
-	},
-	{
-		.wbm_ring_num = 4,
-		.rbm_id = HAL_RX_BUF_RBM_SW4_BM,
-	},
-};
-
-static unsigned int ath12k_hal_reo1_ring_id_offset(struct ath12k_base *ab)
-{
-	return HAL_REO1_RING_ID(ab) - HAL_REO1_RING_BASE_LSB(ab);
-}
-
-static unsigned int ath12k_hal_reo1_ring_msi1_base_lsb_offset(struct ath12k_base *ab)
-{
-	return HAL_REO1_RING_MSI1_BASE_LSB(ab) - HAL_REO1_RING_BASE_LSB(ab);
-}
-
-static unsigned int ath12k_hal_reo1_ring_msi1_base_msb_offset(struct ath12k_base *ab)
-{
-	return HAL_REO1_RING_MSI1_BASE_MSB(ab) - HAL_REO1_RING_BASE_LSB(ab);
-}
-
-static unsigned int ath12k_hal_reo1_ring_msi1_data_offset(struct ath12k_base *ab)
-{
-	return HAL_REO1_RING_MSI1_DATA(ab) - HAL_REO1_RING_BASE_LSB(ab);
-}
-
-static unsigned int ath12k_hal_reo1_ring_base_msb_offset(struct ath12k_base *ab)
-{
-	return HAL_REO1_RING_BASE_MSB(ab) - HAL_REO1_RING_BASE_LSB(ab);
-}
-
-static unsigned int ath12k_hal_reo1_ring_producer_int_setup_offset(struct ath12k_base *ab)
-{
-	return HAL_REO1_RING_PRODUCER_INT_SETUP(ab) - HAL_REO1_RING_BASE_LSB(ab);
-}
-
-static unsigned int ath12k_hal_reo1_ring_hp_addr_lsb_offset(struct ath12k_base *ab)
-{
-	return HAL_REO1_RING_HP_ADDR_LSB(ab) - HAL_REO1_RING_BASE_LSB(ab);
-}
-
-static unsigned int ath12k_hal_reo1_ring_hp_addr_msb_offset(struct ath12k_base *ab)
-{
-	return HAL_REO1_RING_HP_ADDR_MSB(ab) - HAL_REO1_RING_BASE_LSB(ab);
-}
-
-static unsigned int ath12k_hal_reo1_ring_misc_offset(struct ath12k_base *ab)
-{
-	return HAL_REO1_RING_MISC(ab) - HAL_REO1_RING_BASE_LSB(ab);
-}
-
-static bool ath12k_hw_qcn9274_rx_desc_get_first_msdu(struct hal_rx_desc *desc)
-{
-	return !!le16_get_bits(desc->u.qcn9274.msdu_end.info5,
-			       RX_MSDU_END_INFO5_FIRST_MSDU);
-}
-
-static bool ath12k_hw_qcn9274_rx_desc_get_last_msdu(struct hal_rx_desc *desc)
-{
-	return !!le16_get_bits(desc->u.qcn9274.msdu_end.info5,
-			       RX_MSDU_END_INFO5_LAST_MSDU);
-}
-
-static u8 ath12k_hw_qcn9274_rx_desc_get_l3_pad_bytes(struct hal_rx_desc *desc)
-{
-	return le16_get_bits(desc->u.qcn9274.msdu_end.info5,
-			     RX_MSDU_END_INFO5_L3_HDR_PADDING);
-}
-
-static bool ath12k_hw_qcn9274_rx_desc_encrypt_valid(struct hal_rx_desc *desc)
-{
-	return !!le32_get_bits(desc->u.qcn9274.mpdu_start.info4,
-			       RX_MPDU_START_INFO4_ENCRYPT_INFO_VALID);
-}
-
-static u32 ath12k_hw_qcn9274_rx_desc_get_encrypt_type(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274.mpdu_start.info2,
-			     RX_MPDU_START_INFO2_ENC_TYPE);
-}
-
-static u8 ath12k_hw_qcn9274_rx_desc_get_decap_type(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274.msdu_end.info11,
-			     RX_MSDU_END_INFO11_DECAP_FORMAT);
-}
-
-static u8 ath12k_hw_qcn9274_rx_desc_get_mesh_ctl(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274.msdu_end.info11,
-			     RX_MSDU_END_INFO11_MESH_CTRL_PRESENT);
-}
-
-static bool ath12k_hw_qcn9274_rx_desc_get_mpdu_seq_ctl_vld(struct hal_rx_desc *desc)
-{
-	return !!le32_get_bits(desc->u.qcn9274.mpdu_start.info4,
-			       RX_MPDU_START_INFO4_MPDU_SEQ_CTRL_VALID);
-}
-
-static bool ath12k_hw_qcn9274_rx_desc_get_mpdu_fc_valid(struct hal_rx_desc *desc)
-{
-	return !!le32_get_bits(desc->u.qcn9274.mpdu_start.info4,
-			       RX_MPDU_START_INFO4_MPDU_FCTRL_VALID);
-}
-
-static u16 ath12k_hw_qcn9274_rx_desc_get_mpdu_start_seq_no(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274.mpdu_start.info4,
-			     RX_MPDU_START_INFO4_MPDU_SEQ_NUM);
-}
-
-static u16 ath12k_hw_qcn9274_rx_desc_get_msdu_len(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274.msdu_end.info10,
-			     RX_MSDU_END_INFO10_MSDU_LENGTH);
-}
-
-static u8 ath12k_hw_qcn9274_rx_desc_get_msdu_sgi(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274.msdu_end.info12,
-			     RX_MSDU_END_INFO12_SGI);
-}
-
-static u8 ath12k_hw_qcn9274_rx_desc_get_msdu_rate_mcs(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274.msdu_end.info12,
-			     RX_MSDU_END_INFO12_RATE_MCS);
-}
-
-static u8 ath12k_hw_qcn9274_rx_desc_get_msdu_rx_bw(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274.msdu_end.info12,
-			     RX_MSDU_END_INFO12_RECV_BW);
-}
-
-static u32 ath12k_hw_qcn9274_rx_desc_get_msdu_freq(struct hal_rx_desc *desc)
-{
-	return __le32_to_cpu(desc->u.qcn9274.msdu_end.phy_meta_data);
-}
-
-static u8 ath12k_hw_qcn9274_rx_desc_get_msdu_pkt_type(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274.msdu_end.info12,
-			     RX_MSDU_END_INFO12_PKT_TYPE);
-}
-
-static u8 ath12k_hw_qcn9274_rx_desc_get_msdu_nss(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274.msdu_end.info12,
-			     RX_MSDU_END_INFO12_MIMO_SS_BITMAP);
-}
-
-static u8 ath12k_hw_qcn9274_rx_desc_get_mpdu_tid(struct hal_rx_desc *desc)
-{
-	return le16_get_bits(desc->u.qcn9274.msdu_end.info5,
-			    RX_MSDU_END_INFO5_TID);
-}
-
-static u16 ath12k_hw_qcn9274_rx_desc_get_mpdu_peer_id(struct hal_rx_desc *desc)
-{
-	return __le16_to_cpu(desc->u.qcn9274.mpdu_start.sw_peer_id);
-}
-
-static void ath12k_hw_qcn9274_rx_desc_copy_end_tlv(struct hal_rx_desc *fdesc,
-						   struct hal_rx_desc *ldesc)
-{
-	memcpy(&fdesc->u.qcn9274.msdu_end, &ldesc->u.qcn9274.msdu_end,
-	       sizeof(struct rx_msdu_end_qcn9274));
-}
-
-static u32 ath12k_hw_qcn9274_rx_desc_get_mpdu_ppdu_id(struct hal_rx_desc *desc)
-{
-	return __le16_to_cpu(desc->u.qcn9274.mpdu_start.phy_ppdu_id);
-}
-
-static void ath12k_hw_qcn9274_rx_desc_set_msdu_len(struct hal_rx_desc *desc, u16 len)
-{
-	u32 info = __le32_to_cpu(desc->u.qcn9274.msdu_end.info10);
-
-	info &= ~RX_MSDU_END_INFO10_MSDU_LENGTH;
-	info |= u32_encode_bits(len, RX_MSDU_END_INFO10_MSDU_LENGTH);
-
-	desc->u.qcn9274.msdu_end.info10 = __cpu_to_le32(info);
-}
-
-static u8 *ath12k_hw_qcn9274_rx_desc_get_msdu_payload(struct hal_rx_desc *desc)
-{
-	return &desc->u.qcn9274.msdu_payload[0];
-}
-
-static u32 ath12k_hw_qcn9274_rx_desc_get_mpdu_start_offset(void)
-{
-	return offsetof(struct hal_rx_desc_qcn9274, mpdu_start);
-}
-
-static u32 ath12k_hw_qcn9274_rx_desc_get_msdu_end_offset(void)
-{
-	return offsetof(struct hal_rx_desc_qcn9274, msdu_end);
-}
-
-static bool ath12k_hw_qcn9274_rx_desc_mac_addr2_valid(struct hal_rx_desc *desc)
-{
-	return __le32_to_cpu(desc->u.qcn9274.mpdu_start.info4) &
-	       RX_MPDU_START_INFO4_MAC_ADDR2_VALID;
-}
-
-static u8 *ath12k_hw_qcn9274_rx_desc_mpdu_start_addr2(struct hal_rx_desc *desc)
-{
-	return desc->u.qcn9274.mpdu_start.addr2;
-}
-
-static bool ath12k_hw_qcn9274_rx_desc_is_da_mcbc(struct hal_rx_desc *desc)
-{
-	return __le16_to_cpu(desc->u.qcn9274.msdu_end.info5) &
-	       RX_MSDU_END_INFO5_DA_IS_MCBC;
-}
-
-static void ath12k_hw_qcn9274_rx_desc_get_dot11_hdr(struct hal_rx_desc *desc,
-						    struct ieee80211_hdr *hdr)
-{
-	hdr->frame_control = desc->u.qcn9274.mpdu_start.frame_ctrl;
-	hdr->duration_id = desc->u.qcn9274.mpdu_start.duration;
-	ether_addr_copy(hdr->addr1, desc->u.qcn9274.mpdu_start.addr1);
-	ether_addr_copy(hdr->addr2, desc->u.qcn9274.mpdu_start.addr2);
-	ether_addr_copy(hdr->addr3, desc->u.qcn9274.mpdu_start.addr3);
-	if (__le32_to_cpu(desc->u.qcn9274.mpdu_start.info4) &
-			RX_MPDU_START_INFO4_MAC_ADDR4_VALID) {
-		ether_addr_copy(hdr->addr4, desc->u.qcn9274.mpdu_start.addr4);
-	}
-	hdr->seq_ctrl = desc->u.qcn9274.mpdu_start.seq_ctrl;
-}
-
-static void ath12k_hw_qcn9274_rx_desc_get_crypto_hdr(struct hal_rx_desc *desc,
-						     u8 *crypto_hdr,
-						     enum hal_encrypt_type enctype)
-{
-	unsigned int key_id;
-
-	switch (enctype) {
-	case HAL_ENCRYPT_TYPE_OPEN:
-		return;
-	case HAL_ENCRYPT_TYPE_TKIP_NO_MIC:
-	case HAL_ENCRYPT_TYPE_TKIP_MIC:
-		crypto_hdr[0] =
-			HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274.mpdu_start.pn[0]);
-		crypto_hdr[1] = 0;
-		crypto_hdr[2] =
-			HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274.mpdu_start.pn[0]);
-		break;
-	case HAL_ENCRYPT_TYPE_CCMP_128:
-	case HAL_ENCRYPT_TYPE_CCMP_256:
-	case HAL_ENCRYPT_TYPE_GCMP_128:
-	case HAL_ENCRYPT_TYPE_AES_GCMP_256:
-		crypto_hdr[0] =
-			HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274.mpdu_start.pn[0]);
-		crypto_hdr[1] =
-			HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274.mpdu_start.pn[0]);
-		crypto_hdr[2] = 0;
-		break;
-	case HAL_ENCRYPT_TYPE_WEP_40:
-	case HAL_ENCRYPT_TYPE_WEP_104:
-	case HAL_ENCRYPT_TYPE_WEP_128:
-	case HAL_ENCRYPT_TYPE_WAPI_GCM_SM4:
-	case HAL_ENCRYPT_TYPE_WAPI:
-		return;
-	}
-	key_id = le32_get_bits(desc->u.qcn9274.mpdu_start.info5,
-			       RX_MPDU_START_INFO5_KEY_ID);
-	crypto_hdr[3] = 0x20 | (key_id << 6);
-	crypto_hdr[4] = HAL_RX_MPDU_INFO_PN_GET_BYTE3(desc->u.qcn9274.mpdu_start.pn[0]);
-	crypto_hdr[5] = HAL_RX_MPDU_INFO_PN_GET_BYTE4(desc->u.qcn9274.mpdu_start.pn[0]);
-	crypto_hdr[6] = HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274.mpdu_start.pn[1]);
-	crypto_hdr[7] = HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274.mpdu_start.pn[1]);
-}
-
-static int ath12k_hal_srng_create_config_qcn9274(struct ath12k_base *ab)
-{
-	struct ath12k_hal *hal = &ab->hal;
-	struct hal_srng_config *s;
-
-	hal->srng_config = kmemdup(hw_srng_config_template,
-				   sizeof(hw_srng_config_template),
-				   GFP_KERNEL);
-	if (!hal->srng_config)
-		return -ENOMEM;
-
-	s = &hal->srng_config[HAL_REO_DST];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_HP;
-	s->reg_size[0] = HAL_REO2_RING_BASE_LSB(ab) - HAL_REO1_RING_BASE_LSB(ab);
-	s->reg_size[1] = HAL_REO2_RING_HP - HAL_REO1_RING_HP;
-
-	s = &hal->srng_config[HAL_REO_EXCEPTION];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_SW0_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_SW0_RING_HP;
-
-	s = &hal->srng_config[HAL_REO_REINJECT];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_HP;
-	s->reg_size[0] = HAL_SW2REO1_RING_BASE_LSB(ab) - HAL_SW2REO_RING_BASE_LSB(ab);
-	s->reg_size[1] = HAL_SW2REO1_RING_HP - HAL_SW2REO_RING_HP;
-
-	s = &hal->srng_config[HAL_REO_CMD];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_HP;
-
-	s = &hal->srng_config[HAL_REO_STATUS];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_HP;
-
-	s = &hal->srng_config[HAL_TCL_DATA];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_HP;
-	s->reg_size[0] = HAL_TCL2_RING_BASE_LSB(ab) - HAL_TCL1_RING_BASE_LSB(ab);
-	s->reg_size[1] = HAL_TCL2_RING_HP - HAL_TCL1_RING_HP;
-
-	s = &hal->srng_config[HAL_TCL_CMD];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_HP;
-
-	s = &hal->srng_config[HAL_TCL_STATUS];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_HP;
-
-	s = &hal->srng_config[HAL_CE_SRC];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) + HAL_CE_DST_RING_BASE_LSB;
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) + HAL_CE_DST_RING_HP;
-	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) -
-		HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab);
-	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) -
-		HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab);
-
-	s = &hal->srng_config[HAL_CE_DST];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_RING_BASE_LSB;
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_RING_HP;
-	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -
-		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);
-	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -
-		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);
-
-	s = &hal->srng_config[HAL_CE_DST_STATUS];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) +
-		HAL_CE_DST_STATUS_RING_BASE_LSB;
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_STATUS_RING_HP;
-	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -
-		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);
-	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -
-		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);
-
-	s = &hal->srng_config[HAL_WBM_IDLE_LINK];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_HP;
-
-	s = &hal->srng_config[HAL_SW2WBM_RELEASE];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG +
-		HAL_WBM_SW_RELEASE_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_SW_RELEASE_RING_HP;
-	s->reg_size[0] = HAL_WBM_SW1_RELEASE_RING_BASE_LSB(ab) -
-			 HAL_WBM_SW_RELEASE_RING_BASE_LSB(ab);
-	s->reg_size[1] = HAL_WBM_SW1_RELEASE_RING_HP - HAL_WBM_SW_RELEASE_RING_HP;
-
-	s = &hal->srng_config[HAL_WBM2SW_RELEASE];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_HP;
-	s->reg_size[0] = HAL_WBM1_RELEASE_RING_BASE_LSB(ab) -
-		HAL_WBM0_RELEASE_RING_BASE_LSB(ab);
-	s->reg_size[1] = HAL_WBM1_RELEASE_RING_HP - HAL_WBM0_RELEASE_RING_HP;
-
-	/* Some LMAC rings are not accessed from the host:
-	 * RXDMA_BUG, RXDMA_DST, RXDMA_MONITOR_BUF, RXDMA_MONITOR_STATUS,
-	 * RXDMA_MONITOR_DST, RXDMA_MONITOR_DESC, RXDMA_DIR_BUF_SRC,
-	 * RXDMA_RX_MONITOR_BUF, TX_MONITOR_BUF, TX_MONITOR_DST, SW2RXDMA
-	 */
-	s = &hal->srng_config[HAL_PPE2TCL];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_PPE2TCL1_RING_BASE_LSB;
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_PPE2TCL1_RING_HP;
-
-	s = &hal->srng_config[HAL_PPE_RELEASE];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG +
-				HAL_WBM_PPE_RELEASE_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_PPE_RELEASE_RING_HP;
-
-	return 0;
-}
-
-static u16 ath12k_hal_qcn9274_rx_mpdu_start_wmask_get(void)
-{
-	return QCN9274_MPDU_START_WMASK;
-}
-
-static u32 ath12k_hal_qcn9274_rx_msdu_end_wmask_get(void)
-{
-	return QCN9274_MSDU_END_WMASK;
-}
-
-static const struct hal_rx_ops *ath12k_hal_qcn9274_get_hal_rx_compact_ops(void)
-{
-	return &hal_rx_qcn9274_compact_ops;
-}
-
-static bool ath12k_hw_qcn9274_dp_rx_h_msdu_done(struct hal_rx_desc *desc)
-{
-	return !!le32_get_bits(desc->u.qcn9274.msdu_end.info14,
-			       RX_MSDU_END_INFO14_MSDU_DONE);
-}
-
-static bool ath12k_hw_qcn9274_dp_rx_h_l4_cksum_fail(struct hal_rx_desc *desc)
-{
-	return !!le32_get_bits(desc->u.qcn9274.msdu_end.info13,
-			       RX_MSDU_END_INFO13_TCP_UDP_CKSUM_FAIL);
-}
-
-static bool ath12k_hw_qcn9274_dp_rx_h_ip_cksum_fail(struct hal_rx_desc *desc)
-{
-	return !!le32_get_bits(desc->u.qcn9274.msdu_end.info13,
-			       RX_MSDU_END_INFO13_IP_CKSUM_FAIL);
-}
-
-static bool ath12k_hw_qcn9274_dp_rx_h_is_decrypted(struct hal_rx_desc *desc)
-{
-	return (le32_get_bits(desc->u.qcn9274.msdu_end.info14,
-			      RX_MSDU_END_INFO14_DECRYPT_STATUS_CODE) ==
-			      RX_DESC_DECRYPT_STATUS_CODE_OK);
-}
-
-static u32 ath12k_hw_qcn9274_dp_rx_h_mpdu_err(struct hal_rx_desc *desc)
-{
-	u32 info = __le32_to_cpu(desc->u.qcn9274.msdu_end.info13);
-	u32 errmap = 0;
-
-	if (info & RX_MSDU_END_INFO13_FCS_ERR)
-		errmap |= HAL_RX_MPDU_ERR_FCS;
-
-	if (info & RX_MSDU_END_INFO13_DECRYPT_ERR)
-		errmap |= HAL_RX_MPDU_ERR_DECRYPT;
-
-	if (info & RX_MSDU_END_INFO13_TKIP_MIC_ERR)
-		errmap |= HAL_RX_MPDU_ERR_TKIP_MIC;
-
-	if (info & RX_MSDU_END_INFO13_A_MSDU_ERROR)
-		errmap |= HAL_RX_MPDU_ERR_AMSDU_ERR;
-
-	if (info & RX_MSDU_END_INFO13_OVERFLOW_ERR)
-		errmap |= HAL_RX_MPDU_ERR_OVERFLOW;
-
-	if (info & RX_MSDU_END_INFO13_MSDU_LEN_ERR)
-		errmap |= HAL_RX_MPDU_ERR_MSDU_LEN;
-
-	if (info & RX_MSDU_END_INFO13_MPDU_LEN_ERR)
-		errmap |= HAL_RX_MPDU_ERR_MPDU_LEN;
-
-	return errmap;
-}
-
-static u32 ath12k_hw_qcn9274_get_rx_desc_size(void)
-{
-	return sizeof(struct hal_rx_desc_qcn9274);
-}
-
-static u8 ath12k_hw_qcn9274_rx_desc_get_msdu_src_link(struct hal_rx_desc *desc)
-{
-	return 0;
-}
-
-const struct hal_rx_ops hal_rx_qcn9274_ops = {
-	.rx_desc_get_first_msdu = ath12k_hw_qcn9274_rx_desc_get_first_msdu,
-	.rx_desc_get_last_msdu = ath12k_hw_qcn9274_rx_desc_get_last_msdu,
-	.rx_desc_get_l3_pad_bytes = ath12k_hw_qcn9274_rx_desc_get_l3_pad_bytes,
-	.rx_desc_encrypt_valid = ath12k_hw_qcn9274_rx_desc_encrypt_valid,
-	.rx_desc_get_encrypt_type = ath12k_hw_qcn9274_rx_desc_get_encrypt_type,
-	.rx_desc_get_decap_type = ath12k_hw_qcn9274_rx_desc_get_decap_type,
-	.rx_desc_get_mesh_ctl = ath12k_hw_qcn9274_rx_desc_get_mesh_ctl,
-	.rx_desc_get_mpdu_seq_ctl_vld = ath12k_hw_qcn9274_rx_desc_get_mpdu_seq_ctl_vld,
-	.rx_desc_get_mpdu_fc_valid = ath12k_hw_qcn9274_rx_desc_get_mpdu_fc_valid,
-	.rx_desc_get_mpdu_start_seq_no = ath12k_hw_qcn9274_rx_desc_get_mpdu_start_seq_no,
-	.rx_desc_get_msdu_len = ath12k_hw_qcn9274_rx_desc_get_msdu_len,
-	.rx_desc_get_msdu_sgi = ath12k_hw_qcn9274_rx_desc_get_msdu_sgi,
-	.rx_desc_get_msdu_rate_mcs = ath12k_hw_qcn9274_rx_desc_get_msdu_rate_mcs,
-	.rx_desc_get_msdu_rx_bw = ath12k_hw_qcn9274_rx_desc_get_msdu_rx_bw,
-	.rx_desc_get_msdu_freq = ath12k_hw_qcn9274_rx_desc_get_msdu_freq,
-	.rx_desc_get_msdu_pkt_type = ath12k_hw_qcn9274_rx_desc_get_msdu_pkt_type,
-	.rx_desc_get_msdu_nss = ath12k_hw_qcn9274_rx_desc_get_msdu_nss,
-	.rx_desc_get_mpdu_tid = ath12k_hw_qcn9274_rx_desc_get_mpdu_tid,
-	.rx_desc_get_mpdu_peer_id = ath12k_hw_qcn9274_rx_desc_get_mpdu_peer_id,
-	.rx_desc_copy_end_tlv = ath12k_hw_qcn9274_rx_desc_copy_end_tlv,
-	.rx_desc_get_mpdu_ppdu_id = ath12k_hw_qcn9274_rx_desc_get_mpdu_ppdu_id,
-	.rx_desc_set_msdu_len = ath12k_hw_qcn9274_rx_desc_set_msdu_len,
-	.rx_desc_get_msdu_payload = ath12k_hw_qcn9274_rx_desc_get_msdu_payload,
-	.rx_desc_get_mpdu_start_offset = ath12k_hw_qcn9274_rx_desc_get_mpdu_start_offset,
-	.rx_desc_get_msdu_end_offset = ath12k_hw_qcn9274_rx_desc_get_msdu_end_offset,
-	.rx_desc_mac_addr2_valid = ath12k_hw_qcn9274_rx_desc_mac_addr2_valid,
-	.rx_desc_mpdu_start_addr2 = ath12k_hw_qcn9274_rx_desc_mpdu_start_addr2,
-	.rx_desc_is_da_mcbc = ath12k_hw_qcn9274_rx_desc_is_da_mcbc,
-	.rx_desc_get_dot11_hdr = ath12k_hw_qcn9274_rx_desc_get_dot11_hdr,
-	.rx_desc_get_crypto_header = ath12k_hw_qcn9274_rx_desc_get_crypto_hdr,
-	.dp_rx_h_msdu_done = ath12k_hw_qcn9274_dp_rx_h_msdu_done,
-	.dp_rx_h_l4_cksum_fail = ath12k_hw_qcn9274_dp_rx_h_l4_cksum_fail,
-	.dp_rx_h_ip_cksum_fail = ath12k_hw_qcn9274_dp_rx_h_ip_cksum_fail,
-	.dp_rx_h_is_decrypted = ath12k_hw_qcn9274_dp_rx_h_is_decrypted,
-	.dp_rx_h_mpdu_err = ath12k_hw_qcn9274_dp_rx_h_mpdu_err,
-	.rx_desc_get_desc_size = ath12k_hw_qcn9274_get_rx_desc_size,
-	.rx_desc_get_msdu_src_link_id = ath12k_hw_qcn9274_rx_desc_get_msdu_src_link,
-};
-
-static bool ath12k_hw_qcn9274_compact_rx_desc_get_first_msdu(struct hal_rx_desc *desc)
-{
-	return !!le16_get_bits(desc->u.qcn9274_compact.msdu_end.info5,
-			       RX_MSDU_END_INFO5_FIRST_MSDU);
-}
-
-static bool ath12k_hw_qcn9274_compact_rx_desc_get_last_msdu(struct hal_rx_desc *desc)
-{
-	return !!le16_get_bits(desc->u.qcn9274_compact.msdu_end.info5,
-			       RX_MSDU_END_INFO5_LAST_MSDU);
-}
-
-static u8 ath12k_hw_qcn9274_compact_rx_desc_get_l3_pad_bytes(struct hal_rx_desc *desc)
-{
-	return le16_get_bits(desc->u.qcn9274_compact.msdu_end.info5,
-			     RX_MSDU_END_INFO5_L3_HDR_PADDING);
-}
-
-static bool ath12k_hw_qcn9274_compact_rx_desc_encrypt_valid(struct hal_rx_desc *desc)
-{
-	return !!le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info4,
-			       RX_MPDU_START_INFO4_ENCRYPT_INFO_VALID);
-}
-
-static u32 ath12k_hw_qcn9274_compact_rx_desc_get_encrypt_type(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info2,
-			     RX_MPDU_START_INFO2_ENC_TYPE);
-}
-
-static u8 ath12k_hw_qcn9274_compact_rx_desc_get_decap_type(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info11,
-			     RX_MSDU_END_INFO11_DECAP_FORMAT);
-}
-
-static u8 ath12k_hw_qcn9274_compact_rx_desc_get_mesh_ctl(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274.msdu_end.info11,
-			     RX_MSDU_END_INFO11_MESH_CTRL_PRESENT);
-}
-
-static bool
-ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_seq_ctl_vld(struct hal_rx_desc *desc)
-{
-	return !!le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info4,
-			       RX_MPDU_START_INFO4_MPDU_SEQ_CTRL_VALID);
-}
-
-static bool ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_fc_valid(struct hal_rx_desc *desc)
-{
-	return !!le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info4,
-			       RX_MPDU_START_INFO4_MPDU_FCTRL_VALID);
-}
-
-static u16
-ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_start_seq_no(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info4,
-			     RX_MPDU_START_INFO4_MPDU_SEQ_NUM);
-}
-
-static u16 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_len(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info10,
-			     RX_MSDU_END_INFO10_MSDU_LENGTH);
-}
-
-static u8 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_sgi(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12,
-			     RX_MSDU_END_INFO12_SGI);
-}
-
-static u8 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_rate_mcs(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12,
-			     RX_MSDU_END_INFO12_RATE_MCS);
-}
-
-static u8 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_rx_bw(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12,
-			     RX_MSDU_END_INFO12_RECV_BW);
-}
-
-static u32 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_freq(struct hal_rx_desc *desc)
-{
-	return __le32_to_cpu(desc->u.qcn9274_compact.msdu_end.phy_meta_data);
-}
-
-static u8 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_pkt_type(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12,
-			     RX_MSDU_END_INFO12_PKT_TYPE);
-}
-
-static u8 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_nss(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12,
-			     RX_MSDU_END_INFO12_MIMO_SS_BITMAP);
-}
-
-static u8 ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_tid(struct hal_rx_desc *desc)
-{
-	return le16_get_bits(desc->u.qcn9274_compact.msdu_end.info5,
-			     RX_MSDU_END_INFO5_TID);
-}
-
-static u16 ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_peer_id(struct hal_rx_desc *desc)
-{
-	return __le16_to_cpu(desc->u.qcn9274_compact.mpdu_start.sw_peer_id);
-}
-
-static void ath12k_hw_qcn9274_compact_rx_desc_copy_end_tlv(struct hal_rx_desc *fdesc,
-							   struct hal_rx_desc *ldesc)
-{
-	fdesc->u.qcn9274_compact.msdu_end = ldesc->u.qcn9274_compact.msdu_end;
-}
-
-static u32 ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_ppdu_id(struct hal_rx_desc *desc)
-{
-	return __le16_to_cpu(desc->u.qcn9274_compact.mpdu_start.phy_ppdu_id);
-}
-
-static void
-ath12k_hw_qcn9274_compact_rx_desc_set_msdu_len(struct hal_rx_desc *desc, u16 len)
-{
-	u32 info = __le32_to_cpu(desc->u.qcn9274_compact.msdu_end.info10);
-
-	info = u32_replace_bits(info, len, RX_MSDU_END_INFO10_MSDU_LENGTH);
-	desc->u.qcn9274_compact.msdu_end.info10 = __cpu_to_le32(info);
-}
-
-static u8 *ath12k_hw_qcn9274_compact_rx_desc_get_msdu_payload(struct hal_rx_desc *desc)
-{
-	return &desc->u.qcn9274_compact.msdu_payload[0];
-}
-
-static u32 ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_start_offset(void)
-{
-	return offsetof(struct hal_rx_desc_qcn9274_compact, mpdu_start);
-}
-
-static u32 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_end_offset(void)
-{
-	return offsetof(struct hal_rx_desc_qcn9274_compact, msdu_end);
-}
-
-static bool ath12k_hw_qcn9274_compact_rx_desc_mac_addr2_valid(struct hal_rx_desc *desc)
-{
-	return __le32_to_cpu(desc->u.qcn9274_compact.mpdu_start.info4) &
-			     RX_MPDU_START_INFO4_MAC_ADDR2_VALID;
-}
-
-static u8 *ath12k_hw_qcn9274_compact_rx_desc_mpdu_start_addr2(struct hal_rx_desc *desc)
-{
-	return desc->u.qcn9274_compact.mpdu_start.addr2;
-}
-
-static bool ath12k_hw_qcn9274_compact_rx_desc_is_da_mcbc(struct hal_rx_desc *desc)
-{
-	return __le16_to_cpu(desc->u.qcn9274_compact.msdu_end.info5) &
-	       RX_MSDU_END_INFO5_DA_IS_MCBC;
-}
-
-static void ath12k_hw_qcn9274_compact_rx_desc_get_dot11_hdr(struct hal_rx_desc *desc,
-							    struct ieee80211_hdr *hdr)
-{
-	hdr->frame_control = desc->u.qcn9274_compact.mpdu_start.frame_ctrl;
-	hdr->duration_id = desc->u.qcn9274_compact.mpdu_start.duration;
-	ether_addr_copy(hdr->addr1, desc->u.qcn9274_compact.mpdu_start.addr1);
-	ether_addr_copy(hdr->addr2, desc->u.qcn9274_compact.mpdu_start.addr2);
-	ether_addr_copy(hdr->addr3, desc->u.qcn9274_compact.mpdu_start.addr3);
-	if (__le32_to_cpu(desc->u.qcn9274_compact.mpdu_start.info4) &
-			RX_MPDU_START_INFO4_MAC_ADDR4_VALID) {
-		ether_addr_copy(hdr->addr4, desc->u.qcn9274_compact.mpdu_start.addr4);
-	}
-	hdr->seq_ctrl = desc->u.qcn9274_compact.mpdu_start.seq_ctrl;
-}
-
-static void
-ath12k_hw_qcn9274_compact_rx_desc_get_crypto_hdr(struct hal_rx_desc *desc,
-						 u8 *crypto_hdr,
-						 enum hal_encrypt_type enctype)
-{
-	unsigned int key_id;
-
-	switch (enctype) {
-	case HAL_ENCRYPT_TYPE_OPEN:
-		return;
-	case HAL_ENCRYPT_TYPE_TKIP_NO_MIC:
-	case HAL_ENCRYPT_TYPE_TKIP_MIC:
-		crypto_hdr[0] =
-		HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274_compact.mpdu_start.pn[0]);
-		crypto_hdr[1] = 0;
-		crypto_hdr[2] =
-		HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274_compact.mpdu_start.pn[0]);
-		break;
-	case HAL_ENCRYPT_TYPE_CCMP_128:
-	case HAL_ENCRYPT_TYPE_CCMP_256:
-	case HAL_ENCRYPT_TYPE_GCMP_128:
-	case HAL_ENCRYPT_TYPE_AES_GCMP_256:
-		crypto_hdr[0] =
-		HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274_compact.mpdu_start.pn[0]);
-		crypto_hdr[1] =
-		HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274_compact.mpdu_start.pn[0]);
-		crypto_hdr[2] = 0;
-		break;
-	case HAL_ENCRYPT_TYPE_WEP_40:
-	case HAL_ENCRYPT_TYPE_WEP_104:
-	case HAL_ENCRYPT_TYPE_WEP_128:
-	case HAL_ENCRYPT_TYPE_WAPI_GCM_SM4:
-	case HAL_ENCRYPT_TYPE_WAPI:
-		return;
-	}
-	key_id = le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info5,
-			       RX_MPDU_START_INFO5_KEY_ID);
-	crypto_hdr[3] = 0x20 | (key_id << 6);
-	crypto_hdr[4] =
-		HAL_RX_MPDU_INFO_PN_GET_BYTE3(desc->u.qcn9274_compact.mpdu_start.pn[0]);
-	crypto_hdr[5] =
-		HAL_RX_MPDU_INFO_PN_GET_BYTE4(desc->u.qcn9274_compact.mpdu_start.pn[0]);
-	crypto_hdr[6] =
-		HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274_compact.mpdu_start.pn[1]);
-	crypto_hdr[7] =
-		HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274_compact.mpdu_start.pn[1]);
-}
-
-static bool ath12k_hw_qcn9274_compact_dp_rx_h_msdu_done(struct hal_rx_desc *desc)
-{
-	return !!le32_get_bits(desc->u.qcn9274_compact.msdu_end.info14,
-			       RX_MSDU_END_INFO14_MSDU_DONE);
-}
-
-static bool ath12k_hw_qcn9274_compact_dp_rx_h_l4_cksum_fail(struct hal_rx_desc *desc)
-{
-	return !!le32_get_bits(desc->u.qcn9274_compact.msdu_end.info13,
-			       RX_MSDU_END_INFO13_TCP_UDP_CKSUM_FAIL);
-}
-
-static bool ath12k_hw_qcn9274_compact_dp_rx_h_ip_cksum_fail(struct hal_rx_desc *desc)
-{
-	return !!le32_get_bits(desc->u.qcn9274_compact.msdu_end.info13,
-			       RX_MSDU_END_INFO13_IP_CKSUM_FAIL);
-}
-
-static bool ath12k_hw_qcn9274_compact_dp_rx_h_is_decrypted(struct hal_rx_desc *desc)
-{
-	return (le32_get_bits(desc->u.qcn9274_compact.msdu_end.info14,
-			      RX_MSDU_END_INFO14_DECRYPT_STATUS_CODE) ==
-			RX_DESC_DECRYPT_STATUS_CODE_OK);
-}
-
-static u32 ath12k_hw_qcn9274_compact_dp_rx_h_mpdu_err(struct hal_rx_desc *desc)
-{
-	u32 info = __le32_to_cpu(desc->u.qcn9274_compact.msdu_end.info13);
-	u32 errmap = 0;
-
-	if (info & RX_MSDU_END_INFO13_FCS_ERR)
-		errmap |= HAL_RX_MPDU_ERR_FCS;
-
-	if (info & RX_MSDU_END_INFO13_DECRYPT_ERR)
-		errmap |= HAL_RX_MPDU_ERR_DECRYPT;
-
-	if (info & RX_MSDU_END_INFO13_TKIP_MIC_ERR)
-		errmap |= HAL_RX_MPDU_ERR_TKIP_MIC;
-
-	if (info & RX_MSDU_END_INFO13_A_MSDU_ERROR)
-		errmap |= HAL_RX_MPDU_ERR_AMSDU_ERR;
-
-	if (info & RX_MSDU_END_INFO13_OVERFLOW_ERR)
-		errmap |= HAL_RX_MPDU_ERR_OVERFLOW;
-
-	if (info & RX_MSDU_END_INFO13_MSDU_LEN_ERR)
-		errmap |= HAL_RX_MPDU_ERR_MSDU_LEN;
-
-	if (info & RX_MSDU_END_INFO13_MPDU_LEN_ERR)
-		errmap |= HAL_RX_MPDU_ERR_MPDU_LEN;
-
-	return errmap;
-}
-
-static u32 ath12k_hw_qcn9274_compact_get_rx_desc_size(void)
-{
-	return sizeof(struct hal_rx_desc_qcn9274_compact);
-}
-
-static u8 ath12k_hw_qcn9274_compact_rx_desc_get_msdu_src_link(struct hal_rx_desc *desc)
-{
-	return le64_get_bits(desc->u.qcn9274_compact.msdu_end.msdu_end_tag,
-			     RX_MSDU_END_64_TLV_SRC_LINK_ID);
-}
-
-const struct hal_rx_ops hal_rx_qcn9274_compact_ops = {
-	.rx_desc_get_first_msdu = ath12k_hw_qcn9274_compact_rx_desc_get_first_msdu,
-	.rx_desc_get_last_msdu = ath12k_hw_qcn9274_compact_rx_desc_get_last_msdu,
-	.rx_desc_get_l3_pad_bytes = ath12k_hw_qcn9274_compact_rx_desc_get_l3_pad_bytes,
-	.rx_desc_encrypt_valid = ath12k_hw_qcn9274_compact_rx_desc_encrypt_valid,
-	.rx_desc_get_encrypt_type = ath12k_hw_qcn9274_compact_rx_desc_get_encrypt_type,
-	.rx_desc_get_decap_type = ath12k_hw_qcn9274_compact_rx_desc_get_decap_type,
-	.rx_desc_get_mesh_ctl = ath12k_hw_qcn9274_compact_rx_desc_get_mesh_ctl,
-	.rx_desc_get_mpdu_seq_ctl_vld =
-		ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_seq_ctl_vld,
-	.rx_desc_get_mpdu_fc_valid = ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_fc_valid,
-	.rx_desc_get_mpdu_start_seq_no =
-		ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_start_seq_no,
-	.rx_desc_get_msdu_len = ath12k_hw_qcn9274_compact_rx_desc_get_msdu_len,
-	.rx_desc_get_msdu_sgi = ath12k_hw_qcn9274_compact_rx_desc_get_msdu_sgi,
-	.rx_desc_get_msdu_rate_mcs = ath12k_hw_qcn9274_compact_rx_desc_get_msdu_rate_mcs,
-	.rx_desc_get_msdu_rx_bw = ath12k_hw_qcn9274_compact_rx_desc_get_msdu_rx_bw,
-	.rx_desc_get_msdu_freq = ath12k_hw_qcn9274_compact_rx_desc_get_msdu_freq,
-	.rx_desc_get_msdu_pkt_type = ath12k_hw_qcn9274_compact_rx_desc_get_msdu_pkt_type,
-	.rx_desc_get_msdu_nss = ath12k_hw_qcn9274_compact_rx_desc_get_msdu_nss,
-	.rx_desc_get_mpdu_tid = ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_tid,
-	.rx_desc_get_mpdu_peer_id = ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_peer_id,
-	.rx_desc_copy_end_tlv = ath12k_hw_qcn9274_compact_rx_desc_copy_end_tlv,
-	.rx_desc_get_mpdu_ppdu_id = ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_ppdu_id,
-	.rx_desc_set_msdu_len = ath12k_hw_qcn9274_compact_rx_desc_set_msdu_len,
-	.rx_desc_get_msdu_payload = ath12k_hw_qcn9274_compact_rx_desc_get_msdu_payload,
-	.rx_desc_get_mpdu_start_offset =
-		ath12k_hw_qcn9274_compact_rx_desc_get_mpdu_start_offset,
-	.rx_desc_get_msdu_end_offset =
-		ath12k_hw_qcn9274_compact_rx_desc_get_msdu_end_offset,
-	.rx_desc_mac_addr2_valid = ath12k_hw_qcn9274_compact_rx_desc_mac_addr2_valid,
-	.rx_desc_mpdu_start_addr2 = ath12k_hw_qcn9274_compact_rx_desc_mpdu_start_addr2,
-	.rx_desc_is_da_mcbc = ath12k_hw_qcn9274_compact_rx_desc_is_da_mcbc,
-	.rx_desc_get_dot11_hdr = ath12k_hw_qcn9274_compact_rx_desc_get_dot11_hdr,
-	.rx_desc_get_crypto_header = ath12k_hw_qcn9274_compact_rx_desc_get_crypto_hdr,
-	.dp_rx_h_msdu_done = ath12k_hw_qcn9274_compact_dp_rx_h_msdu_done,
-	.dp_rx_h_l4_cksum_fail = ath12k_hw_qcn9274_compact_dp_rx_h_l4_cksum_fail,
-	.dp_rx_h_ip_cksum_fail = ath12k_hw_qcn9274_compact_dp_rx_h_ip_cksum_fail,
-	.dp_rx_h_is_decrypted = ath12k_hw_qcn9274_compact_dp_rx_h_is_decrypted,
-	.dp_rx_h_mpdu_err = ath12k_hw_qcn9274_compact_dp_rx_h_mpdu_err,
-	.rx_desc_get_desc_size = ath12k_hw_qcn9274_compact_get_rx_desc_size,
-	.rx_desc_get_msdu_src_link_id =
-		ath12k_hw_qcn9274_compact_rx_desc_get_msdu_src_link,
-};
-
-const struct hal_ops hal_qcn9274_ops = {
-	.create_srng_config = ath12k_hal_srng_create_config_qcn9274,
-	.tcl_to_wbm_rbm_map = ath12k_hal_qcn9274_tcl_to_wbm_rbm_map,
-	.rxdma_ring_wmask_rx_mpdu_start = ath12k_hal_qcn9274_rx_mpdu_start_wmask_get,
-	.rxdma_ring_wmask_rx_msdu_end = ath12k_hal_qcn9274_rx_msdu_end_wmask_get,
-	.get_hal_rx_compact_ops = ath12k_hal_qcn9274_get_hal_rx_compact_ops,
-};
-
-static bool ath12k_hw_wcn7850_rx_desc_get_first_msdu(struct hal_rx_desc *desc)
-{
-	return !!le16_get_bits(desc->u.wcn7850.msdu_end.info5,
-			       RX_MSDU_END_INFO5_FIRST_MSDU);
-}
-
-static bool ath12k_hw_wcn7850_rx_desc_get_last_msdu(struct hal_rx_desc *desc)
-{
-	return !!le16_get_bits(desc->u.wcn7850.msdu_end.info5,
-			       RX_MSDU_END_INFO5_LAST_MSDU);
-}
-
-static u8 ath12k_hw_wcn7850_rx_desc_get_l3_pad_bytes(struct hal_rx_desc *desc)
-{
-	return le16_get_bits(desc->u.wcn7850.msdu_end.info5,
-			    RX_MSDU_END_INFO5_L3_HDR_PADDING);
-}
-
-static bool ath12k_hw_wcn7850_rx_desc_encrypt_valid(struct hal_rx_desc *desc)
-{
-	return !!le32_get_bits(desc->u.wcn7850.mpdu_start.info4,
-			       RX_MPDU_START_INFO4_ENCRYPT_INFO_VALID);
-}
-
-static u32 ath12k_hw_wcn7850_rx_desc_get_encrypt_type(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.wcn7850.mpdu_start.info2,
-			     RX_MPDU_START_INFO2_ENC_TYPE);
-}
-
-static u8 ath12k_hw_wcn7850_rx_desc_get_decap_type(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.wcn7850.msdu_end.info11,
-			     RX_MSDU_END_INFO11_DECAP_FORMAT);
-}
-
-static u8 ath12k_hw_wcn7850_rx_desc_get_mesh_ctl(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.wcn7850.msdu_end.info11,
-			     RX_MSDU_END_INFO11_MESH_CTRL_PRESENT);
-}
-
-static bool ath12k_hw_wcn7850_rx_desc_get_mpdu_seq_ctl_vld(struct hal_rx_desc *desc)
-{
-	return !!le32_get_bits(desc->u.wcn7850.mpdu_start.info4,
-			       RX_MPDU_START_INFO4_MPDU_SEQ_CTRL_VALID);
-}
-
-static bool ath12k_hw_wcn7850_rx_desc_get_mpdu_fc_valid(struct hal_rx_desc *desc)
-{
-	return !!le32_get_bits(desc->u.wcn7850.mpdu_start.info4,
-			       RX_MPDU_START_INFO4_MPDU_FCTRL_VALID);
-}
-
-static u16 ath12k_hw_wcn7850_rx_desc_get_mpdu_start_seq_no(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.wcn7850.mpdu_start.info4,
-			     RX_MPDU_START_INFO4_MPDU_SEQ_NUM);
-}
-
-static u16 ath12k_hw_wcn7850_rx_desc_get_msdu_len(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.wcn7850.msdu_end.info10,
-			     RX_MSDU_END_INFO10_MSDU_LENGTH);
-}
-
-static u8 ath12k_hw_wcn7850_rx_desc_get_msdu_sgi(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.wcn7850.msdu_end.info12,
-			     RX_MSDU_END_INFO12_SGI);
-}
-
-static u8 ath12k_hw_wcn7850_rx_desc_get_msdu_rate_mcs(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.wcn7850.msdu_end.info12,
-			     RX_MSDU_END_INFO12_RATE_MCS);
-}
-
-static u8 ath12k_hw_wcn7850_rx_desc_get_msdu_rx_bw(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.wcn7850.msdu_end.info12,
-			     RX_MSDU_END_INFO12_RECV_BW);
-}
-
-static u32 ath12k_hw_wcn7850_rx_desc_get_msdu_freq(struct hal_rx_desc *desc)
-{
-	return __le32_to_cpu(desc->u.wcn7850.msdu_end.phy_meta_data);
-}
-
-static u8 ath12k_hw_wcn7850_rx_desc_get_msdu_pkt_type(struct hal_rx_desc *desc)
-{
-	return le32_get_bits(desc->u.wcn7850.msdu_end.info12,
-			     RX_MSDU_END_INFO12_PKT_TYPE);
-}
-
-static u8 ath12k_hw_wcn7850_rx_desc_get_msdu_nss(struct hal_rx_desc *desc)
+static void ath12k_hal_ce_dst_setup(struct ath12k_base *ab,
+				    struct hal_srng *srng, int ring_num)
 {
-	return le32_get_bits(desc->u.wcn7850.msdu_end.info12,
-			     RX_MSDU_END_INFO12_MIMO_SS_BITMAP);
+	ab->hal.ops->ce_dst_setup(ab, srng, ring_num);
 }
 
-static u8 ath12k_hw_wcn7850_rx_desc_get_mpdu_tid(struct hal_rx_desc *desc)
+static void ath12k_hal_srng_src_hw_init(struct ath12k_base *ab,
+					struct hal_srng *srng)
 {
-	return le32_get_bits(desc->u.wcn7850.mpdu_start.info2,
-			     RX_MPDU_START_INFO2_TID);
+	ab->hal.ops->srng_src_hw_init(ab, srng);
 }
 
-static u16 ath12k_hw_wcn7850_rx_desc_get_mpdu_peer_id(struct hal_rx_desc *desc)
+static void ath12k_hal_srng_dst_hw_init(struct ath12k_base *ab,
+					struct hal_srng *srng)
 {
-	return __le16_to_cpu(desc->u.wcn7850.mpdu_start.sw_peer_id);
+	ab->hal.ops->srng_dst_hw_init(ab, srng);
 }
 
-static void ath12k_hw_wcn7850_rx_desc_copy_end_tlv(struct hal_rx_desc *fdesc,
-						   struct hal_rx_desc *ldesc)
+static void ath12k_hal_set_umac_srng_ptr_addr(struct ath12k_base *ab,
+					      struct hal_srng *srng)
 {
-	memcpy(&fdesc->u.wcn7850.msdu_end, &ldesc->u.wcn7850.msdu_end,
-	       sizeof(struct rx_msdu_end_qcn9274));
+	ab->hal.ops->set_umac_srng_ptr_addr(ab, srng);
 }
 
-static u32 ath12k_hw_wcn7850_rx_desc_get_mpdu_start_tag(struct hal_rx_desc *desc)
+static int ath12k_hal_srng_get_ring_id(struct ath12k_hal *hal,
+				       enum hal_ring_type type,
+				       int ring_num, int mac_id)
 {
-	return le64_get_bits(desc->u.wcn7850.mpdu_start_tag,
-			    HAL_TLV_HDR_TAG);
+	return hal->ops->srng_get_ring_id(hal, type, ring_num, mac_id);
 }
 
-static u32 ath12k_hw_wcn7850_rx_desc_get_mpdu_ppdu_id(struct hal_rx_desc *desc)
+int ath12k_hal_srng_update_shadow_config(struct ath12k_base *ab,
+					 enum hal_ring_type ring_type,
+					 int ring_num)
 {
-	return __le16_to_cpu(desc->u.wcn7850.mpdu_start.phy_ppdu_id);
+	return ab->hal.ops->srng_update_shadow_config(ab, ring_type,
+							  ring_num);
 }
 
-static void ath12k_hw_wcn7850_rx_desc_set_msdu_len(struct hal_rx_desc *desc, u16 len)
+u32 ath12k_hal_ce_get_desc_size(struct ath12k_hal *hal, enum hal_ce_desc type)
 {
-	u32 info = __le32_to_cpu(desc->u.wcn7850.msdu_end.info10);
-
-	info &= ~RX_MSDU_END_INFO10_MSDU_LENGTH;
-	info |= u32_encode_bits(len, RX_MSDU_END_INFO10_MSDU_LENGTH);
-
-	desc->u.wcn7850.msdu_end.info10 = __cpu_to_le32(info);
+	return hal->ops->ce_get_desc_size(type);
 }
 
-static u8 *ath12k_hw_wcn7850_rx_desc_get_msdu_payload(struct hal_rx_desc *desc)
+void ath12k_hal_tx_set_dscp_tid_map(struct ath12k_base *ab, int id)
 {
-	return &desc->u.wcn7850.msdu_payload[0];
+	ab->hal.ops->tx_set_dscp_tid_map(ab, id);
 }
 
-static u32 ath12k_hw_wcn7850_rx_desc_get_mpdu_start_offset(void)
+void ath12k_hal_tx_configure_bank_register(struct ath12k_base *ab,
+					   u32 bank_config, u8 bank_id)
 {
-	return offsetof(struct hal_rx_desc_wcn7850, mpdu_start_tag);
+	ab->hal.ops->tx_configure_bank_register(ab, bank_config, bank_id);
 }
 
-static u32 ath12k_hw_wcn7850_rx_desc_get_msdu_end_offset(void)
+void ath12k_hal_reoq_lut_addr_read_enable(struct ath12k_base *ab)
 {
-	return offsetof(struct hal_rx_desc_wcn7850, msdu_end_tag);
+	ab->hal.ops->reoq_lut_addr_read_enable(ab);
 }
 
-static bool ath12k_hw_wcn7850_rx_desc_mac_addr2_valid(struct hal_rx_desc *desc)
+void ath12k_hal_reoq_lut_set_max_peerid(struct ath12k_base *ab)
 {
-	return __le32_to_cpu(desc->u.wcn7850.mpdu_start.info4) &
-	       RX_MPDU_START_INFO4_MAC_ADDR2_VALID;
+	ab->hal.ops->reoq_lut_set_max_peerid(ab);
 }
 
-static u8 *ath12k_hw_wcn7850_rx_desc_mpdu_start_addr2(struct hal_rx_desc *desc)
+void ath12k_hal_write_ml_reoq_lut_addr(struct ath12k_base *ab, dma_addr_t paddr)
 {
-	return desc->u.wcn7850.mpdu_start.addr2;
+	ab->hal.ops->write_ml_reoq_lut_addr(ab, paddr);
 }
 
-static bool ath12k_hw_wcn7850_rx_desc_is_da_mcbc(struct hal_rx_desc *desc)
+void ath12k_hal_write_reoq_lut_addr(struct ath12k_base *ab, dma_addr_t paddr)
 {
-	return __le32_to_cpu(desc->u.wcn7850.msdu_end.info13) &
-	       RX_MSDU_END_INFO13_MCAST_BCAST;
+	ab->hal.ops->write_reoq_lut_addr(ab, paddr);
 }
 
-static void ath12k_hw_wcn7850_rx_desc_get_dot11_hdr(struct hal_rx_desc *desc,
-						    struct ieee80211_hdr *hdr)
+void ath12k_hal_setup_link_idle_list(struct ath12k_base *ab,
+				     struct hal_wbm_idle_scatter_list *sbuf,
+				     u32 nsbufs, u32 tot_link_desc,
+				     u32 end_offset)
 {
-	hdr->frame_control = desc->u.wcn7850.mpdu_start.frame_ctrl;
-	hdr->duration_id = desc->u.wcn7850.mpdu_start.duration;
-	ether_addr_copy(hdr->addr1, desc->u.wcn7850.mpdu_start.addr1);
-	ether_addr_copy(hdr->addr2, desc->u.wcn7850.mpdu_start.addr2);
-	ether_addr_copy(hdr->addr3, desc->u.wcn7850.mpdu_start.addr3);
-	if (__le32_to_cpu(desc->u.wcn7850.mpdu_start.info4) &
-			RX_MPDU_START_INFO4_MAC_ADDR4_VALID) {
-		ether_addr_copy(hdr->addr4, desc->u.wcn7850.mpdu_start.addr4);
-	}
-	hdr->seq_ctrl = desc->u.wcn7850.mpdu_start.seq_ctrl;
+	ab->hal.ops->setup_link_idle_list(ab, sbuf, nsbufs, tot_link_desc,
+					      end_offset);
 }
 
-static void ath12k_hw_wcn7850_rx_desc_get_crypto_hdr(struct hal_rx_desc *desc,
-						     u8 *crypto_hdr,
-						     enum hal_encrypt_type enctype)
+void ath12k_hal_reo_hw_setup(struct ath12k_base *ab, u32 ring_hash_map)
 {
-	unsigned int key_id;
-
-	switch (enctype) {
-	case HAL_ENCRYPT_TYPE_OPEN:
-		return;
-	case HAL_ENCRYPT_TYPE_TKIP_NO_MIC:
-	case HAL_ENCRYPT_TYPE_TKIP_MIC:
-		crypto_hdr[0] =
-			HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.wcn7850.mpdu_start.pn[0]);
-		crypto_hdr[1] = 0;
-		crypto_hdr[2] =
-			HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.wcn7850.mpdu_start.pn[0]);
-		break;
-	case HAL_ENCRYPT_TYPE_CCMP_128:
-	case HAL_ENCRYPT_TYPE_CCMP_256:
-	case HAL_ENCRYPT_TYPE_GCMP_128:
-	case HAL_ENCRYPT_TYPE_AES_GCMP_256:
-		crypto_hdr[0] =
-			HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.wcn7850.mpdu_start.pn[0]);
-		crypto_hdr[1] =
-			HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.wcn7850.mpdu_start.pn[0]);
-		crypto_hdr[2] = 0;
-		break;
-	case HAL_ENCRYPT_TYPE_WEP_40:
-	case HAL_ENCRYPT_TYPE_WEP_104:
-	case HAL_ENCRYPT_TYPE_WEP_128:
-	case HAL_ENCRYPT_TYPE_WAPI_GCM_SM4:
-	case HAL_ENCRYPT_TYPE_WAPI:
-		return;
-	}
-	key_id = u32_get_bits(__le32_to_cpu(desc->u.wcn7850.mpdu_start.info5),
-			      RX_MPDU_START_INFO5_KEY_ID);
-	crypto_hdr[3] = 0x20 | (key_id << 6);
-	crypto_hdr[4] = HAL_RX_MPDU_INFO_PN_GET_BYTE3(desc->u.wcn7850.mpdu_start.pn[0]);
-	crypto_hdr[5] = HAL_RX_MPDU_INFO_PN_GET_BYTE4(desc->u.wcn7850.mpdu_start.pn[0]);
-	crypto_hdr[6] = HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.wcn7850.mpdu_start.pn[1]);
-	crypto_hdr[7] = HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.wcn7850.mpdu_start.pn[1]);
+	ab->hal.ops->reo_hw_setup(ab, ring_hash_map);
 }
 
-static int ath12k_hal_srng_create_config_wcn7850(struct ath12k_base *ab)
+void ath12k_hal_reo_init_cmd_ring(struct ath12k_base *ab, struct hal_srng *srng)
 {
-	struct ath12k_hal *hal = &ab->hal;
-	struct hal_srng_config *s;
-
-	hal->srng_config = kmemdup(hw_srng_config_template,
-				   sizeof(hw_srng_config_template),
-				   GFP_KERNEL);
-	if (!hal->srng_config)
-		return -ENOMEM;
-
-	s = &hal->srng_config[HAL_REO_DST];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_HP;
-	s->reg_size[0] = HAL_REO2_RING_BASE_LSB(ab) - HAL_REO1_RING_BASE_LSB(ab);
-	s->reg_size[1] = HAL_REO2_RING_HP - HAL_REO1_RING_HP;
-
-	s = &hal->srng_config[HAL_REO_EXCEPTION];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_SW0_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_SW0_RING_HP;
-
-	s = &hal->srng_config[HAL_REO_REINJECT];
-	s->max_rings = 1;
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_HP;
-
-	s = &hal->srng_config[HAL_REO_CMD];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_HP;
-
-	s = &hal->srng_config[HAL_REO_STATUS];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_HP;
-
-	s = &hal->srng_config[HAL_TCL_DATA];
-	s->max_rings = 5;
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_HP;
-	s->reg_size[0] = HAL_TCL2_RING_BASE_LSB(ab) - HAL_TCL1_RING_BASE_LSB(ab);
-	s->reg_size[1] = HAL_TCL2_RING_HP - HAL_TCL1_RING_HP;
-
-	s = &hal->srng_config[HAL_TCL_CMD];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_HP;
-
-	s = &hal->srng_config[HAL_TCL_STATUS];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_HP;
-
-	s = &hal->srng_config[HAL_CE_SRC];
-	s->max_rings = 12;
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) + HAL_CE_DST_RING_BASE_LSB;
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) + HAL_CE_DST_RING_HP;
-	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) -
-		HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab);
-	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) -
-		HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab);
-
-	s = &hal->srng_config[HAL_CE_DST];
-	s->max_rings = 12;
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_RING_BASE_LSB;
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_RING_HP;
-	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -
-		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);
-	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -
-		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);
-
-	s = &hal->srng_config[HAL_CE_DST_STATUS];
-	s->max_rings = 12;
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) +
-		HAL_CE_DST_STATUS_RING_BASE_LSB;
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_STATUS_RING_HP;
-	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -
-		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);
-	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -
-		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);
-
-	s = &hal->srng_config[HAL_WBM_IDLE_LINK];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_HP;
-
-	s = &hal->srng_config[HAL_SW2WBM_RELEASE];
-	s->max_rings = 1;
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG +
-		HAL_WBM_SW_RELEASE_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_SW_RELEASE_RING_HP;
-
-	s = &hal->srng_config[HAL_WBM2SW_RELEASE];
-	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_BASE_LSB(ab);
-	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_HP;
-	s->reg_size[0] = HAL_WBM1_RELEASE_RING_BASE_LSB(ab) -
-		HAL_WBM0_RELEASE_RING_BASE_LSB(ab);
-	s->reg_size[1] = HAL_WBM1_RELEASE_RING_HP - HAL_WBM0_RELEASE_RING_HP;
-
-	s = &hal->srng_config[HAL_RXDMA_BUF];
-	s->max_rings = 2;
-	s->mac_type = ATH12K_HAL_SRNG_PMAC;
-
-	s = &hal->srng_config[HAL_RXDMA_DST];
-	s->max_rings = 1;
-	s->entry_size = sizeof(struct hal_reo_entrance_ring) >> 2;
-
-	/* below rings are not used */
-	s = &hal->srng_config[HAL_RXDMA_DIR_BUF];
-	s->max_rings = 0;
-
-	s = &hal->srng_config[HAL_PPE2TCL];
-	s->max_rings = 0;
-
-	s = &hal->srng_config[HAL_PPE_RELEASE];
-	s->max_rings = 0;
-
-	s = &hal->srng_config[HAL_TX_MONITOR_BUF];
-	s->max_rings = 0;
-
-	s = &hal->srng_config[HAL_TX_MONITOR_DST];
-	s->max_rings = 0;
-
-	s = &hal->srng_config[HAL_PPE2TCL];
-	s->max_rings = 0;
-
-	return 0;
+	ab->hal.ops->reo_init_cmd_ring(ab, srng);
 }
 
-static bool ath12k_hw_wcn7850_dp_rx_h_msdu_done(struct hal_rx_desc *desc)
+void ath12k_hal_reo_shared_qaddr_cache_clear(struct ath12k_base *ab)
 {
-	return !!le32_get_bits(desc->u.wcn7850.msdu_end.info14,
-			       RX_MSDU_END_INFO14_MSDU_DONE);
+	ab->hal.ops->reo_shared_qaddr_cache_clear(ab);
 }
+EXPORT_SYMBOL(ath12k_hal_reo_shared_qaddr_cache_clear);
 
-static bool ath12k_hw_wcn7850_dp_rx_h_l4_cksum_fail(struct hal_rx_desc *desc)
+void ath12k_hal_rx_buf_addr_info_set(struct ath12k_hal *hal,
+				     struct ath12k_buffer_addr *binfo,
+				     dma_addr_t paddr, u32 cookie, u8 manager)
 {
-	return !!le32_get_bits(desc->u.wcn7850.msdu_end.info13,
-			       RX_MSDU_END_INFO13_TCP_UDP_CKSUM_FAIL);
+	hal->ops->rx_buf_addr_info_set(binfo, paddr, cookie, manager);
 }
 
-static bool ath12k_hw_wcn7850_dp_rx_h_ip_cksum_fail(struct hal_rx_desc *desc)
+void ath12k_hal_rx_buf_addr_info_get(struct ath12k_hal *hal,
+				     struct ath12k_buffer_addr *binfo,
+				     dma_addr_t *paddr, u32 *msdu_cookies,
+				     u8 *rbm)
 {
-	return !!le32_get_bits(desc->u.wcn7850.msdu_end.info13,
-			      RX_MSDU_END_INFO13_IP_CKSUM_FAIL);
+	hal->ops->rx_buf_addr_info_get(binfo, paddr, msdu_cookies, rbm);
 }
 
-static bool ath12k_hw_wcn7850_dp_rx_h_is_decrypted(struct hal_rx_desc *desc)
+void ath12k_hal_rx_msdu_list_get(struct ath12k_hal *hal, struct ath12k *ar,
+				 void *link_desc,
+				 void *msdu_list,
+				 u16 *num_msdus)
 {
-	return (le32_get_bits(desc->u.wcn7850.msdu_end.info14,
-			      RX_MSDU_END_INFO14_DECRYPT_STATUS_CODE) ==
-			      RX_DESC_DECRYPT_STATUS_CODE_OK);
+	hal->ops->rx_msdu_list_get(ar, link_desc, msdu_list, num_msdus);
 }
 
-static u32 ath12k_hw_wcn7850_dp_rx_h_mpdu_err(struct hal_rx_desc *desc)
+void ath12k_hal_rx_reo_ent_buf_paddr_get(struct ath12k_hal *hal, void *rx_desc,
+					 dma_addr_t *paddr,
+					 u32 *sw_cookie,
+					 struct ath12k_buffer_addr **pp_buf_addr,
+					 u8 *rbm, u32 *msdu_cnt)
 {
-	u32 info = __le32_to_cpu(desc->u.wcn7850.msdu_end.info13);
-	u32 errmap = 0;
-
-	if (info & RX_MSDU_END_INFO13_FCS_ERR)
-		errmap |= HAL_RX_MPDU_ERR_FCS;
-
-	if (info & RX_MSDU_END_INFO13_DECRYPT_ERR)
-		errmap |= HAL_RX_MPDU_ERR_DECRYPT;
-
-	if (info & RX_MSDU_END_INFO13_TKIP_MIC_ERR)
-		errmap |= HAL_RX_MPDU_ERR_TKIP_MIC;
-
-	if (info & RX_MSDU_END_INFO13_A_MSDU_ERROR)
-		errmap |= HAL_RX_MPDU_ERR_AMSDU_ERR;
-
-	if (info & RX_MSDU_END_INFO13_OVERFLOW_ERR)
-		errmap |= HAL_RX_MPDU_ERR_OVERFLOW;
-
-	if (info & RX_MSDU_END_INFO13_MSDU_LEN_ERR)
-		errmap |= HAL_RX_MPDU_ERR_MSDU_LEN;
-
-	if (info & RX_MSDU_END_INFO13_MPDU_LEN_ERR)
-		errmap |= HAL_RX_MPDU_ERR_MPDU_LEN;
-
-	return errmap;
+	hal->ops->rx_reo_ent_buf_paddr_get(rx_desc, paddr, sw_cookie,
+					   pp_buf_addr, rbm, msdu_cnt);
 }
 
-static u32 ath12k_hw_wcn7850_get_rx_desc_size(void)
+void ath12k_hal_cc_config(struct ath12k_base *ab)
 {
-	return sizeof(struct hal_rx_desc_wcn7850);
+	ab->hal.ops->cc_config(ab);
 }
 
-static u8 ath12k_hw_wcn7850_rx_desc_get_msdu_src_link(struct hal_rx_desc *desc)
+enum hal_rx_buf_return_buf_manager
+ath12k_hal_get_idle_link_rbm(struct ath12k_hal *hal, u8 device_id)
 {
-	return 0;
+	return hal->ops->get_idle_link_rbm(hal, device_id);
 }
 
-const struct hal_rx_ops hal_rx_wcn7850_ops = {
-	.rx_desc_get_first_msdu = ath12k_hw_wcn7850_rx_desc_get_first_msdu,
-	.rx_desc_get_last_msdu = ath12k_hw_wcn7850_rx_desc_get_last_msdu,
-	.rx_desc_get_l3_pad_bytes = ath12k_hw_wcn7850_rx_desc_get_l3_pad_bytes,
-	.rx_desc_encrypt_valid = ath12k_hw_wcn7850_rx_desc_encrypt_valid,
-	.rx_desc_get_encrypt_type = ath12k_hw_wcn7850_rx_desc_get_encrypt_type,
-	.rx_desc_get_decap_type = ath12k_hw_wcn7850_rx_desc_get_decap_type,
-	.rx_desc_get_mesh_ctl = ath12k_hw_wcn7850_rx_desc_get_mesh_ctl,
-	.rx_desc_get_mpdu_seq_ctl_vld = ath12k_hw_wcn7850_rx_desc_get_mpdu_seq_ctl_vld,
-	.rx_desc_get_mpdu_fc_valid = ath12k_hw_wcn7850_rx_desc_get_mpdu_fc_valid,
-	.rx_desc_get_mpdu_start_seq_no = ath12k_hw_wcn7850_rx_desc_get_mpdu_start_seq_no,
-	.rx_desc_get_msdu_len = ath12k_hw_wcn7850_rx_desc_get_msdu_len,
-	.rx_desc_get_msdu_sgi = ath12k_hw_wcn7850_rx_desc_get_msdu_sgi,
-	.rx_desc_get_msdu_rate_mcs = ath12k_hw_wcn7850_rx_desc_get_msdu_rate_mcs,
-	.rx_desc_get_msdu_rx_bw = ath12k_hw_wcn7850_rx_desc_get_msdu_rx_bw,
-	.rx_desc_get_msdu_freq = ath12k_hw_wcn7850_rx_desc_get_msdu_freq,
-	.rx_desc_get_msdu_pkt_type = ath12k_hw_wcn7850_rx_desc_get_msdu_pkt_type,
-	.rx_desc_get_msdu_nss = ath12k_hw_wcn7850_rx_desc_get_msdu_nss,
-	.rx_desc_get_mpdu_tid = ath12k_hw_wcn7850_rx_desc_get_mpdu_tid,
-	.rx_desc_get_mpdu_peer_id = ath12k_hw_wcn7850_rx_desc_get_mpdu_peer_id,
-	.rx_desc_copy_end_tlv = ath12k_hw_wcn7850_rx_desc_copy_end_tlv,
-	.rx_desc_get_mpdu_start_tag = ath12k_hw_wcn7850_rx_desc_get_mpdu_start_tag,
-	.rx_desc_get_mpdu_ppdu_id = ath12k_hw_wcn7850_rx_desc_get_mpdu_ppdu_id,
-	.rx_desc_set_msdu_len = ath12k_hw_wcn7850_rx_desc_set_msdu_len,
-	.rx_desc_get_msdu_payload = ath12k_hw_wcn7850_rx_desc_get_msdu_payload,
-	.rx_desc_get_mpdu_start_offset = ath12k_hw_wcn7850_rx_desc_get_mpdu_start_offset,
-	.rx_desc_get_msdu_end_offset = ath12k_hw_wcn7850_rx_desc_get_msdu_end_offset,
-	.rx_desc_mac_addr2_valid = ath12k_hw_wcn7850_rx_desc_mac_addr2_valid,
-	.rx_desc_mpdu_start_addr2 = ath12k_hw_wcn7850_rx_desc_mpdu_start_addr2,
-	.rx_desc_is_da_mcbc = ath12k_hw_wcn7850_rx_desc_is_da_mcbc,
-	.rx_desc_get_dot11_hdr = ath12k_hw_wcn7850_rx_desc_get_dot11_hdr,
-	.rx_desc_get_crypto_header = ath12k_hw_wcn7850_rx_desc_get_crypto_hdr,
-	.dp_rx_h_msdu_done = ath12k_hw_wcn7850_dp_rx_h_msdu_done,
-	.dp_rx_h_l4_cksum_fail = ath12k_hw_wcn7850_dp_rx_h_l4_cksum_fail,
-	.dp_rx_h_ip_cksum_fail = ath12k_hw_wcn7850_dp_rx_h_ip_cksum_fail,
-	.dp_rx_h_is_decrypted = ath12k_hw_wcn7850_dp_rx_h_is_decrypted,
-	.dp_rx_h_mpdu_err = ath12k_hw_wcn7850_dp_rx_h_mpdu_err,
-	.rx_desc_get_desc_size = ath12k_hw_wcn7850_get_rx_desc_size,
-	.rx_desc_get_msdu_src_link_id = ath12k_hw_wcn7850_rx_desc_get_msdu_src_link,
-};
-
-const struct hal_ops hal_wcn7850_ops = {
-	.create_srng_config = ath12k_hal_srng_create_config_wcn7850,
-	.tcl_to_wbm_rbm_map = ath12k_hal_wcn7850_tcl_to_wbm_rbm_map,
-	.rxdma_ring_wmask_rx_mpdu_start = NULL,
-	.rxdma_ring_wmask_rx_msdu_end = NULL,
-	.get_hal_rx_compact_ops = NULL,
-};
-
-static int ath12k_hal_alloc_cont_rdp(struct ath12k_base *ab)
+static int ath12k_hal_alloc_cont_rdp(struct ath12k_hal *hal)
 {
-	struct ath12k_hal *hal = &ab->hal;
 	size_t size;
 
 	size = sizeof(u32) * HAL_SRNG_RING_ID_MAX;
-	hal->rdp.vaddr = dma_alloc_coherent(ab->dev, size, &hal->rdp.paddr,
+	hal->rdp.vaddr = dma_alloc_coherent(hal->dev, size, &hal->rdp.paddr,
 					    GFP_KERNEL);
 	if (!hal->rdp.vaddr)
 		return -ENOMEM;
@@ -1575,27 +164,25 @@ static int ath12k_hal_alloc_cont_rdp(struct ath12k_base *ab)
 	return 0;
 }
 
-static void ath12k_hal_free_cont_rdp(struct ath12k_base *ab)
+static void ath12k_hal_free_cont_rdp(struct ath12k_hal *hal)
 {
-	struct ath12k_hal *hal = &ab->hal;
 	size_t size;
 
 	if (!hal->rdp.vaddr)
 		return;
 
 	size = sizeof(u32) * HAL_SRNG_RING_ID_MAX;
-	dma_free_coherent(ab->dev, size,
+	dma_free_coherent(hal->dev, size,
 			  hal->rdp.vaddr, hal->rdp.paddr);
 	hal->rdp.vaddr = NULL;
 }
 
-static int ath12k_hal_alloc_cont_wrp(struct ath12k_base *ab)
+static int ath12k_hal_alloc_cont_wrp(struct ath12k_hal *hal)
 {
-	struct ath12k_hal *hal = &ab->hal;
 	size_t size;
 
 	size = sizeof(u32) * (HAL_SRNG_NUM_PMAC_RINGS + HAL_SRNG_NUM_DMAC_RINGS);
-	hal->wrp.vaddr = dma_alloc_coherent(ab->dev, size, &hal->wrp.paddr,
+	hal->wrp.vaddr = dma_alloc_coherent(hal->dev, size, &hal->wrp.paddr,
 					    GFP_KERNEL);
 	if (!hal->wrp.vaddr)
 		return -ENOMEM;
@@ -1603,209 +190,19 @@ static int ath12k_hal_alloc_cont_wrp(struct ath12k_base *ab)
 	return 0;
 }
 
-static void ath12k_hal_free_cont_wrp(struct ath12k_base *ab)
+static void ath12k_hal_free_cont_wrp(struct ath12k_hal *hal)
 {
-	struct ath12k_hal *hal = &ab->hal;
 	size_t size;
 
 	if (!hal->wrp.vaddr)
 		return;
 
 	size = sizeof(u32) * (HAL_SRNG_NUM_PMAC_RINGS + HAL_SRNG_NUM_DMAC_RINGS);
-	dma_free_coherent(ab->dev, size,
+	dma_free_coherent(hal->dev, size,
 			  hal->wrp.vaddr, hal->wrp.paddr);
 	hal->wrp.vaddr = NULL;
 }
 
-static void ath12k_hal_ce_dst_setup(struct ath12k_base *ab,
-				    struct hal_srng *srng, int ring_num)
-{
-	struct hal_srng_config *srng_config = &ab->hal.srng_config[HAL_CE_DST];
-	u32 addr;
-	u32 val;
-
-	addr = HAL_CE_DST_RING_CTRL +
-	       srng_config->reg_start[HAL_SRNG_REG_GRP_R0] +
-	       ring_num * srng_config->reg_size[HAL_SRNG_REG_GRP_R0];
-
-	val = ath12k_hif_read32(ab, addr);
-	val &= ~HAL_CE_DST_R0_DEST_CTRL_MAX_LEN;
-	val |= u32_encode_bits(srng->u.dst_ring.max_buffer_length,
-			       HAL_CE_DST_R0_DEST_CTRL_MAX_LEN);
-	ath12k_hif_write32(ab, addr, val);
-}
-
-static void ath12k_hal_srng_dst_hw_init(struct ath12k_base *ab,
-					struct hal_srng *srng)
-{
-	struct ath12k_hal *hal = &ab->hal;
-	u32 val;
-	u64 hp_addr;
-	u32 reg_base;
-
-	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];
-
-	if (srng->flags & HAL_SRNG_FLAGS_MSI_INTR) {
-		ath12k_hif_write32(ab, reg_base +
-				   ath12k_hal_reo1_ring_msi1_base_lsb_offset(ab),
-				   srng->msi_addr);
-
-		val = u32_encode_bits(((u64)srng->msi_addr >> HAL_ADDR_MSB_REG_SHIFT),
-				      HAL_REO1_RING_MSI1_BASE_MSB_ADDR) |
-				      HAL_REO1_RING_MSI1_BASE_MSB_MSI1_ENABLE;
-		ath12k_hif_write32(ab, reg_base +
-				   ath12k_hal_reo1_ring_msi1_base_msb_offset(ab), val);
-
-		ath12k_hif_write32(ab,
-				   reg_base + ath12k_hal_reo1_ring_msi1_data_offset(ab),
-				   srng->msi_data);
-	}
-
-	ath12k_hif_write32(ab, reg_base, srng->ring_base_paddr);
-
-	val = u32_encode_bits(((u64)srng->ring_base_paddr >> HAL_ADDR_MSB_REG_SHIFT),
-			      HAL_REO1_RING_BASE_MSB_RING_BASE_ADDR_MSB) |
-	      u32_encode_bits((srng->entry_size * srng->num_entries),
-			      HAL_REO1_RING_BASE_MSB_RING_SIZE);
-	ath12k_hif_write32(ab, reg_base + ath12k_hal_reo1_ring_base_msb_offset(ab), val);
-
-	val = u32_encode_bits(srng->ring_id, HAL_REO1_RING_ID_RING_ID) |
-	      u32_encode_bits(srng->entry_size, HAL_REO1_RING_ID_ENTRY_SIZE);
-	ath12k_hif_write32(ab, reg_base + ath12k_hal_reo1_ring_id_offset(ab), val);
-
-	/* interrupt setup */
-	val = u32_encode_bits((srng->intr_timer_thres_us >> 3),
-			      HAL_REO1_RING_PRDR_INT_SETUP_INTR_TMR_THOLD);
-
-	val |= u32_encode_bits((srng->intr_batch_cntr_thres_entries * srng->entry_size),
-				HAL_REO1_RING_PRDR_INT_SETUP_BATCH_COUNTER_THOLD);
-
-	ath12k_hif_write32(ab,
-			   reg_base + ath12k_hal_reo1_ring_producer_int_setup_offset(ab),
-			   val);
-
-	hp_addr = hal->rdp.paddr +
-		  ((unsigned long)srng->u.dst_ring.hp_addr -
-		   (unsigned long)hal->rdp.vaddr);
-	ath12k_hif_write32(ab, reg_base + ath12k_hal_reo1_ring_hp_addr_lsb_offset(ab),
-			   hp_addr & HAL_ADDR_LSB_REG_MASK);
-	ath12k_hif_write32(ab, reg_base + ath12k_hal_reo1_ring_hp_addr_msb_offset(ab),
-			   hp_addr >> HAL_ADDR_MSB_REG_SHIFT);
-
-	/* Initialize head and tail pointers to indicate ring is empty */
-	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2];
-	ath12k_hif_write32(ab, reg_base, 0);
-	ath12k_hif_write32(ab, reg_base + HAL_REO1_RING_TP_OFFSET, 0);
-	*srng->u.dst_ring.hp_addr = 0;
-
-	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];
-	val = 0;
-	if (srng->flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP)
-		val |= HAL_REO1_RING_MISC_DATA_TLV_SWAP;
-	if (srng->flags & HAL_SRNG_FLAGS_RING_PTR_SWAP)
-		val |= HAL_REO1_RING_MISC_HOST_FW_SWAP;
-	if (srng->flags & HAL_SRNG_FLAGS_MSI_SWAP)
-		val |= HAL_REO1_RING_MISC_MSI_SWAP;
-	val |= HAL_REO1_RING_MISC_SRNG_ENABLE;
-
-	ath12k_hif_write32(ab, reg_base + ath12k_hal_reo1_ring_misc_offset(ab), val);
-}
-
-static void ath12k_hal_srng_src_hw_init(struct ath12k_base *ab,
-					struct hal_srng *srng)
-{
-	struct ath12k_hal *hal = &ab->hal;
-	u32 val;
-	u64 tp_addr;
-	u32 reg_base;
-
-	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];
-
-	if (srng->flags & HAL_SRNG_FLAGS_MSI_INTR) {
-		ath12k_hif_write32(ab, reg_base +
-				   HAL_TCL1_RING_MSI1_BASE_LSB_OFFSET(ab),
-				   srng->msi_addr);
-
-		val = u32_encode_bits(((u64)srng->msi_addr >> HAL_ADDR_MSB_REG_SHIFT),
-				      HAL_TCL1_RING_MSI1_BASE_MSB_ADDR) |
-				      HAL_TCL1_RING_MSI1_BASE_MSB_MSI1_ENABLE;
-		ath12k_hif_write32(ab, reg_base +
-				       HAL_TCL1_RING_MSI1_BASE_MSB_OFFSET(ab),
-				   val);
-
-		ath12k_hif_write32(ab, reg_base +
-				       HAL_TCL1_RING_MSI1_DATA_OFFSET(ab),
-				   srng->msi_data);
-	}
-
-	ath12k_hif_write32(ab, reg_base, srng->ring_base_paddr);
-
-	val = u32_encode_bits(((u64)srng->ring_base_paddr >> HAL_ADDR_MSB_REG_SHIFT),
-			      HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB) |
-	      u32_encode_bits((srng->entry_size * srng->num_entries),
-			      HAL_TCL1_RING_BASE_MSB_RING_SIZE);
-	ath12k_hif_write32(ab, reg_base + HAL_TCL1_RING_BASE_MSB_OFFSET(ab), val);
-
-	val = u32_encode_bits(srng->entry_size, HAL_REO1_RING_ID_ENTRY_SIZE);
-	ath12k_hif_write32(ab, reg_base + HAL_TCL1_RING_ID_OFFSET(ab), val);
-
-	val = u32_encode_bits(srng->intr_timer_thres_us,
-			      HAL_TCL1_RING_CONSR_INT_SETUP_IX0_INTR_TMR_THOLD);
-
-	val |= u32_encode_bits((srng->intr_batch_cntr_thres_entries * srng->entry_size),
-			       HAL_TCL1_RING_CONSR_INT_SETUP_IX0_BATCH_COUNTER_THOLD);
-
-	ath12k_hif_write32(ab,
-			   reg_base + HAL_TCL1_RING_CONSR_INT_SETUP_IX0_OFFSET(ab),
-			   val);
-
-	val = 0;
-	if (srng->flags & HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN) {
-		val |= u32_encode_bits(srng->u.src_ring.low_threshold,
-				       HAL_TCL1_RING_CONSR_INT_SETUP_IX1_LOW_THOLD);
-	}
-	ath12k_hif_write32(ab,
-			   reg_base + HAL_TCL1_RING_CONSR_INT_SETUP_IX1_OFFSET(ab),
-			   val);
-
-	if (srng->ring_id != HAL_SRNG_RING_ID_WBM_IDLE_LINK) {
-		tp_addr = hal->rdp.paddr +
-			  ((unsigned long)srng->u.src_ring.tp_addr -
-			   (unsigned long)hal->rdp.vaddr);
-		ath12k_hif_write32(ab,
-				   reg_base + HAL_TCL1_RING_TP_ADDR_LSB_OFFSET(ab),
-				   tp_addr & HAL_ADDR_LSB_REG_MASK);
-		ath12k_hif_write32(ab,
-				   reg_base + HAL_TCL1_RING_TP_ADDR_MSB_OFFSET(ab),
-				   tp_addr >> HAL_ADDR_MSB_REG_SHIFT);
-	}
-
-	/* Initialize head and tail pointers to indicate ring is empty */
-	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2];
-	ath12k_hif_write32(ab, reg_base, 0);
-	ath12k_hif_write32(ab, reg_base + HAL_TCL1_RING_TP_OFFSET, 0);
-	*srng->u.src_ring.tp_addr = 0;
-
-	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];
-	val = 0;
-	if (srng->flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP)
-		val |= HAL_TCL1_RING_MISC_DATA_TLV_SWAP;
-	if (srng->flags & HAL_SRNG_FLAGS_RING_PTR_SWAP)
-		val |= HAL_TCL1_RING_MISC_HOST_FW_SWAP;
-	if (srng->flags & HAL_SRNG_FLAGS_MSI_SWAP)
-		val |= HAL_TCL1_RING_MISC_MSI_SWAP;
-
-	/* Loop count is not used for SRC rings */
-	val |= HAL_TCL1_RING_MISC_MSI_LOOPCNT_DISABLE;
-
-	val |= HAL_TCL1_RING_MISC_SRNG_ENABLE;
-
-	if (srng->ring_id == HAL_SRNG_RING_ID_WBM_IDLE_LINK)
-		val |= HAL_TCL1_RING_MISC_MSI_RING_ID_DISABLE;
-
-	ath12k_hif_write32(ab, reg_base + HAL_TCL1_RING_MISC_OFFSET(ab), val);
-}
-
 static void ath12k_hal_srng_hw_init(struct ath12k_base *ab,
 				    struct hal_srng *srng)
 {
@@ -1815,28 +212,6 @@ static void ath12k_hal_srng_hw_init(struct ath12k_base *ab,
 		ath12k_hal_srng_dst_hw_init(ab, srng);
 }
 
-static int ath12k_hal_srng_get_ring_id(struct ath12k_base *ab,
-				       enum hal_ring_type type,
-				       int ring_num, int mac_id)
-{
-	struct hal_srng_config *srng_config = &ab->hal.srng_config[type];
-	int ring_id;
-
-	if (ring_num >= srng_config->max_rings) {
-		ath12k_warn(ab, "invalid ring number :%d\n", ring_num);
-		return -EINVAL;
-	}
-
-	ring_id = srng_config->start_ring_id + ring_num;
-	if (srng_config->mac_type == ATH12K_HAL_SRNG_PMAC)
-		ring_id += mac_id * HAL_SRNG_RINGS_PER_PMAC;
-
-	if (WARN_ON(ring_id >= HAL_SRNG_RING_ID_MAX))
-		return -EINVAL;
-
-	return ring_id;
-}
-
 int ath12k_hal_srng_get_entrysize(struct ath12k_base *ab, u32 ring_type)
 {
 	struct hal_srng_config *srng_config;
@@ -1848,6 +223,7 @@ int ath12k_hal_srng_get_entrysize(struct ath12k_base *ab, u32 ring_type)
 
 	return (srng_config->entry_size << 2);
 }
+EXPORT_SYMBOL(ath12k_hal_srng_get_entrysize);
 
 int ath12k_hal_srng_get_max_entries(struct ath12k_base *ab, u32 ring_type)
 {
@@ -1877,6 +253,7 @@ void ath12k_hal_srng_get_params(struct ath12k_base *ab, struct hal_srng *srng,
 	params->msi2_data = srng->msi2_data;
 	params->flags = srng->flags;
 }
+EXPORT_SYMBOL(ath12k_hal_srng_get_params);
 
 dma_addr_t ath12k_hal_srng_get_hp_addr(struct ath12k_base *ab,
 				       struct hal_srng *srng)
@@ -1910,63 +287,32 @@ dma_addr_t ath12k_hal_srng_get_tp_addr(struct ath12k_base *ab,
 			(unsigned long)ab->hal.wrp.vaddr);
 }
 
-u32 ath12k_hal_ce_get_desc_size(enum hal_ce_desc type)
-{
-	switch (type) {
-	case HAL_CE_DESC_SRC:
-		return sizeof(struct hal_ce_srng_src_desc);
-	case HAL_CE_DESC_DST:
-		return sizeof(struct hal_ce_srng_dest_desc);
-	case HAL_CE_DESC_DST_STATUS:
-		return sizeof(struct hal_ce_srng_dst_status_desc);
-	}
-
-	return 0;
-}
-
-void ath12k_hal_ce_src_set_desc(struct hal_ce_srng_src_desc *desc, dma_addr_t paddr,
-				u32 len, u32 id, u8 byte_swap_data)
+void ath12k_hal_ce_src_set_desc(struct ath12k_hal *hal,
+				struct hal_ce_srng_src_desc *desc,
+				dma_addr_t paddr, u32 len, u32 id,
+				u8 byte_swap_data)
 {
-	desc->buffer_addr_low = cpu_to_le32(paddr & HAL_ADDR_LSB_REG_MASK);
-	desc->buffer_addr_info =
-		le32_encode_bits(((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT),
-				 HAL_CE_SRC_DESC_ADDR_INFO_ADDR_HI) |
-		le32_encode_bits(byte_swap_data,
-				 HAL_CE_SRC_DESC_ADDR_INFO_BYTE_SWAP) |
-		le32_encode_bits(0, HAL_CE_SRC_DESC_ADDR_INFO_GATHER) |
-		le32_encode_bits(len, HAL_CE_SRC_DESC_ADDR_INFO_LEN);
-	desc->meta_info = le32_encode_bits(id, HAL_CE_SRC_DESC_META_INFO_DATA);
+	hal->ops->ce_src_set_desc(desc, paddr, len, id, byte_swap_data);
 }
 
-void ath12k_hal_ce_dst_set_desc(struct hal_ce_srng_dest_desc *desc, dma_addr_t paddr)
+void ath12k_hal_ce_dst_set_desc(struct ath12k_hal *hal,
+				struct hal_ce_srng_dest_desc *desc,
+				dma_addr_t paddr)
 {
-	desc->buffer_addr_low = cpu_to_le32(paddr & HAL_ADDR_LSB_REG_MASK);
-	desc->buffer_addr_info =
-		le32_encode_bits(((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT),
-				 HAL_CE_DEST_DESC_ADDR_INFO_ADDR_HI);
+	hal->ops->ce_dst_set_desc(desc, paddr);
 }
 
-u32 ath12k_hal_ce_dst_status_get_length(struct hal_ce_srng_dst_status_desc *desc)
+u32 ath12k_hal_ce_dst_status_get_length(struct ath12k_hal *hal,
+					struct hal_ce_srng_dst_status_desc *desc)
 {
-	u32 len;
-
-	len = le32_get_bits(desc->flags, HAL_CE_DST_STATUS_DESC_FLAGS_LEN);
-	desc->flags &= ~cpu_to_le32(HAL_CE_DST_STATUS_DESC_FLAGS_LEN);
-
-	return len;
+	return hal->ops->ce_dst_status_get_length(desc);
 }
 
-void ath12k_hal_set_link_desc_addr(struct hal_wbm_link_desc *desc, u32 cookie,
-				   dma_addr_t paddr,
-				   enum hal_rx_buf_return_buf_manager rbm)
+void ath12k_hal_set_link_desc_addr(struct ath12k_hal *hal,
+				   struct hal_wbm_link_desc *desc, u32 cookie,
+				   dma_addr_t paddr, int rbm)
 {
-	desc->buf_addr_info.info0 = le32_encode_bits((paddr & HAL_ADDR_LSB_REG_MASK),
-						     BUFFER_ADDR_INFO0_ADDR);
-	desc->buf_addr_info.info1 =
-			le32_encode_bits(((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT),
-					 BUFFER_ADDR_INFO1_ADDR) |
-			le32_encode_bits(rbm, BUFFER_ADDR_INFO1_RET_BUF_MGR) |
-			le32_encode_bits(cookie, BUFFER_ADDR_INFO1_SW_COOKIE);
+	hal->ops->set_link_desc_addr(desc, cookie, paddr, rbm);
 }
 
 void *ath12k_hal_srng_dst_peek(struct ath12k_base *ab, struct hal_srng *srng)
@@ -1978,6 +324,7 @@ void *ath12k_hal_srng_dst_peek(struct ath12k_base *ab, struct hal_srng *srng)
 
 	return NULL;
 }
+EXPORT_SYMBOL(ath12k_hal_srng_dst_peek);
 
 void *ath12k_hal_srng_dst_get_next_entry(struct ath12k_base *ab,
 					 struct hal_srng *srng)
@@ -1996,6 +343,7 @@ void *ath12k_hal_srng_dst_get_next_entry(struct ath12k_base *ab,
 
 	return desc;
 }
+EXPORT_SYMBOL(ath12k_hal_srng_dst_get_next_entry);
 
 int ath12k_hal_srng_dst_num_free(struct ath12k_base *ab, struct hal_srng *srng,
 				 bool sync_hw_ptr)
@@ -2018,6 +366,7 @@ int ath12k_hal_srng_dst_num_free(struct ath12k_base *ab, struct hal_srng *srng,
 	else
 		return (srng->ring_size - tp + hp) / srng->entry_size;
 }
+EXPORT_SYMBOL(ath12k_hal_srng_dst_num_free);
 
 /* Returns number of available entries in src ring */
 int ath12k_hal_srng_src_num_free(struct ath12k_base *ab, struct hal_srng *srng,
@@ -2059,6 +408,7 @@ void *ath12k_hal_srng_src_next_peek(struct ath12k_base *ab,
 
 	return desc;
 }
+EXPORT_SYMBOL(ath12k_hal_srng_src_next_peek);
 
 void *ath12k_hal_srng_src_get_next_entry(struct ath12k_base *ab,
 					 struct hal_srng *srng)
@@ -2092,6 +442,7 @@ void *ath12k_hal_srng_src_get_next_entry(struct ath12k_base *ab,
 
 	return desc;
 }
+EXPORT_SYMBOL(ath12k_hal_srng_src_get_next_entry);
 
 void *ath12k_hal_srng_src_peek(struct ath12k_base *ab, struct hal_srng *srng)
 {
@@ -2103,6 +454,7 @@ void *ath12k_hal_srng_src_peek(struct ath12k_base *ab, struct hal_srng *srng)
 
 	return srng->ring_base_vaddr + srng->u.src_ring.hp;
 }
+EXPORT_SYMBOL(ath12k_hal_srng_src_peek);
 
 void *ath12k_hal_srng_src_reap_next(struct ath12k_base *ab,
 				    struct hal_srng *srng)
@@ -2162,6 +514,7 @@ void ath12k_hal_srng_access_begin(struct ath12k_base *ab, struct hal_srng *srng)
 		}
 	}
 }
+EXPORT_SYMBOL(ath12k_hal_srng_access_begin);
 
 /* Update cached ring head/tail pointers to HW. ath12k_hal_srng_access_begin()
  * should have been called before this.
@@ -2217,112 +570,7 @@ void ath12k_hal_srng_access_end(struct ath12k_base *ab, struct hal_srng *srng)
 
 	srng->timestamp = jiffies;
 }
-
-void ath12k_hal_setup_link_idle_list(struct ath12k_base *ab,
-				     struct hal_wbm_idle_scatter_list *sbuf,
-				     u32 nsbufs, u32 tot_link_desc,
-				     u32 end_offset)
-{
-	struct ath12k_buffer_addr *link_addr;
-	int i;
-	u32 reg_scatter_buf_sz = HAL_WBM_IDLE_SCATTER_BUF_SIZE / 64;
-	u32 val;
-
-	link_addr = (void *)sbuf[0].vaddr + HAL_WBM_IDLE_SCATTER_BUF_SIZE;
-
-	for (i = 1; i < nsbufs; i++) {
-		link_addr->info0 = cpu_to_le32(sbuf[i].paddr & HAL_ADDR_LSB_REG_MASK);
-
-		link_addr->info1 =
-			le32_encode_bits((u64)sbuf[i].paddr >> HAL_ADDR_MSB_REG_SHIFT,
-					 HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32) |
-			le32_encode_bits(BASE_ADDR_MATCH_TAG_VAL,
-					 HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG);
-
-		link_addr = (void *)sbuf[i].vaddr +
-			     HAL_WBM_IDLE_SCATTER_BUF_SIZE;
-	}
-
-	val = u32_encode_bits(reg_scatter_buf_sz, HAL_WBM_SCATTER_BUFFER_SIZE) |
-	      u32_encode_bits(0x1, HAL_WBM_LINK_DESC_IDLE_LIST_MODE);
-
-	ath12k_hif_write32(ab,
-			   HAL_SEQ_WCSS_UMAC_WBM_REG +
-			   HAL_WBM_R0_IDLE_LIST_CONTROL_ADDR(ab),
-			   val);
-
-	val = u32_encode_bits(reg_scatter_buf_sz * nsbufs,
-			      HAL_WBM_SCATTER_RING_SIZE_OF_IDLE_LINK_DESC_LIST);
-	ath12k_hif_write32(ab,
-			   HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_R0_IDLE_LIST_SIZE_ADDR(ab),
-			   val);
-
-	val = u32_encode_bits(sbuf[0].paddr & HAL_ADDR_LSB_REG_MASK,
-			      BUFFER_ADDR_INFO0_ADDR);
-	ath12k_hif_write32(ab,
-			   HAL_SEQ_WCSS_UMAC_WBM_REG +
-			   HAL_WBM_SCATTERED_RING_BASE_LSB(ab),
-			   val);
-
-	val = u32_encode_bits(BASE_ADDR_MATCH_TAG_VAL,
-			      HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG) |
-	      u32_encode_bits((u64)sbuf[0].paddr >> HAL_ADDR_MSB_REG_SHIFT,
-			      HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32);
-	ath12k_hif_write32(ab,
-			   HAL_SEQ_WCSS_UMAC_WBM_REG +
-			   HAL_WBM_SCATTERED_RING_BASE_MSB(ab),
-			   val);
-
-	/* Setup head and tail pointers for the idle list */
-	val = u32_encode_bits(sbuf[nsbufs - 1].paddr, BUFFER_ADDR_INFO0_ADDR);
-	ath12k_hif_write32(ab,
-			   HAL_SEQ_WCSS_UMAC_WBM_REG +
-			   HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0(ab),
-			   val);
-
-	val = u32_encode_bits(((u64)sbuf[nsbufs - 1].paddr >> HAL_ADDR_MSB_REG_SHIFT),
-			      HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32) |
-	       u32_encode_bits((end_offset >> 2),
-			       HAL_WBM_SCATTERED_DESC_HEAD_P_OFFSET_IX1);
-	ath12k_hif_write32(ab,
-			   HAL_SEQ_WCSS_UMAC_WBM_REG +
-			   HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX1(ab),
-			   val);
-
-	val = u32_encode_bits(sbuf[0].paddr, BUFFER_ADDR_INFO0_ADDR);
-	ath12k_hif_write32(ab,
-			   HAL_SEQ_WCSS_UMAC_WBM_REG +
-			   HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0(ab),
-			   val);
-
-	val = u32_encode_bits(sbuf[0].paddr, BUFFER_ADDR_INFO0_ADDR);
-	ath12k_hif_write32(ab,
-			   HAL_SEQ_WCSS_UMAC_WBM_REG +
-			   HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX0(ab),
-			   val);
-
-	val = u32_encode_bits(((u64)sbuf[0].paddr >> HAL_ADDR_MSB_REG_SHIFT),
-			      HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32) |
-	      u32_encode_bits(0, HAL_WBM_SCATTERED_DESC_TAIL_P_OFFSET_IX1);
-	ath12k_hif_write32(ab,
-			   HAL_SEQ_WCSS_UMAC_WBM_REG +
-			   HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX1(ab),
-			   val);
-
-	val = 2 * tot_link_desc;
-	ath12k_hif_write32(ab,
-			   HAL_SEQ_WCSS_UMAC_WBM_REG +
-			   HAL_WBM_SCATTERED_DESC_PTR_HP_ADDR(ab),
-			   val);
-
-	/* Enable the SRNG */
-	val = u32_encode_bits(1, HAL_WBM_IDLE_LINK_RING_MISC_SRNG_ENABLE) |
-	      u32_encode_bits(1, HAL_WBM_IDLE_LINK_RING_MISC_RIND_ID_DISABLE);
-	ath12k_hif_write32(ab,
-			   HAL_SEQ_WCSS_UMAC_WBM_REG +
-			   HAL_WBM_IDLE_LINK_RING_MISC_ADDR(ab),
-			   val);
-}
+EXPORT_SYMBOL(ath12k_hal_srng_access_end);
 
 int ath12k_hal_srng_setup(struct ath12k_base *ab, enum hal_ring_type type,
 			  int ring_num, int mac_id,
@@ -2334,9 +582,8 @@ int ath12k_hal_srng_setup(struct ath12k_base *ab, enum hal_ring_type type,
 	int ring_id;
 	u32 idx;
 	int i;
-	u32 reg_base;
 
-	ring_id = ath12k_hal_srng_get_ring_id(ab, type, ring_num, mac_id);
+	ring_id = ath12k_hal_srng_get_ring_id(hal, type, ring_num, mac_id);
 	if (ring_id < 0)
 		return ring_id;
 
@@ -2369,8 +616,6 @@ int ath12k_hal_srng_setup(struct ath12k_base *ab, enum hal_ring_type type,
 	memset(srng->ring_base_vaddr, 0,
 	       (srng->entry_size * srng->num_entries) << 2);
 
-	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2];
-
 	if (srng->ring_dir == HAL_SRNG_DIR_SRC) {
 		srng->u.src_ring.hp = 0;
 		srng->u.src_ring.cached_tp = 0;
@@ -2379,16 +624,7 @@ int ath12k_hal_srng_setup(struct ath12k_base *ab, enum hal_ring_type type,
 		srng->u.src_ring.low_threshold = params->low_threshold *
 						 srng->entry_size;
 		if (srng_config->mac_type == ATH12K_HAL_SRNG_UMAC) {
-			if (!ab->hw_params->supports_shadow_regs)
-				srng->u.src_ring.hp_addr =
-					(u32 *)((unsigned long)ab->mem + reg_base);
-			else
-				ath12k_dbg(ab, ATH12K_DBG_HAL,
-					   "hal type %d ring_num %d reg_base 0x%x shadow 0x%lx\n",
-					   type, ring_num,
-					   reg_base,
-					   (unsigned long)srng->u.src_ring.hp_addr -
-					   (unsigned long)ab->mem);
+			ath12k_hal_set_umac_srng_ptr_addr(ab, srng);
 		} else {
 			idx = ring_id - HAL_SRNG_RING_ID_DMAC_CMN_ID_START;
 			srng->u.src_ring.hp_addr = (void *)(hal->wrp.vaddr +
@@ -2409,17 +645,7 @@ int ath12k_hal_srng_setup(struct ath12k_base *ab, enum hal_ring_type type,
 		srng->u.dst_ring.cached_hp = 0;
 		srng->u.dst_ring.hp_addr = (void *)(hal->rdp.vaddr + ring_id);
 		if (srng_config->mac_type == ATH12K_HAL_SRNG_UMAC) {
-			if (!ab->hw_params->supports_shadow_regs)
-				srng->u.dst_ring.tp_addr =
-					(u32 *)((unsigned long)ab->mem + reg_base +
-					(HAL_REO1_RING_TP - HAL_REO1_RING_HP));
-			else
-				ath12k_dbg(ab, ATH12K_DBG_HAL,
-					   "type %d ring_num %d target_reg 0x%x shadow 0x%lx\n",
-					   type, ring_num,
-					   reg_base + HAL_REO1_RING_TP - HAL_REO1_RING_HP,
-					   (unsigned long)srng->u.dst_ring.tp_addr -
-					   (unsigned long)ab->mem);
+			ath12k_hal_set_umac_srng_ptr_addr(ab, srng);
 		} else {
 			/* For PMAC & DMAC rings, tail pointer updates will be done
 			 * through FW by writing to a shared memory location
@@ -2444,68 +670,6 @@ int ath12k_hal_srng_setup(struct ath12k_base *ab, enum hal_ring_type type,
 	return ring_id;
 }
 
-static void ath12k_hal_srng_update_hp_tp_addr(struct ath12k_base *ab,
-					      int shadow_cfg_idx,
-					      enum hal_ring_type ring_type,
-					      int ring_num)
-{
-	struct hal_srng *srng;
-	struct ath12k_hal *hal = &ab->hal;
-	int ring_id;
-	struct hal_srng_config *srng_config = &hal->srng_config[ring_type];
-
-	ring_id = ath12k_hal_srng_get_ring_id(ab, ring_type, ring_num, 0);
-	if (ring_id < 0)
-		return;
-
-	srng = &hal->srng_list[ring_id];
-
-	if (srng_config->ring_dir == HAL_SRNG_DIR_DST)
-		srng->u.dst_ring.tp_addr = (u32 *)(HAL_SHADOW_REG(shadow_cfg_idx) +
-						   (unsigned long)ab->mem);
-	else
-		srng->u.src_ring.hp_addr = (u32 *)(HAL_SHADOW_REG(shadow_cfg_idx) +
-						   (unsigned long)ab->mem);
-}
-
-int ath12k_hal_srng_update_shadow_config(struct ath12k_base *ab,
-					 enum hal_ring_type ring_type,
-					 int ring_num)
-{
-	struct ath12k_hal *hal = &ab->hal;
-	struct hal_srng_config *srng_config = &hal->srng_config[ring_type];
-	int shadow_cfg_idx = hal->num_shadow_reg_configured;
-	u32 target_reg;
-
-	if (shadow_cfg_idx >= HAL_SHADOW_NUM_REGS)
-		return -EINVAL;
-
-	hal->num_shadow_reg_configured++;
-
-	target_reg = srng_config->reg_start[HAL_HP_OFFSET_IN_REG_START];
-	target_reg += srng_config->reg_size[HAL_HP_OFFSET_IN_REG_START] *
-		ring_num;
-
-	/* For destination ring, shadow the TP */
-	if (srng_config->ring_dir == HAL_SRNG_DIR_DST)
-		target_reg += HAL_OFFSET_FROM_HP_TO_TP;
-
-	hal->shadow_reg_addr[shadow_cfg_idx] = target_reg;
-
-	/* update hp/tp addr to hal structure*/
-	ath12k_hal_srng_update_hp_tp_addr(ab, shadow_cfg_idx, ring_type,
-					  ring_num);
-
-	ath12k_dbg(ab, ATH12K_DBG_HAL,
-		   "target_reg %x, shadow reg 0x%x shadow_idx 0x%x, ring_type %d, ring num %d",
-		  target_reg,
-		  HAL_SHADOW_REG(shadow_cfg_idx),
-		  shadow_cfg_idx,
-		  ring_type, ring_num);
-
-	return 0;
-}
-
 void ath12k_hal_srng_shadow_config(struct ath12k_base *ab)
 {
 	struct ath12k_hal *hal = &ab->hal;
@@ -2551,18 +715,16 @@ void ath12k_hal_srng_shadow_update_hp_tp(struct ath12k_base *ab,
 		ath12k_hal_srng_access_end(ab, srng);
 }
 
-static void ath12k_hal_register_srng_lock_keys(struct ath12k_base *ab)
+static void ath12k_hal_register_srng_lock_keys(struct ath12k_hal *hal)
 {
-	struct ath12k_hal *hal = &ab->hal;
 	u32 ring_id;
 
 	for (ring_id = 0; ring_id < HAL_SRNG_RING_ID_MAX; ring_id++)
 		lockdep_register_key(&hal->srng_list[ring_id].lock_key);
 }
 
-static void ath12k_hal_unregister_srng_lock_keys(struct ath12k_base *ab)
+static void ath12k_hal_unregister_srng_lock_keys(struct ath12k_hal *hal)
 {
-	struct ath12k_hal *hal = &ab->hal;
 	u32 ring_id;
 
 	for (ring_id = 0; ring_id < HAL_SRNG_RING_ID_MAX; ring_id++)
@@ -2574,26 +736,26 @@ int ath12k_hal_srng_init(struct ath12k_base *ab)
 	struct ath12k_hal *hal = &ab->hal;
 	int ret;
 
-	memset(hal, 0, sizeof(*hal));
-
-	ret = ab->hw_params->hal_ops->create_srng_config(ab);
+	ret = hal->ops->create_srng_config(hal);
 	if (ret)
 		goto err_hal;
 
-	ret = ath12k_hal_alloc_cont_rdp(ab);
+	hal->dev = ab->dev;
+
+	ret = ath12k_hal_alloc_cont_rdp(hal);
 	if (ret)
 		goto err_hal;
 
-	ret = ath12k_hal_alloc_cont_wrp(ab);
+	ret = ath12k_hal_alloc_cont_wrp(hal);
 	if (ret)
 		goto err_free_cont_rdp;
 
-	ath12k_hal_register_srng_lock_keys(ab);
+	ath12k_hal_register_srng_lock_keys(hal);
 
 	return 0;
 
 err_free_cont_rdp:
-	ath12k_hal_free_cont_rdp(ab);
+	ath12k_hal_free_cont_rdp(hal);
 
 err_hal:
 	return ret;
@@ -2603,9 +765,9 @@ void ath12k_hal_srng_deinit(struct ath12k_base *ab)
 {
 	struct ath12k_hal *hal = &ab->hal;
 
-	ath12k_hal_unregister_srng_lock_keys(ab);
-	ath12k_hal_free_cont_rdp(ab);
-	ath12k_hal_free_cont_wrp(ab);
+	ath12k_hal_unregister_srng_lock_keys(hal);
+	ath12k_hal_free_cont_rdp(hal);
+	ath12k_hal_free_cont_wrp(hal);
 	kfree(hal->srng_config);
 	hal->srng_config = NULL;
 }
diff --git a/drivers/net/wireless/ath/ath12k/hal.h b/drivers/net/wireless/ath/ath12k/hal.h
index efe00e167998..f23ba1f9eaac 100644
--- a/drivers/net/wireless/ath/ath12k/hal.h
+++ b/drivers/net/wireless/ath/ath12k/hal.h
@@ -1,16 +1,43 @@
 /* SPDX-License-Identifier: BSD-3-Clause-Clear */
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #ifndef ATH12K_HAL_H
 #define ATH12K_HAL_H
 
-#include "hal_desc.h"
-#include "rx_desc.h"
+#include "hw.h"
 
 struct ath12k_base;
+
+#define HAL_DESC_REO_NON_QOS_TID	16
+
+#define HAL_INVALID_PEERID	0x3fff
+#define VHT_SIG_SU_NSS_MASK	0x7
+
+#define HAL_TX_ADDRX_EN			1
+#define HAL_TX_ADDRY_EN			2
+
+#define HAL_TX_ADDR_SEARCH_DEFAULT	0
+#define HAL_TX_ADDR_SEARCH_INDEX	1
+
+#define HAL_RX_MAX_MPDU		256
+#define HAL_RX_NUM_WORDS_PER_PPDU_BITMAP	(HAL_RX_MAX_MPDU >> 5)
+
+/* TODO: 16 entries per radio times MAX_VAPS_SUPPORTED */
+#define HAL_DSCP_TID_MAP_TBL_NUM_ENTRIES_MAX	32
+#define HAL_DSCP_TID_TBL_SIZE			24
+
+#define EHT_MAX_USER_INFO	4
+#define HAL_RX_MON_MAX_AGGR_SIZE	128
+#define HAL_MAX_UL_MU_USERS	37
+
+#define MAX_USER_POS 8
+#define MAX_MU_GROUP_ID 64
+#define MAX_MU_GROUP_SHOW 16
+#define MAX_MU_GROUP_LENGTH (6 * MAX_MU_GROUP_SHOW)
+
 #define HAL_CE_REMAP_REG_BASE	(ab->ce_remap_base_addr)
 
 #define HAL_LINK_DESC_SIZE			(32 << 2)
@@ -24,364 +51,37 @@ struct ath12k_base;
 #define HAL_RING_BASE_ALIGN	8
 #define HAL_REO_QLUT_ADDR_ALIGN 256
 
+#define HAL_ADDR_LSB_REG_MASK		0xffffffff
+#define HAL_ADDR_MSB_REG_SHIFT		32
+
+#define HAL_WBM2SW_REL_ERR_RING_NUM 3
+
+#define HAL_SHADOW_NUM_REGS_MAX	40
+
 #define HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX	32704
 /* TODO: Check with hw team on the supported scatter buf size */
 #define HAL_WBM_IDLE_SCATTER_NEXT_PTR_SIZE	8
 #define HAL_WBM_IDLE_SCATTER_BUF_SIZE (HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX - \
 				       HAL_WBM_IDLE_SCATTER_NEXT_PTR_SIZE)
 
-/* TODO: 16 entries per radio times MAX_VAPS_SUPPORTED */
-#define HAL_DSCP_TID_MAP_TBL_NUM_ENTRIES_MAX	32
-#define HAL_DSCP_TID_TBL_SIZE			24
-
-/* calculate the register address from bar0 of shadow register x */
-#define HAL_SHADOW_BASE_ADDR			0x000008fc
-#define HAL_SHADOW_NUM_REGS			40
-#define HAL_HP_OFFSET_IN_REG_START		1
-#define HAL_OFFSET_FROM_HP_TO_TP		4
-
-#define HAL_SHADOW_REG(x) (HAL_SHADOW_BASE_ADDR + (4 * (x)))
-#define HAL_REO_QDESC_MAX_PEERID		8191
-
-/* WCSS Relative address */
-#define HAL_SEQ_WCSS_CMEM_OFFSET		0x00100000
-#define HAL_SEQ_WCSS_UMAC_OFFSET		0x00a00000
-#define HAL_SEQ_WCSS_UMAC_REO_REG		0x00a38000
-#define HAL_SEQ_WCSS_UMAC_TCL_REG		0x00a44000
-#define HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) \
-	((ab)->hw_params->regs->hal_umac_ce0_src_reg_base)
-#define HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) \
-	((ab)->hw_params->regs->hal_umac_ce0_dest_reg_base)
-#define HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) \
-	((ab)->hw_params->regs->hal_umac_ce1_src_reg_base)
-#define HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) \
-	((ab)->hw_params->regs->hal_umac_ce1_dest_reg_base)
-#define HAL_SEQ_WCSS_UMAC_WBM_REG		0x00a34000
-
-#define HAL_CE_WFSS_CE_REG_BASE			0x01b80000
-
-#define HAL_TCL_SW_CONFIG_BANK_ADDR		0x00a4408c
-
-/* SW2TCL(x) R0 ring configuration address */
-#define HAL_TCL1_RING_CMN_CTRL_REG		0x00000020
-#define HAL_TCL1_RING_DSCP_TID_MAP		0x00000240
-#define HAL_TCL1_RING_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_tcl1_ring_base_lsb)
-#define HAL_TCL1_RING_BASE_MSB(ab) \
-	((ab)->hw_params->regs->hal_tcl1_ring_base_msb)
-#define HAL_TCL1_RING_ID(ab)			((ab)->hw_params->regs->hal_tcl1_ring_id)
-#define HAL_TCL1_RING_MISC(ab) \
-	((ab)->hw_params->regs->hal_tcl1_ring_misc)
-#define HAL_TCL1_RING_TP_ADDR_LSB(ab) \
-	((ab)->hw_params->regs->hal_tcl1_ring_tp_addr_lsb)
-#define HAL_TCL1_RING_TP_ADDR_MSB(ab) \
-	((ab)->hw_params->regs->hal_tcl1_ring_tp_addr_msb)
-#define HAL_TCL1_RING_CONSUMER_INT_SETUP_IX0(ab) \
-	((ab)->hw_params->regs->hal_tcl1_ring_consumer_int_setup_ix0)
-#define HAL_TCL1_RING_CONSUMER_INT_SETUP_IX1(ab) \
-	((ab)->hw_params->regs->hal_tcl1_ring_consumer_int_setup_ix1)
-#define HAL_TCL1_RING_MSI1_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_tcl1_ring_msi1_base_lsb)
-#define HAL_TCL1_RING_MSI1_BASE_MSB(ab) \
-	((ab)->hw_params->regs->hal_tcl1_ring_msi1_base_msb)
-#define HAL_TCL1_RING_MSI1_DATA(ab) \
-	((ab)->hw_params->regs->hal_tcl1_ring_msi1_data)
-#define HAL_TCL2_RING_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_tcl2_ring_base_lsb)
-#define HAL_TCL_RING_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_tcl_ring_base_lsb)
-
-#define HAL_TCL1_RING_MSI1_BASE_LSB_OFFSET(ab) ({ typeof(ab) _ab = (ab); \
-	(HAL_TCL1_RING_MSI1_BASE_LSB(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); })
-#define HAL_TCL1_RING_MSI1_BASE_MSB_OFFSET(ab)	({ typeof(ab) _ab = (ab); \
-	(HAL_TCL1_RING_MSI1_BASE_MSB(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); })
-#define HAL_TCL1_RING_MSI1_DATA_OFFSET(ab) ({ typeof(ab) _ab = (ab); \
-	(HAL_TCL1_RING_MSI1_DATA(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); })
-#define HAL_TCL1_RING_BASE_MSB_OFFSET(ab) ({ typeof(ab) _ab = (ab); \
-	(HAL_TCL1_RING_BASE_MSB(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); })
-#define HAL_TCL1_RING_ID_OFFSET(ab) ({ typeof(ab) _ab = (ab); \
-	(HAL_TCL1_RING_ID(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); })
-#define HAL_TCL1_RING_CONSR_INT_SETUP_IX0_OFFSET(ab) ({ typeof(ab) _ab = (ab); \
-	(HAL_TCL1_RING_CONSUMER_INT_SETUP_IX0(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); })
-#define HAL_TCL1_RING_CONSR_INT_SETUP_IX1_OFFSET(ab) ({ typeof(ab) _ab = (ab); \
-	(HAL_TCL1_RING_CONSUMER_INT_SETUP_IX1(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); })
-#define HAL_TCL1_RING_TP_ADDR_LSB_OFFSET(ab) ({ typeof(ab) _ab = (ab); \
-	(HAL_TCL1_RING_TP_ADDR_LSB(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); })
-#define HAL_TCL1_RING_TP_ADDR_MSB_OFFSET(ab) ({ typeof(ab) _ab = (ab); \
-	(HAL_TCL1_RING_TP_ADDR_MSB(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); })
-#define HAL_TCL1_RING_MISC_OFFSET(ab) ({ typeof(ab) _ab = (ab); \
-	(HAL_TCL1_RING_MISC(_ab) - HAL_TCL1_RING_BASE_LSB(_ab)); })
-
-/* SW2TCL(x) R2 ring pointers (head/tail) address */
-#define HAL_TCL1_RING_HP			0x00002000
-#define HAL_TCL1_RING_TP			0x00002004
-#define HAL_TCL2_RING_HP			0x00002008
-#define HAL_TCL_RING_HP				0x00002028
-
-#define HAL_TCL1_RING_TP_OFFSET \
-		(HAL_TCL1_RING_TP - HAL_TCL1_RING_HP)
-
-/* TCL STATUS ring address */
-#define HAL_TCL_STATUS_RING_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_tcl_status_ring_base_lsb)
-#define HAL_TCL_STATUS_RING_HP			0x00002048
-
-/* PPE2TCL1 Ring address */
-#define HAL_TCL_PPE2TCL1_RING_BASE_LSB		0x00000c48
-#define HAL_TCL_PPE2TCL1_RING_HP		0x00002038
-
-/* WBM PPE Release Ring address */
-#define HAL_WBM_PPE_RELEASE_RING_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_ppe_rel_ring_base)
-#define HAL_WBM_PPE_RELEASE_RING_HP		0x00003020
-
-/* REO2SW(x) R0 ring configuration address */
-#define HAL_REO1_GEN_ENABLE			0x00000000
-#define HAL_REO1_MISC_CTRL_ADDR(ab) \
-	((ab)->hw_params->regs->hal_reo1_misc_ctrl_addr)
-#define HAL_REO1_DEST_RING_CTRL_IX_0		0x00000004
-#define HAL_REO1_DEST_RING_CTRL_IX_1		0x00000008
-#define HAL_REO1_DEST_RING_CTRL_IX_2		0x0000000c
-#define HAL_REO1_DEST_RING_CTRL_IX_3		0x00000010
-#define HAL_REO1_QDESC_ADDR(ab)		((ab)->hw_params->regs->hal_reo1_qdesc_addr)
-#define HAL_REO1_QDESC_MAX_PEERID(ab)	((ab)->hw_params->regs->hal_reo1_qdesc_max_peerid)
-#define HAL_REO1_SW_COOKIE_CFG0(ab)	((ab)->hw_params->regs->hal_reo1_sw_cookie_cfg0)
-#define HAL_REO1_SW_COOKIE_CFG1(ab)	((ab)->hw_params->regs->hal_reo1_sw_cookie_cfg1)
-#define HAL_REO1_QDESC_LUT_BASE0(ab)	((ab)->hw_params->regs->hal_reo1_qdesc_lut_base0)
-#define HAL_REO1_QDESC_LUT_BASE1(ab)	((ab)->hw_params->regs->hal_reo1_qdesc_lut_base1)
-#define HAL_REO1_RING_BASE_LSB(ab)	((ab)->hw_params->regs->hal_reo1_ring_base_lsb)
-#define HAL_REO1_RING_BASE_MSB(ab)	((ab)->hw_params->regs->hal_reo1_ring_base_msb)
-#define HAL_REO1_RING_ID(ab)		((ab)->hw_params->regs->hal_reo1_ring_id)
-#define HAL_REO1_RING_MISC(ab)		((ab)->hw_params->regs->hal_reo1_ring_misc)
-#define HAL_REO1_RING_HP_ADDR_LSB(ab)	((ab)->hw_params->regs->hal_reo1_ring_hp_addr_lsb)
-#define HAL_REO1_RING_HP_ADDR_MSB(ab)	((ab)->hw_params->regs->hal_reo1_ring_hp_addr_msb)
-#define HAL_REO1_RING_PRODUCER_INT_SETUP(ab) \
-	((ab)->hw_params->regs->hal_reo1_ring_producer_int_setup)
-#define HAL_REO1_RING_MSI1_BASE_LSB(ab)	\
-	((ab)->hw_params->regs->hal_reo1_ring_msi1_base_lsb)
-#define HAL_REO1_RING_MSI1_BASE_MSB(ab)	\
-	((ab)->hw_params->regs->hal_reo1_ring_msi1_base_msb)
-#define HAL_REO1_RING_MSI1_DATA(ab)	((ab)->hw_params->regs->hal_reo1_ring_msi1_data)
-#define HAL_REO2_RING_BASE_LSB(ab)	((ab)->hw_params->regs->hal_reo2_ring_base)
-#define HAL_REO1_AGING_THRESH_IX_0(ab)	((ab)->hw_params->regs->hal_reo1_aging_thres_ix0)
-#define HAL_REO1_AGING_THRESH_IX_1(ab)	((ab)->hw_params->regs->hal_reo1_aging_thres_ix1)
-#define HAL_REO1_AGING_THRESH_IX_2(ab)	((ab)->hw_params->regs->hal_reo1_aging_thres_ix2)
-#define HAL_REO1_AGING_THRESH_IX_3(ab)	((ab)->hw_params->regs->hal_reo1_aging_thres_ix3)
-
-/* REO2SW(x) R2 ring pointers (head/tail) address */
-#define HAL_REO1_RING_HP			0x00003048
-#define HAL_REO1_RING_TP			0x0000304c
-#define HAL_REO2_RING_HP			0x00003050
-
-#define HAL_REO1_RING_TP_OFFSET			(HAL_REO1_RING_TP - HAL_REO1_RING_HP)
-
-/* REO2SW0 ring configuration address */
-#define HAL_REO_SW0_RING_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_reo2_sw0_ring_base)
-
-/* REO2SW0 R2 ring pointer (head/tail) address */
-#define HAL_REO_SW0_RING_HP			0x00003088
-
-/* REO CMD R0 address */
-#define HAL_REO_CMD_RING_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_reo_cmd_ring_base)
-
-/* REO CMD R2 address */
-#define HAL_REO_CMD_HP				0x00003020
-
-/* SW2REO R0 address */
-#define	HAL_SW2REO_RING_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_sw2reo_ring_base)
-#define HAL_SW2REO1_RING_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_sw2reo1_ring_base)
-
-/* SW2REO R2 address */
-#define HAL_SW2REO_RING_HP			0x00003028
-#define HAL_SW2REO1_RING_HP			0x00003030
-
-/* CE ring R0 address */
-#define HAL_CE_SRC_RING_BASE_LSB                0x00000000
-#define HAL_CE_DST_RING_BASE_LSB		0x00000000
-#define HAL_CE_DST_STATUS_RING_BASE_LSB		0x00000058
-#define HAL_CE_DST_RING_CTRL			0x000000b0
-
-/* CE ring R2 address */
-#define HAL_CE_DST_RING_HP			0x00000400
-#define HAL_CE_DST_STATUS_RING_HP		0x00000408
-
-/* REO status address */
-#define HAL_REO_STATUS_RING_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_reo_status_ring_base)
-#define HAL_REO_STATUS_HP			0x000030a8
-
-/* WBM Idle R0 address */
-#define HAL_WBM_IDLE_LINK_RING_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_wbm_idle_ring_base_lsb)
-#define HAL_WBM_IDLE_LINK_RING_MISC_ADDR(ab) \
-	((ab)->hw_params->regs->hal_wbm_idle_ring_misc_addr)
-#define HAL_WBM_R0_IDLE_LIST_CONTROL_ADDR(ab) \
-	((ab)->hw_params->regs->hal_wbm_r0_idle_list_cntl_addr)
-#define HAL_WBM_R0_IDLE_LIST_SIZE_ADDR(ab) \
-	((ab)->hw_params->regs->hal_wbm_r0_idle_list_size_addr)
-#define HAL_WBM_SCATTERED_RING_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_wbm_scattered_ring_base_lsb)
-#define HAL_WBM_SCATTERED_RING_BASE_MSB(ab) \
-	((ab)->hw_params->regs->hal_wbm_scattered_ring_base_msb)
-#define HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0(ab) \
-	((ab)->hw_params->regs->hal_wbm_scattered_desc_head_info_ix0)
-#define HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX1(ab) \
-	((ab)->hw_params->regs->hal_wbm_scattered_desc_head_info_ix1)
-#define HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX0(ab) \
-	((ab)->hw_params->regs->hal_wbm_scattered_desc_tail_info_ix0)
-#define HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX1(ab) \
-	((ab)->hw_params->regs->hal_wbm_scattered_desc_tail_info_ix1)
-#define HAL_WBM_SCATTERED_DESC_PTR_HP_ADDR(ab) \
-	((ab)->hw_params->regs->hal_wbm_scattered_desc_ptr_hp_addr)
-
-/* WBM Idle R2 address */
-#define HAL_WBM_IDLE_LINK_RING_HP		0x000030b8
-
-/* SW2WBM R0 release address */
-#define HAL_WBM_SW_RELEASE_RING_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_wbm_sw_release_ring_base_lsb)
-#define HAL_WBM_SW1_RELEASE_RING_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_wbm_sw1_release_ring_base_lsb)
-
-/* SW2WBM R2 release address */
-#define HAL_WBM_SW_RELEASE_RING_HP		0x00003010
-#define HAL_WBM_SW1_RELEASE_RING_HP		0x00003018
-
-/* WBM2SW R0 release address */
-#define HAL_WBM0_RELEASE_RING_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_wbm0_release_ring_base_lsb)
-
-#define HAL_WBM1_RELEASE_RING_BASE_LSB(ab) \
-	((ab)->hw_params->regs->hal_wbm1_release_ring_base_lsb)
-
-/* WBM2SW R2 release address */
-#define HAL_WBM0_RELEASE_RING_HP		0x000030c8
-#define HAL_WBM1_RELEASE_RING_HP		0x000030d0
-
-/* WBM cookie config address and mask */
-#define HAL_WBM_SW_COOKIE_CFG0			0x00000040
-#define HAL_WBM_SW_COOKIE_CFG1			0x00000044
-#define HAL_WBM_SW_COOKIE_CFG2			0x00000090
-#define HAL_WBM_SW_COOKIE_CONVERT_CFG		0x00000094
-
-#define HAL_WBM_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB	GENMASK(7, 0)
-#define HAL_WBM_SW_COOKIE_CFG_COOKIE_PPT_MSB		GENMASK(12, 8)
-#define HAL_WBM_SW_COOKIE_CFG_COOKIE_SPT_MSB		GENMASK(17, 13)
-#define HAL_WBM_SW_COOKIE_CFG_ALIGN			BIT(18)
-#define HAL_WBM_SW_COOKIE_CFG_RELEASE_PATH_EN		BIT(0)
-#define HAL_WBM_SW_COOKIE_CFG_ERR_PATH_EN		BIT(1)
-#define HAL_WBM_SW_COOKIE_CFG_CONV_IND_EN		BIT(3)
-
-#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN		BIT(1)
-#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW1_EN		BIT(2)
-#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW2_EN		BIT(3)
-#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW3_EN		BIT(4)
-#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN		BIT(5)
-#define HAL_WBM_SW_COOKIE_CONV_CFG_GLOBAL_EN		BIT(8)
-
-/* TCL ring field mask and offset */
-#define HAL_TCL1_RING_BASE_MSB_RING_SIZE		GENMASK(27, 8)
-#define HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB	GENMASK(7, 0)
-#define HAL_TCL1_RING_ID_ENTRY_SIZE			GENMASK(7, 0)
-#define HAL_TCL1_RING_MISC_MSI_RING_ID_DISABLE		BIT(0)
-#define HAL_TCL1_RING_MISC_MSI_LOOPCNT_DISABLE		BIT(1)
-#define HAL_TCL1_RING_MISC_MSI_SWAP			BIT(3)
-#define HAL_TCL1_RING_MISC_HOST_FW_SWAP			BIT(4)
-#define HAL_TCL1_RING_MISC_DATA_TLV_SWAP		BIT(5)
-#define HAL_TCL1_RING_MISC_SRNG_ENABLE			BIT(6)
-#define HAL_TCL1_RING_CONSR_INT_SETUP_IX0_INTR_TMR_THOLD   GENMASK(31, 16)
-#define HAL_TCL1_RING_CONSR_INT_SETUP_IX0_BATCH_COUNTER_THOLD GENMASK(14, 0)
-#define HAL_TCL1_RING_CONSR_INT_SETUP_IX1_LOW_THOLD	GENMASK(15, 0)
-#define HAL_TCL1_RING_MSI1_BASE_MSB_MSI1_ENABLE		BIT(8)
-#define HAL_TCL1_RING_MSI1_BASE_MSB_ADDR		GENMASK(7, 0)
-#define HAL_TCL1_RING_CMN_CTRL_DSCP_TID_MAP_PROG_EN	BIT(23)
-#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP		GENMASK(31, 0)
-#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP0		GENMASK(2, 0)
-#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP1		GENMASK(5, 3)
-#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP2		GENMASK(8, 6)
-#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP3		GENMASK(11, 9)
-#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP4		GENMASK(14, 12)
-#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP5		GENMASK(17, 15)
-#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP6		GENMASK(20, 18)
-#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP7		GENMASK(23, 21)
-
-/* REO ring field mask and offset */
-#define HAL_REO1_RING_BASE_MSB_RING_SIZE		GENMASK(27, 8)
-#define HAL_REO1_RING_BASE_MSB_RING_BASE_ADDR_MSB	GENMASK(7, 0)
-#define HAL_REO1_RING_ID_RING_ID			GENMASK(15, 8)
-#define HAL_REO1_RING_ID_ENTRY_SIZE			GENMASK(7, 0)
-#define HAL_REO1_RING_MISC_MSI_SWAP			BIT(3)
-#define HAL_REO1_RING_MISC_HOST_FW_SWAP			BIT(4)
-#define HAL_REO1_RING_MISC_DATA_TLV_SWAP		BIT(5)
-#define HAL_REO1_RING_MISC_SRNG_ENABLE			BIT(6)
-#define HAL_REO1_RING_PRDR_INT_SETUP_INTR_TMR_THOLD	GENMASK(31, 16)
-#define HAL_REO1_RING_PRDR_INT_SETUP_BATCH_COUNTER_THOLD GENMASK(14, 0)
-#define HAL_REO1_RING_MSI1_BASE_MSB_MSI1_ENABLE		BIT(8)
-#define HAL_REO1_RING_MSI1_BASE_MSB_ADDR		GENMASK(7, 0)
-#define HAL_REO1_MISC_CTL_FRAG_DST_RING			GENMASK(20, 17)
-#define HAL_REO1_MISC_CTL_BAR_DST_RING			GENMASK(24, 21)
-#define HAL_REO1_GEN_ENABLE_AGING_LIST_ENABLE		BIT(2)
-#define HAL_REO1_GEN_ENABLE_AGING_FLUSH_ENABLE		BIT(3)
-#define HAL_REO1_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB	GENMASK(7, 0)
-#define HAL_REO1_SW_COOKIE_CFG_COOKIE_PPT_MSB		GENMASK(12, 8)
-#define HAL_REO1_SW_COOKIE_CFG_COOKIE_SPT_MSB		GENMASK(17, 13)
-#define HAL_REO1_SW_COOKIE_CFG_ALIGN			BIT(18)
-#define HAL_REO1_SW_COOKIE_CFG_ENABLE			BIT(19)
-#define HAL_REO1_SW_COOKIE_CFG_GLOBAL_ENABLE		BIT(20)
-#define HAL_REO_QDESC_ADDR_READ_LUT_ENABLE		BIT(7)
-#define HAL_REO_QDESC_ADDR_READ_CLEAR_QDESC_ARRAY	BIT(6)
-
-/* CE ring bit field mask and shift */
-#define HAL_CE_DST_R0_DEST_CTRL_MAX_LEN			GENMASK(15, 0)
-
-#define HAL_ADDR_LSB_REG_MASK				0xffffffff
-
-#define HAL_ADDR_MSB_REG_SHIFT				32
-
-/* WBM ring bit field mask and shift */
-#define HAL_WBM_LINK_DESC_IDLE_LIST_MODE		BIT(1)
-#define HAL_WBM_SCATTER_BUFFER_SIZE			GENMASK(10, 2)
-#define HAL_WBM_SCATTER_RING_SIZE_OF_IDLE_LINK_DESC_LIST GENMASK(31, 16)
-#define HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32	GENMASK(7, 0)
-#define HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG	GENMASK(31, 8)
-
-#define HAL_WBM_SCATTERED_DESC_HEAD_P_OFFSET_IX1	GENMASK(20, 8)
-#define HAL_WBM_SCATTERED_DESC_TAIL_P_OFFSET_IX1	GENMASK(20, 8)
-
-#define HAL_WBM_IDLE_LINK_RING_MISC_SRNG_ENABLE		BIT(6)
-#define HAL_WBM_IDLE_LINK_RING_MISC_RIND_ID_DISABLE	BIT(0)
-
-#define BASE_ADDR_MATCH_TAG_VAL 0x5
-
-#define HAL_REO_REO2SW1_RING_BASE_MSB_RING_SIZE		0x000fffff
-#define HAL_REO_REO2SW0_RING_BASE_MSB_RING_SIZE		0x000fffff
-#define HAL_REO_SW2REO_RING_BASE_MSB_RING_SIZE		0x0000ffff
-#define HAL_REO_CMD_RING_BASE_MSB_RING_SIZE		0x0000ffff
-#define HAL_REO_STATUS_RING_BASE_MSB_RING_SIZE		0x0000ffff
-#define HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE		0x000fffff
-#define HAL_SW2TCL1_CMD_RING_BASE_MSB_RING_SIZE		0x000fffff
-#define HAL_TCL_STATUS_RING_BASE_MSB_RING_SIZE		0x0000ffff
-#define HAL_CE_SRC_RING_BASE_MSB_RING_SIZE		0x0000ffff
-#define HAL_CE_DST_RING_BASE_MSB_RING_SIZE		0x0000ffff
-#define HAL_CE_DST_STATUS_RING_BASE_MSB_RING_SIZE	0x0000ffff
-#define HAL_WBM_IDLE_LINK_RING_BASE_MSB_RING_SIZE	0x000fffff
-#define HAL_SW2WBM_RELEASE_RING_BASE_MSB_RING_SIZE	0x0000ffff
-#define HAL_WBM2SW_RELEASE_RING_BASE_MSB_RING_SIZE	0x000fffff
-#define HAL_RXDMA_RING_MAX_SIZE				0x0000ffff
-#define HAL_RXDMA_RING_MAX_SIZE_BE			0x000fffff
-#define HAL_WBM2PPE_RELEASE_RING_BASE_MSB_RING_SIZE	0x000fffff
-
-#define HAL_WBM2SW_REL_ERR_RING_NUM 3
-/* Add any other errors here and return them in
- * ath12k_hal_rx_desc_get_err().
- */
-
-#define HAL_IPQ5332_CE_WFSS_REG_BASE	0x740000
-#define HAL_IPQ5332_CE_SIZE		0x100000
+#define HAL_AST_IDX_INVALID    0xFFFF
+#define HAL_RX_MAX_MCS         12
+#define HAL_RX_MAX_MCS_HT      31
+#define HAL_RX_MAX_MCS_VHT     9
+#define HAL_RX_MAX_MCS_HE      11
+#define HAL_RX_MAX_MCS_BE      15
+#define HAL_RX_MAX_NSS         8
+#define HAL_RX_MAX_NUM_LEGACY_RATES 12
+
+#define HAL_RX_UL_OFDMA_USER_INFO_V0_W0_VALID		BIT(30)
+#define HAL_RX_UL_OFDMA_USER_INFO_V0_W0_VER		BIT(31)
+#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_NSS		GENMASK(2, 0)
+#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_MCS		GENMASK(6, 3)
+#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_LDPC		BIT(7)
+#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_DCM		BIT(8)
+#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_RU_START	GENMASK(15, 9)
+#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_RU_SIZE		GENMASK(18, 16)
+#define HAL_RX_FCS_LEN                          4
 
 enum hal_srng_ring_id {
 	HAL_SRNG_RING_ID_REO2SW0 = 0,
@@ -525,6 +225,64 @@ enum hal_srng_ring_id {
 #define HAL_SRNG_RING_ID_MAX    (HAL_SRNG_RING_ID_DMAC_CMN_ID_END + \
 				 HAL_SRNG_NUM_PMAC_RINGS)
 
+enum hal_rx_su_mu_coding {
+	HAL_RX_SU_MU_CODING_BCC,
+	HAL_RX_SU_MU_CODING_LDPC,
+	HAL_RX_SU_MU_CODING_MAX,
+};
+
+enum hal_rx_gi {
+	HAL_RX_GI_0_8_US,
+	HAL_RX_GI_0_4_US,
+	HAL_RX_GI_1_6_US,
+	HAL_RX_GI_3_2_US,
+	HAL_RX_GI_MAX,
+};
+
+enum hal_rx_bw {
+	HAL_RX_BW_20MHZ,
+	HAL_RX_BW_40MHZ,
+	HAL_RX_BW_80MHZ,
+	HAL_RX_BW_160MHZ,
+	HAL_RX_BW_320MHZ,
+	HAL_RX_BW_MAX,
+};
+
+enum hal_rx_preamble {
+	HAL_RX_PREAMBLE_11A,
+	HAL_RX_PREAMBLE_11B,
+	HAL_RX_PREAMBLE_11N,
+	HAL_RX_PREAMBLE_11AC,
+	HAL_RX_PREAMBLE_11AX,
+	HAL_RX_PREAMBLE_11BA,
+	HAL_RX_PREAMBLE_11BE,
+	HAL_RX_PREAMBLE_MAX,
+};
+
+enum hal_rx_reception_type {
+	HAL_RX_RECEPTION_TYPE_SU,
+	HAL_RX_RECEPTION_TYPE_MU_MIMO,
+	HAL_RX_RECEPTION_TYPE_MU_OFDMA,
+	HAL_RX_RECEPTION_TYPE_MU_OFDMA_MIMO,
+	HAL_RX_RECEPTION_TYPE_MAX,
+};
+
+enum hal_rx_legacy_rate {
+	HAL_RX_LEGACY_RATE_1_MBPS,
+	HAL_RX_LEGACY_RATE_2_MBPS,
+	HAL_RX_LEGACY_RATE_5_5_MBPS,
+	HAL_RX_LEGACY_RATE_6_MBPS,
+	HAL_RX_LEGACY_RATE_9_MBPS,
+	HAL_RX_LEGACY_RATE_11_MBPS,
+	HAL_RX_LEGACY_RATE_12_MBPS,
+	HAL_RX_LEGACY_RATE_18_MBPS,
+	HAL_RX_LEGACY_RATE_24_MBPS,
+	HAL_RX_LEGACY_RATE_36_MBPS,
+	HAL_RX_LEGACY_RATE_48_MBPS,
+	HAL_RX_LEGACY_RATE_54_MBPS,
+	HAL_RX_LEGACY_RATE_INVALID,
+};
+
 enum hal_ring_type {
 	HAL_REO_DST,
 	HAL_REO_EXCEPTION,
@@ -554,11 +312,6 @@ enum hal_ring_type {
 	HAL_MAX_RING_TYPES,
 };
 
-#define HAL_RX_MAX_BA_WINDOW	256
-
-#define HAL_DEFAULT_BE_BK_VI_REO_TIMEOUT_USEC	(100 * 1000)
-#define HAL_DEFAULT_VO_REO_TIMEOUT_USEC		(40 * 1000)
-
 /**
  * enum hal_reo_cmd_type: Enum for REO command type
  * @HAL_REO_CMD_GET_QUEUE_STATS: Get REO queue status/stats
@@ -597,6 +350,128 @@ enum hal_reo_cmd_status {
 	HAL_REO_CMD_DRAIN		= 0xff,
 };
 
+enum hal_tcl_encap_type {
+	HAL_TCL_ENCAP_TYPE_RAW,
+	HAL_TCL_ENCAP_TYPE_NATIVE_WIFI,
+	HAL_TCL_ENCAP_TYPE_ETHERNET,
+	HAL_TCL_ENCAP_TYPE_802_3 = 3,
+	HAL_TCL_ENCAP_TYPE_MAX
+};
+
+enum hal_tcl_desc_type {
+	HAL_TCL_DESC_TYPE_BUFFER,
+	HAL_TCL_DESC_TYPE_EXT_DESC,
+	HAL_TCL_DESC_TYPE_MAX,
+};
+
+enum hal_reo_dest_ring_buffer_type {
+	HAL_REO_DEST_RING_BUFFER_TYPE_MSDU,
+	HAL_REO_DEST_RING_BUFFER_TYPE_LINK_DESC,
+};
+
+enum hal_reo_dest_ring_push_reason {
+	HAL_REO_DEST_RING_PUSH_REASON_ERR_DETECTED,
+	HAL_REO_DEST_RING_PUSH_REASON_ROUTING_INSTRUCTION,
+};
+
+enum hal_reo_entr_rxdma_push_reason {
+	HAL_REO_ENTR_RING_RXDMA_PUSH_REASON_ERR_DETECTED,
+	HAL_REO_ENTR_RING_RXDMA_PUSH_REASON_ROUTING_INSTRUCTION,
+	HAL_REO_ENTR_RING_RXDMA_PUSH_REASON_RX_FLUSH,
+};
+
+enum hal_reo_dest_ring_error_code {
+	HAL_REO_DEST_RING_ERROR_CODE_DESC_ADDR_ZERO,
+	HAL_REO_DEST_RING_ERROR_CODE_DESC_INVALID,
+	HAL_REO_DEST_RING_ERROR_CODE_AMPDU_IN_NON_BA,
+	HAL_REO_DEST_RING_ERROR_CODE_NON_BA_DUPLICATE,
+	HAL_REO_DEST_RING_ERROR_CODE_BA_DUPLICATE,
+	HAL_REO_DEST_RING_ERROR_CODE_FRAME_2K_JUMP,
+	HAL_REO_DEST_RING_ERROR_CODE_BAR_2K_JUMP,
+	HAL_REO_DEST_RING_ERROR_CODE_FRAME_OOR,
+	HAL_REO_DEST_RING_ERROR_CODE_BAR_OOR,
+	HAL_REO_DEST_RING_ERROR_CODE_NO_BA_SESSION,
+	HAL_REO_DEST_RING_ERROR_CODE_FRAME_SN_EQUALS_SSN,
+	HAL_REO_DEST_RING_ERROR_CODE_PN_CHECK_FAILED,
+	HAL_REO_DEST_RING_ERROR_CODE_2K_ERR_FLAG_SET,
+	HAL_REO_DEST_RING_ERROR_CODE_PN_ERR_FLAG_SET,
+	HAL_REO_DEST_RING_ERROR_CODE_DESC_BLOCKED,
+	HAL_REO_DEST_RING_ERROR_CODE_MAX,
+};
+
+enum hal_reo_entr_rxdma_ecode {
+	HAL_REO_ENTR_RING_RXDMA_ECODE_OVERFLOW_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_MPDU_LEN_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_FCS_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_DECRYPT_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_TKIP_MIC_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_UNECRYPTED_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_MSDU_LEN_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_MSDU_LIMIT_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_WIFI_PARSE_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_AMSDU_PARSE_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_SA_TIMEOUT_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_DA_TIMEOUT_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_FLOW_TIMEOUT_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_FLUSH_REQUEST_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_AMSDU_FRAG_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_MULTICAST_ECHO_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_AMSDU_MISMATCH_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_UNAUTH_WDS_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_GRPCAST_AMSDU_WDS_ERR,
+	HAL_REO_ENTR_RING_RXDMA_ECODE_MAX,
+};
+
+enum hal_wbm_htt_tx_comp_status {
+	HAL_WBM_REL_HTT_TX_COMP_STATUS_OK,
+	HAL_WBM_REL_HTT_TX_COMP_STATUS_DROP,
+	HAL_WBM_REL_HTT_TX_COMP_STATUS_TTL,
+	HAL_WBM_REL_HTT_TX_COMP_STATUS_REINJ,
+	HAL_WBM_REL_HTT_TX_COMP_STATUS_INSPECT,
+	HAL_WBM_REL_HTT_TX_COMP_STATUS_MEC_NOTIFY,
+	HAL_WBM_REL_HTT_TX_COMP_STATUS_VDEVID_MISMATCH,
+	HAL_WBM_REL_HTT_TX_COMP_STATUS_MAX,
+};
+
+enum hal_encrypt_type {
+	HAL_ENCRYPT_TYPE_WEP_40,
+	HAL_ENCRYPT_TYPE_WEP_104,
+	HAL_ENCRYPT_TYPE_TKIP_NO_MIC,
+	HAL_ENCRYPT_TYPE_WEP_128,
+	HAL_ENCRYPT_TYPE_TKIP_MIC,
+	HAL_ENCRYPT_TYPE_WAPI,
+	HAL_ENCRYPT_TYPE_CCMP_128,
+	HAL_ENCRYPT_TYPE_OPEN,
+	HAL_ENCRYPT_TYPE_CCMP_256,
+	HAL_ENCRYPT_TYPE_GCMP_128,
+	HAL_ENCRYPT_TYPE_AES_GCMP_256,
+	HAL_ENCRYPT_TYPE_WAPI_GCM_SM4,
+};
+
+enum hal_tx_rate_stats_bw {
+	HAL_TX_RATE_STATS_BW_20,
+	HAL_TX_RATE_STATS_BW_40,
+	HAL_TX_RATE_STATS_BW_80,
+	HAL_TX_RATE_STATS_BW_160,
+};
+
+enum hal_tx_rate_stats_pkt_type {
+	HAL_TX_RATE_STATS_PKT_TYPE_11A,
+	HAL_TX_RATE_STATS_PKT_TYPE_11B,
+	HAL_TX_RATE_STATS_PKT_TYPE_11N,
+	HAL_TX_RATE_STATS_PKT_TYPE_11AC,
+	HAL_TX_RATE_STATS_PKT_TYPE_11AX,
+	HAL_TX_RATE_STATS_PKT_TYPE_11BA,
+	HAL_TX_RATE_STATS_PKT_TYPE_11BE,
+};
+
+enum hal_tx_rate_stats_sgi {
+	HAL_TX_RATE_STATS_SGI_08US,
+	HAL_TX_RATE_STATS_SGI_04US,
+	HAL_TX_RATE_STATS_SGI_16US,
+	HAL_TX_RATE_STATS_SGI_32US,
+};
+
 struct hal_wbm_idle_scatter_list {
 	dma_addr_t paddr;
 	struct hal_wbm_link_desc *vaddr;
@@ -625,6 +500,339 @@ enum hal_srng_dir {
 	HAL_SRNG_DIR_DST
 };
 
+enum rx_msdu_start_pkt_type {
+	RX_MSDU_START_PKT_TYPE_11A,
+	RX_MSDU_START_PKT_TYPE_11B,
+	RX_MSDU_START_PKT_TYPE_11N,
+	RX_MSDU_START_PKT_TYPE_11AC,
+	RX_MSDU_START_PKT_TYPE_11AX,
+	RX_MSDU_START_PKT_TYPE_11BA,
+	RX_MSDU_START_PKT_TYPE_11BE,
+};
+
+enum rx_msdu_start_sgi {
+	RX_MSDU_START_SGI_0_8_US,
+	RX_MSDU_START_SGI_0_4_US,
+	RX_MSDU_START_SGI_1_6_US,
+	RX_MSDU_START_SGI_3_2_US,
+};
+
+enum rx_msdu_start_recv_bw {
+	RX_MSDU_START_RECV_BW_20MHZ,
+	RX_MSDU_START_RECV_BW_40MHZ,
+	RX_MSDU_START_RECV_BW_80MHZ,
+	RX_MSDU_START_RECV_BW_160MHZ,
+};
+
+enum rx_msdu_start_reception_type {
+	RX_MSDU_START_RECEPTION_TYPE_SU,
+	RX_MSDU_START_RECEPTION_TYPE_DL_MU_MIMO,
+	RX_MSDU_START_RECEPTION_TYPE_DL_MU_OFDMA,
+	RX_MSDU_START_RECEPTION_TYPE_DL_MU_OFDMA_MIMO,
+	RX_MSDU_START_RECEPTION_TYPE_UL_MU_MIMO,
+	RX_MSDU_START_RECEPTION_TYPE_UL_MU_OFDMA,
+	RX_MSDU_START_RECEPTION_TYPE_UL_MU_OFDMA_MIMO,
+};
+
+enum rx_desc_decap_type {
+	RX_DESC_DECAP_TYPE_RAW,
+	RX_DESC_DECAP_TYPE_NATIVE_WIFI,
+	RX_DESC_DECAP_TYPE_ETHERNET2_DIX,
+	RX_DESC_DECAP_TYPE_8023,
+};
+
+struct hal_rx_user_status {
+	u32 mcs:4,
+	nss:3,
+	ofdma_info_valid:1,
+	ul_ofdma_ru_start_index:7,
+	ul_ofdma_ru_width:7,
+	ul_ofdma_ru_size:8;
+	u32 ul_ofdma_user_v0_word0;
+	u32 ul_ofdma_user_v0_word1;
+	u32 ast_index;
+	u32 tid;
+	u16 tcp_msdu_count;
+	u16 tcp_ack_msdu_count;
+	u16 udp_msdu_count;
+	u16 other_msdu_count;
+	u16 frame_control;
+	u8 frame_control_info_valid;
+	u8 data_sequence_control_info_valid;
+	u16 first_data_seq_ctrl;
+	u32 preamble_type;
+	u16 ht_flags;
+	u16 vht_flags;
+	u16 he_flags;
+	u8 rs_flags;
+	u8 ldpc;
+	u32 mpdu_cnt_fcs_ok;
+	u32 mpdu_cnt_fcs_err;
+	u32 mpdu_fcs_ok_bitmap[HAL_RX_NUM_WORDS_PER_PPDU_BITMAP];
+	u32 mpdu_ok_byte_count;
+	u32 mpdu_err_byte_count;
+	bool ampdu_present;
+	u16 ampdu_id;
+};
+
+struct hal_rx_u_sig_info {
+	bool ul_dl;
+	u8 bw;
+	u8 ppdu_type_comp_mode;
+	u8 eht_sig_mcs;
+	u8 num_eht_sig_sym;
+	struct ieee80211_radiotap_eht_usig usig;
+};
+
+struct hal_rx_tlv_aggr_info {
+	bool in_progress;
+	u16 cur_len;
+	u16 tlv_tag;
+	u8 buf[HAL_RX_MON_MAX_AGGR_SIZE];
+};
+
+struct hal_rx_radiotap_eht {
+	__le32 known;
+	__le32 data[9];
+};
+
+struct hal_rx_eht_info {
+	u8 num_user_info;
+	struct hal_rx_radiotap_eht eht;
+	u32 user_info[EHT_MAX_USER_INFO];
+};
+
+struct hal_rx_msdu_desc_info {
+	u32 msdu_flags;
+	u16 msdu_len; /* 14 bits for length */
+};
+
+/* hal_mon_buf_ring
+ *	Producer : SW
+ *	Consumer : Monitor
+ *
+ * paddr_lo
+ *	Lower 32-bit physical address of the buffer pointer from the source ring.
+ * paddr_hi
+ *	bit range 7-0 : upper 8 bit of the physical address.
+ *	bit range 31-8 : reserved.
+ * cookie
+ *	Consumer: RxMon/TxMon 64 bit cookie of the buffers.
+ */
+struct hal_mon_buf_ring {
+	__le32 paddr_lo;
+	__le32 paddr_hi;
+	__le64 cookie;
+};
+
+struct hal_rx_mon_ppdu_info {
+	u32 ppdu_id;
+	u32 last_ppdu_id;
+	u64 ppdu_ts;
+	u32 num_mpdu_fcs_ok;
+	u32 num_mpdu_fcs_err;
+	u32 preamble_type;
+	u32 mpdu_len;
+	u16 chan_num;
+	u16 freq;
+	u16 tcp_msdu_count;
+	u16 tcp_ack_msdu_count;
+	u16 udp_msdu_count;
+	u16 other_msdu_count;
+	u16 peer_id;
+	u8 rate;
+	u8 mcs;
+	u8 nss;
+	u8 bw;
+	u8 vht_flag_values1;
+	u8 vht_flag_values2;
+	u8 vht_flag_values3[4];
+	u8 vht_flag_values4;
+	u8 vht_flag_values5;
+	u16 vht_flag_values6;
+	u8 is_stbc;
+	u8 gi;
+	u8 sgi;
+	u8 ldpc;
+	u8 beamformed;
+	u8 rssi_comb;
+	u16 tid;
+	u8 fc_valid;
+	u16 ht_flags;
+	u16 vht_flags;
+	u16 he_flags;
+	u16 he_mu_flags;
+	u8 dcm;
+	u8 ru_alloc;
+	u8 reception_type;
+	u64 tsft;
+	u64 rx_duration;
+	u16 frame_control;
+	u32 ast_index;
+	u8 rs_fcs_err;
+	u8 rs_flags;
+	u8 cck_flag;
+	u8 ofdm_flag;
+	u8 ulofdma_flag;
+	u8 frame_control_info_valid;
+	u16 he_per_user_1;
+	u16 he_per_user_2;
+	u8 he_per_user_position;
+	u8 he_per_user_known;
+	u16 he_flags1;
+	u16 he_flags2;
+	u8 he_RU[4];
+	u16 he_data1;
+	u16 he_data2;
+	u16 he_data3;
+	u16 he_data4;
+	u16 he_data5;
+	u16 he_data6;
+	u32 ppdu_len;
+	u32 prev_ppdu_id;
+	u32 device_id;
+	u16 first_data_seq_ctrl;
+	u8 monitor_direct_used;
+	u8 data_sequence_control_info_valid;
+	u8 ltf_size;
+	u8 rxpcu_filter_pass;
+	s8 rssi_chain[8][8];
+	u32 num_users;
+	u32 mpdu_fcs_ok_bitmap[HAL_RX_NUM_WORDS_PER_PPDU_BITMAP];
+	u8 addr1[ETH_ALEN];
+	u8 addr2[ETH_ALEN];
+	u8 addr3[ETH_ALEN];
+	u8 addr4[ETH_ALEN];
+	struct hal_rx_user_status userstats[HAL_MAX_UL_MU_USERS];
+	u8 userid;
+	bool first_msdu_in_mpdu;
+	bool is_ampdu;
+	u8 medium_prot_type;
+	bool ppdu_continuation;
+	bool eht_usig;
+	struct hal_rx_u_sig_info u_sig_info;
+	bool is_eht;
+	struct hal_rx_eht_info eht_info;
+	struct hal_rx_tlv_aggr_info tlv_aggr;
+};
+
+struct hal_rx_desc_data {
+	struct ieee80211_rx_status *rx_status;
+	u32 phy_meta_data;
+	u32 err_bitmap;
+	u32 enctype;
+	u32 msdu_done:1,
+	    is_decrypted:1,
+	    ip_csum_fail:1,
+	    l4_csum_fail:1,
+	    is_first_msdu:1,
+	    is_last_msdu:1,
+	    mesh_ctrl_present:1,
+	    addr2_present:1,
+	    is_mcbc:1,
+	    seq_ctl_valid:1,
+	    fc_valid:1;
+	u16 msdu_len;
+	u16 peer_id;
+	u16 seq_no;
+	u8 *addr2;
+	u8 pkt_type;
+	u8 l3_pad_bytes;
+	u8 decap_type;
+	u8 bw;
+	u8 rate_mcs;
+	u8 nss;
+	u8 sgi;
+	u8 tid;
+};
+
+#define BUFFER_ADDR_INFO0_ADDR         GENMASK(31, 0)
+
+#define BUFFER_ADDR_INFO1_ADDR         GENMASK(7, 0)
+#define BUFFER_ADDR_INFO1_RET_BUF_MGR  GENMASK(11, 8)
+#define BUFFER_ADDR_INFO1_SW_COOKIE    GENMASK(31, 12)
+
+struct ath12k_buffer_addr {
+	__le32 info0;
+	__le32 info1;
+} __packed;
+
+/* ath12k_buffer_addr
+ *
+ * buffer_addr_31_0
+ *		Address (lower 32 bits) of the MSDU buffer or MSDU_EXTENSION
+ *		descriptor or Link descriptor
+ *
+ * buffer_addr_39_32
+ *		Address (upper 8 bits) of the MSDU buffer or MSDU_EXTENSION
+ *		descriptor or Link descriptor
+ *
+ * return_buffer_manager (RBM)
+ *		Consumer: WBM
+ *		Producer: SW/FW
+ *		Indicates to which buffer manager the buffer or MSDU_EXTENSION
+ *		descriptor or link descriptor that is being pointed to shall be
+ *		returned after the frame has been processed. It is used by WBM
+ *		for routing purposes.
+ *
+ *		Values are defined in enum %HAL_RX_BUF_RBM_
+ *
+ * sw_buffer_cookie
+ *		Cookie field exclusively used by SW. HW ignores the contents,
+ *		accept that it passes the programmed value on to other
+ *		descriptors together with the physical address.
+ *
+ *		Field can be used by SW to for example associate the buffers
+ *		physical address with the virtual address.
+ *
+ *		NOTE1:
+ *		The three most significant bits can have a special meaning
+ *		 in case this struct is embedded in a TX_MPDU_DETAILS STRUCT,
+ *		and field transmit_bw_restriction is set
+ *
+ *		In case of NON punctured transmission:
+ *		Sw_buffer_cookie[19:17] = 3'b000: 20 MHz TX only
+ *		Sw_buffer_cookie[19:17] = 3'b001: 40 MHz TX only
+ *		Sw_buffer_cookie[19:17] = 3'b010: 80 MHz TX only
+ *		Sw_buffer_cookie[19:17] = 3'b011: 160 MHz TX only
+ *		Sw_buffer_cookie[19:17] = 3'b101: 240 MHz TX only
+ *		Sw_buffer_cookie[19:17] = 3'b100: 320 MHz TX only
+ *		Sw_buffer_cookie[19:18] = 2'b11: reserved
+ *
+ *		In case of punctured transmission:
+ *		Sw_buffer_cookie[19:16] = 4'b0000: pattern 0 only
+ *		Sw_buffer_cookie[19:16] = 4'b0001: pattern 1 only
+ *		Sw_buffer_cookie[19:16] = 4'b0010: pattern 2 only
+ *		Sw_buffer_cookie[19:16] = 4'b0011: pattern 3 only
+ *		Sw_buffer_cookie[19:16] = 4'b0100: pattern 4 only
+ *		Sw_buffer_cookie[19:16] = 4'b0101: pattern 5 only
+ *		Sw_buffer_cookie[19:16] = 4'b0110: pattern 6 only
+ *		Sw_buffer_cookie[19:16] = 4'b0111: pattern 7 only
+ *		Sw_buffer_cookie[19:16] = 4'b1000: pattern 8 only
+ *		Sw_buffer_cookie[19:16] = 4'b1001: pattern 9 only
+ *		Sw_buffer_cookie[19:16] = 4'b1010: pattern 10 only
+ *		Sw_buffer_cookie[19:16] = 4'b1011: pattern 11 only
+ *		Sw_buffer_cookie[19:18] = 2'b11: reserved
+ *
+ *		Note: a punctured transmission is indicated by the presence
+ *		 of TLV TX_PUNCTURE_SETUP embedded in the scheduler TLV
+ *
+ *		Sw_buffer_cookie[20:17]: Tid: The TID field in the QoS control
+ *		 field
+ *
+ *		Sw_buffer_cookie[16]: Mpdu_qos_control_valid: This field
+ *		 indicates MPDUs with a QoS control field.
+ *
+ */
+
+struct hal_ce_srng_dest_desc;
+struct hal_ce_srng_dst_status_desc;
+struct hal_ce_srng_src_desc;
+
+struct hal_wbm_link_desc {
+	struct ath12k_buffer_addr buf_addr_info;
+} __packed;
+
 /* srng flags */
 #define HAL_SRNG_FLAGS_MSI_SWAP			0x00000008
 #define HAL_SRNG_FLAGS_RING_PTR_SWAP		0x00000010
@@ -634,9 +842,6 @@ enum hal_srng_dir {
 #define HAL_SRNG_FLAGS_HIGH_THRESH_INTR_EN	0x00080000
 #define HAL_SRNG_FLAGS_LMAC_RING		0x80000000
 
-#define HAL_SRNG_TLV_HDR_TAG		GENMASK(9, 1)
-#define HAL_SRNG_TLV_HDR_LEN		GENMASK(25, 10)
-
 /* Common SRNG ring structure for source and destination rings */
 struct hal_srng {
 	/* Unique SRNG ring ID */
@@ -758,6 +963,51 @@ struct hal_srng {
 	} u;
 };
 
+/* hal_wbm_link_desc
+ *
+ *	Producer: WBM
+ *	Consumer: WBM
+ *
+ * buf_addr_info
+ *		Details of the physical address of a buffer or MSDU
+ *		link descriptor.
+ */
+
+enum hal_wbm_rel_src_module {
+	HAL_WBM_REL_SRC_MODULE_TQM,
+	HAL_WBM_REL_SRC_MODULE_RXDMA,
+	HAL_WBM_REL_SRC_MODULE_REO,
+	HAL_WBM_REL_SRC_MODULE_FW,
+	HAL_WBM_REL_SRC_MODULE_SW,
+	HAL_WBM_REL_SRC_MODULE_MAX,
+};
+
+/* hal_wbm_rel_desc_type
+ *
+ * msdu_buffer
+ *	The address points to an MSDU buffer
+ *
+ * msdu_link_descriptor
+ *	The address points to an Tx MSDU link descriptor
+ *
+ * mpdu_link_descriptor
+ *	The address points to an MPDU link descriptor
+ *
+ * msdu_ext_descriptor
+ *	The address points to an MSDU extension descriptor
+ *
+ * queue_ext_descriptor
+ *	The address points to an TQM queue extension descriptor. WBM should
+ *	treat this is the same way as a link descriptor.
+ */
+enum hal_wbm_rel_desc_type {
+	HAL_WBM_REL_DESC_TYPE_REL_MSDU,
+	HAL_WBM_REL_DESC_TYPE_MSDU_LINK,
+	HAL_WBM_REL_DESC_TYPE_MPDU_LINK,
+	HAL_WBM_REL_DESC_TYPE_MSDU_EXT,
+	HAL_WBM_REL_DESC_TYPE_QUEUE_EXT,
+};
+
 /* Interrupt mitigation - Batch threshold in terms of number of frames */
 #define HAL_SRNG_INT_BATCH_THRESHOLD_TX 256
 #define HAL_SRNG_INT_BATCH_THRESHOLD_RX 128
@@ -821,66 +1071,6 @@ enum hal_rx_buf_return_buf_manager {
 	HAL_RX_BUF_RBM_SW6_BM,
 };
 
-#define HAL_SRNG_DESC_LOOP_CNT		0xf0000000
-
-#define HAL_REO_CMD_FLG_NEED_STATUS		BIT(0)
-#define HAL_REO_CMD_FLG_STATS_CLEAR		BIT(1)
-#define HAL_REO_CMD_FLG_FLUSH_BLOCK_LATER	BIT(2)
-#define HAL_REO_CMD_FLG_FLUSH_RELEASE_BLOCKING	BIT(3)
-#define HAL_REO_CMD_FLG_FLUSH_NO_INVAL		BIT(4)
-#define HAL_REO_CMD_FLG_FLUSH_FWD_ALL_MPDUS	BIT(5)
-#define HAL_REO_CMD_FLG_FLUSH_ALL		BIT(6)
-#define HAL_REO_CMD_FLG_UNBLK_RESOURCE		BIT(7)
-#define HAL_REO_CMD_FLG_UNBLK_CACHE		BIT(8)
-#define HAL_REO_CMD_FLG_FLUSH_QUEUE_1K_DESC	BIT(9)
-
-/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO0_UPD_* fields */
-#define HAL_REO_CMD_UPD0_RX_QUEUE_NUM		BIT(8)
-#define HAL_REO_CMD_UPD0_VLD			BIT(9)
-#define HAL_REO_CMD_UPD0_ALDC			BIT(10)
-#define HAL_REO_CMD_UPD0_DIS_DUP_DETECTION	BIT(11)
-#define HAL_REO_CMD_UPD0_SOFT_REORDER_EN	BIT(12)
-#define HAL_REO_CMD_UPD0_AC			BIT(13)
-#define HAL_REO_CMD_UPD0_BAR			BIT(14)
-#define HAL_REO_CMD_UPD0_RETRY			BIT(15)
-#define HAL_REO_CMD_UPD0_CHECK_2K_MODE		BIT(16)
-#define HAL_REO_CMD_UPD0_OOR_MODE		BIT(17)
-#define HAL_REO_CMD_UPD0_BA_WINDOW_SIZE		BIT(18)
-#define HAL_REO_CMD_UPD0_PN_CHECK		BIT(19)
-#define HAL_REO_CMD_UPD0_EVEN_PN		BIT(20)
-#define HAL_REO_CMD_UPD0_UNEVEN_PN		BIT(21)
-#define HAL_REO_CMD_UPD0_PN_HANDLE_ENABLE	BIT(22)
-#define HAL_REO_CMD_UPD0_PN_SIZE		BIT(23)
-#define HAL_REO_CMD_UPD0_IGNORE_AMPDU_FLG	BIT(24)
-#define HAL_REO_CMD_UPD0_SVLD			BIT(25)
-#define HAL_REO_CMD_UPD0_SSN			BIT(26)
-#define HAL_REO_CMD_UPD0_SEQ_2K_ERR		BIT(27)
-#define HAL_REO_CMD_UPD0_PN_ERR			BIT(28)
-#define HAL_REO_CMD_UPD0_PN_VALID		BIT(29)
-#define HAL_REO_CMD_UPD0_PN			BIT(30)
-
-/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO1_* fields */
-#define HAL_REO_CMD_UPD1_VLD			BIT(16)
-#define HAL_REO_CMD_UPD1_ALDC			GENMASK(18, 17)
-#define HAL_REO_CMD_UPD1_DIS_DUP_DETECTION	BIT(19)
-#define HAL_REO_CMD_UPD1_SOFT_REORDER_EN	BIT(20)
-#define HAL_REO_CMD_UPD1_AC			GENMASK(22, 21)
-#define HAL_REO_CMD_UPD1_BAR			BIT(23)
-#define HAL_REO_CMD_UPD1_RETRY			BIT(24)
-#define HAL_REO_CMD_UPD1_CHECK_2K_MODE		BIT(25)
-#define HAL_REO_CMD_UPD1_OOR_MODE		BIT(26)
-#define HAL_REO_CMD_UPD1_PN_CHECK		BIT(27)
-#define HAL_REO_CMD_UPD1_EVEN_PN		BIT(28)
-#define HAL_REO_CMD_UPD1_UNEVEN_PN		BIT(29)
-#define HAL_REO_CMD_UPD1_PN_HANDLE_ENABLE	BIT(30)
-#define HAL_REO_CMD_UPD1_IGNORE_AMPDU_FLG	BIT(31)
-
-/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO2_* fields */
-#define HAL_REO_CMD_UPD2_SVLD			BIT(10)
-#define HAL_REO_CMD_UPD2_SSN			GENMASK(22, 11)
-#define HAL_REO_CMD_UPD2_SEQ_2K_ERR		BIT(23)
-#define HAL_REO_CMD_UPD2_PN_ERR			BIT(24)
-
 struct ath12k_hal_reo_cmd {
 	u32 addr_lo;
 	u32 flag;
@@ -926,91 +1116,89 @@ struct hal_reo_status_header {
 	u32 timestamp;
 };
 
-struct hal_reo_status_queue_stats {
-	u16 ssn;
-	u16 curr_idx;
-	u32 pn[4];
-	u32 last_rx_queue_ts;
-	u32 last_rx_dequeue_ts;
-	u32 rx_bitmap[8]; /* Bitmap from 0-255 */
-	u32 curr_mpdu_cnt;
-	u32 curr_msdu_cnt;
-	u16 fwd_due_to_bar_cnt;
-	u16 dup_cnt;
-	u32 frames_in_order_cnt;
-	u32 num_mpdu_processed_cnt;
-	u32 num_msdu_processed_cnt;
-	u32 total_num_processed_byte_cnt;
-	u32 late_rx_mpdu_cnt;
-	u32 reorder_hole_cnt;
-	u8 timeout_cnt;
-	u8 bar_rx_cnt;
-	u8 num_window_2k_jump_cnt;
+struct ath12k_hw_hal_params {
+	enum hal_rx_buf_return_buf_manager rx_buf_rbm;
+	u32 wbm2sw_cc_enable;
 };
 
-struct hal_reo_status_flush_queue {
-	bool err_detected;
-};
-
-enum hal_reo_status_flush_cache_err_code {
-	HAL_REO_STATUS_FLUSH_CACHE_ERR_CODE_SUCCESS,
-	HAL_REO_STATUS_FLUSH_CACHE_ERR_CODE_IN_USE,
-	HAL_REO_STATUS_FLUSH_CACHE_ERR_CODE_NOT_FOUND,
-};
-
-struct hal_reo_status_flush_cache {
-	bool err_detected;
-	enum hal_reo_status_flush_cache_err_code err_code;
-	bool cache_controller_flush_status_hit;
-	u8 cache_controller_flush_status_desc_type;
-	u8 cache_controller_flush_status_client_id;
-	u8 cache_controller_flush_status_err;
-	u8 cache_controller_flush_status_cnt;
-};
-
-enum hal_reo_status_unblock_cache_type {
-	HAL_REO_STATUS_UNBLOCK_BLOCKING_RESOURCE,
-	HAL_REO_STATUS_UNBLOCK_ENTIRE_CACHE_USAGE,
-};
-
-struct hal_reo_status_unblock_cache {
-	bool err_detected;
-	enum hal_reo_status_unblock_cache_type unblock_type;
-};
-
-struct hal_reo_status_flush_timeout_list {
-	bool err_detected;
-	bool list_empty;
-	u16 release_desc_cnt;
-	u16 fwd_buf_cnt;
-};
-
-enum hal_reo_threshold_idx {
-	HAL_REO_THRESHOLD_IDX_DESC_COUNTER0,
-	HAL_REO_THRESHOLD_IDX_DESC_COUNTER1,
-	HAL_REO_THRESHOLD_IDX_DESC_COUNTER2,
-	HAL_REO_THRESHOLD_IDX_DESC_COUNTER_SUM,
-};
-
-struct hal_reo_status_desc_thresh_reached {
-	enum hal_reo_threshold_idx threshold_idx;
-	u32 link_desc_counter0;
-	u32 link_desc_counter1;
-	u32 link_desc_counter2;
-	u32 link_desc_counter_sum;
-};
-
-struct hal_reo_status {
-	struct hal_reo_status_header uniform_hdr;
-	u8 loop_cnt;
-	union {
-		struct hal_reo_status_queue_stats queue_stats;
-		struct hal_reo_status_flush_queue flush_queue;
-		struct hal_reo_status_flush_cache flush_cache;
-		struct hal_reo_status_unblock_cache unblock_cache;
-		struct hal_reo_status_flush_timeout_list timeout_list;
-		struct hal_reo_status_desc_thresh_reached desc_thresh_reached;
-	} u;
+struct ath12k_hw_regs {
+	u32 tcl1_ring_id;
+	u32 tcl1_ring_misc;
+	u32 tcl1_ring_tp_addr_lsb;
+	u32 tcl1_ring_tp_addr_msb;
+	u32 tcl1_ring_consumer_int_setup_ix0;
+	u32 tcl1_ring_consumer_int_setup_ix1;
+	u32 tcl1_ring_msi1_base_lsb;
+	u32 tcl1_ring_msi1_base_msb;
+	u32 tcl1_ring_msi1_data;
+	u32 tcl_ring_base_lsb;
+	u32 tcl1_ring_base_lsb;
+	u32 tcl1_ring_base_msb;
+	u32 tcl2_ring_base_lsb;
+
+	u32 tcl_status_ring_base_lsb;
+
+	u32 reo1_qdesc_addr;
+	u32 reo1_qdesc_max_peerid;
+
+	u32 wbm_idle_ring_base_lsb;
+	u32 wbm_idle_ring_misc_addr;
+	u32 wbm_r0_idle_list_cntl_addr;
+	u32 wbm_r0_idle_list_size_addr;
+	u32 wbm_scattered_ring_base_lsb;
+	u32 wbm_scattered_ring_base_msb;
+	u32 wbm_scattered_desc_head_info_ix0;
+	u32 wbm_scattered_desc_head_info_ix1;
+	u32 wbm_scattered_desc_tail_info_ix0;
+	u32 wbm_scattered_desc_tail_info_ix1;
+	u32 wbm_scattered_desc_ptr_hp_addr;
+
+	u32 wbm_sw_release_ring_base_lsb;
+	u32 wbm_sw1_release_ring_base_lsb;
+	u32 wbm0_release_ring_base_lsb;
+	u32 wbm1_release_ring_base_lsb;
+
+	u32 pcie_qserdes_sysclk_en_sel;
+	u32 pcie_pcs_osc_dtct_config_base;
+
+	u32 umac_ce0_src_reg_base;
+	u32 umac_ce0_dest_reg_base;
+	u32 umac_ce1_src_reg_base;
+	u32 umac_ce1_dest_reg_base;
+
+	u32 ppe_rel_ring_base;
+
+	u32 reo2_ring_base;
+	u32 reo1_misc_ctrl_addr;
+	u32 reo1_sw_cookie_cfg0;
+	u32 reo1_sw_cookie_cfg1;
+	u32 reo1_qdesc_lut_base0;
+	u32 reo1_qdesc_lut_base1;
+	u32 reo1_ring_base_lsb;
+	u32 reo1_ring_base_msb;
+	u32 reo1_ring_id;
+	u32 reo1_ring_misc;
+	u32 reo1_ring_hp_addr_lsb;
+	u32 reo1_ring_hp_addr_msb;
+	u32 reo1_ring_producer_int_setup;
+	u32 reo1_ring_msi1_base_lsb;
+	u32 reo1_ring_msi1_base_msb;
+	u32 reo1_ring_msi1_data;
+	u32 reo1_aging_thres_ix0;
+	u32 reo1_aging_thres_ix1;
+	u32 reo1_aging_thres_ix2;
+	u32 reo1_aging_thres_ix3;
+
+	u32 reo2_sw0_ring_base;
+
+	u32 sw2reo_ring_base;
+	u32 sw2reo1_ring_base;
+
+	u32 reo_cmd_ring_base;
+
+	u32 reo_status_ring_base;
+
+	u32 gcc_gcc_pcie_hot_rst;
 };
 
 /* HAL context to be used to access SRNG APIs (currently used by data path
@@ -1036,16 +1224,23 @@ struct ath12k_hal {
 		dma_addr_t paddr;
 	} wrp;
 
+	struct device *dev;
+	const struct hal_ops *ops;
+	const struct ath12k_hw_regs *regs;
+	const struct ath12k_hw_hal_params *hal_params;
 	/* Available REO blocking resources bitmap */
 	u8 avail_blk_resource;
 
 	u8 current_blk_index;
 
 	/* shadow register configuration */
-	u32 shadow_reg_addr[HAL_SHADOW_NUM_REGS];
+	u32 shadow_reg_addr[HAL_SHADOW_NUM_REGS_MAX];
 	int num_shadow_reg_configured;
 
 	u32 hal_desc_sz;
+	u32 hal_wbm_release_ring_tx_size;
+
+	const struct ath12k_hal_tcl_to_wbm_rbm_map *tcl_to_wbm_rbm_map;
 };
 
 /* Maps WBM ring number and Return Buffer Manager Id per TCL ring */
@@ -1054,92 +1249,197 @@ struct ath12k_hal_tcl_to_wbm_rbm_map  {
 	u8 rbm_id;
 };
 
-struct hal_rx_ops {
-	bool (*rx_desc_get_first_msdu)(struct hal_rx_desc *desc);
-	bool (*rx_desc_get_last_msdu)(struct hal_rx_desc *desc);
-	u8 (*rx_desc_get_l3_pad_bytes)(struct hal_rx_desc *desc);
-	u8 *(*rx_desc_get_hdr_status)(struct hal_rx_desc *desc);
-	bool (*rx_desc_encrypt_valid)(struct hal_rx_desc *desc);
-	u32 (*rx_desc_get_encrypt_type)(struct hal_rx_desc *desc);
-	u8 (*rx_desc_get_decap_type)(struct hal_rx_desc *desc);
-	u8 (*rx_desc_get_mesh_ctl)(struct hal_rx_desc *desc);
-	bool (*rx_desc_get_mpdu_seq_ctl_vld)(struct hal_rx_desc *desc);
-	bool (*rx_desc_get_mpdu_fc_valid)(struct hal_rx_desc *desc);
-	u16 (*rx_desc_get_mpdu_start_seq_no)(struct hal_rx_desc *desc);
-	u16 (*rx_desc_get_msdu_len)(struct hal_rx_desc *desc);
-	u8 (*rx_desc_get_msdu_sgi)(struct hal_rx_desc *desc);
-	u8 (*rx_desc_get_msdu_rate_mcs)(struct hal_rx_desc *desc);
-	u8 (*rx_desc_get_msdu_rx_bw)(struct hal_rx_desc *desc);
-	u32 (*rx_desc_get_msdu_freq)(struct hal_rx_desc *desc);
-	u8 (*rx_desc_get_msdu_pkt_type)(struct hal_rx_desc *desc);
-	u8 (*rx_desc_get_msdu_nss)(struct hal_rx_desc *desc);
-	u8 (*rx_desc_get_mpdu_tid)(struct hal_rx_desc *desc);
-	u16 (*rx_desc_get_mpdu_peer_id)(struct hal_rx_desc *desc);
-	void (*rx_desc_copy_end_tlv)(struct hal_rx_desc *fdesc,
-				     struct hal_rx_desc *ldesc);
-	u32 (*rx_desc_get_mpdu_start_tag)(struct hal_rx_desc *desc);
-	u32 (*rx_desc_get_mpdu_ppdu_id)(struct hal_rx_desc *desc);
+enum hal_wbm_rel_bm_act {
+	HAL_WBM_REL_BM_ACT_PUT_IN_IDLE,
+	HAL_WBM_REL_BM_ACT_REL_MSDU,
+};
+
+/* hal_wbm_rel_bm_act
+ *
+ * put_in_idle_list
+ *	Put the buffer or descriptor back in the idle list. In case of MSDU or
+ *	MDPU link descriptor, BM does not need to check to release any
+ *	individual MSDU buffers.
+ *
+ * release_msdu_list
+ *	This BM action can only be used in combination with desc_type being
+ *	msdu_link_descriptor. Field first_msdu_index points out which MSDU
+ *	pointer in the MSDU link descriptor is the first of an MPDU that is
+ *	released. BM shall release all the MSDU buffers linked to this first
+ *	MSDU buffer pointer. All related MSDU buffer pointer entries shall be
+ *	set to value 0, which represents the 'NULL' pointer. When all MSDU
+ *	buffer pointers in the MSDU link descriptor are 'NULL', the MSDU link
+ *	descriptor itself shall also be released.
+ */
+
+#define RU_INVALID		0
+#define RU_26			1
+#define RU_52			2
+#define RU_106			4
+#define RU_242			9
+#define RU_484			18
+#define RU_996			37
+#define RU_2X996		74
+#define RU_3X996		111
+#define RU_4X996		148
+#define RU_52_26		(RU_52 + RU_26)
+#define RU_106_26		(RU_106 + RU_26)
+#define RU_484_242		(RU_484 + RU_242)
+#define RU_996_484		(RU_996 + RU_484)
+#define RU_996_484_242		(RU_996 + RU_484_242)
+#define RU_2X996_484		(RU_2X996 + RU_484)
+#define RU_3X996_484		(RU_3X996 + RU_484)
+
+enum ath12k_eht_ru_size {
+	ATH12K_EHT_RU_26,
+	ATH12K_EHT_RU_52,
+	ATH12K_EHT_RU_106,
+	ATH12K_EHT_RU_242,
+	ATH12K_EHT_RU_484,
+	ATH12K_EHT_RU_996,
+	ATH12K_EHT_RU_996x2,
+	ATH12K_EHT_RU_996x4,
+	ATH12K_EHT_RU_52_26,
+	ATH12K_EHT_RU_106_26,
+	ATH12K_EHT_RU_484_242,
+	ATH12K_EHT_RU_996_484,
+	ATH12K_EHT_RU_996_484_242,
+	ATH12K_EHT_RU_996x2_484,
+	ATH12K_EHT_RU_996x3,
+	ATH12K_EHT_RU_996x3_484,
+
+	/* Keep last */
+	ATH12K_EHT_RU_INVALID,
+};
+
+#define HAL_RX_RU_ALLOC_TYPE_MAX	ATH12K_EHT_RU_INVALID
+
+static inline
+enum nl80211_he_ru_alloc ath12k_he_ru_tones_to_nl80211_he_ru_alloc(u16 ru_tones)
+{
+	enum nl80211_he_ru_alloc ret;
+
+	switch (ru_tones) {
+	case RU_52:
+		ret = NL80211_RATE_INFO_HE_RU_ALLOC_52;
+		break;
+	case RU_106:
+		ret = NL80211_RATE_INFO_HE_RU_ALLOC_106;
+		break;
+	case RU_242:
+		ret = NL80211_RATE_INFO_HE_RU_ALLOC_242;
+		break;
+	case RU_484:
+		ret = NL80211_RATE_INFO_HE_RU_ALLOC_484;
+		break;
+	case RU_996:
+		ret = NL80211_RATE_INFO_HE_RU_ALLOC_996;
+		break;
+	case RU_2X996:
+		ret = NL80211_RATE_INFO_HE_RU_ALLOC_2x996;
+		break;
+	case RU_26:
+		fallthrough;
+	default:
+		ret = NL80211_RATE_INFO_HE_RU_ALLOC_26;
+		break;
+	}
+	return ret;
+}
+
+struct ath12k_hw_version_map {
+	const struct hal_ops *hal_ops;
+	u32 hal_desc_sz;
+	const struct ath12k_hal_tcl_to_wbm_rbm_map *tcl_to_wbm_rbm_map;
+	const struct ath12k_hw_hal_params *hal_params;
+	const struct ath12k_hw_regs *hw_regs;
+};
+
+struct hal_ops {
+	int (*create_srng_config)(struct ath12k_hal *hal);
 	void (*rx_desc_set_msdu_len)(struct hal_rx_desc *desc, u16 len);
-	struct rx_attention *(*rx_desc_get_attention)(struct hal_rx_desc *desc);
-	u8 *(*rx_desc_get_msdu_payload)(struct hal_rx_desc *desc);
-	u32 (*rx_desc_get_mpdu_start_offset)(void);
-	u32 (*rx_desc_get_msdu_end_offset)(void);
-	bool (*rx_desc_mac_addr2_valid)(struct hal_rx_desc *desc);
-	u8* (*rx_desc_mpdu_start_addr2)(struct hal_rx_desc *desc);
-	bool (*rx_desc_is_da_mcbc)(struct hal_rx_desc *desc);
 	void (*rx_desc_get_dot11_hdr)(struct hal_rx_desc *desc,
 				      struct ieee80211_hdr *hdr);
 	void (*rx_desc_get_crypto_header)(struct hal_rx_desc *desc,
 					  u8 *crypto_hdr,
 					  enum hal_encrypt_type enctype);
-	bool (*dp_rx_h_msdu_done)(struct hal_rx_desc *desc);
-	bool (*dp_rx_h_l4_cksum_fail)(struct hal_rx_desc *desc);
-	bool (*dp_rx_h_ip_cksum_fail)(struct hal_rx_desc *desc);
-	bool (*dp_rx_h_is_decrypted)(struct hal_rx_desc *desc);
-	u32 (*dp_rx_h_mpdu_err)(struct hal_rx_desc *desc);
-	u32 (*rx_desc_get_desc_size)(void);
+	void (*rx_desc_copy_end_tlv)(struct hal_rx_desc *fdesc,
+				     struct hal_rx_desc *ldesc);
 	u8 (*rx_desc_get_msdu_src_link_id)(struct hal_rx_desc *desc);
-};
-
-struct hal_ops {
-	int (*create_srng_config)(struct ath12k_base *ab);
-	u16 (*rxdma_ring_wmask_rx_mpdu_start)(void);
-	u32 (*rxdma_ring_wmask_rx_msdu_end)(void);
-	const struct hal_rx_ops *(*get_hal_rx_compact_ops)(void);
-	const struct ath12k_hal_tcl_to_wbm_rbm_map *tcl_to_wbm_rbm_map;
-};
-
-extern const struct hal_ops hal_qcn9274_ops;
-extern const struct hal_ops hal_wcn7850_ops;
-
-extern const struct hal_rx_ops hal_rx_qcn9274_ops;
-extern const struct hal_rx_ops hal_rx_qcn9274_compact_ops;
-extern const struct hal_rx_ops hal_rx_wcn7850_ops;
-
-u32 ath12k_hal_reo_qdesc_size(u32 ba_window_size, u8 tid);
-void ath12k_hal_reo_qdesc_setup(struct hal_rx_reo_queue *qdesc,
-				int tid, u32 ba_window_size,
-				u32 start_seq, enum hal_pn_type type);
-void ath12k_hal_reo_init_cmd_ring(struct ath12k_base *ab,
-				  struct hal_srng *srng);
-void ath12k_hal_reo_hw_setup(struct ath12k_base *ab, u32 ring_hash_map);
-void ath12k_hal_setup_link_idle_list(struct ath12k_base *ab,
+	void (*extract_rx_desc_data)(struct hal_rx_desc_data *rx_desc_data,
+				     struct hal_rx_desc *rx_desc,
+				     struct hal_rx_desc *ldesc);
+	u32 (*rx_desc_get_mpdu_start_tag)(struct hal_rx_desc *desc);
+	u32 (*rx_desc_get_mpdu_ppdu_id)(struct hal_rx_desc *desc);
+	u8 (*rx_desc_get_l3_pad_bytes)(struct hal_rx_desc *desc);
+	u8 *(*rx_desc_get_msdu_payload)(struct hal_rx_desc *desc);
+	void (*ce_dst_setup)(struct ath12k_base *ab,
+			     struct hal_srng *srng, int ring_num);
+	void (*set_umac_srng_ptr_addr)(struct ath12k_base *ab,
+				       struct hal_srng *srng);
+	void (*srng_src_hw_init)(struct ath12k_base *ab, struct hal_srng *srng);
+	void (*srng_dst_hw_init)(struct ath12k_base *ab, struct hal_srng *srng);
+	int (*srng_update_shadow_config)(struct ath12k_base *ab,
+					 enum hal_ring_type ring_type,
+					 int ring_num);
+	int (*srng_get_ring_id)(struct ath12k_hal *hal, enum hal_ring_type type,
+				int ring_num, int mac_id);
+	u32 (*ce_get_desc_size)(enum hal_ce_desc type);
+	void (*ce_src_set_desc)(struct hal_ce_srng_src_desc *desc,
+				dma_addr_t paddr, u32 len, u32 id,
+				u8 byte_swap_data);
+	void (*ce_dst_set_desc)(struct hal_ce_srng_dest_desc *desc,
+				dma_addr_t paddr);
+	u32 (*ce_dst_status_get_length)(struct hal_ce_srng_dst_status_desc *desc);
+	void (*set_link_desc_addr)(struct hal_wbm_link_desc *desc, u32 cookie,
+				   dma_addr_t paddr,
+				   enum hal_rx_buf_return_buf_manager rbm);
+	void (*tx_set_dscp_tid_map)(struct ath12k_base *ab, int id);
+	void (*tx_configure_bank_register)(struct ath12k_base *ab,
+					   u32 bank_config, u8 bank_id);
+	void (*reoq_lut_addr_read_enable)(struct ath12k_base *ab);
+	void (*reoq_lut_set_max_peerid)(struct ath12k_base *ab);
+	void (*write_ml_reoq_lut_addr)(struct ath12k_base *ab,
+				       dma_addr_t paddr);
+	void (*write_reoq_lut_addr)(struct ath12k_base *ab, dma_addr_t paddr);
+	void (*setup_link_idle_list)(struct ath12k_base *ab,
 				     struct hal_wbm_idle_scatter_list *sbuf,
 				     u32 nsbufs, u32 tot_link_desc,
 				     u32 end_offset);
+	void (*reo_init_cmd_ring)(struct ath12k_base *ab,
+				  struct hal_srng *srng);
+	void (*reo_shared_qaddr_cache_clear)(struct ath12k_base *ab);
+	void (*reo_hw_setup)(struct ath12k_base *ab, u32 ring_hash_map);
+	void (*rx_buf_addr_info_set)(struct ath12k_buffer_addr *binfo,
+				     dma_addr_t paddr, u32 cookie, u8 manager);
+	void (*rx_buf_addr_info_get)(struct ath12k_buffer_addr *binfo,
+				     dma_addr_t *paddr, u32 *msdu_cookies,
+				     u8 *rbm);
+	void (*cc_config)(struct ath12k_base *ab);
+	enum hal_rx_buf_return_buf_manager
+		(*get_idle_link_rbm)(struct ath12k_hal *hal, u8 device_id);
+	void (*rx_msdu_list_get)(struct ath12k *ar,
+				 void *link_desc,
+				 void *msdu_list,
+				 u16 *num_msdus);
+	void (*rx_reo_ent_buf_paddr_get)(void *rx_desc, dma_addr_t *paddr,
+					 u32 *sw_cookie,
+					 struct ath12k_buffer_addr **pp_buf_addr,
+					 u8 *rbm, u32 *msdu_cnt);
+};
 
 dma_addr_t ath12k_hal_srng_get_tp_addr(struct ath12k_base *ab,
 				       struct hal_srng *srng);
 dma_addr_t ath12k_hal_srng_get_hp_addr(struct ath12k_base *ab,
 				       struct hal_srng *srng);
-void ath12k_hal_set_link_desc_addr(struct hal_wbm_link_desc *desc, u32 cookie,
-				   dma_addr_t paddr,
-				   enum hal_rx_buf_return_buf_manager rbm);
-u32 ath12k_hal_ce_get_desc_size(enum hal_ce_desc type);
-void ath12k_hal_ce_src_set_desc(struct hal_ce_srng_src_desc *desc, dma_addr_t paddr,
-				u32 len, u32 id, u8 byte_swap_data);
-void ath12k_hal_ce_dst_set_desc(struct hal_ce_srng_dest_desc *desc, dma_addr_t paddr);
-u32 ath12k_hal_ce_dst_status_get_length(struct hal_ce_srng_dst_status_desc *desc);
+u32 ath12k_hal_ce_get_desc_size(struct ath12k_hal *hal, enum hal_ce_desc type);
+void ath12k_hal_ce_dst_set_desc(struct ath12k_hal *hal,
+				struct hal_ce_srng_dest_desc *desc,
+				dma_addr_t paddr);
+void ath12k_hal_ce_src_set_desc(struct ath12k_hal *hal,
+				struct hal_ce_srng_src_desc *desc,
+				dma_addr_t paddr, u32 len, u32 id,
+				u8 byte_swap_data);
 int ath12k_hal_srng_get_entrysize(struct ath12k_base *ab, u32 ring_type);
 int ath12k_hal_srng_get_max_entries(struct ath12k_base *ab, u32 ring_type);
 void ath12k_hal_srng_get_params(struct ath12k_base *ab, struct hal_srng *srng,
@@ -1178,4 +1478,41 @@ void ath12k_hal_srng_shadow_config(struct ath12k_base *ab);
 void ath12k_hal_srng_shadow_update_hp_tp(struct ath12k_base *ab,
 					 struct hal_srng *srng);
 void ath12k_hal_reo_shared_qaddr_cache_clear(struct ath12k_base *ab);
+void ath12k_hal_set_link_desc_addr(struct ath12k_hal *hal,
+				   struct hal_wbm_link_desc *desc, u32 cookie,
+				   dma_addr_t paddr, int rbm);
+void ath12k_hal_setup_link_idle_list(struct ath12k_base *ab,
+				     struct hal_wbm_idle_scatter_list *sbuf,
+				     u32 nsbufs, u32 tot_link_desc,
+				     u32 end_offset);
+u32
+ath12k_hal_ce_dst_status_get_length(struct ath12k_hal *hal,
+				    struct hal_ce_srng_dst_status_desc *desc);
+void ath12k_hal_tx_set_dscp_tid_map(struct ath12k_base *ab, int id);
+void ath12k_hal_tx_configure_bank_register(struct ath12k_base *ab,
+					   u32 bank_config, u8 bank_id);
+void ath12k_hal_reoq_lut_addr_read_enable(struct ath12k_base *ab);
+void ath12k_hal_reoq_lut_set_max_peerid(struct ath12k_base *ab);
+void ath12k_hal_write_reoq_lut_addr(struct ath12k_base *ab, dma_addr_t paddr);
+void
+ath12k_hal_write_ml_reoq_lut_addr(struct ath12k_base *ab, dma_addr_t paddr);
+void ath12k_hal_reo_init_cmd_ring(struct ath12k_base *ab, struct hal_srng *srng);
+void ath12k_hal_reo_hw_setup(struct ath12k_base *ab, u32 ring_hash_map);
+void ath12k_hal_rx_buf_addr_info_set(struct ath12k_hal *hal,
+				     struct ath12k_buffer_addr *binfo,
+				     dma_addr_t paddr, u32 cookie, u8 manager);
+void ath12k_hal_rx_buf_addr_info_get(struct ath12k_hal *hal,
+				     struct ath12k_buffer_addr *binfo,
+				     dma_addr_t *paddr, u32 *msdu_cookies,
+				     u8 *rbm);
+void ath12k_hal_cc_config(struct ath12k_base *ab);
+enum hal_rx_buf_return_buf_manager
+ath12k_hal_get_idle_link_rbm(struct ath12k_hal *hal, u8 device_id);
+void ath12k_hal_rx_msdu_list_get(struct ath12k_hal *hal, struct ath12k *ar,
+				 void *link_desc, void *msdu_list,
+				 u16 *num_msdus);
+void ath12k_hal_rx_reo_ent_buf_paddr_get(struct ath12k_hal *hal, void *rx_desc,
+					 dma_addr_t *paddr, u32 *sw_cookie,
+					 struct ath12k_buffer_addr **pp_buf_addr,
+					 u8 *rbm, u32 *msdu_cnt);
 #endif
diff --git a/drivers/net/wireless/ath/ath12k/hal_desc.h b/drivers/net/wireless/ath/ath12k/hal_desc.h
deleted file mode 100644
index 13ddac4a9412..000000000000
--- a/drivers/net/wireless/ath/ath12k/hal_desc.h
+++ /dev/null
@@ -1,3045 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause-Clear */
-/*
- * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2022, 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved.
- */
-#include "core.h"
-
-#ifndef ATH12K_HAL_DESC_H
-#define ATH12K_HAL_DESC_H
-
-#define BUFFER_ADDR_INFO0_ADDR         GENMASK(31, 0)
-
-#define BUFFER_ADDR_INFO1_ADDR         GENMASK(7, 0)
-#define BUFFER_ADDR_INFO1_RET_BUF_MGR  GENMASK(11, 8)
-#define BUFFER_ADDR_INFO1_SW_COOKIE    GENMASK(31, 12)
-
-struct ath12k_buffer_addr {
-	__le32 info0;
-	__le32 info1;
-} __packed;
-
-/* ath12k_buffer_addr
- *
- * buffer_addr_31_0
- *		Address (lower 32 bits) of the MSDU buffer or MSDU_EXTENSION
- *		descriptor or Link descriptor
- *
- * buffer_addr_39_32
- *		Address (upper 8 bits) of the MSDU buffer or MSDU_EXTENSION
- *		descriptor or Link descriptor
- *
- * return_buffer_manager (RBM)
- *		Consumer: WBM
- *		Producer: SW/FW
- *		Indicates to which buffer manager the buffer or MSDU_EXTENSION
- *		descriptor or link descriptor that is being pointed to shall be
- *		returned after the frame has been processed. It is used by WBM
- *		for routing purposes.
- *
- *		Values are defined in enum %HAL_RX_BUF_RBM_
- *
- * sw_buffer_cookie
- *		Cookie field exclusively used by SW. HW ignores the contents,
- *		accept that it passes the programmed value on to other
- *		descriptors together with the physical address.
- *
- *		Field can be used by SW to for example associate the buffers
- *		physical address with the virtual address.
- *
- *		NOTE1:
- *		The three most significant bits can have a special meaning
- *		 in case this struct is embedded in a TX_MPDU_DETAILS STRUCT,
- *		and field transmit_bw_restriction is set
- *
- *		In case of NON punctured transmission:
- *		Sw_buffer_cookie[19:17] = 3'b000: 20 MHz TX only
- *		Sw_buffer_cookie[19:17] = 3'b001: 40 MHz TX only
- *		Sw_buffer_cookie[19:17] = 3'b010: 80 MHz TX only
- *		Sw_buffer_cookie[19:17] = 3'b011: 160 MHz TX only
- *		Sw_buffer_cookie[19:17] = 3'b101: 240 MHz TX only
- *		Sw_buffer_cookie[19:17] = 3'b100: 320 MHz TX only
- *		Sw_buffer_cookie[19:18] = 2'b11: reserved
- *
- *		In case of punctured transmission:
- *		Sw_buffer_cookie[19:16] = 4'b0000: pattern 0 only
- *		Sw_buffer_cookie[19:16] = 4'b0001: pattern 1 only
- *		Sw_buffer_cookie[19:16] = 4'b0010: pattern 2 only
- *		Sw_buffer_cookie[19:16] = 4'b0011: pattern 3 only
- *		Sw_buffer_cookie[19:16] = 4'b0100: pattern 4 only
- *		Sw_buffer_cookie[19:16] = 4'b0101: pattern 5 only
- *		Sw_buffer_cookie[19:16] = 4'b0110: pattern 6 only
- *		Sw_buffer_cookie[19:16] = 4'b0111: pattern 7 only
- *		Sw_buffer_cookie[19:16] = 4'b1000: pattern 8 only
- *		Sw_buffer_cookie[19:16] = 4'b1001: pattern 9 only
- *		Sw_buffer_cookie[19:16] = 4'b1010: pattern 10 only
- *		Sw_buffer_cookie[19:16] = 4'b1011: pattern 11 only
- *		Sw_buffer_cookie[19:18] = 2'b11: reserved
- *
- *		Note: a punctured transmission is indicated by the presence
- *		 of TLV TX_PUNCTURE_SETUP embedded in the scheduler TLV
- *
- *		Sw_buffer_cookie[20:17]: Tid: The TID field in the QoS control
- *		 field
- *
- *		Sw_buffer_cookie[16]: Mpdu_qos_control_valid: This field
- *		 indicates MPDUs with a QoS control field.
- *
- */
-
-enum hal_tlv_tag {
-	HAL_MACTX_CBF_START					= 0 /* 0x0 */,
-	HAL_PHYRX_DATA						= 1 /* 0x1 */,
-	HAL_PHYRX_CBF_DATA_RESP					= 2 /* 0x2 */,
-	HAL_PHYRX_ABORT_REQUEST					= 3 /* 0x3 */,
-	HAL_PHYRX_USER_ABORT_NOTIFICATION			= 4 /* 0x4 */,
-	HAL_MACTX_DATA_RESP					= 5 /* 0x5 */,
-	HAL_MACTX_CBF_DATA					= 6 /* 0x6 */,
-	HAL_MACTX_CBF_DONE					= 7 /* 0x7 */,
-	HAL_PHYRX_LMR_DATA_RESP					= 8 /* 0x8 */,
-	HAL_RXPCU_TO_UCODE_START				= 9 /* 0x9 */,
-	HAL_RXPCU_TO_UCODE_DELIMITER_FOR_FULL_MPDU		= 10 /* 0xa */,
-	HAL_RXPCU_TO_UCODE_FULL_MPDU_DATA			= 11 /* 0xb */,
-	HAL_RXPCU_TO_UCODE_FCS_STATUS				= 12 /* 0xc */,
-	HAL_RXPCU_TO_UCODE_MPDU_DELIMITER			= 13 /* 0xd */,
-	HAL_RXPCU_TO_UCODE_DELIMITER_FOR_MPDU_HEADER		= 14 /* 0xe */,
-	HAL_RXPCU_TO_UCODE_MPDU_HEADER_DATA			= 15 /* 0xf */,
-	HAL_RXPCU_TO_UCODE_END					= 16 /* 0x10 */,
-	HAL_MACRX_CBF_READ_REQUEST				= 32 /* 0x20 */,
-	HAL_MACRX_CBF_DATA_REQUEST				= 33 /* 0x21 */,
-	HAL_MACRXXPECT_NDP_RECEPTION				= 34 /* 0x22 */,
-	HAL_MACRX_FREEZE_CAPTURE_CHANNEL			= 35 /* 0x23 */,
-	HAL_MACRX_NDP_TIMEOUT					= 36 /* 0x24 */,
-	HAL_MACRX_ABORT_ACK					= 37 /* 0x25 */,
-	HAL_MACRX_REQ_IMPLICIT_FB				= 38 /* 0x26 */,
-	HAL_MACRX_CHAIN_MASK					= 39 /* 0x27 */,
-	HAL_MACRX_NAP_USER					= 40 /* 0x28 */,
-	HAL_MACRX_ABORT_REQUEST					= 41 /* 0x29 */,
-	HAL_PHYTX_OTHER_TRANSMIT_INFO16				= 42 /* 0x2a */,
-	HAL_PHYTX_ABORT_ACK					= 43 /* 0x2b */,
-	HAL_PHYTX_ABORT_REQUEST					= 44 /* 0x2c */,
-	HAL_PHYTX_PKT_END					= 45 /* 0x2d */,
-	HAL_PHYTX_PPDU_HEADER_INFO_REQUEST			= 46 /* 0x2e */,
-	HAL_PHYTX_REQUEST_CTRL_INFO				= 47 /* 0x2f */,
-	HAL_PHYTX_DATA_REQUEST					= 48 /* 0x30 */,
-	HAL_PHYTX_BF_CV_LOADING_DONE				= 49 /* 0x31 */,
-	HAL_PHYTX_NAP_ACK					= 50 /* 0x32 */,
-	HAL_PHYTX_NAP_DONE					= 51 /* 0x33 */,
-	HAL_PHYTX_OFF_ACK					= 52 /* 0x34 */,
-	HAL_PHYTX_ON_ACK					= 53 /* 0x35 */,
-	HAL_PHYTX_SYNTH_OFF_ACK					= 54 /* 0x36 */,
-	HAL_PHYTX_DEBUG16					= 55 /* 0x37 */,
-	HAL_MACTX_ABORT_REQUEST					= 56 /* 0x38 */,
-	HAL_MACTX_ABORT_ACK					= 57 /* 0x39 */,
-	HAL_MACTX_PKT_END					= 58 /* 0x3a */,
-	HAL_MACTX_PRE_PHY_DESC					= 59 /* 0x3b */,
-	HAL_MACTX_BF_PARAMS_COMMON				= 60 /* 0x3c */,
-	HAL_MACTX_BF_PARAMS_PER_USER				= 61 /* 0x3d */,
-	HAL_MACTX_PREFETCH_CV					= 62 /* 0x3e */,
-	HAL_MACTX_USER_DESC_COMMON				= 63 /* 0x3f */,
-	HAL_MACTX_USER_DESC_PER_USER				= 64 /* 0x40 */,
-	HAL_XAMPLE_USER_TLV_16					= 65 /* 0x41 */,
-	HAL_XAMPLE_TLV_16					= 66 /* 0x42 */,
-	HAL_MACTX_PHY_OFF					= 67 /* 0x43 */,
-	HAL_MACTX_PHY_ON					= 68 /* 0x44 */,
-	HAL_MACTX_SYNTH_OFF					= 69 /* 0x45 */,
-	HAL_MACTXXPECT_CBF_COMMON				= 70 /* 0x46 */,
-	HAL_MACTXXPECT_CBF_PER_USER				= 71 /* 0x47 */,
-	HAL_MACTX_PHY_DESC					= 72 /* 0x48 */,
-	HAL_MACTX_L_SIG_A					= 73 /* 0x49 */,
-	HAL_MACTX_L_SIG_B					= 74 /* 0x4a */,
-	HAL_MACTX_HT_SIG					= 75 /* 0x4b */,
-	HAL_MACTX_VHT_SIG_A					= 76 /* 0x4c */,
-	HAL_MACTX_VHT_SIG_B_SU20				= 77 /* 0x4d */,
-	HAL_MACTX_VHT_SIG_B_SU40				= 78 /* 0x4e */,
-	HAL_MACTX_VHT_SIG_B_SU80				= 79 /* 0x4f */,
-	HAL_MACTX_VHT_SIG_B_SU160				= 80 /* 0x50 */,
-	HAL_MACTX_VHT_SIG_B_MU20				= 81 /* 0x51 */,
-	HAL_MACTX_VHT_SIG_B_MU40				= 82 /* 0x52 */,
-	HAL_MACTX_VHT_SIG_B_MU80				= 83 /* 0x53 */,
-	HAL_MACTX_VHT_SIG_B_MU160				= 84 /* 0x54 */,
-	HAL_MACTX_SERVICE					= 85 /* 0x55 */,
-	HAL_MACTX_HE_SIG_A_SU					= 86 /* 0x56 */,
-	HAL_MACTX_HE_SIG_A_MU_DL				= 87 /* 0x57 */,
-	HAL_MACTX_HE_SIG_A_MU_UL				= 88 /* 0x58 */,
-	HAL_MACTX_HE_SIG_B1_MU					= 89 /* 0x59 */,
-	HAL_MACTX_HE_SIG_B2_MU					= 90 /* 0x5a */,
-	HAL_MACTX_HE_SIG_B2_OFDMA				= 91 /* 0x5b */,
-	HAL_MACTX_DELETE_CV					= 92 /* 0x5c */,
-	HAL_MACTX_MU_UPLINK_COMMON				= 93 /* 0x5d */,
-	HAL_MACTX_MU_UPLINK_USER_SETUP				= 94 /* 0x5e */,
-	HAL_MACTX_OTHER_TRANSMIT_INFO				= 95 /* 0x5f */,
-	HAL_MACTX_PHY_NAP					= 96 /* 0x60 */,
-	HAL_MACTX_DEBUG						= 97 /* 0x61 */,
-	HAL_PHYRX_ABORT_ACK					= 98 /* 0x62 */,
-	HAL_PHYRX_GENERATED_CBF_DETAILS				= 99 /* 0x63 */,
-	HAL_PHYRX_RSSI_LEGACY					= 100 /* 0x64 */,
-	HAL_PHYRX_RSSI_HT					= 101 /* 0x65 */,
-	HAL_PHYRX_USER_INFO					= 102 /* 0x66 */,
-	HAL_PHYRX_PKT_END					= 103 /* 0x67 */,
-	HAL_PHYRX_DEBUG						= 104 /* 0x68 */,
-	HAL_PHYRX_CBF_TRANSFER_DONE				= 105 /* 0x69 */,
-	HAL_PHYRX_CBF_TRANSFER_ABORT				= 106 /* 0x6a */,
-	HAL_PHYRX_L_SIG_A					= 107 /* 0x6b */,
-	HAL_PHYRX_L_SIG_B					= 108 /* 0x6c */,
-	HAL_PHYRX_HT_SIG					= 109 /* 0x6d */,
-	HAL_PHYRX_VHT_SIG_A					= 110 /* 0x6e */,
-	HAL_PHYRX_VHT_SIG_B_SU20				= 111 /* 0x6f */,
-	HAL_PHYRX_VHT_SIG_B_SU40				= 112 /* 0x70 */,
-	HAL_PHYRX_VHT_SIG_B_SU80				= 113 /* 0x71 */,
-	HAL_PHYRX_VHT_SIG_B_SU160				= 114 /* 0x72 */,
-	HAL_PHYRX_VHT_SIG_B_MU20				= 115 /* 0x73 */,
-	HAL_PHYRX_VHT_SIG_B_MU40				= 116 /* 0x74 */,
-	HAL_PHYRX_VHT_SIG_B_MU80				= 117 /* 0x75 */,
-	HAL_PHYRX_VHT_SIG_B_MU160				= 118 /* 0x76 */,
-	HAL_PHYRX_HE_SIG_A_SU					= 119 /* 0x77 */,
-	HAL_PHYRX_HE_SIG_A_MU_DL				= 120 /* 0x78 */,
-	HAL_PHYRX_HE_SIG_A_MU_UL				= 121 /* 0x79 */,
-	HAL_PHYRX_HE_SIG_B1_MU					= 122 /* 0x7a */,
-	HAL_PHYRX_HE_SIG_B2_MU					= 123 /* 0x7b */,
-	HAL_PHYRX_HE_SIG_B2_OFDMA				= 124 /* 0x7c */,
-	HAL_PHYRX_OTHER_RECEIVE_INFO				= 125 /* 0x7d */,
-	HAL_PHYRX_COMMON_USER_INFO				= 126 /* 0x7e */,
-	HAL_PHYRX_DATA_DONE					= 127 /* 0x7f */,
-	HAL_COEX_TX_REQ						= 128 /* 0x80 */,
-	HAL_DUMMY						= 129 /* 0x81 */,
-	HALXAMPLE_TLV_32_NAME					= 130 /* 0x82 */,
-	HAL_MPDU_LIMIT						= 131 /* 0x83 */,
-	HAL_NA_LENGTH_END					= 132 /* 0x84 */,
-	HAL_OLE_BUF_STATUS					= 133 /* 0x85 */,
-	HAL_PCU_PPDU_SETUP_DONE					= 134 /* 0x86 */,
-	HAL_PCU_PPDU_SETUP_END					= 135 /* 0x87 */,
-	HAL_PCU_PPDU_SETUP_INIT					= 136 /* 0x88 */,
-	HAL_PCU_PPDU_SETUP_START				= 137 /* 0x89 */,
-	HAL_PDG_FES_SETUP					= 138 /* 0x8a */,
-	HAL_PDG_RESPONSE					= 139 /* 0x8b */,
-	HAL_PDG_TX_REQ						= 140 /* 0x8c */,
-	HAL_SCH_WAIT_INSTR					= 141 /* 0x8d */,
-	HAL_TQM_FLOWMPTY_STATUS					= 143 /* 0x8f */,
-	HAL_TQM_FLOW_NOTMPTY_STATUS				= 144 /* 0x90 */,
-	HAL_TQM_GEN_MPDU_LENGTH_LIST				= 145 /* 0x91 */,
-	HAL_TQM_GEN_MPDU_LENGTH_LIST_STATUS			= 146 /* 0x92 */,
-	HAL_TQM_GEN_MPDUS					= 147 /* 0x93 */,
-	HAL_TQM_GEN_MPDUS_STATUS				= 148 /* 0x94 */,
-	HAL_TQM_REMOVE_MPDU					= 149 /* 0x95 */,
-	HAL_TQM_REMOVE_MPDU_STATUS				= 150 /* 0x96 */,
-	HAL_TQM_REMOVE_MSDU					= 151 /* 0x97 */,
-	HAL_TQM_REMOVE_MSDU_STATUS				= 152 /* 0x98 */,
-	HAL_TQM_UPDATE_TX_MPDU_COUNT				= 153 /* 0x99 */,
-	HAL_TQM_WRITE_CMD					= 154 /* 0x9a */,
-	HAL_OFDMA_TRIGGER_DETAILS				= 155 /* 0x9b */,
-	HAL_TX_DATA						= 156 /* 0x9c */,
-	HAL_TX_FES_SETUP					= 157 /* 0x9d */,
-	HAL_RX_PACKET						= 158 /* 0x9e */,
-	HALXPECTED_RESPONSE					= 159 /* 0x9f */,
-	HAL_TX_MPDU_END						= 160 /* 0xa0 */,
-	HAL_TX_MPDU_START					= 161 /* 0xa1 */,
-	HAL_TX_MSDU_END						= 162 /* 0xa2 */,
-	HAL_TX_MSDU_START					= 163 /* 0xa3 */,
-	HAL_TX_SW_MODE_SETUP					= 164 /* 0xa4 */,
-	HAL_TXPCU_BUFFER_STATUS					= 165 /* 0xa5 */,
-	HAL_TXPCU_USER_BUFFER_STATUS				= 166 /* 0xa6 */,
-	HAL_DATA_TO_TIME_CONFIG					= 167 /* 0xa7 */,
-	HALXAMPLE_USER_TLV_32					= 168 /* 0xa8 */,
-	HAL_MPDU_INFO						= 169 /* 0xa9 */,
-	HAL_PDG_USER_SETUP					= 170 /* 0xaa */,
-	HAL_TX_11AH_SETUP					= 171 /* 0xab */,
-	HAL_REO_UPDATE_RX_REO_QUEUE_STATUS			= 172 /* 0xac */,
-	HAL_TX_PEER_ENTRY					= 173 /* 0xad */,
-	HAL_TX_RAW_OR_NATIVE_FRAME_SETUP			= 174 /* 0xae */,
-	HALXAMPLE_USER_TLV_44					= 175 /* 0xaf */,
-	HAL_TX_FLUSH						= 176 /* 0xb0 */,
-	HAL_TX_FLUSH_REQ					= 177 /* 0xb1 */,
-	HAL_TQM_WRITE_CMD_STATUS				= 178 /* 0xb2 */,
-	HAL_TQM_GET_MPDU_QUEUE_STATS				= 179 /* 0xb3 */,
-	HAL_TQM_GET_MSDU_FLOW_STATS				= 180 /* 0xb4 */,
-	HALXAMPLE_USER_CTLV_44					= 181 /* 0xb5 */,
-	HAL_TX_FES_STATUS_START					= 182 /* 0xb6 */,
-	HAL_TX_FES_STATUS_USER_PPDU				= 183 /* 0xb7 */,
-	HAL_TX_FES_STATUS_USER_RESPONSE				= 184 /* 0xb8 */,
-	HAL_TX_FES_STATUS_END					= 185 /* 0xb9 */,
-	HAL_RX_TRIG_INFO					= 186 /* 0xba */,
-	HAL_RXPCU_TX_SETUP_CLEAR				= 187 /* 0xbb */,
-	HAL_RX_FRAME_BITMAP_REQ					= 188 /* 0xbc */,
-	HAL_RX_FRAME_BITMAP_ACK					= 189 /* 0xbd */,
-	HAL_COEX_RX_STATUS					= 190 /* 0xbe */,
-	HAL_RX_START_PARAM					= 191 /* 0xbf */,
-	HAL_RX_PPDU_START					= 192 /* 0xc0 */,
-	HAL_RX_PPDU_END						= 193 /* 0xc1 */,
-	HAL_RX_MPDU_START					= 194 /* 0xc2 */,
-	HAL_RX_MPDU_END						= 195 /* 0xc3 */,
-	HAL_RX_MSDU_START					= 196 /* 0xc4 */,
-	HAL_RX_MSDU_END						= 197 /* 0xc5 */,
-	HAL_RX_ATTENTION					= 198 /* 0xc6 */,
-	HAL_RECEIVED_RESPONSE_INFO				= 199 /* 0xc7 */,
-	HAL_RX_PHY_SLEEP					= 200 /* 0xc8 */,
-	HAL_RX_HEADER						= 201 /* 0xc9 */,
-	HAL_RX_PEER_ENTRY					= 202 /* 0xca */,
-	HAL_RX_FLUSH						= 203 /* 0xcb */,
-	HAL_RX_RESPONSE_REQUIRED_INFO				= 204 /* 0xcc */,
-	HAL_RX_FRAMELESS_BAR_DETAILS				= 205 /* 0xcd */,
-	HAL_TQM_GET_MPDU_QUEUE_STATS_STATUS			= 206 /* 0xce */,
-	HAL_TQM_GET_MSDU_FLOW_STATS_STATUS			= 207 /* 0xcf */,
-	HAL_TX_CBF_INFO						= 208 /* 0xd0 */,
-	HAL_PCU_PPDU_SETUP_USER					= 209 /* 0xd1 */,
-	HAL_RX_MPDU_PCU_START					= 210 /* 0xd2 */,
-	HAL_RX_PM_INFO						= 211 /* 0xd3 */,
-	HAL_RX_USER_PPDU_END					= 212 /* 0xd4 */,
-	HAL_RX_PRE_PPDU_START					= 213 /* 0xd5 */,
-	HAL_RX_PREAMBLE						= 214 /* 0xd6 */,
-	HAL_TX_FES_SETUP_COMPLETE				= 215 /* 0xd7 */,
-	HAL_TX_LAST_MPDU_FETCHED				= 216 /* 0xd8 */,
-	HAL_TXDMA_STOP_REQUEST					= 217 /* 0xd9 */,
-	HAL_RXPCU_SETUP						= 218 /* 0xda */,
-	HAL_RXPCU_USER_SETUP					= 219 /* 0xdb */,
-	HAL_TX_FES_STATUS_ACK_OR_BA				= 220 /* 0xdc */,
-	HAL_TQM_ACKED_MPDU					= 221 /* 0xdd */,
-	HAL_COEX_TX_RESP					= 222 /* 0xde */,
-	HAL_COEX_TX_STATUS					= 223 /* 0xdf */,
-	HAL_MACTX_COEX_PHY_CTRL					= 224 /* 0xe0 */,
-	HAL_COEX_STATUS_BROADCAST				= 225 /* 0xe1 */,
-	HAL_RESPONSE_START_STATUS				= 226 /* 0xe2 */,
-	HAL_RESPONSEND_STATUS					= 227 /* 0xe3 */,
-	HAL_CRYPTO_STATUS					= 228 /* 0xe4 */,
-	HAL_RECEIVED_TRIGGER_INFO				= 229 /* 0xe5 */,
-	HAL_COEX_TX_STOP_CTRL					= 230 /* 0xe6 */,
-	HAL_RX_PPDU_ACK_REPORT					= 231 /* 0xe7 */,
-	HAL_RX_PPDU_NO_ACK_REPORT				= 232 /* 0xe8 */,
-	HAL_SCH_COEX_STATUS					= 233 /* 0xe9 */,
-	HAL_SCHEDULER_COMMAND_STATUS				= 234 /* 0xea */,
-	HAL_SCHEDULER_RX_PPDU_NO_RESPONSE_STATUS		= 235 /* 0xeb */,
-	HAL_TX_FES_STATUS_PROT					= 236 /* 0xec */,
-	HAL_TX_FES_STATUS_START_PPDU				= 237 /* 0xed */,
-	HAL_TX_FES_STATUS_START_PROT				= 238 /* 0xee */,
-	HAL_TXPCU_PHYTX_DEBUG32					= 239 /* 0xef */,
-	HAL_TXPCU_PHYTX_OTHER_TRANSMIT_INFO32			= 240 /* 0xf0 */,
-	HAL_TX_MPDU_COUNT_TRANSFERND				= 241 /* 0xf1 */,
-	HAL_WHO_ANCHOR_OFFSET					= 242 /* 0xf2 */,
-	HAL_WHO_ANCHOR_VALUE					= 243 /* 0xf3 */,
-	HAL_WHO_CCE_INFO					= 244 /* 0xf4 */,
-	HAL_WHO_COMMIT						= 245 /* 0xf5 */,
-	HAL_WHO_COMMIT_DONE					= 246 /* 0xf6 */,
-	HAL_WHO_FLUSH						= 247 /* 0xf7 */,
-	HAL_WHO_L2_LLC						= 248 /* 0xf8 */,
-	HAL_WHO_L2_PAYLOAD					= 249 /* 0xf9 */,
-	HAL_WHO_L3_CHECKSUM					= 250 /* 0xfa */,
-	HAL_WHO_L3_INFO						= 251 /* 0xfb */,
-	HAL_WHO_L4_CHECKSUM					= 252 /* 0xfc */,
-	HAL_WHO_L4_INFO						= 253 /* 0xfd */,
-	HAL_WHO_MSDU						= 254 /* 0xfe */,
-	HAL_WHO_MSDU_MISC					= 255 /* 0xff */,
-	HAL_WHO_PACKET_DATA					= 256 /* 0x100 */,
-	HAL_WHO_PACKET_HDR					= 257 /* 0x101 */,
-	HAL_WHO_PPDU_END					= 258 /* 0x102 */,
-	HAL_WHO_PPDU_START					= 259 /* 0x103 */,
-	HAL_WHO_TSO						= 260 /* 0x104 */,
-	HAL_WHO_WMAC_HEADER_PV0					= 261 /* 0x105 */,
-	HAL_WHO_WMAC_HEADER_PV1					= 262 /* 0x106 */,
-	HAL_WHO_WMAC_IV						= 263 /* 0x107 */,
-	HAL_MPDU_INFO_END					= 264 /* 0x108 */,
-	HAL_MPDU_INFO_BITMAP					= 265 /* 0x109 */,
-	HAL_TX_QUEUE_EXTENSION					= 266 /* 0x10a */,
-	HAL_SCHEDULER_SELFGEN_RESPONSE_STATUS			= 267 /* 0x10b */,
-	HAL_TQM_UPDATE_TX_MPDU_COUNT_STATUS			= 268 /* 0x10c */,
-	HAL_TQM_ACKED_MPDU_STATUS				= 269 /* 0x10d */,
-	HAL_TQM_ADD_MSDU_STATUS					= 270 /* 0x10e */,
-	HAL_TQM_LIST_GEN_DONE					= 271 /* 0x10f */,
-	HAL_WHO_TERMINATE					= 272 /* 0x110 */,
-	HAL_TX_LAST_MPDU_END					= 273 /* 0x111 */,
-	HAL_TX_CV_DATA						= 274 /* 0x112 */,
-	HAL_PPDU_TX_END						= 275 /* 0x113 */,
-	HAL_PROT_TX_END						= 276 /* 0x114 */,
-	HAL_MPDU_INFO_GLOBAL_END				= 277 /* 0x115 */,
-	HAL_TQM_SCH_INSTR_GLOBAL_END				= 278 /* 0x116 */,
-	HAL_RX_PPDU_END_USER_STATS				= 279 /* 0x117 */,
-	HAL_RX_PPDU_END_USER_STATS_EXT				= 280 /* 0x118 */,
-	HAL_REO_GET_QUEUE_STATS					= 281 /* 0x119 */,
-	HAL_REO_FLUSH_QUEUE					= 282 /* 0x11a */,
-	HAL_REO_FLUSH_CACHE					= 283 /* 0x11b */,
-	HAL_REO_UNBLOCK_CACHE					= 284 /* 0x11c */,
-	HAL_REO_GET_QUEUE_STATS_STATUS				= 285 /* 0x11d */,
-	HAL_REO_FLUSH_QUEUE_STATUS				= 286 /* 0x11e */,
-	HAL_REO_FLUSH_CACHE_STATUS				= 287 /* 0x11f */,
-	HAL_REO_UNBLOCK_CACHE_STATUS				= 288 /* 0x120 */,
-	HAL_TQM_FLUSH_CACHE					= 289 /* 0x121 */,
-	HAL_TQM_UNBLOCK_CACHE					= 290 /* 0x122 */,
-	HAL_TQM_FLUSH_CACHE_STATUS				= 291 /* 0x123 */,
-	HAL_TQM_UNBLOCK_CACHE_STATUS				= 292 /* 0x124 */,
-	HAL_RX_PPDU_END_STATUS_DONE				= 293 /* 0x125 */,
-	HAL_RX_STATUS_BUFFER_DONE				= 294 /* 0x126 */,
-	HAL_TX_DATA_SYNC					= 297 /* 0x129 */,
-	HAL_PHYRX_CBF_READ_REQUEST_ACK				= 298 /* 0x12a */,
-	HAL_TQM_GET_MPDU_HEAD_INFO				= 299 /* 0x12b */,
-	HAL_TQM_SYNC_CMD					= 300 /* 0x12c */,
-	HAL_TQM_GET_MPDU_HEAD_INFO_STATUS			= 301 /* 0x12d */,
-	HAL_TQM_SYNC_CMD_STATUS					= 302 /* 0x12e */,
-	HAL_TQM_THRESHOLD_DROP_NOTIFICATION_STATUS		= 303 /* 0x12f */,
-	HAL_TQM_DESCRIPTOR_THRESHOLD_REACHED_STATUS		= 304 /* 0x130 */,
-	HAL_REO_FLUSH_TIMEOUT_LIST				= 305 /* 0x131 */,
-	HAL_REO_FLUSH_TIMEOUT_LIST_STATUS			= 306 /* 0x132 */,
-	HAL_REO_DESCRIPTOR_THRESHOLD_REACHED_STATUS		= 307 /* 0x133 */,
-	HAL_SCHEDULER_RX_SIFS_RESPONSE_TRIGGER_STATUS		= 308 /* 0x134 */,
-	HALXAMPLE_USER_TLV_32_NAME				= 309 /* 0x135 */,
-	HAL_RX_PPDU_START_USER_INFO				= 310 /* 0x136 */,
-	HAL_RX_RING_MASK					= 311 /* 0x137 */,
-	HAL_COEX_MAC_NAP					= 312 /* 0x138 */,
-	HAL_RXPCU_PPDU_END_INFO					= 313 /* 0x139 */,
-	HAL_WHO_MESH_CONTROL					= 314 /* 0x13a */,
-	HAL_PDG_SW_MODE_BW_START				= 315 /* 0x13b */,
-	HAL_PDG_SW_MODE_BW_END					= 316 /* 0x13c */,
-	HAL_PDG_WAIT_FOR_MAC_REQUEST				= 317 /* 0x13d */,
-	HAL_PDG_WAIT_FOR_PHY_REQUEST				= 318 /* 0x13e */,
-	HAL_SCHEDULER_END					= 319 /* 0x13f */,
-	HAL_RX_PPDU_START_DROPPED				= 320 /* 0x140 */,
-	HAL_RX_PPDU_END_DROPPED					= 321 /* 0x141 */,
-	HAL_RX_PPDU_END_STATUS_DONE_DROPPED			= 322 /* 0x142 */,
-	HAL_RX_MPDU_START_DROPPED				= 323 /* 0x143 */,
-	HAL_RX_MSDU_START_DROPPED				= 324 /* 0x144 */,
-	HAL_RX_MSDU_END_DROPPED					= 325 /* 0x145 */,
-	HAL_RX_MPDU_END_DROPPED					= 326 /* 0x146 */,
-	HAL_RX_ATTENTION_DROPPED				= 327 /* 0x147 */,
-	HAL_TXPCU_USER_SETUP					= 328 /* 0x148 */,
-	HAL_RXPCU_USER_SETUP_EXT				= 329 /* 0x149 */,
-	HAL_CMD_PART_0_END					= 330 /* 0x14a */,
-	HAL_MACTX_SYNTH_ON					= 331 /* 0x14b */,
-	HAL_SCH_CRITICAL_TLV_REFERENCE				= 332 /* 0x14c */,
-	HAL_TQM_MPDU_GLOBAL_START				= 333 /* 0x14d */,
-	HALXAMPLE_TLV_32					= 334 /* 0x14e */,
-	HAL_TQM_UPDATE_TX_MSDU_FLOW				= 335 /* 0x14f */,
-	HAL_TQM_UPDATE_TX_MPDU_QUEUE_HEAD			= 336 /* 0x150 */,
-	HAL_TQM_UPDATE_TX_MSDU_FLOW_STATUS			= 337 /* 0x151 */,
-	HAL_TQM_UPDATE_TX_MPDU_QUEUE_HEAD_STATUS		= 338 /* 0x152 */,
-	HAL_REO_UPDATE_RX_REO_QUEUE				= 339 /* 0x153 */,
-	HAL_TQM_MPDU_QUEUEMPTY_STATUS				= 340 /* 0x154 */,
-	HAL_TQM_2_SCH_MPDU_AVAILABLE				= 341 /* 0x155 */,
-	HAL_PDG_TRIG_RESPONSE					= 342 /* 0x156 */,
-	HAL_TRIGGER_RESPONSE_TX_DONE				= 343 /* 0x157 */,
-	HAL_ABORT_FROM_PHYRX_DETAILS				= 344 /* 0x158 */,
-	HAL_SCH_TQM_CMD_WRAPPER					= 345 /* 0x159 */,
-	HAL_MPDUS_AVAILABLE					= 346 /* 0x15a */,
-	HAL_RECEIVED_RESPONSE_INFO_PART2			= 347 /* 0x15b */,
-	HAL_PHYRX_TX_START_TIMING				= 348 /* 0x15c */,
-	HAL_TXPCU_PREAMBLE_DONE					= 349 /* 0x15d */,
-	HAL_NDP_PREAMBLE_DONE					= 350 /* 0x15e */,
-	HAL_SCH_TQM_CMD_WRAPPER_RBO_DROP			= 351 /* 0x15f */,
-	HAL_SCH_TQM_CMD_WRAPPER_CONT_DROP			= 352 /* 0x160 */,
-	HAL_MACTX_CLEAR_PREV_TX_INFO				= 353 /* 0x161 */,
-	HAL_TX_PUNCTURE_SETUP					= 354 /* 0x162 */,
-	HAL_R2R_STATUS_END					= 355 /* 0x163 */,
-	HAL_MACTX_PREFETCH_CV_COMMON				= 356 /* 0x164 */,
-	HAL_END_OF_FLUSH_MARKER					= 357 /* 0x165 */,
-	HAL_MACTX_MU_UPLINK_COMMON_PUNC				= 358 /* 0x166 */,
-	HAL_MACTX_MU_UPLINK_USER_SETUP_PUNC			= 359 /* 0x167 */,
-	HAL_RECEIVED_RESPONSE_USER_7_0				= 360 /* 0x168 */,
-	HAL_RECEIVED_RESPONSE_USER_15_8				= 361 /* 0x169 */,
-	HAL_RECEIVED_RESPONSE_USER_23_16			= 362 /* 0x16a */,
-	HAL_RECEIVED_RESPONSE_USER_31_24			= 363 /* 0x16b */,
-	HAL_RECEIVED_RESPONSE_USER_36_32			= 364 /* 0x16c */,
-	HAL_TX_LOOPBACK_SETUP					= 365 /* 0x16d */,
-	HAL_PHYRX_OTHER_RECEIVE_INFO_RU_DETAILS			= 366 /* 0x16e */,
-	HAL_SCH_WAIT_INSTR_TX_PATH				= 367 /* 0x16f */,
-	HAL_MACTX_OTHER_TRANSMIT_INFO_TX2TX			= 368 /* 0x170 */,
-	HAL_MACTX_OTHER_TRANSMIT_INFOMUPHY_SETUP		= 369 /* 0x171 */,
-	HAL_PHYRX_OTHER_RECEIVE_INFOVM_DETAILS			= 370 /* 0x172 */,
-	HAL_TX_WUR_DATA						= 371 /* 0x173 */,
-	HAL_RX_PPDU_END_START					= 372 /* 0x174 */,
-	HAL_RX_PPDU_END_MIDDLE					= 373 /* 0x175 */,
-	HAL_RX_PPDU_END_LAST					= 374 /* 0x176 */,
-	HAL_MACTX_BACKOFF_BASED_TRANSMISSION			= 375 /* 0x177 */,
-	HAL_MACTX_OTHER_TRANSMIT_INFO_DL_OFDMA_TX		= 376 /* 0x178 */,
-	HAL_SRP_INFO						= 377 /* 0x179 */,
-	HAL_OBSS_SR_INFO					= 378 /* 0x17a */,
-	HAL_SCHEDULER_SW_MSG_STATUS				= 379 /* 0x17b */,
-	HAL_HWSCH_RXPCU_MAC_INFO_ANNOUNCEMENT			= 380 /* 0x17c */,
-	HAL_RXPCU_SETUP_COMPLETE				= 381 /* 0x17d */,
-	HAL_SNOOP_PPDU_START					= 382 /* 0x17e */,
-	HAL_SNOOP_MPDU_USR_DBG_INFO				= 383 /* 0x17f */,
-	HAL_SNOOP_MSDU_USR_DBG_INFO				= 384 /* 0x180 */,
-	HAL_SNOOP_MSDU_USR_DATA					= 385 /* 0x181 */,
-	HAL_SNOOP_MPDU_USR_STAT_INFO				= 386 /* 0x182 */,
-	HAL_SNOOP_PPDU_END					= 387 /* 0x183 */,
-	HAL_SNOOP_SPARE						= 388 /* 0x184 */,
-	HAL_PHYRX_OTHER_RECEIVE_INFO_MU_RSSI_COMMON		= 390 /* 0x186 */,
-	HAL_PHYRX_OTHER_RECEIVE_INFO_MU_RSSI_USER		= 391 /* 0x187 */,
-	HAL_MACTX_OTHER_TRANSMIT_INFO_SCH_DETAILS		= 392 /* 0x188 */,
-	HAL_PHYRX_OTHER_RECEIVE_INFO_108PVM_DETAILS		= 393 /* 0x189 */,
-	HAL_SCH_TLV_WRAPPER					= 394 /* 0x18a */,
-	HAL_SCHEDULER_STATUS_WRAPPER				= 395 /* 0x18b */,
-	HAL_MPDU_INFO_6X					= 396 /* 0x18c */,
-	HAL_MACTX_11AZ_USER_DESC_PER_USER			= 397 /* 0x18d */,
-	HAL_MACTX_U_SIGHT_SU_MU					= 398 /* 0x18e */,
-	HAL_MACTX_U_SIGHT_TB					= 399 /* 0x18f */,
-	HAL_PHYRX_U_SIGHT_SU_MU					= 403 /* 0x193 */,
-	HAL_PHYRX_U_SIGHT_TB					= 404 /* 0x194 */,
-	HAL_MACRX_LMR_READ_REQUEST				= 408 /* 0x198 */,
-	HAL_MACRX_LMR_DATA_REQUEST				= 409 /* 0x199 */,
-	HAL_PHYRX_LMR_TRANSFER_DONE				= 410 /* 0x19a */,
-	HAL_PHYRX_LMR_TRANSFER_ABORT				= 411 /* 0x19b */,
-	HAL_PHYRX_LMR_READ_REQUEST_ACK				= 412 /* 0x19c */,
-	HAL_MACRX_SECURE_LTF_SEQ_PTR				= 413 /* 0x19d */,
-	HAL_PHYRX_USER_INFO_MU_UL				= 414 /* 0x19e */,
-	HAL_MPDU_QUEUE_OVERVIEW					= 415 /* 0x19f */,
-	HAL_SCHEDULER_NAV_INFO					= 416 /* 0x1a0 */,
-	HAL_LMR_PEER_ENTRY					= 418 /* 0x1a2 */,
-	HAL_LMR_MPDU_START					= 419 /* 0x1a3 */,
-	HAL_LMR_DATA						= 420 /* 0x1a4 */,
-	HAL_LMR_MPDU_END					= 421 /* 0x1a5 */,
-	HAL_REO_GET_QUEUE_1K_STATS_STATUS			= 422 /* 0x1a6 */,
-	HAL_RX_FRAME_1K_BITMAP_ACK				= 423 /* 0x1a7 */,
-	HAL_TX_FES_STATUS_1K_BA					= 424 /* 0x1a8 */,
-	HAL_TQM_ACKED_1K_MPDU					= 425 /* 0x1a9 */,
-	HAL_MACRX_INBSS_OBSS_IND				= 426 /* 0x1aa */,
-	HAL_PHYRX_LOCATION					= 427 /* 0x1ab */,
-	HAL_MLO_TX_NOTIFICATION_SU				= 428 /* 0x1ac */,
-	HAL_MLO_TX_NOTIFICATION_MU				= 429 /* 0x1ad */,
-	HAL_MLO_TX_REQ_SU					= 430 /* 0x1ae */,
-	HAL_MLO_TX_REQ_MU					= 431 /* 0x1af */,
-	HAL_MLO_TX_RESP						= 432 /* 0x1b0 */,
-	HAL_MLO_RX_NOTIFICATION					= 433 /* 0x1b1 */,
-	HAL_MLO_BKOFF_TRUNC_REQ					= 434 /* 0x1b2 */,
-	HAL_MLO_TBTT_NOTIFICATION				= 435 /* 0x1b3 */,
-	HAL_MLO_MESSAGE						= 436 /* 0x1b4 */,
-	HAL_MLO_TS_SYNC_MSG					= 437 /* 0x1b5 */,
-	HAL_MLO_FES_SETUP					= 438 /* 0x1b6 */,
-	HAL_MLO_PDG_FES_SETUP_SU				= 439 /* 0x1b7 */,
-	HAL_MLO_PDG_FES_SETUP_MU				= 440 /* 0x1b8 */,
-	HAL_MPDU_INFO_1K_BITMAP					= 441 /* 0x1b9 */,
-	HAL_MON_BUF_ADDR					= 442 /* 0x1ba */,
-	HAL_TX_FRAG_STATE					= 443 /* 0x1bb */,
-	HAL_MACTXHT_SIG_USR_OFDMA				= 446 /* 0x1be */,
-	HAL_PHYRXHT_SIG_CMN_PUNC				= 448 /* 0x1c0 */,
-	HAL_PHYRXHT_SIG_CMN_OFDMA				= 450 /* 0x1c2 */,
-	HAL_PHYRXHT_SIG_USR_OFDMA				= 454 /* 0x1c6 */,
-	HAL_PHYRX_PKT_END_PART1					= 456 /* 0x1c8 */,
-	HAL_MACTXXPECT_NDP_RECEPTION				= 457 /* 0x1c9 */,
-	HAL_MACTX_SECURE_LTF_SEQ_PTR				= 458 /* 0x1ca */,
-	HAL_MLO_PDG_BKOFF_TRUNC_NOTIFY				= 460 /* 0x1cc */,
-	HAL_PHYRX_11AZ_INTEGRITY_DATA				= 461 /* 0x1cd */,
-	HAL_PHYTX_LOCATION					= 462 /* 0x1ce */,
-	HAL_PHYTX_11AZ_INTEGRITY_DATA				= 463 /* 0x1cf */,
-	HAL_MACTXHT_SIG_USR_SU					= 466 /* 0x1d2 */,
-	HAL_MACTXHT_SIG_USR_MU_MIMO				= 467 /* 0x1d3 */,
-	HAL_PHYRXHT_SIG_USR_SU					= 468 /* 0x1d4 */,
-	HAL_PHYRXHT_SIG_USR_MU_MIMO				= 469 /* 0x1d5 */,
-	HAL_PHYRX_GENERIC_U_SIG					= 470 /* 0x1d6 */,
-	HAL_PHYRX_GENERIC_EHT_SIG				= 471 /* 0x1d7 */,
-	HAL_OVERWRITE_RESP_START				= 472 /* 0x1d8 */,
-	HAL_OVERWRITE_RESP_PREAMBLE_INFO			= 473 /* 0x1d9 */,
-	HAL_OVERWRITE_RESP_FRAME_INFO				= 474 /* 0x1da */,
-	HAL_OVERWRITE_RESP_END					= 475 /* 0x1db */,
-	HAL_RXPCUARLY_RX_INDICATION				= 476 /* 0x1dc */,
-	HAL_MON_DROP						= 477 /* 0x1dd */,
-	HAL_MACRX_MU_UPLINK_COMMON_SNIFF			= 478 /* 0x1de */,
-	HAL_MACRX_MU_UPLINK_USER_SETUP_SNIFF			= 479 /* 0x1df */,
-	HAL_MACRX_MU_UPLINK_USER_SEL_SNIFF			= 480 /* 0x1e0 */,
-	HAL_MACRX_MU_UPLINK_FCS_STATUS_SNIFF			= 481 /* 0x1e1 */,
-	HAL_MACTX_PREFETCH_CV_DMA				= 482 /* 0x1e2 */,
-	HAL_MACTX_PREFETCH_CV_PER_USER				= 483 /* 0x1e3 */,
-	HAL_PHYRX_OTHER_RECEIVE_INFO_ALL_SIGB_DETAILS		= 484 /* 0x1e4 */,
-	HAL_MACTX_BF_PARAMS_UPDATE_COMMON			= 485 /* 0x1e5 */,
-	HAL_MACTX_BF_PARAMS_UPDATE_PER_USER			= 486 /* 0x1e6 */,
-	HAL_RANGING_USER_DETAILS				= 487 /* 0x1e7 */,
-	HAL_PHYTX_CV_CORR_STATUS				= 488 /* 0x1e8 */,
-	HAL_PHYTX_CV_CORR_COMMON				= 489 /* 0x1e9 */,
-	HAL_PHYTX_CV_CORR_USER					= 490 /* 0x1ea */,
-	HAL_MACTX_CV_CORR_COMMON				= 491 /* 0x1eb */,
-	HAL_MACTX_CV_CORR_MAC_INFO_GROUP			= 492 /* 0x1ec */,
-	HAL_BW_PUNCTUREVAL_WRAPPER				= 493 /* 0x1ed */,
-	HAL_MACTX_RX_NOTIFICATION_FOR_PHY			= 494 /* 0x1ee */,
-	HAL_MACTX_TX_NOTIFICATION_FOR_PHY			= 495 /* 0x1ef */,
-	HAL_MACTX_MU_UPLINK_COMMON_PER_BW			= 496 /* 0x1f0 */,
-	HAL_MACTX_MU_UPLINK_USER_SETUP_PER_BW			= 497 /* 0x1f1 */,
-	HAL_RX_PPDU_END_USER_STATS_EXT2				= 498 /* 0x1f2 */,
-	HAL_FW2SW_MON						= 499 /* 0x1f3 */,
-	HAL_WSI_DIRECT_MESSAGE					= 500 /* 0x1f4 */,
-	HAL_MACTXMLSR_PRE_SWITCH				= 501 /* 0x1f5 */,
-	HAL_MACTXMLSR_SWITCH					= 502 /* 0x1f6 */,
-	HAL_MACTXMLSR_SWITCH_BACK				= 503 /* 0x1f7 */,
-	HAL_PHYTXMLSR_SWITCH_ACK				= 504 /* 0x1f8 */,
-	HAL_PHYTXMLSR_SWITCH_BACK_ACK				= 505 /* 0x1f9 */,
-	HAL_SPARE_REUSE_TAG_0					= 506 /* 0x1fa */,
-	HAL_SPARE_REUSE_TAG_1					= 507 /* 0x1fb */,
-	HAL_SPARE_REUSE_TAG_2					= 508 /* 0x1fc */,
-	HAL_SPARE_REUSE_TAG_3					= 509 /* 0x1fd */,
-	/* FIXME: Assign correct value for HAL_TCL_DATA_CMD */
-	HAL_TCL_DATA_CMD					= 510,
-	HAL_TLV_BASE						= 511 /* 0x1ff */,
-};
-
-#define HAL_TLV_HDR_TAG		GENMASK(9, 1)
-#define HAL_TLV_HDR_LEN		GENMASK(25, 10)
-#define HAL_TLV_USR_ID          GENMASK(31, 26)
-
-#define HAL_TLV_ALIGN	4
-
-struct hal_tlv_hdr {
-	__le32 tl;
-	u8 value[];
-} __packed;
-
-#define HAL_TLV_64_HDR_TAG		GENMASK(9, 1)
-#define HAL_TLV_64_HDR_LEN		GENMASK(21, 10)
-#define HAL_TLV_64_USR_ID		GENMASK(31, 26)
-#define HAL_TLV_64_ALIGN		8
-
-struct hal_tlv_64_hdr {
-	__le64 tl;
-	u8 value[];
-} __packed;
-
-#define RX_MPDU_DESC_INFO0_MSDU_COUNT		GENMASK(7, 0)
-#define RX_MPDU_DESC_INFO0_FRAG_FLAG		BIT(8)
-#define RX_MPDU_DESC_INFO0_MPDU_RETRY		BIT(9)
-#define RX_MPDU_DESC_INFO0_AMPDU_FLAG		BIT(10)
-#define RX_MPDU_DESC_INFO0_BAR_FRAME		BIT(11)
-#define RX_MPDU_DESC_INFO0_VALID_PN		BIT(12)
-#define RX_MPDU_DESC_INFO0_RAW_MPDU		BIT(13)
-#define RX_MPDU_DESC_INFO0_MORE_FRAG_FLAG	BIT(14)
-#define RX_MPDU_DESC_INFO0_SRC_INFO		GENMASK(26, 15)
-#define RX_MPDU_DESC_INFO0_MPDU_QOS_CTRL_VALID	BIT(27)
-#define RX_MPDU_DESC_INFO0_TID			GENMASK(31, 28)
-
-/* Peer Metadata classification */
-
-/* Version 0 */
-#define RX_MPDU_DESC_META_DATA_V0_PEER_ID	GENMASK(15, 0)
-#define RX_MPDU_DESC_META_DATA_V0_VDEV_ID	GENMASK(23, 16)
-
-/* Version 1 */
-#define RX_MPDU_DESC_META_DATA_V1_PEER_ID		GENMASK(13, 0)
-#define RX_MPDU_DESC_META_DATA_V1_LOGICAL_LINK_ID	GENMASK(15, 14)
-#define RX_MPDU_DESC_META_DATA_V1_VDEV_ID		GENMASK(23, 16)
-#define RX_MPDU_DESC_META_DATA_V1_LMAC_ID		GENMASK(25, 24)
-#define RX_MPDU_DESC_META_DATA_V1_DEVICE_ID		GENMASK(28, 26)
-
-/* Version 1A */
-#define RX_MPDU_DESC_META_DATA_V1A_PEER_ID		GENMASK(13, 0)
-#define RX_MPDU_DESC_META_DATA_V1A_VDEV_ID		GENMASK(21, 14)
-#define RX_MPDU_DESC_META_DATA_V1A_LOGICAL_LINK_ID	GENMASK(25, 22)
-#define RX_MPDU_DESC_META_DATA_V1A_DEVICE_ID		GENMASK(28, 26)
-
-/* Version 1B */
-#define RX_MPDU_DESC_META_DATA_V1B_PEER_ID	GENMASK(13, 0)
-#define RX_MPDU_DESC_META_DATA_V1B_VDEV_ID	GENMASK(21, 14)
-#define RX_MPDU_DESC_META_DATA_V1B_HW_LINK_ID	GENMASK(25, 22)
-#define RX_MPDU_DESC_META_DATA_V1B_DEVICE_ID	GENMASK(28, 26)
-
-struct rx_mpdu_desc {
-	__le32 info0; /* %RX_MPDU_DESC_INFO */
-	__le32 peer_meta_data;
-} __packed;
-
-/* rx_mpdu_desc
- *		Producer: RXDMA
- *		Consumer: REO/SW/FW
- *
- * msdu_count
- *		The number of MSDUs within the MPDU
- *
- * fragment_flag
- *		When set, this MPDU is a fragment and REO should forward this
- *		fragment MPDU to the REO destination ring without any reorder
- *		checks, pn checks or bitmap update. This implies that REO is
- *		forwarding the pointer to the MSDU link descriptor.
- *
- * mpdu_retry_bit
- *		The retry bit setting from the MPDU header of the received frame
- *
- * ampdu_flag
- *		Indicates the MPDU was received as part of an A-MPDU.
- *
- * bar_frame
- *		Indicates the received frame is a BAR frame. After processing,
- *		this frame shall be pushed to SW or deleted.
- *
- * valid_pn
- *		When not set, REO will not perform a PN sequence number check.
- *
- * raw_mpdu
- *		Field only valid when first_msdu_in_mpdu_flag is set. Indicates
- *		the contents in the MSDU buffer contains a 'RAW' MPDU. This
- *		'RAW' MPDU might be spread out over multiple MSDU buffers.
- *
- * more_fragment_flag
- *		The More Fragment bit setting from the MPDU header of the
- *		received frame
- *
- * src_info
- *		Source (Virtual) device/interface info associated with this peer.
- *		This field gets passed on by REO to PPE in the EDMA descriptor.
- *
- * mpdu_qos_control_valid
- *		When set, the MPDU has a QoS control field
- *
- * tid
- *		Field only valid when mpdu_qos_control_valid is set
- */
-
-enum hal_rx_msdu_desc_reo_dest_ind {
-	HAL_RX_MSDU_DESC_REO_DEST_IND_TCL,
-	HAL_RX_MSDU_DESC_REO_DEST_IND_SW1,
-	HAL_RX_MSDU_DESC_REO_DEST_IND_SW2,
-	HAL_RX_MSDU_DESC_REO_DEST_IND_SW3,
-	HAL_RX_MSDU_DESC_REO_DEST_IND_SW4,
-	HAL_RX_MSDU_DESC_REO_DEST_IND_RELEASE,
-	HAL_RX_MSDU_DESC_REO_DEST_IND_FW,
-	HAL_RX_MSDU_DESC_REO_DEST_IND_SW5,
-	HAL_RX_MSDU_DESC_REO_DEST_IND_SW6,
-	HAL_RX_MSDU_DESC_REO_DEST_IND_SW7,
-	HAL_RX_MSDU_DESC_REO_DEST_IND_SW8,
-};
-
-#define RX_MSDU_DESC_INFO0_FIRST_MSDU_IN_MPDU	BIT(0)
-#define RX_MSDU_DESC_INFO0_LAST_MSDU_IN_MPDU	BIT(1)
-#define RX_MSDU_DESC_INFO0_MSDU_CONTINUATION	BIT(2)
-#define RX_MSDU_DESC_INFO0_MSDU_LENGTH		GENMASK(16, 3)
-#define RX_MSDU_DESC_INFO0_MSDU_DROP		BIT(17)
-#define RX_MSDU_DESC_INFO0_VALID_SA		BIT(18)
-#define RX_MSDU_DESC_INFO0_VALID_DA		BIT(19)
-#define RX_MSDU_DESC_INFO0_DA_MCBC		BIT(20)
-#define RX_MSDU_DESC_INFO0_L3_HDR_PAD_MSB	BIT(21)
-#define RX_MSDU_DESC_INFO0_TCP_UDP_CHKSUM_FAIL	BIT(22)
-#define RX_MSDU_DESC_INFO0_IP_CHKSUM_FAIL	BIT(23)
-#define RX_MSDU_DESC_INFO0_FROM_DS		BIT(24)
-#define RX_MSDU_DESC_INFO0_TO_DS		BIT(25)
-#define RX_MSDU_DESC_INFO0_INTRA_BSS		BIT(26)
-#define RX_MSDU_DESC_INFO0_DST_CHIP_ID		GENMASK(28, 27)
-#define RX_MSDU_DESC_INFO0_DECAP_FORMAT		GENMASK(30, 29)
-
-#define HAL_RX_MSDU_PKT_LENGTH_GET(val)		\
-	(le32_get_bits((val), RX_MSDU_DESC_INFO0_MSDU_LENGTH))
-
-struct rx_msdu_desc {
-	__le32 info0;
-} __packed;
-
-/* rx_msdu_desc
- *
- * first_msdu_in_mpdu
- *		Indicates first msdu in mpdu.
- *
- * last_msdu_in_mpdu
- *		Indicates last msdu in mpdu. This flag can be true only when
- *		'Msdu_continuation' set to 0. This implies that when an msdu
- *		is spread out over multiple buffers and thus msdu_continuation
- *		is set, only for the very last buffer of the msdu, can the
- *		'last_msdu_in_mpdu' be set.
- *
- *		When both first_msdu_in_mpdu and last_msdu_in_mpdu are set,
- *		the MPDU that this MSDU belongs to only contains a single MSDU.
- *
- * msdu_continuation
- *		When set, this MSDU buffer was not able to hold the entire MSDU.
- *		The next buffer will therefore contain additional information
- *		related to this MSDU.
- *
- * msdu_length
- *		Field is only valid in combination with the 'first_msdu_in_mpdu'
- *		being set. Full MSDU length in bytes after decapsulation. This
- *		field is still valid for MPDU frames without A-MSDU. It still
- *		represents MSDU length after decapsulation Or in case of RAW
- *		MPDUs, it indicates the length of the entire MPDU (without FCS
- *		field).
- *
- * msdu_drop
- *		Indicates that REO shall drop this MSDU and not forward it to
- *		any other ring.
- *
- * valid_sa
- *		Indicates OLE found a valid SA entry for this MSDU.
- *
- * valid_da
- *		When set, OLE found a valid DA entry for this MSDU.
- *
- * da_mcbc
- *		Field Only valid if valid_da is set. Indicates the DA address
- *		is a Multicast or Broadcast address for this MSDU.
- *
- * l3_header_padding_msb
- *		Passed on from 'RX_MSDU_END' TLV (only the MSB is reported as
- *		the LSB is always zero). Number of bytes padded to make sure
- *		that the L3 header will always start of a Dword boundary
- *
- * tcp_udp_checksum_fail
- *		Passed on from 'RX_ATTENTION' TLV
- *		Indicates that the computed checksum did not match the checksum
- *		in the TCP/UDP header.
- *
- * ip_checksum_fail
- *		Passed on from 'RX_ATTENTION' TLV
- *		Indicates that the computed checksum did not match the checksum
- *		in the IP header.
- *
- * from_DS
- *		Set if the 'from DS' bit is set in the frame control.
- *
- * to_DS
- *		Set if the 'to DS' bit is set in the frame control.
- *
- * intra_bss
- *		This packet needs intra-BSS routing by SW as the 'vdev_id'
- *		for the destination is the same as the 'vdev_id' that this
- *		MSDU was got in.
- *
- * dest_chip_id
- *		If intra_bss is set, copied by RXOLE/RXDMA from 'ADDR_SEARCH_ENTRY'
- *		to support intra-BSS routing with multi-chip multi-link operation.
- *		This indicates into which chip's TCL the packet should be queued.
- *
- * decap_format
- *		Indicates the format after decapsulation:
- */
-
-#define RX_MSDU_EXT_DESC_INFO0_REO_DEST_IND	GENMASK(4, 0)
-#define RX_MSDU_EXT_DESC_INFO0_SERVICE_CODE	GENMASK(13, 5)
-#define RX_MSDU_EXT_DESC_INFO0_PRIORITY_VALID	BIT(14)
-#define RX_MSDU_EXT_DESC_INFO0_DATA_OFFSET	GENMASK(26, 15)
-#define RX_MSDU_EXT_DESC_INFO0_SRC_LINK_ID	GENMASK(29, 27)
-
-struct rx_msdu_ext_desc {
-	__le32 info0;
-} __packed;
-
-/* rx_msdu_ext_desc
- *
- * reo_destination_indication
- *		The ID of the REO exit ring where the MSDU frame shall push
- *		after (MPDU level) reordering has finished.
- *
- * service_code
- *		Opaque service code between PPE and Wi-Fi
- *
- * priority_valid
- *
- * data_offset
- *		The offset to Rx packet data within the buffer (including
- *		Rx DMA offset programming and L3 header padding inserted
- *		by Rx OLE).
- *
- * src_link_id
- *		Set to the link ID of the PMAC that received the frame
- */
-
-enum hal_reo_dest_ring_buffer_type {
-	HAL_REO_DEST_RING_BUFFER_TYPE_MSDU,
-	HAL_REO_DEST_RING_BUFFER_TYPE_LINK_DESC,
-};
-
-enum hal_reo_dest_ring_push_reason {
-	HAL_REO_DEST_RING_PUSH_REASON_ERR_DETECTED,
-	HAL_REO_DEST_RING_PUSH_REASON_ROUTING_INSTRUCTION,
-};
-
-enum hal_reo_dest_ring_error_code {
-	HAL_REO_DEST_RING_ERROR_CODE_DESC_ADDR_ZERO,
-	HAL_REO_DEST_RING_ERROR_CODE_DESC_INVALID,
-	HAL_REO_DEST_RING_ERROR_CODE_AMPDU_IN_NON_BA,
-	HAL_REO_DEST_RING_ERROR_CODE_NON_BA_DUPLICATE,
-	HAL_REO_DEST_RING_ERROR_CODE_BA_DUPLICATE,
-	HAL_REO_DEST_RING_ERROR_CODE_FRAME_2K_JUMP,
-	HAL_REO_DEST_RING_ERROR_CODE_BAR_2K_JUMP,
-	HAL_REO_DEST_RING_ERROR_CODE_FRAME_OOR,
-	HAL_REO_DEST_RING_ERROR_CODE_BAR_OOR,
-	HAL_REO_DEST_RING_ERROR_CODE_NO_BA_SESSION,
-	HAL_REO_DEST_RING_ERROR_CODE_FRAME_SN_EQUALS_SSN,
-	HAL_REO_DEST_RING_ERROR_CODE_PN_CHECK_FAILED,
-	HAL_REO_DEST_RING_ERROR_CODE_2K_ERR_FLAG_SET,
-	HAL_REO_DEST_RING_ERROR_CODE_PN_ERR_FLAG_SET,
-	HAL_REO_DEST_RING_ERROR_CODE_DESC_BLOCKED,
-	HAL_REO_DEST_RING_ERROR_CODE_MAX,
-};
-
-#define HAL_REO_DEST_RING_INFO0_BUFFER_TYPE		BIT(0)
-#define HAL_REO_DEST_RING_INFO0_PUSH_REASON		GENMASK(2, 1)
-#define HAL_REO_DEST_RING_INFO0_ERROR_CODE		GENMASK(7, 3)
-#define HAL_REO_DEST_RING_INFO0_MSDU_DATA_SIZE		GENMASK(11, 8)
-#define HAL_REO_DEST_RING_INFO0_SW_EXCEPTION		BIT(12)
-#define HAL_REO_DEST_RING_INFO0_SRC_LINK_ID		GENMASK(15, 13)
-#define HAL_REO_DEST_RING_INFO0_SIGNATURE		GENMASK(19, 16)
-#define HAL_REO_DEST_RING_INFO0_RING_ID			GENMASK(27, 20)
-#define HAL_REO_DEST_RING_INFO0_LOOPING_COUNT		GENMASK(31, 28)
-
-struct hal_reo_dest_ring {
-	struct ath12k_buffer_addr buf_addr_info;
-	struct rx_mpdu_desc rx_mpdu_info;
-	struct rx_msdu_desc rx_msdu_info;
-	__le32 buf_va_lo;
-	__le32 buf_va_hi;
-	__le32 info0; /* %HAL_REO_DEST_RING_INFO0_ */
-} __packed;
-
-/* hal_reo_dest_ring
- *
- *		Producer: RXDMA
- *		Consumer: REO/SW/FW
- *
- * buf_addr_info
- *		Details of the physical address of a buffer or MSDU
- *		link descriptor.
- *
- * rx_mpdu_info
- *		General information related to the MPDU that is passed
- *		on from REO entrance ring to the REO destination ring.
- *
- * rx_msdu_info
- *		General information related to the MSDU that is passed
- *		on from RXDMA all the way to the REO destination ring.
- *
- * buf_va_lo
- *		Field only valid if Reo_dest_buffer_type is set to MSDU_buf_address
- *		Lower 32 bits of the 64-bit virtual address corresponding
- *		to Buf_or_link_desc_addr_info
- *
- * buf_va_hi
- *		Address (upper 32 bits) of the REO queue descriptor.
- *		Upper 32 bits of the 64-bit virtual address corresponding
- *		to Buf_or_link_desc_addr_info
- *
- * buffer_type
- *		Indicates the type of address provided in the buf_addr_info.
- *		Values are defined in enum %HAL_REO_DEST_RING_BUFFER_TYPE_.
- *
- * push_reason
- *		Reason for pushing this frame to this exit ring. Values are
- *		defined in enum %HAL_REO_DEST_RING_PUSH_REASON_.
- *
- * error_code
- *		Valid only when 'push_reason' is set. All error codes are
- *		defined in enum %HAL_REO_DEST_RING_ERROR_CODE_.
- *
- * captured_msdu_data_size
- *		The number of following REO_DESTINATION STRUCTs that have
- *		been replaced with msdu_data extracted from the msdu_buffer
- *		and copied into the ring for easy FW/SW access.
- *
- * sw_exception
- *		This field has the same setting as the SW_exception field
- *		in the corresponding REO_entrance_ring descriptor.
- *		When set, the REO entrance descriptor is generated by FW,
- *		and the MPDU was processed in the following way:
- *		- NO re-order function is needed.
- *		- MPDU delinking is determined by the setting of Entrance
- *		  ring field: SW_excection_mpdu_delink
- *		- Destination ring selection is based on the setting of
- *		  the Entrance ring field SW_exception_destination _ring_valid
- *
- * src_link_id
- *		Set to the link ID of the PMAC that received the frame
- *
- * signature
- *		Set to value 0x8 when msdu capture mode is enabled for this ring
- *
- * ring_id
- *		The buffer pointer ring id.
- *		0 - Idle ring
- *		1 - N refers to other rings.
- *
- * looping_count
- *		Indicates the number of times the producer of entries into
- *		this ring has looped around the ring.
- */
-
-#define HAL_REO_TO_PPE_RING_INFO0_DATA_LENGTH	GENMASK(15, 0)
-#define HAL_REO_TO_PPE_RING_INFO0_DATA_OFFSET	GENMASK(23, 16)
-#define HAL_REO_TO_PPE_RING_INFO0_POOL_ID	GENMASK(28, 24)
-#define HAL_REO_TO_PPE_RING_INFO0_PREHEADER	BIT(29)
-#define HAL_REO_TO_PPE_RING_INFO0_TSO_EN	BIT(30)
-#define HAL_REO_TO_PPE_RING_INFO0_MORE	BIT(31)
-
-struct hal_reo_to_ppe_ring {
-	__le32 buffer_addr;
-	__le32 info0; /* %HAL_REO_TO_PPE_RING_INFO0_ */
-} __packed;
-
-/* hal_reo_to_ppe_ring
- *
- *		Producer: REO
- *		Consumer: PPE
- *
- * buf_addr_info
- *		Details of the physical address of a buffer or MSDU
- *		link descriptor.
- *
- * data_length
- *		Length of valid data in bytes
- *
- * data_offset
- *		Offset to the data from buffer pointer. Can be used to
- *		strip header in the data for tunnel termination etc.
- *
- * pool_id
- *		REO has global configuration register for this field.
- *		It may have several free buffer pools, each
- *		RX-Descriptor ring can fetch free buffer from specific
- *		buffer pool; pool id will indicate which pool the buffer
- *		will be released to; POOL_ID Zero returned to SW
- *
- * preheader
- *		Disabled: 0 (Default)
- *		Enabled: 1
- *
- * tso_en
- *		Disabled: 0 (Default)
- *		Enabled: 1
- *
- * more
- *		More Segments followed
- */
-
-enum hal_reo_entr_rxdma_push_reason {
-	HAL_REO_ENTR_RING_RXDMA_PUSH_REASON_ERR_DETECTED,
-	HAL_REO_ENTR_RING_RXDMA_PUSH_REASON_ROUTING_INSTRUCTION,
-	HAL_REO_ENTR_RING_RXDMA_PUSH_REASON_RX_FLUSH,
-};
-
-enum hal_reo_entr_rxdma_ecode {
-	HAL_REO_ENTR_RING_RXDMA_ECODE_OVERFLOW_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_MPDU_LEN_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_FCS_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_DECRYPT_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_TKIP_MIC_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_UNECRYPTED_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_MSDU_LEN_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_MSDU_LIMIT_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_WIFI_PARSE_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_AMSDU_PARSE_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_SA_TIMEOUT_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_DA_TIMEOUT_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_FLOW_TIMEOUT_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_FLUSH_REQUEST_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_AMSDU_FRAG_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_MULTICAST_ECHO_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_AMSDU_MISMATCH_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_UNAUTH_WDS_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_GRPCAST_AMSDU_WDS_ERR,
-	HAL_REO_ENTR_RING_RXDMA_ECODE_MAX,
-};
-
-enum hal_rx_reo_dest_ring {
-	HAL_RX_REO_DEST_RING_TCL,
-	HAL_RX_REO_DEST_RING_SW1,
-	HAL_RX_REO_DEST_RING_SW2,
-	HAL_RX_REO_DEST_RING_SW3,
-	HAL_RX_REO_DEST_RING_SW4,
-	HAL_RX_REO_DEST_RING_RELEASE,
-	HAL_RX_REO_DEST_RING_FW,
-	HAL_RX_REO_DEST_RING_SW5,
-	HAL_RX_REO_DEST_RING_SW6,
-	HAL_RX_REO_DEST_RING_SW7,
-	HAL_RX_REO_DEST_RING_SW8,
-};
-
-#define HAL_REO_ENTR_RING_INFO0_QUEUE_ADDR_HI		GENMASK(7, 0)
-#define HAL_REO_ENTR_RING_INFO0_MPDU_BYTE_COUNT		GENMASK(21, 8)
-#define HAL_REO_ENTR_RING_INFO0_DEST_IND		GENMASK(26, 22)
-#define HAL_REO_ENTR_RING_INFO0_FRAMELESS_BAR		BIT(27)
-
-#define HAL_REO_ENTR_RING_INFO1_RXDMA_PUSH_REASON	GENMASK(1, 0)
-#define HAL_REO_ENTR_RING_INFO1_RXDMA_ERROR_CODE	GENMASK(6, 2)
-#define HAL_REO_ENTR_RING_INFO1_MPDU_FRAG_NUM		GENMASK(10, 7)
-#define HAL_REO_ENTR_RING_INFO1_SW_EXCEPTION		BIT(11)
-#define HAL_REO_ENTR_RING_INFO1_SW_EXCEPT_MPDU_DELINK	BIT(12)
-#define HAL_REO_ENTR_RING_INFO1_SW_EXCEPTION_RING_VLD	BIT(13)
-#define HAL_REO_ENTR_RING_INFO1_SW_EXCEPTION_RING	GENMASK(18, 14)
-#define HAL_REO_ENTR_RING_INFO1_MPDU_SEQ_NUM		GENMASK(30, 19)
-
-#define HAL_REO_ENTR_RING_INFO2_PHY_PPDU_ID		GENMASK(15, 0)
-#define HAL_REO_ENTR_RING_INFO2_SRC_LINK_ID		GENMASK(18, 16)
-#define HAL_REO_ENTR_RING_INFO2_RING_ID			GENMASK(27, 20)
-#define HAL_REO_ENTR_RING_INFO2_LOOPING_COUNT		GENMASK(31, 28)
-
-struct hal_reo_entrance_ring {
-	struct ath12k_buffer_addr buf_addr_info;
-	struct rx_mpdu_desc rx_mpdu_info;
-	__le32 queue_addr_lo;
-	__le32 info0; /* %HAL_REO_ENTR_RING_INFO0_ */
-	__le32 info1; /* %HAL_REO_ENTR_RING_INFO1_ */
-	__le32 info2; /* %HAL_REO_DEST_RING_INFO2_ */
-
-} __packed;
-
-/* hal_reo_entrance_ring
- *
- *		Producer: RXDMA
- *		Consumer: REO
- *
- * buf_addr_info
- *		Details of the physical address of a buffer or MSDU
- *		link descriptor.
- *
- * rx_mpdu_info
- *		General information related to the MPDU that is passed
- *		on from REO entrance ring to the REO destination ring.
- *
- * queue_addr_lo
- *		Address (lower 32 bits) of the REO queue descriptor.
- *
- * queue_addr_hi
- *		Address (upper 8 bits) of the REO queue descriptor.
- *
- * mpdu_byte_count
- *		An approximation of the number of bytes received in this MPDU.
- *		Used to keeps stats on the amount of data flowing
- *		through a queue.
- *
- * reo_destination_indication
- *		The id of the reo exit ring where the msdu frame shall push
- *		after (MPDU level) reordering has finished. Values are defined
- *		in enum %HAL_RX_MSDU_DESC_REO_DEST_IND_.
- *
- * frameless_bar
- *		Indicates that this REO entrance ring struct contains BAR info
- *		from a multi TID BAR frame. The original multi TID BAR frame
- *		itself contained all the REO info for the first TID, but all
- *		the subsequent TID info and their linkage to the REO descriptors
- *		is passed down as 'frameless' BAR info.
- *
- *		The only fields valid in this descriptor when this bit is set
- *		are queue_addr_lo, queue_addr_hi, mpdu_sequence_number,
- *		bar_frame and peer_meta_data.
- *
- * rxdma_push_reason
- *		Reason for pushing this frame to this exit ring. Values are
- *		defined in enum %HAL_REO_ENTR_RING_RXDMA_PUSH_REASON_.
- *
- * rxdma_error_code
- *		Valid only when 'push_reason' is set. All error codes are
- *		defined in enum %HAL_REO_ENTR_RING_RXDMA_ECODE_.
- *
- * mpdu_fragment_number
- *		Field only valid when Reo_level_mpdu_frame_info.
- *		Rx_mpdu_desc_info_details.Fragment_flag is set.
- *
- * sw_exception
- *		When not set, REO is performing all its default MPDU processing
- *		operations,
- *		When set, this REO entrance descriptor is generated by FW, and
- *		should be processed as an exception. This implies:
- *		NO re-order function is needed.
- *		MPDU delinking is determined by the setting of field
- *		SW_excection_mpdu_delink
- *
- * sw_exception_mpdu_delink
- *		Field only valid when SW_exception is set.
- *		1'b0: REO should NOT delink the MPDU, and thus pass this
- *			MPDU on to the destination ring as is. This implies that
- *			in the REO_DESTINATION_RING struct field
- *			Buf_or_link_desc_addr_info should point to an MSDU link
- *			descriptor
- *		1'b1: REO should perform the normal MPDU delink into MSDU operations.
- *
- * sw_exception_dest_ring
- *		Field only valid when fields SW_exception and SW
- *		exception_destination_ring_valid are set. values are defined
- *		in %HAL_RX_REO_DEST_RING_.
- *
- * mpdu_seq_number
- *		The field can have two different meanings based on the setting
- *		of sub-field Reo level mpdu frame info.
- *		Rx_mpdu_desc_info_details. BAR_frame
- *		'BAR_frame' is NOT set:
- *		The MPDU sequence number of the received frame.
- *		'BAR_frame' is set.
- *		The MPDU Start sequence number from the BAR frame
- *
- * phy_ppdu_id
- *		A PPDU counter value that PHY increments for every PPDU received
- *
- * src_link_id
- *		Set to the link ID of the PMAC that received the frame
- *
- * ring_id
- *		The buffer pointer ring id.
- *		0 - Idle ring
- *		1 - N refers to other rings.
- *
- * looping_count
- *		Indicates the number of times the producer of entries into
- *		this ring has looped around the ring.
- */
-
-#define HAL_REO_CMD_HDR_INFO0_CMD_NUMBER	GENMASK(15, 0)
-#define HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED	BIT(16)
-
-struct hal_reo_cmd_hdr {
-	__le32 info0;
-} __packed;
-
-#define HAL_REO_GET_QUEUE_STATS_INFO0_QUEUE_ADDR_HI	GENMASK(7, 0)
-#define HAL_REO_GET_QUEUE_STATS_INFO0_CLEAR_STATS	BIT(8)
-
-struct hal_reo_get_queue_stats {
-	struct hal_reo_cmd_hdr cmd;
-	__le32 queue_addr_lo;
-	__le32 info0;
-	__le32 rsvd0[6];
-	__le32 tlv64_pad;
-} __packed;
-
-/* hal_reo_get_queue_stats
- *		Producer: SW
- *		Consumer: REO
- *
- * cmd
- *		Details for command execution tracking purposes.
- *
- * queue_addr_lo
- *		Address (lower 32 bits) of the REO queue descriptor.
- *
- * queue_addr_hi
- *		Address (upper 8 bits) of the REO queue descriptor.
- *
- * clear_stats
- *		Clear stats settings. When set, Clear the stats after
- *		generating the status.
- *
- *		Following stats will be cleared.
- *		Timeout_count
- *		Forward_due_to_bar_count
- *		Duplicate_count
- *		Frames_in_order_count
- *		BAR_received_count
- *		MPDU_Frames_processed_count
- *		MSDU_Frames_processed_count
- *		Total_processed_byte_count
- *		Late_receive_MPDU_count
- *		window_jump_2k
- *		Hole_count
- */
-
-#define HAL_REO_FLUSH_QUEUE_INFO0_DESC_ADDR_HI		GENMASK(7, 0)
-#define HAL_REO_FLUSH_QUEUE_INFO0_BLOCK_DESC_ADDR	BIT(8)
-#define HAL_REO_FLUSH_QUEUE_INFO0_BLOCK_RESRC_IDX	GENMASK(10, 9)
-
-struct hal_reo_flush_queue {
-	struct hal_reo_cmd_hdr cmd;
-	__le32 desc_addr_lo;
-	__le32 info0;
-	__le32 rsvd0[6];
-} __packed;
-
-#define HAL_REO_FLUSH_CACHE_INFO0_CACHE_ADDR_HI		GENMASK(7, 0)
-#define HAL_REO_FLUSH_CACHE_INFO0_FWD_ALL_MPDUS		BIT(8)
-#define HAL_REO_FLUSH_CACHE_INFO0_RELEASE_BLOCK_IDX	BIT(9)
-#define HAL_REO_FLUSH_CACHE_INFO0_BLOCK_RESRC_IDX	GENMASK(11, 10)
-#define HAL_REO_FLUSH_CACHE_INFO0_FLUSH_WO_INVALIDATE	BIT(12)
-#define HAL_REO_FLUSH_CACHE_INFO0_BLOCK_CACHE_USAGE	BIT(13)
-#define HAL_REO_FLUSH_CACHE_INFO0_FLUSH_ALL		BIT(14)
-#define HAL_REO_FLUSH_CACHE_INFO0_FLUSH_QUEUE_1K_DESC	BIT(15)
-
-struct hal_reo_flush_cache {
-	struct hal_reo_cmd_hdr cmd;
-	__le32 cache_addr_lo;
-	__le32 info0;
-	__le32 rsvd0[6];
-} __packed;
-
-#define HAL_TCL_DATA_CMD_INFO0_CMD_TYPE			BIT(0)
-#define HAL_TCL_DATA_CMD_INFO0_DESC_TYPE		BIT(1)
-#define HAL_TCL_DATA_CMD_INFO0_BANK_ID			GENMASK(7, 2)
-#define HAL_TCL_DATA_CMD_INFO0_TX_NOTIFY_FRAME		GENMASK(10, 8)
-#define HAL_TCL_DATA_CMD_INFO0_HDR_LEN_READ_SEL		BIT(11)
-#define HAL_TCL_DATA_CMD_INFO0_BUF_TIMESTAMP		GENMASK(30, 12)
-#define HAL_TCL_DATA_CMD_INFO0_BUF_TIMESTAMP_VLD	BIT(31)
-
-#define HAL_TCL_DATA_CMD_INFO1_CMD_NUM		GENMASK(31, 16)
-
-#define HAL_TCL_DATA_CMD_INFO2_DATA_LEN		GENMASK(15, 0)
-#define HAL_TCL_DATA_CMD_INFO2_IP4_CKSUM_EN	BIT(16)
-#define HAL_TCL_DATA_CMD_INFO2_UDP4_CKSUM_EN	BIT(17)
-#define HAL_TCL_DATA_CMD_INFO2_UDP6_CKSUM_EN	BIT(18)
-#define HAL_TCL_DATA_CMD_INFO2_TCP4_CKSUM_EN	BIT(19)
-#define HAL_TCL_DATA_CMD_INFO2_TCP6_CKSUM_EN	BIT(20)
-#define HAL_TCL_DATA_CMD_INFO2_TO_FW		BIT(21)
-#define HAL_TCL_DATA_CMD_INFO2_PKT_OFFSET	GENMASK(31, 23)
-
-#define HAL_TCL_DATA_CMD_INFO3_TID_OVERWRITE		BIT(0)
-#define HAL_TCL_DATA_CMD_INFO3_FLOW_OVERRIDE_EN		BIT(1)
-#define HAL_TCL_DATA_CMD_INFO3_CLASSIFY_INFO_SEL	GENMASK(3, 2)
-#define HAL_TCL_DATA_CMD_INFO3_TID			GENMASK(7, 4)
-#define HAL_TCL_DATA_CMD_INFO3_FLOW_OVERRIDE		BIT(8)
-#define HAL_TCL_DATA_CMD_INFO3_PMAC_ID			GENMASK(10, 9)
-#define HAL_TCL_DATA_CMD_INFO3_MSDU_COLOR		GENMASK(12, 11)
-#define HAL_TCL_DATA_CMD_INFO3_VDEV_ID			GENMASK(31, 24)
-
-#define HAL_TCL_DATA_CMD_INFO4_SEARCH_INDEX		GENMASK(19, 0)
-#define HAL_TCL_DATA_CMD_INFO4_CACHE_SET_NUM		GENMASK(23, 20)
-#define HAL_TCL_DATA_CMD_INFO4_IDX_LOOKUP_OVERRIDE	BIT(24)
-
-#define HAL_TCL_DATA_CMD_INFO5_RING_ID			GENMASK(27, 20)
-#define HAL_TCL_DATA_CMD_INFO5_LOOPING_COUNT		GENMASK(31, 28)
-
-enum hal_encrypt_type {
-	HAL_ENCRYPT_TYPE_WEP_40,
-	HAL_ENCRYPT_TYPE_WEP_104,
-	HAL_ENCRYPT_TYPE_TKIP_NO_MIC,
-	HAL_ENCRYPT_TYPE_WEP_128,
-	HAL_ENCRYPT_TYPE_TKIP_MIC,
-	HAL_ENCRYPT_TYPE_WAPI,
-	HAL_ENCRYPT_TYPE_CCMP_128,
-	HAL_ENCRYPT_TYPE_OPEN,
-	HAL_ENCRYPT_TYPE_CCMP_256,
-	HAL_ENCRYPT_TYPE_GCMP_128,
-	HAL_ENCRYPT_TYPE_AES_GCMP_256,
-	HAL_ENCRYPT_TYPE_WAPI_GCM_SM4,
-};
-
-enum hal_tcl_encap_type {
-	HAL_TCL_ENCAP_TYPE_RAW,
-	HAL_TCL_ENCAP_TYPE_NATIVE_WIFI,
-	HAL_TCL_ENCAP_TYPE_ETHERNET,
-	HAL_TCL_ENCAP_TYPE_802_3 = 3,
-	HAL_TCL_ENCAP_TYPE_MAX
-};
-
-enum hal_tcl_desc_type {
-	HAL_TCL_DESC_TYPE_BUFFER,
-	HAL_TCL_DESC_TYPE_EXT_DESC,
-	HAL_TCL_DESC_TYPE_MAX,
-};
-
-enum hal_wbm_htt_tx_comp_status {
-	HAL_WBM_REL_HTT_TX_COMP_STATUS_OK,
-	HAL_WBM_REL_HTT_TX_COMP_STATUS_DROP,
-	HAL_WBM_REL_HTT_TX_COMP_STATUS_TTL,
-	HAL_WBM_REL_HTT_TX_COMP_STATUS_REINJ,
-	HAL_WBM_REL_HTT_TX_COMP_STATUS_INSPECT,
-	HAL_WBM_REL_HTT_TX_COMP_STATUS_MEC_NOTIFY,
-	HAL_WBM_REL_HTT_TX_COMP_STATUS_VDEVID_MISMATCH,
-	HAL_WBM_REL_HTT_TX_COMP_STATUS_MAX,
-};
-
-struct hal_tcl_data_cmd {
-	struct ath12k_buffer_addr buf_addr_info;
-	__le32 info0;
-	__le32 info1;
-	__le32 info2;
-	__le32 info3;
-	__le32 info4;
-	__le32 info5;
-} __packed;
-
-/* hal_tcl_data_cmd
- *
- * buf_addr_info
- *		Details of the physical address of a buffer or MSDU
- *		link descriptor.
- *
- * tcl_cmd_type
- *		used to select the type of TCL Command descriptor
- *
- * desc_type
- *		Indicates the type of address provided in the buf_addr_info.
- *		Values are defined in enum %HAL_REO_DEST_RING_BUFFER_TYPE_.
- *
- * bank_id
- *		used to select one of the TCL register banks for fields removed
- *		from 'TCL_DATA_CMD' that do not change often within one virtual
- *		device or a set of virtual devices:
- *
- * tx_notify_frame
- *		TCL copies this value to 'TQM_ENTRANCE_RING' field FW_tx_notify_frame.
- *
- * hdr_length_read_sel
- *		used to select the per 'encap_type' register set for MSDU header
- *		read length
- *
- * buffer_timestamp
- * buffer_timestamp_valid
- *		Frame system entrance timestamp. It shall be filled by first
- *		module (SW, TCL or TQM) that sees the frames first.
- *
- * cmd_num
- *		This number can be used to match against status.
- *
- * data_length
- *		MSDU length in case of direct descriptor. Length of link
- *		extension descriptor in case of Link extension descriptor.
- *
- * *_checksum_en
- *		Enable checksum replacement for ipv4, udp_over_ipv4, ipv6,
- *		udp_over_ipv6, tcp_over_ipv4 and tcp_over_ipv6.
- *
- * to_fw
- *		Forward packet to FW along with classification result. The
- *		packet will not be forward to TQM when this bit is set.
- *		1'b0: Use classification result to forward the packet.
- *		1'b1: Override classification result & forward packet only to fw
- *
- * packet_offset
- *		Packet offset from Metadata in case of direct buffer descriptor.
- *
- * hlos_tid_overwrite
- *
- *		When set, TCL shall ignore the IP DSCP and VLAN PCP
- *		fields and use HLOS_TID as the final TID. Otherwise TCL
- *		shall consider the DSCP and PCP fields as well as HLOS_TID
- *		and choose a final TID based on the configured priority
- *
- * flow_override_enable
- *		TCL uses this to select the flow pointer from the peer table,
- *		which can be overridden by SW for pre-encrypted raw WiFi packets
- *		that cannot be parsed for UDP or for other MLO
- *		0 - FP_PARSE_IP: Use the flow-pointer based on parsing the IPv4
- *				 or IPv6 header.
- *		1 - FP_USE_OVERRIDE: Use the who_classify_info_sel and
- *				     flow_override fields to select the flow-pointer
- *
- * who_classify_info_sel
- *		Field only valid when flow_override_enable is set to FP_USE_OVERRIDE.
- *		This field is used to select  one of the 'WHO_CLASSIFY_INFO's in the
- *		peer table in case more than 2 flows are mapped to a single TID.
- *		0: To choose Flow 0 and 1 of any TID use this value.
- *		1: To choose Flow 2 and 3 of any TID use this value.
- *		2: To choose Flow 4 and 5 of any TID use this value.
- *		3: To choose Flow 6 and 7 of any TID use this value.
- *
- *		If who_classify_info sel is not in sync with the num_tx_classify_info
- *		field from address search, then TCL will set 'who_classify_info_sel'
- *		to 0 use flows 0 and 1.
- *
- * hlos_tid
- *		HLOS MSDU priority
- *		Field is used when HLOS_TID_overwrite is set.
- *
- * flow_override
- *		Field only valid when flow_override_enable is set to FP_USE_OVERRIDE
- *		TCL uses this to select the flow pointer from the peer table,
- *		which can be overridden by SW for pre-encrypted raw WiFi packets
- *		that cannot be parsed for UDP or for other MLO
- *		0 - FP_USE_NON_UDP: Use the non-UDP flow pointer (flow 0)
- *		1 - FP_USE_UDP: Use the UDP flow pointer (flow 1)
- *
- * pmac_id
- *		TCL uses this PMAC_ID in address search, i.e, while
- *		finding matching entry for the packet in AST corresponding
- *		to given PMAC_ID
- *
- *		If PMAC ID is all 1s (=> value 3), it indicates wildcard
- *		match for any PMAC
- *
- * vdev_id
- *		Virtual device ID to check against the address search entry to
- *		avoid security issues from transmitting packets from an incorrect
- *		virtual device
- *
- * search_index
- *		The index that will be used for index based address or
- *		flow search. The field is valid when 'search_type' is  1 or 2.
- *
- * cache_set_num
- *
- *		Cache set number that should be used to cache the index
- *		based search results, for address and flow search. This
- *		value should be equal to LSB four bits of the hash value of
- *		match data, in case of search index points to an entry which
- *		may be used in content based search also. The value can be
- *		anything when the entry pointed by search index will not be
- *		used for content based search.
- *
- * index_loop_override
- *		When set, address search and packet routing is forced to use
- *		'search_index' instead of following the register configuration
- *		selected by Bank_id.
- *
- * ring_id
- *		The buffer pointer ring ID.
- *		0 refers to the IDLE ring
- *		1 - N refers to other rings
- *
- * looping_count
- *
- *		A count value that indicates the number of times the
- *		producer of entries into the Ring has looped around the
- *		ring.
- *
- *		At initialization time, this value is set to 0. On the
- *		first loop, this value is set to 1. After the max value is
- *		reached allowed by the number of bits for this field, the
- *		count value continues with 0 again.
- *
- *		In case SW is the consumer of the ring entries, it can
- *		use this field to figure out up to where the producer of
- *		entries has created new entries. This eliminates the need to
- *		check where the head pointer' of the ring is located once
- *		the SW starts processing an interrupt indicating that new
- *		entries have been put into this ring...
- *
- *		Also note that SW if it wants only needs to look at the
- *		LSB bit of this count value.
- */
-
-#define HAL_TCL_DESC_LEN sizeof(struct hal_tcl_data_cmd)
-
-#define HAL_TX_MSDU_EXT_INFO0_BUF_PTR_LO	GENMASK(31, 0)
-
-#define HAL_TX_MSDU_EXT_INFO1_BUF_PTR_HI	GENMASK(7, 0)
-#define HAL_TX_MSDU_EXT_INFO1_EXTN_OVERRIDE	BIT(8)
-#define HAL_TX_MSDU_EXT_INFO1_ENCAP_TYPE	GENMASK(10, 9)
-#define HAL_TX_MSDU_EXT_INFO1_ENCRYPT_TYPE	GENMASK(14, 11)
-#define HAL_TX_MSDU_EXT_INFO1_BUF_LEN		GENMASK(31, 16)
-
-struct hal_tx_msdu_ext_desc {
-	__le32 rsvd0[6];
-	__le32 info0;
-	__le32 info1;
-	__le32 rsvd1[10];
-};
-
-struct hal_tcl_gse_cmd {
-	__le32 ctrl_buf_addr_lo;
-	__le32 info0;
-	__le32 meta_data[2];
-	__le32 rsvd0[2];
-	__le32 info1;
-} __packed;
-
-/* hal_tcl_gse_cmd
- *
- * ctrl_buf_addr_lo, ctrl_buf_addr_hi
- *		Address of a control buffer containing additional info needed
- *		for this command execution.
- *
- * meta_data
- *		Meta data to be returned in the status descriptor
- */
-
-enum hal_tcl_cache_op_res {
-	HAL_TCL_CACHE_OP_RES_DONE,
-	HAL_TCL_CACHE_OP_RES_NOT_FOUND,
-	HAL_TCL_CACHE_OP_RES_TIMEOUT,
-};
-
-struct hal_tcl_status_ring {
-	__le32 info0;
-	__le32 msdu_byte_count;
-	__le32 msdu_timestamp;
-	__le32 meta_data[2];
-	__le32 info1;
-	__le32 rsvd0;
-	__le32 info2;
-} __packed;
-
-/* hal_tcl_status_ring
- *
- * msdu_cnt
- * msdu_byte_count
- *		MSDU count of Entry and MSDU byte count for entry 1.
- *
- */
-
-#define HAL_CE_SRC_DESC_ADDR_INFO_ADDR_HI	GENMASK(7, 0)
-#define HAL_CE_SRC_DESC_ADDR_INFO_HASH_EN	BIT(8)
-#define HAL_CE_SRC_DESC_ADDR_INFO_BYTE_SWAP	BIT(9)
-#define HAL_CE_SRC_DESC_ADDR_INFO_DEST_SWAP	BIT(10)
-#define HAL_CE_SRC_DESC_ADDR_INFO_GATHER	BIT(11)
-#define HAL_CE_SRC_DESC_ADDR_INFO_LEN		GENMASK(31, 16)
-
-#define HAL_CE_SRC_DESC_META_INFO_DATA		GENMASK(15, 0)
-
-#define HAL_CE_SRC_DESC_FLAGS_RING_ID		GENMASK(27, 20)
-#define HAL_CE_SRC_DESC_FLAGS_LOOP_CNT		HAL_SRNG_DESC_LOOP_CNT
-
-struct hal_ce_srng_src_desc {
-	__le32 buffer_addr_low;
-	__le32 buffer_addr_info; /* %HAL_CE_SRC_DESC_ADDR_INFO_ */
-	__le32 meta_info; /* %HAL_CE_SRC_DESC_META_INFO_ */
-	__le32 flags; /* %HAL_CE_SRC_DESC_FLAGS_ */
-} __packed;
-
-/* hal_ce_srng_src_desc
- *
- * buffer_addr_lo
- *		LSB 32 bits of the 40 Bit Pointer to the source buffer
- *
- * buffer_addr_hi
- *		MSB 8 bits of the 40 Bit Pointer to the source buffer
- *
- * toeplitz_en
- *		Enable generation of 32-bit Toeplitz-LFSR hash for
- *		data transfer. In case of gather field in first source
- *		ring entry of the gather copy cycle in taken into account.
- *
- * src_swap
- *		Treats source memory organization as big-endian. For
- *		each dword read (4 bytes), the byte 0 is swapped with byte 3
- *		and byte 1 is swapped with byte 2.
- *		In case of gather field in first source ring entry of
- *		the gather copy cycle in taken into account.
- *
- * dest_swap
- *		Treats destination memory organization as big-endian.
- *		For each dword write (4 bytes), the byte 0 is swapped with
- *		byte 3 and byte 1 is swapped with byte 2.
- *		In case of gather field in first source ring entry of
- *		the gather copy cycle in taken into account.
- *
- * gather
- *		Enables gather of multiple copy engine source
- *		descriptors to one destination.
- *
- * ce_res_0
- *		Reserved
- *
- *
- * length
- *		Length of the buffer in units of octets of the current
- *		descriptor
- *
- * fw_metadata
- *		Meta data used by FW.
- *		In case of gather field in first source ring entry of
- *		the gather copy cycle in taken into account.
- *
- * ce_res_1
- *		Reserved
- *
- * ce_res_2
- *		Reserved
- *
- * ring_id
- *		The buffer pointer ring ID.
- *		0 refers to the IDLE ring
- *		1 - N refers to other rings
- *		Helps with debugging when dumping ring contents.
- *
- * looping_count
- *		A count value that indicates the number of times the
- *		producer of entries into the Ring has looped around the
- *		ring.
- *
- *		At initialization time, this value is set to 0. On the
- *		first loop, this value is set to 1. After the max value is
- *		reached allowed by the number of bits for this field, the
- *		count value continues with 0 again.
- *
- *		In case SW is the consumer of the ring entries, it can
- *		use this field to figure out up to where the producer of
- *		entries has created new entries. This eliminates the need to
- *		check where the head pointer' of the ring is located once
- *		the SW starts processing an interrupt indicating that new
- *		entries have been put into this ring...
- *
- *		Also note that SW if it wants only needs to look at the
- *		LSB bit of this count value.
- */
-
-#define HAL_CE_DEST_DESC_ADDR_INFO_ADDR_HI		GENMASK(7, 0)
-#define HAL_CE_DEST_DESC_ADDR_INFO_RING_ID		GENMASK(27, 20)
-#define HAL_CE_DEST_DESC_ADDR_INFO_LOOP_CNT		HAL_SRNG_DESC_LOOP_CNT
-
-struct hal_ce_srng_dest_desc {
-	__le32 buffer_addr_low;
-	__le32 buffer_addr_info; /* %HAL_CE_DEST_DESC_ADDR_INFO_ */
-} __packed;
-
-/* hal_ce_srng_dest_desc
- *
- * dst_buffer_low
- *		LSB 32 bits of the 40 Bit Pointer to the Destination
- *		buffer
- *
- * dst_buffer_high
- *		MSB 8 bits of the 40 Bit Pointer to the Destination
- *		buffer
- *
- * ce_res_4
- *		Reserved
- *
- * ring_id
- *		The buffer pointer ring ID.
- *		0 refers to the IDLE ring
- *		1 - N refers to other rings
- *		Helps with debugging when dumping ring contents.
- *
- * looping_count
- *		A count value that indicates the number of times the
- *		producer of entries into the Ring has looped around the
- *		ring.
- *
- *		At initialization time, this value is set to 0. On the
- *		first loop, this value is set to 1. After the max value is
- *		reached allowed by the number of bits for this field, the
- *		count value continues with 0 again.
- *
- *		In case SW is the consumer of the ring entries, it can
- *		use this field to figure out up to where the producer of
- *		entries has created new entries. This eliminates the need to
- *		check where the head pointer' of the ring is located once
- *		the SW starts processing an interrupt indicating that new
- *		entries have been put into this ring...
- *
- *		Also note that SW if it wants only needs to look at the
- *		LSB bit of this count value.
- */
-
-#define HAL_CE_DST_STATUS_DESC_FLAGS_HASH_EN		BIT(8)
-#define HAL_CE_DST_STATUS_DESC_FLAGS_BYTE_SWAP		BIT(9)
-#define HAL_CE_DST_STATUS_DESC_FLAGS_DEST_SWAP		BIT(10)
-#define HAL_CE_DST_STATUS_DESC_FLAGS_GATHER		BIT(11)
-#define HAL_CE_DST_STATUS_DESC_FLAGS_LEN		GENMASK(31, 16)
-
-#define HAL_CE_DST_STATUS_DESC_META_INFO_DATA		GENMASK(15, 0)
-#define HAL_CE_DST_STATUS_DESC_META_INFO_RING_ID	GENMASK(27, 20)
-#define HAL_CE_DST_STATUS_DESC_META_INFO_LOOP_CNT	HAL_SRNG_DESC_LOOP_CNT
-
-struct hal_ce_srng_dst_status_desc {
-	__le32 flags; /* %HAL_CE_DST_STATUS_DESC_FLAGS_ */
-	__le32 toeplitz_hash0;
-	__le32 toeplitz_hash1;
-	__le32 meta_info; /* HAL_CE_DST_STATUS_DESC_META_INFO_ */
-} __packed;
-
-/* hal_ce_srng_dst_status_desc
- *
- * ce_res_5
- *		Reserved
- *
- * toeplitz_en
- *
- * src_swap
- *		Source memory buffer swapped
- *
- * dest_swap
- *		Destination  memory buffer swapped
- *
- * gather
- *		Gather of multiple copy engine source descriptors to one
- *		destination enabled
- *
- * ce_res_6
- *		Reserved
- *
- * length
- *		Sum of all the Lengths of the source descriptor in the
- *		gather chain
- *
- * toeplitz_hash_0
- *		32 LS bits of 64 bit Toeplitz LFSR hash result
- *
- * toeplitz_hash_1
- *		32 MS bits of 64 bit Toeplitz LFSR hash result
- *
- * fw_metadata
- *		Meta data used by FW
- *		In case of gather field in first source ring entry of
- *		the gather copy cycle in taken into account.
- *
- * ce_res_7
- *		Reserved
- *
- * ring_id
- *		The buffer pointer ring ID.
- *		0 refers to the IDLE ring
- *		1 - N refers to other rings
- *		Helps with debugging when dumping ring contents.
- *
- * looping_count
- *		A count value that indicates the number of times the
- *		producer of entries into the Ring has looped around the
- *		ring.
- *
- *		At initialization time, this value is set to 0. On the
- *		first loop, this value is set to 1. After the max value is
- *		reached allowed by the number of bits for this field, the
- *		count value continues with 0 again.
- *
- *		In case SW is the consumer of the ring entries, it can
- *		use this field to figure out up to where the producer of
- *		entries has created new entries. This eliminates the need to
- *		check where the head pointer' of the ring is located once
- *		the SW starts processing an interrupt indicating that new
- *		entries have been put into this ring...
- *
- *		Also note that SW if it wants only needs to look at the
- *			LSB bit of this count value.
- */
-
-#define HAL_TX_RATE_STATS_INFO0_VALID		BIT(0)
-#define HAL_TX_RATE_STATS_INFO0_BW		GENMASK(3, 1)
-#define HAL_TX_RATE_STATS_INFO0_PKT_TYPE	GENMASK(7, 4)
-#define HAL_TX_RATE_STATS_INFO0_STBC		BIT(8)
-#define HAL_TX_RATE_STATS_INFO0_LDPC		BIT(9)
-#define HAL_TX_RATE_STATS_INFO0_SGI		GENMASK(11, 10)
-#define HAL_TX_RATE_STATS_INFO0_MCS		GENMASK(15, 12)
-#define HAL_TX_RATE_STATS_INFO0_OFDMA_TX	BIT(16)
-#define HAL_TX_RATE_STATS_INFO0_TONES_IN_RU	GENMASK(28, 17)
-
-enum hal_tx_rate_stats_bw {
-	HAL_TX_RATE_STATS_BW_20,
-	HAL_TX_RATE_STATS_BW_40,
-	HAL_TX_RATE_STATS_BW_80,
-	HAL_TX_RATE_STATS_BW_160,
-};
-
-enum hal_tx_rate_stats_pkt_type {
-	HAL_TX_RATE_STATS_PKT_TYPE_11A,
-	HAL_TX_RATE_STATS_PKT_TYPE_11B,
-	HAL_TX_RATE_STATS_PKT_TYPE_11N,
-	HAL_TX_RATE_STATS_PKT_TYPE_11AC,
-	HAL_TX_RATE_STATS_PKT_TYPE_11AX,
-	HAL_TX_RATE_STATS_PKT_TYPE_11BA,
-	HAL_TX_RATE_STATS_PKT_TYPE_11BE,
-};
-
-enum hal_tx_rate_stats_sgi {
-	HAL_TX_RATE_STATS_SGI_08US,
-	HAL_TX_RATE_STATS_SGI_04US,
-	HAL_TX_RATE_STATS_SGI_16US,
-	HAL_TX_RATE_STATS_SGI_32US,
-};
-
-struct hal_tx_rate_stats {
-	__le32 info0;
-	__le32 tsf;
-} __packed;
-
-struct hal_wbm_link_desc {
-	struct ath12k_buffer_addr buf_addr_info;
-} __packed;
-
-/* hal_wbm_link_desc
- *
- *	Producer: WBM
- *	Consumer: WBM
- *
- * buf_addr_info
- *		Details of the physical address of a buffer or MSDU
- *		link descriptor.
- */
-
-enum hal_wbm_rel_src_module {
-	HAL_WBM_REL_SRC_MODULE_TQM,
-	HAL_WBM_REL_SRC_MODULE_RXDMA,
-	HAL_WBM_REL_SRC_MODULE_REO,
-	HAL_WBM_REL_SRC_MODULE_FW,
-	HAL_WBM_REL_SRC_MODULE_SW,
-	HAL_WBM_REL_SRC_MODULE_MAX,
-};
-
-enum hal_wbm_rel_desc_type {
-	HAL_WBM_REL_DESC_TYPE_REL_MSDU,
-	HAL_WBM_REL_DESC_TYPE_MSDU_LINK,
-	HAL_WBM_REL_DESC_TYPE_MPDU_LINK,
-	HAL_WBM_REL_DESC_TYPE_MSDU_EXT,
-	HAL_WBM_REL_DESC_TYPE_QUEUE_EXT,
-};
-
-/* hal_wbm_rel_desc_type
- *
- * msdu_buffer
- *	The address points to an MSDU buffer
- *
- * msdu_link_descriptor
- *	The address points to an Tx MSDU link descriptor
- *
- * mpdu_link_descriptor
- *	The address points to an MPDU link descriptor
- *
- * msdu_ext_descriptor
- *	The address points to an MSDU extension descriptor
- *
- * queue_ext_descriptor
- *	The address points to an TQM queue extension descriptor. WBM should
- *	treat this is the same way as a link descriptor.
- */
-
-enum hal_wbm_rel_bm_act {
-	HAL_WBM_REL_BM_ACT_PUT_IN_IDLE,
-	HAL_WBM_REL_BM_ACT_REL_MSDU,
-};
-
-/* hal_wbm_rel_bm_act
- *
- * put_in_idle_list
- *	Put the buffer or descriptor back in the idle list. In case of MSDU or
- *	MDPU link descriptor, BM does not need to check to release any
- *	individual MSDU buffers.
- *
- * release_msdu_list
- *	This BM action can only be used in combination with desc_type being
- *	msdu_link_descriptor. Field first_msdu_index points out which MSDU
- *	pointer in the MSDU link descriptor is the first of an MPDU that is
- *	released. BM shall release all the MSDU buffers linked to this first
- *	MSDU buffer pointer. All related MSDU buffer pointer entries shall be
- *	set to value 0, which represents the 'NULL' pointer. When all MSDU
- *	buffer pointers in the MSDU link descriptor are 'NULL', the MSDU link
- *	descriptor itself shall also be released.
- */
-#define HAL_WBM_COMPL_RX_INFO0_REL_SRC_MODULE		GENMASK(2, 0)
-#define HAL_WBM_COMPL_RX_INFO0_BM_ACTION		GENMASK(5, 3)
-#define HAL_WBM_COMPL_RX_INFO0_DESC_TYPE		GENMASK(8, 6)
-#define HAL_WBM_COMPL_RX_INFO0_RBM			GENMASK(12, 9)
-#define HAL_WBM_COMPL_RX_INFO0_RXDMA_PUSH_REASON	GENMASK(18, 17)
-#define HAL_WBM_COMPL_RX_INFO0_RXDMA_ERROR_CODE		GENMASK(23, 19)
-#define HAL_WBM_COMPL_RX_INFO0_REO_PUSH_REASON		GENMASK(25, 24)
-#define HAL_WBM_COMPL_RX_INFO0_REO_ERROR_CODE		GENMASK(30, 26)
-#define HAL_WBM_COMPL_RX_INFO0_WBM_INTERNAL_ERROR	BIT(31)
-
-#define HAL_WBM_COMPL_RX_INFO1_PHY_ADDR_HI		GENMASK(7, 0)
-#define HAL_WBM_COMPL_RX_INFO1_SW_COOKIE		GENMASK(27, 8)
-#define HAL_WBM_COMPL_RX_INFO1_LOOPING_COUNT		GENMASK(31, 28)
-
-struct hal_wbm_completion_ring_rx {
-	__le32 addr_lo;
-	__le32 addr_hi;
-	__le32 info0;
-	struct rx_mpdu_desc rx_mpdu_info;
-	struct rx_msdu_desc rx_msdu_info;
-	__le32 phy_addr_lo;
-	__le32 info1;
-} __packed;
-
-#define HAL_WBM_COMPL_TX_INFO0_REL_SRC_MODULE		GENMASK(2, 0)
-#define HAL_WBM_COMPL_TX_INFO0_DESC_TYPE		GENMASK(8, 6)
-#define HAL_WBM_COMPL_TX_INFO0_RBM			GENMASK(12, 9)
-#define HAL_WBM_COMPL_TX_INFO0_TQM_RELEASE_REASON	GENMASK(16, 13)
-#define HAL_WBM_COMPL_TX_INFO0_RBM_OVERRIDE_VLD		BIT(17)
-#define HAL_WBM_COMPL_TX_INFO0_SW_COOKIE_LO		GENMASK(29, 18)
-#define HAL_WBM_COMPL_TX_INFO0_CC_DONE			BIT(30)
-#define HAL_WBM_COMPL_TX_INFO0_WBM_INTERNAL_ERROR	BIT(31)
-
-#define HAL_WBM_COMPL_TX_INFO1_TQM_STATUS_NUMBER	GENMASK(23, 0)
-#define HAL_WBM_COMPL_TX_INFO1_TRANSMIT_COUNT		GENMASK(30, 24)
-#define HAL_WBM_COMPL_TX_INFO1_SW_REL_DETAILS_VALID	BIT(31)
-
-#define HAL_WBM_COMPL_TX_INFO2_ACK_FRAME_RSSI		GENMASK(7, 0)
-#define HAL_WBM_COMPL_TX_INFO2_FIRST_MSDU		BIT(8)
-#define HAL_WBM_COMPL_TX_INFO2_LAST_MSDU		BIT(9)
-#define HAL_WBM_COMPL_TX_INFO2_FW_TX_NOTIF_FRAME	GENMASK(12, 10)
-#define HAL_WBM_COMPL_TX_INFO2_BUFFER_TIMESTAMP		GENMASK(31, 13)
-
-#define HAL_WBM_COMPL_TX_INFO3_PEER_ID			GENMASK(15, 0)
-#define HAL_WBM_COMPL_TX_INFO3_TID			GENMASK(19, 16)
-#define HAL_WBM_COMPL_TX_INFO3_SW_COOKIE_HI		GENMASK(27, 20)
-#define HAL_WBM_COMPL_TX_INFO3_LOOPING_COUNT		GENMASK(31, 28)
-
-struct hal_wbm_completion_ring_tx {
-	__le32 buf_va_lo;
-	__le32 buf_va_hi;
-	__le32 info0;
-	__le32 info1;
-	__le32 info2;
-	struct hal_tx_rate_stats rate_stats;
-	__le32 info3;
-} __packed;
-
-#define HAL_WBM_RELEASE_TX_INFO0_REL_SRC_MODULE		GENMASK(2, 0)
-#define HAL_WBM_RELEASE_TX_INFO0_BM_ACTION		GENMASK(5, 3)
-#define HAL_WBM_RELEASE_TX_INFO0_DESC_TYPE		GENMASK(8, 6)
-#define HAL_WBM_RELEASE_TX_INFO0_FIRST_MSDU_IDX		GENMASK(12, 9)
-#define HAL_WBM_RELEASE_TX_INFO0_TQM_RELEASE_REASON	GENMASK(18, 13)
-#define HAL_WBM_RELEASE_TX_INFO0_RBM_OVERRIDE_VLD	BIT(17)
-#define HAL_WBM_RELEASE_TX_INFO0_SW_BUFFER_COOKIE_11_0	GENMASK(29, 18)
-#define HAL_WBM_RELEASE_TX_INFO0_WBM_INTERNAL_ERROR	BIT(31)
-
-#define HAL_WBM_RELEASE_TX_INFO1_TQM_STATUS_NUMBER	GENMASK(23, 0)
-#define HAL_WBM_RELEASE_TX_INFO1_TRANSMIT_COUNT		GENMASK(30, 24)
-#define HAL_WBM_RELEASE_TX_INFO1_SW_REL_DETAILS_VALID	BIT(31)
-
-#define HAL_WBM_RELEASE_TX_INFO2_ACK_FRAME_RSSI		GENMASK(7, 0)
-#define HAL_WBM_RELEASE_TX_INFO2_FIRST_MSDU		BIT(8)
-#define HAL_WBM_RELEASE_TX_INFO2_LAST_MSDU		BIT(9)
-#define HAL_WBM_RELEASE_TX_INFO2_FW_TX_NOTIF_FRAME	GENMASK(12, 10)
-#define HAL_WBM_RELEASE_TX_INFO2_BUFFER_TIMESTAMP	GENMASK(31, 13)
-
-#define HAL_WBM_RELEASE_TX_INFO3_PEER_ID		GENMASK(15, 0)
-#define HAL_WBM_RELEASE_TX_INFO3_TID			GENMASK(19, 16)
-#define HAL_WBM_RELEASE_TX_INFO3_SW_BUFFER_COOKIE_19_12	GENMASK(27, 20)
-#define HAL_WBM_RELEASE_TX_INFO3_LOOPING_COUNT		GENMASK(31, 28)
-
-struct hal_wbm_release_ring_tx {
-	struct ath12k_buffer_addr buf_addr_info;
-	__le32 info0;
-	__le32 info1;
-	__le32 info2;
-	struct hal_tx_rate_stats rate_stats;
-	__le32 info3;
-} __packed;
-
-#define HAL_WBM_RELEASE_RX_INFO0_REL_SRC_MODULE		GENMASK(2, 0)
-#define HAL_WBM_RELEASE_RX_INFO0_BM_ACTION		GENMASK(5, 3)
-#define HAL_WBM_RELEASE_RX_INFO0_DESC_TYPE		GENMASK(8, 6)
-#define HAL_WBM_RELEASE_RX_INFO0_FIRST_MSDU_IDX		GENMASK(12, 9)
-#define HAL_WBM_RELEASE_RX_INFO0_CC_STATUS		BIT(16)
-#define HAL_WBM_RELEASE_RX_INFO0_RXDMA_PUSH_REASON	GENMASK(18, 17)
-#define HAL_WBM_RELEASE_RX_INFO0_RXDMA_ERROR_CODE	GENMASK(23, 19)
-#define HAL_WBM_RELEASE_RX_INFO0_REO_PUSH_REASON	GENMASK(25, 24)
-#define HAL_WBM_RELEASE_RX_INFO0_REO_ERROR_CODE		GENMASK(30, 26)
-#define HAL_WBM_RELEASE_RX_INFO0_WBM_INTERNAL_ERROR	BIT(31)
-
-#define HAL_WBM_RELEASE_RX_INFO2_RING_ID		GENMASK(27, 20)
-#define HAL_WBM_RELEASE_RX_INFO2_LOOPING_COUNT		GENMASK(31, 28)
-
-struct hal_wbm_release_ring_rx {
-	struct ath12k_buffer_addr buf_addr_info;
-	__le32 info0;
-	struct rx_mpdu_desc rx_mpdu_info;
-	struct rx_msdu_desc rx_msdu_info;
-	__le32 info1;
-	__le32 info2;
-} __packed;
-
-#define HAL_WBM_RELEASE_RX_CC_INFO0_RBM			GENMASK(12, 9)
-#define HAL_WBM_RELEASE_RX_CC_INFO1_COOKIE		GENMASK(27, 8)
-/* Used when hw cc is success */
-struct hal_wbm_release_ring_cc_rx {
-	__le32 buf_va_lo;
-	__le32 buf_va_hi;
-	__le32 info0;
-	struct rx_mpdu_desc rx_mpdu_info;
-	struct rx_msdu_desc rx_msdu_info;
-	__le32 buf_pa_lo;
-	__le32 info1;
-} __packed;
-
-#define HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE		GENMASK(2, 0)
-#define HAL_WBM_RELEASE_INFO0_BM_ACTION			GENMASK(5, 3)
-#define HAL_WBM_RELEASE_INFO0_DESC_TYPE			GENMASK(8, 6)
-#define HAL_WBM_RELEASE_INFO0_RXDMA_PUSH_REASON		GENMASK(18, 17)
-#define HAL_WBM_RELEASE_INFO0_RXDMA_ERROR_CODE		GENMASK(23, 19)
-#define HAL_WBM_RELEASE_INFO0_REO_PUSH_REASON		GENMASK(25, 24)
-#define HAL_WBM_RELEASE_INFO0_REO_ERROR_CODE		GENMASK(30, 26)
-#define HAL_WBM_RELEASE_INFO0_WBM_INTERNAL_ERROR	BIT(31)
-
-#define HAL_WBM_RELEASE_INFO3_FIRST_MSDU		BIT(0)
-#define HAL_WBM_RELEASE_INFO3_LAST_MSDU			BIT(1)
-#define HAL_WBM_RELEASE_INFO3_CONTINUATION		BIT(2)
-
-#define HAL_WBM_RELEASE_INFO5_LOOPING_COUNT		GENMASK(31, 28)
-#define HAL_ENCRYPT_TYPE_MAX 12
-
-struct hal_wbm_release_ring {
-	struct ath12k_buffer_addr buf_addr_info;
-	__le32 info0;
-	__le32 info1;
-	__le32 info2;
-	__le32 info3;
-	__le32 info4;
-	__le32 info5;
-} __packed;
-
-/* hal_wbm_release_ring
- *
- *	Producer: SW/TQM/RXDMA/REO/SWITCH
- *	Consumer: WBM/SW/FW
- *
- * HTT tx status is overlaid on wbm_release ring on 4-byte words 2, 3, 4 and 5
- * for software based completions.
- *
- * buf_addr_info
- *	Details of the physical address of the buffer or link descriptor.
- *
- * release_source_module
- *	Indicates which module initiated the release of this buffer/descriptor.
- *	Values are defined in enum %HAL_WBM_REL_SRC_MODULE_.
- *
- * buffer_or_desc_type
- *	Field only valid when WBM is marked as the return_buffer_manager in
- *	the Released_Buffer_address_info. Indicates that type of buffer or
- *	descriptor is being released. Values are in enum %HAL_WBM_REL_DESC_TYPE.
- *
- * wbm_internal_error
- *	Is set when WBM got a buffer pointer but the action was to push it to
- *	the idle link descriptor ring or do link related activity OR
- *	Is set when WBM got a link buffer pointer but the action was to push it
- *	to the buffer descriptor ring.
- *
- * looping_count
- *	A count value that indicates the number of times the
- *	producer of entries into the Buffer Manager Ring has looped
- *	around the ring.
- *
- *	At initialization time, this value is set to 0. On the
- *	first loop, this value is set to 1. After the max value is
- *	reached allowed by the number of bits for this field, the
- *	count value continues with 0 again.
- *
- *	In case SW is the consumer of the ring entries, it can
- *	use this field to figure out up to where the producer of
- *	entries has created new entries. This eliminates the need to
- *	check where the head pointer' of the ring is located once
- *	the SW starts processing an interrupt indicating that new
- *	entries have been put into this ring...
- *
- *	Also note that SW if it wants only needs to look at the
- *	LSB bit of this count value.
- */
-
-/**
- * enum hal_wbm_tqm_rel_reason - TQM release reason code
- * @HAL_WBM_TQM_REL_REASON_FRAME_ACKED: ACK or BACK received for the frame
- * @HAL_WBM_TQM_REL_REASON_CMD_REMOVE_MPDU: Command remove_mpdus initiated by SW
- * @HAL_WBM_TQM_REL_REASON_CMD_REMOVE_TX: Command remove transmitted_mpdus
- *	initiated by sw.
- * @HAL_WBM_TQM_REL_REASON_CMD_REMOVE_NOTX: Command remove untransmitted_mpdus
- *	initiated by sw.
- * @HAL_WBM_TQM_REL_REASON_CMD_REMOVE_AGED_FRAMES: Command remove aged msdus or
- *	mpdus.
- * @HAL_WBM_TQM_REL_REASON_CMD_REMOVE_RESEAON1: Remove command initiated by
- *	fw with fw_reason1.
- * @HAL_WBM_TQM_REL_REASON_CMD_REMOVE_RESEAON2: Remove command initiated by
- *	fw with fw_reason2.
- * @HAL_WBM_TQM_REL_REASON_CMD_REMOVE_RESEAON3: Remove command initiated by
- *	fw with fw_reason3.
- * @HAL_WBM_TQM_REL_REASON_CMD_DISABLE_QUEUE: Remove command initiated by
- *	fw with disable queue.
- * @HAL_WBM_TQM_REL_REASON_CMD_TILL_NONMATCHING: Remove command initiated by
- *	fw to remove all mpdu until 1st non-match.
- * @HAL_WBM_TQM_REL_REASON_DROP_THRESHOLD: Dropped due to drop threshold
- *	criteria
- * @HAL_WBM_TQM_REL_REASON_DROP_LINK_DESC_UNAVAIL: Dropped due to link desc
- *	not available
- * @HAL_WBM_TQM_REL_REASON_DROP_OR_INVALID_MSDU: Dropped due drop bit set or
- *	null flow
- * @HAL_WBM_TQM_REL_REASON_MULTICAST_DROP: Dropped due mcast drop set for VDEV
- * @HAL_WBM_TQM_REL_REASON_VDEV_MISMATCH_DROP: Dropped due to being set with
- *	'TCL_drop_reason'
- */
-enum hal_wbm_tqm_rel_reason {
-	HAL_WBM_TQM_REL_REASON_FRAME_ACKED,
-	HAL_WBM_TQM_REL_REASON_CMD_REMOVE_MPDU,
-	HAL_WBM_TQM_REL_REASON_CMD_REMOVE_TX,
-	HAL_WBM_TQM_REL_REASON_CMD_REMOVE_NOTX,
-	HAL_WBM_TQM_REL_REASON_CMD_REMOVE_AGED_FRAMES,
-	HAL_WBM_TQM_REL_REASON_CMD_REMOVE_RESEAON1,
-	HAL_WBM_TQM_REL_REASON_CMD_REMOVE_RESEAON2,
-	HAL_WBM_TQM_REL_REASON_CMD_REMOVE_RESEAON3,
-	HAL_WBM_TQM_REL_REASON_CMD_DISABLE_QUEUE,
-	HAL_WBM_TQM_REL_REASON_CMD_TILL_NONMATCHING,
-	HAL_WBM_TQM_REL_REASON_DROP_THRESHOLD,
-	HAL_WBM_TQM_REL_REASON_DROP_LINK_DESC_UNAVAIL,
-	HAL_WBM_TQM_REL_REASON_DROP_OR_INVALID_MSDU,
-	HAL_WBM_TQM_REL_REASON_MULTICAST_DROP,
-	HAL_WBM_TQM_REL_REASON_VDEV_MISMATCH_DROP,
-};
-
-struct hal_wbm_buffer_ring {
-	struct ath12k_buffer_addr buf_addr_info;
-};
-
-enum hal_mon_end_reason {
-	HAL_MON_STATUS_BUFFER_FULL,
-	HAL_MON_FLUSH_DETECTED,
-	HAL_MON_END_OF_PPDU,
-	HAL_MON_PPDU_TRUNCATED,
-};
-
-#define HAL_SW_MONITOR_RING_INFO0_RXDMA_PUSH_REASON	GENMASK(1, 0)
-#define HAL_SW_MONITOR_RING_INFO0_RXDMA_ERROR_CODE	GENMASK(6, 2)
-#define HAL_SW_MONITOR_RING_INFO0_MPDU_FRAGMENT_NUMBER	GENMASK(10, 7)
-#define HAL_SW_MONITOR_RING_INFO0_FRAMELESS_BAR		BIT(11)
-#define HAL_SW_MONITOR_RING_INFO0_STATUS_BUF_COUNT	GENMASK(15, 12)
-#define HAL_SW_MONITOR_RING_INFO0_END_OF_PPDU		BIT(16)
-
-#define HAL_SW_MONITOR_RING_INFO1_PHY_PPDU_ID	GENMASK(15, 0)
-#define HAL_SW_MONITOR_RING_INFO1_RING_ID	GENMASK(27, 20)
-#define HAL_SW_MONITOR_RING_INFO1_LOOPING_COUNT	GENMASK(31, 28)
-
-struct hal_sw_monitor_ring {
-	struct ath12k_buffer_addr buf_addr_info;
-	struct rx_mpdu_desc rx_mpdu_info;
-	struct ath12k_buffer_addr status_buff_addr_info;
-	__le32 info0; /* %HAL_SW_MONITOR_RING_INFO0 */
-	__le32 info1; /* %HAL_SW_MONITOR_RING_INFO1 */
-} __packed;
-
-/* hal_sw_monitor_ring
- *
- *		Producer: RXDMA
- *		Consumer: REO/SW/FW
- * buf_addr_info
- *              Details of the physical address of a buffer or MSDU
- *              link descriptor.
- *
- * rx_mpdu_info
- *              Details related to the MPDU being pushed to SW, valid
- *              only if end_of_ppdu is set to 0.
- *
- * status_buff_addr_info
- *		Details of the physical address of the first status
- *		buffer used for the PPDU (either the PPDU that included the
- *		MPDU being pushed to SW if end_of_ppdu = 0, or the PPDU
- *		whose end is indicated through end_of_ppdu = 1)
- *
- * rxdma_push_reason
- *		Indicates why RXDMA pushed the frame to this ring
- *
- *		<enum 0 rxdma_error_detected> RXDMA detected an error an
- *		pushed this frame to this queue
- *
- *		<enum 1 rxdma_routing_instruction> RXDMA pushed the
- *		frame to this queue per received routing instructions. No
- *		error within RXDMA was detected
- *
- *		<enum 2 rxdma_rx_flush> RXDMA received an RX_FLUSH. As a
- *		result the MSDU link descriptor might not have the
- *		last_msdu_in_mpdu_flag set, but instead WBM might just see a
- *		NULL pointer in the MSDU link descriptor. This is to be
- *		considered a normal condition for this scenario.
- *
- * rxdma_error_code
- *		Field only valid when rxdma_push_reason is set to
- *		'rxdma_error_detected.'
- *
- *		<enum 0 rxdma_overflow_err>MPDU frame is not complete
- *		due to a FIFO overflow error in RXPCU.
- *
- *		<enum 1 rxdma_mpdu_length_err>MPDU frame is not complete
- *		due to receiving incomplete MPDU from the PHY
- *
- *		<enum 3 rxdma_decrypt_err>CRYPTO reported a decryption
- *		error or CRYPTO received an encrypted frame, but did not get
- *		a valid corresponding key id in the peer entry.
- *
- *		<enum 4 rxdma_tkip_mic_err>CRYPTO reported a TKIP MIC
- *		error
- *
- *		<enum 5 rxdma_unecrypted_err>CRYPTO reported an
- *		unencrypted frame error when encrypted was expected
- *
- *		<enum 6 rxdma_msdu_len_err>RX OLE reported an MSDU
- *		length error
- *
- *		<enum 7 rxdma_msdu_limit_err>RX OLE reported that max
- *		number of MSDUs allowed in an MPDU got exceeded
- *
- *		<enum 8 rxdma_wifi_parse_err>RX OLE reported a parsing
- *		error
- *
- *		<enum 9 rxdma_amsdu_parse_err>RX OLE reported an A-MSDU
- *		parsing error
- *
- *		<enum 10 rxdma_sa_timeout_err>RX OLE reported a timeout
- *		during SA search
- *
- *		<enum 11 rxdma_da_timeout_err>RX OLE reported a timeout
- *		during DA search
- *
- *		<enum 12 rxdma_flow_timeout_err>RX OLE reported a
- *		timeout during flow search
- *
- *		<enum 13 rxdma_flush_request>RXDMA received a flush
- *		request
- *
- *		<enum 14 rxdma_amsdu_fragment_err>Rx PCU reported A-MSDU
- *		present as well as a fragmented MPDU.
- *
- * mpdu_fragment_number
- *		Field only valid when Reo_level_mpdu_frame_info.
- *		Rx_mpdu_desc_info_details.Fragment_flag is set and
- *		end_of_ppdu is set to 0.
- *
- *		The fragment number from the 802.11 header.
- *
- *		Note that the sequence number is embedded in the field:
- *		Reo_level_mpdu_frame_info. Rx_mpdu_desc_info_details.
- *		Mpdu_sequence_number
- *
- * frameless_bar
- *		When set, this SW monitor ring struct contains BAR info
- *		from a multi TID BAR frame. The original multi TID BAR frame
- *		itself contained all the REO info for the first TID, but all
- *		the subsequent TID info and their linkage to the REO
- *		descriptors is passed down as 'frameless' BAR info.
- *
- *		The only fields valid in this descriptor when this bit
- *		is within the
- *
- *		Reo_level_mpdu_frame_info:
- *		   Within Rx_mpdu_desc_info_details:
- *			Mpdu_Sequence_number
- *			BAR_frame
- *			Peer_meta_data
- *			All other fields shall be set to 0.
- *
- * status_buf_count
- *		A count of status buffers used so far for the PPDU
- *		(either the PPDU that included the MPDU being pushed to SW
- *		if end_of_ppdu = 0, or the PPDU whose end is indicated
- *		through end_of_ppdu = 1)
- *
- * end_of_ppdu
- *		Some hw RXDMA can be configured to generate a separate
- *		'SW_MONITOR_RING' descriptor at the end of a PPDU (either
- *		through an 'RX_PPDU_END' TLV or through an 'RX_FLUSH') to
- *		demarcate PPDUs.
- *
- *		For such a descriptor, this bit is set to 1 and fields
- *		Reo_level_mpdu_frame_info, mpdu_fragment_number and
- *		Frameless_bar are all set to 0.
- *
- *		Otherwise this bit is set to 0.
- *
- * phy_ppdu_id
- *		A PPDU counter value that PHY increments for every PPDU
- *		received
- *
- *		The counter value wraps around. Some hw RXDMA can be
- *		configured to copy this from the RX_PPDU_START TLV for every
- *		output descriptor.
- *
- * ring_id
- *		For debugging.
- *		This field is filled in by the SRNG module.
- *		It help to identify the ring that is being looked
- *
- * looping_count
- *		For debugging.
- *		This field is filled in by the SRNG module.
- *
- *		A count value that indicates the number of times the
- *		producer of entries into this Ring has looped around the
- *		ring.
- *		At initialization time, this value is set to 0. On the
- *		first loop, this value is set to 1. After the max value is
- *		reached allowed by the number of bits for this field, the
- *		count value continues with 0 again.
- *
- *		In case SW is the consumer of the ring entries, it can
- *		use this field to figure out up to where the producer of
- *		entries has created new entries. This eliminates the need to
- *		check where the head pointer' of the ring is located once
- *		the SW starts processing an interrupt indicating that new
- *		entries have been put into this ring...
- */
-
-enum hal_desc_owner {
-	HAL_DESC_OWNER_WBM,
-	HAL_DESC_OWNER_SW,
-	HAL_DESC_OWNER_TQM,
-	HAL_DESC_OWNER_RXDMA,
-	HAL_DESC_OWNER_REO,
-	HAL_DESC_OWNER_SWITCH,
-};
-
-enum hal_desc_buf_type {
-	HAL_DESC_BUF_TYPE_TX_MSDU_LINK,
-	HAL_DESC_BUF_TYPE_TX_MPDU_LINK,
-	HAL_DESC_BUF_TYPE_TX_MPDU_QUEUE_HEAD,
-	HAL_DESC_BUF_TYPE_TX_MPDU_QUEUE_EXT,
-	HAL_DESC_BUF_TYPE_TX_FLOW,
-	HAL_DESC_BUF_TYPE_TX_BUFFER,
-	HAL_DESC_BUF_TYPE_RX_MSDU_LINK,
-	HAL_DESC_BUF_TYPE_RX_MPDU_LINK,
-	HAL_DESC_BUF_TYPE_RX_REO_QUEUE,
-	HAL_DESC_BUF_TYPE_RX_REO_QUEUE_EXT,
-	HAL_DESC_BUF_TYPE_RX_BUFFER,
-	HAL_DESC_BUF_TYPE_IDLE_LINK,
-};
-
-#define HAL_DESC_REO_OWNED		4
-#define HAL_DESC_REO_QUEUE_DESC		8
-#define HAL_DESC_REO_QUEUE_EXT_DESC	9
-#define HAL_DESC_REO_NON_QOS_TID	16
-
-#define HAL_DESC_HDR_INFO0_OWNER	GENMASK(3, 0)
-#define HAL_DESC_HDR_INFO0_BUF_TYPE	GENMASK(7, 4)
-#define HAL_DESC_HDR_INFO0_DBG_RESERVED	GENMASK(31, 8)
-
-struct hal_desc_header {
-	__le32 info0;
-} __packed;
-
-struct hal_rx_mpdu_link_ptr {
-	struct ath12k_buffer_addr addr_info;
-} __packed;
-
-struct hal_rx_msdu_details {
-	struct ath12k_buffer_addr buf_addr_info;
-	struct rx_msdu_desc rx_msdu_info;
-	struct rx_msdu_ext_desc rx_msdu_ext_info;
-} __packed;
-
-#define HAL_RX_MSDU_LNK_INFO0_RX_QUEUE_NUMBER		GENMASK(15, 0)
-#define HAL_RX_MSDU_LNK_INFO0_FIRST_MSDU_LNK		BIT(16)
-
-struct hal_rx_msdu_link {
-	struct hal_desc_header desc_hdr;
-	struct ath12k_buffer_addr buf_addr_info;
-	__le32 info0;
-	__le32 pn[4];
-	struct hal_rx_msdu_details msdu_link[6];
-} __packed;
-
-struct hal_rx_reo_queue_ext {
-	struct hal_desc_header desc_hdr;
-	__le32 rsvd;
-	struct hal_rx_mpdu_link_ptr mpdu_link[15];
-} __packed;
-
-/* hal_rx_reo_queue_ext
- *	Consumer: REO
- *	Producer: REO
- *
- * descriptor_header
- *	Details about which module owns this struct.
- *
- * mpdu_link
- *	Pointer to the next MPDU_link descriptor in the MPDU queue.
- */
-
-enum hal_rx_reo_queue_pn_size {
-	HAL_RX_REO_QUEUE_PN_SIZE_24,
-	HAL_RX_REO_QUEUE_PN_SIZE_48,
-	HAL_RX_REO_QUEUE_PN_SIZE_128,
-};
-
-#define HAL_RX_REO_QUEUE_RX_QUEUE_NUMBER		GENMASK(15, 0)
-
-#define HAL_RX_REO_QUEUE_INFO0_VLD			BIT(0)
-#define HAL_RX_REO_QUEUE_INFO0_ASSOC_LNK_DESC_COUNTER	GENMASK(2, 1)
-#define HAL_RX_REO_QUEUE_INFO0_DIS_DUP_DETECTION	BIT(3)
-#define HAL_RX_REO_QUEUE_INFO0_SOFT_REORDER_EN		BIT(4)
-#define HAL_RX_REO_QUEUE_INFO0_AC			GENMASK(6, 5)
-#define HAL_RX_REO_QUEUE_INFO0_BAR			BIT(7)
-#define HAL_RX_REO_QUEUE_INFO0_RETRY			BIT(8)
-#define HAL_RX_REO_QUEUE_INFO0_CHECK_2K_MODE		BIT(9)
-#define HAL_RX_REO_QUEUE_INFO0_OOR_MODE			BIT(10)
-#define HAL_RX_REO_QUEUE_INFO0_BA_WINDOW_SIZE		GENMASK(20, 11)
-#define HAL_RX_REO_QUEUE_INFO0_PN_CHECK			BIT(21)
-#define HAL_RX_REO_QUEUE_INFO0_EVEN_PN			BIT(22)
-#define HAL_RX_REO_QUEUE_INFO0_UNEVEN_PN		BIT(23)
-#define HAL_RX_REO_QUEUE_INFO0_PN_HANDLE_ENABLE		BIT(24)
-#define HAL_RX_REO_QUEUE_INFO0_PN_SIZE			GENMASK(26, 25)
-#define HAL_RX_REO_QUEUE_INFO0_IGNORE_AMPDU_FLG		BIT(27)
-
-#define HAL_RX_REO_QUEUE_INFO1_SVLD			BIT(0)
-#define HAL_RX_REO_QUEUE_INFO1_SSN			GENMASK(12, 1)
-#define HAL_RX_REO_QUEUE_INFO1_CURRENT_IDX		GENMASK(22, 13)
-#define HAL_RX_REO_QUEUE_INFO1_SEQ_2K_ERR		BIT(23)
-#define HAL_RX_REO_QUEUE_INFO1_PN_ERR			BIT(24)
-#define HAL_RX_REO_QUEUE_INFO1_PN_VALID			BIT(31)
-
-#define HAL_RX_REO_QUEUE_INFO2_MPDU_COUNT		GENMASK(6, 0)
-#define HAL_RX_REO_QUEUE_INFO2_MSDU_COUNT		(31, 7)
-
-#define HAL_RX_REO_QUEUE_INFO3_TIMEOUT_COUNT		GENMASK(9, 4)
-#define HAL_RX_REO_QUEUE_INFO3_FWD_DUE_TO_BAR_CNT	GENMASK(15, 10)
-#define HAL_RX_REO_QUEUE_INFO3_DUPLICATE_COUNT		GENMASK(31, 16)
-
-#define HAL_RX_REO_QUEUE_INFO4_FRAME_IN_ORD_COUNT	GENMASK(23, 0)
-#define HAL_RX_REO_QUEUE_INFO4_BAR_RECVD_COUNT		GENMASK(31, 24)
-
-#define HAL_RX_REO_QUEUE_INFO5_LATE_RX_MPDU_COUNT	GENMASK(11, 0)
-#define HAL_RX_REO_QUEUE_INFO5_WINDOW_JUMP_2K		GENMASK(15, 12)
-#define HAL_RX_REO_QUEUE_INFO5_HOLE_COUNT		GENMASK(31, 16)
-
-struct hal_rx_reo_queue {
-	struct hal_desc_header desc_hdr;
-	__le32 rx_queue_num;
-	__le32 info0;
-	__le32 info1;
-	__le32 pn[4];
-	__le32 last_rx_enqueue_timestamp;
-	__le32 last_rx_dequeue_timestamp;
-	__le32 next_aging_queue[2];
-	__le32 prev_aging_queue[2];
-	__le32 rx_bitmap[9];
-	__le32 info2;
-	__le32 info3;
-	__le32 info4;
-	__le32 processed_mpdus;
-	__le32 processed_msdus;
-	__le32 processed_total_bytes;
-	__le32 info5;
-	__le32 rsvd[2];
-	struct hal_rx_reo_queue_ext ext_desc[];
-} __packed;
-
-/* hal_rx_reo_queue
- *
- * descriptor_header
- *	Details about which module owns this struct. Note that sub field
- *	Buffer_type shall be set to receive_reo_queue_descriptor.
- *
- * receive_queue_number
- *	Indicates the MPDU queue ID to which this MPDU link descriptor belongs.
- *
- * vld
- *	Valid bit indicating a session is established and the queue descriptor
- *	is valid.
- * associated_link_descriptor_counter
- *	Indicates which of the 3 link descriptor counters shall be incremented
- *	or decremented when link descriptors are added or removed from this
- *	flow queue.
- * disable_duplicate_detection
- *	When set, do not perform any duplicate detection.
- * soft_reorder_enable
- *	When set, REO has been instructed to not perform the actual re-ordering
- *	of frames for this queue, but just to insert the reorder opcodes.
- * ac
- *	Indicates the access category of the queue descriptor.
- * bar
- *	Indicates if BAR has been received.
- * retry
- *	Retry bit is checked if this bit is set.
- * chk_2k_mode
- *	Indicates what type of operation is expected from Reo when the received
- *	frame SN falls within the 2K window.
- * oor_mode
- *	Indicates what type of operation is expected when the received frame
- *	falls within the OOR window.
- * ba_window_size
- *	Indicates the negotiated (window size + 1). Max of 256 bits.
- *
- *	A value 255 means 256 bitmap, 63 means 64 bitmap, 0 (means non-BA
- *	session, with window size of 0). The 3 values here are the main values
- *	validated, but other values should work as well.
- *
- *	A BA window size of 0 (=> one frame entry bitmat), means that there is
- *	no additional rx_reo_queue_ext desc. following rx_reo_queue in memory.
- *	A BA window size of 1 - 105, means that there is 1 rx_reo_queue_ext.
- *	A BA window size of 106 - 210, means that there are 2 rx_reo_queue_ext.
- *	A BA window size of 211 - 256, means that there are 3 rx_reo_queue_ext.
- * pn_check_needed, pn_shall_be_even, pn_shall_be_uneven, pn_handling_enable,
- * pn_size
- *	REO shall perform the PN increment check, even number check, uneven
- *	number check, PN error check and size of the PN field check.
- * ignore_ampdu_flag
- *	REO shall ignore the ampdu_flag on entrance descriptor for this queue.
- *
- * svld
- *	Sequence number in next field is valid one.
- * ssn
- *	 Starting Sequence number of the session.
- * current_index
- *	Points to last forwarded packet
- * seq_2k_error_detected_flag
- *	REO has detected a 2k error jump in the sequence number and from that
- *	moment forward, all new frames are forwarded directly to FW, without
- *	duplicate detect, reordering, etc.
- * pn_error_detected_flag
- *	REO has detected a PN error.
- */
-
-#define HAL_REO_UPD_RX_QUEUE_INFO0_QUEUE_ADDR_HI		GENMASK(7, 0)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_RX_QUEUE_NUM		BIT(8)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_VLD			BIT(9)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_ASSOC_LNK_DESC_CNT	BIT(10)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_DIS_DUP_DETECTION	BIT(11)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_SOFT_REORDER_EN		BIT(12)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_AC			BIT(13)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_BAR			BIT(14)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_RETRY			BIT(15)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_CHECK_2K_MODE		BIT(16)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_OOR_MODE			BIT(17)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_BA_WINDOW_SIZE		BIT(18)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_CHECK			BIT(19)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_EVEN_PN			BIT(20)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_UNEVEN_PN		BIT(21)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_HANDLE_ENABLE		BIT(22)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_SIZE			BIT(23)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_IGNORE_AMPDU_FLG		BIT(24)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_SVLD			BIT(25)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_SSN			BIT(26)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_SEQ_2K_ERR		BIT(27)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_ERR			BIT(28)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_VALID			BIT(29)
-#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN			BIT(30)
-
-#define HAL_REO_UPD_RX_QUEUE_INFO1_RX_QUEUE_NUMBER		GENMASK(15, 0)
-#define HAL_REO_UPD_RX_QUEUE_INFO1_VLD				BIT(16)
-#define HAL_REO_UPD_RX_QUEUE_INFO1_ASSOC_LNK_DESC_COUNTER	GENMASK(18, 17)
-#define HAL_REO_UPD_RX_QUEUE_INFO1_DIS_DUP_DETECTION		BIT(19)
-#define HAL_REO_UPD_RX_QUEUE_INFO1_SOFT_REORDER_EN		BIT(20)
-#define HAL_REO_UPD_RX_QUEUE_INFO1_AC				GENMASK(22, 21)
-#define HAL_REO_UPD_RX_QUEUE_INFO1_BAR				BIT(23)
-#define HAL_REO_UPD_RX_QUEUE_INFO1_RETRY			BIT(24)
-#define HAL_REO_UPD_RX_QUEUE_INFO1_CHECK_2K_MODE		BIT(25)
-#define HAL_REO_UPD_RX_QUEUE_INFO1_OOR_MODE			BIT(26)
-#define HAL_REO_UPD_RX_QUEUE_INFO1_PN_CHECK			BIT(27)
-#define HAL_REO_UPD_RX_QUEUE_INFO1_EVEN_PN			BIT(28)
-#define HAL_REO_UPD_RX_QUEUE_INFO1_UNEVEN_PN			BIT(29)
-#define HAL_REO_UPD_RX_QUEUE_INFO1_PN_HANDLE_ENABLE		BIT(30)
-#define HAL_REO_UPD_RX_QUEUE_INFO1_IGNORE_AMPDU_FLG		BIT(31)
-
-#define HAL_REO_UPD_RX_QUEUE_INFO2_BA_WINDOW_SIZE		GENMASK(9, 0)
-#define HAL_REO_UPD_RX_QUEUE_INFO2_PN_SIZE			GENMASK(11, 10)
-#define HAL_REO_UPD_RX_QUEUE_INFO2_SVLD				BIT(12)
-#define HAL_REO_UPD_RX_QUEUE_INFO2_SSN				GENMASK(24, 13)
-#define HAL_REO_UPD_RX_QUEUE_INFO2_SEQ_2K_ERR			BIT(25)
-#define HAL_REO_UPD_RX_QUEUE_INFO2_PN_ERR			BIT(26)
-#define HAL_REO_UPD_RX_QUEUE_INFO2_PN_VALID			BIT(27)
-
-struct hal_reo_update_rx_queue {
-	struct hal_reo_cmd_hdr cmd;
-	__le32 queue_addr_lo;
-	__le32 info0;
-	__le32 info1;
-	__le32 info2;
-	__le32 pn[4];
-} __packed;
-
-struct hal_rx_reo_queue_1k {
-	struct hal_desc_header desc_hdr;
-	__le32 rx_bitmap_1023_288[23];
-	__le32 reserved[8];
-} __packed;
-
-#define HAL_REO_UNBLOCK_CACHE_INFO0_UNBLK_CACHE		BIT(0)
-#define HAL_REO_UNBLOCK_CACHE_INFO0_RESOURCE_IDX	GENMASK(2, 1)
-
-struct hal_reo_unblock_cache {
-	struct hal_reo_cmd_hdr cmd;
-	__le32 info0;
-	__le32 rsvd[7];
-} __packed;
-
-enum hal_reo_exec_status {
-	HAL_REO_EXEC_STATUS_SUCCESS,
-	HAL_REO_EXEC_STATUS_BLOCKED,
-	HAL_REO_EXEC_STATUS_FAILED,
-	HAL_REO_EXEC_STATUS_RESOURCE_BLOCKED,
-};
-
-#define HAL_REO_STATUS_HDR_INFO0_STATUS_NUM	GENMASK(15, 0)
-#define HAL_REO_STATUS_HDR_INFO0_EXEC_TIME	GENMASK(25, 16)
-#define HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS	GENMASK(27, 26)
-
-struct hal_reo_status_hdr {
-	__le32 info0;
-	__le32 timestamp;
-} __packed;
-
-/* hal_reo_status_hdr
- *		Producer: REO
- *		Consumer: SW
- *
- * status_num
- *		The value in this field is equal to value of the reo command
- *		number. This field helps to correlate the statuses with the REO
- *		commands.
- *
- * execution_time (in us)
- *		The amount of time REO took to execute the command. Note that
- *		this time does not include the duration of the command waiting
- *		in the command ring, before the execution started.
- *
- * execution_status
- *		Execution status of the command. Values are defined in
- *		enum %HAL_REO_EXEC_STATUS_.
- */
-#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO0_SSN		GENMASK(11, 0)
-#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO0_CUR_IDX		GENMASK(21, 12)
-
-#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO1_MPDU_COUNT		GENMASK(6, 0)
-#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO1_MSDU_COUNT		GENMASK(31, 7)
-
-#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO2_WINDOW_JMP2K	GENMASK(3, 0)
-#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO2_TIMEOUT_COUNT	GENMASK(9, 4)
-#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO2_FDTB_COUNT		GENMASK(15, 10)
-#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO2_DUPLICATE_COUNT	GENMASK(31, 16)
-
-#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO3_FIO_COUNT		GENMASK(23, 0)
-#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO3_BAR_RCVD_CNT	GENMASK(31, 24)
-
-#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO4_LATE_RX_MPDU	GENMASK(11, 0)
-#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO4_HOLE_COUNT		GENMASK(27, 12)
-
-#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO5_LOOPING_CNT	GENMASK(31, 28)
-
-struct hal_reo_get_queue_stats_status {
-	struct hal_reo_status_hdr hdr;
-	__le32 info0;
-	__le32 pn[4];
-	__le32 last_rx_enqueue_timestamp;
-	__le32 last_rx_dequeue_timestamp;
-	__le32 rx_bitmap[9];
-	__le32 info1;
-	__le32 info2;
-	__le32 info3;
-	__le32 num_mpdu_frames;
-	__le32 num_msdu_frames;
-	__le32 total_bytes;
-	__le32 info4;
-	__le32 info5;
-} __packed;
-
-/* hal_reo_get_queue_stats_status
- *		Producer: REO
- *		Consumer: SW
- *
- * status_hdr
- *		Details that can link this status with the original command. It
- *		also contains info on how long REO took to execute this command.
- *
- * ssn
- *		Starting Sequence number of the session, this changes whenever
- *		window moves (can be filled by SW then maintained by REO).
- *
- * current_index
- *		Points to last forwarded packet.
- *
- * pn
- *		Bits of the PN number.
- *
- * last_rx_enqueue_timestamp
- * last_rx_dequeue_timestamp
- *		Timestamp of arrival of the last MPDU for this queue and
- *		Timestamp of forwarding an MPDU accordingly.
- *
- * rx_bitmap
- *		When a bit is set, the corresponding frame is currently held
- *		in the re-order queue. The bitmap  is Fully managed by HW.
- *
- * current_mpdu_count
- * current_msdu_count
- *		The number of MPDUs and MSDUs in the queue.
- *
- * timeout_count
- *		The number of times REO started forwarding frames even though
- *		there is a hole in the bitmap. Forwarding reason is timeout.
- *
- * forward_due_to_bar_count
- *		The number of times REO started forwarding frames even though
- *		there is a hole in the bitmap. Fwd reason is reception of BAR.
- *
- * duplicate_count
- *		The number of duplicate frames that have been detected.
- *
- * frames_in_order_count
- *		The number of frames that have been received in order (without
- *		a hole that prevented them from being forwarded immediately).
- *
- * bar_received_count
- *		The number of times a BAR frame is received.
- *
- * mpdu_frames_processed_count
- * msdu_frames_processed_count
- *		The total number of MPDU/MSDU frames that have been processed.
- *
- * total_bytes
- *		An approximation of the number of bytes received for this queue.
- *
- * late_receive_mpdu_count
- *		The number of MPDUs received after the window had already moved
- *		on. The 'late' sequence window is defined as
- *		(Window SSN - 256) - (Window SSN - 1).
- *
- * window_jump_2k
- *		The number of times the window moved more than 2K
- *
- * hole_count
- *		The number of times a hole was created in the receive bitmap.
- *
- * looping_count
- *		A count value that indicates the number of times the producer of
- *		entries into this Ring has looped around the ring.
- */
-
-#define HAL_REO_STATUS_LOOP_CNT			GENMASK(31, 28)
-
-#define HAL_REO_FLUSH_QUEUE_INFO0_ERR_DETECTED	BIT(0)
-#define HAL_REO_FLUSH_QUEUE_INFO0_RSVD		GENMASK(31, 1)
-#define HAL_REO_FLUSH_QUEUE_INFO1_RSVD		GENMASK(27, 0)
-
-struct hal_reo_flush_queue_status {
-	struct hal_reo_status_hdr hdr;
-	__le32 info0;
-	__le32 rsvd0[21];
-	__le32 info1;
-} __packed;
-
-/* hal_reo_flush_queue_status
- *		Producer: REO
- *		Consumer: SW
- *
- * status_hdr
- *		Details that can link this status with the original command. It
- *		also contains info on how long REO took to execute this command.
- *
- * error_detected
- *		Status of blocking resource
- *
- *		0 - No error has been detected while executing this command
- *		1 - Error detected. The resource to be used for blocking was
- *		    already in use.
- *
- * looping_count
- *		A count value that indicates the number of times the producer of
- *		entries into this Ring has looped around the ring.
- */
-
-#define HAL_REO_FLUSH_CACHE_STATUS_INFO0_IS_ERR			BIT(0)
-#define HAL_REO_FLUSH_CACHE_STATUS_INFO0_BLOCK_ERR_CODE		GENMASK(2, 1)
-#define HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_STATUS_HIT	BIT(8)
-#define HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_DESC_TYPE	GENMASK(11, 9)
-#define HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_CLIENT_ID	GENMASK(15, 12)
-#define HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_ERR		GENMASK(17, 16)
-#define HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_COUNT		GENMASK(25, 18)
-
-struct hal_reo_flush_cache_status {
-	struct hal_reo_status_hdr hdr;
-	__le32 info0;
-	__le32 rsvd0[21];
-	__le32 info1;
-} __packed;
-
-/* hal_reo_flush_cache_status
- *		Producer: REO
- *		Consumer: SW
- *
- * status_hdr
- *		Details that can link this status with the original command. It
- *		also contains info on how long REO took to execute this command.
- *
- * error_detected
- *		Status for blocking resource handling
- *
- *		0 - No error has been detected while executing this command
- *		1 - An error in the blocking resource management was detected
- *
- * block_error_details
- *		only valid when error_detected is set
- *
- *		0 - No blocking related errors found
- *		1 - Blocking resource is already in use
- *		2 - Resource requested to be unblocked, was not blocked
- *
- * cache_controller_flush_status_hit
- *		The status that the cache controller returned on executing the
- *		flush command.
- *
- *		0 - miss; 1 - hit
- *
- * cache_controller_flush_status_desc_type
- *		Flush descriptor type
- *
- * cache_controller_flush_status_client_id
- *		Module who made the flush request
- *
- *		In REO, this is always 0
- *
- * cache_controller_flush_status_error
- *		Error condition
- *
- *		0 - No error found
- *		1 - HW interface is still busy
- *		2 - Line currently locked. Used for one line flush command
- *		3 - At least one line is still locked.
- *		    Used for cache flush command.
- *
- * cache_controller_flush_count
- *		The number of lines that were actually flushed out
- *
- * looping_count
- *		A count value that indicates the number of times the producer of
- *		entries into this Ring has looped around the ring.
- */
-
-#define HAL_REO_UNBLOCK_CACHE_STATUS_INFO0_IS_ERR	BIT(0)
-#define HAL_REO_UNBLOCK_CACHE_STATUS_INFO0_TYPE		BIT(1)
-
-struct hal_reo_unblock_cache_status {
-	struct hal_reo_status_hdr hdr;
-	__le32 info0;
-	__le32 rsvd0[21];
-	__le32 info1;
-} __packed;
-
-/* hal_reo_unblock_cache_status
- *		Producer: REO
- *		Consumer: SW
- *
- * status_hdr
- *		Details that can link this status with the original command. It
- *		also contains info on how long REO took to execute this command.
- *
- * error_detected
- *		0 - No error has been detected while executing this command
- *		1 - The blocking resource was not in use, and therefore it could
- *		    not be unblocked.
- *
- * unblock_type
- *		Reference to the type of unblock command
- *		0 - Unblock a blocking resource
- *		1 - The entire cache usage is unblock
- *
- * looping_count
- *		A count value that indicates the number of times the producer of
- *		entries into this Ring has looped around the ring.
- */
-
-#define HAL_REO_FLUSH_TIMEOUT_STATUS_INFO0_IS_ERR		BIT(0)
-#define HAL_REO_FLUSH_TIMEOUT_STATUS_INFO0_LIST_EMPTY		BIT(1)
-
-#define HAL_REO_FLUSH_TIMEOUT_STATUS_INFO1_REL_DESC_COUNT	GENMASK(15, 0)
-#define HAL_REO_FLUSH_TIMEOUT_STATUS_INFO1_FWD_BUF_COUNT	GENMASK(31, 16)
-
-struct hal_reo_flush_timeout_list_status {
-	struct hal_reo_status_hdr hdr;
-	__le32 info0;
-	__le32 info1;
-	__le32 rsvd0[20];
-	__le32 info2;
-} __packed;
-
-/* hal_reo_flush_timeout_list_status
- *		Producer: REO
- *		Consumer: SW
- *
- * status_hdr
- *		Details that can link this status with the original command. It
- *		also contains info on how long REO took to execute this command.
- *
- * error_detected
- *		0 - No error has been detected while executing this command
- *		1 - Command not properly executed and returned with error
- *
- * timeout_list_empty
- *		When set, REO has depleted the timeout list and all entries are
- *		gone.
- *
- * release_desc_count
- *		Producer: SW; Consumer: REO
- *		The number of link descriptor released
- *
- * forward_buf_count
- *		Producer: SW; Consumer: REO
- *		The number of buffers forwarded to the REO destination rings
- *
- * looping_count
- *		A count value that indicates the number of times the producer of
- *		entries into this Ring has looped around the ring.
- */
-
-#define HAL_REO_DESC_THRESH_STATUS_INFO0_THRESH_INDEX		GENMASK(1, 0)
-#define HAL_REO_DESC_THRESH_STATUS_INFO1_LINK_DESC_COUNTER0	GENMASK(23, 0)
-#define HAL_REO_DESC_THRESH_STATUS_INFO2_LINK_DESC_COUNTER1	GENMASK(23, 0)
-#define HAL_REO_DESC_THRESH_STATUS_INFO3_LINK_DESC_COUNTER2	GENMASK(23, 0)
-#define HAL_REO_DESC_THRESH_STATUS_INFO4_LINK_DESC_COUNTER_SUM	GENMASK(25, 0)
-
-struct hal_reo_desc_thresh_reached_status {
-	struct hal_reo_status_hdr hdr;
-	__le32 info0;
-	__le32 info1;
-	__le32 info2;
-	__le32 info3;
-	__le32 info4;
-	__le32 rsvd0[17];
-	__le32 info5;
-} __packed;
-
-/* hal_reo_desc_thresh_reached_status
- *		Producer: REO
- *		Consumer: SW
- *
- * status_hdr
- *		Details that can link this status with the original command. It
- *		also contains info on how long REO took to execute this command.
- *
- * threshold_index
- *		The index of the threshold register whose value got reached
- *
- * link_descriptor_counter0
- * link_descriptor_counter1
- * link_descriptor_counter2
- * link_descriptor_counter_sum
- *		Value of the respective counters at generation of this message
- *
- * looping_count
- *		A count value that indicates the number of times the producer of
- *		entries into this Ring has looped around the ring.
- */
-
-#define HAL_TCL_ENTRANCE_FROM_PPE_RING_INFO0_DATA_LENGTH	GENMASK(13, 0)
-#define HAL_TCL_ENTRANCE_FROM_PPE_RING_INFO0_L4_CSUM_STATUS	BIT(14)
-#define HAL_TCL_ENTRANCE_FROM_PPE_RING_INFO0_L3_CSUM_STATUS	BIT(15)
-#define HAL_TCL_ENTRANCE_FROM_PPE_RING_INFO0_PID		GENMASK(27, 24)
-#define HAL_TCL_ENTRANCE_FROM_PPE_RING_INFO0_QDISC		BIT(28)
-#define HAL_TCL_ENTRANCE_FROM_PPE_RING_INFO0_MULTICAST	BIT(29)
-#define HAL_TCL_ENTRANCE_FROM_PPE_RING_INFO0_MORE		BIT(30)
-#define HAL_TCL_ENTRANCE_FROM_PPE_RING_INFO0_VALID_TOGGLE	BIT(31)
-
-struct hal_tcl_entrance_from_ppe_ring {
-	__le32 buffer_addr;
-	__le32 info0;
-} __packed;
-
-struct hal_mon_buf_ring {
-	__le32 paddr_lo;
-	__le32 paddr_hi;
-	__le64 cookie;
-};
-
-/* hal_mon_buf_ring
- *	Producer : SW
- *	Consumer : Monitor
- *
- * paddr_lo
- *	Lower 32-bit physical address of the buffer pointer from the source ring.
- * paddr_hi
- *	bit range 7-0 : upper 8 bit of the physical address.
- *	bit range 31-8 : reserved.
- * cookie
- *	Consumer: RxMon/TxMon 64 bit cookie of the buffers.
- */
-
-#define HAL_MON_DEST_COOKIE_BUF_ID      GENMASK(17, 0)
-
-#define HAL_MON_DEST_INFO0_END_OFFSET		GENMASK(11, 0)
-#define HAL_MON_DEST_INFO0_END_REASON		GENMASK(17, 16)
-#define HAL_MON_DEST_INFO0_INITIATOR		BIT(18)
-#define HAL_MON_DEST_INFO0_EMPTY_DESC		BIT(19)
-#define HAL_MON_DEST_INFO0_RING_ID		GENMASK(27, 20)
-#define HAL_MON_DEST_INFO0_LOOPING_COUNT	GENMASK(31, 28)
-
-struct hal_mon_dest_desc {
-	__le32 cookie;
-	__le32 reserved;
-	__le32 ppdu_id;
-	__le32 info0;
-};
-
-/* hal_mon_dest_ring
- *	Producer : TxMon/RxMon
- *	Consumer : SW
- * cookie
- *	bit 0 -17 buf_id to track the skb's vaddr.
- * ppdu_id
- *	Phy ppdu_id
- * end_offset
- *	The offset into status buffer where DMA ended, ie., offset to the last
- *	TLV + last TLV size.
- * flush_detected
- *	Indicates whether 'tx_flush' or 'rx_flush' occurred.
- * end_of_ppdu
- *	Indicates end of ppdu.
- * pmac_id
- *	Indicates PMAC that received from frame.
- * empty_descriptor
- *	This descriptor is written on flush or end of ppdu or end of status
- *	buffer.
- * ring_id
- *	updated by SRNG.
- * looping_count
- *	updated by SRNG.
- */
-
-#define HAL_TX_MSDU_METADATA_INFO0_ENCRYPT_FLAG		BIT(8)
-#define HAL_TX_MSDU_METADATA_INFO0_ENCRYPT_TYPE		GENMASK(16, 15)
-#define HAL_TX_MSDU_METADATA_INFO0_HOST_TX_DESC_POOL	BIT(31)
-
-struct hal_tx_msdu_metadata {
-	__le32 info0;
-	__le32 rsvd0[6];
-} __packed;
-
-/* hal_tx_msdu_metadata
- * valid_encrypt_type
- *		if set, encrypt type is valid
- * encrypt_type
- *		0 = NO_ENCRYPT,
- *		1 = ENCRYPT,
- *		2 ~ 3 - Reserved
- * host_tx_desc_pool
- *		If set, Firmware allocates tx_descriptors
- *		in WAL_BUFFERID_TX_HOST_DATA_EXP,instead
- *		of WAL_BUFFERID_TX_TCL_DATA_EXP.
- *		Use cases:
- *		Any time firmware uses TQM-BYPASS for Data
- *		TID, firmware expect host to set this bit.
- */
-
-#endif /* ATH12K_HAL_DESC_H */
diff --git a/drivers/net/wireless/ath/ath12k/hal_rx.c b/drivers/net/wireless/ath/ath12k/hal_rx.c
deleted file mode 100644
index c4443ca05cd6..000000000000
--- a/drivers/net/wireless/ath/ath12k/hal_rx.c
+++ /dev/null
@@ -1,947 +0,0 @@
-// SPDX-License-Identifier: BSD-3-Clause-Clear
-/*
- * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
- */
-
-#include "debug.h"
-#include "hal.h"
-#include "hal_tx.h"
-#include "hal_rx.h"
-#include "hal_desc.h"
-#include "hif.h"
-
-static void ath12k_hal_reo_set_desc_hdr(struct hal_desc_header *hdr,
-					u8 owner, u8 buffer_type, u32 magic)
-{
-	hdr->info0 = le32_encode_bits(owner, HAL_DESC_HDR_INFO0_OWNER) |
-		     le32_encode_bits(buffer_type, HAL_DESC_HDR_INFO0_BUF_TYPE);
-
-	/* Magic pattern in reserved bits for debugging */
-	hdr->info0 |= le32_encode_bits(magic, HAL_DESC_HDR_INFO0_DBG_RESERVED);
-}
-
-static int ath12k_hal_reo_cmd_queue_stats(struct hal_tlv_64_hdr *tlv,
-					  struct ath12k_hal_reo_cmd *cmd)
-{
-	struct hal_reo_get_queue_stats *desc;
-
-	tlv->tl = le64_encode_bits(HAL_REO_GET_QUEUE_STATS, HAL_TLV_HDR_TAG) |
-		  le64_encode_bits(sizeof(*desc), HAL_TLV_HDR_LEN);
-
-	desc = (struct hal_reo_get_queue_stats *)tlv->value;
-	memset_startat(desc, 0, queue_addr_lo);
-
-	desc->cmd.info0 &= ~cpu_to_le32(HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED);
-	if (cmd->flag & HAL_REO_CMD_FLG_NEED_STATUS)
-		desc->cmd.info0 |= cpu_to_le32(HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED);
-
-	desc->queue_addr_lo = cpu_to_le32(cmd->addr_lo);
-	desc->info0 = le32_encode_bits(cmd->addr_hi,
-				       HAL_REO_GET_QUEUE_STATS_INFO0_QUEUE_ADDR_HI);
-	if (cmd->flag & HAL_REO_CMD_FLG_STATS_CLEAR)
-		desc->info0 |= cpu_to_le32(HAL_REO_GET_QUEUE_STATS_INFO0_CLEAR_STATS);
-
-	return le32_get_bits(desc->cmd.info0, HAL_REO_CMD_HDR_INFO0_CMD_NUMBER);
-}
-
-static int ath12k_hal_reo_cmd_flush_cache(struct ath12k_hal *hal,
-					  struct hal_tlv_64_hdr *tlv,
-					  struct ath12k_hal_reo_cmd *cmd)
-{
-	struct hal_reo_flush_cache *desc;
-	u8 avail_slot = ffz(hal->avail_blk_resource);
-
-	if (cmd->flag & HAL_REO_CMD_FLG_FLUSH_BLOCK_LATER) {
-		if (avail_slot >= HAL_MAX_AVAIL_BLK_RES)
-			return -ENOSPC;
-
-		hal->current_blk_index = avail_slot;
-	}
-
-	tlv->tl = le64_encode_bits(HAL_REO_FLUSH_CACHE, HAL_TLV_HDR_TAG) |
-		  le64_encode_bits(sizeof(*desc), HAL_TLV_HDR_LEN);
-
-	desc = (struct hal_reo_flush_cache *)tlv->value;
-	memset_startat(desc, 0, cache_addr_lo);
-
-	desc->cmd.info0 &= ~cpu_to_le32(HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED);
-	if (cmd->flag & HAL_REO_CMD_FLG_NEED_STATUS)
-		desc->cmd.info0 |= cpu_to_le32(HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED);
-
-	desc->cache_addr_lo = cpu_to_le32(cmd->addr_lo);
-	desc->info0 = le32_encode_bits(cmd->addr_hi,
-				       HAL_REO_FLUSH_CACHE_INFO0_CACHE_ADDR_HI);
-
-	if (cmd->flag & HAL_REO_CMD_FLG_FLUSH_FWD_ALL_MPDUS)
-		desc->info0 |= cpu_to_le32(HAL_REO_FLUSH_CACHE_INFO0_FWD_ALL_MPDUS);
-
-	if (cmd->flag & HAL_REO_CMD_FLG_FLUSH_BLOCK_LATER) {
-		desc->info0 |= cpu_to_le32(HAL_REO_FLUSH_CACHE_INFO0_BLOCK_CACHE_USAGE);
-		desc->info0 |=
-			le32_encode_bits(avail_slot,
-					 HAL_REO_FLUSH_CACHE_INFO0_BLOCK_RESRC_IDX);
-	}
-
-	if (cmd->flag & HAL_REO_CMD_FLG_FLUSH_NO_INVAL)
-		desc->info0 |= cpu_to_le32(HAL_REO_FLUSH_CACHE_INFO0_FLUSH_WO_INVALIDATE);
-
-	if (cmd->flag & HAL_REO_CMD_FLG_FLUSH_ALL)
-		desc->info0 |= cpu_to_le32(HAL_REO_FLUSH_CACHE_INFO0_FLUSH_ALL);
-
-	if (cmd->flag & HAL_REO_CMD_FLG_FLUSH_QUEUE_1K_DESC)
-		desc->info0 |= cpu_to_le32(HAL_REO_FLUSH_CACHE_INFO0_FLUSH_QUEUE_1K_DESC);
-
-	return le32_get_bits(desc->cmd.info0, HAL_REO_CMD_HDR_INFO0_CMD_NUMBER);
-}
-
-static int ath12k_hal_reo_cmd_update_rx_queue(struct hal_tlv_64_hdr *tlv,
-					      struct ath12k_hal_reo_cmd *cmd)
-{
-	struct hal_reo_update_rx_queue *desc;
-
-	tlv->tl = le64_encode_bits(HAL_REO_UPDATE_RX_REO_QUEUE, HAL_TLV_HDR_TAG) |
-		  le64_encode_bits(sizeof(*desc), HAL_TLV_HDR_LEN);
-
-	desc = (struct hal_reo_update_rx_queue *)tlv->value;
-	memset_startat(desc, 0, queue_addr_lo);
-
-	desc->cmd.info0 &= ~cpu_to_le32(HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED);
-	if (cmd->flag & HAL_REO_CMD_FLG_NEED_STATUS)
-		desc->cmd.info0 |= cpu_to_le32(HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED);
-
-	desc->queue_addr_lo = cpu_to_le32(cmd->addr_lo);
-	desc->info0 =
-		le32_encode_bits(cmd->addr_hi,
-				 HAL_REO_UPD_RX_QUEUE_INFO0_QUEUE_ADDR_HI) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_RX_QUEUE_NUM),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_RX_QUEUE_NUM) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_VLD),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_VLD) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_ALDC),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_ASSOC_LNK_DESC_CNT) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_DIS_DUP_DETECTION),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_DIS_DUP_DETECTION) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_SOFT_REORDER_EN),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_SOFT_REORDER_EN) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_AC),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_AC) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_BAR),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_BAR) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_RETRY),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_RETRY) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_CHECK_2K_MODE),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_CHECK_2K_MODE) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_OOR_MODE),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_OOR_MODE) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_BA_WINDOW_SIZE),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_BA_WINDOW_SIZE) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_PN_CHECK),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_CHECK) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_EVEN_PN),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_EVEN_PN) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_UNEVEN_PN),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_UNEVEN_PN) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_PN_HANDLE_ENABLE),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_HANDLE_ENABLE) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_PN_SIZE),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_SIZE) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_IGNORE_AMPDU_FLG),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_IGNORE_AMPDU_FLG) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_SVLD),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_SVLD) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_SSN),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_SSN) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_SEQ_2K_ERR),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_SEQ_2K_ERR) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_PN_VALID),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_VALID) |
-		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_PN),
-				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN);
-
-	desc->info1 =
-		le32_encode_bits(cmd->rx_queue_num,
-				 HAL_REO_UPD_RX_QUEUE_INFO1_RX_QUEUE_NUMBER) |
-		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_VLD),
-				 HAL_REO_UPD_RX_QUEUE_INFO1_VLD) |
-		le32_encode_bits(u32_get_bits(cmd->upd1, HAL_REO_CMD_UPD1_ALDC),
-				 HAL_REO_UPD_RX_QUEUE_INFO1_ASSOC_LNK_DESC_COUNTER) |
-		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_DIS_DUP_DETECTION),
-				 HAL_REO_UPD_RX_QUEUE_INFO1_DIS_DUP_DETECTION) |
-		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_SOFT_REORDER_EN),
-				 HAL_REO_UPD_RX_QUEUE_INFO1_SOFT_REORDER_EN) |
-		le32_encode_bits(u32_get_bits(cmd->upd1, HAL_REO_CMD_UPD1_AC),
-				 HAL_REO_UPD_RX_QUEUE_INFO1_AC) |
-		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_BAR),
-				 HAL_REO_UPD_RX_QUEUE_INFO1_BAR) |
-		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_CHECK_2K_MODE),
-				 HAL_REO_UPD_RX_QUEUE_INFO1_CHECK_2K_MODE) |
-		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_RETRY),
-				 HAL_REO_UPD_RX_QUEUE_INFO1_RETRY) |
-		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_OOR_MODE),
-				 HAL_REO_UPD_RX_QUEUE_INFO1_OOR_MODE) |
-		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_PN_CHECK),
-				 HAL_REO_UPD_RX_QUEUE_INFO1_PN_CHECK) |
-		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_EVEN_PN),
-				 HAL_REO_UPD_RX_QUEUE_INFO1_EVEN_PN) |
-		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_UNEVEN_PN),
-				 HAL_REO_UPD_RX_QUEUE_INFO1_UNEVEN_PN) |
-		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_PN_HANDLE_ENABLE),
-				 HAL_REO_UPD_RX_QUEUE_INFO1_PN_HANDLE_ENABLE) |
-		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_IGNORE_AMPDU_FLG),
-				 HAL_REO_UPD_RX_QUEUE_INFO1_IGNORE_AMPDU_FLG);
-
-	if (cmd->pn_size == 24)
-		cmd->pn_size = HAL_RX_REO_QUEUE_PN_SIZE_24;
-	else if (cmd->pn_size == 48)
-		cmd->pn_size = HAL_RX_REO_QUEUE_PN_SIZE_48;
-	else if (cmd->pn_size == 128)
-		cmd->pn_size = HAL_RX_REO_QUEUE_PN_SIZE_128;
-
-	if (cmd->ba_window_size < 1)
-		cmd->ba_window_size = 1;
-
-	if (cmd->ba_window_size == 1)
-		cmd->ba_window_size++;
-
-	desc->info2 =
-		le32_encode_bits(cmd->ba_window_size - 1,
-				 HAL_REO_UPD_RX_QUEUE_INFO2_BA_WINDOW_SIZE) |
-		le32_encode_bits(cmd->pn_size, HAL_REO_UPD_RX_QUEUE_INFO2_PN_SIZE) |
-		le32_encode_bits(!!(cmd->upd2 & HAL_REO_CMD_UPD2_SVLD),
-				 HAL_REO_UPD_RX_QUEUE_INFO2_SVLD) |
-		le32_encode_bits(u32_get_bits(cmd->upd2, HAL_REO_CMD_UPD2_SSN),
-				 HAL_REO_UPD_RX_QUEUE_INFO2_SSN) |
-		le32_encode_bits(!!(cmd->upd2 & HAL_REO_CMD_UPD2_SEQ_2K_ERR),
-				 HAL_REO_UPD_RX_QUEUE_INFO2_SEQ_2K_ERR) |
-		le32_encode_bits(!!(cmd->upd2 & HAL_REO_CMD_UPD2_PN_ERR),
-				 HAL_REO_UPD_RX_QUEUE_INFO2_PN_ERR);
-
-	return le32_get_bits(desc->cmd.info0, HAL_REO_CMD_HDR_INFO0_CMD_NUMBER);
-}
-
-int ath12k_hal_reo_cmd_send(struct ath12k_base *ab, struct hal_srng *srng,
-			    enum hal_reo_cmd_type type,
-			    struct ath12k_hal_reo_cmd *cmd)
-{
-	struct hal_tlv_64_hdr *reo_desc;
-	int ret;
-
-	spin_lock_bh(&srng->lock);
-
-	ath12k_hal_srng_access_begin(ab, srng);
-	reo_desc = ath12k_hal_srng_src_get_next_entry(ab, srng);
-	if (!reo_desc) {
-		ret = -ENOBUFS;
-		goto out;
-	}
-
-	switch (type) {
-	case HAL_REO_CMD_GET_QUEUE_STATS:
-		ret = ath12k_hal_reo_cmd_queue_stats(reo_desc, cmd);
-		break;
-	case HAL_REO_CMD_FLUSH_CACHE:
-		ret = ath12k_hal_reo_cmd_flush_cache(&ab->hal, reo_desc, cmd);
-		break;
-	case HAL_REO_CMD_UPDATE_RX_QUEUE:
-		ret = ath12k_hal_reo_cmd_update_rx_queue(reo_desc, cmd);
-		break;
-	case HAL_REO_CMD_FLUSH_QUEUE:
-	case HAL_REO_CMD_UNBLOCK_CACHE:
-	case HAL_REO_CMD_FLUSH_TIMEOUT_LIST:
-		ath12k_warn(ab, "Unsupported reo command %d\n", type);
-		ret = -EOPNOTSUPP;
-		break;
-	default:
-		ath12k_warn(ab, "Unknown reo command %d\n", type);
-		ret = -EINVAL;
-		break;
-	}
-
-out:
-	ath12k_hal_srng_access_end(ab, srng);
-	spin_unlock_bh(&srng->lock);
-
-	return ret;
-}
-
-void ath12k_hal_rx_buf_addr_info_set(struct ath12k_buffer_addr *binfo,
-				     dma_addr_t paddr, u32 cookie, u8 manager)
-{
-	u32 paddr_lo, paddr_hi;
-
-	paddr_lo = lower_32_bits(paddr);
-	paddr_hi = upper_32_bits(paddr);
-	binfo->info0 = le32_encode_bits(paddr_lo, BUFFER_ADDR_INFO0_ADDR);
-	binfo->info1 = le32_encode_bits(paddr_hi, BUFFER_ADDR_INFO1_ADDR) |
-		       le32_encode_bits(cookie, BUFFER_ADDR_INFO1_SW_COOKIE) |
-		       le32_encode_bits(manager, BUFFER_ADDR_INFO1_RET_BUF_MGR);
-}
-
-void ath12k_hal_rx_buf_addr_info_get(struct ath12k_buffer_addr *binfo,
-				     dma_addr_t *paddr,
-				     u32 *cookie, u8 *rbm)
-{
-	*paddr = (((u64)le32_get_bits(binfo->info1, BUFFER_ADDR_INFO1_ADDR)) << 32) |
-		le32_get_bits(binfo->info0, BUFFER_ADDR_INFO0_ADDR);
-	*cookie = le32_get_bits(binfo->info1, BUFFER_ADDR_INFO1_SW_COOKIE);
-	*rbm = le32_get_bits(binfo->info1, BUFFER_ADDR_INFO1_RET_BUF_MGR);
-}
-
-void ath12k_hal_rx_msdu_link_info_get(struct hal_rx_msdu_link *link, u32 *num_msdus,
-				      u32 *msdu_cookies,
-				      enum hal_rx_buf_return_buf_manager *rbm)
-{
-	struct hal_rx_msdu_details *msdu;
-	u32 val;
-	int i;
-
-	*num_msdus = HAL_NUM_RX_MSDUS_PER_LINK_DESC;
-
-	msdu = &link->msdu_link[0];
-	*rbm = le32_get_bits(msdu->buf_addr_info.info1,
-			     BUFFER_ADDR_INFO1_RET_BUF_MGR);
-
-	for (i = 0; i < *num_msdus; i++) {
-		msdu = &link->msdu_link[i];
-
-		val = le32_get_bits(msdu->buf_addr_info.info0,
-				    BUFFER_ADDR_INFO0_ADDR);
-		if (val == 0) {
-			*num_msdus = i;
-			break;
-		}
-		*msdu_cookies = le32_get_bits(msdu->buf_addr_info.info1,
-					      BUFFER_ADDR_INFO1_SW_COOKIE);
-		msdu_cookies++;
-	}
-}
-
-int ath12k_hal_desc_reo_parse_err(struct ath12k_base *ab,
-				  struct hal_reo_dest_ring *desc,
-				  dma_addr_t *paddr, u32 *desc_bank)
-{
-	enum hal_reo_dest_ring_push_reason push_reason;
-	enum hal_reo_dest_ring_error_code err_code;
-	u32 cookie;
-
-	push_reason = le32_get_bits(desc->info0,
-				    HAL_REO_DEST_RING_INFO0_PUSH_REASON);
-	err_code = le32_get_bits(desc->info0,
-				 HAL_REO_DEST_RING_INFO0_ERROR_CODE);
-	ab->device_stats.reo_error[err_code]++;
-
-	if (push_reason != HAL_REO_DEST_RING_PUSH_REASON_ERR_DETECTED &&
-	    push_reason != HAL_REO_DEST_RING_PUSH_REASON_ROUTING_INSTRUCTION) {
-		ath12k_warn(ab, "expected error push reason code, received %d\n",
-			    push_reason);
-		return -EINVAL;
-	}
-
-	ath12k_hal_rx_reo_ent_paddr_get(ab, &desc->buf_addr_info, paddr, &cookie);
-	*desc_bank = u32_get_bits(cookie, DP_LINK_DESC_BANK_MASK);
-
-	return 0;
-}
-
-int ath12k_hal_wbm_desc_parse_err(struct ath12k_base *ab, void *desc,
-				  struct hal_rx_wbm_rel_info *rel_info)
-{
-	struct hal_wbm_release_ring *wbm_desc = desc;
-	struct hal_wbm_release_ring_cc_rx *wbm_cc_desc = desc;
-	enum hal_wbm_rel_desc_type type;
-	enum hal_wbm_rel_src_module rel_src;
-	bool hw_cc_done;
-	u64 desc_va;
-	u32 val;
-
-	type = le32_get_bits(wbm_desc->info0, HAL_WBM_RELEASE_INFO0_DESC_TYPE);
-	/* We expect only WBM_REL buffer type */
-	if (type != HAL_WBM_REL_DESC_TYPE_REL_MSDU) {
-		WARN_ON(1);
-		return -EINVAL;
-	}
-
-	rel_src = le32_get_bits(wbm_desc->info0,
-				HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE);
-	if (rel_src != HAL_WBM_REL_SRC_MODULE_RXDMA &&
-	    rel_src != HAL_WBM_REL_SRC_MODULE_REO)
-		return -EINVAL;
-
-	/* The format of wbm rel ring desc changes based on the
-	 * hw cookie conversion status
-	 */
-	hw_cc_done = le32_get_bits(wbm_desc->info0,
-				   HAL_WBM_RELEASE_RX_INFO0_CC_STATUS);
-
-	if (!hw_cc_done) {
-		val = le32_get_bits(wbm_desc->buf_addr_info.info1,
-				    BUFFER_ADDR_INFO1_RET_BUF_MGR);
-		if (val != HAL_RX_BUF_RBM_SW3_BM) {
-			ab->device_stats.invalid_rbm++;
-			return -EINVAL;
-		}
-
-		rel_info->cookie = le32_get_bits(wbm_desc->buf_addr_info.info1,
-						 BUFFER_ADDR_INFO1_SW_COOKIE);
-
-		rel_info->rx_desc = NULL;
-	} else {
-		val = le32_get_bits(wbm_cc_desc->info0,
-				    HAL_WBM_RELEASE_RX_CC_INFO0_RBM);
-		if (val != HAL_RX_BUF_RBM_SW3_BM) {
-			ab->device_stats.invalid_rbm++;
-			return -EINVAL;
-		}
-
-		rel_info->cookie = le32_get_bits(wbm_cc_desc->info1,
-						 HAL_WBM_RELEASE_RX_CC_INFO1_COOKIE);
-
-		desc_va = ((u64)le32_to_cpu(wbm_cc_desc->buf_va_hi) << 32 |
-			   le32_to_cpu(wbm_cc_desc->buf_va_lo));
-		rel_info->rx_desc =
-			(struct ath12k_rx_desc_info *)((unsigned long)desc_va);
-	}
-
-	rel_info->err_rel_src = rel_src;
-	rel_info->hw_cc_done = hw_cc_done;
-
-	rel_info->first_msdu = le32_get_bits(wbm_desc->info3,
-					     HAL_WBM_RELEASE_INFO3_FIRST_MSDU);
-	rel_info->last_msdu = le32_get_bits(wbm_desc->info3,
-					    HAL_WBM_RELEASE_INFO3_LAST_MSDU);
-	rel_info->continuation = le32_get_bits(wbm_desc->info3,
-					       HAL_WBM_RELEASE_INFO3_CONTINUATION);
-
-	if (rel_info->err_rel_src == HAL_WBM_REL_SRC_MODULE_REO) {
-		rel_info->push_reason =
-			le32_get_bits(wbm_desc->info0,
-				      HAL_WBM_RELEASE_INFO0_REO_PUSH_REASON);
-		rel_info->err_code =
-			le32_get_bits(wbm_desc->info0,
-				      HAL_WBM_RELEASE_INFO0_REO_ERROR_CODE);
-	} else {
-		rel_info->push_reason =
-			le32_get_bits(wbm_desc->info0,
-				      HAL_WBM_RELEASE_INFO0_RXDMA_PUSH_REASON);
-		rel_info->err_code =
-			le32_get_bits(wbm_desc->info0,
-				      HAL_WBM_RELEASE_INFO0_RXDMA_ERROR_CODE);
-	}
-
-	return 0;
-}
-
-void ath12k_hal_rx_reo_ent_paddr_get(struct ath12k_base *ab,
-				     struct ath12k_buffer_addr *buff_addr,
-				     dma_addr_t *paddr, u32 *cookie)
-{
-	*paddr = ((u64)(le32_get_bits(buff_addr->info1,
-				      BUFFER_ADDR_INFO1_ADDR)) << 32) |
-		le32_get_bits(buff_addr->info0, BUFFER_ADDR_INFO0_ADDR);
-
-	*cookie = le32_get_bits(buff_addr->info1, BUFFER_ADDR_INFO1_SW_COOKIE);
-}
-
-void ath12k_hal_rx_reo_ent_buf_paddr_get(void *rx_desc, dma_addr_t *paddr,
-					 u32 *sw_cookie,
-					 struct ath12k_buffer_addr **pp_buf_addr,
-					 u8 *rbm, u32 *msdu_cnt)
-{
-	struct hal_reo_entrance_ring *reo_ent_ring =
-		(struct hal_reo_entrance_ring *)rx_desc;
-	struct ath12k_buffer_addr *buf_addr_info;
-	struct rx_mpdu_desc *rx_mpdu_desc_info_details;
-
-	rx_mpdu_desc_info_details =
-			(struct rx_mpdu_desc *)&reo_ent_ring->rx_mpdu_info;
-
-	*msdu_cnt = le32_get_bits(rx_mpdu_desc_info_details->info0,
-				  RX_MPDU_DESC_INFO0_MSDU_COUNT);
-
-	buf_addr_info = (struct ath12k_buffer_addr *)&reo_ent_ring->buf_addr_info;
-
-	*paddr = (((u64)le32_get_bits(buf_addr_info->info1,
-				      BUFFER_ADDR_INFO1_ADDR)) << 32) |
-			le32_get_bits(buf_addr_info->info0,
-				      BUFFER_ADDR_INFO0_ADDR);
-
-	*sw_cookie = le32_get_bits(buf_addr_info->info1,
-				   BUFFER_ADDR_INFO1_SW_COOKIE);
-	*rbm = le32_get_bits(buf_addr_info->info1,
-			     BUFFER_ADDR_INFO1_RET_BUF_MGR);
-
-	*pp_buf_addr = (void *)buf_addr_info;
-}
-
-void ath12k_hal_rx_msdu_list_get(struct ath12k *ar,
-				 struct hal_rx_msdu_link *link_desc,
-				 struct hal_rx_msdu_list *msdu_list,
-				 u16 *num_msdus)
-{
-	struct hal_rx_msdu_details *msdu_details = NULL;
-	struct rx_msdu_desc *msdu_desc_info = NULL;
-	u32 last = 0, first = 0;
-	u8 tmp = 0;
-	int i;
-
-	last = u32_encode_bits(last, RX_MSDU_DESC_INFO0_LAST_MSDU_IN_MPDU);
-	first = u32_encode_bits(first, RX_MSDU_DESC_INFO0_FIRST_MSDU_IN_MPDU);
-	msdu_details = &link_desc->msdu_link[0];
-
-	for (i = 0; i < HAL_RX_NUM_MSDU_DESC; i++) {
-		if (!i && le32_get_bits(msdu_details[i].buf_addr_info.info0,
-					BUFFER_ADDR_INFO0_ADDR) == 0)
-			break;
-		if (le32_get_bits(msdu_details[i].buf_addr_info.info0,
-				  BUFFER_ADDR_INFO0_ADDR) == 0) {
-			msdu_desc_info = &msdu_details[i - 1].rx_msdu_info;
-			msdu_desc_info->info0 |= cpu_to_le32(last);
-			break;
-		}
-		msdu_desc_info = &msdu_details[i].rx_msdu_info;
-
-		if (!i)
-			msdu_desc_info->info0 |= cpu_to_le32(first);
-		else if (i == (HAL_RX_NUM_MSDU_DESC - 1))
-			msdu_desc_info->info0 |= cpu_to_le32(last);
-		msdu_list->msdu_info[i].msdu_flags = le32_to_cpu(msdu_desc_info->info0);
-		msdu_list->msdu_info[i].msdu_len =
-			 HAL_RX_MSDU_PKT_LENGTH_GET(msdu_desc_info->info0);
-		msdu_list->sw_cookie[i] =
-			le32_get_bits(msdu_details[i].buf_addr_info.info1,
-				      BUFFER_ADDR_INFO1_SW_COOKIE);
-		tmp = le32_get_bits(msdu_details[i].buf_addr_info.info1,
-				    BUFFER_ADDR_INFO1_RET_BUF_MGR);
-		msdu_list->paddr[i] =
-			((u64)(le32_get_bits(msdu_details[i].buf_addr_info.info1,
-					     BUFFER_ADDR_INFO1_ADDR)) << 32) |
-			le32_get_bits(msdu_details[i].buf_addr_info.info0,
-				      BUFFER_ADDR_INFO0_ADDR);
-		msdu_list->rbm[i] = tmp;
-	}
-	*num_msdus = i;
-}
-
-void ath12k_hal_rx_msdu_link_desc_set(struct ath12k_base *ab,
-				      struct hal_wbm_release_ring *desc,
-				      struct ath12k_buffer_addr *buf_addr_info,
-				      enum hal_wbm_rel_bm_act action)
-{
-	desc->buf_addr_info = *buf_addr_info;
-	desc->info0 |= le32_encode_bits(HAL_WBM_REL_SRC_MODULE_SW,
-					HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE) |
-		    le32_encode_bits(action, HAL_WBM_RELEASE_INFO0_BM_ACTION) |
-		    le32_encode_bits(HAL_WBM_REL_DESC_TYPE_MSDU_LINK,
-				     HAL_WBM_RELEASE_INFO0_DESC_TYPE);
-}
-
-void ath12k_hal_reo_status_queue_stats(struct ath12k_base *ab, struct hal_tlv_64_hdr *tlv,
-				       struct hal_reo_status *status)
-{
-	struct hal_reo_get_queue_stats_status *desc =
-		(struct hal_reo_get_queue_stats_status *)tlv->value;
-
-	status->uniform_hdr.cmd_num =
-				le32_get_bits(desc->hdr.info0,
-					      HAL_REO_STATUS_HDR_INFO0_STATUS_NUM);
-	status->uniform_hdr.cmd_status =
-				le32_get_bits(desc->hdr.info0,
-					      HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS);
-
-	ath12k_dbg(ab, ATH12K_DBG_HAL, "Queue stats status:\n");
-	ath12k_dbg(ab, ATH12K_DBG_HAL, "header: cmd_num %d status %d\n",
-		   status->uniform_hdr.cmd_num,
-		   status->uniform_hdr.cmd_status);
-	ath12k_dbg(ab, ATH12K_DBG_HAL, "ssn %u cur_idx %u\n",
-		   le32_get_bits(desc->info0,
-				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO0_SSN),
-		   le32_get_bits(desc->info0,
-				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO0_CUR_IDX));
-	ath12k_dbg(ab, ATH12K_DBG_HAL, "pn = [%08x, %08x, %08x, %08x]\n",
-		   desc->pn[0], desc->pn[1], desc->pn[2], desc->pn[3]);
-	ath12k_dbg(ab, ATH12K_DBG_HAL, "last_rx: enqueue_tstamp %08x dequeue_tstamp %08x\n",
-		   desc->last_rx_enqueue_timestamp,
-		   desc->last_rx_dequeue_timestamp);
-	ath12k_dbg(ab, ATH12K_DBG_HAL, "rx_bitmap [%08x %08x %08x %08x %08x %08x %08x %08x]\n",
-		   desc->rx_bitmap[0], desc->rx_bitmap[1], desc->rx_bitmap[2],
-		   desc->rx_bitmap[3], desc->rx_bitmap[4], desc->rx_bitmap[5],
-		   desc->rx_bitmap[6], desc->rx_bitmap[7]);
-	ath12k_dbg(ab, ATH12K_DBG_HAL, "count: cur_mpdu %u cur_msdu %u\n",
-		   le32_get_bits(desc->info1,
-				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO1_MPDU_COUNT),
-		   le32_get_bits(desc->info1,
-				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO1_MSDU_COUNT));
-	ath12k_dbg(ab, ATH12K_DBG_HAL, "fwd_timeout %u fwd_bar %u dup_count %u\n",
-		   le32_get_bits(desc->info2,
-				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO2_TIMEOUT_COUNT),
-		   le32_get_bits(desc->info2,
-				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO2_FDTB_COUNT),
-		   le32_get_bits(desc->info2,
-				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO2_DUPLICATE_COUNT));
-	ath12k_dbg(ab, ATH12K_DBG_HAL, "frames_in_order %u bar_rcvd %u\n",
-		   le32_get_bits(desc->info3,
-				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO3_FIO_COUNT),
-		   le32_get_bits(desc->info3,
-				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO3_BAR_RCVD_CNT));
-	ath12k_dbg(ab, ATH12K_DBG_HAL, "num_mpdus %d num_msdus %d total_bytes %d\n",
-		   desc->num_mpdu_frames, desc->num_msdu_frames,
-		   desc->total_bytes);
-	ath12k_dbg(ab, ATH12K_DBG_HAL, "late_rcvd %u win_jump_2k %u hole_cnt %u\n",
-		   le32_get_bits(desc->info4,
-				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO4_LATE_RX_MPDU),
-		   le32_get_bits(desc->info2,
-				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO2_WINDOW_JMP2K),
-		   le32_get_bits(desc->info4,
-				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO4_HOLE_COUNT));
-	ath12k_dbg(ab, ATH12K_DBG_HAL, "looping count %u\n",
-		   le32_get_bits(desc->info5,
-				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO5_LOOPING_CNT));
-}
-
-void ath12k_hal_reo_flush_queue_status(struct ath12k_base *ab, struct hal_tlv_64_hdr *tlv,
-				       struct hal_reo_status *status)
-{
-	struct hal_reo_flush_queue_status *desc =
-		(struct hal_reo_flush_queue_status *)tlv->value;
-
-	status->uniform_hdr.cmd_num =
-			le32_get_bits(desc->hdr.info0,
-				      HAL_REO_STATUS_HDR_INFO0_STATUS_NUM);
-	status->uniform_hdr.cmd_status =
-			le32_get_bits(desc->hdr.info0,
-				      HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS);
-	status->u.flush_queue.err_detected =
-			le32_get_bits(desc->info0,
-				      HAL_REO_FLUSH_QUEUE_INFO0_ERR_DETECTED);
-}
-
-void ath12k_hal_reo_flush_cache_status(struct ath12k_base *ab, struct hal_tlv_64_hdr *tlv,
-				       struct hal_reo_status *status)
-{
-	struct ath12k_hal *hal = &ab->hal;
-	struct hal_reo_flush_cache_status *desc =
-		(struct hal_reo_flush_cache_status *)tlv->value;
-
-	status->uniform_hdr.cmd_num =
-			le32_get_bits(desc->hdr.info0,
-				      HAL_REO_STATUS_HDR_INFO0_STATUS_NUM);
-	status->uniform_hdr.cmd_status =
-			le32_get_bits(desc->hdr.info0,
-				      HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS);
-
-	status->u.flush_cache.err_detected =
-			le32_get_bits(desc->info0,
-				      HAL_REO_FLUSH_CACHE_STATUS_INFO0_IS_ERR);
-	status->u.flush_cache.err_code =
-		le32_get_bits(desc->info0,
-			      HAL_REO_FLUSH_CACHE_STATUS_INFO0_BLOCK_ERR_CODE);
-	if (!status->u.flush_cache.err_code)
-		hal->avail_blk_resource |= BIT(hal->current_blk_index);
-
-	status->u.flush_cache.cache_controller_flush_status_hit =
-		le32_get_bits(desc->info0,
-			      HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_STATUS_HIT);
-
-	status->u.flush_cache.cache_controller_flush_status_desc_type =
-		le32_get_bits(desc->info0,
-			      HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_DESC_TYPE);
-	status->u.flush_cache.cache_controller_flush_status_client_id =
-		le32_get_bits(desc->info0,
-			      HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_CLIENT_ID);
-	status->u.flush_cache.cache_controller_flush_status_err =
-		le32_get_bits(desc->info0,
-			      HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_ERR);
-	status->u.flush_cache.cache_controller_flush_status_cnt =
-		le32_get_bits(desc->info0,
-			      HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_COUNT);
-}
-
-void ath12k_hal_reo_unblk_cache_status(struct ath12k_base *ab, struct hal_tlv_64_hdr *tlv,
-				       struct hal_reo_status *status)
-{
-	struct ath12k_hal *hal = &ab->hal;
-	struct hal_reo_unblock_cache_status *desc =
-		(struct hal_reo_unblock_cache_status *)tlv->value;
-
-	status->uniform_hdr.cmd_num =
-			le32_get_bits(desc->hdr.info0,
-				      HAL_REO_STATUS_HDR_INFO0_STATUS_NUM);
-	status->uniform_hdr.cmd_status =
-			le32_get_bits(desc->hdr.info0,
-				      HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS);
-
-	status->u.unblock_cache.err_detected =
-			le32_get_bits(desc->info0,
-				      HAL_REO_UNBLOCK_CACHE_STATUS_INFO0_IS_ERR);
-	status->u.unblock_cache.unblock_type =
-			le32_get_bits(desc->info0,
-				      HAL_REO_UNBLOCK_CACHE_STATUS_INFO0_TYPE);
-
-	if (!status->u.unblock_cache.err_detected &&
-	    status->u.unblock_cache.unblock_type ==
-	    HAL_REO_STATUS_UNBLOCK_BLOCKING_RESOURCE)
-		hal->avail_blk_resource &= ~BIT(hal->current_blk_index);
-}
-
-void ath12k_hal_reo_flush_timeout_list_status(struct ath12k_base *ab,
-					      struct hal_tlv_64_hdr *tlv,
-					      struct hal_reo_status *status)
-{
-	struct hal_reo_flush_timeout_list_status *desc =
-		(struct hal_reo_flush_timeout_list_status *)tlv->value;
-
-	status->uniform_hdr.cmd_num =
-			le32_get_bits(desc->hdr.info0,
-				      HAL_REO_STATUS_HDR_INFO0_STATUS_NUM);
-	status->uniform_hdr.cmd_status =
-			le32_get_bits(desc->hdr.info0,
-				      HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS);
-
-	status->u.timeout_list.err_detected =
-			le32_get_bits(desc->info0,
-				      HAL_REO_FLUSH_TIMEOUT_STATUS_INFO0_IS_ERR);
-	status->u.timeout_list.list_empty =
-			le32_get_bits(desc->info0,
-				      HAL_REO_FLUSH_TIMEOUT_STATUS_INFO0_LIST_EMPTY);
-
-	status->u.timeout_list.release_desc_cnt =
-		le32_get_bits(desc->info1,
-			      HAL_REO_FLUSH_TIMEOUT_STATUS_INFO1_REL_DESC_COUNT);
-	status->u.timeout_list.fwd_buf_cnt =
-		le32_get_bits(desc->info0,
-			      HAL_REO_FLUSH_TIMEOUT_STATUS_INFO1_FWD_BUF_COUNT);
-}
-
-void ath12k_hal_reo_desc_thresh_reached_status(struct ath12k_base *ab,
-					       struct hal_tlv_64_hdr *tlv,
-					       struct hal_reo_status *status)
-{
-	struct hal_reo_desc_thresh_reached_status *desc =
-		(struct hal_reo_desc_thresh_reached_status *)tlv->value;
-
-	status->uniform_hdr.cmd_num =
-			le32_get_bits(desc->hdr.info0,
-				      HAL_REO_STATUS_HDR_INFO0_STATUS_NUM);
-	status->uniform_hdr.cmd_status =
-			le32_get_bits(desc->hdr.info0,
-				      HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS);
-
-	status->u.desc_thresh_reached.threshold_idx =
-		le32_get_bits(desc->info0,
-			      HAL_REO_DESC_THRESH_STATUS_INFO0_THRESH_INDEX);
-
-	status->u.desc_thresh_reached.link_desc_counter0 =
-		le32_get_bits(desc->info1,
-			      HAL_REO_DESC_THRESH_STATUS_INFO1_LINK_DESC_COUNTER0);
-
-	status->u.desc_thresh_reached.link_desc_counter1 =
-		le32_get_bits(desc->info2,
-			      HAL_REO_DESC_THRESH_STATUS_INFO2_LINK_DESC_COUNTER1);
-
-	status->u.desc_thresh_reached.link_desc_counter2 =
-		le32_get_bits(desc->info3,
-			      HAL_REO_DESC_THRESH_STATUS_INFO3_LINK_DESC_COUNTER2);
-
-	status->u.desc_thresh_reached.link_desc_counter_sum =
-		le32_get_bits(desc->info4,
-			      HAL_REO_DESC_THRESH_STATUS_INFO4_LINK_DESC_COUNTER_SUM);
-}
-
-void ath12k_hal_reo_update_rx_reo_queue_status(struct ath12k_base *ab,
-					       struct hal_tlv_64_hdr *tlv,
-					       struct hal_reo_status *status)
-{
-	struct hal_reo_status_hdr *desc =
-		(struct hal_reo_status_hdr *)tlv->value;
-
-	status->uniform_hdr.cmd_num =
-			le32_get_bits(desc->info0,
-				      HAL_REO_STATUS_HDR_INFO0_STATUS_NUM);
-	status->uniform_hdr.cmd_status =
-			le32_get_bits(desc->info0,
-				      HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS);
-}
-
-u32 ath12k_hal_reo_qdesc_size(u32 ba_window_size, u8 tid)
-{
-	u32 num_ext_desc, num_1k_desc = 0;
-
-	if (ba_window_size <= 1) {
-		if (tid != HAL_DESC_REO_NON_QOS_TID)
-			num_ext_desc = 1;
-		else
-			num_ext_desc = 0;
-
-	} else if (ba_window_size <= 105) {
-		num_ext_desc = 1;
-	} else if (ba_window_size <= 210) {
-		num_ext_desc = 2;
-	} else if (ba_window_size <= 256) {
-		num_ext_desc = 3;
-	} else {
-		num_ext_desc = 10;
-		num_1k_desc = 1;
-	}
-
-	return sizeof(struct hal_rx_reo_queue) +
-		(num_ext_desc * sizeof(struct hal_rx_reo_queue_ext)) +
-		(num_1k_desc * sizeof(struct hal_rx_reo_queue_1k));
-}
-
-void ath12k_hal_reo_qdesc_setup(struct hal_rx_reo_queue *qdesc,
-				int tid, u32 ba_window_size,
-				u32 start_seq, enum hal_pn_type type)
-{
-	struct hal_rx_reo_queue_ext *ext_desc;
-
-	ath12k_hal_reo_set_desc_hdr(&qdesc->desc_hdr, HAL_DESC_REO_OWNED,
-				    HAL_DESC_REO_QUEUE_DESC,
-				    REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_0);
-
-	qdesc->rx_queue_num = le32_encode_bits(tid, HAL_RX_REO_QUEUE_RX_QUEUE_NUMBER);
-
-	qdesc->info0 =
-		le32_encode_bits(1, HAL_RX_REO_QUEUE_INFO0_VLD) |
-		le32_encode_bits(1, HAL_RX_REO_QUEUE_INFO0_ASSOC_LNK_DESC_COUNTER) |
-		le32_encode_bits(ath12k_tid_to_ac(tid), HAL_RX_REO_QUEUE_INFO0_AC);
-
-	if (ba_window_size < 1)
-		ba_window_size = 1;
-
-	if (ba_window_size == 1 && tid != HAL_DESC_REO_NON_QOS_TID)
-		ba_window_size++;
-
-	if (ba_window_size == 1)
-		qdesc->info0 |= le32_encode_bits(1, HAL_RX_REO_QUEUE_INFO0_RETRY);
-
-	qdesc->info0 |= le32_encode_bits(ba_window_size - 1,
-					 HAL_RX_REO_QUEUE_INFO0_BA_WINDOW_SIZE);
-	switch (type) {
-	case HAL_PN_TYPE_NONE:
-	case HAL_PN_TYPE_WAPI_EVEN:
-	case HAL_PN_TYPE_WAPI_UNEVEN:
-		break;
-	case HAL_PN_TYPE_WPA:
-		qdesc->info0 |=
-			le32_encode_bits(1, HAL_RX_REO_QUEUE_INFO0_PN_CHECK) |
-			le32_encode_bits(HAL_RX_REO_QUEUE_PN_SIZE_48,
-					 HAL_RX_REO_QUEUE_INFO0_PN_SIZE);
-		break;
-	}
-
-	/* TODO: Set Ignore ampdu flags based on BA window size and/or
-	 * AMPDU capabilities
-	 */
-	qdesc->info0 |= le32_encode_bits(1, HAL_RX_REO_QUEUE_INFO0_IGNORE_AMPDU_FLG);
-
-	qdesc->info1 |= le32_encode_bits(0, HAL_RX_REO_QUEUE_INFO1_SVLD);
-
-	if (start_seq <= 0xfff)
-		qdesc->info1 = le32_encode_bits(start_seq,
-						HAL_RX_REO_QUEUE_INFO1_SSN);
-
-	if (tid == HAL_DESC_REO_NON_QOS_TID)
-		return;
-
-	ext_desc = qdesc->ext_desc;
-
-	/* TODO: HW queue descriptors are currently allocated for max BA
-	 * window size for all QOS TIDs so that same descriptor can be used
-	 * later when ADDBA request is received. This should be changed to
-	 * allocate HW queue descriptors based on BA window size being
-	 * negotiated (0 for non BA cases), and reallocate when BA window
-	 * size changes and also send WMI message to FW to change the REO
-	 * queue descriptor in Rx peer entry as part of dp_rx_tid_update.
-	 */
-	memset(ext_desc, 0, 3 * sizeof(*ext_desc));
-	ath12k_hal_reo_set_desc_hdr(&ext_desc->desc_hdr, HAL_DESC_REO_OWNED,
-				    HAL_DESC_REO_QUEUE_EXT_DESC,
-				    REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_1);
-	ext_desc++;
-	ath12k_hal_reo_set_desc_hdr(&ext_desc->desc_hdr, HAL_DESC_REO_OWNED,
-				    HAL_DESC_REO_QUEUE_EXT_DESC,
-				    REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_2);
-	ext_desc++;
-	ath12k_hal_reo_set_desc_hdr(&ext_desc->desc_hdr, HAL_DESC_REO_OWNED,
-				    HAL_DESC_REO_QUEUE_EXT_DESC,
-				    REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_3);
-}
-
-void ath12k_hal_reo_init_cmd_ring(struct ath12k_base *ab,
-				  struct hal_srng *srng)
-{
-	struct hal_srng_params params;
-	struct hal_tlv_64_hdr *tlv;
-	struct hal_reo_get_queue_stats *desc;
-	int i, cmd_num = 1;
-	int entry_size;
-	u8 *entry;
-
-	memset(&params, 0, sizeof(params));
-
-	entry_size = ath12k_hal_srng_get_entrysize(ab, HAL_REO_CMD);
-	ath12k_hal_srng_get_params(ab, srng, &params);
-	entry = (u8 *)params.ring_base_vaddr;
-
-	for (i = 0; i < params.num_entries; i++) {
-		tlv = (struct hal_tlv_64_hdr *)entry;
-		desc = (struct hal_reo_get_queue_stats *)tlv->value;
-		desc->cmd.info0 = le32_encode_bits(cmd_num++,
-						   HAL_REO_CMD_HDR_INFO0_CMD_NUMBER);
-		entry += entry_size;
-	}
-}
-
-void ath12k_hal_reo_hw_setup(struct ath12k_base *ab, u32 ring_hash_map)
-{
-	u32 reo_base = HAL_SEQ_WCSS_UMAC_REO_REG;
-	u32 val;
-
-	val = ath12k_hif_read32(ab, reo_base + HAL_REO1_GEN_ENABLE);
-
-	val |= u32_encode_bits(1, HAL_REO1_GEN_ENABLE_AGING_LIST_ENABLE) |
-	       u32_encode_bits(1, HAL_REO1_GEN_ENABLE_AGING_FLUSH_ENABLE);
-	ath12k_hif_write32(ab, reo_base + HAL_REO1_GEN_ENABLE, val);
-
-	val = ath12k_hif_read32(ab, reo_base + HAL_REO1_MISC_CTRL_ADDR(ab));
-
-	val &= ~(HAL_REO1_MISC_CTL_FRAG_DST_RING |
-		 HAL_REO1_MISC_CTL_BAR_DST_RING);
-	val |= u32_encode_bits(HAL_SRNG_RING_ID_REO2SW0,
-			       HAL_REO1_MISC_CTL_FRAG_DST_RING);
-	val |= u32_encode_bits(HAL_SRNG_RING_ID_REO2SW0,
-			       HAL_REO1_MISC_CTL_BAR_DST_RING);
-	ath12k_hif_write32(ab, reo_base + HAL_REO1_MISC_CTRL_ADDR(ab), val);
-
-	ath12k_hif_write32(ab, reo_base + HAL_REO1_AGING_THRESH_IX_0(ab),
-			   HAL_DEFAULT_BE_BK_VI_REO_TIMEOUT_USEC);
-	ath12k_hif_write32(ab, reo_base + HAL_REO1_AGING_THRESH_IX_1(ab),
-			   HAL_DEFAULT_BE_BK_VI_REO_TIMEOUT_USEC);
-	ath12k_hif_write32(ab, reo_base + HAL_REO1_AGING_THRESH_IX_2(ab),
-			   HAL_DEFAULT_BE_BK_VI_REO_TIMEOUT_USEC);
-	ath12k_hif_write32(ab, reo_base + HAL_REO1_AGING_THRESH_IX_3(ab),
-			   HAL_DEFAULT_VO_REO_TIMEOUT_USEC);
-
-	ath12k_hif_write32(ab, reo_base + HAL_REO1_DEST_RING_CTRL_IX_2,
-			   ring_hash_map);
-	ath12k_hif_write32(ab, reo_base + HAL_REO1_DEST_RING_CTRL_IX_3,
-			   ring_hash_map);
-}
-
-void ath12k_hal_reo_shared_qaddr_cache_clear(struct ath12k_base *ab)
-{
-	u32 val;
-
-	lockdep_assert_held(&ab->base_lock);
-	val = ath12k_hif_read32(ab, HAL_SEQ_WCSS_UMAC_REO_REG +
-				HAL_REO1_QDESC_ADDR(ab));
-
-	val |= u32_encode_bits(1, HAL_REO_QDESC_ADDR_READ_CLEAR_QDESC_ARRAY);
-	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG +
-			   HAL_REO1_QDESC_ADDR(ab), val);
-
-	val &= ~HAL_REO_QDESC_ADDR_READ_CLEAR_QDESC_ARRAY;
-	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG +
-			   HAL_REO1_QDESC_ADDR(ab), val);
-}
diff --git a/drivers/net/wireless/ath/ath12k/hal_rx.h b/drivers/net/wireless/ath/ath12k/hal_rx.h
deleted file mode 100644
index d1ad7747b82c..000000000000
--- a/drivers/net/wireless/ath/ath12k/hal_rx.h
+++ /dev/null
@@ -1,1171 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause-Clear */
-/*
- * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
- */
-
-#ifndef ATH12K_HAL_RX_H
-#define ATH12K_HAL_RX_H
-
-struct hal_rx_wbm_rel_info {
-	u32 cookie;
-	enum hal_wbm_rel_src_module err_rel_src;
-	enum hal_reo_dest_ring_push_reason push_reason;
-	u32 err_code;
-	bool first_msdu;
-	bool last_msdu;
-	bool continuation;
-	void *rx_desc;
-	bool hw_cc_done;
-};
-
-#define HAL_INVALID_PEERID	0x3fff
-#define VHT_SIG_SU_NSS_MASK 0x7
-
-#define HAL_RX_MPDU_INFO_PN_GET_BYTE1(__val) \
-	le32_get_bits((__val), GENMASK(7, 0))
-
-#define HAL_RX_MPDU_INFO_PN_GET_BYTE2(__val) \
-	le32_get_bits((__val), GENMASK(15, 8))
-
-#define HAL_RX_MPDU_INFO_PN_GET_BYTE3(__val) \
-	le32_get_bits((__val), GENMASK(23, 16))
-
-#define HAL_RX_MPDU_INFO_PN_GET_BYTE4(__val) \
-	le32_get_bits((__val), GENMASK(31, 24))
-
-struct hal_rx_mon_status_tlv_hdr {
-	u32 hdr;
-	u8 value[];
-};
-
-enum hal_rx_su_mu_coding {
-	HAL_RX_SU_MU_CODING_BCC,
-	HAL_RX_SU_MU_CODING_LDPC,
-	HAL_RX_SU_MU_CODING_MAX,
-};
-
-enum hal_rx_gi {
-	HAL_RX_GI_0_8_US,
-	HAL_RX_GI_0_4_US,
-	HAL_RX_GI_1_6_US,
-	HAL_RX_GI_3_2_US,
-	HAL_RX_GI_MAX,
-};
-
-enum hal_rx_bw {
-	HAL_RX_BW_20MHZ,
-	HAL_RX_BW_40MHZ,
-	HAL_RX_BW_80MHZ,
-	HAL_RX_BW_160MHZ,
-	HAL_RX_BW_320MHZ,
-	HAL_RX_BW_MAX,
-};
-
-enum hal_rx_preamble {
-	HAL_RX_PREAMBLE_11A,
-	HAL_RX_PREAMBLE_11B,
-	HAL_RX_PREAMBLE_11N,
-	HAL_RX_PREAMBLE_11AC,
-	HAL_RX_PREAMBLE_11AX,
-	HAL_RX_PREAMBLE_11BA,
-	HAL_RX_PREAMBLE_11BE,
-	HAL_RX_PREAMBLE_MAX,
-};
-
-enum hal_rx_reception_type {
-	HAL_RX_RECEPTION_TYPE_SU,
-	HAL_RX_RECEPTION_TYPE_MU_MIMO,
-	HAL_RX_RECEPTION_TYPE_MU_OFDMA,
-	HAL_RX_RECEPTION_TYPE_MU_OFDMA_MIMO,
-	HAL_RX_RECEPTION_TYPE_MAX,
-};
-
-enum hal_rx_legacy_rate {
-	HAL_RX_LEGACY_RATE_1_MBPS,
-	HAL_RX_LEGACY_RATE_2_MBPS,
-	HAL_RX_LEGACY_RATE_5_5_MBPS,
-	HAL_RX_LEGACY_RATE_6_MBPS,
-	HAL_RX_LEGACY_RATE_9_MBPS,
-	HAL_RX_LEGACY_RATE_11_MBPS,
-	HAL_RX_LEGACY_RATE_12_MBPS,
-	HAL_RX_LEGACY_RATE_18_MBPS,
-	HAL_RX_LEGACY_RATE_24_MBPS,
-	HAL_RX_LEGACY_RATE_36_MBPS,
-	HAL_RX_LEGACY_RATE_48_MBPS,
-	HAL_RX_LEGACY_RATE_54_MBPS,
-	HAL_RX_LEGACY_RATE_INVALID,
-};
-
-#define HAL_TLV_STATUS_PPDU_NOT_DONE            0
-#define HAL_TLV_STATUS_PPDU_DONE                1
-#define HAL_TLV_STATUS_BUF_DONE                 2
-#define HAL_TLV_STATUS_PPDU_NON_STD_DONE        3
-#define HAL_RX_FCS_LEN                          4
-
-enum hal_rx_mon_status {
-	HAL_RX_MON_STATUS_PPDU_NOT_DONE,
-	HAL_RX_MON_STATUS_PPDU_DONE,
-	HAL_RX_MON_STATUS_BUF_DONE,
-	HAL_RX_MON_STATUS_BUF_ADDR,
-	HAL_RX_MON_STATUS_MPDU_START,
-	HAL_RX_MON_STATUS_MPDU_END,
-	HAL_RX_MON_STATUS_MSDU_END,
-};
-
-#define HAL_RX_MAX_MPDU				1024
-#define HAL_RX_NUM_WORDS_PER_PPDU_BITMAP	(HAL_RX_MAX_MPDU >> 5)
-
-struct hal_rx_user_status {
-	u32 mcs:4,
-	nss:3,
-	ofdma_info_valid:1,
-	ul_ofdma_ru_start_index:7,
-	ul_ofdma_ru_width:7,
-	ul_ofdma_ru_size:8;
-	u32 ul_ofdma_user_v0_word0;
-	u32 ul_ofdma_user_v0_word1;
-	u32 ast_index;
-	u32 tid;
-	u16 tcp_msdu_count;
-	u16 tcp_ack_msdu_count;
-	u16 udp_msdu_count;
-	u16 other_msdu_count;
-	u16 frame_control;
-	u8 frame_control_info_valid;
-	u8 data_sequence_control_info_valid;
-	u16 first_data_seq_ctrl;
-	u32 preamble_type;
-	u16 ht_flags;
-	u16 vht_flags;
-	u16 he_flags;
-	u8 rs_flags;
-	u8 ldpc;
-	u32 mpdu_cnt_fcs_ok;
-	u32 mpdu_cnt_fcs_err;
-	u32 mpdu_fcs_ok_bitmap[HAL_RX_NUM_WORDS_PER_PPDU_BITMAP];
-	u32 mpdu_ok_byte_count;
-	u32 mpdu_err_byte_count;
-	bool ampdu_present;
-	u16 ampdu_id;
-};
-
-#define HAL_MAX_UL_MU_USERS	37
-
-struct hal_rx_u_sig_info {
-	bool ul_dl;
-	u8 bw;
-	u8 ppdu_type_comp_mode;
-	u8 eht_sig_mcs;
-	u8 num_eht_sig_sym;
-	struct ieee80211_radiotap_eht_usig usig;
-};
-
-#define HAL_RX_MON_MAX_AGGR_SIZE	128
-
-struct hal_rx_tlv_aggr_info {
-	bool in_progress;
-	u16 cur_len;
-	u16 tlv_tag;
-	u8 buf[HAL_RX_MON_MAX_AGGR_SIZE];
-};
-
-struct hal_rx_radiotap_eht {
-	__le32 known;
-	__le32 data[9];
-};
-
-#define EHT_MAX_USER_INFO	4
-
-struct hal_rx_eht_info {
-	u8 num_user_info;
-	struct hal_rx_radiotap_eht eht;
-	u32 user_info[EHT_MAX_USER_INFO];
-};
-
-struct hal_rx_mon_ppdu_info {
-	u32 ppdu_id;
-	u32 last_ppdu_id;
-	u64 ppdu_ts;
-	u32 num_mpdu_fcs_ok;
-	u32 num_mpdu_fcs_err;
-	u32 preamble_type;
-	u32 mpdu_len;
-	u16 chan_num;
-	u16 freq;
-	u16 tcp_msdu_count;
-	u16 tcp_ack_msdu_count;
-	u16 udp_msdu_count;
-	u16 other_msdu_count;
-	u16 peer_id;
-	u8 rate;
-	u8 mcs;
-	u8 nss;
-	u8 bw;
-	u8 vht_flag_values1;
-	u8 vht_flag_values2;
-	u8 vht_flag_values3[4];
-	u8 vht_flag_values4;
-	u8 vht_flag_values5;
-	u16 vht_flag_values6;
-	u8 is_stbc;
-	u8 gi;
-	u8 sgi;
-	u8 ldpc;
-	u8 beamformed;
-	u8 rssi_comb;
-	u16 tid;
-	u8 fc_valid;
-	u16 ht_flags;
-	u16 vht_flags;
-	u16 he_flags;
-	u16 he_mu_flags;
-	u8 dcm;
-	u8 ru_alloc;
-	u8 reception_type;
-	u64 tsft;
-	u64 rx_duration;
-	u16 frame_control;
-	u32 ast_index;
-	u8 rs_fcs_err;
-	u8 rs_flags;
-	u8 cck_flag;
-	u8 ofdm_flag;
-	u8 ulofdma_flag;
-	u8 frame_control_info_valid;
-	u16 he_per_user_1;
-	u16 he_per_user_2;
-	u8 he_per_user_position;
-	u8 he_per_user_known;
-	u16 he_flags1;
-	u16 he_flags2;
-	u8 he_RU[4];
-	u16 he_data1;
-	u16 he_data2;
-	u16 he_data3;
-	u16 he_data4;
-	u16 he_data5;
-	u16 he_data6;
-	u32 ppdu_len;
-	u32 prev_ppdu_id;
-	u32 device_id;
-	u16 first_data_seq_ctrl;
-	u8 monitor_direct_used;
-	u8 data_sequence_control_info_valid;
-	u8 ltf_size;
-	u8 rxpcu_filter_pass;
-	s8 rssi_chain[8][8];
-	u32 num_users;
-	u32 mpdu_fcs_ok_bitmap[HAL_RX_NUM_WORDS_PER_PPDU_BITMAP];
-	u8 addr1[ETH_ALEN];
-	u8 addr2[ETH_ALEN];
-	u8 addr3[ETH_ALEN];
-	u8 addr4[ETH_ALEN];
-	struct hal_rx_user_status userstats[HAL_MAX_UL_MU_USERS];
-	u8 userid;
-	bool first_msdu_in_mpdu;
-	bool is_ampdu;
-	u8 medium_prot_type;
-	bool ppdu_continuation;
-	bool eht_usig;
-	struct hal_rx_u_sig_info u_sig_info;
-	bool is_eht;
-	struct hal_rx_eht_info eht_info;
-	struct hal_rx_tlv_aggr_info tlv_aggr;
-};
-
-#define HAL_RX_PPDU_START_INFO0_PPDU_ID			GENMASK(15, 0)
-#define HAL_RX_PPDU_START_INFO1_CHAN_NUM		GENMASK(15, 0)
-#define HAL_RX_PPDU_START_INFO1_CHAN_FREQ		GENMASK(31, 16)
-
-struct hal_rx_ppdu_start {
-	__le32 info0;
-	__le32 info1;
-	__le32 ppdu_start_ts_31_0;
-	__le32 ppdu_start_ts_63_32;
-	__le32 rsvd[2];
-} __packed;
-
-#define HAL_RX_PPDU_END_USER_STATS_INFO0_PEER_ID		GENMASK(13, 0)
-#define HAL_RX_PPDU_END_USER_STATS_INFO0_DEVICE_ID		GENMASK(15, 14)
-#define HAL_RX_PPDU_END_USER_STATS_INFO0_MPDU_CNT_FCS_ERR	GENMASK(26, 16)
-
-#define HAL_RX_PPDU_END_USER_STATS_INFO1_MPDU_CNT_FCS_OK	GENMASK(10, 0)
-#define HAL_RX_PPDU_END_USER_STATS_INFO1_FC_VALID		BIT(11)
-#define HAL_RX_PPDU_END_USER_STATS_INFO1_QOS_CTRL_VALID		BIT(12)
-#define HAL_RX_PPDU_END_USER_STATS_INFO1_HT_CTRL_VALID		BIT(13)
-#define HAL_RX_PPDU_END_USER_STATS_INFO1_PKT_TYPE              GENMASK(24, 21)
-
-#define HAL_RX_PPDU_END_USER_STATS_INFO2_AST_INDEX		GENMASK(15, 0)
-#define HAL_RX_PPDU_END_USER_STATS_INFO2_FRAME_CTRL		GENMASK(31, 16)
-
-#define HAL_RX_PPDU_END_USER_STATS_INFO3_QOS_CTRL		GENMASK(31, 16)
-
-#define HAL_RX_PPDU_END_USER_STATS_INFO4_UDP_MSDU_CNT		GENMASK(15, 0)
-#define HAL_RX_PPDU_END_USER_STATS_INFO4_TCP_MSDU_CNT		GENMASK(31, 16)
-
-#define HAL_RX_PPDU_END_USER_STATS_INFO5_OTHER_MSDU_CNT		GENMASK(15, 0)
-#define HAL_RX_PPDU_END_USER_STATS_INFO5_TCP_ACK_MSDU_CNT	GENMASK(31, 16)
-
-#define HAL_RX_PPDU_END_USER_STATS_INFO6_TID_BITMAP		GENMASK(15, 0)
-#define HAL_RX_PPDU_END_USER_STATS_INFO6_TID_EOSP_BITMAP	GENMASK(31, 16)
-
-#define HAL_RX_PPDU_END_USER_STATS_INFO7_MPDU_OK_BYTE_COUNT    GENMASK(24, 0)
-#define HAL_RX_PPDU_END_USER_STATS_INFO8_MPDU_ERR_BYTE_COUNT   GENMASK(24, 0)
-
-struct hal_rx_ppdu_end_user_stats {
-	__le32 rsvd0[2];
-	__le32 info0;
-	__le32 info1;
-	__le32 info2;
-	__le32 info3;
-	__le32 ht_ctrl;
-	__le32 rsvd1[2];
-	__le32 info4;
-	__le32 info5;
-	__le32 usr_resp_ref;
-	__le32 info6;
-	__le32 rsvd3[4];
-	__le32 info7;
-	__le32 rsvd4;
-	__le32 info8;
-	__le32 rsvd5[2];
-	__le32 usr_resp_ref_ext;
-	__le32 rsvd6;
-} __packed;
-
-struct hal_rx_ppdu_end_user_stats_ext {
-	__le32 info0;
-	__le32 info1;
-	__le32 info2;
-	__le32 info3;
-	__le32 info4;
-	__le32 info5;
-	__le32 info6;
-	__le32 rsvd;
-} __packed;
-
-#define HAL_RX_HT_SIG_INFO_INFO0_MCS		GENMASK(6, 0)
-#define HAL_RX_HT_SIG_INFO_INFO0_BW		BIT(7)
-
-#define HAL_RX_HT_SIG_INFO_INFO1_STBC		GENMASK(5, 4)
-#define HAL_RX_HT_SIG_INFO_INFO1_FEC_CODING	BIT(6)
-#define HAL_RX_HT_SIG_INFO_INFO1_GI		BIT(7)
-
-struct hal_rx_ht_sig_info {
-	__le32 info0;
-	__le32 info1;
-} __packed;
-
-#define HAL_RX_LSIG_B_INFO_INFO0_RATE	GENMASK(3, 0)
-#define HAL_RX_LSIG_B_INFO_INFO0_LEN	GENMASK(15, 4)
-
-struct hal_rx_lsig_b_info {
-	__le32 info0;
-} __packed;
-
-#define HAL_RX_LSIG_A_INFO_INFO0_RATE		GENMASK(3, 0)
-#define HAL_RX_LSIG_A_INFO_INFO0_LEN		GENMASK(16, 5)
-#define HAL_RX_LSIG_A_INFO_INFO0_PKT_TYPE	GENMASK(27, 24)
-
-struct hal_rx_lsig_a_info {
-	__le32 info0;
-} __packed;
-
-#define HAL_RX_VHT_SIG_A_INFO_INFO0_BW		GENMASK(1, 0)
-#define HAL_RX_VHT_SIG_A_INFO_INFO0_STBC	BIT(3)
-#define HAL_RX_VHT_SIG_A_INFO_INFO0_GROUP_ID	GENMASK(9, 4)
-#define HAL_RX_VHT_SIG_A_INFO_INFO0_NSTS	GENMASK(21, 10)
-
-#define HAL_RX_VHT_SIG_A_INFO_INFO1_GI_SETTING		GENMASK(1, 0)
-#define HAL_RX_VHT_SIG_A_INFO_INFO1_SU_MU_CODING	BIT(2)
-#define HAL_RX_VHT_SIG_A_INFO_INFO1_MCS			GENMASK(7, 4)
-#define HAL_RX_VHT_SIG_A_INFO_INFO1_BEAMFORMED		BIT(8)
-
-struct hal_rx_vht_sig_a_info {
-	__le32 info0;
-	__le32 info1;
-} __packed;
-
-enum hal_rx_vht_sig_a_gi_setting {
-	HAL_RX_VHT_SIG_A_NORMAL_GI = 0,
-	HAL_RX_VHT_SIG_A_SHORT_GI = 1,
-	HAL_RX_VHT_SIG_A_SHORT_GI_AMBIGUITY = 3,
-};
-
-#define HE_GI_0_8 0
-#define HE_GI_0_4 1
-#define HE_GI_1_6 2
-#define HE_GI_3_2 3
-
-#define HE_LTF_1_X 0
-#define HE_LTF_2_X 1
-#define HE_LTF_4_X 2
-
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_MCS	GENMASK(6, 3)
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_DCM		BIT(7)
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_BW	GENMASK(20, 19)
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_CP_LTF_SIZE	GENMASK(22, 21)
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_NSTS		GENMASK(25, 23)
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_BSS_COLOR		GENMASK(13, 8)
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_SPATIAL_REUSE	GENMASK(18, 15)
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_FORMAT_IND	BIT(0)
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_BEAM_CHANGE	BIT(1)
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_DL_UL_FLAG	BIT(2)
-
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO1_TXOP_DURATION	GENMASK(6, 0)
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO1_CODING		BIT(7)
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO1_LDPC_EXTRA	BIT(8)
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO1_STBC		BIT(9)
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO1_TXBF		BIT(10)
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO1_PKT_EXT_FACTOR	GENMASK(12, 11)
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO1_PKT_EXT_PE_DISAM	BIT(13)
-#define HAL_RX_HE_SIG_A_SU_INFO_INFO1_DOPPLER_IND	BIT(15)
-
-struct hal_rx_he_sig_a_su_info {
-	__le32 info0;
-	__le32 info1;
-} __packed;
-
-#define HAL_RX_HE_SIG_A_MU_DL_INFO0_UL_FLAG		BIT(1)
-#define HAL_RX_HE_SIG_A_MU_DL_INFO0_MCS_OF_SIGB		GENMASK(3, 1)
-#define HAL_RX_HE_SIG_A_MU_DL_INFO0_DCM_OF_SIGB		BIT(4)
-#define HAL_RX_HE_SIG_A_MU_DL_INFO0_BSS_COLOR		GENMASK(10, 5)
-#define HAL_RX_HE_SIG_A_MU_DL_INFO0_SPATIAL_REUSE	GENMASK(14, 11)
-#define HAL_RX_HE_SIG_A_MU_DL_INFO0_TRANSMIT_BW		GENMASK(17, 15)
-#define HAL_RX_HE_SIG_A_MU_DL_INFO0_NUM_SIGB_SYMB	GENMASK(21, 18)
-#define HAL_RX_HE_SIG_A_MU_DL_INFO0_COMP_MODE_SIGB	BIT(22)
-#define HAL_RX_HE_SIG_A_MU_DL_INFO0_CP_LTF_SIZE		GENMASK(24, 23)
-#define HAL_RX_HE_SIG_A_MU_DL_INFO0_DOPPLER_INDICATION	BIT(25)
-
-#define HAL_RX_HE_SIG_A_MU_DL_INFO1_TXOP_DURATION	GENMASK(6, 0)
-#define HAL_RX_HE_SIG_A_MU_DL_INFO1_NUM_LTF_SYMB	GENMASK(10, 8)
-#define HAL_RX_HE_SIG_A_MU_DL_INFO1_LDPC_EXTRA		BIT(11)
-#define HAL_RX_HE_SIG_A_MU_DL_INFO1_STBC		BIT(12)
-#define HAL_RX_HE_SIG_A_MU_DL_INFO1_PKT_EXT_FACTOR	GENMASK(14, 13)
-#define HAL_RX_HE_SIG_A_MU_DL_INFO1_PKT_EXT_PE_DISAM	BIT(15)
-
-struct hal_rx_he_sig_a_mu_dl_info {
-	__le32 info0;
-	__le32 info1;
-} __packed;
-
-#define HAL_RX_HE_SIG_B1_MU_INFO_INFO0_RU_ALLOCATION	GENMASK(7, 0)
-
-struct hal_rx_he_sig_b1_mu_info {
-	__le32 info0;
-} __packed;
-
-#define HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_ID           GENMASK(10, 0)
-#define HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_MCS		GENMASK(18, 15)
-#define HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_CODING	BIT(20)
-#define HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_NSTS		GENMASK(31, 29)
-
-struct hal_rx_he_sig_b2_mu_info {
-	__le32 info0;
-} __packed;
-
-#define HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_ID	GENMASK(10, 0)
-#define HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_NSTS	GENMASK(13, 11)
-#define HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_TXBF	BIT(14)
-#define HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_MCS	GENMASK(18, 15)
-#define HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_DCM	BIT(19)
-#define HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_CODING	BIT(20)
-
-struct hal_rx_he_sig_b2_ofdma_info {
-	__le32 info0;
-} __packed;
-
-enum hal_rx_ul_reception_type {
-	HAL_RECEPTION_TYPE_ULOFMDA,
-	HAL_RECEPTION_TYPE_ULMIMO,
-	HAL_RECEPTION_TYPE_OTHER,
-	HAL_RECEPTION_TYPE_FRAMELESS
-};
-
-#define HAL_RX_RSSI_LEGACY_INFO_INFO0_RECEPTION		GENMASK(3, 0)
-#define HAL_RX_RSSI_LEGACY_INFO_INFO0_RX_BW		GENMASK(7, 5)
-#define HAL_RX_RSSI_LEGACY_INFO_INFO1_RSSI_COMB		GENMASK(15, 8)
-#define HAL_RX_RSSI_LEGACY_INFO_INFO2_RSSI_COMB_PPDU	GENMASK(7, 0)
-
-struct hal_rx_phyrx_rssi_legacy_info {
-	__le32 info0;
-	__le32 rsvd0[39];
-	__le32 info1;
-	__le32 info2;
-} __packed;
-
-#define HAL_RX_MPDU_START_INFO0_PPDU_ID			GENMASK(31, 16)
-#define HAL_RX_MPDU_START_INFO1_PEERID			GENMASK(29, 16)
-#define HAL_RX_MPDU_START_INFO1_DEVICE_ID		GENMASK(31, 30)
-#define HAL_RX_MPDU_START_INFO2_MPDU_LEN		GENMASK(13, 0)
-struct hal_rx_mpdu_start {
-	__le32 rsvd0[9];
-	__le32 info0;
-	__le32 info1;
-	__le32 rsvd1[2];
-	__le32 info2;
-	__le32 rsvd2[16];
-} __packed;
-
-struct hal_rx_msdu_end {
-	__le32 info0;
-	__le32 rsvd0[9];
-	__le16 info00;
-	__le16 info01;
-	__le32 rsvd00[8];
-	__le32 info1;
-	__le32 rsvd1[10];
-	__le32 info2;
-	__le32 rsvd2;
-} __packed;
-
-#define HAL_RX_PPDU_END_DURATION	GENMASK(23, 0)
-struct hal_rx_ppdu_end_duration {
-	__le32 rsvd0[9];
-	__le32 info0;
-	__le32 rsvd1[18];
-} __packed;
-
-struct hal_rx_rxpcu_classification_overview {
-	u32 rsvd0;
-} __packed;
-
-struct hal_rx_msdu_desc_info {
-	u32 msdu_flags;
-	u16 msdu_len; /* 14 bits for length */
-};
-
-#define HAL_RX_NUM_MSDU_DESC 6
-struct hal_rx_msdu_list {
-	struct hal_rx_msdu_desc_info msdu_info[HAL_RX_NUM_MSDU_DESC];
-	u64 paddr[HAL_RX_NUM_MSDU_DESC];
-	u32 sw_cookie[HAL_RX_NUM_MSDU_DESC];
-	u8 rbm[HAL_RX_NUM_MSDU_DESC];
-};
-
-#define HAL_RX_FBM_ACK_INFO0_ADDR1_31_0		GENMASK(31, 0)
-#define HAL_RX_FBM_ACK_INFO1_ADDR1_47_32	GENMASK(15, 0)
-#define HAL_RX_FBM_ACK_INFO1_ADDR2_15_0		GENMASK(31, 16)
-#define HAL_RX_FBM_ACK_INFO2_ADDR2_47_16	GENMASK(31, 0)
-
-struct hal_rx_frame_bitmap_ack {
-	__le32 reserved;
-	__le32 info0;
-	__le32 info1;
-	__le32 info2;
-	__le32 reserved1[10];
-} __packed;
-
-#define HAL_RX_RESP_REQ_INFO0_PPDU_ID		GENMASK(15, 0)
-#define HAL_RX_RESP_REQ_INFO0_RECEPTION_TYPE	BIT(16)
-#define HAL_RX_RESP_REQ_INFO1_DURATION		GENMASK(15, 0)
-#define HAL_RX_RESP_REQ_INFO1_RATE_MCS		GENMASK(24, 21)
-#define HAL_RX_RESP_REQ_INFO1_SGI		GENMASK(26, 25)
-#define HAL_RX_RESP_REQ_INFO1_STBC		BIT(27)
-#define HAL_RX_RESP_REQ_INFO1_LDPC		BIT(28)
-#define HAL_RX_RESP_REQ_INFO1_IS_AMPDU		BIT(29)
-#define HAL_RX_RESP_REQ_INFO2_NUM_USER		GENMASK(6, 0)
-#define HAL_RX_RESP_REQ_INFO3_ADDR1_31_0	GENMASK(31, 0)
-#define HAL_RX_RESP_REQ_INFO4_ADDR1_47_32	GENMASK(15, 0)
-#define HAL_RX_RESP_REQ_INFO4_ADDR1_15_0	GENMASK(31, 16)
-#define HAL_RX_RESP_REQ_INFO5_ADDR1_47_16	GENMASK(31, 0)
-
-struct hal_rx_resp_req_info {
-	__le32 info0;
-	__le32 reserved[1];
-	__le32 info1;
-	__le32 info2;
-	__le32 reserved1[2];
-	__le32 info3;
-	__le32 info4;
-	__le32 info5;
-	__le32 reserved2[5];
-} __packed;
-
-#define REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_0 0xDDBEEF
-#define REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_1 0xADBEEF
-#define REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_2 0xBDBEEF
-#define REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_3 0xCDBEEF
-
-#define HAL_RX_UL_OFDMA_USER_INFO_V0_W0_VALID		BIT(30)
-#define HAL_RX_UL_OFDMA_USER_INFO_V0_W0_VER		BIT(31)
-#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_NSS		GENMASK(2, 0)
-#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_MCS		GENMASK(6, 3)
-#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_LDPC		BIT(7)
-#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_DCM		BIT(8)
-#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_RU_START	GENMASK(15, 9)
-#define HAL_RX_UL_OFDMA_USER_INFO_V0_W1_RU_SIZE		GENMASK(18, 16)
-
-/* HE Radiotap data1 Mask */
-#define HE_SU_FORMAT_TYPE 0x0000
-#define HE_EXT_SU_FORMAT_TYPE 0x0001
-#define HE_MU_FORMAT_TYPE  0x0002
-#define HE_TRIG_FORMAT_TYPE  0x0003
-#define HE_BEAM_CHANGE_KNOWN 0x0008
-#define HE_DL_UL_KNOWN 0x0010
-#define HE_MCS_KNOWN 0x0020
-#define HE_DCM_KNOWN 0x0040
-#define HE_CODING_KNOWN 0x0080
-#define HE_LDPC_EXTRA_SYMBOL_KNOWN 0x0100
-#define HE_STBC_KNOWN 0x0200
-#define HE_DATA_BW_RU_KNOWN 0x4000
-#define HE_DOPPLER_KNOWN 0x8000
-#define HE_BSS_COLOR_KNOWN 0x0004
-
-/* HE Radiotap data2 Mask */
-#define HE_GI_KNOWN 0x0002
-#define HE_TXBF_KNOWN 0x0010
-#define HE_PE_DISAMBIGUITY_KNOWN 0x0020
-#define HE_TXOP_KNOWN 0x0040
-#define HE_LTF_SYMBOLS_KNOWN 0x0004
-#define HE_PRE_FEC_PADDING_KNOWN 0x0008
-#define HE_MIDABLE_PERIODICITY_KNOWN 0x0080
-
-/* HE radiotap data3 shift values */
-#define HE_BEAM_CHANGE_SHIFT 6
-#define HE_DL_UL_SHIFT 7
-#define HE_TRANSMIT_MCS_SHIFT 8
-#define HE_DCM_SHIFT 12
-#define HE_CODING_SHIFT 13
-#define HE_LDPC_EXTRA_SYMBOL_SHIFT 14
-#define HE_STBC_SHIFT 15
-
-/* HE radiotap data4 shift values */
-#define HE_STA_ID_SHIFT 4
-
-/* HE radiotap data5 */
-#define HE_GI_SHIFT 4
-#define HE_LTF_SIZE_SHIFT 6
-#define HE_LTF_SYM_SHIFT 8
-#define HE_TXBF_SHIFT 14
-#define HE_PE_DISAMBIGUITY_SHIFT 15
-#define HE_PRE_FEC_PAD_SHIFT 12
-
-/* HE radiotap data6 */
-#define HE_DOPPLER_SHIFT 4
-#define HE_TXOP_SHIFT 8
-
-/* HE radiotap HE-MU flags1 */
-#define HE_SIG_B_MCS_KNOWN 0x0010
-#define HE_SIG_B_DCM_KNOWN 0x0040
-#define HE_SIG_B_SYM_NUM_KNOWN 0x8000
-#define HE_RU_0_KNOWN 0x0100
-#define HE_RU_1_KNOWN 0x0200
-#define HE_RU_2_KNOWN 0x0400
-#define HE_RU_3_KNOWN 0x0800
-#define HE_DCM_FLAG_1_SHIFT 5
-#define HE_SPATIAL_REUSE_MU_KNOWN 0x0100
-#define HE_SIG_B_COMPRESSION_FLAG_1_KNOWN 0x4000
-
-/* HE radiotap HE-MU flags2 */
-#define HE_SIG_B_COMPRESSION_FLAG_2_SHIFT 3
-#define HE_BW_KNOWN 0x0004
-#define HE_NUM_SIG_B_SYMBOLS_SHIFT 4
-#define HE_SIG_B_COMPRESSION_FLAG_2_KNOWN 0x0100
-#define HE_NUM_SIG_B_FLAG_2_SHIFT 9
-#define HE_LTF_FLAG_2_SYMBOLS_SHIFT 12
-#define HE_LTF_KNOWN 0x8000
-
-/* HE radiotap per_user_1 */
-#define HE_STA_SPATIAL_SHIFT 11
-#define HE_TXBF_SHIFT 14
-#define HE_RESERVED_SET_TO_1_SHIFT 19
-#define HE_STA_CODING_SHIFT 20
-
-/* HE radiotap per_user_2 */
-#define HE_STA_MCS_SHIFT 4
-#define HE_STA_DCM_SHIFT 5
-
-/* HE radiotap per user known */
-#define HE_USER_FIELD_POSITION_KNOWN 0x01
-#define HE_STA_ID_PER_USER_KNOWN 0x02
-#define HE_STA_NSTS_KNOWN 0x04
-#define HE_STA_TX_BF_KNOWN 0x08
-#define HE_STA_SPATIAL_CONFIG_KNOWN 0x10
-#define HE_STA_MCS_KNOWN 0x20
-#define HE_STA_DCM_KNOWN 0x40
-#define HE_STA_CODING_KNOWN 0x80
-
-#define HAL_RX_MPDU_ERR_FCS			BIT(0)
-#define HAL_RX_MPDU_ERR_DECRYPT			BIT(1)
-#define HAL_RX_MPDU_ERR_TKIP_MIC		BIT(2)
-#define HAL_RX_MPDU_ERR_AMSDU_ERR		BIT(3)
-#define HAL_RX_MPDU_ERR_OVERFLOW		BIT(4)
-#define HAL_RX_MPDU_ERR_MSDU_LEN		BIT(5)
-#define HAL_RX_MPDU_ERR_MPDU_LEN		BIT(6)
-#define HAL_RX_MPDU_ERR_UNENCRYPTED_FRAME	BIT(7)
-
-#define HAL_RX_CMN_USR_INFO0_CP_SETTING			GENMASK(17, 16)
-#define HAL_RX_CMN_USR_INFO0_LTF_SIZE			GENMASK(19, 18)
-
-struct hal_phyrx_common_user_info {
-	__le32 rsvd[2];
-	__le32 info0;
-	__le32 rsvd1;
-} __packed;
-
-#define HAL_RX_EHT_SIG_NDP_CMN_INFO0_SPATIAL_REUSE	GENMASK(3, 0)
-#define HAL_RX_EHT_SIG_NDP_CMN_INFO0_GI_LTF		GENMASK(5, 4)
-#define HAL_RX_EHT_SIG_NDP_CMN_INFO0_NUM_LTF_SYM	GENMASK(8, 6)
-#define HAL_RX_EHT_SIG_NDP_CMN_INFO0_NSS		GENMASK(10, 7)
-#define HAL_RX_EHT_SIG_NDP_CMN_INFO0_BEAMFORMED		BIT(11)
-#define HAL_RX_EHT_SIG_NDP_CMN_INFO0_DISREGARD		GENMASK(13, 12)
-#define HAL_RX_EHT_SIG_NDP_CMN_INFO0_CRC		GENMASK(17, 14)
-
-struct hal_eht_sig_ndp_cmn_eb {
-	__le32 info0;
-} __packed;
-
-#define HAL_RX_EHT_SIG_OVERFLOW_INFO0_SPATIAL_REUSE		GENMASK(3, 0)
-#define HAL_RX_EHT_SIG_OVERFLOW_INFO0_GI_LTF			GENMASK(5, 4)
-#define HAL_RX_EHT_SIG_OVERFLOW_INFO0_NUM_LTF_SYM		GENMASK(8, 6)
-#define HAL_RX_EHT_SIG_OVERFLOW_INFO0_LDPC_EXTA_SYM		BIT(9)
-#define HAL_RX_EHT_SIG_OVERFLOW_INFO0_PRE_FEC_PAD_FACTOR	GENMASK(11, 10)
-#define HAL_RX_EHT_SIG_OVERFLOW_INFO0_DISAMBIGUITY		BIT(12)
-#define HAL_RX_EHT_SIG_OVERFLOW_INFO0_DISREGARD			GENMASK(16, 13)
-
-struct hal_eht_sig_usig_overflow {
-	__le32 info0;
-} __packed;
-
-#define HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_STA_ID	GENMASK(10, 0)
-#define HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_MCS	GENMASK(14, 11)
-#define HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_VALIDATE	BIT(15)
-#define HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_NSS	GENMASK(19, 16)
-#define HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_BEAMFORMED	BIT(20)
-#define HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_CODING	BIT(21)
-#define HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_CRC	GENMASK(25, 22)
-
-struct hal_eht_sig_non_mu_mimo {
-	__le32 info0;
-} __packed;
-
-#define HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_STA_ID		GENMASK(10, 0)
-#define HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_MCS		GENMASK(14, 11)
-#define HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_CODING		BIT(15)
-#define HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_SPATIAL_CODING	GENMASK(22, 16)
-#define HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_CRC		GENMASK(26, 23)
-
-struct hal_eht_sig_mu_mimo {
-	__le32 info0;
-} __packed;
-
-union hal_eht_sig_user_field {
-	struct hal_eht_sig_mu_mimo mu_mimo;
-	struct hal_eht_sig_non_mu_mimo n_mu_mimo;
-};
-
-#define HAL_RX_EHT_SIG_NON_OFDMA_INFO0_SPATIAL_REUSE		GENMASK(3, 0)
-#define HAL_RX_EHT_SIG_NON_OFDMA_INFO0_GI_LTF			GENMASK(5, 4)
-#define HAL_RX_EHT_SIG_NON_OFDMA_INFO0_NUM_LTF_SYM		GENMASK(8, 6)
-#define HAL_RX_EHT_SIG_NON_OFDMA_INFO0_LDPC_EXTA_SYM		BIT(9)
-#define HAL_RX_EHT_SIG_NON_OFDMA_INFO0_PRE_FEC_PAD_FACTOR	GENMASK(11, 10)
-#define HAL_RX_EHT_SIG_NON_OFDMA_INFO0_DISAMBIGUITY		BIT(12)
-#define HAL_RX_EHT_SIG_NON_OFDMA_INFO0_DISREGARD		GENMASK(16, 13)
-#define HAL_RX_EHT_SIG_NON_OFDMA_INFO0_NUM_USERS		GENMASK(19, 17)
-
-struct hal_eht_sig_non_ofdma_cmn_eb {
-	__le32 info0;
-	union hal_eht_sig_user_field user_field;
-} __packed;
-
-#define HAL_RX_EHT_SIG_OFDMA_EB1_SPATIAL_REUSE		GENMASK_ULL(3, 0)
-#define HAL_RX_EHT_SIG_OFDMA_EB1_GI_LTF			GENMASK_ULL(5, 4)
-#define HAL_RX_EHT_SIG_OFDMA_EB1_NUM_LFT_SYM		GENMASK_ULL(8, 6)
-#define HAL_RX_EHT_SIG_OFDMA_EB1_LDPC_EXTRA_SYM		BIT(9)
-#define HAL_RX_EHT_SIG_OFDMA_EB1_PRE_FEC_PAD_FACTOR	GENMASK_ULL(11, 10)
-#define HAL_RX_EHT_SIG_OFDMA_EB1_PRE_DISAMBIGUITY	BIT(12)
-#define HAL_RX_EHT_SIG_OFDMA_EB1_DISREGARD		GENMASK_ULL(16, 13)
-#define HAL_RX_EHT_SIG_OFDMA_EB1_RU_ALLOC_1_1		GENMASK_ULL(25, 17)
-#define HAL_RX_EHT_SIG_OFDMA_EB1_RU_ALLOC_1_2		GENMASK_ULL(34, 26)
-#define HAL_RX_EHT_SIG_OFDMA_EB1_CRC			GENMASK_ULL(30, 27)
-
-struct hal_eht_sig_ofdma_cmn_eb1 {
-	__le64 info0;
-} __packed;
-
-#define HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_1		GENMASK_ULL(8, 0)
-#define HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_2		GENMASK_ULL(17, 9)
-#define HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_3		GENMASK_ULL(26, 18)
-#define HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_4		GENMASK_ULL(35, 27)
-#define HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_5		GENMASK_ULL(44, 36)
-#define HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_6		GENMASK_ULL(53, 45)
-#define HAL_RX_EHT_SIG_OFDMA_EB2_MCS			GNEMASK_ULL(57, 54)
-
-struct hal_eht_sig_ofdma_cmn_eb2 {
-	__le64 info0;
-} __packed;
-
-struct hal_eht_sig_ofdma_cmn_eb {
-	struct hal_eht_sig_ofdma_cmn_eb1 eb1;
-	struct hal_eht_sig_ofdma_cmn_eb2 eb2;
-	union hal_eht_sig_user_field user_field;
-} __packed;
-
-enum hal_eht_bw {
-	HAL_EHT_BW_20,
-	HAL_EHT_BW_40,
-	HAL_EHT_BW_80,
-	HAL_EHT_BW_160,
-	HAL_EHT_BW_320_1,
-	HAL_EHT_BW_320_2,
-};
-
-#define HAL_RX_USIG_CMN_INFO0_PHY_VERSION	GENMASK(2, 0)
-#define HAL_RX_USIG_CMN_INFO0_BW		GENMASK(5, 3)
-#define HAL_RX_USIG_CMN_INFO0_UL_DL		BIT(6)
-#define HAL_RX_USIG_CMN_INFO0_BSS_COLOR		GENMASK(12, 7)
-#define HAL_RX_USIG_CMN_INFO0_TXOP		GENMASK(19, 13)
-#define HAL_RX_USIG_CMN_INFO0_DISREGARD		GENMASK(25, 20)
-#define HAL_RX_USIG_CMN_INFO0_VALIDATE		BIT(26)
-
-struct hal_mon_usig_cmn {
-	__le32 info0;
-} __packed;
-
-#define HAL_RX_USIG_TB_INFO0_PPDU_TYPE_COMP_MODE	GENMASK(1, 0)
-#define HAL_RX_USIG_TB_INFO0_VALIDATE			BIT(2)
-#define HAL_RX_USIG_TB_INFO0_SPATIAL_REUSE_1		GENMASK(6, 3)
-#define HAL_RX_USIG_TB_INFO0_SPATIAL_REUSE_2		GENMASK(10, 7)
-#define HAL_RX_USIG_TB_INFO0_DISREGARD_1		GENMASK(15, 11)
-#define HAL_RX_USIG_TB_INFO0_CRC			GENMASK(19, 16)
-#define HAL_RX_USIG_TB_INFO0_TAIL			GENMASK(25, 20)
-#define HAL_RX_USIG_TB_INFO0_RX_INTEG_CHECK_PASS	BIT(31)
-
-struct hal_mon_usig_tb {
-	__le32 info0;
-} __packed;
-
-#define HAL_RX_USIG_MU_INFO0_PPDU_TYPE_COMP_MODE	GENMASK(1, 0)
-#define HAL_RX_USIG_MU_INFO0_VALIDATE_1			BIT(2)
-#define HAL_RX_USIG_MU_INFO0_PUNC_CH_INFO		GENMASK(7, 3)
-#define HAL_RX_USIG_MU_INFO0_VALIDATE_2			BIT(8)
-#define HAL_RX_USIG_MU_INFO0_EHT_SIG_MCS		GENMASK(10, 9)
-#define HAL_RX_USIG_MU_INFO0_NUM_EHT_SIG_SYM		GENMASK(15, 11)
-#define HAL_RX_USIG_MU_INFO0_CRC			GENMASK(20, 16)
-#define HAL_RX_USIG_MU_INFO0_TAIL			GENMASK(26, 21)
-#define HAL_RX_USIG_MU_INFO0_RX_INTEG_CHECK_PASS	BIT(31)
-
-struct hal_mon_usig_mu {
-	__le32 info0;
-} __packed;
-
-union hal_mon_usig_non_cmn {
-	struct hal_mon_usig_tb tb;
-	struct hal_mon_usig_mu mu;
-};
-
-struct hal_mon_usig_hdr {
-	struct hal_mon_usig_cmn cmn;
-	union hal_mon_usig_non_cmn non_cmn;
-} __packed;
-
-#define HAL_RX_USR_INFO0_PHY_PPDU_ID		GENMASK(15, 0)
-#define HAL_RX_USR_INFO0_USR_RSSI		GENMASK(23, 16)
-#define HAL_RX_USR_INFO0_PKT_TYPE		GENMASK(27, 24)
-#define HAL_RX_USR_INFO0_STBC			BIT(28)
-#define HAL_RX_USR_INFO0_RECEPTION_TYPE		GENMASK(31, 29)
-
-#define HAL_RX_USR_INFO1_MCS			GENMASK(3, 0)
-#define HAL_RX_USR_INFO1_SGI			GENMASK(5, 4)
-#define HAL_RX_USR_INFO1_HE_RANGING_NDP		BIT(6)
-#define HAL_RX_USR_INFO1_MIMO_SS_BITMAP		GENMASK(15, 8)
-#define HAL_RX_USR_INFO1_RX_BW			GENMASK(18, 16)
-#define HAL_RX_USR_INFO1_DL_OFMDA_USR_IDX	GENMASK(31, 24)
-
-#define HAL_RX_USR_INFO2_DL_OFDMA_CONTENT_CHAN	BIT(0)
-#define HAL_RX_USR_INFO2_NSS			GENMASK(10, 8)
-#define HAL_RX_USR_INFO2_STREAM_OFFSET		GENMASK(13, 11)
-#define HAL_RX_USR_INFO2_STA_DCM		BIT(14)
-#define HAL_RX_USR_INFO2_LDPC			BIT(15)
-#define HAL_RX_USR_INFO2_RU_TYPE_80_0		GENMASK(19, 16)
-#define HAL_RX_USR_INFO2_RU_TYPE_80_1		GENMASK(23, 20)
-#define HAL_RX_USR_INFO2_RU_TYPE_80_2		GENMASK(27, 24)
-#define HAL_RX_USR_INFO2_RU_TYPE_80_3		GENMASK(31, 28)
-
-#define HAL_RX_USR_INFO3_RU_START_IDX_80_0	GENMASK(5, 0)
-#define HAL_RX_USR_INFO3_RU_START_IDX_80_1	GENMASK(13, 8)
-#define HAL_RX_USR_INFO3_RU_START_IDX_80_2	GENMASK(21, 16)
-#define HAL_RX_USR_INFO3_RU_START_IDX_80_3	GENMASK(29, 24)
-
-struct hal_receive_user_info {
-	__le32 info0;
-	__le32 info1;
-	__le32 info2;
-	__le32 info3;
-	__le32 user_fd_rssi_seg0;
-	__le32 user_fd_rssi_seg1;
-	__le32 user_fd_rssi_seg2;
-	__le32 user_fd_rssi_seg3;
-} __packed;
-
-enum hal_mon_reception_type {
-	HAL_RECEPTION_TYPE_SU,
-	HAL_RECEPTION_TYPE_DL_MU_MIMO,
-	HAL_RECEPTION_TYPE_DL_MU_OFMA,
-	HAL_RECEPTION_TYPE_DL_MU_OFDMA_MIMO,
-	HAL_RECEPTION_TYPE_UL_MU_MIMO,
-	HAL_RECEPTION_TYPE_UL_MU_OFDMA,
-	HAL_RECEPTION_TYPE_UL_MU_OFDMA_MIMO,
-};
-
-/* Different allowed RU in 11BE */
-#define HAL_EHT_RU_26		0ULL
-#define HAL_EHT_RU_52		1ULL
-#define HAL_EHT_RU_78		2ULL
-#define HAL_EHT_RU_106		3ULL
-#define HAL_EHT_RU_132		4ULL
-#define HAL_EHT_RU_242		5ULL
-#define HAL_EHT_RU_484		6ULL
-#define HAL_EHT_RU_726		7ULL
-#define HAL_EHT_RU_996		8ULL
-#define HAL_EHT_RU_996x2	9ULL
-#define HAL_EHT_RU_996x3	10ULL
-#define HAL_EHT_RU_996x4	11ULL
-#define HAL_EHT_RU_NONE		15ULL
-#define HAL_EHT_RU_INVALID	31ULL
-/* MRUs spanning above 80Mhz
- * HAL_EHT_RU_996_484 = HAL_EHT_RU_484 + HAL_EHT_RU_996 + 4 (reserved)
- */
-#define HAL_EHT_RU_996_484	18ULL
-#define HAL_EHT_RU_996x2_484	28ULL
-#define HAL_EHT_RU_996x3_484	40ULL
-#define HAL_EHT_RU_996_484_242	23ULL
-
-#define NUM_RU_BITS_PER80	16
-#define NUM_RU_BITS_PER20	4
-
-/* Different per_80Mhz band in 320Mhz bandwidth */
-#define HAL_80_0	0
-#define HAL_80_1	1
-#define HAL_80_2	2
-#define HAL_80_3	3
-
-#define HAL_RU_80MHZ(num_band)		((num_band) * NUM_RU_BITS_PER80)
-#define HAL_RU_20MHZ(idx_per_80)	((idx_per_80) * NUM_RU_BITS_PER20)
-
-#define HAL_RU_SHIFT(num_band, idx_per_80)	\
-		(HAL_RU_80MHZ(num_band) + HAL_RU_20MHZ(idx_per_80))
-
-#define HAL_RU(ru, num_band, idx_per_80)	\
-		((u64)(ru) << HAL_RU_SHIFT(num_band, idx_per_80))
-
-/* MRU-996+484 */
-#define HAL_EHT_RU_996_484_0	(HAL_RU(HAL_EHT_RU_484, HAL_80_0, 1) |	\
-				 HAL_RU(HAL_EHT_RU_996, HAL_80_1, 0))
-#define HAL_EHT_RU_996_484_1	(HAL_RU(HAL_EHT_RU_484, HAL_80_0, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996, HAL_80_1, 0))
-#define HAL_EHT_RU_996_484_2	(HAL_RU(HAL_EHT_RU_996, HAL_80_0, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_1, 1))
-#define HAL_EHT_RU_996_484_3	(HAL_RU(HAL_EHT_RU_996, HAL_80_0, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_1, 0))
-#define HAL_EHT_RU_996_484_4	(HAL_RU(HAL_EHT_RU_484, HAL_80_2, 1) |	\
-				 HAL_RU(HAL_EHT_RU_996, HAL_80_3, 0))
-#define HAL_EHT_RU_996_484_5	(HAL_RU(HAL_EHT_RU_484, HAL_80_2, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996, HAL_80_3, 0))
-#define HAL_EHT_RU_996_484_6	(HAL_RU(HAL_EHT_RU_996, HAL_80_2, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_3, 1))
-#define HAL_EHT_RU_996_484_7	(HAL_RU(HAL_EHT_RU_996, HAL_80_2, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_3, 0))
-
-/* MRU-996x2+484 */
-#define HAL_EHT_RU_996x2_484_0	(HAL_RU(HAL_EHT_RU_484, HAL_80_0, 1) |	\
-				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_2, 0))
-#define HAL_EHT_RU_996x2_484_1	(HAL_RU(HAL_EHT_RU_484, HAL_80_0, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_2, 0))
-#define HAL_EHT_RU_996x2_484_2	(HAL_RU(HAL_EHT_RU_996x2, HAL_80_0, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_1, 1) |	\
-				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_2, 0))
-#define HAL_EHT_RU_996x2_484_3	(HAL_RU(HAL_EHT_RU_996x2, HAL_80_0, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_2, 0))
-#define HAL_EHT_RU_996x2_484_4	(HAL_RU(HAL_EHT_RU_996x2, HAL_80_0, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_2, 1))
-#define HAL_EHT_RU_996x2_484_5	(HAL_RU(HAL_EHT_RU_996x2, HAL_80_0, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_2, 0))
-#define HAL_EHT_RU_996x2_484_6	(HAL_RU(HAL_EHT_RU_484, HAL_80_1, 1) |	\
-				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_2, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_3, 0))
-#define HAL_EHT_RU_996x2_484_7	(HAL_RU(HAL_EHT_RU_484, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_2, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_3, 0))
-#define HAL_EHT_RU_996x2_484_8	(HAL_RU(HAL_EHT_RU_996x2, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_2, 1) |	\
-				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_3, 0))
-#define HAL_EHT_RU_996x2_484_9	(HAL_RU(HAL_EHT_RU_996x2, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_2, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_3, 0))
-#define HAL_EHT_RU_996x2_484_10	(HAL_RU(HAL_EHT_RU_996x2, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_2, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_3, 1))
-#define HAL_EHT_RU_996x2_484_11	(HAL_RU(HAL_EHT_RU_996x2, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_2, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_3, 0))
-
-/* MRU-996x3+484 */
-#define HAL_EHT_RU_996x3_484_0	(HAL_RU(HAL_EHT_RU_484, HAL_80_0, 1) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_2, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_3, 0))
-#define HAL_EHT_RU_996x3_484_1	(HAL_RU(HAL_EHT_RU_484, HAL_80_0, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_2, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_3, 0))
-#define HAL_EHT_RU_996x3_484_2	(HAL_RU(HAL_EHT_RU_996x3, HAL_80_0, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_1, 1) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_2, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_3, 0))
-#define HAL_EHT_RU_996x3_484_3	(HAL_RU(HAL_EHT_RU_996x3, HAL_80_0, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_2, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_3, 0))
-#define HAL_EHT_RU_996x3_484_4	(HAL_RU(HAL_EHT_RU_996x3, HAL_80_0, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_2, 1) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_3, 0))
-#define HAL_EHT_RU_996x3_484_5	(HAL_RU(HAL_EHT_RU_996x3, HAL_80_0, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_2, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_3, 0))
-#define HAL_EHT_RU_996x3_484_6	(HAL_RU(HAL_EHT_RU_996x3, HAL_80_0, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_2, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_3, 1))
-#define HAL_EHT_RU_996x3_484_7	(HAL_RU(HAL_EHT_RU_996x3, HAL_80_0, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_1, 0) |	\
-				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_2, 0) |	\
-				 HAL_RU(HAL_EHT_RU_484, HAL_80_3, 0))
-
-#define HAL_RU_PER80(ru_per80, num_80mhz, ru_idx_per80mhz) \
-			(HAL_RU(ru_per80, num_80mhz, ru_idx_per80mhz))
-
-#define RU_INVALID		0
-#define RU_26			1
-#define RU_52			2
-#define RU_106			4
-#define RU_242			9
-#define RU_484			18
-#define RU_996			37
-#define RU_2X996		74
-#define RU_3X996		111
-#define RU_4X996		148
-#define RU_52_26		(RU_52 + RU_26)
-#define RU_106_26		(RU_106 + RU_26)
-#define RU_484_242		(RU_484 + RU_242)
-#define RU_996_484		(RU_996 + RU_484)
-#define RU_996_484_242		(RU_996 + RU_484_242)
-#define RU_2X996_484		(RU_2X996 + RU_484)
-#define RU_3X996_484		(RU_3X996 + RU_484)
-
-enum ath12k_eht_ru_size {
-	ATH12K_EHT_RU_26,
-	ATH12K_EHT_RU_52,
-	ATH12K_EHT_RU_106,
-	ATH12K_EHT_RU_242,
-	ATH12K_EHT_RU_484,
-	ATH12K_EHT_RU_996,
-	ATH12K_EHT_RU_996x2,
-	ATH12K_EHT_RU_996x4,
-	ATH12K_EHT_RU_52_26,
-	ATH12K_EHT_RU_106_26,
-	ATH12K_EHT_RU_484_242,
-	ATH12K_EHT_RU_996_484,
-	ATH12K_EHT_RU_996_484_242,
-	ATH12K_EHT_RU_996x2_484,
-	ATH12K_EHT_RU_996x3,
-	ATH12K_EHT_RU_996x3_484,
-
-	/* Keep last */
-	ATH12K_EHT_RU_INVALID,
-};
-
-#define HAL_RX_RU_ALLOC_TYPE_MAX	ATH12K_EHT_RU_INVALID
-
-static inline
-enum nl80211_he_ru_alloc ath12k_he_ru_tones_to_nl80211_he_ru_alloc(u16 ru_tones)
-{
-	enum nl80211_he_ru_alloc ret;
-
-	switch (ru_tones) {
-	case RU_52:
-		ret = NL80211_RATE_INFO_HE_RU_ALLOC_52;
-		break;
-	case RU_106:
-		ret = NL80211_RATE_INFO_HE_RU_ALLOC_106;
-		break;
-	case RU_242:
-		ret = NL80211_RATE_INFO_HE_RU_ALLOC_242;
-		break;
-	case RU_484:
-		ret = NL80211_RATE_INFO_HE_RU_ALLOC_484;
-		break;
-	case RU_996:
-		ret = NL80211_RATE_INFO_HE_RU_ALLOC_996;
-		break;
-	case RU_2X996:
-		ret = NL80211_RATE_INFO_HE_RU_ALLOC_2x996;
-		break;
-	case RU_26:
-		fallthrough;
-	default:
-		ret = NL80211_RATE_INFO_HE_RU_ALLOC_26;
-		break;
-	}
-	return ret;
-}
-
-void ath12k_hal_reo_status_queue_stats(struct ath12k_base *ab,
-				       struct hal_tlv_64_hdr *tlv,
-				       struct hal_reo_status *status);
-void ath12k_hal_reo_flush_queue_status(struct ath12k_base *ab,
-				       struct hal_tlv_64_hdr *tlv,
-				       struct hal_reo_status *status);
-void ath12k_hal_reo_flush_cache_status(struct ath12k_base *ab,
-				       struct hal_tlv_64_hdr *tlv,
-				       struct hal_reo_status *status);
-void ath12k_hal_reo_unblk_cache_status(struct ath12k_base *ab,
-				       struct hal_tlv_64_hdr *tlv,
-				       struct hal_reo_status *status);
-void ath12k_hal_reo_flush_timeout_list_status(struct ath12k_base *ab,
-					      struct hal_tlv_64_hdr *tlv,
-					      struct hal_reo_status *status);
-void ath12k_hal_reo_desc_thresh_reached_status(struct ath12k_base *ab,
-					       struct hal_tlv_64_hdr *tlv,
-					       struct hal_reo_status *status);
-void ath12k_hal_reo_update_rx_reo_queue_status(struct ath12k_base *ab,
-					       struct hal_tlv_64_hdr *tlv,
-					       struct hal_reo_status *status);
-void ath12k_hal_rx_msdu_link_info_get(struct hal_rx_msdu_link *link, u32 *num_msdus,
-				      u32 *msdu_cookies,
-				      enum hal_rx_buf_return_buf_manager *rbm);
-void ath12k_hal_rx_msdu_link_desc_set(struct ath12k_base *ab,
-				      struct hal_wbm_release_ring *desc,
-				      struct ath12k_buffer_addr *buf_addr_info,
-				      enum hal_wbm_rel_bm_act action);
-void ath12k_hal_rx_buf_addr_info_set(struct ath12k_buffer_addr *binfo,
-				     dma_addr_t paddr, u32 cookie, u8 manager);
-void ath12k_hal_rx_buf_addr_info_get(struct ath12k_buffer_addr *binfo,
-				     dma_addr_t *paddr,
-				     u32 *cookie, u8 *rbm);
-int ath12k_hal_desc_reo_parse_err(struct ath12k_base *ab,
-				  struct hal_reo_dest_ring *desc,
-				  dma_addr_t *paddr, u32 *desc_bank);
-int ath12k_hal_wbm_desc_parse_err(struct ath12k_base *ab, void *desc,
-				  struct hal_rx_wbm_rel_info *rel_info);
-void ath12k_hal_rx_reo_ent_paddr_get(struct ath12k_base *ab,
-				     struct ath12k_buffer_addr *buff_addr,
-				     dma_addr_t *paddr, u32 *cookie);
-void ath12k_hal_rx_reo_ent_buf_paddr_get(void *rx_desc, dma_addr_t *paddr, u32 *sw_cookie,
-					 struct ath12k_buffer_addr **pp_buf_addr,
-					 u8 *rbm, u32 *msdu_cnt);
-void ath12k_hal_rx_msdu_list_get(struct ath12k *ar,
-				 struct hal_rx_msdu_link *link_desc,
-				 struct hal_rx_msdu_list *msdu_list,
-				 u16 *num_msdus);
-
-#endif
diff --git a/drivers/net/wireless/ath/ath12k/hal_tx.c b/drivers/net/wireless/ath/ath12k/hal_tx.c
deleted file mode 100644
index 869e07e406fe..000000000000
--- a/drivers/net/wireless/ath/ath12k/hal_tx.c
+++ /dev/null
@@ -1,145 +0,0 @@
-// SPDX-License-Identifier: BSD-3-Clause-Clear
-/*
- * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
- */
-
-#include "hal_desc.h"
-#include "hal.h"
-#include "hal_tx.h"
-#include "hif.h"
-
-#define DSCP_TID_MAP_TBL_ENTRY_SIZE 64
-
-/* dscp_tid_map - Default DSCP-TID mapping
- *=================
- * DSCP        TID
- *=================
- * 000xxx      0
- * 001xxx      1
- * 010xxx      2
- * 011xxx      3
- * 100xxx      4
- * 101xxx      5
- * 110xxx      6
- * 111xxx      7
- */
-static inline u8 dscp2tid(u8 dscp)
-{
-	return dscp >> 3;
-}
-
-void ath12k_hal_tx_cmd_desc_setup(struct ath12k_base *ab,
-				  struct hal_tcl_data_cmd *tcl_cmd,
-				  struct hal_tx_info *ti)
-{
-	tcl_cmd->buf_addr_info.info0 =
-		le32_encode_bits(ti->paddr, BUFFER_ADDR_INFO0_ADDR);
-	tcl_cmd->buf_addr_info.info1 =
-		le32_encode_bits(((uint64_t)ti->paddr >> HAL_ADDR_MSB_REG_SHIFT),
-				 BUFFER_ADDR_INFO1_ADDR);
-	tcl_cmd->buf_addr_info.info1 |=
-		le32_encode_bits((ti->rbm_id), BUFFER_ADDR_INFO1_RET_BUF_MGR) |
-		le32_encode_bits(ti->desc_id, BUFFER_ADDR_INFO1_SW_COOKIE);
-
-	tcl_cmd->info0 =
-		le32_encode_bits(ti->type, HAL_TCL_DATA_CMD_INFO0_DESC_TYPE) |
-		le32_encode_bits(ti->bank_id, HAL_TCL_DATA_CMD_INFO0_BANK_ID);
-
-	tcl_cmd->info1 =
-		le32_encode_bits(ti->meta_data_flags,
-				 HAL_TCL_DATA_CMD_INFO1_CMD_NUM);
-
-	tcl_cmd->info2 = cpu_to_le32(ti->flags0) |
-		le32_encode_bits(ti->data_len, HAL_TCL_DATA_CMD_INFO2_DATA_LEN) |
-		le32_encode_bits(ti->pkt_offset, HAL_TCL_DATA_CMD_INFO2_PKT_OFFSET);
-
-	tcl_cmd->info3 = cpu_to_le32(ti->flags1) |
-		le32_encode_bits(ti->tid, HAL_TCL_DATA_CMD_INFO3_TID) |
-		le32_encode_bits(ti->lmac_id, HAL_TCL_DATA_CMD_INFO3_PMAC_ID) |
-		le32_encode_bits(ti->vdev_id, HAL_TCL_DATA_CMD_INFO3_VDEV_ID);
-
-	tcl_cmd->info4 = le32_encode_bits(ti->bss_ast_idx,
-					  HAL_TCL_DATA_CMD_INFO4_SEARCH_INDEX) |
-			 le32_encode_bits(ti->bss_ast_hash,
-					  HAL_TCL_DATA_CMD_INFO4_CACHE_SET_NUM);
-	tcl_cmd->info5 = 0;
-}
-
-void ath12k_hal_tx_set_dscp_tid_map(struct ath12k_base *ab, int id)
-{
-	u32 ctrl_reg_val;
-	u32 addr;
-	u8 hw_map_val[HAL_DSCP_TID_TBL_SIZE], dscp, tid;
-	int i;
-	u32 value;
-
-	ctrl_reg_val = ath12k_hif_read32(ab, HAL_SEQ_WCSS_UMAC_TCL_REG +
-					 HAL_TCL1_RING_CMN_CTRL_REG);
-	/* Enable read/write access */
-	ctrl_reg_val |= HAL_TCL1_RING_CMN_CTRL_DSCP_TID_MAP_PROG_EN;
-	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_TCL_REG +
-			   HAL_TCL1_RING_CMN_CTRL_REG, ctrl_reg_val);
-
-	addr = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_DSCP_TID_MAP +
-	       (4 * id * (HAL_DSCP_TID_TBL_SIZE / 4));
-
-	/* Configure each DSCP-TID mapping in three bits there by configure
-	 * three bytes in an iteration.
-	 */
-	for (i = 0, dscp = 0; i < HAL_DSCP_TID_TBL_SIZE; i += 3) {
-		tid = dscp2tid(dscp);
-		value = u32_encode_bits(tid, HAL_TCL1_RING_FIELD_DSCP_TID_MAP0);
-		dscp++;
-
-		tid = dscp2tid(dscp);
-		value |= u32_encode_bits(tid, HAL_TCL1_RING_FIELD_DSCP_TID_MAP1);
-		dscp++;
-
-		tid = dscp2tid(dscp);
-		value |= u32_encode_bits(tid, HAL_TCL1_RING_FIELD_DSCP_TID_MAP2);
-		dscp++;
-
-		tid = dscp2tid(dscp);
-		value |= u32_encode_bits(tid, HAL_TCL1_RING_FIELD_DSCP_TID_MAP3);
-		dscp++;
-
-		tid = dscp2tid(dscp);
-		value |= u32_encode_bits(tid, HAL_TCL1_RING_FIELD_DSCP_TID_MAP4);
-		dscp++;
-
-		tid = dscp2tid(dscp);
-		value |= u32_encode_bits(tid, HAL_TCL1_RING_FIELD_DSCP_TID_MAP5);
-		dscp++;
-
-		tid = dscp2tid(dscp);
-		value |= u32_encode_bits(tid, HAL_TCL1_RING_FIELD_DSCP_TID_MAP6);
-		dscp++;
-
-		tid = dscp2tid(dscp);
-		value |= u32_encode_bits(tid, HAL_TCL1_RING_FIELD_DSCP_TID_MAP7);
-		dscp++;
-
-		memcpy(&hw_map_val[i], &value, 3);
-	}
-
-	for (i = 0; i < HAL_DSCP_TID_TBL_SIZE; i += 4) {
-		ath12k_hif_write32(ab, addr, *(u32 *)&hw_map_val[i]);
-		addr += 4;
-	}
-
-	/* Disable read/write access */
-	ctrl_reg_val = ath12k_hif_read32(ab, HAL_SEQ_WCSS_UMAC_TCL_REG +
-					 HAL_TCL1_RING_CMN_CTRL_REG);
-	ctrl_reg_val &= ~HAL_TCL1_RING_CMN_CTRL_DSCP_TID_MAP_PROG_EN;
-	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_TCL_REG +
-			   HAL_TCL1_RING_CMN_CTRL_REG,
-			   ctrl_reg_val);
-}
-
-void ath12k_hal_tx_configure_bank_register(struct ath12k_base *ab, u32 bank_config,
-					   u8 bank_id)
-{
-	ath12k_hif_write32(ab, HAL_TCL_SW_CONFIG_BANK_ADDR + 4 * bank_id,
-			   bank_config);
-}
diff --git a/drivers/net/wireless/ath/ath12k/hal_tx.h b/drivers/net/wireless/ath/ath12k/hal_tx.h
deleted file mode 100644
index eb065a79f6c6..000000000000
--- a/drivers/net/wireless/ath/ath12k/hal_tx.h
+++ /dev/null
@@ -1,200 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause-Clear */
-/*
- * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2022, 2024-2025 Qualcomm Innovation Center, Inc.
- * All rights reserved.
- */
-
-#ifndef ATH12K_HAL_TX_H
-#define ATH12K_HAL_TX_H
-
-#include "hal_desc.h"
-#include "core.h"
-
-#define HAL_TX_ADDRX_EN			1
-#define HAL_TX_ADDRY_EN			2
-
-#define HAL_TX_ADDR_SEARCH_DEFAULT	0
-#define HAL_TX_ADDR_SEARCH_INDEX	1
-
-/* TODO: check all these data can be managed with struct ath12k_tx_desc_info for perf */
-struct hal_tx_info {
-	u16 meta_data_flags; /* %HAL_TCL_DATA_CMD_INFO0_META_ */
-	u8 ring_id;
-	u8 rbm_id;
-	u32 desc_id;
-	enum hal_tcl_desc_type type;
-	enum hal_tcl_encap_type encap_type;
-	dma_addr_t paddr;
-	u32 data_len;
-	u32 pkt_offset;
-	enum hal_encrypt_type encrypt_type;
-	u32 flags0; /* %HAL_TCL_DATA_CMD_INFO1_ */
-	u32 flags1; /* %HAL_TCL_DATA_CMD_INFO2_ */
-	u16 addr_search_flags; /* %HAL_TCL_DATA_CMD_INFO0_ADDR(X/Y)_ */
-	u16 bss_ast_hash;
-	u16 bss_ast_idx;
-	u8 tid;
-	u8 search_type; /* %HAL_TX_ADDR_SEARCH_ */
-	u8 lmac_id;
-	u8 vdev_id;
-	u8 dscp_tid_tbl_idx;
-	bool enable_mesh;
-	int bank_id;
-};
-
-/* TODO: Check if the actual desc macros can be used instead */
-#define HAL_TX_STATUS_FLAGS_FIRST_MSDU		BIT(0)
-#define HAL_TX_STATUS_FLAGS_LAST_MSDU		BIT(1)
-#define HAL_TX_STATUS_FLAGS_MSDU_IN_AMSDU	BIT(2)
-#define HAL_TX_STATUS_FLAGS_RATE_STATS_VALID	BIT(3)
-#define HAL_TX_STATUS_FLAGS_RATE_LDPC		BIT(4)
-#define HAL_TX_STATUS_FLAGS_RATE_STBC		BIT(5)
-#define HAL_TX_STATUS_FLAGS_OFDMA		BIT(6)
-
-#define HAL_TX_STATUS_DESC_LEN		sizeof(struct hal_wbm_release_ring)
-
-/* Tx status parsed from srng desc */
-struct hal_tx_status {
-	enum hal_wbm_rel_src_module buf_rel_source;
-	enum hal_wbm_tqm_rel_reason status;
-	s8 ack_rssi;
-	u32 flags; /* %HAL_TX_STATUS_FLAGS_ */
-	u32 ppdu_id;
-	u8 try_cnt;
-	u8 tid;
-	u16 peer_id;
-	enum hal_tx_rate_stats_pkt_type pkt_type;
-	enum hal_tx_rate_stats_sgi sgi;
-	enum ath12k_supported_bw bw;
-	u8 mcs;
-	u16 tones;
-	u8 ofdma;
-};
-
-#define HAL_TX_PHY_DESC_INFO0_BF_TYPE		GENMASK(17, 16)
-#define HAL_TX_PHY_DESC_INFO0_PREAMBLE_11B	BIT(20)
-#define HAL_TX_PHY_DESC_INFO0_PKT_TYPE		GENMASK(24, 21)
-#define HAL_TX_PHY_DESC_INFO0_BANDWIDTH		GENMASK(30, 28)
-#define HAL_TX_PHY_DESC_INFO1_MCS		GENMASK(3, 0)
-#define HAL_TX_PHY_DESC_INFO1_STBC		BIT(6)
-#define HAL_TX_PHY_DESC_INFO2_NSS		GENMASK(23, 21)
-#define HAL_TX_PHY_DESC_INFO3_AP_PKT_BW		GENMASK(6, 4)
-#define HAL_TX_PHY_DESC_INFO3_LTF_SIZE		GENMASK(20, 19)
-#define HAL_TX_PHY_DESC_INFO3_ACTIVE_CHANNEL	GENMASK(17, 15)
-
-struct hal_tx_phy_desc {
-	__le32 info0;
-	__le32 info1;
-	__le32 info2;
-	__le32 info3;
-} __packed;
-
-#define HAL_TX_FES_STAT_PROT_INFO0_STRT_FRM_TS_15_0	GENMASK(15, 0)
-#define HAL_TX_FES_STAT_PROT_INFO0_STRT_FRM_TS_31_16	GENMASK(31, 16)
-#define HAL_TX_FES_STAT_PROT_INFO1_END_FRM_TS_15_0	GENMASK(15, 0)
-#define HAL_TX_FES_STAT_PROT_INFO1_END_FRM_TS_31_16	GENMASK(31, 16)
-
-struct hal_tx_fes_status_prot {
-	__le64 reserved;
-	__le32 info0;
-	__le32 info1;
-	__le32 reserved1[11];
-} __packed;
-
-#define HAL_TX_FES_STAT_USR_PPDU_INFO0_DURATION		GENMASK(15, 0)
-
-struct hal_tx_fes_status_user_ppdu {
-	__le64 reserved;
-	__le32 info0;
-	__le32 reserved1[3];
-} __packed;
-
-#define HAL_TX_FES_STAT_STRT_INFO0_PROT_TS_LOWER_32	GENMASK(31, 0)
-#define HAL_TX_FES_STAT_STRT_INFO1_PROT_TS_UPPER_32	GENMASK(31, 0)
-
-struct hal_tx_fes_status_start_prot {
-	__le32 info0;
-	__le32 info1;
-	__le64 reserved;
-} __packed;
-
-#define HAL_TX_FES_STATUS_START_INFO0_MEDIUM_PROT_TYPE	GENMASK(29, 27)
-
-struct hal_tx_fes_status_start {
-	__le32 reserved;
-	__le32 info0;
-	__le64 reserved1;
-} __packed;
-
-#define HAL_TX_Q_EXT_INFO0_FRAME_CTRL		GENMASK(15, 0)
-#define HAL_TX_Q_EXT_INFO0_QOS_CTRL		GENMASK(31, 16)
-#define HAL_TX_Q_EXT_INFO1_AMPDU_FLAG		BIT(0)
-
-struct hal_tx_queue_exten {
-	__le32 info0;
-	__le32 info1;
-} __packed;
-
-#define HAL_TX_FES_SETUP_INFO0_NUM_OF_USERS	GENMASK(28, 23)
-
-struct hal_tx_fes_setup {
-	__le32 schedule_id;
-	__le32 info0;
-	__le64 reserved;
-} __packed;
-
-#define HAL_TX_PPDU_SETUP_INFO0_MEDIUM_PROT_TYPE	GENMASK(2, 0)
-#define HAL_TX_PPDU_SETUP_INFO1_PROT_FRAME_ADDR1_31_0	GENMASK(31, 0)
-#define HAL_TX_PPDU_SETUP_INFO2_PROT_FRAME_ADDR1_47_32	GENMASK(15, 0)
-#define HAL_TX_PPDU_SETUP_INFO2_PROT_FRAME_ADDR2_15_0	GENMASK(31, 16)
-#define HAL_TX_PPDU_SETUP_INFO3_PROT_FRAME_ADDR2_47_16	GENMASK(31, 0)
-#define HAL_TX_PPDU_SETUP_INFO4_PROT_FRAME_ADDR3_31_0	GENMASK(31, 0)
-#define HAL_TX_PPDU_SETUP_INFO5_PROT_FRAME_ADDR3_47_32	GENMASK(15, 0)
-#define HAL_TX_PPDU_SETUP_INFO5_PROT_FRAME_ADDR4_15_0	GENMASK(31, 16)
-#define HAL_TX_PPDU_SETUP_INFO6_PROT_FRAME_ADDR4_47_16	GENMASK(31, 0)
-
-struct hal_tx_pcu_ppdu_setup_init {
-	__le32 info0;
-	__le32 info1;
-	__le32 info2;
-	__le32 info3;
-	__le32 reserved;
-	__le32 info4;
-	__le32 info5;
-	__le32 info6;
-} __packed;
-
-#define HAL_TX_FES_STATUS_END_INFO0_START_TIMESTAMP_15_0	GENMASK(15, 0)
-#define HAL_TX_FES_STATUS_END_INFO0_START_TIMESTAMP_31_16	GENMASK(31, 16)
-
-struct hal_tx_fes_status_end {
-	__le32 reserved[2];
-	__le32 info0;
-	__le32 reserved1[19];
-} __packed;
-
-#define HAL_TX_BANK_CONFIG_EPD			BIT(0)
-#define HAL_TX_BANK_CONFIG_ENCAP_TYPE		GENMASK(2, 1)
-#define HAL_TX_BANK_CONFIG_ENCRYPT_TYPE		GENMASK(6, 3)
-#define HAL_TX_BANK_CONFIG_SRC_BUFFER_SWAP	BIT(7)
-#define HAL_TX_BANK_CONFIG_LINK_META_SWAP	BIT(8)
-#define HAL_TX_BANK_CONFIG_INDEX_LOOKUP_EN	BIT(9)
-#define HAL_TX_BANK_CONFIG_ADDRX_EN		BIT(10)
-#define HAL_TX_BANK_CONFIG_ADDRY_EN		BIT(11)
-#define HAL_TX_BANK_CONFIG_MESH_EN		GENMASK(13, 12)
-#define HAL_TX_BANK_CONFIG_VDEV_ID_CHECK_EN	BIT(14)
-#define HAL_TX_BANK_CONFIG_PMAC_ID		GENMASK(16, 15)
-/* STA mode will have MCAST_PKT_CTRL instead of DSCP_TID_MAP bitfield */
-#define HAL_TX_BANK_CONFIG_DSCP_TIP_MAP_ID	GENMASK(22, 17)
-
-void ath12k_hal_tx_cmd_desc_setup(struct ath12k_base *ab,
-				  struct hal_tcl_data_cmd *tcl_cmd,
-				  struct hal_tx_info *ti);
-void ath12k_hal_tx_set_dscp_tid_map(struct ath12k_base *ab, int id);
-int ath12k_hal_reo_cmd_send(struct ath12k_base *ab, struct hal_srng *srng,
-			    enum hal_reo_cmd_type type,
-			    struct ath12k_hal_reo_cmd *cmd);
-void ath12k_hal_tx_configure_bank_register(struct ath12k_base *ab, u32 bank_config,
-					   u8 bank_id);
-#endif
diff --git a/drivers/net/wireless/ath/ath12k/htc.c b/drivers/net/wireless/ath/ath12k/htc.c
index d13616bf07f4..92138caa2a82 100644
--- a/drivers/net/wireless/ath/ath12k/htc.c
+++ b/drivers/net/wireless/ath/ath12k/htc.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2022, 2024 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 #include <linux/skbuff.h>
 #include <linux/ctype.h>
@@ -376,6 +376,7 @@ void ath12k_htc_rx_completion_handler(struct ath12k_base *ab,
 out:
 	kfree_skb(skb);
 }
+EXPORT_SYMBOL(ath12k_htc_rx_completion_handler);
 
 static void ath12k_htc_control_rx_complete(struct ath12k_base *ab,
 					   struct sk_buff *skb)
diff --git a/drivers/net/wireless/ath/ath12k/hw.c b/drivers/net/wireless/ath/ath12k/hw.c
deleted file mode 100644
index 6791ae1d64e5..000000000000
--- a/drivers/net/wireless/ath/ath12k/hw.c
+++ /dev/null
@@ -1,1680 +0,0 @@
-// SPDX-License-Identifier: BSD-3-Clause-Clear
-/*
- * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
- */
-
-#include <linux/types.h>
-#include <linux/bitops.h>
-#include <linux/bitfield.h>
-
-#include "debug.h"
-#include "core.h"
-#include "ce.h"
-#include "hw.h"
-#include "mhi.h"
-#include "dp_rx.h"
-#include "peer.h"
-
-static const guid_t wcn7850_uuid = GUID_INIT(0xf634f534, 0x6147, 0x11ec,
-					     0x90, 0xd6, 0x02, 0x42,
-					     0xac, 0x12, 0x00, 0x03);
-
-static u8 ath12k_hw_qcn9274_mac_from_pdev_id(int pdev_idx)
-{
-	return pdev_idx;
-}
-
-static int ath12k_hw_mac_id_to_pdev_id_qcn9274(const struct ath12k_hw_params *hw,
-					       int mac_id)
-{
-	return mac_id;
-}
-
-static int ath12k_hw_mac_id_to_srng_id_qcn9274(const struct ath12k_hw_params *hw,
-					       int mac_id)
-{
-	return 0;
-}
-
-static u8 ath12k_hw_get_ring_selector_qcn9274(struct sk_buff *skb)
-{
-	return smp_processor_id();
-}
-
-static bool ath12k_dp_srng_is_comp_ring_qcn9274(int ring_num)
-{
-	if (ring_num < 3 || ring_num == 4)
-		return true;
-
-	return false;
-}
-
-static bool ath12k_is_frame_link_agnostic_qcn9274(struct ath12k_link_vif *arvif,
-						  struct ieee80211_mgmt *mgmt)
-{
-	return ieee80211_is_action(mgmt->frame_control);
-}
-
-static int ath12k_hw_mac_id_to_pdev_id_wcn7850(const struct ath12k_hw_params *hw,
-					       int mac_id)
-{
-	return 0;
-}
-
-static int ath12k_hw_mac_id_to_srng_id_wcn7850(const struct ath12k_hw_params *hw,
-					       int mac_id)
-{
-	return mac_id;
-}
-
-static u8 ath12k_hw_get_ring_selector_wcn7850(struct sk_buff *skb)
-{
-	return skb_get_queue_mapping(skb);
-}
-
-static bool ath12k_dp_srng_is_comp_ring_wcn7850(int ring_num)
-{
-	if (ring_num == 0 || ring_num == 2 || ring_num == 4)
-		return true;
-
-	return false;
-}
-
-static bool ath12k_is_addba_resp_action_code(struct ieee80211_mgmt *mgmt)
-{
-	if (!ieee80211_is_action(mgmt->frame_control))
-		return false;
-
-	if (mgmt->u.action.category != WLAN_CATEGORY_BACK)
-		return false;
-
-	if (mgmt->u.action.u.addba_resp.action_code != WLAN_ACTION_ADDBA_RESP)
-		return false;
-
-	return true;
-}
-
-static bool ath12k_is_frame_link_agnostic_wcn7850(struct ath12k_link_vif *arvif,
-						  struct ieee80211_mgmt *mgmt)
-{
-	struct ieee80211_vif *vif = ath12k_ahvif_to_vif(arvif->ahvif);
-	struct ath12k_hw *ah = ath12k_ar_to_ah(arvif->ar);
-	struct ath12k_base *ab = arvif->ar->ab;
-	__le16 fc = mgmt->frame_control;
-
-	spin_lock_bh(&ab->base_lock);
-	if (!ath12k_peer_find_by_addr(ab, mgmt->da) &&
-	    !ath12k_peer_ml_find(ah, mgmt->da)) {
-		spin_unlock_bh(&ab->base_lock);
-		return false;
-	}
-	spin_unlock_bh(&ab->base_lock);
-
-	if (vif->type == NL80211_IFTYPE_STATION)
-		return arvif->is_up &&
-		       (vif->valid_links == vif->active_links) &&
-		       !ieee80211_is_probe_req(fc) &&
-		       !ieee80211_is_auth(fc) &&
-		       !ieee80211_is_deauth(fc) &&
-		       !ath12k_is_addba_resp_action_code(mgmt);
-
-	if (vif->type == NL80211_IFTYPE_AP)
-		return !(ieee80211_is_probe_resp(fc) || ieee80211_is_auth(fc) ||
-			 ieee80211_is_assoc_resp(fc) || ieee80211_is_reassoc_resp(fc) ||
-			 ath12k_is_addba_resp_action_code(mgmt));
-
-	return false;
-}
-
-static const struct ath12k_hw_ops qcn9274_ops = {
-	.get_hw_mac_from_pdev_id = ath12k_hw_qcn9274_mac_from_pdev_id,
-	.mac_id_to_pdev_id = ath12k_hw_mac_id_to_pdev_id_qcn9274,
-	.mac_id_to_srng_id = ath12k_hw_mac_id_to_srng_id_qcn9274,
-	.rxdma_ring_sel_config = ath12k_dp_rxdma_ring_sel_config_qcn9274,
-	.get_ring_selector = ath12k_hw_get_ring_selector_qcn9274,
-	.dp_srng_is_tx_comp_ring = ath12k_dp_srng_is_comp_ring_qcn9274,
-	.is_frame_link_agnostic = ath12k_is_frame_link_agnostic_qcn9274,
-};
-
-static const struct ath12k_hw_ops wcn7850_ops = {
-	.get_hw_mac_from_pdev_id = ath12k_hw_qcn9274_mac_from_pdev_id,
-	.mac_id_to_pdev_id = ath12k_hw_mac_id_to_pdev_id_wcn7850,
-	.mac_id_to_srng_id = ath12k_hw_mac_id_to_srng_id_wcn7850,
-	.rxdma_ring_sel_config = ath12k_dp_rxdma_ring_sel_config_wcn7850,
-	.get_ring_selector = ath12k_hw_get_ring_selector_wcn7850,
-	.dp_srng_is_tx_comp_ring = ath12k_dp_srng_is_comp_ring_wcn7850,
-	.is_frame_link_agnostic = ath12k_is_frame_link_agnostic_wcn7850,
-};
-
-#define ATH12K_TX_RING_MASK_0 0x1
-#define ATH12K_TX_RING_MASK_1 0x2
-#define ATH12K_TX_RING_MASK_2 0x4
-#define ATH12K_TX_RING_MASK_3 0x8
-#define ATH12K_TX_RING_MASK_4 0x10
-
-#define ATH12K_RX_RING_MASK_0 0x1
-#define ATH12K_RX_RING_MASK_1 0x2
-#define ATH12K_RX_RING_MASK_2 0x4
-#define ATH12K_RX_RING_MASK_3 0x8
-
-#define ATH12K_RX_ERR_RING_MASK_0 0x1
-
-#define ATH12K_RX_WBM_REL_RING_MASK_0 0x1
-
-#define ATH12K_REO_STATUS_RING_MASK_0 0x1
-
-#define ATH12K_HOST2RXDMA_RING_MASK_0 0x1
-
-#define ATH12K_RX_MON_RING_MASK_0 0x1
-#define ATH12K_RX_MON_RING_MASK_1 0x2
-#define ATH12K_RX_MON_RING_MASK_2 0x4
-
-#define ATH12K_TX_MON_RING_MASK_0 0x1
-#define ATH12K_TX_MON_RING_MASK_1 0x2
-
-#define ATH12K_RX_MON_STATUS_RING_MASK_0 0x1
-#define ATH12K_RX_MON_STATUS_RING_MASK_1 0x2
-#define ATH12K_RX_MON_STATUS_RING_MASK_2 0x4
-
-/* Target firmware's Copy Engine configuration. */
-static const struct ce_pipe_config ath12k_target_ce_config_wlan_qcn9274[] = {
-	/* CE0: host->target HTC control and raw streams */
-	{
-		.pipenum = __cpu_to_le32(0),
-		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE1: target->host HTT + HTC control */
-	{
-		.pipenum = __cpu_to_le32(1),
-		.pipedir = __cpu_to_le32(PIPEDIR_IN),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE2: target->host WMI */
-	{
-		.pipenum = __cpu_to_le32(2),
-		.pipedir = __cpu_to_le32(PIPEDIR_IN),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE3: host->target WMI (mac0) */
-	{
-		.pipenum = __cpu_to_le32(3),
-		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE4: host->target HTT */
-	{
-		.pipenum = __cpu_to_le32(4),
-		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
-		.nentries = __cpu_to_le32(256),
-		.nbytes_max = __cpu_to_le32(256),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS | CE_ATTR_DIS_INTR),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE5: target->host Pktlog */
-	{
-		.pipenum = __cpu_to_le32(5),
-		.pipedir = __cpu_to_le32(PIPEDIR_IN),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE6: Reserved for target autonomous hif_memcpy */
-	{
-		.pipenum = __cpu_to_le32(6),
-		.pipedir = __cpu_to_le32(PIPEDIR_INOUT),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(16384),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE7: host->target WMI (mac1) */
-	{
-		.pipenum = __cpu_to_le32(7),
-		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE8: Reserved for target autonomous hif_memcpy */
-	{
-		.pipenum = __cpu_to_le32(8),
-		.pipedir = __cpu_to_le32(PIPEDIR_INOUT),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(16384),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE9, 10 and 11: Reserved for MHI */
-
-	/* CE12: Target CV prefetch */
-	{
-		.pipenum = __cpu_to_le32(12),
-		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE13: Target CV prefetch */
-	{
-		.pipenum = __cpu_to_le32(13),
-		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE14: WMI logging/CFR/Spectral/Radar */
-	{
-		.pipenum = __cpu_to_le32(14),
-		.pipedir = __cpu_to_le32(PIPEDIR_IN),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE15: Reserved */
-};
-
-/* Target firmware's Copy Engine configuration. */
-static const struct ce_pipe_config ath12k_target_ce_config_wlan_wcn7850[] = {
-	/* CE0: host->target HTC control and raw streams */
-	{
-		.pipenum = __cpu_to_le32(0),
-		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE1: target->host HTT + HTC control */
-	{
-		.pipenum = __cpu_to_le32(1),
-		.pipedir = __cpu_to_le32(PIPEDIR_IN),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE2: target->host WMI */
-	{
-		.pipenum = __cpu_to_le32(2),
-		.pipedir = __cpu_to_le32(PIPEDIR_IN),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE3: host->target WMI */
-	{
-		.pipenum = __cpu_to_le32(3),
-		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE4: host->target HTT */
-	{
-		.pipenum = __cpu_to_le32(4),
-		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
-		.nentries = __cpu_to_le32(256),
-		.nbytes_max = __cpu_to_le32(256),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS | CE_ATTR_DIS_INTR),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE5: target->host Pktlog */
-	{
-		.pipenum = __cpu_to_le32(5),
-		.pipedir = __cpu_to_le32(PIPEDIR_IN),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE6: Reserved for target autonomous hif_memcpy */
-	{
-		.pipenum = __cpu_to_le32(6),
-		.pipedir = __cpu_to_le32(PIPEDIR_INOUT),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(16384),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE7 used only by Host */
-	{
-		.pipenum = __cpu_to_le32(7),
-		.pipedir = __cpu_to_le32(PIPEDIR_INOUT_H2H),
-		.nentries = __cpu_to_le32(0),
-		.nbytes_max = __cpu_to_le32(0),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS | CE_ATTR_DIS_INTR),
-		.reserved = __cpu_to_le32(0),
-	},
-
-	/* CE8 target->host used only by IPA */
-	{
-		.pipenum = __cpu_to_le32(8),
-		.pipedir = __cpu_to_le32(PIPEDIR_INOUT),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(16384),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-	/* CE 9, 10, 11 are used by MHI driver */
-};
-
-/* Map from service/endpoint to Copy Engine.
- * This table is derived from the CE_PCI TABLE, above.
- * It is passed to the Target at startup for use by firmware.
- */
-static const struct service_to_pipe ath12k_target_service_to_ce_map_wlan_qcn9274[] = {
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO),
-		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
-		__cpu_to_le32(3),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK),
-		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
-		__cpu_to_le32(3),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE),
-		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
-		__cpu_to_le32(3),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI),
-		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
-		__cpu_to_le32(3),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL),
-		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
-		__cpu_to_le32(3),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL),
-		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
-		__cpu_to_le32(0),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(1),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_TEST_RAW_STREAMS),
-		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
-		__cpu_to_le32(0),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_TEST_RAW_STREAMS),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(1),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG),
-		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
-		__cpu_to_le32(4),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(1),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL_MAC1),
-		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
-		__cpu_to_le32(7),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL_MAC1),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_PKT_LOG),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(5),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL_DIAG),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(14),
-	},
-
-	/* (Additions here) */
-
-	{ /* must be last */
-		__cpu_to_le32(0),
-		__cpu_to_le32(0),
-		__cpu_to_le32(0),
-	},
-};
-
-static const struct service_to_pipe ath12k_target_service_to_ce_map_wlan_wcn7850[] = {
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO),
-		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
-		__cpu_to_le32(3),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK),
-		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
-		__cpu_to_le32(3),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE),
-		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
-		__cpu_to_le32(3),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI),
-		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
-		__cpu_to_le32(3),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL),
-		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
-		__cpu_to_le32(3),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL),
-		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
-		__cpu_to_le32(0),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG),
-		__cpu_to_le32(PIPEDIR_OUT),	/* out = UL = host -> target */
-		__cpu_to_le32(4),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG),
-		__cpu_to_le32(PIPEDIR_IN),	/* in = DL = target -> host */
-		__cpu_to_le32(1),
-	},
-
-	/* (Additions here) */
-
-	{ /* must be last */
-		__cpu_to_le32(0),
-		__cpu_to_le32(0),
-		__cpu_to_le32(0),
-	},
-};
-
-static const struct ce_pipe_config ath12k_target_ce_config_wlan_ipq5332[] = {
-	/* host->target HTC control and raw streams */
-	{
-		.pipenum = __cpu_to_le32(0),
-		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-	/* target->host HTT */
-	{
-		.pipenum = __cpu_to_le32(1),
-		.pipedir = __cpu_to_le32(PIPEDIR_IN),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-	/* target->host WMI  + HTC control */
-	{
-		.pipenum = __cpu_to_le32(2),
-		.pipedir = __cpu_to_le32(PIPEDIR_IN),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-	/* host->target WMI */
-	{
-		.pipenum = __cpu_to_le32(3),
-		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-	/* host->target HTT */
-	{
-		.pipenum = __cpu_to_le32(4),
-		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
-		.nentries = __cpu_to_le32(256),
-		.nbytes_max = __cpu_to_le32(256),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS | CE_ATTR_DIS_INTR),
-		.reserved = __cpu_to_le32(0),
-	},
-	/* Target -> host PKTLOG */
-	{
-		.pipenum = __cpu_to_le32(5),
-		.pipedir = __cpu_to_le32(PIPEDIR_IN),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-	/* Reserved for target autonomous HIF_memcpy */
-	{
-		.pipenum = __cpu_to_le32(6),
-		.pipedir = __cpu_to_le32(PIPEDIR_INOUT),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(16384),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-	/* CE7 Reserved for CV Prefetch */
-	{
-		.pipenum = __cpu_to_le32(7),
-		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-	/* CE8 Reserved for target generic HIF memcpy */
-	{
-		.pipenum = __cpu_to_le32(8),
-		.pipedir = __cpu_to_le32(PIPEDIR_INOUT),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(16384),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-	/* CE9 WMI logging/CFR/Spectral/Radar/ */
-	{
-		.pipenum = __cpu_to_le32(9),
-		.pipedir = __cpu_to_le32(PIPEDIR_IN),
-		.nentries = __cpu_to_le32(32),
-		.nbytes_max = __cpu_to_le32(2048),
-		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
-		.reserved = __cpu_to_le32(0),
-	},
-	/* Unused TBD */
-	{
-		.pipenum = __cpu_to_le32(10),
-		.pipedir = __cpu_to_le32(PIPEDIR_NONE),
-		.nentries = __cpu_to_le32(0),
-		.nbytes_max = __cpu_to_le32(0),
-		.flags = __cpu_to_le32(0),
-		.reserved = __cpu_to_le32(0),
-	},
-	/* Unused TBD */
-	{
-		.pipenum = __cpu_to_le32(11),
-		.pipedir = __cpu_to_le32(PIPEDIR_NONE),
-		.nentries = __cpu_to_le32(0),
-		.nbytes_max = __cpu_to_le32(0),
-		.flags = __cpu_to_le32(0),
-		.reserved = __cpu_to_le32(0),
-	},
-};
-
-static const struct service_to_pipe ath12k_target_service_to_ce_map_wlan_ipq5332[] = {
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO),
-		__cpu_to_le32(PIPEDIR_OUT),
-		__cpu_to_le32(3),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO),
-		__cpu_to_le32(PIPEDIR_IN),
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK),
-		__cpu_to_le32(PIPEDIR_OUT),
-		__cpu_to_le32(3),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK),
-		__cpu_to_le32(PIPEDIR_IN),
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE),
-		__cpu_to_le32(PIPEDIR_OUT),
-		__cpu_to_le32(3),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE),
-		__cpu_to_le32(PIPEDIR_IN),
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI),
-		__cpu_to_le32(PIPEDIR_OUT),
-		__cpu_to_le32(3),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI),
-		__cpu_to_le32(PIPEDIR_IN),
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL),
-		__cpu_to_le32(PIPEDIR_OUT),
-		__cpu_to_le32(3),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL),
-		__cpu_to_le32(PIPEDIR_IN),
-		__cpu_to_le32(2),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL),
-		__cpu_to_le32(PIPEDIR_OUT),
-		__cpu_to_le32(0),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL),
-		__cpu_to_le32(PIPEDIR_IN),
-		__cpu_to_le32(1),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_TEST_RAW_STREAMS),
-		__cpu_to_le32(PIPEDIR_OUT),
-		__cpu_to_le32(0),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_TEST_RAW_STREAMS),
-		__cpu_to_le32(PIPEDIR_IN),
-		__cpu_to_le32(1),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG),
-		__cpu_to_le32(PIPEDIR_OUT),
-		__cpu_to_le32(4),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG),
-		__cpu_to_le32(PIPEDIR_IN),
-		__cpu_to_le32(1),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_PKT_LOG),
-		__cpu_to_le32(PIPEDIR_IN),
-		__cpu_to_le32(5),
-	},
-	{
-		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL_DIAG),
-		__cpu_to_le32(PIPEDIR_IN),
-		__cpu_to_le32(9),
-	},
-	/* (Additions here) */
-
-	{ /* must be last */
-		__cpu_to_le32(0),
-		__cpu_to_le32(0),
-		__cpu_to_le32(0),
-	},
-};
-
-static const struct ath12k_hw_ring_mask ath12k_hw_ring_mask_qcn9274 = {
-	.tx  = {
-		ATH12K_TX_RING_MASK_0,
-		ATH12K_TX_RING_MASK_1,
-		ATH12K_TX_RING_MASK_2,
-		ATH12K_TX_RING_MASK_3,
-	},
-	.rx_mon_dest = {
-		0, 0, 0, 0,
-		0, 0, 0, 0,
-		ATH12K_RX_MON_RING_MASK_0,
-		ATH12K_RX_MON_RING_MASK_1,
-		ATH12K_RX_MON_RING_MASK_2,
-	},
-	.rx = {
-		0, 0, 0, 0,
-		ATH12K_RX_RING_MASK_0,
-		ATH12K_RX_RING_MASK_1,
-		ATH12K_RX_RING_MASK_2,
-		ATH12K_RX_RING_MASK_3,
-	},
-	.rx_err = {
-		0, 0, 0,
-		ATH12K_RX_ERR_RING_MASK_0,
-	},
-	.rx_wbm_rel = {
-		0, 0, 0,
-		ATH12K_RX_WBM_REL_RING_MASK_0,
-	},
-	.reo_status = {
-		0, 0, 0,
-		ATH12K_REO_STATUS_RING_MASK_0,
-	},
-	.host2rxdma = {
-		0, 0, 0,
-		ATH12K_HOST2RXDMA_RING_MASK_0,
-	},
-	.tx_mon_dest = {
-		0, 0, 0,
-	},
-};
-
-static const struct ath12k_hw_ring_mask ath12k_hw_ring_mask_ipq5332 = {
-	.tx  = {
-		ATH12K_TX_RING_MASK_0,
-		ATH12K_TX_RING_MASK_1,
-		ATH12K_TX_RING_MASK_2,
-		ATH12K_TX_RING_MASK_3,
-	},
-	.rx_mon_dest = {
-		0, 0, 0, 0, 0, 0, 0, 0,
-		ATH12K_RX_MON_RING_MASK_0,
-	},
-	.rx = {
-		0, 0, 0, 0,
-		ATH12K_RX_RING_MASK_0,
-		ATH12K_RX_RING_MASK_1,
-		ATH12K_RX_RING_MASK_2,
-		ATH12K_RX_RING_MASK_3,
-	},
-	.rx_err = {
-		0, 0, 0,
-		ATH12K_RX_ERR_RING_MASK_0,
-	},
-	.rx_wbm_rel = {
-		0, 0, 0,
-		ATH12K_RX_WBM_REL_RING_MASK_0,
-	},
-	.reo_status = {
-		0, 0, 0,
-		ATH12K_REO_STATUS_RING_MASK_0,
-	},
-	.host2rxdma = {
-		0, 0, 0,
-		ATH12K_HOST2RXDMA_RING_MASK_0,
-	},
-	.tx_mon_dest = {
-		ATH12K_TX_MON_RING_MASK_0,
-		ATH12K_TX_MON_RING_MASK_1,
-	},
-};
-
-static const struct ath12k_hw_ring_mask ath12k_hw_ring_mask_wcn7850 = {
-	.tx  = {
-		ATH12K_TX_RING_MASK_0,
-		ATH12K_TX_RING_MASK_1,
-		ATH12K_TX_RING_MASK_2,
-	},
-	.rx_mon_dest = {
-	},
-	.rx_mon_status = {
-		0, 0, 0, 0,
-		ATH12K_RX_MON_STATUS_RING_MASK_0,
-		ATH12K_RX_MON_STATUS_RING_MASK_1,
-		ATH12K_RX_MON_STATUS_RING_MASK_2,
-	},
-	.rx = {
-		0, 0, 0,
-		ATH12K_RX_RING_MASK_0,
-		ATH12K_RX_RING_MASK_1,
-		ATH12K_RX_RING_MASK_2,
-		ATH12K_RX_RING_MASK_3,
-	},
-	.rx_err = {
-		ATH12K_RX_ERR_RING_MASK_0,
-	},
-	.rx_wbm_rel = {
-		ATH12K_RX_WBM_REL_RING_MASK_0,
-	},
-	.reo_status = {
-		ATH12K_REO_STATUS_RING_MASK_0,
-	},
-	.host2rxdma = {
-	},
-	.tx_mon_dest = {
-	},
-};
-
-static const struct ath12k_hw_regs qcn9274_v1_regs = {
-	/* SW2TCL(x) R0 ring configuration address */
-	.hal_tcl1_ring_id = 0x00000908,
-	.hal_tcl1_ring_misc = 0x00000910,
-	.hal_tcl1_ring_tp_addr_lsb = 0x0000091c,
-	.hal_tcl1_ring_tp_addr_msb = 0x00000920,
-	.hal_tcl1_ring_consumer_int_setup_ix0 = 0x00000930,
-	.hal_tcl1_ring_consumer_int_setup_ix1 = 0x00000934,
-	.hal_tcl1_ring_msi1_base_lsb = 0x00000948,
-	.hal_tcl1_ring_msi1_base_msb = 0x0000094c,
-	.hal_tcl1_ring_msi1_data = 0x00000950,
-	.hal_tcl_ring_base_lsb = 0x00000b58,
-	.hal_tcl1_ring_base_lsb = 0x00000900,
-	.hal_tcl1_ring_base_msb = 0x00000904,
-	.hal_tcl2_ring_base_lsb = 0x00000978,
-
-	/* TCL STATUS ring address */
-	.hal_tcl_status_ring_base_lsb = 0x00000d38,
-
-	.hal_wbm_idle_ring_base_lsb = 0x00000d0c,
-	.hal_wbm_idle_ring_misc_addr = 0x00000d1c,
-	.hal_wbm_r0_idle_list_cntl_addr = 0x00000210,
-	.hal_wbm_r0_idle_list_size_addr = 0x00000214,
-	.hal_wbm_scattered_ring_base_lsb = 0x00000220,
-	.hal_wbm_scattered_ring_base_msb = 0x00000224,
-	.hal_wbm_scattered_desc_head_info_ix0 = 0x00000230,
-	.hal_wbm_scattered_desc_head_info_ix1 = 0x00000234,
-	.hal_wbm_scattered_desc_tail_info_ix0 = 0x00000240,
-	.hal_wbm_scattered_desc_tail_info_ix1 = 0x00000244,
-	.hal_wbm_scattered_desc_ptr_hp_addr = 0x0000024c,
-
-	.hal_wbm_sw_release_ring_base_lsb = 0x0000034c,
-	.hal_wbm_sw1_release_ring_base_lsb = 0x000003c4,
-	.hal_wbm0_release_ring_base_lsb = 0x00000dd8,
-	.hal_wbm1_release_ring_base_lsb = 0x00000e50,
-
-	/* PCIe base address */
-	.pcie_qserdes_sysclk_en_sel = 0x01e0c0a8,
-	.pcie_pcs_osc_dtct_config_base = 0x01e0d45c,
-
-	/* PPE release ring address */
-	.hal_ppe_rel_ring_base = 0x0000043c,
-
-	/* REO DEST ring address */
-	.hal_reo2_ring_base = 0x0000055c,
-	.hal_reo1_misc_ctrl_addr = 0x00000b7c,
-	.hal_reo1_sw_cookie_cfg0 = 0x00000050,
-	.hal_reo1_sw_cookie_cfg1 = 0x00000054,
-	.hal_reo1_qdesc_lut_base0 = 0x00000058,
-	.hal_reo1_qdesc_lut_base1 = 0x0000005c,
-	.hal_reo1_ring_base_lsb = 0x000004e4,
-	.hal_reo1_ring_base_msb = 0x000004e8,
-	.hal_reo1_ring_id = 0x000004ec,
-	.hal_reo1_ring_misc = 0x000004f4,
-	.hal_reo1_ring_hp_addr_lsb = 0x000004f8,
-	.hal_reo1_ring_hp_addr_msb = 0x000004fc,
-	.hal_reo1_ring_producer_int_setup = 0x00000508,
-	.hal_reo1_ring_msi1_base_lsb = 0x0000052C,
-	.hal_reo1_ring_msi1_base_msb = 0x00000530,
-	.hal_reo1_ring_msi1_data = 0x00000534,
-	.hal_reo1_aging_thres_ix0 = 0x00000b08,
-	.hal_reo1_aging_thres_ix1 = 0x00000b0c,
-	.hal_reo1_aging_thres_ix2 = 0x00000b10,
-	.hal_reo1_aging_thres_ix3 = 0x00000b14,
-
-	/* REO Exception ring address */
-	.hal_reo2_sw0_ring_base = 0x000008a4,
-
-	/* REO Reinject ring address */
-	.hal_sw2reo_ring_base = 0x00000304,
-	.hal_sw2reo1_ring_base = 0x0000037c,
-
-	/* REO cmd ring address */
-	.hal_reo_cmd_ring_base = 0x0000028c,
-
-	/* REO status ring address */
-	.hal_reo_status_ring_base = 0x00000a84,
-
-	/* CE base address */
-	.hal_umac_ce0_src_reg_base = 0x01b80000,
-	.hal_umac_ce0_dest_reg_base = 0x01b81000,
-	.hal_umac_ce1_src_reg_base = 0x01b82000,
-	.hal_umac_ce1_dest_reg_base = 0x01b83000,
-
-	.gcc_gcc_pcie_hot_rst = 0x1e38338,
-};
-
-static const struct ath12k_hw_regs qcn9274_v2_regs = {
-	/* SW2TCL(x) R0 ring configuration address */
-	.hal_tcl1_ring_id = 0x00000908,
-	.hal_tcl1_ring_misc = 0x00000910,
-	.hal_tcl1_ring_tp_addr_lsb = 0x0000091c,
-	.hal_tcl1_ring_tp_addr_msb = 0x00000920,
-	.hal_tcl1_ring_consumer_int_setup_ix0 = 0x00000930,
-	.hal_tcl1_ring_consumer_int_setup_ix1 = 0x00000934,
-	.hal_tcl1_ring_msi1_base_lsb = 0x00000948,
-	.hal_tcl1_ring_msi1_base_msb = 0x0000094c,
-	.hal_tcl1_ring_msi1_data = 0x00000950,
-	.hal_tcl_ring_base_lsb = 0x00000b58,
-	.hal_tcl1_ring_base_lsb = 0x00000900,
-	.hal_tcl1_ring_base_msb = 0x00000904,
-	.hal_tcl2_ring_base_lsb = 0x00000978,
-
-	/* TCL STATUS ring address */
-	.hal_tcl_status_ring_base_lsb = 0x00000d38,
-
-	/* WBM idle link ring address */
-	.hal_wbm_idle_ring_base_lsb = 0x00000d3c,
-	.hal_wbm_idle_ring_misc_addr = 0x00000d4c,
-	.hal_wbm_r0_idle_list_cntl_addr = 0x00000240,
-	.hal_wbm_r0_idle_list_size_addr = 0x00000244,
-	.hal_wbm_scattered_ring_base_lsb = 0x00000250,
-	.hal_wbm_scattered_ring_base_msb = 0x00000254,
-	.hal_wbm_scattered_desc_head_info_ix0 = 0x00000260,
-	.hal_wbm_scattered_desc_head_info_ix1 = 0x00000264,
-	.hal_wbm_scattered_desc_tail_info_ix0 = 0x00000270,
-	.hal_wbm_scattered_desc_tail_info_ix1 = 0x00000274,
-	.hal_wbm_scattered_desc_ptr_hp_addr = 0x0000027c,
-
-	/* SW2WBM release ring address */
-	.hal_wbm_sw_release_ring_base_lsb = 0x0000037c,
-	.hal_wbm_sw1_release_ring_base_lsb = 0x000003f4,
-
-	/* WBM2SW release ring address */
-	.hal_wbm0_release_ring_base_lsb = 0x00000e08,
-	.hal_wbm1_release_ring_base_lsb = 0x00000e80,
-
-	/* PCIe base address */
-	.pcie_qserdes_sysclk_en_sel = 0x01e0c0a8,
-	.pcie_pcs_osc_dtct_config_base = 0x01e0d45c,
-
-	/* PPE release ring address */
-	.hal_ppe_rel_ring_base = 0x0000046c,
-
-	/* REO DEST ring address */
-	.hal_reo2_ring_base = 0x00000578,
-	.hal_reo1_misc_ctrl_addr = 0x00000b9c,
-	.hal_reo1_sw_cookie_cfg0 = 0x0000006c,
-	.hal_reo1_sw_cookie_cfg1 = 0x00000070,
-	.hal_reo1_qdesc_lut_base0 = 0x00000074,
-	.hal_reo1_qdesc_lut_base1 = 0x00000078,
-	.hal_reo1_qdesc_addr = 0x0000007c,
-	.hal_reo1_qdesc_max_peerid = 0x00000088,
-	.hal_reo1_ring_base_lsb = 0x00000500,
-	.hal_reo1_ring_base_msb = 0x00000504,
-	.hal_reo1_ring_id = 0x00000508,
-	.hal_reo1_ring_misc = 0x00000510,
-	.hal_reo1_ring_hp_addr_lsb = 0x00000514,
-	.hal_reo1_ring_hp_addr_msb = 0x00000518,
-	.hal_reo1_ring_producer_int_setup = 0x00000524,
-	.hal_reo1_ring_msi1_base_lsb = 0x00000548,
-	.hal_reo1_ring_msi1_base_msb = 0x0000054C,
-	.hal_reo1_ring_msi1_data = 0x00000550,
-	.hal_reo1_aging_thres_ix0 = 0x00000B28,
-	.hal_reo1_aging_thres_ix1 = 0x00000B2C,
-	.hal_reo1_aging_thres_ix2 = 0x00000B30,
-	.hal_reo1_aging_thres_ix3 = 0x00000B34,
-
-	/* REO Exception ring address */
-	.hal_reo2_sw0_ring_base = 0x000008c0,
-
-	/* REO Reinject ring address */
-	.hal_sw2reo_ring_base = 0x00000320,
-	.hal_sw2reo1_ring_base = 0x00000398,
-
-	/* REO cmd ring address */
-	.hal_reo_cmd_ring_base = 0x000002A8,
-
-	/* REO status ring address */
-	.hal_reo_status_ring_base = 0x00000aa0,
-
-	/* CE base address */
-	.hal_umac_ce0_src_reg_base = 0x01b80000,
-	.hal_umac_ce0_dest_reg_base = 0x01b81000,
-	.hal_umac_ce1_src_reg_base = 0x01b82000,
-	.hal_umac_ce1_dest_reg_base = 0x01b83000,
-
-	.gcc_gcc_pcie_hot_rst = 0x1e38338,
-};
-
-static const struct ath12k_hw_regs ipq5332_regs = {
-	/* SW2TCL(x) R0 ring configuration address */
-	.hal_tcl1_ring_id = 0x00000918,
-	.hal_tcl1_ring_misc = 0x00000920,
-	.hal_tcl1_ring_tp_addr_lsb = 0x0000092c,
-	.hal_tcl1_ring_tp_addr_msb = 0x00000930,
-	.hal_tcl1_ring_consumer_int_setup_ix0 = 0x00000940,
-	.hal_tcl1_ring_consumer_int_setup_ix1 = 0x00000944,
-	.hal_tcl1_ring_msi1_base_lsb = 0x00000958,
-	.hal_tcl1_ring_msi1_base_msb = 0x0000095c,
-	.hal_tcl1_ring_base_lsb = 0x00000910,
-	.hal_tcl1_ring_base_msb = 0x00000914,
-	.hal_tcl1_ring_msi1_data = 0x00000960,
-	.hal_tcl2_ring_base_lsb = 0x00000988,
-	.hal_tcl_ring_base_lsb = 0x00000b68,
-
-	/* TCL STATUS ring address */
-	.hal_tcl_status_ring_base_lsb = 0x00000d48,
-
-	/* REO DEST ring address */
-	.hal_reo2_ring_base = 0x00000578,
-	.hal_reo1_misc_ctrl_addr = 0x00000b9c,
-	.hal_reo1_sw_cookie_cfg0 = 0x0000006c,
-	.hal_reo1_sw_cookie_cfg1 = 0x00000070,
-	.hal_reo1_qdesc_lut_base0 = 0x00000074,
-	.hal_reo1_qdesc_lut_base1 = 0x00000078,
-	.hal_reo1_ring_base_lsb = 0x00000500,
-	.hal_reo1_ring_base_msb = 0x00000504,
-	.hal_reo1_ring_id = 0x00000508,
-	.hal_reo1_ring_misc = 0x00000510,
-	.hal_reo1_ring_hp_addr_lsb = 0x00000514,
-	.hal_reo1_ring_hp_addr_msb = 0x00000518,
-	.hal_reo1_ring_producer_int_setup = 0x00000524,
-	.hal_reo1_ring_msi1_base_lsb = 0x00000548,
-	.hal_reo1_ring_msi1_base_msb = 0x0000054C,
-	.hal_reo1_ring_msi1_data = 0x00000550,
-	.hal_reo1_aging_thres_ix0 = 0x00000B28,
-	.hal_reo1_aging_thres_ix1 = 0x00000B2C,
-	.hal_reo1_aging_thres_ix2 = 0x00000B30,
-	.hal_reo1_aging_thres_ix3 = 0x00000B34,
-
-	/* REO Exception ring address */
-	.hal_reo2_sw0_ring_base = 0x000008c0,
-
-	/* REO Reinject ring address */
-	.hal_sw2reo_ring_base = 0x00000320,
-	.hal_sw2reo1_ring_base = 0x00000398,
-
-	/* REO cmd ring address */
-	.hal_reo_cmd_ring_base = 0x000002A8,
-
-	/* REO status ring address */
-	.hal_reo_status_ring_base = 0x00000aa0,
-
-	/* WBM idle link ring address */
-	.hal_wbm_idle_ring_base_lsb = 0x00000d3c,
-	.hal_wbm_idle_ring_misc_addr = 0x00000d4c,
-	.hal_wbm_r0_idle_list_cntl_addr = 0x00000240,
-	.hal_wbm_r0_idle_list_size_addr = 0x00000244,
-	.hal_wbm_scattered_ring_base_lsb = 0x00000250,
-	.hal_wbm_scattered_ring_base_msb = 0x00000254,
-	.hal_wbm_scattered_desc_head_info_ix0 = 0x00000260,
-	.hal_wbm_scattered_desc_head_info_ix1   = 0x00000264,
-	.hal_wbm_scattered_desc_tail_info_ix0 = 0x00000270,
-	.hal_wbm_scattered_desc_tail_info_ix1 = 0x00000274,
-	.hal_wbm_scattered_desc_ptr_hp_addr = 0x0000027c,
-
-	/* SW2WBM release ring address */
-	.hal_wbm_sw_release_ring_base_lsb = 0x0000037c,
-
-	/* WBM2SW release ring address */
-	.hal_wbm0_release_ring_base_lsb = 0x00000e08,
-	.hal_wbm1_release_ring_base_lsb = 0x00000e80,
-
-	/* PPE release ring address */
-	.hal_ppe_rel_ring_base = 0x0000046c,
-
-	/* CE address */
-	.hal_umac_ce0_src_reg_base = 0x00740000 -
-		HAL_IPQ5332_CE_WFSS_REG_BASE,
-	.hal_umac_ce0_dest_reg_base = 0x00741000 -
-		HAL_IPQ5332_CE_WFSS_REG_BASE,
-	.hal_umac_ce1_src_reg_base = 0x00742000 -
-		HAL_IPQ5332_CE_WFSS_REG_BASE,
-	.hal_umac_ce1_dest_reg_base = 0x00743000 -
-		HAL_IPQ5332_CE_WFSS_REG_BASE,
-};
-
-static const struct ath12k_hw_regs wcn7850_regs = {
-	/* SW2TCL(x) R0 ring configuration address */
-	.hal_tcl1_ring_id = 0x00000908,
-	.hal_tcl1_ring_misc = 0x00000910,
-	.hal_tcl1_ring_tp_addr_lsb = 0x0000091c,
-	.hal_tcl1_ring_tp_addr_msb = 0x00000920,
-	.hal_tcl1_ring_consumer_int_setup_ix0 = 0x00000930,
-	.hal_tcl1_ring_consumer_int_setup_ix1 = 0x00000934,
-	.hal_tcl1_ring_msi1_base_lsb = 0x00000948,
-	.hal_tcl1_ring_msi1_base_msb = 0x0000094c,
-	.hal_tcl1_ring_msi1_data = 0x00000950,
-	.hal_tcl_ring_base_lsb = 0x00000b58,
-	.hal_tcl1_ring_base_lsb = 0x00000900,
-	.hal_tcl1_ring_base_msb = 0x00000904,
-	.hal_tcl2_ring_base_lsb = 0x00000978,
-
-	/* TCL STATUS ring address */
-	.hal_tcl_status_ring_base_lsb = 0x00000d38,
-
-	.hal_wbm_idle_ring_base_lsb = 0x00000d3c,
-	.hal_wbm_idle_ring_misc_addr = 0x00000d4c,
-	.hal_wbm_r0_idle_list_cntl_addr = 0x00000240,
-	.hal_wbm_r0_idle_list_size_addr = 0x00000244,
-	.hal_wbm_scattered_ring_base_lsb = 0x00000250,
-	.hal_wbm_scattered_ring_base_msb = 0x00000254,
-	.hal_wbm_scattered_desc_head_info_ix0 = 0x00000260,
-	.hal_wbm_scattered_desc_head_info_ix1 = 0x00000264,
-	.hal_wbm_scattered_desc_tail_info_ix0 = 0x00000270,
-	.hal_wbm_scattered_desc_tail_info_ix1 = 0x00000274,
-	.hal_wbm_scattered_desc_ptr_hp_addr = 0x00000027c,
-
-	.hal_wbm_sw_release_ring_base_lsb = 0x0000037c,
-	.hal_wbm_sw1_release_ring_base_lsb = 0x00000284,
-	.hal_wbm0_release_ring_base_lsb = 0x00000e08,
-	.hal_wbm1_release_ring_base_lsb = 0x00000e80,
-
-	/* PCIe base address */
-	.pcie_qserdes_sysclk_en_sel = 0x01e0e0a8,
-	.pcie_pcs_osc_dtct_config_base = 0x01e0f45c,
-
-	/* PPE release ring address */
-	.hal_ppe_rel_ring_base = 0x0000043c,
-
-	/* REO DEST ring address */
-	.hal_reo2_ring_base = 0x0000055c,
-	.hal_reo1_misc_ctrl_addr = 0x00000b7c,
-	.hal_reo1_sw_cookie_cfg0 = 0x00000050,
-	.hal_reo1_sw_cookie_cfg1 = 0x00000054,
-	.hal_reo1_qdesc_lut_base0 = 0x00000058,
-	.hal_reo1_qdesc_lut_base1 = 0x0000005c,
-	.hal_reo1_ring_base_lsb = 0x000004e4,
-	.hal_reo1_ring_base_msb = 0x000004e8,
-	.hal_reo1_ring_id = 0x000004ec,
-	.hal_reo1_ring_misc = 0x000004f4,
-	.hal_reo1_ring_hp_addr_lsb = 0x000004f8,
-	.hal_reo1_ring_hp_addr_msb = 0x000004fc,
-	.hal_reo1_ring_producer_int_setup = 0x00000508,
-	.hal_reo1_ring_msi1_base_lsb = 0x0000052C,
-	.hal_reo1_ring_msi1_base_msb = 0x00000530,
-	.hal_reo1_ring_msi1_data = 0x00000534,
-	.hal_reo1_aging_thres_ix0 = 0x00000b08,
-	.hal_reo1_aging_thres_ix1 = 0x00000b0c,
-	.hal_reo1_aging_thres_ix2 = 0x00000b10,
-	.hal_reo1_aging_thres_ix3 = 0x00000b14,
-
-	/* REO Exception ring address */
-	.hal_reo2_sw0_ring_base = 0x000008a4,
-
-	/* REO Reinject ring address */
-	.hal_sw2reo_ring_base = 0x00000304,
-	.hal_sw2reo1_ring_base = 0x0000037c,
-
-	/* REO cmd ring address */
-	.hal_reo_cmd_ring_base = 0x0000028c,
-
-	/* REO status ring address */
-	.hal_reo_status_ring_base = 0x00000a84,
-
-	/* CE base address */
-	.hal_umac_ce0_src_reg_base = 0x01b80000,
-	.hal_umac_ce0_dest_reg_base = 0x01b81000,
-	.hal_umac_ce1_src_reg_base = 0x01b82000,
-	.hal_umac_ce1_dest_reg_base = 0x01b83000,
-
-	.gcc_gcc_pcie_hot_rst = 0x1e40304,
-};
-
-static const struct ath12k_hw_hal_params ath12k_hw_hal_params_qcn9274 = {
-	.rx_buf_rbm = HAL_RX_BUF_RBM_SW3_BM,
-	.wbm2sw_cc_enable = HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN |
-			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW1_EN |
-			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW2_EN |
-			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW3_EN |
-			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN,
-};
-
-static const struct ath12k_hw_hal_params ath12k_hw_hal_params_wcn7850 = {
-	.rx_buf_rbm = HAL_RX_BUF_RBM_SW1_BM,
-	.wbm2sw_cc_enable = HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN |
-			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW2_EN |
-			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW3_EN |
-			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN,
-};
-
-static const struct ath12k_hw_hal_params ath12k_hw_hal_params_ipq5332 = {
-	.rx_buf_rbm = HAL_RX_BUF_RBM_SW3_BM,
-	.wbm2sw_cc_enable = HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN |
-			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW1_EN |
-			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW2_EN |
-			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW3_EN |
-			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN,
-};
-
-static const struct ce_ie_addr ath12k_ce_ie_addr_ipq5332 = {
-	.ie1_reg_addr = CE_HOST_IE_ADDRESS - HAL_IPQ5332_CE_WFSS_REG_BASE,
-	.ie2_reg_addr = CE_HOST_IE_2_ADDRESS - HAL_IPQ5332_CE_WFSS_REG_BASE,
-	.ie3_reg_addr = CE_HOST_IE_3_ADDRESS - HAL_IPQ5332_CE_WFSS_REG_BASE,
-};
-
-static const struct ce_remap ath12k_ce_remap_ipq5332 = {
-	.base = HAL_IPQ5332_CE_WFSS_REG_BASE,
-	.size = HAL_IPQ5332_CE_SIZE,
-};
-
-static const struct ath12k_hw_params ath12k_hw_params[] = {
-	{
-		.name = "qcn9274 hw1.0",
-		.hw_rev = ATH12K_HW_QCN9274_HW10,
-		.fw = {
-			.dir = "QCN9274/hw1.0",
-			.board_size = 256 * 1024,
-			.cal_offset = 128 * 1024,
-			.m3_loader = ath12k_m3_fw_loader_driver,
-		},
-		.max_radios = 1,
-		.single_pdev_only = false,
-		.qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_QCN9274,
-		.internal_sleep_clock = false,
-
-		.hw_ops = &qcn9274_ops,
-		.ring_mask = &ath12k_hw_ring_mask_qcn9274,
-		.regs = &qcn9274_v1_regs,
-
-		.host_ce_config = ath12k_host_ce_config_qcn9274,
-		.ce_count = 16,
-		.target_ce_config = ath12k_target_ce_config_wlan_qcn9274,
-		.target_ce_count = 12,
-		.svc_to_ce_map = ath12k_target_service_to_ce_map_wlan_qcn9274,
-		.svc_to_ce_map_len = 18,
-
-		.hal_params = &ath12k_hw_hal_params_qcn9274,
-
-		.rxdma1_enable = false,
-		.num_rxdma_per_pdev = 1,
-		.num_rxdma_dst_ring = 0,
-		.rx_mac_buf_ring = false,
-		.vdev_start_delay = false,
-
-		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
-					BIT(NL80211_IFTYPE_AP) |
-					BIT(NL80211_IFTYPE_MESH_POINT) |
-					BIT(NL80211_IFTYPE_AP_VLAN),
-		.supports_monitor = false,
-
-		.idle_ps = false,
-		.download_calib = true,
-		.supports_suspend = false,
-		.tcl_ring_retry = true,
-		.reoq_lut_support = true,
-		.supports_shadow_regs = false,
-
-		.num_tcl_banks = 48,
-		.max_tx_ring = 4,
-
-		.mhi_config = &ath12k_mhi_config_qcn9274,
-
-		.wmi_init = ath12k_wmi_init_qcn9274,
-
-		.hal_ops = &hal_qcn9274_ops,
-
-		.qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01),
-
-		.rfkill_pin = 0,
-		.rfkill_cfg = 0,
-		.rfkill_on_level = 0,
-
-		.rddm_size = 0x600000,
-
-		.def_num_link = 0,
-		.max_mlo_peer = 256,
-
-		.otp_board_id_register = QCN9274_QFPROM_RAW_RFA_PDET_ROW13_LSB,
-
-		.supports_sta_ps = false,
-
-		.acpi_guid = NULL,
-		.supports_dynamic_smps_6ghz = true,
-
-		.iova_mask = 0,
-
-		.supports_aspm = false,
-
-		.ce_ie_addr = NULL,
-		.ce_remap = NULL,
-		.bdf_addr_offset = 0,
-
-		.current_cc_support = false,
-
-		.dp_primary_link_only = true,
-	},
-	{
-		.name = "wcn7850 hw2.0",
-		.hw_rev = ATH12K_HW_WCN7850_HW20,
-
-		.fw = {
-			.dir = "WCN7850/hw2.0",
-			.board_size = 256 * 1024,
-			.cal_offset = 256 * 1024,
-			.m3_loader = ath12k_m3_fw_loader_driver,
-		},
-
-		.max_radios = 1,
-		.single_pdev_only = true,
-		.qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_WCN7850,
-		.internal_sleep_clock = true,
-
-		.hw_ops = &wcn7850_ops,
-		.ring_mask = &ath12k_hw_ring_mask_wcn7850,
-		.regs = &wcn7850_regs,
-
-		.host_ce_config = ath12k_host_ce_config_wcn7850,
-		.ce_count = 9,
-		.target_ce_config = ath12k_target_ce_config_wlan_wcn7850,
-		.target_ce_count = 9,
-		.svc_to_ce_map = ath12k_target_service_to_ce_map_wlan_wcn7850,
-		.svc_to_ce_map_len = 14,
-
-		.hal_params = &ath12k_hw_hal_params_wcn7850,
-
-		.rxdma1_enable = false,
-		.num_rxdma_per_pdev = 2,
-		.num_rxdma_dst_ring = 1,
-		.rx_mac_buf_ring = true,
-		.vdev_start_delay = true,
-
-		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
-				   BIT(NL80211_IFTYPE_AP) |
-				   BIT(NL80211_IFTYPE_P2P_DEVICE) |
-				   BIT(NL80211_IFTYPE_P2P_CLIENT) |
-				   BIT(NL80211_IFTYPE_P2P_GO),
-		.supports_monitor = true,
-
-		.idle_ps = true,
-		.download_calib = false,
-		.supports_suspend = true,
-		.tcl_ring_retry = false,
-		.reoq_lut_support = false,
-		.supports_shadow_regs = true,
-
-		.num_tcl_banks = 7,
-		.max_tx_ring = 3,
-
-		.mhi_config = &ath12k_mhi_config_wcn7850,
-
-		.wmi_init = ath12k_wmi_init_wcn7850,
-
-		.hal_ops = &hal_wcn7850_ops,
-
-		.qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01) |
-					   BIT(CNSS_PCIE_PERST_NO_PULL_V01),
-
-		.rfkill_pin = 48,
-		.rfkill_cfg = 0,
-		.rfkill_on_level = 1,
-
-		.rddm_size = 0x780000,
-
-		.def_num_link = 2,
-		.max_mlo_peer = 32,
-
-		.otp_board_id_register = 0,
-
-		.supports_sta_ps = true,
-
-		.acpi_guid = &wcn7850_uuid,
-		.supports_dynamic_smps_6ghz = false,
-
-		.iova_mask = ATH12K_PCIE_MAX_PAYLOAD_SIZE - 1,
-
-		.supports_aspm = true,
-
-		.ce_ie_addr = NULL,
-		.ce_remap = NULL,
-		.bdf_addr_offset = 0,
-
-		.current_cc_support = true,
-
-		.dp_primary_link_only = false,
-	},
-	{
-		.name = "qcn9274 hw2.0",
-		.hw_rev = ATH12K_HW_QCN9274_HW20,
-		.fw = {
-			.dir = "QCN9274/hw2.0",
-			.board_size = 256 * 1024,
-			.cal_offset = 128 * 1024,
-			.m3_loader = ath12k_m3_fw_loader_driver,
-		},
-		.max_radios = 2,
-		.single_pdev_only = false,
-		.qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_QCN9274,
-		.internal_sleep_clock = false,
-
-		.hw_ops = &qcn9274_ops,
-		.ring_mask = &ath12k_hw_ring_mask_qcn9274,
-		.regs = &qcn9274_v2_regs,
-
-		.host_ce_config = ath12k_host_ce_config_qcn9274,
-		.ce_count = 16,
-		.target_ce_config = ath12k_target_ce_config_wlan_qcn9274,
-		.target_ce_count = 12,
-		.svc_to_ce_map = ath12k_target_service_to_ce_map_wlan_qcn9274,
-		.svc_to_ce_map_len = 18,
-
-		.hal_params = &ath12k_hw_hal_params_qcn9274,
-
-		.rxdma1_enable = true,
-		.num_rxdma_per_pdev = 1,
-		.num_rxdma_dst_ring = 0,
-		.rx_mac_buf_ring = false,
-		.vdev_start_delay = false,
-
-		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
-					BIT(NL80211_IFTYPE_AP) |
-					BIT(NL80211_IFTYPE_MESH_POINT) |
-					BIT(NL80211_IFTYPE_AP_VLAN),
-		.supports_monitor = true,
-
-		.idle_ps = false,
-		.download_calib = true,
-		.supports_suspend = false,
-		.tcl_ring_retry = true,
-		.reoq_lut_support = true,
-		.supports_shadow_regs = false,
-
-		.num_tcl_banks = 48,
-		.max_tx_ring = 4,
-
-		.mhi_config = &ath12k_mhi_config_qcn9274,
-
-		.wmi_init = ath12k_wmi_init_qcn9274,
-
-		.hal_ops = &hal_qcn9274_ops,
-
-		.qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01),
-
-		.rfkill_pin = 0,
-		.rfkill_cfg = 0,
-		.rfkill_on_level = 0,
-
-		.rddm_size = 0x600000,
-
-		.def_num_link = 0,
-		.max_mlo_peer = 256,
-
-		.otp_board_id_register = QCN9274_QFPROM_RAW_RFA_PDET_ROW13_LSB,
-
-		.supports_sta_ps = false,
-
-		.acpi_guid = NULL,
-		.supports_dynamic_smps_6ghz = true,
-
-		.iova_mask = 0,
-
-		.supports_aspm = false,
-
-		.ce_ie_addr = NULL,
-		.ce_remap = NULL,
-		.bdf_addr_offset = 0,
-
-		.current_cc_support = false,
-
-		.dp_primary_link_only = true,
-	},
-	{
-		.name = "ipq5332 hw1.0",
-		.hw_rev = ATH12K_HW_IPQ5332_HW10,
-		.fw = {
-			.dir = "IPQ5332/hw1.0",
-			.board_size = 256 * 1024,
-			.cal_offset = 128 * 1024,
-			.m3_loader = ath12k_m3_fw_loader_remoteproc,
-		},
-		.max_radios = 1,
-		.single_pdev_only = false,
-		.qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_IPQ5332,
-		.internal_sleep_clock = false,
-
-		.hw_ops = &qcn9274_ops,
-		.regs = &ipq5332_regs,
-		.ring_mask = &ath12k_hw_ring_mask_ipq5332,
-
-		.host_ce_config = ath12k_host_ce_config_ipq5332,
-		.ce_count = 12,
-		.target_ce_config = ath12k_target_ce_config_wlan_ipq5332,
-		.target_ce_count = 12,
-		.svc_to_ce_map = ath12k_target_service_to_ce_map_wlan_ipq5332,
-		.svc_to_ce_map_len = 18,
-
-		.hal_params = &ath12k_hw_hal_params_ipq5332,
-
-		.rxdma1_enable = false,
-		.num_rxdma_per_pdev = 1,
-		.num_rxdma_dst_ring = 0,
-		.rx_mac_buf_ring = false,
-		.vdev_start_delay = false,
-
-		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
-				   BIT(NL80211_IFTYPE_AP) |
-				   BIT(NL80211_IFTYPE_MESH_POINT),
-		.supports_monitor = false,
-
-		.idle_ps = false,
-		.download_calib = true,
-		.supports_suspend = false,
-		.tcl_ring_retry = true,
-		.reoq_lut_support = false,
-		.supports_shadow_regs = false,
-
-		.num_tcl_banks = 48,
-		.max_tx_ring = 4,
-
-		.wmi_init = &ath12k_wmi_init_qcn9274,
-
-		.hal_ops = &hal_qcn9274_ops,
-
-		.qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01),
-
-		.rfkill_pin = 0,
-		.rfkill_cfg = 0,
-		.rfkill_on_level = 0,
-
-		.rddm_size = 0,
-
-		.def_num_link = 0,
-		.max_mlo_peer = 256,
-
-		.otp_board_id_register = 0,
-
-		.supports_sta_ps = false,
-
-		.acpi_guid = NULL,
-		.supports_dynamic_smps_6ghz = false,
-		.iova_mask = 0,
-		.supports_aspm = false,
-
-		.ce_ie_addr = &ath12k_ce_ie_addr_ipq5332,
-		.ce_remap = &ath12k_ce_remap_ipq5332,
-		.bdf_addr_offset = 0xC00000,
-
-		.dp_primary_link_only = true,
-	},
-};
-
-int ath12k_hw_init(struct ath12k_base *ab)
-{
-	const struct ath12k_hw_params *hw_params = NULL;
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(ath12k_hw_params); i++) {
-		hw_params = &ath12k_hw_params[i];
-
-		if (hw_params->hw_rev == ab->hw_rev)
-			break;
-	}
-
-	if (i == ARRAY_SIZE(ath12k_hw_params)) {
-		ath12k_err(ab, "Unsupported hardware version: 0x%x\n", ab->hw_rev);
-		return -EINVAL;
-	}
-
-	ab->hw_params = hw_params;
-
-	ath12k_info(ab, "Hardware name: %s\n", ab->hw_params->name);
-
-	return 0;
-}
diff --git a/drivers/net/wireless/ath/ath12k/hw.h b/drivers/net/wireless/ath/ath12k/hw.h
index 8ce11c3e6d5c..655753d0413a 100644
--- a/drivers/net/wireless/ath/ath12k/hw.h
+++ b/drivers/net/wireless/ath/ath12k/hw.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause-Clear */
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #ifndef ATH12K_HW_H
@@ -128,11 +128,6 @@ struct ath12k_hw_ring_mask {
 	u8 tx_mon_dest[ATH12K_EXT_IRQ_GRP_NUM_MAX];
 };
 
-struct ath12k_hw_hal_params {
-	enum hal_rx_buf_return_buf_manager rx_buf_rbm;
-	u32	  wbm2sw_cc_enable;
-};
-
 enum ath12k_m3_fw_loaders {
 	ath12k_m3_fw_loader_driver,
 	ath12k_m3_fw_loader_remoteproc,
@@ -156,7 +151,6 @@ struct ath12k_hw_params {
 
 	const struct ath12k_hw_ops *hw_ops;
 	const struct ath12k_hw_ring_mask *ring_mask;
-	const struct ath12k_hw_regs *regs;
 
 	const struct ce_attr *host_ce_config;
 	u32 ce_count;
@@ -165,8 +159,6 @@ struct ath12k_hw_params {
 	const struct service_to_pipe *svc_to_ce_map;
 	u32 svc_to_ce_map_len;
 
-	const struct ath12k_hw_hal_params *hal_params;
-
 	bool rxdma1_enable:1;
 	int num_rxdma_per_pdev;
 	int num_rxdma_dst_ring;
@@ -193,8 +185,6 @@ struct ath12k_hw_params {
 	void (*wmi_init)(struct ath12k_base *ab,
 			 struct ath12k_wmi_resource_config_arg *config);
 
-	const struct hal_ops *hal_ops;
-
 	u64 qmi_cnss_feature_bitmap;
 
 	u32 rfkill_pin;
@@ -285,86 +275,6 @@ enum ath12k_bd_ie_type {
 	ATH12K_BD_IE_REGDB = 1,
 };
 
-struct ath12k_hw_regs {
-	u32 hal_tcl1_ring_id;
-	u32 hal_tcl1_ring_misc;
-	u32 hal_tcl1_ring_tp_addr_lsb;
-	u32 hal_tcl1_ring_tp_addr_msb;
-	u32 hal_tcl1_ring_consumer_int_setup_ix0;
-	u32 hal_tcl1_ring_consumer_int_setup_ix1;
-	u32 hal_tcl1_ring_msi1_base_lsb;
-	u32 hal_tcl1_ring_msi1_base_msb;
-	u32 hal_tcl1_ring_msi1_data;
-	u32 hal_tcl_ring_base_lsb;
-	u32 hal_tcl1_ring_base_lsb;
-	u32 hal_tcl1_ring_base_msb;
-	u32 hal_tcl2_ring_base_lsb;
-
-	u32 hal_tcl_status_ring_base_lsb;
-
-	u32 hal_reo1_qdesc_addr;
-	u32 hal_reo1_qdesc_max_peerid;
-
-	u32 hal_wbm_idle_ring_base_lsb;
-	u32 hal_wbm_idle_ring_misc_addr;
-	u32 hal_wbm_r0_idle_list_cntl_addr;
-	u32 hal_wbm_r0_idle_list_size_addr;
-	u32 hal_wbm_scattered_ring_base_lsb;
-	u32 hal_wbm_scattered_ring_base_msb;
-	u32 hal_wbm_scattered_desc_head_info_ix0;
-	u32 hal_wbm_scattered_desc_head_info_ix1;
-	u32 hal_wbm_scattered_desc_tail_info_ix0;
-	u32 hal_wbm_scattered_desc_tail_info_ix1;
-	u32 hal_wbm_scattered_desc_ptr_hp_addr;
-
-	u32 hal_wbm_sw_release_ring_base_lsb;
-	u32 hal_wbm_sw1_release_ring_base_lsb;
-	u32 hal_wbm0_release_ring_base_lsb;
-	u32 hal_wbm1_release_ring_base_lsb;
-
-	u32 pcie_qserdes_sysclk_en_sel;
-	u32 pcie_pcs_osc_dtct_config_base;
-
-	u32 hal_umac_ce0_src_reg_base;
-	u32 hal_umac_ce0_dest_reg_base;
-	u32 hal_umac_ce1_src_reg_base;
-	u32 hal_umac_ce1_dest_reg_base;
-
-	u32 hal_ppe_rel_ring_base;
-
-	u32 hal_reo2_ring_base;
-	u32 hal_reo1_misc_ctrl_addr;
-	u32 hal_reo1_sw_cookie_cfg0;
-	u32 hal_reo1_sw_cookie_cfg1;
-	u32 hal_reo1_qdesc_lut_base0;
-	u32 hal_reo1_qdesc_lut_base1;
-	u32 hal_reo1_ring_base_lsb;
-	u32 hal_reo1_ring_base_msb;
-	u32 hal_reo1_ring_id;
-	u32 hal_reo1_ring_misc;
-	u32 hal_reo1_ring_hp_addr_lsb;
-	u32 hal_reo1_ring_hp_addr_msb;
-	u32 hal_reo1_ring_producer_int_setup;
-	u32 hal_reo1_ring_msi1_base_lsb;
-	u32 hal_reo1_ring_msi1_base_msb;
-	u32 hal_reo1_ring_msi1_data;
-	u32 hal_reo1_aging_thres_ix0;
-	u32 hal_reo1_aging_thres_ix1;
-	u32 hal_reo1_aging_thres_ix2;
-	u32 hal_reo1_aging_thres_ix3;
-
-	u32 hal_reo2_sw0_ring_base;
-
-	u32 hal_sw2reo_ring_base;
-	u32 hal_sw2reo1_ring_base;
-
-	u32 hal_reo_cmd_ring_base;
-
-	u32 hal_reo_status_ring_base;
-
-	u32 gcc_gcc_pcie_hot_rst;
-};
-
 static inline const char *ath12k_bd_ie_type_str(enum ath12k_bd_ie_type type)
 {
 	switch (type) {
@@ -377,6 +287,4 @@ static inline const char *ath12k_bd_ie_type_str(enum ath12k_bd_ie_type type)
 	return "unknown";
 }
 
-int ath12k_hw_init(struct ath12k_base *ab);
-
 #endif
diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c
index f7a2a544bef2..2f4daee9e2f0 100644
--- a/drivers/net/wireless/ath/ath12k/mac.c
+++ b/drivers/net/wireless/ath/ath12k/mac.c
@@ -21,6 +21,8 @@
 #include "hif.h"
 #include "wow.h"
 #include "debugfs_sta.h"
+#include "dp.h"
+#include "dp_cmn.h"
 
 #define CHAN2G(_channel, _freq, _flags) { \
 	.band                   = NL80211_BAND_2GHZ, \
@@ -361,6 +363,7 @@ u16 ath12k_mac_he_convert_tones_to_ru_tones(u16 tones)
 		return RU_26;
 	}
 }
+EXPORT_SYMBOL(ath12k_mac_he_convert_tones_to_ru_tones);
 
 enum nl80211_eht_gi ath12k_mac_eht_gi_to_nl80211_eht_gi(u8 sgi)
 {
@@ -375,6 +378,7 @@ enum nl80211_eht_gi ath12k_mac_eht_gi_to_nl80211_eht_gi(u8 sgi)
 		return NL80211_RATE_INFO_EHT_GI_0_8;
 	}
 }
+EXPORT_SYMBOL(ath12k_mac_eht_gi_to_nl80211_eht_gi);
 
 enum nl80211_eht_ru_alloc ath12k_mac_eht_ru_tones_to_nl80211_eht_ru_alloc(u16 ru_tones)
 {
@@ -415,6 +419,7 @@ enum nl80211_eht_ru_alloc ath12k_mac_eht_ru_tones_to_nl80211_eht_ru_alloc(u16 ru
 		return NL80211_RATE_INFO_EHT_RU_ALLOC_26;
 	}
 }
+EXPORT_SYMBOL(ath12k_mac_eht_ru_tones_to_nl80211_eht_ru_alloc);
 
 enum rate_info_bw
 ath12k_mac_bw_to_mac80211_bw(enum ath12k_supported_bw bw)
@@ -441,6 +446,7 @@ ath12k_mac_bw_to_mac80211_bw(enum ath12k_supported_bw bw)
 
 	return ret;
 }
+EXPORT_SYMBOL(ath12k_mac_bw_to_mac80211_bw);
 
 enum ath12k_supported_bw ath12k_mac_mac80211_bw_to_ath12k_bw(enum rate_info_bw bw)
 {
@@ -484,6 +490,7 @@ int ath12k_mac_hw_ratecode_to_legacy_rate(u8 hw_rc, u8 preamble, u8 *rateidx,
 
 	return -EINVAL;
 }
+EXPORT_SYMBOL(ath12k_mac_hw_ratecode_to_legacy_rate);
 
 u8 ath12k_mac_bitrate_to_idx(const struct ieee80211_supported_band *sband,
 			     u32 bitrate)
@@ -1167,26 +1174,119 @@ static int ath12k_mac_set_kickout(struct ath12k_link_vif *arvif)
 	return 0;
 }
 
+static void ath12k_mac_link_sta_rhash_cleanup(void *data, struct ieee80211_sta *sta)
+{
+	u8 link_id;
+	unsigned long links_map;
+	struct ath12k_sta *ahsta;
+	struct ath12k *ar = data;
+	struct ath12k_link_sta *arsta;
+	struct ath12k_link_vif *arvif;
+	struct ath12k_base *ab = ar->ab;
+
+	ahsta = ath12k_sta_to_ahsta(sta);
+	links_map = ahsta->links_map;
+
+	rcu_read_lock();
+	for_each_set_bit(link_id, &links_map, IEEE80211_MLD_MAX_NUM_LINKS) {
+		arsta = rcu_dereference(ahsta->link[link_id]);
+		if (!arsta)
+			continue;
+		arvif = arsta->arvif;
+		if (!(arvif->ar == ar))
+			continue;
+
+		spin_lock_bh(&ab->base_lock);
+		ath12k_link_sta_rhash_delete(ab, arsta);
+		spin_unlock_bh(&ab->base_lock);
+	}
+	rcu_read_unlock();
+}
+
 void ath12k_mac_peer_cleanup_all(struct ath12k *ar)
 {
-	struct ath12k_peer *peer, *tmp;
+	struct ath12k_dp_link_peer *peer, *tmp;
 	struct ath12k_base *ab = ar->ab;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct ath12k_link_vif *arvif, *tmp_vif;
+	struct ath12k_dp_hw *dp_hw = &ar->ah->dp_hw;
+	struct ath12k_dp_peer *dp_peer = NULL;
+	u16 peerid_index;
+	struct list_head peers;
+
+	INIT_LIST_HEAD(&peers);
 
 	lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy);
 
-	spin_lock_bh(&ab->base_lock);
-	list_for_each_entry_safe(peer, tmp, &ab->peers, list) {
+	spin_lock_bh(&dp->dp_lock);
+	list_for_each_entry_safe(peer, tmp, &dp->peers, list) {
 		/* Skip Rx TID cleanup for self peer */
-		if (peer->sta)
+		if (peer->sta && peer->dp_peer)
 			ath12k_dp_rx_peer_tid_cleanup(ar, peer);
 
-		list_del(&peer->list);
-		kfree(peer);
+		/* cleanup dp peer */
+		spin_lock_bh(&dp_hw->peer_lock);
+		dp_peer = peer->dp_peer;
+		peerid_index = ath12k_dp_peer_get_peerid_index(dp, peer->peer_id);
+		rcu_assign_pointer(dp_peer->link_peers[peer->link_id], NULL);
+		rcu_assign_pointer(dp_hw->dp_peers[peerid_index], NULL);
+		spin_unlock_bh(&dp_hw->peer_lock);
+
+		ath12k_dp_link_peer_rhash_delete(dp, peer);
+
+		list_move(&peer->list, &peers);
+	}
+	spin_unlock_bh(&dp->dp_lock);
+
+	synchronize_rcu();
+
+	list_for_each_entry_safe(peer, tmp, &peers, list) {
+		ath12k_dp_link_peer_free(peer);
 	}
-	spin_unlock_bh(&ab->base_lock);
 
 	ar->num_peers = 0;
 	ar->num_stations = 0;
+
+	/* Cleanup rhash table maintained for arsta by iterating over sta */
+	ieee80211_iterate_stations_mtx(ar->ah->hw, ath12k_mac_link_sta_rhash_cleanup,
+				       ar);
+
+	/* Delete all the self dp_peers on asserted radio */
+	list_for_each_entry_safe_reverse(arvif, tmp_vif, &ar->arvifs, list) {
+		if ((arvif->ahvif->vdev_type == WMI_VDEV_TYPE_AP) &&
+		    (arvif->link_id < IEEE80211_MLD_MAX_NUM_LINKS)) {
+			ath12k_dp_peer_delete(dp_hw, arvif->bssid, NULL);
+			arvif->num_stations = 0;
+		}
+	}
+}
+
+void ath12k_mac_dp_peer_cleanup(struct ath12k_hw *ah)
+{
+	struct list_head peers;
+	struct ath12k_dp_peer *dp_peer, *tmp;
+	struct ath12k_dp_hw *dp_hw = &ah->dp_hw;
+
+	INIT_LIST_HEAD(&peers);
+
+	spin_lock_bh(&dp_hw->peer_lock);
+	list_for_each_entry_safe(dp_peer, tmp, &dp_hw->dp_peers_list, list) {
+		if (dp_peer->is_mlo) {
+			rcu_assign_pointer(dp_hw->dp_peers[dp_peer->peer_id], NULL);
+			clear_bit(dp_peer->peer_id, ah->free_ml_peer_id_map);
+		}
+
+		list_move(&dp_peer->list, &peers);
+	}
+
+	spin_unlock_bh(&dp_hw->peer_lock);
+
+	synchronize_rcu();
+
+	list_for_each_entry_safe(dp_peer, tmp, &peers, list) {
+		list_del(&dp_peer->list);
+		kfree(dp_peer);
+	}
 }
 
 static int ath12k_mac_vdev_setup_sync(struct ath12k *ar)
@@ -1458,10 +1558,11 @@ int ath12k_mac_vdev_stop(struct ath12k_link_vif *arvif)
 	return ret;
 }
 
-static int ath12k_mac_op_config(struct ieee80211_hw *hw, int radio_idx, u32 changed)
+int ath12k_mac_op_config(struct ieee80211_hw *hw, int radio_idx, u32 changed)
 {
 	return 0;
 }
+EXPORT_SYMBOL(ath12k_mac_op_config);
 
 static int ath12k_mac_setup_bcn_p2p_ie(struct ath12k_link_vif *arvif,
 				       struct sk_buff *bcn)
@@ -3728,10 +3829,11 @@ static void ath12k_bss_assoc(struct ath12k *ar,
 	struct ath12k_link_sta *arsta;
 	struct ieee80211_sta *ap_sta;
 	struct ath12k_sta *ahsta;
-	struct ath12k_peer *peer;
+	struct ath12k_dp_link_peer *peer;
 	bool is_auth = false;
 	u32 hemode = 0;
 	int ret;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ar->ab);
 
 	lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy);
 
@@ -3843,13 +3945,14 @@ static void ath12k_bss_assoc(struct ath12k *ar,
 		   "mac vdev %d up (associated) bssid %pM aid %d\n",
 		   arvif->vdev_id, bss_conf->bssid, vif->cfg.aid);
 
-	spin_lock_bh(&ar->ab->base_lock);
+	spin_lock_bh(&dp->dp_lock);
 
-	peer = ath12k_peer_find(ar->ab, arvif->vdev_id, arvif->bssid);
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id,
+							 arvif->bssid);
 	if (peer && peer->is_authorized)
 		is_auth = true;
 
-	spin_unlock_bh(&ar->ab->base_lock);
+	spin_unlock_bh(&dp->dp_lock);
 
 	/* Authorize BSS Peer */
 	if (is_auth) {
@@ -4085,6 +4188,9 @@ static void ath12k_mac_remove_link_interface(struct ieee80211_hw *hw,
 		if (ret)
 			ath12k_warn(ar->ab, "failed to submit AP self-peer removal on vdev %d link id %d: %d",
 				    arvif->vdev_id, arvif->link_id, ret);
+
+		if (arvif->link_id < IEEE80211_MLD_MAX_NUM_LINKS)
+			ath12k_dp_peer_delete(&ah->dp_hw, arvif->bssid, NULL);
 	}
 	ath12k_mac_vdev_delete(ar, arvif);
 }
@@ -4138,7 +4244,7 @@ static void ath12k_mac_unassign_link_vif(struct ath12k_link_vif *arvif)
 		memset(arvif, 0, sizeof(*arvif));
 }
 
-static int
+int
 ath12k_mac_op_change_vif_links(struct ieee80211_hw *hw,
 			       struct ieee80211_vif *vif,
 			       u16 old_links, u16 new_links,
@@ -4187,6 +4293,7 @@ ath12k_mac_op_change_vif_links(struct ieee80211_hw *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL(ath12k_mac_op_change_vif_links);
 
 static int ath12k_mac_fils_discovery(struct ath12k_link_vif *arvif,
 				     struct ieee80211_bss_conf *info)
@@ -4235,9 +4342,9 @@ static int ath12k_mac_fils_discovery(struct ath12k_link_vif *arvif,
 	return ret;
 }
 
-static void ath12k_mac_op_vif_cfg_changed(struct ieee80211_hw *hw,
-					  struct ieee80211_vif *vif,
-					  u64 changed)
+void ath12k_mac_op_vif_cfg_changed(struct ieee80211_hw *hw,
+				   struct ieee80211_vif *vif,
+				   u64 changed)
 {
 	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
 	unsigned long links = ahvif->links_map;
@@ -4305,6 +4412,7 @@ static void ath12k_mac_op_vif_cfg_changed(struct ieee80211_hw *hw,
 		}
 	}
 }
+EXPORT_SYMBOL(ath12k_mac_op_vif_cfg_changed);
 
 static void ath12k_mac_vif_setup_ps(struct ath12k_link_vif *arvif)
 {
@@ -4780,10 +4888,10 @@ static void ath12k_ahvif_put_link_cache(struct ath12k_vif *ahvif, u8 link_id)
 	ahvif->cache[link_id] = NULL;
 }
 
-static void ath12k_mac_op_link_info_changed(struct ieee80211_hw *hw,
-					    struct ieee80211_vif *vif,
-					    struct ieee80211_bss_conf *info,
-					    u64 changed)
+void ath12k_mac_op_link_info_changed(struct ieee80211_hw *hw,
+				     struct ieee80211_vif *vif,
+				     struct ieee80211_bss_conf *info,
+				     u64 changed)
 {
 	struct ath12k *ar;
 	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
@@ -4813,6 +4921,7 @@ static void ath12k_mac_op_link_info_changed(struct ieee80211_hw *hw,
 
 	ath12k_mac_bss_info_changed(ar, arvif, info, changed);
 }
+EXPORT_SYMBOL(ath12k_mac_op_link_info_changed);
 
 static struct ath12k*
 ath12k_mac_select_scan_device(struct ieee80211_hw *hw,
@@ -5114,10 +5223,10 @@ int ath12k_mac_get_fw_stats(struct ath12k *ar,
 	return 0;
 }
 
-static int ath12k_mac_op_get_txpower(struct ieee80211_hw *hw,
-				     struct ieee80211_vif *vif,
-				     unsigned int link_id,
-				     int *dbm)
+int ath12k_mac_op_get_txpower(struct ieee80211_hw *hw,
+			      struct ieee80211_vif *vif,
+			      unsigned int link_id,
+			      int *dbm)
 {
 	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
 	struct ath12k_fw_stats_req_params params = {};
@@ -5191,6 +5300,7 @@ static int ath12k_mac_op_get_txpower(struct ieee80211_hw *hw,
 		   *dbm);
 	return 0;
 }
+EXPORT_SYMBOL(ath12k_mac_op_get_txpower);
 
 static u8
 ath12k_mac_find_link_id_by_ar(struct ath12k_vif *ahvif, struct ath12k *ar)
@@ -5402,9 +5512,9 @@ static int ath12k_mac_initiate_hw_scan(struct ieee80211_hw *hw,
 	return ret;
 }
 
-static int ath12k_mac_op_hw_scan(struct ieee80211_hw *hw,
-				 struct ieee80211_vif *vif,
-				 struct ieee80211_scan_request *hw_req)
+int ath12k_mac_op_hw_scan(struct ieee80211_hw *hw,
+			  struct ieee80211_vif *vif,
+			  struct ieee80211_scan_request *hw_req)
 {
 	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
 	struct ieee80211_channel **chan_list, *chan;
@@ -5482,9 +5592,10 @@ static int ath12k_mac_op_hw_scan(struct ieee80211_hw *hw,
 	kfree(chan_list);
 	return ret;
 }
+EXPORT_SYMBOL(ath12k_mac_op_hw_scan);
 
-static void ath12k_mac_op_cancel_hw_scan(struct ieee80211_hw *hw,
-					 struct ieee80211_vif *vif)
+void ath12k_mac_op_cancel_hw_scan(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif)
 {
 	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
 	unsigned long link_id, links_map = ahvif->links_map;
@@ -5505,6 +5616,7 @@ static void ath12k_mac_op_cancel_hw_scan(struct ieee80211_hw *hw,
 		cancel_delayed_work_sync(&ar->scan.timeout);
 	}
 }
+EXPORT_SYMBOL(ath12k_mac_op_cancel_hw_scan);
 
 static int ath12k_install_key(struct ath12k_link_vif *arvif,
 			      struct ieee80211_key_conf *key,
@@ -5612,7 +5724,7 @@ static int ath12k_install_key(struct ath12k_link_vif *arvif,
 		return -ETIMEDOUT;
 
 	if (ether_addr_equal(arg.macaddr, arvif->bssid))
-		ahvif->key_cipher = arg.ieee80211_key_cipher;
+		ahvif->dp_vif.key_cipher = arg.ieee80211_key_cipher;
 
 	if (ar->install_key_status) {
 		ret = -EINVAL;
@@ -5655,27 +5767,40 @@ static int ath12k_clear_peer_keys(struct ath12k_link_vif *arvif,
 {
 	struct ath12k *ar = arvif->ar;
 	struct ath12k_base *ab = ar->ab;
-	struct ath12k_peer *peer;
+	struct ath12k_dp_link_peer *peer;
 	int first_errno = 0;
 	int ret;
-	int i;
+	int i, len;
 	u32 flags = 0;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct ieee80211_key_conf *keys[WMI_MAX_KEY_INDEX + 1] = {};
 
 	lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy);
 
-	spin_lock_bh(&ab->base_lock);
-	peer = ath12k_peer_find(ab, arvif->vdev_id, addr);
-	spin_unlock_bh(&ab->base_lock);
-
-	if (!peer)
+	spin_lock_bh(&dp->dp_lock);
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id, addr);
+	if (!peer || !peer->dp_peer) {
+		spin_unlock_bh(&dp->dp_lock);
 		return -ENOENT;
+	}
+
+	len = ARRAY_SIZE(peer->dp_peer->keys);
+	for (i = 0; i < len; i++) {
+		if (!peer->dp_peer->keys[i])
+			continue;
+
+		keys[i] = peer->dp_peer->keys[i];
+		peer->dp_peer->keys[i] = NULL;
+	}
+
+	spin_unlock_bh(&dp->dp_lock);
 
-	for (i = 0; i < ARRAY_SIZE(peer->keys); i++) {
-		if (!peer->keys[i])
+	for (i = 0; i < len; i++) {
+		if (!keys[i])
 			continue;
 
 		/* key flags are not required to delete the key */
-		ret = ath12k_install_key(arvif, peer->keys[i],
+		ret = ath12k_install_key(arvif, keys[i],
 					 DISABLE_KEY, addr, flags);
 		if (ret < 0 && first_errno == 0)
 			first_errno = ret;
@@ -5683,10 +5808,6 @@ static int ath12k_clear_peer_keys(struct ath12k_link_vif *arvif,
 		if (ret < 0)
 			ath12k_warn(ab, "failed to remove peer key %d: %d\n",
 				    i, ret);
-
-		spin_lock_bh(&ab->base_lock);
-		peer->keys[i] = NULL;
-		spin_unlock_bh(&ab->base_lock);
 	}
 
 	return first_errno;
@@ -5699,11 +5820,12 @@ static int ath12k_mac_set_key(struct ath12k *ar, enum set_key_cmd cmd,
 {
 	struct ieee80211_sta *sta = NULL;
 	struct ath12k_base *ab = ar->ab;
-	struct ath12k_peer *peer;
+	struct ath12k_dp_link_peer *peer;
 	struct ath12k_sta *ahsta;
 	const u8 *peer_addr;
 	int ret;
 	u32 flags = 0;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 
 	lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy);
 
@@ -5723,11 +5845,12 @@ static int ath12k_mac_set_key(struct ath12k *ar, enum set_key_cmd cmd,
 	/* the peer should not disappear in mid-way (unless FW goes awry) since
 	 * we already hold wiphy lock. we just make sure its there now.
 	 */
-	spin_lock_bh(&ab->base_lock);
-	peer = ath12k_peer_find(ab, arvif->vdev_id, peer_addr);
-	spin_unlock_bh(&ab->base_lock);
+	spin_lock_bh(&dp->dp_lock);
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id,
+							 peer_addr);
+	if (!peer || !peer->dp_peer) {
+		spin_unlock_bh(&dp->dp_lock);
 
-	if (!peer) {
 		if (cmd == SET_KEY) {
 			ath12k_warn(ab, "cannot install key for non-existent peer %pM\n",
 				    peer_addr);
@@ -5740,6 +5863,8 @@ static int ath12k_mac_set_key(struct ath12k *ar, enum set_key_cmd cmd,
 		return 0;
 	}
 
+	spin_unlock_bh(&dp->dp_lock);
+
 	if (key->flags & IEEE80211_KEY_FLAG_PAIRWISE)
 		flags = WMI_KEY_PAIRWISE;
 	else
@@ -5757,23 +5882,26 @@ static int ath12k_mac_set_key(struct ath12k *ar, enum set_key_cmd cmd,
 		return ret;
 	}
 
-	spin_lock_bh(&ab->base_lock);
-	peer = ath12k_peer_find(ab, arvif->vdev_id, peer_addr);
-	if (peer && cmd == SET_KEY) {
-		peer->keys[key->keyidx] = key;
+	spin_lock_bh(&dp->dp_lock);
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id,
+							 peer_addr);
+	if (peer && peer->dp_peer && cmd == SET_KEY) {
+		peer->dp_peer->keys[key->keyidx] = key;
 		if (key->flags & IEEE80211_KEY_FLAG_PAIRWISE) {
-			peer->ucast_keyidx = key->keyidx;
-			peer->sec_type = ath12k_dp_tx_get_encrypt_type(key->cipher);
+			peer->dp_peer->ucast_keyidx = key->keyidx;
+			peer->dp_peer->sec_type =
+					ath12k_dp_tx_get_encrypt_type(key->cipher);
 		} else {
-			peer->mcast_keyidx = key->keyidx;
-			peer->sec_type_grp = ath12k_dp_tx_get_encrypt_type(key->cipher);
+			peer->dp_peer->mcast_keyidx = key->keyidx;
+			peer->dp_peer->sec_type_grp =
+					ath12k_dp_tx_get_encrypt_type(key->cipher);
 		}
-	} else if (peer && cmd == DISABLE_KEY) {
-		peer->keys[key->keyidx] = NULL;
+	} else if (peer && peer->dp_peer && cmd == DISABLE_KEY) {
+		peer->dp_peer->keys[key->keyidx] = NULL;
 		if (key->flags & IEEE80211_KEY_FLAG_PAIRWISE)
-			peer->ucast_keyidx = 0;
+			peer->dp_peer->ucast_keyidx = 0;
 		else
-			peer->mcast_keyidx = 0;
+			peer->dp_peer->mcast_keyidx = 0;
 	} else if (!peer)
 		/* impossible unless FW goes crazy */
 		ath12k_warn(ab, "peer %pM disappeared!\n", peer_addr);
@@ -5798,7 +5926,7 @@ static int ath12k_mac_set_key(struct ath12k *ar, enum set_key_cmd cmd,
 		}
 	}
 
-	spin_unlock_bh(&ab->base_lock);
+	spin_unlock_bh(&dp->dp_lock);
 
 	return 0;
 }
@@ -5842,9 +5970,9 @@ static int ath12k_mac_update_key_cache(struct ath12k_vif_cache *cache,
 	return 0;
 }
 
-static int ath12k_mac_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
-				 struct ieee80211_vif *vif, struct ieee80211_sta *sta,
-				 struct ieee80211_key_conf *key)
+int ath12k_mac_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
+			  struct ieee80211_vif *vif, struct ieee80211_sta *sta,
+			  struct ieee80211_key_conf *key)
 {
 	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
 	struct ath12k_link_vif *arvif;
@@ -5931,6 +6059,7 @@ static int ath12k_mac_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 
 	return 0;
 }
+EXPORT_SYMBOL(ath12k_mac_op_set_key);
 
 static int
 ath12k_mac_bitrate_mask_num_vht_rates(struct ath12k *ar,
@@ -6556,46 +6685,47 @@ static void ath12k_mac_station_post_remove(struct ath12k *ar,
 {
 	struct ieee80211_vif *vif = ath12k_ahvif_to_vif(arvif->ahvif);
 	struct ieee80211_sta *sta = ath12k_ahsta_to_sta(arsta->ahsta);
-	struct ath12k_peer *peer;
+	struct ath12k_dp_link_peer *peer;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ar->ab);
 
 	lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy);
 
 	ath12k_mac_dec_num_stations(arvif, arsta);
 
-	spin_lock_bh(&ar->ab->base_lock);
+	spin_lock_bh(&dp->dp_lock);
 
-	peer = ath12k_peer_find(ar->ab, arvif->vdev_id, arsta->addr);
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id,
+							 arsta->addr);
 	if (peer && peer->sta == sta) {
 		ath12k_warn(ar->ab, "Found peer entry %pM n vdev %i after it was supposedly removed\n",
 			    vif->addr, arvif->vdev_id);
 		peer->sta = NULL;
-		list_del(&peer->list);
-		kfree(peer);
+
+		ath12k_dp_link_peer_free(peer);
 		ar->num_peers--;
 	}
 
-	spin_unlock_bh(&ar->ab->base_lock);
-
-	kfree(arsta->rx_stats);
-	arsta->rx_stats = NULL;
+	spin_unlock_bh(&dp->dp_lock);
 }
 
 static int ath12k_mac_station_unauthorize(struct ath12k *ar,
 					  struct ath12k_link_vif *arvif,
 					  struct ath12k_link_sta *arsta)
 {
-	struct ath12k_peer *peer;
+	struct ath12k_dp_link_peer *peer;
 	int ret;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ar->ab);
 
 	lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy);
 
-	spin_lock_bh(&ar->ab->base_lock);
+	spin_lock_bh(&dp->dp_lock);
 
-	peer = ath12k_peer_find(ar->ab, arvif->vdev_id, arsta->addr);
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id,
+							 arsta->addr);
 	if (peer)
 		peer->is_authorized = false;
 
-	spin_unlock_bh(&ar->ab->base_lock);
+	spin_unlock_bh(&dp->dp_lock);
 
 	/* Driver must clear the keys during the state change from
 	 * IEEE80211_STA_AUTHORIZED to IEEE80211_STA_ASSOC, since after
@@ -6617,19 +6747,21 @@ static int ath12k_mac_station_authorize(struct ath12k *ar,
 					struct ath12k_link_vif *arvif,
 					struct ath12k_link_sta *arsta)
 {
-	struct ath12k_peer *peer;
+	struct ath12k_dp_link_peer *peer;
 	struct ieee80211_vif *vif = ath12k_ahvif_to_vif(arvif->ahvif);
 	int ret;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ar->ab);
 
 	lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy);
 
-	spin_lock_bh(&ar->ab->base_lock);
+	spin_lock_bh(&dp->dp_lock);
 
-	peer = ath12k_peer_find(ar->ab, arvif->vdev_id, arsta->addr);
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id,
+							 arsta->addr);
 	if (peer)
 		peer->is_authorized = true;
 
-	spin_unlock_bh(&ar->ab->base_lock);
+	spin_unlock_bh(&dp->dp_lock);
 
 	if (vif->type == NL80211_IFTYPE_STATION && arvif->is_up) {
 		ret = ath12k_wmi_set_peer_param(ar, arsta->addr,
@@ -6653,6 +6785,7 @@ static int ath12k_mac_station_remove(struct ath12k *ar,
 	struct ieee80211_sta *sta = ath12k_ahsta_to_sta(arsta->ahsta);
 	struct ath12k_vif *ahvif = arvif->ahvif;
 	int ret = 0;
+	struct ath12k_link_sta *temp_arsta;
 
 	lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy);
 
@@ -6681,6 +6814,15 @@ static int ath12k_mac_station_remove(struct ath12k *ar,
 
 	ath12k_mac_station_post_remove(ar, arvif, arsta);
 
+	spin_lock_bh(&ar->ab->base_lock);
+
+	/* To handle roaming and split phy scenario */
+	temp_arsta = ath12k_link_sta_find_by_addr(ar->ab, arsta->addr);
+	if (temp_arsta && temp_arsta->arvif->ar == ar)
+		ath12k_link_sta_rhash_delete(ar->ab, arsta);
+
+	spin_unlock_bh(&ar->ab->base_lock);
+
 	if (sta->valid_links)
 		ath12k_mac_free_unassign_link_sta(ahvif->ah,
 						  arsta->ahsta, arsta->link_id);
@@ -6697,6 +6839,7 @@ static int ath12k_mac_station_add(struct ath12k *ar,
 	struct ieee80211_sta *sta = ath12k_ahsta_to_sta(arsta->ahsta);
 	struct ath12k_wmi_peer_create_arg peer_param = {};
 	int ret;
+	struct ath12k_link_sta *temp_arsta;
 
 	lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy);
 
@@ -6707,12 +6850,24 @@ static int ath12k_mac_station_add(struct ath12k *ar,
 		goto exit;
 	}
 
-	if (ath12k_debugfs_is_extd_rx_stats_enabled(ar) && !arsta->rx_stats) {
-		arsta->rx_stats = kzalloc(sizeof(*arsta->rx_stats), GFP_KERNEL);
-		if (!arsta->rx_stats) {
-			ret = -ENOMEM;
-			goto dec_num_station;
-		}
+	spin_lock_bh(&ab->base_lock);
+
+	/*
+	 * In case of Split PHY and roaming scenario, pdev idx
+	 * might differ but both the pdev will share same rhash
+	 * table. In that case update the rhash table if link_sta is
+	 * already present
+	 */
+	temp_arsta = ath12k_link_sta_find_by_addr(ab, arsta->addr);
+	if (temp_arsta && temp_arsta->arvif->ar != ar)
+		ath12k_link_sta_rhash_delete(ab, temp_arsta);
+
+	ret = ath12k_link_sta_rhash_add(ab, arsta);
+	spin_unlock_bh(&ab->base_lock);
+	if (ret) {
+		ath12k_warn(ab, "Failed to add arsta: %pM to hash table, ret: %d",
+			    arsta->addr, ret);
+		goto dec_num_station;
 	}
 
 	peer_param.vdev_id = arvif->vdev_id;
@@ -6758,13 +6913,13 @@ static int ath12k_mac_station_add(struct ath12k *ar,
 		}
 	}
 
-	ewma_avg_rssi_init(&arsta->avg_rssi);
 	return 0;
 
 free_peer:
 	ath12k_peer_delete(ar, arvif->vdev_id, arsta->addr);
-	kfree(arsta->rx_stats);
-	arsta->rx_stats = NULL;
+	spin_lock_bh(&ab->base_lock);
+	ath12k_link_sta_rhash_delete(ab, arsta);
+	spin_unlock_bh(&ab->base_lock);
 dec_num_station:
 	ath12k_mac_dec_num_stations(arvif, arsta);
 exit:
@@ -6841,10 +6996,17 @@ static void ath12k_mac_ml_station_remove(struct ath12k_vif *ahvif,
 
 		ath12k_mac_station_post_remove(ar, arvif, arsta);
 
+		spin_lock_bh(&ar->ab->base_lock);
+		ath12k_link_sta_rhash_delete(ar->ab, arsta);
+		spin_unlock_bh(&ar->ab->base_lock);
+
 		ath12k_mac_free_unassign_link_sta(ah, ahsta, link_id);
 	}
 
-	ath12k_peer_ml_delete(ah, sta);
+	if (sta->mlo) {
+		clear_bit(ahsta->ml_peer_id, ah->free_ml_peer_id_map);
+		ahsta->ml_peer_id = ATH12K_MLO_PEER_ID_INVALID;
+	}
 }
 
 static int ath12k_mac_handle_link_sta_state(struct ieee80211_hw *hw,
@@ -7267,11 +7429,11 @@ static int ath12k_mac_select_links(struct ath12k_base *ab,
 	return 0;
 }
 
-static int ath12k_mac_op_sta_state(struct ieee80211_hw *hw,
-				   struct ieee80211_vif *vif,
-				   struct ieee80211_sta *sta,
-				   enum ieee80211_sta_state old_state,
-				   enum ieee80211_sta_state new_state)
+int ath12k_mac_op_sta_state(struct ieee80211_hw *hw,
+			    struct ieee80211_vif *vif,
+			    struct ieee80211_sta *sta,
+			    enum ieee80211_sta_state old_state,
+			    enum ieee80211_sta_state new_state)
 {
 	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
 	struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(sta);
@@ -7283,7 +7445,8 @@ static int ath12k_mac_op_sta_state(struct ieee80211_hw *hw,
 	u16 selected_links = 0;
 	u8 link_id = 0, i;
 	struct ath12k *ar;
-	int ret;
+	int ret = -EINVAL;
+	struct ath12k_dp_peer_create_params dp_params = {};
 
 	lockdep_assert_wiphy(hw->wiphy);
 
@@ -7306,12 +7469,28 @@ static int ath12k_mac_op_sta_state(struct ieee80211_hw *hw,
 		/* ML sta */
 		if (sta->mlo && !ahsta->links_map &&
 		    (hweight16(sta->valid_links) == 1)) {
-			ret = ath12k_peer_ml_create(ah, sta);
-			if (ret) {
-				ath12k_hw_warn(ah, "unable to create ML peer for sta %pM",
+			ahsta->ml_peer_id = ath12k_peer_ml_alloc(ah);
+			if (ahsta->ml_peer_id == ATH12K_MLO_PEER_ID_INVALID) {
+				ath12k_hw_warn(ah, "unable to allocate ML peer id for sta %pM",
 					       sta->addr);
 				goto exit;
 			}
+
+			dp_params.is_mlo = true;
+			dp_params.peer_id = ahsta->ml_peer_id | ATH12K_PEER_ML_ID_VALID;
+		}
+
+		dp_params.sta = sta;
+
+		if (vif->type == NL80211_IFTYPE_AP)
+			dp_params.ucast_ra_only = true;
+
+		ret = ath12k_dp_peer_create(&ah->dp_hw, sta->addr, &dp_params);
+		if (ret) {
+			ath12k_hw_warn(ah, "unable to create ath12k_dp_peer for sta %pM, ret: %d",
+				       sta->addr, ret);
+
+			goto ml_peer_id_clear;
 		}
 
 		ret = ath12k_mac_assign_link_sta(ah, ahsta, arsta, ahvif,
@@ -7319,7 +7498,7 @@ static int ath12k_mac_op_sta_state(struct ieee80211_hw *hw,
 		if (ret) {
 			ath12k_hw_warn(ah, "unable assign link %d for sta %pM",
 				       link_id, sta->addr);
-			goto exit;
+			goto peer_delete;
 		}
 
 		/* above arsta will get memset, hence do this after assign
@@ -7389,7 +7568,12 @@ static int ath12k_mac_op_sta_state(struct ieee80211_hw *hw,
 		if (ret) {
 			ath12k_hw_warn(ah, "unable to move link sta %d of sta %pM from state %d to %d",
 				       link_id, arsta->addr, old_state, new_state);
-			goto exit;
+
+			if (old_state == IEEE80211_STA_NOTEXIST &&
+			    new_state == IEEE80211_STA_NONE)
+				goto peer_delete;
+			else
+				goto exit;
 		}
 	}
 
@@ -7417,11 +7601,23 @@ static int ath12k_mac_op_sta_state(struct ieee80211_hw *hw,
 	 * handler below
 	 */
 	if (old_state == IEEE80211_STA_NONE &&
-	    new_state == IEEE80211_STA_NOTEXIST && sta->mlo)
-		ath12k_mac_ml_station_remove(ahvif, ahsta);
+	    new_state == IEEE80211_STA_NOTEXIST) {
+		if (sta->mlo)
+			ath12k_mac_ml_station_remove(ahvif, ahsta);
+
+		ath12k_dp_peer_delete(&ah->dp_hw, sta->addr, sta);
+	}
 
 	ret = 0;
+	goto exit;
 
+peer_delete:
+	ath12k_dp_peer_delete(&ah->dp_hw, sta->addr, sta);
+ml_peer_id_clear:
+	if (sta->mlo) {
+		clear_bit(ahsta->ml_peer_id, ah->free_ml_peer_id_map);
+		ahsta->ml_peer_id = ATH12K_MLO_PEER_ID_INVALID;
+	}
 exit:
 	/* update the state if everything went well */
 	if (!ret)
@@ -7429,10 +7625,11 @@ static int ath12k_mac_op_sta_state(struct ieee80211_hw *hw,
 
 	return ret;
 }
+EXPORT_SYMBOL(ath12k_mac_op_sta_state);
 
-static int ath12k_mac_op_sta_set_txpwr(struct ieee80211_hw *hw,
-				       struct ieee80211_vif *vif,
-				       struct ieee80211_sta *sta)
+int ath12k_mac_op_sta_set_txpwr(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif,
+				struct ieee80211_sta *sta)
 {
 	struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(sta);
 	struct ath12k *ar;
@@ -7479,11 +7676,12 @@ static int ath12k_mac_op_sta_set_txpwr(struct ieee80211_hw *hw,
 out:
 	return ret;
 }
+EXPORT_SYMBOL(ath12k_mac_op_sta_set_txpwr);
 
-static void ath12k_mac_op_link_sta_rc_update(struct ieee80211_hw *hw,
-					     struct ieee80211_vif *vif,
-					     struct ieee80211_link_sta *link_sta,
-					     u32 changed)
+void ath12k_mac_op_link_sta_rc_update(struct ieee80211_hw *hw,
+				      struct ieee80211_vif *vif,
+				      struct ieee80211_link_sta *link_sta,
+				      u32 changed)
 {
 	struct ieee80211_sta *sta = link_sta->sta;
 	struct ath12k *ar;
@@ -7492,8 +7690,9 @@ static void ath12k_mac_op_link_sta_rc_update(struct ieee80211_hw *hw,
 	struct ath12k_hw *ah = ath12k_hw_to_ah(hw);
 	struct ath12k_link_sta *arsta;
 	struct ath12k_link_vif *arvif;
-	struct ath12k_peer *peer;
+	struct ath12k_dp_link_peer *peer;
 	u32 bw, smps;
+	struct ath12k_dp *dp;
 
 	rcu_read_lock();
 	arvif = rcu_dereference(ahvif->link[link_sta->link_id]);
@@ -7505,6 +7704,7 @@ static void ath12k_mac_op_link_sta_rc_update(struct ieee80211_hw *hw,
 	}
 
 	ar = arvif->ar;
+	dp = ath12k_ab_to_dp(ar->ab);
 
 	arsta = rcu_dereference(ahsta->link[link_sta->link_id]);
 	if (!arsta) {
@@ -7513,18 +7713,19 @@ static void ath12k_mac_op_link_sta_rc_update(struct ieee80211_hw *hw,
 			    link_sta->link_id, sta->addr);
 		return;
 	}
-	spin_lock_bh(&ar->ab->base_lock);
+	spin_lock_bh(&dp->dp_lock);
 
-	peer = ath12k_peer_find(ar->ab, arvif->vdev_id, arsta->addr);
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arvif->vdev_id,
+							 arsta->addr);
 	if (!peer) {
-		spin_unlock_bh(&ar->ab->base_lock);
+		spin_unlock_bh(&dp->dp_lock);
 		rcu_read_unlock();
 		ath12k_warn(ar->ab, "mac sta rc update failed to find peer %pM on vdev %i\n",
 			    arsta->addr, arvif->vdev_id);
 		return;
 	}
 
-	spin_unlock_bh(&ar->ab->base_lock);
+	spin_unlock_bh(&dp->dp_lock);
 
 	if (arsta->link_id >= IEEE80211_MLD_MAX_NUM_LINKS) {
 		rcu_read_unlock();
@@ -7587,6 +7788,7 @@ static void ath12k_mac_op_link_sta_rc_update(struct ieee80211_hw *hw,
 
 	rcu_read_unlock();
 }
+EXPORT_SYMBOL(ath12k_mac_op_link_sta_rc_update);
 
 static struct ath12k_link_sta *ath12k_mac_alloc_assign_link_sta(struct ath12k_hw *ah,
 								struct ath12k_sta *ahsta,
@@ -7618,10 +7820,10 @@ static struct ath12k_link_sta *ath12k_mac_alloc_assign_link_sta(struct ath12k_hw
 	return arsta;
 }
 
-static int ath12k_mac_op_change_sta_links(struct ieee80211_hw *hw,
-					  struct ieee80211_vif *vif,
-					  struct ieee80211_sta *sta,
-					  u16 old_links, u16 new_links)
+int ath12k_mac_op_change_sta_links(struct ieee80211_hw *hw,
+				   struct ieee80211_vif *vif,
+				   struct ieee80211_sta *sta,
+				   u16 old_links, u16 new_links)
 {
 	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
 	struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(sta);
@@ -7682,15 +7884,17 @@ static int ath12k_mac_op_change_sta_links(struct ieee80211_hw *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL(ath12k_mac_op_change_sta_links);
 
-static bool ath12k_mac_op_can_activate_links(struct ieee80211_hw *hw,
-					     struct ieee80211_vif *vif,
-					     u16 active_links)
+bool ath12k_mac_op_can_activate_links(struct ieee80211_hw *hw,
+				      struct ieee80211_vif *vif,
+				      u16 active_links)
 {
 	/* TODO: Handle recovery case */
 
 	return true;
 }
+EXPORT_SYMBOL(ath12k_mac_op_can_activate_links);
 
 static int ath12k_conf_tx_uapsd(struct ath12k_link_vif *arvif,
 				u16 ac, bool enable)
@@ -7802,10 +8006,10 @@ static int ath12k_mac_conf_tx(struct ath12k_link_vif *arvif, u16 ac,
 	return ret;
 }
 
-static int ath12k_mac_op_conf_tx(struct ieee80211_hw *hw,
-				 struct ieee80211_vif *vif,
-				 unsigned int link_id, u16 ac,
-				 const struct ieee80211_tx_queue_params *params)
+int ath12k_mac_op_conf_tx(struct ieee80211_hw *hw,
+			  struct ieee80211_vif *vif,
+			  unsigned int link_id, u16 ac,
+			  const struct ieee80211_tx_queue_params *params)
 {
 	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
 	struct ath12k_link_vif *arvif;
@@ -7834,6 +8038,7 @@ static int ath12k_mac_op_conf_tx(struct ieee80211_hw *hw,
 
 	return ret;
 }
+EXPORT_SYMBOL(ath12k_mac_op_conf_tx);
 
 static struct ieee80211_sta_ht_cap
 ath12k_create_ht_cap(struct ath12k *ar, u32 ar_ht_cap, u32 rate_cap_rx_chainmask)
@@ -8685,7 +8890,7 @@ static int ath12k_mac_mgmt_tx_wmi(struct ath12k *ar, struct ath12k_link_vif *arv
 		     ieee80211_is_disassoc(hdr->frame_control)) &&
 		     ieee80211_has_protected(hdr->frame_control)) {
 			enctype = ath12k_dp_tx_get_encrypt_type(skb_cb->cipher);
-			mic_len = ath12k_dp_rx_crypto_mic_len(ar, enctype);
+			mic_len = ath12k_dp_rx_crypto_mic_len(ab->dp, enctype);
 			skb_put(skb, mic_len);
 		}
 	}
@@ -8956,8 +9161,8 @@ static void ath12k_mgmt_over_wmi_tx_work(struct wiphy *wiphy, struct wiphy_work
 	}
 }
 
-static int ath12k_mac_mgmt_tx(struct ath12k *ar, struct sk_buff *skb,
-			      bool is_prb_rsp)
+int ath12k_mac_mgmt_tx(struct ath12k *ar, struct sk_buff *skb,
+		       bool is_prb_rsp)
 {
 	struct sk_buff_head *q = &ar->wmi_mgmt_tx_queue;
 
@@ -8987,11 +9192,12 @@ static int ath12k_mac_mgmt_tx(struct ath12k *ar, struct sk_buff *skb,
 
 	return 0;
 }
+EXPORT_SYMBOL(ath12k_mac_mgmt_tx);
 
-static void ath12k_mac_add_p2p_noa_ie(struct ath12k *ar,
-				      struct ieee80211_vif *vif,
-				      struct sk_buff *skb,
-				      bool is_prb_rsp)
+void ath12k_mac_add_p2p_noa_ie(struct ath12k *ar,
+			       struct ieee80211_vif *vif,
+			       struct sk_buff *skb,
+			       bool is_prb_rsp)
 {
 	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
 
@@ -9008,11 +9214,12 @@ static void ath12k_mac_add_p2p_noa_ie(struct ath12k *ar,
 
 	spin_unlock_bh(&ar->data_lock);
 }
+EXPORT_SYMBOL(ath12k_mac_add_p2p_noa_ie);
 
 /* Note: called under rcu_read_lock() */
-static void ath12k_mlo_mcast_update_tx_link_address(struct ieee80211_vif *vif,
-						    u8 link_id, struct sk_buff *skb,
-						    u32 info_flags)
+void ath12k_mlo_mcast_update_tx_link_address(struct ieee80211_vif *vif,
+					     u8 link_id, struct sk_buff *skb,
+					     u32 info_flags)
 {
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
 	struct ieee80211_bss_conf *bss_conf;
@@ -9024,10 +9231,11 @@ static void ath12k_mlo_mcast_update_tx_link_address(struct ieee80211_vif *vif,
 	if (bss_conf)
 		ether_addr_copy(hdr->addr2, bss_conf->addr);
 }
+EXPORT_SYMBOL(ath12k_mlo_mcast_update_tx_link_address);
 
 /* Note: called under rcu_read_lock() */
-static u8 ath12k_mac_get_tx_link(struct ieee80211_sta *sta, struct ieee80211_vif *vif,
-				 u8 link, struct sk_buff *skb, u32 info_flags)
+u8 ath12k_mac_get_tx_link(struct ieee80211_sta *sta, struct ieee80211_vif *vif,
+			  u8 link, struct sk_buff *skb, u32 info_flags)
 {
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
 	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
@@ -9122,190 +9330,7 @@ static u8 ath12k_mac_get_tx_link(struct ieee80211_sta *sta, struct ieee80211_vif
 
 	return link;
 }
-
-/* Note: called under rcu_read_lock() */
-static void ath12k_mac_op_tx(struct ieee80211_hw *hw,
-			     struct ieee80211_tx_control *control,
-			     struct sk_buff *skb)
-{
-	struct ath12k_skb_cb *skb_cb = ATH12K_SKB_CB(skb);
-	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
-	struct ieee80211_vif *vif = info->control.vif;
-	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
-	struct ath12k_link_vif *arvif = &ahvif->deflink;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
-	struct ieee80211_key_conf *key = info->control.hw_key;
-	struct ieee80211_sta *sta = control->sta;
-	struct ath12k_link_vif *tmp_arvif;
-	u32 info_flags = info->flags;
-	struct sk_buff *msdu_copied;
-	struct ath12k *ar, *tmp_ar;
-	struct ath12k_peer *peer;
-	unsigned long links_map;
-	bool is_mcast = false;
-	bool is_dvlan = false;
-	struct ethhdr *eth;
-	bool is_prb_rsp;
-	u16 mcbc_gsn;
-	u8 link_id;
-	int ret;
-
-	if (ahvif->vdev_type == WMI_VDEV_TYPE_MONITOR) {
-		ieee80211_free_txskb(hw, skb);
-		return;
-	}
-
-	link_id = u32_get_bits(info->control.flags, IEEE80211_TX_CTRL_MLO_LINK);
-	memset(skb_cb, 0, sizeof(*skb_cb));
-	skb_cb->vif = vif;
-
-	if (key) {
-		skb_cb->cipher = key->cipher;
-		skb_cb->flags |= ATH12K_SKB_CIPHER_SET;
-	}
-
-	/* handle only for MLO case, use deflink for non MLO case */
-	if (ieee80211_vif_is_mld(vif)) {
-		link_id = ath12k_mac_get_tx_link(sta, vif, link_id, skb, info_flags);
-		if (link_id >= IEEE80211_MLD_MAX_NUM_LINKS) {
-			ieee80211_free_txskb(hw, skb);
-			return;
-		}
-	} else {
-		link_id = 0;
-	}
-
-	arvif = rcu_dereference(ahvif->link[link_id]);
-	if (!arvif || !arvif->ar) {
-		ath12k_warn(ahvif->ah, "failed to find arvif link id %u for frame transmission",
-			    link_id);
-		ieee80211_free_txskb(hw, skb);
-		return;
-	}
-
-	ar = arvif->ar;
-	skb_cb->link_id = link_id;
-	is_prb_rsp = ieee80211_is_probe_resp(hdr->frame_control);
-
-	if (info_flags & IEEE80211_TX_CTL_HW_80211_ENCAP) {
-		eth = (struct ethhdr *)skb->data;
-		is_mcast = is_multicast_ether_addr(eth->h_dest);
-
-		skb_cb->flags |= ATH12K_SKB_HW_80211_ENCAP;
-	} else if (ieee80211_is_mgmt(hdr->frame_control)) {
-		if (sta && sta->mlo)
-			skb_cb->flags |= ATH12K_SKB_MLO_STA;
-
-		ret = ath12k_mac_mgmt_tx(ar, skb, is_prb_rsp);
-		if (ret) {
-			ath12k_warn(ar->ab, "failed to queue management frame %d\n",
-				    ret);
-			ieee80211_free_txskb(hw, skb);
-		}
-		return;
-	}
-
-	if (!(info_flags & IEEE80211_TX_CTL_HW_80211_ENCAP))
-		is_mcast = is_multicast_ether_addr(hdr->addr1);
-
-	/* This is case only for P2P_GO */
-	if (vif->type == NL80211_IFTYPE_AP && vif->p2p)
-		ath12k_mac_add_p2p_noa_ie(ar, vif, skb, is_prb_rsp);
-
-	/* Checking if it is a DVLAN frame */
-	if (!test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ar->ab->dev_flags) &&
-	    !(skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) &&
-	    !(skb_cb->flags & ATH12K_SKB_CIPHER_SET) &&
-	    ieee80211_has_protected(hdr->frame_control))
-		is_dvlan = true;
-
-	if (!vif->valid_links || !is_mcast || is_dvlan ||
-	    (skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) ||
-	    test_bit(ATH12K_FLAG_RAW_MODE, &ar->ab->dev_flags)) {
-		ret = ath12k_dp_tx(ar, arvif, skb, false, 0, is_mcast);
-		if (unlikely(ret)) {
-			ath12k_warn(ar->ab, "failed to transmit frame %d\n", ret);
-			ieee80211_free_txskb(ar->ah->hw, skb);
-			return;
-		}
-	} else {
-		mcbc_gsn = atomic_inc_return(&ahvif->mcbc_gsn) & 0xfff;
-
-		links_map = ahvif->links_map;
-		for_each_set_bit(link_id, &links_map,
-				 IEEE80211_MLD_MAX_NUM_LINKS) {
-			tmp_arvif = rcu_dereference(ahvif->link[link_id]);
-			if (!tmp_arvif || !tmp_arvif->is_up)
-				continue;
-
-			tmp_ar = tmp_arvif->ar;
-			msdu_copied = skb_copy(skb, GFP_ATOMIC);
-			if (!msdu_copied) {
-				ath12k_err(ar->ab,
-					   "skb copy failure link_id 0x%X vdevid 0x%X\n",
-					   link_id, tmp_arvif->vdev_id);
-				continue;
-			}
-
-			ath12k_mlo_mcast_update_tx_link_address(vif, link_id,
-								msdu_copied,
-								info_flags);
-
-			skb_cb = ATH12K_SKB_CB(msdu_copied);
-			skb_cb->link_id = link_id;
-
-			/* For open mode, skip peer find logic */
-			if (unlikely(!ahvif->key_cipher))
-				goto skip_peer_find;
-
-			spin_lock_bh(&tmp_ar->ab->base_lock);
-			peer = ath12k_peer_find_by_addr(tmp_ar->ab, tmp_arvif->bssid);
-			if (!peer) {
-				spin_unlock_bh(&tmp_ar->ab->base_lock);
-				ath12k_warn(tmp_ar->ab,
-					    "failed to find peer for vdev_id 0x%X addr %pM link_map 0x%X\n",
-					    tmp_arvif->vdev_id, tmp_arvif->bssid,
-					    ahvif->links_map);
-				dev_kfree_skb_any(msdu_copied);
-				continue;
-			}
-
-			key = peer->keys[peer->mcast_keyidx];
-			if (key) {
-				skb_cb->cipher = key->cipher;
-				skb_cb->flags |= ATH12K_SKB_CIPHER_SET;
-
-				hdr = (struct ieee80211_hdr *)msdu_copied->data;
-				if (!ieee80211_has_protected(hdr->frame_control))
-					hdr->frame_control |=
-						cpu_to_le16(IEEE80211_FCTL_PROTECTED);
-			}
-			spin_unlock_bh(&tmp_ar->ab->base_lock);
-
-skip_peer_find:
-			ret = ath12k_dp_tx(tmp_ar, tmp_arvif,
-					   msdu_copied, true, mcbc_gsn, is_mcast);
-			if (unlikely(ret)) {
-				if (ret == -ENOMEM) {
-					/* Drops are expected during heavy multicast
-					 * frame flood. Print with debug log
-					 * level to avoid lot of console prints
-					 */
-					ath12k_dbg(ar->ab, ATH12K_DBG_MAC,
-						   "failed to transmit frame %d\n",
-						   ret);
-				} else {
-					ath12k_warn(ar->ab,
-						    "failed to transmit frame %d\n",
-						    ret);
-				}
-
-				dev_kfree_skb_any(msdu_copied);
-			}
-		}
-		ieee80211_free_txskb(ar->ah->hw, skb);
-	}
-}
+EXPORT_SYMBOL(ath12k_mac_get_tx_link);
 
 void ath12k_mac_drain_tx(struct ath12k *ar)
 {
@@ -9480,7 +9505,7 @@ static void ath12k_drain_tx(struct ath12k_hw *ah)
 		ath12k_mac_drain_tx(ar);
 }
 
-static int ath12k_mac_op_start(struct ieee80211_hw *hw)
+int ath12k_mac_op_start(struct ieee80211_hw *hw)
 {
 	struct ath12k_hw *ah = ath12k_hw_to_ah(hw);
 	struct ath12k *ar;
@@ -9533,6 +9558,7 @@ static int ath12k_mac_op_start(struct ieee80211_hw *hw)
 
 	return ret;
 }
+EXPORT_SYMBOL(ath12k_mac_op_start);
 
 int ath12k_mac_rfkill_config(struct ath12k *ar)
 {
@@ -9593,6 +9619,7 @@ int ath12k_mac_rfkill_enable_radio(struct ath12k *ar, bool enable)
 
 static void ath12k_mac_stop(struct ath12k *ar)
 {
+	struct ath12k_pdev_dp *dp_pdev = &ar->dp;
 	struct ath12k_hw *ah = ar->ah;
 	struct htt_ppdu_stats_info *ppdu_stats, *tmp;
 	struct ath12k_wmi_scan_chan_list_arg *arg;
@@ -9617,13 +9644,14 @@ static void ath12k_mac_stop(struct ath12k *ar)
 	ar->state_11d = ATH12K_11D_IDLE;
 	complete(&ar->completed_11d_scan);
 
-	spin_lock_bh(&ar->data_lock);
-
-	list_for_each_entry_safe(ppdu_stats, tmp, &ar->ppdu_stats_info, list) {
+	spin_lock_bh(&dp_pdev->ppdu_list_lock);
+	list_for_each_entry_safe(ppdu_stats, tmp, &dp_pdev->ppdu_stats_info, list) {
 		list_del(&ppdu_stats->list);
 		kfree(ppdu_stats);
 	}
+	spin_unlock_bh(&dp_pdev->ppdu_list_lock);
 
+	spin_lock_bh(&ar->data_lock);
 	while ((arg = list_first_entry_or_null(&ar->regd_channel_update_queue,
 					       struct ath12k_wmi_scan_chan_list_arg,
 					       list))) {
@@ -9639,7 +9667,7 @@ static void ath12k_mac_stop(struct ath12k *ar)
 	atomic_set(&ar->num_pending_mgmt_tx, 0);
 }
 
-static void ath12k_mac_op_stop(struct ieee80211_hw *hw, bool suspend)
+void ath12k_mac_op_stop(struct ieee80211_hw *hw, bool suspend)
 {
 	struct ath12k_hw *ah = ath12k_hw_to_ah(hw);
 	struct ath12k *ar;
@@ -9658,6 +9686,7 @@ static void ath12k_mac_op_stop(struct ieee80211_hw *hw, bool suspend)
 
 	mutex_unlock(&ah->hw_mutex);
 }
+EXPORT_SYMBOL(ath12k_mac_op_stop);
 
 static u8
 ath12k_mac_get_vdev_stats_id(struct ath12k_link_vif *arvif)
@@ -9791,14 +9820,14 @@ static void ath12k_mac_update_vif_offload(struct ath12k_link_vif *arvif)
 					IEEE80211_OFFLOAD_DECAP_ENABLED);
 
 	if (vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED)
-		ahvif->tx_encap_type = ATH12K_HW_TXRX_ETHERNET;
+		ahvif->dp_vif.tx_encap_type = ATH12K_HW_TXRX_ETHERNET;
 	else if (test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags))
-		ahvif->tx_encap_type = ATH12K_HW_TXRX_RAW;
+		ahvif->dp_vif.tx_encap_type = ATH12K_HW_TXRX_RAW;
 	else
-		ahvif->tx_encap_type = ATH12K_HW_TXRX_NATIVE_WIFI;
+		ahvif->dp_vif.tx_encap_type = ATH12K_HW_TXRX_NATIVE_WIFI;
 
 	ret = ath12k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id,
-					    param_id, ahvif->tx_encap_type);
+					    param_id, ahvif->dp_vif.tx_encap_type);
 	if (ret) {
 		ath12k_warn(ab, "failed to set vdev %d tx encap mode: %d\n",
 			    arvif->vdev_id, ret);
@@ -9822,8 +9851,8 @@ static void ath12k_mac_update_vif_offload(struct ath12k_link_vif *arvif)
 	}
 }
 
-static void ath12k_mac_op_update_vif_offload(struct ieee80211_hw *hw,
-					     struct ieee80211_vif *vif)
+void ath12k_mac_op_update_vif_offload(struct ieee80211_hw *hw,
+				      struct ieee80211_vif *vif)
 {
 	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
 	struct ath12k_link_vif *arvif;
@@ -9847,6 +9876,7 @@ static void ath12k_mac_op_update_vif_offload(struct ieee80211_hw *hw,
 
 	ath12k_mac_update_vif_offload(&ahvif->deflink);
 }
+EXPORT_SYMBOL(ath12k_mac_op_update_vif_offload);
 
 static bool ath12k_mac_vif_ap_active_any(struct ath12k_base *ab)
 {
@@ -10012,6 +10042,9 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif)
 	int i;
 	int ret, vdev_id;
 	u8 link_id;
+	struct ath12k_dp_link_vif *dp_link_vif = NULL;
+	struct ath12k_dp_peer_create_params params = {};
+	bool dp_peer_created = false;
 
 	lockdep_assert_wiphy(hw->wiphy);
 
@@ -10093,8 +10126,26 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif)
 		goto err_vdev_del;
 	}
 
+	dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, arvif->link_id);
+
+	dp_link_vif->vdev_id = arvif->vdev_id;
+	dp_link_vif->lmac_id = ar->lmac_id;
+	dp_link_vif->pdev_idx = ar->pdev_idx;
+
 	switch (ahvif->vdev_type) {
 	case WMI_VDEV_TYPE_AP:
+		params.ucast_ra_only = true;
+
+		if (arvif->link_id < IEEE80211_MLD_MAX_NUM_LINKS) {
+			ret = ath12k_dp_peer_create(&ah->dp_hw, arvif->bssid, &params);
+			if (ret) {
+				ath12k_warn(ab, "failed to vdev %d create dp_peer for AP: %d\n",
+					    arvif->vdev_id, ret);
+				goto err_vdev_del;
+			}
+			dp_peer_created = true;
+		}
+
 		peer_param.vdev_id = arvif->vdev_id;
 		peer_param.peer_addr = arvif->bssid;
 		peer_param.peer_type = WMI_PEER_TYPE_DEFAULT;
@@ -10102,7 +10153,7 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif)
 		if (ret) {
 			ath12k_warn(ab, "failed to vdev %d create peer for AP: %d\n",
 				    arvif->vdev_id, ret);
-			goto err_vdev_del;
+			goto err_dp_peer_del;
 		}
 
 		ret = ath12k_mac_set_kickout(arvif);
@@ -10208,6 +10259,10 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif)
 		ar->num_peers--;
 	}
 
+err_dp_peer_del:
+	if (dp_peer_created)
+		ath12k_dp_peer_delete(&ah->dp_hw, arvif->bssid, NULL);
+
 err_vdev_del:
 	if (ahvif->vdev_type == WMI_VDEV_TYPE_MONITOR) {
 		ar->monitor_vdev_id = -1;
@@ -10402,8 +10457,8 @@ static struct ath12k *ath12k_mac_assign_vif_to_vdev(struct ieee80211_hw *hw,
 	return arvif->ar;
 }
 
-static int ath12k_mac_op_add_interface(struct ieee80211_hw *hw,
-				       struct ieee80211_vif *vif)
+int ath12k_mac_op_add_interface(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif)
 {
 	struct ath12k_hw *ah = ath12k_hw_to_ah(hw);
 	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
@@ -10450,6 +10505,7 @@ static int ath12k_mac_op_add_interface(struct ieee80211_hw *hw,
 	 */
 	return 0;
 }
+EXPORT_SYMBOL(ath12k_mac_op_add_interface);
 
 static void ath12k_mac_vif_unref(struct ath12k_dp *dp, struct ieee80211_vif *vif)
 {
@@ -10480,6 +10536,7 @@ static int ath12k_mac_vdev_delete(struct ath12k *ar, struct ath12k_link_vif *arv
 {
 	struct ath12k_vif *ahvif = arvif->ahvif;
 	struct ieee80211_vif *vif = ath12k_ahvif_to_vif(ahvif);
+	struct ath12k_dp_link_vif *dp_link_vif;
 	struct ath12k_base *ab = ar->ab;
 	unsigned long time_left;
 	int ret;
@@ -10525,8 +10582,10 @@ static int ath12k_mac_vdev_delete(struct ath12k *ar, struct ath12k_link_vif *arv
 	idr_for_each(&ar->txmgmt_idr,
 		     ath12k_mac_vif_txmgmt_idr_remove, vif);
 
-	ath12k_mac_vif_unref(&ab->dp, vif);
-	ath12k_dp_tx_put_bank_profile(&ab->dp, arvif->bank_id);
+	ath12k_mac_vif_unref(ath12k_ab_to_dp(ab), vif);
+
+	dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, arvif->link_id);
+	ath12k_dp_tx_put_bank_profile(ath12k_ab_to_dp(ab), dp_link_vif->bank_id);
 
 	/* Recalc txpower for remaining vdev */
 	ath12k_mac_txpower_recalc(ar);
@@ -10538,8 +10597,8 @@ static int ath12k_mac_vdev_delete(struct ath12k *ar, struct ath12k_link_vif *arv
 	return ret;
 }
 
-static void ath12k_mac_op_remove_interface(struct ieee80211_hw *hw,
-					   struct ieee80211_vif *vif)
+void ath12k_mac_op_remove_interface(struct ieee80211_hw *hw,
+				    struct ieee80211_vif *vif)
 {
 	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
 	struct ath12k_link_vif *arvif;
@@ -10586,6 +10645,7 @@ static void ath12k_mac_op_remove_interface(struct ieee80211_hw *hw,
 		ath12k_mac_unassign_link_vif(arvif);
 	}
 }
+EXPORT_SYMBOL(ath12k_mac_op_remove_interface);
 
 /* FIXME: Has to be verified. */
 #define SUPPORTED_FILTERS			\
@@ -10597,10 +10657,10 @@ static void ath12k_mac_op_remove_interface(struct ieee80211_hw *hw,
 	FIF_PROBE_REQ |				\
 	FIF_FCSFAIL)
 
-static void ath12k_mac_op_configure_filter(struct ieee80211_hw *hw,
-					   unsigned int changed_flags,
-					   unsigned int *total_flags,
-					   u64 multicast)
+void ath12k_mac_op_configure_filter(struct ieee80211_hw *hw,
+				    unsigned int changed_flags,
+				    unsigned int *total_flags,
+				    u64 multicast)
 {
 	struct ath12k_hw *ah = ath12k_hw_to_ah(hw);
 	struct ath12k *ar;
@@ -10612,9 +10672,10 @@ static void ath12k_mac_op_configure_filter(struct ieee80211_hw *hw,
 	*total_flags &= SUPPORTED_FILTERS;
 	ar->filter_flags = *total_flags;
 }
+EXPORT_SYMBOL(ath12k_mac_op_configure_filter);
 
-static int ath12k_mac_op_get_antenna(struct ieee80211_hw *hw, int radio_idx,
-				     u32 *tx_ant, u32 *rx_ant)
+int ath12k_mac_op_get_antenna(struct ieee80211_hw *hw, int radio_idx,
+			      u32 *tx_ant, u32 *rx_ant)
 {
 	struct ath12k_hw *ah = ath12k_hw_to_ah(hw);
 	int antennas_rx = 0, antennas_tx = 0;
@@ -10633,9 +10694,10 @@ static int ath12k_mac_op_get_antenna(struct ieee80211_hw *hw, int radio_idx,
 
 	return 0;
 }
+EXPORT_SYMBOL(ath12k_mac_op_get_antenna);
 
-static int ath12k_mac_op_set_antenna(struct ieee80211_hw *hw, int radio_idx,
-				     u32 tx_ant, u32 rx_ant)
+int ath12k_mac_op_set_antenna(struct ieee80211_hw *hw, int radio_idx,
+			      u32 tx_ant, u32 rx_ant)
 {
 	struct ath12k_hw *ah = ath12k_hw_to_ah(hw);
 	struct ath12k *ar;
@@ -10652,6 +10714,7 @@ static int ath12k_mac_op_set_antenna(struct ieee80211_hw *hw, int radio_idx,
 
 	return ret;
 }
+EXPORT_SYMBOL(ath12k_mac_op_set_antenna);
 
 static int ath12k_mac_ampdu_action(struct ieee80211_hw *hw,
 				   struct ieee80211_vif *vif,
@@ -10693,9 +10756,9 @@ static int ath12k_mac_ampdu_action(struct ieee80211_hw *hw,
 	return ret;
 }
 
-static int ath12k_mac_op_ampdu_action(struct ieee80211_hw *hw,
-				      struct ieee80211_vif *vif,
-				      struct ieee80211_ampdu_params *params)
+int ath12k_mac_op_ampdu_action(struct ieee80211_hw *hw,
+			       struct ieee80211_vif *vif,
+			       struct ieee80211_ampdu_params *params)
 {
 	struct ieee80211_sta *sta = params->sta;
 	struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(sta);
@@ -10716,9 +10779,10 @@ static int ath12k_mac_op_ampdu_action(struct ieee80211_hw *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL(ath12k_mac_op_ampdu_action);
 
-static int ath12k_mac_op_add_chanctx(struct ieee80211_hw *hw,
-				     struct ieee80211_chanctx_conf *ctx)
+int ath12k_mac_op_add_chanctx(struct ieee80211_hw *hw,
+			      struct ieee80211_chanctx_conf *ctx)
 {
 	struct ath12k *ar;
 	struct ath12k_base *ab;
@@ -10745,9 +10809,10 @@ static int ath12k_mac_op_add_chanctx(struct ieee80211_hw *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL(ath12k_mac_op_add_chanctx);
 
-static void ath12k_mac_op_remove_chanctx(struct ieee80211_hw *hw,
-					 struct ieee80211_chanctx_conf *ctx)
+void ath12k_mac_op_remove_chanctx(struct ieee80211_hw *hw,
+				  struct ieee80211_chanctx_conf *ctx)
 {
 	struct ath12k *ar;
 	struct ath12k_base *ab;
@@ -10772,6 +10837,7 @@ static void ath12k_mac_op_remove_chanctx(struct ieee80211_hw *hw,
 	spin_unlock_bh(&ar->data_lock);
 	ar->chan_tx_pwr = ATH12K_PDEV_TX_POWER_INVALID;
 }
+EXPORT_SYMBOL(ath12k_mac_op_remove_chanctx);
 
 static enum wmi_phy_mode
 ath12k_mac_check_down_grade_phy_mode(struct ath12k *ar,
@@ -11330,9 +11396,9 @@ ath12k_mac_update_active_vif_chan(struct ath12k *ar,
 	kfree(arg.vifs);
 }
 
-static void ath12k_mac_op_change_chanctx(struct ieee80211_hw *hw,
-					 struct ieee80211_chanctx_conf *ctx,
-					 u32 changed)
+void ath12k_mac_op_change_chanctx(struct ieee80211_hw *hw,
+				  struct ieee80211_chanctx_conf *ctx,
+				  u32 changed)
 {
 	struct ath12k *ar;
 	struct ath12k_base *ab;
@@ -11362,6 +11428,7 @@ static void ath12k_mac_op_change_chanctx(struct ieee80211_hw *hw,
 
 	/* TODO: Recalc radar detection */
 }
+EXPORT_SYMBOL(ath12k_mac_op_change_chanctx);
 
 static int ath12k_start_vdev_delay(struct ath12k *ar,
 				   struct ath12k_link_vif *arvif)
@@ -11803,7 +11870,7 @@ static void ath12k_mac_parse_tx_pwr_env(struct ath12k *ar,
 	}
 }
 
-static int
+int
 ath12k_mac_op_assign_vif_chanctx(struct ieee80211_hw *hw,
 				 struct ieee80211_vif *vif,
 				 struct ieee80211_bss_conf *link_conf,
@@ -11852,7 +11919,7 @@ ath12k_mac_op_assign_vif_chanctx(struct ieee80211_hw *hw,
 	if (ab->hw_params->vdev_start_delay &&
 	    ahvif->vdev_type != WMI_VDEV_TYPE_AP &&
 	    ahvif->vdev_type != WMI_VDEV_TYPE_MONITOR &&
-	    !ath12k_peer_exist_by_vdev_id(ab, arvif->vdev_id)) {
+	    !ath12k_dp_link_peer_exist_by_vdev_id(ath12k_ab_to_dp(ab), arvif->vdev_id)) {
 		ret = 0;
 		goto out;
 	}
@@ -11888,8 +11955,9 @@ ath12k_mac_op_assign_vif_chanctx(struct ieee80211_hw *hw,
 out:
 	return ret;
 }
+EXPORT_SYMBOL(ath12k_mac_op_assign_vif_chanctx);
 
-static void
+void
 ath12k_mac_op_unassign_vif_chanctx(struct ieee80211_hw *hw,
 				   struct ieee80211_vif *vif,
 				   struct ieee80211_bss_conf *link_conf,
@@ -11956,8 +12024,9 @@ ath12k_mac_op_unassign_vif_chanctx(struct ieee80211_hw *hw,
 		ar->scan.arvif = NULL;
 	}
 }
+EXPORT_SYMBOL(ath12k_mac_op_unassign_vif_chanctx);
 
-static int
+int
 ath12k_mac_op_switch_vif_chanctx(struct ieee80211_hw *hw,
 				 struct ieee80211_vif_chanctx_switch *vifs,
 				 int n_vifs,
@@ -11982,6 +12051,7 @@ ath12k_mac_op_switch_vif_chanctx(struct ieee80211_hw *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL(ath12k_mac_op_switch_vif_chanctx);
 
 static int
 ath12k_set_vdev_param_to_all_vifs(struct ath12k *ar, int param, u32 value)
@@ -12010,8 +12080,8 @@ ath12k_set_vdev_param_to_all_vifs(struct ath12k *ar, int param, u32 value)
 /* mac80211 stores device specific RTS/Fragmentation threshold value,
  * this is set interface specific to firmware from ath12k driver
  */
-static int ath12k_mac_op_set_rts_threshold(struct ieee80211_hw *hw,
-					   int radio_idx, u32 value)
+int ath12k_mac_op_set_rts_threshold(struct ieee80211_hw *hw,
+				    int radio_idx, u32 value)
 {
 	struct ath12k_hw *ah = ath12k_hw_to_ah(hw);
 	struct wiphy *wiphy = hw->wiphy;
@@ -12070,9 +12140,10 @@ static int ath12k_mac_op_set_rts_threshold(struct ieee80211_hw *hw,
 
 	return ret;
 }
+EXPORT_SYMBOL(ath12k_mac_op_set_rts_threshold);
 
-static int ath12k_mac_op_set_frag_threshold(struct ieee80211_hw *hw,
-					    int radio_idx, u32 value)
+int ath12k_mac_op_set_frag_threshold(struct ieee80211_hw *hw,
+				     int radio_idx, u32 value)
 {
 	/* Even though there's a WMI vdev param for fragmentation threshold no
 	 * known firmware actually implements it. Moreover it is not possible to
@@ -12089,6 +12160,7 @@ static int ath12k_mac_op_set_frag_threshold(struct ieee80211_hw *hw,
 
 	return -EOPNOTSUPP;
 }
+EXPORT_SYMBOL(ath12k_mac_op_set_frag_threshold);
 
 static int ath12k_mac_flush(struct ath12k *ar)
 {
@@ -12126,8 +12198,8 @@ int ath12k_mac_wait_tx_complete(struct ath12k *ar)
 	return ath12k_mac_flush(ar);
 }
 
-static void ath12k_mac_op_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
-				u32 queues, bool drop)
+void ath12k_mac_op_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			 u32 queues, bool drop)
 {
 	struct ath12k_hw *ah = ath12k_hw_to_ah(hw);
 	struct ath12k_link_vif *arvif;
@@ -12162,6 +12234,7 @@ static void ath12k_mac_op_flush(struct ieee80211_hw *hw, struct ieee80211_vif *v
 		ath12k_mac_flush(arvif->ar);
 	}
 }
+EXPORT_SYMBOL(ath12k_mac_op_flush);
 
 static int
 ath12k_mac_bitrate_mask_num_ht_rates(struct ath12k *ar,
@@ -12725,9 +12798,11 @@ ath12k_mac_validate_fixed_rate_settings(struct ath12k *ar, enum nl80211_band ban
 	bool eht_fixed_rate = false, he_fixed_rate = false, vht_fixed_rate = false;
 	const u16 *vht_mcs_mask, *he_mcs_mask, *eht_mcs_mask;
 	struct ieee80211_link_sta *link_sta;
-	struct ath12k_peer *peer, *tmp;
+	struct ath12k_dp_link_peer *peer, *tmp;
 	u8 vht_nss, he_nss, eht_nss;
 	int ret = true;
+	struct ath12k_base *ab = ar->ab;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 
 	vht_mcs_mask = mask->control[band].vht_mcs;
 	he_mcs_mask = mask->control[band].he_mcs;
@@ -12750,8 +12825,8 @@ ath12k_mac_validate_fixed_rate_settings(struct ath12k *ar, enum nl80211_band ban
 	eht_nss = ath12k_mac_max_eht_nss(eht_mcs_mask);
 
 	rcu_read_lock();
-	spin_lock_bh(&ar->ab->base_lock);
-	list_for_each_entry_safe(peer, tmp, &ar->ab->peers, list) {
+	spin_lock_bh(&dp->dp_lock);
+	list_for_each_entry_safe(peer, tmp, &dp->peers, list) {
 		if (peer->sta) {
 			link_sta = rcu_dereference(peer->sta->link[link_id]);
 			if (!link_sta) {
@@ -12777,12 +12852,12 @@ ath12k_mac_validate_fixed_rate_settings(struct ath12k *ar, enum nl80211_band ban
 		}
 	}
 exit:
-	spin_unlock_bh(&ar->ab->base_lock);
+	spin_unlock_bh(&dp->dp_lock);
 	rcu_read_unlock();
 	return ret;
 }
 
-static int
+int
 ath12k_mac_op_set_bitrate_mask(struct ieee80211_hw *hw,
 			       struct ieee80211_vif *vif,
 			       const struct cfg80211_bitrate_mask *mask)
@@ -12961,8 +13036,9 @@ ath12k_mac_op_set_bitrate_mask(struct ieee80211_hw *hw,
 out:
 	return ret;
 }
+EXPORT_SYMBOL(ath12k_mac_op_set_bitrate_mask);
 
-static void
+void
 ath12k_mac_op_reconfig_complete(struct ieee80211_hw *hw,
 				enum ieee80211_reconfig_type reconfig_type)
 {
@@ -13023,7 +13099,7 @@ ath12k_mac_op_reconfig_complete(struct ieee80211_hw *hw,
 			ahvif = arvif->ahvif;
 			ath12k_dbg(ab, ATH12K_DBG_BOOT,
 				   "reconfig cipher %d up %d vdev type %d\n",
-				   ahvif->key_cipher,
+				   ahvif->dp_vif.key_cipher,
 				   arvif->is_up,
 				   ahvif->vdev_type);
 
@@ -13043,6 +13119,7 @@ ath12k_mac_op_reconfig_complete(struct ieee80211_hw *hw,
 		}
 	}
 }
+EXPORT_SYMBOL(ath12k_mac_op_reconfig_complete);
 
 static void
 ath12k_mac_update_bss_chan_survey(struct ath12k *ar,
@@ -13076,8 +13153,8 @@ ath12k_mac_update_bss_chan_survey(struct ath12k *ar,
 		ath12k_warn(ar->ab, "bss channel survey timed out\n");
 }
 
-static int ath12k_mac_op_get_survey(struct ieee80211_hw *hw, int idx,
-				    struct survey_info *survey)
+int ath12k_mac_op_get_survey(struct ieee80211_hw *hw, int idx,
+			     struct survey_info *survey)
 {
 	struct ath12k *ar;
 	struct ieee80211_supported_band *sband;
@@ -13131,6 +13208,7 @@ static int ath12k_mac_op_get_survey(struct ieee80211_hw *hw, int idx,
 
 	return 0;
 }
+EXPORT_SYMBOL(ath12k_mac_op_get_survey);
 
 static void ath12k_mac_put_chain_rssi(struct station_info *sinfo,
 				      struct ath12k_link_sta *arsta)
@@ -13153,15 +13231,18 @@ static void ath12k_mac_put_chain_rssi(struct station_info *sinfo,
 	}
 }
 
-static void ath12k_mac_op_sta_statistics(struct ieee80211_hw *hw,
-					 struct ieee80211_vif *vif,
-					 struct ieee80211_sta *sta,
-					 struct station_info *sinfo)
+void ath12k_mac_op_sta_statistics(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_sta *sta,
+				  struct station_info *sinfo)
 {
 	struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(sta);
+	struct ath12k_dp_link_peer_rate_info rate_info = {};
 	struct ath12k_fw_stats_req_params params = {};
+	struct ath12k_dp_link_peer *peer;
 	struct ath12k_link_sta *arsta;
 	s8 signal, noise_floor;
+	struct ath12k_dp *dp;
 	struct ath12k *ar;
 	bool db2dbm;
 
@@ -13172,34 +13253,37 @@ static void ath12k_mac_op_sta_statistics(struct ieee80211_hw *hw,
 	if (!ar)
 		return;
 
+	dp = ath12k_ab_to_dp(ar->ab);
+	ath12k_dp_link_peer_get_sta_rate_info_stats(dp, arsta->addr, &rate_info);
+
 	db2dbm = test_bit(WMI_TLV_SERVICE_HW_DB2DBM_CONVERSION_SUPPORT,
 			  ar->ab->wmi_ab.svc_map);
 
-	sinfo->rx_duration = arsta->rx_duration;
+	sinfo->rx_duration = rate_info.rx_duration;
 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_DURATION);
 
-	sinfo->tx_duration = arsta->tx_duration;
+	sinfo->tx_duration = rate_info.tx_duration;
 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_DURATION);
 
-	if (arsta->txrate.legacy || arsta->txrate.nss) {
-		if (arsta->txrate.legacy) {
-			sinfo->txrate.legacy = arsta->txrate.legacy;
+	if (rate_info.txrate.legacy || rate_info.txrate.nss) {
+		if (rate_info.txrate.legacy) {
+			sinfo->txrate.legacy = rate_info.txrate.legacy;
 		} else {
-			sinfo->txrate.mcs = arsta->txrate.mcs;
-			sinfo->txrate.nss = arsta->txrate.nss;
-			sinfo->txrate.bw = arsta->txrate.bw;
-			sinfo->txrate.he_gi = arsta->txrate.he_gi;
-			sinfo->txrate.he_dcm = arsta->txrate.he_dcm;
-			sinfo->txrate.he_ru_alloc = arsta->txrate.he_ru_alloc;
-			sinfo->txrate.eht_gi = arsta->txrate.eht_gi;
-			sinfo->txrate.eht_ru_alloc = arsta->txrate.eht_ru_alloc;
-		}
-		sinfo->txrate.flags = arsta->txrate.flags;
+			sinfo->txrate.mcs = rate_info.txrate.mcs;
+			sinfo->txrate.nss = rate_info.txrate.nss;
+			sinfo->txrate.bw = rate_info.txrate.bw;
+			sinfo->txrate.he_gi = rate_info.txrate.he_gi;
+			sinfo->txrate.he_dcm = rate_info.txrate.he_dcm;
+			sinfo->txrate.he_ru_alloc = rate_info.txrate.he_ru_alloc;
+			sinfo->txrate.eht_gi = rate_info.txrate.eht_gi;
+			sinfo->txrate.eht_ru_alloc = rate_info.txrate.eht_ru_alloc;
+		}
+		sinfo->txrate.flags = rate_info.txrate.flags;
 		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE);
 	}
 
 	/* TODO: Use real NF instead of default one. */
-	signal = arsta->rssi_comb;
+	signal = rate_info.rssi_comb;
 
 	params.pdev_id = ar->pdev->pdev_id;
 	params.vdev_id = 0;
@@ -13229,26 +13313,37 @@ static void ath12k_mac_op_sta_statistics(struct ieee80211_hw *hw,
 		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL);
 	}
 
-	sinfo->signal_avg = ewma_avg_rssi_read(&arsta->avg_rssi);
+	sinfo->signal_avg = rate_info.signal_avg;
 
 	if (!db2dbm)
 		sinfo->signal_avg += noise_floor;
 
 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL_AVG);
 
-	sinfo->tx_retries = arsta->tx_retry_count;
-	sinfo->tx_failed = arsta->tx_retry_failed;
+	spin_lock_bh(&dp->dp_lock);
+	peer = ath12k_dp_link_peer_find_by_addr(dp, arsta->addr);
+	if (!peer) {
+		spin_unlock_bh(&dp->dp_lock);
+		return;
+	}
+
+	sinfo->tx_retries = peer->tx_retry_count;
+	sinfo->tx_failed = peer->tx_retry_failed;
 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_RETRIES);
 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED);
+
+	spin_unlock_bh(&dp->dp_lock);
 }
+EXPORT_SYMBOL(ath12k_mac_op_sta_statistics);
 
-static void ath12k_mac_op_link_sta_statistics(struct ieee80211_hw *hw,
-					      struct ieee80211_vif *vif,
-					      struct ieee80211_link_sta *link_sta,
-					      struct link_station_info *link_sinfo)
+void ath12k_mac_op_link_sta_statistics(struct ieee80211_hw *hw,
+				       struct ieee80211_vif *vif,
+				       struct ieee80211_link_sta *link_sta,
+				       struct link_station_info *link_sinfo)
 {
 	struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(link_sta->sta);
 	struct ath12k_fw_stats_req_params params = {};
+	struct ath12k_dp_link_peer *peer;
 	struct ath12k_link_sta *arsta;
 	struct ath12k *ar;
 	s8 signal;
@@ -13268,43 +13363,64 @@ static void ath12k_mac_op_link_sta_statistics(struct ieee80211_hw *hw,
 	db2dbm = test_bit(WMI_TLV_SERVICE_HW_DB2DBM_CONVERSION_SUPPORT,
 			  ar->ab->wmi_ab.svc_map);
 
-	link_sinfo->rx_duration = arsta->rx_duration;
+	spin_lock_bh(&ar->ab->dp->dp_lock);
+	peer = ath12k_dp_link_peer_find_by_addr(ar->ab->dp, arsta->addr);
+	if (!peer) {
+		spin_unlock_bh(&ar->ab->dp->dp_lock);
+		return;
+	}
+
+	link_sinfo->rx_duration = peer->rx_duration;
 	link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_DURATION);
 
-	link_sinfo->tx_duration = arsta->tx_duration;
+	link_sinfo->tx_duration = peer->tx_duration;
 	link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_DURATION);
 
-	if (arsta->txrate.legacy || arsta->txrate.nss) {
-		if (arsta->txrate.legacy) {
-			link_sinfo->txrate.legacy = arsta->txrate.legacy;
+	if (peer->txrate.legacy || peer->txrate.nss) {
+		if (peer->txrate.legacy) {
+			link_sinfo->txrate.legacy = peer->txrate.legacy;
 		} else {
-			link_sinfo->txrate.mcs = arsta->txrate.mcs;
-			link_sinfo->txrate.nss = arsta->txrate.nss;
-			link_sinfo->txrate.bw = arsta->txrate.bw;
-			link_sinfo->txrate.he_gi = arsta->txrate.he_gi;
-			link_sinfo->txrate.he_dcm = arsta->txrate.he_dcm;
+			link_sinfo->txrate.mcs = peer->txrate.mcs;
+			link_sinfo->txrate.nss = peer->txrate.nss;
+			link_sinfo->txrate.bw = peer->txrate.bw;
+			link_sinfo->txrate.he_gi = peer->txrate.he_gi;
+			link_sinfo->txrate.he_dcm = peer->txrate.he_dcm;
 			link_sinfo->txrate.he_ru_alloc =
-				arsta->txrate.he_ru_alloc;
-			link_sinfo->txrate.eht_gi = arsta->txrate.eht_gi;
+				peer->txrate.he_ru_alloc;
+			link_sinfo->txrate.eht_gi = peer->txrate.eht_gi;
 			link_sinfo->txrate.eht_ru_alloc =
-				arsta->txrate.eht_ru_alloc;
+				peer->txrate.eht_ru_alloc;
 		}
-		link_sinfo->txrate.flags = arsta->txrate.flags;
+		link_sinfo->txrate.flags = peer->txrate.flags;
 		link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE);
 	}
 
+	link_sinfo->signal_avg = ewma_avg_rssi_read(&peer->avg_rssi);
+
+	if (!db2dbm)
+		link_sinfo->signal_avg += ATH12K_DEFAULT_NOISE_FLOOR;
+
+	link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL_AVG);
+
+	link_sinfo->tx_retries = peer->tx_retry_count;
+	link_sinfo->tx_failed = peer->tx_retry_failed;
+	link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_RETRIES);
+	link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED);
+
 	/* TODO: Use real NF instead of default one. */
-	signal = arsta->rssi_comb;
+	signal = peer->rssi_comb;
 
-	params.pdev_id = ar->pdev->pdev_id;
-	params.vdev_id = 0;
-	params.stats_id = WMI_REQUEST_VDEV_STAT;
+	spin_unlock_bh(&ar->ab->dp->dp_lock);
 
-	if (!signal &&
-	    ahsta->ahvif->vdev_type == WMI_VDEV_TYPE_STA &&
-	    !(ath12k_mac_get_fw_stats(ar, &params))) {
-		signal = arsta->rssi_beacon;
-		ath12k_fw_stats_reset(ar);
+	if (!signal && ahsta->ahvif->vdev_type == WMI_VDEV_TYPE_STA) {
+		params.pdev_id = ar->pdev->pdev_id;
+		params.vdev_id = 0;
+		params.stats_id = WMI_REQUEST_VDEV_STAT;
+
+		if (!ath12k_mac_get_fw_stats(ar, &params)) {
+			signal = arsta->rssi_beacon;
+			ath12k_fw_stats_reset(ar);
+		}
 	}
 
 	if (signal) {
@@ -13312,22 +13428,11 @@ static void ath12k_mac_op_link_sta_statistics(struct ieee80211_hw *hw,
 			db2dbm ? signal : signal + ATH12K_DEFAULT_NOISE_FLOOR;
 		link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL);
 	}
-
-	link_sinfo->signal_avg = ewma_avg_rssi_read(&arsta->avg_rssi);
-
-	if (!db2dbm)
-		link_sinfo->signal_avg += ATH12K_DEFAULT_NOISE_FLOOR;
-
-	link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL_AVG);
-
-	link_sinfo->tx_retries = arsta->tx_retry_count;
-	link_sinfo->tx_failed = arsta->tx_retry_failed;
-	link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_RETRIES);
-	link_sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED);
 }
+EXPORT_SYMBOL(ath12k_mac_op_link_sta_statistics);
 
-static int ath12k_mac_op_cancel_remain_on_channel(struct ieee80211_hw *hw,
-						  struct ieee80211_vif *vif)
+int ath12k_mac_op_cancel_remain_on_channel(struct ieee80211_hw *hw,
+					   struct ieee80211_vif *vif)
 {
 	struct ath12k_hw *ah = ath12k_hw_to_ah(hw);
 	struct ath12k *ar;
@@ -13347,12 +13452,13 @@ static int ath12k_mac_op_cancel_remain_on_channel(struct ieee80211_hw *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL(ath12k_mac_op_cancel_remain_on_channel);
 
-static int ath12k_mac_op_remain_on_channel(struct ieee80211_hw *hw,
-					   struct ieee80211_vif *vif,
-					   struct ieee80211_channel *chan,
-					   int duration,
-					   enum ieee80211_roc_type type)
+int ath12k_mac_op_remain_on_channel(struct ieee80211_hw *hw,
+				    struct ieee80211_vif *vif,
+				    struct ieee80211_channel *chan,
+				    int duration,
+				    enum ieee80211_roc_type type)
 {
 	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
 	struct ath12k_hw *ah = ath12k_hw_to_ah(hw);
@@ -13487,10 +13593,11 @@ static int ath12k_mac_op_remain_on_channel(struct ieee80211_hw *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL(ath12k_mac_op_remain_on_channel);
 
-static void ath12k_mac_op_set_rekey_data(struct ieee80211_hw *hw,
-					 struct ieee80211_vif *vif,
-					 struct cfg80211_gtk_rekey_data *data)
+void ath12k_mac_op_set_rekey_data(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif,
+				  struct cfg80211_gtk_rekey_data *data)
 {
 	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
 	struct ath12k_rekey_data *rekey_data;
@@ -13523,63 +13630,7 @@ static void ath12k_mac_op_set_rekey_data(struct ieee80211_hw *hw,
 	ath12k_dbg_dump(ar->ab, ATH12K_DBG_MAC, "replay ctr", NULL,
 			&rekey_data->replay_ctr, sizeof(rekey_data->replay_ctr));
 }
-
-static const struct ieee80211_ops ath12k_ops = {
-	.tx				= ath12k_mac_op_tx,
-	.wake_tx_queue			= ieee80211_handle_wake_tx_queue,
-	.start                          = ath12k_mac_op_start,
-	.stop                           = ath12k_mac_op_stop,
-	.reconfig_complete              = ath12k_mac_op_reconfig_complete,
-	.add_interface                  = ath12k_mac_op_add_interface,
-	.remove_interface		= ath12k_mac_op_remove_interface,
-	.update_vif_offload		= ath12k_mac_op_update_vif_offload,
-	.config                         = ath12k_mac_op_config,
-	.link_info_changed              = ath12k_mac_op_link_info_changed,
-	.vif_cfg_changed		= ath12k_mac_op_vif_cfg_changed,
-	.change_vif_links               = ath12k_mac_op_change_vif_links,
-	.configure_filter		= ath12k_mac_op_configure_filter,
-	.hw_scan                        = ath12k_mac_op_hw_scan,
-	.cancel_hw_scan                 = ath12k_mac_op_cancel_hw_scan,
-	.set_key                        = ath12k_mac_op_set_key,
-	.set_rekey_data	                = ath12k_mac_op_set_rekey_data,
-	.sta_state                      = ath12k_mac_op_sta_state,
-	.sta_set_txpwr			= ath12k_mac_op_sta_set_txpwr,
-	.link_sta_rc_update		= ath12k_mac_op_link_sta_rc_update,
-	.conf_tx                        = ath12k_mac_op_conf_tx,
-	.set_antenna			= ath12k_mac_op_set_antenna,
-	.get_antenna			= ath12k_mac_op_get_antenna,
-	.ampdu_action			= ath12k_mac_op_ampdu_action,
-	.add_chanctx			= ath12k_mac_op_add_chanctx,
-	.remove_chanctx			= ath12k_mac_op_remove_chanctx,
-	.change_chanctx			= ath12k_mac_op_change_chanctx,
-	.assign_vif_chanctx		= ath12k_mac_op_assign_vif_chanctx,
-	.unassign_vif_chanctx		= ath12k_mac_op_unassign_vif_chanctx,
-	.switch_vif_chanctx		= ath12k_mac_op_switch_vif_chanctx,
-	.get_txpower			= ath12k_mac_op_get_txpower,
-	.set_rts_threshold		= ath12k_mac_op_set_rts_threshold,
-	.set_frag_threshold		= ath12k_mac_op_set_frag_threshold,
-	.set_bitrate_mask		= ath12k_mac_op_set_bitrate_mask,
-	.get_survey			= ath12k_mac_op_get_survey,
-	.flush				= ath12k_mac_op_flush,
-	.sta_statistics			= ath12k_mac_op_sta_statistics,
-	.link_sta_statistics		= ath12k_mac_op_link_sta_statistics,
-	.remain_on_channel              = ath12k_mac_op_remain_on_channel,
-	.cancel_remain_on_channel       = ath12k_mac_op_cancel_remain_on_channel,
-	.change_sta_links               = ath12k_mac_op_change_sta_links,
-	.can_activate_links             = ath12k_mac_op_can_activate_links,
-#ifdef CONFIG_PM
-	.suspend			= ath12k_wow_op_suspend,
-	.resume				= ath12k_wow_op_resume,
-	.set_wakeup			= ath12k_wow_op_set_wakeup,
-#endif
-#ifdef CONFIG_ATH12K_DEBUGFS
-	.vif_add_debugfs                = ath12k_debugfs_op_vif_add,
-#endif
-	CFG80211_TESTMODE_CMD(ath12k_tm_cmd)
-#ifdef CONFIG_ATH12K_DEBUGFS
-	.link_sta_add_debugfs           = ath12k_debugfs_link_sta_op_add,
-#endif
-};
+EXPORT_SYMBOL(ath12k_mac_op_set_rekey_data);
 
 void ath12k_mac_update_freq_range(struct ath12k *ar,
 				  u32 freq_low, u32 freq_high)
@@ -14558,8 +14609,9 @@ static void ath12k_mac_setup(struct ath12k *ar)
 	ar->vdev_id_11d_scan = ATH12K_11D_INVALID_VDEV_ID;
 
 	spin_lock_init(&ar->data_lock);
+	spin_lock_init(&ar->dp.ppdu_list_lock);
 	INIT_LIST_HEAD(&ar->arvifs);
-	INIT_LIST_HEAD(&ar->ppdu_stats_info);
+	INIT_LIST_HEAD(&ar->dp.ppdu_stats_info);
 
 	init_completion(&ar->vdev_setup_done);
 	init_completion(&ar->vdev_delete_done);
@@ -14798,7 +14850,7 @@ static struct ath12k_hw *ath12k_mac_hw_allocate(struct ath12k_hw_group *ag,
 	u8 pdev_idx;
 
 	hw = ieee80211_alloc_hw(struct_size(ah, radio, num_pdev_map),
-				&ath12k_ops);
+				pdev_map->ab->ath12k_ops);
 	if (!hw)
 		return NULL;
 
@@ -14807,7 +14859,9 @@ static struct ath12k_hw *ath12k_mac_hw_allocate(struct ath12k_hw_group *ag,
 	ah->num_radio = num_pdev_map;
 
 	mutex_init(&ah->hw_mutex);
-	INIT_LIST_HEAD(&ah->ml_peers);
+
+	spin_lock_init(&ah->dp_hw.peer_lock);
+	INIT_LIST_HEAD(&ah->dp_hw.dp_peers_list);
 
 	for (i = 0; i < num_pdev_map; i++) {
 		ab = pdev_map[i].ab;
diff --git a/drivers/net/wireless/ath/ath12k/mac.h b/drivers/net/wireless/ath/ath12k/mac.h
index 1f689e367c8a..422bd3b095cd 100644
--- a/drivers/net/wireless/ath/ath12k/mac.h
+++ b/drivers/net/wireless/ath/ath12k/mac.h
@@ -54,9 +54,6 @@ struct ath12k_generic_iter {
  * for driver usage purpose.
  */
 #define ATH12K_FIRST_SCAN_LINK	IEEE80211_MLD_MAX_NUM_LINKS
-#define ATH12K_SCAN_MAX_LINKS	ATH12K_GROUP_MAX_RADIO
-/* Define 1 scan link for each radio for parallel scan purposes */
-#define ATH12K_NUM_MAX_LINKS (IEEE80211_MLD_MAX_NUM_LINKS + ATH12K_SCAN_MAX_LINKS)
 #define ATH12K_SCAN_LINKS_MASK	GENMASK(ATH12K_NUM_MAX_LINKS, IEEE80211_MLD_MAX_NUM_LINKS)
 
 #define ATH12K_NUM_MAX_ACTIVE_LINKS_PER_DEVICE	2
@@ -172,6 +169,7 @@ struct ath12k *ath12k_mac_get_ar_by_pdev_id(struct ath12k_base *ab, u32 pdev_id)
 
 void ath12k_mac_drain_tx(struct ath12k *ar);
 void ath12k_mac_peer_cleanup_all(struct ath12k *ar);
+void ath12k_mac_dp_peer_cleanup(struct ath12k_hw *ah);
 int ath12k_mac_tx_mgmt_pending_free(int buf_id, void *skb, void *ctx);
 enum rate_info_bw ath12k_mac_bw_to_mac80211_bw(enum ath12k_supported_bw bw);
 enum ath12k_supported_bw ath12k_mac_mac80211_bw_to_ath12k_bw(enum rate_info_bw bw);
@@ -206,4 +204,139 @@ void ath12k_mac_update_freq_range(struct ath12k *ar,
 void ath12k_mac_fill_reg_tpc_info(struct ath12k *ar,
 				  struct ath12k_link_vif *arvif,
 				  struct ieee80211_chanctx_conf *ctx);
+int ath12k_mac_op_start(struct ieee80211_hw *hw);
+void ath12k_mac_op_stop(struct ieee80211_hw *hw, bool suspend);
+void
+ath12k_mac_op_reconfig_complete(struct ieee80211_hw *hw,
+				enum ieee80211_reconfig_type reconfig_type);
+int ath12k_mac_op_add_interface(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif);
+void ath12k_mac_op_remove_interface(struct ieee80211_hw *hw,
+				    struct ieee80211_vif *vif);
+void ath12k_mac_op_update_vif_offload(struct ieee80211_hw *hw,
+				      struct ieee80211_vif *vif);
+int ath12k_mac_op_config(struct ieee80211_hw *hw, int radio_idx, u32 changed);
+void ath12k_mac_op_link_info_changed(struct ieee80211_hw *hw,
+				     struct ieee80211_vif *vif,
+				     struct ieee80211_bss_conf *info,
+				     u64 changed);
+void ath12k_mac_op_vif_cfg_changed(struct ieee80211_hw *hw,
+				   struct ieee80211_vif *vif,
+				   u64 changed);
+int
+ath12k_mac_op_change_vif_links
+			(struct ieee80211_hw *hw,
+			 struct ieee80211_vif *vif,
+			 u16 old_links, u16 new_links,
+			 struct ieee80211_bss_conf *ol[IEEE80211_MLD_MAX_NUM_LINKS]);
+void ath12k_mac_op_configure_filter(struct ieee80211_hw *hw,
+				    unsigned int changed_flags,
+				    unsigned int *total_flags,
+				    u64 multicast);
+int ath12k_mac_op_hw_scan(struct ieee80211_hw *hw,
+			  struct ieee80211_vif *vif,
+			  struct ieee80211_scan_request *hw_req);
+void ath12k_mac_op_cancel_hw_scan(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif);
+int ath12k_mac_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
+			  struct ieee80211_vif *vif, struct ieee80211_sta *sta,
+			  struct ieee80211_key_conf *key);
+void ath12k_mac_op_set_rekey_data(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif,
+				  struct cfg80211_gtk_rekey_data *data);
+int ath12k_mac_op_sta_state(struct ieee80211_hw *hw,
+			    struct ieee80211_vif *vif,
+			    struct ieee80211_sta *sta,
+			    enum ieee80211_sta_state old_state,
+			    enum ieee80211_sta_state new_state);
+int ath12k_mac_op_sta_set_txpwr(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif,
+				struct ieee80211_sta *sta);
+void ath12k_mac_op_link_sta_rc_update(struct ieee80211_hw *hw,
+				      struct ieee80211_vif *vif,
+				      struct ieee80211_link_sta *link_sta,
+				      u32 changed);
+int ath12k_mac_op_conf_tx(struct ieee80211_hw *hw,
+			  struct ieee80211_vif *vif,
+			  unsigned int link_id, u16 ac,
+			  const struct ieee80211_tx_queue_params *params);
+int ath12k_mac_op_set_antenna(struct ieee80211_hw *hw, int radio_idx,
+			      u32 tx_ant, u32 rx_ant);
+int ath12k_mac_op_get_antenna(struct ieee80211_hw *hw, int radio_idx,
+			      u32 *tx_ant, u32 *rx_ant);
+int ath12k_mac_op_ampdu_action(struct ieee80211_hw *hw,
+			       struct ieee80211_vif *vif,
+			       struct ieee80211_ampdu_params *params);
+int ath12k_mac_op_add_chanctx(struct ieee80211_hw *hw,
+			      struct ieee80211_chanctx_conf *ctx);
+void ath12k_mac_op_remove_chanctx(struct ieee80211_hw *hw,
+				  struct ieee80211_chanctx_conf *ctx);
+void ath12k_mac_op_change_chanctx(struct ieee80211_hw *hw,
+				  struct ieee80211_chanctx_conf *ctx,
+				  u32 changed);
+int
+ath12k_mac_op_assign_vif_chanctx(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif,
+				 struct ieee80211_bss_conf *link_conf,
+				 struct ieee80211_chanctx_conf *ctx);
+void
+ath12k_mac_op_unassign_vif_chanctx(struct ieee80211_hw *hw,
+				   struct ieee80211_vif *vif,
+				   struct ieee80211_bss_conf *link_conf,
+				   struct ieee80211_chanctx_conf *ctx);
+int
+ath12k_mac_op_switch_vif_chanctx(struct ieee80211_hw *hw,
+				 struct ieee80211_vif_chanctx_switch *vifs,
+				 int n_vifs,
+				 enum ieee80211_chanctx_switch_mode mode);
+int ath12k_mac_op_set_rts_threshold(struct ieee80211_hw *hw,
+				    int radio_idx, u32 value);
+int ath12k_mac_op_set_frag_threshold(struct ieee80211_hw *hw,
+				     int radio_idx, u32 value);
+int
+ath12k_mac_op_set_bitrate_mask(struct ieee80211_hw *hw,
+			       struct ieee80211_vif *vif,
+			       const struct cfg80211_bitrate_mask *mask);
+int ath12k_mac_op_get_survey(struct ieee80211_hw *hw, int idx,
+			     struct survey_info *survey);
+void ath12k_mac_op_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			 u32 queues, bool drop);
+void ath12k_mac_op_sta_statistics(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_sta *sta,
+				  struct station_info *sinfo);
+void ath12k_mac_op_link_sta_statistics(struct ieee80211_hw *hw,
+				       struct ieee80211_vif *vif,
+				       struct ieee80211_link_sta *link_sta,
+				       struct link_station_info *link_sinfo);
+int ath12k_mac_op_remain_on_channel(struct ieee80211_hw *hw,
+				    struct ieee80211_vif *vif,
+				    struct ieee80211_channel *chan,
+				    int duration,
+				    enum ieee80211_roc_type type);
+int ath12k_mac_op_cancel_remain_on_channel(struct ieee80211_hw *hw,
+					   struct ieee80211_vif *vif);
+int ath12k_mac_op_change_sta_links(struct ieee80211_hw *hw,
+				   struct ieee80211_vif *vif,
+				   struct ieee80211_sta *sta,
+				   u16 old_links, u16 new_links);
+bool ath12k_mac_op_can_activate_links(struct ieee80211_hw *hw,
+				      struct ieee80211_vif *vif,
+				      u16 active_links);
+int ath12k_mac_op_get_txpower(struct ieee80211_hw *hw,
+			      struct ieee80211_vif *vif,
+			      unsigned int link_id,
+			      int *dbm);
+int ath12k_mac_mgmt_tx(struct ath12k *ar, struct sk_buff *skb,
+		       bool is_prb_rsp);
+void ath12k_mac_add_p2p_noa_ie(struct ath12k *ar,
+			       struct ieee80211_vif *vif,
+			       struct sk_buff *skb,
+			       bool is_prb_rsp);
+u8 ath12k_mac_get_tx_link(struct ieee80211_sta *sta, struct ieee80211_vif *vif,
+			  u8 link, struct sk_buff *skb, u32 info_flags);
+
+void ath12k_mlo_mcast_update_tx_link_address(struct ieee80211_vif *vif,
+					     u8 link_id, struct sk_buff *skb,
+					     u32 info_flags);
 #endif
diff --git a/drivers/net/wireless/ath/ath12k/mhi.c b/drivers/net/wireless/ath/ath12k/mhi.c
index 08f44baf182a..45c0f66dcc5e 100644
--- a/drivers/net/wireless/ath/ath12k/mhi.c
+++ b/drivers/net/wireless/ath/ath12k/mhi.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #include <linux/msi.h>
@@ -18,136 +18,6 @@
 #define OTP_VALID_DUALMAC_BOARD_ID_MASK		0x1000
 #define MHI_CB_INVALID	0xff
 
-static const struct mhi_channel_config ath12k_mhi_channels_qcn9274[] = {
-	{
-		.num = 20,
-		.name = "IPCR",
-		.num_elements = 32,
-		.event_ring = 1,
-		.dir = DMA_TO_DEVICE,
-		.ee_mask = 0x4,
-		.pollcfg = 0,
-		.doorbell = MHI_DB_BRST_DISABLE,
-		.lpm_notify = false,
-		.offload_channel = false,
-		.doorbell_mode_switch = false,
-		.auto_queue = false,
-	},
-	{
-		.num = 21,
-		.name = "IPCR",
-		.num_elements = 32,
-		.event_ring = 1,
-		.dir = DMA_FROM_DEVICE,
-		.ee_mask = 0x4,
-		.pollcfg = 0,
-		.doorbell = MHI_DB_BRST_DISABLE,
-		.lpm_notify = false,
-		.offload_channel = false,
-		.doorbell_mode_switch = false,
-		.auto_queue = true,
-	},
-};
-
-static struct mhi_event_config ath12k_mhi_events_qcn9274[] = {
-	{
-		.num_elements = 32,
-		.irq_moderation_ms = 0,
-		.irq = 1,
-		.data_type = MHI_ER_CTRL,
-		.mode = MHI_DB_BRST_DISABLE,
-		.hardware_event = false,
-		.client_managed = false,
-		.offload_channel = false,
-	},
-	{
-		.num_elements = 256,
-		.irq_moderation_ms = 1,
-		.irq = 2,
-		.mode = MHI_DB_BRST_DISABLE,
-		.priority = 1,
-		.hardware_event = false,
-		.client_managed = false,
-		.offload_channel = false,
-	},
-};
-
-const struct mhi_controller_config ath12k_mhi_config_qcn9274 = {
-	.max_channels = 30,
-	.timeout_ms = 10000,
-	.use_bounce_buf = false,
-	.buf_len = 0,
-	.num_channels = ARRAY_SIZE(ath12k_mhi_channels_qcn9274),
-	.ch_cfg = ath12k_mhi_channels_qcn9274,
-	.num_events = ARRAY_SIZE(ath12k_mhi_events_qcn9274),
-	.event_cfg = ath12k_mhi_events_qcn9274,
-};
-
-static const struct mhi_channel_config ath12k_mhi_channels_wcn7850[] = {
-	{
-		.num = 20,
-		.name = "IPCR",
-		.num_elements = 64,
-		.event_ring = 1,
-		.dir = DMA_TO_DEVICE,
-		.ee_mask = 0x4,
-		.pollcfg = 0,
-		.doorbell = MHI_DB_BRST_DISABLE,
-		.lpm_notify = false,
-		.offload_channel = false,
-		.doorbell_mode_switch = false,
-		.auto_queue = false,
-	},
-	{
-		.num = 21,
-		.name = "IPCR",
-		.num_elements = 64,
-		.event_ring = 1,
-		.dir = DMA_FROM_DEVICE,
-		.ee_mask = 0x4,
-		.pollcfg = 0,
-		.doorbell = MHI_DB_BRST_DISABLE,
-		.lpm_notify = false,
-		.offload_channel = false,
-		.doorbell_mode_switch = false,
-		.auto_queue = true,
-	},
-};
-
-static struct mhi_event_config ath12k_mhi_events_wcn7850[] = {
-	{
-		.num_elements = 32,
-		.irq_moderation_ms = 0,
-		.irq = 1,
-		.mode = MHI_DB_BRST_DISABLE,
-		.data_type = MHI_ER_CTRL,
-		.hardware_event = false,
-		.client_managed = false,
-		.offload_channel = false,
-	},
-	{
-		.num_elements = 256,
-		.irq_moderation_ms = 1,
-		.irq = 2,
-		.mode = MHI_DB_BRST_DISABLE,
-		.priority = 1,
-		.hardware_event = false,
-		.client_managed = false,
-		.offload_channel = false,
-	},
-};
-
-const struct mhi_controller_config ath12k_mhi_config_wcn7850 = {
-	.max_channels = 128,
-	.timeout_ms = 2000,
-	.use_bounce_buf = false,
-	.buf_len = 8192,
-	.num_channels = ARRAY_SIZE(ath12k_mhi_channels_wcn7850),
-	.ch_cfg = ath12k_mhi_channels_wcn7850,
-	.num_events = ARRAY_SIZE(ath12k_mhi_events_wcn7850),
-	.event_cfg = ath12k_mhi_events_wcn7850,
-};
-
 void ath12k_mhi_set_mhictrl_reset(struct ath12k_base *ab)
 {
 	u32 val;
diff --git a/drivers/net/wireless/ath/ath12k/mhi.h b/drivers/net/wireless/ath/ath12k/mhi.h
index 7358b8477536..367432676385 100644
--- a/drivers/net/wireless/ath/ath12k/mhi.h
+++ b/drivers/net/wireless/ath/ath12k/mhi.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause-Clear */
 /*
  * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2022, 2024 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 #ifndef _ATH12K_MHI_H
 #define _ATH12K_MHI_H
@@ -31,9 +31,6 @@ enum ath12k_mhi_state {
 	ATH12K_MHI_RDDM_DONE,
 };
 
-extern const struct mhi_controller_config ath12k_mhi_config_qcn9274;
-extern const struct mhi_controller_config ath12k_mhi_config_wcn7850;
-
 int ath12k_mhi_start(struct ath12k_pci *ar_pci);
 void ath12k_mhi_stop(struct ath12k_pci *ar_pci, bool is_suspend);
 int ath12k_mhi_register(struct ath12k_pci *ar_pci);
diff --git a/drivers/net/wireless/ath/ath12k/pci.c b/drivers/net/wireless/ath/ath12k/pci.c
index a12c8379cb46..a42c4289c6b2 100644
--- a/drivers/net/wireless/ath/ath12k/pci.c
+++ b/drivers/net/wireless/ath/ath12k/pci.c
@@ -15,6 +15,7 @@
 #include "hif.h"
 #include "mhi.h"
 #include "debug.h"
+#include "hal.h"
 
 #define ATH12K_PCI_BAR_NUM		0
 #define ATH12K_PCI_DMA_MASK		36
@@ -28,44 +29,17 @@
 #define WINDOW_RANGE_MASK		GENMASK(18, 0)
 #define WINDOW_STATIC_MASK		GENMASK(31, 6)
 
-#define TCSR_SOC_HW_VERSION		0x1B00000
-#define TCSR_SOC_HW_VERSION_MAJOR_MASK	GENMASK(11, 8)
-#define TCSR_SOC_HW_VERSION_MINOR_MASK	GENMASK(7, 4)
-
 /* BAR0 + 4k is always accessible, and no
  * need to force wakeup.
  * 4K - 32 = 0xFE0
  */
 #define ACCESS_ALWAYS_OFF 0xFE0
 
-#define QCN9274_DEVICE_ID		0x1109
-#define WCN7850_DEVICE_ID		0x1107
-
 #define PCIE_LOCAL_REG_QRTR_NODE_ID	0x1E03164
 #define DOMAIN_NUMBER_MASK		GENMASK(7, 4)
 #define BUS_NUMBER_MASK			GENMASK(3, 0)
 
-static const struct pci_device_id ath12k_pci_id_table[] = {
-	{ PCI_VDEVICE(QCOM, QCN9274_DEVICE_ID) },
-	{ PCI_VDEVICE(QCOM, WCN7850_DEVICE_ID) },
-	{}
-};
-
-MODULE_DEVICE_TABLE(pci, ath12k_pci_id_table);
-
-/* TODO: revisit IRQ mapping for new SRNG's */
-static const struct ath12k_msi_config ath12k_msi_config[] = {
-	{
-		.total_vectors = 16,
-		.total_users = 3,
-		.users = (struct ath12k_msi_user[]) {
-			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
-			{ .name = "CE", .num_vectors = 5, .base_vector = 3 },
-			{ .name = "DP", .num_vectors = 8, .base_vector = 8 },
-		},
-	},
-};
-
+static struct ath12k_pci_driver *ath12k_pci_family_drivers[ATH12K_DEVICE_FAMILY_MAX];
 static const struct ath12k_msi_config msi_config_one_msi = {
 	.total_vectors = 1,
 	.total_users = 4,
@@ -136,30 +110,6 @@ static const char *irq_name[ATH12K_IRQ_NUM_MAX] = {
 	"tcl2host-status-ring",
 };
 
-static int ath12k_pci_bus_wake_up(struct ath12k_base *ab)
-{
-	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);
-
-	return mhi_device_get_sync(ab_pci->mhi_ctrl->mhi_dev);
-}
-
-static void ath12k_pci_bus_release(struct ath12k_base *ab)
-{
-	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);
-
-	mhi_device_put(ab_pci->mhi_ctrl->mhi_dev);
-}
-
-static const struct ath12k_pci_ops ath12k_pci_ops_qcn9274 = {
-	.wakeup = NULL,
-	.release = NULL,
-};
-
-static const struct ath12k_pci_ops ath12k_pci_ops_wcn7850 = {
-	.wakeup = ath12k_pci_bus_wake_up,
-	.release = ath12k_pci_bus_release,
-};
-
 static void ath12k_pci_select_window(struct ath12k_pci *ab_pci, u32 offset)
 {
 	struct ath12k_base *ab = ab_pci->ab;
@@ -183,10 +133,12 @@ static void ath12k_pci_select_window(struct ath12k_pci *ab_pci, u32 offset)
 
 static void ath12k_pci_select_static_window(struct ath12k_pci *ab_pci)
 {
-	u32 umac_window = u32_get_bits(HAL_SEQ_WCSS_UMAC_OFFSET, WINDOW_VALUE_MASK);
-	u32 ce_window = u32_get_bits(HAL_CE_WFSS_CE_REG_BASE, WINDOW_VALUE_MASK);
+	u32 umac_window;
+	u32 ce_window;
 	u32 window;
 
+	umac_window = u32_get_bits(ab_pci->reg_base->umac_base, WINDOW_VALUE_MASK);
+	ce_window = u32_get_bits(ab_pci->reg_base->ce_reg_base, WINDOW_VALUE_MASK);
 	window = (umac_window << 12) | (ce_window << 6);
 
 	spin_lock_bh(&ab_pci->window_lock);
@@ -199,13 +151,14 @@ static void ath12k_pci_select_static_window(struct ath12k_pci *ab_pci)
 static u32 ath12k_pci_get_window_start(struct ath12k_base *ab,
 				       u32 offset)
 {
+	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);
 	u32 window_start;
 
 	/* If offset lies within DP register range, use 3rd window */
-	if ((offset ^ HAL_SEQ_WCSS_UMAC_OFFSET) < WINDOW_RANGE_MASK)
+	if ((offset ^ ab_pci->reg_base->umac_base) < WINDOW_RANGE_MASK)
 		window_start = 3 * WINDOW_START;
 	/* If offset lies within CE register range, use 2nd window */
-	else if ((offset ^ HAL_CE_WFSS_CE_REG_BASE) < WINDOW_RANGE_MASK)
+	else if ((offset ^ ab_pci->reg_base->ce_reg_base) < WINDOW_RANGE_MASK)
 		window_start = 2 * WINDOW_START;
 	else
 		window_start = WINDOW_START;
@@ -544,10 +497,11 @@ static int ath12k_pci_ext_grp_napi_poll(struct napi_struct *napi, int budget)
 						struct ath12k_ext_irq_grp,
 						napi);
 	struct ath12k_base *ab = irq_grp->ab;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	int work_done;
 	int i;
 
-	work_done = ath12k_dp_service_srng(ab, irq_grp, budget);
+	work_done = ath12k_dp_service_srng(dp, irq_grp, budget);
 	if (work_done < budget) {
 		napi_complete_done(napi, work_done);
 		for (i = 0; i < irq_grp->num_irq; i++)
@@ -1244,6 +1198,7 @@ u32 ath12k_pci_read32(struct ath12k_base *ab, u32 offset)
 		ab_pci->pci_ops->release(ab);
 	return val;
 }
+EXPORT_SYMBOL(ath12k_pci_read32);
 
 void ath12k_pci_write32(struct ath12k_base *ab, u32 offset, u32 value)
 {
@@ -1549,28 +1504,34 @@ static const struct ath12k_hif_ops ath12k_pci_hif_ops = {
 #endif
 };
 
-static
-void ath12k_pci_read_hw_version(struct ath12k_base *ab, u32 *major, u32 *minor)
+static enum ath12k_device_family
+ath12k_get_device_family(const struct pci_device_id *pci_dev)
 {
-	u32 soc_hw_version;
+	enum ath12k_device_family device_family_id;
+	const struct pci_device_id *id;
+
+	for (device_family_id = ATH12K_DEVICE_FAMILY_START;
+	     device_family_id < ATH12K_DEVICE_FAMILY_MAX; device_family_id++) {
+		if (!ath12k_pci_family_drivers[device_family_id])
+			continue;
 
-	soc_hw_version = ath12k_pci_read32(ab, TCSR_SOC_HW_VERSION);
-	*major = FIELD_GET(TCSR_SOC_HW_VERSION_MAJOR_MASK,
-			   soc_hw_version);
-	*minor = FIELD_GET(TCSR_SOC_HW_VERSION_MINOR_MASK,
-			   soc_hw_version);
+		id = ath12k_pci_family_drivers[device_family_id]->id_table;
+		while (id->device) {
+			if (id->device == pci_dev->device)
+				return device_family_id;
+			id += 1;
+		}
+	}
 
-	ath12k_dbg(ab, ATH12K_DBG_PCI,
-		   "pci tcsr_soc_hw_version major %d minor %d\n",
-		    *major, *minor);
+	return ATH12K_DEVICE_FAMILY_MAX;
 }
 
 static int ath12k_pci_probe(struct pci_dev *pdev,
 			    const struct pci_device_id *pci_dev)
 {
-	struct ath12k_base *ab;
+	enum ath12k_device_family device_id;
 	struct ath12k_pci *ab_pci;
-	u32 soc_hw_version_major, soc_hw_version_minor;
+	struct ath12k_base *ab;
 	int ret;
 
 	ab = ath12k_core_alloc(&pdev->dev, sizeof(*ab_pci), ATH12K_BUS_PCI);
@@ -1605,56 +1566,25 @@ static int ath12k_pci_probe(struct pci_dev *pdev,
 	ab->id.subsystem_vendor = pdev->subsystem_vendor;
 	ab->id.subsystem_device = pdev->subsystem_device;
 
-	switch (pci_dev->device) {
-	case QCN9274_DEVICE_ID:
-		ab_pci->msi_config = &ath12k_msi_config[0];
-		ab->static_window_map = true;
-		ab_pci->pci_ops = &ath12k_pci_ops_qcn9274;
-		ab->hal_rx_ops = &hal_rx_qcn9274_ops;
-		ath12k_pci_read_hw_version(ab, &soc_hw_version_major,
-					   &soc_hw_version_minor);
-		ab->target_mem_mode = ath12k_core_get_memory_mode(ab);
-		switch (soc_hw_version_major) {
-		case ATH12K_PCI_SOC_HW_VERSION_2:
-			ab->hw_rev = ATH12K_HW_QCN9274_HW20;
-			break;
-		case ATH12K_PCI_SOC_HW_VERSION_1:
-			ab->hw_rev = ATH12K_HW_QCN9274_HW10;
-			break;
-		default:
-			dev_err(&pdev->dev,
-				"Unknown hardware version found for QCN9274: 0x%x\n",
-				soc_hw_version_major);
-			ret = -EOPNOTSUPP;
-			goto err_pci_free_region;
-		}
-		break;
-	case WCN7850_DEVICE_ID:
-		ab->id.bdf_search = ATH12K_BDF_SEARCH_BUS_AND_BOARD;
-		ab_pci->msi_config = &ath12k_msi_config[0];
-		ab->static_window_map = false;
-		ab_pci->pci_ops = &ath12k_pci_ops_wcn7850;
-		ab->hal_rx_ops = &hal_rx_wcn7850_ops;
-		ath12k_pci_read_hw_version(ab, &soc_hw_version_major,
-					   &soc_hw_version_minor);
-		ab->target_mem_mode = ATH12K_QMI_MEMORY_MODE_DEFAULT;
-		switch (soc_hw_version_major) {
-		case ATH12K_PCI_SOC_HW_VERSION_2:
-			ab->hw_rev = ATH12K_HW_WCN7850_HW20;
-			break;
-		default:
-			dev_err(&pdev->dev,
-				"Unknown hardware version found for WCN7850: 0x%x\n",
-				soc_hw_version_major);
-			ret = -EOPNOTSUPP;
-			goto err_pci_free_region;
-		}
-		break;
+	device_id = ath12k_get_device_family(pci_dev);
+	if (device_id >= ATH12K_DEVICE_FAMILY_MAX) {
+		ath12k_err(ab, "failed to get device family id\n");
+		ret = -EINVAL;
+		goto err_pci_free_region;
+	}
 
-	default:
-		dev_err(&pdev->dev, "Unknown PCI device found: 0x%x\n",
-			pci_dev->device);
-		ret = -EOPNOTSUPP;
+	ath12k_dbg(ab, ATH12K_DBG_PCI, "PCI device family id: %d\n", device_id);
+
+	ab_pci->device_family_ops = &ath12k_pci_family_drivers[device_id]->ops;
+	ab_pci->reg_base = ath12k_pci_family_drivers[device_id]->reg_base;
+
+	/* Call device specific probe. This is the callback that can
+	 * be used to override any ops in future
+	 * probe is validated for NULL during registration.
+	 */
+	ret = ab_pci->device_family_ops->probe(pdev, pci_dev);
+	if (ret) {
+		ath12k_err(ab, "failed to probe device: %d\n", ret);
 		goto err_pci_free_region;
 	}
 
@@ -1709,13 +1639,25 @@ static int ath12k_pci_probe(struct pci_dev *pdev,
 		goto err_free_irq;
 	}
 
+	/* Invoke arch_init here so that arch-specific init operations
+	 * can utilize already initialized ab fields, such as HAL SRNGs.
+	 */
+	ret = ab_pci->device_family_ops->arch_init(ab);
+	if (ret) {
+		ath12k_err(ab, "PCI arch_init failed %d\n", ret);
+		goto err_pci_msi_free;
+	}
+
 	ret = ath12k_core_init(ab);
 	if (ret) {
 		ath12k_err(ab, "failed to init core: %d\n", ret);
-		goto err_free_irq;
+		goto err_deinit_arch;
 	}
 	return 0;
 
+err_deinit_arch:
+	ab_pci->device_family_ops->arch_deinit(ab);
+
 err_free_irq:
 	/* __free_irq() expects the caller to have cleared the affinity hint */
 	ath12k_pci_set_irq_affinity_hint(ab_pci, NULL);
@@ -1774,6 +1716,9 @@ static void ath12k_pci_remove(struct pci_dev *pdev)
 
 	ath12k_hal_srng_deinit(ab);
 	ath12k_ce_free_pipes(ab);
+
+	ab_pci->device_family_ops->arch_deinit(ab);
+
 	ath12k_core_free(ab);
 }
 
@@ -1862,33 +1807,47 @@ static const struct dev_pm_ops __maybe_unused ath12k_pci_pm_ops = {
 				     ath12k_pci_pm_resume_early)
 };
 
-static struct pci_driver ath12k_pci_driver = {
-	.name = "ath12k_pci",
-	.id_table = ath12k_pci_id_table,
-	.probe = ath12k_pci_probe,
-	.remove = ath12k_pci_remove,
-	.shutdown = ath12k_pci_shutdown,
-	.driver.pm = &ath12k_pci_pm_ops,
-};
-
-int ath12k_pci_init(void)
+int ath12k_pci_register_driver(const enum ath12k_device_family device_id,
+			       struct ath12k_pci_driver *driver)
 {
-	int ret;
+	struct pci_driver *pci_driver;
 
-	ret = pci_register_driver(&ath12k_pci_driver);
-	if (ret) {
-		pr_err("failed to register ath12k pci driver: %d\n",
-		       ret);
-		return ret;
+	if (device_id >= ATH12K_DEVICE_FAMILY_MAX)
+		return -EINVAL;
+
+	if (!driver || !driver->ops.probe ||
+	    !driver->ops.arch_init || !driver->ops.arch_deinit)
+		return -EINVAL;
+
+	if (ath12k_pci_family_drivers[device_id]) {
+		pr_err("Driver already registered for %d\n", device_id);
+		return -EALREADY;
 	}
 
-	return 0;
+	ath12k_pci_family_drivers[device_id] = driver;
+
+	pci_driver = &ath12k_pci_family_drivers[device_id]->driver;
+	pci_driver->name = driver->name;
+	pci_driver->id_table = driver->id_table;
+	pci_driver->probe = ath12k_pci_probe;
+	pci_driver->remove = ath12k_pci_remove;
+	pci_driver->shutdown = ath12k_pci_shutdown;
+	pci_driver->driver.pm = &ath12k_pci_pm_ops;
+
+	return pci_register_driver(pci_driver);
 }
+EXPORT_SYMBOL(ath12k_pci_register_driver);
 
-void ath12k_pci_exit(void)
+void ath12k_pci_unregister_driver(const enum ath12k_device_family device_id)
 {
-	pci_unregister_driver(&ath12k_pci_driver);
+	if (device_id >= ATH12K_DEVICE_FAMILY_MAX ||
+	    !ath12k_pci_family_drivers[device_id])
+		return;
+
+	pci_unregister_driver(&ath12k_pci_family_drivers[device_id]->driver);
+	ath12k_pci_family_drivers[device_id] = NULL;
 }
+EXPORT_SYMBOL(ath12k_pci_unregister_driver);
 
 /* firmware files */
 MODULE_FIRMWARE(ATH12K_FW_DIR "/QCN9274/hw2.0/*");
diff --git a/drivers/net/wireless/ath/ath12k/pci.h b/drivers/net/wireless/ath/ath12k/pci.h
index d1ec8aad7f6c..1cc4f0e050f9 100644
--- a/drivers/net/wireless/ath/ath12k/pci.h
+++ b/drivers/net/wireless/ath/ath12k/pci.h
@@ -1,12 +1,13 @@
 /* SPDX-License-Identifier: BSD-3-Clause-Clear */
 /*
  * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 #ifndef ATH12K_PCI_H
 #define ATH12K_PCI_H
 
 #include <linux/mhi.h>
+#include <linux/pci.h>
 
 #include "core.h"
 
@@ -29,7 +30,7 @@
 #define PARM_LTSSM_VALUE			0x111
 
 #define GCC_GCC_PCIE_HOT_RST(ab) \
-	((ab)->hw_params->regs->gcc_gcc_pcie_hot_rst)
+	((ab)->hal.regs->gcc_gcc_pcie_hot_rst)
 
 #define GCC_GCC_PCIE_HOT_RST_VAL		0x10
 
@@ -38,17 +39,17 @@
 #define PCIE_INT_CLEAR_ALL			0xffffffff
 
 #define PCIE_QSERDES_COM_SYSCLK_EN_SEL_REG(ab) \
-	((ab)->hw_params->regs->pcie_qserdes_sysclk_en_sel)
+	((ab)->hal.regs->pcie_qserdes_sysclk_en_sel)
 #define PCIE_QSERDES_COM_SYSCLK_EN_SEL_VAL	0x10
 #define PCIE_QSERDES_COM_SYSCLK_EN_SEL_MSK	0xffffffff
 #define PCIE_PCS_OSC_DTCT_CONFIG1_REG(ab) \
-	((ab)->hw_params->regs->pcie_pcs_osc_dtct_config_base)
+	((ab)->hal.regs->pcie_pcs_osc_dtct_config_base)
 #define PCIE_PCS_OSC_DTCT_CONFIG1_VAL		0x02
 #define PCIE_PCS_OSC_DTCT_CONFIG2_REG(ab) \
-	((ab)->hw_params->regs->pcie_pcs_osc_dtct_config_base + 0x4)
+	((ab)->hal.regs->pcie_pcs_osc_dtct_config_base + 0x4)
 #define PCIE_PCS_OSC_DTCT_CONFIG2_VAL		0x52
 #define PCIE_PCS_OSC_DTCT_CONFIG4_REG(ab) \
-	((ab)->hw_params->regs->pcie_pcs_osc_dtct_config_base + 0xc)
+	((ab)->hal.regs->pcie_pcs_osc_dtct_config_base + 0xc)
 #define PCIE_PCS_OSC_DTCT_CONFIG4_VAL		0xff
 #define PCIE_PCS_OSC_DTCT_CONFIG_MSK		0x000000ff
 
@@ -70,9 +71,6 @@
 #define QRTR_PCI_DOMAIN_NR_MASK		GENMASK(7, 4)
 #define QRTR_PCI_BUS_NUMBER_MASK	GENMASK(3, 0)
 
-#define ATH12K_PCI_SOC_HW_VERSION_1	1
-#define ATH12K_PCI_SOC_HW_VERSION_2	2
-
 struct ath12k_msi_user {
 	const char *name;
 	int num_vectors;
@@ -97,6 +95,17 @@ struct ath12k_pci_ops {
 	void (*release)(struct ath12k_base *ab);
 };
 
+struct ath12k_pci_device_family_ops {
+	int (*probe)(struct pci_dev *pdev, const struct pci_device_id *pci_dev);
+	int (*arch_init)(struct ath12k_base *ab);
+	void (*arch_deinit)(struct ath12k_base *ab);
+};
+
+struct ath12k_pci_reg_base {
+	u32 umac_base;
+	u32 ce_reg_base;
+};
+
 struct ath12k_pci {
 	struct pci_dev *pdev;
 	struct ath12k_base *ab;
@@ -119,6 +128,16 @@ struct ath12k_pci {
 	const struct ath12k_pci_ops *pci_ops;
 	u32 qmi_instance;
 	u64 dma_mask;
+	const struct ath12k_pci_device_family_ops *device_family_ops;
+	const struct ath12k_pci_reg_base *reg_base;
+};
+
+struct ath12k_pci_driver {
+	const char *name;
+	const struct pci_device_id *id_table;
+	struct ath12k_pci_device_family_ops ops;
+	struct pci_driver driver;
+	const struct ath12k_pci_reg_base *reg_base;
 };
 
 static inline struct ath12k_pci *ath12k_pci_priv(struct ath12k_base *ab)
@@ -148,6 +167,7 @@ void ath12k_pci_stop(struct ath12k_base *ab);
 int ath12k_pci_start(struct ath12k_base *ab);
 int ath12k_pci_power_up(struct ath12k_base *ab);
 void ath12k_pci_power_down(struct ath12k_base *ab, bool is_suspend);
-int ath12k_pci_init(void);
-void ath12k_pci_exit(void);
+int ath12k_pci_register_driver(const enum ath12k_device_family device_id,
+			       struct ath12k_pci_driver *driver);
+void ath12k_pci_unregister_driver(const enum ath12k_device_family device_id);
 #endif /* ATH12K_PCI_H */
diff --git a/drivers/net/wireless/ath/ath12k/peer.c b/drivers/net/wireless/ath/ath12k/peer.c
index f1ae9e5b5af7..5f3bd3b9a3e9 100644
--- a/drivers/net/wireless/ath/ath12k/peer.c
+++ b/drivers/net/wireless/ath/ath12k/peer.c
@@ -1,211 +1,28 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2022, 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #include "core.h"
 #include "peer.h"
 #include "debug.h"
+#include "debugfs.h"
 
-struct ath12k_ml_peer *ath12k_peer_ml_find(struct ath12k_hw *ah, const u8 *addr)
-{
-	struct ath12k_ml_peer *ml_peer;
-
-	lockdep_assert_wiphy(ah->hw->wiphy);
-
-	list_for_each_entry(ml_peer, &ah->ml_peers, list) {
-		if (!ether_addr_equal(ml_peer->addr, addr))
-			continue;
-
-		return ml_peer;
-	}
-
-	return NULL;
-}
-
-struct ath12k_peer *ath12k_peer_find(struct ath12k_base *ab, int vdev_id,
-				     const u8 *addr)
-{
-	struct ath12k_peer *peer;
-
-	lockdep_assert_held(&ab->base_lock);
-
-	list_for_each_entry(peer, &ab->peers, list) {
-		if (peer->vdev_id != vdev_id)
-			continue;
-		if (!ether_addr_equal(peer->addr, addr))
-			continue;
-
-		return peer;
-	}
-
-	return NULL;
-}
-
-static struct ath12k_peer *ath12k_peer_find_by_pdev_idx(struct ath12k_base *ab,
-							u8 pdev_idx, const u8 *addr)
-{
-	struct ath12k_peer *peer;
-
-	lockdep_assert_held(&ab->base_lock);
-
-	list_for_each_entry(peer, &ab->peers, list) {
-		if (peer->pdev_idx != pdev_idx)
-			continue;
-		if (!ether_addr_equal(peer->addr, addr))
-			continue;
-
-		return peer;
-	}
-
-	return NULL;
-}
-
-struct ath12k_peer *ath12k_peer_find_by_addr(struct ath12k_base *ab,
-					     const u8 *addr)
-{
-	struct ath12k_peer *peer;
-
-	lockdep_assert_held(&ab->base_lock);
-
-	list_for_each_entry(peer, &ab->peers, list) {
-		if (!ether_addr_equal(peer->addr, addr))
-			continue;
-
-		return peer;
-	}
-
-	return NULL;
-}
-
-static struct ath12k_peer *ath12k_peer_find_by_ml_id(struct ath12k_base *ab,
-						     int ml_peer_id)
-{
-	struct ath12k_peer *peer;
-
-	lockdep_assert_held(&ab->base_lock);
-
-	list_for_each_entry(peer, &ab->peers, list)
-		if (ml_peer_id == peer->ml_id)
-			return peer;
-
-	return NULL;
-}
-
-struct ath12k_peer *ath12k_peer_find_by_id(struct ath12k_base *ab,
-					   int peer_id)
-{
-	struct ath12k_peer *peer;
-
-	lockdep_assert_held(&ab->base_lock);
-
-	if (peer_id == HAL_INVALID_PEERID)
-		return NULL;
-
-	if (peer_id & ATH12K_PEER_ML_ID_VALID)
-		return ath12k_peer_find_by_ml_id(ab, peer_id);
-
-	list_for_each_entry(peer, &ab->peers, list)
-		if (peer_id == peer->peer_id)
-			return peer;
-
-	return NULL;
-}
-
-bool ath12k_peer_exist_by_vdev_id(struct ath12k_base *ab, int vdev_id)
-{
-	struct ath12k_peer *peer;
-
-	spin_lock_bh(&ab->base_lock);
-
-	list_for_each_entry(peer, &ab->peers, list) {
-		if (vdev_id == peer->vdev_id) {
-			spin_unlock_bh(&ab->base_lock);
-			return true;
-		}
-	}
-	spin_unlock_bh(&ab->base_lock);
-	return false;
-}
-
-struct ath12k_peer *ath12k_peer_find_by_ast(struct ath12k_base *ab,
-					    int ast_hash)
-{
-	struct ath12k_peer *peer;
-
-	lockdep_assert_held(&ab->base_lock);
-
-	list_for_each_entry(peer, &ab->peers, list)
-		if (ast_hash == peer->ast_hash)
-			return peer;
-
-	return NULL;
-}
-
-void ath12k_peer_unmap_event(struct ath12k_base *ab, u16 peer_id)
-{
-	struct ath12k_peer *peer;
-
-	spin_lock_bh(&ab->base_lock);
-
-	peer = ath12k_peer_find_by_id(ab, peer_id);
-	if (!peer) {
-		ath12k_warn(ab, "peer-unmap-event: unknown peer id %d\n",
-			    peer_id);
-		goto exit;
-	}
-
-	ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "htt peer unmap vdev %d peer %pM id %d\n",
-		   peer->vdev_id, peer->addr, peer_id);
-
-	list_del(&peer->list);
-	kfree(peer);
-	wake_up(&ab->peer_mapping_wq);
-
-exit:
-	spin_unlock_bh(&ab->base_lock);
-}
-
-void ath12k_peer_map_event(struct ath12k_base *ab, u8 vdev_id, u16 peer_id,
-			   u8 *mac_addr, u16 ast_hash, u16 hw_peer_id)
-{
-	struct ath12k_peer *peer;
-
-	spin_lock_bh(&ab->base_lock);
-	peer = ath12k_peer_find(ab, vdev_id, mac_addr);
-	if (!peer) {
-		peer = kzalloc(sizeof(*peer), GFP_ATOMIC);
-		if (!peer)
-			goto exit;
-
-		peer->vdev_id = vdev_id;
-		peer->peer_id = peer_id;
-		peer->ast_hash = ast_hash;
-		peer->hw_peer_id = hw_peer_id;
-		ether_addr_copy(peer->addr, mac_addr);
-		list_add(&peer->list, &ab->peers);
-		wake_up(&ab->peer_mapping_wq);
-	}
-
-	ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "htt peer map vdev %d peer %pM id %d\n",
-		   vdev_id, mac_addr, peer_id);
-
-exit:
-	spin_unlock_bh(&ab->base_lock);
-}
-
-static int ath12k_wait_for_peer_common(struct ath12k_base *ab, int vdev_id,
-				       const u8 *addr, bool expect_mapped)
+static int ath12k_wait_for_dp_link_peer_common(struct ath12k_base *ab, int vdev_id,
+					       const u8 *addr, bool expect_mapped)
 {
 	int ret;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 
 	ret = wait_event_timeout(ab->peer_mapping_wq, ({
 				bool mapped;
 
-				spin_lock_bh(&ab->base_lock);
-				mapped = !!ath12k_peer_find(ab, vdev_id, addr);
-				spin_unlock_bh(&ab->base_lock);
+				spin_lock_bh(&dp->dp_lock);
+				mapped = !!ath12k_dp_link_peer_find_by_vdev_and_addr(dp,
+										vdev_id,
+										addr);
+				spin_unlock_bh(&dp->dp_lock);
 
 				(mapped == expect_mapped ||
 				 test_bit(ATH12K_FLAG_CRASH_FLUSH, &ab->dev_flags));
@@ -219,30 +36,30 @@ static int ath12k_wait_for_peer_common(struct ath12k_base *ab, int vdev_id,
 
 void ath12k_peer_cleanup(struct ath12k *ar, u32 vdev_id)
 {
-	struct ath12k_peer *peer, *tmp;
+	struct ath12k_dp_link_peer *peer, *tmp;
 	struct ath12k_base *ab = ar->ab;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 
 	lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy);
 
-	spin_lock_bh(&ab->base_lock);
-	list_for_each_entry_safe(peer, tmp, &ab->peers, list) {
+	spin_lock_bh(&dp->dp_lock);
+	list_for_each_entry_safe(peer, tmp, &dp->peers, list) {
 		if (peer->vdev_id != vdev_id)
 			continue;
 
 		ath12k_warn(ab, "removing stale peer %pM from vdev_id %d\n",
 			    peer->addr, vdev_id);
 
-		list_del(&peer->list);
-		kfree(peer);
+		ath12k_dp_link_peer_free(peer);
 		ar->num_peers--;
 	}
 
-	spin_unlock_bh(&ab->base_lock);
+	spin_unlock_bh(&dp->dp_lock);
 }
 
 static int ath12k_wait_for_peer_deleted(struct ath12k *ar, int vdev_id, const u8 *addr)
 {
-	return ath12k_wait_for_peer_common(ar->ab, vdev_id, addr, false);
+	return ath12k_wait_for_dp_link_peer_common(ar->ab, vdev_id, addr, false);
 }
 
 int ath12k_wait_for_peer_delete_done(struct ath12k *ar, u32 vdev_id,
@@ -293,6 +110,10 @@ int ath12k_peer_delete(struct ath12k *ar, u32 vdev_id, u8 *addr)
 
 	lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy);
 
+	ath12k_dp_link_peer_unassign(ath12k_ab_to_dp(ar->ab),
+				     &(ath12k_ar_to_ah(ar)->dp_hw), vdev_id,
+				     addr, ar->hw_link_id);
+
 	ret = ath12k_peer_delete_send(ar, vdev_id, addr);
 	if (ret)
 		return ret;
@@ -308,7 +129,7 @@ int ath12k_peer_delete(struct ath12k *ar, u32 vdev_id, u8 *addr)
 
 static int ath12k_wait_for_peer_created(struct ath12k *ar, int vdev_id, const u8 *addr)
 {
-	return ath12k_wait_for_peer_common(ar->ab, vdev_id, addr, true);
+	return ath12k_wait_for_dp_link_peer_common(ar->ab, vdev_id, addr, true);
 }
 
 int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif,
@@ -316,28 +137,34 @@ int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif,
 		       struct ath12k_wmi_peer_create_arg *arg)
 {
 	struct ieee80211_vif *vif = ath12k_ahvif_to_vif(arvif->ahvif);
+	struct ath12k_vif *ahvif = arvif->ahvif;
+	struct ath12k_dp_link_vif *dp_link_vif;
 	struct ath12k_link_sta *arsta;
 	u8 link_id = arvif->link_id;
-	struct ath12k_peer *peer;
+	struct ath12k_dp_link_peer *peer;
 	struct ath12k_sta *ahsta;
 	u16 ml_peer_id;
 	int ret;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ar->ab);
 
 	lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy);
 
+	dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, link_id);
+
 	if (ar->num_peers > (ar->max_num_peers - 1)) {
 		ath12k_warn(ar->ab,
 			    "failed to create peer due to insufficient peer entry resource in firmware\n");
 		return -ENOBUFS;
 	}
 
-	spin_lock_bh(&ar->ab->base_lock);
-	peer = ath12k_peer_find_by_pdev_idx(ar->ab, ar->pdev_idx, arg->peer_addr);
+	spin_lock_bh(&dp->dp_lock);
+	peer = ath12k_dp_link_peer_find_by_pdev_and_addr(dp, ar->pdev_idx,
+							 arg->peer_addr);
 	if (peer) {
-		spin_unlock_bh(&ar->ab->base_lock);
+		spin_unlock_bh(&dp->dp_lock);
 		return -EINVAL;
 	}
-	spin_unlock_bh(&ar->ab->base_lock);
+	spin_unlock_bh(&dp->dp_lock);
 
 	ret = ath12k_wmi_send_peer_create_cmd(ar, arg);
 	if (ret) {
@@ -352,11 +179,12 @@ int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif,
 	if (ret)
 		return ret;
 
-	spin_lock_bh(&ar->ab->base_lock);
+	spin_lock_bh(&dp->dp_lock);
 
-	peer = ath12k_peer_find(ar->ab, arg->vdev_id, arg->peer_addr);
+	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arg->vdev_id,
+							 arg->peer_addr);
 	if (!peer) {
-		spin_unlock_bh(&ar->ab->base_lock);
+		spin_unlock_bh(&dp->dp_lock);
 		ath12k_warn(ar->ab, "failed to find peer %pM on vdev %i after creation\n",
 			    arg->peer_addr, arg->vdev_id);
 
@@ -382,13 +210,10 @@ int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif,
 	peer->sta = sta;
 
 	if (vif->type == NL80211_IFTYPE_STATION) {
-		arvif->ast_hash = peer->ast_hash;
-		arvif->ast_idx = peer->hw_peer_id;
+		dp_link_vif->ast_hash = peer->ast_hash;
+		dp_link_vif->ast_idx = peer->hw_peer_id;
 	}
 
-	if (vif->type == NL80211_IFTYPE_AP)
-		peer->ucast_ra_only = true;
-
 	if (sta) {
 		ahsta = ath12k_sta_to_ahsta(sta);
 		arsta = wiphy_dereference(ath12k_ar_to_hw(ar)->wiphy,
@@ -412,17 +237,22 @@ int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif,
 		}
 	}
 
-	peer->sec_type = HAL_ENCRYPT_TYPE_OPEN;
-	peer->sec_type_grp = HAL_ENCRYPT_TYPE_OPEN;
-
 	ar->num_peers++;
 
-	spin_unlock_bh(&ar->ab->base_lock);
+	spin_unlock_bh(&dp->dp_lock);
 
-	return 0;
+	if (arvif->link_id < IEEE80211_MLD_MAX_NUM_LINKS) {
+		ret = ath12k_dp_link_peer_assign(ath12k_ab_to_dp(ar->ab),
+						 &(ath12k_ar_to_ah(ar)->dp_hw),
+						 arvif->vdev_id, sta,
+						 (u8 *)arg->peer_addr, link_id,
+						 ar->hw_link_id);
+	}
+
+	return ret;
 }
 
-static u16 ath12k_peer_ml_alloc(struct ath12k_hw *ah)
+u16 ath12k_peer_ml_alloc(struct ath12k_hw *ah)
 {
 	u16 ml_peer_id;
 
@@ -442,68 +272,6 @@ static u16 ath12k_peer_ml_alloc(struct ath12k_hw *ah)
 	return ml_peer_id;
 }
 
-int ath12k_peer_ml_create(struct ath12k_hw *ah, struct ieee80211_sta *sta)
-{
-	struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(sta);
-	struct ath12k_ml_peer *ml_peer;
-
-	lockdep_assert_wiphy(ah->hw->wiphy);
-
-	if (!sta->mlo)
-		return -EINVAL;
-
-	ml_peer = ath12k_peer_ml_find(ah, sta->addr);
-	if (ml_peer) {
-		ath12k_hw_warn(ah, "ML peer %d exists already, unable to add new entry for %pM",
-			       ml_peer->id, sta->addr);
-		return -EEXIST;
-	}
-
-	ml_peer = kzalloc(sizeof(*ml_peer), GFP_ATOMIC);
-	if (!ml_peer)
-		return -ENOMEM;
-
-	ahsta->ml_peer_id = ath12k_peer_ml_alloc(ah);
-
-	if (ahsta->ml_peer_id == ATH12K_MLO_PEER_ID_INVALID) {
-		ath12k_hw_warn(ah, "unable to allocate ML peer id for sta %pM",
-			       sta->addr);
-		kfree(ml_peer);
-		return -ENOMEM;
-	}
-
-	ether_addr_copy(ml_peer->addr, sta->addr);
-	ml_peer->id = ahsta->ml_peer_id;
-	list_add(&ml_peer->list, &ah->ml_peers);
-
-	return 0;
-}
-
-int ath12k_peer_ml_delete(struct ath12k_hw *ah, struct ieee80211_sta *sta)
-{
-	struct ath12k_sta *ahsta = ath12k_sta_to_ahsta(sta);
-	struct ath12k_ml_peer *ml_peer;
-
-	lockdep_assert_wiphy(ah->hw->wiphy);
-
-	if (!sta->mlo)
-		return -EINVAL;
-
-	clear_bit(ahsta->ml_peer_id, ah->free_ml_peer_id_map);
-	ahsta->ml_peer_id = ATH12K_MLO_PEER_ID_INVALID;
-
-	ml_peer = ath12k_peer_ml_find(ah, sta->addr);
-	if (!ml_peer) {
-		ath12k_hw_warn(ah, "ML peer for %pM not found", sta->addr);
-		return -EINVAL;
-	}
-
-	list_del(&ml_peer->list);
-	kfree(ml_peer);
-
-	return 0;
-}
-
 int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_sta *ahsta)
 {
 	struct ieee80211_sta *sta = ath12k_ahsta_to_sta(ahsta);
@@ -536,6 +304,11 @@ int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_st
 
 		ath12k_dp_peer_cleanup(ar, arvif->vdev_id, arsta->addr);
 
+		ath12k_dp_link_peer_unassign(ath12k_ab_to_dp(ar->ab),
+					     &(ath12k_ar_to_ah(ar)->dp_hw),
+					     arvif->vdev_id, arsta->addr,
+					     ar->hw_link_id);
+
 		ret = ath12k_peer_delete_send(ar, arvif->vdev_id, arsta->addr);
 		if (ret) {
 			ath12k_warn(ar->ab,
@@ -568,3 +341,119 @@ int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_st
 
 	return err_ret;
 }
+
+static int ath12k_link_sta_rhash_insert(struct ath12k_base *ab,
+					struct ath12k_link_sta *arsta)
+{
+	struct ath12k_link_sta *tmp;
+
+	lockdep_assert_held(&ab->base_lock);
+
+	tmp = rhashtable_lookup_get_insert_fast(ab->rhead_sta_addr, &arsta->rhash_addr,
+						ab->rhash_sta_addr_param);
+	if (!tmp)
+		return 0;
+	else if (IS_ERR(tmp))
+		return PTR_ERR(tmp);
+	else
+		return -EEXIST;
+}
+
+static int ath12k_link_sta_rhash_remove(struct ath12k_base *ab,
+					struct ath12k_link_sta *arsta)
+{
+	int ret;
+
+	lockdep_assert_held(&ab->base_lock);
+
+	ret = rhashtable_remove_fast(ab->rhead_sta_addr, &arsta->rhash_addr,
+				     ab->rhash_sta_addr_param);
+	if (ret && ret != -ENOENT)
+		return ret;
+
+	return 0;
+}
+
+int ath12k_link_sta_rhash_add(struct ath12k_base *ab,
+			      struct ath12k_link_sta *arsta)
+{
+	int ret;
+
+	lockdep_assert_held(&ab->base_lock);
+
+	ret = ath12k_link_sta_rhash_insert(ab, arsta);
+	if (ret)
+		ath12k_warn(ab, "failed to add arsta %pM in rhash_addr ret %d\n",
+			    arsta->addr, ret);
+
+	return ret;
+}
+
+void ath12k_link_sta_rhash_delete(struct ath12k_base *ab,
+				  struct ath12k_link_sta *arsta)
+{
+	/*
+	 * Return type of this function is void since there is nothing to be
+	 * done in failure case
+	 */
+	int ret;
+
+	lockdep_assert_held(&ab->base_lock);
+
+	ret = ath12k_link_sta_rhash_remove(ab, arsta);
+	if (ret)
+		ath12k_warn(ab,
+			    "failed to remove arsta %pM in rhash_addr ret %d\n",
+			    arsta->addr, ret);
+}
+
+int ath12k_link_sta_rhash_tbl_init(struct ath12k_base *ab)
+{
+	struct rhashtable_params *param;
+	struct rhashtable *rhash_addr_tbl;
+	int ret;
+
+	rhash_addr_tbl = kzalloc(sizeof(*ab->rhead_sta_addr), GFP_KERNEL);
+	if (!rhash_addr_tbl)
+		return -ENOMEM;
+
+	param = &ab->rhash_sta_addr_param;
+
+	param->key_offset = offsetof(struct ath12k_link_sta, addr);
+	param->head_offset = offsetof(struct ath12k_link_sta, rhash_addr);
+	param->key_len = sizeof_field(struct ath12k_link_sta, addr);
+	param->automatic_shrinking = true;
+	param->nelem_hint = ab->num_radios * ath12k_core_get_max_peers_per_radio(ab);
+
+	ret = rhashtable_init(rhash_addr_tbl, param);
+	if (ret) {
+		ath12k_warn(ab, "failed to init peer addr rhash table %d\n",
+			    ret);
+		goto err_free;
+	}
+
+	ab->rhead_sta_addr = rhash_addr_tbl;
+
+	return 0;
+
+err_free:
+	kfree(rhash_addr_tbl);
+
+	return ret;
+}
+
+void ath12k_link_sta_rhash_tbl_destroy(struct ath12k_base *ab)
+{
+	rhashtable_destroy(ab->rhead_sta_addr);
+	kfree(ab->rhead_sta_addr);
+	ab->rhead_sta_addr = NULL;
+}
+
+struct ath12k_link_sta *ath12k_link_sta_find_by_addr(struct ath12k_base *ab,
+						     const u8 *addr)
+{
+	lockdep_assert_held(&ab->base_lock);
+
+	return rhashtable_lookup_fast(ab->rhead_sta_addr, addr,
+				      ab->rhash_sta_addr_param);
+}
diff --git a/drivers/net/wireless/ath/ath12k/peer.h b/drivers/net/wireless/ath/ath12k/peer.h
index 44afc0b7dd53..49d89796bc46 100644
--- a/drivers/net/wireless/ath/ath12k/peer.h
+++ b/drivers/net/wireless/ath/ath12k/peer.h
@@ -1,84 +1,14 @@
 /* SPDX-License-Identifier: BSD-3-Clause-Clear */
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #ifndef ATH12K_PEER_H
 #define ATH12K_PEER_H
 
-#include "dp_rx.h"
+#include "dp_peer.h"
 
-struct ppdu_user_delayba {
-	u16 sw_peer_id;
-	u32 info0;
-	u16 ru_end;
-	u16 ru_start;
-	u32 info1;
-	u32 rate_flags;
-	u32 resp_rate_flags;
-};
-
-#define ATH12K_PEER_ML_ID_VALID         BIT(13)
-
-struct ath12k_peer {
-	struct list_head list;
-	struct ieee80211_sta *sta;
-	int vdev_id;
-	u8 addr[ETH_ALEN];
-	int peer_id;
-	u16 ast_hash;
-	u8 pdev_idx;
-	u16 hw_peer_id;
-
-	/* protected by ab->data_lock */
-	struct ieee80211_key_conf *keys[WMI_MAX_KEY_INDEX + 1];
-	struct ath12k_dp_rx_tid rx_tid[IEEE80211_NUM_TIDS + 1];
-
-	/* Info used in MMIC verification of
-	 * RX fragments
-	 */
-	struct crypto_shash *tfm_mmic;
-	u8 mcast_keyidx;
-	u8 ucast_keyidx;
-	u16 sec_type;
-	u16 sec_type_grp;
-	struct ppdu_user_delayba ppdu_stats_delayba;
-	bool delayba_flag;
-	bool is_authorized;
-	bool mlo;
-	/* protected by ab->data_lock */
-	bool dp_setup_done;
-
-	u16 ml_id;
-
-	/* any other ML info common for all partners can be added
-	 * here and would be same for all partner peers.
-	 */
-	u8 ml_addr[ETH_ALEN];
-
-	/* To ensure only certain work related to dp is done once */
-	bool primary_link;
-
-	/* for reference to ath12k_link_sta */
-	u8 link_id;
-	bool ucast_ra_only;
-};
-
-struct ath12k_ml_peer {
-	struct list_head list;
-	u8 addr[ETH_ALEN];
-	u16 id;
-};
-
-void ath12k_peer_unmap_event(struct ath12k_base *ab, u16 peer_id);
-void ath12k_peer_map_event(struct ath12k_base *ab, u8 vdev_id, u16 peer_id,
-			   u8 *mac_addr, u16 ast_hash, u16 hw_peer_id);
-struct ath12k_peer *ath12k_peer_find(struct ath12k_base *ab, int vdev_id,
-				     const u8 *addr);
-struct ath12k_peer *ath12k_peer_find_by_addr(struct ath12k_base *ab,
-					     const u8 *addr);
-struct ath12k_peer *ath12k_peer_find_by_id(struct ath12k_base *ab, int peer_id);
 void ath12k_peer_cleanup(struct ath12k *ar, u32 vdev_id);
 int ath12k_peer_delete(struct ath12k *ar, u32 vdev_id, u8 *addr);
 int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif,
@@ -86,38 +16,14 @@ int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif,
 		       struct ath12k_wmi_peer_create_arg *arg);
 int ath12k_wait_for_peer_delete_done(struct ath12k *ar, u32 vdev_id,
 				     const u8 *addr);
-bool ath12k_peer_exist_by_vdev_id(struct ath12k_base *ab, int vdev_id);
-struct ath12k_peer *ath12k_peer_find_by_ast(struct ath12k_base *ab, int ast_hash);
-int ath12k_peer_ml_create(struct ath12k_hw *ah, struct ieee80211_sta *sta);
-int ath12k_peer_ml_delete(struct ath12k_hw *ah, struct ieee80211_sta *sta);
 int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_sta *ahsta);
 struct ath12k_ml_peer *ath12k_peer_ml_find(struct ath12k_hw *ah,
 					   const u8 *addr);
-static inline
-struct ath12k_link_sta *ath12k_peer_get_link_sta(struct ath12k_base *ab,
-						 struct ath12k_peer *peer)
-{
-	struct ath12k_sta *ahsta;
-	struct ath12k_link_sta *arsta;
-
-	if (!peer->sta)
-		return NULL;
-
-	ahsta = ath12k_sta_to_ahsta(peer->sta);
-	if (peer->ml_id & ATH12K_PEER_ML_ID_VALID) {
-		if (!(ahsta->links_map & BIT(peer->link_id))) {
-			ath12k_warn(ab, "peer %pM id %d link_id %d can't found in STA link_map 0x%x\n",
-				    peer->addr, peer->peer_id, peer->link_id,
-				    ahsta->links_map);
-			return NULL;
-		}
-		arsta = rcu_dereference(ahsta->link[peer->link_id]);
-		if (!arsta)
-			return NULL;
-	} else {
-		arsta =  &ahsta->deflink;
-	}
-	return arsta;
-}
-
+int ath12k_link_sta_rhash_tbl_init(struct ath12k_base *ab);
+void ath12k_link_sta_rhash_tbl_destroy(struct ath12k_base *ab);
+void ath12k_link_sta_rhash_delete(struct ath12k_base *ab, struct ath12k_link_sta *arsta);
+int ath12k_link_sta_rhash_add(struct ath12k_base *ab, struct ath12k_link_sta *arsta);
+struct ath12k_link_sta *ath12k_link_sta_find_by_addr(struct ath12k_base *ab,
+						     const u8 *addr);
+u16 ath12k_peer_ml_alloc(struct ath12k_hw *ah);
 #endif /* _PEER_H_ */
diff --git a/drivers/net/wireless/ath/ath12k/rx_desc.h b/drivers/net/wireless/ath/ath12k/rx_desc.h
deleted file mode 100644
index 6c600473b402..000000000000
--- a/drivers/net/wireless/ath/ath12k/rx_desc.h
+++ /dev/null
@@ -1,1544 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause-Clear */
-/*
- * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
- */
-#ifndef ATH12K_RX_DESC_H
-#define ATH12K_RX_DESC_H
-
-enum rx_desc_decap_type {
-	RX_DESC_DECAP_TYPE_RAW,
-	RX_DESC_DECAP_TYPE_NATIVE_WIFI,
-	RX_DESC_DECAP_TYPE_ETHERNET2_DIX,
-	RX_DESC_DECAP_TYPE_8023,
-};
-
-enum rx_desc_decrypt_status_code {
-	RX_DESC_DECRYPT_STATUS_CODE_OK,
-	RX_DESC_DECRYPT_STATUS_CODE_UNPROTECTED_FRAME,
-	RX_DESC_DECRYPT_STATUS_CODE_DATA_ERR,
-	RX_DESC_DECRYPT_STATUS_CODE_KEY_INVALID,
-	RX_DESC_DECRYPT_STATUS_CODE_PEER_ENTRY_INVALID,
-	RX_DESC_DECRYPT_STATUS_CODE_OTHER,
-};
-
-#define RX_MPDU_START_INFO0_REO_DEST_IND		GENMASK(4, 0)
-#define RX_MPDU_START_INFO0_LMAC_PEER_ID_MSB		GENMASK(6, 5)
-#define RX_MPDU_START_INFO0_FLOW_ID_TOEPLITZ		BIT(7)
-#define RX_MPDU_START_INFO0_PKT_SEL_FP_UCAST_DATA	BIT(8)
-#define RX_MPDU_START_INFO0_PKT_SEL_FP_MCAST_DATA	BIT(9)
-#define RX_MPDU_START_INFO0_PKT_SEL_FP_CTRL_BAR		BIT(10)
-#define RX_MPDU_START_INFO0_RXDMA0_SRC_RING_SEL		GENMASK(13, 11)
-#define RX_MPDU_START_INFO0_RXDMA0_DST_RING_SEL		GENMASK(16, 14)
-#define RX_MPDU_START_INFO0_MCAST_ECHO_DROP_EN		BIT(17)
-#define RX_MPDU_START_INFO0_WDS_LEARN_DETECT_EN		BIT(18)
-#define RX_MPDU_START_INFO0_INTRA_BSS_CHECK_EN		BIT(19)
-#define RX_MPDU_START_INFO0_USE_PPE			BIT(20)
-#define RX_MPDU_START_INFO0_PPE_ROUTING_EN		BIT(21)
-
-#define RX_MPDU_START_INFO1_REO_QUEUE_DESC_HI		GENMASK(7, 0)
-#define RX_MPDU_START_INFO1_RECV_QUEUE_NUM		GENMASK(23, 8)
-#define RX_MPDU_START_INFO1_PRE_DELIM_ERR_WARN		BIT(24)
-#define RX_MPDU_START_INFO1_FIRST_DELIM_ERR		BIT(25)
-
-#define RX_MPDU_START_INFO2_EPD_EN			BIT(0)
-#define RX_MPDU_START_INFO2_ALL_FRAME_ENCPD		BIT(1)
-#define RX_MPDU_START_INFO2_ENC_TYPE			GENMASK(5, 2)
-#define RX_MPDU_START_INFO2_VAR_WEP_KEY_WIDTH		GENMASK(7, 6)
-#define RX_MPDU_START_INFO2_MESH_STA			GENMASK(9, 8)
-#define RX_MPDU_START_INFO2_BSSID_HIT			BIT(10)
-#define RX_MPDU_START_INFO2_BSSID_NUM			GENMASK(14, 11)
-#define RX_MPDU_START_INFO2_TID				GENMASK(18, 15)
-
-#define RX_MPDU_START_INFO3_RXPCU_MPDU_FLTR		GENMASK(1, 0)
-#define RX_MPDU_START_INFO3_SW_FRAME_GRP_ID		GENMASK(8, 2)
-#define RX_MPDU_START_INFO3_NDP_FRAME			BIT(9)
-#define RX_MPDU_START_INFO3_PHY_ERR			BIT(10)
-#define RX_MPDU_START_INFO3_PHY_ERR_MPDU_HDR		BIT(11)
-#define RX_MPDU_START_INFO3_PROTO_VER_ERR		BIT(12)
-#define RX_MPDU_START_INFO3_AST_LOOKUP_VALID		BIT(13)
-#define RX_MPDU_START_INFO3_RANGING			BIT(14)
-
-#define RX_MPDU_START_INFO4_MPDU_FCTRL_VALID		BIT(0)
-#define RX_MPDU_START_INFO4_MPDU_DUR_VALID		BIT(1)
-#define RX_MPDU_START_INFO4_MAC_ADDR1_VALID		BIT(2)
-#define RX_MPDU_START_INFO4_MAC_ADDR2_VALID		BIT(3)
-#define RX_MPDU_START_INFO4_MAC_ADDR3_VALID		BIT(4)
-#define RX_MPDU_START_INFO4_MAC_ADDR4_VALID		BIT(5)
-#define RX_MPDU_START_INFO4_MPDU_SEQ_CTRL_VALID		BIT(6)
-#define RX_MPDU_START_INFO4_MPDU_QOS_CTRL_VALID		BIT(7)
-#define RX_MPDU_START_INFO4_MPDU_HT_CTRL_VALID		BIT(8)
-#define RX_MPDU_START_INFO4_ENCRYPT_INFO_VALID		BIT(9)
-#define RX_MPDU_START_INFO4_MPDU_FRAG_NUMBER		GENMASK(13, 10)
-#define RX_MPDU_START_INFO4_MORE_FRAG_FLAG		BIT(14)
-#define RX_MPDU_START_INFO4_FROM_DS			BIT(16)
-#define RX_MPDU_START_INFO4_TO_DS			BIT(17)
-#define RX_MPDU_START_INFO4_ENCRYPTED			BIT(18)
-#define RX_MPDU_START_INFO4_MPDU_RETRY			BIT(19)
-#define RX_MPDU_START_INFO4_MPDU_SEQ_NUM		GENMASK(31, 20)
-
-#define RX_MPDU_START_INFO5_KEY_ID			GENMASK(7, 0)
-#define RX_MPDU_START_INFO5_NEW_PEER_ENTRY		BIT(8)
-#define RX_MPDU_START_INFO5_DECRYPT_NEEDED		BIT(9)
-#define RX_MPDU_START_INFO5_DECAP_TYPE			GENMASK(11, 10)
-#define RX_MPDU_START_INFO5_VLAN_TAG_C_PADDING		BIT(12)
-#define RX_MPDU_START_INFO5_VLAN_TAG_S_PADDING		BIT(13)
-#define RX_MPDU_START_INFO5_STRIP_VLAN_TAG_C		BIT(14)
-#define RX_MPDU_START_INFO5_STRIP_VLAN_TAG_S		BIT(15)
-#define RX_MPDU_START_INFO5_PRE_DELIM_COUNT		GENMASK(27, 16)
-#define RX_MPDU_START_INFO5_AMPDU_FLAG			BIT(28)
-#define RX_MPDU_START_INFO5_BAR_FRAME			BIT(29)
-#define RX_MPDU_START_INFO5_RAW_MPDU			BIT(30)
-
-#define RX_MPDU_START_INFO6_MPDU_LEN			GENMASK(13, 0)
-#define RX_MPDU_START_INFO6_FIRST_MPDU			BIT(14)
-#define RX_MPDU_START_INFO6_MCAST_BCAST			BIT(15)
-#define RX_MPDU_START_INFO6_AST_IDX_NOT_FOUND		BIT(16)
-#define RX_MPDU_START_INFO6_AST_IDX_TIMEOUT		BIT(17)
-#define RX_MPDU_START_INFO6_POWER_MGMT			BIT(18)
-#define RX_MPDU_START_INFO6_NON_QOS			BIT(19)
-#define RX_MPDU_START_INFO6_NULL_DATA			BIT(20)
-#define RX_MPDU_START_INFO6_MGMT_TYPE			BIT(21)
-#define RX_MPDU_START_INFO6_CTRL_TYPE			BIT(22)
-#define RX_MPDU_START_INFO6_MORE_DATA			BIT(23)
-#define RX_MPDU_START_INFO6_EOSP			BIT(24)
-#define RX_MPDU_START_INFO6_FRAGMENT			BIT(25)
-#define RX_MPDU_START_INFO6_ORDER			BIT(26)
-#define RX_MPDU_START_INFO6_UAPSD_TRIGGER		BIT(27)
-#define RX_MPDU_START_INFO6_ENCRYPT_REQUIRED		BIT(28)
-#define RX_MPDU_START_INFO6_DIRECTED			BIT(29)
-#define RX_MPDU_START_INFO6_AMSDU_PRESENT		BIT(30)
-
-#define RX_MPDU_START_INFO7_VDEV_ID			GENMASK(7, 0)
-#define RX_MPDU_START_INFO7_SERVICE_CODE		GENMASK(16, 8)
-#define RX_MPDU_START_INFO7_PRIORITY_VALID		BIT(17)
-#define RX_MPDU_START_INFO7_SRC_INFO			GENMASK(29, 18)
-
-#define RX_MPDU_START_INFO8_AUTH_TO_SEND_WDS		BIT(0)
-
-struct rx_mpdu_start_qcn9274 {
-	__le32 info0;
-	__le32 reo_queue_desc_lo;
-	__le32 info1;
-	__le32 pn[4];
-	__le32 info2;
-	__le32 peer_meta_data;
-	__le16 info3;
-	__le16 phy_ppdu_id;
-	__le16 ast_index;
-	__le16 sw_peer_id;
-	__le32 info4;
-	__le32 info5;
-	__le32 info6;
-	__le16 frame_ctrl;
-	__le16 duration;
-	u8 addr1[ETH_ALEN];
-	u8 addr2[ETH_ALEN];
-	u8 addr3[ETH_ALEN];
-	__le16 seq_ctrl;
-	u8 addr4[ETH_ALEN];
-	__le16 qos_ctrl;
-	__le32 ht_ctrl;
-	__le32 info7;
-	u8 multi_link_addr1[ETH_ALEN];
-	u8 multi_link_addr2[ETH_ALEN];
-	__le32 info8;
-	__le32 res0;
-	__le32 res1;
-} __packed;
-
-#define QCN9274_MPDU_START_SELECT_MPDU_START_TAG			BIT(0)
-#define QCN9274_MPDU_START_SELECT_INFO0_REO_QUEUE_DESC_LO		BIT(1)
-#define QCN9274_MPDU_START_SELECT_INFO1_PN_31_0				BIT(2)
-#define QCN9274_MPDU_START_SELECT_PN_95_32				BIT(3)
-#define QCN9274_MPDU_START_SELECT_PN_127_96_INFO2			BIT(4)
-#define QCN9274_MPDU_START_SELECT_PEER_MDATA_INFO3_PHY_PPDU_ID		BIT(5)
-#define QCN9274_MPDU_START_SELECT_AST_IDX_SW_PEER_ID_INFO4		BIT(6)
-#define QCN9274_MPDU_START_SELECT_INFO5_INFO6				BIT(7)
-#define QCN9274_MPDU_START_SELECT_FRAME_CTRL_DURATION_ADDR1_31_0	BIT(8)
-#define QCN9274_MPDU_START_SELECT_ADDR2_47_0_ADDR1_47_32		BIT(9)
-#define QCN9274_MPDU_START_SELECT_ADDR3_47_0_SEQ_CTRL			BIT(10)
-#define QCN9274_MPDU_START_SELECT_ADDR4_47_0_QOS_CTRL			BIT(11)
-#define QCN9274_MPDU_START_SELECT_HT_CTRL_INFO7				BIT(12)
-#define QCN9274_MPDU_START_SELECT_ML_ADDR1_47_0_ML_ADDR2_15_0		BIT(13)
-#define QCN9274_MPDU_START_SELECT_ML_ADDR2_47_16_INFO8			BIT(14)
-#define QCN9274_MPDU_START_SELECT_RES_0_RES_1				BIT(15)
-
-#define QCN9274_MPDU_START_WMASK (QCN9274_MPDU_START_SELECT_INFO1_PN_31_0 |	\
-		QCN9274_MPDU_START_SELECT_PN_95_32 |				\
-		QCN9274_MPDU_START_SELECT_PN_127_96_INFO2 |			\
-		QCN9274_MPDU_START_SELECT_PEER_MDATA_INFO3_PHY_PPDU_ID |	\
-		QCN9274_MPDU_START_SELECT_AST_IDX_SW_PEER_ID_INFO4 |		\
-		QCN9274_MPDU_START_SELECT_INFO5_INFO6 |				\
-		QCN9274_MPDU_START_SELECT_FRAME_CTRL_DURATION_ADDR1_31_0 |	\
-		QCN9274_MPDU_START_SELECT_ADDR2_47_0_ADDR1_47_32 |		\
-		QCN9274_MPDU_START_SELECT_ADDR3_47_0_SEQ_CTRL |			\
-		QCN9274_MPDU_START_SELECT_ADDR4_47_0_QOS_CTRL)
-
-/* The below rx_mpdu_start_qcn9274_compact structure is tied with the mask
- * value QCN9274_MPDU_START_WMASK. If the mask value changes the structure
- * will also change.
- */
-
-struct rx_mpdu_start_qcn9274_compact {
-	__le32 info1;
-	__le32 pn[4];
-	__le32 info2;
-	__le32 peer_meta_data;
-	__le16 info3;
-	__le16 phy_ppdu_id;
-	__le16 ast_index;
-	__le16 sw_peer_id;
-	__le32 info4;
-	__le32 info5;
-	__le32 info6;
-	__le16 frame_ctrl;
-	__le16 duration;
-	u8 addr1[ETH_ALEN];
-	u8 addr2[ETH_ALEN];
-	u8 addr3[ETH_ALEN];
-	__le16 seq_ctrl;
-	u8 addr4[ETH_ALEN];
-	__le16 qos_ctrl;
-} __packed;
-
-/* rx_mpdu_start
- *
- * reo_destination_indication
- *		The id of the reo exit ring where the msdu frame shall push
- *		after (MPDU level) reordering has finished. Values are defined
- *		in enum %HAL_RX_MSDU_DESC_REO_DEST_IND_.
- *
- * lmac_peer_id_msb
- *
- *		If use_flow_id_toeplitz_clfy is set and lmac_peer_id_'sb
- *		is 2'b00, Rx OLE uses a REO destination indicati'n of {1'b1,
- *		hash[3:0]} using the chosen Toeplitz hash from Common Parser
- *		if flow search fails.
- *		If use_flow_id_toeplitz_clfy is set and lmac_peer_id_msb
- *		's not 2'b00, Rx OLE uses a REO destination indication of
- *		{lmac_peer_id_msb, hash[2:0]} using the chosen Toeplitz
- *		hash from Common Parser if flow search fails.
- *
- * use_flow_id_toeplitz_clfy
- *		Indication to Rx OLE to enable REO destination routing based
- *		on the chosen Toeplitz hash from Common Parser, in case
- *		flow search fails
- *
- * pkt_selection_fp_ucast_data
- *		Filter pass Unicast data frame (matching rxpcu_filter_pass
- *		and sw_frame_group_Unicast_data) routing selection
- *
- * pkt_selection_fp_mcast_data
- *		Filter pass Multicast data frame (matching rxpcu_filter_pass
- *		and sw_frame_group_Multicast_data) routing selection
- *
- * pkt_selection_fp_ctrl_bar
- *		Filter pass BAR frame (matching rxpcu_filter_pass
- *		and sw_frame_group_ctrl_1000) routing selection
- *
- * rxdma0_src_ring_selection
- *		Field only valid when for the received frame type the corresponding
- *		pkt_selection_fp_... bit is set
- *
- * rxdma0_dst_ring_selection
- *		Field only valid when for the received frame type the corresponding
- *		pkt_selection_fp_... bit is set
- *
- * mcast_echo_drop_enable
- *		If set, for multicast packets, multicast echo check (i.e.
- *		SA search with mcast_echo_check = 1) shall be performed
- *		by RXOLE, and any multicast echo packets should be indicated
- *		 to RXDMA for release to WBM
- *
- * wds_learning_detect_en
- *		If set, WDS learning detection based on SA search and notification
- *		to FW (using RXDMA0 status ring) is enabled and the "timestamp"
- *		field in address search failure cache-only entry should
- *		be used to avoid multiple WDS learning notifications.
- *
- * intrabss_check_en
- *		If set, intra-BSS routing detection is enabled
- *
- * use_ppe
- *		Indicates to RXDMA to ignore the REO_destination_indication
- *		and use a programmed value corresponding to the REO2PPE
- *		ring
- *		This override to REO2PPE for packets requiring multiple
- *		buffers shall be disabled based on an RXDMA configuration,
- *		as PPE may not support such packets.
- *
- *		Supported only in full AP chips, not in client/soft
- *		chips
- *
- * ppe_routing_enable
- *		Global enable/disable bit for routing to PPE, used to disable
- *		PPE routing even if RXOLE CCE or flow search indicate 'Use_PPE'
- *		This is set by SW for peers which are being handled by a
- *		host SW/accelerator subsystem that also handles packet
- *		buffer management for WiFi-to-PPE routing.
- *
- *		This is cleared by SW for peers which are being handled
- *		by a different subsystem, completely disabling WiFi-to-PPE
- *		routing for such peers.
- *
- * rx_reo_queue_desc_addr_lo
- *		Address (lower 32 bits) of the REO queue descriptor.
- *
- * rx_reo_queue_desc_addr_hi
- *		Address (upper 8 bits) of the REO queue descriptor.
- *
- * receive_queue_number
- *		Indicates the MPDU queue ID to which this MPDU link
- *		descriptor belongs.
- *
- * pre_delim_err_warning
- *		Indicates that a delimiter FCS error was found in between the
- *		previous MPDU and this MPDU. Note that this is just a warning,
- *		and does not mean that this MPDU is corrupted in any way. If
- *		it is, there will be other errors indicated such as FCS or
- *		decrypt errors.
- *
- * first_delim_err
- *		Indicates that the first delimiter had a FCS failure.
- *
- * pn
- *		The PN number.
- *
- * epd_en
- *		Field only valid when AST_based_lookup_valid == 1.
- *		In case of ndp or phy_err or AST_based_lookup_valid == 0,
- *		this field will be set to 0
- *		If set to one use EPD instead of LPD
- *		In case of ndp or phy_err, this field will never be set.
- *
- * all_frames_shall_be_encrypted
- *		In case of ndp or phy_err or AST_based_lookup_valid == 0,
- *		this field will be set to 0
- *
- *		When set, all frames (data only ?) shall be encrypted. If
- *		not, RX CRYPTO shall set an error flag.
- *
- *
- * encrypt_type
- *		In case of ndp or phy_err or AST_based_lookup_valid == 0,
- *		this field will be set to 0
- *
- *		Indicates type of decrypt cipher used (as defined in the
- *		peer entry)
- *
- * wep_key_width_for_variable_key
- *
- *		Field only valid when key_type is set to wep_varied_width.
- *
- * mesh_sta
- *
- * bssid_hit
- *		When set, the BSSID of the incoming frame matched one of
- *		 the 8 BSSID register values
- * bssid_number
- *		Field only valid when bssid_hit is set.
- *		This number indicates which one out of the 8 BSSID register
- *		values matched the incoming frame
- *
- * tid
- *		Field only valid when mpdu_qos_control_valid is set
- *		The TID field in the QoS control field
- *
- * peer_meta_data
- *		Meta data that SW has programmed in the Peer table entry
- *		of the transmitting STA.
- *
- * rxpcu_mpdu_filter_in_category
- *		Field indicates what the reason was that this mpdu frame
- *		was allowed to come into the receive path by rxpcu. Values
- *		are defined in enum %RX_DESC_RXPCU_FILTER_*.
- *
- * sw_frame_group_id
- *		SW processes frames based on certain classifications. Values
- *		are defined in enum %RX_DESC_SW_FRAME_GRP_ID_*.
- *
- * ndp_frame
- *		When set, the received frame was an NDP frame, and thus
- *		there will be no MPDU data.
- * phy_err
- *		When set, a PHY error was received before MAC received any
- *		data, and thus there will be no MPDU data.
- *
- * phy_err_during_mpdu_header
- *		When set, a PHY error was received before MAC received the
- *		complete MPDU header which was needed for proper decoding
- *
- * protocol_version_err
- *		Set when RXPCU detected a version error in the Frame control
- *		field
- *
- * ast_based_lookup_valid
- *		When set, AST based lookup for this frame has found a valid
- *		result.
- *
- * ranging
- *		When set, a ranging NDPA or a ranging NDP was received.
- *
- * phy_ppdu_id
- *		A ppdu counter value that PHY increments for every PPDU
- *		received. The counter value wraps around.
- *
- * ast_index
- *
- *		This field indicates the index of the AST entry corresponding
- *		to this MPDU. It is provided by the GSE module instantiated
- *		in RXPCU.
- *		A value of 0xFFFF indicates an invalid AST index, meaning
- *		that No AST entry was found or NO AST search was performed
- *
- * sw_peer_id
- *		In case of ndp or phy_err or AST_based_lookup_valid == 0,
- *		this field will be set to 0
- *		This field indicates a unique peer identifier. It is set
- *		equal to field 'sw_peer_id' from the AST entry
- *
- * frame_control_valid
- *		When set, the field Mpdu_Frame_control_field has valid information
- *
- * frame_duration_valid
- *		When set, the field Mpdu_duration_field has valid information
- *
- * mac_addr_ad1..4_valid
- *		When set, the fields mac_addr_adx_..... have valid information
- *
- * mpdu_seq_ctrl_valid
- *
- *		When set, the fields mpdu_sequence_control_field and mpdu_sequence_number
- *		have valid information as well as field
- *		For MPDUs without a sequence control field, this field will
- *		not be set.
- *
- * mpdu_qos_ctrl_valid, mpdu_ht_ctrl_valid
- *
- *		When set, the field mpdu_qos_control_field, mpdu_ht_control has valid
- *		information, For MPDUs without a QoS,HT control field, this field
- *		will not be set.
- *
- * frame_encryption_info_valid
- *
- *		When set, the encryption related info fields, like IV and
- *		PN are valid
- *		For MPDUs that are not encrypted, this will not be set.
- *
- * mpdu_fragment_number
- *
- *		Field only valid when Mpdu_sequence_control_valid is set
- *		AND Fragment_flag is set. The fragment number from the 802.11 header
- *
- * more_fragment_flag
- *
- *		The More Fragment bit setting from the MPDU header of the
- *		received frame
- *
- * fr_ds
- *
- *		Field only valid when Mpdu_frame_control_valid is set
- *		Set if the from DS bit is set in the frame control.
- *
- * to_ds
- *
- *		Field only valid when Mpdu_frame_control_valid is set
- *		Set if the to DS bit is set in the frame control.
- *
- * encrypted
- *
- *		Field only valid when Mpdu_frame_control_valid is set.
- *		Protected bit from the frame control.
- *
- * mpdu_retry
- *		Field only valid when Mpdu_frame_control_valid is set.
- *		Retry bit from the frame control.  Only valid when first_msdu is set
- *
- * mpdu_sequence_number
- *		Field only valid when Mpdu_sequence_control_valid is set.
- *
- *		The sequence number from the 802.11 header.
- * key_id
- *		The key ID octet from the IV.
- *		Field only valid when Frame_encryption_info_valid is set
- *
- * new_peer_entry
- *		Set if new RX_PEER_ENTRY TLV follows. If clear, RX_PEER_ENTRY
- *		doesn't follow so RX DECRYPTION module either uses old peer
- *		entry or not decrypt.
- *
- * decrypt_needed
- *		When RXPCU sets bit 'ast_index_not_found or ast_index_timeout',
- *		RXPCU will also ensure that this bit is NOT set. CRYPTO for that
- *		reason only needs to evaluate this bit and non of the other ones
- *
- * decap_type
- *		Used by the OLE during decapsulation. Values are defined in
- *		enum %MPDU_START_DECAP_TYPE_*.
- *
- * rx_insert_vlan_c_tag_padding
- * rx_insert_vlan_s_tag_padding
- *		Insert 4 byte of all zeros as VLAN tag or double VLAN tag if
- *		the rx payload does not have VLAN.
- *
- * strip_vlan_c_tag_decap
- * strip_vlan_s_tag_decap
- *		Strip VLAN or double VLAN during decapsulation.
- *
- * pre_delim_count
- *		The number of delimiters before this MPDU. Note that this
- *		number is cleared at PPDU start. If this MPDU is the first
- *		received MPDU in the PPDU and this MPDU gets filtered-in,
- *		this field will indicate the number of delimiters located
- *		after the last MPDU in the previous PPDU.
- *
- *		If this MPDU is located after the first received MPDU in
- *		an PPDU, this field will indicate the number of delimiters
- *		located between the previous MPDU and this MPDU.
- *
- * ampdu_flag
- *		Received frame was part of an A-MPDU.
- *
- * bar_frame
- *		Received frame is a BAR frame
- *
- * raw_mpdu
- *		Set when no 802.11 to nwifi/ethernet hdr conversion is done
- *
- * mpdu_length
- *		MPDU length before decapsulation.
- *
- * first_mpdu
- *		Indicates the first MSDU of the PPDU.  If both first_mpdu
- *		and last_mpdu are set in the MSDU then this is a not an
- *		A-MPDU frame but a stand alone MPDU.  Interior MPDU in an
- *		A-MPDU shall have both first_mpdu and last_mpdu bits set to
- *		0.  The PPDU start status will only be valid when this bit
- *		is set.
- *
- * mcast_bcast
- *		Multicast / broadcast indicator.  Only set when the MAC
- *		address 1 bit 0 is set indicating mcast/bcast and the BSSID
- *		matches one of the 4 BSSID registers. Only set when
- *		first_msdu is set.
- *
- * ast_index_not_found
- *		Only valid when first_msdu is set. Indicates no AST matching
- *		entries within the max search count.
- *
- * ast_index_timeout
- *		Only valid when first_msdu is set. Indicates an unsuccessful
- *		search in the address search table due to timeout.
- *
- * power_mgmt
- *		Power management bit set in the 802.11 header.  Only set
- *		when first_msdu is set.
- *
- * non_qos
- *		Set if packet is not a non-QoS data frame.  Only set when
- *		first_msdu is set.
- *
- * null_data
- *		Set if frame type indicates either null data or QoS null
- *		data format.  Only set when first_msdu is set.
- *
- * mgmt_type
- *		Set if packet is a management packet.  Only set when
- *		first_msdu is set.
- *
- * ctrl_type
- *		Set if packet is a control packet.  Only set when first_msdu
- *		is set.
- *
- * more_data
- *		Set if more bit in frame control is set.  Only set when
- *		first_msdu is set.
- *
- * eosp
- *		Set if the EOSP (end of service period) bit in the QoS
- *		control field is set.  Only set when first_msdu is set.
- *
- *
- * fragment_flag
- *		Fragment indication
- *
- * order
- *		Set if the order bit in the frame control is set.  Only
- *		set when first_msdu is set.
- *
- * u_apsd_trigger
- *		U-APSD trigger frame
- *
- * encrypt_required
- *		Indicates that this data type frame is not encrypted even if
- *		the policy for this MPDU requires encryption as indicated in
- *		the peer table key type.
- *
- * directed
- *		MPDU is a directed packet which means that the RA matched
- *		our STA addresses.  In proxySTA it means that the TA matched
- *		an entry in our address search table with the corresponding
- *		'no_ack' bit is the address search entry cleared.
- * amsdu_present
- *		AMSDU present
- *
- * mpdu_frame_control_field
- *		Frame control field in header. Only valid when the field is marked valid.
- *
- * mpdu_duration_field
- *		Duration field in header. Only valid when the field is marked valid.
- *
- * mac_addr_adx
- *		MAC addresses in the received frame. Only valid when corresponding
- *		address valid bit is set
- *
- * mpdu_qos_control_field, mpdu_ht_control_field
- *		QoS/HT control fields from header. Valid only when corresponding fields
- *		are marked valid
- *
- * vdev_id
- *		Virtual device associated with this peer
- *		RXOLE uses this to determine intra-BSS routing.
- *
- * service_code
- *		Opaque service code between PPE and Wi-Fi
- *		This field gets passed on by REO to PPE in the EDMA descriptor
- *		('REO_TO_PPE_RING').
- *
- * priority_valid
- *		This field gets passed on by REO to PPE in the EDMA descriptor
- *		('REO_TO_PPE_RING').
- *
- * src_info
- *		Source (virtual) device/interface info. associated with
- *		this peer
- *		This field gets passed on by REO to PPE in the EDMA descriptor
- *		('REO_TO_PPE_RING').
- *
- * multi_link_addr_ad1_ad2_valid
- *		If set, Rx OLE shall convert Address1 and Address2 of received
- *		data frames to multi-link addresses during decapsulation to eth/nwifi
- *
- * multi_link_addr_ad1,ad2
- *		Multi-link receiver address1,2. Only valid when corresponding
- *		valid bit is set
- *
- * authorize_to_send_wds
- *		If not set, RXDMA shall perform error-routing for WDS packets
- *		as the sender is not authorized and might misuse WDS frame
- *		format to inject packets with arbitrary DA/SA.
- *
- */
-
-enum rx_msdu_start_pkt_type {
-	RX_MSDU_START_PKT_TYPE_11A,
-	RX_MSDU_START_PKT_TYPE_11B,
-	RX_MSDU_START_PKT_TYPE_11N,
-	RX_MSDU_START_PKT_TYPE_11AC,
-	RX_MSDU_START_PKT_TYPE_11AX,
-	RX_MSDU_START_PKT_TYPE_11BA,
-	RX_MSDU_START_PKT_TYPE_11BE,
-};
-
-enum rx_msdu_start_sgi {
-	RX_MSDU_START_SGI_0_8_US,
-	RX_MSDU_START_SGI_0_4_US,
-	RX_MSDU_START_SGI_1_6_US,
-	RX_MSDU_START_SGI_3_2_US,
-};
-
-enum rx_msdu_start_recv_bw {
-	RX_MSDU_START_RECV_BW_20MHZ,
-	RX_MSDU_START_RECV_BW_40MHZ,
-	RX_MSDU_START_RECV_BW_80MHZ,
-	RX_MSDU_START_RECV_BW_160MHZ,
-};
-
-enum rx_msdu_start_reception_type {
-	RX_MSDU_START_RECEPTION_TYPE_SU,
-	RX_MSDU_START_RECEPTION_TYPE_DL_MU_MIMO,
-	RX_MSDU_START_RECEPTION_TYPE_DL_MU_OFDMA,
-	RX_MSDU_START_RECEPTION_TYPE_DL_MU_OFDMA_MIMO,
-	RX_MSDU_START_RECEPTION_TYPE_UL_MU_MIMO,
-	RX_MSDU_START_RECEPTION_TYPE_UL_MU_OFDMA,
-	RX_MSDU_START_RECEPTION_TYPE_UL_MU_OFDMA_MIMO,
-};
-
-#define RX_MSDU_END_64_TLV_SRC_LINK_ID		GENMASK(24, 22)
-
-#define RX_MSDU_END_INFO0_RXPCU_MPDU_FITLER	GENMASK(1, 0)
-#define RX_MSDU_END_INFO0_SW_FRAME_GRP_ID	GENMASK(8, 2)
-
-#define RX_MSDU_END_INFO1_REPORTED_MPDU_LENGTH	GENMASK(13, 0)
-
-#define RX_MSDU_END_INFO2_CCE_SUPER_RULE	GENMASK(13, 8)
-#define RX_MSDU_END_INFO2_CCND_TRUNCATE		BIT(14)
-#define RX_MSDU_END_INFO2_CCND_CCE_DIS		BIT(15)
-
-#define RX_MSDU_END_INFO3_DA_OFFSET		GENMASK(5, 0)
-#define RX_MSDU_END_INFO3_SA_OFFSET		GENMASK(11, 6)
-#define RX_MSDU_END_INFO3_DA_OFFSET_VALID	BIT(12)
-#define RX_MSDU_END_INFO3_SA_OFFSET_VALID	BIT(13)
-
-#define RX_MSDU_END_INFO4_TCP_FLAG		GENMASK(8, 0)
-#define RX_MSDU_END_INFO4_LRO_ELIGIBLE		BIT(9)
-
-#define RX_MSDU_END_INFO5_SA_IDX_TIMEOUT	BIT(0)
-#define RX_MSDU_END_INFO5_DA_IDX_TIMEOUT	BIT(1)
-#define RX_MSDU_END_INFO5_TO_DS			BIT(2)
-#define RX_MSDU_END_INFO5_TID			GENMASK(6, 3)
-#define RX_MSDU_END_INFO5_SA_IS_VALID		BIT(7)
-#define RX_MSDU_END_INFO5_DA_IS_VALID		BIT(8)
-#define RX_MSDU_END_INFO5_DA_IS_MCBC		BIT(9)
-#define RX_MSDU_END_INFO5_L3_HDR_PADDING	GENMASK(11, 10)
-#define RX_MSDU_END_INFO5_FIRST_MSDU		BIT(12)
-#define RX_MSDU_END_INFO5_LAST_MSDU		BIT(13)
-#define RX_MSDU_END_INFO5_FROM_DS		BIT(14)
-#define RX_MSDU_END_INFO5_IP_CHKSUM_FAIL_COPY	BIT(15)
-
-#define RX_MSDU_END_INFO6_MSDU_DROP		BIT(0)
-#define RX_MSDU_END_INFO6_REO_DEST_IND		GENMASK(5, 1)
-#define RX_MSDU_END_INFO6_FLOW_IDX		GENMASK(25, 6)
-#define RX_MSDU_END_INFO6_USE_PPE		BIT(26)
-#define RX_MSDU_END_INFO6_MESH_STA		GENMASK(28, 27)
-#define RX_MSDU_END_INFO6_VLAN_CTAG_STRIPPED	BIT(29)
-#define RX_MSDU_END_INFO6_VLAN_STAG_STRIPPED	BIT(30)
-#define RX_MSDU_END_INFO6_FRAGMENT_FLAG		BIT(31)
-
-#define RX_MSDU_END_INFO7_AGGR_COUNT		GENMASK(7, 0)
-#define RX_MSDU_END_INFO7_FLOW_AGGR_CONTN	BIT(8)
-#define RX_MSDU_END_INFO7_FISA_TIMEOUT		BIT(9)
-
-#define RX_MSDU_END_INFO7_TCPUDP_CSUM_FAIL_CPY	BIT(10)
-#define RX_MSDU_END_INFO7_MSDU_LIMIT_ERROR	BIT(11)
-#define RX_MSDU_END_INFO7_FLOW_IDX_TIMEOUT	BIT(12)
-#define RX_MSDU_END_INFO7_FLOW_IDX_INVALID	BIT(13)
-#define RX_MSDU_END_INFO7_CCE_MATCH		BIT(14)
-#define RX_MSDU_END_INFO7_AMSDU_PARSER_ERR	BIT(15)
-
-#define RX_MSDU_END_INFO8_KEY_ID		GENMASK(7, 0)
-
-#define RX_MSDU_END_INFO9_SERVICE_CODE		GENMASK(14, 6)
-#define RX_MSDU_END_INFO9_PRIORITY_VALID	BIT(15)
-#define RX_MSDU_END_INFO9_INRA_BSS		BIT(16)
-#define RX_MSDU_END_INFO9_DEST_CHIP_ID		GENMASK(18, 17)
-#define RX_MSDU_END_INFO9_MCAST_ECHO		BIT(19)
-#define RX_MSDU_END_INFO9_WDS_LEARN_EVENT	BIT(20)
-#define RX_MSDU_END_INFO9_WDS_ROAM_EVENT	BIT(21)
-#define RX_MSDU_END_INFO9_WDS_KEEP_ALIVE_EVENT	BIT(22)
-
-#define RX_MSDU_END_INFO10_MSDU_LENGTH		GENMASK(13, 0)
-#define RX_MSDU_END_INFO10_STBC			BIT(14)
-#define RX_MSDU_END_INFO10_IPSEC_ESP		BIT(15)
-#define RX_MSDU_END_INFO10_L3_OFFSET		GENMASK(22, 16)
-#define RX_MSDU_END_INFO10_IPSEC_AH		BIT(23)
-#define RX_MSDU_END_INFO10_L4_OFFSET		GENMASK(31, 24)
-
-#define RX_MSDU_END_INFO11_MSDU_NUMBER		GENMASK(7, 0)
-#define RX_MSDU_END_INFO11_DECAP_FORMAT		GENMASK(9, 8)
-#define RX_MSDU_END_INFO11_IPV4			BIT(10)
-#define RX_MSDU_END_INFO11_IPV6			BIT(11)
-#define RX_MSDU_END_INFO11_TCP			BIT(12)
-#define RX_MSDU_END_INFO11_UDP			BIT(13)
-#define RX_MSDU_END_INFO11_IP_FRAG		BIT(14)
-#define RX_MSDU_END_INFO11_TCP_ONLY_ACK		BIT(15)
-#define RX_MSDU_END_INFO11_DA_IS_BCAST_MCAST	BIT(16)
-#define RX_MSDU_END_INFO11_SEL_TOEPLITZ_HASH	GENMASK(18, 17)
-#define RX_MSDU_END_INFO11_IP_FIXED_HDR_VALID	BIT(19)
-#define RX_MSDU_END_INFO11_IP_EXTN_HDR_VALID	BIT(20)
-#define RX_MSDU_END_INFO11_IP_TCP_UDP_HDR_VALID	BIT(21)
-#define RX_MSDU_END_INFO11_MESH_CTRL_PRESENT	BIT(22)
-#define RX_MSDU_END_INFO11_LDPC			BIT(23)
-#define RX_MSDU_END_INFO11_IP4_IP6_NXT_HDR	GENMASK(31, 24)
-
-#define RX_MSDU_END_INFO12_USER_RSSI		GENMASK(7, 0)
-#define RX_MSDU_END_INFO12_PKT_TYPE		GENMASK(11, 8)
-#define RX_MSDU_END_INFO12_SGI			GENMASK(13, 12)
-#define RX_MSDU_END_INFO12_RATE_MCS		GENMASK(17, 14)
-#define RX_MSDU_END_INFO12_RECV_BW		GENMASK(20, 18)
-#define RX_MSDU_END_INFO12_RECEPTION_TYPE	GENMASK(23, 21)
-
-#define RX_MSDU_END_INFO12_MIMO_SS_BITMAP	GENMASK(30, 24)
-#define RX_MSDU_END_INFO12_MIMO_DONE_COPY	BIT(31)
-
-#define RX_MSDU_END_INFO13_FIRST_MPDU		BIT(0)
-#define RX_MSDU_END_INFO13_MCAST_BCAST		BIT(2)
-#define RX_MSDU_END_INFO13_AST_IDX_NOT_FOUND	BIT(3)
-#define RX_MSDU_END_INFO13_AST_IDX_TIMEDOUT	BIT(4)
-#define RX_MSDU_END_INFO13_POWER_MGMT		BIT(5)
-#define RX_MSDU_END_INFO13_NON_QOS		BIT(6)
-#define RX_MSDU_END_INFO13_NULL_DATA		BIT(7)
-#define RX_MSDU_END_INFO13_MGMT_TYPE		BIT(8)
-#define RX_MSDU_END_INFO13_CTRL_TYPE		BIT(9)
-#define RX_MSDU_END_INFO13_MORE_DATA		BIT(10)
-#define RX_MSDU_END_INFO13_EOSP			BIT(11)
-#define RX_MSDU_END_INFO13_A_MSDU_ERROR		BIT(12)
-#define RX_MSDU_END_INFO13_ORDER		BIT(14)
-#define RX_MSDU_END_INFO13_OVERFLOW_ERR		BIT(16)
-#define RX_MSDU_END_INFO13_MSDU_LEN_ERR		BIT(17)
-#define RX_MSDU_END_INFO13_TCP_UDP_CKSUM_FAIL	BIT(18)
-#define RX_MSDU_END_INFO13_IP_CKSUM_FAIL	BIT(19)
-#define RX_MSDU_END_INFO13_SA_IDX_INVALID	BIT(20)
-#define RX_MSDU_END_INFO13_DA_IDX_INVALID	BIT(21)
-#define RX_MSDU_END_INFO13_AMSDU_ADDR_MISMATCH	BIT(22)
-#define RX_MSDU_END_INFO13_RX_IN_TX_DECRYPT_BYP	BIT(23)
-#define RX_MSDU_END_INFO13_ENCRYPT_REQUIRED	BIT(24)
-#define RX_MSDU_END_INFO13_DIRECTED		BIT(25)
-#define RX_MSDU_END_INFO13_BUFFER_FRAGMENT	BIT(26)
-#define RX_MSDU_END_INFO13_MPDU_LEN_ERR		BIT(27)
-#define RX_MSDU_END_INFO13_TKIP_MIC_ERR		BIT(28)
-#define RX_MSDU_END_INFO13_DECRYPT_ERR		BIT(29)
-#define RX_MSDU_END_INFO13_UNDECRYPT_FRAME_ERR	BIT(30)
-#define RX_MSDU_END_INFO13_FCS_ERR		BIT(31)
-
-#define RX_MSDU_END_INFO13_WIFI_PARSER_ERR      BIT(15)
-
-#define RX_MSDU_END_INFO14_DECRYPT_STATUS_CODE	GENMASK(12, 10)
-#define RX_MSDU_END_INFO14_RX_BITMAP_NOT_UPDED	BIT(13)
-#define RX_MSDU_END_INFO14_MSDU_DONE		BIT(31)
-
-struct rx_msdu_end_qcn9274 {
-	__le16 info0;
-	__le16 phy_ppdu_id;
-	__le16 ip_hdr_cksum;
-	__le16 info1;
-	__le16 info2;
-	__le16 cumulative_l3_checksum;
-	__le32 rule_indication0;
-	__le32 ipv6_options_crc;
-	__le16 info3;
-	__le16 l3_type;
-	__le32 rule_indication1;
-	__le32 tcp_seq_num;
-	__le32 tcp_ack_num;
-	__le16 info4;
-	__le16 window_size;
-	__le16 sa_sw_peer_id;
-	__le16 info5;
-	__le16 sa_idx;
-	__le16 da_idx_or_sw_peer_id;
-	__le32 info6;
-	__le32 fse_metadata;
-	__le16 cce_metadata;
-	__le16 tcp_udp_cksum;
-	__le16 info7;
-	__le16 cumulative_ip_length;
-	__le32 info8;
-	__le32 info9;
-	__le32 info10;
-	__le32 info11;
-	__le16 vlan_ctag_ci;
-	__le16 vlan_stag_ci;
-	__le32 peer_meta_data;
-	__le32 info12;
-	__le32 flow_id_toeplitz;
-	__le32 ppdu_start_timestamp_63_32;
-	__le32 phy_meta_data;
-	__le32 ppdu_start_timestamp_31_0;
-	__le32 toeplitz_hash_2_or_4;
-	__le16 res0;
-	__le16 sa_15_0;
-	__le32 sa_47_16;
-	__le32 info13;
-	__le32 info14;
-} __packed;
-
-#define QCN9274_MSDU_END_SELECT_MSDU_END_TAG				BIT(0)
-#define QCN9274_MSDU_END_SELECT_INFO0_PHY_PPDUID_IP_HDR_CSUM_INFO1	BIT(1)
-#define QCN9274_MSDU_END_SELECT_INFO2_CUMULATIVE_CSUM_RULE_IND_0	BIT(2)
-#define QCN9274_MSDU_END_SELECT_IPV6_OP_CRC_INFO3_TYPE13		BIT(3)
-#define QCN9274_MSDU_END_SELECT_RULE_IND_1_TCP_SEQ_NUM			BIT(4)
-#define QCN9274_MSDU_END_SELECT_TCP_ACK_NUM_INFO4_WINDOW_SIZE		BIT(5)
-#define QCN9274_MSDU_END_SELECT_SA_SW_PER_ID_INFO5_SA_DA_ID		BIT(6)
-#define QCN9274_MSDU_END_SELECT_INFO6_FSE_METADATA			BIT(7)
-#define QCN9274_MSDU_END_SELECT_CCE_MDATA_TCP_UDP_CSUM_INFO7_IP_LEN	BIT(8)
-#define QCN9274_MSDU_END_SELECT_INFO8_INFO9				BIT(9)
-#define QCN9274_MSDU_END_SELECT_INFO10_INFO11				BIT(10)
-#define QCN9274_MSDU_END_SELECT_VLAN_CTAG_STAG_CI_PEER_MDATA		BIT(11)
-#define QCN9274_MSDU_END_SELECT_INFO12_AND_FLOW_ID_TOEPLITZ		BIT(12)
-#define QCN9274_MSDU_END_SELECT_PPDU_START_TS_63_32_PHY_MDATA		BIT(13)
-#define QCN9274_MSDU_END_SELECT_PPDU_START_TS_31_0_TOEPLITZ_HASH_2_4	BIT(14)
-#define QCN9274_MSDU_END_SELECT_RES0_SA_47_0				BIT(15)
-#define QCN9274_MSDU_END_SELECT_INFO13_INFO14				BIT(16)
-
-#define QCN9274_MSDU_END_WMASK (QCN9274_MSDU_END_SELECT_MSDU_END_TAG |	\
-		QCN9274_MSDU_END_SELECT_SA_SW_PER_ID_INFO5_SA_DA_ID |	\
-		QCN9274_MSDU_END_SELECT_INFO10_INFO11 |			\
-		QCN9274_MSDU_END_SELECT_INFO12_AND_FLOW_ID_TOEPLITZ |	\
-		QCN9274_MSDU_END_SELECT_PPDU_START_TS_63_32_PHY_MDATA |	\
-		QCN9274_MSDU_END_SELECT_INFO13_INFO14)
-
-/* The below rx_msdu_end_qcn9274_compact structure is tied with the mask value
- * QCN9274_MSDU_END_WMASK. If the mask value changes the structure will also
- * change.
- */
-
-struct rx_msdu_end_qcn9274_compact {
-	__le64 msdu_end_tag;
-	__le16 sa_sw_peer_id;
-	__le16 info5;
-	__le16 sa_idx;
-	__le16 da_idx_or_sw_peer_id;
-	__le32 info10;
-	__le32 info11;
-	__le32 info12;
-	__le32 flow_id_toeplitz;
-	__le32 ppdu_start_timestamp_63_32;
-	__le32 phy_meta_data;
-	__le32 info13;
-	__le32 info14;
-} __packed;
-
-/* rx_msdu_end
- *
- * rxpcu_mpdu_filter_in_category
- *		Field indicates what the reason was that this mpdu frame
- *		was allowed to come into the receive path by rxpcu. Values
- *		are defined in enum %RX_DESC_RXPCU_FILTER_*.
- *
- * sw_frame_group_id
- *		SW processes frames based on certain classifications. Values
- *		are defined in enum %RX_DESC_SW_FRAME_GRP_ID_*.
- *
- * phy_ppdu_id
- *		A ppdu counter value that PHY increments for every PPDU
- *		received. The counter value wraps around.
- *
- * ip_hdr_cksum
- *		This can include the IP header checksum or the pseudo
- *		header checksum used by TCP/UDP checksum.
- *
- * reported_mpdu_length
- *		MPDU length before decapsulation. Only valid when first_msdu is
- *		set. This field is taken directly from the length field of the
- *		A-MPDU delimiter or the preamble length field for non-A-MPDU
- *		frames.
- *
- * cce_super_rule
- *		Indicates the super filter rule.
- *
- * cce_classify_not_done_truncate
- *		Classification failed due to truncated frame.
- *
- * cce_classify_not_done_cce_dis
- *		Classification failed due to CCE global disable
- *
- * cumulative_l3_checksum
- *		FISA: IP header checksum including the total MSDU length
- *		that is part of this flow aggregated so far, reported if
- *		'RXOLE_R0_FISA_CTRL. CHKSUM_CUM_IP_LEN_EN' is set
- *
- * rule_indication
- *		Bitmap indicating which of rules have matched.
- *
- * ipv6_options_crc
- *		32 bit CRC computed out of  IP v6 extension headers.
- *
- * da_offset
- *		Offset into MSDU buffer for DA.
- *
- * sa_offset
- *		Offset into MSDU buffer for SA.
- *
- * da_offset_valid
- *		da_offset field is valid. This will be set to 0 in case
- *		of a dynamic A-MSDU when DA is compressed.
- *
- * sa_offset_valid
- *		sa_offset field is valid. This will be set to 0 in case
- *		of a dynamic A-MSDU when SA is compressed.
- *
- * l3_type
- *		The 16-bit type value indicating the type of L3 later
- *		extracted from LLC/SNAP, set to zero if SNAP is not
- *		available.
- *
- * tcp_seq_number
- *		TCP sequence number.
- *
- * tcp_ack_number
- *		TCP acknowledge number.
- *
- * tcp_flag
- *		TCP flags {NS, CWR, ECE, URG, ACK, PSH, RST, SYN, FIN}.
- *
- * lro_eligible
- *		Computed out of TCP and IP fields to indicate that this
- *		MSDU is eligible for LRO.
- *
- * window_size
- *		TCP receive window size.
- *
- * sa_sw_peer_id
- *		sw_peer_id from the address search entry corresponding to the
- *		source address of the MSDU.
- *
- * sa_idx_timeout
- *		Indicates an unsuccessful MAC source address search due to the
- *		expiring of the search timer.
- *
- * da_idx_timeout
- *		Indicates an unsuccessful MAC destination address search due to
- *		the expiring of the search timer.
- *
- * to_ds
- *		Set if the to DS bit is set in the frame control.
- *
- * tid
- *		TID field in the QoS control field
- *
- * sa_is_valid
- *		Indicates that OLE found a valid SA entry.
- *
- * da_is_valid
- *		Indicates that OLE found a valid DA entry.
- *
- * da_is_mcbc
- *		Field Only valid if da_is_valid is set. Indicates the DA address
- *		was a Multicast of Broadcast address.
- *
- * l3_header_padding
- *		Number of bytes padded  to make sure that the L3 header will
- *		always start of a Dword boundary.
- *
- * first_msdu
- *		Indicates the first MSDU of A-MSDU. If both first_msdu and
- *		last_msdu are set in the MSDU then this is a non-aggregated MSDU
- *		frame: normal MPDU. Interior MSDU in an A-MSDU shall have both
- *		first_mpdu and last_mpdu bits set to 0.
- *
- * last_msdu
- *		Indicates the last MSDU of the A-MSDU. MPDU end status is only
- *		valid when last_msdu is set.
- *
- * fr_ds
- *		Set if the from DS bit is set in the frame control.
- *
- * ip_chksum_fail_copy
- *		Indicates that the computed checksum did not match the
- *		checksum in the IP header.
- *
- * sa_idx
- *		The offset in the address table which matches the MAC source
- *		address.
- *
- * da_idx_or_sw_peer_id
- *		Based on a register configuration in RXOLE, this field will
- *		contain:
- *		The offset in the address table which matches the MAC destination
- *		address
- *		OR:
- *		sw_peer_id from the address search entry corresponding to
- *		the destination address of the MSDU
- *
- * msdu_drop
- *		REO shall drop this MSDU and not forward it to any other ring.
- *
- *		The id of the reo exit ring where the msdu frame shall push
- *		after (MPDU level) reordering has finished. Values are defined
- *		in enum %HAL_RX_MSDU_DESC_REO_DEST_IND_.
- *
- * flow_idx
- *		Flow table index.
- *
- * use_ppe
- *		Indicates to RXDMA to ignore the REO_destination_indication
- *		and use a programmed value corresponding to the REO2PPE
- *		ring
- *
- * mesh_sta
- *		When set, this is a Mesh (11s) STA.
- *
- * vlan_ctag_stripped
- *		Set by RXOLE if it stripped 4-bytes of C-VLAN Tag from the
- *		packet
- *
- * vlan_stag_stripped
- *		Set by RXOLE if it stripped 4-bytes of S-VLAN Tag from the
- *		packet
- *
- * fragment_flag
- *		Indicates that this is an 802.11 fragment frame.  This is
- *		set when either the more_frag bit is set in the frame control
- *		or the fragment number is not zero.  Only set when first_msdu
- *		is set.
- *
- * fse_metadata
- *		FSE related meta data.
- *
- * cce_metadata
- *		CCE related meta data.
- *
- * tcp_udp_chksum
- *		The value of the computed TCP/UDP checksum.  A mode bit
- *		selects whether this checksum is the full checksum or the
- *		partial checksum which does not include the pseudo header.
- *
- * aggregation_count
- *		Number of MSDU's aggregated so far
- *
- * flow_aggregation_continuation
- *		To indicate that this MSDU can be aggregated with
- *		the previous packet with the same flow id
- *
- * fisa_timeout
- *		To indicate that the aggregation has restarted for
- *		this flow due to timeout
- *
- * tcp_udp_chksum_fail
- *		Indicates that the computed checksum (tcp_udp_chksum) did
- *		not match the checksum in the TCP/UDP header.
- *
- * msdu_limit_error
- *		Indicates that the MSDU threshold was exceeded and thus all the
- *		rest of the MSDUs will not be scattered and will not be
- *		decapsulated but will be DMA'ed in RAW format as a single MSDU.
- *
- * flow_idx_timeout
- *		Indicates an unsuccessful flow search due to the expiring of
- *		the search timer.
- *
- * flow_idx_invalid
- *		flow id is not valid.
- *
- * cce_match
- *		Indicates that this status has a corresponding MSDU that
- *		requires FW processing. The OLE will have classification
- *		ring mask registers which will indicate the ring(s) for
- *		packets and descriptors which need FW attention.
- *
- * amsdu_parser_error
- *		A-MSDU could not be properly de-agregated.
- *
- * cumulative_ip_length
- *		Total MSDU length that is part of this flow aggregated
- *		so far
- *
- * key_id
- *		The key ID octet from the IV. Only valid when first_msdu is set.
- *
- * service_code
- *		Opaque service code between PPE and Wi-Fi
- *
- * priority_valid
- *		This field gets passed on by REO to PPE in the EDMA descriptor
- *
- * intra_bss
- *		This packet needs intra-BSS routing by SW as the 'vdev_id'
- *		for the destination is the same as 'vdev_id' (from 'RX_MPDU_PCU_START')
- *		that this MSDU was got in.
- *
- * dest_chip_id
- *		If intra_bss is set, copied by RXOLE from 'ADDR_SEARCH_ENTRY'
- *		to support intra-BSS routing with multi-chip multi-link
- *		operation. This indicates into which chip's TCL the packet should be
- *		queueued
- *
- * multicast_echo
- *		If set, this packet is a multicast echo, i.e. the DA is
- *		multicast and Rx OLE SA search with mcast_echo_check = 1
- *		passed. RXDMA should release such packets to WBM.
- *
- * wds_learning_event
- *		If set, this packet has an SA search failure with WDS learning
- *		enabled for the peer. RXOLE should route this TLV to the
- *		RXDMA0 status ring to notify FW.
- *
- * wds_roaming_event
- *		If set, this packet's SA 'Sw_peer_id' mismatches the 'Sw_peer_id'
- *		of the peer through which the packet was got, indicating
- *		the SA node has roamed. RXOLE should route this TLV to
- *		the RXDMA0 status ring to notify FW.
- *
- * wds_keep_alive_event
- *		If set, the AST timestamp for this packet's SA is older
- *		than the current timestamp by more than a threshold programmed
- *		in RXOLE. RXOLE should route this TLV to the RXDMA0 status
- *		ring to notify FW to keep the AST entry for the SA alive.
- *
- * msdu_length
- *		MSDU length in bytes after decapsulation.
- *		This field is still valid for MPDU frames without A-MSDU.
- *		It still represents MSDU length after decapsulation
- *
- * stbc
- *		When set, use STBC transmission rates.
- *
- * ipsec_esp
- *		Set if IPv4/v6 packet is using IPsec ESP.
- *
- * l3_offset
- *		Depending upon mode bit, this field either indicates the
- *		L3 offset in bytes from the start of the RX_HEADER or the IP
- *		offset in bytes from the start of the packet after
- *		decapsulation. The latter is only valid if ipv4_proto or
- *		ipv6_proto is set.
- *
- * ipsec_ah
- *		Set if IPv4/v6 packet is using IPsec AH
- *
- * l4_offset
- *		Depending upon mode bit, this field either indicates the
- *		L4 offset in bytes from the start of RX_HEADER (only valid
- *		if either ipv4_proto or ipv6_proto is set to 1) or indicates
- *		the offset in bytes to the start of TCP or UDP header from
- *		the start of the IP header after decapsulation (Only valid if
- *		tcp_proto or udp_proto is set). The value 0 indicates that
- *		the offset is longer than 127 bytes.
- *
- * msdu_number
- *		Indicates the MSDU number within a MPDU.  This value is
- *		reset to zero at the start of each MPDU.  If the number of
- *		MSDU exceeds 255 this number will wrap using modulo 256.
- *
- * decap_type
- *		Indicates the format after decapsulation. Values are defined in
- *		enum %MPDU_START_DECAP_TYPE_*.
- *
- * ipv4_proto
- *		Set if L2 layer indicates IPv4 protocol.
- *
- * ipv6_proto
- *		Set if L2 layer indicates IPv6 protocol.
- *
- * tcp_proto
- *		Set if the ipv4_proto or ipv6_proto are set and the IP protocol
- *		indicates TCP.
- *
- * udp_proto
- *		Set if the ipv4_proto or ipv6_proto are set and the IP protocol
- *		indicates UDP.
- *
- * ip_frag
- *		Indicates that either the IP More frag bit is set or IP frag
- *		number is non-zero.  If set indicates that this is a fragmented
- *		IP packet.
- *
- * tcp_only_ack
- *		Set if only the TCP Ack bit is set in the TCP flags and if
- *		the TCP payload is 0.
- *
- * da_is_bcast_mcast
- *		The destination address is broadcast or multicast.
- *
- * toeplitz_hash
- *		Actual chosen Hash.
- *		0 - Toeplitz hash of 2-tuple (IP source address, IP
- *		    destination address)
- *		1 - Toeplitz hash of 4-tuple (IP source	address,
- *		    IP destination address, L4 (TCP/UDP) source port,
- *		    L4 (TCP/UDP) destination port)
- *		2 - Toeplitz of flow_id
- *		3 - Zero is used
- *
- * ip_fixed_header_valid
- *		Fixed 20-byte IPv4 header or 40-byte IPv6 header parsed
- *		fully within first 256 bytes of the packet
- *
- * ip_extn_header_valid
- *		IPv6/IPv6 header, including IPv4 options and
- *		recognizable extension headers parsed fully within first 256
- *		bytes of the packet
- *
- * tcp_udp_header_valid
- *		Fixed 20-byte TCP (excluding TCP options) or 8-byte UDP
- *		header parsed fully within first 256 bytes of the packet
- *
- * mesh_control_present
- *		When set, this MSDU includes the 'Mesh Control' field
- *
- * ldpc
- *
- * ip4_protocol_ip6_next_header
- *		For IPv4, this is the 8 bit protocol field set). For IPv6 this
- *		is the 8 bit next_header field.
- *
- *
- * vlan_ctag_ci
- *		2 bytes of C-VLAN Tag Control Information from WHO_L2_LLC
- *
- * vlan_stag_ci
- *		2 bytes of S-VLAN Tag Control Information from WHO_L2_LLC
- *		in case of double VLAN
- *
- * peer_meta_data
- *		Meta data that SW has programmed in the Peer table entry
- *		of the transmitting STA.
- *
- * user_rssi
- *		RSSI for this user
- *
- * pkt_type
- *		Values are defined in enum %RX_MSDU_START_PKT_TYPE_*.
- *
- * sgi
- *		Field only valid when pkt type is HT, VHT or HE. Values are
- *		defined in enum %RX_MSDU_START_SGI_*.
- *
- * rate_mcs
- *		MCS Rate used.
- *
- * receive_bandwidth
- *		Full receive Bandwidth. Values are defined in enum
- *		%RX_MSDU_START_RECV_*.
- *
- * reception_type
- *		Indicates what type of reception this is and defined in enum
- *		%RX_MSDU_START_RECEPTION_TYPE_*.
- *
- * mimo_ss_bitmap
- *		Field only valid when
- *		Reception_type is RX_MSDU_START_RECEPTION_TYPE_DL_MU_MIMO or
- *		RX_MSDU_START_RECEPTION_TYPE_DL_MU_OFDMA_MIMO.
- *
- *		Bitmap, with each bit indicating if the related spatial
- *		stream is used for this STA
- *
- *		LSB related to SS 0
- *
- *		0 - spatial stream not used for this reception
- *		1 - spatial stream used for this reception
- *
- * msdu_done_copy
- *		If set indicates that the RX packet data, RX header data,
- *		RX PPDU start descriptor, RX MPDU start/end descriptor,
- *		RX MSDU start/end descriptors and RX Attention descriptor
- *		are all valid.  This bit is in the last 64-bit of the descriptor
- *		expected to be subscribed in future hardware.
- *
- * flow_id_toeplitz
- *		Toeplitz hash of 5-tuple
- *		{IP source address, IP destination address, IP source port, IP
- *		destination port, L4 protocol}  in case of non-IPSec.
- *
- *		In case of IPSec - Toeplitz hash of 4-tuple
- *		{IP source address, IP destination address, SPI, L4 protocol}
- *
- *		The relevant Toeplitz key registers are provided in RxOLE's
- *		instance of common parser module. These registers are separate
- *		from the Toeplitz keys used by ASE/FSE modules inside RxOLE.
- *		The actual value will be passed on from common parser module
- *		to RxOLE in one of the WHO_* TLVs.
- *
- * ppdu_start_timestamp
- *		Timestamp that indicates when the PPDU that contained this MPDU
- *		started on the medium.
- *
- * phy_meta_data
- *		SW programmed Meta data provided by the PHY. Can be used for SW
- *		to indicate the channel the device is on.
- *
- * toeplitz_hash_2_or_4
- *		Controlled by multiple RxOLE registers for TCP/UDP over
- *		IPv4/IPv6 - Either, Toeplitz hash computed over 2-tuple
- *		IPv4 or IPv6 src/dest addresses is reported; or, Toeplitz
- *		hash computed over 4-tuple IPv4 or IPv6 src/dest addresses
- *		and src/dest ports is reported. The Flow_id_toeplitz hash
- *		can also be reported here. Usually the hash reported here
- *		is the one used for hash-based REO routing (see use_flow_id_toeplitz_clfy
- *		in 'RXPT_CLASSIFY_INFO').
- *
- * sa
- *		Source MAC address
- *
- * first_mpdu
- *		Indicates the first MSDU of the PPDU.  If both first_mpdu
- *		and last_mpdu are set in the MSDU then this is a not an
- *		A-MPDU frame but a stand alone MPDU.  Interior MPDU in an
- *		A-MPDU shall have both first_mpdu and last_mpdu bits set to
- *		0.  The PPDU start status will only be valid when this bit
- *		is set.
- *
- * mcast_bcast
- *		Multicast / broadcast indicator.  Only set when the MAC
- *		address 1 bit 0 is set indicating mcast/bcast and the BSSID
- *		matches one of the 4 BSSID registers. Only set when
- *		first_msdu is set.
- *
- * ast_index_not_found
- *		Only valid when first_msdu is set. Indicates no AST matching
- *		entries within the max search count.
- *
- * ast_index_timeout
- *		Only valid when first_msdu is set. Indicates an unsuccessful
- *		search in the address search table due to timeout.
- *
- * power_mgmt
- *		Power management bit set in the 802.11 header.  Only set
- *		when first_msdu is set.
- *
- * non_qos
- *		Set if packet is not a non-QoS data frame.  Only set when
- *		first_msdu is set.
- *
- * null_data
- *		Set if frame type indicates either null data or QoS null
- *		data format.  Only set when first_msdu is set.
- *
- * mgmt_type
- *		Set if packet is a management packet.  Only set when
- *		first_msdu is set.
- *
- * ctrl_type
- *		Set if packet is a control packet.  Only set when first_msdu
- *		is set.
- *
- * more_data
- *		Set if more bit in frame control is set.  Only set when
- *		first_msdu is set.
- *
- * eosp
- *		Set if the EOSP (end of service period) bit in the QoS
- *		control field is set.  Only set when first_msdu is set.
- *
- * a_msdu_error
- *		Set if number of MSDUs in A-MSDU is above a threshold or if the
- *		size of the MSDU is invalid. This receive buffer will contain
- *		all of the remainder of MSDUs in this MPDU w/o decapsulation.
- *
- * order
- *		Set if the order bit in the frame control is set.  Only
- *		set when first_msdu is set.
- *
- * wifi_parser_error
- *		Indicates that the WiFi frame has one of the following errors
- *
- * overflow_err
- *		RXPCU Receive FIFO ran out of space to receive the full MPDU.
- *		Therefore this MPDU is terminated early and is thus corrupted.
- *
- *		This MPDU will not be ACKed.
- *
- *		RXPCU might still be able to correctly receive the following
- *		MPDUs in the PPDU if enough fifo space became available in time.
- *
- * mpdu_length_err
- *		Set by RXPCU if the expected MPDU length does not correspond
- *		with the actually received number of bytes in the MPDU.
- *
- * tcp_udp_chksum_fail
- *		Indicates that the computed checksum (tcp_udp_chksum) did
- *		not match the checksum in the TCP/UDP header.
- *
- * ip_chksum_fail
- *		Indicates that the computed checksum did not match the
- *		checksum in the IP header.
- *
- * sa_idx_invalid
- *		Indicates no matching entry was found in the address search
- *		table for the source MAC address.
- *
- * da_idx_invalid
- *		Indicates no matching entry was found in the address search
- *		table for the destination MAC address.
- *
- * amsdu_addr_mismatch
- *		Indicates that an A-MSDU with 'from DS = 0' had an SA mismatching
- *		TA or an A-MDU with 'to DS = 0' had a DA mismatching RA
- *
- * rx_in_tx_decrypt_byp
- *		Indicates that RX packet is not decrypted as Crypto is busy
- *		with TX packet processing.
- *
- * encrypt_required
- *		Indicates that this data type frame is not encrypted even if
- *		the policy for this MPDU requires encryption as indicated in
- *		the peer table key type.
- *
- * directed
- *		MPDU is a directed packet which means that the RA matched
- *		our STA addresses.  In proxySTA it means that the TA matched
- *		an entry in our address search table with the corresponding
- *		'no_ack' bit is the address search entry cleared.
- *
- * buffer_fragment
- *		Indicates that at least one of the rx buffers has been
- *		fragmented.  If set the FW should look at the rx_frag_info
- *		descriptor described below.
- *
- * mpdu_length_err
- *		Indicates that the MPDU was pre-maturely terminated
- *		resulting in a truncated MPDU.  Don't trust the MPDU length
- *		field.
- *
- * tkip_mic_err
- *		Indicates that the MPDU Michael integrity check failed
- *
- * decrypt_err
- *		Indicates that the MPDU decrypt integrity check failed
- *
- * fcs_err
- *		Indicates that the MPDU FCS check failed
- *
- * flow_idx_timeout
- *		Indicates an unsuccessful flow search due to the expiring of
- *		the search timer.
- *
- * flow_idx_invalid
- *		flow id is not valid.
- *
- * decrypt_status_code
- *		Field provides insight into the decryption performed. Values
- *		are defined in enum %RX_DESC_DECRYPT_STATUS_CODE_*.
- *
- * rx_bitmap_not_updated
- *		Frame is received, but RXPCU could not update the receive bitmap
- *		due to (temporary) fifo constraints.
- *
- * msdu_done
- *		If set indicates that the RX packet data, RX header data, RX
- *		PPDU start descriptor, RX MPDU start/end descriptor, RX MSDU
- *		start/end descriptors and RX Attention descriptor are all
- *		valid.  This bit must be in the last octet of the
- *		descriptor.
- *
- */
-
-struct hal_rx_desc_qcn9274 {
-	struct rx_msdu_end_qcn9274 msdu_end;
-	struct rx_mpdu_start_qcn9274 mpdu_start;
-	u8 msdu_payload[];
-} __packed;
-
-struct hal_rx_desc_qcn9274_compact {
-	struct rx_msdu_end_qcn9274_compact msdu_end;
-	struct rx_mpdu_start_qcn9274_compact mpdu_start;
-	u8 msdu_payload[];
-} __packed;
-
-#define RX_BE_PADDING0_BYTES 8
-#define RX_BE_PADDING1_BYTES 8
-
-#define HAL_RX_BE_PKT_HDR_TLV_LEN		112
-
-struct rx_pkt_hdr_tlv {
-	__le64 tag;
-	__le64 phy_ppdu_id;
-	u8 rx_pkt_hdr[HAL_RX_BE_PKT_HDR_TLV_LEN];
-};
-
-struct hal_rx_desc_wcn7850 {
-	__le64 msdu_end_tag;
-	struct rx_msdu_end_qcn9274 msdu_end;
-	u8 rx_padding0[RX_BE_PADDING0_BYTES];
-	__le64 mpdu_start_tag;
-	struct rx_mpdu_start_qcn9274 mpdu_start;
-	struct rx_pkt_hdr_tlv	 pkt_hdr_tlv;
-	u8 msdu_payload[];
-};
-
-struct hal_rx_desc {
-	union {
-		struct hal_rx_desc_qcn9274 qcn9274;
-		struct hal_rx_desc_qcn9274_compact qcn9274_compact;
-		struct hal_rx_desc_wcn7850 wcn7850;
-	} u;
-} __packed;
-
-#define MAX_USER_POS 8
-#define MAX_MU_GROUP_ID 64
-#define MAX_MU_GROUP_SHOW 16
-#define MAX_MU_GROUP_LENGTH (6 * MAX_MU_GROUP_SHOW)
-
-#endif /* ATH12K_RX_DESC_H */
diff --git a/drivers/net/wireless/ath/ath12k/testmode.c b/drivers/net/wireless/ath/ath12k/testmode.c
index fb6af7ccf71f..05a65970c862 100644
--- a/drivers/net/wireless/ath/ath12k/testmode.c
+++ b/drivers/net/wireless/ath/ath12k/testmode.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #include "testmode.h"
@@ -393,3 +393,4 @@ int ath12k_tm_cmd(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		return -EOPNOTSUPP;
 	}
 }
+EXPORT_SYMBOL(ath12k_tm_cmd);
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/Makefile b/drivers/net/wireless/ath/ath12k/wifi7/Makefile
new file mode 100644
index 000000000000..dcfa732bb95b
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/Makefile
@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+obj-$(CONFIG_ATH12K) += ath12k_wifi7.o
+ath12k_wifi7-y += core.o \
+	          pci.o \
+	          wmi.o \
+	          mhi.o \
+	          ce.o \
+	          hw.o \
+		  hal_tx.o \
+		  hal_rx.o \
+		  dp_rx.o \
+		  dp_tx.o \
+		  dp.o \
+		  dp_mon.o \
+		  hal.o \
+		  hal_qcn9274.o \
+		  hal_wcn7850.o
+
+ath12k_wifi7-$(CONFIG_ATH12K_AHB) += ahb.o
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/ahb.c b/drivers/net/wireless/ath/ath12k/wifi7/ahb.c
new file mode 100644
index 000000000000..a6c5f7689edd
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/ahb.c
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/soc/qcom/mdt_loader.h>
+#include "../ahb.h"
+#include "ahb.h"
+#include "../debug.h"
+#include "../hif.h"
+#include "hw.h"
+#include "dp.h"
+#include "core.h"
+
+static const struct of_device_id ath12k_wifi7_ahb_of_match[] = {
+	{ .compatible = "qcom,ipq5332-wifi",
+	  .data = (void *)ATH12K_HW_IPQ5332_HW10,
+	},
+	{ }
+};
+
+MODULE_DEVICE_TABLE(of, ath12k_wifi7_ahb_of_match);
+
+static int ath12k_wifi7_ahb_probe(struct platform_device *pdev)
+{
+	struct ath12k_ahb *ab_ahb;
+	enum ath12k_hw_rev hw_rev;
+	struct ath12k_base *ab;
+	int ret;
+
+	ab = platform_get_drvdata(pdev);
+	ab_ahb = ath12k_ab_to_ahb(ab);
+
+	hw_rev = (enum ath12k_hw_rev)(kernel_ulong_t)of_device_get_match_data(&pdev->dev);
+	switch (hw_rev) {
+	case ATH12K_HW_IPQ5332_HW10:
+		ab_ahb->userpd_id = ATH12K_IPQ5332_USERPD_ID;
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	ab->target_mem_mode = ATH12K_QMI_MEMORY_MODE_DEFAULT;
+	ab->hw_rev = hw_rev;
+
+	ret = ath12k_wifi7_hw_init(ab);
+	if (ret) {
+		ath12k_err(ab, "WiFi-7 hw_init for AHB failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static struct ath12k_ahb_driver ath12k_wifi7_ahb_driver = {
+	.name = "ath12k_wifi7_ahb",
+	.id_table = ath12k_wifi7_ahb_of_match,
+	.ops.probe = ath12k_wifi7_ahb_probe,
+	.ops.arch_init = ath12k_wifi7_arch_init,
+	.ops.arch_deinit = ath12k_wifi7_arch_deinit,
+};
+
+int ath12k_wifi7_ahb_init(void)
+{
+	return ath12k_ahb_register_driver(ATH12K_DEVICE_FAMILY_WIFI7,
+					  &ath12k_wifi7_ahb_driver);
+}
+
+void ath12k_wifi7_ahb_exit(void)
+{
+	ath12k_ahb_unregister_driver(ATH12K_DEVICE_FAMILY_WIFI7);
+}
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/ahb.h b/drivers/net/wireless/ath/ath12k/wifi7/ahb.h
new file mode 100644
index 000000000000..5974c7cad69a
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/ahb.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+#ifndef ATH12K_AHB_WIFI7_H
+#define ATH12K_AHB_WIFI7_H
+
+#ifdef CONFIG_ATH12K_AHB
+int ath12k_wifi7_ahb_init(void);
+void ath12k_wifi7_ahb_exit(void);
+#else
+static inline int ath12k_wifi7_ahb_init(void)
+{
+	return 0;
+}
+
+static inline void ath12k_wifi7_ahb_exit(void) {}
+#endif
+#endif /* ATH12K_AHB_WIFI7_H */
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/ce.c b/drivers/net/wireless/ath/ath12k/wifi7/ce.c
new file mode 100644
index 000000000000..952d6c39c333
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/ce.c
@@ -0,0 +1,973 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/types.h>
+#include <linux/bitops.h>
+#include <linux/bitfield.h>
+
+#include "../core.h"
+#include "../ce.h"
+#include "ce.h"
+#include "../dp_rx.h"
+
+/* Copy Engine (CE) configs for QCN9274 */
+/* Target firmware's Copy Engine configuration. */
+const struct ce_pipe_config ath12k_wifi7_target_ce_config_wlan_qcn9274[] = {
+	/* CE0: host->target HTC control and raw streams */
+	{
+		.pipenum = __cpu_to_le32(0),
+		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE1: target->host HTT + HTC control */
+	{
+		.pipenum = __cpu_to_le32(1),
+		.pipedir = __cpu_to_le32(PIPEDIR_IN),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE2: target->host WMI */
+	{
+		.pipenum = __cpu_to_le32(2),
+		.pipedir = __cpu_to_le32(PIPEDIR_IN),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE3: host->target WMI (mac0) */
+	{
+		.pipenum = __cpu_to_le32(3),
+		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE4: host->target HTT */
+	{
+		.pipenum = __cpu_to_le32(4),
+		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
+		.nentries = __cpu_to_le32(256),
+		.nbytes_max = __cpu_to_le32(256),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS | CE_ATTR_DIS_INTR),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE5: target->host Pktlog */
+	{
+		.pipenum = __cpu_to_le32(5),
+		.pipedir = __cpu_to_le32(PIPEDIR_IN),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE6: Reserved for target autonomous hif_memcpy */
+	{
+		.pipenum = __cpu_to_le32(6),
+		.pipedir = __cpu_to_le32(PIPEDIR_INOUT),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(16384),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE7: host->target WMI (mac1) */
+	{
+		.pipenum = __cpu_to_le32(7),
+		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE8: Reserved for target autonomous hif_memcpy */
+	{
+		.pipenum = __cpu_to_le32(8),
+		.pipedir = __cpu_to_le32(PIPEDIR_INOUT),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(16384),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE9, 10 and 11: Reserved for MHI */
+
+	/* CE12: Target CV prefetch */
+	{
+		.pipenum = __cpu_to_le32(12),
+		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE13: Target CV prefetch */
+	{
+		.pipenum = __cpu_to_le32(13),
+		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE14: WMI logging/CFR/Spectral/Radar */
+	{
+		.pipenum = __cpu_to_le32(14),
+		.pipedir = __cpu_to_le32(PIPEDIR_IN),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE15: Reserved */
+};
+
+/* Map from service/endpoint to Copy Engine.
+ * This table is derived from the CE_PCI TABLE, above.
+ * It is passed to the Target at startup for use by firmware.
+ * Pipe direction:
+ *      PIPEDIR_OUT = UL = host -> target
+ *      PIPEDIR_IN = DL = target -> host
+ */
+const struct service_to_pipe
+ath12k_wifi7_target_service_to_ce_map_wlan_qcn9274[] = {
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(0),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(1),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_TEST_RAW_STREAMS),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(0),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_TEST_RAW_STREAMS),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(1),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(4),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(1),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL_MAC1),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(7),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL_MAC1),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_PKT_LOG),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(5),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL_DIAG),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(14),
+	},
+
+	/* (Additions here) */
+
+	{ /* must be last */
+		__cpu_to_le32(0),
+		__cpu_to_le32(0),
+		__cpu_to_le32(0),
+	},
+};
+
+const struct ce_attr ath12k_wifi7_host_ce_config_qcn9274[] = {
+	/* CE0: host->target HTC control and raw streams */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 16,
+		.src_sz_max = 2048,
+		.dest_nentries = 0,
+	},
+
+	/* CE1: target->host HTT + HTC control */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 512,
+		.recv_cb = ath12k_htc_rx_completion_handler,
+	},
+
+	/* CE2: target->host WMI */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 128,
+		.recv_cb = ath12k_htc_rx_completion_handler,
+	},
+
+	/* CE3: host->target WMI (mac0) */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 32,
+		.src_sz_max = 2048,
+		.dest_nentries = 0,
+	},
+
+	/* CE4: host->target HTT */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 2048,
+		.src_sz_max = 256,
+		.dest_nentries = 0,
+	},
+
+	/* CE5: target->host pktlog */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 512,
+		.recv_cb = ath12k_dp_htt_htc_t2h_msg_handler,
+	},
+
+	/* CE6: target autonomous hif_memcpy */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+
+	/* CE7: host->target WMI (mac1) */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 32,
+		.src_sz_max = 2048,
+		.dest_nentries = 0,
+	},
+
+	/* CE8: target autonomous hif_memcpy */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+
+	/* CE9: MHI */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+
+	/* CE10: MHI */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+
+	/* CE11: MHI */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+
+	/* CE12: CV Prefetch */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+
+	/* CE13: CV Prefetch */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+
+	/* CE14: target->host dbg log */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 512,
+		.recv_cb = ath12k_htc_rx_completion_handler,
+	},
+
+	/* CE15: reserved for future use */
+	{
+		.flags = (CE_ATTR_FLAGS | CE_ATTR_DIS_INTR),
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+};
+
+/* Copy Engine (CE) configs for WCN7850 */
+/* Target firmware's Copy Engine configuration. */
+const struct ce_pipe_config ath12k_wifi7_target_ce_config_wlan_wcn7850[] = {
+	/* CE0: host->target HTC control and raw streams */
+	{
+		.pipenum = __cpu_to_le32(0),
+		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE1: target->host HTT + HTC control */
+	{
+		.pipenum = __cpu_to_le32(1),
+		.pipedir = __cpu_to_le32(PIPEDIR_IN),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE2: target->host WMI */
+	{
+		.pipenum = __cpu_to_le32(2),
+		.pipedir = __cpu_to_le32(PIPEDIR_IN),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE3: host->target WMI */
+	{
+		.pipenum = __cpu_to_le32(3),
+		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE4: host->target HTT */
+	{
+		.pipenum = __cpu_to_le32(4),
+		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
+		.nentries = __cpu_to_le32(256),
+		.nbytes_max = __cpu_to_le32(256),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS | CE_ATTR_DIS_INTR),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE5: target->host Pktlog */
+	{
+		.pipenum = __cpu_to_le32(5),
+		.pipedir = __cpu_to_le32(PIPEDIR_IN),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE6: Reserved for target autonomous hif_memcpy */
+	{
+		.pipenum = __cpu_to_le32(6),
+		.pipedir = __cpu_to_le32(PIPEDIR_INOUT),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(16384),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE7 used only by Host */
+	{
+		.pipenum = __cpu_to_le32(7),
+		.pipedir = __cpu_to_le32(PIPEDIR_INOUT_H2H),
+		.nentries = __cpu_to_le32(0),
+		.nbytes_max = __cpu_to_le32(0),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS | CE_ATTR_DIS_INTR),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE8 target->host used only by IPA */
+	{
+		.pipenum = __cpu_to_le32(8),
+		.pipedir = __cpu_to_le32(PIPEDIR_INOUT),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(16384),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+	/* CE 9, 10, 11 are used by MHI driver */
+};
+
+const struct service_to_pipe
+ath12k_wifi7_target_service_to_ce_map_wlan_wcn7850[] = {
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(0),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(4),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(1),
+	},
+
+	/* (Additions here) */
+
+	{ /* must be last */
+		__cpu_to_le32(0),
+		__cpu_to_le32(0),
+		__cpu_to_le32(0),
+	},
+};
+
+const struct ce_attr ath12k_wifi7_host_ce_config_wcn7850[] = {
+	/* CE0: host->target HTC control and raw streams */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 16,
+		.src_sz_max = 2048,
+		.dest_nentries = 0,
+	},
+
+	/* CE1: target->host HTT + HTC control */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 512,
+		.recv_cb = ath12k_htc_rx_completion_handler,
+	},
+
+	/* CE2: target->host WMI */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 64,
+		.recv_cb = ath12k_htc_rx_completion_handler,
+	},
+
+	/* CE3: host->target WMI (mac0) */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 32,
+		.src_sz_max = 2048,
+		.dest_nentries = 0,
+	},
+
+	/* CE4: host->target HTT */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 2048,
+		.src_sz_max = 256,
+		.dest_nentries = 0,
+	},
+
+	/* CE5: target->host pktlog */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+
+	/* CE6: target autonomous hif_memcpy */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+
+	/* CE7: host->target WMI (mac1) */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 0,
+	},
+
+	/* CE8: target autonomous hif_memcpy */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+};
+
+/* Copy Engine (CE) configs for IPQ5332 */
+/* Target firmware's Copy Engine configuration. */
+const struct ce_pipe_config ath12k_wifi7_target_ce_config_wlan_ipq5332[] = {
+	/* CE0: host->target HTC control and raw streams */
+	{
+		.pipenum = __cpu_to_le32(0),
+		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE1: target->host HTT */
+	{
+		.pipenum = __cpu_to_le32(1),
+		.pipedir = __cpu_to_le32(PIPEDIR_IN),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE2: target->host WMI  + HTC control */
+	{
+		.pipenum = __cpu_to_le32(2),
+		.pipedir = __cpu_to_le32(PIPEDIR_IN),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE3: host->target WMI */
+	{
+		.pipenum = __cpu_to_le32(3),
+		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE4: host->target HTT */
+	{
+		.pipenum = __cpu_to_le32(4),
+		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
+		.nentries = __cpu_to_le32(256),
+		.nbytes_max = __cpu_to_le32(256),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS | CE_ATTR_DIS_INTR),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE5: Target -> host PKTLOG */
+	{
+		.pipenum = __cpu_to_le32(5),
+		.pipedir = __cpu_to_le32(PIPEDIR_IN),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE6: Reserved for target autonomous HIF_memcpy */
+	{
+		.pipenum = __cpu_to_le32(6),
+		.pipedir = __cpu_to_le32(PIPEDIR_INOUT),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(16384),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE7: Reserved for CV Prefetch */
+	{
+		.pipenum = __cpu_to_le32(7),
+		.pipedir = __cpu_to_le32(PIPEDIR_OUT),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE8: Reserved for target generic HIF memcpy */
+	{
+		.pipenum = __cpu_to_le32(8),
+		.pipedir = __cpu_to_le32(PIPEDIR_INOUT),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(16384),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE9: WMI logging/CFR/Spectral/Radar/ */
+	{
+		.pipenum = __cpu_to_le32(9),
+		.pipedir = __cpu_to_le32(PIPEDIR_IN),
+		.nentries = __cpu_to_le32(32),
+		.nbytes_max = __cpu_to_le32(2048),
+		.flags = __cpu_to_le32(CE_ATTR_FLAGS),
+		.reserved = __cpu_to_le32(0),
+	},
+
+	/* CE10: Unused TBD */
+	{
+		.pipenum = __cpu_to_le32(10),
+		.pipedir = __cpu_to_le32(PIPEDIR_NONE),
+		.nentries = __cpu_to_le32(0),
+		.nbytes_max = __cpu_to_le32(0),
+		.flags = __cpu_to_le32(0),
+		.reserved = __cpu_to_le32(0),
+	},
+	/* CE11: Unused TBD */
+	{
+		.pipenum = __cpu_to_le32(11),
+		.pipedir = __cpu_to_le32(PIPEDIR_NONE),
+		.nentries = __cpu_to_le32(0),
+		.nbytes_max = __cpu_to_le32(0),
+		.flags = __cpu_to_le32(0),
+		.reserved = __cpu_to_le32(0),
+	},
+};
+
+const struct service_to_pipe
+ath12k_wifi7_target_service_to_ce_map_wlan_ipq5332[] = {
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VO),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BK),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_BE),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_DATA_VI),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(3),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(2),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(0),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_RSVD_CTRL),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(1),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_TEST_RAW_STREAMS),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(0),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_TEST_RAW_STREAMS),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(1),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG),
+		__cpu_to_le32(PIPEDIR_OUT),
+		__cpu_to_le32(4),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_HTT_DATA_MSG),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(1),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_PKT_LOG),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(5),
+	},
+	{
+		__cpu_to_le32(ATH12K_HTC_SVC_ID_WMI_CONTROL_DIAG),
+		__cpu_to_le32(PIPEDIR_IN),
+		__cpu_to_le32(9),
+	},
+	/* (Additions here) */
+
+	{ /* must be last */
+		__cpu_to_le32(0),
+		__cpu_to_le32(0),
+		__cpu_to_le32(0),
+	},
+};
+
+const struct ce_attr ath12k_wifi7_host_ce_config_ipq5332[] = {
+	/* CE0: host->target HTC control and raw streams */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 16,
+		.src_sz_max = 2048,
+		.dest_nentries = 0,
+	},
+
+	/* CE1: target->host HTT + HTC control */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 512,
+		.recv_cb = ath12k_htc_rx_completion_handler,
+	},
+
+	/* CE2: target->host WMI */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 128,
+		.recv_cb = ath12k_htc_rx_completion_handler,
+	},
+
+	/* CE3: host->target WMI */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 32,
+		.src_sz_max = 2048,
+		.dest_nentries = 0,
+	},
+
+	/* CE4: host->target HTT */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 2048,
+		.src_sz_max = 256,
+		.dest_nentries = 0,
+	},
+
+	/* CE5: target -> host PKTLOG */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 512,
+		.recv_cb = ath12k_dp_htt_htc_t2h_msg_handler,
+	},
+
+	/* CE6: Target autonomous HIF_memcpy */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+
+	/* CE7: CV Prefetch */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+
+	/* CE8: Target HIF memcpy (Generic HIF memcypy) */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+
+	/* CE9: WMI logging/CFR/Spectral/Radar */
+	{
+		.flags = CE_ATTR_FLAGS,
+		.src_nentries = 0,
+		.src_sz_max = 2048,
+		.dest_nentries = 128,
+	},
+
+	/* CE10: Unused */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+
+	/* CE11: Unused */
+	{
+		.flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR,
+		.src_nentries = 0,
+		.src_sz_max = 0,
+		.dest_nentries = 0,
+	},
+};
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/ce.h b/drivers/net/wireless/ath/ath12k/wifi7/ce.h
new file mode 100644
index 000000000000..369a14472913
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/ce.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef ATH12K_WIFI7_CE_H
+#define ATH12K_WIFI7_CE_H
+
+extern const struct ce_pipe_config ath12k_wifi7_target_ce_config_wlan_qcn9274[];
+extern const struct ce_pipe_config ath12k_wifi7_target_ce_config_wlan_wcn7850[];
+extern const struct ce_pipe_config ath12k_wifi7_target_ce_config_wlan_ipq5332[];
+
+extern const struct service_to_pipe ath12k_wifi7_target_service_to_ce_map_wlan_qcn9274[];
+extern const struct service_to_pipe ath12k_wifi7_target_service_to_ce_map_wlan_wcn7850[];
+extern const struct service_to_pipe ath12k_wifi7_target_service_to_ce_map_wlan_ipq5332[];
+
+extern const struct ce_attr ath12k_wifi7_host_ce_config_qcn9274[];
+extern const struct ce_attr ath12k_wifi7_host_ce_config_wcn7850[];
+extern const struct ce_attr ath12k_wifi7_host_ce_config_ipq5332[];
+
+#endif /* ATH12K_WIFI7_CE_H */
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/core.c b/drivers/net/wireless/ath/ath12k/wifi7/core.c
new file mode 100644
index 000000000000..a02c57acf137
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/core.c
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/module.h>
+#include "../ahb.h"
+#include "../pci.h"
+#include "pci.h"
+#include "ahb.h"
+#include "core.h"
+#include "dp.h"
+#include "../debug.h"
+
+static int ahb_err, pci_err;
+
+int ath12k_wifi7_arch_init(struct ath12k_base *ab)
+{
+	struct ath12k_dp *dp;
+
+	dp = ath12k_wifi7_dp_device_alloc(ab);
+	if (!dp) {
+		ath12k_err(ab, "dp alloc failed");
+		return -EINVAL;
+	}
+
+	ab->dp = dp;
+
+	return 0;
+}
+
+void ath12k_wifi7_arch_deinit(struct ath12k_base *ab)
+{
+	ath12k_wifi7_dp_device_free(ab->dp);
+	ab->dp = NULL;
+}
+
+static int ath12k_wifi7_init(void)
+{
+	ahb_err = ath12k_wifi7_ahb_init();
+	if (ahb_err)
+		pr_warn("Failed to initialize ath12k Wi-Fi 7 AHB device: %d\n",
+			ahb_err);
+
+	pci_err = ath12k_wifi7_pci_init();
+	if (pci_err)
+		pr_warn("Failed to initialize ath12k Wi-Fi 7 PCI device: %d\n",
+			pci_err);
+
+	/* If both failed, return one of the failures (arbitrary) */
+	return ahb_err && pci_err ? ahb_err : 0;
+}
+
+static void ath12k_wifi7_exit(void)
+{
+	if (!pci_err)
+		ath12k_wifi7_pci_exit();
+
+	if (!ahb_err)
+		ath12k_wifi7_ahb_exit();
+}
+
+module_init(ath12k_wifi7_init);
+module_exit(ath12k_wifi7_exit);
+
+MODULE_DESCRIPTION("Driver support for Qualcomm Technologies 802.11be WLAN devices");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/core.h b/drivers/net/wireless/ath/ath12k/wifi7/core.h
new file mode 100644
index 000000000000..7e9689d2ddd7
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/core.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+#ifndef ATH12K_CORE_WIFI7_H
+#define ATH12K_CORE_WIFI7_H
+
+int ath12k_wifi7_arch_init(struct ath12k_base *ab);
+void ath12k_wifi7_arch_deinit(struct ath12k_base *ab);
+
+#endif
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp.c b/drivers/net/wireless/ath/ath12k/wifi7/dp.c
new file mode 100644
index 000000000000..2b194879ee80
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp.c
@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+#include "../core.h"
+#include "../debug.h"
+#include "../dp_rx.h"
+#include "../dp_tx.h"
+#include "hal_desc.h"
+#include "../dp_mon.h"
+#include "dp_mon.h"
+#include "../dp_cmn.h"
+#include "dp_rx.h"
+#include "dp.h"
+#include "dp_tx.h"
+#include "hal.h"
+
+static int ath12k_wifi7_dp_service_srng(struct ath12k_dp *dp,
+					struct ath12k_ext_irq_grp *irq_grp,
+					int budget)
+{
+	struct napi_struct *napi = &irq_grp->napi;
+	int grp_id = irq_grp->grp_id;
+	int work_done = 0;
+	int i = 0, j;
+	int tot_work_done = 0;
+	enum dp_monitor_mode monitor_mode;
+	u8 ring_mask;
+
+	if (dp->hw_params->ring_mask->tx[grp_id]) {
+		i = fls(dp->hw_params->ring_mask->tx[grp_id]) - 1;
+		ath12k_wifi7_dp_tx_completion_handler(dp, i);
+	}
+
+	if (dp->hw_params->ring_mask->rx_err[grp_id]) {
+		work_done = ath12k_wifi7_dp_rx_process_err(dp, napi, budget);
+		budget -= work_done;
+		tot_work_done += work_done;
+		if (budget <= 0)
+			goto done;
+	}
+
+	if (dp->hw_params->ring_mask->rx_wbm_rel[grp_id]) {
+		work_done = ath12k_wifi7_dp_rx_process_wbm_err(dp, napi, budget);
+		budget -= work_done;
+		tot_work_done += work_done;
+
+		if (budget <= 0)
+			goto done;
+	}
+
+	if (dp->hw_params->ring_mask->rx[grp_id]) {
+		i = fls(dp->hw_params->ring_mask->rx[grp_id]) - 1;
+		work_done = ath12k_wifi7_dp_rx_process(dp, i, napi, budget);
+		budget -= work_done;
+		tot_work_done += work_done;
+		if (budget <= 0)
+			goto done;
+	}
+
+	if (dp->hw_params->ring_mask->rx_mon_status[grp_id]) {
+		ring_mask = dp->hw_params->ring_mask->rx_mon_status[grp_id];
+		for (i = 0; i < dp->ab->num_radios; i++) {
+			for (j = 0; j < dp->hw_params->num_rxdma_per_pdev; j++) {
+				int id = i * dp->hw_params->num_rxdma_per_pdev + j;
+
+				if (ring_mask & BIT(id)) {
+					work_done =
+					ath12k_wifi7_dp_mon_process_ring(dp, id, napi,
+									 budget,
+									 0);
+					budget -= work_done;
+					tot_work_done += work_done;
+					if (budget <= 0)
+						goto done;
+				}
+			}
+		}
+	}
+
+	if (dp->hw_params->ring_mask->rx_mon_dest[grp_id]) {
+		monitor_mode = ATH12K_DP_RX_MONITOR_MODE;
+		ring_mask = dp->hw_params->ring_mask->rx_mon_dest[grp_id];
+		for (i = 0; i < dp->ab->num_radios; i++) {
+			for (j = 0; j < dp->hw_params->num_rxdma_per_pdev; j++) {
+				int id = i * dp->hw_params->num_rxdma_per_pdev + j;
+
+				if (ring_mask & BIT(id)) {
+					work_done =
+					ath12k_wifi7_dp_mon_process_ring(dp, id, napi,
+									 budget,
+									 monitor_mode);
+					budget -= work_done;
+					tot_work_done += work_done;
+
+					if (budget <= 0)
+						goto done;
+				}
+			}
+		}
+	}
+
+	if (dp->hw_params->ring_mask->tx_mon_dest[grp_id]) {
+		monitor_mode = ATH12K_DP_TX_MONITOR_MODE;
+		ring_mask = dp->hw_params->ring_mask->tx_mon_dest[grp_id];
+		for (i = 0; i < dp->ab->num_radios; i++) {
+			for (j = 0; j < dp->hw_params->num_rxdma_per_pdev; j++) {
+				int id = i * dp->hw_params->num_rxdma_per_pdev + j;
+
+				if (ring_mask & BIT(id)) {
+					work_done =
+					ath12k_wifi7_dp_mon_process_ring(dp, id,
+									 napi, budget,
+									 monitor_mode);
+					budget -= work_done;
+					tot_work_done += work_done;
+
+					if (budget <= 0)
+						goto done;
+				}
+			}
+		}
+	}
+
+	if (dp->hw_params->ring_mask->reo_status[grp_id])
+		ath12k_wifi7_dp_rx_process_reo_status(dp);
+
+	if (dp->hw_params->ring_mask->host2rxdma[grp_id]) {
+		struct dp_rxdma_ring *rx_ring = &dp->rx_refill_buf_ring;
+		LIST_HEAD(list);
+
+		ath12k_dp_rx_bufs_replenish(dp, rx_ring, &list, 0);
+	}
+
+	/* TODO: Implement handler for other interrupts */
+
+done:
+	return tot_work_done;
+}
+
+static struct ath12k_dp_arch_ops ath12k_wifi7_dp_arch_ops = {
+	.service_srng = ath12k_wifi7_dp_service_srng,
+	.tx_get_vdev_bank_config = ath12k_wifi7_dp_tx_get_vdev_bank_config,
+	.reo_cmd_send = ath12k_wifi7_dp_reo_cmd_send,
+	.setup_pn_check_reo_cmd = ath12k_wifi7_dp_setup_pn_check_reo_cmd,
+	.rx_peer_tid_delete = ath12k_wifi7_dp_rx_peer_tid_delete,
+	.reo_cache_flush = ath12k_wifi7_dp_reo_cache_flush,
+	.rx_link_desc_return = ath12k_wifi7_dp_rx_link_desc_return,
+	.rx_frags_cleanup = ath12k_wifi7_dp_rx_frags_cleanup,
+	.peer_rx_tid_reo_update = ath12k_wifi7_peer_rx_tid_reo_update,
+	.rx_assign_reoq = ath12k_wifi7_dp_rx_assign_reoq,
+	.peer_rx_tid_qref_setup = ath12k_wifi7_peer_rx_tid_qref_setup,
+	.peer_rx_tid_qref_reset = ath12k_wifi7_peer_rx_tid_qref_reset,
+	.rx_tid_delete_handler = ath12k_wifi7_dp_rx_tid_delete_handler,
+};
+
+/* TODO: remove export once this file is built with wifi7 ko */
+struct ath12k_dp *ath12k_wifi7_dp_device_alloc(struct ath12k_base *ab)
+{
+	struct ath12k_dp *dp;
+
+	/* TODO: align dp later if cache alignment becomes a bottleneck */
+	dp = kzalloc(sizeof(*dp), GFP_KERNEL);
+	if (!dp)
+		return NULL;
+
+	dp->ab = ab;
+	dp->dev = ab->dev;
+	dp->hw_params = ab->hw_params;
+	dp->hal = &ab->hal;
+
+	dp->ops = &ath12k_wifi7_dp_arch_ops;
+
+	return dp;
+}
+
+void ath12k_wifi7_dp_device_free(struct ath12k_dp *dp)
+{
+	kfree(dp);
+}
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp.h b/drivers/net/wireless/ath/ath12k/wifi7/dp.h
new file mode 100644
index 000000000000..a5f0941d34e2
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef ATH12K_DP_WIFI7_H
+#define ATH12K_DP_WIFI7_H
+
+#include "../dp_cmn.h"
+#include "hw.h"
+
+struct ath12k_base;
+struct ath12k_dp;
+enum dp_monitor_mode;
+
+struct ath12k_dp *ath12k_wifi7_dp_device_alloc(struct ath12k_base *ab);
+void ath12k_wifi7_dp_device_free(struct ath12k_dp *dp);
+
+#endif
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c
new file mode 100644
index 000000000000..bd741532b7dc
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c
@@ -0,0 +1,3385 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include "hal_desc.h"
+#include "../dp_mon.h"
+#include "dp_mon.h"
+#include "../debug.h"
+#include "hal_qcn9274.h"
+#include "dp_rx.h"
+#include "../dp_tx.h"
+#include "../peer.h"
+
+static void
+ath12k_wifi7_dp_mon_hal_aggr_tlv(struct hal_rx_mon_ppdu_info *ppdu_info,
+				 u16 tlv_len, const void *tlv_data)
+{
+	if (tlv_len <= HAL_RX_MON_MAX_AGGR_SIZE - ppdu_info->tlv_aggr.cur_len) {
+		memcpy(ppdu_info->tlv_aggr.buf + ppdu_info->tlv_aggr.cur_len,
+		       tlv_data, tlv_len);
+		ppdu_info->tlv_aggr.cur_len += tlv_len;
+	}
+}
+
+static void
+ath12k_wifi7_dp_mon_rx_memset_ppdu_info(struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	memset(ppdu_info, 0, sizeof(*ppdu_info));
+	ppdu_info->peer_id = HAL_INVALID_PEERID;
+}
+
+/* Hardware fill buffer with 128 bytes aligned. So need to reap it
+ * with 128 bytes aligned.
+ */
+#define RXDMA_DATA_DMA_BLOCK_SIZE 128
+
+static void
+ath12k_wifi7_dp_mon_get_buf_len(struct hal_rx_msdu_desc_info *info,
+				bool *is_frag, u32 *total_len,
+				u32 *frag_len, u32 *msdu_cnt)
+{
+	if (info->msdu_flags & RX_MSDU_DESC_INFO0_MSDU_CONTINUATION) {
+		*is_frag = true;
+		*frag_len = (RX_MON_STATUS_BASE_BUF_SIZE -
+			     sizeof(struct hal_rx_desc)) &
+			     ~(RXDMA_DATA_DMA_BLOCK_SIZE - 1);
+		*total_len += *frag_len;
+	} else {
+		if (*is_frag)
+			*frag_len = info->msdu_len - *total_len;
+		else
+			*frag_len = info->msdu_len;
+
+		*msdu_cnt -= 1;
+	}
+}
+
+static void
+ath12k_wifi7_dp_mon_rx_handle_ofdma_info(const struct hal_rx_ppdu_end_user_stats *ppdu_end_user,
+					 struct hal_rx_user_status *rx_user_status)
+{
+	rx_user_status->ul_ofdma_user_v0_word0 =
+		__le32_to_cpu(ppdu_end_user->usr_resp_ref);
+	rx_user_status->ul_ofdma_user_v0_word1 =
+		__le32_to_cpu(ppdu_end_user->usr_resp_ref_ext);
+}
+
+static void
+ath12k_wifi7_dp_mon_rx_populate_byte_count(const struct hal_rx_ppdu_end_user_stats *stats,
+					   void *ppduinfo,
+					   struct hal_rx_user_status *rx_user_status)
+{
+	rx_user_status->mpdu_ok_byte_count =
+		le32_get_bits(stats->info7,
+			      HAL_RX_PPDU_END_USER_STATS_INFO7_MPDU_OK_BYTE_COUNT);
+	rx_user_status->mpdu_err_byte_count =
+		le32_get_bits(stats->info8,
+			      HAL_RX_PPDU_END_USER_STATS_INFO8_MPDU_ERR_BYTE_COUNT);
+}
+
+static void
+ath12k_wifi7_dp_mon_rx_populate_mu_user_info(const struct hal_rx_ppdu_end_user_stats *rx_tlv,
+					     struct hal_rx_mon_ppdu_info *ppdu_info,
+					     struct hal_rx_user_status *rx_user_status)
+{
+	rx_user_status->ast_index = ppdu_info->ast_index;
+	rx_user_status->tid = ppdu_info->tid;
+	rx_user_status->tcp_ack_msdu_count =
+		ppdu_info->tcp_ack_msdu_count;
+	rx_user_status->tcp_msdu_count =
+		ppdu_info->tcp_msdu_count;
+	rx_user_status->udp_msdu_count =
+		ppdu_info->udp_msdu_count;
+	rx_user_status->other_msdu_count =
+		ppdu_info->other_msdu_count;
+	rx_user_status->frame_control = ppdu_info->frame_control;
+	rx_user_status->frame_control_info_valid =
+		ppdu_info->frame_control_info_valid;
+	rx_user_status->data_sequence_control_info_valid =
+		ppdu_info->data_sequence_control_info_valid;
+	rx_user_status->first_data_seq_ctrl =
+		ppdu_info->first_data_seq_ctrl;
+	rx_user_status->preamble_type = ppdu_info->preamble_type;
+	rx_user_status->ht_flags = ppdu_info->ht_flags;
+	rx_user_status->vht_flags = ppdu_info->vht_flags;
+	rx_user_status->he_flags = ppdu_info->he_flags;
+	rx_user_status->rs_flags = ppdu_info->rs_flags;
+
+	rx_user_status->mpdu_cnt_fcs_ok =
+		ppdu_info->num_mpdu_fcs_ok;
+	rx_user_status->mpdu_cnt_fcs_err =
+		ppdu_info->num_mpdu_fcs_err;
+	memcpy(&rx_user_status->mpdu_fcs_ok_bitmap[0], &ppdu_info->mpdu_fcs_ok_bitmap[0],
+	       HAL_RX_NUM_WORDS_PER_PPDU_BITMAP *
+	       sizeof(ppdu_info->mpdu_fcs_ok_bitmap[0]));
+
+	ath12k_wifi7_dp_mon_rx_populate_byte_count(rx_tlv, ppdu_info, rx_user_status);
+}
+
+static inline enum ath12k_eht_ru_size
+hal_rx_mon_hal_ru_size_to_ath12k_ru_size(u32 hal_ru_size)
+{
+	switch (hal_ru_size) {
+	case HAL_EHT_RU_26:
+		return ATH12K_EHT_RU_26;
+	case HAL_EHT_RU_52:
+		return ATH12K_EHT_RU_52;
+	case HAL_EHT_RU_78:
+		return ATH12K_EHT_RU_52_26;
+	case HAL_EHT_RU_106:
+		return ATH12K_EHT_RU_106;
+	case HAL_EHT_RU_132:
+		return ATH12K_EHT_RU_106_26;
+	case HAL_EHT_RU_242:
+		return ATH12K_EHT_RU_242;
+	case HAL_EHT_RU_484:
+		return ATH12K_EHT_RU_484;
+	case HAL_EHT_RU_726:
+		return ATH12K_EHT_RU_484_242;
+	case HAL_EHT_RU_996:
+		return ATH12K_EHT_RU_996;
+	case HAL_EHT_RU_996x2:
+		return ATH12K_EHT_RU_996x2;
+	case HAL_EHT_RU_996x3:
+		return ATH12K_EHT_RU_996x3;
+	case HAL_EHT_RU_996x4:
+		return ATH12K_EHT_RU_996x4;
+	case HAL_EHT_RU_NONE:
+		return ATH12K_EHT_RU_INVALID;
+	case HAL_EHT_RU_996_484:
+		return ATH12K_EHT_RU_996_484;
+	case HAL_EHT_RU_996x2_484:
+		return ATH12K_EHT_RU_996x2_484;
+	case HAL_EHT_RU_996x3_484:
+		return ATH12K_EHT_RU_996x3_484;
+	case HAL_EHT_RU_996_484_242:
+		return ATH12K_EHT_RU_996_484_242;
+	default:
+		return ATH12K_EHT_RU_INVALID;
+	}
+}
+
+static inline u32
+hal_rx_ul_ofdma_ru_size_to_width(enum ath12k_eht_ru_size ru_size)
+{
+	switch (ru_size) {
+	case ATH12K_EHT_RU_26:
+		return RU_26;
+	case ATH12K_EHT_RU_52:
+		return RU_52;
+	case ATH12K_EHT_RU_52_26:
+		return RU_52_26;
+	case ATH12K_EHT_RU_106:
+		return RU_106;
+	case ATH12K_EHT_RU_106_26:
+		return RU_106_26;
+	case ATH12K_EHT_RU_242:
+		return RU_242;
+	case ATH12K_EHT_RU_484:
+		return RU_484;
+	case ATH12K_EHT_RU_484_242:
+		return RU_484_242;
+	case ATH12K_EHT_RU_996:
+		return RU_996;
+	case ATH12K_EHT_RU_996_484:
+		return RU_996_484;
+	case ATH12K_EHT_RU_996_484_242:
+		return RU_996_484_242;
+	case ATH12K_EHT_RU_996x2:
+		return RU_2X996;
+	case ATH12K_EHT_RU_996x2_484:
+		return RU_2X996_484;
+	case ATH12K_EHT_RU_996x3:
+		return RU_3X996;
+	case ATH12K_EHT_RU_996x3_484:
+		return RU_3X996_484;
+	case ATH12K_EHT_RU_996x4:
+		return RU_4X996;
+	default:
+		return RU_INVALID;
+	}
+}
+
+static void
+ath12k_wifi7_dp_mon_hal_rx_parse_user_info(const struct hal_receive_user_info *rx_usr_info,
+					   u16 user_id,
+					   struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	struct hal_rx_user_status *mon_rx_user_status = NULL;
+	struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht;
+	enum ath12k_eht_ru_size rtap_ru_size = ATH12K_EHT_RU_INVALID;
+	u32 ru_width, reception_type, ru_index = HAL_EHT_RU_INVALID;
+	u32 ru_type_80_0, ru_start_index_80_0;
+	u32 ru_type_80_1, ru_start_index_80_1;
+	u32 ru_type_80_2, ru_start_index_80_2;
+	u32 ru_type_80_3, ru_start_index_80_3;
+	u32 ru_size = 0, num_80mhz_with_ru = 0;
+	u64 ru_index_320mhz = 0;
+	u32 ru_index_per80mhz;
+
+	reception_type = le32_get_bits(rx_usr_info->info0,
+				       HAL_RX_USR_INFO0_RECEPTION_TYPE);
+
+	switch (reception_type) {
+	case HAL_RECEPTION_TYPE_SU:
+		ppdu_info->reception_type = HAL_RX_RECEPTION_TYPE_SU;
+		break;
+	case HAL_RECEPTION_TYPE_DL_MU_MIMO:
+	case HAL_RECEPTION_TYPE_UL_MU_MIMO:
+		ppdu_info->reception_type = HAL_RX_RECEPTION_TYPE_MU_MIMO;
+		break;
+	case HAL_RECEPTION_TYPE_DL_MU_OFMA:
+	case HAL_RECEPTION_TYPE_UL_MU_OFDMA:
+		ppdu_info->reception_type = HAL_RX_RECEPTION_TYPE_MU_OFDMA;
+		break;
+	case HAL_RECEPTION_TYPE_DL_MU_OFDMA_MIMO:
+	case HAL_RECEPTION_TYPE_UL_MU_OFDMA_MIMO:
+		ppdu_info->reception_type = HAL_RX_RECEPTION_TYPE_MU_OFDMA_MIMO;
+	}
+
+	ppdu_info->is_stbc = le32_get_bits(rx_usr_info->info0, HAL_RX_USR_INFO0_STBC);
+	ppdu_info->ldpc = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_LDPC);
+	ppdu_info->dcm = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_STA_DCM);
+	ppdu_info->bw = le32_get_bits(rx_usr_info->info1, HAL_RX_USR_INFO1_RX_BW);
+	ppdu_info->mcs = le32_get_bits(rx_usr_info->info1, HAL_RX_USR_INFO1_MCS);
+	ppdu_info->nss = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_NSS) + 1;
+
+	if (user_id < HAL_MAX_UL_MU_USERS) {
+		mon_rx_user_status = &ppdu_info->userstats[user_id];
+		mon_rx_user_status->mcs = ppdu_info->mcs;
+		mon_rx_user_status->nss = ppdu_info->nss;
+	}
+
+	if (!(ppdu_info->reception_type == HAL_RX_RECEPTION_TYPE_MU_MIMO ||
+	      ppdu_info->reception_type == HAL_RX_RECEPTION_TYPE_MU_OFDMA ||
+	      ppdu_info->reception_type == HAL_RX_RECEPTION_TYPE_MU_OFDMA_MIMO))
+		return;
+
+	/* RU allocation present only for OFDMA reception */
+	ru_type_80_0 = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_RU_TYPE_80_0);
+	ru_start_index_80_0 = le32_get_bits(rx_usr_info->info3,
+					    HAL_RX_USR_INFO3_RU_START_IDX_80_0);
+	if (ru_type_80_0 != HAL_EHT_RU_NONE) {
+		ru_size += ru_type_80_0;
+		ru_index_per80mhz = ru_start_index_80_0;
+		ru_index = ru_index_per80mhz;
+		ru_index_320mhz |= HAL_RU_PER80(ru_type_80_0, 0, ru_index_per80mhz);
+		num_80mhz_with_ru++;
+	}
+
+	ru_type_80_1 = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_RU_TYPE_80_1);
+	ru_start_index_80_1 = le32_get_bits(rx_usr_info->info3,
+					    HAL_RX_USR_INFO3_RU_START_IDX_80_1);
+	if (ru_type_80_1 != HAL_EHT_RU_NONE) {
+		ru_size += ru_type_80_1;
+		ru_index_per80mhz = ru_start_index_80_1;
+		ru_index = ru_index_per80mhz;
+		ru_index_320mhz |= HAL_RU_PER80(ru_type_80_1, 1, ru_index_per80mhz);
+		num_80mhz_with_ru++;
+	}
+
+	ru_type_80_2 = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_RU_TYPE_80_2);
+	ru_start_index_80_2 = le32_get_bits(rx_usr_info->info3,
+					    HAL_RX_USR_INFO3_RU_START_IDX_80_2);
+	if (ru_type_80_2 != HAL_EHT_RU_NONE) {
+		ru_size += ru_type_80_2;
+		ru_index_per80mhz = ru_start_index_80_2;
+		ru_index = ru_index_per80mhz;
+		ru_index_320mhz |= HAL_RU_PER80(ru_type_80_2, 2, ru_index_per80mhz);
+		num_80mhz_with_ru++;
+	}
+
+	ru_type_80_3 = le32_get_bits(rx_usr_info->info2, HAL_RX_USR_INFO2_RU_TYPE_80_3);
+	ru_start_index_80_3 = le32_get_bits(rx_usr_info->info2,
+					    HAL_RX_USR_INFO3_RU_START_IDX_80_3);
+	if (ru_type_80_3 != HAL_EHT_RU_NONE) {
+		ru_size += ru_type_80_3;
+		ru_index_per80mhz = ru_start_index_80_3;
+		ru_index = ru_index_per80mhz;
+		ru_index_320mhz |= HAL_RU_PER80(ru_type_80_3, 3, ru_index_per80mhz);
+		num_80mhz_with_ru++;
+	}
+
+	if (num_80mhz_with_ru > 1) {
+		/* Calculate the MRU index */
+		switch (ru_index_320mhz) {
+		case HAL_EHT_RU_996_484_0:
+		case HAL_EHT_RU_996x2_484_0:
+		case HAL_EHT_RU_996x3_484_0:
+			ru_index = 0;
+			break;
+		case HAL_EHT_RU_996_484_1:
+		case HAL_EHT_RU_996x2_484_1:
+		case HAL_EHT_RU_996x3_484_1:
+			ru_index = 1;
+			break;
+		case HAL_EHT_RU_996_484_2:
+		case HAL_EHT_RU_996x2_484_2:
+		case HAL_EHT_RU_996x3_484_2:
+			ru_index = 2;
+			break;
+		case HAL_EHT_RU_996_484_3:
+		case HAL_EHT_RU_996x2_484_3:
+		case HAL_EHT_RU_996x3_484_3:
+			ru_index = 3;
+			break;
+		case HAL_EHT_RU_996_484_4:
+		case HAL_EHT_RU_996x2_484_4:
+		case HAL_EHT_RU_996x3_484_4:
+			ru_index = 4;
+			break;
+		case HAL_EHT_RU_996_484_5:
+		case HAL_EHT_RU_996x2_484_5:
+		case HAL_EHT_RU_996x3_484_5:
+			ru_index = 5;
+			break;
+		case HAL_EHT_RU_996_484_6:
+		case HAL_EHT_RU_996x2_484_6:
+		case HAL_EHT_RU_996x3_484_6:
+			ru_index = 6;
+			break;
+		case HAL_EHT_RU_996_484_7:
+		case HAL_EHT_RU_996x2_484_7:
+		case HAL_EHT_RU_996x3_484_7:
+			ru_index = 7;
+			break;
+		case HAL_EHT_RU_996x2_484_8:
+			ru_index = 8;
+			break;
+		case HAL_EHT_RU_996x2_484_9:
+			ru_index = 9;
+			break;
+		case HAL_EHT_RU_996x2_484_10:
+			ru_index = 10;
+			break;
+		case HAL_EHT_RU_996x2_484_11:
+			ru_index = 11;
+			break;
+		default:
+			ru_index = HAL_EHT_RU_INVALID;
+			break;
+		}
+
+		ru_size += 4;
+	}
+
+	rtap_ru_size = hal_rx_mon_hal_ru_size_to_ath12k_ru_size(ru_size);
+	if (rtap_ru_size != ATH12K_EHT_RU_INVALID) {
+		u32 known, data;
+
+		known = __le32_to_cpu(eht->known);
+		known |= IEEE80211_RADIOTAP_EHT_KNOWN_RU_MRU_SIZE_OM;
+		eht->known = cpu_to_le32(known);
+
+		data = __le32_to_cpu(eht->data[1]);
+		data |=	u32_encode_bits(rtap_ru_size,
+					IEEE80211_RADIOTAP_EHT_DATA1_RU_SIZE);
+		eht->data[1] = cpu_to_le32(data);
+	}
+
+	if (ru_index != HAL_EHT_RU_INVALID) {
+		u32 known, data;
+
+		known = __le32_to_cpu(eht->known);
+		known |= IEEE80211_RADIOTAP_EHT_KNOWN_RU_MRU_INDEX_OM;
+		eht->known = cpu_to_le32(known);
+
+		data = __le32_to_cpu(eht->data[1]);
+		data |=	u32_encode_bits(rtap_ru_size,
+					IEEE80211_RADIOTAP_EHT_DATA1_RU_INDEX);
+		eht->data[1] = cpu_to_le32(data);
+	}
+
+	if (mon_rx_user_status && ru_index != HAL_EHT_RU_INVALID &&
+	    rtap_ru_size != ATH12K_EHT_RU_INVALID) {
+		mon_rx_user_status->ul_ofdma_ru_start_index = ru_index;
+		mon_rx_user_status->ul_ofdma_ru_size = rtap_ru_size;
+
+		ru_width = hal_rx_ul_ofdma_ru_size_to_width(rtap_ru_size);
+
+		mon_rx_user_status->ul_ofdma_ru_width = ru_width;
+		mon_rx_user_status->ofdma_info_valid = 1;
+	}
+}
+
+static void
+ath12k_wifi7_dp_mon_parse_l_sig_b(const struct hal_rx_lsig_b_info *lsigb,
+				  struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	u32 info0 = __le32_to_cpu(lsigb->info0);
+	u8 rate;
+
+	rate = u32_get_bits(info0, HAL_RX_LSIG_B_INFO_INFO0_RATE);
+	switch (rate) {
+	case 1:
+		rate = HAL_RX_LEGACY_RATE_1_MBPS;
+		break;
+	case 2:
+	case 5:
+		rate = HAL_RX_LEGACY_RATE_2_MBPS;
+		break;
+	case 3:
+	case 6:
+		rate = HAL_RX_LEGACY_RATE_5_5_MBPS;
+		break;
+	case 4:
+	case 7:
+		rate = HAL_RX_LEGACY_RATE_11_MBPS;
+		break;
+	default:
+		rate = HAL_RX_LEGACY_RATE_INVALID;
+	}
+
+	ppdu_info->rate = rate;
+	ppdu_info->cck_flag = 1;
+}
+
+static void
+ath12k_wifi7_dp_mon_parse_l_sig_a(const struct hal_rx_lsig_a_info *lsiga,
+				  struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	u32 info0 = __le32_to_cpu(lsiga->info0);
+	u8 rate;
+
+	rate = u32_get_bits(info0, HAL_RX_LSIG_A_INFO_INFO0_RATE);
+	switch (rate) {
+	case 8:
+		rate = HAL_RX_LEGACY_RATE_48_MBPS;
+		break;
+	case 9:
+		rate = HAL_RX_LEGACY_RATE_24_MBPS;
+		break;
+	case 10:
+		rate = HAL_RX_LEGACY_RATE_12_MBPS;
+		break;
+	case 11:
+		rate = HAL_RX_LEGACY_RATE_6_MBPS;
+		break;
+	case 12:
+		rate = HAL_RX_LEGACY_RATE_54_MBPS;
+		break;
+	case 13:
+		rate = HAL_RX_LEGACY_RATE_36_MBPS;
+		break;
+	case 14:
+		rate = HAL_RX_LEGACY_RATE_18_MBPS;
+		break;
+	case 15:
+		rate = HAL_RX_LEGACY_RATE_9_MBPS;
+		break;
+	default:
+		rate = HAL_RX_LEGACY_RATE_INVALID;
+	}
+
+	ppdu_info->rate = rate;
+}
+
+static void
+ath12k_wifi7_dp_mon_hal_rx_parse_u_sig_cmn(const struct hal_mon_usig_cmn *cmn,
+					   struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	u32 common;
+
+	ppdu_info->u_sig_info.bw = le32_get_bits(cmn->info0,
+						 HAL_RX_USIG_CMN_INFO0_BW);
+	ppdu_info->u_sig_info.ul_dl = le32_get_bits(cmn->info0,
+						    HAL_RX_USIG_CMN_INFO0_UL_DL);
+
+	common = __le32_to_cpu(ppdu_info->u_sig_info.usig.common);
+	common |= IEEE80211_RADIOTAP_EHT_USIG_COMMON_PHY_VER_KNOWN |
+		  IEEE80211_RADIOTAP_EHT_USIG_COMMON_BW_KNOWN |
+		  IEEE80211_RADIOTAP_EHT_USIG_COMMON_UL_DL_KNOWN |
+		  IEEE80211_RADIOTAP_EHT_USIG_COMMON_BSS_COLOR_KNOWN |
+		  IEEE80211_RADIOTAP_EHT_USIG_COMMON_TXOP_KNOWN |
+		  ATH12K_LE32_DEC_ENC(cmn->info0,
+				      HAL_RX_USIG_CMN_INFO0_PHY_VERSION,
+				      IEEE80211_RADIOTAP_EHT_USIG_COMMON_PHY_VER) |
+		  u32_encode_bits(ppdu_info->u_sig_info.bw,
+				  IEEE80211_RADIOTAP_EHT_USIG_COMMON_BW) |
+		  u32_encode_bits(ppdu_info->u_sig_info.ul_dl,
+				  IEEE80211_RADIOTAP_EHT_USIG_COMMON_UL_DL) |
+		  ATH12K_LE32_DEC_ENC(cmn->info0,
+				      HAL_RX_USIG_CMN_INFO0_BSS_COLOR,
+				      IEEE80211_RADIOTAP_EHT_USIG_COMMON_BSS_COLOR) |
+		  ATH12K_LE32_DEC_ENC(cmn->info0,
+				      HAL_RX_USIG_CMN_INFO0_TXOP,
+				      IEEE80211_RADIOTAP_EHT_USIG_COMMON_TXOP);
+	ppdu_info->u_sig_info.usig.common = cpu_to_le32(common);
+
+	switch (ppdu_info->u_sig_info.bw) {
+	default:
+		fallthrough;
+	case HAL_EHT_BW_20:
+		ppdu_info->bw = HAL_RX_BW_20MHZ;
+		break;
+	case HAL_EHT_BW_40:
+		ppdu_info->bw = HAL_RX_BW_40MHZ;
+		break;
+	case HAL_EHT_BW_80:
+		ppdu_info->bw = HAL_RX_BW_80MHZ;
+		break;
+	case HAL_EHT_BW_160:
+		ppdu_info->bw = HAL_RX_BW_160MHZ;
+		break;
+	case HAL_EHT_BW_320_1:
+	case HAL_EHT_BW_320_2:
+		ppdu_info->bw = HAL_RX_BW_320MHZ;
+		break;
+	}
+}
+
+static void
+ath12k_wifi7_dp_mon_hal_rx_parse_u_sig_tb(const struct hal_mon_usig_tb *usig_tb,
+					  struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	struct ieee80211_radiotap_eht_usig *usig = &ppdu_info->u_sig_info.usig;
+	enum ieee80211_radiotap_eht_usig_tb spatial_reuse1, spatial_reuse2;
+	u32 common, value, mask;
+
+	spatial_reuse1 = IEEE80211_RADIOTAP_EHT_USIG2_TB_B3_B6_SPATIAL_REUSE_1;
+	spatial_reuse2 = IEEE80211_RADIOTAP_EHT_USIG2_TB_B7_B10_SPATIAL_REUSE_2;
+
+	common = __le32_to_cpu(usig->common);
+	value = __le32_to_cpu(usig->value);
+	mask = __le32_to_cpu(usig->mask);
+
+	ppdu_info->u_sig_info.ppdu_type_comp_mode =
+				le32_get_bits(usig_tb->info0,
+					      HAL_RX_USIG_TB_INFO0_PPDU_TYPE_COMP_MODE);
+
+	common |= ATH12K_LE32_DEC_ENC(usig_tb->info0,
+				      HAL_RX_USIG_TB_INFO0_RX_INTEG_CHECK_PASS,
+				      IEEE80211_RADIOTAP_EHT_USIG_COMMON_BAD_USIG_CRC);
+
+	value |= IEEE80211_RADIOTAP_EHT_USIG1_TB_B20_B25_DISREGARD |
+		 u32_encode_bits(ppdu_info->u_sig_info.ppdu_type_comp_mode,
+				 IEEE80211_RADIOTAP_EHT_USIG2_TB_B0_B1_PPDU_TYPE) |
+		 IEEE80211_RADIOTAP_EHT_USIG2_TB_B2_VALIDATE |
+		 ATH12K_LE32_DEC_ENC(usig_tb->info0,
+				     HAL_RX_USIG_TB_INFO0_SPATIAL_REUSE_1,
+				     spatial_reuse1) |
+		 ATH12K_LE32_DEC_ENC(usig_tb->info0,
+				     HAL_RX_USIG_TB_INFO0_SPATIAL_REUSE_2,
+				     spatial_reuse2) |
+		 IEEE80211_RADIOTAP_EHT_USIG2_TB_B11_B15_DISREGARD |
+		 ATH12K_LE32_DEC_ENC(usig_tb->info0,
+				     HAL_RX_USIG_TB_INFO0_CRC,
+				     IEEE80211_RADIOTAP_EHT_USIG2_TB_B16_B19_CRC) |
+		 ATH12K_LE32_DEC_ENC(usig_tb->info0,
+				     HAL_RX_USIG_TB_INFO0_TAIL,
+				     IEEE80211_RADIOTAP_EHT_USIG2_TB_B20_B25_TAIL);
+
+	mask |= IEEE80211_RADIOTAP_EHT_USIG1_TB_B20_B25_DISREGARD |
+		IEEE80211_RADIOTAP_EHT_USIG2_TB_B0_B1_PPDU_TYPE |
+		IEEE80211_RADIOTAP_EHT_USIG2_TB_B2_VALIDATE |
+		spatial_reuse1 | spatial_reuse2 |
+		IEEE80211_RADIOTAP_EHT_USIG2_TB_B11_B15_DISREGARD |
+		IEEE80211_RADIOTAP_EHT_USIG2_TB_B16_B19_CRC |
+		IEEE80211_RADIOTAP_EHT_USIG2_TB_B20_B25_TAIL;
+
+	usig->common = cpu_to_le32(common);
+	usig->value = cpu_to_le32(value);
+	usig->mask = cpu_to_le32(mask);
+}
+
+static void
+ath12k_wifi7_dp_mon_hal_rx_parse_u_sig_mu(const struct hal_mon_usig_mu *usig_mu,
+					  struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	struct ieee80211_radiotap_eht_usig *usig = &ppdu_info->u_sig_info.usig;
+	enum ieee80211_radiotap_eht_usig_mu sig_symb, punc;
+	u32 common, value, mask;
+
+	sig_symb = IEEE80211_RADIOTAP_EHT_USIG2_MU_B11_B15_EHT_SIG_SYMBOLS;
+	punc = IEEE80211_RADIOTAP_EHT_USIG2_MU_B3_B7_PUNCTURED_INFO;
+
+	common = __le32_to_cpu(usig->common);
+	value = __le32_to_cpu(usig->value);
+	mask = __le32_to_cpu(usig->mask);
+
+	ppdu_info->u_sig_info.ppdu_type_comp_mode =
+				le32_get_bits(usig_mu->info0,
+					      HAL_RX_USIG_MU_INFO0_PPDU_TYPE_COMP_MODE);
+	ppdu_info->u_sig_info.eht_sig_mcs =
+				le32_get_bits(usig_mu->info0,
+					      HAL_RX_USIG_MU_INFO0_EHT_SIG_MCS);
+	ppdu_info->u_sig_info.num_eht_sig_sym =
+				le32_get_bits(usig_mu->info0,
+					      HAL_RX_USIG_MU_INFO0_NUM_EHT_SIG_SYM);
+
+	common |= ATH12K_LE32_DEC_ENC(usig_mu->info0,
+				      HAL_RX_USIG_MU_INFO0_RX_INTEG_CHECK_PASS,
+				      IEEE80211_RADIOTAP_EHT_USIG_COMMON_BAD_USIG_CRC);
+
+	value |= IEEE80211_RADIOTAP_EHT_USIG1_MU_B20_B24_DISREGARD |
+		 IEEE80211_RADIOTAP_EHT_USIG1_MU_B25_VALIDATE |
+		 u32_encode_bits(ppdu_info->u_sig_info.ppdu_type_comp_mode,
+				 IEEE80211_RADIOTAP_EHT_USIG2_MU_B0_B1_PPDU_TYPE) |
+		 IEEE80211_RADIOTAP_EHT_USIG2_MU_B2_VALIDATE |
+		 ATH12K_LE32_DEC_ENC(usig_mu->info0,
+				     HAL_RX_USIG_MU_INFO0_PUNC_CH_INFO,
+				     punc) |
+		 IEEE80211_RADIOTAP_EHT_USIG2_MU_B8_VALIDATE |
+		 u32_encode_bits(ppdu_info->u_sig_info.eht_sig_mcs,
+				 IEEE80211_RADIOTAP_EHT_USIG2_MU_B9_B10_SIG_MCS) |
+		 u32_encode_bits(ppdu_info->u_sig_info.num_eht_sig_sym,
+				 sig_symb) |
+		 ATH12K_LE32_DEC_ENC(usig_mu->info0,
+				     HAL_RX_USIG_MU_INFO0_CRC,
+				     IEEE80211_RADIOTAP_EHT_USIG2_MU_B16_B19_CRC) |
+		 ATH12K_LE32_DEC_ENC(usig_mu->info0,
+				     HAL_RX_USIG_MU_INFO0_TAIL,
+				     IEEE80211_RADIOTAP_EHT_USIG2_MU_B20_B25_TAIL);
+
+	mask |= IEEE80211_RADIOTAP_EHT_USIG1_MU_B20_B24_DISREGARD |
+		IEEE80211_RADIOTAP_EHT_USIG1_MU_B25_VALIDATE |
+		IEEE80211_RADIOTAP_EHT_USIG2_MU_B0_B1_PPDU_TYPE |
+		IEEE80211_RADIOTAP_EHT_USIG2_MU_B2_VALIDATE |
+		punc |
+		IEEE80211_RADIOTAP_EHT_USIG2_MU_B8_VALIDATE |
+		IEEE80211_RADIOTAP_EHT_USIG2_MU_B9_B10_SIG_MCS |
+		sig_symb |
+		IEEE80211_RADIOTAP_EHT_USIG2_MU_B16_B19_CRC |
+		IEEE80211_RADIOTAP_EHT_USIG2_MU_B20_B25_TAIL;
+
+	usig->common = cpu_to_le32(common);
+	usig->value = cpu_to_le32(value);
+	usig->mask = cpu_to_le32(mask);
+}
+
+static void
+ath12k_wifi7_dp_mon_hal_rx_parse_u_sig_hdr(const struct hal_mon_usig_hdr *usig,
+					   struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	u8 comp_mode;
+
+	ppdu_info->eht_usig = true;
+
+	ath12k_wifi7_dp_mon_hal_rx_parse_u_sig_cmn(&usig->cmn, ppdu_info);
+
+	comp_mode = le32_get_bits(usig->non_cmn.mu.info0,
+				  HAL_RX_USIG_MU_INFO0_PPDU_TYPE_COMP_MODE);
+
+	if (comp_mode == 0 && ppdu_info->u_sig_info.ul_dl)
+		ath12k_wifi7_dp_mon_hal_rx_parse_u_sig_tb(&usig->non_cmn.tb, ppdu_info);
+	else
+		ath12k_wifi7_dp_mon_hal_rx_parse_u_sig_mu(&usig->non_cmn.mu, ppdu_info);
+}
+
+static void
+ath12k_wifi7_dp_mon_parse_vht_sig_a(const struct hal_rx_vht_sig_a_info *vht_sig,
+				    struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	u32 nsts, info0, info1;
+	u8 gi_setting;
+
+	info0 = __le32_to_cpu(vht_sig->info0);
+	info1 = __le32_to_cpu(vht_sig->info1);
+
+	ppdu_info->ldpc = u32_get_bits(info1, HAL_RX_VHT_SIG_A_INFO_INFO1_SU_MU_CODING);
+	ppdu_info->mcs = u32_get_bits(info1, HAL_RX_VHT_SIG_A_INFO_INFO1_MCS);
+	gi_setting = u32_get_bits(info1, HAL_RX_VHT_SIG_A_INFO_INFO1_GI_SETTING);
+	switch (gi_setting) {
+	case HAL_RX_VHT_SIG_A_NORMAL_GI:
+		ppdu_info->gi = HAL_RX_GI_0_8_US;
+		break;
+	case HAL_RX_VHT_SIG_A_SHORT_GI:
+	case HAL_RX_VHT_SIG_A_SHORT_GI_AMBIGUITY:
+		ppdu_info->gi = HAL_RX_GI_0_4_US;
+		break;
+	}
+
+	ppdu_info->is_stbc = u32_get_bits(info0, HAL_RX_VHT_SIG_A_INFO_INFO0_STBC);
+	nsts = u32_get_bits(info0, HAL_RX_VHT_SIG_A_INFO_INFO0_NSTS);
+	if (ppdu_info->is_stbc && nsts > 0)
+		nsts = ((nsts + 1) >> 1) - 1;
+
+	ppdu_info->nss = u32_get_bits(nsts, VHT_SIG_SU_NSS_MASK) + 1;
+	ppdu_info->bw = u32_get_bits(info0, HAL_RX_VHT_SIG_A_INFO_INFO0_BW);
+	ppdu_info->beamformed = u32_get_bits(info1,
+					     HAL_RX_VHT_SIG_A_INFO_INFO1_BEAMFORMED);
+	ppdu_info->vht_flag_values5 = u32_get_bits(info0,
+						   HAL_RX_VHT_SIG_A_INFO_INFO0_GROUP_ID);
+	ppdu_info->vht_flag_values3[0] = (((ppdu_info->mcs) << 4) |
+					    ppdu_info->nss);
+	ppdu_info->vht_flag_values2 = ppdu_info->bw;
+	ppdu_info->vht_flag_values4 =
+		u32_get_bits(info1, HAL_RX_VHT_SIG_A_INFO_INFO1_SU_MU_CODING);
+}
+
+static void
+ath12k_wifi7_dp_mon_parse_ht_sig(const struct hal_rx_ht_sig_info *ht_sig,
+				 struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	u32 info0 = __le32_to_cpu(ht_sig->info0);
+	u32 info1 = __le32_to_cpu(ht_sig->info1);
+
+	ppdu_info->mcs = u32_get_bits(info0, HAL_RX_HT_SIG_INFO_INFO0_MCS);
+	ppdu_info->bw = u32_get_bits(info0, HAL_RX_HT_SIG_INFO_INFO0_BW);
+	ppdu_info->is_stbc = u32_get_bits(info1, HAL_RX_HT_SIG_INFO_INFO1_STBC);
+	ppdu_info->ldpc = u32_get_bits(info1, HAL_RX_HT_SIG_INFO_INFO1_FEC_CODING);
+	ppdu_info->gi = u32_get_bits(info1, HAL_RX_HT_SIG_INFO_INFO1_GI);
+	ppdu_info->nss = (ppdu_info->mcs >> 3) + 1;
+}
+
+static void
+ath12k_wifi7_dp_mon_parse_he_sig_b2_ofdma(const struct hal_rx_he_sig_b2_ofdma_info *ofdma,
+					  struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	u32 info0, value;
+
+	info0 = __le32_to_cpu(ofdma->info0);
+
+	ppdu_info->he_data1 |= HE_MCS_KNOWN | HE_DCM_KNOWN | HE_CODING_KNOWN;
+
+	/* HE-data2 */
+	ppdu_info->he_data2 |= HE_TXBF_KNOWN;
+
+	ppdu_info->mcs = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_MCS);
+	value = ppdu_info->mcs << HE_TRANSMIT_MCS_SHIFT;
+	ppdu_info->he_data3 |= value;
+
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_DCM);
+	value = value << HE_DCM_SHIFT;
+	ppdu_info->he_data3 |= value;
+
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_CODING);
+	ppdu_info->ldpc = value;
+	value = value << HE_CODING_SHIFT;
+	ppdu_info->he_data3 |= value;
+
+	/* HE-data4 */
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_ID);
+	value = value << HE_STA_ID_SHIFT;
+	ppdu_info->he_data4 |= value;
+
+	ppdu_info->nss =
+		u32_get_bits(info0,
+			     HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_NSTS) + 1;
+	ppdu_info->beamformed = u32_get_bits(info0,
+					     HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_TXBF);
+}
+
+static void
+ath12k_wifi7_dp_mon_parse_he_sig_b2_mu(const struct hal_rx_he_sig_b2_mu_info *he_sig_b2_mu,
+				       struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	u32 info0, value;
+
+	info0 = __le32_to_cpu(he_sig_b2_mu->info0);
+
+	ppdu_info->he_data1 |= HE_MCS_KNOWN | HE_CODING_KNOWN;
+
+	ppdu_info->mcs = u32_get_bits(info0, HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_MCS);
+	value = ppdu_info->mcs << HE_TRANSMIT_MCS_SHIFT;
+	ppdu_info->he_data3 |= value;
+
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_CODING);
+	ppdu_info->ldpc = value;
+	value = value << HE_CODING_SHIFT;
+	ppdu_info->he_data3 |= value;
+
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_ID);
+	value = value << HE_STA_ID_SHIFT;
+	ppdu_info->he_data4 |= value;
+
+	ppdu_info->nss =
+		u32_get_bits(info0,
+			     HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_NSTS) + 1;
+}
+
+static void
+ath12k_wifi7_dp_mon_parse_he_sig_b1_mu(const struct hal_rx_he_sig_b1_mu_info *he_sig_b1_mu,
+				       struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	u32 info0 = __le32_to_cpu(he_sig_b1_mu->info0);
+	u16 ru_tones;
+
+	ru_tones = u32_get_bits(info0,
+				HAL_RX_HE_SIG_B1_MU_INFO_INFO0_RU_ALLOCATION);
+	ppdu_info->ru_alloc = ath12k_he_ru_tones_to_nl80211_he_ru_alloc(ru_tones);
+	ppdu_info->he_RU[0] = ru_tones;
+}
+
+static void
+ath12k_wifi7_dp_mon_parse_he_sig_mu(const struct hal_rx_he_sig_a_mu_dl_info *he_sig_a_mu_dl,
+				    struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	u32 info0, info1, value;
+	u16 he_gi = 0, he_ltf = 0;
+
+	info0 = __le32_to_cpu(he_sig_a_mu_dl->info0);
+	info1 = __le32_to_cpu(he_sig_a_mu_dl->info1);
+
+	ppdu_info->he_mu_flags = 1;
+
+	ppdu_info->he_data1 = HE_MU_FORMAT_TYPE;
+	ppdu_info->he_data1 |=
+			HE_BSS_COLOR_KNOWN |
+			HE_DL_UL_KNOWN |
+			HE_LDPC_EXTRA_SYMBOL_KNOWN |
+			HE_STBC_KNOWN |
+			HE_DATA_BW_RU_KNOWN |
+			HE_DOPPLER_KNOWN;
+
+	ppdu_info->he_data2 =
+			HE_GI_KNOWN |
+			HE_LTF_SYMBOLS_KNOWN |
+			HE_PRE_FEC_PADDING_KNOWN |
+			HE_PE_DISAMBIGUITY_KNOWN |
+			HE_TXOP_KNOWN |
+			HE_MIDABLE_PERIODICITY_KNOWN;
+
+	/* data3 */
+	ppdu_info->he_data3 = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_BSS_COLOR);
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_UL_FLAG);
+	value = value << HE_DL_UL_SHIFT;
+	ppdu_info->he_data3 |= value;
+
+	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_LDPC_EXTRA);
+	value = value << HE_LDPC_EXTRA_SYMBOL_SHIFT;
+	ppdu_info->he_data3 |= value;
+
+	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_STBC);
+	value = value << HE_STBC_SHIFT;
+	ppdu_info->he_data3 |= value;
+
+	/* data4 */
+	ppdu_info->he_data4 = u32_get_bits(info0,
+					   HAL_RX_HE_SIG_A_MU_DL_INFO0_SPATIAL_REUSE);
+	ppdu_info->he_data4 = value;
+
+	/* data5 */
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_TRANSMIT_BW);
+	ppdu_info->he_data5 = value;
+	ppdu_info->bw = value;
+
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_CP_LTF_SIZE);
+	switch (value) {
+	case 0:
+		he_gi = HE_GI_0_8;
+		he_ltf = HE_LTF_4_X;
+		break;
+	case 1:
+		he_gi = HE_GI_0_8;
+		he_ltf = HE_LTF_2_X;
+		break;
+	case 2:
+		he_gi = HE_GI_1_6;
+		he_ltf = HE_LTF_2_X;
+		break;
+	case 3:
+		he_gi = HE_GI_3_2;
+		he_ltf = HE_LTF_4_X;
+		break;
+	}
+
+	ppdu_info->gi = he_gi;
+	value = he_gi << HE_GI_SHIFT;
+	ppdu_info->he_data5 |= value;
+
+	value = he_ltf << HE_LTF_SIZE_SHIFT;
+	ppdu_info->he_data5 |= value;
+
+	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_NUM_LTF_SYMB);
+	value = (value << HE_LTF_SYM_SHIFT);
+	ppdu_info->he_data5 |= value;
+
+	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_PKT_EXT_FACTOR);
+	value = value << HE_PRE_FEC_PAD_SHIFT;
+	ppdu_info->he_data5 |= value;
+
+	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_PKT_EXT_PE_DISAM);
+	value = value << HE_PE_DISAMBIGUITY_SHIFT;
+	ppdu_info->he_data5 |= value;
+
+	/*data6*/
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_DOPPLER_INDICATION);
+	value = value << HE_DOPPLER_SHIFT;
+	ppdu_info->he_data6 |= value;
+
+	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_MU_DL_INFO1_TXOP_DURATION);
+	value = value << HE_TXOP_SHIFT;
+	ppdu_info->he_data6 |= value;
+
+	/* HE-MU Flags */
+	/* HE-MU-flags1 */
+	ppdu_info->he_flags1 =
+		HE_SIG_B_MCS_KNOWN |
+		HE_SIG_B_DCM_KNOWN |
+		HE_SIG_B_COMPRESSION_FLAG_1_KNOWN |
+		HE_SIG_B_SYM_NUM_KNOWN |
+		HE_RU_0_KNOWN;
+
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_MCS_OF_SIGB);
+	ppdu_info->he_flags1 |= value;
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_DCM_OF_SIGB);
+	value = value << HE_DCM_FLAG_1_SHIFT;
+	ppdu_info->he_flags1 |= value;
+
+	/* HE-MU-flags2 */
+	ppdu_info->he_flags2 = HE_BW_KNOWN;
+
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_TRANSMIT_BW);
+	ppdu_info->he_flags2 |= value;
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_COMP_MODE_SIGB);
+	value = value << HE_SIG_B_COMPRESSION_FLAG_2_SHIFT;
+	ppdu_info->he_flags2 |= value;
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_MU_DL_INFO0_NUM_SIGB_SYMB);
+	value = value - 1;
+	value = value << HE_NUM_SIG_B_SYMBOLS_SHIFT;
+	ppdu_info->he_flags2 |= value;
+
+	ppdu_info->is_stbc = info1 &
+			     HAL_RX_HE_SIG_A_MU_DL_INFO1_STBC;
+}
+
+static void
+ath12k_wifi7_dp_mon_parse_he_sig_su(const struct hal_rx_he_sig_a_su_info *he_sig_a,
+				    struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	u32 info0, info1, value;
+	u32 dcm;
+	u8 he_dcm = 0, he_stbc = 0;
+	u16 he_gi = 0, he_ltf = 0;
+
+	ppdu_info->he_flags = 1;
+
+	info0 = __le32_to_cpu(he_sig_a->info0);
+	info1 = __le32_to_cpu(he_sig_a->info1);
+
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_FORMAT_IND);
+	if (value == 0)
+		ppdu_info->he_data1 = HE_TRIG_FORMAT_TYPE;
+	else
+		ppdu_info->he_data1 = HE_SU_FORMAT_TYPE;
+
+	ppdu_info->he_data1 |=
+			HE_BSS_COLOR_KNOWN |
+			HE_BEAM_CHANGE_KNOWN |
+			HE_DL_UL_KNOWN |
+			HE_MCS_KNOWN |
+			HE_DCM_KNOWN |
+			HE_CODING_KNOWN |
+			HE_LDPC_EXTRA_SYMBOL_KNOWN |
+			HE_STBC_KNOWN |
+			HE_DATA_BW_RU_KNOWN |
+			HE_DOPPLER_KNOWN;
+
+	ppdu_info->he_data2 |=
+			HE_GI_KNOWN |
+			HE_TXBF_KNOWN |
+			HE_PE_DISAMBIGUITY_KNOWN |
+			HE_TXOP_KNOWN |
+			HE_LTF_SYMBOLS_KNOWN |
+			HE_PRE_FEC_PADDING_KNOWN |
+			HE_MIDABLE_PERIODICITY_KNOWN;
+
+	ppdu_info->he_data3 = u32_get_bits(info0,
+					   HAL_RX_HE_SIG_A_SU_INFO_INFO0_BSS_COLOR);
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_BEAM_CHANGE);
+	value = value << HE_BEAM_CHANGE_SHIFT;
+	ppdu_info->he_data3 |= value;
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_DL_UL_FLAG);
+	value = value << HE_DL_UL_SHIFT;
+	ppdu_info->he_data3 |= value;
+
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_MCS);
+	ppdu_info->mcs = value;
+	value = value << HE_TRANSMIT_MCS_SHIFT;
+	ppdu_info->he_data3 |= value;
+
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_DCM);
+	he_dcm = value;
+	value = value << HE_DCM_SHIFT;
+	ppdu_info->he_data3 |= value;
+	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_CODING);
+	value = value << HE_CODING_SHIFT;
+	ppdu_info->he_data3 |= value;
+	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_LDPC_EXTRA);
+	value = value << HE_LDPC_EXTRA_SYMBOL_SHIFT;
+	ppdu_info->he_data3 |= value;
+	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_STBC);
+	he_stbc = value;
+	value = value << HE_STBC_SHIFT;
+	ppdu_info->he_data3 |= value;
+
+	/* data4 */
+	ppdu_info->he_data4 = u32_get_bits(info0,
+					   HAL_RX_HE_SIG_A_SU_INFO_INFO0_SPATIAL_REUSE);
+
+	/* data5 */
+	value = u32_get_bits(info0,
+			     HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_BW);
+	ppdu_info->he_data5 = value;
+	ppdu_info->bw = value;
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_CP_LTF_SIZE);
+	switch (value) {
+	case 0:
+		he_gi = HE_GI_0_8;
+		he_ltf = HE_LTF_1_X;
+		break;
+	case 1:
+		he_gi = HE_GI_0_8;
+		he_ltf = HE_LTF_2_X;
+		break;
+	case 2:
+		he_gi = HE_GI_1_6;
+		he_ltf = HE_LTF_2_X;
+		break;
+	case 3:
+		if (he_dcm && he_stbc) {
+			he_gi = HE_GI_0_8;
+			he_ltf = HE_LTF_4_X;
+		} else {
+			he_gi = HE_GI_3_2;
+			he_ltf = HE_LTF_4_X;
+		}
+		break;
+	}
+	ppdu_info->gi = he_gi;
+	value = he_gi << HE_GI_SHIFT;
+	ppdu_info->he_data5 |= value;
+	value = he_ltf << HE_LTF_SIZE_SHIFT;
+	ppdu_info->ltf_size = he_ltf;
+	ppdu_info->he_data5 |= value;
+
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_NSTS);
+	value = (value << HE_LTF_SYM_SHIFT);
+	ppdu_info->he_data5 |= value;
+
+	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_PKT_EXT_FACTOR);
+	value = value << HE_PRE_FEC_PAD_SHIFT;
+	ppdu_info->he_data5 |= value;
+
+	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_TXBF);
+	value = value << HE_TXBF_SHIFT;
+	ppdu_info->he_data5 |= value;
+	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_PKT_EXT_PE_DISAM);
+	value = value << HE_PE_DISAMBIGUITY_SHIFT;
+	ppdu_info->he_data5 |= value;
+
+	/* data6 */
+	value = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_NSTS);
+	value++;
+	ppdu_info->he_data6 = value;
+	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_DOPPLER_IND);
+	value = value << HE_DOPPLER_SHIFT;
+	ppdu_info->he_data6 |= value;
+	value = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_TXOP_DURATION);
+	value = value << HE_TXOP_SHIFT;
+	ppdu_info->he_data6 |= value;
+
+	ppdu_info->mcs =
+		u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_MCS);
+	ppdu_info->bw =
+		u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_BW);
+	ppdu_info->ldpc = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_CODING);
+	ppdu_info->is_stbc = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_STBC);
+	ppdu_info->beamformed = u32_get_bits(info1, HAL_RX_HE_SIG_A_SU_INFO_INFO1_TXBF);
+	dcm = u32_get_bits(info0, HAL_RX_HE_SIG_A_SU_INFO_INFO0_DCM);
+	ppdu_info->nss = u32_get_bits(info0,
+				      HAL_RX_HE_SIG_A_SU_INFO_INFO0_NSTS) + 1;
+	ppdu_info->dcm = dcm;
+}
+
+static inline bool
+ath12k_wifi7_dp_mon_hal_rx_is_non_ofdma(const struct hal_rx_u_sig_info *usig_info)
+{
+	u32 ppdu_type_comp_mode = usig_info->ppdu_type_comp_mode;
+	u32 ul_dl = usig_info->ul_dl;
+
+	if ((ppdu_type_comp_mode == HAL_RX_RECEPTION_TYPE_MU_MIMO && ul_dl == 0) ||
+	    (ppdu_type_comp_mode == HAL_RX_RECEPTION_TYPE_MU_OFDMA && ul_dl == 0) ||
+	    (ppdu_type_comp_mode == HAL_RX_RECEPTION_TYPE_MU_MIMO  && ul_dl == 1))
+		return true;
+
+	return false;
+}
+
+static inline bool
+ath12k_wifi7_dp_mon_hal_rx_is_ofdma(const struct hal_rx_u_sig_info *usig_info)
+{
+	if (usig_info->ppdu_type_comp_mode == 0 && usig_info->ul_dl == 0)
+		return true;
+
+	return false;
+}
+
+static inline bool
+ath12k_wifi7_dp_mon_hal_rx_is_frame_type_ndp(const struct hal_rx_u_sig_info *usig_info)
+{
+	if (usig_info->ppdu_type_comp_mode == 1 &&
+	    usig_info->eht_sig_mcs == 0 &&
+	    usig_info->num_eht_sig_sym == 0)
+		return true;
+
+	return false;
+}
+
+static void
+ath12k_wifi7_dp_mon_hal_rx_parse_eht_sig_ndp(const struct hal_eht_sig_ndp_cmn_eb *eht_sig_ndp,
+					     struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht;
+	u32 known, data;
+
+	known = __le32_to_cpu(eht->known);
+	known |= IEEE80211_RADIOTAP_EHT_KNOWN_SPATIAL_REUSE |
+		 IEEE80211_RADIOTAP_EHT_KNOWN_EHT_LTF |
+		 IEEE80211_RADIOTAP_EHT_KNOWN_NSS_S |
+		 IEEE80211_RADIOTAP_EHT_KNOWN_BEAMFORMED_S |
+		 IEEE80211_RADIOTAP_EHT_KNOWN_DISREGARD_S |
+		 IEEE80211_RADIOTAP_EHT_KNOWN_CRC1 |
+		 IEEE80211_RADIOTAP_EHT_KNOWN_TAIL1;
+	eht->known = cpu_to_le32(known);
+
+	data = __le32_to_cpu(eht->data[0]);
+	data |= ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0,
+				    HAL_RX_EHT_SIG_NDP_CMN_INFO0_SPATIAL_REUSE,
+				    IEEE80211_RADIOTAP_EHT_DATA0_SPATIAL_REUSE);
+	/* GI and LTF size are separately indicated in radiotap header
+	 * and hence will be parsed from other TLV
+	 */
+	data |=	ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0,
+				    HAL_RX_EHT_SIG_NDP_CMN_INFO0_NUM_LTF_SYM,
+				    IEEE80211_RADIOTAP_EHT_DATA0_EHT_LTF);
+
+	data |=	ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0,
+				    HAL_RX_EHT_SIG_NDP_CMN_INFO0_CRC,
+				    IEEE80211_RADIOTAP_EHT_DATA0_CRC1_O);
+
+	data |= ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0,
+				    HAL_RX_EHT_SIG_NDP_CMN_INFO0_DISREGARD,
+				    IEEE80211_RADIOTAP_EHT_DATA0_DISREGARD_S);
+	eht->data[0] = cpu_to_le32(data);
+
+	data = __le32_to_cpu(eht->data[7]);
+	data |=	ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0,
+				    HAL_RX_EHT_SIG_NDP_CMN_INFO0_NSS,
+				    IEEE80211_RADIOTAP_EHT_DATA7_NSS_S);
+
+	data |=	ATH12K_LE32_DEC_ENC(eht_sig_ndp->info0,
+				    HAL_RX_EHT_SIG_NDP_CMN_INFO0_BEAMFORMED,
+				    IEEE80211_RADIOTAP_EHT_DATA7_BEAMFORMED_S);
+	eht->data[7] = cpu_to_le32(data);
+}
+
+static void
+ath12k_wifi7_dp_mon_hal_rx_parse_usig_overflow(const struct hal_eht_sig_usig_overflow *ovflow,
+					       struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht;
+	u32 known, data;
+
+	known = __le32_to_cpu(eht->known);
+	known |= IEEE80211_RADIOTAP_EHT_KNOWN_SPATIAL_REUSE |
+		 IEEE80211_RADIOTAP_EHT_KNOWN_EHT_LTF |
+		 IEEE80211_RADIOTAP_EHT_KNOWN_LDPC_EXTRA_SYM_OM |
+		 IEEE80211_RADIOTAP_EHT_KNOWN_PRE_PADD_FACOR_OM |
+		 IEEE80211_RADIOTAP_EHT_KNOWN_PE_DISAMBIGUITY_OM |
+		 IEEE80211_RADIOTAP_EHT_KNOWN_DISREGARD_O;
+	eht->known = cpu_to_le32(known);
+
+	data = __le32_to_cpu(eht->data[0]);
+	data |=	ATH12K_LE32_DEC_ENC(ovflow->info0,
+				    HAL_RX_EHT_SIG_OVERFLOW_INFO0_SPATIAL_REUSE,
+				    IEEE80211_RADIOTAP_EHT_DATA0_SPATIAL_REUSE);
+
+	/* GI and LTF size are separately indicated in radiotap header
+	 * and hence will be parsed from other TLV
+	 */
+	data |=	ATH12K_LE32_DEC_ENC(ovflow->info0,
+				    HAL_RX_EHT_SIG_OVERFLOW_INFO0_NUM_LTF_SYM,
+				    IEEE80211_RADIOTAP_EHT_DATA0_EHT_LTF);
+
+	data |=	ATH12K_LE32_DEC_ENC(ovflow->info0,
+				    HAL_RX_EHT_SIG_OVERFLOW_INFO0_LDPC_EXTA_SYM,
+				    IEEE80211_RADIOTAP_EHT_DATA0_LDPC_EXTRA_SYM_OM);
+
+	data |=	ATH12K_LE32_DEC_ENC(ovflow->info0,
+				    HAL_RX_EHT_SIG_OVERFLOW_INFO0_PRE_FEC_PAD_FACTOR,
+				    IEEE80211_RADIOTAP_EHT_DATA0_PRE_PADD_FACOR_OM);
+
+	data |=	ATH12K_LE32_DEC_ENC(ovflow->info0,
+				    HAL_RX_EHT_SIG_OVERFLOW_INFO0_DISAMBIGUITY,
+				    IEEE80211_RADIOTAP_EHT_DATA0_PE_DISAMBIGUITY_OM);
+
+	data |=	ATH12K_LE32_DEC_ENC(ovflow->info0,
+				    HAL_RX_EHT_SIG_OVERFLOW_INFO0_DISREGARD,
+				    IEEE80211_RADIOTAP_EHT_DATA0_DISREGARD_O);
+	eht->data[0] = cpu_to_le32(data);
+}
+
+static void
+ath12k_wifi7_dp_mon_hal_rx_parse_non_ofdma_users(const struct hal_eht_sig_non_ofdma_cmn_eb *eb,
+						 struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht;
+	u32 known, data;
+
+	known = __le32_to_cpu(eht->known);
+	known |= IEEE80211_RADIOTAP_EHT_KNOWN_NR_NON_OFDMA_USERS_M;
+	eht->known = cpu_to_le32(known);
+
+	data = __le32_to_cpu(eht->data[7]);
+	data |=	ATH12K_LE32_DEC_ENC(eb->info0,
+				    HAL_RX_EHT_SIG_NON_OFDMA_INFO0_NUM_USERS,
+				    IEEE80211_RADIOTAP_EHT_DATA7_NUM_OF_NON_OFDMA_USERS);
+	eht->data[7] = cpu_to_le32(data);
+}
+
+static void
+ath12k_wifi7_dp_mon_hal_rx_parse_eht_mumimo_user(const struct hal_eht_sig_mu_mimo *user,
+						 struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	struct hal_rx_eht_info *eht_info = &ppdu_info->eht_info;
+	u32 user_idx;
+
+	if (eht_info->num_user_info >= ARRAY_SIZE(eht_info->user_info))
+		return;
+
+	user_idx = eht_info->num_user_info++;
+
+	eht_info->user_info[user_idx] |=
+		IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID_KNOWN |
+		IEEE80211_RADIOTAP_EHT_USER_INFO_MCS_KNOWN |
+		IEEE80211_RADIOTAP_EHT_USER_INFO_CODING_KNOWN |
+		IEEE80211_RADIOTAP_EHT_USER_INFO_SPATIAL_CONFIG_KNOWN_M |
+		ATH12K_LE32_DEC_ENC(user->info0,
+				    HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_STA_ID,
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID) |
+		ATH12K_LE32_DEC_ENC(user->info0,
+				    HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_CODING,
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_CODING) |
+		ATH12K_LE32_DEC_ENC(user->info0,
+				    HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_MCS,
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_MCS) |
+		ATH12K_LE32_DEC_ENC(user->info0,
+				    HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_SPATIAL_CODING,
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_SPATIAL_CONFIG_M);
+
+	ppdu_info->mcs = le32_get_bits(user->info0,
+				       HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_MCS);
+}
+
+static void
+ath12k_wifi7_dp_mon_hal_rx_parse_eht_non_mumimo_user(const struct hal_eht_sig_non_mu_mimo *user,
+						     struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	struct hal_rx_eht_info *eht_info = &ppdu_info->eht_info;
+	u32 user_idx;
+
+	if (eht_info->num_user_info >= ARRAY_SIZE(eht_info->user_info))
+		return;
+
+	user_idx = eht_info->num_user_info++;
+
+	eht_info->user_info[user_idx] |=
+		IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID_KNOWN |
+		IEEE80211_RADIOTAP_EHT_USER_INFO_MCS_KNOWN |
+		IEEE80211_RADIOTAP_EHT_USER_INFO_CODING_KNOWN |
+		IEEE80211_RADIOTAP_EHT_USER_INFO_NSS_KNOWN_O |
+		IEEE80211_RADIOTAP_EHT_USER_INFO_BEAMFORMING_KNOWN_O |
+		ATH12K_LE32_DEC_ENC(user->info0,
+				    HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_STA_ID,
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID) |
+		ATH12K_LE32_DEC_ENC(user->info0,
+				    HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_CODING,
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_CODING) |
+		ATH12K_LE32_DEC_ENC(user->info0,
+				    HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_MCS,
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_MCS) |
+		ATH12K_LE32_DEC_ENC(user->info0,
+				    HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_NSS,
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_NSS_O) |
+		ATH12K_LE32_DEC_ENC(user->info0,
+				    HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_BEAMFORMED,
+				    IEEE80211_RADIOTAP_EHT_USER_INFO_BEAMFORMING_O);
+
+	ppdu_info->mcs = le32_get_bits(user->info0,
+				       HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_MCS);
+
+	ppdu_info->nss = le32_get_bits(user->info0,
+				       HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_NSS) + 1;
+}
+
+static inline bool
+ath12k_wifi7_dp_mon_hal_rx_is_mu_mimo_user(const struct hal_rx_u_sig_info *usig_info)
+{
+	if (usig_info->ppdu_type_comp_mode == HAL_RX_RECEPTION_TYPE_SU &&
+	    usig_info->ul_dl == 1)
+		return true;
+
+	return false;
+}
+
+static void
+ath12k_wifi7_dp_mon_hal_rx_parse_eht_sig_non_ofdma(const void *tlv,
+						   struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	const struct hal_eht_sig_non_ofdma_cmn_eb *eb = tlv;
+
+	ath12k_wifi7_dp_mon_hal_rx_parse_usig_overflow(tlv, ppdu_info);
+	ath12k_wifi7_dp_mon_hal_rx_parse_non_ofdma_users(eb, ppdu_info);
+
+	if (ath12k_wifi7_dp_mon_hal_rx_is_mu_mimo_user(&ppdu_info->u_sig_info))
+		ath12k_wifi7_dp_mon_hal_rx_parse_eht_mumimo_user(&eb->user_field.mu_mimo,
+								 ppdu_info);
+	else
+		ath12k_wifi7_dp_mon_hal_rx_parse_eht_non_mumimo_user(&eb->user_field.n_mu_mimo,
+								     ppdu_info);
+}
+
+static void
+ath12k_wifi7_dp_mon_hal_rx_parse_ru_allocation(const struct hal_eht_sig_ofdma_cmn_eb *eb,
+					       struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	const struct hal_eht_sig_ofdma_cmn_eb1 *ofdma_cmn_eb1 = &eb->eb1;
+	const struct hal_eht_sig_ofdma_cmn_eb2 *ofdma_cmn_eb2 = &eb->eb2;
+	struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht;
+	enum ieee80211_radiotap_eht_data ru_123, ru_124, ru_125, ru_126;
+	enum ieee80211_radiotap_eht_data ru_121, ru_122, ru_112, ru_111;
+	u32 data;
+
+	ru_123 = IEEE80211_RADIOTAP_EHT_DATA4_RU_ALLOC_CC_1_2_3;
+	ru_124 = IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_4;
+	ru_125 = IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_5;
+	ru_126 = IEEE80211_RADIOTAP_EHT_DATA6_RU_ALLOC_CC_1_2_6;
+	ru_121 = IEEE80211_RADIOTAP_EHT_DATA3_RU_ALLOC_CC_1_2_1;
+	ru_122 = IEEE80211_RADIOTAP_EHT_DATA3_RU_ALLOC_CC_1_2_2;
+	ru_112 = IEEE80211_RADIOTAP_EHT_DATA2_RU_ALLOC_CC_1_1_2;
+	ru_111 = IEEE80211_RADIOTAP_EHT_DATA1_RU_ALLOC_CC_1_1_1;
+
+	switch (ppdu_info->u_sig_info.bw) {
+	case HAL_EHT_BW_320_2:
+	case HAL_EHT_BW_320_1:
+		data = __le32_to_cpu(eht->data[4]);
+		/* CC1 2::3 */
+		data |=	IEEE80211_RADIOTAP_EHT_DATA4_RU_ALLOC_CC_1_2_3_KNOWN |
+			ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0,
+					    HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_3,
+					    ru_123);
+		eht->data[4] = cpu_to_le32(data);
+
+		data = __le32_to_cpu(eht->data[5]);
+		/* CC1 2::4 */
+		data |=	IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_4_KNOWN |
+			ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0,
+					    HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_4,
+					    ru_124);
+
+		/* CC1 2::5 */
+		data |=	IEEE80211_RADIOTAP_EHT_DATA5_RU_ALLOC_CC_1_2_5_KNOWN |
+			ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0,
+					    HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_5,
+					    ru_125);
+		eht->data[5] = cpu_to_le32(data);
+
+		data = __le32_to_cpu(eht->data[6]);
+		/* CC1 2::6 */
+		data |=	IEEE80211_RADIOTAP_EHT_DATA6_RU_ALLOC_CC_1_2_6_KNOWN |
+			ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0,
+					    HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_6,
+					    ru_126);
+		eht->data[6] = cpu_to_le32(data);
+
+		fallthrough;
+	case HAL_EHT_BW_160:
+		data = __le32_to_cpu(eht->data[3]);
+		/* CC1 2::1 */
+		data |=	IEEE80211_RADIOTAP_EHT_DATA3_RU_ALLOC_CC_1_2_1_KNOWN |
+			ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0,
+					    HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_1,
+					    ru_121);
+		/* CC1 2::2 */
+		data |=	IEEE80211_RADIOTAP_EHT_DATA3_RU_ALLOC_CC_1_2_2_KNOWN |
+			ATH12K_LE64_DEC_ENC(ofdma_cmn_eb2->info0,
+					    HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_2,
+					    ru_122);
+		eht->data[3] = cpu_to_le32(data);
+
+		fallthrough;
+	case HAL_EHT_BW_80:
+		data = __le32_to_cpu(eht->data[2]);
+		/* CC1 1::2 */
+		data |=	IEEE80211_RADIOTAP_EHT_DATA2_RU_ALLOC_CC_1_1_2_KNOWN |
+			ATH12K_LE64_DEC_ENC(ofdma_cmn_eb1->info0,
+					    HAL_RX_EHT_SIG_OFDMA_EB1_RU_ALLOC_1_2,
+					    ru_112);
+		eht->data[2] = cpu_to_le32(data);
+
+		fallthrough;
+	case HAL_EHT_BW_40:
+		fallthrough;
+	case HAL_EHT_BW_20:
+		data = __le32_to_cpu(eht->data[1]);
+		/* CC1 1::1 */
+		data |=	IEEE80211_RADIOTAP_EHT_DATA1_RU_ALLOC_CC_1_1_1_KNOWN |
+			ATH12K_LE64_DEC_ENC(ofdma_cmn_eb1->info0,
+					    HAL_RX_EHT_SIG_OFDMA_EB1_RU_ALLOC_1_1,
+					    ru_111);
+		eht->data[1] = cpu_to_le32(data);
+		break;
+	default:
+		break;
+	}
+}
+
+static void
+ath12k_wifi7_dp_mon_hal_rx_parse_eht_sig_ofdma(const void *tlv,
+					       struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	const struct hal_eht_sig_ofdma_cmn_eb *ofdma = tlv;
+
+	ath12k_wifi7_dp_mon_hal_rx_parse_usig_overflow(tlv, ppdu_info);
+	ath12k_wifi7_dp_mon_hal_rx_parse_ru_allocation(ofdma, ppdu_info);
+
+	ath12k_wifi7_dp_mon_hal_rx_parse_eht_non_mumimo_user(&ofdma->user_field.n_mu_mimo,
+							     ppdu_info);
+}
+
+static void
+ath12k_wifi7_dp_mon_parse_eht_sig_hdr(struct hal_rx_mon_ppdu_info *ppdu_info,
+				      const void *tlv_data)
+{
+	ppdu_info->is_eht = true;
+
+	if (ath12k_wifi7_dp_mon_hal_rx_is_frame_type_ndp(&ppdu_info->u_sig_info))
+		ath12k_wifi7_dp_mon_hal_rx_parse_eht_sig_ndp(tlv_data, ppdu_info);
+	else if (ath12k_wifi7_dp_mon_hal_rx_is_non_ofdma(&ppdu_info->u_sig_info))
+		ath12k_wifi7_dp_mon_hal_rx_parse_eht_sig_non_ofdma(tlv_data, ppdu_info);
+	else if (ath12k_wifi7_dp_mon_hal_rx_is_ofdma(&ppdu_info->u_sig_info))
+		ath12k_wifi7_dp_mon_hal_rx_parse_eht_sig_ofdma(tlv_data, ppdu_info);
+}
+
+static void ath12k_wifi7_dp_mon_parse_rx_msdu_end_err(u32 info, u32 *errmap)
+{
+	if (info & RX_MSDU_END_INFO13_FCS_ERR)
+		*errmap |= HAL_RX_MPDU_ERR_FCS;
+
+	if (info & RX_MSDU_END_INFO13_DECRYPT_ERR)
+		*errmap |= HAL_RX_MPDU_ERR_DECRYPT;
+
+	if (info & RX_MSDU_END_INFO13_TKIP_MIC_ERR)
+		*errmap |= HAL_RX_MPDU_ERR_TKIP_MIC;
+
+	if (info & RX_MSDU_END_INFO13_A_MSDU_ERROR)
+		*errmap |= HAL_RX_MPDU_ERR_AMSDU_ERR;
+
+	if (info & RX_MSDU_END_INFO13_OVERFLOW_ERR)
+		*errmap |= HAL_RX_MPDU_ERR_OVERFLOW;
+
+	if (info & RX_MSDU_END_INFO13_MSDU_LEN_ERR)
+		*errmap |= HAL_RX_MPDU_ERR_MSDU_LEN;
+
+	if (info & RX_MSDU_END_INFO13_MPDU_LEN_ERR)
+		*errmap |= HAL_RX_MPDU_ERR_MPDU_LEN;
+}
+
+static void
+ath12k_wifi7_parse_cmn_usr_info(const struct hal_phyrx_common_user_info *cmn_usr_info,
+				struct hal_rx_mon_ppdu_info *ppdu_info)
+{
+	struct hal_rx_radiotap_eht *eht = &ppdu_info->eht_info.eht;
+	u32 known, data, cp_setting, ltf_size;
+
+	known = __le32_to_cpu(eht->known);
+	known |= IEEE80211_RADIOTAP_EHT_KNOWN_GI |
+		IEEE80211_RADIOTAP_EHT_KNOWN_EHT_LTF;
+	eht->known = cpu_to_le32(known);
+
+	cp_setting = le32_get_bits(cmn_usr_info->info0,
+				   HAL_RX_CMN_USR_INFO0_CP_SETTING);
+	ltf_size = le32_get_bits(cmn_usr_info->info0,
+				 HAL_RX_CMN_USR_INFO0_LTF_SIZE);
+
+	data = __le32_to_cpu(eht->data[0]);
+	data |= u32_encode_bits(cp_setting, IEEE80211_RADIOTAP_EHT_DATA0_GI);
+	data |= u32_encode_bits(ltf_size, IEEE80211_RADIOTAP_EHT_DATA0_LTF);
+	eht->data[0] = cpu_to_le32(data);
+
+	if (!ppdu_info->ltf_size)
+		ppdu_info->ltf_size = ltf_size;
+	if (!ppdu_info->gi)
+		ppdu_info->gi = cp_setting;
+}
+
+static void
+ath12k_wifi7_dp_mon_parse_status_msdu_end(struct ath12k_mon_data *pmon,
+					  const struct hal_rx_msdu_end *msdu_end)
+{
+	ath12k_wifi7_dp_mon_parse_rx_msdu_end_err(__le32_to_cpu(msdu_end->info2),
+						  &pmon->err_bitmap);
+	pmon->decap_format = le32_get_bits(msdu_end->info1,
+					   RX_MSDU_END_INFO11_DECAP_FORMAT);
+}
+
+static enum hal_rx_mon_status
+ath12k_wifi7_dp_mon_rx_parse_status_tlv(struct ath12k_pdev_dp *dp_pdev,
+					struct ath12k_mon_data *pmon,
+					const struct hal_tlv_64_hdr *tlv)
+{
+	struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info;
+	const void *tlv_data = tlv->value;
+	u32 info[7], userid;
+	u16 tlv_tag, tlv_len;
+
+	tlv_tag = le64_get_bits(tlv->tl, HAL_TLV_64_HDR_TAG);
+	tlv_len = le64_get_bits(tlv->tl, HAL_TLV_64_HDR_LEN);
+	userid = le64_get_bits(tlv->tl, HAL_TLV_64_USR_ID);
+
+	if (ppdu_info->tlv_aggr.in_progress && ppdu_info->tlv_aggr.tlv_tag != tlv_tag) {
+		ath12k_wifi7_dp_mon_parse_eht_sig_hdr(ppdu_info,
+						      ppdu_info->tlv_aggr.buf);
+
+		ppdu_info->tlv_aggr.in_progress = false;
+		ppdu_info->tlv_aggr.cur_len = 0;
+	}
+
+	switch (tlv_tag) {
+	case HAL_RX_PPDU_START: {
+		const struct hal_rx_ppdu_start *ppdu_start = tlv_data;
+
+		u64 ppdu_ts = ath12k_le32hilo_to_u64(ppdu_start->ppdu_start_ts_63_32,
+						     ppdu_start->ppdu_start_ts_31_0);
+
+		info[0] = __le32_to_cpu(ppdu_start->info0);
+
+		ppdu_info->ppdu_id = u32_get_bits(info[0],
+						  HAL_RX_PPDU_START_INFO0_PPDU_ID);
+
+		info[1] = __le32_to_cpu(ppdu_start->info1);
+		ppdu_info->chan_num = u32_get_bits(info[1],
+						   HAL_RX_PPDU_START_INFO1_CHAN_NUM);
+		ppdu_info->freq = u32_get_bits(info[1],
+					       HAL_RX_PPDU_START_INFO1_CHAN_FREQ);
+		ppdu_info->ppdu_ts = ppdu_ts;
+
+		if (ppdu_info->ppdu_id != ppdu_info->last_ppdu_id) {
+			ppdu_info->last_ppdu_id = ppdu_info->ppdu_id;
+			ppdu_info->num_users = 0;
+			memset(&ppdu_info->mpdu_fcs_ok_bitmap, 0,
+			       HAL_RX_NUM_WORDS_PER_PPDU_BITMAP *
+			       sizeof(ppdu_info->mpdu_fcs_ok_bitmap[0]));
+		}
+		break;
+	}
+	case HAL_RX_PPDU_END_USER_STATS: {
+		const struct hal_rx_ppdu_end_user_stats *eu_stats = tlv_data;
+		u32 tid_bitmap;
+
+		info[0] = __le32_to_cpu(eu_stats->info0);
+		info[1] = __le32_to_cpu(eu_stats->info1);
+		info[2] = __le32_to_cpu(eu_stats->info2);
+		info[4] = __le32_to_cpu(eu_stats->info4);
+		info[5] = __le32_to_cpu(eu_stats->info5);
+		info[6] = __le32_to_cpu(eu_stats->info6);
+
+		ppdu_info->ast_index =
+			u32_get_bits(info[2], HAL_RX_PPDU_END_USER_STATS_INFO2_AST_INDEX);
+		ppdu_info->fc_valid =
+			u32_get_bits(info[1], HAL_RX_PPDU_END_USER_STATS_INFO1_FC_VALID);
+		tid_bitmap = u32_get_bits(info[6],
+					  HAL_RX_PPDU_END_USER_STATS_INFO6_TID_BITMAP);
+		ppdu_info->tid = ffs(tid_bitmap) - 1;
+		ppdu_info->tcp_msdu_count =
+			u32_get_bits(info[4],
+				     HAL_RX_PPDU_END_USER_STATS_INFO4_TCP_MSDU_CNT);
+		ppdu_info->udp_msdu_count =
+			u32_get_bits(info[4],
+				     HAL_RX_PPDU_END_USER_STATS_INFO4_UDP_MSDU_CNT);
+		ppdu_info->other_msdu_count =
+			u32_get_bits(info[5],
+				     HAL_RX_PPDU_END_USER_STATS_INFO5_OTHER_MSDU_CNT);
+		ppdu_info->tcp_ack_msdu_count =
+			u32_get_bits(info[5],
+				     HAL_RX_PPDU_END_USER_STATS_INFO5_TCP_ACK_MSDU_CNT);
+		ppdu_info->preamble_type =
+			u32_get_bits(info[1],
+				     HAL_RX_PPDU_END_USER_STATS_INFO1_PKT_TYPE);
+		ppdu_info->num_mpdu_fcs_ok =
+			u32_get_bits(info[1],
+				     HAL_RX_PPDU_END_USER_STATS_INFO1_MPDU_CNT_FCS_OK);
+		ppdu_info->num_mpdu_fcs_err =
+			u32_get_bits(info[0],
+				     HAL_RX_PPDU_END_USER_STATS_INFO0_MPDU_CNT_FCS_ERR);
+		ppdu_info->peer_id =
+			u32_get_bits(info[0], HAL_RX_PPDU_END_USER_STATS_INFO0_PEER_ID);
+
+		switch (ppdu_info->preamble_type) {
+		case HAL_RX_PREAMBLE_11N:
+			ppdu_info->ht_flags = 1;
+			break;
+		case HAL_RX_PREAMBLE_11AC:
+			ppdu_info->vht_flags = 1;
+			break;
+		case HAL_RX_PREAMBLE_11AX:
+			ppdu_info->he_flags = 1;
+			break;
+		case HAL_RX_PREAMBLE_11BE:
+			ppdu_info->is_eht = true;
+			break;
+		default:
+			break;
+		}
+
+		if (userid < HAL_MAX_UL_MU_USERS) {
+			struct hal_rx_user_status *rxuser_stats =
+				&ppdu_info->userstats[userid];
+
+			if (ppdu_info->num_mpdu_fcs_ok > 1 ||
+			    ppdu_info->num_mpdu_fcs_err > 1)
+				ppdu_info->userstats[userid].ampdu_present = true;
+
+			ppdu_info->num_users += 1;
+
+			ath12k_wifi7_dp_mon_rx_handle_ofdma_info(eu_stats, rxuser_stats);
+			ath12k_wifi7_dp_mon_rx_populate_mu_user_info(eu_stats, ppdu_info,
+								     rxuser_stats);
+		}
+		ppdu_info->mpdu_fcs_ok_bitmap[0] = __le32_to_cpu(eu_stats->rsvd1[0]);
+		ppdu_info->mpdu_fcs_ok_bitmap[1] = __le32_to_cpu(eu_stats->rsvd1[1]);
+		break;
+	}
+	case HAL_RX_PPDU_END_USER_STATS_EXT: {
+		const struct hal_rx_ppdu_end_user_stats_ext *eu_stats = tlv_data;
+
+		ppdu_info->mpdu_fcs_ok_bitmap[2] = __le32_to_cpu(eu_stats->info1);
+		ppdu_info->mpdu_fcs_ok_bitmap[3] = __le32_to_cpu(eu_stats->info2);
+		ppdu_info->mpdu_fcs_ok_bitmap[4] = __le32_to_cpu(eu_stats->info3);
+		ppdu_info->mpdu_fcs_ok_bitmap[5] = __le32_to_cpu(eu_stats->info4);
+		ppdu_info->mpdu_fcs_ok_bitmap[6] = __le32_to_cpu(eu_stats->info5);
+		ppdu_info->mpdu_fcs_ok_bitmap[7] = __le32_to_cpu(eu_stats->info6);
+		break;
+	}
+	case HAL_PHYRX_HT_SIG:
+		ath12k_wifi7_dp_mon_parse_ht_sig(tlv_data, ppdu_info);
+		break;
+
+	case HAL_PHYRX_L_SIG_B:
+		ath12k_wifi7_dp_mon_parse_l_sig_b(tlv_data, ppdu_info);
+		break;
+
+	case HAL_PHYRX_L_SIG_A:
+		ath12k_wifi7_dp_mon_parse_l_sig_a(tlv_data, ppdu_info);
+		break;
+
+	case HAL_PHYRX_VHT_SIG_A:
+		ath12k_wifi7_dp_mon_parse_vht_sig_a(tlv_data, ppdu_info);
+		break;
+
+	case HAL_PHYRX_HE_SIG_A_SU:
+		ath12k_wifi7_dp_mon_parse_he_sig_su(tlv_data, ppdu_info);
+		break;
+
+	case HAL_PHYRX_HE_SIG_A_MU_DL:
+		ath12k_wifi7_dp_mon_parse_he_sig_mu(tlv_data, ppdu_info);
+		break;
+
+	case HAL_PHYRX_HE_SIG_B1_MU:
+		ath12k_wifi7_dp_mon_parse_he_sig_b1_mu(tlv_data, ppdu_info);
+		break;
+
+	case HAL_PHYRX_HE_SIG_B2_MU:
+		ath12k_wifi7_dp_mon_parse_he_sig_b2_mu(tlv_data, ppdu_info);
+		break;
+
+	case HAL_PHYRX_HE_SIG_B2_OFDMA:
+		ath12k_wifi7_dp_mon_parse_he_sig_b2_ofdma(tlv_data, ppdu_info);
+		break;
+
+	case HAL_PHYRX_RSSI_LEGACY: {
+		const struct hal_rx_phyrx_rssi_legacy_info *rssi = tlv_data;
+
+		info[0] = __le32_to_cpu(rssi->info0);
+		info[2] = __le32_to_cpu(rssi->info2);
+
+		/* TODO: Please note that the combined rssi will not be accurate
+		 * in MU case. Rssi in MU needs to be retrieved from
+		 * PHYRX_OTHER_RECEIVE_INFO TLV.
+		 */
+		ppdu_info->rssi_comb =
+			u32_get_bits(info[2],
+				     HAL_RX_RSSI_LEGACY_INFO_INFO2_RSSI_COMB_PPDU);
+
+		ppdu_info->bw = u32_get_bits(info[0],
+					     HAL_RX_RSSI_LEGACY_INFO_INFO0_RX_BW);
+		break;
+	}
+	case HAL_PHYRX_COMMON_USER_INFO: {
+		ath12k_wifi7_parse_cmn_usr_info(tlv_data, ppdu_info);
+		break;
+	}
+	case HAL_RX_PPDU_START_USER_INFO:
+		ath12k_wifi7_dp_mon_hal_rx_parse_user_info(tlv_data, userid, ppdu_info);
+		break;
+
+	case HAL_RXPCU_PPDU_END_INFO: {
+		const struct hal_rx_ppdu_end_duration *ppdu_rx_duration = tlv_data;
+
+		info[0] = __le32_to_cpu(ppdu_rx_duration->info0);
+		ppdu_info->rx_duration =
+			u32_get_bits(info[0], HAL_RX_PPDU_END_DURATION);
+		ppdu_info->tsft = __le32_to_cpu(ppdu_rx_duration->rsvd0[1]);
+		ppdu_info->tsft = (ppdu_info->tsft << 32) |
+				   __le32_to_cpu(ppdu_rx_duration->rsvd0[0]);
+		break;
+	}
+	case HAL_RX_MPDU_START: {
+		const struct hal_rx_mpdu_start *mpdu_start = tlv_data;
+		u16 peer_id;
+
+		info[1] = __le32_to_cpu(mpdu_start->info1);
+		peer_id = u32_get_bits(info[1], HAL_RX_MPDU_START_INFO1_PEERID);
+		if (peer_id)
+			ppdu_info->peer_id = peer_id;
+
+		ppdu_info->mpdu_len += u32_get_bits(info[1],
+						    HAL_RX_MPDU_START_INFO2_MPDU_LEN);
+		if (userid < HAL_MAX_UL_MU_USERS) {
+			info[0] = __le32_to_cpu(mpdu_start->info0);
+			ppdu_info->userid = userid;
+			ppdu_info->userstats[userid].ampdu_id =
+				u32_get_bits(info[0], HAL_RX_MPDU_START_INFO0_PPDU_ID);
+		}
+
+		return HAL_RX_MON_STATUS_MPDU_START;
+	}
+	case HAL_RX_MSDU_START:
+		/* TODO: add msdu start parsing logic */
+		break;
+	case HAL_MON_BUF_ADDR:
+		return HAL_RX_MON_STATUS_BUF_ADDR;
+	case HAL_RX_MSDU_END:
+		ath12k_wifi7_dp_mon_parse_status_msdu_end(pmon, tlv_data);
+		return HAL_RX_MON_STATUS_MSDU_END;
+	case HAL_RX_MPDU_END:
+		return HAL_RX_MON_STATUS_MPDU_END;
+	case HAL_PHYRX_GENERIC_U_SIG:
+		ath12k_wifi7_dp_mon_hal_rx_parse_u_sig_hdr(tlv_data, ppdu_info);
+		break;
+	case HAL_PHYRX_GENERIC_EHT_SIG:
+		/* Handle the case where aggregation is in progress
+		 * or the current TLV is one of the TLVs which should be
+		 * aggregated
+		 */
+		if (!ppdu_info->tlv_aggr.in_progress) {
+			ppdu_info->tlv_aggr.in_progress = true;
+			ppdu_info->tlv_aggr.tlv_tag = tlv_tag;
+			ppdu_info->tlv_aggr.cur_len = 0;
+		}
+
+		ppdu_info->is_eht = true;
+
+		ath12k_wifi7_dp_mon_hal_aggr_tlv(ppdu_info, tlv_len, tlv_data);
+		break;
+	case HAL_DUMMY:
+		return HAL_RX_MON_STATUS_BUF_DONE;
+	case HAL_RX_PPDU_END_STATUS_DONE:
+	case 0:
+		return HAL_RX_MON_STATUS_PPDU_DONE;
+	default:
+		break;
+	}
+
+	return HAL_RX_MON_STATUS_PPDU_NOT_DONE;
+}
+
+static int
+ath12k_wifi7_dp_mon_parse_rx_dest_tlv(struct ath12k_pdev_dp *dp_pdev,
+				      struct ath12k_mon_data *pmon,
+				      enum hal_rx_mon_status hal_status,
+				      const void *tlv_data)
+{
+	switch (hal_status) {
+	case HAL_RX_MON_STATUS_MPDU_START:
+		if (WARN_ON_ONCE(pmon->mon_mpdu))
+			break;
+
+		pmon->mon_mpdu = kzalloc(sizeof(*pmon->mon_mpdu), GFP_ATOMIC);
+		if (!pmon->mon_mpdu)
+			return -ENOMEM;
+		break;
+	case HAL_RX_MON_STATUS_BUF_ADDR:
+		return ath12k_dp_mon_parse_status_buf(dp_pdev, pmon, tlv_data);
+	case HAL_RX_MON_STATUS_MPDU_END:
+		/* If no MSDU then free empty MPDU */
+		if (pmon->mon_mpdu->tail) {
+			pmon->mon_mpdu->tail->next = NULL;
+			list_add_tail(&pmon->mon_mpdu->list, &pmon->dp_rx_mon_mpdu_list);
+		} else {
+			kfree(pmon->mon_mpdu);
+		}
+		pmon->mon_mpdu = NULL;
+		break;
+	case HAL_RX_MON_STATUS_MSDU_END:
+		pmon->mon_mpdu->decap_format = pmon->decap_format;
+		pmon->mon_mpdu->err_bitmap = pmon->err_bitmap;
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static struct dp_mon_tx_ppdu_info *
+ath12k_wifi7_dp_mon_tx_get_ppdu_info(struct ath12k_mon_data *pmon,
+				     unsigned int ppdu_id,
+				     enum dp_mon_tx_ppdu_info_type type)
+{
+	struct dp_mon_tx_ppdu_info *tx_ppdu_info;
+
+	if (type == DP_MON_TX_PROT_PPDU_INFO) {
+		tx_ppdu_info = pmon->tx_prot_ppdu_info;
+
+		if (tx_ppdu_info && !tx_ppdu_info->is_used)
+			return tx_ppdu_info;
+		kfree(tx_ppdu_info);
+	} else {
+		tx_ppdu_info = pmon->tx_data_ppdu_info;
+
+		if (tx_ppdu_info && !tx_ppdu_info->is_used)
+			return tx_ppdu_info;
+		kfree(tx_ppdu_info);
+	}
+
+	/* allocate new tx_ppdu_info */
+	tx_ppdu_info = kzalloc(sizeof(*tx_ppdu_info), GFP_ATOMIC);
+	if (!tx_ppdu_info)
+		return NULL;
+
+	tx_ppdu_info->is_used = 0;
+	tx_ppdu_info->ppdu_id = ppdu_id;
+
+	if (type == DP_MON_TX_PROT_PPDU_INFO)
+		pmon->tx_prot_ppdu_info = tx_ppdu_info;
+	else
+		pmon->tx_data_ppdu_info = tx_ppdu_info;
+
+	return tx_ppdu_info;
+}
+
+static struct dp_mon_tx_ppdu_info *
+ath12k_wifi7_dp_mon_hal_tx_ppdu_info(struct ath12k_mon_data *pmon,
+				     u16 tlv_tag)
+{
+	switch (tlv_tag) {
+	case HAL_TX_FES_SETUP:
+	case HAL_TX_FLUSH:
+	case HAL_PCU_PPDU_SETUP_INIT:
+	case HAL_TX_PEER_ENTRY:
+	case HAL_TX_QUEUE_EXTENSION:
+	case HAL_TX_MPDU_START:
+	case HAL_TX_MSDU_START:
+	case HAL_TX_DATA:
+	case HAL_MON_BUF_ADDR:
+	case HAL_TX_MPDU_END:
+	case HAL_TX_LAST_MPDU_FETCHED:
+	case HAL_TX_LAST_MPDU_END:
+	case HAL_COEX_TX_REQ:
+	case HAL_TX_RAW_OR_NATIVE_FRAME_SETUP:
+	case HAL_SCH_CRITICAL_TLV_REFERENCE:
+	case HAL_TX_FES_SETUP_COMPLETE:
+	case HAL_TQM_MPDU_GLOBAL_START:
+	case HAL_SCHEDULER_END:
+	case HAL_TX_FES_STATUS_USER_PPDU:
+		break;
+	case HAL_TX_FES_STATUS_PROT: {
+		if (!pmon->tx_prot_ppdu_info->is_used)
+			pmon->tx_prot_ppdu_info->is_used = true;
+
+		return pmon->tx_prot_ppdu_info;
+	}
+	}
+
+	if (!pmon->tx_data_ppdu_info->is_used)
+		pmon->tx_data_ppdu_info->is_used = true;
+
+	return pmon->tx_data_ppdu_info;
+}
+
+#define MAX_MONITOR_HEADER 512
+#define MAX_DUMMY_FRM_BODY 128
+
+static struct
+sk_buff *ath12k_wifi7_dp_mon_tx_alloc_skb(void)
+{
+	struct sk_buff *skb;
+
+	skb = dev_alloc_skb(MAX_MONITOR_HEADER + MAX_DUMMY_FRM_BODY);
+	if (!skb)
+		return NULL;
+
+	skb_reserve(skb, MAX_MONITOR_HEADER);
+
+	if (!IS_ALIGNED((unsigned long)skb->data, 4))
+		skb_pull(skb, PTR_ALIGN(skb->data, 4) - skb->data);
+
+	return skb;
+}
+
+static int
+ath12k_wifi7_dp_mon_tx_gen_cts2self_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info)
+{
+	struct sk_buff *skb;
+	struct ieee80211_cts *cts;
+
+	skb = ath12k_wifi7_dp_mon_tx_alloc_skb();
+	if (!skb)
+		return -ENOMEM;
+
+	cts = (struct ieee80211_cts *)skb->data;
+	memset(cts, 0, MAX_DUMMY_FRM_BODY);
+	cts->frame_control =
+		cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_CTS);
+	cts->duration = cpu_to_le16(tx_ppdu_info->rx_status.rx_duration);
+	memcpy(cts->ra, tx_ppdu_info->rx_status.addr1, sizeof(cts->ra));
+
+	skb_put(skb, sizeof(*cts));
+	tx_ppdu_info->tx_mon_mpdu->head = skb;
+	tx_ppdu_info->tx_mon_mpdu->tail = NULL;
+	list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list,
+		      &tx_ppdu_info->dp_tx_mon_mpdu_list);
+
+	return 0;
+}
+
+static int
+ath12k_wifi7_dp_mon_tx_gen_rts_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info)
+{
+	struct sk_buff *skb;
+	struct ieee80211_rts *rts;
+
+	skb = ath12k_wifi7_dp_mon_tx_alloc_skb();
+	if (!skb)
+		return -ENOMEM;
+
+	rts = (struct ieee80211_rts *)skb->data;
+	memset(rts, 0, MAX_DUMMY_FRM_BODY);
+	rts->frame_control =
+		cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_RTS);
+	rts->duration = cpu_to_le16(tx_ppdu_info->rx_status.rx_duration);
+	memcpy(rts->ra, tx_ppdu_info->rx_status.addr1, sizeof(rts->ra));
+	memcpy(rts->ta, tx_ppdu_info->rx_status.addr2, sizeof(rts->ta));
+
+	skb_put(skb, sizeof(*rts));
+	tx_ppdu_info->tx_mon_mpdu->head = skb;
+	tx_ppdu_info->tx_mon_mpdu->tail = NULL;
+	list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list,
+		      &tx_ppdu_info->dp_tx_mon_mpdu_list);
+
+	return 0;
+}
+
+static int
+ath12k_wifi7_dp_mon_tx_gen_3addr_qos_null_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info)
+{
+	struct sk_buff *skb;
+	struct ieee80211_qos_hdr *qhdr;
+
+	skb = ath12k_wifi7_dp_mon_tx_alloc_skb();
+	if (!skb)
+		return -ENOMEM;
+
+	qhdr = (struct ieee80211_qos_hdr *)skb->data;
+	memset(qhdr, 0, MAX_DUMMY_FRM_BODY);
+	qhdr->frame_control =
+		cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_NULLFUNC);
+	qhdr->duration_id = cpu_to_le16(tx_ppdu_info->rx_status.rx_duration);
+	memcpy(qhdr->addr1, tx_ppdu_info->rx_status.addr1, ETH_ALEN);
+	memcpy(qhdr->addr2, tx_ppdu_info->rx_status.addr2, ETH_ALEN);
+	memcpy(qhdr->addr3, tx_ppdu_info->rx_status.addr3, ETH_ALEN);
+
+	skb_put(skb, sizeof(*qhdr));
+	tx_ppdu_info->tx_mon_mpdu->head = skb;
+	tx_ppdu_info->tx_mon_mpdu->tail = NULL;
+	list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list,
+		      &tx_ppdu_info->dp_tx_mon_mpdu_list);
+
+	return 0;
+}
+
+static int
+ath12k_wifi7_dp_mon_tx_gen_4addr_qos_null_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info)
+{
+	struct sk_buff *skb;
+	struct dp_mon_qosframe_addr4 *qhdr;
+
+	skb = ath12k_wifi7_dp_mon_tx_alloc_skb();
+	if (!skb)
+		return -ENOMEM;
+
+	qhdr = (struct dp_mon_qosframe_addr4 *)skb->data;
+	memset(qhdr, 0, MAX_DUMMY_FRM_BODY);
+	qhdr->frame_control =
+		cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_NULLFUNC);
+	qhdr->duration = cpu_to_le16(tx_ppdu_info->rx_status.rx_duration);
+	memcpy(qhdr->addr1, tx_ppdu_info->rx_status.addr1, ETH_ALEN);
+	memcpy(qhdr->addr2, tx_ppdu_info->rx_status.addr2, ETH_ALEN);
+	memcpy(qhdr->addr3, tx_ppdu_info->rx_status.addr3, ETH_ALEN);
+	memcpy(qhdr->addr4, tx_ppdu_info->rx_status.addr4, ETH_ALEN);
+
+	skb_put(skb, sizeof(*qhdr));
+	tx_ppdu_info->tx_mon_mpdu->head = skb;
+	tx_ppdu_info->tx_mon_mpdu->tail = NULL;
+	list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list,
+		      &tx_ppdu_info->dp_tx_mon_mpdu_list);
+
+	return 0;
+}
+
+static int
+ath12k_wifi7_dp_mon_tx_gen_ack_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info)
+{
+	struct sk_buff *skb;
+	struct dp_mon_frame_min_one *fbmhdr;
+
+	skb = ath12k_wifi7_dp_mon_tx_alloc_skb();
+	if (!skb)
+		return -ENOMEM;
+
+	fbmhdr = (struct dp_mon_frame_min_one *)skb->data;
+	memset(fbmhdr, 0, MAX_DUMMY_FRM_BODY);
+	fbmhdr->frame_control =
+		cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_CFACK);
+	memcpy(fbmhdr->addr1, tx_ppdu_info->rx_status.addr1, ETH_ALEN);
+
+	/* set duration zero for ack frame */
+	fbmhdr->duration = 0;
+
+	skb_put(skb, sizeof(*fbmhdr));
+	tx_ppdu_info->tx_mon_mpdu->head = skb;
+	tx_ppdu_info->tx_mon_mpdu->tail = NULL;
+	list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list,
+		      &tx_ppdu_info->dp_tx_mon_mpdu_list);
+
+	return 0;
+}
+
+static int
+ath12k_wifi7_dp_mon_tx_gen_prot_frame(struct dp_mon_tx_ppdu_info *tx_ppdu_info)
+{
+	int ret = 0;
+
+	switch (tx_ppdu_info->rx_status.medium_prot_type) {
+	case DP_MON_TX_MEDIUM_RTS_LEGACY:
+	case DP_MON_TX_MEDIUM_RTS_11AC_STATIC_BW:
+	case DP_MON_TX_MEDIUM_RTS_11AC_DYNAMIC_BW:
+		ret = ath12k_wifi7_dp_mon_tx_gen_rts_frame(tx_ppdu_info);
+		break;
+	case DP_MON_TX_MEDIUM_CTS2SELF:
+		ret = ath12k_wifi7_dp_mon_tx_gen_cts2self_frame(tx_ppdu_info);
+		break;
+	case DP_MON_TX_MEDIUM_QOS_NULL_NO_ACK_3ADDR:
+		ret = ath12k_wifi7_dp_mon_tx_gen_3addr_qos_null_frame(tx_ppdu_info);
+		break;
+	case DP_MON_TX_MEDIUM_QOS_NULL_NO_ACK_4ADDR:
+		ret = ath12k_wifi7_dp_mon_tx_gen_4addr_qos_null_frame(tx_ppdu_info);
+		break;
+	}
+
+	return ret;
+}
+
+static enum dp_mon_tx_tlv_status
+ath12k_wifi7_dp_mon_tx_parse_status_tlv(struct ath12k_base *ab,
+					struct ath12k_mon_data *pmon,
+					u16 tlv_tag, const void *tlv_data,
+					u32 userid)
+{
+	struct dp_mon_tx_ppdu_info *tx_ppdu_info;
+	enum dp_mon_tx_tlv_status status = DP_MON_TX_STATUS_PPDU_NOT_DONE;
+	u32 info[7];
+
+	tx_ppdu_info = ath12k_wifi7_dp_mon_hal_tx_ppdu_info(pmon, tlv_tag);
+
+	switch (tlv_tag) {
+	case HAL_TX_FES_SETUP: {
+		const struct hal_tx_fes_setup *tx_fes_setup = tlv_data;
+
+		info[0] = __le32_to_cpu(tx_fes_setup->info0);
+		tx_ppdu_info->ppdu_id = __le32_to_cpu(tx_fes_setup->schedule_id);
+		tx_ppdu_info->num_users =
+			u32_get_bits(info[0], HAL_TX_FES_SETUP_INFO0_NUM_OF_USERS);
+		status = DP_MON_TX_FES_SETUP;
+		break;
+	}
+
+	case HAL_TX_FES_STATUS_END: {
+		const struct hal_tx_fes_status_end *tx_fes_status_end = tlv_data;
+		u32 tst_15_0, tst_31_16;
+
+		info[0] = __le32_to_cpu(tx_fes_status_end->info0);
+		tst_15_0 =
+			u32_get_bits(info[0],
+				     HAL_TX_FES_STATUS_END_INFO0_START_TIMESTAMP_15_0);
+		tst_31_16 =
+			u32_get_bits(info[0],
+				     HAL_TX_FES_STATUS_END_INFO0_START_TIMESTAMP_31_16);
+
+		tx_ppdu_info->rx_status.ppdu_ts = (tst_15_0 | (tst_31_16 << 16));
+		status = DP_MON_TX_FES_STATUS_END;
+		break;
+	}
+
+	case HAL_RX_RESPONSE_REQUIRED_INFO: {
+		const struct hal_rx_resp_req_info *rx_resp_req_info = tlv_data;
+		u32 addr_32;
+		u16 addr_16;
+
+		info[0] = __le32_to_cpu(rx_resp_req_info->info0);
+		info[1] = __le32_to_cpu(rx_resp_req_info->info1);
+		info[2] = __le32_to_cpu(rx_resp_req_info->info2);
+		info[3] = __le32_to_cpu(rx_resp_req_info->info3);
+		info[4] = __le32_to_cpu(rx_resp_req_info->info4);
+		info[5] = __le32_to_cpu(rx_resp_req_info->info5);
+
+		tx_ppdu_info->rx_status.ppdu_id =
+			u32_get_bits(info[0], HAL_RX_RESP_REQ_INFO0_PPDU_ID);
+		tx_ppdu_info->rx_status.reception_type =
+			u32_get_bits(info[0], HAL_RX_RESP_REQ_INFO0_RECEPTION_TYPE);
+		tx_ppdu_info->rx_status.rx_duration =
+			u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_DURATION);
+		tx_ppdu_info->rx_status.mcs =
+			u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_RATE_MCS);
+		tx_ppdu_info->rx_status.sgi =
+			u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_SGI);
+		tx_ppdu_info->rx_status.is_stbc =
+			u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_STBC);
+		tx_ppdu_info->rx_status.ldpc =
+			u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_LDPC);
+		tx_ppdu_info->rx_status.is_ampdu =
+			u32_get_bits(info[1], HAL_RX_RESP_REQ_INFO1_IS_AMPDU);
+		tx_ppdu_info->rx_status.num_users =
+			u32_get_bits(info[2], HAL_RX_RESP_REQ_INFO2_NUM_USER);
+
+		addr_32 = u32_get_bits(info[3], HAL_RX_RESP_REQ_INFO3_ADDR1_31_0);
+		addr_16 = u32_get_bits(info[3], HAL_RX_RESP_REQ_INFO4_ADDR1_47_32);
+		ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr1);
+
+		addr_16 = u32_get_bits(info[4], HAL_RX_RESP_REQ_INFO4_ADDR1_15_0);
+		addr_32 = u32_get_bits(info[5], HAL_RX_RESP_REQ_INFO5_ADDR1_47_16);
+		ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr2);
+
+		if (tx_ppdu_info->rx_status.reception_type == 0)
+			ath12k_wifi7_dp_mon_tx_gen_cts2self_frame(tx_ppdu_info);
+		status = DP_MON_RX_RESPONSE_REQUIRED_INFO;
+		break;
+	}
+
+	case HAL_PCU_PPDU_SETUP_INIT: {
+		const struct hal_tx_pcu_ppdu_setup_init *ppdu_setup = tlv_data;
+		u32 addr_32;
+		u16 addr_16;
+
+		info[0] = __le32_to_cpu(ppdu_setup->info0);
+		info[1] = __le32_to_cpu(ppdu_setup->info1);
+		info[2] = __le32_to_cpu(ppdu_setup->info2);
+		info[3] = __le32_to_cpu(ppdu_setup->info3);
+		info[4] = __le32_to_cpu(ppdu_setup->info4);
+		info[5] = __le32_to_cpu(ppdu_setup->info5);
+		info[6] = __le32_to_cpu(ppdu_setup->info6);
+
+		/* protection frame address 1 */
+		addr_32 = u32_get_bits(info[1],
+				       HAL_TX_PPDU_SETUP_INFO1_PROT_FRAME_ADDR1_31_0);
+		addr_16 = u32_get_bits(info[2],
+				       HAL_TX_PPDU_SETUP_INFO2_PROT_FRAME_ADDR1_47_32);
+		ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr1);
+
+		/* protection frame address 2 */
+		addr_16 = u32_get_bits(info[2],
+				       HAL_TX_PPDU_SETUP_INFO2_PROT_FRAME_ADDR2_15_0);
+		addr_32 = u32_get_bits(info[3],
+				       HAL_TX_PPDU_SETUP_INFO3_PROT_FRAME_ADDR2_47_16);
+		ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr2);
+
+		/* protection frame address 3 */
+		addr_32 = u32_get_bits(info[4],
+				       HAL_TX_PPDU_SETUP_INFO4_PROT_FRAME_ADDR3_31_0);
+		addr_16 = u32_get_bits(info[5],
+				       HAL_TX_PPDU_SETUP_INFO5_PROT_FRAME_ADDR3_47_32);
+		ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr3);
+
+		/* protection frame address 4 */
+		addr_16 = u32_get_bits(info[5],
+				       HAL_TX_PPDU_SETUP_INFO5_PROT_FRAME_ADDR4_15_0);
+		addr_32 = u32_get_bits(info[6],
+				       HAL_TX_PPDU_SETUP_INFO6_PROT_FRAME_ADDR4_47_16);
+		ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr4);
+
+		status = u32_get_bits(info[0],
+				      HAL_TX_PPDU_SETUP_INFO0_MEDIUM_PROT_TYPE);
+		break;
+	}
+
+	case HAL_TX_QUEUE_EXTENSION: {
+		const struct hal_tx_queue_exten *tx_q_exten = tlv_data;
+
+		info[0] = __le32_to_cpu(tx_q_exten->info0);
+
+		tx_ppdu_info->rx_status.frame_control =
+			u32_get_bits(info[0],
+				     HAL_TX_Q_EXT_INFO0_FRAME_CTRL);
+		tx_ppdu_info->rx_status.fc_valid = true;
+		break;
+	}
+
+	case HAL_TX_FES_STATUS_START: {
+		const struct hal_tx_fes_status_start *tx_fes_start = tlv_data;
+
+		info[0] = __le32_to_cpu(tx_fes_start->info0);
+
+		tx_ppdu_info->rx_status.medium_prot_type =
+			u32_get_bits(info[0],
+				     HAL_TX_FES_STATUS_START_INFO0_MEDIUM_PROT_TYPE);
+		break;
+	}
+
+	case HAL_TX_FES_STATUS_PROT: {
+		const struct hal_tx_fes_status_prot *tx_fes_status = tlv_data;
+		u32 start_timestamp;
+		u32 end_timestamp;
+
+		info[0] = __le32_to_cpu(tx_fes_status->info0);
+		info[1] = __le32_to_cpu(tx_fes_status->info1);
+
+		start_timestamp =
+			u32_get_bits(info[0],
+				     HAL_TX_FES_STAT_PROT_INFO0_STRT_FRM_TS_15_0);
+		start_timestamp |=
+			u32_get_bits(info[0],
+				     HAL_TX_FES_STAT_PROT_INFO0_STRT_FRM_TS_31_16) << 15;
+		end_timestamp =
+			u32_get_bits(info[1],
+				     HAL_TX_FES_STAT_PROT_INFO1_END_FRM_TS_15_0);
+		end_timestamp |=
+			u32_get_bits(info[1],
+				     HAL_TX_FES_STAT_PROT_INFO1_END_FRM_TS_31_16) << 15;
+		tx_ppdu_info->rx_status.rx_duration = end_timestamp - start_timestamp;
+
+		ath12k_wifi7_dp_mon_tx_gen_prot_frame(tx_ppdu_info);
+		break;
+	}
+
+	case HAL_TX_FES_STATUS_START_PPDU:
+	case HAL_TX_FES_STATUS_START_PROT: {
+		const struct hal_tx_fes_status_start_prot *tx_fes_stat_start = tlv_data;
+		u64 ppdu_ts;
+
+		info[0] = __le32_to_cpu(tx_fes_stat_start->info0);
+
+		tx_ppdu_info->rx_status.ppdu_ts =
+			u32_get_bits(info[0],
+				     HAL_TX_FES_STAT_STRT_INFO0_PROT_TS_LOWER_32);
+		ppdu_ts = (u32_get_bits(info[1],
+					HAL_TX_FES_STAT_STRT_INFO1_PROT_TS_UPPER_32));
+		tx_ppdu_info->rx_status.ppdu_ts |= ppdu_ts << 32;
+		break;
+	}
+
+	case HAL_TX_FES_STATUS_USER_PPDU: {
+		const struct hal_tx_fes_status_user_ppdu *tx_fes_usr_ppdu = tlv_data;
+
+		info[0] = __le32_to_cpu(tx_fes_usr_ppdu->info0);
+
+		tx_ppdu_info->rx_status.rx_duration =
+			u32_get_bits(info[0],
+				     HAL_TX_FES_STAT_USR_PPDU_INFO0_DURATION);
+		break;
+	}
+
+	case HAL_MACTX_HE_SIG_A_SU:
+		ath12k_wifi7_dp_mon_parse_he_sig_su(tlv_data,
+						    &tx_ppdu_info->rx_status);
+		break;
+
+	case HAL_MACTX_HE_SIG_A_MU_DL:
+		ath12k_wifi7_dp_mon_parse_he_sig_mu(tlv_data, &tx_ppdu_info->rx_status);
+		break;
+
+	case HAL_MACTX_HE_SIG_B1_MU:
+		ath12k_wifi7_dp_mon_parse_he_sig_b1_mu(tlv_data,
+						       &tx_ppdu_info->rx_status);
+		break;
+
+	case HAL_MACTX_HE_SIG_B2_MU:
+		ath12k_wifi7_dp_mon_parse_he_sig_b2_mu(tlv_data,
+						       &tx_ppdu_info->rx_status);
+		break;
+
+	case HAL_MACTX_HE_SIG_B2_OFDMA:
+		ath12k_wifi7_dp_mon_parse_he_sig_b2_ofdma(tlv_data,
+							  &tx_ppdu_info->rx_status);
+		break;
+
+	case HAL_MACTX_VHT_SIG_A:
+		ath12k_wifi7_dp_mon_parse_vht_sig_a(tlv_data, &tx_ppdu_info->rx_status);
+		break;
+
+	case HAL_MACTX_L_SIG_A:
+		ath12k_wifi7_dp_mon_parse_l_sig_a(tlv_data, &tx_ppdu_info->rx_status);
+		break;
+
+	case HAL_MACTX_L_SIG_B:
+		ath12k_wifi7_dp_mon_parse_l_sig_b(tlv_data, &tx_ppdu_info->rx_status);
+		break;
+
+	case HAL_RX_FRAME_BITMAP_ACK: {
+		const struct hal_rx_frame_bitmap_ack *fbm_ack = tlv_data;
+		u32 addr_32;
+		u16 addr_16;
+
+		info[0] = __le32_to_cpu(fbm_ack->info0);
+		info[1] = __le32_to_cpu(fbm_ack->info1);
+
+		addr_32 = u32_get_bits(info[0],
+				       HAL_RX_FBM_ACK_INFO0_ADDR1_31_0);
+		addr_16 = u32_get_bits(info[1],
+				       HAL_RX_FBM_ACK_INFO1_ADDR1_47_32);
+		ath12k_dp_get_mac_addr(addr_32, addr_16, tx_ppdu_info->rx_status.addr1);
+
+		ath12k_wifi7_dp_mon_tx_gen_ack_frame(tx_ppdu_info);
+		break;
+	}
+
+	case HAL_MACTX_PHY_DESC: {
+		const struct hal_tx_phy_desc *tx_phy_desc = tlv_data;
+
+		info[0] = __le32_to_cpu(tx_phy_desc->info0);
+		info[1] = __le32_to_cpu(tx_phy_desc->info1);
+		info[2] = __le32_to_cpu(tx_phy_desc->info2);
+		info[3] = __le32_to_cpu(tx_phy_desc->info3);
+
+		tx_ppdu_info->rx_status.beamformed =
+			u32_get_bits(info[0],
+				     HAL_TX_PHY_DESC_INFO0_BF_TYPE);
+		tx_ppdu_info->rx_status.preamble_type =
+			u32_get_bits(info[0],
+				     HAL_TX_PHY_DESC_INFO0_PREAMBLE_11B);
+		tx_ppdu_info->rx_status.mcs =
+			u32_get_bits(info[1],
+				     HAL_TX_PHY_DESC_INFO1_MCS);
+		tx_ppdu_info->rx_status.ltf_size =
+			u32_get_bits(info[3],
+				     HAL_TX_PHY_DESC_INFO3_LTF_SIZE);
+		tx_ppdu_info->rx_status.nss =
+			u32_get_bits(info[2],
+				     HAL_TX_PHY_DESC_INFO2_NSS);
+		tx_ppdu_info->rx_status.chan_num =
+			u32_get_bits(info[3],
+				     HAL_TX_PHY_DESC_INFO3_ACTIVE_CHANNEL);
+		tx_ppdu_info->rx_status.bw =
+			u32_get_bits(info[0],
+				     HAL_TX_PHY_DESC_INFO0_BANDWIDTH);
+		break;
+	}
+
+	case HAL_TX_MPDU_START: {
+		struct dp_mon_mpdu *mon_mpdu = tx_ppdu_info->tx_mon_mpdu;
+
+		mon_mpdu = kzalloc(sizeof(*mon_mpdu), GFP_ATOMIC);
+		if (!mon_mpdu)
+			return DP_MON_TX_STATUS_PPDU_NOT_DONE;
+		status = DP_MON_TX_MPDU_START;
+		break;
+	}
+
+	case HAL_TX_MPDU_END:
+		list_add_tail(&tx_ppdu_info->tx_mon_mpdu->list,
+			      &tx_ppdu_info->dp_tx_mon_mpdu_list);
+		break;
+	}
+
+	return status;
+}
+
+static enum dp_mon_tx_tlv_status
+ath12k_wifi7_dp_mon_tx_status_get_num_user(u16 tlv_tag,
+					   struct hal_tlv_hdr *tx_tlv,
+					   u8 *num_users)
+{
+	u32 tlv_status = DP_MON_TX_STATUS_PPDU_NOT_DONE;
+	u32 info0;
+
+	switch (tlv_tag) {
+	case HAL_TX_FES_SETUP: {
+		struct hal_tx_fes_setup *tx_fes_setup =
+				(struct hal_tx_fes_setup *)tx_tlv;
+
+		info0 = __le32_to_cpu(tx_fes_setup->info0);
+
+		*num_users = u32_get_bits(info0, HAL_TX_FES_SETUP_INFO0_NUM_OF_USERS);
+		tlv_status = DP_MON_TX_FES_SETUP;
+		break;
+	}
+
+	case HAL_RX_RESPONSE_REQUIRED_INFO: {
+		/* TODO: need to update *num_users */
+		tlv_status = DP_MON_RX_RESPONSE_REQUIRED_INFO;
+		break;
+	}
+	}
+
+	return tlv_status;
+}
+
+static int
+ath12k_wifi7_dp_mon_rx_deliver(struct ath12k_pdev_dp *dp_pdev,
+			       struct dp_mon_mpdu *mon_mpdu,
+			       struct hal_rx_mon_ppdu_info *ppduinfo,
+			       struct napi_struct *napi)
+{
+	struct sk_buff *mon_skb, *skb_next, *header;
+	struct ieee80211_rx_status *rxs = &dp_pdev->rx_status;
+	u8 decap = DP_RX_DECAP_TYPE_RAW;
+
+	mon_skb = ath12k_dp_mon_rx_merg_msdus(dp_pdev, mon_mpdu, ppduinfo, rxs);
+	if (!mon_skb)
+		goto mon_deliver_fail;
+
+	header = mon_skb;
+	rxs->flag = 0;
+
+	if (mon_mpdu->err_bitmap & HAL_RX_MPDU_ERR_FCS)
+		rxs->flag = RX_FLAG_FAILED_FCS_CRC;
+
+	do {
+		skb_next = mon_skb->next;
+		if (!skb_next)
+			rxs->flag &= ~RX_FLAG_AMSDU_MORE;
+		else
+			rxs->flag |= RX_FLAG_AMSDU_MORE;
+
+		if (mon_skb == header) {
+			header = NULL;
+			rxs->flag &= ~RX_FLAG_ALLOW_SAME_PN;
+		} else {
+			rxs->flag |= RX_FLAG_ALLOW_SAME_PN;
+		}
+		rxs->flag |= RX_FLAG_ONLY_MONITOR;
+
+		if (!(rxs->flag & RX_FLAG_ONLY_MONITOR))
+			decap = mon_mpdu->decap_format;
+
+		ath12k_dp_mon_update_radiotap(dp_pdev, ppduinfo, mon_skb, rxs);
+		ath12k_dp_mon_rx_deliver_msdu(dp_pdev, napi, mon_skb, ppduinfo,
+					      rxs, decap);
+		mon_skb = skb_next;
+	} while (mon_skb);
+	rxs->flag = 0;
+
+	return 0;
+
+mon_deliver_fail:
+	mon_skb = mon_mpdu->head;
+	while (mon_skb) {
+		skb_next = mon_skb->next;
+		dev_kfree_skb_any(mon_skb);
+		mon_skb = skb_next;
+	}
+	return -EINVAL;
+}
+
+static void
+ath12k_wifi7_dp_mon_tx_process_ppdu_info(struct ath12k_pdev_dp *dp_pdev,
+					 struct napi_struct *napi,
+					 struct dp_mon_tx_ppdu_info *tx_ppdu_info)
+{
+	struct dp_mon_mpdu *tmp, *mon_mpdu;
+
+	list_for_each_entry_safe(mon_mpdu, tmp,
+				 &tx_ppdu_info->dp_tx_mon_mpdu_list, list) {
+		list_del(&mon_mpdu->list);
+
+		if (mon_mpdu->head)
+			ath12k_wifi7_dp_mon_rx_deliver(dp_pdev, mon_mpdu,
+						       &tx_ppdu_info->rx_status, napi);
+
+		kfree(mon_mpdu);
+	}
+}
+
+enum hal_rx_mon_status
+ath12k_wifi7_dp_mon_tx_parse_mon_status(struct ath12k_pdev_dp *dp_pdev,
+					struct ath12k_mon_data *pmon,
+					struct sk_buff *skb,
+					struct napi_struct *napi,
+					u32 ppdu_id)
+{
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_base *ab = dp->ab;
+	struct dp_mon_tx_ppdu_info *tx_prot_ppdu_info, *tx_data_ppdu_info;
+	struct hal_tlv_hdr *tlv;
+	u8 *ptr = skb->data;
+	u16 tlv_tag;
+	u16 tlv_len;
+	u32 tlv_userid = 0;
+	u8 num_user;
+	u32 tlv_status = DP_MON_TX_STATUS_PPDU_NOT_DONE;
+
+	tx_prot_ppdu_info =
+		ath12k_wifi7_dp_mon_tx_get_ppdu_info(pmon, ppdu_id,
+						     DP_MON_TX_PROT_PPDU_INFO);
+	if (!tx_prot_ppdu_info)
+		return -ENOMEM;
+
+	tlv = (struct hal_tlv_hdr *)ptr;
+	tlv_tag = le32_get_bits(tlv->tl, HAL_TLV_HDR_TAG);
+
+	tlv_status = ath12k_wifi7_dp_mon_tx_status_get_num_user(tlv_tag, tlv,
+								&num_user);
+	if (tlv_status == DP_MON_TX_STATUS_PPDU_NOT_DONE || !num_user)
+		return -EINVAL;
+
+	tx_data_ppdu_info =
+		ath12k_wifi7_dp_mon_tx_get_ppdu_info(pmon, ppdu_id,
+						     DP_MON_TX_DATA_PPDU_INFO);
+	if (!tx_data_ppdu_info)
+		return -ENOMEM;
+
+	do {
+		tlv = (struct hal_tlv_hdr *)ptr;
+		tlv_tag = le32_get_bits(tlv->tl, HAL_TLV_HDR_TAG);
+		tlv_len = le32_get_bits(tlv->tl, HAL_TLV_HDR_LEN);
+		tlv_userid = le32_get_bits(tlv->tl, HAL_TLV_USR_ID);
+
+		tlv_status = ath12k_wifi7_dp_mon_tx_parse_status_tlv(ab, pmon,
+								     tlv_tag, ptr,
+								     tlv_userid);
+		ptr += tlv_len;
+		ptr = PTR_ALIGN(ptr, HAL_TLV_ALIGN);
+		if ((ptr - skb->data) >= DP_TX_MONITOR_BUF_SIZE)
+			break;
+	} while (tlv_status != DP_MON_TX_FES_STATUS_END);
+
+	ath12k_wifi7_dp_mon_tx_process_ppdu_info(dp_pdev, napi, tx_data_ppdu_info);
+	ath12k_wifi7_dp_mon_tx_process_ppdu_info(dp_pdev, napi, tx_prot_ppdu_info);
+
+	return tlv_status;
+}
+
+static void
+ath12k_wifi7_dp_mon_next_link_desc_get(struct ath12k_base *ab,
+				       struct hal_rx_msdu_link *msdu_link,
+				       dma_addr_t *paddr, u32 *sw_cookie, u8 *rbm,
+				       struct ath12k_buffer_addr **pp_buf_addr_info)
+{
+	struct ath12k_buffer_addr *buf_addr_info;
+
+	buf_addr_info = &msdu_link->buf_addr_info;
+
+	ath12k_wifi7_hal_rx_buf_addr_info_get(buf_addr_info, paddr, sw_cookie, rbm);
+
+	*pp_buf_addr_info = buf_addr_info;
+}
+
+static u32
+ath12k_wifi7_dp_rx_mon_mpdu_pop(struct ath12k *ar, int mac_id,
+				void *ring_entry, struct sk_buff **head_msdu,
+				struct sk_buff **tail_msdu,
+				struct list_head *used_list,
+				u32 *npackets, u32 *ppdu_id)
+{
+	struct ath12k_mon_data *pmon = (struct ath12k_mon_data *)&ar->dp.mon_data;
+	struct ath12k_base *ab = ar->ab;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct ath12k_buffer_addr *p_buf_addr_info, *p_last_buf_addr_info;
+	u32 msdu_ppdu_id = 0, msdu_cnt = 0, total_len = 0, frag_len = 0;
+	u32 rx_buf_size, rx_pkt_offset, sw_cookie;
+	bool is_frag, is_first_msdu, drop_mpdu = false;
+	struct hal_reo_entrance_ring *ent_desc =
+		(struct hal_reo_entrance_ring *)ring_entry;
+	u32 rx_bufs_used = 0, i = 0, desc_bank = 0;
+	struct hal_rx_desc *rx_desc, *tail_rx_desc;
+	struct hal_rx_msdu_link *msdu_link_desc;
+	struct sk_buff *msdu = NULL, *last = NULL;
+	struct ath12k_rx_desc_info *desc_info;
+	struct ath12k_buffer_addr buf_info;
+	struct hal_rx_msdu_list msdu_list;
+	struct ath12k_skb_rxcb *rxcb;
+	u16 num_msdus = 0;
+	dma_addr_t paddr;
+	u8 rbm;
+
+	ath12k_wifi7_hal_rx_reo_ent_buf_paddr_get(ring_entry, &paddr,
+						  &sw_cookie,
+						  &p_last_buf_addr_info,
+						  &rbm,
+						  &msdu_cnt);
+
+	spin_lock_bh(&pmon->mon_lock);
+
+	if (le32_get_bits(ent_desc->info1,
+			  HAL_REO_ENTR_RING_INFO1_RXDMA_PUSH_REASON) ==
+			  HAL_REO_DEST_RING_PUSH_REASON_ERR_DETECTED) {
+		u8 rxdma_err = le32_get_bits(ent_desc->info1,
+					     HAL_REO_ENTR_RING_INFO1_RXDMA_ERROR_CODE);
+		if (rxdma_err == HAL_REO_ENTR_RING_RXDMA_ECODE_FLUSH_REQUEST_ERR ||
+		    rxdma_err == HAL_REO_ENTR_RING_RXDMA_ECODE_MPDU_LEN_ERR ||
+		    rxdma_err == HAL_REO_ENTR_RING_RXDMA_ECODE_OVERFLOW_ERR) {
+			drop_mpdu = true;
+			pmon->rx_mon_stats.dest_mpdu_drop++;
+		}
+	}
+
+	is_frag = false;
+	is_first_msdu = true;
+	rx_pkt_offset = sizeof(struct hal_rx_desc);
+
+	do {
+		if (pmon->mon_last_linkdesc_paddr == paddr) {
+			pmon->rx_mon_stats.dup_mon_linkdesc_cnt++;
+			spin_unlock_bh(&pmon->mon_lock);
+			return rx_bufs_used;
+		}
+
+		desc_bank = u32_get_bits(sw_cookie, DP_LINK_DESC_BANK_MASK);
+		msdu_link_desc =
+			dp->link_desc_banks[desc_bank].vaddr +
+			(paddr - dp->link_desc_banks[desc_bank].paddr);
+
+		ath12k_wifi7_hal_rx_msdu_list_get(ar, msdu_link_desc, &msdu_list,
+						  &num_msdus);
+		desc_info = ath12k_dp_get_rx_desc(ar->ab->dp,
+						  msdu_list.sw_cookie[num_msdus - 1]);
+		tail_rx_desc = (struct hal_rx_desc *)(desc_info->skb)->data;
+
+		for (i = 0; i < num_msdus; i++) {
+			u32 l2_hdr_offset;
+
+			if (pmon->mon_last_buf_cookie == msdu_list.sw_cookie[i]) {
+				ath12k_dbg(ar->ab, ATH12K_DBG_DATA,
+					   "i %d last_cookie %d is same\n",
+					   i, pmon->mon_last_buf_cookie);
+				drop_mpdu = true;
+				pmon->rx_mon_stats.dup_mon_buf_cnt++;
+				continue;
+			}
+
+			desc_info =
+				ath12k_dp_get_rx_desc(ar->ab->dp, msdu_list.sw_cookie[i]);
+			msdu = desc_info->skb;
+
+			if (!msdu) {
+				ath12k_dbg(ar->ab, ATH12K_DBG_DATA,
+					   "msdu_pop: invalid msdu (%d/%d)\n",
+					   i + 1, num_msdus);
+				goto next_msdu;
+			}
+			rxcb = ATH12K_SKB_RXCB(msdu);
+			if (rxcb->paddr != msdu_list.paddr[i]) {
+				ath12k_dbg(ar->ab, ATH12K_DBG_DATA,
+					   "i %d paddr %lx != %lx\n",
+					   i, (unsigned long)rxcb->paddr,
+					   (unsigned long)msdu_list.paddr[i]);
+				drop_mpdu = true;
+				continue;
+			}
+			if (!rxcb->unmapped) {
+				dma_unmap_single(ar->ab->dev, rxcb->paddr,
+						 msdu->len +
+						 skb_tailroom(msdu),
+						 DMA_FROM_DEVICE);
+				rxcb->unmapped = 1;
+			}
+			if (drop_mpdu) {
+				ath12k_dbg(ar->ab, ATH12K_DBG_DATA,
+					   "i %d drop msdu %p *ppdu_id %x\n",
+					   i, msdu, *ppdu_id);
+				dev_kfree_skb_any(msdu);
+				msdu = NULL;
+				goto next_msdu;
+			}
+
+			rx_desc = (struct hal_rx_desc *)msdu->data;
+			l2_hdr_offset = ath12k_dp_rx_h_l3pad(ar->ab, tail_rx_desc);
+			if (is_first_msdu) {
+				if (!ath12k_wifi7_dp_rxdesc_mpdu_valid(ar->ab,
+								       rx_desc)) {
+					drop_mpdu = true;
+					dev_kfree_skb_any(msdu);
+					msdu = NULL;
+					pmon->mon_last_linkdesc_paddr = paddr;
+					goto next_msdu;
+				}
+				msdu_ppdu_id =
+					ath12k_dp_rxdesc_get_ppduid(ar->ab, rx_desc);
+
+				if (ath12k_dp_mon_comp_ppduid(msdu_ppdu_id,
+							      ppdu_id)) {
+					spin_unlock_bh(&pmon->mon_lock);
+					return rx_bufs_used;
+				}
+				pmon->mon_last_linkdesc_paddr = paddr;
+				is_first_msdu = false;
+			}
+			ath12k_wifi7_dp_mon_get_buf_len(&msdu_list.msdu_info[i],
+							&is_frag, &total_len,
+							&frag_len, &msdu_cnt);
+			rx_buf_size = rx_pkt_offset + l2_hdr_offset + frag_len;
+
+			if (ath12k_dp_pkt_set_pktlen(msdu, rx_buf_size)) {
+				dev_kfree_skb_any(msdu);
+				goto next_msdu;
+			}
+
+			if (!(*head_msdu))
+				*head_msdu = msdu;
+			else if (last)
+				last->next = msdu;
+
+			last = msdu;
+next_msdu:
+			pmon->mon_last_buf_cookie = msdu_list.sw_cookie[i];
+			rx_bufs_used++;
+			desc_info->skb = NULL;
+			list_add_tail(&desc_info->list, used_list);
+		}
+
+		ath12k_wifi7_hal_rx_buf_addr_info_set(&buf_info, paddr,
+						      sw_cookie, rbm);
+
+		ath12k_wifi7_dp_mon_next_link_desc_get(ab,
+						       msdu_link_desc, &paddr,
+						       &sw_cookie, &rbm,
+						       &p_buf_addr_info);
+
+		ath12k_dp_arch_rx_link_desc_return(ar->ab->dp, &buf_info,
+						   HAL_WBM_REL_BM_ACT_PUT_IN_IDLE);
+
+		p_last_buf_addr_info = p_buf_addr_info;
+
+	} while (paddr && msdu_cnt);
+
+	spin_unlock_bh(&pmon->mon_lock);
+
+	if (last)
+		last->next = NULL;
+
+	*tail_msdu = msdu;
+
+	if (msdu_cnt == 0)
+		*npackets = 1;
+
+	return rx_bufs_used;
+}
+
+/* The destination ring processing is stuck if the destination is not
+ * moving while status ring moves 16 PPDU. The destination ring processing
+ * skips this destination ring PPDU as a workaround.
+ */
+#define MON_DEST_RING_STUCK_MAX_CNT 16
+
+static void
+ath12k_wifi7_dp_rx_mon_dest_process(struct ath12k *ar, int mac_id,
+				    u32 quota, struct napi_struct *napi)
+{
+	struct ath12k_mon_data *pmon = (struct ath12k_mon_data *)&ar->dp.mon_data;
+	struct ath12k_pdev_mon_stats *rx_mon_stats;
+	u32 ppdu_id, rx_bufs_used = 0, ring_id;
+	u32 mpdu_rx_bufs_used, npackets = 0;
+	struct ath12k_base *ab = ar->ab;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	void *ring_entry, *mon_dst_srng;
+	struct dp_mon_mpdu *tmp_mpdu;
+	LIST_HEAD(rx_desc_used_list);
+	struct hal_srng *srng;
+
+	ring_id = dp->rxdma_err_dst_ring[mac_id].ring_id;
+	srng = &ab->hal.srng_list[ring_id];
+
+	mon_dst_srng = &ab->hal.srng_list[ring_id];
+
+	spin_lock_bh(&srng->lock);
+
+	ath12k_hal_srng_access_begin(ab, mon_dst_srng);
+
+	ppdu_id = pmon->mon_ppdu_info.ppdu_id;
+	rx_mon_stats = &pmon->rx_mon_stats;
+
+	while ((ring_entry = ath12k_hal_srng_dst_peek(ar->ab, mon_dst_srng))) {
+		struct sk_buff *head_msdu, *tail_msdu;
+
+		head_msdu = NULL;
+		tail_msdu = NULL;
+
+		mpdu_rx_bufs_used = ath12k_wifi7_dp_rx_mon_mpdu_pop(ar, mac_id,
+								    ring_entry,
+								    &head_msdu,
+								    &tail_msdu,
+								    &rx_desc_used_list,
+								    &npackets,
+								    &ppdu_id);
+
+		rx_bufs_used += mpdu_rx_bufs_used;
+
+		if (mpdu_rx_bufs_used) {
+			dp->mon_dest_ring_stuck_cnt = 0;
+		} else {
+			dp->mon_dest_ring_stuck_cnt++;
+			rx_mon_stats->dest_mon_not_reaped++;
+		}
+
+		if (dp->mon_dest_ring_stuck_cnt > MON_DEST_RING_STUCK_MAX_CNT) {
+			rx_mon_stats->dest_mon_stuck++;
+			ath12k_dbg(ar->ab, ATH12K_DBG_DATA,
+				   "status ring ppdu_id=%d dest ring ppdu_id=%d mon_dest_ring_stuck_cnt=%d dest_mon_not_reaped=%u dest_mon_stuck=%u\n",
+				   pmon->mon_ppdu_info.ppdu_id, ppdu_id,
+				   dp->mon_dest_ring_stuck_cnt,
+				   rx_mon_stats->dest_mon_not_reaped,
+				   rx_mon_stats->dest_mon_stuck);
+			spin_lock_bh(&pmon->mon_lock);
+			pmon->mon_ppdu_info.ppdu_id = ppdu_id;
+			spin_unlock_bh(&pmon->mon_lock);
+			continue;
+		}
+
+		if (ppdu_id != pmon->mon_ppdu_info.ppdu_id) {
+			spin_lock_bh(&pmon->mon_lock);
+			pmon->mon_ppdu_status = DP_PPDU_STATUS_START;
+			spin_unlock_bh(&pmon->mon_lock);
+			ath12k_dbg(ar->ab, ATH12K_DBG_DATA,
+				   "dest_rx: new ppdu_id %x != status ppdu_id %x dest_mon_not_reaped = %u dest_mon_stuck = %u\n",
+				   ppdu_id, pmon->mon_ppdu_info.ppdu_id,
+				   rx_mon_stats->dest_mon_not_reaped,
+				   rx_mon_stats->dest_mon_stuck);
+			break;
+		}
+
+		if (head_msdu && tail_msdu) {
+			tmp_mpdu = kzalloc(sizeof(*tmp_mpdu), GFP_ATOMIC);
+			if (!tmp_mpdu)
+				break;
+
+			tmp_mpdu->head = head_msdu;
+			tmp_mpdu->tail = tail_msdu;
+			tmp_mpdu->err_bitmap = pmon->err_bitmap;
+			tmp_mpdu->decap_format = pmon->decap_format;
+			ath12k_wifi7_dp_mon_rx_deliver(&ar->dp, tmp_mpdu,
+						       &pmon->mon_ppdu_info, napi);
+			rx_mon_stats->dest_mpdu_done++;
+			kfree(tmp_mpdu);
+		}
+
+		ring_entry = ath12k_hal_srng_dst_get_next_entry(ar->ab,
+								mon_dst_srng);
+	}
+	ath12k_hal_srng_access_end(ar->ab, mon_dst_srng);
+
+	spin_unlock_bh(&srng->lock);
+
+	if (rx_bufs_used) {
+		rx_mon_stats->dest_ppdu_done++;
+		ath12k_dp_rx_bufs_replenish(ar->ab->dp,
+					    &dp->rx_refill_buf_ring,
+					    &rx_desc_used_list,
+					    rx_bufs_used);
+	}
+}
+
+static enum dp_mon_status_buf_state
+ath12k_wifi7_dp_rx_mon_buf_done(struct ath12k_base *ab, struct hal_srng *srng,
+				struct dp_rxdma_mon_ring *rx_ring)
+{
+	struct ath12k_skb_rxcb *rxcb;
+	struct hal_tlv_64_hdr *tlv;
+	struct sk_buff *skb;
+	void *status_desc;
+	dma_addr_t paddr;
+	u32 cookie;
+	int buf_id;
+	u8 rbm;
+
+	status_desc = ath12k_hal_srng_src_next_peek(ab, srng);
+	if (!status_desc)
+		return DP_MON_STATUS_NO_DMA;
+
+	ath12k_wifi7_hal_rx_buf_addr_info_get(status_desc, &paddr, &cookie, &rbm);
+
+	buf_id = u32_get_bits(cookie, DP_RXDMA_BUF_COOKIE_BUF_ID);
+
+	spin_lock_bh(&rx_ring->idr_lock);
+	skb = idr_find(&rx_ring->bufs_idr, buf_id);
+	spin_unlock_bh(&rx_ring->idr_lock);
+
+	if (!skb)
+		return DP_MON_STATUS_NO_DMA;
+
+	rxcb = ATH12K_SKB_RXCB(skb);
+	dma_sync_single_for_cpu(ab->dev, rxcb->paddr,
+				skb->len + skb_tailroom(skb),
+				DMA_FROM_DEVICE);
+
+	tlv = (struct hal_tlv_64_hdr *)skb->data;
+	if (le64_get_bits(tlv->tl, HAL_TLV_HDR_TAG) != HAL_RX_STATUS_BUFFER_DONE)
+		return DP_MON_STATUS_NO_DMA;
+
+	return DP_MON_STATUS_REPLINISH;
+}
+
+static enum hal_rx_mon_status
+ath12k_wifi7_dp_mon_parse_rx_dest(struct ath12k_pdev_dp *dp_pdev,
+				  struct ath12k_mon_data *pmon,
+				  struct sk_buff *skb)
+{
+	struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev);
+	struct hal_tlv_64_hdr *tlv;
+	struct ath12k_skb_rxcb *rxcb;
+	enum hal_rx_mon_status hal_status;
+	u16 tlv_tag, tlv_len;
+	u8 *ptr = skb->data;
+
+	do {
+		tlv = (struct hal_tlv_64_hdr *)ptr;
+		tlv_tag = le64_get_bits(tlv->tl, HAL_TLV_64_HDR_TAG);
+
+		/* The actual length of PPDU_END is the combined length of many PHY
+		 * TLVs that follow. Skip the TLV header and
+		 * rx_rxpcu_classification_overview that follows the header to get to
+		 * next TLV.
+		 */
+
+		if (tlv_tag == HAL_RX_PPDU_END)
+			tlv_len = sizeof(struct hal_rx_rxpcu_classification_overview);
+		else
+			tlv_len = le64_get_bits(tlv->tl, HAL_TLV_64_HDR_LEN);
+
+		hal_status = ath12k_wifi7_dp_mon_rx_parse_status_tlv(dp_pdev, pmon,
+								     tlv);
+
+		if (ar->monitor_started && ar->ab->hw_params->rxdma1_enable &&
+		    ath12k_wifi7_dp_mon_parse_rx_dest_tlv(dp_pdev, pmon, hal_status,
+							  tlv->value))
+			return HAL_RX_MON_STATUS_PPDU_DONE;
+
+		ptr += sizeof(*tlv) + tlv_len;
+		ptr = PTR_ALIGN(ptr, HAL_TLV_64_ALIGN);
+
+		if ((ptr - skb->data) > skb->len)
+			break;
+
+	} while ((hal_status == HAL_RX_MON_STATUS_PPDU_NOT_DONE) ||
+		 (hal_status == HAL_RX_MON_STATUS_BUF_ADDR) ||
+		 (hal_status == HAL_RX_MON_STATUS_MPDU_START) ||
+		 (hal_status == HAL_RX_MON_STATUS_MPDU_END) ||
+		 (hal_status == HAL_RX_MON_STATUS_MSDU_END));
+
+	rxcb = ATH12K_SKB_RXCB(skb);
+	if (rxcb->is_end_of_ppdu)
+		hal_status = HAL_RX_MON_STATUS_PPDU_DONE;
+
+	return hal_status;
+}
+
+static enum hal_rx_mon_status
+ath12k_wifi7_dp_mon_rx_parse_mon_status(struct ath12k_pdev_dp *dp_pdev,
+					struct ath12k_mon_data *pmon,
+					struct sk_buff *skb,
+					struct napi_struct *napi)
+{
+	struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info;
+	struct dp_mon_mpdu *tmp;
+	struct dp_mon_mpdu *mon_mpdu = pmon->mon_mpdu;
+	enum hal_rx_mon_status hal_status;
+
+	hal_status = ath12k_wifi7_dp_mon_parse_rx_dest(dp_pdev, pmon, skb);
+	if (hal_status != HAL_RX_MON_STATUS_PPDU_DONE)
+		return hal_status;
+
+	list_for_each_entry_safe(mon_mpdu, tmp, &pmon->dp_rx_mon_mpdu_list, list) {
+		list_del(&mon_mpdu->list);
+
+		if (mon_mpdu->head && mon_mpdu->tail)
+			ath12k_wifi7_dp_mon_rx_deliver(dp_pdev, mon_mpdu,
+						       ppdu_info, napi);
+
+		kfree(mon_mpdu);
+	}
+
+	return hal_status;
+}
+
+static int
+ath12k_wifi7_dp_rx_reap_mon_status_ring(struct ath12k_base *ab, int mac_id,
+					int *budget, struct sk_buff_head *skb_list)
+{
+	const struct ath12k_hw_hal_params *hal_params;
+	int buf_id, srng_id, num_buffs_reaped = 0;
+	enum dp_mon_status_buf_state reap_status;
+	struct dp_rxdma_mon_ring *rx_ring;
+	struct ath12k_mon_data *pmon;
+	struct ath12k_skb_rxcb *rxcb;
+	struct hal_tlv_64_hdr *tlv;
+	void *rx_mon_status_desc;
+	struct hal_srng *srng;
+	struct ath12k_dp *dp;
+	struct sk_buff *skb;
+	struct ath12k *ar;
+	dma_addr_t paddr;
+	u32 cookie;
+	u8 rbm;
+
+	ar = ab->pdevs[ath12k_hw_mac_id_to_pdev_id(ab->hw_params, mac_id)].ar;
+	dp = ath12k_ab_to_dp(ab);
+	pmon = &ar->dp.mon_data;
+	srng_id = ath12k_hw_mac_id_to_srng_id(ab->hw_params, mac_id);
+	rx_ring = &dp->rx_mon_status_refill_ring[srng_id];
+
+	srng = &ab->hal.srng_list[rx_ring->refill_buf_ring.ring_id];
+
+	spin_lock_bh(&srng->lock);
+
+	ath12k_hal_srng_access_begin(ab, srng);
+
+	while (*budget) {
+		*budget -= 1;
+		rx_mon_status_desc = ath12k_hal_srng_src_peek(ab, srng);
+		if (!rx_mon_status_desc) {
+			pmon->buf_state = DP_MON_STATUS_REPLINISH;
+			break;
+		}
+		ath12k_wifi7_hal_rx_buf_addr_info_get(rx_mon_status_desc, &paddr,
+						      &cookie, &rbm);
+		if (paddr) {
+			buf_id = u32_get_bits(cookie, DP_RXDMA_BUF_COOKIE_BUF_ID);
+
+			spin_lock_bh(&rx_ring->idr_lock);
+			skb = idr_find(&rx_ring->bufs_idr, buf_id);
+			spin_unlock_bh(&rx_ring->idr_lock);
+
+			if (!skb) {
+				ath12k_warn(ab, "rx monitor status with invalid buf_id %d\n",
+					    buf_id);
+				pmon->buf_state = DP_MON_STATUS_REPLINISH;
+				goto move_next;
+			}
+
+			rxcb = ATH12K_SKB_RXCB(skb);
+
+			dma_sync_single_for_cpu(ab->dev, rxcb->paddr,
+						skb->len + skb_tailroom(skb),
+						DMA_FROM_DEVICE);
+
+			tlv = (struct hal_tlv_64_hdr *)skb->data;
+			if (le64_get_bits(tlv->tl, HAL_TLV_HDR_TAG) !=
+					HAL_RX_STATUS_BUFFER_DONE) {
+				pmon->buf_state = DP_MON_STATUS_NO_DMA;
+				ath12k_warn(ab,
+					    "mon status DONE not set %llx, buf_id %d\n",
+					    le64_get_bits(tlv->tl, HAL_TLV_HDR_TAG),
+					    buf_id);
+				/* RxDMA status done bit might not be set even
+				 * though tp is moved by HW.
+				 */
+
+				/* If done status is missing:
+				 * 1. As per MAC team's suggestion,
+				 *    when HP + 1 entry is peeked and if DMA
+				 *    is not done and if HP + 2 entry's DMA done
+				 *    is set. skip HP + 1 entry and
+				 *    start processing in next interrupt.
+				 * 2. If HP + 2 entry's DMA done is not set,
+				 *    poll onto HP + 1 entry DMA done to be set.
+				 *    Check status for same buffer for next time
+				 *    dp_rx_mon_status_srng_process
+				 */
+				reap_status = ath12k_wifi7_dp_rx_mon_buf_done(ab, srng,
+									      rx_ring);
+				if (reap_status == DP_MON_STATUS_NO_DMA)
+					continue;
+
+				spin_lock_bh(&rx_ring->idr_lock);
+				idr_remove(&rx_ring->bufs_idr, buf_id);
+				spin_unlock_bh(&rx_ring->idr_lock);
+
+				dma_unmap_single(ab->dev, rxcb->paddr,
+						 skb->len + skb_tailroom(skb),
+						 DMA_FROM_DEVICE);
+
+				dev_kfree_skb_any(skb);
+				pmon->buf_state = DP_MON_STATUS_REPLINISH;
+				goto move_next;
+			}
+
+			spin_lock_bh(&rx_ring->idr_lock);
+			idr_remove(&rx_ring->bufs_idr, buf_id);
+			spin_unlock_bh(&rx_ring->idr_lock);
+
+			dma_unmap_single(ab->dev, rxcb->paddr,
+					 skb->len + skb_tailroom(skb),
+					 DMA_FROM_DEVICE);
+
+			if (ath12k_dp_pkt_set_pktlen(skb, RX_MON_STATUS_BUF_SIZE)) {
+				dev_kfree_skb_any(skb);
+				goto move_next;
+			}
+			__skb_queue_tail(skb_list, skb);
+		} else {
+			pmon->buf_state = DP_MON_STATUS_REPLINISH;
+		}
+move_next:
+		skb = ath12k_dp_rx_alloc_mon_status_buf(ab, rx_ring,
+							&buf_id);
+		hal_params = ab->hal.hal_params;
+
+		if (!skb) {
+			ath12k_warn(ab, "failed to alloc buffer for status ring\n");
+			ath12k_wifi7_hal_rx_buf_addr_info_set(rx_mon_status_desc,
+							      0, 0,
+							      hal_params->rx_buf_rbm);
+			num_buffs_reaped++;
+			break;
+		}
+		rxcb = ATH12K_SKB_RXCB(skb);
+
+		cookie = u32_encode_bits(mac_id, DP_RXDMA_BUF_COOKIE_PDEV_ID) |
+			 u32_encode_bits(buf_id, DP_RXDMA_BUF_COOKIE_BUF_ID);
+
+		ath12k_wifi7_hal_rx_buf_addr_info_set(rx_mon_status_desc, rxcb->paddr,
+						      cookie, hal_params->rx_buf_rbm);
+		ath12k_hal_srng_src_get_next_entry(ab, srng);
+		num_buffs_reaped++;
+	}
+	ath12k_hal_srng_access_end(ab, srng);
+	spin_unlock_bh(&srng->lock);
+
+	return num_buffs_reaped;
+}
+
+static int
+__ath12k_wifi7_dp_mon_process_ring(struct ath12k *ar, int mac_id,
+				   struct napi_struct *napi, int *budget)
+{
+	struct ath12k_mon_data *pmon = (struct ath12k_mon_data *)&ar->dp.mon_data;
+	struct ath12k_pdev_mon_stats *rx_mon_stats = &pmon->rx_mon_stats;
+	struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info;
+	enum hal_rx_mon_status hal_status;
+	struct sk_buff_head skb_list;
+	int num_buffs_reaped;
+	struct sk_buff *skb;
+
+	__skb_queue_head_init(&skb_list);
+
+	num_buffs_reaped = ath12k_wifi7_dp_rx_reap_mon_status_ring(ar->ab, mac_id,
+								   budget, &skb_list);
+	if (!num_buffs_reaped)
+		goto exit;
+
+	while ((skb = __skb_dequeue(&skb_list))) {
+		memset(ppdu_info, 0, sizeof(*ppdu_info));
+		ppdu_info->peer_id = HAL_INVALID_PEERID;
+
+		hal_status = ath12k_wifi7_dp_mon_parse_rx_dest(&ar->dp, pmon, skb);
+
+		if (ar->monitor_started &&
+		    pmon->mon_ppdu_status == DP_PPDU_STATUS_START &&
+		    hal_status == HAL_TLV_STATUS_PPDU_DONE) {
+			rx_mon_stats->status_ppdu_done++;
+			pmon->mon_ppdu_status = DP_PPDU_STATUS_DONE;
+			ath12k_wifi7_dp_rx_mon_dest_process(ar, mac_id, *budget, napi);
+			pmon->mon_ppdu_status = DP_PPDU_STATUS_START;
+		}
+
+		dev_kfree_skb_any(skb);
+	}
+
+exit:
+	return num_buffs_reaped;
+}
+
+static int
+ath12k_wifi7_dp_mon_srng_process(struct ath12k_pdev_dp *pdev_dp, int *budget,
+				 struct napi_struct *napi)
+{
+	struct ath12k_dp *dp = pdev_dp->dp;
+	struct ath12k_base *ab = dp->ab;
+	struct ath12k_mon_data *pmon = (struct ath12k_mon_data *)&pdev_dp->mon_data;
+	struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info;
+	struct hal_mon_dest_desc *mon_dst_desc;
+	struct sk_buff *skb;
+	struct ath12k_skb_rxcb *rxcb;
+	struct dp_srng *mon_dst_ring;
+	struct hal_srng *srng;
+	struct dp_rxdma_mon_ring *buf_ring;
+	struct ath12k_dp_link_peer *peer;
+	struct sk_buff_head skb_list;
+	u64 cookie;
+	int num_buffs_reaped = 0, srng_id, buf_id;
+	u32 hal_status, end_offset, info0, end_reason;
+	u8 pdev_idx = ath12k_hw_mac_id_to_pdev_id(ab->hw_params, pdev_dp->mac_id);
+
+	__skb_queue_head_init(&skb_list);
+	srng_id = ath12k_hw_mac_id_to_srng_id(ab->hw_params, pdev_idx);
+	mon_dst_ring = &pdev_dp->rxdma_mon_dst_ring[srng_id];
+	buf_ring = &dp->rxdma_mon_buf_ring;
+
+	srng = &ab->hal.srng_list[mon_dst_ring->ring_id];
+	spin_lock_bh(&srng->lock);
+	ath12k_hal_srng_access_begin(ab, srng);
+
+	while (likely(*budget)) {
+		mon_dst_desc = ath12k_hal_srng_dst_peek(ab, srng);
+		if (unlikely(!mon_dst_desc))
+			break;
+
+		/* In case of empty descriptor, the cookie in the ring descriptor
+		 * is invalid. Therefore, this entry is skipped, and ring processing
+		 * continues.
+		 */
+		info0 = le32_to_cpu(mon_dst_desc->info0);
+		if (u32_get_bits(info0, HAL_MON_DEST_INFO0_EMPTY_DESC))
+			goto move_next;
+
+		cookie = le32_to_cpu(mon_dst_desc->cookie);
+		buf_id = u32_get_bits(cookie, DP_RXDMA_BUF_COOKIE_BUF_ID);
+
+		spin_lock_bh(&buf_ring->idr_lock);
+		skb = idr_remove(&buf_ring->bufs_idr, buf_id);
+		spin_unlock_bh(&buf_ring->idr_lock);
+
+		if (unlikely(!skb)) {
+			ath12k_warn(ab, "monitor destination with invalid buf_id %d\n",
+				    buf_id);
+			goto move_next;
+		}
+
+		rxcb = ATH12K_SKB_RXCB(skb);
+		dma_unmap_single(ab->dev, rxcb->paddr,
+				 skb->len + skb_tailroom(skb),
+				 DMA_FROM_DEVICE);
+
+		end_reason = u32_get_bits(info0, HAL_MON_DEST_INFO0_END_REASON);
+
+		/* HAL_MON_FLUSH_DETECTED implies that an rx flush received at the end of
+		 * rx PPDU and HAL_MON_PPDU_TRUNCATED implies that the PPDU got
+		 * truncated due to a system level error. In both the cases, buffer data
+		 * can be discarded
+		 */
+		if ((end_reason == HAL_MON_FLUSH_DETECTED) ||
+		    (end_reason == HAL_MON_PPDU_TRUNCATED)) {
+			ath12k_dbg(ab, ATH12K_DBG_DATA,
+				   "Monitor dest descriptor end reason %d", end_reason);
+			dev_kfree_skb_any(skb);
+			goto move_next;
+		}
+
+		/* Calculate the budget when the ring descriptor with the
+		 * HAL_MON_END_OF_PPDU to ensure that one PPDU worth of data is always
+		 * reaped. This helps to efficiently utilize the NAPI budget.
+		 */
+		if (end_reason == HAL_MON_END_OF_PPDU) {
+			*budget -= 1;
+			rxcb->is_end_of_ppdu = true;
+		}
+
+		end_offset = u32_get_bits(info0, HAL_MON_DEST_INFO0_END_OFFSET);
+		if (likely(end_offset <= DP_RX_BUFFER_SIZE)) {
+			skb_put(skb, end_offset);
+		} else {
+			ath12k_warn(ab,
+				    "invalid offset on mon stats destination %u\n",
+				    end_offset);
+			skb_put(skb, DP_RX_BUFFER_SIZE);
+		}
+
+		__skb_queue_tail(&skb_list, skb);
+
+move_next:
+		ath12k_dp_mon_buf_replenish(ab, buf_ring, 1);
+		ath12k_hal_srng_dst_get_next_entry(ab, srng);
+		num_buffs_reaped++;
+	}
+
+	ath12k_hal_srng_access_end(ab, srng);
+	spin_unlock_bh(&srng->lock);
+
+	if (!num_buffs_reaped)
+		return 0;
+
+	/* In some cases, one PPDU worth of data can be spread across multiple NAPI
+	 * schedules, To avoid losing existing parsed ppdu_info information, skip
+	 * the memset of the ppdu_info structure and continue processing it.
+	 */
+	if (!ppdu_info->ppdu_continuation)
+		ath12k_wifi7_dp_mon_rx_memset_ppdu_info(ppdu_info);
+
+	while ((skb = __skb_dequeue(&skb_list))) {
+		hal_status = ath12k_wifi7_dp_mon_rx_parse_mon_status(pdev_dp, pmon,
+								     skb, napi);
+		if (hal_status != HAL_RX_MON_STATUS_PPDU_DONE) {
+			ppdu_info->ppdu_continuation = true;
+			dev_kfree_skb_any(skb);
+			continue;
+		}
+
+		if (ppdu_info->peer_id == HAL_INVALID_PEERID)
+			goto free_skb;
+
+		rcu_read_lock();
+		peer = ath12k_dp_link_peer_find_by_peerid(pdev_dp, ppdu_info->peer_id);
+		if (!peer || !peer->sta) {
+			ath12k_dbg(ab, ATH12K_DBG_DATA,
+				   "failed to find the peer with monitor peer_id %d\n",
+				   ppdu_info->peer_id);
+			goto next_skb;
+		}
+
+		if (ppdu_info->reception_type == HAL_RX_RECEPTION_TYPE_SU) {
+			ath12k_dp_mon_rx_update_peer_su_stats(peer, ppdu_info);
+		} else if ((ppdu_info->fc_valid) &&
+			   (ppdu_info->ast_index != HAL_AST_IDX_INVALID)) {
+			ath12k_dp_mon_rx_process_ulofdma(ppdu_info);
+			ath12k_dp_mon_rx_update_peer_mu_stats(ab, ppdu_info);
+		}
+
+next_skb:
+		rcu_read_unlock();
+free_skb:
+		dev_kfree_skb_any(skb);
+		ath12k_wifi7_dp_mon_rx_memset_ppdu_info(ppdu_info);
+	}
+
+	return num_buffs_reaped;
+}
+
+int ath12k_wifi7_dp_mon_process_ring(struct ath12k_dp *dp, int mac_id,
+				     struct napi_struct *napi, int budget,
+				     enum dp_monitor_mode monitor_mode)
+{
+	u8 pdev_idx = ath12k_hw_mac_id_to_pdev_id(dp->hw_params, mac_id);
+	struct ath12k_pdev_dp *dp_pdev;
+	struct ath12k *ar;
+	int num_buffs_reaped = 0;
+
+	rcu_read_lock();
+
+	dp_pdev = ath12k_dp_to_pdev_dp(dp, pdev_idx);
+	if (!dp_pdev) {
+		rcu_read_unlock();
+		return 0;
+	}
+
+	if (dp->hw_params->rxdma1_enable) {
+		if (monitor_mode == ATH12K_DP_RX_MONITOR_MODE)
+			num_buffs_reaped = ath12k_wifi7_dp_mon_srng_process(dp_pdev,
+									    &budget,
+									    napi);
+	} else {
+		ar = ath12k_pdev_dp_to_ar(dp_pdev);
+
+		if (ar->monitor_started)
+			num_buffs_reaped =
+				__ath12k_wifi7_dp_mon_process_ring(ar, mac_id, napi,
+								   &budget);
+	}
+
+	rcu_read_unlock();
+
+	return num_buffs_reaped;
+}
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.h b/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.h
new file mode 100644
index 000000000000..148d1e0b70fe
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef ATH12K_DP_MON_WIFI7_H
+#define ATH12K_DP_MON_WIFI7_H
+
+#include "hw.h"
+
+enum dp_monitor_mode;
+
+int ath12k_wifi7_dp_mon_process_ring(struct ath12k_dp *dp, int mac_id,
+				     struct napi_struct *napi, int budget,
+				     enum dp_monitor_mode monitor_mode);
+enum hal_rx_mon_status
+ath12k_wifi7_dp_mon_tx_parse_mon_status(struct ath12k_pdev_dp *dp_pdev,
+					struct ath12k_mon_data *pmon,
+					struct sk_buff *skb,
+					struct napi_struct *napi,
+					u32 ppdu_id);
+#endif
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c
new file mode 100644
index 000000000000..a1ca55fe51c0
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c
@@ -0,0 +1,2202 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include "dp_rx.h"
+#include "../dp_tx.h"
+#include "../peer.h"
+#include "hal_qcn9274.h"
+#include "hal_wcn7850.h"
+
+static u16 ath12k_wifi7_dp_rx_get_peer_id(struct ath12k_dp *dp,
+					  enum ath12k_peer_metadata_version ver,
+					  __le32 peer_metadata)
+{
+	switch (ver) {
+	default:
+		ath12k_warn(dp->ab, "Unknown peer metadata version: %d", ver);
+		fallthrough;
+	case ATH12K_PEER_METADATA_V0:
+		return le32_get_bits(peer_metadata,
+				     RX_MPDU_DESC_META_DATA_V0_PEER_ID);
+	case ATH12K_PEER_METADATA_V1:
+		return le32_get_bits(peer_metadata,
+				     RX_MPDU_DESC_META_DATA_V1_PEER_ID);
+	case ATH12K_PEER_METADATA_V1A:
+		return le32_get_bits(peer_metadata,
+				     RX_MPDU_DESC_META_DATA_V1A_PEER_ID);
+	case ATH12K_PEER_METADATA_V1B:
+		return le32_get_bits(peer_metadata,
+				     RX_MPDU_DESC_META_DATA_V1B_PEER_ID);
+	}
+}
+
+void ath12k_wifi7_peer_rx_tid_qref_setup(struct ath12k_base *ab, u16 peer_id, u16 tid,
+					 dma_addr_t paddr)
+{
+	struct ath12k_reo_queue_ref *qref;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	bool ml_peer = false;
+
+	if (!ab->hw_params->reoq_lut_support)
+		return;
+
+	if (peer_id & ATH12K_PEER_ML_ID_VALID) {
+		peer_id &= ~ATH12K_PEER_ML_ID_VALID;
+		ml_peer = true;
+	}
+
+	if (ml_peer)
+		qref = (struct ath12k_reo_queue_ref *)dp->ml_reoq_lut.vaddr +
+				(peer_id * (IEEE80211_NUM_TIDS + 1) + tid);
+	else
+		qref = (struct ath12k_reo_queue_ref *)dp->reoq_lut.vaddr +
+				(peer_id * (IEEE80211_NUM_TIDS + 1) + tid);
+
+	qref->info0 = u32_encode_bits(lower_32_bits(paddr),
+				      BUFFER_ADDR_INFO0_ADDR);
+	qref->info1 = u32_encode_bits(upper_32_bits(paddr),
+				      BUFFER_ADDR_INFO1_ADDR) |
+		      u32_encode_bits(tid, DP_REO_QREF_NUM);
+
+	ath12k_hal_reo_shared_qaddr_cache_clear(ab);
+}
+
+void ath12k_wifi7_peer_rx_tid_qref_reset(struct ath12k_base *ab,
+					 u16 peer_id, u16 tid)
+{
+	struct ath12k_reo_queue_ref *qref;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	bool ml_peer = false;
+
+	if (!ab->hw_params->reoq_lut_support)
+		return;
+
+	if (peer_id & ATH12K_PEER_ML_ID_VALID) {
+		peer_id &= ~ATH12K_PEER_ML_ID_VALID;
+		ml_peer = true;
+	}
+
+	if (ml_peer)
+		qref = (struct ath12k_reo_queue_ref *)dp->ml_reoq_lut.vaddr +
+				(peer_id * (IEEE80211_NUM_TIDS + 1) + tid);
+	else
+		qref = (struct ath12k_reo_queue_ref *)dp->reoq_lut.vaddr +
+				(peer_id * (IEEE80211_NUM_TIDS + 1) + tid);
+
+	qref->info0 = u32_encode_bits(0, BUFFER_ADDR_INFO0_ADDR);
+	qref->info1 = u32_encode_bits(0, BUFFER_ADDR_INFO1_ADDR) |
+		      u32_encode_bits(tid, DP_REO_QREF_NUM);
+}
+
+void ath12k_wifi7_dp_rx_peer_tid_delete(struct ath12k_base *ab,
+					struct ath12k_dp_link_peer *peer, u8 tid)
+{
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+
+	if (!(peer->rx_tid_active_bitmask & (1 << tid)))
+		return;
+
+	ath12k_dp_mark_tid_as_inactive(dp, peer->peer_id, tid);
+	ath12k_dp_rx_process_reo_cmd_update_rx_queue_list(dp);
+}
+
+int ath12k_wifi7_dp_rx_link_desc_return(struct ath12k_dp *dp,
+					struct ath12k_buffer_addr *buf_addr_info,
+					enum hal_wbm_rel_bm_act action)
+{
+	struct ath12k_base *ab = dp->ab;
+	struct hal_wbm_release_ring *desc;
+	struct hal_srng *srng;
+	int ret = 0;
+
+	srng = &dp->hal->srng_list[dp->wbm_desc_rel_ring.ring_id];
+
+	spin_lock_bh(&srng->lock);
+
+	ath12k_hal_srng_access_begin(ab, srng);
+
+	desc = ath12k_hal_srng_src_get_next_entry(ab, srng);
+	if (!desc) {
+		ret = -ENOBUFS;
+		goto exit;
+	}
+
+	ath12k_wifi7_hal_rx_msdu_link_desc_set(ab, desc, buf_addr_info, action);
+
+exit:
+	ath12k_hal_srng_access_end(ab, srng);
+
+	spin_unlock_bh(&srng->lock);
+
+	return ret;
+}
+
+int ath12k_wifi7_dp_reo_cmd_send(struct ath12k_base *ab,
+				 struct ath12k_dp_rx_tid_rxq *rx_tid,
+				 enum hal_reo_cmd_type type,
+				 struct ath12k_hal_reo_cmd *cmd,
+				 void (*cb)(struct ath12k_dp *dp, void *ctx,
+					    enum hal_reo_cmd_status status))
+{
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct ath12k_dp_rx_reo_cmd *dp_cmd;
+	struct hal_srng *cmd_ring;
+	int cmd_num;
+
+	cmd_ring = &ab->hal.srng_list[dp->reo_cmd_ring.ring_id];
+	cmd_num = ath12k_wifi7_hal_reo_cmd_send(ab, cmd_ring, type, cmd);
+
+	/* cmd_num should start from 1, during failure return the error code */
+	if (cmd_num < 0)
+		return cmd_num;
+
+	/* reo cmd ring descriptors has cmd_num starting from 1 */
+	if (cmd_num == 0)
+		return -EINVAL;
+
+	if (!cb)
+		return 0;
+
+	/* Can this be optimized so that we keep the pending command list only
+	 * for tid delete command to free up the resource on the command status
+	 * indication?
+	 */
+	dp_cmd = kzalloc(sizeof(*dp_cmd), GFP_ATOMIC);
+
+	if (!dp_cmd)
+		return -ENOMEM;
+
+	memcpy(&dp_cmd->data, rx_tid, sizeof(*rx_tid));
+	dp_cmd->cmd_num = cmd_num;
+	dp_cmd->handler = cb;
+
+	spin_lock_bh(&dp->reo_cmd_lock);
+	list_add_tail(&dp_cmd->list, &dp->reo_cmd_list);
+	spin_unlock_bh(&dp->reo_cmd_lock);
+
+	return 0;
+}
+
+int ath12k_wifi7_peer_rx_tid_reo_update(struct ath12k_dp *dp,
+					struct ath12k_dp_link_peer *peer,
+					struct ath12k_dp_rx_tid *rx_tid,
+					u32 ba_win_sz, u16 ssn,
+					bool update_ssn)
+{
+	struct ath12k_hal_reo_cmd cmd = {};
+	struct ath12k_base *ab = dp->ab;
+	int ret;
+	struct ath12k_dp_rx_tid_rxq rx_tid_rxq;
+
+	ath12k_dp_init_rx_tid_rxq(&rx_tid_rxq, rx_tid,
+				  (peer->rx_tid_active_bitmask & (1 << rx_tid->tid)));
+
+	cmd.addr_lo = lower_32_bits(rx_tid_rxq.qbuf.paddr_aligned);
+	cmd.addr_hi = upper_32_bits(rx_tid_rxq.qbuf.paddr_aligned);
+	cmd.flag = HAL_REO_CMD_FLG_NEED_STATUS;
+	cmd.upd0 = HAL_REO_CMD_UPD0_BA_WINDOW_SIZE;
+	cmd.ba_window_size = ba_win_sz;
+
+	if (update_ssn) {
+		cmd.upd0 |= HAL_REO_CMD_UPD0_SSN;
+		cmd.upd2 = u32_encode_bits(ssn, HAL_REO_CMD_UPD2_SSN);
+	}
+
+	ret = ath12k_wifi7_dp_reo_cmd_send(ab, &rx_tid_rxq,
+					   HAL_REO_CMD_UPDATE_RX_QUEUE, &cmd,
+					   NULL);
+	if (ret) {
+		ath12k_warn(ab, "failed to update rx tid queue, tid %d (%d)\n",
+			    rx_tid_rxq.tid, ret);
+		return ret;
+	}
+
+	rx_tid->ba_win_sz = ba_win_sz;
+
+	return 0;
+}
+
+int ath12k_wifi7_dp_reo_cache_flush(struct ath12k_base *ab,
+				    struct ath12k_dp_rx_tid_rxq *rx_tid)
+{
+	struct ath12k_hal_reo_cmd cmd = {};
+	int ret;
+
+	cmd.addr_lo = lower_32_bits(rx_tid->qbuf.paddr_aligned);
+	cmd.addr_hi = upper_32_bits(rx_tid->qbuf.paddr_aligned);
+	/* HAL_REO_CMD_FLG_FLUSH_FWD_ALL_MPDUS - all pending MPDUs
+	 *in the bitmap will be forwarded/flushed to REO output rings
+	 */
+	cmd.flag = HAL_REO_CMD_FLG_NEED_STATUS |
+		   HAL_REO_CMD_FLG_FLUSH_FWD_ALL_MPDUS;
+
+	/* For all QoS TIDs (except NON_QOS), the driver allocates a maximum
+	 * window size of 1024. In such cases, the driver can issue a single
+	 * 1KB descriptor flush command instead of sending multiple 128-byte
+	 * flush commands for each QoS TID, improving efficiency.
+	 */
+
+	if (rx_tid->tid != HAL_DESC_REO_NON_QOS_TID)
+		cmd.flag |= HAL_REO_CMD_FLG_FLUSH_QUEUE_1K_DESC;
+
+	ret = ath12k_wifi7_dp_reo_cmd_send(ab, rx_tid,
+					   HAL_REO_CMD_FLUSH_CACHE,
+					   &cmd, ath12k_dp_reo_cmd_free);
+	return ret;
+}
+
+int ath12k_wifi7_dp_rx_assign_reoq(struct ath12k_base *ab, struct ath12k_dp_peer *dp_peer,
+				   struct ath12k_dp_rx_tid *rx_tid,
+				   u16 ssn, enum hal_pn_type pn_type)
+{
+	u32 ba_win_sz = rx_tid->ba_win_sz;
+	struct ath12k_reoq_buf *buf;
+	void *vaddr, *vaddr_aligned;
+	dma_addr_t paddr_aligned;
+	u8 tid = rx_tid->tid;
+	u32 hw_desc_sz;
+	int ret;
+
+	buf = &dp_peer->reoq_bufs[tid];
+	if (!buf->vaddr) {
+		/* TODO: Optimize the memory allocation for qos tid based on
+		 * the actual BA window size in REO tid update path.
+		 */
+		if (tid == HAL_DESC_REO_NON_QOS_TID)
+			hw_desc_sz = ath12k_wifi7_hal_reo_qdesc_size(ba_win_sz, tid);
+		else
+			hw_desc_sz = ath12k_wifi7_hal_reo_qdesc_size(DP_BA_WIN_SZ_MAX,
+								     tid);
+
+		vaddr = kzalloc(hw_desc_sz + HAL_LINK_DESC_ALIGN - 1, GFP_ATOMIC);
+		if (!vaddr)
+			return -ENOMEM;
+
+		vaddr_aligned = PTR_ALIGN(vaddr, HAL_LINK_DESC_ALIGN);
+
+		ath12k_wifi7_hal_reo_qdesc_setup(vaddr_aligned, tid, ba_win_sz,
+						 ssn, pn_type);
+
+		paddr_aligned = dma_map_single(ab->dev, vaddr_aligned, hw_desc_sz,
+					       DMA_BIDIRECTIONAL);
+		ret = dma_mapping_error(ab->dev, paddr_aligned);
+		if (ret) {
+			kfree(vaddr);
+			return ret;
+		}
+
+		buf->vaddr = vaddr;
+		buf->paddr_aligned = paddr_aligned;
+		buf->size = hw_desc_sz;
+	}
+
+	rx_tid->qbuf = *buf;
+
+	return 0;
+}
+
+int ath12k_wifi7_dp_rx_tid_delete_handler(struct ath12k_base *ab,
+					  struct ath12k_dp_rx_tid_rxq *rx_tid)
+{
+	struct ath12k_hal_reo_cmd cmd = {};
+
+	cmd.flag = HAL_REO_CMD_FLG_NEED_STATUS;
+	cmd.addr_lo = lower_32_bits(rx_tid->qbuf.paddr_aligned);
+	cmd.addr_hi = upper_32_bits(rx_tid->qbuf.paddr_aligned);
+	cmd.upd0 |= HAL_REO_CMD_UPD0_VLD;
+	/* Observed flush cache failure, to avoid that set vld bit during delete */
+	cmd.upd1 |= HAL_REO_CMD_UPD1_VLD;
+
+	return ath12k_wifi7_dp_reo_cmd_send(ab, rx_tid,
+					    HAL_REO_CMD_UPDATE_RX_QUEUE, &cmd,
+					    ath12k_dp_rx_tid_del_func);
+}
+
+static void ath12k_wifi7_dp_rx_h_csum_offload(struct sk_buff *msdu,
+					      struct hal_rx_desc_data *rx_info)
+{
+	msdu->ip_summed = (rx_info->ip_csum_fail || rx_info->l4_csum_fail) ?
+			   CHECKSUM_NONE : CHECKSUM_UNNECESSARY;
+}
+
+static void ath12k_wifi7_dp_rx_h_mpdu(struct ath12k_pdev_dp *dp_pdev,
+				      struct sk_buff *msdu,
+				      struct hal_rx_desc *rx_desc,
+				      struct hal_rx_desc_data *rx_info)
+{
+	struct ath12k_skb_rxcb *rxcb;
+	enum hal_encrypt_type enctype;
+	bool is_decrypted = false;
+	struct ieee80211_hdr *hdr;
+	struct ath12k_dp_peer *peer;
+	struct ieee80211_rx_status *rx_status = rx_info->rx_status;
+	u32 err_bitmap = rx_info->err_bitmap;
+
+	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
+			 "dp_rx_h_mpdu called without rcu lock");
+
+	/* PN for multicast packets will be checked in mac80211 */
+	rxcb = ATH12K_SKB_RXCB(msdu);
+	rxcb->is_mcbc = rx_info->is_mcbc;
+
+	if (rxcb->is_mcbc)
+		rxcb->peer_id = rx_info->peer_id;
+
+	peer = ath12k_dp_peer_find_by_peerid(dp_pdev, rxcb->peer_id);
+	if (peer) {
+		/* resetting mcbc bit because mcbc packets are unicast
+		 * packets only for AP as STA sends unicast packets.
+		 */
+		rxcb->is_mcbc = rxcb->is_mcbc && !peer->ucast_ra_only;
+
+		if (rxcb->is_mcbc)
+			enctype = peer->sec_type_grp;
+		else
+			enctype = peer->sec_type;
+	} else {
+		enctype = HAL_ENCRYPT_TYPE_OPEN;
+	}
+
+	if (enctype != HAL_ENCRYPT_TYPE_OPEN && !err_bitmap)
+		is_decrypted = rx_info->is_decrypted;
+
+	/* Clear per-MPDU flags while leaving per-PPDU flags intact */
+	rx_status->flag &= ~(RX_FLAG_FAILED_FCS_CRC |
+			     RX_FLAG_MMIC_ERROR |
+			     RX_FLAG_DECRYPTED |
+			     RX_FLAG_IV_STRIPPED |
+			     RX_FLAG_MMIC_STRIPPED);
+
+	if (err_bitmap & HAL_RX_MPDU_ERR_FCS)
+		rx_status->flag |= RX_FLAG_FAILED_FCS_CRC;
+	if (err_bitmap & HAL_RX_MPDU_ERR_TKIP_MIC)
+		rx_status->flag |= RX_FLAG_MMIC_ERROR;
+
+	if (is_decrypted) {
+		rx_status->flag |= RX_FLAG_DECRYPTED | RX_FLAG_MMIC_STRIPPED;
+
+		if (rx_info->is_mcbc)
+			rx_status->flag |= RX_FLAG_MIC_STRIPPED |
+					   RX_FLAG_ICV_STRIPPED;
+		else
+			rx_status->flag |= RX_FLAG_IV_STRIPPED |
+					   RX_FLAG_PN_VALIDATED;
+	}
+
+	ath12k_wifi7_dp_rx_h_csum_offload(msdu, rx_info);
+	ath12k_dp_rx_h_undecap(dp_pdev, msdu, rx_desc,
+			       enctype, is_decrypted, rx_info);
+
+	if (!is_decrypted || rx_info->is_mcbc)
+		return;
+
+	if (rx_info->decap_type != DP_RX_DECAP_TYPE_ETHERNET2_DIX) {
+		hdr = (void *)msdu->data;
+		hdr->frame_control &= ~__cpu_to_le16(IEEE80211_FCTL_PROTECTED);
+	}
+}
+
+static int ath12k_wifi7_dp_rx_msdu_coalesce(struct ath12k_hal *hal,
+					    struct sk_buff_head *msdu_list,
+					    struct sk_buff *first, struct sk_buff *last,
+					    u8 l3pad_bytes, int msdu_len,
+					    struct hal_rx_desc_data *rx_info)
+{
+	struct sk_buff *skb;
+	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(first);
+	int buf_first_hdr_len, buf_first_len;
+	struct hal_rx_desc *ldesc;
+	int space_extra, rem_len, buf_len;
+	u32 hal_rx_desc_sz = hal->hal_desc_sz;
+	bool is_continuation;
+
+	/* As the msdu is spread across multiple rx buffers,
+	 * find the offset to the start of msdu for computing
+	 * the length of the msdu in the first buffer.
+	 */
+	buf_first_hdr_len = hal_rx_desc_sz + l3pad_bytes;
+	buf_first_len = DP_RX_BUFFER_SIZE - buf_first_hdr_len;
+
+	if (WARN_ON_ONCE(msdu_len <= buf_first_len)) {
+		skb_put(first, buf_first_hdr_len + msdu_len);
+		skb_pull(first, buf_first_hdr_len);
+		return 0;
+	}
+
+	ldesc = (struct hal_rx_desc *)last->data;
+	rxcb->is_first_msdu = rx_info->is_first_msdu;
+	rxcb->is_last_msdu = rx_info->is_last_msdu;
+
+	/* MSDU spans over multiple buffers because the length of the MSDU
+	 * exceeds DP_RX_BUFFER_SIZE - HAL_RX_DESC_SIZE. So assume the data
+	 * in the first buf is of length DP_RX_BUFFER_SIZE - HAL_RX_DESC_SIZE.
+	 */
+	skb_put(first, DP_RX_BUFFER_SIZE);
+	skb_pull(first, buf_first_hdr_len);
+
+	/* When an MSDU spread over multiple buffers MSDU_END
+	 * tlvs are valid only in the last buffer. Copy those tlvs.
+	 */
+	ath12k_dp_rx_desc_end_tlv_copy(hal, rxcb->rx_desc, ldesc);
+
+	space_extra = msdu_len - (buf_first_len + skb_tailroom(first));
+	if (space_extra > 0 &&
+	    (pskb_expand_head(first, 0, space_extra, GFP_ATOMIC) < 0)) {
+		/* Free up all buffers of the MSDU */
+		while ((skb = __skb_dequeue(msdu_list)) != NULL) {
+			rxcb = ATH12K_SKB_RXCB(skb);
+			if (!rxcb->is_continuation) {
+				dev_kfree_skb_any(skb);
+				break;
+			}
+			dev_kfree_skb_any(skb);
+		}
+		return -ENOMEM;
+	}
+
+	rem_len = msdu_len - buf_first_len;
+	while ((skb = __skb_dequeue(msdu_list)) != NULL && rem_len > 0) {
+		rxcb = ATH12K_SKB_RXCB(skb);
+		is_continuation = rxcb->is_continuation;
+		if (is_continuation)
+			buf_len = DP_RX_BUFFER_SIZE - hal_rx_desc_sz;
+		else
+			buf_len = rem_len;
+
+		if (buf_len > (DP_RX_BUFFER_SIZE - hal_rx_desc_sz)) {
+			WARN_ON_ONCE(1);
+			dev_kfree_skb_any(skb);
+			return -EINVAL;
+		}
+
+		skb_put(skb, buf_len + hal_rx_desc_sz);
+		skb_pull(skb, hal_rx_desc_sz);
+		skb_copy_from_linear_data(skb, skb_put(first, buf_len),
+					  buf_len);
+		dev_kfree_skb_any(skb);
+
+		rem_len -= buf_len;
+		if (!is_continuation)
+			break;
+	}
+
+	return 0;
+}
+
+static int ath12k_wifi7_dp_rx_process_msdu(struct ath12k_pdev_dp *dp_pdev,
+					   struct sk_buff *msdu,
+					   struct sk_buff_head *msdu_list,
+					   struct hal_rx_desc_data *rx_info)
+{
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct hal_rx_desc *rx_desc, *lrx_desc;
+	struct ath12k_skb_rxcb *rxcb;
+	struct sk_buff *last_buf;
+	struct ath12k_hal *hal = dp->hal;
+	u8 l3_pad_bytes;
+	u16 msdu_len;
+	int ret;
+	u32 hal_rx_desc_sz = hal->hal_desc_sz;
+
+	last_buf = ath12k_dp_rx_get_msdu_last_buf(msdu_list, msdu);
+	if (!last_buf) {
+		ath12k_warn(dp->ab,
+			    "No valid Rx buffer to access MSDU_END tlv\n");
+		ret = -EIO;
+		goto free_out;
+	}
+
+	rx_desc = (struct hal_rx_desc *)msdu->data;
+	lrx_desc = (struct hal_rx_desc *)last_buf->data;
+
+	ath12k_dp_extract_rx_desc_data(hal, rx_info, rx_desc, lrx_desc);
+	if (!rx_info->msdu_done) {
+		ath12k_warn(dp->ab, "msdu_done bit in msdu_end is not set\n");
+		ret = -EIO;
+		goto free_out;
+	}
+
+	rxcb = ATH12K_SKB_RXCB(msdu);
+	rxcb->rx_desc = rx_desc;
+	msdu_len = rx_info->msdu_len;
+	l3_pad_bytes = rx_info->l3_pad_bytes;
+
+	if (rxcb->is_frag) {
+		skb_pull(msdu, hal_rx_desc_sz);
+	} else if (!rxcb->is_continuation) {
+		if ((msdu_len + hal_rx_desc_sz) > DP_RX_BUFFER_SIZE) {
+			ret = -EINVAL;
+			ath12k_warn(dp->ab, "invalid msdu len %u\n", msdu_len);
+			ath12k_dbg_dump(dp->ab, ATH12K_DBG_DATA, NULL, "", rx_desc,
+					sizeof(*rx_desc));
+			goto free_out;
+		}
+		skb_put(msdu, hal_rx_desc_sz + l3_pad_bytes + msdu_len);
+		skb_pull(msdu, hal_rx_desc_sz + l3_pad_bytes);
+	} else {
+		ret = ath12k_wifi7_dp_rx_msdu_coalesce(hal, msdu_list,
+						       msdu, last_buf,
+						       l3_pad_bytes, msdu_len,
+						       rx_info);
+		if (ret) {
+			ath12k_warn(dp->ab,
+				    "failed to coalesce msdu rx buffer%d\n", ret);
+			goto free_out;
+		}
+	}
+
+	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, rx_desc, msdu,
+							     rx_info))) {
+		ret = -EINVAL;
+		goto free_out;
+	}
+
+	ath12k_dp_rx_h_ppdu(dp_pdev, rx_info);
+	ath12k_wifi7_dp_rx_h_mpdu(dp_pdev, msdu, rx_desc, rx_info);
+
+	rx_info->rx_status->flag |= RX_FLAG_SKIP_MONITOR | RX_FLAG_DUP_VALIDATED;
+
+	return 0;
+
+free_out:
+	return ret;
+}
+
+static void
+ath12k_wifi7_dp_rx_process_received_packets(struct ath12k_dp *dp,
+					    struct napi_struct *napi,
+					    struct sk_buff_head *msdu_list,
+					    int ring_id)
+{
+	struct ath12k_hw_group *ag = dp->ag;
+	struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp;
+	struct ieee80211_rx_status rx_status = {};
+	struct ath12k_skb_rxcb *rxcb;
+	struct sk_buff *msdu;
+	struct ath12k *ar;
+	struct ath12k_pdev_dp *dp_pdev;
+	struct ath12k_hw_link *hw_links = ag->hw_links;
+	struct ath12k_base *partner_ab;
+	struct hal_rx_desc_data rx_info;
+	struct ath12k_dp *partner_dp;
+	u8 hw_link_id, pdev_idx;
+	int ret;
+
+	if (skb_queue_empty(msdu_list))
+		return;
+
+	rx_info.addr2_present = false;
+	rx_info.rx_status = &rx_status;
+
+	rcu_read_lock();
+
+	while ((msdu = __skb_dequeue(msdu_list))) {
+		rxcb = ATH12K_SKB_RXCB(msdu);
+		hw_link_id = rxcb->hw_link_id;
+		partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp,
+						    hw_links[hw_link_id].device_id);
+		pdev_idx = ath12k_hw_mac_id_to_pdev_id(partner_dp->hw_params,
+						       hw_links[hw_link_id].pdev_idx);
+		partner_ab = partner_dp->ab;
+		ar = partner_ab->pdevs[pdev_idx].ar;
+		if (!rcu_dereference(partner_ab->pdevs_active[pdev_idx])) {
+			dev_kfree_skb_any(msdu);
+			continue;
+		}
+
+		if (test_bit(ATH12K_FLAG_CAC_RUNNING, &ar->dev_flags)) {
+			dev_kfree_skb_any(msdu);
+			continue;
+		}
+
+		dp_pdev = ath12k_dp_to_pdev_dp(partner_dp, pdev_idx);
+		if (!dp_pdev) {
+			dev_kfree_skb_any(msdu);
+			continue;
+		}
+
+		ret = ath12k_wifi7_dp_rx_process_msdu(dp_pdev, msdu, msdu_list, &rx_info);
+		if (ret) {
+			ath12k_dbg(dp->ab, ATH12K_DBG_DATA,
+				   "Unable to process msdu %d", ret);
+			dev_kfree_skb_any(msdu);
+			continue;
+		}
+
+		ath12k_dp_rx_deliver_msdu(dp_pdev, napi, msdu, &rx_info);
+	}
+
+	rcu_read_unlock();
+}
+
+int ath12k_wifi7_dp_rx_process(struct ath12k_dp *dp, int ring_id,
+			       struct napi_struct *napi, int budget)
+{
+	struct ath12k_hw_group *ag = dp->ag;
+	struct ath12k_base *ab = dp->ab;
+	struct ath12k_hal *hal = dp->hal;
+	struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp;
+	struct list_head rx_desc_used_list[ATH12K_MAX_DEVICES];
+	struct ath12k_hw_link *hw_links = ag->hw_links;
+	int num_buffs_reaped[ATH12K_MAX_DEVICES] = {};
+	struct ath12k_rx_desc_info *desc_info;
+	struct dp_rxdma_ring *rx_ring = &dp->rx_refill_buf_ring;
+	struct hal_reo_dest_ring *desc;
+	struct ath12k_dp *partner_dp;
+	struct sk_buff_head msdu_list;
+	struct ath12k_skb_rxcb *rxcb;
+	int total_msdu_reaped = 0;
+	u8 hw_link_id, device_id;
+	struct hal_srng *srng;
+	struct sk_buff *msdu;
+	bool done = false;
+	u64 desc_va;
+
+	__skb_queue_head_init(&msdu_list);
+
+	for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++)
+		INIT_LIST_HEAD(&rx_desc_used_list[device_id]);
+
+	srng = &hal->srng_list[dp->reo_dst_ring[ring_id].ring_id];
+
+	spin_lock_bh(&srng->lock);
+
+try_again:
+	ath12k_hal_srng_access_begin(ab, srng);
+
+	while ((desc = ath12k_hal_srng_dst_get_next_entry(ab, srng))) {
+		struct rx_mpdu_desc *mpdu_info;
+		struct rx_msdu_desc *msdu_info;
+		enum hal_reo_dest_ring_push_reason push_reason;
+		u32 cookie;
+
+		cookie = le32_get_bits(desc->buf_addr_info.info1,
+				       BUFFER_ADDR_INFO1_SW_COOKIE);
+
+		hw_link_id = le32_get_bits(desc->info0,
+					   HAL_REO_DEST_RING_INFO0_SRC_LINK_ID);
+
+		desc_va = ((u64)le32_to_cpu(desc->buf_va_hi) << 32 |
+			   le32_to_cpu(desc->buf_va_lo));
+		desc_info = (struct ath12k_rx_desc_info *)((unsigned long)desc_va);
+
+		device_id = hw_links[hw_link_id].device_id;
+		partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp, device_id);
+		if (unlikely(!partner_dp)) {
+			if (desc_info->skb) {
+				dev_kfree_skb_any(desc_info->skb);
+				desc_info->skb = NULL;
+			}
+
+			continue;
+		}
+
+		/* retry manual desc retrieval */
+		if (!desc_info) {
+			desc_info = ath12k_dp_get_rx_desc(partner_dp, cookie);
+			if (!desc_info) {
+				ath12k_warn(partner_dp->ab, "Invalid cookie in manual descriptor retrieval: 0x%x\n",
+					    cookie);
+				continue;
+			}
+		}
+
+		if (desc_info->magic != ATH12K_DP_RX_DESC_MAGIC)
+			ath12k_warn(ab, "Check HW CC implementation");
+
+		msdu = desc_info->skb;
+		desc_info->skb = NULL;
+
+		list_add_tail(&desc_info->list, &rx_desc_used_list[device_id]);
+
+		rxcb = ATH12K_SKB_RXCB(msdu);
+		dma_unmap_single(partner_dp->dev, rxcb->paddr,
+				 msdu->len + skb_tailroom(msdu),
+				 DMA_FROM_DEVICE);
+
+		num_buffs_reaped[device_id]++;
+		dp->device_stats.reo_rx[ring_id][dp->device_id]++;
+
+		push_reason = le32_get_bits(desc->info0,
+					    HAL_REO_DEST_RING_INFO0_PUSH_REASON);
+		if (push_reason !=
+		    HAL_REO_DEST_RING_PUSH_REASON_ROUTING_INSTRUCTION) {
+			dev_kfree_skb_any(msdu);
+			dp->device_stats.hal_reo_error[ring_id]++;
+			continue;
+		}
+
+		msdu_info = &desc->rx_msdu_info;
+		mpdu_info = &desc->rx_mpdu_info;
+
+		rxcb->is_first_msdu = !!(le32_to_cpu(msdu_info->info0) &
+					 RX_MSDU_DESC_INFO0_FIRST_MSDU_IN_MPDU);
+		rxcb->is_last_msdu = !!(le32_to_cpu(msdu_info->info0) &
+					RX_MSDU_DESC_INFO0_LAST_MSDU_IN_MPDU);
+		rxcb->is_continuation = !!(le32_to_cpu(msdu_info->info0) &
+					   RX_MSDU_DESC_INFO0_MSDU_CONTINUATION);
+		rxcb->hw_link_id = hw_link_id;
+		rxcb->peer_id = ath12k_wifi7_dp_rx_get_peer_id(dp, dp->peer_metadata_ver,
+							       mpdu_info->peer_meta_data);
+		rxcb->tid = le32_get_bits(mpdu_info->info0,
+					  RX_MPDU_DESC_INFO0_TID);
+
+		__skb_queue_tail(&msdu_list, msdu);
+
+		if (!rxcb->is_continuation) {
+			total_msdu_reaped++;
+			done = true;
+		} else {
+			done = false;
+		}
+
+		if (total_msdu_reaped >= budget)
+			break;
+	}
+
+	/* Hw might have updated the head pointer after we cached it.
+	 * In this case, even though there are entries in the ring we'll
+	 * get rx_desc NULL. Give the read another try with updated cached
+	 * head pointer so that we can reap complete MPDU in the current
+	 * rx processing.
+	 */
+	if (!done && ath12k_hal_srng_dst_num_free(ab, srng, true)) {
+		ath12k_hal_srng_access_end(ab, srng);
+		goto try_again;
+	}
+
+	ath12k_hal_srng_access_end(ab, srng);
+
+	spin_unlock_bh(&srng->lock);
+
+	if (!total_msdu_reaped)
+		goto exit;
+
+	for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) {
+		if (!num_buffs_reaped[device_id])
+			continue;
+
+		partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp, device_id);
+		rx_ring = &partner_dp->rx_refill_buf_ring;
+
+		ath12k_dp_rx_bufs_replenish(partner_dp, rx_ring,
+					    &rx_desc_used_list[device_id],
+					    num_buffs_reaped[device_id]);
+	}
+
+	ath12k_wifi7_dp_rx_process_received_packets(dp, napi, &msdu_list,
+						    ring_id);
+
+exit:
+	return total_msdu_reaped;
+}
+
+static bool
+ath12k_wifi7_dp_rx_h_defrag_validate_incr_pn(struct ath12k_pdev_dp *dp_pdev,
+					     struct ath12k_dp_rx_tid *rx_tid,
+					     enum hal_encrypt_type encrypt_type)
+{
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct sk_buff *first_frag, *skb;
+	u64 last_pn;
+	u64 cur_pn;
+
+	first_frag = skb_peek(&rx_tid->rx_frags);
+
+	if (encrypt_type != HAL_ENCRYPT_TYPE_CCMP_128 &&
+	    encrypt_type != HAL_ENCRYPT_TYPE_CCMP_256 &&
+	    encrypt_type != HAL_ENCRYPT_TYPE_GCMP_128 &&
+	    encrypt_type != HAL_ENCRYPT_TYPE_AES_GCMP_256)
+		return true;
+
+	last_pn = ath12k_dp_rx_h_get_pn(dp, first_frag);
+	skb_queue_walk(&rx_tid->rx_frags, skb) {
+		if (skb == first_frag)
+			continue;
+
+		cur_pn = ath12k_dp_rx_h_get_pn(dp, skb);
+		if (cur_pn != last_pn + 1)
+			return false;
+		last_pn = cur_pn;
+	}
+	return true;
+}
+
+static int ath12k_wifi7_dp_rx_h_defrag_reo_reinject(struct ath12k_dp *dp,
+						    struct ath12k_dp_rx_tid *rx_tid,
+						    struct sk_buff *defrag_skb)
+{
+	struct ath12k_base *ab = dp->ab;
+	struct ath12k_hal *hal = dp->hal;
+	struct hal_rx_desc *rx_desc = (struct hal_rx_desc *)defrag_skb->data;
+	struct hal_reo_entrance_ring *reo_ent_ring;
+	struct hal_reo_dest_ring *reo_dest_ring;
+	struct dp_link_desc_bank *link_desc_banks;
+	struct hal_rx_msdu_link *msdu_link;
+	struct hal_rx_msdu_details *msdu0;
+	struct hal_srng *srng;
+	dma_addr_t link_paddr, buf_paddr;
+	u32 desc_bank, msdu_info, msdu_ext_info, mpdu_info;
+	u32 cookie, hal_rx_desc_sz, dest_ring_info0, queue_addr_hi;
+	int ret;
+	struct ath12k_rx_desc_info *desc_info;
+	enum hal_rx_buf_return_buf_manager idle_link_rbm = dp->idle_link_rbm;
+	u8 dst_ind;
+
+	hal_rx_desc_sz = hal->hal_desc_sz;
+	link_desc_banks = dp->link_desc_banks;
+	reo_dest_ring = rx_tid->dst_ring_desc;
+
+	ath12k_wifi7_hal_rx_reo_ent_paddr_get(&reo_dest_ring->buf_addr_info,
+					      &link_paddr, &cookie);
+	desc_bank = u32_get_bits(cookie, DP_LINK_DESC_BANK_MASK);
+
+	msdu_link = (struct hal_rx_msdu_link *)(link_desc_banks[desc_bank].vaddr +
+			(link_paddr - link_desc_banks[desc_bank].paddr));
+	msdu0 = &msdu_link->msdu_link[0];
+	msdu_ext_info = le32_to_cpu(msdu0->rx_msdu_ext_info.info0);
+	dst_ind = u32_get_bits(msdu_ext_info, RX_MSDU_EXT_DESC_INFO0_REO_DEST_IND);
+
+	memset(msdu0, 0, sizeof(*msdu0));
+
+	msdu_info = u32_encode_bits(1, RX_MSDU_DESC_INFO0_FIRST_MSDU_IN_MPDU) |
+		    u32_encode_bits(1, RX_MSDU_DESC_INFO0_LAST_MSDU_IN_MPDU) |
+		    u32_encode_bits(0, RX_MSDU_DESC_INFO0_MSDU_CONTINUATION) |
+		    u32_encode_bits(defrag_skb->len - hal_rx_desc_sz,
+				    RX_MSDU_DESC_INFO0_MSDU_LENGTH) |
+		    u32_encode_bits(1, RX_MSDU_DESC_INFO0_VALID_SA) |
+		    u32_encode_bits(1, RX_MSDU_DESC_INFO0_VALID_DA);
+	msdu0->rx_msdu_info.info0 = cpu_to_le32(msdu_info);
+	msdu0->rx_msdu_ext_info.info0 = cpu_to_le32(msdu_ext_info);
+
+	/* change msdu len in hal rx desc */
+	ath12k_dp_rxdesc_set_msdu_len(hal, rx_desc, defrag_skb->len - hal_rx_desc_sz);
+
+	buf_paddr = dma_map_single(dp->dev, defrag_skb->data,
+				   defrag_skb->len + skb_tailroom(defrag_skb),
+				   DMA_TO_DEVICE);
+	if (dma_mapping_error(dp->dev, buf_paddr))
+		return -ENOMEM;
+
+	spin_lock_bh(&dp->rx_desc_lock);
+	desc_info = list_first_entry_or_null(&dp->rx_desc_free_list,
+					     struct ath12k_rx_desc_info,
+					     list);
+	if (!desc_info) {
+		spin_unlock_bh(&dp->rx_desc_lock);
+		ath12k_warn(ab, "failed to find rx desc for reinject\n");
+		ret = -ENOMEM;
+		goto err_unmap_dma;
+	}
+
+	desc_info->skb = defrag_skb;
+	desc_info->in_use = true;
+
+	list_del(&desc_info->list);
+	spin_unlock_bh(&dp->rx_desc_lock);
+
+	ATH12K_SKB_RXCB(defrag_skb)->paddr = buf_paddr;
+
+	ath12k_wifi7_hal_rx_buf_addr_info_set(&msdu0->buf_addr_info, buf_paddr,
+					      desc_info->cookie,
+					      HAL_RX_BUF_RBM_SW3_BM);
+
+	/* Fill mpdu details into reo entrance ring */
+	srng = &hal->srng_list[dp->reo_reinject_ring.ring_id];
+
+	spin_lock_bh(&srng->lock);
+	ath12k_hal_srng_access_begin(ab, srng);
+
+	reo_ent_ring = ath12k_hal_srng_src_get_next_entry(ab, srng);
+	if (!reo_ent_ring) {
+		ath12k_hal_srng_access_end(ab, srng);
+		spin_unlock_bh(&srng->lock);
+		ret = -ENOSPC;
+		goto err_free_desc;
+	}
+	memset(reo_ent_ring, 0, sizeof(*reo_ent_ring));
+
+	ath12k_wifi7_hal_rx_buf_addr_info_set(&reo_ent_ring->buf_addr_info, link_paddr,
+					      cookie, idle_link_rbm);
+
+	mpdu_info = u32_encode_bits(1, RX_MPDU_DESC_INFO0_MSDU_COUNT) |
+		    u32_encode_bits(0, RX_MPDU_DESC_INFO0_FRAG_FLAG) |
+		    u32_encode_bits(1, RX_MPDU_DESC_INFO0_RAW_MPDU) |
+		    u32_encode_bits(1, RX_MPDU_DESC_INFO0_VALID_PN) |
+		    u32_encode_bits(rx_tid->tid, RX_MPDU_DESC_INFO0_TID);
+
+	reo_ent_ring->rx_mpdu_info.info0 = cpu_to_le32(mpdu_info);
+	reo_ent_ring->rx_mpdu_info.peer_meta_data =
+		reo_dest_ring->rx_mpdu_info.peer_meta_data;
+
+	if (dp->hw_params->reoq_lut_support) {
+		reo_ent_ring->queue_addr_lo = reo_dest_ring->rx_mpdu_info.peer_meta_data;
+		queue_addr_hi = 0;
+	} else {
+		reo_ent_ring->queue_addr_lo =
+				cpu_to_le32(lower_32_bits(rx_tid->qbuf.paddr_aligned));
+		queue_addr_hi = upper_32_bits(rx_tid->qbuf.paddr_aligned);
+	}
+
+	reo_ent_ring->info0 = le32_encode_bits(queue_addr_hi,
+					       HAL_REO_ENTR_RING_INFO0_QUEUE_ADDR_HI) |
+			      le32_encode_bits(dst_ind,
+					       HAL_REO_ENTR_RING_INFO0_DEST_IND);
+
+	reo_ent_ring->info1 = le32_encode_bits(rx_tid->cur_sn,
+					       HAL_REO_ENTR_RING_INFO1_MPDU_SEQ_NUM);
+	dest_ring_info0 = le32_get_bits(reo_dest_ring->info0,
+					HAL_REO_DEST_RING_INFO0_SRC_LINK_ID);
+	reo_ent_ring->info2 =
+		cpu_to_le32(u32_get_bits(dest_ring_info0,
+					 HAL_REO_ENTR_RING_INFO2_SRC_LINK_ID));
+
+	ath12k_hal_srng_access_end(ab, srng);
+	spin_unlock_bh(&srng->lock);
+
+	return 0;
+
+err_free_desc:
+	spin_lock_bh(&dp->rx_desc_lock);
+	desc_info->in_use = false;
+	desc_info->skb = NULL;
+	list_add_tail(&desc_info->list, &dp->rx_desc_free_list);
+	spin_unlock_bh(&dp->rx_desc_lock);
+err_unmap_dma:
+	dma_unmap_single(dp->dev, buf_paddr, defrag_skb->len + skb_tailroom(defrag_skb),
+			 DMA_TO_DEVICE);
+	return ret;
+}
+
+static int ath12k_wifi7_dp_rx_h_verify_tkip_mic(struct ath12k_pdev_dp *dp_pdev,
+						struct ath12k_dp_peer *peer,
+						enum hal_encrypt_type enctype,
+						struct sk_buff *msdu,
+						struct hal_rx_desc_data *rx_info)
+{
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_hal *hal = dp->hal;
+	struct hal_rx_desc *rx_desc = (struct hal_rx_desc *)msdu->data;
+	struct ieee80211_rx_status *rxs = IEEE80211_SKB_RXCB(msdu);
+	struct ieee80211_key_conf *key_conf;
+	struct ieee80211_hdr *hdr;
+	u8 mic[IEEE80211_CCMP_MIC_LEN];
+	int head_len, tail_len, ret;
+	size_t data_len;
+	u32 hdr_len, hal_rx_desc_sz = hal->hal_desc_sz;
+	u8 *key, *data;
+	u8 key_idx;
+
+	if (enctype != HAL_ENCRYPT_TYPE_TKIP_MIC)
+		return 0;
+
+	rx_info->addr2_present = false;
+	rx_info->rx_status = rxs;
+
+	hdr = (struct ieee80211_hdr *)(msdu->data + hal_rx_desc_sz);
+	hdr_len = ieee80211_hdrlen(hdr->frame_control);
+	head_len = hdr_len + hal_rx_desc_sz + IEEE80211_TKIP_IV_LEN;
+	tail_len = IEEE80211_CCMP_MIC_LEN + IEEE80211_TKIP_ICV_LEN + FCS_LEN;
+
+	if (!is_multicast_ether_addr(hdr->addr1))
+		key_idx = peer->ucast_keyidx;
+	else
+		key_idx = peer->mcast_keyidx;
+
+	key_conf = peer->keys[key_idx];
+
+	data = msdu->data + head_len;
+	data_len = msdu->len - head_len - tail_len;
+	key = &key_conf->key[NL80211_TKIP_DATA_OFFSET_RX_MIC_KEY];
+
+	ret = ath12k_dp_rx_h_michael_mic(peer->tfm_mmic, key, hdr, data,
+					 data_len, mic);
+	if (ret || memcmp(mic, data + data_len, IEEE80211_CCMP_MIC_LEN))
+		goto mic_fail;
+
+	return 0;
+
+mic_fail:
+	(ATH12K_SKB_RXCB(msdu))->is_first_msdu = true;
+	(ATH12K_SKB_RXCB(msdu))->is_last_msdu = true;
+
+	ath12k_dp_extract_rx_desc_data(hal, rx_info, rx_desc, rx_desc);
+
+	rxs->flag |= RX_FLAG_MMIC_ERROR | RX_FLAG_MMIC_STRIPPED |
+		    RX_FLAG_IV_STRIPPED | RX_FLAG_DECRYPTED;
+	skb_pull(msdu, hal_rx_desc_sz);
+
+	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, rx_desc, msdu,
+							     rx_info)))
+		return -EINVAL;
+
+	ath12k_dp_rx_h_ppdu(dp_pdev, rx_info);
+	ath12k_dp_rx_h_undecap(dp_pdev, msdu, rx_desc,
+			       HAL_ENCRYPT_TYPE_TKIP_MIC, true, rx_info);
+	ieee80211_rx(ath12k_pdev_dp_to_hw(dp_pdev), msdu);
+	return -EINVAL;
+}
+
+static int ath12k_wifi7_dp_rx_h_defrag(struct ath12k_pdev_dp *dp_pdev,
+				       struct ath12k_dp_peer *peer,
+				       struct ath12k_dp_rx_tid *rx_tid,
+				       struct sk_buff **defrag_skb,
+				       enum hal_encrypt_type enctype,
+				       struct hal_rx_desc_data *rx_info)
+{
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_base *ab = dp->ab;
+	struct sk_buff *skb, *first_frag, *last_frag;
+	struct ieee80211_hdr *hdr;
+	bool is_decrypted = false;
+	int msdu_len = 0;
+	int extra_space;
+	u32 flags, hal_rx_desc_sz = ab->hal.hal_desc_sz;
+
+	first_frag = skb_peek(&rx_tid->rx_frags);
+	last_frag = skb_peek_tail(&rx_tid->rx_frags);
+
+	skb_queue_walk(&rx_tid->rx_frags, skb) {
+		flags = 0;
+		hdr = (struct ieee80211_hdr *)(skb->data + hal_rx_desc_sz);
+
+		if (enctype != HAL_ENCRYPT_TYPE_OPEN)
+			is_decrypted = rx_info->is_decrypted;
+
+		if (is_decrypted) {
+			if (skb != first_frag)
+				flags |= RX_FLAG_IV_STRIPPED;
+			if (skb != last_frag)
+				flags |= RX_FLAG_ICV_STRIPPED |
+					RX_FLAG_MIC_STRIPPED;
+		}
+
+		/* RX fragments are always raw packets */
+		if (skb != last_frag)
+			skb_trim(skb, skb->len - FCS_LEN);
+		ath12k_dp_rx_h_undecap_frag(dp_pdev, skb, enctype, flags);
+
+		if (skb != first_frag)
+			skb_pull(skb, hal_rx_desc_sz +
+				      ieee80211_hdrlen(hdr->frame_control));
+		msdu_len += skb->len;
+	}
+
+	extra_space = msdu_len - (DP_RX_BUFFER_SIZE + skb_tailroom(first_frag));
+	if (extra_space > 0 &&
+	    (pskb_expand_head(first_frag, 0, extra_space, GFP_ATOMIC) < 0))
+		return -ENOMEM;
+
+	__skb_unlink(first_frag, &rx_tid->rx_frags);
+	while ((skb = __skb_dequeue(&rx_tid->rx_frags))) {
+		skb_put_data(first_frag, skb->data, skb->len);
+		dev_kfree_skb_any(skb);
+	}
+
+	hdr = (struct ieee80211_hdr *)(first_frag->data + hal_rx_desc_sz);
+	hdr->frame_control &= ~__cpu_to_le16(IEEE80211_FCTL_MOREFRAGS);
+	ATH12K_SKB_RXCB(first_frag)->is_frag = 1;
+
+	if (ath12k_wifi7_dp_rx_h_verify_tkip_mic(dp_pdev, peer, enctype, first_frag,
+						 rx_info))
+		first_frag = NULL;
+
+	*defrag_skb = first_frag;
+	return 0;
+}
+
+void ath12k_wifi7_dp_rx_frags_cleanup(struct ath12k_dp_rx_tid *rx_tid,
+				      bool rel_link_desc)
+{
+	enum hal_wbm_rel_bm_act act = HAL_WBM_REL_BM_ACT_PUT_IN_IDLE;
+	struct ath12k_buffer_addr *buf_addr_info;
+	struct ath12k_dp *dp = rx_tid->dp;
+
+	lockdep_assert_held(&dp->dp_lock);
+
+	if (rx_tid->dst_ring_desc) {
+		if (rel_link_desc) {
+			buf_addr_info = &rx_tid->dst_ring_desc->buf_addr_info;
+			ath12k_wifi7_dp_rx_link_desc_return(dp, buf_addr_info, act);
+		}
+		kfree(rx_tid->dst_ring_desc);
+		rx_tid->dst_ring_desc = NULL;
+	}
+
+	rx_tid->cur_sn = 0;
+	rx_tid->last_frag_no = 0;
+	rx_tid->rx_frag_bitmap = 0;
+	__skb_queue_purge(&rx_tid->rx_frags);
+}
+
+static int ath12k_wifi7_dp_rx_frag_h_mpdu(struct ath12k_pdev_dp *dp_pdev,
+					  struct sk_buff *msdu,
+					  struct hal_reo_dest_ring *ring_desc,
+					  struct hal_rx_desc_data *rx_info)
+{
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_hal *hal = dp->hal;
+	struct ath12k_base *ab = dp->ab;
+	struct ath12k_dp_peer *peer;
+	struct ath12k_dp_rx_tid *rx_tid;
+	struct sk_buff *defrag_skb = NULL;
+	u32 peer_id = rx_info->peer_id;
+	u16 seqno, frag_no;
+	u8 tid = rx_info->tid;
+	int ret = 0;
+	bool more_frags;
+	enum hal_encrypt_type enctype = rx_info->enctype;
+
+	frag_no = ath12k_dp_rx_h_frag_no(hal, msdu);
+	more_frags = ath12k_dp_rx_h_more_frags(hal, msdu);
+	seqno = rx_info->seq_no;
+
+	if (!rx_info->seq_ctl_valid || !rx_info->fc_valid ||
+	    tid > IEEE80211_NUM_TIDS)
+		return -EINVAL;
+
+	/* received unfragmented packet in reo
+	 * exception ring, this shouldn't happen
+	 * as these packets typically come from
+	 * reo2sw srngs.
+	 */
+	if (WARN_ON_ONCE(!frag_no && !more_frags))
+		return -EINVAL;
+
+	spin_lock_bh(&dp->dp_lock);
+	peer = ath12k_dp_peer_find_by_peerid(dp_pdev, peer_id);
+	if (!peer) {
+		ath12k_warn(ab, "failed to find the peer to de-fragment received fragment peer_id %d\n",
+			    peer_id);
+		ret = -ENOENT;
+		goto out_unlock;
+	}
+
+	if (!peer->dp_setup_done) {
+		ath12k_warn(ab, "The peer %pM [%d] has uninitialized datapath\n",
+			    peer->addr, peer_id);
+		ret = -ENOENT;
+		goto out_unlock;
+	}
+
+	rx_tid = &peer->rx_tid[tid];
+
+	if ((!skb_queue_empty(&rx_tid->rx_frags) && seqno != rx_tid->cur_sn) ||
+	    skb_queue_empty(&rx_tid->rx_frags)) {
+		/* Flush stored fragments and start a new sequence */
+		ath12k_wifi7_dp_rx_frags_cleanup(rx_tid, true);
+		rx_tid->cur_sn = seqno;
+	}
+
+	if (rx_tid->rx_frag_bitmap & BIT(frag_no)) {
+		/* Fragment already present */
+		ret = -EINVAL;
+		goto out_unlock;
+	}
+
+	if ((!rx_tid->rx_frag_bitmap || frag_no > __fls(rx_tid->rx_frag_bitmap)))
+		__skb_queue_tail(&rx_tid->rx_frags, msdu);
+	else
+		ath12k_dp_rx_h_sort_frags(hal, &rx_tid->rx_frags, msdu);
+
+	rx_tid->rx_frag_bitmap |= BIT(frag_no);
+	if (!more_frags)
+		rx_tid->last_frag_no = frag_no;
+
+	if (frag_no == 0) {
+		rx_tid->dst_ring_desc = kmemdup(ring_desc,
+						sizeof(*rx_tid->dst_ring_desc),
+						GFP_ATOMIC);
+		if (!rx_tid->dst_ring_desc) {
+			ret = -ENOMEM;
+			goto out_unlock;
+		}
+	} else {
+		ath12k_wifi7_dp_rx_link_desc_return(dp, &ring_desc->buf_addr_info,
+						    HAL_WBM_REL_BM_ACT_PUT_IN_IDLE);
+	}
+
+	if (!rx_tid->last_frag_no ||
+	    rx_tid->rx_frag_bitmap != GENMASK(rx_tid->last_frag_no, 0)) {
+		mod_timer(&rx_tid->frag_timer, jiffies +
+					       ATH12K_DP_RX_FRAGMENT_TIMEOUT_MS);
+		goto out_unlock;
+	}
+
+	spin_unlock_bh(&dp->dp_lock);
+	timer_delete_sync(&rx_tid->frag_timer);
+	spin_lock_bh(&dp->dp_lock);
+
+	peer = ath12k_dp_peer_find_by_peerid(dp_pdev, peer_id);
+	if (!peer)
+		goto err_frags_cleanup;
+
+	if (!ath12k_wifi7_dp_rx_h_defrag_validate_incr_pn(dp_pdev, rx_tid, enctype))
+		goto err_frags_cleanup;
+
+	if (ath12k_wifi7_dp_rx_h_defrag(dp_pdev, peer, rx_tid, &defrag_skb,
+					enctype, rx_info))
+		goto err_frags_cleanup;
+
+	if (!defrag_skb)
+		goto err_frags_cleanup;
+
+	if (ath12k_wifi7_dp_rx_h_defrag_reo_reinject(dp, rx_tid, defrag_skb))
+		goto err_frags_cleanup;
+
+	ath12k_wifi7_dp_rx_frags_cleanup(rx_tid, false);
+	goto out_unlock;
+
+err_frags_cleanup:
+	dev_kfree_skb_any(defrag_skb);
+	ath12k_wifi7_dp_rx_frags_cleanup(rx_tid, true);
+out_unlock:
+	spin_unlock_bh(&dp->dp_lock);
+	return ret;
+}
+
+static int
+ath12k_wifi7_dp_process_rx_err_buf(struct ath12k_pdev_dp *dp_pdev,
+				   struct hal_reo_dest_ring *desc,
+				   struct list_head *used_list,
+				   bool drop, u32 cookie)
+{
+	struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev);
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_hal *hal = dp->hal;
+	struct sk_buff *msdu;
+	struct ath12k_skb_rxcb *rxcb;
+	struct hal_rx_desc_data rx_info;
+	struct hal_rx_desc *rx_desc;
+	u16 msdu_len;
+	u32 hal_rx_desc_sz = hal->hal_desc_sz;
+	struct ath12k_rx_desc_info *desc_info;
+	u64 desc_va;
+
+	desc_va = ((u64)le32_to_cpu(desc->buf_va_hi) << 32 |
+		   le32_to_cpu(desc->buf_va_lo));
+	desc_info = (struct ath12k_rx_desc_info *)((unsigned long)desc_va);
+
+	/* retry manual desc retrieval */
+	if (!desc_info) {
+		desc_info = ath12k_dp_get_rx_desc(dp, cookie);
+		if (!desc_info) {
+			ath12k_warn(dp->ab,
+				    "Invalid cookie in DP rx error descriptor retrieval: 0x%x\n",
+				    cookie);
+			return -EINVAL;
+		}
+	}
+
+	if (desc_info->magic != ATH12K_DP_RX_DESC_MAGIC)
+		ath12k_warn(dp->ab, "RX Exception, Check HW CC implementation");
+
+	msdu = desc_info->skb;
+	desc_info->skb = NULL;
+
+	list_add_tail(&desc_info->list, used_list);
+
+	rxcb = ATH12K_SKB_RXCB(msdu);
+	dma_unmap_single(dp->dev, rxcb->paddr,
+			 msdu->len + skb_tailroom(msdu),
+			 DMA_FROM_DEVICE);
+
+	if (drop) {
+		dev_kfree_skb_any(msdu);
+		return 0;
+	}
+
+	rcu_read_lock();
+	if (!rcu_dereference(ar->ab->pdevs_active[ar->pdev_idx])) {
+		dev_kfree_skb_any(msdu);
+		goto exit;
+	}
+
+	if (test_bit(ATH12K_FLAG_CAC_RUNNING, &ar->dev_flags)) {
+		dev_kfree_skb_any(msdu);
+		goto exit;
+	}
+
+	rx_desc = (struct hal_rx_desc *)msdu->data;
+	ath12k_dp_extract_rx_desc_data(hal, &rx_info, rx_desc, rx_desc);
+
+	msdu_len = rx_info.msdu_len;
+	if ((msdu_len + hal_rx_desc_sz) > DP_RX_BUFFER_SIZE) {
+		ath12k_warn(dp->ab, "invalid msdu leng %u", msdu_len);
+		ath12k_dbg_dump(dp->ab, ATH12K_DBG_DATA, NULL, "", rx_desc,
+				sizeof(*rx_desc));
+		dev_kfree_skb_any(msdu);
+		goto exit;
+	}
+
+	skb_put(msdu, hal_rx_desc_sz + msdu_len);
+
+	if (ath12k_wifi7_dp_rx_frag_h_mpdu(dp_pdev, msdu, desc, &rx_info)) {
+		dev_kfree_skb_any(msdu);
+		ath12k_wifi7_dp_rx_link_desc_return(dp, &desc->buf_addr_info,
+						    HAL_WBM_REL_BM_ACT_PUT_IN_IDLE);
+	}
+exit:
+	rcu_read_unlock();
+	return 0;
+}
+
+static int ath12k_dp_h_msdu_buffer_type(struct ath12k_dp *dp,
+					struct list_head *list,
+					struct hal_reo_dest_ring *desc)
+{
+	struct ath12k_rx_desc_info *desc_info;
+	struct ath12k_skb_rxcb *rxcb;
+	struct sk_buff *msdu;
+	u64 desc_va;
+
+	dp->device_stats.reo_excep_msdu_buf_type++;
+
+	desc_va = (u64)le32_to_cpu(desc->buf_va_hi) << 32 |
+		  le32_to_cpu(desc->buf_va_lo);
+	desc_info = (struct ath12k_rx_desc_info *)(uintptr_t)desc_va;
+	if (!desc_info) {
+		u32 cookie;
+
+		cookie = le32_get_bits(desc->buf_addr_info.info1,
+				       BUFFER_ADDR_INFO1_SW_COOKIE);
+		desc_info = ath12k_dp_get_rx_desc(dp, cookie);
+		if (!desc_info) {
+			ath12k_warn(dp->ab, "Invalid cookie in manual descriptor retrieval: 0x%x\n",
+				    cookie);
+			return -EINVAL;
+		}
+	}
+
+	if (desc_info->magic != ATH12K_DP_RX_DESC_MAGIC) {
+		ath12k_warn(dp->ab, "rx exception, magic check failed with value: %u\n",
+			    desc_info->magic);
+		return -EINVAL;
+	}
+
+	msdu = desc_info->skb;
+	desc_info->skb = NULL;
+	list_add_tail(&desc_info->list, list);
+	rxcb = ATH12K_SKB_RXCB(msdu);
+	dma_unmap_single(dp->dev, rxcb->paddr, msdu->len + skb_tailroom(msdu),
+			 DMA_FROM_DEVICE);
+	dev_kfree_skb_any(msdu);
+
+	return 0;
+}
+
+int ath12k_wifi7_dp_rx_process_err(struct ath12k_dp *dp, struct napi_struct *napi,
+				   int budget)
+{
+	struct ath12k_base *ab = dp->ab;
+	struct ath12k_hal *hal = dp->hal;
+	struct ath12k_hw_group *ag = dp->ag;
+	struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp;
+	struct ath12k_dp *partner_dp;
+	struct list_head rx_desc_used_list[ATH12K_MAX_DEVICES];
+	u32 msdu_cookies[HAL_NUM_RX_MSDUS_PER_LINK_DESC];
+	int num_buffs_reaped[ATH12K_MAX_DEVICES] = {};
+	struct dp_link_desc_bank *link_desc_banks;
+	enum hal_rx_buf_return_buf_manager rbm;
+	struct hal_rx_msdu_link *link_desc_va;
+	int tot_n_bufs_reaped, quota, ret, i;
+	struct hal_reo_dest_ring *reo_desc;
+	struct dp_rxdma_ring *rx_ring;
+	struct dp_srng *reo_except;
+	struct ath12k_hw_link *hw_links = ag->hw_links;
+	struct ath12k_pdev_dp *dp_pdev;
+	u8 hw_link_id, device_id;
+	u32 desc_bank, num_msdus;
+	struct hal_srng *srng;
+	dma_addr_t paddr;
+	bool is_frag;
+	bool drop;
+	int pdev_idx;
+	struct list_head *used_list;
+	enum hal_wbm_rel_bm_act act;
+
+	tot_n_bufs_reaped = 0;
+	quota = budget;
+
+	for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++)
+		INIT_LIST_HEAD(&rx_desc_used_list[device_id]);
+
+	reo_except = &dp->reo_except_ring;
+
+	srng = &hal->srng_list[reo_except->ring_id];
+
+	spin_lock_bh(&srng->lock);
+
+	ath12k_hal_srng_access_begin(ab, srng);
+
+	while (budget &&
+	       (reo_desc = ath12k_hal_srng_dst_get_next_entry(ab, srng))) {
+		drop = false;
+		dp->device_stats.err_ring_pkts++;
+
+		hw_link_id = le32_get_bits(reo_desc->info0,
+					   HAL_REO_DEST_RING_INFO0_SRC_LINK_ID);
+		device_id = hw_links[hw_link_id].device_id;
+		partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp, device_id);
+
+		/* Below case is added to handle data packet from un-associated clients.
+		 * As it is expected that AST lookup will fail for
+		 * un-associated station's data packets.
+		 */
+		if (le32_get_bits(reo_desc->info0, HAL_REO_DEST_RING_INFO0_BUFFER_TYPE) ==
+		    HAL_REO_DEST_RING_BUFFER_TYPE_MSDU) {
+			if (!ath12k_dp_h_msdu_buffer_type(partner_dp,
+							  &rx_desc_used_list[device_id],
+							  reo_desc)) {
+				num_buffs_reaped[device_id]++;
+				tot_n_bufs_reaped++;
+			}
+			goto next_desc;
+		}
+
+		ret = ath12k_wifi7_hal_desc_reo_parse_err(dp, reo_desc, &paddr,
+							  &desc_bank);
+		if (ret) {
+			ath12k_warn(ab, "failed to parse error reo desc %d\n",
+				    ret);
+			continue;
+		}
+
+		pdev_idx = ath12k_hw_mac_id_to_pdev_id(partner_dp->hw_params,
+						       hw_links[hw_link_id].pdev_idx);
+
+		link_desc_banks = partner_dp->link_desc_banks;
+		link_desc_va = link_desc_banks[desc_bank].vaddr +
+			       (paddr - link_desc_banks[desc_bank].paddr);
+		ath12k_wifi7_hal_rx_msdu_link_info_get(link_desc_va, &num_msdus,
+						       msdu_cookies, &rbm);
+		if (rbm != partner_dp->idle_link_rbm &&
+		    rbm != HAL_RX_BUF_RBM_SW3_BM &&
+		    rbm != partner_dp->hal->hal_params->rx_buf_rbm) {
+			act = HAL_WBM_REL_BM_ACT_REL_MSDU;
+			dp->device_stats.invalid_rbm++;
+			ath12k_warn(ab, "invalid return buffer manager %d\n", rbm);
+			ath12k_wifi7_dp_rx_link_desc_return(partner_dp,
+							    &reo_desc->buf_addr_info,
+							    act);
+			continue;
+		}
+
+		is_frag = !!(le32_to_cpu(reo_desc->rx_mpdu_info.info0) &
+			     RX_MPDU_DESC_INFO0_FRAG_FLAG);
+
+		/* Process only rx fragments with one msdu per link desc below, and drop
+		 * msdu's indicated due to error reasons.
+		 * Dynamic fragmentation not supported in Multi-link client, so drop the
+		 * partner device buffers.
+		 */
+		if (!is_frag || num_msdus > 1 ||
+		    partner_dp->device_id != dp->device_id) {
+			drop = true;
+			act = HAL_WBM_REL_BM_ACT_PUT_IN_IDLE;
+
+			/* Return the link desc back to wbm idle list */
+			ath12k_wifi7_dp_rx_link_desc_return(partner_dp,
+							    &reo_desc->buf_addr_info,
+							    act);
+		}
+
+		rcu_read_lock();
+
+		dp_pdev = ath12k_dp_to_pdev_dp(dp, pdev_idx);
+		if (!dp_pdev) {
+			rcu_read_unlock();
+			continue;
+		}
+
+		for (i = 0; i < num_msdus; i++) {
+			used_list = &rx_desc_used_list[device_id];
+
+			if (!ath12k_wifi7_dp_process_rx_err_buf(dp_pdev, reo_desc,
+								used_list,
+								drop,
+								msdu_cookies[i])) {
+				num_buffs_reaped[device_id]++;
+				tot_n_bufs_reaped++;
+			}
+		}
+
+		rcu_read_unlock();
+
+next_desc:
+		if (tot_n_bufs_reaped >= quota) {
+			tot_n_bufs_reaped = quota;
+			goto exit;
+		}
+
+		budget = quota - tot_n_bufs_reaped;
+	}
+
+exit:
+	ath12k_hal_srng_access_end(ab, srng);
+
+	spin_unlock_bh(&srng->lock);
+
+	for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) {
+		if (!num_buffs_reaped[device_id])
+			continue;
+
+		partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp, device_id);
+		rx_ring = &partner_dp->rx_refill_buf_ring;
+
+		ath12k_dp_rx_bufs_replenish(partner_dp, rx_ring,
+					    &rx_desc_used_list[device_id],
+					    num_buffs_reaped[device_id]);
+	}
+
+	return tot_n_bufs_reaped;
+}
+
+static void
+ath12k_wifi7_dp_rx_null_q_desc_sg_drop(struct ath12k_dp *dp, int msdu_len,
+				       struct sk_buff_head *msdu_list)
+{
+	struct sk_buff *skb, *tmp;
+	struct ath12k_skb_rxcb *rxcb;
+	int n_buffs;
+
+	n_buffs = DIV_ROUND_UP(msdu_len,
+			       (DP_RX_BUFFER_SIZE - dp->ab->hal.hal_desc_sz));
+
+	skb_queue_walk_safe(msdu_list, skb, tmp) {
+		rxcb = ATH12K_SKB_RXCB(skb);
+		if (rxcb->err_rel_src == HAL_WBM_REL_SRC_MODULE_REO &&
+		    rxcb->err_code == HAL_REO_DEST_RING_ERROR_CODE_DESC_ADDR_ZERO) {
+			if (!n_buffs)
+				break;
+			__skb_unlink(skb, msdu_list);
+			dev_kfree_skb_any(skb);
+			n_buffs--;
+		}
+	}
+}
+
+static int ath12k_wifi7_dp_rx_h_null_q_desc(struct ath12k_pdev_dp *dp_pdev,
+					    struct sk_buff *msdu,
+					    struct hal_rx_desc_data *rx_info,
+					    struct sk_buff_head *msdu_list)
+{
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_base *ab = dp->ab;
+	u16 msdu_len = rx_info->msdu_len;
+	struct hal_rx_desc *desc = (struct hal_rx_desc *)msdu->data;
+	u8 l3pad_bytes = rx_info->l3_pad_bytes;
+	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
+	u32 hal_rx_desc_sz = dp->ab->hal.hal_desc_sz;
+
+	if (!rxcb->is_frag && ((msdu_len + hal_rx_desc_sz) > DP_RX_BUFFER_SIZE)) {
+		/* First buffer will be freed by the caller, so deduct it's length */
+		msdu_len = msdu_len - (DP_RX_BUFFER_SIZE - hal_rx_desc_sz);
+		ath12k_wifi7_dp_rx_null_q_desc_sg_drop(dp, msdu_len, msdu_list);
+		return -EINVAL;
+	}
+
+	/* Even after cleaning up the sg buffers in the msdu list with above check
+	 * any msdu received with continuation flag needs to be dropped as invalid.
+	 * This protects against some random err frame with continuation flag.
+	 */
+	if (rxcb->is_continuation)
+		return -EINVAL;
+
+	if (!rx_info->msdu_done) {
+		ath12k_warn(ab,
+			    "msdu_done bit not set in null_q_des processing\n");
+		__skb_queue_purge(msdu_list);
+		return -EIO;
+	}
+
+	/* Handle NULL queue descriptor violations arising out a missing
+	 * REO queue for a given peer or a given TID. This typically
+	 * may happen if a packet is received on a QOS enabled TID before the
+	 * ADDBA negotiation for that TID, when the TID queue is setup. Or
+	 * it may also happen for MC/BC frames if they are not routed to the
+	 * non-QOS TID queue, in the absence of any other default TID queue.
+	 * This error can show up both in a REO destination or WBM release ring.
+	 */
+
+	if (rxcb->is_frag) {
+		skb_pull(msdu, hal_rx_desc_sz);
+	} else {
+		if ((hal_rx_desc_sz + l3pad_bytes + msdu_len) > DP_RX_BUFFER_SIZE)
+			return -EINVAL;
+
+		skb_put(msdu, hal_rx_desc_sz + l3pad_bytes + msdu_len);
+		skb_pull(msdu, hal_rx_desc_sz + l3pad_bytes);
+	}
+	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, desc, msdu, rx_info)))
+		return -EINVAL;
+
+	ath12k_dp_rx_h_ppdu(dp_pdev, rx_info);
+	ath12k_wifi7_dp_rx_h_mpdu(dp_pdev, msdu, desc, rx_info);
+
+	rxcb->tid = rx_info->tid;
+
+	/* Please note that caller will having the access to msdu and completing
+	 * rx with mac80211. Need not worry about cleaning up amsdu_list.
+	 */
+
+	return 0;
+}
+
+static bool ath12k_wifi7_dp_rx_h_tkip_mic_err(struct ath12k_pdev_dp *dp_pdev,
+					      struct sk_buff *msdu,
+					      struct hal_rx_desc_data *rx_info)
+{
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_base *ab = dp->ab;
+	u16 msdu_len = rx_info->msdu_len;
+	struct hal_rx_desc *desc = (struct hal_rx_desc *)msdu->data;
+	u8 l3pad_bytes = rx_info->l3_pad_bytes;
+	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
+	u32 hal_rx_desc_sz = ab->hal.hal_desc_sz;
+
+	rxcb->is_first_msdu = rx_info->is_first_msdu;
+	rxcb->is_last_msdu = rx_info->is_last_msdu;
+
+	if ((hal_rx_desc_sz + l3pad_bytes + msdu_len) > DP_RX_BUFFER_SIZE) {
+		ath12k_dbg(ab, ATH12K_DBG_DATA,
+			   "invalid msdu len in tkip mic err %u\n", msdu_len);
+		ath12k_dbg_dump(ab, ATH12K_DBG_DATA, NULL, "", desc,
+				sizeof(*desc));
+		return true;
+	}
+
+	skb_put(msdu, hal_rx_desc_sz + l3pad_bytes + msdu_len);
+	skb_pull(msdu, hal_rx_desc_sz + l3pad_bytes);
+
+	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, desc, msdu, rx_info)))
+		return true;
+
+	ath12k_dp_rx_h_ppdu(dp_pdev, rx_info);
+
+	rx_info->rx_status->flag |= (RX_FLAG_MMIC_STRIPPED | RX_FLAG_MMIC_ERROR |
+				     RX_FLAG_DECRYPTED);
+
+	ath12k_dp_rx_h_undecap(dp_pdev, msdu, desc,
+			       HAL_ENCRYPT_TYPE_TKIP_MIC, false, rx_info);
+	return false;
+}
+
+static bool ath12k_wifi7_dp_rx_h_rxdma_err(struct ath12k_pdev_dp *dp_pdev,
+					   struct sk_buff *msdu,
+					   struct hal_rx_desc_data *rx_info)
+{
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
+	bool drop = false;
+
+	dp->device_stats.rxdma_error[rxcb->err_code]++;
+
+	switch (rxcb->err_code) {
+	case HAL_REO_ENTR_RING_RXDMA_ECODE_DECRYPT_ERR:
+	case HAL_REO_ENTR_RING_RXDMA_ECODE_TKIP_MIC_ERR:
+		if (rx_info->err_bitmap & HAL_RX_MPDU_ERR_TKIP_MIC) {
+			drop = ath12k_wifi7_dp_rx_h_tkip_mic_err(dp_pdev, msdu, rx_info);
+			break;
+		}
+		fallthrough;
+	default:
+		/* TODO: Review other rxdma error code to check if anything is
+		 * worth reporting to mac80211
+		 */
+		drop = true;
+		break;
+	}
+
+	return drop;
+}
+
+static bool ath12k_wifi7_dp_rx_h_reo_err(struct ath12k_pdev_dp *dp_pdev,
+					 struct sk_buff *msdu,
+					 struct hal_rx_desc_data *rx_info,
+					 struct sk_buff_head *msdu_list)
+{
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
+	bool drop = false;
+
+	dp->device_stats.reo_error[rxcb->err_code]++;
+
+	switch (rxcb->err_code) {
+	case HAL_REO_DEST_RING_ERROR_CODE_DESC_ADDR_ZERO:
+		if (ath12k_wifi7_dp_rx_h_null_q_desc(dp_pdev, msdu, rx_info, msdu_list))
+			drop = true;
+		break;
+	case HAL_REO_DEST_RING_ERROR_CODE_PN_CHECK_FAILED:
+		/* TODO: Do not drop PN failed packets in the driver;
+		 * instead, it is good to drop such packets in mac80211
+		 * after incrementing the replay counters.
+		 */
+		fallthrough;
+	default:
+		/* TODO: Review other errors and process them to mac80211
+		 * as appropriate.
+		 */
+		drop = true;
+		break;
+	}
+
+	return drop;
+}
+
+static void ath12k_wifi7_dp_rx_wbm_err(struct ath12k_pdev_dp *dp_pdev,
+				       struct napi_struct *napi,
+				       struct sk_buff *msdu,
+				       struct sk_buff_head *msdu_list)
+{
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct hal_rx_desc *rx_desc = (struct hal_rx_desc *)msdu->data;
+	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
+	struct ieee80211_rx_status rxs = {};
+	struct hal_rx_desc_data rx_info;
+	bool drop = true;
+
+	rx_info.addr2_present = false;
+	rx_info.rx_status = &rxs;
+
+	ath12k_dp_extract_rx_desc_data(dp->hal, &rx_info, rx_desc, rx_desc);
+
+	switch (rxcb->err_rel_src) {
+	case HAL_WBM_REL_SRC_MODULE_REO:
+		drop = ath12k_wifi7_dp_rx_h_reo_err(dp_pdev, msdu, &rx_info, msdu_list);
+		break;
+	case HAL_WBM_REL_SRC_MODULE_RXDMA:
+		drop = ath12k_wifi7_dp_rx_h_rxdma_err(dp_pdev, msdu, &rx_info);
+		break;
+	default:
+		/* msdu will get freed */
+		break;
+	}
+
+	if (drop) {
+		dev_kfree_skb_any(msdu);
+		return;
+	}
+
+	rx_info.rx_status->flag |= RX_FLAG_SKIP_MONITOR;
+
+	ath12k_dp_rx_deliver_msdu(dp_pdev, napi, msdu, &rx_info);
+}
+
+void ath12k_wifi7_dp_setup_pn_check_reo_cmd(struct ath12k_hal_reo_cmd *cmd,
+					    struct ath12k_dp_rx_tid *rx_tid,
+					    u32 cipher, enum set_key_cmd key_cmd)
+{
+	cmd->flag = HAL_REO_CMD_FLG_NEED_STATUS;
+	cmd->upd0 = HAL_REO_CMD_UPD0_PN |
+			HAL_REO_CMD_UPD0_PN_SIZE |
+			HAL_REO_CMD_UPD0_PN_VALID |
+			HAL_REO_CMD_UPD0_PN_CHECK |
+			HAL_REO_CMD_UPD0_SVLD;
+
+	switch (cipher) {
+	case WLAN_CIPHER_SUITE_TKIP:
+	case WLAN_CIPHER_SUITE_CCMP:
+	case WLAN_CIPHER_SUITE_CCMP_256:
+	case WLAN_CIPHER_SUITE_GCMP:
+	case WLAN_CIPHER_SUITE_GCMP_256:
+		if (key_cmd == SET_KEY) {
+			cmd->upd1 |= HAL_REO_CMD_UPD1_PN_CHECK;
+			cmd->pn_size = 48;
+		}
+		break;
+	default:
+		break;
+	}
+
+	cmd->addr_lo = lower_32_bits(rx_tid->qbuf.paddr_aligned);
+	cmd->addr_hi = upper_32_bits(rx_tid->qbuf.paddr_aligned);
+}
+
+int ath12k_wifi7_dp_rx_process_wbm_err(struct ath12k_dp *dp,
+				       struct napi_struct *napi, int budget)
+{
+	struct list_head rx_desc_used_list[ATH12K_MAX_DEVICES];
+	struct ath12k_base *ab = dp->ab;
+	struct ath12k_hal *hal = dp->hal;
+	struct ath12k *ar;
+	struct ath12k_pdev_dp *dp_pdev;
+	struct ath12k_hw_group *ag = dp->ag;
+	struct ath12k_dp_hw_group *dp_hw_grp = &ag->dp_hw_grp;
+	struct ath12k_dp *partner_dp;
+	struct dp_rxdma_ring *rx_ring;
+	struct hal_rx_wbm_rel_info err_info;
+	struct hal_srng *srng;
+	struct sk_buff *msdu;
+	struct sk_buff_head msdu_list, scatter_msdu_list;
+	struct ath12k_skb_rxcb *rxcb;
+	void *rx_desc;
+	int num_buffs_reaped[ATH12K_MAX_DEVICES] = {};
+	int total_num_buffs_reaped = 0;
+	struct ath12k_rx_desc_info *desc_info;
+	struct ath12k_device_dp_stats *device_stats = &dp->device_stats;
+	struct ath12k_hw_link *hw_links = ag->hw_links;
+	u8 hw_link_id, device_id;
+	int ret, pdev_idx;
+	struct hal_rx_desc *msdu_data;
+
+	__skb_queue_head_init(&msdu_list);
+	__skb_queue_head_init(&scatter_msdu_list);
+
+	for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++)
+		INIT_LIST_HEAD(&rx_desc_used_list[device_id]);
+
+	srng = &hal->srng_list[dp->rx_rel_ring.ring_id];
+	spin_lock_bh(&srng->lock);
+
+	ath12k_hal_srng_access_begin(ab, srng);
+
+	while (budget) {
+		rx_desc = ath12k_hal_srng_dst_get_next_entry(ab, srng);
+		if (!rx_desc)
+			break;
+
+		ret = ath12k_wifi7_hal_wbm_desc_parse_err(dp, rx_desc,
+							  &err_info);
+		if (ret) {
+			ath12k_warn(ab, "failed to parse rx error in wbm_rel ring desc %d\n",
+				    ret);
+			continue;
+		}
+
+		desc_info = err_info.rx_desc;
+
+		/* retry manual desc retrieval if hw cc is not done */
+		if (!desc_info) {
+			desc_info = ath12k_dp_get_rx_desc(dp, err_info.cookie);
+			if (!desc_info) {
+				ath12k_warn(ab, "Invalid cookie in DP WBM rx error descriptor retrieval: 0x%x\n",
+					    err_info.cookie);
+				continue;
+			}
+		}
+
+		if (desc_info->magic != ATH12K_DP_RX_DESC_MAGIC)
+			ath12k_warn(ab, "WBM RX err, Check HW CC implementation");
+
+		msdu = desc_info->skb;
+		desc_info->skb = NULL;
+
+		device_id = desc_info->device_id;
+		partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp, device_id);
+		if (unlikely(!partner_dp)) {
+			dev_kfree_skb_any(msdu);
+
+			/* In any case continuation bit is set
+			 * in the previous record, cleanup scatter_msdu_list
+			 */
+			ath12k_dp_clean_up_skb_list(&scatter_msdu_list);
+			continue;
+		}
+
+		list_add_tail(&desc_info->list, &rx_desc_used_list[device_id]);
+
+		rxcb = ATH12K_SKB_RXCB(msdu);
+		dma_unmap_single(partner_dp->dev, rxcb->paddr,
+				 msdu->len + skb_tailroom(msdu),
+				 DMA_FROM_DEVICE);
+
+		num_buffs_reaped[device_id]++;
+		total_num_buffs_reaped++;
+
+		if (!err_info.continuation)
+			budget--;
+
+		if (err_info.push_reason !=
+		    HAL_REO_DEST_RING_PUSH_REASON_ERR_DETECTED) {
+			dev_kfree_skb_any(msdu);
+			continue;
+		}
+
+		msdu_data = (struct hal_rx_desc *)msdu->data;
+		rxcb->err_rel_src = err_info.err_rel_src;
+		rxcb->err_code = err_info.err_code;
+		rxcb->is_first_msdu = err_info.first_msdu;
+		rxcb->is_last_msdu = err_info.last_msdu;
+		rxcb->is_continuation = err_info.continuation;
+		rxcb->rx_desc = msdu_data;
+		rxcb->peer_id = ath12k_wifi7_dp_rx_get_peer_id(dp, dp->peer_metadata_ver,
+							       err_info.peer_metadata);
+
+		if (err_info.continuation) {
+			__skb_queue_tail(&scatter_msdu_list, msdu);
+			continue;
+		}
+
+		hw_link_id = ath12k_dp_rx_get_msdu_src_link(partner_dp->hal,
+							    msdu_data);
+		if (hw_link_id >= ATH12K_GROUP_MAX_RADIO) {
+			dev_kfree_skb_any(msdu);
+
+			/* In any case continuation bit is set
+			 * in the previous record, cleanup scatter_msdu_list
+			 */
+			ath12k_dp_clean_up_skb_list(&scatter_msdu_list);
+			continue;
+		}
+
+		if (!skb_queue_empty(&scatter_msdu_list)) {
+			struct sk_buff *msdu;
+
+			skb_queue_walk(&scatter_msdu_list, msdu) {
+				rxcb = ATH12K_SKB_RXCB(msdu);
+				rxcb->hw_link_id = hw_link_id;
+			}
+
+			skb_queue_splice_tail_init(&scatter_msdu_list,
+						   &msdu_list);
+		}
+
+		rxcb = ATH12K_SKB_RXCB(msdu);
+		rxcb->hw_link_id = hw_link_id;
+		__skb_queue_tail(&msdu_list, msdu);
+	}
+
+	/* In any case continuation bit is set in the
+	 * last record, cleanup scatter_msdu_list
+	 */
+	ath12k_dp_clean_up_skb_list(&scatter_msdu_list);
+
+	ath12k_hal_srng_access_end(ab, srng);
+
+	spin_unlock_bh(&srng->lock);
+
+	if (!total_num_buffs_reaped)
+		goto done;
+
+	for (device_id = 0; device_id < ATH12K_MAX_DEVICES; device_id++) {
+		if (!num_buffs_reaped[device_id])
+			continue;
+
+		partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp, device_id);
+		rx_ring = &partner_dp->rx_refill_buf_ring;
+
+		ath12k_dp_rx_bufs_replenish(dp, rx_ring,
+					    &rx_desc_used_list[device_id],
+					    num_buffs_reaped[device_id]);
+	}
+
+	rcu_read_lock();
+	while ((msdu = __skb_dequeue(&msdu_list))) {
+		rxcb = ATH12K_SKB_RXCB(msdu);
+		hw_link_id = rxcb->hw_link_id;
+
+		device_id = hw_links[hw_link_id].device_id;
+		partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp, device_id);
+		if (unlikely(!partner_dp)) {
+			ath12k_dbg(ab, ATH12K_DBG_DATA,
+				   "Unable to process WBM error msdu due to invalid hw link id %d device id %d\n",
+				   hw_link_id, device_id);
+			dev_kfree_skb_any(msdu);
+			continue;
+		}
+
+		pdev_idx = ath12k_hw_mac_id_to_pdev_id(partner_dp->hw_params,
+						       hw_links[hw_link_id].pdev_idx);
+
+		dp_pdev = ath12k_dp_to_pdev_dp(partner_dp, pdev_idx);
+		if (!dp_pdev) {
+			dev_kfree_skb_any(msdu);
+			continue;
+		}
+		ar = ath12k_pdev_dp_to_ar(dp_pdev);
+
+		if (!ar || !rcu_dereference(ar->ab->pdevs_active[pdev_idx])) {
+			dev_kfree_skb_any(msdu);
+			continue;
+		}
+
+		if (test_bit(ATH12K_FLAG_CAC_RUNNING, &ar->dev_flags)) {
+			dev_kfree_skb_any(msdu);
+			continue;
+		}
+
+		if (rxcb->err_rel_src < HAL_WBM_REL_SRC_MODULE_MAX) {
+			device_id = dp_pdev->dp->device_id;
+			device_stats->rx_wbm_rel_source[rxcb->err_rel_src][device_id]++;
+		}
+
+		ath12k_wifi7_dp_rx_wbm_err(dp_pdev, napi, msdu, &msdu_list);
+	}
+	rcu_read_unlock();
+done:
+	return total_num_buffs_reaped;
+}
+
+int ath12k_dp_rxdma_ring_sel_config_qcn9274(struct ath12k_base *ab)
+{
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct htt_rx_ring_tlv_filter tlv_filter = {};
+	u32 ring_id;
+	int ret;
+	u32 hal_rx_desc_sz = ab->hal.hal_desc_sz;
+
+	ring_id = dp->rx_refill_buf_ring.refill_buf_ring.ring_id;
+
+	tlv_filter.rx_filter = HTT_RX_TLV_FLAGS_RXDMA_RING;
+	tlv_filter.pkt_filter_flags2 = HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR;
+	tlv_filter.pkt_filter_flags3 = HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST |
+					HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST |
+					HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA;
+	tlv_filter.offset_valid = true;
+	tlv_filter.rx_packet_offset = hal_rx_desc_sz;
+
+	tlv_filter.rx_mpdu_start_offset =
+		ath12k_hal_rx_desc_get_mpdu_start_offset_qcn9274();
+	tlv_filter.rx_msdu_end_offset =
+		ath12k_hal_rx_desc_get_msdu_end_offset_qcn9274();
+
+	tlv_filter.rx_mpdu_start_wmask = ath12k_hal_rx_mpdu_start_wmask_get_qcn9274();
+	tlv_filter.rx_msdu_end_wmask = ath12k_hal_rx_msdu_end_wmask_get_qcn9274();
+	ath12k_dbg(ab, ATH12K_DBG_DATA,
+		   "Configuring compact tlv masks rx_mpdu_start_wmask 0x%x rx_msdu_end_wmask 0x%x\n",
+		   tlv_filter.rx_mpdu_start_wmask, tlv_filter.rx_msdu_end_wmask);
+
+	ret = ath12k_dp_tx_htt_rx_filter_setup(ab, ring_id, 0,
+					       HAL_RXDMA_BUF,
+					       DP_RXDMA_REFILL_RING_SIZE,
+					       &tlv_filter);
+
+	return ret;
+}
+
+int ath12k_dp_rxdma_ring_sel_config_wcn7850(struct ath12k_base *ab)
+{
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct htt_rx_ring_tlv_filter tlv_filter = {};
+	u32 ring_id;
+	int ret = 0;
+	u32 hal_rx_desc_sz = ab->hal.hal_desc_sz;
+	int i;
+
+	ring_id = dp->rx_refill_buf_ring.refill_buf_ring.ring_id;
+
+	tlv_filter.rx_filter = HTT_RX_TLV_FLAGS_RXDMA_RING;
+	tlv_filter.pkt_filter_flags2 = HTT_RX_FP_CTRL_PKT_FILTER_TLV_FLAGS2_BAR;
+	tlv_filter.pkt_filter_flags3 = HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_MCAST |
+					HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_UCAST |
+					HTT_RX_FP_DATA_PKT_FILTER_TLV_FLASG3_NULL_DATA;
+	tlv_filter.offset_valid = true;
+	tlv_filter.rx_packet_offset = hal_rx_desc_sz;
+
+	tlv_filter.rx_header_offset = offsetof(struct hal_rx_desc_wcn7850, pkt_hdr_tlv);
+
+	tlv_filter.rx_mpdu_start_offset =
+		ath12k_hal_rx_desc_get_mpdu_start_offset_wcn7850();
+	tlv_filter.rx_msdu_end_offset =
+		ath12k_hal_rx_desc_get_msdu_end_offset_wcn7850();
+
+	/* TODO: Selectively subscribe to required qwords within msdu_end
+	 * and mpdu_start and setup the mask in below msg
+	 * and modify the rx_desc struct
+	 */
+
+	for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) {
+		ring_id = dp->rx_mac_buf_ring[i].ring_id;
+		ret = ath12k_dp_tx_htt_rx_filter_setup(ab, ring_id, i,
+						       HAL_RXDMA_BUF,
+						       DP_RXDMA_REFILL_RING_SIZE,
+						       &tlv_filter);
+	}
+
+	return ret;
+}
+
+void ath12k_wifi7_dp_rx_process_reo_status(struct ath12k_dp *dp)
+{
+	struct ath12k_base *ab = dp->ab;
+	struct ath12k_hal *hal = dp->hal;
+	struct hal_tlv_64_hdr *hdr;
+	struct hal_srng *srng;
+	struct ath12k_dp_rx_reo_cmd *cmd, *tmp;
+	bool found = false;
+	u16 tag;
+	struct hal_reo_status reo_status;
+
+	srng = &hal->srng_list[dp->reo_status_ring.ring_id];
+
+	memset(&reo_status, 0, sizeof(reo_status));
+
+	spin_lock_bh(&srng->lock);
+
+	ath12k_hal_srng_access_begin(ab, srng);
+
+	while ((hdr = ath12k_hal_srng_dst_get_next_entry(ab, srng))) {
+		tag = le64_get_bits(hdr->tl, HAL_SRNG_TLV_HDR_TAG);
+
+		switch (tag) {
+		case HAL_REO_GET_QUEUE_STATS_STATUS:
+			ath12k_wifi7_hal_reo_status_queue_stats(ab, hdr,
+								&reo_status);
+			break;
+		case HAL_REO_FLUSH_QUEUE_STATUS:
+			ath12k_wifi7_hal_reo_flush_queue_status(ab, hdr,
+								&reo_status);
+			break;
+		case HAL_REO_FLUSH_CACHE_STATUS:
+			ath12k_wifi7_hal_reo_flush_cache_status(ab, hdr,
+								&reo_status);
+			break;
+		case HAL_REO_UNBLOCK_CACHE_STATUS:
+			ath12k_wifi7_hal_reo_unblk_cache_status(ab, hdr,
+								&reo_status);
+			break;
+		case HAL_REO_FLUSH_TIMEOUT_LIST_STATUS:
+			ath12k_wifi7_hal_reo_flush_timeout_list_status(ab, hdr,
+								       &reo_status);
+			break;
+		case HAL_REO_DESCRIPTOR_THRESHOLD_REACHED_STATUS:
+			ath12k_wifi7_hal_reo_desc_thresh_reached_status(ab, hdr,
+									&reo_status);
+			break;
+		case HAL_REO_UPDATE_RX_REO_QUEUE_STATUS:
+			ath12k_wifi7_hal_reo_update_rx_reo_queue_status(ab, hdr,
+									&reo_status);
+			break;
+		default:
+			ath12k_warn(ab, "Unknown reo status type %d\n", tag);
+			continue;
+		}
+
+		spin_lock_bh(&dp->reo_cmd_lock);
+		list_for_each_entry_safe(cmd, tmp, &dp->reo_cmd_list, list) {
+			if (reo_status.uniform_hdr.cmd_num == cmd->cmd_num) {
+				found = true;
+				list_del(&cmd->list);
+				break;
+			}
+		}
+		spin_unlock_bh(&dp->reo_cmd_lock);
+
+		if (found) {
+			cmd->handler(dp, (void *)&cmd->data,
+				     reo_status.uniform_hdr.cmd_status);
+			kfree(cmd);
+		}
+
+		found = false;
+	}
+
+	ath12k_hal_srng_access_end(ab, srng);
+
+	spin_unlock_bh(&srng->lock);
+}
+
+bool
+ath12k_wifi7_dp_rxdesc_mpdu_valid(struct ath12k_base *ab,
+				  struct hal_rx_desc *rx_desc)
+{
+	u32 tlv_tag;
+
+	tlv_tag = ab->hal.ops->rx_desc_get_mpdu_start_tag(rx_desc);
+
+	return tlv_tag == HAL_RX_MPDU_START;
+}
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h
new file mode 100644
index 000000000000..a98836b83f48
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+#ifndef ATH12K_DP_RX_WIFI7_H
+#define ATH12K_DP_RX_WIFI7_H
+
+#include "../core.h"
+#include "../dp_rx.h"
+#include "hal_rx_desc.h"
+
+struct ath12k_hal_reo_cmd;
+
+int ath12k_wifi7_dp_rx_process_wbm_err(struct ath12k_dp *dp,
+				       struct napi_struct *napi, int budget);
+int ath12k_wifi7_dp_rx_process_err(struct ath12k_dp *dp, struct napi_struct *napi,
+				   int budget);
+int ath12k_wifi7_dp_rx_process(struct ath12k_dp *dp, int mac_id,
+			       struct napi_struct *napi,
+			       int budget);
+void ath12k_wifi7_dp_rx_process_reo_status(struct ath12k_dp *dp);
+int ath12k_dp_rxdma_ring_sel_config_qcn9274(struct ath12k_base *ab);
+int ath12k_dp_rxdma_ring_sel_config_wcn7850(struct ath12k_base *ab);
+void ath12k_wifi7_dp_setup_pn_check_reo_cmd(struct ath12k_hal_reo_cmd *cmd,
+					    struct ath12k_dp_rx_tid *rx_tid,
+					    u32 cipher, enum set_key_cmd key_cmd);
+int ath12k_wifi7_dp_rx_assign_reoq(struct ath12k_base *ab, struct ath12k_dp_peer *dp_peer,
+				   struct ath12k_dp_rx_tid *rx_tid,
+				   u16 ssn, enum hal_pn_type pn_type);
+int ath12k_wifi7_dp_rx_link_desc_return(struct ath12k_dp *dp,
+					struct ath12k_buffer_addr *buf_addr_info,
+					enum hal_wbm_rel_bm_act action);
+void ath12k_wifi7_dp_rx_frags_cleanup(struct ath12k_dp_rx_tid *rx_tid,
+				      bool rel_link_desc);
+void ath12k_wifi7_peer_rx_tid_qref_setup(struct ath12k_base *ab, u16 peer_id, u16 tid,
+					 dma_addr_t paddr);
+void ath12k_wifi7_dp_rx_peer_tid_delete(struct ath12k_base *ab,
+					struct ath12k_dp_link_peer *peer, u8 tid);
+int ath12k_wifi7_dp_reo_cmd_send(struct ath12k_base *ab,
+				 struct ath12k_dp_rx_tid_rxq *rx_tid,
+				 enum hal_reo_cmd_type type,
+				 struct ath12k_hal_reo_cmd *cmd,
+				 void (*cb)(struct ath12k_dp *dp, void *ctx,
+					    enum hal_reo_cmd_status status));
+int ath12k_wifi7_dp_reo_cache_flush(struct ath12k_base *ab,
+				    struct ath12k_dp_rx_tid_rxq *rx_tid);
+int ath12k_wifi7_peer_rx_tid_reo_update(struct ath12k_dp *dp,
+					struct ath12k_dp_link_peer *peer,
+					struct ath12k_dp_rx_tid *rx_tid,
+					u32 ba_win_sz, u16 ssn,
+					bool update_ssn);
+void ath12k_wifi7_peer_rx_tid_qref_reset(struct ath12k_base *ab, u16 peer_id, u16 tid);
+bool
+ath12k_wifi7_dp_rxdesc_mpdu_valid(struct ath12k_base *ab,
+				  struct hal_rx_desc *rx_desc);
+int ath12k_wifi7_dp_rx_tid_delete_handler(struct ath12k_base *ab,
+					  struct ath12k_dp_rx_tid_rxq *rx_tid);
+#endif
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c
new file mode 100644
index 000000000000..629084aa36d8
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c
@@ -0,0 +1,978 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include "../core.h"
+#include "../debug.h"
+#include "../dp_tx.h"
+#include "../peer.h"
+#include "dp_tx.h"
+#include "hal_desc.h"
+#include "hal.h"
+#include "hal_tx.h"
+
+static void
+ath12k_wifi7_hal_tx_cmd_ext_desc_setup(struct ath12k_base *ab,
+				       struct hal_tx_msdu_ext_desc *tcl_ext_cmd,
+				       struct hal_tx_info *ti)
+{
+	tcl_ext_cmd->info0 = le32_encode_bits(ti->paddr,
+					      HAL_TX_MSDU_EXT_INFO0_BUF_PTR_LO);
+	tcl_ext_cmd->info1 = le32_encode_bits(0x0,
+					      HAL_TX_MSDU_EXT_INFO1_BUF_PTR_HI) |
+			       le32_encode_bits(ti->data_len,
+						HAL_TX_MSDU_EXT_INFO1_BUF_LEN);
+
+	tcl_ext_cmd->info1 |= le32_encode_bits(1, HAL_TX_MSDU_EXT_INFO1_EXTN_OVERRIDE) |
+				le32_encode_bits(ti->encap_type,
+						 HAL_TX_MSDU_EXT_INFO1_ENCAP_TYPE) |
+				le32_encode_bits(ti->encrypt_type,
+						 HAL_TX_MSDU_EXT_INFO1_ENCRYPT_TYPE);
+}
+
+#define HTT_META_DATA_ALIGNMENT 0x8
+
+/* Preparing HTT Metadata when utilized with ext MSDU */
+static int ath12k_wifi7_dp_prepare_htt_metadata(struct sk_buff *skb)
+{
+	struct hal_tx_msdu_metadata *desc_ext;
+	u8 htt_desc_size;
+	/* Size rounded of multiple of 8 bytes */
+	u8 htt_desc_size_aligned;
+
+	htt_desc_size = sizeof(struct hal_tx_msdu_metadata);
+	htt_desc_size_aligned = ALIGN(htt_desc_size, HTT_META_DATA_ALIGNMENT);
+
+	desc_ext = ath12k_dp_metadata_align_skb(skb, htt_desc_size_aligned);
+	if (!desc_ext)
+		return -ENOMEM;
+
+	desc_ext->info0 = le32_encode_bits(1, HAL_TX_MSDU_METADATA_INFO0_ENCRYPT_FLAG) |
+			  le32_encode_bits(0, HAL_TX_MSDU_METADATA_INFO0_ENCRYPT_TYPE) |
+			  le32_encode_bits(1,
+					   HAL_TX_MSDU_METADATA_INFO0_HOST_TX_DESC_POOL);
+
+	return 0;
+}
+
+/* TODO: Remove the export once this file is built with wifi7 ko */
+int ath12k_wifi7_dp_tx(struct ath12k_pdev_dp *dp_pdev, struct ath12k_link_vif *arvif,
+		       struct sk_buff *skb, bool gsn_valid, int mcbc_gsn,
+		       bool is_mcast)
+{
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_hal *hal = dp->hal;
+	struct ath12k_base *ab = dp->ab;
+	struct hal_tx_info ti = {};
+	struct ath12k_tx_desc_info *tx_desc;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct ath12k_skb_cb *skb_cb = ATH12K_SKB_CB(skb);
+	struct hal_tcl_data_cmd *hal_tcl_desc;
+	struct hal_tx_msdu_ext_desc *msg;
+	struct sk_buff *skb_ext_desc = NULL;
+	struct hal_srng *tcl_ring;
+	struct ieee80211_hdr *hdr = (void *)skb->data;
+	struct ath12k_vif *ahvif = arvif->ahvif;
+	struct ath12k_dp_vif *dp_vif = &ahvif->dp_vif;
+	struct ath12k_dp_link_vif *dp_link_vif;
+	struct dp_tx_ring *tx_ring;
+	u8 pool_id;
+	u8 hal_ring_id;
+	int ret;
+	u8 ring_selector, ring_map = 0;
+	bool tcl_ring_retry;
+	bool msdu_ext_desc = false;
+	bool add_htt_metadata = false;
+	u32 iova_mask = dp->hw_params->iova_mask;
+	bool is_diff_encap = false;
+	bool is_null_frame = false;
+
+	if (test_bit(ATH12K_FLAG_CRASH_FLUSH, &ab->dev_flags))
+		return -ESHUTDOWN;
+
+	if (!(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) &&
+	    !ieee80211_is_data(hdr->frame_control))
+		return -EOPNOTSUPP;
+
+	pool_id = skb_get_queue_mapping(skb) & (ATH12K_HW_MAX_QUEUES - 1);
+
+	/* Let the default ring selection be based on current processor
+	 * number, where one of the 3 tcl rings are selected based on
+	 * the smp_processor_id(). In case that ring
+	 * is full/busy, we resort to other available rings.
+	 * If all rings are full, we drop the packet.
+	 * TODO: Add throttling logic when all rings are full
+	 */
+	ring_selector = dp->hw_params->hw_ops->get_ring_selector(skb);
+
+tcl_ring_sel:
+	tcl_ring_retry = false;
+	ti.ring_id = ring_selector % dp->hw_params->max_tx_ring;
+
+	ring_map |= BIT(ti.ring_id);
+	ti.rbm_id = hal->tcl_to_wbm_rbm_map[ti.ring_id].rbm_id;
+
+	tx_ring = &dp->tx_ring[ti.ring_id];
+
+	tx_desc = ath12k_dp_tx_assign_buffer(dp, pool_id);
+	if (!tx_desc)
+		return -ENOMEM;
+
+	dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, arvif->link_id);
+
+	ti.bank_id = dp_link_vif->bank_id;
+	ti.meta_data_flags = dp_link_vif->tcl_metadata;
+
+	if (dp_vif->tx_encap_type == HAL_TCL_ENCAP_TYPE_RAW &&
+	    test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ab->dev_flags)) {
+		if (skb_cb->flags & ATH12K_SKB_CIPHER_SET) {
+			ti.encrypt_type =
+				ath12k_dp_tx_get_encrypt_type(skb_cb->cipher);
+
+			if (ieee80211_has_protected(hdr->frame_control))
+				skb_put(skb, IEEE80211_CCMP_MIC_LEN);
+		} else {
+			ti.encrypt_type = HAL_ENCRYPT_TYPE_OPEN;
+		}
+
+		msdu_ext_desc = true;
+	}
+
+	if (gsn_valid) {
+		/* Reset and Initialize meta_data_flags with Global Sequence
+		 * Number (GSN) info.
+		 */
+		ti.meta_data_flags =
+			u32_encode_bits(HTT_TCL_META_DATA_TYPE_GLOBAL_SEQ_NUM,
+					HTT_TCL_META_DATA_TYPE) |
+			u32_encode_bits(mcbc_gsn, HTT_TCL_META_DATA_GLOBAL_SEQ_NUM);
+	}
+
+	ti.encap_type = ath12k_dp_tx_get_encap_type(ab, skb);
+	ti.addr_search_flags = dp_link_vif->hal_addr_search_flags;
+	ti.search_type = dp_link_vif->search_type;
+	ti.type = HAL_TCL_DESC_TYPE_BUFFER;
+	ti.pkt_offset = 0;
+	ti.lmac_id = dp_link_vif->lmac_id;
+
+	ti.vdev_id = dp_link_vif->vdev_id;
+	if (gsn_valid)
+		ti.vdev_id += HTT_TX_MLO_MCAST_HOST_REINJECT_BASE_VDEV_ID;
+
+	ti.bss_ast_hash = dp_link_vif->ast_hash;
+	ti.bss_ast_idx = dp_link_vif->ast_idx;
+	ti.dscp_tid_tbl_idx = 0;
+
+	if (skb->ip_summed == CHECKSUM_PARTIAL &&
+	    ti.encap_type != HAL_TCL_ENCAP_TYPE_RAW) {
+		ti.flags0 |= u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_IP4_CKSUM_EN) |
+			     u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_UDP4_CKSUM_EN) |
+			     u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_UDP6_CKSUM_EN) |
+			     u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_TCP4_CKSUM_EN) |
+			     u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_TCP6_CKSUM_EN);
+	}
+
+	ti.flags1 |= u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO3_TID_OVERWRITE);
+
+	ti.tid = ath12k_dp_tx_get_tid(skb);
+
+	switch (ti.encap_type) {
+	case HAL_TCL_ENCAP_TYPE_NATIVE_WIFI:
+		is_null_frame = ieee80211_is_nullfunc(hdr->frame_control);
+		if (ahvif->vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED) {
+			if (skb->protocol == cpu_to_be16(ETH_P_PAE) || is_null_frame)
+				is_diff_encap = true;
+
+			/* Firmware expects msdu ext descriptor for nwifi/raw packets
+			 * received in ETH mode. Without this, observed tx fail for
+			 * Multicast packets in ETH mode.
+			 */
+			msdu_ext_desc = true;
+		} else {
+			ath12k_dp_tx_encap_nwifi(skb);
+		}
+		break;
+	case HAL_TCL_ENCAP_TYPE_RAW:
+		if (!test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags)) {
+			ret = -EINVAL;
+			goto fail_remove_tx_buf;
+		}
+		break;
+	case HAL_TCL_ENCAP_TYPE_ETHERNET:
+		/* no need to encap */
+		break;
+	case HAL_TCL_ENCAP_TYPE_802_3:
+	default:
+		/* TODO: Take care of other encap modes as well */
+		ret = -EINVAL;
+		atomic_inc(&dp->device_stats.tx_err.misc_fail);
+		goto fail_remove_tx_buf;
+	}
+
+	if (iova_mask &&
+	    (unsigned long)skb->data & iova_mask) {
+		ret = ath12k_dp_tx_align_payload(dp, &skb);
+		if (ret) {
+			ath12k_warn(ab, "failed to align TX buffer %d\n", ret);
+			/* don't bail out, give original buffer
+			 * a chance even unaligned.
+			 */
+			goto map;
+		}
+
+		/* hdr is pointing to a wrong place after alignment,
+		 * so refresh it for later use.
+		 */
+		hdr = (void *)skb->data;
+	}
+map:
+	ti.paddr = dma_map_single(dp->dev, skb->data, skb->len, DMA_TO_DEVICE);
+	if (dma_mapping_error(dp->dev, ti.paddr)) {
+		atomic_inc(&dp->device_stats.tx_err.misc_fail);
+		ath12k_warn(ab, "failed to DMA map data Tx buffer\n");
+		ret = -ENOMEM;
+		goto fail_remove_tx_buf;
+	}
+
+	if ((!test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ab->dev_flags) &&
+	     !(skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) &&
+	     !(skb_cb->flags & ATH12K_SKB_CIPHER_SET) &&
+	     ieee80211_has_protected(hdr->frame_control)) ||
+	    is_diff_encap) {
+		/* Firmware is not expecting meta data for qos null
+		 * nwifi packet received in ETH encap mode.
+		 */
+		if (is_null_frame && msdu_ext_desc)
+			goto skip_htt_meta;
+
+		/* Add metadata for sw encrypted vlan group traffic
+		 * and EAPOL nwifi packet received in ETH encap mode.
+		 */
+		add_htt_metadata = true;
+		msdu_ext_desc = true;
+		ti.meta_data_flags |= HTT_TCL_META_DATA_VALID_HTT;
+skip_htt_meta:
+		ti.flags0 |= u32_encode_bits(1, HAL_TCL_DATA_CMD_INFO2_TO_FW);
+		ti.encap_type = HAL_TCL_ENCAP_TYPE_RAW;
+		ti.encrypt_type = HAL_ENCRYPT_TYPE_OPEN;
+	}
+
+	tx_desc->skb = skb;
+	tx_desc->mac_id = dp_link_vif->pdev_idx;
+	ti.desc_id = tx_desc->desc_id;
+	ti.data_len = skb->len;
+	skb_cb->paddr = ti.paddr;
+
+	if (msdu_ext_desc) {
+		skb_ext_desc = dev_alloc_skb(sizeof(struct hal_tx_msdu_ext_desc));
+		if (!skb_ext_desc) {
+			ret = -ENOMEM;
+			goto fail_unmap_dma;
+		}
+
+		skb_put(skb_ext_desc, sizeof(struct hal_tx_msdu_ext_desc));
+		memset(skb_ext_desc->data, 0, skb_ext_desc->len);
+
+		msg = (struct hal_tx_msdu_ext_desc *)skb_ext_desc->data;
+		ath12k_wifi7_hal_tx_cmd_ext_desc_setup(ab, msg, &ti);
+
+		if (add_htt_metadata) {
+			ret = ath12k_wifi7_dp_prepare_htt_metadata(skb_ext_desc);
+			if (ret < 0) {
+				ath12k_dbg(ab, ATH12K_DBG_DP_TX,
+					   "Failed to add HTT meta data, dropping packet\n");
+				goto fail_free_ext_skb;
+			}
+		}
+
+		ti.paddr = dma_map_single(dp->dev, skb_ext_desc->data,
+					  skb_ext_desc->len, DMA_TO_DEVICE);
+		ret = dma_mapping_error(dp->dev, ti.paddr);
+		if (ret)
+			goto fail_free_ext_skb;
+
+		ti.data_len = skb_ext_desc->len;
+		ti.type = HAL_TCL_DESC_TYPE_EXT_DESC;
+
+		skb_cb->paddr_ext_desc = ti.paddr;
+		tx_desc->skb_ext_desc = skb_ext_desc;
+	}
+
+	hal_ring_id = tx_ring->tcl_data_ring.ring_id;
+	tcl_ring = &hal->srng_list[hal_ring_id];
+
+	spin_lock_bh(&tcl_ring->lock);
+
+	ath12k_hal_srng_access_begin(ab, tcl_ring);
+
+	hal_tcl_desc = ath12k_hal_srng_src_get_next_entry(ab, tcl_ring);
+	if (!hal_tcl_desc) {
+		/* NOTE: It is highly unlikely we'll be running out of tcl_ring
+		 * desc because the desc is directly enqueued onto hw queue.
+		 */
+		ath12k_hal_srng_access_end(ab, tcl_ring);
+		dp->device_stats.tx_err.desc_na[ti.ring_id]++;
+		spin_unlock_bh(&tcl_ring->lock);
+		ret = -ENOMEM;
+
+		/* Checking for available tcl descriptors in another ring in
+		 * case of failure due to full tcl ring now, is better than
+		 * checking this ring earlier for each pkt tx.
+		 * Restart ring selection if some rings are not checked yet.
+		 */
+		if (ring_map != (BIT(dp->hw_params->max_tx_ring) - 1) &&
+		    dp->hw_params->tcl_ring_retry) {
+			tcl_ring_retry = true;
+			ring_selector++;
+		}
+
+		goto fail_unmap_dma_ext;
+	}
+
+	spin_lock_bh(&arvif->link_stats_lock);
+	arvif->link_stats.tx_encap_type[ti.encap_type]++;
+	arvif->link_stats.tx_encrypt_type[ti.encrypt_type]++;
+	arvif->link_stats.tx_desc_type[ti.type]++;
+
+	if (is_mcast)
+		arvif->link_stats.tx_bcast_mcast++;
+	else
+		arvif->link_stats.tx_enqueued++;
+	spin_unlock_bh(&arvif->link_stats_lock);
+
+	dp->device_stats.tx_enqueued[ti.ring_id]++;
+
+	ath12k_wifi7_hal_tx_cmd_desc_setup(ab, hal_tcl_desc, &ti);
+
+	ath12k_hal_srng_access_end(ab, tcl_ring);
+
+	spin_unlock_bh(&tcl_ring->lock);
+
+	ath12k_dbg_dump(ab, ATH12K_DBG_DP_TX, NULL, "dp tx msdu: ",
+			skb->data, skb->len);
+
+	atomic_inc(&dp_pdev->num_tx_pending);
+
+	return 0;
+
+fail_unmap_dma_ext:
+	if (skb_cb->paddr_ext_desc)
+		dma_unmap_single(dp->dev, skb_cb->paddr_ext_desc,
+				 skb_ext_desc->len,
+				 DMA_TO_DEVICE);
+fail_free_ext_skb:
+	kfree_skb(skb_ext_desc);
+
+fail_unmap_dma:
+	dma_unmap_single(dp->dev, ti.paddr, ti.data_len, DMA_TO_DEVICE);
+
+fail_remove_tx_buf:
+	ath12k_dp_tx_release_txbuf(dp, tx_desc, pool_id);
+
+	spin_lock_bh(&arvif->link_stats_lock);
+	arvif->link_stats.tx_dropped++;
+	spin_unlock_bh(&arvif->link_stats_lock);
+
+	if (tcl_ring_retry)
+		goto tcl_ring_sel;
+
+	return ret;
+}
+
+static void
+ath12k_dp_tx_htt_tx_complete_buf(struct ath12k_dp *dp,
+				 struct ath12k_tx_desc_params *desc_params,
+				 struct dp_tx_ring *tx_ring,
+				 struct ath12k_dp_htt_wbm_tx_status *ts,
+				 u16 peer_id)
+{
+	struct ath12k_base *ab = dp->ab;
+	struct ieee80211_tx_info *info;
+	struct ath12k_link_vif *arvif;
+	struct ath12k_skb_cb *skb_cb;
+	struct ieee80211_vif *vif;
+	struct ath12k_vif *ahvif;
+	struct sk_buff *msdu = desc_params->skb;
+	s32 noise_floor;
+	struct ieee80211_tx_status status = {};
+	struct ath12k_dp_link_peer *peer;
+	struct ath12k_pdev_dp *dp_pdev;
+	u8 pdev_id;
+
+	skb_cb = ATH12K_SKB_CB(msdu);
+	info = IEEE80211_SKB_CB(msdu);
+
+	pdev_id = ath12k_hw_mac_id_to_pdev_id(dp->hw_params, desc_params->mac_id);
+
+	rcu_read_lock();
+	dp_pdev = ath12k_dp_to_pdev_dp(dp, pdev_id);
+	if (!dp_pdev) {
+		rcu_read_unlock();
+		return;
+	}
+
+	dp->device_stats.tx_completed[tx_ring->tcl_data_ring_id]++;
+
+	if (atomic_dec_and_test(&dp_pdev->num_tx_pending))
+		wake_up(&dp_pdev->tx_empty_waitq);
+
+	dma_unmap_single(dp->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE);
+	if (skb_cb->paddr_ext_desc) {
+		dma_unmap_single(dp->dev, skb_cb->paddr_ext_desc,
+				 desc_params->skb_ext_desc->len, DMA_TO_DEVICE);
+		dev_kfree_skb_any(desc_params->skb_ext_desc);
+	}
+
+	vif = skb_cb->vif;
+	if (vif) {
+		ahvif = ath12k_vif_to_ahvif(vif);
+		arvif = rcu_dereference(ahvif->link[skb_cb->link_id]);
+		if (arvif) {
+			spin_lock_bh(&arvif->link_stats_lock);
+			arvif->link_stats.tx_completed++;
+			spin_unlock_bh(&arvif->link_stats_lock);
+		}
+	}
+
+	memset(&info->status, 0, sizeof(info->status));
+
+	if (ts->acked) {
+		if (!(info->flags & IEEE80211_TX_CTL_NO_ACK)) {
+			info->flags |= IEEE80211_TX_STAT_ACK;
+			info->status.ack_signal = ts->ack_rssi;
+
+			if (!test_bit(WMI_TLV_SERVICE_HW_DB2DBM_CONVERSION_SUPPORT,
+				      ab->wmi_ab.svc_map)) {
+				struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev);
+
+				spin_lock_bh(&ar->data_lock);
+				noise_floor = ath12k_pdev_get_noise_floor(ar);
+				spin_unlock_bh(&ar->data_lock);
+
+				info->status.ack_signal += noise_floor;
+			}
+
+			info->status.flags = IEEE80211_TX_STATUS_ACK_SIGNAL_VALID;
+		} else {
+			info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
+		}
+	}
+
+	peer = ath12k_dp_link_peer_find_by_peerid(dp_pdev, peer_id);
+	if (!peer || !peer->sta) {
+		ath12k_dbg(ab, ATH12K_DBG_DATA,
+			   "dp_tx: failed to find the peer with peer_id %d\n", peer_id);
+		ieee80211_free_txskb(ath12k_pdev_dp_to_hw(dp_pdev), msdu);
+		goto exit;
+	} else {
+		status.sta = peer->sta;
+	}
+
+	status.info = info;
+	status.skb = msdu;
+	ieee80211_tx_status_ext(ath12k_pdev_dp_to_hw(dp_pdev), &status);
+exit:
+	rcu_read_unlock();
+}
+
+static void
+ath12k_dp_tx_process_htt_tx_complete(struct ath12k_dp *dp, void *desc,
+				     struct dp_tx_ring *tx_ring,
+				     struct ath12k_tx_desc_params *desc_params)
+{
+	struct htt_tx_wbm_completion *status_desc;
+	struct ath12k_dp_htt_wbm_tx_status ts = {};
+	enum hal_wbm_htt_tx_comp_status wbm_status;
+	u16 peer_id;
+
+	status_desc = desc;
+
+	wbm_status = le32_get_bits(status_desc->info0,
+				   HTT_TX_WBM_COMP_INFO0_STATUS);
+	dp->device_stats.fw_tx_status[wbm_status]++;
+
+	switch (wbm_status) {
+	case HAL_WBM_REL_HTT_TX_COMP_STATUS_OK:
+		ts.acked = (wbm_status == HAL_WBM_REL_HTT_TX_COMP_STATUS_OK);
+		ts.ack_rssi = le32_get_bits(status_desc->info2,
+					    HTT_TX_WBM_COMP_INFO2_ACK_RSSI);
+
+		peer_id = le32_get_bits(((struct hal_wbm_completion_ring_tx *)desc)->
+				info3, HAL_WBM_COMPL_TX_INFO3_PEER_ID);
+
+		ath12k_dp_tx_htt_tx_complete_buf(dp, desc_params, tx_ring, &ts, peer_id);
+		break;
+	case HAL_WBM_REL_HTT_TX_COMP_STATUS_DROP:
+	case HAL_WBM_REL_HTT_TX_COMP_STATUS_TTL:
+	case HAL_WBM_REL_HTT_TX_COMP_STATUS_REINJ:
+	case HAL_WBM_REL_HTT_TX_COMP_STATUS_INSPECT:
+	case HAL_WBM_REL_HTT_TX_COMP_STATUS_VDEVID_MISMATCH:
+		ath12k_dp_tx_free_txbuf(dp, tx_ring, desc_params);
+		break;
+	case HAL_WBM_REL_HTT_TX_COMP_STATUS_MEC_NOTIFY:
+		/* This event is to be handled only when the driver decides to
+		 * use WDS offload functionality.
+		 */
+		break;
+	default:
+		ath12k_warn(dp->ab, "Unknown htt wbm tx status %d\n", wbm_status);
+		break;
+	}
+}
+
+static void ath12k_wifi7_dp_tx_update_txcompl(struct ath12k_pdev_dp *dp_pdev,
+					      struct hal_tx_status *ts)
+{
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_dp_link_peer *peer;
+	struct ath12k_link_sta *arsta;
+	struct rate_info txrate = {};
+	struct ieee80211_sta *sta;
+	struct ath12k_sta *ahsta;
+	u16 rate, ru_tones;
+	u8 rate_idx = 0;
+	int ret;
+
+	peer = ath12k_dp_link_peer_find_by_peerid(dp_pdev, ts->peer_id);
+	if (!peer || !peer->sta) {
+		ath12k_dbg(dp->ab, ATH12K_DBG_DP_TX,
+			   "failed to find the peer by id %u\n", ts->peer_id);
+		return;
+	}
+
+	spin_lock_bh(&dp->dp_lock);
+
+	sta = peer->sta;
+	ahsta = ath12k_sta_to_ahsta(sta);
+	arsta = &ahsta->deflink;
+
+	spin_unlock_bh(&dp->dp_lock);
+
+	/* This is to prefer choose the real NSS value arsta->last_txrate.nss,
+	 * if it is invalid, then choose the NSS value while assoc.
+	 */
+	if (peer->last_txrate.nss)
+		txrate.nss = peer->last_txrate.nss;
+	else
+		txrate.nss = arsta->peer_nss;
+
+	switch (ts->pkt_type) {
+	case HAL_TX_RATE_STATS_PKT_TYPE_11A:
+	case HAL_TX_RATE_STATS_PKT_TYPE_11B:
+		ret = ath12k_mac_hw_ratecode_to_legacy_rate(ts->mcs,
+							    ts->pkt_type,
+							    &rate_idx,
+							    &rate);
+		if (ret < 0) {
+			ath12k_warn(dp->ab, "Invalid tx legacy rate %d\n", ret);
+			return;
+		}
+
+		txrate.legacy = rate;
+		break;
+	case HAL_TX_RATE_STATS_PKT_TYPE_11N:
+		if (ts->mcs > ATH12K_HT_MCS_MAX) {
+			ath12k_warn(dp->ab, "Invalid HT mcs index %d\n", ts->mcs);
+			return;
+		}
+
+		if (txrate.nss != 0)
+			txrate.mcs = ts->mcs + 8 * (txrate.nss - 1);
+
+		txrate.flags = RATE_INFO_FLAGS_MCS;
+
+		if (ts->sgi)
+			txrate.flags |= RATE_INFO_FLAGS_SHORT_GI;
+		break;
+	case HAL_TX_RATE_STATS_PKT_TYPE_11AC:
+		if (ts->mcs > ATH12K_VHT_MCS_MAX) {
+			ath12k_warn(dp->ab, "Invalid VHT mcs index %d\n", ts->mcs);
+			return;
+		}
+
+		txrate.mcs = ts->mcs;
+		txrate.flags = RATE_INFO_FLAGS_VHT_MCS;
+
+		if (ts->sgi)
+			txrate.flags |= RATE_INFO_FLAGS_SHORT_GI;
+		break;
+	case HAL_TX_RATE_STATS_PKT_TYPE_11AX:
+		if (ts->mcs > ATH12K_HE_MCS_MAX) {
+			ath12k_warn(dp->ab, "Invalid HE mcs index %d\n", ts->mcs);
+			return;
+		}
+
+		txrate.mcs = ts->mcs;
+		txrate.flags = RATE_INFO_FLAGS_HE_MCS;
+		txrate.he_gi = ath12k_he_gi_to_nl80211_he_gi(ts->sgi);
+		break;
+	case HAL_TX_RATE_STATS_PKT_TYPE_11BE:
+		if (ts->mcs > ATH12K_EHT_MCS_MAX) {
+			ath12k_warn(dp->ab, "Invalid EHT mcs index %d\n", ts->mcs);
+			return;
+		}
+
+		txrate.mcs = ts->mcs;
+		txrate.flags = RATE_INFO_FLAGS_EHT_MCS;
+		txrate.eht_gi = ath12k_mac_eht_gi_to_nl80211_eht_gi(ts->sgi);
+		break;
+	default:
+		ath12k_warn(dp->ab, "Invalid tx pkt type: %d\n", ts->pkt_type);
+		return;
+	}
+
+	txrate.bw = ath12k_mac_bw_to_mac80211_bw(ts->bw);
+
+	if (ts->ofdma && ts->pkt_type == HAL_TX_RATE_STATS_PKT_TYPE_11AX) {
+		txrate.bw = RATE_INFO_BW_HE_RU;
+		ru_tones = ath12k_mac_he_convert_tones_to_ru_tones(ts->tones);
+		txrate.he_ru_alloc =
+			ath12k_he_ru_tones_to_nl80211_he_ru_alloc(ru_tones);
+	}
+
+	if (ts->ofdma && ts->pkt_type == HAL_TX_RATE_STATS_PKT_TYPE_11BE) {
+		txrate.bw = RATE_INFO_BW_EHT_RU;
+		txrate.eht_ru_alloc =
+			ath12k_mac_eht_ru_tones_to_nl80211_eht_ru_alloc(ts->tones);
+	}
+
+	spin_lock_bh(&dp->dp_lock);
+	peer->txrate = txrate;
+	spin_unlock_bh(&dp->dp_lock);
+}
+
+static void ath12k_wifi7_dp_tx_complete_msdu(struct ath12k_pdev_dp *dp_pdev,
+					     struct ath12k_tx_desc_params *desc_params,
+					     struct hal_tx_status *ts,
+					     int ring)
+{
+	struct ath12k_dp *dp = dp_pdev->dp;
+	struct ath12k_base *ab = dp->ab;
+	struct ieee80211_tx_info *info;
+	struct ath12k_link_vif *arvif;
+	struct ath12k_skb_cb *skb_cb;
+	struct ieee80211_vif *vif;
+	struct ath12k_vif *ahvif;
+	struct sk_buff *msdu = desc_params->skb;
+	s32 noise_floor;
+	struct ieee80211_tx_status status = {};
+	struct ieee80211_rate_status status_rate = {};
+	struct ath12k_dp_link_peer *peer;
+	struct rate_info rate;
+
+	if (WARN_ON_ONCE(ts->buf_rel_source != HAL_WBM_REL_SRC_MODULE_TQM)) {
+		/* Must not happen */
+		return;
+	}
+
+	skb_cb = ATH12K_SKB_CB(msdu);
+	dp->device_stats.tx_completed[ring]++;
+
+	dma_unmap_single(dp->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE);
+	if (skb_cb->paddr_ext_desc) {
+		dma_unmap_single(dp->dev, skb_cb->paddr_ext_desc,
+				 desc_params->skb_ext_desc->len, DMA_TO_DEVICE);
+		dev_kfree_skb_any(desc_params->skb_ext_desc);
+	}
+
+	rcu_read_lock();
+
+	if (!rcu_dereference(ab->pdevs_active[dp_pdev->mac_id])) {
+		ieee80211_free_txskb(ath12k_pdev_dp_to_hw(dp_pdev), msdu);
+		goto exit;
+	}
+
+	if (!skb_cb->vif) {
+		ieee80211_free_txskb(ath12k_pdev_dp_to_hw(dp_pdev), msdu);
+		goto exit;
+	}
+
+	vif = skb_cb->vif;
+	if (vif) {
+		ahvif = ath12k_vif_to_ahvif(vif);
+		arvif = rcu_dereference(ahvif->link[skb_cb->link_id]);
+		if (arvif) {
+			spin_lock_bh(&arvif->link_stats_lock);
+			arvif->link_stats.tx_completed++;
+			spin_unlock_bh(&arvif->link_stats_lock);
+		}
+	}
+
+	info = IEEE80211_SKB_CB(msdu);
+	memset(&info->status, 0, sizeof(info->status));
+
+	/* skip tx rate update from ieee80211_status*/
+	info->status.rates[0].idx = -1;
+
+	switch (ts->status) {
+	case HAL_WBM_TQM_REL_REASON_FRAME_ACKED:
+		if (!(info->flags & IEEE80211_TX_CTL_NO_ACK)) {
+			info->flags |= IEEE80211_TX_STAT_ACK;
+			info->status.ack_signal = ts->ack_rssi;
+
+			if (!test_bit(WMI_TLV_SERVICE_HW_DB2DBM_CONVERSION_SUPPORT,
+				      ab->wmi_ab.svc_map)) {
+				struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev);
+
+				spin_lock_bh(&ar->data_lock);
+				noise_floor = ath12k_pdev_get_noise_floor(ar);
+				spin_unlock_bh(&ar->data_lock);
+
+				info->status.ack_signal += noise_floor;
+			}
+
+			info->status.flags = IEEE80211_TX_STATUS_ACK_SIGNAL_VALID;
+		}
+		break;
+	case HAL_WBM_TQM_REL_REASON_CMD_REMOVE_TX:
+		if (info->flags & IEEE80211_TX_CTL_NO_ACK) {
+			info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
+			break;
+		}
+		fallthrough;
+	case HAL_WBM_TQM_REL_REASON_CMD_REMOVE_MPDU:
+	case HAL_WBM_TQM_REL_REASON_DROP_THRESHOLD:
+	case HAL_WBM_TQM_REL_REASON_CMD_REMOVE_AGED_FRAMES:
+		/* The failure status is due to internal firmware tx failure
+		 * hence drop the frame; do not update the status of frame to
+		 * the upper layer
+		 */
+		ieee80211_free_txskb(ath12k_pdev_dp_to_hw(dp_pdev), msdu);
+		goto exit;
+	default:
+		ath12k_dbg(ab, ATH12K_DBG_DP_TX, "tx frame is not acked status %d\n",
+			   ts->status);
+		break;
+	}
+
+	/* NOTE: Tx rate status reporting. Tx completion status does not have
+	 * necessary information (for example nss) to build the tx rate.
+	 * Might end up reporting it out-of-band from HTT stats.
+	 */
+
+	ath12k_wifi7_dp_tx_update_txcompl(dp_pdev, ts);
+
+	peer = ath12k_dp_link_peer_find_by_peerid(dp_pdev, ts->peer_id);
+	if (!peer || !peer->sta) {
+		ath12k_err(ab,
+			   "dp_tx: failed to find the peer with peer_id %d\n",
+			   ts->peer_id);
+		ieee80211_free_txskb(ath12k_pdev_dp_to_hw(dp_pdev), msdu);
+		goto exit;
+	}
+
+	status.sta = peer->sta;
+	status.info = info;
+	status.skb = msdu;
+	rate = peer->last_txrate;
+
+	status_rate.rate_idx = rate;
+	status_rate.try_count = 1;
+
+	status.rates = &status_rate;
+	status.n_rates = 1;
+	ieee80211_tx_status_ext(ath12k_pdev_dp_to_hw(dp_pdev), &status);
+
+exit:
+	rcu_read_unlock();
+}
+
+static void
+ath12k_wifi7_dp_tx_status_parse(struct ath12k_dp *dp,
+				struct hal_wbm_completion_ring_tx *desc,
+				struct hal_tx_status *ts)
+{
+	u32 info0 = le32_to_cpu(desc->rate_stats.info0);
+
+	ts->buf_rel_source =
+		le32_get_bits(desc->info0, HAL_WBM_COMPL_TX_INFO0_REL_SRC_MODULE);
+	if (ts->buf_rel_source != HAL_WBM_REL_SRC_MODULE_FW &&
+	    ts->buf_rel_source != HAL_WBM_REL_SRC_MODULE_TQM)
+		return;
+
+	if (ts->buf_rel_source == HAL_WBM_REL_SRC_MODULE_FW)
+		return;
+
+	ts->status = le32_get_bits(desc->info0,
+				   HAL_WBM_COMPL_TX_INFO0_TQM_RELEASE_REASON);
+
+	ts->ppdu_id = le32_get_bits(desc->info1,
+				    HAL_WBM_COMPL_TX_INFO1_TQM_STATUS_NUMBER);
+
+	ts->peer_id = le32_get_bits(desc->info3, HAL_WBM_COMPL_TX_INFO3_PEER_ID);
+
+	ts->ack_rssi = le32_get_bits(desc->info2,
+				     HAL_WBM_COMPL_TX_INFO2_ACK_FRAME_RSSI);
+
+	if (info0 & HAL_TX_RATE_STATS_INFO0_VALID) {
+		ts->pkt_type = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_PKT_TYPE);
+		ts->mcs = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_MCS);
+		ts->sgi = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_SGI);
+		ts->bw = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_BW);
+		ts->tones = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_TONES_IN_RU);
+		ts->ofdma = u32_get_bits(info0, HAL_TX_RATE_STATS_INFO0_OFDMA_TX);
+	}
+}
+
+void ath12k_wifi7_dp_tx_completion_handler(struct ath12k_dp *dp, int ring_id)
+{
+	struct ath12k_base *ab = dp->ab;
+	struct ath12k_pdev_dp *dp_pdev;
+	int hal_ring_id = dp->tx_ring[ring_id].tcl_comp_ring.ring_id;
+	struct hal_srng *status_ring = &dp->hal->srng_list[hal_ring_id];
+	struct ath12k_tx_desc_info *tx_desc = NULL;
+	struct hal_tx_status ts = {};
+	struct ath12k_tx_desc_params desc_params;
+	struct dp_tx_ring *tx_ring = &dp->tx_ring[ring_id];
+	struct hal_wbm_release_ring *desc;
+	u8 pdev_idx;
+	u64 desc_va;
+	enum hal_wbm_rel_src_module buf_rel_source;
+	enum hal_wbm_tqm_rel_reason rel_status;
+
+	spin_lock_bh(&status_ring->lock);
+
+	ath12k_hal_srng_access_begin(ab, status_ring);
+
+	while (ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_head) !=
+	       tx_ring->tx_status_tail) {
+		desc = ath12k_hal_srng_dst_get_next_entry(ab, status_ring);
+		if (!desc)
+			break;
+
+		memcpy(&tx_ring->tx_status[tx_ring->tx_status_head],
+		       desc, sizeof(*desc));
+		tx_ring->tx_status_head =
+			ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_head);
+	}
+
+	if (ath12k_hal_srng_dst_peek(ab, status_ring) &&
+	    (ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_head) ==
+	     tx_ring->tx_status_tail)) {
+		/* TODO: Process pending tx_status messages when kfifo_is_full() */
+		ath12k_warn(ab, "Unable to process some of the tx_status ring desc because status_fifo is full\n");
+	}
+
+	ath12k_hal_srng_access_end(ab, status_ring);
+
+	spin_unlock_bh(&status_ring->lock);
+
+	while (ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_tail) !=
+	       tx_ring->tx_status_head) {
+		struct hal_wbm_completion_ring_tx *tx_status;
+		u32 desc_id;
+
+		tx_ring->tx_status_tail =
+			ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_tail);
+		tx_status = &tx_ring->tx_status[tx_ring->tx_status_tail];
+		ath12k_wifi7_dp_tx_status_parse(dp, tx_status, &ts);
+
+		if (le32_get_bits(tx_status->info0, HAL_WBM_COMPL_TX_INFO0_CC_DONE)) {
+			/* HW done cookie conversion */
+			desc_va = ((u64)le32_to_cpu(tx_status->buf_va_hi) << 32 |
+				   le32_to_cpu(tx_status->buf_va_lo));
+			tx_desc = (struct ath12k_tx_desc_info *)((unsigned long)desc_va);
+		} else {
+			/* SW does cookie conversion to VA */
+			desc_id = le32_get_bits(tx_status->buf_va_hi,
+						BUFFER_ADDR_INFO1_SW_COOKIE);
+
+			tx_desc = ath12k_dp_get_tx_desc(dp, desc_id);
+		}
+		if (!tx_desc) {
+			ath12k_warn(ab, "unable to retrieve tx_desc!");
+			continue;
+		}
+
+		desc_params.mac_id = tx_desc->mac_id;
+		desc_params.skb = tx_desc->skb;
+		desc_params.skb_ext_desc = tx_desc->skb_ext_desc;
+
+		/* Find the HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE value */
+		buf_rel_source = le32_get_bits(tx_status->info0,
+					       HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE);
+		dp->device_stats.tx_wbm_rel_source[buf_rel_source]++;
+
+		rel_status = le32_get_bits(tx_status->info0,
+					   HAL_WBM_COMPL_TX_INFO0_TQM_RELEASE_REASON);
+		dp->device_stats.tqm_rel_reason[rel_status]++;
+
+		/* Release descriptor as soon as extracting necessary info
+		 * to reduce contention
+		 */
+		ath12k_dp_tx_release_txbuf(dp, tx_desc, tx_desc->pool_id);
+		if (ts.buf_rel_source == HAL_WBM_REL_SRC_MODULE_FW) {
+			ath12k_dp_tx_process_htt_tx_complete(dp, (void *)tx_status,
+							     tx_ring, &desc_params);
+			continue;
+		}
+
+		pdev_idx = ath12k_hw_mac_id_to_pdev_id(dp->hw_params, desc_params.mac_id);
+
+		rcu_read_lock();
+
+		dp_pdev = ath12k_dp_to_pdev_dp(dp, pdev_idx);
+		if (!dp_pdev) {
+			rcu_read_unlock();
+			continue;
+		}
+
+		if (atomic_dec_and_test(&dp_pdev->num_tx_pending))
+			wake_up(&dp_pdev->tx_empty_waitq);
+
+		ath12k_wifi7_dp_tx_complete_msdu(dp_pdev, &desc_params, &ts,
+						 tx_ring->tcl_data_ring_id);
+		rcu_read_unlock();
+	}
+}
+
+u32 ath12k_wifi7_dp_tx_get_vdev_bank_config(struct ath12k_base *ab,
+					    struct ath12k_link_vif *arvif)
+{
+	u32 bank_config = 0;
+	u8 link_id = arvif->link_id;
+	struct ath12k_vif *ahvif = arvif->ahvif;
+	struct ath12k_dp_vif *dp_vif = &ahvif->dp_vif;
+	struct ath12k_dp_link_vif *dp_link_vif;
+
+	dp_link_vif = ath12k_dp_vif_to_dp_link_vif(dp_vif, link_id);
+
+	/* Only valid for raw frames with HW crypto enabled.
+	 * With SW crypto, mac80211 sets key per packet
+	 */
+	if (dp_vif->tx_encap_type == HAL_TCL_ENCAP_TYPE_RAW &&
+	    test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ab->dev_flags))
+		bank_config |=
+			u32_encode_bits(ath12k_dp_tx_get_encrypt_type(dp_vif->key_cipher),
+					HAL_TX_BANK_CONFIG_ENCRYPT_TYPE);
+
+	bank_config |= u32_encode_bits(dp_vif->tx_encap_type,
+					HAL_TX_BANK_CONFIG_ENCAP_TYPE);
+	bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_SRC_BUFFER_SWAP) |
+			u32_encode_bits(0, HAL_TX_BANK_CONFIG_LINK_META_SWAP) |
+			u32_encode_bits(0, HAL_TX_BANK_CONFIG_EPD);
+
+	/* only valid if idx_lookup_override is not set in tcl_data_cmd */
+	if (ahvif->vdev_type == WMI_VDEV_TYPE_STA)
+		bank_config |= u32_encode_bits(1, HAL_TX_BANK_CONFIG_INDEX_LOOKUP_EN);
+	else
+		bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_INDEX_LOOKUP_EN);
+
+	bank_config |= u32_encode_bits(dp_link_vif->hal_addr_search_flags &
+				       HAL_TX_ADDRX_EN,
+				       HAL_TX_BANK_CONFIG_ADDRX_EN) |
+			u32_encode_bits(!!(dp_link_vif->hal_addr_search_flags &
+					HAL_TX_ADDRY_EN),
+					HAL_TX_BANK_CONFIG_ADDRY_EN);
+
+	bank_config |= u32_encode_bits(ieee80211_vif_is_mesh(ahvif->vif) ? 3 : 0,
+					HAL_TX_BANK_CONFIG_MESH_EN) |
+			u32_encode_bits(dp_link_vif->vdev_id_check_en,
+					HAL_TX_BANK_CONFIG_VDEV_ID_CHECK_EN);
+
+	bank_config |= u32_encode_bits(0, HAL_TX_BANK_CONFIG_DSCP_TIP_MAP_ID);
+
+	return bank_config;
+}
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.h b/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.h
new file mode 100644
index 000000000000..24cf7972d41b
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef ATH12K_DP_TX_WIFI7_H
+#define ATH12K_DP_TX_WIFI7_H
+
+int ath12k_wifi7_dp_tx(struct ath12k_pdev_dp *dp_pdev, struct ath12k_link_vif *arvif,
+		       struct sk_buff *skb, bool gsn_valid, int mcbc_gsn,
+		       bool is_mcast);
+void ath12k_wifi7_dp_tx_completion_handler(struct ath12k_dp *dp, int ring_id);
+u32 ath12k_wifi7_dp_tx_get_vdev_bank_config(struct ath12k_base *ab,
+					    struct ath12k_link_vif *arvif);
+#endif
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal.c b/drivers/net/wireless/ath/ath12k/wifi7/hal.c
new file mode 100644
index 000000000000..03a007dd6857
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal.c
@@ -0,0 +1,705 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+#include "hw.h"
+#include "hal_desc.h"
+#include "../hal.h"
+#include "hal.h"
+#include "hal_tx.h"
+#include "../debug.h"
+#include "../hif.h"
+#include "hal_qcn9274.h"
+#include "hal_wcn7850.h"
+
+static const struct ath12k_hw_version_map ath12k_wifi7_hw_ver_map[] = {
+	[ATH12K_HW_QCN9274_HW10] = {
+		.hal_ops = &hal_qcn9274_ops,
+		.hal_desc_sz = sizeof(struct hal_rx_desc_qcn9274_compact),
+		.tcl_to_wbm_rbm_map = ath12k_hal_tcl_to_wbm_rbm_map_qcn9274,
+		.hal_params = &ath12k_hw_hal_params_qcn9274,
+		.hw_regs = &qcn9274_v1_regs,
+	},
+	[ATH12K_HW_QCN9274_HW20] = {
+		.hal_ops = &hal_qcn9274_ops,
+		.hal_desc_sz = sizeof(struct hal_rx_desc_qcn9274_compact),
+		.tcl_to_wbm_rbm_map = ath12k_hal_tcl_to_wbm_rbm_map_qcn9274,
+		.hal_params = &ath12k_hw_hal_params_qcn9274,
+		.hw_regs = &qcn9274_v2_regs,
+	},
+	[ATH12K_HW_WCN7850_HW20] = {
+		.hal_ops = &hal_wcn7850_ops,
+		.hal_desc_sz = sizeof(struct hal_rx_desc_wcn7850),
+		.tcl_to_wbm_rbm_map = ath12k_hal_tcl_to_wbm_rbm_map_wcn7850,
+		.hal_params = &ath12k_hw_hal_params_wcn7850,
+		.hw_regs = &wcn7850_regs,
+	},
+	[ATH12K_HW_IPQ5332_HW10] = {
+		.hal_ops = &hal_qcn9274_ops,
+		.hal_desc_sz = sizeof(struct hal_rx_desc_qcn9274_compact),
+		.tcl_to_wbm_rbm_map = ath12k_hal_tcl_to_wbm_rbm_map_qcn9274,
+		.hal_params = &ath12k_hw_hal_params_ipq5332,
+		.hw_regs = &ipq5332_regs,
+	},
+};
+
+int ath12k_wifi7_hal_init(struct ath12k_base *ab)
+{
+	struct ath12k_hal *hal = &ab->hal;
+
+	memset(hal, 0, sizeof(*hal));
+
+	hal->ops = ath12k_wifi7_hw_ver_map[ab->hw_rev].hal_ops;
+	hal->hal_desc_sz = ath12k_wifi7_hw_ver_map[ab->hw_rev].hal_desc_sz;
+	hal->tcl_to_wbm_rbm_map = ath12k_wifi7_hw_ver_map[ab->hw_rev].tcl_to_wbm_rbm_map;
+	hal->regs = ath12k_wifi7_hw_ver_map[ab->hw_rev].hw_regs;
+	hal->hal_params = ath12k_wifi7_hw_ver_map[ab->hw_rev].hal_params;
+	hal->hal_wbm_release_ring_tx_size = sizeof(struct hal_wbm_release_ring_tx);
+
+	return 0;
+}
+
+static unsigned int ath12k_wifi7_hal_reo1_ring_id_offset(struct ath12k_hal *hal)
+{
+	return HAL_REO1_RING_ID(hal) - HAL_REO1_RING_BASE_LSB(hal);
+}
+
+static unsigned
+int ath12k_wifi7_hal_reo1_ring_msi1_base_lsb_offset(struct ath12k_hal *hal)
+{
+	return HAL_REO1_RING_MSI1_BASE_LSB(hal) - HAL_REO1_RING_BASE_LSB(hal);
+}
+
+static unsigned
+int ath12k_wifi7_hal_reo1_ring_msi1_base_msb_offset(struct ath12k_hal *hal)
+{
+	return HAL_REO1_RING_MSI1_BASE_MSB(hal) - HAL_REO1_RING_BASE_LSB(hal);
+}
+
+static unsigned int ath12k_wifi7_hal_reo1_ring_msi1_data_offset(struct ath12k_hal *hal)
+{
+	return HAL_REO1_RING_MSI1_DATA(hal) - HAL_REO1_RING_BASE_LSB(hal);
+}
+
+static unsigned int ath12k_wifi7_hal_reo1_ring_base_msb_offset(struct ath12k_hal *hal)
+{
+	return HAL_REO1_RING_BASE_MSB(hal) - HAL_REO1_RING_BASE_LSB(hal);
+}
+
+static unsigned
+int ath12k_wifi7_hal_reo1_ring_producer_int_setup_offset(struct ath12k_hal *hal)
+{
+	return HAL_REO1_RING_PRODUCER_INT_SETUP(hal) - HAL_REO1_RING_BASE_LSB(hal);
+}
+
+static unsigned int ath12k_wifi7_hal_reo1_ring_hp_addr_lsb_offset(struct ath12k_hal *hal)
+{
+	return HAL_REO1_RING_HP_ADDR_LSB(hal) - HAL_REO1_RING_BASE_LSB(hal);
+}
+
+static unsigned int ath12k_wifi7_hal_reo1_ring_hp_addr_msb_offset(struct ath12k_hal *hal)
+{
+	return HAL_REO1_RING_HP_ADDR_MSB(hal) - HAL_REO1_RING_BASE_LSB(hal);
+}
+
+static unsigned int ath12k_wifi7_hal_reo1_ring_misc_offset(struct ath12k_hal *hal)
+{
+	return HAL_REO1_RING_MISC(hal) - HAL_REO1_RING_BASE_LSB(hal);
+}
+
+void ath12k_wifi7_hal_ce_dst_setup(struct ath12k_base *ab,
+				   struct hal_srng *srng, int ring_num)
+{
+	struct hal_srng_config *srng_config = &ab->hal.srng_config[HAL_CE_DST];
+	u32 addr;
+	u32 val;
+
+	addr = HAL_CE_DST_RING_CTRL +
+	       srng_config->reg_start[HAL_SRNG_REG_GRP_R0] +
+	       ring_num * srng_config->reg_size[HAL_SRNG_REG_GRP_R0];
+
+	val = ath12k_hif_read32(ab, addr);
+	val &= ~HAL_CE_DST_R0_DEST_CTRL_MAX_LEN;
+	val |= u32_encode_bits(srng->u.dst_ring.max_buffer_length,
+			       HAL_CE_DST_R0_DEST_CTRL_MAX_LEN);
+	ath12k_hif_write32(ab, addr, val);
+}
+
+void ath12k_wifi7_hal_srng_dst_hw_init(struct ath12k_base *ab,
+				       struct hal_srng *srng)
+{
+	struct ath12k_hal *hal = &ab->hal;
+	u32 val;
+	u64 hp_addr;
+	u32 reg_base;
+
+	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];
+
+	if (srng->flags & HAL_SRNG_FLAGS_MSI_INTR) {
+		ath12k_hif_write32(ab, reg_base +
+				   ath12k_wifi7_hal_reo1_ring_msi1_base_lsb_offset(hal),
+				   srng->msi_addr);
+
+		val = u32_encode_bits(((u64)srng->msi_addr >> HAL_ADDR_MSB_REG_SHIFT),
+				      HAL_REO1_RING_MSI1_BASE_MSB_ADDR) |
+				      HAL_REO1_RING_MSI1_BASE_MSB_MSI1_ENABLE;
+		ath12k_hif_write32(ab, reg_base +
+				   ath12k_wifi7_hal_reo1_ring_msi1_base_msb_offset(hal),
+				   val);
+
+		ath12k_hif_write32(ab,
+				   reg_base +
+				   ath12k_wifi7_hal_reo1_ring_msi1_data_offset(hal),
+				   srng->msi_data);
+	}
+
+	ath12k_hif_write32(ab, reg_base, srng->ring_base_paddr);
+
+	val = u32_encode_bits(((u64)srng->ring_base_paddr >> HAL_ADDR_MSB_REG_SHIFT),
+			      HAL_REO1_RING_BASE_MSB_RING_BASE_ADDR_MSB) |
+	      u32_encode_bits((srng->entry_size * srng->num_entries),
+			      HAL_REO1_RING_BASE_MSB_RING_SIZE);
+	ath12k_hif_write32(ab, reg_base + ath12k_wifi7_hal_reo1_ring_base_msb_offset(hal),
+			   val);
+
+	val = u32_encode_bits(srng->ring_id, HAL_REO1_RING_ID_RING_ID) |
+	      u32_encode_bits(srng->entry_size, HAL_REO1_RING_ID_ENTRY_SIZE);
+	ath12k_hif_write32(ab, reg_base + ath12k_wifi7_hal_reo1_ring_id_offset(hal), val);
+
+	/* interrupt setup */
+	val = u32_encode_bits((srng->intr_timer_thres_us >> 3),
+			      HAL_REO1_RING_PRDR_INT_SETUP_INTR_TMR_THOLD);
+
+	val |= u32_encode_bits((srng->intr_batch_cntr_thres_entries * srng->entry_size),
+				HAL_REO1_RING_PRDR_INT_SETUP_BATCH_COUNTER_THOLD);
+
+	ath12k_hif_write32(ab,
+			   reg_base +
+			   ath12k_wifi7_hal_reo1_ring_producer_int_setup_offset(hal),
+			   val);
+
+	hp_addr = hal->rdp.paddr +
+		  ((unsigned long)srng->u.dst_ring.hp_addr -
+		   (unsigned long)hal->rdp.vaddr);
+	ath12k_hif_write32(ab, reg_base +
+			   ath12k_wifi7_hal_reo1_ring_hp_addr_lsb_offset(hal),
+			   hp_addr & HAL_ADDR_LSB_REG_MASK);
+	ath12k_hif_write32(ab, reg_base +
+			   ath12k_wifi7_hal_reo1_ring_hp_addr_msb_offset(hal),
+			   hp_addr >> HAL_ADDR_MSB_REG_SHIFT);
+
+	/* Initialize head and tail pointers to indicate ring is empty */
+	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2];
+	ath12k_hif_write32(ab, reg_base, 0);
+	ath12k_hif_write32(ab, reg_base + HAL_REO1_RING_TP_OFFSET, 0);
+	*srng->u.dst_ring.hp_addr = 0;
+
+	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];
+	val = 0;
+	if (srng->flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP)
+		val |= HAL_REO1_RING_MISC_DATA_TLV_SWAP;
+	if (srng->flags & HAL_SRNG_FLAGS_RING_PTR_SWAP)
+		val |= HAL_REO1_RING_MISC_HOST_FW_SWAP;
+	if (srng->flags & HAL_SRNG_FLAGS_MSI_SWAP)
+		val |= HAL_REO1_RING_MISC_MSI_SWAP;
+	val |= HAL_REO1_RING_MISC_SRNG_ENABLE;
+
+	ath12k_hif_write32(ab, reg_base + ath12k_wifi7_hal_reo1_ring_misc_offset(hal),
+			   val);
+}
+
+void ath12k_wifi7_hal_srng_src_hw_init(struct ath12k_base *ab,
+				       struct hal_srng *srng)
+{
+	struct ath12k_hal *hal = &ab->hal;
+	u32 val;
+	u64 tp_addr;
+	u32 reg_base;
+
+	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];
+
+	if (srng->flags & HAL_SRNG_FLAGS_MSI_INTR) {
+		ath12k_hif_write32(ab, reg_base +
+				   HAL_TCL1_RING_MSI1_BASE_LSB_OFFSET(hal),
+				   srng->msi_addr);
+
+		val = u32_encode_bits(((u64)srng->msi_addr >> HAL_ADDR_MSB_REG_SHIFT),
+				      HAL_TCL1_RING_MSI1_BASE_MSB_ADDR) |
+				      HAL_TCL1_RING_MSI1_BASE_MSB_MSI1_ENABLE;
+		ath12k_hif_write32(ab, reg_base +
+				       HAL_TCL1_RING_MSI1_BASE_MSB_OFFSET(hal),
+				   val);
+
+		ath12k_hif_write32(ab, reg_base +
+				       HAL_TCL1_RING_MSI1_DATA_OFFSET(hal),
+				   srng->msi_data);
+	}
+
+	ath12k_hif_write32(ab, reg_base, srng->ring_base_paddr);
+
+	val = u32_encode_bits(((u64)srng->ring_base_paddr >> HAL_ADDR_MSB_REG_SHIFT),
+			      HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB) |
+	      u32_encode_bits((srng->entry_size * srng->num_entries),
+			      HAL_TCL1_RING_BASE_MSB_RING_SIZE);
+	ath12k_hif_write32(ab, reg_base + HAL_TCL1_RING_BASE_MSB_OFFSET(hal), val);
+
+	val = u32_encode_bits(srng->entry_size, HAL_REO1_RING_ID_ENTRY_SIZE);
+	ath12k_hif_write32(ab, reg_base + HAL_TCL1_RING_ID_OFFSET(hal), val);
+
+	val = u32_encode_bits(srng->intr_timer_thres_us,
+			      HAL_TCL1_RING_CONSR_INT_SETUP_IX0_INTR_TMR_THOLD);
+
+	val |= u32_encode_bits((srng->intr_batch_cntr_thres_entries * srng->entry_size),
+			       HAL_TCL1_RING_CONSR_INT_SETUP_IX0_BATCH_COUNTER_THOLD);
+
+	ath12k_hif_write32(ab,
+			   reg_base + HAL_TCL1_RING_CONSR_INT_SETUP_IX0_OFFSET(hal),
+			   val);
+
+	val = 0;
+	if (srng->flags & HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN) {
+		val |= u32_encode_bits(srng->u.src_ring.low_threshold,
+				       HAL_TCL1_RING_CONSR_INT_SETUP_IX1_LOW_THOLD);
+	}
+	ath12k_hif_write32(ab,
+			   reg_base + HAL_TCL1_RING_CONSR_INT_SETUP_IX1_OFFSET(hal),
+			   val);
+
+	if (srng->ring_id != HAL_SRNG_RING_ID_WBM_IDLE_LINK) {
+		tp_addr = hal->rdp.paddr +
+			  ((unsigned long)srng->u.src_ring.tp_addr -
+			   (unsigned long)hal->rdp.vaddr);
+		ath12k_hif_write32(ab,
+				   reg_base + HAL_TCL1_RING_TP_ADDR_LSB_OFFSET(hal),
+				   tp_addr & HAL_ADDR_LSB_REG_MASK);
+		ath12k_hif_write32(ab,
+				   reg_base + HAL_TCL1_RING_TP_ADDR_MSB_OFFSET(hal),
+				   tp_addr >> HAL_ADDR_MSB_REG_SHIFT);
+	}
+
+	/* Initialize head and tail pointers to indicate ring is empty */
+	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2];
+	ath12k_hif_write32(ab, reg_base, 0);
+	ath12k_hif_write32(ab, reg_base + HAL_TCL1_RING_TP_OFFSET, 0);
+	*srng->u.src_ring.tp_addr = 0;
+
+	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];
+	val = 0;
+	if (srng->flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP)
+		val |= HAL_TCL1_RING_MISC_DATA_TLV_SWAP;
+	if (srng->flags & HAL_SRNG_FLAGS_RING_PTR_SWAP)
+		val |= HAL_TCL1_RING_MISC_HOST_FW_SWAP;
+	if (srng->flags & HAL_SRNG_FLAGS_MSI_SWAP)
+		val |= HAL_TCL1_RING_MISC_MSI_SWAP;
+
+	/* Loop count is not used for SRC rings */
+	val |= HAL_TCL1_RING_MISC_MSI_LOOPCNT_DISABLE;
+
+	val |= HAL_TCL1_RING_MISC_SRNG_ENABLE;
+
+	if (srng->ring_id == HAL_SRNG_RING_ID_WBM_IDLE_LINK)
+		val |= HAL_TCL1_RING_MISC_MSI_RING_ID_DISABLE;
+
+	ath12k_hif_write32(ab, reg_base + HAL_TCL1_RING_MISC_OFFSET(hal), val);
+}
+
+void ath12k_wifi7_hal_set_umac_srng_ptr_addr(struct ath12k_base *ab,
+					     struct hal_srng *srng)
+{
+	u32 reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2];
+
+	if (srng->ring_dir == HAL_SRNG_DIR_SRC) {
+		if (!ab->hw_params->supports_shadow_regs) {
+			srng->u.src_ring.hp_addr =
+				(u32 *)((unsigned long)ab->mem + reg_base);
+		} else {
+			ath12k_dbg(ab, ATH12K_DBG_HAL,
+				   "hal reg_base 0x%x shadow 0x%lx\n",
+				   reg_base,
+				   (unsigned long)srng->u.src_ring.hp_addr -
+				   (unsigned long)ab->mem);
+		}
+	} else  {
+		if (!ab->hw_params->supports_shadow_regs) {
+			srng->u.dst_ring.tp_addr =
+				(u32 *)((unsigned long)ab->mem + reg_base +
+						(HAL_REO1_RING_TP - HAL_REO1_RING_HP));
+		} else {
+			ath12k_dbg(ab, ATH12K_DBG_HAL,
+				   "target_reg 0x%x shadow 0x%lx\n",
+				   reg_base + HAL_REO1_RING_TP - HAL_REO1_RING_HP,
+				   (unsigned long)srng->u.dst_ring.tp_addr -
+				   (unsigned long)ab->mem);
+		}
+	}
+}
+
+int ath12k_wifi7_hal_srng_get_ring_id(struct ath12k_hal *hal,
+				      enum hal_ring_type type,
+				      int ring_num, int mac_id)
+{
+	struct hal_srng_config *srng_config = &hal->srng_config[type];
+	int ring_id;
+
+	if (ring_num >= srng_config->max_rings) {
+		ath12k_warn(hal, "invalid ring number :%d\n", ring_num);
+		return -EINVAL;
+	}
+
+	ring_id = srng_config->start_ring_id + ring_num;
+	if (srng_config->mac_type == ATH12K_HAL_SRNG_PMAC)
+		ring_id += mac_id * HAL_SRNG_RINGS_PER_PMAC;
+
+	if (WARN_ON(ring_id >= HAL_SRNG_RING_ID_MAX))
+		return -EINVAL;
+
+	return ring_id;
+}
+
+static
+void ath12k_wifi7_hal_srng_update_hp_tp_addr(struct ath12k_base *ab,
+					     int shadow_cfg_idx,
+					     enum hal_ring_type ring_type,
+					     int ring_num)
+{
+	struct hal_srng *srng;
+	struct ath12k_hal *hal = &ab->hal;
+	int ring_id;
+	struct hal_srng_config *srng_config = &hal->srng_config[ring_type];
+
+	ring_id = ath12k_wifi7_hal_srng_get_ring_id(hal, ring_type, ring_num,
+						    0);
+	if (ring_id < 0)
+		return;
+
+	srng = &hal->srng_list[ring_id];
+
+	if (srng_config->ring_dir == HAL_SRNG_DIR_DST)
+		srng->u.dst_ring.tp_addr = (u32 *)(HAL_SHADOW_REG(shadow_cfg_idx) +
+						   (unsigned long)ab->mem);
+	else
+		srng->u.src_ring.hp_addr = (u32 *)(HAL_SHADOW_REG(shadow_cfg_idx) +
+						   (unsigned long)ab->mem);
+}
+
+u32 ath12k_wifi7_hal_ce_get_desc_size(enum hal_ce_desc type)
+{
+	switch (type) {
+	case HAL_CE_DESC_SRC:
+		return sizeof(struct hal_ce_srng_src_desc);
+	case HAL_CE_DESC_DST:
+		return sizeof(struct hal_ce_srng_dest_desc);
+	case HAL_CE_DESC_DST_STATUS:
+		return sizeof(struct hal_ce_srng_dst_status_desc);
+	}
+
+	return 0;
+}
+
+int ath12k_wifi7_hal_srng_update_shadow_config(struct ath12k_base *ab,
+					       enum hal_ring_type ring_type,
+					       int ring_num)
+{
+	struct ath12k_hal *hal = &ab->hal;
+	struct hal_srng_config *srng_config = &hal->srng_config[ring_type];
+	int shadow_cfg_idx = hal->num_shadow_reg_configured;
+	u32 target_reg;
+
+	if (shadow_cfg_idx >= HAL_SHADOW_NUM_REGS_MAX)
+		return -EINVAL;
+
+	hal->num_shadow_reg_configured++;
+
+	target_reg = srng_config->reg_start[HAL_HP_OFFSET_IN_REG_START];
+	target_reg += srng_config->reg_size[HAL_HP_OFFSET_IN_REG_START] *
+		ring_num;
+
+	/* For destination ring, shadow the TP */
+	if (srng_config->ring_dir == HAL_SRNG_DIR_DST)
+		target_reg += HAL_OFFSET_FROM_HP_TO_TP;
+
+	hal->shadow_reg_addr[shadow_cfg_idx] = target_reg;
+
+	/* update hp/tp addr to hal structure*/
+	ath12k_wifi7_hal_srng_update_hp_tp_addr(ab, shadow_cfg_idx, ring_type,
+						ring_num);
+
+	ath12k_dbg(ab, ATH12K_DBG_HAL,
+		   "target_reg %x, shadow reg 0x%x shadow_idx 0x%x, ring_type %d, ring num %d",
+		  target_reg,
+		  HAL_SHADOW_REG(shadow_cfg_idx),
+		  shadow_cfg_idx,
+		  ring_type, ring_num);
+
+	return 0;
+}
+
+void ath12k_wifi7_hal_ce_src_set_desc(struct hal_ce_srng_src_desc *desc,
+				      dma_addr_t paddr,
+				      u32 len, u32 id, u8 byte_swap_data)
+{
+	desc->buffer_addr_low = cpu_to_le32(paddr & HAL_ADDR_LSB_REG_MASK);
+	desc->buffer_addr_info =
+		le32_encode_bits(((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT),
+				 HAL_CE_SRC_DESC_ADDR_INFO_ADDR_HI) |
+		le32_encode_bits(byte_swap_data,
+				 HAL_CE_SRC_DESC_ADDR_INFO_BYTE_SWAP) |
+		le32_encode_bits(0, HAL_CE_SRC_DESC_ADDR_INFO_GATHER) |
+		le32_encode_bits(len, HAL_CE_SRC_DESC_ADDR_INFO_LEN);
+	desc->meta_info = le32_encode_bits(id, HAL_CE_SRC_DESC_META_INFO_DATA);
+}
+
+void ath12k_wifi7_hal_ce_dst_set_desc(struct hal_ce_srng_dest_desc *desc,
+				      dma_addr_t paddr)
+{
+	desc->buffer_addr_low = cpu_to_le32(paddr & HAL_ADDR_LSB_REG_MASK);
+	desc->buffer_addr_info =
+		le32_encode_bits(((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT),
+				 HAL_CE_DEST_DESC_ADDR_INFO_ADDR_HI);
+}
+
+void ath12k_wifi7_hal_set_link_desc_addr(struct hal_wbm_link_desc *desc,
+					 u32 cookie, dma_addr_t paddr,
+					 enum hal_rx_buf_return_buf_manager rbm)
+{
+	desc->buf_addr_info.info0 = le32_encode_bits((paddr & HAL_ADDR_LSB_REG_MASK),
+						     BUFFER_ADDR_INFO0_ADDR);
+	desc->buf_addr_info.info1 =
+			le32_encode_bits(((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT),
+					 BUFFER_ADDR_INFO1_ADDR) |
+			le32_encode_bits(rbm, BUFFER_ADDR_INFO1_RET_BUF_MGR) |
+			le32_encode_bits(cookie, BUFFER_ADDR_INFO1_SW_COOKIE);
+}
+
+u32 ath12k_wifi7_hal_ce_dst_status_get_length(struct hal_ce_srng_dst_status_desc *desc)
+{
+	u32 len;
+
+	len = le32_get_bits(READ_ONCE(desc->flags), HAL_CE_DST_STATUS_DESC_FLAGS_LEN);
+	desc->flags &= ~cpu_to_le32(HAL_CE_DST_STATUS_DESC_FLAGS_LEN);
+
+	return len;
+}
+
+void
+ath12k_wifi7_hal_setup_link_idle_list(struct ath12k_base *ab,
+				      struct hal_wbm_idle_scatter_list *sbuf,
+				      u32 nsbufs, u32 tot_link_desc,
+				      u32 end_offset)
+{
+	struct ath12k_hal *hal = &ab->hal;
+	struct ath12k_buffer_addr *link_addr;
+	int i;
+	u32 reg_scatter_buf_sz = HAL_WBM_IDLE_SCATTER_BUF_SIZE / 64;
+	u32 val;
+
+	link_addr = (void *)sbuf[0].vaddr + HAL_WBM_IDLE_SCATTER_BUF_SIZE;
+
+	for (i = 1; i < nsbufs; i++) {
+		link_addr->info0 = cpu_to_le32(sbuf[i].paddr & HAL_ADDR_LSB_REG_MASK);
+
+		link_addr->info1 =
+			le32_encode_bits((u64)sbuf[i].paddr >> HAL_ADDR_MSB_REG_SHIFT,
+					 HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32) |
+			le32_encode_bits(BASE_ADDR_MATCH_TAG_VAL,
+					 HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG);
+
+		link_addr = (void *)sbuf[i].vaddr +
+			     HAL_WBM_IDLE_SCATTER_BUF_SIZE;
+	}
+
+	val = u32_encode_bits(reg_scatter_buf_sz, HAL_WBM_SCATTER_BUFFER_SIZE) |
+	      u32_encode_bits(0x1, HAL_WBM_LINK_DESC_IDLE_LIST_MODE);
+
+	ath12k_hif_write32(ab,
+			   HAL_SEQ_WCSS_UMAC_WBM_REG +
+			   HAL_WBM_R0_IDLE_LIST_CONTROL_ADDR(hal),
+			   val);
+
+	val = u32_encode_bits(reg_scatter_buf_sz * nsbufs,
+			      HAL_WBM_SCATTER_RING_SIZE_OF_IDLE_LINK_DESC_LIST);
+	ath12k_hif_write32(ab,
+			   HAL_SEQ_WCSS_UMAC_WBM_REG +
+			   HAL_WBM_R0_IDLE_LIST_SIZE_ADDR(hal),
+			   val);
+
+	val = u32_encode_bits(sbuf[0].paddr & HAL_ADDR_LSB_REG_MASK,
+			      BUFFER_ADDR_INFO0_ADDR);
+	ath12k_hif_write32(ab,
+			   HAL_SEQ_WCSS_UMAC_WBM_REG +
+			   HAL_WBM_SCATTERED_RING_BASE_LSB(hal),
+			   val);
+
+	val = u32_encode_bits(BASE_ADDR_MATCH_TAG_VAL,
+			      HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG) |
+	      u32_encode_bits((u64)sbuf[0].paddr >> HAL_ADDR_MSB_REG_SHIFT,
+			      HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32);
+	ath12k_hif_write32(ab,
+			   HAL_SEQ_WCSS_UMAC_WBM_REG +
+			   HAL_WBM_SCATTERED_RING_BASE_MSB(hal),
+			   val);
+
+	/* Setup head and tail pointers for the idle list */
+	val = u32_encode_bits(sbuf[nsbufs - 1].paddr, BUFFER_ADDR_INFO0_ADDR);
+	ath12k_hif_write32(ab,
+			   HAL_SEQ_WCSS_UMAC_WBM_REG +
+			   HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0(hal),
+			   val);
+
+	val = u32_encode_bits(((u64)sbuf[nsbufs - 1].paddr >> HAL_ADDR_MSB_REG_SHIFT),
+			      HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32) |
+	       u32_encode_bits((end_offset >> 2),
+			       HAL_WBM_SCATTERED_DESC_HEAD_P_OFFSET_IX1);
+	ath12k_hif_write32(ab,
+			   HAL_SEQ_WCSS_UMAC_WBM_REG +
+			   HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX1(hal),
+			   val);
+
+	val = u32_encode_bits(sbuf[0].paddr, BUFFER_ADDR_INFO0_ADDR);
+	ath12k_hif_write32(ab,
+			   HAL_SEQ_WCSS_UMAC_WBM_REG +
+			   HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0(hal),
+			   val);
+
+	val = u32_encode_bits(sbuf[0].paddr, BUFFER_ADDR_INFO0_ADDR);
+	ath12k_hif_write32(ab,
+			   HAL_SEQ_WCSS_UMAC_WBM_REG +
+			   HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX0(hal),
+			   val);
+
+	val = u32_encode_bits(((u64)sbuf[0].paddr >> HAL_ADDR_MSB_REG_SHIFT),
+			      HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32) |
+	      u32_encode_bits(0, HAL_WBM_SCATTERED_DESC_TAIL_P_OFFSET_IX1);
+	ath12k_hif_write32(ab,
+			   HAL_SEQ_WCSS_UMAC_WBM_REG +
+			   HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX1(hal),
+			   val);
+
+	val = 2 * tot_link_desc;
+	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_WBM_REG +
+			   HAL_WBM_SCATTERED_DESC_PTR_HP_ADDR(hal),
+			   val);
+
+	/* Enable the SRNG */
+	val = u32_encode_bits(1, HAL_WBM_IDLE_LINK_RING_MISC_SRNG_ENABLE) |
+	      u32_encode_bits(1, HAL_WBM_IDLE_LINK_RING_MISC_RIND_ID_DISABLE);
+	ath12k_hif_write32(ab,
+			   HAL_SEQ_WCSS_UMAC_WBM_REG +
+			   HAL_WBM_IDLE_LINK_RING_MISC_ADDR(hal),
+			   val);
+}
+
+void ath12k_wifi7_hal_tx_configure_bank_register(struct ath12k_base *ab,
+						 u32 bank_config,
+						 u8 bank_id)
+{
+	ath12k_hif_write32(ab, HAL_TCL_SW_CONFIG_BANK_ADDR + 4 * bank_id,
+			   bank_config);
+}
+
+void ath12k_wifi7_hal_reoq_lut_addr_read_enable(struct ath12k_base *ab)
+{
+	struct ath12k_hal *hal = &ab->hal;
+
+	u32 val = ath12k_hif_read32(ab, HAL_SEQ_WCSS_UMAC_REO_REG +
+				    HAL_REO1_QDESC_ADDR(hal));
+
+	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_ADDR(hal),
+			   val | HAL_REO_QDESC_ADDR_READ_LUT_ENABLE);
+}
+
+void ath12k_wifi7_hal_reoq_lut_set_max_peerid(struct ath12k_base *ab)
+{
+	struct ath12k_hal *hal = &ab->hal;
+
+	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_MAX_PEERID(hal),
+			   HAL_REO_QDESC_MAX_PEERID);
+}
+
+void ath12k_wifi7_hal_write_reoq_lut_addr(struct ath12k_base *ab,
+					  dma_addr_t paddr)
+{
+	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG +
+			   HAL_REO1_QDESC_LUT_BASE0(&ab->hal), paddr);
+}
+
+void ath12k_wifi7_hal_write_ml_reoq_lut_addr(struct ath12k_base *ab,
+					     dma_addr_t paddr)
+{
+	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG +
+			   HAL_REO1_QDESC_LUT_BASE1(&ab->hal), paddr);
+}
+
+void ath12k_wifi7_hal_cc_config(struct ath12k_base *ab)
+{
+	u32 cmem_base = ab->qmi.dev_mem[ATH12K_QMI_DEVMEM_CMEM_INDEX].start;
+	u32 reo_base = HAL_SEQ_WCSS_UMAC_REO_REG;
+	u32 wbm_base = HAL_SEQ_WCSS_UMAC_WBM_REG;
+	u32 val = 0;
+	struct ath12k_hal *hal = &ab->hal;
+
+	if (ath12k_ftm_mode)
+		return;
+
+	ath12k_hif_write32(ab, reo_base + HAL_REO1_SW_COOKIE_CFG0(hal), cmem_base);
+
+	val |= u32_encode_bits(ATH12K_CMEM_ADDR_MSB,
+			       HAL_REO1_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB) |
+		u32_encode_bits(ATH12K_CC_PPT_MSB,
+				HAL_REO1_SW_COOKIE_CFG_COOKIE_PPT_MSB) |
+		u32_encode_bits(ATH12K_CC_SPT_MSB,
+				HAL_REO1_SW_COOKIE_CFG_COOKIE_SPT_MSB) |
+		u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_ALIGN) |
+		u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_ENABLE) |
+		u32_encode_bits(1, HAL_REO1_SW_COOKIE_CFG_GLOBAL_ENABLE);
+
+	ath12k_hif_write32(ab, reo_base + HAL_REO1_SW_COOKIE_CFG1(hal), val);
+
+	/* Enable HW CC for WBM */
+	ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG0, cmem_base);
+
+	val = u32_encode_bits(ATH12K_CMEM_ADDR_MSB,
+			      HAL_WBM_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB) |
+		u32_encode_bits(ATH12K_CC_PPT_MSB,
+				HAL_WBM_SW_COOKIE_CFG_COOKIE_PPT_MSB) |
+		u32_encode_bits(ATH12K_CC_SPT_MSB,
+				HAL_WBM_SW_COOKIE_CFG_COOKIE_SPT_MSB) |
+		u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_ALIGN);
+
+	ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG1, val);
+
+	/* Enable conversion complete indication */
+	val = ath12k_hif_read32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG2);
+	val |= u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_RELEASE_PATH_EN) |
+		u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_ERR_PATH_EN) |
+		u32_encode_bits(1, HAL_WBM_SW_COOKIE_CFG_CONV_IND_EN);
+
+	ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CFG2, val);
+
+	/* Enable Cookie conversion for WBM2SW Rings */
+	val = ath12k_hif_read32(ab, wbm_base + HAL_WBM_SW_COOKIE_CONVERT_CFG);
+	val |= u32_encode_bits(1, HAL_WBM_SW_COOKIE_CONV_CFG_GLOBAL_EN) |
+	       hal->hal_params->wbm2sw_cc_enable;
+
+	ath12k_hif_write32(ab, wbm_base + HAL_WBM_SW_COOKIE_CONVERT_CFG, val);
+}
+
+enum hal_rx_buf_return_buf_manager
+ath12k_wifi7_hal_get_idle_link_rbm(struct ath12k_hal *hal, u8 device_id)
+{
+	switch (device_id) {
+	case 0:
+		return HAL_RX_BUF_RBM_WBM_DEV0_IDLE_DESC_LIST;
+	case 1:
+		return HAL_RX_BUF_RBM_WBM_DEV1_IDLE_DESC_LIST;
+	case 2:
+		return HAL_RX_BUF_RBM_WBM_DEV2_IDLE_DESC_LIST;
+	default:
+		ath12k_warn(hal,
+			    "invalid %d device id, so choose default rbm\n",
+			    device_id);
+		WARN_ON(1);
+		return HAL_RX_BUF_RBM_WBM_DEV0_IDLE_DESC_LIST;
+	}
+}
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal.h b/drivers/net/wireless/ath/ath12k/wifi7/hal.h
new file mode 100644
index 000000000000..7d65b82c61f2
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal.h
@@ -0,0 +1,564 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef ATH12K_HAL_WIFI7_H
+#define ATH12K_HAL_WIFI7_H
+
+#include "../core.h"
+#include "../hal.h"
+#include "hal_desc.h"
+#include "hal_tx.h"
+#include "hal_rx.h"
+#include "hal_rx_desc.h"
+
+/* calculate the register address from bar0 of shadow register x */
+#define HAL_SHADOW_BASE_ADDR			0x000008fc
+#define HAL_SHADOW_NUM_REGS			40
+#define HAL_HP_OFFSET_IN_REG_START		1
+#define HAL_OFFSET_FROM_HP_TO_TP		4
+
+#define HAL_SHADOW_REG(x) (HAL_SHADOW_BASE_ADDR + (4 * (x)))
+#define HAL_REO_QDESC_MAX_PEERID		8191
+
+/* WCSS Relative address */
+#define HAL_SEQ_WCSS_CMEM_OFFSET		0x00100000
+#define HAL_SEQ_WCSS_UMAC_OFFSET		0x00a00000
+#define HAL_SEQ_WCSS_UMAC_REO_REG		0x00a38000
+#define HAL_SEQ_WCSS_UMAC_TCL_REG		0x00a44000
+#define HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal) \
+	((hal)->regs->umac_ce0_src_reg_base)
+#define HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) \
+	((hal)->regs->umac_ce0_dest_reg_base)
+#define HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(hal) \
+	((hal)->regs->umac_ce1_src_reg_base)
+#define HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) \
+	((hal)->regs->umac_ce1_dest_reg_base)
+#define HAL_SEQ_WCSS_UMAC_WBM_REG		0x00a34000
+
+#define HAL_CE_WFSS_CE_REG_BASE			0x01b80000
+
+#define HAL_TCL_SW_CONFIG_BANK_ADDR		0x00a4408c
+
+/* SW2TCL(x) R0 ring configuration address */
+#define HAL_TCL1_RING_CMN_CTRL_REG		0x00000020
+#define HAL_TCL1_RING_DSCP_TID_MAP		0x00000240
+
+#define HAL_TCL1_RING_BASE_LSB(hal) \
+	((hal)->regs->tcl1_ring_base_lsb)
+#define HAL_TCL1_RING_BASE_MSB(hal) \
+	((hal)->regs->tcl1_ring_base_msb)
+#define HAL_TCL1_RING_ID(hal)		((hal)->regs->tcl1_ring_id)
+#define HAL_TCL1_RING_MISC(hal) \
+	((hal)->regs->tcl1_ring_misc)
+#define HAL_TCL1_RING_TP_ADDR_LSB(hal) \
+	((hal)->regs->tcl1_ring_tp_addr_lsb)
+#define HAL_TCL1_RING_TP_ADDR_MSB(hal) \
+	((hal)->regs->tcl1_ring_tp_addr_msb)
+#define HAL_TCL1_RING_CONSUMER_INT_SETUP_IX0(hal) \
+	((hal)->regs->tcl1_ring_consumer_int_setup_ix0)
+#define HAL_TCL1_RING_CONSUMER_INT_SETUP_IX1(hal) \
+	((hal)->regs->tcl1_ring_consumer_int_setup_ix1)
+#define HAL_TCL1_RING_MSI1_BASE_LSB(hal) \
+	((hal)->regs->tcl1_ring_msi1_base_lsb)
+#define HAL_TCL1_RING_MSI1_BASE_MSB(hal) \
+	((hal)->regs->tcl1_ring_msi1_base_msb)
+#define HAL_TCL1_RING_MSI1_DATA(hal) \
+	((hal)->regs->tcl1_ring_msi1_data)
+#define HAL_TCL2_RING_BASE_LSB(hal) \
+	((hal)->regs->tcl2_ring_base_lsb)
+#define HAL_TCL_RING_BASE_LSB(hal) \
+	((hal)->regs->tcl_ring_base_lsb)
+
+#define HAL_TCL1_RING_MSI1_BASE_LSB_OFFSET(hal) ({ typeof(hal) _hal = (hal); \
+	(HAL_TCL1_RING_MSI1_BASE_LSB(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); })
+#define HAL_TCL1_RING_MSI1_BASE_MSB_OFFSET(hal)	({ typeof(hal) _hal = (hal); \
+	(HAL_TCL1_RING_MSI1_BASE_MSB(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); })
+#define HAL_TCL1_RING_MSI1_DATA_OFFSET(hal) ({ typeof(hal) _hal = (hal); \
+	(HAL_TCL1_RING_MSI1_DATA(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); })
+#define HAL_TCL1_RING_BASE_MSB_OFFSET(hal) ({ typeof(hal) _hal = (hal); \
+	(HAL_TCL1_RING_BASE_MSB(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); })
+#define HAL_TCL1_RING_ID_OFFSET(hal) ({ typeof(hal) _hal = (hal); \
+	(HAL_TCL1_RING_ID(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); })
+#define HAL_TCL1_RING_CONSR_INT_SETUP_IX0_OFFSET(hal) ({ typeof(hal) _hal = (hal); \
+	(HAL_TCL1_RING_CONSUMER_INT_SETUP_IX0(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); })
+#define HAL_TCL1_RING_CONSR_INT_SETUP_IX1_OFFSET(hal) ({ typeof(hal) _hal = (hal); \
+	(HAL_TCL1_RING_CONSUMER_INT_SETUP_IX1(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); })
+#define HAL_TCL1_RING_TP_ADDR_LSB_OFFSET(hal) ({ typeof(hal) _hal = (hal); \
+	(HAL_TCL1_RING_TP_ADDR_LSB(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); })
+#define HAL_TCL1_RING_TP_ADDR_MSB_OFFSET(hal) ({ typeof(hal) _hal = (hal); \
+	(HAL_TCL1_RING_TP_ADDR_MSB(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); })
+#define HAL_TCL1_RING_MISC_OFFSET(hal) ({ typeof(hal) _hal = (hal); \
+	(HAL_TCL1_RING_MISC(_hal) - HAL_TCL1_RING_BASE_LSB(_hal)); })
+
+/* SW2TCL(x) R2 ring pointers (head/tail) address */
+#define HAL_TCL1_RING_HP			0x00002000
+#define HAL_TCL1_RING_TP			0x00002004
+#define HAL_TCL2_RING_HP			0x00002008
+#define HAL_TCL_RING_HP				0x00002028
+
+#define HAL_TCL1_RING_TP_OFFSET \
+		(HAL_TCL1_RING_TP - HAL_TCL1_RING_HP)
+
+/* TCL STATUS ring address */
+#define HAL_TCL_STATUS_RING_BASE_LSB(hal) \
+	((hal)->regs->tcl_status_ring_base_lsb)
+#define HAL_TCL_STATUS_RING_HP			0x00002048
+
+/* PPE2TCL1 Ring address */
+#define HAL_TCL_PPE2TCL1_RING_BASE_LSB		0x00000c48
+#define HAL_TCL_PPE2TCL1_RING_HP		0x00002038
+
+/* WBM PPE Release Ring address */
+#define HAL_WBM_PPE_RELEASE_RING_BASE_LSB(hal) \
+	((hal)->regs->ppe_rel_ring_base)
+#define HAL_WBM_PPE_RELEASE_RING_HP		0x00003020
+
+/* REO2SW(x) R0 ring configuration address */
+#define HAL_REO1_GEN_ENABLE			0x00000000
+#define HAL_REO1_MISC_CTRL_ADDR(hal) \
+	((hal)->regs->reo1_misc_ctrl_addr)
+#define HAL_REO1_DEST_RING_CTRL_IX_0		0x00000004
+#define HAL_REO1_DEST_RING_CTRL_IX_1		0x00000008
+#define HAL_REO1_DEST_RING_CTRL_IX_2		0x0000000c
+#define HAL_REO1_DEST_RING_CTRL_IX_3		0x00000010
+#define HAL_REO1_QDESC_ADDR(hal)		((hal)->regs->reo1_qdesc_addr)
+#define HAL_REO1_QDESC_MAX_PEERID(hal)	((hal)->regs->reo1_qdesc_max_peerid)
+#define HAL_REO1_SW_COOKIE_CFG0(hal)	((hal)->regs->reo1_sw_cookie_cfg0)
+#define HAL_REO1_SW_COOKIE_CFG1(hal)	((hal)->regs->reo1_sw_cookie_cfg1)
+#define HAL_REO1_QDESC_LUT_BASE0(hal)	((hal)->regs->reo1_qdesc_lut_base0)
+#define HAL_REO1_QDESC_LUT_BASE1(hal)	((hal)->regs->reo1_qdesc_lut_base1)
+#define HAL_REO1_RING_BASE_LSB(hal)	((hal)->regs->reo1_ring_base_lsb)
+#define HAL_REO1_RING_BASE_MSB(hal)	((hal)->regs->reo1_ring_base_msb)
+#define HAL_REO1_RING_ID(hal)		((hal)->regs->reo1_ring_id)
+#define HAL_REO1_RING_MISC(hal)		((hal)->regs->reo1_ring_misc)
+#define HAL_REO1_RING_HP_ADDR_LSB(hal)	((hal)->regs->reo1_ring_hp_addr_lsb)
+#define HAL_REO1_RING_HP_ADDR_MSB(hal)	((hal)->regs->reo1_ring_hp_addr_msb)
+#define HAL_REO1_RING_PRODUCER_INT_SETUP(hal) \
+	((hal)->regs->reo1_ring_producer_int_setup)
+#define HAL_REO1_RING_MSI1_BASE_LSB(hal)	\
+	((hal)->regs->reo1_ring_msi1_base_lsb)
+#define HAL_REO1_RING_MSI1_BASE_MSB(hal)	\
+	((hal)->regs->reo1_ring_msi1_base_msb)
+#define HAL_REO1_RING_MSI1_DATA(hal)	((hal)->regs->reo1_ring_msi1_data)
+#define HAL_REO2_RING_BASE_LSB(hal)	((hal)->regs->reo2_ring_base)
+#define HAL_REO1_AGING_THRESH_IX_0(hal)	((hal)->regs->reo1_aging_thres_ix0)
+#define HAL_REO1_AGING_THRESH_IX_1(hal)	((hal)->regs->reo1_aging_thres_ix1)
+#define HAL_REO1_AGING_THRESH_IX_2(hal)	((hal)->regs->reo1_aging_thres_ix2)
+#define HAL_REO1_AGING_THRESH_IX_3(hal)	((hal)->regs->reo1_aging_thres_ix3)
+
+/* REO2SW(x) R2 ring pointers (head/tail) address */
+#define HAL_REO1_RING_HP			0x00003048
+#define HAL_REO1_RING_TP			0x0000304c
+#define HAL_REO2_RING_HP			0x00003050
+
+#define HAL_REO1_RING_TP_OFFSET			(HAL_REO1_RING_TP - HAL_REO1_RING_HP)
+
+/* REO2SW0 ring configuration address */
+#define HAL_REO_SW0_RING_BASE_LSB(hal) \
+	((hal)->regs->reo2_sw0_ring_base)
+
+/* REO2SW0 R2 ring pointer (head/tail) address */
+#define HAL_REO_SW0_RING_HP			0x00003088
+
+/* REO CMD R0 address */
+#define HAL_REO_CMD_RING_BASE_LSB(hal) \
+	((hal)->regs->reo_cmd_ring_base)
+
+/* REO CMD R2 address */
+#define HAL_REO_CMD_HP				0x00003020
+
+/* SW2REO R0 address */
+#define	HAL_SW2REO_RING_BASE_LSB(hal) \
+	((hal)->regs->sw2reo_ring_base)
+#define HAL_SW2REO1_RING_BASE_LSB(hal) \
+	((hal)->regs->sw2reo1_ring_base)
+
+/* SW2REO R2 address */
+#define HAL_SW2REO_RING_HP			0x00003028
+#define HAL_SW2REO1_RING_HP			0x00003030
+
+/* CE ring R0 address */
+#define HAL_CE_SRC_RING_BASE_LSB                0x00000000
+#define HAL_CE_DST_RING_BASE_LSB		0x00000000
+#define HAL_CE_DST_STATUS_RING_BASE_LSB		0x00000058
+#define HAL_CE_DST_RING_CTRL			0x000000b0
+
+/* CE ring R2 address */
+#define HAL_CE_DST_RING_HP			0x00000400
+#define HAL_CE_DST_STATUS_RING_HP		0x00000408
+
+/* REO status address */
+#define HAL_REO_STATUS_RING_BASE_LSB(hal) \
+	((hal)->regs->reo_status_ring_base)
+#define HAL_REO_STATUS_HP			0x000030a8
+
+/* WBM Idle R0 address */
+#define HAL_WBM_IDLE_LINK_RING_BASE_LSB(hal) \
+	((hal)->regs->wbm_idle_ring_base_lsb)
+#define HAL_WBM_IDLE_LINK_RING_MISC_ADDR(hal) \
+	((hal)->regs->wbm_idle_ring_misc_addr)
+#define HAL_WBM_R0_IDLE_LIST_CONTROL_ADDR(hal) \
+	((hal)->regs->wbm_r0_idle_list_cntl_addr)
+#define HAL_WBM_R0_IDLE_LIST_SIZE_ADDR(hal) \
+	((hal)->regs->wbm_r0_idle_list_size_addr)
+#define HAL_WBM_SCATTERED_RING_BASE_LSB(hal) \
+	((hal)->regs->wbm_scattered_ring_base_lsb)
+#define HAL_WBM_SCATTERED_RING_BASE_MSB(hal) \
+	((hal)->regs->wbm_scattered_ring_base_msb)
+#define HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0(hal) \
+	((hal)->regs->wbm_scattered_desc_head_info_ix0)
+#define HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX1(hal) \
+	((hal)->regs->wbm_scattered_desc_head_info_ix1)
+#define HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX0(hal) \
+	((hal)->regs->wbm_scattered_desc_tail_info_ix0)
+#define HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX1(hal) \
+	((hal)->regs->wbm_scattered_desc_tail_info_ix1)
+#define HAL_WBM_SCATTERED_DESC_PTR_HP_ADDR(hal) \
+	((hal)->regs->wbm_scattered_desc_ptr_hp_addr)
+
+/* WBM Idle R2 address */
+#define HAL_WBM_IDLE_LINK_RING_HP		0x000030b8
+
+/* SW2WBM R0 release address */
+#define HAL_WBM_SW_RELEASE_RING_BASE_LSB(hal) \
+	((hal)->regs->wbm_sw_release_ring_base_lsb)
+#define HAL_WBM_SW1_RELEASE_RING_BASE_LSB(hal) \
+	((hal)->regs->wbm_sw1_release_ring_base_lsb)
+
+/* SW2WBM R2 release address */
+#define HAL_WBM_SW_RELEASE_RING_HP		0x00003010
+#define HAL_WBM_SW1_RELEASE_RING_HP		0x00003018
+
+/* WBM2SW R0 release address */
+#define HAL_WBM0_RELEASE_RING_BASE_LSB(hal) \
+	((hal)->regs->wbm0_release_ring_base_lsb)
+
+#define HAL_WBM1_RELEASE_RING_BASE_LSB(hal) \
+	((hal)->regs->wbm1_release_ring_base_lsb)
+
+/* WBM2SW R2 release address */
+#define HAL_WBM0_RELEASE_RING_HP		0x000030c8
+#define HAL_WBM1_RELEASE_RING_HP		0x000030d0
+
+/* WBM cookie config address and mask */
+#define HAL_WBM_SW_COOKIE_CFG0			0x00000040
+#define HAL_WBM_SW_COOKIE_CFG1			0x00000044
+#define HAL_WBM_SW_COOKIE_CFG2			0x00000090
+#define HAL_WBM_SW_COOKIE_CONVERT_CFG		0x00000094
+
+#define HAL_WBM_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB	GENMASK(7, 0)
+#define HAL_WBM_SW_COOKIE_CFG_COOKIE_PPT_MSB		GENMASK(12, 8)
+#define HAL_WBM_SW_COOKIE_CFG_COOKIE_SPT_MSB		GENMASK(17, 13)
+#define HAL_WBM_SW_COOKIE_CFG_ALIGN			BIT(18)
+#define HAL_WBM_SW_COOKIE_CFG_RELEASE_PATH_EN		BIT(0)
+#define HAL_WBM_SW_COOKIE_CFG_ERR_PATH_EN		BIT(1)
+#define HAL_WBM_SW_COOKIE_CFG_CONV_IND_EN		BIT(3)
+
+#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN		BIT(1)
+#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW1_EN		BIT(2)
+#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW2_EN		BIT(3)
+#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW3_EN		BIT(4)
+#define HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN		BIT(5)
+#define HAL_WBM_SW_COOKIE_CONV_CFG_GLOBAL_EN		BIT(8)
+
+/* TCL ring field mask and offset */
+#define HAL_TCL1_RING_BASE_MSB_RING_SIZE		GENMASK(27, 8)
+#define HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB	GENMASK(7, 0)
+#define HAL_TCL1_RING_ID_ENTRY_SIZE			GENMASK(7, 0)
+#define HAL_TCL1_RING_MISC_MSI_RING_ID_DISABLE		BIT(0)
+#define HAL_TCL1_RING_MISC_MSI_LOOPCNT_DISABLE		BIT(1)
+#define HAL_TCL1_RING_MISC_MSI_SWAP			BIT(3)
+#define HAL_TCL1_RING_MISC_HOST_FW_SWAP			BIT(4)
+#define HAL_TCL1_RING_MISC_DATA_TLV_SWAP		BIT(5)
+#define HAL_TCL1_RING_MISC_SRNG_ENABLE			BIT(6)
+#define HAL_TCL1_RING_CONSR_INT_SETUP_IX0_INTR_TMR_THOLD   GENMASK(31, 16)
+#define HAL_TCL1_RING_CONSR_INT_SETUP_IX0_BATCH_COUNTER_THOLD GENMASK(14, 0)
+#define HAL_TCL1_RING_CONSR_INT_SETUP_IX1_LOW_THOLD	GENMASK(15, 0)
+#define HAL_TCL1_RING_MSI1_BASE_MSB_MSI1_ENABLE		BIT(8)
+#define HAL_TCL1_RING_MSI1_BASE_MSB_ADDR		GENMASK(7, 0)
+#define HAL_TCL1_RING_CMN_CTRL_DSCP_TID_MAP_PROG_EN	BIT(23)
+#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP		GENMASK(31, 0)
+#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP0		GENMASK(2, 0)
+#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP1		GENMASK(5, 3)
+#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP2		GENMASK(8, 6)
+#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP3		GENMASK(11, 9)
+#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP4		GENMASK(14, 12)
+#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP5		GENMASK(17, 15)
+#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP6		GENMASK(20, 18)
+#define HAL_TCL1_RING_FIELD_DSCP_TID_MAP7		GENMASK(23, 21)
+
+/* REO ring field mask and offset */
+#define HAL_REO1_RING_BASE_MSB_RING_SIZE		GENMASK(27, 8)
+#define HAL_REO1_RING_BASE_MSB_RING_BASE_ADDR_MSB	GENMASK(7, 0)
+#define HAL_REO1_RING_ID_RING_ID			GENMASK(15, 8)
+#define HAL_REO1_RING_ID_ENTRY_SIZE			GENMASK(7, 0)
+#define HAL_REO1_RING_MISC_MSI_SWAP			BIT(3)
+#define HAL_REO1_RING_MISC_HOST_FW_SWAP			BIT(4)
+#define HAL_REO1_RING_MISC_DATA_TLV_SWAP		BIT(5)
+#define HAL_REO1_RING_MISC_SRNG_ENABLE			BIT(6)
+#define HAL_REO1_RING_PRDR_INT_SETUP_INTR_TMR_THOLD	GENMASK(31, 16)
+#define HAL_REO1_RING_PRDR_INT_SETUP_BATCH_COUNTER_THOLD GENMASK(14, 0)
+#define HAL_REO1_RING_MSI1_BASE_MSB_MSI1_ENABLE		BIT(8)
+#define HAL_REO1_RING_MSI1_BASE_MSB_ADDR		GENMASK(7, 0)
+#define HAL_REO1_MISC_CTL_FRAG_DST_RING			GENMASK(20, 17)
+#define HAL_REO1_MISC_CTL_BAR_DST_RING			GENMASK(24, 21)
+#define HAL_REO1_GEN_ENABLE_AGING_LIST_ENABLE		BIT(2)
+#define HAL_REO1_GEN_ENABLE_AGING_FLUSH_ENABLE		BIT(3)
+#define HAL_REO1_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB	GENMASK(7, 0)
+#define HAL_REO1_SW_COOKIE_CFG_COOKIE_PPT_MSB		GENMASK(12, 8)
+#define HAL_REO1_SW_COOKIE_CFG_COOKIE_SPT_MSB		GENMASK(17, 13)
+#define HAL_REO1_SW_COOKIE_CFG_ALIGN			BIT(18)
+#define HAL_REO1_SW_COOKIE_CFG_ENABLE			BIT(19)
+#define HAL_REO1_SW_COOKIE_CFG_GLOBAL_ENABLE		BIT(20)
+#define HAL_REO_QDESC_ADDR_READ_LUT_ENABLE		BIT(7)
+#define HAL_REO_QDESC_ADDR_READ_CLEAR_QDESC_ARRAY	BIT(6)
+
+/* CE ring bit field mask and shift */
+#define HAL_CE_DST_R0_DEST_CTRL_MAX_LEN			GENMASK(15, 0)
+
+#define HAL_ADDR_LSB_REG_MASK				0xffffffff
+
+#define HAL_ADDR_MSB_REG_SHIFT				32
+
+/* WBM ring bit field mask and shift */
+#define HAL_WBM_LINK_DESC_IDLE_LIST_MODE		BIT(1)
+#define HAL_WBM_SCATTER_BUFFER_SIZE			GENMASK(10, 2)
+#define HAL_WBM_SCATTER_RING_SIZE_OF_IDLE_LINK_DESC_LIST GENMASK(31, 16)
+#define HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32	GENMASK(7, 0)
+#define HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG	GENMASK(31, 8)
+
+#define HAL_WBM_SCATTERED_DESC_HEAD_P_OFFSET_IX1	GENMASK(20, 8)
+#define HAL_WBM_SCATTERED_DESC_TAIL_P_OFFSET_IX1	GENMASK(20, 8)
+
+#define HAL_WBM_IDLE_LINK_RING_MISC_SRNG_ENABLE		BIT(6)
+#define HAL_WBM_IDLE_LINK_RING_MISC_RIND_ID_DISABLE	BIT(0)
+
+#define BASE_ADDR_MATCH_TAG_VAL 0x5
+
+#define HAL_REO_REO2SW1_RING_BASE_MSB_RING_SIZE		0x000fffff
+#define HAL_REO_REO2SW0_RING_BASE_MSB_RING_SIZE		0x000fffff
+#define HAL_REO_SW2REO_RING_BASE_MSB_RING_SIZE		0x0000ffff
+#define HAL_REO_CMD_RING_BASE_MSB_RING_SIZE		0x0000ffff
+#define HAL_REO_STATUS_RING_BASE_MSB_RING_SIZE		0x0000ffff
+#define HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE		0x000fffff
+#define HAL_SW2TCL1_CMD_RING_BASE_MSB_RING_SIZE		0x000fffff
+#define HAL_TCL_STATUS_RING_BASE_MSB_RING_SIZE		0x0000ffff
+#define HAL_CE_SRC_RING_BASE_MSB_RING_SIZE		0x0000ffff
+#define HAL_CE_DST_RING_BASE_MSB_RING_SIZE		0x0000ffff
+#define HAL_CE_DST_STATUS_RING_BASE_MSB_RING_SIZE	0x0000ffff
+#define HAL_WBM_IDLE_LINK_RING_BASE_MSB_RING_SIZE	0x000fffff
+#define HAL_SW2WBM_RELEASE_RING_BASE_MSB_RING_SIZE	0x0000ffff
+#define HAL_WBM2SW_RELEASE_RING_BASE_MSB_RING_SIZE	0x000fffff
+#define HAL_RXDMA_RING_MAX_SIZE				0x0000ffff
+#define HAL_RXDMA_RING_MAX_SIZE_BE			0x000fffff
+#define HAL_WBM2PPE_RELEASE_RING_BASE_MSB_RING_SIZE	0x000fffff
+
+#define HAL_WBM2SW_REL_ERR_RING_NUM 3
+/* Add any other errors here and return them in
+ * ath12k_hal_rx_desc_get_err().
+ */
+
+#define HAL_IPQ5332_CE_WFSS_REG_BASE	0x740000
+#define HAL_IPQ5332_CE_SIZE		0x100000
+
+#define HAL_RX_MAX_BA_WINDOW	256
+
+#define HAL_DEFAULT_BE_BK_VI_REO_TIMEOUT_USEC	(100 * 1000)
+#define HAL_DEFAULT_VO_REO_TIMEOUT_USEC		(40 * 1000)
+
+#define HAL_SRNG_TLV_HDR_TAG		GENMASK(9, 1)
+#define HAL_SRNG_TLV_HDR_LEN		GENMASK(25, 10)
+
+#define HAL_SRNG_DESC_LOOP_CNT		0xf0000000
+
+#define HAL_REO_CMD_FLG_NEED_STATUS		BIT(0)
+#define HAL_REO_CMD_FLG_STATS_CLEAR		BIT(1)
+#define HAL_REO_CMD_FLG_FLUSH_BLOCK_LATER	BIT(2)
+#define HAL_REO_CMD_FLG_FLUSH_RELEASE_BLOCKING	BIT(3)
+#define HAL_REO_CMD_FLG_FLUSH_NO_INVAL		BIT(4)
+#define HAL_REO_CMD_FLG_FLUSH_FWD_ALL_MPDUS	BIT(5)
+#define HAL_REO_CMD_FLG_FLUSH_ALL		BIT(6)
+#define HAL_REO_CMD_FLG_UNBLK_RESOURCE		BIT(7)
+#define HAL_REO_CMD_FLG_UNBLK_CACHE		BIT(8)
+#define HAL_REO_CMD_FLG_FLUSH_QUEUE_1K_DESC	BIT(9)
+
+/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO0_UPD_* fields */
+#define HAL_REO_CMD_UPD0_RX_QUEUE_NUM		BIT(8)
+#define HAL_REO_CMD_UPD0_VLD			BIT(9)
+#define HAL_REO_CMD_UPD0_ALDC			BIT(10)
+#define HAL_REO_CMD_UPD0_DIS_DUP_DETECTION	BIT(11)
+#define HAL_REO_CMD_UPD0_SOFT_REORDER_EN	BIT(12)
+#define HAL_REO_CMD_UPD0_AC			BIT(13)
+#define HAL_REO_CMD_UPD0_BAR			BIT(14)
+#define HAL_REO_CMD_UPD0_RETRY			BIT(15)
+#define HAL_REO_CMD_UPD0_CHECK_2K_MODE		BIT(16)
+#define HAL_REO_CMD_UPD0_OOR_MODE		BIT(17)
+#define HAL_REO_CMD_UPD0_BA_WINDOW_SIZE		BIT(18)
+#define HAL_REO_CMD_UPD0_PN_CHECK		BIT(19)
+#define HAL_REO_CMD_UPD0_EVEN_PN		BIT(20)
+#define HAL_REO_CMD_UPD0_UNEVEN_PN		BIT(21)
+#define HAL_REO_CMD_UPD0_PN_HANDLE_ENABLE	BIT(22)
+#define HAL_REO_CMD_UPD0_PN_SIZE		BIT(23)
+#define HAL_REO_CMD_UPD0_IGNORE_AMPDU_FLG	BIT(24)
+#define HAL_REO_CMD_UPD0_SVLD			BIT(25)
+#define HAL_REO_CMD_UPD0_SSN			BIT(26)
+#define HAL_REO_CMD_UPD0_SEQ_2K_ERR		BIT(27)
+#define HAL_REO_CMD_UPD0_PN_ERR			BIT(28)
+#define HAL_REO_CMD_UPD0_PN_VALID		BIT(29)
+#define HAL_REO_CMD_UPD0_PN			BIT(30)
+
+/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO1_* fields */
+#define HAL_REO_CMD_UPD1_VLD			BIT(16)
+#define HAL_REO_CMD_UPD1_ALDC			GENMASK(18, 17)
+#define HAL_REO_CMD_UPD1_DIS_DUP_DETECTION	BIT(19)
+#define HAL_REO_CMD_UPD1_SOFT_REORDER_EN	BIT(20)
+#define HAL_REO_CMD_UPD1_AC			GENMASK(22, 21)
+#define HAL_REO_CMD_UPD1_BAR			BIT(23)
+#define HAL_REO_CMD_UPD1_RETRY			BIT(24)
+#define HAL_REO_CMD_UPD1_CHECK_2K_MODE		BIT(25)
+#define HAL_REO_CMD_UPD1_OOR_MODE		BIT(26)
+#define HAL_REO_CMD_UPD1_PN_CHECK		BIT(27)
+#define HAL_REO_CMD_UPD1_EVEN_PN		BIT(28)
+#define HAL_REO_CMD_UPD1_UNEVEN_PN		BIT(29)
+#define HAL_REO_CMD_UPD1_PN_HANDLE_ENABLE	BIT(30)
+#define HAL_REO_CMD_UPD1_IGNORE_AMPDU_FLG	BIT(31)
+
+/* Should be matching with HAL_REO_UPD_RX_QUEUE_INFO2_* fields */
+#define HAL_REO_CMD_UPD2_SVLD			BIT(10)
+#define HAL_REO_CMD_UPD2_SSN			GENMASK(22, 11)
+#define HAL_REO_CMD_UPD2_SEQ_2K_ERR		BIT(23)
+#define HAL_REO_CMD_UPD2_PN_ERR			BIT(24)
+
+struct hal_reo_status_queue_stats {
+	u16 ssn;
+	u16 curr_idx;
+	u32 pn[4];
+	u32 last_rx_queue_ts;
+	u32 last_rx_dequeue_ts;
+	u32 rx_bitmap[8]; /* Bitmap from 0-255 */
+	u32 curr_mpdu_cnt;
+	u32 curr_msdu_cnt;
+	u16 fwd_due_to_bar_cnt;
+	u16 dup_cnt;
+	u32 frames_in_order_cnt;
+	u32 num_mpdu_processed_cnt;
+	u32 num_msdu_processed_cnt;
+	u32 total_num_processed_byte_cnt;
+	u32 late_rx_mpdu_cnt;
+	u32 reorder_hole_cnt;
+	u8 timeout_cnt;
+	u8 bar_rx_cnt;
+	u8 num_window_2k_jump_cnt;
+};
+
+struct hal_reo_status_flush_queue {
+	bool err_detected;
+};
+
+enum hal_reo_status_flush_cache_err_code {
+	HAL_REO_STATUS_FLUSH_CACHE_ERR_CODE_SUCCESS,
+	HAL_REO_STATUS_FLUSH_CACHE_ERR_CODE_IN_USE,
+	HAL_REO_STATUS_FLUSH_CACHE_ERR_CODE_NOT_FOUND,
+};
+
+struct hal_reo_status_flush_cache {
+	bool err_detected;
+	enum hal_reo_status_flush_cache_err_code err_code;
+	bool cache_controller_flush_status_hit;
+	u8 cache_controller_flush_status_desc_type;
+	u8 cache_controller_flush_status_client_id;
+	u8 cache_controller_flush_status_err;
+	u8 cache_controller_flush_status_cnt;
+};
+
+enum hal_reo_status_unblock_cache_type {
+	HAL_REO_STATUS_UNBLOCK_BLOCKING_RESOURCE,
+	HAL_REO_STATUS_UNBLOCK_ENTIRE_CACHE_USAGE,
+};
+
+struct hal_reo_status_unblock_cache {
+	bool err_detected;
+	enum hal_reo_status_unblock_cache_type unblock_type;
+};
+
+struct hal_reo_status_flush_timeout_list {
+	bool err_detected;
+	bool list_empty;
+	u16 release_desc_cnt;
+	u16 fwd_buf_cnt;
+};
+
+enum hal_reo_threshold_idx {
+	HAL_REO_THRESHOLD_IDX_DESC_COUNTER0,
+	HAL_REO_THRESHOLD_IDX_DESC_COUNTER1,
+	HAL_REO_THRESHOLD_IDX_DESC_COUNTER2,
+	HAL_REO_THRESHOLD_IDX_DESC_COUNTER_SUM,
+};
+
+struct hal_reo_status_desc_thresh_reached {
+	enum hal_reo_threshold_idx threshold_idx;
+	u32 link_desc_counter0;
+	u32 link_desc_counter1;
+	u32 link_desc_counter2;
+	u32 link_desc_counter_sum;
+};
+
+struct hal_reo_status {
+	struct hal_reo_status_header uniform_hdr;
+	u8 loop_cnt;
+	union {
+		struct hal_reo_status_queue_stats queue_stats;
+		struct hal_reo_status_flush_queue flush_queue;
+		struct hal_reo_status_flush_cache flush_cache;
+		struct hal_reo_status_unblock_cache unblock_cache;
+		struct hal_reo_status_flush_timeout_list timeout_list;
+		struct hal_reo_status_desc_thresh_reached desc_thresh_reached;
+	} u;
+};
+
+int ath12k_wifi7_hal_init(struct ath12k_base *ab);
+void ath12k_wifi7_hal_ce_dst_setup(struct ath12k_base *ab,
+				   struct hal_srng *srng, int ring_num);
+void ath12k_wifi7_hal_srng_dst_hw_init(struct ath12k_base *ab,
+				       struct hal_srng *srng);
+void ath12k_wifi7_hal_srng_src_hw_init(struct ath12k_base *ab,
+				       struct hal_srng *srng);
+void ath12k_wifi7_hal_set_umac_srng_ptr_addr(struct ath12k_base *ab,
+					     struct hal_srng *srng);
+int ath12k_wifi7_hal_srng_update_shadow_config(struct ath12k_base *ab,
+					       enum hal_ring_type ring_type,
+					       int ring_num);
+int ath12k_wifi7_hal_srng_get_ring_id(struct ath12k_hal *hal,
+				      enum hal_ring_type type,
+				      int ring_num, int mac_id);
+u32 ath12k_wifi7_hal_ce_get_desc_size(enum hal_ce_desc type);
+void ath12k_wifi7_hal_cc_config(struct ath12k_base *ab);
+enum hal_rx_buf_return_buf_manager
+ath12k_wifi7_hal_get_idle_link_rbm(struct ath12k_hal *hal, u8 device_id);
+void ath12k_wifi7_hal_ce_src_set_desc(struct hal_ce_srng_src_desc *desc,
+				      dma_addr_t paddr,
+				      u32 len, u32 id, u8 byte_swap_data);
+void ath12k_wifi7_hal_ce_dst_set_desc(struct hal_ce_srng_dest_desc *desc,
+				      dma_addr_t paddr);
+void
+ath12k_wifi7_hal_set_link_desc_addr(struct hal_wbm_link_desc *desc,
+				    u32 cookie, dma_addr_t paddr,
+				    enum hal_rx_buf_return_buf_manager rbm);
+u32
+ath12k_wifi7_hal_ce_dst_status_get_length(struct hal_ce_srng_dst_status_desc *desc);
+void
+ath12k_wifi7_hal_setup_link_idle_list(struct ath12k_base *ab,
+				      struct hal_wbm_idle_scatter_list *sbuf,
+				      u32 nsbufs, u32 tot_link_desc,
+				      u32 end_offset);
+void ath12k_wifi7_hal_reoq_lut_addr_read_enable(struct ath12k_base *ab);
+void ath12k_wifi7_hal_reoq_lut_set_max_peerid(struct ath12k_base *ab);
+void ath12k_wifi7_hal_write_reoq_lut_addr(struct ath12k_base *ab,
+					  dma_addr_t paddr);
+void ath12k_wifi7_hal_write_ml_reoq_lut_addr(struct ath12k_base *ab,
+					     dma_addr_t paddr);
+u32 ath12k_wifi7_hal_reo_qdesc_size(u32 ba_window_size, u8 tid);
+#endif
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_desc.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_desc.h
new file mode 100644
index 000000000000..9dad8f7ca9f2
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_desc.h
@@ -0,0 +1,2751 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+#include "../core.h"
+
+#ifndef ATH12K_HAL_DESC_H
+#define ATH12K_HAL_DESC_H
+
+enum hal_tlv_tag {
+	HAL_MACTX_CBF_START					= 0 /* 0x0 */,
+	HAL_PHYRX_DATA						= 1 /* 0x1 */,
+	HAL_PHYRX_CBF_DATA_RESP					= 2 /* 0x2 */,
+	HAL_PHYRX_ABORT_REQUEST					= 3 /* 0x3 */,
+	HAL_PHYRX_USER_ABORT_NOTIFICATION			= 4 /* 0x4 */,
+	HAL_MACTX_DATA_RESP					= 5 /* 0x5 */,
+	HAL_MACTX_CBF_DATA					= 6 /* 0x6 */,
+	HAL_MACTX_CBF_DONE					= 7 /* 0x7 */,
+	HAL_PHYRX_LMR_DATA_RESP					= 8 /* 0x8 */,
+	HAL_RXPCU_TO_UCODE_START				= 9 /* 0x9 */,
+	HAL_RXPCU_TO_UCODE_DELIMITER_FOR_FULL_MPDU		= 10 /* 0xa */,
+	HAL_RXPCU_TO_UCODE_FULL_MPDU_DATA			= 11 /* 0xb */,
+	HAL_RXPCU_TO_UCODE_FCS_STATUS				= 12 /* 0xc */,
+	HAL_RXPCU_TO_UCODE_MPDU_DELIMITER			= 13 /* 0xd */,
+	HAL_RXPCU_TO_UCODE_DELIMITER_FOR_MPDU_HEADER		= 14 /* 0xe */,
+	HAL_RXPCU_TO_UCODE_MPDU_HEADER_DATA			= 15 /* 0xf */,
+	HAL_RXPCU_TO_UCODE_END					= 16 /* 0x10 */,
+	HAL_MACRX_CBF_READ_REQUEST				= 32 /* 0x20 */,
+	HAL_MACRX_CBF_DATA_REQUEST				= 33 /* 0x21 */,
+	HAL_MACRXXPECT_NDP_RECEPTION				= 34 /* 0x22 */,
+	HAL_MACRX_FREEZE_CAPTURE_CHANNEL			= 35 /* 0x23 */,
+	HAL_MACRX_NDP_TIMEOUT					= 36 /* 0x24 */,
+	HAL_MACRX_ABORT_ACK					= 37 /* 0x25 */,
+	HAL_MACRX_REQ_IMPLICIT_FB				= 38 /* 0x26 */,
+	HAL_MACRX_CHAIN_MASK					= 39 /* 0x27 */,
+	HAL_MACRX_NAP_USER					= 40 /* 0x28 */,
+	HAL_MACRX_ABORT_REQUEST					= 41 /* 0x29 */,
+	HAL_PHYTX_OTHER_TRANSMIT_INFO16				= 42 /* 0x2a */,
+	HAL_PHYTX_ABORT_ACK					= 43 /* 0x2b */,
+	HAL_PHYTX_ABORT_REQUEST					= 44 /* 0x2c */,
+	HAL_PHYTX_PKT_END					= 45 /* 0x2d */,
+	HAL_PHYTX_PPDU_HEADER_INFO_REQUEST			= 46 /* 0x2e */,
+	HAL_PHYTX_REQUEST_CTRL_INFO				= 47 /* 0x2f */,
+	HAL_PHYTX_DATA_REQUEST					= 48 /* 0x30 */,
+	HAL_PHYTX_BF_CV_LOADING_DONE				= 49 /* 0x31 */,
+	HAL_PHYTX_NAP_ACK					= 50 /* 0x32 */,
+	HAL_PHYTX_NAP_DONE					= 51 /* 0x33 */,
+	HAL_PHYTX_OFF_ACK					= 52 /* 0x34 */,
+	HAL_PHYTX_ON_ACK					= 53 /* 0x35 */,
+	HAL_PHYTX_SYNTH_OFF_ACK					= 54 /* 0x36 */,
+	HAL_PHYTX_DEBUG16					= 55 /* 0x37 */,
+	HAL_MACTX_ABORT_REQUEST					= 56 /* 0x38 */,
+	HAL_MACTX_ABORT_ACK					= 57 /* 0x39 */,
+	HAL_MACTX_PKT_END					= 58 /* 0x3a */,
+	HAL_MACTX_PRE_PHY_DESC					= 59 /* 0x3b */,
+	HAL_MACTX_BF_PARAMS_COMMON				= 60 /* 0x3c */,
+	HAL_MACTX_BF_PARAMS_PER_USER				= 61 /* 0x3d */,
+	HAL_MACTX_PREFETCH_CV					= 62 /* 0x3e */,
+	HAL_MACTX_USER_DESC_COMMON				= 63 /* 0x3f */,
+	HAL_MACTX_USER_DESC_PER_USER				= 64 /* 0x40 */,
+	HAL_XAMPLE_USER_TLV_16					= 65 /* 0x41 */,
+	HAL_XAMPLE_TLV_16					= 66 /* 0x42 */,
+	HAL_MACTX_PHY_OFF					= 67 /* 0x43 */,
+	HAL_MACTX_PHY_ON					= 68 /* 0x44 */,
+	HAL_MACTX_SYNTH_OFF					= 69 /* 0x45 */,
+	HAL_MACTXXPECT_CBF_COMMON				= 70 /* 0x46 */,
+	HAL_MACTXXPECT_CBF_PER_USER				= 71 /* 0x47 */,
+	HAL_MACTX_PHY_DESC					= 72 /* 0x48 */,
+	HAL_MACTX_L_SIG_A					= 73 /* 0x49 */,
+	HAL_MACTX_L_SIG_B					= 74 /* 0x4a */,
+	HAL_MACTX_HT_SIG					= 75 /* 0x4b */,
+	HAL_MACTX_VHT_SIG_A					= 76 /* 0x4c */,
+	HAL_MACTX_VHT_SIG_B_SU20				= 77 /* 0x4d */,
+	HAL_MACTX_VHT_SIG_B_SU40				= 78 /* 0x4e */,
+	HAL_MACTX_VHT_SIG_B_SU80				= 79 /* 0x4f */,
+	HAL_MACTX_VHT_SIG_B_SU160				= 80 /* 0x50 */,
+	HAL_MACTX_VHT_SIG_B_MU20				= 81 /* 0x51 */,
+	HAL_MACTX_VHT_SIG_B_MU40				= 82 /* 0x52 */,
+	HAL_MACTX_VHT_SIG_B_MU80				= 83 /* 0x53 */,
+	HAL_MACTX_VHT_SIG_B_MU160				= 84 /* 0x54 */,
+	HAL_MACTX_SERVICE					= 85 /* 0x55 */,
+	HAL_MACTX_HE_SIG_A_SU					= 86 /* 0x56 */,
+	HAL_MACTX_HE_SIG_A_MU_DL				= 87 /* 0x57 */,
+	HAL_MACTX_HE_SIG_A_MU_UL				= 88 /* 0x58 */,
+	HAL_MACTX_HE_SIG_B1_MU					= 89 /* 0x59 */,
+	HAL_MACTX_HE_SIG_B2_MU					= 90 /* 0x5a */,
+	HAL_MACTX_HE_SIG_B2_OFDMA				= 91 /* 0x5b */,
+	HAL_MACTX_DELETE_CV					= 92 /* 0x5c */,
+	HAL_MACTX_MU_UPLINK_COMMON				= 93 /* 0x5d */,
+	HAL_MACTX_MU_UPLINK_USER_SETUP				= 94 /* 0x5e */,
+	HAL_MACTX_OTHER_TRANSMIT_INFO				= 95 /* 0x5f */,
+	HAL_MACTX_PHY_NAP					= 96 /* 0x60 */,
+	HAL_MACTX_DEBUG						= 97 /* 0x61 */,
+	HAL_PHYRX_ABORT_ACK					= 98 /* 0x62 */,
+	HAL_PHYRX_GENERATED_CBF_DETAILS				= 99 /* 0x63 */,
+	HAL_PHYRX_RSSI_LEGACY					= 100 /* 0x64 */,
+	HAL_PHYRX_RSSI_HT					= 101 /* 0x65 */,
+	HAL_PHYRX_USER_INFO					= 102 /* 0x66 */,
+	HAL_PHYRX_PKT_END					= 103 /* 0x67 */,
+	HAL_PHYRX_DEBUG						= 104 /* 0x68 */,
+	HAL_PHYRX_CBF_TRANSFER_DONE				= 105 /* 0x69 */,
+	HAL_PHYRX_CBF_TRANSFER_ABORT				= 106 /* 0x6a */,
+	HAL_PHYRX_L_SIG_A					= 107 /* 0x6b */,
+	HAL_PHYRX_L_SIG_B					= 108 /* 0x6c */,
+	HAL_PHYRX_HT_SIG					= 109 /* 0x6d */,
+	HAL_PHYRX_VHT_SIG_A					= 110 /* 0x6e */,
+	HAL_PHYRX_VHT_SIG_B_SU20				= 111 /* 0x6f */,
+	HAL_PHYRX_VHT_SIG_B_SU40				= 112 /* 0x70 */,
+	HAL_PHYRX_VHT_SIG_B_SU80				= 113 /* 0x71 */,
+	HAL_PHYRX_VHT_SIG_B_SU160				= 114 /* 0x72 */,
+	HAL_PHYRX_VHT_SIG_B_MU20				= 115 /* 0x73 */,
+	HAL_PHYRX_VHT_SIG_B_MU40				= 116 /* 0x74 */,
+	HAL_PHYRX_VHT_SIG_B_MU80				= 117 /* 0x75 */,
+	HAL_PHYRX_VHT_SIG_B_MU160				= 118 /* 0x76 */,
+	HAL_PHYRX_HE_SIG_A_SU					= 119 /* 0x77 */,
+	HAL_PHYRX_HE_SIG_A_MU_DL				= 120 /* 0x78 */,
+	HAL_PHYRX_HE_SIG_A_MU_UL				= 121 /* 0x79 */,
+	HAL_PHYRX_HE_SIG_B1_MU					= 122 /* 0x7a */,
+	HAL_PHYRX_HE_SIG_B2_MU					= 123 /* 0x7b */,
+	HAL_PHYRX_HE_SIG_B2_OFDMA				= 124 /* 0x7c */,
+	HAL_PHYRX_OTHER_RECEIVE_INFO				= 125 /* 0x7d */,
+	HAL_PHYRX_COMMON_USER_INFO				= 126 /* 0x7e */,
+	HAL_PHYRX_DATA_DONE					= 127 /* 0x7f */,
+	HAL_COEX_TX_REQ						= 128 /* 0x80 */,
+	HAL_DUMMY						= 129 /* 0x81 */,
+	HALXAMPLE_TLV_32_NAME					= 130 /* 0x82 */,
+	HAL_MPDU_LIMIT						= 131 /* 0x83 */,
+	HAL_NA_LENGTH_END					= 132 /* 0x84 */,
+	HAL_OLE_BUF_STATUS					= 133 /* 0x85 */,
+	HAL_PCU_PPDU_SETUP_DONE					= 134 /* 0x86 */,
+	HAL_PCU_PPDU_SETUP_END					= 135 /* 0x87 */,
+	HAL_PCU_PPDU_SETUP_INIT					= 136 /* 0x88 */,
+	HAL_PCU_PPDU_SETUP_START				= 137 /* 0x89 */,
+	HAL_PDG_FES_SETUP					= 138 /* 0x8a */,
+	HAL_PDG_RESPONSE					= 139 /* 0x8b */,
+	HAL_PDG_TX_REQ						= 140 /* 0x8c */,
+	HAL_SCH_WAIT_INSTR					= 141 /* 0x8d */,
+	HAL_TQM_FLOWMPTY_STATUS					= 143 /* 0x8f */,
+	HAL_TQM_FLOW_NOTMPTY_STATUS				= 144 /* 0x90 */,
+	HAL_TQM_GEN_MPDU_LENGTH_LIST				= 145 /* 0x91 */,
+	HAL_TQM_GEN_MPDU_LENGTH_LIST_STATUS			= 146 /* 0x92 */,
+	HAL_TQM_GEN_MPDUS					= 147 /* 0x93 */,
+	HAL_TQM_GEN_MPDUS_STATUS				= 148 /* 0x94 */,
+	HAL_TQM_REMOVE_MPDU					= 149 /* 0x95 */,
+	HAL_TQM_REMOVE_MPDU_STATUS				= 150 /* 0x96 */,
+	HAL_TQM_REMOVE_MSDU					= 151 /* 0x97 */,
+	HAL_TQM_REMOVE_MSDU_STATUS				= 152 /* 0x98 */,
+	HAL_TQM_UPDATE_TX_MPDU_COUNT				= 153 /* 0x99 */,
+	HAL_TQM_WRITE_CMD					= 154 /* 0x9a */,
+	HAL_OFDMA_TRIGGER_DETAILS				= 155 /* 0x9b */,
+	HAL_TX_DATA						= 156 /* 0x9c */,
+	HAL_TX_FES_SETUP					= 157 /* 0x9d */,
+	HAL_RX_PACKET						= 158 /* 0x9e */,
+	HALXPECTED_RESPONSE					= 159 /* 0x9f */,
+	HAL_TX_MPDU_END						= 160 /* 0xa0 */,
+	HAL_TX_MPDU_START					= 161 /* 0xa1 */,
+	HAL_TX_MSDU_END						= 162 /* 0xa2 */,
+	HAL_TX_MSDU_START					= 163 /* 0xa3 */,
+	HAL_TX_SW_MODE_SETUP					= 164 /* 0xa4 */,
+	HAL_TXPCU_BUFFER_STATUS					= 165 /* 0xa5 */,
+	HAL_TXPCU_USER_BUFFER_STATUS				= 166 /* 0xa6 */,
+	HAL_DATA_TO_TIME_CONFIG					= 167 /* 0xa7 */,
+	HALXAMPLE_USER_TLV_32					= 168 /* 0xa8 */,
+	HAL_MPDU_INFO						= 169 /* 0xa9 */,
+	HAL_PDG_USER_SETUP					= 170 /* 0xaa */,
+	HAL_TX_11AH_SETUP					= 171 /* 0xab */,
+	HAL_REO_UPDATE_RX_REO_QUEUE_STATUS			= 172 /* 0xac */,
+	HAL_TX_PEER_ENTRY					= 173 /* 0xad */,
+	HAL_TX_RAW_OR_NATIVE_FRAME_SETUP			= 174 /* 0xae */,
+	HALXAMPLE_USER_TLV_44					= 175 /* 0xaf */,
+	HAL_TX_FLUSH						= 176 /* 0xb0 */,
+	HAL_TX_FLUSH_REQ					= 177 /* 0xb1 */,
+	HAL_TQM_WRITE_CMD_STATUS				= 178 /* 0xb2 */,
+	HAL_TQM_GET_MPDU_QUEUE_STATS				= 179 /* 0xb3 */,
+	HAL_TQM_GET_MSDU_FLOW_STATS				= 180 /* 0xb4 */,
+	HALXAMPLE_USER_CTLV_44					= 181 /* 0xb5 */,
+	HAL_TX_FES_STATUS_START					= 182 /* 0xb6 */,
+	HAL_TX_FES_STATUS_USER_PPDU				= 183 /* 0xb7 */,
+	HAL_TX_FES_STATUS_USER_RESPONSE				= 184 /* 0xb8 */,
+	HAL_TX_FES_STATUS_END					= 185 /* 0xb9 */,
+	HAL_RX_TRIG_INFO					= 186 /* 0xba */,
+	HAL_RXPCU_TX_SETUP_CLEAR				= 187 /* 0xbb */,
+	HAL_RX_FRAME_BITMAP_REQ					= 188 /* 0xbc */,
+	HAL_RX_FRAME_BITMAP_ACK					= 189 /* 0xbd */,
+	HAL_COEX_RX_STATUS					= 190 /* 0xbe */,
+	HAL_RX_START_PARAM					= 191 /* 0xbf */,
+	HAL_RX_PPDU_START					= 192 /* 0xc0 */,
+	HAL_RX_PPDU_END						= 193 /* 0xc1 */,
+	HAL_RX_MPDU_START					= 194 /* 0xc2 */,
+	HAL_RX_MPDU_END						= 195 /* 0xc3 */,
+	HAL_RX_MSDU_START					= 196 /* 0xc4 */,
+	HAL_RX_MSDU_END						= 197 /* 0xc5 */,
+	HAL_RX_ATTENTION					= 198 /* 0xc6 */,
+	HAL_RECEIVED_RESPONSE_INFO				= 199 /* 0xc7 */,
+	HAL_RX_PHY_SLEEP					= 200 /* 0xc8 */,
+	HAL_RX_HEADER						= 201 /* 0xc9 */,
+	HAL_RX_PEER_ENTRY					= 202 /* 0xca */,
+	HAL_RX_FLUSH						= 203 /* 0xcb */,
+	HAL_RX_RESPONSE_REQUIRED_INFO				= 204 /* 0xcc */,
+	HAL_RX_FRAMELESS_BAR_DETAILS				= 205 /* 0xcd */,
+	HAL_TQM_GET_MPDU_QUEUE_STATS_STATUS			= 206 /* 0xce */,
+	HAL_TQM_GET_MSDU_FLOW_STATS_STATUS			= 207 /* 0xcf */,
+	HAL_TX_CBF_INFO						= 208 /* 0xd0 */,
+	HAL_PCU_PPDU_SETUP_USER					= 209 /* 0xd1 */,
+	HAL_RX_MPDU_PCU_START					= 210 /* 0xd2 */,
+	HAL_RX_PM_INFO						= 211 /* 0xd3 */,
+	HAL_RX_USER_PPDU_END					= 212 /* 0xd4 */,
+	HAL_RX_PRE_PPDU_START					= 213 /* 0xd5 */,
+	HAL_RX_PREAMBLE						= 214 /* 0xd6 */,
+	HAL_TX_FES_SETUP_COMPLETE				= 215 /* 0xd7 */,
+	HAL_TX_LAST_MPDU_FETCHED				= 216 /* 0xd8 */,
+	HAL_TXDMA_STOP_REQUEST					= 217 /* 0xd9 */,
+	HAL_RXPCU_SETUP						= 218 /* 0xda */,
+	HAL_RXPCU_USER_SETUP					= 219 /* 0xdb */,
+	HAL_TX_FES_STATUS_ACK_OR_BA				= 220 /* 0xdc */,
+	HAL_TQM_ACKED_MPDU					= 221 /* 0xdd */,
+	HAL_COEX_TX_RESP					= 222 /* 0xde */,
+	HAL_COEX_TX_STATUS					= 223 /* 0xdf */,
+	HAL_MACTX_COEX_PHY_CTRL					= 224 /* 0xe0 */,
+	HAL_COEX_STATUS_BROADCAST				= 225 /* 0xe1 */,
+	HAL_RESPONSE_START_STATUS				= 226 /* 0xe2 */,
+	HAL_RESPONSEND_STATUS					= 227 /* 0xe3 */,
+	HAL_CRYPTO_STATUS					= 228 /* 0xe4 */,
+	HAL_RECEIVED_TRIGGER_INFO				= 229 /* 0xe5 */,
+	HAL_COEX_TX_STOP_CTRL					= 230 /* 0xe6 */,
+	HAL_RX_PPDU_ACK_REPORT					= 231 /* 0xe7 */,
+	HAL_RX_PPDU_NO_ACK_REPORT				= 232 /* 0xe8 */,
+	HAL_SCH_COEX_STATUS					= 233 /* 0xe9 */,
+	HAL_SCHEDULER_COMMAND_STATUS				= 234 /* 0xea */,
+	HAL_SCHEDULER_RX_PPDU_NO_RESPONSE_STATUS		= 235 /* 0xeb */,
+	HAL_TX_FES_STATUS_PROT					= 236 /* 0xec */,
+	HAL_TX_FES_STATUS_START_PPDU				= 237 /* 0xed */,
+	HAL_TX_FES_STATUS_START_PROT				= 238 /* 0xee */,
+	HAL_TXPCU_PHYTX_DEBUG32					= 239 /* 0xef */,
+	HAL_TXPCU_PHYTX_OTHER_TRANSMIT_INFO32			= 240 /* 0xf0 */,
+	HAL_TX_MPDU_COUNT_TRANSFERND				= 241 /* 0xf1 */,
+	HAL_WHO_ANCHOR_OFFSET					= 242 /* 0xf2 */,
+	HAL_WHO_ANCHOR_VALUE					= 243 /* 0xf3 */,
+	HAL_WHO_CCE_INFO					= 244 /* 0xf4 */,
+	HAL_WHO_COMMIT						= 245 /* 0xf5 */,
+	HAL_WHO_COMMIT_DONE					= 246 /* 0xf6 */,
+	HAL_WHO_FLUSH						= 247 /* 0xf7 */,
+	HAL_WHO_L2_LLC						= 248 /* 0xf8 */,
+	HAL_WHO_L2_PAYLOAD					= 249 /* 0xf9 */,
+	HAL_WHO_L3_CHECKSUM					= 250 /* 0xfa */,
+	HAL_WHO_L3_INFO						= 251 /* 0xfb */,
+	HAL_WHO_L4_CHECKSUM					= 252 /* 0xfc */,
+	HAL_WHO_L4_INFO						= 253 /* 0xfd */,
+	HAL_WHO_MSDU						= 254 /* 0xfe */,
+	HAL_WHO_MSDU_MISC					= 255 /* 0xff */,
+	HAL_WHO_PACKET_DATA					= 256 /* 0x100 */,
+	HAL_WHO_PACKET_HDR					= 257 /* 0x101 */,
+	HAL_WHO_PPDU_END					= 258 /* 0x102 */,
+	HAL_WHO_PPDU_START					= 259 /* 0x103 */,
+	HAL_WHO_TSO						= 260 /* 0x104 */,
+	HAL_WHO_WMAC_HEADER_PV0					= 261 /* 0x105 */,
+	HAL_WHO_WMAC_HEADER_PV1					= 262 /* 0x106 */,
+	HAL_WHO_WMAC_IV						= 263 /* 0x107 */,
+	HAL_MPDU_INFO_END					= 264 /* 0x108 */,
+	HAL_MPDU_INFO_BITMAP					= 265 /* 0x109 */,
+	HAL_TX_QUEUE_EXTENSION					= 266 /* 0x10a */,
+	HAL_SCHEDULER_SELFGEN_RESPONSE_STATUS			= 267 /* 0x10b */,
+	HAL_TQM_UPDATE_TX_MPDU_COUNT_STATUS			= 268 /* 0x10c */,
+	HAL_TQM_ACKED_MPDU_STATUS				= 269 /* 0x10d */,
+	HAL_TQM_ADD_MSDU_STATUS					= 270 /* 0x10e */,
+	HAL_TQM_LIST_GEN_DONE					= 271 /* 0x10f */,
+	HAL_WHO_TERMINATE					= 272 /* 0x110 */,
+	HAL_TX_LAST_MPDU_END					= 273 /* 0x111 */,
+	HAL_TX_CV_DATA						= 274 /* 0x112 */,
+	HAL_PPDU_TX_END						= 275 /* 0x113 */,
+	HAL_PROT_TX_END						= 276 /* 0x114 */,
+	HAL_MPDU_INFO_GLOBAL_END				= 277 /* 0x115 */,
+	HAL_TQM_SCH_INSTR_GLOBAL_END				= 278 /* 0x116 */,
+	HAL_RX_PPDU_END_USER_STATS				= 279 /* 0x117 */,
+	HAL_RX_PPDU_END_USER_STATS_EXT				= 280 /* 0x118 */,
+	HAL_REO_GET_QUEUE_STATS					= 281 /* 0x119 */,
+	HAL_REO_FLUSH_QUEUE					= 282 /* 0x11a */,
+	HAL_REO_FLUSH_CACHE					= 283 /* 0x11b */,
+	HAL_REO_UNBLOCK_CACHE					= 284 /* 0x11c */,
+	HAL_REO_GET_QUEUE_STATS_STATUS				= 285 /* 0x11d */,
+	HAL_REO_FLUSH_QUEUE_STATUS				= 286 /* 0x11e */,
+	HAL_REO_FLUSH_CACHE_STATUS				= 287 /* 0x11f */,
+	HAL_REO_UNBLOCK_CACHE_STATUS				= 288 /* 0x120 */,
+	HAL_TQM_FLUSH_CACHE					= 289 /* 0x121 */,
+	HAL_TQM_UNBLOCK_CACHE					= 290 /* 0x122 */,
+	HAL_TQM_FLUSH_CACHE_STATUS				= 291 /* 0x123 */,
+	HAL_TQM_UNBLOCK_CACHE_STATUS				= 292 /* 0x124 */,
+	HAL_RX_PPDU_END_STATUS_DONE				= 293 /* 0x125 */,
+	HAL_RX_STATUS_BUFFER_DONE				= 294 /* 0x126 */,
+	HAL_TX_DATA_SYNC					= 297 /* 0x129 */,
+	HAL_PHYRX_CBF_READ_REQUEST_ACK				= 298 /* 0x12a */,
+	HAL_TQM_GET_MPDU_HEAD_INFO				= 299 /* 0x12b */,
+	HAL_TQM_SYNC_CMD					= 300 /* 0x12c */,
+	HAL_TQM_GET_MPDU_HEAD_INFO_STATUS			= 301 /* 0x12d */,
+	HAL_TQM_SYNC_CMD_STATUS					= 302 /* 0x12e */,
+	HAL_TQM_THRESHOLD_DROP_NOTIFICATION_STATUS		= 303 /* 0x12f */,
+	HAL_TQM_DESCRIPTOR_THRESHOLD_REACHED_STATUS		= 304 /* 0x130 */,
+	HAL_REO_FLUSH_TIMEOUT_LIST				= 305 /* 0x131 */,
+	HAL_REO_FLUSH_TIMEOUT_LIST_STATUS			= 306 /* 0x132 */,
+	HAL_REO_DESCRIPTOR_THRESHOLD_REACHED_STATUS		= 307 /* 0x133 */,
+	HAL_SCHEDULER_RX_SIFS_RESPONSE_TRIGGER_STATUS		= 308 /* 0x134 */,
+	HALXAMPLE_USER_TLV_32_NAME				= 309 /* 0x135 */,
+	HAL_RX_PPDU_START_USER_INFO				= 310 /* 0x136 */,
+	HAL_RX_RING_MASK					= 311 /* 0x137 */,
+	HAL_COEX_MAC_NAP					= 312 /* 0x138 */,
+	HAL_RXPCU_PPDU_END_INFO					= 313 /* 0x139 */,
+	HAL_WHO_MESH_CONTROL					= 314 /* 0x13a */,
+	HAL_PDG_SW_MODE_BW_START				= 315 /* 0x13b */,
+	HAL_PDG_SW_MODE_BW_END					= 316 /* 0x13c */,
+	HAL_PDG_WAIT_FOR_MAC_REQUEST				= 317 /* 0x13d */,
+	HAL_PDG_WAIT_FOR_PHY_REQUEST				= 318 /* 0x13e */,
+	HAL_SCHEDULER_END					= 319 /* 0x13f */,
+	HAL_RX_PPDU_START_DROPPED				= 320 /* 0x140 */,
+	HAL_RX_PPDU_END_DROPPED					= 321 /* 0x141 */,
+	HAL_RX_PPDU_END_STATUS_DONE_DROPPED			= 322 /* 0x142 */,
+	HAL_RX_MPDU_START_DROPPED				= 323 /* 0x143 */,
+	HAL_RX_MSDU_START_DROPPED				= 324 /* 0x144 */,
+	HAL_RX_MSDU_END_DROPPED					= 325 /* 0x145 */,
+	HAL_RX_MPDU_END_DROPPED					= 326 /* 0x146 */,
+	HAL_RX_ATTENTION_DROPPED				= 327 /* 0x147 */,
+	HAL_TXPCU_USER_SETUP					= 328 /* 0x148 */,
+	HAL_RXPCU_USER_SETUP_EXT				= 329 /* 0x149 */,
+	HAL_CMD_PART_0_END					= 330 /* 0x14a */,
+	HAL_MACTX_SYNTH_ON					= 331 /* 0x14b */,
+	HAL_SCH_CRITICAL_TLV_REFERENCE				= 332 /* 0x14c */,
+	HAL_TQM_MPDU_GLOBAL_START				= 333 /* 0x14d */,
+	HALXAMPLE_TLV_32					= 334 /* 0x14e */,
+	HAL_TQM_UPDATE_TX_MSDU_FLOW				= 335 /* 0x14f */,
+	HAL_TQM_UPDATE_TX_MPDU_QUEUE_HEAD			= 336 /* 0x150 */,
+	HAL_TQM_UPDATE_TX_MSDU_FLOW_STATUS			= 337 /* 0x151 */,
+	HAL_TQM_UPDATE_TX_MPDU_QUEUE_HEAD_STATUS		= 338 /* 0x152 */,
+	HAL_REO_UPDATE_RX_REO_QUEUE				= 339 /* 0x153 */,
+	HAL_TQM_MPDU_QUEUEMPTY_STATUS				= 340 /* 0x154 */,
+	HAL_TQM_2_SCH_MPDU_AVAILABLE				= 341 /* 0x155 */,
+	HAL_PDG_TRIG_RESPONSE					= 342 /* 0x156 */,
+	HAL_TRIGGER_RESPONSE_TX_DONE				= 343 /* 0x157 */,
+	HAL_ABORT_FROM_PHYRX_DETAILS				= 344 /* 0x158 */,
+	HAL_SCH_TQM_CMD_WRAPPER					= 345 /* 0x159 */,
+	HAL_MPDUS_AVAILABLE					= 346 /* 0x15a */,
+	HAL_RECEIVED_RESPONSE_INFO_PART2			= 347 /* 0x15b */,
+	HAL_PHYRX_TX_START_TIMING				= 348 /* 0x15c */,
+	HAL_TXPCU_PREAMBLE_DONE					= 349 /* 0x15d */,
+	HAL_NDP_PREAMBLE_DONE					= 350 /* 0x15e */,
+	HAL_SCH_TQM_CMD_WRAPPER_RBO_DROP			= 351 /* 0x15f */,
+	HAL_SCH_TQM_CMD_WRAPPER_CONT_DROP			= 352 /* 0x160 */,
+	HAL_MACTX_CLEAR_PREV_TX_INFO				= 353 /* 0x161 */,
+	HAL_TX_PUNCTURE_SETUP					= 354 /* 0x162 */,
+	HAL_R2R_STATUS_END					= 355 /* 0x163 */,
+	HAL_MACTX_PREFETCH_CV_COMMON				= 356 /* 0x164 */,
+	HAL_END_OF_FLUSH_MARKER					= 357 /* 0x165 */,
+	HAL_MACTX_MU_UPLINK_COMMON_PUNC				= 358 /* 0x166 */,
+	HAL_MACTX_MU_UPLINK_USER_SETUP_PUNC			= 359 /* 0x167 */,
+	HAL_RECEIVED_RESPONSE_USER_7_0				= 360 /* 0x168 */,
+	HAL_RECEIVED_RESPONSE_USER_15_8				= 361 /* 0x169 */,
+	HAL_RECEIVED_RESPONSE_USER_23_16			= 362 /* 0x16a */,
+	HAL_RECEIVED_RESPONSE_USER_31_24			= 363 /* 0x16b */,
+	HAL_RECEIVED_RESPONSE_USER_36_32			= 364 /* 0x16c */,
+	HAL_TX_LOOPBACK_SETUP					= 365 /* 0x16d */,
+	HAL_PHYRX_OTHER_RECEIVE_INFO_RU_DETAILS			= 366 /* 0x16e */,
+	HAL_SCH_WAIT_INSTR_TX_PATH				= 367 /* 0x16f */,
+	HAL_MACTX_OTHER_TRANSMIT_INFO_TX2TX			= 368 /* 0x170 */,
+	HAL_MACTX_OTHER_TRANSMIT_INFOMUPHY_SETUP		= 369 /* 0x171 */,
+	HAL_PHYRX_OTHER_RECEIVE_INFOVM_DETAILS			= 370 /* 0x172 */,
+	HAL_TX_WUR_DATA						= 371 /* 0x173 */,
+	HAL_RX_PPDU_END_START					= 372 /* 0x174 */,
+	HAL_RX_PPDU_END_MIDDLE					= 373 /* 0x175 */,
+	HAL_RX_PPDU_END_LAST					= 374 /* 0x176 */,
+	HAL_MACTX_BACKOFF_BASED_TRANSMISSION			= 375 /* 0x177 */,
+	HAL_MACTX_OTHER_TRANSMIT_INFO_DL_OFDMA_TX		= 376 /* 0x178 */,
+	HAL_SRP_INFO						= 377 /* 0x179 */,
+	HAL_OBSS_SR_INFO					= 378 /* 0x17a */,
+	HAL_SCHEDULER_SW_MSG_STATUS				= 379 /* 0x17b */,
+	HAL_HWSCH_RXPCU_MAC_INFO_ANNOUNCEMENT			= 380 /* 0x17c */,
+	HAL_RXPCU_SETUP_COMPLETE				= 381 /* 0x17d */,
+	HAL_SNOOP_PPDU_START					= 382 /* 0x17e */,
+	HAL_SNOOP_MPDU_USR_DBG_INFO				= 383 /* 0x17f */,
+	HAL_SNOOP_MSDU_USR_DBG_INFO				= 384 /* 0x180 */,
+	HAL_SNOOP_MSDU_USR_DATA					= 385 /* 0x181 */,
+	HAL_SNOOP_MPDU_USR_STAT_INFO				= 386 /* 0x182 */,
+	HAL_SNOOP_PPDU_END					= 387 /* 0x183 */,
+	HAL_SNOOP_SPARE						= 388 /* 0x184 */,
+	HAL_PHYRX_OTHER_RECEIVE_INFO_MU_RSSI_COMMON		= 390 /* 0x186 */,
+	HAL_PHYRX_OTHER_RECEIVE_INFO_MU_RSSI_USER		= 391 /* 0x187 */,
+	HAL_MACTX_OTHER_TRANSMIT_INFO_SCH_DETAILS		= 392 /* 0x188 */,
+	HAL_PHYRX_OTHER_RECEIVE_INFO_108PVM_DETAILS		= 393 /* 0x189 */,
+	HAL_SCH_TLV_WRAPPER					= 394 /* 0x18a */,
+	HAL_SCHEDULER_STATUS_WRAPPER				= 395 /* 0x18b */,
+	HAL_MPDU_INFO_6X					= 396 /* 0x18c */,
+	HAL_MACTX_11AZ_USER_DESC_PER_USER			= 397 /* 0x18d */,
+	HAL_MACTX_U_SIGHT_SU_MU					= 398 /* 0x18e */,
+	HAL_MACTX_U_SIGHT_TB					= 399 /* 0x18f */,
+	HAL_PHYRX_U_SIGHT_SU_MU					= 403 /* 0x193 */,
+	HAL_PHYRX_U_SIGHT_TB					= 404 /* 0x194 */,
+	HAL_MACRX_LMR_READ_REQUEST				= 408 /* 0x198 */,
+	HAL_MACRX_LMR_DATA_REQUEST				= 409 /* 0x199 */,
+	HAL_PHYRX_LMR_TRANSFER_DONE				= 410 /* 0x19a */,
+	HAL_PHYRX_LMR_TRANSFER_ABORT				= 411 /* 0x19b */,
+	HAL_PHYRX_LMR_READ_REQUEST_ACK				= 412 /* 0x19c */,
+	HAL_MACRX_SECURE_LTF_SEQ_PTR				= 413 /* 0x19d */,
+	HAL_PHYRX_USER_INFO_MU_UL				= 414 /* 0x19e */,
+	HAL_MPDU_QUEUE_OVERVIEW					= 415 /* 0x19f */,
+	HAL_SCHEDULER_NAV_INFO					= 416 /* 0x1a0 */,
+	HAL_LMR_PEER_ENTRY					= 418 /* 0x1a2 */,
+	HAL_LMR_MPDU_START					= 419 /* 0x1a3 */,
+	HAL_LMR_DATA						= 420 /* 0x1a4 */,
+	HAL_LMR_MPDU_END					= 421 /* 0x1a5 */,
+	HAL_REO_GET_QUEUE_1K_STATS_STATUS			= 422 /* 0x1a6 */,
+	HAL_RX_FRAME_1K_BITMAP_ACK				= 423 /* 0x1a7 */,
+	HAL_TX_FES_STATUS_1K_BA					= 424 /* 0x1a8 */,
+	HAL_TQM_ACKED_1K_MPDU					= 425 /* 0x1a9 */,
+	HAL_MACRX_INBSS_OBSS_IND				= 426 /* 0x1aa */,
+	HAL_PHYRX_LOCATION					= 427 /* 0x1ab */,
+	HAL_MLO_TX_NOTIFICATION_SU				= 428 /* 0x1ac */,
+	HAL_MLO_TX_NOTIFICATION_MU				= 429 /* 0x1ad */,
+	HAL_MLO_TX_REQ_SU					= 430 /* 0x1ae */,
+	HAL_MLO_TX_REQ_MU					= 431 /* 0x1af */,
+	HAL_MLO_TX_RESP						= 432 /* 0x1b0 */,
+	HAL_MLO_RX_NOTIFICATION					= 433 /* 0x1b1 */,
+	HAL_MLO_BKOFF_TRUNC_REQ					= 434 /* 0x1b2 */,
+	HAL_MLO_TBTT_NOTIFICATION				= 435 /* 0x1b3 */,
+	HAL_MLO_MESSAGE						= 436 /* 0x1b4 */,
+	HAL_MLO_TS_SYNC_MSG					= 437 /* 0x1b5 */,
+	HAL_MLO_FES_SETUP					= 438 /* 0x1b6 */,
+	HAL_MLO_PDG_FES_SETUP_SU				= 439 /* 0x1b7 */,
+	HAL_MLO_PDG_FES_SETUP_MU				= 440 /* 0x1b8 */,
+	HAL_MPDU_INFO_1K_BITMAP					= 441 /* 0x1b9 */,
+	HAL_MON_BUF_ADDR					= 442 /* 0x1ba */,
+	HAL_TX_FRAG_STATE					= 443 /* 0x1bb */,
+	HAL_MACTXHT_SIG_USR_OFDMA				= 446 /* 0x1be */,
+	HAL_PHYRXHT_SIG_CMN_PUNC				= 448 /* 0x1c0 */,
+	HAL_PHYRXHT_SIG_CMN_OFDMA				= 450 /* 0x1c2 */,
+	HAL_PHYRXHT_SIG_USR_OFDMA				= 454 /* 0x1c6 */,
+	HAL_PHYRX_PKT_END_PART1					= 456 /* 0x1c8 */,
+	HAL_MACTXXPECT_NDP_RECEPTION				= 457 /* 0x1c9 */,
+	HAL_MACTX_SECURE_LTF_SEQ_PTR				= 458 /* 0x1ca */,
+	HAL_MLO_PDG_BKOFF_TRUNC_NOTIFY				= 460 /* 0x1cc */,
+	HAL_PHYRX_11AZ_INTEGRITY_DATA				= 461 /* 0x1cd */,
+	HAL_PHYTX_LOCATION					= 462 /* 0x1ce */,
+	HAL_PHYTX_11AZ_INTEGRITY_DATA				= 463 /* 0x1cf */,
+	HAL_MACTXHT_SIG_USR_SU					= 466 /* 0x1d2 */,
+	HAL_MACTXHT_SIG_USR_MU_MIMO				= 467 /* 0x1d3 */,
+	HAL_PHYRXHT_SIG_USR_SU					= 468 /* 0x1d4 */,
+	HAL_PHYRXHT_SIG_USR_MU_MIMO				= 469 /* 0x1d5 */,
+	HAL_PHYRX_GENERIC_U_SIG					= 470 /* 0x1d6 */,
+	HAL_PHYRX_GENERIC_EHT_SIG				= 471 /* 0x1d7 */,
+	HAL_OVERWRITE_RESP_START				= 472 /* 0x1d8 */,
+	HAL_OVERWRITE_RESP_PREAMBLE_INFO			= 473 /* 0x1d9 */,
+	HAL_OVERWRITE_RESP_FRAME_INFO				= 474 /* 0x1da */,
+	HAL_OVERWRITE_RESP_END					= 475 /* 0x1db */,
+	HAL_RXPCUARLY_RX_INDICATION				= 476 /* 0x1dc */,
+	HAL_MON_DROP						= 477 /* 0x1dd */,
+	HAL_MACRX_MU_UPLINK_COMMON_SNIFF			= 478 /* 0x1de */,
+	HAL_MACRX_MU_UPLINK_USER_SETUP_SNIFF			= 479 /* 0x1df */,
+	HAL_MACRX_MU_UPLINK_USER_SEL_SNIFF			= 480 /* 0x1e0 */,
+	HAL_MACRX_MU_UPLINK_FCS_STATUS_SNIFF			= 481 /* 0x1e1 */,
+	HAL_MACTX_PREFETCH_CV_DMA				= 482 /* 0x1e2 */,
+	HAL_MACTX_PREFETCH_CV_PER_USER				= 483 /* 0x1e3 */,
+	HAL_PHYRX_OTHER_RECEIVE_INFO_ALL_SIGB_DETAILS		= 484 /* 0x1e4 */,
+	HAL_MACTX_BF_PARAMS_UPDATE_COMMON			= 485 /* 0x1e5 */,
+	HAL_MACTX_BF_PARAMS_UPDATE_PER_USER			= 486 /* 0x1e6 */,
+	HAL_RANGING_USER_DETAILS				= 487 /* 0x1e7 */,
+	HAL_PHYTX_CV_CORR_STATUS				= 488 /* 0x1e8 */,
+	HAL_PHYTX_CV_CORR_COMMON				= 489 /* 0x1e9 */,
+	HAL_PHYTX_CV_CORR_USER					= 490 /* 0x1ea */,
+	HAL_MACTX_CV_CORR_COMMON				= 491 /* 0x1eb */,
+	HAL_MACTX_CV_CORR_MAC_INFO_GROUP			= 492 /* 0x1ec */,
+	HAL_BW_PUNCTUREVAL_WRAPPER				= 493 /* 0x1ed */,
+	HAL_MACTX_RX_NOTIFICATION_FOR_PHY			= 494 /* 0x1ee */,
+	HAL_MACTX_TX_NOTIFICATION_FOR_PHY			= 495 /* 0x1ef */,
+	HAL_MACTX_MU_UPLINK_COMMON_PER_BW			= 496 /* 0x1f0 */,
+	HAL_MACTX_MU_UPLINK_USER_SETUP_PER_BW			= 497 /* 0x1f1 */,
+	HAL_RX_PPDU_END_USER_STATS_EXT2				= 498 /* 0x1f2 */,
+	HAL_FW2SW_MON						= 499 /* 0x1f3 */,
+	HAL_WSI_DIRECT_MESSAGE					= 500 /* 0x1f4 */,
+	HAL_MACTXMLSR_PRE_SWITCH				= 501 /* 0x1f5 */,
+	HAL_MACTXMLSR_SWITCH					= 502 /* 0x1f6 */,
+	HAL_MACTXMLSR_SWITCH_BACK				= 503 /* 0x1f7 */,
+	HAL_PHYTXMLSR_SWITCH_ACK				= 504 /* 0x1f8 */,
+	HAL_PHYTXMLSR_SWITCH_BACK_ACK				= 505 /* 0x1f9 */,
+	HAL_SPARE_REUSE_TAG_0					= 506 /* 0x1fa */,
+	HAL_SPARE_REUSE_TAG_1					= 507 /* 0x1fb */,
+	HAL_SPARE_REUSE_TAG_2					= 508 /* 0x1fc */,
+	HAL_SPARE_REUSE_TAG_3					= 509 /* 0x1fd */,
+	/* FIXME: Assign correct value for HAL_TCL_DATA_CMD */
+	HAL_TCL_DATA_CMD					= 510,
+	HAL_TLV_BASE						= 511 /* 0x1ff */,
+};
+
+#define HAL_TLV_HDR_TAG		GENMASK(9, 1)
+#define HAL_TLV_HDR_LEN		GENMASK(25, 10)
+#define HAL_TLV_USR_ID          GENMASK(31, 26)
+
+#define HAL_TLV_ALIGN	4
+
+struct hal_tlv_hdr {
+	__le32 tl;
+	u8 value[];
+} __packed;
+
+#define HAL_TLV_64_HDR_TAG		GENMASK(9, 1)
+#define HAL_TLV_64_HDR_LEN		GENMASK(21, 10)
+#define HAL_TLV_64_USR_ID		GENMASK(31, 26)
+#define HAL_TLV_64_ALIGN		8
+
+struct hal_tlv_64_hdr {
+	__le64 tl;
+	u8 value[];
+} __packed;
+
+#define RX_MPDU_DESC_INFO0_MSDU_COUNT		GENMASK(7, 0)
+#define RX_MPDU_DESC_INFO0_FRAG_FLAG		BIT(8)
+#define RX_MPDU_DESC_INFO0_MPDU_RETRY		BIT(9)
+#define RX_MPDU_DESC_INFO0_AMPDU_FLAG		BIT(10)
+#define RX_MPDU_DESC_INFO0_BAR_FRAME		BIT(11)
+#define RX_MPDU_DESC_INFO0_VALID_PN		BIT(12)
+#define RX_MPDU_DESC_INFO0_RAW_MPDU		BIT(13)
+#define RX_MPDU_DESC_INFO0_MORE_FRAG_FLAG	BIT(14)
+#define RX_MPDU_DESC_INFO0_SRC_INFO		GENMASK(26, 15)
+#define RX_MPDU_DESC_INFO0_MPDU_QOS_CTRL_VALID	BIT(27)
+#define RX_MPDU_DESC_INFO0_TID			GENMASK(31, 28)
+
+/* Peer Metadata classification */
+
+/* Version 0 */
+#define RX_MPDU_DESC_META_DATA_V0_PEER_ID	GENMASK(15, 0)
+#define RX_MPDU_DESC_META_DATA_V0_VDEV_ID	GENMASK(23, 16)
+
+/* Version 1 */
+#define RX_MPDU_DESC_META_DATA_V1_PEER_ID		GENMASK(13, 0)
+#define RX_MPDU_DESC_META_DATA_V1_LOGICAL_LINK_ID	GENMASK(15, 14)
+#define RX_MPDU_DESC_META_DATA_V1_VDEV_ID		GENMASK(23, 16)
+#define RX_MPDU_DESC_META_DATA_V1_LMAC_ID		GENMASK(25, 24)
+#define RX_MPDU_DESC_META_DATA_V1_DEVICE_ID		GENMASK(28, 26)
+
+/* Version 1A */
+#define RX_MPDU_DESC_META_DATA_V1A_PEER_ID		GENMASK(13, 0)
+#define RX_MPDU_DESC_META_DATA_V1A_VDEV_ID		GENMASK(21, 14)
+#define RX_MPDU_DESC_META_DATA_V1A_LOGICAL_LINK_ID	GENMASK(25, 22)
+#define RX_MPDU_DESC_META_DATA_V1A_DEVICE_ID		GENMASK(28, 26)
+
+/* Version 1B */
+#define RX_MPDU_DESC_META_DATA_V1B_PEER_ID	GENMASK(13, 0)
+#define RX_MPDU_DESC_META_DATA_V1B_VDEV_ID	GENMASK(21, 14)
+#define RX_MPDU_DESC_META_DATA_V1B_HW_LINK_ID	GENMASK(25, 22)
+#define RX_MPDU_DESC_META_DATA_V1B_DEVICE_ID	GENMASK(28, 26)
+
+struct rx_mpdu_desc {
+	__le32 info0; /* %RX_MPDU_DESC_INFO */
+	__le32 peer_meta_data;
+} __packed;
+
+/* rx_mpdu_desc
+ *		Producer: RXDMA
+ *		Consumer: REO/SW/FW
+ *
+ * msdu_count
+ *		The number of MSDUs within the MPDU
+ *
+ * fragment_flag
+ *		When set, this MPDU is a fragment and REO should forward this
+ *		fragment MPDU to the REO destination ring without any reorder
+ *		checks, pn checks or bitmap update. This implies that REO is
+ *		forwarding the pointer to the MSDU link descriptor.
+ *
+ * mpdu_retry_bit
+ *		The retry bit setting from the MPDU header of the received frame
+ *
+ * ampdu_flag
+ *		Indicates the MPDU was received as part of an A-MPDU.
+ *
+ * bar_frame
+ *		Indicates the received frame is a BAR frame. After processing,
+ *		this frame shall be pushed to SW or deleted.
+ *
+ * valid_pn
+ *		When not set, REO will not perform a PN sequence number check.
+ *
+ * raw_mpdu
+ *		Field only valid when first_msdu_in_mpdu_flag is set. Indicates
+ *		the contents in the MSDU buffer contains a 'RAW' MPDU. This
+ *		'RAW' MPDU might be spread out over multiple MSDU buffers.
+ *
+ * more_fragment_flag
+ *		The More Fragment bit setting from the MPDU header of the
+ *		received frame
+ *
+ * src_info
+ *		Source (Virtual) device/interface info associated with this peer.
+ *		This field gets passed on by REO to PPE in the EDMA descriptor.
+ *
+ * mpdu_qos_control_valid
+ *		When set, the MPDU has a QoS control field
+ *
+ * tid
+ *		Field only valid when mpdu_qos_control_valid is set
+ */
+
+enum hal_rx_msdu_desc_reo_dest_ind {
+	HAL_RX_MSDU_DESC_REO_DEST_IND_TCL,
+	HAL_RX_MSDU_DESC_REO_DEST_IND_SW1,
+	HAL_RX_MSDU_DESC_REO_DEST_IND_SW2,
+	HAL_RX_MSDU_DESC_REO_DEST_IND_SW3,
+	HAL_RX_MSDU_DESC_REO_DEST_IND_SW4,
+	HAL_RX_MSDU_DESC_REO_DEST_IND_RELEASE,
+	HAL_RX_MSDU_DESC_REO_DEST_IND_FW,
+	HAL_RX_MSDU_DESC_REO_DEST_IND_SW5,
+	HAL_RX_MSDU_DESC_REO_DEST_IND_SW6,
+	HAL_RX_MSDU_DESC_REO_DEST_IND_SW7,
+	HAL_RX_MSDU_DESC_REO_DEST_IND_SW8,
+};
+
+#define RX_MSDU_DESC_INFO0_FIRST_MSDU_IN_MPDU	BIT(0)
+#define RX_MSDU_DESC_INFO0_LAST_MSDU_IN_MPDU	BIT(1)
+#define RX_MSDU_DESC_INFO0_MSDU_CONTINUATION	BIT(2)
+#define RX_MSDU_DESC_INFO0_MSDU_LENGTH		GENMASK(16, 3)
+#define RX_MSDU_DESC_INFO0_MSDU_DROP		BIT(17)
+#define RX_MSDU_DESC_INFO0_VALID_SA		BIT(18)
+#define RX_MSDU_DESC_INFO0_VALID_DA		BIT(19)
+#define RX_MSDU_DESC_INFO0_DA_MCBC		BIT(20)
+#define RX_MSDU_DESC_INFO0_L3_HDR_PAD_MSB	BIT(21)
+#define RX_MSDU_DESC_INFO0_TCP_UDP_CHKSUM_FAIL	BIT(22)
+#define RX_MSDU_DESC_INFO0_IP_CHKSUM_FAIL	BIT(23)
+#define RX_MSDU_DESC_INFO0_FROM_DS		BIT(24)
+#define RX_MSDU_DESC_INFO0_TO_DS		BIT(25)
+#define RX_MSDU_DESC_INFO0_INTRA_BSS		BIT(26)
+#define RX_MSDU_DESC_INFO0_DST_CHIP_ID		GENMASK(28, 27)
+#define RX_MSDU_DESC_INFO0_DECAP_FORMAT		GENMASK(30, 29)
+
+#define HAL_RX_MSDU_PKT_LENGTH_GET(val)		\
+	(le32_get_bits((val), RX_MSDU_DESC_INFO0_MSDU_LENGTH))
+
+struct rx_msdu_desc {
+	__le32 info0;
+} __packed;
+
+/* rx_msdu_desc
+ *
+ * first_msdu_in_mpdu
+ *		Indicates first msdu in mpdu.
+ *
+ * last_msdu_in_mpdu
+ *		Indicates last msdu in mpdu. This flag can be true only when
+ *		'Msdu_continuation' set to 0. This implies that when an msdu
+ *		is spread out over multiple buffers and thus msdu_continuation
+ *		is set, only for the very last buffer of the msdu, can the
+ *		'last_msdu_in_mpdu' be set.
+ *
+ *		When both first_msdu_in_mpdu and last_msdu_in_mpdu are set,
+ *		the MPDU that this MSDU belongs to only contains a single MSDU.
+ *
+ * msdu_continuation
+ *		When set, this MSDU buffer was not able to hold the entire MSDU.
+ *		The next buffer will therefore contain additional information
+ *		related to this MSDU.
+ *
+ * msdu_length
+ *		Field is only valid in combination with the 'first_msdu_in_mpdu'
+ *		being set. Full MSDU length in bytes after decapsulation. This
+ *		field is still valid for MPDU frames without A-MSDU. It still
+ *		represents MSDU length after decapsulation Or in case of RAW
+ *		MPDUs, it indicates the length of the entire MPDU (without FCS
+ *		field).
+ *
+ * msdu_drop
+ *		Indicates that REO shall drop this MSDU and not forward it to
+ *		any other ring.
+ *
+ * valid_sa
+ *		Indicates OLE found a valid SA entry for this MSDU.
+ *
+ * valid_da
+ *		When set, OLE found a valid DA entry for this MSDU.
+ *
+ * da_mcbc
+ *		Field Only valid if valid_da is set. Indicates the DA address
+ *		is a Multicast or Broadcast address for this MSDU.
+ *
+ * l3_header_padding_msb
+ *		Passed on from 'RX_MSDU_END' TLV (only the MSB is reported as
+ *		the LSB is always zero). Number of bytes padded to make sure
+ *		that the L3 header will always start of a Dword boundary
+ *
+ * tcp_udp_checksum_fail
+ *		Passed on from 'RX_ATTENTION' TLV
+ *		Indicates that the computed checksum did not match the checksum
+ *		in the TCP/UDP header.
+ *
+ * ip_checksum_fail
+ *		Passed on from 'RX_ATTENTION' TLV
+ *		Indicates that the computed checksum did not match the checksum
+ *		in the IP header.
+ *
+ * from_DS
+ *		Set if the 'from DS' bit is set in the frame control.
+ *
+ * to_DS
+ *		Set if the 'to DS' bit is set in the frame control.
+ *
+ * intra_bss
+ *		This packet needs intra-BSS routing by SW as the 'vdev_id'
+ *		for the destination is the same as the 'vdev_id' that this
+ *		MSDU was got in.
+ *
+ * dest_chip_id
+ *		If intra_bss is set, copied by RXOLE/RXDMA from 'ADDR_SEARCH_ENTRY'
+ *		to support intra-BSS routing with multi-chip multi-link operation.
+ *		This indicates into which chip's TCL the packet should be queued.
+ *
+ * decap_format
+ *		Indicates the format after decapsulation:
+ */
+
+#define RX_MSDU_EXT_DESC_INFO0_REO_DEST_IND	GENMASK(4, 0)
+#define RX_MSDU_EXT_DESC_INFO0_SERVICE_CODE	GENMASK(13, 5)
+#define RX_MSDU_EXT_DESC_INFO0_PRIORITY_VALID	BIT(14)
+#define RX_MSDU_EXT_DESC_INFO0_DATA_OFFSET	GENMASK(26, 15)
+#define RX_MSDU_EXT_DESC_INFO0_SRC_LINK_ID	GENMASK(29, 27)
+
+struct rx_msdu_ext_desc {
+	__le32 info0;
+} __packed;
+
+/* rx_msdu_ext_desc
+ *
+ * reo_destination_indication
+ *		The ID of the REO exit ring where the MSDU frame shall push
+ *		after (MPDU level) reordering has finished.
+ *
+ * service_code
+ *		Opaque service code between PPE and Wi-Fi
+ *
+ * priority_valid
+ *
+ * data_offset
+ *		The offset to Rx packet data within the buffer (including
+ *		Rx DMA offset programming and L3 header padding inserted
+ *		by Rx OLE).
+ *
+ * src_link_id
+ *		Set to the link ID of the PMAC that received the frame
+ */
+
+#define HAL_REO_DEST_RING_INFO0_BUFFER_TYPE		BIT(0)
+#define HAL_REO_DEST_RING_INFO0_PUSH_REASON		GENMASK(2, 1)
+#define HAL_REO_DEST_RING_INFO0_ERROR_CODE		GENMASK(7, 3)
+#define HAL_REO_DEST_RING_INFO0_MSDU_DATA_SIZE		GENMASK(11, 8)
+#define HAL_REO_DEST_RING_INFO0_SW_EXCEPTION		BIT(12)
+#define HAL_REO_DEST_RING_INFO0_SRC_LINK_ID		GENMASK(15, 13)
+#define HAL_REO_DEST_RING_INFO0_SIGNATURE		GENMASK(19, 16)
+#define HAL_REO_DEST_RING_INFO0_RING_ID			GENMASK(27, 20)
+#define HAL_REO_DEST_RING_INFO0_LOOPING_COUNT		GENMASK(31, 28)
+
+struct hal_reo_dest_ring {
+	struct ath12k_buffer_addr buf_addr_info;
+	struct rx_mpdu_desc rx_mpdu_info;
+	struct rx_msdu_desc rx_msdu_info;
+	__le32 buf_va_lo;
+	__le32 buf_va_hi;
+	__le32 info0; /* %HAL_REO_DEST_RING_INFO0_ */
+} __packed;
+
+/* hal_reo_dest_ring
+ *
+ *		Producer: RXDMA
+ *		Consumer: REO/SW/FW
+ *
+ * buf_addr_info
+ *		Details of the physical address of a buffer or MSDU
+ *		link descriptor.
+ *
+ * rx_mpdu_info
+ *		General information related to the MPDU that is passed
+ *		on from REO entrance ring to the REO destination ring.
+ *
+ * rx_msdu_info
+ *		General information related to the MSDU that is passed
+ *		on from RXDMA all the way to the REO destination ring.
+ *
+ * buf_va_lo
+ *		Field only valid if Reo_dest_buffer_type is set to MSDU_buf_address
+ *		Lower 32 bits of the 64-bit virtual address corresponding
+ *		to Buf_or_link_desc_addr_info
+ *
+ * buf_va_hi
+ *		Address (upper 32 bits) of the REO queue descriptor.
+ *		Upper 32 bits of the 64-bit virtual address corresponding
+ *		to Buf_or_link_desc_addr_info
+ *
+ * buffer_type
+ *		Indicates the type of address provided in the buf_addr_info.
+ *		Values are defined in enum %HAL_REO_DEST_RING_BUFFER_TYPE_.
+ *
+ * push_reason
+ *		Reason for pushing this frame to this exit ring. Values are
+ *		defined in enum %HAL_REO_DEST_RING_PUSH_REASON_.
+ *
+ * error_code
+ *		Valid only when 'push_reason' is set. All error codes are
+ *		defined in enum %HAL_REO_DEST_RING_ERROR_CODE_.
+ *
+ * captured_msdu_data_size
+ *		The number of following REO_DESTINATION STRUCTs that have
+ *		been replaced with msdu_data extracted from the msdu_buffer
+ *		and copied into the ring for easy FW/SW access.
+ *
+ * sw_exception
+ *		This field has the same setting as the SW_exception field
+ *		in the corresponding REO_entrance_ring descriptor.
+ *		When set, the REO entrance descriptor is generated by FW,
+ *		and the MPDU was processed in the following way:
+ *		- NO re-order function is needed.
+ *		- MPDU delinking is determined by the setting of Entrance
+ *		  ring field: SW_excection_mpdu_delink
+ *		- Destination ring selection is based on the setting of
+ *		  the Entrance ring field SW_exception_destination _ring_valid
+ *
+ * src_link_id
+ *		Set to the link ID of the PMAC that received the frame
+ *
+ * signature
+ *		Set to value 0x8 when msdu capture mode is enabled for this ring
+ *
+ * ring_id
+ *		The buffer pointer ring id.
+ *		0 - Idle ring
+ *		1 - N refers to other rings.
+ *
+ * looping_count
+ *		Indicates the number of times the producer of entries into
+ *		this ring has looped around the ring.
+ */
+
+#define HAL_REO_TO_PPE_RING_INFO0_DATA_LENGTH	GENMASK(15, 0)
+#define HAL_REO_TO_PPE_RING_INFO0_DATA_OFFSET	GENMASK(23, 16)
+#define HAL_REO_TO_PPE_RING_INFO0_POOL_ID	GENMASK(28, 24)
+#define HAL_REO_TO_PPE_RING_INFO0_PREHEADER	BIT(29)
+#define HAL_REO_TO_PPE_RING_INFO0_TSO_EN	BIT(30)
+#define HAL_REO_TO_PPE_RING_INFO0_MORE	BIT(31)
+
+struct hal_reo_to_ppe_ring {
+	__le32 buffer_addr;
+	__le32 info0; /* %HAL_REO_TO_PPE_RING_INFO0_ */
+} __packed;
+
+/* hal_reo_to_ppe_ring
+ *
+ *		Producer: REO
+ *		Consumer: PPE
+ *
+ * buf_addr_info
+ *		Details of the physical address of a buffer or MSDU
+ *		link descriptor.
+ *
+ * data_length
+ *		Length of valid data in bytes
+ *
+ * data_offset
+ *		Offset to the data from buffer pointer. Can be used to
+ *		strip header in the data for tunnel termination etc.
+ *
+ * pool_id
+ *		REO has global configuration register for this field.
+ *		It may have several free buffer pools, each
+ *		RX-Descriptor ring can fetch free buffer from specific
+ *		buffer pool; pool id will indicate which pool the buffer
+ *		will be released to; POOL_ID Zero returned to SW
+ *
+ * preheader
+ *		Disabled: 0 (Default)
+ *		Enabled: 1
+ *
+ * tso_en
+ *		Disabled: 0 (Default)
+ *		Enabled: 1
+ *
+ * more
+ *		More Segments followed
+ */
+
+enum hal_rx_reo_dest_ring {
+	HAL_RX_REO_DEST_RING_TCL,
+	HAL_RX_REO_DEST_RING_SW1,
+	HAL_RX_REO_DEST_RING_SW2,
+	HAL_RX_REO_DEST_RING_SW3,
+	HAL_RX_REO_DEST_RING_SW4,
+	HAL_RX_REO_DEST_RING_RELEASE,
+	HAL_RX_REO_DEST_RING_FW,
+	HAL_RX_REO_DEST_RING_SW5,
+	HAL_RX_REO_DEST_RING_SW6,
+	HAL_RX_REO_DEST_RING_SW7,
+	HAL_RX_REO_DEST_RING_SW8,
+};
+
+#define HAL_REO_ENTR_RING_INFO0_QUEUE_ADDR_HI		GENMASK(7, 0)
+#define HAL_REO_ENTR_RING_INFO0_MPDU_BYTE_COUNT		GENMASK(21, 8)
+#define HAL_REO_ENTR_RING_INFO0_DEST_IND		GENMASK(26, 22)
+#define HAL_REO_ENTR_RING_INFO0_FRAMELESS_BAR		BIT(27)
+
+#define HAL_REO_ENTR_RING_INFO1_RXDMA_PUSH_REASON	GENMASK(1, 0)
+#define HAL_REO_ENTR_RING_INFO1_RXDMA_ERROR_CODE	GENMASK(6, 2)
+#define HAL_REO_ENTR_RING_INFO1_MPDU_FRAG_NUM		GENMASK(10, 7)
+#define HAL_REO_ENTR_RING_INFO1_SW_EXCEPTION		BIT(11)
+#define HAL_REO_ENTR_RING_INFO1_SW_EXCEPT_MPDU_DELINK	BIT(12)
+#define HAL_REO_ENTR_RING_INFO1_SW_EXCEPTION_RING_VLD	BIT(13)
+#define HAL_REO_ENTR_RING_INFO1_SW_EXCEPTION_RING	GENMASK(18, 14)
+#define HAL_REO_ENTR_RING_INFO1_MPDU_SEQ_NUM		GENMASK(30, 19)
+
+#define HAL_REO_ENTR_RING_INFO2_PHY_PPDU_ID		GENMASK(15, 0)
+#define HAL_REO_ENTR_RING_INFO2_SRC_LINK_ID		GENMASK(18, 16)
+#define HAL_REO_ENTR_RING_INFO2_RING_ID			GENMASK(27, 20)
+#define HAL_REO_ENTR_RING_INFO2_LOOPING_COUNT		GENMASK(31, 28)
+
+struct hal_reo_entrance_ring {
+	struct ath12k_buffer_addr buf_addr_info;
+	struct rx_mpdu_desc rx_mpdu_info;
+	__le32 queue_addr_lo;
+	__le32 info0; /* %HAL_REO_ENTR_RING_INFO0_ */
+	__le32 info1; /* %HAL_REO_ENTR_RING_INFO1_ */
+	__le32 info2; /* %HAL_REO_DEST_RING_INFO2_ */
+
+} __packed;
+
+/* hal_reo_entrance_ring
+ *
+ *		Producer: RXDMA
+ *		Consumer: REO
+ *
+ * buf_addr_info
+ *		Details of the physical address of a buffer or MSDU
+ *		link descriptor.
+ *
+ * rx_mpdu_info
+ *		General information related to the MPDU that is passed
+ *		on from REO entrance ring to the REO destination ring.
+ *
+ * queue_addr_lo
+ *		Address (lower 32 bits) of the REO queue descriptor.
+ *
+ * queue_addr_hi
+ *		Address (upper 8 bits) of the REO queue descriptor.
+ *
+ * mpdu_byte_count
+ *		An approximation of the number of bytes received in this MPDU.
+ *		Used to keeps stats on the amount of data flowing
+ *		through a queue.
+ *
+ * reo_destination_indication
+ *		The id of the reo exit ring where the msdu frame shall push
+ *		after (MPDU level) reordering has finished. Values are defined
+ *		in enum %HAL_RX_MSDU_DESC_REO_DEST_IND_.
+ *
+ * frameless_bar
+ *		Indicates that this REO entrance ring struct contains BAR info
+ *		from a multi TID BAR frame. The original multi TID BAR frame
+ *		itself contained all the REO info for the first TID, but all
+ *		the subsequent TID info and their linkage to the REO descriptors
+ *		is passed down as 'frameless' BAR info.
+ *
+ *		The only fields valid in this descriptor when this bit is set
+ *		are queue_addr_lo, queue_addr_hi, mpdu_sequence_number,
+ *		bar_frame and peer_meta_data.
+ *
+ * rxdma_push_reason
+ *		Reason for pushing this frame to this exit ring. Values are
+ *		defined in enum %HAL_REO_ENTR_RING_RXDMA_PUSH_REASON_.
+ *
+ * rxdma_error_code
+ *		Valid only when 'push_reason' is set. All error codes are
+ *		defined in enum %HAL_REO_ENTR_RING_RXDMA_ECODE_.
+ *
+ * mpdu_fragment_number
+ *		Field only valid when Reo_level_mpdu_frame_info.
+ *		Rx_mpdu_desc_info_details.Fragment_flag is set.
+ *
+ * sw_exception
+ *		When not set, REO is performing all its default MPDU processing
+ *		operations,
+ *		When set, this REO entrance descriptor is generated by FW, and
+ *		should be processed as an exception. This implies:
+ *		NO re-order function is needed.
+ *		MPDU delinking is determined by the setting of field
+ *		SW_excection_mpdu_delink
+ *
+ * sw_exception_mpdu_delink
+ *		Field only valid when SW_exception is set.
+ *		1'b0: REO should NOT delink the MPDU, and thus pass this
+ *			MPDU on to the destination ring as is. This implies that
+ *			in the REO_DESTINATION_RING struct field
+ *			Buf_or_link_desc_addr_info should point to an MSDU link
+ *			descriptor
+ *		1'b1: REO should perform the normal MPDU delink into MSDU operations.
+ *
+ * sw_exception_dest_ring
+ *		Field only valid when fields SW_exception and SW
+ *		exception_destination_ring_valid are set. values are defined
+ *		in %HAL_RX_REO_DEST_RING_.
+ *
+ * mpdu_seq_number
+ *		The field can have two different meanings based on the setting
+ *		of sub-field Reo level mpdu frame info.
+ *		Rx_mpdu_desc_info_details. BAR_frame
+ *		'BAR_frame' is NOT set:
+ *		The MPDU sequence number of the received frame.
+ *		'BAR_frame' is set.
+ *		The MPDU Start sequence number from the BAR frame
+ *
+ * phy_ppdu_id
+ *		A PPDU counter value that PHY increments for every PPDU received
+ *
+ * src_link_id
+ *		Set to the link ID of the PMAC that received the frame
+ *
+ * ring_id
+ *		The buffer pointer ring id.
+ *		0 - Idle ring
+ *		1 - N refers to other rings.
+ *
+ * looping_count
+ *		Indicates the number of times the producer of entries into
+ *		this ring has looped around the ring.
+ */
+
+#define HAL_REO_CMD_HDR_INFO0_CMD_NUMBER	GENMASK(15, 0)
+#define HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED	BIT(16)
+
+struct hal_reo_cmd_hdr {
+	__le32 info0;
+} __packed;
+
+#define HAL_REO_GET_QUEUE_STATS_INFO0_QUEUE_ADDR_HI	GENMASK(7, 0)
+#define HAL_REO_GET_QUEUE_STATS_INFO0_CLEAR_STATS	BIT(8)
+
+struct hal_reo_get_queue_stats {
+	struct hal_reo_cmd_hdr cmd;
+	__le32 queue_addr_lo;
+	__le32 info0;
+	__le32 rsvd0[6];
+	__le32 tlv64_pad;
+} __packed;
+
+/* hal_reo_get_queue_stats
+ *		Producer: SW
+ *		Consumer: REO
+ *
+ * cmd
+ *		Details for command execution tracking purposes.
+ *
+ * queue_addr_lo
+ *		Address (lower 32 bits) of the REO queue descriptor.
+ *
+ * queue_addr_hi
+ *		Address (upper 8 bits) of the REO queue descriptor.
+ *
+ * clear_stats
+ *		Clear stats settings. When set, Clear the stats after
+ *		generating the status.
+ *
+ *		Following stats will be cleared.
+ *		Timeout_count
+ *		Forward_due_to_bar_count
+ *		Duplicate_count
+ *		Frames_in_order_count
+ *		BAR_received_count
+ *		MPDU_Frames_processed_count
+ *		MSDU_Frames_processed_count
+ *		Total_processed_byte_count
+ *		Late_receive_MPDU_count
+ *		window_jump_2k
+ *		Hole_count
+ */
+
+#define HAL_REO_FLUSH_QUEUE_INFO0_DESC_ADDR_HI		GENMASK(7, 0)
+#define HAL_REO_FLUSH_QUEUE_INFO0_BLOCK_DESC_ADDR	BIT(8)
+#define HAL_REO_FLUSH_QUEUE_INFO0_BLOCK_RESRC_IDX	GENMASK(10, 9)
+
+struct hal_reo_flush_queue {
+	struct hal_reo_cmd_hdr cmd;
+	__le32 desc_addr_lo;
+	__le32 info0;
+	__le32 rsvd0[6];
+} __packed;
+
+#define HAL_REO_FLUSH_CACHE_INFO0_CACHE_ADDR_HI		GENMASK(7, 0)
+#define HAL_REO_FLUSH_CACHE_INFO0_FWD_ALL_MPDUS		BIT(8)
+#define HAL_REO_FLUSH_CACHE_INFO0_RELEASE_BLOCK_IDX	BIT(9)
+#define HAL_REO_FLUSH_CACHE_INFO0_BLOCK_RESRC_IDX	GENMASK(11, 10)
+#define HAL_REO_FLUSH_CACHE_INFO0_FLUSH_WO_INVALIDATE	BIT(12)
+#define HAL_REO_FLUSH_CACHE_INFO0_BLOCK_CACHE_USAGE	BIT(13)
+#define HAL_REO_FLUSH_CACHE_INFO0_FLUSH_ALL		BIT(14)
+#define HAL_REO_FLUSH_CACHE_INFO0_FLUSH_QUEUE_1K_DESC	BIT(15)
+
+struct hal_reo_flush_cache {
+	struct hal_reo_cmd_hdr cmd;
+	__le32 cache_addr_lo;
+	__le32 info0;
+	__le32 rsvd0[6];
+} __packed;
+
+#define HAL_TCL_DATA_CMD_INFO0_CMD_TYPE			BIT(0)
+#define HAL_TCL_DATA_CMD_INFO0_DESC_TYPE		BIT(1)
+#define HAL_TCL_DATA_CMD_INFO0_BANK_ID			GENMASK(7, 2)
+#define HAL_TCL_DATA_CMD_INFO0_TX_NOTIFY_FRAME		GENMASK(10, 8)
+#define HAL_TCL_DATA_CMD_INFO0_HDR_LEN_READ_SEL		BIT(11)
+#define HAL_TCL_DATA_CMD_INFO0_BUF_TIMESTAMP		GENMASK(30, 12)
+#define HAL_TCL_DATA_CMD_INFO0_BUF_TIMESTAMP_VLD	BIT(31)
+
+#define HAL_TCL_DATA_CMD_INFO1_CMD_NUM		GENMASK(31, 16)
+
+#define HAL_TCL_DATA_CMD_INFO2_DATA_LEN		GENMASK(15, 0)
+#define HAL_TCL_DATA_CMD_INFO2_IP4_CKSUM_EN	BIT(16)
+#define HAL_TCL_DATA_CMD_INFO2_UDP4_CKSUM_EN	BIT(17)
+#define HAL_TCL_DATA_CMD_INFO2_UDP6_CKSUM_EN	BIT(18)
+#define HAL_TCL_DATA_CMD_INFO2_TCP4_CKSUM_EN	BIT(19)
+#define HAL_TCL_DATA_CMD_INFO2_TCP6_CKSUM_EN	BIT(20)
+#define HAL_TCL_DATA_CMD_INFO2_TO_FW		BIT(21)
+#define HAL_TCL_DATA_CMD_INFO2_PKT_OFFSET	GENMASK(31, 23)
+
+#define HAL_TCL_DATA_CMD_INFO3_TID_OVERWRITE		BIT(0)
+#define HAL_TCL_DATA_CMD_INFO3_FLOW_OVERRIDE_EN		BIT(1)
+#define HAL_TCL_DATA_CMD_INFO3_CLASSIFY_INFO_SEL	GENMASK(3, 2)
+#define HAL_TCL_DATA_CMD_INFO3_TID			GENMASK(7, 4)
+#define HAL_TCL_DATA_CMD_INFO3_FLOW_OVERRIDE		BIT(8)
+#define HAL_TCL_DATA_CMD_INFO3_PMAC_ID			GENMASK(10, 9)
+#define HAL_TCL_DATA_CMD_INFO3_MSDU_COLOR		GENMASK(12, 11)
+#define HAL_TCL_DATA_CMD_INFO3_VDEV_ID			GENMASK(31, 24)
+
+#define HAL_TCL_DATA_CMD_INFO4_SEARCH_INDEX		GENMASK(19, 0)
+#define HAL_TCL_DATA_CMD_INFO4_CACHE_SET_NUM		GENMASK(23, 20)
+#define HAL_TCL_DATA_CMD_INFO4_IDX_LOOKUP_OVERRIDE	BIT(24)
+
+#define HAL_TCL_DATA_CMD_INFO5_RING_ID			GENMASK(27, 20)
+#define HAL_TCL_DATA_CMD_INFO5_LOOPING_COUNT		GENMASK(31, 28)
+
+struct hal_tcl_data_cmd {
+	struct ath12k_buffer_addr buf_addr_info;
+	__le32 info0;
+	__le32 info1;
+	__le32 info2;
+	__le32 info3;
+	__le32 info4;
+	__le32 info5;
+} __packed;
+
+/* hal_tcl_data_cmd
+ *
+ * buf_addr_info
+ *		Details of the physical address of a buffer or MSDU
+ *		link descriptor.
+ *
+ * tcl_cmd_type
+ *		used to select the type of TCL Command descriptor
+ *
+ * desc_type
+ *		Indicates the type of address provided in the buf_addr_info.
+ *		Values are defined in enum %HAL_REO_DEST_RING_BUFFER_TYPE_.
+ *
+ * bank_id
+ *		used to select one of the TCL register banks for fields removed
+ *		from 'TCL_DATA_CMD' that do not change often within one virtual
+ *		device or a set of virtual devices:
+ *
+ * tx_notify_frame
+ *		TCL copies this value to 'TQM_ENTRANCE_RING' field FW_tx_notify_frame.
+ *
+ * hdr_length_read_sel
+ *		used to select the per 'encap_type' register set for MSDU header
+ *		read length
+ *
+ * buffer_timestamp
+ * buffer_timestamp_valid
+ *		Frame system entrance timestamp. It shall be filled by first
+ *		module (SW, TCL or TQM) that sees the frames first.
+ *
+ * cmd_num
+ *		This number can be used to match against status.
+ *
+ * data_length
+ *		MSDU length in case of direct descriptor. Length of link
+ *		extension descriptor in case of Link extension descriptor.
+ *
+ * *_checksum_en
+ *		Enable checksum replacement for ipv4, udp_over_ipv4, ipv6,
+ *		udp_over_ipv6, tcp_over_ipv4 and tcp_over_ipv6.
+ *
+ * to_fw
+ *		Forward packet to FW along with classification result. The
+ *		packet will not be forward to TQM when this bit is set.
+ *		1'b0: Use classification result to forward the packet.
+ *		1'b1: Override classification result & forward packet only to fw
+ *
+ * packet_offset
+ *		Packet offset from Metadata in case of direct buffer descriptor.
+ *
+ * hlos_tid_overwrite
+ *
+ *		When set, TCL shall ignore the IP DSCP and VLAN PCP
+ *		fields and use HLOS_TID as the final TID. Otherwise TCL
+ *		shall consider the DSCP and PCP fields as well as HLOS_TID
+ *		and choose a final TID based on the configured priority
+ *
+ * flow_override_enable
+ *		TCL uses this to select the flow pointer from the peer table,
+ *		which can be overridden by SW for pre-encrypted raw WiFi packets
+ *		that cannot be parsed for UDP or for other MLO
+ *		0 - FP_PARSE_IP: Use the flow-pointer based on parsing the IPv4
+ *				 or IPv6 header.
+ *		1 - FP_USE_OVERRIDE: Use the who_classify_info_sel and
+ *				     flow_override fields to select the flow-pointer
+ *
+ * who_classify_info_sel
+ *		Field only valid when flow_override_enable is set to FP_USE_OVERRIDE.
+ *		This field is used to select  one of the 'WHO_CLASSIFY_INFO's in the
+ *		peer table in case more than 2 flows are mapped to a single TID.
+ *		0: To choose Flow 0 and 1 of any TID use this value.
+ *		1: To choose Flow 2 and 3 of any TID use this value.
+ *		2: To choose Flow 4 and 5 of any TID use this value.
+ *		3: To choose Flow 6 and 7 of any TID use this value.
+ *
+ *		If who_classify_info sel is not in sync with the num_tx_classify_info
+ *		field from address search, then TCL will set 'who_classify_info_sel'
+ *		to 0 use flows 0 and 1.
+ *
+ * hlos_tid
+ *		HLOS MSDU priority
+ *		Field is used when HLOS_TID_overwrite is set.
+ *
+ * flow_override
+ *		Field only valid when flow_override_enable is set to FP_USE_OVERRIDE
+ *		TCL uses this to select the flow pointer from the peer table,
+ *		which can be overridden by SW for pre-encrypted raw WiFi packets
+ *		that cannot be parsed for UDP or for other MLO
+ *		0 - FP_USE_NON_UDP: Use the non-UDP flow pointer (flow 0)
+ *		1 - FP_USE_UDP: Use the UDP flow pointer (flow 1)
+ *
+ * pmac_id
+ *		TCL uses this PMAC_ID in address search, i.e, while
+ *		finding matching entry for the packet in AST corresponding
+ *		to given PMAC_ID
+ *
+ *		If PMAC ID is all 1s (=> value 3), it indicates wildcard
+ *		match for any PMAC
+ *
+ * vdev_id
+ *		Virtual device ID to check against the address search entry to
+ *		avoid security issues from transmitting packets from an incorrect
+ *		virtual device
+ *
+ * search_index
+ *		The index that will be used for index based address or
+ *		flow search. The field is valid when 'search_type' is  1 or 2.
+ *
+ * cache_set_num
+ *
+ *		Cache set number that should be used to cache the index
+ *		based search results, for address and flow search. This
+ *		value should be equal to LSB four bits of the hash value of
+ *		match data, in case of search index points to an entry which
+ *		may be used in content based search also. The value can be
+ *		anything when the entry pointed by search index will not be
+ *		used for content based search.
+ *
+ * index_loop_override
+ *		When set, address search and packet routing is forced to use
+ *		'search_index' instead of following the register configuration
+ *		selected by Bank_id.
+ *
+ * ring_id
+ *		The buffer pointer ring ID.
+ *		0 refers to the IDLE ring
+ *		1 - N refers to other rings
+ *
+ * looping_count
+ *
+ *		A count value that indicates the number of times the
+ *		producer of entries into the Ring has looped around the
+ *		ring.
+ *
+ *		At initialization time, this value is set to 0. On the
+ *		first loop, this value is set to 1. After the max value is
+ *		reached allowed by the number of bits for this field, the
+ *		count value continues with 0 again.
+ *
+ *		In case SW is the consumer of the ring entries, it can
+ *		use this field to figure out up to where the producer of
+ *		entries has created new entries. This eliminates the need to
+ *		check where the head pointer' of the ring is located once
+ *		the SW starts processing an interrupt indicating that new
+ *		entries have been put into this ring...
+ *
+ *		Also note that SW if it wants only needs to look at the
+ *		LSB bit of this count value.
+ */
+
+#define HAL_TCL_DESC_LEN sizeof(struct hal_tcl_data_cmd)
+
+#define HAL_TX_MSDU_EXT_INFO0_BUF_PTR_LO	GENMASK(31, 0)
+
+#define HAL_TX_MSDU_EXT_INFO1_BUF_PTR_HI	GENMASK(7, 0)
+#define HAL_TX_MSDU_EXT_INFO1_EXTN_OVERRIDE	BIT(8)
+#define HAL_TX_MSDU_EXT_INFO1_ENCAP_TYPE	GENMASK(10, 9)
+#define HAL_TX_MSDU_EXT_INFO1_ENCRYPT_TYPE	GENMASK(14, 11)
+#define HAL_TX_MSDU_EXT_INFO1_BUF_LEN		GENMASK(31, 16)
+
+struct hal_tx_msdu_ext_desc {
+	__le32 rsvd0[6];
+	__le32 info0;
+	__le32 info1;
+	__le32 rsvd1[10];
+};
+
+struct hal_tcl_gse_cmd {
+	__le32 ctrl_buf_addr_lo;
+	__le32 info0;
+	__le32 meta_data[2];
+	__le32 rsvd0[2];
+	__le32 info1;
+} __packed;
+
+/* hal_tcl_gse_cmd
+ *
+ * ctrl_buf_addr_lo, ctrl_buf_addr_hi
+ *		Address of a control buffer containing additional info needed
+ *		for this command execution.
+ *
+ * meta_data
+ *		Meta data to be returned in the status descriptor
+ */
+
+enum hal_tcl_cache_op_res {
+	HAL_TCL_CACHE_OP_RES_DONE,
+	HAL_TCL_CACHE_OP_RES_NOT_FOUND,
+	HAL_TCL_CACHE_OP_RES_TIMEOUT,
+};
+
+struct hal_tcl_status_ring {
+	__le32 info0;
+	__le32 msdu_byte_count;
+	__le32 msdu_timestamp;
+	__le32 meta_data[2];
+	__le32 info1;
+	__le32 rsvd0;
+	__le32 info2;
+} __packed;
+
+/* hal_tcl_status_ring
+ *
+ * msdu_cnt
+ * msdu_byte_count
+ *		MSDU count of Entry and MSDU byte count for entry 1.
+ *
+ */
+
+#define HAL_CE_SRC_DESC_ADDR_INFO_ADDR_HI	GENMASK(7, 0)
+#define HAL_CE_SRC_DESC_ADDR_INFO_HASH_EN	BIT(8)
+#define HAL_CE_SRC_DESC_ADDR_INFO_BYTE_SWAP	BIT(9)
+#define HAL_CE_SRC_DESC_ADDR_INFO_DEST_SWAP	BIT(10)
+#define HAL_CE_SRC_DESC_ADDR_INFO_GATHER	BIT(11)
+#define HAL_CE_SRC_DESC_ADDR_INFO_LEN		GENMASK(31, 16)
+
+#define HAL_CE_SRC_DESC_META_INFO_DATA		GENMASK(15, 0)
+
+#define HAL_CE_SRC_DESC_FLAGS_RING_ID		GENMASK(27, 20)
+#define HAL_CE_SRC_DESC_FLAGS_LOOP_CNT		HAL_SRNG_DESC_LOOP_CNT
+
+struct hal_ce_srng_src_desc {
+	__le32 buffer_addr_low;
+	__le32 buffer_addr_info; /* %HAL_CE_SRC_DESC_ADDR_INFO_ */
+	__le32 meta_info; /* %HAL_CE_SRC_DESC_META_INFO_ */
+	__le32 flags; /* %HAL_CE_SRC_DESC_FLAGS_ */
+} __packed;
+
+/* hal_ce_srng_src_desc
+ *
+ * buffer_addr_lo
+ *		LSB 32 bits of the 40 Bit Pointer to the source buffer
+ *
+ * buffer_addr_hi
+ *		MSB 8 bits of the 40 Bit Pointer to the source buffer
+ *
+ * toeplitz_en
+ *		Enable generation of 32-bit Toeplitz-LFSR hash for
+ *		data transfer. In case of gather field in first source
+ *		ring entry of the gather copy cycle in taken into account.
+ *
+ * src_swap
+ *		Treats source memory organization as big-endian. For
+ *		each dword read (4 bytes), the byte 0 is swapped with byte 3
+ *		and byte 1 is swapped with byte 2.
+ *		In case of gather field in first source ring entry of
+ *		the gather copy cycle in taken into account.
+ *
+ * dest_swap
+ *		Treats destination memory organization as big-endian.
+ *		For each dword write (4 bytes), the byte 0 is swapped with
+ *		byte 3 and byte 1 is swapped with byte 2.
+ *		In case of gather field in first source ring entry of
+ *		the gather copy cycle in taken into account.
+ *
+ * gather
+ *		Enables gather of multiple copy engine source
+ *		descriptors to one destination.
+ *
+ * ce_res_0
+ *		Reserved
+ *
+ *
+ * length
+ *		Length of the buffer in units of octets of the current
+ *		descriptor
+ *
+ * fw_metadata
+ *		Meta data used by FW.
+ *		In case of gather field in first source ring entry of
+ *		the gather copy cycle in taken into account.
+ *
+ * ce_res_1
+ *		Reserved
+ *
+ * ce_res_2
+ *		Reserved
+ *
+ * ring_id
+ *		The buffer pointer ring ID.
+ *		0 refers to the IDLE ring
+ *		1 - N refers to other rings
+ *		Helps with debugging when dumping ring contents.
+ *
+ * looping_count
+ *		A count value that indicates the number of times the
+ *		producer of entries into the Ring has looped around the
+ *		ring.
+ *
+ *		At initialization time, this value is set to 0. On the
+ *		first loop, this value is set to 1. After the max value is
+ *		reached allowed by the number of bits for this field, the
+ *		count value continues with 0 again.
+ *
+ *		In case SW is the consumer of the ring entries, it can
+ *		use this field to figure out up to where the producer of
+ *		entries has created new entries. This eliminates the need to
+ *		check where the head pointer' of the ring is located once
+ *		the SW starts processing an interrupt indicating that new
+ *		entries have been put into this ring...
+ *
+ *		Also note that SW if it wants only needs to look at the
+ *		LSB bit of this count value.
+ */
+
+#define HAL_CE_DEST_DESC_ADDR_INFO_ADDR_HI		GENMASK(7, 0)
+#define HAL_CE_DEST_DESC_ADDR_INFO_RING_ID		GENMASK(27, 20)
+#define HAL_CE_DEST_DESC_ADDR_INFO_LOOP_CNT		HAL_SRNG_DESC_LOOP_CNT
+
+struct hal_ce_srng_dest_desc {
+	__le32 buffer_addr_low;
+	__le32 buffer_addr_info; /* %HAL_CE_DEST_DESC_ADDR_INFO_ */
+} __packed;
+
+/* hal_ce_srng_dest_desc
+ *
+ * dst_buffer_low
+ *		LSB 32 bits of the 40 Bit Pointer to the Destination
+ *		buffer
+ *
+ * dst_buffer_high
+ *		MSB 8 bits of the 40 Bit Pointer to the Destination
+ *		buffer
+ *
+ * ce_res_4
+ *		Reserved
+ *
+ * ring_id
+ *		The buffer pointer ring ID.
+ *		0 refers to the IDLE ring
+ *		1 - N refers to other rings
+ *		Helps with debugging when dumping ring contents.
+ *
+ * looping_count
+ *		A count value that indicates the number of times the
+ *		producer of entries into the Ring has looped around the
+ *		ring.
+ *
+ *		At initialization time, this value is set to 0. On the
+ *		first loop, this value is set to 1. After the max value is
+ *		reached allowed by the number of bits for this field, the
+ *		count value continues with 0 again.
+ *
+ *		In case SW is the consumer of the ring entries, it can
+ *		use this field to figure out up to where the producer of
+ *		entries has created new entries. This eliminates the need to
+ *		check where the head pointer' of the ring is located once
+ *		the SW starts processing an interrupt indicating that new
+ *		entries have been put into this ring...
+ *
+ *		Also note that SW if it wants only needs to look at the
+ *		LSB bit of this count value.
+ */
+
+#define HAL_CE_DST_STATUS_DESC_FLAGS_HASH_EN		BIT(8)
+#define HAL_CE_DST_STATUS_DESC_FLAGS_BYTE_SWAP		BIT(9)
+#define HAL_CE_DST_STATUS_DESC_FLAGS_DEST_SWAP		BIT(10)
+#define HAL_CE_DST_STATUS_DESC_FLAGS_GATHER		BIT(11)
+#define HAL_CE_DST_STATUS_DESC_FLAGS_LEN		GENMASK(31, 16)
+
+#define HAL_CE_DST_STATUS_DESC_META_INFO_DATA		GENMASK(15, 0)
+#define HAL_CE_DST_STATUS_DESC_META_INFO_RING_ID	GENMASK(27, 20)
+#define HAL_CE_DST_STATUS_DESC_META_INFO_LOOP_CNT	HAL_SRNG_DESC_LOOP_CNT
+
+struct hal_ce_srng_dst_status_desc {
+	__le32 flags; /* %HAL_CE_DST_STATUS_DESC_FLAGS_ */
+	__le32 toeplitz_hash0;
+	__le32 toeplitz_hash1;
+	__le32 meta_info; /* HAL_CE_DST_STATUS_DESC_META_INFO_ */
+} __packed;
+
+/* hal_ce_srng_dst_status_desc
+ *
+ * ce_res_5
+ *		Reserved
+ *
+ * toeplitz_en
+ *
+ * src_swap
+ *		Source memory buffer swapped
+ *
+ * dest_swap
+ *		Destination  memory buffer swapped
+ *
+ * gather
+ *		Gather of multiple copy engine source descriptors to one
+ *		destination enabled
+ *
+ * ce_res_6
+ *		Reserved
+ *
+ * length
+ *		Sum of all the Lengths of the source descriptor in the
+ *		gather chain
+ *
+ * toeplitz_hash_0
+ *		32 LS bits of 64 bit Toeplitz LFSR hash result
+ *
+ * toeplitz_hash_1
+ *		32 MS bits of 64 bit Toeplitz LFSR hash result
+ *
+ * fw_metadata
+ *		Meta data used by FW
+ *		In case of gather field in first source ring entry of
+ *		the gather copy cycle in taken into account.
+ *
+ * ce_res_7
+ *		Reserved
+ *
+ * ring_id
+ *		The buffer pointer ring ID.
+ *		0 refers to the IDLE ring
+ *		1 - N refers to other rings
+ *		Helps with debugging when dumping ring contents.
+ *
+ * looping_count
+ *		A count value that indicates the number of times the
+ *		producer of entries into the Ring has looped around the
+ *		ring.
+ *
+ *		At initialization time, this value is set to 0. On the
+ *		first loop, this value is set to 1. After the max value is
+ *		reached allowed by the number of bits for this field, the
+ *		count value continues with 0 again.
+ *
+ *		In case SW is the consumer of the ring entries, it can
+ *		use this field to figure out up to where the producer of
+ *		entries has created new entries. This eliminates the need to
+ *		check where the head pointer' of the ring is located once
+ *		the SW starts processing an interrupt indicating that new
+ *		entries have been put into this ring...
+ *
+ *		Also note that SW if it wants only needs to look at the
+ *			LSB bit of this count value.
+ */
+
+#define HAL_TX_RATE_STATS_INFO0_VALID		BIT(0)
+#define HAL_TX_RATE_STATS_INFO0_BW		GENMASK(3, 1)
+#define HAL_TX_RATE_STATS_INFO0_PKT_TYPE	GENMASK(7, 4)
+#define HAL_TX_RATE_STATS_INFO0_STBC		BIT(8)
+#define HAL_TX_RATE_STATS_INFO0_LDPC		BIT(9)
+#define HAL_TX_RATE_STATS_INFO0_SGI		GENMASK(11, 10)
+#define HAL_TX_RATE_STATS_INFO0_MCS		GENMASK(15, 12)
+#define HAL_TX_RATE_STATS_INFO0_OFDMA_TX	BIT(16)
+#define HAL_TX_RATE_STATS_INFO0_TONES_IN_RU	GENMASK(28, 17)
+
+struct hal_tx_rate_stats {
+	__le32 info0;
+	__le32 tsf;
+} __packed;
+
+#define HAL_WBM_COMPL_RX_INFO0_REL_SRC_MODULE		GENMASK(2, 0)
+#define HAL_WBM_COMPL_RX_INFO0_BM_ACTION		GENMASK(5, 3)
+#define HAL_WBM_COMPL_RX_INFO0_DESC_TYPE		GENMASK(8, 6)
+#define HAL_WBM_COMPL_RX_INFO0_RBM			GENMASK(12, 9)
+#define HAL_WBM_COMPL_RX_INFO0_RXDMA_PUSH_REASON	GENMASK(18, 17)
+#define HAL_WBM_COMPL_RX_INFO0_RXDMA_ERROR_CODE		GENMASK(23, 19)
+#define HAL_WBM_COMPL_RX_INFO0_REO_PUSH_REASON		GENMASK(25, 24)
+#define HAL_WBM_COMPL_RX_INFO0_REO_ERROR_CODE		GENMASK(30, 26)
+#define HAL_WBM_COMPL_RX_INFO0_WBM_INTERNAL_ERROR	BIT(31)
+
+#define HAL_WBM_COMPL_RX_INFO1_PHY_ADDR_HI		GENMASK(7, 0)
+#define HAL_WBM_COMPL_RX_INFO1_SW_COOKIE		GENMASK(27, 8)
+#define HAL_WBM_COMPL_RX_INFO1_LOOPING_COUNT		GENMASK(31, 28)
+
+struct hal_wbm_completion_ring_rx {
+	__le32 addr_lo;
+	__le32 addr_hi;
+	__le32 info0;
+	struct rx_mpdu_desc rx_mpdu_info;
+	struct rx_msdu_desc rx_msdu_info;
+	__le32 phy_addr_lo;
+	__le32 info1;
+} __packed;
+
+#define HAL_WBM_COMPL_TX_INFO0_REL_SRC_MODULE		GENMASK(2, 0)
+#define HAL_WBM_COMPL_TX_INFO0_DESC_TYPE		GENMASK(8, 6)
+#define HAL_WBM_COMPL_TX_INFO0_RBM			GENMASK(12, 9)
+#define HAL_WBM_COMPL_TX_INFO0_TQM_RELEASE_REASON	GENMASK(16, 13)
+#define HAL_WBM_COMPL_TX_INFO0_RBM_OVERRIDE_VLD		BIT(17)
+#define HAL_WBM_COMPL_TX_INFO0_SW_COOKIE_LO		GENMASK(29, 18)
+#define HAL_WBM_COMPL_TX_INFO0_CC_DONE			BIT(30)
+#define HAL_WBM_COMPL_TX_INFO0_WBM_INTERNAL_ERROR	BIT(31)
+
+#define HAL_WBM_COMPL_TX_INFO1_TQM_STATUS_NUMBER	GENMASK(23, 0)
+#define HAL_WBM_COMPL_TX_INFO1_TRANSMIT_COUNT		GENMASK(30, 24)
+#define HAL_WBM_COMPL_TX_INFO1_SW_REL_DETAILS_VALID	BIT(31)
+
+#define HAL_WBM_COMPL_TX_INFO2_ACK_FRAME_RSSI		GENMASK(7, 0)
+#define HAL_WBM_COMPL_TX_INFO2_FIRST_MSDU		BIT(8)
+#define HAL_WBM_COMPL_TX_INFO2_LAST_MSDU		BIT(9)
+#define HAL_WBM_COMPL_TX_INFO2_FW_TX_NOTIF_FRAME	GENMASK(12, 10)
+#define HAL_WBM_COMPL_TX_INFO2_BUFFER_TIMESTAMP		GENMASK(31, 13)
+
+#define HAL_WBM_COMPL_TX_INFO3_PEER_ID			GENMASK(15, 0)
+#define HAL_WBM_COMPL_TX_INFO3_TID			GENMASK(19, 16)
+#define HAL_WBM_COMPL_TX_INFO3_SW_COOKIE_HI		GENMASK(27, 20)
+#define HAL_WBM_COMPL_TX_INFO3_LOOPING_COUNT		GENMASK(31, 28)
+
+struct hal_wbm_completion_ring_tx {
+	__le32 buf_va_lo;
+	__le32 buf_va_hi;
+	__le32 info0;
+	__le32 info1;
+	__le32 info2;
+	struct hal_tx_rate_stats rate_stats;
+	__le32 info3;
+} __packed;
+
+#define HAL_WBM_RELEASE_TX_INFO0_REL_SRC_MODULE		GENMASK(2, 0)
+#define HAL_WBM_RELEASE_TX_INFO0_BM_ACTION		GENMASK(5, 3)
+#define HAL_WBM_RELEASE_TX_INFO0_DESC_TYPE		GENMASK(8, 6)
+#define HAL_WBM_RELEASE_TX_INFO0_FIRST_MSDU_IDX		GENMASK(12, 9)
+#define HAL_WBM_RELEASE_TX_INFO0_TQM_RELEASE_REASON	GENMASK(18, 13)
+#define HAL_WBM_RELEASE_TX_INFO0_RBM_OVERRIDE_VLD	BIT(17)
+#define HAL_WBM_RELEASE_TX_INFO0_SW_BUFFER_COOKIE_11_0	GENMASK(29, 18)
+#define HAL_WBM_RELEASE_TX_INFO0_WBM_INTERNAL_ERROR	BIT(31)
+
+#define HAL_WBM_RELEASE_TX_INFO1_TQM_STATUS_NUMBER	GENMASK(23, 0)
+#define HAL_WBM_RELEASE_TX_INFO1_TRANSMIT_COUNT		GENMASK(30, 24)
+#define HAL_WBM_RELEASE_TX_INFO1_SW_REL_DETAILS_VALID	BIT(31)
+
+#define HAL_WBM_RELEASE_TX_INFO2_ACK_FRAME_RSSI		GENMASK(7, 0)
+#define HAL_WBM_RELEASE_TX_INFO2_FIRST_MSDU		BIT(8)
+#define HAL_WBM_RELEASE_TX_INFO2_LAST_MSDU		BIT(9)
+#define HAL_WBM_RELEASE_TX_INFO2_FW_TX_NOTIF_FRAME	GENMASK(12, 10)
+#define HAL_WBM_RELEASE_TX_INFO2_BUFFER_TIMESTAMP	GENMASK(31, 13)
+
+#define HAL_WBM_RELEASE_TX_INFO3_PEER_ID		GENMASK(15, 0)
+#define HAL_WBM_RELEASE_TX_INFO3_TID			GENMASK(19, 16)
+#define HAL_WBM_RELEASE_TX_INFO3_SW_BUFFER_COOKIE_19_12	GENMASK(27, 20)
+#define HAL_WBM_RELEASE_TX_INFO3_LOOPING_COUNT		GENMASK(31, 28)
+
+struct hal_wbm_release_ring_tx {
+	struct ath12k_buffer_addr buf_addr_info;
+	__le32 info0;
+	__le32 info1;
+	__le32 info2;
+	struct hal_tx_rate_stats rate_stats;
+	__le32 info3;
+} __packed;
+
+#define HAL_WBM_RELEASE_RX_INFO0_REL_SRC_MODULE		GENMASK(2, 0)
+#define HAL_WBM_RELEASE_RX_INFO0_BM_ACTION		GENMASK(5, 3)
+#define HAL_WBM_RELEASE_RX_INFO0_DESC_TYPE		GENMASK(8, 6)
+#define HAL_WBM_RELEASE_RX_INFO0_FIRST_MSDU_IDX		GENMASK(12, 9)
+#define HAL_WBM_RELEASE_RX_INFO0_CC_STATUS		BIT(16)
+#define HAL_WBM_RELEASE_RX_INFO0_RXDMA_PUSH_REASON	GENMASK(18, 17)
+#define HAL_WBM_RELEASE_RX_INFO0_RXDMA_ERROR_CODE	GENMASK(23, 19)
+#define HAL_WBM_RELEASE_RX_INFO0_REO_PUSH_REASON	GENMASK(25, 24)
+#define HAL_WBM_RELEASE_RX_INFO0_REO_ERROR_CODE		GENMASK(30, 26)
+#define HAL_WBM_RELEASE_RX_INFO0_WBM_INTERNAL_ERROR	BIT(31)
+
+#define HAL_WBM_RELEASE_RX_INFO2_RING_ID		GENMASK(27, 20)
+#define HAL_WBM_RELEASE_RX_INFO2_LOOPING_COUNT		GENMASK(31, 28)
+
+struct hal_wbm_release_ring_rx {
+	struct ath12k_buffer_addr buf_addr_info;
+	__le32 info0;
+	struct rx_mpdu_desc rx_mpdu_info;
+	struct rx_msdu_desc rx_msdu_info;
+	__le32 info1;
+	__le32 info2;
+} __packed;
+
+#define HAL_WBM_RELEASE_RX_CC_INFO0_RBM			GENMASK(12, 9)
+#define HAL_WBM_RELEASE_RX_CC_INFO1_COOKIE		GENMASK(27, 8)
+/* Used when hw cc is success */
+struct hal_wbm_release_ring_cc_rx {
+	__le32 buf_va_lo;
+	__le32 buf_va_hi;
+	__le32 info0;
+	struct rx_mpdu_desc rx_mpdu_info;
+	struct rx_msdu_desc rx_msdu_info;
+	__le32 buf_pa_lo;
+	__le32 info1;
+} __packed;
+
+#define HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE		GENMASK(2, 0)
+#define HAL_WBM_RELEASE_INFO0_BM_ACTION			GENMASK(5, 3)
+#define HAL_WBM_RELEASE_INFO0_DESC_TYPE			GENMASK(8, 6)
+#define HAL_WBM_RELEASE_INFO0_RXDMA_PUSH_REASON		GENMASK(18, 17)
+#define HAL_WBM_RELEASE_INFO0_RXDMA_ERROR_CODE		GENMASK(23, 19)
+#define HAL_WBM_RELEASE_INFO0_REO_PUSH_REASON		GENMASK(25, 24)
+#define HAL_WBM_RELEASE_INFO0_REO_ERROR_CODE		GENMASK(30, 26)
+#define HAL_WBM_RELEASE_INFO0_WBM_INTERNAL_ERROR	BIT(31)
+
+#define HAL_WBM_RELEASE_INFO3_FIRST_MSDU		BIT(0)
+#define HAL_WBM_RELEASE_INFO3_LAST_MSDU			BIT(1)
+#define HAL_WBM_RELEASE_INFO3_CONTINUATION		BIT(2)
+
+#define HAL_WBM_RELEASE_INFO5_LOOPING_COUNT		GENMASK(31, 28)
+
+struct hal_wbm_release_ring {
+	struct ath12k_buffer_addr buf_addr_info;
+	__le32 info0;
+	__le32 info1;
+	__le32 info2;
+	__le32 info3;
+	__le32 info4;
+	__le32 info5;
+} __packed;
+
+/* hal_wbm_release_ring
+ *
+ *	Producer: SW/TQM/RXDMA/REO/SWITCH
+ *	Consumer: WBM/SW/FW
+ *
+ * HTT tx status is overlaid on wbm_release ring on 4-byte words 2, 3, 4 and 5
+ * for software based completions.
+ *
+ * buf_addr_info
+ *	Details of the physical address of the buffer or link descriptor.
+ *
+ * release_source_module
+ *	Indicates which module initiated the release of this buffer/descriptor.
+ *	Values are defined in enum %HAL_WBM_REL_SRC_MODULE_.
+ *
+ * buffer_or_desc_type
+ *	Field only valid when WBM is marked as the return_buffer_manager in
+ *	the Released_Buffer_address_info. Indicates that type of buffer or
+ *	descriptor is being released. Values are in enum %HAL_WBM_REL_DESC_TYPE.
+ *
+ * wbm_internal_error
+ *	Is set when WBM got a buffer pointer but the action was to push it to
+ *	the idle link descriptor ring or do link related activity OR
+ *	Is set when WBM got a link buffer pointer but the action was to push it
+ *	to the buffer descriptor ring.
+ *
+ * looping_count
+ *	A count value that indicates the number of times the
+ *	producer of entries into the Buffer Manager Ring has looped
+ *	around the ring.
+ *
+ *	At initialization time, this value is set to 0. On the
+ *	first loop, this value is set to 1. After the max value is
+ *	reached allowed by the number of bits for this field, the
+ *	count value continues with 0 again.
+ *
+ *	In case SW is the consumer of the ring entries, it can
+ *	use this field to figure out up to where the producer of
+ *	entries has created new entries. This eliminates the need to
+ *	check where the head pointer' of the ring is located once
+ *	the SW starts processing an interrupt indicating that new
+ *	entries have been put into this ring...
+ *
+ *	Also note that SW if it wants only needs to look at the
+ *	LSB bit of this count value.
+ */
+
+/**
+ * enum hal_wbm_tqm_rel_reason - TQM release reason code
+ * @HAL_WBM_TQM_REL_REASON_FRAME_ACKED: ACK or BACK received for the frame
+ * @HAL_WBM_TQM_REL_REASON_CMD_REMOVE_MPDU: Command remove_mpdus initiated by SW
+ * @HAL_WBM_TQM_REL_REASON_CMD_REMOVE_TX: Command remove transmitted_mpdus
+ *	initiated by sw.
+ * @HAL_WBM_TQM_REL_REASON_CMD_REMOVE_NOTX: Command remove untransmitted_mpdus
+ *	initiated by sw.
+ * @HAL_WBM_TQM_REL_REASON_CMD_REMOVE_AGED_FRAMES: Command remove aged msdus or
+ *	mpdus.
+ * @HAL_WBM_TQM_REL_REASON_CMD_REMOVE_RESEAON1: Remove command initiated by
+ *	fw with fw_reason1.
+ * @HAL_WBM_TQM_REL_REASON_CMD_REMOVE_RESEAON2: Remove command initiated by
+ *	fw with fw_reason2.
+ * @HAL_WBM_TQM_REL_REASON_CMD_REMOVE_RESEAON3: Remove command initiated by
+ *	fw with fw_reason3.
+ * @HAL_WBM_TQM_REL_REASON_CMD_DISABLE_QUEUE: Remove command initiated by
+ *	fw with disable queue.
+ * @HAL_WBM_TQM_REL_REASON_CMD_TILL_NONMATCHING: Remove command initiated by
+ *	fw to remove all mpdu until 1st non-match.
+ * @HAL_WBM_TQM_REL_REASON_DROP_THRESHOLD: Dropped due to drop threshold
+ *	criteria
+ * @HAL_WBM_TQM_REL_REASON_DROP_LINK_DESC_UNAVAIL: Dropped due to link desc
+ *	not available
+ * @HAL_WBM_TQM_REL_REASON_DROP_OR_INVALID_MSDU: Dropped due drop bit set or
+ *	null flow
+ * @HAL_WBM_TQM_REL_REASON_MULTICAST_DROP: Dropped due mcast drop set for VDEV
+ * @HAL_WBM_TQM_REL_REASON_VDEV_MISMATCH_DROP: Dropped due to being set with
+ *	'TCL_drop_reason'
+ */
+enum hal_wbm_tqm_rel_reason {
+	HAL_WBM_TQM_REL_REASON_FRAME_ACKED,
+	HAL_WBM_TQM_REL_REASON_CMD_REMOVE_MPDU,
+	HAL_WBM_TQM_REL_REASON_CMD_REMOVE_TX,
+	HAL_WBM_TQM_REL_REASON_CMD_REMOVE_NOTX,
+	HAL_WBM_TQM_REL_REASON_CMD_REMOVE_AGED_FRAMES,
+	HAL_WBM_TQM_REL_REASON_CMD_REMOVE_RESEAON1,
+	HAL_WBM_TQM_REL_REASON_CMD_REMOVE_RESEAON2,
+	HAL_WBM_TQM_REL_REASON_CMD_REMOVE_RESEAON3,
+	HAL_WBM_TQM_REL_REASON_CMD_DISABLE_QUEUE,
+	HAL_WBM_TQM_REL_REASON_CMD_TILL_NONMATCHING,
+	HAL_WBM_TQM_REL_REASON_DROP_THRESHOLD,
+	HAL_WBM_TQM_REL_REASON_DROP_LINK_DESC_UNAVAIL,
+	HAL_WBM_TQM_REL_REASON_DROP_OR_INVALID_MSDU,
+	HAL_WBM_TQM_REL_REASON_MULTICAST_DROP,
+	HAL_WBM_TQM_REL_REASON_VDEV_MISMATCH_DROP,
+};
+
+struct hal_wbm_buffer_ring {
+	struct ath12k_buffer_addr buf_addr_info;
+};
+
+enum hal_mon_end_reason {
+	HAL_MON_STATUS_BUFFER_FULL,
+	HAL_MON_FLUSH_DETECTED,
+	HAL_MON_END_OF_PPDU,
+	HAL_MON_PPDU_TRUNCATED,
+};
+
+#define HAL_SW_MONITOR_RING_INFO0_RXDMA_PUSH_REASON	GENMASK(1, 0)
+#define HAL_SW_MONITOR_RING_INFO0_RXDMA_ERROR_CODE	GENMASK(6, 2)
+#define HAL_SW_MONITOR_RING_INFO0_MPDU_FRAGMENT_NUMBER	GENMASK(10, 7)
+#define HAL_SW_MONITOR_RING_INFO0_FRAMELESS_BAR		BIT(11)
+#define HAL_SW_MONITOR_RING_INFO0_STATUS_BUF_COUNT	GENMASK(15, 12)
+#define HAL_SW_MONITOR_RING_INFO0_END_OF_PPDU		BIT(16)
+
+#define HAL_SW_MONITOR_RING_INFO1_PHY_PPDU_ID	GENMASK(15, 0)
+#define HAL_SW_MONITOR_RING_INFO1_RING_ID	GENMASK(27, 20)
+#define HAL_SW_MONITOR_RING_INFO1_LOOPING_COUNT	GENMASK(31, 28)
+
+struct hal_sw_monitor_ring {
+	struct ath12k_buffer_addr buf_addr_info;
+	struct rx_mpdu_desc rx_mpdu_info;
+	struct ath12k_buffer_addr status_buff_addr_info;
+	__le32 info0; /* %HAL_SW_MONITOR_RING_INFO0 */
+	__le32 info1; /* %HAL_SW_MONITOR_RING_INFO1 */
+} __packed;
+
+/* hal_sw_monitor_ring
+ *
+ *		Producer: RXDMA
+ *		Consumer: REO/SW/FW
+ * buf_addr_info
+ *              Details of the physical address of a buffer or MSDU
+ *              link descriptor.
+ *
+ * rx_mpdu_info
+ *              Details related to the MPDU being pushed to SW, valid
+ *              only if end_of_ppdu is set to 0.
+ *
+ * status_buff_addr_info
+ *		Details of the physical address of the first status
+ *		buffer used for the PPDU (either the PPDU that included the
+ *		MPDU being pushed to SW if end_of_ppdu = 0, or the PPDU
+ *		whose end is indicated through end_of_ppdu = 1)
+ *
+ * rxdma_push_reason
+ *		Indicates why RXDMA pushed the frame to this ring
+ *
+ *		<enum 0 rxdma_error_detected> RXDMA detected an error an
+ *		pushed this frame to this queue
+ *
+ *		<enum 1 rxdma_routing_instruction> RXDMA pushed the
+ *		frame to this queue per received routing instructions. No
+ *		error within RXDMA was detected
+ *
+ *		<enum 2 rxdma_rx_flush> RXDMA received an RX_FLUSH. As a
+ *		result the MSDU link descriptor might not have the
+ *		last_msdu_in_mpdu_flag set, but instead WBM might just see a
+ *		NULL pointer in the MSDU link descriptor. This is to be
+ *		considered a normal condition for this scenario.
+ *
+ * rxdma_error_code
+ *		Field only valid when rxdma_push_reason is set to
+ *		'rxdma_error_detected.'
+ *
+ *		<enum 0 rxdma_overflow_err>MPDU frame is not complete
+ *		due to a FIFO overflow error in RXPCU.
+ *
+ *		<enum 1 rxdma_mpdu_length_err>MPDU frame is not complete
+ *		due to receiving incomplete MPDU from the PHY
+ *
+ *		<enum 3 rxdma_decrypt_err>CRYPTO reported a decryption
+ *		error or CRYPTO received an encrypted frame, but did not get
+ *		a valid corresponding key id in the peer entry.
+ *
+ *		<enum 4 rxdma_tkip_mic_err>CRYPTO reported a TKIP MIC
+ *		error
+ *
+ *		<enum 5 rxdma_unecrypted_err>CRYPTO reported an
+ *		unencrypted frame error when encrypted was expected
+ *
+ *		<enum 6 rxdma_msdu_len_err>RX OLE reported an MSDU
+ *		length error
+ *
+ *		<enum 7 rxdma_msdu_limit_err>RX OLE reported that max
+ *		number of MSDUs allowed in an MPDU got exceeded
+ *
+ *		<enum 8 rxdma_wifi_parse_err>RX OLE reported a parsing
+ *		error
+ *
+ *		<enum 9 rxdma_amsdu_parse_err>RX OLE reported an A-MSDU
+ *		parsing error
+ *
+ *		<enum 10 rxdma_sa_timeout_err>RX OLE reported a timeout
+ *		during SA search
+ *
+ *		<enum 11 rxdma_da_timeout_err>RX OLE reported a timeout
+ *		during DA search
+ *
+ *		<enum 12 rxdma_flow_timeout_err>RX OLE reported a
+ *		timeout during flow search
+ *
+ *		<enum 13 rxdma_flush_request>RXDMA received a flush
+ *		request
+ *
+ *		<enum 14 rxdma_amsdu_fragment_err>Rx PCU reported A-MSDU
+ *		present as well as a fragmented MPDU.
+ *
+ * mpdu_fragment_number
+ *		Field only valid when Reo_level_mpdu_frame_info.
+ *		Rx_mpdu_desc_info_details.Fragment_flag is set and
+ *		end_of_ppdu is set to 0.
+ *
+ *		The fragment number from the 802.11 header.
+ *
+ *		Note that the sequence number is embedded in the field:
+ *		Reo_level_mpdu_frame_info. Rx_mpdu_desc_info_details.
+ *		Mpdu_sequence_number
+ *
+ * frameless_bar
+ *		When set, this SW monitor ring struct contains BAR info
+ *		from a multi TID BAR frame. The original multi TID BAR frame
+ *		itself contained all the REO info for the first TID, but all
+ *		the subsequent TID info and their linkage to the REO
+ *		descriptors is passed down as 'frameless' BAR info.
+ *
+ *		The only fields valid in this descriptor when this bit
+ *		is within the
+ *
+ *		Reo_level_mpdu_frame_info:
+ *		   Within Rx_mpdu_desc_info_details:
+ *			Mpdu_Sequence_number
+ *			BAR_frame
+ *			Peer_meta_data
+ *			All other fields shall be set to 0.
+ *
+ * status_buf_count
+ *		A count of status buffers used so far for the PPDU
+ *		(either the PPDU that included the MPDU being pushed to SW
+ *		if end_of_ppdu = 0, or the PPDU whose end is indicated
+ *		through end_of_ppdu = 1)
+ *
+ * end_of_ppdu
+ *		Some hw RXDMA can be configured to generate a separate
+ *		'SW_MONITOR_RING' descriptor at the end of a PPDU (either
+ *		through an 'RX_PPDU_END' TLV or through an 'RX_FLUSH') to
+ *		demarcate PPDUs.
+ *
+ *		For such a descriptor, this bit is set to 1 and fields
+ *		Reo_level_mpdu_frame_info, mpdu_fragment_number and
+ *		Frameless_bar are all set to 0.
+ *
+ *		Otherwise this bit is set to 0.
+ *
+ * phy_ppdu_id
+ *		A PPDU counter value that PHY increments for every PPDU
+ *		received
+ *
+ *		The counter value wraps around. Some hw RXDMA can be
+ *		configured to copy this from the RX_PPDU_START TLV for every
+ *		output descriptor.
+ *
+ * ring_id
+ *		For debugging.
+ *		This field is filled in by the SRNG module.
+ *		It help to identify the ring that is being looked
+ *
+ * looping_count
+ *		For debugging.
+ *		This field is filled in by the SRNG module.
+ *
+ *		A count value that indicates the number of times the
+ *		producer of entries into this Ring has looped around the
+ *		ring.
+ *		At initialization time, this value is set to 0. On the
+ *		first loop, this value is set to 1. After the max value is
+ *		reached allowed by the number of bits for this field, the
+ *		count value continues with 0 again.
+ *
+ *		In case SW is the consumer of the ring entries, it can
+ *		use this field to figure out up to where the producer of
+ *		entries has created new entries. This eliminates the need to
+ *		check where the head pointer' of the ring is located once
+ *		the SW starts processing an interrupt indicating that new
+ *		entries have been put into this ring...
+ */
+
+enum hal_desc_owner {
+	HAL_DESC_OWNER_WBM,
+	HAL_DESC_OWNER_SW,
+	HAL_DESC_OWNER_TQM,
+	HAL_DESC_OWNER_RXDMA,
+	HAL_DESC_OWNER_REO,
+	HAL_DESC_OWNER_SWITCH,
+};
+
+enum hal_desc_buf_type {
+	HAL_DESC_BUF_TYPE_TX_MSDU_LINK,
+	HAL_DESC_BUF_TYPE_TX_MPDU_LINK,
+	HAL_DESC_BUF_TYPE_TX_MPDU_QUEUE_HEAD,
+	HAL_DESC_BUF_TYPE_TX_MPDU_QUEUE_EXT,
+	HAL_DESC_BUF_TYPE_TX_FLOW,
+	HAL_DESC_BUF_TYPE_TX_BUFFER,
+	HAL_DESC_BUF_TYPE_RX_MSDU_LINK,
+	HAL_DESC_BUF_TYPE_RX_MPDU_LINK,
+	HAL_DESC_BUF_TYPE_RX_REO_QUEUE,
+	HAL_DESC_BUF_TYPE_RX_REO_QUEUE_EXT,
+	HAL_DESC_BUF_TYPE_RX_BUFFER,
+	HAL_DESC_BUF_TYPE_IDLE_LINK,
+};
+
+#define HAL_DESC_REO_OWNED		4
+#define HAL_DESC_REO_QUEUE_DESC		8
+#define HAL_DESC_REO_QUEUE_EXT_DESC	9
+
+#define HAL_DESC_HDR_INFO0_OWNER	GENMASK(3, 0)
+#define HAL_DESC_HDR_INFO0_BUF_TYPE	GENMASK(7, 4)
+#define HAL_DESC_HDR_INFO0_DBG_RESERVED	GENMASK(31, 8)
+
+struct hal_desc_header {
+	__le32 info0;
+} __packed;
+
+struct hal_rx_mpdu_link_ptr {
+	struct ath12k_buffer_addr addr_info;
+} __packed;
+
+struct hal_rx_msdu_details {
+	struct ath12k_buffer_addr buf_addr_info;
+	struct rx_msdu_desc rx_msdu_info;
+	struct rx_msdu_ext_desc rx_msdu_ext_info;
+} __packed;
+
+#define HAL_RX_MSDU_LNK_INFO0_RX_QUEUE_NUMBER		GENMASK(15, 0)
+#define HAL_RX_MSDU_LNK_INFO0_FIRST_MSDU_LNK		BIT(16)
+
+struct hal_rx_msdu_link {
+	struct hal_desc_header desc_hdr;
+	struct ath12k_buffer_addr buf_addr_info;
+	__le32 info0;
+	__le32 pn[4];
+	struct hal_rx_msdu_details msdu_link[6];
+} __packed;
+
+struct hal_rx_reo_queue_ext {
+	struct hal_desc_header desc_hdr;
+	__le32 rsvd;
+	struct hal_rx_mpdu_link_ptr mpdu_link[15];
+} __packed;
+
+/* hal_rx_reo_queue_ext
+ *	Consumer: REO
+ *	Producer: REO
+ *
+ * descriptor_header
+ *	Details about which module owns this struct.
+ *
+ * mpdu_link
+ *	Pointer to the next MPDU_link descriptor in the MPDU queue.
+ */
+
+enum hal_rx_reo_queue_pn_size {
+	HAL_RX_REO_QUEUE_PN_SIZE_24,
+	HAL_RX_REO_QUEUE_PN_SIZE_48,
+	HAL_RX_REO_QUEUE_PN_SIZE_128,
+};
+
+#define HAL_RX_REO_QUEUE_RX_QUEUE_NUMBER		GENMASK(15, 0)
+
+#define HAL_RX_REO_QUEUE_INFO0_VLD			BIT(0)
+#define HAL_RX_REO_QUEUE_INFO0_ASSOC_LNK_DESC_COUNTER	GENMASK(2, 1)
+#define HAL_RX_REO_QUEUE_INFO0_DIS_DUP_DETECTION	BIT(3)
+#define HAL_RX_REO_QUEUE_INFO0_SOFT_REORDER_EN		BIT(4)
+#define HAL_RX_REO_QUEUE_INFO0_AC			GENMASK(6, 5)
+#define HAL_RX_REO_QUEUE_INFO0_BAR			BIT(7)
+#define HAL_RX_REO_QUEUE_INFO0_RETRY			BIT(8)
+#define HAL_RX_REO_QUEUE_INFO0_CHECK_2K_MODE		BIT(9)
+#define HAL_RX_REO_QUEUE_INFO0_OOR_MODE			BIT(10)
+#define HAL_RX_REO_QUEUE_INFO0_BA_WINDOW_SIZE		GENMASK(20, 11)
+#define HAL_RX_REO_QUEUE_INFO0_PN_CHECK			BIT(21)
+#define HAL_RX_REO_QUEUE_INFO0_EVEN_PN			BIT(22)
+#define HAL_RX_REO_QUEUE_INFO0_UNEVEN_PN		BIT(23)
+#define HAL_RX_REO_QUEUE_INFO0_PN_HANDLE_ENABLE		BIT(24)
+#define HAL_RX_REO_QUEUE_INFO0_PN_SIZE			GENMASK(26, 25)
+#define HAL_RX_REO_QUEUE_INFO0_IGNORE_AMPDU_FLG		BIT(27)
+
+#define HAL_RX_REO_QUEUE_INFO1_SVLD			BIT(0)
+#define HAL_RX_REO_QUEUE_INFO1_SSN			GENMASK(12, 1)
+#define HAL_RX_REO_QUEUE_INFO1_CURRENT_IDX		GENMASK(22, 13)
+#define HAL_RX_REO_QUEUE_INFO1_SEQ_2K_ERR		BIT(23)
+#define HAL_RX_REO_QUEUE_INFO1_PN_ERR			BIT(24)
+#define HAL_RX_REO_QUEUE_INFO1_PN_VALID			BIT(31)
+
+#define HAL_RX_REO_QUEUE_INFO2_MPDU_COUNT		GENMASK(6, 0)
+#define HAL_RX_REO_QUEUE_INFO2_MSDU_COUNT		(31, 7)
+
+#define HAL_RX_REO_QUEUE_INFO3_TIMEOUT_COUNT		GENMASK(9, 4)
+#define HAL_RX_REO_QUEUE_INFO3_FWD_DUE_TO_BAR_CNT	GENMASK(15, 10)
+#define HAL_RX_REO_QUEUE_INFO3_DUPLICATE_COUNT		GENMASK(31, 16)
+
+#define HAL_RX_REO_QUEUE_INFO4_FRAME_IN_ORD_COUNT	GENMASK(23, 0)
+#define HAL_RX_REO_QUEUE_INFO4_BAR_RECVD_COUNT		GENMASK(31, 24)
+
+#define HAL_RX_REO_QUEUE_INFO5_LATE_RX_MPDU_COUNT	GENMASK(11, 0)
+#define HAL_RX_REO_QUEUE_INFO5_WINDOW_JUMP_2K		GENMASK(15, 12)
+#define HAL_RX_REO_QUEUE_INFO5_HOLE_COUNT		GENMASK(31, 16)
+
+struct hal_rx_reo_queue {
+	struct hal_desc_header desc_hdr;
+	__le32 rx_queue_num;
+	__le32 info0;
+	__le32 info1;
+	__le32 pn[4];
+	__le32 last_rx_enqueue_timestamp;
+	__le32 last_rx_dequeue_timestamp;
+	__le32 next_aging_queue[2];
+	__le32 prev_aging_queue[2];
+	__le32 rx_bitmap[9];
+	__le32 info2;
+	__le32 info3;
+	__le32 info4;
+	__le32 processed_mpdus;
+	__le32 processed_msdus;
+	__le32 processed_total_bytes;
+	__le32 info5;
+	__le32 rsvd[2];
+	struct hal_rx_reo_queue_ext ext_desc[];
+} __packed;
+
+/* hal_rx_reo_queue
+ *
+ * descriptor_header
+ *	Details about which module owns this struct. Note that sub field
+ *	Buffer_type shall be set to receive_reo_queue_descriptor.
+ *
+ * receive_queue_number
+ *	Indicates the MPDU queue ID to which this MPDU link descriptor belongs.
+ *
+ * vld
+ *	Valid bit indicating a session is established and the queue descriptor
+ *	is valid.
+ * associated_link_descriptor_counter
+ *	Indicates which of the 3 link descriptor counters shall be incremented
+ *	or decremented when link descriptors are added or removed from this
+ *	flow queue.
+ * disable_duplicate_detection
+ *	When set, do not perform any duplicate detection.
+ * soft_reorder_enable
+ *	When set, REO has been instructed to not perform the actual re-ordering
+ *	of frames for this queue, but just to insert the reorder opcodes.
+ * ac
+ *	Indicates the access category of the queue descriptor.
+ * bar
+ *	Indicates if BAR has been received.
+ * retry
+ *	Retry bit is checked if this bit is set.
+ * chk_2k_mode
+ *	Indicates what type of operation is expected from Reo when the received
+ *	frame SN falls within the 2K window.
+ * oor_mode
+ *	Indicates what type of operation is expected when the received frame
+ *	falls within the OOR window.
+ * ba_window_size
+ *	Indicates the negotiated (window size + 1). Max of 256 bits.
+ *
+ *	A value 255 means 256 bitmap, 63 means 64 bitmap, 0 (means non-BA
+ *	session, with window size of 0). The 3 values here are the main values
+ *	validated, but other values should work as well.
+ *
+ *	A BA window size of 0 (=> one frame entry bitmat), means that there is
+ *	no additional rx_reo_queue_ext desc. following rx_reo_queue in memory.
+ *	A BA window size of 1 - 105, means that there is 1 rx_reo_queue_ext.
+ *	A BA window size of 106 - 210, means that there are 2 rx_reo_queue_ext.
+ *	A BA window size of 211 - 256, means that there are 3 rx_reo_queue_ext.
+ * pn_check_needed, pn_shall_be_even, pn_shall_be_uneven, pn_handling_enable,
+ * pn_size
+ *	REO shall perform the PN increment check, even number check, uneven
+ *	number check, PN error check and size of the PN field check.
+ * ignore_ampdu_flag
+ *	REO shall ignore the ampdu_flag on entrance descriptor for this queue.
+ *
+ * svld
+ *	Sequence number in next field is valid one.
+ * ssn
+ *	 Starting Sequence number of the session.
+ * current_index
+ *	Points to last forwarded packet
+ * seq_2k_error_detected_flag
+ *	REO has detected a 2k error jump in the sequence number and from that
+ *	moment forward, all new frames are forwarded directly to FW, without
+ *	duplicate detect, reordering, etc.
+ * pn_error_detected_flag
+ *	REO has detected a PN error.
+ */
+
+#define HAL_REO_UPD_RX_QUEUE_INFO0_QUEUE_ADDR_HI		GENMASK(7, 0)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_RX_QUEUE_NUM		BIT(8)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_VLD			BIT(9)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_ASSOC_LNK_DESC_CNT	BIT(10)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_DIS_DUP_DETECTION	BIT(11)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_SOFT_REORDER_EN		BIT(12)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_AC			BIT(13)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_BAR			BIT(14)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_RETRY			BIT(15)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_CHECK_2K_MODE		BIT(16)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_OOR_MODE			BIT(17)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_BA_WINDOW_SIZE		BIT(18)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_CHECK			BIT(19)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_EVEN_PN			BIT(20)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_UNEVEN_PN		BIT(21)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_HANDLE_ENABLE		BIT(22)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_SIZE			BIT(23)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_IGNORE_AMPDU_FLG		BIT(24)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_SVLD			BIT(25)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_SSN			BIT(26)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_SEQ_2K_ERR		BIT(27)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_ERR			BIT(28)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_VALID			BIT(29)
+#define HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN			BIT(30)
+
+#define HAL_REO_UPD_RX_QUEUE_INFO1_RX_QUEUE_NUMBER		GENMASK(15, 0)
+#define HAL_REO_UPD_RX_QUEUE_INFO1_VLD				BIT(16)
+#define HAL_REO_UPD_RX_QUEUE_INFO1_ASSOC_LNK_DESC_COUNTER	GENMASK(18, 17)
+#define HAL_REO_UPD_RX_QUEUE_INFO1_DIS_DUP_DETECTION		BIT(19)
+#define HAL_REO_UPD_RX_QUEUE_INFO1_SOFT_REORDER_EN		BIT(20)
+#define HAL_REO_UPD_RX_QUEUE_INFO1_AC				GENMASK(22, 21)
+#define HAL_REO_UPD_RX_QUEUE_INFO1_BAR				BIT(23)
+#define HAL_REO_UPD_RX_QUEUE_INFO1_RETRY			BIT(24)
+#define HAL_REO_UPD_RX_QUEUE_INFO1_CHECK_2K_MODE		BIT(25)
+#define HAL_REO_UPD_RX_QUEUE_INFO1_OOR_MODE			BIT(26)
+#define HAL_REO_UPD_RX_QUEUE_INFO1_PN_CHECK			BIT(27)
+#define HAL_REO_UPD_RX_QUEUE_INFO1_EVEN_PN			BIT(28)
+#define HAL_REO_UPD_RX_QUEUE_INFO1_UNEVEN_PN			BIT(29)
+#define HAL_REO_UPD_RX_QUEUE_INFO1_PN_HANDLE_ENABLE		BIT(30)
+#define HAL_REO_UPD_RX_QUEUE_INFO1_IGNORE_AMPDU_FLG		BIT(31)
+
+#define HAL_REO_UPD_RX_QUEUE_INFO2_BA_WINDOW_SIZE		GENMASK(9, 0)
+#define HAL_REO_UPD_RX_QUEUE_INFO2_PN_SIZE			GENMASK(11, 10)
+#define HAL_REO_UPD_RX_QUEUE_INFO2_SVLD				BIT(12)
+#define HAL_REO_UPD_RX_QUEUE_INFO2_SSN				GENMASK(24, 13)
+#define HAL_REO_UPD_RX_QUEUE_INFO2_SEQ_2K_ERR			BIT(25)
+#define HAL_REO_UPD_RX_QUEUE_INFO2_PN_ERR			BIT(26)
+#define HAL_REO_UPD_RX_QUEUE_INFO2_PN_VALID			BIT(27)
+
+struct hal_reo_update_rx_queue {
+	struct hal_reo_cmd_hdr cmd;
+	__le32 queue_addr_lo;
+	__le32 info0;
+	__le32 info1;
+	__le32 info2;
+	__le32 pn[4];
+} __packed;
+
+struct hal_rx_reo_queue_1k {
+	struct hal_desc_header desc_hdr;
+	__le32 rx_bitmap_1023_288[23];
+	__le32 reserved[8];
+} __packed;
+
+#define HAL_REO_UNBLOCK_CACHE_INFO0_UNBLK_CACHE		BIT(0)
+#define HAL_REO_UNBLOCK_CACHE_INFO0_RESOURCE_IDX	GENMASK(2, 1)
+
+struct hal_reo_unblock_cache {
+	struct hal_reo_cmd_hdr cmd;
+	__le32 info0;
+	__le32 rsvd[7];
+} __packed;
+
+enum hal_reo_exec_status {
+	HAL_REO_EXEC_STATUS_SUCCESS,
+	HAL_REO_EXEC_STATUS_BLOCKED,
+	HAL_REO_EXEC_STATUS_FAILED,
+	HAL_REO_EXEC_STATUS_RESOURCE_BLOCKED,
+};
+
+#define HAL_REO_STATUS_HDR_INFO0_STATUS_NUM	GENMASK(15, 0)
+#define HAL_REO_STATUS_HDR_INFO0_EXEC_TIME	GENMASK(25, 16)
+#define HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS	GENMASK(27, 26)
+
+struct hal_reo_status_hdr {
+	__le32 info0;
+	__le32 timestamp;
+} __packed;
+
+/* hal_reo_status_hdr
+ *		Producer: REO
+ *		Consumer: SW
+ *
+ * status_num
+ *		The value in this field is equal to value of the reo command
+ *		number. This field helps to correlate the statuses with the REO
+ *		commands.
+ *
+ * execution_time (in us)
+ *		The amount of time REO took to execute the command. Note that
+ *		this time does not include the duration of the command waiting
+ *		in the command ring, before the execution started.
+ *
+ * execution_status
+ *		Execution status of the command. Values are defined in
+ *		enum %HAL_REO_EXEC_STATUS_.
+ */
+#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO0_SSN		GENMASK(11, 0)
+#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO0_CUR_IDX		GENMASK(21, 12)
+
+#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO1_MPDU_COUNT		GENMASK(6, 0)
+#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO1_MSDU_COUNT		GENMASK(31, 7)
+
+#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO2_WINDOW_JMP2K	GENMASK(3, 0)
+#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO2_TIMEOUT_COUNT	GENMASK(9, 4)
+#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO2_FDTB_COUNT		GENMASK(15, 10)
+#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO2_DUPLICATE_COUNT	GENMASK(31, 16)
+
+#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO3_FIO_COUNT		GENMASK(23, 0)
+#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO3_BAR_RCVD_CNT	GENMASK(31, 24)
+
+#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO4_LATE_RX_MPDU	GENMASK(11, 0)
+#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO4_HOLE_COUNT		GENMASK(27, 12)
+
+#define HAL_REO_GET_QUEUE_STATS_STATUS_INFO5_LOOPING_CNT	GENMASK(31, 28)
+
+struct hal_reo_get_queue_stats_status {
+	struct hal_reo_status_hdr hdr;
+	__le32 info0;
+	__le32 pn[4];
+	__le32 last_rx_enqueue_timestamp;
+	__le32 last_rx_dequeue_timestamp;
+	__le32 rx_bitmap[9];
+	__le32 info1;
+	__le32 info2;
+	__le32 info3;
+	__le32 num_mpdu_frames;
+	__le32 num_msdu_frames;
+	__le32 total_bytes;
+	__le32 info4;
+	__le32 info5;
+} __packed;
+
+/* hal_reo_get_queue_stats_status
+ *		Producer: REO
+ *		Consumer: SW
+ *
+ * status_hdr
+ *		Details that can link this status with the original command. It
+ *		also contains info on how long REO took to execute this command.
+ *
+ * ssn
+ *		Starting Sequence number of the session, this changes whenever
+ *		window moves (can be filled by SW then maintained by REO).
+ *
+ * current_index
+ *		Points to last forwarded packet.
+ *
+ * pn
+ *		Bits of the PN number.
+ *
+ * last_rx_enqueue_timestamp
+ * last_rx_dequeue_timestamp
+ *		Timestamp of arrival of the last MPDU for this queue and
+ *		Timestamp of forwarding an MPDU accordingly.
+ *
+ * rx_bitmap
+ *		When a bit is set, the corresponding frame is currently held
+ *		in the re-order queue. The bitmap  is Fully managed by HW.
+ *
+ * current_mpdu_count
+ * current_msdu_count
+ *		The number of MPDUs and MSDUs in the queue.
+ *
+ * timeout_count
+ *		The number of times REO started forwarding frames even though
+ *		there is a hole in the bitmap. Forwarding reason is timeout.
+ *
+ * forward_due_to_bar_count
+ *		The number of times REO started forwarding frames even though
+ *		there is a hole in the bitmap. Fwd reason is reception of BAR.
+ *
+ * duplicate_count
+ *		The number of duplicate frames that have been detected.
+ *
+ * frames_in_order_count
+ *		The number of frames that have been received in order (without
+ *		a hole that prevented them from being forwarded immediately).
+ *
+ * bar_received_count
+ *		The number of times a BAR frame is received.
+ *
+ * mpdu_frames_processed_count
+ * msdu_frames_processed_count
+ *		The total number of MPDU/MSDU frames that have been processed.
+ *
+ * total_bytes
+ *		An approximation of the number of bytes received for this queue.
+ *
+ * late_receive_mpdu_count
+ *		The number of MPDUs received after the window had already moved
+ *		on. The 'late' sequence window is defined as
+ *		(Window SSN - 256) - (Window SSN - 1).
+ *
+ * window_jump_2k
+ *		The number of times the window moved more than 2K
+ *
+ * hole_count
+ *		The number of times a hole was created in the receive bitmap.
+ *
+ * looping_count
+ *		A count value that indicates the number of times the producer of
+ *		entries into this Ring has looped around the ring.
+ */
+
+#define HAL_REO_STATUS_LOOP_CNT			GENMASK(31, 28)
+
+#define HAL_REO_FLUSH_QUEUE_INFO0_ERR_DETECTED	BIT(0)
+#define HAL_REO_FLUSH_QUEUE_INFO0_RSVD		GENMASK(31, 1)
+#define HAL_REO_FLUSH_QUEUE_INFO1_RSVD		GENMASK(27, 0)
+
+struct hal_reo_flush_queue_status {
+	struct hal_reo_status_hdr hdr;
+	__le32 info0;
+	__le32 rsvd0[21];
+	__le32 info1;
+} __packed;
+
+/* hal_reo_flush_queue_status
+ *		Producer: REO
+ *		Consumer: SW
+ *
+ * status_hdr
+ *		Details that can link this status with the original command. It
+ *		also contains info on how long REO took to execute this command.
+ *
+ * error_detected
+ *		Status of blocking resource
+ *
+ *		0 - No error has been detected while executing this command
+ *		1 - Error detected. The resource to be used for blocking was
+ *		    already in use.
+ *
+ * looping_count
+ *		A count value that indicates the number of times the producer of
+ *		entries into this Ring has looped around the ring.
+ */
+
+#define HAL_REO_FLUSH_CACHE_STATUS_INFO0_IS_ERR			BIT(0)
+#define HAL_REO_FLUSH_CACHE_STATUS_INFO0_BLOCK_ERR_CODE		GENMASK(2, 1)
+#define HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_STATUS_HIT	BIT(8)
+#define HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_DESC_TYPE	GENMASK(11, 9)
+#define HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_CLIENT_ID	GENMASK(15, 12)
+#define HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_ERR		GENMASK(17, 16)
+#define HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_COUNT		GENMASK(25, 18)
+
+struct hal_reo_flush_cache_status {
+	struct hal_reo_status_hdr hdr;
+	__le32 info0;
+	__le32 rsvd0[21];
+	__le32 info1;
+} __packed;
+
+/* hal_reo_flush_cache_status
+ *		Producer: REO
+ *		Consumer: SW
+ *
+ * status_hdr
+ *		Details that can link this status with the original command. It
+ *		also contains info on how long REO took to execute this command.
+ *
+ * error_detected
+ *		Status for blocking resource handling
+ *
+ *		0 - No error has been detected while executing this command
+ *		1 - An error in the blocking resource management was detected
+ *
+ * block_error_details
+ *		only valid when error_detected is set
+ *
+ *		0 - No blocking related errors found
+ *		1 - Blocking resource is already in use
+ *		2 - Resource requested to be unblocked, was not blocked
+ *
+ * cache_controller_flush_status_hit
+ *		The status that the cache controller returned on executing the
+ *		flush command.
+ *
+ *		0 - miss; 1 - hit
+ *
+ * cache_controller_flush_status_desc_type
+ *		Flush descriptor type
+ *
+ * cache_controller_flush_status_client_id
+ *		Module who made the flush request
+ *
+ *		In REO, this is always 0
+ *
+ * cache_controller_flush_status_error
+ *		Error condition
+ *
+ *		0 - No error found
+ *		1 - HW interface is still busy
+ *		2 - Line currently locked. Used for one line flush command
+ *		3 - At least one line is still locked.
+ *		    Used for cache flush command.
+ *
+ * cache_controller_flush_count
+ *		The number of lines that were actually flushed out
+ *
+ * looping_count
+ *		A count value that indicates the number of times the producer of
+ *		entries into this Ring has looped around the ring.
+ */
+
+#define HAL_REO_UNBLOCK_CACHE_STATUS_INFO0_IS_ERR	BIT(0)
+#define HAL_REO_UNBLOCK_CACHE_STATUS_INFO0_TYPE		BIT(1)
+
+struct hal_reo_unblock_cache_status {
+	struct hal_reo_status_hdr hdr;
+	__le32 info0;
+	__le32 rsvd0[21];
+	__le32 info1;
+} __packed;
+
+/* hal_reo_unblock_cache_status
+ *		Producer: REO
+ *		Consumer: SW
+ *
+ * status_hdr
+ *		Details that can link this status with the original command. It
+ *		also contains info on how long REO took to execute this command.
+ *
+ * error_detected
+ *		0 - No error has been detected while executing this command
+ *		1 - The blocking resource was not in use, and therefore it could
+ *		    not be unblocked.
+ *
+ * unblock_type
+ *		Reference to the type of unblock command
+ *		0 - Unblock a blocking resource
+ *		1 - The entire cache usage is unblock
+ *
+ * looping_count
+ *		A count value that indicates the number of times the producer of
+ *		entries into this Ring has looped around the ring.
+ */
+
+#define HAL_REO_FLUSH_TIMEOUT_STATUS_INFO0_IS_ERR		BIT(0)
+#define HAL_REO_FLUSH_TIMEOUT_STATUS_INFO0_LIST_EMPTY		BIT(1)
+
+#define HAL_REO_FLUSH_TIMEOUT_STATUS_INFO1_REL_DESC_COUNT	GENMASK(15, 0)
+#define HAL_REO_FLUSH_TIMEOUT_STATUS_INFO1_FWD_BUF_COUNT	GENMASK(31, 16)
+
+struct hal_reo_flush_timeout_list_status {
+	struct hal_reo_status_hdr hdr;
+	__le32 info0;
+	__le32 info1;
+	__le32 rsvd0[20];
+	__le32 info2;
+} __packed;
+
+/* hal_reo_flush_timeout_list_status
+ *		Producer: REO
+ *		Consumer: SW
+ *
+ * status_hdr
+ *		Details that can link this status with the original command. It
+ *		also contains info on how long REO took to execute this command.
+ *
+ * error_detected
+ *		0 - No error has been detected while executing this command
+ *		1 - Command not properly executed and returned with error
+ *
+ * timeout_list_empty
+ *		When set, REO has depleted the timeout list and all entries are
+ *		gone.
+ *
+ * release_desc_count
+ *		Producer: SW; Consumer: REO
+ *		The number of link descriptor released
+ *
+ * forward_buf_count
+ *		Producer: SW; Consumer: REO
+ *		The number of buffers forwarded to the REO destination rings
+ *
+ * looping_count
+ *		A count value that indicates the number of times the producer of
+ *		entries into this Ring has looped around the ring.
+ */
+
+#define HAL_REO_DESC_THRESH_STATUS_INFO0_THRESH_INDEX		GENMASK(1, 0)
+#define HAL_REO_DESC_THRESH_STATUS_INFO1_LINK_DESC_COUNTER0	GENMASK(23, 0)
+#define HAL_REO_DESC_THRESH_STATUS_INFO2_LINK_DESC_COUNTER1	GENMASK(23, 0)
+#define HAL_REO_DESC_THRESH_STATUS_INFO3_LINK_DESC_COUNTER2	GENMASK(23, 0)
+#define HAL_REO_DESC_THRESH_STATUS_INFO4_LINK_DESC_COUNTER_SUM	GENMASK(25, 0)
+
+struct hal_reo_desc_thresh_reached_status {
+	struct hal_reo_status_hdr hdr;
+	__le32 info0;
+	__le32 info1;
+	__le32 info2;
+	__le32 info3;
+	__le32 info4;
+	__le32 rsvd0[17];
+	__le32 info5;
+} __packed;
+
+/* hal_reo_desc_thresh_reached_status
+ *		Producer: REO
+ *		Consumer: SW
+ *
+ * status_hdr
+ *		Details that can link this status with the original command. It
+ *		also contains info on how long REO took to execute this command.
+ *
+ * threshold_index
+ *		The index of the threshold register whose value got reached
+ *
+ * link_descriptor_counter0
+ * link_descriptor_counter1
+ * link_descriptor_counter2
+ * link_descriptor_counter_sum
+ *		Value of the respective counters at generation of this message
+ *
+ * looping_count
+ *		A count value that indicates the number of times the producer of
+ *		entries into this Ring has looped around the ring.
+ */
+
+#define HAL_TCL_ENTRANCE_FROM_PPE_RING_INFO0_DATA_LENGTH	GENMASK(13, 0)
+#define HAL_TCL_ENTRANCE_FROM_PPE_RING_INFO0_L4_CSUM_STATUS	BIT(14)
+#define HAL_TCL_ENTRANCE_FROM_PPE_RING_INFO0_L3_CSUM_STATUS	BIT(15)
+#define HAL_TCL_ENTRANCE_FROM_PPE_RING_INFO0_PID		GENMASK(27, 24)
+#define HAL_TCL_ENTRANCE_FROM_PPE_RING_INFO0_QDISC		BIT(28)
+#define HAL_TCL_ENTRANCE_FROM_PPE_RING_INFO0_MULTICAST	BIT(29)
+#define HAL_TCL_ENTRANCE_FROM_PPE_RING_INFO0_MORE		BIT(30)
+#define HAL_TCL_ENTRANCE_FROM_PPE_RING_INFO0_VALID_TOGGLE	BIT(31)
+
+struct hal_tcl_entrance_from_ppe_ring {
+	__le32 buffer_addr;
+	__le32 info0;
+} __packed;
+
+#define HAL_MON_DEST_COOKIE_BUF_ID      GENMASK(17, 0)
+
+#define HAL_MON_DEST_INFO0_END_OFFSET		GENMASK(11, 0)
+#define HAL_MON_DEST_INFO0_END_REASON		GENMASK(17, 16)
+#define HAL_MON_DEST_INFO0_INITIATOR		BIT(18)
+#define HAL_MON_DEST_INFO0_EMPTY_DESC		BIT(19)
+#define HAL_MON_DEST_INFO0_RING_ID		GENMASK(27, 20)
+#define HAL_MON_DEST_INFO0_LOOPING_COUNT	GENMASK(31, 28)
+
+struct hal_mon_dest_desc {
+	__le32 cookie;
+	__le32 reserved;
+	__le32 ppdu_id;
+	__le32 info0;
+};
+
+/* hal_mon_dest_ring
+ *	Producer : TxMon/RxMon
+ *	Consumer : SW
+ * cookie
+ *	bit 0 -17 buf_id to track the skb's vaddr.
+ * ppdu_id
+ *	Phy ppdu_id
+ * end_offset
+ *	The offset into status buffer where DMA ended, ie., offset to the last
+ *	TLV + last TLV size.
+ * flush_detected
+ *	Indicates whether 'tx_flush' or 'rx_flush' occurred.
+ * end_of_ppdu
+ *	Indicates end of ppdu.
+ * pmac_id
+ *	Indicates PMAC that received from frame.
+ * empty_descriptor
+ *	This descriptor is written on flush or end of ppdu or end of status
+ *	buffer.
+ * ring_id
+ *	updated by SRNG.
+ * looping_count
+ *	updated by SRNG.
+ */
+
+#define HAL_TX_MSDU_METADATA_INFO0_ENCRYPT_FLAG		BIT(8)
+#define HAL_TX_MSDU_METADATA_INFO0_ENCRYPT_TYPE		GENMASK(16, 15)
+#define HAL_TX_MSDU_METADATA_INFO0_HOST_TX_DESC_POOL	BIT(31)
+
+struct hal_tx_msdu_metadata {
+	__le32 info0;
+	__le32 rsvd0[6];
+} __packed;
+
+/* hal_tx_msdu_metadata
+ * valid_encrypt_type
+ *		if set, encrypt type is valid
+ * encrypt_type
+ *		0 = NO_ENCRYPT,
+ *		1 = ENCRYPT,
+ *		2 ~ 3 - Reserved
+ * host_tx_desc_pool
+ *		If set, Firmware allocates tx_descriptors
+ *		in WAL_BUFFERID_TX_HOST_DATA_EXP,instead
+ *		of WAL_BUFFERID_TX_TCL_DATA_EXP.
+ *		Use cases:
+ *		Any time firmware uses TQM-BYPASS for Data
+ *		TID, firmware expect host to set this bit.
+ */
+
+#endif /* ATH12K_HAL_DESC_H */
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.c b/drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.c
new file mode 100644
index 000000000000..c129e937132b
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.c
@@ -0,0 +1,1032 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+#include "hal_desc.h"
+#include "hal_qcn9274.h"
+#include "hw.h"
+#include "hal.h"
+#include "hal_tx.h"
+
+static const struct hal_srng_config hw_srng_config_template[] = {
+	/* TODO: max_rings can populated by querying HW capabilities */
+	[HAL_REO_DST] = {
+		.start_ring_id = HAL_SRNG_RING_ID_REO2SW1,
+		.max_rings = 8,
+		.entry_size = sizeof(struct hal_reo_dest_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_DST,
+		.max_size = HAL_REO_REO2SW1_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_REO_EXCEPTION] = {
+		/* Designating REO2SW0 ring as exception ring.
+		 * Any of theREO2SW rings can be used as exception ring.
+		 */
+		.start_ring_id = HAL_SRNG_RING_ID_REO2SW0,
+		.max_rings = 1,
+		.entry_size = sizeof(struct hal_reo_dest_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_DST,
+		.max_size = HAL_REO_REO2SW0_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_REO_REINJECT] = {
+		.start_ring_id = HAL_SRNG_RING_ID_SW2REO,
+		.max_rings = 4,
+		.entry_size = sizeof(struct hal_reo_entrance_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_REO_SW2REO_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_REO_CMD] = {
+		.start_ring_id = HAL_SRNG_RING_ID_REO_CMD,
+		.max_rings = 1,
+		.entry_size = (sizeof(struct hal_tlv_64_hdr) +
+			sizeof(struct hal_reo_get_queue_stats)) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_REO_CMD_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_REO_STATUS] = {
+		.start_ring_id = HAL_SRNG_RING_ID_REO_STATUS,
+		.max_rings = 1,
+		.entry_size = (sizeof(struct hal_tlv_64_hdr) +
+			sizeof(struct hal_reo_get_queue_stats_status)) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_DST,
+		.max_size = HAL_REO_STATUS_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_TCL_DATA] = {
+		.start_ring_id = HAL_SRNG_RING_ID_SW2TCL1,
+		.max_rings = 6,
+		.entry_size = sizeof(struct hal_tcl_data_cmd) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_TCL_CMD] = {
+		.start_ring_id = HAL_SRNG_RING_ID_SW2TCL_CMD,
+		.max_rings = 1,
+		.entry_size = sizeof(struct hal_tcl_gse_cmd) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_SW2TCL1_CMD_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_TCL_STATUS] = {
+		.start_ring_id = HAL_SRNG_RING_ID_TCL_STATUS,
+		.max_rings = 1,
+		.entry_size = (sizeof(struct hal_tlv_hdr) +
+			     sizeof(struct hal_tcl_status_ring)) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_DST,
+		.max_size = HAL_TCL_STATUS_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_CE_SRC] = {
+		.start_ring_id = HAL_SRNG_RING_ID_CE0_SRC,
+		.max_rings = 16,
+		.entry_size = sizeof(struct hal_ce_srng_src_desc) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_CE_SRC_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_CE_DST] = {
+		.start_ring_id = HAL_SRNG_RING_ID_CE0_DST,
+		.max_rings = 16,
+		.entry_size = sizeof(struct hal_ce_srng_dest_desc) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_CE_DST_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_CE_DST_STATUS] = {
+		.start_ring_id = HAL_SRNG_RING_ID_CE0_DST_STATUS,
+		.max_rings = 16,
+		.entry_size = sizeof(struct hal_ce_srng_dst_status_desc) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_DST,
+		.max_size = HAL_CE_DST_STATUS_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_WBM_IDLE_LINK] = {
+		.start_ring_id = HAL_SRNG_RING_ID_WBM_IDLE_LINK,
+		.max_rings = 1,
+		.entry_size = sizeof(struct hal_wbm_link_desc) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_WBM_IDLE_LINK_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_SW2WBM_RELEASE] = {
+		.start_ring_id = HAL_SRNG_RING_ID_WBM_SW0_RELEASE,
+		.max_rings = 2,
+		.entry_size = sizeof(struct hal_wbm_release_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_SW2WBM_RELEASE_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_WBM2SW_RELEASE] = {
+		.start_ring_id = HAL_SRNG_RING_ID_WBM2SW0_RELEASE,
+		.max_rings = 8,
+		.entry_size = sizeof(struct hal_wbm_release_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_DST,
+		.max_size = HAL_WBM2SW_RELEASE_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_RXDMA_BUF] = {
+		.start_ring_id = HAL_SRNG_SW2RXDMA_BUF0,
+		.max_rings = 1,
+		.entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_DMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
+	},
+	[HAL_RXDMA_DST] = {
+		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_RXDMA2SW0,
+		.max_rings = 0,
+		.entry_size = 0,
+		.mac_type = ATH12K_HAL_SRNG_PMAC,
+		.ring_dir = HAL_SRNG_DIR_DST,
+		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
+	},
+	[HAL_RXDMA_MONITOR_BUF] = {
+		.start_ring_id = HAL_SRNG_SW2RXMON_BUF0,
+		.max_rings = 1,
+		.entry_size = sizeof(struct hal_mon_buf_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_PMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
+	},
+	[HAL_RXDMA_MONITOR_STATUS] = {
+		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA1_STATBUF,
+		.max_rings = 1,
+		.entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_PMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
+	},
+	[HAL_RXDMA_MONITOR_DESC] = { 0, },
+	[HAL_RXDMA_DIR_BUF] = {
+		.start_ring_id = HAL_SRNG_RING_ID_RXDMA_DIR_BUF,
+		.max_rings = 2,
+		.entry_size = 8 >> 2, /* TODO: Define the struct */
+		.mac_type = ATH12K_HAL_SRNG_PMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
+	},
+	[HAL_PPE2TCL] = {
+		.start_ring_id = HAL_SRNG_RING_ID_PPE2TCL1,
+		.max_rings = 1,
+		.entry_size = sizeof(struct hal_tcl_entrance_from_ppe_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_PMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_PPE_RELEASE] = {
+		.start_ring_id = HAL_SRNG_RING_ID_WBM_PPE_RELEASE,
+		.max_rings = 1,
+		.entry_size = sizeof(struct hal_wbm_release_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_PMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_WBM2PPE_RELEASE_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_TX_MONITOR_BUF] = {
+		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2TXMON_BUF0,
+		.max_rings = 1,
+		.entry_size = sizeof(struct hal_mon_buf_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_PMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
+	},
+	[HAL_RXDMA_MONITOR_DST] = {
+		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXMON_BUF0,
+		.max_rings = 1,
+		.entry_size = sizeof(struct hal_mon_dest_desc) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_PMAC,
+		.ring_dir = HAL_SRNG_DIR_DST,
+		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
+	},
+	[HAL_TX_MONITOR_DST] = {
+		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_TXMON2SW0_BUF0,
+		.max_rings = 1,
+		.entry_size = sizeof(struct hal_mon_dest_desc) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_PMAC,
+		.ring_dir = HAL_SRNG_DIR_DST,
+		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
+	}
+};
+
+const struct ath12k_hw_regs qcn9274_v1_regs = {
+	/* SW2TCL(x) R0 ring configuration address */
+	.tcl1_ring_id = 0x00000908,
+	.tcl1_ring_misc = 0x00000910,
+	.tcl1_ring_tp_addr_lsb = 0x0000091c,
+	.tcl1_ring_tp_addr_msb = 0x00000920,
+	.tcl1_ring_consumer_int_setup_ix0 = 0x00000930,
+	.tcl1_ring_consumer_int_setup_ix1 = 0x00000934,
+	.tcl1_ring_msi1_base_lsb = 0x00000948,
+	.tcl1_ring_msi1_base_msb = 0x0000094c,
+	.tcl1_ring_msi1_data = 0x00000950,
+	.tcl_ring_base_lsb = 0x00000b58,
+	.tcl1_ring_base_lsb = 0x00000900,
+	.tcl1_ring_base_msb = 0x00000904,
+	.tcl2_ring_base_lsb = 0x00000978,
+
+	/* TCL STATUS ring address */
+	.tcl_status_ring_base_lsb = 0x00000d38,
+
+	.wbm_idle_ring_base_lsb = 0x00000d0c,
+	.wbm_idle_ring_misc_addr = 0x00000d1c,
+	.wbm_r0_idle_list_cntl_addr = 0x00000210,
+	.wbm_r0_idle_list_size_addr = 0x00000214,
+	.wbm_scattered_ring_base_lsb = 0x00000220,
+	.wbm_scattered_ring_base_msb = 0x00000224,
+	.wbm_scattered_desc_head_info_ix0 = 0x00000230,
+	.wbm_scattered_desc_head_info_ix1 = 0x00000234,
+	.wbm_scattered_desc_tail_info_ix0 = 0x00000240,
+	.wbm_scattered_desc_tail_info_ix1 = 0x00000244,
+	.wbm_scattered_desc_ptr_hp_addr = 0x0000024c,
+
+	.wbm_sw_release_ring_base_lsb = 0x0000034c,
+	.wbm_sw1_release_ring_base_lsb = 0x000003c4,
+	.wbm0_release_ring_base_lsb = 0x00000dd8,
+	.wbm1_release_ring_base_lsb = 0x00000e50,
+
+	/* PCIe base address */
+	.pcie_qserdes_sysclk_en_sel = 0x01e0c0a8,
+	.pcie_pcs_osc_dtct_config_base = 0x01e0d45c,
+
+	/* PPE release ring address */
+	.ppe_rel_ring_base = 0x0000043c,
+
+	/* REO DEST ring address */
+	.reo2_ring_base = 0x0000055c,
+	.reo1_misc_ctrl_addr = 0x00000b7c,
+	.reo1_sw_cookie_cfg0 = 0x00000050,
+	.reo1_sw_cookie_cfg1 = 0x00000054,
+	.reo1_qdesc_lut_base0 = 0x00000058,
+	.reo1_qdesc_lut_base1 = 0x0000005c,
+	.reo1_ring_base_lsb = 0x000004e4,
+	.reo1_ring_base_msb = 0x000004e8,
+	.reo1_ring_id = 0x000004ec,
+	.reo1_ring_misc = 0x000004f4,
+	.reo1_ring_hp_addr_lsb = 0x000004f8,
+	.reo1_ring_hp_addr_msb = 0x000004fc,
+	.reo1_ring_producer_int_setup = 0x00000508,
+	.reo1_ring_msi1_base_lsb = 0x0000052C,
+	.reo1_ring_msi1_base_msb = 0x00000530,
+	.reo1_ring_msi1_data = 0x00000534,
+	.reo1_aging_thres_ix0 = 0x00000b08,
+	.reo1_aging_thres_ix1 = 0x00000b0c,
+	.reo1_aging_thres_ix2 = 0x00000b10,
+	.reo1_aging_thres_ix3 = 0x00000b14,
+
+	/* REO Exception ring address */
+	.reo2_sw0_ring_base = 0x000008a4,
+
+	/* REO Reinject ring address */
+	.sw2reo_ring_base = 0x00000304,
+	.sw2reo1_ring_base = 0x0000037c,
+
+	/* REO cmd ring address */
+	.reo_cmd_ring_base = 0x0000028c,
+
+	/* REO status ring address */
+	.reo_status_ring_base = 0x00000a84,
+
+	/* CE base address */
+	.umac_ce0_src_reg_base = 0x01b80000,
+	.umac_ce0_dest_reg_base = 0x01b81000,
+	.umac_ce1_src_reg_base = 0x01b82000,
+	.umac_ce1_dest_reg_base = 0x01b83000,
+
+	.gcc_gcc_pcie_hot_rst = 0x1e38338,
+};
+
+const struct ath12k_hw_regs qcn9274_v2_regs = {
+	/* SW2TCL(x) R0 ring configuration address */
+	.tcl1_ring_id = 0x00000908,
+	.tcl1_ring_misc = 0x00000910,
+	.tcl1_ring_tp_addr_lsb = 0x0000091c,
+	.tcl1_ring_tp_addr_msb = 0x00000920,
+	.tcl1_ring_consumer_int_setup_ix0 = 0x00000930,
+	.tcl1_ring_consumer_int_setup_ix1 = 0x00000934,
+	.tcl1_ring_msi1_base_lsb = 0x00000948,
+	.tcl1_ring_msi1_base_msb = 0x0000094c,
+	.tcl1_ring_msi1_data = 0x00000950,
+	.tcl_ring_base_lsb = 0x00000b58,
+	.tcl1_ring_base_lsb = 0x00000900,
+	.tcl1_ring_base_msb = 0x00000904,
+	.tcl2_ring_base_lsb = 0x00000978,
+
+	/* TCL STATUS ring address */
+	.tcl_status_ring_base_lsb = 0x00000d38,
+
+	/* WBM idle link ring address */
+	.wbm_idle_ring_base_lsb = 0x00000d3c,
+	.wbm_idle_ring_misc_addr = 0x00000d4c,
+	.wbm_r0_idle_list_cntl_addr = 0x00000240,
+	.wbm_r0_idle_list_size_addr = 0x00000244,
+	.wbm_scattered_ring_base_lsb = 0x00000250,
+	.wbm_scattered_ring_base_msb = 0x00000254,
+	.wbm_scattered_desc_head_info_ix0 = 0x00000260,
+	.wbm_scattered_desc_head_info_ix1 = 0x00000264,
+	.wbm_scattered_desc_tail_info_ix0 = 0x00000270,
+	.wbm_scattered_desc_tail_info_ix1 = 0x00000274,
+	.wbm_scattered_desc_ptr_hp_addr = 0x0000027c,
+
+	/* SW2WBM release ring address */
+	.wbm_sw_release_ring_base_lsb = 0x0000037c,
+	.wbm_sw1_release_ring_base_lsb = 0x000003f4,
+
+	/* WBM2SW release ring address */
+	.wbm0_release_ring_base_lsb = 0x00000e08,
+	.wbm1_release_ring_base_lsb = 0x00000e80,
+
+	/* PCIe base address */
+	.pcie_qserdes_sysclk_en_sel = 0x01e0c0a8,
+	.pcie_pcs_osc_dtct_config_base = 0x01e0d45c,
+
+	/* PPE release ring address */
+	.ppe_rel_ring_base = 0x0000046c,
+
+	/* REO DEST ring address */
+	.reo2_ring_base = 0x00000578,
+	.reo1_misc_ctrl_addr = 0x00000b9c,
+	.reo1_sw_cookie_cfg0 = 0x0000006c,
+	.reo1_sw_cookie_cfg1 = 0x00000070,
+	.reo1_qdesc_lut_base0 = 0x00000074,
+	.reo1_qdesc_lut_base1 = 0x00000078,
+	.reo1_qdesc_addr = 0x0000007c,
+	.reo1_qdesc_max_peerid = 0x00000088,
+	.reo1_ring_base_lsb = 0x00000500,
+	.reo1_ring_base_msb = 0x00000504,
+	.reo1_ring_id = 0x00000508,
+	.reo1_ring_misc = 0x00000510,
+	.reo1_ring_hp_addr_lsb = 0x00000514,
+	.reo1_ring_hp_addr_msb = 0x00000518,
+	.reo1_ring_producer_int_setup = 0x00000524,
+	.reo1_ring_msi1_base_lsb = 0x00000548,
+	.reo1_ring_msi1_base_msb = 0x0000054C,
+	.reo1_ring_msi1_data = 0x00000550,
+	.reo1_aging_thres_ix0 = 0x00000B28,
+	.reo1_aging_thres_ix1 = 0x00000B2C,
+	.reo1_aging_thres_ix2 = 0x00000B30,
+	.reo1_aging_thres_ix3 = 0x00000B34,
+
+	/* REO Exception ring address */
+	.reo2_sw0_ring_base = 0x000008c0,
+
+	/* REO Reinject ring address */
+	.sw2reo_ring_base = 0x00000320,
+	.sw2reo1_ring_base = 0x00000398,
+
+	/* REO cmd ring address */
+	.reo_cmd_ring_base = 0x000002A8,
+
+	/* REO status ring address */
+	.reo_status_ring_base = 0x00000aa0,
+
+	/* CE base address */
+	.umac_ce0_src_reg_base = 0x01b80000,
+	.umac_ce0_dest_reg_base = 0x01b81000,
+	.umac_ce1_src_reg_base = 0x01b82000,
+	.umac_ce1_dest_reg_base = 0x01b83000,
+
+	.gcc_gcc_pcie_hot_rst = 0x1e38338,
+};
+
+const struct ath12k_hw_regs ipq5332_regs = {
+	/* SW2TCL(x) R0 ring configuration address */
+	.tcl1_ring_id = 0x00000918,
+	.tcl1_ring_misc = 0x00000920,
+	.tcl1_ring_tp_addr_lsb = 0x0000092c,
+	.tcl1_ring_tp_addr_msb = 0x00000930,
+	.tcl1_ring_consumer_int_setup_ix0 = 0x00000940,
+	.tcl1_ring_consumer_int_setup_ix1 = 0x00000944,
+	.tcl1_ring_msi1_base_lsb = 0x00000958,
+	.tcl1_ring_msi1_base_msb = 0x0000095c,
+	.tcl1_ring_base_lsb = 0x00000910,
+	.tcl1_ring_base_msb = 0x00000914,
+	.tcl1_ring_msi1_data = 0x00000960,
+	.tcl2_ring_base_lsb = 0x00000988,
+	.tcl_ring_base_lsb = 0x00000b68,
+
+	/* TCL STATUS ring address */
+	.tcl_status_ring_base_lsb = 0x00000d48,
+
+	/* REO DEST ring address */
+	.reo2_ring_base = 0x00000578,
+	.reo1_misc_ctrl_addr = 0x00000b9c,
+	.reo1_sw_cookie_cfg0 = 0x0000006c,
+	.reo1_sw_cookie_cfg1 = 0x00000070,
+	.reo1_qdesc_lut_base0 = 0x00000074,
+	.reo1_qdesc_lut_base1 = 0x00000078,
+	.reo1_ring_base_lsb = 0x00000500,
+	.reo1_ring_base_msb = 0x00000504,
+	.reo1_ring_id = 0x00000508,
+	.reo1_ring_misc = 0x00000510,
+	.reo1_ring_hp_addr_lsb = 0x00000514,
+	.reo1_ring_hp_addr_msb = 0x00000518,
+	.reo1_ring_producer_int_setup = 0x00000524,
+	.reo1_ring_msi1_base_lsb = 0x00000548,
+	.reo1_ring_msi1_base_msb = 0x0000054C,
+	.reo1_ring_msi1_data = 0x00000550,
+	.reo1_aging_thres_ix0 = 0x00000B28,
+	.reo1_aging_thres_ix1 = 0x00000B2C,
+	.reo1_aging_thres_ix2 = 0x00000B30,
+	.reo1_aging_thres_ix3 = 0x00000B34,
+
+	/* REO Exception ring address */
+	.reo2_sw0_ring_base = 0x000008c0,
+
+	/* REO Reinject ring address */
+	.sw2reo_ring_base = 0x00000320,
+	.sw2reo1_ring_base = 0x00000398,
+
+	/* REO cmd ring address */
+	.reo_cmd_ring_base = 0x000002A8,
+
+	/* REO status ring address */
+	.reo_status_ring_base = 0x00000aa0,
+
+	/* WBM idle link ring address */
+	.wbm_idle_ring_base_lsb = 0x00000d3c,
+	.wbm_idle_ring_misc_addr = 0x00000d4c,
+	.wbm_r0_idle_list_cntl_addr = 0x00000240,
+	.wbm_r0_idle_list_size_addr = 0x00000244,
+	.wbm_scattered_ring_base_lsb = 0x00000250,
+	.wbm_scattered_ring_base_msb = 0x00000254,
+	.wbm_scattered_desc_head_info_ix0 = 0x00000260,
+	.wbm_scattered_desc_head_info_ix1   = 0x00000264,
+	.wbm_scattered_desc_tail_info_ix0 = 0x00000270,
+	.wbm_scattered_desc_tail_info_ix1 = 0x00000274,
+	.wbm_scattered_desc_ptr_hp_addr = 0x0000027c,
+
+	/* SW2WBM release ring address */
+	.wbm_sw_release_ring_base_lsb = 0x0000037c,
+
+	/* WBM2SW release ring address */
+	.wbm0_release_ring_base_lsb = 0x00000e08,
+	.wbm1_release_ring_base_lsb = 0x00000e80,
+
+	/* PPE release ring address */
+	.ppe_rel_ring_base = 0x0000046c,
+
+	/* CE address */
+	.umac_ce0_src_reg_base = 0x00740000 -
+		HAL_IPQ5332_CE_WFSS_REG_BASE,
+	.umac_ce0_dest_reg_base = 0x00741000 -
+		HAL_IPQ5332_CE_WFSS_REG_BASE,
+	.umac_ce1_src_reg_base = 0x00742000 -
+		HAL_IPQ5332_CE_WFSS_REG_BASE,
+	.umac_ce1_dest_reg_base = 0x00743000 -
+		HAL_IPQ5332_CE_WFSS_REG_BASE,
+};
+
+static inline
+bool ath12k_hal_rx_desc_get_first_msdu_qcn9274(struct hal_rx_desc *desc)
+{
+	return !!le16_get_bits(desc->u.qcn9274_compact.msdu_end.info5,
+			       RX_MSDU_END_INFO5_FIRST_MSDU);
+}
+
+static inline
+bool ath12k_hal_rx_desc_get_last_msdu_qcn9274(struct hal_rx_desc *desc)
+{
+	return !!le16_get_bits(desc->u.qcn9274_compact.msdu_end.info5,
+			       RX_MSDU_END_INFO5_LAST_MSDU);
+}
+
+u8 ath12k_hal_rx_desc_get_l3_pad_bytes_qcn9274(struct hal_rx_desc *desc)
+{
+	return le16_get_bits(desc->u.qcn9274_compact.msdu_end.info5,
+			     RX_MSDU_END_INFO5_L3_HDR_PADDING);
+}
+
+static inline
+bool ath12k_hal_rx_desc_encrypt_valid_qcn9274(struct hal_rx_desc *desc)
+{
+	return !!le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info4,
+			       RX_MPDU_START_INFO4_ENCRYPT_INFO_VALID);
+}
+
+static inline
+u32 ath12k_hal_rx_desc_get_encrypt_type_qcn9274(struct hal_rx_desc *desc)
+{
+	if (!ath12k_hal_rx_desc_encrypt_valid_qcn9274(desc))
+		return HAL_ENCRYPT_TYPE_OPEN;
+
+	return le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info2,
+			     RX_MPDU_START_INFO2_ENC_TYPE);
+}
+
+static inline
+u8 ath12k_hal_rx_desc_get_decap_type_qcn9274(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info11,
+			     RX_MSDU_END_INFO11_DECAP_FORMAT);
+}
+
+static inline
+u8 ath12k_hal_rx_desc_get_mesh_ctl_qcn9274(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info11,
+			     RX_MSDU_END_INFO11_MESH_CTRL_PRESENT);
+}
+
+static inline
+bool ath12k_hal_rx_desc_get_mpdu_seq_ctl_vld_qcn9274(struct hal_rx_desc *desc)
+{
+	return !!le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info4,
+			       RX_MPDU_START_INFO4_MPDU_SEQ_CTRL_VALID);
+}
+
+static inline
+bool ath12k_hal_rx_desc_get_mpdu_fc_valid_qcn9274(struct hal_rx_desc *desc)
+{
+	return !!le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info4,
+			       RX_MPDU_START_INFO4_MPDU_FCTRL_VALID);
+}
+
+static inline
+u16 ath12k_hal_rx_desc_get_mpdu_start_seq_no_qcn9274(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info4,
+			     RX_MPDU_START_INFO4_MPDU_SEQ_NUM);
+}
+
+static inline
+u16 ath12k_hal_rx_desc_get_msdu_len_qcn9274(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info10,
+			     RX_MSDU_END_INFO10_MSDU_LENGTH);
+}
+
+static inline
+u8 ath12k_hal_rx_desc_get_msdu_sgi_qcn9274(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12,
+			     RX_MSDU_END_INFO12_SGI);
+}
+
+static inline
+u8 ath12k_hal_rx_desc_get_msdu_rate_mcs_qcn9274(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12,
+			     RX_MSDU_END_INFO12_RATE_MCS);
+}
+
+static inline
+u8 ath12k_hal_rx_desc_get_msdu_rx_bw_qcn9274(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12,
+			     RX_MSDU_END_INFO12_RECV_BW);
+}
+
+static inline
+u32 ath12k_hal_rx_desc_get_msdu_freq_qcn9274(struct hal_rx_desc *desc)
+{
+	return __le32_to_cpu(desc->u.qcn9274_compact.msdu_end.phy_meta_data);
+}
+
+static inline
+u8 ath12k_hal_rx_desc_get_msdu_pkt_type_qcn9274(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12,
+			     RX_MSDU_END_INFO12_PKT_TYPE);
+}
+
+static inline
+u8 ath12k_hal_rx_desc_get_msdu_nss_qcn9274(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.qcn9274_compact.msdu_end.info12,
+			     RX_MSDU_END_INFO12_MIMO_SS_BITMAP);
+}
+
+static inline
+u8 ath12k_hal_rx_desc_get_mpdu_tid_qcn9274(struct hal_rx_desc *desc)
+{
+	return le16_get_bits(desc->u.qcn9274_compact.msdu_end.info5,
+			     RX_MSDU_END_INFO5_TID);
+}
+
+static inline
+u16 ath12k_hal_rx_desc_get_mpdu_peer_id_qcn9274(struct hal_rx_desc *desc)
+{
+	return __le16_to_cpu(desc->u.qcn9274_compact.mpdu_start.sw_peer_id);
+}
+
+void ath12k_hal_rx_desc_copy_end_tlv_qcn9274(struct hal_rx_desc *fdesc,
+					     struct hal_rx_desc *ldesc)
+{
+	fdesc->u.qcn9274_compact.msdu_end = ldesc->u.qcn9274_compact.msdu_end;
+}
+
+u32 ath12k_hal_rx_desc_get_mpdu_ppdu_id_qcn9274(struct hal_rx_desc *desc)
+{
+	return __le16_to_cpu(desc->u.qcn9274_compact.mpdu_start.phy_ppdu_id);
+}
+
+void ath12k_hal_rx_desc_set_msdu_len_qcn9274(struct hal_rx_desc *desc, u16 len)
+{
+	u32 info = __le32_to_cpu(desc->u.qcn9274_compact.msdu_end.info10);
+
+	info = u32_replace_bits(info, len, RX_MSDU_END_INFO10_MSDU_LENGTH);
+	desc->u.qcn9274_compact.msdu_end.info10 = __cpu_to_le32(info);
+}
+
+u8 *ath12k_hal_rx_desc_get_msdu_payload_qcn9274(struct hal_rx_desc *desc)
+{
+	return &desc->u.qcn9274_compact.msdu_payload[0];
+}
+
+u32 ath12k_hal_rx_desc_get_mpdu_start_offset_qcn9274(void)
+{
+	return offsetof(struct hal_rx_desc_qcn9274_compact, mpdu_start);
+}
+
+u32 ath12k_hal_rx_desc_get_msdu_end_offset_qcn9274(void)
+{
+	return offsetof(struct hal_rx_desc_qcn9274_compact, msdu_end);
+}
+
+static inline
+bool ath12k_hal_rx_desc_mac_addr2_valid_qcn9274(struct hal_rx_desc *desc)
+{
+	return __le32_to_cpu(desc->u.qcn9274_compact.mpdu_start.info4) &
+			     RX_MPDU_START_INFO4_MAC_ADDR2_VALID;
+}
+
+static inline
+u8 *ath12k_hal_rx_desc_mpdu_start_addr2_qcn9274(struct hal_rx_desc *desc)
+{
+	return desc->u.qcn9274_compact.mpdu_start.addr2;
+}
+
+static inline
+bool ath12k_hal_rx_desc_is_da_mcbc_qcn9274(struct hal_rx_desc *desc)
+{
+	return __le16_to_cpu(desc->u.qcn9274_compact.msdu_end.info5) &
+	       RX_MSDU_END_INFO5_DA_IS_MCBC;
+}
+
+static inline
+bool ath12k_hal_rx_h_msdu_done_qcn9274(struct hal_rx_desc *desc)
+{
+	return !!le32_get_bits(desc->u.qcn9274_compact.msdu_end.info14,
+			       RX_MSDU_END_INFO14_MSDU_DONE);
+}
+
+static inline
+bool ath12k_hal_rx_h_l4_cksum_fail_qcn9274(struct hal_rx_desc *desc)
+{
+	return !!le32_get_bits(desc->u.qcn9274_compact.msdu_end.info13,
+			       RX_MSDU_END_INFO13_TCP_UDP_CKSUM_FAIL);
+}
+
+static inline
+bool ath12k_hal_rx_h_ip_cksum_fail_qcn9274(struct hal_rx_desc *desc)
+{
+	return !!le32_get_bits(desc->u.qcn9274_compact.msdu_end.info13,
+			       RX_MSDU_END_INFO13_IP_CKSUM_FAIL);
+}
+
+static inline
+bool ath12k_hal_rx_h_is_decrypted_qcn9274(struct hal_rx_desc *desc)
+{
+	return (le32_get_bits(desc->u.qcn9274_compact.msdu_end.info14,
+			      RX_MSDU_END_INFO14_DECRYPT_STATUS_CODE) ==
+			RX_DESC_DECRYPT_STATUS_CODE_OK);
+}
+
+u32 ath12k_hal_get_rx_desc_size_qcn9274(void)
+{
+	return sizeof(struct hal_rx_desc_qcn9274_compact);
+}
+
+u8 ath12k_hal_rx_desc_get_msdu_src_link_qcn9274(struct hal_rx_desc *desc)
+{
+	return le64_get_bits(desc->u.qcn9274_compact.msdu_end.msdu_end_tag,
+			     RX_MSDU_END_64_TLV_SRC_LINK_ID);
+}
+
+u16 ath12k_hal_rx_mpdu_start_wmask_get_qcn9274(void)
+{
+	return QCN9274_MPDU_START_WMASK;
+}
+
+u32 ath12k_hal_rx_msdu_end_wmask_get_qcn9274(void)
+{
+	return QCN9274_MSDU_END_WMASK;
+}
+
+static u32 ath12k_hal_rx_h_mpdu_err_qcn9274(struct hal_rx_desc *desc)
+{
+	u32 info = __le32_to_cpu(desc->u.qcn9274_compact.msdu_end.info13);
+	u32 errmap = 0;
+
+	if (info & RX_MSDU_END_INFO13_FCS_ERR)
+		errmap |= HAL_RX_MPDU_ERR_FCS;
+
+	if (info & RX_MSDU_END_INFO13_DECRYPT_ERR)
+		errmap |= HAL_RX_MPDU_ERR_DECRYPT;
+
+	if (info & RX_MSDU_END_INFO13_TKIP_MIC_ERR)
+		errmap |= HAL_RX_MPDU_ERR_TKIP_MIC;
+
+	if (info & RX_MSDU_END_INFO13_A_MSDU_ERROR)
+		errmap |= HAL_RX_MPDU_ERR_AMSDU_ERR;
+
+	if (info & RX_MSDU_END_INFO13_OVERFLOW_ERR)
+		errmap |= HAL_RX_MPDU_ERR_OVERFLOW;
+
+	if (info & RX_MSDU_END_INFO13_MSDU_LEN_ERR)
+		errmap |= HAL_RX_MPDU_ERR_MSDU_LEN;
+
+	if (info & RX_MSDU_END_INFO13_MPDU_LEN_ERR)
+		errmap |= HAL_RX_MPDU_ERR_MPDU_LEN;
+
+	return errmap;
+}
+
+void ath12k_hal_rx_desc_get_crypto_hdr_qcn9274(struct hal_rx_desc *desc,
+					       u8 *crypto_hdr,
+					       enum hal_encrypt_type enctype)
+{
+	unsigned int key_id;
+
+	switch (enctype) {
+	case HAL_ENCRYPT_TYPE_OPEN:
+		return;
+	case HAL_ENCRYPT_TYPE_TKIP_NO_MIC:
+	case HAL_ENCRYPT_TYPE_TKIP_MIC:
+		crypto_hdr[0] =
+		HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274_compact.mpdu_start.pn[0]);
+		crypto_hdr[1] = 0;
+		crypto_hdr[2] =
+		HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274_compact.mpdu_start.pn[0]);
+		break;
+	case HAL_ENCRYPT_TYPE_CCMP_128:
+	case HAL_ENCRYPT_TYPE_CCMP_256:
+	case HAL_ENCRYPT_TYPE_GCMP_128:
+	case HAL_ENCRYPT_TYPE_AES_GCMP_256:
+		crypto_hdr[0] =
+		HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274_compact.mpdu_start.pn[0]);
+		crypto_hdr[1] =
+		HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274_compact.mpdu_start.pn[0]);
+		crypto_hdr[2] = 0;
+		break;
+	case HAL_ENCRYPT_TYPE_WEP_40:
+	case HAL_ENCRYPT_TYPE_WEP_104:
+	case HAL_ENCRYPT_TYPE_WEP_128:
+	case HAL_ENCRYPT_TYPE_WAPI_GCM_SM4:
+	case HAL_ENCRYPT_TYPE_WAPI:
+		return;
+	}
+	key_id = le32_get_bits(desc->u.qcn9274_compact.mpdu_start.info5,
+			       RX_MPDU_START_INFO5_KEY_ID);
+	crypto_hdr[3] = 0x20 | (key_id << 6);
+	crypto_hdr[4] =
+		HAL_RX_MPDU_INFO_PN_GET_BYTE3(desc->u.qcn9274_compact.mpdu_start.pn[0]);
+	crypto_hdr[5] =
+		HAL_RX_MPDU_INFO_PN_GET_BYTE4(desc->u.qcn9274_compact.mpdu_start.pn[0]);
+	crypto_hdr[6] =
+		HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.qcn9274_compact.mpdu_start.pn[1]);
+	crypto_hdr[7] =
+		HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.qcn9274_compact.mpdu_start.pn[1]);
+}
+
+void ath12k_hal_rx_desc_get_dot11_hdr_qcn9274(struct hal_rx_desc *desc,
+					      struct ieee80211_hdr *hdr)
+{
+	hdr->frame_control = desc->u.qcn9274_compact.mpdu_start.frame_ctrl;
+	hdr->duration_id = desc->u.qcn9274_compact.mpdu_start.duration;
+	ether_addr_copy(hdr->addr1, desc->u.qcn9274_compact.mpdu_start.addr1);
+	ether_addr_copy(hdr->addr2, desc->u.qcn9274_compact.mpdu_start.addr2);
+	ether_addr_copy(hdr->addr3, desc->u.qcn9274_compact.mpdu_start.addr3);
+	if (__le32_to_cpu(desc->u.qcn9274_compact.mpdu_start.info4) &
+			RX_MPDU_START_INFO4_MAC_ADDR4_VALID) {
+		ether_addr_copy(hdr->addr4, desc->u.qcn9274_compact.mpdu_start.addr4);
+	}
+	hdr->seq_ctrl = desc->u.qcn9274_compact.mpdu_start.seq_ctrl;
+}
+
+void ath12k_hal_extract_rx_desc_data_qcn9274(struct hal_rx_desc_data *rx_desc_data,
+					     struct hal_rx_desc *rx_desc,
+					     struct hal_rx_desc *ldesc)
+{
+	rx_desc_data->is_first_msdu = ath12k_hal_rx_desc_get_first_msdu_qcn9274(ldesc);
+	rx_desc_data->is_last_msdu = ath12k_hal_rx_desc_get_last_msdu_qcn9274(ldesc);
+	rx_desc_data->l3_pad_bytes = ath12k_hal_rx_desc_get_l3_pad_bytes_qcn9274(ldesc);
+	rx_desc_data->enctype = ath12k_hal_rx_desc_get_encrypt_type_qcn9274(rx_desc);
+	rx_desc_data->decap_type = ath12k_hal_rx_desc_get_decap_type_qcn9274(rx_desc);
+	rx_desc_data->mesh_ctrl_present =
+		ath12k_hal_rx_desc_get_mesh_ctl_qcn9274(rx_desc);
+	rx_desc_data->seq_ctl_valid =
+		ath12k_hal_rx_desc_get_mpdu_seq_ctl_vld_qcn9274(rx_desc);
+	rx_desc_data->fc_valid = ath12k_hal_rx_desc_get_mpdu_fc_valid_qcn9274(rx_desc);
+	rx_desc_data->seq_no = ath12k_hal_rx_desc_get_mpdu_start_seq_no_qcn9274(rx_desc);
+	rx_desc_data->msdu_len = ath12k_hal_rx_desc_get_msdu_len_qcn9274(ldesc);
+	rx_desc_data->sgi = ath12k_hal_rx_desc_get_msdu_sgi_qcn9274(rx_desc);
+	rx_desc_data->rate_mcs = ath12k_hal_rx_desc_get_msdu_rate_mcs_qcn9274(rx_desc);
+	rx_desc_data->bw = ath12k_hal_rx_desc_get_msdu_rx_bw_qcn9274(rx_desc);
+	rx_desc_data->phy_meta_data = ath12k_hal_rx_desc_get_msdu_freq_qcn9274(rx_desc);
+	rx_desc_data->pkt_type = ath12k_hal_rx_desc_get_msdu_pkt_type_qcn9274(rx_desc);
+	rx_desc_data->nss = hweight8(ath12k_hal_rx_desc_get_msdu_nss_qcn9274(rx_desc));
+	rx_desc_data->tid = ath12k_hal_rx_desc_get_mpdu_tid_qcn9274(rx_desc);
+	rx_desc_data->peer_id = ath12k_hal_rx_desc_get_mpdu_peer_id_qcn9274(rx_desc);
+	rx_desc_data->addr2_present = ath12k_hal_rx_desc_mac_addr2_valid_qcn9274(rx_desc);
+	rx_desc_data->addr2 = ath12k_hal_rx_desc_mpdu_start_addr2_qcn9274(rx_desc);
+	rx_desc_data->is_mcbc = ath12k_hal_rx_desc_is_da_mcbc_qcn9274(rx_desc);
+	rx_desc_data->msdu_done = ath12k_hal_rx_h_msdu_done_qcn9274(ldesc);
+	rx_desc_data->l4_csum_fail = ath12k_hal_rx_h_l4_cksum_fail_qcn9274(rx_desc);
+	rx_desc_data->ip_csum_fail = ath12k_hal_rx_h_ip_cksum_fail_qcn9274(rx_desc);
+	rx_desc_data->is_decrypted = ath12k_hal_rx_h_is_decrypted_qcn9274(rx_desc);
+	rx_desc_data->err_bitmap = ath12k_hal_rx_h_mpdu_err_qcn9274(rx_desc);
+}
+
+const struct ath12k_hw_hal_params ath12k_hw_hal_params_qcn9274 = {
+	.rx_buf_rbm = HAL_RX_BUF_RBM_SW3_BM,
+	.wbm2sw_cc_enable = HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN |
+			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW1_EN |
+			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW2_EN |
+			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW3_EN |
+			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN,
+};
+
+const struct ath12k_hw_hal_params ath12k_hw_hal_params_ipq5332 = {
+	.rx_buf_rbm = HAL_RX_BUF_RBM_SW3_BM,
+	.wbm2sw_cc_enable = HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN |
+			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW1_EN |
+			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW2_EN |
+			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW3_EN |
+			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN,
+};
+
+static int ath12k_hal_srng_create_config_qcn9274(struct ath12k_hal *hal)
+{
+	struct hal_srng_config *s;
+
+	hal->srng_config = kmemdup(hw_srng_config_template,
+				   sizeof(hw_srng_config_template),
+				   GFP_KERNEL);
+	if (!hal->srng_config)
+		return -ENOMEM;
+
+	s = &hal->srng_config[HAL_REO_DST];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_HP;
+	s->reg_size[0] = HAL_REO2_RING_BASE_LSB(hal) - HAL_REO1_RING_BASE_LSB(hal);
+	s->reg_size[1] = HAL_REO2_RING_HP - HAL_REO1_RING_HP;
+
+	s = &hal->srng_config[HAL_REO_EXCEPTION];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_SW0_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_SW0_RING_HP;
+
+	s = &hal->srng_config[HAL_REO_REINJECT];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_HP;
+	s->reg_size[0] = HAL_SW2REO1_RING_BASE_LSB(hal) - HAL_SW2REO_RING_BASE_LSB(hal);
+	s->reg_size[1] = HAL_SW2REO1_RING_HP - HAL_SW2REO_RING_HP;
+
+	s = &hal->srng_config[HAL_REO_CMD];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_HP;
+
+	s = &hal->srng_config[HAL_REO_STATUS];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_HP;
+
+	s = &hal->srng_config[HAL_TCL_DATA];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_HP;
+	s->reg_size[0] = HAL_TCL2_RING_BASE_LSB(hal) - HAL_TCL1_RING_BASE_LSB(hal);
+	s->reg_size[1] = HAL_TCL2_RING_HP - HAL_TCL1_RING_HP;
+
+	s = &hal->srng_config[HAL_TCL_CMD];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_HP;
+
+	s = &hal->srng_config[HAL_TCL_STATUS];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_HP;
+
+	s = &hal->srng_config[HAL_CE_SRC];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal) + HAL_CE_DST_RING_BASE_LSB;
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal) + HAL_CE_DST_RING_HP;
+	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(hal) -
+		HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal);
+	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(hal) -
+		HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal);
+
+	s = &hal->srng_config[HAL_CE_DST];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) + HAL_CE_DST_RING_BASE_LSB;
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) + HAL_CE_DST_RING_HP;
+	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) -
+		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal);
+	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) -
+		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal);
+
+	s = &hal->srng_config[HAL_CE_DST_STATUS];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) +
+		HAL_CE_DST_STATUS_RING_BASE_LSB;
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) + HAL_CE_DST_STATUS_RING_HP;
+	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) -
+		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal);
+	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) -
+		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal);
+
+	s = &hal->srng_config[HAL_WBM_IDLE_LINK];
+	s->reg_start[0] =
+		HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_HP;
+
+	s = &hal->srng_config[HAL_SW2WBM_RELEASE];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG +
+		HAL_WBM_SW_RELEASE_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_SW_RELEASE_RING_HP;
+	s->reg_size[0] = HAL_WBM_SW1_RELEASE_RING_BASE_LSB(hal) -
+			 HAL_WBM_SW_RELEASE_RING_BASE_LSB(hal);
+	s->reg_size[1] = HAL_WBM_SW1_RELEASE_RING_HP - HAL_WBM_SW_RELEASE_RING_HP;
+
+	s = &hal->srng_config[HAL_WBM2SW_RELEASE];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_HP;
+	s->reg_size[0] = HAL_WBM1_RELEASE_RING_BASE_LSB(hal) -
+		HAL_WBM0_RELEASE_RING_BASE_LSB(hal);
+	s->reg_size[1] = HAL_WBM1_RELEASE_RING_HP - HAL_WBM0_RELEASE_RING_HP;
+
+	/* Some LMAC rings are not accessed from the host:
+	 * RXDMA_BUG, RXDMA_DST, RXDMA_MONITOR_BUF, RXDMA_MONITOR_STATUS,
+	 * RXDMA_MONITOR_DST, RXDMA_MONITOR_DESC, RXDMA_DIR_BUF_SRC,
+	 * RXDMA_RX_MONITOR_BUF, TX_MONITOR_BUF, TX_MONITOR_DST, SW2RXDMA
+	 */
+	s = &hal->srng_config[HAL_PPE2TCL];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_PPE2TCL1_RING_BASE_LSB;
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_PPE2TCL1_RING_HP;
+
+	s = &hal->srng_config[HAL_PPE_RELEASE];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG +
+				HAL_WBM_PPE_RELEASE_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_PPE_RELEASE_RING_HP;
+
+	return 0;
+}
+
+const struct ath12k_hal_tcl_to_wbm_rbm_map
+ath12k_hal_tcl_to_wbm_rbm_map_qcn9274[DP_TCL_NUM_RING_MAX] = {
+	{
+		.wbm_ring_num = 0,
+		.rbm_id = HAL_RX_BUF_RBM_SW0_BM,
+	},
+	{
+		.wbm_ring_num = 1,
+		.rbm_id = HAL_RX_BUF_RBM_SW1_BM,
+	},
+	{
+		.wbm_ring_num = 2,
+		.rbm_id = HAL_RX_BUF_RBM_SW2_BM,
+	},
+	{
+		.wbm_ring_num = 4,
+		.rbm_id = HAL_RX_BUF_RBM_SW4_BM,
+	},
+};
+
+const struct hal_ops hal_qcn9274_ops = {
+	.create_srng_config = ath12k_hal_srng_create_config_qcn9274,
+	.rx_desc_set_msdu_len = ath12k_hal_rx_desc_set_msdu_len_qcn9274,
+	.rx_desc_get_dot11_hdr = ath12k_hal_rx_desc_get_dot11_hdr_qcn9274,
+	.rx_desc_get_crypto_header = ath12k_hal_rx_desc_get_crypto_hdr_qcn9274,
+	.rx_desc_copy_end_tlv = ath12k_hal_rx_desc_copy_end_tlv_qcn9274,
+	.rx_desc_get_msdu_src_link_id = ath12k_hal_rx_desc_get_msdu_src_link_qcn9274,
+	.extract_rx_desc_data = ath12k_hal_extract_rx_desc_data_qcn9274,
+	.rx_desc_get_l3_pad_bytes = ath12k_hal_rx_desc_get_l3_pad_bytes_qcn9274,
+	.rx_desc_get_mpdu_ppdu_id = ath12k_hal_rx_desc_get_mpdu_ppdu_id_qcn9274,
+	.rx_desc_get_msdu_payload = ath12k_hal_rx_desc_get_msdu_payload_qcn9274,
+	.ce_dst_setup = ath12k_wifi7_hal_ce_dst_setup,
+	.srng_src_hw_init = ath12k_wifi7_hal_srng_src_hw_init,
+	.srng_dst_hw_init = ath12k_wifi7_hal_srng_dst_hw_init,
+	.set_umac_srng_ptr_addr = ath12k_wifi7_hal_set_umac_srng_ptr_addr,
+	.srng_update_shadow_config = ath12k_wifi7_hal_srng_update_shadow_config,
+	.srng_get_ring_id = ath12k_wifi7_hal_srng_get_ring_id,
+	.ce_get_desc_size = ath12k_wifi7_hal_ce_get_desc_size,
+	.ce_src_set_desc = ath12k_wifi7_hal_ce_src_set_desc,
+	.ce_dst_set_desc = ath12k_wifi7_hal_ce_dst_set_desc,
+	.ce_dst_status_get_length = ath12k_wifi7_hal_ce_dst_status_get_length,
+	.set_link_desc_addr = ath12k_wifi7_hal_set_link_desc_addr,
+	.tx_set_dscp_tid_map = ath12k_wifi7_hal_tx_set_dscp_tid_map,
+	.tx_configure_bank_register =
+				ath12k_wifi7_hal_tx_configure_bank_register,
+	.reoq_lut_addr_read_enable = ath12k_wifi7_hal_reoq_lut_addr_read_enable,
+	.reoq_lut_set_max_peerid = ath12k_wifi7_hal_reoq_lut_set_max_peerid,
+	.write_reoq_lut_addr = ath12k_wifi7_hal_write_reoq_lut_addr,
+	.write_ml_reoq_lut_addr = ath12k_wifi7_hal_write_ml_reoq_lut_addr,
+	.setup_link_idle_list = ath12k_wifi7_hal_setup_link_idle_list,
+	.reo_init_cmd_ring = ath12k_wifi7_hal_reo_init_cmd_ring,
+	.reo_hw_setup = ath12k_wifi7_hal_reo_hw_setup,
+	.reo_shared_qaddr_cache_clear = ath12k_wifi7_hal_reo_shared_qaddr_cache_clear,
+	.rx_buf_addr_info_set = ath12k_wifi7_hal_rx_buf_addr_info_set,
+	.rx_buf_addr_info_get = ath12k_wifi7_hal_rx_buf_addr_info_get,
+	.cc_config = ath12k_wifi7_hal_cc_config,
+	.get_idle_link_rbm = ath12k_wifi7_hal_get_idle_link_rbm,
+	.rx_msdu_list_get = ath12k_wifi7_hal_rx_msdu_list_get,
+	.rx_reo_ent_buf_paddr_get = ath12k_wifi7_hal_rx_reo_ent_buf_paddr_get,
+};
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.h
new file mode 100644
index 000000000000..08c0a0469474
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.h
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear*/
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef ATH12K_HAL_QCN9274_H
+#define ATH12K_HAL_QCN9274_H
+
+#include <linux/ieee80211.h>
+#include <linux/etherdevice.h>
+#include "../hal.h"
+#include "hal_rx.h"
+#include "hal.h"
+
+extern const struct hal_ops hal_qcn9274_ops;
+extern const struct ath12k_hw_regs qcn9274_v1_regs;
+extern const struct ath12k_hw_regs qcn9274_v2_regs;
+extern const struct ath12k_hw_regs ipq5332_regs;
+extern const struct ath12k_hal_tcl_to_wbm_rbm_map
+ath12k_hal_tcl_to_wbm_rbm_map_qcn9274[DP_TCL_NUM_RING_MAX];
+extern const struct ath12k_hw_hal_params ath12k_hw_hal_params_qcn9274;
+extern const struct ath12k_hw_hal_params ath12k_hw_hal_params_ipq5332;
+
+u8 ath12k_hal_rx_desc_get_l3_pad_bytes_qcn9274(struct hal_rx_desc *desc);
+void ath12k_hal_rx_desc_copy_end_tlv_qcn9274(struct hal_rx_desc *fdesc,
+					     struct hal_rx_desc *ldesc);
+u32 ath12k_hal_rx_desc_get_mpdu_ppdu_id_qcn9274(struct hal_rx_desc *desc);
+void ath12k_hal_rx_desc_set_msdu_len_qcn9274(struct hal_rx_desc *desc, u16 len);
+u8 *ath12k_hal_rx_desc_get_msdu_payload_qcn9274(struct hal_rx_desc *desc);
+u32 ath12k_hal_rx_desc_get_mpdu_start_offset_qcn9274(void);
+u32 ath12k_hal_rx_desc_get_msdu_end_offset_qcn9274(void);
+u32 ath12k_hal_get_rx_desc_size_qcn9274(void);
+u8 ath12k_hal_rx_desc_get_msdu_src_link_qcn9274(struct hal_rx_desc *desc);
+u16 ath12k_hal_rx_mpdu_start_wmask_get_qcn9274(void);
+u32 ath12k_hal_rx_msdu_end_wmask_get_qcn9274(void);
+void ath12k_hal_rx_desc_get_crypto_hdr_qcn9274(struct hal_rx_desc *desc,
+					       u8 *crypto_hdr,
+					       enum hal_encrypt_type enctype);
+void ath12k_hal_rx_desc_get_dot11_hdr_qcn9274(struct hal_rx_desc *desc,
+					      struct ieee80211_hdr *hdr);
+void ath12k_hal_extract_rx_desc_data_qcn9274(struct hal_rx_desc_data *rx_desc_data,
+					     struct hal_rx_desc *rx_desc,
+					     struct hal_rx_desc *ldesc);
+#endif
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.c b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.c
new file mode 100644
index 000000000000..903fb52a03bf
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.c
@@ -0,0 +1,974 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include "../debug.h"
+#include "../hal.h"
+#include "../hif.h"
+#include "hal_tx.h"
+#include "hal_rx.h"
+#include "hal_desc.h"
+#include "hal.h"
+
+static
+void ath12k_wifi7_hal_reo_set_desc_hdr(struct hal_desc_header *hdr,
+				       u8 owner, u8 buffer_type, u32 magic)
+{
+	hdr->info0 = le32_encode_bits(owner, HAL_DESC_HDR_INFO0_OWNER) |
+		     le32_encode_bits(buffer_type, HAL_DESC_HDR_INFO0_BUF_TYPE);
+
+	/* Magic pattern in reserved bits for debugging */
+	hdr->info0 |= le32_encode_bits(magic, HAL_DESC_HDR_INFO0_DBG_RESERVED);
+}
+
+static int ath12k_wifi7_hal_reo_cmd_queue_stats(struct hal_tlv_64_hdr *tlv,
+						struct ath12k_hal_reo_cmd *cmd)
+{
+	struct hal_reo_get_queue_stats *desc;
+
+	tlv->tl = le64_encode_bits(HAL_REO_GET_QUEUE_STATS, HAL_TLV_HDR_TAG) |
+		  le64_encode_bits(sizeof(*desc), HAL_TLV_HDR_LEN);
+
+	desc = (struct hal_reo_get_queue_stats *)tlv->value;
+	memset_startat(desc, 0, queue_addr_lo);
+
+	desc->cmd.info0 &= ~cpu_to_le32(HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED);
+	if (cmd->flag & HAL_REO_CMD_FLG_NEED_STATUS)
+		desc->cmd.info0 |= cpu_to_le32(HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED);
+
+	desc->queue_addr_lo = cpu_to_le32(cmd->addr_lo);
+	desc->info0 = le32_encode_bits(cmd->addr_hi,
+				       HAL_REO_GET_QUEUE_STATS_INFO0_QUEUE_ADDR_HI);
+	if (cmd->flag & HAL_REO_CMD_FLG_STATS_CLEAR)
+		desc->info0 |= cpu_to_le32(HAL_REO_GET_QUEUE_STATS_INFO0_CLEAR_STATS);
+
+	return le32_get_bits(desc->cmd.info0, HAL_REO_CMD_HDR_INFO0_CMD_NUMBER);
+}
+
+static int ath12k_wifi7_hal_reo_cmd_flush_cache(struct ath12k_hal *hal,
+						struct hal_tlv_64_hdr *tlv,
+						struct ath12k_hal_reo_cmd *cmd)
+{
+	struct hal_reo_flush_cache *desc;
+	u8 avail_slot = ffz(hal->avail_blk_resource);
+
+	if (cmd->flag & HAL_REO_CMD_FLG_FLUSH_BLOCK_LATER) {
+		if (avail_slot >= HAL_MAX_AVAIL_BLK_RES)
+			return -ENOSPC;
+
+		hal->current_blk_index = avail_slot;
+	}
+
+	tlv->tl = le64_encode_bits(HAL_REO_FLUSH_CACHE, HAL_TLV_HDR_TAG) |
+		  le64_encode_bits(sizeof(*desc), HAL_TLV_HDR_LEN);
+
+	desc = (struct hal_reo_flush_cache *)tlv->value;
+	memset_startat(desc, 0, cache_addr_lo);
+
+	desc->cmd.info0 &= ~cpu_to_le32(HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED);
+	if (cmd->flag & HAL_REO_CMD_FLG_NEED_STATUS)
+		desc->cmd.info0 |= cpu_to_le32(HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED);
+
+	desc->cache_addr_lo = cpu_to_le32(cmd->addr_lo);
+	desc->info0 = le32_encode_bits(cmd->addr_hi,
+				       HAL_REO_FLUSH_CACHE_INFO0_CACHE_ADDR_HI);
+
+	if (cmd->flag & HAL_REO_CMD_FLG_FLUSH_FWD_ALL_MPDUS)
+		desc->info0 |= cpu_to_le32(HAL_REO_FLUSH_CACHE_INFO0_FWD_ALL_MPDUS);
+
+	if (cmd->flag & HAL_REO_CMD_FLG_FLUSH_BLOCK_LATER) {
+		desc->info0 |= cpu_to_le32(HAL_REO_FLUSH_CACHE_INFO0_BLOCK_CACHE_USAGE);
+		desc->info0 |=
+			le32_encode_bits(avail_slot,
+					 HAL_REO_FLUSH_CACHE_INFO0_BLOCK_RESRC_IDX);
+	}
+
+	if (cmd->flag & HAL_REO_CMD_FLG_FLUSH_NO_INVAL)
+		desc->info0 |= cpu_to_le32(HAL_REO_FLUSH_CACHE_INFO0_FLUSH_WO_INVALIDATE);
+
+	if (cmd->flag & HAL_REO_CMD_FLG_FLUSH_ALL)
+		desc->info0 |= cpu_to_le32(HAL_REO_FLUSH_CACHE_INFO0_FLUSH_ALL);
+
+	if (cmd->flag & HAL_REO_CMD_FLG_FLUSH_QUEUE_1K_DESC)
+		desc->info0 |= cpu_to_le32(HAL_REO_FLUSH_CACHE_INFO0_FLUSH_QUEUE_1K_DESC);
+
+	return le32_get_bits(desc->cmd.info0, HAL_REO_CMD_HDR_INFO0_CMD_NUMBER);
+}
+
+static int
+ath12k_wifi7_hal_reo_cmd_update_rx_queue(struct hal_tlv_64_hdr *tlv,
+					 struct ath12k_hal_reo_cmd *cmd)
+{
+	struct hal_reo_update_rx_queue *desc;
+
+	tlv->tl = le64_encode_bits(HAL_REO_UPDATE_RX_REO_QUEUE, HAL_TLV_HDR_TAG) |
+		  le64_encode_bits(sizeof(*desc), HAL_TLV_HDR_LEN);
+
+	desc = (struct hal_reo_update_rx_queue *)tlv->value;
+	memset_startat(desc, 0, queue_addr_lo);
+
+	desc->cmd.info0 &= ~cpu_to_le32(HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED);
+	if (cmd->flag & HAL_REO_CMD_FLG_NEED_STATUS)
+		desc->cmd.info0 |= cpu_to_le32(HAL_REO_CMD_HDR_INFO0_STATUS_REQUIRED);
+
+	desc->queue_addr_lo = cpu_to_le32(cmd->addr_lo);
+	desc->info0 =
+		le32_encode_bits(cmd->addr_hi,
+				 HAL_REO_UPD_RX_QUEUE_INFO0_QUEUE_ADDR_HI) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_RX_QUEUE_NUM),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_RX_QUEUE_NUM) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_VLD),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_VLD) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_ALDC),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_ASSOC_LNK_DESC_CNT) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_DIS_DUP_DETECTION),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_DIS_DUP_DETECTION) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_SOFT_REORDER_EN),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_SOFT_REORDER_EN) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_AC),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_AC) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_BAR),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_BAR) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_RETRY),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_RETRY) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_CHECK_2K_MODE),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_CHECK_2K_MODE) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_OOR_MODE),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_OOR_MODE) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_BA_WINDOW_SIZE),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_BA_WINDOW_SIZE) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_PN_CHECK),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_CHECK) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_EVEN_PN),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_EVEN_PN) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_UNEVEN_PN),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_UNEVEN_PN) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_PN_HANDLE_ENABLE),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_HANDLE_ENABLE) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_PN_SIZE),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_SIZE) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_IGNORE_AMPDU_FLG),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_IGNORE_AMPDU_FLG) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_SVLD),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_SVLD) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_SSN),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_SSN) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_SEQ_2K_ERR),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_SEQ_2K_ERR) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_PN_VALID),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN_VALID) |
+		le32_encode_bits(!!(cmd->upd0 & HAL_REO_CMD_UPD0_PN),
+				 HAL_REO_UPD_RX_QUEUE_INFO0_UPD_PN);
+
+	desc->info1 =
+		le32_encode_bits(cmd->rx_queue_num,
+				 HAL_REO_UPD_RX_QUEUE_INFO1_RX_QUEUE_NUMBER) |
+		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_VLD),
+				 HAL_REO_UPD_RX_QUEUE_INFO1_VLD) |
+		le32_encode_bits(u32_get_bits(cmd->upd1, HAL_REO_CMD_UPD1_ALDC),
+				 HAL_REO_UPD_RX_QUEUE_INFO1_ASSOC_LNK_DESC_COUNTER) |
+		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_DIS_DUP_DETECTION),
+				 HAL_REO_UPD_RX_QUEUE_INFO1_DIS_DUP_DETECTION) |
+		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_SOFT_REORDER_EN),
+				 HAL_REO_UPD_RX_QUEUE_INFO1_SOFT_REORDER_EN) |
+		le32_encode_bits(u32_get_bits(cmd->upd1, HAL_REO_CMD_UPD1_AC),
+				 HAL_REO_UPD_RX_QUEUE_INFO1_AC) |
+		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_BAR),
+				 HAL_REO_UPD_RX_QUEUE_INFO1_BAR) |
+		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_CHECK_2K_MODE),
+				 HAL_REO_UPD_RX_QUEUE_INFO1_CHECK_2K_MODE) |
+		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_RETRY),
+				 HAL_REO_UPD_RX_QUEUE_INFO1_RETRY) |
+		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_OOR_MODE),
+				 HAL_REO_UPD_RX_QUEUE_INFO1_OOR_MODE) |
+		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_PN_CHECK),
+				 HAL_REO_UPD_RX_QUEUE_INFO1_PN_CHECK) |
+		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_EVEN_PN),
+				 HAL_REO_UPD_RX_QUEUE_INFO1_EVEN_PN) |
+		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_UNEVEN_PN),
+				 HAL_REO_UPD_RX_QUEUE_INFO1_UNEVEN_PN) |
+		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_PN_HANDLE_ENABLE),
+				 HAL_REO_UPD_RX_QUEUE_INFO1_PN_HANDLE_ENABLE) |
+		le32_encode_bits(!!(cmd->upd1 & HAL_REO_CMD_UPD1_IGNORE_AMPDU_FLG),
+				 HAL_REO_UPD_RX_QUEUE_INFO1_IGNORE_AMPDU_FLG);
+
+	if (cmd->pn_size == 24)
+		cmd->pn_size = HAL_RX_REO_QUEUE_PN_SIZE_24;
+	else if (cmd->pn_size == 48)
+		cmd->pn_size = HAL_RX_REO_QUEUE_PN_SIZE_48;
+	else if (cmd->pn_size == 128)
+		cmd->pn_size = HAL_RX_REO_QUEUE_PN_SIZE_128;
+
+	if (cmd->ba_window_size < 1)
+		cmd->ba_window_size = 1;
+
+	if (cmd->ba_window_size == 1)
+		cmd->ba_window_size++;
+
+	desc->info2 =
+		le32_encode_bits(cmd->ba_window_size - 1,
+				 HAL_REO_UPD_RX_QUEUE_INFO2_BA_WINDOW_SIZE) |
+		le32_encode_bits(cmd->pn_size, HAL_REO_UPD_RX_QUEUE_INFO2_PN_SIZE) |
+		le32_encode_bits(!!(cmd->upd2 & HAL_REO_CMD_UPD2_SVLD),
+				 HAL_REO_UPD_RX_QUEUE_INFO2_SVLD) |
+		le32_encode_bits(u32_get_bits(cmd->upd2, HAL_REO_CMD_UPD2_SSN),
+				 HAL_REO_UPD_RX_QUEUE_INFO2_SSN) |
+		le32_encode_bits(!!(cmd->upd2 & HAL_REO_CMD_UPD2_SEQ_2K_ERR),
+				 HAL_REO_UPD_RX_QUEUE_INFO2_SEQ_2K_ERR) |
+		le32_encode_bits(!!(cmd->upd2 & HAL_REO_CMD_UPD2_PN_ERR),
+				 HAL_REO_UPD_RX_QUEUE_INFO2_PN_ERR);
+
+	return le32_get_bits(desc->cmd.info0, HAL_REO_CMD_HDR_INFO0_CMD_NUMBER);
+}
+
+int ath12k_wifi7_hal_reo_cmd_send(struct ath12k_base *ab, struct hal_srng *srng,
+				  enum hal_reo_cmd_type type,
+				  struct ath12k_hal_reo_cmd *cmd)
+{
+	struct hal_tlv_64_hdr *reo_desc;
+	int ret;
+
+	spin_lock_bh(&srng->lock);
+
+	ath12k_hal_srng_access_begin(ab, srng);
+	reo_desc = ath12k_hal_srng_src_get_next_entry(ab, srng);
+	if (!reo_desc) {
+		ret = -ENOBUFS;
+		goto out;
+	}
+
+	switch (type) {
+	case HAL_REO_CMD_GET_QUEUE_STATS:
+		ret = ath12k_wifi7_hal_reo_cmd_queue_stats(reo_desc, cmd);
+		break;
+	case HAL_REO_CMD_FLUSH_CACHE:
+		ret = ath12k_wifi7_hal_reo_cmd_flush_cache(&ab->hal, reo_desc,
+							   cmd);
+		break;
+	case HAL_REO_CMD_UPDATE_RX_QUEUE:
+		ret = ath12k_wifi7_hal_reo_cmd_update_rx_queue(reo_desc, cmd);
+		break;
+	case HAL_REO_CMD_FLUSH_QUEUE:
+	case HAL_REO_CMD_UNBLOCK_CACHE:
+	case HAL_REO_CMD_FLUSH_TIMEOUT_LIST:
+		ath12k_warn(ab, "Unsupported reo command %d\n", type);
+		ret = -EOPNOTSUPP;
+		break;
+	default:
+		ath12k_warn(ab, "Unknown reo command %d\n", type);
+		ret = -EINVAL;
+		break;
+	}
+
+out:
+	ath12k_hal_srng_access_end(ab, srng);
+	spin_unlock_bh(&srng->lock);
+
+	return ret;
+}
+
+void ath12k_wifi7_hal_rx_buf_addr_info_set(struct ath12k_buffer_addr *binfo,
+					   dma_addr_t paddr, u32 cookie,
+					   u8 manager)
+{
+	u32 paddr_lo, paddr_hi;
+
+	paddr_lo = lower_32_bits(paddr);
+	paddr_hi = upper_32_bits(paddr);
+	binfo->info0 = le32_encode_bits(paddr_lo, BUFFER_ADDR_INFO0_ADDR);
+	binfo->info1 = le32_encode_bits(paddr_hi, BUFFER_ADDR_INFO1_ADDR) |
+		       le32_encode_bits(cookie, BUFFER_ADDR_INFO1_SW_COOKIE) |
+		       le32_encode_bits(manager, BUFFER_ADDR_INFO1_RET_BUF_MGR);
+}
+
+void ath12k_wifi7_hal_rx_buf_addr_info_get(struct ath12k_buffer_addr *binfo,
+					   dma_addr_t *paddr,
+					   u32 *cookie, u8 *rbm)
+{
+	*paddr = (((u64)le32_get_bits(binfo->info1, BUFFER_ADDR_INFO1_ADDR)) << 32) |
+		le32_get_bits(binfo->info0, BUFFER_ADDR_INFO0_ADDR);
+	*cookie = le32_get_bits(binfo->info1, BUFFER_ADDR_INFO1_SW_COOKIE);
+	*rbm = le32_get_bits(binfo->info1, BUFFER_ADDR_INFO1_RET_BUF_MGR);
+}
+
+void
+ath12k_wifi7_hal_rx_msdu_link_info_get(struct hal_rx_msdu_link *link,
+				       u32 *num_msdus, u32 *msdu_cookies,
+				       enum hal_rx_buf_return_buf_manager *rbm)
+{
+	struct hal_rx_msdu_details *msdu;
+	u32 val;
+	int i;
+
+	*num_msdus = HAL_NUM_RX_MSDUS_PER_LINK_DESC;
+
+	msdu = &link->msdu_link[0];
+	*rbm = le32_get_bits(msdu->buf_addr_info.info1,
+			     BUFFER_ADDR_INFO1_RET_BUF_MGR);
+
+	for (i = 0; i < *num_msdus; i++) {
+		msdu = &link->msdu_link[i];
+
+		val = le32_get_bits(msdu->buf_addr_info.info0,
+				    BUFFER_ADDR_INFO0_ADDR);
+		if (val == 0) {
+			*num_msdus = i;
+			break;
+		}
+		*msdu_cookies = le32_get_bits(msdu->buf_addr_info.info1,
+					      BUFFER_ADDR_INFO1_SW_COOKIE);
+		msdu_cookies++;
+	}
+}
+
+int ath12k_wifi7_hal_desc_reo_parse_err(struct ath12k_dp *dp,
+					struct hal_reo_dest_ring *desc,
+					dma_addr_t *paddr, u32 *desc_bank)
+{
+	struct ath12k_base *ab = dp->ab;
+	enum hal_reo_dest_ring_push_reason push_reason;
+	enum hal_reo_dest_ring_error_code err_code;
+	u32 cookie;
+
+	push_reason = le32_get_bits(desc->info0,
+				    HAL_REO_DEST_RING_INFO0_PUSH_REASON);
+	err_code = le32_get_bits(desc->info0,
+				 HAL_REO_DEST_RING_INFO0_ERROR_CODE);
+	dp->device_stats.reo_error[err_code]++;
+
+	if (push_reason != HAL_REO_DEST_RING_PUSH_REASON_ERR_DETECTED &&
+	    push_reason != HAL_REO_DEST_RING_PUSH_REASON_ROUTING_INSTRUCTION) {
+		ath12k_warn(ab, "expected error push reason code, received %d\n",
+			    push_reason);
+		return -EINVAL;
+	}
+
+	ath12k_wifi7_hal_rx_reo_ent_paddr_get(&desc->buf_addr_info, paddr,
+					      &cookie);
+	*desc_bank = u32_get_bits(cookie, DP_LINK_DESC_BANK_MASK);
+
+	return 0;
+}
+
+int ath12k_wifi7_hal_wbm_desc_parse_err(struct ath12k_dp *dp, void *desc,
+					struct hal_rx_wbm_rel_info *rel_info)
+{
+	struct hal_wbm_release_ring *wbm_desc = desc;
+	struct hal_wbm_release_ring_cc_rx *wbm_cc_desc = desc;
+	enum hal_wbm_rel_desc_type type;
+	enum hal_wbm_rel_src_module rel_src;
+	bool hw_cc_done;
+	u64 desc_va;
+	u32 val;
+
+	type = le32_get_bits(wbm_desc->info0, HAL_WBM_RELEASE_INFO0_DESC_TYPE);
+	/* We expect only WBM_REL buffer type */
+	if (type != HAL_WBM_REL_DESC_TYPE_REL_MSDU) {
+		WARN_ON(1);
+		return -EINVAL;
+	}
+
+	rel_src = le32_get_bits(wbm_desc->info0,
+				HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE);
+	if (rel_src != HAL_WBM_REL_SRC_MODULE_RXDMA &&
+	    rel_src != HAL_WBM_REL_SRC_MODULE_REO)
+		return -EINVAL;
+
+	/* The format of wbm rel ring desc changes based on the
+	 * hw cookie conversion status
+	 */
+	hw_cc_done = le32_get_bits(wbm_desc->info0,
+				   HAL_WBM_RELEASE_RX_INFO0_CC_STATUS);
+
+	if (!hw_cc_done) {
+		val = le32_get_bits(wbm_desc->buf_addr_info.info1,
+				    BUFFER_ADDR_INFO1_RET_BUF_MGR);
+		if (val != HAL_RX_BUF_RBM_SW3_BM) {
+			dp->device_stats.invalid_rbm++;
+			return -EINVAL;
+		}
+
+		rel_info->cookie = le32_get_bits(wbm_desc->buf_addr_info.info1,
+						 BUFFER_ADDR_INFO1_SW_COOKIE);
+
+		rel_info->rx_desc = NULL;
+	} else {
+		val = le32_get_bits(wbm_cc_desc->info0,
+				    HAL_WBM_RELEASE_RX_CC_INFO0_RBM);
+		if (val != HAL_RX_BUF_RBM_SW3_BM) {
+			dp->device_stats.invalid_rbm++;
+			return -EINVAL;
+		}
+
+		rel_info->cookie = le32_get_bits(wbm_cc_desc->info1,
+						 HAL_WBM_RELEASE_RX_CC_INFO1_COOKIE);
+
+		desc_va = ((u64)le32_to_cpu(wbm_cc_desc->buf_va_hi) << 32 |
+			   le32_to_cpu(wbm_cc_desc->buf_va_lo));
+		rel_info->rx_desc =
+			(struct ath12k_rx_desc_info *)((unsigned long)desc_va);
+	}
+
+	rel_info->err_rel_src = rel_src;
+	rel_info->hw_cc_done = hw_cc_done;
+
+	rel_info->first_msdu = le32_get_bits(wbm_desc->info3,
+					     HAL_WBM_RELEASE_INFO3_FIRST_MSDU);
+	rel_info->last_msdu = le32_get_bits(wbm_desc->info3,
+					    HAL_WBM_RELEASE_INFO3_LAST_MSDU);
+	rel_info->continuation = le32_get_bits(wbm_desc->info3,
+					       HAL_WBM_RELEASE_INFO3_CONTINUATION);
+
+	if (rel_info->err_rel_src == HAL_WBM_REL_SRC_MODULE_REO) {
+		rel_info->push_reason =
+			le32_get_bits(wbm_desc->info0,
+				      HAL_WBM_RELEASE_INFO0_REO_PUSH_REASON);
+		rel_info->err_code =
+			le32_get_bits(wbm_desc->info0,
+				      HAL_WBM_RELEASE_INFO0_REO_ERROR_CODE);
+	} else {
+		rel_info->push_reason =
+			le32_get_bits(wbm_desc->info0,
+				      HAL_WBM_RELEASE_INFO0_RXDMA_PUSH_REASON);
+		rel_info->err_code =
+			le32_get_bits(wbm_desc->info0,
+				      HAL_WBM_RELEASE_INFO0_RXDMA_ERROR_CODE);
+	}
+
+	rel_info->peer_metadata = wbm_desc->info2;
+
+	return 0;
+}
+
+void ath12k_wifi7_hal_rx_reo_ent_paddr_get(struct ath12k_buffer_addr *buff_addr,
+					   dma_addr_t *paddr, u32 *cookie)
+{
+	*paddr = ((u64)(le32_get_bits(buff_addr->info1,
+				      BUFFER_ADDR_INFO1_ADDR)) << 32) |
+		le32_get_bits(buff_addr->info0, BUFFER_ADDR_INFO0_ADDR);
+
+	*cookie = le32_get_bits(buff_addr->info1, BUFFER_ADDR_INFO1_SW_COOKIE);
+}
+
+void ath12k_wifi7_hal_rx_reo_ent_buf_paddr_get(void *rx_desc, dma_addr_t *paddr,
+					       u32 *sw_cookie,
+					       struct ath12k_buffer_addr **pp_buf_addr,
+					       u8 *rbm, u32 *msdu_cnt)
+{
+	struct hal_reo_entrance_ring *reo_ent_ring =
+		(struct hal_reo_entrance_ring *)rx_desc;
+	struct ath12k_buffer_addr *buf_addr_info;
+	struct rx_mpdu_desc *rx_mpdu_desc_info_details;
+
+	rx_mpdu_desc_info_details =
+			(struct rx_mpdu_desc *)&reo_ent_ring->rx_mpdu_info;
+
+	*msdu_cnt = le32_get_bits(rx_mpdu_desc_info_details->info0,
+				  RX_MPDU_DESC_INFO0_MSDU_COUNT);
+
+	buf_addr_info = (struct ath12k_buffer_addr *)&reo_ent_ring->buf_addr_info;
+
+	*paddr = (((u64)le32_get_bits(buf_addr_info->info1,
+				      BUFFER_ADDR_INFO1_ADDR)) << 32) |
+			le32_get_bits(buf_addr_info->info0,
+				      BUFFER_ADDR_INFO0_ADDR);
+
+	*sw_cookie = le32_get_bits(buf_addr_info->info1,
+				   BUFFER_ADDR_INFO1_SW_COOKIE);
+	*rbm = le32_get_bits(buf_addr_info->info1,
+			     BUFFER_ADDR_INFO1_RET_BUF_MGR);
+
+	*pp_buf_addr = (void *)buf_addr_info;
+}
+
+void ath12k_wifi7_hal_rx_msdu_list_get(struct ath12k *ar,
+				       void *link_desc_opaque,
+				       void *msdu_list_opaque, u16 *num_msdus)
+{
+	struct hal_rx_msdu_link *link_desc =
+				(struct hal_rx_msdu_link *)link_desc_opaque;
+	struct hal_rx_msdu_list *msdu_list =
+				(struct hal_rx_msdu_list *)msdu_list_opaque;
+	struct hal_rx_msdu_details *msdu_details = NULL;
+	struct rx_msdu_desc *msdu_desc_info = NULL;
+	u32 last = 0, first = 0;
+	u8 tmp = 0;
+	int i;
+
+	last = u32_encode_bits(last, RX_MSDU_DESC_INFO0_LAST_MSDU_IN_MPDU);
+	first = u32_encode_bits(first, RX_MSDU_DESC_INFO0_FIRST_MSDU_IN_MPDU);
+	msdu_details = &link_desc->msdu_link[0];
+
+	for (i = 0; i < HAL_RX_NUM_MSDU_DESC; i++) {
+		if (!i && le32_get_bits(msdu_details[i].buf_addr_info.info0,
+					BUFFER_ADDR_INFO0_ADDR) == 0)
+			break;
+		if (le32_get_bits(msdu_details[i].buf_addr_info.info0,
+				  BUFFER_ADDR_INFO0_ADDR) == 0) {
+			msdu_desc_info = &msdu_details[i - 1].rx_msdu_info;
+			msdu_desc_info->info0 |= cpu_to_le32(last);
+			break;
+		}
+		msdu_desc_info = &msdu_details[i].rx_msdu_info;
+
+		if (!i)
+			msdu_desc_info->info0 |= cpu_to_le32(first);
+		else if (i == (HAL_RX_NUM_MSDU_DESC - 1))
+			msdu_desc_info->info0 |= cpu_to_le32(last);
+		msdu_list->msdu_info[i].msdu_flags = le32_to_cpu(msdu_desc_info->info0);
+		msdu_list->msdu_info[i].msdu_len =
+			 HAL_RX_MSDU_PKT_LENGTH_GET(msdu_desc_info->info0);
+		msdu_list->sw_cookie[i] =
+			le32_get_bits(msdu_details[i].buf_addr_info.info1,
+				      BUFFER_ADDR_INFO1_SW_COOKIE);
+		tmp = le32_get_bits(msdu_details[i].buf_addr_info.info1,
+				    BUFFER_ADDR_INFO1_RET_BUF_MGR);
+		msdu_list->paddr[i] =
+			((u64)(le32_get_bits(msdu_details[i].buf_addr_info.info1,
+					     BUFFER_ADDR_INFO1_ADDR)) << 32) |
+			le32_get_bits(msdu_details[i].buf_addr_info.info0,
+				      BUFFER_ADDR_INFO0_ADDR);
+		msdu_list->rbm[i] = tmp;
+	}
+	*num_msdus = i;
+}
+
+void
+ath12k_wifi7_hal_rx_msdu_link_desc_set(struct ath12k_base *ab,
+				       struct hal_wbm_release_ring *desc,
+				       struct ath12k_buffer_addr *buf_addr_info,
+				       enum hal_wbm_rel_bm_act action)
+{
+	desc->buf_addr_info = *buf_addr_info;
+	desc->info0 |= le32_encode_bits(HAL_WBM_REL_SRC_MODULE_SW,
+					HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE) |
+		    le32_encode_bits(action, HAL_WBM_RELEASE_INFO0_BM_ACTION) |
+		    le32_encode_bits(HAL_WBM_REL_DESC_TYPE_MSDU_LINK,
+				     HAL_WBM_RELEASE_INFO0_DESC_TYPE);
+}
+
+void ath12k_wifi7_hal_reo_status_queue_stats(struct ath12k_base *ab,
+					     struct hal_tlv_64_hdr *tlv,
+					     struct hal_reo_status *status)
+{
+	struct hal_reo_get_queue_stats_status *desc =
+		(struct hal_reo_get_queue_stats_status *)tlv->value;
+
+	status->uniform_hdr.cmd_num =
+				le32_get_bits(desc->hdr.info0,
+					      HAL_REO_STATUS_HDR_INFO0_STATUS_NUM);
+	status->uniform_hdr.cmd_status =
+				le32_get_bits(desc->hdr.info0,
+					      HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS);
+
+	ath12k_dbg(ab, ATH12K_DBG_HAL, "Queue stats status:\n");
+	ath12k_dbg(ab, ATH12K_DBG_HAL, "header: cmd_num %d status %d\n",
+		   status->uniform_hdr.cmd_num,
+		   status->uniform_hdr.cmd_status);
+	ath12k_dbg(ab, ATH12K_DBG_HAL, "ssn %u cur_idx %u\n",
+		   le32_get_bits(desc->info0,
+				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO0_SSN),
+		   le32_get_bits(desc->info0,
+				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO0_CUR_IDX));
+	ath12k_dbg(ab, ATH12K_DBG_HAL, "pn = [%08x, %08x, %08x, %08x]\n",
+		   desc->pn[0], desc->pn[1], desc->pn[2], desc->pn[3]);
+	ath12k_dbg(ab, ATH12K_DBG_HAL, "last_rx: enqueue_tstamp %08x dequeue_tstamp %08x\n",
+		   desc->last_rx_enqueue_timestamp,
+		   desc->last_rx_dequeue_timestamp);
+	ath12k_dbg(ab, ATH12K_DBG_HAL, "rx_bitmap [%08x %08x %08x %08x %08x %08x %08x %08x]\n",
+		   desc->rx_bitmap[0], desc->rx_bitmap[1], desc->rx_bitmap[2],
+		   desc->rx_bitmap[3], desc->rx_bitmap[4], desc->rx_bitmap[5],
+		   desc->rx_bitmap[6], desc->rx_bitmap[7]);
+	ath12k_dbg(ab, ATH12K_DBG_HAL, "count: cur_mpdu %u cur_msdu %u\n",
+		   le32_get_bits(desc->info1,
+				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO1_MPDU_COUNT),
+		   le32_get_bits(desc->info1,
+				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO1_MSDU_COUNT));
+	ath12k_dbg(ab, ATH12K_DBG_HAL, "fwd_timeout %u fwd_bar %u dup_count %u\n",
+		   le32_get_bits(desc->info2,
+				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO2_TIMEOUT_COUNT),
+		   le32_get_bits(desc->info2,
+				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO2_FDTB_COUNT),
+		   le32_get_bits(desc->info2,
+				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO2_DUPLICATE_COUNT));
+	ath12k_dbg(ab, ATH12K_DBG_HAL, "frames_in_order %u bar_rcvd %u\n",
+		   le32_get_bits(desc->info3,
+				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO3_FIO_COUNT),
+		   le32_get_bits(desc->info3,
+				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO3_BAR_RCVD_CNT));
+	ath12k_dbg(ab, ATH12K_DBG_HAL, "num_mpdus %d num_msdus %d total_bytes %d\n",
+		   desc->num_mpdu_frames, desc->num_msdu_frames,
+		   desc->total_bytes);
+	ath12k_dbg(ab, ATH12K_DBG_HAL, "late_rcvd %u win_jump_2k %u hole_cnt %u\n",
+		   le32_get_bits(desc->info4,
+				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO4_LATE_RX_MPDU),
+		   le32_get_bits(desc->info2,
+				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO2_WINDOW_JMP2K),
+		   le32_get_bits(desc->info4,
+				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO4_HOLE_COUNT));
+	ath12k_dbg(ab, ATH12K_DBG_HAL, "looping count %u\n",
+		   le32_get_bits(desc->info5,
+				 HAL_REO_GET_QUEUE_STATS_STATUS_INFO5_LOOPING_CNT));
+}
+
+void ath12k_wifi7_hal_reo_flush_queue_status(struct ath12k_base *ab,
+					     struct hal_tlv_64_hdr *tlv,
+					     struct hal_reo_status *status)
+{
+	struct hal_reo_flush_queue_status *desc =
+		(struct hal_reo_flush_queue_status *)tlv->value;
+
+	status->uniform_hdr.cmd_num =
+			le32_get_bits(desc->hdr.info0,
+				      HAL_REO_STATUS_HDR_INFO0_STATUS_NUM);
+	status->uniform_hdr.cmd_status =
+			le32_get_bits(desc->hdr.info0,
+				      HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS);
+	status->u.flush_queue.err_detected =
+			le32_get_bits(desc->info0,
+				      HAL_REO_FLUSH_QUEUE_INFO0_ERR_DETECTED);
+}
+
+void
+ath12k_wifi7_hal_reo_flush_cache_status(struct ath12k_base *ab,
+					struct hal_tlv_64_hdr *tlv,
+					struct hal_reo_status *status)
+{
+	struct ath12k_hal *hal = &ab->hal;
+	struct hal_reo_flush_cache_status *desc =
+		(struct hal_reo_flush_cache_status *)tlv->value;
+
+	status->uniform_hdr.cmd_num =
+			le32_get_bits(desc->hdr.info0,
+				      HAL_REO_STATUS_HDR_INFO0_STATUS_NUM);
+	status->uniform_hdr.cmd_status =
+			le32_get_bits(desc->hdr.info0,
+				      HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS);
+
+	status->u.flush_cache.err_detected =
+			le32_get_bits(desc->info0,
+				      HAL_REO_FLUSH_CACHE_STATUS_INFO0_IS_ERR);
+	status->u.flush_cache.err_code =
+		le32_get_bits(desc->info0,
+			      HAL_REO_FLUSH_CACHE_STATUS_INFO0_BLOCK_ERR_CODE);
+	if (!status->u.flush_cache.err_code)
+		hal->avail_blk_resource |= BIT(hal->current_blk_index);
+
+	status->u.flush_cache.cache_controller_flush_status_hit =
+		le32_get_bits(desc->info0,
+			      HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_STATUS_HIT);
+
+	status->u.flush_cache.cache_controller_flush_status_desc_type =
+		le32_get_bits(desc->info0,
+			      HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_DESC_TYPE);
+	status->u.flush_cache.cache_controller_flush_status_client_id =
+		le32_get_bits(desc->info0,
+			      HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_CLIENT_ID);
+	status->u.flush_cache.cache_controller_flush_status_err =
+		le32_get_bits(desc->info0,
+			      HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_ERR);
+	status->u.flush_cache.cache_controller_flush_status_cnt =
+		le32_get_bits(desc->info0,
+			      HAL_REO_FLUSH_CACHE_STATUS_INFO0_FLUSH_COUNT);
+}
+
+void ath12k_wifi7_hal_reo_unblk_cache_status(struct ath12k_base *ab,
+					     struct hal_tlv_64_hdr *tlv,
+					     struct hal_reo_status *status)
+{
+	struct ath12k_hal *hal = &ab->hal;
+	struct hal_reo_unblock_cache_status *desc =
+		(struct hal_reo_unblock_cache_status *)tlv->value;
+
+	status->uniform_hdr.cmd_num =
+			le32_get_bits(desc->hdr.info0,
+				      HAL_REO_STATUS_HDR_INFO0_STATUS_NUM);
+	status->uniform_hdr.cmd_status =
+			le32_get_bits(desc->hdr.info0,
+				      HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS);
+
+	status->u.unblock_cache.err_detected =
+			le32_get_bits(desc->info0,
+				      HAL_REO_UNBLOCK_CACHE_STATUS_INFO0_IS_ERR);
+	status->u.unblock_cache.unblock_type =
+			le32_get_bits(desc->info0,
+				      HAL_REO_UNBLOCK_CACHE_STATUS_INFO0_TYPE);
+
+	if (!status->u.unblock_cache.err_detected &&
+	    status->u.unblock_cache.unblock_type ==
+	    HAL_REO_STATUS_UNBLOCK_BLOCKING_RESOURCE)
+		hal->avail_blk_resource &= ~BIT(hal->current_blk_index);
+}
+
+void
+ath12k_wifi7_hal_reo_flush_timeout_list_status(struct ath12k_base *ab,
+					       struct hal_tlv_64_hdr *tlv,
+					       struct hal_reo_status *status)
+{
+	struct hal_reo_flush_timeout_list_status *desc =
+		(struct hal_reo_flush_timeout_list_status *)tlv->value;
+
+	status->uniform_hdr.cmd_num =
+			le32_get_bits(desc->hdr.info0,
+				      HAL_REO_STATUS_HDR_INFO0_STATUS_NUM);
+	status->uniform_hdr.cmd_status =
+			le32_get_bits(desc->hdr.info0,
+				      HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS);
+
+	status->u.timeout_list.err_detected =
+			le32_get_bits(desc->info0,
+				      HAL_REO_FLUSH_TIMEOUT_STATUS_INFO0_IS_ERR);
+	status->u.timeout_list.list_empty =
+			le32_get_bits(desc->info0,
+				      HAL_REO_FLUSH_TIMEOUT_STATUS_INFO0_LIST_EMPTY);
+
+	status->u.timeout_list.release_desc_cnt =
+		le32_get_bits(desc->info1,
+			      HAL_REO_FLUSH_TIMEOUT_STATUS_INFO1_REL_DESC_COUNT);
+	status->u.timeout_list.fwd_buf_cnt =
+		le32_get_bits(desc->info0,
+			      HAL_REO_FLUSH_TIMEOUT_STATUS_INFO1_FWD_BUF_COUNT);
+}
+
+void
+ath12k_wifi7_hal_reo_desc_thresh_reached_status(struct ath12k_base *ab,
+						struct hal_tlv_64_hdr *tlv,
+						struct hal_reo_status *status)
+{
+	struct hal_reo_desc_thresh_reached_status *desc =
+		(struct hal_reo_desc_thresh_reached_status *)tlv->value;
+
+	status->uniform_hdr.cmd_num =
+			le32_get_bits(desc->hdr.info0,
+				      HAL_REO_STATUS_HDR_INFO0_STATUS_NUM);
+	status->uniform_hdr.cmd_status =
+			le32_get_bits(desc->hdr.info0,
+				      HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS);
+
+	status->u.desc_thresh_reached.threshold_idx =
+		le32_get_bits(desc->info0,
+			      HAL_REO_DESC_THRESH_STATUS_INFO0_THRESH_INDEX);
+
+	status->u.desc_thresh_reached.link_desc_counter0 =
+		le32_get_bits(desc->info1,
+			      HAL_REO_DESC_THRESH_STATUS_INFO1_LINK_DESC_COUNTER0);
+
+	status->u.desc_thresh_reached.link_desc_counter1 =
+		le32_get_bits(desc->info2,
+			      HAL_REO_DESC_THRESH_STATUS_INFO2_LINK_DESC_COUNTER1);
+
+	status->u.desc_thresh_reached.link_desc_counter2 =
+		le32_get_bits(desc->info3,
+			      HAL_REO_DESC_THRESH_STATUS_INFO3_LINK_DESC_COUNTER2);
+
+	status->u.desc_thresh_reached.link_desc_counter_sum =
+		le32_get_bits(desc->info4,
+			      HAL_REO_DESC_THRESH_STATUS_INFO4_LINK_DESC_COUNTER_SUM);
+}
+
+void ath12k_wifi7_hal_reo_update_rx_reo_queue_status(struct ath12k_base *ab,
+						     struct hal_tlv_64_hdr *tlv,
+						     struct hal_reo_status *status)
+{
+	struct hal_reo_status_hdr *desc =
+		(struct hal_reo_status_hdr *)tlv->value;
+
+	status->uniform_hdr.cmd_num =
+			le32_get_bits(desc->info0,
+				      HAL_REO_STATUS_HDR_INFO0_STATUS_NUM);
+	status->uniform_hdr.cmd_status =
+			le32_get_bits(desc->info0,
+				      HAL_REO_STATUS_HDR_INFO0_EXEC_STATUS);
+}
+
+u32 ath12k_wifi7_hal_reo_qdesc_size(u32 ba_window_size, u8 tid)
+{
+	u32 num_ext_desc, num_1k_desc = 0;
+
+	if (ba_window_size <= 1) {
+		if (tid != HAL_DESC_REO_NON_QOS_TID)
+			num_ext_desc = 1;
+		else
+			num_ext_desc = 0;
+
+	} else if (ba_window_size <= 105) {
+		num_ext_desc = 1;
+	} else if (ba_window_size <= 210) {
+		num_ext_desc = 2;
+	} else if (ba_window_size <= 256) {
+		num_ext_desc = 3;
+	} else {
+		num_ext_desc = 10;
+		num_1k_desc = 1;
+	}
+
+	return sizeof(struct hal_rx_reo_queue) +
+		(num_ext_desc * sizeof(struct hal_rx_reo_queue_ext)) +
+		(num_1k_desc * sizeof(struct hal_rx_reo_queue_1k));
+}
+
+void ath12k_wifi7_hal_reo_qdesc_setup(struct hal_rx_reo_queue *qdesc,
+				      int tid, u32 ba_window_size,
+				      u32 start_seq, enum hal_pn_type type)
+{
+	struct hal_rx_reo_queue_ext *ext_desc;
+
+	ath12k_wifi7_hal_reo_set_desc_hdr(&qdesc->desc_hdr, HAL_DESC_REO_OWNED,
+					  HAL_DESC_REO_QUEUE_DESC,
+					  REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_0);
+
+	qdesc->rx_queue_num = le32_encode_bits(tid, HAL_RX_REO_QUEUE_RX_QUEUE_NUMBER);
+
+	qdesc->info0 =
+		le32_encode_bits(1, HAL_RX_REO_QUEUE_INFO0_VLD) |
+		le32_encode_bits(1, HAL_RX_REO_QUEUE_INFO0_ASSOC_LNK_DESC_COUNTER) |
+		le32_encode_bits(ath12k_tid_to_ac(tid), HAL_RX_REO_QUEUE_INFO0_AC);
+
+	if (ba_window_size < 1)
+		ba_window_size = 1;
+
+	if (ba_window_size == 1 && tid != HAL_DESC_REO_NON_QOS_TID)
+		ba_window_size++;
+
+	if (ba_window_size == 1)
+		qdesc->info0 |= le32_encode_bits(1, HAL_RX_REO_QUEUE_INFO0_RETRY);
+
+	qdesc->info0 |= le32_encode_bits(ba_window_size - 1,
+					 HAL_RX_REO_QUEUE_INFO0_BA_WINDOW_SIZE);
+	switch (type) {
+	case HAL_PN_TYPE_NONE:
+	case HAL_PN_TYPE_WAPI_EVEN:
+	case HAL_PN_TYPE_WAPI_UNEVEN:
+		break;
+	case HAL_PN_TYPE_WPA:
+		qdesc->info0 |=
+			le32_encode_bits(1, HAL_RX_REO_QUEUE_INFO0_PN_CHECK) |
+			le32_encode_bits(HAL_RX_REO_QUEUE_PN_SIZE_48,
+					 HAL_RX_REO_QUEUE_INFO0_PN_SIZE);
+		break;
+	}
+
+	/* TODO: Set Ignore ampdu flags based on BA window size and/or
+	 * AMPDU capabilities
+	 */
+	qdesc->info0 |= le32_encode_bits(1, HAL_RX_REO_QUEUE_INFO0_IGNORE_AMPDU_FLG);
+
+	qdesc->info1 |= le32_encode_bits(0, HAL_RX_REO_QUEUE_INFO1_SVLD);
+
+	if (start_seq <= 0xfff)
+		qdesc->info1 = le32_encode_bits(start_seq,
+						HAL_RX_REO_QUEUE_INFO1_SSN);
+
+	if (tid == HAL_DESC_REO_NON_QOS_TID)
+		return;
+
+	ext_desc = qdesc->ext_desc;
+
+	/* TODO: HW queue descriptors are currently allocated for max BA
+	 * window size for all QOS TIDs so that same descriptor can be used
+	 * later when ADDBA request is received. This should be changed to
+	 * allocate HW queue descriptors based on BA window size being
+	 * negotiated (0 for non BA cases), and reallocate when BA window
+	 * size changes and also send WMI message to FW to change the REO
+	 * queue descriptor in Rx peer entry as part of dp_rx_tid_update.
+	 */
+	memset(ext_desc, 0, 3 * sizeof(*ext_desc));
+	ath12k_wifi7_hal_reo_set_desc_hdr(&ext_desc->desc_hdr,
+					  HAL_DESC_REO_OWNED,
+					  HAL_DESC_REO_QUEUE_EXT_DESC,
+					  REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_1);
+	ext_desc++;
+	ath12k_wifi7_hal_reo_set_desc_hdr(&ext_desc->desc_hdr,
+					  HAL_DESC_REO_OWNED,
+					  HAL_DESC_REO_QUEUE_EXT_DESC,
+					  REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_2);
+	ext_desc++;
+	ath12k_wifi7_hal_reo_set_desc_hdr(&ext_desc->desc_hdr,
+					  HAL_DESC_REO_OWNED,
+					  HAL_DESC_REO_QUEUE_EXT_DESC,
+					  REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_3);
+}
+
+void ath12k_wifi7_hal_reo_init_cmd_ring(struct ath12k_base *ab,
+					struct hal_srng *srng)
+{
+	struct hal_srng_params params;
+	struct hal_tlv_64_hdr *tlv;
+	struct hal_reo_get_queue_stats *desc;
+	int i, cmd_num = 1;
+	int entry_size;
+	u8 *entry;
+
+	memset(&params, 0, sizeof(params));
+
+	entry_size = ath12k_hal_srng_get_entrysize(ab, HAL_REO_CMD);
+	ath12k_hal_srng_get_params(ab, srng, &params);
+	entry = (u8 *)params.ring_base_vaddr;
+
+	for (i = 0; i < params.num_entries; i++) {
+		tlv = (struct hal_tlv_64_hdr *)entry;
+		desc = (struct hal_reo_get_queue_stats *)tlv->value;
+		desc->cmd.info0 = le32_encode_bits(cmd_num++,
+						   HAL_REO_CMD_HDR_INFO0_CMD_NUMBER);
+		entry += entry_size;
+	}
+}
+
+void ath12k_wifi7_hal_reo_hw_setup(struct ath12k_base *ab, u32 ring_hash_map)
+{
+	struct ath12k_hal *hal = &ab->hal;
+
+	u32 reo_base = HAL_SEQ_WCSS_UMAC_REO_REG;
+	u32 val;
+
+	val = ath12k_hif_read32(ab, reo_base + HAL_REO1_GEN_ENABLE);
+
+	val |= u32_encode_bits(1, HAL_REO1_GEN_ENABLE_AGING_LIST_ENABLE) |
+	       u32_encode_bits(1, HAL_REO1_GEN_ENABLE_AGING_FLUSH_ENABLE);
+	ath12k_hif_write32(ab, reo_base + HAL_REO1_GEN_ENABLE, val);
+
+	val = ath12k_hif_read32(ab, reo_base + HAL_REO1_MISC_CTRL_ADDR(hal));
+
+	val &= ~(HAL_REO1_MISC_CTL_FRAG_DST_RING |
+		 HAL_REO1_MISC_CTL_BAR_DST_RING);
+	val |= u32_encode_bits(HAL_SRNG_RING_ID_REO2SW0,
+			       HAL_REO1_MISC_CTL_FRAG_DST_RING);
+	val |= u32_encode_bits(HAL_SRNG_RING_ID_REO2SW0,
+			       HAL_REO1_MISC_CTL_BAR_DST_RING);
+	ath12k_hif_write32(ab, reo_base + HAL_REO1_MISC_CTRL_ADDR(hal), val);
+
+	ath12k_hif_write32(ab, reo_base + HAL_REO1_AGING_THRESH_IX_0(hal),
+			   HAL_DEFAULT_BE_BK_VI_REO_TIMEOUT_USEC);
+	ath12k_hif_write32(ab, reo_base + HAL_REO1_AGING_THRESH_IX_1(hal),
+			   HAL_DEFAULT_BE_BK_VI_REO_TIMEOUT_USEC);
+	ath12k_hif_write32(ab, reo_base + HAL_REO1_AGING_THRESH_IX_2(hal),
+			   HAL_DEFAULT_BE_BK_VI_REO_TIMEOUT_USEC);
+	ath12k_hif_write32(ab, reo_base + HAL_REO1_AGING_THRESH_IX_3(hal),
+			   HAL_DEFAULT_VO_REO_TIMEOUT_USEC);
+
+	ath12k_hif_write32(ab, reo_base + HAL_REO1_DEST_RING_CTRL_IX_2,
+			   ring_hash_map);
+	ath12k_hif_write32(ab, reo_base + HAL_REO1_DEST_RING_CTRL_IX_3,
+			   ring_hash_map);
+}
+
+void ath12k_wifi7_hal_reo_shared_qaddr_cache_clear(struct ath12k_base *ab)
+{
+	u32 val;
+	struct ath12k_hal *hal = &ab->hal;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+
+	lockdep_assert_held(&dp->dp_lock);
+	val = ath12k_hif_read32(ab, HAL_SEQ_WCSS_UMAC_REO_REG +
+				HAL_REO1_QDESC_ADDR(hal));
+
+	val |= u32_encode_bits(1, HAL_REO_QDESC_ADDR_READ_CLEAR_QDESC_ARRAY);
+	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG +
+			   HAL_REO1_QDESC_ADDR(hal), val);
+
+	val &= ~HAL_REO_QDESC_ADDR_READ_CLEAR_QDESC_ARRAY;
+	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG +
+			   HAL_REO1_QDESC_ADDR(hal), val);
+}
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.h
new file mode 100644
index 000000000000..8a0f4a781d8a
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.h
@@ -0,0 +1,871 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef ATH12K_HAL_RX_H
+#define ATH12K_HAL_RX_H
+
+#include "hal_desc.h"
+
+struct hal_reo_status;
+
+struct hal_rx_wbm_rel_info {
+	u32 cookie;
+	enum hal_wbm_rel_src_module err_rel_src;
+	enum hal_reo_dest_ring_push_reason push_reason;
+	u32 err_code;
+	bool first_msdu;
+	bool last_msdu;
+	bool continuation;
+	void *rx_desc;
+	bool hw_cc_done;
+	__le32 peer_metadata;
+};
+
+#define HAL_RX_MPDU_INFO_PN_GET_BYTE1(__val) \
+	le32_get_bits((__val), GENMASK(7, 0))
+
+#define HAL_RX_MPDU_INFO_PN_GET_BYTE2(__val) \
+	le32_get_bits((__val), GENMASK(15, 8))
+
+#define HAL_RX_MPDU_INFO_PN_GET_BYTE3(__val) \
+	le32_get_bits((__val), GENMASK(23, 16))
+
+#define HAL_RX_MPDU_INFO_PN_GET_BYTE4(__val) \
+	le32_get_bits((__val), GENMASK(31, 24))
+
+struct hal_rx_mon_status_tlv_hdr {
+	u32 hdr;
+	u8 value[];
+};
+
+#define HAL_TLV_STATUS_PPDU_NOT_DONE            0
+#define HAL_TLV_STATUS_PPDU_DONE                1
+#define HAL_TLV_STATUS_BUF_DONE                 2
+#define HAL_TLV_STATUS_PPDU_NON_STD_DONE        3
+
+enum hal_rx_mon_status {
+	HAL_RX_MON_STATUS_PPDU_NOT_DONE,
+	HAL_RX_MON_STATUS_PPDU_DONE,
+	HAL_RX_MON_STATUS_BUF_DONE,
+	HAL_RX_MON_STATUS_BUF_ADDR,
+	HAL_RX_MON_STATUS_MPDU_START,
+	HAL_RX_MON_STATUS_MPDU_END,
+	HAL_RX_MON_STATUS_MSDU_END,
+};
+
+#define HAL_RX_PPDU_START_INFO0_PPDU_ID			GENMASK(15, 0)
+#define HAL_RX_PPDU_START_INFO1_CHAN_NUM		GENMASK(15, 0)
+#define HAL_RX_PPDU_START_INFO1_CHAN_FREQ		GENMASK(31, 16)
+
+struct hal_rx_ppdu_start {
+	__le32 info0;
+	__le32 info1;
+	__le32 ppdu_start_ts_31_0;
+	__le32 ppdu_start_ts_63_32;
+	__le32 rsvd[2];
+} __packed;
+
+#define HAL_RX_PPDU_END_USER_STATS_INFO0_PEER_ID		GENMASK(13, 0)
+#define HAL_RX_PPDU_END_USER_STATS_INFO0_DEVICE_ID		GENMASK(15, 14)
+#define HAL_RX_PPDU_END_USER_STATS_INFO0_MPDU_CNT_FCS_ERR	GENMASK(26, 16)
+
+#define HAL_RX_PPDU_END_USER_STATS_INFO1_MPDU_CNT_FCS_OK	GENMASK(10, 0)
+#define HAL_RX_PPDU_END_USER_STATS_INFO1_FC_VALID		BIT(11)
+#define HAL_RX_PPDU_END_USER_STATS_INFO1_QOS_CTRL_VALID		BIT(12)
+#define HAL_RX_PPDU_END_USER_STATS_INFO1_HT_CTRL_VALID		BIT(13)
+#define HAL_RX_PPDU_END_USER_STATS_INFO1_PKT_TYPE              GENMASK(24, 21)
+
+#define HAL_RX_PPDU_END_USER_STATS_INFO2_AST_INDEX		GENMASK(15, 0)
+#define HAL_RX_PPDU_END_USER_STATS_INFO2_FRAME_CTRL		GENMASK(31, 16)
+
+#define HAL_RX_PPDU_END_USER_STATS_INFO3_QOS_CTRL		GENMASK(31, 16)
+
+#define HAL_RX_PPDU_END_USER_STATS_INFO4_UDP_MSDU_CNT		GENMASK(15, 0)
+#define HAL_RX_PPDU_END_USER_STATS_INFO4_TCP_MSDU_CNT		GENMASK(31, 16)
+
+#define HAL_RX_PPDU_END_USER_STATS_INFO5_OTHER_MSDU_CNT		GENMASK(15, 0)
+#define HAL_RX_PPDU_END_USER_STATS_INFO5_TCP_ACK_MSDU_CNT	GENMASK(31, 16)
+
+#define HAL_RX_PPDU_END_USER_STATS_INFO6_TID_BITMAP		GENMASK(15, 0)
+#define HAL_RX_PPDU_END_USER_STATS_INFO6_TID_EOSP_BITMAP	GENMASK(31, 16)
+
+#define HAL_RX_PPDU_END_USER_STATS_INFO7_MPDU_OK_BYTE_COUNT    GENMASK(24, 0)
+#define HAL_RX_PPDU_END_USER_STATS_INFO8_MPDU_ERR_BYTE_COUNT   GENMASK(24, 0)
+
+struct hal_rx_ppdu_end_user_stats {
+	__le32 rsvd0[2];
+	__le32 info0;
+	__le32 info1;
+	__le32 info2;
+	__le32 info3;
+	__le32 ht_ctrl;
+	__le32 rsvd1[2];
+	__le32 info4;
+	__le32 info5;
+	__le32 usr_resp_ref;
+	__le32 info6;
+	__le32 rsvd3[4];
+	__le32 info7;
+	__le32 rsvd4;
+	__le32 info8;
+	__le32 rsvd5[2];
+	__le32 usr_resp_ref_ext;
+	__le32 rsvd6;
+} __packed;
+
+struct hal_rx_ppdu_end_user_stats_ext {
+	__le32 info0;
+	__le32 info1;
+	__le32 info2;
+	__le32 info3;
+	__le32 info4;
+	__le32 info5;
+	__le32 info6;
+	__le32 rsvd;
+} __packed;
+
+#define HAL_RX_HT_SIG_INFO_INFO0_MCS		GENMASK(6, 0)
+#define HAL_RX_HT_SIG_INFO_INFO0_BW		BIT(7)
+
+#define HAL_RX_HT_SIG_INFO_INFO1_STBC		GENMASK(5, 4)
+#define HAL_RX_HT_SIG_INFO_INFO1_FEC_CODING	BIT(6)
+#define HAL_RX_HT_SIG_INFO_INFO1_GI		BIT(7)
+
+struct hal_rx_ht_sig_info {
+	__le32 info0;
+	__le32 info1;
+} __packed;
+
+#define HAL_RX_LSIG_B_INFO_INFO0_RATE	GENMASK(3, 0)
+#define HAL_RX_LSIG_B_INFO_INFO0_LEN	GENMASK(15, 4)
+
+struct hal_rx_lsig_b_info {
+	__le32 info0;
+} __packed;
+
+#define HAL_RX_LSIG_A_INFO_INFO0_RATE		GENMASK(3, 0)
+#define HAL_RX_LSIG_A_INFO_INFO0_LEN		GENMASK(16, 5)
+#define HAL_RX_LSIG_A_INFO_INFO0_PKT_TYPE	GENMASK(27, 24)
+
+struct hal_rx_lsig_a_info {
+	__le32 info0;
+} __packed;
+
+#define HAL_RX_VHT_SIG_A_INFO_INFO0_BW		GENMASK(1, 0)
+#define HAL_RX_VHT_SIG_A_INFO_INFO0_STBC	BIT(3)
+#define HAL_RX_VHT_SIG_A_INFO_INFO0_GROUP_ID	GENMASK(9, 4)
+#define HAL_RX_VHT_SIG_A_INFO_INFO0_NSTS	GENMASK(21, 10)
+
+#define HAL_RX_VHT_SIG_A_INFO_INFO1_GI_SETTING		GENMASK(1, 0)
+#define HAL_RX_VHT_SIG_A_INFO_INFO1_SU_MU_CODING	BIT(2)
+#define HAL_RX_VHT_SIG_A_INFO_INFO1_MCS			GENMASK(7, 4)
+#define HAL_RX_VHT_SIG_A_INFO_INFO1_BEAMFORMED		BIT(8)
+
+struct hal_rx_vht_sig_a_info {
+	__le32 info0;
+	__le32 info1;
+} __packed;
+
+enum hal_rx_vht_sig_a_gi_setting {
+	HAL_RX_VHT_SIG_A_NORMAL_GI = 0,
+	HAL_RX_VHT_SIG_A_SHORT_GI = 1,
+	HAL_RX_VHT_SIG_A_SHORT_GI_AMBIGUITY = 3,
+};
+
+#define HE_GI_0_8 0
+#define HE_GI_0_4 1
+#define HE_GI_1_6 2
+#define HE_GI_3_2 3
+
+#define HE_LTF_1_X 0
+#define HE_LTF_2_X 1
+#define HE_LTF_4_X 2
+
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_MCS	GENMASK(6, 3)
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_DCM		BIT(7)
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_TRANSMIT_BW	GENMASK(20, 19)
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_CP_LTF_SIZE	GENMASK(22, 21)
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_NSTS		GENMASK(25, 23)
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_BSS_COLOR		GENMASK(13, 8)
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_SPATIAL_REUSE	GENMASK(18, 15)
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_FORMAT_IND	BIT(0)
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_BEAM_CHANGE	BIT(1)
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO0_DL_UL_FLAG	BIT(2)
+
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO1_TXOP_DURATION	GENMASK(6, 0)
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO1_CODING		BIT(7)
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO1_LDPC_EXTRA	BIT(8)
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO1_STBC		BIT(9)
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO1_TXBF		BIT(10)
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO1_PKT_EXT_FACTOR	GENMASK(12, 11)
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO1_PKT_EXT_PE_DISAM	BIT(13)
+#define HAL_RX_HE_SIG_A_SU_INFO_INFO1_DOPPLER_IND	BIT(15)
+
+struct hal_rx_he_sig_a_su_info {
+	__le32 info0;
+	__le32 info1;
+} __packed;
+
+#define HAL_RX_HE_SIG_A_MU_DL_INFO0_UL_FLAG		BIT(1)
+#define HAL_RX_HE_SIG_A_MU_DL_INFO0_MCS_OF_SIGB		GENMASK(3, 1)
+#define HAL_RX_HE_SIG_A_MU_DL_INFO0_DCM_OF_SIGB		BIT(4)
+#define HAL_RX_HE_SIG_A_MU_DL_INFO0_BSS_COLOR		GENMASK(10, 5)
+#define HAL_RX_HE_SIG_A_MU_DL_INFO0_SPATIAL_REUSE	GENMASK(14, 11)
+#define HAL_RX_HE_SIG_A_MU_DL_INFO0_TRANSMIT_BW		GENMASK(17, 15)
+#define HAL_RX_HE_SIG_A_MU_DL_INFO0_NUM_SIGB_SYMB	GENMASK(21, 18)
+#define HAL_RX_HE_SIG_A_MU_DL_INFO0_COMP_MODE_SIGB	BIT(22)
+#define HAL_RX_HE_SIG_A_MU_DL_INFO0_CP_LTF_SIZE		GENMASK(24, 23)
+#define HAL_RX_HE_SIG_A_MU_DL_INFO0_DOPPLER_INDICATION	BIT(25)
+
+#define HAL_RX_HE_SIG_A_MU_DL_INFO1_TXOP_DURATION	GENMASK(6, 0)
+#define HAL_RX_HE_SIG_A_MU_DL_INFO1_NUM_LTF_SYMB	GENMASK(10, 8)
+#define HAL_RX_HE_SIG_A_MU_DL_INFO1_LDPC_EXTRA		BIT(11)
+#define HAL_RX_HE_SIG_A_MU_DL_INFO1_STBC		BIT(12)
+#define HAL_RX_HE_SIG_A_MU_DL_INFO1_PKT_EXT_FACTOR	GENMASK(14, 13)
+#define HAL_RX_HE_SIG_A_MU_DL_INFO1_PKT_EXT_PE_DISAM	BIT(15)
+
+struct hal_rx_he_sig_a_mu_dl_info {
+	__le32 info0;
+	__le32 info1;
+} __packed;
+
+#define HAL_RX_HE_SIG_B1_MU_INFO_INFO0_RU_ALLOCATION	GENMASK(7, 0)
+
+struct hal_rx_he_sig_b1_mu_info {
+	__le32 info0;
+} __packed;
+
+#define HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_ID           GENMASK(10, 0)
+#define HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_MCS		GENMASK(18, 15)
+#define HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_CODING	BIT(20)
+#define HAL_RX_HE_SIG_B2_MU_INFO_INFO0_STA_NSTS		GENMASK(31, 29)
+
+struct hal_rx_he_sig_b2_mu_info {
+	__le32 info0;
+} __packed;
+
+#define HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_ID	GENMASK(10, 0)
+#define HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_NSTS	GENMASK(13, 11)
+#define HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_TXBF	BIT(14)
+#define HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_MCS	GENMASK(18, 15)
+#define HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_DCM	BIT(19)
+#define HAL_RX_HE_SIG_B2_OFDMA_INFO_INFO0_STA_CODING	BIT(20)
+
+struct hal_rx_he_sig_b2_ofdma_info {
+	__le32 info0;
+} __packed;
+
+enum hal_rx_ul_reception_type {
+	HAL_RECEPTION_TYPE_ULOFMDA,
+	HAL_RECEPTION_TYPE_ULMIMO,
+	HAL_RECEPTION_TYPE_OTHER,
+	HAL_RECEPTION_TYPE_FRAMELESS
+};
+
+#define HAL_RX_RSSI_LEGACY_INFO_INFO0_RECEPTION		GENMASK(3, 0)
+#define HAL_RX_RSSI_LEGACY_INFO_INFO0_RX_BW		GENMASK(7, 5)
+#define HAL_RX_RSSI_LEGACY_INFO_INFO1_RSSI_COMB		GENMASK(15, 8)
+#define HAL_RX_RSSI_LEGACY_INFO_INFO2_RSSI_COMB_PPDU	GENMASK(7, 0)
+
+struct hal_rx_phyrx_rssi_legacy_info {
+	__le32 info0;
+	__le32 rsvd0[39];
+	__le32 info1;
+	__le32 info2;
+} __packed;
+
+#define HAL_RX_MPDU_START_INFO0_PPDU_ID			GENMASK(31, 16)
+#define HAL_RX_MPDU_START_INFO1_PEERID			GENMASK(29, 16)
+#define HAL_RX_MPDU_START_INFO1_DEVICE_ID		GENMASK(31, 30)
+#define HAL_RX_MPDU_START_INFO2_MPDU_LEN		GENMASK(13, 0)
+struct hal_rx_mpdu_start {
+	__le32 rsvd0[9];
+	__le32 info0;
+	__le32 info1;
+	__le32 rsvd1[2];
+	__le32 info2;
+	__le32 rsvd2[16];
+} __packed;
+
+struct hal_rx_msdu_end {
+	__le32 info0;
+	__le32 rsvd0[9];
+	__le16 info00;
+	__le16 info01;
+	__le32 rsvd00[8];
+	__le32 info1;
+	__le32 rsvd1[10];
+	__le32 info2;
+	__le32 rsvd2;
+} __packed;
+
+#define HAL_RX_PPDU_END_DURATION	GENMASK(23, 0)
+struct hal_rx_ppdu_end_duration {
+	__le32 rsvd0[9];
+	__le32 info0;
+	__le32 rsvd1[18];
+} __packed;
+
+struct hal_rx_rxpcu_classification_overview {
+	u32 rsvd0;
+} __packed;
+
+#define HAL_RX_NUM_MSDU_DESC 6
+struct hal_rx_msdu_list {
+	struct hal_rx_msdu_desc_info msdu_info[HAL_RX_NUM_MSDU_DESC];
+	u64 paddr[HAL_RX_NUM_MSDU_DESC];
+	u32 sw_cookie[HAL_RX_NUM_MSDU_DESC];
+	u8 rbm[HAL_RX_NUM_MSDU_DESC];
+};
+
+#define HAL_RX_FBM_ACK_INFO0_ADDR1_31_0		GENMASK(31, 0)
+#define HAL_RX_FBM_ACK_INFO1_ADDR1_47_32	GENMASK(15, 0)
+#define HAL_RX_FBM_ACK_INFO1_ADDR2_15_0		GENMASK(31, 16)
+#define HAL_RX_FBM_ACK_INFO2_ADDR2_47_16	GENMASK(31, 0)
+
+struct hal_rx_frame_bitmap_ack {
+	__le32 reserved;
+	__le32 info0;
+	__le32 info1;
+	__le32 info2;
+	__le32 reserved1[10];
+} __packed;
+
+#define HAL_RX_RESP_REQ_INFO0_PPDU_ID		GENMASK(15, 0)
+#define HAL_RX_RESP_REQ_INFO0_RECEPTION_TYPE	BIT(16)
+#define HAL_RX_RESP_REQ_INFO1_DURATION		GENMASK(15, 0)
+#define HAL_RX_RESP_REQ_INFO1_RATE_MCS		GENMASK(24, 21)
+#define HAL_RX_RESP_REQ_INFO1_SGI		GENMASK(26, 25)
+#define HAL_RX_RESP_REQ_INFO1_STBC		BIT(27)
+#define HAL_RX_RESP_REQ_INFO1_LDPC		BIT(28)
+#define HAL_RX_RESP_REQ_INFO1_IS_AMPDU		BIT(29)
+#define HAL_RX_RESP_REQ_INFO2_NUM_USER		GENMASK(6, 0)
+#define HAL_RX_RESP_REQ_INFO3_ADDR1_31_0	GENMASK(31, 0)
+#define HAL_RX_RESP_REQ_INFO4_ADDR1_47_32	GENMASK(15, 0)
+#define HAL_RX_RESP_REQ_INFO4_ADDR1_15_0	GENMASK(31, 16)
+#define HAL_RX_RESP_REQ_INFO5_ADDR1_47_16	GENMASK(31, 0)
+
+struct hal_rx_resp_req_info {
+	__le32 info0;
+	__le32 reserved[1];
+	__le32 info1;
+	__le32 info2;
+	__le32 reserved1[2];
+	__le32 info3;
+	__le32 info4;
+	__le32 info5;
+	__le32 reserved2[5];
+} __packed;
+
+#define REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_0 0xDDBEEF
+#define REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_1 0xADBEEF
+#define REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_2 0xBDBEEF
+#define REO_QUEUE_DESC_MAGIC_DEBUG_PATTERN_3 0xCDBEEF
+
+/* HE Radiotap data1 Mask */
+#define HE_SU_FORMAT_TYPE 0x0000
+#define HE_EXT_SU_FORMAT_TYPE 0x0001
+#define HE_MU_FORMAT_TYPE  0x0002
+#define HE_TRIG_FORMAT_TYPE  0x0003
+#define HE_BEAM_CHANGE_KNOWN 0x0008
+#define HE_DL_UL_KNOWN 0x0010
+#define HE_MCS_KNOWN 0x0020
+#define HE_DCM_KNOWN 0x0040
+#define HE_CODING_KNOWN 0x0080
+#define HE_LDPC_EXTRA_SYMBOL_KNOWN 0x0100
+#define HE_STBC_KNOWN 0x0200
+#define HE_DATA_BW_RU_KNOWN 0x4000
+#define HE_DOPPLER_KNOWN 0x8000
+#define HE_BSS_COLOR_KNOWN 0x0004
+
+/* HE Radiotap data2 Mask */
+#define HE_GI_KNOWN 0x0002
+#define HE_TXBF_KNOWN 0x0010
+#define HE_PE_DISAMBIGUITY_KNOWN 0x0020
+#define HE_TXOP_KNOWN 0x0040
+#define HE_LTF_SYMBOLS_KNOWN 0x0004
+#define HE_PRE_FEC_PADDING_KNOWN 0x0008
+#define HE_MIDABLE_PERIODICITY_KNOWN 0x0080
+
+/* HE radiotap data3 shift values */
+#define HE_BEAM_CHANGE_SHIFT 6
+#define HE_DL_UL_SHIFT 7
+#define HE_TRANSMIT_MCS_SHIFT 8
+#define HE_DCM_SHIFT 12
+#define HE_CODING_SHIFT 13
+#define HE_LDPC_EXTRA_SYMBOL_SHIFT 14
+#define HE_STBC_SHIFT 15
+
+/* HE radiotap data4 shift values */
+#define HE_STA_ID_SHIFT 4
+
+/* HE radiotap data5 */
+#define HE_GI_SHIFT 4
+#define HE_LTF_SIZE_SHIFT 6
+#define HE_LTF_SYM_SHIFT 8
+#define HE_TXBF_SHIFT 14
+#define HE_PE_DISAMBIGUITY_SHIFT 15
+#define HE_PRE_FEC_PAD_SHIFT 12
+
+/* HE radiotap data6 */
+#define HE_DOPPLER_SHIFT 4
+#define HE_TXOP_SHIFT 8
+
+/* HE radiotap HE-MU flags1 */
+#define HE_SIG_B_MCS_KNOWN 0x0010
+#define HE_SIG_B_DCM_KNOWN 0x0040
+#define HE_SIG_B_SYM_NUM_KNOWN 0x8000
+#define HE_RU_0_KNOWN 0x0100
+#define HE_RU_1_KNOWN 0x0200
+#define HE_RU_2_KNOWN 0x0400
+#define HE_RU_3_KNOWN 0x0800
+#define HE_DCM_FLAG_1_SHIFT 5
+#define HE_SPATIAL_REUSE_MU_KNOWN 0x0100
+#define HE_SIG_B_COMPRESSION_FLAG_1_KNOWN 0x4000
+
+/* HE radiotap HE-MU flags2 */
+#define HE_SIG_B_COMPRESSION_FLAG_2_SHIFT 3
+#define HE_BW_KNOWN 0x0004
+#define HE_NUM_SIG_B_SYMBOLS_SHIFT 4
+#define HE_SIG_B_COMPRESSION_FLAG_2_KNOWN 0x0100
+#define HE_NUM_SIG_B_FLAG_2_SHIFT 9
+#define HE_LTF_FLAG_2_SYMBOLS_SHIFT 12
+#define HE_LTF_KNOWN 0x8000
+
+/* HE radiotap per_user_1 */
+#define HE_STA_SPATIAL_SHIFT 11
+#define HE_TXBF_SHIFT 14
+#define HE_RESERVED_SET_TO_1_SHIFT 19
+#define HE_STA_CODING_SHIFT 20
+
+/* HE radiotap per_user_2 */
+#define HE_STA_MCS_SHIFT 4
+#define HE_STA_DCM_SHIFT 5
+
+/* HE radiotap per user known */
+#define HE_USER_FIELD_POSITION_KNOWN 0x01
+#define HE_STA_ID_PER_USER_KNOWN 0x02
+#define HE_STA_NSTS_KNOWN 0x04
+#define HE_STA_TX_BF_KNOWN 0x08
+#define HE_STA_SPATIAL_CONFIG_KNOWN 0x10
+#define HE_STA_MCS_KNOWN 0x20
+#define HE_STA_DCM_KNOWN 0x40
+#define HE_STA_CODING_KNOWN 0x80
+
+#define HAL_RX_MPDU_ERR_FCS			BIT(0)
+#define HAL_RX_MPDU_ERR_DECRYPT			BIT(1)
+#define HAL_RX_MPDU_ERR_TKIP_MIC		BIT(2)
+#define HAL_RX_MPDU_ERR_AMSDU_ERR		BIT(3)
+#define HAL_RX_MPDU_ERR_OVERFLOW		BIT(4)
+#define HAL_RX_MPDU_ERR_MSDU_LEN		BIT(5)
+#define HAL_RX_MPDU_ERR_MPDU_LEN		BIT(6)
+#define HAL_RX_MPDU_ERR_UNENCRYPTED_FRAME	BIT(7)
+
+#define HAL_RX_CMN_USR_INFO0_CP_SETTING			GENMASK(17, 16)
+#define HAL_RX_CMN_USR_INFO0_LTF_SIZE			GENMASK(19, 18)
+
+struct hal_phyrx_common_user_info {
+	__le32 rsvd[2];
+	__le32 info0;
+	__le32 rsvd1;
+} __packed;
+
+#define HAL_RX_EHT_SIG_NDP_CMN_INFO0_SPATIAL_REUSE	GENMASK(3, 0)
+#define HAL_RX_EHT_SIG_NDP_CMN_INFO0_GI_LTF		GENMASK(5, 4)
+#define HAL_RX_EHT_SIG_NDP_CMN_INFO0_NUM_LTF_SYM	GENMASK(8, 6)
+#define HAL_RX_EHT_SIG_NDP_CMN_INFO0_NSS		GENMASK(10, 7)
+#define HAL_RX_EHT_SIG_NDP_CMN_INFO0_BEAMFORMED		BIT(11)
+#define HAL_RX_EHT_SIG_NDP_CMN_INFO0_DISREGARD		GENMASK(13, 12)
+#define HAL_RX_EHT_SIG_NDP_CMN_INFO0_CRC		GENMASK(17, 14)
+
+struct hal_eht_sig_ndp_cmn_eb {
+	__le32 info0;
+} __packed;
+
+#define HAL_RX_EHT_SIG_OVERFLOW_INFO0_SPATIAL_REUSE		GENMASK(3, 0)
+#define HAL_RX_EHT_SIG_OVERFLOW_INFO0_GI_LTF			GENMASK(5, 4)
+#define HAL_RX_EHT_SIG_OVERFLOW_INFO0_NUM_LTF_SYM		GENMASK(8, 6)
+#define HAL_RX_EHT_SIG_OVERFLOW_INFO0_LDPC_EXTA_SYM		BIT(9)
+#define HAL_RX_EHT_SIG_OVERFLOW_INFO0_PRE_FEC_PAD_FACTOR	GENMASK(11, 10)
+#define HAL_RX_EHT_SIG_OVERFLOW_INFO0_DISAMBIGUITY		BIT(12)
+#define HAL_RX_EHT_SIG_OVERFLOW_INFO0_DISREGARD			GENMASK(16, 13)
+
+struct hal_eht_sig_usig_overflow {
+	__le32 info0;
+} __packed;
+
+#define HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_STA_ID	GENMASK(10, 0)
+#define HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_MCS	GENMASK(14, 11)
+#define HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_VALIDATE	BIT(15)
+#define HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_NSS	GENMASK(19, 16)
+#define HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_BEAMFORMED	BIT(20)
+#define HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_CODING	BIT(21)
+#define HAL_RX_EHT_SIG_NON_MUMIMO_USER_INFO0_CRC	GENMASK(25, 22)
+
+struct hal_eht_sig_non_mu_mimo {
+	__le32 info0;
+} __packed;
+
+#define HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_STA_ID		GENMASK(10, 0)
+#define HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_MCS		GENMASK(14, 11)
+#define HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_CODING		BIT(15)
+#define HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_SPATIAL_CODING	GENMASK(22, 16)
+#define HAL_RX_EHT_SIG_MUMIMO_USER_INFO0_CRC		GENMASK(26, 23)
+
+struct hal_eht_sig_mu_mimo {
+	__le32 info0;
+} __packed;
+
+union hal_eht_sig_user_field {
+	struct hal_eht_sig_mu_mimo mu_mimo;
+	struct hal_eht_sig_non_mu_mimo n_mu_mimo;
+};
+
+#define HAL_RX_EHT_SIG_NON_OFDMA_INFO0_SPATIAL_REUSE		GENMASK(3, 0)
+#define HAL_RX_EHT_SIG_NON_OFDMA_INFO0_GI_LTF			GENMASK(5, 4)
+#define HAL_RX_EHT_SIG_NON_OFDMA_INFO0_NUM_LTF_SYM		GENMASK(8, 6)
+#define HAL_RX_EHT_SIG_NON_OFDMA_INFO0_LDPC_EXTA_SYM		BIT(9)
+#define HAL_RX_EHT_SIG_NON_OFDMA_INFO0_PRE_FEC_PAD_FACTOR	GENMASK(11, 10)
+#define HAL_RX_EHT_SIG_NON_OFDMA_INFO0_DISAMBIGUITY		BIT(12)
+#define HAL_RX_EHT_SIG_NON_OFDMA_INFO0_DISREGARD		GENMASK(16, 13)
+#define HAL_RX_EHT_SIG_NON_OFDMA_INFO0_NUM_USERS		GENMASK(19, 17)
+
+struct hal_eht_sig_non_ofdma_cmn_eb {
+	__le32 info0;
+	union hal_eht_sig_user_field user_field;
+} __packed;
+
+#define HAL_RX_EHT_SIG_OFDMA_EB1_SPATIAL_REUSE		GENMASK_ULL(3, 0)
+#define HAL_RX_EHT_SIG_OFDMA_EB1_GI_LTF			GENMASK_ULL(5, 4)
+#define HAL_RX_EHT_SIG_OFDMA_EB1_NUM_LFT_SYM		GENMASK_ULL(8, 6)
+#define HAL_RX_EHT_SIG_OFDMA_EB1_LDPC_EXTRA_SYM		BIT(9)
+#define HAL_RX_EHT_SIG_OFDMA_EB1_PRE_FEC_PAD_FACTOR	GENMASK_ULL(11, 10)
+#define HAL_RX_EHT_SIG_OFDMA_EB1_PRE_DISAMBIGUITY	BIT(12)
+#define HAL_RX_EHT_SIG_OFDMA_EB1_DISREGARD		GENMASK_ULL(16, 13)
+#define HAL_RX_EHT_SIG_OFDMA_EB1_RU_ALLOC_1_1		GENMASK_ULL(25, 17)
+#define HAL_RX_EHT_SIG_OFDMA_EB1_RU_ALLOC_1_2		GENMASK_ULL(34, 26)
+#define HAL_RX_EHT_SIG_OFDMA_EB1_CRC			GENMASK_ULL(30, 27)
+
+struct hal_eht_sig_ofdma_cmn_eb1 {
+	__le64 info0;
+} __packed;
+
+#define HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_1		GENMASK_ULL(8, 0)
+#define HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_2		GENMASK_ULL(17, 9)
+#define HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_3		GENMASK_ULL(26, 18)
+#define HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_4		GENMASK_ULL(35, 27)
+#define HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_5		GENMASK_ULL(44, 36)
+#define HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_6		GENMASK_ULL(53, 45)
+#define HAL_RX_EHT_SIG_OFDMA_EB2_MCS			GNEMASK_ULL(57, 54)
+
+struct hal_eht_sig_ofdma_cmn_eb2 {
+	__le64 info0;
+} __packed;
+
+struct hal_eht_sig_ofdma_cmn_eb {
+	struct hal_eht_sig_ofdma_cmn_eb1 eb1;
+	struct hal_eht_sig_ofdma_cmn_eb2 eb2;
+	union hal_eht_sig_user_field user_field;
+} __packed;
+
+enum hal_eht_bw {
+	HAL_EHT_BW_20,
+	HAL_EHT_BW_40,
+	HAL_EHT_BW_80,
+	HAL_EHT_BW_160,
+	HAL_EHT_BW_320_1,
+	HAL_EHT_BW_320_2,
+};
+
+#define HAL_RX_USIG_CMN_INFO0_PHY_VERSION	GENMASK(2, 0)
+#define HAL_RX_USIG_CMN_INFO0_BW		GENMASK(5, 3)
+#define HAL_RX_USIG_CMN_INFO0_UL_DL		BIT(6)
+#define HAL_RX_USIG_CMN_INFO0_BSS_COLOR		GENMASK(12, 7)
+#define HAL_RX_USIG_CMN_INFO0_TXOP		GENMASK(19, 13)
+#define HAL_RX_USIG_CMN_INFO0_DISREGARD		GENMASK(25, 20)
+#define HAL_RX_USIG_CMN_INFO0_VALIDATE		BIT(26)
+
+struct hal_mon_usig_cmn {
+	__le32 info0;
+} __packed;
+
+#define HAL_RX_USIG_TB_INFO0_PPDU_TYPE_COMP_MODE	GENMASK(1, 0)
+#define HAL_RX_USIG_TB_INFO0_VALIDATE			BIT(2)
+#define HAL_RX_USIG_TB_INFO0_SPATIAL_REUSE_1		GENMASK(6, 3)
+#define HAL_RX_USIG_TB_INFO0_SPATIAL_REUSE_2		GENMASK(10, 7)
+#define HAL_RX_USIG_TB_INFO0_DISREGARD_1		GENMASK(15, 11)
+#define HAL_RX_USIG_TB_INFO0_CRC			GENMASK(19, 16)
+#define HAL_RX_USIG_TB_INFO0_TAIL			GENMASK(25, 20)
+#define HAL_RX_USIG_TB_INFO0_RX_INTEG_CHECK_PASS	BIT(31)
+
+struct hal_mon_usig_tb {
+	__le32 info0;
+} __packed;
+
+#define HAL_RX_USIG_MU_INFO0_PPDU_TYPE_COMP_MODE	GENMASK(1, 0)
+#define HAL_RX_USIG_MU_INFO0_VALIDATE_1			BIT(2)
+#define HAL_RX_USIG_MU_INFO0_PUNC_CH_INFO		GENMASK(7, 3)
+#define HAL_RX_USIG_MU_INFO0_VALIDATE_2			BIT(8)
+#define HAL_RX_USIG_MU_INFO0_EHT_SIG_MCS		GENMASK(10, 9)
+#define HAL_RX_USIG_MU_INFO0_NUM_EHT_SIG_SYM		GENMASK(15, 11)
+#define HAL_RX_USIG_MU_INFO0_CRC			GENMASK(20, 16)
+#define HAL_RX_USIG_MU_INFO0_TAIL			GENMASK(26, 21)
+#define HAL_RX_USIG_MU_INFO0_RX_INTEG_CHECK_PASS	BIT(31)
+
+struct hal_mon_usig_mu {
+	__le32 info0;
+} __packed;
+
+union hal_mon_usig_non_cmn {
+	struct hal_mon_usig_tb tb;
+	struct hal_mon_usig_mu mu;
+};
+
+struct hal_mon_usig_hdr {
+	struct hal_mon_usig_cmn cmn;
+	union hal_mon_usig_non_cmn non_cmn;
+} __packed;
+
+#define HAL_RX_USR_INFO0_PHY_PPDU_ID		GENMASK(15, 0)
+#define HAL_RX_USR_INFO0_USR_RSSI		GENMASK(23, 16)
+#define HAL_RX_USR_INFO0_PKT_TYPE		GENMASK(27, 24)
+#define HAL_RX_USR_INFO0_STBC			BIT(28)
+#define HAL_RX_USR_INFO0_RECEPTION_TYPE		GENMASK(31, 29)
+
+#define HAL_RX_USR_INFO1_MCS			GENMASK(3, 0)
+#define HAL_RX_USR_INFO1_SGI			GENMASK(5, 4)
+#define HAL_RX_USR_INFO1_HE_RANGING_NDP		BIT(6)
+#define HAL_RX_USR_INFO1_MIMO_SS_BITMAP		GENMASK(15, 8)
+#define HAL_RX_USR_INFO1_RX_BW			GENMASK(18, 16)
+#define HAL_RX_USR_INFO1_DL_OFMDA_USR_IDX	GENMASK(31, 24)
+
+#define HAL_RX_USR_INFO2_DL_OFDMA_CONTENT_CHAN	BIT(0)
+#define HAL_RX_USR_INFO2_NSS			GENMASK(10, 8)
+#define HAL_RX_USR_INFO2_STREAM_OFFSET		GENMASK(13, 11)
+#define HAL_RX_USR_INFO2_STA_DCM		BIT(14)
+#define HAL_RX_USR_INFO2_LDPC			BIT(15)
+#define HAL_RX_USR_INFO2_RU_TYPE_80_0		GENMASK(19, 16)
+#define HAL_RX_USR_INFO2_RU_TYPE_80_1		GENMASK(23, 20)
+#define HAL_RX_USR_INFO2_RU_TYPE_80_2		GENMASK(27, 24)
+#define HAL_RX_USR_INFO2_RU_TYPE_80_3		GENMASK(31, 28)
+
+#define HAL_RX_USR_INFO3_RU_START_IDX_80_0	GENMASK(5, 0)
+#define HAL_RX_USR_INFO3_RU_START_IDX_80_1	GENMASK(13, 8)
+#define HAL_RX_USR_INFO3_RU_START_IDX_80_2	GENMASK(21, 16)
+#define HAL_RX_USR_INFO3_RU_START_IDX_80_3	GENMASK(29, 24)
+
+struct hal_receive_user_info {
+	__le32 info0;
+	__le32 info1;
+	__le32 info2;
+	__le32 info3;
+	__le32 user_fd_rssi_seg0;
+	__le32 user_fd_rssi_seg1;
+	__le32 user_fd_rssi_seg2;
+	__le32 user_fd_rssi_seg3;
+} __packed;
+
+enum hal_mon_reception_type {
+	HAL_RECEPTION_TYPE_SU,
+	HAL_RECEPTION_TYPE_DL_MU_MIMO,
+	HAL_RECEPTION_TYPE_DL_MU_OFMA,
+	HAL_RECEPTION_TYPE_DL_MU_OFDMA_MIMO,
+	HAL_RECEPTION_TYPE_UL_MU_MIMO,
+	HAL_RECEPTION_TYPE_UL_MU_OFDMA,
+	HAL_RECEPTION_TYPE_UL_MU_OFDMA_MIMO,
+};
+
+/* Different allowed RU in 11BE */
+#define HAL_EHT_RU_26		0ULL
+#define HAL_EHT_RU_52		1ULL
+#define HAL_EHT_RU_78		2ULL
+#define HAL_EHT_RU_106		3ULL
+#define HAL_EHT_RU_132		4ULL
+#define HAL_EHT_RU_242		5ULL
+#define HAL_EHT_RU_484		6ULL
+#define HAL_EHT_RU_726		7ULL
+#define HAL_EHT_RU_996		8ULL
+#define HAL_EHT_RU_996x2	9ULL
+#define HAL_EHT_RU_996x3	10ULL
+#define HAL_EHT_RU_996x4	11ULL
+#define HAL_EHT_RU_NONE		15ULL
+#define HAL_EHT_RU_INVALID	31ULL
+/* MRUs spanning above 80Mhz
+ * HAL_EHT_RU_996_484 = HAL_EHT_RU_484 + HAL_EHT_RU_996 + 4 (reserved)
+ */
+#define HAL_EHT_RU_996_484	18ULL
+#define HAL_EHT_RU_996x2_484	28ULL
+#define HAL_EHT_RU_996x3_484	40ULL
+#define HAL_EHT_RU_996_484_242	23ULL
+
+#define NUM_RU_BITS_PER80	16
+#define NUM_RU_BITS_PER20	4
+
+/* Different per_80Mhz band in 320Mhz bandwidth */
+#define HAL_80_0	0
+#define HAL_80_1	1
+#define HAL_80_2	2
+#define HAL_80_3	3
+
+#define HAL_RU_80MHZ(num_band)		((num_band) * NUM_RU_BITS_PER80)
+#define HAL_RU_20MHZ(idx_per_80)	((idx_per_80) * NUM_RU_BITS_PER20)
+
+#define HAL_RU_SHIFT(num_band, idx_per_80)	\
+		(HAL_RU_80MHZ(num_band) + HAL_RU_20MHZ(idx_per_80))
+
+#define HAL_RU(ru, num_band, idx_per_80)	\
+		((u64)(ru) << HAL_RU_SHIFT(num_band, idx_per_80))
+
+/* MRU-996+484 */
+#define HAL_EHT_RU_996_484_0	(HAL_RU(HAL_EHT_RU_484, HAL_80_0, 1) |	\
+				 HAL_RU(HAL_EHT_RU_996, HAL_80_1, 0))
+#define HAL_EHT_RU_996_484_1	(HAL_RU(HAL_EHT_RU_484, HAL_80_0, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996, HAL_80_1, 0))
+#define HAL_EHT_RU_996_484_2	(HAL_RU(HAL_EHT_RU_996, HAL_80_0, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_1, 1))
+#define HAL_EHT_RU_996_484_3	(HAL_RU(HAL_EHT_RU_996, HAL_80_0, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_1, 0))
+#define HAL_EHT_RU_996_484_4	(HAL_RU(HAL_EHT_RU_484, HAL_80_2, 1) |	\
+				 HAL_RU(HAL_EHT_RU_996, HAL_80_3, 0))
+#define HAL_EHT_RU_996_484_5	(HAL_RU(HAL_EHT_RU_484, HAL_80_2, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996, HAL_80_3, 0))
+#define HAL_EHT_RU_996_484_6	(HAL_RU(HAL_EHT_RU_996, HAL_80_2, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_3, 1))
+#define HAL_EHT_RU_996_484_7	(HAL_RU(HAL_EHT_RU_996, HAL_80_2, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_3, 0))
+
+/* MRU-996x2+484 */
+#define HAL_EHT_RU_996x2_484_0	(HAL_RU(HAL_EHT_RU_484, HAL_80_0, 1) |	\
+				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_2, 0))
+#define HAL_EHT_RU_996x2_484_1	(HAL_RU(HAL_EHT_RU_484, HAL_80_0, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_2, 0))
+#define HAL_EHT_RU_996x2_484_2	(HAL_RU(HAL_EHT_RU_996x2, HAL_80_0, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_1, 1) |	\
+				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_2, 0))
+#define HAL_EHT_RU_996x2_484_3	(HAL_RU(HAL_EHT_RU_996x2, HAL_80_0, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_2, 0))
+#define HAL_EHT_RU_996x2_484_4	(HAL_RU(HAL_EHT_RU_996x2, HAL_80_0, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_2, 1))
+#define HAL_EHT_RU_996x2_484_5	(HAL_RU(HAL_EHT_RU_996x2, HAL_80_0, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_2, 0))
+#define HAL_EHT_RU_996x2_484_6	(HAL_RU(HAL_EHT_RU_484, HAL_80_1, 1) |	\
+				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_2, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_3, 0))
+#define HAL_EHT_RU_996x2_484_7	(HAL_RU(HAL_EHT_RU_484, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_2, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_3, 0))
+#define HAL_EHT_RU_996x2_484_8	(HAL_RU(HAL_EHT_RU_996x2, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_2, 1) |	\
+				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_3, 0))
+#define HAL_EHT_RU_996x2_484_9	(HAL_RU(HAL_EHT_RU_996x2, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_2, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_3, 0))
+#define HAL_EHT_RU_996x2_484_10	(HAL_RU(HAL_EHT_RU_996x2, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_2, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_3, 1))
+#define HAL_EHT_RU_996x2_484_11	(HAL_RU(HAL_EHT_RU_996x2, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x2, HAL_80_2, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_3, 0))
+
+/* MRU-996x3+484 */
+#define HAL_EHT_RU_996x3_484_0	(HAL_RU(HAL_EHT_RU_484, HAL_80_0, 1) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_2, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_3, 0))
+#define HAL_EHT_RU_996x3_484_1	(HAL_RU(HAL_EHT_RU_484, HAL_80_0, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_2, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_3, 0))
+#define HAL_EHT_RU_996x3_484_2	(HAL_RU(HAL_EHT_RU_996x3, HAL_80_0, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_1, 1) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_2, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_3, 0))
+#define HAL_EHT_RU_996x3_484_3	(HAL_RU(HAL_EHT_RU_996x3, HAL_80_0, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_2, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_3, 0))
+#define HAL_EHT_RU_996x3_484_4	(HAL_RU(HAL_EHT_RU_996x3, HAL_80_0, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_2, 1) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_3, 0))
+#define HAL_EHT_RU_996x3_484_5	(HAL_RU(HAL_EHT_RU_996x3, HAL_80_0, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_2, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_3, 0))
+#define HAL_EHT_RU_996x3_484_6	(HAL_RU(HAL_EHT_RU_996x3, HAL_80_0, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_2, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_3, 1))
+#define HAL_EHT_RU_996x3_484_7	(HAL_RU(HAL_EHT_RU_996x3, HAL_80_0, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_1, 0) |	\
+				 HAL_RU(HAL_EHT_RU_996x3, HAL_80_2, 0) |	\
+				 HAL_RU(HAL_EHT_RU_484, HAL_80_3, 0))
+
+#define HAL_RU_PER80(ru_per80, num_80mhz, ru_idx_per80mhz) \
+			(HAL_RU(ru_per80, num_80mhz, ru_idx_per80mhz))
+
+void ath12k_wifi7_hal_reo_status_queue_stats(struct ath12k_base *ab,
+					     struct hal_tlv_64_hdr *tlv,
+					     struct hal_reo_status *status);
+void ath12k_wifi7_hal_reo_flush_queue_status(struct ath12k_base *ab,
+					     struct hal_tlv_64_hdr *tlv,
+					     struct hal_reo_status *status);
+void ath12k_wifi7_hal_reo_flush_cache_status(struct ath12k_base *ab,
+					     struct hal_tlv_64_hdr *tlv,
+					     struct hal_reo_status *status);
+void ath12k_wifi7_hal_reo_unblk_cache_status(struct ath12k_base *ab,
+					     struct hal_tlv_64_hdr *tlv,
+					     struct hal_reo_status *status);
+void ath12k_wifi7_hal_reo_flush_timeout_list_status(struct ath12k_base *ab,
+						    struct hal_tlv_64_hdr *tlv,
+						    struct hal_reo_status *status);
+void ath12k_wifi7_hal_reo_desc_thresh_reached_status(struct ath12k_base *ab,
+						     struct hal_tlv_64_hdr *tlv,
+						     struct hal_reo_status *status);
+void ath12k_wifi7_hal_reo_update_rx_reo_queue_status(struct ath12k_base *ab,
+						     struct hal_tlv_64_hdr *tlv,
+						     struct hal_reo_status *status);
+void ath12k_wifi7_hal_rx_msdu_link_info_get(struct hal_rx_msdu_link *link, u32 *num_msdus,
+					    u32 *msdu_cookies,
+					    enum hal_rx_buf_return_buf_manager *rbm);
+void ath12k_wifi7_hal_rx_msdu_link_desc_set(struct ath12k_base *ab,
+					    struct hal_wbm_release_ring *desc,
+					    struct ath12k_buffer_addr *buf_addr_info,
+					    enum hal_wbm_rel_bm_act action);
+void ath12k_wifi7_hal_rx_buf_addr_info_set(struct ath12k_buffer_addr *binfo,
+					   dma_addr_t paddr, u32 cookie, u8 manager);
+void ath12k_wifi7_hal_rx_buf_addr_info_get(struct ath12k_buffer_addr *binfo,
+					   dma_addr_t *paddr,
+					   u32 *cookie, u8 *rbm);
+int ath12k_wifi7_hal_desc_reo_parse_err(struct ath12k_dp *dp,
+					struct hal_reo_dest_ring *desc,
+					dma_addr_t *paddr, u32 *desc_bank);
+int ath12k_wifi7_hal_wbm_desc_parse_err(struct ath12k_dp *dp, void *desc,
+					struct hal_rx_wbm_rel_info *rel_info);
+void ath12k_wifi7_hal_rx_reo_ent_paddr_get(struct ath12k_buffer_addr *buff_addr,
+					   dma_addr_t *paddr, u32 *cookie);
+void ath12k_wifi7_hal_rx_reo_ent_buf_paddr_get(void *rx_desc, dma_addr_t *paddr,
+					       u32 *sw_cookie,
+					       struct ath12k_buffer_addr **pp_buf_addr,
+					       u8 *rbm, u32 *msdu_cnt);
+void ath12k_wifi7_hal_rx_msdu_list_get(struct ath12k *ar,
+				       void *link_desc,
+				       void *msdu_list_opaque,
+				       u16 *num_msdus);
+void ath12k_wifi7_hal_reo_init_cmd_ring(struct ath12k_base *ab,
+					struct hal_srng *srng);
+void ath12k_wifi7_hal_reo_shared_qaddr_cache_clear(struct ath12k_base *ab);
+void ath12k_wifi7_hal_reo_hw_setup(struct ath12k_base *ab, u32 ring_hash_map);
+void ath12k_wifi7_hal_reo_qdesc_setup(struct hal_rx_reo_queue *qdesc,
+				      int tid, u32 ba_window_size,
+				      u32 start_seq, enum hal_pn_type type);
+
+#endif
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h
new file mode 100644
index 000000000000..cc5e1d336376
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h
@@ -0,0 +1,1491 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+#ifndef ATH12K_RX_DESC_H
+#define ATH12K_RX_DESC_H
+
+enum rx_desc_decrypt_status_code {
+	RX_DESC_DECRYPT_STATUS_CODE_OK,
+	RX_DESC_DECRYPT_STATUS_CODE_UNPROTECTED_FRAME,
+	RX_DESC_DECRYPT_STATUS_CODE_DATA_ERR,
+	RX_DESC_DECRYPT_STATUS_CODE_KEY_INVALID,
+	RX_DESC_DECRYPT_STATUS_CODE_PEER_ENTRY_INVALID,
+	RX_DESC_DECRYPT_STATUS_CODE_OTHER,
+};
+
+#define RX_MPDU_START_INFO0_REO_DEST_IND		GENMASK(4, 0)
+#define RX_MPDU_START_INFO0_LMAC_PEER_ID_MSB		GENMASK(6, 5)
+#define RX_MPDU_START_INFO0_FLOW_ID_TOEPLITZ		BIT(7)
+#define RX_MPDU_START_INFO0_PKT_SEL_FP_UCAST_DATA	BIT(8)
+#define RX_MPDU_START_INFO0_PKT_SEL_FP_MCAST_DATA	BIT(9)
+#define RX_MPDU_START_INFO0_PKT_SEL_FP_CTRL_BAR		BIT(10)
+#define RX_MPDU_START_INFO0_RXDMA0_SRC_RING_SEL		GENMASK(13, 11)
+#define RX_MPDU_START_INFO0_RXDMA0_DST_RING_SEL		GENMASK(16, 14)
+#define RX_MPDU_START_INFO0_MCAST_ECHO_DROP_EN		BIT(17)
+#define RX_MPDU_START_INFO0_WDS_LEARN_DETECT_EN		BIT(18)
+#define RX_MPDU_START_INFO0_INTRA_BSS_CHECK_EN		BIT(19)
+#define RX_MPDU_START_INFO0_USE_PPE			BIT(20)
+#define RX_MPDU_START_INFO0_PPE_ROUTING_EN		BIT(21)
+
+#define RX_MPDU_START_INFO1_REO_QUEUE_DESC_HI		GENMASK(7, 0)
+#define RX_MPDU_START_INFO1_RECV_QUEUE_NUM		GENMASK(23, 8)
+#define RX_MPDU_START_INFO1_PRE_DELIM_ERR_WARN		BIT(24)
+#define RX_MPDU_START_INFO1_FIRST_DELIM_ERR		BIT(25)
+
+#define RX_MPDU_START_INFO2_EPD_EN			BIT(0)
+#define RX_MPDU_START_INFO2_ALL_FRAME_ENCPD		BIT(1)
+#define RX_MPDU_START_INFO2_ENC_TYPE			GENMASK(5, 2)
+#define RX_MPDU_START_INFO2_VAR_WEP_KEY_WIDTH		GENMASK(7, 6)
+#define RX_MPDU_START_INFO2_MESH_STA			GENMASK(9, 8)
+#define RX_MPDU_START_INFO2_BSSID_HIT			BIT(10)
+#define RX_MPDU_START_INFO2_BSSID_NUM			GENMASK(14, 11)
+#define RX_MPDU_START_INFO2_TID				GENMASK(18, 15)
+
+#define RX_MPDU_START_INFO3_RXPCU_MPDU_FLTR		GENMASK(1, 0)
+#define RX_MPDU_START_INFO3_SW_FRAME_GRP_ID		GENMASK(8, 2)
+#define RX_MPDU_START_INFO3_NDP_FRAME			BIT(9)
+#define RX_MPDU_START_INFO3_PHY_ERR			BIT(10)
+#define RX_MPDU_START_INFO3_PHY_ERR_MPDU_HDR		BIT(11)
+#define RX_MPDU_START_INFO3_PROTO_VER_ERR		BIT(12)
+#define RX_MPDU_START_INFO3_AST_LOOKUP_VALID		BIT(13)
+#define RX_MPDU_START_INFO3_RANGING			BIT(14)
+
+#define RX_MPDU_START_INFO4_MPDU_FCTRL_VALID		BIT(0)
+#define RX_MPDU_START_INFO4_MPDU_DUR_VALID		BIT(1)
+#define RX_MPDU_START_INFO4_MAC_ADDR1_VALID		BIT(2)
+#define RX_MPDU_START_INFO4_MAC_ADDR2_VALID		BIT(3)
+#define RX_MPDU_START_INFO4_MAC_ADDR3_VALID		BIT(4)
+#define RX_MPDU_START_INFO4_MAC_ADDR4_VALID		BIT(5)
+#define RX_MPDU_START_INFO4_MPDU_SEQ_CTRL_VALID		BIT(6)
+#define RX_MPDU_START_INFO4_MPDU_QOS_CTRL_VALID		BIT(7)
+#define RX_MPDU_START_INFO4_MPDU_HT_CTRL_VALID		BIT(8)
+#define RX_MPDU_START_INFO4_ENCRYPT_INFO_VALID		BIT(9)
+#define RX_MPDU_START_INFO4_MPDU_FRAG_NUMBER		GENMASK(13, 10)
+#define RX_MPDU_START_INFO4_MORE_FRAG_FLAG		BIT(14)
+#define RX_MPDU_START_INFO4_FROM_DS			BIT(16)
+#define RX_MPDU_START_INFO4_TO_DS			BIT(17)
+#define RX_MPDU_START_INFO4_ENCRYPTED			BIT(18)
+#define RX_MPDU_START_INFO4_MPDU_RETRY			BIT(19)
+#define RX_MPDU_START_INFO4_MPDU_SEQ_NUM		GENMASK(31, 20)
+
+#define RX_MPDU_START_INFO5_KEY_ID			GENMASK(7, 0)
+#define RX_MPDU_START_INFO5_NEW_PEER_ENTRY		BIT(8)
+#define RX_MPDU_START_INFO5_DECRYPT_NEEDED		BIT(9)
+#define RX_MPDU_START_INFO5_DECAP_TYPE			GENMASK(11, 10)
+#define RX_MPDU_START_INFO5_VLAN_TAG_C_PADDING		BIT(12)
+#define RX_MPDU_START_INFO5_VLAN_TAG_S_PADDING		BIT(13)
+#define RX_MPDU_START_INFO5_STRIP_VLAN_TAG_C		BIT(14)
+#define RX_MPDU_START_INFO5_STRIP_VLAN_TAG_S		BIT(15)
+#define RX_MPDU_START_INFO5_PRE_DELIM_COUNT		GENMASK(27, 16)
+#define RX_MPDU_START_INFO5_AMPDU_FLAG			BIT(28)
+#define RX_MPDU_START_INFO5_BAR_FRAME			BIT(29)
+#define RX_MPDU_START_INFO5_RAW_MPDU			BIT(30)
+
+#define RX_MPDU_START_INFO6_MPDU_LEN			GENMASK(13, 0)
+#define RX_MPDU_START_INFO6_FIRST_MPDU			BIT(14)
+#define RX_MPDU_START_INFO6_MCAST_BCAST			BIT(15)
+#define RX_MPDU_START_INFO6_AST_IDX_NOT_FOUND		BIT(16)
+#define RX_MPDU_START_INFO6_AST_IDX_TIMEOUT		BIT(17)
+#define RX_MPDU_START_INFO6_POWER_MGMT			BIT(18)
+#define RX_MPDU_START_INFO6_NON_QOS			BIT(19)
+#define RX_MPDU_START_INFO6_NULL_DATA			BIT(20)
+#define RX_MPDU_START_INFO6_MGMT_TYPE			BIT(21)
+#define RX_MPDU_START_INFO6_CTRL_TYPE			BIT(22)
+#define RX_MPDU_START_INFO6_MORE_DATA			BIT(23)
+#define RX_MPDU_START_INFO6_EOSP			BIT(24)
+#define RX_MPDU_START_INFO6_FRAGMENT			BIT(25)
+#define RX_MPDU_START_INFO6_ORDER			BIT(26)
+#define RX_MPDU_START_INFO6_UAPSD_TRIGGER		BIT(27)
+#define RX_MPDU_START_INFO6_ENCRYPT_REQUIRED		BIT(28)
+#define RX_MPDU_START_INFO6_DIRECTED			BIT(29)
+#define RX_MPDU_START_INFO6_AMSDU_PRESENT		BIT(30)
+
+#define RX_MPDU_START_INFO7_VDEV_ID			GENMASK(7, 0)
+#define RX_MPDU_START_INFO7_SERVICE_CODE		GENMASK(16, 8)
+#define RX_MPDU_START_INFO7_PRIORITY_VALID		BIT(17)
+#define RX_MPDU_START_INFO7_SRC_INFO			GENMASK(29, 18)
+
+#define RX_MPDU_START_INFO8_AUTH_TO_SEND_WDS		BIT(0)
+
+struct rx_mpdu_start_qcn9274 {
+	__le32 info0;
+	__le32 reo_queue_desc_lo;
+	__le32 info1;
+	__le32 pn[4];
+	__le32 info2;
+	__le32 peer_meta_data;
+	__le16 info3;
+	__le16 phy_ppdu_id;
+	__le16 ast_index;
+	__le16 sw_peer_id;
+	__le32 info4;
+	__le32 info5;
+	__le32 info6;
+	__le16 frame_ctrl;
+	__le16 duration;
+	u8 addr1[ETH_ALEN];
+	u8 addr2[ETH_ALEN];
+	u8 addr3[ETH_ALEN];
+	__le16 seq_ctrl;
+	u8 addr4[ETH_ALEN];
+	__le16 qos_ctrl;
+	__le32 ht_ctrl;
+	__le32 info7;
+	u8 multi_link_addr1[ETH_ALEN];
+	u8 multi_link_addr2[ETH_ALEN];
+	__le32 info8;
+	__le32 res0;
+	__le32 res1;
+} __packed;
+
+#define QCN9274_MPDU_START_SELECT_MPDU_START_TAG			BIT(0)
+#define QCN9274_MPDU_START_SELECT_INFO0_REO_QUEUE_DESC_LO		BIT(1)
+#define QCN9274_MPDU_START_SELECT_INFO1_PN_31_0				BIT(2)
+#define QCN9274_MPDU_START_SELECT_PN_95_32				BIT(3)
+#define QCN9274_MPDU_START_SELECT_PN_127_96_INFO2			BIT(4)
+#define QCN9274_MPDU_START_SELECT_PEER_MDATA_INFO3_PHY_PPDU_ID		BIT(5)
+#define QCN9274_MPDU_START_SELECT_AST_IDX_SW_PEER_ID_INFO4		BIT(6)
+#define QCN9274_MPDU_START_SELECT_INFO5_INFO6				BIT(7)
+#define QCN9274_MPDU_START_SELECT_FRAME_CTRL_DURATION_ADDR1_31_0	BIT(8)
+#define QCN9274_MPDU_START_SELECT_ADDR2_47_0_ADDR1_47_32		BIT(9)
+#define QCN9274_MPDU_START_SELECT_ADDR3_47_0_SEQ_CTRL			BIT(10)
+#define QCN9274_MPDU_START_SELECT_ADDR4_47_0_QOS_CTRL			BIT(11)
+#define QCN9274_MPDU_START_SELECT_HT_CTRL_INFO7				BIT(12)
+#define QCN9274_MPDU_START_SELECT_ML_ADDR1_47_0_ML_ADDR2_15_0		BIT(13)
+#define QCN9274_MPDU_START_SELECT_ML_ADDR2_47_16_INFO8			BIT(14)
+#define QCN9274_MPDU_START_SELECT_RES_0_RES_1				BIT(15)
+
+#define QCN9274_MPDU_START_WMASK (QCN9274_MPDU_START_SELECT_INFO1_PN_31_0 |	\
+		QCN9274_MPDU_START_SELECT_PN_95_32 |				\
+		QCN9274_MPDU_START_SELECT_PN_127_96_INFO2 |			\
+		QCN9274_MPDU_START_SELECT_PEER_MDATA_INFO3_PHY_PPDU_ID |	\
+		QCN9274_MPDU_START_SELECT_AST_IDX_SW_PEER_ID_INFO4 |		\
+		QCN9274_MPDU_START_SELECT_INFO5_INFO6 |				\
+		QCN9274_MPDU_START_SELECT_FRAME_CTRL_DURATION_ADDR1_31_0 |	\
+		QCN9274_MPDU_START_SELECT_ADDR2_47_0_ADDR1_47_32 |		\
+		QCN9274_MPDU_START_SELECT_ADDR3_47_0_SEQ_CTRL |			\
+		QCN9274_MPDU_START_SELECT_ADDR4_47_0_QOS_CTRL)
+
+/* The below rx_mpdu_start_qcn9274_compact structure is tied with the mask
+ * value QCN9274_MPDU_START_WMASK. If the mask value changes the structure
+ * will also change.
+ */
+
+struct rx_mpdu_start_qcn9274_compact {
+	__le32 info1;
+	__le32 pn[4];
+	__le32 info2;
+	__le32 peer_meta_data;
+	__le16 info3;
+	__le16 phy_ppdu_id;
+	__le16 ast_index;
+	__le16 sw_peer_id;
+	__le32 info4;
+	__le32 info5;
+	__le32 info6;
+	__le16 frame_ctrl;
+	__le16 duration;
+	u8 addr1[ETH_ALEN];
+	u8 addr2[ETH_ALEN];
+	u8 addr3[ETH_ALEN];
+	__le16 seq_ctrl;
+	u8 addr4[ETH_ALEN];
+	__le16 qos_ctrl;
+} __packed;
+
+/* rx_mpdu_start
+ *
+ * reo_destination_indication
+ *		The id of the reo exit ring where the msdu frame shall push
+ *		after (MPDU level) reordering has finished. Values are defined
+ *		in enum %HAL_RX_MSDU_DESC_REO_DEST_IND_.
+ *
+ * lmac_peer_id_msb
+ *
+ *		If use_flow_id_toeplitz_clfy is set and lmac_peer_id_'sb
+ *		is 2'b00, Rx OLE uses a REO destination indicati'n of {1'b1,
+ *		hash[3:0]} using the chosen Toeplitz hash from Common Parser
+ *		if flow search fails.
+ *		If use_flow_id_toeplitz_clfy is set and lmac_peer_id_msb
+ *		's not 2'b00, Rx OLE uses a REO destination indication of
+ *		{lmac_peer_id_msb, hash[2:0]} using the chosen Toeplitz
+ *		hash from Common Parser if flow search fails.
+ *
+ * use_flow_id_toeplitz_clfy
+ *		Indication to Rx OLE to enable REO destination routing based
+ *		on the chosen Toeplitz hash from Common Parser, in case
+ *		flow search fails
+ *
+ * pkt_selection_fp_ucast_data
+ *		Filter pass Unicast data frame (matching rxpcu_filter_pass
+ *		and sw_frame_group_Unicast_data) routing selection
+ *
+ * pkt_selection_fp_mcast_data
+ *		Filter pass Multicast data frame (matching rxpcu_filter_pass
+ *		and sw_frame_group_Multicast_data) routing selection
+ *
+ * pkt_selection_fp_ctrl_bar
+ *		Filter pass BAR frame (matching rxpcu_filter_pass
+ *		and sw_frame_group_ctrl_1000) routing selection
+ *
+ * rxdma0_src_ring_selection
+ *		Field only valid when for the received frame type the corresponding
+ *		pkt_selection_fp_... bit is set
+ *
+ * rxdma0_dst_ring_selection
+ *		Field only valid when for the received frame type the corresponding
+ *		pkt_selection_fp_... bit is set
+ *
+ * mcast_echo_drop_enable
+ *		If set, for multicast packets, multicast echo check (i.e.
+ *		SA search with mcast_echo_check = 1) shall be performed
+ *		by RXOLE, and any multicast echo packets should be indicated
+ *		 to RXDMA for release to WBM
+ *
+ * wds_learning_detect_en
+ *		If set, WDS learning detection based on SA search and notification
+ *		to FW (using RXDMA0 status ring) is enabled and the "timestamp"
+ *		field in address search failure cache-only entry should
+ *		be used to avoid multiple WDS learning notifications.
+ *
+ * intrabss_check_en
+ *		If set, intra-BSS routing detection is enabled
+ *
+ * use_ppe
+ *		Indicates to RXDMA to ignore the REO_destination_indication
+ *		and use a programmed value corresponding to the REO2PPE
+ *		ring
+ *		This override to REO2PPE for packets requiring multiple
+ *		buffers shall be disabled based on an RXDMA configuration,
+ *		as PPE may not support such packets.
+ *
+ *		Supported only in full AP chips, not in client/soft
+ *		chips
+ *
+ * ppe_routing_enable
+ *		Global enable/disable bit for routing to PPE, used to disable
+ *		PPE routing even if RXOLE CCE or flow search indicate 'Use_PPE'
+ *		This is set by SW for peers which are being handled by a
+ *		host SW/accelerator subsystem that also handles packet
+ *		buffer management for WiFi-to-PPE routing.
+ *
+ *		This is cleared by SW for peers which are being handled
+ *		by a different subsystem, completely disabling WiFi-to-PPE
+ *		routing for such peers.
+ *
+ * rx_reo_queue_desc_addr_lo
+ *		Address (lower 32 bits) of the REO queue descriptor.
+ *
+ * rx_reo_queue_desc_addr_hi
+ *		Address (upper 8 bits) of the REO queue descriptor.
+ *
+ * receive_queue_number
+ *		Indicates the MPDU queue ID to which this MPDU link
+ *		descriptor belongs.
+ *
+ * pre_delim_err_warning
+ *		Indicates that a delimiter FCS error was found in between the
+ *		previous MPDU and this MPDU. Note that this is just a warning,
+ *		and does not mean that this MPDU is corrupted in any way. If
+ *		it is, there will be other errors indicated such as FCS or
+ *		decrypt errors.
+ *
+ * first_delim_err
+ *		Indicates that the first delimiter had a FCS failure.
+ *
+ * pn
+ *		The PN number.
+ *
+ * epd_en
+ *		Field only valid when AST_based_lookup_valid == 1.
+ *		In case of ndp or phy_err or AST_based_lookup_valid == 0,
+ *		this field will be set to 0
+ *		If set to one use EPD instead of LPD
+ *		In case of ndp or phy_err, this field will never be set.
+ *
+ * all_frames_shall_be_encrypted
+ *		In case of ndp or phy_err or AST_based_lookup_valid == 0,
+ *		this field will be set to 0
+ *
+ *		When set, all frames (data only ?) shall be encrypted. If
+ *		not, RX CRYPTO shall set an error flag.
+ *
+ *
+ * encrypt_type
+ *		In case of ndp or phy_err or AST_based_lookup_valid == 0,
+ *		this field will be set to 0
+ *
+ *		Indicates type of decrypt cipher used (as defined in the
+ *		peer entry)
+ *
+ * wep_key_width_for_variable_key
+ *
+ *		Field only valid when key_type is set to wep_varied_width.
+ *
+ * mesh_sta
+ *
+ * bssid_hit
+ *		When set, the BSSID of the incoming frame matched one of
+ *		 the 8 BSSID register values
+ * bssid_number
+ *		Field only valid when bssid_hit is set.
+ *		This number indicates which one out of the 8 BSSID register
+ *		values matched the incoming frame
+ *
+ * tid
+ *		Field only valid when mpdu_qos_control_valid is set
+ *		The TID field in the QoS control field
+ *
+ * peer_meta_data
+ *		Meta data that SW has programmed in the Peer table entry
+ *		of the transmitting STA.
+ *
+ * rxpcu_mpdu_filter_in_category
+ *		Field indicates what the reason was that this mpdu frame
+ *		was allowed to come into the receive path by rxpcu. Values
+ *		are defined in enum %RX_DESC_RXPCU_FILTER_*.
+ *
+ * sw_frame_group_id
+ *		SW processes frames based on certain classifications. Values
+ *		are defined in enum %RX_DESC_SW_FRAME_GRP_ID_*.
+ *
+ * ndp_frame
+ *		When set, the received frame was an NDP frame, and thus
+ *		there will be no MPDU data.
+ * phy_err
+ *		When set, a PHY error was received before MAC received any
+ *		data, and thus there will be no MPDU data.
+ *
+ * phy_err_during_mpdu_header
+ *		When set, a PHY error was received before MAC received the
+ *		complete MPDU header which was needed for proper decoding
+ *
+ * protocol_version_err
+ *		Set when RXPCU detected a version error in the Frame control
+ *		field
+ *
+ * ast_based_lookup_valid
+ *		When set, AST based lookup for this frame has found a valid
+ *		result.
+ *
+ * ranging
+ *		When set, a ranging NDPA or a ranging NDP was received.
+ *
+ * phy_ppdu_id
+ *		A ppdu counter value that PHY increments for every PPDU
+ *		received. The counter value wraps around.
+ *
+ * ast_index
+ *
+ *		This field indicates the index of the AST entry corresponding
+ *		to this MPDU. It is provided by the GSE module instantiated
+ *		in RXPCU.
+ *		A value of 0xFFFF indicates an invalid AST index, meaning
+ *		that No AST entry was found or NO AST search was performed
+ *
+ * sw_peer_id
+ *		In case of ndp or phy_err or AST_based_lookup_valid == 0,
+ *		this field will be set to 0
+ *		This field indicates a unique peer identifier. It is set
+ *		equal to field 'sw_peer_id' from the AST entry
+ *
+ * frame_control_valid
+ *		When set, the field Mpdu_Frame_control_field has valid information
+ *
+ * frame_duration_valid
+ *		When set, the field Mpdu_duration_field has valid information
+ *
+ * mac_addr_ad1..4_valid
+ *		When set, the fields mac_addr_adx_..... have valid information
+ *
+ * mpdu_seq_ctrl_valid
+ *
+ *		When set, the fields mpdu_sequence_control_field and mpdu_sequence_number
+ *		have valid information as well as field
+ *		For MPDUs without a sequence control field, this field will
+ *		not be set.
+ *
+ * mpdu_qos_ctrl_valid, mpdu_ht_ctrl_valid
+ *
+ *		When set, the field mpdu_qos_control_field, mpdu_ht_control has valid
+ *		information, For MPDUs without a QoS,HT control field, this field
+ *		will not be set.
+ *
+ * frame_encryption_info_valid
+ *
+ *		When set, the encryption related info fields, like IV and
+ *		PN are valid
+ *		For MPDUs that are not encrypted, this will not be set.
+ *
+ * mpdu_fragment_number
+ *
+ *		Field only valid when Mpdu_sequence_control_valid is set
+ *		AND Fragment_flag is set. The fragment number from the 802.11 header
+ *
+ * more_fragment_flag
+ *
+ *		The More Fragment bit setting from the MPDU header of the
+ *		received frame
+ *
+ * fr_ds
+ *
+ *		Field only valid when Mpdu_frame_control_valid is set
+ *		Set if the from DS bit is set in the frame control.
+ *
+ * to_ds
+ *
+ *		Field only valid when Mpdu_frame_control_valid is set
+ *		Set if the to DS bit is set in the frame control.
+ *
+ * encrypted
+ *
+ *		Field only valid when Mpdu_frame_control_valid is set.
+ *		Protected bit from the frame control.
+ *
+ * mpdu_retry
+ *		Field only valid when Mpdu_frame_control_valid is set.
+ *		Retry bit from the frame control.  Only valid when first_msdu is set
+ *
+ * mpdu_sequence_number
+ *		Field only valid when Mpdu_sequence_control_valid is set.
+ *
+ *		The sequence number from the 802.11 header.
+ * key_id
+ *		The key ID octet from the IV.
+ *		Field only valid when Frame_encryption_info_valid is set
+ *
+ * new_peer_entry
+ *		Set if new RX_PEER_ENTRY TLV follows. If clear, RX_PEER_ENTRY
+ *		doesn't follow so RX DECRYPTION module either uses old peer
+ *		entry or not decrypt.
+ *
+ * decrypt_needed
+ *		When RXPCU sets bit 'ast_index_not_found or ast_index_timeout',
+ *		RXPCU will also ensure that this bit is NOT set. CRYPTO for that
+ *		reason only needs to evaluate this bit and non of the other ones
+ *
+ * decap_type
+ *		Used by the OLE during decapsulation. Values are defined in
+ *		enum %MPDU_START_DECAP_TYPE_*.
+ *
+ * rx_insert_vlan_c_tag_padding
+ * rx_insert_vlan_s_tag_padding
+ *		Insert 4 byte of all zeros as VLAN tag or double VLAN tag if
+ *		the rx payload does not have VLAN.
+ *
+ * strip_vlan_c_tag_decap
+ * strip_vlan_s_tag_decap
+ *		Strip VLAN or double VLAN during decapsulation.
+ *
+ * pre_delim_count
+ *		The number of delimiters before this MPDU. Note that this
+ *		number is cleared at PPDU start. If this MPDU is the first
+ *		received MPDU in the PPDU and this MPDU gets filtered-in,
+ *		this field will indicate the number of delimiters located
+ *		after the last MPDU in the previous PPDU.
+ *
+ *		If this MPDU is located after the first received MPDU in
+ *		an PPDU, this field will indicate the number of delimiters
+ *		located between the previous MPDU and this MPDU.
+ *
+ * ampdu_flag
+ *		Received frame was part of an A-MPDU.
+ *
+ * bar_frame
+ *		Received frame is a BAR frame
+ *
+ * raw_mpdu
+ *		Set when no 802.11 to nwifi/ethernet hdr conversion is done
+ *
+ * mpdu_length
+ *		MPDU length before decapsulation.
+ *
+ * first_mpdu
+ *		Indicates the first MSDU of the PPDU.  If both first_mpdu
+ *		and last_mpdu are set in the MSDU then this is a not an
+ *		A-MPDU frame but a stand alone MPDU.  Interior MPDU in an
+ *		A-MPDU shall have both first_mpdu and last_mpdu bits set to
+ *		0.  The PPDU start status will only be valid when this bit
+ *		is set.
+ *
+ * mcast_bcast
+ *		Multicast / broadcast indicator.  Only set when the MAC
+ *		address 1 bit 0 is set indicating mcast/bcast and the BSSID
+ *		matches one of the 4 BSSID registers. Only set when
+ *		first_msdu is set.
+ *
+ * ast_index_not_found
+ *		Only valid when first_msdu is set. Indicates no AST matching
+ *		entries within the max search count.
+ *
+ * ast_index_timeout
+ *		Only valid when first_msdu is set. Indicates an unsuccessful
+ *		search in the address search table due to timeout.
+ *
+ * power_mgmt
+ *		Power management bit set in the 802.11 header.  Only set
+ *		when first_msdu is set.
+ *
+ * non_qos
+ *		Set if packet is not a non-QoS data frame.  Only set when
+ *		first_msdu is set.
+ *
+ * null_data
+ *		Set if frame type indicates either null data or QoS null
+ *		data format.  Only set when first_msdu is set.
+ *
+ * mgmt_type
+ *		Set if packet is a management packet.  Only set when
+ *		first_msdu is set.
+ *
+ * ctrl_type
+ *		Set if packet is a control packet.  Only set when first_msdu
+ *		is set.
+ *
+ * more_data
+ *		Set if more bit in frame control is set.  Only set when
+ *		first_msdu is set.
+ *
+ * eosp
+ *		Set if the EOSP (end of service period) bit in the QoS
+ *		control field is set.  Only set when first_msdu is set.
+ *
+ *
+ * fragment_flag
+ *		Fragment indication
+ *
+ * order
+ *		Set if the order bit in the frame control is set.  Only
+ *		set when first_msdu is set.
+ *
+ * u_apsd_trigger
+ *		U-APSD trigger frame
+ *
+ * encrypt_required
+ *		Indicates that this data type frame is not encrypted even if
+ *		the policy for this MPDU requires encryption as indicated in
+ *		the peer table key type.
+ *
+ * directed
+ *		MPDU is a directed packet which means that the RA matched
+ *		our STA addresses.  In proxySTA it means that the TA matched
+ *		an entry in our address search table with the corresponding
+ *		'no_ack' bit is the address search entry cleared.
+ * amsdu_present
+ *		AMSDU present
+ *
+ * mpdu_frame_control_field
+ *		Frame control field in header. Only valid when the field is marked valid.
+ *
+ * mpdu_duration_field
+ *		Duration field in header. Only valid when the field is marked valid.
+ *
+ * mac_addr_adx
+ *		MAC addresses in the received frame. Only valid when corresponding
+ *		address valid bit is set
+ *
+ * mpdu_qos_control_field, mpdu_ht_control_field
+ *		QoS/HT control fields from header. Valid only when corresponding fields
+ *		are marked valid
+ *
+ * vdev_id
+ *		Virtual device associated with this peer
+ *		RXOLE uses this to determine intra-BSS routing.
+ *
+ * service_code
+ *		Opaque service code between PPE and Wi-Fi
+ *		This field gets passed on by REO to PPE in the EDMA descriptor
+ *		('REO_TO_PPE_RING').
+ *
+ * priority_valid
+ *		This field gets passed on by REO to PPE in the EDMA descriptor
+ *		('REO_TO_PPE_RING').
+ *
+ * src_info
+ *		Source (virtual) device/interface info. associated with
+ *		this peer
+ *		This field gets passed on by REO to PPE in the EDMA descriptor
+ *		('REO_TO_PPE_RING').
+ *
+ * multi_link_addr_ad1_ad2_valid
+ *		If set, Rx OLE shall convert Address1 and Address2 of received
+ *		data frames to multi-link addresses during decapsulation to eth/nwifi
+ *
+ * multi_link_addr_ad1,ad2
+ *		Multi-link receiver address1,2. Only valid when corresponding
+ *		valid bit is set
+ *
+ * authorize_to_send_wds
+ *		If not set, RXDMA shall perform error-routing for WDS packets
+ *		as the sender is not authorized and might misuse WDS frame
+ *		format to inject packets with arbitrary DA/SA.
+ *
+ */
+
+#define RX_MSDU_END_64_TLV_SRC_LINK_ID		GENMASK(24, 22)
+
+#define RX_MSDU_END_INFO0_RXPCU_MPDU_FITLER	GENMASK(1, 0)
+#define RX_MSDU_END_INFO0_SW_FRAME_GRP_ID	GENMASK(8, 2)
+
+#define RX_MSDU_END_INFO1_REPORTED_MPDU_LENGTH	GENMASK(13, 0)
+
+#define RX_MSDU_END_INFO2_CCE_SUPER_RULE	GENMASK(13, 8)
+#define RX_MSDU_END_INFO2_CCND_TRUNCATE		BIT(14)
+#define RX_MSDU_END_INFO2_CCND_CCE_DIS		BIT(15)
+
+#define RX_MSDU_END_INFO3_DA_OFFSET		GENMASK(5, 0)
+#define RX_MSDU_END_INFO3_SA_OFFSET		GENMASK(11, 6)
+#define RX_MSDU_END_INFO3_DA_OFFSET_VALID	BIT(12)
+#define RX_MSDU_END_INFO3_SA_OFFSET_VALID	BIT(13)
+
+#define RX_MSDU_END_INFO4_TCP_FLAG		GENMASK(8, 0)
+#define RX_MSDU_END_INFO4_LRO_ELIGIBLE		BIT(9)
+
+#define RX_MSDU_END_INFO5_SA_IDX_TIMEOUT	BIT(0)
+#define RX_MSDU_END_INFO5_DA_IDX_TIMEOUT	BIT(1)
+#define RX_MSDU_END_INFO5_TO_DS			BIT(2)
+#define RX_MSDU_END_INFO5_TID			GENMASK(6, 3)
+#define RX_MSDU_END_INFO5_SA_IS_VALID		BIT(7)
+#define RX_MSDU_END_INFO5_DA_IS_VALID		BIT(8)
+#define RX_MSDU_END_INFO5_DA_IS_MCBC		BIT(9)
+#define RX_MSDU_END_INFO5_L3_HDR_PADDING	GENMASK(11, 10)
+#define RX_MSDU_END_INFO5_FIRST_MSDU		BIT(12)
+#define RX_MSDU_END_INFO5_LAST_MSDU		BIT(13)
+#define RX_MSDU_END_INFO5_FROM_DS		BIT(14)
+#define RX_MSDU_END_INFO5_IP_CHKSUM_FAIL_COPY	BIT(15)
+
+#define RX_MSDU_END_INFO6_MSDU_DROP		BIT(0)
+#define RX_MSDU_END_INFO6_REO_DEST_IND		GENMASK(5, 1)
+#define RX_MSDU_END_INFO6_FLOW_IDX		GENMASK(25, 6)
+#define RX_MSDU_END_INFO6_USE_PPE		BIT(26)
+#define RX_MSDU_END_INFO6_MESH_STA		GENMASK(28, 27)
+#define RX_MSDU_END_INFO6_VLAN_CTAG_STRIPPED	BIT(29)
+#define RX_MSDU_END_INFO6_VLAN_STAG_STRIPPED	BIT(30)
+#define RX_MSDU_END_INFO6_FRAGMENT_FLAG		BIT(31)
+
+#define RX_MSDU_END_INFO7_AGGR_COUNT		GENMASK(7, 0)
+#define RX_MSDU_END_INFO7_FLOW_AGGR_CONTN	BIT(8)
+#define RX_MSDU_END_INFO7_FISA_TIMEOUT		BIT(9)
+
+#define RX_MSDU_END_INFO7_TCPUDP_CSUM_FAIL_CPY	BIT(10)
+#define RX_MSDU_END_INFO7_MSDU_LIMIT_ERROR	BIT(11)
+#define RX_MSDU_END_INFO7_FLOW_IDX_TIMEOUT	BIT(12)
+#define RX_MSDU_END_INFO7_FLOW_IDX_INVALID	BIT(13)
+#define RX_MSDU_END_INFO7_CCE_MATCH		BIT(14)
+#define RX_MSDU_END_INFO7_AMSDU_PARSER_ERR	BIT(15)
+
+#define RX_MSDU_END_INFO8_KEY_ID		GENMASK(7, 0)
+
+#define RX_MSDU_END_INFO9_SERVICE_CODE		GENMASK(14, 6)
+#define RX_MSDU_END_INFO9_PRIORITY_VALID	BIT(15)
+#define RX_MSDU_END_INFO9_INRA_BSS		BIT(16)
+#define RX_MSDU_END_INFO9_DEST_CHIP_ID		GENMASK(18, 17)
+#define RX_MSDU_END_INFO9_MCAST_ECHO		BIT(19)
+#define RX_MSDU_END_INFO9_WDS_LEARN_EVENT	BIT(20)
+#define RX_MSDU_END_INFO9_WDS_ROAM_EVENT	BIT(21)
+#define RX_MSDU_END_INFO9_WDS_KEEP_ALIVE_EVENT	BIT(22)
+
+#define RX_MSDU_END_INFO10_MSDU_LENGTH		GENMASK(13, 0)
+#define RX_MSDU_END_INFO10_STBC			BIT(14)
+#define RX_MSDU_END_INFO10_IPSEC_ESP		BIT(15)
+#define RX_MSDU_END_INFO10_L3_OFFSET		GENMASK(22, 16)
+#define RX_MSDU_END_INFO10_IPSEC_AH		BIT(23)
+#define RX_MSDU_END_INFO10_L4_OFFSET		GENMASK(31, 24)
+
+#define RX_MSDU_END_INFO11_MSDU_NUMBER		GENMASK(7, 0)
+#define RX_MSDU_END_INFO11_DECAP_FORMAT		GENMASK(9, 8)
+#define RX_MSDU_END_INFO11_IPV4			BIT(10)
+#define RX_MSDU_END_INFO11_IPV6			BIT(11)
+#define RX_MSDU_END_INFO11_TCP			BIT(12)
+#define RX_MSDU_END_INFO11_UDP			BIT(13)
+#define RX_MSDU_END_INFO11_IP_FRAG		BIT(14)
+#define RX_MSDU_END_INFO11_TCP_ONLY_ACK		BIT(15)
+#define RX_MSDU_END_INFO11_DA_IS_BCAST_MCAST	BIT(16)
+#define RX_MSDU_END_INFO11_SEL_TOEPLITZ_HASH	GENMASK(18, 17)
+#define RX_MSDU_END_INFO11_IP_FIXED_HDR_VALID	BIT(19)
+#define RX_MSDU_END_INFO11_IP_EXTN_HDR_VALID	BIT(20)
+#define RX_MSDU_END_INFO11_IP_TCP_UDP_HDR_VALID	BIT(21)
+#define RX_MSDU_END_INFO11_MESH_CTRL_PRESENT	BIT(22)
+#define RX_MSDU_END_INFO11_LDPC			BIT(23)
+#define RX_MSDU_END_INFO11_IP4_IP6_NXT_HDR	GENMASK(31, 24)
+
+#define RX_MSDU_END_INFO12_USER_RSSI		GENMASK(7, 0)
+#define RX_MSDU_END_INFO12_PKT_TYPE		GENMASK(11, 8)
+#define RX_MSDU_END_INFO12_SGI			GENMASK(13, 12)
+#define RX_MSDU_END_INFO12_RATE_MCS		GENMASK(17, 14)
+#define RX_MSDU_END_INFO12_RECV_BW		GENMASK(20, 18)
+#define RX_MSDU_END_INFO12_RECEPTION_TYPE	GENMASK(23, 21)
+
+#define RX_MSDU_END_INFO12_MIMO_SS_BITMAP	GENMASK(30, 24)
+#define RX_MSDU_END_INFO12_MIMO_DONE_COPY	BIT(31)
+
+#define RX_MSDU_END_INFO13_FIRST_MPDU		BIT(0)
+#define RX_MSDU_END_INFO13_MCAST_BCAST		BIT(2)
+#define RX_MSDU_END_INFO13_AST_IDX_NOT_FOUND	BIT(3)
+#define RX_MSDU_END_INFO13_AST_IDX_TIMEDOUT	BIT(4)
+#define RX_MSDU_END_INFO13_POWER_MGMT		BIT(5)
+#define RX_MSDU_END_INFO13_NON_QOS		BIT(6)
+#define RX_MSDU_END_INFO13_NULL_DATA		BIT(7)
+#define RX_MSDU_END_INFO13_MGMT_TYPE		BIT(8)
+#define RX_MSDU_END_INFO13_CTRL_TYPE		BIT(9)
+#define RX_MSDU_END_INFO13_MORE_DATA		BIT(10)
+#define RX_MSDU_END_INFO13_EOSP			BIT(11)
+#define RX_MSDU_END_INFO13_A_MSDU_ERROR		BIT(12)
+#define RX_MSDU_END_INFO13_ORDER		BIT(14)
+#define RX_MSDU_END_INFO13_OVERFLOW_ERR		BIT(16)
+#define RX_MSDU_END_INFO13_MSDU_LEN_ERR		BIT(17)
+#define RX_MSDU_END_INFO13_TCP_UDP_CKSUM_FAIL	BIT(18)
+#define RX_MSDU_END_INFO13_IP_CKSUM_FAIL	BIT(19)
+#define RX_MSDU_END_INFO13_SA_IDX_INVALID	BIT(20)
+#define RX_MSDU_END_INFO13_DA_IDX_INVALID	BIT(21)
+#define RX_MSDU_END_INFO13_AMSDU_ADDR_MISMATCH	BIT(22)
+#define RX_MSDU_END_INFO13_RX_IN_TX_DECRYPT_BYP	BIT(23)
+#define RX_MSDU_END_INFO13_ENCRYPT_REQUIRED	BIT(24)
+#define RX_MSDU_END_INFO13_DIRECTED		BIT(25)
+#define RX_MSDU_END_INFO13_BUFFER_FRAGMENT	BIT(26)
+#define RX_MSDU_END_INFO13_MPDU_LEN_ERR		BIT(27)
+#define RX_MSDU_END_INFO13_TKIP_MIC_ERR		BIT(28)
+#define RX_MSDU_END_INFO13_DECRYPT_ERR		BIT(29)
+#define RX_MSDU_END_INFO13_UNDECRYPT_FRAME_ERR	BIT(30)
+#define RX_MSDU_END_INFO13_FCS_ERR		BIT(31)
+
+#define RX_MSDU_END_INFO13_WIFI_PARSER_ERR      BIT(15)
+
+#define RX_MSDU_END_INFO14_DECRYPT_STATUS_CODE	GENMASK(12, 10)
+#define RX_MSDU_END_INFO14_RX_BITMAP_NOT_UPDED	BIT(13)
+#define RX_MSDU_END_INFO14_MSDU_DONE		BIT(31)
+
+struct rx_msdu_end_qcn9274 {
+	__le16 info0;
+	__le16 phy_ppdu_id;
+	__le16 ip_hdr_cksum;
+	__le16 info1;
+	__le16 info2;
+	__le16 cumulative_l3_checksum;
+	__le32 rule_indication0;
+	__le32 ipv6_options_crc;
+	__le16 info3;
+	__le16 l3_type;
+	__le32 rule_indication1;
+	__le32 tcp_seq_num;
+	__le32 tcp_ack_num;
+	__le16 info4;
+	__le16 window_size;
+	__le16 sa_sw_peer_id;
+	__le16 info5;
+	__le16 sa_idx;
+	__le16 da_idx_or_sw_peer_id;
+	__le32 info6;
+	__le32 fse_metadata;
+	__le16 cce_metadata;
+	__le16 tcp_udp_cksum;
+	__le16 info7;
+	__le16 cumulative_ip_length;
+	__le32 info8;
+	__le32 info9;
+	__le32 info10;
+	__le32 info11;
+	__le16 vlan_ctag_ci;
+	__le16 vlan_stag_ci;
+	__le32 peer_meta_data;
+	__le32 info12;
+	__le32 flow_id_toeplitz;
+	__le32 ppdu_start_timestamp_63_32;
+	__le32 phy_meta_data;
+	__le32 ppdu_start_timestamp_31_0;
+	__le32 toeplitz_hash_2_or_4;
+	__le16 res0;
+	__le16 sa_15_0;
+	__le32 sa_47_16;
+	__le32 info13;
+	__le32 info14;
+} __packed;
+
+#define QCN9274_MSDU_END_SELECT_MSDU_END_TAG				BIT(0)
+#define QCN9274_MSDU_END_SELECT_INFO0_PHY_PPDUID_IP_HDR_CSUM_INFO1	BIT(1)
+#define QCN9274_MSDU_END_SELECT_INFO2_CUMULATIVE_CSUM_RULE_IND_0	BIT(2)
+#define QCN9274_MSDU_END_SELECT_IPV6_OP_CRC_INFO3_TYPE13		BIT(3)
+#define QCN9274_MSDU_END_SELECT_RULE_IND_1_TCP_SEQ_NUM			BIT(4)
+#define QCN9274_MSDU_END_SELECT_TCP_ACK_NUM_INFO4_WINDOW_SIZE		BIT(5)
+#define QCN9274_MSDU_END_SELECT_SA_SW_PER_ID_INFO5_SA_DA_ID		BIT(6)
+#define QCN9274_MSDU_END_SELECT_INFO6_FSE_METADATA			BIT(7)
+#define QCN9274_MSDU_END_SELECT_CCE_MDATA_TCP_UDP_CSUM_INFO7_IP_LEN	BIT(8)
+#define QCN9274_MSDU_END_SELECT_INFO8_INFO9				BIT(9)
+#define QCN9274_MSDU_END_SELECT_INFO10_INFO11				BIT(10)
+#define QCN9274_MSDU_END_SELECT_VLAN_CTAG_STAG_CI_PEER_MDATA		BIT(11)
+#define QCN9274_MSDU_END_SELECT_INFO12_AND_FLOW_ID_TOEPLITZ		BIT(12)
+#define QCN9274_MSDU_END_SELECT_PPDU_START_TS_63_32_PHY_MDATA		BIT(13)
+#define QCN9274_MSDU_END_SELECT_PPDU_START_TS_31_0_TOEPLITZ_HASH_2_4	BIT(14)
+#define QCN9274_MSDU_END_SELECT_RES0_SA_47_0				BIT(15)
+#define QCN9274_MSDU_END_SELECT_INFO13_INFO14				BIT(16)
+
+#define QCN9274_MSDU_END_WMASK (QCN9274_MSDU_END_SELECT_MSDU_END_TAG |	\
+		QCN9274_MSDU_END_SELECT_SA_SW_PER_ID_INFO5_SA_DA_ID |	\
+		QCN9274_MSDU_END_SELECT_INFO10_INFO11 |			\
+		QCN9274_MSDU_END_SELECT_INFO12_AND_FLOW_ID_TOEPLITZ |	\
+		QCN9274_MSDU_END_SELECT_PPDU_START_TS_63_32_PHY_MDATA |	\
+		QCN9274_MSDU_END_SELECT_INFO13_INFO14)
+
+/* The below rx_msdu_end_qcn9274_compact structure is tied with the mask value
+ * QCN9274_MSDU_END_WMASK. If the mask value changes the structure will also
+ * change.
+ */
+
+struct rx_msdu_end_qcn9274_compact {
+	__le64 msdu_end_tag;
+	__le16 sa_sw_peer_id;
+	__le16 info5;
+	__le16 sa_idx;
+	__le16 da_idx_or_sw_peer_id;
+	__le32 info10;
+	__le32 info11;
+	__le32 info12;
+	__le32 flow_id_toeplitz;
+	__le32 ppdu_start_timestamp_63_32;
+	__le32 phy_meta_data;
+	__le32 info13;
+	__le32 info14;
+} __packed;
+
+/* rx_msdu_end
+ *
+ * rxpcu_mpdu_filter_in_category
+ *		Field indicates what the reason was that this mpdu frame
+ *		was allowed to come into the receive path by rxpcu. Values
+ *		are defined in enum %RX_DESC_RXPCU_FILTER_*.
+ *
+ * sw_frame_group_id
+ *		SW processes frames based on certain classifications. Values
+ *		are defined in enum %RX_DESC_SW_FRAME_GRP_ID_*.
+ *
+ * phy_ppdu_id
+ *		A ppdu counter value that PHY increments for every PPDU
+ *		received. The counter value wraps around.
+ *
+ * ip_hdr_cksum
+ *		This can include the IP header checksum or the pseudo
+ *		header checksum used by TCP/UDP checksum.
+ *
+ * reported_mpdu_length
+ *		MPDU length before decapsulation. Only valid when first_msdu is
+ *		set. This field is taken directly from the length field of the
+ *		A-MPDU delimiter or the preamble length field for non-A-MPDU
+ *		frames.
+ *
+ * cce_super_rule
+ *		Indicates the super filter rule.
+ *
+ * cce_classify_not_done_truncate
+ *		Classification failed due to truncated frame.
+ *
+ * cce_classify_not_done_cce_dis
+ *		Classification failed due to CCE global disable
+ *
+ * cumulative_l3_checksum
+ *		FISA: IP header checksum including the total MSDU length
+ *		that is part of this flow aggregated so far, reported if
+ *		'RXOLE_R0_FISA_CTRL. CHKSUM_CUM_IP_LEN_EN' is set
+ *
+ * rule_indication
+ *		Bitmap indicating which of rules have matched.
+ *
+ * ipv6_options_crc
+ *		32 bit CRC computed out of  IP v6 extension headers.
+ *
+ * da_offset
+ *		Offset into MSDU buffer for DA.
+ *
+ * sa_offset
+ *		Offset into MSDU buffer for SA.
+ *
+ * da_offset_valid
+ *		da_offset field is valid. This will be set to 0 in case
+ *		of a dynamic A-MSDU when DA is compressed.
+ *
+ * sa_offset_valid
+ *		sa_offset field is valid. This will be set to 0 in case
+ *		of a dynamic A-MSDU when SA is compressed.
+ *
+ * l3_type
+ *		The 16-bit type value indicating the type of L3 later
+ *		extracted from LLC/SNAP, set to zero if SNAP is not
+ *		available.
+ *
+ * tcp_seq_number
+ *		TCP sequence number.
+ *
+ * tcp_ack_number
+ *		TCP acknowledge number.
+ *
+ * tcp_flag
+ *		TCP flags {NS, CWR, ECE, URG, ACK, PSH, RST, SYN, FIN}.
+ *
+ * lro_eligible
+ *		Computed out of TCP and IP fields to indicate that this
+ *		MSDU is eligible for LRO.
+ *
+ * window_size
+ *		TCP receive window size.
+ *
+ * sa_sw_peer_id
+ *		sw_peer_id from the address search entry corresponding to the
+ *		source address of the MSDU.
+ *
+ * sa_idx_timeout
+ *		Indicates an unsuccessful MAC source address search due to the
+ *		expiring of the search timer.
+ *
+ * da_idx_timeout
+ *		Indicates an unsuccessful MAC destination address search due to
+ *		the expiring of the search timer.
+ *
+ * to_ds
+ *		Set if the to DS bit is set in the frame control.
+ *
+ * tid
+ *		TID field in the QoS control field
+ *
+ * sa_is_valid
+ *		Indicates that OLE found a valid SA entry.
+ *
+ * da_is_valid
+ *		Indicates that OLE found a valid DA entry.
+ *
+ * da_is_mcbc
+ *		Field Only valid if da_is_valid is set. Indicates the DA address
+ *		was a Multicast of Broadcast address.
+ *
+ * l3_header_padding
+ *		Number of bytes padded  to make sure that the L3 header will
+ *		always start of a Dword boundary.
+ *
+ * first_msdu
+ *		Indicates the first MSDU of A-MSDU. If both first_msdu and
+ *		last_msdu are set in the MSDU then this is a non-aggregated MSDU
+ *		frame: normal MPDU. Interior MSDU in an A-MSDU shall have both
+ *		first_mpdu and last_mpdu bits set to 0.
+ *
+ * last_msdu
+ *		Indicates the last MSDU of the A-MSDU. MPDU end status is only
+ *		valid when last_msdu is set.
+ *
+ * fr_ds
+ *		Set if the from DS bit is set in the frame control.
+ *
+ * ip_chksum_fail_copy
+ *		Indicates that the computed checksum did not match the
+ *		checksum in the IP header.
+ *
+ * sa_idx
+ *		The offset in the address table which matches the MAC source
+ *		address.
+ *
+ * da_idx_or_sw_peer_id
+ *		Based on a register configuration in RXOLE, this field will
+ *		contain:
+ *		The offset in the address table which matches the MAC destination
+ *		address
+ *		OR:
+ *		sw_peer_id from the address search entry corresponding to
+ *		the destination address of the MSDU
+ *
+ * msdu_drop
+ *		REO shall drop this MSDU and not forward it to any other ring.
+ *
+ *		The id of the reo exit ring where the msdu frame shall push
+ *		after (MPDU level) reordering has finished. Values are defined
+ *		in enum %HAL_RX_MSDU_DESC_REO_DEST_IND_.
+ *
+ * flow_idx
+ *		Flow table index.
+ *
+ * use_ppe
+ *		Indicates to RXDMA to ignore the REO_destination_indication
+ *		and use a programmed value corresponding to the REO2PPE
+ *		ring
+ *
+ * mesh_sta
+ *		When set, this is a Mesh (11s) STA.
+ *
+ * vlan_ctag_stripped
+ *		Set by RXOLE if it stripped 4-bytes of C-VLAN Tag from the
+ *		packet
+ *
+ * vlan_stag_stripped
+ *		Set by RXOLE if it stripped 4-bytes of S-VLAN Tag from the
+ *		packet
+ *
+ * fragment_flag
+ *		Indicates that this is an 802.11 fragment frame.  This is
+ *		set when either the more_frag bit is set in the frame control
+ *		or the fragment number is not zero.  Only set when first_msdu
+ *		is set.
+ *
+ * fse_metadata
+ *		FSE related meta data.
+ *
+ * cce_metadata
+ *		CCE related meta data.
+ *
+ * tcp_udp_chksum
+ *		The value of the computed TCP/UDP checksum.  A mode bit
+ *		selects whether this checksum is the full checksum or the
+ *		partial checksum which does not include the pseudo header.
+ *
+ * aggregation_count
+ *		Number of MSDU's aggregated so far
+ *
+ * flow_aggregation_continuation
+ *		To indicate that this MSDU can be aggregated with
+ *		the previous packet with the same flow id
+ *
+ * fisa_timeout
+ *		To indicate that the aggregation has restarted for
+ *		this flow due to timeout
+ *
+ * tcp_udp_chksum_fail
+ *		Indicates that the computed checksum (tcp_udp_chksum) did
+ *		not match the checksum in the TCP/UDP header.
+ *
+ * msdu_limit_error
+ *		Indicates that the MSDU threshold was exceeded and thus all the
+ *		rest of the MSDUs will not be scattered and will not be
+ *		decapsulated but will be DMA'ed in RAW format as a single MSDU.
+ *
+ * flow_idx_timeout
+ *		Indicates an unsuccessful flow search due to the expiring of
+ *		the search timer.
+ *
+ * flow_idx_invalid
+ *		flow id is not valid.
+ *
+ * cce_match
+ *		Indicates that this status has a corresponding MSDU that
+ *		requires FW processing. The OLE will have classification
+ *		ring mask registers which will indicate the ring(s) for
+ *		packets and descriptors which need FW attention.
+ *
+ * amsdu_parser_error
+ *		A-MSDU could not be properly de-agregated.
+ *
+ * cumulative_ip_length
+ *		Total MSDU length that is part of this flow aggregated
+ *		so far
+ *
+ * key_id
+ *		The key ID octet from the IV. Only valid when first_msdu is set.
+ *
+ * service_code
+ *		Opaque service code between PPE and Wi-Fi
+ *
+ * priority_valid
+ *		This field gets passed on by REO to PPE in the EDMA descriptor
+ *
+ * intra_bss
+ *		This packet needs intra-BSS routing by SW as the 'vdev_id'
+ *		for the destination is the same as 'vdev_id' (from 'RX_MPDU_PCU_START')
+ *		that this MSDU was got in.
+ *
+ * dest_chip_id
+ *		If intra_bss is set, copied by RXOLE from 'ADDR_SEARCH_ENTRY'
+ *		to support intra-BSS routing with multi-chip multi-link
+ *		operation. This indicates into which chip's TCL the packet should be
+ *		queueued
+ *
+ * multicast_echo
+ *		If set, this packet is a multicast echo, i.e. the DA is
+ *		multicast and Rx OLE SA search with mcast_echo_check = 1
+ *		passed. RXDMA should release such packets to WBM.
+ *
+ * wds_learning_event
+ *		If set, this packet has an SA search failure with WDS learning
+ *		enabled for the peer. RXOLE should route this TLV to the
+ *		RXDMA0 status ring to notify FW.
+ *
+ * wds_roaming_event
+ *		If set, this packet's SA 'Sw_peer_id' mismatches the 'Sw_peer_id'
+ *		of the peer through which the packet was got, indicating
+ *		the SA node has roamed. RXOLE should route this TLV to
+ *		the RXDMA0 status ring to notify FW.
+ *
+ * wds_keep_alive_event
+ *		If set, the AST timestamp for this packet's SA is older
+ *		than the current timestamp by more than a threshold programmed
+ *		in RXOLE. RXOLE should route this TLV to the RXDMA0 status
+ *		ring to notify FW to keep the AST entry for the SA alive.
+ *
+ * msdu_length
+ *		MSDU length in bytes after decapsulation.
+ *		This field is still valid for MPDU frames without A-MSDU.
+ *		It still represents MSDU length after decapsulation
+ *
+ * stbc
+ *		When set, use STBC transmission rates.
+ *
+ * ipsec_esp
+ *		Set if IPv4/v6 packet is using IPsec ESP.
+ *
+ * l3_offset
+ *		Depending upon mode bit, this field either indicates the
+ *		L3 offset in bytes from the start of the RX_HEADER or the IP
+ *		offset in bytes from the start of the packet after
+ *		decapsulation. The latter is only valid if ipv4_proto or
+ *		ipv6_proto is set.
+ *
+ * ipsec_ah
+ *		Set if IPv4/v6 packet is using IPsec AH
+ *
+ * l4_offset
+ *		Depending upon mode bit, this field either indicates the
+ *		L4 offset in bytes from the start of RX_HEADER (only valid
+ *		if either ipv4_proto or ipv6_proto is set to 1) or indicates
+ *		the offset in bytes to the start of TCP or UDP header from
+ *		the start of the IP header after decapsulation (Only valid if
+ *		tcp_proto or udp_proto is set). The value 0 indicates that
+ *		the offset is longer than 127 bytes.
+ *
+ * msdu_number
+ *		Indicates the MSDU number within a MPDU.  This value is
+ *		reset to zero at the start of each MPDU.  If the number of
+ *		MSDU exceeds 255 this number will wrap using modulo 256.
+ *
+ * decap_type
+ *		Indicates the format after decapsulation. Values are defined in
+ *		enum %MPDU_START_DECAP_TYPE_*.
+ *
+ * ipv4_proto
+ *		Set if L2 layer indicates IPv4 protocol.
+ *
+ * ipv6_proto
+ *		Set if L2 layer indicates IPv6 protocol.
+ *
+ * tcp_proto
+ *		Set if the ipv4_proto or ipv6_proto are set and the IP protocol
+ *		indicates TCP.
+ *
+ * udp_proto
+ *		Set if the ipv4_proto or ipv6_proto are set and the IP protocol
+ *		indicates UDP.
+ *
+ * ip_frag
+ *		Indicates that either the IP More frag bit is set or IP frag
+ *		number is non-zero.  If set indicates that this is a fragmented
+ *		IP packet.
+ *
+ * tcp_only_ack
+ *		Set if only the TCP Ack bit is set in the TCP flags and if
+ *		the TCP payload is 0.
+ *
+ * da_is_bcast_mcast
+ *		The destination address is broadcast or multicast.
+ *
+ * toeplitz_hash
+ *		Actual chosen Hash.
+ *		0 - Toeplitz hash of 2-tuple (IP source address, IP
+ *		    destination address)
+ *		1 - Toeplitz hash of 4-tuple (IP source	address,
+ *		    IP destination address, L4 (TCP/UDP) source port,
+ *		    L4 (TCP/UDP) destination port)
+ *		2 - Toeplitz of flow_id
+ *		3 - Zero is used
+ *
+ * ip_fixed_header_valid
+ *		Fixed 20-byte IPv4 header or 40-byte IPv6 header parsed
+ *		fully within first 256 bytes of the packet
+ *
+ * ip_extn_header_valid
+ *		IPv6/IPv6 header, including IPv4 options and
+ *		recognizable extension headers parsed fully within first 256
+ *		bytes of the packet
+ *
+ * tcp_udp_header_valid
+ *		Fixed 20-byte TCP (excluding TCP options) or 8-byte UDP
+ *		header parsed fully within first 256 bytes of the packet
+ *
+ * mesh_control_present
+ *		When set, this MSDU includes the 'Mesh Control' field
+ *
+ * ldpc
+ *
+ * ip4_protocol_ip6_next_header
+ *		For IPv4, this is the 8 bit protocol field set). For IPv6 this
+ *		is the 8 bit next_header field.
+ *
+ *
+ * vlan_ctag_ci
+ *		2 bytes of C-VLAN Tag Control Information from WHO_L2_LLC
+ *
+ * vlan_stag_ci
+ *		2 bytes of S-VLAN Tag Control Information from WHO_L2_LLC
+ *		in case of double VLAN
+ *
+ * peer_meta_data
+ *		Meta data that SW has programmed in the Peer table entry
+ *		of the transmitting STA.
+ *
+ * user_rssi
+ *		RSSI for this user
+ *
+ * pkt_type
+ *		Values are defined in enum %RX_MSDU_START_PKT_TYPE_*.
+ *
+ * sgi
+ *		Field only valid when pkt type is HT, VHT or HE. Values are
+ *		defined in enum %RX_MSDU_START_SGI_*.
+ *
+ * rate_mcs
+ *		MCS Rate used.
+ *
+ * receive_bandwidth
+ *		Full receive Bandwidth. Values are defined in enum
+ *		%RX_MSDU_START_RECV_*.
+ *
+ * reception_type
+ *		Indicates what type of reception this is and defined in enum
+ *		%RX_MSDU_START_RECEPTION_TYPE_*.
+ *
+ * mimo_ss_bitmap
+ *		Field only valid when
+ *		Reception_type is RX_MSDU_START_RECEPTION_TYPE_DL_MU_MIMO or
+ *		RX_MSDU_START_RECEPTION_TYPE_DL_MU_OFDMA_MIMO.
+ *
+ *		Bitmap, with each bit indicating if the related spatial
+ *		stream is used for this STA
+ *
+ *		LSB related to SS 0
+ *
+ *		0 - spatial stream not used for this reception
+ *		1 - spatial stream used for this reception
+ *
+ * msdu_done_copy
+ *		If set indicates that the RX packet data, RX header data,
+ *		RX PPDU start descriptor, RX MPDU start/end descriptor,
+ *		RX MSDU start/end descriptors and RX Attention descriptor
+ *		are all valid.  This bit is in the last 64-bit of the descriptor
+ *		expected to be subscribed in future hardware.
+ *
+ * flow_id_toeplitz
+ *		Toeplitz hash of 5-tuple
+ *		{IP source address, IP destination address, IP source port, IP
+ *		destination port, L4 protocol}  in case of non-IPSec.
+ *
+ *		In case of IPSec - Toeplitz hash of 4-tuple
+ *		{IP source address, IP destination address, SPI, L4 protocol}
+ *
+ *		The relevant Toeplitz key registers are provided in RxOLE's
+ *		instance of common parser module. These registers are separate
+ *		from the Toeplitz keys used by ASE/FSE modules inside RxOLE.
+ *		The actual value will be passed on from common parser module
+ *		to RxOLE in one of the WHO_* TLVs.
+ *
+ * ppdu_start_timestamp
+ *		Timestamp that indicates when the PPDU that contained this MPDU
+ *		started on the medium.
+ *
+ * phy_meta_data
+ *		SW programmed Meta data provided by the PHY. Can be used for SW
+ *		to indicate the channel the device is on.
+ *
+ * toeplitz_hash_2_or_4
+ *		Controlled by multiple RxOLE registers for TCP/UDP over
+ *		IPv4/IPv6 - Either, Toeplitz hash computed over 2-tuple
+ *		IPv4 or IPv6 src/dest addresses is reported; or, Toeplitz
+ *		hash computed over 4-tuple IPv4 or IPv6 src/dest addresses
+ *		and src/dest ports is reported. The Flow_id_toeplitz hash
+ *		can also be reported here. Usually the hash reported here
+ *		is the one used for hash-based REO routing (see use_flow_id_toeplitz_clfy
+ *		in 'RXPT_CLASSIFY_INFO').
+ *
+ * sa
+ *		Source MAC address
+ *
+ * first_mpdu
+ *		Indicates the first MSDU of the PPDU.  If both first_mpdu
+ *		and last_mpdu are set in the MSDU then this is a not an
+ *		A-MPDU frame but a stand alone MPDU.  Interior MPDU in an
+ *		A-MPDU shall have both first_mpdu and last_mpdu bits set to
+ *		0.  The PPDU start status will only be valid when this bit
+ *		is set.
+ *
+ * mcast_bcast
+ *		Multicast / broadcast indicator.  Only set when the MAC
+ *		address 1 bit 0 is set indicating mcast/bcast and the BSSID
+ *		matches one of the 4 BSSID registers. Only set when
+ *		first_msdu is set.
+ *
+ * ast_index_not_found
+ *		Only valid when first_msdu is set. Indicates no AST matching
+ *		entries within the max search count.
+ *
+ * ast_index_timeout
+ *		Only valid when first_msdu is set. Indicates an unsuccessful
+ *		search in the address search table due to timeout.
+ *
+ * power_mgmt
+ *		Power management bit set in the 802.11 header.  Only set
+ *		when first_msdu is set.
+ *
+ * non_qos
+ *		Set if packet is not a non-QoS data frame.  Only set when
+ *		first_msdu is set.
+ *
+ * null_data
+ *		Set if frame type indicates either null data or QoS null
+ *		data format.  Only set when first_msdu is set.
+ *
+ * mgmt_type
+ *		Set if packet is a management packet.  Only set when
+ *		first_msdu is set.
+ *
+ * ctrl_type
+ *		Set if packet is a control packet.  Only set when first_msdu
+ *		is set.
+ *
+ * more_data
+ *		Set if more bit in frame control is set.  Only set when
+ *		first_msdu is set.
+ *
+ * eosp
+ *		Set if the EOSP (end of service period) bit in the QoS
+ *		control field is set.  Only set when first_msdu is set.
+ *
+ * a_msdu_error
+ *		Set if number of MSDUs in A-MSDU is above a threshold or if the
+ *		size of the MSDU is invalid. This receive buffer will contain
+ *		all of the remainder of MSDUs in this MPDU w/o decapsulation.
+ *
+ * order
+ *		Set if the order bit in the frame control is set.  Only
+ *		set when first_msdu is set.
+ *
+ * wifi_parser_error
+ *		Indicates that the WiFi frame has one of the following errors
+ *
+ * overflow_err
+ *		RXPCU Receive FIFO ran out of space to receive the full MPDU.
+ *		Therefore this MPDU is terminated early and is thus corrupted.
+ *
+ *		This MPDU will not be ACKed.
+ *
+ *		RXPCU might still be able to correctly receive the following
+ *		MPDUs in the PPDU if enough fifo space became available in time.
+ *
+ * mpdu_length_err
+ *		Set by RXPCU if the expected MPDU length does not correspond
+ *		with the actually received number of bytes in the MPDU.
+ *
+ * tcp_udp_chksum_fail
+ *		Indicates that the computed checksum (tcp_udp_chksum) did
+ *		not match the checksum in the TCP/UDP header.
+ *
+ * ip_chksum_fail
+ *		Indicates that the computed checksum did not match the
+ *		checksum in the IP header.
+ *
+ * sa_idx_invalid
+ *		Indicates no matching entry was found in the address search
+ *		table for the source MAC address.
+ *
+ * da_idx_invalid
+ *		Indicates no matching entry was found in the address search
+ *		table for the destination MAC address.
+ *
+ * amsdu_addr_mismatch
+ *		Indicates that an A-MSDU with 'from DS = 0' had an SA mismatching
+ *		TA or an A-MDU with 'to DS = 0' had a DA mismatching RA
+ *
+ * rx_in_tx_decrypt_byp
+ *		Indicates that RX packet is not decrypted as Crypto is busy
+ *		with TX packet processing.
+ *
+ * encrypt_required
+ *		Indicates that this data type frame is not encrypted even if
+ *		the policy for this MPDU requires encryption as indicated in
+ *		the peer table key type.
+ *
+ * directed
+ *		MPDU is a directed packet which means that the RA matched
+ *		our STA addresses.  In proxySTA it means that the TA matched
+ *		an entry in our address search table with the corresponding
+ *		'no_ack' bit is the address search entry cleared.
+ *
+ * buffer_fragment
+ *		Indicates that at least one of the rx buffers has been
+ *		fragmented.  If set the FW should look at the rx_frag_info
+ *		descriptor described below.
+ *
+ * mpdu_length_err
+ *		Indicates that the MPDU was pre-maturely terminated
+ *		resulting in a truncated MPDU.  Don't trust the MPDU length
+ *		field.
+ *
+ * tkip_mic_err
+ *		Indicates that the MPDU Michael integrity check failed
+ *
+ * decrypt_err
+ *		Indicates that the MPDU decrypt integrity check failed
+ *
+ * fcs_err
+ *		Indicates that the MPDU FCS check failed
+ *
+ * flow_idx_timeout
+ *		Indicates an unsuccessful flow search due to the expiring of
+ *		the search timer.
+ *
+ * flow_idx_invalid
+ *		flow id is not valid.
+ *
+ * decrypt_status_code
+ *		Field provides insight into the decryption performed. Values
+ *		are defined in enum %RX_DESC_DECRYPT_STATUS_CODE_*.
+ *
+ * rx_bitmap_not_updated
+ *		Frame is received, but RXPCU could not update the receive bitmap
+ *		due to (temporary) fifo constraints.
+ *
+ * msdu_done
+ *		If set indicates that the RX packet data, RX header data, RX
+ *		PPDU start descriptor, RX MPDU start/end descriptor, RX MSDU
+ *		start/end descriptors and RX Attention descriptor are all
+ *		valid.  This bit must be in the last octet of the
+ *		descriptor.
+ *
+ */
+
+struct hal_rx_desc_qcn9274_compact {
+	struct rx_msdu_end_qcn9274_compact msdu_end;
+	struct rx_mpdu_start_qcn9274_compact mpdu_start;
+	u8 msdu_payload[];
+} __packed;
+
+#define RX_BE_PADDING0_BYTES 8
+#define RX_BE_PADDING1_BYTES 8
+
+#define HAL_RX_BE_PKT_HDR_TLV_LEN		112
+
+struct rx_pkt_hdr_tlv {
+	__le64 tag;
+	__le64 phy_ppdu_id;
+	u8 rx_pkt_hdr[HAL_RX_BE_PKT_HDR_TLV_LEN];
+};
+
+struct hal_rx_desc_wcn7850 {
+	__le64 msdu_end_tag;
+	struct rx_msdu_end_qcn9274 msdu_end;
+	u8 rx_padding0[RX_BE_PADDING0_BYTES];
+	__le64 mpdu_start_tag;
+	struct rx_mpdu_start_qcn9274 mpdu_start;
+	struct rx_pkt_hdr_tlv	 pkt_hdr_tlv;
+	u8 msdu_payload[];
+};
+
+struct hal_rx_desc {
+	union {
+		struct hal_rx_desc_qcn9274_compact qcn9274_compact;
+		struct hal_rx_desc_wcn7850 wcn7850;
+	} u;
+} __packed;
+
+#endif /* ATH12K_RX_DESC_H */
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_tx.c b/drivers/net/wireless/ath/ath12k/wifi7/hal_tx.c
new file mode 100644
index 000000000000..02d3cadf03fe
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_tx.c
@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include "../hal.h"
+#include "hal_tx.h"
+#include "../hif.h"
+#include "hal.h"
+
+#define DSCP_TID_MAP_TBL_ENTRY_SIZE 64
+
+/* dscp_tid_map - Default DSCP-TID mapping
+ *=================
+ * DSCP        TID
+ *=================
+ * 000xxx      0
+ * 001xxx      1
+ * 010xxx      2
+ * 011xxx      3
+ * 100xxx      4
+ * 101xxx      5
+ * 110xxx      6
+ * 111xxx      7
+ */
+static inline u8 dscp2tid(u8 dscp)
+{
+	return dscp >> 3;
+}
+
+void ath12k_wifi7_hal_tx_cmd_desc_setup(struct ath12k_base *ab,
+					struct hal_tcl_data_cmd *tcl_cmd,
+					struct hal_tx_info *ti)
+{
+	tcl_cmd->buf_addr_info.info0 =
+		le32_encode_bits(ti->paddr, BUFFER_ADDR_INFO0_ADDR);
+	tcl_cmd->buf_addr_info.info1 =
+		le32_encode_bits(((uint64_t)ti->paddr >> HAL_ADDR_MSB_REG_SHIFT),
+				 BUFFER_ADDR_INFO1_ADDR);
+	tcl_cmd->buf_addr_info.info1 |=
+		le32_encode_bits((ti->rbm_id), BUFFER_ADDR_INFO1_RET_BUF_MGR) |
+		le32_encode_bits(ti->desc_id, BUFFER_ADDR_INFO1_SW_COOKIE);
+
+	tcl_cmd->info0 =
+		le32_encode_bits(ti->type, HAL_TCL_DATA_CMD_INFO0_DESC_TYPE) |
+		le32_encode_bits(ti->bank_id, HAL_TCL_DATA_CMD_INFO0_BANK_ID);
+
+	tcl_cmd->info1 =
+		le32_encode_bits(ti->meta_data_flags,
+				 HAL_TCL_DATA_CMD_INFO1_CMD_NUM);
+
+	tcl_cmd->info2 = cpu_to_le32(ti->flags0) |
+		le32_encode_bits(ti->data_len, HAL_TCL_DATA_CMD_INFO2_DATA_LEN) |
+		le32_encode_bits(ti->pkt_offset, HAL_TCL_DATA_CMD_INFO2_PKT_OFFSET);
+
+	tcl_cmd->info3 = cpu_to_le32(ti->flags1) |
+		le32_encode_bits(ti->tid, HAL_TCL_DATA_CMD_INFO3_TID) |
+		le32_encode_bits(ti->lmac_id, HAL_TCL_DATA_CMD_INFO3_PMAC_ID) |
+		le32_encode_bits(ti->vdev_id, HAL_TCL_DATA_CMD_INFO3_VDEV_ID);
+
+	tcl_cmd->info4 = le32_encode_bits(ti->bss_ast_idx,
+					  HAL_TCL_DATA_CMD_INFO4_SEARCH_INDEX) |
+			 le32_encode_bits(ti->bss_ast_hash,
+					  HAL_TCL_DATA_CMD_INFO4_CACHE_SET_NUM);
+	tcl_cmd->info5 = 0;
+}
+
+void ath12k_wifi7_hal_tx_set_dscp_tid_map(struct ath12k_base *ab, int id)
+{
+	u32 ctrl_reg_val;
+	u32 addr;
+	u8 hw_map_val[HAL_DSCP_TID_TBL_SIZE], dscp, tid;
+	int i;
+	u32 value;
+
+	ctrl_reg_val = ath12k_hif_read32(ab, HAL_SEQ_WCSS_UMAC_TCL_REG +
+					 HAL_TCL1_RING_CMN_CTRL_REG);
+	/* Enable read/write access */
+	ctrl_reg_val |= HAL_TCL1_RING_CMN_CTRL_DSCP_TID_MAP_PROG_EN;
+	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_TCL_REG +
+			   HAL_TCL1_RING_CMN_CTRL_REG, ctrl_reg_val);
+
+	addr = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_DSCP_TID_MAP +
+	       (4 * id * (HAL_DSCP_TID_TBL_SIZE / 4));
+
+	/* Configure each DSCP-TID mapping in three bits there by configure
+	 * three bytes in an iteration.
+	 */
+	for (i = 0, dscp = 0; i < HAL_DSCP_TID_TBL_SIZE; i += 3) {
+		tid = dscp2tid(dscp);
+		value = u32_encode_bits(tid, HAL_TCL1_RING_FIELD_DSCP_TID_MAP0);
+		dscp++;
+
+		tid = dscp2tid(dscp);
+		value |= u32_encode_bits(tid, HAL_TCL1_RING_FIELD_DSCP_TID_MAP1);
+		dscp++;
+
+		tid = dscp2tid(dscp);
+		value |= u32_encode_bits(tid, HAL_TCL1_RING_FIELD_DSCP_TID_MAP2);
+		dscp++;
+
+		tid = dscp2tid(dscp);
+		value |= u32_encode_bits(tid, HAL_TCL1_RING_FIELD_DSCP_TID_MAP3);
+		dscp++;
+
+		tid = dscp2tid(dscp);
+		value |= u32_encode_bits(tid, HAL_TCL1_RING_FIELD_DSCP_TID_MAP4);
+		dscp++;
+
+		tid = dscp2tid(dscp);
+		value |= u32_encode_bits(tid, HAL_TCL1_RING_FIELD_DSCP_TID_MAP5);
+		dscp++;
+
+		tid = dscp2tid(dscp);
+		value |= u32_encode_bits(tid, HAL_TCL1_RING_FIELD_DSCP_TID_MAP6);
+		dscp++;
+
+		tid = dscp2tid(dscp);
+		value |= u32_encode_bits(tid, HAL_TCL1_RING_FIELD_DSCP_TID_MAP7);
+		dscp++;
+
+		memcpy(&hw_map_val[i], &value, 3);
+	}
+
+	for (i = 0; i < HAL_DSCP_TID_TBL_SIZE; i += 4) {
+		ath12k_hif_write32(ab, addr, *(u32 *)&hw_map_val[i]);
+		addr += 4;
+	}
+
+	/* Disable read/write access */
+	ctrl_reg_val = ath12k_hif_read32(ab, HAL_SEQ_WCSS_UMAC_TCL_REG +
+					 HAL_TCL1_RING_CMN_CTRL_REG);
+	ctrl_reg_val &= ~HAL_TCL1_RING_CMN_CTRL_DSCP_TID_MAP_PROG_EN;
+	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_TCL_REG +
+			   HAL_TCL1_RING_CMN_CTRL_REG,
+			   ctrl_reg_val);
+}
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_tx.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_tx.h
new file mode 100644
index 000000000000..9d2b1552c2f5
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_tx.h
@@ -0,0 +1,194 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef ATH12K_HAL_TX_H
+#define ATH12K_HAL_TX_H
+
+#include "../mac.h"
+#include "hal_desc.h"
+
+/* TODO: check all these data can be managed with struct ath12k_tx_desc_info for perf */
+struct hal_tx_info {
+	u16 meta_data_flags; /* %HAL_TCL_DATA_CMD_INFO0_META_ */
+	u8 ring_id;
+	u8 rbm_id;
+	u32 desc_id;
+	enum hal_tcl_desc_type type;
+	enum hal_tcl_encap_type encap_type;
+	dma_addr_t paddr;
+	u32 data_len;
+	u32 pkt_offset;
+	enum hal_encrypt_type encrypt_type;
+	u32 flags0; /* %HAL_TCL_DATA_CMD_INFO1_ */
+	u32 flags1; /* %HAL_TCL_DATA_CMD_INFO2_ */
+	u16 addr_search_flags; /* %HAL_TCL_DATA_CMD_INFO0_ADDR(X/Y)_ */
+	u16 bss_ast_hash;
+	u16 bss_ast_idx;
+	u8 tid;
+	u8 search_type; /* %HAL_TX_ADDR_SEARCH_ */
+	u8 lmac_id;
+	u8 vdev_id;
+	u8 dscp_tid_tbl_idx;
+	bool enable_mesh;
+	int bank_id;
+};
+
+/* TODO: Check if the actual desc macros can be used instead */
+#define HAL_TX_STATUS_FLAGS_FIRST_MSDU		BIT(0)
+#define HAL_TX_STATUS_FLAGS_LAST_MSDU		BIT(1)
+#define HAL_TX_STATUS_FLAGS_MSDU_IN_AMSDU	BIT(2)
+#define HAL_TX_STATUS_FLAGS_RATE_STATS_VALID	BIT(3)
+#define HAL_TX_STATUS_FLAGS_RATE_LDPC		BIT(4)
+#define HAL_TX_STATUS_FLAGS_RATE_STBC		BIT(5)
+#define HAL_TX_STATUS_FLAGS_OFDMA		BIT(6)
+
+#define HAL_TX_STATUS_DESC_LEN		sizeof(struct hal_wbm_release_ring)
+
+/* Tx status parsed from srng desc */
+struct hal_tx_status {
+	enum hal_wbm_rel_src_module buf_rel_source;
+	enum hal_wbm_tqm_rel_reason status;
+	s8 ack_rssi;
+	u32 flags; /* %HAL_TX_STATUS_FLAGS_ */
+	u32 ppdu_id;
+	u8 try_cnt;
+	u8 tid;
+	u16 peer_id;
+	enum hal_tx_rate_stats_pkt_type pkt_type;
+	enum hal_tx_rate_stats_sgi sgi;
+	enum ath12k_supported_bw bw;
+	u8 mcs;
+	u16 tones;
+	u8 ofdma;
+};
+
+#define HAL_TX_PHY_DESC_INFO0_BF_TYPE		GENMASK(17, 16)
+#define HAL_TX_PHY_DESC_INFO0_PREAMBLE_11B	BIT(20)
+#define HAL_TX_PHY_DESC_INFO0_PKT_TYPE		GENMASK(24, 21)
+#define HAL_TX_PHY_DESC_INFO0_BANDWIDTH		GENMASK(30, 28)
+#define HAL_TX_PHY_DESC_INFO1_MCS		GENMASK(3, 0)
+#define HAL_TX_PHY_DESC_INFO1_STBC		BIT(6)
+#define HAL_TX_PHY_DESC_INFO2_NSS		GENMASK(23, 21)
+#define HAL_TX_PHY_DESC_INFO3_AP_PKT_BW		GENMASK(6, 4)
+#define HAL_TX_PHY_DESC_INFO3_LTF_SIZE		GENMASK(20, 19)
+#define HAL_TX_PHY_DESC_INFO3_ACTIVE_CHANNEL	GENMASK(17, 15)
+
+struct hal_tx_phy_desc {
+	__le32 info0;
+	__le32 info1;
+	__le32 info2;
+	__le32 info3;
+} __packed;
+
+#define HAL_TX_FES_STAT_PROT_INFO0_STRT_FRM_TS_15_0	GENMASK(15, 0)
+#define HAL_TX_FES_STAT_PROT_INFO0_STRT_FRM_TS_31_16	GENMASK(31, 16)
+#define HAL_TX_FES_STAT_PROT_INFO1_END_FRM_TS_15_0	GENMASK(15, 0)
+#define HAL_TX_FES_STAT_PROT_INFO1_END_FRM_TS_31_16	GENMASK(31, 16)
+
+struct hal_tx_fes_status_prot {
+	__le64 reserved;
+	__le32 info0;
+	__le32 info1;
+	__le32 reserved1[11];
+} __packed;
+
+#define HAL_TX_FES_STAT_USR_PPDU_INFO0_DURATION		GENMASK(15, 0)
+
+struct hal_tx_fes_status_user_ppdu {
+	__le64 reserved;
+	__le32 info0;
+	__le32 reserved1[3];
+} __packed;
+
+#define HAL_TX_FES_STAT_STRT_INFO0_PROT_TS_LOWER_32	GENMASK(31, 0)
+#define HAL_TX_FES_STAT_STRT_INFO1_PROT_TS_UPPER_32	GENMASK(31, 0)
+
+struct hal_tx_fes_status_start_prot {
+	__le32 info0;
+	__le32 info1;
+	__le64 reserved;
+} __packed;
+
+#define HAL_TX_FES_STATUS_START_INFO0_MEDIUM_PROT_TYPE	GENMASK(29, 27)
+
+struct hal_tx_fes_status_start {
+	__le32 reserved;
+	__le32 info0;
+	__le64 reserved1;
+} __packed;
+
+#define HAL_TX_Q_EXT_INFO0_FRAME_CTRL		GENMASK(15, 0)
+#define HAL_TX_Q_EXT_INFO0_QOS_CTRL		GENMASK(31, 16)
+#define HAL_TX_Q_EXT_INFO1_AMPDU_FLAG		BIT(0)
+
+struct hal_tx_queue_exten {
+	__le32 info0;
+	__le32 info1;
+} __packed;
+
+#define HAL_TX_FES_SETUP_INFO0_NUM_OF_USERS	GENMASK(28, 23)
+
+struct hal_tx_fes_setup {
+	__le32 schedule_id;
+	__le32 info0;
+	__le64 reserved;
+} __packed;
+
+#define HAL_TX_PPDU_SETUP_INFO0_MEDIUM_PROT_TYPE	GENMASK(2, 0)
+#define HAL_TX_PPDU_SETUP_INFO1_PROT_FRAME_ADDR1_31_0	GENMASK(31, 0)
+#define HAL_TX_PPDU_SETUP_INFO2_PROT_FRAME_ADDR1_47_32	GENMASK(15, 0)
+#define HAL_TX_PPDU_SETUP_INFO2_PROT_FRAME_ADDR2_15_0	GENMASK(31, 16)
+#define HAL_TX_PPDU_SETUP_INFO3_PROT_FRAME_ADDR2_47_16	GENMASK(31, 0)
+#define HAL_TX_PPDU_SETUP_INFO4_PROT_FRAME_ADDR3_31_0	GENMASK(31, 0)
+#define HAL_TX_PPDU_SETUP_INFO5_PROT_FRAME_ADDR3_47_32	GENMASK(15, 0)
+#define HAL_TX_PPDU_SETUP_INFO5_PROT_FRAME_ADDR4_15_0	GENMASK(31, 16)
+#define HAL_TX_PPDU_SETUP_INFO6_PROT_FRAME_ADDR4_47_16	GENMASK(31, 0)
+
+struct hal_tx_pcu_ppdu_setup_init {
+	__le32 info0;
+	__le32 info1;
+	__le32 info2;
+	__le32 info3;
+	__le32 reserved;
+	__le32 info4;
+	__le32 info5;
+	__le32 info6;
+} __packed;
+
+#define HAL_TX_FES_STATUS_END_INFO0_START_TIMESTAMP_15_0	GENMASK(15, 0)
+#define HAL_TX_FES_STATUS_END_INFO0_START_TIMESTAMP_31_16	GENMASK(31, 16)
+
+struct hal_tx_fes_status_end {
+	__le32 reserved[2];
+	__le32 info0;
+	__le32 reserved1[19];
+} __packed;
+
+#define HAL_TX_BANK_CONFIG_EPD			BIT(0)
+#define HAL_TX_BANK_CONFIG_ENCAP_TYPE		GENMASK(2, 1)
+#define HAL_TX_BANK_CONFIG_ENCRYPT_TYPE		GENMASK(6, 3)
+#define HAL_TX_BANK_CONFIG_SRC_BUFFER_SWAP	BIT(7)
+#define HAL_TX_BANK_CONFIG_LINK_META_SWAP	BIT(8)
+#define HAL_TX_BANK_CONFIG_INDEX_LOOKUP_EN	BIT(9)
+#define HAL_TX_BANK_CONFIG_ADDRX_EN		BIT(10)
+#define HAL_TX_BANK_CONFIG_ADDRY_EN		BIT(11)
+#define HAL_TX_BANK_CONFIG_MESH_EN		GENMASK(13, 12)
+#define HAL_TX_BANK_CONFIG_VDEV_ID_CHECK_EN	BIT(14)
+#define HAL_TX_BANK_CONFIG_PMAC_ID		GENMASK(16, 15)
+/* STA mode will have MCAST_PKT_CTRL instead of DSCP_TID_MAP bitfield */
+#define HAL_TX_BANK_CONFIG_DSCP_TIP_MAP_ID	GENMASK(22, 17)
+
+void ath12k_wifi7_hal_tx_set_dscp_tid_map(struct ath12k_base *ab, int id);
+void ath12k_wifi7_hal_tx_cmd_desc_setup(struct ath12k_base *ab,
+					struct hal_tcl_data_cmd *tcl_cmd,
+					struct hal_tx_info *ti);
+int ath12k_wifi7_hal_reo_cmd_send(struct ath12k_base *ab, struct hal_srng *srng,
+				  enum hal_reo_cmd_type type,
+				  struct ath12k_hal_reo_cmd *cmd);
+void ath12k_wifi7_hal_tx_configure_bank_register(struct ath12k_base *ab,
+						 u32 bank_config,
+						 u8 bank_id);
+#endif
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.c b/drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.c
new file mode 100644
index 000000000000..7108cc41536d
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.c
@@ -0,0 +1,805 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include "hal_desc.h"
+#include "hal_wcn7850.h"
+#include "hw.h"
+#include "hal.h"
+#include "hal_tx.h"
+
+static const struct hal_srng_config hw_srng_config_template[] = {
+	/* TODO: max_rings can populated by querying HW capabilities */
+	[HAL_REO_DST] = {
+		.start_ring_id = HAL_SRNG_RING_ID_REO2SW1,
+		.max_rings = 8,
+		.entry_size = sizeof(struct hal_reo_dest_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_DST,
+		.max_size = HAL_REO_REO2SW1_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_REO_EXCEPTION] = {
+		/* Designating REO2SW0 ring as exception ring.
+		 * Any of theREO2SW rings can be used as exception ring.
+		 */
+		.start_ring_id = HAL_SRNG_RING_ID_REO2SW0,
+		.max_rings = 1,
+		.entry_size = sizeof(struct hal_reo_dest_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_DST,
+		.max_size = HAL_REO_REO2SW0_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_REO_REINJECT] = {
+		.start_ring_id = HAL_SRNG_RING_ID_SW2REO,
+		.max_rings = 4,
+		.entry_size = sizeof(struct hal_reo_entrance_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_REO_SW2REO_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_REO_CMD] = {
+		.start_ring_id = HAL_SRNG_RING_ID_REO_CMD,
+		.max_rings = 1,
+		.entry_size = (sizeof(struct hal_tlv_64_hdr) +
+			sizeof(struct hal_reo_get_queue_stats)) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_REO_CMD_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_REO_STATUS] = {
+		.start_ring_id = HAL_SRNG_RING_ID_REO_STATUS,
+		.max_rings = 1,
+		.entry_size = (sizeof(struct hal_tlv_64_hdr) +
+			sizeof(struct hal_reo_get_queue_stats_status)) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_DST,
+		.max_size = HAL_REO_STATUS_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_TCL_DATA] = {
+		.start_ring_id = HAL_SRNG_RING_ID_SW2TCL1,
+		.max_rings = 6,
+		.entry_size = sizeof(struct hal_tcl_data_cmd) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_TCL_CMD] = {
+		.start_ring_id = HAL_SRNG_RING_ID_SW2TCL_CMD,
+		.max_rings = 1,
+		.entry_size = sizeof(struct hal_tcl_gse_cmd) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_SW2TCL1_CMD_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_TCL_STATUS] = {
+		.start_ring_id = HAL_SRNG_RING_ID_TCL_STATUS,
+		.max_rings = 1,
+		.entry_size = (sizeof(struct hal_tlv_hdr) +
+			     sizeof(struct hal_tcl_status_ring)) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_DST,
+		.max_size = HAL_TCL_STATUS_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_CE_SRC] = {
+		.start_ring_id = HAL_SRNG_RING_ID_CE0_SRC,
+		.max_rings = 16,
+		.entry_size = sizeof(struct hal_ce_srng_src_desc) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_CE_SRC_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_CE_DST] = {
+		.start_ring_id = HAL_SRNG_RING_ID_CE0_DST,
+		.max_rings = 16,
+		.entry_size = sizeof(struct hal_ce_srng_dest_desc) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_CE_DST_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_CE_DST_STATUS] = {
+		.start_ring_id = HAL_SRNG_RING_ID_CE0_DST_STATUS,
+		.max_rings = 16,
+		.entry_size = sizeof(struct hal_ce_srng_dst_status_desc) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_DST,
+		.max_size = HAL_CE_DST_STATUS_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_WBM_IDLE_LINK] = {
+		.start_ring_id = HAL_SRNG_RING_ID_WBM_IDLE_LINK,
+		.max_rings = 1,
+		.entry_size = sizeof(struct hal_wbm_link_desc) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_WBM_IDLE_LINK_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_SW2WBM_RELEASE] = {
+		.start_ring_id = HAL_SRNG_RING_ID_WBM_SW0_RELEASE,
+		.max_rings = 2,
+		.entry_size = sizeof(struct hal_wbm_release_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_SW2WBM_RELEASE_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_WBM2SW_RELEASE] = {
+		.start_ring_id = HAL_SRNG_RING_ID_WBM2SW0_RELEASE,
+		.max_rings = 8,
+		.entry_size = sizeof(struct hal_wbm_release_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_UMAC,
+		.ring_dir = HAL_SRNG_DIR_DST,
+		.max_size = HAL_WBM2SW_RELEASE_RING_BASE_MSB_RING_SIZE,
+	},
+	[HAL_RXDMA_BUF] = {
+		.start_ring_id = HAL_SRNG_SW2RXDMA_BUF0,
+		.max_rings = 1,
+		.entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_DMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
+	},
+	[HAL_RXDMA_DST] = {
+		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_RXDMA2SW0,
+		.max_rings = 0,
+		.entry_size = 0,
+		.mac_type = ATH12K_HAL_SRNG_PMAC,
+		.ring_dir = HAL_SRNG_DIR_DST,
+		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
+	},
+	[HAL_RXDMA_MONITOR_BUF] = {},
+	[HAL_RXDMA_MONITOR_STATUS] = {
+		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA1_STATBUF,
+		.max_rings = 1,
+		.entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,
+		.mac_type = ATH12K_HAL_SRNG_PMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
+	},
+	[HAL_RXDMA_MONITOR_DESC] = { 0, },
+	[HAL_RXDMA_DIR_BUF] = {
+		.start_ring_id = HAL_SRNG_RING_ID_RXDMA_DIR_BUF,
+		.max_rings = 2,
+		.entry_size = 8 >> 2, /* TODO: Define the struct */
+		.mac_type = ATH12K_HAL_SRNG_PMAC,
+		.ring_dir = HAL_SRNG_DIR_SRC,
+		.max_size = HAL_RXDMA_RING_MAX_SIZE_BE,
+	},
+	[HAL_PPE2TCL] = {},
+	[HAL_PPE_RELEASE] = {},
+	[HAL_TX_MONITOR_BUF] = {},
+	[HAL_RXDMA_MONITOR_DST] = {},
+	[HAL_TX_MONITOR_DST] = {}
+};
+
+const struct ath12k_hw_regs wcn7850_regs = {
+	/* SW2TCL(x) R0 ring configuration address */
+	.tcl1_ring_id = 0x00000908,
+	.tcl1_ring_misc = 0x00000910,
+	.tcl1_ring_tp_addr_lsb = 0x0000091c,
+	.tcl1_ring_tp_addr_msb = 0x00000920,
+	.tcl1_ring_consumer_int_setup_ix0 = 0x00000930,
+	.tcl1_ring_consumer_int_setup_ix1 = 0x00000934,
+	.tcl1_ring_msi1_base_lsb = 0x00000948,
+	.tcl1_ring_msi1_base_msb = 0x0000094c,
+	.tcl1_ring_msi1_data = 0x00000950,
+	.tcl_ring_base_lsb = 0x00000b58,
+	.tcl1_ring_base_lsb = 0x00000900,
+	.tcl1_ring_base_msb = 0x00000904,
+	.tcl2_ring_base_lsb = 0x00000978,
+
+	/* TCL STATUS ring address */
+	.tcl_status_ring_base_lsb = 0x00000d38,
+
+	.wbm_idle_ring_base_lsb = 0x00000d3c,
+	.wbm_idle_ring_misc_addr = 0x00000d4c,
+	.wbm_r0_idle_list_cntl_addr = 0x00000240,
+	.wbm_r0_idle_list_size_addr = 0x00000244,
+	.wbm_scattered_ring_base_lsb = 0x00000250,
+	.wbm_scattered_ring_base_msb = 0x00000254,
+	.wbm_scattered_desc_head_info_ix0 = 0x00000260,
+	.wbm_scattered_desc_head_info_ix1 = 0x00000264,
+	.wbm_scattered_desc_tail_info_ix0 = 0x00000270,
+	.wbm_scattered_desc_tail_info_ix1 = 0x00000274,
+	.wbm_scattered_desc_ptr_hp_addr = 0x00000027c,
+
+	.wbm_sw_release_ring_base_lsb = 0x0000037c,
+	.wbm_sw1_release_ring_base_lsb = 0x00000284,
+	.wbm0_release_ring_base_lsb = 0x00000e08,
+	.wbm1_release_ring_base_lsb = 0x00000e80,
+
+	/* PCIe base address */
+	.pcie_qserdes_sysclk_en_sel = 0x01e0e0a8,
+	.pcie_pcs_osc_dtct_config_base = 0x01e0f45c,
+
+	/* PPE release ring address */
+	.ppe_rel_ring_base = 0x0000043c,
+
+	/* REO DEST ring address */
+	.reo2_ring_base = 0x0000055c,
+	.reo1_misc_ctrl_addr = 0x00000b7c,
+	.reo1_sw_cookie_cfg0 = 0x00000050,
+	.reo1_sw_cookie_cfg1 = 0x00000054,
+	.reo1_qdesc_lut_base0 = 0x00000058,
+	.reo1_qdesc_lut_base1 = 0x0000005c,
+	.reo1_ring_base_lsb = 0x000004e4,
+	.reo1_ring_base_msb = 0x000004e8,
+	.reo1_ring_id = 0x000004ec,
+	.reo1_ring_misc = 0x000004f4,
+	.reo1_ring_hp_addr_lsb = 0x000004f8,
+	.reo1_ring_hp_addr_msb = 0x000004fc,
+	.reo1_ring_producer_int_setup = 0x00000508,
+	.reo1_ring_msi1_base_lsb = 0x0000052C,
+	.reo1_ring_msi1_base_msb = 0x00000530,
+	.reo1_ring_msi1_data = 0x00000534,
+	.reo1_aging_thres_ix0 = 0x00000b08,
+	.reo1_aging_thres_ix1 = 0x00000b0c,
+	.reo1_aging_thres_ix2 = 0x00000b10,
+	.reo1_aging_thres_ix3 = 0x00000b14,
+
+	/* REO Exception ring address */
+	.reo2_sw0_ring_base = 0x000008a4,
+
+	/* REO Reinject ring address */
+	.sw2reo_ring_base = 0x00000304,
+	.sw2reo1_ring_base = 0x0000037c,
+
+	/* REO cmd ring address */
+	.reo_cmd_ring_base = 0x0000028c,
+
+	/* REO status ring address */
+	.reo_status_ring_base = 0x00000a84,
+
+	/* CE base address */
+	.umac_ce0_src_reg_base = 0x01b80000,
+	.umac_ce0_dest_reg_base = 0x01b81000,
+	.umac_ce1_src_reg_base = 0x01b82000,
+	.umac_ce1_dest_reg_base = 0x01b83000,
+
+	.gcc_gcc_pcie_hot_rst = 0x1e40304,
+};
+
+static inline
+bool ath12k_hal_rx_desc_get_first_msdu_wcn7850(struct hal_rx_desc *desc)
+{
+	return !!le16_get_bits(desc->u.wcn7850.msdu_end.info5,
+			       RX_MSDU_END_INFO5_FIRST_MSDU);
+}
+
+static inline
+bool ath12k_hal_rx_desc_get_last_msdu_wcn7850(struct hal_rx_desc *desc)
+{
+	return !!le16_get_bits(desc->u.wcn7850.msdu_end.info5,
+			       RX_MSDU_END_INFO5_LAST_MSDU);
+}
+
+u8 ath12k_hal_rx_desc_get_l3_pad_bytes_wcn7850(struct hal_rx_desc *desc)
+{
+	return le16_get_bits(desc->u.wcn7850.msdu_end.info5,
+			    RX_MSDU_END_INFO5_L3_HDR_PADDING);
+}
+
+static inline
+bool ath12k_hal_rx_desc_encrypt_valid_wcn7850(struct hal_rx_desc *desc)
+{
+	return !!le32_get_bits(desc->u.wcn7850.mpdu_start.info4,
+			       RX_MPDU_START_INFO4_ENCRYPT_INFO_VALID);
+}
+
+static inline
+u32 ath12k_hal_rx_desc_get_encrypt_type_wcn7850(struct hal_rx_desc *desc)
+{
+	if (!ath12k_hal_rx_desc_encrypt_valid_wcn7850(desc))
+		return HAL_ENCRYPT_TYPE_OPEN;
+
+	return le32_get_bits(desc->u.wcn7850.mpdu_start.info2,
+			     RX_MPDU_START_INFO2_ENC_TYPE);
+}
+
+static inline
+u8 ath12k_hal_rx_desc_get_decap_type_wcn7850(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.wcn7850.msdu_end.info11,
+			     RX_MSDU_END_INFO11_DECAP_FORMAT);
+}
+
+static inline
+u8 ath12k_hal_rx_desc_get_mesh_ctl_wcn7850(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.wcn7850.msdu_end.info11,
+			     RX_MSDU_END_INFO11_MESH_CTRL_PRESENT);
+}
+
+static inline
+bool ath12k_hal_rx_desc_get_mpdu_seq_ctl_vld_wcn7850(struct hal_rx_desc *desc)
+{
+	return !!le32_get_bits(desc->u.wcn7850.mpdu_start.info4,
+			       RX_MPDU_START_INFO4_MPDU_SEQ_CTRL_VALID);
+}
+
+static inline
+bool ath12k_hal_rx_desc_get_mpdu_fc_valid_wcn7850(struct hal_rx_desc *desc)
+{
+	return !!le32_get_bits(desc->u.wcn7850.mpdu_start.info4,
+			       RX_MPDU_START_INFO4_MPDU_FCTRL_VALID);
+}
+
+static inline
+u16 ath12k_hal_rx_desc_get_mpdu_start_seq_no_wcn7850(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.wcn7850.mpdu_start.info4,
+			     RX_MPDU_START_INFO4_MPDU_SEQ_NUM);
+}
+
+static inline
+u16 ath12k_hal_rx_desc_get_msdu_len_wcn7850(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.wcn7850.msdu_end.info10,
+			     RX_MSDU_END_INFO10_MSDU_LENGTH);
+}
+
+static inline
+u8 ath12k_hal_rx_desc_get_msdu_sgi_wcn7850(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.wcn7850.msdu_end.info12,
+			     RX_MSDU_END_INFO12_SGI);
+}
+
+static inline
+u8 ath12k_hal_rx_desc_get_msdu_rate_mcs_wcn7850(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.wcn7850.msdu_end.info12,
+			     RX_MSDU_END_INFO12_RATE_MCS);
+}
+
+static inline
+u8 ath12k_hal_rx_desc_get_msdu_rx_bw_wcn7850(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.wcn7850.msdu_end.info12,
+			     RX_MSDU_END_INFO12_RECV_BW);
+}
+
+static inline
+u32 ath12k_hal_rx_desc_get_msdu_freq_wcn7850(struct hal_rx_desc *desc)
+{
+	return __le32_to_cpu(desc->u.wcn7850.msdu_end.phy_meta_data);
+}
+
+static inline
+u8 ath12k_hal_rx_desc_get_msdu_pkt_type_wcn7850(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.wcn7850.msdu_end.info12,
+			     RX_MSDU_END_INFO12_PKT_TYPE);
+}
+
+static inline
+u8 ath12k_hal_rx_desc_get_msdu_nss_wcn7850(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.wcn7850.msdu_end.info12,
+			     RX_MSDU_END_INFO12_MIMO_SS_BITMAP);
+}
+
+static inline
+u8 ath12k_hal_rx_desc_get_mpdu_tid_wcn7850(struct hal_rx_desc *desc)
+{
+	return le32_get_bits(desc->u.wcn7850.mpdu_start.info2,
+			     RX_MPDU_START_INFO2_TID);
+}
+
+static inline
+u16 ath12k_hal_rx_desc_get_mpdu_peer_id_wcn7850(struct hal_rx_desc *desc)
+{
+	return __le16_to_cpu(desc->u.wcn7850.mpdu_start.sw_peer_id);
+}
+
+void ath12k_hal_rx_desc_copy_end_tlv_wcn7850(struct hal_rx_desc *fdesc,
+					     struct hal_rx_desc *ldesc)
+{
+	memcpy(&fdesc->u.wcn7850.msdu_end, &ldesc->u.wcn7850.msdu_end,
+	       sizeof(struct rx_msdu_end_qcn9274));
+}
+
+u32 ath12k_hal_rx_desc_get_mpdu_start_tag_wcn7850(struct hal_rx_desc *desc)
+{
+	return le64_get_bits(desc->u.wcn7850.mpdu_start_tag,
+			    HAL_TLV_HDR_TAG);
+}
+
+u32 ath12k_hal_rx_desc_get_mpdu_ppdu_id_wcn7850(struct hal_rx_desc *desc)
+{
+	return __le16_to_cpu(desc->u.wcn7850.mpdu_start.phy_ppdu_id);
+}
+
+void ath12k_hal_rx_desc_set_msdu_len_wcn7850(struct hal_rx_desc *desc, u16 len)
+{
+	u32 info = __le32_to_cpu(desc->u.wcn7850.msdu_end.info10);
+
+	info &= ~RX_MSDU_END_INFO10_MSDU_LENGTH;
+	info |= u32_encode_bits(len, RX_MSDU_END_INFO10_MSDU_LENGTH);
+
+	desc->u.wcn7850.msdu_end.info10 = __cpu_to_le32(info);
+}
+
+u8 *ath12k_hal_rx_desc_get_msdu_payload_wcn7850(struct hal_rx_desc *desc)
+{
+	return &desc->u.wcn7850.msdu_payload[0];
+}
+
+u32 ath12k_hal_rx_desc_get_mpdu_start_offset_wcn7850(void)
+{
+	return offsetof(struct hal_rx_desc_wcn7850, mpdu_start_tag);
+}
+
+u32 ath12k_hal_rx_desc_get_msdu_end_offset_wcn7850(void)
+{
+	return offsetof(struct hal_rx_desc_wcn7850, msdu_end_tag);
+}
+
+static inline
+bool ath12k_hal_rx_desc_mac_addr2_valid_wcn7850(struct hal_rx_desc *desc)
+{
+	return __le32_to_cpu(desc->u.wcn7850.mpdu_start.info4) &
+	       RX_MPDU_START_INFO4_MAC_ADDR2_VALID;
+}
+
+static inline
+u8 *ath12k_hal_rx_desc_mpdu_start_addr2_wcn7850(struct hal_rx_desc *desc)
+{
+	return desc->u.wcn7850.mpdu_start.addr2;
+}
+
+static inline
+bool ath12k_hal_rx_desc_is_da_mcbc_wcn7850(struct hal_rx_desc *desc)
+{
+	return __le32_to_cpu(desc->u.wcn7850.msdu_end.info13) &
+	       RX_MSDU_END_INFO13_MCAST_BCAST;
+}
+
+static inline
+bool ath12k_hal_rx_h_msdu_done_wcn7850(struct hal_rx_desc *desc)
+{
+	return !!le32_get_bits(desc->u.wcn7850.msdu_end.info14,
+			       RX_MSDU_END_INFO14_MSDU_DONE);
+}
+
+static inline
+bool ath12k_hal_rx_h_l4_cksum_fail_wcn7850(struct hal_rx_desc *desc)
+{
+	return !!le32_get_bits(desc->u.wcn7850.msdu_end.info13,
+			       RX_MSDU_END_INFO13_TCP_UDP_CKSUM_FAIL);
+}
+
+static inline
+bool ath12k_hal_rx_h_ip_cksum_fail_wcn7850(struct hal_rx_desc *desc)
+{
+	return !!le32_get_bits(desc->u.wcn7850.msdu_end.info13,
+			      RX_MSDU_END_INFO13_IP_CKSUM_FAIL);
+}
+
+static inline
+bool ath12k_hal_rx_h_is_decrypted_wcn7850(struct hal_rx_desc *desc)
+{
+	return (le32_get_bits(desc->u.wcn7850.msdu_end.info14,
+			      RX_MSDU_END_INFO14_DECRYPT_STATUS_CODE) ==
+			      RX_DESC_DECRYPT_STATUS_CODE_OK);
+}
+
+u32 ath12k_hal_get_rx_desc_size_wcn7850(void)
+{
+	return sizeof(struct hal_rx_desc_wcn7850);
+}
+
+u8 ath12k_hal_rx_desc_get_msdu_src_link_wcn7850(struct hal_rx_desc *desc)
+{
+	return 0;
+}
+
+static u32 ath12k_hal_rx_h_mpdu_err_wcn7850(struct hal_rx_desc *desc)
+{
+	u32 info = __le32_to_cpu(desc->u.wcn7850.msdu_end.info13);
+	u32 errmap = 0;
+
+	if (info & RX_MSDU_END_INFO13_FCS_ERR)
+		errmap |= HAL_RX_MPDU_ERR_FCS;
+
+	if (info & RX_MSDU_END_INFO13_DECRYPT_ERR)
+		errmap |= HAL_RX_MPDU_ERR_DECRYPT;
+
+	if (info & RX_MSDU_END_INFO13_TKIP_MIC_ERR)
+		errmap |= HAL_RX_MPDU_ERR_TKIP_MIC;
+
+	if (info & RX_MSDU_END_INFO13_A_MSDU_ERROR)
+		errmap |= HAL_RX_MPDU_ERR_AMSDU_ERR;
+
+	if (info & RX_MSDU_END_INFO13_OVERFLOW_ERR)
+		errmap |= HAL_RX_MPDU_ERR_OVERFLOW;
+
+	if (info & RX_MSDU_END_INFO13_MSDU_LEN_ERR)
+		errmap |= HAL_RX_MPDU_ERR_MSDU_LEN;
+
+	if (info & RX_MSDU_END_INFO13_MPDU_LEN_ERR)
+		errmap |= HAL_RX_MPDU_ERR_MPDU_LEN;
+
+	return errmap;
+}
+
+void ath12k_hal_rx_desc_get_crypto_hdr_wcn7850(struct hal_rx_desc *desc,
+					       u8 *crypto_hdr,
+					       enum hal_encrypt_type enctype)
+{
+	unsigned int key_id;
+
+	switch (enctype) {
+	case HAL_ENCRYPT_TYPE_OPEN:
+		return;
+	case HAL_ENCRYPT_TYPE_TKIP_NO_MIC:
+	case HAL_ENCRYPT_TYPE_TKIP_MIC:
+		crypto_hdr[0] =
+			HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.wcn7850.mpdu_start.pn[0]);
+		crypto_hdr[1] = 0;
+		crypto_hdr[2] =
+			HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.wcn7850.mpdu_start.pn[0]);
+		break;
+	case HAL_ENCRYPT_TYPE_CCMP_128:
+	case HAL_ENCRYPT_TYPE_CCMP_256:
+	case HAL_ENCRYPT_TYPE_GCMP_128:
+	case HAL_ENCRYPT_TYPE_AES_GCMP_256:
+		crypto_hdr[0] =
+			HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.wcn7850.mpdu_start.pn[0]);
+		crypto_hdr[1] =
+			HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.wcn7850.mpdu_start.pn[0]);
+		crypto_hdr[2] = 0;
+		break;
+	case HAL_ENCRYPT_TYPE_WEP_40:
+	case HAL_ENCRYPT_TYPE_WEP_104:
+	case HAL_ENCRYPT_TYPE_WEP_128:
+	case HAL_ENCRYPT_TYPE_WAPI_GCM_SM4:
+	case HAL_ENCRYPT_TYPE_WAPI:
+		return;
+	}
+	key_id = u32_get_bits(__le32_to_cpu(desc->u.wcn7850.mpdu_start.info5),
+			      RX_MPDU_START_INFO5_KEY_ID);
+	crypto_hdr[3] = 0x20 | (key_id << 6);
+	crypto_hdr[4] = HAL_RX_MPDU_INFO_PN_GET_BYTE3(desc->u.wcn7850.mpdu_start.pn[0]);
+	crypto_hdr[5] = HAL_RX_MPDU_INFO_PN_GET_BYTE4(desc->u.wcn7850.mpdu_start.pn[0]);
+	crypto_hdr[6] = HAL_RX_MPDU_INFO_PN_GET_BYTE1(desc->u.wcn7850.mpdu_start.pn[1]);
+	crypto_hdr[7] = HAL_RX_MPDU_INFO_PN_GET_BYTE2(desc->u.wcn7850.mpdu_start.pn[1]);
+}
+
+void ath12k_hal_rx_desc_get_dot11_hdr_wcn7850(struct hal_rx_desc *desc,
+					      struct ieee80211_hdr *hdr)
+{
+	hdr->frame_control = desc->u.wcn7850.mpdu_start.frame_ctrl;
+	hdr->duration_id = desc->u.wcn7850.mpdu_start.duration;
+	ether_addr_copy(hdr->addr1, desc->u.wcn7850.mpdu_start.addr1);
+	ether_addr_copy(hdr->addr2, desc->u.wcn7850.mpdu_start.addr2);
+	ether_addr_copy(hdr->addr3, desc->u.wcn7850.mpdu_start.addr3);
+	if (__le32_to_cpu(desc->u.wcn7850.mpdu_start.info4) &
+			RX_MPDU_START_INFO4_MAC_ADDR4_VALID) {
+		ether_addr_copy(hdr->addr4, desc->u.wcn7850.mpdu_start.addr4);
+	}
+	hdr->seq_ctrl = desc->u.wcn7850.mpdu_start.seq_ctrl;
+}
+
+void ath12k_hal_extract_rx_desc_data_wcn7850(struct hal_rx_desc_data *rx_desc_data,
+					     struct hal_rx_desc *rx_desc,
+					     struct hal_rx_desc *ldesc)
+{
+	rx_desc_data->is_first_msdu = ath12k_hal_rx_desc_get_first_msdu_wcn7850(ldesc);
+	rx_desc_data->is_last_msdu = ath12k_hal_rx_desc_get_last_msdu_wcn7850(ldesc);
+	rx_desc_data->l3_pad_bytes = ath12k_hal_rx_desc_get_l3_pad_bytes_wcn7850(ldesc);
+	rx_desc_data->enctype = ath12k_hal_rx_desc_get_encrypt_type_wcn7850(rx_desc);
+	rx_desc_data->decap_type = ath12k_hal_rx_desc_get_decap_type_wcn7850(rx_desc);
+	rx_desc_data->mesh_ctrl_present =
+		ath12k_hal_rx_desc_get_mesh_ctl_wcn7850(rx_desc);
+	rx_desc_data->seq_ctl_valid =
+		ath12k_hal_rx_desc_get_mpdu_seq_ctl_vld_wcn7850(rx_desc);
+	rx_desc_data->fc_valid = ath12k_hal_rx_desc_get_mpdu_fc_valid_wcn7850(rx_desc);
+	rx_desc_data->seq_no = ath12k_hal_rx_desc_get_mpdu_start_seq_no_wcn7850(rx_desc);
+	rx_desc_data->msdu_len = ath12k_hal_rx_desc_get_msdu_len_wcn7850(ldesc);
+	rx_desc_data->sgi = ath12k_hal_rx_desc_get_msdu_sgi_wcn7850(rx_desc);
+	rx_desc_data->rate_mcs = ath12k_hal_rx_desc_get_msdu_rate_mcs_wcn7850(rx_desc);
+	rx_desc_data->bw = ath12k_hal_rx_desc_get_msdu_rx_bw_wcn7850(rx_desc);
+	rx_desc_data->phy_meta_data = ath12k_hal_rx_desc_get_msdu_freq_wcn7850(rx_desc);
+	rx_desc_data->pkt_type = ath12k_hal_rx_desc_get_msdu_pkt_type_wcn7850(rx_desc);
+	rx_desc_data->nss = hweight8(ath12k_hal_rx_desc_get_msdu_nss_wcn7850(rx_desc));
+	rx_desc_data->tid = ath12k_hal_rx_desc_get_mpdu_tid_wcn7850(rx_desc);
+	rx_desc_data->peer_id = ath12k_hal_rx_desc_get_mpdu_peer_id_wcn7850(rx_desc);
+	rx_desc_data->addr2_present = ath12k_hal_rx_desc_mac_addr2_valid_wcn7850(rx_desc);
+	rx_desc_data->addr2 = ath12k_hal_rx_desc_mpdu_start_addr2_wcn7850(rx_desc);
+	rx_desc_data->is_mcbc = ath12k_hal_rx_desc_is_da_mcbc_wcn7850(rx_desc);
+	rx_desc_data->msdu_done = ath12k_hal_rx_h_msdu_done_wcn7850(ldesc);
+	rx_desc_data->l4_csum_fail = ath12k_hal_rx_h_l4_cksum_fail_wcn7850(rx_desc);
+	rx_desc_data->ip_csum_fail = ath12k_hal_rx_h_ip_cksum_fail_wcn7850(rx_desc);
+	rx_desc_data->is_decrypted = ath12k_hal_rx_h_is_decrypted_wcn7850(rx_desc);
+	rx_desc_data->err_bitmap = ath12k_hal_rx_h_mpdu_err_wcn7850(rx_desc);
+}
+
+static int ath12k_hal_srng_create_config_wcn7850(struct ath12k_hal *hal)
+{
+	struct hal_srng_config *s;
+
+	hal->srng_config = kmemdup(hw_srng_config_template,
+				   sizeof(hw_srng_config_template),
+				   GFP_KERNEL);
+	if (!hal->srng_config)
+		return -ENOMEM;
+
+	s = &hal->srng_config[HAL_REO_DST];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_HP;
+	s->reg_size[0] = HAL_REO2_RING_BASE_LSB(hal) - HAL_REO1_RING_BASE_LSB(hal);
+	s->reg_size[1] = HAL_REO2_RING_HP - HAL_REO1_RING_HP;
+
+	s = &hal->srng_config[HAL_REO_EXCEPTION];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_SW0_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_SW0_RING_HP;
+
+	s = &hal->srng_config[HAL_REO_REINJECT];
+	s->max_rings = 1;
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_HP;
+
+	s = &hal->srng_config[HAL_REO_CMD];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_HP;
+
+	s = &hal->srng_config[HAL_REO_STATUS];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_HP;
+
+	s = &hal->srng_config[HAL_TCL_DATA];
+	s->max_rings = 5;
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_HP;
+	s->reg_size[0] = HAL_TCL2_RING_BASE_LSB(hal) - HAL_TCL1_RING_BASE_LSB(hal);
+	s->reg_size[1] = HAL_TCL2_RING_HP - HAL_TCL1_RING_HP;
+
+	s = &hal->srng_config[HAL_TCL_CMD];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_HP;
+
+	s = &hal->srng_config[HAL_TCL_STATUS];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_HP;
+
+	s = &hal->srng_config[HAL_CE_SRC];
+	s->max_rings = 12;
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal) + HAL_CE_DST_RING_BASE_LSB;
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal) + HAL_CE_DST_RING_HP;
+	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(hal) -
+		HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal);
+	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(hal) -
+		HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(hal);
+
+	s = &hal->srng_config[HAL_CE_DST];
+	s->max_rings = 12;
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) + HAL_CE_DST_RING_BASE_LSB;
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) + HAL_CE_DST_RING_HP;
+	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) -
+		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal);
+	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) -
+		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal);
+
+	s = &hal->srng_config[HAL_CE_DST_STATUS];
+	s->max_rings = 12;
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) +
+		HAL_CE_DST_STATUS_RING_BASE_LSB;
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal) + HAL_CE_DST_STATUS_RING_HP;
+	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) -
+		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal);
+	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(hal) -
+		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(hal);
+
+	s = &hal->srng_config[HAL_WBM_IDLE_LINK];
+	s->reg_start[0] =
+		HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_HP;
+
+	s = &hal->srng_config[HAL_SW2WBM_RELEASE];
+	s->max_rings = 1;
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG +
+		HAL_WBM_SW_RELEASE_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_SW_RELEASE_RING_HP;
+
+	s = &hal->srng_config[HAL_WBM2SW_RELEASE];
+	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_BASE_LSB(hal);
+	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_HP;
+	s->reg_size[0] = HAL_WBM1_RELEASE_RING_BASE_LSB(hal) -
+		HAL_WBM0_RELEASE_RING_BASE_LSB(hal);
+	s->reg_size[1] = HAL_WBM1_RELEASE_RING_HP - HAL_WBM0_RELEASE_RING_HP;
+
+	s = &hal->srng_config[HAL_RXDMA_BUF];
+	s->max_rings = 2;
+	s->mac_type = ATH12K_HAL_SRNG_PMAC;
+
+	s = &hal->srng_config[HAL_RXDMA_DST];
+	s->max_rings = 1;
+	s->entry_size = sizeof(struct hal_reo_entrance_ring) >> 2;
+
+	/* below rings are not used */
+	s = &hal->srng_config[HAL_RXDMA_DIR_BUF];
+	s->max_rings = 0;
+
+	s = &hal->srng_config[HAL_PPE2TCL];
+	s->max_rings = 0;
+
+	s = &hal->srng_config[HAL_PPE_RELEASE];
+	s->max_rings = 0;
+
+	s = &hal->srng_config[HAL_TX_MONITOR_BUF];
+	s->max_rings = 0;
+
+	s = &hal->srng_config[HAL_TX_MONITOR_DST];
+	s->max_rings = 0;
+
+	s = &hal->srng_config[HAL_PPE2TCL];
+	s->max_rings = 0;
+
+	return 0;
+}
+
+const struct ath12k_hal_tcl_to_wbm_rbm_map
+ath12k_hal_tcl_to_wbm_rbm_map_wcn7850[DP_TCL_NUM_RING_MAX] = {
+	{
+		.wbm_ring_num = 0,
+		.rbm_id = HAL_RX_BUF_RBM_SW0_BM,
+	},
+	{
+		.wbm_ring_num = 2,
+		.rbm_id = HAL_RX_BUF_RBM_SW2_BM,
+	},
+	{
+		.wbm_ring_num = 4,
+		.rbm_id = HAL_RX_BUF_RBM_SW4_BM,
+	},
+};
+
+const struct ath12k_hw_hal_params ath12k_hw_hal_params_wcn7850 = {
+	.rx_buf_rbm = HAL_RX_BUF_RBM_SW1_BM,
+	.wbm2sw_cc_enable = HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN |
+			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW2_EN |
+			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW3_EN |
+			    HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW4_EN,
+};
+
+const struct hal_ops hal_wcn7850_ops = {
+	.create_srng_config = ath12k_hal_srng_create_config_wcn7850,
+	.rx_desc_set_msdu_len = ath12k_hal_rx_desc_set_msdu_len_wcn7850,
+	.rx_desc_get_dot11_hdr = ath12k_hal_rx_desc_get_dot11_hdr_wcn7850,
+	.rx_desc_get_crypto_header = ath12k_hal_rx_desc_get_crypto_hdr_wcn7850,
+	.rx_desc_copy_end_tlv = ath12k_hal_rx_desc_copy_end_tlv_wcn7850,
+	.rx_desc_get_msdu_src_link_id = ath12k_hal_rx_desc_get_msdu_src_link_wcn7850,
+	.extract_rx_desc_data = ath12k_hal_extract_rx_desc_data_wcn7850,
+	.rx_desc_get_l3_pad_bytes = ath12k_hal_rx_desc_get_l3_pad_bytes_wcn7850,
+	.rx_desc_get_mpdu_start_tag = ath12k_hal_rx_desc_get_mpdu_start_tag_wcn7850,
+	.rx_desc_get_mpdu_ppdu_id = ath12k_hal_rx_desc_get_mpdu_ppdu_id_wcn7850,
+	.rx_desc_get_msdu_payload = ath12k_hal_rx_desc_get_msdu_payload_wcn7850,
+	.ce_dst_setup = ath12k_wifi7_hal_ce_dst_setup,
+	.srng_src_hw_init = ath12k_wifi7_hal_srng_src_hw_init,
+	.srng_dst_hw_init = ath12k_wifi7_hal_srng_dst_hw_init,
+	.set_umac_srng_ptr_addr = ath12k_wifi7_hal_set_umac_srng_ptr_addr,
+	.srng_update_shadow_config = ath12k_wifi7_hal_srng_update_shadow_config,
+	.srng_get_ring_id = ath12k_wifi7_hal_srng_get_ring_id,
+	.ce_get_desc_size = ath12k_wifi7_hal_ce_get_desc_size,
+	.ce_src_set_desc = ath12k_wifi7_hal_ce_src_set_desc,
+	.ce_dst_set_desc = ath12k_wifi7_hal_ce_dst_set_desc,
+	.ce_dst_status_get_length = ath12k_wifi7_hal_ce_dst_status_get_length,
+	.set_link_desc_addr = ath12k_wifi7_hal_set_link_desc_addr,
+	.tx_set_dscp_tid_map = ath12k_wifi7_hal_tx_set_dscp_tid_map,
+	.tx_configure_bank_register =
+		ath12k_wifi7_hal_tx_configure_bank_register,
+	.reoq_lut_addr_read_enable = ath12k_wifi7_hal_reoq_lut_addr_read_enable,
+	.reoq_lut_set_max_peerid = ath12k_wifi7_hal_reoq_lut_set_max_peerid,
+	.write_reoq_lut_addr = ath12k_wifi7_hal_write_reoq_lut_addr,
+	.write_ml_reoq_lut_addr = ath12k_wifi7_hal_write_ml_reoq_lut_addr,
+	.setup_link_idle_list = ath12k_wifi7_hal_setup_link_idle_list,
+	.reo_init_cmd_ring = ath12k_wifi7_hal_reo_init_cmd_ring,
+	.reo_shared_qaddr_cache_clear = ath12k_wifi7_hal_reo_shared_qaddr_cache_clear,
+	.reo_hw_setup = ath12k_wifi7_hal_reo_hw_setup,
+	.rx_buf_addr_info_set = ath12k_wifi7_hal_rx_buf_addr_info_set,
+	.rx_buf_addr_info_get = ath12k_wifi7_hal_rx_buf_addr_info_get,
+	.cc_config = ath12k_wifi7_hal_cc_config,
+	.get_idle_link_rbm = ath12k_wifi7_hal_get_idle_link_rbm,
+	.rx_msdu_list_get = ath12k_wifi7_hal_rx_msdu_list_get,
+	.rx_reo_ent_buf_paddr_get = ath12k_wifi7_hal_rx_reo_ent_buf_paddr_get,
+};
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.h
new file mode 100644
index 000000000000..46047fd6a312
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef ATH12K_HAL_WCN7850_H
+#define ATH12K_HAL_WCN7850_H
+
+#include "../hal.h"
+#include "hal_rx.h"
+#include "hal.h"
+
+extern const struct hal_ops hal_wcn7850_ops;
+extern const struct ath12k_hw_regs wcn7850_regs;
+extern const struct ath12k_hal_tcl_to_wbm_rbm_map
+ath12k_hal_tcl_to_wbm_rbm_map_wcn7850[DP_TCL_NUM_RING_MAX];
+extern const struct ath12k_hw_hal_params ath12k_hw_hal_params_wcn7850;
+
+u8 ath12k_hal_rx_desc_get_l3_pad_bytes_wcn7850(struct hal_rx_desc *desc);
+void ath12k_hal_rx_desc_copy_end_tlv_wcn7850(struct hal_rx_desc *fdesc,
+					     struct hal_rx_desc *ldesc);
+u32 ath12k_hal_rx_desc_get_mpdu_start_tag_wcn7850(struct hal_rx_desc *desc);
+u32 ath12k_hal_rx_desc_get_mpdu_ppdu_id_wcn7850(struct hal_rx_desc *desc);
+void ath12k_hal_rx_desc_set_msdu_len_wcn7850(struct hal_rx_desc *desc, u16 len);
+u8 *ath12k_hal_rx_desc_get_msdu_payload_wcn7850(struct hal_rx_desc *desc);
+u32 ath12k_hal_rx_desc_get_mpdu_start_offset_wcn7850(void);
+u32 ath12k_hal_rx_desc_get_msdu_end_offset_wcn7850(void);
+u32 ath12k_hal_get_rx_desc_size_wcn7850(void);
+u8 ath12k_hal_rx_desc_get_msdu_src_link_wcn7850(struct hal_rx_desc *desc);
+void ath12k_hal_rx_desc_get_crypto_hdr_wcn7850(struct hal_rx_desc *desc,
+					       u8 *crypto_hdr,
+					       enum hal_encrypt_type enctype);
+void ath12k_hal_rx_desc_get_dot11_hdr_wcn7850(struct hal_rx_desc *desc,
+					      struct ieee80211_hdr *hdr);
+void ath12k_hal_extract_rx_desc_data_wcn7850(struct hal_rx_desc_data *rx_desc_data,
+					     struct hal_rx_desc *rx_desc,
+					     struct hal_rx_desc *ldesc);
+#endif
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hw.c b/drivers/net/wireless/ath/ath12k/wifi7/hw.c
new file mode 100644
index 000000000000..1f5dda73230d
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hw.c
@@ -0,0 +1,945 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/types.h>
+#include <linux/bitops.h>
+#include <linux/bitfield.h>
+
+#include "../debug.h"
+#include "../core.h"
+#include "../ce.h"
+#include "ce.h"
+#include "../hw.h"
+#include "hw.h"
+#include "../mhi.h"
+#include "mhi.h"
+#include "dp_rx.h"
+#include "../peer.h"
+#include "wmi.h"
+#include "../wow.h"
+#include "../debugfs.h"
+#include "../debugfs_sta.h"
+#include "../testmode.h"
+#include "hal.h"
+#include "dp_tx.h"
+
+static const guid_t wcn7850_uuid = GUID_INIT(0xf634f534, 0x6147, 0x11ec,
+					     0x90, 0xd6, 0x02, 0x42,
+					     0xac, 0x12, 0x00, 0x03);
+
+static u8 ath12k_wifi7_hw_qcn9274_mac_from_pdev_id(int pdev_idx)
+{
+	return pdev_idx;
+}
+
+static int
+ath12k_wifi7_hw_mac_id_to_pdev_id_qcn9274(const struct ath12k_hw_params *hw,
+					  int mac_id)
+{
+	return mac_id;
+}
+
+static int
+ath12k_wifi7_hw_mac_id_to_srng_id_qcn9274(const struct ath12k_hw_params *hw,
+					  int mac_id)
+{
+	return 0;
+}
+
+static u8 ath12k_wifi7_hw_get_ring_selector_qcn9274(struct sk_buff *skb)
+{
+	return smp_processor_id();
+}
+
+static bool ath12k_wifi7_dp_srng_is_comp_ring_qcn9274(int ring_num)
+{
+	if (ring_num < 3 || ring_num == 4)
+		return true;
+
+	return false;
+}
+
+static bool
+ath12k_wifi7_is_frame_link_agnostic_qcn9274(struct ath12k_link_vif *arvif,
+					    struct ieee80211_mgmt *mgmt)
+{
+	return ieee80211_is_action(mgmt->frame_control);
+}
+
+static int
+ath12k_wifi7_hw_mac_id_to_pdev_id_wcn7850(const struct ath12k_hw_params *hw,
+					  int mac_id)
+{
+	return 0;
+}
+
+static int
+ath12k_wifi7_hw_mac_id_to_srng_id_wcn7850(const struct ath12k_hw_params *hw,
+					  int mac_id)
+{
+	return mac_id;
+}
+
+static u8 ath12k_wifi7_hw_get_ring_selector_wcn7850(struct sk_buff *skb)
+{
+	return skb_get_queue_mapping(skb);
+}
+
+static bool ath12k_wifi7_dp_srng_is_comp_ring_wcn7850(int ring_num)
+{
+	if (ring_num == 0 || ring_num == 2 || ring_num == 4)
+		return true;
+
+	return false;
+}
+
+static bool ath12k_is_addba_resp_action_code(struct ieee80211_mgmt *mgmt)
+{
+	if (!ieee80211_is_action(mgmt->frame_control))
+		return false;
+
+	if (mgmt->u.action.category != WLAN_CATEGORY_BACK)
+		return false;
+
+	if (mgmt->u.action.u.addba_resp.action_code != WLAN_ACTION_ADDBA_RESP)
+		return false;
+
+	return true;
+}
+
+static bool
+ath12k_wifi7_is_frame_link_agnostic_wcn7850(struct ath12k_link_vif *arvif,
+					    struct ieee80211_mgmt *mgmt)
+{
+	struct ieee80211_vif *vif = ath12k_ahvif_to_vif(arvif->ahvif);
+	struct ath12k_hw *ah = ath12k_ar_to_ah(arvif->ar);
+	struct ath12k_base *ab = arvif->ar->ab;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct ath12k_dp_peer *peer;
+	__le16 fc = mgmt->frame_control;
+
+	spin_lock_bh(&dp->dp_lock);
+	if (!ath12k_dp_link_peer_find_by_addr(dp, mgmt->da)) {
+		spin_lock_bh(&ah->dp_hw.peer_lock);
+		peer = ath12k_dp_peer_find_by_addr(&ah->dp_hw, mgmt->da);
+		if (!peer || (peer && !peer->is_mlo)) {
+			spin_unlock_bh(&ah->dp_hw.peer_lock);
+			spin_unlock_bh(&dp->dp_lock);
+			return false;
+		}
+		spin_unlock_bh(&ah->dp_hw.peer_lock);
+	}
+	spin_unlock_bh(&dp->dp_lock);
+
+	if (vif->type == NL80211_IFTYPE_STATION)
+		return arvif->is_up &&
+		       (vif->valid_links == vif->active_links) &&
+		       !ieee80211_is_probe_req(fc) &&
+		       !ieee80211_is_auth(fc) &&
+		       !ieee80211_is_deauth(fc) &&
+		       !ath12k_is_addba_resp_action_code(mgmt);
+
+	if (vif->type == NL80211_IFTYPE_AP)
+		return !(ieee80211_is_probe_resp(fc) || ieee80211_is_auth(fc) ||
+			 ieee80211_is_assoc_resp(fc) || ieee80211_is_reassoc_resp(fc) ||
+			 ath12k_is_addba_resp_action_code(mgmt));
+
+	return false;
+}
+
+static const struct ath12k_hw_ops qcn9274_ops = {
+	.get_hw_mac_from_pdev_id = ath12k_wifi7_hw_qcn9274_mac_from_pdev_id,
+	.mac_id_to_pdev_id = ath12k_wifi7_hw_mac_id_to_pdev_id_qcn9274,
+	.mac_id_to_srng_id = ath12k_wifi7_hw_mac_id_to_srng_id_qcn9274,
+	.rxdma_ring_sel_config = ath12k_dp_rxdma_ring_sel_config_qcn9274,
+	.get_ring_selector = ath12k_wifi7_hw_get_ring_selector_qcn9274,
+	.dp_srng_is_tx_comp_ring = ath12k_wifi7_dp_srng_is_comp_ring_qcn9274,
+	.is_frame_link_agnostic = ath12k_wifi7_is_frame_link_agnostic_qcn9274,
+};
+
+static const struct ath12k_hw_ops wcn7850_ops = {
+	.get_hw_mac_from_pdev_id = ath12k_wifi7_hw_qcn9274_mac_from_pdev_id,
+	.mac_id_to_pdev_id = ath12k_wifi7_hw_mac_id_to_pdev_id_wcn7850,
+	.mac_id_to_srng_id = ath12k_wifi7_hw_mac_id_to_srng_id_wcn7850,
+	.rxdma_ring_sel_config = ath12k_dp_rxdma_ring_sel_config_wcn7850,
+	.get_ring_selector = ath12k_wifi7_hw_get_ring_selector_wcn7850,
+	.dp_srng_is_tx_comp_ring = ath12k_wifi7_dp_srng_is_comp_ring_wcn7850,
+	.is_frame_link_agnostic = ath12k_wifi7_is_frame_link_agnostic_wcn7850,
+};
+
+#define ATH12K_TX_RING_MASK_0 0x1
+#define ATH12K_TX_RING_MASK_1 0x2
+#define ATH12K_TX_RING_MASK_2 0x4
+#define ATH12K_TX_RING_MASK_3 0x8
+#define ATH12K_TX_RING_MASK_4 0x10
+
+#define ATH12K_RX_RING_MASK_0 0x1
+#define ATH12K_RX_RING_MASK_1 0x2
+#define ATH12K_RX_RING_MASK_2 0x4
+#define ATH12K_RX_RING_MASK_3 0x8
+
+#define ATH12K_RX_ERR_RING_MASK_0 0x1
+
+#define ATH12K_RX_WBM_REL_RING_MASK_0 0x1
+
+#define ATH12K_REO_STATUS_RING_MASK_0 0x1
+
+#define ATH12K_HOST2RXDMA_RING_MASK_0 0x1
+
+#define ATH12K_RX_MON_RING_MASK_0 0x1
+#define ATH12K_RX_MON_RING_MASK_1 0x2
+#define ATH12K_RX_MON_RING_MASK_2 0x4
+
+#define ATH12K_TX_MON_RING_MASK_0 0x1
+#define ATH12K_TX_MON_RING_MASK_1 0x2
+
+#define ATH12K_RX_MON_STATUS_RING_MASK_0 0x1
+#define ATH12K_RX_MON_STATUS_RING_MASK_1 0x2
+#define ATH12K_RX_MON_STATUS_RING_MASK_2 0x4
+
+static const struct ath12k_hw_ring_mask ath12k_wifi7_hw_ring_mask_qcn9274 = {
+	.tx  = {
+		ATH12K_TX_RING_MASK_0,
+		ATH12K_TX_RING_MASK_1,
+		ATH12K_TX_RING_MASK_2,
+		ATH12K_TX_RING_MASK_3,
+	},
+	.rx_mon_dest = {
+		0, 0, 0, 0,
+		0, 0, 0, 0,
+		ATH12K_RX_MON_RING_MASK_0,
+		ATH12K_RX_MON_RING_MASK_1,
+		ATH12K_RX_MON_RING_MASK_2,
+	},
+	.rx = {
+		0, 0, 0, 0,
+		ATH12K_RX_RING_MASK_0,
+		ATH12K_RX_RING_MASK_1,
+		ATH12K_RX_RING_MASK_2,
+		ATH12K_RX_RING_MASK_3,
+	},
+	.rx_err = {
+		0, 0, 0,
+		ATH12K_RX_ERR_RING_MASK_0,
+	},
+	.rx_wbm_rel = {
+		0, 0, 0,
+		ATH12K_RX_WBM_REL_RING_MASK_0,
+	},
+	.reo_status = {
+		0, 0, 0,
+		ATH12K_REO_STATUS_RING_MASK_0,
+	},
+	.host2rxdma = {
+		0, 0, 0,
+		ATH12K_HOST2RXDMA_RING_MASK_0,
+	},
+	.tx_mon_dest = {
+		0, 0, 0,
+	},
+};
+
+static const struct ath12k_hw_ring_mask ath12k_wifi7_hw_ring_mask_ipq5332 = {
+	.tx  = {
+		ATH12K_TX_RING_MASK_0,
+		ATH12K_TX_RING_MASK_1,
+		ATH12K_TX_RING_MASK_2,
+		ATH12K_TX_RING_MASK_3,
+	},
+	.rx_mon_dest = {
+		0, 0, 0, 0, 0, 0, 0, 0,
+		ATH12K_RX_MON_RING_MASK_0,
+	},
+	.rx = {
+		0, 0, 0, 0,
+		ATH12K_RX_RING_MASK_0,
+		ATH12K_RX_RING_MASK_1,
+		ATH12K_RX_RING_MASK_2,
+		ATH12K_RX_RING_MASK_3,
+	},
+	.rx_err = {
+		0, 0, 0,
+		ATH12K_RX_ERR_RING_MASK_0,
+	},
+	.rx_wbm_rel = {
+		0, 0, 0,
+		ATH12K_RX_WBM_REL_RING_MASK_0,
+	},
+	.reo_status = {
+		0, 0, 0,
+		ATH12K_REO_STATUS_RING_MASK_0,
+	},
+	.host2rxdma = {
+		0, 0, 0,
+		ATH12K_HOST2RXDMA_RING_MASK_0,
+	},
+	.tx_mon_dest = {
+		ATH12K_TX_MON_RING_MASK_0,
+		ATH12K_TX_MON_RING_MASK_1,
+	},
+};
+
+static const struct ath12k_hw_ring_mask ath12k_wifi7_hw_ring_mask_wcn7850 = {
+	.tx  = {
+		ATH12K_TX_RING_MASK_0,
+		ATH12K_TX_RING_MASK_1,
+		ATH12K_TX_RING_MASK_2,
+	},
+	.rx_mon_dest = {
+	},
+	.rx_mon_status = {
+		0, 0, 0, 0,
+		ATH12K_RX_MON_STATUS_RING_MASK_0,
+		ATH12K_RX_MON_STATUS_RING_MASK_1,
+		ATH12K_RX_MON_STATUS_RING_MASK_2,
+	},
+	.rx = {
+		0, 0, 0,
+		ATH12K_RX_RING_MASK_0,
+		ATH12K_RX_RING_MASK_1,
+		ATH12K_RX_RING_MASK_2,
+		ATH12K_RX_RING_MASK_3,
+	},
+	.rx_err = {
+		ATH12K_RX_ERR_RING_MASK_0,
+	},
+	.rx_wbm_rel = {
+		ATH12K_RX_WBM_REL_RING_MASK_0,
+	},
+	.reo_status = {
+		ATH12K_REO_STATUS_RING_MASK_0,
+	},
+	.host2rxdma = {
+	},
+	.tx_mon_dest = {
+	},
+};
+
+static const struct ce_ie_addr ath12k_wifi7_ce_ie_addr_ipq5332 = {
+	.ie1_reg_addr = CE_HOST_IE_ADDRESS - HAL_IPQ5332_CE_WFSS_REG_BASE,
+	.ie2_reg_addr = CE_HOST_IE_2_ADDRESS - HAL_IPQ5332_CE_WFSS_REG_BASE,
+	.ie3_reg_addr = CE_HOST_IE_3_ADDRESS - HAL_IPQ5332_CE_WFSS_REG_BASE,
+};
+
+static const struct ce_remap ath12k_wifi7_ce_remap_ipq5332 = {
+	.base = HAL_IPQ5332_CE_WFSS_REG_BASE,
+	.size = HAL_IPQ5332_CE_SIZE,
+	.cmem_offset = HAL_SEQ_WCSS_CMEM_OFFSET,
+};
+
+static const struct ath12k_hw_params ath12k_wifi7_hw_params[] = {
+	{
+		.name = "qcn9274 hw1.0",
+		.hw_rev = ATH12K_HW_QCN9274_HW10,
+		.fw = {
+			.dir = "QCN9274/hw1.0",
+			.board_size = 256 * 1024,
+			.cal_offset = 128 * 1024,
+			.m3_loader = ath12k_m3_fw_loader_driver,
+		},
+		.max_radios = 1,
+		.single_pdev_only = false,
+		.qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_QCN9274,
+		.internal_sleep_clock = false,
+
+		.hw_ops = &qcn9274_ops,
+		.ring_mask = &ath12k_wifi7_hw_ring_mask_qcn9274,
+
+		.host_ce_config = ath12k_wifi7_host_ce_config_qcn9274,
+		.ce_count = 16,
+		.target_ce_config = ath12k_wifi7_target_ce_config_wlan_qcn9274,
+		.target_ce_count = 12,
+		.svc_to_ce_map =
+			ath12k_wifi7_target_service_to_ce_map_wlan_qcn9274,
+		.svc_to_ce_map_len = 18,
+
+		.rxdma1_enable = false,
+		.num_rxdma_per_pdev = 1,
+		.num_rxdma_dst_ring = 0,
+		.rx_mac_buf_ring = false,
+		.vdev_start_delay = false,
+
+		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
+					BIT(NL80211_IFTYPE_AP) |
+					BIT(NL80211_IFTYPE_MESH_POINT) |
+					BIT(NL80211_IFTYPE_AP_VLAN),
+		.supports_monitor = false,
+
+		.idle_ps = false,
+		.download_calib = true,
+		.supports_suspend = false,
+		.tcl_ring_retry = true,
+		.reoq_lut_support = true,
+		.supports_shadow_regs = false,
+
+		.num_tcl_banks = 48,
+		.max_tx_ring = 4,
+
+		.mhi_config = &ath12k_wifi7_mhi_config_qcn9274,
+
+		.wmi_init = ath12k_wifi7_wmi_init_qcn9274,
+
+		.qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01),
+
+		.rfkill_pin = 0,
+		.rfkill_cfg = 0,
+		.rfkill_on_level = 0,
+
+		.rddm_size = 0x600000,
+
+		.def_num_link = 0,
+		.max_mlo_peer = 256,
+
+		.otp_board_id_register = QCN9274_QFPROM_RAW_RFA_PDET_ROW13_LSB,
+
+		.supports_sta_ps = false,
+
+		.acpi_guid = NULL,
+		.supports_dynamic_smps_6ghz = true,
+
+		.iova_mask = 0,
+
+		.supports_aspm = false,
+
+		.ce_ie_addr = NULL,
+		.ce_remap = NULL,
+		.bdf_addr_offset = 0,
+
+		.current_cc_support = false,
+
+		.dp_primary_link_only = true,
+	},
+	{
+		.name = "wcn7850 hw2.0",
+		.hw_rev = ATH12K_HW_WCN7850_HW20,
+
+		.fw = {
+			.dir = "WCN7850/hw2.0",
+			.board_size = 256 * 1024,
+			.cal_offset = 256 * 1024,
+			.m3_loader = ath12k_m3_fw_loader_driver,
+		},
+
+		.max_radios = 1,
+		.single_pdev_only = true,
+		.qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_WCN7850,
+		.internal_sleep_clock = true,
+
+		.hw_ops = &wcn7850_ops,
+		.ring_mask = &ath12k_wifi7_hw_ring_mask_wcn7850,
+
+		.host_ce_config = ath12k_wifi7_host_ce_config_wcn7850,
+		.ce_count = 9,
+		.target_ce_config = ath12k_wifi7_target_ce_config_wlan_wcn7850,
+		.target_ce_count = 9,
+		.svc_to_ce_map =
+			ath12k_wifi7_target_service_to_ce_map_wlan_wcn7850,
+		.svc_to_ce_map_len = 14,
+
+		.rxdma1_enable = false,
+		.num_rxdma_per_pdev = 2,
+		.num_rxdma_dst_ring = 1,
+		.rx_mac_buf_ring = true,
+		.vdev_start_delay = true,
+
+		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
+				   BIT(NL80211_IFTYPE_AP) |
+				   BIT(NL80211_IFTYPE_P2P_DEVICE) |
+				   BIT(NL80211_IFTYPE_P2P_CLIENT) |
+				   BIT(NL80211_IFTYPE_P2P_GO),
+		.supports_monitor = true,
+
+		.idle_ps = true,
+		.download_calib = false,
+		.supports_suspend = true,
+		.tcl_ring_retry = false,
+		.reoq_lut_support = false,
+		.supports_shadow_regs = true,
+
+		.num_tcl_banks = 7,
+		.max_tx_ring = 3,
+
+		.mhi_config = &ath12k_wifi7_mhi_config_wcn7850,
+
+		.wmi_init = ath12k_wifi7_wmi_init_wcn7850,
+
+		.qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01) |
+					   BIT(CNSS_PCIE_PERST_NO_PULL_V01),
+
+		.rfkill_pin = 48,
+		.rfkill_cfg = 0,
+		.rfkill_on_level = 1,
+
+		.rddm_size = 0x780000,
+
+		.def_num_link = 2,
+		.max_mlo_peer = 32,
+
+		.otp_board_id_register = 0,
+
+		.supports_sta_ps = true,
+
+		.acpi_guid = &wcn7850_uuid,
+		.supports_dynamic_smps_6ghz = false,
+
+		.iova_mask = ATH12K_PCIE_MAX_PAYLOAD_SIZE - 1,
+
+		.supports_aspm = true,
+
+		.ce_ie_addr = NULL,
+		.ce_remap = NULL,
+		.bdf_addr_offset = 0,
+
+		.current_cc_support = true,
+
+		.dp_primary_link_only = false,
+	},
+	{
+		.name = "qcn9274 hw2.0",
+		.hw_rev = ATH12K_HW_QCN9274_HW20,
+		.fw = {
+			.dir = "QCN9274/hw2.0",
+			.board_size = 256 * 1024,
+			.cal_offset = 128 * 1024,
+			.m3_loader = ath12k_m3_fw_loader_driver,
+		},
+		.max_radios = 2,
+		.single_pdev_only = false,
+		.qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_QCN9274,
+		.internal_sleep_clock = false,
+
+		.hw_ops = &qcn9274_ops,
+		.ring_mask = &ath12k_wifi7_hw_ring_mask_qcn9274,
+
+		.host_ce_config = ath12k_wifi7_host_ce_config_qcn9274,
+		.ce_count = 16,
+		.target_ce_config = ath12k_wifi7_target_ce_config_wlan_qcn9274,
+		.target_ce_count = 12,
+		.svc_to_ce_map =
+			ath12k_wifi7_target_service_to_ce_map_wlan_qcn9274,
+		.svc_to_ce_map_len = 18,
+
+		.rxdma1_enable = true,
+		.num_rxdma_per_pdev = 1,
+		.num_rxdma_dst_ring = 0,
+		.rx_mac_buf_ring = false,
+		.vdev_start_delay = false,
+
+		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
+					BIT(NL80211_IFTYPE_AP) |
+					BIT(NL80211_IFTYPE_MESH_POINT) |
+					BIT(NL80211_IFTYPE_AP_VLAN),
+		.supports_monitor = true,
+
+		.idle_ps = false,
+		.download_calib = true,
+		.supports_suspend = false,
+		.tcl_ring_retry = true,
+		.reoq_lut_support = true,
+		.supports_shadow_regs = false,
+
+		.num_tcl_banks = 48,
+		.max_tx_ring = 4,
+
+		.mhi_config = &ath12k_wifi7_mhi_config_qcn9274,
+
+		.wmi_init = ath12k_wifi7_wmi_init_qcn9274,
+
+		.qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01),
+
+		.rfkill_pin = 0,
+		.rfkill_cfg = 0,
+		.rfkill_on_level = 0,
+
+		.rddm_size = 0x600000,
+
+		.def_num_link = 0,
+		.max_mlo_peer = 256,
+
+		.otp_board_id_register = QCN9274_QFPROM_RAW_RFA_PDET_ROW13_LSB,
+
+		.supports_sta_ps = false,
+
+		.acpi_guid = NULL,
+		.supports_dynamic_smps_6ghz = true,
+
+		.iova_mask = 0,
+
+		.supports_aspm = false,
+
+		.ce_ie_addr = NULL,
+		.ce_remap = NULL,
+		.bdf_addr_offset = 0,
+
+		.current_cc_support = false,
+
+		.dp_primary_link_only = true,
+	},
+	{
+		.name = "ipq5332 hw1.0",
+		.hw_rev = ATH12K_HW_IPQ5332_HW10,
+		.fw = {
+			.dir = "IPQ5332/hw1.0",
+			.board_size = 256 * 1024,
+			.cal_offset = 128 * 1024,
+			.m3_loader = ath12k_m3_fw_loader_remoteproc,
+		},
+		.max_radios = 1,
+		.single_pdev_only = false,
+		.qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_IPQ5332,
+		.internal_sleep_clock = false,
+
+		.hw_ops = &qcn9274_ops,
+		.ring_mask = &ath12k_wifi7_hw_ring_mask_ipq5332,
+
+		.host_ce_config = ath12k_wifi7_host_ce_config_ipq5332,
+		.ce_count = 12,
+		.target_ce_config = ath12k_wifi7_target_ce_config_wlan_ipq5332,
+		.target_ce_count = 12,
+		.svc_to_ce_map =
+			ath12k_wifi7_target_service_to_ce_map_wlan_ipq5332,
+		.svc_to_ce_map_len = 18,
+
+		.rxdma1_enable = false,
+		.num_rxdma_per_pdev = 1,
+		.num_rxdma_dst_ring = 0,
+		.rx_mac_buf_ring = false,
+		.vdev_start_delay = false,
+
+		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
+				   BIT(NL80211_IFTYPE_AP) |
+				   BIT(NL80211_IFTYPE_MESH_POINT),
+		.supports_monitor = false,
+
+		.idle_ps = false,
+		.download_calib = true,
+		.supports_suspend = false,
+		.tcl_ring_retry = true,
+		.reoq_lut_support = false,
+		.supports_shadow_regs = false,
+
+		.num_tcl_banks = 48,
+		.max_tx_ring = 4,
+
+		.wmi_init = &ath12k_wifi7_wmi_init_qcn9274,
+
+		.qmi_cnss_feature_bitmap = BIT(CNSS_QDSS_CFG_MISS_V01),
+
+		.rfkill_pin = 0,
+		.rfkill_cfg = 0,
+		.rfkill_on_level = 0,
+
+		.rddm_size = 0,
+
+		.def_num_link = 0,
+		.max_mlo_peer = 256,
+
+		.otp_board_id_register = 0,
+
+		.supports_sta_ps = false,
+
+		.acpi_guid = NULL,
+		.supports_dynamic_smps_6ghz = false,
+		.iova_mask = 0,
+		.supports_aspm = false,
+
+		.ce_ie_addr = &ath12k_wifi7_ce_ie_addr_ipq5332,
+		.ce_remap = &ath12k_wifi7_ce_remap_ipq5332,
+		.bdf_addr_offset = 0xC00000,
+
+		.dp_primary_link_only = true,
+	},
+};
+
+/* Note: called under rcu_read_lock() */
+static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw,
+				   struct ieee80211_tx_control *control,
+				   struct sk_buff *skb)
+{
+	struct ath12k_skb_cb *skb_cb = ATH12K_SKB_CB(skb);
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct ieee80211_vif *vif = info->control.vif;
+	struct ath12k_vif *ahvif = ath12k_vif_to_ahvif(vif);
+	struct ath12k_link_vif *arvif = &ahvif->deflink;
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
+	struct ieee80211_key_conf *key = info->control.hw_key;
+	struct ieee80211_sta *sta = control->sta;
+	struct ath12k_link_vif *tmp_arvif;
+	u32 info_flags = info->flags;
+	struct sk_buff *msdu_copied;
+	struct ath12k *ar, *tmp_ar;
+	struct ath12k_pdev_dp *dp_pdev, *tmp_dp_pdev;
+	struct ath12k_dp_link_peer *peer;
+	unsigned long links_map;
+	bool is_mcast = false;
+	bool is_dvlan = false;
+	struct ethhdr *eth;
+	bool is_prb_rsp;
+	u16 mcbc_gsn;
+	u8 link_id;
+	int ret;
+	struct ath12k_dp *tmp_dp;
+
+	if (ahvif->vdev_type == WMI_VDEV_TYPE_MONITOR) {
+		ieee80211_free_txskb(hw, skb);
+		return;
+	}
+
+	link_id = u32_get_bits(info->control.flags, IEEE80211_TX_CTRL_MLO_LINK);
+	memset(skb_cb, 0, sizeof(*skb_cb));
+	skb_cb->vif = vif;
+
+	if (key) {
+		skb_cb->cipher = key->cipher;
+		skb_cb->flags |= ATH12K_SKB_CIPHER_SET;
+	}
+
+	/* handle only for MLO case, use deflink for non MLO case */
+	if (ieee80211_vif_is_mld(vif)) {
+		link_id = ath12k_mac_get_tx_link(sta, vif, link_id, skb, info_flags);
+		if (link_id >= IEEE80211_MLD_MAX_NUM_LINKS) {
+			ieee80211_free_txskb(hw, skb);
+			return;
+		}
+	} else {
+		link_id = 0;
+	}
+
+	arvif = rcu_dereference(ahvif->link[link_id]);
+	if (!arvif || !arvif->ar) {
+		ath12k_warn(ahvif->ah, "failed to find arvif link id %u for frame transmission",
+			    link_id);
+		ieee80211_free_txskb(hw, skb);
+		return;
+	}
+
+	ar = arvif->ar;
+	skb_cb->link_id = link_id;
+	/*
+	 * as skb_cb is common currently for dp and mgmt tx processing
+	 * set this in the common mac op tx function.
+	 */
+	skb_cb->ar = ar;
+	is_prb_rsp = ieee80211_is_probe_resp(hdr->frame_control);
+
+	if (info_flags & IEEE80211_TX_CTL_HW_80211_ENCAP) {
+		eth = (struct ethhdr *)skb->data;
+		is_mcast = is_multicast_ether_addr(eth->h_dest);
+
+		skb_cb->flags |= ATH12K_SKB_HW_80211_ENCAP;
+	} else if (ieee80211_is_mgmt(hdr->frame_control)) {
+		if (sta && sta->mlo)
+			skb_cb->flags |= ATH12K_SKB_MLO_STA;
+
+		ret = ath12k_mac_mgmt_tx(ar, skb, is_prb_rsp);
+		if (ret) {
+			ath12k_warn(ar->ab, "failed to queue management frame %d\n",
+				    ret);
+			ieee80211_free_txskb(hw, skb);
+		}
+		return;
+	}
+
+	if (!(info_flags & IEEE80211_TX_CTL_HW_80211_ENCAP))
+		is_mcast = is_multicast_ether_addr(hdr->addr1);
+
+	/* This is case only for P2P_GO */
+	if (vif->type == NL80211_IFTYPE_AP && vif->p2p)
+		ath12k_mac_add_p2p_noa_ie(ar, vif, skb, is_prb_rsp);
+
+	dp_pdev = ath12k_dp_to_pdev_dp(ar->ab->dp, ar->pdev_idx);
+	if (!dp_pdev) {
+		ieee80211_free_txskb(hw, skb);
+		return;
+	}
+
+	/* Checking if it is a DVLAN frame */
+	if (!test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ar->ab->dev_flags) &&
+	    !(skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) &&
+	    !(skb_cb->flags & ATH12K_SKB_CIPHER_SET) &&
+	    ieee80211_has_protected(hdr->frame_control))
+		is_dvlan = true;
+
+	if (!vif->valid_links || !is_mcast || is_dvlan ||
+	    (skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) ||
+	    test_bit(ATH12K_FLAG_RAW_MODE, &ar->ab->dev_flags)) {
+		ret = ath12k_wifi7_dp_tx(dp_pdev, arvif, skb, false, 0, is_mcast);
+		if (unlikely(ret)) {
+			ath12k_warn(ar->ab, "failed to transmit frame %d\n", ret);
+			ieee80211_free_txskb(ar->ah->hw, skb);
+			return;
+		}
+	} else {
+		mcbc_gsn = atomic_inc_return(&ahvif->dp_vif.mcbc_gsn) & 0xfff;
+
+		links_map = ahvif->links_map;
+		for_each_set_bit(link_id, &links_map,
+				 IEEE80211_MLD_MAX_NUM_LINKS) {
+			tmp_arvif = rcu_dereference(ahvif->link[link_id]);
+			if (!tmp_arvif || !tmp_arvif->is_up)
+				continue;
+
+			tmp_ar = tmp_arvif->ar;
+			tmp_dp_pdev = ath12k_dp_to_pdev_dp(tmp_ar->ab->dp,
+							   tmp_ar->pdev_idx);
+			if (!tmp_dp_pdev)
+				continue;
+			msdu_copied = skb_copy(skb, GFP_ATOMIC);
+			if (!msdu_copied) {
+				ath12k_err(ar->ab,
+					   "skb copy failure link_id 0x%X vdevid 0x%X\n",
+					   link_id, tmp_arvif->vdev_id);
+				continue;
+			}
+
+			ath12k_mlo_mcast_update_tx_link_address(vif, link_id,
+								msdu_copied,
+								info_flags);
+
+			skb_cb = ATH12K_SKB_CB(msdu_copied);
+			skb_cb->link_id = link_id;
+			skb_cb->vif = vif;
+			skb_cb->ar = tmp_ar;
+
+			/* For open mode, skip peer find logic */
+			if (unlikely(!ahvif->dp_vif.key_cipher))
+				goto skip_peer_find;
+
+			tmp_dp = ath12k_ab_to_dp(tmp_ar->ab);
+			spin_lock_bh(&tmp_dp->dp_lock);
+			peer = ath12k_dp_link_peer_find_by_addr(tmp_dp,
+								tmp_arvif->bssid);
+			if (!peer || !peer->dp_peer) {
+				spin_unlock_bh(&tmp_dp->dp_lock);
+				ath12k_warn(tmp_ar->ab,
+					    "failed to find peer for vdev_id 0x%X addr %pM link_map 0x%X\n",
+					    tmp_arvif->vdev_id, tmp_arvif->bssid,
+					    ahvif->links_map);
+				dev_kfree_skb_any(msdu_copied);
+				continue;
+			}
+
+			key = peer->dp_peer->keys[peer->dp_peer->mcast_keyidx];
+			if (key) {
+				skb_cb->cipher = key->cipher;
+				skb_cb->flags |= ATH12K_SKB_CIPHER_SET;
+
+				hdr = (struct ieee80211_hdr *)msdu_copied->data;
+				if (!ieee80211_has_protected(hdr->frame_control))
+					hdr->frame_control |=
+						cpu_to_le16(IEEE80211_FCTL_PROTECTED);
+			}
+			spin_unlock_bh(&tmp_dp->dp_lock);
+
+skip_peer_find:
+			ret = ath12k_wifi7_dp_tx(tmp_dp_pdev, tmp_arvif,
+						 msdu_copied, true, mcbc_gsn, is_mcast);
+			if (unlikely(ret)) {
+				if (ret == -ENOMEM) {
+					/* Drops are expected during heavy multicast
+					 * frame flood. Print with debug log
+					 * level to avoid lot of console prints
+					 */
+					ath12k_dbg(ar->ab, ATH12K_DBG_MAC,
+						   "failed to transmit frame %d\n",
+						   ret);
+				} else {
+					ath12k_warn(ar->ab,
+						    "failed to transmit frame %d\n",
+						    ret);
+				}
+
+				dev_kfree_skb_any(msdu_copied);
+			}
+		}
+		ieee80211_free_txskb(ar->ah->hw, skb);
+	}
+}
+
+static const struct ieee80211_ops ath12k_ops_wifi7 = {
+	.tx				= ath12k_wifi7_mac_op_tx,
+	.wake_tx_queue			= ieee80211_handle_wake_tx_queue,
+	.start                          = ath12k_mac_op_start,
+	.stop                           = ath12k_mac_op_stop,
+	.reconfig_complete              = ath12k_mac_op_reconfig_complete,
+	.add_interface                  = ath12k_mac_op_add_interface,
+	.remove_interface		= ath12k_mac_op_remove_interface,
+	.update_vif_offload		= ath12k_mac_op_update_vif_offload,
+	.config                         = ath12k_mac_op_config,
+	.link_info_changed              = ath12k_mac_op_link_info_changed,
+	.vif_cfg_changed		= ath12k_mac_op_vif_cfg_changed,
+	.change_vif_links               = ath12k_mac_op_change_vif_links,
+	.configure_filter		= ath12k_mac_op_configure_filter,
+	.hw_scan                        = ath12k_mac_op_hw_scan,
+	.cancel_hw_scan                 = ath12k_mac_op_cancel_hw_scan,
+	.set_key                        = ath12k_mac_op_set_key,
+	.set_rekey_data	                = ath12k_mac_op_set_rekey_data,
+	.sta_state                      = ath12k_mac_op_sta_state,
+	.sta_set_txpwr			= ath12k_mac_op_sta_set_txpwr,
+	.link_sta_rc_update		= ath12k_mac_op_link_sta_rc_update,
+	.conf_tx                        = ath12k_mac_op_conf_tx,
+	.set_antenna			= ath12k_mac_op_set_antenna,
+	.get_antenna			= ath12k_mac_op_get_antenna,
+	.ampdu_action			= ath12k_mac_op_ampdu_action,
+	.add_chanctx			= ath12k_mac_op_add_chanctx,
+	.remove_chanctx			= ath12k_mac_op_remove_chanctx,
+	.change_chanctx			= ath12k_mac_op_change_chanctx,
+	.assign_vif_chanctx		= ath12k_mac_op_assign_vif_chanctx,
+	.unassign_vif_chanctx		= ath12k_mac_op_unassign_vif_chanctx,
+	.switch_vif_chanctx		= ath12k_mac_op_switch_vif_chanctx,
+	.get_txpower			= ath12k_mac_op_get_txpower,
+	.set_rts_threshold		= ath12k_mac_op_set_rts_threshold,
+	.set_frag_threshold		= ath12k_mac_op_set_frag_threshold,
+	.set_bitrate_mask		= ath12k_mac_op_set_bitrate_mask,
+	.get_survey			= ath12k_mac_op_get_survey,
+	.flush				= ath12k_mac_op_flush,
+	.sta_statistics			= ath12k_mac_op_sta_statistics,
+	.link_sta_statistics		= ath12k_mac_op_link_sta_statistics,
+	.remain_on_channel              = ath12k_mac_op_remain_on_channel,
+	.cancel_remain_on_channel       = ath12k_mac_op_cancel_remain_on_channel,
+	.change_sta_links               = ath12k_mac_op_change_sta_links,
+	.can_activate_links             = ath12k_mac_op_can_activate_links,
+#ifdef CONFIG_PM
+	.suspend			= ath12k_wow_op_suspend,
+	.resume				= ath12k_wow_op_resume,
+	.set_wakeup			= ath12k_wow_op_set_wakeup,
+#endif
+#ifdef CONFIG_ATH12K_DEBUGFS
+	.vif_add_debugfs                = ath12k_debugfs_op_vif_add,
+#endif
+	CFG80211_TESTMODE_CMD(ath12k_tm_cmd)
+#ifdef CONFIG_ATH12K_DEBUGFS
+	.link_sta_add_debugfs           = ath12k_debugfs_link_sta_op_add,
+#endif
+};
+
+int ath12k_wifi7_hw_init(struct ath12k_base *ab)
+{
+	const struct ath12k_hw_params *hw_params = NULL;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ath12k_wifi7_hw_params); i++) {
+		hw_params = &ath12k_wifi7_hw_params[i];
+
+		if (hw_params->hw_rev == ab->hw_rev)
+			break;
+	}
+
+	if (i == ARRAY_SIZE(ath12k_wifi7_hw_params)) {
+		ath12k_err(ab, "Unsupported Wi-Fi 7 hardware version: 0x%x\n",
+			   ab->hw_rev);
+		return -EINVAL;
+	}
+
+	ab->hw_params = hw_params;
+	ab->ath12k_ops = &ath12k_ops_wifi7;
+
+	ath12k_wifi7_hal_init(ab);
+
+	ath12k_info(ab, "Wi-Fi 7 Hardware name: %s\n", ab->hw_params->name);
+
+	return 0;
+}
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hw.h b/drivers/net/wireless/ath/ath12k/wifi7/hw.h
new file mode 100644
index 000000000000..643b6fdfdb66
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hw.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef ATH12K_WIFI7_HW_H
+#define ATH12K_WIFI7_HW_H
+
+struct ath12k_base;
+int ath12k_wifi7_hw_init(struct ath12k_base *ab);
+
+#endif /* ATH12K_WIFI7_HW_H */
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/mhi.c b/drivers/net/wireless/ath/ath12k/wifi7/mhi.c
new file mode 100644
index 000000000000..b8d972659314
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/mhi.c
@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include "../mhi.h"
+#include "mhi.h"
+
+static const struct mhi_channel_config ath12k_wifi7_mhi_channels_qcn9274[] = {
+	{
+		.num = 20,
+		.name = "IPCR",
+		.num_elements = 32,
+		.event_ring = 1,
+		.dir = DMA_TO_DEVICE,
+		.ee_mask = 0x4,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+	},
+	{
+		.num = 21,
+		.name = "IPCR",
+		.num_elements = 32,
+		.event_ring = 1,
+		.dir = DMA_FROM_DEVICE,
+		.ee_mask = 0x4,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = true,
+	},
+};
+
+static struct mhi_event_config ath12k_wifi7_mhi_events_qcn9274[] = {
+	{
+		.num_elements = 32,
+		.irq_moderation_ms = 0,
+		.irq = 1,
+		.data_type = MHI_ER_CTRL,
+		.mode = MHI_DB_BRST_DISABLE,
+		.hardware_event = false,
+		.client_managed = false,
+		.offload_channel = false,
+	},
+	{
+		.num_elements = 256,
+		.irq_moderation_ms = 1,
+		.irq = 2,
+		.mode = MHI_DB_BRST_DISABLE,
+		.priority = 1,
+		.hardware_event = false,
+		.client_managed = false,
+		.offload_channel = false,
+	},
+};
+
+const struct mhi_controller_config ath12k_wifi7_mhi_config_qcn9274 = {
+	.max_channels = 30,
+	.timeout_ms = 10000,
+	.use_bounce_buf = false,
+	.buf_len = 0,
+	.num_channels = ARRAY_SIZE(ath12k_wifi7_mhi_channels_qcn9274),
+	.ch_cfg = ath12k_wifi7_mhi_channels_qcn9274,
+	.num_events = ARRAY_SIZE(ath12k_wifi7_mhi_events_qcn9274),
+	.event_cfg = ath12k_wifi7_mhi_events_qcn9274,
+};
+
+static const struct mhi_channel_config ath12k_wifi7_mhi_channels_wcn7850[] = {
+	{
+		.num = 20,
+		.name = "IPCR",
+		.num_elements = 64,
+		.event_ring = 1,
+		.dir = DMA_TO_DEVICE,
+		.ee_mask = 0x4,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+	},
+	{
+		.num = 21,
+		.name = "IPCR",
+		.num_elements = 64,
+		.event_ring = 1,
+		.dir = DMA_FROM_DEVICE,
+		.ee_mask = 0x4,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = true,
+	},
+};
+
+static struct mhi_event_config ath12k_wifi7_mhi_events_wcn7850[] = {
+	{
+		.num_elements = 32,
+		.irq_moderation_ms = 0,
+		.irq = 1,
+		.mode = MHI_DB_BRST_DISABLE,
+		.data_type = MHI_ER_CTRL,
+		.hardware_event = false,
+		.client_managed = false,
+		.offload_channel = false,
+	},
+	{
+		.num_elements = 256,
+		.irq_moderation_ms = 1,
+		.irq = 2,
+		.mode = MHI_DB_BRST_DISABLE,
+		.priority = 1,
+		.hardware_event = false,
+		.client_managed = false,
+		.offload_channel = false,
+	},
+};
+
+const struct mhi_controller_config ath12k_wifi7_mhi_config_wcn7850 = {
+	.max_channels = 128,
+	.timeout_ms = 2000,
+	.use_bounce_buf = false,
+	.buf_len = 8192,
+	.num_channels = ARRAY_SIZE(ath12k_wifi7_mhi_channels_wcn7850),
+	.ch_cfg = ath12k_wifi7_mhi_channels_wcn7850,
+	.num_events = ARRAY_SIZE(ath12k_wifi7_mhi_events_wcn7850),
+	.event_cfg = ath12k_wifi7_mhi_events_wcn7850,
+};
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/mhi.h b/drivers/net/wireless/ath/ath12k/wifi7/mhi.h
new file mode 100644
index 000000000000..2e2dd3503d83
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/mhi.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _ATH12K_WIFI7_MHI_H
+#define _ATH12K_WIFI7_MHI_H
+extern const struct mhi_controller_config ath12k_wifi7_mhi_config_qcn9274;
+extern const struct mhi_controller_config ath12k_wifi7_mhi_config_wcn7850;
+#endif /* _ATH12K_WIFI7_MHI_H */
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/pci.c b/drivers/net/wireless/ath/ath12k/wifi7/pci.c
new file mode 100644
index 000000000000..dedc88858bb0
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/pci.c
@@ -0,0 +1,191 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/pci.h>
+
+#include "../pci.h"
+#include "pci.h"
+#include "../core.h"
+#include "../hif.h"
+#include "../mhi.h"
+#include "hw.h"
+#include "../hal.h"
+#include "dp.h"
+#include "core.h"
+#include "hal.h"
+
+#define QCN9274_DEVICE_ID		0x1109
+#define WCN7850_DEVICE_ID		0x1107
+
+#define ATH12K_PCI_W7_SOC_HW_VERSION_1	1
+#define ATH12K_PCI_W7_SOC_HW_VERSION_2	2
+
+#define TCSR_SOC_HW_VERSION		0x1B00000
+#define TCSR_SOC_HW_VERSION_MAJOR_MASK	GENMASK(11, 8)
+#define TCSR_SOC_HW_VERSION_MINOR_MASK	GENMASK(7, 4)
+
+static const struct pci_device_id ath12k_wifi7_pci_id_table[] = {
+	{ PCI_VDEVICE(QCOM, QCN9274_DEVICE_ID) },
+	{ PCI_VDEVICE(QCOM, WCN7850_DEVICE_ID) },
+	{}
+};
+
+MODULE_DEVICE_TABLE(pci, ath12k_wifi7_pci_id_table);
+
+/* TODO: revisit IRQ mapping for new SRNG's */
+static const struct ath12k_msi_config ath12k_wifi7_msi_config[] = {
+	{
+		.total_vectors = 16,
+		.total_users = 3,
+		.users = (struct ath12k_msi_user[]) {
+			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
+			{ .name = "CE", .num_vectors = 5, .base_vector = 3 },
+			{ .name = "DP", .num_vectors = 8, .base_vector = 8 },
+		},
+	},
+};
+
+static const struct ath12k_pci_ops ath12k_wifi7_pci_ops_qcn9274 = {
+	.wakeup = NULL,
+	.release = NULL,
+};
+
+static int ath12k_wifi7_pci_bus_wake_up(struct ath12k_base *ab)
+{
+	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);
+
+	return mhi_device_get_sync(ab_pci->mhi_ctrl->mhi_dev);
+}
+
+static void ath12k_wifi7_pci_bus_release(struct ath12k_base *ab)
+{
+	struct ath12k_pci *ab_pci = ath12k_pci_priv(ab);
+
+	mhi_device_put(ab_pci->mhi_ctrl->mhi_dev);
+}
+
+static const struct ath12k_pci_ops ath12k_wifi7_pci_ops_wcn7850 = {
+	.wakeup = ath12k_wifi7_pci_bus_wake_up,
+	.release = ath12k_wifi7_pci_bus_release,
+};
+
+static
+void ath12k_wifi7_pci_read_hw_version(struct ath12k_base *ab,
+				      u32 *major, u32 *minor)
+{
+	u32 soc_hw_version;
+
+	soc_hw_version = ath12k_pci_read32(ab, TCSR_SOC_HW_VERSION);
+	*major = u32_get_bits(soc_hw_version, TCSR_SOC_HW_VERSION_MAJOR_MASK);
+	*minor = u32_get_bits(soc_hw_version, TCSR_SOC_HW_VERSION_MINOR_MASK);
+}
+
+static int ath12k_wifi7_pci_probe(struct pci_dev *pdev,
+				  const struct pci_device_id *pci_dev)
+{
+	u32 soc_hw_version_major, soc_hw_version_minor;
+	struct ath12k_pci *ab_pci;
+	struct ath12k_base *ab;
+	int ret;
+
+	ab = pci_get_drvdata(pdev);
+	if (!ab)
+		return -EINVAL;
+
+	ab_pci = ath12k_pci_priv(ab);
+	if (!ab_pci)
+		return -EINVAL;
+
+	switch (pci_dev->device) {
+	case QCN9274_DEVICE_ID:
+		ab_pci->msi_config = &ath12k_wifi7_msi_config[0];
+		ab->static_window_map = true;
+		ab_pci->pci_ops = &ath12k_wifi7_pci_ops_qcn9274;
+		ath12k_wifi7_pci_read_hw_version(ab, &soc_hw_version_major,
+						 &soc_hw_version_minor);
+		ab->target_mem_mode = ath12k_core_get_memory_mode(ab);
+		switch (soc_hw_version_major) {
+		case ATH12K_PCI_W7_SOC_HW_VERSION_2:
+			ab->hw_rev = ATH12K_HW_QCN9274_HW20;
+			break;
+		case ATH12K_PCI_W7_SOC_HW_VERSION_1:
+			ab->hw_rev = ATH12K_HW_QCN9274_HW10;
+			break;
+		default:
+			dev_err(&pdev->dev,
+				"Unknown hardware version found for QCN9274: 0x%x\n",
+				soc_hw_version_major);
+			return -EOPNOTSUPP;
+		}
+		break;
+	case WCN7850_DEVICE_ID:
+		ab->id.bdf_search = ATH12K_BDF_SEARCH_BUS_AND_BOARD;
+		ab_pci->msi_config = &ath12k_wifi7_msi_config[0];
+		ab->static_window_map = false;
+		ab_pci->pci_ops = &ath12k_wifi7_pci_ops_wcn7850;
+		ath12k_wifi7_pci_read_hw_version(ab, &soc_hw_version_major,
+						 &soc_hw_version_minor);
+		ab->target_mem_mode = ATH12K_QMI_MEMORY_MODE_DEFAULT;
+		switch (soc_hw_version_major) {
+		case ATH12K_PCI_W7_SOC_HW_VERSION_2:
+			ab->hw_rev = ATH12K_HW_WCN7850_HW20;
+			break;
+		default:
+			dev_err(&pdev->dev,
+				"Unknown hardware version found for WCN7850: 0x%x\n",
+				soc_hw_version_major);
+			return -EOPNOTSUPP;
+		}
+		break;
+
+	default:
+		dev_err(&pdev->dev, "Unknown Wi-Fi 7 PCI device found: 0x%x\n",
+			pci_dev->device);
+		return -EOPNOTSUPP;
+	}
+
+	ret = ath12k_wifi7_hw_init(ab);
+	if (ret) {
+		dev_err(&pdev->dev, "WiFi-7 hw_init for PCI failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct ath12k_pci_reg_base ath12k_wifi7_reg_base = {
+	.umac_base = HAL_SEQ_WCSS_UMAC_OFFSET,
+	.ce_reg_base = HAL_CE_WFSS_CE_REG_BASE,
+};
+
+static struct ath12k_pci_driver ath12k_wifi7_pci_driver = {
+	.name = "ath12k_wifi7_pci",
+	.id_table = ath12k_wifi7_pci_id_table,
+	.ops.probe = ath12k_wifi7_pci_probe,
+	.reg_base = &ath12k_wifi7_reg_base,
+	.ops.arch_init = ath12k_wifi7_arch_init,
+	.ops.arch_deinit = ath12k_wifi7_arch_deinit,
+};
+
+int ath12k_wifi7_pci_init(void)
+{
+	int ret;
+
+	ret = ath12k_pci_register_driver(ATH12K_DEVICE_FAMILY_WIFI7,
+					 &ath12k_wifi7_pci_driver);
+	if (ret) {
+		pr_err("Failed to register ath12k Wi-Fi 7 driver: %d\n",
+		       ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+void ath12k_wifi7_pci_exit(void)
+{
+	ath12k_pci_unregister_driver(ATH12K_DEVICE_FAMILY_WIFI7);
+}
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/pci.h b/drivers/net/wireless/ath/ath12k/wifi7/pci.h
new file mode 100644
index 000000000000..662a8bab0ce7
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/pci.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+#ifndef ATH12K_PCI_WIFI7_H
+#define ATH12K_PCI_WIFI7_H
+
+int ath12k_wifi7_pci_init(void);
+void ath12k_wifi7_pci_exit(void);
+
+#endif /* ATH12K_PCI_WIFI7_H */
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/wmi.c b/drivers/net/wireless/ath/ath12k/wifi7/wmi.c
new file mode 100644
index 000000000000..c575b44a33f3
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/wmi.c
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include "../core.h"
+#include "wmi.h"
+
+void ath12k_wifi7_wmi_init_qcn9274(struct ath12k_base *ab,
+				   struct ath12k_wmi_resource_config_arg *config)
+{
+	config->num_vdevs = ab->num_radios * TARGET_NUM_VDEVS(ab);
+	config->num_peers = ab->num_radios *
+		ath12k_core_get_max_peers_per_radio(ab);
+	config->num_offload_peers = TARGET_NUM_OFFLD_PEERS;
+	config->num_offload_reorder_buffs = TARGET_NUM_OFFLD_REORDER_BUFFS;
+	config->num_peer_keys = TARGET_NUM_PEER_KEYS;
+	config->ast_skid_limit = TARGET_AST_SKID_LIMIT;
+	config->tx_chain_mask = (1 << ab->target_caps.num_rf_chains) - 1;
+	config->rx_chain_mask = (1 << ab->target_caps.num_rf_chains) - 1;
+	config->rx_timeout_pri[0] = TARGET_RX_TIMEOUT_LO_PRI;
+	config->rx_timeout_pri[1] = TARGET_RX_TIMEOUT_LO_PRI;
+	config->rx_timeout_pri[2] = TARGET_RX_TIMEOUT_LO_PRI;
+	config->rx_timeout_pri[3] = TARGET_RX_TIMEOUT_HI_PRI;
+
+	if (test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags))
+		config->rx_decap_mode = TARGET_DECAP_MODE_RAW;
+	else
+		config->rx_decap_mode = TARGET_DECAP_MODE_NATIVE_WIFI;
+
+	config->scan_max_pending_req = TARGET_SCAN_MAX_PENDING_REQS;
+	config->bmiss_offload_max_vdev = TARGET_BMISS_OFFLOAD_MAX_VDEV;
+	config->roam_offload_max_vdev = TARGET_ROAM_OFFLOAD_MAX_VDEV;
+	config->roam_offload_max_ap_profiles = TARGET_ROAM_OFFLOAD_MAX_AP_PROFILES;
+	config->num_mcast_groups = TARGET_NUM_MCAST_GROUPS;
+	config->num_mcast_table_elems = TARGET_NUM_MCAST_TABLE_ELEMS;
+	config->mcast2ucast_mode = TARGET_MCAST2UCAST_MODE;
+	config->tx_dbg_log_size = TARGET_TX_DBG_LOG_SIZE;
+	config->num_wds_entries = TARGET_NUM_WDS_ENTRIES;
+	config->dma_burst_size = TARGET_DMA_BURST_SIZE;
+	config->rx_skip_defrag_timeout_dup_detection_check =
+		TARGET_RX_SKIP_DEFRAG_TIMEOUT_DUP_DETECTION_CHECK;
+	config->vow_config = TARGET_VOW_CONFIG;
+	config->gtk_offload_max_vdev = TARGET_GTK_OFFLOAD_MAX_VDEV;
+	config->num_msdu_desc = TARGET_NUM_MSDU_DESC;
+	config->beacon_tx_offload_max_vdev = ab->num_radios * TARGET_MAX_BCN_OFFLD;
+	config->rx_batchmode = TARGET_RX_BATCHMODE;
+	/* Indicates host supports peer map v3 and unmap v2 support */
+	config->peer_map_unmap_version = 0x32;
+	config->twt_ap_pdev_count = ab->num_radios;
+	config->twt_ap_sta_count = 1000;
+	config->ema_max_vap_cnt = ab->num_radios;
+	config->ema_max_profile_period = TARGET_EMA_MAX_PROFILE_PERIOD;
+	config->beacon_tx_offload_max_vdev += config->ema_max_vap_cnt;
+
+	if (test_bit(WMI_TLV_SERVICE_PEER_METADATA_V1A_V1B_SUPPORT, ab->wmi_ab.svc_map))
+		config->peer_metadata_ver = ATH12K_PEER_METADATA_V1B;
+}
+
+void ath12k_wifi7_wmi_init_wcn7850(struct ath12k_base *ab,
+				   struct ath12k_wmi_resource_config_arg *config)
+{
+	config->num_vdevs = 4;
+	config->num_peers = 16;
+	config->num_tids = 32;
+
+	config->num_offload_peers = 3;
+	config->num_offload_reorder_buffs = 3;
+	config->num_peer_keys = TARGET_NUM_PEER_KEYS;
+	config->ast_skid_limit = TARGET_AST_SKID_LIMIT;
+	config->tx_chain_mask = (1 << ab->target_caps.num_rf_chains) - 1;
+	config->rx_chain_mask = (1 << ab->target_caps.num_rf_chains) - 1;
+	config->rx_timeout_pri[0] = TARGET_RX_TIMEOUT_LO_PRI;
+	config->rx_timeout_pri[1] = TARGET_RX_TIMEOUT_LO_PRI;
+	config->rx_timeout_pri[2] = TARGET_RX_TIMEOUT_LO_PRI;
+	config->rx_timeout_pri[3] = TARGET_RX_TIMEOUT_HI_PRI;
+	config->rx_decap_mode = TARGET_DECAP_MODE_NATIVE_WIFI;
+	config->scan_max_pending_req = TARGET_SCAN_MAX_PENDING_REQS;
+	config->bmiss_offload_max_vdev = TARGET_BMISS_OFFLOAD_MAX_VDEV;
+	config->roam_offload_max_vdev = TARGET_ROAM_OFFLOAD_MAX_VDEV;
+	config->roam_offload_max_ap_profiles = TARGET_ROAM_OFFLOAD_MAX_AP_PROFILES;
+	config->num_mcast_groups = 0;
+	config->num_mcast_table_elems = 0;
+	config->mcast2ucast_mode = 0;
+	config->tx_dbg_log_size = TARGET_TX_DBG_LOG_SIZE;
+	config->num_wds_entries = 0;
+	config->dma_burst_size = 0;
+	config->rx_skip_defrag_timeout_dup_detection_check = 0;
+	config->vow_config = TARGET_VOW_CONFIG;
+	config->gtk_offload_max_vdev = 2;
+	config->num_msdu_desc = 0x400;
+	config->beacon_tx_offload_max_vdev = 2;
+	config->rx_batchmode = TARGET_RX_BATCHMODE;
+
+	config->peer_map_unmap_version = 0x1;
+	config->use_pdev_id = 1;
+	config->max_frag_entries = 0xa;
+	config->num_tdls_vdevs = 0x1;
+	config->num_tdls_conn_table_entries = 8;
+	config->beacon_tx_offload_max_vdev = 0x2;
+	config->num_multicast_filter_entries = 0x20;
+	config->num_wow_filters = 0x16;
+	config->num_keep_alive_pattern = 0;
+}
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/wmi.h b/drivers/net/wireless/ath/ath12k/wifi7/wmi.h
new file mode 100644
index 000000000000..ae74e176fa2d
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/wifi7/wmi.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef ATH12K_WMI_WIFI7_H
+#define ATH12K_WMI_WIFI7_H
+
+void ath12k_wifi7_wmi_init_qcn9274(struct ath12k_base *ab,
+				   struct ath12k_wmi_resource_config_arg *config);
+void ath12k_wifi7_wmi_init_wcn7850(struct ath12k_base *ab,
+				   struct ath12k_wmi_resource_config_arg *config);
+
+#endif
diff --git a/drivers/net/wireless/ath/ath12k/wmi.c b/drivers/net/wireless/ath/ath12k/wmi.c
index be8b2943094f..150b04d0a21c 100644
--- a/drivers/net/wireless/ath/ath12k/wmi.c
+++ b/drivers/net/wireless/ath/ath12k/wmi.c
@@ -206,103 +206,6 @@ static __le32 ath12k_wmi_tlv_cmd_hdr(u32 cmd, u32 len)
 	return ath12k_wmi_tlv_hdr(cmd, len - TLV_HDR_SIZE);
 }
 
-void ath12k_wmi_init_qcn9274(struct ath12k_base *ab,
-			     struct ath12k_wmi_resource_config_arg *config)
-{
-	config->num_vdevs = ab->num_radios * TARGET_NUM_VDEVS(ab);
-	config->num_peers = ab->num_radios *
-		ath12k_core_get_max_peers_per_radio(ab);
-	config->num_offload_peers = TARGET_NUM_OFFLD_PEERS;
-	config->num_offload_reorder_buffs = TARGET_NUM_OFFLD_REORDER_BUFFS;
-	config->num_peer_keys = TARGET_NUM_PEER_KEYS;
-	config->ast_skid_limit = TARGET_AST_SKID_LIMIT;
-	config->tx_chain_mask = (1 << ab->target_caps.num_rf_chains) - 1;
-	config->rx_chain_mask = (1 << ab->target_caps.num_rf_chains) - 1;
-	config->rx_timeout_pri[0] = TARGET_RX_TIMEOUT_LO_PRI;
-	config->rx_timeout_pri[1] = TARGET_RX_TIMEOUT_LO_PRI;
-	config->rx_timeout_pri[2] = TARGET_RX_TIMEOUT_LO_PRI;
-	config->rx_timeout_pri[3] = TARGET_RX_TIMEOUT_HI_PRI;
-
-	if (test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags))
-		config->rx_decap_mode = TARGET_DECAP_MODE_RAW;
-	else
-		config->rx_decap_mode = TARGET_DECAP_MODE_NATIVE_WIFI;
-
-	config->scan_max_pending_req = TARGET_SCAN_MAX_PENDING_REQS;
-	config->bmiss_offload_max_vdev = TARGET_BMISS_OFFLOAD_MAX_VDEV;
-	config->roam_offload_max_vdev = TARGET_ROAM_OFFLOAD_MAX_VDEV;
-	config->roam_offload_max_ap_profiles = TARGET_ROAM_OFFLOAD_MAX_AP_PROFILES;
-	config->num_mcast_groups = TARGET_NUM_MCAST_GROUPS;
-	config->num_mcast_table_elems = TARGET_NUM_MCAST_TABLE_ELEMS;
-	config->mcast2ucast_mode = TARGET_MCAST2UCAST_MODE;
-	config->tx_dbg_log_size = TARGET_TX_DBG_LOG_SIZE;
-	config->num_wds_entries = TARGET_NUM_WDS_ENTRIES;
-	config->dma_burst_size = TARGET_DMA_BURST_SIZE;
-	config->rx_skip_defrag_timeout_dup_detection_check =
-		TARGET_RX_SKIP_DEFRAG_TIMEOUT_DUP_DETECTION_CHECK;
-	config->vow_config = TARGET_VOW_CONFIG;
-	config->gtk_offload_max_vdev = TARGET_GTK_OFFLOAD_MAX_VDEV;
-	config->num_msdu_desc = TARGET_NUM_MSDU_DESC;
-	config->beacon_tx_offload_max_vdev = ab->num_radios * TARGET_MAX_BCN_OFFLD;
-	config->rx_batchmode = TARGET_RX_BATCHMODE;
-	/* Indicates host supports peer map v3 and unmap v2 support */
-	config->peer_map_unmap_version = 0x32;
-	config->twt_ap_pdev_count = ab->num_radios;
-	config->twt_ap_sta_count = 1000;
-	config->ema_max_vap_cnt = ab->num_radios;
-	config->ema_max_profile_period = TARGET_EMA_MAX_PROFILE_PERIOD;
-	config->beacon_tx_offload_max_vdev += config->ema_max_vap_cnt;
-
-	if (test_bit(WMI_TLV_SERVICE_PEER_METADATA_V1A_V1B_SUPPORT, ab->wmi_ab.svc_map))
-		config->peer_metadata_ver = ATH12K_PEER_METADATA_V1B;
-}
-
-void ath12k_wmi_init_wcn7850(struct ath12k_base *ab,
-			     struct ath12k_wmi_resource_config_arg *config)
-{
-	config->num_vdevs = 4;
-	config->num_peers = 16;
-	config->num_tids = 32;
-
-	config->num_offload_peers = 3;
-	config->num_offload_reorder_buffs = 3;
-	config->num_peer_keys = TARGET_NUM_PEER_KEYS;
-	config->ast_skid_limit = TARGET_AST_SKID_LIMIT;
-	config->tx_chain_mask = (1 << ab->target_caps.num_rf_chains) - 1;
-	config->rx_chain_mask = (1 << ab->target_caps.num_rf_chains) - 1;
-	config->rx_timeout_pri[0] = TARGET_RX_TIMEOUT_LO_PRI;
-	config->rx_timeout_pri[1] = TARGET_RX_TIMEOUT_LO_PRI;
-	config->rx_timeout_pri[2] = TARGET_RX_TIMEOUT_LO_PRI;
-	config->rx_timeout_pri[3] = TARGET_RX_TIMEOUT_HI_PRI;
-	config->rx_decap_mode = TARGET_DECAP_MODE_NATIVE_WIFI;
-	config->scan_max_pending_req = TARGET_SCAN_MAX_PENDING_REQS;
-	config->bmiss_offload_max_vdev = TARGET_BMISS_OFFLOAD_MAX_VDEV;
-	config->roam_offload_max_vdev = TARGET_ROAM_OFFLOAD_MAX_VDEV;
-	config->roam_offload_max_ap_profiles = TARGET_ROAM_OFFLOAD_MAX_AP_PROFILES;
-	config->num_mcast_groups = 0;
-	config->num_mcast_table_elems = 0;
-	config->mcast2ucast_mode = 0;
-	config->tx_dbg_log_size = TARGET_TX_DBG_LOG_SIZE;
-	config->num_wds_entries = 0;
-	config->dma_burst_size = 0;
-	config->rx_skip_defrag_timeout_dup_detection_check = 0;
-	config->vow_config = TARGET_VOW_CONFIG;
-	config->gtk_offload_max_vdev = 2;
-	config->num_msdu_desc = 0x400;
-	config->beacon_tx_offload_max_vdev = 2;
-	config->rx_batchmode = TARGET_RX_BATCHMODE;
-
-	config->peer_map_unmap_version = 0x1;
-	config->use_pdev_id = 1;
-	config->max_frag_entries = 0xa;
-	config->num_tdls_vdevs = 0x1;
-	config->num_tdls_conn_table_entries = 8;
-	config->beacon_tx_offload_max_vdev = 0x2;
-	config->num_multicast_filter_entries = 0x20;
-	config->num_wow_filters = 0x16;
-	config->num_keep_alive_pattern = 0;
-}
-
 #define PRIMAP(_hw_mode_) \
 	[_hw_mode_] = _hw_mode_##_PRI
 
@@ -4217,6 +4120,7 @@ int ath12k_wmi_set_hw_mode(struct ath12k_base *ab,
 
 int ath12k_wmi_cmd_init(struct ath12k_base *ab)
 {
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	struct ath12k_wmi_base *wmi_ab = &ab->wmi_ab;
 	struct ath12k_wmi_init_cmd_arg arg = {};
 
@@ -4237,7 +4141,7 @@ int ath12k_wmi_cmd_init(struct ath12k_base *ab)
 	arg.num_band_to_mac = ab->num_radios;
 	ath12k_fill_band_to_mac_param(ab, arg.band_to_mac);
 
-	ab->dp.peer_metadata_ver = arg.res_cfg.peer_metadata_ver;
+	dp->peer_metadata_ver = arg.res_cfg.peer_metadata_ver;
 
 	return ath12k_init_cmd_send(&wmi_ab->wmi[0], &arg);
 }
@@ -7379,8 +7283,8 @@ static void ath12k_peer_sta_kickout_event(struct ath12k_base *ab, struct sk_buff
 	struct wmi_peer_sta_kickout_arg arg = {};
 	struct ath12k_link_vif *arvif;
 	struct ieee80211_sta *sta;
-	struct ath12k_peer *peer;
-	unsigned int link_id;
+	struct ath12k_sta *ahsta;
+	struct ath12k_link_sta *arsta;
 	struct ath12k *ar;
 
 	if (ath12k_pull_peer_sta_kickout_ev(ab, skb, &arg) != 0) {
@@ -7392,42 +7296,24 @@ static void ath12k_peer_sta_kickout_event(struct ath12k_base *ab, struct sk_buff
 
 	spin_lock_bh(&ab->base_lock);
 
-	peer = ath12k_peer_find_by_addr(ab, arg.mac_addr);
+	arsta = ath12k_link_sta_find_by_addr(ab, arg.mac_addr);
 
-	if (!peer) {
-		ath12k_warn(ab, "peer not found %pM\n",
+	if (!arsta) {
+		ath12k_warn(ab, "arsta not found %pM\n",
 			    arg.mac_addr);
 		goto exit;
 	}
 
-	arvif = ath12k_mac_get_arvif_by_vdev_id(ab, peer->vdev_id);
+	arvif = arsta->arvif;
 	if (!arvif) {
-		ath12k_warn(ab, "invalid vdev id in peer sta kickout ev %d",
-			    peer->vdev_id);
+		ath12k_warn(ab, "invalid arvif in peer sta kickout ev for STA %pM",
+			    arg.mac_addr);
 		goto exit;
 	}
 
 	ar = arvif->ar;
-
-	if (peer->mlo) {
-		sta = ieee80211_find_sta_by_link_addrs(ath12k_ar_to_hw(ar),
-						       arg.mac_addr,
-						       NULL, &link_id);
-		if (peer->link_id != link_id) {
-			ath12k_warn(ab,
-				    "Spurious quick kickout for MLO STA %pM with invalid link_id, peer: %d, sta: %d\n",
-				    arg.mac_addr, peer->link_id, link_id);
-			goto exit;
-		}
-	} else {
-		sta = ieee80211_find_sta_by_ifaddr(ath12k_ar_to_hw(ar),
-						   arg.mac_addr, NULL);
-	}
-	if (!sta) {
-		ath12k_warn(ab, "Spurious quick kickout for %sSTA %pM\n",
-			    peer->mlo ? "MLO " : "", arg.mac_addr);
-		goto exit;
-	}
+	ahsta = arsta->ahsta;
+	sta = ath12k_ahsta_to_sta(ahsta);
 
 	ath12k_dbg(ab, ATH12K_DBG_WMI,
 		   "peer sta kickout event %pM reason: %d rssi: %d\n",
diff --git a/drivers/net/wireless/ath/ath12k/wmi.h b/drivers/net/wireless/ath/ath12k/wmi.h
index f99fced1610e..65f0d7ed4178 100644
--- a/drivers/net/wireless/ath/ath12k/wmi.h
+++ b/drivers/net/wireless/ath/ath12k/wmi.h
@@ -9,6 +9,7 @@
 
 #include <net/mac80211.h>
 #include "htc.h"
+#include "cmn_defs.h"
 
 /* Naming conventions for structures:
  *
@@ -5151,8 +5152,6 @@ struct wmi_probe_tmpl_cmd {
 	__le32 buf_len;
 } __packed;
 
-#define MAX_RADIOS 2
-
 #define WMI_MLO_CMD_TIMEOUT_HZ (5 * HZ)
 #define WMI_SERVICE_READY_TIMEOUT_HZ (5 * HZ)
 #define WMI_SEND_TIMEOUT_HZ (3 * HZ)
@@ -6323,10 +6322,6 @@ struct ath12k_wmi_rssi_dbm_conv_info_arg {
 	s8 min_nf_dbm;
 };
 
-void ath12k_wmi_init_qcn9274(struct ath12k_base *ab,
-			     struct ath12k_wmi_resource_config_arg *config);
-void ath12k_wmi_init_wcn7850(struct ath12k_base *ab,
-			     struct ath12k_wmi_resource_config_arg *config);
 int ath12k_wmi_cmd_send(struct ath12k_wmi_pdev *wmi, struct sk_buff *skb,
 			u32 cmd_id);
 struct sk_buff *ath12k_wmi_alloc_skb(struct ath12k_wmi_base *wmi_sc, u32 len);
diff --git a/drivers/net/wireless/ath/ath12k/wow.c b/drivers/net/wireless/ath/ath12k/wow.c
index e8481626f194..f56ec44a1636 100644
--- a/drivers/net/wireless/ath/ath12k/wow.c
+++ b/drivers/net/wireless/ath/ath12k/wow.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2020 The Linux Foundation. All rights reserved.
- * Copyright (c) 2022-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #include <linux/delay.h>
@@ -919,6 +919,7 @@ int ath12k_wow_op_suspend(struct ieee80211_hw *hw,
 exit:
 	return ret ? 1 : 0;
 }
+EXPORT_SYMBOL(ath12k_wow_op_suspend);
 
 void ath12k_wow_op_set_wakeup(struct ieee80211_hw *hw, bool enabled)
 {
@@ -929,6 +930,7 @@ void ath12k_wow_op_set_wakeup(struct ieee80211_hw *hw, bool enabled)
 
 	device_set_wakeup_enable(ar->ab->dev, enabled);
 }
+EXPORT_SYMBOL(ath12k_wow_op_set_wakeup);
 
 int ath12k_wow_op_resume(struct ieee80211_hw *hw)
 {
@@ -1001,6 +1003,7 @@ int ath12k_wow_op_resume(struct ieee80211_hw *hw)
 
 	return ret;
 }
+EXPORT_SYMBOL(ath12k_wow_op_resume);
 
 int ath12k_wow_init(struct ath12k *ar)
 {
diff --git a/drivers/net/wireless/ath/ath6kl/debug.c b/drivers/net/wireless/ath/ath6kl/debug.c
index b837d31416df..84403aab21c0 100644
--- a/drivers/net/wireless/ath/ath6kl/debug.c
+++ b/drivers/net/wireless/ath/ath6kl/debug.c
@@ -19,6 +19,7 @@
 
 #include <linux/skbuff.h>
 #include <linux/fs.h>
+#include <linux/hex.h>
 #include <linux/vmalloc.h>
 #include <linux/export.h>
 
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/bz.c b/drivers/net/wireless/intel/iwlwifi/cfg/bz.c
index d25445bd1e5c..77db8c75e6e2 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/bz.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/bz.c
@@ -19,12 +19,6 @@
 #define IWL_BZ_SMEM_OFFSET		0x400000
 #define IWL_BZ_SMEM_LEN			0xD0000
 
-#define IWL_BZ_A_FM_B_FW_PRE		"iwlwifi-bz-a0-fm-b0"
-#define IWL_BZ_A_FM_C_FW_PRE		"iwlwifi-bz-a0-fm-c0"
-#define IWL_BZ_A_FM4_B_FW_PRE		"iwlwifi-bz-a0-fm4-b0"
-#define IWL_GL_B_FM_B_FW_PRE		"iwlwifi-gl-b0-fm-b0"
-#define IWL_GL_C_FM_C_FW_PRE		"iwlwifi-gl-c0-fm-c0"
-
 static const struct iwl_family_base_params iwl_bz_base = {
 	.num_of_queues = 512,
 	.max_tfd_queue_size = 65536,
@@ -100,9 +94,3 @@ const struct iwl_mac_cfg iwl_gl_mac_cfg = {
 	.xtal_latency = 12000,
 	.low_latency_xtal = true,
 };
-
-IWL_CORE_FW(IWL_BZ_A_FM_B_FW_PRE, IWL_BZ_UCODE_CORE_MAX);
-IWL_CORE_FW(IWL_BZ_A_FM_C_FW_PRE, IWL_BZ_UCODE_CORE_MAX);
-IWL_CORE_FW(IWL_BZ_A_FM4_B_FW_PRE, IWL_BZ_UCODE_CORE_MAX);
-IWL_CORE_FW(IWL_GL_B_FM_B_FW_PRE, IWL_BZ_UCODE_CORE_MAX);
-IWL_CORE_FW(IWL_GL_C_FM_C_FW_PRE, IWL_BZ_UCODE_CORE_MAX);
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c b/drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c
index fd82050e33a3..ad2536f53084 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c
@@ -5,6 +5,12 @@
  */
 #include "iwl-config.h"
 
+#define IWL_BZ_A_FM_B_FW_PRE		"iwlwifi-bz-a0-fm-b0"
+#define IWL_BZ_A_FM_C_FW_PRE		"iwlwifi-bz-a0-fm-c0"
+#define IWL_BZ_A_FM4_B_FW_PRE		"iwlwifi-bz-a0-fm4-b0"
+#define IWL_GL_B_FM_B_FW_PRE		"iwlwifi-gl-b0-fm-b0"
+#define IWL_GL_C_FM_C_FW_PRE		"iwlwifi-gl-c0-fm-c0"
+
 /* NVM versions */
 #define IWL_FM_NVM_VERSION		0x0a1d
 
@@ -50,3 +56,9 @@ const char iwl_be201_name[] = "Intel(R) Wi-Fi 7 BE201 320MHz";
 const char iwl_be200_name[] = "Intel(R) Wi-Fi 7 BE200 320MHz";
 const char iwl_be202_name[] = "Intel(R) Wi-Fi 7 BE202 160MHz";
 const char iwl_be401_name[] = "Intel(R) Wi-Fi 7 BE401 320MHz";
+
+IWL_CORE_FW(IWL_BZ_A_FM_B_FW_PRE, IWL_BZ_UCODE_CORE_MAX);
+IWL_CORE_FW(IWL_BZ_A_FM_C_FW_PRE, IWL_BZ_UCODE_CORE_MAX);
+IWL_CORE_FW(IWL_BZ_A_FM4_B_FW_PRE, IWL_BZ_UCODE_CORE_MAX);
+IWL_CORE_FW(IWL_GL_B_FM_B_FW_PRE, IWL_BZ_UCODE_CORE_MAX);
+IWL_CORE_FW(IWL_GL_C_FM_C_FW_PRE, IWL_BZ_UCODE_CORE_MAX);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h b/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
index f76cea6e9ec8..f260115abc5a 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
@@ -65,6 +65,18 @@ enum iwl_mac_conf_subcmd_ids {
 	 * @TWT_OPERATION_CMD: &struct iwl_twt_operation_cmd
 	 */
 	TWT_OPERATION_CMD = 0x10,
+	/**
+	 * @NAN_CFG_CMD: &struct iwl_nan_config_cmd
+	 */
+	NAN_CFG_CMD = 0x12,
+	/**
+	 * @NAN_DW_END_NOTIF: &struct iwl_nan_dw_end_notif
+	 */
+	NAN_DW_END_NOTIF = 0xf4,
+	/**
+	 * @NAN_JOINED_CLUSTER_NOTIF: &struct iwl_nan_cluster_notif
+	 */
+	NAN_JOINED_CLUSTER_NOTIF = 0xf5,
 	/**
 	 * @MISSED_BEACONS_NOTIF: &struct iwl_missed_beacons_notif
 	 */
@@ -491,23 +503,37 @@ enum iwl_link_modify_bandwidth {
 	IWL_LINK_MODIFY_BW_320,
 };
 
+/**
+ * enum iwl_npca_flags - NPCA flags
+ * @IWL_NPCA_FLAG_MAC_HDR_BASED: MAC header based NPCA operation
+ *	permitted in the BSS (MOPLEN)
+ */
+enum iwl_npca_flags {
+	IWL_NPCA_FLAG_MAC_HDR_BASED = BIT(0),
+}; /* NPCA_FLAG_E */
+
 /**
  * struct iwl_npca_params - NPCA parameters (non-primary channel access)
  *
+ * @dis_subch_bmap: disabled subchannel bitmap for NPCA
  * @switch_delay: after switch, delay TX according to destination AP
  * @switch_back_delay: switch back to control channel before OBSS frame end
+ * @initial_qsrc: Indicates the value that is used to initialize the
+ *	EDCAF QSRC[AC] variables
  * @min_dur_threshold: minimum PPDU time to switch to the non-primary
- *	NPCA channel
- * @flags: NPCA flags - bit 0: puncturing allowed, bit 1: new TX allowed
+ *	NPCA channel (usec)
+ * @flags: NPCA flags, see &enum iwl_npca_flags
  * @reserved: reserved for alignment purposes
  */
 struct iwl_npca_params {
+	__le16 dis_subch_bmap;
 	u8 switch_delay;
 	u8 switch_back_delay;
-	__le16 min_dur_threshold;
-	__le16 flags;
-	__le16 reserved;
-} __packed; /* NPCA_PARAM_API_S_VER_1 */
+	u8 initial_qsrc;
+	u8 min_dur_threshold;
+	u8 flags;
+	u8 reserved;
+} __packed; /* NPCA_PARAM_API_S_VER_2 */
 
 /**
  * struct iwl_link_config_cmd - command structure to configure the LINK context
@@ -618,7 +644,8 @@ struct iwl_link_config_cmd {
 	struct iwl_npca_params npca_params; /* since _VER_7 */
 	struct iwl_ac_qos prio_edca_params; /* since _VER_7 */
 	__le32 reserved3[4];
-} __packed; /* LINK_CONTEXT_CONFIG_CMD_API_S_VER_1, _VER_2, _VER_3, _VER_4, _VER_5, _VER_6, _VER_7 */
+} __packed; /* LINK_CONTEXT_CONFIG_CMD_API_S_VER_1, _VER_2, _VER_3, _VER_4,
+	     *				    _VER_5, _VER_6, _VER_7, _VER_8 */
 
 /* Currently FW supports link ids in the range 0-3 and can have
  * at most two active links for each vif.
@@ -990,4 +1017,121 @@ struct iwl_twt_operation_cmd {
 	u8 ul_tid_bitmap;
 } __packed; /* TWT_OPERATION_API_S_VER_1 */
 
+enum iwl_nan_band {
+	IWL_NAN_BAND_5GHZ = 0,
+	IWL_NAN_BAND_2GHZ = 1,
+	IWL_NUM_NAN_BANDS,
+};
+
+/**
+ * struct iwl_nan_band_config - NAN band configuration
+ *
+ * @rssi_close: RSSI threshold for close proximity in dBm
+ * @rssi_middle: RSSI threshold for middle proximity in dBm
+ * @dw_interval: Discovery Window (DW) interval for synchronization beacons and
+ *	SDFs. Valid values of DW interval are: 1, 2, 3, 4 and 5 corresponding to
+ *	1, 2, 4, 8, and 16 DWs.
+ * @reserved: reserved
+ */
+struct iwl_nan_band_config {
+	u8 rssi_close;
+	u8 rssi_middle;
+	u8 dw_interval;
+	u8 reserved;
+}; /* NAN_BAND_SPECIFIC_CONFIG_API_S_VER_1 */
+
+/**
+ * enum iwl_nan_flags - flags for NAN configuration
+ *
+ * @IWL_NAN_FLAG_DW_END_NOTIF_ENABLED: indicates that the host wants to receive
+ *	notifications when a DW ends.
+ */
+enum iwl_nan_flags {
+	IWL_NAN_FLAG_DW_END_NOTIF_ENABLED = BIT(0),
+};
+
+/**
+ * struct iwl_nan_config_cmd - NAN configuration command
+ *
+ * @action: action to perform, see &enum iwl_ctxt_action
+ * @nmi_addr: NAN Management Interface (NMI) address
+ * @reserved_for_nmi_addr: reserved
+ * @discovery_beacon_interval: discovery beacon interval in TUs
+ * @cluster_id: local cluster ID, in case the local device starts a cluster
+ * @sta_id: station ID of the NAN station
+ * @hb_channel: channel for 5 GHz if the device supports operation on 5 GHz.
+ *	Valid values are 44 and 149, which correspond to the 5 GHz channel, and
+ *	0 which means that NAN operation on the 5 GHz band is disabled.
+ * @master_pref: master preference
+ * @dwell_time: dwell time on the discovery channel during scan (milliseconds).
+ *	If set to 0, the dwell time is determined by the firmware.
+ * @scan_period: scan period in seconds. If set to 0, the scan period is
+ *	determined by the firmware.
+ * @flags: flags for NAN configuration, see &enum iwl_nan_flags
+ * @band_config: band configuration for NAN, one for each band
+ * @nan_attr_len: length of the NAN attributes to be added to the beacon (bytes)
+ * @nan_vendor_elems_len: length of the NAN vendor elements to be added to the
+ *	beacon (bytes)
+ * @beacon_data: variable length data that contains the NAN attributes
+ *	(&nan_attr_len) followed by the NAN vendor elements
+ *	(&nan_vendor_elems_len).
+ */
+struct iwl_nan_config_cmd {
+	__le32 action;
+	u8 nmi_addr[6];
+	__le16 reserved_for_nmi_addr;
+	__le32 discovery_beacon_interval;
+
+	__le16 cluster_id;
+	u8 sta_id;
+	u8 hb_channel;
+
+	u8 master_pref;
+	u8 dwell_time;
+	u8 scan_period;
+	u8 flags;
+
+	struct iwl_nan_band_config band_config[IWL_NUM_NAN_BANDS];
+
+	__le32 nan_attr_len;
+	__le32 nan_vendor_elems_len;
+	u8 beacon_data[];
+} __packed; /*  NAN_CONFIG_CMD_API_S_VER_1 */
+
+/**
+ * enum iwl_nan_cluster_notif_flags - flags for the cluster notification
+ *
+ * @IWL_NAN_CLUSTER_NOTIF_FLAG_NEW_CLUSTER: indicates that the device has
+ *	started a new cluster. If not set, the device has joined an existing
+ *	cluster.
+ */
+enum iwl_nan_cluster_notif_flags {
+	IWL_NAN_CLUSTER_NOTIF_FLAG_NEW_CLUSTER	= BIT(0),
+}; /* NAN_JOINED_CLUSTER_FLAG_E_VER_1 */
+
+/**
+ * struct iwl_nan_cluster_notif - event sent when the device starts or joins a
+ *	NAN cluster.
+ *
+ * @cluster_id: cluster ID
+ * @flags: combination of &enum iwl_nan_cluster_notif_flags
+ * @reserved: reserved
+ */
+struct iwl_nan_cluster_notif {
+	__le16 cluster_id;
+	u8 flags;
+	u8 reserved;
+}; /* NAN_JOINED_CLUSTER_NTF_API_S_VER_1 */
+
+/**
+ * struct iwl_nan_dw_end_notif - sent to notify the host the end of a DW.
+ *
+ * @band: band on which the DW ended. See &enum iwl_nan_band.
+ * @reserved: reserved
+ */
+struct iwl_nan_dw_end_notif {
+	u8 band;
+	u8 reserved[3];
+} __packed; /* NAN_DW_END_NTF_API_S_VER_1 */
+
 #endif /* __iwl_fw_api_mac_cfg_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/power.h b/drivers/net/wireless/intel/iwlwifi/fw/api/power.h
index 535864e22626..0cd8a12e0f7c 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/power.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/power.h
@@ -766,7 +766,7 @@ enum iwl_6ghz_ap_type {
  * AP_TX_POWER_CONSTRAINTS_CMD
  * Used for VLP/LPI/AFC Access Point power constraints for 6GHz channels
  * @link_id: linkId
- * @ap_type: see &enum iwl_ap_type
+ * @ap_type: see &enum iwl_6ghz_ap_type
  * @eirp_pwr: 8-bit 2s complement signed integer in the range
  *	-64 dBm to 63 dBm with a 0.5 dB step
  *	default &DEFAULT_TPE_TX_POWER (no maximum limit)
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h b/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h
index 9c464e7aba10..ae6be3ed32f8 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h
@@ -73,6 +73,7 @@ enum iwl_tlc_mng_cfg_chains {
  * @IWL_TLC_MNG_MODE_VHT: enable VHT
  * @IWL_TLC_MNG_MODE_HE: enable HE
  * @IWL_TLC_MNG_MODE_EHT: enable EHT
+ * @IWL_TLC_MNG_MODE_UHR: enable UHR
  */
 enum iwl_tlc_mng_cfg_mode {
 	IWL_TLC_MNG_MODE_CCK = 0,
@@ -82,6 +83,7 @@ enum iwl_tlc_mng_cfg_mode {
 	IWL_TLC_MNG_MODE_VHT,
 	IWL_TLC_MNG_MODE_HE,
 	IWL_TLC_MNG_MODE_EHT,
+	IWL_TLC_MNG_MODE_UHR,
 };
 
 /**
@@ -205,7 +207,7 @@ struct iwl_tlc_config_cmd_v4 {
 } __packed; /* TLC_MNG_CONFIG_CMD_API_S_VER_4 */
 
 /**
- * struct iwl_tlc_config_cmd - TLC configuration
+ * struct iwl_tlc_config_cmd_v5 - TLC configuration
  * @sta_id: station id
  * @reserved1: reserved
  * @max_ch_width: max supported channel width from &enum iwl_tlc_mng_cfg_cw
@@ -221,7 +223,7 @@ struct iwl_tlc_config_cmd_v4 {
  * @max_tx_op: max TXOP in uSecs for all AC (BK, BE, VO, VI),
  *	       set zero for no limit.
  */
-struct iwl_tlc_config_cmd {
+struct iwl_tlc_config_cmd_v5 {
 	u8 sta_id;
 	u8 reserved1[3];
 	u8 max_ch_width;
@@ -235,6 +237,38 @@ struct iwl_tlc_config_cmd {
 	__le16 max_tx_op;
 } __packed; /* TLC_MNG_CONFIG_CMD_API_S_VER_5 */
 
+/**
+ * struct iwl_tlc_config_cmd - TLC configuration
+ * @sta_mask: station mask (in NAN we can have multiple logical stations of
+ *	the same peer (with the same TLC configuration)).
+ * @phy_id: the phy id to used for this TLC configuration
+ * @max_ch_width: max supported channel width from &enum iwl_tlc_mng_cfg_cw
+ * @mode: &enum iwl_tlc_mng_cfg_mode
+ * @chains: bitmask of &enum iwl_tlc_mng_cfg_chains
+ * @sgi_ch_width_supp: bitmap of SGI support per channel width
+ *		       use BIT(&enum iwl_tlc_mng_cfg_cw)
+ * @flags: bitmask of &enum iwl_tlc_mng_cfg_flags
+ * @non_ht_rates: bitmap of supported legacy rates
+ * @ht_rates: bitmap of &enum iwl_tlc_mng_ht_rates, per <nss, channel-width>
+ *	      pair (0 - 80mhz width and below, 1 - 160mhz, 2 - 320mhz).
+ * @max_mpdu_len: max MPDU length, in bytes
+ * @max_tx_op: max TXOP in uSecs for all AC (BK, BE, VO, VI),
+ *	       set zero for no limit.
+ */
+struct iwl_tlc_config_cmd {
+	__le32 sta_mask;
+	__le32 phy_id;
+	u8 max_ch_width;
+	u8 mode;
+	u8 chains;
+	u8 sgi_ch_width_supp;
+	__le16 flags;
+	__le16 non_ht_rates;
+	__le32 ht_rates[IWL_TLC_NSS_MAX][IWL_TLC_MCS_PER_BW_NUM_V4];
+	__le16 max_mpdu_len;
+	__le16 max_tx_op;
+} __packed; /* TLC_MNG_CONFIG_CMD_API_S_VER_6 */
+
 /**
  * enum iwl_tlc_update_flags - updated fields
  * @IWL_TLC_NOTIF_FLAG_RATE: last initial rate update
@@ -706,10 +740,11 @@ enum {
 #define RATE_MCS_HE_SU_4_LTF		3
 #define RATE_MCS_HE_SU_4_LTF_08_GI	4
 
-/* Bit 24-23: HE type. (0) SU, (1) SU_EXT, (2) MU, (3) trigger based */
+/* Bit 24-23: HE type. (0) SU, (1) HE SU_EXT/UHR ELR, (2) MU, (3) trigger based */
 #define RATE_MCS_HE_TYPE_POS		23
 #define RATE_MCS_HE_TYPE_SU		(0 << RATE_MCS_HE_TYPE_POS)
 #define RATE_MCS_HE_TYPE_EXT_SU		(1 << RATE_MCS_HE_TYPE_POS)
+#define RATE_MCS_HE_TYPE_UHR_ELR	(1 << RATE_MCS_HE_TYPE_POS)
 #define RATE_MCS_HE_TYPE_MU		(2 << RATE_MCS_HE_TYPE_POS)
 #define RATE_MCS_HE_TYPE_TRIG		(3 << RATE_MCS_HE_TYPE_POS)
 #define RATE_MCS_HE_TYPE_MSK		(3 << RATE_MCS_HE_TYPE_POS)
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h b/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
index 3ed7e0807b90..ac6c1ef2cbcd 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
@@ -1062,13 +1062,37 @@ struct iwl_vht_sigs {
 #define OFDM_RX_FRAME_VHT_NUM_OF_DATA_SYM		0x000007ff
 #define OFDM_RX_FRAME_VHT_NUM_OF_DATA_SYM_VALID		0x80000000
 	__le32 a0;
-	__le32 a1, a2;
+#define OFDM_RX_FRAME_VHT_BANDWIDTH			0x00000003
+#define OFDM_RX_FRAME_VHT_STBC				0x00000008
+#define OFDM_RX_FRAME_VHT_GRP_ID			0x000003f0
+#define OFDM_RX_FRAME_VHT_STS_USER0			0x00001c00
+#define OFDM_RX_FRAME_VHT_MU_STS_USER1			0x0000e000
+#define OFDM_RX_FRAME_VHT_MU_STS_USER2			0x00070000
+#define OFDM_RX_FRAME_VHT_MU_STS_USER3			0x00380000
+#define OFDM_RX_FRAME_VHT_PARTIAL_AID_OR_MU_STS		0x003fe000
+#define OFDM_RX_FRAME_VHT_MU_MIMO_USER_POSITION		0x03000000
+#define OFDM_RX_FRAME_VHT_NO_STREAMS			0x04000000
+#define OFDM_RX_FRAME_VHT_STS				0x38000000
+	__le32 a1;
+#define OFDM_RX_FRAME_VHT_SHORT_GI			0x00000001
+#define OFDM_RX_FRAME_VHT_SHORT_GI_AMBIG		0x00000002
+#define OFDM_RX_FRAME_VHT_CODING			0x00000004
+#define OFDM_RX_FRAME_VHT_CODING_EXTRA_SYM		0x00000008
+#define OFDM_RX_FRAME_VHT_MCS_OR_MU_CODING		0x000000f0
+#define OFDM_RX_FRAME_VHT_BF_OR_MU_RESERVED		0x00000100
+#define OFDM_RX_FRAME_VHT_CRC				0x0003fc00
+#define OFDM_RX_FRAME_VHT_CRC_OK_BIT			0x00040000
+#define OFDM_RX_FRAME_VHT_CUR_USER_CODING		0x00080000
+#define OFDM_RX_FRAME_VHT_CUR_USER_STS			0x00700000
+	__le32 a2;
 };
 
 struct iwl_he_sigs {
 #define OFDM_RX_FRAME_HE_BEAM_CHANGE			0x00000001
 #define OFDM_RX_FRAME_HE_UL_FLAG			0x00000002
+/* SU/ER-SU: MCS, MU: SIG-B MCS */
 #define OFDM_RX_FRAME_HE_MCS				0x0000003c
+/* SU/ER-SU: DCM, MU: SIG-B DCM */
 #define OFDM_RX_FRAME_HE_DCM				0x00000040
 #define OFDM_RX_FRAME_HE_BSS_COLOR			0x00001f80
 #define OFDM_RX_FRAME_HE_SPATIAL_REUSE			0x0001e000
@@ -1247,19 +1271,82 @@ struct iwl_eht_tb_sigs {
 };
 
 struct iwl_uhr_sigs {
-	__le32 usig_a1, usig_a1_uhr, usig_a2_uhr, b1, b2;
+	/* same as EHT above */
+	__le32 usig_a1;
+#define OFDM_RX_FRAME_UHR_BSS_COLOR2			0x0000003f
+	__le32 usig_a1_uhr;
+#define	OFDM_RX_FRAME_UHR_PPDU_TYPE			0x00000003
+#define	OFDM_RX_FRAME_UHR_COBF_CSR_DISABLE		0x00000004
+#define	OFDM_RX_FRAME_UHR_PUNC_CHANNEL			0x000000f8
+#define	OFDM_RX_FRAME_UHR_USIG2_VALIDATE_B8		0x00000100
+#define	OFDM_RX_FRAME_UHR_SIG_MCS			0x00000600
+#define	OFDM_RX_FRAME_UHR_SIG_SYM_NUM			0x0000f800
+#define	OFDM_RX_FRAME_UHR_TRIG_SPATIAL_REUSE_1		0x000f0000
+#define	OFDM_RX_FRAME_UHR_TRIG_SPATIAL_REUSE_2		0x00f00000
+#define	OFDM_RX_FRAME_UHR_TRIG_USIG2_DISREGARD		0x1f000000
+	__le32 usig_a2_uhr;
+#define OFDM_RX_FRAME_UHR_SPATIAL_REUSE			0x0000000f
+#define OFDM_RX_FRAME_UHR_GI_LTF_TYPE			0x00000030
+#define OFDM_RX_FRAME_UHR_NUM_OF_LTF_SYM		0x000001c0
+#define OFDM_RX_FRAME_UHR_CODING_EXTRA_SYM		0x00000200
+#define OFDM_RX_FRAME_UHR_PE_A_FACTOR			0x00000c00
+#define OFDM_RX_FRAME_UHR_PE_DISAMBIGUITY		0x00001000
+#define OFDM_RX_FRAME_UHR_IM_DISABLE			0x00002000
+#define OFDM_RX_FRAME_UHR_USIG_OVF_DISREGARD		0x0000c000
+#define OFDM_RX_FRAME_UHR_NUM_OF_USERS			0x00070000
+#define OFDM_RX_FRAME_UHR_NSTS				0x00f00000
+#define OFDM_RX_FRAME_UHR_BF				0x01000000
+#define OFDM_RX_FRAME_UHR_USIG_OVF_NDP_DISREGARD	0x06000000
+#define OFDM_RX_FRAME_UHR_COMM_CC1_CRC_OK		0x08000000
+#define OFDM_RX_FRAME_UHR_COMM_CC2_CRC_OK		0x10000000
+#define OFDM_RX_FRAME_UHR_NON_VALID_RU_ALLOC		0x20000000
+	__le32 b1;
+#define OFDM_RX_FRAME_UHR_MCS				0x000001f0
+#define OFDM_RX_FRAME_UHR_CODING			0x00000200
+#define OFDM_RX_FRAME_UHR_SPATIAL_CONFIG		0x00003c00
+#define OFDM_RX_FRAME_UHR_STA_RU			0x003fc000
+#define OFDM_RX_FRAME_UHR_STA_RU_PS160			0x00400000
+#define OFDM_RX_FRAME_UHR_UEQM				0x00800000
+#define OFDM_RX_FRAME_UHR_2XLDPC			0x01000000
+#define OFDM_RX_FRAME_UHR_UEQM_PATTERN			0x06000000
+#define OFDM_RX_FRAME_UHR_IS_MU_MIMO_USER_FIELD		0x08000000
+#define OFDM_RX_FRAME_UHR_USER_FIELD_CRC_OK		0x40000000
+	__le32 b2;
+#define OFDM_RX_UHR_NUM_OF_DATA_SYM			0x000007ff
+#define OFDM_RX_UHR_PE_DURATION				0x00003800
 	__le32 sig2;
+	/* same as EHT above: OFDM_RX_FRAME_EHT_RU_ALLOC_* */
 	__le32 cmn[6];
+#define OFDM_RX_FRAME_UHR_USER_FIELD_ID			0x000007ff
 	__le32 user_id;
 };
 
 struct iwl_uhr_tb_sigs {
-	__le32 usig_a1, usig_a2_uhr, tb_rx0, tb_rx1;
+	/* same as UHR above */
+	__le32 usig_a1, usig_a2_uhr;
+	/* same as HE above */
+	__le32 tb_rx0, tb_rx1;
 };
 
 struct iwl_uhr_elr_sigs {
+	/* same as UHR above */
 	__le32 usig_a1, usig_a2_uhr;
-	__le32 uhr_sig_elr1, uhr_sig_elr2;
+#define OFDM_RX_VECTOR_UHR_ELR_VER_ID			0x00000007
+#define OFDM_RX_VECTOR_UHR_ELR_UPLINK_FLAG		0x00000008
+#define OFDM_RX_VECTOR_UHR_ELR_MCS			0x00000010
+#define OFDM_RX_VECTOR_UHR_ELR_CODING			0x00000020
+#define OFDM_RX_VECTOR_UHR_ELR_LENGTH_IN_SYM		0x00007fc0
+#define OFDM_RX_VECTOR_UHR_ELR_CODING_EXTRA_SYM		0x00008000
+#define OFDM_RX_VECTOR_UHR_ELR_SIG1_CRC_OK		0x00010000
+#define OFDM_RX_VECTOR_UHR_ELR_STA_ID			0x0ffe0000
+#define OFDM_RX_VECTOR_UHR_ELR_DISREGARD		0x70000000
+	__le32 uhr_sig_elr1;
+#define OFDM_RX_VECTOR_UHR_ELR_MARK_BSS_COLOR		0x0000003f
+#define OFDM_RX_VECTOR_UHR_ELR_SIG_ID_INDX		0x00000e00
+#define OFDM_RX_VECTOR_UHR_ELR_STA_RU			0x000ff000
+#define OFDM_RX_VECTOR_UHR_ELR_STA_RU_PS160		0x00100000
+#define OFDM_RX_VECTOR_UHR_ELR_SIG2_CRC_OK		0x00200000
+	__le32 uhr_sig_elr2;
 };
 
 union iwl_sigs {
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c b/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c
index 3b0e8c43ba4a..3c4bee85b825 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c
@@ -7,6 +7,7 @@
 #include "api/commands.h"
 #include "debugfs.h"
 #include "dbg.h"
+#include <linux/hex.h>
 #include <linux/seq_file.h>
 
 #define FWRT_DEBUGFS_OPEN_WRAPPER(name, buflen, argtype)		\
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/file.h b/drivers/net/wireless/intel/iwlwifi/fw/file.h
index b9e0b69c6680..378788de1d74 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/file.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/file.h
@@ -447,6 +447,7 @@ typedef unsigned int __bitwise iwl_ucode_tlv_capa_t;
  *	during assert handling even if the dump isn't split
  * @IWL_UCODE_TLV_CAPA_FW_ACCEPTS_RAW_DSM_TABLE: Firmware has capability of
  *	handling raw DSM table data.
+ * @IWL_UCODE_TLV_CAPA_NAN_SYNC_SUPPORT: Supports NAN synchronization
  *
  * @NUM_IWL_UCODE_TLV_CAPA: number of bits used
  */
@@ -550,6 +551,7 @@ enum iwl_ucode_tlv_capa {
 
 	IWL_UCODE_TLV_CAPA_RESET_DURING_ASSERT		= (__force iwl_ucode_tlv_capa_t)(4 * 32 + 0),
 	IWL_UCODE_TLV_CAPA_FW_ACCEPTS_RAW_DSM_TABLE 	= (__force iwl_ucode_tlv_capa_t)(4 * 32 + 1),
+	IWL_UCODE_TLV_CAPA_NAN_SYNC_SUPPORT		= (__force iwl_ucode_tlv_capa_t)(4 * 32 + 2),
 	NUM_IWL_UCODE_TLV_CAPA
 /*
  * This construction make both sparse (which cannot increment the previous
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/smem.c b/drivers/net/wireless/intel/iwlwifi/fw/smem.c
index 90fd69b4860c..344ddde85b18 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/smem.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/smem.c
@@ -6,6 +6,7 @@
  */
 #include "iwl-drv.h"
 #include "runtime.h"
+#include "dbg.h"
 #include "fw/api/commands.h"
 
 static void iwl_parse_shared_mem_22000(struct iwl_fw_runtime *fwrt,
@@ -17,7 +18,9 @@ static void iwl_parse_shared_mem_22000(struct iwl_fw_runtime *fwrt,
 	u8 api_ver = iwl_fw_lookup_notif_ver(fwrt->fw, SYSTEM_GROUP,
 					     SHARED_MEM_CFG_CMD, 0);
 
-	if (WARN_ON(lmac_num > ARRAY_SIZE(mem_cfg->lmac_smem)))
+	/* Note: notification has 3 entries, but we only expect 2 */
+	if (IWL_FW_CHECK(fwrt, lmac_num > ARRAY_SIZE(fwrt->smem_cfg.lmac),
+			 "FW advertises %d LMACs\n", lmac_num))
 		return;
 
 	fwrt->smem_cfg.num_lmacs = lmac_num;
@@ -26,7 +29,8 @@ static void iwl_parse_shared_mem_22000(struct iwl_fw_runtime *fwrt,
 	fwrt->smem_cfg.rxfifo2_size = le32_to_cpu(mem_cfg->rxfifo2_size);
 
 	if (api_ver >= 4 &&
-	    !WARN_ON_ONCE(iwl_rx_packet_payload_len(pkt) < sizeof(*mem_cfg))) {
+	    !IWL_FW_CHECK(fwrt, iwl_rx_packet_payload_len(pkt) < sizeof(*mem_cfg),
+			  "bad shared mem notification size\n")) {
 		fwrt->smem_cfg.rxfifo2_control_size =
 			le32_to_cpu(mem_cfg->rxfifo2_control_size);
 	}
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
index 3391f07b01de..f8fc6f30fbe5 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
@@ -1597,7 +1597,7 @@ static void _iwl_op_mode_stop(struct iwl_drv *drv)
  */
 static void iwl_req_fw_callback(const struct firmware *ucode_raw, void *context)
 {
-	unsigned int min_core, max_core, loaded_core;
+	int min_core, max_core, loaded_core;
 	struct iwl_drv *drv = context;
 	struct iwl_fw *fw = &drv->fw;
 	const struct iwl_ucode_header *ucode;
@@ -1676,7 +1676,7 @@ static void iwl_req_fw_callback(const struct firmware *ucode_raw, void *context)
 	if (loaded_core < min_core || loaded_core > max_core) {
 		IWL_ERR(drv,
 			"Driver unable to support your firmware API. "
-			"Driver supports FW core %u..%u, firmware is %u.\n",
+			"Driver supports FW core %d..%d, firmware is %d.\n",
 			min_core, max_core, loaded_core);
 		goto try_again;
 	}
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/Makefile b/drivers/net/wireless/intel/iwlwifi/mld/Makefile
index c966e573f430..5740c0510b61 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/Makefile
+++ b/drivers/net/wireless/intel/iwlwifi/mld/Makefile
@@ -4,7 +4,7 @@ obj-$(CONFIG_IWLWIFI_KUNIT_TESTS) += tests/
 
 iwlmld-y += mld.o notif.o mac80211.o fw.o power.o iface.o link.o rx.o mcc.o session-protect.o phy.o
 iwlmld-y += scan.o sta.o tx.o coex.o tlc.o agg.o key.o regulatory.o ap.o thermal.o roc.o stats.o
-iwlmld-y += low_latency.o mlo.o ptp.o time_sync.o ftm-initiator.o
+iwlmld-y += low_latency.o mlo.o ptp.o time_sync.o ftm-initiator.o nan.o
 iwlmld-$(CONFIG_IWLWIFI_DEBUGFS) += debugfs.o
 iwlmld-$(CONFIG_IWLWIFI_LEDS) += led.o
 iwlmld-$(CONFIG_PM_SLEEP) += d3.o
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/d3.c b/drivers/net/wireless/intel/iwlwifi/mld/d3.c
index dd85be94433c..6595542e95cf 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/d3.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/d3.c
@@ -996,8 +996,6 @@ static void iwl_mld_mlo_rekey(struct iwl_mld *mld,
 			      struct iwl_mld_wowlan_status *wowlan_status,
 			      struct ieee80211_vif *vif)
 {
-	struct iwl_mld_old_mlo_keys *old_keys __free(kfree) = NULL;
-
 	IWL_DEBUG_WOWLAN(mld, "Num of MLO Keys: %d\n", wowlan_status->num_mlo_keys);
 
 	if (!wowlan_status->num_mlo_keys)
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/debugfs.c b/drivers/net/wireless/intel/iwlwifi/mld/debugfs.c
index b9c9cd3f44e4..ce2fc98782c3 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/debugfs.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/debugfs.c
@@ -24,6 +24,7 @@
 #include "fw/api/rfi.h"
 #include "fw/dhc-utils.h"
 #include <linux/dmi.h>
+#include <linux/hex.h>
 
 #define MLD_DEBUGFS_READ_FILE_OPS(name, bufsz)				\
 	_MLD_DEBUGFS_READ_FILE_OPS(name, bufsz, struct iwl_mld)
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/iface.c b/drivers/net/wireless/intel/iwlwifi/mld/iface.c
index a5ececfc13e4..3ca3e169738e 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/iface.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/iface.c
@@ -339,6 +339,10 @@ int iwl_mld_mac_fw_action(struct iwl_mld *mld, struct ieee80211_vif *vif,
 
 	lockdep_assert_wiphy(mld->wiphy);
 
+	/* NAN interface type is not known to FW */
+	if (vif->type == NL80211_IFTYPE_NAN)
+		return 0;
+
 	if (action == FW_CTXT_ACTION_REMOVE)
 		return iwl_mld_rm_mac_from_fw(mld, vif);
 
@@ -387,21 +391,16 @@ static void iwl_mld_mlo_scan_start_wk(struct wiphy *wiphy,
 IWL_MLD_ALLOC_FN(vif, vif)
 
 /* Constructor function for struct iwl_mld_vif */
-static int
+static void
 iwl_mld_init_vif(struct iwl_mld *mld, struct ieee80211_vif *vif)
 {
 	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
-	int ret;
 
 	lockdep_assert_wiphy(mld->wiphy);
 
 	mld_vif->mld = mld;
 	mld_vif->roc_activity = ROC_NUM_ACTIVITIES;
 
-	ret = iwl_mld_allocate_vif_fw_id(mld, &mld_vif->fw_id, vif);
-	if (ret)
-		return ret;
-
 	if (!mld->fw_status.in_hw_restart) {
 		wiphy_work_init(&mld_vif->emlsr.unblock_tpt_wk,
 				iwl_mld_emlsr_unblock_tpt_wk);
@@ -415,8 +414,6 @@ iwl_mld_init_vif(struct iwl_mld *mld, struct ieee80211_vif *vif)
 					iwl_mld_mlo_scan_start_wk);
 	}
 	iwl_mld_init_internal_sta(&mld_vif->aux_sta);
-
-	return 0;
 }
 
 int iwl_mld_add_vif(struct iwl_mld *mld, struct ieee80211_vif *vif)
@@ -426,7 +423,13 @@ int iwl_mld_add_vif(struct iwl_mld *mld, struct ieee80211_vif *vif)
 
 	lockdep_assert_wiphy(mld->wiphy);
 
-	ret = iwl_mld_init_vif(mld, vif);
+	iwl_mld_init_vif(mld, vif);
+
+	/* NAN interface type is not known to FW */
+	if (vif->type == NL80211_IFTYPE_NAN)
+		return 0;
+
+	ret = iwl_mld_allocate_vif_fw_id(mld, &mld_vif->fw_id, vif);
 	if (ret)
 		return ret;
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
index 55b484c16280..8466345d7f21 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
@@ -50,7 +50,7 @@
 	{							\
 		.max = 1,					\
 		.types = BIT(NL80211_IFTYPE_P2P_DEVICE),	\
-	}
+	},
 
 static const struct ieee80211_iface_limit iwl_mld_limits[] = {
 	IWL_MLD_LIMITS(0)
@@ -60,6 +60,22 @@ static const struct ieee80211_iface_limit iwl_mld_limits_ap[] = {
 	IWL_MLD_LIMITS(BIT(NL80211_IFTYPE_AP))
 };
 
+static const struct ieee80211_iface_limit iwl_mld_limits_nan[] = {
+	{
+		.max = 2,
+		.types = BIT(NL80211_IFTYPE_STATION),
+	},
+	{
+		.max = 1,
+		.types = BIT(NL80211_IFTYPE_NAN),
+	},
+	/* Removed when two channels are permitted */
+	{
+		.max = 1,
+		.types = BIT(NL80211_IFTYPE_AP),
+	},
+};
+
 static const struct ieee80211_iface_combination
 iwl_mld_iface_combinations[] = {
 	{
@@ -74,6 +90,19 @@ iwl_mld_iface_combinations[] = {
 		.limits = iwl_mld_limits_ap,
 		.n_limits = ARRAY_SIZE(iwl_mld_limits_ap),
 	},
+	/* NAN combinations follow, these exclude P2P */
+	{
+		.num_different_channels = 2,
+		.max_interfaces = 3,
+		.limits = iwl_mld_limits_nan,
+		.n_limits = ARRAY_SIZE(iwl_mld_limits_nan) - 1,
+	},
+	{
+		.num_different_channels = 1,
+		.max_interfaces = 4,
+		.limits = iwl_mld_limits_nan,
+		.n_limits = ARRAY_SIZE(iwl_mld_limits_nan),
+	}
 };
 
 static const u8 ext_capa_base[IWL_MLD_STA_EXT_CAPA_SIZE] = {
@@ -305,8 +334,38 @@ static void iwl_mac_hw_set_wiphy(struct iwl_mld *mld)
 
 	wiphy->hw_timestamp_max_peers = 1;
 
-	wiphy->iface_combinations = iwl_mld_iface_combinations;
-	wiphy->n_iface_combinations = ARRAY_SIZE(iwl_mld_iface_combinations);
+	if (iwl_mld_nan_supported(mld)) {
+		hw->wiphy->interface_modes |= BIT(NL80211_IFTYPE_NAN);
+		hw->wiphy->iface_combinations = iwl_mld_iface_combinations;
+		hw->wiphy->n_iface_combinations =
+			ARRAY_SIZE(iwl_mld_iface_combinations);
+
+		hw->wiphy->nan_supported_bands = BIT(NL80211_BAND_2GHZ);
+		if (mld->nvm_data->bands[NL80211_BAND_5GHZ].n_channels)
+			hw->wiphy->nan_supported_bands |=
+				BIT(NL80211_BAND_5GHZ);
+
+		hw->wiphy->nan_capa.flags = WIPHY_NAN_FLAGS_CONFIGURABLE_SYNC |
+					    WIPHY_NAN_FLAGS_USERSPACE_DE;
+
+		hw->wiphy->nan_capa.op_mode = NAN_OP_MODE_PHY_MODE_MASK |
+					      NAN_OP_MODE_80P80MHZ |
+					      NAN_OP_MODE_160MHZ;
+
+		/* Support 2 antenna's for Tx and Rx */
+		hw->wiphy->nan_capa.n_antennas = 0x22;
+
+		/* Maximal channel switch time is 4 msec */
+		hw->wiphy->nan_capa.max_channel_switch_time = 4;
+		hw->wiphy->nan_capa.dev_capabilities =
+			NAN_DEV_CAPA_EXT_KEY_ID_SUPPORTED |
+			NAN_DEV_CAPA_NDPE_SUPPORTED;
+	} else {
+		wiphy->iface_combinations = iwl_mld_iface_combinations;
+		/* Do not include NAN combinations */
+		wiphy->n_iface_combinations =
+			ARRAY_SIZE(iwl_mld_iface_combinations) - 2;
+	}
 
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_VHT_IBSS);
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_DFS_CONCURRENT);
@@ -318,6 +377,8 @@ static void iwl_mac_hw_set_wiphy(struct iwl_mld *mld)
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_FILS_MAX_CHANNEL_TIME);
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_OCE_PROBE_REQ_HIGH_TX_RATE);
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_MU_MIMO_AIR_SNIFFER);
+	wiphy_ext_feature_set(wiphy,
+			      NL80211_EXT_FEATURE_MULTICAST_REGISTRATIONS);
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_SPP_AMSDU_SUPPORT);
 
 	if (fw_has_capa(ucode_capa, IWL_UCODE_TLV_CAPA_PROTECTED_TWT))
@@ -616,10 +677,11 @@ int iwl_mld_mac80211_add_interface(struct ieee80211_hw *hw,
 	 * Add the default link, but not if this is an MLD vif as that implies
 	 * the HW is restarting and it will be configured by change_vif_links.
 	 */
-	if (!ieee80211_vif_is_mld(vif))
+	if (vif->type != NL80211_IFTYPE_NAN && !ieee80211_vif_is_mld(vif)) {
 		ret = iwl_mld_add_link(mld, &vif->bss_conf);
-	if (ret)
-		goto err;
+		if (ret)
+			goto err;
+	}
 
 	if (vif->type == NL80211_IFTYPE_STATION) {
 		vif->driver_flags |= IEEE80211_VIF_REMOVE_AP_AFTER_DISASSOC;
@@ -647,6 +709,9 @@ int iwl_mld_mac80211_add_interface(struct ieee80211_hw *hw,
 	if (vif->type == NL80211_IFTYPE_P2P_DEVICE)
 		mld->p2p_device_vif = vif;
 
+	if (vif->type == NL80211_IFTYPE_NAN)
+		mld->nan_device_vif = vif;
+
 	return 0;
 
 err:
@@ -674,7 +739,10 @@ void iwl_mld_mac80211_remove_interface(struct ieee80211_hw *hw,
 	if (vif->type == NL80211_IFTYPE_P2P_DEVICE)
 		mld->p2p_device_vif = NULL;
 
-	iwl_mld_remove_link(mld, &vif->bss_conf);
+	if (vif->type == NL80211_IFTYPE_NAN)
+		mld->nan_device_vif = NULL;
+	else
+		iwl_mld_remove_link(mld, &vif->bss_conf);
 
 #ifdef CONFIG_IWLWIFI_DEBUGFS
 	debugfs_remove(iwl_mld_vif_from_mac80211(vif)->dbgfs_slink);
@@ -1506,6 +1574,9 @@ iwl_mld_mac80211_conf_tx(struct ieee80211_hw *hw,
 
 	lockdep_assert_wiphy(mld->wiphy);
 
+	if (vif->type == NL80211_IFTYPE_NAN)
+		return 0;
+
 	link = iwl_mld_link_dereference_check(mld_vif, link_id);
 	if (!link)
 		return -EINVAL;
@@ -2699,4 +2770,7 @@ const struct ieee80211_ops iwl_mld_hw_ops = {
 	.set_hw_timestamp = iwl_mld_set_hw_timestamp,
 	.start_pmsr = iwl_mld_start_pmsr,
 	.can_neg_ttlm = iwl_mld_can_neg_ttlm,
+	.start_nan = iwl_mld_start_nan,
+	.stop_nan = iwl_mld_stop_nan,
+	.nan_change_conf = iwl_mld_nan_change_config,
 };
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mld.c b/drivers/net/wireless/intel/iwlwifi/mld/mld.c
index 8a4c96385640..495e9d8f3af6 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mld.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mld.c
@@ -234,6 +234,9 @@ static const struct iwl_hcmd_names iwl_mld_mac_conf_names[] = {
 	HCMD_NAME(AUX_STA_CMD),
 	HCMD_NAME(STA_REMOVE_CMD),
 	HCMD_NAME(ROC_CMD),
+	HCMD_NAME(NAN_CFG_CMD),
+	HCMD_NAME(NAN_DW_END_NOTIF),
+	HCMD_NAME(NAN_JOINED_CLUSTER_NOTIF),
 	HCMD_NAME(MISSED_BEACONS_NOTIF),
 	HCMD_NAME(EMLSR_TRANS_FAIL_NOTIF),
 	HCMD_NAME(ROC_NOTIF),
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mld.h b/drivers/net/wireless/intel/iwlwifi/mld/mld.h
index 22efe8e10f53..66c7a7d31409 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mld.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mld.h
@@ -35,6 +35,7 @@
 #include "ptp.h"
 #include "time_sync.h"
 #include "ftm-initiator.h"
+#include "nan.h"
 
 /**
  * DOC: Introduction
@@ -199,6 +200,7 @@
  * @ptp_data: data of the PTP clock
  * @time_sync: time sync data.
  * @ftm_initiator: FTM initiator data
+ * @nan_device_vif: points to the NAN device vif if exists
  */
 struct iwl_mld {
 	/* Add here fields that need clean up on restart */
@@ -228,6 +230,7 @@ struct iwl_mld {
 #endif /* CONFIG_PM_SLEEP */
 		struct ieee80211_vif *p2p_device_vif;
 		bool bt_is_active;
+		struct ieee80211_vif *nan_device_vif;
 	);
 	struct ieee80211_link_sta __rcu *fw_id_to_link_sta[IWL_STATION_COUNT_MAX];
 	/* And here fields that survive a fw restart */
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/nan.c b/drivers/net/wireless/intel/iwlwifi/mld/nan.c
new file mode 100644
index 000000000000..e7a2c1ec7494
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/mld/nan.c
@@ -0,0 +1,287 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2025 Intel Corporation
+ */
+
+#include "mld.h"
+#include "iface.h"
+#include "fw/api/mac-cfg.h"
+
+#define IWL_NAN_DISOVERY_BEACON_INTERNVAL_TU 512
+#define IWL_NAN_RSSI_CLOSE 55
+#define IWL_NAN_RSSI_MIDDLE 70
+
+bool iwl_mld_nan_supported(struct iwl_mld *mld)
+{
+	return fw_has_capa(&mld->fw->ucode_capa,
+			   IWL_UCODE_TLV_CAPA_NAN_SYNC_SUPPORT);
+}
+
+static int iwl_mld_nan_send_config_cmd(struct iwl_mld *mld,
+				       struct iwl_nan_config_cmd *cmd,
+				       u8 *beacon_data, size_t beacon_data_len)
+{
+	struct iwl_host_cmd hcmd = {
+		.id = WIDE_ID(MAC_CONF_GROUP, NAN_CFG_CMD),
+	};
+
+	hcmd.len[0] = sizeof(*cmd);
+	hcmd.data[0] = cmd;
+
+	if (beacon_data_len) {
+		hcmd.len[1] = beacon_data_len;
+		hcmd.data[1] = beacon_data;
+		hcmd.dataflags[1] = IWL_HCMD_DFL_DUP;
+	}
+
+	return iwl_mld_send_cmd(mld, &hcmd);
+}
+
+static int iwl_mld_nan_config(struct iwl_mld *mld,
+			      struct ieee80211_vif *vif,
+			      struct cfg80211_nan_conf *conf,
+			      enum iwl_ctxt_action action)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_nan_config_cmd cmd = {
+		.action = cpu_to_le32(action),
+	};
+	u8 *data __free(kfree) = NULL;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	ether_addr_copy(cmd.nmi_addr, vif->addr);
+	cmd.master_pref = conf->master_pref;
+
+	if (conf->cluster_id)
+		cmd.cluster_id =
+			cpu_to_le16(*(const u16 *)(conf->cluster_id + 4));
+
+	cmd.scan_period = conf->scan_period < 255 ? conf->scan_period : 255;
+	cmd.dwell_time =
+		conf->scan_dwell_time < 255 ? conf->scan_dwell_time : 255;
+
+	if (conf->discovery_beacon_interval)
+		cmd.discovery_beacon_interval =
+			cpu_to_le32(conf->discovery_beacon_interval);
+	else
+		cmd.discovery_beacon_interval =
+			cpu_to_le32(IWL_NAN_DISOVERY_BEACON_INTERNVAL_TU);
+
+	if (conf->enable_dw_notification)
+		cmd.flags = IWL_NAN_FLAG_DW_END_NOTIF_ENABLED;
+
+	/* 2 GHz band must be supported */
+	cmd.band_config[IWL_NAN_BAND_2GHZ].rssi_close =
+		abs(conf->band_cfgs[NL80211_BAND_2GHZ].rssi_close);
+	cmd.band_config[IWL_NAN_BAND_2GHZ].rssi_middle =
+		abs(conf->band_cfgs[NL80211_BAND_2GHZ].rssi_middle);
+	cmd.band_config[IWL_NAN_BAND_2GHZ].dw_interval =
+		conf->band_cfgs[NL80211_BAND_2GHZ].awake_dw_interval;
+
+	/* 5 GHz band operation is optional. Configure its operation if
+	 * supported. Note that conf->bands might be zero, so we need to check
+	 * the channel pointer, not the band mask.
+	 */
+	if (conf->band_cfgs[NL80211_BAND_5GHZ].chan) {
+		cmd.hb_channel =
+			conf->band_cfgs[NL80211_BAND_5GHZ].chan->hw_value;
+
+		cmd.band_config[IWL_NAN_BAND_5GHZ].rssi_close =
+			abs(conf->band_cfgs[NL80211_BAND_5GHZ].rssi_close);
+		cmd.band_config[IWL_NAN_BAND_5GHZ].rssi_middle =
+			abs(conf->band_cfgs[NL80211_BAND_5GHZ].rssi_middle);
+		cmd.band_config[IWL_NAN_BAND_5GHZ].dw_interval =
+			conf->band_cfgs[NL80211_BAND_5GHZ].awake_dw_interval;
+	}
+
+	if (conf->extra_nan_attrs_len || conf->vendor_elems_len) {
+		data = kmalloc(conf->extra_nan_attrs_len +
+			       conf->vendor_elems_len, GFP_KERNEL);
+		if (!data)
+			return -ENOMEM;
+
+		cmd.nan_attr_len = cpu_to_le32(conf->extra_nan_attrs_len);
+		cmd.nan_vendor_elems_len = cpu_to_le32(conf->vendor_elems_len);
+
+		if (conf->extra_nan_attrs_len)
+			memcpy(data, conf->extra_nan_attrs,
+			       conf->extra_nan_attrs_len);
+
+		if (conf->vendor_elems_len)
+			memcpy(data + conf->extra_nan_attrs_len,
+			       conf->vendor_elems,
+			       conf->vendor_elems_len);
+	}
+
+	cmd.sta_id = mld_vif->aux_sta.sta_id;
+	return iwl_mld_nan_send_config_cmd(mld, &cmd, data,
+					   conf->extra_nan_attrs_len +
+					   conf->vendor_elems_len);
+}
+
+int iwl_mld_start_nan(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		      struct cfg80211_nan_conf *conf)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_mld_int_sta *aux_sta = &mld_vif->aux_sta;
+	int ret;
+
+	IWL_DEBUG_MAC80211(mld, "NAN: start: bands=0x%x\n", conf->bands);
+
+	ret = iwl_mld_add_aux_sta(mld, aux_sta);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_nan_config(mld, vif, conf, FW_CTXT_ACTION_ADD);
+	if (ret) {
+		IWL_ERR(mld, "Failed to start NAN. ret=%d\n", ret);
+		iwl_mld_remove_aux_sta(mld, vif);
+	}
+
+	return ret;
+}
+
+int iwl_mld_nan_change_config(struct ieee80211_hw *hw,
+			      struct ieee80211_vif *vif,
+			      struct cfg80211_nan_conf *conf,
+			      u32 changes)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	IWL_DEBUG_MAC80211(mld, "NAN: change: changes=0x%x, bands=0x%x\n",
+			   changes, conf->bands);
+
+	/* Note that we do not use 'changes' as the FW always expects the
+	 * complete configuration, and mac80211 always provides the complete
+	 * configuration.
+	 */
+	return iwl_mld_nan_config(mld, vif, conf, FW_CTXT_ACTION_MODIFY);
+}
+
+int iwl_mld_stop_nan(struct ieee80211_hw *hw,
+		     struct ieee80211_vif *vif)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct iwl_nan_config_cmd cmd = {
+		.action = cpu_to_le32(FW_CTXT_ACTION_REMOVE),
+	};
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	ret = iwl_mld_send_cmd_pdu(mld,
+				   WIDE_ID(MAC_CONF_GROUP, NAN_CFG_CMD),
+				   &cmd);
+	if (ret)
+		IWL_ERR(mld, "NAN: Failed to stop NAN. ret=%d\n", ret);
+
+	/* assume that higher layer guarantees that no additional frames are
+	 * added before calling this callback
+	 */
+	iwl_mld_flush_link_sta_txqs(mld, mld_vif->aux_sta.sta_id);
+	iwl_mld_remove_aux_sta(mld, vif);
+
+	/* cancel based on object type being NAN, as the NAN objects do
+	 * not have a unique identifier associated with them
+	 */
+	iwl_mld_cancel_notifications_of_object(mld,
+					       IWL_MLD_OBJECT_TYPE_NAN,
+					       0);
+	return 0;
+}
+
+void iwl_mld_handle_nan_cluster_notif(struct iwl_mld *mld,
+				      struct iwl_rx_packet *pkt)
+{
+	struct iwl_nan_cluster_notif *notif = (void *)pkt->data;
+	struct wireless_dev *wdev = mld->nan_device_vif ?
+		ieee80211_vif_to_wdev(mld->nan_device_vif) : NULL;
+	bool new_cluster = !!(notif->flags &
+			      IWL_NAN_CLUSTER_NOTIF_FLAG_NEW_CLUSTER);
+	u8 cluster_id[ETH_ALEN] __aligned(2) = {
+		0x50, 0x6f, 0x9a, 0x01, 0x00, 0x00
+	};
+	u16 id = le16_to_cpu(notif->cluster_id);
+
+	IWL_DEBUG_INFO(mld,
+		       "NAN: cluster event: cluster_id=0x%x, flags=0x%x\n",
+		       id, notif->flags);
+
+	if (IWL_FW_CHECK(mld, !wdev, "NAN: cluster event without wdev\n"))
+		return;
+
+	if (IWL_FW_CHECK(mld, !ieee80211_vif_nan_started(mld->nan_device_vif),
+			 "NAN: cluster event without NAN started\n"))
+		return;
+
+	*((u16 *)(cluster_id + 4)) = id;
+
+	cfg80211_nan_cluster_joined(wdev, cluster_id, new_cluster, GFP_KERNEL);
+}
+
+bool iwl_mld_cancel_nan_cluster_notif(struct iwl_mld *mld,
+				      struct iwl_rx_packet *pkt,
+				      u32 obj_id)
+{
+	return true;
+}
+
+bool iwl_mld_cancel_nan_dw_end_notif(struct iwl_mld *mld,
+				     struct iwl_rx_packet *pkt,
+				     u32 obj_id)
+{
+	return true;
+}
+
+void iwl_mld_handle_nan_dw_end_notif(struct iwl_mld *mld,
+				     struct iwl_rx_packet *pkt)
+{
+	struct iwl_nan_dw_end_notif *notif = (void *)pkt->data;
+	struct iwl_mld_vif *mld_vif = mld->nan_device_vif ?
+		iwl_mld_vif_from_mac80211(mld->nan_device_vif) :
+		NULL;
+	struct wireless_dev *wdev;
+	struct ieee80211_channel *chan;
+
+	IWL_INFO(mld, "NAN: DW end: band=%u\n", notif->band);
+
+	if (IWL_FW_CHECK(mld, !mld_vif, "NAN: DW end without mld_vif\n"))
+		return;
+
+	if (IWL_FW_CHECK(mld, !ieee80211_vif_nan_started(mld->nan_device_vif),
+			 "NAN: DW end without NAN started\n"))
+		return;
+
+	if (WARN_ON(mld_vif->aux_sta.sta_id == IWL_INVALID_STA))
+		return;
+
+	IWL_DEBUG_INFO(mld, "NAN: flush queues for aux sta=%u\n",
+		       mld_vif->aux_sta.sta_id);
+
+	iwl_mld_flush_link_sta_txqs(mld, mld_vif->aux_sta.sta_id);
+
+	/* TODO: currently the notification specified the band on which the DW
+	 * ended. Need to change that to the actual channel on which the next DW
+	 * will be started.
+	 */
+	switch (notif->band) {
+	case IWL_NAN_BAND_2GHZ:
+		chan = ieee80211_get_channel(mld->wiphy, 2437);
+		break;
+	case IWL_NAN_BAND_5GHZ:
+		/* TODO: use the actual channel */
+		chan = ieee80211_get_channel(mld->wiphy, 5745);
+		break;
+	default:
+		IWL_FW_CHECK(mld, false,
+			     "NAN: Invalid band %u in DW end notif\n",
+			     notif->band);
+		return;
+	}
+
+	wdev = ieee80211_vif_to_wdev(mld->nan_device_vif);
+	cfg80211_next_nan_dw_notif(wdev, chan, GFP_KERNEL);
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/nan.h b/drivers/net/wireless/intel/iwlwifi/mld/nan.h
new file mode 100644
index 000000000000..c9c83d1012f0
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/mld/nan.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * Copyright (C) 2025 Intel Corporation
+ */
+
+#include <net/cfg80211.h>
+#include <linux/etherdevice.h>
+
+bool iwl_mld_nan_supported(struct iwl_mld *mld);
+int iwl_mld_start_nan(struct ieee80211_hw *hw,
+		      struct ieee80211_vif *vif,
+		      struct cfg80211_nan_conf *conf);
+int iwl_mld_nan_change_config(struct ieee80211_hw *hw,
+			      struct ieee80211_vif *vif,
+			      struct cfg80211_nan_conf *conf,
+			      u32 changes);
+int iwl_mld_stop_nan(struct ieee80211_hw *hw,
+		     struct ieee80211_vif *vif);
+void iwl_mld_handle_nan_cluster_notif(struct iwl_mld *mld,
+				      struct iwl_rx_packet *pkt);
+void iwl_mld_handle_nan_dw_end_notif(struct iwl_mld *mld,
+				     struct iwl_rx_packet *pkt);
+bool iwl_mld_cancel_nan_cluster_notif(struct iwl_mld *mld,
+				      struct iwl_rx_packet *pkt,
+				      u32 obj_id);
+bool iwl_mld_cancel_nan_dw_end_notif(struct iwl_mld *mld,
+				     struct iwl_rx_packet *pkt,
+				     u32 obj_id);
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/notif.c b/drivers/net/wireless/intel/iwlwifi/mld/notif.c
index 4cf3920b005f..35356b244c0a 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/notif.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/notif.c
@@ -111,6 +111,9 @@ static bool iwl_mld_cancel_##name##_notif(struct iwl_mld *mld,			\
 #define RX_HANDLER_OF_FTM_REQ(_grp, _cmd, _name)				\
 	RX_HANDLER_OF_OBJ(_grp, _cmd, _name, FTM_REQ)
 
+#define RX_HANDLER_OF_NAN(_grp, _cmd, _name)				\
+	RX_HANDLER_OF_OBJ(_grp, _cmd, _name, NAN)
+
 static void iwl_mld_handle_mfuart_notif(struct iwl_mld *mld,
 					struct iwl_rx_packet *pkt)
 {
@@ -344,6 +347,8 @@ CMD_VERSIONS(time_sync_confirm_notif,
 	     CMD_VER_ENTRY(1, iwl_time_msmt_cfm_notify))
 CMD_VERSIONS(ftm_resp_notif, CMD_VER_ENTRY(10, iwl_tof_range_rsp_ntfy))
 CMD_VERSIONS(beacon_filter_notif, CMD_VER_ENTRY(2, iwl_beacon_filter_notif))
+CMD_VERSIONS(nan_cluster_notif, CMD_VER_ENTRY(1, iwl_nan_cluster_notif))
+CMD_VERSIONS(nan_dw_end_notif, CMD_VER_ENTRY(1, iwl_nan_dw_end_notif))
 
 DEFINE_SIMPLE_CANCELLATION(session_prot, iwl_session_prot_notif, mac_link_id)
 DEFINE_SIMPLE_CANCELLATION(tlc, iwl_tlc_update_notif, sta_id)
@@ -459,6 +464,10 @@ const struct iwl_rx_handler iwl_mld_rx_handlers[] = {
 			   beacon_filter_notif)
 	RX_HANDLER_OF_FTM_REQ(LOCATION_GROUP, TOF_RANGE_RESPONSE_NOTIF,
 			      ftm_resp_notif)
+	RX_HANDLER_OF_NAN(MAC_CONF_GROUP, NAN_JOINED_CLUSTER_NOTIF,
+			  nan_cluster_notif)
+	RX_HANDLER_OF_NAN(MAC_CONF_GROUP, NAN_DW_END_NOTIF,
+			  nan_dw_end_notif)
 };
 EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_rx_handlers);
 
@@ -531,6 +540,8 @@ static void iwl_mld_rx_notif(struct iwl_mld *mld,
 			     struct iwl_rx_cmd_buffer *rxb,
 			     struct iwl_rx_packet *pkt)
 {
+	union iwl_dbg_tlv_tp_data tp_data = { .fw_pkt = pkt };
+
 	for (int i = 0; i < ARRAY_SIZE(iwl_mld_rx_handlers); i++) {
 		const struct iwl_rx_handler *rx_h = &iwl_mld_rx_handlers[i];
 		struct iwl_async_handler_entry *entry;
@@ -571,6 +582,8 @@ static void iwl_mld_rx_notif(struct iwl_mld *mld,
 	}
 
 	iwl_notification_wait_notify(&mld->notif_wait, pkt);
+	iwl_dbg_tlv_time_point(&mld->fwrt,
+			       IWL_FW_INI_TIME_POINT_FW_RSP_OR_NOTIF, &tp_data);
 }
 
 void iwl_mld_rx(struct iwl_op_mode *op_mode, struct napi_struct *napi,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/notif.h b/drivers/net/wireless/intel/iwlwifi/mld/notif.h
index adcdd9dec192..373c1a90d98e 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/notif.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/notif.h
@@ -25,6 +25,7 @@ enum iwl_mld_object_type {
 	IWL_MLD_OBJECT_TYPE_ROC,
 	IWL_MLD_OBJECT_TYPE_SCAN,
 	IWL_MLD_OBJECT_TYPE_FTM_REQ,
+	IWL_MLD_OBJECT_TYPE_NAN,
 };
 
 void iwl_mld_cancel_notifications_of_object(struct iwl_mld *mld,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/power.c b/drivers/net/wireless/intel/iwlwifi/mld/power.c
index f664b277adf7..c3318e84f4a2 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/power.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/power.c
@@ -328,6 +328,33 @@ iwl_mld_tpe_sta_cmd_data(struct iwl_txpower_constraints_cmd *cmd,
 				       link->tpe.max_reg_client[0].power[i]);
 }
 
+static int
+iwl_mld_set_ap_power_type(struct iwl_txpower_constraints_cmd *cmd,
+			  struct ieee80211_vif *vif,
+			  struct ieee80211_bss_conf *link)
+{
+	if (vif->type == NL80211_IFTYPE_AP) {
+		cmd->ap_type = cpu_to_le16(IWL_6GHZ_AP_TYPE_VLP);
+		return 0;
+	}
+
+	switch (link->power_type) {
+	case IEEE80211_REG_LPI_AP:
+		cmd->ap_type = cpu_to_le16(IWL_6GHZ_AP_TYPE_LPI);
+		break;
+	case IEEE80211_REG_SP_AP:
+		cmd->ap_type = cpu_to_le16(IWL_6GHZ_AP_TYPE_SP);
+		break;
+	case IEEE80211_REG_VLP_AP:
+		cmd->ap_type = cpu_to_le16(IWL_6GHZ_AP_TYPE_VLP);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 void
 iwl_mld_send_ap_tx_power_constraint_cmd(struct iwl_mld *mld,
 					struct ieee80211_vif *vif,
@@ -349,15 +376,13 @@ iwl_mld_send_ap_tx_power_constraint_cmd(struct iwl_mld *mld,
 	memset(cmd.psd_pwr, DEFAULT_TPE_TX_POWER, sizeof(cmd.psd_pwr));
 	memset(cmd.eirp_pwr, DEFAULT_TPE_TX_POWER, sizeof(cmd.eirp_pwr));
 
-	if (vif->type == NL80211_IFTYPE_AP) {
-		cmd.ap_type = cpu_to_le16(IWL_6GHZ_AP_TYPE_VLP);
-	} else if (link->power_type == IEEE80211_REG_UNSET_AP) {
+	if (iwl_mld_set_ap_power_type(&cmd, vif, link))
 		return;
-	} else {
-		cmd.ap_type = cpu_to_le16(link->power_type - 1);
+
+	if (vif->type != NL80211_IFTYPE_AP)
 		iwl_mld_tpe_sta_cmd_data(&cmd, link);
-	}
 
+	IWL_DEBUG_POWER(mld, "AP power type: %d\n", le16_to_cpu(cmd.ap_type));
 	ret = iwl_mld_send_cmd_pdu(mld,
 				   WIDE_ID(PHY_OPS_GROUP,
 					   AP_TX_POWER_CONSTRAINTS_CMD),
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/ptp.c b/drivers/net/wireless/intel/iwlwifi/mld/ptp.c
index ffeb37a7f830..231920425c06 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/ptp.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/ptp.c
@@ -121,6 +121,12 @@ static int iwl_mld_ptp_gettime(struct ptp_clock_info *ptp,
 	return 0;
 }
 
+static int iwl_mld_ptp_settime(struct ptp_clock_info *ptp,
+			       const struct timespec64 *ts)
+{
+	return -EOPNOTSUPP;
+}
+
 static int iwl_mld_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
 {
 	struct iwl_mld *mld = container_of(ptp, struct iwl_mld,
@@ -279,6 +285,7 @@ void iwl_mld_ptp_init(struct iwl_mld *mld)
 
 	mld->ptp_data.ptp_clock_info.owner = THIS_MODULE;
 	mld->ptp_data.ptp_clock_info.gettime64 = iwl_mld_ptp_gettime;
+	mld->ptp_data.ptp_clock_info.settime64 = iwl_mld_ptp_settime;
 	mld->ptp_data.ptp_clock_info.max_adj = 0x7fffffff;
 	mld->ptp_data.ptp_clock_info.adjtime = iwl_mld_ptp_adjtime;
 	mld->ptp_data.ptp_clock_info.adjfine = iwl_mld_ptp_adjfine;
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/rx.c b/drivers/net/wireless/intel/iwlwifi/mld/rx.c
index 6a76e3fcb581..214dcfde2fb4 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/rx.c
@@ -207,6 +207,134 @@ static void iwl_mld_fill_signal(struct iwl_mld *mld, int link_id,
 	rx_status->chain_signal[1] = energy_b;
 }
 
+static void
+iwl_mld_decode_vht_phy_data(struct iwl_mld_rx_phy_data *phy_data,
+			    struct ieee80211_radiotap_vht *vht,
+			    struct ieee80211_rx_status *rx_status)
+{
+	bool stbc;
+
+	vht->known = cpu_to_le16(IEEE80211_RADIOTAP_VHT_KNOWN_BANDWIDTH |
+				 IEEE80211_RADIOTAP_VHT_KNOWN_GROUP_ID |
+				 IEEE80211_RADIOTAP_VHT_KNOWN_STBC |
+				 IEEE80211_RADIOTAP_VHT_KNOWN_GI |
+				 IEEE80211_RADIOTAP_VHT_KNOWN_SGI_NSYM_DIS |
+				 IEEE80211_RADIOTAP_VHT_KNOWN_LDPC_EXTRA_OFDM_SYM |
+				 IEEE80211_RADIOTAP_VHT_KNOWN_BEAMFORMED);
+
+	switch (le32_get_bits(phy_data->ntfy->sigs.vht.a1,
+			      OFDM_RX_FRAME_VHT_BANDWIDTH)) {
+	case 0:
+		vht->bandwidth = IEEE80211_RADIOTAP_VHT_BW_20;
+		break;
+	case 1:
+		vht->bandwidth = IEEE80211_RADIOTAP_VHT_BW_40;
+		break;
+	case 2:
+		vht->bandwidth = IEEE80211_RADIOTAP_VHT_BW_80;
+		break;
+	case 3:
+		vht->bandwidth = IEEE80211_RADIOTAP_VHT_BW_160;
+		break;
+	}
+
+	vht->group_id = le32_get_bits(phy_data->ntfy->sigs.vht.a1,
+				      OFDM_RX_FRAME_VHT_GRP_ID);
+
+	stbc = le32_get_bits(phy_data->ntfy->sigs.vht.a1,
+			     OFDM_RX_FRAME_VHT_STBC);
+	if (stbc)
+		vht->flags |= IEEE80211_RADIOTAP_VHT_FLAG_STBC;
+
+	if (le32_get_bits(phy_data->ntfy->sigs.vht.a2,
+			  OFDM_RX_FRAME_VHT_SHORT_GI))
+		vht->flags |= IEEE80211_RADIOTAP_VHT_FLAG_SGI;
+
+	if (le32_get_bits(phy_data->ntfy->sigs.vht.a2,
+			  OFDM_RX_FRAME_VHT_SHORT_GI_AMBIG))
+		vht->flags |= IEEE80211_RADIOTAP_VHT_FLAG_SGI_NSYM_M10_9;
+
+	if (le32_get_bits(phy_data->ntfy->sigs.vht.a2,
+			  OFDM_RX_FRAME_VHT_CODING_EXTRA_SYM))
+		vht->flags |= IEEE80211_RADIOTAP_VHT_FLAG_LDPC_EXTRA_OFDM_SYM;
+
+	if (vht->group_id != 0 && vht->group_id != 63) {
+		/* MU frame */
+		int user = le32_get_bits(phy_data->ntfy->sigs.vht.a1,
+					 OFDM_RX_FRAME_VHT_MU_MIMO_USER_POSITION);
+		int nsts;
+
+		/* Always beamformed */
+		vht->flags |= IEEE80211_RADIOTAP_VHT_FLAG_BEAMFORMED;
+
+		/* No MCS information in the a1/a2 data for MU frames */
+		nsts = le32_get_bits(phy_data->ntfy->sigs.vht.a1,
+				     OFDM_RX_FRAME_VHT_STS_USER0);
+		vht->mcs_nss[0] = (stbc ? nsts / 2 : nsts) | 0xf0;
+
+		nsts = le32_get_bits(phy_data->ntfy->sigs.vht.a1,
+				     OFDM_RX_FRAME_VHT_MU_STS_USER1);
+		vht->mcs_nss[1] = (stbc ? nsts / 2 : nsts) | 0xf0;
+
+		nsts = le32_get_bits(phy_data->ntfy->sigs.vht.a1,
+				     OFDM_RX_FRAME_VHT_MU_STS_USER2);
+		vht->mcs_nss[2] = (stbc ? nsts / 2 : nsts) | 0xf0;
+
+		nsts = le32_get_bits(phy_data->ntfy->sigs.vht.a1,
+				      OFDM_RX_FRAME_VHT_MU_STS_USER3);
+		vht->mcs_nss[3] = (stbc ? nsts / 2 : nsts) | 0xf0;
+
+		/* Report current user MCS from rate_n_flags via rx_status */
+		vht->mcs_nss[user] &= 0x0f;
+		vht->mcs_nss[user] |= rx_status->rate_idx << 4;
+
+		/* Report LDPC for current user */
+		if (rx_status->enc_flags & RX_ENC_FLAG_LDPC)
+			vht->coding = 0x1 << user;
+	} else {
+		int nsts;
+
+		/* SU frame */
+		vht->known |= cpu_to_le16(IEEE80211_RADIOTAP_VHT_KNOWN_PARTIAL_AID);
+
+		if (le32_get_bits(phy_data->ntfy->sigs.vht.a2,
+				  OFDM_RX_FRAME_VHT_BF_OR_MU_RESERVED))
+			vht->flags |= IEEE80211_RADIOTAP_VHT_FLAG_BEAMFORMED;
+
+		vht->partial_aid =
+			cpu_to_le16(le32_get_bits(phy_data->ntfy->sigs.vht.a1,
+						  OFDM_RX_FRAME_VHT_PARTIAL_AID_OR_MU_STS));
+
+		nsts = le32_get_bits(phy_data->ntfy->sigs.vht.a1,
+				     OFDM_RX_FRAME_VHT_STS) + 1;
+		vht->mcs_nss[0] =
+			(stbc ? nsts / 2 : nsts) |
+			le32_get_bits(phy_data->ntfy->sigs.vht.a2,
+				      OFDM_RX_FRAME_VHT_MCS_OR_MU_CODING) << 4;
+		vht->mcs_nss[1] = 0;
+		vht->mcs_nss[2] = 0;
+		vht->mcs_nss[3] = 0;
+
+		if (rx_status->enc_flags & RX_ENC_FLAG_LDPC)
+			vht->coding = 0x1;
+	}
+}
+
+static void iwl_mld_rx_vht(struct sk_buff *skb,
+			   struct iwl_mld_rx_phy_data *phy_data)
+{
+	struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
+	struct ieee80211_radiotap_vht *vht;
+
+	if (likely(!phy_data->ntfy))
+		return;
+
+	vht = skb_put_zero(skb, sizeof(*vht));
+	rx_status->flag |= RX_FLAG_RADIOTAP_VHT;
+
+	iwl_mld_decode_vht_phy_data(phy_data, vht, rx_status);
+}
+
 static void
 iwl_mld_he_set_ru_alloc(struct ieee80211_rx_status *rx_status,
 			struct ieee80211_radiotap_he *he,
@@ -268,11 +396,11 @@ iwl_mld_decode_he_mu(struct iwl_mld_rx_phy_data *phy_data,
 {
 	u32 rate_n_flags = phy_data->rate_n_flags;
 
-	he_mu->flags1 |= RTAP_ENC_HE(phy_data->ntfy->sigs.he.b,
-				     OFDM_RX_FRAME_HE_SIGB_DCM,
+	he_mu->flags1 |= RTAP_ENC_HE(phy_data->ntfy->sigs.he.a1,
+				     OFDM_RX_FRAME_HE_DCM,
 				     IEEE80211_RADIOTAP_HE_MU_FLAGS1_SIG_B_DCM);
-	he_mu->flags1 |= RTAP_ENC_HE(phy_data->ntfy->sigs.he.b,
-				     OFDM_RX_FRAME_HE_SIGB_MCS,
+	he_mu->flags1 |= RTAP_ENC_HE(phy_data->ntfy->sigs.he.a1,
+				     OFDM_RX_FRAME_HE_MCS,
 				     IEEE80211_RADIOTAP_HE_MU_FLAGS1_SIG_B_MCS);
 	he_mu->flags2 |= RTAP_ENC_HE(phy_data->ntfy->sigs.he.a1,
 				     OFDM_RX_FRAME_HE_PRMBL_PUNC_TYPE,
@@ -280,7 +408,7 @@ iwl_mld_decode_he_mu(struct iwl_mld_rx_phy_data *phy_data,
 	he_mu->flags2 |= RTAP_ENC_HE(phy_data->ntfy->sigs.he.a2,
 				     OFDM_RX_FRAME_HE_MU_NUM_OF_SIGB_SYM_OR_USER_NUM,
 				     IEEE80211_RADIOTAP_HE_MU_FLAGS2_SIG_B_SYMS_USERS);
-	he_mu->flags2 |= RTAP_ENC_HE(phy_data->ntfy->sigs.he.b,
+	he_mu->flags2 |= RTAP_ENC_HE(phy_data->ntfy->sigs.he.a2,
 				     OFDM_RX_FRAME_HE_MU_SIGB_COMP,
 				     IEEE80211_RADIOTAP_HE_MU_FLAGS2_SIG_B_COMP);
 
@@ -1377,6 +1505,10 @@ static void iwl_mld_rx_fill_status(struct iwl_mld *mld, int link_id,
 
 	iwl_mld_set_rx_rate(mld, phy_data, rx_status);
 
+	/* must be before HE data (radiotap field order) */
+	if (format == RATE_MCS_MOD_TYPE_VHT)
+		iwl_mld_rx_vht(skb, phy_data);
+
 	/* must be before L-SIG data (radiotap field order) */
 	if (format == RATE_MCS_MOD_TYPE_HE)
 		iwl_mld_rx_he(skb, phy_data);
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/scan.c b/drivers/net/wireless/intel/iwlwifi/mld/scan.c
index fd1022ddc912..16f48087a888 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/scan.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/scan.c
@@ -1063,14 +1063,15 @@ static int
 iwl_mld_scan_cmd_set_6ghz_chan_params(struct iwl_mld *mld,
 				      struct iwl_mld_scan_params *params,
 				      struct ieee80211_vif *vif,
-				      struct iwl_scan_req_params_v17 *scan_p,
-				      enum iwl_mld_scan_status scan_status)
+				      struct iwl_scan_req_params_v17 *scan_p)
 {
 	struct iwl_scan_channel_params_v7 *chan_p = &scan_p->channel_params;
 	struct iwl_scan_probe_params_v4 *probe_p = &scan_p->probe_params;
 
-	chan_p->flags = iwl_mld_scan_get_cmd_gen_flags(mld, params, vif,
-						       scan_status);
+	/* Explicitly clear the flags since most of them are not
+	 * relevant for 6 GHz scan.
+	 */
+	chan_p->flags = 0;
 	chan_p->count = iwl_mld_scan_cfg_channels_6g(mld, params,
 						     params->n_channels,
 						     probe_p, chan_p,
@@ -1106,8 +1107,7 @@ iwl_mld_scan_cmd_set_chan_params(struct iwl_mld *mld,
 
 	if (params->scan_6ghz)
 		return iwl_mld_scan_cmd_set_6ghz_chan_params(mld, params,
-							     vif, scan_p,
-							     scan_status);
+							     vif, scan_p);
 
 	/* relevant only for 2.4 GHz/5 GHz scan */
 	cp->flags = iwl_mld_scan_cmd_set_chan_flags(mld, params, vif,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/sta.c b/drivers/net/wireless/intel/iwlwifi/mld/sta.c
index 61ecc33116cf..6056a306f7cb 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/sta.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/sta.c
@@ -1163,7 +1163,8 @@ void iwl_mld_remove_aux_sta(struct iwl_mld *mld,
 	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
 
 	if (WARN_ON(vif->type != NL80211_IFTYPE_P2P_DEVICE &&
-		    vif->type != NL80211_IFTYPE_STATION))
+		    vif->type != NL80211_IFTYPE_STATION &&
+		    vif->type != NL80211_IFTYPE_NAN))
 		return;
 
 	iwl_mld_remove_internal_sta(mld, &mld_vif->aux_sta, false,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/tlc.c b/drivers/net/wireless/intel/iwlwifi/mld/tlc.c
index 0e172281b0c8..62a54c37a98c 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/tlc.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/tlc.c
@@ -8,6 +8,7 @@
 #include "tlc.h"
 #include "hcmd.h"
 #include "sta.h"
+#include "phy.h"
 
 #include "fw/api/rs.h"
 #include "fw/api/context.h"
@@ -447,11 +448,48 @@ iwl_mld_fill_supp_rates(struct iwl_mld *mld, struct ieee80211_vif *vif,
 	}
 }
 
-static void iwl_mld_convert_tlc_cmd_to_v4(struct iwl_tlc_config_cmd *cmd,
-					  struct iwl_tlc_config_cmd_v4 *cmd_v4)
+static int iwl_mld_convert_tlc_cmd_to_v5(struct iwl_tlc_config_cmd *cmd,
+					 struct iwl_tlc_config_cmd_v5 *cmd_v5)
 {
+	if (WARN_ON_ONCE(hweight32(le32_to_cpu(cmd->sta_mask)) != 1))
+		return -EINVAL;
+
+	/* Convert sta_mask to sta_id */
+	cmd_v5->sta_id = __ffs(le32_to_cpu(cmd->sta_mask));
+
+	/* Copy all the rest */
+	cmd_v5->max_ch_width = cmd->max_ch_width;
+	cmd_v5->mode = cmd->mode;
+	cmd_v5->chains = cmd->chains;
+	cmd_v5->sgi_ch_width_supp = cmd->sgi_ch_width_supp;
+	cmd_v5->flags = cmd->flags;
+	cmd_v5->non_ht_rates = cmd->non_ht_rates;
+
+	BUILD_BUG_ON(sizeof(cmd_v5->ht_rates) != sizeof(cmd->ht_rates));
+	memcpy(cmd_v5->ht_rates, cmd->ht_rates, sizeof(cmd->ht_rates));
+
+	cmd_v5->max_mpdu_len = cmd->max_mpdu_len;
+	cmd_v5->max_tx_op = cmd->max_tx_op;
+
+	return 0;
+}
+
+static int iwl_mld_convert_tlc_cmd_to_v4(struct iwl_tlc_config_cmd *cmd,
+					 struct iwl_tlc_config_cmd_v4 *cmd_v4)
+{
+	if (WARN_ON_ONCE(hweight32(le32_to_cpu(cmd->sta_mask)) != 1))
+		return -EINVAL;
+
+	/* Convert sta_mask to sta_id */
+	cmd_v4->sta_id = __ffs(le32_to_cpu(cmd->sta_mask));
+
 	/* Copy everything until ht_rates */
-	memcpy(cmd_v4, cmd, offsetof(struct iwl_tlc_config_cmd, ht_rates));
+	cmd_v4->max_ch_width = cmd->max_ch_width;
+	cmd_v4->mode = cmd->mode;
+	cmd_v4->chains = cmd->chains;
+	cmd_v4->sgi_ch_width_supp = cmd->sgi_ch_width_supp;
+	cmd_v4->flags = cmd->flags;
+	cmd_v4->non_ht_rates = cmd->non_ht_rates;
 
 	/* Convert ht_rates from __le32 to __le16 */
 	BUILD_BUG_ON(ARRAY_SIZE(cmd_v4->ht_rates) != ARRAY_SIZE(cmd->ht_rates));
@@ -465,14 +503,17 @@ static void iwl_mld_convert_tlc_cmd_to_v4(struct iwl_tlc_config_cmd *cmd,
 	/* Copy the rest */
 	cmd_v4->max_mpdu_len = cmd->max_mpdu_len;
 	cmd_v4->max_tx_op = cmd->max_tx_op;
+
+	return 0;
 }
 
 static void iwl_mld_send_tlc_cmd(struct iwl_mld *mld,
 				 struct ieee80211_vif *vif,
 				 struct ieee80211_link_sta *link_sta,
-				 enum nl80211_band band)
+				 struct ieee80211_bss_conf *link)
 {
 	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(link_sta->sta);
+	enum nl80211_band band = link->chanreq.oper.chan->band;
 	struct ieee80211_supported_band *sband = mld->hw->wiphy->bands[band];
 	const struct ieee80211_sta_he_cap *own_he_cap =
 		ieee80211_get_he_iftype_cap_vif(sband, vif);
@@ -492,25 +533,44 @@ static void iwl_mld_send_tlc_cmd(struct iwl_mld *mld,
 	int fw_sta_id = iwl_mld_fw_sta_id_from_link_sta(mld, link_sta);
 	u32 cmd_id = WIDE_ID(DATA_PATH_GROUP, TLC_MNG_CONFIG_CMD);
 	u8 cmd_ver = iwl_fw_lookup_cmd_ver(mld->fw, cmd_id, 0);
-	struct iwl_tlc_config_cmd_v4 cmd_v4;
+	struct ieee80211_chanctx_conf *chan_ctx;
+	struct iwl_tlc_config_cmd_v5 cmd_v5 = {};
+	struct iwl_tlc_config_cmd_v4 cmd_v4 = {};
 	void *cmd_ptr;
 	u8 cmd_size;
+	u32 phy_id;
 	int ret;
 
 	if (fw_sta_id < 0)
 		return;
 
-	cmd.sta_id = fw_sta_id;
+	cmd.sta_mask = cpu_to_le32(BIT(fw_sta_id));
+
+	chan_ctx = rcu_dereference_wiphy(mld->wiphy, link->chanctx_conf);
+	if (WARN_ON(!chan_ctx))
+		return;
+
+	phy_id = iwl_mld_phy_from_mac80211(chan_ctx)->fw_id;
+	cmd.phy_id = cpu_to_le32(phy_id);
 
 	iwl_mld_fill_supp_rates(mld, vif, link_sta, sband,
 				own_he_cap, own_eht_cap,
 				&cmd);
 
-	if (cmd_ver == 5) {
+	if (cmd_ver == 6) {
 		cmd_ptr = &cmd;
 		cmd_size = sizeof(cmd);
+	} else if (cmd_ver == 5) {
+		/* TODO: remove support once FW moves to version 6 */
+		ret = iwl_mld_convert_tlc_cmd_to_v5(&cmd, &cmd_v5);
+		if (ret)
+			return;
+		cmd_ptr = &cmd_v5;
+		cmd_size = sizeof(cmd_v5);
 	} else if (cmd_ver == 4) {
-		iwl_mld_convert_tlc_cmd_to_v4(&cmd, &cmd_v4);
+		ret = iwl_mld_convert_tlc_cmd_to_v4(&cmd, &cmd_v4);
+		if (ret)
+			return;
 		cmd_ptr = &cmd_v4;
 		cmd_size = sizeof(cmd_v4);
 	} else {
@@ -520,8 +580,9 @@ static void iwl_mld_send_tlc_cmd(struct iwl_mld *mld,
 	}
 
 	IWL_DEBUG_RATE(mld,
-		       "TLC CONFIG CMD, sta_id=%d, max_ch_width=%d, mode=%d\n",
-		       cmd.sta_id, cmd.max_ch_width, cmd.mode);
+		       "TLC CONFIG CMD, sta_mask=0x%x, max_ch_width=%d, mode=%d, phy_id=%d\n",
+		       le32_to_cpu(cmd.sta_mask), cmd.max_ch_width, cmd.mode,
+		       le32_to_cpu(cmd.phy_id));
 
 	/* Send async since this can be called within a RCU-read section */
 	ret = iwl_mld_send_cmd_with_flags_pdu(mld, cmd_id, CMD_ASYNC, cmd_ptr,
@@ -561,7 +622,6 @@ void iwl_mld_config_tlc_link(struct iwl_mld *mld,
 			     struct ieee80211_link_sta *link_sta)
 {
 	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(link_sta->sta);
-	enum nl80211_band band;
 
 	if (WARN_ON_ONCE(!link_conf->chanreq.oper.chan))
 		return;
@@ -575,8 +635,7 @@ void iwl_mld_config_tlc_link(struct iwl_mld *mld,
 		ieee80211_sta_recalc_aggregates(link_sta->sta);
 	}
 
-	band = link_conf->chanreq.oper.chan->band;
-	iwl_mld_send_tlc_cmd(mld, vif, link_sta, band);
+	iwl_mld_send_tlc_cmd(mld, vif, link_sta, link_conf);
 }
 
 void iwl_mld_config_tlc(struct iwl_mld *mld, struct ieee80211_vif *vif,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/tx.c b/drivers/net/wireless/intel/iwlwifi/mld/tx.c
index 3b4b575aadaa..546d09a38dab 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/tx.c
@@ -345,6 +345,11 @@ u8 iwl_mld_get_lowest_rate(struct iwl_mld *mld,
 
 	iwl_mld_get_basic_rates_and_band(mld, vif, info, &basic_rates, &band);
 
+	if (band >= NUM_NL80211_BANDS) {
+		WARN_ON(vif->type != NL80211_IFTYPE_NAN);
+		return IWL_FIRST_OFDM_RATE;
+	}
+
 	sband = mld->hw->wiphy->bands[band];
 	for_each_set_bit(i, &basic_rates, BITS_PER_LONG) {
 		u16 hw = sband->bitrates[i].hw_value;
@@ -666,6 +671,12 @@ iwl_mld_get_tx_queue_id(struct iwl_mld *mld, struct ieee80211_txq *txq,
 		}
 
 		WARN_ON(!ieee80211_is_mgmt(fc));
+		return mld_vif->aux_sta.queue_id;
+	case NL80211_IFTYPE_NAN:
+		mld_vif = iwl_mld_vif_from_mac80211(info->control.vif);
+
+		WARN_ON(!ieee80211_is_mgmt(fc));
+
 		return mld_vif->aux_sta.queue_id;
 	default:
 		WARN_ONCE(1, "Unsupported vif type\n");
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
index 683c0ba5fb39..e6b9896dc4ac 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
@@ -6,6 +6,7 @@
  */
 #include <linux/vmalloc.h>
 #include <linux/err.h>
+#include <linux/hex.h>
 #include <linux/ieee80211.h>
 #include <linux/netdevice.h>
 #include <linux/dmi.h>
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
index 867807abde66..49ffc4ecee85 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
@@ -1761,6 +1761,20 @@ void iwl_mvm_probe_resp_data_notif(struct iwl_mvm *mvm,
 
 	mvmvif = iwl_mvm_vif_from_mac80211(vif);
 
+	/*
+	 * len_low should be 2 + n*13 (where n is the number of descriptors.
+	 * 13 is the size of a NoA descriptor). We can have either one or two
+	 * descriptors.
+	 */
+	if (IWL_FW_CHECK(mvm, notif->noa_active &&
+			 notif->noa_attr.len_low != 2 +
+			 sizeof(struct ieee80211_p2p_noa_desc) &&
+			 notif->noa_attr.len_low != 2 +
+			 sizeof(struct ieee80211_p2p_noa_desc) * 2,
+			 "Invalid noa_attr.len_low (%d)\n",
+			 notif->noa_attr.len_low))
+		return;
+
 	new_data = kzalloc(sizeof(*new_data), GFP_KERNEL);
 	if (!new_data)
 		return;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
index 301d590fe0bd..d7e9c2b7980e 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
@@ -11,6 +11,7 @@
 #include <linux/spinlock.h>
 #include <linux/cleanup.h>
 #include <linux/leds.h>
+#include <linux/hex.h>
 #include <linux/in6.h>
 
 #ifdef CONFIG_THERMAL
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c b/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c
index 06a4c9f74797..ad156b82eaa9 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c
@@ -220,6 +220,12 @@ static int iwl_mvm_ptp_gettime(struct ptp_clock_info *ptp,
 	return 0;
 }
 
+static int iwl_mvm_ptp_settime(struct ptp_clock_info *ptp,
+			       const struct timespec64 *ts)
+{
+	return -EOPNOTSUPP;
+}
+
 static int iwl_mvm_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
 {
 	struct iwl_mvm *mvm = container_of(ptp, struct iwl_mvm,
@@ -281,6 +287,7 @@ void iwl_mvm_ptp_init(struct iwl_mvm *mvm)
 	mvm->ptp_data.ptp_clock_info.adjfine = iwl_mvm_ptp_adjfine;
 	mvm->ptp_data.ptp_clock_info.adjtime = iwl_mvm_ptp_adjtime;
 	mvm->ptp_data.ptp_clock_info.gettime64 = iwl_mvm_ptp_gettime;
+	mvm->ptp_data.ptp_clock_info.settime64 = iwl_mvm_ptp_settime;
 	mvm->ptp_data.scaled_freq = SCALE_FACTOR;
 
 	/* Give a short 'friendly name' to identify the PHC clock */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
index b588f1dcf20d..59331f1d144b 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
@@ -2568,16 +2568,16 @@ static int iwl_mvm_scan_umac_v14_and_above(struct iwl_mvm *mvm,
 					       bitmap_ssid,
 					       version);
 		return 0;
-	} else {
-		pb->preq = params->preq;
 	}
 
-	cp->flags = iwl_mvm_scan_umac_chan_flags_v2(mvm, params, vif);
-	cp->n_aps_override[0] = IWL_SCAN_ADWELL_N_APS_GO_FRIENDLY;
-	cp->n_aps_override[1] = IWL_SCAN_ADWELL_N_APS_SOCIAL_CHS;
+	pb->preq = params->preq;
 
 	iwl_mvm_umac_scan_fill_6g_chan_list(mvm, params, pb);
 
+	/* Explicitly clear the flags since most of them are not
+	 * relevant for 6 GHz scan.
+	 */
+	cp->flags = 0;
 	cp->count = iwl_mvm_umac_scan_cfg_channels_v7_6g(mvm, params,
 							 params->n_channels,
 							 pb, cp, vif->type,
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h b/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h
index e16865dd8e52..c93fd245c90f 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h
@@ -5,6 +5,7 @@
 #define __MT7615_H
 
 #include <linux/completion.h>
+#include <linux/hex.h>
 #include <linux/interrupt.h>
 #include <linux/ktime.h>
 #include <linux/regmap.h>
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.c
index aa702ba7c9f5..d6c35e8d02a5 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.c
@@ -511,7 +511,8 @@ void rtl92cu_tx_fill_desc(struct ieee80211_hw *hw,
 	if (sta) {
 		sta_entry = (struct rtl_sta_info *)sta->drv_priv;
 		tid = ieee80211_get_tid(hdr);
-		agg_state = sta_entry->tids[tid].agg.agg_state;
+		if (tid < MAX_TID_COUNT)
+			agg_state = sta_entry->tids[tid].agg.agg_state;
 		ampdu_density = sta->deflink.ht_cap.ampdu_density;
 	}
 
diff --git a/drivers/net/wireless/realtek/rtw88/sdio.c b/drivers/net/wireless/realtek/rtw88/sdio.c
index 99d7c629eac6..e35de52d8eb4 100644
--- a/drivers/net/wireless/realtek/rtw88/sdio.c
+++ b/drivers/net/wireless/realtek/rtw88/sdio.c
@@ -144,8 +144,10 @@ static u32 rtw_sdio_to_io_address(struct rtw_dev *rtwdev, u32 addr,
 
 static bool rtw_sdio_use_direct_io(struct rtw_dev *rtwdev, u32 addr)
 {
+	bool might_indirect_under_power_off = rtwdev->chip->id == RTW_CHIP_TYPE_8822C;
+
 	if (!test_bit(RTW_FLAG_POWERON, rtwdev->flags) &&
-	    !rtw_sdio_is_bus_addr(addr))
+	    !rtw_sdio_is_bus_addr(addr) && might_indirect_under_power_off)
 		return false;
 
 	return !rtw_sdio_is_sdio30_supported(rtwdev) ||
diff --git a/drivers/net/wireless/realtek/rtw88/usb.c b/drivers/net/wireless/realtek/rtw88/usb.c
index 009202c627d2..3b5126ffc81a 100644
--- a/drivers/net/wireless/realtek/rtw88/usb.c
+++ b/drivers/net/wireless/realtek/rtw88/usb.c
@@ -965,8 +965,7 @@ static int rtw_usb_init_rx(struct rtw_dev *rtwdev)
 	struct sk_buff *rx_skb;
 	int i;
 
-	rtwusb->rxwq = alloc_workqueue("rtw88_usb: rx wq", WQ_BH | WQ_UNBOUND,
-				       0);
+	rtwusb->rxwq = alloc_workqueue("rtw88_usb: rx wq", WQ_BH, 0);
 	if (!rtwusb->rxwq) {
 		rtw_err(rtwdev, "failed to create RX work queue\n");
 		return -ENOMEM;
diff --git a/drivers/net/wireless/realtek/rtw89/debug.c b/drivers/net/wireless/realtek/rtw89/debug.c
index 1264c2f82600..8666e26c8c80 100644
--- a/drivers/net/wireless/realtek/rtw89/debug.c
+++ b/drivers/net/wireless/realtek/rtw89/debug.c
@@ -2,6 +2,7 @@
 /* Copyright(c) 2019-2020  Realtek Corporation
  */
 
+#include <linux/hex.h>
 #include <linux/vmalloc.h>
 
 #include "coex.h"
diff --git a/drivers/net/wireless/silabs/wfx/fwio.c b/drivers/net/wireless/silabs/wfx/fwio.c
index 52c7f560b062..edd5ac30ed19 100644
--- a/drivers/net/wireless/silabs/wfx/fwio.c
+++ b/drivers/net/wireless/silabs/wfx/fwio.c
@@ -6,6 +6,7 @@
  * Copyright (c) 2010, ST-Ericsson
  */
 #include <linux/firmware.h>
+#include <linux/hex.h>
 #include <linux/slab.h>
 #include <linux/mm.h>
 #include <linux/bitfield.h>
diff --git a/drivers/net/wireless/ti/wlcore/tx.c b/drivers/net/wireless/ti/wlcore/tx.c
index f76087be2f75..6241866d39df 100644
--- a/drivers/net/wireless/ti/wlcore/tx.c
+++ b/drivers/net/wireless/ti/wlcore/tx.c
@@ -207,6 +207,11 @@ static int wl1271_tx_allocate(struct wl1271 *wl, struct wl12xx_vif *wlvif,
 	total_blocks = wlcore_hw_calc_tx_blocks(wl, total_len, spare_blocks);
 
 	if (total_blocks <= wl->tx_blocks_available) {
+		if (skb_headroom(skb) < (total_len - skb->len) &&
+		    pskb_expand_head(skb, (total_len - skb->len), 0, GFP_ATOMIC)) {
+			wl1271_free_tx_id(wl, id);
+			return -EAGAIN;
+		}
 		desc = skb_push(skb, total_len - skb->len);
 
 		wlcore_hw_set_tx_desc_blocks(wl, desc, total_blocks,
diff --git a/drivers/nfc/pn533/usb.c b/drivers/nfc/pn533/usb.c
index ffd7367ce119..018a80674f06 100644
--- a/drivers/nfc/pn533/usb.c
+++ b/drivers/nfc/pn533/usb.c
@@ -406,7 +406,7 @@ static int pn533_acr122_poweron_rdr(struct pn533_usb_phy *phy)
 	if (rc || (transferred != sizeof(cmd))) {
 		nfc_err(&phy->udev->dev,
 			"Reader power on cmd error %d\n", rc);
-		return rc;
+		return rc ?: -EINVAL;
 	}
 
 	rc =  usb_submit_urb(phy->in_urb, GFP_KERNEL);
diff --git a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c
index f851397b65d6..e38540b92716 100644
--- a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c
+++ b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c
@@ -29,7 +29,7 @@ MODULE_PARM_DESC(use_lut_mws,
 		 "Enable the use of the LUT based memory windows");
 
 #define SWITCHTEC_NTB_MAGIC 0x45CC0001
-#define MAX_MWS     128
+#define MAX_MWS     256
 
 struct shared_mw {
 	u32 magic;
@@ -1202,7 +1202,8 @@ static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)
 				       sndev->mmio_self_ctrl);
 
 	sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries);
-	sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw);
+	if (sndev->nr_lut_mw)
+		sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw);
 
 	dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n",
 		sndev->nr_direct_mw, sndev->nr_lut_mw);
@@ -1212,7 +1213,8 @@ static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)
 
 	sndev->peer_nr_lut_mw =
 		ioread16(&sndev->mmio_peer_ctrl->lut_table_entries);
-	sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw);
+	if (sndev->peer_nr_lut_mw)
+		sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw);
 
 	dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n",
 		sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw);
@@ -1314,6 +1316,12 @@ static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev)
 	for (i = 0; i < sndev->nr_lut_mw; i++) {
 		int idx = sndev->nr_direct_mw + i;
 
+		if (idx >= MAX_MWS) {
+			dev_err(&sndev->stdev->dev,
+				"Total number of MW cannot be bigger than %d", MAX_MWS);
+			break;
+		}
+
 		sndev->self_shared->mw_sizes[idx] = LUT_SIZE;
 	}
 }
diff --git a/drivers/ntb/msi.c b/drivers/ntb/msi.c
index 368f6d894bba..6817d504c12a 100644
--- a/drivers/ntb/msi.c
+++ b/drivers/ntb/msi.c
@@ -315,37 +315,6 @@ int ntbm_msi_request_threaded_irq(struct ntb_dev *ntb, irq_handler_t handler,
 }
 EXPORT_SYMBOL(ntbm_msi_request_threaded_irq);
 
-static int ntbm_msi_callback_match(struct device *dev, void *res, void *data)
-{
-	struct ntb_dev *ntb = dev_ntb(dev);
-	struct ntb_msi_devres *dr = res;
-
-	return dr->ntb == ntb && dr->entry == data;
-}
-
-/**
- * ntbm_msi_free_irq() - free an interrupt
- * @ntb:	NTB device context
- * @irq:	Interrupt line to free
- * @dev_id:	Device identity to free
- *
- * This function should be used to manually free IRQs allocated with
- * ntbm_request_[threaded_]irq().
- */
-void ntbm_msi_free_irq(struct ntb_dev *ntb, unsigned int irq, void *dev_id)
-{
-	struct msi_desc *entry = irq_get_msi_desc(irq);
-
-	entry->write_msi_msg = NULL;
-	entry->write_msi_msg_data = NULL;
-
-	WARN_ON(devres_destroy(&ntb->dev, ntbm_msi_callback_release,
-			       ntbm_msi_callback_match, entry));
-
-	devm_free_irq(&ntb->dev, irq, dev_id);
-}
-EXPORT_SYMBOL(ntbm_msi_free_irq);
-
 /**
  * ntb_msi_peer_trigger() - Trigger an interrupt handler on a peer
  * @ntb:	NTB device context
@@ -373,36 +342,3 @@ int ntb_msi_peer_trigger(struct ntb_dev *ntb, int peer,
 	return 0;
 }
 EXPORT_SYMBOL(ntb_msi_peer_trigger);
-
-/**
- * ntb_msi_peer_addr() - Get the DMA address to trigger a peer's MSI interrupt
- * @ntb:	NTB device context
- * @peer:	Peer index
- * @desc:	MSI descriptor data which triggers the interrupt
- * @msi_addr:   Physical address to trigger the interrupt
- *
- * This function allows using DMA engines to trigger an interrupt
- * (for example, trigger an interrupt to process the data after
- * sending it). To trigger the interrupt, write @desc.data to the address
- * returned in @msi_addr
- *
- * Return: Zero on success, otherwise a negative error number.
- */
-int ntb_msi_peer_addr(struct ntb_dev *ntb, int peer,
-		      struct ntb_msi_desc *desc,
-		      phys_addr_t *msi_addr)
-{
-	int peer_widx = ntb_peer_mw_count(ntb) - 1 - peer;
-	phys_addr_t mw_phys_addr;
-	int ret;
-
-	ret = ntb_peer_mw_get_addr(ntb, peer_widx, &mw_phys_addr, NULL);
-	if (ret)
-		return ret;
-
-	if (msi_addr)
-		*msi_addr = mw_phys_addr + desc->addr_offset;
-
-	return 0;
-}
-EXPORT_SYMBOL(ntb_msi_peer_addr);
diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c
index e44ef69dffc2..127dae51fec1 100644
--- a/drivers/nvme/target/configfs.c
+++ b/drivers/nvme/target/configfs.c
@@ -4,6 +4,7 @@
  * Copyright (c) 2015-2016 HGST, a Western Digital Company.
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/hex.h>
 #include <linux/kstrtox.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index cc88e5a28c8a..eab3e4fc0f74 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -4,6 +4,7 @@
  * Copyright (c) 2015-2016 HGST, a Western Digital Company.
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/hex.h>
 #include <linux/module.h>
 #include <linux/random.h>
 #include <linux/rculist.h>
diff --git a/drivers/nvmem/brcm_nvram.c b/drivers/nvmem/brcm_nvram.c
index b4cf245fb246..2dce6a7b8039 100644
--- a/drivers/nvmem/brcm_nvram.c
+++ b/drivers/nvmem/brcm_nvram.c
@@ -5,6 +5,7 @@
 
 #include <linux/bcm47xx_nvram.h>
 #include <linux/etherdevice.h>
+#include <linux/hex.h>
 #include <linux/if_ether.h>
 #include <linux/io.h>
 #include <linux/mod_devicetable.h>
diff --git a/drivers/nvmem/layouts/u-boot-env.c b/drivers/nvmem/layouts/u-boot-env.c
index ab32bf1291af..f27f387bb52a 100644
--- a/drivers/nvmem/layouts/u-boot-env.c
+++ b/drivers/nvmem/layouts/u-boot-env.c
@@ -6,6 +6,7 @@
 #include <linux/crc32.h>
 #include <linux/etherdevice.h>
 #include <linux/export.h>
+#include <linux/hex.h>
 #include <linux/if_ether.h>
 #include <linux/nvmem-consumer.h>
 #include <linux/nvmem-provider.h>
diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c
index d378d4b4109f..331646d667b9 100644
--- a/drivers/of/fdt.c
+++ b/drivers/of/fdt.c
@@ -503,8 +503,8 @@ void __init early_init_fdt_scan_reserved_mem(void)
 	if (!initial_boot_params)
 		return;
 
-	fdt_scan_reserved_mem();
 	fdt_reserve_elfcorehdr();
+	fdt_scan_reserved_mem();
 
 	/* Process header /memreserve/ fields */
 	for (n = 0; ; n++) {
diff --git a/drivers/of/property.c b/drivers/of/property.c
index 4e3524227720..ce5ada040d7e 100644
--- a/drivers/of/property.c
+++ b/drivers/of/property.c
@@ -1294,17 +1294,6 @@ static struct device_node *parse_##fname(struct device_node *np,	  \
 	return parse_prop_cells(np, prop_name, index, name, cells);	  \
 }
 
-static int strcmp_suffix(const char *str, const char *suffix)
-{
-	unsigned int len, suffix_len;
-
-	len = strlen(str);
-	suffix_len = strlen(suffix);
-	if (len <= suffix_len)
-		return -1;
-	return strcmp(str + len - suffix_len, suffix);
-}
-
 /**
  * parse_suffix_prop_cells - Suffix property parsing function for suppliers
  *
@@ -1331,7 +1320,7 @@ static struct device_node *parse_suffix_prop_cells(struct device_node *np,
 {
 	struct of_phandle_args sup_args;
 
-	if (strcmp_suffix(prop_name, suffix))
+	if (!strends(prop_name, suffix))
 		return NULL;
 
 	if (of_parse_phandle_with_args(np, prop_name, cells_name, index,
@@ -1416,7 +1405,7 @@ DEFINE_SUFFIX_PROP(gpio, "-gpio", "#gpio-cells")
 static struct device_node *parse_gpios(struct device_node *np,
 				       const char *prop_name, int index)
 {
-	if (!strcmp_suffix(prop_name, ",nr-gpios"))
+	if (strends(prop_name, ",nr-gpios"))
 		return NULL;
 
 	return parse_suffix_prop_cells(np, prop_name, index, "-gpios",
diff --git a/drivers/parisc/sba_iommu.c b/drivers/parisc/sba_iommu.c
index a6eb6bffa5ea..eefb2bac8443 100644
--- a/drivers/parisc/sba_iommu.c
+++ b/drivers/parisc/sba_iommu.c
@@ -578,8 +578,8 @@ sba_io_pdir_entry(__le64 *pdir_ptr, space_t sid, phys_addr_t pba,
 	pba &= IOVP_MASK;
 	pba |= (ci >> PAGE_SHIFT) & 0xff;  /* move CI (8 bits) into lowest byte */
 
-	pba |= SBA_PDIR_VALID_BIT;	/* set "valid" bit */
-	*pdir_ptr = cpu_to_le64(pba);	/* swap and store into I/O Pdir */
+	/* set "valid" bit, swap and store into I/O Pdir */
+	*pdir_ptr = cpu_to_le64((unsigned long)pba | SBA_PDIR_VALID_BIT);
 
 	/*
 	 * If the PDC_MODEL capabilities has Non-coherent IO-PDIR bit set
diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
index 372207c33a85..f116591975ff 100644
--- a/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -26,9 +26,27 @@ static struct pci_ops dw_pcie_ops;
 static struct pci_ops dw_pcie_ecam_ops;
 static struct pci_ops dw_child_pcie_ops;
 
+#ifdef CONFIG_SMP
+static void dw_irq_noop(struct irq_data *d) { }
+#endif
+
+static bool dw_pcie_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
+				      struct irq_domain *real_parent, struct msi_domain_info *info)
+{
+	if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info))
+		return false;
+
+#ifdef CONFIG_SMP
+	info->chip->irq_ack = dw_irq_noop;
+	info->chip->irq_pre_redirect = irq_chip_pre_redirect_parent;
+#else
+	info->chip->irq_ack = irq_chip_ack_parent;
+#endif
+	return true;
+}
+
 #define DW_PCIE_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS		| \
 				    MSI_FLAG_USE_DEF_CHIP_OPS		| \
-				    MSI_FLAG_NO_AFFINITY		| \
 				    MSI_FLAG_PCI_MSI_MASK_PARENT)
 #define DW_PCIE_MSI_FLAGS_SUPPORTED (MSI_FLAG_MULTI_PCI_MSI		| \
 				     MSI_FLAG_PCI_MSIX			| \
@@ -40,41 +58,30 @@ static const struct msi_parent_ops dw_pcie_msi_parent_ops = {
 	.required_flags		= DW_PCIE_MSI_FLAGS_REQUIRED,
 	.supported_flags	= DW_PCIE_MSI_FLAGS_SUPPORTED,
 	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
-	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
 	.prefix			= "DW-",
-	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
+	.init_dev_msi_info	= dw_pcie_init_dev_msi_info,
 };
 
 /* MSI int handler */
-irqreturn_t dw_handle_msi_irq(struct dw_pcie_rp *pp)
+void dw_handle_msi_irq(struct dw_pcie_rp *pp)
 {
-	int i, pos;
-	unsigned long val;
-	u32 status, num_ctrls;
-	irqreturn_t ret = IRQ_NONE;
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	unsigned int i, num_ctrls;
 
 	num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
 
 	for (i = 0; i < num_ctrls; i++) {
-		status = dw_pcie_readl_dbi(pci, PCIE_MSI_INTR0_STATUS +
-					   (i * MSI_REG_CTRL_BLOCK_SIZE));
+		unsigned int reg_off = i * MSI_REG_CTRL_BLOCK_SIZE;
+		unsigned int irq_off = i * MAX_MSI_IRQS_PER_CTRL;
+		unsigned long status, pos;
+
+		status = dw_pcie_readl_dbi(pci, PCIE_MSI_INTR0_STATUS + reg_off);
 		if (!status)
 			continue;
 
-		ret = IRQ_HANDLED;
-		val = status;
-		pos = 0;
-		while ((pos = find_next_bit(&val, MAX_MSI_IRQS_PER_CTRL,
-					    pos)) != MAX_MSI_IRQS_PER_CTRL) {
-			generic_handle_domain_irq(pp->irq_domain,
-						  (i * MAX_MSI_IRQS_PER_CTRL) +
-						  pos);
-			pos++;
-		}
+		for_each_set_bit(pos, &status, MAX_MSI_IRQS_PER_CTRL)
+			generic_handle_demux_domain_irq(pp->irq_domain, irq_off + pos);
 	}
-
-	return ret;
 }
 
 /* Chained MSI interrupt service routine */
@@ -95,13 +102,10 @@ static void dw_pci_setup_msi_msg(struct irq_data *d, struct msi_msg *msg)
 {
 	struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d);
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
-	u64 msi_target;
-
-	msi_target = (u64)pp->msi_data;
+	u64 msi_target = (u64)pp->msi_data;
 
 	msg->address_lo = lower_32_bits(msi_target);
 	msg->address_hi = upper_32_bits(msi_target);
-
 	msg->data = d->hwirq;
 
 	dev_dbg(pci->dev, "msi#%d address_hi %#x address_lo %#x\n",
@@ -113,18 +117,14 @@ static void dw_pci_bottom_mask(struct irq_data *d)
 	struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d);
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	unsigned int res, bit, ctrl;
-	unsigned long flags;
-
-	raw_spin_lock_irqsave(&pp->lock, flags);
 
+	guard(raw_spinlock)(&pp->lock);
 	ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
 	res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
 	bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
 
 	pp->irq_mask[ctrl] |= BIT(bit);
 	dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]);
-
-	raw_spin_unlock_irqrestore(&pp->lock, flags);
 }
 
 static void dw_pci_bottom_unmask(struct irq_data *d)
@@ -132,18 +132,14 @@ static void dw_pci_bottom_unmask(struct irq_data *d)
 	struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d);
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	unsigned int res, bit, ctrl;
-	unsigned long flags;
-
-	raw_spin_lock_irqsave(&pp->lock, flags);
 
+	guard(raw_spinlock)(&pp->lock);
 	ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
 	res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
 	bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
 
 	pp->irq_mask[ctrl] &= ~BIT(bit);
 	dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]);
-
-	raw_spin_unlock_irqrestore(&pp->lock, flags);
 }
 
 static void dw_pci_bottom_ack(struct irq_data *d)
@@ -160,54 +156,48 @@ static void dw_pci_bottom_ack(struct irq_data *d)
 }
 
 static struct irq_chip dw_pci_msi_bottom_irq_chip = {
-	.name = "DWPCI-MSI",
-	.irq_ack = dw_pci_bottom_ack,
-	.irq_compose_msi_msg = dw_pci_setup_msi_msg,
-	.irq_mask = dw_pci_bottom_mask,
-	.irq_unmask = dw_pci_bottom_unmask,
+	.name			= "DWPCI-MSI",
+	.irq_compose_msi_msg	= dw_pci_setup_msi_msg,
+	.irq_mask		= dw_pci_bottom_mask,
+	.irq_unmask		= dw_pci_bottom_unmask,
+#ifdef CONFIG_SMP
+	.irq_ack		= dw_irq_noop,
+	.irq_pre_redirect	= dw_pci_bottom_ack,
+	.irq_set_affinity	= irq_chip_redirect_set_affinity,
+#else
+	.irq_ack		= dw_pci_bottom_ack,
+#endif
 };
 
-static int dw_pcie_irq_domain_alloc(struct irq_domain *domain,
-				    unsigned int virq, unsigned int nr_irqs,
-				    void *args)
+static int dw_pcie_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
+				    unsigned int nr_irqs, void *args)
 {
 	struct dw_pcie_rp *pp = domain->host_data;
-	unsigned long flags;
-	u32 i;
 	int bit;
 
-	raw_spin_lock_irqsave(&pp->lock, flags);
-
-	bit = bitmap_find_free_region(pp->msi_irq_in_use, pp->num_vectors,
-				      order_base_2(nr_irqs));
-
-	raw_spin_unlock_irqrestore(&pp->lock, flags);
+	scoped_guard (raw_spinlock_irq, &pp->lock) {
+		bit = bitmap_find_free_region(pp->msi_irq_in_use, pp->num_vectors,
+					      order_base_2(nr_irqs));
+	}
 
 	if (bit < 0)
 		return -ENOSPC;
 
-	for (i = 0; i < nr_irqs; i++)
-		irq_domain_set_info(domain, virq + i, bit + i,
-				    pp->msi_irq_chip,
-				    pp, handle_edge_irq,
-				    NULL, NULL);
-
+	for (unsigned int i = 0; i < nr_irqs; i++) {
+		irq_domain_set_info(domain, virq + i, bit + i, pp->msi_irq_chip,
+				    pp, handle_edge_irq, NULL, NULL);
+	}
 	return 0;
 }
 
-static void dw_pcie_irq_domain_free(struct irq_domain *domain,
-				    unsigned int virq, unsigned int nr_irqs)
+static void dw_pcie_irq_domain_free(struct irq_domain *domain, unsigned int virq,
+				    unsigned int nr_irqs)
 {
 	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
 	struct dw_pcie_rp *pp = domain->host_data;
-	unsigned long flags;
 
-	raw_spin_lock_irqsave(&pp->lock, flags);
-
-	bitmap_release_region(pp->msi_irq_in_use, d->hwirq,
-			      order_base_2(nr_irqs));
-
-	raw_spin_unlock_irqrestore(&pp->lock, flags);
+	guard(raw_spinlock_irq)(&pp->lock);
+	bitmap_release_region(pp->msi_irq_in_use, d->hwirq, order_base_2(nr_irqs));
 }
 
 static const struct irq_domain_ops dw_pcie_msi_domain_ops = {
@@ -241,8 +231,7 @@ void dw_pcie_free_msi(struct dw_pcie_rp *pp)
 
 	for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) {
 		if (pp->msi_irq[ctrl] > 0)
-			irq_set_chained_handler_and_data(pp->msi_irq[ctrl],
-							 NULL, NULL);
+			irq_set_chained_handler_and_data(pp->msi_irq[ctrl], NULL, NULL);
 	}
 
 	irq_domain_remove(pp->irq_domain);
diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
index 31685951a080..403f6cfe8191 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -821,7 +821,7 @@ static inline enum dw_pcie_ltssm dw_pcie_get_ltssm(struct dw_pcie *pci)
 #ifdef CONFIG_PCIE_DW_HOST
 int dw_pcie_suspend_noirq(struct dw_pcie *pci);
 int dw_pcie_resume_noirq(struct dw_pcie *pci);
-irqreturn_t dw_handle_msi_irq(struct dw_pcie_rp *pp);
+void dw_handle_msi_irq(struct dw_pcie_rp *pp);
 void dw_pcie_msi_init(struct dw_pcie_rp *pp);
 int dw_pcie_msi_host_init(struct dw_pcie_rp *pp);
 void dw_pcie_free_msi(struct dw_pcie_rp *pp);
@@ -842,10 +842,7 @@ static inline int dw_pcie_resume_noirq(struct dw_pcie *pci)
 	return 0;
 }
 
-static inline irqreturn_t dw_handle_msi_irq(struct dw_pcie_rp *pp)
-{
-	return IRQ_NONE;
-}
+static inline void dw_handle_msi_irq(struct dw_pcie_rp *pp) { }
 
 static inline void dw_pcie_msi_init(struct dw_pcie_rp *pp)
 { }
diff --git a/drivers/pci/pci-acpi.c b/drivers/pci/pci-acpi.c
index 9369377725fa..79980203a1fb 100644
--- a/drivers/pci/pci-acpi.c
+++ b/drivers/pci/pci-acpi.c
@@ -847,12 +847,7 @@ bool shpchp_is_native(struct pci_dev *bridge)
  */
 static void pci_acpi_wake_bus(struct acpi_device_wakeup_context *context)
 {
-	struct acpi_device *adev;
-	struct acpi_pci_root *root;
-
-	adev = container_of(context, struct acpi_device, wakeup.context);
-	root = acpi_driver_data(adev);
-	pci_pme_wakeup_bus(root->bus);
+	pci_pme_wakeup_bus(to_pci_host_bridge(context->dev)->bus);
 }
 
 /**
@@ -885,12 +880,14 @@ static void pci_acpi_wake_dev(struct acpi_device_wakeup_context *context)
 }
 
 /**
- * pci_acpi_add_bus_pm_notifier - Register PM notifier for root PCI bus.
+ * pci_acpi_add_root_pm_notifier - Register PM notifier for root PCI bus.
  * @dev: PCI root bridge ACPI device.
+ * @root: PCI root corresponding to @dev.
  */
-acpi_status pci_acpi_add_bus_pm_notifier(struct acpi_device *dev)
+acpi_status pci_acpi_add_root_pm_notifier(struct acpi_device *dev,
+					  struct acpi_pci_root *root)
 {
-	return acpi_add_pm_notifier(dev, NULL, pci_acpi_wake_bus);
+	return acpi_add_pm_notifier(dev, root->bus->bridge, pci_acpi_wake_bus);
 }
 
 /**
diff --git a/drivers/pci/vgaarb.c b/drivers/pci/vgaarb.c
index 436fa7f4c387..805be9ea4a34 100644
--- a/drivers/pci/vgaarb.c
+++ b/drivers/pci/vgaarb.c
@@ -26,7 +26,7 @@
 #include <linux/poll.h>
 #include <linux/miscdevice.h>
 #include <linux/slab.h>
-#include <linux/screen_info.h>
+#include <linux/sysfb.h>
 #include <linux/vt.h>
 #include <linux/console.h>
 #include <linux/acpi.h>
@@ -557,7 +557,7 @@ EXPORT_SYMBOL(vga_put);
 static bool vga_is_firmware_default(struct pci_dev *pdev)
 {
 #if defined CONFIG_X86
-	return pdev == screen_info_pci_dev(&screen_info);
+	return pdev == screen_info_pci_dev(&sysfb_primary_display.screen);
 #else
 	return false;
 #endif
diff --git a/drivers/platform/surface/aggregator/core.c b/drivers/platform/surface/aggregator/core.c
index c7e05f7bc199..82e531023911 100644
--- a/drivers/platform/surface/aggregator/core.c
+++ b/drivers/platform/surface/aggregator/core.c
@@ -380,9 +380,9 @@ static int ssam_serdev_setup(struct acpi_device *ssh, struct serdev_device *serd
 
 /* -- Power management. ----------------------------------------------------- */
 
-static void ssam_serial_hub_shutdown(struct device *dev)
+static void ssam_serial_hub_shutdown(struct serdev_device *serdev)
 {
-	struct ssam_controller *c = dev_get_drvdata(dev);
+	struct ssam_controller *c = dev_get_drvdata(&serdev->dev);
 	int status;
 
 	/*
@@ -834,12 +834,12 @@ MODULE_DEVICE_TABLE(of, ssam_serial_hub_of_match);
 static struct serdev_device_driver ssam_serial_hub = {
 	.probe = ssam_serial_hub_probe,
 	.remove = ssam_serial_hub_remove,
+	.shutdown = ssam_serial_hub_shutdown,
 	.driver = {
 		.name = "surface_serial_hub",
 		.acpi_match_table = ACPI_PTR(ssam_serial_hub_acpi_match),
 		.of_match_table = of_match_ptr(ssam_serial_hub_of_match),
 		.pm = &ssam_serial_hub_pm_ops,
-		.shutdown = ssam_serial_hub_shutdown,
 		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
diff --git a/drivers/platform/x86/dell/alienware-wmi-wmax.c b/drivers/platform/x86/dell/alienware-wmi-wmax.c
index 1418bd326edf..e69b50162bb1 100644
--- a/drivers/platform/x86/dell/alienware-wmi-wmax.c
+++ b/drivers/platform/x86/dell/alienware-wmi-wmax.c
@@ -89,6 +89,30 @@ static struct awcc_quirks generic_quirks = {
 static struct awcc_quirks empty_quirks;
 
 static const struct dmi_system_id awcc_dmi_table[] __initconst = {
+	{
+		.ident = "Alienware 16 Area-51",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Alienware"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Alienware 16 Area-51"),
+		},
+		.driver_data = &g_series_quirks,
+	},
+	{
+		.ident = "Alienware 16X Aurora",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Alienware"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Alienware 16X Aurora"),
+		},
+		.driver_data = &g_series_quirks,
+	},
+	{
+		.ident = "Alienware 18 Area-51",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Alienware"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Alienware 18 Area-51"),
+		},
+		.driver_data = &g_series_quirks,
+	},
 	{
 		.ident = "Alienware 16 Aurora",
 		.matches = {
@@ -161,6 +185,14 @@ static const struct dmi_system_id awcc_dmi_table[] __initconst = {
 		},
 		.driver_data = &generic_quirks,
 	},
+	{
+		.ident = "Alienware x16",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Alienware"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Alienware x16"),
+		},
+		.driver_data = &g_series_quirks,
+	},
 	{
 		.ident = "Alienware x17",
 		.matches = {
diff --git a/drivers/platform/x86/intel/hid.c b/drivers/platform/x86/intel/hid.c
index 560cc063198e..0f8684f4464b 100644
--- a/drivers/platform/x86/intel/hid.c
+++ b/drivers/platform/x86/intel/hid.c
@@ -779,43 +779,4 @@ static struct platform_driver intel_hid_pl_driver = {
 	.remove = intel_hid_remove,
 };
 
-/*
- * Unfortunately, some laptops provide a _HID="INT33D5" device with
- * _CID="PNP0C02".  This causes the pnpacpi scan driver to claim the
- * ACPI node, so no platform device will be created.  The pnpacpi
- * driver rejects this device in subsequent processing, so no physical
- * node is created at all.
- *
- * As a workaround until the ACPI core figures out how to handle
- * this corner case, manually ask the ACPI platform device code to
- * claim the ACPI node.
- */
-static acpi_status __init
-check_acpi_dev(acpi_handle handle, u32 lvl, void *context, void **rv)
-{
-	const struct acpi_device_id *ids = context;
-	struct acpi_device *dev = acpi_fetch_acpi_dev(handle);
-
-	if (dev && acpi_match_device_ids(dev, ids) == 0)
-		if (!IS_ERR_OR_NULL(acpi_create_platform_device(dev, NULL)))
-			dev_info(&dev->dev,
-				 "intel-hid: created platform device\n");
-
-	return AE_OK;
-}
-
-static int __init intel_hid_init(void)
-{
-	acpi_walk_namespace(ACPI_TYPE_DEVICE, ACPI_ROOT_OBJECT,
-			    ACPI_UINT32_MAX, check_acpi_dev, NULL,
-			    (void *)intel_hid_ids, NULL);
-
-	return platform_driver_register(&intel_hid_pl_driver);
-}
-module_init(intel_hid_init);
-
-static void __exit intel_hid_exit(void)
-{
-	platform_driver_unregister(&intel_hid_pl_driver);
-}
-module_exit(intel_hid_exit);
+module_platform_driver(intel_hid_pl_driver);
diff --git a/drivers/platform/x86/intel/vbtn.c b/drivers/platform/x86/intel/vbtn.c
index 232cd12e3c9f..9ca87e707582 100644
--- a/drivers/platform/x86/intel/vbtn.c
+++ b/drivers/platform/x86/intel/vbtn.c
@@ -390,32 +390,4 @@ static struct platform_driver intel_vbtn_pl_driver = {
 	.remove = intel_vbtn_remove,
 };
 
-static acpi_status __init
-check_acpi_dev(acpi_handle handle, u32 lvl, void *context, void **rv)
-{
-	const struct acpi_device_id *ids = context;
-	struct acpi_device *dev = acpi_fetch_acpi_dev(handle);
-
-	if (dev && acpi_match_device_ids(dev, ids) == 0)
-		if (!IS_ERR_OR_NULL(acpi_create_platform_device(dev, NULL)))
-			dev_info(&dev->dev,
-				 "intel-vbtn: created platform device\n");
-
-	return AE_OK;
-}
-
-static int __init intel_vbtn_init(void)
-{
-	acpi_walk_namespace(ACPI_TYPE_DEVICE, ACPI_ROOT_OBJECT,
-			    ACPI_UINT32_MAX, check_acpi_dev, NULL,
-			    (void *)intel_vbtn_ids, NULL);
-
-	return platform_driver_register(&intel_vbtn_pl_driver);
-}
-module_init(intel_vbtn_init);
-
-static void __exit intel_vbtn_exit(void)
-{
-	platform_driver_unregister(&intel_vbtn_pl_driver);
-}
-module_exit(intel_vbtn_exit);
+module_platform_driver(intel_vbtn_pl_driver);
diff --git a/drivers/platform/x86/intel/wmi/thunderbolt.c b/drivers/platform/x86/intel/wmi/thunderbolt.c
index 08df560a2c7a..15e5763a20dd 100644
--- a/drivers/platform/x86/intel/wmi/thunderbolt.c
+++ b/drivers/platform/x86/intel/wmi/thunderbolt.c
@@ -10,6 +10,7 @@
 #include <linux/acpi.h>
 #include <linux/device.h>
 #include <linux/fs.h>
+#include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/string.h>
diff --git a/drivers/platform/x86/uniwill/uniwill-acpi.c b/drivers/platform/x86/uniwill/uniwill-acpi.c
index bd7e63dd5181..0f935532f250 100644
--- a/drivers/platform/x86/uniwill/uniwill-acpi.c
+++ b/drivers/platform/x86/uniwill/uniwill-acpi.c
@@ -1844,6 +1844,13 @@ static const struct dmi_system_id uniwill_dmi_table[] __initconst = {
 			DMI_EXACT_MATCH(DMI_BOARD_NAME, "X6AR5xxY_mLED"),
 		},
 	},
+	{
+		.ident = "TUXEDO Book BA15 Gen10 AMD",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "TUXEDO"),
+			DMI_EXACT_MATCH(DMI_BOARD_NAME, "PF5PU1G"),
+		},
+	},
 	{
 		.ident = "TUXEDO Pulse 14 Gen1 AMD",
 		.matches = {
diff --git a/drivers/pmdomain/qcom/rpmhpd.c b/drivers/pmdomain/qcom/rpmhpd.c
index a8b37037c6fe..19849703be4a 100644
--- a/drivers/pmdomain/qcom/rpmhpd.c
+++ b/drivers/pmdomain/qcom/rpmhpd.c
@@ -246,6 +246,8 @@ static struct rpmhpd *sa8540p_rpmhpds[] = {
 	[SC8280XP_MMCX_AO] = &mmcx_ao,
 	[SC8280XP_MX] = &mx,
 	[SC8280XP_MX_AO] = &mx_ao,
+	[SC8280XP_MXC] = &mxc,
+	[SC8280XP_MXC_AO] = &mxc_ao,
 	[SC8280XP_NSP] = &nsp,
 };
 
@@ -700,6 +702,8 @@ static struct rpmhpd *sc8280xp_rpmhpds[] = {
 	[SC8280XP_MMCX_AO] = &mmcx_ao,
 	[SC8280XP_MX] = &mx,
 	[SC8280XP_MX_AO] = &mx_ao,
+	[SC8280XP_MXC] = &mxc,
+	[SC8280XP_MXC_AO] = &mxc_ao,
 	[SC8280XP_NSP] = &nsp,
 	[SC8280XP_QPHY] = &qphy,
 };
diff --git a/drivers/pnp/support.c b/drivers/pnp/support.c
index a6073db10ec6..f6c866851769 100644
--- a/drivers/pnp/support.c
+++ b/drivers/pnp/support.c
@@ -9,6 +9,7 @@
 
 #include <linux/module.h>
 #include <linux/ctype.h>
+#include <linux/hex.h>
 #include <linux/pnp.h>
 #include "base.h"
 
diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c
index b9d87e56cbbc..3ff6da3bf4e6 100644
--- a/drivers/powercap/intel_rapl_common.c
+++ b/drivers/powercap/intel_rapl_common.c
@@ -2032,7 +2032,7 @@ static int rapl_pmu_update(struct rapl_package *rp)
 	return ret;
 }
 
-int rapl_package_add_pmu(struct rapl_package *rp)
+int rapl_package_add_pmu_locked(struct rapl_package *rp)
 {
 	struct rapl_package_pmu_data *data = &rp->pmu_data;
 	int idx;
@@ -2040,8 +2040,6 @@ int rapl_package_add_pmu(struct rapl_package *rp)
 	if (rp->has_pmu)
 		return -EEXIST;
 
-	guard(cpus_read_lock)();
-
 	for (idx = 0; idx < rp->nr_domains; idx++) {
 		struct rapl_domain *rd = &rp->domains[idx];
 		int domain = rd->id;
@@ -2091,17 +2089,23 @@ int rapl_package_add_pmu(struct rapl_package *rp)
 
 	return rapl_pmu_update(rp);
 }
+EXPORT_SYMBOL_GPL(rapl_package_add_pmu_locked);
+
+int rapl_package_add_pmu(struct rapl_package *rp)
+{
+	guard(cpus_read_lock)();
+
+	return rapl_package_add_pmu_locked(rp);
+}
 EXPORT_SYMBOL_GPL(rapl_package_add_pmu);
 
-void rapl_package_remove_pmu(struct rapl_package *rp)
+void rapl_package_remove_pmu_locked(struct rapl_package *rp)
 {
 	struct rapl_package *pos;
 
 	if (!rp->has_pmu)
 		return;
 
-	guard(cpus_read_lock)();
-
 	list_for_each_entry(pos, &rapl_packages, plist) {
 		/* PMU is still needed */
 		if (pos->has_pmu && pos != rp)
@@ -2111,6 +2115,14 @@ void rapl_package_remove_pmu(struct rapl_package *rp)
 	perf_pmu_unregister(&rapl_pmu.pmu);
 	memset(&rapl_pmu, 0, sizeof(struct rapl_pmu));
 }
+EXPORT_SYMBOL_GPL(rapl_package_remove_pmu_locked);
+
+void rapl_package_remove_pmu(struct rapl_package *rp)
+{
+	guard(cpus_read_lock)();
+
+	rapl_package_remove_pmu_locked(rp);
+}
 EXPORT_SYMBOL_GPL(rapl_package_remove_pmu);
 #endif
 
diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c
index 0ce1096b6314..9a7e150b3536 100644
--- a/drivers/powercap/intel_rapl_msr.c
+++ b/drivers/powercap/intel_rapl_msr.c
@@ -82,7 +82,7 @@ static int rapl_cpu_online(unsigned int cpu)
 		if (IS_ERR(rp))
 			return PTR_ERR(rp);
 		if (rapl_msr_pmu)
-			rapl_package_add_pmu(rp);
+			rapl_package_add_pmu_locked(rp);
 	}
 	cpumask_set_cpu(cpu, &rp->cpumask);
 	return 0;
@@ -101,7 +101,7 @@ static int rapl_cpu_down_prep(unsigned int cpu)
 	lead_cpu = cpumask_first(&rp->cpumask);
 	if (lead_cpu >= nr_cpu_ids) {
 		if (rapl_msr_pmu)
-			rapl_package_remove_pmu(rp);
+			rapl_package_remove_pmu_locked(rp);
 		rapl_remove_package_cpuslocked(rp);
 	} else if (rp->lead_cpu == cpu) {
 		rp->lead_cpu = lead_cpu;
diff --git a/drivers/powercap/powercap_sys.c b/drivers/powercap/powercap_sys.c
index 4112a0097338..1ff369880beb 100644
--- a/drivers/powercap/powercap_sys.c
+++ b/drivers/powercap/powercap_sys.c
@@ -68,7 +68,7 @@ static ssize_t show_constraint_##_attr(struct device *dev, \
 	int id; \
 	struct powercap_zone_constraint *pconst;\
 	\
-	if (!sscanf(dev_attr->attr.name, "constraint_%d_", &id)) \
+	if (sscanf(dev_attr->attr.name, "constraint_%d_", &id) != 1) \
 		return -EINVAL; \
 	if (id >= power_zone->const_id_cnt)	\
 		return -EINVAL; \
@@ -93,7 +93,7 @@ static ssize_t store_constraint_##_attr(struct device *dev,\
 	int id; \
 	struct powercap_zone_constraint *pconst;\
 	\
-	if (!sscanf(dev_attr->attr.name, "constraint_%d_", &id)) \
+	if (sscanf(dev_attr->attr.name, "constraint_%d_", &id) != 1) \
 		return -EINVAL; \
 	if (id >= power_zone->const_id_cnt)	\
 		return -EINVAL; \
@@ -162,7 +162,7 @@ static ssize_t show_constraint_name(struct device *dev,
 	ssize_t len = -ENODATA;
 	struct powercap_zone_constraint *pconst;
 
-	if (!sscanf(dev_attr->attr.name, "constraint_%d_", &id))
+	if (sscanf(dev_attr->attr.name, "constraint_%d_", &id) != 1)
 		return -EINVAL;
 	if (id >= power_zone->const_id_cnt)
 		return -EINVAL;
@@ -625,17 +625,23 @@ struct powercap_control_type *powercap_register_control_type(
 	INIT_LIST_HEAD(&control_type->node);
 	control_type->dev.class = &powercap_class;
 	dev_set_name(&control_type->dev, "%s", name);
-	result = device_register(&control_type->dev);
-	if (result) {
-		put_device(&control_type->dev);
-		return ERR_PTR(result);
-	}
 	idr_init(&control_type->idr);
 
 	mutex_lock(&powercap_cntrl_list_lock);
 	list_add_tail(&control_type->node, &powercap_cntrl_list);
 	mutex_unlock(&powercap_cntrl_list_lock);
 
+	result = device_register(&control_type->dev);
+	if (result) {
+		mutex_lock(&powercap_cntrl_list_lock);
+		list_del(&control_type->node);
+		mutex_unlock(&powercap_cntrl_list_lock);
+
+		idr_destroy(&control_type->idr);
+		put_device(&control_type->dev);
+		return ERR_PTR(result);
+	}
+
 	return control_type;
 }
 EXPORT_SYMBOL_GPL(powercap_register_control_type);
diff --git a/drivers/ptp/ptp_pch.c b/drivers/ptp/ptp_pch.c
index b8a9a54a176c..f854da2fd812 100644
--- a/drivers/ptp/ptp_pch.c
+++ b/drivers/ptp/ptp_pch.c
@@ -10,6 +10,7 @@
 
 #include <linux/device.h>
 #include <linux/err.h>
+#include <linux/hex.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
diff --git a/drivers/pwm/pwm_th1520.rs b/drivers/pwm/pwm_th1520.rs
index e3b7e77356fc..71b80761ec31 100644
--- a/drivers/pwm/pwm_th1520.rs
+++ b/drivers/pwm/pwm_th1520.rs
@@ -372,7 +372,7 @@ fn probe(
             }),
         )?;
 
-        pwm::Registration::register(dev, chip)?;
+        chip.register()?;
 
         Ok(Th1520PwmPlatformDriver)
     }
diff --git a/drivers/rapidio/rio-scan.c b/drivers/rapidio/rio-scan.c
index c12941f71e2c..03615eb2d3d7 100644
--- a/drivers/rapidio/rio-scan.c
+++ b/drivers/rapidio/rio-scan.c
@@ -854,7 +854,7 @@ static struct rio_net *rio_scan_alloc_net(struct rio_mport *mport,
 
 		if (idtab == NULL) {
 			pr_err("RIO: failed to allocate destID table\n");
-			rio_free_net(net);
+			kfree(net);
 			net = NULL;
 		} else {
 			net->enum_data = idtab;
diff --git a/drivers/regulator/Kconfig b/drivers/regulator/Kconfig
index d2335276cce5..98966ed36f8e 100644
--- a/drivers/regulator/Kconfig
+++ b/drivers/regulator/Kconfig
@@ -659,6 +659,15 @@ config REGULATOR_MAX77650
 	  Semiconductor. This device has a SIMO with three independent
 	  power rails and an LDO.
 
+config REGULATOR_MAX77675
+	tristate "Maxim MAX77675 regulator driver"
+	depends on I2C && OF
+	select REGMAP_I2C
+	help
+	  This driver controls the Maxim MAX77675 power regulator via I2C.
+	  It supports four programmable buck-boost outputs.
+	  Say Y here to enable the regulator driver
+
 config REGULATOR_MAX77857
 	tristate "ADI MAX77857/MAX77831 regulator support"
 	depends on I2C
diff --git a/drivers/regulator/Makefile b/drivers/regulator/Makefile
index 1beba1493241..1836fedfeb5d 100644
--- a/drivers/regulator/Makefile
+++ b/drivers/regulator/Makefile
@@ -79,6 +79,7 @@ obj-$(CONFIG_REGULATOR_MAX77503) += max77503-regulator.o
 obj-$(CONFIG_REGULATOR_MAX77541) += max77541-regulator.o
 obj-$(CONFIG_REGULATOR_MAX77620) += max77620-regulator.o
 obj-$(CONFIG_REGULATOR_MAX77650) += max77650-regulator.o
+obj-$(CONFIG_REGULATOR_MAX77675) += max77675-regulator.o
 obj-$(CONFIG_REGULATOR_MAX8649)	+= max8649.o
 obj-$(CONFIG_REGULATOR_MAX8660) += max8660.o
 obj-$(CONFIG_REGULATOR_MAX8893) += max8893.o
diff --git a/drivers/regulator/max77675-regulator.c b/drivers/regulator/max77675-regulator.c
new file mode 100644
index 000000000000..af3eb7174875
--- /dev/null
+++ b/drivers/regulator/max77675-regulator.c
@@ -0,0 +1,1056 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2025 Analog Devices, Inc.
+ * ADI regulator driver for MAX77675.
+ */
+
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/cleanup.h>
+#include <linux/slab.h>
+#include <linux/of.h>
+#include <linux/i2c.h>
+#include <linux/regmap.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/consumer.h>
+#include <linux/regulator/of_regulator.h>
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+
+/* Register Addresses */
+#define MAX77675_REG_CNFG_GLBL_A     0x00
+#define MAX77675_REG_CNFG_GLBL_B     0x01
+#define MAX77675_REG_INT_GLBL        0x02
+#define MAX77675_REG_INTM_GLBL       0x03
+#define MAX77675_REG_STAT_GLBL       0x04
+#define MAX77675_REG_ERCF_GLBL       0x05
+#define MAX77675_REG_CID             0x06
+#define MAX77675_REG_CNFG_SBB_TOP_A  0x07
+#define MAX77675_REG_CNFG_SBB0_A     0x08
+#define MAX77675_REG_CNFG_SBB0_B     0x09
+#define MAX77675_REG_CNFG_SBB1_A     0x0A
+#define MAX77675_REG_CNFG_SBB1_B     0x0B
+#define MAX77675_REG_CNFG_SBB2_A     0x0C
+#define MAX77675_REG_CNFG_SBB2_B     0x0D
+#define MAX77675_REG_CNFG_SBB3_A     0x0E
+#define MAX77675_REG_CNFG_SBB3_B     0x0F
+#define MAX77675_REG_CNFG_SBB_TOP_B  0x10
+
+/* CNFG_GLBL_A (0x00) bit masks and shifts */
+#define MAX77675_MRT_MASK           GENMASK(7, 6)    /* Manual Reset Time (bits 7:6) */
+#define MAX77675_MRT_SHIFT          6
+#define MAX77675_PU_DIS_BIT         BIT(5)           /* Pullup Disable (bit 5) */
+#define MAX77675_PU_DIS_SHIFT       5
+#define MAX77675_BIAS_LPM_BIT       BIT(4)           /* Bias Low Power Mode (bit 4) */
+#define MAX77675_BIAS_LPM_SHIFT     4
+#define MAX77675_SIMO_CH_DIS_BIT    BIT(3)           /* SIMO Internal Channel Disable (bit 3) */
+#define MAX77675_SIMO_CH_DIS_SHIFT  3
+#define MAX77675_EN_MODE_MASK       GENMASK(2, 1)    /* nEN Mode (bits 2:1) */
+#define MAX77675_EN_MODE_SHIFT      1
+#define MAX77675_DBEN_EN_BIT        BIT(0)           /* Debounce Enable (bit 0) */
+#define MAX77675_DBEN_EN_SHIFT      0
+
+/* CNFG_GLBL_B (0x01) */
+#define MAX77675_SFT_CTRL_MASK      GENMASK(2, 0)    /* Soft Start Control */
+#define MAX77675_SFT_CTRL_SHIFT     0
+
+/* INT_GLBL (0x02) bit bits and shifts */
+#define MAX77675_INT_SBB3_F_BIT     BIT(7)
+#define MAX77675_INT_SBB3_F_SHIFT   7
+#define MAX77675_INT_SBB2_F_BIT     BIT(6)
+#define MAX77675_INT_SBB2_F_SHIFT   6
+#define MAX77675_INT_SBB1_F_BIT     BIT(5)
+#define MAX77675_INT_SBB1_F_SHIFT   5
+#define MAX77675_INT_SBB0_F_BIT     BIT(4)
+#define MAX77675_INT_SBB0_F_SHIFT   4
+#define MAX77675_INT_TJAL2_R_BIT    BIT(3)
+#define MAX77675_INT_TJAL2_R_SHIFT  3
+#define MAX77675_INT_TJAL1_R_BIT    BIT(2)
+#define MAX77675_INT_TJAL1_R_SHIFT  2
+#define MAX77675_INT_EN_R_BIT       BIT(1)
+#define MAX77675_INT_EN_R_SHIFT     1
+#define MAX77675_INT_EN_F_BIT       BIT(0)
+#define MAX77675_INT_EN_F_SHIFT     0
+
+/* INTM_GLBL (0x03) bits and shifts */
+#define MAX77675_INTM_SBB3_F_BIT    BIT(7)
+#define MAX77675_INTM_SBB3_F_SHIFT  7
+#define MAX77675_INTM_SBB2_F_BIT    BIT(6)
+#define MAX77675_INTM_SBB2_F_SHIFT  6
+#define MAX77675_INTM_SBB1_F_BIT    BIT(5)
+#define MAX77675_INTM_SBB1_F_SHIFT  5
+#define MAX77675_INTM_SBB0_F_BIT    BIT(4)
+#define MAX77675_INTM_SBB0_F_SHIFT  4
+#define MAX77675_INTM_TJAL2_R_BIT   BIT(3)
+#define MAX77675_INTM_TJAL2_R_SHIFT 3
+#define MAX77675_INTM_TJAL1_R_BIT   BIT(2)
+#define MAX77675_INTM_TJAL1_R_SHIFT 2
+#define MAX77675_INTM_EN_R_BIT      BIT(1)
+#define MAX77675_INTM_EN_R_SHIFT    1
+#define MAX77675_INTM_EN_F_BIT      BIT(0)
+#define MAX77675_INTM_EN_F_SHIFT    0
+
+/* STAT_GLBL (0x04) bits and shifts */
+#define MAX77675_STAT_SBB3_S_BIT    BIT(7)
+#define MAX77675_STAT_SBB3_S_SHIFT  7
+#define MAX77675_STAT_SBB2_S_BIT    BIT(6)
+#define MAX77675_STAT_SBB2_S_SHIFT  6
+#define MAX77675_STAT_SBB1_S_BIT    BIT(5)
+#define MAX77675_STAT_SBB1_S_SHIFT  5
+#define MAX77675_STAT_SBB0_S_BIT    BIT(4)
+#define MAX77675_STAT_SBB0_S_SHIFT  4
+#define MAX77675_STAT_TJAL2_S_BIT   BIT(2)
+#define MAX77675_STAT_TJAL2_S_SHIFT 2
+#define MAX77675_STAT_TJAL1_S_BIT   BIT(1)
+#define MAX77675_STAT_TJAL1_S_SHIFT 1
+#define MAX77675_STAT_STAT_EN_BIT   BIT(0)
+#define MAX77675_STAT_STAT_EN_SHIFT 0
+
+#define MAX77675_STAT_STAT_EN_BIT   BIT(0)
+#define MAX77675_STAT_STAT_EN_SHIFT 0
+
+/* ERCFLAG (0x05) bits and shifts */
+#define MAX77675_SFT_CRST_F_BIT     BIT(5)  /* Software Cold Reset Flag */
+#define MAX77675_SFT_CRST_F_SHIFT   5
+#define MAX77675_SFT_OFF_F_BIT      BIT(4)  /* Software Off Flag */
+#define MAX77675_SFT_OFF_F_SHIFT    4
+#define MAX77675_MRST_BIT           BIT(3)  /* Manual Reset Timer Flag */
+#define MAX77675_MRST_SHIFT         3
+#define MAX77675_UVLO_BIT           BIT(2)  /* Undervoltage Lockout Flag */
+#define MAX77675_UVLO_SHIFT         2
+#define MAX77675_OVLO_BIT           BIT(1)  /* Overvoltage Lockout Flag */
+#define MAX77675_OVLO_SHIFT         1
+#define MAX77675_TOVLD_BIT          BIT(0)  /* Thermal Overload Flag */
+#define MAX77675_TOVLD_SHIFT        0
+
+/* CID (0x06) bits and shifts */
+#define MAX77675_CID_MASK           GENMASK(4, 0)  /* Chip Identification Code mask */
+#define MAX77675_CID_SHIFT          0              /* Starts at bit 0 */
+
+/* CNFG_SBB_TOP_A (0x07) bits and shifts */
+#define MAX77675_STEP_SZ_SBB3_BIT   BIT(5)
+#define MAX77675_STEP_SZ_SBB3_SHIFT 5
+#define MAX77675_STEP_SZ_SBB2_BIT   BIT(4)
+#define MAX77675_STEP_SZ_SBB2_SHIFT 4
+#define MAX77675_STEP_SZ_SBB1_BIT   BIT(3)
+#define MAX77675_STEP_SZ_SBB1_SHIFT 3
+#define MAX77675_STEP_SZ_SBB0_BIT   BIT(2)
+#define MAX77675_STEP_SZ_SBB0_SHIFT 2
+#define MAX77675_DRV_SBB_MASK       GENMASK(1, 0)
+#define MAX77675_DRV_SBB_SHIFT      0
+
+/* CNFG_SBB0_A (0x08) bits and shifts */
+#define MAX77675_TV_SBB0_MASK       GENMASK(7, 0)
+#define MAX77675_TV_SBB0_SHIFT      0
+
+/* CNFG_SBB0_B (0x09) bits and shifts */
+#define MAX77675_ADE_SBB0_BIT       BIT(3)
+#define MAX77675_ADE_SBB0_SHIFT     3
+#define MAX77675_EN_SBB0_MASK       GENMASK(2, 0)
+#define MAX77675_EN_SBB0_SHIFT      0
+
+/* CNFG_SBB1_A (0x0A) bits and shifts */
+#define MAX77675_TV_SBB1_MASK       GENMASK(7, 0)
+#define MAX77675_TV_SBB1_SHIFT      0
+
+/* CNFG_SBB1_B (0x0B) bits and shifts */
+#define MAX77675_ADE_SBB1_BIT       BIT(3)
+#define MAX77675_ADE_SBB1_SHIFT     3
+#define MAX77675_EN_SBB1_MASK       GENMASK(2, 0)
+#define MAX77675_EN_SBB1_SHIFT      0
+
+/* CNFG_SBB2_A (0x0C) bits and shifts */
+#define MAX77675_TV_SBB2_MASK       GENMASK(7, 0)
+#define MAX77675_TV_SBB2_SHIFT      0
+
+/* CNFG_SBB2_B (0x0D) bits and shifts */
+#define MAX77675_ADE_SBB2_BIT       BIT(3)
+#define MAX77675_ADE_SBB2_SHIFT     3
+#define MAX77675_EN_SBB2_MASK       GENMASK(2, 0)
+#define MAX77675_EN_SBB2_SHIFT      0
+
+/* CNFG_SBB3_A (0x0E) bits and shifts */
+#define MAX77675_TV_SBB3_MASK       GENMASK(7, 0)
+#define MAX77675_TV_SBB3_SHIFT      0
+
+/* CNFG_SBB3_B (0x0F) bits and shifts */
+#define MAX77675_ADE_SBB3_BIT       BIT(3)
+#define MAX77675_ADE_SBB3_SHIFT     3
+#define MAX77675_EN_SBB3_MASK       GENMASK(2, 0)
+#define MAX77675_EN_SBB3_SHIFT      0
+
+#define MAX77675_EN_SBB_MASK        GENMASK(2, 0)
+
+/* CNFG_SBB_TOP_B (0x10) bits and shifts */
+#define MAX77675_DVS_SLEW_BIT       BIT(5)
+#define MAX77675_DVS_SLEW_SHIFT     5
+#define MAX77675_LAT_MODE_BIT       BIT(4)
+#define MAX77675_LAT_MODE_SHIFT     4
+#define MAX77675_SR_SBB3_BIT        BIT(3)
+#define MAX77675_SR_SBB3_SHIFT      3
+#define MAX77675_SR_SBB2_BIT        BIT(2)
+#define MAX77675_SR_SBB2_SHIFT      2
+#define MAX77675_SR_SBB1_BIT        BIT(1)
+#define MAX77675_SR_SBB1_SHIFT      1
+#define MAX77675_SR_SBB0_BIT        BIT(0)
+#define MAX77675_SR_SBB0_SHIFT      0
+
+#define MAX77675_MAX_REGISTER       0x10
+
+/* Common minimum voltage (in microvolts) */
+#define MAX77675_MIN_UV             500000     // 500 mV
+
+/* Voltage step configuration for 25mV mode */
+#define MAX77675_STEP_25MV          25000      // Step size: 25 mV
+#define MAX77675_MAX_UV_25MV        5500000    // Max voltage: 5.5 V
+#define MAX77675_NUM_LEVELS_25MV    201        // levels = (5500mV - 500mV) / 25mV + 1
+
+/* Voltage step configuration for 12.5mV mode */
+#define MAX77675_STEP_12_5MV        12500      // Step size: 12.5 mV
+#define MAX77675_MAX_UV_12_5MV      3687500    // Max voltage: 3.6875 V
+#define MAX77675_NUM_LEVELS_12_5MV  255        // levels = (3687.5mV - 500mV) / 12.5mV + 1
+
+#define MAX77675_ENABLE_OFF         0x04
+#define MAX77675_ENABLE_ON          0x06
+
+#define MAX77675_REGULATOR_AD_OFF   0x00
+#define MAX77675_REGULATOR_AD_ON    BIT(3)
+
+/* FPS source */
+#define MAX77675_FPS_SLOT_0         0x0
+#define MAX77675_FPS_SLOT_1         0x1
+#define MAX77675_FPS_SLOT_2         0x2
+#define MAX77675_FPS_SLOT_3         0x3
+#define MAX77675_FPS_DEF            0x4
+
+/* nEN Manual Reset Time Configuration (MRT) */
+#define MAX77675_MRT_4S             0x0
+#define MAX77675_MRT_8S             0x1
+#define MAX77675_MRT_12S            0x2
+#define MAX77675_MRT_16S            0x3
+
+/* nEN Mode Configuration */
+#define MAX77675_EN_PUSH_BUTTON     0x0
+#define MAX77675_EN_SLIDE_SWITCH    0x1
+#define MAX77675_EN_LOGIC           0x2
+
+/* Debounce Timer Enable (DBEN_nEN) */
+#define MAX77675_DBEN_100US         0x0
+#define MAX77675_DBEN_30000US       0x1
+
+/* Rising slew rate control for SBB0 when ramping up */
+#define MAX77675_SR_2MV_PER_US      0x0  // 2 mV/us
+#define MAX77675_SR_USE_DVS         0x1  // Use DVS slew rate setting (adi,dvs-slew-rate)
+
+/* Latency Mode */
+#define MAX77675_HIGH_LATENCY_MODE  0x0   // High latency, low quiescent current (~100us)
+#define MAX77675_LOW_LATENCY_MODE   0x1   // Low latency, high quiescent current (~10us)
+
+/* Dynamic Voltage Scaling (DVS) Slew Rate */
+#define MAX77675_DVS_SLEW_5MV_PER_US   0x0  // 5 mV/us
+#define MAX77675_DVS_SLEW_10MV_PER_US  0x1  // 10 mV/us
+
+/* SIMO Buck-Boost Drive Strength (All Channels) */
+#define MAX77675_DRV_SBB_STRENGTH_MAX  0x0  // Maximum drive strength (~0.6 ns transition time)
+#define MAX77675_DRV_SBB_STRENGTH_HIGH 0x1  // High drive strength (~1.2 ns transition time)
+#define MAX77675_DRV_SBB_STRENGTH_LOW  0x2  // Low drive strength (~1.8 ns transition time)
+#define MAX77675_DRV_SBB_STRENGTH_MIN  0x3  // Minimum drive strength (~8 ns transition time)
+
+/* Regulator ID enumeration */
+enum max77675_regulator_id {
+	MAX77675_ID_SBB0 = 0,
+	MAX77675_ID_SBB1,
+	MAX77675_ID_SBB2,
+	MAX77675_ID_SBB3,
+	MAX77675_ID_NUM_MAX,
+};
+
+struct max77675_regulator_sbb_setting {
+	u8   fps_slot;
+	bool fixed_slew_rate;
+};
+
+struct max77675_config {
+	u8   en_mode;
+	u8   voltage_change_latency;
+	u8   drv_sbb_strength;
+	u8   dvs_slew_rate;
+	u8   debounce_time;
+	u8   manual_reset_time;
+	bool en_pullup_disable;
+	bool bias_low_power_request;
+	bool simo_ldo_always_on;
+};
+
+struct max77675_regulator {
+	struct device *dev;
+	struct regmap *regmap;
+	struct max77675_config config;
+	struct max77675_regulator_sbb_setting sbb_setting[MAX77675_ID_NUM_MAX];
+};
+
+static int max77675_regulator_get_fps_src(struct max77675_regulator *maxreg, int id)
+{
+	unsigned int reg_addr;
+	unsigned int val;
+	int ret;
+
+	switch (id) {
+	case MAX77675_ID_SBB0:
+		reg_addr = MAX77675_REG_CNFG_SBB0_B;
+		break;
+	case MAX77675_ID_SBB1:
+		reg_addr = MAX77675_REG_CNFG_SBB1_B;
+		break;
+	case MAX77675_ID_SBB2:
+		reg_addr = MAX77675_REG_CNFG_SBB2_B;
+		break;
+	case MAX77675_ID_SBB3:
+		reg_addr = MAX77675_REG_CNFG_SBB3_B;
+		break;
+	default:
+		dev_err(maxreg->dev, "Invalid regulator id: %d\n", id);
+		return -EINVAL;
+	}
+
+	ret = regmap_read(maxreg->regmap, reg_addr, &val);
+	if (ret < 0) {
+		dev_err(maxreg->dev, "Failed to read FPS source (reg 0x%02x): %d\n",
+			reg_addr, ret);
+		return ret;
+	}
+
+	return FIELD_GET(MAX77675_EN_SBB_MASK, val);
+}
+
+static int max77675_regulator_set_fps_src(struct max77675_regulator *maxreg, int id, u8 fps_src)
+{
+	unsigned int reg_addr;
+
+	switch (id) {
+	case MAX77675_ID_SBB0:
+		reg_addr = MAX77675_REG_CNFG_SBB0_B;
+		break;
+	case MAX77675_ID_SBB1:
+		reg_addr = MAX77675_REG_CNFG_SBB1_B;
+		break;
+	case MAX77675_ID_SBB2:
+		reg_addr = MAX77675_REG_CNFG_SBB2_B;
+		break;
+	case MAX77675_ID_SBB3:
+		reg_addr = MAX77675_REG_CNFG_SBB3_B;
+		break;
+	default:
+		dev_err(maxreg->dev, "Invalid regulator id: %d\n", id);
+		return -EINVAL;
+	}
+
+	return regmap_update_bits(maxreg->regmap, reg_addr, MAX77675_EN_SBB_MASK, fps_src);
+}
+
+static int max77675_set_sbb_slew_rate_fixed(struct max77675_regulator *maxreg, int id, bool fixed)
+{
+	u8 mask, value;
+	u8 slew_src_ctrl_bit = fixed ? 0 : 1;
+
+	switch (id) {
+	case MAX77675_ID_SBB0:
+		mask = MAX77675_SR_SBB0_BIT;
+		value = FIELD_PREP(MAX77675_SR_SBB0_BIT, slew_src_ctrl_bit);
+		break;
+
+	case MAX77675_ID_SBB1:
+		mask = MAX77675_SR_SBB1_BIT;
+		value = FIELD_PREP(MAX77675_SR_SBB1_BIT, slew_src_ctrl_bit);
+		break;
+
+	case MAX77675_ID_SBB2:
+		mask = MAX77675_SR_SBB2_BIT;
+		value = FIELD_PREP(MAX77675_SR_SBB2_BIT, slew_src_ctrl_bit);
+		break;
+
+	case MAX77675_ID_SBB3:
+		mask = MAX77675_SR_SBB3_BIT;
+		value = FIELD_PREP(MAX77675_SR_SBB3_BIT, slew_src_ctrl_bit);
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return regmap_update_bits(maxreg->regmap, MAX77675_REG_CNFG_SBB_TOP_B, mask, value);
+}
+
+static int max77675_init_regulator(struct max77675_regulator *maxreg, int id)
+{
+	struct max77675_regulator_sbb_setting *sbb_setting = &maxreg->sbb_setting[id];
+	int ret;
+
+	if (sbb_setting->fps_slot == MAX77675_FPS_DEF) {
+		ret = max77675_regulator_get_fps_src(maxreg, id);
+		if (ret < 0)
+			return ret;
+
+		sbb_setting->fps_slot = ret;
+	} else {
+		ret = max77675_regulator_set_fps_src(maxreg, id, sbb_setting->fps_slot);
+		if (ret < 0)
+			return ret;
+	}
+
+	ret = max77675_set_sbb_slew_rate_fixed(maxreg, id, sbb_setting->fixed_slew_rate);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int max77675_of_parse_cb(struct device_node *np,
+				const struct regulator_desc *desc,
+				struct regulator_config *config)
+{
+	struct max77675_regulator *maxreg = config->driver_data;
+	struct max77675_regulator_sbb_setting *sbb_setting = &maxreg->sbb_setting[desc->id];
+	static const char * const fps_slots[] = { "slot0", "slot1", "slot2", "slot3", "default" };
+	const char *fps_str;
+	int slot;
+
+	/* Parse FPS slot from DT */
+	if (of_property_read_string(np, "adi,fps-slot", &fps_str)) {
+		/* Property not set, use default */
+		sbb_setting->fps_slot = MAX77675_FPS_DEF;
+	} else {
+		/* Match string to index */
+		slot = match_string(fps_slots, ARRAY_SIZE(fps_slots), fps_str);
+		if (slot < 0) {
+			dev_dbg(maxreg->dev, "Invalid fps-slot '%s', using default\n", fps_str);
+			sbb_setting->fps_slot = MAX77675_FPS_DEF;
+		} else {
+			sbb_setting->fps_slot = slot;
+		}
+	}
+
+	/* Parse slew rate control source */
+	sbb_setting->fixed_slew_rate = of_property_read_bool(np, "adi,fixed-slew-rate");
+
+	/* Apply parsed configuration */
+	return max77675_init_regulator(maxreg, desc->id);
+}
+
+static int max77675_get_error_flags(struct regulator_dev *rdev, unsigned int *flags)
+{
+	struct max77675_regulator *maxreg = rdev_get_drvdata(rdev);
+	unsigned int int_flags;
+	int id = rdev_get_id(rdev);
+	int ret;
+
+	ret = regmap_read(maxreg->regmap, MAX77675_REG_INT_GLBL, &int_flags);
+	if (ret) {
+		dev_err(maxreg->dev, "Failed to read INT_GLBL: %d\n", ret);
+		return ret;
+	}
+
+	*flags = 0;
+
+	switch (id) {
+	case MAX77675_ID_SBB0:
+		if (int_flags & MAX77675_INT_SBB0_F_BIT)
+			*flags |= REGULATOR_ERROR_FAIL;
+		break;
+	case MAX77675_ID_SBB1:
+		if (int_flags & MAX77675_INT_SBB1_F_BIT)
+			*flags |= REGULATOR_ERROR_FAIL;
+		break;
+	case MAX77675_ID_SBB2:
+		if (int_flags & MAX77675_INT_SBB2_F_BIT)
+			*flags |= REGULATOR_ERROR_FAIL;
+		break;
+	case MAX77675_ID_SBB3:
+		if (int_flags & MAX77675_INT_SBB3_F_BIT)
+			*flags |= REGULATOR_ERROR_FAIL;
+		break;
+	default:
+		dev_warn(maxreg->dev, "Unsupported regulator ID: %d\n", id);
+		break;
+	}
+
+	if (int_flags & MAX77675_INT_TJAL2_R_BIT) {
+		/* TJAL2 interrupt: Over-temperature condition (above 120 degree) */
+		*flags |= REGULATOR_ERROR_OVER_TEMP;
+	}
+
+	return 0;
+}
+
+static const struct regulator_ops max77675_regulator_ops = {
+	.list_voltage         = regulator_list_voltage_linear,
+	.enable               = regulator_enable_regmap,
+	.disable              = regulator_disable_regmap,
+	.is_enabled           = regulator_is_enabled_regmap,
+	.map_voltage          = regulator_map_voltage_linear,
+	.set_voltage_sel      = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel      = regulator_get_voltage_sel_regmap,
+	.set_active_discharge = regulator_set_active_discharge_regmap,
+	.get_error_flags      = max77675_get_error_flags,
+};
+
+static struct regulator_desc max77675_regulators[MAX77675_ID_NUM_MAX] = {
+	{
+		.name                  = "sbb0",
+		.of_match              = of_match_ptr("sbb0"),
+		.regulators_node       = of_match_ptr("regulators"),
+		.of_parse_cb           = max77675_of_parse_cb,
+		.id                    = MAX77675_ID_SBB0,
+		.ops                   = &max77675_regulator_ops,
+		.type                  = REGULATOR_VOLTAGE,
+		.owner                 = THIS_MODULE,
+		.n_voltages            = MAX77675_NUM_LEVELS_25MV,
+		.min_uV                = MAX77675_MIN_UV,
+		.uV_step               = MAX77675_STEP_25MV,
+		.vsel_reg              = MAX77675_REG_CNFG_SBB0_A,
+		.vsel_mask             = MAX77675_TV_SBB0_MASK,
+		.enable_reg            = MAX77675_REG_CNFG_SBB0_B,
+		.enable_mask           = MAX77675_EN_SBB0_MASK,
+		.enable_val            = MAX77675_ENABLE_ON,
+		.disable_val           = MAX77675_ENABLE_OFF,
+		.active_discharge_off  = MAX77675_REGULATOR_AD_OFF,
+		.active_discharge_on   = MAX77675_REGULATOR_AD_ON,
+		.active_discharge_mask = MAX77675_ADE_SBB0_BIT,
+		.active_discharge_reg  = MAX77675_REG_CNFG_SBB0_B,
+	},
+	{
+		.name                  = "sbb1",
+		.of_match              = of_match_ptr("sbb1"),
+		.regulators_node       = of_match_ptr("regulators"),
+		.of_parse_cb           = max77675_of_parse_cb,
+		.id                    = MAX77675_ID_SBB1,
+		.ops                   = &max77675_regulator_ops,
+		.type                  = REGULATOR_VOLTAGE,
+		.owner                 = THIS_MODULE,
+		.n_voltages            = MAX77675_NUM_LEVELS_25MV,
+		.min_uV                = MAX77675_MIN_UV,
+		.uV_step               = MAX77675_STEP_25MV,
+		.vsel_reg              = MAX77675_REG_CNFG_SBB1_A,
+		.vsel_mask             = MAX77675_TV_SBB1_MASK,
+		.enable_reg            = MAX77675_REG_CNFG_SBB1_B,
+		.enable_mask           = MAX77675_EN_SBB1_MASK,
+		.enable_val            = MAX77675_ENABLE_ON,
+		.disable_val           = MAX77675_ENABLE_OFF,
+		.active_discharge_off  = MAX77675_REGULATOR_AD_OFF,
+		.active_discharge_on   = MAX77675_REGULATOR_AD_ON,
+		.active_discharge_mask = MAX77675_ADE_SBB1_BIT,
+		.active_discharge_reg  = MAX77675_REG_CNFG_SBB1_B,
+	},
+	{
+		.name                  = "sbb2",
+		.of_match              = of_match_ptr("sbb2"),
+		.regulators_node       = of_match_ptr("regulators"),
+		.of_parse_cb           = max77675_of_parse_cb,
+		.id                    = MAX77675_ID_SBB2,
+		.ops                   = &max77675_regulator_ops,
+		.type                  = REGULATOR_VOLTAGE,
+		.owner                 = THIS_MODULE,
+		.n_voltages            = MAX77675_NUM_LEVELS_25MV,
+		.min_uV                = MAX77675_MIN_UV,
+		.uV_step               = MAX77675_STEP_25MV,
+		.vsel_reg              = MAX77675_REG_CNFG_SBB2_A,
+		.vsel_mask             = MAX77675_TV_SBB2_MASK,
+		.enable_reg            = MAX77675_REG_CNFG_SBB2_B,
+		.enable_mask           = MAX77675_EN_SBB2_MASK,
+		.enable_val            = MAX77675_ENABLE_ON,
+		.disable_val           = MAX77675_ENABLE_OFF,
+		.active_discharge_off  = MAX77675_REGULATOR_AD_OFF,
+		.active_discharge_on   = MAX77675_REGULATOR_AD_ON,
+		.active_discharge_mask = MAX77675_ADE_SBB2_BIT,
+		.active_discharge_reg  = MAX77675_REG_CNFG_SBB2_B,
+	},
+	{
+		.name                  = "sbb3",
+		.of_match              = of_match_ptr("sbb3"),
+		.regulators_node       = of_match_ptr("regulators"),
+		.of_parse_cb           = max77675_of_parse_cb,
+		.id                    = MAX77675_ID_SBB3,
+		.ops                   = &max77675_regulator_ops,
+		.type                  = REGULATOR_VOLTAGE,
+		.owner                 = THIS_MODULE,
+		.n_voltages            = MAX77675_NUM_LEVELS_25MV,
+		.min_uV                = MAX77675_MIN_UV,
+		.uV_step               = MAX77675_STEP_25MV,
+		.vsel_reg              = MAX77675_REG_CNFG_SBB3_A,
+		.vsel_mask             = MAX77675_TV_SBB3_MASK,
+		.enable_reg            = MAX77675_REG_CNFG_SBB3_B,
+		.enable_mask           = MAX77675_EN_SBB3_MASK,
+		.enable_val            = MAX77675_ENABLE_ON,
+		.disable_val           = MAX77675_ENABLE_OFF,
+		.active_discharge_off  = MAX77675_REGULATOR_AD_OFF,
+		.active_discharge_on   = MAX77675_REGULATOR_AD_ON,
+		.active_discharge_mask = MAX77675_ADE_SBB3_BIT,
+		.active_discharge_reg  = MAX77675_REG_CNFG_SBB3_B,
+	},
+};
+
+static bool max77675_volatile_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case MAX77675_REG_CNFG_GLBL_B:
+		/* This register can be updated by an internal state machine */
+	case MAX77675_REG_INT_GLBL:
+	case MAX77675_REG_STAT_GLBL:
+	case MAX77675_REG_ERCF_GLBL:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static const struct regmap_config max77675_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = MAX77675_MAX_REGISTER,
+	.cache_type = REGCACHE_MAPLE,
+	.volatile_reg = max77675_volatile_reg,
+};
+
+static int max77675_apply_config(struct max77675_regulator *maxreg)
+{
+	const struct max77675_config *cfg = &maxreg->config;
+	int ret;
+
+	/* Set EN pin mode */
+	ret = regmap_update_bits(maxreg->regmap, MAX77675_REG_CNFG_GLBL_A,
+				 MAX77675_EN_MODE_MASK,
+				 FIELD_PREP(MAX77675_EN_MODE_MASK, cfg->en_mode));
+	if (ret) {
+		dev_err(maxreg->dev, "Failed to set EN mode: %d\n", ret);
+		return ret;
+	}
+
+	/* Set the latency between output voltage change and SBBx voltage ramp start */
+	ret = regmap_update_bits(maxreg->regmap, MAX77675_REG_CNFG_SBB_TOP_B,
+				 MAX77675_LAT_MODE_BIT,
+				 FIELD_PREP(MAX77675_LAT_MODE_BIT, cfg->voltage_change_latency));
+	if (ret) {
+		dev_err(maxreg->dev, "Failed to set latency mode: %d\n", ret);
+		return ret;
+	}
+
+	/* Set drive strength */
+	ret = regmap_update_bits(maxreg->regmap, MAX77675_REG_CNFG_SBB_TOP_A,
+				 MAX77675_DRV_SBB_MASK,
+				 FIELD_PREP(MAX77675_DRV_SBB_MASK, cfg->drv_sbb_strength));
+	if (ret) {
+		dev_err(maxreg->dev, "Failed to set drive strength: %d\n", ret);
+		return ret;
+	}
+
+	/* Set DVS slew rate */
+	ret = regmap_update_bits(maxreg->regmap, MAX77675_REG_CNFG_SBB_TOP_B,
+				 MAX77675_DVS_SLEW_BIT,
+				 FIELD_PREP(MAX77675_DVS_SLEW_BIT, cfg->dvs_slew_rate));
+	if (ret) {
+		dev_err(maxreg->dev, "Failed to set DVS slew rate: %d\n", ret);
+		return ret;
+	}
+
+	/* Set debounce time for EN pin */
+	ret = regmap_update_bits(maxreg->regmap, MAX77675_REG_CNFG_GLBL_A,
+				 MAX77675_DBEN_EN_BIT,
+				 FIELD_PREP(MAX77675_DBEN_EN_BIT, cfg->debounce_time));
+	if (ret) {
+		dev_err(maxreg->dev, "Failed to set EN debounce time: %d\n", ret);
+		return ret;
+	}
+
+	/* Set manual reset time (MRT) for EN pin */
+	ret = regmap_update_bits(maxreg->regmap, MAX77675_REG_CNFG_GLBL_A,
+				 MAX77675_MRT_MASK,
+				 FIELD_PREP(MAX77675_MRT_MASK, cfg->manual_reset_time));
+	if (ret) {
+		dev_err(maxreg->dev, "Failed to set manual reset time: %d\n", ret);
+		return ret;
+	}
+
+	/* Enable or disable internal pull-up resistor on EN pin */
+	ret = regmap_update_bits(maxreg->regmap, MAX77675_REG_CNFG_GLBL_A,
+				 MAX77675_PU_DIS_BIT,
+				 FIELD_PREP(MAX77675_PU_DIS_BIT, cfg->en_pullup_disable));
+	if (ret) {
+		dev_err(maxreg->dev, "Failed to set EN pull-up disable: %d\n", ret);
+		return ret;
+	}
+
+	/* Request main bias to enter low-power mode */
+	ret = regmap_update_bits(maxreg->regmap, MAX77675_REG_CNFG_GLBL_A,
+				 MAX77675_BIAS_LPM_BIT,
+				 FIELD_PREP(MAX77675_BIAS_LPM_BIT, cfg->bias_low_power_request));
+	if (ret) {
+		dev_err(maxreg->dev, "Failed to set bias low-power request: %d\n", ret);
+		return ret;
+	}
+
+	/* Force SIMO internal LDO to always supply 1.8V */
+	ret = regmap_update_bits(maxreg->regmap, MAX77675_REG_CNFG_GLBL_A,
+				 MAX77675_SIMO_CH_DIS_BIT,
+				 FIELD_PREP(MAX77675_SIMO_CH_DIS_BIT, cfg->simo_ldo_always_on));
+	if (ret) {
+		dev_err(maxreg->dev, "Failed to set SIMO internal LDO always-on: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int max77675_parse_en_mode(struct device *dev,
+				  struct device_node *np,
+				  u8 *en_mode)
+{
+	static const char * const en_modes[] = {"push-button", "slide-switch", "logic"};
+	const char *str;
+	int index;
+
+	*en_mode = MAX77675_EN_SLIDE_SWITCH;
+
+	if (of_property_read_string(np, "adi,en-mode", &str))
+		return 0;
+
+	index = match_string(en_modes, ARRAY_SIZE(en_modes), str);
+	if (index < 0) {
+		dev_err(dev, "Invalid 'adi,en-mode' value '%s'\n", str);
+		return -EINVAL;
+	}
+
+	*en_mode = index;
+
+	return 0;
+}
+
+static int max77675_parse_voltage_change_latency(struct device *dev,
+						 struct device_node *np,
+						 u8 *latency_mode)
+{
+	u32 val;
+
+	*latency_mode = MAX77675_HIGH_LATENCY_MODE;
+
+	if (!of_property_read_u32(np, "adi,voltage-change-latency-us", &val)) {
+		switch (val) {
+		case 10:
+			*latency_mode = MAX77675_LOW_LATENCY_MODE;
+			break;
+		case 100:
+			*latency_mode = MAX77675_HIGH_LATENCY_MODE;
+			break;
+		default:
+			dev_err(dev, "Invalid voltage-change-latency-us value: %u\n", val);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int max77675_parse_manual_reset_time(struct device *dev,
+					    struct device_node *np,
+					    u8 *reset_time)
+{
+	u32 val;
+
+	*reset_time = MAX77675_MRT_4S;
+
+	if (!of_property_read_u32(np, "reset-time-sec", &val)) {
+		switch (val) {
+		case 4:
+			*reset_time = MAX77675_MRT_4S;
+			break;
+		case 8:
+			*reset_time = MAX77675_MRT_8S;
+			break;
+		case 12:
+			*reset_time = MAX77675_MRT_12S;
+			break;
+		case 16:
+			*reset_time = MAX77675_MRT_16S;
+			break;
+		default:
+			dev_err(dev, "Invalid reset-time-sec value: %u\n", val);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int max77675_parse_dvs_slew_rate(struct device *dev, struct device_node *np, u8 *slew_rate)
+{
+	u32 val;
+
+	/* Set default: 5 mV/us */
+	*slew_rate = MAX77675_DVS_SLEW_5MV_PER_US;
+
+	if (!of_property_read_u32(np, "adi,dvs-slew-rate-mv-per-us", &val)) {
+		switch (val) {
+		case 5:
+			*slew_rate = MAX77675_DVS_SLEW_5MV_PER_US;
+			break;
+		case 10:
+			*slew_rate = MAX77675_DVS_SLEW_10MV_PER_US;
+			break;
+		default:
+			dev_err(dev, "Invalid dvs-slew-rate-mv-per-us value: %u\n", val);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int max77675_parse_drv_sbb_strength(struct device *dev, struct device_node *np, u8 *strength)
+{
+	static const char * const strength_names[] = {"max", "high", "low", "min"};
+	const char *str;
+	int index;
+
+	/* Set default: maximum drive strength */
+	*strength = MAX77675_DRV_SBB_STRENGTH_MAX;
+
+	if (of_property_read_string(np, "adi,drv-sbb-strength", &str))
+		return 0;
+
+	index = match_string(strength_names, ARRAY_SIZE(strength_names), str);
+	if (index < 0) {
+		dev_err(dev, "Invalid 'adi,drv-sbb-strength' value: '%s'\n", str);
+		return -EINVAL;
+	}
+
+	*strength = index;
+
+	return 0;
+}
+
+static int max77675_parse_debounce_time_us(struct device *dev,
+					   struct device_node *np,
+					   u8 *debounce_time)
+{
+	u32 val;
+
+	*debounce_time = MAX77675_DBEN_100US;
+
+	if (!of_property_read_u32(np, "input-debounce", &val)) {
+		switch (val) {
+		case 100:
+			*debounce_time = MAX77675_DBEN_100US;
+			break;
+		case 30000:
+			*debounce_time = MAX77675_DBEN_30000US;
+			break;
+		default:
+			dev_err(dev, "Invalid input-debounce value: %u\n", val);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int max77675_parse_config(struct max77675_regulator *maxreg)
+{
+	struct device_node *np = maxreg->dev->of_node;
+	struct max77675_config *cfg = &maxreg->config;
+	int ret;
+
+	/* EN pin mode */
+	ret = max77675_parse_en_mode(maxreg->dev, np, &cfg->en_mode);
+	if (ret < 0)
+		return ret;
+
+	/* voltage change latency */
+	ret = max77675_parse_voltage_change_latency(maxreg->dev, np, &cfg->voltage_change_latency);
+	if (ret < 0)
+		return ret;
+
+	/* drive strength */
+	ret = max77675_parse_drv_sbb_strength(maxreg->dev, np, &cfg->drv_sbb_strength);
+	if (ret < 0)
+		return ret;
+
+	/* dvs slew rate */
+	ret = max77675_parse_dvs_slew_rate(maxreg->dev, np, &cfg->dvs_slew_rate);
+	if (ret < 0)
+		return ret;
+
+	/* Debounce time for EN pin */
+	ret = max77675_parse_debounce_time_us(maxreg->dev, np, &cfg->debounce_time);
+	if (ret < 0)
+		return ret;
+
+	/* Manual reset time for EN pin */
+	ret = max77675_parse_manual_reset_time(maxreg->dev, np, &cfg->manual_reset_time);
+	if (ret < 0)
+		return ret;
+
+	/* Disable internal pull-up resistor on EN pin */
+	cfg->en_pullup_disable = of_property_read_bool(np, "bias-disable");
+
+	/* Request low-power mode for main bias */
+	cfg->bias_low_power_request = of_property_read_bool(np, "adi,bias-low-power-request");
+
+	/* Force internal LDO to always supply 1.8V */
+	cfg->simo_ldo_always_on = of_property_read_bool(np, "adi,simo-ldo-always-on");
+
+	return ret;
+}
+
+static int max77675_init_event(struct max77675_regulator *maxreg)
+{
+	unsigned int ercflag, int_glbl;
+	int ret;
+
+	ret = regmap_read(maxreg->regmap, MAX77675_REG_ERCF_GLBL, &ercflag);
+	if (ret) {
+		dev_err(maxreg->dev, "Failed to read CID register: %d\n", ret);
+		return ret;
+	}
+
+	ret = regmap_read(maxreg->regmap, MAX77675_REG_INT_GLBL, &int_glbl);
+	if (ret) {
+		dev_err(maxreg->dev, "Failed to read INT_GLBL register: %d\n", ret);
+		return ret;
+	}
+
+	if (ercflag & MAX77675_SFT_CRST_F_BIT)
+		dev_dbg(maxreg->dev, "Software Cold Reset Flag is set\n");
+
+	if (ercflag & MAX77675_SFT_OFF_F_BIT)
+		dev_dbg(maxreg->dev, "Software Off Flag is set\n");
+
+	if (ercflag & MAX77675_MRST_BIT)
+		dev_dbg(maxreg->dev, "Manual Reset Timer Flag is set\n");
+
+	if (ercflag & MAX77675_UVLO_BIT)
+		dev_dbg(maxreg->dev, "Undervoltage Lockout Flag is set\n");
+
+	if (ercflag & MAX77675_OVLO_BIT)
+		dev_dbg(maxreg->dev, "Overvoltage Lockout Flag is set\n");
+
+	if (ercflag & MAX77675_TOVLD_BIT)
+		dev_dbg(maxreg->dev, "Thermal Overload Flag is set\n");
+
+	if (int_glbl & MAX77675_INT_SBB3_F_BIT)
+		dev_dbg(maxreg->dev, "SBB3 Channel Fault Interrupt occurred\n");
+
+	if (int_glbl & MAX77675_INT_SBB2_F_BIT)
+		dev_dbg(maxreg->dev, "SBB2 Channel Fault Interrupt occurred\n");
+
+	if (int_glbl & MAX77675_INT_SBB1_F_BIT)
+		dev_dbg(maxreg->dev, "SBB1 Channel Fault Interrupt occurred\n");
+
+	if (int_glbl & MAX77675_INT_SBB0_F_BIT)
+		dev_dbg(maxreg->dev, "SBB0 Channel Fault Interrupt occurred\n");
+
+	if (int_glbl & MAX77675_INT_TJAL2_R_BIT)
+		dev_dbg(maxreg->dev, "Thermal Alarm 2 Rising Interrupt occurred\n");
+
+	if (int_glbl & MAX77675_INT_TJAL1_R_BIT)
+		dev_dbg(maxreg->dev, "Thermal Alarm 1 Rising Interrupt occurred\n");
+
+	if (int_glbl & MAX77675_INT_EN_R_BIT)
+		dev_dbg(maxreg->dev, "nEN Rising Edge Interrupt occurred\n");
+
+	if (int_glbl & MAX77675_INT_EN_F_BIT)
+		dev_dbg(maxreg->dev, "nEN Falling Edge Interrupt occurred\n");
+
+	return 0;
+}
+
+static int max77675_regulator_probe(struct i2c_client *client)
+{
+	struct max77675_regulator *maxreg;
+	struct regulator_config config = {};
+	int i, ret;
+
+	maxreg = devm_kzalloc(&client->dev, sizeof(*maxreg), GFP_KERNEL);
+	if (!maxreg)
+		return -ENOMEM;
+
+	maxreg->dev = &client->dev;
+
+	maxreg->regmap = devm_regmap_init_i2c(client, &max77675_regmap_config);
+	if (IS_ERR(maxreg->regmap))
+		return dev_err_probe(maxreg->dev,
+				     PTR_ERR(maxreg->regmap),
+				     "Failed to init regmap\n");
+
+	ret = max77675_init_event(maxreg);
+	if (ret < 0)
+		return dev_err_probe(maxreg->dev, ret, "Failed to init event\n");
+
+	ret = max77675_parse_config(maxreg);
+	if (ret < 0)
+		return dev_err_probe(maxreg->dev, ret, "Failed to parse config\n");
+
+	ret = max77675_apply_config(maxreg);
+	if (ret < 0)
+		return dev_err_probe(maxreg->dev, ret, "Failed to apply config\n");
+
+	config.dev = &client->dev;
+	config.regmap = maxreg->regmap;
+	config.driver_data = maxreg;
+
+	struct device_node *regulators_np __free(device_node) =
+		of_get_child_by_name(client->dev.of_node, "regulators");
+	if (!regulators_np) {
+		dev_err(maxreg->dev, "No 'regulators' subnode found in DT\n");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < MAX77675_ID_NUM_MAX; i++) {
+		const struct regulator_desc *desc = &max77675_regulators[i];
+		struct regulator_dev *rdev;
+
+		struct device_node *child_np __free(device_node) =
+			of_get_child_by_name(regulators_np, desc->name);
+		if (!child_np) {
+			dev_warn(maxreg->dev, "No DT node for regulator %s\n", desc->name);
+			continue;
+		}
+
+		config.of_node = child_np;
+
+		rdev = devm_regulator_register(&client->dev, desc, &config);
+		if (IS_ERR(rdev)) {
+			return dev_err_probe(maxreg->dev, PTR_ERR(rdev),
+				"Failed to register regulator %d (%s)\n",
+				i, desc->name);
+		}
+	}
+
+	return 0;
+}
+
+static const struct i2c_device_id max77675_i2c_id[] = {
+	{ "max77675", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, max77675_i2c_id);
+
+static const struct of_device_id __maybe_unused max77675_of_match[] = {
+	{ .compatible = "adi,max77675", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, max77675_of_match);
+
+static struct i2c_driver max77675_regulator_driver = {
+	.driver = {
+		.name = "max77675",
+		.of_match_table = of_match_ptr(max77675_of_match),
+	},
+	.probe = max77675_regulator_probe,
+	.id_table = max77675_i2c_id,
+};
+
+module_i2c_driver(max77675_regulator_driver);
+
+MODULE_DESCRIPTION("MAX77675 Regulator Driver");
+MODULE_AUTHOR("Joan Na <joan.na@analog.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c
index 5130a35214c9..8426d7af9c75 100644
--- a/drivers/remoteproc/imx_dsp_rproc.c
+++ b/drivers/remoteproc/imx_dsp_rproc.c
@@ -38,15 +38,15 @@ MODULE_PARM_DESC(no_mailboxes,
 
 /* Flag indicating that the remote is up and running */
 #define REMOTE_IS_READY				BIT(0)
-/* Flag indicating that the host should wait for a firmware-ready response */
-#define WAIT_FW_READY				BIT(1)
+/* Flag indicating that the host should wait for a firmware-confirmation response */
+#define WAIT_FW_CONFIRMATION				BIT(1)
 #define REMOTE_READY_WAIT_MAX_RETRIES		500
 
 /*
  * This flag is set in the DSP resource table's features field to indicate
- * that the firmware requires the host NOT to wait for a FW_READY response.
+ * that the firmware requires the host NOT to wait for a FW_CONFIRMATION response.
  */
-#define FEATURE_DONT_WAIT_FW_READY		BIT(0)
+#define FEATURE_SKIP_FW_CONFIRMATION		BIT(0)
 
 /* att flags */
 /* DSP own area */
@@ -287,7 +287,7 @@ static int imx_dsp_rproc_ready(struct rproc *rproc)
  * @avail: available space in the resource table
  *
  * Parse the DSP-specific resource entry and update flags accordingly.
- * If the WAIT_FW_READY feature is set, the host must wait for the firmware
+ * If the WAIT_FW_CONFIRMATION feature is set, the host must wait for the firmware
  * to signal readiness before proceeding with execution.
  *
  * Return: RSC_HANDLED if processed successfully, RSC_IGNORED otherwise.
@@ -322,7 +322,7 @@ static int imx_dsp_rproc_handle_rsc(struct rproc *rproc, u32 rsc_type,
 
 	/*
 	 * For now, in struct fw_rsc_imx_dsp, version 0,
-	 * only FEATURE_DONT_WAIT_FW_READY is valid.
+	 * only FEATURE_SKIP_FW_CONFIRMATION is valid.
 	 *
 	 * When adding new features, please upgrade version.
 	 */
@@ -332,8 +332,8 @@ static int imx_dsp_rproc_handle_rsc(struct rproc *rproc, u32 rsc_type,
 		return RSC_IGNORED;
 	}
 
-	if (imx_dsp_rsc->features & FEATURE_DONT_WAIT_FW_READY)
-		priv->flags &= ~WAIT_FW_READY;
+	if (imx_dsp_rsc->features & FEATURE_SKIP_FW_CONFIRMATION)
+		priv->flags &= ~WAIT_FW_CONFIRMATION;
 
 	return RSC_HANDLED;
 }
@@ -385,7 +385,7 @@ static int imx_dsp_rproc_start(struct rproc *rproc)
 		return ret;
 	}
 
-	if (priv->flags & WAIT_FW_READY)
+	if (priv->flags & WAIT_FW_CONFIRMATION)
 		return imx_dsp_rproc_ready(rproc);
 
 	return 0;
@@ -644,6 +644,32 @@ static void imx_dsp_rproc_free_mbox(struct imx_dsp_rproc *priv)
 	mbox_free_channel(priv->rxdb_ch);
 }
 
+static int imx_dsp_rproc_mem_alloc(struct rproc *rproc,
+				   struct rproc_mem_entry *mem)
+{
+	struct device *dev = rproc->dev.parent;
+	void *va;
+
+	va = ioremap_wc(mem->dma, mem->len);
+	if (!va) {
+		dev_err(dev, "Unable to map memory region: %pa+%zx\n",
+			&mem->dma, mem->len);
+		return -ENOMEM;
+	}
+
+	mem->va = va;
+
+	return 0;
+}
+
+static int imx_dsp_rproc_mem_release(struct rproc *rproc,
+				     struct rproc_mem_entry *mem)
+{
+	iounmap(mem->va);
+
+	return 0;
+}
+
 /**
  * imx_dsp_rproc_add_carveout() - request mailbox channels
  * @priv: private data pointer
@@ -659,7 +685,6 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv)
 	struct device *dev = rproc->dev.parent;
 	struct device_node *np = dev->of_node;
 	struct rproc_mem_entry *mem;
-	void __iomem *cpu_addr;
 	int a, i = 0;
 	u64 da;
 
@@ -673,15 +698,10 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv)
 		if (imx_dsp_rproc_sys_to_da(priv, att->sa, att->size, &da))
 			return -EINVAL;
 
-		cpu_addr = devm_ioremap_wc(dev, att->sa, att->size);
-		if (!cpu_addr) {
-			dev_err(dev, "failed to map memory %p\n", &att->sa);
-			return -ENOMEM;
-		}
-
 		/* Register memory region */
-		mem = rproc_mem_entry_init(dev, (void __force *)cpu_addr, (dma_addr_t)att->sa,
-					   att->size, da, NULL, NULL, "dsp_mem");
+		mem = rproc_mem_entry_init(dev, NULL, (dma_addr_t)att->sa,
+					   att->size, da, imx_dsp_rproc_mem_alloc,
+					   imx_dsp_rproc_mem_release, "dsp_mem");
 
 		if (mem)
 			rproc_coredump_add_segment(rproc, da, att->size);
@@ -709,15 +729,11 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv)
 		if (imx_dsp_rproc_sys_to_da(priv, res.start, resource_size(&res), &da))
 			return -EINVAL;
 
-		cpu_addr = devm_ioremap_resource_wc(dev, &res);
-		if (IS_ERR(cpu_addr)) {
-			dev_err(dev, "failed to map memory %pR\n", &res);
-			return PTR_ERR(cpu_addr);
-		}
-
 		/* Register memory region */
-		mem = rproc_mem_entry_init(dev, (void __force *)cpu_addr, (dma_addr_t)res.start,
-					   resource_size(&res), da, NULL, NULL,
+		mem = rproc_mem_entry_init(dev, NULL, (dma_addr_t)res.start,
+					   resource_size(&res), da,
+					    imx_dsp_rproc_mem_alloc,
+					    imx_dsp_rproc_mem_release,
 					   "%.*s", strchrnul(res.name, '@') - res.name, res.name);
 		if (!mem)
 			return -ENOMEM;
@@ -1131,8 +1147,8 @@ static int imx_dsp_rproc_probe(struct platform_device *pdev)
 	priv = rproc->priv;
 	priv->rproc = rproc;
 	priv->dsp_dcfg = dsp_dcfg;
-	/* By default, host waits for fw_ready reply */
-	priv->flags |= WAIT_FW_READY;
+	/* By default, host waits for fw_confirmation reply */
+	priv->flags |= WAIT_FW_CONFIRMATION;
 
 	if (no_mailboxes)
 		imx_dsp_rproc_mbox_init = imx_dsp_rproc_mbox_no_alloc;
@@ -1242,6 +1258,21 @@ static int imx_dsp_suspend(struct device *dev)
 	if (rproc->state != RPROC_RUNNING)
 		goto out;
 
+	/*
+	 * No channel available for sending messages;
+	 * indicates no mailboxes present, so trigger PM runtime suspend
+	 */
+	if (!priv->tx_ch) {
+		dev_dbg(dev, "No initialized mbox tx channel, suspend directly.\n");
+		goto out;
+	}
+
+	/* No fw confirmation expected, so trigger PM runtime suspend */
+	if (!(priv->flags & WAIT_FW_CONFIRMATION)) {
+		dev_dbg(dev, "No FW_CONFIRMATION needed, suspend directly.\n");
+		goto out;
+	}
+
 	reinit_completion(&priv->pm_comp);
 
 	/* Tell DSP that suspend is happening */
diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c
index 3be8790c14a2..33f21ab24c92 100644
--- a/drivers/remoteproc/imx_rproc.c
+++ b/drivers/remoteproc/imx_rproc.c
@@ -694,7 +694,7 @@ static int imx_rproc_addr_init(struct imx_rproc *priv,
 		}
 		priv->mem[b].sys_addr = res.start;
 		priv->mem[b].size = resource_size(&res);
-		if (!strcmp(res.name, "rsc-table"))
+		if (strstarts(res.name, "rsc-table"))
 			priv->rsc_table = priv->mem[b].cpu_addr;
 		b++;
 	}
diff --git a/drivers/reset/core.c b/drivers/reset/core.c
index 0135dd0ae204..0666dfc41ca9 100644
--- a/drivers/reset/core.c
+++ b/drivers/reset/core.c
@@ -868,11 +868,11 @@ static int reset_add_gpio_aux_device(struct device *parent,
  */
 static int __reset_add_reset_gpio_device(const struct of_phandle_args *args)
 {
-	struct property_entry properties[2] = { };
+	struct property_entry properties[3] = { };
 	unsigned int offset, of_flags, lflags;
 	struct reset_gpio_lookup *rgpio_dev;
 	struct device *parent;
-	int id, ret;
+	int id, ret, prop = 0;
 
 	/*
 	 * Currently only #gpio-cells=2 is supported with the meaning of:
@@ -923,7 +923,8 @@ static int __reset_add_reset_gpio_device(const struct of_phandle_args *args)
 
 	lflags = GPIO_PERSISTENT | (of_flags & GPIO_ACTIVE_LOW);
 	parent = gpio_device_to_device(gdev);
-	properties[0] = PROPERTY_ENTRY_GPIO("reset-gpios", parent->fwnode, offset, lflags);
+	properties[prop++] = PROPERTY_ENTRY_STRING("compatible", "reset-gpio");
+	properties[prop++] = PROPERTY_ENTRY_GPIO("reset-gpios", parent->fwnode, offset, lflags);
 
 	id = ida_alloc(&reset_gpio_ida, GFP_KERNEL);
 	if (id < 0)
diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c
index 5d661681a9b6..96964745065b 100644
--- a/drivers/rpmsg/rpmsg_core.c
+++ b/drivers/rpmsg/rpmsg_core.c
@@ -352,50 +352,38 @@ field##_show(struct device *dev,					\
 }									\
 static DEVICE_ATTR_RO(field);
 
-#define rpmsg_string_attr(field, member)				\
-static ssize_t								\
-field##_store(struct device *dev, struct device_attribute *attr,	\
-	      const char *buf, size_t sz)				\
-{									\
-	struct rpmsg_device *rpdev = to_rpmsg_device(dev);		\
-	const char *old;						\
-	char *new;							\
-									\
-	new = kstrndup(buf, sz, GFP_KERNEL);				\
-	if (!new)							\
-		return -ENOMEM;						\
-	new[strcspn(new, "\n")] = '\0';					\
-									\
-	device_lock(dev);						\
-	old = rpdev->member;						\
-	if (strlen(new)) {						\
-		rpdev->member = new;					\
-	} else {							\
-		kfree(new);						\
-		rpdev->member = NULL;					\
-	}								\
-	device_unlock(dev);						\
-									\
-	kfree(old);							\
-									\
-	return sz;							\
-}									\
-static ssize_t								\
-field##_show(struct device *dev,					\
-	     struct device_attribute *attr, char *buf)			\
-{									\
-	struct rpmsg_device *rpdev = to_rpmsg_device(dev);		\
-									\
-	return sprintf(buf, "%s\n", rpdev->member);			\
-}									\
-static DEVICE_ATTR_RW(field)
-
 /* for more info, see Documentation/ABI/testing/sysfs-bus-rpmsg */
 rpmsg_show_attr(name, id.name, "%s\n");
 rpmsg_show_attr(src, src, "0x%x\n");
 rpmsg_show_attr(dst, dst, "0x%x\n");
 rpmsg_show_attr(announce, announce ? "true" : "false", "%s\n");
-rpmsg_string_attr(driver_override, driver_override);
+
+static ssize_t driver_override_store(struct device *dev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t count)
+{
+	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
+	int ret;
+
+	ret = driver_set_override(dev, &rpdev->driver_override, buf, count);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static ssize_t driver_override_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
+	ssize_t len;
+
+	device_lock(dev);
+	len = sysfs_emit(buf, "%s\n", rpdev->driver_override);
+	device_unlock(dev);
+	return len;
+}
+static DEVICE_ATTR_RW(driver_override);
 
 static ssize_t modalias_show(struct device *dev,
 			     struct device_attribute *attr, char *buf)
diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c
index 484890b4a6a7..79d983055b4d 100644
--- a/drivers/rpmsg/virtio_rpmsg_bus.c
+++ b/drivers/rpmsg/virtio_rpmsg_bus.c
@@ -41,13 +41,12 @@
  * @buf_size:   size of one rx or tx buffer
  * @last_sbuf:	index of last tx buffer used
  * @bufs_dma:	dma base addr of the buffers
- * @tx_lock:	protects svq, sbufs and sleepers, to allow concurrent senders.
+ * @tx_lock:	protects svq and sbufs, to allow concurrent senders.
  *		sending a message might require waking up a dozing remote
  *		processor, which involves sleeping, hence the mutex.
  * @endpoints:	idr of local endpoints, allows fast retrieval
  * @endpoints_lock: lock of the endpoints set
  * @sendq:	wait queue of sending contexts waiting for a tx buffers
- * @sleepers:	number of senders that are waiting for a tx buffer
  *
  * This structure stores the rpmsg state of a given virtio remote processor
  * device (there might be several virtio proc devices for each physical
@@ -65,7 +64,6 @@ struct virtproc_info {
 	struct idr endpoints;
 	struct mutex endpoints_lock;
 	wait_queue_head_t sendq;
-	atomic_t sleepers;
 };
 
 /* The feature bitmap for virtio rpmsg */
@@ -144,6 +142,8 @@ static int virtio_rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len,
 static int virtio_rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len);
 static int virtio_rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data,
 				  int len, u32 dst);
+static __poll_t virtio_rpmsg_poll(struct rpmsg_endpoint *ept, struct file *filp,
+				  poll_table *wait);
 static ssize_t virtio_rpmsg_get_mtu(struct rpmsg_endpoint *ept);
 static struct rpmsg_device *__rpmsg_create_channel(struct virtproc_info *vrp,
 						   struct rpmsg_channel_info *chinfo);
@@ -154,6 +154,7 @@ static const struct rpmsg_endpoint_ops virtio_endpoint_ops = {
 	.sendto = virtio_rpmsg_sendto,
 	.trysend = virtio_rpmsg_trysend,
 	.trysendto = virtio_rpmsg_trysendto,
+	.poll = virtio_rpmsg_poll,
 	.get_mtu = virtio_rpmsg_get_mtu,
 };
 
@@ -436,7 +437,6 @@ static void *get_a_tx_buf(struct virtproc_info *vrp)
 	unsigned int len;
 	void *ret;
 
-	/* support multiple concurrent senders */
 	mutex_lock(&vrp->tx_lock);
 
 	/*
@@ -454,62 +454,6 @@ static void *get_a_tx_buf(struct virtproc_info *vrp)
 	return ret;
 }
 
-/**
- * rpmsg_upref_sleepers() - enable "tx-complete" interrupts, if needed
- * @vrp: virtual remote processor state
- *
- * This function is called before a sender is blocked, waiting for
- * a tx buffer to become available.
- *
- * If we already have blocking senders, this function merely increases
- * the "sleepers" reference count, and exits.
- *
- * Otherwise, if this is the first sender to block, we also enable
- * virtio's tx callbacks, so we'd be immediately notified when a tx
- * buffer is consumed (we rely on virtio's tx callback in order
- * to wake up sleeping senders as soon as a tx buffer is used by the
- * remote processor).
- */
-static void rpmsg_upref_sleepers(struct virtproc_info *vrp)
-{
-	/* support multiple concurrent senders */
-	mutex_lock(&vrp->tx_lock);
-
-	/* are we the first sleeping context waiting for tx buffers ? */
-	if (atomic_inc_return(&vrp->sleepers) == 1)
-		/* enable "tx-complete" interrupts before dozing off */
-		virtqueue_enable_cb(vrp->svq);
-
-	mutex_unlock(&vrp->tx_lock);
-}
-
-/**
- * rpmsg_downref_sleepers() - disable "tx-complete" interrupts, if needed
- * @vrp: virtual remote processor state
- *
- * This function is called after a sender, that waited for a tx buffer
- * to become available, is unblocked.
- *
- * If we still have blocking senders, this function merely decreases
- * the "sleepers" reference count, and exits.
- *
- * Otherwise, if there are no more blocking senders, we also disable
- * virtio's tx callbacks, to avoid the overhead incurred with handling
- * those (now redundant) interrupts.
- */
-static void rpmsg_downref_sleepers(struct virtproc_info *vrp)
-{
-	/* support multiple concurrent senders */
-	mutex_lock(&vrp->tx_lock);
-
-	/* are we the last sleeping context waiting for tx buffers ? */
-	if (atomic_dec_and_test(&vrp->sleepers))
-		/* disable "tx-complete" interrupts */
-		virtqueue_disable_cb(vrp->svq);
-
-	mutex_unlock(&vrp->tx_lock);
-}
-
 /**
  * rpmsg_send_offchannel_raw() - send a message across to the remote processor
  * @rpdev: the rpmsg channel
@@ -582,9 +526,6 @@ static int rpmsg_send_offchannel_raw(struct rpmsg_device *rpdev,
 
 	/* no free buffer ? wait for one (but bail after 15 seconds) */
 	while (!msg) {
-		/* enable "tx-complete" interrupts, if not already enabled */
-		rpmsg_upref_sleepers(vrp);
-
 		/*
 		 * sleep until a free buffer is available or 15 secs elapse.
 		 * the timeout period is not configurable because there's
@@ -595,9 +536,6 @@ static int rpmsg_send_offchannel_raw(struct rpmsg_device *rpdev,
 					(msg = get_a_tx_buf(vrp)),
 					msecs_to_jiffies(15000));
 
-		/* disable "tx-complete" interrupts if we're the last sleeper */
-		rpmsg_downref_sleepers(vrp);
-
 		/* timeout ? */
 		if (!err) {
 			dev_err(dev, "timeout waiting for a tx buffer\n");
@@ -676,6 +614,34 @@ static int virtio_rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data,
 	return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, false);
 }
 
+static __poll_t virtio_rpmsg_poll(struct rpmsg_endpoint *ept, struct file *filp,
+				  poll_table *wait)
+{
+	struct rpmsg_device *rpdev = ept->rpdev;
+	struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev);
+	struct virtproc_info *vrp = vch->vrp;
+	__poll_t mask = 0;
+
+	poll_wait(filp, &vrp->sendq, wait);
+
+	/* support multiple concurrent senders */
+	mutex_lock(&vrp->tx_lock);
+
+	/*
+	 * check for a free buffer, either:
+	 * - we haven't used all of the available transmit buffers (half of the
+	 *   allocated buffers are used for transmit, hence num_bufs / 2), or,
+	 * - we ask the virtqueue if there's a buffer available
+	 */
+	if (vrp->last_sbuf < vrp->num_bufs / 2 ||
+	    !virtqueue_enable_cb(vrp->svq))
+		mask |= EPOLLOUT;
+
+	mutex_unlock(&vrp->tx_lock);
+
+	return mask;
+}
+
 static ssize_t virtio_rpmsg_get_mtu(struct rpmsg_endpoint *ept)
 {
 	struct rpmsg_device *rpdev = ept->rpdev;
@@ -922,9 +888,6 @@ static int rpmsg_probe(struct virtio_device *vdev)
 		WARN_ON(err); /* sanity check; this can't really happen */
 	}
 
-	/* suppress "tx-complete" interrupts */
-	virtqueue_disable_cb(vrp->svq);
-
 	vdev->priv = vrp;
 
 	rpdev_ctrl = rpmsg_virtio_add_ctrl_dev(vdev);
diff --git a/drivers/rtc/rtc-optee.c b/drivers/rtc/rtc-optee.c
index 184c6d142801..eefde789d194 100644
--- a/drivers/rtc/rtc-optee.c
+++ b/drivers/rtc/rtc-optee.c
@@ -547,9 +547,9 @@ static int optee_ctx_match(struct tee_ioctl_version_data *ver, const void *data)
 		return 0;
 }
 
-static int optee_rtc_probe(struct device *dev)
+static int optee_rtc_probe(struct tee_client_device *rtc_device)
 {
-	struct tee_client_device *rtc_device = to_tee_client_device(dev);
+	struct device *dev = &rtc_device->dev;
 	struct tee_ioctl_open_session_arg sess2_arg = {0};
 	struct tee_ioctl_open_session_arg sess_arg = {0};
 	struct optee_rtc *priv;
@@ -682,8 +682,9 @@ static int optee_rtc_probe(struct device *dev)
 	return err;
 }
 
-static int optee_rtc_remove(struct device *dev)
+static void optee_rtc_remove(struct tee_client_device *rtc_device)
 {
+	struct device *dev = &rtc_device->dev;
 	struct optee_rtc *priv = dev_get_drvdata(dev);
 
 	if (priv->features & TA_RTC_FEATURE_ALARM) {
@@ -696,8 +697,6 @@ static int optee_rtc_remove(struct device *dev)
 	tee_shm_free(priv->shm);
 	tee_client_close_session(priv->ctx, priv->session_id);
 	tee_client_close_context(priv->ctx);
-
-	return 0;
 }
 
 static int optee_rtc_suspend(struct device *dev)
@@ -724,27 +723,15 @@ MODULE_DEVICE_TABLE(tee, optee_rtc_id_table);
 
 static struct tee_client_driver optee_rtc_driver = {
 	.id_table	= optee_rtc_id_table,
+	.probe		= optee_rtc_probe,
+	.remove		= optee_rtc_remove,
 	.driver		= {
 		.name		= "optee_rtc",
-		.bus		= &tee_bus_type,
-		.probe		= optee_rtc_probe,
-		.remove		= optee_rtc_remove,
 		.pm		= pm_sleep_ptr(&optee_rtc_pm_ops),
 	},
 };
 
-static int __init optee_rtc_mod_init(void)
-{
-	return driver_register(&optee_rtc_driver.driver);
-}
-
-static void __exit optee_rtc_mod_exit(void)
-{
-	driver_unregister(&optee_rtc_driver.driver);
-}
-
-module_init(optee_rtc_mod_init);
-module_exit(optee_rtc_mod_exit);
+module_tee_client_driver(optee_rtc_driver);
 
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Clément Léger <clement.leger@bootlin.com>");
diff --git a/drivers/s390/cio/blacklist.c b/drivers/s390/cio/blacklist.c
index 738d5e2d5304..020d210bde9f 100644
--- a/drivers/s390/cio/blacklist.c
+++ b/drivers/s390/cio/blacklist.c
@@ -10,6 +10,7 @@
 
 #define pr_fmt(fmt) "cio: " fmt
 
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/vmalloc.h>
 #include <linux/proc_fs.h>
diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c
index a445494fd2be..d652df96a507 100644
--- a/drivers/s390/crypto/ap_bus.c
+++ b/drivers/s390/crypto/ap_bus.c
@@ -16,6 +16,7 @@
 #include <linux/kernel_stat.h>
 #include <linux/moduleparam.h>
 #include <linux/export.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/delay.h>
 #include <linux/err.h>
@@ -52,24 +53,24 @@ MODULE_LICENSE("GPL");
 
 int ap_domain_index = -1;	/* Adjunct Processor Domain Index */
 static DEFINE_SPINLOCK(ap_domain_lock);
-module_param_named(domain, ap_domain_index, int, 0440);
+module_param_named(domain, ap_domain_index, int, 0444);
 MODULE_PARM_DESC(domain, "domain index for ap devices");
 EXPORT_SYMBOL(ap_domain_index);
 
 static int ap_thread_flag;
-module_param_named(poll_thread, ap_thread_flag, int, 0440);
+module_param_named(poll_thread, ap_thread_flag, int, 0444);
 MODULE_PARM_DESC(poll_thread, "Turn on/off poll thread, default is 0 (off).");
 
 static char *apm_str;
-module_param_named(apmask, apm_str, charp, 0440);
+module_param_named(apmask, apm_str, charp, 0444);
 MODULE_PARM_DESC(apmask, "AP bus adapter mask.");
 
 static char *aqm_str;
-module_param_named(aqmask, aqm_str, charp, 0440);
+module_param_named(aqmask, aqm_str, charp, 0444);
 MODULE_PARM_DESC(aqmask, "AP bus domain mask.");
 
 static int ap_useirq = 1;
-module_param_named(useirq, ap_useirq, int, 0440);
+module_param_named(useirq, ap_useirq, int, 0444);
 MODULE_PARM_DESC(useirq, "Use interrupt if available, default is 1 (on).");
 
 atomic_t ap_max_msg_size = ATOMIC_INIT(AP_DEFAULT_MAX_MSG_SIZE);
@@ -130,7 +131,7 @@ debug_info_t *ap_dbf_info;
  */
 static mempool_t *ap_msg_pool;
 static unsigned int ap_msg_pool_min_items = 8;
-module_param_named(msgpool_min_items, ap_msg_pool_min_items, uint, 0440);
+module_param_named(msgpool_min_items, ap_msg_pool_min_items, uint, 0400);
 MODULE_PARM_DESC(msgpool_min_items, "AP message pool minimal items");
 
 /*
diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c
index 7a3b99f065f2..c796773fbce8 100644
--- a/drivers/s390/crypto/zcrypt_api.c
+++ b/drivers/s390/crypto/zcrypt_api.c
@@ -50,7 +50,7 @@ MODULE_DESCRIPTION("Cryptographic Coprocessor interface, " \
 MODULE_LICENSE("GPL");
 
 unsigned int zcrypt_mempool_threshold = 5;
-module_param_named(mempool_threshold, zcrypt_mempool_threshold, uint, 0440);
+module_param_named(mempool_threshold, zcrypt_mempool_threshold, uint, 0400);
 MODULE_PARM_DESC(mempool_threshold, "CCA and EP11 request/reply mempool minimal items (min: 1)");
 
 /*
diff --git a/drivers/s390/crypto/zcrypt_cex4.c b/drivers/s390/crypto/zcrypt_cex4.c
index 6ba7fbddd3f7..e9a984903bff 100644
--- a/drivers/s390/crypto/zcrypt_cex4.c
+++ b/drivers/s390/crypto/zcrypt_cex4.c
@@ -6,6 +6,7 @@
 
 #include <linux/module.h>
 #include <linux/slab.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/err.h>
 #include <linux/atomic.h>
diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c
index 4904b831c0a7..1653cc668dcf 100644
--- a/drivers/s390/virtio/virtio_ccw.c
+++ b/drivers/s390/virtio/virtio_ccw.c
@@ -8,6 +8,7 @@
  */
 
 #include <linux/kernel_stat.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/memblock.h>
 #include <linux/err.h>
diff --git a/drivers/scsi/aacraid/rx.c b/drivers/scsi/aacraid/rx.c
index e06ff83b69ce..ba9f3256c258 100644
--- a/drivers/scsi/aacraid/rx.c
+++ b/drivers/scsi/aacraid/rx.c
@@ -17,6 +17,7 @@
  */
 
 #include <linux/kernel.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/types.h>
 #include <linux/pci.h>
diff --git a/drivers/scsi/ips.c b/drivers/scsi/ips.c
index 3393a288fd23..40af961382dc 100644
--- a/drivers/scsi/ips.c
+++ b/drivers/scsi/ips.c
@@ -167,6 +167,7 @@
 #include <linux/stddef.h>
 #include <linux/string.h>
 #include <linux/errno.h>
+#include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/ioport.h>
 #include <linux/slab.h>
diff --git a/drivers/scsi/libsas/sas_scsi_host.c b/drivers/scsi/libsas/sas_scsi_host.c
index ffa5b49aaf08..da02457f0b09 100644
--- a/drivers/scsi/libsas/sas_scsi_host.c
+++ b/drivers/scsi/libsas/sas_scsi_host.c
@@ -10,6 +10,7 @@
 #include <linux/firmware.h>
 #include <linux/export.h>
 #include <linux/ctype.h>
+#include <linux/hex.h>
 #include <linux/kernel.h>
 
 #include "sas_internal.h"
diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c
index 33582d48ec09..4af5c069635a 100644
--- a/drivers/scsi/lpfc/lpfc_attr.c
+++ b/drivers/scsi/lpfc/lpfc_attr.c
@@ -6979,6 +6979,42 @@ lpfc_reset_stats(struct Scsi_Host *shost)
 	return;
 }
 
+/**
+ * lpfc_get_enc_info - Return encryption information about the session for
+ *                     a given remote port.
+ * @rport: ptr to fc_rport from scsi transport fc
+ *
+ * Given an rport object, iterate through the fc_nodes list to find node
+ * corresponding with rport. Pass the encryption information from the node to
+ * rport's encryption attribute for reporting to upper layers. Information is
+ * passed through nlp_enc_info struct which contains encryption status.
+ *
+ * Returns:
+ * - Address of rport's fc_encryption_info struct
+ * - NULL when not found
+ **/
+static struct fc_encryption_info *
+lpfc_get_enc_info(struct fc_rport *rport)
+{
+	struct Scsi_Host *shost = rport_to_shost(rport);
+	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
+	struct fc_encryption_info *ef = NULL;
+	struct lpfc_nodelist *ndlp, *next_ndlp;
+	unsigned long iflags;
+
+	spin_lock_irqsave(&vport->fc_nodes_list_lock, iflags);
+	list_for_each_entry_safe(ndlp, next_ndlp, &vport->fc_nodes, nlp_listp) {
+		if (ndlp->rport && ndlp->rport == rport) {
+			ef = &rport->enc_info;
+			ef->status = ndlp->nlp_enc_info.status;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&vport->fc_nodes_list_lock, iflags);
+	return ef;
+}
+
+
 /*
  * The LPFC driver treats linkdown handling as target loss events so there
  * are no sysfs handlers for link_down_tmo.
@@ -7196,6 +7232,8 @@ struct fc_function_template lpfc_transport_functions = {
 	.get_fc_host_stats = lpfc_get_stats,
 	.reset_fc_host_stats = lpfc_reset_stats,
 
+	.get_fc_rport_enc_info = lpfc_get_enc_info,
+
 	.dd_fcrport_size = sizeof(struct lpfc_rport_data),
 	.show_rport_maxframe_size = 1,
 	.show_rport_supported_classes = 1,
@@ -7265,6 +7303,8 @@ struct fc_function_template lpfc_vport_transport_functions = {
 	.get_fc_host_stats = lpfc_get_stats,
 	.reset_fc_host_stats = lpfc_reset_stats,
 
+	.get_fc_rport_enc_info = lpfc_get_enc_info,
+
 	.dd_fcrport_size = sizeof(struct lpfc_rport_data),
 	.show_rport_maxframe_size = 1,
 	.show_rport_supported_classes = 1,
diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c
index 92b5b2dbe847..646f88c776f5 100644
--- a/drivers/scsi/lpfc/lpfc_debugfs.c
+++ b/drivers/scsi/lpfc/lpfc_debugfs.c
@@ -872,6 +872,13 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
 				 ndlp->nlp_rpi);
 		len += scnprintf(buf+len, size-len, "flag:x%08lx ",
 				 ndlp->nlp_flag);
+		if (ndlp->nlp_enc_info.status) {
+			len += scnprintf(buf + len,
+					 size - len, "ENCRYPTED");
+			len += scnprintf(buf + len, size - len,
+					 ndlp->nlp_enc_info.level
+					 ? "(CNSA2.0) " : "(CNSA1.0) ");
+		}
 		if (!ndlp->nlp_type)
 			len += scnprintf(buf+len, size-len, "UNKNOWN_TYPE ");
 		if (ndlp->nlp_type & NLP_FC_NODE)
diff --git a/drivers/scsi/lpfc/lpfc_disc.h b/drivers/scsi/lpfc/lpfc_disc.h
index 51cb8571c049..de0adeecf668 100644
--- a/drivers/scsi/lpfc/lpfc_disc.h
+++ b/drivers/scsi/lpfc/lpfc_disc.h
@@ -77,6 +77,11 @@ struct lpfc_node_rrqs {
 	unsigned long xri_bitmap[XRI_BITMAP_ULONGS];
 };
 
+struct lpfc_enc_info {
+	u8 status; /* encryption status for session */
+	u8 level; /* CNSA encryption level */
+};
+
 enum lpfc_fc4_xpt_flags {
 	NLP_XPT_REGD		= 0x1,
 	SCSI_XPT_REGD		= 0x2,
@@ -138,6 +143,8 @@ struct lpfc_nodelist {
 	uint8_t		vmid_support;		/* destination VMID support */
 #define NLP_NVME_NSLER     0x1			/* NVME NSLER device */
 
+	struct lpfc_enc_info nlp_enc_info; /* Encryption information struct */
+
 	struct timer_list   nlp_delayfunc;	/* Used for delayed ELS cmds */
 	struct lpfc_hba *phba;
 	struct fc_rport *rport;		/* scsi_transport_fc port structure */
diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c
index 02b6d31b9ad9..32da3c23c7f4 100644
--- a/drivers/scsi/lpfc/lpfc_els.c
+++ b/drivers/scsi/lpfc/lpfc_els.c
@@ -2014,6 +2014,58 @@ lpfc_cmpl_els_rrq(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 	lpfc_nlp_put(ndlp);
 	return;
 }
+
+/**
+ * lpfc_check_encryption - Reports an ndlp's encryption information
+ * @phba: pointer to lpfc hba data structure.
+ * @ndlp: pointer to a node-list data structure.
+ * @cmdiocb: pointer to lpfc command iocbq data structure.
+ * @rspiocb: pointer to lpfc response iocbq data structure.
+ *
+ * This routine is called in the completion callback function for issuing
+ * or receiving a Port Login (PLOGI) command. In a PLOGI completion, if FEDIF
+ * is supported, encryption information will be provided in completion status
+ * data. If @phba supports FEDIF, a log message containing encryption
+ * information will be logged. Encryption status is also saved for encryption
+ * reporting with upper layer through the rport encryption attribute.
+ **/
+static void
+lpfc_check_encryption(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
+		      struct lpfc_iocbq *cmdiocb, struct lpfc_iocbq *rspiocb)
+{
+	struct lpfc_vport *vport = cmdiocb->vport;
+	u32 did = ndlp->nlp_DID;
+	struct lpfc_enc_info *nlp_enc_info = &ndlp->nlp_enc_info;
+	char enc_status[FC_RPORT_ENCRYPTION_STATUS_MAX_LEN] = {0};
+	char enc_level[8] = "N/A";
+	u8 encryption;
+
+	if (phba->sli4_hba.encryption_support &&
+	    ((did & Fabric_DID_MASK) != Fabric_DID_MASK)) {
+		encryption = bf_get(lpfc_wcqe_c_enc,
+				    &rspiocb->wcqe_cmpl);
+		nlp_enc_info->status = encryption;
+
+		strscpy(enc_status, encryption ? "Encrypted" : "Unencrypted",
+			sizeof(enc_status));
+
+		if (encryption) {
+			nlp_enc_info->level = bf_get(lpfc_wcqe_c_enc_lvl,
+						     &rspiocb->wcqe_cmpl);
+			strscpy(enc_level, nlp_enc_info->level ? "CNSA2.0" :
+								 "CNSA1.0",
+				sizeof(enc_level));
+		}
+
+		lpfc_printf_vlog(vport, KERN_INFO, LOG_ENCRYPTION,
+				 "0924 DID:x%06x %s Session "
+				 "Established, Encryption Level:%s "
+				 "rpi:x%x\n",
+				 ndlp->nlp_DID, enc_status, enc_level,
+				 ndlp->nlp_rpi);
+	}
+}
+
 /**
  * lpfc_cmpl_els_plogi - Completion callback function for plogi
  * @phba: pointer to lpfc hba data structure.
@@ -2153,6 +2205,8 @@ lpfc_cmpl_els_plogi(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 			goto out;
 		ndlp = lpfc_plogi_confirm_nport(phba, prsp->virt, ndlp);
 
+		lpfc_check_encryption(phba, ndlp, cmdiocb, rspiocb);
+
 		sp = (struct serv_parm *)((u8 *)prsp->virt +
 					  sizeof(u32));
 
@@ -5407,6 +5461,9 @@ lpfc_cmpl_els_rsp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 		goto out;
 	}
 
+	if (!ulp_status && test_bit(NLP_RCV_PLOGI, &ndlp->nlp_flag))
+		lpfc_check_encryption(phba, ndlp, cmdiocb, rspiocb);
+
 	lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_ELS_RSP,
 		"ELS rsp cmpl:    status:x%x/x%x did:x%x",
 		ulp_status, ulp_word4, did);
diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c
index bb803f32bc1b..1aeebdc08073 100644
--- a/drivers/scsi/lpfc/lpfc_hbadisc.c
+++ b/drivers/scsi/lpfc/lpfc_hbadisc.c
@@ -5340,6 +5340,7 @@ lpfc_unreg_rpi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 		clear_bit(NLP_NPR_ADISC, &ndlp->nlp_flag);
 		if (acc_plogi)
 			clear_bit(NLP_LOGO_ACC, &ndlp->nlp_flag);
+		memset(&ndlp->nlp_enc_info, 0, sizeof(ndlp->nlp_enc_info));
 		return 1;
 	}
 	clear_bit(NLP_LOGO_ACC, &ndlp->nlp_flag);
diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h
index a7f7ed86d2b0..c000474c3066 100644
--- a/drivers/scsi/lpfc/lpfc_hw4.h
+++ b/drivers/scsi/lpfc/lpfc_hw4.h
@@ -437,6 +437,12 @@ struct lpfc_wcqe_complete {
 #define lpfc_wcqe_c_cmf_bw_MASK		0x0FFFFFFF
 #define lpfc_wcqe_c_cmf_bw_WORD		total_data_placed
 	uint32_t parameter;
+#define lpfc_wcqe_c_enc_SHIFT		31
+#define lpfc_wcqe_c_enc_MASK		0x00000001
+#define lpfc_wcqe_c_enc_WORD		parameter
+#define lpfc_wcqe_c_enc_lvl_SHIFT	30
+#define lpfc_wcqe_c_enc_lvl_MASK	0x00000001
+#define lpfc_wcqe_c_enc_lvl_WORD	parameter
 #define lpfc_wcqe_c_bg_edir_SHIFT	5
 #define lpfc_wcqe_c_bg_edir_MASK	0x00000001
 #define lpfc_wcqe_c_bg_edir_WORD	parameter
@@ -2942,7 +2948,10 @@ struct lpfc_mbx_read_config {
 #define lpfc_mbx_rd_conf_topology_SHIFT		24
 #define lpfc_mbx_rd_conf_topology_MASK		0x000000FF
 #define lpfc_mbx_rd_conf_topology_WORD		word2
-	uint32_t rsvd_3;
+	uint32_t word3;
+#define lpfc_mbx_rd_conf_fedif_SHIFT		6
+#define lpfc_mbx_rd_conf_fedif_MASK		0x00000001
+#define lpfc_mbx_rd_conf_fedif_WORD		word3
 	uint32_t word4;
 #define lpfc_mbx_rd_conf_e_d_tov_SHIFT		0
 #define lpfc_mbx_rd_conf_e_d_tov_MASK		0x0000FFFF
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index b1460b16dd91..a116a16c4a6f 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -9999,6 +9999,11 @@ lpfc_sli4_read_config(struct lpfc_hba *phba)
 				(phba->sli4_hba.max_cfg_param.max_vpi - 1) : 0;
 		phba->max_vports = phba->max_vpi;
 
+		if (bf_get(lpfc_mbx_rd_conf_fedif, rd_config))
+			phba->sli4_hba.encryption_support = true;
+		else
+			phba->sli4_hba.encryption_support = false;
+
 		/* Next decide on FPIN or Signal E2E CGN support
 		 * For congestion alarms and warnings valid combination are:
 		 * 1. FPIN alarms / FPIN warnings
diff --git a/drivers/scsi/lpfc/lpfc_logmsg.h b/drivers/scsi/lpfc/lpfc_logmsg.h
index 59bd2bafc73f..e00d101d548c 100644
--- a/drivers/scsi/lpfc/lpfc_logmsg.h
+++ b/drivers/scsi/lpfc/lpfc_logmsg.h
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2023 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2025 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2004-2009 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -47,6 +47,7 @@
 #define LOG_RSVD1	0x01000000	/* Reserved */
 #define LOG_RSVD2	0x02000000	/* Reserved */
 #define LOG_CGN_MGMT    0x04000000	/* Congestion Mgmt events */
+#define LOG_ENCRYPTION  0x40000000      /* EDIF Encryption events. */
 #define LOG_TRACE_EVENT 0x80000000	/* Dmp the DBG log on this err */
 #define LOG_ALL_MSG	0x7fffffff	/* LOG all messages */
 
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c
index 73d77cfab5f8..734af3d039f8 100644
--- a/drivers/scsi/lpfc/lpfc_sli.c
+++ b/drivers/scsi/lpfc/lpfc_sli.c
@@ -20432,62 +20432,36 @@ lpfc_check_next_fcf_pri_level(struct lpfc_hba *phba)
 uint16_t
 lpfc_sli4_fcf_rr_next_index_get(struct lpfc_hba *phba)
 {
-	uint16_t next_fcf_index;
+	uint16_t next;
 
-initial_priority:
-	/* Search start from next bit of currently registered FCF index */
-	next_fcf_index = phba->fcf.current_rec.fcf_indx;
-
-next_priority:
-	/* Determine the next fcf index to check */
-	next_fcf_index = (next_fcf_index + 1) % LPFC_SLI4_FCF_TBL_INDX_MAX;
-	next_fcf_index = find_next_bit(phba->fcf.fcf_rr_bmask,
-				       LPFC_SLI4_FCF_TBL_INDX_MAX,
-				       next_fcf_index);
+	do {
+		for_each_set_bit_wrap(next, phba->fcf.fcf_rr_bmask,
+				LPFC_SLI4_FCF_TBL_INDX_MAX, phba->fcf.current_rec.fcf_indx) {
+			if (next == phba->fcf.current_rec.fcf_indx)
+				continue;
 
-	/* Wrap around condition on phba->fcf.fcf_rr_bmask */
-	if (next_fcf_index >= LPFC_SLI4_FCF_TBL_INDX_MAX) {
-		/*
-		 * If we have wrapped then we need to clear the bits that
-		 * have been tested so that we can detect when we should
-		 * change the priority level.
-		 */
-		next_fcf_index = find_first_bit(phba->fcf.fcf_rr_bmask,
-					       LPFC_SLI4_FCF_TBL_INDX_MAX);
-	}
+			if (!(phba->fcf.fcf_pri[next].fcf_rec.flag & LPFC_FCF_FLOGI_FAILED)) {
+				lpfc_printf_log(phba, KERN_INFO, LOG_FIP,
+					"2845 Get next roundrobin failover FCF (x%x)\n", next);
+				return next;
+			}
 
+			if (list_is_singular(&phba->fcf.fcf_pri_list))
+				return LPFC_FCOE_FCF_NEXT_NONE;
+		}
 
-	/* Check roundrobin failover list empty condition */
-	if (next_fcf_index >= LPFC_SLI4_FCF_TBL_INDX_MAX ||
-		next_fcf_index == phba->fcf.current_rec.fcf_indx) {
 		/*
 		 * If next fcf index is not found check if there are lower
 		 * Priority level fcf's in the fcf_priority list.
 		 * Set up the rr_bmask with all of the avaiable fcf bits
 		 * at that level and continue the selection process.
 		 */
-		if (lpfc_check_next_fcf_pri_level(phba))
-			goto initial_priority;
-		lpfc_printf_log(phba, KERN_WARNING, LOG_FIP,
-				"2844 No roundrobin failover FCF available\n");
-
-		return LPFC_FCOE_FCF_NEXT_NONE;
-	}
-
-	if (next_fcf_index < LPFC_SLI4_FCF_TBL_INDX_MAX &&
-		phba->fcf.fcf_pri[next_fcf_index].fcf_rec.flag &
-		LPFC_FCF_FLOGI_FAILED) {
-		if (list_is_singular(&phba->fcf.fcf_pri_list))
-			return LPFC_FCOE_FCF_NEXT_NONE;
+	} while (lpfc_check_next_fcf_pri_level(phba));
 
-		goto next_priority;
-	}
-
-	lpfc_printf_log(phba, KERN_INFO, LOG_FIP,
-			"2845 Get next roundrobin failover FCF (x%x)\n",
-			next_fcf_index);
+	lpfc_printf_log(phba, KERN_WARNING, LOG_FIP,
+			"2844 No roundrobin failover FCF available\n");
 
-	return next_fcf_index;
+	return LPFC_FCOE_FCF_NEXT_NONE;
 }
 
 /**
diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h
index fd6dab157887..ee58383492b2 100644
--- a/drivers/scsi/lpfc/lpfc_sli4.h
+++ b/drivers/scsi/lpfc/lpfc_sli4.h
@@ -888,6 +888,10 @@ struct lpfc_sli4_hba {
 #define LPFC_FP_EQ_MAX_INTR_SEC         10000
 
 	uint32_t intr_enable;
+
+	 /* Indicates whether SLI Port supports FEDIF */
+	bool encryption_support;
+
 	struct lpfc_bmbx bmbx;
 	struct lpfc_max_cfg_param max_cfg_param;
 	uint16_t extents_in_use; /* must allocate resource extents. */
diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h
index f3dada5bf7c1..c4ca8bf5843a 100644
--- a/drivers/scsi/lpfc/lpfc_version.h
+++ b/drivers/scsi/lpfc/lpfc_version.h
@@ -20,7 +20,7 @@
  * included with this package.                                     *
  *******************************************************************/
 
-#define LPFC_DRIVER_VERSION "14.4.0.12"
+#define LPFC_DRIVER_VERSION "14.4.0.13"
 #define LPFC_DRIVER_NAME		"lpfc"
 
 /* Used for SLI 2/3 */
diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h b/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h
index b42933fcd423..6561f98c3cb2 100644
--- a/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h
+++ b/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h
@@ -166,6 +166,7 @@ struct mpi3_ioc_facts_data {
 #define MPI3_IOCFACTS_FLAGS_SIGNED_NVDATA_REQUIRED            (0x00010000)
 #define MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_MASK            (0x0000ff00)
 #define MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_SHIFT           (8)
+#define MPI3_IOCFACTS_FLAGS_MAX_REQ_PER_REPLY_QUEUE_LIMIT     (0x00000040)
 #define MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_MASK          (0x00000030)
 #define MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_SHIFT		(4)
 #define MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_NOT_STARTED   (0x00000000)
diff --git a/drivers/scsi/mpi3mr/mpi3mr_fw.c b/drivers/scsi/mpi3mr/mpi3mr_fw.c
index 8fe6e0bf342e..8c4bb7169a87 100644
--- a/drivers/scsi/mpi3mr/mpi3mr_fw.c
+++ b/drivers/scsi/mpi3mr/mpi3mr_fw.c
@@ -3158,6 +3158,8 @@ static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
 	mrioc->facts.dma_mask = (facts_flags &
 	    MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_MASK) >>
 	    MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_SHIFT;
+	mrioc->facts.max_req_limit = (facts_flags &
+			MPI3_IOCFACTS_FLAGS_MAX_REQ_PER_REPLY_QUEUE_LIMIT);
 	mrioc->facts.protocol_flags = facts_data->protocol_flags;
 	mrioc->facts.mpi_version = le32_to_cpu(facts_data->mpi_version.word);
 	mrioc->facts.max_reqs = le16_to_cpu(facts_data->max_outstanding_requests);
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c
index 0d652db8fe24..e4e22cb0e277 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.c
@@ -843,7 +843,7 @@ mpt3sas_base_start_watchdog(struct MPT3SAS_ADAPTER *ioc)
 	/* initialize fault polling */
 
 	INIT_DELAYED_WORK(&ioc->fault_reset_work, _base_fault_reset_work);
-	snprintf(ioc->fault_reset_work_q_name,
+	scnprintf(ioc->fault_reset_work_q_name,
 	    sizeof(ioc->fault_reset_work_q_name), "poll_%s%d_status",
 	    ioc->driver_name, ioc->id);
 	ioc->fault_reset_work_q = alloc_ordered_workqueue(
@@ -1564,6 +1564,8 @@ _base_get_cb_idx(struct MPT3SAS_ADAPTER *ioc, u16 smid)
 	int i;
 	u16 ctl_smid = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT + 1;
 	u8 cb_idx = 0xFF;
+	u16 discovery_smid =
+	    ioc->shost->can_queue + INTERNAL_SCSIIO_FOR_DISCOVERY;
 
 	if (smid < ioc->hi_priority_smid) {
 		struct scsiio_tracker *st;
@@ -1572,8 +1574,10 @@ _base_get_cb_idx(struct MPT3SAS_ADAPTER *ioc, u16 smid)
 			st = _get_st_from_smid(ioc, smid);
 			if (st)
 				cb_idx = st->cb_idx;
-		} else if (smid == ctl_smid)
+		} else if (smid < discovery_smid)
 			cb_idx = ioc->ctl_cb_idx;
+		else
+			cb_idx = ioc->scsih_cb_idx;
 	} else if (smid < ioc->internal_smid) {
 		i = smid - ioc->hi_priority_smid;
 		cb_idx = ioc->hpr_lookup[i].cb_idx;
@@ -3174,7 +3178,7 @@ _base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index)
 
 	if (index >= ioc->iopoll_q_start_index) {
 		qid = index - ioc->iopoll_q_start_index;
-		snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d-mq-poll%d",
+		scnprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d-mq-poll%d",
 		    ioc->driver_name, ioc->id, qid);
 		reply_q->is_iouring_poll_q = 1;
 		ioc->io_uring_poll_queues[qid].reply_q = reply_q;
@@ -3183,10 +3187,10 @@ _base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index)
 
 
 	if (ioc->msix_enable)
-		snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d-msix%d",
+		scnprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d-msix%d",
 		    ioc->driver_name, ioc->id, index);
 	else
-		snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d",
+		scnprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d",
 		    ioc->driver_name, ioc->id);
 	r = request_irq(pci_irq_vector(pdev, index), _base_interrupt,
 			IRQF_SHARED, reply_q->name, reply_q);
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h
index e6a6f21d309b..de37fa5ac073 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.h
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.h
@@ -147,6 +147,7 @@
 #define INTERNAL_CMDS_COUNT		10	/* reserved cmds */
 /* reserved for issuing internally framed scsi io cmds */
 #define INTERNAL_SCSIIO_CMDS_COUNT	3
+#define INTERNAL_SCSIIO_FOR_DISCOVERY	2
 
 #define MPI3_HIM_MASK			0xFFFFFFFF /* mask every bit*/
 
@@ -480,6 +481,7 @@ struct MPT3SAS_DEVICE {
 	u32	flags;
 	u8	configured_lun;
 	u8	block;
+	u8	deleted;
 	u8	tlr_snoop_check;
 	u8	ignore_delay_remove;
 	/* Iopriority Command Handling */
@@ -577,7 +579,9 @@ struct _sas_device {
 	u8	chassis_slot;
 	u8	is_chassis_slot_valid;
 	u8	connector_name[5];
+	u8	ssd_device;
 	struct kref refcount;
+
 	u8	port_type;
 	struct hba_port *port;
 	struct sas_rphy *rphy;
diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
index 7092d0debef3..ac69a5abe2e2 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
@@ -61,6 +61,8 @@
 
 #define PCIE_CHANNEL 2
 
+#define MPT3_MAX_LUNS (255)
+
 /* forward proto's */
 static void _scsih_expander_node_remove(struct MPT3SAS_ADAPTER *ioc,
 	struct _sas_node *sas_expander);
@@ -70,13 +72,24 @@ static void _scsih_remove_device(struct MPT3SAS_ADAPTER *ioc,
 	struct _sas_device *sas_device);
 static int _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle,
 	u8 retry_count, u8 is_pd);
-static int _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle);
+static int _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle,
+	u8 retry_count);
 static void _scsih_pcie_device_remove_from_sml(struct MPT3SAS_ADAPTER *ioc,
 	struct _pcie_device *pcie_device);
 static void
 _scsih_pcie_check_device(struct MPT3SAS_ADAPTER *ioc, u16 handle);
 static u8 _scsih_check_for_pending_tm(struct MPT3SAS_ADAPTER *ioc, u16 smid);
 static void _scsih_complete_devices_scanning(struct MPT3SAS_ADAPTER *ioc);
+static enum device_responsive_state
+_scsih_wait_for_target_to_become_ready(struct MPT3SAS_ADAPTER *ioc, u16 handle,
+	u8 retry_count, u8 is_pd, u8 tr_timeout, u8 tr_method);
+static enum device_responsive_state
+_scsih_ata_pass_thru_idd(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 *is_ssd_device,
+	u8 tr_timeout, u8 tr_method);
+static enum device_responsive_state
+_scsih_wait_for_device_to_become_ready(struct MPT3SAS_ADAPTER *ioc, u16 handle,
+	u8 retry_count, u8 is_pd, int lun, u8 tr_timeout, u8 tr_method);
+static void _firmware_event_work_delayed(struct work_struct *work);
 
 /* global parameters */
 LIST_HEAD(mpt3sas_ioc_list);
@@ -159,6 +172,15 @@ module_param(enable_sdev_max_qd, bool, 0444);
 MODULE_PARM_DESC(enable_sdev_max_qd,
 	"Enable sdev max qd as can_queue, def=disabled(0)");
 
+/*
+ * permit overriding the SCSI command issuing capability of
+ * the driver to bring the drive to READY state
+ */
+static int issue_scsi_cmd_to_bringup_drive = 1;
+module_param(issue_scsi_cmd_to_bringup_drive, int, 0444);
+MODULE_PARM_DESC(issue_scsi_cmd_to_bringup_drive, "allow host driver to\n"
+	"issue SCSI commands to bring the drive to READY state, default=1 ");
+
 static int multipath_on_hba = -1;
 module_param(multipath_on_hba, int, 0);
 MODULE_PARM_DESC(multipath_on_hba,
@@ -173,10 +195,33 @@ module_param(host_tagset_enable, int, 0444);
 MODULE_PARM_DESC(host_tagset_enable,
 	"Shared host tagset enable/disable Default: enable(1)");
 
+static int command_retry_count = 144;
+module_param(command_retry_count, int, 0444);
+MODULE_PARM_DESC(command_retry_count, "Device discovery TUR command retry\n"
+	"count: (default=144)");
+
 /* raid transport support */
 static struct raid_template *mpt3sas_raid_template;
 static struct raid_template *mpt2sas_raid_template;
 
+/**
+ * enum device_responsive_state - responsive state
+ * @DEVICE_READY: device is ready to be added
+ * @DEVICE_RETRY: device can be retried later
+ * @DEVICE_RETRY_UA: retry unit attentions
+ * @DEVICE_START_UNIT: requires start unit
+ * @DEVICE_STOP_UNIT: requires stop unit
+ * @DEVICE_ERROR: device reported some fatal error
+ *
+ */
+enum device_responsive_state {
+	DEVICE_READY,
+	DEVICE_RETRY,
+	DEVICE_RETRY_UA,
+	DEVICE_START_UNIT,
+	DEVICE_STOP_UNIT,
+	DEVICE_ERROR,
+};
 
 /**
  * struct sense_info - common structure for obtaining sense keys
@@ -205,6 +250,9 @@ struct sense_info {
 
 /**
  * struct fw_event_work - firmware event struct
+ * @retries: retry count for processing the event
+ * @delayed_work_active: flag indicating if delayed work is active
+ * @delayed_work: delayed work item for deferred event handling
  * @list: link list framework
  * @work: work object (ioc->fault_reset_work_q)
  * @ioc: per adapter object
@@ -219,6 +267,9 @@ struct sense_info {
  * This object stored on ioc->fw_event_list.
  */
 struct fw_event_work {
+	u8			*retries;
+	u8                      delayed_work_active;
+	struct delayed_work     delayed_work;
 	struct list_head	list;
 	struct work_struct	work;
 
@@ -230,11 +281,16 @@ struct fw_event_work {
 	u16			event;
 	struct kref		refcount;
 	char			event_data[] __aligned(4);
+
 };
 
 static void fw_event_work_free(struct kref *r)
 {
-	kfree(container_of(r, struct fw_event_work, refcount));
+	struct fw_event_work *fw_work;
+
+	fw_work = container_of(r, struct fw_event_work, refcount);
+	kfree(fw_work->retries);
+	kfree(fw_work);
 }
 
 static void fw_event_work_get(struct fw_event_work *fw_work)
@@ -955,6 +1011,7 @@ _scsih_sas_device_remove(struct MPT3SAS_ADAPTER *ioc,
 		sas_device_put(sas_device);
 	}
 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
+
 }
 
 /**
@@ -2528,6 +2585,8 @@ scsih_sdev_configure(struct scsi_device *sdev, struct queue_limits *lim)
 	char *r_level = "";
 	u16 handle, volume_handle = 0;
 	u64 volume_wwid = 0;
+	enum device_responsive_state retval;
+	u8 count = 0;
 
 	qdepth = 1;
 	sas_device_priv_data = sdev->hostdata;
@@ -2686,6 +2745,7 @@ scsih_sdev_configure(struct scsi_device *sdev, struct queue_limits *lim)
 
 		pcie_device_put(pcie_device);
 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
+
 		mpt3sas_scsih_change_queue_depth(sdev, qdepth);
 		lim->virt_boundary_mask = ioc->page_size - 1;
 		return 0;
@@ -2737,9 +2797,16 @@ scsih_sdev_configure(struct scsi_device *sdev, struct queue_limits *lim)
 	sas_device_put(sas_device);
 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
 
-	if (!ssp_target)
+	if (!ssp_target) {
 		_scsih_display_sata_capabilities(ioc, handle, sdev);
 
+		do {
+			retval = _scsih_ata_pass_thru_idd(ioc, handle,
+				    &sas_device->ssd_device, 30, 0);
+		} while ((retval == DEVICE_RETRY || retval == DEVICE_RETRY_UA)
+			&& count++ < 3);
+	}
+
 
 	mpt3sas_scsih_change_queue_depth(sdev, qdepth);
 
@@ -3595,6 +3662,37 @@ _scsih_fw_event_del_from_list(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work
 	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
 }
 
+/**
+ * _scsih_fw_event_requeue - requeue an event
+ * @ioc: per adapter object
+ * @fw_event: object describing the event
+ * @delay: time in milliseconds to wait before retrying the event
+ *
+ * Context: This function will acquire ioc->fw_event_lock.
+ *
+ * Return nothing.
+ */
+static void
+_scsih_fw_event_requeue(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work
+	*fw_event, unsigned long delay)
+{
+	unsigned long flags;
+
+	if (ioc->firmware_event_thread == NULL)
+		return;
+
+	spin_lock_irqsave(&ioc->fw_event_lock, flags);
+	fw_event_work_get(fw_event);
+	list_add_tail(&fw_event->list, &ioc->fw_event_list);
+	if (!fw_event->delayed_work_active) {
+		fw_event->delayed_work_active = 1;
+		INIT_DELAYED_WORK(&fw_event->delayed_work,
+		    _firmware_event_work_delayed);
+	}
+	queue_delayed_work(ioc->firmware_event_thread, &fw_event->delayed_work,
+	    msecs_to_jiffies(delay));
+	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
+}
 
  /**
  * mpt3sas_send_trigger_data_event - send event for processing trigger data
@@ -3825,29 +3923,235 @@ _scsih_internal_device_unblock(struct scsi_device *sdev,
 /**
  * _scsih_ublock_io_all_device - unblock every device
  * @ioc: per adapter object
+ * @no_turs: flag to disable TEST UNIT READY checks during device unblocking
  *
  * change the device state from block to running
  */
 static void
-_scsih_ublock_io_all_device(struct MPT3SAS_ADAPTER *ioc)
+_scsih_ublock_io_all_device(struct MPT3SAS_ADAPTER *ioc, u8 no_turs)
 {
 	struct MPT3SAS_DEVICE *sas_device_priv_data;
 	struct scsi_device *sdev;
+	struct MPT3SAS_TARGET *sas_target;
+	enum device_responsive_state rc;
+	struct _sas_device *sas_device = NULL;
+	struct _pcie_device *pcie_device = NULL;
+	int count = 0;
+	u8 tr_method = 0;
+	u8 tr_timeout = 30;
+
 
 	shost_for_each_device(sdev, ioc->shost) {
 		sas_device_priv_data = sdev->hostdata;
 		if (!sas_device_priv_data)
 			continue;
+
+		sas_target = sas_device_priv_data->sas_target;
+		if (!sas_target || sas_target->deleted)
+			continue;
+
 		if (!sas_device_priv_data->block)
 			continue;
 
-		dewtprintk(ioc, sdev_printk(KERN_INFO, sdev,
-			"device_running, handle(0x%04x)\n",
-		    sas_device_priv_data->sas_target->handle));
+		if ((no_turs) || (!issue_scsi_cmd_to_bringup_drive)) {
+			sdev_printk(KERN_WARNING, sdev, "device_unblocked handle(0x%04x)\n",
+				sas_device_priv_data->sas_target->handle);
+			_scsih_internal_device_unblock(sdev, sas_device_priv_data);
+			continue;
+		}
+
+		do {
+			pcie_device = mpt3sas_get_pdev_by_handle(ioc, sas_target->handle);
+			if (pcie_device && (!ioc->tm_custom_handling) &&
+				(!(mpt3sas_scsih_is_pcie_scsi_device(pcie_device->device_info)))) {
+				tr_timeout = pcie_device->reset_timeout;
+				tr_method = MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
+			}
+			rc = _scsih_wait_for_device_to_become_ready(ioc,
+			    sas_target->handle, 0, (sas_target->flags &
+			    MPT_TARGET_FLAGS_RAID_COMPONENT), sdev->lun, tr_timeout, tr_method);
+			if (rc == DEVICE_RETRY || rc == DEVICE_START_UNIT ||
+			    rc == DEVICE_STOP_UNIT || rc == DEVICE_RETRY_UA)
+				ssleep(1);
+			if (pcie_device)
+				pcie_device_put(pcie_device);
+		} while ((rc == DEVICE_RETRY || rc == DEVICE_START_UNIT ||
+		    rc == DEVICE_STOP_UNIT || rc == DEVICE_RETRY_UA)
+			&& count++ < command_retry_count);
+		sas_device_priv_data->block = 0;
+		if (rc != DEVICE_READY)
+			sas_device_priv_data->deleted = 1;
+
 		_scsih_internal_device_unblock(sdev, sas_device_priv_data);
+
+		if (rc != DEVICE_READY) {
+			sdev_printk(KERN_WARNING, sdev, "%s: device_offlined,\n"
+			    "handle(0x%04x)\n",
+			    __func__, sas_device_priv_data->sas_target->handle);
+			scsi_device_set_state(sdev, SDEV_OFFLINE);
+			sas_device = mpt3sas_get_sdev_by_addr(ioc,
+					sas_device_priv_data->sas_target->sas_address,
+					sas_device_priv_data->sas_target->port);
+			if (sas_device) {
+				_scsih_display_enclosure_chassis_info(NULL, sas_device, sdev, NULL);
+				sas_device_put(sas_device);
+			} else {
+				pcie_device = mpt3sas_get_pdev_by_wwid(ioc,
+						    sas_device_priv_data->sas_target->sas_address);
+				if (pcie_device) {
+					if (pcie_device->enclosure_handle != 0)
+						sdev_printk(KERN_INFO, sdev, "enclosure logical id\n"
+						    "(0x%016llx), slot(%d)\n", (unsigned long long)
+							pcie_device->enclosure_logical_id,
+							pcie_device->slot);
+					if (pcie_device->connector_name[0] != '\0')
+						sdev_printk(KERN_INFO, sdev, "enclosure level(0x%04x),\n"
+							" connector name( %s)\n",
+							pcie_device->enclosure_level,
+							pcie_device->connector_name);
+					pcie_device_put(pcie_device);
+				}
+			}
+		} else
+			sdev_printk(KERN_WARNING, sdev, "device_unblocked,\n"
+			    "handle(0x%04x)\n",
+			    sas_device_priv_data->sas_target->handle);
 	}
 }
 
+/**
+ * _scsih_ublock_io_device_wait - unblock IO for target
+ * @ioc: per adapter object
+ * @sas_address: sas address
+ * @port: hba port entry
+ *
+ * make sure device is reponsponding before unblocking
+ */
+static void
+_scsih_ublock_io_device_wait(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,
+			     struct hba_port *port)
+{
+	struct MPT3SAS_DEVICE *sas_device_priv_data;
+	struct MPT3SAS_TARGET *sas_target;
+	enum device_responsive_state rc;
+	struct scsi_device *sdev;
+	int host_reset_completion_count;
+	struct _sas_device *sas_device;
+	struct _pcie_device *pcie_device;
+	u8 tr_timeout = 30;
+	u8 tr_method = 0;
+	int count = 0;
+
+	/* moving devices from SDEV_OFFLINE to SDEV_BLOCK */
+	shost_for_each_device(sdev, ioc->shost) {
+		sas_device_priv_data = sdev->hostdata;
+		if (!sas_device_priv_data)
+			continue;
+		sas_target = sas_device_priv_data->sas_target;
+		if (!sas_target)
+			continue;
+		if (sas_target->sas_address != sas_address ||
+		    sas_target->port != port)
+			continue;
+		if (sdev->sdev_state == SDEV_OFFLINE) {
+			sas_device_priv_data->block = 1;
+			sas_device_priv_data->deleted = 0;
+			scsi_device_set_state(sdev, SDEV_RUNNING);
+			scsi_internal_device_block_nowait(sdev);
+		}
+	}
+
+	/* moving devices from SDEV_BLOCK to SDEV_RUNNING state */
+	shost_for_each_device(sdev, ioc->shost) {
+		sas_device_priv_data = sdev->hostdata;
+		if (!sas_device_priv_data)
+			continue;
+		sas_target = sas_device_priv_data->sas_target;
+		if (!sas_target)
+			continue;
+		if (sas_target->sas_address != sas_address ||
+		    sas_target->port != port)
+			continue;
+		if (!sas_device_priv_data->block)
+			continue;
+
+		do {
+			host_reset_completion_count = 0;
+			pcie_device = mpt3sas_get_pdev_by_handle(ioc, sas_target->handle);
+			if (pcie_device && (!ioc->tm_custom_handling) &&
+				(!(mpt3sas_scsih_is_pcie_scsi_device(pcie_device->device_info)))) {
+				tr_timeout = pcie_device->reset_timeout;
+				tr_method = MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
+			}
+			rc = _scsih_wait_for_device_to_become_ready(ioc,
+			      sas_target->handle, 0, (sas_target->flags &
+			      MPT_TARGET_FLAGS_RAID_COMPONENT), sdev->lun, tr_timeout, tr_method);
+			if (rc == DEVICE_RETRY || rc == DEVICE_START_UNIT ||
+			    rc == DEVICE_STOP_UNIT || rc == DEVICE_RETRY_UA) {
+				do {
+					msleep(500);
+					host_reset_completion_count++;
+				} while (rc == DEVICE_RETRY &&
+							ioc->shost_recovery);
+				if (host_reset_completion_count > 1) {
+					rc = _scsih_wait_for_device_to_become_ready(ioc,
+						sas_target->handle, 0, (sas_target->flags &
+						MPT_TARGET_FLAGS_RAID_COMPONENT), sdev->lun,
+						tr_timeout, tr_method);
+					if (rc == DEVICE_RETRY || rc == DEVICE_START_UNIT ||
+					    rc == DEVICE_STOP_UNIT || rc == DEVICE_RETRY_UA)
+						msleep(500);
+				}
+				continue;
+			}
+			if (pcie_device)
+				pcie_device_put(pcie_device);
+		} while ((rc == DEVICE_RETRY || rc == DEVICE_START_UNIT ||
+		    rc == DEVICE_STOP_UNIT || rc == DEVICE_RETRY_UA)
+			&& count++ <= command_retry_count);
+
+		sas_device_priv_data->block = 0;
+		if (rc != DEVICE_READY)
+			sas_device_priv_data->deleted = 1;
+		scsi_internal_device_unblock_nowait(sdev, SDEV_RUNNING);
+
+		if (rc != DEVICE_READY) {
+			sdev_printk(KERN_WARNING, sdev,
+			    "%s: device_offlined, handle(0x%04x)\n",
+			    __func__, sas_device_priv_data->sas_target->handle);
+
+			sas_device = mpt3sas_get_sdev_by_handle(ioc,
+				sas_device_priv_data->sas_target->handle);
+			if (sas_device) {
+				_scsih_display_enclosure_chassis_info(NULL, sas_device, sdev, NULL);
+				sas_device_put(sas_device);
+			} else {
+				pcie_device = mpt3sas_get_pdev_by_handle(ioc,
+							sas_device_priv_data->sas_target->handle);
+				if (pcie_device) {
+					if (pcie_device->enclosure_handle != 0)
+						sdev_printk(KERN_INFO, sdev,
+							"device_offlined, enclosure logical id(0x%016llx),\n"
+							" slot(%d)\n", (unsigned long long)
+							pcie_device->enclosure_logical_id,
+							pcie_device->slot);
+					if (pcie_device->connector_name[0] != '\0')
+						sdev_printk(KERN_WARNING, sdev,
+							"device_offlined, enclosure level(0x%04x),\n"
+							"connector name( %s)\n",
+							pcie_device->enclosure_level,
+							pcie_device->connector_name);
+					pcie_device_put(pcie_device);
+				}
+			}
+			scsi_device_set_state(sdev, SDEV_OFFLINE);
+		} else {
+			sdev_printk(KERN_WARNING, sdev,
+				"device_unblocked, handle(0x%04x)\n",
+				sas_device_priv_data->sas_target->handle);
+		}
+	}
+}
 
 /**
  * _scsih_ublock_io_device - prepare device to be deleted
@@ -7108,91 +7412,850 @@ _scsih_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)
 	return 1;
 }
 
-
-
-
-#define MPT3_MAX_LUNS (255)
-
-
 /**
- * _scsih_check_access_status - check access flags
+ * _scsi_send_scsi_io - send internal SCSI_IO to target
  * @ioc: per adapter object
- * @sas_address: sas address
- * @handle: sas device handle
- * @access_status: errors returned during discovery of the device
+ * @transfer_packet: packet describing the transfer
+ * @tr_timeout: Target Reset Timeout
+ * @tr_method: Target Reset Method
+ * Context: user
  *
- * Return: 0 for success, else failure
+ * Returns 0 for success, non-zero for failure.
  */
-static u8
-_scsih_check_access_status(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,
-	u16 handle, u8 access_status)
+static int
+_scsi_send_scsi_io(struct MPT3SAS_ADAPTER *ioc, struct _scsi_io_transfer
+	*transfer_packet, u8 tr_timeout, u8 tr_method)
 {
-	u8 rc = 1;
-	char *desc = NULL;
+	Mpi2SCSIIOReply_t *mpi_reply;
+	Mpi2SCSIIORequest_t *mpi_request;
+	u16 smid;
+	u8 issue_reset = 0;
+	int rc;
+	void *priv_sense;
+	u32 mpi_control;
+	void *psge;
+	dma_addr_t data_out_dma = 0;
+	dma_addr_t data_in_dma = 0;
+	size_t data_in_sz = 0;
+	size_t data_out_sz = 0;
+	u16 handle;
+	u8 retry_count = 0, host_reset_count = 0;
+	int tm_return_code;
 
-	switch (access_status) {
-	case MPI2_SAS_DEVICE0_ASTATUS_NO_ERRORS:
-	case MPI2_SAS_DEVICE0_ASTATUS_SATA_NEEDS_INITIALIZATION:
-		rc = 0;
-		break;
-	case MPI2_SAS_DEVICE0_ASTATUS_SATA_CAPABILITY_FAILED:
-		desc = "sata capability failed";
-		break;
-	case MPI2_SAS_DEVICE0_ASTATUS_SATA_AFFILIATION_CONFLICT:
-		desc = "sata affiliation conflict";
-		break;
-	case MPI2_SAS_DEVICE0_ASTATUS_ROUTE_NOT_ADDRESSABLE:
-		desc = "route not addressable";
-		break;
-	case MPI2_SAS_DEVICE0_ASTATUS_SMP_ERROR_NOT_ADDRESSABLE:
-		desc = "smp error not addressable";
+	if (ioc->pci_error_recovery) {
+		pr_info("%s: pci error recovery in progress!\n", __func__);
+		return -EFAULT;
+	}
+
+	if (ioc->shost_recovery) {
+		pr_info("%s: host recovery in progress!\n", __func__);
+		return -EAGAIN;
+	}
+
+	handle = transfer_packet->handle;
+	if (handle == MPT3SAS_INVALID_DEVICE_HANDLE) {
+		pr_info("%s: no device!\n",  __func__);
+		return -EFAULT;
+	}
+
+	mutex_lock(&ioc->scsih_cmds.mutex);
+
+	if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {
+		pr_err("%s: scsih_cmd in use\n", __func__);
+		rc = -EAGAIN;
+		goto out;
+	}
+
+ retry_loop:
+	if (test_bit(handle, ioc->device_remove_in_progress)) {
+		pr_info("%s: device removal in progress\n", __func__);
+		rc = -EFAULT;
+		goto out;
+	}
+
+	ioc->scsih_cmds.status = MPT3_CMD_PENDING;
+
+	rc = mpt3sas_wait_for_ioc(ioc, 10);
+	if (rc)
+		goto out;
+
+	/* Use second reserved smid for discovery related IOs */
+	smid = ioc->shost->can_queue + INTERNAL_SCSIIO_FOR_DISCOVERY;
+
+	rc = 0;
+	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
+	ioc->scsih_cmds.smid = smid;
+	memset(mpi_request, 0, sizeof(Mpi2SCSIIORequest_t));
+	if (transfer_packet->is_raid)
+		mpi_request->Function = MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH;
+	else
+		mpi_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
+	mpi_request->DevHandle = cpu_to_le16(handle);
+
+	switch (transfer_packet->dir) {
+	case DMA_TO_DEVICE:
+		mpi_control = MPI2_SCSIIO_CONTROL_WRITE;
+		data_out_dma = transfer_packet->data_dma;
+		data_out_sz = transfer_packet->data_length;
 		break;
-	case MPI2_SAS_DEVICE0_ASTATUS_DEVICE_BLOCKED:
-		desc = "device blocked";
+	case DMA_FROM_DEVICE:
+		mpi_control = MPI2_SCSIIO_CONTROL_READ;
+		data_in_dma = transfer_packet->data_dma;
+		data_in_sz = transfer_packet->data_length;
 		break;
-	case MPI2_SAS_DEVICE0_ASTATUS_SATA_INIT_FAILED:
-	case MPI2_SAS_DEVICE0_ASTATUS_SIF_UNKNOWN:
-	case MPI2_SAS_DEVICE0_ASTATUS_SIF_AFFILIATION_CONFLICT:
-	case MPI2_SAS_DEVICE0_ASTATUS_SIF_DIAG:
-	case MPI2_SAS_DEVICE0_ASTATUS_SIF_IDENTIFICATION:
-	case MPI2_SAS_DEVICE0_ASTATUS_SIF_CHECK_POWER:
-	case MPI2_SAS_DEVICE0_ASTATUS_SIF_PIO_SN:
-	case MPI2_SAS_DEVICE0_ASTATUS_SIF_MDMA_SN:
-	case MPI2_SAS_DEVICE0_ASTATUS_SIF_UDMA_SN:
-	case MPI2_SAS_DEVICE0_ASTATUS_SIF_ZONING_VIOLATION:
-	case MPI2_SAS_DEVICE0_ASTATUS_SIF_NOT_ADDRESSABLE:
-	case MPI2_SAS_DEVICE0_ASTATUS_SIF_MAX:
-		desc = "sata initialization failed";
+	case DMA_BIDIRECTIONAL:
+		mpi_control = MPI2_SCSIIO_CONTROL_BIDIRECTIONAL;
+		/* TODO - is BIDI support needed ?? */
+		WARN_ON_ONCE(true);
 		break;
 	default:
-		desc = "unknown";
+	case DMA_NONE:
+		mpi_control = MPI2_SCSIIO_CONTROL_NODATATRANSFER;
 		break;
 	}
 
-	if (!rc)
-		return 0;
+	psge = &mpi_request->SGL;
+	ioc->build_sg(ioc, psge, data_out_dma, data_out_sz, data_in_dma,
+	    data_in_sz);
 
-	ioc_err(ioc, "discovery errors(%s): sas_address(0x%016llx), handle(0x%04x)\n",
-		desc, (u64)sas_address, handle);
+	mpi_request->Control = cpu_to_le32(mpi_control |
+	    MPI2_SCSIIO_CONTROL_SIMPLEQ);
+	mpi_request->DataLength = cpu_to_le32(transfer_packet->data_length);
+	mpi_request->MsgFlags = MPI2_SCSIIO_MSGFLAGS_SYSTEM_SENSE_ADDR;
+	mpi_request->SenseBufferLength = SCSI_SENSE_BUFFERSIZE;
+	mpi_request->SenseBufferLowAddress =
+	    mpt3sas_base_get_sense_buffer_dma(ioc, smid);
+	priv_sense = mpt3sas_base_get_sense_buffer(ioc, smid);
+	mpi_request->SGLOffset0 = offsetof(Mpi2SCSIIORequest_t, SGL) / 4;
+	mpi_request->IoFlags = cpu_to_le16(transfer_packet->cdb_length);
+	int_to_scsilun(transfer_packet->lun, (struct scsi_lun *)
+	    mpi_request->LUN);
+	memcpy(mpi_request->CDB.CDB32, transfer_packet->cdb,
+	    transfer_packet->cdb_length);
+	init_completion(&ioc->scsih_cmds.done);
+	if (likely(mpi_request->Function == MPI2_FUNCTION_SCSI_IO_REQUEST))
+		ioc->put_smid_scsi_io(ioc, smid, handle);
+	else
+		ioc->put_smid_default(ioc, smid);
+	wait_for_completion_timeout(&ioc->scsih_cmds.done,
+	    transfer_packet->timeout*HZ);
+	if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {
+		mpt3sas_check_cmd_timeout(ioc,
+		    ioc->scsih_cmds.status, mpi_request,
+		    sizeof(Mpi2SCSIIORequest_t)/4, issue_reset);
+		goto issue_target_reset;
+	}
+	if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {
+		transfer_packet->valid_reply = 1;
+		mpi_reply = ioc->scsih_cmds.reply;
+		transfer_packet->sense_length =
+		   le32_to_cpu(mpi_reply->SenseCount);
+		if (transfer_packet->sense_length)
+			memcpy(transfer_packet->sense, priv_sense,
+			    transfer_packet->sense_length);
+		transfer_packet->transfer_length =
+		    le32_to_cpu(mpi_reply->TransferCount);
+		transfer_packet->ioc_status =
+		    le16_to_cpu(mpi_reply->IOCStatus) &
+		    MPI2_IOCSTATUS_MASK;
+		transfer_packet->scsi_state = mpi_reply->SCSIState;
+		transfer_packet->scsi_status = mpi_reply->SCSIStatus;
+		transfer_packet->log_info =
+		    le32_to_cpu(mpi_reply->IOCLogInfo);
+	}
+	goto out;
+
+ issue_target_reset:
+	if (issue_reset) {
+		pr_info("issue target reset: handle (0x%04x)\n", handle);
+		tm_return_code =
+			mpt3sas_scsih_issue_locked_tm(ioc, handle,
+				0xFFFFFFFF, 0xFFFFFFFF, 0,
+				MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, smid, 0,
+				tr_timeout, tr_method);
+
+		if (tm_return_code == SUCCESS) {
+			pr_info("target reset completed: handle (0x%04x)\n", handle);
+			/* If the command is successfully aborted due to
+			 * target reset TM then do up to three retries else
+			 * command will be terminated by the host reset TM and
+			 * hence retry once.
+			 */
+			if (((ioc->scsih_cmds.status & MPT3_CMD_COMPLETE) &&
+			    retry_count++ < 3) ||
+			    ((ioc->scsih_cmds.status & MPT3_CMD_RESET) &&
+			    host_reset_count++ == 0)) {
+				pr_info("issue retry: handle (0x%04x)\n", handle);
+				goto retry_loop;
+			}
+		} else
+			pr_info("target reset didn't complete:  handle(0x%04x)\n", handle);
+		rc = -EFAULT;
+	} else
+		rc = -EAGAIN;
+
+ out:
+	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
+	mutex_unlock(&ioc->scsih_cmds.mutex);
 	return rc;
 }
 
 /**
- * _scsih_check_device - checking device responsiveness
+ * _scsih_determine_disposition -
  * @ioc: per adapter object
- * @parent_sas_address: sas address of parent expander or sas host
- * @handle: attached device handle
- * @phy_number: phy number
- * @link_rate: new link rate
+ * @transfer_packet: packet describing the transfer
+ * Context: user
+ *
+ * Determines if an internal generated scsi_io is good data, or
+ * whether it needs to be retried or treated as an error.
+ *
+ * Returns device_responsive_state
  */
-static void
-_scsih_check_device(struct MPT3SAS_ADAPTER *ioc,
-	u64 parent_sas_address, u16 handle, u8 phy_number, u8 link_rate)
+static enum device_responsive_state
+_scsih_determine_disposition(struct MPT3SAS_ADAPTER *ioc,
+	struct _scsi_io_transfer *transfer_packet)
 {
-	Mpi2ConfigReply_t mpi_reply;
-	Mpi2SasDevicePage0_t sas_device_pg0;
-	struct _sas_device *sas_device = NULL;
-	struct _enclosure_node *enclosure_dev = NULL;
+	static enum device_responsive_state rc;
+	struct sense_info sense_info = {0, 0, 0};
+	u8 check_sense = 0;
+	char *desc = NULL;
+
+	if (!transfer_packet->valid_reply)
+		return DEVICE_READY;
+
+	switch (transfer_packet->ioc_status) {
+	case MPI2_IOCSTATUS_BUSY:
+	case MPI2_IOCSTATUS_INSUFFICIENT_RESOURCES:
+	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
+	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
+	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
+		rc = DEVICE_RETRY;
+		break;
+	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
+		if (transfer_packet->log_info ==  0x31170000) {
+			rc = DEVICE_RETRY;
+			break;
+		}
+		if (transfer_packet->cdb[0] == REPORT_LUNS)
+			rc = DEVICE_READY;
+		else
+			rc = DEVICE_RETRY;
+		break;
+	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
+	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
+	case MPI2_IOCSTATUS_SUCCESS:
+		if (!transfer_packet->scsi_state &&
+		    !transfer_packet->scsi_status) {
+			rc = DEVICE_READY;
+			break;
+		}
+		if (transfer_packet->scsi_state &
+		    MPI2_SCSI_STATE_AUTOSENSE_VALID) {
+			rc = DEVICE_ERROR;
+			check_sense = 1;
+			break;
+		}
+		if (transfer_packet->scsi_state &
+		    (MPI2_SCSI_STATE_AUTOSENSE_FAILED |
+		    MPI2_SCSI_STATE_NO_SCSI_STATUS |
+		    MPI2_SCSI_STATE_TERMINATED)) {
+			rc = DEVICE_RETRY;
+			break;
+		}
+		if (transfer_packet->scsi_status >=
+		    MPI2_SCSI_STATUS_BUSY) {
+			rc = DEVICE_RETRY;
+			break;
+		}
+		rc = DEVICE_READY;
+		break;
+	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
+		if (transfer_packet->scsi_state &
+		    MPI2_SCSI_STATE_TERMINATED)
+			rc = DEVICE_RETRY;
+		else
+			rc = DEVICE_ERROR;
+		break;
+	case MPI2_IOCSTATUS_INSUFFICIENT_POWER:
+	default:
+		rc = DEVICE_ERROR;
+		break;
+	}
+
+	if (check_sense) {
+		_scsih_normalize_sense(transfer_packet->sense, &sense_info);
+		if (sense_info.skey == UNIT_ATTENTION)
+			rc = DEVICE_RETRY_UA;
+		else if (sense_info.skey == NOT_READY) {
+			/* medium isn't present */
+			if (sense_info.asc == 0x3a)
+				rc = DEVICE_READY;
+			/* LOGICAL UNIT NOT READY */
+			else if (sense_info.asc == 0x04) {
+				if (sense_info.ascq == 0x03 ||
+				   sense_info.ascq == 0x0b ||
+				   sense_info.ascq == 0x0c) {
+					rc = DEVICE_ERROR;
+				} else
+					rc = DEVICE_START_UNIT;
+			}
+			/* LOGICAL UNIT HAS NOT SELF-CONFIGURED YET */
+			else if (sense_info.asc == 0x3e && !sense_info.ascq)
+				rc = DEVICE_START_UNIT;
+		} else if (sense_info.skey == ILLEGAL_REQUEST &&
+		    transfer_packet->cdb[0] == REPORT_LUNS) {
+			rc = DEVICE_READY;
+		} else if (sense_info.skey == MEDIUM_ERROR) {
+
+			/* medium is corrupt, lets add the device so
+			 * users can collect some info as needed
+			 */
+
+			if (sense_info.asc == 0x31)
+				rc = DEVICE_READY;
+		} else if (sense_info.skey == HARDWARE_ERROR) {
+			/* Defect List Error, still add the device */
+			if (sense_info.asc == 0x19)
+				rc = DEVICE_READY;
+		}
+	}
+
+	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK) {
+		switch (rc) {
+		case DEVICE_READY:
+			desc = "ready";
+			break;
+		case DEVICE_RETRY:
+			desc = "retry";
+			break;
+		case DEVICE_RETRY_UA:
+			desc = "retry_ua";
+			break;
+		case DEVICE_START_UNIT:
+			desc = "start_unit";
+			break;
+		case DEVICE_STOP_UNIT:
+			desc = "stop_unit";
+			break;
+		case DEVICE_ERROR:
+			desc = "error";
+			break;
+		}
+
+		pr_info("ioc_status(0x%04x),\n"
+		    "loginfo(0x%08x), scsi_status(0x%02x),\n"
+		    "scsi_state(0x%02x), rc(%s)\n",
+			transfer_packet->ioc_status,
+			transfer_packet->log_info, transfer_packet->scsi_status,
+			transfer_packet->scsi_state, desc);
+
+		if (check_sense)
+			pr_info("\t[sense_key,asc,ascq]:\n"
+			    "[0x%02x,0x%02x,0x%02x]\n",
+			    sense_info.skey, sense_info.asc, sense_info.ascq);
+	}
+	return rc;
+}
+
+/**
+ * _scsih_report_luns - send REPORT_LUNS to target
+ * @ioc: per adapter object
+ * @handle: expander handle
+ * @data: report luns data payload
+ * @data_length: length of data in bytes
+ * @retry_count: Requeue count
+ * @is_pd: is this hidden raid component
+ * @tr_timeout: Target Reset Timeout
+ * @tr_method: Target Reset Method
+ * Context: user
+ *
+ * Returns device_responsive_state
+ */
+static enum device_responsive_state
+_scsih_report_luns(struct MPT3SAS_ADAPTER *ioc, u16 handle, void *data,
+	u32 data_length, u8 retry_count, u8 is_pd, u8 tr_timeout, u8 tr_method)
+{
+	struct _scsi_io_transfer *transfer_packet;
+	enum device_responsive_state rc;
+	void *lun_data;
+	int return_code;
+	int retries;
+
+	lun_data = NULL;
+	transfer_packet = kzalloc(sizeof(struct _scsi_io_transfer), GFP_KERNEL);
+	if (!transfer_packet) {
+
+		ioc_err(ioc, "failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__);
+		rc = DEVICE_RETRY;
+		goto out;
+	}
+
+	lun_data = dma_alloc_coherent(&ioc->pdev->dev, data_length,
+		&transfer_packet->data_dma, GFP_ATOMIC);
+	if (!lun_data) {
+
+		ioc_err(ioc, "failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__);
+		rc = DEVICE_RETRY;
+		goto out;
+	}
+
+	for (retries = 0; retries < 4; retries++) {
+		rc = DEVICE_ERROR;
+		ioc_info(ioc, "REPORT_LUNS: handle(0x%04x),\n"
+		    "retries(%d)\n", handle, retries);
+		memset(lun_data, 0, data_length);
+		transfer_packet->handle = handle;
+		transfer_packet->dir = DMA_FROM_DEVICE;
+		transfer_packet->data_length = data_length;
+		transfer_packet->cdb_length = 12;
+		transfer_packet->cdb[0] = REPORT_LUNS;
+		transfer_packet->cdb[6] = (data_length >> 24) & 0xFF;
+		transfer_packet->cdb[7] = (data_length >> 16) & 0xFF;
+		transfer_packet->cdb[8] = (data_length >>  8) & 0xFF;
+		transfer_packet->cdb[9] = data_length & 0xFF;
+		transfer_packet->timeout = 30;
+		transfer_packet->is_raid = is_pd;
+
+		return_code = _scsi_send_scsi_io(ioc, transfer_packet, tr_timeout, tr_method);
+		switch (return_code) {
+		case 0:
+			rc = _scsih_determine_disposition(ioc, transfer_packet);
+			if (rc == DEVICE_READY) {
+				memcpy(data, lun_data, data_length);
+				goto out;
+			} else if (rc == DEVICE_ERROR)
+				goto out;
+			break;
+		case -EAGAIN:
+			rc = DEVICE_RETRY;
+			break;
+		case -EFAULT:
+		default:
+			ioc_err(ioc, "failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__);
+			goto out;
+		}
+	}
+ out:
+
+	if (lun_data)
+		dma_free_coherent(&ioc->pdev->dev, data_length, lun_data,
+		    transfer_packet->data_dma);
+	kfree(transfer_packet);
+
+	if ((rc == DEVICE_RETRY || rc == DEVICE_START_UNIT ||
+	    rc == DEVICE_RETRY_UA) && retry_count >= command_retry_count)
+		rc = DEVICE_ERROR;
+
+	return rc;
+}
+
+/**
+ * _scsih_start_unit - send START_UNIT to target
+ * @ioc: per adapter object
+ * @handle: expander handle
+ * @lun: lun number
+ * @is_pd: is this hidden raid component
+ * @tr_timeout: Target Reset Timeout
+ * @tr_method: Target Reset Method
+ * Context: user
+ *
+ * Returns device_responsive_state
+ */
+static enum device_responsive_state
+_scsih_start_unit(struct MPT3SAS_ADAPTER *ioc, u16 handle, u32 lun, u8 is_pd,
+	u8 tr_timeout, u8 tr_method)
+{
+	struct _scsi_io_transfer *transfer_packet;
+	enum device_responsive_state rc;
+	int return_code;
+
+	transfer_packet = kzalloc(sizeof(struct _scsi_io_transfer), GFP_KERNEL);
+	if (!transfer_packet) {
+
+		pr_info("failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__);
+		rc = DEVICE_RETRY;
+		goto out;
+	}
+
+	rc = DEVICE_READY;
+	transfer_packet->handle = handle;
+	transfer_packet->dir = DMA_NONE;
+	transfer_packet->lun = lun;
+	transfer_packet->cdb_length = 6;
+	transfer_packet->cdb[0] = START_STOP;
+	transfer_packet->cdb[1] = 1;
+	transfer_packet->cdb[4] = 1;
+	transfer_packet->timeout = 30;
+	transfer_packet->is_raid = is_pd;
+
+	pr_info("START_UNIT: handle(0x%04x), lun(%d)\n", handle, lun);
+
+	return_code = _scsi_send_scsi_io(ioc, transfer_packet, tr_timeout, tr_method);
+	switch (return_code) {
+	case 0:
+		rc = _scsih_determine_disposition(ioc, transfer_packet);
+		break;
+	case -EAGAIN:
+		rc = DEVICE_RETRY;
+		break;
+	case -EFAULT:
+	default:
+		pr_err("failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__);
+		rc = DEVICE_ERROR;
+		break;
+	}
+ out:
+	kfree(transfer_packet);
+	return rc;
+}
+
+/**
+ * _scsih_test_unit_ready - send TUR to target
+ * @ioc: per adapter object
+ * @handle: expander handle
+ * @lun: lun number
+ * @is_pd: is this hidden raid component
+ * @tr_timeout: Target Reset timeout value for Pcie devie
+ * @tr_method: pcie device Target reset method
+ * Context: user
+ *
+ * Returns device_responsive_state
+ */
+static enum device_responsive_state
+_scsih_test_unit_ready(struct MPT3SAS_ADAPTER *ioc, u16 handle, u32 lun,
+	u8 is_pd, u8 tr_timeout, u8 tr_method)
+{
+	struct _scsi_io_transfer *transfer_packet;
+	enum device_responsive_state rc;
+	int return_code;
+	int sata_init_failure = 0;
+
+	transfer_packet = kzalloc(sizeof(struct _scsi_io_transfer), GFP_KERNEL);
+	if (!transfer_packet) {
+
+		pr_info("failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__);
+		rc = DEVICE_RETRY;
+		goto out;
+	}
+
+	rc = DEVICE_READY;
+	transfer_packet->handle = handle;
+	transfer_packet->dir = DMA_NONE;
+	transfer_packet->lun = lun;
+	transfer_packet->cdb_length = 6;
+	transfer_packet->cdb[0] = TEST_UNIT_READY;
+	transfer_packet->timeout = 30;
+	transfer_packet->is_raid = is_pd;
+
+ sata_init_retry:
+	pr_info("TEST_UNIT_READY: handle(0x%04x) lun(%d)\n", handle, lun);
+
+	return_code = _scsi_send_scsi_io(ioc, transfer_packet, tr_timeout, tr_method);
+	switch (return_code) {
+	case 0:
+		rc = _scsih_determine_disposition(ioc, transfer_packet);
+		if (rc == DEVICE_RETRY &&
+		    transfer_packet->log_info == 0x31111000) {
+			if (!sata_init_failure++) {
+				pr_info("SATA Initialization Timeout sending a retry\n");
+				rc = DEVICE_READY;
+				goto sata_init_retry;
+			} else {
+				pr_err("SATA Initialization Failed\n");
+				rc = DEVICE_ERROR;
+			}
+		}
+		break;
+	case -EAGAIN:
+		rc = DEVICE_RETRY;
+		break;
+	case -EFAULT:
+	default:
+		pr_err("failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__);
+		rc = DEVICE_ERROR;
+		break;
+	}
+ out:
+	kfree(transfer_packet);
+	return rc;
+}
+
+/**
+ * _scsih_ata_pass_thru_idd - obtain SATA device Identify Device Data
+ * @ioc: per adapter object
+ * @handle: device handle
+ * @is_ssd_device : is this SATA SSD device
+ * @tr_timeout: Target Reset Timeout
+ * @tr_method: Target Reset Method
+ * Context: user
+ *
+ * Returns device_responsive_state
+ */
+static enum device_responsive_state
+_scsih_ata_pass_thru_idd(struct MPT3SAS_ADAPTER *ioc, u16 handle,
+	u8 *is_ssd_device, u8 tr_timeout, u8 tr_method)
+{
+	struct _scsi_io_transfer *transfer_packet;
+	enum device_responsive_state rc;
+	u16 *idd_data;
+	int return_code;
+	u32 data_length;
+
+	idd_data = NULL;
+	transfer_packet = kzalloc(sizeof(struct _scsi_io_transfer), GFP_KERNEL);
+	if (!transfer_packet) {
+
+		ioc_err(ioc, "failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__);
+		rc = DEVICE_RETRY;
+		goto out;
+	}
+	data_length = 512;
+	idd_data = dma_alloc_coherent(&ioc->pdev->dev, data_length,
+		&transfer_packet->data_dma, GFP_ATOMIC);
+	if (!idd_data) {
+
+		ioc_err(ioc, "failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__);
+		rc = DEVICE_RETRY;
+		goto out;
+	}
+	rc = DEVICE_READY;
+	memset(idd_data, 0, data_length);
+	transfer_packet->handle = handle;
+	transfer_packet->dir = DMA_FROM_DEVICE;
+	transfer_packet->data_length = data_length;
+	transfer_packet->cdb_length = 12;
+	transfer_packet->cdb[0] = ATA_12;
+	transfer_packet->cdb[1] = 0x8;
+	transfer_packet->cdb[2] = 0xd;
+	transfer_packet->cdb[3] = 0x1;
+	transfer_packet->cdb[9] = 0xec;
+	transfer_packet->timeout = 30;
+
+	return_code = _scsi_send_scsi_io(ioc, transfer_packet, 30, 0);
+	switch (return_code) {
+	case 0:
+		rc = _scsih_determine_disposition(ioc, transfer_packet);
+		if (rc == DEVICE_READY) {
+			// Check if nominal media rotation rate is set to 1 i.e. SSD device
+			if (idd_data[217] == 1)
+				*is_ssd_device = 1;
+		}
+		break;
+	case -EAGAIN:
+		rc = DEVICE_RETRY;
+		break;
+	case -EFAULT:
+	default:
+
+		ioc_err(ioc, "failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__);
+		rc = DEVICE_ERROR;
+		break;
+	}
+
+ out:
+	if (idd_data) {
+		dma_free_coherent(&ioc->pdev->dev, data_length, idd_data,
+		    transfer_packet->data_dma);
+	}
+	kfree(transfer_packet);
+	return rc;
+}
+
+/**
+ * _scsih_wait_for_device_to_become_ready - handle busy devices
+ * @ioc: per adapter object
+ * @handle: expander handle
+ * @retry_count: number of times this event has been retried
+ * @is_pd: is this hidden raid component
+ * @lun: lun number
+ * @tr_timeout: Target Reset Timeout
+ * @tr_method: Target Reset Method
+ *
+ * Some devices spend too much time in busy state, queue event later
+ *
+ * Return the device_responsive_state.
+ */
+
+static enum device_responsive_state
+_scsih_wait_for_device_to_become_ready(struct MPT3SAS_ADAPTER *ioc, u16 handle,
+	u8 retry_count, u8 is_pd, int lun, u8 tr_timeout, u8 tr_method)
+{
+	enum device_responsive_state rc;
+
+	if (ioc->pci_error_recovery)
+		return DEVICE_ERROR;
+
+	if (ioc->shost_recovery)
+		return DEVICE_RETRY;
+
+	rc = _scsih_test_unit_ready(ioc, handle, lun, is_pd, tr_timeout, tr_method);
+	if (rc == DEVICE_READY || rc == DEVICE_ERROR)
+		return rc;
+	else if (rc == DEVICE_START_UNIT) {
+		rc = _scsih_start_unit(ioc, handle, lun, is_pd, tr_timeout, tr_method);
+		if (rc == DEVICE_ERROR)
+			return rc;
+		rc = _scsih_test_unit_ready(ioc, handle, lun, is_pd, tr_timeout, tr_method);
+	}
+
+	if ((rc == DEVICE_RETRY || rc == DEVICE_START_UNIT ||
+	    rc == DEVICE_RETRY_UA) && retry_count >= command_retry_count)
+		rc = DEVICE_ERROR;
+	return rc;
+}
+
+static inline int mpt_scsilun_to_int(struct scsi_lun *scsilun)
+{
+	return scsilun_to_int(scsilun);
+}
+
+/**
+ * _scsih_wait_for_target_to_become_ready - handle busy devices
+ * @ioc: per adapter object
+ * @handle: expander handle
+ * @retry_count: number of times this event has been retried
+ * @is_pd: is this hidden raid component
+ * @tr_timeout: Target Reset timeout value
+ * @tr_method: Target Reset method Hot/Protocol level.
+ *
+ * Some devices spend too much time in busy state, queue event later
+ *
+ * Return the device_responsive_state.
+ */
+static enum device_responsive_state
+_scsih_wait_for_target_to_become_ready(struct MPT3SAS_ADAPTER *ioc, u16 handle,
+	u8 retry_count, u8 is_pd, u8 tr_timeout, u8 tr_method)
+{
+	enum device_responsive_state rc;
+	struct scsi_lun *lun_data;
+	u32 length, num_luns;
+	u8 *data;
+	int lun;
+	struct scsi_lun *lunp;
+
+	lun_data = kcalloc(MPT3_MAX_LUNS, sizeof(struct scsi_lun), GFP_KERNEL);
+	if (!lun_data) {
+
+		ioc_err(ioc, "failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__);
+		return DEVICE_RETRY;
+	}
+
+	rc = _scsih_report_luns(ioc, handle, lun_data,
+	    MPT3_MAX_LUNS * sizeof(struct scsi_lun), retry_count, is_pd,
+	    tr_timeout, tr_method);
+
+	if (rc != DEVICE_READY)
+		goto out;
+
+	/* some debug bits*/
+	data = (u8 *)lun_data;
+	length = ((data[0] << 24) | (data[1] << 16) |
+		(data[2] << 8) | (data[3] << 0));
+
+	num_luns = (length / sizeof(struct scsi_lun));
+
+	lunp = &lun_data[1];
+	lun = (num_luns) ? mpt_scsilun_to_int(&lun_data[1]) : 0;
+	rc = _scsih_wait_for_device_to_become_ready(ioc, handle, retry_count,
+	    is_pd, lun, tr_timeout, tr_method);
+
+	if (rc == DEVICE_ERROR) {
+		struct scsi_lun *lunq;
+
+		for (lunq = lunp++; lunq <= &lun_data[num_luns]; lunq++) {
+
+			rc = _scsih_wait_for_device_to_become_ready(ioc, handle,
+					retry_count, is_pd, mpt_scsilun_to_int(lunq),
+					tr_timeout, tr_method);
+			if (rc != DEVICE_ERROR)
+				goto out;
+		}
+	}
+out:
+	kfree(lun_data);
+	return rc;
+}
+
+
+/**
+ * _scsih_check_access_status - check access flags
+ * @ioc: per adapter object
+ * @sas_address: sas address
+ * @handle: sas device handle
+ * @access_status: errors returned during discovery of the device
+ *
+ * Return: 0 for success, else failure
+ */
+static u8
+_scsih_check_access_status(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,
+	u16 handle, u8 access_status)
+{
+	u8 rc = 1;
+	char *desc = NULL;
+
+	switch (access_status) {
+	case MPI2_SAS_DEVICE0_ASTATUS_NO_ERRORS:
+	case MPI2_SAS_DEVICE0_ASTATUS_SATA_NEEDS_INITIALIZATION:
+		rc = 0;
+		break;
+	case MPI2_SAS_DEVICE0_ASTATUS_SATA_CAPABILITY_FAILED:
+		desc = "sata capability failed";
+		break;
+	case MPI2_SAS_DEVICE0_ASTATUS_SATA_AFFILIATION_CONFLICT:
+		desc = "sata affiliation conflict";
+		break;
+	case MPI2_SAS_DEVICE0_ASTATUS_ROUTE_NOT_ADDRESSABLE:
+		desc = "route not addressable";
+		break;
+	case MPI2_SAS_DEVICE0_ASTATUS_SMP_ERROR_NOT_ADDRESSABLE:
+		desc = "smp error not addressable";
+		break;
+	case MPI2_SAS_DEVICE0_ASTATUS_DEVICE_BLOCKED:
+		desc = "device blocked";
+		break;
+	case MPI2_SAS_DEVICE0_ASTATUS_SATA_INIT_FAILED:
+	case MPI2_SAS_DEVICE0_ASTATUS_SIF_UNKNOWN:
+	case MPI2_SAS_DEVICE0_ASTATUS_SIF_AFFILIATION_CONFLICT:
+	case MPI2_SAS_DEVICE0_ASTATUS_SIF_DIAG:
+	case MPI2_SAS_DEVICE0_ASTATUS_SIF_IDENTIFICATION:
+	case MPI2_SAS_DEVICE0_ASTATUS_SIF_CHECK_POWER:
+	case MPI2_SAS_DEVICE0_ASTATUS_SIF_PIO_SN:
+	case MPI2_SAS_DEVICE0_ASTATUS_SIF_MDMA_SN:
+	case MPI2_SAS_DEVICE0_ASTATUS_SIF_UDMA_SN:
+	case MPI2_SAS_DEVICE0_ASTATUS_SIF_ZONING_VIOLATION:
+	case MPI2_SAS_DEVICE0_ASTATUS_SIF_NOT_ADDRESSABLE:
+	case MPI2_SAS_DEVICE0_ASTATUS_SIF_MAX:
+		desc = "sata initialization failed";
+		break;
+	default:
+		desc = "unknown";
+		break;
+	}
+
+	if (!rc)
+		return 0;
+
+	ioc_err(ioc, "discovery errors(%s): sas_address(0x%016llx), handle(0x%04x)\n",
+		desc, (u64)sas_address, handle);
+	return rc;
+}
+
+/**
+ * _scsih_check_device - checking device responsiveness
+ * @ioc: per adapter object
+ * @parent_sas_address: sas address of parent expander or sas host
+ * @handle: attached device handle
+ * @phy_number: phy number
+ * @link_rate: new link rate
+ */
+static void
+_scsih_check_device(struct MPT3SAS_ADAPTER *ioc,
+	u64 parent_sas_address, u16 handle, u8 phy_number, u8 link_rate)
+{
+	Mpi2ConfigReply_t mpi_reply;
+	Mpi2SasDevicePage0_t sas_device_pg0;
+	struct _sas_device *sas_device = NULL;
+	struct _enclosure_node *enclosure_dev = NULL;
 	u32 ioc_status;
 	unsigned long flags;
 	u64 sas_address;
@@ -7239,8 +8302,8 @@ _scsih_check_device(struct MPT3SAS_ADAPTER *ioc,
 			sas_device->handle, handle);
 		sas_target_priv_data->handle = handle;
 		sas_device->handle = handle;
-		if (le16_to_cpu(sas_device_pg0.Flags) &
-		     MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
+		if ((le16_to_cpu(sas_device_pg0.Flags) & MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID)
+		    && (ioc->hba_mpi_version_belonged != MPI2_VERSION)) {
 			sas_device->enclosure_level =
 				sas_device_pg0.EnclosureLevel;
 			memcpy(sas_device->connector_name,
@@ -7282,7 +8345,11 @@ _scsih_check_device(struct MPT3SAS_ADAPTER *ioc,
 		goto out_unlock;
 
 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
-	_scsih_ublock_io_device(ioc, sas_address, port);
+
+	if (issue_scsi_cmd_to_bringup_drive)
+		_scsih_ublock_io_device_wait(ioc, sas_address, port);
+	else
+		_scsih_ublock_io_device(ioc, sas_address, port);
 
 	if (sas_device)
 		sas_device_put(sas_device);
@@ -7298,7 +8365,7 @@ _scsih_check_device(struct MPT3SAS_ADAPTER *ioc,
  * _scsih_add_device -  creating sas device object
  * @ioc: per adapter object
  * @handle: sas device handle
- * @phy_num: phy number end device attached to
+ * @retry_count: number of times this event has been retried
  * @is_pd: is this hidden raid component
  *
  * Creating end device object, stored in ioc->sas_device_list.
@@ -7306,16 +8373,18 @@ _scsih_check_device(struct MPT3SAS_ADAPTER *ioc,
  * Return: 0 for success, non-zero for failure.
  */
 static int
-_scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,
+_scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 retry_count,
 	u8 is_pd)
 {
 	Mpi2ConfigReply_t mpi_reply;
 	Mpi2SasDevicePage0_t sas_device_pg0;
 	struct _sas_device *sas_device;
 	struct _enclosure_node *enclosure_dev = NULL;
+	enum device_responsive_state rc;
 	u32 ioc_status;
 	u64 sas_address;
 	u32 device_info;
+	u8 connector_name[5];
 	u8 port_id;
 
 	if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,
@@ -7371,6 +8440,48 @@ _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,
 				 sas_device_pg0.EnclosureHandle);
 	}
 
+	/*
+	 * Wait for device that is becoming ready
+	 * queue request later if device is busy.
+	 */
+	if ((!ioc->wait_for_discovery_to_complete) &&
+		(issue_scsi_cmd_to_bringup_drive)) {
+		ioc_info(ioc, "detecting: handle(0x%04x),\n"
+				"sas_address(0x%016llx), phy(%d)\n", handle,
+				(unsigned long long)sas_address, sas_device_pg0.PhyNum);
+		rc = _scsih_wait_for_target_to_become_ready(ioc, handle,
+		    retry_count, is_pd, 30, 0);
+		if (rc != DEVICE_READY) {
+			if (le16_to_cpu(sas_device_pg0.EnclosureHandle) != 0)
+				dewtprintk(ioc, ioc_info(ioc, "%s:\n"
+				    "device not ready: slot(%d)\n", __func__,
+				    le16_to_cpu(sas_device_pg0.Slot)));
+			if ((le16_to_cpu(sas_device_pg0.Flags) &
+			    MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) &&
+			    (ioc->hba_mpi_version_belonged != MPI2_VERSION)) {
+				memcpy(connector_name,
+					sas_device_pg0.ConnectorName, 4);
+				connector_name[4] = '\0';
+				dewtprintk(ioc, ioc_info(ioc, "%s:\n"
+				    "device not ready:\n"
+				    "enclosure level(0x%04x),\n"
+				    "connector name( %s)\n",  __func__,
+				    sas_device_pg0.EnclosureLevel, connector_name));
+			}
+
+			if ((enclosure_dev) && (le16_to_cpu(enclosure_dev->pg0.Flags) &
+			    MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID))
+				ioc_info(ioc, "chassis slot(0x%04x)\n",
+						enclosure_dev->pg0.ChassisSlot);
+
+			if (rc == DEVICE_RETRY || rc == DEVICE_START_UNIT ||
+			    rc == DEVICE_STOP_UNIT || rc == DEVICE_RETRY_UA)
+				return 1;
+			else if (rc == DEVICE_ERROR)
+				return 0;
+		}
+	}
+
 	sas_device = kzalloc(sizeof(struct _sas_device),
 	    GFP_KERNEL);
 	if (!sas_device) {
@@ -7586,10 +8697,13 @@ _scsih_sas_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
 	struct fw_event_work *fw_event)
 {
 	int i;
+	int rc;
+	int requeue_event;
 	u16 parent_handle, handle;
 	u16 reason_code;
 	u8 phy_number, max_phys;
 	struct _sas_node *sas_expander;
+	struct _sas_device *sas_device;
 	u64 sas_address;
 	unsigned long flags;
 	u8 link_rate, prev_link_rate;
@@ -7639,7 +8753,7 @@ _scsih_sas_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
 	spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
 
 	/* handle siblings events */
-	for (i = 0; i < event_data->NumEntries; i++) {
+	for (i = 0, requeue_event = 0; i < event_data->NumEntries; i++) {
 		if (fw_event->ignore) {
 			dewtprintk(ioc,
 				   ioc_info(ioc, "ignoring expander event\n"));
@@ -7656,6 +8770,20 @@ _scsih_sas_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
 		    MPI2_EVENT_SAS_TOPO_PHYSTATUS_VACANT) && (reason_code !=
 		    MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING))
 				continue;
+		if (fw_event->delayed_work_active && (reason_code ==
+		    MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING)) {
+			dewtprintk(ioc, ioc_info(ioc, "ignoring\n"
+			    "Target not responding event phy in re-queued event processing\n"));
+			continue;
+		}
+
+		if (fw_event->delayed_work_active && (reason_code ==
+		    MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING)) {
+			dewtprintk(ioc, ioc_info(ioc, "ignoring Target not responding\n"
+						"event phy in re-queued event processing\n"));
+			continue;
+		}
+
 		handle = le16_to_cpu(event_data->PHY[i].AttachedDevHandle);
 		if (!handle)
 			continue;
@@ -7679,9 +8807,32 @@ _scsih_sas_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
 			_scsih_check_device(ioc, sas_address, handle,
 			    phy_number, link_rate);
 
+			/* This code after this point handles the test case
+			 * where a device has been added, however its returning
+			 * BUSY for sometime.  Then before the Device Missing
+			 * Delay expires and the device becomes READY, the
+			 * device is removed and added back.
+			 */
+			spin_lock_irqsave(&ioc->sas_device_lock, flags);
+			sas_device = __mpt3sas_get_sdev_by_handle(ioc,
+			    handle);
+			spin_unlock_irqrestore(&ioc->sas_device_lock,
+			    flags);
+
+			if (sas_device) {
+				sas_device_put(sas_device);
+				break;
+			}
+
 			if (!test_bit(handle, ioc->pend_os_device_add))
 				break;
 
+			dewtprintk(ioc, ioc_info(ioc, "handle(0x%04x) device not found: convert\n"
+			    "event to a device add\n", handle));
+			event_data->PHY[i].PhyStatus &= 0xF0;
+			event_data->PHY[i].PhyStatus |=
+						MPI2_EVENT_SAS_TOPO_RC_TARG_ADDED;
+
 			fallthrough;
 
 		case MPI2_EVENT_SAS_TOPO_RC_TARG_ADDED:
@@ -7692,7 +8843,18 @@ _scsih_sas_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
 			mpt3sas_transport_update_links(ioc, sas_address,
 			    handle, phy_number, link_rate, port);
 
-			_scsih_add_device(ioc, handle, phy_number, 0);
+			if (link_rate < MPI2_SAS_NEG_LINK_RATE_1_5)
+				break;
+
+			rc = _scsih_add_device(ioc, handle,
+			    fw_event->retries[i], 0);
+			if (rc) {/* retry due to busy device */
+				fw_event->retries[i]++;
+				requeue_event = 1;
+			} else {/* mark entry vacant */
+				event_data->PHY[i].PhyStatus |=
+			    MPI2_EVENT_SAS_TOPO_PHYSTATUS_VACANT;
+			}
 
 			break;
 		case MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING:
@@ -7707,7 +8869,7 @@ _scsih_sas_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
 	    sas_expander)
 		mpt3sas_expander_remove(ioc, sas_address, port);
 
-	return 0;
+	return requeue_event;
 }
 
 /**
@@ -8078,7 +9240,10 @@ _scsih_pcie_check_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
 	pcie_device_put(pcie_device);
 
-	_scsih_ublock_io_device(ioc, wwid, NULL);
+	if (issue_scsi_cmd_to_bringup_drive)
+		_scsih_ublock_io_device_wait(ioc, wwid, NULL);
+	else
+		_scsih_ublock_io_device(ioc, wwid, NULL);
 
 	return;
 }
@@ -8087,19 +9252,24 @@ _scsih_pcie_check_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
  * _scsih_pcie_add_device -  creating pcie device object
  * @ioc: per adapter object
  * @handle: pcie device handle
+ * @retry_count: number of times this event has been retried
  *
  * Creating end device object, stored in ioc->pcie_device_list.
  *
  * Return: 1 means queue the event later, 0 means complete the event
  */
 static int
-_scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
+_scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 retry_count)
 {
 	Mpi26PCIeDevicePage0_t pcie_device_pg0;
 	Mpi26PCIeDevicePage2_t pcie_device_pg2;
 	Mpi2ConfigReply_t mpi_reply;
 	struct _pcie_device *pcie_device;
 	struct _enclosure_node *enclosure_dev;
+	enum device_responsive_state rc;
+	u8 connector_name[5];
+	u8 tr_timeout = 30;
+	u8 tr_method = 0;
 	u32 ioc_status;
 	u64 wwid;
 
@@ -8167,6 +9337,53 @@ _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 			    __LINE__, __func__);
 			return 0;
 		}
+
+		if (!ioc->tm_custom_handling) {
+			tr_method = MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
+			if (pcie_device_pg2.ControllerResetTO)
+				tr_timeout = pcie_device_pg2.ControllerResetTO;
+
+		}
+	}
+
+	/*
+	 * Wait for device that is becoming ready
+	 * queue request later if device is busy.
+	 */
+	if ((!ioc->wait_for_discovery_to_complete) &&
+		(issue_scsi_cmd_to_bringup_drive) &&
+		(pcie_device_pg0.AccessStatus !=
+			MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED)) {
+		ioc_info(ioc, "detecting: handle(0x%04x),\n"
+		    "wwid(0x%016llx), port(%d)\n", handle,
+		    (unsigned long long)wwid, pcie_device_pg0.PortNum);
+
+		rc = _scsih_wait_for_target_to_become_ready(ioc, handle,
+		    retry_count, 0, tr_timeout, tr_method);
+		if (rc != DEVICE_READY) {
+			if (le16_to_cpu(pcie_device_pg0.EnclosureHandle) != 0)
+				dewtprintk(ioc, ioc_info(ioc, "%s:\n"
+				    "device not ready: slot(%d)\n",
+				    __func__,
+				    le16_to_cpu(pcie_device_pg0.Slot)));
+
+			if (le32_to_cpu(pcie_device_pg0.Flags) &
+			    MPI26_PCIEDEV0_FLAGS_ENCL_LEVEL_VALID) {
+				memcpy(connector_name,
+				    pcie_device_pg0.ConnectorName, 4);
+				connector_name[4] = '\0';
+				dewtprintk(ioc, ioc_info(ioc, "%s: device not ready: enclosure\n"
+				    "level(0x%04x), connector name( %s)\n", __func__,
+				    pcie_device_pg0.EnclosureLevel,
+				    connector_name));
+			}
+
+			if (rc == DEVICE_RETRY || rc == DEVICE_START_UNIT ||
+				rc == DEVICE_STOP_UNIT || rc == DEVICE_RETRY_UA)
+				return 1;
+			else if (rc == DEVICE_ERROR)
+				return 0;
+		}
 	}
 
 	pcie_device = kzalloc(sizeof(struct _pcie_device), GFP_KERNEL);
@@ -8330,7 +9547,7 @@ _scsih_pcie_topology_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
  * Context: user.
  *
  */
-static void
+static int
 _scsih_pcie_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
 	struct fw_event_work *fw_event)
 {
@@ -8340,6 +9557,7 @@ _scsih_pcie_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
 	u8 link_rate, prev_link_rate;
 	unsigned long flags;
 	int rc;
+	int requeue_event;
 	Mpi26EventDataPCIeTopologyChangeList_t *event_data =
 		(Mpi26EventDataPCIeTopologyChangeList_t *) fw_event->event_data;
 	struct _pcie_device *pcie_device;
@@ -8349,22 +9567,22 @@ _scsih_pcie_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
 
 	if (ioc->shost_recovery || ioc->remove_host ||
 		ioc->pci_error_recovery)
-		return;
+		return 0;
 
 	if (fw_event->ignore) {
 		dewtprintk(ioc, ioc_info(ioc, "ignoring switch event\n"));
-		return;
+		return 0;
 	}
 
 	/* handle siblings events */
-	for (i = 0; i < event_data->NumEntries; i++) {
+	for (i = 0, requeue_event = 0; i < event_data->NumEntries; i++) {
 		if (fw_event->ignore) {
 			dewtprintk(ioc,
 				   ioc_info(ioc, "ignoring switch event\n"));
-			return;
+			return 0;
 		}
 		if (ioc->remove_host || ioc->pci_error_recovery)
-			return;
+			return 0;
 		reason_code = event_data->PortEntry[i].PortStatus;
 		handle =
 			le16_to_cpu(event_data->PortEntry[i].AttachedDevHandle);
@@ -8418,8 +9636,11 @@ _scsih_pcie_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
 			if (link_rate < MPI26_EVENT_PCIE_TOPO_PI_RATE_2_5)
 				break;
 
-			rc = _scsih_pcie_add_device(ioc, handle);
-			if (!rc) {
+			rc = _scsih_pcie_add_device(ioc, handle, fw_event->retries[i]);
+			if (rc) {/* retry due to busy device */
+				fw_event->retries[i]++;
+				requeue_event = 1;
+			} else {
 				/* mark entry vacant */
 				/* TODO This needs to be reviewed and fixed,
 				 * we dont have an entry
@@ -8434,11 +9655,12 @@ _scsih_pcie_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
 			break;
 		}
 	}
+	return requeue_event;
 }
 
 /**
  * _scsih_pcie_device_status_change_event_debug - debug for device event
- * @ioc: ?
+ * @ioc: per adapter object
  * @event_data: event data payload
  * Context: user.
  */
@@ -8810,7 +10032,7 @@ _scsih_sas_broadcast_primitive_event(struct MPT3SAS_ADAPTER *ioc,
 
 	ioc->broadcast_aen_busy = 0;
 	if (!ioc->shost_recovery)
-		_scsih_ublock_io_all_device(ioc);
+		_scsih_ublock_io_all_device(ioc, 1);
 	mutex_unlock(&ioc->tm_cmds.mutex);
 }
 
@@ -10344,7 +11566,7 @@ _scsih_remove_unresponding_devices(struct MPT3SAS_ADAPTER *ioc)
 	ioc_info(ioc, "removing unresponding devices: complete\n");
 
 	/* unblock devices */
-	_scsih_ublock_io_all_device(ioc);
+	_scsih_ublock_io_all_device(ioc, 0);
 }
 
 static void
@@ -10624,7 +11846,8 @@ _scsih_scan_for_devices_after_reset(struct MPT3SAS_ADAPTER *ioc)
 		}
 		retry_count = 0;
 		parent_handle = le16_to_cpu(pcie_device_pg0.ParentDevHandle);
-		_scsih_pcie_add_device(ioc, handle);
+		while (_scsih_pcie_add_device(ioc, handle, retry_count++))
+			ssleep(1);
 
 		ioc_info(ioc, "\tAFTER adding pcie end device: handle (0x%04x), wwid(0x%016llx)\n",
 			 handle, (u64)le64_to_cpu(pcie_device_pg0.WWID));
@@ -10768,7 +11991,11 @@ _mpt3sas_fw_work(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work *fw_event)
 		_scsih_turn_on_pfa_led(ioc, fw_event->device_handle);
 		break;
 	case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
-		_scsih_sas_topology_change_event(ioc, fw_event);
+		if (_scsih_sas_topology_change_event(ioc, fw_event)) {
+			_scsih_fw_event_requeue(ioc, fw_event, 1000);
+			ioc->current_event = NULL;
+			return;
+		}
 		break;
 	case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
 		if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
@@ -10808,7 +12035,11 @@ _mpt3sas_fw_work(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work *fw_event)
 		_scsih_pcie_enumeration_event(ioc, fw_event);
 		break;
 	case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
-		_scsih_pcie_topology_change_event(ioc, fw_event);
+		if (_scsih_pcie_topology_change_event(ioc, fw_event)) {
+			_scsih_fw_event_requeue(ioc, fw_event, 1000);
+			ioc->current_event = NULL;
+			return;
+		}
 		break;
 	}
 out:
@@ -10833,6 +12064,15 @@ _firmware_event_work(struct work_struct *work)
 	_mpt3sas_fw_work(fw_event->ioc, fw_event);
 }
 
+static void
+_firmware_event_work_delayed(struct work_struct *work)
+{
+	struct fw_event_work *fw_event = container_of(work,
+	    struct fw_event_work, delayed_work.work);
+
+	_mpt3sas_fw_work(fw_event->ioc, fw_event);
+}
+
 /**
  * mpt3sas_scsih_event_callback - firmware event handler (called at ISR time)
  * @ioc: per adapter object
@@ -11013,6 +12253,34 @@ mpt3sas_scsih_event_callback(struct MPT3SAS_ADAPTER *ioc, u8 msix_index,
 		return 1;
 	}
 
+	if (event == MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST) {
+		Mpi2EventDataSasTopologyChangeList_t *topo_event_data =
+		    (Mpi2EventDataSasTopologyChangeList_t *)
+		    mpi_reply->EventData;
+		fw_event->retries = kzalloc(topo_event_data->NumEntries,
+		    GFP_ATOMIC);
+		if (!fw_event->retries) {
+
+			ioc_err(ioc, "failure at %s:%d/%s()!\n",  __FILE__, __LINE__, __func__);
+			kfree(fw_event->event_data);
+			fw_event_work_put(fw_event);
+			return 1;
+		}
+	}
+
+	if (event == MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST) {
+		Mpi26EventDataPCIeTopologyChangeList_t *topo_event_data =
+			(Mpi26EventDataPCIeTopologyChangeList_t *) mpi_reply->EventData;
+		fw_event->retries = kzalloc(topo_event_data->NumEntries,
+			GFP_ATOMIC);
+		if (!fw_event->retries) {
+
+			ioc_err(ioc, "failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__);
+			fw_event_work_put(fw_event);
+			return 1;
+		}
+	}
+
 	memcpy(fw_event->event_data, mpi_reply->EventData, sz);
 	fw_event->ioc = ioc;
 	fw_event->VF_ID = mpi_reply->VF_ID;
diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c
index ccfc2d26dd37..2c44a379cb23 100644
--- a/drivers/scsi/qla2xxx/qla_bsg.c
+++ b/drivers/scsi/qla2xxx/qla_bsg.c
@@ -11,6 +11,8 @@
 #include <linux/delay.h>
 #include <linux/bsg-lib.h>
 
+static int qla28xx_validate_flash_image(struct bsg_job *bsg_job);
+
 static void qla2xxx_free_fcport_work(struct work_struct *work)
 {
 	struct fc_port *fcport = container_of(work, typeof(*fcport),
@@ -1546,8 +1548,9 @@ qla2x00_update_optrom(struct bsg_job *bsg_job)
 	ha->optrom_buffer = NULL;
 	ha->optrom_state = QLA_SWAITING;
 	mutex_unlock(&ha->optrom_mutex);
-	bsg_job_done(bsg_job, bsg_reply->result,
-		       bsg_reply->reply_payload_rcv_len);
+	if (!rval)
+		bsg_job_done(bsg_job, bsg_reply->result,
+			     bsg_reply->reply_payload_rcv_len);
 	return rval;
 }
 
@@ -2549,6 +2552,30 @@ qla2x00_get_flash_image_status(struct bsg_job *bsg_job)
 	return 0;
 }
 
+static int
+qla2x00_get_drv_attr(struct bsg_job *bsg_job)
+{
+	struct qla_drv_attr drv_attr;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+
+	memset(&drv_attr, 0, sizeof(struct qla_drv_attr));
+	drv_attr.ext_attributes |= QLA_IMG_SET_VALID_SUPPORT;
+
+
+	sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
+			bsg_job->reply_payload.sg_cnt, &drv_attr,
+			sizeof(struct qla_drv_attr));
+
+	bsg_reply->reply_payload_rcv_len = sizeof(struct qla_drv_attr);
+	bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_OK;
+
+	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
+	bsg_reply->result = DID_OK << 16;
+	bsg_job_done(bsg_job, bsg_reply->result, bsg_reply->reply_payload_rcv_len);
+
+	return 0;
+}
+
 static int
 qla2x00_manage_host_stats(struct bsg_job *bsg_job)
 {
@@ -2612,8 +2639,9 @@ qla2x00_manage_host_stats(struct bsg_job *bsg_job)
 				    sizeof(struct ql_vnd_mng_host_stats_resp));
 
 	bsg_reply->result = DID_OK;
-	bsg_job_done(bsg_job, bsg_reply->result,
-		     bsg_reply->reply_payload_rcv_len);
+	if (!ret)
+		bsg_job_done(bsg_job, bsg_reply->result,
+			     bsg_reply->reply_payload_rcv_len);
 
 	return ret;
 }
@@ -2702,8 +2730,9 @@ qla2x00_get_host_stats(struct bsg_job *bsg_job)
 							       bsg_job->reply_payload.sg_cnt,
 							       data, response_len);
 	bsg_reply->result = DID_OK;
-	bsg_job_done(bsg_job, bsg_reply->result,
-		     bsg_reply->reply_payload_rcv_len);
+	if (!ret)
+		bsg_job_done(bsg_job, bsg_reply->result,
+			     bsg_reply->reply_payload_rcv_len);
 
 	kfree(data);
 host_stat_out:
@@ -2802,8 +2831,9 @@ qla2x00_get_tgt_stats(struct bsg_job *bsg_job)
 				    bsg_job->reply_payload.sg_cnt, data,
 				    response_len);
 	bsg_reply->result = DID_OK;
-	bsg_job_done(bsg_job, bsg_reply->result,
-		     bsg_reply->reply_payload_rcv_len);
+	if (!ret)
+		bsg_job_done(bsg_job, bsg_reply->result,
+			     bsg_reply->reply_payload_rcv_len);
 
 tgt_stat_out:
 	kfree(data);
@@ -2864,8 +2894,9 @@ qla2x00_manage_host_port(struct bsg_job *bsg_job)
 				    bsg_job->reply_payload.sg_cnt, &rsp_data,
 				    sizeof(struct ql_vnd_mng_host_port_resp));
 	bsg_reply->result = DID_OK;
-	bsg_job_done(bsg_job, bsg_reply->result,
-		     bsg_reply->reply_payload_rcv_len);
+	if (!ret)
+		bsg_job_done(bsg_job, bsg_reply->result,
+			     bsg_reply->reply_payload_rcv_len);
 
 	return ret;
 }
@@ -2933,6 +2964,12 @@ qla2x00_process_vendor_specific(struct scsi_qla_host *vha, struct bsg_job *bsg_j
 	case QL_VND_GET_FLASH_UPDATE_CAPS:
 		return qla27xx_get_flash_upd_cap(bsg_job);
 
+	case QL_VND_GET_DRV_ATTR:
+		return qla2x00_get_drv_attr(bsg_job);
+
+	case QL_VND_IMG_SET_VALID:
+		return qla28xx_validate_flash_image(bsg_job);
+
 	case QL_VND_SET_FLASH_UPDATE_CAPS:
 		return qla27xx_set_flash_upd_cap(bsg_job);
 
@@ -3240,9 +3277,97 @@ int qla2x00_mailbox_passthru(struct bsg_job *bsg_job)
 
 	bsg_job->reply_len = sizeof(*bsg_job->reply);
 	bsg_reply->result = DID_OK << 16;
-	bsg_job_done(bsg_job, bsg_reply->result, bsg_reply->reply_payload_rcv_len);
+	if (!ret)
+		bsg_job_done(bsg_job, bsg_reply->result, bsg_reply->reply_payload_rcv_len);
 
 	kfree(req_data);
 
 	return ret;
 }
+
+static int
+qla28xx_do_validate_flash_image(struct bsg_job *bsg_job, uint16_t *state)
+{
+	struct fc_bsg_request *bsg_request = bsg_job->request;
+	scsi_qla_host_t *vha = shost_priv(fc_bsg_to_shost(bsg_job));
+	uint16_t mstate[16];
+	uint16_t mpi_state = 0;
+	uint16_t img_idx;
+	int rval = QLA_SUCCESS;
+
+	memset(mstate, 0, sizeof(mstate));
+
+	rval = qla2x00_get_firmware_state(vha, mstate);
+	if (rval != QLA_SUCCESS) {
+		ql_log(ql_log_warn, vha, 0xffff,
+				"MBC to get MPI state failed (%d)\n", rval);
+		rval = -EINVAL;
+		goto exit_flash_img;
+	}
+
+	mpi_state = mstate[11];
+
+	if (!(mpi_state & BIT_9 && mpi_state & BIT_8 && mpi_state & BIT_15)) {
+		ql_log(ql_log_warn, vha, 0xffff,
+				"MPI firmware state failed (0x%02x)\n", mpi_state);
+		rval = -EINVAL;
+		goto exit_flash_img;
+	}
+
+	rval = qla81xx_fac_semaphore_access(vha, FAC_SEMAPHORE_LOCK);
+	if (rval != QLA_SUCCESS) {
+		ql_log(ql_log_warn, vha, 0xffff,
+				"Unable to lock flash semaphore.");
+		goto exit_flash_img;
+	}
+
+	img_idx = bsg_request->rqst_data.h_vendor.vendor_cmd[1];
+
+	rval = qla_mpipt_validate_fw(vha, img_idx, state);
+	if (rval != QLA_SUCCESS) {
+		ql_log(ql_log_warn, vha, 0xffff,
+				"Failed to validate Firmware image index [0x%x].\n",
+				img_idx);
+	}
+
+	qla81xx_fac_semaphore_access(vha, FAC_SEMAPHORE_UNLOCK);
+
+exit_flash_img:
+	return rval;
+}
+
+static int qla28xx_validate_flash_image(struct bsg_job *bsg_job)
+{
+	scsi_qla_host_t *vha = shost_priv(fc_bsg_to_shost(bsg_job));
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct qla_hw_data *ha = vha->hw;
+	uint16_t state = 0;
+	int rval = 0;
+
+	if (!IS_QLA28XX(ha) || vha->vp_idx != 0)
+		return -EPERM;
+
+	mutex_lock(&ha->optrom_mutex);
+	rval = qla28xx_do_validate_flash_image(bsg_job, &state);
+	if (rval)
+		rval = -EINVAL;
+	mutex_unlock(&ha->optrom_mutex);
+
+	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
+
+	if (rval)
+		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
+			(state == 39) ? EXT_STATUS_IMG_SET_VALID_ERR :
+			EXT_STATUS_IMG_SET_CONFIG_ERR;
+	else
+		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_OK;
+
+	bsg_reply->result = DID_OK << 16;
+	bsg_reply->reply_payload_rcv_len = 0;
+	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
+	if (!rval)
+		bsg_job_done(bsg_job, bsg_reply->result,
+			     bsg_reply->reply_payload_rcv_len);
+
+	return QLA_SUCCESS;
+}
diff --git a/drivers/scsi/qla2xxx/qla_bsg.h b/drivers/scsi/qla2xxx/qla_bsg.h
index d38dab0a07e8..a920c8e482bc 100644
--- a/drivers/scsi/qla2xxx/qla_bsg.h
+++ b/drivers/scsi/qla2xxx/qla_bsg.h
@@ -32,12 +32,14 @@
 #define QL_VND_GET_PRIV_STATS_EX	0x1A
 #define QL_VND_SS_GET_FLASH_IMAGE_STATUS	0x1E
 #define QL_VND_EDIF_MGMT                0X1F
+#define QL_VND_GET_DRV_ATTR		0x22
 #define QL_VND_MANAGE_HOST_STATS	0x23
 #define QL_VND_GET_HOST_STATS		0x24
 #define QL_VND_GET_TGT_STATS		0x25
 #define QL_VND_MANAGE_HOST_PORT		0x26
 #define QL_VND_MBX_PASSTHRU		0x2B
 #define QL_VND_DPORT_DIAGNOSTICS_V2	0x2C
+#define QL_VND_IMG_SET_VALID	0x30
 
 /* BSG Vendor specific subcode returns */
 #define EXT_STATUS_OK			0
@@ -50,6 +52,8 @@
 #define EXT_STATUS_BUFFER_TOO_SMALL	16
 #define EXT_STATUS_NO_MEMORY		17
 #define EXT_STATUS_DEVICE_OFFLINE	22
+#define EXT_STATUS_IMG_SET_VALID_ERR	47
+#define EXT_STATUS_IMG_SET_CONFIG_ERR	48
 
 /*
  * To support bidirectional iocb
@@ -318,6 +322,14 @@ struct qla_active_regions {
 	uint8_t reserved[31];
 } __packed;
 
+struct qla_drv_attr {
+        uint32_t        attributes;
+        u32             ext_attributes;
+#define QLA_IMG_SET_VALID_SUPPORT       BIT_4
+        u32             status_flags;
+        uint8_t         reserved[20];
+} __packed;
+
 #include "qla_edif_bsg.h"
 
 #endif
diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index b3265952c4be..5593ad7fad27 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -1270,6 +1270,7 @@ static inline bool qla2xxx_is_valid_mbs(unsigned int mbs)
  */
 #define MBC_LOAD_RAM			1	/* Load RAM. */
 #define MBC_EXECUTE_FIRMWARE		2	/* Execute firmware. */
+#define MBC_LOAD_FLASH_FIRMWARE		3	/* Load flash firmware. */
 #define MBC_READ_RAM_WORD		5	/* Read RAM word. */
 #define MBC_MAILBOX_REGISTER_TEST	6	/* Wrap incoming mailboxes */
 #define MBC_VERIFY_CHECKSUM		7	/* Verify checksum. */
@@ -1385,6 +1386,26 @@ static inline bool qla2xxx_is_valid_mbs(unsigned int mbs)
 #define HCS_WRITE_SERDES		0x3
 #define HCS_READ_SERDES			0x4
 
+/*
+ * ISP2[7|8]xx mailbox commands.
+ */
+#define MBC_MPI_PASSTHROUGH            0x200
+
+/* MBC_MPI_PASSTHROUGH */
+#define MPIPT_REQ_V1 1
+enum {
+       MPIPT_SUBCMD_GET_SUP_CMD = 0x10,
+       MPIPT_SUBCMD_GET_SUP_FEATURE,
+       MPIPT_SUBCMD_GET_STATUS,
+       MPIPT_SUBCMD_VALIDATE_FW,
+};
+
+enum {
+       MPIPT_MPI_STATUS = 1,
+       MPIPT_FCORE_STATUS,
+       MPIPT_LOCKDOWN_STATUS,
+};
+
 /* Firmware return data sizes */
 #define FCAL_MAP_SIZE	128
 
@@ -4149,6 +4170,7 @@ struct qla_hw_data {
 		uint32_t	eeh_flush:2;
 #define EEH_FLUSH_RDY  1
 #define EEH_FLUSH_DONE 2
+		uint32_t	secure_mcu:1;
 	} flags;
 
 	uint16_t max_exchg;
@@ -4414,6 +4436,8 @@ struct qla_hw_data {
 	((IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) &&\
 	 (ha->zio_mode == QLA_ZIO_MODE_6))
 
+#define IS_QLA28XX_SECURED(ha)	(IS_QLA28XX(ha) && ha->flags.secure_mcu)
+
 	/* HBA serial number */
 	uint8_t		serial0;
 	uint8_t		serial1;
@@ -5368,7 +5392,7 @@ struct edif_sa_index_entry {
 	struct list_head next;
 };
 
-/* Refer to SNIA SFF 8247 */
+/* Refer to SNIA SFF 8472 */
 struct sff_8247_a0 {
 	u8 txid;	/* transceiver id */
 	u8 ext_txid;
@@ -5412,6 +5436,7 @@ struct sff_8247_a0 {
 #define FC_SP_32 BIT_3
 #define FC_SP_2  BIT_2
 #define FC_SP_1  BIT_0
+#define FC_SPEED_2	BIT_1
 	u8 fc_sp_cc10;
 	u8 encode;
 	u8 bitrate;
@@ -5430,7 +5455,8 @@ struct sff_8247_a0 {
 	u8 vendor_pn[SFF_PART_NAME_LEN];	/* part number */
 	u8 vendor_rev[4];
 	u8 wavelength[2];
-	u8 resv;
+#define FC_SP_64	BIT_0
+	u8 fiber_channel_speed2;
 	u8 cc_base;
 	u8 options[2];	/* offset 64 */
 	u8 br_max;
diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h
index 55d531c19e6b..9e328c235e39 100644
--- a/drivers/scsi/qla2xxx/qla_gbl.h
+++ b/drivers/scsi/qla2xxx/qla_gbl.h
@@ -344,6 +344,9 @@ qla2x00_dump_ram(scsi_qla_host_t *, dma_addr_t, uint32_t, uint32_t);
 extern int
 qla2x00_execute_fw(scsi_qla_host_t *, uint32_t);
 
+extern int
+qla28xx_load_flash_firmware(scsi_qla_host_t *vha);
+
 extern int
 qla2x00_get_fw_version(scsi_qla_host_t *);
 
@@ -839,6 +842,8 @@ extern int qla82xx_write_optrom_data(struct scsi_qla_host *, void *,
 extern int qla82xx_abort_isp(scsi_qla_host_t *);
 extern int qla82xx_restart_isp(scsi_qla_host_t *);
 
+extern int qla_mpipt_validate_fw(scsi_qla_host_t *vha, u16 img_idx, u16 *state);
+
 /* IOCB related functions */
 extern int qla82xx_start_scsi(srb_t *);
 extern void qla2x00_sp_free(srb_t *sp);
diff --git a/drivers/scsi/qla2xxx/qla_gs.c b/drivers/scsi/qla2xxx/qla_gs.c
index 51c7cea71f90..880cd73feaca 100644
--- a/drivers/scsi/qla2xxx/qla_gs.c
+++ b/drivers/scsi/qla2xxx/qla_gs.c
@@ -3266,9 +3266,6 @@ void qla_fab_scan_finish(scsi_qla_host_t *vha, srb_t *sp)
 			    atomic_read(&fcport->state) == FCS_ONLINE) ||
 				do_delete) {
 				if (fcport->loop_id != FC_NO_LOOP_ID) {
-					if (fcport->flags & FCF_FCP2_DEVICE)
-						continue;
-
 					ql_log(ql_log_warn, vha, 0x20f0,
 					       "%s %d %8phC post del sess\n",
 					       __func__, __LINE__,
@@ -3535,8 +3532,8 @@ int qla_fab_async_scan(scsi_qla_host_t *vha, srb_t *sp)
 	if (vha->scan.scan_flags & SF_SCANNING) {
 		spin_unlock_irqrestore(&vha->work_lock, flags);
 		ql_dbg(ql_dbg_disc + ql_dbg_verbose, vha, 0x2012,
-		    "%s: scan active\n", __func__);
-		return rval;
+		    "%s: scan active for sp:%p\n", __func__, sp);
+		goto done_free_sp;
 	}
 	vha->scan.scan_flags |= SF_SCANNING;
 	if (!sp)
@@ -3701,23 +3698,25 @@ int qla_fab_async_scan(scsi_qla_host_t *vha, srb_t *sp)
 	return rval;
 
 done_free_sp:
-	if (sp->u.iocb_cmd.u.ctarg.req) {
-		dma_free_coherent(&vha->hw->pdev->dev,
-		    sp->u.iocb_cmd.u.ctarg.req_allocated_size,
-		    sp->u.iocb_cmd.u.ctarg.req,
-		    sp->u.iocb_cmd.u.ctarg.req_dma);
-		sp->u.iocb_cmd.u.ctarg.req = NULL;
-	}
-	if (sp->u.iocb_cmd.u.ctarg.rsp) {
-		dma_free_coherent(&vha->hw->pdev->dev,
-		    sp->u.iocb_cmd.u.ctarg.rsp_allocated_size,
-		    sp->u.iocb_cmd.u.ctarg.rsp,
-		    sp->u.iocb_cmd.u.ctarg.rsp_dma);
-		sp->u.iocb_cmd.u.ctarg.rsp = NULL;
-	}
+	if (sp) {
+		if (sp->u.iocb_cmd.u.ctarg.req) {
+			dma_free_coherent(&vha->hw->pdev->dev,
+			    sp->u.iocb_cmd.u.ctarg.req_allocated_size,
+			    sp->u.iocb_cmd.u.ctarg.req,
+			    sp->u.iocb_cmd.u.ctarg.req_dma);
+			sp->u.iocb_cmd.u.ctarg.req = NULL;
+		}
+		if (sp->u.iocb_cmd.u.ctarg.rsp) {
+			dma_free_coherent(&vha->hw->pdev->dev,
+			    sp->u.iocb_cmd.u.ctarg.rsp_allocated_size,
+			    sp->u.iocb_cmd.u.ctarg.rsp,
+			    sp->u.iocb_cmd.u.ctarg.rsp_dma);
+			sp->u.iocb_cmd.u.ctarg.rsp = NULL;
+		}
 
-	/* ref: INIT */
-	kref_put(&sp->cmd_kref, qla2x00_sp_release);
+		/* ref: INIT */
+		kref_put(&sp->cmd_kref, qla2x00_sp_release);
+	}
 
 	spin_lock_irqsave(&vha->work_lock, flags);
 	vha->scan.scan_flags &= ~SF_SCANNING;
diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c
index d395cbfe6802..689f909943b4 100644
--- a/drivers/scsi/qla2xxx/qla_init.c
+++ b/drivers/scsi/qla2xxx/qla_init.c
@@ -1859,15 +1859,6 @@ void qla2x00_handle_rscn(scsi_qla_host_t *vha, struct event_arg *ea)
 	case RSCN_PORT_ADDR:
 		fcport = qla2x00_find_fcport_by_nportid(vha, &ea->id, 1);
 		if (fcport) {
-			if (ql2xfc2target &&
-			    fcport->flags & FCF_FCP2_DEVICE &&
-			    atomic_read(&fcport->state) == FCS_ONLINE) {
-				ql_dbg(ql_dbg_disc, vha, 0x2115,
-				       "Delaying session delete for FCP2 portid=%06x %8phC ",
-					fcport->d_id.b24, fcport->port_name);
-				return;
-			}
-
 			if (vha->hw->flags.edif_enabled && DBELL_ACTIVE(vha)) {
 				/*
 				 * On ipsec start by remote port, Target port
@@ -2471,8 +2462,23 @@ qla24xx_handle_plogi_done_event(struct scsi_qla_host *vha, struct event_arg *ea)
 	    ea->sp->gen1, fcport->rscn_gen,
 	    ea->data[0], ea->data[1], ea->iop[0], ea->iop[1]);
 
-	if ((fcport->fw_login_state == DSC_LS_PLOGI_PEND) ||
-	    (fcport->fw_login_state == DSC_LS_PRLI_PEND)) {
+	if (fcport->fw_login_state == DSC_LS_PLOGI_PEND) {
+		ql_dbg(ql_dbg_disc, vha, 0x20ea,
+		    "%s %d %8phC Remote is trying to login\n",
+		    __func__, __LINE__, fcport->port_name);
+		/*
+		 * If we get here, there is port thats already logged in,
+		 * but it's state has not moved ahead. Recheck with FW on
+		 * what state it is in and proceed ahead
+		 */
+		if (!N2N_TOPO(vha->hw)) {
+			fcport->fw_login_state = DSC_LS_PRLI_COMP;
+			qla24xx_post_gpdb_work(vha, fcport, 0);
+		}
+		return;
+	}
+
+	if (fcport->fw_login_state == DSC_LS_PRLI_PEND) {
 		ql_dbg(ql_dbg_disc, vha, 0x20ea,
 		    "%s %d %8phC Remote is trying to login\n",
 		    __func__, __LINE__, fcport->port_name);
@@ -4074,6 +4080,22 @@ static void qla2xxx_print_sfp_info(struct scsi_qla_host *vha)
 	u8 str[STR_LEN], *ptr, p;
 	int leftover, len;
 
+	ql_dbg(ql_dbg_init, vha, 0x015a,
+	    "SFP: %.*s -> %.*s ->%s%s%s%s%s%s%s\n",
+	    (int)sizeof(a0->vendor_name), a0->vendor_name,
+	    (int)sizeof(a0->vendor_pn), a0->vendor_pn,
+	    a0->fc_sp_cc10 & FC_SP_2 ? a0->fiber_channel_speed2  &  FC_SP_64 ?
+					" 64G" : "" : "",
+	    a0->fc_sp_cc10 & FC_SP_32 ? " 32G" : "",
+	    a0->fc_sp_cc10 & FC_SP_16 ? " 16G" : "",
+	    a0->fc_sp_cc10 & FC_SP_8  ?  " 8G" : "",
+	    a0->fc_sp_cc10 & FC_SP_4  ?  " 4G" : "",
+	    a0->fc_sp_cc10 & FC_SP_2  ?  " 2G" : "",
+	    a0->fc_sp_cc10 & FC_SP_1  ?  " 1G" : "");
+
+	if (!(ql2xextended_error_logging & ql_dbg_verbose))
+		return;
+
 	memset(str, 0, STR_LEN);
 	snprintf(str, SFF_VEN_NAME_LEN+1, a0->vendor_name);
 	ql_dbg(ql_dbg_init, vha, 0x015a,
@@ -8442,6 +8464,148 @@ bool qla24xx_risc_firmware_invalid(uint32_t *dword)
 	    !(~dword[4] | ~dword[5] | ~dword[6] | ~dword[7]);
 }
 
+static int
+qla28xx_get_srisc_addr(scsi_qla_host_t *vha, uint32_t *srisc_addr,
+		       uint32_t faddr)
+{
+	struct qla_hw_data *ha = vha->hw;
+	struct req_que *req = ha->req_q_map[0];
+	uint32_t *dcode;
+	int rval;
+
+	*srisc_addr = 0;
+	dcode = (uint32_t *)req->ring;
+
+	rval = qla24xx_read_flash_data(vha, dcode, faddr, 10);
+	if (rval) {
+		ql_log(ql_log_fatal, vha, 0x01aa,
+		    "-> Failed to read flash addr + size .\n");
+		return QLA_FUNCTION_FAILED;
+	}
+
+	*srisc_addr = be32_to_cpu((__force __be32)dcode[2]);
+	return QLA_SUCCESS;
+}
+
+static int
+qla28xx_load_fw_template(scsi_qla_host_t *vha, uint32_t faddr)
+{
+	struct qla_hw_data *ha = vha->hw;
+	struct fwdt *fwdt = ha->fwdt;
+	struct req_que *req = ha->req_q_map[0];
+	uint32_t risc_size, risc_attr = 0;
+	uint templates, segments, fragment;
+	uint32_t *dcode;
+	ulong dlen;
+	int rval;
+	uint j;
+
+	dcode = (uint32_t *)req->ring;
+	segments = FA_RISC_CODE_SEGMENTS;
+
+	for (j = 0; j < segments; j++) {
+		rval = qla24xx_read_flash_data(vha, dcode, faddr, 10);
+		if (rval) {
+			ql_log(ql_log_fatal, vha, 0x01a1,
+			       "-> Failed to read flash addr + size .\n");
+			return QLA_FUNCTION_FAILED;
+		}
+
+		risc_size = be32_to_cpu((__force __be32)dcode[3]);
+
+		if (risc_attr == 0)
+			risc_attr = be32_to_cpu((__force __be32)dcode[9]);
+
+		dlen = ha->fw_transfer_size >> 2;
+		for (fragment = 0; fragment < risc_size; fragment++) {
+			if (dlen > risc_size)
+				dlen = risc_size;
+
+			faddr += dlen;
+			risc_size -= dlen;
+		}
+	}
+
+	templates = (risc_attr & BIT_9) ? 2 : 1;
+
+	ql_dbg(ql_dbg_init, vha, 0x01a1, "-> templates = %u\n", templates);
+
+	for (j = 0; j < templates; j++, fwdt++) {
+		vfree(fwdt->template);
+		fwdt->template = NULL;
+		fwdt->length = 0;
+
+		dcode = (uint32_t *)req->ring;
+
+		rval = qla24xx_read_flash_data(vha, dcode, faddr, 7);
+		if (rval) {
+			ql_log(ql_log_fatal, vha, 0x01a2,
+			    "-> Unable to read template size.\n");
+			goto failed;
+		}
+
+		risc_size = be32_to_cpu((__force __be32)dcode[2]);
+		ql_dbg(ql_dbg_init, vha, 0x01a3,
+		    "-> fwdt%u template array at %#x (%#x dwords)\n",
+		    j, faddr, risc_size);
+		if (!risc_size || !~risc_size) {
+			ql_dbg(ql_dbg_init, vha, 0x01a4,
+			    "-> fwdt%u failed to read array\n", j);
+			goto failed;
+		}
+
+		/* skip header and ignore checksum */
+		faddr += 7;
+		risc_size -= 8;
+
+		ql_dbg(ql_dbg_init, vha, 0x01a5,
+		    "-> fwdt%u template allocate template %#x words...\n",
+		    j, risc_size);
+		fwdt->template = vmalloc(risc_size * sizeof(*dcode));
+		if (!fwdt->template) {
+			ql_log(ql_log_warn, vha, 0x01a6,
+			    "-> fwdt%u failed allocate template.\n", j);
+			goto failed;
+		}
+
+		dcode = fwdt->template;
+		rval = qla24xx_read_flash_data(vha, dcode, faddr, risc_size);
+
+		if (rval || !qla27xx_fwdt_template_valid(dcode)) {
+			ql_log(ql_log_warn, vha, 0x01a7,
+			    "-> fwdt%u failed template validate (rval %x)\n",
+			    j, rval);
+			goto failed;
+		}
+
+		dlen = qla27xx_fwdt_template_size(dcode);
+		ql_dbg(ql_dbg_init, vha, 0x01a7,
+		    "-> fwdt%u template size %#lx bytes (%#lx words)\n",
+		    j, dlen, dlen / sizeof(*dcode));
+		if (dlen > risc_size * sizeof(*dcode)) {
+			ql_log(ql_log_warn, vha, 0x01a8,
+			    "-> fwdt%u template exceeds array (%-lu bytes)\n",
+			    j, dlen - risc_size * sizeof(*dcode));
+			goto failed;
+		}
+
+		fwdt->length = dlen;
+		ql_dbg(ql_dbg_init, vha, 0x01a9,
+		    "-> fwdt%u loaded template ok\n", j);
+
+		faddr += risc_size + 1;
+	}
+
+	return QLA_SUCCESS;
+
+failed:
+	vfree(fwdt->template);
+	fwdt->template = NULL;
+	fwdt->length = 0;
+
+	return QLA_SUCCESS;
+}
+
 static int
 qla24xx_load_risc_flash(scsi_qla_host_t *vha, uint32_t *srisc_addr,
     uint32_t faddr)
@@ -8881,16 +9045,18 @@ int
 qla81xx_load_risc(scsi_qla_host_t *vha, uint32_t *srisc_addr)
 {
 	int rval;
+	uint32_t f_region = 0;
 	struct qla_hw_data *ha = vha->hw;
 	struct active_regions active_regions = { };
 
-	if (ql2xfwloadbin == 2)
+	if (ql2xfwloadbin == 2 && !IS_QLA28XX(ha))
 		goto try_blob_fw;
 
 	/* FW Load priority:
-	 * 1) Firmware residing in flash.
-	 * 2) Firmware via request-firmware interface (.bin file).
-	 * 3) Golden-Firmware residing in flash -- (limited operation).
+	 * 1) If 28xxx, ROM cmd to load flash firmware.
+	 * 2) Firmware residing in flash.
+	 * 3) Firmware via request-firmware interface (.bin file).
+	 * 4) Golden-Firmware residing in flash -- (limited operation).
 	 */
 
 	if (!IS_QLA27XX(ha) && !IS_QLA28XX(ha))
@@ -8898,6 +9064,40 @@ qla81xx_load_risc(scsi_qla_host_t *vha, uint32_t *srisc_addr)
 
 	qla27xx_get_active_image(vha, &active_regions);
 
+	/* For 28XXX, always load the flash firmware using rom mbx */
+	if (IS_QLA28XX_SECURED(ha)) {
+		rval = qla28xx_load_flash_firmware(vha);
+		if (rval != QLA_SUCCESS) {
+			ql_log(ql_log_fatal, vha, 0x019e,
+			       "Failed to load flash firmware.\n");
+			goto exit_load_risc;
+		}
+
+		f_region =
+		(active_regions.global != QLA27XX_SECONDARY_IMAGE) ?
+		 ha->flt_region_fw : ha->flt_region_fw_sec;
+
+		ql_log(ql_log_info, vha, 0x019f,
+		       "Load flash firmware successful (%s).\n",
+		       ((active_regions.global != QLA27XX_SECONDARY_IMAGE) ?
+		       "Primary" : "Secondary"));
+
+		rval = qla28xx_get_srisc_addr(vha, srisc_addr, f_region);
+		if (rval != QLA_SUCCESS) {
+			ql_log(ql_log_warn, vha, 0x019f,
+			       "failed to read srisc address\n");
+			goto exit_load_risc;
+		}
+
+		rval = qla28xx_load_fw_template(vha, f_region);
+		if (rval != QLA_SUCCESS) {
+			ql_log(ql_log_warn, vha, 0x01a0,
+			       "failed to read firmware template\n");
+		}
+
+		goto exit_load_risc;
+	}
+
 	if (active_regions.global != QLA27XX_SECONDARY_IMAGE)
 		goto try_primary_fw;
 
@@ -8927,6 +9127,8 @@ qla81xx_load_risc(scsi_qla_host_t *vha, uint32_t *srisc_addr)
 
 	ql_log(ql_log_info, vha, 0x009a, "Need firmware flash update.\n");
 	ha->flags.running_gold_fw = 1;
+
+exit_load_risc:
 	return rval;
 }
 
diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c
index a3971afc2dd1..a4dda4fcb52c 100644
--- a/drivers/scsi/qla2xxx/qla_isr.c
+++ b/drivers/scsi/qla2xxx/qla_isr.c
@@ -1669,13 +1669,28 @@ qla2x00_async_event(scsi_qla_host_t *vha, struct rsp_que *rsp, uint16_t *mb)
 
 			/* Port logout */
 			fcport = qla2x00_find_fcport_by_loopid(vha, mb[1]);
-			if (!fcport)
+			if (!fcport) {
+				ql_dbg(ql_dbg_async, vha, 0x5011,
+					"Could not find fcport:%04x %04x %04x\n",
+					mb[1], mb[2], mb[3]);
 				break;
-			if (atomic_read(&fcport->state) != FCS_ONLINE)
+			}
+
+			if (atomic_read(&fcport->state) != FCS_ONLINE) {
+				ql_dbg(ql_dbg_async, vha, 0x5012,
+					"Port state is not online State:0x%x \n",
+					atomic_read(&fcport->state));
+				ql_dbg(ql_dbg_async, vha, 0x5012,
+					"Scheduling session for deletion \n");
+				fcport->logout_on_delete = 0;
+				qlt_schedule_sess_for_deletion(fcport);
 				break;
+			}
+
 			ql_dbg(ql_dbg_async, vha, 0x508a,
 			    "Marking port lost loopid=%04x portid=%06x.\n",
 			    fcport->loop_id, fcport->d_id.b24);
+
 			if (qla_ini_mode_enabled(vha)) {
 				fcport->logout_on_delete = 0;
 				qlt_schedule_sess_for_deletion(fcport);
diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c
index 1f01576f044b..0d598be6f3ea 100644
--- a/drivers/scsi/qla2xxx/qla_mbx.c
+++ b/drivers/scsi/qla2xxx/qla_mbx.c
@@ -43,6 +43,7 @@ static struct rom_cmd {
 } rom_cmds[] = {
 	{ MBC_LOAD_RAM },
 	{ MBC_EXECUTE_FIRMWARE },
+	{ MBC_LOAD_FLASH_FIRMWARE },
 	{ MBC_READ_RAM_WORD },
 	{ MBC_MAILBOX_REGISTER_TEST },
 	{ MBC_VERIFY_CHECKSUM },
@@ -824,6 +825,53 @@ qla2x00_execute_fw(scsi_qla_host_t *vha, uint32_t risc_addr)
 	return rval;
 }
 
+/*
+ * qla2x00_load_flash_firmware
+ *	Load firmware from flash.
+ *
+ * Input:
+ *	vha = adapter block pointer.
+ *
+ * Returns:
+ *	qla28xx local function return status code.
+ *
+ * Context:
+ *	Kernel context.
+ */
+int
+qla28xx_load_flash_firmware(scsi_qla_host_t *vha)
+{
+	struct qla_hw_data *ha = vha->hw;
+	int rval = QLA_COMMAND_ERROR;
+	mbx_cmd_t mc;
+	mbx_cmd_t *mcp = &mc;
+
+	if (!IS_QLA28XX(ha))
+		return rval;
+
+	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x11a6,
+	       "Entered %s.\n", __func__);
+
+	mcp->mb[0] = MBC_LOAD_FLASH_FIRMWARE;
+	mcp->out_mb = MBX_2 | MBX_1 | MBX_0;
+	mcp->in_mb = MBX_0;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+	rval = qla2x00_mailbox_command(vha, mcp);
+
+	if (rval != QLA_SUCCESS) {
+		ql_dbg(ql_log_info, vha, 0x11a7,
+		       "Failed=%x cmd error=%x img error=%x.\n",
+		       rval, mcp->mb[1], mcp->mb[2]);
+	} else {
+		ql_dbg(ql_log_info, vha, 0x11a8,
+		       "Done %s.\n", __func__);
+	}
+
+	return rval;
+}
+
+
 /*
  * qla_get_exlogin_status
  *	Get extended login status
@@ -7157,3 +7205,43 @@ int qla_mailbox_passthru(scsi_qla_host_t *vha,
 
 	return rval;
 }
+
+int qla_mpipt_validate_fw(scsi_qla_host_t *vha, u16 img_idx, uint16_t *state)
+{
+	struct qla_hw_data *ha = vha->hw;
+	mbx_cmd_t mc;
+	mbx_cmd_t *mcp = &mc;
+	int rval;
+
+	if (!IS_QLA28XX(ha)) {
+		ql_dbg(ql_dbg_mbx, vha, 0xffff, "%s %d\n", __func__, __LINE__);
+		return QLA_FUNCTION_FAILED;
+	}
+
+	if (img_idx > 1) {
+		ql_log(ql_log_info, vha, 0xffff,
+				"%s %d Invalid flash image index [%d]\n",
+				__func__, __LINE__, img_idx);
+		return QLA_INVALID_COMMAND;
+	}
+
+	memset(&mc, 0, sizeof(mc));
+	mcp->mb[0] = MBC_MPI_PASSTHROUGH;
+	mcp->mb[1] = MPIPT_SUBCMD_VALIDATE_FW;
+	mcp->mb[2] = img_idx;
+	mcp->out_mb = MBX_1|MBX_0;
+	mcp->in_mb = MBX_2|MBX_1|MBX_0;
+
+	/* send mb via iocb */
+	rval = qla24xx_send_mb_cmd(vha, &mc);
+	if (rval) {
+		ql_log(ql_log_info, vha, 0xffff, "%s:Failed %x (mb=%x,%x)\n",
+				__func__, rval, mcp->mb[0], mcp->mb[1]);
+		*state = mcp->mb[1];
+	} else {
+		ql_log(ql_log_info, vha, 0xffff, "%s: mb=%x,%x,%x\n", __func__,
+				mcp->mb[0], mcp->mb[1], mcp->mb[2]);
+	}
+
+	return rval;
+}
diff --git a/drivers/scsi/qla2xxx/qla_nx.h b/drivers/scsi/qla2xxx/qla_nx.h
index 5d1bdc15b75c..8e7a7f5f0adb 100644
--- a/drivers/scsi/qla2xxx/qla_nx.h
+++ b/drivers/scsi/qla2xxx/qla_nx.h
@@ -892,6 +892,7 @@ struct ct6_dsd {
 #define	FA_VPD_SIZE_82XX	0x400
 
 #define FA_FLASH_LAYOUT_ADDR_82	0xFC400
+#define FA_FLASH_MCU_OFF	0x13000
 
 /******************************************************************************
 *
diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index 16a44c0917e1..c9aff70e7357 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -1183,7 +1183,8 @@ qla2x00_wait_for_hba_ready(scsi_qla_host_t *vha)
 	while ((qla2x00_reset_active(vha) || ha->dpc_active ||
 		ha->flags.mbox_busy) ||
 	       test_bit(FX00_RESET_RECOVERY, &vha->dpc_flags) ||
-	       test_bit(FX00_TARGET_SCAN, &vha->dpc_flags)) {
+	       test_bit(FX00_TARGET_SCAN, &vha->dpc_flags) ||
+	       (vha->scan.scan_flags & SF_SCANNING)) {
 		if (test_bit(UNLOADING, &base_vha->dpc_flags))
 			break;
 		msleep(1000);
diff --git a/drivers/scsi/qla2xxx/qla_sup.c b/drivers/scsi/qla2xxx/qla_sup.c
index 9e7a407ba1b9..b6c36a8a2d60 100644
--- a/drivers/scsi/qla2xxx/qla_sup.c
+++ b/drivers/scsi/qla2xxx/qla_sup.c
@@ -1084,6 +1084,32 @@ qla2xxx_get_idc_param(scsi_qla_host_t *vha)
 	return;
 }
 
+static int qla28xx_validate_mcu_signature(scsi_qla_host_t *vha)
+{
+	struct qla_hw_data *ha = vha->hw;
+	struct req_que *req = ha->req_q_map[0];
+	uint32_t *dcode = (uint32_t *)req->ring;
+	uint32_t signature[2] = {0x000c0000, 0x00050000};
+	int ret = QLA_SUCCESS;
+
+	ret = qla24xx_read_flash_data(vha, dcode, FA_FLASH_MCU_OFF >> 2, 2);
+	if (ret) {
+		ql_log(ql_log_fatal, vha, 0x01ab,
+		       "-> Failed to read flash mcu signature.\n");
+		ret = QLA_FUNCTION_FAILED;
+		goto done;
+	}
+
+	ql_dbg(ql_dbg_init, vha, 0x01ac,
+		"Flash data 0x%08x 0x%08x.\n", dcode[0], dcode[1]);
+
+	if (!(dcode[0] == signature[0] && dcode[1] == signature[1]))
+		ret = QLA_FUNCTION_FAILED;
+
+done:
+	return ret;
+}
+
 int
 qla2xxx_get_flash_info(scsi_qla_host_t *vha)
 {
@@ -1096,6 +1122,9 @@ qla2xxx_get_flash_info(scsi_qla_host_t *vha)
 	    !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 		return QLA_SUCCESS;
 
+	if (IS_QLA28XX(ha) && !qla28xx_validate_mcu_signature(vha))
+		ha->flags.secure_mcu = 1;
+
 	ret = qla2xxx_find_flt_start(vha, &flt_addr);
 	if (ret != QLA_SUCCESS)
 		return ret;
diff --git a/drivers/scsi/qla2xxx/qla_version.h b/drivers/scsi/qla2xxx/qla_version.h
index a491d6ee5c94..9564beafdab7 100644
--- a/drivers/scsi/qla2xxx/qla_version.h
+++ b/drivers/scsi/qla2xxx/qla_version.h
@@ -6,9 +6,9 @@
 /*
  * Driver version
  */
-#define QLA2XXX_VERSION      "10.02.09.400-k"
+#define QLA2XXX_VERSION      "10.02.10.100-k"
 
 #define QLA_DRIVER_MAJOR_VER	10
-#define QLA_DRIVER_MINOR_VER	2
-#define QLA_DRIVER_PATCH_VER	9
-#define QLA_DRIVER_BETA_VER	400
+#define QLA_DRIVER_MINOR_VER	02
+#define QLA_DRIVER_PATCH_VER	10
+#define QLA_DRIVER_BETA_VER	100
diff --git a/drivers/scsi/qla2xxx/tcm_qla2xxx.c b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
index 2fff68935338..9f16164faa1e 100644
--- a/drivers/scsi/qla2xxx/tcm_qla2xxx.c
+++ b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
@@ -18,6 +18,7 @@
 #include <linux/module.h>
 #include <linux/utsname.h>
 #include <linux/vmalloc.h>
+#include <linux/hex.h>
 #include <linux/list.h>
 #include <linux/slab.h>
 #include <linux/types.h>
diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c
index 1f2a53ba5dd9..f41791801def 100644
--- a/drivers/scsi/scsi_debug.c
+++ b/drivers/scsi/scsi_debug.c
@@ -1371,8 +1371,7 @@ static void mk_sense_invalid_fld(struct scsi_cmnd *scp,
 
 	sbuff = scp->sense_buffer;
 	if (!sbuff) {
-		sdev_printk(KERN_ERR, scp->device,
-			    "%s: sense_buffer is NULL\n", __func__);
+		sdev_printk(KERN_ERR, scp->device, "sense_buffer is NULL\n");
 		return;
 	}
 	asc = c_d ? INVALID_FIELD_IN_CDB : INVALID_FIELD_IN_PARAM_LIST;
@@ -1404,8 +1403,7 @@ static void mk_sense_invalid_fld(struct scsi_cmnd *scp,
 static void mk_sense_buffer(struct scsi_cmnd *scp, int key, int asc, int asq)
 {
 	if (!scp->sense_buffer) {
-		sdev_printk(KERN_ERR, scp->device,
-			    "%s: sense_buffer is NULL\n", __func__);
+		sdev_printk(KERN_ERR, scp->device, "sense_buffer is NULL\n");
 		return;
 	}
 	memset(scp->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
@@ -1423,8 +1421,7 @@ static void mk_sense_info_tape(struct scsi_cmnd *scp, int key, int asc, int asq,
 			unsigned int information, unsigned char tape_flags)
 {
 	if (!scp->sense_buffer) {
-		sdev_printk(KERN_ERR, scp->device,
-			    "%s: sense_buffer is NULL\n", __func__);
+		sdev_printk(KERN_ERR, scp->device, "sense_buffer is NULL\n");
 		return;
 	}
 	memset(scp->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
@@ -1452,15 +1449,12 @@ static int scsi_debug_ioctl(struct scsi_device *dev, unsigned int cmd,
 {
 	if (sdebug_verbose) {
 		if (0x1261 == cmd)
-			sdev_printk(KERN_INFO, dev,
-				    "%s: BLKFLSBUF [0x1261]\n", __func__);
+			sdev_printk(KERN_INFO, dev, "BLKFLSBUF [0x1261]\n");
 		else if (0x5331 == cmd)
 			sdev_printk(KERN_INFO, dev,
-				    "%s: CDROM_GET_CAPABILITY [0x5331]\n",
-				    __func__);
+				    "CDROM_GET_CAPABILITY [0x5331]\n");
 		else
-			sdev_printk(KERN_INFO, dev, "%s: cmd=0x%x\n",
-				    __func__, cmd);
+			sdev_printk(KERN_INFO, dev, "cmd=0x%x\n", cmd);
 	}
 	return -EINVAL;
 	/* return -ENOTTY; // correct return but upsets fdisk */
@@ -1664,8 +1658,8 @@ static int p_fill_from_dev_buffer(struct scsi_cmnd *scp, const void *arr,
 
 	act_len = sg_pcopy_from_buffer(sdb->table.sgl, sdb->table.nents,
 				       arr, arr_len, skip);
-	pr_debug("%s: off_dst=%u, scsi_bufflen=%u, act_len=%u, resid=%d\n",
-		 __func__, off_dst, scsi_bufflen(scp), act_len,
+	pr_debug("off_dst=%u, scsi_bufflen=%u, act_len=%u, resid=%d\n",
+		 off_dst, scsi_bufflen(scp), act_len,
 		 scsi_get_resid(scp));
 	n = scsi_bufflen(scp) - (off_dst + act_len);
 	scsi_set_resid(scp, min_t(u32, scsi_get_resid(scp), n));
@@ -3188,8 +3182,8 @@ static int resp_mode_select(struct scsi_cmnd *scp,
 		return DID_ERROR << 16;
 	else if (sdebug_verbose && (res < param_len))
 		sdev_printk(KERN_INFO, scp->device,
-			    "%s: cdb indicated=%d, IO sent=%d bytes\n",
-			    __func__, param_len, res);
+			    "cdb indicated=%d, IO sent=%d bytes\n",
+			    param_len, res);
 	md_len = mselect6 ? (arr[0] + 1) : (get_unaligned_be16(arr + 0) + 2);
 	bd_len = mselect6 ? arr[3] : get_unaligned_be16(arr + 6);
 	off = (mselect6 ? 4 : 8);
@@ -5133,8 +5127,7 @@ static int resp_write_scat(struct scsi_cmnd *scp,
 	if (lbdof == 0) {
 		if (sdebug_verbose)
 			sdev_printk(KERN_INFO, scp->device,
-				"%s: %s: LB Data Offset field bad\n",
-				my_name, __func__);
+				"%s: LB Data Offset field bad\n", my_name);
 		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
 		return illegal_condition_result;
 	}
@@ -5142,8 +5135,7 @@ static int resp_write_scat(struct scsi_cmnd *scp,
 	if ((lrd_size + (num_lrd * lrd_size)) > lbdof_blen) {
 		if (sdebug_verbose)
 			sdev_printk(KERN_INFO, scp->device,
-				"%s: %s: LBA range descriptors don't fit\n",
-				my_name, __func__);
+				"%s: LBA range descriptors don't fit\n", my_name);
 		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
 		return illegal_condition_result;
 	}
@@ -5152,8 +5144,8 @@ static int resp_write_scat(struct scsi_cmnd *scp,
 		return SCSI_MLQUEUE_HOST_BUSY;
 	if (sdebug_verbose)
 		sdev_printk(KERN_INFO, scp->device,
-			"%s: %s: Fetch header+scatter_list, lbdof_blen=%u\n",
-			my_name, __func__, lbdof_blen);
+			"%s: Fetch header+scatter_list, lbdof_blen=%u\n",
+			my_name, lbdof_blen);
 	res = fetch_to_dev_buffer(scp, lrdp, lbdof_blen);
 	if (res == -1) {
 		ret = DID_ERROR << 16;
@@ -5170,8 +5162,8 @@ static int resp_write_scat(struct scsi_cmnd *scp,
 		num = get_unaligned_be32(up + 8);
 		if (sdebug_verbose)
 			sdev_printk(KERN_INFO, scp->device,
-				"%s: %s: k=%d  LBA=0x%llx num=%u  sg_off=%u\n",
-				my_name, __func__, k, lba, num, sg_off);
+				"%s: k=%d  LBA=0x%llx num=%u  sg_off=%u\n",
+				my_name, k, lba, num, sg_off);
 		if (num == 0)
 			continue;
 		ret = check_device_access_params(scp, lba, num, true);
@@ -5183,8 +5175,8 @@ static int resp_write_scat(struct scsi_cmnd *scp,
 		if ((cum_lb + num) > bt_len) {
 			if (sdebug_verbose)
 				sdev_printk(KERN_INFO, scp->device,
-				    "%s: %s: sum of blocks > data provided\n",
-				    my_name, __func__);
+				    "%s: sum of blocks > data provided\n",
+				    my_name);
 			mk_sense_buffer(scp, ILLEGAL_REQUEST, WRITE_ERROR_ASC,
 					0);
 			ret = illegal_condition_result;
@@ -5876,8 +5868,8 @@ static int resp_verify(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 		goto cleanup;
 	} else if (sdebug_verbose && (ret < (a_num * lb_size))) {
 		sdev_printk(KERN_INFO, scp->device,
-			    "%s: %s: cdb indicated=%u, IO sent=%d bytes\n",
-			    my_name, __func__, a_num * lb_size, ret);
+			    "%s: cdb indicated=%u, IO sent=%d bytes\n",
+			    my_name, a_num * lb_size, ret);
 	}
 	if (is_bytchk3) {
 		for (j = 1, off = lb_size; j < vnum; ++j, off += lb_size)
@@ -6404,11 +6396,6 @@ static void sdebug_q_cmd_complete(struct sdebug_defer *sd_dp)
 			atomic_inc(&sdebug_miss_cpus);
 	}
 
-	if (!scp) {
-		pr_err("scmd=NULL\n");
-		return;
-	}
-
 	spin_lock_irqsave(&sdsc->lock, flags);
 	aborted = sd_dp->aborted;
 	if (unlikely(aborted))
@@ -6685,14 +6672,14 @@ static int scsi_debug_sdev_configure(struct scsi_device *sdp,
 	devip->debugfs_entry = debugfs_create_dir(dev_name(&sdp->sdev_dev),
 				sdebug_debugfs_root);
 	if (IS_ERR_OR_NULL(devip->debugfs_entry))
-		pr_info("%s: failed to create debugfs directory for device %s\n",
-			__func__, dev_name(&sdp->sdev_gendev));
+		pr_info("failed to create debugfs directory for device %s\n",
+			dev_name(&sdp->sdev_gendev));
 
 	dentry = debugfs_create_file("error", 0600, devip->debugfs_entry, sdp,
 				&sdebug_error_fops);
 	if (IS_ERR_OR_NULL(dentry))
-		pr_info("%s: failed to create error file for device %s\n",
-			__func__, dev_name(&sdp->sdev_gendev));
+		pr_info("failed to create error file for device %s\n",
+			dev_name(&sdp->sdev_gendev));
 
 	return 0;
 }
@@ -6734,7 +6721,7 @@ static bool scsi_debug_stop_cmnd(struct scsi_cmnd *cmnd)
 {
 	struct sdebug_scsi_cmd *sdsc = scsi_cmd_priv(cmnd);
 	struct sdebug_defer *sd_dp = &sdsc->sd_dp;
-	enum sdeb_defer_type defer_t = READ_ONCE(sd_dp->defer_t);
+	enum sdeb_defer_type defer_t = sd_dp->defer_t;
 
 	lockdep_assert_held(&sdsc->lock);
 
@@ -6880,7 +6867,7 @@ static int scsi_debug_abort(struct scsi_cmnd *SCpnt)
 
 	if (SDEBUG_OPT_ALL_NOISE & sdebug_opts)
 		sdev_printk(KERN_INFO, SCpnt->device,
-			    "%s: command%s found\n", __func__,
+			    "command%s found\n",
 			    aborted ? "" : " not");
 
 
@@ -6968,7 +6955,7 @@ static int scsi_debug_device_reset(struct scsi_cmnd *SCpnt)
 	++num_dev_resets;
 
 	if (SDEBUG_OPT_ALL_NOISE & sdebug_opts)
-		sdev_printk(KERN_INFO, sdp, "%s\n", __func__);
+		sdev_printk(KERN_INFO, sdp, "doing device reset");
 
 	scsi_debug_stop_all_queued(sdp);
 	if (devip) {
@@ -7008,7 +6995,7 @@ static int scsi_debug_target_reset(struct scsi_cmnd *SCpnt)
 
 	++num_target_resets;
 	if (SDEBUG_OPT_ALL_NOISE & sdebug_opts)
-		sdev_printk(KERN_INFO, sdp, "%s\n", __func__);
+		sdev_printk(KERN_INFO, sdp, "doing target reset\n");
 
 	list_for_each_entry(devip, &sdbg_host->dev_info_list, dev_list) {
 		if (devip->target == sdp->id) {
@@ -7021,7 +7008,7 @@ static int scsi_debug_target_reset(struct scsi_cmnd *SCpnt)
 
 	if (SDEBUG_OPT_RESET_NOISE & sdebug_opts)
 		sdev_printk(KERN_INFO, sdp,
-			    "%s: %d device(s) found in target\n", __func__, k);
+			    "%d device(s) found in target\n", k);
 
 	if (sdebug_fail_target_reset(SCpnt)) {
 		scmd_printk(KERN_INFO, SCpnt, "fail target reset 0x%x\n",
@@ -7042,7 +7029,7 @@ static int scsi_debug_bus_reset(struct scsi_cmnd *SCpnt)
 	++num_bus_resets;
 
 	if (SDEBUG_OPT_ALL_NOISE & sdebug_opts)
-		sdev_printk(KERN_INFO, sdp, "%s\n", __func__);
+		sdev_printk(KERN_INFO, sdp, "doing bus reset\n");
 
 	list_for_each_entry(devip, &sdbg_host->dev_info_list, dev_list) {
 		set_bit(SDEBUG_UA_BUS_RESET, devip->uas_bm);
@@ -7053,7 +7040,7 @@ static int scsi_debug_bus_reset(struct scsi_cmnd *SCpnt)
 
 	if (SDEBUG_OPT_RESET_NOISE & sdebug_opts)
 		sdev_printk(KERN_INFO, sdp,
-			    "%s: %d device(s) found in host\n", __func__, k);
+			    "%d device(s) found in host\n", k);
 	return SUCCESS;
 }
 
@@ -7065,7 +7052,7 @@ static int scsi_debug_host_reset(struct scsi_cmnd *SCpnt)
 
 	++num_host_resets;
 	if (SDEBUG_OPT_ALL_NOISE & sdebug_opts)
-		sdev_printk(KERN_INFO, SCpnt->device, "%s\n", __func__);
+		sdev_printk(KERN_INFO, SCpnt->device, "doing host reset\n");
 	mutex_lock(&sdebug_host_list_mutex);
 	list_for_each_entry(sdbg_host, &sdebug_host_list, host_list) {
 		list_for_each_entry(devip, &sdbg_host->dev_info_list,
@@ -7080,7 +7067,7 @@ static int scsi_debug_host_reset(struct scsi_cmnd *SCpnt)
 	stop_all_queued();
 	if (SDEBUG_OPT_RESET_NOISE & sdebug_opts)
 		sdev_printk(KERN_INFO, SCpnt->device,
-			    "%s: %d device(s) found\n", __func__, k);
+			"%d device(s) found\n", k);
 	return SUCCESS;
 }
 
@@ -7231,8 +7218,8 @@ static int schedule_resp(struct scsi_cmnd *cmnd, struct sdebug_dev_info *devip,
 			scsi_result = device_qfull_result;
 
 			if (unlikely(SDEBUG_OPT_Q_NOISE & sdebug_opts))
-				sdev_printk(KERN_INFO, sdp, "%s: num_in_q=%d +1, <inject> status: TASK SET FULL\n",
-					    __func__, num_in_q);
+				sdev_printk(KERN_INFO, sdp, "num_in_q=%d +1, <inject> status: TASK SET FULL\n",
+					    num_in_q);
 		}
 	}
 
@@ -7258,8 +7245,8 @@ static int schedule_resp(struct scsi_cmnd *cmnd, struct sdebug_dev_info *devip,
 	}
 
 	if (unlikely(sdebug_verbose && cmnd->result))
-		sdev_printk(KERN_INFO, sdp, "%s: non-zero result=0x%x\n",
-			    __func__, cmnd->result);
+		sdev_printk(KERN_INFO, sdp, "non-zero result=0x%x\n",
+			    cmnd->result);
 
 	if (delta_jiff > 0 || ndelay > 0) {
 		ktime_t kt;
@@ -7296,12 +7283,12 @@ static int schedule_resp(struct scsi_cmnd *cmnd, struct sdebug_dev_info *devip,
 		if (polled) {
 			spin_lock_irqsave(&sdsc->lock, flags);
 			sd_dp->cmpl_ts = ktime_add(ns_to_ktime(ns_from_boot), kt);
-			WRITE_ONCE(sd_dp->defer_t, SDEB_DEFER_POLL);
+			sd_dp->defer_t = SDEB_DEFER_POLL;
 			spin_unlock_irqrestore(&sdsc->lock, flags);
 		} else {
 			/* schedule the invocation of scsi_done() for a later time */
 			spin_lock_irqsave(&sdsc->lock, flags);
-			WRITE_ONCE(sd_dp->defer_t, SDEB_DEFER_HRT);
+			sd_dp->defer_t = SDEB_DEFER_HRT;
 			hrtimer_start(&sd_dp->hrt, kt, HRTIMER_MODE_REL_PINNED);
 			/*
 			 * The completion handler will try to grab sqcp->lock,
@@ -7325,11 +7312,11 @@ static int schedule_resp(struct scsi_cmnd *cmnd, struct sdebug_dev_info *devip,
 		if (polled) {
 			spin_lock_irqsave(&sdsc->lock, flags);
 			sd_dp->cmpl_ts = ns_to_ktime(ns_from_boot);
-			WRITE_ONCE(sd_dp->defer_t, SDEB_DEFER_POLL);
+			sd_dp->defer_t = SDEB_DEFER_POLL;
 			spin_unlock_irqrestore(&sdsc->lock, flags);
 		} else {
 			spin_lock_irqsave(&sdsc->lock, flags);
-			WRITE_ONCE(sd_dp->defer_t, SDEB_DEFER_WQ);
+			sd_dp->defer_t = SDEB_DEFER_WQ;
 			schedule_work(&sd_dp->ew.work);
 			spin_unlock_irqrestore(&sdsc->lock, flags);
 		}
@@ -7459,7 +7446,7 @@ MODULE_PARM_DESC(lbprz,
 MODULE_PARM_DESC(lbpu, "enable LBP, support UNMAP command (def=0)");
 MODULE_PARM_DESC(lbpws, "enable LBP, support WRITE SAME(16) with UNMAP bit (def=0)");
 MODULE_PARM_DESC(lbpws10, "enable LBP, support WRITE SAME(10) with UNMAP bit (def=0)");
-MODULE_PARM_DESC(atomic_write, "enable ATOMIC WRITE support, support WRITE ATOMIC(16) (def=0)");
+MODULE_PARM_DESC(atomic_wr, "enable ATOMIC WRITE support, support WRITE ATOMIC(16) (def=0)");
 MODULE_PARM_DESC(lowest_aligned, "lowest aligned lba (def=0)");
 MODULE_PARM_DESC(lun_format, "LUN format: 0->peripheral (def); 1 --> flat address method");
 MODULE_PARM_DESC(max_luns, "number of LUNs per target to simulate(def=1)");
@@ -8697,7 +8684,7 @@ static int __init scsi_debug_init(void)
 
 	sdebug_debugfs_root = debugfs_create_dir("scsi_debug", NULL);
 	if (IS_ERR_OR_NULL(sdebug_debugfs_root))
-		pr_info("%s: failed to create initial debugfs directory\n", __func__);
+		pr_info("failed to create initial debugfs directory\n");
 
 	for (k = 0; k < hosts_to_add; k++) {
 		if (want_store && k == 0) {
@@ -8813,7 +8800,7 @@ static int sdebug_add_store(void)
 	if (unlikely(res < 0)) {
 		xa_unlock_irqrestore(per_store_ap, iflags);
 		kfree(sip);
-		pr_warn("%s: xa_alloc() errno=%d\n", __func__, -res);
+		pr_warn("xa_alloc() errno=%d\n", -res);
 		return res;
 	}
 	sdeb_most_recent_idx = n_idx;
@@ -8870,7 +8857,7 @@ static int sdebug_add_store(void)
 	return (int)n_idx;
 err:
 	sdebug_erase_store((int)n_idx, sip);
-	pr_warn("%s: failed, errno=%d\n", __func__, -res);
+	pr_warn("failed, errno=%d\n", -res);
 	return res;
 }
 
@@ -8929,7 +8916,7 @@ static int sdebug_add_host_helper(int per_host_idx)
 		put_device(&sdbg_host->dev);
 	else
 		kfree(sdbg_host);
-	pr_warn("%s: failed, errno=%d\n", __func__, -error);
+	pr_warn("failed, errno=%d\n", -error);
 	return error;
 }
 
@@ -8997,7 +8984,7 @@ static int sdebug_change_qdepth(struct scsi_device *sdev, int qdepth)
 
 	if (qdepth > SDEBUG_CANQUEUE) {
 		qdepth = SDEBUG_CANQUEUE;
-		pr_warn("%s: requested qdepth [%d] exceeds canqueue [%d], trim\n", __func__,
+		pr_warn("requested qdepth [%d] exceeds canqueue [%d], trim\n",
 			qdepth, SDEBUG_CANQUEUE);
 	}
 	if (qdepth < 1)
@@ -9009,7 +8996,7 @@ static int sdebug_change_qdepth(struct scsi_device *sdev, int qdepth)
 	mutex_unlock(&sdebug_host_list_mutex);
 
 	if (SDEBUG_OPT_Q_NOISE & sdebug_opts)
-		sdev_printk(KERN_INFO, sdev, "%s: qdepth=%d\n", __func__, qdepth);
+		sdev_printk(KERN_INFO, sdev, "qdepth=%d\n", qdepth);
 
 	return sdev->queue_depth;
 }
@@ -9133,7 +9120,7 @@ static bool sdebug_blk_mq_poll_iter(struct request *rq, void *opaque)
 
 	spin_lock_irqsave(&sdsc->lock, flags);
 	sd_dp = &sdsc->sd_dp;
-	if (READ_ONCE(sd_dp->defer_t) != SDEB_DEFER_POLL) {
+	if (sd_dp->defer_t != SDEB_DEFER_POLL) {
 		spin_unlock_irqrestore(&sdsc->lock, flags);
 		return true;
 	}
@@ -9282,8 +9269,7 @@ static void scsi_debug_abort_cmd(struct Scsi_Host *shost, struct scsi_cmnd *scp)
 	bool res = false;
 
 	if (!to_be_aborted_scmd) {
-		pr_err("%s: command with tag %#x not found\n", __func__,
-		       unique_tag);
+		pr_err("command with tag %#x not found\n", unique_tag);
 		return;
 	}
 
@@ -9291,11 +9277,9 @@ static void scsi_debug_abort_cmd(struct Scsi_Host *shost, struct scsi_cmnd *scp)
 		res = scsi_debug_stop_cmnd(to_be_aborted_scmd);
 
 	if (res)
-		pr_info("%s: aborted command with tag %#x\n",
-			__func__, unique_tag);
+		pr_info("aborted command with tag %#x\n", unique_tag);
 	else
-		pr_err("%s: failed to abort command with tag %#x\n",
-		       __func__, unique_tag);
+		pr_err("failed to abort command with tag %#x\n", unique_tag);
 
 	set_host_byte(scp, res ? DID_OK : DID_ERROR);
 }
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 93031326ac3e..3b860b369692 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -76,7 +76,7 @@ int scsi_init_sense_cache(struct Scsi_Host *shost)
 }
 
 static void
-scsi_set_blocked(struct scsi_cmnd *cmd, int reason)
+scsi_set_blocked(struct scsi_cmnd *cmd, enum scsi_qc_status reason)
 {
 	struct Scsi_Host *host = cmd->device->host;
 	struct scsi_device *device = cmd->device;
@@ -139,7 +139,8 @@ static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd, unsigned long msecs)
  * for a requeue after completion, which should only occur in this
  * file.
  */
-static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, bool unbusy)
+static void __scsi_queue_insert(struct scsi_cmnd *cmd,
+				enum scsi_qc_status reason, bool unbusy)
 {
 	struct scsi_device *device = cmd->device;
 
@@ -179,7 +180,7 @@ static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, bool unbusy)
  * Context: This could be called either from an interrupt context or a normal
  * process context.
  */
-void scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
+void scsi_queue_insert(struct scsi_cmnd *cmd, enum scsi_qc_status reason)
 {
 	__scsi_queue_insert(cmd, reason, true);
 }
@@ -1577,7 +1578,7 @@ static void scsi_complete(struct request *rq)
  * Return: nonzero return request was rejected and device's queue needs to be
  * plugged.
  */
-static int scsi_dispatch_cmd(struct scsi_cmnd *cmd)
+static enum scsi_qc_status scsi_dispatch_cmd(struct scsi_cmnd *cmd)
 {
 	struct Scsi_Host *host = cmd->device->host;
 	int rtn = 0;
@@ -1826,7 +1827,7 @@ static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
 	struct Scsi_Host *shost = sdev->host;
 	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
 	blk_status_t ret;
-	int reason;
+	enum scsi_qc_status reason;
 
 	WARN_ON_ONCE(cmd->budget_token < 0);
 
diff --git a/drivers/scsi/scsi_priv.h b/drivers/scsi/scsi_priv.h
index d07ec15d6c00..7a193cc04e5b 100644
--- a/drivers/scsi/scsi_priv.h
+++ b/drivers/scsi/scsi_priv.h
@@ -102,7 +102,8 @@ void scsi_eh_done(struct scsi_cmnd *scmd);
 
 /* scsi_lib.c */
 extern void scsi_device_unbusy(struct scsi_device *sdev, struct scsi_cmnd *cmd);
-extern void scsi_queue_insert(struct scsi_cmnd *cmd, int reason);
+extern void scsi_queue_insert(struct scsi_cmnd *cmd,
+			      enum scsi_qc_status reason);
 extern void scsi_io_completion(struct scsi_cmnd *, unsigned int);
 extern void scsi_run_host_queues(struct Scsi_Host *shost);
 extern void scsi_requeue_run_queue(struct work_struct *work);
diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c
index 987befb02408..9532138105c1 100644
--- a/drivers/scsi/scsi_transport_fc.c
+++ b/drivers/scsi/scsi_transport_fc.c
@@ -12,6 +12,7 @@
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
+#include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/bsg-lib.h>
 #include <scsi/scsi_device.h>
@@ -1328,6 +1329,46 @@ store_fc_rport_fast_io_fail_tmo(struct device *dev,
 static FC_DEVICE_ATTR(rport, fast_io_fail_tmo, S_IRUGO | S_IWUSR,
 	show_fc_rport_fast_io_fail_tmo, store_fc_rport_fast_io_fail_tmo);
 
+#define fc_rport_encryption(name)							\
+static ssize_t fc_rport_encinfo_##name(struct device *cd,				\
+				       struct device_attribute *attr,			\
+				       char *buf)					\
+{											\
+	struct fc_rport *rport = transport_class_to_rport(cd);				\
+	struct Scsi_Host *shost = rport_to_shost(rport);				\
+	struct fc_internal *i = to_fc_internal(shost->transportt);			\
+	struct fc_encryption_info *info;						\
+	ssize_t ret = -ENOENT;								\
+	u32 data;									\
+											\
+	if (i->f->get_fc_rport_enc_info) {						\
+		info = (i->f->get_fc_rport_enc_info)(rport);				\
+		if (info) {								\
+			data = info->name;						\
+			if (!strcmp(#name, "status")) {					\
+				ret = scnprintf(buf,					\
+						FC_RPORT_ENCRYPTION_STATUS_MAX_LEN,	\
+						"%s\n",					\
+						data ? "Encrypted" : "Unencrypted");	\
+			}								\
+		}									\
+	}										\
+	return ret;									\
+}											\
+static FC_DEVICE_ATTR(rport, encryption_##name, 0444, fc_rport_encinfo_##name, NULL)	\
+
+fc_rport_encryption(status);
+
+static struct attribute *fc_rport_encryption_attrs[] = {
+	&device_attr_rport_encryption_status.attr,
+	NULL
+};
+
+static struct attribute_group fc_rport_encryption_group = {
+	.name = "encryption",
+	.attrs = fc_rport_encryption_attrs,
+};
+
 #define fc_rport_fpin_statistic(name)					\
 static ssize_t fc_rport_fpinstat_##name(struct device *cd,		\
 				  struct device_attribute *attr,	\
@@ -2633,6 +2674,8 @@ fc_attach_transport(struct fc_function_template *ft)
 	i->rport_attr_cont.ac.attrs = &i->rport_attrs[0];
 	i->rport_attr_cont.ac.class = &fc_rport_class.class;
 	i->rport_attr_cont.ac.match = fc_rport_match;
+	if (ft->get_fc_rport_enc_info)
+		i->rport_attr_cont.encryption = &fc_rport_encryption_group;
 	i->rport_attr_cont.statistics = &fc_rport_statistics_group;
 	transport_container_register(&i->rport_attr_cont);
 
diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c
index b3af9b78fa12..57fba34832ad 100644
--- a/drivers/scsi/sg.c
+++ b/drivers/scsi/sg.c
@@ -731,6 +731,8 @@ sg_new_write(Sg_fd *sfp, struct file *file, const char __user *buf,
 		sg_remove_request(sfp, srp);
 		return -EFAULT;
 	}
+	hp->duration = jiffies_to_msecs(jiffies);
+
 	if (hp->interface_id != 'S') {
 		sg_remove_request(sfp, srp);
 		return -ENOSYS;
@@ -815,7 +817,6 @@ sg_common_write(Sg_fd * sfp, Sg_request * srp,
 		return -ENODEV;
 	}
 
-	hp->duration = jiffies_to_msecs(jiffies);
 	if (hp->interface_id != '\0' &&	/* v3 (or later) interface */
 	    (SG_FLAG_Q_AT_TAIL & hp->flags))
 		at_head = 0;
@@ -1338,9 +1339,6 @@ sg_rq_end_io(struct request *rq, blk_status_t status)
 				      "sg_cmd_done: pack_id=%d, res=0x%x\n",
 				      srp->header.pack_id, result));
 	srp->header.resid = resid;
-	ms = jiffies_to_msecs(jiffies);
-	srp->header.duration = (ms > srp->header.duration) ?
-				(ms - srp->header.duration) : 0;
 	if (0 != result) {
 		struct scsi_sense_hdr sshdr;
 
@@ -1389,6 +1387,9 @@ sg_rq_end_io(struct request *rq, blk_status_t status)
 			done = 0;
 	}
 	srp->done = done;
+	ms = jiffies_to_msecs(jiffies);
+	srp->header.duration = (ms > srp->header.duration) ?
+				(ms - srp->header.duration) : 0;
 	write_unlock_irqrestore(&sfp->rq_list_lock, iflags);
 
 	if (likely(done)) {
@@ -2533,6 +2534,7 @@ static void sg_proc_debug_helper(struct seq_file *s, Sg_device * sdp)
 	const sg_io_hdr_t *hp;
 	const char * cp;
 	unsigned int ms;
+	unsigned int duration;
 
 	k = 0;
 	list_for_each_entry(fp, &sdp->sfds, sfd_siblings) {
@@ -2570,13 +2572,17 @@ static void sg_proc_debug_helper(struct seq_file *s, Sg_device * sdp)
 			seq_printf(s, " id=%d blen=%d",
 				   srp->header.pack_id, blen);
 			if (srp->done)
-				seq_printf(s, " dur=%d", hp->duration);
+				seq_printf(s, " dur=%u", hp->duration);
 			else {
 				ms = jiffies_to_msecs(jiffies);
-				seq_printf(s, " t_o/elap=%d/%d",
+				duration = READ_ONCE(hp->duration);
+				if (duration)
+					duration = (ms > duration ?
+						    ms - duration : 0);
+				seq_printf(s, " t_o/elap=%u/%u",
 					(new_interface ? hp->timeout :
 						  jiffies_to_msecs(fp->timeout)),
-					(ms > hp->duration ? ms - hp->duration : 0));
+					duration);
 			}
 			seq_printf(s, "ms sgat=%d op=0x%02x\n", usg,
 				   (int) srp->data.cmd_opcode);
diff --git a/drivers/soc/qcom/llcc-qcom.c b/drivers/soc/qcom/llcc-qcom.c
index 13e174267294..1abfda7a58f2 100644
--- a/drivers/soc/qcom/llcc-qcom.c
+++ b/drivers/soc/qcom/llcc-qcom.c
@@ -182,6 +182,197 @@ enum llcc_reg_offset {
 	LLCC_TRP_WRS_CACHEABLE_EN,
 };
 
+static const struct llcc_slice_config glymur_data[] = {
+	{
+		.usecase_id = LLCC_CPUSS,
+		.slice_id = 1,
+		.max_cap = 7680,
+		.priority = 1,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.vict_prio = true,
+		.activate_on_init = true,
+	}, {
+		.usecase_id = LLCC_VIDSC0,
+		.slice_id = 2,
+		.max_cap = 512,
+		.priority = 3,
+		.fixed_size = true,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_AUDIO,
+		.slice_id = 6,
+		.max_cap = 1024,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_VIDSC1,
+		.slice_id = 4,
+		.max_cap = 512,
+		.priority = 3,
+		.fixed_size = true,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_CMPT,
+		.slice_id = 10,
+		.max_cap = 7680,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_GPUHTW,
+		.slice_id = 11,
+		.max_cap = 512,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_GPU,
+		.slice_id = 9,
+		.max_cap = 7680,
+		.priority = 1,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.write_scid_en = true,
+		.write_scid_cacheable_en = true,
+		.stale_en = true,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_MMUHWT,
+		.slice_id = 18,
+		.max_cap = 768,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.vict_prio = true,
+		.activate_on_init = true,
+	}, {
+		.usecase_id = LLCC_AUDHW,
+		.slice_id = 22,
+		.max_cap = 1024,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_CVP,
+		.slice_id = 8,
+		.max_cap = 64,
+		.priority = 3,
+		.fixed_size = true,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_WRCACHE,
+		.slice_id = 31,
+		.max_cap = 1536,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.vict_prio = true,
+		.activate_on_init = true,
+	}, {
+		.usecase_id = LLCC_CMPTHCP,
+		.slice_id = 17,
+		.max_cap = 256,
+		.priority = 3,
+		.fixed_size = true,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_LCPDARE,
+		.slice_id = 30,
+		.max_cap = 768,
+		.priority = 3,
+		.fixed_size = true,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.alloc_oneway_en = true,
+		.vict_prio = true,
+		.activate_on_init = true,
+	}, {
+		.usecase_id = LLCC_AENPU,
+		.slice_id = 3,
+		.max_cap = 3072,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.cache_mode = 2,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_ISLAND1,
+		.slice_id = 12,
+		.max_cap = 5632,
+		.priority = 7,
+		.fixed_size = true,
+		.bonus_ways = 0x0,
+		.res_ways = 0x7FF,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_VIDVSP,
+		.slice_id = 28,
+		.max_cap = 256,
+		.priority = 3,
+		.fixed_size = true,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_OOBM_NS,
+		.slice_id = 5,
+		.max_cap = 512,
+		.priority = 1,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_CPUSS_OPP,
+		.slice_id = 32,
+		.max_cap = 0,
+		.fixed_size = true,
+		.bonus_ways = 0x0,
+		.res_ways = 0x0,
+		.vict_prio = true,
+		.activate_on_init = true,
+	}, {
+		.usecase_id = LLCC_PCIE_TCU,
+		.slice_id = 19,
+		.max_cap = 256,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.vict_prio = true,
+		.activate_on_init = true,
+	}, {
+		.usecase_id = LLCC_VIDSC_VSP1,
+		.slice_id = 29,
+		.max_cap = 256,
+		.priority = 3,
+		.fixed_size = true,
+		.bonus_ways = 0xFFF,
+		.res_ways = 0x0,
+		.vict_prio = true,
+	}
+};
+
 static const struct llcc_slice_config ipq5424_data[] =  {
 	{
 		.usecase_id = LLCC_CPUSS,
@@ -3872,6 +4063,16 @@ static const struct qcom_llcc_config kaanapali_cfg[] = {
 	},
 };
 
+static const struct qcom_llcc_config glymur_cfg[] = {
+	{
+		.sct_data	= glymur_data,
+		.size		= ARRAY_SIZE(glymur_data),
+		.reg_offset	= llcc_v6_reg_offset,
+		.edac_reg_offset = &llcc_v2_1_edac_reg_offset,
+		.no_edac	= true,
+	},
+};
+
 static const struct qcom_llcc_config qcs615_cfg[] = {
 	{
 		.sct_data	= qcs615_data,
@@ -4103,6 +4304,11 @@ static const struct qcom_sct_config kaanapali_cfgs = {
 	.num_config	= ARRAY_SIZE(kaanapali_cfg),
 };
 
+static const struct qcom_sct_config glymur_cfgs = {
+	.llcc_config	= glymur_cfg,
+	.num_config	= ARRAY_SIZE(glymur_cfg),
+};
+
 static const struct qcom_sct_config qcs615_cfgs = {
 	.llcc_config	= qcs615_cfg,
 	.num_config	= ARRAY_SIZE(qcs615_cfg),
@@ -4941,6 +5147,7 @@ static int qcom_llcc_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id qcom_llcc_of_match[] = {
+	{ .compatible = "qcom,glymur-llcc", .data = &glymur_cfgs },
 	{ .compatible = "qcom,ipq5424-llcc", .data = &ipq5424_cfgs},
 	{ .compatible = "qcom,kaanapali-llcc", .data = &kaanapali_cfgs},
 	{ .compatible = "qcom,qcs615-llcc", .data = &qcs615_cfgs},
diff --git a/drivers/soc/qcom/smem.c b/drivers/soc/qcom/smem.c
index fef840b54574..088b2bbee9e6 100644
--- a/drivers/soc/qcom/smem.c
+++ b/drivers/soc/qcom/smem.c
@@ -1219,7 +1219,9 @@ static int qcom_smem_probe(struct platform_device *pdev)
 		smem->item_count = qcom_smem_get_item_count(smem);
 		break;
 	case SMEM_GLOBAL_HEAP_VERSION:
-		qcom_smem_map_global(smem, size);
+		ret = qcom_smem_map_global(smem, size);
+		if (ret < 0)
+			return ret;
 		smem->item_count = SMEM_ITEM_COUNT;
 		break;
 	default:
diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig
index 340a1ff7e92b..198baf890b14 100644
--- a/drivers/soc/renesas/Kconfig
+++ b/drivers/soc/renesas/Kconfig
@@ -423,6 +423,7 @@ config ARCH_R9A09G057
 config ARCH_R9A09G077
 	bool "ARM64 Platform support for R9A09G077 (RZ/T2H)"
 	default y if ARCH_RENESAS
+	select RENESAS_RZT2H_ICU
 	help
 	  This enables support for the Renesas RZ/T2H SoC variants.
 
diff --git a/drivers/soc/xilinx/zynqmp_power.c b/drivers/soc/xilinx/zynqmp_power.c
index 9b7b2858b22a..9085db1b480a 100644
--- a/drivers/soc/xilinx/zynqmp_power.c
+++ b/drivers/soc/xilinx/zynqmp_power.c
@@ -82,7 +82,7 @@ static void subsystem_restart_event_callback(const u32 *payload, void *data)
 	memcpy(zynqmp_pm_init_restart_work->args, &payload[0],
 	       sizeof(zynqmp_pm_init_restart_work->args));
 
-	queue_work(system_unbound_wq, &zynqmp_pm_init_restart_work->callback_work);
+	queue_work(system_dfl_wq, &zynqmp_pm_init_restart_work->callback_work);
 }
 
 static void suspend_event_callback(const u32 *payload, void *data)
@@ -95,7 +95,7 @@ static void suspend_event_callback(const u32 *payload, void *data)
 	memcpy(zynqmp_pm_init_suspend_work->args, &payload[1],
 	       sizeof(zynqmp_pm_init_suspend_work->args));
 
-	queue_work(system_unbound_wq, &zynqmp_pm_init_suspend_work->callback_work);
+	queue_work(system_dfl_wq, &zynqmp_pm_init_suspend_work->callback_work);
 }
 
 static irqreturn_t zynqmp_pm_isr(int irq, void *data)
@@ -140,7 +140,7 @@ static void ipi_receive_callback(struct mbox_client *cl, void *data)
 		memcpy(zynqmp_pm_init_suspend_work->args, &payload[1],
 		       sizeof(zynqmp_pm_init_suspend_work->args));
 
-		queue_work(system_unbound_wq,
+		queue_work(system_dfl_wq,
 			   &zynqmp_pm_init_suspend_work->callback_work);
 
 		/* Send NULL message to mbox controller to ack the message */
diff --git a/drivers/soundwire/dmi-quirks.c b/drivers/soundwire/dmi-quirks.c
index 91ab97a456fa..5854218e1a27 100644
--- a/drivers/soundwire/dmi-quirks.c
+++ b/drivers/soundwire/dmi-quirks.c
@@ -122,6 +122,17 @@ static const struct dmi_system_id adr_remap_quirk_table[] = {
 		},
 		.driver_data = (void *)intel_tgl_bios,
 	},
+	{
+		/*
+		 * quirk used for Avell B.ON (OEM rebrand of NUC15 'Bishop County'
+		 * LAPBC510 and LAPBC710)
+		 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Avell High Performance"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "B.ON"),
+		},
+		.driver_data = (void *)intel_tgl_bios,
+	},
 	{
 		/* quirk used for NUC15 'Rooks County' LAPRC510 and LAPRC710 skews */
 		.matches = {
diff --git a/drivers/soundwire/intel_auxdevice.c b/drivers/soundwire/intel_auxdevice.c
index 6df2601fff90..8752b0e3ce74 100644
--- a/drivers/soundwire/intel_auxdevice.c
+++ b/drivers/soundwire/intel_auxdevice.c
@@ -52,6 +52,7 @@ struct wake_capable_part {
 
 static struct wake_capable_part wake_capable_list[] = {
 	{0x01fa, 0x4243},
+	{0x01fa, 0x4245},
 	{0x025d, 0x5682},
 	{0x025d, 0x700},
 	{0x025d, 0x711},
diff --git a/drivers/soundwire/qcom.c b/drivers/soundwire/qcom.c
index 17afc5aa8b44..8102a1b0d516 100644
--- a/drivers/soundwire/qcom.c
+++ b/drivers/soundwire/qcom.c
@@ -1228,7 +1228,7 @@ static int qcom_swrm_stream_alloc_ports(struct qcom_swrm_ctrl *ctrl,
 	struct sdw_port_runtime *p_rt;
 	struct sdw_slave *slave;
 	unsigned long *port_mask;
-	int maxport, pn, nports = 0, ret = 0;
+	int maxport, pn, nports = 0;
 	unsigned int m_port;
 	struct sdw_port_config *pconfig __free(kfree) = kcalloc(ctrl->nports,
 								sizeof(*pconfig), GFP_KERNEL);
@@ -1246,7 +1246,8 @@ static int qcom_swrm_stream_alloc_ports(struct qcom_swrm_ctrl *ctrl,
 	sconfig.type = stream->type;
 	sconfig.bps = 1;
 
-	mutex_lock(&ctrl->port_lock);
+	guard(mutex)(&ctrl->port_lock);
+
 	list_for_each_entry(m_rt, &stream->master_list, stream_node) {
 		/*
 		 * For streams with multiple masters:
@@ -1272,8 +1273,7 @@ static int qcom_swrm_stream_alloc_ports(struct qcom_swrm_ctrl *ctrl,
 
 				if (pn > maxport) {
 					dev_err(ctrl->dev, "All ports busy\n");
-					ret = -EBUSY;
-					goto out;
+					return -EBUSY;
 				}
 				set_bit(pn, port_mask);
 				pconfig[nports].num = pn;
@@ -1285,10 +1285,8 @@ static int qcom_swrm_stream_alloc_ports(struct qcom_swrm_ctrl *ctrl,
 
 	sdw_stream_add_master(&ctrl->bus, &sconfig, pconfig,
 			      nports, stream);
-out:
-	mutex_unlock(&ctrl->port_lock);
 
-	return ret;
+	return 0;
 }
 
 static int qcom_swrm_hw_params(struct snd_pcm_substream *substream,
diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig
index 5520403896fc..69659b71ca49 100644
--- a/drivers/spi/Kconfig
+++ b/drivers/spi/Kconfig
@@ -136,6 +136,15 @@ config SPI_AR934X
 	  This enables support for the SPI controller present on the
 	  Qualcomm Atheros AR934X/QCA95XX SoCs.
 
+config SPI_ATCSPI200
+	tristate "Andes ATCSPI200 SPI controller"
+	depends on ARCH_ANDES
+	help
+	  SPI driver for Andes ATCSPI200 SPI controller.
+	  ATCSPI200 controller supports DMA and PIO modes. When DMA
+	  is not available, the driver automatically falls back to
+	  PIO mode.
+
 config SPI_ATH79
 	tristate "Atheros AR71XX/AR724X/AR913X SPI controller driver"
 	depends on ATH79 || COMPILE_TEST
@@ -481,6 +490,16 @@ config SPI_NXP_FLEXSPI
 	  This controller does not support generic SPI messages and only
 	  supports the high-level SPI memory interface.
 
+config SPI_NXP_XSPI
+        tristate "NXP xSPI controller"
+	depends on ARCH_MXC || COMPILE_TEST
+	depends on HAS_IOMEM
+	help
+	  This enables support for the xSPI controller. Up to two devices
+	  can be connected to one host.
+	  This controller does not support generic SPI messages and only
+	  supports the high-level SPI memory interface.
+
 config SPI_GPIO
 	tristate "GPIO-based bitbanging SPI Master"
 	depends on GPIOLIB || COMPILE_TEST
diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile
index 863b628ff1ec..ce427a5c57db 100644
--- a/drivers/spi/Makefile
+++ b/drivers/spi/Makefile
@@ -26,6 +26,7 @@ obj-$(CONFIG_SPI_APPLE)			+= spi-apple.o
 obj-$(CONFIG_SPI_AR934X)		+= spi-ar934x.o
 obj-$(CONFIG_SPI_ARMADA_3700)		+= spi-armada-3700.o
 obj-$(CONFIG_SPI_ASPEED_SMC)		+= spi-aspeed-smc.o
+obj-$(CONFIG_SPI_ATCSPI200)		+= spi-atcspi200.o
 obj-$(CONFIG_SPI_ATMEL)			+= spi-atmel.o
 obj-$(CONFIG_SPI_ATMEL_QUADSPI)		+= atmel-quadspi.o
 obj-$(CONFIG_SPI_AT91_USART)		+= spi-at91-usart.o
@@ -102,6 +103,7 @@ obj-$(CONFIG_SPI_WPCM_FIU)		+= spi-wpcm-fiu.o
 obj-$(CONFIG_SPI_NPCM_FIU)		+= spi-npcm-fiu.o
 obj-$(CONFIG_SPI_NPCM_PSPI)		+= spi-npcm-pspi.o
 obj-$(CONFIG_SPI_NXP_FLEXSPI)		+= spi-nxp-fspi.o
+obj-$(CONFIG_SPI_NXP_XSPI)		+= spi-nxp-xspi.o
 obj-$(CONFIG_SPI_OC_TINY)		+= spi-oc-tiny.o
 spi-octeon-objs				:= spi-cavium.o spi-cavium-octeon.o
 obj-$(CONFIG_SPI_OCTEON)		+= spi-octeon.o
diff --git a/drivers/spi/spi-atcspi200.c b/drivers/spi/spi-atcspi200.c
new file mode 100644
index 000000000000..0af7446642e5
--- /dev/null
+++ b/drivers/spi/spi-atcspi200.c
@@ -0,0 +1,680 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for Andes ATCSPI200 SPI Controller
+ *
+ * Copyright (C) 2025 Andes Technology Corporation.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/dev_printk.h>
+#include <linux/dmaengine.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/jiffies.h>
+#include <linux/minmax.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/spi/spi.h>
+#include <linux/spi/spi-mem.h>
+
+/* Register definitions  */
+#define ATCSPI_TRANS_FMT	0x10	/* SPI transfer format register */
+#define ATCSPI_TRANS_CTRL	0x20	/* SPI transfer control register */
+#define ATCSPI_CMD		0x24	/* SPI command register */
+#define ATCSPI_ADDR		0x28	/* SPI address register */
+#define ATCSPI_DATA		0x2C	/* SPI data register */
+#define ATCSPI_CTRL		0x30	/* SPI control register */
+#define ATCSPI_STATUS		0x34	/* SPI status register */
+#define ATCSPI_TIMING		0x40	/* SPI interface timing register */
+#define ATCSPI_CONFIG		0x7C	/* SPI configuration register */
+
+/* Transfer format register */
+#define TRANS_FMT_CPHA		BIT(0)
+#define TRANS_FMT_CPOL		BIT(1)
+#define TRANS_FMT_DATA_MERGE_EN	BIT(7)
+#define TRANS_FMT_DATA_LEN_MASK	GENMASK(12, 8)
+#define TRANS_FMT_ADDR_LEN_MASK	GENMASK(17, 16)
+#define TRANS_FMT_DATA_LEN(x)	FIELD_PREP(TRANS_FMT_DATA_LEN_MASK, (x) - 1)
+#define TRANS_FMT_ADDR_LEN(x)	FIELD_PREP(TRANS_FMT_ADDR_LEN_MASK, (x) - 1)
+
+/* Transfer control register */
+#define TRANS_MODE_MASK		GENMASK(27, 24)
+#define TRANS_MODE_W_ONLY	FIELD_PREP(TRANS_MODE_MASK, 1)
+#define TRANS_MODE_R_ONLY	FIELD_PREP(TRANS_MODE_MASK, 2)
+#define TRANS_MODE_NONE_DATA	FIELD_PREP(TRANS_MODE_MASK, 7)
+#define TRANS_MODE_DMY_READ	FIELD_PREP(TRANS_MODE_MASK, 9)
+#define TRANS_FIELD_DECNZ(m, x)	((x) ? FIELD_PREP(m, (x) - 1) : 0)
+#define TRANS_RD_TRANS_CNT(x)	TRANS_FIELD_DECNZ(GENMASK(8, 0), x)
+#define TRANS_DUMMY_CNT(x)	TRANS_FIELD_DECNZ(GENMASK(10, 9), x)
+#define TRANS_WR_TRANS_CNT(x)	TRANS_FIELD_DECNZ(GENMASK(20, 12), x)
+#define TRANS_DUAL_QUAD(x)	FIELD_PREP(GENMASK(23, 22), (x))
+#define TRANS_ADDR_FMT		BIT(28)
+#define TRANS_ADDR_EN		BIT(29)
+#define TRANS_CMD_EN		BIT(30)
+
+/* Control register */
+#define CTRL_SPI_RST		BIT(0)
+#define CTRL_RX_FIFO_RST	BIT(1)
+#define CTRL_TX_FIFO_RST	BIT(2)
+#define CTRL_RX_DMA_EN		BIT(3)
+#define CTRL_TX_DMA_EN		BIT(4)
+
+/* Status register */
+#define ATCSPI_ACTIVE		BIT(0)
+#define ATCSPI_RX_EMPTY		BIT(14)
+#define ATCSPI_TX_FULL		BIT(23)
+
+/* Interface timing setting */
+#define TIMING_SCLK_DIV_MASK	GENMASK(7, 0)
+#define TIMING_SCLK_DIV_MAX	0xFE
+
+/* Configuration register */
+#define RXFIFO_SIZE(x)		FIELD_GET(GENMASK(3, 0), (x))
+#define TXFIFO_SIZE(x)		FIELD_GET(GENMASK(7, 4), (x))
+
+/* driver configurations */
+#define ATCSPI_MAX_TRANS_LEN	512
+#define ATCSPI_MAX_SPEED_HZ	50000000
+#define ATCSPI_RDY_TIMEOUT_US	1000000
+#define ATCSPI_XFER_TIMEOUT(n)	((n) * 10)
+#define ATCSPI_MAX_CS_NUM	1
+#define ATCSPI_DMA_THRESHOLD	256
+#define ATCSPI_BITS_PER_UINT	8
+#define ATCSPI_DATA_MERGE_EN	1
+#define ATCSPI_DMA_SUPPORT	1
+
+/**
+ * struct atcspi_dev - Andes ATCSPI200 SPI controller private data
+ * @host:           Pointer to the SPI controller structure.
+ * @mutex_lock:     A mutex to protect concurrent access to the controller.
+ * @dma_completion: A completion to signal the end of a DMA transfer.
+ * @dev:            Pointer to the device structure.
+ * @regmap:         Register map for accessing controller registers.
+ * @clk:            Pointer to the controller's functional clock.
+ * @dma_addr:       The physical address of the SPI data register for DMA.
+ * @clk_rate:       The cached frequency of the functional clock.
+ * @sclk_rate:      The target frequency for the SPI clock (SCLK).
+ * @txfifo_size:    The size of the transmit FIFO in bytes.
+ * @rxfifo_size:    The size of the receive FIFO in bytes.
+ * @data_merge:     A flag indicating if the data merge mode is enabled for
+ *                  the current transfer.
+ * @use_dma:        Enable DMA mode if ATCSPI_DMA_SUPPORT is set and DMA is
+ *                  successfully configured.
+ */
+struct atcspi_dev {
+	struct spi_controller	*host;
+	struct mutex		mutex_lock;
+	struct completion	dma_completion;
+	struct device		*dev;
+	struct regmap		*regmap;
+	struct clk		*clk;
+	dma_addr_t		dma_addr;
+	unsigned int		clk_rate;
+	unsigned int		sclk_rate;
+	unsigned int		txfifo_size;
+	unsigned int		rxfifo_size;
+	bool			data_merge;
+	bool			use_dma;
+};
+
+static int atcspi_wait_fifo_ready(struct atcspi_dev *spi,
+				  enum spi_mem_data_dir dir)
+{
+	unsigned int val;
+	unsigned int mask;
+	int ret;
+
+	mask = (dir == SPI_MEM_DATA_OUT) ? ATCSPI_TX_FULL : ATCSPI_RX_EMPTY;
+	ret = regmap_read_poll_timeout(spi->regmap,
+				       ATCSPI_STATUS,
+				       val,
+				       !(val & mask),
+				       0,
+				       ATCSPI_RDY_TIMEOUT_US);
+	if (ret)
+		dev_info(spi->dev, "Timed out waiting for FIFO ready\n");
+
+	return ret;
+}
+
+static int atcspi_xfer_data_poll(struct atcspi_dev *spi,
+				 const struct spi_mem_op *op)
+{
+	void *rx_buf = op->data.buf.in;
+	const void *tx_buf = op->data.buf.out;
+	unsigned int val;
+	int trans_bytes = op->data.nbytes;
+	int num_byte;
+	int ret = 0;
+
+	num_byte = spi->data_merge ? 4 : 1;
+	while (trans_bytes) {
+		if (op->data.dir == SPI_MEM_DATA_OUT) {
+			ret = atcspi_wait_fifo_ready(spi, SPI_MEM_DATA_OUT);
+			if (ret)
+				return ret;
+
+			if (spi->data_merge)
+				val = *(unsigned int *)tx_buf;
+			else
+				val = *(unsigned char *)tx_buf;
+			regmap_write(spi->regmap, ATCSPI_DATA, val);
+			tx_buf = (unsigned char *)tx_buf + num_byte;
+		} else {
+			ret = atcspi_wait_fifo_ready(spi, SPI_MEM_DATA_IN);
+			if (ret)
+				return ret;
+
+			regmap_read(spi->regmap, ATCSPI_DATA, &val);
+			if (spi->data_merge)
+				*(unsigned int *)rx_buf = val;
+			else
+				*(unsigned char *)rx_buf = (unsigned char)val;
+			rx_buf = (unsigned char *)rx_buf + num_byte;
+		}
+		trans_bytes -= num_byte;
+	}
+
+	return ret;
+}
+
+static void atcspi_set_trans_ctl(struct atcspi_dev *spi,
+				 const struct spi_mem_op *op)
+{
+	unsigned int tc = 0;
+
+	if (op->cmd.nbytes)
+		tc |= TRANS_CMD_EN;
+	if (op->addr.nbytes)
+		tc |= TRANS_ADDR_EN;
+	if (op->addr.buswidth > 1)
+		tc |= TRANS_ADDR_FMT;
+	if (op->data.nbytes) {
+		tc |= TRANS_DUAL_QUAD(ffs(op->data.buswidth) - 1);
+		if (op->data.dir == SPI_MEM_DATA_IN) {
+			if (op->dummy.nbytes)
+				tc |= TRANS_MODE_DMY_READ |
+				      TRANS_DUMMY_CNT(op->dummy.nbytes);
+			else
+				tc |= TRANS_MODE_R_ONLY;
+			tc |= TRANS_RD_TRANS_CNT(op->data.nbytes);
+		} else {
+			tc |= TRANS_MODE_W_ONLY |
+			      TRANS_WR_TRANS_CNT(op->data.nbytes);
+		}
+	} else {
+		tc |= TRANS_MODE_NONE_DATA;
+	}
+	regmap_write(spi->regmap, ATCSPI_TRANS_CTRL, tc);
+}
+
+static void atcspi_set_trans_fmt(struct atcspi_dev *spi,
+				 const struct spi_mem_op *op)
+{
+	unsigned int val;
+
+	regmap_read(spi->regmap, ATCSPI_TRANS_FMT, &val);
+	if (op->data.nbytes) {
+		if (ATCSPI_DATA_MERGE_EN && ATCSPI_BITS_PER_UINT == 8 &&
+		    !(op->data.nbytes % 4)) {
+			val |= TRANS_FMT_DATA_MERGE_EN;
+			spi->data_merge = true;
+		} else {
+			val &= ~TRANS_FMT_DATA_MERGE_EN;
+			spi->data_merge = false;
+		}
+	}
+
+	val = (val & ~TRANS_FMT_ADDR_LEN_MASK) |
+	      TRANS_FMT_ADDR_LEN(op->addr.nbytes);
+	regmap_write(spi->regmap, ATCSPI_TRANS_FMT, val);
+}
+
+static void atcspi_prepare_trans(struct atcspi_dev *spi,
+				 const struct spi_mem_op *op)
+{
+	atcspi_set_trans_fmt(spi, op);
+	atcspi_set_trans_ctl(spi, op);
+	if (op->addr.nbytes)
+		regmap_write(spi->regmap, ATCSPI_ADDR, op->addr.val);
+	regmap_write(spi->regmap, ATCSPI_CMD, op->cmd.opcode);
+}
+
+static int atcspi_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op)
+{
+	struct atcspi_dev *spi;
+
+	spi = spi_controller_get_devdata(mem->spi->controller);
+	op->data.nbytes = min(op->data.nbytes, ATCSPI_MAX_TRANS_LEN);
+
+	/* DMA needs to be aligned to 4 byte */
+	if (spi->use_dma && op->data.nbytes >= ATCSPI_DMA_THRESHOLD)
+		op->data.nbytes = ALIGN_DOWN(op->data.nbytes, 4);
+
+	return 0;
+}
+
+static int atcspi_dma_config(struct atcspi_dev *spi, bool is_rx)
+{
+	struct dma_slave_config conf = { 0 };
+	struct dma_chan *chan;
+
+	if (is_rx) {
+		chan = spi->host->dma_rx;
+		conf.direction = DMA_DEV_TO_MEM;
+		conf.src_addr = spi->dma_addr;
+	} else {
+		chan = spi->host->dma_tx;
+		conf.direction = DMA_MEM_TO_DEV;
+		conf.dst_addr = spi->dma_addr;
+	}
+	conf.dst_maxburst = spi->rxfifo_size / 2;
+	conf.src_maxburst = spi->txfifo_size / 2;
+
+	if (spi->data_merge) {
+		conf.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+		conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+	} else {
+		conf.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
+		conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
+	}
+
+	return dmaengine_slave_config(chan, &conf);
+}
+
+static void atcspi_dma_callback(void *arg)
+{
+	struct completion *dma_completion = arg;
+
+	complete(dma_completion);
+}
+
+static int atcspi_dma_trans(struct atcspi_dev *spi,
+			    const struct spi_mem_op *op)
+{
+	struct dma_async_tx_descriptor *desc;
+	struct dma_chan *dma_ch;
+	struct sg_table sgt;
+	enum dma_transfer_direction dma_dir;
+	dma_cookie_t cookie;
+	unsigned int ctrl;
+	int timeout;
+	int ret;
+
+	regmap_read(spi->regmap, ATCSPI_CTRL, &ctrl);
+	ctrl |= CTRL_TX_DMA_EN | CTRL_RX_DMA_EN;
+	regmap_write(spi->regmap, ATCSPI_CTRL, ctrl);
+	if (op->data.dir == SPI_MEM_DATA_IN) {
+		ret = atcspi_dma_config(spi, TRUE);
+		dma_dir = DMA_DEV_TO_MEM;
+		dma_ch = spi->host->dma_rx;
+	} else {
+		ret = atcspi_dma_config(spi, FALSE);
+		dma_dir = DMA_MEM_TO_DEV;
+		dma_ch = spi->host->dma_tx;
+	}
+	if (ret)
+		return ret;
+
+	ret = spi_controller_dma_map_mem_op_data(spi->host, op, &sgt);
+	if (ret)
+		return ret;
+
+	desc = dmaengine_prep_slave_sg(dma_ch, sgt.sgl, sgt.nents, dma_dir,
+				       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+	if (!desc) {
+		ret = -ENOMEM;
+		goto exit_unmap;
+	}
+
+	reinit_completion(&spi->dma_completion);
+	desc->callback = atcspi_dma_callback;
+	desc->callback_param = &spi->dma_completion;
+	cookie = dmaengine_submit(desc);
+	ret = dma_submit_error(cookie);
+	if (ret)
+		goto exit_unmap;
+
+	dma_async_issue_pending(dma_ch);
+	timeout = msecs_to_jiffies(ATCSPI_XFER_TIMEOUT(op->data.nbytes));
+	if (!wait_for_completion_timeout(&spi->dma_completion, timeout)) {
+		ret = -ETIMEDOUT;
+		dmaengine_terminate_all(dma_ch);
+	}
+
+exit_unmap:
+	spi_controller_dma_unmap_mem_op_data(spi->host, op, &sgt);
+
+	return ret;
+}
+
+static int atcspi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op)
+{
+	struct spi_device *spi_dev = mem->spi;
+	struct atcspi_dev *spi;
+	unsigned int val;
+	int ret;
+
+	spi = spi_controller_get_devdata(spi_dev->controller);
+	mutex_lock(&spi->mutex_lock);
+	atcspi_prepare_trans(spi, op);
+	if (op->data.nbytes) {
+		if (spi->use_dma && op->data.nbytes >= ATCSPI_DMA_THRESHOLD)
+			ret = atcspi_dma_trans(spi, op);
+		else
+			ret = atcspi_xfer_data_poll(spi, op);
+		if (ret) {
+			dev_info(spi->dev, "SPI transmission failed\n");
+			goto exec_mem_exit;
+		}
+	}
+
+	ret = regmap_read_poll_timeout(spi->regmap,
+				       ATCSPI_STATUS,
+				       val,
+				       !(val & ATCSPI_ACTIVE),
+				       0,
+				       ATCSPI_RDY_TIMEOUT_US);
+	if (ret)
+		dev_info(spi->dev, "Timed out waiting for ATCSPI_ACTIVE\n");
+
+exec_mem_exit:
+	mutex_unlock(&spi->mutex_lock);
+
+	return ret;
+}
+
+static const struct spi_controller_mem_ops atcspi_mem_ops = {
+	.exec_op = atcspi_exec_mem_op,
+	.adjust_op_size = atcspi_adjust_op_size,
+};
+
+static int atcspi_setup(struct atcspi_dev *spi)
+{
+	unsigned int ctrl_val;
+	unsigned int val;
+	int actual_spi_sclk_f;
+	int ret;
+	unsigned char div;
+
+	ctrl_val = CTRL_TX_FIFO_RST | CTRL_RX_FIFO_RST | CTRL_SPI_RST;
+	regmap_write(spi->regmap, ATCSPI_CTRL, ctrl_val);
+	ret = regmap_read_poll_timeout(spi->regmap,
+				       ATCSPI_CTRL,
+				       val,
+				       !(val & ctrl_val),
+				       0,
+				       ATCSPI_RDY_TIMEOUT_US);
+	if (ret)
+		return dev_err_probe(spi->dev, ret,
+				     "Timed out waiting for ATCSPI_CTRL\n");
+
+	val = TRANS_FMT_DATA_LEN(ATCSPI_BITS_PER_UINT) |
+	      TRANS_FMT_CPHA | TRANS_FMT_CPOL;
+	regmap_write(spi->regmap, ATCSPI_TRANS_FMT, val);
+
+	regmap_read(spi->regmap, ATCSPI_CONFIG, &val);
+	spi->txfifo_size = BIT(TXFIFO_SIZE(val) + 1);
+	spi->rxfifo_size = BIT(RXFIFO_SIZE(val) + 1);
+
+	regmap_read(spi->regmap, ATCSPI_TIMING, &val);
+	val &= ~TIMING_SCLK_DIV_MASK;
+
+	/*
+	 * The SCLK_DIV value 0xFF is special and indicates that the
+	 * SCLK rate should be the same as the SPI clock rate.
+	 */
+	if (spi->sclk_rate >= spi->clk_rate) {
+		div = TIMING_SCLK_DIV_MASK;
+	} else {
+		/*
+		 * The divider value is determined as follows:
+		 * 1. If the divider can generate the exact target frequency,
+		 *    use that setting.
+		 * 2. If an exact match is not possible, select the closest
+		 *    available setting that is lower than the target frequency.
+		 */
+		div = (spi->clk_rate + (spi->sclk_rate * 2 - 1)) /
+		      (spi->sclk_rate * 2) - 1;
+
+		/* Check if the actual SPI clock is lower than the target */
+		actual_spi_sclk_f = spi->clk_rate / ((div + 1) * 2);
+		if (actual_spi_sclk_f < spi->sclk_rate)
+			dev_info(spi->dev,
+				 "Clock adjusted %d to %d due to divider limitation",
+				 spi->sclk_rate, actual_spi_sclk_f);
+
+		if (div > TIMING_SCLK_DIV_MAX)
+			return dev_err_probe(spi->dev, -EINVAL,
+					     "Unsupported SPI clock %d\n",
+					     spi->sclk_rate);
+	}
+	val |= div;
+	regmap_write(spi->regmap, ATCSPI_TIMING, val);
+
+	return ret;
+}
+
+static int atcspi_init_resources(struct platform_device *pdev,
+				 struct atcspi_dev *spi,
+				 struct resource **mem_res)
+{
+	void __iomem *base;
+	const struct regmap_config atcspi_regmap_cfg = {
+		.name = "atcspi",
+		.reg_bits = 32,
+		.val_bits = 32,
+		.cache_type = REGCACHE_NONE,
+		.reg_stride = 4,
+		.pad_bits = 0,
+		.max_register = ATCSPI_CONFIG
+	};
+
+	base = devm_platform_get_and_ioremap_resource(pdev, 0, mem_res);
+	if (IS_ERR(base))
+		return dev_err_probe(spi->dev, PTR_ERR(base),
+				     "Failed to get ioremap resource\n");
+
+	spi->regmap = devm_regmap_init_mmio(spi->dev, base,
+					    &atcspi_regmap_cfg);
+	if (IS_ERR(spi->regmap))
+		return dev_err_probe(spi->dev, PTR_ERR(spi->regmap),
+				     "Failed to init regmap\n");
+
+	spi->clk = devm_clk_get(spi->dev, NULL);
+	if (IS_ERR(spi->clk))
+		return dev_err_probe(spi->dev, PTR_ERR(spi->clk),
+				     "Failed to get SPI clock\n");
+
+	spi->sclk_rate = ATCSPI_MAX_SPEED_HZ;
+	return 0;
+}
+
+static int atcspi_configure_dma(struct atcspi_dev *spi)
+{
+	struct dma_chan *dma_chan;
+	int ret = 0;
+
+	dma_chan = devm_dma_request_chan(spi->dev, "rx");
+	if (IS_ERR(dma_chan)) {
+		ret = PTR_ERR(dma_chan);
+		goto err_exit;
+	}
+	spi->host->dma_rx = dma_chan;
+
+	dma_chan = devm_dma_request_chan(spi->dev, "tx");
+	if (IS_ERR(dma_chan)) {
+		ret = PTR_ERR(dma_chan);
+		goto free_rx;
+	}
+	spi->host->dma_tx = dma_chan;
+	init_completion(&spi->dma_completion);
+
+	return ret;
+
+free_rx:
+	dma_release_channel(spi->host->dma_rx);
+	spi->host->dma_rx = NULL;
+err_exit:
+	return ret;
+}
+
+static int atcspi_enable_clk(struct atcspi_dev *spi)
+{
+	int ret;
+
+	ret = clk_prepare_enable(spi->clk);
+	if (ret)
+		return dev_err_probe(spi->dev, ret,
+				     "Failed to enable clock\n");
+
+	spi->clk_rate = clk_get_rate(spi->clk);
+	if (!spi->clk_rate)
+		return dev_err_probe(spi->dev, -EINVAL,
+				     "Failed to get SPI clock rate\n");
+
+	return 0;
+}
+
+static void atcspi_init_controller(struct platform_device *pdev,
+				   struct atcspi_dev *spi,
+				   struct spi_controller *host,
+				   struct resource *mem_res)
+{
+	/* Get the physical address of the data register for DMA transfers. */
+	spi->dma_addr = (dma_addr_t)(mem_res->start + ATCSPI_DATA);
+
+	/* Initialize controller properties */
+	host->bus_num = pdev->id;
+	host->mode_bits = SPI_CPOL | SPI_CPHA | SPI_RX_QUAD | SPI_TX_QUAD;
+	host->dev.of_node = pdev->dev.of_node;
+	host->num_chipselect = ATCSPI_MAX_CS_NUM;
+	host->mem_ops = &atcspi_mem_ops;
+	host->max_speed_hz = spi->sclk_rate;
+}
+
+static int atcspi_probe(struct platform_device *pdev)
+{
+	struct spi_controller *host;
+	struct atcspi_dev *spi;
+	struct resource *mem_res;
+	int ret;
+
+	host = spi_alloc_host(&pdev->dev, sizeof(*spi));
+	if (!host)
+		return -ENOMEM;
+
+	spi = spi_controller_get_devdata(host);
+	spi->host = host;
+	spi->dev = &pdev->dev;
+	dev_set_drvdata(&pdev->dev, host);
+
+	ret = atcspi_init_resources(pdev, spi, &mem_res);
+	if (ret)
+		goto free_controller;
+
+	ret = atcspi_enable_clk(spi);
+	if (ret)
+		goto free_controller;
+
+	atcspi_init_controller(pdev, spi, host, mem_res);
+
+	ret = atcspi_setup(spi);
+	if (ret)
+		goto disable_clk;
+
+	ret = devm_spi_register_controller(&pdev->dev, host);
+	if (ret) {
+		dev_err_probe(spi->dev, ret,
+			      "Failed to register SPI controller\n");
+		goto disable_clk;
+	}
+
+	spi->use_dma = false;
+	if (ATCSPI_DMA_SUPPORT) {
+		ret = atcspi_configure_dma(spi);
+		if (ret)
+			dev_info(spi->dev,
+				 "Failed to init DMA, fallback to PIO mode\n");
+		else
+			spi->use_dma = true;
+	}
+	mutex_init(&spi->mutex_lock);
+
+	return 0;
+
+disable_clk:
+	clk_disable_unprepare(spi->clk);
+
+free_controller:
+	spi_controller_put(host);
+	return ret;
+}
+
+static int atcspi_suspend(struct device *dev)
+{
+	struct spi_controller *host = dev_get_drvdata(dev);
+	struct atcspi_dev *spi = spi_controller_get_devdata(host);
+
+	spi_controller_suspend(host);
+
+	clk_disable_unprepare(spi->clk);
+
+	return 0;
+}
+
+static int atcspi_resume(struct device *dev)
+{
+	struct spi_controller *host = dev_get_drvdata(dev);
+	struct atcspi_dev *spi = spi_controller_get_devdata(host);
+	int ret;
+
+	ret = clk_prepare_enable(spi->clk);
+	if (ret)
+		return ret;
+
+	ret = atcspi_setup(spi);
+	if (ret)
+		goto disable_clk;
+
+	ret = spi_controller_resume(host);
+	if (ret)
+		goto disable_clk;
+
+	return ret;
+
+disable_clk:
+	clk_disable_unprepare(spi->clk);
+
+	return ret;
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(atcspi_pm_ops, atcspi_suspend, atcspi_resume);
+
+static const struct of_device_id atcspi_of_match[] = {
+	{ .compatible = "andestech,qilai-spi", },
+	{ .compatible = "andestech,ae350-spi", },
+	{ /* sentinel */ }
+};
+
+MODULE_DEVICE_TABLE(of, atcspi_of_match);
+
+static struct platform_driver atcspi_driver = {
+	.probe = atcspi_probe,
+	.driver = {
+		.name = "atcspi200",
+		.owner	= THIS_MODULE,
+		.of_match_table = atcspi_of_match,
+		.pm = pm_sleep_ptr(&atcspi_pm_ops)
+	}
+};
+module_platform_driver(atcspi_driver);
+
+MODULE_AUTHOR("CL Wang <cl634@andestech.com>");
+MODULE_DESCRIPTION("Andes ATCSPI200 SPI controller driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c
index af6d050da1c8..e93ca632befb 100644
--- a/drivers/spi/spi-cadence-quadspi.c
+++ b/drivers/spi/spi-cadence-quadspi.c
@@ -1845,6 +1845,12 @@ static int cqspi_probe(struct platform_device *pdev)
 		return -ENODEV;
 	}
 
+	ret = cqspi_setup_flash(cqspi);
+	if (ret) {
+		dev_err(dev, "failed to setup flash parameters %d\n", ret);
+		return ret;
+	}
+
 	/* Obtain QSPI clock. */
 	cqspi->clk = devm_clk_get(dev, NULL);
 	if (IS_ERR(cqspi->clk)) {
@@ -1988,12 +1994,6 @@ static int cqspi_probe(struct platform_device *pdev)
 		pm_runtime_get_noresume(dev);
 	}
 
-	ret = cqspi_setup_flash(cqspi);
-	if (ret) {
-		dev_err(dev, "failed to setup flash parameters %d\n", ret);
-		goto probe_setup_failed;
-	}
-
 	host->num_chipselect = cqspi->num_chipselect;
 
 	if (ddata && (ddata->quirks & CQSPI_SUPPORT_DEVICE_RESET))
@@ -2001,8 +2001,10 @@ static int cqspi_probe(struct platform_device *pdev)
 
 	if (cqspi->use_direct_mode) {
 		ret = cqspi_request_mmap_dma(cqspi);
-		if (ret == -EPROBE_DEFER)
+		if (ret == -EPROBE_DEFER) {
+			dev_err_probe(&pdev->dev, ret, "Failed to request mmap DMA\n");
 			goto probe_setup_failed;
+		}
 	}
 
 	ret = spi_register_controller(host);
@@ -2011,10 +2013,8 @@ static int cqspi_probe(struct platform_device *pdev)
 		goto probe_setup_failed;
 	}
 
-	if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) {
-		pm_runtime_mark_last_busy(dev);
+	if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM)))
 		pm_runtime_put_autosuspend(dev);
-	}
 
 	return 0;
 probe_setup_failed:
@@ -2024,7 +2024,9 @@ static int cqspi_probe(struct platform_device *pdev)
 probe_reset_failed:
 	if (cqspi->is_jh7110)
 		cqspi_jh7110_disable_clk(pdev, cqspi);
-	clk_disable_unprepare(cqspi->clk);
+
+	if (pm_runtime_get_sync(&pdev->dev) >= 0)
+		clk_disable_unprepare(cqspi->clk);
 probe_clk_failed:
 	return ret;
 }
diff --git a/drivers/spi/spi-cadence-xspi.c b/drivers/spi/spi-cadence-xspi.c
index 6dcba0e0ddaa..c4ab6b2fb43f 100644
--- a/drivers/spi/spi-cadence-xspi.c
+++ b/drivers/spi/spi-cadence-xspi.c
@@ -2,7 +2,6 @@
 // Cadence XSPI flash controller driver
 // Copyright (C) 2020-21 Cadence
 
-#include <linux/acpi.h>
 #include <linux/completion.h>
 #include <linux/delay.h>
 #include <linux/err.h>
@@ -12,15 +11,16 @@
 #include <linux/iopoll.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/property.h>
 #include <linux/spi/spi.h>
 #include <linux/spi/spi-mem.h>
 #include <linux/bitfield.h>
 #include <linux/limits.h>
 #include <linux/log2.h>
 #include <linux/bitrev.h>
+#include <linux/util_macros.h>
 
 #define CDNS_XSPI_MAGIC_NUM_VALUE	0x6522
 #define CDNS_XSPI_MAX_BANKS		8
@@ -774,19 +774,15 @@ static int marvell_xspi_mem_op_execute(struct spi_mem *mem,
 	return ret;
 }
 
-#ifdef CONFIG_ACPI
 static bool cdns_xspi_supports_op(struct spi_mem *mem,
 				  const struct spi_mem_op *op)
 {
 	struct spi_device *spi = mem->spi;
-	const union acpi_object *obj;
-	struct acpi_device *adev;
+	struct device *dev = &spi->dev;
+	u32 value;
 
-	adev = ACPI_COMPANION(&spi->dev);
-
-	if (!acpi_dev_get_property(adev, "spi-tx-bus-width", ACPI_TYPE_INTEGER,
-				   &obj)) {
-		switch (obj->integer.value) {
+	if (!device_property_read_u32(dev, "spi-tx-bus-width", &value)) {
+		switch (value) {
 		case 1:
 			break;
 		case 2:
@@ -799,16 +795,13 @@ static bool cdns_xspi_supports_op(struct spi_mem *mem,
 			spi->mode |= SPI_TX_OCTAL;
 			break;
 		default:
-			dev_warn(&spi->dev,
-				 "spi-tx-bus-width %lld not supported\n",
-				 obj->integer.value);
+			dev_warn(dev, "spi-tx-bus-width %u not supported\n", value);
 			break;
 		}
 	}
 
-	if (!acpi_dev_get_property(adev, "spi-rx-bus-width", ACPI_TYPE_INTEGER,
-				   &obj)) {
-		switch (obj->integer.value) {
+	if (!device_property_read_u32(dev, "spi-rx-bus-width", &value)) {
+		switch (value) {
 		case 1:
 			break;
 		case 2:
@@ -821,9 +814,7 @@ static bool cdns_xspi_supports_op(struct spi_mem *mem,
 			spi->mode |= SPI_RX_OCTAL;
 			break;
 		default:
-			dev_warn(&spi->dev,
-				 "spi-rx-bus-width %lld not supported\n",
-				 obj->integer.value);
+			dev_warn(dev, "spi-rx-bus-width %u not supported\n", value);
 			break;
 		}
 	}
@@ -833,7 +824,6 @@ static bool cdns_xspi_supports_op(struct spi_mem *mem,
 
 	return true;
 }
-#endif
 
 static int cdns_xspi_adjust_mem_op_size(struct spi_mem *mem, struct spi_mem_op *op)
 {
@@ -846,17 +836,13 @@ static int cdns_xspi_adjust_mem_op_size(struct spi_mem *mem, struct spi_mem_op *
 }
 
 static const struct spi_controller_mem_ops cadence_xspi_mem_ops = {
-#ifdef CONFIG_ACPI
-	.supports_op = cdns_xspi_supports_op,
-#endif
+	.supports_op = PTR_IF(IS_ENABLED(CONFIG_ACPI), cdns_xspi_supports_op),
 	.exec_op = cdns_xspi_mem_op_execute,
 	.adjust_op_size = cdns_xspi_adjust_mem_op_size,
 };
 
 static const struct spi_controller_mem_ops marvell_xspi_mem_ops = {
-#ifdef CONFIG_ACPI
-	.supports_op = cdns_xspi_supports_op,
-#endif
+	.supports_op = PTR_IF(IS_ENABLED(CONFIG_ACPI), cdns_xspi_supports_op),
 	.exec_op = marvell_xspi_mem_op_execute,
 	.adjust_op_size = cdns_xspi_adjust_mem_op_size,
 };
@@ -1157,12 +1143,9 @@ static int cdns_xspi_probe(struct platform_device *pdev)
 		SPI_MODE_0  | SPI_MODE_3;
 
 	cdns_xspi = spi_controller_get_devdata(host);
-	cdns_xspi->driver_data = of_device_get_match_data(dev);
-	if (!cdns_xspi->driver_data) {
-		cdns_xspi->driver_data = acpi_device_get_match_data(dev);
-		if (!cdns_xspi->driver_data)
-			return -ENODEV;
-	}
+	cdns_xspi->driver_data = device_get_match_data(dev);
+	if (!cdns_xspi->driver_data)
+		return -ENODEV;
 
 	if (cdns_xspi->driver_data->mrvl_hw_overlay) {
 		host->mem_ops = &marvell_xspi_mem_ops;
diff --git a/drivers/spi/spi-fsl-lpspi.c b/drivers/spi/spi-fsl-lpspi.c
index 065456aba2ae..8f45ead23836 100644
--- a/drivers/spi/spi-fsl-lpspi.c
+++ b/drivers/spi/spi-fsl-lpspi.c
@@ -486,15 +486,7 @@ static int fsl_lpspi_setup_transfer(struct spi_controller *controller,
 		fsl_lpspi->tx = fsl_lpspi_buf_tx_u32;
 	}
 
-	/*
-	 * t->len is 'unsigned' and txfifosize and watermrk is 'u8', force
-	 * type cast is inevitable. When len > 255, len will be truncated in min_t(),
-	 * it caused wrong watermark set. 'unsigned int' is as the designated type
-	 * for min_t() to avoid truncation.
-	 */
-	fsl_lpspi->watermark = min_t(unsigned int,
-				     fsl_lpspi->txfifosize,
-				     t->len);
+	fsl_lpspi->watermark = min(fsl_lpspi->txfifosize, t->len);
 
 	if (fsl_lpspi_can_dma(controller, spi, t))
 		fsl_lpspi->usedma = true;
diff --git a/drivers/spi/spi-fsl-spi.c b/drivers/spi/spi-fsl-spi.c
index 2f2082652a1a..481a7b28aacd 100644
--- a/drivers/spi/spi-fsl-spi.c
+++ b/drivers/spi/spi-fsl-spi.c
@@ -335,7 +335,7 @@ static int fsl_spi_prepare_message(struct spi_controller *ctlr,
 			if (t->bits_per_word == 16 || t->bits_per_word == 32)
 				t->bits_per_word = 8; /* pretend its 8 bits */
 			if (t->bits_per_word == 8 && t->len >= 256 &&
-			    (mpc8xxx_spi->flags & SPI_CPM1))
+			    !(t->len & 1) && (mpc8xxx_spi->flags & SPI_CPM1))
 				t->bits_per_word = 16;
 		}
 	}
diff --git a/drivers/spi/spi-geni-qcom.c b/drivers/spi/spi-geni-qcom.c
index a0d8d3425c6c..5cca356cb673 100644
--- a/drivers/spi/spi-geni-qcom.c
+++ b/drivers/spi/spi-geni-qcom.c
@@ -82,6 +82,7 @@ struct spi_geni_master {
 	u32 fifo_width_bits;
 	u32 tx_wm;
 	u32 last_mode;
+	u8 last_cs;
 	unsigned long cur_speed_hz;
 	unsigned long cur_sclk_hz;
 	unsigned int cur_bits_per_word;
@@ -284,55 +285,6 @@ static bool spi_geni_is_abort_still_pending(struct spi_geni_master *mas)
 	return false;
 }
 
-static void spi_geni_set_cs(struct spi_device *slv, bool set_flag)
-{
-	struct spi_geni_master *mas = spi_controller_get_devdata(slv->controller);
-	struct spi_controller *spi = dev_get_drvdata(mas->dev);
-	struct geni_se *se = &mas->se;
-	unsigned long time_left;
-
-	if (!(slv->mode & SPI_CS_HIGH))
-		set_flag = !set_flag;
-
-	if (set_flag == mas->cs_flag)
-		return;
-
-	pm_runtime_get_sync(mas->dev);
-
-	if (spi_geni_is_abort_still_pending(mas)) {
-		dev_err(mas->dev, "Can't set chip select\n");
-		goto exit;
-	}
-
-	spin_lock_irq(&mas->lock);
-	if (mas->cur_xfer) {
-		dev_err(mas->dev, "Can't set CS when prev xfer running\n");
-		spin_unlock_irq(&mas->lock);
-		goto exit;
-	}
-
-	mas->cs_flag = set_flag;
-	/* set xfer_mode to FIFO to complete cs_done in isr */
-	mas->cur_xfer_mode = GENI_SE_FIFO;
-	geni_se_select_mode(se, mas->cur_xfer_mode);
-
-	reinit_completion(&mas->cs_done);
-	if (set_flag)
-		geni_se_setup_m_cmd(se, SPI_CS_ASSERT, 0);
-	else
-		geni_se_setup_m_cmd(se, SPI_CS_DEASSERT, 0);
-	spin_unlock_irq(&mas->lock);
-
-	time_left = wait_for_completion_timeout(&mas->cs_done, HZ);
-	if (!time_left) {
-		dev_warn(mas->dev, "Timeout setting chip select\n");
-		handle_se_timeout(spi, NULL);
-	}
-
-exit:
-	pm_runtime_put(mas->dev);
-}
-
 static void spi_setup_word_len(struct spi_geni_master *mas, u16 mode,
 					unsigned int bits_per_word)
 {
@@ -399,36 +351,27 @@ static int setup_fifo_params(struct spi_device *spi_slv,
 {
 	struct spi_geni_master *mas = spi_controller_get_devdata(spi);
 	struct geni_se *se = &mas->se;
-	u32 loopback_cfg = 0, cpol = 0, cpha = 0, demux_output_inv = 0;
-	u32 demux_sel;
-
-	if (mas->last_mode != spi_slv->mode) {
-		if (spi_slv->mode & SPI_LOOP)
-			loopback_cfg = LOOPBACK_ENABLE;
-
-		if (spi_slv->mode & SPI_CPOL)
-			cpol = CPOL;
-
-		if (spi_slv->mode & SPI_CPHA)
-			cpha = CPHA;
-
-		if (spi_slv->mode & SPI_CS_HIGH)
-			demux_output_inv = BIT(spi_get_chipselect(spi_slv, 0));
-
-		demux_sel = spi_get_chipselect(spi_slv, 0);
-		mas->cur_bits_per_word = spi_slv->bits_per_word;
+	u8 chipselect = spi_get_chipselect(spi_slv, 0);
+	bool cs_changed = (mas->last_cs != chipselect);
+	u32 mode_changed = mas->last_mode ^ spi_slv->mode;
+
+	mas->last_cs = chipselect;
+	mas->last_mode = spi_slv->mode;
+
+	if (mode_changed & SPI_LSB_FIRST)
+		mas->cur_bits_per_word = 0; /* force next setup_se_xfer to call spi_setup_word_len */
+	if (mode_changed & SPI_LOOP)
+		writel((spi_slv->mode & SPI_LOOP) ? LOOPBACK_ENABLE : 0, se->base + SE_SPI_LOOPBACK);
+	if (cs_changed)
+		writel(chipselect, se->base + SE_SPI_DEMUX_SEL);
+	if (mode_changed & SE_SPI_CPHA)
+		writel((spi_slv->mode & SPI_CPHA) ? CPHA : 0, se->base + SE_SPI_CPHA);
+	if (mode_changed & SE_SPI_CPOL)
+		writel((spi_slv->mode & SPI_CPOL) ? CPOL : 0, se->base + SE_SPI_CPOL);
+	if ((mode_changed & SPI_CS_HIGH) || (cs_changed && (spi_slv->mode & SPI_CS_HIGH)))
+		writel((spi_slv->mode & SPI_CS_HIGH) ? BIT(chipselect) : 0, se->base + SE_SPI_DEMUX_OUTPUT_INV);
 
-		spi_setup_word_len(mas, spi_slv->mode, spi_slv->bits_per_word);
-		writel(loopback_cfg, se->base + SE_SPI_LOOPBACK);
-		writel(demux_sel, se->base + SE_SPI_DEMUX_SEL);
-		writel(cpha, se->base + SE_SPI_CPHA);
-		writel(cpol, se->base + SE_SPI_CPOL);
-		writel(demux_output_inv, se->base + SE_SPI_DEMUX_OUTPUT_INV);
-
-		mas->last_mode = spi_slv->mode;
-	}
-
-	return geni_spi_set_clock_and_bw(mas, spi_slv->max_speed_hz);
+	return 0;
 }
 
 static void
@@ -548,10 +491,10 @@ static u32 get_xfer_len_in_words(struct spi_transfer *xfer,
 {
 	u32 len;
 
-	if (!(mas->cur_bits_per_word % MIN_WORD_LEN))
-		len = xfer->len * BITS_PER_BYTE / mas->cur_bits_per_word;
+	if (!(xfer->bits_per_word % MIN_WORD_LEN))
+		len = xfer->len * BITS_PER_BYTE / xfer->bits_per_word;
 	else
-		len = xfer->len / (mas->cur_bits_per_word / BITS_PER_BYTE + 1);
+		len = xfer->len / (xfer->bits_per_word / BITS_PER_BYTE + 1);
 	len &= TRANS_LEN_MSK;
 
 	return len;
@@ -571,7 +514,7 @@ static bool geni_can_dma(struct spi_controller *ctlr,
 		return true;
 
 	len = get_xfer_len_in_words(xfer, mas);
-	fifo_size = mas->tx_fifo_depth * mas->fifo_width_bits / mas->cur_bits_per_word;
+	fifo_size = mas->tx_fifo_depth * mas->fifo_width_bits / xfer->bits_per_word;
 
 	if (len > fifo_size)
 		return true;
@@ -724,11 +667,17 @@ static int spi_geni_init(struct spi_geni_master *mas)
 	case 0:
 		mas->cur_xfer_mode = GENI_SE_FIFO;
 		geni_se_select_mode(se, GENI_SE_FIFO);
+		/* setup_fifo_params assumes that these registers start with a zero value */
+		writel(0, se->base + SE_SPI_LOOPBACK);
+		writel(0, se->base + SE_SPI_DEMUX_SEL);
+		writel(0, se->base + SE_SPI_CPHA);
+		writel(0, se->base + SE_SPI_CPOL);
+		writel(0, se->base + SE_SPI_DEMUX_OUTPUT_INV);
 		ret = 0;
 		break;
 	}
 
-	/* We always control CS manually */
+	/* We never control CS manually */
 	if (!spi->target) {
 		spi_tx_cfg = readl(se->base + SE_SPI_TRANS_CFG);
 		spi_tx_cfg &= ~CS_TOGGLE;
@@ -841,6 +790,7 @@ static int setup_se_xfer(struct spi_transfer *xfer,
 				u16 mode, struct spi_controller *spi)
 {
 	u32 m_cmd = 0;
+	u32 m_params = 0;
 	u32 len;
 	struct geni_se *se = &mas->se;
 	int ret;
@@ -904,12 +854,17 @@ static int setup_se_xfer(struct spi_transfer *xfer,
 		mas->cur_xfer_mode = GENI_SE_DMA;
 	geni_se_select_mode(se, mas->cur_xfer_mode);
 
+	if (!xfer->cs_change) {
+		if (!list_is_last(&xfer->transfer_list, &spi->cur_msg->transfers))
+			m_params = FRAGMENTATION;
+	}
+
 	/*
 	 * Lock around right before we start the transfer since our
 	 * interrupt could come in at any time now.
 	 */
 	spin_lock_irq(&mas->lock);
-	geni_se_setup_m_cmd(se, m_cmd, FRAGMENTATION);
+	geni_se_setup_m_cmd(se, m_cmd, m_params);
 
 	if (mas->cur_xfer_mode == GENI_SE_DMA) {
 		if (m_cmd & SPI_RX_ONLY)
@@ -1148,14 +1103,6 @@ static int spi_geni_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	/*
-	 * check the mode supported and set_cs for fifo mode only
-	 * for dma (gsi) mode, the gsi will set cs based on params passed in
-	 * TRE
-	 */
-	if (!spi->target && mas->cur_xfer_mode == GENI_SE_FIFO)
-		spi->set_cs = spi_geni_set_cs;
-
 	/*
 	 * TX is required per GSI spec, see setup_gsi_xfer().
 	 */
diff --git a/drivers/spi/spi-imx.c b/drivers/spi/spi-imx.c
index b8b79bb7fec3..69c288c8c737 100644
--- a/drivers/spi/spi-imx.c
+++ b/drivers/spi/spi-imx.c
@@ -60,6 +60,7 @@ MODULE_PARM_DESC(polling_limit_us,
 #define MX51_ECSPI_CTRL_MAX_BURST	512
 /* The maximum bytes that IMX53_ECSPI can transfer in target mode.*/
 #define MX53_MAX_TRANSFER_BYTES		512
+#define BYTES_PER_32BITS_WORD		4
 
 enum spi_imx_devtype {
 	IMX1_CSPI,
@@ -95,6 +96,16 @@ struct spi_imx_devtype_data {
 	enum spi_imx_devtype devtype;
 };
 
+struct dma_data_package {
+	u32 cmd_word;
+	void *dma_rx_buf;
+	void *dma_tx_buf;
+	dma_addr_t dma_tx_addr;
+	dma_addr_t dma_rx_addr;
+	int dma_len;
+	int data_len;
+};
+
 struct spi_imx_data {
 	struct spi_controller *controller;
 	struct device *dev;
@@ -130,6 +141,9 @@ struct spi_imx_data {
 	u32 wml;
 	struct completion dma_rx_completion;
 	struct completion dma_tx_completion;
+	size_t dma_package_num;
+	struct dma_data_package *dma_data;
+	int rx_offset;
 
 	const struct spi_imx_devtype_data *devtype_data;
 };
@@ -189,6 +203,9 @@ MXC_SPI_BUF_TX(u16)
 MXC_SPI_BUF_RX(u32)
 MXC_SPI_BUF_TX(u32)
 
+/* Align to cache line to avoid swiotlo bounce */
+#define DMA_CACHE_ALIGNED_LEN(x) ALIGN((x), dma_get_cache_alignment())
+
 /* First entry is reserved, second entry is valid only if SDHC_SPIEN is set
  * (which is currently not the case in this driver)
  */
@@ -247,12 +264,26 @@ static bool spi_imx_can_dma(struct spi_controller *controller, struct spi_device
 	if (!controller->dma_rx)
 		return false;
 
-	if (spi_imx->target_mode)
+	/*
+	 * Due to Freescale errata ERR003775 "eCSPI: Burst completion by Chip
+	 * Select (SS) signal in Slave mode is not functional" burst size must
+	 * be set exactly to the size of the transfer. This limit SPI transaction
+	 * with maximum 2^12 bits.
+	 */
+	if (transfer->len > MX53_MAX_TRANSFER_BYTES && spi_imx->target_mode)
 		return false;
 
 	if (transfer->len < spi_imx->devtype_data->fifo_size)
 		return false;
 
+	/* DMA only can transmit data in bytes */
+	if (spi_imx->bits_per_word != 8 && spi_imx->bits_per_word != 16 &&
+	    spi_imx->bits_per_word != 32)
+		return false;
+
+	if (transfer->len >= MAX_SDMA_BD_BYTES)
+		return false;
+
 	spi_imx->dynamic_burst = 0;
 
 	return true;
@@ -1282,50 +1313,6 @@ static irqreturn_t spi_imx_isr(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static int spi_imx_dma_configure(struct spi_controller *controller)
-{
-	int ret;
-	enum dma_slave_buswidth buswidth;
-	struct dma_slave_config rx = {}, tx = {};
-	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
-
-	switch (spi_imx_bytes_per_word(spi_imx->bits_per_word)) {
-	case 4:
-		buswidth = DMA_SLAVE_BUSWIDTH_4_BYTES;
-		break;
-	case 2:
-		buswidth = DMA_SLAVE_BUSWIDTH_2_BYTES;
-		break;
-	case 1:
-		buswidth = DMA_SLAVE_BUSWIDTH_1_BYTE;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	tx.direction = DMA_MEM_TO_DEV;
-	tx.dst_addr = spi_imx->base_phys + MXC_CSPITXDATA;
-	tx.dst_addr_width = buswidth;
-	tx.dst_maxburst = spi_imx->wml;
-	ret = dmaengine_slave_config(controller->dma_tx, &tx);
-	if (ret) {
-		dev_err(spi_imx->dev, "TX dma configuration failed with %d\n", ret);
-		return ret;
-	}
-
-	rx.direction = DMA_DEV_TO_MEM;
-	rx.src_addr = spi_imx->base_phys + MXC_CSPIRXDATA;
-	rx.src_addr_width = buswidth;
-	rx.src_maxburst = spi_imx->wml;
-	ret = dmaengine_slave_config(controller->dma_rx, &rx);
-	if (ret) {
-		dev_err(spi_imx->dev, "RX dma configuration failed with %d\n", ret);
-		return ret;
-	}
-
-	return 0;
-}
-
 static int spi_imx_setupxfer(struct spi_device *spi,
 				 struct spi_transfer *t)
 {
@@ -1442,8 +1429,6 @@ static int spi_imx_sdma_init(struct device *dev, struct spi_imx_data *spi_imx,
 
 	init_completion(&spi_imx->dma_rx_completion);
 	init_completion(&spi_imx->dma_tx_completion);
-	controller->can_dma = spi_imx_can_dma;
-	controller->max_dma_len = MAX_SDMA_BD_BYTES;
 	spi_imx->controller->flags = SPI_CONTROLLER_MUST_RX |
 					 SPI_CONTROLLER_MUST_TX;
 
@@ -1481,71 +1466,302 @@ static int spi_imx_calculate_timeout(struct spi_imx_data *spi_imx, int size)
 	return secs_to_jiffies(2 * timeout);
 }
 
-static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
-				struct spi_transfer *transfer)
+static void spi_imx_dma_unmap(struct spi_imx_data *spi_imx,
+			      struct dma_data_package *dma_data)
+{
+	struct device *tx_dev = spi_imx->controller->dma_tx->device->dev;
+	struct device *rx_dev = spi_imx->controller->dma_rx->device->dev;
+
+	dma_unmap_single(tx_dev, dma_data->dma_tx_addr,
+			 DMA_CACHE_ALIGNED_LEN(dma_data->dma_len),
+			 DMA_TO_DEVICE);
+	dma_unmap_single(rx_dev, dma_data->dma_rx_addr,
+			 DMA_CACHE_ALIGNED_LEN(dma_data->dma_len),
+			 DMA_FROM_DEVICE);
+}
+
+static void spi_imx_dma_rx_data_handle(struct spi_imx_data *spi_imx,
+				       struct dma_data_package *dma_data, void *rx_buf,
+				       bool word_delay)
+{
+	void *copy_ptr;
+	int unaligned;
+
+	/*
+	 * On little-endian CPUs, adjust byte order:
+	 * - Swap bytes when bpw = 8
+	 * - Swap half-words when bpw = 16
+	 * This ensures correct data ordering for DMA transfers.
+	 */
+#ifdef __LITTLE_ENDIAN
+	if (!word_delay) {
+		unsigned int bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);
+		u32 *temp = dma_data->dma_rx_buf;
+
+		for (int i = 0; i < DIV_ROUND_UP(dma_data->dma_len, sizeof(*temp)); i++) {
+			if (bytes_per_word == 1)
+				swab32s(temp + i);
+			else if (bytes_per_word == 2)
+				swahw32s(temp + i);
+		}
+	}
+#endif
+
+	/*
+	 * When dynamic burst enabled, DMA RX always receives 32-bit words from RXFIFO with
+	 * buswidth = 4, but when data_len is not 4-bytes alignment, the RM shows when
+	 * burst length = 32*n + m bits, a SPI burst contains the m LSB in first word and all
+	 * 32 bits in other n words. So if garbage bytes in the first word, trim first word then
+	 * copy the actual data to rx_buf.
+	 */
+	if (dma_data->data_len % BYTES_PER_32BITS_WORD && !word_delay) {
+		unaligned = dma_data->data_len % BYTES_PER_32BITS_WORD;
+		copy_ptr = (u8 *)dma_data->dma_rx_buf + BYTES_PER_32BITS_WORD - unaligned;
+	} else {
+		copy_ptr = dma_data->dma_rx_buf;
+	}
+
+	memcpy(rx_buf, copy_ptr, dma_data->data_len);
+}
+
+static int spi_imx_dma_map(struct spi_imx_data *spi_imx,
+			   struct dma_data_package *dma_data)
 {
-	struct dma_async_tx_descriptor *desc_tx, *desc_rx;
-	unsigned long transfer_timeout;
-	unsigned long time_left;
 	struct spi_controller *controller = spi_imx->controller;
-	struct sg_table *tx = &transfer->tx_sg, *rx = &transfer->rx_sg;
-	struct scatterlist *last_sg = sg_last(rx->sgl, rx->nents);
-	unsigned int bytes_per_word, i;
+	struct device *tx_dev = controller->dma_tx->device->dev;
+	struct device *rx_dev = controller->dma_rx->device->dev;
 	int ret;
 
-	/* Get the right burst length from the last sg to ensure no tail data */
-	bytes_per_word = spi_imx_bytes_per_word(transfer->bits_per_word);
-	for (i = spi_imx->devtype_data->fifo_size / 2; i > 0; i--) {
-		if (!(sg_dma_len(last_sg) % (i * bytes_per_word)))
-			break;
+	dma_data->dma_tx_addr = dma_map_single(tx_dev, dma_data->dma_tx_buf,
+					       DMA_CACHE_ALIGNED_LEN(dma_data->dma_len),
+					       DMA_TO_DEVICE);
+	ret = dma_mapping_error(tx_dev, dma_data->dma_tx_addr);
+	if (ret < 0) {
+		dev_err(spi_imx->dev, "DMA TX map failed %d\n", ret);
+		return ret;
 	}
-	/* Use 1 as wml in case no available burst length got */
-	if (i == 0)
-		i = 1;
 
-	spi_imx->wml =  i;
+	dma_data->dma_rx_addr = dma_map_single(rx_dev, dma_data->dma_rx_buf,
+					       DMA_CACHE_ALIGNED_LEN(dma_data->dma_len),
+					       DMA_FROM_DEVICE);
+	ret = dma_mapping_error(rx_dev, dma_data->dma_rx_addr);
+	if (ret < 0) {
+		dev_err(spi_imx->dev, "DMA RX map failed %d\n", ret);
+		dma_unmap_single(tx_dev, dma_data->dma_tx_addr,
+				 DMA_CACHE_ALIGNED_LEN(dma_data->dma_len),
+				 DMA_TO_DEVICE);
+		return ret;
+	}
 
-	ret = spi_imx_dma_configure(controller);
-	if (ret)
-		goto dma_failure_no_start;
+	return 0;
+}
 
-	if (!spi_imx->devtype_data->setup_wml) {
-		dev_err(spi_imx->dev, "No setup_wml()?\n");
-		ret = -EINVAL;
-		goto dma_failure_no_start;
+static int spi_imx_dma_tx_data_handle(struct spi_imx_data *spi_imx,
+				      struct dma_data_package *dma_data,
+				      const void *tx_buf,
+				      bool word_delay)
+{
+	void *copy_ptr;
+	int unaligned;
+
+	if (word_delay) {
+		dma_data->dma_len = dma_data->data_len;
+	} else {
+		/*
+		 * As per the reference manual, when burst length = 32*n + m bits, ECSPI
+		 * sends m LSB bits in the first word, followed by n full 32-bit words.
+		 * Since actual data may not be 4-byte aligned, allocate DMA TX/RX buffers
+		 * to ensure alignment. For TX, DMA pushes 4-byte aligned words to TXFIFO,
+		 * while ECSPI uses BURST_LENGTH settings to maintain correct bit count.
+		 * For RX, DMA always receives 32-bit words from RXFIFO, when data len is
+		 * not 4-byte aligned, trim the first word to drop garbage bytes, then group
+		 * all transfer DMA bounse buffer and copy all valid data to rx_buf.
+		 */
+		dma_data->dma_len = ALIGN(dma_data->data_len, BYTES_PER_32BITS_WORD);
 	}
-	spi_imx->devtype_data->setup_wml(spi_imx);
+
+	dma_data->dma_tx_buf = kzalloc(dma_data->dma_len, GFP_KERNEL);
+	if (!dma_data->dma_tx_buf)
+		return -ENOMEM;
+
+	dma_data->dma_rx_buf = kzalloc(dma_data->dma_len, GFP_KERNEL);
+	if (!dma_data->dma_rx_buf) {
+		kfree(dma_data->dma_tx_buf);
+		return -ENOMEM;
+	}
+
+	if (dma_data->data_len % BYTES_PER_32BITS_WORD && !word_delay) {
+		unaligned = dma_data->data_len % BYTES_PER_32BITS_WORD;
+		copy_ptr = (u8 *)dma_data->dma_tx_buf + BYTES_PER_32BITS_WORD - unaligned;
+	} else {
+		copy_ptr = dma_data->dma_tx_buf;
+	}
+
+	memcpy(copy_ptr, tx_buf, dma_data->data_len);
+
+	/*
+	 * When word_delay is enabled, DMA transfers an entire word in one minor loop.
+	 * In this case, no data requires additional handling.
+	 */
+	if (word_delay)
+		return 0;
+
+#ifdef __LITTLE_ENDIAN
+	/*
+	 * On little-endian CPUs, adjust byte order:
+	 * - Swap bytes when bpw = 8
+	 * - Swap half-words when bpw = 16
+	 * This ensures correct data ordering for DMA transfers.
+	 */
+	unsigned int bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);
+	u32 *temp = dma_data->dma_tx_buf;
+
+	for (int i = 0; i < DIV_ROUND_UP(dma_data->dma_len, sizeof(*temp)); i++) {
+		if (bytes_per_word == 1)
+			swab32s(temp + i);
+		else if (bytes_per_word == 2)
+			swahw32s(temp + i);
+	}
+#endif
+
+	return 0;
+}
+
+static int spi_imx_dma_data_prepare(struct spi_imx_data *spi_imx,
+				    struct spi_transfer *transfer,
+				    bool word_delay)
+{
+	u32 pre_bl, tail_bl;
+	u32 ctrl;
+	int ret;
+
+	/*
+	 * ECSPI supports a maximum burst of 512 bytes. When xfer->len exceeds 512
+	 * and is not a multiple of 512, a tail transfer is required. BURST_LEGTH
+	 * is used for SPI HW to maintain correct bit count. BURST_LENGTH should
+	 * update with data length. After DMA request submit, SPI can not update the
+	 * BURST_LENGTH, in this case, we must split two package, update the register
+	 * then setup second DMA transfer.
+	 */
+	ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);
+	if (word_delay) {
+		/*
+		 * When SPI IMX need to support word delay, according to "Sample Period Control
+		 * Register" shows, The Sample Period Control Register (ECSPI_PERIODREG)
+		 * provides software a way to insert delays (wait states) between consecutive
+		 * SPI transfers. As a result, ECSPI can only transfer one word per frame, and
+		 * the delay occurs between frames.
+		 */
+		spi_imx->dma_package_num = 1;
+		pre_bl = spi_imx->bits_per_word - 1;
+	} else if (transfer->len <= MX51_ECSPI_CTRL_MAX_BURST) {
+		spi_imx->dma_package_num = 1;
+		pre_bl = transfer->len * BITS_PER_BYTE - 1;
+	} else if (!(transfer->len % MX51_ECSPI_CTRL_MAX_BURST)) {
+		spi_imx->dma_package_num = 1;
+		pre_bl = MX51_ECSPI_CTRL_MAX_BURST * BITS_PER_BYTE - 1;
+	} else {
+		spi_imx->dma_package_num = 2;
+		pre_bl = MX51_ECSPI_CTRL_MAX_BURST * BITS_PER_BYTE - 1;
+		tail_bl = (transfer->len % MX51_ECSPI_CTRL_MAX_BURST) * BITS_PER_BYTE - 1;
+	}
+
+	spi_imx->dma_data = kmalloc_array(spi_imx->dma_package_num,
+					  sizeof(struct dma_data_package),
+					  GFP_KERNEL | __GFP_ZERO);
+	if (!spi_imx->dma_data) {
+		dev_err(spi_imx->dev, "Failed to allocate DMA package buffer!\n");
+		return -ENOMEM;
+	}
+
+	if (spi_imx->dma_package_num == 1) {
+		ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;
+		ctrl |= pre_bl << MX51_ECSPI_CTRL_BL_OFFSET;
+		spi_imx->dma_data[0].cmd_word = ctrl;
+		spi_imx->dma_data[0].data_len = transfer->len;
+		ret = spi_imx_dma_tx_data_handle(spi_imx, &spi_imx->dma_data[0], transfer->tx_buf,
+						 word_delay);
+		if (ret) {
+			kfree(spi_imx->dma_data);
+			return ret;
+		}
+	} else {
+		ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;
+		ctrl |= pre_bl << MX51_ECSPI_CTRL_BL_OFFSET;
+		spi_imx->dma_data[0].cmd_word = ctrl;
+		spi_imx->dma_data[0].data_len = round_down(transfer->len,
+							   MX51_ECSPI_CTRL_MAX_BURST);
+		ret = spi_imx_dma_tx_data_handle(spi_imx, &spi_imx->dma_data[0], transfer->tx_buf,
+						 false);
+		if (ret) {
+			kfree(spi_imx->dma_data);
+			return ret;
+		}
+
+		ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;
+		ctrl |= tail_bl << MX51_ECSPI_CTRL_BL_OFFSET;
+		spi_imx->dma_data[1].cmd_word = ctrl;
+		spi_imx->dma_data[1].data_len = transfer->len % MX51_ECSPI_CTRL_MAX_BURST;
+		ret = spi_imx_dma_tx_data_handle(spi_imx, &spi_imx->dma_data[1],
+						 transfer->tx_buf + spi_imx->dma_data[0].data_len,
+						 false);
+		if (ret) {
+			kfree(spi_imx->dma_data[0].dma_tx_buf);
+			kfree(spi_imx->dma_data[0].dma_rx_buf);
+			kfree(spi_imx->dma_data);
+		}
+	}
+
+	return 0;
+}
+
+static int spi_imx_dma_submit(struct spi_imx_data *spi_imx,
+			      struct dma_data_package *dma_data,
+			      struct spi_transfer *transfer)
+{
+	struct spi_controller *controller = spi_imx->controller;
+	struct dma_async_tx_descriptor *desc_tx, *desc_rx;
+	unsigned long transfer_timeout;
+	unsigned long time_left;
+	dma_cookie_t cookie;
 
 	/*
 	 * The TX DMA setup starts the transfer, so make sure RX is configured
 	 * before TX.
 	 */
-	desc_rx = dmaengine_prep_slave_sg(controller->dma_rx,
-				rx->sgl, rx->nents, DMA_DEV_TO_MEM,
-				DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+	desc_rx = dmaengine_prep_slave_single(controller->dma_rx, dma_data->dma_rx_addr,
+					      dma_data->dma_len, DMA_DEV_TO_MEM,
+					      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 	if (!desc_rx) {
-		ret = -EINVAL;
-		goto dma_failure_no_start;
+		transfer->error |= SPI_TRANS_FAIL_NO_START;
+		return -EINVAL;
 	}
 
 	desc_rx->callback = spi_imx_dma_rx_callback;
 	desc_rx->callback_param = (void *)spi_imx;
-	dmaengine_submit(desc_rx);
+	cookie = dmaengine_submit(desc_rx);
+	if (dma_submit_error(cookie)) {
+		dev_err(spi_imx->dev, "submitting DMA RX failed\n");
+		transfer->error |= SPI_TRANS_FAIL_NO_START;
+		goto dmaengine_terminate_rx;
+	}
+
 	reinit_completion(&spi_imx->dma_rx_completion);
 	dma_async_issue_pending(controller->dma_rx);
 
-	desc_tx = dmaengine_prep_slave_sg(controller->dma_tx,
-				tx->sgl, tx->nents, DMA_MEM_TO_DEV,
-				DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
-	if (!desc_tx) {
-		dmaengine_terminate_all(controller->dma_tx);
-		dmaengine_terminate_all(controller->dma_rx);
-		return -EINVAL;
-	}
+	desc_tx = dmaengine_prep_slave_single(controller->dma_tx, dma_data->dma_tx_addr,
+					      dma_data->dma_len, DMA_MEM_TO_DEV,
+					      DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+	if (!desc_tx)
+		goto dmaengine_terminate_rx;
 
 	desc_tx->callback = spi_imx_dma_tx_callback;
 	desc_tx->callback_param = (void *)spi_imx;
-	dmaengine_submit(desc_tx);
+	cookie = dmaengine_submit(desc_tx);
+	if (dma_submit_error(cookie)) {
+		dev_err(spi_imx->dev, "submitting DMA TX failed\n");
+		goto dmaengine_terminate_tx;
+	}
 	reinit_completion(&spi_imx->dma_tx_completion);
 	dma_async_issue_pending(controller->dma_tx);
 
@@ -1553,25 +1769,163 @@ static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
 
 	transfer_timeout = spi_imx_calculate_timeout(spi_imx, transfer->len);
 
-	/* Wait SDMA to finish the data transfer.*/
-	time_left = wait_for_completion_timeout(&spi_imx->dma_tx_completion,
-						transfer_timeout);
-	if (!time_left) {
-		dev_err(spi_imx->dev, "I/O Error in DMA TX\n");
-		dmaengine_terminate_all(controller->dma_tx);
-		dmaengine_terminate_all(controller->dma_rx);
-		return -ETIMEDOUT;
+	if (!spi_imx->target_mode) {
+		/* Wait SDMA to finish the data transfer.*/
+		time_left = wait_for_completion_timeout(&spi_imx->dma_tx_completion,
+							transfer_timeout);
+		if (!time_left) {
+			dev_err(spi_imx->dev, "I/O Error in DMA TX\n");
+			dmaengine_terminate_all(controller->dma_tx);
+			dmaengine_terminate_all(controller->dma_rx);
+			return -ETIMEDOUT;
+		}
+
+		time_left = wait_for_completion_timeout(&spi_imx->dma_rx_completion,
+							transfer_timeout);
+		if (!time_left) {
+			dev_err(&controller->dev, "I/O Error in DMA RX\n");
+			spi_imx->devtype_data->reset(spi_imx);
+			dmaengine_terminate_all(controller->dma_rx);
+			return -ETIMEDOUT;
+		}
+	} else {
+		spi_imx->target_aborted = false;
+
+		if (wait_for_completion_interruptible(&spi_imx->dma_tx_completion) ||
+		    READ_ONCE(spi_imx->target_aborted)) {
+			dev_dbg(spi_imx->dev, "I/O Error in DMA TX interrupted\n");
+			dmaengine_terminate_all(controller->dma_tx);
+			dmaengine_terminate_all(controller->dma_rx);
+			return -EINTR;
+		}
+
+		if (wait_for_completion_interruptible(&spi_imx->dma_rx_completion) ||
+		    READ_ONCE(spi_imx->target_aborted)) {
+			dev_dbg(spi_imx->dev, "I/O Error in DMA RX interrupted\n");
+			dmaengine_terminate_all(controller->dma_rx);
+			return -EINTR;
+		}
+
+		/*
+		 * ECSPI has a HW issue when works in Target mode, after 64 words
+		 * writtern to TXFIFO, even TXFIFO becomes empty, ECSPI_TXDATA keeps
+		 * shift out the last word data, so we have to disable ECSPI when in
+		 * target mode after the transfer completes.
+		 */
+		if (spi_imx->devtype_data->disable)
+			spi_imx->devtype_data->disable(spi_imx);
 	}
 
-	time_left = wait_for_completion_timeout(&spi_imx->dma_rx_completion,
-						transfer_timeout);
-	if (!time_left) {
-		dev_err(&controller->dev, "I/O Error in DMA RX\n");
-		spi_imx->devtype_data->reset(spi_imx);
-		dmaengine_terminate_all(controller->dma_rx);
-		return -ETIMEDOUT;
+	return 0;
+
+dmaengine_terminate_tx:
+	dmaengine_terminate_all(controller->dma_tx);
+dmaengine_terminate_rx:
+	dmaengine_terminate_all(controller->dma_rx);
+
+	return -EINVAL;
+}
+
+static void spi_imx_dma_max_wml_find(struct spi_imx_data *spi_imx,
+				     struct dma_data_package *dma_data,
+				     bool word_delay)
+{
+	unsigned int bytes_per_word = word_delay ?
+				      spi_imx_bytes_per_word(spi_imx->bits_per_word) :
+				      BYTES_PER_32BITS_WORD;
+	unsigned int i;
+
+	for (i = spi_imx->devtype_data->fifo_size / 2; i > 0; i--) {
+		if (!dma_data->dma_len % (i * bytes_per_word))
+			break;
+	}
+	/* Use 1 as wml in case no available burst length got */
+	if (i == 0)
+		i = 1;
+
+	spi_imx->wml = i;
+}
+
+static int spi_imx_dma_configure(struct spi_controller *controller, bool word_delay)
+{
+	int ret;
+	enum dma_slave_buswidth buswidth;
+	struct dma_slave_config rx = {}, tx = {};
+	struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller);
+
+	if (word_delay) {
+		switch (spi_imx_bytes_per_word(spi_imx->bits_per_word)) {
+		case 4:
+			buswidth = DMA_SLAVE_BUSWIDTH_4_BYTES;
+			break;
+		case 2:
+			buswidth = DMA_SLAVE_BUSWIDTH_2_BYTES;
+			break;
+		case 1:
+			buswidth = DMA_SLAVE_BUSWIDTH_1_BYTE;
+			break;
+		default:
+			return -EINVAL;
+		}
+	} else {
+		buswidth = DMA_SLAVE_BUSWIDTH_4_BYTES;
+	}
+
+	tx.direction = DMA_MEM_TO_DEV;
+	tx.dst_addr = spi_imx->base_phys + MXC_CSPITXDATA;
+	tx.dst_addr_width = buswidth;
+	tx.dst_maxburst = spi_imx->wml;
+	ret = dmaengine_slave_config(controller->dma_tx, &tx);
+	if (ret) {
+		dev_err(spi_imx->dev, "TX dma configuration failed with %d\n", ret);
+		return ret;
+	}
+
+	rx.direction = DMA_DEV_TO_MEM;
+	rx.src_addr = spi_imx->base_phys + MXC_CSPIRXDATA;
+	rx.src_addr_width = buswidth;
+	rx.src_maxburst = spi_imx->wml;
+	ret = dmaengine_slave_config(controller->dma_rx, &rx);
+	if (ret) {
+		dev_err(spi_imx->dev, "RX dma configuration failed with %d\n", ret);
+		return ret;
 	}
 
+	return 0;
+}
+
+static int spi_imx_dma_package_transfer(struct spi_imx_data *spi_imx,
+					struct dma_data_package *dma_data,
+					struct spi_transfer *transfer,
+					bool word_delay)
+{
+	struct spi_controller *controller = spi_imx->controller;
+	int ret;
+
+	spi_imx_dma_max_wml_find(spi_imx, dma_data, word_delay);
+
+	ret = spi_imx_dma_configure(controller, word_delay);
+	if (ret)
+		goto dma_failure_no_start;
+
+	if (!spi_imx->devtype_data->setup_wml) {
+		dev_err(spi_imx->dev, "No setup_wml()?\n");
+		ret = -EINVAL;
+		goto dma_failure_no_start;
+	}
+	spi_imx->devtype_data->setup_wml(spi_imx);
+
+	ret = spi_imx_dma_submit(spi_imx, dma_data, transfer);
+	if (ret)
+		return ret;
+
+	/* Trim the DMA RX buffer and copy the actual data to rx_buf */
+	dma_sync_single_for_cpu(controller->dma_rx->device->dev, dma_data->dma_rx_addr,
+				dma_data->dma_len, DMA_FROM_DEVICE);
+	spi_imx_dma_rx_data_handle(spi_imx, dma_data, transfer->rx_buf + spi_imx->rx_offset,
+				   word_delay);
+	spi_imx->rx_offset += dma_data->data_len;
+
 	return 0;
 /* fallback to pio */
 dma_failure_no_start:
@@ -1579,6 +1933,57 @@ static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
 	return ret;
 }
 
+static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
+				struct spi_transfer *transfer)
+{
+	bool word_delay = transfer->word_delay.value != 0 && !spi_imx->target_mode;
+	int ret;
+	int i;
+
+	ret = spi_imx_dma_data_prepare(spi_imx, transfer, word_delay);
+	if (ret < 0) {
+		transfer->error |= SPI_TRANS_FAIL_NO_START;
+		dev_err(spi_imx->dev, "DMA data prepare fail\n");
+		goto fallback_pio;
+	}
+
+	spi_imx->rx_offset = 0;
+
+	/* Each dma_package performs a separate DMA transfer once */
+	for (i = 0; i < spi_imx->dma_package_num; i++) {
+		ret = spi_imx_dma_map(spi_imx, &spi_imx->dma_data[i]);
+		if (ret < 0) {
+			if (i == 0)
+				transfer->error |= SPI_TRANS_FAIL_NO_START;
+			dev_err(spi_imx->dev, "DMA map fail\n");
+			break;
+		}
+
+		/* Update the CTRL register BL field */
+		writel(spi_imx->dma_data[i].cmd_word, spi_imx->base + MX51_ECSPI_CTRL);
+
+		ret = spi_imx_dma_package_transfer(spi_imx, &spi_imx->dma_data[i],
+						   transfer, word_delay);
+
+		/* Whether the dma transmission is successful or not, dma unmap is necessary */
+		spi_imx_dma_unmap(spi_imx, &spi_imx->dma_data[i]);
+
+		if (ret < 0) {
+			dev_dbg(spi_imx->dev, "DMA %d transfer not really finish\n", i);
+			break;
+		}
+	}
+
+	for (int j = 0; j < spi_imx->dma_package_num; j++) {
+		kfree(spi_imx->dma_data[j].dma_tx_buf);
+		kfree(spi_imx->dma_data[j].dma_rx_buf);
+	}
+	kfree(spi_imx->dma_data);
+
+fallback_pio:
+	return ret;
+}
+
 static int spi_imx_pio_transfer(struct spi_device *spi,
 				struct spi_transfer *transfer)
 {
@@ -1737,7 +2142,7 @@ static int spi_imx_transfer_one(struct spi_controller *controller,
 	while (spi_imx->devtype_data->rx_available(spi_imx))
 		readl(spi_imx->base + MXC_CSPIRXDATA);
 
-	if (spi_imx->target_mode)
+	if (spi_imx->target_mode && !spi_imx->usedma)
 		return spi_imx_pio_transfer_target(spi, transfer);
 
 	/*
@@ -1745,9 +2150,17 @@ static int spi_imx_transfer_one(struct spi_controller *controller,
 	 * transfer, the SPI transfer has already been mapped, so we
 	 * have to do the DMA transfer here.
 	 */
-	if (spi_imx->usedma)
-		return spi_imx_dma_transfer(spi_imx, transfer);
-
+	if (spi_imx->usedma) {
+		ret = spi_imx_dma_transfer(spi_imx, transfer);
+		if (transfer->error & SPI_TRANS_FAIL_NO_START) {
+			spi_imx->usedma = false;
+			if (spi_imx->target_mode)
+				return spi_imx_pio_transfer_target(spi, transfer);
+			else
+				return spi_imx_pio_transfer(spi, transfer);
+		}
+		return ret;
+	}
 	/* run in polling mode for short transfers */
 	if (transfer->len == 1 || (polling_limit_us &&
 				   spi_imx_transfer_estimate_time_us(transfer) < polling_limit_us))
diff --git a/drivers/spi/spi-mpfs.c b/drivers/spi/spi-mpfs.c
index 9a14d1732a15..7e9e64d8e6c8 100644
--- a/drivers/spi/spi-mpfs.c
+++ b/drivers/spi/spi-mpfs.c
@@ -577,6 +577,7 @@ static int mpfs_spi_probe(struct platform_device *pdev)
 
 	ret = devm_spi_register_controller(&pdev->dev, host);
 	if (ret) {
+		mpfs_spi_disable_ints(spi);
 		mpfs_spi_disable(spi);
 		return dev_err_probe(&pdev->dev, ret,
 				     "unable to register host for SPI controller\n");
diff --git a/drivers/spi/spi-mt65xx.c b/drivers/spi/spi-mt65xx.c
index 4b40985af1ea..90e5813cfdc3 100644
--- a/drivers/spi/spi-mt65xx.c
+++ b/drivers/spi/spi-mt65xx.c
@@ -1320,7 +1320,7 @@ static int mtk_spi_probe(struct platform_device *pdev)
 
 	ret = devm_request_threaded_irq(dev, irq, mtk_spi_interrupt,
 					mtk_spi_interrupt_thread,
-					IRQF_TRIGGER_NONE, dev_name(dev), host);
+					IRQF_ONESHOT, dev_name(dev), host);
 	if (ret)
 		return dev_err_probe(dev, ret, "failed to register irq\n");
 
diff --git a/drivers/spi/spi-nxp-xspi.c b/drivers/spi/spi-nxp-xspi.c
new file mode 100644
index 000000000000..25339492bf3a
--- /dev/null
+++ b/drivers/spi/spi-nxp-xspi.c
@@ -0,0 +1,1385 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/*
+ * NXP xSPI controller driver.
+ *
+ * Copyright 2025 NXP
+ *
+ * xSPI is a flexible SPI host controller which supports single
+ * external devices. This device can have up to eight bidirectional
+ * data lines, this means xSPI support Single/Dual/Quad/Octal mode
+ * data transfer (1/2/4/8 bidirectional data lines).
+ *
+ * xSPI controller is driven by the LUT(Look-up Table) registers
+ * LUT registers are a look-up-table for sequences of instructions.
+ * A valid sequence consists of five LUT registers.
+ * Maximum 16 LUT sequences can be programmed simultaneously.
+ *
+ * LUTs are being created at run-time based on the commands passed
+ * from the spi-mem framework, thus using single LUT index.
+ *
+ * Software triggered Flash read/write access by IP Bus.
+ *
+ * Memory mapped read access by AHB Bus.
+ *
+ * Based on SPI MEM interface and spi-nxp-fspi.c driver.
+ *
+ * Author:
+ *     Haibo Chen <haibo.chen@nxp.com>
+ * Co-author:
+ *     Han Xu <han.xu@nxp.com>
+ */
+
+#include <linux/bitops.h>
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/jiffies.h>
+#include <linux/kernel.h>
+#include <linux/log2.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/pm_runtime.h>
+#include <linux/spi/spi.h>
+#include <linux/spi/spi-mem.h>
+
+/* Runtime pm timeout */
+#define XSPI_RPM_TIMEOUT_MS 50	/* 50ms */
+/*
+ * The driver only uses one single LUT entry, that is updated on
+ * each call of exec_op(). Index 0 is preset at boot with a basic
+ * read operation, so let's use the last entry (15).
+ */
+#define	XSPI_SEQID_LUT			15
+
+#define XSPI_MCR			0x0
+#define XSPI_MCR_CKN_FA_EN		BIT(26)
+#define XSPI_MCR_DQS_FA_SEL_MASK	GENMASK(25, 24)
+#define XSPI_MCR_ISD3FA			BIT(17)
+#define XSPI_MCR_ISD2FA			BIT(16)
+#define XSPI_MCR_DOZE			BIT(15)
+#define XSPI_MCR_MDIS			BIT(14)
+#define XSPI_MCR_DLPEN			BIT(12)
+#define XSPI_MCR_CLR_TXF		BIT(11)
+#define XSPI_MCR_CLR_RXF		BIT(10)
+#define XSPI_MCR_IPS_TG_RST		BIT(9)
+#define XSPI_MCR_VAR_LAT_EN		BIT(8)
+#define XSPI_MCR_DDR_EN			BIT(7)
+#define XSPI_MCR_DQS_EN			BIT(6)
+#define XSPI_MCR_DQS_LAT_EN		BIT(5)
+#define XSPI_MCR_DQS_OUT_EN		BIT(4)
+#define XSPI_MCR_SWRSTHD		BIT(1)
+#define XSPI_MCR_SWRSTSD		BIT(0)
+
+#define XSPI_IPCR			0x8
+
+#define XSPI_FLSHCR			0xC
+#define XSPI_FLSHCR_TDH_MASK		GENMASK(17, 16)
+#define XSPI_FLSHCR_TCSH_MASK		GENMASK(11, 8)
+#define XSPI_FLSHCR_TCSS_MASK		GENMASK(3, 0)
+
+#define XSPI_BUF0CR			0x10
+#define XSPI_BUF1CR			0x14
+#define XSPI_BUF2CR			0x18
+#define XSPI_BUF3CR			0x1C
+#define XSPI_BUF3CR_ALLMST		BIT(31)
+#define XSPI_BUF3CR_ADATSZ_MASK		GENMASK(17, 8)
+#define XSPI_BUF3CR_MSTRID_MASK		GENMASK(3, 0)
+
+#define XSPI_BFGENCR			0x20
+#define XSPI_BFGENCR_SEQID_WR_MASK	GENMASK(31, 28)
+#define XSPI_BFGENCR_ALIGN_MASK		GENMASK(24, 22)
+#define XSPI_BFGENCR_PPWF_CLR		BIT(20)
+#define XSPI_BFGENCR_WR_FLUSH_EN	BIT(21)
+#define XSPI_BFGENCR_SEQID_WR_EN	BIT(17)
+#define XSPI_BFGENCR_SEQID_MASK		GENMASK(15, 12)
+
+#define XSPI_BUF0IND			0x30
+#define XSPI_BUF1IND			0x34
+#define XSPI_BUF2IND			0x38
+
+#define XSPI_DLLCRA			0x60
+#define XSPI_DLLCRA_DLLEN		BIT(31)
+#define XSPI_DLLCRA_FREQEN		BIT(30)
+#define XSPI_DLLCRA_DLL_REFCNTR_MASK	GENMASK(27, 24)
+#define XSPI_DLLCRA_DLLRES_MASK		GENMASK(23, 20)
+#define XSPI_DLLCRA_SLV_FINE_MASK	GENMASK(19, 16)
+#define XSPI_DLLCRA_SLV_DLY_MASK	GENMASK(14, 12)
+#define XSPI_DLLCRA_SLV_DLY_COARSE_MASK	GENMASK(11,  8)
+#define XSPI_DLLCRA_SLV_DLY_FINE_MASK	GENMASK(7, 5)
+#define XSPI_DLLCRA_DLL_CDL8		BIT(4)
+#define XSPI_DLLCRA_SLAVE_AUTO_UPDT	BIT(3)
+#define XSPI_DLLCRA_SLV_EN		BIT(2)
+#define XSPI_DLLCRA_SLV_DLL_BYPASS	BIT(1)
+#define XSPI_DLLCRA_SLV_UPD		BIT(0)
+
+#define XSPI_SFAR			0x100
+
+#define XSPI_SFACR			0x104
+#define XSPI_SFACR_FORCE_A10		BIT(22)
+#define XSPI_SFACR_WA_4B_EN		BIT(21)
+#define XSPI_SFACR_CAS_INTRLVD		BIT(20)
+#define XSPI_SFACR_RX_BP_EN		BIT(18)
+#define XSPI_SFACR_BYTE_SWAP		BIT(17)
+#define XSPI_SFACR_WA			BIT(16)
+#define XSPI_SFACR_CAS_MASK		GENMASK(3, 0)
+
+#define XSPI_SMPR			0x108
+#define XSPI_SMPR_DLLFSMPFA_MASK	GENMASK(26, 24)
+#define XSPI_SMPR_FSDLY			BIT(6)
+#define XSPI_SMPR_FSPHS			BIT(5)
+
+#define XSPI_RBSR			0x10C
+
+#define XSPI_RBCT			0x110
+#define XSPI_RBCT_WMRK_MASK		GENMASK(6, 0)
+
+#define XSPI_DLLSR			0x12C
+#define XSPI_DLLSR_DLLA_LOCK		BIT(15)
+#define XSPI_DLLSR_SLVA_LOCK		BIT(14)
+#define XSPI_DLLSR_DLLA_RANGE_ERR	BIT(13)
+#define XSPI_DLLSR_DLLA_FINE_UNDERFLOW	BIT(12)
+
+#define XSPI_TBSR			0x150
+
+#define XSPI_TBDR			0x154
+
+#define XSPI_TBCT			0x158
+#define XSPI_TBCT_WMRK_MASK		GENMASK(7, 0)
+
+#define XSPI_SR				0x15C
+#define XSPI_SR_TXFULL			BIT(27)
+#define XSPI_SR_TXDMA			BIT(26)
+#define XSPI_SR_TXWA			BIT(25)
+#define XSPI_SR_TXNE			BIT(24)
+#define XSPI_SR_RXDMA			BIT(23)
+#define XSPI_SR_ARB_STATE_MASK		GENMASK(23, 20)
+#define XSPI_SR_RXFULL			BIT(19)
+#define XSPI_SR_RXWE			BIT(16)
+#define XSPI_SR_ARB_LCK			BIT(15)
+#define XSPI_SR_AHBnFUL			BIT(11)
+#define XSPI_SR_AHBnNE			BIT(7)
+#define XSPI_SR_AHBTRN			BIT(6)
+#define XSPI_SR_AWRACC			BIT(4)
+#define XSPI_SR_AHB_ACC			BIT(2)
+#define XSPI_SR_IP_ACC			BIT(1)
+#define XSPI_SR_BUSY			BIT(0)
+
+#define XSPI_FR				0x160
+#define XSPI_FR_DLPFF			BIT(31)
+#define XSPI_FR_DLLABRT			BIT(28)
+#define XSPI_FR_TBFF			BIT(27)
+#define XSPI_FR_TBUF			BIT(26)
+#define XSPI_FR_DLLUNLCK		BIT(24)
+#define XSPI_FR_ILLINE			BIT(23)
+#define XSPI_FR_RBOF			BIT(17)
+#define XSPI_FR_RBDF			BIT(16)
+#define XSPI_FR_AAEF			BIT(15)
+#define XSPI_FR_AITEF			BIT(14)
+#define XSPI_FR_AIBSEF			BIT(13)
+#define XSPI_FR_ABOF			BIT(12)
+#define XSPI_FR_CRCAEF			BIT(10)
+#define XSPI_FR_PPWF			BIT(8)
+#define XSPI_FR_IPIEF			BIT(6)
+#define XSPI_FR_IPEDERR			BIT(5)
+#define XSPI_FR_PERFOVF			BIT(2)
+#define XSPI_FR_RDADDR			BIT(1)
+#define XSPI_FR_TFF			BIT(0)
+
+#define XSPI_RSER			0x164
+#define XSPI_RSER_TFIE			BIT(0)
+
+#define XSPI_SFA1AD			0x180
+
+#define XSPI_SFA2AD			0x184
+
+#define XSPI_RBDR0			0x200
+
+#define XSPI_LUTKEY			0x300
+#define XSPI_LUT_KEY_VAL		(0x5AF05AF0UL)
+
+#define XSPI_LCKCR			0x304
+#define XSPI_LOKCR_LOCK			BIT(0)
+#define XSPI_LOKCR_UNLOCK		BIT(1)
+
+#define XSPI_LUT			0x310
+#define XSPI_LUT_OFFSET			(XSPI_SEQID_LUT * 5 * 4)
+#define XSPI_LUT_REG(idx) \
+	(XSPI_LUT + XSPI_LUT_OFFSET + (idx) * 4)
+
+#define XSPI_MCREXT			0x4FC
+#define XSPI_MCREXT_RST_MASK		GENMASK(3, 0)
+
+
+#define XSPI_FRAD0_WORD2		0x808
+#define XSPI_FRAD0_WORD2_MD0ACP_MASK	GENMASK(2, 0)
+
+#define XSPI_FRAD0_WORD3		0x80C
+#define XSPI_FRAD0_WORD3_VLD		BIT(31)
+
+#define XSPI_TG0MDAD			0x900
+#define XSPI_TG0MDAD_VLD		BIT(31)
+
+#define XSPI_TG1MDAD			0x910
+
+#define XSPI_MGC			0x920
+#define XSPI_MGC_GVLD			BIT(31)
+#define XSPI_MGC_GVLDMDAD		BIT(29)
+#define XSPI_MGC_GVLDFRAD		BIT(27)
+
+#define XSPI_MTO			0x928
+
+#define XSPI_ERRSTAT			0x938
+#define XSPI_INT_EN			0x93C
+
+#define XSPI_SFP_TG_IPCR		0x958
+#define XSPI_SFP_TG_IPCR_SEQID_MASK	GENMASK(27, 24)
+#define XSPI_SFP_TG_IPCR_ARB_UNLOCK	BIT(23)
+#define XSPI_SFP_TG_IPCR_ARB_LOCK	BIT(22)
+#define XSPI_SFP_TG_IPCR_IDATSZ_MASK	GENMASK(15, 0)
+
+#define XSPI_SFP_TG_SFAR 0x95C
+
+/* Register map end */
+
+/********* XSPI CMD definitions ***************************/
+#define LUT_STOP	0x00
+#define LUT_CMD_SDR	0x01
+#define LUT_ADDR_SDR	0x02
+#define LUT_DUMMY	0x03
+#define LUT_MODE8_SDR	0x04
+#define LUT_MODE2_SDR	0x05
+#define LUT_MODE4_SDR	0x06
+#define LUT_READ_SDR	0x07
+#define LUT_WRITE_SDR	0x08
+#define LUT_JMP_ON_CS	0x09
+#define LUT_ADDR_DDR	0x0A
+#define LUT_MODE8_DDR	0x0B
+#define LUT_MODE2_DDR	0x0C
+#define LUT_MODE4_DDR	0x0D
+#define LUT_READ_DDR	0x0E
+#define LUT_WRITE_DDR	0x0F
+#define LUT_DATA_LEARN	0x10
+#define LUT_CMD_DDR	0x11
+#define LUT_CADDR_SDR	0x12
+#define LUT_CADDR_DDR	0x13
+#define JMP_TO_SEQ	0x14
+
+#define XSPI_64BIT_LE	0x3
+/*
+ * Calculate number of required PAD bits for LUT register.
+ *
+ * The pad stands for the number of IO lines [0:7].
+ * For example, the octal read needs eight IO lines,
+ * so you should use LUT_PAD(8). This macro
+ * returns 3 i.e. use eight (2^3) IP lines for read.
+ */
+#define LUT_PAD(x) (fls(x) - 1)
+
+/*
+ * Macro for constructing the LUT entries with the following
+ * register layout:
+ *
+ *  ---------------------------------------------------
+ *  | INSTR1 | PAD1 | OPRND1 | INSTR0 | PAD0 | OPRND0 |
+ *  ---------------------------------------------------
+ */
+#define PAD_SHIFT		8
+#define INSTR_SHIFT		10
+#define OPRND_SHIFT		16
+
+/* Macros for constructing the LUT register. */
+#define LUT_DEF(idx, ins, pad, opr)			  \
+	((((ins) << INSTR_SHIFT) | ((pad) << PAD_SHIFT) | \
+	(opr)) << (((idx) % 2) * OPRND_SHIFT))
+
+#define NXP_XSPI_MIN_IOMAP	SZ_4M
+#define NXP_XSPI_MAX_CHIPSELECT		2
+#define POLL_TOUT_US		5000
+
+/* Access flash memory using IP bus only */
+#define XSPI_QUIRK_USE_IP_ONLY	BIT(0)
+
+struct nxp_xspi_devtype_data {
+	unsigned int rxfifo;
+	unsigned int txfifo;
+	unsigned int ahb_buf_size;
+	unsigned int quirks;
+};
+
+static struct nxp_xspi_devtype_data imx94_data = {
+	.rxfifo = SZ_512,       /* (128 * 4 bytes)  */
+	.txfifo = SZ_1K,        /* (256 * 4 bytes)  */
+	.ahb_buf_size = SZ_4K,  /* (1024 * 4 bytes)  */
+};
+
+struct nxp_xspi {
+	void __iomem *iobase;
+	void __iomem *ahb_addr;
+	u32 memmap_phy;
+	u32 memmap_phy_size;
+	u32 memmap_start;
+	u32 memmap_len;
+	struct clk *clk;
+	struct device *dev;
+	struct completion c;
+	const struct nxp_xspi_devtype_data *devtype_data;
+	/* mutex lock for each operation */
+	struct mutex lock;
+	int selected;
+#define XSPI_DTR_PROTO		BIT(0)
+	int flags;
+	/* Save the previous operation clock rate */
+	unsigned long pre_op_rate;
+	/* The max clock rate xspi supported output to device */
+	unsigned long support_max_rate;
+};
+
+static inline int needs_ip_only(struct nxp_xspi *xspi)
+{
+	return xspi->devtype_data->quirks & XSPI_QUIRK_USE_IP_ONLY;
+}
+
+static irqreturn_t nxp_xspi_irq_handler(int irq, void *dev_id)
+{
+	struct nxp_xspi *xspi = dev_id;
+	u32 reg;
+
+	reg = readl(xspi->iobase + XSPI_FR);
+	if (reg & XSPI_FR_TFF) {
+		/* Clear interrupt */
+		writel(XSPI_FR_TFF, xspi->iobase + XSPI_FR);
+		complete(&xspi->c);
+		return IRQ_HANDLED;
+	}
+
+	return IRQ_NONE;
+}
+
+static int nxp_xspi_check_buswidth(struct nxp_xspi *xspi, u8 width)
+{
+	return (is_power_of_2(width) && width <= 8) ? 0 : -EOPNOTSUPP;
+}
+
+static bool nxp_xspi_supports_op(struct spi_mem *mem,
+				 const struct spi_mem_op *op)
+{
+	struct nxp_xspi *xspi = spi_controller_get_devdata(mem->spi->controller);
+	int ret;
+
+	ret = nxp_xspi_check_buswidth(xspi, op->cmd.buswidth);
+
+	if (op->addr.nbytes)
+		ret |= nxp_xspi_check_buswidth(xspi, op->addr.buswidth);
+
+	if (op->dummy.nbytes)
+		ret |= nxp_xspi_check_buswidth(xspi, op->dummy.buswidth);
+
+	if (op->data.nbytes)
+		ret |= nxp_xspi_check_buswidth(xspi, op->data.buswidth);
+
+	if (ret)
+		return false;
+
+	/*
+	 * The number of address bytes should be equal to or less than 4 bytes.
+	 */
+	if (op->addr.nbytes > 4)
+		return false;
+
+	/* Max 32 dummy clock cycles supported */
+	if (op->dummy.buswidth &&
+	    (op->dummy.nbytes * 8 / op->dummy.buswidth > 64))
+		return false;
+
+	if (needs_ip_only(xspi) && op->data.dir == SPI_MEM_DATA_IN &&
+	    op->data.nbytes > xspi->devtype_data->rxfifo)
+		return false;
+
+	if (op->data.dir == SPI_MEM_DATA_OUT &&
+			op->data.nbytes > xspi->devtype_data->txfifo)
+		return false;
+
+	return spi_mem_default_supports_op(mem, op);
+}
+
+static void nxp_xspi_prepare_lut(struct nxp_xspi *xspi,
+				 const struct spi_mem_op *op)
+{
+	void __iomem *base = xspi->iobase;
+	u32 lutval[5] = {};
+	int lutidx = 1, i;
+
+	/* cmd */
+	if (op->cmd.dtr) {
+		lutval[0] |= LUT_DEF(0, LUT_CMD_DDR, LUT_PAD(op->cmd.buswidth),
+				     op->cmd.opcode >> 8);
+		lutval[lutidx / 2] |= LUT_DEF(lutidx, LUT_CMD_DDR,
+					      LUT_PAD(op->cmd.buswidth),
+					      op->cmd.opcode & 0x00ff);
+		lutidx++;
+	} else {
+		lutval[0] |= LUT_DEF(0, LUT_CMD_SDR, LUT_PAD(op->cmd.buswidth),
+				     op->cmd.opcode);
+	}
+
+	/* Addr bytes */
+	if (op->addr.nbytes) {
+		lutval[lutidx / 2] |= LUT_DEF(lutidx, op->addr.dtr ?
+					      LUT_ADDR_DDR : LUT_ADDR_SDR,
+					      LUT_PAD(op->addr.buswidth),
+					      op->addr.nbytes * 8);
+		lutidx++;
+	}
+
+	/* Dummy bytes, if needed */
+	if (op->dummy.nbytes) {
+		lutval[lutidx / 2] |= LUT_DEF(lutidx, LUT_DUMMY,
+					      LUT_PAD(op->data.buswidth),
+					      op->dummy.nbytes * 8 /
+						/* need distinguish ddr mode */
+					      op->dummy.buswidth / (op->dummy.dtr ? 2 : 1));
+		lutidx++;
+	}
+
+	/* Read/Write data bytes */
+	if (op->data.nbytes) {
+		lutval[lutidx / 2] |= LUT_DEF(lutidx,
+					      op->data.dir == SPI_MEM_DATA_IN ?
+					      (op->data.dtr ? LUT_READ_DDR : LUT_READ_SDR) :
+					      (op->data.dtr ? LUT_WRITE_DDR : LUT_WRITE_SDR),
+					      LUT_PAD(op->data.buswidth),
+					      0);
+		lutidx++;
+	}
+
+	/* Stop condition. */
+	lutval[lutidx / 2] |= LUT_DEF(lutidx, LUT_STOP, 0, 0);
+
+	/* Unlock LUT */
+	writel(XSPI_LUT_KEY_VAL, xspi->iobase + XSPI_LUTKEY);
+	writel(XSPI_LOKCR_UNLOCK, xspi->iobase + XSPI_LCKCR);
+
+	/* Fill LUT */
+	for (i = 0; i < ARRAY_SIZE(lutval); i++)
+		writel(lutval[i], base + XSPI_LUT_REG(i));
+
+	dev_dbg(xspi->dev, "CMD[%02x] lutval[0:%08x 1:%08x 2:%08x 3:%08x 4:%08x], size: 0x%08x\n",
+		op->cmd.opcode, lutval[0], lutval[1], lutval[2], lutval[3], lutval[4],
+		op->data.nbytes);
+
+	/* Lock LUT */
+	writel(XSPI_LUT_KEY_VAL, xspi->iobase + XSPI_LUTKEY);
+	writel(XSPI_LOKCR_LOCK, xspi->iobase + XSPI_LCKCR);
+}
+
+static void nxp_xspi_disable_ddr(struct nxp_xspi *xspi)
+{
+	void __iomem *base = xspi->iobase;
+	u32 reg;
+
+	/* Disable module */
+	reg = readl(base + XSPI_MCR);
+	reg |= XSPI_MCR_MDIS;
+	writel(reg, base + XSPI_MCR);
+
+	reg &= ~XSPI_MCR_DDR_EN;
+	reg &= ~XSPI_MCR_DQS_FA_SEL_MASK;
+	/* Use dummy pad loopback mode to sample data */
+	reg |= FIELD_PREP(XSPI_MCR_DQS_FA_SEL_MASK, 0x01);
+	writel(reg, base + XSPI_MCR);
+	xspi->support_max_rate = 133000000;
+
+	reg = readl(base + XSPI_FLSHCR);
+	reg &= ~XSPI_FLSHCR_TDH_MASK;
+	writel(reg, base + XSPI_FLSHCR);
+
+	/* Select sampling at inverted clock */
+	reg = FIELD_PREP(XSPI_SMPR_DLLFSMPFA_MASK, 0) | XSPI_SMPR_FSPHS;
+	writel(reg, base + XSPI_SMPR);
+
+	/* Enable module */
+	reg = readl(base + XSPI_MCR);
+	reg &= ~XSPI_MCR_MDIS;
+	writel(reg, base + XSPI_MCR);
+}
+
+static void nxp_xspi_enable_ddr(struct nxp_xspi *xspi)
+{
+	void __iomem *base = xspi->iobase;
+	u32 reg;
+
+	/* Disable module */
+	reg = readl(base + XSPI_MCR);
+	reg |= XSPI_MCR_MDIS;
+	writel(reg, base + XSPI_MCR);
+
+	reg |= XSPI_MCR_DDR_EN;
+	reg &= ~XSPI_MCR_DQS_FA_SEL_MASK;
+	/* Use external dqs to sample data */
+	reg |= FIELD_PREP(XSPI_MCR_DQS_FA_SEL_MASK, 0x03);
+	writel(reg, base + XSPI_MCR);
+	xspi->support_max_rate = 200000000;
+
+	reg = readl(base + XSPI_FLSHCR);
+	reg &= ~XSPI_FLSHCR_TDH_MASK;
+	reg |= FIELD_PREP(XSPI_FLSHCR_TDH_MASK, 0x01);
+	writel(reg, base + XSPI_FLSHCR);
+
+	reg = FIELD_PREP(XSPI_SMPR_DLLFSMPFA_MASK, 0x04);
+	writel(reg, base + XSPI_SMPR);
+
+	/* Enable module */
+	reg = readl(base + XSPI_MCR);
+	reg &= ~XSPI_MCR_MDIS;
+	writel(reg, base + XSPI_MCR);
+}
+
+static void nxp_xspi_sw_reset(struct nxp_xspi *xspi)
+{
+	void __iomem *base = xspi->iobase;
+	bool mdis_flag = false;
+	u32 reg;
+	int ret;
+
+	reg = readl(base + XSPI_MCR);
+
+	/*
+	 * Per RM, when reset SWRSTSD and SWRSTHD, XSPI must be
+	 * enabled (MDIS = 0).
+	 * So if MDIS is 1, should clear it before assert SWRSTSD
+	 * and SWRSTHD.
+	 */
+	if (reg & XSPI_MCR_MDIS) {
+		reg &= ~XSPI_MCR_MDIS;
+		writel(reg, base + XSPI_MCR);
+		mdis_flag = true;
+	}
+
+	/* Software reset for AHB domain and Serial flash memory domain */
+	reg |= XSPI_MCR_SWRSTHD | XSPI_MCR_SWRSTSD;
+	/* Software Reset for IPS Target Group Queue 0 */
+	reg |= XSPI_MCR_IPS_TG_RST;
+	writel(reg, base + XSPI_MCR);
+
+	/* IPS_TG_RST will self-clear to 0 once IPS_TG_RST complete */
+	ret = readl_poll_timeout(base + XSPI_MCR, reg, !(reg & XSPI_MCR_IPS_TG_RST),
+			      100, 5000);
+	if (ret == -ETIMEDOUT)
+		dev_warn(xspi->dev, "XSPI_MCR_IPS_TG_RST do not self-clear in 5ms!");
+
+	/*
+	 * Per RM, must wait for at least three system cycles and
+	 * three flash cycles after changing the value of reset field.
+	 * delay 5us for safe.
+	 */
+	fsleep(5);
+
+	/*
+	 * Per RM, before dessert SWRSTSD and SWRSTHD, XSPI must be
+	 * disabled (MIDS = 1).
+	 */
+	reg = readl(base + XSPI_MCR);
+	reg |= XSPI_MCR_MDIS;
+	writel(reg, base + XSPI_MCR);
+
+	/* deassert software reset */
+	reg &= ~(XSPI_MCR_SWRSTHD | XSPI_MCR_SWRSTSD);
+	writel(reg, base + XSPI_MCR);
+
+	/*
+	 * Per RM, must wait for at least three system cycles and
+	 * three flash cycles after changing the value of reset field.
+	 * delay 5us for safe.
+	 */
+	fsleep(5);
+
+	/* Re-enable XSPI if it is enabled at beginning */
+	if (!mdis_flag) {
+		reg &= ~XSPI_MCR_MDIS;
+		writel(reg, base + XSPI_MCR);
+	}
+}
+
+static void nxp_xspi_dll_bypass(struct nxp_xspi *xspi)
+{
+	void __iomem *base = xspi->iobase;
+	int ret;
+	u32 reg;
+
+	nxp_xspi_sw_reset(xspi);
+
+	writel(0, base + XSPI_DLLCRA);
+
+	/* Set SLV EN first */
+	reg = XSPI_DLLCRA_SLV_EN;
+	writel(reg, base + XSPI_DLLCRA);
+
+	reg = XSPI_DLLCRA_FREQEN |
+	      FIELD_PREP(XSPI_DLLCRA_SLV_DLY_COARSE_MASK, 0x0) |
+	      XSPI_DLLCRA_SLV_EN | XSPI_DLLCRA_SLV_DLL_BYPASS;
+	writel(reg, base + XSPI_DLLCRA);
+
+	reg |= XSPI_DLLCRA_SLV_UPD;
+	writel(reg, base + XSPI_DLLCRA);
+
+	ret = readl_poll_timeout(base + XSPI_DLLSR, reg,
+			      reg & XSPI_DLLSR_SLVA_LOCK, 0, POLL_TOUT_US);
+	if (ret)
+		dev_err(xspi->dev,
+			"DLL SLVA unlock, the DLL status is %x, need to check!\n",
+			readl(base + XSPI_DLLSR));
+}
+
+static void nxp_xspi_dll_auto(struct nxp_xspi *xspi, unsigned long rate)
+{
+	void __iomem *base = xspi->iobase;
+	int ret;
+	u32 reg;
+
+	nxp_xspi_sw_reset(xspi);
+
+	writel(0, base + XSPI_DLLCRA);
+
+	/* Set SLV EN first */
+	reg = XSPI_DLLCRA_SLV_EN;
+	writel(reg, base + XSPI_DLLCRA);
+
+	reg = FIELD_PREP(XSPI_DLLCRA_DLL_REFCNTR_MASK, 0x02) |
+	      FIELD_PREP(XSPI_DLLCRA_DLLRES_MASK, 0x08) |
+	      XSPI_DLLCRA_SLAVE_AUTO_UPDT | XSPI_DLLCRA_SLV_EN;
+	if (rate > 133000000)
+		reg |= XSPI_DLLCRA_FREQEN;
+
+	writel(reg, base + XSPI_DLLCRA);
+
+	reg |= XSPI_DLLCRA_SLV_UPD;
+	writel(reg, base + XSPI_DLLCRA);
+
+	reg |= XSPI_DLLCRA_DLLEN;
+	writel(reg, base + XSPI_DLLCRA);
+
+	ret = readl_poll_timeout(base + XSPI_DLLSR, reg,
+			      reg & XSPI_DLLSR_DLLA_LOCK, 0, POLL_TOUT_US);
+	if (ret)
+		dev_err(xspi->dev,
+			"DLL unlock, the DLL status is %x, need to check!\n",
+			readl(base + XSPI_DLLSR));
+
+	ret = readl_poll_timeout(base + XSPI_DLLSR, reg,
+			      reg & XSPI_DLLSR_SLVA_LOCK, 0, POLL_TOUT_US);
+	if (ret)
+		dev_err(xspi->dev,
+			"DLL SLVA unlock, the DLL status is %x, need to check!\n",
+			readl(base + XSPI_DLLSR));
+}
+
+static void nxp_xspi_select_mem(struct nxp_xspi *xspi, struct spi_device *spi,
+				const struct spi_mem_op *op)
+{
+	/* xspi only support one DTR mode: 8D-8D-8D */
+	bool op_is_dtr = op->cmd.dtr && op->addr.dtr && op->dummy.dtr && op->data.dtr;
+	unsigned long root_clk_rate, rate;
+	uint64_t cs0_top_address;
+	uint64_t cs1_top_address;
+	u32 reg;
+	int ret;
+
+	/*
+	 * Return when following condition all meet,
+	 * 1, if previously selected target device is same as current
+	 *    requested target device.
+	 * 2, the DTR or STR mode do not change.
+	 * 3, previous operation max rate equals current one.
+	 *
+	 * For other case, need to re-config.
+	 */
+	if (xspi->selected == spi_get_chipselect(spi, 0) &&
+	    (!!(xspi->flags & XSPI_DTR_PROTO) == op_is_dtr) &&
+	    (xspi->pre_op_rate == op->max_freq))
+		return;
+
+	if (op_is_dtr) {
+		nxp_xspi_enable_ddr(xspi);
+		xspi->flags |= XSPI_DTR_PROTO;
+	} else {
+		nxp_xspi_disable_ddr(xspi);
+		xspi->flags &= ~XSPI_DTR_PROTO;
+	}
+	rate = min_t(unsigned long, xspi->support_max_rate, op->max_freq);
+	/*
+	 * There is two dividers between xspi_clk_root(from SoC CCM) and xspi_sfif.
+	 * xspi_clk_root ---->divider1 ----> ipg_clk_2xsfif
+	 *                              |
+	 *                              |
+	 *                              |---> divider2 ---> ipg_clk_sfif
+	 * divider1 is controlled by SOCCR, SOCCR default value is 0.
+	 * divider2 fix to divide 2.
+	 * when SOCCR = 0:
+	 *        ipg_clk_2xsfif = xspi_clk_root
+	 *        ipg_clk_sfif = ipg_clk_2xsfif / 2 = xspi_clk_root / 2
+	 * ipg_clk_2xsfif is used for DTR mode.
+	 * xspi_sck(output to device) is defined based on xspi_sfif clock.
+	 */
+	root_clk_rate = rate * 2;
+
+	clk_disable_unprepare(xspi->clk);
+
+	ret = clk_set_rate(xspi->clk, root_clk_rate);
+	if (ret)
+		return;
+
+	ret = clk_prepare_enable(xspi->clk);
+	if (ret)
+		return;
+
+	xspi->pre_op_rate = op->max_freq;
+	xspi->selected = spi_get_chipselect(spi, 0);
+
+	if (xspi->selected) {		/* CS1 select */
+		cs0_top_address = xspi->memmap_phy;
+		cs1_top_address = SZ_4G - 1;
+	} else {			/* CS0 select */
+		cs0_top_address = SZ_4G - 1;
+		cs1_top_address = SZ_4G - 1;
+	}
+	writel(cs0_top_address, xspi->iobase + XSPI_SFA1AD);
+	writel(cs1_top_address, xspi->iobase + XSPI_SFA2AD);
+
+	reg = readl(xspi->iobase + XSPI_SFACR);
+	if (op->data.swap16)
+		reg |= XSPI_SFACR_BYTE_SWAP;
+	else
+		reg &= ~XSPI_SFACR_BYTE_SWAP;
+	writel(reg, xspi->iobase + XSPI_SFACR);
+
+	if (!op_is_dtr || rate < 60000000)
+		nxp_xspi_dll_bypass(xspi);
+	else
+		nxp_xspi_dll_auto(xspi, rate);
+}
+
+static int nxp_xspi_ahb_read(struct nxp_xspi *xspi, const struct spi_mem_op *op)
+{
+	u32 start = op->addr.val;
+	u32 len = op->data.nbytes;
+
+	/* If necessary, ioremap before AHB read */
+	if ((!xspi->ahb_addr) || start < xspi->memmap_start ||
+	     start + len > xspi->memmap_start + xspi->memmap_len) {
+		if (xspi->ahb_addr)
+			iounmap(xspi->ahb_addr);
+
+		xspi->memmap_start = start;
+		xspi->memmap_len = len > NXP_XSPI_MIN_IOMAP ?
+				len : NXP_XSPI_MIN_IOMAP;
+
+		xspi->ahb_addr = ioremap(xspi->memmap_phy + xspi->memmap_start,
+					 xspi->memmap_len);
+
+		if (!xspi->ahb_addr) {
+			dev_err(xspi->dev, "failed to alloc memory\n");
+			return -ENOMEM;
+		}
+	}
+
+	/* Read out the data directly from the AHB buffer. */
+	memcpy_fromio(op->data.buf.in,
+			xspi->ahb_addr + start - xspi->memmap_start, len);
+
+	return 0;
+}
+
+static int nxp_xspi_fill_txfifo(struct nxp_xspi *xspi,
+				 const struct spi_mem_op *op)
+{
+	void __iomem *base = xspi->iobase;
+	u8 *buf = (u8 *)op->data.buf.out;
+	u32 reg, left;
+	int i;
+
+	for (i = 0; i < ALIGN(op->data.nbytes, 4); i += 4) {
+		reg = readl(base + XSPI_FR);
+		reg |= XSPI_FR_TBFF;
+		writel(reg, base + XSPI_FR);
+		/* Read again to check whether the tx fifo has rom */
+		reg = readl(base + XSPI_FR);
+		if (!(reg & XSPI_FR_TBFF)) {
+			WARN_ON(1);
+			return -EIO;
+		}
+
+		if (i == ALIGN_DOWN(op->data.nbytes, 4)) {
+			/* Use 0xFF for extra bytes */
+			left = 0xFFFFFFFF;
+			/* The last 1 to 3 bytes */
+			memcpy((u8 *)&left, buf + i, op->data.nbytes - i);
+			writel(left, base + XSPI_TBDR);
+		} else {
+			writel(*(u32 *)(buf + i), base + XSPI_TBDR);
+		}
+	}
+
+	return 0;
+}
+
+static int nxp_xspi_read_rxfifo(struct nxp_xspi *xspi,
+				const struct spi_mem_op *op)
+{
+	u32 watermark, watermark_bytes, reg;
+	void __iomem *base = xspi->iobase;
+	u8 *buf = (u8 *) op->data.buf.in;
+	int i, ret, len;
+
+	/*
+	 * Config the rx watermark half of the 64 memory-mapped RX data buffer RBDRn
+	 * refer to the RBCT config in nxp_xspi_do_op()
+	 */
+	watermark = 32;
+	watermark_bytes = watermark * 4;
+
+	len = op->data.nbytes;
+
+	while (len >= watermark_bytes) {
+		/* Make sure the RX FIFO contains valid data before read */
+		ret = readl_poll_timeout(base + XSPI_FR, reg,
+				      reg & XSPI_FR_RBDF, 0, POLL_TOUT_US);
+		if (ret) {
+			WARN_ON(1);
+			return ret;
+		}
+
+		for (i = 0; i < watermark; i++)
+			*(u32 *)(buf + i * 4) = readl(base + XSPI_RBDR0 + i * 4);
+
+		len = len - watermark_bytes;
+		buf = buf + watermark_bytes;
+		/* Pop up data to RXFIFO for next read. */
+		reg = readl(base + XSPI_FR);
+		reg |= XSPI_FR_RBDF;
+		writel(reg, base + XSPI_FR);
+	}
+
+	/* Wait for the total data transfer finished */
+	ret = readl_poll_timeout(base + XSPI_SR, reg, !(reg & XSPI_SR_BUSY), 0, POLL_TOUT_US);
+	if (ret) {
+		WARN_ON(1);
+		return ret;
+	}
+
+	i = 0;
+	while (len >= 4) {
+		*(u32 *)(buf) = readl(base + XSPI_RBDR0 + i);
+		i += 4;
+		len -= 4;
+		buf += 4;
+	}
+
+	if (len > 0) {
+		reg = readl(base + XSPI_RBDR0 + i);
+		memcpy(buf, (u8 *)&reg, len);
+	}
+
+	/* Invalid RXFIFO first */
+	reg = readl(base + XSPI_MCR);
+	reg |= XSPI_MCR_CLR_RXF;
+	writel(reg, base + XSPI_MCR);
+	/* Wait for the CLR_RXF clear */
+	ret = readl_poll_timeout(base + XSPI_MCR, reg,
+			      !(reg & XSPI_MCR_CLR_RXF), 1, POLL_TOUT_US);
+	WARN_ON(ret);
+
+	return ret;
+}
+
+static int nxp_xspi_do_op(struct nxp_xspi *xspi, const struct spi_mem_op *op)
+{
+	void __iomem *base = xspi->iobase;
+	int watermark, err = 0;
+	u32 reg, len;
+
+	len = op->data.nbytes;
+	if (op->data.nbytes && op->data.dir == SPI_MEM_DATA_OUT) {
+		/* Clear the TX FIFO. */
+		reg = readl(base + XSPI_MCR);
+		reg |= XSPI_MCR_CLR_TXF;
+		writel(reg, base + XSPI_MCR);
+		/* Wait for the CLR_TXF clear */
+		err = readl_poll_timeout(base + XSPI_MCR, reg,
+				      !(reg & XSPI_MCR_CLR_TXF), 1, POLL_TOUT_US);
+		if (err) {
+			WARN_ON(1);
+			return err;
+		}
+
+		/* Cover the no 4bytes alignment data length */
+		watermark = (xspi->devtype_data->txfifo - ALIGN(op->data.nbytes, 4)) / 4 + 1;
+		reg = FIELD_PREP(XSPI_TBCT_WMRK_MASK, watermark);
+		writel(reg, base + XSPI_TBCT);
+		/*
+		 * According to the RM, for TBDR register, a write transaction on the
+		 * flash memory with data size of less than 32 bits leads to the removal
+		 * of one data entry from the TX buffer. The valid bits are used and the
+		 * rest of the bits are discarded.
+		 * But for data size large than 32 bits, according to test, for no 4bytes
+		 * alignment data, the last 1~3 bytes will lost, because TX buffer use
+		 * 4 bytes entries.
+		 * So here adjust the transfer data length to make it 4bytes alignment.
+		 * then will meet the upper watermark setting, trigger the 4bytes entries
+		 * pop out.
+		 * Will use extra 0xff to append, refer to nxp_xspi_fill_txfifo().
+		 */
+		if (len > 4)
+			len = ALIGN(op->data.nbytes, 4);
+
+	} else if (op->data.nbytes && op->data.dir == SPI_MEM_DATA_IN) {
+		/* Invalid RXFIFO first */
+		reg = readl(base + XSPI_MCR);
+		reg |= XSPI_MCR_CLR_RXF;
+		writel(reg, base + XSPI_MCR);
+		/* Wait for the CLR_RXF clear */
+		err = readl_poll_timeout(base + XSPI_MCR, reg,
+				      !(reg & XSPI_MCR_CLR_RXF), 1, POLL_TOUT_US);
+		if (err) {
+			WARN_ON(1);
+			return err;
+		}
+
+		reg = FIELD_PREP(XSPI_RBCT_WMRK_MASK, 31);
+		writel(reg, base + XSPI_RBCT);
+	}
+
+	init_completion(&xspi->c);
+
+	/* Config the data address */
+	writel(op->addr.val + xspi->memmap_phy, base + XSPI_SFP_TG_SFAR);
+
+	/* Config the data size and lut id, trigger the transfer */
+	reg = FIELD_PREP(XSPI_SFP_TG_IPCR_SEQID_MASK, XSPI_SEQID_LUT) |
+	      FIELD_PREP(XSPI_SFP_TG_IPCR_IDATSZ_MASK, len);
+	writel(reg, base + XSPI_SFP_TG_IPCR);
+
+	if (op->data.nbytes && op->data.dir == SPI_MEM_DATA_OUT) {
+		err = nxp_xspi_fill_txfifo(xspi, op);
+		if (err)
+			return err;
+	}
+
+	/* Wait for the interrupt. */
+	if (!wait_for_completion_timeout(&xspi->c, msecs_to_jiffies(1000)))
+		err = -ETIMEDOUT;
+
+	/* Invoke IP data read. */
+	if (!err && op->data.nbytes && op->data.dir == SPI_MEM_DATA_IN)
+		err = nxp_xspi_read_rxfifo(xspi, op);
+
+	return err;
+}
+
+static int nxp_xspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op)
+{
+	struct nxp_xspi *xspi = spi_controller_get_devdata(mem->spi->controller);
+	void __iomem *base = xspi->iobase;
+	u32 reg;
+	int err;
+
+	guard(mutex)(&xspi->lock);
+
+	PM_RUNTIME_ACQUIRE_AUTOSUSPEND(xspi->dev, pm);
+	err = PM_RUNTIME_ACQUIRE_ERR(&pm);
+	if (err)
+		return err;
+
+	/* Wait for controller being ready. */
+	err = readl_poll_timeout(base + XSPI_SR, reg,
+			      !(reg & XSPI_SR_BUSY), 1, POLL_TOUT_US);
+	if (err) {
+		dev_err(xspi->dev, "SR keeps in BUSY!");
+		return err;
+	}
+
+	nxp_xspi_select_mem(xspi, mem->spi, op);
+
+	nxp_xspi_prepare_lut(xspi, op);
+
+	/*
+	 * For read:
+	 *     the address in AHB mapped range will use AHB read.
+	 *     the address out of AHB mapped range will use IP read.
+	 * For write:
+	 *     all use IP write.
+	 */
+	if ((op->data.dir == SPI_MEM_DATA_IN) && !needs_ip_only(xspi)
+		&& ((op->addr.val + op->data.nbytes) <= xspi->memmap_phy_size))
+		err = nxp_xspi_ahb_read(xspi, op);
+	else
+		err = nxp_xspi_do_op(xspi, op);
+
+	nxp_xspi_sw_reset(xspi);
+
+	return err;
+}
+
+static int nxp_xspi_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op)
+{
+	struct nxp_xspi *xspi = spi_controller_get_devdata(mem->spi->controller);
+
+	if (op->data.dir == SPI_MEM_DATA_OUT) {
+		if (op->data.nbytes > xspi->devtype_data->txfifo)
+			op->data.nbytes = xspi->devtype_data->txfifo;
+	} else {
+		/* Limit data bytes to RX FIFO in case of IP read only */
+		if (needs_ip_only(xspi) && (op->data.nbytes > xspi->devtype_data->rxfifo))
+			op->data.nbytes = xspi->devtype_data->rxfifo;
+
+		/* Address in AHB mapped range prefer to use AHB read. */
+		if (!needs_ip_only(xspi) && (op->addr.val < xspi->memmap_phy_size)
+			&& ((op->addr.val + op->data.nbytes) > xspi->memmap_phy_size))
+			op->data.nbytes = xspi->memmap_phy_size - op->addr.val;
+	}
+
+	return 0;
+}
+
+static void nxp_xspi_config_ahb_buffer(struct nxp_xspi *xspi)
+{
+	void __iomem *base = xspi->iobase;
+	u32 ahb_data_trans_size;
+	u32 reg;
+
+	writel(0xA, base + XSPI_BUF0CR);
+	writel(0x2, base + XSPI_BUF1CR);
+	writel(0xD, base + XSPI_BUF2CR);
+
+	/* Configure buffer3 for All Master Access */
+	reg = FIELD_PREP(XSPI_BUF3CR_MSTRID_MASK, 0x06) |
+	      XSPI_BUF3CR_ALLMST;
+
+	ahb_data_trans_size = xspi->devtype_data->ahb_buf_size / 8;
+	reg |= FIELD_PREP(XSPI_BUF3CR_ADATSZ_MASK, ahb_data_trans_size);
+	writel(reg, base + XSPI_BUF3CR);
+
+	/* Only the buffer3 is used */
+	writel(0, base + XSPI_BUF0IND);
+	writel(0, base + XSPI_BUF1IND);
+	writel(0, base + XSPI_BUF2IND);
+
+	/* AHB only use ID=15 for read */
+	reg = FIELD_PREP(XSPI_BFGENCR_SEQID_MASK, XSPI_SEQID_LUT);
+	reg |= XSPI_BFGENCR_WR_FLUSH_EN;
+	/* No limit for align */
+	reg |= FIELD_PREP(XSPI_BFGENCR_ALIGN_MASK, 0);
+	writel(reg, base + XSPI_BFGENCR);
+}
+
+static int nxp_xspi_default_setup(struct nxp_xspi *xspi)
+{
+	void __iomem *base = xspi->iobase;
+	u32 reg;
+
+	/* Bypass SFP check, clear MGC_GVLD, MGC_GVLDMDAD, MGC_GVLDFRAD */
+	writel(0, base + XSPI_MGC);
+
+	/* Enable the EENV0 SFP check */
+	reg = readl(base + XSPI_TG0MDAD);
+	reg |= XSPI_TG0MDAD_VLD;
+	writel(reg, base + XSPI_TG0MDAD);
+
+	/* Give read/write access right to EENV0 */
+	reg = readl(base + XSPI_FRAD0_WORD2);
+	reg &= ~XSPI_FRAD0_WORD2_MD0ACP_MASK;
+	reg |= FIELD_PREP(XSPI_FRAD0_WORD2_MD0ACP_MASK, 0x03);
+	writel(reg, base + XSPI_FRAD0_WORD2);
+
+	/* Enable the FRAD check for EENV0 */
+	reg = readl(base + XSPI_FRAD0_WORD3);
+	reg |= XSPI_FRAD0_WORD3_VLD;
+	writel(reg, base + XSPI_FRAD0_WORD3);
+
+	/*
+	 * Config the timeout to max value, this timeout will affect the
+	 * TBDR and RBDRn access right after IP cmd triggered.
+	 */
+	writel(0xFFFFFFFF, base + XSPI_MTO);
+
+	/* Disable module */
+	reg = readl(base + XSPI_MCR);
+	reg |= XSPI_MCR_MDIS;
+	writel(reg, base + XSPI_MCR);
+
+	nxp_xspi_sw_reset(xspi);
+
+	reg = readl(base + XSPI_MCR);
+	reg &= ~(XSPI_MCR_CKN_FA_EN | XSPI_MCR_DQS_FA_SEL_MASK |
+		 XSPI_MCR_DOZE | XSPI_MCR_VAR_LAT_EN |
+		 XSPI_MCR_DDR_EN | XSPI_MCR_DQS_OUT_EN);
+	reg |= XSPI_MCR_DQS_EN;
+	reg |= XSPI_MCR_ISD3FA | XSPI_MCR_ISD2FA;
+	writel(reg, base + XSPI_MCR);
+
+	reg = readl(base + XSPI_SFACR);
+	reg &= ~(XSPI_SFACR_FORCE_A10 | XSPI_SFACR_WA_4B_EN |
+		 XSPI_SFACR_BYTE_SWAP | XSPI_SFACR_WA |
+		 XSPI_SFACR_CAS_MASK);
+	reg |= XSPI_SFACR_FORCE_A10;
+	writel(reg, base + XSPI_SFACR);
+
+	nxp_xspi_config_ahb_buffer(xspi);
+
+	reg = FIELD_PREP(XSPI_FLSHCR_TCSH_MASK, 0x03) |
+	      FIELD_PREP(XSPI_FLSHCR_TCSS_MASK, 0x03);
+	writel(reg, base + XSPI_FLSHCR);
+
+	/* Enable module */
+	reg = readl(base + XSPI_MCR);
+	reg &= ~XSPI_MCR_MDIS;
+	writel(reg, base + XSPI_MCR);
+
+	xspi->selected = -1;
+
+	/* Enable the interrupt */
+	writel(XSPI_RSER_TFIE, base + XSPI_RSER);
+
+	return 0;
+}
+
+static const char *nxp_xspi_get_name(struct spi_mem *mem)
+{
+	struct nxp_xspi *xspi = spi_controller_get_devdata(mem->spi->controller);
+	struct device *dev = &mem->spi->dev;
+	const char *name;
+
+	/* Set custom name derived from the platform_device of the controller. */
+	if (of_get_available_child_count(xspi->dev->of_node) == 1)
+		return dev_name(xspi->dev);
+
+	name = devm_kasprintf(dev, GFP_KERNEL,
+			      "%s-%d", dev_name(xspi->dev),
+			      spi_get_chipselect(mem->spi, 0));
+
+	if (!name) {
+		dev_err(dev, "failed to get memory for custom flash name\n");
+		return ERR_PTR(-ENOMEM);
+	}
+
+	return name;
+}
+
+static const struct spi_controller_mem_ops nxp_xspi_mem_ops = {
+	.adjust_op_size = nxp_xspi_adjust_op_size,
+	.supports_op = nxp_xspi_supports_op,
+	.exec_op = nxp_xspi_exec_op,
+	.get_name = nxp_xspi_get_name,
+};
+
+static const struct spi_controller_mem_caps nxp_xspi_mem_caps = {
+	.dtr = true,
+	.per_op_freq = true,
+	.swap16 = true,
+};
+
+static void nxp_xspi_cleanup(void *data)
+{
+	struct nxp_xspi *xspi = data;
+	u32 reg;
+
+	pm_runtime_get_sync(xspi->dev);
+
+	/* Disable interrupt */
+	writel(0, xspi->iobase + XSPI_RSER);
+	/* Clear all the internal logic flags */
+	writel(0xFFFFFFFF, xspi->iobase + XSPI_FR);
+	/* Disable the hardware */
+	reg = readl(xspi->iobase + XSPI_MCR);
+	reg |= XSPI_MCR_MDIS;
+	writel(reg, xspi->iobase + XSPI_MCR);
+
+	pm_runtime_put_sync(xspi->dev);
+
+	if (xspi->ahb_addr)
+		iounmap(xspi->ahb_addr);
+}
+
+static int nxp_xspi_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct spi_controller *ctlr;
+	struct nxp_xspi *xspi;
+	struct resource *res;
+	int ret, irq;
+
+	ctlr = devm_spi_alloc_host(dev, sizeof(*xspi));
+	if (!ctlr)
+		return -ENOMEM;
+
+	ctlr->mode_bits = SPI_RX_DUAL | SPI_RX_QUAD | SPI_RX_OCTAL |
+			  SPI_TX_DUAL | SPI_TX_QUAD | SPI_TX_OCTAL;
+
+	xspi = spi_controller_get_devdata(ctlr);
+	xspi->dev = dev;
+	xspi->devtype_data = device_get_match_data(dev);
+	if (!xspi->devtype_data)
+		return -ENODEV;
+
+	platform_set_drvdata(pdev, xspi);
+
+	/* Find the resources - configuration register address space */
+	xspi->iobase = devm_platform_ioremap_resource_byname(pdev, "base");
+	if (IS_ERR(xspi->iobase))
+		return PTR_ERR(xspi->iobase);
+
+	/* Find the resources - controller memory mapped space */
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mmap");
+	if (!res)
+		return -ENODEV;
+
+	/* Assign memory mapped starting address and mapped size. */
+	xspi->memmap_phy = res->start;
+	xspi->memmap_phy_size = resource_size(res);
+
+	/* Find the clocks */
+	xspi->clk = devm_clk_get(dev, "per");
+	if (IS_ERR(xspi->clk))
+		return PTR_ERR(xspi->clk);
+
+	/* Find the irq */
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return dev_err_probe(dev, irq,  "Failed to get irq source");
+
+	pm_runtime_set_autosuspend_delay(dev, XSPI_RPM_TIMEOUT_MS);
+	pm_runtime_use_autosuspend(dev);
+	ret = devm_pm_runtime_enable(dev);
+	if (ret)
+		return ret;
+
+	PM_RUNTIME_ACQUIRE_AUTOSUSPEND(dev, pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to enable clock");
+
+	/* Clear potential interrupt by write xspi errstat */
+	writel(0xFFFFFFFF, xspi->iobase + XSPI_ERRSTAT);
+	writel(0xFFFFFFFF, xspi->iobase + XSPI_FR);
+
+	nxp_xspi_default_setup(xspi);
+
+	ret = devm_request_irq(dev, irq,
+			nxp_xspi_irq_handler, 0, pdev->name, xspi);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to request irq");
+
+	ret = devm_mutex_init(dev, &xspi->lock);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(dev, nxp_xspi_cleanup, xspi);
+	if (ret)
+		return ret;
+
+	ctlr->bus_num = -1;
+	ctlr->num_chipselect = NXP_XSPI_MAX_CHIPSELECT;
+	ctlr->mem_ops = &nxp_xspi_mem_ops;
+	ctlr->mem_caps = &nxp_xspi_mem_caps;
+	ctlr->dev.of_node = dev->of_node;
+
+	return devm_spi_register_controller(dev, ctlr);
+}
+
+static int nxp_xspi_runtime_suspend(struct device *dev)
+{
+	struct nxp_xspi *xspi = dev_get_drvdata(dev);
+	u32 reg;
+
+	reg = readl(xspi->iobase + XSPI_MCR);
+	reg |= XSPI_MCR_MDIS;
+	writel(reg, xspi->iobase + XSPI_MCR);
+
+	clk_disable_unprepare(xspi->clk);
+
+	return 0;
+}
+
+static int nxp_xspi_runtime_resume(struct device *dev)
+{
+	struct nxp_xspi *xspi = dev_get_drvdata(dev);
+	u32 reg;
+	int ret;
+
+	ret = clk_prepare_enable(xspi->clk);
+	if (ret)
+		return ret;
+
+	reg = readl(xspi->iobase + XSPI_MCR);
+	reg &= ~XSPI_MCR_MDIS;
+	writel(reg, xspi->iobase + XSPI_MCR);
+
+	return 0;
+}
+
+static int nxp_xspi_suspend(struct device *dev)
+{
+	int ret;
+
+	ret = pinctrl_pm_select_sleep_state(dev);
+	if (ret) {
+		dev_err(dev, "select flexspi sleep pinctrl failed!\n");
+		return ret;
+	}
+
+	return pm_runtime_force_suspend(dev);
+}
+
+static int nxp_xspi_resume(struct device *dev)
+{
+	struct nxp_xspi *xspi = dev_get_drvdata(dev);
+	int ret;
+
+	ret = pm_runtime_force_resume(dev);
+	if (ret)
+		return ret;
+
+	nxp_xspi_default_setup(xspi);
+
+	ret = pinctrl_pm_select_default_state(dev);
+	if (ret)
+		dev_err(dev, "select flexspi default pinctrl failed!\n");
+
+	return ret;
+}
+
+
+static const struct dev_pm_ops nxp_xspi_pm_ops = {
+	RUNTIME_PM_OPS(nxp_xspi_runtime_suspend, nxp_xspi_runtime_resume, NULL)
+	SYSTEM_SLEEP_PM_OPS(nxp_xspi_suspend, nxp_xspi_resume)
+};
+
+static const struct of_device_id nxp_xspi_dt_ids[] = {
+	{ .compatible = "nxp,imx94-xspi", .data = (void *)&imx94_data, },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, nxp_xspi_dt_ids);
+
+static struct platform_driver nxp_xspi_driver = {
+	.driver = {
+		.name	= "nxp-xspi",
+		.of_match_table = nxp_xspi_dt_ids,
+		.pm =   pm_ptr(&nxp_xspi_pm_ops),
+	},
+	.probe          = nxp_xspi_probe,
+};
+module_platform_driver(nxp_xspi_driver);
+
+MODULE_DESCRIPTION("NXP xSPI Controller Driver");
+MODULE_AUTHOR("NXP Semiconductor");
+MODULE_AUTHOR("Haibo Chen <haibo.chen@nxp.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/spi/spi-qpic-snand.c b/drivers/spi/spi-qpic-snand.c
index 7681a91d67d5..0334ba738bef 100644
--- a/drivers/spi/spi-qpic-snand.c
+++ b/drivers/spi/spi-qpic-snand.c
@@ -850,8 +850,6 @@ static int qcom_spi_read_page_ecc(struct qcom_nand_controller *snandc,
 	snandc->regs->ecc_bch_cfg = cpu_to_le32(ecc_bch_cfg);
 	snandc->regs->exec = cpu_to_le32(1);
 
-	qcom_spi_set_read_loc(snandc, 0, 0, 0, ecc_cfg->cw_data, 1);
-
 	qcom_clear_bam_transaction(snandc);
 
 	qcom_write_reg_dma(snandc, &snandc->regs->addr0, NAND_ADDR0, 2, 0);
@@ -941,8 +939,6 @@ static int qcom_spi_read_page_oob(struct qcom_nand_controller *snandc,
 	snandc->regs->ecc_bch_cfg = cpu_to_le32(ecc_bch_cfg);
 	snandc->regs->exec = cpu_to_le32(1);
 
-	qcom_spi_set_read_loc(snandc, 0, 0, 0, ecc_cfg->cw_data, 1);
-
 	qcom_write_reg_dma(snandc, &snandc->regs->addr0, NAND_ADDR0, 2, 0);
 	qcom_write_reg_dma(snandc, &snandc->regs->cfg0, NAND_DEV0_CFG0, 3, 0);
 	qcom_write_reg_dma(snandc, &snandc->regs->erased_cw_detect_cfg_clr,
diff --git a/drivers/spi/spi-rzv2h-rspi.c b/drivers/spi/spi-rzv2h-rspi.c
index 1db7e4e5d64e..aae916882915 100644
--- a/drivers/spi/spi-rzv2h-rspi.c
+++ b/drivers/spi/spi-rzv2h-rspi.c
@@ -9,6 +9,7 @@
 #include <linux/bitops.h>
 #include <linux/bits.h>
 #include <linux/clk.h>
+#include <linux/dmaengine.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/limits.h>
@@ -21,6 +22,8 @@
 #include <linux/spi/spi.h>
 #include <linux/wait.h>
 
+#include "internals.h"
+
 /* Registers */
 #define RSPI_SPDR		0x00
 #define RSPI_SPCR		0x08
@@ -37,6 +40,7 @@
 /* Register SPCR */
 #define RSPI_SPCR_BPEN		BIT(31)
 #define RSPI_SPCR_MSTR		BIT(30)
+#define RSPI_SPCR_SPTIE		BIT(20)
 #define RSPI_SPCR_SPRIE		BIT(17)
 #define RSPI_SPCR_SCKASE	BIT(12)
 #define RSPI_SPCR_SPE		BIT(0)
@@ -93,31 +97,29 @@ struct rzv2h_rspi_info {
 };
 
 struct rzv2h_rspi_priv {
-	struct reset_control_bulk_data resets[RSPI_RESET_NUM];
 	struct spi_controller *controller;
 	const struct rzv2h_rspi_info *info;
+	struct platform_device *pdev;
 	void __iomem *base;
 	struct clk *tclk;
 	struct clk *pclk;
 	wait_queue_head_t wait;
 	unsigned int bytes_per_word;
+	int irq_rx;
 	u32 last_speed_hz;
 	u32 freq;
 	u16 status;
 	u8 spr;
 	u8 brdv;
 	bool use_pclk;
+	bool dma_callbacked;
 };
 
 #define RZV2H_RSPI_TX(func, type)					\
 static inline void rzv2h_rspi_tx_##type(struct rzv2h_rspi_priv *rspi,	\
 					const void *txbuf,		\
 					unsigned int index) {		\
-	type buf = 0;							\
-									\
-	if (txbuf)							\
-		buf = ((type *)txbuf)[index];				\
-									\
+	type buf = ((type *)txbuf)[index];				\
 	func(buf, rspi->base + RSPI_SPDR);				\
 }
 
@@ -126,9 +128,7 @@ static inline void rzv2h_rspi_rx_##type(struct rzv2h_rspi_priv *rspi,	\
 					void *rxbuf,			\
 					unsigned int index) {		\
 	type buf = func(rspi->base + RSPI_SPDR);			\
-									\
-	if (rxbuf)							\
-		((type *)rxbuf)[index] = buf;				\
+	((type *)rxbuf)[index] = buf;					\
 }
 
 RZV2H_RSPI_TX(writel, u32)
@@ -224,16 +224,27 @@ static int rzv2h_rspi_receive(struct rzv2h_rspi_priv *rspi, void *rxbuf,
 	return 0;
 }
 
-static int rzv2h_rspi_transfer_one(struct spi_controller *controller,
-				  struct spi_device *spi,
-				  struct spi_transfer *transfer)
+static bool rzv2h_rspi_can_dma(struct spi_controller *ctlr, struct spi_device *spi,
+			       struct spi_transfer *xfer)
 {
-	struct rzv2h_rspi_priv *rspi = spi_controller_get_devdata(controller);
-	unsigned int words_to_transfer, i;
-	int ret = 0;
+	struct rzv2h_rspi_priv *rspi = spi_controller_get_devdata(ctlr);
 
-	transfer->effective_speed_hz = rspi->freq;
-	words_to_transfer = transfer->len / rspi->bytes_per_word;
+	if (ctlr->fallback)
+		return false;
+
+	if (!ctlr->dma_tx || !ctlr->dma_rx)
+		return false;
+
+	return xfer->len > rspi->info->fifo_size;
+}
+
+static int rzv2h_rspi_transfer_pio(struct rzv2h_rspi_priv *rspi,
+				   struct spi_device *spi,
+				   struct spi_transfer *transfer,
+				   unsigned int words_to_transfer)
+{
+	unsigned int i;
+	int ret = 0;
 
 	for (i = 0; i < words_to_transfer; i++) {
 		rzv2h_rspi_clear_all_irqs(rspi);
@@ -245,12 +256,151 @@ static int rzv2h_rspi_transfer_one(struct spi_controller *controller,
 			break;
 	}
 
-	rzv2h_rspi_clear_all_irqs(rspi);
+	return ret;
+}
+
+static void rzv2h_rspi_dma_complete(void *arg)
+{
+	struct rzv2h_rspi_priv *rspi = arg;
+
+	rspi->dma_callbacked = 1;
+	wake_up_interruptible(&rspi->wait);
+}
+
+static struct dma_async_tx_descriptor *
+rzv2h_rspi_setup_dma_channel(struct rzv2h_rspi_priv *rspi,
+			     struct dma_chan *chan, struct sg_table *sg,
+			     enum dma_slave_buswidth width,
+			     enum dma_transfer_direction direction)
+{
+	struct dma_slave_config config = {
+		.dst_addr = rspi->pdev->resource->start + RSPI_SPDR,
+		.src_addr = rspi->pdev->resource->start + RSPI_SPDR,
+		.dst_addr_width = width,
+		.src_addr_width = width,
+		.direction = direction,
+	};
+	struct dma_async_tx_descriptor *desc;
+	int ret;
 
+	ret = dmaengine_slave_config(chan, &config);
 	if (ret)
-		transfer->error = SPI_TRANS_FAIL_IO;
+		return ERR_PTR(ret);
+
+	desc = dmaengine_prep_slave_sg(chan, sg->sgl, sg->nents, direction,
+				       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+	if (!desc)
+		return ERR_PTR(-EAGAIN);
+
+	if (direction == DMA_DEV_TO_MEM) {
+		desc->callback = rzv2h_rspi_dma_complete;
+		desc->callback_param = rspi;
+	}
+
+	return desc;
+}
+
+static enum dma_slave_buswidth
+rzv2h_rspi_dma_width(struct rzv2h_rspi_priv *rspi)
+{
+	switch (rspi->bytes_per_word) {
+	case 4:
+		return DMA_SLAVE_BUSWIDTH_4_BYTES;
+	case 2:
+		return DMA_SLAVE_BUSWIDTH_2_BYTES;
+	case 1:
+		return DMA_SLAVE_BUSWIDTH_1_BYTE;
+	default:
+		return DMA_SLAVE_BUSWIDTH_UNDEFINED;
+	}
+}
+
+static int rzv2h_rspi_transfer_dma(struct rzv2h_rspi_priv *rspi,
+				   struct spi_device *spi,
+				   struct spi_transfer *transfer,
+				   unsigned int words_to_transfer)
+{
+	struct dma_async_tx_descriptor *tx_desc = NULL, *rx_desc = NULL;
+	enum dma_slave_buswidth width;
+	dma_cookie_t cookie;
+	int ret;
+
+	width = rzv2h_rspi_dma_width(rspi);
+	if (width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
+		return -EINVAL;
+
+	rx_desc = rzv2h_rspi_setup_dma_channel(rspi, rspi->controller->dma_rx,
+					       &transfer->rx_sg, width,
+					       DMA_DEV_TO_MEM);
+	if (IS_ERR(rx_desc))
+		return PTR_ERR(rx_desc);
+
+	tx_desc = rzv2h_rspi_setup_dma_channel(rspi, rspi->controller->dma_tx,
+					       &transfer->tx_sg, width,
+					       DMA_MEM_TO_DEV);
+	if (IS_ERR(tx_desc))
+		return PTR_ERR(tx_desc);
+
+	cookie = dmaengine_submit(rx_desc);
+	if (dma_submit_error(cookie))
+		return cookie;
+
+	cookie = dmaengine_submit(tx_desc);
+	if (dma_submit_error(cookie)) {
+		dmaengine_terminate_sync(rspi->controller->dma_rx);
+		return cookie;
+	}
 
-	spi_finalize_current_transfer(controller);
+	/*
+	 * DMA transfer does not need IRQs to be enabled.
+	 * For PIO, we only use RX IRQ, so disable that.
+	 */
+	disable_irq(rspi->irq_rx);
+
+	rspi->dma_callbacked = 0;
+
+	dma_async_issue_pending(rspi->controller->dma_rx);
+	dma_async_issue_pending(rspi->controller->dma_tx);
+	rzv2h_rspi_clear_all_irqs(rspi);
+
+	ret = wait_event_interruptible_timeout(rspi->wait, rspi->dma_callbacked, HZ);
+	if (ret) {
+		dmaengine_synchronize(rspi->controller->dma_tx);
+		dmaengine_synchronize(rspi->controller->dma_rx);
+		ret = 0;
+	} else {
+		dmaengine_terminate_sync(rspi->controller->dma_tx);
+		dmaengine_terminate_sync(rspi->controller->dma_rx);
+		ret = -ETIMEDOUT;
+	}
+
+	enable_irq(rspi->irq_rx);
+
+	return ret;
+}
+
+static int rzv2h_rspi_transfer_one(struct spi_controller *controller,
+				   struct spi_device *spi,
+				   struct spi_transfer *transfer)
+{
+	struct rzv2h_rspi_priv *rspi = spi_controller_get_devdata(controller);
+	bool is_dma = spi_xfer_is_dma_mapped(controller, spi, transfer);
+	unsigned int words_to_transfer;
+	int ret;
+
+	transfer->effective_speed_hz = rspi->freq;
+	words_to_transfer = transfer->len / rspi->bytes_per_word;
+
+	if (is_dma)
+		ret = rzv2h_rspi_transfer_dma(rspi, spi, transfer, words_to_transfer);
+	else
+		ret = rzv2h_rspi_transfer_pio(rspi, spi, transfer, words_to_transfer);
+
+	rzv2h_rspi_clear_all_irqs(rspi);
+
+	if (is_dma && ret == -EAGAIN)
+		/* Retry with PIO */
+		transfer->error = SPI_TRANS_FAIL_NO_START;
 
 	return ret;
 }
@@ -485,6 +635,9 @@ static int rzv2h_rspi_prepare_message(struct spi_controller *ctlr,
 	/* SPI receive buffer full interrupt enable */
 	conf32 |= RSPI_SPCR_SPRIE;
 
+	/* SPI transmit buffer empty interrupt enable */
+	conf32 |= RSPI_SPCR_SPTIE;
+
 	/* Bypass synchronization circuit */
 	conf32 |= FIELD_PREP(RSPI_SPCR_BPEN, rspi->use_pclk);
 
@@ -512,7 +665,7 @@ static int rzv2h_rspi_prepare_message(struct spi_controller *ctlr,
 		writeb(0, rspi->base + RSPI_SSLP);
 
 	/* Setup FIFO thresholds */
-	conf16 = FIELD_PREP(RSPI_SPDCR2_TTRG, rspi->info->fifo_size - 1);
+	conf16 = FIELD_PREP(RSPI_SPDCR2_TTRG, 0);
 	conf16 |= FIELD_PREP(RSPI_SPDCR2_RTRG, 0);
 	writew(conf16, rspi->base + RSPI_SPDCR2);
 
@@ -538,9 +691,10 @@ static int rzv2h_rspi_probe(struct platform_device *pdev)
 	struct spi_controller *controller;
 	struct device *dev = &pdev->dev;
 	struct rzv2h_rspi_priv *rspi;
+	struct reset_control *reset;
 	struct clk_bulk_data *clks;
-	int irq_rx, ret, i;
 	long tclk_rate;
+	int ret, i;
 
 	controller = devm_spi_alloc_host(dev, sizeof(*rspi));
 	if (!controller)
@@ -550,6 +704,7 @@ static int rzv2h_rspi_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, rspi);
 
 	rspi->controller = controller;
+	rspi->pdev = pdev;
 
 	rspi->info = device_get_match_data(dev);
 
@@ -573,83 +728,84 @@ static int rzv2h_rspi_probe(struct platform_device *pdev)
 	if (!rspi->tclk)
 		return dev_err_probe(dev, -EINVAL, "Failed to get tclk\n");
 
-	rspi->resets[0].id = "presetn";
-	rspi->resets[1].id = "tresetn";
-	ret = devm_reset_control_bulk_get_optional_exclusive(dev, RSPI_RESET_NUM,
-							     rspi->resets);
-	if (ret)
-		return dev_err_probe(dev, ret, "cannot get resets\n");
+	reset = devm_reset_control_get_optional_exclusive_deasserted(&pdev->dev,
+								     "presetn");
+	if (IS_ERR(reset))
+		return dev_err_probe(&pdev->dev, PTR_ERR(reset),
+				     "cannot get presetn reset\n");
 
-	irq_rx = platform_get_irq_byname(pdev, "rx");
-	if (irq_rx < 0)
-		return dev_err_probe(dev, irq_rx, "cannot get IRQ 'rx'\n");
+	reset = devm_reset_control_get_optional_exclusive_deasserted(&pdev->dev,
+								     "tresetn");
+	if (IS_ERR(reset))
+		return dev_err_probe(&pdev->dev, PTR_ERR(reset),
+				     "cannot get tresetn reset\n");
 
-	ret = reset_control_bulk_deassert(RSPI_RESET_NUM, rspi->resets);
-	if (ret)
-		return dev_err_probe(dev, ret, "failed to deassert resets\n");
+	rspi->irq_rx = platform_get_irq_byname(pdev, "rx");
+	if (rspi->irq_rx < 0)
+		return dev_err_probe(dev, rspi->irq_rx, "cannot get IRQ 'rx'\n");
 
 	init_waitqueue_head(&rspi->wait);
 
-	ret = devm_request_irq(dev, irq_rx, rzv2h_rx_irq_handler, 0,
+	ret = devm_request_irq(dev, rspi->irq_rx, rzv2h_rx_irq_handler, 0,
 			       dev_name(dev), rspi);
 	if (ret) {
 		dev_err(dev, "cannot request `rx` IRQ\n");
-		goto quit_resets;
+		return ret;
 	}
 
 	controller->mode_bits = SPI_CPHA | SPI_CPOL | SPI_CS_HIGH |
 				SPI_LSB_FIRST | SPI_LOOP;
+	controller->flags = SPI_CONTROLLER_MUST_RX | SPI_CONTROLLER_MUST_TX;
 	controller->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);
 	controller->prepare_message = rzv2h_rspi_prepare_message;
 	controller->unprepare_message = rzv2h_rspi_unprepare_message;
 	controller->num_chipselect = 4;
 	controller->transfer_one = rzv2h_rspi_transfer_one;
+	controller->can_dma = rzv2h_rspi_can_dma;
 
 	tclk_rate = clk_round_rate(rspi->tclk, 0);
-	if (tclk_rate < 0) {
-		ret = tclk_rate;
-		goto quit_resets;
-	}
+	if (tclk_rate < 0)
+		return tclk_rate;
 
 	controller->min_speed_hz = rzv2h_rspi_calc_bitrate(tclk_rate,
 							   RSPI_SPBR_SPR_MAX,
 							   RSPI_SPCMD_BRDV_MAX);
 
 	tclk_rate = clk_round_rate(rspi->tclk, ULONG_MAX);
-	if (tclk_rate < 0) {
-		ret = tclk_rate;
-		goto quit_resets;
-	}
+	if (tclk_rate < 0)
+		return tclk_rate;
 
 	controller->max_speed_hz = rzv2h_rspi_calc_bitrate(tclk_rate,
 							   RSPI_SPBR_SPR_MIN,
 							   RSPI_SPCMD_BRDV_MIN);
 
-	device_set_node(&controller->dev, dev_fwnode(dev));
+	controller->dma_tx = devm_dma_request_chan(dev, "tx");
+	if (IS_ERR(controller->dma_tx)) {
+		ret = dev_warn_probe(dev, PTR_ERR(controller->dma_tx),
+				     "failed to request TX DMA channel\n");
+		if (ret == -EPROBE_DEFER)
+			return ret;
+		controller->dma_tx = NULL;
+	}
 
-	ret = spi_register_controller(controller);
-	if (ret) {
-		dev_err(dev, "register controller failed\n");
-		goto quit_resets;
+	controller->dma_rx = devm_dma_request_chan(dev, "rx");
+	if (IS_ERR(controller->dma_rx)) {
+		ret = dev_warn_probe(dev, PTR_ERR(controller->dma_rx),
+				     "failed to request RX DMA channel\n");
+		if (ret == -EPROBE_DEFER)
+			return ret;
+		controller->dma_rx = NULL;
 	}
 
-	return 0;
+	device_set_node(&controller->dev, dev_fwnode(dev));
 
-quit_resets:
-	reset_control_bulk_assert(RSPI_RESET_NUM, rspi->resets);
+	ret = devm_spi_register_controller(dev, controller);
+	if (ret)
+		dev_err(dev, "register controller failed\n");
 
 	return ret;
 }
 
-static void rzv2h_rspi_remove(struct platform_device *pdev)
-{
-	struct rzv2h_rspi_priv *rspi = platform_get_drvdata(pdev);
-
-	spi_unregister_controller(rspi->controller);
-
-	reset_control_bulk_assert(RSPI_RESET_NUM, rspi->resets);
-}
-
 static const struct rzv2h_rspi_info rzv2h_info = {
 	.find_tclk_rate = rzv2h_rspi_find_rate_fixed,
 	.tclk_name = "tclk",
@@ -674,7 +830,6 @@ MODULE_DEVICE_TABLE(of, rzv2h_rspi_match);
 
 static struct platform_driver rzv2h_rspi_drv = {
 	.probe = rzv2h_rspi_probe,
-	.remove = rzv2h_rspi_remove,
 	.driver = {
 		.name = "rzv2h_rspi",
 		.of_match_table = rzv2h_rspi_match,
diff --git a/drivers/spi/spi-stm32-ospi.c b/drivers/spi/spi-stm32-ospi.c
index f36fd36da269..f396439e2281 100644
--- a/drivers/spi/spi-stm32-ospi.c
+++ b/drivers/spi/spi-stm32-ospi.c
@@ -34,8 +34,6 @@
 #define CR_ABORT		BIT(1)
 #define CR_DMAEN		BIT(2)
 #define CR_FTHRES_SHIFT		8
-#define CR_TEIE			BIT(16)
-#define CR_TCIE			BIT(17)
 #define CR_SMIE			BIT(19)
 #define CR_APMS			BIT(22)
 #define CR_CSSEL		BIT(24)
@@ -106,7 +104,7 @@
 #define STM32_ABT_TIMEOUT_US		100000
 #define STM32_COMP_TIMEOUT_MS		5000
 #define STM32_BUSY_TIMEOUT_US		100000
-
+#define STM32_WAIT_CMD_TIMEOUT_US	5000
 
 #define STM32_AUTOSUSPEND_DELAY -1
 
@@ -116,7 +114,6 @@ struct stm32_ospi {
 	struct clk *clk;
 	struct reset_control *rstc;
 
-	struct completion data_completion;
 	struct completion match_completion;
 
 	struct dma_chan *dma_chtx;
@@ -142,14 +139,32 @@ struct stm32_ospi {
 	struct mutex lock;
 };
 
-static void stm32_ospi_read_fifo(u8 *val, void __iomem *addr)
+static void stm32_ospi_read_fifo(void *val, void __iomem *addr, u8 len)
 {
-	*val = readb_relaxed(addr);
+	switch (len) {
+	case sizeof(u32):
+		*((u32 *)val) = readl_relaxed(addr);
+		break;
+	case sizeof(u16):
+		*((u16 *)val) = readw_relaxed(addr);
+		break;
+	case sizeof(u8):
+		*((u8 *)val) = readb_relaxed(addr);
+	}
 }
 
-static void stm32_ospi_write_fifo(u8 *val, void __iomem *addr)
+static void stm32_ospi_write_fifo(void *val, void __iomem *addr, u8 len)
 {
-	writeb_relaxed(*val, addr);
+	switch (len) {
+	case sizeof(u32):
+		writel_relaxed(*((u32 *)val), addr);
+		break;
+	case sizeof(u16):
+		writew_relaxed(*((u16 *)val), addr);
+		break;
+	case sizeof(u8):
+		writeb_relaxed(*((u8 *)val), addr);
+	}
 }
 
 static int stm32_ospi_abort(struct stm32_ospi *ospi)
@@ -172,19 +187,20 @@ static int stm32_ospi_abort(struct stm32_ospi *ospi)
 	return timeout;
 }
 
-static int stm32_ospi_poll(struct stm32_ospi *ospi, u8 *buf, u32 len, bool read)
+static int stm32_ospi_poll(struct stm32_ospi *ospi, void *buf, u32 len, bool read)
 {
 	void __iomem *regs_base = ospi->regs_base;
-	void (*fifo)(u8 *val, void __iomem *addr);
+	void (*fifo)(void *val, void __iomem *addr, u8 len);
 	u32 sr;
 	int ret;
+	u8 step;
 
 	if (read)
 		fifo = stm32_ospi_read_fifo;
 	else
 		fifo = stm32_ospi_write_fifo;
 
-	while (len--) {
+	while (len) {
 		ret = readl_relaxed_poll_timeout_atomic(regs_base + OSPI_SR,
 							sr, sr & SR_FTF, 1,
 							STM32_FIFO_TIMEOUT_US);
@@ -193,7 +209,17 @@ static int stm32_ospi_poll(struct stm32_ospi *ospi, u8 *buf, u32 len, bool read)
 				len, sr);
 			return ret;
 		}
-		fifo(buf++, regs_base + OSPI_DR);
+
+		if (len >= sizeof(u32))
+			step = sizeof(u32);
+		else if (len >= sizeof(u16))
+			step = sizeof(u16);
+		else
+			step = sizeof(u8);
+
+		fifo(buf, regs_base + OSPI_DR, step);
+		len -= step;
+		buf += step;
 	}
 
 	return 0;
@@ -211,22 +237,16 @@ static int stm32_ospi_wait_nobusy(struct stm32_ospi *ospi)
 static int stm32_ospi_wait_cmd(struct stm32_ospi *ospi)
 {
 	void __iomem *regs_base = ospi->regs_base;
-	u32 cr, sr;
+	u32 sr;
 	int err = 0;
 
-	if ((readl_relaxed(regs_base + OSPI_SR) & SR_TCF) ||
-	    ospi->fmode == CR_FMODE_APM)
+	if (ospi->fmode == CR_FMODE_APM)
 		goto out;
 
-	reinit_completion(&ospi->data_completion);
-	cr = readl_relaxed(regs_base + OSPI_CR);
-	writel_relaxed(cr | CR_TCIE | CR_TEIE, regs_base + OSPI_CR);
-
-	if (!wait_for_completion_timeout(&ospi->data_completion,
-				msecs_to_jiffies(STM32_COMP_TIMEOUT_MS)))
-		err = -ETIMEDOUT;
+	err = readl_relaxed_poll_timeout_atomic(ospi->regs_base + OSPI_SR, sr,
+						(sr & (SR_TEF | SR_TCF)), 1,
+						STM32_WAIT_CMD_TIMEOUT_US);
 
-	sr = readl_relaxed(regs_base + OSPI_SR);
 	if (sr & SR_TCF)
 		/* avoid false timeout */
 		err = 0;
@@ -259,29 +279,29 @@ static irqreturn_t stm32_ospi_irq(int irq, void *dev_id)
 	cr = readl_relaxed(regs_base + OSPI_CR);
 	sr = readl_relaxed(regs_base + OSPI_SR);
 
-	if (cr & CR_SMIE && sr & SR_SMF) {
+	if (sr & SR_SMF) {
 		/* disable irq */
 		cr &= ~CR_SMIE;
 		writel_relaxed(cr, regs_base + OSPI_CR);
 		complete(&ospi->match_completion);
-
-		return IRQ_HANDLED;
-	}
-
-	if (sr & (SR_TEF | SR_TCF)) {
-		/* disable irq */
-		cr &= ~CR_TCIE & ~CR_TEIE;
-		writel_relaxed(cr, regs_base + OSPI_CR);
-		complete(&ospi->data_completion);
 	}
 
 	return IRQ_HANDLED;
 }
 
-static void stm32_ospi_dma_setup(struct stm32_ospi *ospi,
-			 struct dma_slave_config *dma_cfg)
+static int stm32_ospi_dma_setup(struct stm32_ospi *ospi,
+				struct dma_slave_config *dma_cfg)
 {
+	struct dma_slave_caps caps;
+	int ret = 0;
+
 	if (dma_cfg && ospi->dma_chrx) {
+		ret = dma_get_slave_caps(ospi->dma_chrx, &caps);
+		if (ret)
+			return ret;
+
+		dma_cfg->src_maxburst = caps.max_burst / dma_cfg->src_addr_width;
+
 		if (dmaengine_slave_config(ospi->dma_chrx, dma_cfg)) {
 			dev_err(ospi->dev, "dma rx config failed\n");
 			dma_release_channel(ospi->dma_chrx);
@@ -290,6 +310,12 @@ static void stm32_ospi_dma_setup(struct stm32_ospi *ospi,
 	}
 
 	if (dma_cfg && ospi->dma_chtx) {
+		ret = dma_get_slave_caps(ospi->dma_chtx, &caps);
+		if (ret)
+			return ret;
+
+		dma_cfg->dst_maxburst = caps.max_burst / dma_cfg->dst_addr_width;
+
 		if (dmaengine_slave_config(ospi->dma_chtx, dma_cfg)) {
 			dev_err(ospi->dev, "dma tx config failed\n");
 			dma_release_channel(ospi->dma_chtx);
@@ -298,6 +324,8 @@ static void stm32_ospi_dma_setup(struct stm32_ospi *ospi,
 	}
 
 	init_completion(&ospi->dma_completion);
+
+	return ret;
 }
 
 static int stm32_ospi_tx_mm(struct stm32_ospi *ospi,
@@ -391,7 +419,7 @@ static int stm32_ospi_xfer(struct stm32_ospi *ospi, const struct spi_mem_op *op)
 	if (op->data.dir == SPI_MEM_DATA_IN)
 		buf = op->data.buf.in;
 	else
-		buf = (u8 *)op->data.buf.out;
+		buf = (void *)op->data.buf.out;
 
 	return stm32_ospi_poll(ospi, buf, op->data.nbytes,
 			       op->data.dir == SPI_MEM_DATA_IN);
@@ -838,7 +866,6 @@ static int stm32_ospi_get_resources(struct platform_device *pdev)
 		dev_info(dev, "No memory-map region found\n");
 	}
 
-	init_completion(&ospi->data_completion);
 	init_completion(&ospi->match_completion);
 
 	return 0;
@@ -899,9 +926,9 @@ static int stm32_ospi_probe(struct platform_device *pdev)
 	dma_cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 	dma_cfg.src_addr = ospi->regs_phys_base + OSPI_DR;
 	dma_cfg.dst_addr = ospi->regs_phys_base + OSPI_DR;
-	dma_cfg.src_maxburst = 4;
-	dma_cfg.dst_maxburst = 4;
-	stm32_ospi_dma_setup(ospi, &dma_cfg);
+	ret = stm32_ospi_dma_setup(ospi, &dma_cfg);
+	if (ret)
+		return ret;
 
 	mutex_init(&ospi->lock);
 
diff --git a/drivers/spi/spi-stm32-qspi.c b/drivers/spi/spi-stm32-qspi.c
index f2d19f1c5ab1..d1df66875809 100644
--- a/drivers/spi/spi-stm32-qspi.c
+++ b/drivers/spi/spi-stm32-qspi.c
@@ -31,8 +31,6 @@
 #define CR_DFM			BIT(6)
 #define CR_FSEL			BIT(7)
 #define CR_FTHRES_SHIFT		8
-#define CR_TEIE			BIT(16)
-#define CR_TCIE			BIT(17)
 #define CR_FTIE			BIT(18)
 #define CR_SMIE			BIT(19)
 #define CR_TOIE			BIT(20)
@@ -86,11 +84,12 @@
 #define STM32_QSPI_MAX_MMAP_SZ	SZ_256M
 #define STM32_QSPI_MAX_NORCHIP	2
 
-#define STM32_FIFO_TIMEOUT_US 30000
-#define STM32_BUSY_TIMEOUT_US 100000
-#define STM32_ABT_TIMEOUT_US 100000
-#define STM32_COMP_TIMEOUT_MS 1000
-#define STM32_AUTOSUSPEND_DELAY -1
+#define STM32_FIFO_TIMEOUT_US		30000
+#define STM32_BUSY_TIMEOUT_US		100000
+#define STM32_ABT_TIMEOUT_US		100000
+#define STM32_WAIT_CMD_TIMEOUT_US	5000
+#define STM32_COMP_TIMEOUT_MS		1000
+#define STM32_AUTOSUSPEND_DELAY		-1
 
 struct stm32_qspi_flash {
 	u32 cs;
@@ -107,7 +106,6 @@ struct stm32_qspi {
 	struct clk *clk;
 	u32 clk_rate;
 	struct stm32_qspi_flash flash[STM32_QSPI_MAX_NORCHIP];
-	struct completion data_completion;
 	struct completion match_completion;
 	u32 fmode;
 
@@ -134,53 +132,63 @@ static irqreturn_t stm32_qspi_irq(int irq, void *dev_id)
 	cr = readl_relaxed(qspi->io_base + QSPI_CR);
 	sr = readl_relaxed(qspi->io_base + QSPI_SR);
 
-	if (cr & CR_SMIE && sr & SR_SMF) {
+	if (sr & SR_SMF) {
 		/* disable irq */
 		cr &= ~CR_SMIE;
 		writel_relaxed(cr, qspi->io_base + QSPI_CR);
 		complete(&qspi->match_completion);
-
-		return IRQ_HANDLED;
-	}
-
-	if (sr & (SR_TEF | SR_TCF)) {
-		/* disable irq */
-		cr &= ~CR_TCIE & ~CR_TEIE;
-		writel_relaxed(cr, qspi->io_base + QSPI_CR);
-		complete(&qspi->data_completion);
 	}
 
 	return IRQ_HANDLED;
 }
 
-static void stm32_qspi_read_fifo(u8 *val, void __iomem *addr)
+static void stm32_qspi_read_fifo(void *val, void __iomem *addr, u8 len)
 {
-	*val = readb_relaxed(addr);
+	switch (len) {
+	case sizeof(u32):
+		*((u32 *)val) = readl_relaxed(addr);
+		break;
+	case sizeof(u16):
+		*((u16 *)val) = readw_relaxed(addr);
+		break;
+	case sizeof(u8):
+		*((u8 *)val) = readb_relaxed(addr);
+	}
 }
 
-static void stm32_qspi_write_fifo(u8 *val, void __iomem *addr)
+static void stm32_qspi_write_fifo(void *val, void __iomem *addr, u8 len)
 {
-	writeb_relaxed(*val, addr);
+	switch (len) {
+	case sizeof(u32):
+		writel_relaxed(*((u32 *)val), addr);
+		break;
+	case sizeof(u16):
+		writew_relaxed(*((u16 *)val), addr);
+		break;
+	case sizeof(u8):
+		writeb_relaxed(*((u8 *)val), addr);
+	}
 }
 
 static int stm32_qspi_tx_poll(struct stm32_qspi *qspi,
 			      const struct spi_mem_op *op)
 {
-	void (*tx_fifo)(u8 *val, void __iomem *addr);
+	void (*fifo)(void *val, void __iomem *addr, u8 len);
 	u32 len = op->data.nbytes, sr;
-	u8 *buf;
+	void *buf;
 	int ret;
+	u8 step;
 
 	if (op->data.dir == SPI_MEM_DATA_IN) {
-		tx_fifo = stm32_qspi_read_fifo;
+		fifo = stm32_qspi_read_fifo;
 		buf = op->data.buf.in;
 
 	} else {
-		tx_fifo = stm32_qspi_write_fifo;
-		buf = (u8 *)op->data.buf.out;
+		fifo = stm32_qspi_write_fifo;
+		buf = (void *)op->data.buf.out;
 	}
 
-	while (len--) {
+	while (len) {
 		ret = readl_relaxed_poll_timeout_atomic(qspi->io_base + QSPI_SR,
 							sr, (sr & SR_FTF), 1,
 							STM32_FIFO_TIMEOUT_US);
@@ -189,7 +197,17 @@ static int stm32_qspi_tx_poll(struct stm32_qspi *qspi,
 				len, sr);
 			return ret;
 		}
-		tx_fifo(buf++, qspi->io_base + QSPI_DR);
+
+		if (len >= sizeof(u32))
+			step = sizeof(u32);
+		else if (len >= sizeof(u16))
+			step = sizeof(u16);
+		else
+			step = sizeof(u8);
+
+		fifo(buf, qspi->io_base + QSPI_DR, step);
+		len -= step;
+		buf += step;
 	}
 
 	return 0;
@@ -301,25 +319,18 @@ static int stm32_qspi_wait_nobusy(struct stm32_qspi *qspi)
 
 static int stm32_qspi_wait_cmd(struct stm32_qspi *qspi)
 {
-	u32 cr, sr;
+	u32 sr;
 	int err = 0;
 
-	if ((readl_relaxed(qspi->io_base + QSPI_SR) & SR_TCF) ||
-	    qspi->fmode == CCR_FMODE_APM)
+	if (qspi->fmode == CCR_FMODE_APM)
 		goto out;
 
-	reinit_completion(&qspi->data_completion);
-	cr = readl_relaxed(qspi->io_base + QSPI_CR);
-	writel_relaxed(cr | CR_TCIE | CR_TEIE, qspi->io_base + QSPI_CR);
+	err = readl_relaxed_poll_timeout_atomic(qspi->io_base + QSPI_SR, sr,
+						(sr & (SR_TEF | SR_TCF)), 1,
+						STM32_WAIT_CMD_TIMEOUT_US);
 
-	if (!wait_for_completion_timeout(&qspi->data_completion,
-				msecs_to_jiffies(STM32_COMP_TIMEOUT_MS))) {
-		err = -ETIMEDOUT;
-	} else {
-		sr = readl_relaxed(qspi->io_base + QSPI_SR);
-		if (sr & SR_TEF)
-			err = -EIO;
-	}
+	if (sr & SR_TEF)
+		err = -EIO;
 
 out:
 	/* clear flags */
@@ -689,6 +700,7 @@ static int stm32_qspi_dma_setup(struct stm32_qspi *qspi)
 {
 	struct dma_slave_config dma_cfg;
 	struct device *dev = qspi->dev;
+	struct dma_slave_caps caps;
 	int ret = 0;
 
 	memset(&dma_cfg, 0, sizeof(dma_cfg));
@@ -697,8 +709,6 @@ static int stm32_qspi_dma_setup(struct stm32_qspi *qspi)
 	dma_cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
 	dma_cfg.src_addr = qspi->phys_base + QSPI_DR;
 	dma_cfg.dst_addr = qspi->phys_base + QSPI_DR;
-	dma_cfg.src_maxburst = 4;
-	dma_cfg.dst_maxburst = 4;
 
 	qspi->dma_chrx = dma_request_chan(dev, "rx");
 	if (IS_ERR(qspi->dma_chrx)) {
@@ -707,6 +717,11 @@ static int stm32_qspi_dma_setup(struct stm32_qspi *qspi)
 		if (ret == -EPROBE_DEFER)
 			goto out;
 	} else {
+		ret = dma_get_slave_caps(qspi->dma_chrx, &caps);
+		if (ret)
+			return ret;
+
+		dma_cfg.src_maxburst = caps.max_burst / dma_cfg.src_addr_width;
 		if (dmaengine_slave_config(qspi->dma_chrx, &dma_cfg)) {
 			dev_err(dev, "dma rx config failed\n");
 			dma_release_channel(qspi->dma_chrx);
@@ -719,6 +734,11 @@ static int stm32_qspi_dma_setup(struct stm32_qspi *qspi)
 		ret = PTR_ERR(qspi->dma_chtx);
 		qspi->dma_chtx = NULL;
 	} else {
+		ret = dma_get_slave_caps(qspi->dma_chtx, &caps);
+		if (ret)
+			return ret;
+
+		dma_cfg.dst_maxburst = caps.max_burst / dma_cfg.dst_addr_width;
 		if (dmaengine_slave_config(qspi->dma_chtx, &dma_cfg)) {
 			dev_err(dev, "dma tx config failed\n");
 			dma_release_channel(qspi->dma_chtx);
@@ -797,7 +817,6 @@ static int stm32_qspi_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	init_completion(&qspi->data_completion);
 	init_completion(&qspi->match_completion);
 
 	qspi->clk = devm_clk_get(dev, NULL);
diff --git a/drivers/spi/spi-virtio.c b/drivers/spi/spi-virtio.c
index 2acb929b2c69..6aad9f1fd016 100644
--- a/drivers/spi/spi-virtio.c
+++ b/drivers/spi/spi-virtio.c
@@ -150,7 +150,6 @@ static int virtio_spi_transfer_one(struct spi_controller *ctrl,
 				   struct spi_transfer *xfer)
 {
 	struct virtio_spi_priv *priv = spi_controller_get_devdata(ctrl);
-	struct virtio_spi_req *spi_req __free(kfree) = NULL;
 	struct spi_transfer_head *th;
 	struct scatterlist sg_out_head, sg_out_payload;
 	struct scatterlist sg_in_result, sg_in_payload;
@@ -159,7 +158,8 @@ static int virtio_spi_transfer_one(struct spi_controller *ctrl,
 	unsigned int incnt = 0;
 	int ret;
 
-	spi_req = kzalloc(sizeof(*spi_req), GFP_KERNEL);
+	struct virtio_spi_req *spi_req __free(kfree) = kzalloc(sizeof(*spi_req),
+							       GFP_KERNEL);
 	if (!spi_req)
 		return -ENOMEM;
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c
index 8fdeeda88a6d..e89b24fa5e05 100644
--- a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c
+++ b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c
@@ -6,6 +6,7 @@
  ******************************************************************************/
 
 #include <drv_types.h>
+#include <linux/hex.h>
 #include <linux/of.h>
 #include <linux/unaligned.h>
 
diff --git a/drivers/target/iscsi/iscsi_target_auth.c b/drivers/target/iscsi/iscsi_target_auth.c
index c8a248bd11be..2c4d583fe3e6 100644
--- a/drivers/target/iscsi/iscsi_target_auth.c
+++ b/drivers/target/iscsi/iscsi_target_auth.c
@@ -12,6 +12,7 @@
 #include <linux/kernel.h>
 #include <linux/string.h>
 #include <linux/err.h>
+#include <linux/hex.h>
 #include <linux/random.h>
 #include <linux/scatterlist.h>
 #include <target/iscsi/iscsi_target_core.h>
diff --git a/drivers/target/target_core_configfs.c b/drivers/target/target_core_configfs.c
index f7868b41c5e6..eb5bc3f1673b 100644
--- a/drivers/target/target_core_configfs.c
+++ b/drivers/target/target_core_configfs.c
@@ -1741,6 +1741,54 @@ static ssize_t target_wwn_vpd_protocol_identifier_show(struct config_item *item,
 	return len;
 }
 
+static ssize_t target_wwn_pd_text_id_info_show(struct config_item *item,
+		char *page)
+{
+	return sysfs_emit(page, "%s\n", &to_t10_wwn(item)->pd_text_id_info[0]);
+}
+
+static ssize_t target_wwn_pd_text_id_info_store(struct config_item *item,
+		const char *page, size_t count)
+{
+	struct t10_wwn *t10_wwn = to_t10_wwn(item);
+	struct se_device *dev = t10_wwn->t10_dev;
+
+	/* +2 to allow for a trailing (stripped) '\n' and null-terminator */
+	unsigned char buf[PD_TEXT_ID_INFO_LEN + 2];
+	char *stripped;
+
+	/*
+	 * Check to see if any active exports exist.  If they do exist, fail
+	 * here as changing this information on the fly (underneath the
+	 * initiator side OS dependent multipath code) could cause negative
+	 * effects.
+	 */
+	if (dev->export_count) {
+		pr_err("Unable to set the peripheral device text id info while active %d exports exist\n",
+			dev->export_count);
+		return -EINVAL;
+	}
+
+	if (strscpy(buf, page, sizeof(buf)) < 0)
+		return -EOVERFLOW;
+
+	/* Strip any newline added from userspace. */
+	stripped = strstrip(buf);
+	if (strlen(stripped) >= PD_TEXT_ID_INFO_LEN) {
+		pr_err("Emulated peripheral device text id info exceeds PD_TEXT_ID_INFO_LEN: " __stringify(PD_TEXT_ID_INFO_LEN "\n"));
+		return -EOVERFLOW;
+	}
+
+	BUILD_BUG_ON(sizeof(dev->t10_wwn.pd_text_id_info) != PD_TEXT_ID_INFO_LEN);
+	strscpy(dev->t10_wwn.pd_text_id_info, stripped,
+	       sizeof(dev->t10_wwn.pd_text_id_info));
+
+	pr_debug("Target_Core_ConfigFS: Set emulated peripheral dev text id info:"
+		  " %s\n", dev->t10_wwn.pd_text_id_info);
+
+	return count;
+}
+
 /*
  * Generic wrapper for dumping VPD identifiers by association.
  */
@@ -1797,6 +1845,7 @@ CONFIGFS_ATTR_RO(target_wwn_, vpd_protocol_identifier);
 CONFIGFS_ATTR_RO(target_wwn_, vpd_assoc_logical_unit);
 CONFIGFS_ATTR_RO(target_wwn_, vpd_assoc_target_port);
 CONFIGFS_ATTR_RO(target_wwn_, vpd_assoc_scsi_target_device);
+CONFIGFS_ATTR(target_wwn_, pd_text_id_info);
 
 static struct configfs_attribute *target_core_dev_wwn_attrs[] = {
 	&target_wwn_attr_vendor_id,
@@ -1808,6 +1857,7 @@ static struct configfs_attribute *target_core_dev_wwn_attrs[] = {
 	&target_wwn_attr_vpd_assoc_logical_unit,
 	&target_wwn_attr_vpd_assoc_target_port,
 	&target_wwn_attr_vpd_assoc_scsi_target_device,
+	&target_wwn_attr_pd_text_id_info,
 	NULL,
 };
 
diff --git a/drivers/target/target_core_fabric_lib.c b/drivers/target/target_core_fabric_lib.c
index ec7bc6e30228..87c5d26a5089 100644
--- a/drivers/target/target_core_fabric_lib.c
+++ b/drivers/target/target_core_fabric_lib.c
@@ -16,6 +16,7 @@
  * on the formats implemented in this file.
  */
 
+#include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/string.h>
 #include <linux/ctype.h>
diff --git a/drivers/target/target_core_spc.c b/drivers/target/target_core_spc.c
index fe2b888bcb43..f964a531e053 100644
--- a/drivers/target/target_core_spc.c
+++ b/drivers/target/target_core_spc.c
@@ -7,6 +7,7 @@
  * Nicholas A. Bellinger <nab@kernel.org>
  */
 
+#include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/unaligned.h>
@@ -25,6 +26,8 @@
 #include "target_core_ua.h"
 #include "target_core_xcopy.h"
 
+#define PD_TEXT_ID_INFO_HDR_LEN	4
+
 static void spc_fill_alua_data(struct se_lun *lun, unsigned char *buf)
 {
 	struct t10_alua_tg_pt_gp *tg_pt_gp;
@@ -1999,6 +2002,18 @@ static const struct target_opcode_descriptor tcm_opcode_report_supp_opcodes = {
 	.enabled = spc_rsoc_enabled,
 };
 
+static struct target_opcode_descriptor tcm_opcode_report_identifying_information = {
+	.support = SCSI_SUPPORT_FULL,
+	.serv_action_valid = 1,
+	.opcode = MAINTENANCE_IN,
+	.service_action = MI_REPORT_IDENTIFYING_INFORMATION,
+	.cdb_size = 12,
+	.usage_bits = {MAINTENANCE_IN, MI_REPORT_IDENTIFYING_INFORMATION,
+		       0x00, 0x00,
+		       0x00, 0x00, 0xff, 0xff,
+		       0xff, 0xff, 0xff, SCSI_CONTROL_MASK},
+};
+
 static bool tcm_is_set_tpg_enabled(const struct target_opcode_descriptor *descr,
 				   struct se_cmd *cmd)
 {
@@ -2086,6 +2101,7 @@ static const struct target_opcode_descriptor *tcm_supported_opcodes[] = {
 	&tcm_opcode_report_target_pgs,
 	&tcm_opcode_report_supp_opcodes,
 	&tcm_opcode_set_tpg,
+	&tcm_opcode_report_identifying_information,
 };
 
 static int
@@ -2303,6 +2319,72 @@ spc_emulate_report_supp_op_codes(struct se_cmd *cmd)
 	return ret;
 }
 
+static sense_reason_t
+spc_fill_pd_text_id_info(struct se_cmd *cmd, u8 *cdb)
+{
+	struct se_device *dev = cmd->se_dev;
+	unsigned char *buf;
+	unsigned char *rbuf;
+	u32 buf_len;
+	u16 data_len;
+
+	buf_len = get_unaligned_be32(&cdb[6]);
+	if (buf_len < PD_TEXT_ID_INFO_HDR_LEN)
+		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
+
+	data_len = strlen(dev->t10_wwn.pd_text_id_info);
+	if (data_len > 0)
+		/* trailing null */
+		data_len += 1;
+
+	data_len = data_len + PD_TEXT_ID_INFO_HDR_LEN;
+
+	if (data_len < buf_len)
+		buf_len = data_len;
+
+	buf = kzalloc(buf_len, GFP_KERNEL);
+	if (!buf) {
+		pr_err("Unable to allocate response buffer for IDENTITY INFO\n");
+		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
+	}
+
+	scnprintf(&buf[PD_TEXT_ID_INFO_HDR_LEN], buf_len - PD_TEXT_ID_INFO_HDR_LEN, "%s",
+		 dev->t10_wwn.pd_text_id_info);
+
+	put_unaligned_be16(data_len, &buf[2]);
+
+	rbuf = transport_kmap_data_sg(cmd);
+	if (!rbuf) {
+		pr_err("transport_kmap_data_sg() failed in %s\n", __func__);
+		kfree(buf);
+		return TCM_OUT_OF_RESOURCES;
+	}
+
+	memcpy(rbuf, buf, buf_len);
+	transport_kunmap_data_sg(cmd);
+	kfree(buf);
+
+	target_complete_cmd_with_length(cmd, SAM_STAT_GOOD, buf_len);
+	return TCM_NO_SENSE;
+}
+
+static sense_reason_t
+spc_emulate_report_id_info(struct se_cmd *cmd)
+{
+	u8 *cdb = cmd->t_task_cdb;
+	sense_reason_t rc;
+
+	switch ((cdb[10] >> 1)) {
+	case 2:
+		rc = spc_fill_pd_text_id_info(cmd, cdb);
+		break;
+	default:
+		return TCM_UNSUPPORTED_SCSI_OPCODE;
+	}
+
+	return rc;
+}
+
 sense_reason_t
 spc_parse_cdb(struct se_cmd *cmd, unsigned int *size)
 {
@@ -2442,6 +2524,11 @@ spc_parse_cdb(struct se_cmd *cmd, unsigned int *size)
 			    MI_REPORT_SUPPORTED_OPERATION_CODES)
 				cmd->execute_cmd =
 					spc_emulate_report_supp_op_codes;
+			if ((cdb[1] & 0x1f) ==
+			    MI_REPORT_IDENTIFYING_INFORMATION) {
+				cmd->execute_cmd =
+					spc_emulate_report_id_info;
+			}
 			*size = get_unaligned_be32(&cdb[6]);
 		} else {
 			/*
diff --git a/drivers/target/tcm_fc/tfc_conf.c b/drivers/target/tcm_fc/tfc_conf.c
index f686d95d3273..a29b20b5f78e 100644
--- a/drivers/target/tcm_fc/tfc_conf.c
+++ b/drivers/target/tcm_fc/tfc_conf.c
@@ -17,6 +17,7 @@
 #include <linux/moduleparam.h>
 #include <generated/utsrelease.h>
 #include <linux/utsname.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/kthread.h>
diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c
index d65d47cc154e..a015730664c7 100644
--- a/drivers/tee/tee_core.c
+++ b/drivers/tee/tee_core.c
@@ -1398,13 +1398,97 @@ static int tee_client_device_uevent(const struct device *dev,
 	return add_uevent_var(env, "MODALIAS=tee:%pUb", dev_id);
 }
 
+static int tee_client_device_probe(struct device *dev)
+{
+	struct tee_client_device *tcdev = to_tee_client_device(dev);
+	struct tee_client_driver *drv = to_tee_client_driver(dev->driver);
+
+	if (drv->probe)
+		return drv->probe(tcdev);
+	else
+		return 0;
+}
+
+static void tee_client_device_remove(struct device *dev)
+{
+	struct tee_client_device *tcdev = to_tee_client_device(dev);
+	struct tee_client_driver *drv = to_tee_client_driver(dev->driver);
+
+	if (drv->remove)
+		drv->remove(tcdev);
+}
+
+static void tee_client_device_shutdown(struct device *dev)
+{
+	struct tee_client_device *tcdev = to_tee_client_device(dev);
+	struct tee_client_driver *drv = to_tee_client_driver(dev->driver);
+
+	if (dev->driver && drv->shutdown)
+		drv->shutdown(tcdev);
+}
+
 const struct bus_type tee_bus_type = {
 	.name		= "tee",
 	.match		= tee_client_device_match,
 	.uevent		= tee_client_device_uevent,
+	.probe		= tee_client_device_probe,
+	.remove		= tee_client_device_remove,
+	.shutdown	= tee_client_device_shutdown,
 };
 EXPORT_SYMBOL_GPL(tee_bus_type);
 
+static int tee_client_device_probe_legacy(struct tee_client_device *tcdev)
+{
+	struct device *dev = &tcdev->dev;
+	struct device_driver *driver = dev->driver;
+
+	return driver->probe(dev);
+}
+
+static void tee_client_device_remove_legacy(struct tee_client_device *tcdev)
+{
+	struct device *dev = &tcdev->dev;
+	struct device_driver *driver = dev->driver;
+
+	driver->remove(dev);
+}
+
+static void tee_client_device_shutdown_legacy(struct tee_client_device *tcdev)
+{
+	struct device *dev = &tcdev->dev;
+	struct device_driver *driver = dev->driver;
+
+	driver->shutdown(dev);
+}
+
+int __tee_client_driver_register(struct tee_client_driver *tee_driver,
+				 struct module *owner)
+{
+	tee_driver->driver.owner = owner;
+	tee_driver->driver.bus = &tee_bus_type;
+
+	/*
+	 * Drivers that have callbacks set for tee_driver->driver need updating
+	 * to use the callbacks in tee_driver instead. driver_register() warns
+	 * about that, so no need to warn here, too.
+	 */
+	if (!tee_driver->probe && tee_driver->driver.probe)
+		tee_driver->probe = tee_client_device_probe_legacy;
+	if (!tee_driver->remove && tee_driver->driver.remove)
+		tee_driver->remove = tee_client_device_remove_legacy;
+	if (!tee_driver->shutdown && tee_driver->driver.probe)
+		tee_driver->shutdown = tee_client_device_shutdown_legacy;
+
+	return driver_register(&tee_driver->driver);
+}
+EXPORT_SYMBOL_GPL(__tee_client_driver_register);
+
+void tee_client_driver_unregister(struct tee_client_driver *tee_driver)
+{
+	driver_unregister(&tee_driver->driver);
+}
+EXPORT_SYMBOL_GPL(tee_client_driver_unregister);
+
 static int __init tee_init(void)
 {
 	int rc;
diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c
index 0d4dcc66e097..c693d934103a 100644
--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c
+++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c
@@ -503,7 +503,8 @@ static const struct pci_device_id proc_thermal_pci_ids[] = {
 	{ PCI_DEVICE_DATA(INTEL, WCL_THERMAL, PROC_THERMAL_FEATURE_MSI_SUPPORT |
 	  PROC_THERMAL_FEATURE_RAPL | PROC_THERMAL_FEATURE_DLVR |
 	  PROC_THERMAL_FEATURE_DVFS | PROC_THERMAL_FEATURE_WT_HINT |
-	  PROC_THERMAL_FEATURE_POWER_FLOOR | PROC_THERMAL_FEATURE_PTC) },
+	  PROC_THERMAL_FEATURE_POWER_FLOOR | PROC_THERMAL_FEATURE_PTC |
+	  PROC_THERMAL_FEATURE_SOC_POWER_SLIDER) },
 	{ PCI_DEVICE_DATA(INTEL, NVL_H_THERMAL, PROC_THERMAL_FEATURE_RAPL |
 	  PROC_THERMAL_FEATURE_DLVR | PROC_THERMAL_FEATURE_DVFS |
 	  PROC_THERMAL_FEATURE_MSI_SUPPORT | PROC_THERMAL_FEATURE_WT_HINT |
diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c
index 17ca5c082643..89758c9934ec 100644
--- a/drivers/thermal/thermal_core.c
+++ b/drivers/thermal/thermal_core.c
@@ -500,7 +500,7 @@ void thermal_zone_set_trip_hyst(struct thermal_zone_device *tz,
 	WRITE_ONCE(trip->hysteresis, hyst);
 	thermal_notify_tz_trip_change(tz, trip);
 	/*
-	 * If the zone temperature is above or at the trip tmperature, the trip
+	 * If the zone temperature is above or at the trip temperature, the trip
 	 * is in the trips_reached list and its threshold is equal to its low
 	 * temperature.  It needs to stay in that list, but its threshold needs
 	 * to be updated and the list ordering may need to be restored.
@@ -1043,7 +1043,7 @@ static void thermal_cooling_device_init_complete(struct thermal_cooling_device *
  * @np:		a pointer to a device tree node.
  * @type:	the thermal cooling device type.
  * @devdata:	device private data.
- * @ops:		standard thermal cooling devices callbacks.
+ * @ops:	standard thermal cooling devices callbacks.
  *
  * This interface function adds a new thermal cooling device (fan/processor/...)
  * to /sys/class/thermal/ folder as cooling_device[0-*]. It tries to bind itself
diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c
index b3948aad0b95..e2732c575bad 100644
--- a/drivers/thunderbolt/switch.c
+++ b/drivers/thunderbolt/switch.c
@@ -7,6 +7,7 @@
  */
 
 #include <linux/delay.h>
+#include <linux/hex.h>
 #include <linux/idr.h>
 #include <linux/module.h>
 #include <linux/nvmem-provider.h>
diff --git a/drivers/tty/n_hdlc.c b/drivers/tty/n_hdlc.c
index 3c9dcb0928c6..f242d73ee4e0 100644
--- a/drivers/tty/n_hdlc.c
+++ b/drivers/tty/n_hdlc.c
@@ -127,6 +127,8 @@ struct n_hdlc_buf_list {
  * @rx_buf_list: list of received frame buffers
  * @tx_free_buf_list: list unused transmit frame buffers
  * @rx_free_buf_list: list unused received frame buffers
+ * @write_work: work struct for deferred frame transmission
+ * @tty_for_write_work: pointer to tty instance used by the @write_work
  */
 struct n_hdlc {
 	bool			tbusy;
diff --git a/drivers/tty/serdev/core.c b/drivers/tty/serdev/core.c
index b33e708cb245..40eedc15277c 100644
--- a/drivers/tty/serdev/core.c
+++ b/drivers/tty/serdev/core.c
@@ -414,11 +414,21 @@ static void serdev_drv_remove(struct device *dev)
 		sdrv->remove(to_serdev_device(dev));
 }
 
+static void serdev_drv_shutdown(struct device *dev)
+{
+	const struct serdev_device_driver *sdrv =
+		to_serdev_device_driver(dev->driver);
+
+	if (dev->driver && sdrv->shutdown)
+		sdrv->shutdown(to_serdev_device(dev));
+}
+
 static const struct bus_type serdev_bus_type = {
 	.name		= "serial",
 	.match		= serdev_device_match,
 	.probe		= serdev_drv_probe,
 	.remove		= serdev_drv_remove,
+	.shutdown	= serdev_drv_shutdown,
 };
 
 /**
@@ -814,6 +824,14 @@ void serdev_controller_remove(struct serdev_controller *ctrl)
 }
 EXPORT_SYMBOL_GPL(serdev_controller_remove);
 
+static void serdev_legacy_shutdown(struct serdev_device *serdev)
+{
+	struct device *dev = &serdev->dev;
+	struct device_driver *driver = dev->driver;
+
+	driver->shutdown(dev);
+}
+
 /**
  * __serdev_device_driver_register() - Register client driver with serdev core
  * @sdrv:	client driver to be associated with client-device.
@@ -830,6 +848,9 @@ int __serdev_device_driver_register(struct serdev_device_driver *sdrv, struct mo
 	/* force drivers to async probe so I/O is possible in probe */
         sdrv->driver.probe_type = PROBE_PREFER_ASYNCHRONOUS;
 
+	if (!sdrv->shutdown && sdrv->driver.shutdown)
+		sdrv->shutdown = serdev_legacy_shutdown;
+
 	return driver_register(&sdrv->driver);
 }
 EXPORT_SYMBOL_GPL(__serdev_device_driver_register);
diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c
index 27af83f0ff46..db73b2ae17fa 100644
--- a/drivers/tty/serial/8250/8250_dw.c
+++ b/drivers/tty/serial/8250/8250_dw.c
@@ -643,6 +643,10 @@ static int dw8250_probe(struct platform_device *pdev)
 	if (err)
 		return err;
 
+	err = pm_runtime_set_active(dev);
+	if (err)
+		return dev_err_probe(dev, err, "Failed to set the runtime suspend as active\n");
+
 	data->uart_16550_compatible = device_property_read_bool(dev, "snps,uart-16550-compatible");
 
 	data->pdata = device_get_match_data(p->dev);
@@ -685,7 +689,6 @@ static int dw8250_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, data);
 
-	pm_runtime_set_active(dev);
 	pm_runtime_enable(dev);
 
 	return 0;
@@ -741,19 +744,25 @@ static int dw8250_runtime_suspend(struct device *dev)
 
 static int dw8250_runtime_resume(struct device *dev)
 {
+	int ret;
 	struct dw8250_data *data = dev_get_drvdata(dev);
 
-	clk_prepare_enable(data->pclk);
+	ret = clk_prepare_enable(data->pclk);
+	if (ret)
+		return ret;
 
-	clk_prepare_enable(data->clk);
+	ret = clk_prepare_enable(data->clk);
+	if (ret) {
+		clk_disable_unprepare(data->pclk);
+		return ret;
+	}
 
 	return 0;
 }
 
-static const struct dev_pm_ops dw8250_pm_ops = {
-	SYSTEM_SLEEP_PM_OPS(dw8250_suspend, dw8250_resume)
-	RUNTIME_PM_OPS(dw8250_runtime_suspend, dw8250_runtime_resume, NULL)
-};
+static _DEFINE_DEV_PM_OPS(dw8250_pm_ops, dw8250_suspend, dw8250_resume,
+			  dw8250_runtime_suspend, dw8250_runtime_resume,
+			  NULL);
 
 static const struct dw8250_platform_data dw8250_dw_apb = {
 	.usr_reg = DW_UART_USR,
diff --git a/drivers/tty/serial/8250/8250_keba.c b/drivers/tty/serial/8250/8250_keba.c
index c05b89551b12..f94d97e69dc5 100644
--- a/drivers/tty/serial/8250/8250_keba.c
+++ b/drivers/tty/serial/8250/8250_keba.c
@@ -6,10 +6,18 @@
  */
 
 #include <linux/auxiliary_bus.h>
-#include <linux/device.h>
+#include <linux/bits.h>
+#include <linux/container_of.h>
+#include <linux/dev_printk.h>
+#include <linux/device/devres.h>
+#include <linux/err.h>
 #include <linux/io.h>
 #include <linux/misc/keba.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
+#include <linux/serial_core.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
 
 #include "8250.h"
 
@@ -43,6 +51,10 @@ enum kuart_mode {
 #define KUART_CAPABILITY_RS232	BIT(KUART_MODE_RS232)
 #define KUART_CAPABILITY_MASK	GENMASK(3, 0)
 
+/* registers for Indexed Control Register access in enhanced mode */
+#define KUART_EMODE_ICR_OFFSET	UART_SCR
+#define KUART_EMODE_ICR_VALUE	UART_LSR
+
 /* Additional Control Register DTR line configuration */
 #define UART_ACR_DTRLC_MASK		0x18
 #define UART_ACR_DTRLC_COMPAT		0x00
@@ -90,13 +102,13 @@ static void kuart_dtr_line_config(struct uart_8250_port *up, u8 dtrlc)
 	u8 acr;
 
 	/* set index register to 0 to access ACR register */
-	serial_out(up, UART_SCR, UART_ACR);
+	serial_out(up, KUART_EMODE_ICR_OFFSET, UART_ACR);
 
 	/* set value register to 0x10 writing DTR mode (1,0) */
-	acr = serial_in(up, UART_LSR);
+	acr = serial_in(up, KUART_EMODE_ICR_VALUE);
 	acr &= ~UART_ACR_DTRLC_MASK;
 	acr |= dtrlc;
-	serial_out(up, UART_LSR, acr);
+	serial_out(up, KUART_EMODE_ICR_VALUE, acr);
 }
 
 static int kuart_rs485_config(struct uart_port *port, struct ktermios *termios,
@@ -240,10 +252,9 @@ static int kuart_probe(struct auxiliary_device *auxdev,
 	}
 
 	retval = serial8250_register_8250_port(&uart);
-	if (retval < 0) {
-		dev_err(&auxdev->dev, "UART registration failed!\n");
-		return retval;
-	}
+	if (retval < 0)
+		return dev_err_probe(&auxdev->dev, retval,
+				     "UART registration failed!\n");
 	kuart->line = retval;
 
 	return 0;
diff --git a/drivers/tty/serial/8250/8250_loongson.c b/drivers/tty/serial/8250/8250_loongson.c
index 53153a116c01..47df3c4c9d21 100644
--- a/drivers/tty/serial/8250/8250_loongson.c
+++ b/drivers/tty/serial/8250/8250_loongson.c
@@ -128,8 +128,8 @@ static int loongson_uart_probe(struct platform_device *pdev)
 	port->private_data = priv;
 
 	port->membase = devm_platform_get_and_ioremap_resource(pdev, 0, &priv->res);
-	if (!port->membase)
-		return -ENOMEM;
+	if (IS_ERR(port->membase))
+		return PTR_ERR(port->membase);
 
 	port->mapbase = priv->res->start;
 	port->mapsize = resource_size(priv->res);
diff --git a/drivers/tty/serial/8250/8250_men_mcb.c b/drivers/tty/serial/8250/8250_men_mcb.c
index a78ef35c8187..6ba7464bb2dd 100644
--- a/drivers/tty/serial/8250/8250_men_mcb.c
+++ b/drivers/tty/serial/8250/8250_men_mcb.c
@@ -28,8 +28,6 @@
 #define MEN_UART3_MASK	0x04
 #define MEN_UART4_MASK	0x08
 
-#define MEN_Z125_UARTS_AVAILABLE	0x01
-
 #define MEN_Z025_MAX_UARTS		4
 #define MEN_UART_MEM_SIZE		0x10
 #define MEM_UART_REGISTER_SIZE		0x01
@@ -42,12 +40,10 @@
 
 #define MEN_READ_REGISTER(addr)	readb(addr)
 
-#define MAX_PORTS	4
-
 struct serial_8250_men_mcb_data {
 	int num_ports;
-	int line[MAX_PORTS];
-	unsigned int offset[MAX_PORTS];
+	int line[MEN_Z025_MAX_UARTS];
+	unsigned int offset[MEN_Z025_MAX_UARTS];
 };
 
 /*
@@ -126,7 +122,7 @@ static int read_serial_data(struct mcb_device *mdev,
 	if (res < 0)
 		return res;
 
-	for (i = 0; i < MAX_PORTS; i++) {
+	for (i = 0; i < MEN_Z025_MAX_UARTS; i++) {
 		mask = 0x1 << i;
 		switch (uarts_available & mask) {
 		case MEN_UART1_MASK:
@@ -150,7 +146,7 @@ static int read_serial_data(struct mcb_device *mdev,
 		}
 	}
 
-	if (count <= 0 || count > MAX_PORTS) {
+	if (count <= 0 || count > MEN_Z025_MAX_UARTS) {
 		dev_err(&mdev->dev, "unexpected number of ports: %u\n",
 			count);
 		return -ENODEV;
diff --git a/drivers/tty/serial/rsci.c b/drivers/tty/serial/rsci.c
index b3c48dc1e07d..1ef7c6d61707 100644
--- a/drivers/tty/serial/rsci.c
+++ b/drivers/tty/serial/rsci.c
@@ -11,6 +11,8 @@
 #include <linux/serial_core.h>
 #include <linux/serial_sci.h>
 #include <linux/tty_flip.h>
+
+#include "serial_mctrl_gpio.h"
 #include "rsci.h"
 
 MODULE_IMPORT_NS("SH_SCI");
@@ -24,7 +26,6 @@ MODULE_IMPORT_NS("SH_SCI");
 #define CCR3	0x14
 #define CCR4	0x18
 #define FCR	0x24
-#define DCR	0x30
 #define CSR	0x48
 #define FRSR	0x50
 #define FTSR	0x54
@@ -36,12 +37,6 @@ MODULE_IMPORT_NS("SH_SCI");
 #define RDR_FPER		BIT(11) /* FIFO Parity Error */
 #define RDR_RDAT_MSK		GENMASK(8, 0)
 
-/* TDR (Transmit Data Register) */
-#define TDR_MPBT		BIT(9)	/* Multiprocessor Transfer */
-#define TDR_TDAT_9BIT_LSHIFT	0
-#define TDR_TDAT_9BIT_VAL	0x1FF
-#define TDR_TDAT_9BIT_MSK	(TDR_TDAT_9BIT_VAL << TDR_TDAT_9BIT_LSHIFT)
-
 /* CCR0 (Common Control Register 0) */
 #define CCR0_SSE		BIT(24)	/* SSn# Pin Function Enable */
 #define CCR0_TEIE		BIT(21)	/* Transmit End Interrupt Enable */
@@ -66,6 +61,41 @@ MODULE_IMPORT_NS("SH_SCI");
 #define CCR1_CTSPEN		BIT(1)	/* CTS External Pin Enable */
 #define CCR1_CTSE		BIT(0)	/* CTS Enable */
 
+/* CCR2 (Common Control Register 2) */
+#define CCR2_INIT			0xFF000004
+#define CCR2_CKS_TCLK			(0)	/* TCLK clock */
+#define CCR2_CKS_TCLK_DIV4		BIT(20)	/* TCLK/4 clock */
+#define CCR2_CKS_TCLK_DIV16		BIT(21)	/* TCLK16 clock */
+#define CCR2_CKS_TCLK_DIV64		(BIT(21) | BIT(20)) /* TCLK/64 clock */
+#define CCR2_BRME			BIT(16)	/* Bitrate Modulation Enable */
+#define CCR2_ABCSE			BIT(6)	/* Asynchronous Mode Extended Base Clock Select */
+#define CCR2_ABCS			BIT(5)	/* Asynchronous Mode Base Clock Select */
+#define CCR2_BGDM			BIT(4)	/* Baud Rate Generator Double-Speed Mode Select */
+
+/* CCR3 (Common Control Register 3) */
+#define CCR3_INIT			0x1203
+#define CCR3_BLK			BIT(29)	/* Block Transfer Mode */
+#define CCR3_GM				BIT(28)	/* GSM Mode */
+#define CCR3_CKE1			BIT(25)	/* Clock Enable 1 */
+#define CCR3_CKE0			BIT(24)	/* Clock Enable 0 */
+#define CCR3_DEN			BIT(21)	/* Driver Enabled */
+#define CCR3_FM				BIT(20)	/* FIFO Mode Select */
+#define CCR3_MP				BIT(19)	/* Multi-Processor Mode */
+#define CCR3_MOD_ASYNC			0	/* Asynchronous mode (Multi-processor mode) */
+#define CCR3_MOD_IRDA			BIT(16)	/* Smart card interface mode */
+#define CCR3_MOD_CLK_SYNC		BIT(17)	/* Clock synchronous mode */
+#define CCR3_MOD_SPI			(BIT(17) | BIT(16)) /* Simple SPI mode */
+#define CCR3_MOD_I2C			BIT(18)	/* Simple I2C mode */
+#define CCR3_RXDESEL			BIT(15)	/* Asynchronous Start Bit Edge Detection Select */
+#define CCR3_STP			BIT(14)	/* Stop bit Length */
+#define CCR3_SINV			BIT(13)	/* Transmitted/Received Data Invert */
+#define CCR3_LSBF			BIT(12)	/* LSB First select */
+#define CCR3_CHR1			BIT(9)	/* Character Length */
+#define CCR3_CHR0			BIT(8)	/* Character Length */
+#define CCR3_BPEN			BIT(7)	/* Synchronizer Bypass Enable */
+#define CCR3_CPOL			BIT(1)	/* Clock Polarity Select */
+#define CCR3_CPHA			BIT(0)	/* Clock Phase Select */
+
 /* FCR (FIFO Control Register) */
 #define FCR_RFRST		BIT(23)	/* Receive FIFO Data Register Reset */
 #define FCR_TFRST		BIT(15)	/* Transmit FIFO Data Register Reset */
@@ -119,8 +149,6 @@ MODULE_IMPORT_NS("SH_SCI");
 /* FFCLR (FIFO Flag CLear Register) */
 #define FFCLR_DRC		BIT(0)	/* DR Clear */
 
-#define DCR_DEPOL		BIT(0)
-
 static u32 rsci_serial_in(struct uart_port *p, int offset)
 {
 	return readl(p->membase + offset);
@@ -137,10 +165,6 @@ static void rsci_clear_DRxC(struct uart_port *port)
 	rsci_serial_out(port, FFCLR, FFCLR_DRC);
 }
 
-static void rsci_clear_SCxSR(struct uart_port *port, unsigned int mask)
-{
-	rsci_serial_out(port, CFCLR, mask);
-}
 
 static void rsci_start_rx(struct uart_port *port)
 {
@@ -151,21 +175,161 @@ static void rsci_start_rx(struct uart_port *port)
 	rsci_serial_out(port, CCR0, ctrl);
 }
 
+static void rsci_enable_ms(struct uart_port *port)
+{
+	mctrl_gpio_enable_ms(to_sci_port(port)->gpios);
+}
+
+static void rsci_init_pins(struct uart_port *port, unsigned int cflag)
+{
+	struct sci_port *s = to_sci_port(port);
+
+	/* Use port-specific handler if provided */
+	if (s->cfg->ops && s->cfg->ops->init_pins) {
+		s->cfg->ops->init_pins(port, cflag);
+		return;
+	}
+
+	if (!s->has_rtscts)
+		return;
+
+	if (s->autorts)
+		rsci_serial_out(port, CCR1, rsci_serial_in(port, CCR1) |
+				CCR1_CTSE | CCR1_CTSPEN);
+}
+
+static int rsci_scif_set_rtrg(struct uart_port *port, int rx_trig)
+{
+	u32 fcr = rsci_serial_in(port, FCR);
+
+	if (rx_trig >= port->fifosize)
+		rx_trig = port->fifosize - 1;
+	else if (rx_trig < 1)
+		rx_trig = 0;
+
+	fcr &= ~FCR_RTRG4_0;
+	fcr |= field_prep(FCR_RTRG4_0, rx_trig);
+	rsci_serial_out(port, FCR, fcr);
+
+	return rx_trig;
+}
+
 static void rsci_set_termios(struct uart_port *port, struct ktermios *termios,
 			     const struct ktermios *old)
 {
+	unsigned int ccr2_val = CCR2_INIT, ccr3_val = CCR3_INIT;
+	unsigned int ccr0_val = 0, ccr1_val = 0, ccr4_val = 0;
+	unsigned int brr1 = 255, cks1 = 0, srr1 = 15;
 	struct sci_port *s = to_sci_port(port);
+	unsigned int brr = 255, cks = 0;
+	int min_err = INT_MAX, err;
+	unsigned long max_freq = 0;
+	unsigned int baud, i;
 	unsigned long flags;
+	unsigned int ctrl;
+	int best_clk = -1;
+
+	if ((termios->c_cflag & CSIZE) == CS7) {
+		ccr3_val |= CCR3_CHR0;
+	} else {
+		termios->c_cflag &= ~CSIZE;
+		termios->c_cflag |= CS8;
+	}
+
+	if (termios->c_cflag & PARENB)
+		ccr1_val |= CCR1_PE;
+
+	if (termios->c_cflag & PARODD)
+		ccr1_val |= (CCR1_PE | CCR1_PM);
+
+	if (termios->c_cflag & CSTOPB)
+		ccr3_val |= CCR3_STP;
+
+	/* Enable noise filter function */
+	ccr1_val |= CCR1_NFEN;
+
+	/*
+	 * earlyprintk comes here early on with port->uartclk set to zero.
+	 * the clock framework is not up and running at this point so here
+	 * we assume that 115200 is the maximum baud rate. please note that
+	 * the baud rate is not programmed during earlyprintk - it is assumed
+	 * that the previous boot loader has enabled required clocks and
+	 * setup the baud rate generator hardware for us already.
+	 */
+	if (!port->uartclk) {
+		max_freq = 115200;
+	} else {
+		for (i = 0; i < SCI_NUM_CLKS; i++)
+			max_freq = max(max_freq, s->clk_rates[i]);
+
+		max_freq /= min_sr(s);
+	}
+
+	baud = uart_get_baud_rate(port, termios, old, 0, max_freq);
+	if (!baud)
+		goto done;
+
+	/* Divided Functional Clock using standard Bit Rate Register */
+	err = sci_scbrr_calc(s, baud, &brr1, &srr1, &cks1);
+	if (abs(err) < abs(min_err)) {
+		best_clk = SCI_FCK;
+		ccr0_val = 0;
+		min_err = err;
+		brr = brr1;
+		cks = cks1;
+	}
+
+done:
+	if (best_clk >= 0)
+		dev_dbg(port->dev, "Using clk %pC for %u%+d bps\n",
+			s->clks[best_clk], baud, min_err);
 
 	sci_port_enable(s);
 	uart_port_lock_irqsave(port, &flags);
 
-	/* For now, only RX enabling is supported */
-	if (termios->c_cflag & CREAD)
+	uart_update_timeout(port, termios->c_cflag, baud);
+
+	rsci_serial_out(port, CCR0, ccr0_val);
+
+	ccr3_val |= CCR3_FM;
+	rsci_serial_out(port, CCR3, ccr3_val);
+
+	ccr2_val |= (cks << 20) | (brr << 8);
+	rsci_serial_out(port, CCR2, ccr2_val);
+
+	rsci_serial_out(port, CCR1, ccr1_val);
+	rsci_serial_out(port, CCR4, ccr4_val);
+
+	ctrl = rsci_serial_in(port, FCR);
+	ctrl |= (FCR_RFRST | FCR_TFRST);
+	rsci_serial_out(port, FCR, ctrl);
+
+	if (s->rx_trigger > 1)
+		rsci_scif_set_rtrg(port, s->rx_trigger);
+
+	port->status &= ~UPSTAT_AUTOCTS;
+	s->autorts = false;
+
+	if ((port->flags & UPF_HARD_FLOW) && (termios->c_cflag & CRTSCTS)) {
+		port->status |= UPSTAT_AUTOCTS;
+		s->autorts = true;
+	}
+
+	rsci_init_pins(port, termios->c_cflag);
+	rsci_serial_out(port, CFCLR, CFCLR_CLRFLAG);
+	rsci_serial_out(port, FFCLR, FFCLR_DRC);
+
+	ccr0_val |= CCR0_RE;
+	rsci_serial_out(port, CCR0, ccr0_val);
+
+	if ((termios->c_cflag & CREAD) != 0)
 		rsci_start_rx(port);
 
 	uart_port_unlock_irqrestore(port, flags);
 	sci_port_disable(s);
+
+	if (UART_ENABLE_MS(port, termios->c_cflag))
+		rsci_enable_ms(port);
 }
 
 static int rsci_txfill(struct uart_port *port)
@@ -190,13 +354,34 @@ static unsigned int rsci_tx_empty(struct uart_port *port)
 
 static void rsci_set_mctrl(struct uart_port *port, unsigned int mctrl)
 {
-	/* Not supported yet */
+	if (mctrl & TIOCM_LOOP) {
+		/* Standard loopback mode */
+		rsci_serial_out(port, CCR1, rsci_serial_in(port, CCR1) | CCR1_SPLP);
+	}
 }
 
 static unsigned int rsci_get_mctrl(struct uart_port *port)
 {
-	/* Not supported yet */
-	return 0;
+	struct sci_port *s = to_sci_port(port);
+	struct mctrl_gpios *gpios = s->gpios;
+	unsigned int mctrl = 0;
+
+	mctrl_gpio_get(gpios, &mctrl);
+
+	/*
+	 * CTS/RTS is handled in hardware when supported, while nothing
+	 * else is wired up.
+	 */
+	if (!mctrl_gpio_to_gpiod(gpios, UART_GPIO_CTS))
+		mctrl |= TIOCM_CTS;
+
+	if (!mctrl_gpio_to_gpiod(gpios, UART_GPIO_DSR))
+		mctrl |= TIOCM_DSR;
+
+	if (!mctrl_gpio_to_gpiod(gpios, UART_GPIO_DCD))
+		mctrl |= TIOCM_CAR;
+
+	return mctrl;
 }
 
 static void rsci_clear_CFC(struct uart_port *port, unsigned int mask)
@@ -326,7 +511,8 @@ static void rsci_receive_chars(struct uart_port *port)
 				continue;
 			}
 
-			/* Store data and status.
+			/*
+			 * Store data and status.
 			 * Non FIFO mode is not supported
 			 */
 			if (rdat & RDR_FFER) {
@@ -360,6 +546,28 @@ static void rsci_receive_chars(struct uart_port *port)
 	}
 }
 
+static void rsci_break_ctl(struct uart_port *port, int break_state)
+{
+	unsigned short ccr0_val, ccr1_val;
+	unsigned long flags;
+
+	uart_port_lock_irqsave(port, &flags);
+	ccr1_val = rsci_serial_in(port, CCR1);
+	ccr0_val = rsci_serial_in(port, CCR0);
+
+	if (break_state == -1) {
+		ccr1_val = (ccr1_val | CCR1_SPB2IO) & ~CCR1_SPB2DT;
+		ccr0_val &= ~CCR0_TE;
+	} else {
+		ccr1_val = (ccr1_val | CCR1_SPB2DT) & ~CCR1_SPB2IO;
+		ccr0_val |= CCR0_TE;
+	}
+
+	rsci_serial_out(port, CCR1, ccr1_val);
+	rsci_serial_out(port, CCR0, ccr0_val);
+	uart_port_unlock_irqrestore(port, flags);
+}
+
 static void rsci_poll_put_char(struct uart_port *port, unsigned char c)
 {
 	u32 status;
@@ -375,18 +583,27 @@ static void rsci_poll_put_char(struct uart_port *port, unsigned char c)
 	}
 	rsci_serial_out(port, TDR, c);
 done:
-	rsci_clear_SCxSR(port, CFCLR_TDREC);
+	rsci_clear_CFC(port, CFCLR_TDREC);
 }
 
 static void rsci_prepare_console_write(struct uart_port *port, u32 ctrl)
 {
 	struct sci_port *s = to_sci_port(port);
-	u32 ctrl_temp =
-		s->params->param_bits->rxtx_enable | CCR0_TIE |
-		s->hscif_tot;
+	u32 ctrl_temp = s->params->param_bits->rxtx_enable;
+
+	if (s->type == RSCI_PORT_SCIF16)
+		ctrl_temp |= CCR0_TIE | s->hscif_tot;
+
 	rsci_serial_out(port, CCR0, ctrl_temp);
 }
 
+static void rsci_finish_console_write(struct uart_port *port, u32 ctrl)
+{
+	/* First set TE = 0 and then restore the CCR0 value */
+	rsci_serial_out(port, CCR0, ctrl & ~CCR0_TE);
+	rsci_serial_out(port, CCR0, ctrl);
+}
+
 static const char *rsci_type(struct uart_port *port)
 {
 	return "rsci";
@@ -416,7 +633,18 @@ static const struct sci_port_params_bits rsci_port_param_bits = {
 	.poll_sent_bits = CSR_TDRE | CSR_TEND,
 };
 
-static const struct sci_port_params rsci_port_params = {
+static const struct sci_port_params rsci_rzg3e_port_params = {
+	.fifosize = 32,
+	.overrun_reg = CSR,
+	.overrun_mask = CSR_ORER,
+	.sampling_rate_mask = SCI_SR(32),
+	.error_mask = RSCI_DEFAULT_ERROR_MASK,
+	.error_clear = RSCI_ERROR_CLEAR,
+	.param_bits = &rsci_port_param_bits,
+	.common_regs = &rsci_common_regs,
+};
+
+static const struct sci_port_params rsci_rzt2h_port_params = {
 	.fifosize = 16,
 	.overrun_reg = CSR,
 	.overrun_mask = CSR_ORER,
@@ -434,6 +662,8 @@ static const struct uart_ops rsci_uart_ops = {
 	.start_tx	= rsci_start_tx,
 	.stop_tx	= rsci_stop_tx,
 	.stop_rx	= rsci_stop_rx,
+	.enable_ms	= rsci_enable_ms,
+	.break_ctl	= rsci_break_ctl,
 	.startup	= sci_startup,
 	.shutdown	= sci_shutdown,
 	.set_termios	= rsci_set_termios,
@@ -448,31 +678,47 @@ static const struct uart_ops rsci_uart_ops = {
 static const struct sci_port_ops rsci_port_ops = {
 	.read_reg		= rsci_serial_in,
 	.write_reg		= rsci_serial_out,
-	.clear_SCxSR		= rsci_clear_SCxSR,
+	.clear_SCxSR		= rsci_clear_CFC,
 	.transmit_chars		= rsci_transmit_chars,
 	.receive_chars		= rsci_receive_chars,
 	.poll_put_char		= rsci_poll_put_char,
 	.prepare_console_write	= rsci_prepare_console_write,
+	.finish_console_write	= rsci_finish_console_write,
 	.suspend_regs_size	= rsci_suspend_regs_size,
+	.set_rtrg		= rsci_scif_set_rtrg,
 	.shutdown_complete	= rsci_shutdown_complete,
 };
 
-struct sci_of_data of_sci_rsci_data = {
-	.type = SCI_PORT_RSCI,
+struct sci_of_data of_rsci_rzg3e_data = {
+	.type = RSCI_PORT_SCIF32,
+	.ops = &rsci_port_ops,
+	.uart_ops = &rsci_uart_ops,
+	.params = &rsci_rzg3e_port_params,
+};
+
+struct sci_of_data of_rsci_rzt2h_data = {
+	.type = RSCI_PORT_SCIF16,
 	.ops = &rsci_port_ops,
 	.uart_ops = &rsci_uart_ops,
-	.params = &rsci_port_params,
+	.params = &rsci_rzt2h_port_params,
 };
 
 #ifdef CONFIG_SERIAL_SH_SCI_EARLYCON
 
-static int __init rsci_early_console_setup(struct earlycon_device *device,
-					   const char *opt)
+static int __init rsci_rzg3e_early_console_setup(struct earlycon_device *device,
+						 const char *opt)
+{
+	return scix_early_console_setup(device, &of_rsci_rzg3e_data);
+}
+
+static int __init rsci_rzt2h_early_console_setup(struct earlycon_device *device,
+						 const char *opt)
 {
-	return scix_early_console_setup(device, &of_sci_rsci_data);
+	return scix_early_console_setup(device, &of_rsci_rzt2h_data);
 }
 
-OF_EARLYCON_DECLARE(rsci, "renesas,r9a09g077-rsci", rsci_early_console_setup);
+OF_EARLYCON_DECLARE(rsci, "renesas,r9a09g047-rsci", rsci_rzg3e_early_console_setup);
+OF_EARLYCON_DECLARE(rsci, "renesas,r9a09g077-rsci", rsci_rzt2h_early_console_setup);
 
 #endif /* CONFIG_SERIAL_SH_SCI_EARLYCON */
 
diff --git a/drivers/tty/serial/rsci.h b/drivers/tty/serial/rsci.h
index 2af3f28b465a..2aa2ba3973ee 100644
--- a/drivers/tty/serial/rsci.h
+++ b/drivers/tty/serial/rsci.h
@@ -5,6 +5,7 @@
 
 #include "sh-sci-common.h"
 
-extern struct sci_of_data of_sci_rsci_data;
+extern struct sci_of_data of_rsci_rzg3e_data;
+extern struct sci_of_data of_rsci_rzt2h_data;
 
 #endif /* __RSCI_H__ */
diff --git a/drivers/tty/serial/serial_base_bus.c b/drivers/tty/serial/serial_base_bus.c
index 22749ab0428a..8e891984cdc0 100644
--- a/drivers/tty/serial/serial_base_bus.c
+++ b/drivers/tty/serial/serial_base_bus.c
@@ -13,7 +13,7 @@
 #include <linux/device.h>
 #include <linux/idr.h>
 #include <linux/module.h>
-#include <linux/of.h>
+#include <linux/property.h>
 #include <linux/serial_core.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
@@ -60,6 +60,7 @@ void serial_base_driver_unregister(struct device_driver *driver)
 	driver_unregister(driver);
 }
 
+/* On failure the caller must put device @dev with put_device() */
 static int serial_base_device_init(struct uart_port *port,
 				   struct device *dev,
 				   struct device *parent_dev,
@@ -73,7 +74,8 @@ static int serial_base_device_init(struct uart_port *port,
 	dev->parent = parent_dev;
 	dev->bus = &serial_base_bus_type;
 	dev->release = release;
-	device_set_of_node_from_dev(dev, parent_dev);
+
+	device_set_node(dev, fwnode_handle_get(dev_fwnode(parent_dev)));
 
 	if (!serial_base_initialized) {
 		dev_dbg(port->dev, "uart_add_one_port() called before arch_initcall()?\n");
@@ -94,7 +96,7 @@ static void serial_base_ctrl_release(struct device *dev)
 {
 	struct serial_ctrl_device *ctrl_dev = to_serial_base_ctrl_device(dev);
 
-	of_node_put(dev->of_node);
+	fwnode_handle_put(dev_fwnode(dev));
 	kfree(ctrl_dev);
 }
 
@@ -142,7 +144,7 @@ static void serial_base_port_release(struct device *dev)
 {
 	struct serial_port_device *port_dev = to_serial_base_port_device(dev);
 
-	of_node_put(dev->of_node);
+	fwnode_handle_put(dev_fwnode(dev));
 	kfree(port_dev);
 }
 
diff --git a/drivers/tty/serial/sh-sci-common.h b/drivers/tty/serial/sh-sci-common.h
index e3c028df14f1..f363a659c46a 100644
--- a/drivers/tty/serial/sh-sci-common.h
+++ b/drivers/tty/serial/sh-sci-common.h
@@ -7,7 +7,8 @@
 
 /* Private port IDs */
 enum SCI_PORT_TYPE {
-	SCI_PORT_RSCI = BIT(7) | 0,
+	RSCI_PORT_SCIF16 = BIT(7) | 0,
+	RSCI_PORT_SCIF32 = BIT(7) | 1,
 };
 
 enum SCI_CLKS {
@@ -15,6 +16,9 @@ enum SCI_CLKS {
 	SCI_SCK,		/* Optional External Clock */
 	SCI_BRG_INT,		/* Optional BRG Internal Clock Source */
 	SCI_SCIF_CLK,		/* Optional BRG External Clock Source */
+	SCI_FCK_DIV4,		/* Optional Functional Clock frequency-divided by 4 */
+	SCI_FCK_DIV16,		/* Optional Functional Clock frequency-divided by 16 */
+	SCI_FCK_DIV64,		/* Optional Functional Clock frequency-divided by 64 */
 	SCI_NUM_CLKS
 };
 
@@ -89,6 +93,7 @@ struct sci_port_ops {
 	void (*shutdown_complete)(struct uart_port *port);
 
 	void (*prepare_console_write)(struct uart_port *port, u32 ctrl);
+	void (*finish_console_write)(struct uart_port *port, u32 ctrl);
 	void (*console_save)(struct uart_port *port);
 	void (*console_restore)(struct uart_port *port);
 	size_t (*suspend_regs_size)(void);
@@ -165,6 +170,9 @@ void sci_port_enable(struct sci_port *sci_port);
 int sci_startup(struct uart_port *port);
 void sci_shutdown(struct uart_port *port);
 
+int sci_scbrr_calc(struct sci_port *s, unsigned int bps, unsigned int *brr,
+		   unsigned int *srr, unsigned int *cks);
+
 #define min_sr(_port)		ffs((_port)->sampling_rate_mask)
 #define max_sr(_port)		fls((_port)->sampling_rate_mask)
 
diff --git a/drivers/tty/serial/sh-sci.c b/drivers/tty/serial/sh-sci.c
index 53edbf1d8963..a75d2113752b 100644
--- a/drivers/tty/serial/sh-sci.c
+++ b/drivers/tty/serial/sh-sci.c
@@ -1182,6 +1182,11 @@ static int sci_handle_errors(struct uart_port *port)
 	return copied;
 }
 
+static bool sci_is_rsci_type(u8 type)
+{
+	return (type == RSCI_PORT_SCIF16 || type == RSCI_PORT_SCIF32);
+}
+
 static int sci_handle_fifo_overrun(struct uart_port *port)
 {
 	struct tty_port *tport = &port->state->port;
@@ -1190,7 +1195,7 @@ static int sci_handle_fifo_overrun(struct uart_port *port)
 	int copied = 0;
 	u32 status;
 
-	if (s->type != SCI_PORT_RSCI) {
+	if (!sci_is_rsci_type(s->type)) {
 		reg = sci_getreg(port, s->params->overrun_reg);
 		if (!reg->size)
 			return 0;
@@ -1198,7 +1203,7 @@ static int sci_handle_fifo_overrun(struct uart_port *port)
 
 	status = s->ops->read_reg(port, s->params->overrun_reg);
 	if (status & s->params->overrun_mask) {
-		if (s->type == SCI_PORT_RSCI) {
+		if (sci_is_rsci_type(s->type)) {
 			/*
 			 * All of the CFCLR_*C clearing bits match the corresponding
 			 * CSR_*status bits. So, reuse the overrun mask for clearing.
@@ -2015,7 +2020,7 @@ static irqreturn_t sci_tx_end_interrupt(int irq, void *ptr)
 	unsigned long flags;
 	u32 ctrl;
 
-	if (s->type != PORT_SCI && s->type != SCI_PORT_RSCI)
+	if (s->type != PORT_SCI && !sci_is_rsci_type(s->type))
 		return sci_tx_interrupt(irq, ptr);
 
 	uart_port_lock_irqsave(port, &flags);
@@ -2568,9 +2573,8 @@ static int sci_brg_calc(struct sci_port *s, unsigned int bps,
 }
 
 /* calculate sample rate, BRR, and clock select */
-static int sci_scbrr_calc(struct sci_port *s, unsigned int bps,
-		   unsigned int *brr, unsigned int *srr,
-		   unsigned int *cks)
+int sci_scbrr_calc(struct sci_port *s, unsigned int bps, unsigned int *brr,
+		   unsigned int *srr, unsigned int *cks)
 {
 	unsigned long freq = s->clk_rates[SCI_FCK];
 	unsigned int sr, br, prediv, scrate, c;
@@ -2634,6 +2638,7 @@ static int sci_scbrr_calc(struct sci_port *s, unsigned int bps,
 		min_err, *brr, *srr + 1, *cks);
 	return min_err;
 }
+EXPORT_SYMBOL_NS_GPL(sci_scbrr_calc, "SH_SCI");
 
 static void sci_reset(struct uart_port *port)
 {
@@ -3167,15 +3172,21 @@ static int sci_init_clocks(struct sci_port *sci_port, struct device *dev)
 		[SCI_SCK] = "sck",
 		[SCI_BRG_INT] = "brg_int",
 		[SCI_SCIF_CLK] = "scif_clk",
+		[SCI_FCK_DIV4] = "tclk_div4",
+		[SCI_FCK_DIV16] = "tclk_div16",
+		[SCI_FCK_DIV64] = "tclk_div64",
 	};
 	struct clk *clk;
 	unsigned int i;
 
 	if (sci_port->type == PORT_HSCIF) {
 		clk_names[SCI_SCK] = "hsck";
-	} else if (sci_port->type == SCI_PORT_RSCI) {
+	} else if (sci_port->type == RSCI_PORT_SCIF16) {
 		clk_names[SCI_FCK] = "operation";
 		clk_names[SCI_BRG_INT] = "bus";
+	} else if (sci_port->type == RSCI_PORT_SCIF32) {
+		clk_names[SCI_FCK] = "tclk";
+		clk_names[SCI_BRG_INT] = "pclk";
 	}
 
 	for (i = 0; i < SCI_NUM_CLKS; i++) {
@@ -3185,12 +3196,13 @@ static int sci_init_clocks(struct sci_port *sci_port, struct device *dev)
 		if (IS_ERR(clk))
 			return PTR_ERR(clk);
 
-		if (!clk && sci_port->type == SCI_PORT_RSCI &&
-		    (i == SCI_FCK || i == SCI_BRG_INT)) {
-			return dev_err_probe(dev, -ENODEV,
-					     "failed to get %s\n",
-					     name);
-		}
+		if (!clk && sci_port->type == RSCI_PORT_SCIF16 &&
+		    (i == SCI_FCK || i == SCI_BRG_INT))
+			return dev_err_probe(dev, -ENODEV, "failed to get %s\n", name);
+
+		if (!clk && sci_port->type == RSCI_PORT_SCIF32 &&
+		    (i != SCI_SCK && i != SCI_SCIF_CLK))
+			return dev_err_probe(dev, -ENODEV, "failed to get %s\n", name);
 
 		if (!clk && i == SCI_FCK) {
 			/*
@@ -3200,16 +3212,13 @@ static int sci_init_clocks(struct sci_port *sci_port, struct device *dev)
 			 */
 			clk = devm_clk_get(dev, "peripheral_clk");
 			if (IS_ERR(clk))
-				return dev_err_probe(dev, PTR_ERR(clk),
-						     "failed to get %s\n",
-						     name);
+				return dev_err_probe(dev, PTR_ERR(clk), "failed to get %s\n", name);
 		}
 
 		if (!clk)
 			dev_dbg(dev, "failed to get %s\n", name);
 		else
-			dev_dbg(dev, "clk %s is %pC rate %lu\n", name,
-				clk, clk_get_rate(clk));
+			dev_dbg(dev, "clk %s is %pC rate %lu\n", name, clk, clk_get_rate(clk));
 		sci_port->clks[i] = clk;
 	}
 	return 0;
@@ -3295,7 +3304,7 @@ static int sci_init_single(struct platform_device *dev,
 	 * The fourth interrupt on SCI and RSCI port is transmit end interrupt, so
 	 * shuffle the interrupts.
 	 */
-	if (p->type == PORT_SCI || p->type == SCI_PORT_RSCI)
+	if (p->type == PORT_SCI || sci_is_rsci_type(p->type))
 		swap(sci_port->irqs[SCIx_BRI_IRQ], sci_port->irqs[SCIx_TEI_IRQ]);
 
 	/* The SCI generates several interrupts. They can be muxed together or
@@ -3320,6 +3329,7 @@ static int sci_init_single(struct platform_device *dev,
 		sci_port->rx_trigger = 64;
 		break;
 	case PORT_SCIFA:
+	case RSCI_PORT_SCIF32:
 		sci_port->rx_trigger = 32;
 		break;
 	case PORT_SCIF:
@@ -3329,8 +3339,8 @@ static int sci_init_single(struct platform_device *dev,
 		else
 			sci_port->rx_trigger = 8;
 		break;
-	case SCI_PORT_RSCI:
-		sci_port->rx_trigger = 15;
+	case RSCI_PORT_SCIF16:
+		sci_port->rx_trigger = 16;
 		break;
 	default:
 		sci_port->rx_trigger = 1;
@@ -3422,7 +3432,10 @@ static void serial_console_write(struct console *co, const char *s,
 		cpu_relax();
 
 	/* restore the SCSCR */
-	sci_port->ops->write_reg(port, regs->control, ctrl);
+	if (sci_port->ops->finish_console_write)
+		sci_port->ops->finish_console_write(port, ctrl);
+	else
+		sci_port->ops->write_reg(port, regs->control, ctrl);
 
 	if (locked)
 		uart_port_unlock_irqrestore(port, flags);
@@ -3549,16 +3562,14 @@ static struct uart_driver sci_uart_driver = {
 static void sci_remove(struct platform_device *dev)
 {
 	struct sci_port *s = platform_get_drvdata(dev);
-	unsigned int type = s->type;	/* uart_remove_... clears it */
 
 	sci_ports_in_use &= ~BIT(s->port.line);
 	uart_remove_one_port(&sci_uart_driver, &s->port);
 
-	if (s->port.fifosize > 1)
+	if (s->port.fifosize > 1) {
 		device_remove_file(&dev->dev, &dev_attr_rx_fifo_trigger);
-	if (type == PORT_SCIFA || type == PORT_SCIFB || type == PORT_HSCIF ||
-	    type == SCI_PORT_RSCI)
 		device_remove_file(&dev->dev, &dev_attr_rx_fifo_timeout);
+	}
 }
 
 static const struct sci_of_data of_sci_scif_sh2 = {
@@ -3652,9 +3663,13 @@ static const struct of_device_id of_sci_match[] __maybe_unused = {
 		.data = &of_sci_scif_rzv2h,
 	},
 #ifdef CONFIG_SERIAL_RSCI
+	{
+		.compatible = "renesas,r9a09g047-rsci",
+		.data = &of_rsci_rzg3e_data,
+	},
 	{
 		.compatible = "renesas,r9a09g077-rsci",
-		.data = &of_sci_rsci_data,
+		.data = &of_rsci_rzt2h_data,
 	},
 #endif	/* CONFIG_SERIAL_RSCI */
 	/* Family-specific types */
@@ -3716,7 +3731,7 @@ static struct plat_sci_port *sci_parse_dt(struct platform_device *pdev,
 
 	data = of_device_get_match_data(&pdev->dev);
 
-	rstc = devm_reset_control_get_optional_exclusive(&pdev->dev, NULL);
+	rstc = devm_reset_control_array_get_optional_exclusive(&pdev->dev);
 	if (IS_ERR(rstc))
 		return ERR_PTR(dev_err_probe(&pdev->dev, PTR_ERR(rstc),
 					     "failed to get reset ctrl\n"));
@@ -3917,15 +3932,10 @@ static int sci_probe(struct platform_device *dev)
 		ret = device_create_file(&dev->dev, &dev_attr_rx_fifo_trigger);
 		if (ret)
 			return ret;
-	}
-	if (sp->type == PORT_SCIFA || sp->type == PORT_SCIFB ||
-	    sp->type == PORT_HSCIF || sp->type == SCI_PORT_RSCI) {
+
 		ret = device_create_file(&dev->dev, &dev_attr_rx_fifo_timeout);
 		if (ret) {
-			if (sp->port.fifosize > 1) {
-				device_remove_file(&dev->dev,
-						   &dev_attr_rx_fifo_trigger);
-			}
+			device_remove_file(&dev->dev, &dev_attr_rx_fifo_trigger);
 			return ret;
 		}
 	}
diff --git a/drivers/tty/vt/keyboard.c b/drivers/tty/vt/keyboard.c
index d65fc60dd7be..3538d54d6a6a 100644
--- a/drivers/tty/vt/keyboard.c
+++ b/drivers/tty/vt/keyboard.c
@@ -1649,134 +1649,143 @@ int __init kbd_init(void)
 
 /* Ioctl support code */
 
-/**
- *	vt_do_diacrit		-	diacritical table updates
- *	@cmd: ioctl request
- *	@udp: pointer to user data for ioctl
- *	@perm: permissions check computed by caller
- *
- *	Update the diacritical tables atomically and safely. Lock them
- *	against simultaneous keypresses
- */
-int vt_do_diacrit(unsigned int cmd, void __user *udp, int perm)
+static int vt_do_kdgkbdiacr(void __user *udp)
 {
-	int asize;
-
-	switch (cmd) {
-	case KDGKBDIACR:
-	{
-		struct kbdiacrs __user *a = udp;
-		int i;
+	struct kbdiacrs __user *a = udp;
+	int i, asize;
 
-		struct kbdiacr __free(kfree) *dia = kmalloc_array(MAX_DIACR, sizeof(struct kbdiacr),
-								  GFP_KERNEL);
-		if (!dia)
-			return -ENOMEM;
+	struct kbdiacr __free(kfree) *dia = kmalloc_array(MAX_DIACR, sizeof(struct kbdiacr),
+							  GFP_KERNEL);
+	if (!dia)
+		return -ENOMEM;
 
-		/* Lock the diacriticals table, make a copy and then
-		   copy it after we unlock */
-		scoped_guard(spinlock_irqsave, &kbd_event_lock) {
-			asize = accent_table_size;
-			for (i = 0; i < asize; i++) {
-				dia[i].diacr = conv_uni_to_8bit(accent_table[i].diacr);
-				dia[i].base = conv_uni_to_8bit(accent_table[i].base);
-				dia[i].result = conv_uni_to_8bit(accent_table[i].result);
-			}
+	/* Lock the diacriticals table, make a copy and then
+	   copy it after we unlock */
+	scoped_guard(spinlock_irqsave, &kbd_event_lock) {
+		asize = accent_table_size;
+		for (i = 0; i < asize; i++) {
+			dia[i].diacr = conv_uni_to_8bit(accent_table[i].diacr);
+			dia[i].base = conv_uni_to_8bit(accent_table[i].base);
+			dia[i].result = conv_uni_to_8bit(accent_table[i].result);
 		}
-
-		if (put_user(asize, &a->kb_cnt))
-			return -EFAULT;
-		if (copy_to_user(a->kbdiacr, dia, asize * sizeof(struct kbdiacr)))
-			return -EFAULT;
-		return 0;
 	}
-	case KDGKBDIACRUC:
-	{
-		struct kbdiacrsuc __user *a = udp;
 
-		void __free(kfree) *buf = kmalloc_array(MAX_DIACR, sizeof(struct kbdiacruc),
-							GFP_KERNEL);
-		if (buf == NULL)
-			return -ENOMEM;
+	if (put_user(asize, &a->kb_cnt))
+		return -EFAULT;
+	if (copy_to_user(a->kbdiacr, dia, asize * sizeof(struct kbdiacr)))
+		return -EFAULT;
+	return 0;
+}
 
-		/* Lock the diacriticals table, make a copy and then
-		   copy it after we unlock */
-		scoped_guard(spinlock_irqsave, &kbd_event_lock) {
-			asize = accent_table_size;
-			memcpy(buf, accent_table, asize * sizeof(struct kbdiacruc));
-		}
+static int vt_do_kdgkbdiacruc(void __user *udp)
+{
+	struct kbdiacrsuc __user *a = udp;
+	int asize;
 
-		if (put_user(asize, &a->kb_cnt))
-			return -EFAULT;
-		if (copy_to_user(a->kbdiacruc, buf, asize * sizeof(struct kbdiacruc)))
-			return -EFAULT;
+	void __free(kfree) *buf = kmalloc_array(MAX_DIACR, sizeof(struct kbdiacruc),
+						GFP_KERNEL);
+	if (buf == NULL)
+		return -ENOMEM;
 
-		return 0;
+	/* Lock the diacriticals table, make a copy and then
+	   copy it after we unlock */
+	scoped_guard(spinlock_irqsave, &kbd_event_lock) {
+		asize = accent_table_size;
+		memcpy(buf, accent_table, asize * sizeof(struct kbdiacruc));
 	}
 
-	case KDSKBDIACR:
-	{
-		struct kbdiacrs __user *a = udp;
-		struct kbdiacr __free(kfree) *dia = NULL;
-		unsigned int ct;
-		int i;
+	if (put_user(asize, &a->kb_cnt))
+		return -EFAULT;
+	if (copy_to_user(a->kbdiacruc, buf, asize * sizeof(struct kbdiacruc)))
+		return -EFAULT;
 
-		if (!perm)
-			return -EPERM;
-		if (get_user(ct, &a->kb_cnt))
-			return -EFAULT;
-		if (ct >= MAX_DIACR)
-			return -EINVAL;
+	return 0;
+}
 
-		if (ct) {
-			dia = memdup_array_user(a->kbdiacr,
-						ct, sizeof(struct kbdiacr));
-			if (IS_ERR(dia))
-				return PTR_ERR(dia);
-		}
+static int vt_do_kdskbdiacr(void __user *udp, int perm)
+{
+	struct kbdiacrs __user *a = udp;
+	struct kbdiacr __free(kfree) *dia = NULL;
+	unsigned int ct;
+	int i;
 
-		guard(spinlock_irqsave)(&kbd_event_lock);
-		accent_table_size = ct;
-		for (i = 0; i < ct; i++) {
-			accent_table[i].diacr =
-					conv_8bit_to_uni(dia[i].diacr);
-			accent_table[i].base =
-					conv_8bit_to_uni(dia[i].base);
-			accent_table[i].result =
-					conv_8bit_to_uni(dia[i].result);
-		}
+	if (!perm)
+		return -EPERM;
+	if (get_user(ct, &a->kb_cnt))
+		return -EFAULT;
+	if (ct >= MAX_DIACR)
+		return -EINVAL;
 
-		return 0;
+	if (ct) {
+		dia = memdup_array_user(a->kbdiacr,
+					ct, sizeof(struct kbdiacr));
+		if (IS_ERR(dia))
+			return PTR_ERR(dia);
 	}
 
-	case KDSKBDIACRUC:
-	{
-		struct kbdiacrsuc __user *a = udp;
-		unsigned int ct;
-		void __free(kfree) *buf = NULL;
+	guard(spinlock_irqsave)(&kbd_event_lock);
+	accent_table_size = ct;
+	for (i = 0; i < ct; i++) {
+		accent_table[i].diacr =
+				conv_8bit_to_uni(dia[i].diacr);
+		accent_table[i].base =
+				conv_8bit_to_uni(dia[i].base);
+		accent_table[i].result =
+				conv_8bit_to_uni(dia[i].result);
+	}
 
-		if (!perm)
-			return -EPERM;
+	return 0;
+}
 
-		if (get_user(ct, &a->kb_cnt))
-			return -EFAULT;
+static int vt_do_kdskbdiacruc(void __user *udp, int perm)
+{
+	struct kbdiacrsuc __user *a = udp;
+	unsigned int ct;
+	void __free(kfree) *buf = NULL;
 
-		if (ct >= MAX_DIACR)
-			return -EINVAL;
+	if (!perm)
+		return -EPERM;
 
-		if (ct) {
-			buf = memdup_array_user(a->kbdiacruc,
-						ct, sizeof(struct kbdiacruc));
-			if (IS_ERR(buf))
-				return PTR_ERR(buf);
-		}
-		guard(spinlock_irqsave)(&kbd_event_lock);
-		if (ct)
-			memcpy(accent_table, buf,
-					ct * sizeof(struct kbdiacruc));
-		accent_table_size = ct;
-		return 0;
+	if (get_user(ct, &a->kb_cnt))
+		return -EFAULT;
+
+	if (ct >= MAX_DIACR)
+		return -EINVAL;
+
+	if (ct) {
+		buf = memdup_array_user(a->kbdiacruc,
+					ct, sizeof(struct kbdiacruc));
+		if (IS_ERR(buf))
+			return PTR_ERR(buf);
 	}
+	guard(spinlock_irqsave)(&kbd_event_lock);
+	if (ct)
+		memcpy(accent_table, buf,
+				ct * sizeof(struct kbdiacruc));
+	accent_table_size = ct;
+	return 0;
+}
+
+/**
+ *	vt_do_diacrit		-	diacritical table updates
+ *	@cmd: ioctl request
+ *	@udp: pointer to user data for ioctl
+ *	@perm: permissions check computed by caller
+ *
+ *	Update the diacritical tables atomically and safely. Lock them
+ *	against simultaneous keypresses
+ */
+int vt_do_diacrit(unsigned int cmd, void __user *udp, int perm)
+{
+	switch (cmd) {
+	case KDGKBDIACR:
+		return vt_do_kdgkbdiacr(udp);
+	case KDGKBDIACRUC:
+		return vt_do_kdgkbdiacruc(udp);
+	case KDSKBDIACR:
+		return vt_do_kdskbdiacr(udp, perm);
+	case KDSKBDIACRUC:
+		return vt_do_kdskbdiacruc(udp, perm);
 	}
 	return 0;
 }
diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c
index 59b4b5e126ba..edda91bfdf62 100644
--- a/drivers/tty/vt/vt.c
+++ b/drivers/tty/vt/vt.c
@@ -79,6 +79,7 @@
 #include <linux/kernel.h>
 #include <linux/string.h>
 #include <linux/errno.h>
+#include <linux/hex.h>
 #include <linux/kd.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c
index 9ab91b4c05b0..64c234096e23 100644
--- a/drivers/ufs/core/ufs-mcq.c
+++ b/drivers/ufs/core/ufs-mcq.c
@@ -431,8 +431,7 @@ void ufshcd_mcq_disable(struct ufs_hba *hba)
 
 void ufshcd_mcq_enable_esi(struct ufs_hba *hba)
 {
-	ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x2,
-		      REG_UFS_MEM_CFG);
+	ufshcd_rmwl(hba, ESI_ENABLE, ESI_ENABLE, REG_UFS_MEM_CFG);
 }
 EXPORT_SYMBOL_GPL(ufshcd_mcq_enable_esi);
 
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
index 80c0b49f30b0..c9f2cf1730c2 100644
--- a/drivers/ufs/core/ufshcd.c
+++ b/drivers/ufs/core/ufshcd.c
@@ -18,6 +18,7 @@
 #include <linux/blkdev.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
+#include <linux/hex.h>
 #include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/pm_opp.h>
@@ -10359,7 +10360,7 @@ static int ufshcd_suspend(struct ufs_hba *hba)
 	ret = ufshcd_setup_clocks(hba, false);
 	if (ret) {
 		ufshcd_enable_irq(hba);
-		return ret;
+		goto out;
 	}
 	if (ufshcd_is_clkgating_allowed(hba)) {
 		hba->clk_gating.state = CLKS_OFF;
@@ -10371,6 +10372,9 @@ static int ufshcd_suspend(struct ufs_hba *hba)
 	/* Put the host controller in low power mode if possible */
 	ufshcd_hba_vreg_set_lpm(hba);
 	ufshcd_pm_qos_update(hba, false);
+out:
+	if (ret)
+		ufshcd_update_evt_hist(hba, UFS_EVT_SUSPEND_ERR, (u32)ret);
 	return ret;
 }
 
diff --git a/drivers/usb/atm/speedtch.c b/drivers/usb/atm/speedtch.c
index 773ac2725532..e6b610a87482 100644
--- a/drivers/usb/atm/speedtch.c
+++ b/drivers/usb/atm/speedtch.c
@@ -13,6 +13,7 @@
 #include <linux/device.h>
 #include <linux/errno.h>
 #include <linux/firmware.h>
+#include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
diff --git a/drivers/usb/atm/ueagle-atm.c b/drivers/usb/atm/ueagle-atm.c
index cd0f7b4bd82a..78a2585f33ec 100644
--- a/drivers/usb/atm/ueagle-atm.c
+++ b/drivers/usb/atm/ueagle-atm.c
@@ -15,6 +15,7 @@
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/crc32.h>
+#include <linux/hex.h>
 #include <linux/usb.h>
 #include <linux/firmware.h>
 #include <linux/ctype.h>
diff --git a/drivers/usb/dwc3/dwc3-of-simple.c b/drivers/usb/dwc3/dwc3-of-simple.c
index a4954a21be93..c116143335d9 100644
--- a/drivers/usb/dwc3/dwc3-of-simple.c
+++ b/drivers/usb/dwc3/dwc3-of-simple.c
@@ -70,11 +70,11 @@ static int dwc3_of_simple_probe(struct platform_device *pdev)
 	simple->num_clocks = ret;
 	ret = clk_bulk_prepare_enable(simple->num_clocks, simple->clks);
 	if (ret)
-		goto err_resetc_assert;
+		goto err_clk_put_all;
 
 	ret = of_platform_populate(np, NULL, NULL, dev);
 	if (ret)
-		goto err_clk_put;
+		goto err_clk_disable;
 
 	pm_runtime_set_active(dev);
 	pm_runtime_enable(dev);
@@ -82,8 +82,9 @@ static int dwc3_of_simple_probe(struct platform_device *pdev)
 
 	return 0;
 
-err_clk_put:
+err_clk_disable:
 	clk_bulk_disable_unprepare(simple->num_clocks, simple->clks);
+err_clk_put_all:
 	clk_bulk_put_all(simple->num_clocks, simple->clks);
 
 err_resetc_assert:
diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c
index bc3fe31638b9..8a35a6901db7 100644
--- a/drivers/usb/dwc3/gadget.c
+++ b/drivers/usb/dwc3/gadget.c
@@ -4826,7 +4826,7 @@ void dwc3_gadget_exit(struct dwc3 *dwc)
 	if (!dwc->gadget)
 		return;
 
-	dwc3_enable_susphy(dwc, false);
+	dwc3_enable_susphy(dwc, true);
 	usb_del_gadget(dwc->gadget);
 	dwc3_gadget_free_endpoints(dwc);
 	usb_put_gadget(dwc->gadget);
diff --git a/drivers/usb/dwc3/host.c b/drivers/usb/dwc3/host.c
index cf6512ed17a6..96b588bd08cd 100644
--- a/drivers/usb/dwc3/host.c
+++ b/drivers/usb/dwc3/host.c
@@ -227,7 +227,7 @@ void dwc3_host_exit(struct dwc3 *dwc)
 	if (dwc->sys_wakeup)
 		device_init_wakeup(&dwc->xhci->dev, false);
 
-	dwc3_enable_susphy(dwc, false);
+	dwc3_enable_susphy(dwc, true);
 	platform_device_unregister(dwc->xhci);
 	dwc->xhci = NULL;
 }
diff --git a/drivers/usb/gadget/function/f_midi.c b/drivers/usb/gadget/function/f_midi.c
index da82598fcef8..ad679a6ecac1 100644
--- a/drivers/usb/gadget/function/f_midi.c
+++ b/drivers/usb/gadget/function/f_midi.c
@@ -875,6 +875,7 @@ static int f_midi_bind(struct usb_configuration *c, struct usb_function *f)
 	struct usb_composite_dev *cdev = c->cdev;
 	struct f_midi *midi = func_to_midi(f);
 	struct usb_string *us;
+	struct f_midi_opts *opts;
 	int status, n, jack = 1, i = 0, endpoint_descriptor_index = 0;
 
 	midi->gadget = cdev->gadget;
@@ -883,6 +884,10 @@ static int f_midi_bind(struct usb_configuration *c, struct usb_function *f)
 	if (status < 0)
 		goto fail_register;
 
+	opts = container_of(f->fi, struct f_midi_opts, func_inst);
+	if (opts->interface_string)
+		midi_string_defs[STRING_FUNC_IDX].s = opts->interface_string;
+
 	/* maybe allocate device-global string ID */
 	us = usb_gstrings_attach(c->cdev, midi_strings,
 				 ARRAY_SIZE(midi_string_defs));
@@ -1178,59 +1183,60 @@ end:									\
 									\
 CONFIGFS_ATTR(f_midi_opts_, name);
 
+#define F_MIDI_OPT_STRING(name)						\
+static ssize_t f_midi_opts_##name##_show(struct config_item *item, char *page) \
+{									\
+	struct f_midi_opts *opts = to_f_midi_opts(item);		\
+	ssize_t result;							\
+									\
+	mutex_lock(&opts->lock);					\
+	if (opts->name) {						\
+		result = strscpy(page, opts->name, PAGE_SIZE);		\
+	} else {							\
+		page[0] = 0;						\
+		result = 0;						\
+	}								\
+									\
+	mutex_unlock(&opts->lock);					\
+									\
+	return result;							\
+}									\
+									\
+static ssize_t f_midi_opts_##name##_store(struct config_item *item,	\
+					 const char *page, size_t len)	\
+{									\
+	struct f_midi_opts *opts = to_f_midi_opts(item);		\
+	int ret;							\
+	char *c;							\
+									\
+	mutex_lock(&opts->lock);					\
+	if (opts->refcnt > 1) {						\
+		ret = -EBUSY;						\
+		goto end;						\
+	}								\
+									\
+	c = kstrndup(page, len, GFP_KERNEL);				\
+	if (!c) {							\
+		ret = -ENOMEM;						\
+		goto end;						\
+	}								\
+	kfree(opts->name);						\
+	opts->name = c;							\
+	ret = len;							\
+end:									\
+	mutex_unlock(&opts->lock);					\
+	return ret;							\
+}									\
+									\
+CONFIGFS_ATTR(f_midi_opts_, name)
+
 F_MIDI_OPT_SIGNED(index, true, SNDRV_CARDS);
 F_MIDI_OPT(buflen, false, 0);
 F_MIDI_OPT(qlen, false, 0);
 F_MIDI_OPT(in_ports, true, MAX_PORTS);
 F_MIDI_OPT(out_ports, true, MAX_PORTS);
-
-static ssize_t f_midi_opts_id_show(struct config_item *item, char *page)
-{
-	struct f_midi_opts *opts = to_f_midi_opts(item);
-	ssize_t result;
-
-	mutex_lock(&opts->lock);
-	if (opts->id) {
-		result = strscpy(page, opts->id, PAGE_SIZE);
-	} else {
-		page[0] = 0;
-		result = 0;
-	}
-
-	mutex_unlock(&opts->lock);
-
-	return result;
-}
-
-static ssize_t f_midi_opts_id_store(struct config_item *item,
-				    const char *page, size_t len)
-{
-	struct f_midi_opts *opts = to_f_midi_opts(item);
-	int ret;
-	char *c;
-
-	mutex_lock(&opts->lock);
-	if (opts->refcnt > 1) {
-		ret = -EBUSY;
-		goto end;
-	}
-
-	c = kstrndup(page, len, GFP_KERNEL);
-	if (!c) {
-		ret = -ENOMEM;
-		goto end;
-	}
-	if (opts->id_allocated)
-		kfree(opts->id);
-	opts->id = c;
-	opts->id_allocated = true;
-	ret = len;
-end:
-	mutex_unlock(&opts->lock);
-	return ret;
-}
-
-CONFIGFS_ATTR(f_midi_opts_, id);
+F_MIDI_OPT_STRING(id);
+F_MIDI_OPT_STRING(interface_string);
 
 static struct configfs_attribute *midi_attrs[] = {
 	&f_midi_opts_attr_index,
@@ -1239,6 +1245,7 @@ static struct configfs_attribute *midi_attrs[] = {
 	&f_midi_opts_attr_in_ports,
 	&f_midi_opts_attr_out_ports,
 	&f_midi_opts_attr_id,
+	&f_midi_opts_attr_interface_string,
 	NULL,
 };
 
@@ -1262,8 +1269,8 @@ static void f_midi_free_inst(struct usb_function_instance *f)
 	mutex_unlock(&opts->lock);
 
 	if (free) {
-		if (opts->id_allocated)
-			kfree(opts->id);
+		kfree(opts->id);
+		kfree(opts->interface_string);
 		kfree(opts);
 	}
 }
@@ -1279,7 +1286,8 @@ static struct usb_function_instance *f_midi_alloc_inst(void)
 	mutex_init(&opts->lock);
 	opts->func_inst.free_func_inst = f_midi_free_inst;
 	opts->index = SNDRV_DEFAULT_IDX1;
-	opts->id = SNDRV_DEFAULT_STR1;
+	opts->id = NULL;
+	opts->interface_string = NULL;
 	opts->buflen = 512;
 	opts->qlen = 32;
 	opts->in_ports = 1;
diff --git a/drivers/usb/gadget/function/u_ether.c b/drivers/usb/gadget/function/u_ether.c
index f58590bf5e02..c47965d850d4 100644
--- a/drivers/usb/gadget/function/u_ether.c
+++ b/drivers/usb/gadget/function/u_ether.c
@@ -16,6 +16,7 @@
 #include <linux/ctype.h>
 #include <linux/etherdevice.h>
 #include <linux/ethtool.h>
+#include <linux/hex.h>
 #include <linux/if_vlan.h>
 #include <linux/string_helpers.h>
 #include <linux/usb/composite.h>
diff --git a/drivers/usb/gadget/function/u_midi.h b/drivers/usb/gadget/function/u_midi.h
index 2e400b495cb8..41cb8aa73f09 100644
--- a/drivers/usb/gadget/function/u_midi.h
+++ b/drivers/usb/gadget/function/u_midi.h
@@ -19,7 +19,7 @@ struct f_midi_opts {
 	struct usb_function_instance	func_inst;
 	int				index;
 	char				*id;
-	bool				id_allocated;
+	char				*interface_string;
 	unsigned int			in_ports;
 	unsigned int			out_ports;
 	unsigned int			buflen;
diff --git a/drivers/usb/gadget/function/uvc_configfs.c b/drivers/usb/gadget/function/uvc_configfs.c
index a4a2d3dcb0d6..5a87516ddb31 100644
--- a/drivers/usb/gadget/function/uvc_configfs.c
+++ b/drivers/usb/gadget/function/uvc_configfs.c
@@ -12,6 +12,7 @@
 
 #include "uvc_configfs.h"
 
+#include <linux/hex.h>
 #include <linux/sort.h>
 #include <linux/usb/uvc.h>
 #include <linux/usb/video.h>
diff --git a/drivers/usb/gadget/udc/aspeed-vhub/core.c b/drivers/usb/gadget/udc/aspeed-vhub/core.c
index f2685f89b3e5..19c1849ae665 100644
--- a/drivers/usb/gadget/udc/aspeed-vhub/core.c
+++ b/drivers/usb/gadget/udc/aspeed-vhub/core.c
@@ -23,6 +23,7 @@
 #include <linux/of.h>
 #include <linux/regmap.h>
 #include <linux/dma-mapping.h>
+#include <linux/reset.h>
 
 #include "vhub.h"
 
@@ -280,6 +281,8 @@ static void ast_vhub_remove(struct platform_device *pdev)
 	if (vhub->clk)
 		clk_disable_unprepare(vhub->clk);
 
+	reset_control_assert(vhub->rst);
+
 	spin_unlock_irqrestore(&vhub->lock, flags);
 
 	if (vhub->ep0_bufs)
@@ -294,6 +297,7 @@ static void ast_vhub_remove(struct platform_device *pdev)
 static int ast_vhub_probe(struct platform_device *pdev)
 {
 	enum usb_device_speed max_speed;
+	const u64 *dma_mask_ptr;
 	struct ast_vhub *vhub;
 	struct resource *res;
 	int i, rc = 0;
@@ -348,6 +352,16 @@ static int ast_vhub_probe(struct platform_device *pdev)
 		goto err;
 	}
 
+	vhub->rst = devm_reset_control_get_optional_shared(&pdev->dev, NULL);
+	if (IS_ERR(vhub->rst)) {
+		rc = PTR_ERR(vhub->rst);
+		goto err;
+	}
+
+	rc = reset_control_deassert(vhub->rst);
+	if (rc)
+		goto err;
+
 	/* Check if we need to limit the HW to USB1 */
 	max_speed = usb_get_maximum_speed(&pdev->dev);
 	if (max_speed != USB_SPEED_UNKNOWN && max_speed < USB_SPEED_HIGH)
@@ -370,6 +384,12 @@ static int ast_vhub_probe(struct platform_device *pdev)
 		goto err;
 	}
 
+	dma_mask_ptr = (u64 *)of_device_get_match_data(&pdev->dev);
+	if (dma_mask_ptr) {
+		rc = dma_coerce_mask_and_coherent(&pdev->dev, *dma_mask_ptr);
+		if (rc)
+			goto err;
+	}
 	/*
 	 * Allocate DMA buffers for all EP0s in one chunk,
 	 * one per port and one for the vHub itself
@@ -412,15 +432,25 @@ static int ast_vhub_probe(struct platform_device *pdev)
 	return rc;
 }
 
+static const u64 dma_mask_32 =	DMA_BIT_MASK(32);
+static const u64 dma_mask_64 =	DMA_BIT_MASK(64);
+
 static const struct of_device_id ast_vhub_dt_ids[] = {
 	{
 		.compatible = "aspeed,ast2400-usb-vhub",
+		.data = &dma_mask_32,
 	},
 	{
 		.compatible = "aspeed,ast2500-usb-vhub",
+		.data = &dma_mask_32,
 	},
 	{
 		.compatible = "aspeed,ast2600-usb-vhub",
+		.data = &dma_mask_32,
+	},
+	{
+		.compatible = "aspeed,ast2700-usb-vhub",
+		.data = &dma_mask_64,
 	},
 	{ }
 };
diff --git a/drivers/usb/gadget/udc/aspeed-vhub/vhub.h b/drivers/usb/gadget/udc/aspeed-vhub/vhub.h
index 6b9dfa6e10eb..aca2050e2db0 100644
--- a/drivers/usb/gadget/udc/aspeed-vhub/vhub.h
+++ b/drivers/usb/gadget/udc/aspeed-vhub/vhub.h
@@ -388,6 +388,7 @@ struct ast_vhub {
 	spinlock_t			lock;
 	struct work_struct		wake_work;
 	struct clk			*clk;
+	struct reset_control		*rst;
 
 	/* EP0 DMA buffers allocated in one chunk */
 	void				*ep0_bufs;
diff --git a/drivers/usb/gadget/udc/lpc32xx_udc.c b/drivers/usb/gadget/udc/lpc32xx_udc.c
index 1a7d3c4f652f..73c0f28a8585 100644
--- a/drivers/usb/gadget/udc/lpc32xx_udc.c
+++ b/drivers/usb/gadget/udc/lpc32xx_udc.c
@@ -3020,7 +3020,7 @@ static int lpc32xx_udc_probe(struct platform_device *pdev)
 	pdev->dev.dma_mask = &lpc32xx_usbd_dmamask;
 	retval = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
 	if (retval)
-		return retval;
+		goto i2c_fail;
 
 	udc->board = &lpc32xx_usbddata;
 
@@ -3038,28 +3038,32 @@ static int lpc32xx_udc_probe(struct platform_device *pdev)
 	/* Get IRQs */
 	for (i = 0; i < 4; i++) {
 		udc->udp_irq[i] = platform_get_irq(pdev, i);
-		if (udc->udp_irq[i] < 0)
-			return udc->udp_irq[i];
+		if (udc->udp_irq[i] < 0) {
+			retval = udc->udp_irq[i];
+			goto i2c_fail;
+		}
 	}
 
 	udc->udp_baseaddr = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(udc->udp_baseaddr)) {
 		dev_err(udc->dev, "IO map failure\n");
-		return PTR_ERR(udc->udp_baseaddr);
+		retval = PTR_ERR(udc->udp_baseaddr);
+		goto i2c_fail;
 	}
 
 	/* Get USB device clock */
 	udc->usb_slv_clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(udc->usb_slv_clk)) {
 		dev_err(udc->dev, "failed to acquire USB device clock\n");
-		return PTR_ERR(udc->usb_slv_clk);
+		retval = PTR_ERR(udc->usb_slv_clk);
+		goto i2c_fail;
 	}
 
 	/* Enable USB device clock */
 	retval = clk_prepare_enable(udc->usb_slv_clk);
 	if (retval < 0) {
 		dev_err(udc->dev, "failed to start USB device clock\n");
-		return retval;
+		goto i2c_fail;
 	}
 
 	/* Setup deferred workqueue data */
@@ -3161,6 +3165,8 @@ static int lpc32xx_udc_probe(struct platform_device *pdev)
 	dma_free_coherent(&pdev->dev, UDCA_BUFF_SIZE,
 			  udc->udca_v_base, udc->udca_p_base);
 i2c_fail:
+	if (udc->isp1301_i2c_client)
+		put_device(&udc->isp1301_i2c_client->dev);
 	clk_disable_unprepare(udc->usb_slv_clk);
 	dev_err(udc->dev, "%s probe failed, %d\n", driver_name, retval);
 
@@ -3189,6 +3195,9 @@ static void lpc32xx_udc_remove(struct platform_device *pdev)
 	dma_free_coherent(&pdev->dev, UDCA_BUFF_SIZE,
 			  udc->udca_v_base, udc->udca_p_base);
 
+	if (udc->isp1301_i2c_client)
+		put_device(&udc->isp1301_i2c_client->dev);
+
 	clk_disable_unprepare(udc->usb_slv_clk);
 }
 
diff --git a/drivers/usb/host/xhci-dbgtty.c b/drivers/usb/host/xhci-dbgtty.c
index 57cdda4e09c8..90282e51e23e 100644
--- a/drivers/usb/host/xhci-dbgtty.c
+++ b/drivers/usb/host/xhci-dbgtty.c
@@ -554,7 +554,7 @@ static void xhci_dbc_tty_unregister_device(struct xhci_dbc *dbc)
 	 * Hang up the TTY. This wakes up any blocked
 	 * writers and causes subsequent writes to fail.
 	 */
-	tty_vhangup(port->port.tty);
+	tty_port_tty_vhangup(&port->port);
 
 	tty_unregister_device(dbc_tty_driver, port->minor);
 	xhci_dbc_tty_exit_port(port);
diff --git a/drivers/usb/phy/phy-fsl-usb.c b/drivers/usb/phy/phy-fsl-usb.c
index 40ac68e52cee..e266a47c4d48 100644
--- a/drivers/usb/phy/phy-fsl-usb.c
+++ b/drivers/usb/phy/phy-fsl-usb.c
@@ -988,6 +988,7 @@ static void fsl_otg_remove(struct platform_device *pdev)
 {
 	struct fsl_usb2_platform_data *pdata = dev_get_platdata(&pdev->dev);
 
+	disable_delayed_work_sync(&fsl_otg_dev->otg_event);
 	usb_remove_phy(&fsl_otg_dev->phy);
 	free_irq(fsl_otg_dev->irq, fsl_otg_dev);
 
diff --git a/drivers/usb/renesas_usbhs/pipe.c b/drivers/usb/renesas_usbhs/pipe.c
index 75fff2e4cbc6..56fc3ff5016f 100644
--- a/drivers/usb/renesas_usbhs/pipe.c
+++ b/drivers/usb/renesas_usbhs/pipe.c
@@ -713,11 +713,13 @@ struct usbhs_pipe *usbhs_pipe_malloc(struct usbhs_priv *priv,
 	/* make sure pipe is not busy */
 	ret = usbhsp_pipe_barrier(pipe);
 	if (ret < 0) {
+		usbhsp_put_pipe(pipe);
 		dev_err(dev, "pipe setup failed %d\n", usbhs_pipe_number(pipe));
 		return NULL;
 	}
 
 	if (usbhsp_setup_pipecfg(pipe, is_host, dir_in, &pipecfg)) {
+		usbhsp_put_pipe(pipe);
 		dev_err(dev, "can't setup pipe\n");
 		return NULL;
 	}
diff --git a/drivers/usb/typec/altmodes/displayport.c b/drivers/usb/typec/altmodes/displayport.c
index 8d111ad3b71b..d96ab106a980 100644
--- a/drivers/usb/typec/altmodes/displayport.c
+++ b/drivers/usb/typec/altmodes/displayport.c
@@ -766,12 +766,16 @@ int dp_altmode_probe(struct typec_altmode *alt)
 	if (!(DP_CAP_PIN_ASSIGN_DFP_D(port->vdo) &
 	      DP_CAP_PIN_ASSIGN_UFP_D(alt->vdo)) &&
 	    !(DP_CAP_PIN_ASSIGN_UFP_D(port->vdo) &
-	      DP_CAP_PIN_ASSIGN_DFP_D(alt->vdo)))
+	      DP_CAP_PIN_ASSIGN_DFP_D(alt->vdo))) {
+		typec_altmode_put_plug(plug);
 		return -ENODEV;
+	}
 
 	dp = devm_kzalloc(&alt->dev, sizeof(*dp), GFP_KERNEL);
-	if (!dp)
+	if (!dp) {
+		typec_altmode_put_plug(plug);
 		return -ENOMEM;
+	}
 
 	INIT_WORK(&dp->work, dp_altmode_work);
 	mutex_init(&dp->lock);
diff --git a/drivers/usb/typec/bus.c b/drivers/usb/typec/bus.c
index a884cec9ab7e..e84b134a3381 100644
--- a/drivers/usb/typec/bus.c
+++ b/drivers/usb/typec/bus.c
@@ -445,7 +445,23 @@ static struct attribute *typec_attrs[] = {
 	&dev_attr_description.attr,
 	NULL
 };
-ATTRIBUTE_GROUPS(typec);
+
+static umode_t typec_is_visible(struct kobject *kobj, struct attribute *attr, int n)
+{
+	if (is_typec_partner_altmode(kobj_to_dev(kobj)))
+		return attr->mode;
+	return 0;
+}
+
+static const struct attribute_group typec_group = {
+	.is_visible = typec_is_visible,
+	.attrs = typec_attrs,
+};
+
+static const struct attribute_group *typec_groups[] = {
+	&typec_group,
+	NULL
+};
 
 static int typec_match(struct device *dev, const struct device_driver *driver)
 {
@@ -453,6 +469,9 @@ static int typec_match(struct device *dev, const struct device_driver *driver)
 	struct typec_altmode *altmode = to_typec_altmode(dev);
 	const struct typec_device_id *id;
 
+	if (!is_typec_partner_altmode(dev))
+		return 0;
+
 	for (id = drv->id_table; id->svid; id++)
 		if (id->svid == altmode->svid)
 			return 1;
@@ -463,6 +482,9 @@ static int typec_uevent(const struct device *dev, struct kobj_uevent_env *env)
 {
 	const struct typec_altmode *altmode = to_typec_altmode(dev);
 
+	if (!is_typec_partner_altmode(dev))
+		return 0;
+
 	if (add_uevent_var(env, "SVID=%04X", altmode->svid))
 		return -ENOMEM;
 
@@ -547,3 +569,4 @@ const struct bus_type typec_bus = {
 	.probe = typec_probe,
 	.remove = typec_remove,
 };
+EXPORT_SYMBOL_GPL(typec_bus);
diff --git a/drivers/usb/typec/bus.h b/drivers/usb/typec/bus.h
index 643b8c81786d..7df5deb1dd3a 100644
--- a/drivers/usb/typec/bus.h
+++ b/drivers/usb/typec/bus.h
@@ -5,7 +5,6 @@
 
 #include <linux/usb/typec_altmode.h>
 
-struct bus_type;
 struct typec_mux;
 struct typec_retimer;
 
@@ -28,9 +27,4 @@ struct altmode {
 
 #define to_altmode(d) container_of(d, struct altmode, adev)
 
-extern const struct bus_type typec_bus;
-extern const struct device_type typec_altmode_dev_type;
-
-#define is_typec_altmode(_dev_) (_dev_->type == &typec_altmode_dev_type)
-
 #endif /* __USB_TYPEC_ALTMODE_H__ */
diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c
index 9b2647cb199b..c4ff4310ff58 100644
--- a/drivers/usb/typec/class.c
+++ b/drivers/usb/typec/class.c
@@ -235,7 +235,7 @@ static int altmode_match(struct device *dev, const void *data)
 	struct typec_altmode *adev = to_typec_altmode(dev);
 	const struct typec_device_id *id = data;
 
-	if (!is_typec_altmode(dev))
+	if (!is_typec_port_altmode(dev))
 		return 0;
 
 	return (adev->svid == id->svid);
@@ -532,15 +532,31 @@ static void typec_altmode_release(struct device *dev)
 	kfree(alt);
 }
 
-const struct device_type typec_altmode_dev_type = {
-	.name = "typec_alternate_mode",
+const struct device_type typec_port_altmode_dev_type = {
+	.name = "typec_port_alternate_mode",
 	.groups = typec_altmode_groups,
 	.release = typec_altmode_release,
 };
+EXPORT_SYMBOL_GPL(typec_port_altmode_dev_type);
+
+const struct device_type typec_plug_altmode_dev_type = {
+	.name = "typec_plug_alternate_mode",
+	.groups = typec_altmode_groups,
+	.release = typec_altmode_release,
+};
+EXPORT_SYMBOL_GPL(typec_plug_altmode_dev_type);
+
+const struct device_type typec_partner_altmode_dev_type = {
+	.name = "typec_partner_alternate_mode",
+	.groups = typec_altmode_groups,
+	.release = typec_altmode_release,
+};
+EXPORT_SYMBOL_GPL(typec_partner_altmode_dev_type);
 
 static struct typec_altmode *
 typec_register_altmode(struct device *parent,
-		       const struct typec_altmode_desc *desc)
+		       const struct typec_altmode_desc *desc,
+		       const struct device_type *type)
 {
 	unsigned int id = altmode_id_get(parent);
 	bool is_port = is_typec_port(parent);
@@ -575,7 +591,7 @@ typec_register_altmode(struct device *parent,
 
 	alt->adev.dev.parent = parent;
 	alt->adev.dev.groups = alt->groups;
-	alt->adev.dev.type = &typec_altmode_dev_type;
+	alt->adev.dev.type = type;
 	dev_set_name(&alt->adev.dev, "%s.%u", dev_name(parent), id);
 
 	get_device(alt->adev.dev.parent);
@@ -584,9 +600,7 @@ typec_register_altmode(struct device *parent,
 	if (!is_port)
 		typec_altmode_set_partner(alt);
 
-	/* The partners are bind to drivers */
-	if (is_typec_partner(parent))
-		alt->adev.dev.bus = &typec_bus;
+	alt->adev.dev.bus = &typec_bus;
 
 	/* Plug alt modes need a class to generate udev events. */
 	if (is_typec_plug(parent))
@@ -963,7 +977,7 @@ struct typec_altmode *
 typec_partner_register_altmode(struct typec_partner *partner,
 			       const struct typec_altmode_desc *desc)
 {
-	return typec_register_altmode(&partner->dev, desc);
+	return typec_register_altmode(&partner->dev, desc, &typec_partner_altmode_dev_type);
 }
 EXPORT_SYMBOL_GPL(typec_partner_register_altmode);
 
@@ -1193,7 +1207,7 @@ struct typec_altmode *
 typec_plug_register_altmode(struct typec_plug *plug,
 			    const struct typec_altmode_desc *desc)
 {
-	return typec_register_altmode(&plug->dev, desc);
+	return typec_register_altmode(&plug->dev, desc, &typec_plug_altmode_dev_type);
 }
 EXPORT_SYMBOL_GPL(typec_plug_register_altmode);
 
@@ -2493,7 +2507,7 @@ typec_port_register_altmode(struct typec_port *port,
 		return ERR_CAST(retimer);
 	}
 
-	adev = typec_register_altmode(&port->dev, desc);
+	adev = typec_register_altmode(&port->dev, desc, &typec_port_altmode_dev_type);
 	if (IS_ERR(adev)) {
 		typec_retimer_put(retimer);
 		typec_mux_put(mux);
diff --git a/drivers/usb/typec/ucsi/Kconfig b/drivers/usb/typec/ucsi/Kconfig
index 7fcb1e1de5d6..b812be4d0e67 100644
--- a/drivers/usb/typec/ucsi/Kconfig
+++ b/drivers/usb/typec/ucsi/Kconfig
@@ -96,6 +96,7 @@ config UCSI_LENOVO_YOGA_C630
 config UCSI_HUAWEI_GAOKUN
 	tristate "UCSI Interface Driver for Huawei Matebook E Go"
 	depends on EC_HUAWEI_GAOKUN
+	depends on DRM || !DRM
 	select DRM_AUX_HPD_BRIDGE if DRM_BRIDGE && OF
 	help
 	  This driver enables UCSI support on the Huawei Matebook E Go tablet,
diff --git a/drivers/usb/typec/ucsi/debugfs.c b/drivers/usb/typec/ucsi/debugfs.c
index 174f4d53b777..b71fde30d153 100644
--- a/drivers/usb/typec/ucsi/debugfs.c
+++ b/drivers/usb/typec/ucsi/debugfs.c
@@ -8,6 +8,7 @@
  *	    Gopal Saranya <saranya.gopal@intel.com>
  */
 #include <linux/debugfs.h>
+#include <linux/hex.h>
 #include <linux/slab.h>
 #include <linux/string.h>
 #include <linux/types.h>
diff --git a/drivers/usb/typec/ucsi/psy.c b/drivers/usb/typec/ucsi/psy.c
index 3abe9370ffaa..c8ebab8ba7e7 100644
--- a/drivers/usb/typec/ucsi/psy.c
+++ b/drivers/usb/typec/ucsi/psy.c
@@ -112,15 +112,20 @@ static int ucsi_psy_get_voltage_max(struct ucsi_connector *con,
 				    union power_supply_propval *val)
 {
 	u32 pdo;
+	int max_voltage = 0;
 
 	switch (UCSI_CONSTAT(con, PWR_OPMODE)) {
 	case UCSI_CONSTAT_PWR_OPMODE_PD:
-		if (con->num_pdos > 0) {
-			pdo = con->src_pdos[con->num_pdos - 1];
-			val->intval = pdo_fixed_voltage(pdo) * 1000;
-		} else {
-			val->intval = 0;
+		for (int i = 0; i < con->num_pdos; i++) {
+			int pdo_voltage = 0;
+
+			pdo = con->src_pdos[i];
+			if (pdo_type(pdo) == PDO_TYPE_FIXED)
+				pdo_voltage = pdo_fixed_voltage(pdo) * 1000;
+			max_voltage = (pdo_voltage > max_voltage) ? pdo_voltage
+								  : max_voltage;
 		}
+		val->intval = max_voltage;
 		break;
 	case UCSI_CONSTAT_PWR_OPMODE_TYPEC3_0:
 	case UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5:
@@ -168,6 +173,7 @@ static int ucsi_psy_get_current_max(struct ucsi_connector *con,
 				    union power_supply_propval *val)
 {
 	u32 pdo;
+	int max_current = 0;
 
 	if (!UCSI_CONSTAT(con, CONNECTED)) {
 		val->intval = 0;
@@ -176,12 +182,16 @@ static int ucsi_psy_get_current_max(struct ucsi_connector *con,
 
 	switch (UCSI_CONSTAT(con, PWR_OPMODE)) {
 	case UCSI_CONSTAT_PWR_OPMODE_PD:
-		if (con->num_pdos > 0) {
-			pdo = con->src_pdos[con->num_pdos - 1];
-			val->intval = pdo_max_current(pdo) * 1000;
-		} else {
-			val->intval = 0;
+		for (int i = 0; i < con->num_pdos; i++) {
+			int pdo_current = 0;
+
+			pdo = con->src_pdos[i];
+			if (pdo_type(pdo) == PDO_TYPE_FIXED)
+				pdo_current = pdo_max_current(pdo) * 1000;
+			max_current = (pdo_current > max_current) ? pdo_current
+								  : max_current;
 		}
+		val->intval = max_current;
 		break;
 	case UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5:
 		val->intval = UCSI_TYPEC_1_5_CURRENT * 1000;
@@ -202,10 +212,28 @@ static int ucsi_psy_get_current_max(struct ucsi_connector *con,
 static int ucsi_psy_get_current_now(struct ucsi_connector *con,
 				    union power_supply_propval *val)
 {
-	if (UCSI_CONSTAT(con, PWR_OPMODE) == UCSI_CONSTAT_PWR_OPMODE_PD)
-		val->intval = rdo_op_current(con->rdo) * 1000;
-	else
+	if (!UCSI_CONSTAT(con, CONNECTED)) {
 		val->intval = 0;
+		return 0;
+	}
+
+	switch (UCSI_CONSTAT(con, PWR_OPMODE)) {
+	case UCSI_CONSTAT_PWR_OPMODE_PD:
+		val->intval = rdo_op_current(con->rdo) * 1000;
+		break;
+	case UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5:
+		val->intval = UCSI_TYPEC_1_5_CURRENT * 1000;
+		break;
+	case UCSI_CONSTAT_PWR_OPMODE_TYPEC3_0:
+		val->intval = UCSI_TYPEC_3_0_CURRENT * 1000;
+		break;
+	case UCSI_CONSTAT_PWR_OPMODE_BC:
+	case UCSI_CONSTAT_PWR_OPMODE_DEFAULT:
+	/* UCSI can't tell b/w DCP/CDP or USB2/3x1/3x2 SDP chargers */
+	default:
+		val->intval = UCSI_TYPEC_DEFAULT_CURRENT * 1000;
+		break;
+	}
 	return 0;
 }
 
diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c
index 9b3df776137a..7129973f19e7 100644
--- a/drivers/usb/typec/ucsi/ucsi.c
+++ b/drivers/usb/typec/ucsi/ucsi.c
@@ -97,7 +97,7 @@ int ucsi_sync_control_common(struct ucsi *ucsi, u64 command, u32 *cci)
 	if (!ret && cci)
 		ret = ucsi->ops->read_cci(ucsi, cci);
 
-	if (!ret && ucsi->message_in_size > 0 &&
+	if (!ret && cci && ucsi->message_in_size > 0 &&
 	    (*cci & UCSI_CCI_COMMAND_COMPLETE))
 		ret = ucsi->ops->read_message_in(ucsi, ucsi->message_in,
 						 ucsi->message_in_size);
diff --git a/drivers/usb/typec/ucsi/ucsi_ccg.c b/drivers/usb/typec/ucsi/ucsi_ccg.c
index ead1b2a25c79..cf90b280ada3 100644
--- a/drivers/usb/typec/ucsi/ucsi_ccg.c
+++ b/drivers/usb/typec/ucsi/ucsi_ccg.c
@@ -10,6 +10,7 @@
 #include <linux/acpi.h>
 #include <linux/delay.h>
 #include <linux/firmware.h>
+#include <linux/hex.h>
 #include <linux/i2c.h>
 #include <linux/module.h>
 #include <linux/pci.h>
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index bccdc9eab267..fcf7f10adbbf 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -1444,13 +1444,13 @@ static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq,
 ({ \
 	int ret; \
 	if (!vq->iotlb) { \
-		ret = __put_user(x, ptr); \
+		ret = put_user(x, ptr); \
 	} else { \
 		__typeof__(ptr) to = \
 			(__typeof__(ptr)) __vhost_get_user(vq, ptr,	\
 					  sizeof(*ptr), VHOST_ADDR_USED); \
 		if (to != NULL) \
-			ret = __put_user(x, to); \
+			ret = put_user(x, to); \
 		else \
 			ret = -EFAULT;	\
 	} \
@@ -1489,14 +1489,14 @@ static inline int vhost_put_used_idx(struct vhost_virtqueue *vq)
 ({ \
 	int ret; \
 	if (!vq->iotlb) { \
-		ret = __get_user(x, ptr); \
+		ret = get_user(x, ptr); \
 	} else { \
 		__typeof__(ptr) from = \
 			(__typeof__(ptr)) __vhost_get_user(vq, ptr, \
 							   sizeof(*ptr), \
 							   type); \
 		if (from != NULL) \
-			ret = __get_user(x, from); \
+			ret = get_user(x, from); \
 		else \
 			ret = -EFAULT; \
 	} \
diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c
index 0298ddc34824..552cfb53498a 100644
--- a/drivers/vhost/vsock.c
+++ b/drivers/vhost/vsock.c
@@ -66,14 +66,15 @@ static u32 vhost_transport_get_local_cid(void)
 	return VHOST_VSOCK_DEFAULT_HOST_CID;
 }
 
-/* Callers that dereference the return value must hold vhost_vsock_mutex or the
- * RCU read lock.
+/* Callers must be in an RCU read section or hold the vhost_vsock_mutex.
+ * The return value can only be dereferenced while within the section.
  */
 static struct vhost_vsock *vhost_vsock_get(u32 guest_cid)
 {
 	struct vhost_vsock *vsock;
 
-	hash_for_each_possible_rcu(vhost_vsock_hash, vsock, hash, guest_cid) {
+	hash_for_each_possible_rcu(vhost_vsock_hash, vsock, hash, guest_cid,
+				   lockdep_is_held(&vhost_vsock_mutex)) {
 		u32 other_cid = vsock->guest_cid;
 
 		/* Skip instances that have no CID yet */
@@ -709,9 +710,15 @@ static void vhost_vsock_reset_orphans(struct sock *sk)
 	 * executing.
 	 */
 
+	rcu_read_lock();
+
 	/* If the peer is still valid, no need to reset connection */
-	if (vhost_vsock_get(vsk->remote_addr.svm_cid))
+	if (vhost_vsock_get(vsk->remote_addr.svm_cid)) {
+		rcu_read_unlock();
 		return;
+	}
+
+	rcu_read_unlock();
 
 	/* If the close timeout is pending, let it expire.  This avoids races
 	 * with the timeout callback.
diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig
index d51777df12d1..9884f003247d 100644
--- a/drivers/video/Kconfig
+++ b/drivers/video/Kconfig
@@ -63,11 +63,13 @@ endif # HAS_IOMEM
 
 config FIRMWARE_EDID
 	bool "Enable firmware EDID"
-	depends on X86
+	depends on EFI_GENERIC_STUB || X86
 	help
 	  This enables access to the EDID transferred from the firmware.
-	  On x86, this is from the VESA BIOS. DRM display drivers will
-	  be able to export the information to userspace.
+	  On EFI systems, the EDID comes from the same device as the
+	  primary GOP. On x86 with BIOS, it comes from the VESA BIOS.
+	  DRM display drivers will be able to export the information
+	  to userspace.
 
 	  Also enable this if DDC/I2C transfers do not work for your driver
 	  and if you are using nvidiafb, i810fb or savagefb.
diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c
index 7be9e865325d..34ea14412ace 100644
--- a/drivers/video/fbdev/core/fbcon.c
+++ b/drivers/video/fbdev/core/fbcon.c
@@ -2289,28 +2289,6 @@ static bool fbcon_blank(struct vc_data *vc, enum vesa_blank_mode blank,
 	return false;
 }
 
-static void fbcon_debug_enter(struct vc_data *vc)
-{
-	struct fb_info *info = fbcon_info_from_console(vc->vc_num);
-	struct fbcon_par *par = info->fbcon_par;
-
-	par->save_graphics = par->graphics;
-	par->graphics = 0;
-	if (info->fbops->fb_debug_enter)
-		info->fbops->fb_debug_enter(info);
-	fbcon_set_palette(vc, color_table);
-}
-
-static void fbcon_debug_leave(struct vc_data *vc)
-{
-	struct fb_info *info = fbcon_info_from_console(vc->vc_num);
-	struct fbcon_par *par = info->fbcon_par;
-
-	par->graphics = par->save_graphics;
-	if (info->fbops->fb_debug_leave)
-		info->fbops->fb_debug_leave(info);
-}
-
 static int fbcon_get_font(struct vc_data *vc, struct console_font *font, unsigned int vpitch)
 {
 	u8 *fontdata = vc->vc_font.data;
@@ -3214,8 +3192,6 @@ static const struct consw fb_con = {
 	.con_set_palette 	= fbcon_set_palette,
 	.con_invert_region 	= fbcon_invert_region,
 	.con_resize             = fbcon_resize,
-	.con_debug_enter	= fbcon_debug_enter,
-	.con_debug_leave	= fbcon_debug_leave,
 };
 
 static ssize_t rotate_store(struct device *device,
diff --git a/drivers/video/fbdev/core/fbcon.h b/drivers/video/fbdev/core/fbcon.h
index 44ea4ae4bba0..1cd10a7faab0 100644
--- a/drivers/video/fbdev/core/fbcon.h
+++ b/drivers/video/fbdev/core/fbcon.h
@@ -79,7 +79,6 @@ struct fbcon_par {
 	int    cursor_reset;
 	int    blank_state;
 	int    graphics;
-	int    save_graphics; /* for debug enter/leave */
 	bool   initialized;
 	int    rotate;
 	int    cur_rotate;
diff --git a/drivers/video/fbdev/core/fbmon.c b/drivers/video/fbdev/core/fbmon.c
index 0a65bef01e3c..07df7e98f8a3 100644
--- a/drivers/video/fbdev/core/fbmon.c
+++ b/drivers/video/fbdev/core/fbmon.c
@@ -32,11 +32,13 @@
 #include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/slab.h>
-#include <video/edid.h>
+#include <linux/string_choices.h>
+#include <linux/sysfb.h>
+
 #include <video/of_videomode.h>
 #include <video/videomode.h>
+
 #include "../edid.h"
-#include <linux/string_choices.h>
 
 /*
  * EDID parser
@@ -1504,7 +1506,7 @@ const unsigned char *fb_firmware_edid(struct device *device)
 		res = &dev->resource[PCI_ROM_RESOURCE];
 
 	if (res && res->flags & IORESOURCE_ROM_SHADOW)
-		edid = edid_info.dummy;
+		edid = sysfb_primary_display.edid.dummy;
 
 	return edid;
 }
diff --git a/drivers/video/fbdev/efifb.c b/drivers/video/fbdev/efifb.c
index 0e1bd3dba255..47ebc0107209 100644
--- a/drivers/video/fbdev/efifb.c
+++ b/drivers/video/fbdev/efifb.c
@@ -15,7 +15,7 @@
 #include <linux/fb.h>
 #include <linux/platform_device.h>
 #include <linux/printk.h>
-#include <linux/screen_info.h>
+#include <linux/sysfb.h>
 #include <video/vga.h>
 #include <asm/efi.h>
 #include <drm/drm_utils.h> /* For drm_get_panel_orientation_quirk */
@@ -345,6 +345,7 @@ ATTRIBUTE_GROUPS(efifb);
 
 static int efifb_probe(struct platform_device *dev)
 {
+	struct sysfb_display_info *dpy;
 	struct screen_info *si;
 	struct fb_info *info;
 	struct efifb_par *par;
@@ -360,10 +361,11 @@ static int efifb_probe(struct platform_device *dev)
 	 * driver. We get a copy of the attached screen_info, so that we can
 	 * modify its values without affecting later drivers.
 	 */
-	si = dev_get_platdata(&dev->dev);
-	if (!si)
+	dpy = dev_get_platdata(&dev->dev);
+	if (!dpy)
 		return -ENODEV;
-	si = devm_kmemdup(&dev->dev, si, sizeof(*si), GFP_KERNEL);
+
+	si = devm_kmemdup(&dev->dev, &dpy->screen, sizeof(*si), GFP_KERNEL);
 	if (!si)
 		return -ENOMEM;
 
diff --git a/drivers/video/fbdev/vesafb.c b/drivers/video/fbdev/vesafb.c
index f135033c22fb..f84f4db244bf 100644
--- a/drivers/video/fbdev/vesafb.c
+++ b/drivers/video/fbdev/vesafb.c
@@ -20,7 +20,7 @@
 #include <linux/ioport.h>
 #include <linux/init.h>
 #include <linux/platform_device.h>
-#include <linux/screen_info.h>
+#include <linux/sysfb.h>
 #include <linux/io.h>
 
 #include <video/vga.h>
@@ -243,6 +243,7 @@ static int vesafb_setup(char *options)
 
 static int vesafb_probe(struct platform_device *dev)
 {
+	struct sysfb_display_info *dpy;
 	struct screen_info *si;
 	struct fb_info *info;
 	struct vesafb_par *par;
@@ -257,10 +258,11 @@ static int vesafb_probe(struct platform_device *dev)
 	 * driver. We get a copy of the attached screen_info, so that we can
 	 * modify its values without affecting later drivers.
 	 */
-	si = dev_get_platdata(&dev->dev);
-	if (!si)
+	dpy = dev_get_platdata(&dev->dev);
+	if (!dpy)
 		return -ENODEV;
-	si = devm_kmemdup(&dev->dev, si, sizeof(*si), GFP_KERNEL);
+
+	si = devm_kmemdup(&dev->dev, &dpy->screen, sizeof(*si), GFP_KERNEL);
 	if (!si)
 		return -ENOMEM;
 
diff --git a/drivers/video/fbdev/vga16fb.c b/drivers/video/fbdev/vga16fb.c
index 6b81337a4909..22085d3668e8 100644
--- a/drivers/video/fbdev/vga16fb.c
+++ b/drivers/video/fbdev/vga16fb.c
@@ -21,7 +21,7 @@
 #include <linux/ioport.h>
 #include <linux/init.h>
 #include <linux/platform_device.h>
-#include <linux/screen_info.h>
+#include <linux/sysfb.h>
 
 #include <asm/io.h>
 #include <video/vga.h>
@@ -1305,15 +1305,17 @@ static const struct fb_ops vga16fb_ops = {
 
 static int vga16fb_probe(struct platform_device *dev)
 {
+	struct sysfb_display_info *dpy;
 	struct screen_info *si;
 	struct fb_info *info;
 	struct vga16fb_par *par;
 	int i;
 	int ret = 0;
 
-	si = dev_get_platdata(&dev->dev);
-	if (!si)
+	dpy = dev_get_platdata(&dev->dev);
+	if (!dpy)
 		return -ENODEV;
+	si = &dpy->screen;
 
 	ret = check_mode_supported(si);
 	if (ret)
diff --git a/drivers/video/screen_info_pci.c b/drivers/video/screen_info_pci.c
index 66bfc1d0a6dc..8f34d8a74f09 100644
--- a/drivers/video/screen_info_pci.c
+++ b/drivers/video/screen_info_pci.c
@@ -4,6 +4,7 @@
 #include <linux/printk.h>
 #include <linux/screen_info.h>
 #include <linux/string.h>
+#include <linux/sysfb.h>
 
 static struct pci_dev *screen_info_lfb_pdev;
 static size_t screen_info_lfb_bar;
@@ -26,7 +27,7 @@ static bool __screen_info_relocation_is_valid(const struct screen_info *si, stru
 
 void screen_info_apply_fixups(void)
 {
-	struct screen_info *si = &screen_info;
+	struct screen_info *si = &sysfb_primary_display.screen;
 
 	if (screen_info_lfb_pdev) {
 		struct resource *pr = &screen_info_lfb_pdev->resource[screen_info_lfb_bar];
@@ -75,7 +76,7 @@ static void screen_info_fixup_lfb(struct pci_dev *pdev)
 		.flags = IORESOURCE_MEM,
 	};
 	const struct resource *pr;
-	const struct screen_info *si = &screen_info;
+	const struct screen_info *si = &sysfb_primary_display.screen;
 
 	if (screen_info_lfb_pdev)
 		return; // already found
diff --git a/drivers/w1/slaves/w1_therm.c b/drivers/w1/slaves/w1_therm.c
index 9ccedb3264fb..832e3da94b20 100644
--- a/drivers/w1/slaves/w1_therm.c
+++ b/drivers/w1/slaves/w1_therm.c
@@ -1836,53 +1836,35 @@ static ssize_t alarms_store(struct device *device,
 	struct w1_slave *sl = dev_to_w1_slave(device);
 	struct therm_info info;
 	u8 new_config_register[3];	/* array of data to be written */
-	int temp, ret;
-	char *token = NULL;
+	long long temp;
+	int ret = 0;
 	s8 tl, th;	/* 1 byte per value + temp ring order */
-	char *p_args, *orig;
-
-	p_args = orig = kmalloc(size, GFP_KERNEL);
-	/* Safe string copys as buf is const */
-	if (!p_args) {
-		dev_warn(device,
-			"%s: error unable to allocate memory %d\n",
-			__func__, -ENOMEM);
-		return size;
-	}
-	strcpy(p_args, buf);
-
-	/* Split string using space char */
-	token = strsep(&p_args, " ");
-
-	if (!token)	{
-		dev_info(device,
-			"%s: error parsing args %d\n", __func__, -EINVAL);
-		goto free_m;
-	}
-
-	/* Convert 1st entry to int */
-	ret = kstrtoint (token, 10, &temp);
+	const char *p = buf;
+	char *endp;
+
+	temp = simple_strtoll(p, &endp, 10);
+	if (p == endp || *endp != ' ')
+		ret = -EINVAL;
+	else if (temp < INT_MIN || temp > INT_MAX)
+		ret = -ERANGE;
 	if (ret) {
 		dev_info(device,
 			"%s: error parsing args %d\n", __func__, ret);
-		goto free_m;
+		return size;
 	}
 
 	tl = int_to_short(temp);
 
-	/* Split string using space char */
-	token = strsep(&p_args, " ");
-	if (!token)	{
-		dev_info(device,
-			"%s: error parsing args %d\n", __func__, -EINVAL);
-		goto free_m;
-	}
-	/* Convert 2nd entry to int */
-	ret = kstrtoint (token, 10, &temp);
+	p = endp + 1;
+	temp = simple_strtoll(p, &endp, 10);
+	if (p == endp)
+		ret = -EINVAL;
+	else if (temp < INT_MIN || temp > INT_MAX)
+		ret = -ERANGE;
 	if (ret) {
 		dev_info(device,
 			"%s: error parsing args %d\n", __func__, ret);
-		goto free_m;
+		return size;
 	}
 
 	/* Prepare to cast to short by eliminating out of range values */
@@ -1905,7 +1887,7 @@ static ssize_t alarms_store(struct device *device,
 		dev_info(device,
 			"%s: error reading from the slave device %d\n",
 			__func__, ret);
-		goto free_m;
+		return size;
 	}
 
 	/* Write data in the device RAM */
@@ -1913,7 +1895,7 @@ static ssize_t alarms_store(struct device *device,
 		dev_info(device,
 			"%s: Device not supported by the driver %d\n",
 			__func__, -ENODEV);
-		goto free_m;
+		return size;
 	}
 
 	ret = SLAVE_SPECIFIC_FUNC(sl)->write_data(sl, new_config_register);
@@ -1922,10 +1904,6 @@ static ssize_t alarms_store(struct device *device,
 			"%s: error writing to the slave device %d\n",
 			__func__, ret);
 
-free_m:
-	/* free allocated memory */
-	kfree(orig);
-
 	return size;
 }
 
diff --git a/drivers/watchdog/hpwdt.c b/drivers/watchdog/hpwdt.c
index ae30e394d176..2a848c35c14d 100644
--- a/drivers/watchdog/hpwdt.c
+++ b/drivers/watchdog/hpwdt.c
@@ -12,6 +12,7 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/device.h>
+#include <linux/hex.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
diff --git a/fs/adfs/dir.c b/fs/adfs/dir.c
index 77fbd196008f..4f9dc276da6f 100644
--- a/fs/adfs/dir.c
+++ b/fs/adfs/dir.c
@@ -6,6 +6,7 @@
  *
  *  Common directory handling for ADFS
  */
+#include <linux/hex.h>
 #include <linux/slab.h>
 #include "adfs.h"
 
diff --git a/fs/binfmt_misc.c b/fs/binfmt_misc.c
index 8cb1a94339b8..2b772613a74c 100644
--- a/fs/binfmt_misc.c
+++ b/fs/binfmt_misc.c
@@ -12,6 +12,7 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/sched/mm.h>
 #include <linux/magic.h>
diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig
index 4438637c8900..4814cb233835 100644
--- a/fs/btrfs/Kconfig
+++ b/fs/btrfs/Kconfig
@@ -1,14 +1,12 @@
 # SPDX-License-Identifier: GPL-2.0
+# misc-next marker
 
 config BTRFS_FS
 	tristate "Btrfs filesystem support"
 	select BLK_CGROUP_PUNT_BIO
 	select CRC32
-	select CRYPTO
-	select CRYPTO_CRC32C
-	select CRYPTO_XXHASH
-	select CRYPTO_SHA256
-	select CRYPTO_BLAKE2B
+	select CRYPTO_LIB_BLAKE2B
+	select CRYPTO_LIB_SHA256
 	select ZLIB_INFLATE
 	select ZLIB_DEFLATE
 	select LZO_COMPRESS
@@ -18,6 +16,7 @@ config BTRFS_FS
 	select FS_IOMAP
 	select RAID6_PQ
 	select XOR_BLOCKS
+	select XXHASH
 	depends on PAGE_SIZE_LESS_THAN_256KB
 
 	help
diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c
index fa1d321a2fb8..e4d382d3a7ae 100644
--- a/fs/btrfs/bio.c
+++ b/fs/btrfs/bio.c
@@ -480,6 +480,8 @@ static void btrfs_clone_write_end_io(struct bio *bio)
 
 static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio)
 {
+	u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT;
+
 	if (!dev || !dev->bdev ||
 	    test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) ||
 	    (btrfs_op(bio) == BTRFS_MAP_WRITE &&
@@ -494,12 +496,13 @@ static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio)
 	 * For zone append writing, bi_sector must point the beginning of the
 	 * zone
 	 */
-	if (bio_op(bio) == REQ_OP_ZONE_APPEND) {
-		u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT;
+	if (btrfs_bio(bio)->can_use_append && btrfs_dev_is_sequential(dev, physical)) {
 		u64 zone_start = round_down(physical, dev->fs_info->zone_size);
 
 		ASSERT(btrfs_dev_is_sequential(dev, physical));
 		bio->bi_iter.bi_sector = zone_start >> SECTOR_SHIFT;
+		bio->bi_opf &= ~REQ_OP_WRITE;
+		bio->bi_opf |= REQ_OP_ZONE_APPEND;
 	}
 	btrfs_debug(dev->fs_info,
 	"%s: rw %d 0x%x, sector=%llu, dev=%lu (%s id %llu), size=%u",
@@ -747,7 +750,6 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num)
 	u64 logical = bio->bi_iter.bi_sector << SECTOR_SHIFT;
 	u64 length = bio->bi_iter.bi_size;
 	u64 map_length = length;
-	bool use_append = btrfs_use_zone_append(bbio);
 	struct btrfs_io_context *bioc = NULL;
 	struct btrfs_io_stripe smap;
 	blk_status_t status;
@@ -775,8 +777,10 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num)
 	if (bio_op(bio) == REQ_OP_WRITE && is_data_bbio(bbio))
 		bbio->orig_logical = logical;
 
+	bbio->can_use_append = btrfs_use_zone_append(bbio);
+
 	map_length = min(map_length, length);
-	if (use_append)
+	if (bbio->can_use_append)
 		map_length = btrfs_append_map_length(bbio, map_length);
 
 	if (map_length < length) {
@@ -805,11 +809,6 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num)
 	}
 
 	if (btrfs_op(bio) == BTRFS_MAP_WRITE) {
-		if (use_append) {
-			bio->bi_opf &= ~REQ_OP_WRITE;
-			bio->bi_opf |= REQ_OP_ZONE_APPEND;
-		}
-
 		if (is_data_bbio(bbio) && bioc && bioc->use_rst) {
 			/*
 			 * No locking for the list update, as we only add to
@@ -836,7 +835,7 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num)
 			status = errno_to_blk_status(ret);
 			if (status)
 				goto fail;
-		} else if (use_append ||
+		} else if (bbio->can_use_append ||
 			   (btrfs_is_zoned(fs_info) && inode &&
 			    inode->flags & BTRFS_INODE_NODATASUM)) {
 			ret = btrfs_alloc_dummy_sum(bbio);
diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h
index 1be74209f0b8..157cdfa2f78a 100644
--- a/fs/btrfs/bio.h
+++ b/fs/btrfs/bio.h
@@ -68,29 +68,33 @@ struct btrfs_bio {
 		struct btrfs_tree_parent_check parent_check;
 	};
 
+	/* For internal use in read end I/O handling */
+	struct work_struct end_io_work;
+
 	/* End I/O information supplied to btrfs_bio_alloc */
 	btrfs_bio_end_io_t end_io;
 	void *private;
 
-	/* For internal use in read end I/O handling */
-	unsigned int mirror_num;
 	atomic_t pending_ios;
-	struct work_struct end_io_work;
+	u16 mirror_num;
 
 	/* Save the first error status of split bio. */
 	blk_status_t status;
 
 	/* Use the commit root to look up csums (data read bio only). */
-	bool csum_search_commit_root;
+	bool csum_search_commit_root:1;
 
 	/*
 	 * Since scrub will reuse btree inode, we need this flag to distinguish
 	 * scrub bios.
 	 */
-	bool is_scrub;
+	bool is_scrub:1;
 
 	/* Whether the csum generation for data write is async. */
-	bool async_csum;
+	bool async_csum:1;
+
+	/* Whether the bio is written using zone append. */
+	bool can_use_append:1;
 
 	/*
 	 * This member must come last, bio_alloc_bioset will allocate enough
diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c
index 08b14449fabe..e417aba4c4c7 100644
--- a/fs/btrfs/block-group.c
+++ b/fs/btrfs/block-group.c
@@ -1057,8 +1057,7 @@ static int remove_block_group_item(struct btrfs_trans_handle *trans,
 	if (ret < 0)
 		return ret;
 
-	ret = btrfs_del_item(trans, root, path);
-	return ret;
+	return btrfs_del_item(trans, root, path);
 }
 
 int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
@@ -2266,7 +2265,7 @@ static int exclude_super_stripes(struct btrfs_block_group *cache)
 	return 0;
 }
 
-static struct btrfs_block_group *btrfs_create_block_group_cache(
+static struct btrfs_block_group *btrfs_create_block_group(
 		struct btrfs_fs_info *fs_info, u64 start)
 {
 	struct btrfs_block_group *cache;
@@ -2370,7 +2369,7 @@ static int read_one_block_group(struct btrfs_fs_info *info,
 
 	ASSERT(key->type == BTRFS_BLOCK_GROUP_ITEM_KEY);
 
-	cache = btrfs_create_block_group_cache(info, key->objectid);
+	cache = btrfs_create_block_group(info, key->objectid);
 	if (!cache)
 		return -ENOMEM;
 
@@ -2491,7 +2490,7 @@ static int fill_dummy_bgs(struct btrfs_fs_info *fs_info)
 		struct btrfs_block_group *bg;
 
 		map = rb_entry(node, struct btrfs_chunk_map, rb_node);
-		bg = btrfs_create_block_group_cache(fs_info, map->start);
+		bg = btrfs_create_block_group(fs_info, map->start);
 		if (!bg) {
 			ret = -ENOMEM;
 			break;
@@ -2886,7 +2885,7 @@ struct btrfs_block_group *btrfs_make_block_group(struct btrfs_trans_handle *tran
 
 	btrfs_set_log_full_commit(trans);
 
-	cache = btrfs_create_block_group_cache(fs_info, chunk_offset);
+	cache = btrfs_create_block_group(fs_info, chunk_offset);
 	if (!cache)
 		return ERR_PTR(-ENOMEM);
 
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
index 6b3357287b42..4323d4172c7b 100644
--- a/fs/btrfs/compression.c
+++ b/fs/btrfs/compression.c
@@ -21,7 +21,6 @@
 #include <linux/sched/mm.h>
 #include <linux/log2.h>
 #include <linux/shrinker.h>
-#include <crypto/hash.h>
 #include "misc.h"
 #include "ctree.h"
 #include "fs.h"
diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c
index a48b4befbee7..7267b2502665 100644
--- a/fs/btrfs/ctree.c
+++ b/fs/btrfs/ctree.c
@@ -249,6 +249,7 @@ int btrfs_copy_root(struct btrfs_trans_handle *trans,
 	int ret = 0;
 	int level;
 	struct btrfs_disk_key disk_key;
+	const bool is_reloc_root = (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID);
 	u64 reloc_src_root = 0;
 
 	WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) &&
@@ -262,7 +263,7 @@ int btrfs_copy_root(struct btrfs_trans_handle *trans,
 	else
 		btrfs_node_key(buf, &disk_key, 0);
 
-	if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID)
+	if (is_reloc_root)
 		reloc_src_root = btrfs_header_owner(buf);
 	cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid,
 				     &disk_key, level, buf->start, 0,
@@ -276,7 +277,7 @@ int btrfs_copy_root(struct btrfs_trans_handle *trans,
 	btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV);
 	btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN |
 				     BTRFS_HEADER_FLAG_RELOC);
-	if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID)
+	if (is_reloc_root)
 		btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC);
 	else
 		btrfs_set_header_owner(cow, new_root_objectid);
@@ -291,16 +292,9 @@ int btrfs_copy_root(struct btrfs_trans_handle *trans,
 		return ret;
 	}
 
-	if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
-		ret = btrfs_inc_ref(trans, root, cow, 1);
-		if (unlikely(ret))
-			btrfs_abort_transaction(trans, ret);
-	} else {
-		ret = btrfs_inc_ref(trans, root, cow, 0);
-		if (unlikely(ret))
-			btrfs_abort_transaction(trans, ret);
-	}
-	if (ret) {
+	ret = btrfs_inc_ref(trans, root, cow, is_reloc_root);
+	if (unlikely(ret)) {
+		btrfs_abort_transaction(trans, ret);
 		btrfs_tree_unlock(cow);
 		free_extent_buffer(cow);
 		return ret;
@@ -362,6 +356,7 @@ static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans,
 	u64 owner;
 	u64 flags;
 	int ret;
+	const bool is_reloc_root = (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID);
 
 	/*
 	 * Backrefs update rules:
@@ -397,8 +392,7 @@ static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans,
 		}
 	} else {
 		refs = 1;
-		if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID ||
-		    btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
+		if (is_reloc_root || btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
 			flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
 		else
 			flags = 0;
@@ -417,18 +411,17 @@ static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans,
 	}
 
 	if (refs > 1) {
-		if ((owner == btrfs_root_id(root) ||
-		     btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID) &&
+		if ((owner == btrfs_root_id(root) || is_reloc_root) &&
 		    !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) {
-			ret = btrfs_inc_ref(trans, root, buf, 1);
+			ret = btrfs_inc_ref(trans, root, buf, true);
 			if (ret)
 				return ret;
 
-			if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID) {
-				ret = btrfs_dec_ref(trans, root, buf, 0);
+			if (is_reloc_root) {
+				ret = btrfs_dec_ref(trans, root, buf, false);
 				if (ret)
 					return ret;
-				ret = btrfs_inc_ref(trans, root, cow, 1);
+				ret = btrfs_inc_ref(trans, root, cow, true);
 				if (ret)
 					return ret;
 			}
@@ -437,23 +430,16 @@ static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans,
 			if (ret)
 				return ret;
 		} else {
-
-			if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID)
-				ret = btrfs_inc_ref(trans, root, cow, 1);
-			else
-				ret = btrfs_inc_ref(trans, root, cow, 0);
+			ret = btrfs_inc_ref(trans, root, cow, is_reloc_root);
 			if (ret)
 				return ret;
 		}
 	} else {
 		if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
-			if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID)
-				ret = btrfs_inc_ref(trans, root, cow, 1);
-			else
-				ret = btrfs_inc_ref(trans, root, cow, 0);
+			ret = btrfs_inc_ref(trans, root, cow, is_reloc_root);
 			if (ret)
 				return ret;
-			ret = btrfs_dec_ref(trans, root, buf, 1);
+			ret = btrfs_dec_ref(trans, root, buf, true);
 			if (ret)
 				return ret;
 		}
@@ -4016,8 +4002,7 @@ int btrfs_split_item(struct btrfs_trans_handle *trans,
 	if (ret)
 		return ret;
 
-	ret = split_item(trans, path, new_key, split_offset);
-	return ret;
+	return split_item(trans, path, new_key, split_offset);
 }
 
 /*
diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
index 692370fc07b2..6de7ad191e04 100644
--- a/fs/btrfs/ctree.h
+++ b/fs/btrfs/ctree.h
@@ -85,6 +85,14 @@ struct btrfs_path {
 #define BTRFS_PATH_AUTO_FREE(path_name)					\
 	struct btrfs_path *path_name __free(btrfs_free_path) = NULL
 
+/*
+ * This defines an on-stack path that will be auto released when exiting the scope.
+ *
+ * It is compatible with any existing manual btrfs_release_path() calls.
+ */
+#define BTRFS_PATH_AUTO_RELEASE(path_name)					\
+	struct btrfs_path path_name __free(btrfs_release_path) = { 0 }
+
 /*
  * The state of btrfs root
  */
@@ -601,6 +609,7 @@ void btrfs_release_path(struct btrfs_path *p);
 struct btrfs_path *btrfs_alloc_path(void);
 void btrfs_free_path(struct btrfs_path *p);
 DEFINE_FREE(btrfs_free_path, struct btrfs_path *, btrfs_free_path(_T))
+DEFINE_FREE(btrfs_release_path, struct btrfs_path, btrfs_release_path(&_T))
 
 int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root,
 		   struct btrfs_path *path, int slot, int nr);
diff --git a/fs/btrfs/defrag.c b/fs/btrfs/defrag.c
index b81e224d4a27..bcc6656ad034 100644
--- a/fs/btrfs/defrag.c
+++ b/fs/btrfs/defrag.c
@@ -609,7 +609,7 @@ static struct extent_map *defrag_get_extent(struct btrfs_inode *inode,
 {
 	struct btrfs_root *root = inode->root;
 	struct btrfs_file_extent_item *fi;
-	struct btrfs_path path = { 0 };
+	BTRFS_PATH_AUTO_RELEASE(path);
 	struct extent_map *em;
 	struct btrfs_key key;
 	u64 ino = btrfs_ino(inode);
@@ -720,16 +720,13 @@ static struct extent_map *defrag_get_extent(struct btrfs_inode *inode,
 		if (ret > 0)
 			goto not_found;
 	}
-	btrfs_release_path(&path);
 	return em;
 
 not_found:
-	btrfs_release_path(&path);
 	btrfs_free_extent_map(em);
 	return NULL;
 
 err:
-	btrfs_release_path(&path);
 	btrfs_free_extent_map(em);
 	return ERR_PTR(ret);
 }
diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c
index ce6e9f8812e0..2286bee2c6d3 100644
--- a/fs/btrfs/delayed-inode.c
+++ b/fs/btrfs/delayed-inode.c
@@ -152,37 +152,39 @@ static struct btrfs_delayed_node *btrfs_get_or_create_delayed_node(
 		return ERR_PTR(-ENOMEM);
 	btrfs_init_delayed_node(node, root, ino);
 
+	/* Cached in the inode and can be accessed. */
+	refcount_set(&node->refs, 2);
+	btrfs_delayed_node_ref_tracker_alloc(node, tracker, GFP_NOFS);
+	btrfs_delayed_node_ref_tracker_alloc(node, &node->inode_cache_tracker, GFP_NOFS);
+
 	/* Allocate and reserve the slot, from now it can return a NULL from xa_load(). */
 	ret = xa_reserve(&root->delayed_nodes, ino, GFP_NOFS);
-	if (ret == -ENOMEM) {
-		btrfs_delayed_node_ref_tracker_dir_exit(node);
-		kmem_cache_free(delayed_node_cache, node);
-		return ERR_PTR(-ENOMEM);
-	}
+	if (ret == -ENOMEM)
+		goto cleanup;
+
 	xa_lock(&root->delayed_nodes);
 	ptr = xa_load(&root->delayed_nodes, ino);
 	if (ptr) {
 		/* Somebody inserted it, go back and read it. */
 		xa_unlock(&root->delayed_nodes);
-		btrfs_delayed_node_ref_tracker_dir_exit(node);
-		kmem_cache_free(delayed_node_cache, node);
-		node = NULL;
-		goto again;
+		goto cleanup;
 	}
 	ptr = __xa_store(&root->delayed_nodes, ino, node, GFP_ATOMIC);
 	ASSERT(xa_err(ptr) != -EINVAL);
 	ASSERT(xa_err(ptr) != -ENOMEM);
 	ASSERT(ptr == NULL);
-
-	/* Cached in the inode and can be accessed. */
-	refcount_set(&node->refs, 2);
-	btrfs_delayed_node_ref_tracker_alloc(node, tracker, GFP_ATOMIC);
-	btrfs_delayed_node_ref_tracker_alloc(node, &node->inode_cache_tracker, GFP_ATOMIC);
-
 	btrfs_inode->delayed_node = node;
 	xa_unlock(&root->delayed_nodes);
 
 	return node;
+cleanup:
+	btrfs_delayed_node_ref_tracker_free(node, tracker);
+	btrfs_delayed_node_ref_tracker_free(node, &node->inode_cache_tracker);
+	btrfs_delayed_node_ref_tracker_dir_exit(node);
+	kmem_cache_free(delayed_node_cache, node);
+	if (ret)
+		return ERR_PTR(ret);
+	goto again;
 }
 
 /*
@@ -1135,8 +1137,8 @@ __btrfs_commit_inode_delayed_items(struct btrfs_trans_handle *trans,
 	ret = btrfs_record_root_in_trans(trans, node->root);
 	if (ret)
 		return ret;
-	ret = btrfs_update_delayed_inode(trans, node->root, path, node);
-	return ret;
+
+	return btrfs_update_delayed_inode(trans, node->root, path, node);
 }
 
 /*
diff --git a/fs/btrfs/direct-io.c b/fs/btrfs/direct-io.c
index 07e19e88ba4b..9a63200d7a53 100644
--- a/fs/btrfs/direct-io.c
+++ b/fs/btrfs/direct-io.c
@@ -763,7 +763,7 @@ static ssize_t btrfs_dio_read(struct kiocb *iocb, struct iov_iter *iter,
 	struct btrfs_dio_data data = { 0 };
 
 	return iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dio_ops,
-			    IOMAP_DIO_PARTIAL, &data, done_before);
+			    IOMAP_DIO_PARTIAL | IOMAP_DIO_FSBLOCK_ALIGNED, &data, done_before);
 }
 
 static struct iomap_dio *btrfs_dio_write(struct kiocb *iocb, struct iov_iter *iter,
@@ -772,7 +772,7 @@ static struct iomap_dio *btrfs_dio_write(struct kiocb *iocb, struct iov_iter *it
 	struct btrfs_dio_data data = { 0 };
 
 	return __iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dio_ops,
-			    IOMAP_DIO_PARTIAL, &data, done_before);
+			    IOMAP_DIO_PARTIAL | IOMAP_DIO_FSBLOCK_ALIGNED, &data, done_before);
 }
 
 static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info,
@@ -785,19 +785,6 @@ static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info,
 
 	if (iov_iter_alignment(iter) & blocksize_mask)
 		return -EINVAL;
-
-	/*
-	 * For bs > ps support, we heavily rely on large folios to make sure no
-	 * block will cross large folio boundaries.
-	 *
-	 * But memory provided by direct IO is only virtually contiguous, not
-	 * physically contiguous, and will break the btrfs' large folio requirement.
-	 *
-	 * So for bs > ps support, all direct IOs should fallback to buffered ones.
-	 */
-	if (fs_info->sectorsize > PAGE_SIZE)
-		return -EINVAL;
-
 	return 0;
 }
 
@@ -814,6 +801,8 @@ ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from)
 	ssize_t ret;
 	unsigned int ilock_flags = 0;
 	struct iomap_dio *dio;
+	const u64 data_profile = btrfs_data_alloc_profile(fs_info) &
+				 BTRFS_BLOCK_GROUP_PROFILE_MASK;
 
 	if (iocb->ki_flags & IOCB_NOWAIT)
 		ilock_flags |= BTRFS_ILOCK_TRY;
@@ -827,6 +816,16 @@ ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from)
 	if (iocb->ki_pos + iov_iter_count(from) <= i_size_read(inode) && IS_NOSEC(inode))
 		ilock_flags |= BTRFS_ILOCK_SHARED;
 
+	/*
+	 * If our data profile has duplication (either extra mirrors or RAID56),
+	 * we can not trust the direct IO buffer, the content may change during
+	 * writeback and cause different contents written to different mirrors.
+	 *
+	 * Thus only RAID0 and SINGLE can go true zero-copy direct IO.
+	 */
+	if (data_profile != BTRFS_BLOCK_GROUP_RAID0 && data_profile != 0)
+		goto buffered;
+
 relock:
 	ret = btrfs_inode_lock(BTRFS_I(inode), ilock_flags);
 	if (ret < 0)
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 89149fac804c..e5535bdc5b0c 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -18,7 +18,6 @@
 #include <linux/crc32c.h>
 #include <linux/sched/mm.h>
 #include <linux/unaligned.h>
-#include <crypto/hash.h>
 #include "ctree.h"
 #include "disk-io.h"
 #include "transaction.h"
@@ -62,12 +61,6 @@
 static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info);
 static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info);
 
-static void btrfs_free_csum_hash(struct btrfs_fs_info *fs_info)
-{
-	if (fs_info->csum_shash)
-		crypto_free_shash(fs_info->csum_shash);
-}
-
 /*
  * Compute the csum of a btree block and store the result to provided buffer.
  */
@@ -76,12 +69,11 @@ static void csum_tree_block(struct extent_buffer *buf, u8 *result)
 	struct btrfs_fs_info *fs_info = buf->fs_info;
 	int num_pages;
 	u32 first_page_part;
-	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
+	struct btrfs_csum_ctx csum;
 	char *kaddr;
 	int i;
 
-	shash->tfm = fs_info->csum_shash;
-	crypto_shash_init(shash);
+	btrfs_csum_init(&csum, fs_info->csum_type);
 
 	if (buf->addr) {
 		/* Pages are contiguous, handle them as a big one. */
@@ -94,21 +86,21 @@ static void csum_tree_block(struct extent_buffer *buf, u8 *result)
 		num_pages = num_extent_pages(buf);
 	}
 
-	crypto_shash_update(shash, kaddr + BTRFS_CSUM_SIZE,
-			    first_page_part - BTRFS_CSUM_SIZE);
+	btrfs_csum_update(&csum, kaddr + BTRFS_CSUM_SIZE,
+			  first_page_part - BTRFS_CSUM_SIZE);
 
 	/*
 	 * Multiple single-page folios case would reach here.
 	 *
 	 * nodesize <= PAGE_SIZE and large folio all handled by above
-	 * crypto_shash_update() already.
+	 * btrfs_csum_update() already.
 	 */
 	for (i = 1; i < num_pages && INLINE_EXTENT_BUFFER_PAGES > 1; i++) {
 		kaddr = folio_address(buf->folios[i]);
-		crypto_shash_update(shash, kaddr, PAGE_SIZE);
+		btrfs_csum_update(&csum, kaddr, PAGE_SIZE);
 	}
 	memset(result, 0, BTRFS_CSUM_SIZE);
-	crypto_shash_final(shash, result);
+	btrfs_csum_final(&csum, result);
 }
 
 /*
@@ -160,18 +152,15 @@ static bool btrfs_supported_super_csum(u16 csum_type)
 int btrfs_check_super_csum(struct btrfs_fs_info *fs_info,
 			   const struct btrfs_super_block *disk_sb)
 {
-	char result[BTRFS_CSUM_SIZE];
-	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
-
-	shash->tfm = fs_info->csum_shash;
+	u8 result[BTRFS_CSUM_SIZE];
 
 	/*
 	 * The super_block structure does not span the whole
 	 * BTRFS_SUPER_INFO_SIZE range, we expect that the unused space is
 	 * filled with zeros and is included in the checksum.
 	 */
-	crypto_shash_digest(shash, (const u8 *)disk_sb + BTRFS_CSUM_SIZE,
-			    BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, result);
+	btrfs_csum(fs_info->csum_type, (const u8 *)disk_sb + BTRFS_CSUM_SIZE,
+		   BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, result);
 
 	if (memcmp(disk_sb->csum, result, fs_info->csum_size))
 		return 1;
@@ -186,7 +175,6 @@ static int btrfs_repair_eb_io_failure(const struct extent_buffer *eb,
 	const u32 step = min(fs_info->nodesize, PAGE_SIZE);
 	const u32 nr_steps = eb->len / step;
 	phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE];
-	int ret = 0;
 
 	if (sb_rdonly(fs_info->sb))
 		return -EROFS;
@@ -208,9 +196,8 @@ static int btrfs_repair_eb_io_failure(const struct extent_buffer *eb,
 		paddrs[i] = page_to_phys(&folio->page) + offset_in_page(eb->start);
 	}
 
-	ret = btrfs_repair_io_failure(fs_info, 0, eb->start, eb->len, eb->start,
-				      paddrs, step, mirror_num);
-	return ret;
+	return btrfs_repair_io_failure(fs_info, 0, eb->start, eb->len,
+				       eb->start, paddrs, step, mirror_num);
 }
 
 /*
@@ -837,7 +824,6 @@ struct btrfs_root *btrfs_create_tree(struct btrfs_trans_handle *trans,
 	struct extent_buffer *leaf;
 	struct btrfs_root *tree_root = fs_info->tree_root;
 	struct btrfs_root *root;
-	struct btrfs_key key;
 	unsigned int nofs_flag;
 	int ret = 0;
 
@@ -886,10 +872,7 @@ struct btrfs_root *btrfs_create_tree(struct btrfs_trans_handle *trans,
 
 	btrfs_tree_unlock(leaf);
 
-	key.objectid = objectid;
-	key.type = BTRFS_ROOT_ITEM_KEY;
-	key.offset = 0;
-	ret = btrfs_insert_root(trans, tree_root, &key, &root->root_item);
+	ret = btrfs_insert_root(trans, tree_root, &root->root_key, &root->root_item);
 	if (ret)
 		goto fail;
 
@@ -1251,7 +1234,6 @@ void btrfs_free_fs_info(struct btrfs_fs_info *fs_info)
 		ASSERT(percpu_counter_sum_positive(em_counter) == 0);
 	percpu_counter_destroy(em_counter);
 	percpu_counter_destroy(&fs_info->dev_replace.bio_counter);
-	btrfs_free_csum_hash(fs_info);
 	btrfs_free_stripe_hash_table(fs_info);
 	btrfs_free_ref_cache(fs_info);
 	kfree(fs_info->balance_ctl);
@@ -2005,21 +1987,8 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info)
 	return 0;
 }
 
-static int btrfs_init_csum_hash(struct btrfs_fs_info *fs_info, u16 csum_type)
+static void btrfs_init_csum_hash(struct btrfs_fs_info *fs_info, u16 csum_type)
 {
-	struct crypto_shash *csum_shash;
-	const char *csum_driver = btrfs_super_csum_driver(csum_type);
-
-	csum_shash = crypto_alloc_shash(csum_driver, 0, 0);
-
-	if (IS_ERR(csum_shash)) {
-		btrfs_err(fs_info, "error allocating %s hash for checksum",
-			  csum_driver);
-		return PTR_ERR(csum_shash);
-	}
-
-	fs_info->csum_shash = csum_shash;
-
 	/* Check if the checksum implementation is a fast accelerated one. */
 	switch (csum_type) {
 	case BTRFS_CSUM_TYPE_CRC32:
@@ -2033,10 +2002,8 @@ static int btrfs_init_csum_hash(struct btrfs_fs_info *fs_info, u16 csum_type)
 		break;
 	}
 
-	btrfs_info(fs_info, "using %s (%s) checksum algorithm",
-			btrfs_super_csum_name(csum_type),
-			crypto_shash_driver_name(csum_shash));
-	return 0;
+	btrfs_info(fs_info, "using %s checksum algorithm",
+		   btrfs_super_csum_name(csum_type));
 }
 
 static int btrfs_replay_log(struct btrfs_fs_info *fs_info,
@@ -2194,11 +2161,10 @@ static int load_global_roots(struct btrfs_root *tree_root)
 		return ret;
 	if (!btrfs_fs_compat_ro(tree_root->fs_info, FREE_SPACE_TREE))
 		return ret;
-	ret = load_global_roots_objectid(tree_root, path,
-					 BTRFS_FREE_SPACE_TREE_OBJECTID,
-					 "free space");
 
-	return ret;
+	return load_global_roots_objectid(tree_root, path,
+					  BTRFS_FREE_SPACE_TREE_OBJECTID,
+					  "free space");
 }
 
 static int btrfs_read_roots(struct btrfs_fs_info *fs_info)
@@ -3314,12 +3280,9 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
 	}
 
 	fs_info->csum_size = btrfs_super_csum_size(disk_super);
+	fs_info->csum_type = csum_type;
 
-	ret = btrfs_init_csum_hash(fs_info, csum_type);
-	if (ret) {
-		btrfs_release_disk_super(disk_super);
-		goto fail_alloc;
-	}
+	btrfs_init_csum_hash(fs_info, csum_type);
 
 	/*
 	 * We want to check superblock checksum, the type is stored inside.
@@ -3713,7 +3676,6 @@ static int write_dev_supers(struct btrfs_device *device,
 {
 	struct btrfs_fs_info *fs_info = device->fs_info;
 	struct address_space *mapping = device->bdev->bd_mapping;
-	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
 	int i;
 	int ret;
 	u64 bytenr, bytenr_orig;
@@ -3723,8 +3685,6 @@ static int write_dev_supers(struct btrfs_device *device,
 	if (max_mirrors == 0)
 		max_mirrors = BTRFS_SUPER_MIRROR_MAX;
 
-	shash->tfm = fs_info->csum_shash;
-
 	for (i = 0; i < max_mirrors; i++) {
 		struct folio *folio;
 		struct bio *bio;
@@ -3748,9 +3708,8 @@ static int write_dev_supers(struct btrfs_device *device,
 
 		btrfs_set_super_bytenr(sb, bytenr_orig);
 
-		crypto_shash_digest(shash, (const char *)sb + BTRFS_CSUM_SIZE,
-				    BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE,
-				    sb->csum);
+		btrfs_csum(fs_info->csum_type, (const u8 *)sb + BTRFS_CSUM_SIZE,
+			   BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, sb->csum);
 
 		folio = __filemap_get_folio(mapping, bytenr >> PAGE_SHIFT,
 					    FGP_LOCK | FGP_ACCESSED | FGP_CREAT,
diff --git a/fs/btrfs/extent-io-tree.c b/fs/btrfs/extent-io-tree.c
index bb2ca1c9c7b0..d0dd50f7d279 100644
--- a/fs/btrfs/extent-io-tree.c
+++ b/fs/btrfs/extent-io-tree.c
@@ -187,8 +187,6 @@ static int add_extent_changeset(struct extent_state *state, u32 bits,
 				 struct extent_changeset *changeset,
 				 int set)
 {
-	int ret;
-
 	if (!changeset)
 		return 0;
 	if (set && (state->state & bits) == bits)
@@ -196,9 +194,8 @@ static int add_extent_changeset(struct extent_state *state, u32 bits,
 	if (!set && (state->state & bits) == 0)
 		return 0;
 	changeset->bytes_changed += state->end - state->start + 1;
-	ret = ulist_add(&changeset->range_changed, state->start, state->end,
-			GFP_ATOMIC);
-	return ret;
+
+	return ulist_add(&changeset->range_changed, state->start, state->end, GFP_ATOMIC);
 }
 
 static inline struct extent_state *next_state(struct extent_state *state)
diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c
index e4cae34620d1..1dcd69fe97ed 100644
--- a/fs/btrfs/extent-tree.c
+++ b/fs/btrfs/extent-tree.c
@@ -2559,7 +2559,6 @@ static u64 get_alloc_profile_by_root(struct btrfs_root *root, int data)
 {
 	struct btrfs_fs_info *fs_info = root->fs_info;
 	u64 flags;
-	u64 ret;
 
 	if (data)
 		flags = BTRFS_BLOCK_GROUP_DATA;
@@ -2568,8 +2567,7 @@ static u64 get_alloc_profile_by_root(struct btrfs_root *root, int data)
 	else
 		flags = BTRFS_BLOCK_GROUP_METADATA;
 
-	ret = btrfs_get_alloc_profile(fs_info, flags);
-	return ret;
+	return btrfs_get_alloc_profile(fs_info, flags);
 }
 
 static u64 first_logical_byte(struct btrfs_fs_info *fs_info)
@@ -4191,10 +4189,8 @@ static int find_free_extent_update_loop(struct btrfs_fs_info *fs_info,
 			else
 				trans = btrfs_join_transaction(root);
 
-			if (IS_ERR(trans)) {
-				ret = PTR_ERR(trans);
-				return ret;
-			}
+			if (IS_ERR(trans))
+				return PTR_ERR(trans);
 
 			ret = btrfs_chunk_alloc(trans, space_info, ffe_ctl->flags,
 						CHUNK_ALLOC_FORCE_FOR_EXTENT);
@@ -4288,36 +4284,43 @@ static int prepare_allocation_zoned(struct btrfs_fs_info *fs_info,
 				    struct find_free_extent_ctl *ffe_ctl,
 				    struct btrfs_space_info *space_info)
 {
+	struct btrfs_block_group *block_group;
+
 	if (ffe_ctl->for_treelog) {
 		spin_lock(&fs_info->treelog_bg_lock);
 		if (fs_info->treelog_bg)
 			ffe_ctl->hint_byte = fs_info->treelog_bg;
 		spin_unlock(&fs_info->treelog_bg_lock);
-	} else if (ffe_ctl->for_data_reloc) {
+		return 0;
+	}
+
+	if (ffe_ctl->for_data_reloc) {
 		spin_lock(&fs_info->relocation_bg_lock);
 		if (fs_info->data_reloc_bg)
 			ffe_ctl->hint_byte = fs_info->data_reloc_bg;
 		spin_unlock(&fs_info->relocation_bg_lock);
-	} else if (ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA) {
-		struct btrfs_block_group *block_group;
+		return 0;
+	}
 
-		spin_lock(&fs_info->zone_active_bgs_lock);
-		list_for_each_entry(block_group, &fs_info->zone_active_bgs, active_bg_list) {
-			/*
-			 * No lock is OK here because avail is monotonically
-			 * decreasing, and this is just a hint.
-			 */
-			u64 avail = block_group->zone_capacity - block_group->alloc_offset;
+	if (!(ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA))
+		return 0;
 
-			if (block_group_bits(block_group, ffe_ctl->flags) &&
-			    block_group->space_info == space_info &&
-			    avail >= ffe_ctl->num_bytes) {
-				ffe_ctl->hint_byte = block_group->start;
-				break;
-			}
+	spin_lock(&fs_info->zone_active_bgs_lock);
+	list_for_each_entry(block_group, &fs_info->zone_active_bgs, active_bg_list) {
+		/*
+		 * No lock is OK here because avail is monotonically
+		 * decreasing, and this is just a hint.
+		 */
+		u64 avail = block_group->zone_capacity - block_group->alloc_offset;
+
+		if (block_group_bits(block_group, ffe_ctl->flags) &&
+		    block_group->space_info == space_info &&
+		    avail >= ffe_ctl->num_bytes) {
+			ffe_ctl->hint_byte = block_group->start;
+			break;
 		}
-		spin_unlock(&fs_info->zone_active_bgs_lock);
 	}
+	spin_unlock(&fs_info->zone_active_bgs_lock);
 
 	return 0;
 }
@@ -5263,7 +5266,6 @@ struct walk_control {
  * @root:	the root we are currently deleting
  * @wc:		the walk control for this deletion
  * @eb:		the parent eb that we're currently visiting
- * @refs:	the number of refs for wc->level - 1
  * @flags:	the flags for wc->level - 1
  * @slot:	the slot in the eb that we're currently checking
  *
@@ -5458,12 +5460,12 @@ static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
 	/* wc->stage == UPDATE_BACKREF */
 	if (!(wc->flags[level] & flag)) {
 		ASSERT(path->locks[level]);
-		ret = btrfs_inc_ref(trans, root, eb, 1);
+		ret = btrfs_inc_ref(trans, root, eb, true);
 		if (unlikely(ret)) {
 			btrfs_abort_transaction(trans, ret);
 			return ret;
 		}
-		ret = btrfs_dec_ref(trans, root, eb, 0);
+		ret = btrfs_dec_ref(trans, root, eb, false);
 		if (unlikely(ret)) {
 			btrfs_abort_transaction(trans, ret);
 			return ret;
@@ -5864,18 +5866,12 @@ static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
 
 	if (wc->refs[level] == 1) {
 		if (level == 0) {
-			if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
-				ret = btrfs_dec_ref(trans, root, eb, 1);
-				if (ret) {
-					btrfs_abort_transaction(trans, ret);
-					return ret;
-				}
-			} else {
-				ret = btrfs_dec_ref(trans, root, eb, 0);
-				if (unlikely(ret)) {
-					btrfs_abort_transaction(trans, ret);
-					return ret;
-				}
+			const bool full_backref = (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF);
+
+			ret = btrfs_dec_ref(trans, root, eb, full_backref);
+			if (unlikely(ret)) {
+				btrfs_abort_transaction(trans, ret);
+				return ret;
 			}
 			if (btrfs_is_fstree(btrfs_root_id(root))) {
 				ret = btrfs_qgroup_trace_leaf_items(trans, eb);
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 629fd5af4286..bbc55873cb16 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -998,11 +998,17 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached,
 	u64 start = folio_pos(folio);
 	const u64 end = start + folio_size(folio) - 1;
 	u64 extent_offset;
+	u64 locked_end;
 	u64 last_byte = i_size_read(inode);
 	struct extent_map *em;
 	int ret = 0;
 	const size_t blocksize = fs_info->sectorsize;
 
+	if (bio_ctrl->ractl)
+		locked_end = readahead_pos(bio_ctrl->ractl) + readahead_length(bio_ctrl->ractl) - 1;
+	else
+		locked_end = end;
+
 	ret = set_folio_extent_mapped(folio);
 	if (ret < 0) {
 		folio_unlock(folio);
@@ -1036,7 +1042,14 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached,
 			end_folio_read(folio, true, cur, blocksize);
 			continue;
 		}
-		em = get_extent_map(BTRFS_I(inode), folio, cur, end - cur + 1, em_cached);
+		/*
+		 * Search extent map for the whole locked range.
+		 * This will allow btrfs_get_extent() to return a larger hole
+		 * when possible.
+		 * This can reduce duplicated btrfs_get_extent() calls for large
+		 * holes.
+		 */
+		em = get_extent_map(BTRFS_I(inode), folio, cur, locked_end - cur + 1, em_cached);
 		if (IS_ERR(em)) {
 			end_folio_read(folio, false, cur, end + 1 - cur);
 			return PTR_ERR(em);
@@ -1426,8 +1439,9 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
 	u64 delalloc_start = page_start;
 	u64 delalloc_end = page_end;
 	u64 delalloc_to_write = 0;
+	unsigned int start_bit;
+	unsigned int end_bit;
 	int ret = 0;
-	int bit;
 
 	/* Save the dirty bitmap as our submission bitmap will be a subset of it. */
 	if (btrfs_is_subpage(fs_info, folio)) {
@@ -1437,10 +1451,12 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
 		bio_ctrl->submit_bitmap = 1;
 	}
 
-	for_each_set_bit(bit, &bio_ctrl->submit_bitmap, blocks_per_folio) {
-		u64 start = page_start + (bit << fs_info->sectorsize_bits);
+	for_each_set_bitrange(start_bit, end_bit, &bio_ctrl->submit_bitmap,
+			      blocks_per_folio) {
+		u64 start = page_start + (start_bit << fs_info->sectorsize_bits);
+		u32 len = (end_bit - start_bit) << fs_info->sectorsize_bits;
 
-		btrfs_folio_set_lock(fs_info, folio, start, fs_info->sectorsize);
+		btrfs_folio_set_lock(fs_info, folio, start, len);
 	}
 
 	/* Lock all (subpage) delalloc ranges inside the folio first. */
@@ -1557,10 +1573,13 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
 				fs_info->sectorsize_bits,
 				blocks_per_folio);
 
-		for_each_set_bit(bit, &bio_ctrl->submit_bitmap, bitmap_size)
-			btrfs_mark_ordered_io_finished(inode, folio,
-				page_start + (bit << fs_info->sectorsize_bits),
-				fs_info->sectorsize, false);
+		for_each_set_bitrange(start_bit, end_bit, &bio_ctrl->submit_bitmap,
+				      bitmap_size) {
+			u64 start = page_start + (start_bit << fs_info->sectorsize_bits);
+			u32 len = (end_bit - start_bit) << fs_info->sectorsize_bits;
+
+			btrfs_mark_ordered_io_finished(inode, folio, start, len, false);
+		}
 		return ret;
 	}
 out:
@@ -1598,7 +1617,7 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
 
 /*
  * Return 0 if we have submitted or queued the sector for submission.
- * Return <0 for critical errors, and the sector will have its dirty flag cleared.
+ * Return <0 for critical errors, and the involved sector will be cleaned up.
  *
  * Caller should make sure filepos < i_size and handle filepos >= i_size case.
  */
@@ -1622,6 +1641,13 @@ static int submit_one_sector(struct btrfs_inode *inode,
 
 	em = btrfs_get_extent(inode, NULL, filepos, sectorsize);
 	if (IS_ERR(em)) {
+		/*
+		 * bio_ctrl may contain a bio crossing several folios.
+		 * Submit it immediately so that the bio has a chance
+		 * to finish normally, other than marked as error.
+		 */
+		submit_one_bio(bio_ctrl);
+
 		/*
 		 * When submission failed, we should still clear the folio dirty.
 		 * Or the folio will be written back again but without any
@@ -1630,6 +1656,13 @@ static int submit_one_sector(struct btrfs_inode *inode,
 		btrfs_folio_clear_dirty(fs_info, folio, filepos, sectorsize);
 		btrfs_folio_set_writeback(fs_info, folio, filepos, sectorsize);
 		btrfs_folio_clear_writeback(fs_info, folio, filepos, sectorsize);
+
+		/*
+		 * Since there is no bio submitted to finish the ordered
+		 * extent, we have to manually finish this sector.
+		 */
+		btrfs_mark_ordered_io_finished(inode, folio, filepos,
+					       fs_info->sectorsize, false);
 		return PTR_ERR(em);
 	}
 
@@ -1714,8 +1747,8 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode,
 		return ret;
 	}
 
-	for (cur = start; cur < end; cur += fs_info->sectorsize)
-		set_bit((cur - folio_start) >> fs_info->sectorsize_bits, &range_bitmap);
+	bitmap_set(&range_bitmap, (start - folio_pos(folio)) >> fs_info->sectorsize_bits,
+		   len >> fs_info->sectorsize_bits);
 	bitmap_and(&bio_ctrl->submit_bitmap, &bio_ctrl->submit_bitmap, &range_bitmap,
 		   blocks_per_folio);
 
@@ -1728,7 +1761,7 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode,
 			struct btrfs_ordered_extent *ordered;
 
 			ordered = btrfs_lookup_first_ordered_range(inode, cur,
-								   folio_end - cur);
+								   fs_info->sectorsize);
 			/*
 			 * We have just run delalloc before getting here, so
 			 * there must be an ordered extent.
@@ -1742,7 +1775,7 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode,
 			btrfs_put_ordered_extent(ordered);
 
 			btrfs_mark_ordered_io_finished(inode, folio, cur,
-						       end - cur, true);
+						       fs_info->sectorsize, true);
 			/*
 			 * This range is beyond i_size, thus we don't need to
 			 * bother writing back.
@@ -1751,24 +1784,11 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode,
 			 * writeback the sectors with subpage dirty bits,
 			 * causing writeback without ordered extent.
 			 */
-			btrfs_folio_clear_dirty(fs_info, folio, cur, end - cur);
-			break;
+			btrfs_folio_clear_dirty(fs_info, folio, cur, fs_info->sectorsize);
+			continue;
 		}
 		ret = submit_one_sector(inode, folio, cur, bio_ctrl, i_size);
 		if (unlikely(ret < 0)) {
-			/*
-			 * bio_ctrl may contain a bio crossing several folios.
-			 * Submit it immediately so that the bio has a chance
-			 * to finish normally, other than marked as error.
-			 */
-			submit_one_bio(bio_ctrl);
-			/*
-			 * Failed to grab the extent map which should be very rare.
-			 * Since there is no bio submitted to finish the ordered
-			 * extent, we have to manually finish this sector.
-			 */
-			btrfs_mark_ordered_io_finished(inode, folio, cur,
-						       fs_info->sectorsize, false);
 			if (!found_error)
 				found_error = ret;
 			continue;
diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c
index 14e5257f0f04..568f0e0ebdf6 100644
--- a/fs/btrfs/file-item.c
+++ b/fs/btrfs/file-item.c
@@ -8,7 +8,6 @@
 #include <linux/pagemap.h>
 #include <linux/highmem.h>
 #include <linux/sched/mm.h>
-#include <crypto/hash.h>
 #include "messages.h"
 #include "ctree.h"
 #include "disk-io.h"
@@ -769,7 +768,6 @@ static void csum_one_bio(struct btrfs_bio *bbio, struct bvec_iter *src)
 {
 	struct btrfs_inode *inode = bbio->inode;
 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
-	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
 	struct bio *bio = &bbio->bio;
 	struct btrfs_ordered_sum *sums = bbio->sums;
 	struct bvec_iter iter = *src;
@@ -781,8 +779,6 @@ static void csum_one_bio(struct btrfs_bio *bbio, struct bvec_iter *src)
 	u32 offset = 0;
 	int index = 0;
 
-	shash->tfm = fs_info->csum_shash;
-
 	btrfs_bio_for_each_block(paddr, bio, &iter, step) {
 		paddrs[(offset / step) % nr_steps] = paddr;
 		offset += step;
diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c
index 7a501e73d880..69edf5f44bda 100644
--- a/fs/btrfs/file.c
+++ b/fs/btrfs/file.c
@@ -859,7 +859,7 @@ static noinline int prepare_one_folio(struct inode *inode, struct folio **folio_
 	fgf_t fgp_flags = (nowait ? FGP_WRITEBEGIN | FGP_NOWAIT : FGP_WRITEBEGIN) |
 			  fgf_set_order(write_bytes);
 	struct folio *folio;
-	int ret = 0;
+	int ret;
 
 again:
 	folio = __filemap_get_folio(inode->i_mapping, index, fgp_flags, mask);
@@ -876,10 +876,8 @@ static noinline int prepare_one_folio(struct inode *inode, struct folio **folio_
 	if (ret) {
 		/* The folio is already unlocked. */
 		folio_put(folio);
-		if (!nowait && ret == -EAGAIN) {
-			ret = 0;
+		if (!nowait && ret == -EAGAIN)
 			goto again;
-		}
 		return ret;
 	}
 	*folio_ret = folio;
@@ -1274,8 +1272,7 @@ static int copy_one_range(struct btrfs_inode *inode, struct iov_iter *iter,
 		btrfs_delalloc_release_extents(inode, reserved_len);
 		release_space(inode, *data_reserved, reserved_start, reserved_len,
 			      only_release_metadata);
-		ret = extents_locked;
-		return ret;
+		return extents_locked;
 	}
 
 	copied = copy_folio_from_iter_atomic(folio, offset_in_folio(folio, start),
@@ -2019,13 +2016,14 @@ static vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf)
 	else
 		btrfs_delalloc_release_space(inode, data_reserved, page_start,
 					     reserved_space, true);
-	extent_changeset_free(data_reserved);
 out_noreserve:
 	if (only_release_metadata)
 		btrfs_check_nocow_unlock(inode);
 
 	sb_end_pagefault(inode->vfs_inode.i_sb);
 
+	extent_changeset_free(data_reserved);
+
 	if (ret < 0)
 		return vmf_error(ret);
 
diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c
index 1ad2ad384b9e..a66ce9ef3aff 100644
--- a/fs/btrfs/free-space-tree.c
+++ b/fs/btrfs/free-space-tree.c
@@ -1525,9 +1525,7 @@ int btrfs_remove_block_group_free_space(struct btrfs_trans_handle *trans,
 		btrfs_release_path(path);
 	}
 
-	ret = 0;
-
-	return ret;
+	return 0;
 }
 
 static int load_free_space_bitmaps(struct btrfs_caching_control *caching_ctl,
diff --git a/fs/btrfs/fs.c b/fs/btrfs/fs.c
index feb0a2faa837..14d83565cdee 100644
--- a/fs/btrfs/fs.c
+++ b/fs/btrfs/fs.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 
+#include <linux/crc32.h>
 #include "messages.h"
 #include "fs.h"
 #include "accessors.h"
@@ -8,13 +9,11 @@
 static const struct btrfs_csums {
 	u16		size;
 	const char	name[10];
-	const char	driver[12];
 } btrfs_csums[] = {
 	[BTRFS_CSUM_TYPE_CRC32] = { .size = 4, .name = "crc32c" },
 	[BTRFS_CSUM_TYPE_XXHASH] = { .size = 8, .name = "xxhash64" },
 	[BTRFS_CSUM_TYPE_SHA256] = { .size = 32, .name = "sha256" },
-	[BTRFS_CSUM_TYPE_BLAKE2] = { .size = 32, .name = "blake2b",
-				     .driver = "blake2b-256" },
+	[BTRFS_CSUM_TYPE_BLAKE2] = { .size = 32, .name = "blake2b" },
 };
 
 /* This exists for btrfs-progs usages. */
@@ -37,21 +36,94 @@ const char *btrfs_super_csum_name(u16 csum_type)
 	return btrfs_csums[csum_type].name;
 }
 
-/*
- * Return driver name if defined, otherwise the name that's also a valid driver
- * name.
- */
-const char *btrfs_super_csum_driver(u16 csum_type)
+size_t __attribute_const__ btrfs_get_num_csums(void)
 {
-	/* csum type is validated at mount time */
-	return btrfs_csums[csum_type].driver[0] ?
-		btrfs_csums[csum_type].driver :
-		btrfs_csums[csum_type].name;
+	return ARRAY_SIZE(btrfs_csums);
 }
 
-size_t __attribute_const__ btrfs_get_num_csums(void)
+void btrfs_csum(u16 csum_type, const u8 *data, size_t len, u8 *out)
 {
-	return ARRAY_SIZE(btrfs_csums);
+	switch (csum_type) {
+	case BTRFS_CSUM_TYPE_CRC32:
+		put_unaligned_le32(~crc32c(~0, data, len), out);
+		break;
+	case BTRFS_CSUM_TYPE_XXHASH:
+		put_unaligned_le64(xxh64(data, len, 0), out);
+		break;
+	case BTRFS_CSUM_TYPE_SHA256:
+		sha256(data, len, out);
+		break;
+	case BTRFS_CSUM_TYPE_BLAKE2:
+		blake2b(NULL, 0, data, len, out, 32);
+		break;
+	default:
+		/* Checksum type is validated at mount time. */
+		BUG();
+	}
+}
+
+void btrfs_csum_init(struct btrfs_csum_ctx *ctx, u16 csum_type)
+{
+	ctx->csum_type = csum_type;
+	switch (ctx->csum_type) {
+	case BTRFS_CSUM_TYPE_CRC32:
+		ctx->crc32 = ~0;
+		break;
+	case BTRFS_CSUM_TYPE_XXHASH:
+		xxh64_reset(&ctx->xxh64, 0);
+		break;
+	case BTRFS_CSUM_TYPE_SHA256:
+		sha256_init(&ctx->sha256);
+		break;
+	case BTRFS_CSUM_TYPE_BLAKE2:
+		blake2b_init(&ctx->blake2b, 32);
+		break;
+	default:
+		/* Checksume type is validated at mount time. */
+		BUG();
+	}
+}
+
+void btrfs_csum_update(struct btrfs_csum_ctx *ctx, const u8 *data, size_t len)
+{
+	switch (ctx->csum_type) {
+	case BTRFS_CSUM_TYPE_CRC32:
+		ctx->crc32 = crc32c(ctx->crc32, data, len);
+		break;
+	case BTRFS_CSUM_TYPE_XXHASH:
+		xxh64_update(&ctx->xxh64, data, len);
+		break;
+	case BTRFS_CSUM_TYPE_SHA256:
+		sha256_update(&ctx->sha256, data, len);
+		break;
+	case BTRFS_CSUM_TYPE_BLAKE2:
+		blake2b_update(&ctx->blake2b, data, len);
+		break;
+	default:
+		/* Checksum type is validated at mount time. */
+		BUG();
+	}
+}
+
+void btrfs_csum_final(struct btrfs_csum_ctx *ctx, u8 *out)
+{
+	switch (ctx->csum_type) {
+	case BTRFS_CSUM_TYPE_CRC32:
+		put_unaligned_le32(~ctx->crc32, out);
+		break;
+	case BTRFS_CSUM_TYPE_XXHASH:
+		put_unaligned_le64(xxh64_digest(&ctx->xxh64), out);
+		break;
+	case BTRFS_CSUM_TYPE_SHA256:
+		sha256_final(&ctx->sha256, out);
+		break;
+	case BTRFS_CSUM_TYPE_BLAKE2:
+		blake2b_final(&ctx->blake2b, out);
+		break;
+	default:
+		/* Checksum type is validated at mount time. */
+		BUG();
+	}
 }
 
 /*
diff --git a/fs/btrfs/fs.h b/fs/btrfs/fs.h
index 0f7e1ef27891..dcb08bec98c4 100644
--- a/fs/btrfs/fs.h
+++ b/fs/btrfs/fs.h
@@ -3,6 +3,8 @@
 #ifndef BTRFS_FS_H
 #define BTRFS_FS_H
 
+#include <crypto/blake2b.h>
+#include <crypto/sha2.h>
 #include <linux/blkdev.h>
 #include <linux/sizes.h>
 #include <linux/time64.h>
@@ -24,6 +26,7 @@
 #include <linux/wait_bit.h>
 #include <linux/sched.h>
 #include <linux/rbtree.h>
+#include <linux/xxhash.h>
 #include <uapi/linux/btrfs.h>
 #include <uapi/linux/btrfs_tree.h>
 #include "extent-io-tree.h"
@@ -35,7 +38,6 @@ struct inode;
 struct super_block;
 struct kobject;
 struct reloc_control;
-struct crypto_shash;
 struct ulist;
 struct btrfs_device;
 struct btrfs_block_group;
@@ -842,9 +844,10 @@ struct btrfs_fs_info {
 	u32 sectorsize_bits;
 	u32 block_min_order;
 	u32 block_max_order;
+	u32 stripesize;
 	u32 csum_size;
 	u32 csums_per_leaf;
-	u32 stripesize;
+	u32 csum_type;
 
 	/*
 	 * Maximum size of an extent. BTRFS_MAX_EXTENT_SIZE on regular
@@ -856,8 +859,6 @@ struct btrfs_fs_info {
 	spinlock_t swapfile_pins_lock;
 	struct rb_root swapfile_pins;
 
-	struct crypto_shash *csum_shash;
-
 	/* Type of exclusive operation running, protected by super_lock */
 	enum btrfs_exclusive_operation exclusive_operation;
 
@@ -1049,8 +1050,20 @@ int btrfs_check_ioctl_vol_args_path(const struct btrfs_ioctl_vol_args *vol_args)
 u16 btrfs_csum_type_size(u16 type);
 int btrfs_super_csum_size(const struct btrfs_super_block *s);
 const char *btrfs_super_csum_name(u16 csum_type);
-const char *btrfs_super_csum_driver(u16 csum_type);
 size_t __attribute_const__ btrfs_get_num_csums(void);
+struct btrfs_csum_ctx {
+	u16 csum_type;
+	union {
+		u32 crc32;
+		struct xxh64_state xxh64;
+		struct sha256_ctx sha256;
+		struct blake2b_ctx blake2b;
+	};
+};
+void btrfs_csum(u16 csum_type, const u8 *data, size_t len, u8 *out);
+void btrfs_csum_init(struct btrfs_csum_ctx *ctx, u16 csum_type);
+void btrfs_csum_update(struct btrfs_csum_ctx *ctx, const u8 *data, size_t len);
+void btrfs_csum_final(struct btrfs_csum_ctx *ctx, u8 *out);
 
 static inline bool btrfs_is_empty_uuid(const u8 *uuid)
 {
diff --git a/fs/btrfs/inode-item.c b/fs/btrfs/inode-item.c
index b73e1dd97208..a864f8c99729 100644
--- a/fs/btrfs/inode-item.c
+++ b/fs/btrfs/inode-item.c
@@ -371,14 +371,13 @@ int btrfs_insert_empty_inode(struct btrfs_trans_handle *trans,
 			     struct btrfs_path *path, u64 objectid)
 {
 	struct btrfs_key key;
-	int ret;
+
 	key.objectid = objectid;
 	key.type = BTRFS_INODE_ITEM_KEY;
 	key.offset = 0;
 
-	ret = btrfs_insert_empty_item(trans, root, path, &key,
-				      sizeof(struct btrfs_inode_item));
-	return ret;
+	return btrfs_insert_empty_item(trans, root, path, &key,
+				       sizeof(struct btrfs_inode_item));
 }
 
 int btrfs_lookup_inode(struct btrfs_trans_handle *trans, struct btrfs_root
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index c4bee47829ed..312917cd666c 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -3,7 +3,6 @@
  * Copyright (C) 2007 Oracle.  All rights reserved.
  */
 
-#include <crypto/hash.h>
 #include <linux/kernel.h>
 #include <linux/bio.h>
 #include <linux/blk-cgroup.h>
@@ -218,7 +217,7 @@ static void print_data_reloc_error(const struct btrfs_inode *inode, u64 file_off
 				   int mirror_num)
 {
 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
-	struct btrfs_path path = { 0 };
+	BTRFS_PATH_AUTO_RELEASE(path);
 	struct btrfs_key found_key = { 0 };
 	struct extent_buffer *eb;
 	struct btrfs_extent_item *ei;
@@ -285,11 +284,14 @@ static void print_data_reloc_error(const struct btrfs_inode *inode, u64 file_off
 				(ref_level ? "node" : "leaf"),
 				ref_level, ref_root);
 		}
-		btrfs_release_path(&path);
 	} else {
 		struct btrfs_backref_walk_ctx ctx = { 0 };
 		struct data_reloc_warn reloc_warn = { 0 };
 
+		/*
+		 * Do not hold the path as later iterate_extent_inodes() call
+		 * can be time consuming.
+		 */
 		btrfs_release_path(&path);
 
 		ctx.bytenr = found_key.objectid;
@@ -480,13 +482,15 @@ static int insert_inline_extent(struct btrfs_trans_handle *trans,
 	ASSERT(size <= sectorsize);
 
 	/*
-	 * The compressed size also needs to be no larger than a sector.
-	 * That's also why we only need one page as the parameter.
+	 * The compressed size also needs to be no larger than a page.
+	 * That's also why we only need one folio as the parameter.
 	 */
-	if (compressed_folio)
+	if (compressed_folio) {
 		ASSERT(compressed_size <= sectorsize);
-	else
+		ASSERT(compressed_size <= PAGE_SIZE);
+	} else {
 		ASSERT(compressed_size == 0);
+	}
 
 	if (compressed_size && compressed_folio)
 		cur_size = compressed_size;
@@ -573,6 +577,18 @@ static bool can_cow_file_range_inline(struct btrfs_inode *inode,
 	if (offset != 0)
 		return false;
 
+	/*
+	 * Even for bs > ps cases, cow_file_range_inline() can only accept a
+	 * single folio.
+	 *
+	 * This can be problematic and cause access beyond page boundary if a
+	 * page sized folio is passed into that function.
+	 * And encoded write is doing exactly that.
+	 * So here limits the inlined extent size to PAGE_SIZE.
+	 */
+	if (size > PAGE_SIZE || compressed_size > PAGE_SIZE)
+		return false;
+
 	/* Inline extents are limited to sectorsize. */
 	if (size > fs_info->sectorsize)
 		return false;
@@ -617,19 +633,22 @@ static noinline int __cow_file_range_inline(struct btrfs_inode *inode,
 	struct btrfs_drop_extents_args drop_args = { 0 };
 	struct btrfs_root *root = inode->root;
 	struct btrfs_fs_info *fs_info = root->fs_info;
-	struct btrfs_trans_handle *trans;
+	struct btrfs_trans_handle *trans = NULL;
 	u64 data_len = (compressed_size ?: size);
 	int ret;
 	struct btrfs_path *path;
 
 	path = btrfs_alloc_path();
-	if (!path)
-		return -ENOMEM;
+	if (!path) {
+		ret = -ENOMEM;
+		goto out;
+	}
 
 	trans = btrfs_join_transaction(root);
 	if (IS_ERR(trans)) {
-		btrfs_free_path(path);
-		return PTR_ERR(trans);
+		ret = PTR_ERR(trans);
+		trans = NULL;
+		goto out;
 	}
 	trans->block_rsv = &inode->block_rsv;
 
@@ -671,12 +690,17 @@ static noinline int __cow_file_range_inline(struct btrfs_inode *inode,
 	/*
 	 * Don't forget to free the reserved space, as for inlined extent
 	 * it won't count as data extent, free them directly here.
-	 * And at reserve time, it's always aligned to page size, so
-	 * just free one page here.
+	 * And at reserve time, it's always aligned to sector size, so
+	 * just free one sector here.
+	 *
+	 * If we fallback to non-inline (ret == 1) due to -ENOSPC, then we need
+	 * to keep the data reservation.
 	 */
-	btrfs_qgroup_free_data(inode, NULL, 0, fs_info->sectorsize, NULL);
+	if (ret <= 0)
+		btrfs_qgroup_free_data(inode, NULL, 0, fs_info->sectorsize, NULL);
 	btrfs_free_path(path);
-	btrfs_end_transaction(trans);
+	if (trans)
+		btrfs_end_transaction(trans);
 	return ret;
 }
 
@@ -1250,6 +1274,135 @@ u64 btrfs_get_extent_allocation_hint(struct btrfs_inode *inode, u64 start,
 	return alloc_hint;
 }
 
+/*
+ * Handle COW for one range.
+ *
+ * @ins:		The key representing the allocated range.
+ * @file_offset:	The file offset of the COW range
+ * @num_bytes:		The expected length of the COW range
+ *			The actually allocated length can be smaller than it.
+ * @min_alloc_size:	The minimal extent size.
+ * @alloc_hint:		The hint for the extent allocator.
+ * @ret_alloc_size:	The COW range handles by this function.
+ *
+ * Return 0 if everything is fine and update @ret_alloc_size updated.
+ * The range is still locked, and caller should unlock the range after
+ * everything is done or for error handling.
+ *
+ * Return <0 for error and @is updated for where the extra cleanup
+ * should happen. The range [file_offset, file_offset + ret_alloc_size) will
+ * be cleaned up by this function.
+ */
+static int cow_one_range(struct btrfs_inode *inode, struct folio *locked_folio,
+			 struct btrfs_key *ins, struct extent_state **cached,
+			 u64 file_offset, u32 num_bytes, u32 min_alloc_size,
+			 u64 alloc_hint, u32 *ret_alloc_size)
+{
+	struct btrfs_root *root = inode->root;
+	struct btrfs_fs_info *fs_info = root->fs_info;
+	struct btrfs_ordered_extent *ordered;
+	struct btrfs_file_extent file_extent;
+	struct extent_map *em;
+	u32 cur_len = 0;
+	u64 cur_end;
+	int ret;
+
+	ret = btrfs_reserve_extent(root, num_bytes, num_bytes, min_alloc_size,
+				   0, alloc_hint, ins, true, true);
+	if (ret < 0) {
+		*ret_alloc_size = cur_len;
+		return ret;
+	}
+
+	cur_len = ins->offset;
+	cur_end = file_offset + cur_len - 1;
+
+	file_extent.disk_bytenr = ins->objectid;
+	file_extent.disk_num_bytes = ins->offset;
+	file_extent.num_bytes = ins->offset;
+	file_extent.ram_bytes = ins->offset;
+	file_extent.offset = 0;
+	file_extent.compression = BTRFS_COMPRESS_NONE;
+
+	/*
+	 * Locked range will be released either during error clean up (inside
+	 * this function or by the caller for previously successful ranges) or
+	 * after the whole range is finished.
+	 */
+	btrfs_lock_extent(&inode->io_tree, file_offset, cur_end, cached);
+	em = btrfs_create_io_em(inode, file_offset, &file_extent,
+				BTRFS_ORDERED_REGULAR);
+	if (IS_ERR(em)) {
+		ret = PTR_ERR(em);
+		goto free_reserved;
+	}
+	btrfs_free_extent_map(em);
+
+	ordered = btrfs_alloc_ordered_extent(inode, file_offset, &file_extent,
+					     1U << BTRFS_ORDERED_REGULAR);
+	if (IS_ERR(ordered)) {
+		btrfs_drop_extent_map_range(inode, file_offset, cur_end, false);
+		ret = PTR_ERR(ordered);
+		goto free_reserved;
+	}
+
+	if (btrfs_is_data_reloc_root(root)) {
+		ret = btrfs_reloc_clone_csums(ordered);
+
+		/*
+		 * Only drop cache here, and process as normal.
+		 *
+		 * We must not allow extent_clear_unlock_delalloc()
+		 * at free_reserved: label to free meta of this ordered
+		 * extent, as its meta should be freed by
+		 * btrfs_finish_ordered_io().
+		 *
+		 * So we must continue until @start is increased to
+		 * skip current ordered extent.
+		 */
+		if (ret)
+			btrfs_drop_extent_map_range(inode, file_offset,
+						    cur_end, false);
+	}
+	btrfs_put_ordered_extent(ordered);
+	btrfs_dec_block_group_reservations(fs_info, ins->objectid);
+	/*
+	 * Error handling for btrfs_reloc_clone_csums().
+	 *
+	 * Treat the range as finished, thus only clear EXTENT_LOCKED | EXTENT_DELALLOC.
+	 * The accounting will be done by ordered extents.
+	 */
+	if (unlikely(ret < 0)) {
+		btrfs_cleanup_ordered_extents(inode, file_offset, cur_len);
+		extent_clear_unlock_delalloc(inode, file_offset, cur_end, locked_folio, cached,
+					     EXTENT_LOCKED | EXTENT_DELALLOC,
+					     PAGE_UNLOCK | PAGE_START_WRITEBACK |
+					     PAGE_END_WRITEBACK);
+		mapping_set_error(inode->vfs_inode.i_mapping, -EIO);
+	}
+	*ret_alloc_size = cur_len;
+	return ret;
+
+free_reserved:
+	extent_clear_unlock_delalloc(inode, file_offset, cur_end, locked_folio, cached,
+				     EXTENT_LOCKED | EXTENT_DELALLOC |
+				     EXTENT_DELALLOC_NEW |
+				     EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING,
+				     PAGE_UNLOCK | PAGE_START_WRITEBACK |
+				     PAGE_END_WRITEBACK);
+	btrfs_qgroup_free_data(inode, NULL, file_offset, cur_len, NULL);
+	btrfs_dec_block_group_reservations(fs_info, ins->objectid);
+	btrfs_free_reserved_extent(fs_info, ins->objectid, ins->offset, true);
+	mapping_set_error(inode->vfs_inode.i_mapping, -EIO);
+	*ret_alloc_size = cur_len;
+	/*
+	 * We should not return -EAGAIN where it's a special return code for
+	 * zoned to catch btrfs_reserved_extent().
+	 */
+	ASSERT(ret != -EAGAIN);
+	return ret;
+}
+
 /*
  * when extent_io.c finds a delayed allocation range in the file,
  * the call backs end up in this code.  The basic idea is to
@@ -1286,11 +1439,10 @@ static noinline int cow_file_range(struct btrfs_inode *inode,
 	u64 alloc_hint = 0;
 	u64 orig_start = start;
 	u64 num_bytes;
-	u64 cur_alloc_size = 0;
-	u64 min_alloc_size;
-	u64 blocksize = fs_info->sectorsize;
+	u32 min_alloc_size;
+	u32 blocksize = fs_info->sectorsize;
+	u32 cur_alloc_size = 0;
 	struct btrfs_key ins;
-	struct extent_map *em;
 	unsigned clear_bits;
 	unsigned long page_ops;
 	int ret = 0;
@@ -1359,16 +1511,14 @@ static noinline int cow_file_range(struct btrfs_inode *inode,
 		min_alloc_size = fs_info->sectorsize;
 
 	while (num_bytes > 0) {
-		struct btrfs_ordered_extent *ordered;
-		struct btrfs_file_extent file_extent;
+		ret = cow_one_range(inode, locked_folio, &ins, &cached, start,
+				    num_bytes, min_alloc_size, alloc_hint, &cur_alloc_size);
 
-		ret = btrfs_reserve_extent(root, num_bytes, num_bytes,
-					   min_alloc_size, 0, alloc_hint,
-					   &ins, true, true);
 		if (ret == -EAGAIN) {
 			/*
-			 * btrfs_reserve_extent only returns -EAGAIN for zoned
-			 * file systems, which is an indication that there are
+			 * cow_one_range() only returns -EAGAIN for zoned
+			 * file systems (from btrfs_reserve_extent()), which
+			 * is an indication that there are
 			 * no active zones to allocate from at the moment.
 			 *
 			 * If this is the first loop iteration, wait for at
@@ -1397,79 +1547,14 @@ static noinline int cow_file_range(struct btrfs_inode *inode,
 		}
 		if (ret < 0)
 			goto out_unlock;
-		cur_alloc_size = ins.offset;
 
-		file_extent.disk_bytenr = ins.objectid;
-		file_extent.disk_num_bytes = ins.offset;
-		file_extent.num_bytes = ins.offset;
-		file_extent.ram_bytes = ins.offset;
-		file_extent.offset = 0;
-		file_extent.compression = BTRFS_COMPRESS_NONE;
-
-		/*
-		 * Locked range will be released either during error clean up or
-		 * after the whole range is finished.
-		 */
-		btrfs_lock_extent(&inode->io_tree, start, start + cur_alloc_size - 1,
-				  &cached);
-
-		em = btrfs_create_io_em(inode, start, &file_extent,
-					BTRFS_ORDERED_REGULAR);
-		if (IS_ERR(em)) {
-			btrfs_unlock_extent(&inode->io_tree, start,
-					    start + cur_alloc_size - 1, &cached);
-			ret = PTR_ERR(em);
-			goto out_reserve;
-		}
-		btrfs_free_extent_map(em);
-
-		ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent,
-						     1U << BTRFS_ORDERED_REGULAR);
-		if (IS_ERR(ordered)) {
-			btrfs_unlock_extent(&inode->io_tree, start,
-					    start + cur_alloc_size - 1, &cached);
-			ret = PTR_ERR(ordered);
-			goto out_drop_extent_cache;
-		}
-
-		if (btrfs_is_data_reloc_root(root)) {
-			ret = btrfs_reloc_clone_csums(ordered);
-
-			/*
-			 * Only drop cache here, and process as normal.
-			 *
-			 * We must not allow extent_clear_unlock_delalloc()
-			 * at out_unlock label to free meta of this ordered
-			 * extent, as its meta should be freed by
-			 * btrfs_finish_ordered_io().
-			 *
-			 * So we must continue until @start is increased to
-			 * skip current ordered extent.
-			 */
-			if (ret)
-				btrfs_drop_extent_map_range(inode, start,
-							    start + cur_alloc_size - 1,
-							    false);
-		}
-		btrfs_put_ordered_extent(ordered);
+		/* We should not allocate an extent larger than requested.*/
+		ASSERT(cur_alloc_size <= num_bytes);
 
-		btrfs_dec_block_group_reservations(fs_info, ins.objectid);
-
-		if (num_bytes < cur_alloc_size)
-			num_bytes = 0;
-		else
-			num_bytes -= cur_alloc_size;
+		num_bytes -= cur_alloc_size;
 		alloc_hint = ins.objectid + ins.offset;
 		start += cur_alloc_size;
 		cur_alloc_size = 0;
-
-		/*
-		 * btrfs_reloc_clone_csums() error, since start is increased
-		 * extent_clear_unlock_delalloc() at out_unlock label won't
-		 * free metadata of current ordered extent, we're OK to exit.
-		 */
-		if (ret)
-			goto out_unlock;
 	}
 	extent_clear_unlock_delalloc(inode, orig_start, end, locked_folio, &cached,
 				     EXTENT_LOCKED | EXTENT_DELALLOC, page_ops);
@@ -1478,11 +1563,6 @@ static noinline int cow_file_range(struct btrfs_inode *inode,
 		*done_offset = end;
 	return ret;
 
-out_drop_extent_cache:
-	btrfs_drop_extent_map_range(inode, start, start + cur_alloc_size - 1, false);
-out_reserve:
-	btrfs_dec_block_group_reservations(fs_info, ins.objectid);
-	btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, true);
 out_unlock:
 	/*
 	 * Now, we have three regions to clean up:
@@ -1519,24 +1599,9 @@ static noinline int cow_file_range(struct btrfs_inode *inode,
 	page_ops = PAGE_UNLOCK | PAGE_START_WRITEBACK | PAGE_END_WRITEBACK;
 
 	/*
-	 * For the range (2). If we reserved an extent for our delalloc range
-	 * (or a subrange) and failed to create the respective ordered extent,
-	 * then it means that when we reserved the extent we decremented the
-	 * extent's size from the data space_info's bytes_may_use counter and
-	 * incremented the space_info's bytes_reserved counter by the same
-	 * amount. We must make sure extent_clear_unlock_delalloc() does not try
-	 * to decrement again the data space_info's bytes_may_use counter,
-	 * therefore we do not pass it the flag EXTENT_CLEAR_DATA_RESV.
-	 */
-	if (cur_alloc_size) {
-		extent_clear_unlock_delalloc(inode, start,
-					     start + cur_alloc_size - 1,
-					     locked_folio, &cached, clear_bits,
-					     page_ops);
-		btrfs_qgroup_free_data(inode, NULL, start, cur_alloc_size, NULL);
-	}
-
-	/*
+	 * For the range (2) the error handling is done by cow_one_range() itself.
+	 * Nothing needs to be done.
+	 *
 	 * For the range (3). We never touched the region. In addition to the
 	 * clear_bits above, we add EXTENT_CLEAR_DATA_RESV to release the data
 	 * space_info's bytes_may_use counter, reserved in
@@ -1551,7 +1616,7 @@ static noinline int cow_file_range(struct btrfs_inode *inode,
 				       end - start - cur_alloc_size + 1, NULL);
 	}
 	btrfs_err(fs_info,
-"%s failed, root=%llu inode=%llu start=%llu len=%llu cur_offset=%llu cur_alloc_size=%llu: %d",
+"%s failed, root=%llu inode=%llu start=%llu len=%llu cur_offset=%llu cur_alloc_size=%u: %d",
 		  __func__, btrfs_root_id(inode->root),
 		  btrfs_ino(inode), orig_start, end + 1 - orig_start,
 		  start, cur_alloc_size, ret);
@@ -2348,7 +2413,6 @@ int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_fol
 			     u64 start, u64 end, struct writeback_control *wbc)
 {
 	const bool zoned = btrfs_is_zoned(inode->root->fs_info);
-	int ret;
 
 	/*
 	 * The range must cover part of the @locked_folio, or a return of 1
@@ -2357,10 +2421,8 @@ int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_fol
 	ASSERT(!(end <= folio_pos(locked_folio) ||
 		 start >= folio_next_pos(locked_folio)));
 
-	if (should_nocow(inode, start, end)) {
-		ret = run_delalloc_nocow(inode, locked_folio, start, end);
-		return ret;
-	}
+	if (should_nocow(inode, start, end))
+		return run_delalloc_nocow(inode, locked_folio, start, end);
 
 	if (btrfs_inode_can_compress(inode) &&
 	    inode_need_compress(inode, start, end) &&
@@ -2368,11 +2430,9 @@ int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_fol
 		return 1;
 
 	if (zoned)
-		ret = run_delalloc_cow(inode, locked_folio, start, end, wbc,
-				       true);
+		return run_delalloc_cow(inode, locked_folio, start, end, wbc, true);
 	else
-		ret = cow_file_range(inode, locked_folio, start, end, NULL, 0);
-	return ret;
+		return cow_file_range(inode, locked_folio, start, end, NULL, 0);
 }
 
 void btrfs_split_delalloc_extent(struct btrfs_inode *inode,
@@ -3224,6 +3284,21 @@ int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent)
 		goto out;
 	}
 
+	/*
+	 * If we have no data checksum, either the OE is:
+	 * - Fully truncated
+	 *   Those ones won't reach here.
+	 *
+	 * - No data checksum
+	 *
+	 * - Belongs to data reloc inode
+	 *   Which doesn't have csum attached to OE, but cloned
+	 *   from original chunk.
+	 */
+	if (list_empty(&ordered_extent->list))
+		ASSERT(inode->flags & BTRFS_INODE_NODATASUM ||
+		       btrfs_is_data_reloc_root(inode->root));
+
 	ret = add_pending_csums(trans, &ordered_extent->list);
 	if (unlikely(ret)) {
 		btrfs_abort_transaction(trans, ret);
@@ -3394,20 +3469,19 @@ void btrfs_calculate_block_csum_pages(struct btrfs_fs_info *fs_info,
 	const u32 blocksize = fs_info->sectorsize;
 	const u32 step = min(blocksize, PAGE_SIZE);
 	const u32 nr_steps = blocksize / step;
-	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
+	struct btrfs_csum_ctx csum;
 
-	shash->tfm = fs_info->csum_shash;
-	crypto_shash_init(shash);
+	btrfs_csum_init(&csum, fs_info->csum_type);
 	for (int i = 0; i < nr_steps; i++) {
 		const phys_addr_t paddr = paddrs[i];
 		void *kaddr;
 
 		ASSERT(offset_in_page(paddr) + step <= PAGE_SIZE);
 		kaddr = kmap_local_page(phys_to_page(paddr)) + offset_in_page(paddr);
-		crypto_shash_update(shash, kaddr, step);
+		btrfs_csum_update(&csum, kaddr, step);
 		kunmap_local(kaddr);
 	}
-	crypto_shash_final(shash, dest);
+	btrfs_csum_final(&csum, dest);
 }
 
 /*
@@ -4025,11 +4099,6 @@ static int btrfs_read_locked_inode(struct btrfs_inode *inode, struct btrfs_path
 	btrfs_set_inode_mapping_order(inode);
 
 cache_index:
-	ret = btrfs_init_file_extent_tree(inode);
-	if (ret)
-		goto out;
-	btrfs_inode_set_file_extent_range(inode, 0,
-			round_up(i_size_read(vfs_inode), fs_info->sectorsize));
 	/*
 	 * If we were modified in the current generation and evicted from memory
 	 * and then re-read we need to do a full sync since we don't have any
@@ -4116,6 +4185,20 @@ static int btrfs_read_locked_inode(struct btrfs_inode *inode, struct btrfs_path
 				  btrfs_ino(inode), btrfs_root_id(root), ret);
 	}
 
+	/*
+	 * We don't need the path anymore, so release it to avoid holding a read
+	 * lock on a leaf while calling btrfs_init_file_extent_tree(), which can
+	 * allocate memory that triggers reclaim (GFP_KERNEL) and cause a locking
+	 * dependency.
+	 */
+	btrfs_release_path(path);
+
+	ret = btrfs_init_file_extent_tree(inode);
+	if (ret)
+		goto out;
+	btrfs_inode_set_file_extent_range(inode, 0,
+			  round_up(i_size_read(vfs_inode), fs_info->sectorsize));
+
 	if (!maybe_acls)
 		cache_no_acl(vfs_inode);
 
diff --git a/fs/btrfs/messages.c b/fs/btrfs/messages.c
index 2f853de44473..6190777924bf 100644
--- a/fs/btrfs/messages.c
+++ b/fs/btrfs/messages.c
@@ -211,33 +211,19 @@ static struct ratelimit_state printk_limits[] = {
 	RATELIMIT_STATE_INIT(printk_limits[7], DEFAULT_RATELIMIT_INTERVAL, 100),
 };
 
-void __cold _btrfs_printk(const struct btrfs_fs_info *fs_info, const char *fmt, ...)
+__printf(3, 4) __cold
+void _btrfs_printk(const struct btrfs_fs_info *fs_info, unsigned int level, const char *fmt, ...)
 {
-	char lvl[PRINTK_MAX_SINGLE_HEADER_LEN + 1] = "\0";
 	struct va_format vaf;
 	va_list args;
-	int kern_level;
-	const char *type = logtypes[4];
-	struct ratelimit_state *ratelimit = &printk_limits[4];
+	const char *type = logtypes[level];
+	struct ratelimit_state *ratelimit = &printk_limits[level];
 
 #ifdef CONFIG_PRINTK_INDEX
 	printk_index_subsys_emit("%sBTRFS %s (device %s): ", NULL, fmt);
 #endif
 
 	va_start(args, fmt);
-
-	while ((kern_level = printk_get_level(fmt)) != 0) {
-		size_t size = printk_skip_level(fmt) - fmt;
-
-		if (kern_level >= '0' && kern_level <= '7') {
-			memcpy(lvl, fmt,  size);
-			lvl[size] = '\0';
-			type = logtypes[kern_level - '0'];
-			ratelimit = &printk_limits[kern_level - '0'];
-		}
-		fmt += size;
-	}
-
 	vaf.fmt = fmt;
 	vaf.va = &args;
 
@@ -247,10 +233,10 @@ void __cold _btrfs_printk(const struct btrfs_fs_info *fs_info, const char *fmt,
 			char statestr[STATE_STRING_BUF_LEN];
 
 			btrfs_state_to_string(fs_info, statestr);
-			_printk("%sBTRFS %s (device %s%s): %pV\n", lvl, type,
+			_printk(KERN_SOH "%dBTRFS %s (device %s%s): %pV\n", level, type,
 				fs_info->sb->s_id, statestr, &vaf);
 		} else {
-			_printk("%sBTRFS %s: %pV\n", lvl, type, &vaf);
+			_printk(KERN_SOH "%dBTRFS %s: %pV\n", level, type, &vaf);
 		}
 	}
 
diff --git a/fs/btrfs/messages.h b/fs/btrfs/messages.h
index d8c0bd17dcda..943e53980945 100644
--- a/fs/btrfs/messages.h
+++ b/fs/btrfs/messages.h
@@ -23,69 +23,74 @@ void btrfs_no_printk(const struct btrfs_fs_info *fs_info, const char *fmt, ...)
 
 #ifdef CONFIG_PRINTK
 
-#define btrfs_printk(fs_info, fmt, args...)				\
-	_btrfs_printk(fs_info, fmt, ##args)
-
-__printf(2, 3)
-__cold
-void _btrfs_printk(const struct btrfs_fs_info *fs_info, const char *fmt, ...);
+__printf(3, 4) __cold
+void _btrfs_printk(const struct btrfs_fs_info *fs_info, unsigned int level, const char *fmt, ...);
 
 #else
 
-#define btrfs_printk(fs_info, fmt, args...) \
+#define btrfs_printk_in_rcu(fs_info, level, fmt, args...)		\
+	btrfs_no_printk(fs_info, fmt, ##args)
+
+#define btrfs_printk_in_rcu(fs_info, level, fmt, args...)		\
 	btrfs_no_printk(fs_info, fmt, ##args)
+
+#define btrfs_printk_rl_in_rcu(fs_info, level, fmt, args...)		\
+	btrfs_no_printk(fs_info, fmt, ##args)
+
 #endif
 
 /*
  * Print a message with filesystem info, enclosed in RCU protection.
  */
 #define btrfs_crit(fs_info, fmt, args...) \
-	btrfs_printk_in_rcu(fs_info, KERN_CRIT fmt, ##args)
+	btrfs_printk_in_rcu(fs_info, LOGLEVEL_CRIT, fmt, ##args)
 #define btrfs_err(fs_info, fmt, args...) \
-	btrfs_printk_in_rcu(fs_info, KERN_ERR fmt, ##args)
+	btrfs_printk_in_rcu(fs_info, LOGLEVEL_ERR, fmt, ##args)
 #define btrfs_warn(fs_info, fmt, args...) \
-	btrfs_printk_in_rcu(fs_info, KERN_WARNING fmt, ##args)
+	btrfs_printk_in_rcu(fs_info, LOGLEVEL_WARNING, fmt, ##args)
 #define btrfs_info(fs_info, fmt, args...) \
-	btrfs_printk_in_rcu(fs_info, KERN_INFO fmt, ##args)
+	btrfs_printk_in_rcu(fs_info, LOGLEVEL_INFO, fmt, ##args)
 
 /*
  * Wrappers that use a ratelimited printk
  */
 #define btrfs_crit_rl(fs_info, fmt, args...) \
-	btrfs_printk_rl_in_rcu(fs_info, KERN_CRIT fmt, ##args)
+	btrfs_printk_rl_in_rcu(fs_info, LOGLEVEL_CRIT, fmt, ##args)
 #define btrfs_err_rl(fs_info, fmt, args...) \
-	btrfs_printk_rl_in_rcu(fs_info, KERN_ERR fmt, ##args)
+	btrfs_printk_rl_in_rcu(fs_info, LOGLEVEL_ERR, fmt, ##args)
 #define btrfs_warn_rl(fs_info, fmt, args...) \
-	btrfs_printk_rl_in_rcu(fs_info, KERN_WARNING fmt, ##args)
+	btrfs_printk_rl_in_rcu(fs_info, LOGLEVEL_WARNING, fmt, ##args)
 #define btrfs_info_rl(fs_info, fmt, args...) \
-	btrfs_printk_rl_in_rcu(fs_info, KERN_INFO fmt, ##args)
+	btrfs_printk_rl_in_rcu(fs_info, LOGLEVEL_INFO, fmt, ##args)
 
 #if defined(CONFIG_DYNAMIC_DEBUG)
 #define btrfs_debug(fs_info, fmt, args...)				\
 	_dynamic_func_call_no_desc(fmt, btrfs_printk_in_rcu,		\
-				   fs_info, KERN_DEBUG fmt, ##args)
+				   fs_info, LOGLEVEL_DEBUG, fmt, ##args)
 #define btrfs_debug_rl(fs_info, fmt, args...)				\
 	_dynamic_func_call_no_desc(fmt, btrfs_printk_rl_in_rcu,		\
-				   fs_info, KERN_DEBUG fmt, ##args)
+				   fs_info, LOGLEVEL_DEBUG, fmt, ##args)
 #elif defined(DEBUG)
 #define btrfs_debug(fs_info, fmt, args...) \
-	btrfs_printk_in_rcu(fs_info, KERN_DEBUG fmt, ##args)
+	btrfs_printk_in_rcu(fs_info, LOGLEVEL_DEBUG, fmt, ##args)
 #define btrfs_debug_rl(fs_info, fmt, args...) \
-	btrfs_printk_rl_in_rcu(fs_info, KERN_DEBUG fmt, ##args)
+	btrfs_printk_rl_in_rcu(fs_info, LOGLEVEl_DEBUG, fmt, ##args)
 #else
 /* When printk() is no_printk(), expand to no-op. */
 #define btrfs_debug(fs_info, fmt, args...)	do { (void)(fs_info); } while(0)
 #define btrfs_debug_rl(fs_info, fmt, args...)	do { (void)(fs_info); } while(0)
 #endif
 
-#define btrfs_printk_in_rcu(fs_info, fmt, args...)	\
-do {							\
-	rcu_read_lock();				\
-	btrfs_printk(fs_info, fmt, ##args);		\
-	rcu_read_unlock();				\
+#ifdef CONFIG_PRINTK
+
+#define btrfs_printk_in_rcu(fs_info, level, fmt, args...)	\
+do {								\
+	rcu_read_lock();					\
+	_btrfs_printk(fs_info, level, fmt, ##args);		\
+	rcu_read_unlock();					\
 } while (0)
 
-#define btrfs_printk_rl_in_rcu(fs_info, fmt, args...)		\
+#define btrfs_printk_rl_in_rcu(fs_info, level, fmt, args...)	\
 do {								\
 	static DEFINE_RATELIMIT_STATE(_rs,			\
 		DEFAULT_RATELIMIT_INTERVAL,			\
@@ -93,10 +98,12 @@ do {								\
 								\
 	rcu_read_lock();					\
 	if (__ratelimit(&_rs))					\
-		btrfs_printk(fs_info, fmt, ##args);		\
+		_btrfs_printk(fs_info, level, fmt, ##args);	\
 	rcu_read_unlock();					\
 } while (0)
 
+#endif
+
 #ifdef CONFIG_BTRFS_ASSERT
 
 __printf(1, 2)
@@ -113,7 +120,6 @@ static inline void verify_assert_printk_format(const char *fmt, ...) {
  */
 #define __REST_ARGS(_, ... ) __VA_OPT__(,) __VA_ARGS__
 
-#if defined(CONFIG_CC_IS_CLANG) || GCC_VERSION >= 80000
 /*
  * Assertion with optional printk() format.
  *
@@ -151,22 +157,6 @@ do {										\
 	}									\
 } while(0)
 
-#else
-
-/* For GCC < 8.x only the simple output. */
-
-#define ASSERT(cond, args...)							\
-do {										\
-	verify_assert_printk_format("check the format string" args);		\
-	if (!likely(cond)) {							\
-		pr_err("assertion failed: %s :: %ld, in %s:%d\n",		\
-			#cond, (long)(cond), __FILE__, __LINE__);		\
-		BUG();								\
-	}									\
-} while(0)
-
-#endif
-
 #else
 /* Compile check the @cond expression but don't generate any code. */
 #define ASSERT(cond, args...)			BUILD_BUG_ON_INVALID(cond)
diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c
index 9e2b53e90dcb..14d393a5853d 100644
--- a/fs/btrfs/qgroup.c
+++ b/fs/btrfs/qgroup.c
@@ -346,6 +346,42 @@ int btrfs_verify_qgroup_counts(const struct btrfs_fs_info *fs_info, u64 qgroupid
 }
 #endif
 
+static bool squota_check_parent_usage(struct btrfs_fs_info *fs_info, struct btrfs_qgroup *parent)
+{
+	u64 excl_sum = 0;
+	u64 rfer_sum = 0;
+	u64 excl_cmpr_sum = 0;
+	u64 rfer_cmpr_sum = 0;
+	struct btrfs_qgroup_list *glist;
+	int nr_members = 0;
+	bool mismatch;
+
+	if (btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_SIMPLE)
+		return false;
+	if (btrfs_qgroup_level(parent->qgroupid) == 0)
+		return false;
+
+	/* Eligible parent qgroup. Squota; level > 0; empty members list. */
+	list_for_each_entry(glist, &parent->members, next_member) {
+		excl_sum += glist->member->excl;
+		rfer_sum += glist->member->rfer;
+		excl_cmpr_sum += glist->member->excl_cmpr;
+		rfer_cmpr_sum += glist->member->rfer_cmpr;
+		nr_members++;
+	}
+	mismatch = (parent->excl != excl_sum || parent->rfer != rfer_sum ||
+		    parent->excl_cmpr != excl_cmpr_sum || parent->rfer_cmpr != excl_cmpr_sum);
+
+	WARN(mismatch,
+	     "parent squota qgroup %hu/%llu has mismatched usage from its %d members. "
+	     "%llu %llu %llu %llu vs %llu %llu %llu %llu\n",
+	     btrfs_qgroup_level(parent->qgroupid),
+	     btrfs_qgroup_subvolid(parent->qgroupid), nr_members, parent->excl,
+	     parent->rfer, parent->excl_cmpr, parent->rfer_cmpr, excl_sum,
+	     rfer_sum, excl_cmpr_sum, rfer_cmpr_sum);
+	return mismatch;
+}
+
 __printf(2, 3)
 static void qgroup_mark_inconsistent(struct btrfs_fs_info *fs_info, const char *fmt, ...)
 {
@@ -658,7 +694,6 @@ void btrfs_free_qgroup_config(struct btrfs_fs_info *fs_info)
 static int add_qgroup_relation_item(struct btrfs_trans_handle *trans, u64 src,
 				    u64 dst)
 {
-	int ret;
 	struct btrfs_root *quota_root = trans->fs_info->quota_root;
 	BTRFS_PATH_AUTO_FREE(path);
 	struct btrfs_key key;
@@ -671,8 +706,7 @@ static int add_qgroup_relation_item(struct btrfs_trans_handle *trans, u64 src,
 	key.type = BTRFS_QGROUP_RELATION_KEY;
 	key.offset = dst;
 
-	ret = btrfs_insert_empty_item(trans, quota_root, path, &key, 0);
-	return ret;
+	return btrfs_insert_empty_item(trans, quota_root, path, &key, 0);
 }
 
 static int del_qgroup_relation_item(struct btrfs_trans_handle *trans, u64 src,
@@ -797,9 +831,7 @@ static int del_qgroup_item(struct btrfs_trans_handle *trans, u64 qgroupid)
 	if (ret > 0)
 		return -ENOENT;
 
-	ret = btrfs_del_item(trans, quota_root, path);
-
-	return ret;
+	return btrfs_del_item(trans, quota_root, path);
 }
 
 static int update_qgroup_limit_item(struct btrfs_trans_handle *trans,
@@ -1243,14 +1275,7 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info,
 		btrfs_end_transaction(trans);
 	else if (trans)
 		ret = btrfs_end_transaction(trans);
-
-	/*
-	 * At this point we either failed at allocating prealloc, or we
-	 * succeeded and passed the ownership to it to add_qgroup_rb(). In any
-	 * case, this needs to be NULL or there is something wrong.
-	 */
-	ASSERT(prealloc == NULL);
-
+	kfree(prealloc);
 	return ret;
 }
 
@@ -1569,6 +1594,7 @@ int btrfs_add_qgroup_relation(struct btrfs_trans_handle *trans, u64 src, u64 dst
 		goto out;
 	}
 	ret = quick_update_accounting(fs_info, src, dst, 1);
+	squota_check_parent_usage(fs_info, parent);
 	spin_unlock(&fs_info->qgroup_lock);
 out:
 	kfree(prealloc);
@@ -1625,6 +1651,8 @@ static int __del_qgroup_relation(struct btrfs_trans_handle *trans, u64 src,
 		spin_lock(&fs_info->qgroup_lock);
 		del_relation_rb(fs_info, src, dst);
 		ret = quick_update_accounting(fs_info, src, dst, -1);
+		ASSERT(parent);
+		squota_check_parent_usage(fs_info, parent);
 		spin_unlock(&fs_info->qgroup_lock);
 	}
 out:
@@ -1682,15 +1710,40 @@ int btrfs_create_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid)
 	ret = btrfs_sysfs_add_one_qgroup(fs_info, qgroup);
 out:
 	mutex_unlock(&fs_info->qgroup_ioctl_lock);
-	/*
-	 * At this point we either failed at allocating prealloc, or we
-	 * succeeded and passed the ownership to it to add_qgroup_rb(). In any
-	 * case, this needs to be NULL or there is something wrong.
-	 */
-	ASSERT(prealloc == NULL);
+	kfree(prealloc);
 	return ret;
 }
 
+static bool can_delete_parent_qgroup(struct btrfs_qgroup *qgroup)
+
+{
+	ASSERT(btrfs_qgroup_level(qgroup->qgroupid));
+	return list_empty(&qgroup->members);
+}
+
+/*
+ * Return true if we can delete the squota qgroup and false otherwise.
+ *
+ * Rules for whether we can delete:
+ *
+ * A subvolume qgroup can be removed iff the subvolume is fully deleted, which
+ * is iff there is 0 usage in the qgroup.
+ *
+ * A higher level qgroup can be removed iff it has no members.
+ * Note: We audit its usage to warn on inconsitencies without blocking deletion.
+ */
+static bool can_delete_squota_qgroup(struct btrfs_fs_info *fs_info, struct btrfs_qgroup *qgroup)
+{
+	ASSERT(btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE);
+
+	if (btrfs_qgroup_level(qgroup->qgroupid) > 0) {
+		squota_check_parent_usage(fs_info, qgroup);
+		return can_delete_parent_qgroup(qgroup);
+	}
+
+	return !(qgroup->rfer || qgroup->excl || qgroup->rfer_cmpr || qgroup->excl_cmpr);
+}
+
 /*
  * Return 0 if we can not delete the qgroup (not empty or has children etc).
  * Return >0 if we can delete the qgroup.
@@ -1701,23 +1754,13 @@ static int can_delete_qgroup(struct btrfs_fs_info *fs_info, struct btrfs_qgroup
 	struct btrfs_key key;
 	BTRFS_PATH_AUTO_FREE(path);
 
-	/*
-	 * Squota would never be inconsistent, but there can still be case
-	 * where a dropped subvolume still has qgroup numbers, and squota
-	 * relies on such qgroup for future accounting.
-	 *
-	 * So for squota, do not allow dropping any non-zero qgroup.
-	 */
-	if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE &&
-	    (qgroup->rfer || qgroup->excl || qgroup->excl_cmpr || qgroup->rfer_cmpr))
-		return 0;
+	/* Since squotas cannot be inconsistent, they have special rules for deletion. */
+	if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE)
+		return can_delete_squota_qgroup(fs_info, qgroup);
 
 	/* For higher level qgroup, we can only delete it if it has no child. */
-	if (btrfs_qgroup_level(qgroup->qgroupid)) {
-		if (!list_empty(&qgroup->members))
-			return 0;
-		return 1;
-	}
+	if (btrfs_qgroup_level(qgroup->qgroupid))
+		return can_delete_parent_qgroup(qgroup);
 
 	/*
 	 * For level-0 qgroups, we can only delete it if it has no subvolume
@@ -2608,10 +2651,8 @@ int btrfs_qgroup_trace_subtree(struct btrfs_trans_handle *trans,
 			return ret;
 	}
 
-	if (root_level == 0) {
-		ret = btrfs_qgroup_trace_leaf_items(trans, root_eb);
-		return ret;
-	}
+	if (root_level == 0)
+		return btrfs_qgroup_trace_leaf_items(trans, root_eb);
 
 	path = btrfs_alloc_path();
 	if (!path)
@@ -3220,9 +3261,15 @@ static int qgroup_snapshot_quick_inherit(struct btrfs_fs_info *fs_info,
 {
 	struct btrfs_qgroup *src;
 	struct btrfs_qgroup *parent;
+	struct btrfs_qgroup *qgroup;
 	struct btrfs_qgroup_list *list;
+	LIST_HEAD(qgroup_list);
+	const u32 nodesize = fs_info->nodesize;
 	int nr_parents = 0;
 
+	if (btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_FULL)
+		return 0;
+
 	src = find_qgroup_rb(fs_info, srcid);
 	if (!src)
 		return -ENOENT;
@@ -3257,8 +3304,19 @@ static int qgroup_snapshot_quick_inherit(struct btrfs_fs_info *fs_info,
 	if (parent->excl != parent->rfer)
 		return 1;
 
-	parent->excl += fs_info->nodesize;
-	parent->rfer += fs_info->nodesize;
+	qgroup_iterator_add(&qgroup_list, parent);
+	list_for_each_entry(qgroup, &qgroup_list, iterator) {
+		qgroup->rfer += nodesize;
+		qgroup->rfer_cmpr += nodesize;
+		qgroup->excl += nodesize;
+		qgroup->excl_cmpr += nodesize;
+		qgroup_dirty(fs_info, qgroup);
+
+		/* Append parent qgroups to @qgroup_list. */
+		list_for_each_entry(list, &qgroup->groups, next_group)
+			qgroup_iterator_add(&qgroup_list, list->group);
+	}
+	qgroup_iterator_clean(&qgroup_list);
 	return 0;
 }
 
@@ -3279,7 +3337,7 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid,
 	struct btrfs_root *quota_root;
 	struct btrfs_qgroup *srcgroup;
 	struct btrfs_qgroup *dstgroup;
-	struct btrfs_qgroup *prealloc = NULL;
+	struct btrfs_qgroup *prealloc;
 	struct btrfs_qgroup_list **qlist_prealloc = NULL;
 	bool free_inherit = false;
 	bool need_rescan = false;
@@ -3520,14 +3578,7 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid,
 	}
 	if (free_inherit)
 		kfree(inherit);
-
-	/*
-	 * At this point we either failed at allocating prealloc, or we
-	 * succeeded and passed the ownership to it to add_qgroup_rb(). In any
-	 * case, this needs to be NULL or there is something wrong.
-	 */
-	ASSERT(prealloc == NULL);
-
+	kfree(prealloc);
 	return ret;
 }
 
diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c
index 5bfefc3e9c06..310b7d817a27 100644
--- a/fs/btrfs/relocation.c
+++ b/fs/btrfs/relocation.c
@@ -3254,7 +3254,6 @@ static int delete_v1_space_cache(struct extent_buffer *leaf,
 	struct btrfs_key key;
 	bool found = false;
 	int i;
-	int ret;
 
 	if (btrfs_header_owner(leaf) != BTRFS_ROOT_TREE_OBJECTID)
 		return 0;
@@ -3278,8 +3277,8 @@ static int delete_v1_space_cache(struct extent_buffer *leaf,
 	}
 	if (!found)
 		return -ENOENT;
-	ret = delete_block_group_cache(block_group, NULL, space_cache_ino);
-	return ret;
+
+	return delete_block_group_cache(block_group, NULL, space_cache_ino);
 }
 
 /*
diff --git a/fs/btrfs/root-tree.c b/fs/btrfs/root-tree.c
index 6a7e297ab0a7..37a4173c0a0b 100644
--- a/fs/btrfs/root-tree.c
+++ b/fs/btrfs/root-tree.c
@@ -217,8 +217,6 @@ int btrfs_find_orphan_roots(struct btrfs_fs_info *fs_info)
 	BTRFS_PATH_AUTO_FREE(path);
 	struct btrfs_key key;
 	struct btrfs_root *root;
-	int err = 0;
-	int ret;
 
 	path = btrfs_alloc_path();
 	if (!path)
@@ -230,20 +228,19 @@ int btrfs_find_orphan_roots(struct btrfs_fs_info *fs_info)
 
 	while (1) {
 		u64 root_objectid;
+		int ret;
 
 		ret = btrfs_search_slot(NULL, tree_root, &key, path, 0, 0);
-		if (ret < 0) {
-			err = ret;
-			break;
-		}
+		if (ret < 0)
+			return ret;
 
 		leaf = path->nodes[0];
 		if (path->slots[0] >= btrfs_header_nritems(leaf)) {
 			ret = btrfs_next_leaf(tree_root, path);
 			if (ret < 0)
-				err = ret;
-			if (ret != 0)
-				break;
+				return ret;
+			else if (ret > 0)
+				return 0;
 			leaf = path->nodes[0];
 		}
 
@@ -252,34 +249,32 @@ int btrfs_find_orphan_roots(struct btrfs_fs_info *fs_info)
 
 		if (key.objectid != BTRFS_ORPHAN_OBJECTID ||
 		    key.type != BTRFS_ORPHAN_ITEM_KEY)
-			break;
+			return 0;
 
 		root_objectid = key.offset;
 		key.offset++;
 
 		root = btrfs_get_fs_root(fs_info, root_objectid, false);
-		err = PTR_ERR_OR_ZERO(root);
-		if (err && err != -ENOENT) {
+		ret = PTR_ERR_OR_ZERO(root);
+		if (ret && ret != -ENOENT) {
 			break;
-		} else if (err == -ENOENT) {
+		} else if (ret == -ENOENT) {
 			struct btrfs_trans_handle *trans;
 
-			btrfs_release_path(path);
-
 			trans = btrfs_join_transaction(tree_root);
 			if (IS_ERR(trans)) {
-				err = PTR_ERR(trans);
-				btrfs_handle_fs_error(fs_info, err,
-					    "Failed to start trans to delete orphan item");
-				break;
+				ret = PTR_ERR(trans);
+				btrfs_err(fs_info,
+			  "failed to join transaction to delete orphan item: %d",
+					  ret);
+				return ret;
 			}
-			err = btrfs_del_orphan_item(trans, tree_root,
-						    root_objectid);
+			ret = btrfs_del_orphan_item(trans, tree_root, root_objectid);
 			btrfs_end_transaction(trans);
-			if (err) {
-				btrfs_handle_fs_error(fs_info, err,
-					    "Failed to delete root orphan item");
-				break;
+			if (ret) {
+				btrfs_err(fs_info,
+				  "failed to delete root orphan item: %d", ret);
+				return ret;
 			}
 			continue;
 		}
@@ -307,7 +302,7 @@ int btrfs_find_orphan_roots(struct btrfs_fs_info *fs_info)
 		btrfs_put_root(root);
 	}
 
-	return err;
+	return 0;
 }
 
 /* drop the root item for 'key' from the tree root */
diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c
index a40ee41f42c6..2372084cf6c5 100644
--- a/fs/btrfs/scrub.c
+++ b/fs/btrfs/scrub.c
@@ -6,7 +6,6 @@
 #include <linux/blkdev.h>
 #include <linux/ratelimit.h>
 #include <linux/sched/mm.h>
-#include <crypto/hash.h>
 #include "ctree.h"
 #include "discard.h"
 #include "volumes.h"
@@ -718,7 +717,7 @@ static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr
 	const u64 logical = stripe->logical + (sector_nr << fs_info->sectorsize_bits);
 	void *first_kaddr = scrub_stripe_get_kaddr(stripe, sector_nr);
 	struct btrfs_header *header = first_kaddr;
-	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
+	struct btrfs_csum_ctx csum;
 	u8 on_disk_csum[BTRFS_CSUM_SIZE];
 	u8 calculated_csum[BTRFS_CSUM_SIZE];
 
@@ -760,17 +759,16 @@ static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr
 	}
 
 	/* Now check tree block csum. */
-	shash->tfm = fs_info->csum_shash;
-	crypto_shash_init(shash);
-	crypto_shash_update(shash, first_kaddr + BTRFS_CSUM_SIZE,
-			    fs_info->sectorsize - BTRFS_CSUM_SIZE);
+	btrfs_csum_init(&csum, fs_info->csum_type);
+	btrfs_csum_update(&csum, first_kaddr + BTRFS_CSUM_SIZE,
+			  fs_info->sectorsize - BTRFS_CSUM_SIZE);
 
 	for (int i = sector_nr + 1; i < sector_nr + sectors_per_tree; i++) {
-		crypto_shash_update(shash, scrub_stripe_get_kaddr(stripe, i),
-				    fs_info->sectorsize);
+		btrfs_csum_update(&csum, scrub_stripe_get_kaddr(stripe, i),
+				  fs_info->sectorsize);
 	}
 
-	crypto_shash_final(shash, calculated_csum);
+	btrfs_csum_final(&csum, calculated_csum);
 	if (memcmp(calculated_csum, on_disk_csum, fs_info->csum_size) != 0) {
 		scrub_bitmap_set_meta_error(stripe, sector_nr, sectors_per_tree);
 		scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree);
@@ -2173,8 +2171,8 @@ static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx,
 				      u64 full_stripe_start)
 {
 	struct btrfs_fs_info *fs_info = sctx->fs_info;
-	struct btrfs_path extent_path = { 0 };
-	struct btrfs_path csum_path = { 0 };
+	BTRFS_PATH_AUTO_RELEASE(extent_path);
+	BTRFS_PATH_AUTO_RELEASE(csum_path);
 	struct scrub_stripe *stripe;
 	bool all_empty = true;
 	const int data_stripes = nr_data_stripes(map);
@@ -2226,7 +2224,7 @@ static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx,
 				full_stripe_start + btrfs_stripe_nr_to_offset(i),
 				BTRFS_STRIPE_LEN, stripe);
 		if (ret < 0)
-			goto out;
+			return ret;
 		/*
 		 * No extent in this data stripe, need to manually mark them
 		 * initialized to make later read submission happy.
@@ -2248,10 +2246,8 @@ static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx,
 			break;
 		}
 	}
-	if (all_empty) {
-		ret = 0;
-		goto out;
-	}
+	if (all_empty)
+		return 0;
 
 	for (int i = 0; i < data_stripes; i++) {
 		stripe = &sctx->raid56_data_stripes[i];
@@ -2292,20 +2288,15 @@ static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx,
 "scrub: unrepaired sectors detected, full stripe %llu data stripe %u errors %*pbl",
 				  full_stripe_start, i, stripe->nr_sectors,
 				  &error);
-			ret = -EIO;
-			goto out;
+			return ret;
 		}
 		bitmap_or(&extent_bitmap, &extent_bitmap, &has_extent,
 			  stripe->nr_sectors);
 	}
 
 	/* Now we can check and regenerate the P/Q stripe. */
-	ret = scrub_raid56_cached_parity(sctx, scrub_dev, map, full_stripe_start,
-					 &extent_bitmap);
-out:
-	btrfs_release_path(&extent_path);
-	btrfs_release_path(&csum_path);
-	return ret;
+	return scrub_raid56_cached_parity(sctx, scrub_dev, map, full_stripe_start,
+					  &extent_bitmap);
 }
 
 /*
diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c
index 6babbe333741..7b7b7255f7d8 100644
--- a/fs/btrfs/space-info.c
+++ b/fs/btrfs/space-info.c
@@ -602,22 +602,6 @@ do {									\
 	spin_unlock(&__rsv->lock);					\
 } while (0)
 
-static const char *space_info_flag_to_str(const struct btrfs_space_info *space_info)
-{
-	switch (space_info->flags) {
-	case BTRFS_BLOCK_GROUP_SYSTEM:
-		return "SYSTEM";
-	case BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA:
-		return "DATA+METADATA";
-	case BTRFS_BLOCK_GROUP_DATA:
-		return "DATA";
-	case BTRFS_BLOCK_GROUP_METADATA:
-		return "METADATA";
-	default:
-		return "UNKNOWN";
-	}
-}
-
 static void dump_global_block_rsv(struct btrfs_fs_info *fs_info)
 {
 	DUMP_BLOCK_RSV(fs_info, global_block_rsv);
@@ -630,7 +614,7 @@ static void dump_global_block_rsv(struct btrfs_fs_info *fs_info)
 static void __btrfs_dump_space_info(const struct btrfs_space_info *info)
 {
 	const struct btrfs_fs_info *fs_info = info->fs_info;
-	const char *flag_str = space_info_flag_to_str(info);
+	const char *flag_str = btrfs_space_info_type_str(info);
 	lockdep_assert_held(&info->lock);
 
 	/* The free space could be negative in case of overcommit */
diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h
index 446c0614ad4a..0703f24b23f7 100644
--- a/fs/btrfs/space-info.h
+++ b/fs/btrfs/space-info.h
@@ -307,4 +307,20 @@ int btrfs_calc_reclaim_threshold(const struct btrfs_space_info *space_info);
 void btrfs_reclaim_sweep(const struct btrfs_fs_info *fs_info);
 void btrfs_return_free_space(struct btrfs_space_info *space_info, u64 len);
 
+static inline const char *btrfs_space_info_type_str(const struct btrfs_space_info *space_info)
+{
+	switch (space_info->flags) {
+	case BTRFS_BLOCK_GROUP_SYSTEM:
+		return "SYSTEM";
+	case BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA:
+		return "DATA+METADATA";
+	case BTRFS_BLOCK_GROUP_DATA:
+		return "DATA";
+	case BTRFS_BLOCK_GROUP_METADATA:
+		return "METADATA";
+	default:
+		return "UNKNOWN";
+	}
+}
+
 #endif /* BTRFS_SPACE_INFO_H */
diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
index 1999533b52be..c141b7e1ee81 100644
--- a/fs/btrfs/super.c
+++ b/fs/btrfs/super.c
@@ -736,14 +736,12 @@ bool btrfs_check_options(const struct btrfs_fs_info *info,
  */
 void btrfs_set_free_space_cache_settings(struct btrfs_fs_info *fs_info)
 {
-	if (fs_info->sectorsize < PAGE_SIZE) {
+	if (fs_info->sectorsize != PAGE_SIZE && btrfs_test_opt(fs_info, SPACE_CACHE)) {
+		btrfs_info(fs_info,
+			   "forcing free space tree for sector size %u with page size %lu",
+			   fs_info->sectorsize, PAGE_SIZE);
 		btrfs_clear_opt(fs_info->mount_opt, SPACE_CACHE);
-		if (!btrfs_test_opt(fs_info, FREE_SPACE_TREE)) {
-			btrfs_info(fs_info,
-				   "forcing free space tree for sector size %u with page size %lu",
-				   fs_info->sectorsize, PAGE_SIZE);
-			btrfs_set_opt(fs_info->mount_opt, FREE_SPACE_TREE);
-		}
+		btrfs_set_opt(fs_info->mount_opt, FREE_SPACE_TREE);
 	}
 
 	/*
@@ -2485,6 +2483,18 @@ static void btrfs_shutdown(struct super_block *sb)
 }
 #endif
 
+static int btrfs_show_stats(struct seq_file *s, struct dentry *root)
+{
+	struct btrfs_fs_info *fs_info = btrfs_sb(root->d_sb);
+
+	if (btrfs_is_zoned(fs_info)) {
+		btrfs_show_zoned_stats(fs_info, s);
+		return 0;
+	}
+
+	return 0;
+}
+
 static const struct super_operations btrfs_super_ops = {
 	.drop_inode	= btrfs_drop_inode,
 	.evict_inode	= btrfs_evict_inode,
@@ -2500,6 +2510,7 @@ static const struct super_operations btrfs_super_ops = {
 	.unfreeze_fs	= btrfs_unfreeze,
 	.nr_cached_objects = btrfs_nr_cached_objects,
 	.free_cached_objects = btrfs_free_cached_objects,
+	.show_stats	= btrfs_show_stats,
 #ifdef CONFIG_BTRFS_EXPERIMENTAL
 	.remove_bdev	= btrfs_remove_bdev,
 	.shutdown	= btrfs_shutdown,
@@ -2702,7 +2713,3 @@ module_exit(exit_btrfs_fs)
 
 MODULE_DESCRIPTION("B-Tree File System (BTRFS)");
 MODULE_LICENSE("GPL");
-MODULE_SOFTDEP("pre: crc32c");
-MODULE_SOFTDEP("pre: xxhash64");
-MODULE_SOFTDEP("pre: sha256");
-MODULE_SOFTDEP("pre: blake2b-256");
diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index 1f64c132b387..f0974f4c0ae4 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -11,7 +11,6 @@
 #include <linux/bug.h>
 #include <linux/list.h>
 #include <linux/string_choices.h>
-#include <crypto/hash.h>
 #include "messages.h"
 #include "ctree.h"
 #include "discard.h"
@@ -26,7 +25,6 @@
 #include "misc.h"
 #include "fs.h"
 #include "accessors.h"
-#include "zoned.h"
 
 /*
  * Structure name                       Path
@@ -1189,56 +1187,6 @@ static ssize_t btrfs_commit_stats_store(struct kobject *kobj,
 }
 BTRFS_ATTR_RW(, commit_stats, btrfs_commit_stats_show, btrfs_commit_stats_store);
 
-static ssize_t btrfs_zoned_stats_show(struct kobject *kobj,
-				      struct kobj_attribute *a, char *buf)
-{
-	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
-	struct btrfs_block_group *bg;
-	size_t ret = 0;
-
-
-	if (!btrfs_is_zoned(fs_info))
-		return ret;
-
-	spin_lock(&fs_info->zone_active_bgs_lock);
-	ret += sysfs_emit_at(buf, ret, "active block-groups: %zu\n",
-			     list_count_nodes(&fs_info->zone_active_bgs));
-	spin_unlock(&fs_info->zone_active_bgs_lock);
-
-	mutex_lock(&fs_info->reclaim_bgs_lock);
-	spin_lock(&fs_info->unused_bgs_lock);
-	ret += sysfs_emit_at(buf, ret, "\treclaimable: %zu\n",
-			     list_count_nodes(&fs_info->reclaim_bgs));
-	ret += sysfs_emit_at(buf, ret, "\tunused: %zu\n",
-			     list_count_nodes(&fs_info->unused_bgs));
-	spin_unlock(&fs_info->unused_bgs_lock);
-	mutex_unlock(&fs_info->reclaim_bgs_lock);
-
-	ret += sysfs_emit_at(buf, ret, "\tneed reclaim: %s\n",
-			     str_true_false(btrfs_zoned_should_reclaim(fs_info)));
-
-	if (fs_info->data_reloc_bg)
-		ret += sysfs_emit_at(buf, ret,
-				     "data relocation block-group: %llu\n",
-				     fs_info->data_reloc_bg);
-	if (fs_info->treelog_bg)
-		ret += sysfs_emit_at(buf, ret,
-				     "tree-log block-group: %llu\n",
-				     fs_info->treelog_bg);
-
-	spin_lock(&fs_info->zone_active_bgs_lock);
-	ret += sysfs_emit_at(buf, ret, "active zones:\n");
-	list_for_each_entry(bg, &fs_info->zone_active_bgs, active_bg_list) {
-		ret += sysfs_emit_at(buf, ret,
-				     "\tstart: %llu, wp: %llu used: %llu, reserved: %llu, unusable: %llu\n",
-				     bg->start, bg->alloc_offset, bg->used,
-				     bg->reserved, bg->zone_unusable);
-	}
-	spin_unlock(&fs_info->zone_active_bgs_lock);
-	return ret;
-}
-BTRFS_ATTR(, zoned_stats, btrfs_zoned_stats_show);
-
 static ssize_t btrfs_clone_alignment_show(struct kobject *kobj,
 				struct kobj_attribute *a, char *buf)
 {
@@ -1304,10 +1252,9 @@ static ssize_t btrfs_checksum_show(struct kobject *kobj,
 {
 	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
 	u16 csum_type = btrfs_super_csum_type(fs_info->super_copy);
+	const char *csum_name = btrfs_super_csum_name(csum_type);
 
-	return sysfs_emit(buf, "%s (%s)\n",
-			  btrfs_super_csum_name(csum_type),
-			  crypto_shash_driver_name(fs_info->csum_shash));
+	return sysfs_emit(buf, "%s (%s-lib)\n", csum_name, csum_name);
 }
 
 BTRFS_ATTR(, checksum, btrfs_checksum_show);
@@ -1651,7 +1598,6 @@ static const struct attribute *btrfs_attrs[] = {
 	BTRFS_ATTR_PTR(, bg_reclaim_threshold),
 	BTRFS_ATTR_PTR(, commit_stats),
 	BTRFS_ATTR_PTR(, temp_fsid),
-	BTRFS_ATTR_PTR(, zoned_stats),
 #ifdef CONFIG_BTRFS_EXPERIMENTAL
 	BTRFS_ATTR_PTR(, offload_csum),
 #endif
diff --git a/fs/btrfs/tests/qgroup-tests.c b/fs/btrfs/tests/qgroup-tests.c
index 05cfda8af422..e9124605974b 100644
--- a/fs/btrfs/tests/qgroup-tests.c
+++ b/fs/btrfs/tests/qgroup-tests.c
@@ -187,7 +187,6 @@ static int remove_extent_ref(struct btrfs_root *root, u64 bytenr,
 	ret = btrfs_search_slot(&trans, root, &key, path, -1, 1);
 	if (ret) {
 		test_err("couldn't find backref %d", ret);
-		btrfs_free_path(path);
 		return ret;
 	}
 	btrfs_del_item(&trans, root, path);
diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c
index 05ee4391c83a..206872d757c8 100644
--- a/fs/btrfs/transaction.c
+++ b/fs/btrfs/transaction.c
@@ -520,13 +520,14 @@ static inline int is_transaction_blocked(struct btrfs_transaction *trans)
  * when this is done, it is safe to start a new transaction, but the current
  * transaction might not be fully on disk.
  */
-static void wait_current_trans(struct btrfs_fs_info *fs_info)
+static void wait_current_trans(struct btrfs_fs_info *fs_info, unsigned int type)
 {
 	struct btrfs_transaction *cur_trans;
 
 	spin_lock(&fs_info->trans_lock);
 	cur_trans = fs_info->running_transaction;
-	if (cur_trans && is_transaction_blocked(cur_trans)) {
+	if (cur_trans && is_transaction_blocked(cur_trans) &&
+	    (btrfs_blocked_trans_types[cur_trans->state] & type)) {
 		refcount_inc(&cur_trans->use_count);
 		spin_unlock(&fs_info->trans_lock);
 
@@ -701,12 +702,12 @@ start_transaction(struct btrfs_root *root, unsigned int num_items,
 		sb_start_intwrite(fs_info->sb);
 
 	if (may_wait_transaction(fs_info, type))
-		wait_current_trans(fs_info);
+		wait_current_trans(fs_info, type);
 
 	do {
 		ret = join_transaction(fs_info, type);
 		if (ret == -EBUSY) {
-			wait_current_trans(fs_info);
+			wait_current_trans(fs_info, type);
 			if (unlikely(type == TRANS_ATTACH ||
 				     type == TRANS_JOIN_NOSTART))
 				ret = -ENOENT;
@@ -1003,7 +1004,7 @@ int btrfs_wait_for_commit(struct btrfs_fs_info *fs_info, u64 transid)
 
 void btrfs_throttle(struct btrfs_fs_info *fs_info)
 {
-	wait_current_trans(fs_info);
+	wait_current_trans(fs_info, TRANS_START);
 }
 
 bool btrfs_should_end_transaction(struct btrfs_trans_handle *trans)
@@ -1620,9 +1621,9 @@ static int qgroup_account_snapshot(struct btrfs_trans_handle *trans,
 		goto out;
 	switch_commit_roots(trans);
 	ret = btrfs_write_and_wait_transaction(trans);
-	if (ret)
-		btrfs_handle_fs_error(fs_info, ret,
-			"Error while writing out transaction for qgroup");
+	if (unlikely(ret))
+		btrfs_err(fs_info,
+"error while writing out transaction duing qgroup snapshot accounting: %d", ret);
 
 out:
 	/*
@@ -2549,9 +2550,8 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans)
 		wake_up_process(fs_info->cleaner_kthread);
 
 	ret = btrfs_write_and_wait_transaction(trans);
-	if (ret) {
-		btrfs_handle_fs_error(fs_info, ret,
-				      "Error while writing out transaction");
+	if (unlikely(ret)) {
+		btrfs_err(fs_info, "error while writing out transaction: %d", ret);
 		mutex_unlock(&fs_info->tree_log_mutex);
 		goto scrub_continue;
 	}
diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c
index fff37c8d96a4..5831754bb01c 100644
--- a/fs/btrfs/tree-log.c
+++ b/fs/btrfs/tree-log.c
@@ -5865,14 +5865,6 @@ static int log_new_dir_dentries(struct btrfs_trans_handle *trans,
 	struct btrfs_inode *curr_inode = start_inode;
 	int ret = 0;
 
-	/*
-	 * If we are logging a new name, as part of a link or rename operation,
-	 * don't bother logging new dentries, as we just want to log the names
-	 * of an inode and that any new parents exist.
-	 */
-	if (ctx->logging_new_name)
-		return 0;
-
 	path = btrfs_alloc_path();
 	if (!path)
 		return -ENOMEM;
@@ -6051,6 +6043,33 @@ static int conflicting_inode_is_dir(struct btrfs_root *root, u64 ino,
 	return ret;
 }
 
+static bool can_log_conflicting_inode(const struct btrfs_trans_handle *trans,
+				      const struct btrfs_inode *inode)
+{
+	if (!S_ISDIR(inode->vfs_inode.i_mode))
+		return true;
+
+	if (inode->last_unlink_trans < trans->transid)
+		return true;
+
+	/*
+	 * If this is a directory and its unlink_trans is not from a past
+	 * transaction then we must fallback to a transaction commit in order
+	 * to avoid getting a directory with 2 hard links after log replay.
+	 *
+	 * This happens if a directory A is renamed, moved from one parent
+	 * directory to another one, a new file is created in the old parent
+	 * directory with the old name of our directory A, the new file is
+	 * fsynced, then we moved the new file to some other parent directory
+	 * and fsync again the new file. This results in a log tree where we
+	 * logged that directory A existed, with the INODE_REF item for the
+	 * new location but without having logged its old parent inode, so
+	 * that on log replay we add a new link for the new location but the
+	 * old link remains, resulting in a link count of 2.
+	 */
+	return false;
+}
+
 static int add_conflicting_inode(struct btrfs_trans_handle *trans,
 				 struct btrfs_root *root,
 				 struct btrfs_path *path,
@@ -6154,6 +6173,11 @@ static int add_conflicting_inode(struct btrfs_trans_handle *trans,
 		return 0;
 	}
 
+	if (!can_log_conflicting_inode(trans, inode)) {
+		btrfs_add_delayed_iput(inode);
+		return BTRFS_LOG_FORCE_COMMIT;
+	}
+
 	btrfs_add_delayed_iput(inode);
 
 	ino_elem = kmalloc(sizeof(*ino_elem), GFP_NOFS);
@@ -6218,6 +6242,12 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans,
 				break;
 			}
 
+			if (!can_log_conflicting_inode(trans, inode)) {
+				btrfs_add_delayed_iput(inode);
+				ret = BTRFS_LOG_FORCE_COMMIT;
+				break;
+			}
+
 			/*
 			 * Always log the directory, we cannot make this
 			 * conditional on need_log_inode() because the directory
@@ -6311,10 +6341,8 @@ static int copy_inode_items_to_log(struct btrfs_trans_handle *trans,
 			 * and no keys greater than that, so bail out.
 			 */
 			break;
-		} else if ((min_key->type == BTRFS_INODE_REF_KEY ||
-			    min_key->type == BTRFS_INODE_EXTREF_KEY) &&
-			   (inode->generation == trans->transid ||
-			    ctx->logging_conflict_inodes)) {
+		} else if (min_key->type == BTRFS_INODE_REF_KEY ||
+			   min_key->type == BTRFS_INODE_EXTREF_KEY) {
 			u64 other_ino = 0;
 			u64 other_parent = 0;
 
diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c
index ae1742a35e76..072f01b973a1 100644
--- a/fs/btrfs/volumes.c
+++ b/fs/btrfs/volumes.c
@@ -2314,9 +2314,7 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info,
 		free_fs_devices(cur_devices);
 	}
 
-	ret = btrfs_commit_transaction(trans);
-
-	return ret;
+	return btrfs_commit_transaction(trans);
 
 error_undo:
 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
@@ -3691,7 +3689,7 @@ static int del_balance_item(struct btrfs_fs_info *fs_info)
 	struct btrfs_trans_handle *trans;
 	struct btrfs_path *path;
 	struct btrfs_key key;
-	int ret, err;
+	int ret;
 
 	path = btrfs_alloc_path();
 	if (!path)
@@ -3718,9 +3716,11 @@ static int del_balance_item(struct btrfs_fs_info *fs_info)
 	ret = btrfs_del_item(trans, root, path);
 out:
 	btrfs_free_path(path);
-	err = btrfs_commit_transaction(trans);
-	if (err && !ret)
-		ret = err;
+	if (ret == 0)
+		ret = btrfs_commit_transaction(trans);
+	else
+		btrfs_end_transaction(trans);
+
 	return ret;
 }
 
@@ -7128,6 +7128,7 @@ static struct btrfs_fs_devices *open_seed_devices(struct btrfs_fs_info *fs_info,
 
 		fs_devices->seeding = true;
 		fs_devices->opened = 1;
+		list_add(&fs_devices->seed_list, &fs_info->fs_devices->seed_list);
 		return fs_devices;
 	}
 
@@ -7164,7 +7165,6 @@ static int read_one_dev(struct extent_buffer *leaf,
 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
 	struct btrfs_device *device;
 	u64 devid;
-	int ret;
 	u8 fs_uuid[BTRFS_FSID_SIZE];
 	u8 dev_uuid[BTRFS_UUID_SIZE];
 
@@ -7263,8 +7263,8 @@ static int read_one_dev(struct extent_buffer *leaf,
 		atomic64_add(device->total_bytes - device->bytes_used,
 				&fs_info->free_chunk_space);
 	}
-	ret = 0;
-	return ret;
+
+	return 0;
 }
 
 int btrfs_read_sys_array(struct btrfs_fs_info *fs_info)
diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c
index 359a98e6de85..60543cbee6d0 100644
--- a/fs/btrfs/zoned.c
+++ b/fs/btrfs/zoned.c
@@ -2984,3 +2984,58 @@ int btrfs_reset_unused_block_groups(struct btrfs_space_info *space_info, u64 num
 
 	return 0;
 }
+
+void btrfs_show_zoned_stats(struct btrfs_fs_info *fs_info, struct seq_file *s)
+{
+	struct btrfs_block_group *bg;
+	u64 data_reloc_bg;
+	u64 treelog_bg;
+
+	seq_puts(s, "\n  zoned statistics:\n");
+
+	spin_lock(&fs_info->zone_active_bgs_lock);
+	seq_printf(s, "\tactive block-groups: %zu\n",
+			     list_count_nodes(&fs_info->zone_active_bgs));
+	spin_unlock(&fs_info->zone_active_bgs_lock);
+
+	spin_lock(&fs_info->unused_bgs_lock);
+	seq_printf(s, "\t  reclaimable: %zu\n",
+			     list_count_nodes(&fs_info->reclaim_bgs));
+	seq_printf(s, "\t  unused: %zu\n", list_count_nodes(&fs_info->unused_bgs));
+	spin_unlock(&fs_info->unused_bgs_lock);
+
+	seq_printf(s,"\t  need reclaim: %s\n",
+		   str_true_false(btrfs_zoned_should_reclaim(fs_info)));
+
+	data_reloc_bg = data_race(fs_info->data_reloc_bg);
+	if (data_reloc_bg)
+		seq_printf(s, "\tdata relocation block-group: %llu\n",
+			   data_reloc_bg);
+	treelog_bg = data_race(fs_info->treelog_bg);
+	if (treelog_bg)
+		seq_printf(s, "\ttree-log block-group: %llu\n", treelog_bg);
+
+	spin_lock(&fs_info->zone_active_bgs_lock);
+	seq_puts(s, "\tactive zones:\n");
+	list_for_each_entry(bg, &fs_info->zone_active_bgs, active_bg_list) {
+		u64 start;
+		u64 alloc_offset;
+		u64 used;
+		u64 reserved;
+		u64 zone_unusable;
+		const char *typestr = btrfs_space_info_type_str(bg->space_info);
+
+		spin_lock(&bg->lock);
+		start = bg->start;
+		alloc_offset = bg->alloc_offset;
+		used = bg->used;
+		reserved = bg->reserved;
+		zone_unusable = bg->zone_unusable;
+		spin_unlock(&bg->lock);
+
+		seq_printf(s,
+			   "\t  start: %llu, wp: %llu used: %llu, reserved: %llu, unusable: %llu (%s)\n",
+			   start, alloc_offset, used, reserved, zone_unusable, typestr);
+	}
+	spin_unlock(&fs_info->zone_active_bgs_lock);
+}
diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h
index 5cefdeb08b7b..e54b951359ea 100644
--- a/fs/btrfs/zoned.h
+++ b/fs/btrfs/zoned.h
@@ -10,6 +10,7 @@
 #include <linux/errno.h>
 #include <linux/spinlock.h>
 #include <linux/mutex.h>
+#include <linux/seq_file.h>
 #include "messages.h"
 #include "volumes.h"
 #include "disk-io.h"
@@ -96,6 +97,8 @@ int btrfs_zone_finish_one_bg(struct btrfs_fs_info *fs_info);
 int btrfs_zoned_activate_one_bg(struct btrfs_space_info *space_info, bool do_finish);
 void btrfs_check_active_zone_reservation(struct btrfs_fs_info *fs_info);
 int btrfs_reset_unused_block_groups(struct btrfs_space_info *space_info, u64 num_bytes);
+void btrfs_show_zoned_stats(struct btrfs_fs_info *fs_info, struct seq_file *s);
+
 #else /* CONFIG_BLK_DEV_ZONED */
 
 static inline int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info)
@@ -275,6 +278,11 @@ static inline int btrfs_reset_unused_block_groups(struct btrfs_space_info *space
 	return 0;
 }
 
+static inline int btrfs_show_zoned_stats(struct btrfs_fs_info *fs_info, struct seq_file *s)
+{
+	return 0;
+}
+
 #endif
 
 static inline bool btrfs_dev_is_sequential(struct btrfs_device *device, u64 pos)
diff --git a/fs/buffer.c b/fs/buffer.c
index 838c0c571022..fd53b806ab7e 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -2354,7 +2354,7 @@ bool block_is_partially_uptodate(struct folio *folio, size_t from, size_t count)
 	if (!head)
 		return false;
 	blocksize = head->b_size;
-	to = min_t(unsigned, folio_size(folio) - from, count);
+	to = min(folio_size(folio) - from, count);
 	to = from + to;
 	if (from < blocksize && to > folio_size(folio) - blocksize)
 		return false;
@@ -2948,6 +2948,10 @@ bool try_to_free_buffers(struct folio *folio)
 	if (folio_test_writeback(folio))
 		return false;
 
+	/* Misconfigured folio check */
+	if (WARN_ON_ONCE(!folio_buffers(folio)))
+		return true;
+
 	if (mapping == NULL) {		/* can this still happen? */
 		ret = drop_buffers(folio, &buffers_to_free);
 		goto out;
diff --git a/fs/char_dev.c b/fs/char_dev.c
index c2ddb998f3c9..84a5a0699373 100644
--- a/fs/char_dev.c
+++ b/fs/char_dev.c
@@ -343,7 +343,7 @@ void __unregister_chrdev(unsigned int major, unsigned int baseminor,
 	kfree(cd);
 }
 
-static DEFINE_SPINLOCK(cdev_lock);
+static __cacheline_aligned_in_smp DEFINE_SPINLOCK(cdev_lock);
 
 static struct kobject *cdev_get(struct cdev *p)
 {
diff --git a/fs/ecryptfs/crypto.c b/fs/ecryptfs/crypto.c
index 260f8a4938b0..b80f60863b5a 100644
--- a/fs/ecryptfs/crypto.c
+++ b/fs/ecryptfs/crypto.c
@@ -1220,7 +1220,7 @@ static int ecryptfs_read_headers_virt(char *page_virt,
 
 /**
  * ecryptfs_read_xattr_region
- * @page_virt: The vitual address into which to read the xattr data
+ * @page_virt: The virtual address into which to read the xattr data
  * @ecryptfs_inode: The eCryptfs inode
  *
  * Attempts to read the crypto metadata from the extended attribute
diff --git a/fs/ecryptfs/ecryptfs_kernel.h b/fs/ecryptfs/ecryptfs_kernel.h
index 62a2ea7f59ed..5088069d5b22 100644
--- a/fs/ecryptfs/ecryptfs_kernel.h
+++ b/fs/ecryptfs/ecryptfs_kernel.h
@@ -21,6 +21,7 @@
 #include <linux/kernel.h>
 #include <linux/fs.h>
 #include <linux/fs_stack.h>
+#include <linux/hex.h>
 #include <linux/namei.h>
 #include <linux/scatterlist.h>
 #include <linux/hash.h>
@@ -542,7 +543,6 @@ int ecryptfs_decode_and_decrypt_filename(char **decrypted_name,
 					 size_t *decrypted_name_size,
 					 struct super_block *sb,
 					 const char *name, size_t name_size);
-int ecryptfs_fill_zeros(struct file *file, loff_t new_length);
 int ecryptfs_encrypt_and_encode_filename(
 	char **encoded_name,
 	size_t *encoded_name_size,
diff --git a/fs/ecryptfs/keystore.c b/fs/ecryptfs/keystore.c
index bbf8603242fa..3a6283477d61 100644
--- a/fs/ecryptfs/keystore.c
+++ b/fs/ecryptfs/keystore.c
@@ -354,7 +354,7 @@ parse_tag_67_packet(struct ecryptfs_key_record *key_rec,
 	int rc;
 
 	/*
-	 *              ***** TAG 65 Packet Format *****
+	 *              ***** TAG 67 Packet Format *****
 	 *    | Content Type                       | 1 byte       |
 	 *    | Status Indicator                   | 1 byte       |
 	 *    | Encrypted File Encryption Key Size | 1 or 2 bytes |
@@ -837,7 +837,7 @@ struct ecryptfs_parse_tag_70_packet_silly_stack {
  * @filename: This function kmalloc's the memory for the filename
  * @filename_size: This function sets this to the amount of memory
  *                 kmalloc'd for the filename
- * @packet_size: This function sets this to the the number of octets
+ * @packet_size: This function sets this to the number of octets
  *               in the packet parsed
  * @mount_crypt_stat: The mount-wide cryptographic context
  * @data: The memory location containing the start of the tag 70
diff --git a/fs/efivarfs/vars.c b/fs/efivarfs/vars.c
index 6edc10958ecf..798a1bc36022 100644
--- a/fs/efivarfs/vars.c
+++ b/fs/efivarfs/vars.c
@@ -9,6 +9,7 @@
 #include <linux/capability.h>
 #include <linux/types.h>
 #include <linux/errno.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/mm.h>
 #include <linux/module.h>
diff --git a/fs/erofs/decompressor_crypto.c b/fs/erofs/decompressor_crypto.c
index 5ef6f71d3b7f..77c6bd535df3 100644
--- a/fs/erofs/decompressor_crypto.c
+++ b/fs/erofs/decompressor_crypto.c
@@ -62,7 +62,7 @@ struct z_erofs_crypto_engine {
 	struct crypto_acomp *tfm;
 };
 
-struct z_erofs_crypto_engine *z_erofs_crypto[Z_EROFS_COMPRESSION_MAX] = {
+static struct z_erofs_crypto_engine *z_erofs_crypto[Z_EROFS_COMPRESSION_MAX] = {
 	[Z_EROFS_COMPRESSION_LZ4] = (struct z_erofs_crypto_engine[]) {
 		{},
 	},
diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c
index 65da21504632..27b1f44d10ce 100644
--- a/fs/erofs/zdata.c
+++ b/fs/erofs/zdata.c
@@ -1324,8 +1324,8 @@ static int z_erofs_decompress_pcluster(struct z_erofs_backend *be, int err)
 						GFP_NOWAIT | __GFP_NORETRY
 				 }, be->pagepool);
 		if (IS_ERR(reason)) {
-			erofs_err(be->sb, "failed to decompress (%s) %ld @ pa %llu size %u => %u",
-				  alg->name, PTR_ERR(reason), pcl->pos,
+			erofs_err(be->sb, "failed to decompress (%s) %pe @ pa %llu size %u => %u",
+				  alg->name, reason, pcl->pos,
 				  pcl->pclustersize, pcl->length);
 			err = PTR_ERR(reason);
 		} else if (unlikely(reason)) {
diff --git a/fs/exec.c b/fs/exec.c
index 9d5ebc9d15b0..d0606e53376f 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -555,7 +555,7 @@ int copy_string_kernel(const char *arg, struct linux_binprm *bprm)
 		return -E2BIG;
 
 	while (len > 0) {
-		unsigned int bytes_to_copy = min_t(unsigned int, len,
+		unsigned int bytes_to_copy = min(len,
 				min_not_zero(offset_in_page(pos), PAGE_SIZE));
 		struct page *page;
 
diff --git a/fs/exfat/namei.c b/fs/exfat/namei.c
index dfe957493d49..670116ae9ec8 100644
--- a/fs/exfat/namei.c
+++ b/fs/exfat/namei.c
@@ -304,8 +304,8 @@ int exfat_find_empty_entry(struct inode *inode,
 		struct exfat_chain *p_dir, int num_entries,
 		struct exfat_entry_set_cache *es)
 {
-	int dentry;
-	unsigned int ret, last_clu;
+	int dentry, ret;
+	unsigned int last_clu;
 	loff_t size = 0;
 	struct exfat_chain clu;
 	struct super_block *sb = inode->i_sb;
diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c
index 2cf5759ba689..27eb2c1df012 100644
--- a/fs/ext4/extents.c
+++ b/fs/ext4/extents.c
@@ -43,8 +43,13 @@
 #define EXT4_EXT_MARK_UNWRIT1	0x2  /* mark first half unwritten */
 #define EXT4_EXT_MARK_UNWRIT2	0x4  /* mark second half unwritten */
 
-#define EXT4_EXT_DATA_VALID1	0x8  /* first half contains valid data */
-#define EXT4_EXT_DATA_VALID2	0x10 /* second half contains valid data */
+/* first half contains valid data */
+#define EXT4_EXT_DATA_ENTIRE_VALID1	0x8   /* has entirely valid data */
+#define EXT4_EXT_DATA_PARTIAL_VALID1	0x10  /* has partially valid data */
+#define EXT4_EXT_DATA_VALID1		(EXT4_EXT_DATA_ENTIRE_VALID1 | \
+					 EXT4_EXT_DATA_PARTIAL_VALID1)
+
+#define EXT4_EXT_DATA_VALID2	0x20 /* second half contains valid data */
 
 static __le32 ext4_extent_block_csum(struct inode *inode,
 				     struct ext4_extent_header *eh)
@@ -3190,8 +3195,12 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle,
 	unsigned int ee_len, depth;
 	int err = 0;
 
-	BUG_ON((split_flag & (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2)) ==
-	       (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2));
+	BUG_ON((split_flag & EXT4_EXT_DATA_VALID1) == EXT4_EXT_DATA_VALID1);
+	BUG_ON((split_flag & EXT4_EXT_DATA_VALID1) &&
+	       (split_flag & EXT4_EXT_DATA_VALID2));
+
+	/* Do not cache extents that are in the process of being modified. */
+	flags |= EXT4_EX_NOCACHE;
 
 	ext_debug(inode, "logical block %llu\n", (unsigned long long)split);
 
@@ -3258,7 +3267,7 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle,
 
 	err = PTR_ERR(path);
 	if (err != -ENOSPC && err != -EDQUOT && err != -ENOMEM)
-		return path;
+		goto out_path;
 
 	/*
 	 * Get a new path to try to zeroout or fix the extent length.
@@ -3272,52 +3281,54 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle,
 	if (IS_ERR(path)) {
 		EXT4_ERROR_INODE(inode, "Failed split extent on %u, err %ld",
 				 split, PTR_ERR(path));
-		return path;
+		goto out_path;
 	}
 	depth = ext_depth(inode);
 	ex = path[depth].p_ext;
 
 	if (EXT4_EXT_MAY_ZEROOUT & split_flag) {
-		if (split_flag & (EXT4_EXT_DATA_VALID1|EXT4_EXT_DATA_VALID2)) {
-			if (split_flag & EXT4_EXT_DATA_VALID1) {
-				err = ext4_ext_zeroout(inode, ex2);
-				zero_ex.ee_block = ex2->ee_block;
-				zero_ex.ee_len = cpu_to_le16(
-						ext4_ext_get_actual_len(ex2));
-				ext4_ext_store_pblock(&zero_ex,
-						      ext4_ext_pblock(ex2));
-			} else {
-				err = ext4_ext_zeroout(inode, ex);
-				zero_ex.ee_block = ex->ee_block;
-				zero_ex.ee_len = cpu_to_le16(
-						ext4_ext_get_actual_len(ex));
-				ext4_ext_store_pblock(&zero_ex,
-						      ext4_ext_pblock(ex));
-			}
-		} else {
-			err = ext4_ext_zeroout(inode, &orig_ex);
-			zero_ex.ee_block = orig_ex.ee_block;
-			zero_ex.ee_len = cpu_to_le16(
-						ext4_ext_get_actual_len(&orig_ex));
-			ext4_ext_store_pblock(&zero_ex,
-					      ext4_ext_pblock(&orig_ex));
-		}
+		if (split_flag & EXT4_EXT_DATA_VALID1)
+			memcpy(&zero_ex, ex2, sizeof(zero_ex));
+		else if (split_flag & EXT4_EXT_DATA_VALID2)
+			memcpy(&zero_ex, ex, sizeof(zero_ex));
+		else
+			memcpy(&zero_ex, &orig_ex, sizeof(zero_ex));
+		ext4_ext_mark_initialized(&zero_ex);
 
-		if (!err) {
-			/* update the extent length and mark as initialized */
-			ex->ee_len = cpu_to_le16(ee_len);
-			ext4_ext_try_to_merge(handle, inode, path, ex);
-			err = ext4_ext_dirty(handle, inode, path + path->p_depth);
-			if (!err)
-				/* update extent status tree */
-				ext4_zeroout_es(inode, &zero_ex);
-			/* If we failed at this point, we don't know in which
-			 * state the extent tree exactly is so don't try to fix
-			 * length of the original extent as it may do even more
-			 * damage.
+		err = ext4_ext_zeroout(inode, &zero_ex);
+		if (err)
+			goto fix_extent_len;
+
+		/*
+		 * The first half contains partially valid data, the splitting
+		 * of this extent has not been completed, fix extent length
+		 * and ext4_split_extent() split will the first half again.
+		 */
+		if (split_flag & EXT4_EXT_DATA_PARTIAL_VALID1) {
+			/*
+			 * Drop extent cache to prevent stale unwritten
+			 * extents remaining after zeroing out.
 			 */
-			goto out;
+			ext4_es_remove_extent(inode,
+					le32_to_cpu(zero_ex.ee_block),
+					ext4_ext_get_actual_len(&zero_ex));
+			goto fix_extent_len;
 		}
+
+		/* update the extent length and mark as initialized */
+		ex->ee_len = cpu_to_le16(ee_len);
+		ext4_ext_try_to_merge(handle, inode, path, ex);
+		err = ext4_ext_dirty(handle, inode, path + path->p_depth);
+		if (!err)
+			/* update extent status tree */
+			ext4_zeroout_es(inode, &zero_ex);
+		/*
+		 * If we failed at this point, we don't know in which
+		 * state the extent tree exactly is so don't try to fix
+		 * length of the original extent as it may do even more
+		 * damage.
+		 */
+		goto out;
 	}
 
 fix_extent_len:
@@ -3332,6 +3343,10 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle,
 		ext4_free_ext_path(path);
 		path = ERR_PTR(err);
 	}
+out_path:
+	if (IS_ERR(path))
+		/* Remove all remaining potentially stale extents. */
+		ext4_es_remove_extent(inode, ee_block, ee_len);
 	ext4_ext_show_leaf(inode, path);
 	return path;
 }
@@ -3366,6 +3381,9 @@ static struct ext4_ext_path *ext4_split_extent(handle_t *handle,
 	ee_len = ext4_ext_get_actual_len(ex);
 	unwritten = ext4_ext_is_unwritten(ex);
 
+	/* Do not cache extents that are in the process of being modified. */
+	flags |= EXT4_EX_NOCACHE;
+
 	if (map->m_lblk + map->m_len < ee_block + ee_len) {
 		split_flag1 = split_flag & EXT4_EXT_MAY_ZEROOUT;
 		flags1 = flags | EXT4_GET_BLOCKS_SPLIT_NOMERGE;
@@ -3373,7 +3391,9 @@ static struct ext4_ext_path *ext4_split_extent(handle_t *handle,
 			split_flag1 |= EXT4_EXT_MARK_UNWRIT1 |
 				       EXT4_EXT_MARK_UNWRIT2;
 		if (split_flag & EXT4_EXT_DATA_VALID2)
-			split_flag1 |= EXT4_EXT_DATA_VALID1;
+			split_flag1 |= map->m_lblk > ee_block ?
+				       EXT4_EXT_DATA_PARTIAL_VALID1 :
+				       EXT4_EXT_DATA_ENTIRE_VALID1;
 		path = ext4_split_extent_at(handle, inode, path,
 				map->m_lblk + map->m_len, split_flag1, flags1);
 		if (IS_ERR(path))
@@ -3728,16 +3748,20 @@ static struct ext4_ext_path *ext4_split_convert_extents(handle_t *handle,
 
 	/* Convert to unwritten */
 	if (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN) {
-		split_flag |= EXT4_EXT_DATA_VALID1;
-	/* Convert to initialized */
-	} else if (flags & EXT4_GET_BLOCKS_CONVERT) {
+		split_flag |= EXT4_EXT_DATA_ENTIRE_VALID1;
+	/* Split the existing unwritten extent */
+	} else if (flags & (EXT4_GET_BLOCKS_UNWRIT_EXT |
+			    EXT4_GET_BLOCKS_CONVERT)) {
 		/*
 		 * It is safe to convert extent to initialized via explicit
 		 * zeroout only if extent is fully inside i_size or new_size.
 		 */
 		split_flag |= ee_block + ee_len <= eof_block ?
 			      EXT4_EXT_MAY_ZEROOUT : 0;
-		split_flag |= (EXT4_EXT_MARK_UNWRIT2 | EXT4_EXT_DATA_VALID2);
+		split_flag |= EXT4_EXT_MARK_UNWRIT2;
+		/* Convert to initialized */
+		if (flags & EXT4_GET_BLOCKS_CONVERT)
+			split_flag |= EXT4_EXT_DATA_VALID2;
 	}
 	flags |= EXT4_GET_BLOCKS_SPLIT_NOMERGE;
 	return ext4_split_extent(handle, inode, path, map, split_flag, flags,
@@ -3912,8 +3936,10 @@ ext4_ext_handle_unwritten_extents(handle_t *handle, struct inode *inode,
 
 	/* get_block() before submitting IO, split the extent */
 	if (flags & EXT4_GET_BLOCKS_SPLIT_NOMERGE) {
+		int depth;
+
 		path = ext4_split_convert_extents(handle, inode, map, path,
-				flags | EXT4_GET_BLOCKS_CONVERT, allocated);
+						  flags, allocated);
 		if (IS_ERR(path))
 			return path;
 		/*
@@ -3927,7 +3953,13 @@ ext4_ext_handle_unwritten_extents(handle_t *handle, struct inode *inode,
 			err = -EFSCORRUPTED;
 			goto errout;
 		}
-		map->m_flags |= EXT4_MAP_UNWRITTEN;
+		/* Don't mark unwritten if the extent has been zeroed out. */
+		path = ext4_find_extent(inode, map->m_lblk, path, flags);
+		if (IS_ERR(path))
+			return path;
+		depth = ext_depth(inode);
+		if (ext4_ext_is_unwritten(path[depth].p_ext))
+			map->m_flags |= EXT4_MAP_UNWRITTEN;
 		goto out;
 	}
 	/* IO end_io complete, convert the filled extent to written */
@@ -4160,8 +4192,7 @@ static ext4_lblk_t ext4_ext_determine_insert_hole(struct inode *inode,
 insert_hole:
 	/* Put just found gap into cache to speed up subsequent requests */
 	ext_debug(inode, " -> %u:%u\n", hole_start, len);
-	ext4_es_insert_extent(inode, hole_start, len, ~0,
-			      EXTENT_STATUS_HOLE, false);
+	ext4_es_cache_extent(inode, hole_start, len, ~0, EXTENT_STATUS_HOLE);
 
 	/* Update hole_len to reflect hole size after lblk */
 	if (hole_start != lblk)
diff --git a/fs/ext4/extents_status.c b/fs/ext4/extents_status.c
index e04fbf10fe4f..fc83e7e2ca9e 100644
--- a/fs/ext4/extents_status.c
+++ b/fs/ext4/extents_status.c
@@ -178,7 +178,8 @@ static struct kmem_cache *ext4_pending_cachep;
 static int __es_insert_extent(struct inode *inode, struct extent_status *newes,
 			      struct extent_status *prealloc);
 static int __es_remove_extent(struct inode *inode, ext4_lblk_t lblk,
-			      ext4_lblk_t end, int *reserved,
+			      ext4_lblk_t end, unsigned int status,
+			      int *reserved, struct extent_status *res,
 			      struct extent_status *prealloc);
 static int es_reclaim_extents(struct ext4_inode_info *ei, int *nr_to_scan);
 static int __es_shrink(struct ext4_sb_info *sbi, int nr_to_scan,
@@ -242,6 +243,21 @@ static inline void ext4_es_inc_seq(struct inode *inode)
 	WRITE_ONCE(ei->i_es_seq, ei->i_es_seq + 1);
 }
 
+static inline int __es_check_extent_status(struct extent_status *es,
+					   unsigned int status,
+					   struct extent_status *res)
+{
+	if (ext4_es_type(es) & status)
+		return 0;
+
+	if (res) {
+		res->es_lblk = es->es_lblk;
+		res->es_len = es->es_len;
+		res->es_pblk = es->es_pblk;
+	}
+	return -EINVAL;
+}
+
 /*
  * search through the tree for an delayed extent with a given offset.  If
  * it can't be found, try to find next extent.
@@ -882,7 +898,8 @@ static int __es_insert_extent(struct inode *inode, struct extent_status *newes,
 
 /*
  * ext4_es_insert_extent() adds information to an inode's extent
- * status tree.
+ * status tree. This interface is used for modifying extents. To cache
+ * on-disk extents, use ext4_es_cache_extent() instead.
  */
 void ext4_es_insert_extent(struct inode *inode, ext4_lblk_t lblk,
 			   ext4_lblk_t len, ext4_fsblk_t pblk,
@@ -929,7 +946,7 @@ void ext4_es_insert_extent(struct inode *inode, ext4_lblk_t lblk,
 		pr = __alloc_pending(true);
 	write_lock(&EXT4_I(inode)->i_es_lock);
 
-	err1 = __es_remove_extent(inode, lblk, end, &resv_used, es1);
+	err1 = __es_remove_extent(inode, lblk, end, 0, &resv_used, NULL, es1);
 	if (err1 != 0)
 		goto error;
 	/* Free preallocated extent if it didn't get used. */
@@ -961,10 +978,6 @@ void ext4_es_insert_extent(struct inode *inode, ext4_lblk_t lblk,
 		}
 		pending = err3;
 	}
-	/*
-	 * TODO: For cache on-disk extents, there is no need to increment
-	 * the sequence counter, this requires future optimization.
-	 */
 	ext4_es_inc_seq(inode);
 error:
 	write_unlock(&EXT4_I(inode)->i_es_lock);
@@ -998,17 +1011,24 @@ void ext4_es_insert_extent(struct inode *inode, ext4_lblk_t lblk,
 }
 
 /*
- * ext4_es_cache_extent() inserts information into the extent status
- * tree if and only if there isn't information about the range in
- * question already.
+ * ext4_es_cache_extent() inserts information into the extent status tree
+ * only if there is no existing information about the specified range or
+ * if the existing extents have the same status.
+ *
+ * Note that this interface is only used for caching on-disk extent
+ * information and cannot be used to convert existing extents in the extent
+ * status tree. To convert existing extents, use ext4_es_insert_extent()
+ * instead.
  */
 void ext4_es_cache_extent(struct inode *inode, ext4_lblk_t lblk,
 			  ext4_lblk_t len, ext4_fsblk_t pblk,
 			  unsigned int status)
 {
 	struct extent_status *es;
-	struct extent_status newes;
+	struct extent_status chkes, newes;
 	ext4_lblk_t end = lblk + len - 1;
+	bool conflict = false;
+	int err;
 
 	if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
 		return;
@@ -1016,7 +1036,6 @@ void ext4_es_cache_extent(struct inode *inode, ext4_lblk_t lblk,
 	newes.es_lblk = lblk;
 	newes.es_len = len;
 	ext4_es_store_pblock_status(&newes, pblk, status);
-	trace_ext4_es_cache_extent(inode, &newes);
 
 	if (!len)
 		return;
@@ -1024,11 +1043,42 @@ void ext4_es_cache_extent(struct inode *inode, ext4_lblk_t lblk,
 	BUG_ON(end < lblk);
 
 	write_lock(&EXT4_I(inode)->i_es_lock);
-
 	es = __es_tree_search(&EXT4_I(inode)->i_es_tree.root, lblk);
-	if (!es || es->es_lblk > end)
-		__es_insert_extent(inode, &newes, NULL);
+	if (es && es->es_lblk <= end) {
+		/* Found an extent that covers the entire range. */
+		if (es->es_lblk <= lblk && es->es_lblk + es->es_len > end) {
+			if (__es_check_extent_status(es, status, &chkes))
+				conflict = true;
+			goto unlock;
+		}
+		/* Check and remove all extents in range. */
+		err = __es_remove_extent(inode, lblk, end, status, NULL,
+					 &chkes, NULL);
+		if (err) {
+			if (err == -EINVAL)
+				conflict = true;
+			goto unlock;
+		}
+	}
+	__es_insert_extent(inode, &newes, NULL);
+	trace_ext4_es_cache_extent(inode, &newes);
+	ext4_es_print_tree(inode);
+unlock:
 	write_unlock(&EXT4_I(inode)->i_es_lock);
+	if (!conflict)
+		return;
+	/*
+	 * A hole in the on-disk extent but a delayed extent in the extent
+	 * status tree, is allowed.
+	 */
+	if (status == EXTENT_STATUS_HOLE &&
+	    ext4_es_type(&chkes) == EXTENT_STATUS_DELAYED)
+		return;
+
+	ext4_warning_inode(inode,
+			   "ES cache extent failed: add [%d,%d,%llu,0x%x] conflict with existing [%d,%d,%llu,0x%x]\n",
+			   lblk, len, pblk, status, chkes.es_lblk, chkes.es_len,
+			   ext4_es_pblock(&chkes), ext4_es_status(&chkes));
 }
 
 /*
@@ -1409,23 +1459,27 @@ static unsigned int get_rsvd(struct inode *inode, ext4_lblk_t end,
 	return rc->ndelayed;
 }
 
-
 /*
  * __es_remove_extent - removes block range from extent status tree
  *
  * @inode - file containing range
  * @lblk - first block in range
  * @end - last block in range
+ * @status - the extent status to be checked
  * @reserved - number of cluster reservations released
+ * @res - return the extent if the status is not match
  * @prealloc - pre-allocated es to avoid memory allocation failures
  *
  * If @reserved is not NULL and delayed allocation is enabled, counts
  * block/cluster reservations freed by removing range and if bigalloc
- * enabled cancels pending reservations as needed. Returns 0 on success,
- * error code on failure.
+ * enabled cancels pending reservations as needed. If @status is not
+ * zero, check extent status type while removing extent, return -EINVAL
+ * and pass out the extent through @res if not match.  Returns 0 on
+ * success, error code on failure.
  */
 static int __es_remove_extent(struct inode *inode, ext4_lblk_t lblk,
-			      ext4_lblk_t end, int *reserved,
+			      ext4_lblk_t end, unsigned int status,
+			      int *reserved, struct extent_status *res,
 			      struct extent_status *prealloc)
 {
 	struct ext4_es_tree *tree = &EXT4_I(inode)->i_es_tree;
@@ -1434,18 +1488,24 @@ static int __es_remove_extent(struct inode *inode, ext4_lblk_t lblk,
 	struct extent_status orig_es;
 	ext4_lblk_t len1, len2;
 	ext4_fsblk_t block;
-	int err = 0;
+	int err;
 	bool count_reserved = true;
 	struct rsvd_count rc;
 
 	if (reserved == NULL || !test_opt(inode->i_sb, DELALLOC))
 		count_reserved = false;
+	if (status == 0)
+		status = ES_TYPE_MASK;
 
 	es = __es_tree_search(&tree->root, lblk);
 	if (!es)
-		goto out;
+		return 0;
 	if (es->es_lblk > end)
-		goto out;
+		return 0;
+
+	err = __es_check_extent_status(es, status, res);
+	if (err)
+		return err;
 
 	/* Simply invalidate cache_es. */
 	tree->cache_es = NULL;
@@ -1480,7 +1540,7 @@ static int __es_remove_extent(struct inode *inode, ext4_lblk_t lblk,
 
 				es->es_lblk = orig_es.es_lblk;
 				es->es_len = orig_es.es_len;
-				goto out;
+				return err;
 			}
 		} else {
 			es->es_lblk = end + 1;
@@ -1494,7 +1554,7 @@ static int __es_remove_extent(struct inode *inode, ext4_lblk_t lblk,
 		if (count_reserved)
 			count_rsvd(inode, orig_es.es_lblk + len1,
 				   orig_es.es_len - len1 - len2, &orig_es, &rc);
-		goto out_get_reserved;
+		goto out;
 	}
 
 	if (len1 > 0) {
@@ -1509,6 +1569,9 @@ static int __es_remove_extent(struct inode *inode, ext4_lblk_t lblk,
 	}
 
 	while (es && ext4_es_end(es) <= end) {
+		err = __es_check_extent_status(es, status, res);
+		if (err)
+			return err;
 		if (count_reserved)
 			count_rsvd(inode, es->es_lblk, es->es_len, es, &rc);
 		node = rb_next(&es->rb_node);
@@ -1524,6 +1587,10 @@ static int __es_remove_extent(struct inode *inode, ext4_lblk_t lblk,
 	if (es && es->es_lblk < end + 1) {
 		ext4_lblk_t orig_len = es->es_len;
 
+		err = __es_check_extent_status(es, status, res);
+		if (err)
+			return err;
+
 		len1 = ext4_es_end(es) - end;
 		if (count_reserved)
 			count_rsvd(inode, es->es_lblk, orig_len - len1,
@@ -1536,11 +1603,10 @@ static int __es_remove_extent(struct inode *inode, ext4_lblk_t lblk,
 		}
 	}
 
-out_get_reserved:
+out:
 	if (count_reserved)
 		*reserved = get_rsvd(inode, end, es, &rc);
-out:
-	return err;
+	return 0;
 }
 
 /*
@@ -1582,7 +1648,7 @@ void ext4_es_remove_extent(struct inode *inode, ext4_lblk_t lblk,
 	 * is reclaimed.
 	 */
 	write_lock(&EXT4_I(inode)->i_es_lock);
-	err = __es_remove_extent(inode, lblk, end, &reserved, es);
+	err = __es_remove_extent(inode, lblk, end, 0, &reserved, NULL, es);
 	if (err)
 		goto error;
 	/* Free preallocated extent if it didn't get used. */
@@ -2174,7 +2240,7 @@ void ext4_es_insert_delayed_extent(struct inode *inode, ext4_lblk_t lblk,
 	}
 	write_lock(&EXT4_I(inode)->i_es_lock);
 
-	err1 = __es_remove_extent(inode, lblk, end, NULL, es1);
+	err1 = __es_remove_extent(inode, lblk, end, 0, NULL, NULL, es1);
 	if (err1 != 0)
 		goto error;
 	/* Free preallocated extent if it didn't get used. */
diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
index 0c466ccbed69..08a7f2b54a9e 100644
--- a/fs/ext4/inode.c
+++ b/fs/ext4/inode.c
@@ -503,8 +503,8 @@ static int ext4_map_query_blocks_next_in_leaf(handle_t *handle,
 	retval = ext4_ext_map_blocks(handle, inode, &map2, 0);
 
 	if (retval <= 0) {
-		ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
-				      map->m_pblk, status, false);
+		ext4_es_cache_extent(inode, map->m_lblk, map->m_len,
+				     map->m_pblk, status);
 		return map->m_len;
 	}
 
@@ -525,13 +525,13 @@ static int ext4_map_query_blocks_next_in_leaf(handle_t *handle,
 	 */
 	if (map->m_pblk + map->m_len == map2.m_pblk &&
 			status == status2) {
-		ext4_es_insert_extent(inode, map->m_lblk,
-				      map->m_len + map2.m_len, map->m_pblk,
-				      status, false);
+		ext4_es_cache_extent(inode, map->m_lblk,
+				     map->m_len + map2.m_len, map->m_pblk,
+				     status);
 		map->m_len += map2.m_len;
 	} else {
-		ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
-				      map->m_pblk, status, false);
+		ext4_es_cache_extent(inode, map->m_lblk, map->m_len,
+				     map->m_pblk, status);
 	}
 
 	return map->m_len;
@@ -573,8 +573,8 @@ static int ext4_map_query_blocks(handle_t *handle, struct inode *inode,
 			map->m_len == orig_mlen) {
 		status = map->m_flags & EXT4_MAP_UNWRITTEN ?
 				EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
-		ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
-				      map->m_pblk, status, false);
+		ext4_es_cache_extent(inode, map->m_lblk, map->m_len,
+				     map->m_pblk, status);
 	} else {
 		retval = ext4_map_query_blocks_next_in_leaf(handle, inode, map,
 							    orig_mlen);
diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c
index 56d50fd3310b..e817a758801d 100644
--- a/fs/ext4/mballoc.c
+++ b/fs/ext4/mballoc.c
@@ -4276,8 +4276,7 @@ void ext4_mb_mark_bb(struct super_block *sb, ext4_fsblk_t block,
 		 * get the corresponding group metadata to work with.
 		 * For this we have goto again loop.
 		 */
-		thisgrp_len = min_t(unsigned int, (unsigned int)len,
-			EXT4_BLOCKS_PER_GROUP(sb) - EXT4_C2B(sbi, blkoff));
+		thisgrp_len = min(len, EXT4_BLOCKS_PER_GROUP(sb) - EXT4_C2B(sbi, blkoff));
 		clen = EXT4_NUM_B2C(sbi, thisgrp_len);
 
 		if (!ext4_sb_block_valid(sb, NULL, block, thisgrp_len)) {
diff --git a/fs/ext4/resize.c b/fs/ext4/resize.c
index 050f26168d97..76842f0957b5 100644
--- a/fs/ext4/resize.c
+++ b/fs/ext4/resize.c
@@ -1479,7 +1479,7 @@ static void ext4_update_super(struct super_block *sb,
 
 	/* Update the global fs size fields */
 	sbi->s_groups_count += flex_gd->count;
-	sbi->s_blockfile_groups = min_t(ext4_group_t, sbi->s_groups_count,
+	sbi->s_blockfile_groups = min(sbi->s_groups_count,
 			(EXT4_MAX_BLOCK_FILE_PHYS / EXT4_BLOCKS_PER_GROUP(sb)));
 
 	/* Update the reserved block counts only once the new group is
diff --git a/fs/ext4/super.c b/fs/ext4/super.c
index 87205660c5d0..79762c3e0dff 100644
--- a/fs/ext4/super.c
+++ b/fs/ext4/super.c
@@ -4832,7 +4832,7 @@ static int ext4_check_geometry(struct super_block *sb,
 		return -EINVAL;
 	}
 	sbi->s_groups_count = blocks_count;
-	sbi->s_blockfile_groups = min_t(ext4_group_t, sbi->s_groups_count,
+	sbi->s_blockfile_groups = min(sbi->s_groups_count,
 			(EXT4_MAX_BLOCK_FILE_PHYS / EXT4_BLOCKS_PER_GROUP(sb)));
 	if (((u64)sbi->s_groups_count * sbi->s_inodes_per_group) !=
 	    le32_to_cpu(es->s_inodes_count)) {
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index c30e69392a62..12bf4b6e0075 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -31,9 +31,15 @@
 
 static struct kmem_cache *bio_post_read_ctx_cache;
 static struct kmem_cache *bio_entry_slab;
+static struct kmem_cache *ffs_entry_slab;
 static mempool_t *bio_post_read_ctx_pool;
 static struct bio_set f2fs_bioset;
 
+struct f2fs_folio_state {
+	spinlock_t		state_lock;
+	unsigned int		read_pages_pending;
+};
+
 #define	F2FS_BIO_POOL_SIZE	NR_CURSEG_TYPE
 
 int __init f2fs_init_bioset(void)
@@ -138,11 +144,15 @@ static void f2fs_finish_read_bio(struct bio *bio, bool in_task)
 {
 	struct folio_iter fi;
 	struct bio_post_read_ctx *ctx = bio->bi_private;
+	unsigned long flags;
 
 	bio_for_each_folio_all(fi, bio) {
 		struct folio *folio = fi.folio;
+		unsigned nr_pages = fi.length >> PAGE_SHIFT;
+		bool finished = true;
 
-		if (f2fs_is_compressed_page(folio)) {
+		if (!folio_test_large(folio) &&
+		    f2fs_is_compressed_page(folio)) {
 			if (ctx && !ctx->decompression_attempted)
 				f2fs_end_read_compressed_page(folio, true, 0,
 							in_task);
@@ -150,8 +160,20 @@ static void f2fs_finish_read_bio(struct bio *bio, bool in_task)
 			continue;
 		}
 
-		dec_page_count(F2FS_F_SB(folio), __read_io_type(folio));
-		folio_end_read(folio, bio->bi_status == BLK_STS_OK);
+		if (folio_test_large(folio)) {
+			struct f2fs_folio_state *ffs = folio->private;
+
+			spin_lock_irqsave(&ffs->state_lock, flags);
+			ffs->read_pages_pending -= nr_pages;
+			finished = !ffs->read_pages_pending;
+			spin_unlock_irqrestore(&ffs->state_lock, flags);
+		}
+
+		while (nr_pages--)
+			dec_page_count(F2FS_F_SB(folio), __read_io_type(folio));
+
+		if (finished)
+			folio_end_read(folio, bio->bi_status == BLK_STS_OK);
 	}
 
 	if (ctx)
@@ -509,6 +531,9 @@ static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode,
 void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio,
 				 enum page_type type)
 {
+	if (!bio)
+		return;
+
 	WARN_ON_ONCE(!is_read_io(bio_op(bio)));
 	trace_f2fs_submit_read_bio(sbi->sb, type, bio);
 
@@ -1200,11 +1225,21 @@ struct folio *f2fs_get_read_data_folio(struct inode *inode, pgoff_t index,
 	struct dnode_of_data dn;
 	struct folio *folio;
 	int err;
-
+retry:
 	folio = f2fs_grab_cache_folio(mapping, index, for_write);
 	if (IS_ERR(folio))
 		return folio;
 
+	if (folio_test_large(folio)) {
+		pgoff_t folio_index = mapping_align_index(mapping, index);
+
+		f2fs_folio_put(folio, true);
+		invalidate_inode_pages2_range(mapping, folio_index,
+				folio_index + folio_nr_pages(folio) - 1);
+		f2fs_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT);
+		goto retry;
+	}
+
 	if (f2fs_lookup_read_extent_cache_block(inode, index,
 						&dn.data_blkaddr)) {
 		if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), dn.data_blkaddr,
@@ -2332,6 +2367,180 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
 }
 #endif
 
+static struct f2fs_folio_state *ffs_find_or_alloc(struct folio *folio)
+{
+	struct f2fs_folio_state *ffs = folio->private;
+
+	if (ffs)
+		return ffs;
+
+	ffs = f2fs_kmem_cache_alloc(ffs_entry_slab, GFP_NOIO, true, NULL);
+
+	spin_lock_init(&ffs->state_lock);
+	folio_attach_private(folio, ffs);
+	return ffs;
+}
+
+static void ffs_detach_free(struct folio *folio)
+{
+	struct f2fs_folio_state *ffs;
+
+	if (!folio_test_large(folio)) {
+		folio_detach_private(folio);
+		return;
+	}
+
+	ffs = folio_detach_private(folio);
+	if (!ffs)
+		return;
+
+	WARN_ON_ONCE(ffs->read_pages_pending != 0);
+	kmem_cache_free(ffs_entry_slab, ffs);
+}
+
+static int f2fs_read_data_large_folio(struct inode *inode,
+		struct readahead_control *rac, struct folio *folio)
+{
+	struct bio *bio = NULL;
+	sector_t last_block_in_bio = 0;
+	struct f2fs_map_blocks map = {0, };
+	pgoff_t index, offset;
+	unsigned max_nr_pages = rac ? readahead_count(rac) :
+				folio_nr_pages(folio);
+	unsigned nrpages;
+	struct f2fs_folio_state *ffs;
+	int ret = 0;
+
+	if (!IS_IMMUTABLE(inode))
+		return -EOPNOTSUPP;
+
+	if (f2fs_compressed_file(inode))
+		return -EOPNOTSUPP;
+
+	map.m_seg_type = NO_CHECK_TYPE;
+
+	if (rac)
+		folio = readahead_folio(rac);
+next_folio:
+	if (!folio)
+		goto out;
+
+	index = folio->index;
+	offset = 0;
+	ffs = NULL;
+	nrpages = folio_nr_pages(folio);
+
+	for (; nrpages; nrpages--) {
+		sector_t block_nr;
+		/*
+		 * Map blocks using the previous result first.
+		 */
+		if ((map.m_flags & F2FS_MAP_MAPPED) &&
+				index > map.m_lblk &&
+				index < (map.m_lblk + map.m_len))
+			goto got_it;
+
+		/*
+		 * Then do more f2fs_map_blocks() calls until we are
+		 * done with this page.
+		 */
+		memset(&map, 0, sizeof(map));
+		map.m_seg_type = NO_CHECK_TYPE;
+		map.m_lblk = index;
+		map.m_len = max_nr_pages;
+
+		ret = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DEFAULT);
+		if (ret)
+			goto err_out;
+got_it:
+		if ((map.m_flags & F2FS_MAP_MAPPED)) {
+			block_nr = map.m_pblk + index - map.m_lblk;
+			if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr,
+						DATA_GENERIC_ENHANCE_READ)) {
+				ret = -EFSCORRUPTED;
+				goto err_out;
+			}
+		} else {
+			folio_zero_range(folio, offset << PAGE_SHIFT, PAGE_SIZE);
+			if (f2fs_need_verity(inode, index) &&
+			    !fsverity_verify_page(folio_file_page(folio,
+								index))) {
+				ret = -EIO;
+				goto err_out;
+			}
+			continue;
+		}
+
+		/*
+		 * This page will go to BIO.  Do we need to send this
+		 * BIO off first?
+		 */
+		if (bio && (!page_is_mergeable(F2FS_I_SB(inode), bio,
+						last_block_in_bio, block_nr) ||
+			!f2fs_crypt_mergeable_bio(bio, inode, index, NULL))) {
+submit_and_realloc:
+			f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA);
+			bio = NULL;
+		}
+		if (bio == NULL)
+			bio = f2fs_grab_read_bio(inode, block_nr,
+					max_nr_pages,
+					f2fs_ra_op_flags(rac),
+					index, false);
+
+		/*
+		 * If the page is under writeback, we need to wait for
+		 * its completion to see the correct decrypted data.
+		 */
+		f2fs_wait_on_block_writeback(inode, block_nr);
+
+		if (!bio_add_folio(bio, folio, F2FS_BLKSIZE,
+					offset << PAGE_SHIFT))
+			goto submit_and_realloc;
+
+		if (folio_test_large(folio)) {
+			ffs = ffs_find_or_alloc(folio);
+
+			/* set the bitmap to wait */
+			spin_lock_irq(&ffs->state_lock);
+			ffs->read_pages_pending++;
+			spin_unlock_irq(&ffs->state_lock);
+		}
+
+		inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA);
+		f2fs_update_iostat(F2FS_I_SB(inode), NULL, FS_DATA_READ_IO,
+				F2FS_BLKSIZE);
+		last_block_in_bio = block_nr;
+		index++;
+		offset++;
+	}
+	trace_f2fs_read_folio(folio, DATA);
+	if (rac) {
+		folio = readahead_folio(rac);
+		goto next_folio;
+	}
+err_out:
+	/* Nothing was submitted. */
+	if (!bio) {
+		if (!ret)
+			folio_mark_uptodate(folio);
+		folio_unlock(folio);
+		return ret;
+	}
+
+	if (ret) {
+		f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA);
+
+		/* Wait bios and clear uptodate. */
+		folio_lock(folio);
+		folio_clear_uptodate(folio);
+		folio_unlock(folio);
+	}
+out:
+	f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA);
+	return ret;
+}
+
 /*
  * This function was originally taken from fs/mpage.c, and customized for f2fs.
  * Major change was from block_size == page_size in f2fs by default.
@@ -2357,9 +2566,13 @@ static int f2fs_mpage_readpages(struct inode *inode,
 	pgoff_t index;
 #endif
 	unsigned nr_pages = rac ? readahead_count(rac) : 1;
+	struct address_space *mapping = rac ? rac->mapping : folio->mapping;
 	unsigned max_nr_pages = nr_pages;
 	int ret = 0;
 
+	if (mapping_large_folio_support(mapping))
+		return f2fs_read_data_large_folio(inode, rac, folio);
+
 #ifdef CONFIG_F2FS_FS_COMPRESSION
 	if (f2fs_compressed_file(inode)) {
 		index = rac ? readahead_index(rac) : folio->index;
@@ -2450,8 +2663,7 @@ static int f2fs_mpage_readpages(struct inode *inode,
 		}
 #endif
 	}
-	if (bio)
-		f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA);
+	f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA);
 	return ret;
 }
 
@@ -3735,7 +3947,12 @@ void f2fs_invalidate_folio(struct folio *folio, size_t offset, size_t length)
 			f2fs_remove_dirty_inode(inode);
 		}
 	}
-	folio_detach_private(folio);
+
+	if (offset || length != folio_size(folio))
+		return;
+
+	folio_cancel_dirty(folio);
+	ffs_detach_free(folio);
 }
 
 bool f2fs_release_folio(struct folio *folio, gfp_t wait)
@@ -3744,7 +3961,7 @@ bool f2fs_release_folio(struct folio *folio, gfp_t wait)
 	if (folio_test_dirty(folio))
 		return false;
 
-	folio_detach_private(folio);
+	ffs_detach_free(folio);
 	return true;
 }
 
@@ -4150,12 +4367,25 @@ int __init f2fs_init_bio_entry_cache(void)
 {
 	bio_entry_slab = f2fs_kmem_cache_create("f2fs_bio_entry_slab",
 			sizeof(struct bio_entry));
-	return bio_entry_slab ? 0 : -ENOMEM;
+
+	if (!bio_entry_slab)
+		return -ENOMEM;
+
+	ffs_entry_slab = f2fs_kmem_cache_create("f2fs_ffs_slab",
+			sizeof(struct f2fs_folio_state));
+
+	if (!ffs_entry_slab) {
+		kmem_cache_destroy(bio_entry_slab);
+		return -ENOMEM;
+	}
+
+	return 0;
 }
 
 void f2fs_destroy_bio_entry_cache(void)
 {
 	kmem_cache_destroy(bio_entry_slab);
+	kmem_cache_destroy(ffs_entry_slab);
 }
 
 static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index 20edbb99b814..53cbce96f126 100644
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -4917,6 +4917,22 @@ static inline bool is_journalled_quota(struct f2fs_sb_info *sbi)
 	return false;
 }
 
+static inline bool f2fs_quota_file(struct inode *inode)
+{
+#ifdef CONFIG_QUOTA
+	int i;
+
+	if (!f2fs_sb_has_quota_ino(F2FS_I_SB(inode)))
+		return false;
+
+	for (i = 0; i < MAXQUOTAS; i++) {
+		if (f2fs_qf_ino(F2FS_I_SB(inode)->sb, i) == inode->i_ino)
+			return true;
+	}
+#endif
+	return false;
+}
+
 static inline bool f2fs_block_unit_discard(struct f2fs_sb_info *sbi)
 {
 	return F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_BLOCK;
diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
index d7047ca6b98d..e75e61ac50d7 100644
--- a/fs/f2fs/file.c
+++ b/fs/f2fs/file.c
@@ -624,6 +624,10 @@ static int f2fs_file_open(struct inode *inode, struct file *filp)
 	if (!f2fs_is_compress_backend_ready(inode))
 		return -EOPNOTSUPP;
 
+	if (mapping_large_folio_support(inode->i_mapping) &&
+	    filp->f_mode & FMODE_WRITE)
+		return -EOPNOTSUPP;
+
 	err = fsverity_file_open(inode, filp);
 	if (err)
 		return err;
diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c
index 384fa7e2085b..6afd57fa5387 100644
--- a/fs/f2fs/gc.c
+++ b/fs/f2fs/gc.c
@@ -2000,7 +2000,7 @@ int f2fs_gc(struct f2fs_sb_info *sbi, struct f2fs_gc_control *gc_control)
 		goto stop;
 	}
 
-	__get_secs_required(sbi, NULL, &upper_secs, NULL);
+	upper_secs = __get_secs_required(sbi);
 
 	/*
 	 * Write checkpoint to reclaim prefree segments.
diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
index 38b8994bc1b2..921fb02c0f49 100644
--- a/fs/f2fs/inode.c
+++ b/fs/f2fs/inode.c
@@ -597,6 +597,8 @@ struct inode *f2fs_iget(struct super_block *sb, unsigned long ino)
 	if (ret)
 		goto bad_inode;
 make_now:
+	f2fs_set_inode_flags(inode);
+
 	if (ino == F2FS_NODE_INO(sbi)) {
 		inode->i_mapping->a_ops = &f2fs_node_aops;
 		mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS);
@@ -618,6 +620,9 @@ struct inode *f2fs_iget(struct super_block *sb, unsigned long ino)
 		inode->i_op = &f2fs_file_inode_operations;
 		inode->i_fop = &f2fs_file_operations;
 		inode->i_mapping->a_ops = &f2fs_dblock_aops;
+		if (IS_IMMUTABLE(inode) && !f2fs_compressed_file(inode) &&
+		    !f2fs_quota_file(inode))
+			mapping_set_folio_min_order(inode->i_mapping, 0);
 	} else if (S_ISDIR(inode->i_mode)) {
 		inode->i_op = &f2fs_dir_inode_operations;
 		inode->i_fop = &f2fs_dir_operations;
@@ -638,7 +643,6 @@ struct inode *f2fs_iget(struct super_block *sb, unsigned long ino)
 		ret = -EIO;
 		goto bad_inode;
 	}
-	f2fs_set_inode_flags(inode);
 
 	unlock_new_inode(inode);
 	trace_f2fs_iget(inode);
diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h
index 07dcbcbeb7c6..20daaccb34a5 100644
--- a/fs/f2fs/segment.h
+++ b/fs/f2fs/segment.h
@@ -621,97 +621,90 @@ static inline unsigned int get_left_section_blocks(struct f2fs_sb_info *sbi,
 	return CAP_BLKS_PER_SEC(sbi) - get_ckpt_valid_blocks(sbi, segno, true);
 }
 
-static inline bool has_curseg_enough_space(struct f2fs_sb_info *sbi,
-			unsigned int node_blocks, unsigned int data_blocks,
-			unsigned int dent_blocks)
+static inline void get_additional_blocks_required(struct f2fs_sb_info *sbi,
+			unsigned int *total_node_blocks, unsigned int *total_data_blocks,
+			unsigned int *total_dent_blocks, bool separate_dent)
 {
-	unsigned int segno, left_blocks, blocks;
+	unsigned int segno, left_blocks;
 	int i;
+	unsigned int min_free_node_blocks = CAP_BLKS_PER_SEC(sbi);
+	unsigned int min_free_dent_blocks = CAP_BLKS_PER_SEC(sbi);
+	unsigned int min_free_data_blocks = CAP_BLKS_PER_SEC(sbi);
 
 	/* check current data/node sections in the worst case. */
 	for (i = CURSEG_HOT_DATA; i < NR_PERSISTENT_LOG; i++) {
 		segno = CURSEG_I(sbi, i)->segno;
 
 		if (unlikely(segno == NULL_SEGNO))
-			return false;
+			return;
 
 		left_blocks = get_left_section_blocks(sbi, i, segno);
 
-		blocks = i <= CURSEG_COLD_DATA ? data_blocks : node_blocks;
-		if (blocks > left_blocks)
-			return false;
+		if (i > CURSEG_COLD_DATA)
+			min_free_node_blocks = min(min_free_node_blocks, left_blocks);
+		else if (i == CURSEG_HOT_DATA && separate_dent)
+			min_free_dent_blocks = left_blocks;
+		else
+			min_free_data_blocks = min(min_free_data_blocks, left_blocks);
 	}
 
-	/* check current data section for dentry blocks. */
-	segno = CURSEG_I(sbi, CURSEG_HOT_DATA)->segno;
-
-	if (unlikely(segno == NULL_SEGNO))
-		return false;
-
-	left_blocks = get_left_section_blocks(sbi, CURSEG_HOT_DATA, segno);
-
-	if (dent_blocks > left_blocks)
-		return false;
-	return true;
+	*total_node_blocks = (*total_node_blocks > min_free_node_blocks) ?
+			*total_node_blocks - min_free_node_blocks : 0;
+	*total_dent_blocks = (*total_dent_blocks > min_free_dent_blocks) ?
+			*total_dent_blocks - min_free_dent_blocks : 0;
+	*total_data_blocks = (*total_data_blocks > min_free_data_blocks) ?
+			*total_data_blocks - min_free_data_blocks : 0;
 }
 
 /*
- * calculate needed sections for dirty node/dentry and call
- * has_curseg_enough_space, please note that, it needs to account
- * dirty data as well in lfs mode when checkpoint is disabled.
+ * call get_additional_blocks_required to calculate dirty blocks
+ * needing to be placed in free sections, please note that, it
+ * needs to account dirty data as well in lfs mode when checkpoint
+ * is disabled.
  */
-static inline void __get_secs_required(struct f2fs_sb_info *sbi,
-		unsigned int *lower_p, unsigned int *upper_p, bool *curseg_p)
+static inline int __get_secs_required(struct f2fs_sb_info *sbi)
 {
 	unsigned int total_node_blocks = get_pages(sbi, F2FS_DIRTY_NODES) +
 					get_pages(sbi, F2FS_DIRTY_DENTS) +
 					get_pages(sbi, F2FS_DIRTY_IMETA);
 	unsigned int total_dent_blocks = get_pages(sbi, F2FS_DIRTY_DENTS);
 	unsigned int total_data_blocks = 0;
-	unsigned int node_secs = total_node_blocks / CAP_BLKS_PER_SEC(sbi);
-	unsigned int dent_secs = total_dent_blocks / CAP_BLKS_PER_SEC(sbi);
-	unsigned int data_secs = 0;
-	unsigned int node_blocks = total_node_blocks % CAP_BLKS_PER_SEC(sbi);
-	unsigned int dent_blocks = total_dent_blocks % CAP_BLKS_PER_SEC(sbi);
-	unsigned int data_blocks = 0;
+	bool separate_dent = true;
 
-	if (f2fs_lfs_mode(sbi)) {
+	if (f2fs_lfs_mode(sbi))
 		total_data_blocks = get_pages(sbi, F2FS_DIRTY_DATA);
-		data_secs = total_data_blocks / CAP_BLKS_PER_SEC(sbi);
-		data_blocks = total_data_blocks % CAP_BLKS_PER_SEC(sbi);
+
+	/*
+	 * When active_logs != 4, dentry blocks and data blocks can be
+	 * mixed in the same logs, so check their space together.
+	 */
+	if (F2FS_OPTION(sbi).active_logs != 4) {
+		total_data_blocks += total_dent_blocks;
+		total_dent_blocks = 0;
+		separate_dent = false;
 	}
 
-	if (lower_p)
-		*lower_p = node_secs + dent_secs + data_secs;
-	if (upper_p)
-		*upper_p = node_secs + dent_secs + data_secs +
-			(node_blocks ? 1 : 0) + (dent_blocks ? 1 : 0) +
-			(data_blocks ? 1 : 0);
-	if (curseg_p)
-		*curseg_p = has_curseg_enough_space(sbi,
-				node_blocks, data_blocks, dent_blocks);
+	get_additional_blocks_required(sbi, &total_node_blocks, &total_dent_blocks,
+			&total_data_blocks, separate_dent);
+
+	return DIV_ROUND_UP(total_node_blocks, CAP_BLKS_PER_SEC(sbi)) +
+			DIV_ROUND_UP(total_dent_blocks, CAP_BLKS_PER_SEC(sbi)) +
+			DIV_ROUND_UP(total_data_blocks, CAP_BLKS_PER_SEC(sbi));
 }
 
 static inline bool has_not_enough_free_secs(struct f2fs_sb_info *sbi,
 					int freed, int needed)
 {
-	unsigned int free_secs, lower_secs, upper_secs;
-	bool curseg_space;
+	unsigned int free_secs, required_secs;
 
 	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
 		return false;
 
-	__get_secs_required(sbi, &lower_secs, &upper_secs, &curseg_space);
-
 	free_secs = free_sections(sbi) + freed;
-	lower_secs += needed + reserved_sections(sbi);
-	upper_secs += needed + reserved_sections(sbi);
+	required_secs = needed + reserved_sections(sbi) +
+			__get_secs_required(sbi);
 
-	if (free_secs > upper_secs)
-		return false;
-	if (free_secs <= lower_secs)
-		return true;
-	return !curseg_space;
+	return free_secs < required_secs;
 }
 
 static inline bool has_enough_free_secs(struct f2fs_sb_info *sbi,
diff --git a/fs/fat/cache.c b/fs/fat/cache.c
index 630f3056658e..1b87354e24ba 100644
--- a/fs/fat/cache.c
+++ b/fs/fat/cache.c
@@ -54,7 +54,7 @@ void fat_cache_destroy(void)
 	kmem_cache_destroy(fat_cache_cachep);
 }
 
-static inline struct fat_cache *fat_cache_alloc(struct inode *inode)
+static inline struct fat_cache *fat_cache_alloc(void)
 {
 	return kmem_cache_alloc(fat_cache_cachep, GFP_NOFS);
 }
@@ -144,7 +144,7 @@ static void fat_cache_add(struct inode *inode, struct fat_cache_id *new)
 			MSDOS_I(inode)->nr_caches++;
 			spin_unlock(&MSDOS_I(inode)->cache_lru_lock);
 
-			tmp = fat_cache_alloc(inode);
+			tmp = fat_cache_alloc();
 			if (!tmp) {
 				spin_lock(&MSDOS_I(inode)->cache_lru_lock);
 				MSDOS_I(inode)->nr_caches--;
diff --git a/fs/fat/dir.c b/fs/fat/dir.c
index 92b091783966..a7cff436dfbe 100644
--- a/fs/fat/dir.c
+++ b/fs/fat/dir.c
@@ -16,6 +16,7 @@
 
 #include <linux/slab.h>
 #include <linux/compat.h>
+#include <linux/hex.h>
 #include <linux/uaccess.h>
 #include <linux/iversion.h>
 #include "fat.h"
@@ -1353,7 +1354,7 @@ int fat_add_entries(struct inode *dir, void *slots, int nr_slots,
 
 		/* Fill the long name slots. */
 		for (i = 0; i < long_bhs; i++) {
-			int copy = min_t(int, sb->s_blocksize - offset, size);
+			int copy = umin(sb->s_blocksize - offset, size);
 			memcpy(bhs[i]->b_data + offset, slots, copy);
 			mark_buffer_dirty_inode(bhs[i], dir);
 			offset = 0;
@@ -1364,7 +1365,7 @@ int fat_add_entries(struct inode *dir, void *slots, int nr_slots,
 			err = fat_sync_bhs(bhs, long_bhs);
 		if (!err && i < nr_bhs) {
 			/* Fill the short name slot. */
-			int copy = min_t(int, sb->s_blocksize - offset, size);
+			int copy = umin(sb->s_blocksize - offset, size);
 			memcpy(bhs[i]->b_data + offset, slots, copy);
 			mark_buffer_dirty_inode(bhs[i], dir);
 			if (IS_DIRSYNC(dir))
diff --git a/fs/fat/file.c b/fs/fat/file.c
index 4fc49a614fb8..f48435e586c7 100644
--- a/fs/fat/file.c
+++ b/fs/fat/file.c
@@ -140,8 +140,7 @@ static int fat_ioctl_fitrim(struct inode *inode, unsigned long arg)
 	if (copy_from_user(&range, user_range, sizeof(range)))
 		return -EFAULT;
 
-	range.minlen = max_t(unsigned int, range.minlen,
-			     bdev_discard_granularity(sb->s_bdev));
+	range.minlen = max(range.minlen, bdev_discard_granularity(sb->s_bdev));
 
 	err = fat_trim_fs(inode, &range);
 	if (err < 0)
diff --git a/fs/fat/namei_vfat.c b/fs/fat/namei_vfat.c
index 5dbc4cbb8fce..4f3cc2b3089e 100644
--- a/fs/fat/namei_vfat.c
+++ b/fs/fat/namei_vfat.c
@@ -20,6 +20,7 @@
 #include <linux/ctype.h>
 #include <linux/slab.h>
 #include <linux/namei.h>
+#include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/iversion.h>
 #include "fat.h"
diff --git a/fs/file_attr.c b/fs/file_attr.c
index 4c4916632f11..13cdb31a3e94 100644
--- a/fs/file_attr.c
+++ b/fs/file_attr.c
@@ -2,6 +2,7 @@
 #include <linux/fs.h>
 #include <linux/security.h>
 #include <linux/fscrypt.h>
+#include <linux/fsnotify.h>
 #include <linux/fileattr.h>
 #include <linux/export.h>
 #include <linux/syscalls.h>
@@ -298,6 +299,7 @@ int vfs_fileattr_set(struct mnt_idmap *idmap, struct dentry *dentry,
 		err = inode->i_op->fileattr_set(idmap, dentry, fa);
 		if (err)
 			goto out;
+		fsnotify_xattr(dentry);
 	}
 
 out:
diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
index 6800886c4d10..5dd6e89a6d29 100644
--- a/fs/fs-writeback.c
+++ b/fs/fs-writeback.c
@@ -981,7 +981,7 @@ void wbc_account_cgroup_owner(struct writeback_control *wbc, struct folio *folio
 
 	css = mem_cgroup_css_from_folio(folio);
 	/* dead cgroups shouldn't contribute to inode ownership arbitration */
-	if (!(css->flags & CSS_ONLINE))
+	if (!css_is_online(css))
 		return;
 
 	id = css->id;
diff --git a/fs/fs_context.c b/fs/fs_context.c
index 93b7ebf8d927..81ed94f46cac 100644
--- a/fs/fs_context.c
+++ b/fs/fs_context.c
@@ -24,20 +24,6 @@
 #include "mount.h"
 #include "internal.h"
 
-enum legacy_fs_param {
-	LEGACY_FS_UNSET_PARAMS,
-	LEGACY_FS_MONOLITHIC_PARAMS,
-	LEGACY_FS_INDIVIDUAL_PARAMS,
-};
-
-struct legacy_fs_context {
-	char			*legacy_data;	/* Data page for legacy filesystems */
-	size_t			data_size;
-	enum legacy_fs_param	param_type;
-};
-
-static int legacy_init_fs_context(struct fs_context *fc);
-
 static const struct constant_table common_set_sb_flag[] = {
 	{ "dirsync",	SB_DIRSYNC },
 	{ "lazytime",	SB_LAZYTIME },
@@ -275,7 +261,6 @@ static struct fs_context *alloc_fs_context(struct file_system_type *fs_type,
 				      unsigned int sb_flags_mask,
 				      enum fs_context_purpose purpose)
 {
-	int (*init_fs_context)(struct fs_context *);
 	struct fs_context *fc;
 	int ret = -ENOMEM;
 
@@ -307,12 +292,7 @@ static struct fs_context *alloc_fs_context(struct file_system_type *fs_type,
 		break;
 	}
 
-	/* TODO: Make all filesystems support this unconditionally */
-	init_fs_context = fc->fs_type->init_fs_context;
-	if (!init_fs_context)
-		init_fs_context = legacy_init_fs_context;
-
-	ret = init_fs_context(fc);
+	ret = fc->fs_type->init_fs_context(fc);
 	if (ret < 0)
 		goto err_fc;
 	fc->need_free = true;
@@ -376,8 +356,6 @@ void fc_drop_locked(struct fs_context *fc)
 	deactivate_locked_super(sb);
 }
 
-static void legacy_fs_context_free(struct fs_context *fc);
-
 /**
  * vfs_dup_fs_context - Duplicate a filesystem context.
  * @src_fc: The context to copy.
@@ -531,184 +509,6 @@ void put_fs_context(struct fs_context *fc)
 }
 EXPORT_SYMBOL(put_fs_context);
 
-/*
- * Free the config for a filesystem that doesn't support fs_context.
- */
-static void legacy_fs_context_free(struct fs_context *fc)
-{
-	struct legacy_fs_context *ctx = fc->fs_private;
-
-	if (ctx) {
-		if (ctx->param_type == LEGACY_FS_INDIVIDUAL_PARAMS)
-			kfree(ctx->legacy_data);
-		kfree(ctx);
-	}
-}
-
-/*
- * Duplicate a legacy config.
- */
-static int legacy_fs_context_dup(struct fs_context *fc, struct fs_context *src_fc)
-{
-	struct legacy_fs_context *ctx;
-	struct legacy_fs_context *src_ctx = src_fc->fs_private;
-
-	ctx = kmemdup(src_ctx, sizeof(*src_ctx), GFP_KERNEL);
-	if (!ctx)
-		return -ENOMEM;
-
-	if (ctx->param_type == LEGACY_FS_INDIVIDUAL_PARAMS) {
-		ctx->legacy_data = kmemdup(src_ctx->legacy_data,
-					   src_ctx->data_size, GFP_KERNEL);
-		if (!ctx->legacy_data) {
-			kfree(ctx);
-			return -ENOMEM;
-		}
-	}
-
-	fc->fs_private = ctx;
-	return 0;
-}
-
-/*
- * Add a parameter to a legacy config.  We build up a comma-separated list of
- * options.
- */
-static int legacy_parse_param(struct fs_context *fc, struct fs_parameter *param)
-{
-	struct legacy_fs_context *ctx = fc->fs_private;
-	unsigned int size = ctx->data_size;
-	size_t len = 0;
-	int ret;
-
-	ret = vfs_parse_fs_param_source(fc, param);
-	if (ret != -ENOPARAM)
-		return ret;
-
-	if (ctx->param_type == LEGACY_FS_MONOLITHIC_PARAMS)
-		return invalf(fc, "VFS: Legacy: Can't mix monolithic and individual options");
-
-	switch (param->type) {
-	case fs_value_is_string:
-		len = 1 + param->size;
-		fallthrough;
-	case fs_value_is_flag:
-		len += strlen(param->key);
-		break;
-	default:
-		return invalf(fc, "VFS: Legacy: Parameter type for '%s' not supported",
-			      param->key);
-	}
-
-	if (size + len + 2 > PAGE_SIZE)
-		return invalf(fc, "VFS: Legacy: Cumulative options too large");
-	if (strchr(param->key, ',') ||
-	    (param->type == fs_value_is_string &&
-	     memchr(param->string, ',', param->size)))
-		return invalf(fc, "VFS: Legacy: Option '%s' contained comma",
-			      param->key);
-	if (!ctx->legacy_data) {
-		ctx->legacy_data = kmalloc(PAGE_SIZE, GFP_KERNEL);
-		if (!ctx->legacy_data)
-			return -ENOMEM;
-	}
-
-	if (size)
-		ctx->legacy_data[size++] = ',';
-	len = strlen(param->key);
-	memcpy(ctx->legacy_data + size, param->key, len);
-	size += len;
-	if (param->type == fs_value_is_string) {
-		ctx->legacy_data[size++] = '=';
-		memcpy(ctx->legacy_data + size, param->string, param->size);
-		size += param->size;
-	}
-	ctx->legacy_data[size] = '\0';
-	ctx->data_size = size;
-	ctx->param_type = LEGACY_FS_INDIVIDUAL_PARAMS;
-	return 0;
-}
-
-/*
- * Add monolithic mount data.
- */
-static int legacy_parse_monolithic(struct fs_context *fc, void *data)
-{
-	struct legacy_fs_context *ctx = fc->fs_private;
-
-	if (ctx->param_type != LEGACY_FS_UNSET_PARAMS) {
-		pr_warn("VFS: Can't mix monolithic and individual options\n");
-		return -EINVAL;
-	}
-
-	ctx->legacy_data = data;
-	ctx->param_type = LEGACY_FS_MONOLITHIC_PARAMS;
-	if (!ctx->legacy_data)
-		return 0;
-
-	if (fc->fs_type->fs_flags & FS_BINARY_MOUNTDATA)
-		return 0;
-	return security_sb_eat_lsm_opts(ctx->legacy_data, &fc->security);
-}
-
-/*
- * Get a mountable root with the legacy mount command.
- */
-static int legacy_get_tree(struct fs_context *fc)
-{
-	struct legacy_fs_context *ctx = fc->fs_private;
-	struct super_block *sb;
-	struct dentry *root;
-
-	root = fc->fs_type->mount(fc->fs_type, fc->sb_flags,
-				      fc->source, ctx->legacy_data);
-	if (IS_ERR(root))
-		return PTR_ERR(root);
-
-	sb = root->d_sb;
-	BUG_ON(!sb);
-
-	fc->root = root;
-	return 0;
-}
-
-/*
- * Handle remount.
- */
-static int legacy_reconfigure(struct fs_context *fc)
-{
-	struct legacy_fs_context *ctx = fc->fs_private;
-	struct super_block *sb = fc->root->d_sb;
-
-	if (!sb->s_op->remount_fs)
-		return 0;
-
-	return sb->s_op->remount_fs(sb, &fc->sb_flags,
-				    ctx ? ctx->legacy_data : NULL);
-}
-
-const struct fs_context_operations legacy_fs_context_ops = {
-	.free			= legacy_fs_context_free,
-	.dup			= legacy_fs_context_dup,
-	.parse_param		= legacy_parse_param,
-	.parse_monolithic	= legacy_parse_monolithic,
-	.get_tree		= legacy_get_tree,
-	.reconfigure		= legacy_reconfigure,
-};
-
-/*
- * Initialise a legacy context for a filesystem that doesn't support
- * fs_context.
- */
-static int legacy_init_fs_context(struct fs_context *fc)
-{
-	fc->fs_private = kzalloc(sizeof(struct legacy_fs_context), GFP_KERNEL_ACCOUNT);
-	if (!fc->fs_private)
-		return -ENOMEM;
-	fc->ops = &legacy_fs_context_ops;
-	return 0;
-}
-
 int parse_monolithic_mount_data(struct fs_context *fc, void *data)
 {
 	int (*monolithic_mount_data)(struct fs_context *, void *);
@@ -757,10 +557,8 @@ int finish_clean_context(struct fs_context *fc)
 	if (fc->phase != FS_CONTEXT_AWAITING_RECONF)
 		return 0;
 
-	if (fc->fs_type->init_fs_context)
-		error = fc->fs_type->init_fs_context(fc);
-	else
-		error = legacy_init_fs_context(fc);
+	error = fc->fs_type->init_fs_context(fc);
+
 	if (unlikely(error)) {
 		fc->phase = FS_CONTEXT_FAILED;
 		return error;
diff --git a/fs/fsopen.c b/fs/fsopen.c
index f645c99204eb..622ee3926cd5 100644
--- a/fs/fsopen.c
+++ b/fs/fsopen.c
@@ -404,16 +404,6 @@ SYSCALL_DEFINE5(fsconfig,
 		return -EINVAL;
 
 	fc = fd_file(f)->private_data;
-	if (fc->ops == &legacy_fs_context_ops) {
-		switch (cmd) {
-		case FSCONFIG_SET_BINARY:
-		case FSCONFIG_SET_PATH:
-		case FSCONFIG_SET_PATH_EMPTY:
-		case FSCONFIG_SET_FD:
-		case FSCONFIG_CMD_CREATE_EXCL:
-			return -EOPNOTSUPP;
-		}
-	}
 
 	if (_key) {
 		param.key = strndup_user(_key, 256);
diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
index 6d59cbc877c6..a30c8b57d478 100644
--- a/fs/fuse/dev.c
+++ b/fs/fuse/dev.c
@@ -1813,7 +1813,7 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
 			goto out_iput;
 
 		folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset;
-		nr_bytes = min_t(unsigned, num, folio_size(folio) - folio_offset);
+		nr_bytes = min(num, folio_size(folio) - folio_offset);
 		nr_pages = (offset + nr_bytes + PAGE_SIZE - 1) >> PAGE_SHIFT;
 
 		err = fuse_copy_folio(cs, &folio, folio_offset, nr_bytes, 0);
diff --git a/fs/fuse/file.c b/fs/fuse/file.c
index 01bc894e9c2b..4f71eb5a9bac 100644
--- a/fs/fuse/file.c
+++ b/fs/fuse/file.c
@@ -1323,10 +1323,8 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,
 static inline unsigned int fuse_wr_pages(loff_t pos, size_t len,
 				     unsigned int max_pages)
 {
-	return min_t(unsigned int,
-		     ((pos + len - 1) >> PAGE_SHIFT) -
-		     (pos >> PAGE_SHIFT) + 1,
-		     max_pages);
+	return min(((pos + len - 1) >> PAGE_SHIFT) - (pos >> PAGE_SHIFT) + 1,
+		   max_pages);
 }
 
 static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii)
@@ -1607,7 +1605,7 @@ static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
 			struct folio *folio = page_folio(pages[i]);
 			unsigned int offset = start +
 				(folio_page_idx(folio, pages[i]) << PAGE_SHIFT);
-			unsigned int len = min_t(unsigned int, ret, PAGE_SIZE - start);
+			unsigned int len = umin(ret, PAGE_SIZE - start);
 
 			ap->descs[ap->num_folios].offset = offset;
 			ap->descs[ap->num_folios].length = len;
diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c
index 92e029104d8a..97ad59675488 100644
--- a/fs/gfs2/glock.c
+++ b/fs/gfs2/glock.c
@@ -146,7 +146,7 @@ static void __gfs2_glock_free(struct gfs2_glock *gl)
 }
 
 void gfs2_glock_free(struct gfs2_glock *gl) {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 
 	__gfs2_glock_free(gl);
 	if (atomic_dec_and_test(&sdp->sd_glock_disposal))
@@ -154,7 +154,7 @@ void gfs2_glock_free(struct gfs2_glock *gl) {
 }
 
 void gfs2_glock_free_later(struct gfs2_glock *gl) {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 
 	spin_lock(&lru_lock);
 	list_add(&gl->gl_lru, &sdp->sd_dead_glocks);
@@ -184,8 +184,7 @@ static void gfs2_free_dead_glocks(struct gfs2_sbd *sdp)
 
 struct gfs2_glock *gfs2_glock_hold(struct gfs2_glock *gl)
 {
-	GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
-	lockref_get(&gl->gl_lockref);
+	GLOCK_BUG_ON(gl, !lockref_get_not_dead(&gl->gl_lockref));
 	return gl;
 }
 
@@ -218,7 +217,7 @@ static void gfs2_glock_remove_from_lru(struct gfs2_glock *gl)
  * work queue.
  */
 static void gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 
 	if (!queue_delayed_work(sdp->sd_glock_wq, &gl->gl_work, delay)) {
 		/*
@@ -234,7 +233,7 @@ static void gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
 
 static void __gfs2_glock_put(struct gfs2_glock *gl)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct address_space *mapping = gfs2_glock2aspace(gl);
 
 	lockref_mark_dead(&gl->gl_lockref);
@@ -356,7 +355,7 @@ static void gfs2_holder_wake(struct gfs2_holder *gh)
 	smp_mb__after_atomic();
 	wake_up_bit(&gh->gh_iflags, HIF_WAIT);
 	if (gh->gh_flags & GL_ASYNC) {
-		struct gfs2_sbd *sdp = gh->gh_gl->gl_name.ln_sbd;
+		struct gfs2_sbd *sdp = glock_sbd(gh->gh_gl);
 
 		wake_up(&sdp->sd_async_glock_wait);
 	}
@@ -458,7 +457,7 @@ int gfs2_instantiate(struct gfs2_holder *gh)
 
 static void do_promote(struct gfs2_glock *gl)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct gfs2_holder *gh, *current_gh;
 
 	if (gfs2_withdrawn(sdp)) {
@@ -538,7 +537,7 @@ static void state_change(struct gfs2_glock *gl, unsigned int new_state)
 
 static void gfs2_set_demote(int nr, struct gfs2_glock *gl)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 
 	set_bit(nr, &gl->gl_flags);
 	smp_mb();
@@ -608,9 +607,9 @@ static void finish_xmote(struct gfs2_glock *gl, unsigned int ret)
 			do_xmote(gl, gh, LM_ST_UNLOCKED);
 			break;
 		default: /* Everything else */
-			fs_err(gl->gl_name.ln_sbd,
+			fs_err(glock_sbd(gl),
 			       "glock %u:%llu requested=%u ret=%u\n",
-			       gl->gl_name.ln_type, gl->gl_name.ln_number,
+			       glock_type(gl), glock_number(gl),
 			       gl->gl_req, ret);
 			GLOCK_BUG_ON(gl, 1);
 		}
@@ -655,7 +654,7 @@ __releases(&gl->gl_lockref.lock)
 __acquires(&gl->gl_lockref.lock)
 {
 	const struct gfs2_glock_operations *glops = gl->gl_ops;
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct lm_lockstruct *ls = &sdp->sd_lockstruct;
 	int ret;
 
@@ -818,7 +817,7 @@ void glock_set_object(struct gfs2_glock *gl, void *object)
 	prev_object = gl->gl_object;
 	gl->gl_object = object;
 	spin_unlock(&gl->gl_lockref.lock);
-	if (gfs2_assert_warn(gl->gl_name.ln_sbd, prev_object == NULL))
+	if (gfs2_assert_warn(glock_sbd(gl), prev_object == NULL))
 		gfs2_dump_glock(NULL, gl, true);
 }
 
@@ -835,7 +834,7 @@ void glock_clear_object(struct gfs2_glock *gl, void *object)
 	prev_object = gl->gl_object;
 	gl->gl_object = NULL;
 	spin_unlock(&gl->gl_lockref.lock);
-	if (gfs2_assert_warn(gl->gl_name.ln_sbd, prev_object == object))
+	if (gfs2_assert_warn(glock_sbd(gl), prev_object == object))
 		gfs2_dump_glock(NULL, gl, true);
 }
 
@@ -925,7 +924,7 @@ static void gfs2_try_to_evict(struct gfs2_glock *gl)
 
 bool gfs2_queue_try_to_evict(struct gfs2_glock *gl)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 
 	if (test_and_set_bit(GLF_TRY_TO_EVICT, &gl->gl_flags))
 		return false;
@@ -934,7 +933,7 @@ bool gfs2_queue_try_to_evict(struct gfs2_glock *gl)
 
 bool gfs2_queue_verify_delete(struct gfs2_glock *gl, bool later)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	unsigned long delay;
 
 	if (test_and_set_bit(GLF_VERIFY_DELETE, &gl->gl_flags))
@@ -947,7 +946,7 @@ static void delete_work_func(struct work_struct *work)
 {
 	struct delayed_work *dwork = to_delayed_work(work);
 	struct gfs2_glock *gl = container_of(dwork, struct gfs2_glock, gl_delete);
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	bool verify_delete = test_and_clear_bit(GLF_VERIFY_DELETE, &gl->gl_flags);
 
 	/*
@@ -960,7 +959,7 @@ static void delete_work_func(struct work_struct *work)
 		gfs2_try_to_evict(gl);
 
 	if (verify_delete) {
-		u64 no_addr = gl->gl_name.ln_number;
+		u64 no_addr = glock_number(gl);
 		struct inode *inode;
 
 		inode = gfs2_lookup_by_inum(sdp, no_addr, gl->gl_no_formal_ino,
@@ -994,7 +993,7 @@ static void glock_work_func(struct work_struct *work)
 	if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
 	    gl->gl_state != LM_ST_UNLOCKED &&
 	    gl->gl_demote_state != LM_ST_EXCLUSIVE) {
-		if (gl->gl_name.ln_type == LM_TYPE_INODE) {
+		if (glock_type(gl) == LM_TYPE_INODE) {
 			unsigned long holdtime, now = jiffies;
 
 			holdtime = gl->gl_tchange + gl->gl_hold_time;
@@ -1136,7 +1135,7 @@ int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
 	gl->gl_object = NULL;
 	gl->gl_hold_time = GL_GLOCK_DFT_HOLD;
 	INIT_DELAYED_WORK(&gl->gl_work, glock_work_func);
-	if (gl->gl_name.ln_type == LM_TYPE_IOPEN)
+	if (glock_type(gl) == LM_TYPE_IOPEN)
 		INIT_DELAYED_WORK(&gl->gl_delete, delete_work_func);
 
 	mapping = gfs2_glock2aspace(gl);
@@ -1292,7 +1291,7 @@ static int glocks_pending(unsigned int num_gh, struct gfs2_holder *ghs)
 
 int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs)
 {
-	struct gfs2_sbd *sdp = ghs[0].gh_gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(ghs[0].gh_gl);
 	int i, ret = 0, timeout = 0;
 	unsigned long start_time = jiffies;
 
@@ -1418,7 +1417,7 @@ static inline bool pid_is_meaningful(const struct gfs2_holder *gh)
 static inline void add_to_queue(struct gfs2_holder *gh)
 {
 	struct gfs2_glock *gl = gh->gh_gl;
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct gfs2_holder *gh2;
 
 	GLOCK_BUG_ON(gl, gh->gh_owner_pid == NULL);
@@ -1451,11 +1450,11 @@ static inline void add_to_queue(struct gfs2_holder *gh)
 	fs_err(sdp, "original: %pSR\n", (void *)gh2->gh_ip);
 	fs_err(sdp, "pid: %d\n", pid_nr(gh2->gh_owner_pid));
 	fs_err(sdp, "lock type: %d req lock state : %d\n",
-	       gh2->gh_gl->gl_name.ln_type, gh2->gh_state);
+	       glock_type(gh2->gh_gl), gh2->gh_state);
 	fs_err(sdp, "new: %pSR\n", (void *)gh->gh_ip);
 	fs_err(sdp, "pid: %d\n", pid_nr(gh->gh_owner_pid));
 	fs_err(sdp, "lock type: %d req lock state : %d\n",
-	       gh->gh_gl->gl_name.ln_type, gh->gh_state);
+	       glock_type(gh->gh_gl), gh->gh_state);
 	gfs2_dump_glock(NULL, gl, true);
 	BUG();
 }
@@ -1472,7 +1471,7 @@ static inline void add_to_queue(struct gfs2_holder *gh)
 int gfs2_glock_nq(struct gfs2_holder *gh)
 {
 	struct gfs2_glock *gl = gh->gh_gl;
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	int error;
 
 	if (gfs2_withdrawn(sdp))
@@ -1559,7 +1558,7 @@ static void __gfs2_glock_dq(struct gfs2_holder *gh)
 		gl->gl_lockref.count++;
 		if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
 		    !test_bit(GLF_DEMOTE, &gl->gl_flags) &&
-		    gl->gl_name.ln_type == LM_TYPE_INODE)
+		    glock_type(gl) == LM_TYPE_INODE)
 			delay = gl->gl_hold_time;
 		gfs2_glock_queue_work(gl, delay);
 	}
@@ -1590,7 +1589,7 @@ void gfs2_glock_dq(struct gfs2_holder *gh)
 	    !test_bit(GLF_CANCELING, &gl->gl_flags)) {
 		set_bit(GLF_CANCELING, &gl->gl_flags);
 		spin_unlock(&gl->gl_lockref.lock);
-		gl->gl_name.ln_sbd->sd_lockstruct.ls_ops->lm_cancel(gl);
+		glock_sbd(gl)->sd_lockstruct.ls_ops->lm_cancel(gl);
 		wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE);
 		spin_lock(&gl->gl_lockref.lock);
 		clear_bit(GLF_CANCELING, &gl->gl_flags);
@@ -1764,7 +1763,7 @@ void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state)
 	gfs2_glock_hold(gl);
 	spin_lock(&gl->gl_lockref.lock);
 	if (!list_empty(&gl->gl_holders) &&
-	    gl->gl_name.ln_type == LM_TYPE_INODE) {
+	    glock_type(gl) == LM_TYPE_INODE) {
 		unsigned long now = jiffies;
 		unsigned long holdtime;
 
@@ -1821,7 +1820,7 @@ static int gfs2_should_freeze(const struct gfs2_glock *gl)
 
 void gfs2_glock_complete(struct gfs2_glock *gl, int ret)
 {
-	struct lm_lockstruct *ls = &gl->gl_name.ln_sbd->sd_lockstruct;
+	struct lm_lockstruct *ls = &glock_sbd(gl)->sd_lockstruct;
 
 	spin_lock(&gl->gl_lockref.lock);
 	clear_bit(GLF_PENDING_REPLY, &gl->gl_flags);
@@ -1849,9 +1848,9 @@ static int glock_cmp(void *priv, const struct list_head *a,
 	gla = list_entry(a, struct gfs2_glock, gl_lru);
 	glb = list_entry(b, struct gfs2_glock, gl_lru);
 
-	if (gla->gl_name.ln_number > glb->gl_name.ln_number)
+	if (glock_number(gla) > glock_number(glb))
 		return 1;
-	if (gla->gl_name.ln_number < glb->gl_name.ln_number)
+	if (glock_number(gla) < glock_number(glb))
 		return -1;
 
 	return 0;
@@ -1859,7 +1858,7 @@ static int glock_cmp(void *priv, const struct list_head *a,
 
 static bool can_free_glock(struct gfs2_glock *gl)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 
 	return !test_bit(GLF_LOCK, &gl->gl_flags) &&
 	       !gl->gl_lockref.count &&
@@ -1981,7 +1980,7 @@ static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp)
 		rhashtable_walk_start(&iter);
 
 		while ((gl = rhashtable_walk_next(&iter)) && !IS_ERR(gl)) {
-			if (gl->gl_name.ln_sbd == sdp)
+			if (glock_sbd(gl) == sdp)
 				examiner(gl);
 		}
 
@@ -2001,8 +2000,8 @@ void gfs2_cancel_delete_work(struct gfs2_glock *gl)
 
 static void flush_delete_work(struct gfs2_glock *gl)
 {
-	if (gl->gl_name.ln_type == LM_TYPE_IOPEN) {
-		struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	if (glock_type(gl) == LM_TYPE_IOPEN) {
+		struct gfs2_sbd *sdp = glock_sbd(gl);
 
 		if (cancel_delayed_work(&gl->gl_delete)) {
 			queue_delayed_work(sdp->sd_delete_wq,
@@ -2287,7 +2286,7 @@ void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
 	unsigned long long dtime;
 	const struct gfs2_holder *gh;
 	char gflags_buf[32];
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	char fs_id_buf[sizeof(sdp->sd_fsname) + 7];
 	unsigned long nrpages = 0;
 
@@ -2306,8 +2305,8 @@ void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
 	gfs2_print_dbg(seq, "%sG:  s:%s n:%u/%llx f:%s t:%s d:%s/%llu a:%d "
 		       "v:%d r:%d m:%ld p:%lu\n",
 		       fs_id_buf, state2str(gl->gl_state),
-		       gl->gl_name.ln_type,
-		       (unsigned long long)gl->gl_name.ln_number,
+		       glock_type(gl),
+		       (unsigned long long) glock_number(gl),
 		       gflags2str(gflags_buf, gl),
 		       state2str(gl->gl_target),
 		       state2str(gl->gl_demote_state), dtime,
@@ -2327,8 +2326,8 @@ static int gfs2_glstats_seq_show(struct seq_file *seq, void *iter_ptr)
 	struct gfs2_glock *gl = iter_ptr;
 
 	seq_printf(seq, "G: n:%u/%llx rtt:%llu/%llu rttb:%llu/%llu irt:%llu/%llu dcnt: %llu qcnt: %llu\n",
-		   gl->gl_name.ln_type,
-		   (unsigned long long)gl->gl_name.ln_number,
+		   glock_type(gl),
+		   (unsigned long long) glock_number(gl),
 		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTT],
 		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVAR],
 		   (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTB],
@@ -2444,7 +2443,7 @@ static void gfs2_glock_iter_next(struct gfs2_glock_iter *gi, loff_t n)
 			gl = NULL;
 			break;
 		}
-		if (gl->gl_name.ln_sbd != gi->sdp)
+		if (glock_sbd(gl) != gi->sdp)
 			continue;
 		if (n <= 1) {
 			if (!lockref_get_not_dead(&gl->gl_lockref))
@@ -2740,8 +2739,8 @@ static int gfs2_glockfd_seq_show(struct seq_file *seq, void *iter_ptr)
 	gl = GFS2_I(inode)->i_iopen_gh.gh_gl;
 	if (gl) {
 		seq_printf(seq, "%d %u %u/%llx\n",
-			   i->tgid, i->fd, gl->gl_name.ln_type,
-			   (unsigned long long)gl->gl_name.ln_number);
+			   i->tgid, i->fd, glock_type(gl),
+			   (unsigned long long) glock_number(gl));
 	}
 	gfs2_glockfd_seq_show_flock(seq, i);
 	inode_unlock_shared(inode);
diff --git a/fs/gfs2/glock.h b/fs/gfs2/glock.h
index 55d5985f32a0..7a97467de555 100644
--- a/fs/gfs2/glock.h
+++ b/fs/gfs2/glock.h
@@ -126,7 +126,7 @@ enum {
 
 #define GL_GLOCK_MAX_HOLD        (long)(HZ / 5)
 #define GL_GLOCK_DFT_HOLD        (long)(HZ / 5)
-#define GL_GLOCK_MIN_HOLD        (long)(10)
+#define GL_GLOCK_MIN_HOLD        (long)(HZ / 100)
 #define GL_GLOCK_HOLD_INCR       (long)(HZ / 20)
 #define GL_GLOCK_HOLD_DECR       (long)(HZ / 40)
 
@@ -221,11 +221,11 @@ void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl,
 			BUG(); } } while(0)
 #define gfs2_glock_assert_warn(gl, x) do { if (unlikely(!(x))) {	\
 			gfs2_dump_glock(NULL, gl, true);		\
-			gfs2_assert_warn((gl)->gl_name.ln_sbd, (x)); } } \
+			gfs2_assert_warn(glock_sbd(gl), (x)); } } \
 	while (0)
 #define gfs2_glock_assert_withdraw(gl, x) do { if (unlikely(!(x))) {	\
 			gfs2_dump_glock(NULL, gl, true);		\
-			gfs2_assert_withdraw((gl)->gl_name.ln_sbd, (x)); } } \
+			gfs2_assert_withdraw(glock_sbd(gl), (x)); } } \
 	while (0)
 
 __printf(2, 3)
diff --git a/fs/gfs2/glops.c b/fs/gfs2/glops.c
index 2173ccf5034b..ba61649368bf 100644
--- a/fs/gfs2/glops.c
+++ b/fs/gfs2/glops.c
@@ -32,7 +32,7 @@ struct workqueue_struct *gfs2_freeze_wq;
 
 static void gfs2_ail_error(struct gfs2_glock *gl, const struct buffer_head *bh)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 
 	fs_err(sdp,
 	       "AIL buffer %p: blocknr %llu state 0x%08lx mapping %p page "
@@ -40,7 +40,7 @@ static void gfs2_ail_error(struct gfs2_glock *gl, const struct buffer_head *bh)
 	       bh, (unsigned long long)bh->b_blocknr, bh->b_state,
 	       bh->b_folio->mapping, bh->b_folio->flags.f);
 	fs_err(sdp, "AIL glock %u:%llu mapping %p\n",
-	       gl->gl_name.ln_type, gl->gl_name.ln_number,
+	       glock_type(gl), glock_number(gl),
 	       gfs2_glock2aspace(gl));
 	gfs2_lm(sdp, "AIL error\n");
 	gfs2_withdraw(sdp);
@@ -58,7 +58,7 @@ static void gfs2_ail_error(struct gfs2_glock *gl, const struct buffer_head *bh)
 static void __gfs2_ail_flush(struct gfs2_glock *gl, bool fsync,
 			     unsigned int nr_revokes)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct list_head *head = &gl->gl_ail_list;
 	struct gfs2_bufdata *bd, *tmp;
 	struct buffer_head *bh;
@@ -86,7 +86,7 @@ static void __gfs2_ail_flush(struct gfs2_glock *gl, bool fsync,
 
 static int gfs2_ail_empty_gl(struct gfs2_glock *gl)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct gfs2_trans tr;
 	unsigned int revokes;
 	int ret = 0;
@@ -139,7 +139,7 @@ static int gfs2_ail_empty_gl(struct gfs2_glock *gl)
 
 void gfs2_ail_flush(struct gfs2_glock *gl, bool fsync)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	unsigned int revokes = atomic_read(&gl->gl_ail_count);
 	int ret;
 
@@ -163,7 +163,7 @@ void gfs2_ail_flush(struct gfs2_glock *gl, bool fsync)
 
 static int gfs2_rgrp_metasync(struct gfs2_glock *gl)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct address_space *metamapping = gfs2_aspace(sdp);
 	struct gfs2_rgrpd *rgd = gfs2_glock2rgrp(gl);
 	const unsigned bsize = sdp->sd_sb.sb_bsize;
@@ -191,7 +191,7 @@ static int gfs2_rgrp_metasync(struct gfs2_glock *gl)
 
 static int rgrp_go_sync(struct gfs2_glock *gl)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct gfs2_rgrpd *rgd = gfs2_glock2rgrp(gl);
 	int error;
 
@@ -220,7 +220,7 @@ static int rgrp_go_sync(struct gfs2_glock *gl)
 
 static void rgrp_go_inval(struct gfs2_glock *gl, int flags)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct address_space *mapping = gfs2_aspace(sdp);
 	struct gfs2_rgrpd *rgd = gfs2_glock2rgrp(gl);
 	const unsigned bsize = sdp->sd_sb.sb_bsize;
@@ -290,7 +290,7 @@ int gfs2_inode_metasync(struct gfs2_glock *gl)
 	filemap_fdatawrite(metamapping);
 	error = filemap_fdatawait(metamapping);
 	if (error)
-		gfs2_io_error(gl->gl_name.ln_sbd);
+		gfs2_io_error(glock_sbd(gl));
 	return error;
 }
 
@@ -317,7 +317,7 @@ static int inode_go_sync(struct gfs2_glock *gl)
 
 	GLOCK_BUG_ON(gl, gl->gl_state != LM_ST_EXCLUSIVE);
 
-	gfs2_log_flush(gl->gl_name.ln_sbd, gl, GFS2_LOG_HEAD_FLUSH_NORMAL |
+	gfs2_log_flush(glock_sbd(gl), gl, GFS2_LOG_HEAD_FLUSH_NORMAL |
 		       GFS2_LFC_INODE_GO_SYNC);
 	filemap_fdatawrite(metamapping);
 	if (isreg) {
@@ -359,7 +359,7 @@ static void inode_go_inval(struct gfs2_glock *gl, int flags)
 {
 	struct gfs2_inode *ip = gfs2_glock2inode(gl);
 
-	gfs2_assert_withdraw(gl->gl_name.ln_sbd, !atomic_read(&gl->gl_ail_count));
+	gfs2_assert_withdraw(glock_sbd(gl), !atomic_read(&gl->gl_ail_count));
 
 	if (flags & DIO_METADATA) {
 		struct address_space *mapping = gfs2_glock2aspace(gl);
@@ -372,11 +372,11 @@ static void inode_go_inval(struct gfs2_glock *gl, int flags)
 		}
 	}
 
-	if (ip == GFS2_I(gl->gl_name.ln_sbd->sd_rindex)) {
-		gfs2_log_flush(gl->gl_name.ln_sbd, NULL,
+	if (ip == GFS2_I(glock_sbd(gl)->sd_rindex)) {
+		gfs2_log_flush(glock_sbd(gl), NULL,
 			       GFS2_LOG_HEAD_FLUSH_NORMAL |
 			       GFS2_LFC_INODE_GO_INVAL);
-		gl->gl_name.ln_sbd->sd_rindex_uptodate = 0;
+		glock_sbd(gl)->sd_rindex_uptodate = 0;
 	}
 	if (ip && S_ISREG(ip->i_inode.i_mode))
 		truncate_inode_pages(ip->i_inode.i_mapping, 0);
@@ -567,7 +567,7 @@ static void inode_go_dump(struct seq_file *seq, const struct gfs2_glock *gl,
 
 static void freeze_go_callback(struct gfs2_glock *gl, bool remote)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct super_block *sb = sdp->sd_vfs;
 
 	if (!remote ||
@@ -596,7 +596,7 @@ static void freeze_go_callback(struct gfs2_glock *gl, bool remote)
  */
 static int freeze_go_xmote_bh(struct gfs2_glock *gl)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct gfs2_inode *ip = GFS2_I(sdp->sd_jdesc->jd_inode);
 	struct gfs2_glock *j_gl = ip->i_gl;
 	struct gfs2_log_header_host head;
@@ -626,7 +626,7 @@ static int freeze_go_xmote_bh(struct gfs2_glock *gl)
 static void iopen_go_callback(struct gfs2_glock *gl, bool remote)
 {
 	struct gfs2_inode *ip = gl->gl_object;
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 
 	if (!remote || test_bit(SDF_KILL, &sdp->sd_flags))
 		return;
diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h
index d05d8fe4e456..db5fbf02c886 100644
--- a/fs/gfs2/incore.h
+++ b/fs/gfs2/incore.h
@@ -369,6 +369,16 @@ struct gfs2_glock {
 	struct rhash_head gl_node;
 };
 
+static inline unsigned int glock_type(const struct gfs2_glock *gl)
+{
+	return gl->gl_name.ln_type;
+}
+
+static inline u64 glock_number(const struct gfs2_glock *gl)
+{
+	return gl->gl_name.ln_number;
+}
+
 enum {
 	GIF_QD_LOCKED		= 1,
 	GIF_SW_PAGED		= 3,
@@ -839,6 +849,8 @@ struct gfs2_sbd {
 	struct dentry *debugfs_dir;    /* debugfs directory */
 };
 
+#define glock_sbd(gl) ((gl)->gl_name.ln_sbd)
+
 #define GFS2_BAD_INO 1
 
 static inline struct address_space *gfs2_aspace(struct gfs2_sbd *sdp)
@@ -853,9 +865,9 @@ static inline void gfs2_glstats_inc(struct gfs2_glock *gl, int which)
 
 static inline void gfs2_sbstats_inc(const struct gfs2_glock *gl, int which)
 {
-	const struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	const struct gfs2_sbd *sdp = glock_sbd(gl);
 	preempt_disable();
-	this_cpu_ptr(sdp->sd_lkstats)->lkstats[gl->gl_name.ln_type].stats[which]++;
+	this_cpu_ptr(sdp->sd_lkstats)->lkstats[glock_type(gl)].stats[which]++;
 	preempt_enable();
 }
 
diff --git a/fs/gfs2/lock_dlm.c b/fs/gfs2/lock_dlm.c
index b8d249925395..7828ad0b6f5a 100644
--- a/fs/gfs2/lock_dlm.c
+++ b/fs/gfs2/lock_dlm.c
@@ -8,6 +8,7 @@
 
 #include <linux/fs.h>
 #include <linux/dlm.h>
+#include <linux/hex.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/delay.h>
@@ -74,13 +75,13 @@ static inline void gfs2_update_reply_times(struct gfs2_glock *gl,
 					   bool blocking)
 {
 	struct gfs2_pcpu_lkstats *lks;
-	const unsigned gltype = gl->gl_name.ln_type;
+	const unsigned gltype = glock_type(gl);
 	unsigned index = blocking ? GFS2_LKS_SRTTB : GFS2_LKS_SRTT;
 	s64 rtt;
 
 	preempt_disable();
 	rtt = ktime_to_ns(ktime_sub(ktime_get_real(), gl->gl_dstamp));
-	lks = this_cpu_ptr(gl->gl_name.ln_sbd->sd_lkstats);
+	lks = this_cpu_ptr(glock_sbd(gl)->sd_lkstats);
 	gfs2_update_stats(&gl->gl_stats, index, rtt);		/* Local */
 	gfs2_update_stats(&lks->lkstats[gltype], index, rtt);	/* Global */
 	preempt_enable();
@@ -100,7 +101,7 @@ static inline void gfs2_update_reply_times(struct gfs2_glock *gl,
 static inline void gfs2_update_request_times(struct gfs2_glock *gl)
 {
 	struct gfs2_pcpu_lkstats *lks;
-	const unsigned gltype = gl->gl_name.ln_type;
+	const unsigned gltype = glock_type(gl);
 	ktime_t dstamp;
 	s64 irt;
 
@@ -108,7 +109,7 @@ static inline void gfs2_update_request_times(struct gfs2_glock *gl)
 	dstamp = gl->gl_dstamp;
 	gl->gl_dstamp = ktime_get_real();
 	irt = ktime_to_ns(ktime_sub(gl->gl_dstamp, dstamp));
-	lks = this_cpu_ptr(gl->gl_name.ln_sbd->sd_lkstats);
+	lks = this_cpu_ptr(glock_sbd(gl)->sd_lkstats);
 	gfs2_update_stats(&gl->gl_stats, GFS2_LKS_SIRT, irt);		/* Local */
 	gfs2_update_stats(&lks->lkstats[gltype], GFS2_LKS_SIRT, irt);	/* Global */
 	preempt_enable();
@@ -195,7 +196,7 @@ static void gdlm_bast(void *arg, int mode)
 		gfs2_glock_cb(gl, LM_ST_SHARED);
 		break;
 	default:
-		fs_err(gl->gl_name.ln_sbd, "unknown bast mode %d\n", mode);
+		fs_err(glock_sbd(gl), "unknown bast mode %d\n", mode);
 		BUG();
 	}
 }
@@ -276,7 +277,7 @@ static void gfs2_reverse_hex(char *c, u64 value)
 static int gdlm_lock(struct gfs2_glock *gl, unsigned int req_state,
 		     unsigned int flags)
 {
-	struct lm_lockstruct *ls = &gl->gl_name.ln_sbd->sd_lockstruct;
+	struct lm_lockstruct *ls = &glock_sbd(gl)->sd_lockstruct;
 	bool blocking;
 	int cur, req;
 	u32 lkf;
@@ -284,8 +285,8 @@ static int gdlm_lock(struct gfs2_glock *gl, unsigned int req_state,
 	int error;
 
 	gl->gl_req = req_state;
-	cur = make_mode(gl->gl_name.ln_sbd, gl->gl_state);
-	req = make_mode(gl->gl_name.ln_sbd, req_state);
+	cur = make_mode(glock_sbd(gl), gl->gl_state);
+	req = make_mode(glock_sbd(gl), req_state);
 	blocking = !down_conversion(cur, req) &&
 		   !(flags & (LM_FLAG_TRY|LM_FLAG_TRY_1CB));
 	lkf = make_flags(gl, flags, req, blocking);
@@ -296,8 +297,8 @@ static int gdlm_lock(struct gfs2_glock *gl, unsigned int req_state,
 	if (test_bit(GLF_INITIAL, &gl->gl_flags)) {
 		memset(strname, ' ', GDLM_STRNAME_BYTES - 1);
 		strname[GDLM_STRNAME_BYTES - 1] = '\0';
-		gfs2_reverse_hex(strname + 7, gl->gl_name.ln_type);
-		gfs2_reverse_hex(strname + 23, gl->gl_name.ln_number);
+		gfs2_reverse_hex(strname + 7, glock_type(gl));
+		gfs2_reverse_hex(strname + 23, glock_number(gl));
 		gl->gl_dstamp = ktime_get_real();
 	} else {
 		gfs2_update_request_times(gl);
@@ -323,7 +324,7 @@ static int gdlm_lock(struct gfs2_glock *gl, unsigned int req_state,
 
 static void gdlm_put_lock(struct gfs2_glock *gl)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct lm_lockstruct *ls = &sdp->sd_lockstruct;
 	uint32_t flags = 0;
 	int error;
@@ -375,14 +376,14 @@ static void gdlm_put_lock(struct gfs2_glock *gl)
 
 	if (error) {
 		fs_err(sdp, "gdlm_unlock %x,%llx err=%d\n",
-		       gl->gl_name.ln_type,
-		       (unsigned long long)gl->gl_name.ln_number, error);
+		       glock_type(gl),
+		       (unsigned long long) glock_number(gl), error);
 	}
 }
 
 static void gdlm_cancel(struct gfs2_glock *gl)
 {
-	struct lm_lockstruct *ls = &gl->gl_name.ln_sbd->sd_lockstruct;
+	struct lm_lockstruct *ls = &glock_sbd(gl)->sd_lockstruct;
 
 	down_read(&ls->ls_sem);
 	if (likely(ls->ls_dlm != NULL)) {
diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c
index 8312cd2cdae4..9df1c3da1889 100644
--- a/fs/gfs2/log.c
+++ b/fs/gfs2/log.c
@@ -888,8 +888,9 @@ void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd,
 		     sb->s_blocksize - LH_V1_SIZE - 4);
 	lh->lh_crc = cpu_to_be32(crc);
 
-	gfs2_log_write(sdp, jd, page, sb->s_blocksize, 0, dblock);
-	gfs2_log_submit_bio(&jd->jd_log_bio, REQ_OP_WRITE | op_flags);
+	gfs2_log_write(sdp, jd, page, sb->s_blocksize, 0, dblock,
+		       REQ_OP_WRITE | op_flags);
+	gfs2_log_submit_write(sdp, &jd->jd_log_bio);
 }
 
 /**
@@ -1096,7 +1097,7 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags)
 	if (gfs2_withdrawn(sdp))
 		goto out_withdraw;
 	if (sdp->sd_jdesc)
-		gfs2_log_submit_bio(&sdp->sd_jdesc->jd_log_bio, REQ_OP_WRITE);
+		gfs2_log_submit_write(sdp, &sdp->sd_jdesc->jd_log_bio);
 	if (gfs2_withdrawn(sdp))
 		goto out_withdraw;
 
diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c
index 97ebe457c00a..848d53a3c3da 100644
--- a/fs/gfs2/lops.c
+++ b/fs/gfs2/lops.c
@@ -65,15 +65,15 @@ void gfs2_pin(struct gfs2_sbd *sdp, struct buffer_head *bh)
 
 static bool buffer_is_rgrp(const struct gfs2_bufdata *bd)
 {
-	return bd->bd_gl->gl_name.ln_type == LM_TYPE_RGRP;
+	return glock_type(bd->bd_gl) == LM_TYPE_RGRP;
 }
 
 static void maybe_release_space(struct gfs2_bufdata *bd)
 {
 	struct gfs2_glock *gl = bd->bd_gl;
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct gfs2_rgrpd *rgd = gfs2_glock2rgrp(gl);
-	unsigned int index = bd->bd_bh->b_blocknr - gl->gl_name.ln_number;
+	unsigned int index = bd->bd_bh->b_blocknr - glock_number(gl);
 	struct gfs2_bitmap *bi = rgd->rd_bits + index;
 
 	rgrp_lock_local(rgd);
@@ -229,21 +229,21 @@ static void gfs2_end_log_write(struct bio *bio)
 }
 
 /**
- * gfs2_log_submit_bio - Submit any pending log bio
+ * gfs2_log_submit_write - Submit a pending log write bio
+ * @sdp: The super block
  * @biop: Address of the bio pointer
- * @opf: REQ_OP | op_flags
  *
  * Submit any pending part-built or full bio to the block device. If
  * there is no pending bio, then this is a no-op.
  */
 
-void gfs2_log_submit_bio(struct bio **biop, blk_opf_t opf)
+void gfs2_log_submit_write(struct gfs2_sbd *sdp, struct bio **biop)
 {
 	struct bio *bio = *biop;
 	if (bio) {
-		struct gfs2_sbd *sdp = bio->bi_private;
 		atomic_inc(&sdp->sd_log_in_flight);
-		bio->bi_opf = opf;
+		bio->bi_private = sdp;
+		bio->bi_end_io = gfs2_end_log_write;
 		submit_bio(bio);
 		*biop = NULL;
 	}
@@ -253,7 +253,7 @@ void gfs2_log_submit_bio(struct bio **biop, blk_opf_t opf)
  * gfs2_log_alloc_bio - Allocate a bio
  * @sdp: The super block
  * @blkno: The device block number we want to write to
- * @end_io: The bi_end_io callback
+ * @opf: REQ_OP | op_flags
  *
  * Allocate a new bio, initialize it with the given parameters and return it.
  *
@@ -261,14 +261,12 @@ void gfs2_log_submit_bio(struct bio **biop, blk_opf_t opf)
  */
 
 static struct bio *gfs2_log_alloc_bio(struct gfs2_sbd *sdp, u64 blkno,
-				      bio_end_io_t *end_io)
+				      blk_opf_t opf)
 {
 	struct super_block *sb = sdp->sd_vfs;
-	struct bio *bio = bio_alloc(sb->s_bdev, BIO_MAX_VECS, 0, GFP_NOIO);
+	struct bio *bio = bio_alloc(sb->s_bdev, BIO_MAX_VECS, opf, GFP_NOIO);
 
 	bio->bi_iter.bi_sector = blkno << sdp->sd_fsb2bb_shift;
-	bio->bi_end_io = end_io;
-	bio->bi_private = sdp;
 
 	return bio;
 }
@@ -278,7 +276,7 @@ static struct bio *gfs2_log_alloc_bio(struct gfs2_sbd *sdp, u64 blkno,
  * @sdp: The super block
  * @blkno: The device block number we want to write to
  * @biop: The bio to get or allocate
- * @op: REQ_OP
+ * @opf: REQ_OP | op_flags
  * @end_io: The bi_end_io callback
  * @flush: Always flush the current bio and allocate a new one?
  *
@@ -291,8 +289,8 @@ static struct bio *gfs2_log_alloc_bio(struct gfs2_sbd *sdp, u64 blkno,
  */
 
 static struct bio *gfs2_log_get_bio(struct gfs2_sbd *sdp, u64 blkno,
-				    struct bio **biop, enum req_op op,
-				    bio_end_io_t *end_io, bool flush)
+				    struct bio **biop, blk_opf_t opf,
+				    bool flush)
 {
 	struct bio *bio = *biop;
 
@@ -303,10 +301,10 @@ static struct bio *gfs2_log_get_bio(struct gfs2_sbd *sdp, u64 blkno,
 		nblk >>= sdp->sd_fsb2bb_shift;
 		if (blkno == nblk && !flush)
 			return bio;
-		gfs2_log_submit_bio(biop, op);
+		gfs2_log_submit_write(sdp, biop);
 	}
 
-	*biop = gfs2_log_alloc_bio(sdp, blkno, end_io);
+	*biop = gfs2_log_alloc_bio(sdp, blkno, opf);
 	return *biop;
 }
 
@@ -318,6 +316,7 @@ static struct bio *gfs2_log_get_bio(struct gfs2_sbd *sdp, u64 blkno,
  * @size: the size of the data to write
  * @offset: the offset within the page 
  * @blkno: block number of the log entry
+ * @opf: REQ_OP | op_flags
  *
  * Try and add the page segment to the current bio. If that fails,
  * submit the current bio to the device and create a new one, and
@@ -326,17 +325,15 @@ static struct bio *gfs2_log_get_bio(struct gfs2_sbd *sdp, u64 blkno,
 
 void gfs2_log_write(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd,
 		    struct page *page, unsigned size, unsigned offset,
-		    u64 blkno)
+		    u64 blkno, blk_opf_t opf)
 {
 	struct bio *bio;
 	int ret;
 
-	bio = gfs2_log_get_bio(sdp, blkno, &jd->jd_log_bio, REQ_OP_WRITE,
-			       gfs2_end_log_write, false);
+	bio = gfs2_log_get_bio(sdp, blkno, &jd->jd_log_bio, opf, false);
 	ret = bio_add_page(bio, page, size, offset);
 	if (ret == 0) {
-		bio = gfs2_log_get_bio(sdp, blkno, &jd->jd_log_bio,
-				       REQ_OP_WRITE, gfs2_end_log_write, true);
+		bio = gfs2_log_get_bio(sdp, blkno, &jd->jd_log_bio, opf, true);
 		ret = bio_add_page(bio, page, size, offset);
 		WARN_ON(ret == 0);
 	}
@@ -359,7 +356,7 @@ static void gfs2_log_write_bh(struct gfs2_sbd *sdp, struct buffer_head *bh)
 	dblock = gfs2_log_bmap(sdp->sd_jdesc, sdp->sd_log_flush_head);
 	gfs2_log_incr_head(sdp);
 	gfs2_log_write(sdp, sdp->sd_jdesc, folio_page(bh->b_folio, 0),
-			bh->b_size, bh_offset(bh), dblock);
+			bh->b_size, bh_offset(bh), dblock, REQ_OP_WRITE);
 }
 
 /**
@@ -380,7 +377,8 @@ static void gfs2_log_write_page(struct gfs2_sbd *sdp, struct page *page)
 
 	dblock = gfs2_log_bmap(sdp->sd_jdesc, sdp->sd_log_flush_head);
 	gfs2_log_incr_head(sdp);
-	gfs2_log_write(sdp, sdp->sd_jdesc, page, sb->s_blocksize, 0, dblock);
+	gfs2_log_write(sdp, sdp->sd_jdesc, page, sb->s_blocksize, 0, dblock,
+		       REQ_OP_WRITE);
 }
 
 /**
@@ -477,11 +475,12 @@ static void gfs2_jhead_process_page(struct gfs2_jdesc *jd, unsigned long index,
 	folio_put_refs(folio, 2);
 }
 
-static struct bio *gfs2_chain_bio(struct bio *prev, unsigned int nr_iovecs)
+static struct bio *gfs2_chain_bio(struct bio *prev, unsigned int nr_iovecs,
+				  blk_opf_t opf)
 {
 	struct bio *new;
 
-	new = bio_alloc(prev->bi_bdev, nr_iovecs, prev->bi_opf, GFP_NOIO);
+	new = bio_alloc(prev->bi_bdev, nr_iovecs, opf, GFP_NOIO);
 	bio_clone_blkg_association(new, prev);
 	new->bi_iter.bi_sector = bio_end_sector(prev);
 	bio_chain(prev, new);
@@ -489,6 +488,13 @@ static struct bio *gfs2_chain_bio(struct bio *prev, unsigned int nr_iovecs)
 	return new;
 }
 
+static void gfs2_log_submit_read(struct gfs2_sbd *sdp, struct bio *bio)
+{
+	bio->bi_private = sdp;
+	bio->bi_end_io = gfs2_end_log_read;
+	submit_bio(bio);
+}
+
 /**
  * gfs2_find_jhead - find the head of a log
  * @jd: The journal descriptor
@@ -546,18 +552,18 @@ int gfs2_find_jhead(struct gfs2_jdesc *jd, struct gfs2_log_header_host *head)
 					unsigned int blocks =
 						(PAGE_SIZE - off) >> bsize_shift;
 
-					bio = gfs2_chain_bio(bio, blocks);
+					bio = gfs2_chain_bio(bio, blocks,
+							     REQ_OP_READ);
 					goto add_block_to_new_bio;
 				}
 			}
 
 			if (bio) {
 				blocks_submitted = block;
-				submit_bio(bio);
+				gfs2_log_submit_read(sdp, bio);
 			}
 
-			bio = gfs2_log_alloc_bio(sdp, dblock, gfs2_end_log_read);
-			bio->bi_opf = REQ_OP_READ;
+			bio = gfs2_log_alloc_bio(sdp, dblock, REQ_OP_READ);
 add_block_to_new_bio:
 			bio_add_folio_nofail(bio, folio, bsize, off);
 block_added:
@@ -578,7 +584,7 @@ int gfs2_find_jhead(struct gfs2_jdesc *jd, struct gfs2_log_header_host *head)
 
 out:
 	if (bio)
-		submit_bio(bio);
+		gfs2_log_submit_read(sdp, bio);
 	while (blocks_read < block) {
 		gfs2_jhead_process_page(jd, blocks_read >> shift, head, &done);
 		blocks_read += PAGE_SIZE >> bsize_shift;
diff --git a/fs/gfs2/lops.h b/fs/gfs2/lops.h
index be740bf33666..2f6707591825 100644
--- a/fs/gfs2/lops.h
+++ b/fs/gfs2/lops.h
@@ -16,8 +16,8 @@ void gfs2_log_incr_head(struct gfs2_sbd *sdp);
 u64 gfs2_log_bmap(struct gfs2_jdesc *jd, unsigned int lbn);
 void gfs2_log_write(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd,
 		    struct page *page, unsigned size, unsigned offset,
-		    u64 blkno);
-void gfs2_log_submit_bio(struct bio **biop, blk_opf_t opf);
+		    u64 blkno, blk_opf_t opf);
+void gfs2_log_submit_write(struct gfs2_sbd *sdp, struct bio **biop);
 void gfs2_pin(struct gfs2_sbd *sdp, struct buffer_head *bh);
 int gfs2_find_jhead(struct gfs2_jdesc *jd,
 		    struct gfs2_log_header_host *head);
diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c
index e4356198d8d8..3c8e4553102d 100644
--- a/fs/gfs2/meta_io.c
+++ b/fs/gfs2/meta_io.c
@@ -126,7 +126,7 @@ const struct address_space_operations gfs2_rgrp_aops = {
 struct buffer_head *gfs2_getbuf(struct gfs2_glock *gl, u64 blkno, int create)
 {
 	struct address_space *mapping = gfs2_glock2aspace(gl);
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct folio *folio;
 	struct buffer_head *bh;
 	unsigned int shift;
@@ -259,7 +259,7 @@ static void gfs2_submit_bhs(blk_opf_t opf, struct buffer_head *bhs[], int num)
 int gfs2_meta_read(struct gfs2_glock *gl, u64 blkno, int flags,
 		   int rahead, struct buffer_head **bhp)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct buffer_head *bh, *bhs[2];
 	int num = 0;
 
@@ -513,7 +513,7 @@ int gfs2_meta_buffer(struct gfs2_inode *ip, u32 mtype, u64 num,
 
 struct buffer_head *gfs2_meta_ra(struct gfs2_glock *gl, u64 dblock, u32 extlen)
 {
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct buffer_head *first_bh, *bh;
 	u32 max_ra = gfs2_tune_get(sdp, gt_max_readahead) >>
 			  sdp->sd_sb.sb_bsize_shift;
diff --git a/fs/gfs2/meta_io.h b/fs/gfs2/meta_io.h
index b7c8a6684d02..2fe5dec193ed 100644
--- a/fs/gfs2/meta_io.h
+++ b/fs/gfs2/meta_io.h
@@ -43,7 +43,7 @@ static inline struct gfs2_sbd *gfs2_mapping2sbd(struct address_space *mapping)
 	if (mapping->a_ops == &gfs2_meta_aops) {
 		struct gfs2_glock_aspace *gla =
 			container_of(mapping, struct gfs2_glock_aspace, mapping);
-		return gla->glock.gl_name.ln_sbd;
+		return glock_sbd(&gla->glock);
 	} else
 		return inode->i_sb->s_fs_info;
 }
diff --git a/fs/gfs2/quota.c b/fs/gfs2/quota.c
index b1692f12a602..1c3455093ae8 100644
--- a/fs/gfs2/quota.c
+++ b/fs/gfs2/quota.c
@@ -334,6 +334,7 @@ static void qd_put(struct gfs2_quota_data *qd)
 		lockref_mark_dead(&qd->qd_lockref);
 		spin_unlock(&qd->qd_lockref.lock);
 
+		list_lru_del_obj(&gfs2_qd_lru, &qd->qd_lru);
 		gfs2_qd_dispose(qd);
 		return;
 	}
@@ -978,7 +979,7 @@ static int do_sync(unsigned int num_qd, struct gfs2_quota_data **qda,
 		gfs2_glock_dq_uninit(&ghs[qx]);
 	inode_unlock(&ip->i_inode);
 	kfree(ghs);
-	gfs2_log_flush(ip->i_gl->gl_name.ln_sbd, ip->i_gl,
+	gfs2_log_flush(glock_sbd(ip->i_gl), ip->i_gl,
 		       GFS2_LOG_HEAD_FLUSH_NORMAL | GFS2_LFC_DO_SYNC);
 	if (!error) {
 		for (x = 0; x < num_qd; x++) {
@@ -1027,7 +1028,7 @@ static int do_glock(struct gfs2_quota_data *qd, int force_refresh,
 	struct gfs2_holder i_gh;
 	int error;
 
-	gfs2_assert_warn(sdp, sdp == qd->qd_gl->gl_name.ln_sbd);
+	gfs2_assert_warn(sdp, sdp == glock_sbd(qd->qd_gl));
 restart:
 	error = gfs2_glock_nq_init(qd->qd_gl, LM_ST_SHARED, 0, q_gh);
 	if (error)
diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c
index b14e54b38ee8..8a97ca734afc 100644
--- a/fs/gfs2/rgrp.c
+++ b/fs/gfs2/rgrp.c
@@ -1923,7 +1923,7 @@ static void try_rgrp_unlink(struct gfs2_rgrpd *rgd, u64 *last_unlinked, u64 skip
 static bool gfs2_rgrp_congested(const struct gfs2_rgrpd *rgd, int loops)
 {
 	const struct gfs2_glock *gl = rgd->rd_gl;
-	const struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	const struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct gfs2_lkstats *st;
 	u64 r_dcount, l_dcount;
 	u64 l_srttb, a_srttb = 0;
diff --git a/fs/gfs2/trace_gfs2.h b/fs/gfs2/trace_gfs2.h
index fcfbf68ec725..4a146cd9d289 100644
--- a/fs/gfs2/trace_gfs2.h
+++ b/fs/gfs2/trace_gfs2.h
@@ -111,9 +111,9 @@ TRACE_EVENT(gfs2_glock_state_change,
 	),
 
 	TP_fast_assign(
-		__entry->dev		= gl->gl_name.ln_sbd->sd_vfs->s_dev;
-		__entry->glnum		= gl->gl_name.ln_number;
-		__entry->gltype		= gl->gl_name.ln_type;
+		__entry->dev		= glock_sbd(gl)->sd_vfs->s_dev;
+		__entry->glnum		= glock_number(gl);
+		__entry->gltype		= glock_type(gl);
 		__entry->cur_state	= glock_trace_state(gl->gl_state);
 		__entry->new_state	= glock_trace_state(new_state);
 		__entry->tgt_state	= glock_trace_state(gl->gl_target);
@@ -147,9 +147,9 @@ TRACE_EVENT(gfs2_glock_put,
 	),
 
 	TP_fast_assign(
-		__entry->dev		= gl->gl_name.ln_sbd->sd_vfs->s_dev;
-		__entry->gltype		= gl->gl_name.ln_type;
-		__entry->glnum		= gl->gl_name.ln_number;
+		__entry->dev		= glock_sbd(gl)->sd_vfs->s_dev;
+		__entry->gltype		= glock_type(gl);
+		__entry->glnum		= glock_number(gl);
 		__entry->cur_state	= glock_trace_state(gl->gl_state);
 		__entry->flags		= gl->gl_flags  | (gl->gl_object ? (1UL<<GLF_OBJECT) : 0);
 	),
@@ -181,9 +181,9 @@ TRACE_EVENT(gfs2_demote_rq,
 	),
 
 	TP_fast_assign(
-		__entry->dev		= gl->gl_name.ln_sbd->sd_vfs->s_dev;
-		__entry->gltype		= gl->gl_name.ln_type;
-		__entry->glnum		= gl->gl_name.ln_number;
+		__entry->dev		= glock_sbd(gl)->sd_vfs->s_dev;
+		__entry->gltype		= glock_type(gl);
+		__entry->glnum		= glock_number(gl);
 		__entry->cur_state	= glock_trace_state(gl->gl_state);
 		__entry->dmt_state	= glock_trace_state(gl->gl_demote_state);
 		__entry->flags		= gl->gl_flags  | (gl->gl_object ? (1UL<<GLF_OBJECT) : 0);
@@ -215,9 +215,9 @@ TRACE_EVENT(gfs2_promote,
 	),
 
 	TP_fast_assign(
-		__entry->dev	= gh->gh_gl->gl_name.ln_sbd->sd_vfs->s_dev;
-		__entry->glnum	= gh->gh_gl->gl_name.ln_number;
-		__entry->gltype	= gh->gh_gl->gl_name.ln_type;
+		__entry->dev	= glock_sbd(gh->gh_gl)->sd_vfs->s_dev;
+		__entry->glnum	= glock_number(gh->gh_gl);
+		__entry->gltype	= glock_type(gh->gh_gl);
 		__entry->state	= glock_trace_state(gh->gh_state);
 	),
 
@@ -243,9 +243,9 @@ TRACE_EVENT(gfs2_glock_queue,
 	),
 
 	TP_fast_assign(
-		__entry->dev	= gh->gh_gl->gl_name.ln_sbd->sd_vfs->s_dev;
-		__entry->glnum	= gh->gh_gl->gl_name.ln_number;
-		__entry->gltype	= gh->gh_gl->gl_name.ln_type;
+		__entry->dev	= glock_sbd(gh->gh_gl)->sd_vfs->s_dev;
+		__entry->glnum	= glock_number(gh->gh_gl);
+		__entry->gltype	= glock_type(gh->gh_gl);
 		__entry->queue	= queue;
 		__entry->state	= glock_trace_state(gh->gh_state);
 	),
@@ -282,9 +282,9 @@ TRACE_EVENT(gfs2_glock_lock_time,
 	),
 
 	TP_fast_assign(
-		__entry->dev            = gl->gl_name.ln_sbd->sd_vfs->s_dev;
-		__entry->glnum          = gl->gl_name.ln_number;
-		__entry->gltype         = gl->gl_name.ln_type;
+		__entry->dev            = glock_sbd(gl)->sd_vfs->s_dev;
+		__entry->glnum          = glock_number(gl);
+		__entry->gltype         = glock_type(gl);
 		__entry->status		= gl->gl_lksb.sb_status;
 		__entry->flags		= gl->gl_lksb.sb_flags;
 		__entry->tdiff		= tdiff;
@@ -337,11 +337,11 @@ TRACE_EVENT(gfs2_pin,
 	),
 
 	TP_fast_assign(
-		__entry->dev		= bd->bd_gl->gl_name.ln_sbd->sd_vfs->s_dev;
+		__entry->dev		= glock_sbd(bd->bd_gl)->sd_vfs->s_dev;
 		__entry->pin		= pin;
 		__entry->len		= bd->bd_bh->b_size;
 		__entry->block		= bd->bd_bh->b_blocknr;
-		__entry->ino		= bd->bd_gl->gl_name.ln_number;
+		__entry->ino		= glock_number(bd->bd_gl);
 	),
 
 	TP_printk("%u,%u log %s %llu/%lu inode %llu",
@@ -458,7 +458,7 @@ TRACE_EVENT(gfs2_bmap,
 	),
 
 	TP_fast_assign(
-		__entry->dev            = ip->i_gl->gl_name.ln_sbd->sd_vfs->s_dev;
+		__entry->dev            = glock_sbd(ip->i_gl)->sd_vfs->s_dev;
 		__entry->lblock		= lblock;
 		__entry->pblock		= buffer_mapped(bh) ?  bh->b_blocknr : 0;
 		__entry->inum		= ip->i_no_addr;
@@ -494,7 +494,7 @@ TRACE_EVENT(gfs2_iomap_start,
 	),
 
 	TP_fast_assign(
-		__entry->dev            = ip->i_gl->gl_name.ln_sbd->sd_vfs->s_dev;
+		__entry->dev            = glock_sbd(ip->i_gl)->sd_vfs->s_dev;
 		__entry->inum		= ip->i_no_addr;
 		__entry->pos		= pos;
 		__entry->length		= length;
@@ -526,7 +526,7 @@ TRACE_EVENT(gfs2_iomap_end,
 	),
 
 	TP_fast_assign(
-		__entry->dev            = ip->i_gl->gl_name.ln_sbd->sd_vfs->s_dev;
+		__entry->dev            = glock_sbd(ip->i_gl)->sd_vfs->s_dev;
 		__entry->inum		= ip->i_no_addr;
 		__entry->offset		= iomap->offset;
 		__entry->length		= iomap->length;
@@ -568,7 +568,7 @@ TRACE_EVENT(gfs2_block_alloc,
 	),
 
 	TP_fast_assign(
-		__entry->dev		= rgd->rd_gl->gl_name.ln_sbd->sd_vfs->s_dev;
+		__entry->dev		= glock_sbd(rgd->rd_gl)->sd_vfs->s_dev;
 		__entry->start		= block;
 		__entry->inum		= ip->i_no_addr;
 		__entry->len		= len;
diff --git a/fs/gfs2/trans.c b/fs/gfs2/trans.c
index 6df65540e13d..95f2632cdb01 100644
--- a/fs/gfs2/trans.c
+++ b/fs/gfs2/trans.c
@@ -197,7 +197,7 @@ static struct gfs2_bufdata *gfs2_alloc_bufdata(struct gfs2_glock *gl,
 void gfs2_trans_add_data(struct gfs2_glock *gl, struct buffer_head *bh)
 {
 	struct gfs2_trans *tr = current->journal_info;
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct gfs2_bufdata *bd;
 
 	lock_buffer(bh);
@@ -255,7 +255,7 @@ void gfs2_trans_add_databufs(struct gfs2_glock *gl, struct folio *folio,
 void gfs2_trans_add_meta(struct gfs2_glock *gl, struct buffer_head *bh)
 {
 
-	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
+	struct gfs2_sbd *sdp = glock_sbd(gl);
 	struct super_block *sb = sdp->sd_vfs;
 	struct gfs2_bufdata *bd;
 	struct gfs2_meta_header *mh;
diff --git a/fs/init.c b/fs/init.c
index e0f5429c0a49..9b56ebca8cc6 100644
--- a/fs/init.c
+++ b/fs/init.c
@@ -27,20 +27,6 @@ int __init init_mount(const char *dev_name, const char *dir_name,
 	return ret;
 }
 
-int __init init_umount(const char *name, int flags)
-{
-	int lookup_flags = LOOKUP_MOUNTPOINT;
-	struct path path;
-	int ret;
-
-	if (!(flags & UMOUNT_NOFOLLOW))
-		lookup_flags |= LOOKUP_FOLLOW;
-	ret = kern_path(name, lookup_flags, &path);
-	if (ret)
-		return ret;
-	return path_umount(&path, flags);
-}
-
 int __init init_chdir(const char *filename)
 {
 	struct path path;
diff --git a/fs/internal.h b/fs/internal.h
index ab638d41ab81..e333b105337a 100644
--- a/fs/internal.h
+++ b/fs/internal.h
@@ -44,7 +44,6 @@ extern void __init chrdev_init(void);
 /*
  * fs_context.c
  */
-extern const struct fs_context_operations legacy_fs_context_ops;
 extern int parse_monolithic_mount_data(struct fs_context *, void *);
 extern void vfs_clean_context(struct fs_context *fc);
 extern int finish_clean_context(struct fs_context *fc);
diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c
index e5c1ca440d93..fd9a2cf95620 100644
--- a/fs/iomap/buffered-io.c
+++ b/fs/iomap/buffered-io.c
@@ -832,7 +832,7 @@ static struct folio *__iomap_get_folio(struct iomap_iter *iter,
 	if (!mapping_large_folio_support(iter->inode->i_mapping))
 		len = min_t(size_t, len, PAGE_SIZE - offset_in_page(pos));
 
-	if (iter->fbatch) {
+	if (iter->iomap.flags & IOMAP_F_FOLIO_BATCH) {
 		struct folio *folio = folio_batch_next(iter->fbatch);
 
 		if (!folio)
@@ -929,7 +929,7 @@ static int iomap_write_begin(struct iomap_iter *iter,
 	 * process so return and let the caller iterate and refill the batch.
 	 */
 	if (!folio) {
-		WARN_ON_ONCE(!iter->fbatch);
+		WARN_ON_ONCE(!(iter->iomap.flags & IOMAP_F_FOLIO_BATCH));
 		return 0;
 	}
 
@@ -1544,23 +1544,39 @@ static int iomap_zero_iter(struct iomap_iter *iter, bool *did_zero,
 	return status;
 }
 
-loff_t
+/**
+ * iomap_fill_dirty_folios - fill a folio batch with dirty folios
+ * @iter: Iteration structure
+ * @start: Start offset of range. Updated based on lookup progress.
+ * @end: End offset of range
+ * @iomap_flags: Flags to set on the associated iomap to track the batch.
+ *
+ * Returns the folio count directly. Also returns the associated control flag if
+ * the the batch lookup is performed and the expected offset of a subsequent
+ * lookup via out params. The caller is responsible to set the flag on the
+ * associated iomap.
+ */
+unsigned int
 iomap_fill_dirty_folios(
 	struct iomap_iter	*iter,
-	loff_t			offset,
-	loff_t			length)
+	loff_t			*start,
+	loff_t			end,
+	unsigned int		*iomap_flags)
 {
 	struct address_space	*mapping = iter->inode->i_mapping;
-	pgoff_t			start = offset >> PAGE_SHIFT;
-	pgoff_t			end = (offset + length - 1) >> PAGE_SHIFT;
+	pgoff_t			pstart = *start >> PAGE_SHIFT;
+	pgoff_t			pend = (end - 1) >> PAGE_SHIFT;
+	unsigned int		count;
 
-	iter->fbatch = kmalloc(sizeof(struct folio_batch), GFP_KERNEL);
-	if (!iter->fbatch)
-		return offset + length;
-	folio_batch_init(iter->fbatch);
+	if (!iter->fbatch) {
+		*start = end;
+		return 0;
+	}
 
-	filemap_get_folios_dirty(mapping, &start, end, iter->fbatch);
-	return (start << PAGE_SHIFT);
+	count = filemap_get_folios_dirty(mapping, &pstart, pend, iter->fbatch);
+	*start = (pstart << PAGE_SHIFT);
+	*iomap_flags |= IOMAP_F_FOLIO_BATCH;
+	return count;
 }
 EXPORT_SYMBOL_GPL(iomap_fill_dirty_folios);
 
@@ -1569,17 +1585,21 @@ iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
 		const struct iomap_ops *ops,
 		const struct iomap_write_ops *write_ops, void *private)
 {
+	struct folio_batch fbatch;
 	struct iomap_iter iter = {
 		.inode		= inode,
 		.pos		= pos,
 		.len		= len,
 		.flags		= IOMAP_ZERO,
 		.private	= private,
+		.fbatch		= &fbatch,
 	};
 	struct address_space *mapping = inode->i_mapping;
 	int ret;
 	bool range_dirty;
 
+	folio_batch_init(&fbatch);
+
 	/*
 	 * To avoid an unconditional flush, check pagecache state and only flush
 	 * if dirty and the fs returns a mapping that might convert on
@@ -1590,11 +1610,11 @@ iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
 	while ((ret = iomap_iter(&iter, ops)) > 0) {
 		const struct iomap *srcmap = iomap_iter_srcmap(&iter);
 
-		if (WARN_ON_ONCE(iter.fbatch &&
+		if (WARN_ON_ONCE((iter.iomap.flags & IOMAP_F_FOLIO_BATCH) &&
 				 srcmap->type != IOMAP_UNWRITTEN))
 			return -EIO;
 
-		if (!iter.fbatch &&
+		if (!(iter.iomap.flags & IOMAP_F_FOLIO_BATCH) &&
 		    (srcmap->type == IOMAP_HOLE ||
 		     srcmap->type == IOMAP_UNWRITTEN)) {
 			s64 status;
diff --git a/fs/iomap/iter.c b/fs/iomap/iter.c
index 8692e5e41c6d..c04796f6e57f 100644
--- a/fs/iomap/iter.c
+++ b/fs/iomap/iter.c
@@ -8,10 +8,10 @@
 
 static inline void iomap_iter_reset_iomap(struct iomap_iter *iter)
 {
-	if (iter->fbatch) {
+	if (iter->iomap.flags & IOMAP_F_FOLIO_BATCH) {
 		folio_batch_release(iter->fbatch);
-		kfree(iter->fbatch);
-		iter->fbatch = NULL;
+		folio_batch_reinit(iter->fbatch);
+		iter->iomap.flags &= ~IOMAP_F_FOLIO_BATCH;
 	}
 
 	iter->status = 0;
diff --git a/fs/jfs/jfs_logmgr.c b/fs/jfs/jfs_logmgr.c
index b343c5ea1159..5b1c5da04163 100644
--- a/fs/jfs/jfs_logmgr.c
+++ b/fs/jfs/jfs_logmgr.c
@@ -2311,6 +2311,7 @@ int jfsIOWait(void *arg)
 {
 	struct lbuf *bp;
 
+	set_freezable();
 	do {
 		spin_lock_irq(&log_redrive_lock);
 		while ((bp = log_redrive_list)) {
diff --git a/fs/jfs/namei.c b/fs/jfs/namei.c
index 65a218eba8fa..7879c049632b 100644
--- a/fs/jfs/namei.c
+++ b/fs/jfs/namei.c
@@ -1228,7 +1228,7 @@ static int jfs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 				jfs_err("jfs_rename: dtInsert returned -EIO");
 			goto out_tx;
 		}
-		if (S_ISDIR(old_ip->i_mode))
+		if (S_ISDIR(old_ip->i_mode) && old_dir != new_dir)
 			inc_nlink(new_dir);
 	}
 	/*
@@ -1244,7 +1244,9 @@ static int jfs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 		goto out_tx;
 	}
 	if (S_ISDIR(old_ip->i_mode)) {
-		drop_nlink(old_dir);
+		if (new_ip || old_dir != new_dir)
+			drop_nlink(old_dir);
+
 		if (old_dir != new_dir) {
 			/*
 			 * Change inode number of parent for moved directory
diff --git a/fs/libfs.c b/fs/libfs.c
index 9264523be85c..591eb649ebba 100644
--- a/fs/libfs.c
+++ b/fs/libfs.c
@@ -346,22 +346,22 @@ void simple_offset_remove(struct offset_ctx *octx, struct dentry *dentry)
  * User space expects the directory offset value of the replaced
  * (new) directory entry to be unchanged after a rename.
  *
- * Returns zero on success, a negative errno value on failure.
+ * Caller must have grabbed a slot for new_dentry in the maple_tree
+ * associated with new_dir, even if dentry is negative.
  */
-int simple_offset_rename(struct inode *old_dir, struct dentry *old_dentry,
-			 struct inode *new_dir, struct dentry *new_dentry)
+void simple_offset_rename(struct inode *old_dir, struct dentry *old_dentry,
+			  struct inode *new_dir, struct dentry *new_dentry)
 {
 	struct offset_ctx *old_ctx = old_dir->i_op->get_offset_ctx(old_dir);
 	struct offset_ctx *new_ctx = new_dir->i_op->get_offset_ctx(new_dir);
 	long new_offset = dentry2offset(new_dentry);
 
-	simple_offset_remove(old_ctx, old_dentry);
+	if (WARN_ON(!new_offset))
+		return;
 
-	if (new_offset) {
-		offset_set(new_dentry, 0);
-		return simple_offset_replace(new_ctx, old_dentry, new_offset);
-	}
-	return simple_offset_add(new_ctx, old_dentry);
+	simple_offset_remove(old_ctx, old_dentry);
+	offset_set(new_dentry, 0);
+	WARN_ON(simple_offset_replace(new_ctx, old_dentry, new_offset));
 }
 
 /**
@@ -388,31 +388,23 @@ int simple_offset_rename_exchange(struct inode *old_dir,
 	long new_index = dentry2offset(new_dentry);
 	int ret;
 
-	simple_offset_remove(old_ctx, old_dentry);
-	simple_offset_remove(new_ctx, new_dentry);
+	if (WARN_ON(!old_index || !new_index))
+		return -EINVAL;
 
-	ret = simple_offset_replace(new_ctx, old_dentry, new_index);
-	if (ret)
-		goto out_restore;
+	ret = mtree_store(&new_ctx->mt, new_index, old_dentry, GFP_KERNEL);
+	if (WARN_ON(ret))
+		return ret;
 
-	ret = simple_offset_replace(old_ctx, new_dentry, old_index);
-	if (ret) {
-		simple_offset_remove(new_ctx, old_dentry);
-		goto out_restore;
+	ret = mtree_store(&old_ctx->mt, old_index, new_dentry, GFP_KERNEL);
+	if (WARN_ON(ret)) {
+		mtree_store(&new_ctx->mt, new_index, new_dentry, GFP_KERNEL);
+		return ret;
 	}
 
-	ret = simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry);
-	if (ret) {
-		simple_offset_remove(new_ctx, old_dentry);
-		simple_offset_remove(old_ctx, new_dentry);
-		goto out_restore;
-	}
+	offset_set(old_dentry, new_index);
+	offset_set(new_dentry, old_index);
+	simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry);
 	return 0;
-
-out_restore:
-	(void)simple_offset_replace(old_ctx, old_dentry, old_index);
-	(void)simple_offset_replace(new_ctx, new_dentry, new_index);
-	return ret;
 }
 
 /**
diff --git a/fs/locks.c b/fs/locks.c
index 9f565802a88c..a684b367e18b 100644
--- a/fs/locks.c
+++ b/fs/locks.c
@@ -178,7 +178,6 @@ locks_get_lock_context(struct inode *inode, int type)
 {
 	struct file_lock_context *ctx;
 
-	/* paired with cmpxchg() below */
 	ctx = locks_inode_context(inode);
 	if (likely(ctx) || type == F_UNLCK)
 		goto out;
@@ -196,7 +195,18 @@ locks_get_lock_context(struct inode *inode, int type)
 	 * Assign the pointer if it's not already assigned. If it is, then
 	 * free the context we just allocated.
 	 */
-	if (cmpxchg(&inode->i_flctx, NULL, ctx)) {
+	spin_lock(&inode->i_lock);
+	if (!(inode->i_opflags & IOP_FLCTX)) {
+		VFS_BUG_ON_INODE(inode->i_flctx, inode);
+		WRITE_ONCE(inode->i_flctx, ctx);
+		/*
+		 * Paired with locks_inode_context().
+		 */
+		smp_store_release(&inode->i_opflags, inode->i_opflags | IOP_FLCTX);
+		spin_unlock(&inode->i_lock);
+	} else {
+		VFS_BUG_ON_INODE(!inode->i_flctx, inode);
+		spin_unlock(&inode->i_lock);
 		kmem_cache_free(flctx_cache, ctx);
 		ctx = locks_inode_context(inode);
 	}
@@ -369,10 +379,19 @@ locks_dispose_list(struct list_head *dispose)
 	while (!list_empty(dispose)) {
 		flc = list_first_entry(dispose, struct file_lock_core, flc_list);
 		list_del_init(&flc->flc_list);
-		if (flc->flc_flags & (FL_LEASE|FL_DELEG|FL_LAYOUT))
-			locks_free_lease(file_lease(flc));
-		else
-			locks_free_lock(file_lock(flc));
+		locks_free_lock(file_lock(flc));
+	}
+}
+
+static void
+lease_dispose_list(struct list_head *dispose)
+{
+	struct file_lock_core *flc;
+
+	while (!list_empty(dispose)) {
+		flc = list_first_entry(dispose, struct file_lock_core, flc_list);
+		list_del_init(&flc->flc_list);
+		locks_free_lease(file_lease(flc));
 	}
 }
 
@@ -576,10 +595,50 @@ lease_setup(struct file_lease *fl, void **priv)
 	__f_setown(filp, task_pid(current), PIDTYPE_TGID, 0);
 }
 
+/**
+ * lease_open_conflict - see if the given file points to an inode that has
+ *			 an existing open that would conflict with the
+ *			 desired lease.
+ * @filp:	file to check
+ * @arg:	type of lease that we're trying to acquire
+ *
+ * Check to see if there's an existing open fd on this file that would
+ * conflict with the lease we're trying to set.
+ */
+static int
+lease_open_conflict(struct file *filp, const int arg)
+{
+	struct inode *inode = file_inode(filp);
+	int self_wcount = 0, self_rcount = 0;
+
+	if (arg == F_RDLCK)
+		return inode_is_open_for_write(inode) ? -EAGAIN : 0;
+	else if (arg != F_WRLCK)
+		return 0;
+
+	/*
+	 * Make sure that only read/write count is from lease requestor.
+	 * Note that this will result in denying write leases when i_writecount
+	 * is negative, which is what we want.  (We shouldn't grant write leases
+	 * on files open for execution.)
+	 */
+	if (filp->f_mode & FMODE_WRITE)
+		self_wcount = 1;
+	else if (filp->f_mode & FMODE_READ)
+		self_rcount = 1;
+
+	if (atomic_read(&inode->i_writecount) != self_wcount ||
+	    atomic_read(&inode->i_readcount) != self_rcount)
+		return -EAGAIN;
+
+	return 0;
+}
+
 static const struct lease_manager_operations lease_manager_ops = {
 	.lm_break = lease_break_callback,
 	.lm_change = lease_modify,
 	.lm_setup = lease_setup,
+	.lm_open_conflict = lease_open_conflict,
 };
 
 /*
@@ -1620,7 +1679,7 @@ int __break_lease(struct inode *inode, unsigned int flags)
 	spin_unlock(&ctx->flc_lock);
 	percpu_up_read(&file_rwsem);
 
-	locks_dispose_list(&dispose);
+	lease_dispose_list(&dispose);
 	error = wait_event_interruptible_timeout(new_fl->c.flc_wait,
 						 list_empty(&new_fl->c.flc_blocked_member),
 						 break_time);
@@ -1643,7 +1702,7 @@ int __break_lease(struct inode *inode, unsigned int flags)
 out:
 	spin_unlock(&ctx->flc_lock);
 	percpu_up_read(&file_rwsem);
-	locks_dispose_list(&dispose);
+	lease_dispose_list(&dispose);
 free_lock:
 	locks_free_lease(new_fl);
 	return error;
@@ -1727,7 +1786,7 @@ static int __fcntl_getlease(struct file *filp, unsigned int flavor)
 		spin_unlock(&ctx->flc_lock);
 		percpu_up_read(&file_rwsem);
 
-		locks_dispose_list(&dispose);
+		lease_dispose_list(&dispose);
 	}
 	return type;
 }
@@ -1745,52 +1804,6 @@ int fcntl_getdeleg(struct file *filp, struct delegation *deleg)
 	return 0;
 }
 
-/**
- * check_conflicting_open - see if the given file points to an inode that has
- *			    an existing open that would conflict with the
- *			    desired lease.
- * @filp:	file to check
- * @arg:	type of lease that we're trying to acquire
- * @flags:	current lock flags
- *
- * Check to see if there's an existing open fd on this file that would
- * conflict with the lease we're trying to set.
- */
-static int
-check_conflicting_open(struct file *filp, const int arg, int flags)
-{
-	struct inode *inode = file_inode(filp);
-	int self_wcount = 0, self_rcount = 0;
-
-	if (flags & FL_LAYOUT)
-		return 0;
-	if (flags & FL_DELEG)
-		/* We leave these checks to the caller */
-		return 0;
-
-	if (arg == F_RDLCK)
-		return inode_is_open_for_write(inode) ? -EAGAIN : 0;
-	else if (arg != F_WRLCK)
-		return 0;
-
-	/*
-	 * Make sure that only read/write count is from lease requestor.
-	 * Note that this will result in denying write leases when i_writecount
-	 * is negative, which is what we want.  (We shouldn't grant write leases
-	 * on files open for execution.)
-	 */
-	if (filp->f_mode & FMODE_WRITE)
-		self_wcount = 1;
-	else if (filp->f_mode & FMODE_READ)
-		self_rcount = 1;
-
-	if (atomic_read(&inode->i_writecount) != self_wcount ||
-	    atomic_read(&inode->i_readcount) != self_rcount)
-		return -EAGAIN;
-
-	return 0;
-}
-
 static int
 generic_add_lease(struct file *filp, int arg, struct file_lease **flp, void **priv)
 {
@@ -1827,7 +1840,7 @@ generic_add_lease(struct file *filp, int arg, struct file_lease **flp, void **pr
 	percpu_down_read(&file_rwsem);
 	spin_lock(&ctx->flc_lock);
 	time_out_leases(inode, &dispose);
-	error = check_conflicting_open(filp, arg, lease->c.flc_flags);
+	error = lease->fl_lmops->lm_open_conflict(filp, arg);
 	if (error)
 		goto out;
 
@@ -1884,7 +1897,7 @@ generic_add_lease(struct file *filp, int arg, struct file_lease **flp, void **pr
 	 * precedes these checks.
 	 */
 	smp_mb();
-	error = check_conflicting_open(filp, arg, lease->c.flc_flags);
+	error = lease->fl_lmops->lm_open_conflict(filp, arg);
 	if (error) {
 		locks_unlink_lock_ctx(&lease->c);
 		goto out;
@@ -1896,7 +1909,7 @@ generic_add_lease(struct file *filp, int arg, struct file_lease **flp, void **pr
 out:
 	spin_unlock(&ctx->flc_lock);
 	percpu_up_read(&file_rwsem);
-	locks_dispose_list(&dispose);
+	lease_dispose_list(&dispose);
 	if (is_deleg)
 		inode_unlock(inode);
 	if (!error && !my_fl)
@@ -1932,7 +1945,7 @@ static int generic_delete_lease(struct file *filp, void *owner)
 		error = fl->fl_lmops->lm_change(victim, F_UNLCK, &dispose);
 	spin_unlock(&ctx->flc_lock);
 	percpu_up_read(&file_rwsem);
-	locks_dispose_list(&dispose);
+	lease_dispose_list(&dispose);
 	return error;
 }
 
@@ -2727,7 +2740,7 @@ locks_remove_lease(struct file *filp, struct file_lock_context *ctx)
 	spin_unlock(&ctx->flc_lock);
 	percpu_up_read(&file_rwsem);
 
-	locks_dispose_list(&dispose);
+	lease_dispose_list(&dispose);
 }
 
 /*
diff --git a/fs/namei.c b/fs/namei.c
index bf0f66f0e9b9..aefb21bc0944 100644
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -4279,19 +4279,16 @@ static int may_o_create(struct mnt_idmap *idmap,
  *
  * Returns an error code otherwise.
  */
-static struct dentry *atomic_open(struct nameidata *nd, struct dentry *dentry,
+static struct dentry *atomic_open(const struct path *path, struct dentry *dentry,
 				  struct file *file,
 				  int open_flag, umode_t mode)
 {
 	struct dentry *const DENTRY_NOT_SET = (void *) -1UL;
-	struct inode *dir =  nd->path.dentry->d_inode;
+	struct inode *dir =  path->dentry->d_inode;
 	int error;
 
-	if (nd->flags & LOOKUP_DIRECTORY)
-		open_flag |= O_DIRECTORY;
-
 	file->__f_path.dentry = DENTRY_NOT_SET;
-	file->__f_path.mnt = nd->path.mnt;
+	file->__f_path.mnt = path->mnt;
 	error = dir->i_op->atomic_open(dir, dentry, file,
 				       open_to_namei_flags(open_flag), mode);
 	d_lookup_done(dentry);
@@ -4403,7 +4400,9 @@ static struct dentry *lookup_open(struct nameidata *nd, struct file *file,
 	if (create_error)
 		open_flag &= ~O_CREAT;
 	if (dir_inode->i_op->atomic_open) {
-		dentry = atomic_open(nd, dentry, file, open_flag, mode);
+		if (nd->flags & LOOKUP_DIRECTORY)
+			open_flag |= O_DIRECTORY;
+		dentry = atomic_open(&nd->path, dentry, file, open_flag, mode);
 		if (unlikely(create_error) && dentry == ERR_PTR(-ENOENT))
 			dentry = ERR_PTR(create_error);
 		return dentry;
@@ -4937,6 +4936,73 @@ inline struct dentry *start_creating_user_path(
 }
 EXPORT_SYMBOL(start_creating_user_path);
 
+/**
+ * dentry_create - Create and open a file
+ * @path: path to create
+ * @flags: O_ flags
+ * @mode: mode bits for new file
+ * @cred: credentials to use
+ *
+ * Caller must hold the parent directory's lock, and have prepared
+ * a negative dentry, placed in @path->dentry, for the new file.
+ *
+ * Caller sets @path->mnt to the vfsmount of the filesystem where
+ * the new file is to be created. The parent directory and the
+ * negative dentry must reside on the same filesystem instance.
+ *
+ * On success, returns a "struct file *". Otherwise a ERR_PTR
+ * is returned.
+ */
+struct file *dentry_create(struct path *path, int flags, umode_t mode,
+			   const struct cred *cred)
+{
+	struct file *file __free(fput) = NULL;
+	struct dentry *dentry = path->dentry;
+	struct dentry *dir = dentry->d_parent;
+	struct inode *dir_inode = d_inode(dir);
+	struct mnt_idmap *idmap;
+	int error, create_error;
+
+	file = alloc_empty_file(flags, cred);
+	if (IS_ERR(file))
+		return file;
+
+	idmap = mnt_idmap(path->mnt);
+
+	if (dir_inode->i_op->atomic_open) {
+		path->dentry = dir;
+		mode = vfs_prepare_mode(idmap, dir_inode, mode, S_IALLUGO, S_IFREG);
+
+		create_error = may_o_create(idmap, path, dentry, mode);
+		if (create_error)
+			flags &= ~O_CREAT;
+
+		dentry = atomic_open(path, dentry, file, flags, mode);
+		error = PTR_ERR_OR_ZERO(dentry);
+
+		if (unlikely(create_error) && error == -ENOENT)
+			error = create_error;
+
+		if (!error) {
+			if (file->f_mode & FMODE_CREATED)
+				fsnotify_create(dir->d_inode, dentry);
+			if (file->f_mode & FMODE_OPENED)
+				fsnotify_open(file);
+		}
+
+		path->dentry = dentry;
+
+	} else {
+		error = vfs_create(mnt_idmap(path->mnt), path->dentry, mode, NULL);
+		if (!error)
+			error = vfs_open(path, file);
+	}
+	if (unlikely(error))
+		return ERR_PTR(error);
+
+	return no_free_ptr(file);
+}
+EXPORT_SYMBOL(dentry_create);
 
 /**
  * vfs_mknod - create device node or file
diff --git a/fs/namespace.c b/fs/namespace.c
index c58674a20cad..ec3b16fedd9f 100644
--- a/fs/namespace.c
+++ b/fs/namespace.c
@@ -5547,31 +5547,49 @@ static int grab_requested_root(struct mnt_namespace *ns, struct path *root)
 
 /* locks: namespace_shared */
 static int do_statmount(struct kstatmount *s, u64 mnt_id, u64 mnt_ns_id,
-			struct mnt_namespace *ns)
+                        struct file *mnt_file, struct mnt_namespace *ns)
 {
-	struct mount *m;
 	int err;
 
-	/* Has the namespace already been emptied? */
-	if (mnt_ns_id && mnt_ns_empty(ns))
-		return -ENOENT;
+	if (mnt_file) {
+		WARN_ON_ONCE(ns != NULL);
 
-	s->mnt = lookup_mnt_in_ns(mnt_id, ns);
-	if (!s->mnt)
-		return -ENOENT;
+		s->mnt = mnt_file->f_path.mnt;
+		ns = real_mount(s->mnt)->mnt_ns;
+		if (!ns)
+			/*
+			 * We can't set mount point and mnt_ns_id since we don't have a
+			 * ns for the mount. This can happen if the mount is unmounted
+			 * with MNT_DETACH.
+			 */
+			s->mask &= ~(STATMOUNT_MNT_POINT | STATMOUNT_MNT_NS_ID);
+	} else {
+		/* Has the namespace already been emptied? */
+		if (mnt_ns_id && mnt_ns_empty(ns))
+			return -ENOENT;
 
-	err = grab_requested_root(ns, &s->root);
-	if (err)
-		return err;
+		s->mnt = lookup_mnt_in_ns(mnt_id, ns);
+		if (!s->mnt)
+			return -ENOENT;
+	}
 
-	/*
-	 * Don't trigger audit denials. We just want to determine what
-	 * mounts to show users.
-	 */
-	m = real_mount(s->mnt);
-	if (!is_path_reachable(m, m->mnt.mnt_root, &s->root) &&
-	    !ns_capable_noaudit(ns->user_ns, CAP_SYS_ADMIN))
-		return -EPERM;
+	if (ns) {
+		err = grab_requested_root(ns, &s->root);
+		if (err)
+			return err;
+
+		if (!mnt_file) {
+			struct mount *m;
+			/*
+			 * Don't trigger audit denials. We just want to determine what
+			 * mounts to show users.
+			 */
+			m = real_mount(s->mnt);
+			if (!is_path_reachable(m, m->mnt.mnt_root, &s->root) &&
+			    !ns_capable_noaudit(ns->user_ns, CAP_SYS_ADMIN))
+				return -EPERM;
+		}
+	}
 
 	err = security_sb_statfs(s->mnt->mnt_root);
 	if (err)
@@ -5693,7 +5711,7 @@ static int prepare_kstatmount(struct kstatmount *ks, struct mnt_id_req *kreq,
 }
 
 static int copy_mnt_id_req(const struct mnt_id_req __user *req,
-			   struct mnt_id_req *kreq)
+			   struct mnt_id_req *kreq, unsigned int flags)
 {
 	int ret;
 	size_t usize;
@@ -5711,11 +5729,17 @@ static int copy_mnt_id_req(const struct mnt_id_req __user *req,
 	ret = copy_struct_from_user(kreq, sizeof(*kreq), req, usize);
 	if (ret)
 		return ret;
-	if (kreq->mnt_ns_fd != 0 && kreq->mnt_ns_id)
-		return -EINVAL;
-	/* The first valid unique mount id is MNT_UNIQUE_ID_OFFSET + 1. */
-	if (kreq->mnt_id <= MNT_UNIQUE_ID_OFFSET)
-		return -EINVAL;
+
+	if (flags & STATMOUNT_BY_FD) {
+		if (kreq->mnt_id || kreq->mnt_ns_id)
+			return -EINVAL;
+	} else {
+		if (kreq->mnt_ns_fd != 0 && kreq->mnt_ns_id)
+			return -EINVAL;
+		/* The first valid unique mount id is MNT_UNIQUE_ID_OFFSET + 1. */
+		if (kreq->mnt_id <= MNT_UNIQUE_ID_OFFSET)
+			return -EINVAL;
+	}
 	return 0;
 }
 
@@ -5762,25 +5786,33 @@ SYSCALL_DEFINE4(statmount, const struct mnt_id_req __user *, req,
 {
 	struct mnt_namespace *ns __free(mnt_ns_release) = NULL;
 	struct kstatmount *ks __free(kfree) = NULL;
+	struct file *mnt_file __free(fput) = NULL;
 	struct mnt_id_req kreq;
 	/* We currently support retrieval of 3 strings. */
 	size_t seq_size = 3 * PATH_MAX;
 	int ret;
 
-	if (flags)
+	if (flags & ~STATMOUNT_BY_FD)
 		return -EINVAL;
 
-	ret = copy_mnt_id_req(req, &kreq);
+	ret = copy_mnt_id_req(req, &kreq, flags);
 	if (ret)
 		return ret;
 
-	ns = grab_requested_mnt_ns(&kreq);
-	if (IS_ERR(ns))
-		return PTR_ERR(ns);
+	if (flags & STATMOUNT_BY_FD) {
+		mnt_file = fget_raw(kreq.mnt_fd);
+		if (!mnt_file)
+			return -EBADF;
+		/* do_statmount sets ns in case of STATMOUNT_BY_FD */
+	} else {
+		ns = grab_requested_mnt_ns(&kreq);
+		if (IS_ERR(ns))
+			return PTR_ERR(ns);
 
-	if (kreq.mnt_ns_id && (ns != current->nsproxy->mnt_ns) &&
-	    !ns_capable_noaudit(ns->user_ns, CAP_SYS_ADMIN))
-		return -ENOENT;
+		if (kreq.mnt_ns_id && (ns != current->nsproxy->mnt_ns) &&
+		    !ns_capable_noaudit(ns->user_ns, CAP_SYS_ADMIN))
+			return -EPERM;
+	}
 
 	ks = kmalloc(sizeof(*ks), GFP_KERNEL_ACCOUNT);
 	if (!ks)
@@ -5792,7 +5824,7 @@ SYSCALL_DEFINE4(statmount, const struct mnt_id_req __user *, req,
 		return ret;
 
 	scoped_guard(namespace_shared)
-		ret = do_statmount(ks, kreq.mnt_id, kreq.mnt_ns_id, ns);
+		ret = do_statmount(ks, kreq.mnt_id, kreq.mnt_ns_id, mnt_file, ns);
 
 	if (!ret)
 		ret = copy_statmount_to_user(ks);
@@ -5932,7 +5964,7 @@ SYSCALL_DEFINE4(listmount, const struct mnt_id_req __user *, req,
 	if (!access_ok(mnt_ids, nr_mnt_ids * sizeof(*mnt_ids)))
 		return -EFAULT;
 
-	ret = copy_mnt_id_req(req, &kreq);
+	ret = copy_mnt_id_req(req, &kreq, 0);
 	if (ret)
 		return ret;
 
diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c
index 9d55512d0cc9..2a1499f2ad19 100644
--- a/fs/nfsd/export.c
+++ b/fs/nfsd/export.c
@@ -1024,7 +1024,7 @@ exp_rootfh(struct net *net, struct auth_domain *clp, char *name,
 {
 	struct svc_export	*exp;
 	struct path		path;
-	struct inode		*inode;
+	struct inode		*inode __maybe_unused;
 	struct svc_fh		fh;
 	int			err;
 	struct nfsd_net		*nn = net_generic(net, nfsd_net_id);
diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c
index 683bd1130afe..ad7af8cfcf1f 100644
--- a/fs/nfsd/nfs4layouts.c
+++ b/fs/nfsd/nfs4layouts.c
@@ -764,9 +764,28 @@ nfsd4_layout_lm_change(struct file_lease *onlist, int arg,
 	return lease_modify(onlist, arg, dispose);
 }
 
+/**
+ *  nfsd4_layout_lm_open_conflict - see if the given file points to an inode that has
+ *				    an existing open that would conflict with the
+ *				    desired lease.
+ * @filp:	file to check
+ * @arg:	type of lease that we're trying to acquire
+ *
+ * The kernel will call into this operation to determine whether there
+ * are conflicting opens that may prevent the layout from being granted.
+ * For nfsd, that check is done at a higher level, so this trivially
+ * returns 0.
+ */
+static int
+nfsd4_layout_lm_open_conflict(struct file *filp, int arg)
+{
+	return 0;
+}
+
 static const struct lease_manager_operations nfsd4_layouts_lm_ops = {
-	.lm_break	= nfsd4_layout_lm_break,
-	.lm_change	= nfsd4_layout_lm_change,
+	.lm_break		= nfsd4_layout_lm_break,
+	.lm_change		= nfsd4_layout_lm_change,
+	.lm_open_conflict	= nfsd4_layout_lm_open_conflict,
 };
 
 int
diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index b74800917583..6aa22b3b2f43 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -194,7 +194,7 @@ static inline bool nfsd4_create_is_exclusive(int createmode)
 }
 
 static __be32
-nfsd4_vfs_create(struct svc_fh *fhp, struct dentry *child,
+nfsd4_vfs_create(struct svc_fh *fhp, struct dentry **child,
 		 struct nfsd4_open *open)
 {
 	struct file *filp;
@@ -202,6 +202,9 @@ nfsd4_vfs_create(struct svc_fh *fhp, struct dentry *child,
 	int oflags;
 
 	oflags = O_CREAT | O_LARGEFILE;
+	if (nfsd4_create_is_exclusive(open->op_createmode))
+		oflags |= O_EXCL;
+
 	switch (open->op_share_access & NFS4_SHARE_ACCESS_BOTH) {
 	case NFS4_SHARE_ACCESS_WRITE:
 		oflags |= O_WRONLY;
@@ -214,9 +217,11 @@ nfsd4_vfs_create(struct svc_fh *fhp, struct dentry *child,
 	}
 
 	path.mnt = fhp->fh_export->ex_path.mnt;
-	path.dentry = child;
+	path.dentry = *child;
 	filp = dentry_create(&path, oflags, open->op_iattr.ia_mode,
 			     current_cred());
+	*child = path.dentry;
+
 	if (IS_ERR(filp))
 		return nfserrno(PTR_ERR(filp));
 
@@ -350,7 +355,7 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	status = fh_fill_pre_attrs(fhp);
 	if (status != nfs_ok)
 		goto out;
-	status = nfsd4_vfs_create(fhp, child, open);
+	status = nfsd4_vfs_create(fhp, &child, open);
 	if (status != nfs_ok)
 		goto out;
 	open->op_created = true;
diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c
index 441dfbfe2d2b..1e6b2dd47ba7 100644
--- a/fs/nfsd/nfs4recover.c
+++ b/fs/nfsd/nfs4recover.c
@@ -39,6 +39,7 @@
 #include <linux/namei.h>
 #include <linux/sched.h>
 #include <linux/fs.h>
+#include <linux/hex.h>
 #include <linux/module.h>
 #include <net/net_namespace.h>
 #include <linux/sunrpc/rpc_pipe_fs.h>
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 808c24fb5c9a..19d6d6db107f 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -5552,10 +5552,29 @@ nfsd_change_deleg_cb(struct file_lease *onlist, int arg,
 		return -EAGAIN;
 }
 
+/**
+ *  nfsd4_deleg_lm_open_conflict - see if the given file points to an inode that has
+ *				   an existing open that would conflict with the
+ *				   desired lease.
+ * @filp:	file to check
+ * @arg:	type of lease that we're trying to acquire
+ *
+ * The kernel will call into this operation to determine whether there
+ * are conflicting opens that may prevent the deleg from being granted.
+ * For nfsd, that check is done at a higher level, so this trivially
+ * returns 0.
+ */
+static int
+nfsd4_deleg_lm_open_conflict(struct file *filp, int arg)
+{
+	return 0;
+}
+
 static const struct lease_manager_operations nfsd_lease_mng_ops = {
 	.lm_breaker_owns_lease = nfsd_breaker_owns_lease,
 	.lm_break = nfsd_break_deleg_cb,
 	.lm_change = nfsd_change_deleg_cb,
+	.lm_open_conflict = nfsd4_deleg_lm_open_conflict,
 };
 
 static __be32 nfsd4_check_seqid(struct nfsd4_compound_state *cstate, struct nfs4_stateowner *so, u32 seqid)
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 30ce5851fe4c..51ef97c25456 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -3375,6 +3375,11 @@ static __be32 nfsd4_encode_fattr4_suppattr_exclcreat(struct xdr_stream *xdr,
 	u32 supp[3];
 
 	memcpy(supp, nfsd_suppattrs[resp->cstate.minorversion], sizeof(supp));
+	if (!IS_POSIXACL(d_inode(args->dentry)))
+		supp[0] &= ~FATTR4_WORD0_ACL;
+	if (!args->contextsupport)
+		supp[2] &= ~FATTR4_WORD2_SECURITY_LABEL;
+
 	supp[0] &= NFSD_SUPPATTR_EXCLCREAT_WORD0;
 	supp[1] &= NFSD_SUPPATTR_EXCLCREAT_WORD1;
 	supp[2] &= NFSD_SUPPATTR_EXCLCREAT_WORD2;
diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h
index e4263326ca4a..50be785f1d2c 100644
--- a/fs/nfsd/nfsd.h
+++ b/fs/nfsd/nfsd.h
@@ -547,8 +547,14 @@ static inline bool nfsd_attrs_supported(u32 minorversion, const u32 *bmval)
 #define NFSD_SUPPATTR_EXCLCREAT_WORD1 \
 	(NFSD_WRITEABLE_ATTRS_WORD1 & \
 	 ~(FATTR4_WORD1_TIME_ACCESS_SET | FATTR4_WORD1_TIME_MODIFY_SET))
+/*
+ * The FATTR4_WORD2_TIME_DELEG attributes are not to be allowed for
+ * OPEN(create) with EXCLUSIVE4_1. It doesn't make sense to set a
+ * delegated timestamp on a new file.
+ */
 #define NFSD_SUPPATTR_EXCLCREAT_WORD2 \
-	NFSD_WRITEABLE_ATTRS_WORD2
+	(NFSD_WRITEABLE_ATTRS_WORD2 & \
+	~(FATTR4_WORD2_TIME_DELEG_ACCESS | FATTR4_WORD2_TIME_DELEG_MODIFY))
 
 extern int nfsd4_is_junction(struct dentry *dentry);
 extern int register_cld_notifier(void);
diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c
index b08ae85d53ef..f6cae4430ba4 100644
--- a/fs/nfsd/nfssvc.c
+++ b/fs/nfsd/nfssvc.c
@@ -615,12 +615,15 @@ int nfsd_create_serv(struct net *net)
 	serv = svc_create_pooled(nfsd_programs, ARRAY_SIZE(nfsd_programs),
 				 &nn->nfsd_svcstats,
 				 nfsd_max_blksize, nfsd);
-	if (serv == NULL)
+	if (serv == NULL) {
+		percpu_ref_exit(&nn->nfsd_net_ref);
 		return -ENOMEM;
+	}
 
 	error = svc_bind(serv, net);
 	if (error < 0) {
 		svc_destroy(&serv);
+		percpu_ref_exit(&nn->nfsd_net_ref);
 		return error;
 	}
 	spin_lock(&nfsd_notifier_lock);
diff --git a/fs/nfsd/vfs.h b/fs/nfsd/vfs.h
index ded2900d423f..e192dca4a679 100644
--- a/fs/nfsd/vfs.h
+++ b/fs/nfsd/vfs.h
@@ -67,7 +67,8 @@ static inline bool nfsd_attrs_valid(struct nfsd_attrs *attrs)
 	struct iattr *iap = attrs->na_iattr;
 
 	return (iap->ia_valid || (attrs->na_seclabel &&
-		attrs->na_seclabel->len));
+		attrs->na_seclabel->len) ||
+		attrs->na_pacl || attrs->na_dpacl);
 }
 
 __be32		nfserrno (int errno);
diff --git a/fs/notify/fsnotify.c b/fs/notify/fsnotify.c
index d27ff5e5f165..71bd44e5ab6d 100644
--- a/fs/notify/fsnotify.c
+++ b/fs/notify/fsnotify.c
@@ -270,8 +270,15 @@ int __fsnotify_parent(struct dentry *dentry, __u32 mask, const void *data,
 	/*
 	 * Include parent/name in notification either if some notification
 	 * groups require parent info or the parent is interested in this event.
+	 * The parent interest in ACCESS/MODIFY events does not apply to special
+	 * files, where read/write are not on the filesystem of the parent and
+	 * events can provide an undesirable side-channel for information
+	 * exfiltration.
 	 */
-	parent_interested = mask & p_mask & ALL_FSNOTIFY_EVENTS;
+	parent_interested = mask & p_mask & ALL_FSNOTIFY_EVENTS &&
+			    !(data_type == FSNOTIFY_EVENT_PATH &&
+			      d_is_special(dentry) &&
+			      (mask & (FS_ACCESS | FS_MODIFY)));
 	if (parent_needed || parent_interested) {
 		/* When notifying parent, child should be passed as data */
 		WARN_ON_ONCE(inode != fsnotify_data_inode(data, data_type));
diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h
index a4559c9f64e6..f18349689458 100644
--- a/fs/ntfs3/ntfs_fs.h
+++ b/fs/ntfs3/ntfs_fs.h
@@ -14,6 +14,7 @@
 #include <linux/fs.h>
 #include <linux/highmem.h>
 #include <linux/kernel.h>
+#include <linux/hex.h>
 #include <linux/mm.h>
 #include <linux/mutex.h>
 #include <linux/page-flags.h>
diff --git a/fs/ocfs2/cluster/heartbeat.c b/fs/ocfs2/cluster/heartbeat.c
index 724350925aff..8e9cbc334cf4 100644
--- a/fs/ocfs2/cluster/heartbeat.c
+++ b/fs/ocfs2/cluster/heartbeat.c
@@ -1942,7 +1942,7 @@ static struct configfs_attribute *o2hb_region_attrs[] = {
 	NULL,
 };
 
-static struct configfs_item_operations o2hb_region_item_ops = {
+static const struct configfs_item_operations o2hb_region_item_ops = {
 	.release		= o2hb_region_release,
 };
 
@@ -2193,7 +2193,7 @@ static struct configfs_attribute *o2hb_heartbeat_group_attrs[] = {
 	NULL,
 };
 
-static struct configfs_group_operations o2hb_heartbeat_group_group_ops = {
+static const struct configfs_group_operations o2hb_heartbeat_group_group_ops = {
 	.make_item	= o2hb_heartbeat_group_make_item,
 	.drop_item	= o2hb_heartbeat_group_drop_item,
 };
diff --git a/fs/ocfs2/cluster/nodemanager.c b/fs/ocfs2/cluster/nodemanager.c
index 6bc4e064ace4..c5e83c774d73 100644
--- a/fs/ocfs2/cluster/nodemanager.c
+++ b/fs/ocfs2/cluster/nodemanager.c
@@ -396,7 +396,7 @@ static struct configfs_attribute *o2nm_node_attrs[] = {
 	NULL,
 };
 
-static struct configfs_item_operations o2nm_node_item_ops = {
+static const struct configfs_item_operations o2nm_node_item_ops = {
 	.release		= o2nm_node_release,
 };
 
@@ -638,7 +638,7 @@ static void o2nm_node_group_drop_item(struct config_group *group,
 	config_item_put(item);
 }
 
-static struct configfs_group_operations o2nm_node_group_group_ops = {
+static const struct configfs_group_operations o2nm_node_group_group_ops = {
 	.make_item	= o2nm_node_group_make_item,
 	.drop_item	= o2nm_node_group_drop_item,
 };
@@ -657,7 +657,7 @@ static void o2nm_cluster_release(struct config_item *item)
 	kfree(cluster);
 }
 
-static struct configfs_item_operations o2nm_cluster_item_ops = {
+static const struct configfs_item_operations o2nm_cluster_item_ops = {
 	.release	= o2nm_cluster_release,
 };
 
@@ -741,7 +741,7 @@ static void o2nm_cluster_group_drop_item(struct config_group *group, struct conf
 	config_item_put(item);
 }
 
-static struct configfs_group_operations o2nm_cluster_group_group_ops = {
+static const struct configfs_group_operations o2nm_cluster_group_group_ops = {
 	.make_group	= o2nm_cluster_group_make_group,
 	.drop_item	= o2nm_cluster_group_drop_item,
 };
diff --git a/fs/ocfs2/export.c b/fs/ocfs2/export.c
index b95724b767e1..9c2665dd24e2 100644
--- a/fs/ocfs2/export.c
+++ b/fs/ocfs2/export.c
@@ -74,8 +74,9 @@ static struct dentry *ocfs2_get_dentry(struct super_block *sb,
 			 * nice
 			 */
 			status = -ESTALE;
-		} else
+		} else if (status != -ESTALE) {
 			mlog(ML_ERROR, "test inode bit failed %d\n", status);
+		}
 		goto unlock_nfs_sync;
 	}
 
@@ -162,8 +163,9 @@ static struct dentry *ocfs2_get_parent(struct dentry *child)
 	if (status < 0) {
 		if (status == -EINVAL) {
 			status = -ESTALE;
-		} else
+		} else if (status != -ESTALE) {
 			mlog(ML_ERROR, "test inode bit failed %d\n", status);
+		}
 		parent = ERR_PTR(status);
 		goto bail_unlock;
 	}
diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c
index b5fcc2725a29..03a51662ea8e 100644
--- a/fs/ocfs2/inode.c
+++ b/fs/ocfs2/inode.c
@@ -1494,12 +1494,25 @@ int ocfs2_validate_inode_block(struct super_block *sb,
 		goto bail;
 	}
 
-	if ((le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) &&
-	    le32_to_cpu(di->i_clusters)) {
-		rc = ocfs2_error(sb, "Invalid dinode %llu: %u clusters\n",
-				 (unsigned long long)bh->b_blocknr,
-				 le32_to_cpu(di->i_clusters));
-		goto bail;
+	if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) {
+		struct ocfs2_inline_data *data = &di->id2.i_data;
+
+		if (le32_to_cpu(di->i_clusters)) {
+			rc = ocfs2_error(sb,
+					 "Invalid dinode %llu: %u clusters\n",
+					 (unsigned long long)bh->b_blocknr,
+					 le32_to_cpu(di->i_clusters));
+			goto bail;
+		}
+
+		if (le64_to_cpu(di->i_size) > le16_to_cpu(data->id_count)) {
+			rc = ocfs2_error(sb,
+					 "Invalid dinode #%llu: inline data i_size %llu exceeds id_count %u\n",
+					 (unsigned long long)bh->b_blocknr,
+					 (unsigned long long)le64_to_cpu(di->i_size),
+					 le16_to_cpu(data->id_count));
+			goto bail;
+		}
 	}
 
 	if (le32_to_cpu(di->i_flags) & OCFS2_CHAIN_FL) {
@@ -1529,6 +1542,13 @@ int ocfs2_validate_inode_block(struct super_block *sb,
 		}
 	}
 
+	if ((le16_to_cpu(di->i_dyn_features) & OCFS2_HAS_REFCOUNT_FL) &&
+	    !di->i_refcount_loc) {
+		rc = ocfs2_error(sb, "Inode #%llu has refcount flag but no i_refcount_loc\n",
+				(unsigned long long)bh->b_blocknr);
+		goto bail;
+	}
+
 	rc = 0;
 
 bail:
diff --git a/fs/ocfs2/slot_map.c b/fs/ocfs2/slot_map.c
index e544c704b583..ea4a68abc25b 100644
--- a/fs/ocfs2/slot_map.c
+++ b/fs/ocfs2/slot_map.c
@@ -44,6 +44,9 @@ struct ocfs2_slot_info {
 static int __ocfs2_node_num_to_slot(struct ocfs2_slot_info *si,
 				    unsigned int node_num);
 
+static int ocfs2_validate_slot_map_block(struct super_block *sb,
+					  struct buffer_head *bh);
+
 static void ocfs2_invalidate_slot(struct ocfs2_slot_info *si,
 				  int slot_num)
 {
@@ -132,7 +135,8 @@ int ocfs2_refresh_slot_info(struct ocfs2_super *osb)
 	 * this is not true, the read of -1 (UINT64_MAX) will fail.
 	 */
 	ret = ocfs2_read_blocks(INODE_CACHE(si->si_inode), -1, si->si_blocks,
-				si->si_bh, OCFS2_BH_IGNORE_CACHE, NULL);
+				si->si_bh, OCFS2_BH_IGNORE_CACHE,
+				ocfs2_validate_slot_map_block);
 	if (ret == 0) {
 		spin_lock(&osb->osb_lock);
 		ocfs2_update_slot_info(si);
@@ -332,6 +336,24 @@ int ocfs2_clear_slot(struct ocfs2_super *osb, int slot_num)
 	return ocfs2_update_disk_slot(osb, osb->slot_info, slot_num);
 }
 
+static int ocfs2_validate_slot_map_block(struct super_block *sb,
+					  struct buffer_head *bh)
+{
+	int rc;
+
+	BUG_ON(!buffer_uptodate(bh));
+
+	if (bh->b_blocknr < OCFS2_SUPER_BLOCK_BLKNO) {
+		rc = ocfs2_error(sb,
+				 "Invalid Slot Map Buffer Head "
+				 "Block Number : %llu, Should be >= %d",
+				 (unsigned long long)bh->b_blocknr,
+				 OCFS2_SUPER_BLOCK_BLKNO);
+		return rc;
+	}
+	return 0;
+}
+
 static int ocfs2_map_slot_buffers(struct ocfs2_super *osb,
 				  struct ocfs2_slot_info *si)
 {
@@ -383,7 +405,8 @@ static int ocfs2_map_slot_buffers(struct ocfs2_super *osb,
 
 		bh = NULL;  /* Acquire a fresh bh */
 		status = ocfs2_read_blocks(INODE_CACHE(si->si_inode), blkno,
-					   1, &bh, OCFS2_BH_IGNORE_CACHE, NULL);
+					   1, &bh, OCFS2_BH_IGNORE_CACHE,
+					   ocfs2_validate_slot_map_block);
 		if (status < 0) {
 			mlog_errno(status);
 			goto bail;
diff --git a/fs/ocfs2/suballoc.c b/fs/ocfs2/suballoc.c
index 8e6e5235b30c..72c7559c1a2b 100644
--- a/fs/ocfs2/suballoc.c
+++ b/fs/ocfs2/suballoc.c
@@ -295,6 +295,74 @@ static int ocfs2_validate_group_descriptor(struct super_block *sb,
 	return ocfs2_validate_gd_self(sb, bh, 0);
 }
 
+/*
+ * The hint group descriptor (gd) may already have been released
+ * in _ocfs2_free_suballoc_bits(). We first check the gd signature,
+ * then perform the standard ocfs2_read_group_descriptor() jobs.
+ *
+ * If the gd signature is invalid, we return 'rc=0' and set
+ * '*released=1'. The caller is expected to handle this specific case.
+ * Otherwise, we return the actual error code.
+ *
+ * We treat gd signature corruption case as a release case. The
+ * caller ocfs2_claim_suballoc_bits() will use ocfs2_search_chain()
+ * to search each gd block. The code will eventually find this
+ * corrupted gd block - Late, but not missed.
+ *
+ * Note:
+ * The caller is responsible for initializing the '*released' status.
+ */
+static int ocfs2_read_hint_group_descriptor(struct inode *inode,
+			struct ocfs2_dinode *di, u64 gd_blkno,
+			struct buffer_head **bh, int *released)
+{
+	int rc;
+	struct buffer_head *tmp = *bh;
+	struct ocfs2_group_desc *gd;
+
+	rc = ocfs2_read_block(INODE_CACHE(inode), gd_blkno, &tmp, NULL);
+	if (rc)
+		goto out;
+
+	gd = (struct ocfs2_group_desc *) tmp->b_data;
+	if (!OCFS2_IS_VALID_GROUP_DESC(gd)) {
+		/*
+		 * Invalid gd cache was set in ocfs2_read_block(),
+		 * which will affect block_group allocation.
+		 * Path:
+		 * ocfs2_reserve_suballoc_bits
+		 *  ocfs2_block_group_alloc
+		 *   ocfs2_block_group_alloc_contig
+		 *    ocfs2_set_new_buffer_uptodate
+		 */
+		ocfs2_remove_from_cache(INODE_CACHE(inode), tmp);
+		*released = 1; /* we return 'rc=0' for this case */
+		goto free_bh;
+	}
+
+	/* below jobs same with ocfs2_read_group_descriptor() */
+	if (!buffer_jbd(tmp)) {
+		rc = ocfs2_validate_group_descriptor(inode->i_sb, tmp);
+		if (rc)
+			goto free_bh;
+	}
+
+	rc = ocfs2_validate_gd_parent(inode->i_sb, di, tmp, 0);
+	if (rc)
+		goto free_bh;
+
+	/* If ocfs2_read_block() got us a new bh, pass it up. */
+	if (!*bh)
+		*bh = tmp;
+
+	return rc;
+
+free_bh:
+	brelse(tmp);
+out:
+	return rc;
+}
+
 int ocfs2_read_group_descriptor(struct inode *inode, struct ocfs2_dinode *di,
 				u64 gd_blkno, struct buffer_head **bh)
 {
@@ -1725,7 +1793,7 @@ static int ocfs2_search_one_group(struct ocfs2_alloc_context *ac,
 				  u32 bits_wanted,
 				  u32 min_bits,
 				  struct ocfs2_suballoc_result *res,
-				  u16 *bits_left)
+				  u16 *bits_left, int *released)
 {
 	int ret;
 	struct buffer_head *group_bh = NULL;
@@ -1733,9 +1801,11 @@ static int ocfs2_search_one_group(struct ocfs2_alloc_context *ac,
 	struct ocfs2_dinode *di = (struct ocfs2_dinode *)ac->ac_bh->b_data;
 	struct inode *alloc_inode = ac->ac_inode;
 
-	ret = ocfs2_read_group_descriptor(alloc_inode, di,
-					  res->sr_bg_blkno, &group_bh);
-	if (ret < 0) {
+	ret = ocfs2_read_hint_group_descriptor(alloc_inode, di,
+				res->sr_bg_blkno, &group_bh, released);
+	if (*released) {
+		return 0;
+	} else if (ret < 0) {
 		mlog_errno(ret);
 		return ret;
 	}
@@ -1950,6 +2020,7 @@ static int ocfs2_claim_suballoc_bits(struct ocfs2_alloc_context *ac,
 				     struct ocfs2_suballoc_result *res)
 {
 	int status;
+	int released = 0;
 	u16 victim, i;
 	u16 bits_left = 0;
 	u64 hint = ac->ac_last_group;
@@ -1976,6 +2047,7 @@ static int ocfs2_claim_suballoc_bits(struct ocfs2_alloc_context *ac,
 		goto bail;
 	}
 
+	/* the hint bg may already be released, we quiet search this group. */
 	res->sr_bg_blkno = hint;
 	if (res->sr_bg_blkno) {
 		/* Attempt to short-circuit the usual search mechanism
@@ -1983,7 +2055,12 @@ static int ocfs2_claim_suballoc_bits(struct ocfs2_alloc_context *ac,
 		 * allocation group. This helps us maintain some
 		 * contiguousness across allocations. */
 		status = ocfs2_search_one_group(ac, handle, bits_wanted,
-						min_bits, res, &bits_left);
+						min_bits, res, &bits_left,
+						&released);
+		if (released) {
+			res->sr_bg_blkno = 0;
+			goto chain_search;
+		}
 		if (!status)
 			goto set_hint;
 		if (status < 0 && status != -ENOSPC) {
@@ -1991,7 +2068,7 @@ static int ocfs2_claim_suballoc_bits(struct ocfs2_alloc_context *ac,
 			goto bail;
 		}
 	}
-
+chain_search:
 	cl = (struct ocfs2_chain_list *) &fe->id2.i_chain;
 	if (!le16_to_cpu(cl->cl_next_free_rec) ||
 	    le16_to_cpu(cl->cl_next_free_rec) > le16_to_cpu(cl->cl_count)) {
@@ -2113,6 +2190,12 @@ int ocfs2_claim_metadata(handle_t *handle,
 	return status;
 }
 
+/*
+ * after ocfs2 has the ability to release block group unused space,
+ * the ->ip_last_used_group may be invalid. so this function returns
+ * ac->ac_last_group need to verify.
+ * refer the 'hint' in ocfs2_claim_suballoc_bits() for more details.
+ */
 static void ocfs2_init_inode_ac_group(struct inode *dir,
 				      struct buffer_head *parent_di_bh,
 				      struct ocfs2_alloc_context *ac)
@@ -2551,6 +2634,198 @@ static int ocfs2_block_group_clear_bits(handle_t *handle,
 	return status;
 }
 
+/*
+ * Reclaim the suballocator managed space to main bitmap.
+ * This function first works on the suballocator to perform the
+ * cleanup rec/alloc_inode job, then switches to the main bitmap
+ * to reclaim released space.
+ *
+ * handle: The transaction handle
+ * alloc_inode: The suballoc inode
+ * alloc_bh: The buffer_head of suballoc inode
+ * group_bh: The group descriptor buffer_head of suballocator managed.
+ *           Caller should release the input group_bh.
+ */
+static int _ocfs2_reclaim_suballoc_to_main(handle_t *handle,
+			struct inode *alloc_inode,
+			struct buffer_head *alloc_bh,
+			struct buffer_head *group_bh)
+{
+	int idx, status = 0;
+	int i, next_free_rec, len = 0;
+	__le16 old_bg_contig_free_bits = 0;
+	u16 start_bit;
+	u32 tmp_used;
+	u64 bg_blkno, start_blk;
+	unsigned int count;
+	struct ocfs2_chain_rec *rec;
+	struct buffer_head *main_bm_bh = NULL;
+	struct inode *main_bm_inode = NULL;
+	struct ocfs2_super *osb = OCFS2_SB(alloc_inode->i_sb);
+	struct ocfs2_dinode *fe = (struct ocfs2_dinode *) alloc_bh->b_data;
+	struct ocfs2_chain_list *cl = &fe->id2.i_chain;
+	struct ocfs2_group_desc *group = (struct ocfs2_group_desc *) group_bh->b_data;
+
+	idx = le16_to_cpu(group->bg_chain);
+	rec = &(cl->cl_recs[idx]);
+
+	status = ocfs2_extend_trans(handle,
+				ocfs2_calc_group_alloc_credits(osb->sb,
+						 le16_to_cpu(cl->cl_cpg)));
+	if (status) {
+		mlog_errno(status);
+		goto bail;
+	}
+	status = ocfs2_journal_access_di(handle, INODE_CACHE(alloc_inode),
+					 alloc_bh, OCFS2_JOURNAL_ACCESS_WRITE);
+	if (status < 0) {
+		mlog_errno(status);
+		goto bail;
+	}
+
+	/*
+	 * Only clear the suballocator rec item in-place.
+	 *
+	 * If idx is not the last, we don't compress (remove the empty item)
+	 * the cl_recs[]. If not, we need to do lots jobs.
+	 *
+	 * Compress cl_recs[] code example:
+	 * if (idx != cl->cl_next_free_rec - 1)
+	 *     memmove(&cl->cl_recs[idx], &cl->cl_recs[idx + 1],
+	 *         sizeof(struct ocfs2_chain_rec) *
+	 *         (cl->cl_next_free_rec - idx - 1));
+	 * for(i = idx; i < cl->cl_next_free_rec-1; i++) {
+	 *     group->bg_chain = "later group->bg_chain";
+	 *     group->bg_blkno = xxx;
+	 *     ... ...
+	 * }
+	 */
+
+	tmp_used = le32_to_cpu(fe->id1.bitmap1.i_total);
+	fe->id1.bitmap1.i_total = cpu_to_le32(tmp_used - le32_to_cpu(rec->c_total));
+
+	/* Substraction 1 for the block group itself */
+	tmp_used = le32_to_cpu(fe->id1.bitmap1.i_used);
+	fe->id1.bitmap1.i_used = cpu_to_le32(tmp_used - 1);
+
+	tmp_used = le32_to_cpu(fe->i_clusters);
+	fe->i_clusters = cpu_to_le32(tmp_used - le16_to_cpu(cl->cl_cpg));
+
+	spin_lock(&OCFS2_I(alloc_inode)->ip_lock);
+	OCFS2_I(alloc_inode)->ip_clusters -= le32_to_cpu(fe->i_clusters);
+	fe->i_size = cpu_to_le64(ocfs2_clusters_to_bytes(alloc_inode->i_sb,
+					     le32_to_cpu(fe->i_clusters)));
+	spin_unlock(&OCFS2_I(alloc_inode)->ip_lock);
+	i_size_write(alloc_inode, le64_to_cpu(fe->i_size));
+	alloc_inode->i_blocks = ocfs2_inode_sector_count(alloc_inode);
+
+	ocfs2_journal_dirty(handle, alloc_bh);
+	ocfs2_update_inode_fsync_trans(handle, alloc_inode, 0);
+
+	start_blk = le64_to_cpu(rec->c_blkno);
+	count = le32_to_cpu(rec->c_total) / le16_to_cpu(cl->cl_bpc);
+
+	/*
+	 * If the rec is the last one, let's compress the chain list by
+	 * removing the empty cl_recs[] at the end.
+	 */
+	next_free_rec = le16_to_cpu(cl->cl_next_free_rec);
+	if (idx == (next_free_rec - 1)) {
+		len++; /* the last item should be counted first */
+		for (i = (next_free_rec - 2); i > 0; i--) {
+			if (cl->cl_recs[i].c_free == cl->cl_recs[i].c_total)
+				len++;
+			else
+				break;
+		}
+	}
+	le16_add_cpu(&cl->cl_next_free_rec, -len);
+
+	rec->c_free = 0;
+	rec->c_total = 0;
+	rec->c_blkno = 0;
+	ocfs2_remove_from_cache(INODE_CACHE(alloc_inode), group_bh);
+	memset(group, 0, sizeof(struct ocfs2_group_desc));
+
+	/* prepare job for reclaim clusters */
+	main_bm_inode = ocfs2_get_system_file_inode(osb,
+						    GLOBAL_BITMAP_SYSTEM_INODE,
+						    OCFS2_INVALID_SLOT);
+	if (!main_bm_inode)
+		goto bail; /* ignore the error in reclaim path */
+
+	inode_lock(main_bm_inode);
+
+	status = ocfs2_inode_lock(main_bm_inode, &main_bm_bh, 1);
+	if (status < 0)
+		goto free_bm_inode; /* ignore the error in reclaim path */
+
+	ocfs2_block_to_cluster_group(main_bm_inode, start_blk, &bg_blkno,
+				     &start_bit);
+	fe = (struct ocfs2_dinode *) main_bm_bh->b_data;
+	cl = &fe->id2.i_chain;
+	/* reuse group_bh, caller will release the input group_bh */
+	group_bh = NULL;
+
+	/* reclaim clusters to global_bitmap */
+	status = ocfs2_read_group_descriptor(main_bm_inode, fe, bg_blkno,
+					     &group_bh);
+	if (status < 0) {
+		mlog_errno(status);
+		goto free_bm_bh;
+	}
+	group = (struct ocfs2_group_desc *) group_bh->b_data;
+
+	if ((count + start_bit) > le16_to_cpu(group->bg_bits)) {
+		ocfs2_error(alloc_inode->i_sb,
+			"reclaim length (%d) beyands block group length (%d)",
+			count + start_bit, le16_to_cpu(group->bg_bits));
+		goto free_group_bh;
+	}
+
+	old_bg_contig_free_bits = group->bg_contig_free_bits;
+	status = ocfs2_block_group_clear_bits(handle, main_bm_inode,
+					      group, group_bh,
+					      start_bit, count, 0,
+					      _ocfs2_clear_bit);
+	if (status < 0) {
+		mlog_errno(status);
+		goto free_group_bh;
+	}
+
+	status = ocfs2_journal_access_di(handle, INODE_CACHE(main_bm_inode),
+					 main_bm_bh, OCFS2_JOURNAL_ACCESS_WRITE);
+	if (status < 0) {
+		mlog_errno(status);
+		ocfs2_block_group_set_bits(handle, main_bm_inode, group, group_bh,
+				start_bit, count,
+				le16_to_cpu(old_bg_contig_free_bits), 1);
+		goto free_group_bh;
+	}
+
+	idx = le16_to_cpu(group->bg_chain);
+	rec = &(cl->cl_recs[idx]);
+
+	le32_add_cpu(&rec->c_free, count);
+	tmp_used = le32_to_cpu(fe->id1.bitmap1.i_used);
+	fe->id1.bitmap1.i_used = cpu_to_le32(tmp_used - count);
+	ocfs2_journal_dirty(handle, main_bm_bh);
+
+free_group_bh:
+	brelse(group_bh);
+
+free_bm_bh:
+	ocfs2_inode_unlock(main_bm_inode, 1);
+	brelse(main_bm_bh);
+
+free_bm_inode:
+	inode_unlock(main_bm_inode);
+	iput(main_bm_inode);
+
+bail:
+	return status;
+}
+
 /*
  * expects the suballoc inode to already be locked.
  */
@@ -2563,12 +2838,13 @@ static int _ocfs2_free_suballoc_bits(handle_t *handle,
 				     void (*undo_fn)(unsigned int bit,
 						     unsigned long *bitmap))
 {
-	int status = 0;
+	int idx, status = 0;
 	u32 tmp_used;
 	struct ocfs2_dinode *fe = (struct ocfs2_dinode *) alloc_bh->b_data;
 	struct ocfs2_chain_list *cl = &fe->id2.i_chain;
 	struct buffer_head *group_bh = NULL;
 	struct ocfs2_group_desc *group;
+	struct ocfs2_chain_rec *rec;
 	__le16 old_bg_contig_free_bits = 0;
 
 	/* The alloc_bh comes from ocfs2_free_dinode() or
@@ -2614,12 +2890,26 @@ static int _ocfs2_free_suballoc_bits(handle_t *handle,
 		goto bail;
 	}
 
-	le32_add_cpu(&cl->cl_recs[le16_to_cpu(group->bg_chain)].c_free,
-		     count);
+	idx = le16_to_cpu(group->bg_chain);
+	rec = &(cl->cl_recs[idx]);
+
+	le32_add_cpu(&rec->c_free, count);
 	tmp_used = le32_to_cpu(fe->id1.bitmap1.i_used);
 	fe->id1.bitmap1.i_used = cpu_to_le32(tmp_used - count);
 	ocfs2_journal_dirty(handle, alloc_bh);
 
+	/*
+	 * Reclaim suballocator free space.
+	 * Bypass: global_bitmap, non empty rec, first rec in cl_recs[]
+	 */
+	if (ocfs2_is_cluster_bitmap(alloc_inode) ||
+	    (le32_to_cpu(rec->c_free) != (le32_to_cpu(rec->c_total) - 1)) ||
+	    (le16_to_cpu(cl->cl_next_free_rec) == 1)) {
+		goto bail;
+	}
+
+	_ocfs2_reclaim_suballoc_to_main(handle, alloc_inode, alloc_bh, group_bh);
+
 bail:
 	brelse(group_bh);
 	return status;
@@ -2873,7 +3163,7 @@ static int ocfs2_test_suballoc_bit(struct ocfs2_super *osb,
 	struct ocfs2_group_desc *group;
 	struct buffer_head *group_bh = NULL;
 	u64 bg_blkno;
-	int status;
+	int status, quiet = 0, released;
 
 	trace_ocfs2_test_suballoc_bit((unsigned long long)blkno,
 				      (unsigned int)bit);
@@ -2889,9 +3179,13 @@ static int ocfs2_test_suballoc_bit(struct ocfs2_super *osb,
 
 	bg_blkno = group_blkno ? group_blkno :
 		   ocfs2_which_suballoc_group(blkno, bit);
-	status = ocfs2_read_group_descriptor(suballoc, alloc_di, bg_blkno,
-					     &group_bh);
-	if (status < 0) {
+	status = ocfs2_read_hint_group_descriptor(suballoc, alloc_di, bg_blkno,
+					     &group_bh, &released);
+	if (released) {
+		quiet = 1;
+		status = -ESTALE;
+		goto bail;
+	} else if (status < 0) {
 		mlog(ML_ERROR, "read group %llu failed %d\n",
 		     (unsigned long long)bg_blkno, status);
 		goto bail;
@@ -2903,7 +3197,7 @@ static int ocfs2_test_suballoc_bit(struct ocfs2_super *osb,
 bail:
 	brelse(group_bh);
 
-	if (status)
+	if (status && !quiet)
 		mlog_errno(status);
 	return status;
 }
@@ -2923,7 +3217,7 @@ static int ocfs2_test_suballoc_bit(struct ocfs2_super *osb,
  */
 int ocfs2_test_inode_bit(struct ocfs2_super *osb, u64 blkno, int *res)
 {
-	int status;
+	int status, quiet = 0;
 	u64 group_blkno = 0;
 	u16 suballoc_bit = 0, suballoc_slot = 0;
 	struct inode *inode_alloc_inode;
@@ -2965,8 +3259,12 @@ int ocfs2_test_inode_bit(struct ocfs2_super *osb, u64 blkno, int *res)
 
 	status = ocfs2_test_suballoc_bit(osb, inode_alloc_inode, alloc_bh,
 					 group_blkno, blkno, suballoc_bit, res);
-	if (status < 0)
-		mlog(ML_ERROR, "test suballoc bit failed %d\n", status);
+	if (status < 0) {
+		if (status == -ESTALE)
+			quiet = 1;
+		else
+			mlog(ML_ERROR, "test suballoc bit failed %d\n", status);
+	}
 
 	ocfs2_inode_unlock(inode_alloc_inode, 0);
 	inode_unlock(inode_alloc_inode);
@@ -2974,7 +3272,7 @@ int ocfs2_test_inode_bit(struct ocfs2_super *osb, u64 blkno, int *res)
 	iput(inode_alloc_inode);
 	brelse(alloc_bh);
 bail:
-	if (status)
+	if (status && !quiet)
 		mlog_errno(status);
 	return status;
 }
diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c
index 1b21fbc16d73..5fd85f517868 100644
--- a/fs/ocfs2/xattr.c
+++ b/fs/ocfs2/xattr.c
@@ -1971,8 +1971,7 @@ static void ocfs2_xa_remove_entry(struct ocfs2_xa_loc *loc)
 	ocfs2_xa_wipe_namevalue(loc);
 	loc->xl_entry = NULL;
 
-	le16_add_cpu(&xh->xh_count, -1);
-	count = le16_to_cpu(xh->xh_count);
+	count = le16_to_cpu(xh->xh_count) - 1;
 
 	/*
 	 * Only zero out the entry if there are more remaining.  This is
@@ -1987,6 +1986,8 @@ static void ocfs2_xa_remove_entry(struct ocfs2_xa_loc *loc)
 		memset(&xh->xh_entries[count], 0,
 		       sizeof(struct ocfs2_xattr_entry));
 	}
+
+	xh->xh_count = cpu_to_le16(count);
 }
 
 /*
diff --git a/fs/open.c b/fs/open.c
index f328622061c5..74c4c1462b3e 100644
--- a/fs/open.c
+++ b/fs/open.c
@@ -1141,45 +1141,6 @@ struct file *dentry_open_nonotify(const struct path *path, int flags,
 	return f;
 }
 
-/**
- * dentry_create - Create and open a file
- * @path: path to create
- * @flags: O_ flags
- * @mode: mode bits for new file
- * @cred: credentials to use
- *
- * Caller must hold the parent directory's lock, and have prepared
- * a negative dentry, placed in @path->dentry, for the new file.
- *
- * Caller sets @path->mnt to the vfsmount of the filesystem where
- * the new file is to be created. The parent directory and the
- * negative dentry must reside on the same filesystem instance.
- *
- * On success, returns a "struct file *". Otherwise a ERR_PTR
- * is returned.
- */
-struct file *dentry_create(const struct path *path, int flags, umode_t mode,
-			   const struct cred *cred)
-{
-	struct file *f;
-	int error;
-
-	f = alloc_empty_file(flags, cred);
-	if (IS_ERR(f))
-		return f;
-
-	error = vfs_create(mnt_idmap(path->mnt), path->dentry, mode, NULL);
-	if (!error)
-		error = vfs_open(path, f);
-
-	if (unlikely(error)) {
-		fput(f);
-		return ERR_PTR(error);
-	}
-	return f;
-}
-EXPORT_SYMBOL(dentry_create);
-
 /**
  * kernel_file_open - open a file for kernel internal use
  * @path:	path of the file to open
diff --git a/fs/overlayfs/namei.c b/fs/overlayfs/namei.c
index e9a69c95be91..cda26bdef3b9 100644
--- a/fs/overlayfs/namei.c
+++ b/fs/overlayfs/namei.c
@@ -7,6 +7,7 @@
 #include <linux/fs.h>
 #include <linux/cred.h>
 #include <linux/ctype.h>
+#include <linux/hex.h>
 #include <linux/namei.h>
 #include <linux/xattr.h>
 #include <linux/ratelimit.h>
diff --git a/fs/proc/array.c b/fs/proc/array.c
index 42932f88141a..39e9246f6e4a 100644
--- a/fs/proc/array.c
+++ b/fs/proc/array.c
@@ -55,6 +55,7 @@
 
 #include <linux/types.h>
 #include <linux/errno.h>
+#include <linux/hex.h>
 #include <linux/time.h>
 #include <linux/time_namespace.h>
 #include <linux/kernel.h>
diff --git a/fs/proc/base.c b/fs/proc/base.c
index 4eec684baca9..d75d0ce97032 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -589,7 +589,7 @@ static int proc_oom_score(struct seq_file *m, struct pid_namespace *ns,
 	unsigned long points = 0;
 	long badness;
 
-	badness = oom_badness(task, totalpages);
+	badness = oom_badness(task, totalpages, false, NULL, NULL);
 	/*
 	 * Special case OOM_SCORE_ADJ_MIN for all others scale the
 	 * badness value into [0, 2000] range which we have been
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 81dfc26bfae8..480db575553e 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -2739,7 +2739,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,
 		return 0;
 	}
 
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 
 	if ((p->arg.flags & PM_SCAN_WP_MATCHING) && !p->vec_out) {
 		/* Fast path for performing exclusive WP */
@@ -2809,7 +2809,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,
 	if (flush_end)
 		flush_tlb_range(vma, start, addr);
 
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 	pte_unmap_unlock(start_pte, ptl);
 
 	cond_resched();
diff --git a/fs/seq_file.c b/fs/seq_file.c
index 8bbb1ad46335..8894cbde8d3a 100644
--- a/fs/seq_file.c
+++ b/fs/seq_file.c
@@ -11,6 +11,7 @@
 #include <linux/cache.h>
 #include <linux/fs.h>
 #include <linux/export.h>
+#include <linux/hex.h>
 #include <linux/seq_file.h>
 #include <linux/vmalloc.h>
 #include <linux/slab.h>
diff --git a/fs/smb/client/cifsfs.h b/fs/smb/client/cifsfs.h
index e9534258d1ef..75d372ceb655 100644
--- a/fs/smb/client/cifsfs.h
+++ b/fs/smb/client/cifsfs.h
@@ -145,6 +145,6 @@ extern const struct export_operations cifs_export_ops;
 #endif /* CONFIG_CIFS_NFSD_EXPORT */
 
 /* when changing internal version - update following two lines at same time */
-#define SMB3_PRODUCT_BUILD 57
-#define CIFS_VERSION   "2.57"
+#define SMB3_PRODUCT_BUILD 58
+#define CIFS_VERSION   "2.58"
 #endif				/* _CIFSFS_H */
diff --git a/fs/smb/client/cifspdu.h b/fs/smb/client/cifspdu.h
index eeb4011cb217..fdd84369e7b8 100644
--- a/fs/smb/client/cifspdu.h
+++ b/fs/smb/client/cifspdu.h
@@ -12,7 +12,7 @@
 #include <net/sock.h>
 #include <linux/unaligned.h>
 #include "../common/smbfsctl.h"
-#include "../common/smb2pdu.h"
+#include "../common/smb1pdu.h"
 
 #define CIFS_PROT   0
 #define POSIX_PROT  (CIFS_PROT+1)
diff --git a/fs/smb/common/smb1pdu.h b/fs/smb/common/smb1pdu.h
new file mode 100644
index 000000000000..df6d4e11ae92
--- /dev/null
+++ b/fs/smb/common/smb1pdu.h
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: LGPL-2.1 */
+/*
+ *
+ *   Copyright (C) International Business Machines  Corp., 2002,2009
+ *                 2018 Samsung Electronics Co., Ltd.
+ *   Author(s): Steve French <sfrench@us.ibm.com>
+ *              Namjae Jeon <linkinjeon@kernel.org>
+ *
+ */
+
+#ifndef _COMMON_SMB1_PDU_H
+#define _COMMON_SMB1_PDU_H
+
+#define SMB1_PROTO_NUMBER		cpu_to_le32(0x424d53ff)
+
+/*
+ * See MS-CIFS 2.2.3.1
+ *     MS-SMB 2.2.3.1
+ */
+struct smb_hdr {
+	__u8 Protocol[4];
+	__u8 Command;
+	union {
+		struct {
+			__u8 ErrorClass;
+			__u8 Reserved;
+			__le16 Error;
+		} __packed DosError;
+		__le32 CifsError;
+	} __packed Status;
+	__u8 Flags;
+	__le16 Flags2;		/* note: le */
+	__le16 PidHigh;
+	union {
+		struct {
+			__le32 SequenceNumber;  /* le */
+			__u32 Reserved; /* zero */
+		} __packed Sequence;
+		__u8 SecuritySignature[8];	/* le */
+	} __packed Signature;
+	__u8 pad[2];
+	__u16 Tid;
+	__le16 Pid;
+	__u16 Uid;
+	__le16 Mid;
+	__u8 WordCount;
+} __packed;
+
+/* See MS-CIFS 2.2.4.52.1 */
+typedef struct smb_negotiate_req {
+	struct smb_hdr hdr;	/* wct = 0 */
+	__le16 ByteCount;
+	unsigned char DialectsArray[];
+} __packed SMB_NEGOTIATE_REQ;
+
+#endif /* _COMMON_SMB1_PDU_H */
diff --git a/fs/smb/common/smb2pdu.h b/fs/smb/common/smb2pdu.h
index 3c8d8a4e7439..f5ebbe31384a 100644
--- a/fs/smb/common/smb2pdu.h
+++ b/fs/smb/common/smb2pdu.h
@@ -1293,6 +1293,7 @@ struct create_durable_handle_reconnect_v2 {
 	struct create_context_hdr ccontext;
 	__u8   Name[8];
 	struct durable_reconnect_context_v2 dcontext;
+	__u8 Pad[4];
 } __packed;
 
 /* See MS-SMB2 2.2.14.2.12 */
@@ -1985,39 +1986,6 @@ struct smb2_lease_ack {
 	__le64 LeaseDuration;
 } __packed;
 
-/*
- * See MS-CIFS 2.2.3.1
- *     MS-SMB 2.2.3.1
- */
-struct smb_hdr {
-	__u8 Protocol[4];
-	__u8 Command;
-	union {
-		struct {
-			__u8 ErrorClass;
-			__u8 Reserved;
-			__le16 Error;
-		} __packed DosError;
-		__le32 CifsError;
-	} __packed Status;
-	__u8 Flags;
-	__le16 Flags2;		/* note: le */
-	__le16 PidHigh;
-	union {
-		struct {
-			__le32 SequenceNumber;  /* le */
-			__u32 Reserved; /* zero */
-		} __packed Sequence;
-		__u8 SecuritySignature[8];	/* le */
-	} __packed Signature;
-	__u8 pad[2];
-	__u16 Tid;
-	__le16 Pid;
-	__u16 Uid;
-	__le16 Mid;
-	__u8 WordCount;
-} __packed;
-
 #define OP_BREAK_STRUCT_SIZE_20		24
 #define OP_BREAK_STRUCT_SIZE_21		36
 
@@ -2122,11 +2090,4 @@ struct smb_hdr {
 #define SET_MINIMUM_RIGHTS (FILE_READ_EA | FILE_READ_ATTRIBUTES \
 				| READ_CONTROL | SYNCHRONIZE)
 
-/* See MS-CIFS 2.2.4.52.1 */
-typedef struct smb_negotiate_req {
-	struct smb_hdr hdr;	/* wct = 0 */
-	__le16 ByteCount;
-	unsigned char DialectsArray[];
-} __packed SMB_NEGOTIATE_REQ;
-
 #endif				/* _COMMON_SMB2PDU_H */
diff --git a/fs/smb/common/smbdirect/smbdirect_socket.h b/fs/smb/common/smbdirect/smbdirect_socket.h
index 384b19177e1c..ee4c2726771a 100644
--- a/fs/smb/common/smbdirect/smbdirect_socket.h
+++ b/fs/smb/common/smbdirect/smbdirect_socket.h
@@ -132,6 +132,14 @@ struct smbdirect_socket {
 
 	struct smbdirect_socket_parameters parameters;
 
+	/*
+	 * The state for connect/negotiation
+	 */
+	struct {
+		spinlock_t lock;
+		struct work_struct work;
+	} connect;
+
 	/*
 	 * The state for keepalive and timeout handling
 	 */
@@ -353,6 +361,10 @@ static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc)
 	INIT_WORK(&sc->disconnect_work, __smbdirect_socket_disabled_work);
 	disable_work_sync(&sc->disconnect_work);
 
+	spin_lock_init(&sc->connect.lock);
+	INIT_WORK(&sc->connect.work, __smbdirect_socket_disabled_work);
+	disable_work_sync(&sc->connect.work);
+
 	INIT_WORK(&sc->idle.immediate_work, __smbdirect_socket_disabled_work);
 	disable_work_sync(&sc->idle.immediate_work);
 	INIT_DELAYED_WORK(&sc->idle.timer_work, __smbdirect_socket_disabled_work);
diff --git a/fs/smb/common/smbglob.h b/fs/smb/common/smbglob.h
index 9562845a5617..4e33d91cdc9d 100644
--- a/fs/smb/common/smbglob.h
+++ b/fs/smb/common/smbglob.h
@@ -11,8 +11,6 @@
 #ifndef _COMMON_SMB_GLOB_H
 #define _COMMON_SMB_GLOB_H
 
-#define SMB1_PROTO_NUMBER		cpu_to_le32(0x424d53ff)
-
 struct smb_version_values {
 	char		*version_string;
 	__u16		protocol_id;
diff --git a/fs/smb/server/mgmt/user_session.c b/fs/smb/server/mgmt/user_session.c
index 1c181ef99929..7d880ff34402 100644
--- a/fs/smb/server/mgmt/user_session.c
+++ b/fs/smb/server/mgmt/user_session.c
@@ -325,8 +325,10 @@ struct ksmbd_session *ksmbd_session_lookup_all(struct ksmbd_conn *conn,
 	sess = ksmbd_session_lookup(conn, id);
 	if (!sess && conn->binding)
 		sess = ksmbd_session_lookup_slowpath(id);
-	if (sess && sess->state != SMB2_SESSION_VALID)
+	if (sess && sess->state != SMB2_SESSION_VALID) {
+		ksmbd_user_session_put(sess);
 		sess = NULL;
+	}
 	return sess;
 }
 
diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c
index 27f87a13f20a..8aa483800014 100644
--- a/fs/smb/server/smb2pdu.c
+++ b/fs/smb/server/smb2pdu.c
@@ -2363,7 +2363,7 @@ static int smb2_set_ea(struct smb2_ea_info *eabuf, unsigned int buf_len,
 	int rc = 0;
 	unsigned int next = 0;
 
-	if (buf_len < sizeof(struct smb2_ea_info) + eabuf->EaNameLength +
+	if (buf_len < sizeof(struct smb2_ea_info) + eabuf->EaNameLength + 1 +
 			le16_to_cpu(eabuf->EaValueLength))
 		return -EINVAL;
 
@@ -2440,7 +2440,7 @@ static int smb2_set_ea(struct smb2_ea_info *eabuf, unsigned int buf_len,
 			break;
 		}
 
-		if (buf_len < sizeof(struct smb2_ea_info) + eabuf->EaNameLength +
+		if (buf_len < sizeof(struct smb2_ea_info) + eabuf->EaNameLength + 1 +
 				le16_to_cpu(eabuf->EaValueLength)) {
 			rc = -EINVAL;
 			break;
diff --git a/fs/smb/server/smb_common.h b/fs/smb/server/smb_common.h
index 067b45048c73..95bf1465387b 100644
--- a/fs/smb/server/smb_common.h
+++ b/fs/smb/server/smb_common.h
@@ -10,6 +10,7 @@
 
 #include "glob.h"
 #include "../common/smbglob.h"
+#include "../common/smb1pdu.h"
 #include "../common/smb2pdu.h"
 #include "../common/fscc.h"
 #include "smb2pdu.h"
diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c
index 5aa7a66334d9..05598d994a68 100644
--- a/fs/smb/server/smbacl.c
+++ b/fs/smb/server/smbacl.c
@@ -1307,9 +1307,6 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path,
 			granted |= le32_to_cpu(ace->access_req);
 			ace = (struct smb_ace *)((char *)ace + le16_to_cpu(ace->size));
 		}
-
-		if (!pdacl->num_aces)
-			granted = GENERIC_ALL_FLAGS;
 	}
 
 	if (!uid)
diff --git a/fs/smb/server/transport_rdma.c b/fs/smb/server/transport_rdma.c
index 4e7ab8d9314f..f585359684d4 100644
--- a/fs/smb/server/transport_rdma.c
+++ b/fs/smb/server/transport_rdma.c
@@ -242,6 +242,7 @@ static void smb_direct_disconnect_rdma_work(struct work_struct *work)
 	 * disable[_delayed]_work_sync()
 	 */
 	disable_work(&sc->disconnect_work);
+	disable_work(&sc->connect.work);
 	disable_work(&sc->recv_io.posted.refill_work);
 	disable_delayed_work(&sc->idle.timer_work);
 	disable_work(&sc->idle.immediate_work);
@@ -297,6 +298,7 @@ smb_direct_disconnect_rdma_connection(struct smbdirect_socket *sc)
 	 * not queued again but here we don't block and avoid
 	 * disable[_delayed]_work_sync()
 	 */
+	disable_work(&sc->connect.work);
 	disable_work(&sc->recv_io.posted.refill_work);
 	disable_work(&sc->idle.immediate_work);
 	disable_delayed_work(&sc->idle.timer_work);
@@ -467,6 +469,7 @@ static void free_transport(struct smb_direct_transport *t)
 	 */
 	smb_direct_disconnect_wake_up_all(sc);
 
+	disable_work_sync(&sc->connect.work);
 	disable_work_sync(&sc->recv_io.posted.refill_work);
 	disable_delayed_work_sync(&sc->idle.timer_work);
 	disable_work_sync(&sc->idle.immediate_work);
@@ -635,28 +638,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
 
 	switch (sc->recv_io.expected) {
 	case SMBDIRECT_EXPECT_NEGOTIATE_REQ:
-		if (wc->byte_len < sizeof(struct smbdirect_negotiate_req)) {
-			put_recvmsg(sc, recvmsg);
-			smb_direct_disconnect_rdma_connection(sc);
-			return;
-		}
-		sc->recv_io.reassembly.full_packet_received = true;
-		/*
-		 * Some drivers (at least mlx5_ib) might post a
-		 * recv completion before RDMA_CM_EVENT_ESTABLISHED,
-		 * we need to adjust our expectation in that case.
-		 */
-		if (!sc->first_error && sc->status == SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING)
-			sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED;
-		if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)) {
-			put_recvmsg(sc, recvmsg);
-			smb_direct_disconnect_rdma_connection(sc);
-			return;
-		}
-		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING;
-		enqueue_reassembly(sc, recvmsg, 0);
-		wake_up(&sc->status_wait);
-		return;
+		/* see smb_direct_negotiate_recv_done */
+		break;
 	case SMBDIRECT_EXPECT_DATA_TRANSFER: {
 		struct smbdirect_data_transfer *data_transfer =
 			(struct smbdirect_data_transfer *)recvmsg->packet;
@@ -742,6 +725,126 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
 	smb_direct_disconnect_rdma_connection(sc);
 }
 
+static void smb_direct_negotiate_recv_work(struct work_struct *work);
+
+static void smb_direct_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct smbdirect_recv_io *recv_io =
+		container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe);
+	struct smbdirect_socket *sc = recv_io->socket;
+	unsigned long flags;
+
+	/*
+	 * reset the common recv_done for later reuse.
+	 */
+	recv_io->cqe.done = recv_done;
+
+	if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) {
+		put_recvmsg(sc, recv_io);
+		if (wc->status != IB_WC_WR_FLUSH_ERR) {
+			pr_err("Negotiate Recv error. status='%s (%d)' opcode=%d\n",
+			       ib_wc_status_msg(wc->status), wc->status,
+			       wc->opcode);
+			smb_direct_disconnect_rdma_connection(sc);
+		}
+		return;
+	}
+
+	ksmbd_debug(RDMA, "Negotiate Recv completed. status='%s (%d)', opcode=%d\n",
+		    ib_wc_status_msg(wc->status), wc->status,
+		    wc->opcode);
+
+	ib_dma_sync_single_for_cpu(sc->ib.dev,
+				   recv_io->sge.addr,
+				   recv_io->sge.length,
+				   DMA_FROM_DEVICE);
+
+	/*
+	 * This is an internal error!
+	 */
+	if (WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_NEGOTIATE_REQ)) {
+		put_recvmsg(sc, recv_io);
+		smb_direct_disconnect_rdma_connection(sc);
+		return;
+	}
+
+	/*
+	 * Don't reset timer to the keepalive interval in
+	 * this will be done in smb_direct_negotiate_recv_work.
+	 */
+
+	/*
+	 * Only remember the recv_io if it has enough bytes,
+	 * this gives smb_direct_negotiate_recv_work enough
+	 * information in order to disconnect if it was not
+	 * valid.
+	 */
+	sc->recv_io.reassembly.full_packet_received = true;
+	if (wc->byte_len >= sizeof(struct smbdirect_negotiate_req))
+		enqueue_reassembly(sc, recv_io, 0);
+	else
+		put_recvmsg(sc, recv_io);
+
+	/*
+	 * Some drivers (at least mlx5_ib and irdma in roce mode)
+	 * might post a recv completion before RDMA_CM_EVENT_ESTABLISHED,
+	 * we need to adjust our expectation in that case.
+	 *
+	 * So we defer further processing of the negotiation
+	 * to smb_direct_negotiate_recv_work().
+	 *
+	 * If we are already in SMBDIRECT_SOCKET_NEGOTIATE_NEEDED
+	 * we queue the work directly otherwise
+	 * smb_direct_cm_handler() will do it, when
+	 * RDMA_CM_EVENT_ESTABLISHED arrived.
+	 */
+	spin_lock_irqsave(&sc->connect.lock, flags);
+	if (!sc->first_error) {
+		INIT_WORK(&sc->connect.work, smb_direct_negotiate_recv_work);
+		if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)
+			queue_work(sc->workqueue, &sc->connect.work);
+	}
+	spin_unlock_irqrestore(&sc->connect.lock, flags);
+}
+
+static void smb_direct_negotiate_recv_work(struct work_struct *work)
+{
+	struct smbdirect_socket *sc =
+		container_of(work, struct smbdirect_socket, connect.work);
+	const struct smbdirect_socket_parameters *sp = &sc->parameters;
+	struct smbdirect_recv_io *recv_io;
+
+	if (sc->first_error)
+		return;
+
+	ksmbd_debug(RDMA, "Negotiate Recv Work running\n");
+
+	/*
+	 * Reset timer to the keepalive interval in
+	 * order to trigger our next keepalive message.
+	 */
+	sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE;
+	mod_delayed_work(sc->workqueue, &sc->idle.timer_work,
+			 msecs_to_jiffies(sp->keepalive_interval_msec));
+
+	/*
+	 * If smb_direct_negotiate_recv_done() detected an
+	 * invalid request we want to disconnect.
+	 */
+	recv_io = get_first_reassembly(sc);
+	if (!recv_io) {
+		smb_direct_disconnect_rdma_connection(sc);
+		return;
+	}
+
+	if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)) {
+		smb_direct_disconnect_rdma_connection(sc);
+		return;
+	}
+	sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING;
+	wake_up(&sc->status_wait);
+}
+
 static int smb_direct_post_recv(struct smbdirect_socket *sc,
 				struct smbdirect_recv_io *recvmsg)
 {
@@ -758,7 +861,6 @@ static int smb_direct_post_recv(struct smbdirect_socket *sc,
 		return ret;
 	recvmsg->sge.length = sp->max_recv_size;
 	recvmsg->sge.lkey = sc->ib.pd->local_dma_lkey;
-	recvmsg->cqe.done = recv_done;
 
 	wr.wr_cqe = &recvmsg->cqe;
 	wr.next = NULL;
@@ -1732,6 +1834,7 @@ static int smb_direct_cm_handler(struct rdma_cm_id *cm_id,
 				 struct rdma_cm_event *event)
 {
 	struct smbdirect_socket *sc = cm_id->context;
+	unsigned long flags;
 
 	ksmbd_debug(RDMA, "RDMA CM event. cm_id=%p event=%s (%d)\n",
 		    cm_id, rdma_event_msg(event->event), event->event);
@@ -1739,18 +1842,27 @@ static int smb_direct_cm_handler(struct rdma_cm_id *cm_id,
 	switch (event->event) {
 	case RDMA_CM_EVENT_ESTABLISHED: {
 		/*
-		 * Some drivers (at least mlx5_ib) might post a
-		 * recv completion before RDMA_CM_EVENT_ESTABLISHED,
+		 * Some drivers (at least mlx5_ib and irdma in roce mode)
+		 * might post a recv completion before RDMA_CM_EVENT_ESTABLISHED,
 		 * we need to adjust our expectation in that case.
 		 *
-		 * As we already started the negotiation, we just
-		 * ignore RDMA_CM_EVENT_ESTABLISHED here.
+		 * If smb_direct_negotiate_recv_done was called first
+		 * it initialized sc->connect.work only for us to
+		 * start, so that we turned into
+		 * SMBDIRECT_SOCKET_NEGOTIATE_NEEDED, before
+		 * smb_direct_negotiate_recv_work() runs.
+		 *
+		 * If smb_direct_negotiate_recv_done didn't happen
+		 * yet. sc->connect.work is still be disabled and
+		 * queue_work() is a no-op.
 		 */
-		if (!sc->first_error && sc->status > SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING)
-			break;
 		if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING))
 			break;
 		sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED;
+		spin_lock_irqsave(&sc->connect.lock, flags);
+		if (!sc->first_error)
+			queue_work(sc->workqueue, &sc->connect.work);
+		spin_unlock_irqrestore(&sc->connect.lock, flags);
 		wake_up(&sc->status_wait);
 		break;
 	}
@@ -1921,6 +2033,7 @@ static int smb_direct_prepare_negotiation(struct smbdirect_socket *sc)
 	recvmsg = get_free_recvmsg(sc);
 	if (!recvmsg)
 		return -ENOMEM;
+	recvmsg->cqe.done = smb_direct_negotiate_recv_done;
 
 	ret = smb_direct_post_recv(sc, recvmsg);
 	if (ret) {
@@ -2339,6 +2452,7 @@ static int smb_direct_prepare(struct ksmbd_transport *t)
 
 static int smb_direct_connect(struct smbdirect_socket *sc)
 {
+	struct smbdirect_recv_io *recv_io;
 	int ret;
 
 	ret = smb_direct_init_params(sc);
@@ -2353,6 +2467,9 @@ static int smb_direct_connect(struct smbdirect_socket *sc)
 		return ret;
 	}
 
+	list_for_each_entry(recv_io, &sc->recv_io.free.list, list)
+		recv_io->cqe.done = recv_done;
+
 	ret = smb_direct_create_qpair(sc);
 	if (ret) {
 		pr_err("Can't accept RDMA client: %d\n", ret);
diff --git a/fs/smb/server/vfs.c b/fs/smb/server/vfs.c
index 98b0eb966d91..f891344bd76b 100644
--- a/fs/smb/server/vfs.c
+++ b/fs/smb/server/vfs.c
@@ -702,7 +702,7 @@ int ksmbd_vfs_rename(struct ksmbd_work *work, const struct path *old_path,
 	rd.old_parent		= NULL;
 	rd.new_parent		= new_path.dentry;
 	rd.flags		= flags;
-	rd.delegated_inode	= NULL,
+	rd.delegated_inode	= NULL;
 	err = start_renaming_dentry(&rd, lookup_flags, old_child, &new_last);
 	if (err)
 		goto out_drop_write;
diff --git a/fs/splice.c b/fs/splice.c
index d338fe56b50b..5fb07c01936f 100644
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -1467,7 +1467,7 @@ static ssize_t iter_to_pipe(struct iov_iter *from,
 
 		n = DIV_ROUND_UP(left + start, PAGE_SIZE);
 		for (i = 0; i < n; i++) {
-			int size = min_t(int, left, PAGE_SIZE - start);
+			int size = umin(left, PAGE_SIZE - start);
 
 			buf.page = pages[i];
 			buf.offset = start;
diff --git a/fs/udf/unicode.c b/fs/udf/unicode.c
index 32c7f3d27f74..87580ff827ee 100644
--- a/fs/udf/unicode.c
+++ b/fs/udf/unicode.c
@@ -16,6 +16,7 @@
 
 #include "udfdecl.h"
 
+#include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/string.h>	/* for memset */
 #include <linux/nls.h>
diff --git a/fs/xfs/libxfs/xfs_log_format.h b/fs/xfs/libxfs/xfs_log_format.h
index 908e7060428c..3f5a24dda907 100644
--- a/fs/xfs/libxfs/xfs_log_format.h
+++ b/fs/xfs/libxfs/xfs_log_format.h
@@ -184,13 +184,6 @@ struct xlog_rec_header {
 #define XLOG_REC_SIZE_OTHER	offsetofend(struct xlog_rec_header, h_size)
 #endif /* __i386__ */
 
-/* not an on-disk structure, but needed by log recovery in userspace */
-struct xfs_log_iovec {
-	void		*i_addr;	/* beginning address of region */
-	int		i_len;		/* length in bytes of region */
-	uint		i_type;		/* type of region */
-};
-
 /*
  * Transaction Header definitions.
  *
diff --git a/fs/xfs/libxfs/xfs_sb.c b/fs/xfs/libxfs/xfs_sb.c
index cdd16dd805d7..94c272a2ae26 100644
--- a/fs/xfs/libxfs/xfs_sb.c
+++ b/fs/xfs/libxfs/xfs_sb.c
@@ -301,6 +301,21 @@ xfs_validate_rt_geometry(
 	    sbp->sb_rbmblocks != xfs_expected_rbmblocks(sbp))
 		return false;
 
+	if (xfs_sb_is_v5(sbp) &&
+	    (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_ZONED)) {
+		uint32_t		mod;
+
+		/*
+		 * Zoned RT devices must be aligned to the RT group size,
+		 * because garbage collection assumes that all zones have the
+		 * same size to avoid insane complexity if that weren't the
+		 * case.
+		 */
+		div_u64_rem(sbp->sb_rextents, sbp->sb_rgextents, &mod);
+		if (mod)
+			return false;
+	}
+
 	return true;
 }
 
diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c
index c7eb94069caf..09d63aa10314 100644
--- a/fs/xfs/scrub/attr_repair.c
+++ b/fs/xfs/scrub/attr_repair.c
@@ -333,7 +333,6 @@ xrep_xattr_salvage_remote_attr(
 		.attr_filter		= ent->flags & XFS_ATTR_NSP_ONDISK_MASK,
 		.namelen		= rentry->namelen,
 		.name			= rentry->name,
-		.value			= ab->value,
 		.valuelen		= be32_to_cpu(rentry->valuelen),
 	};
 	unsigned int			namesize;
@@ -363,6 +362,7 @@ xrep_xattr_salvage_remote_attr(
 		error = -EDEADLOCK;
 	if (error)
 		return error;
+	args.value = ab->value;
 
 	/* Look up the remote value and stash it for reconstruction. */
 	error = xfs_attr3_leaf_getvalue(leaf_bp, &args);
diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c
index c3a593319bee..ad2956f78eca 100644
--- a/fs/xfs/xfs_attr_item.c
+++ b/fs/xfs/xfs_attr_item.c
@@ -192,10 +192,9 @@ xfs_attri_item_size(
 STATIC void
 xfs_attri_item_format(
 	struct xfs_log_item		*lip,
-	struct xfs_log_vec		*lv)
+	struct xlog_format_buf		*lfb)
 {
 	struct xfs_attri_log_item	*attrip = ATTRI_ITEM(lip);
-	struct xfs_log_iovec		*vecp = NULL;
 	struct xfs_attri_log_nameval	*nv = attrip->attri_nameval;
 
 	attrip->attri_format.alfi_type = XFS_LI_ATTRI;
@@ -220,24 +219,23 @@ xfs_attri_item_format(
 	if (nv->new_value.iov_len > 0)
 		attrip->attri_format.alfi_size++;
 
-	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_ATTRI_FORMAT,
-			&attrip->attri_format,
+	xlog_format_copy(lfb, XLOG_REG_TYPE_ATTRI_FORMAT, &attrip->attri_format,
 			sizeof(struct xfs_attri_log_format));
 
-	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_ATTR_NAME, nv->name.iov_base,
+	xlog_format_copy(lfb, XLOG_REG_TYPE_ATTR_NAME, nv->name.iov_base,
 			nv->name.iov_len);
 
 	if (nv->new_name.iov_len > 0)
-		xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_ATTR_NEWNAME,
-			nv->new_name.iov_base, nv->new_name.iov_len);
+		xlog_format_copy(lfb, XLOG_REG_TYPE_ATTR_NEWNAME,
+				nv->new_name.iov_base, nv->new_name.iov_len);
 
 	if (nv->value.iov_len > 0)
-		xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_ATTR_VALUE,
-			nv->value.iov_base, nv->value.iov_len);
+		xlog_format_copy(lfb, XLOG_REG_TYPE_ATTR_VALUE,
+				nv->value.iov_base, nv->value.iov_len);
 
 	if (nv->new_value.iov_len > 0)
-		xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_ATTR_NEWVALUE,
-			nv->new_value.iov_base, nv->new_value.iov_len);
+		xlog_format_copy(lfb, XLOG_REG_TYPE_ATTR_NEWVALUE,
+				nv->new_value.iov_base, nv->new_value.iov_len);
 }
 
 /*
@@ -322,16 +320,15 @@ xfs_attrd_item_size(
  */
 STATIC void
 xfs_attrd_item_format(
-	struct xfs_log_item	*lip,
-	struct xfs_log_vec	*lv)
+	struct xfs_log_item		*lip,
+	struct xlog_format_buf		*lfb)
 {
 	struct xfs_attrd_log_item	*attrdp = ATTRD_ITEM(lip);
-	struct xfs_log_iovec		*vecp = NULL;
 
 	attrdp->attrd_format.alfd_type = XFS_LI_ATTRD;
 	attrdp->attrd_format.alfd_size = 1;
 
-	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_ATTRD_FORMAT,
+	xlog_format_copy(lfb, XLOG_REG_TYPE_ATTRD_FORMAT,
 			&attrdp->attrd_format,
 			sizeof(struct xfs_attrd_log_format));
 }
@@ -737,7 +734,7 @@ xfs_attr_recover_work(
 	struct xfs_attri_log_item	*attrip = ATTRI_ITEM(lip);
 	struct xfs_attr_intent		*attr;
 	struct xfs_mount		*mp = lip->li_log->l_mp;
-	struct xfs_inode		*ip;
+	struct xfs_inode		*ip = NULL;
 	struct xfs_da_args		*args;
 	struct xfs_trans		*tp;
 	struct xfs_trans_res		resv;
diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c
index 80f0c4bcc483..f38ed63fe86b 100644
--- a/fs/xfs/xfs_bmap_item.c
+++ b/fs/xfs/xfs_bmap_item.c
@@ -92,10 +92,9 @@ unsigned int xfs_bui_log_space(unsigned int nr)
 STATIC void
 xfs_bui_item_format(
 	struct xfs_log_item	*lip,
-	struct xfs_log_vec	*lv)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_bui_log_item	*buip = BUI_ITEM(lip);
-	struct xfs_log_iovec	*vecp = NULL;
 
 	ASSERT(atomic_read(&buip->bui_next_extent) ==
 			buip->bui_format.bui_nextents);
@@ -103,7 +102,7 @@ xfs_bui_item_format(
 	buip->bui_format.bui_type = XFS_LI_BUI;
 	buip->bui_format.bui_size = 1;
 
-	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_BUI_FORMAT, &buip->bui_format,
+	xlog_format_copy(lfb, XLOG_REG_TYPE_BUI_FORMAT, &buip->bui_format,
 			xfs_bui_log_format_sizeof(buip->bui_format.bui_nextents));
 }
 
@@ -188,15 +187,14 @@ unsigned int xfs_bud_log_space(void)
 STATIC void
 xfs_bud_item_format(
 	struct xfs_log_item	*lip,
-	struct xfs_log_vec	*lv)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_bud_log_item	*budp = BUD_ITEM(lip);
-	struct xfs_log_iovec	*vecp = NULL;
 
 	budp->bud_format.bud_type = XFS_LI_BUD;
 	budp->bud_format.bud_size = 1;
 
-	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_BUD_FORMAT, &budp->bud_format,
+	xlog_format_copy(lfb, XLOG_REG_TYPE_BUD_FORMAT, &budp->bud_format,
 			sizeof(struct xfs_bud_log_format));
 }
 
diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c
index 8d85b5eee444..cb2a36374ed4 100644
--- a/fs/xfs/xfs_buf_item.c
+++ b/fs/xfs/xfs_buf_item.c
@@ -263,24 +263,21 @@ xfs_buf_item_size(
 
 static inline void
 xfs_buf_item_copy_iovec(
-	struct xfs_log_vec	*lv,
-	struct xfs_log_iovec	**vecp,
+	struct xlog_format_buf	*lfb,
 	struct xfs_buf		*bp,
 	uint			offset,
 	int			first_bit,
 	uint			nbits)
 {
 	offset += first_bit * XFS_BLF_CHUNK;
-	xlog_copy_iovec(lv, vecp, XLOG_REG_TYPE_BCHUNK,
-			xfs_buf_offset(bp, offset),
+	xlog_format_copy(lfb, XLOG_REG_TYPE_BCHUNK, xfs_buf_offset(bp, offset),
 			nbits * XFS_BLF_CHUNK);
 }
 
 static void
 xfs_buf_item_format_segment(
 	struct xfs_buf_log_item	*bip,
-	struct xfs_log_vec	*lv,
-	struct xfs_log_iovec	**vecp,
+	struct xlog_format_buf	*lfb,
 	uint			offset,
 	struct xfs_buf_log_format *blfp)
 {
@@ -308,7 +305,7 @@ xfs_buf_item_format_segment(
 		return;
 	}
 
-	blfp = xlog_copy_iovec(lv, vecp, XLOG_REG_TYPE_BFORMAT, blfp, base_size);
+	blfp = xlog_format_copy(lfb, XLOG_REG_TYPE_BFORMAT, blfp, base_size);
 	blfp->blf_size = 1;
 
 	if (bip->bli_flags & XFS_BLI_STALE) {
@@ -331,8 +328,7 @@ xfs_buf_item_format_segment(
 		nbits = xfs_contig_bits(blfp->blf_data_map,
 					blfp->blf_map_size, first_bit);
 		ASSERT(nbits > 0);
-		xfs_buf_item_copy_iovec(lv, vecp, bp, offset,
-					first_bit, nbits);
+		xfs_buf_item_copy_iovec(lfb, bp, offset, first_bit, nbits);
 		blfp->blf_size++;
 
 		/*
@@ -357,11 +353,10 @@ xfs_buf_item_format_segment(
 STATIC void
 xfs_buf_item_format(
 	struct xfs_log_item	*lip,
-	struct xfs_log_vec	*lv)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_buf_log_item	*bip = BUF_ITEM(lip);
 	struct xfs_buf		*bp = bip->bli_buf;
-	struct xfs_log_iovec	*vecp = NULL;
 	uint			offset = 0;
 	int			i;
 
@@ -398,7 +393,7 @@ xfs_buf_item_format(
 	}
 
 	for (i = 0; i < bip->bli_format_count; i++) {
-		xfs_buf_item_format_segment(bip, lv, &vecp, offset,
+		xfs_buf_item_format_segment(bip, lfb, offset,
 					    &bip->bli_formats[i]);
 		offset += BBTOB(bp->b_maps[i].bm_len);
 	}
@@ -896,6 +891,7 @@ xfs_buf_item_init(
 		map_size = DIV_ROUND_UP(chunks, NBWORD);
 
 		if (map_size > XFS_BLF_DATAMAP_SIZE) {
+			xfs_buf_item_free_format(bip);
 			kmem_cache_free(xfs_buf_item_cache, bip);
 			xfs_err(mp,
 	"buffer item dirty bitmap (%u uints) too small to reflect %u bytes!",
diff --git a/fs/xfs/xfs_discard.c b/fs/xfs/xfs_discard.c
index 6917de832191..b6ffe4807a11 100644
--- a/fs/xfs/xfs_discard.c
+++ b/fs/xfs/xfs_discard.c
@@ -108,7 +108,7 @@ xfs_discard_endio(
  * list. We plug and chain the bios so that we only need a single completion
  * call to clear all the busy extents once the discards are complete.
  */
-int
+void
 xfs_discard_extents(
 	struct xfs_mount	*mp,
 	struct xfs_busy_extents	*extents)
@@ -116,7 +116,6 @@ xfs_discard_extents(
 	struct xfs_extent_busy	*busyp;
 	struct bio		*bio = NULL;
 	struct blk_plug		plug;
-	int			error = 0;
 
 	blk_start_plug(&plug);
 	list_for_each_entry(busyp, &extents->extent_list, list) {
@@ -126,18 +125,10 @@ xfs_discard_extents(
 
 		trace_xfs_discard_extent(xg, busyp->bno, busyp->length);
 
-		error = __blkdev_issue_discard(btp->bt_bdev,
+		__blkdev_issue_discard(btp->bt_bdev,
 				xfs_gbno_to_daddr(xg, busyp->bno),
 				XFS_FSB_TO_BB(mp, busyp->length),
 				GFP_KERNEL, &bio);
-		if (error && error != -EOPNOTSUPP) {
-			xfs_info(mp,
-	 "discard failed for extent [0x%llx,%u], error %d",
-				 (unsigned long long)busyp->bno,
-				 busyp->length,
-				 error);
-			break;
-		}
 	}
 
 	if (bio) {
@@ -148,8 +139,6 @@ xfs_discard_extents(
 		xfs_discard_endio_work(&extents->endio_work);
 	}
 	blk_finish_plug(&plug);
-
-	return error;
 }
 
 /*
@@ -385,9 +374,7 @@ xfs_trim_perag_extents(
 		 * list  after this function call, as it may have been freed by
 		 * the time control returns to us.
 		 */
-		error = xfs_discard_extents(pag_mount(pag), extents);
-		if (error)
-			break;
+		xfs_discard_extents(pag_mount(pag), extents);
 
 		if (xfs_trim_should_stop())
 			break;
@@ -496,12 +483,10 @@ xfs_discard_rtdev_extents(
 
 		trace_xfs_discard_rtextent(mp, busyp->bno, busyp->length);
 
-		error = __blkdev_issue_discard(bdev,
+		__blkdev_issue_discard(bdev,
 				xfs_rtb_to_daddr(mp, busyp->bno),
 				XFS_FSB_TO_BB(mp, busyp->length),
 				GFP_NOFS, &bio);
-		if (error)
-			break;
 	}
 	xfs_discard_free_rtdev_extents(tr);
 
@@ -741,9 +726,7 @@ xfs_trim_rtgroup_extents(
 		 * list  after this function call, as it may have been freed by
 		 * the time control returns to us.
 		 */
-		error = xfs_discard_extents(rtg_mount(rtg), tr.extents);
-		if (error)
-			break;
+		xfs_discard_extents(rtg_mount(rtg), tr.extents);
 
 		low = tr.restart_rtx;
 	} while (!xfs_trim_should_stop() && low <= high);
diff --git a/fs/xfs/xfs_discard.h b/fs/xfs/xfs_discard.h
index 2b1a85223a56..8c5cc4af6a07 100644
--- a/fs/xfs/xfs_discard.h
+++ b/fs/xfs/xfs_discard.h
@@ -6,7 +6,7 @@ struct fstrim_range;
 struct xfs_mount;
 struct xfs_busy_extents;
 
-int xfs_discard_extents(struct xfs_mount *mp, struct xfs_busy_extents *busy);
+void xfs_discard_extents(struct xfs_mount *mp, struct xfs_busy_extents *busy);
 int xfs_ioc_trim(struct xfs_mount *mp, struct fstrim_range __user *fstrim);
 
 #endif /* XFS_DISCARD_H */
diff --git a/fs/xfs/xfs_dquot_item.c b/fs/xfs/xfs_dquot_item.c
index b374cd9f1900..8bc7f43093a2 100644
--- a/fs/xfs/xfs_dquot_item.c
+++ b/fs/xfs/xfs_dquot_item.c
@@ -44,25 +44,24 @@ xfs_qm_dquot_logitem_size(
 STATIC void
 xfs_qm_dquot_logitem_format(
 	struct xfs_log_item	*lip,
-	struct xfs_log_vec	*lv)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_disk_dquot	ddq;
 	struct xfs_dq_logitem	*qlip = DQUOT_ITEM(lip);
-	struct xfs_log_iovec	*vecp = NULL;
 	struct xfs_dq_logformat	*qlf;
 
-	qlf = xlog_prepare_iovec(lv, &vecp, XLOG_REG_TYPE_QFORMAT);
+	qlf = xlog_format_start(lfb, XLOG_REG_TYPE_QFORMAT);
 	qlf->qlf_type = XFS_LI_DQUOT;
 	qlf->qlf_size = 2;
 	qlf->qlf_id = qlip->qli_dquot->q_id;
 	qlf->qlf_blkno = qlip->qli_dquot->q_blkno;
 	qlf->qlf_len = 1;
 	qlf->qlf_boffset = qlip->qli_dquot->q_bufoffset;
-	xlog_finish_iovec(lv, vecp, sizeof(struct xfs_dq_logformat));
+	xlog_format_commit(lfb, sizeof(struct xfs_dq_logformat));
 
 	xfs_dquot_to_disk(&ddq, qlip->qli_dquot);
 
-	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_DQUOT, &ddq,
+	xlog_format_copy(lfb, XLOG_REG_TYPE_DQUOT, &ddq,
 			sizeof(struct xfs_disk_dquot));
 }
 
diff --git a/fs/xfs/xfs_exchmaps_item.c b/fs/xfs/xfs_exchmaps_item.c
index 229cbe0adf17..10d6fbeff651 100644
--- a/fs/xfs/xfs_exchmaps_item.c
+++ b/fs/xfs/xfs_exchmaps_item.c
@@ -83,16 +83,14 @@ xfs_xmi_item_size(
 STATIC void
 xfs_xmi_item_format(
 	struct xfs_log_item	*lip,
-	struct xfs_log_vec	*lv)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_xmi_log_item	*xmi_lip = XMI_ITEM(lip);
-	struct xfs_log_iovec	*vecp = NULL;
 
 	xmi_lip->xmi_format.xmi_type = XFS_LI_XMI;
 	xmi_lip->xmi_format.xmi_size = 1;
 
-	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_XMI_FORMAT,
-			&xmi_lip->xmi_format,
+	xlog_format_copy(lfb, XLOG_REG_TYPE_XMI_FORMAT, &xmi_lip->xmi_format,
 			sizeof(struct xfs_xmi_log_format));
 }
 
@@ -166,15 +164,14 @@ xfs_xmd_item_size(
 STATIC void
 xfs_xmd_item_format(
 	struct xfs_log_item	*lip,
-	struct xfs_log_vec	*lv)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_xmd_log_item	*xmd_lip = XMD_ITEM(lip);
-	struct xfs_log_iovec	*vecp = NULL;
 
 	xmd_lip->xmd_format.xmd_type = XFS_LI_XMD;
 	xmd_lip->xmd_format.xmd_size = 1;
 
-	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_XMD_FORMAT, &xmd_lip->xmd_format,
+	xlog_format_copy(lfb, XLOG_REG_TYPE_XMD_FORMAT, &xmd_lip->xmd_format,
 			sizeof(struct xfs_xmd_log_format));
 }
 
diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c
index 418ddab590e0..3d1edc43e6fb 100644
--- a/fs/xfs/xfs_extfree_item.c
+++ b/fs/xfs/xfs_extfree_item.c
@@ -98,10 +98,9 @@ unsigned int xfs_efi_log_space(unsigned int nr)
 STATIC void
 xfs_efi_item_format(
 	struct xfs_log_item	*lip,
-	struct xfs_log_vec	*lv)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_efi_log_item	*efip = EFI_ITEM(lip);
-	struct xfs_log_iovec	*vecp = NULL;
 
 	ASSERT(atomic_read(&efip->efi_next_extent) ==
 				efip->efi_format.efi_nextents);
@@ -110,7 +109,7 @@ xfs_efi_item_format(
 	efip->efi_format.efi_type = lip->li_type;
 	efip->efi_format.efi_size = 1;
 
-	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_EFI_FORMAT, &efip->efi_format,
+	xlog_format_copy(lfb, XLOG_REG_TYPE_EFI_FORMAT, &efip->efi_format,
 			xfs_efi_log_format_sizeof(efip->efi_format.efi_nextents));
 }
 
@@ -277,10 +276,9 @@ unsigned int xfs_efd_log_space(unsigned int nr)
 STATIC void
 xfs_efd_item_format(
 	struct xfs_log_item	*lip,
-	struct xfs_log_vec	*lv)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_efd_log_item	*efdp = EFD_ITEM(lip);
-	struct xfs_log_iovec	*vecp = NULL;
 
 	ASSERT(efdp->efd_next_extent == efdp->efd_format.efd_nextents);
 	ASSERT(lip->li_type == XFS_LI_EFD || lip->li_type == XFS_LI_EFD_RT);
@@ -288,7 +286,7 @@ xfs_efd_item_format(
 	efdp->efd_format.efd_type = lip->li_type;
 	efdp->efd_format.efd_size = 1;
 
-	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_EFD_FORMAT, &efdp->efd_format,
+	xlog_format_copy(lfb, XLOG_REG_TYPE_EFD_FORMAT, &efdp->efd_format,
 			xfs_efd_log_format_sizeof(efdp->efd_format.efd_nextents));
 }
 
diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c
index 6108612182e2..7874cf745af3 100644
--- a/fs/xfs/xfs_file.c
+++ b/fs/xfs/xfs_file.c
@@ -1240,6 +1240,38 @@ xfs_falloc_insert_range(
 	return xfs_insert_file_space(XFS_I(inode), offset, len);
 }
 
+/*
+ * For various operations we need to zero up to one block at each end of
+ * the affected range.  For zoned file systems this will require a space
+ * allocation, for which we need a reservation ahead of time.
+ */
+#define XFS_ZONED_ZERO_EDGE_SPACE_RES		2
+
+/*
+ * Zero range implements a full zeroing mechanism but is only used in limited
+ * situations. It is more efficient to allocate unwritten extents than to
+ * perform zeroing here, so use an errortag to randomly force zeroing on DEBUG
+ * kernels for added test coverage.
+ *
+ * On zoned file systems, the error is already injected by
+ * xfs_file_zoned_fallocate, which then reserves the additional space needed.
+ * We only check for this extra space reservation here.
+ */
+static inline bool
+xfs_falloc_force_zero(
+	struct xfs_inode		*ip,
+	struct xfs_zone_alloc_ctx	*ac)
+{
+	if (xfs_is_zoned_inode(ip)) {
+		if (ac->reserved_blocks > XFS_ZONED_ZERO_EDGE_SPACE_RES) {
+			ASSERT(IS_ENABLED(CONFIG_XFS_DEBUG));
+			return true;
+		}
+		return false;
+	}
+	return XFS_TEST_ERROR(ip->i_mount, XFS_ERRTAG_FORCE_ZERO_RANGE);
+}
+
 /*
  * Punch a hole and prealloc the range.  We use a hole punch rather than
  * unwritten extent conversion for two reasons:
@@ -1268,14 +1300,7 @@ xfs_falloc_zero_range(
 	if (error)
 		return error;
 
-	/*
-	 * Zero range implements a full zeroing mechanism but is only used in
-	 * limited situations. It is more efficient to allocate unwritten
-	 * extents than to perform zeroing here, so use an errortag to randomly
-	 * force zeroing on DEBUG kernels for added test coverage.
-	 */
-	if (XFS_TEST_ERROR(ip->i_mount,
-			   XFS_ERRTAG_FORCE_ZERO_RANGE)) {
+	if (xfs_falloc_force_zero(ip, ac)) {
 		error = xfs_zero_range(ip, offset, len, ac, NULL);
 	} else {
 		error = xfs_free_file_space(ip, offset, len, ac);
@@ -1423,13 +1448,26 @@ xfs_file_zoned_fallocate(
 {
 	struct xfs_zone_alloc_ctx ac = { };
 	struct xfs_inode	*ip = XFS_I(file_inode(file));
+	struct xfs_mount	*mp = ip->i_mount;
+	xfs_filblks_t		count_fsb;
 	int			error;
 
-	error = xfs_zoned_space_reserve(ip->i_mount, 2, XFS_ZR_RESERVED, &ac);
+	/*
+	 * If full zeroing is forced by the error injection knob, we need a
+	 * space reservation that covers the entire range.  See the comment in
+	 * xfs_zoned_write_space_reserve for the rationale for the calculation.
+	 * Otherwise just reserve space for the two boundary blocks.
+	 */
+	count_fsb = XFS_ZONED_ZERO_EDGE_SPACE_RES;
+	if ((mode & FALLOC_FL_MODE_MASK) == FALLOC_FL_ZERO_RANGE &&
+	    XFS_TEST_ERROR(mp, XFS_ERRTAG_FORCE_ZERO_RANGE))
+		count_fsb += XFS_B_TO_FSB(mp, len) + 1;
+
+	error = xfs_zoned_space_reserve(mp, count_fsb, XFS_ZR_RESERVED, &ac);
 	if (error)
 		return error;
 	error = __xfs_file_fallocate(file, mode, offset, len, &ac);
-	xfs_zoned_space_unreserve(ip->i_mount, &ac);
+	xfs_zoned_space_unreserve(mp, &ac);
 	return error;
 }
 
diff --git a/fs/xfs/xfs_icreate_item.c b/fs/xfs/xfs_icreate_item.c
index f83ec2bd0583..004dd22393dc 100644
--- a/fs/xfs/xfs_icreate_item.c
+++ b/fs/xfs/xfs_icreate_item.c
@@ -49,13 +49,11 @@ xfs_icreate_item_size(
 STATIC void
 xfs_icreate_item_format(
 	struct xfs_log_item	*lip,
-	struct xfs_log_vec	*lv)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_icreate_item	*icp = ICR_ITEM(lip);
-	struct xfs_log_iovec	*vecp = NULL;
 
-	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_ICREATE,
-			&icp->ic_format,
+	xlog_format_copy(lfb, XLOG_REG_TYPE_ICREATE, &icp->ic_format,
 			sizeof(struct xfs_icreate_log));
 }
 
diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c
index 2eb0c6011a2e..81dfe70e173d 100644
--- a/fs/xfs/xfs_inode_item.c
+++ b/fs/xfs/xfs_inode_item.c
@@ -336,8 +336,7 @@ STATIC void
 xfs_inode_item_format_data_fork(
 	struct xfs_inode_log_item *iip,
 	struct xfs_inode_log_format *ilf,
-	struct xfs_log_vec	*lv,
-	struct xfs_log_iovec	**vecp)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_inode	*ip = iip->ili_inode;
 	size_t			data_bytes;
@@ -354,9 +353,9 @@ xfs_inode_item_format_data_fork(
 
 			ASSERT(xfs_iext_count(&ip->i_df) > 0);
 
-			p = xlog_prepare_iovec(lv, vecp, XLOG_REG_TYPE_IEXT);
+			p = xlog_format_start(lfb, XLOG_REG_TYPE_IEXT);
 			data_bytes = xfs_iextents_copy(ip, p, XFS_DATA_FORK);
-			xlog_finish_iovec(lv, *vecp, data_bytes);
+			xlog_format_commit(lfb, data_bytes);
 
 			ASSERT(data_bytes <= ip->i_df.if_bytes);
 
@@ -374,7 +373,7 @@ xfs_inode_item_format_data_fork(
 		if ((iip->ili_fields & XFS_ILOG_DBROOT) &&
 		    ip->i_df.if_broot_bytes > 0) {
 			ASSERT(ip->i_df.if_broot != NULL);
-			xlog_copy_iovec(lv, vecp, XLOG_REG_TYPE_IBROOT,
+			xlog_format_copy(lfb, XLOG_REG_TYPE_IBROOT,
 					ip->i_df.if_broot,
 					ip->i_df.if_broot_bytes);
 			ilf->ilf_dsize = ip->i_df.if_broot_bytes;
@@ -392,8 +391,9 @@ xfs_inode_item_format_data_fork(
 		    ip->i_df.if_bytes > 0) {
 			ASSERT(ip->i_df.if_data != NULL);
 			ASSERT(ip->i_disk_size > 0);
-			xlog_copy_iovec(lv, vecp, XLOG_REG_TYPE_ILOCAL,
-					ip->i_df.if_data, ip->i_df.if_bytes);
+			xlog_format_copy(lfb, XLOG_REG_TYPE_ILOCAL,
+					ip->i_df.if_data,
+					ip->i_df.if_bytes);
 			ilf->ilf_dsize = (unsigned)ip->i_df.if_bytes;
 			ilf->ilf_size++;
 		} else {
@@ -416,8 +416,7 @@ STATIC void
 xfs_inode_item_format_attr_fork(
 	struct xfs_inode_log_item *iip,
 	struct xfs_inode_log_format *ilf,
-	struct xfs_log_vec	*lv,
-	struct xfs_log_iovec	**vecp)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_inode	*ip = iip->ili_inode;
 	size_t			data_bytes;
@@ -435,9 +434,9 @@ xfs_inode_item_format_attr_fork(
 			ASSERT(xfs_iext_count(&ip->i_af) ==
 				ip->i_af.if_nextents);
 
-			p = xlog_prepare_iovec(lv, vecp, XLOG_REG_TYPE_IATTR_EXT);
+			p = xlog_format_start(lfb, XLOG_REG_TYPE_IATTR_EXT);
 			data_bytes = xfs_iextents_copy(ip, p, XFS_ATTR_FORK);
-			xlog_finish_iovec(lv, *vecp, data_bytes);
+			xlog_format_commit(lfb, data_bytes);
 
 			ilf->ilf_asize = data_bytes;
 			ilf->ilf_size++;
@@ -453,7 +452,7 @@ xfs_inode_item_format_attr_fork(
 		    ip->i_af.if_broot_bytes > 0) {
 			ASSERT(ip->i_af.if_broot != NULL);
 
-			xlog_copy_iovec(lv, vecp, XLOG_REG_TYPE_IATTR_BROOT,
+			xlog_format_copy(lfb, XLOG_REG_TYPE_IATTR_BROOT,
 					ip->i_af.if_broot,
 					ip->i_af.if_broot_bytes);
 			ilf->ilf_asize = ip->i_af.if_broot_bytes;
@@ -469,8 +468,9 @@ xfs_inode_item_format_attr_fork(
 		if ((iip->ili_fields & XFS_ILOG_ADATA) &&
 		    ip->i_af.if_bytes > 0) {
 			ASSERT(ip->i_af.if_data != NULL);
-			xlog_copy_iovec(lv, vecp, XLOG_REG_TYPE_IATTR_LOCAL,
-					ip->i_af.if_data, ip->i_af.if_bytes);
+			xlog_format_copy(lfb, XLOG_REG_TYPE_IATTR_LOCAL,
+					ip->i_af.if_data,
+					ip->i_af.if_bytes);
 			ilf->ilf_asize = (unsigned)ip->i_af.if_bytes;
 			ilf->ilf_size++;
 		} else {
@@ -619,14 +619,13 @@ xfs_inode_to_log_dinode(
 static void
 xfs_inode_item_format_core(
 	struct xfs_inode	*ip,
-	struct xfs_log_vec	*lv,
-	struct xfs_log_iovec	**vecp)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_log_dinode	*dic;
 
-	dic = xlog_prepare_iovec(lv, vecp, XLOG_REG_TYPE_ICORE);
+	dic = xlog_format_start(lfb, XLOG_REG_TYPE_ICORE);
 	xfs_inode_to_log_dinode(ip, dic, ip->i_itemp->ili_item.li_lsn);
-	xlog_finish_iovec(lv, *vecp, xfs_log_dinode_size(ip->i_mount));
+	xlog_format_commit(lfb, xfs_log_dinode_size(ip->i_mount));
 }
 
 /*
@@ -644,14 +643,13 @@ xfs_inode_item_format_core(
 STATIC void
 xfs_inode_item_format(
 	struct xfs_log_item	*lip,
-	struct xfs_log_vec	*lv)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_inode_log_item *iip = INODE_ITEM(lip);
 	struct xfs_inode	*ip = iip->ili_inode;
-	struct xfs_log_iovec	*vecp = NULL;
 	struct xfs_inode_log_format *ilf;
 
-	ilf = xlog_prepare_iovec(lv, &vecp, XLOG_REG_TYPE_IFORMAT);
+	ilf = xlog_format_start(lfb, XLOG_REG_TYPE_IFORMAT);
 	ilf->ilf_type = XFS_LI_INODE;
 	ilf->ilf_ino = ip->i_ino;
 	ilf->ilf_blkno = ip->i_imap.im_blkno;
@@ -668,13 +666,12 @@ xfs_inode_item_format(
 	ilf->ilf_asize = 0;
 	ilf->ilf_pad = 0;
 	memset(&ilf->ilf_u, 0, sizeof(ilf->ilf_u));
+	xlog_format_commit(lfb, sizeof(*ilf));
 
-	xlog_finish_iovec(lv, vecp, sizeof(*ilf));
-
-	xfs_inode_item_format_core(ip, lv, &vecp);
-	xfs_inode_item_format_data_fork(iip, ilf, lv, &vecp);
+	xfs_inode_item_format_core(ip, lfb);
+	xfs_inode_item_format_data_fork(iip, ilf, lfb);
 	if (xfs_inode_has_attr_fork(ip)) {
-		xfs_inode_item_format_attr_fork(iip, ilf, lv, &vecp);
+		xfs_inode_item_format_attr_fork(iip, ilf, lfb);
 	} else {
 		iip->ili_fields &=
 			~(XFS_ILOG_ADATA | XFS_ILOG_ABROOT | XFS_ILOG_AEXT);
diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c
index 04f39ea15898..37a1b33e9045 100644
--- a/fs/xfs/xfs_iomap.c
+++ b/fs/xfs/xfs_iomap.c
@@ -1831,7 +1831,6 @@ xfs_buffered_write_iomap_begin(
 	 */
 	if (flags & IOMAP_ZERO) {
 		xfs_fileoff_t eof_fsb = XFS_B_TO_FSB(mp, XFS_ISIZE(ip));
-		u64 end;
 
 		if (isnullstartblock(imap.br_startblock) &&
 		    offset_fsb >= eof_fsb)
@@ -1851,12 +1850,14 @@ xfs_buffered_write_iomap_begin(
 		 */
 		if (imap.br_state == XFS_EXT_UNWRITTEN &&
 		    offset_fsb < eof_fsb) {
-			loff_t len = min(count,
-					 XFS_FSB_TO_B(mp, imap.br_blockcount));
+			loff_t foffset = offset, fend;
 
-			end = iomap_fill_dirty_folios(iter, offset, len);
+			fend = offset +
+			       min(count, XFS_FSB_TO_B(mp, imap.br_blockcount));
+			iomap_fill_dirty_folios(iter, &foffset, fend,
+						&iomap_flags);
 			end_fsb = min_t(xfs_fileoff_t, end_fsb,
-					XFS_B_TO_FSB(mp, end));
+					XFS_B_TO_FSB(mp, foffset));
 		}
 
 		xfs_trim_extent(&imap, offset_fsb, end_fsb - offset_fsb);
diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c
index a311385b23d8..511756429336 100644
--- a/fs/xfs/xfs_log.c
+++ b/fs/xfs/xfs_log.c
@@ -22,6 +22,15 @@
 #include "xfs_health.h"
 #include "xfs_zone_alloc.h"
 
+struct xlog_write_data {
+	struct xlog_ticket	*ticket;
+	struct xlog_in_core	*iclog;
+	uint32_t		bytes_left;
+	uint32_t		record_cnt;
+	uint32_t		data_cnt;
+	int			log_offset;
+};
+
 struct kmem_cache	*xfs_log_ticket_cache;
 
 /* Local miscellaneous function prototypes */
@@ -43,10 +52,7 @@ STATIC void xlog_state_do_callback(
 STATIC int
 xlog_state_get_iclog_space(
 	struct xlog		*log,
-	int			len,
-	struct xlog_in_core	**iclog,
-	struct xlog_ticket	*ticket,
-	int			*logoffsetp);
+	struct xlog_write_data	*data);
 STATIC void
 xlog_sync(
 	struct xlog		*log,
@@ -74,62 +80,6 @@ xlog_iclogs_empty(
 static int
 xfs_log_cover(struct xfs_mount *);
 
-/*
- * We need to make sure the buffer pointer returned is naturally aligned for the
- * biggest basic data type we put into it. We have already accounted for this
- * padding when sizing the buffer.
- *
- * However, this padding does not get written into the log, and hence we have to
- * track the space used by the log vectors separately to prevent log space hangs
- * due to inaccurate accounting (i.e. a leak) of the used log space through the
- * CIL context ticket.
- *
- * We also add space for the xlog_op_header that describes this region in the
- * log. This prepends the data region we return to the caller to copy their data
- * into, so do all the static initialisation of the ophdr now. Because the ophdr
- * is not 8 byte aligned, we have to be careful to ensure that we align the
- * start of the buffer such that the region we return to the call is 8 byte
- * aligned and packed against the tail of the ophdr.
- */
-void *
-xlog_prepare_iovec(
-	struct xfs_log_vec	*lv,
-	struct xfs_log_iovec	**vecp,
-	uint			type)
-{
-	struct xfs_log_iovec	*vec = *vecp;
-	struct xlog_op_header	*oph;
-	uint32_t		len;
-	void			*buf;
-
-	if (vec) {
-		ASSERT(vec - lv->lv_iovecp < lv->lv_niovecs);
-		vec++;
-	} else {
-		vec = &lv->lv_iovecp[0];
-	}
-
-	len = lv->lv_buf_used + sizeof(struct xlog_op_header);
-	if (!IS_ALIGNED(len, sizeof(uint64_t))) {
-		lv->lv_buf_used = round_up(len, sizeof(uint64_t)) -
-					sizeof(struct xlog_op_header);
-	}
-
-	vec->i_type = type;
-	vec->i_addr = lv->lv_buf + lv->lv_buf_used;
-
-	oph = vec->i_addr;
-	oph->oh_clientid = XFS_TRANSACTION;
-	oph->oh_res2 = 0;
-	oph->oh_flags = 0;
-
-	buf = vec->i_addr + sizeof(struct xlog_op_header);
-	ASSERT(IS_ALIGNED((unsigned long)buf, sizeof(uint64_t)));
-
-	*vecp = vec;
-	return buf;
-}
-
 static inline void
 xlog_grant_sub_space(
 	struct xlog_grant_head	*head,
@@ -848,6 +798,27 @@ xlog_wait_on_iclog(
 	return 0;
 }
 
+int
+xlog_write_one_vec(
+	struct xlog		*log,
+	struct xfs_cil_ctx	*ctx,
+	struct xfs_log_iovec	*reg,
+	struct xlog_ticket	*ticket)
+{
+	struct xfs_log_vec	lv = {
+		.lv_niovecs	= 1,
+		.lv_iovecp	= reg,
+		.lv_bytes	= reg->i_len,
+	};
+	LIST_HEAD		(lv_chain);
+
+	/* account for space used by record data */
+	ticket->t_curr_res -= lv.lv_bytes;
+
+	list_add(&lv.lv_list, &lv_chain);
+	return xlog_write(log, ctx, &lv_chain, ticket, lv.lv_bytes);
+}
+
 /*
  * Write out an unmount record using the ticket provided. We have to account for
  * the data space used in the unmount ticket as this write is not done from a
@@ -876,21 +847,8 @@ xlog_write_unmount_record(
 		.i_len = sizeof(unmount_rec),
 		.i_type = XLOG_REG_TYPE_UNMOUNT,
 	};
-	struct xfs_log_vec vec = {
-		.lv_niovecs = 1,
-		.lv_iovecp = &reg,
-	};
-	LIST_HEAD(lv_chain);
-	list_add(&vec.lv_list, &lv_chain);
-
-	BUILD_BUG_ON((sizeof(struct xlog_op_header) +
-		      sizeof(struct xfs_unmount_log_format)) !=
-							sizeof(unmount_rec));
 
-	/* account for space used by record data */
-	ticket->t_curr_res -= sizeof(unmount_rec);
-
-	return xlog_write(log, NULL, &lv_chain, ticket, reg.i_len);
+	return xlog_write_one_vec(log, NULL, &reg, ticket);
 }
 
 /*
@@ -1920,25 +1878,36 @@ xlog_print_trans(
 	}
 }
 
+static inline uint32_t xlog_write_space_left(struct xlog_write_data *data)
+{
+	return data->iclog->ic_size - data->log_offset;
+}
+
+static void *
+xlog_write_space_advance(
+	struct xlog_write_data	*data,
+	unsigned int		len)
+{
+	void			*p = data->iclog->ic_datap + data->log_offset;
+
+	ASSERT(xlog_write_space_left(data) >= len);
+	ASSERT(data->log_offset % sizeof(int32_t) == 0);
+	ASSERT(len % sizeof(int32_t) == 0);
+
+	data->data_cnt += len;
+	data->log_offset += len;
+	data->bytes_left -= len;
+	return p;
+}
+
 static inline void
 xlog_write_iovec(
-	struct xlog_in_core	*iclog,
-	uint32_t		*log_offset,
-	void			*data,
-	uint32_t		write_len,
-	int			*bytes_left,
-	uint32_t		*record_cnt,
-	uint32_t		*data_cnt)
+	struct xlog_write_data	*data,
+	void			*buf,
+	uint32_t		buf_len)
 {
-	ASSERT(*log_offset < iclog->ic_log->l_iclog_size);
-	ASSERT(*log_offset % sizeof(int32_t) == 0);
-	ASSERT(write_len % sizeof(int32_t) == 0);
-
-	memcpy(iclog->ic_datap + *log_offset, data, write_len);
-	*log_offset += write_len;
-	*bytes_left -= write_len;
-	(*record_cnt)++;
-	*data_cnt += write_len;
+	memcpy(xlog_write_space_advance(data, buf_len), buf, buf_len);
+	data->record_cnt++;
 }
 
 /*
@@ -1948,17 +1917,12 @@ xlog_write_iovec(
 static void
 xlog_write_full(
 	struct xfs_log_vec	*lv,
-	struct xlog_ticket	*ticket,
-	struct xlog_in_core	*iclog,
-	uint32_t		*log_offset,
-	uint32_t		*len,
-	uint32_t		*record_cnt,
-	uint32_t		*data_cnt)
+	struct xlog_write_data	*data)
 {
 	int			index;
 
-	ASSERT(*log_offset + *len <= iclog->ic_size ||
-		iclog->ic_state == XLOG_STATE_WANT_SYNC);
+	ASSERT(data->bytes_left <= xlog_write_space_left(data) ||
+		data->iclog->ic_state == XLOG_STATE_WANT_SYNC);
 
 	/*
 	 * Ordered log vectors have no regions to write so this
@@ -1968,40 +1932,32 @@ xlog_write_full(
 		struct xfs_log_iovec	*reg = &lv->lv_iovecp[index];
 		struct xlog_op_header	*ophdr = reg->i_addr;
 
-		ophdr->oh_tid = cpu_to_be32(ticket->t_tid);
-		xlog_write_iovec(iclog, log_offset, reg->i_addr,
-				reg->i_len, len, record_cnt, data_cnt);
+		ophdr->oh_tid = cpu_to_be32(data->ticket->t_tid);
+		xlog_write_iovec(data, reg->i_addr, reg->i_len);
 	}
 }
 
 static int
 xlog_write_get_more_iclog_space(
-	struct xlog_ticket	*ticket,
-	struct xlog_in_core	**iclogp,
-	uint32_t		*log_offset,
-	uint32_t		len,
-	uint32_t		*record_cnt,
-	uint32_t		*data_cnt)
+	struct xlog_write_data	*data)
 {
-	struct xlog_in_core	*iclog = *iclogp;
-	struct xlog		*log = iclog->ic_log;
+	struct xlog		*log = data->iclog->ic_log;
 	int			error;
 
 	spin_lock(&log->l_icloglock);
-	ASSERT(iclog->ic_state == XLOG_STATE_WANT_SYNC);
-	xlog_state_finish_copy(log, iclog, *record_cnt, *data_cnt);
-	error = xlog_state_release_iclog(log, iclog, ticket);
+	ASSERT(data->iclog->ic_state == XLOG_STATE_WANT_SYNC);
+	xlog_state_finish_copy(log, data->iclog, data->record_cnt,
+			data->data_cnt);
+	error = xlog_state_release_iclog(log, data->iclog, data->ticket);
 	spin_unlock(&log->l_icloglock);
 	if (error)
 		return error;
 
-	error = xlog_state_get_iclog_space(log, len, &iclog, ticket,
-					log_offset);
+	error = xlog_state_get_iclog_space(log, data);
 	if (error)
 		return error;
-	*record_cnt = 0;
-	*data_cnt = 0;
-	*iclogp = iclog;
+	data->record_cnt = 0;
+	data->data_cnt = 0;
 	return 0;
 }
 
@@ -2014,14 +1970,8 @@ xlog_write_get_more_iclog_space(
 static int
 xlog_write_partial(
 	struct xfs_log_vec	*lv,
-	struct xlog_ticket	*ticket,
-	struct xlog_in_core	**iclogp,
-	uint32_t		*log_offset,
-	uint32_t		*len,
-	uint32_t		*record_cnt,
-	uint32_t		*data_cnt)
+	struct xlog_write_data	*data)
 {
-	struct xlog_in_core	*iclog = *iclogp;
 	struct xlog_op_header	*ophdr;
 	int			index = 0;
 	uint32_t		rlen;
@@ -2043,25 +1993,22 @@ xlog_write_partial(
 		 * Hence if there isn't space for region data after the
 		 * opheader, then we need to start afresh with a new iclog.
 		 */
-		if (iclog->ic_size - *log_offset <=
+		if (xlog_write_space_left(data) <=
 					sizeof(struct xlog_op_header)) {
-			error = xlog_write_get_more_iclog_space(ticket,
-					&iclog, log_offset, *len, record_cnt,
-					data_cnt);
+			error = xlog_write_get_more_iclog_space(data);
 			if (error)
 				return error;
 		}
 
 		ophdr = reg->i_addr;
-		rlen = min_t(uint32_t, reg->i_len, iclog->ic_size - *log_offset);
+		rlen = min_t(uint32_t, reg->i_len, xlog_write_space_left(data));
 
-		ophdr->oh_tid = cpu_to_be32(ticket->t_tid);
+		ophdr->oh_tid = cpu_to_be32(data->ticket->t_tid);
 		ophdr->oh_len = cpu_to_be32(rlen - sizeof(struct xlog_op_header));
 		if (rlen != reg->i_len)
 			ophdr->oh_flags |= XLOG_CONTINUE_TRANS;
 
-		xlog_write_iovec(iclog, log_offset, reg->i_addr,
-				rlen, len, record_cnt, data_cnt);
+		xlog_write_iovec(data, reg->i_addr, rlen);
 
 		/* If we wrote the whole region, move to the next. */
 		if (rlen == reg->i_len)
@@ -2096,22 +2043,20 @@ xlog_write_partial(
 			 * consumes hasn't been accounted to the lv we are
 			 * writing.
 			 */
-			error = xlog_write_get_more_iclog_space(ticket,
-					&iclog, log_offset,
-					*len + sizeof(struct xlog_op_header),
-					record_cnt, data_cnt);
+			data->bytes_left += sizeof(struct xlog_op_header);
+			error = xlog_write_get_more_iclog_space(data);
 			if (error)
 				return error;
 
-			ophdr = iclog->ic_datap + *log_offset;
-			ophdr->oh_tid = cpu_to_be32(ticket->t_tid);
+			ophdr = xlog_write_space_advance(data,
+					sizeof(struct xlog_op_header));
+			ophdr->oh_tid = cpu_to_be32(data->ticket->t_tid);
 			ophdr->oh_clientid = XFS_TRANSACTION;
 			ophdr->oh_res2 = 0;
 			ophdr->oh_flags = XLOG_WAS_CONT_TRANS;
 
-			ticket->t_curr_res -= sizeof(struct xlog_op_header);
-			*log_offset += sizeof(struct xlog_op_header);
-			*data_cnt += sizeof(struct xlog_op_header);
+			data->ticket->t_curr_res -=
+				sizeof(struct xlog_op_header);
 
 			/*
 			 * If rlen fits in the iclog, then end the region
@@ -2119,26 +2064,19 @@ xlog_write_partial(
 			 */
 			reg_offset += rlen;
 			rlen = reg->i_len - reg_offset;
-			if (rlen <= iclog->ic_size - *log_offset)
+			if (rlen <= xlog_write_space_left(data))
 				ophdr->oh_flags |= XLOG_END_TRANS;
 			else
 				ophdr->oh_flags |= XLOG_CONTINUE_TRANS;
 
-			rlen = min_t(uint32_t, rlen, iclog->ic_size - *log_offset);
+			rlen = min_t(uint32_t, rlen,
+					xlog_write_space_left(data));
 			ophdr->oh_len = cpu_to_be32(rlen);
 
-			xlog_write_iovec(iclog, log_offset,
-					reg->i_addr + reg_offset,
-					rlen, len, record_cnt, data_cnt);
-
+			xlog_write_iovec(data, reg->i_addr + reg_offset, rlen);
 		} while (ophdr->oh_flags & XLOG_CONTINUE_TRANS);
 	}
 
-	/*
-	 * No more iovecs remain in this logvec so return the next log vec to
-	 * the caller so it can go back to fast path copying.
-	 */
-	*iclogp = iclog;
 	return 0;
 }
 
@@ -2191,12 +2129,12 @@ xlog_write(
 	uint32_t		len)
 
 {
-	struct xlog_in_core	*iclog = NULL;
 	struct xfs_log_vec	*lv;
-	uint32_t		record_cnt = 0;
-	uint32_t		data_cnt = 0;
-	int			error = 0;
-	int			log_offset;
+	struct xlog_write_data	data = {
+		.ticket		= ticket,
+		.bytes_left	= len,
+	};
+	int			error;
 
 	if (ticket->t_curr_res < 0) {
 		xfs_alert_tag(log->l_mp, XFS_PTAG_LOGRES,
@@ -2205,12 +2143,11 @@ xlog_write(
 		xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR);
 	}
 
-	error = xlog_state_get_iclog_space(log, len, &iclog, ticket,
-					   &log_offset);
+	error = xlog_state_get_iclog_space(log, &data);
 	if (error)
 		return error;
 
-	ASSERT(log_offset <= iclog->ic_size - 1);
+	ASSERT(xlog_write_space_left(&data) > 0);
 
 	/*
 	 * If we have a context pointer, pass it the first iclog we are
@@ -2218,7 +2155,7 @@ xlog_write(
 	 * ordering.
 	 */
 	if (ctx)
-		xlog_cil_set_ctx_write_state(ctx, iclog);
+		xlog_cil_set_ctx_write_state(ctx, data.iclog);
 
 	list_for_each_entry(lv, lv_chain, lv_list) {
 		/*
@@ -2226,10 +2163,8 @@ xlog_write(
 		 * the partial copy loop which can handle this case.
 		 */
 		if (lv->lv_niovecs &&
-		    lv->lv_bytes > iclog->ic_size - log_offset) {
-			error = xlog_write_partial(lv, ticket, &iclog,
-					&log_offset, &len, &record_cnt,
-					&data_cnt);
+		    lv->lv_bytes > xlog_write_space_left(&data)) {
+			error = xlog_write_partial(lv, &data);
 			if (error) {
 				/*
 				 * We have no iclog to release, so just return
@@ -2238,11 +2173,10 @@ xlog_write(
 				return error;
 			}
 		} else {
-			xlog_write_full(lv, ticket, iclog, &log_offset,
-					 &len, &record_cnt, &data_cnt);
+			xlog_write_full(lv, &data);
 		}
 	}
-	ASSERT(len == 0);
+	ASSERT(data.bytes_left == 0);
 
 	/*
 	 * We've already been guaranteed that the last writes will fit inside
@@ -2251,8 +2185,8 @@ xlog_write(
 	 * iclog with the number of bytes written here.
 	 */
 	spin_lock(&log->l_icloglock);
-	xlog_state_finish_copy(log, iclog, record_cnt, 0);
-	error = xlog_state_release_iclog(log, iclog, ticket);
+	xlog_state_finish_copy(log, data.iclog, data.record_cnt, 0);
+	error = xlog_state_release_iclog(log, data.iclog, ticket);
 	spin_unlock(&log->l_icloglock);
 
 	return error;
@@ -2574,10 +2508,7 @@ xlog_state_done_syncing(
 STATIC int
 xlog_state_get_iclog_space(
 	struct xlog		*log,
-	int			len,
-	struct xlog_in_core	**iclogp,
-	struct xlog_ticket	*ticket,
-	int			*logoffsetp)
+	struct xlog_write_data	*data)
 {
 	int			log_offset;
 	struct xlog_rec_header	*head;
@@ -2612,7 +2543,7 @@ xlog_state_get_iclog_space(
 	 * must be written.
 	 */
 	if (log_offset == 0) {
-		ticket->t_curr_res -= log->l_iclog_hsize;
+		data->ticket->t_curr_res -= log->l_iclog_hsize;
 		head->h_cycle = cpu_to_be32(log->l_curr_cycle);
 		head->h_lsn = cpu_to_be64(
 			xlog_assign_lsn(log->l_curr_cycle, log->l_curr_block));
@@ -2642,7 +2573,8 @@ xlog_state_get_iclog_space(
 		 * reference to the iclog.
 		 */
 		if (!atomic_add_unless(&iclog->ic_refcnt, -1, 1))
-			error = xlog_state_release_iclog(log, iclog, ticket);
+			error = xlog_state_release_iclog(log, iclog,
+					data->ticket);
 		spin_unlock(&log->l_icloglock);
 		if (error)
 			return error;
@@ -2655,16 +2587,16 @@ xlog_state_get_iclog_space(
 	 * iclogs (to mark it taken), this particular iclog will release/sync
 	 * to disk in xlog_write().
 	 */
-	if (len <= iclog->ic_size - iclog->ic_offset)
-		iclog->ic_offset += len;
+	if (data->bytes_left <= iclog->ic_size - iclog->ic_offset)
+		iclog->ic_offset += data->bytes_left;
 	else
 		xlog_state_switch_iclogs(log, iclog, iclog->ic_size);
-	*iclogp = iclog;
+	data->iclog = iclog;
 
 	ASSERT(iclog->ic_offset <= iclog->ic_size);
 	spin_unlock(&log->l_icloglock);
 
-	*logoffsetp = log_offset;
+	data->log_offset = log_offset;
 	return 0;
 }
 
diff --git a/fs/xfs/xfs_log.h b/fs/xfs/xfs_log.h
index dcc1f44ed68f..0f23812b0b31 100644
--- a/fs/xfs/xfs_log.h
+++ b/fs/xfs/xfs_log.h
@@ -6,20 +6,9 @@
 #ifndef	__XFS_LOG_H__
 #define __XFS_LOG_H__
 
+struct xlog_format_buf;
 struct xfs_cil_ctx;
 
-struct xfs_log_vec {
-	struct list_head	lv_list;	/* CIL lv chain ptrs */
-	uint32_t		lv_order_id;	/* chain ordering info */
-	int			lv_niovecs;	/* number of iovecs in lv */
-	struct xfs_log_iovec	*lv_iovecp;	/* iovec array */
-	struct xfs_log_item	*lv_item;	/* owner */
-	char			*lv_buf;	/* formatted buffer */
-	int			lv_bytes;	/* accounted space in buffer */
-	int			lv_buf_used;	/* buffer space used so far */
-	int			lv_alloc_size;	/* size of allocated lv */
-};
-
 /* Region types for iovec's i_type */
 #define XLOG_REG_TYPE_BFORMAT		1
 #define XLOG_REG_TYPE_BCHUNK		2
@@ -70,58 +59,24 @@ xlog_calc_iovec_len(int len)
 	return roundup(len, sizeof(uint32_t));
 }
 
-void *xlog_prepare_iovec(struct xfs_log_vec *lv, struct xfs_log_iovec **vecp,
-		uint type);
-
-static inline void
-xlog_finish_iovec(struct xfs_log_vec *lv, struct xfs_log_iovec *vec,
-		int data_len)
-{
-	struct xlog_op_header	*oph = vec->i_addr;
-	int			len;
-
-	/*
-	 * Always round up the length to the correct alignment so callers don't
-	 * need to know anything about this log vec layout requirement. This
-	 * means we have to zero the area the data to be written does not cover.
-	 * This is complicated by fact the payload region is offset into the
-	 * logvec region by the opheader that tracks the payload.
-	 */
-	len = xlog_calc_iovec_len(data_len);
-	if (len - data_len != 0) {
-		char	*buf = vec->i_addr + sizeof(struct xlog_op_header);
-
-		memset(buf + data_len, 0, len - data_len);
-	}
-
-	/*
-	 * The opheader tracks aligned payload length, whilst the logvec tracks
-	 * the overall region length.
-	 */
-	oph->oh_len = cpu_to_be32(len);
-
-	len += sizeof(struct xlog_op_header);
-	lv->lv_buf_used += len;
-	lv->lv_bytes += len;
-	vec->i_len = len;
-
-	/* Catch buffer overruns */
-	ASSERT((void *)lv->lv_buf + lv->lv_bytes <=
-		(void *)lv + lv->lv_alloc_size);
-}
+void *xlog_format_start(struct xlog_format_buf *lfb, uint16_t type);
+void xlog_format_commit(struct xlog_format_buf *lfb, unsigned int data_len);
 
 /*
  * Copy the amount of data requested by the caller into a new log iovec.
  */
 static inline void *
-xlog_copy_iovec(struct xfs_log_vec *lv, struct xfs_log_iovec **vecp,
-		uint type, void *data, int len)
+xlog_format_copy(
+	struct xlog_format_buf	*lfb,
+	uint16_t		type,
+	void			*data,
+	unsigned int		len)
 {
 	void *buf;
 
-	buf = xlog_prepare_iovec(lv, vecp, type);
+	buf = xlog_format_start(lfb, type);
 	memcpy(buf, data, len);
-	xlog_finish_iovec(lv, *vecp, len);
+	xlog_format_commit(lfb, len);
 	return buf;
 }
 
diff --git a/fs/xfs/xfs_log_cil.c b/fs/xfs/xfs_log_cil.c
index 778ac47adb8c..bc25012ac5c0 100644
--- a/fs/xfs/xfs_log_cil.c
+++ b/fs/xfs/xfs_log_cil.c
@@ -409,6 +409,102 @@ xfs_cil_prepare_item(
 		lv->lv_item->li_seq = log->l_cilp->xc_ctx->sequence;
 }
 
+struct xlog_format_buf {
+	struct xfs_log_vec	*lv;
+	unsigned int		idx;
+};
+
+/*
+ * We need to make sure the buffer pointer returned is naturally aligned for the
+ * biggest basic data type we put into it. We have already accounted for this
+ * padding when sizing the buffer.
+ *
+ * However, this padding does not get written into the log, and hence we have to
+ * track the space used by the log vectors separately to prevent log space hangs
+ * due to inaccurate accounting (i.e. a leak) of the used log space through the
+ * CIL context ticket.
+ *
+ * We also add space for the xlog_op_header that describes this region in the
+ * log. This prepends the data region we return to the caller to copy their data
+ * into, so do all the static initialisation of the ophdr now. Because the ophdr
+ * is not 8 byte aligned, we have to be careful to ensure that we align the
+ * start of the buffer such that the region we return to the call is 8 byte
+ * aligned and packed against the tail of the ophdr.
+ */
+void *
+xlog_format_start(
+	struct xlog_format_buf	*lfb,
+	uint16_t		type)
+{
+	struct xfs_log_vec	*lv = lfb->lv;
+	struct xfs_log_iovec	*vec = &lv->lv_iovecp[lfb->idx];
+	struct xlog_op_header	*oph;
+	uint32_t		len;
+	void			*buf;
+
+	ASSERT(lfb->idx < lv->lv_niovecs);
+
+	len = lv->lv_buf_used + sizeof(struct xlog_op_header);
+	if (!IS_ALIGNED(len, sizeof(uint64_t))) {
+		lv->lv_buf_used = round_up(len, sizeof(uint64_t)) -
+					sizeof(struct xlog_op_header);
+	}
+
+	vec->i_type = type;
+	vec->i_addr = lv->lv_buf + lv->lv_buf_used;
+
+	oph = vec->i_addr;
+	oph->oh_clientid = XFS_TRANSACTION;
+	oph->oh_res2 = 0;
+	oph->oh_flags = 0;
+
+	buf = vec->i_addr + sizeof(struct xlog_op_header);
+	ASSERT(IS_ALIGNED((unsigned long)buf, sizeof(uint64_t)));
+	return buf;
+}
+
+void
+xlog_format_commit(
+	struct xlog_format_buf	*lfb,
+	unsigned int		data_len)
+{
+	struct xfs_log_vec	*lv = lfb->lv;
+	struct xfs_log_iovec	*vec = &lv->lv_iovecp[lfb->idx];
+	struct xlog_op_header	*oph = vec->i_addr;
+	int			len;
+
+	/*
+	 * Always round up the length to the correct alignment so callers don't
+	 * need to know anything about this log vec layout requirement. This
+	 * means we have to zero the area the data to be written does not cover.
+	 * This is complicated by fact the payload region is offset into the
+	 * logvec region by the opheader that tracks the payload.
+	 */
+	len = xlog_calc_iovec_len(data_len);
+	if (len - data_len != 0) {
+		char	*buf = vec->i_addr + sizeof(struct xlog_op_header);
+
+		memset(buf + data_len, 0, len - data_len);
+	}
+
+	/*
+	 * The opheader tracks aligned payload length, whilst the logvec tracks
+	 * the overall region length.
+	 */
+	oph->oh_len = cpu_to_be32(len);
+
+	len += sizeof(struct xlog_op_header);
+	lv->lv_buf_used += len;
+	lv->lv_bytes += len;
+	vec->i_len = len;
+
+	/* Catch buffer overruns */
+	ASSERT((void *)lv->lv_buf + lv->lv_bytes <=
+		(void *)lv + lv->lv_alloc_size);
+
+	lfb->idx++;
+}
+
 /*
  * Format log item into a flat buffers
  *
@@ -454,6 +550,7 @@ xlog_cil_insert_format_items(
 	list_for_each_entry(lip, &tp->t_items, li_trans) {
 		struct xfs_log_vec *lv = lip->li_lv;
 		struct xfs_log_vec *shadow = lip->li_lv_shadow;
+		struct xlog_format_buf lfb = { };
 
 		/* Skip items which aren't dirty in this transaction. */
 		if (!test_bit(XFS_LI_DIRTY, &lip->li_flags))
@@ -501,8 +598,9 @@ xlog_cil_insert_format_items(
 			lv->lv_item = lip;
 		}
 
+		lfb.lv = lv;
 		ASSERT(IS_ALIGNED((unsigned long)lv->lv_buf, sizeof(uint64_t)));
-		lip->li_ops->iop_format(lip, lv);
+		lip->li_ops->iop_format(lip, &lfb);
 		xfs_cil_prepare_item(log, lip, lv, diff_len);
 	}
 }
@@ -1098,13 +1196,7 @@ xlog_cil_write_commit_record(
 		.i_len = sizeof(struct xlog_op_header),
 		.i_type = XLOG_REG_TYPE_COMMIT,
 	};
-	struct xfs_log_vec	vec = {
-		.lv_niovecs = 1,
-		.lv_iovecp = &reg,
-	};
 	int			error;
-	LIST_HEAD(lv_chain);
-	list_add(&vec.lv_list, &lv_chain);
 
 	if (xlog_is_shutdown(log))
 		return -EIO;
@@ -1112,10 +1204,7 @@ xlog_cil_write_commit_record(
 	error = xlog_cil_order_write(ctx->cil, ctx->sequence, _COMMIT_RECORD);
 	if (error)
 		return error;
-
-	/* account for space used by record data */
-	ctx->ticket->t_curr_res -= reg.i_len;
-	error = xlog_write(log, ctx, &lv_chain, ctx->ticket, reg.i_len);
+	error = xlog_write_one_vec(log, ctx, &reg, ctx->ticket);
 	if (error)
 		xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR);
 	return error;
diff --git a/fs/xfs/xfs_log_priv.h b/fs/xfs/xfs_log_priv.h
index 0fe59f0525aa..cf1e4ce61a8c 100644
--- a/fs/xfs/xfs_log_priv.h
+++ b/fs/xfs/xfs_log_priv.h
@@ -13,6 +13,24 @@ struct xlog;
 struct xlog_ticket;
 struct xfs_mount;
 
+struct xfs_log_iovec {
+	void			*i_addr;/* beginning address of region */
+	int			i_len;	/* length in bytes of region */
+	uint			i_type;	/* type of region */
+};
+
+struct xfs_log_vec {
+	struct list_head	lv_list;	/* CIL lv chain ptrs */
+	uint32_t		lv_order_id;	/* chain ordering info */
+	int			lv_niovecs;	/* number of iovecs in lv */
+	struct xfs_log_iovec	*lv_iovecp;	/* iovec array */
+	struct xfs_log_item	*lv_item;	/* owner */
+	char			*lv_buf;	/* formatted buffer */
+	int			lv_bytes;	/* accounted space in buffer */
+	int			lv_buf_used;	/* buffer space used so far */
+	int			lv_alloc_size;	/* size of allocated lv */
+};
+
 /*
  * get client id from packed copy.
  *
@@ -507,6 +525,8 @@ void	xlog_print_trans(struct xfs_trans *);
 int	xlog_write(struct xlog *log, struct xfs_cil_ctx *ctx,
 		struct list_head *lv_chain, struct xlog_ticket *tic,
 		uint32_t len);
+int	xlog_write_one_vec(struct xlog *log, struct xfs_cil_ctx *ctx,
+		struct xfs_log_iovec *reg, struct xlog_ticket *ticket);
 void	xfs_log_ticket_ungrant(struct xlog *log, struct xlog_ticket *ticket);
 void	xfs_log_ticket_regrant(struct xlog *log, struct xlog_ticket *ticket);
 
diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c
index 3728234699a2..a41f5b577e22 100644
--- a/fs/xfs/xfs_refcount_item.c
+++ b/fs/xfs/xfs_refcount_item.c
@@ -93,10 +93,9 @@ unsigned int xfs_cui_log_space(unsigned int nr)
 STATIC void
 xfs_cui_item_format(
 	struct xfs_log_item	*lip,
-	struct xfs_log_vec	*lv)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_cui_log_item	*cuip = CUI_ITEM(lip);
-	struct xfs_log_iovec	*vecp = NULL;
 
 	ASSERT(atomic_read(&cuip->cui_next_extent) ==
 			cuip->cui_format.cui_nextents);
@@ -105,7 +104,7 @@ xfs_cui_item_format(
 	cuip->cui_format.cui_type = lip->li_type;
 	cuip->cui_format.cui_size = 1;
 
-	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_CUI_FORMAT, &cuip->cui_format,
+	xlog_format_copy(lfb, XLOG_REG_TYPE_CUI_FORMAT, &cuip->cui_format,
 			xfs_cui_log_format_sizeof(cuip->cui_format.cui_nextents));
 }
 
@@ -199,17 +198,16 @@ unsigned int xfs_cud_log_space(void)
 STATIC void
 xfs_cud_item_format(
 	struct xfs_log_item	*lip,
-	struct xfs_log_vec	*lv)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_cud_log_item	*cudp = CUD_ITEM(lip);
-	struct xfs_log_iovec	*vecp = NULL;
 
 	ASSERT(lip->li_type == XFS_LI_CUD || lip->li_type == XFS_LI_CUD_RT);
 
 	cudp->cud_format.cud_type = lip->li_type;
 	cudp->cud_format.cud_size = 1;
 
-	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_CUD_FORMAT, &cudp->cud_format,
+	xlog_format_copy(lfb, XLOG_REG_TYPE_CUD_FORMAT, &cudp->cud_format,
 			sizeof(struct xfs_cud_log_format));
 }
 
diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c
index 15f0903f6fd4..8bf04b101156 100644
--- a/fs/xfs/xfs_rmap_item.c
+++ b/fs/xfs/xfs_rmap_item.c
@@ -92,10 +92,9 @@ unsigned int xfs_rui_log_space(unsigned int nr)
 STATIC void
 xfs_rui_item_format(
 	struct xfs_log_item	*lip,
-	struct xfs_log_vec	*lv)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_rui_log_item	*ruip = RUI_ITEM(lip);
-	struct xfs_log_iovec	*vecp = NULL;
 
 	ASSERT(atomic_read(&ruip->rui_next_extent) ==
 			ruip->rui_format.rui_nextents);
@@ -105,7 +104,7 @@ xfs_rui_item_format(
 	ruip->rui_format.rui_type = lip->li_type;
 	ruip->rui_format.rui_size = 1;
 
-	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_RUI_FORMAT, &ruip->rui_format,
+	xlog_format_copy(lfb, XLOG_REG_TYPE_RUI_FORMAT, &ruip->rui_format,
 			xfs_rui_log_format_sizeof(ruip->rui_format.rui_nextents));
 }
 
@@ -200,17 +199,16 @@ unsigned int xfs_rud_log_space(void)
 STATIC void
 xfs_rud_item_format(
 	struct xfs_log_item	*lip,
-	struct xfs_log_vec	*lv)
+	struct xlog_format_buf	*lfb)
 {
 	struct xfs_rud_log_item	*rudp = RUD_ITEM(lip);
-	struct xfs_log_iovec	*vecp = NULL;
 
 	ASSERT(lip->li_type == XFS_LI_RUD || lip->li_type == XFS_LI_RUD_RT);
 
 	rudp->rud_format.rud_type = lip->li_type;
 	rudp->rud_format.rud_size = 1;
 
-	xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_RUD_FORMAT, &rudp->rud_format,
+	xlog_format_copy(lfb, XLOG_REG_TYPE_RUD_FORMAT, &rudp->rud_format,
 			sizeof(struct xfs_rud_log_format));
 }
 
diff --git a/fs/xfs/xfs_rtalloc.c b/fs/xfs/xfs_rtalloc.c
index 6907e871fa15..e063f4f2f2e6 100644
--- a/fs/xfs/xfs_rtalloc.c
+++ b/fs/xfs/xfs_rtalloc.c
@@ -1255,12 +1255,10 @@ xfs_growfs_check_rtgeom(
 	min_logfsbs = min_t(xfs_extlen_t, xfs_log_calc_minimum_size(nmp),
 			nmp->m_rsumblocks * 2);
 
-	kfree(nmp);
-
 	trace_xfs_growfs_check_rtgeom(mp, min_logfsbs);
 
 	if (min_logfsbs > mp->m_sb.sb_logblocks)
-		return -EINVAL;
+		goto out_inval;
 
 	if (xfs_has_zoned(mp)) {
 		uint32_t	gblocks = mp->m_groups[XG_TYPE_RTG].blocks;
@@ -1268,16 +1266,20 @@ xfs_growfs_check_rtgeom(
 
 		if (rextsize != 1)
 			return -EINVAL;
-		div_u64_rem(mp->m_sb.sb_rblocks, gblocks, &rem);
+		div_u64_rem(nmp->m_sb.sb_rblocks, gblocks, &rem);
 		if (rem) {
 			xfs_warn(mp,
 "new RT volume size (%lld) not aligned to RT group size (%d)",
-				mp->m_sb.sb_rblocks, gblocks);
-			return -EINVAL;
+				nmp->m_sb.sb_rblocks, gblocks);
+			goto out_inval;
 		}
 	}
 
+	kfree(nmp);
 	return 0;
+out_inval:
+	kfree(nmp);
+	return -EINVAL;
 }
 
 /*
diff --git a/fs/xfs/xfs_trans.h b/fs/xfs/xfs_trans.h
index 7fb860f645a3..8830600b3e72 100644
--- a/fs/xfs/xfs_trans.h
+++ b/fs/xfs/xfs_trans.h
@@ -9,6 +9,7 @@
 /* kernel only transaction subsystem defines */
 
 struct xlog;
+struct xlog_format_buf;
 struct xfs_buf;
 struct xfs_buftarg;
 struct xfs_efd_log_item;
@@ -70,7 +71,8 @@ struct xfs_log_item {
 struct xfs_item_ops {
 	unsigned flags;
 	void (*iop_size)(struct xfs_log_item *, int *, int *);
-	void (*iop_format)(struct xfs_log_item *, struct xfs_log_vec *);
+	void (*iop_format)(struct xfs_log_item *lip,
+			struct xlog_format_buf *lfb);
 	void (*iop_pin)(struct xfs_log_item *);
 	void (*iop_unpin)(struct xfs_log_item *, int remove);
 	uint64_t (*iop_sort)(struct xfs_log_item *lip);
diff --git a/include/acpi/pcc.h b/include/acpi/pcc.h
index 9af3b502f839..840bfc95bae3 100644
--- a/include/acpi/pcc.h
+++ b/include/acpi/pcc.h
@@ -17,35 +17,6 @@ struct pcc_mbox_chan {
 	u32 latency;
 	u32 max_access_rate;
 	u16 min_turnaround_time;
-
-	/* Set to true to indicate that the mailbox should manage
-	 * writing the dat to the shared buffer. This differs from
-	 * the case where the drivesr are writing to the buffer and
-	 * using send_data only to  ring the doorbell.  If this flag
-	 * is set, then the void * data parameter of send_data must
-	 * point to a kernel-memory buffer formatted in accordance with
-	 * the PCC specification.
-	 *
-	 * The active buffer management will include reading the
-	 * notify_on_completion flag, and will then
-	 * call mbox_chan_txdone when the acknowledgment interrupt is
-	 * received.
-	 */
-	bool manage_writes;
-
-	/* Optional callback that allows the driver
-	 * to allocate the memory used for receiving
-	 * messages.  The return value is the location
-	 * inside the buffer where the mailbox should write the data.
-	 */
-	void *(*rx_alloc)(struct mbox_client *cl,  int size);
-};
-
-struct pcc_header {
-	u32 signature;
-	u32 flags;
-	u32 length;
-	u32 command;
 };
 
 /* Generic Communications Channel Shared Memory Region */
diff --git a/include/asm-generic/atomic64.h b/include/asm-generic/atomic64.h
index 100d24b02e52..f22ccfc0df98 100644
--- a/include/asm-generic/atomic64.h
+++ b/include/asm-generic/atomic64.h
@@ -10,7 +10,7 @@
 #include <linux/types.h>
 
 typedef struct {
-	s64 counter;
+	s64 __aligned(sizeof(s64)) counter;
 } atomic64_t;
 
 #define ATOMIC64_INIT(i)	{ (i) }
diff --git a/include/asm-generic/audit_change_attr.h b/include/asm-generic/audit_change_attr.h
index cc840537885f..ddd90bbe40df 100644
--- a/include/asm-generic/audit_change_attr.h
+++ b/include/asm-generic/audit_change_attr.h
@@ -26,6 +26,9 @@ __NR_fremovexattr,
 __NR_fchownat,
 __NR_fchmodat,
 #endif
+#ifdef __NR_fchmodat2
+__NR_fchmodat2,
+#endif
 #ifdef __NR_chown32
 __NR_chown32,
 __NR_fchown32,
diff --git a/include/asm-generic/rqspinlock.h b/include/asm-generic/rqspinlock.h
index 0f2dcbbfee2f..dd36ac96bf66 100644
--- a/include/asm-generic/rqspinlock.h
+++ b/include/asm-generic/rqspinlock.h
@@ -28,7 +28,7 @@ struct rqspinlock {
  */
 struct bpf_res_spin_lock {
 	u32 val;
-};
+} __aligned(__alignof__(struct rqspinlock));
 
 struct qspinlock;
 #ifdef CONFIG_QUEUED_SPINLOCKS
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index 1fff717cae51..706416babb3d 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -364,6 +364,20 @@ struct mmu_gather {
 	unsigned int		vma_huge : 1;
 	unsigned int		vma_pfn  : 1;
 
+	/*
+	 * Did we unshare (unmap) any shared page tables? For now only
+	 * used for hugetlb PMD table sharing.
+	 */
+	unsigned int		unshared_tables : 1;
+
+	/*
+	 * Did we unshare any page tables such that they are now exclusive
+	 * and could get reused+modified by the new owner? When setting this
+	 * flag, "unshared_tables" will be set as well. For now only used
+	 * for hugetlb PMD table sharing.
+	 */
+	unsigned int		fully_unshared_tables : 1;
+
 	unsigned int		batch_count;
 
 #ifndef CONFIG_MMU_GATHER_NO_GATHER
@@ -400,6 +414,7 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb)
 	tlb->cleared_pmds = 0;
 	tlb->cleared_puds = 0;
 	tlb->cleared_p4ds = 0;
+	tlb->unshared_tables = 0;
 	/*
 	 * Do not reset mmu_gather::vma_* fields here, we do not
 	 * call into tlb_start_vma() again to set them if there is an
@@ -484,7 +499,7 @@ static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
 	 * these bits.
 	 */
 	if (!(tlb->freed_tables || tlb->cleared_ptes || tlb->cleared_pmds ||
-	      tlb->cleared_puds || tlb->cleared_p4ds))
+	      tlb->cleared_puds || tlb->cleared_p4ds || tlb->unshared_tables))
 		return;
 
 	tlb_flush(tlb);
@@ -773,6 +788,63 @@ static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd)
 }
 #endif
 
+#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
+static inline void tlb_unshare_pmd_ptdesc(struct mmu_gather *tlb, struct ptdesc *pt,
+					  unsigned long addr)
+{
+	/*
+	 * The caller must make sure that concurrent unsharing + exclusive
+	 * reuse is impossible until tlb_flush_unshared_tables() was called.
+	 */
+	VM_WARN_ON_ONCE(!ptdesc_pmd_is_shared(pt));
+	ptdesc_pmd_pts_dec(pt);
+
+	/* Clearing a PUD pointing at a PMD table with PMD leaves. */
+	tlb_flush_pmd_range(tlb, addr & PUD_MASK, PUD_SIZE);
+
+	/*
+	 * If the page table is now exclusively owned, we fully unshared
+	 * a page table.
+	 */
+	if (!ptdesc_pmd_is_shared(pt))
+		tlb->fully_unshared_tables = true;
+	tlb->unshared_tables = true;
+}
+
+static inline void tlb_flush_unshared_tables(struct mmu_gather *tlb)
+{
+	/*
+	 * As soon as the caller drops locks to allow for reuse of
+	 * previously-shared tables, these tables could get modified and
+	 * even reused outside of hugetlb context, so we have to make sure that
+	 * any page table walkers (incl. TLB, GUP-fast) are aware of that
+	 * change.
+	 *
+	 * Even if we are not fully unsharing a PMD table, we must
+	 * flush the TLB for the unsharer now.
+	 */
+	if (tlb->unshared_tables)
+		tlb_flush_mmu_tlbonly(tlb);
+
+	/*
+	 * Similarly, we must make sure that concurrent GUP-fast will not
+	 * walk previously-shared page tables that are getting modified+reused
+	 * elsewhere. So broadcast an IPI to wait for any concurrent GUP-fast.
+	 *
+	 * We only perform this when we are the last sharer of a page table,
+	 * as the IPI will reach all CPUs: any GUP-fast.
+	 *
+	 * Note that on configs where tlb_remove_table_sync_one() is a NOP,
+	 * the expectation is that the tlb_flush_mmu_tlbonly() would have issued
+	 * required IPIs already for us.
+	 */
+	if (tlb->fully_unshared_tables) {
+		tlb_remove_table_sync_one();
+		tlb->fully_unshared_tables = false;
+	}
+}
+#endif /* CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING */
+
 #endif /* CONFIG_MMU */
 
 #endif /* _ASM_GENERIC__TLB_H */
diff --git a/include/crypto/mldsa.h b/include/crypto/mldsa.h
new file mode 100644
index 000000000000..cf30aef29970
--- /dev/null
+++ b/include/crypto/mldsa.h
@@ -0,0 +1,60 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Support for verifying ML-DSA signatures
+ *
+ * Copyright 2025 Google LLC
+ */
+#ifndef _CRYPTO_MLDSA_H
+#define _CRYPTO_MLDSA_H
+
+#include <linux/types.h>
+
+/* Identifier for an ML-DSA parameter set */
+enum mldsa_alg {
+	MLDSA44, /* ML-DSA-44 */
+	MLDSA65, /* ML-DSA-65 */
+	MLDSA87, /* ML-DSA-87 */
+};
+
+/* Lengths of ML-DSA public keys and signatures in bytes */
+#define MLDSA44_PUBLIC_KEY_SIZE 1312
+#define MLDSA65_PUBLIC_KEY_SIZE 1952
+#define MLDSA87_PUBLIC_KEY_SIZE 2592
+#define MLDSA44_SIGNATURE_SIZE 2420
+#define MLDSA65_SIGNATURE_SIZE 3309
+#define MLDSA87_SIGNATURE_SIZE 4627
+
+/**
+ * mldsa_verify() - Verify an ML-DSA signature
+ * @alg: The ML-DSA parameter set to use
+ * @sig: The signature
+ * @sig_len: Length of the signature in bytes.  Should match the
+ *	     MLDSA*_SIGNATURE_SIZE constant associated with @alg,
+ *	     otherwise -EBADMSG will be returned.
+ * @msg: The message
+ * @msg_len: Length of the message in bytes
+ * @pk: The public key
+ * @pk_len: Length of the public key in bytes.  Should match the
+ *	    MLDSA*_PUBLIC_KEY_SIZE constant associated with @alg,
+ *	    otherwise -EBADMSG will be returned.
+ *
+ * This verifies a signature using pure ML-DSA with the specified parameter set.
+ * The context string is assumed to be empty.
+ *
+ * Context: Might sleep
+ *
+ * Return:
+ * * 0 if the signature is valid
+ * * -EBADMSG if the signature and/or public key is malformed
+ * * -EKEYREJECTED if the signature is invalid but otherwise well-formed
+ * * -ENOMEM if out of memory so the validity of the signature is unknown
+ */
+int mldsa_verify(enum mldsa_alg alg, const u8 *sig, size_t sig_len,
+		 const u8 *msg, size_t msg_len, const u8 *pk, size_t pk_len);
+
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST)
+/* Internal function, exposed only for unit testing */
+s32 mldsa_use_hint(u8 h, s32 r, s32 gamma2);
+#endif
+
+#endif /* _CRYPTO_MLDSA_H */
diff --git a/include/crypto/nh.h b/include/crypto/nh.h
new file mode 100644
index 000000000000..465e85bf203f
--- /dev/null
+++ b/include/crypto/nh.h
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * NH hash function for Adiantum
+ */
+
+#ifndef _CRYPTO_NH_H
+#define _CRYPTO_NH_H
+
+#include <linux/types.h>
+
+/* NH parameterization: */
+
+/* Endianness: little */
+/* Word size: 32 bits (works well on NEON, SSE2, AVX2) */
+
+/* Stride: 2 words (optimal on ARM32 NEON; works okay on other CPUs too) */
+#define NH_PAIR_STRIDE		2
+#define NH_MESSAGE_UNIT		(NH_PAIR_STRIDE * 2 * sizeof(u32))
+
+/* Num passes (Toeplitz iteration count): 4, to give ε = 2^{-128} */
+#define NH_NUM_PASSES		4
+#define NH_HASH_BYTES		(NH_NUM_PASSES * sizeof(u64))
+
+/* Max message size: 1024 bytes (32x compression factor) */
+#define NH_NUM_STRIDES		64
+#define NH_MESSAGE_WORDS	(NH_PAIR_STRIDE * 2 * NH_NUM_STRIDES)
+#define NH_MESSAGE_BYTES	(NH_MESSAGE_WORDS * sizeof(u32))
+#define NH_KEY_WORDS		(NH_MESSAGE_WORDS + \
+				 NH_PAIR_STRIDE * 2 * (NH_NUM_PASSES - 1))
+#define NH_KEY_BYTES		(NH_KEY_WORDS * sizeof(u32))
+
+/**
+ * nh() - NH hash function for Adiantum
+ * @key: The key.  @message_len + 48 bytes of it are used.  This is NH_KEY_BYTES
+ *	 if @message_len has its maximum length of NH_MESSAGE_BYTES.
+ * @message: The message
+ * @message_len: The message length in bytes.  Must be a multiple of 16
+ *		 (NH_MESSAGE_UNIT) and at most 1024 (NH_MESSAGE_BYTES).
+ * @hash: (output) The resulting hash value
+ *
+ * Note: the pseudocode for NH in the Adiantum paper iterates over 1024-byte
+ * segments of the message, computes a 32-byte hash for each, and returns all
+ * the hashes concatenated together.  In contrast, this function just hashes one
+ * segment and returns one hash.  It's the caller's responsibility to call this
+ * function for each 1024-byte segment and collect all the hashes.
+ *
+ * Context: Any context.
+ */
+void nh(const u32 *key, const u8 *message, size_t message_len,
+	__le64 hash[NH_NUM_PASSES]);
+
+#endif /* _CRYPTO_NH_H */
diff --git a/include/crypto/nhpoly1305.h b/include/crypto/nhpoly1305.h
deleted file mode 100644
index 306925fea190..000000000000
--- a/include/crypto/nhpoly1305.h
+++ /dev/null
@@ -1,74 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Common values and helper functions for the NHPoly1305 hash function.
- */
-
-#ifndef _NHPOLY1305_H
-#define _NHPOLY1305_H
-
-#include <crypto/hash.h>
-#include <crypto/internal/poly1305.h>
-
-/* NH parameterization: */
-
-/* Endianness: little */
-/* Word size: 32 bits (works well on NEON, SSE2, AVX2) */
-
-/* Stride: 2 words (optimal on ARM32 NEON; works okay on other CPUs too) */
-#define NH_PAIR_STRIDE		2
-#define NH_MESSAGE_UNIT		(NH_PAIR_STRIDE * 2 * sizeof(u32))
-
-/* Num passes (Toeplitz iteration count): 4, to give ε = 2^{-128} */
-#define NH_NUM_PASSES		4
-#define NH_HASH_BYTES		(NH_NUM_PASSES * sizeof(u64))
-
-/* Max message size: 1024 bytes (32x compression factor) */
-#define NH_NUM_STRIDES		64
-#define NH_MESSAGE_WORDS	(NH_PAIR_STRIDE * 2 * NH_NUM_STRIDES)
-#define NH_MESSAGE_BYTES	(NH_MESSAGE_WORDS * sizeof(u32))
-#define NH_KEY_WORDS		(NH_MESSAGE_WORDS + \
-				 NH_PAIR_STRIDE * 2 * (NH_NUM_PASSES - 1))
-#define NH_KEY_BYTES		(NH_KEY_WORDS * sizeof(u32))
-
-#define NHPOLY1305_KEY_SIZE	(POLY1305_BLOCK_SIZE + NH_KEY_BYTES)
-
-struct nhpoly1305_key {
-	struct poly1305_core_key poly_key;
-	u32 nh_key[NH_KEY_WORDS];
-};
-
-struct nhpoly1305_state {
-
-	/* Running total of polynomial evaluation */
-	struct poly1305_state poly_state;
-
-	/* Partial block buffer */
-	u8 buffer[NH_MESSAGE_UNIT];
-	unsigned int buflen;
-
-	/*
-	 * Number of bytes remaining until the current NH message reaches
-	 * NH_MESSAGE_BYTES.  When nonzero, 'nh_hash' holds the partial NH hash.
-	 */
-	unsigned int nh_remaining;
-
-	__le64 nh_hash[NH_NUM_PASSES];
-};
-
-typedef void (*nh_t)(const u32 *key, const u8 *message, size_t message_len,
-		     __le64 hash[NH_NUM_PASSES]);
-
-int crypto_nhpoly1305_setkey(struct crypto_shash *tfm,
-			     const u8 *key, unsigned int keylen);
-
-int crypto_nhpoly1305_init(struct shash_desc *desc);
-int crypto_nhpoly1305_update(struct shash_desc *desc,
-			     const u8 *src, unsigned int srclen);
-int crypto_nhpoly1305_update_helper(struct shash_desc *desc,
-				    const u8 *src, unsigned int srclen,
-				    nh_t nh_fn);
-int crypto_nhpoly1305_final(struct shash_desc *desc, u8 *dst);
-int crypto_nhpoly1305_final_helper(struct shash_desc *desc, u8 *dst,
-				   nh_t nh_fn);
-
-#endif /* _NHPOLY1305_H */
diff --git a/include/drm/drm_atomic.h b/include/drm/drm_atomic.h
index 43783891d359..74ce26fa8838 100644
--- a/include/drm/drm_atomic.h
+++ b/include/drm/drm_atomic.h
@@ -339,6 +339,11 @@ struct drm_private_state_funcs {
  * drm_atomic_helper_wait_for_dependencies().
  */
 struct drm_private_obj {
+	/**
+	 * @dev: parent DRM device
+	 */
+	struct drm_device *dev;
+
 	/**
 	 * @head: List entry used to attach a private object to a &drm_device
 	 * (queued to &drm_mode_config.privobj_list).
diff --git a/include/drm/drm_bridge.h b/include/drm/drm_bridge.h
index 0ff7ab4aa868..d2683846cc61 100644
--- a/include/drm/drm_bridge.h
+++ b/include/drm/drm_bridge.h
@@ -1143,6 +1143,14 @@ struct drm_bridge {
 	 */
 	struct kref refcount;
 
+	/**
+	 * @unplugged:
+	 *
+	 * Flag to tell if the bridge has been unplugged.
+	 * See drm_bridge_enter() and drm_bridge_unplug().
+	 */
+	bool unplugged;
+
 	/** @driver_private: pointer to the bridge driver's internal context */
 	void *driver_private;
 	/** @ops: bitmask of operations supported by the bridge */
@@ -1278,6 +1286,10 @@ drm_priv_to_bridge(struct drm_private_obj *priv)
 	return container_of(priv, struct drm_bridge, base);
 }
 
+bool drm_bridge_enter(struct drm_bridge *bridge, int *idx);
+void drm_bridge_exit(int idx);
+void drm_bridge_unplug(struct drm_bridge *bridge);
+
 struct drm_bridge *drm_bridge_get(struct drm_bridge *bridge);
 void drm_bridge_put(struct drm_bridge *bridge);
 
diff --git a/include/drm/drm_device.h b/include/drm/drm_device.h
index 5af49c5c3778..bc78fb77cc27 100644
--- a/include/drm/drm_device.h
+++ b/include/drm/drm_device.h
@@ -3,6 +3,9 @@
 
 #include <linux/list.h>
 #include <linux/kref.h>
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+#include <linux/mount.h>
+#endif
 #include <linux/mutex.h>
 #include <linux/idr.h>
 #include <linux/sched.h>
@@ -168,6 +171,18 @@ struct drm_device {
 	 */
 	struct drm_master *master;
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+	/**
+	 * @huge_mnt:
+	 *
+	 * Huge tmpfs mountpoint used at GEM object initialization
+	 * drm_gem_object_init(). Drivers can call drm_gem_huge_mnt_create() to
+	 * create, mount and use it. The default tmpfs mountpoint (`shm_mnt`) is
+	 * used if NULL.
+	 */
+	struct vfsmount *huge_mnt;
+#endif
+
 	/**
 	 * @driver_features: per-device driver features
 	 *
diff --git a/include/drm/drm_fb_helper.h b/include/drm/drm_fb_helper.h
index dd9a18f8de5a..05cca77b7249 100644
--- a/include/drm/drm_fb_helper.h
+++ b/include/drm/drm_fb_helper.h
@@ -166,13 +166,6 @@ struct drm_fb_helper {
 	 */
 	struct mutex lock;
 
-	/**
-	 * @kernel_fb_list:
-	 *
-	 * Entry on the global kernel_fb_helper_list, used for kgdb entry/exit.
-	 */
-	struct list_head kernel_fb_list;
-
 	/**
 	 * @delayed_hotplug:
 	 *
@@ -236,8 +229,6 @@ drm_fb_helper_from_client(struct drm_client_dev *client)
 	.fb_setcmap	= drm_fb_helper_setcmap, \
 	.fb_blank	= drm_fb_helper_blank, \
 	.fb_pan_display	= drm_fb_helper_pan_display, \
-	.fb_debug_enter = drm_fb_helper_debug_enter, \
-	.fb_debug_leave = drm_fb_helper_debug_leave, \
 	.fb_ioctl	= drm_fb_helper_ioctl
 
 #ifdef CONFIG_DRM_FBDEV_EMULATION
@@ -280,8 +271,6 @@ int drm_fb_helper_ioctl(struct fb_info *info, unsigned int cmd,
 
 int drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper);
 int drm_fb_helper_initial_config(struct drm_fb_helper *fb_helper);
-int drm_fb_helper_debug_enter(struct fb_info *info);
-int drm_fb_helper_debug_leave(struct fb_info *info);
 #else
 static inline void drm_fb_helper_prepare(struct drm_device *dev,
 					 struct drm_fb_helper *helper,
@@ -387,16 +376,6 @@ static inline int drm_fb_helper_initial_config(struct drm_fb_helper *fb_helper)
 {
 	return 0;
 }
-
-static inline int drm_fb_helper_debug_enter(struct fb_info *info)
-{
-	return 0;
-}
-
-static inline int drm_fb_helper_debug_leave(struct fb_info *info)
-{
-	return 0;
-}
 #endif
 
 #endif
diff --git a/include/drm/drm_file.h b/include/drm/drm_file.h
index 1a3018e4a537..6ee70ad65e1f 100644
--- a/include/drm/drm_file.h
+++ b/include/drm/drm_file.h
@@ -33,6 +33,7 @@
 #include <linux/types.h>
 #include <linux/completion.h>
 #include <linux/idr.h>
+#include <linux/xarray.h>
 
 #include <uapi/drm/drm.h>
 
@@ -316,10 +317,8 @@ struct drm_file {
 	/** @table_lock: Protects @object_idr. */
 	spinlock_t table_lock;
 
-	/** @syncobj_idr: Mapping of sync object handles to object pointers. */
-	struct idr syncobj_idr;
-	/** @syncobj_table_lock: Protects @syncobj_idr. */
-	spinlock_t syncobj_table_lock;
+	/** @syncobj_xa: Mapping of sync object handles to object pointers. */
+	struct xarray syncobj_xa;
 
 	/** @filp: Pointer to the core file structure. */
 	struct file *filp;
diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h
index 8d48d2af2649..86f5846154f7 100644
--- a/include/drm/drm_gem.h
+++ b/include/drm/drm_gem.h
@@ -40,6 +40,9 @@
 #include <linux/list.h>
 #include <linux/mutex.h>
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+#include <drm/drm_device.h>
+#endif
 #include <drm/drm_vma_manager.h>
 
 struct iosys_map;
@@ -469,6 +472,7 @@ struct drm_gem_object {
 	.poll		= drm_poll,\
 	.read		= drm_read,\
 	.llseek		= noop_llseek,\
+	.get_unmapped_area	= drm_gem_get_unmapped_area,\
 	.mmap		= drm_gem_mmap, \
 	.fop_flags	= FOP_UNSIGNED_OFFSET
 
@@ -491,13 +495,40 @@ struct drm_gem_object {
 		DRM_GEM_FOPS,\
 	}
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+int drm_gem_huge_mnt_create(struct drm_device *dev, const char *value);
+#else
+static inline int drm_gem_huge_mnt_create(struct drm_device *dev,
+					  const char *value)
+{
+	return 0;
+}
+#endif
+
+/**
+ * drm_gem_get_huge_mnt - Get the huge tmpfs mountpoint used by a DRM device
+ * @dev: DRM device
+ *
+ * This function gets the huge tmpfs mountpoint used by DRM device @dev. A huge
+ * tmpfs mountpoint is used instead of `shm_mnt` after a successful call to
+ * drm_gem_huge_mnt_create() when CONFIG_TRANSPARENT_HUGEPAGE is enabled.
+ *
+ * Returns:
+ * The huge tmpfs mountpoint in use, NULL otherwise.
+ */
+static inline struct vfsmount *drm_gem_get_huge_mnt(struct drm_device *dev)
+{
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+	return dev->huge_mnt;
+#else
+	return NULL;
+#endif
+}
+
 void drm_gem_object_release(struct drm_gem_object *obj);
 void drm_gem_object_free(struct kref *kref);
 int drm_gem_object_init(struct drm_device *dev,
 			struct drm_gem_object *obj, size_t size);
-int drm_gem_object_init_with_mnt(struct drm_device *dev,
-				 struct drm_gem_object *obj, size_t size,
-				 struct vfsmount *gemfs);
 void drm_gem_private_object_init(struct drm_device *dev,
 				 struct drm_gem_object *obj, size_t size);
 void drm_gem_private_object_fini(struct drm_gem_object *obj);
@@ -507,6 +538,14 @@ int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size,
 		     struct vm_area_struct *vma);
 int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma);
 
+#ifdef CONFIG_MMU
+unsigned long drm_gem_get_unmapped_area(struct file *filp, unsigned long uaddr,
+					unsigned long len, unsigned long pgoff,
+					unsigned long flags);
+#else
+#define drm_gem_get_unmapped_area NULL
+#endif
+
 /**
  * drm_gem_object_get - acquire a GEM buffer object reference
  * @obj: GEM buffer object
diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h
index 589f7bfe7506..6b6478f5ca24 100644
--- a/include/drm/drm_gem_shmem_helper.h
+++ b/include/drm/drm_gem_shmem_helper.h
@@ -109,9 +109,6 @@ struct drm_gem_shmem_object {
 
 int drm_gem_shmem_init(struct drm_device *dev, struct drm_gem_shmem_object *shmem, size_t size);
 struct drm_gem_shmem_object *drm_gem_shmem_create(struct drm_device *dev, size_t size);
-struct drm_gem_shmem_object *drm_gem_shmem_create_with_mnt(struct drm_device *dev,
-							   size_t size,
-							   struct vfsmount *gemfs);
 void drm_gem_shmem_release(struct drm_gem_shmem_object *shmem);
 void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem);
 
diff --git a/include/drm/drm_modeset_helper_vtables.h b/include/drm/drm_modeset_helper_vtables.h
index fe32854b7ffe..3e68213958dd 100644
--- a/include/drm/drm_modeset_helper_vtables.h
+++ b/include/drm/drm_modeset_helper_vtables.h
@@ -52,11 +52,6 @@ struct drm_scanout_buffer;
 struct drm_writeback_connector;
 struct drm_writeback_job;
 
-enum mode_set_atomic {
-	LEAVE_ATOMIC_MODE_SET,
-	ENTER_ATOMIC_MODE_SET,
-};
-
 /**
  * struct drm_crtc_helper_funcs - helper operations for CRTCs
  *
@@ -253,24 +248,6 @@ struct drm_crtc_helper_funcs {
 	int (*mode_set_base)(struct drm_crtc *crtc, int x, int y,
 			     struct drm_framebuffer *old_fb);
 
-	/**
-	 * @mode_set_base_atomic:
-	 *
-	 * This callback is used by the fbdev helpers to set a new framebuffer
-	 * and scanout without sleeping, i.e. from an atomic calling context. It
-	 * is only used to implement kgdb support.
-	 *
-	 * This callback is optional and only needed for kgdb support in the fbdev
-	 * helpers.
-	 *
-	 * RETURNS:
-	 *
-	 * 0 on success or a negative error code on failure.
-	 */
-	int (*mode_set_base_atomic)(struct drm_crtc *crtc,
-				    struct drm_framebuffer *fb, int x, int y,
-				    enum mode_set_atomic);
-
 	/**
 	 * @disable:
 	 *
diff --git a/include/drm/drm_vblank.h b/include/drm/drm_vblank.h
index ffa564d79638..2fcef9c0f5b1 100644
--- a/include/drm/drm_vblank.h
+++ b/include/drm/drm_vblank.h
@@ -302,8 +302,7 @@ bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe);
 bool drm_crtc_handle_vblank(struct drm_crtc *crtc);
 int drm_crtc_vblank_get(struct drm_crtc *crtc);
 void drm_crtc_vblank_put(struct drm_crtc *crtc);
-void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe);
-void drm_crtc_wait_one_vblank(struct drm_crtc *crtc);
+int drm_crtc_wait_one_vblank(struct drm_crtc *crtc);
 void drm_crtc_vblank_off(struct drm_crtc *crtc);
 void drm_crtc_vblank_reset(struct drm_crtc *crtc);
 void drm_crtc_vblank_on_config(struct drm_crtc *crtc,
diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
index fb88301b3c45..78e07c2507c7 100644
--- a/include/drm/gpu_scheduler.h
+++ b/include/drm/gpu_scheduler.h
@@ -645,6 +645,7 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad);
 void drm_sched_start(struct drm_gpu_scheduler *sched, int errno);
 void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
 void drm_sched_fault(struct drm_gpu_scheduler *sched);
+bool drm_sched_is_stopped(struct drm_gpu_scheduler *sched);
 
 struct drm_gpu_scheduler *
 drm_sched_pick_best(struct drm_gpu_scheduler **sched_list,
@@ -674,6 +675,7 @@ bool drm_sched_job_has_dependency(struct drm_sched_job *job,
 				  struct dma_fence *fence);
 void drm_sched_job_cleanup(struct drm_sched_job *job);
 void drm_sched_increase_karma(struct drm_sched_job *bad);
+bool drm_sched_job_is_signaled(struct drm_sched_job *job);
 
 static inline bool drm_sched_invalidate_job(struct drm_sched_job *s_job,
 					    int threshold)
@@ -698,4 +700,54 @@ void drm_sched_entity_modify_sched(struct drm_sched_entity *entity,
 				   struct drm_gpu_scheduler **sched_list,
 				   unsigned int num_sched_list);
 
+/**
+ * struct drm_sched_pending_job_iter - DRM scheduler pending job iterator state
+ * @sched: DRM scheduler associated with pending job iterator
+ */
+struct drm_sched_pending_job_iter {
+	struct drm_gpu_scheduler *sched;
+};
+
+/* Drivers should never call this directly */
+static inline struct drm_sched_pending_job_iter
+__drm_sched_pending_job_iter_begin(struct drm_gpu_scheduler *sched)
+{
+	struct drm_sched_pending_job_iter iter = {
+		.sched = sched,
+	};
+
+	WARN_ON(!drm_sched_is_stopped(sched));
+	return iter;
+}
+
+/* Drivers should never call this directly */
+static inline void
+__drm_sched_pending_job_iter_end(const struct drm_sched_pending_job_iter iter)
+{
+	WARN_ON(!drm_sched_is_stopped(iter.sched));
+}
+
+DEFINE_CLASS(drm_sched_pending_job_iter, struct drm_sched_pending_job_iter,
+	     __drm_sched_pending_job_iter_end(_T),
+	     __drm_sched_pending_job_iter_begin(__sched),
+	     struct drm_gpu_scheduler *__sched);
+static inline void *
+class_drm_sched_pending_job_iter_lock_ptr(class_drm_sched_pending_job_iter_t *_T)
+{ return _T; }
+#define class_drm_sched_pending_job_iter_is_conditional false
+
+/**
+ * drm_sched_for_each_pending_job() - Iterator for each pending job in scheduler
+ * @__job: Current pending job being iterated over
+ * @__sched: DRM scheduler to iterate over pending jobs
+ * @__entity: DRM scheduler entity to filter jobs, NULL indicates no filter
+ *
+ * Iterator for each pending job in scheduler, filtering on an entity, and
+ * enforcing scheduler is fully stopped
+ */
+#define drm_sched_for_each_pending_job(__job, __sched, __entity)		\
+	scoped_guard(drm_sched_pending_job_iter, (__sched))			\
+		list_for_each_entry((__job), &(__sched)->pending_list, list)	\
+			for_each_if(!(__entity) || (__job)->entity == (__entity))
+
 #endif
diff --git a/include/drm/intel/display_parent_interface.h b/include/drm/intel/display_parent_interface.h
index 26bedc360044..55d4df714645 100644
--- a/include/drm/intel/display_parent_interface.h
+++ b/include/drm/intel/display_parent_interface.h
@@ -6,9 +6,36 @@
 
 #include <linux/types.h>
 
+struct dma_fence;
 struct drm_device;
+struct drm_scanout_buffer;
+struct intel_hdcp_gsc_context;
+struct intel_panic;
+struct intel_stolen_node;
 struct ref_tracker;
 
+/* Keep struct definitions sorted */
+
+struct intel_display_hdcp_interface {
+	ssize_t (*gsc_msg_send)(struct intel_hdcp_gsc_context *gsc_context,
+				void *msg_in, size_t msg_in_len,
+				void *msg_out, size_t msg_out_len);
+	bool (*gsc_check_status)(struct drm_device *drm);
+	struct intel_hdcp_gsc_context *(*gsc_context_alloc)(struct drm_device *drm);
+	void (*gsc_context_free)(struct intel_hdcp_gsc_context *gsc_context);
+};
+
+struct intel_display_irq_interface {
+	bool (*enabled)(struct drm_device *drm);
+	void (*synchronize)(struct drm_device *drm);
+};
+
+struct intel_display_panic_interface {
+	struct intel_panic *(*alloc)(void);
+	int (*setup)(struct intel_panic *panic, struct drm_scanout_buffer *sb);
+	void (*finish)(struct intel_panic *panic);
+};
+
 struct intel_display_rpm_interface {
 	struct ref_tracker *(*get)(const struct drm_device *drm);
 	struct ref_tracker *(*get_raw)(const struct drm_device *drm);
@@ -25,6 +52,28 @@ struct intel_display_rpm_interface {
 	void (*assert_unblock)(const struct drm_device *drm);
 };
 
+struct intel_display_rps_interface {
+	void (*boost_if_not_started)(struct dma_fence *fence);
+	void (*mark_interactive)(struct drm_device *drm, bool interactive);
+	void (*ilk_irq_handler)(struct drm_device *drm);
+};
+
+struct intel_display_stolen_interface {
+	int (*insert_node_in_range)(struct intel_stolen_node *node, u64 size,
+				    unsigned int align, u64 start, u64 end);
+	int (*insert_node)(struct intel_stolen_node *node, u64 size, unsigned int align); /* Optional */
+	void (*remove_node)(struct intel_stolen_node *node);
+	bool (*initialized)(struct drm_device *drm);
+	bool (*node_allocated)(const struct intel_stolen_node *node);
+	u64 (*node_offset)(const struct intel_stolen_node *node);
+	u64 (*area_address)(struct drm_device *drm); /* Optional */
+	u64 (*area_size)(struct drm_device *drm); /* Optional */
+	u64 (*node_address)(const struct intel_stolen_node *node);
+	u64 (*node_size)(const struct intel_stolen_node *node);
+	struct intel_stolen_node *(*node_alloc)(struct drm_device *drm);
+	void (*node_free)(const struct intel_stolen_node *node);
+};
+
 /**
  * struct intel_display_parent_interface - services parent driver provides to display
  *
@@ -38,8 +87,38 @@ struct intel_display_rpm_interface {
  * check the optional pointers.
  */
 struct intel_display_parent_interface {
+	/** @hdcp: HDCP GSC interface */
+	const struct intel_display_hdcp_interface *hdcp;
+
+	/** @irq: IRQ interface */
+	const struct intel_display_irq_interface *irq;
+
+	/** @panic: Panic interface */
+	const struct intel_display_panic_interface *panic;
+
 	/** @rpm: Runtime PM functions */
 	const struct intel_display_rpm_interface *rpm;
+
+	/** @rps: RPS interface. Optional. */
+	const struct intel_display_rps_interface *rps;
+
+	/** @stolen: Stolen memory. */
+	const struct intel_display_stolen_interface *stolen;
+
+	/* Generic independent functions */
+	struct {
+		/** @fence_priority_display: Set display priority. Optional. */
+		void (*fence_priority_display)(struct dma_fence *fence);
+
+		/** @has_auxccs: Are AuxCCS formats supported by the parent. Optional. */
+		bool (*has_auxccs)(struct drm_device *drm);
+
+		/** @has_fenced_regions: Support legacy fencing? Optional. */
+		bool (*has_fenced_regions)(struct drm_device *drm);
+
+		/** @vgpu_active: Is vGPU active? Optional. */
+		bool (*vgpu_active)(struct drm_device *drm);
+	};
 };
 
 #endif
diff --git a/include/dt-bindings/clock/oxsemi,ox810se.h b/include/dt-bindings/clock/oxsemi,ox810se.h
deleted file mode 100644
index 7256365160f8..000000000000
--- a/include/dt-bindings/clock/oxsemi,ox810se.h
+++ /dev/null
@@ -1,19 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2016 Neil Armstrong <narmstrong@baylibre.com>
- */
-
-#ifndef DT_CLOCK_OXSEMI_OX810SE_H
-#define DT_CLOCK_OXSEMI_OX810SE_H
-
-#define CLK_810_LEON		0
-#define CLK_810_DMA_SGDMA	1
-#define CLK_810_CIPHER		2
-#define CLK_810_SATA		3
-#define CLK_810_AUDIO		4
-#define CLK_810_USBMPH		5
-#define CLK_810_ETHA		6
-#define CLK_810_PCIEA		7
-#define CLK_810_NAND		8
-
-#endif /* DT_CLOCK_OXSEMI_OX810SE_H */
diff --git a/include/dt-bindings/clock/oxsemi,ox820.h b/include/dt-bindings/clock/oxsemi,ox820.h
deleted file mode 100644
index 55f4226e2f3f..000000000000
--- a/include/dt-bindings/clock/oxsemi,ox820.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2016 Neil Armstrong <narmstrong@baylibre.com>
- */
-
-#ifndef DT_CLOCK_OXSEMI_OX820_H
-#define DT_CLOCK_OXSEMI_OX820_H
-
-/* PLLs */
-#define CLK_820_PLLA		0
-#define CLK_820_PLLB		1
-
-/* Gate Clocks */
-#define CLK_820_LEON		2
-#define CLK_820_DMA_SGDMA	3
-#define CLK_820_CIPHER		4
-#define CLK_820_SD		5
-#define CLK_820_SATA		6
-#define CLK_820_AUDIO		7
-#define CLK_820_USBMPH		8
-#define CLK_820_ETHA		9
-#define CLK_820_PCIEA		10
-#define CLK_820_NAND		11
-#define CLK_820_PCIEB		12
-#define CLK_820_ETHB		13
-#define CLK_820_REF600		14
-#define CLK_820_USBDEV		15
-
-#endif /* DT_CLOCK_OXSEMI_OX820_H */
diff --git a/include/dt-bindings/clock/qcom,mss-sc7180.h b/include/dt-bindings/clock/qcom,mss-sc7180.h
deleted file mode 100644
index f15a9ded2961..000000000000
--- a/include/dt-bindings/clock/qcom,mss-sc7180.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2020, The Linux Foundation. All rights reserved.
- */
-
-#ifndef _DT_BINDINGS_CLK_QCOM_MSS_SC7180_H
-#define _DT_BINDINGS_CLK_QCOM_MSS_SC7180_H
-
-#define MSS_AXI_CRYPTO_CLK	0
-#define MSS_AXI_NAV_CLK		1
-
-#endif
diff --git a/include/dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h b/include/dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h
index 2a805e06487b..9eaedca6a616 100644
--- a/include/dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h
+++ b/include/dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h
@@ -31,5 +31,7 @@
 #define R9A09G077_ETCLKC		19
 #define R9A09G077_ETCLKD		20
 #define R9A09G077_ETCLKE		21
+#define R9A09G077_XSPI_CLK0		22
+#define R9A09G077_XSPI_CLK1		23
 
 #endif /* __DT_BINDINGS_CLOCK_RENESAS_R9A09G077_CPG_H__ */
diff --git a/include/dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h b/include/dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h
index 09da0ad33be6..606468ac49a4 100644
--- a/include/dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h
+++ b/include/dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h
@@ -31,5 +31,7 @@
 #define R9A09G087_ETCLKC		19
 #define R9A09G087_ETCLKD		20
 #define R9A09G087_ETCLKE		21
+#define R9A09G087_XSPI_CLK0		22
+#define R9A09G087_XSPI_CLK1		23
 
 #endif /* __DT_BINDINGS_CLOCK_RENESAS_R9A09G087_CPG_H__ */
diff --git a/include/dt-bindings/clock/thead,th1520-clk-ap.h b/include/dt-bindings/clock/thead,th1520-clk-ap.h
index 09a9aa7b3ab1..68b35cc61204 100644
--- a/include/dt-bindings/clock/thead,th1520-clk-ap.h
+++ b/include/dt-bindings/clock/thead,th1520-clk-ap.h
@@ -93,6 +93,7 @@
 #define CLK_SRAM3		83
 #define CLK_PLL_GMAC_100M	84
 #define CLK_UART_SCLK		85
+#define CLK_C910_BUS		86
 
 /* VO clocks */
 #define CLK_AXI4_VO_ACLK		0
diff --git a/include/dt-bindings/clock/xlnx-versal-clk.h b/include/dt-bindings/clock/xlnx-versal-clk.h
deleted file mode 100644
index 264d634d226e..000000000000
--- a/include/dt-bindings/clock/xlnx-versal-clk.h
+++ /dev/null
@@ -1,123 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- *  Copyright (C) 2019 Xilinx Inc.
- *
- */
-
-#ifndef _DT_BINDINGS_CLK_VERSAL_H
-#define _DT_BINDINGS_CLK_VERSAL_H
-
-#define PMC_PLL					1
-#define APU_PLL					2
-#define RPU_PLL					3
-#define CPM_PLL					4
-#define NOC_PLL					5
-#define PLL_MAX					6
-#define PMC_PRESRC				7
-#define PMC_POSTCLK				8
-#define PMC_PLL_OUT				9
-#define PPLL					10
-#define NOC_PRESRC				11
-#define NOC_POSTCLK				12
-#define NOC_PLL_OUT				13
-#define NPLL					14
-#define APU_PRESRC				15
-#define APU_POSTCLK				16
-#define APU_PLL_OUT				17
-#define APLL					18
-#define RPU_PRESRC				19
-#define RPU_POSTCLK				20
-#define RPU_PLL_OUT				21
-#define RPLL					22
-#define CPM_PRESRC				23
-#define CPM_POSTCLK				24
-#define CPM_PLL_OUT				25
-#define CPLL					26
-#define PPLL_TO_XPD				27
-#define NPLL_TO_XPD				28
-#define APLL_TO_XPD				29
-#define RPLL_TO_XPD				30
-#define EFUSE_REF				31
-#define SYSMON_REF				32
-#define IRO_SUSPEND_REF				33
-#define USB_SUSPEND				34
-#define SWITCH_TIMEOUT				35
-#define RCLK_PMC				36
-#define RCLK_LPD				37
-#define WDT					38
-#define TTC0					39
-#define TTC1					40
-#define TTC2					41
-#define TTC3					42
-#define GEM_TSU					43
-#define GEM_TSU_LB				44
-#define MUXED_IRO_DIV2				45
-#define MUXED_IRO_DIV4				46
-#define PSM_REF					47
-#define GEM0_RX					48
-#define GEM0_TX					49
-#define GEM1_RX					50
-#define GEM1_TX					51
-#define CPM_CORE_REF				52
-#define CPM_LSBUS_REF				53
-#define CPM_DBG_REF				54
-#define CPM_AUX0_REF				55
-#define CPM_AUX1_REF				56
-#define QSPI_REF				57
-#define OSPI_REF				58
-#define SDIO0_REF				59
-#define SDIO1_REF				60
-#define PMC_LSBUS_REF				61
-#define I2C_REF					62
-#define TEST_PATTERN_REF			63
-#define DFT_OSC_REF				64
-#define PMC_PL0_REF				65
-#define PMC_PL1_REF				66
-#define PMC_PL2_REF				67
-#define PMC_PL3_REF				68
-#define CFU_REF					69
-#define SPARE_REF				70
-#define NPI_REF					71
-#define HSM0_REF				72
-#define HSM1_REF				73
-#define SD_DLL_REF				74
-#define FPD_TOP_SWITCH				75
-#define FPD_LSBUS				76
-#define ACPU					77
-#define DBG_TRACE				78
-#define DBG_FPD					79
-#define LPD_TOP_SWITCH				80
-#define ADMA					81
-#define LPD_LSBUS				82
-#define CPU_R5					83
-#define CPU_R5_CORE				84
-#define CPU_R5_OCM				85
-#define CPU_R5_OCM2				86
-#define IOU_SWITCH				87
-#define GEM0_REF				88
-#define GEM1_REF				89
-#define GEM_TSU_REF				90
-#define USB0_BUS_REF				91
-#define UART0_REF				92
-#define UART1_REF				93
-#define SPI0_REF				94
-#define SPI1_REF				95
-#define CAN0_REF				96
-#define CAN1_REF				97
-#define I2C0_REF				98
-#define I2C1_REF				99
-#define DBG_LPD					100
-#define TIMESTAMP_REF				101
-#define DBG_TSTMP				102
-#define CPM_TOPSW_REF				103
-#define USB3_DUAL_REF				104
-#define OUTCLK_MAX				105
-#define REF_CLK					106
-#define PL_ALT_REF_CLK				107
-#define MUXED_IRO				108
-#define PL_EXT					109
-#define PL_LB					110
-#define MIO_50_OR_51				111
-#define MIO_24_OR_25				112
-
-#endif
diff --git a/include/dt-bindings/clock/xlnx-zynqmp-clk.h b/include/dt-bindings/clock/xlnx-zynqmp-clk.h
deleted file mode 100644
index f0f7ddd3dcbd..000000000000
--- a/include/dt-bindings/clock/xlnx-zynqmp-clk.h
+++ /dev/null
@@ -1,133 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Xilinx Zynq MPSoC Firmware layer
- *
- *  Copyright (C) 2014-2018 Xilinx, Inc.
- *
- */
-
-#ifndef _DT_BINDINGS_CLK_ZYNQMP_H
-#define _DT_BINDINGS_CLK_ZYNQMP_H
-
-/*
- * These bindings are deprecated, because they do not match the actual
- * concept of bindings but rather contain pure firmware values.
- * Instead include the header in the DTS source directory.
- */
-#warning "These bindings are deprecated. Instead use the header in the DTS source directory."
-
-#define IOPLL			0
-#define RPLL			1
-#define APLL			2
-#define DPLL			3
-#define VPLL			4
-#define IOPLL_TO_FPD		5
-#define RPLL_TO_FPD		6
-#define APLL_TO_LPD		7
-#define DPLL_TO_LPD		8
-#define VPLL_TO_LPD		9
-#define ACPU			10
-#define ACPU_HALF		11
-#define DBF_FPD			12
-#define DBF_LPD			13
-#define DBG_TRACE		14
-#define DBG_TSTMP		15
-#define DP_VIDEO_REF		16
-#define DP_AUDIO_REF		17
-#define DP_STC_REF		18
-#define GDMA_REF		19
-#define DPDMA_REF		20
-#define DDR_REF			21
-#define SATA_REF		22
-#define PCIE_REF		23
-#define GPU_REF			24
-#define GPU_PP0_REF		25
-#define GPU_PP1_REF		26
-#define TOPSW_MAIN		27
-#define TOPSW_LSBUS		28
-#define GTGREF0_REF		29
-#define LPD_SWITCH		30
-#define LPD_LSBUS		31
-#define USB0_BUS_REF		32
-#define USB1_BUS_REF		33
-#define USB3_DUAL_REF		34
-#define USB0			35
-#define USB1			36
-#define CPU_R5			37
-#define CPU_R5_CORE		38
-#define CSU_SPB			39
-#define CSU_PLL			40
-#define PCAP			41
-#define IOU_SWITCH		42
-#define GEM_TSU_REF		43
-#define GEM_TSU			44
-#define GEM0_TX			45
-#define GEM1_TX			46
-#define GEM2_TX			47
-#define GEM3_TX			48
-#define GEM0_RX			49
-#define GEM1_RX			50
-#define GEM2_RX			51
-#define GEM3_RX			52
-#define QSPI_REF		53
-#define SDIO0_REF		54
-#define SDIO1_REF		55
-#define UART0_REF		56
-#define UART1_REF		57
-#define SPI0_REF		58
-#define SPI1_REF		59
-#define NAND_REF		60
-#define I2C0_REF		61
-#define I2C1_REF		62
-#define CAN0_REF		63
-#define CAN1_REF		64
-#define CAN0			65
-#define CAN1			66
-#define DLL_REF			67
-#define ADMA_REF		68
-#define TIMESTAMP_REF		69
-#define AMS_REF			70
-#define PL0_REF			71
-#define PL1_REF			72
-#define PL2_REF			73
-#define PL3_REF			74
-#define WDT			75
-#define IOPLL_INT		76
-#define IOPLL_PRE_SRC		77
-#define IOPLL_HALF		78
-#define IOPLL_INT_MUX		79
-#define IOPLL_POST_SRC		80
-#define RPLL_INT		81
-#define RPLL_PRE_SRC		82
-#define RPLL_HALF		83
-#define RPLL_INT_MUX		84
-#define RPLL_POST_SRC		85
-#define APLL_INT		86
-#define APLL_PRE_SRC		87
-#define APLL_HALF		88
-#define APLL_INT_MUX		89
-#define APLL_POST_SRC		90
-#define DPLL_INT		91
-#define DPLL_PRE_SRC		92
-#define DPLL_HALF		93
-#define DPLL_INT_MUX		94
-#define DPLL_POST_SRC		95
-#define VPLL_INT		96
-#define VPLL_PRE_SRC		97
-#define VPLL_HALF		98
-#define VPLL_INT_MUX		99
-#define VPLL_POST_SRC		100
-#define CAN0_MIO		101
-#define CAN1_MIO		102
-#define ACPU_FULL		103
-#define GEM0_REF		104
-#define GEM1_REF		105
-#define GEM2_REF		106
-#define GEM3_REF		107
-#define GEM0_REF_UNG		108
-#define GEM1_REF_UNG		109
-#define GEM2_REF_UNG		110
-#define GEM3_REF_UNG		111
-#define LPD_WDT			112
-
-#endif
diff --git a/include/dt-bindings/dma/jz4775-dma.h b/include/dt-bindings/dma/jz4775-dma.h
deleted file mode 100644
index 8d27e2c69dca..000000000000
--- a/include/dt-bindings/dma/jz4775-dma.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * This header provides macros for JZ4775 DMA bindings.
- *
- * Copyright (c) 2020 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
- */
-
-#ifndef __DT_BINDINGS_DMA_JZ4775_DMA_H__
-#define __DT_BINDINGS_DMA_JZ4775_DMA_H__
-
-/*
- * Request type numbers for the JZ4775 DMA controller (written to the DRTn
- * register for the channel).
- */
-#define JZ4775_DMA_I2S0_TX	0x6
-#define JZ4775_DMA_I2S0_RX	0x7
-#define JZ4775_DMA_AUTO		0x8
-#define JZ4775_DMA_SADC_RX	0x9
-#define JZ4775_DMA_UART3_TX	0x0e
-#define JZ4775_DMA_UART3_RX	0x0f
-#define JZ4775_DMA_UART2_TX	0x10
-#define JZ4775_DMA_UART2_RX	0x11
-#define JZ4775_DMA_UART1_TX	0x12
-#define JZ4775_DMA_UART1_RX	0x13
-#define JZ4775_DMA_UART0_TX	0x14
-#define JZ4775_DMA_UART0_RX	0x15
-#define JZ4775_DMA_SSI0_TX	0x16
-#define JZ4775_DMA_SSI0_RX	0x17
-#define JZ4775_DMA_MSC0_TX	0x1a
-#define JZ4775_DMA_MSC0_RX	0x1b
-#define JZ4775_DMA_MSC1_TX	0x1c
-#define JZ4775_DMA_MSC1_RX	0x1d
-#define JZ4775_DMA_MSC2_TX	0x1e
-#define JZ4775_DMA_MSC2_RX	0x1f
-#define JZ4775_DMA_PCM0_TX	0x20
-#define JZ4775_DMA_PCM0_RX	0x21
-#define JZ4775_DMA_SMB0_TX	0x24
-#define JZ4775_DMA_SMB0_RX	0x25
-#define JZ4775_DMA_SMB1_TX	0x26
-#define JZ4775_DMA_SMB1_RX	0x27
-#define JZ4775_DMA_SMB2_TX	0x28
-#define JZ4775_DMA_SMB2_RX	0x29
-
-#endif /* __DT_BINDINGS_DMA_JZ4775_DMA_H__ */
diff --git a/include/dt-bindings/dma/x2000-dma.h b/include/dt-bindings/dma/x2000-dma.h
deleted file mode 100644
index db2cd4830b00..000000000000
--- a/include/dt-bindings/dma/x2000-dma.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * This header provides macros for X2000 DMA bindings.
- *
- * Copyright (c) 2020 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
- */
-
-#ifndef __DT_BINDINGS_DMA_X2000_DMA_H__
-#define __DT_BINDINGS_DMA_X2000_DMA_H__
-
-/*
- * Request type numbers for the X2000 DMA controller (written to the DRTn
- * register for the channel).
- */
-#define X2000_DMA_AUTO		0x8
-#define X2000_DMA_UART5_TX	0xa
-#define X2000_DMA_UART5_RX	0xb
-#define X2000_DMA_UART4_TX	0xc
-#define X2000_DMA_UART4_RX	0xd
-#define X2000_DMA_UART3_TX	0xe
-#define X2000_DMA_UART3_RX	0xf
-#define X2000_DMA_UART2_TX	0x10
-#define X2000_DMA_UART2_RX	0x11
-#define X2000_DMA_UART1_TX	0x12
-#define X2000_DMA_UART1_RX	0x13
-#define X2000_DMA_UART0_TX	0x14
-#define X2000_DMA_UART0_RX	0x15
-#define X2000_DMA_SSI0_TX	0x16
-#define X2000_DMA_SSI0_RX	0x17
-#define X2000_DMA_SSI1_TX	0x18
-#define X2000_DMA_SSI1_RX	0x19
-#define X2000_DMA_I2C0_TX	0x24
-#define X2000_DMA_I2C0_RX	0x25
-#define X2000_DMA_I2C1_TX	0x26
-#define X2000_DMA_I2C1_RX	0x27
-#define X2000_DMA_I2C2_TX	0x28
-#define X2000_DMA_I2C2_RX	0x29
-#define X2000_DMA_I2C3_TX	0x2a
-#define X2000_DMA_I2C3_RX	0x2b
-#define X2000_DMA_I2C4_TX	0x2c
-#define X2000_DMA_I2C4_RX	0x2d
-#define X2000_DMA_I2C5_TX	0x2e
-#define X2000_DMA_I2C5_RX	0x2f
-#define X2000_DMA_UART6_TX	0x30
-#define X2000_DMA_UART6_RX	0x31
-#define X2000_DMA_UART7_TX	0x32
-#define X2000_DMA_UART7_RX	0x33
-#define X2000_DMA_UART8_TX	0x34
-#define X2000_DMA_UART8_RX	0x35
-#define X2000_DMA_UART9_TX	0x36
-#define X2000_DMA_UART9_RX	0x37
-#define X2000_DMA_SADC_RX	0x38
-
-#endif /* __DT_BINDINGS_DMA_X2000_DMA_H__ */
diff --git a/include/dt-bindings/gce/mt6779-gce.h b/include/dt-bindings/gce/mt6779-gce.h
deleted file mode 100644
index 06101316ace4..000000000000
--- a/include/dt-bindings/gce/mt6779-gce.h
+++ /dev/null
@@ -1,222 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 2019 MediaTek Inc.
- * Author: Dennis-YC Hsieh <dennis-yc.hsieh@mediatek.com>
- */
-
-#ifndef _DT_BINDINGS_GCE_MT6779_H
-#define _DT_BINDINGS_GCE_MT6779_H
-
-#define CMDQ_NO_TIMEOUT		0xffffffff
-
-/* GCE HW thread priority */
-#define CMDQ_THR_PRIO_LOWEST	0
-#define CMDQ_THR_PRIO_1		1
-#define CMDQ_THR_PRIO_2		2
-#define CMDQ_THR_PRIO_3		3
-#define CMDQ_THR_PRIO_4		4
-#define CMDQ_THR_PRIO_5		5
-#define CMDQ_THR_PRIO_6		6
-#define CMDQ_THR_PRIO_HIGHEST	7
-
-/* GCE subsys table */
-#define SUBSYS_1300XXXX		0
-#define SUBSYS_1400XXXX		1
-#define SUBSYS_1401XXXX		2
-#define SUBSYS_1402XXXX		3
-#define SUBSYS_1502XXXX		4
-#define SUBSYS_1880XXXX		5
-#define SUBSYS_1881XXXX		6
-#define SUBSYS_1882XXXX		7
-#define SUBSYS_1883XXXX		8
-#define SUBSYS_1884XXXX		9
-#define SUBSYS_1000XXXX		10
-#define SUBSYS_1001XXXX		11
-#define SUBSYS_1002XXXX		12
-#define SUBSYS_1003XXXX		13
-#define SUBSYS_1004XXXX		14
-#define SUBSYS_1005XXXX		15
-#define SUBSYS_1020XXXX		16
-#define SUBSYS_1028XXXX		17
-#define SUBSYS_1700XXXX		18
-#define SUBSYS_1701XXXX		19
-#define SUBSYS_1702XXXX		20
-#define SUBSYS_1703XXXX		21
-#define SUBSYS_1800XXXX		22
-#define SUBSYS_1801XXXX		23
-#define SUBSYS_1802XXXX		24
-#define SUBSYS_1804XXXX		25
-#define SUBSYS_1805XXXX		26
-#define SUBSYS_1808XXXX		27
-#define SUBSYS_180aXXXX		28
-#define SUBSYS_180bXXXX		29
-#define CMDQ_SUBSYS_OFF		32
-
-/* GCE hardware events */
-#define CMDQ_EVENT_DISP_RDMA0_SOF		0
-#define CMDQ_EVENT_DISP_RDMA1_SOF		1
-#define CMDQ_EVENT_MDP_RDMA0_SOF		2
-#define CMDQ_EVENT_MDP_RDMA1_SOF		3
-#define CMDQ_EVENT_MDP_RSZ0_SOF			4
-#define CMDQ_EVENT_MDP_RSZ1_SOF			5
-#define CMDQ_EVENT_MDP_TDSHP_SOF		6
-#define CMDQ_EVENT_MDP_WROT0_SOF		7
-#define CMDQ_EVENT_MDP_WROT1_SOF		8
-#define CMDQ_EVENT_DISP_OVL0_SOF		9
-#define CMDQ_EVENT_DISP_2L_OVL0_SOF		10
-#define CMDQ_EVENT_DISP_2L_OVL1_SOF		11
-#define CMDQ_EVENT_DISP_WDMA0_SOF		12
-#define CMDQ_EVENT_DISP_COLOR0_SOF		13
-#define CMDQ_EVENT_DISP_CCORR0_SOF		14
-#define CMDQ_EVENT_DISP_AAL0_SOF		15
-#define CMDQ_EVENT_DISP_GAMMA0_SOF		16
-#define CMDQ_EVENT_DISP_DITHER0_SOF		17
-#define CMDQ_EVENT_DISP_PWM0_SOF		18
-#define CMDQ_EVENT_DISP_DSI0_SOF		19
-#define CMDQ_EVENT_DISP_DPI0_SOF		20
-#define CMDQ_EVENT_DISP_POSTMASK0_SOF		21
-#define CMDQ_EVENT_DISP_RSZ0_SOF		22
-#define CMDQ_EVENT_MDP_AAL_SOF			23
-#define CMDQ_EVENT_MDP_CCORR_SOF		24
-#define CMDQ_EVENT_DISP_DBI0_SOF		25
-#define CMDQ_EVENT_ISP_RELAY_SOF		26
-#define CMDQ_EVENT_IPU_RELAY_SOF		27
-#define CMDQ_EVENT_DISP_RDMA0_EOF		28
-#define CMDQ_EVENT_DISP_RDMA1_EOF		29
-#define CMDQ_EVENT_MDP_RDMA0_EOF		30
-#define CMDQ_EVENT_MDP_RDMA1_EOF		31
-#define CMDQ_EVENT_MDP_RSZ0_EOF			32
-#define CMDQ_EVENT_MDP_RSZ1_EOF			33
-#define CMDQ_EVENT_MDP_TDSHP_EOF		34
-#define CMDQ_EVENT_MDP_WROT0_W_EOF		35
-#define CMDQ_EVENT_MDP_WROT1_W_EOF		36
-#define CMDQ_EVENT_DISP_OVL0_EOF		37
-#define CMDQ_EVENT_DISP_2L_OVL0_EOF		38
-#define CMDQ_EVENT_DISP_2L_OVL1_EOF		39
-#define CMDQ_EVENT_DISP_WDMA0_EOF		40
-#define CMDQ_EVENT_DISP_COLOR0_EOF		41
-#define CMDQ_EVENT_DISP_CCORR0_EOF		42
-#define CMDQ_EVENT_DISP_AAL0_EOF		43
-#define CMDQ_EVENT_DISP_GAMMA0_EOF		44
-#define CMDQ_EVENT_DISP_DITHER0_EOF		45
-#define CMDQ_EVENT_DISP_DSI0_EOF		46
-#define CMDQ_EVENT_DISP_DPI0_EOF		47
-#define CMDQ_EVENT_DISP_RSZ0_EOF		49
-#define CMDQ_EVENT_MDP_AAL_FRAME_DONE		50
-#define CMDQ_EVENT_MDP_CCORR_FRAME_DONE		51
-#define CMDQ_EVENT_DISP_POSTMASK0_FRAME_DONE	52
-#define CMDQ_EVENT_MUTEX0_STREAM_EOF		130
-#define CMDQ_EVENT_MUTEX1_STREAM_EOF		131
-#define CMDQ_EVENT_MUTEX2_STREAM_EOF		132
-#define CMDQ_EVENT_MUTEX3_STREAM_EOF		133
-#define CMDQ_EVENT_MUTEX4_STREAM_EOF		134
-#define CMDQ_EVENT_MUTEX5_STREAM_EOF		135
-#define CMDQ_EVENT_MUTEX6_STREAM_EOF		136
-#define CMDQ_EVENT_MUTEX7_STREAM_EOF		137
-#define CMDQ_EVENT_MUTEX8_STREAM_EOF		138
-#define CMDQ_EVENT_MUTEX9_STREAM_EOF		139
-#define CMDQ_EVENT_MUTEX10_STREAM_EOF		140
-#define CMDQ_EVENT_MUTEX11_STREAM_EOF		141
-#define CMDQ_EVENT_DISP_RDMA0_UNDERRUN		142
-#define CMDQ_EVENT_DISP_RDMA1_UNDERRUN		143
-#define CMDQ_EVENT_DISP_RDMA2_UNDERRUN		144
-#define CMDQ_EVENT_DISP_RDMA3_UNDERRUN		145
-#define CMDQ_EVENT_DSI0_TE			146
-#define CMDQ_EVENT_DSI0_IRQ_EVENT		147
-#define CMDQ_EVENT_DSI0_DONE_EVENT		148
-#define CMDQ_EVENT_DISP_POSTMASK0_RST_DONE	150
-#define CMDQ_EVENT_DISP_WDMA0_RST_DONE		151
-#define CMDQ_EVENT_MDP_WROT0_RST_DONE		153
-#define CMDQ_EVENT_MDP_RDMA0_RST_DONE		154
-#define CMDQ_EVENT_DISP_OVL0_RST_DONE		155
-#define CMDQ_EVENT_DISP_OVL0_2L_RST_DONE	156
-#define CMDQ_EVENT_DISP_OVL1_2L_RST_DONE	157
-#define CMDQ_EVENT_DIP_CQ_THREAD0_EOF		257
-#define CMDQ_EVENT_DIP_CQ_THREAD1_EOF		258
-#define CMDQ_EVENT_DIP_CQ_THREAD2_EOF		259
-#define CMDQ_EVENT_DIP_CQ_THREAD3_EOF		260
-#define CMDQ_EVENT_DIP_CQ_THREAD4_EOF		261
-#define CMDQ_EVENT_DIP_CQ_THREAD5_EOF		262
-#define CMDQ_EVENT_DIP_CQ_THREAD6_EOF		263
-#define CMDQ_EVENT_DIP_CQ_THREAD7_EOF		264
-#define CMDQ_EVENT_DIP_CQ_THREAD8_EOF		265
-#define CMDQ_EVENT_DIP_CQ_THREAD9_EOF		266
-#define CMDQ_EVENT_DIP_CQ_THREAD10_EOF		267
-#define CMDQ_EVENT_DIP_CQ_THREAD11_EOF		268
-#define CMDQ_EVENT_DIP_CQ_THREAD12_EOF		269
-#define CMDQ_EVENT_DIP_CQ_THREAD13_EOF		270
-#define CMDQ_EVENT_DIP_CQ_THREAD14_EOF		271
-#define CMDQ_EVENT_DIP_CQ_THREAD15_EOF		272
-#define CMDQ_EVENT_DIP_CQ_THREAD16_EOF		273
-#define CMDQ_EVENT_DIP_CQ_THREAD17_EOF		274
-#define CMDQ_EVENT_DIP_CQ_THREAD18_EOF		275
-#define CMDQ_EVENT_DIP_DMA_ERR_EVENT		276
-#define CMDQ_EVENT_AMD_FRAME_DONE		277
-#define CMDQ_EVENT_MFB_DONE			278
-#define CMDQ_EVENT_WPE_A_EOF			279
-#define CMDQ_EVENT_VENC_EOF			289
-#define CMDQ_EVENT_VENC_CMDQ_PAUSE_DONE		290
-#define CMDQ_EVENT_JPEG_ENC_EOF			291
-#define CMDQ_EVENT_VENC_MB_DONE			292
-#define CMDQ_EVENT_VENC_128BYTE_CNT_DONE	293
-#define CMDQ_EVENT_ISP_FRAME_DONE_A		321
-#define CMDQ_EVENT_ISP_FRAME_DONE_B		322
-#define CMDQ_EVENT_ISP_FRAME_DONE_C		323
-#define CMDQ_EVENT_ISP_CAMSV_0_PASS1_DONE	324
-#define CMDQ_EVENT_ISP_CAMSV_0_2_PASS1_DONE	325
-#define CMDQ_EVENT_ISP_CAMSV_1_PASS1_DONE	326
-#define CMDQ_EVENT_ISP_CAMSV_2_PASS1_DONE	327
-#define CMDQ_EVENT_ISP_CAMSV_3_PASS1_DONE	328
-#define CMDQ_EVENT_ISP_TSF_DONE			329
-#define CMDQ_EVENT_SENINF_0_FIFO_FULL		330
-#define CMDQ_EVENT_SENINF_1_FIFO_FULL		331
-#define CMDQ_EVENT_SENINF_2_FIFO_FULL		332
-#define CMDQ_EVENT_SENINF_3_FIFO_FULL		333
-#define CMDQ_EVENT_SENINF_4_FIFO_FULL		334
-#define CMDQ_EVENT_SENINF_5_FIFO_FULL		335
-#define CMDQ_EVENT_SENINF_6_FIFO_FULL		336
-#define CMDQ_EVENT_SENINF_7_FIFO_FULL		337
-#define CMDQ_EVENT_TG_OVRUN_A_INT_DLY		338
-#define CMDQ_EVENT_TG_OVRUN_B_INT_DLY		339
-#define CMDQ_EVENT_TG_OVRUN_C_INT		340
-#define CMDQ_EVENT_TG_GRABERR_A_INT_DLY		341
-#define CMDQ_EVENT_TG_GRABERR_B_INT_DLY		342
-#define CMDQ_EVENT_TG_GRABERR_C_INT		343
-#define CMDQ_EVENT_CQ_VR_SNAP_A_INT_DLY		344
-#define CMDQ_EVENT_CQ_VR_SNAP_B_INT_DLY		345
-#define CMDQ_EVENT_CQ_VR_SNAP_C_INT		346
-#define CMDQ_EVENT_DMA_R1_ERROR_A_INT_DLY	347
-#define CMDQ_EVENT_DMA_R1_ERROR_B_INT_DLY	348
-#define CMDQ_EVENT_DMA_R1_ERROR_C_INT		349
-#define CMDQ_EVENT_APU_GCE_CORE0_EVENT_0	353
-#define CMDQ_EVENT_APU_GCE_CORE0_EVENT_1	354
-#define CMDQ_EVENT_APU_GCE_CORE0_EVENT_2	355
-#define CMDQ_EVENT_APU_GCE_CORE0_EVENT_3	356
-#define CMDQ_EVENT_APU_GCE_CORE1_EVENT_0	385
-#define CMDQ_EVENT_APU_GCE_CORE1_EVENT_1	386
-#define CMDQ_EVENT_APU_GCE_CORE1_EVENT_2	387
-#define CMDQ_EVENT_APU_GCE_CORE1_EVENT_3	388
-#define CMDQ_EVENT_VDEC_EVENT_0			416
-#define CMDQ_EVENT_VDEC_EVENT_1			417
-#define CMDQ_EVENT_VDEC_EVENT_2			418
-#define CMDQ_EVENT_VDEC_EVENT_3			419
-#define CMDQ_EVENT_VDEC_EVENT_4			420
-#define CMDQ_EVENT_VDEC_EVENT_5			421
-#define CMDQ_EVENT_VDEC_EVENT_6			422
-#define CMDQ_EVENT_VDEC_EVENT_7			423
-#define CMDQ_EVENT_VDEC_EVENT_8			424
-#define CMDQ_EVENT_VDEC_EVENT_9			425
-#define CMDQ_EVENT_VDEC_EVENT_10		426
-#define CMDQ_EVENT_VDEC_EVENT_11		427
-#define CMDQ_EVENT_VDEC_EVENT_12		428
-#define CMDQ_EVENT_VDEC_EVENT_13		429
-#define CMDQ_EVENT_VDEC_EVENT_14		430
-#define CMDQ_EVENT_VDEC_EVENT_15		431
-#define CMDQ_EVENT_FDVT_DONE			449
-#define CMDQ_EVENT_FE_DONE			450
-#define CMDQ_EVENT_RSC_EOF			451
-#define CMDQ_EVENT_DVS_DONE_ASYNC_SHOT		452
-#define CMDQ_EVENT_DVP_DONE_ASYNC_SHOT		453
-#define CMDQ_EVENT_DSI0_TE_INFRA		898
-
-#endif
diff --git a/include/dt-bindings/memory/mt6779-larb-port.h b/include/dt-bindings/memory/mt6779-larb-port.h
deleted file mode 100644
index 3fb438a96e35..000000000000
--- a/include/dt-bindings/memory/mt6779-larb-port.h
+++ /dev/null
@@ -1,206 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 2019 MediaTek Inc.
- * Author: Chao Hao <chao.hao@mediatek.com>
- */
-
-#ifndef _DT_BINDINGS_MEMORY_MT6779_LARB_PORT_H_
-#define _DT_BINDINGS_MEMORY_MT6779_LARB_PORT_H_
-
-#include <dt-bindings/memory/mtk-memory-port.h>
-
-#define M4U_LARB0_ID			 0
-#define M4U_LARB1_ID			 1
-#define M4U_LARB2_ID			 2
-#define M4U_LARB3_ID			 3
-#define M4U_LARB4_ID			 4
-#define M4U_LARB5_ID			 5
-#define M4U_LARB6_ID			 6
-#define M4U_LARB7_ID			 7
-#define M4U_LARB8_ID			 8
-#define M4U_LARB9_ID			 9
-#define M4U_LARB10_ID			 10
-#define M4U_LARB11_ID			 11
-
-/* larb0 */
-#define M4U_PORT_DISP_POSTMASK0		 MTK_M4U_ID(M4U_LARB0_ID, 0)
-#define M4U_PORT_DISP_OVL0_HDR		 MTK_M4U_ID(M4U_LARB0_ID, 1)
-#define M4U_PORT_DISP_OVL1_HDR		 MTK_M4U_ID(M4U_LARB0_ID, 2)
-#define M4U_PORT_DISP_OVL0		 MTK_M4U_ID(M4U_LARB0_ID, 3)
-#define M4U_PORT_DISP_OVL1		 MTK_M4U_ID(M4U_LARB0_ID, 4)
-#define M4U_PORT_DISP_PVRIC0		 MTK_M4U_ID(M4U_LARB0_ID, 5)
-#define M4U_PORT_DISP_RDMA0		 MTK_M4U_ID(M4U_LARB0_ID, 6)
-#define M4U_PORT_DISP_WDMA0		 MTK_M4U_ID(M4U_LARB0_ID, 7)
-#define M4U_PORT_DISP_FAKE0		 MTK_M4U_ID(M4U_LARB0_ID, 8)
-
-/* larb1 */
-#define M4U_PORT_DISP_OVL0_2L_HDR	 MTK_M4U_ID(M4U_LARB1_ID, 0)
-#define M4U_PORT_DISP_OVL1_2L_HDR	 MTK_M4U_ID(M4U_LARB1_ID, 1)
-#define M4U_PORT_DISP_OVL0_2L		 MTK_M4U_ID(M4U_LARB1_ID, 2)
-#define M4U_PORT_DISP_OVL1_2L		 MTK_M4U_ID(M4U_LARB1_ID, 3)
-#define M4U_PORT_DISP_RDMA1		 MTK_M4U_ID(M4U_LARB1_ID, 4)
-#define M4U_PORT_MDP_PVRIC0		 MTK_M4U_ID(M4U_LARB1_ID, 5)
-#define M4U_PORT_MDP_PVRIC1		 MTK_M4U_ID(M4U_LARB1_ID, 6)
-#define M4U_PORT_MDP_RDMA0		 MTK_M4U_ID(M4U_LARB1_ID, 7)
-#define M4U_PORT_MDP_RDMA1		 MTK_M4U_ID(M4U_LARB1_ID, 8)
-#define M4U_PORT_MDP_WROT0_R		 MTK_M4U_ID(M4U_LARB1_ID, 9)
-#define M4U_PORT_MDP_WROT0_W		 MTK_M4U_ID(M4U_LARB1_ID, 10)
-#define M4U_PORT_MDP_WROT1_R		 MTK_M4U_ID(M4U_LARB1_ID, 11)
-#define M4U_PORT_MDP_WROT1_W		 MTK_M4U_ID(M4U_LARB1_ID, 12)
-#define M4U_PORT_DISP_FAKE1		 MTK_M4U_ID(M4U_LARB1_ID, 13)
-
-/* larb2-VDEC */
-#define M4U_PORT_HW_VDEC_MC_EXT          MTK_M4U_ID(M4U_LARB2_ID, 0)
-#define M4U_PORT_HW_VDEC_UFO_EXT         MTK_M4U_ID(M4U_LARB2_ID, 1)
-#define M4U_PORT_HW_VDEC_PP_EXT          MTK_M4U_ID(M4U_LARB2_ID, 2)
-#define M4U_PORT_HW_VDEC_PRED_RD_EXT     MTK_M4U_ID(M4U_LARB2_ID, 3)
-#define M4U_PORT_HW_VDEC_PRED_WR_EXT     MTK_M4U_ID(M4U_LARB2_ID, 4)
-#define M4U_PORT_HW_VDEC_PPWRAP_EXT      MTK_M4U_ID(M4U_LARB2_ID, 5)
-#define M4U_PORT_HW_VDEC_TILE_EXT        MTK_M4U_ID(M4U_LARB2_ID, 6)
-#define M4U_PORT_HW_VDEC_VLD_EXT         MTK_M4U_ID(M4U_LARB2_ID, 7)
-#define M4U_PORT_HW_VDEC_VLD2_EXT        MTK_M4U_ID(M4U_LARB2_ID, 8)
-#define M4U_PORT_HW_VDEC_AVC_MV_EXT      MTK_M4U_ID(M4U_LARB2_ID, 9)
-#define M4U_PORT_HW_VDEC_UFO_ENC_EXT     MTK_M4U_ID(M4U_LARB2_ID, 10)
-#define M4U_PORT_HW_VDEC_RG_CTRL_DMA_EXT MTK_M4U_ID(M4U_LARB2_ID, 11)
-
-/* larb3-VENC */
-#define M4U_PORT_VENC_RCPU		 MTK_M4U_ID(M4U_LARB3_ID, 0)
-#define M4U_PORT_VENC_REC		 MTK_M4U_ID(M4U_LARB3_ID, 1)
-#define M4U_PORT_VENC_BSDMA		 MTK_M4U_ID(M4U_LARB3_ID, 2)
-#define M4U_PORT_VENC_SV_COMV		 MTK_M4U_ID(M4U_LARB3_ID, 3)
-#define M4U_PORT_VENC_RD_COMV		 MTK_M4U_ID(M4U_LARB3_ID, 4)
-#define M4U_PORT_VENC_NBM_RDMA		 MTK_M4U_ID(M4U_LARB3_ID, 5)
-#define M4U_PORT_VENC_NBM_RDMA_LITE	 MTK_M4U_ID(M4U_LARB3_ID, 6)
-#define M4U_PORT_JPGENC_Y_RDMA		 MTK_M4U_ID(M4U_LARB3_ID, 7)
-#define M4U_PORT_JPGENC_C_RDMA		 MTK_M4U_ID(M4U_LARB3_ID, 8)
-#define M4U_PORT_JPGENC_Q_TABLE		 MTK_M4U_ID(M4U_LARB3_ID, 9)
-#define M4U_PORT_JPGENC_BSDMA		 MTK_M4U_ID(M4U_LARB3_ID, 10)
-#define M4U_PORT_JPGDEC_WDMA		 MTK_M4U_ID(M4U_LARB3_ID, 11)
-#define M4U_PORT_JPGDEC_BSDMA		 MTK_M4U_ID(M4U_LARB3_ID, 12)
-#define M4U_PORT_VENC_NBM_WDMA		 MTK_M4U_ID(M4U_LARB3_ID, 13)
-#define M4U_PORT_VENC_NBM_WDMA_LITE	 MTK_M4U_ID(M4U_LARB3_ID, 14)
-#define M4U_PORT_VENC_CUR_LUMA		 MTK_M4U_ID(M4U_LARB3_ID, 15)
-#define M4U_PORT_VENC_CUR_CHROMA	 MTK_M4U_ID(M4U_LARB3_ID, 16)
-#define M4U_PORT_VENC_REF_LUMA		 MTK_M4U_ID(M4U_LARB3_ID, 17)
-#define M4U_PORT_VENC_REF_CHROMA	 MTK_M4U_ID(M4U_LARB3_ID, 18)
-
-/* larb4-dummy */
-
-/* larb5-IMG */
-#define M4U_PORT_IMGI_D1		 MTK_M4U_ID(M4U_LARB5_ID, 0)
-#define M4U_PORT_IMGBI_D1		 MTK_M4U_ID(M4U_LARB5_ID, 1)
-#define M4U_PORT_DMGI_D1		 MTK_M4U_ID(M4U_LARB5_ID, 2)
-#define M4U_PORT_DEPI_D1		 MTK_M4U_ID(M4U_LARB5_ID, 3)
-#define M4U_PORT_LCEI_D1		 MTK_M4U_ID(M4U_LARB5_ID, 4)
-#define M4U_PORT_SMTI_D1		 MTK_M4U_ID(M4U_LARB5_ID, 5)
-#define M4U_PORT_SMTO_D2		 MTK_M4U_ID(M4U_LARB5_ID, 6)
-#define M4U_PORT_SMTO_D1		 MTK_M4U_ID(M4U_LARB5_ID, 7)
-#define M4U_PORT_CRZO_D1		 MTK_M4U_ID(M4U_LARB5_ID, 8)
-#define M4U_PORT_IMG3O_D1		 MTK_M4U_ID(M4U_LARB5_ID, 9)
-#define M4U_PORT_VIPI_D1		 MTK_M4U_ID(M4U_LARB5_ID, 10)
-#define M4U_PORT_WPE_RDMA1		 MTK_M4U_ID(M4U_LARB5_ID, 11)
-#define M4U_PORT_WPE_RDMA0		 MTK_M4U_ID(M4U_LARB5_ID, 12)
-#define M4U_PORT_WPE_WDMA		 MTK_M4U_ID(M4U_LARB5_ID, 13)
-#define M4U_PORT_TIMGO_D1		 MTK_M4U_ID(M4U_LARB5_ID, 14)
-#define M4U_PORT_MFB_RDMA0		 MTK_M4U_ID(M4U_LARB5_ID, 15)
-#define M4U_PORT_MFB_RDMA1		 MTK_M4U_ID(M4U_LARB5_ID, 16)
-#define M4U_PORT_MFB_RDMA2		 MTK_M4U_ID(M4U_LARB5_ID, 17)
-#define M4U_PORT_MFB_RDMA3		 MTK_M4U_ID(M4U_LARB5_ID, 18)
-#define M4U_PORT_MFB_WDMA		 MTK_M4U_ID(M4U_LARB5_ID, 19)
-#define M4U_PORT_RESERVE1		 MTK_M4U_ID(M4U_LARB5_ID, 20)
-#define M4U_PORT_RESERVE2		 MTK_M4U_ID(M4U_LARB5_ID, 21)
-#define M4U_PORT_RESERVE3		 MTK_M4U_ID(M4U_LARB5_ID, 22)
-#define M4U_PORT_RESERVE4		 MTK_M4U_ID(M4U_LARB5_ID, 23)
-#define M4U_PORT_RESERVE5		 MTK_M4U_ID(M4U_LARB5_ID, 24)
-#define M4U_PORT_RESERVE6		 MTK_M4U_ID(M4U_LARB5_ID, 25)
-
-/* larb6-IMG-VPU */
-#define M4U_PORT_IMG_IPUO		 MTK_M4U_ID(M4U_LARB6_ID, 0)
-#define M4U_PORT_IMG_IPU3O		 MTK_M4U_ID(M4U_LARB6_ID, 1)
-#define M4U_PORT_IMG_IPUI		 MTK_M4U_ID(M4U_LARB6_ID, 2)
-
-/* larb7-DVS */
-#define M4U_PORT_DVS_RDMA		 MTK_M4U_ID(M4U_LARB7_ID, 0)
-#define M4U_PORT_DVS_WDMA		 MTK_M4U_ID(M4U_LARB7_ID, 1)
-#define M4U_PORT_DVP_RDMA		 MTK_M4U_ID(M4U_LARB7_ID, 2)
-#define M4U_PORT_DVP_WDMA		 MTK_M4U_ID(M4U_LARB7_ID, 3)
-
-/* larb8-IPESYS */
-#define M4U_PORT_FDVT_RDA		 MTK_M4U_ID(M4U_LARB8_ID, 0)
-#define M4U_PORT_FDVT_RDB		 MTK_M4U_ID(M4U_LARB8_ID, 1)
-#define M4U_PORT_FDVT_WRA		 MTK_M4U_ID(M4U_LARB8_ID, 2)
-#define M4U_PORT_FDVT_WRB		 MTK_M4U_ID(M4U_LARB8_ID, 3)
-#define M4U_PORT_FE_RD0			 MTK_M4U_ID(M4U_LARB8_ID, 4)
-#define M4U_PORT_FE_RD1			 MTK_M4U_ID(M4U_LARB8_ID, 5)
-#define M4U_PORT_FE_WR0			 MTK_M4U_ID(M4U_LARB8_ID, 6)
-#define M4U_PORT_FE_WR1			 MTK_M4U_ID(M4U_LARB8_ID, 7)
-#define M4U_PORT_RSC_RDMA0		 MTK_M4U_ID(M4U_LARB8_ID, 8)
-#define M4U_PORT_RSC_WDMA		 MTK_M4U_ID(M4U_LARB8_ID, 9)
-
-/* larb9-CAM */
-#define M4U_PORT_CAM_IMGO_R1_C		 MTK_M4U_ID(M4U_LARB9_ID, 0)
-#define M4U_PORT_CAM_RRZO_R1_C		 MTK_M4U_ID(M4U_LARB9_ID, 1)
-#define M4U_PORT_CAM_LSCI_R1_C		 MTK_M4U_ID(M4U_LARB9_ID, 2)
-#define M4U_PORT_CAM_BPCI_R1_C		 MTK_M4U_ID(M4U_LARB9_ID, 3)
-#define M4U_PORT_CAM_YUVO_R1_C		 MTK_M4U_ID(M4U_LARB9_ID, 4)
-#define M4U_PORT_CAM_UFDI_R2_C		 MTK_M4U_ID(M4U_LARB9_ID, 5)
-#define M4U_PORT_CAM_RAWI_R2_C		 MTK_M4U_ID(M4U_LARB9_ID, 6)
-#define M4U_PORT_CAM_RAWI_R5_C		 MTK_M4U_ID(M4U_LARB9_ID, 7)
-#define M4U_PORT_CAM_CAMSV_1		 MTK_M4U_ID(M4U_LARB9_ID, 8)
-#define M4U_PORT_CAM_CAMSV_2		 MTK_M4U_ID(M4U_LARB9_ID, 9)
-#define M4U_PORT_CAM_CAMSV_3		 MTK_M4U_ID(M4U_LARB9_ID, 10)
-#define M4U_PORT_CAM_CAMSV_4		 MTK_M4U_ID(M4U_LARB9_ID, 11)
-#define M4U_PORT_CAM_CAMSV_5		 MTK_M4U_ID(M4U_LARB9_ID, 12)
-#define M4U_PORT_CAM_CAMSV_6		 MTK_M4U_ID(M4U_LARB9_ID, 13)
-#define M4U_PORT_CAM_AAO_R1_C		 MTK_M4U_ID(M4U_LARB9_ID, 14)
-#define M4U_PORT_CAM_AFO_R1_C		 MTK_M4U_ID(M4U_LARB9_ID, 15)
-#define M4U_PORT_CAM_FLKO_R1_C		 MTK_M4U_ID(M4U_LARB9_ID, 16)
-#define M4U_PORT_CAM_LCESO_R1_C		 MTK_M4U_ID(M4U_LARB9_ID, 17)
-#define M4U_PORT_CAM_CRZO_R1_C		 MTK_M4U_ID(M4U_LARB9_ID, 18)
-#define M4U_PORT_CAM_LTMSO_R1_C		 MTK_M4U_ID(M4U_LARB9_ID, 19)
-#define M4U_PORT_CAM_RSSO_R1_C		 MTK_M4U_ID(M4U_LARB9_ID, 20)
-#define M4U_PORT_CAM_CCUI		 MTK_M4U_ID(M4U_LARB9_ID, 21)
-#define M4U_PORT_CAM_CCUO		 MTK_M4U_ID(M4U_LARB9_ID, 22)
-#define M4U_PORT_CAM_FAKE		 MTK_M4U_ID(M4U_LARB9_ID, 23)
-
-/* larb10-CAM_A */
-#define M4U_PORT_CAM_IMGO_R1_A		 MTK_M4U_ID(M4U_LARB10_ID, 0)
-#define M4U_PORT_CAM_RRZO_R1_A		 MTK_M4U_ID(M4U_LARB10_ID, 1)
-#define M4U_PORT_CAM_LSCI_R1_A		 MTK_M4U_ID(M4U_LARB10_ID, 2)
-#define M4U_PORT_CAM_BPCI_R1_A		 MTK_M4U_ID(M4U_LARB10_ID, 3)
-#define M4U_PORT_CAM_YUVO_R1_A		 MTK_M4U_ID(M4U_LARB10_ID, 4)
-#define M4U_PORT_CAM_UFDI_R2_A		 MTK_M4U_ID(M4U_LARB10_ID, 5)
-#define M4U_PORT_CAM_RAWI_R2_A		 MTK_M4U_ID(M4U_LARB10_ID, 6)
-#define M4U_PORT_CAM_RAWI_R5_A		 MTK_M4U_ID(M4U_LARB10_ID, 7)
-#define M4U_PORT_CAM_IMGO_R1_B		 MTK_M4U_ID(M4U_LARB10_ID, 8)
-#define M4U_PORT_CAM_RRZO_R1_B		 MTK_M4U_ID(M4U_LARB10_ID, 9)
-#define M4U_PORT_CAM_LSCI_R1_B		 MTK_M4U_ID(M4U_LARB10_ID, 10)
-#define M4U_PORT_CAM_BPCI_R1_B		 MTK_M4U_ID(M4U_LARB10_ID, 11)
-#define M4U_PORT_CAM_YUVO_R1_B		 MTK_M4U_ID(M4U_LARB10_ID, 12)
-#define M4U_PORT_CAM_UFDI_R2_B		 MTK_M4U_ID(M4U_LARB10_ID, 13)
-#define M4U_PORT_CAM_RAWI_R2_B		 MTK_M4U_ID(M4U_LARB10_ID, 14)
-#define M4U_PORT_CAM_RAWI_R5_B		 MTK_M4U_ID(M4U_LARB10_ID, 15)
-#define M4U_PORT_CAM_CAMSV_0		 MTK_M4U_ID(M4U_LARB10_ID, 16)
-#define M4U_PORT_CAM_AAO_R1_A		 MTK_M4U_ID(M4U_LARB10_ID, 17)
-#define M4U_PORT_CAM_AFO_R1_A		 MTK_M4U_ID(M4U_LARB10_ID, 18)
-#define M4U_PORT_CAM_FLKO_R1_A		 MTK_M4U_ID(M4U_LARB10_ID, 19)
-#define M4U_PORT_CAM_LCESO_R1_A		 MTK_M4U_ID(M4U_LARB10_ID, 20)
-#define M4U_PORT_CAM_CRZO_R1_A		 MTK_M4U_ID(M4U_LARB10_ID, 21)
-#define M4U_PORT_CAM_AAO_R1_B		 MTK_M4U_ID(M4U_LARB10_ID, 22)
-#define M4U_PORT_CAM_AFO_R1_B		 MTK_M4U_ID(M4U_LARB10_ID, 23)
-#define M4U_PORT_CAM_FLKO_R1_B		 MTK_M4U_ID(M4U_LARB10_ID, 24)
-#define M4U_PORT_CAM_LCESO_R1_B		 MTK_M4U_ID(M4U_LARB10_ID, 25)
-#define M4U_PORT_CAM_CRZO_R1_B		 MTK_M4U_ID(M4U_LARB10_ID, 26)
-#define M4U_PORT_CAM_LTMSO_R1_A		 MTK_M4U_ID(M4U_LARB10_ID, 27)
-#define M4U_PORT_CAM_RSSO_R1_A		 MTK_M4U_ID(M4U_LARB10_ID, 28)
-#define M4U_PORT_CAM_LTMSO_R1_B		 MTK_M4U_ID(M4U_LARB10_ID, 29)
-#define M4U_PORT_CAM_RSSO_R1_B		 MTK_M4U_ID(M4U_LARB10_ID, 30)
-
-/* larb11-CAM-VPU */
-#define M4U_PORT_CAM_IPUO		 MTK_M4U_ID(M4U_LARB11_ID, 0)
-#define M4U_PORT_CAM_IPU2O		 MTK_M4U_ID(M4U_LARB11_ID, 1)
-#define M4U_PORT_CAM_IPU3O		 MTK_M4U_ID(M4U_LARB11_ID, 2)
-#define M4U_PORT_CAM_IPUI		 MTK_M4U_ID(M4U_LARB11_ID, 3)
-#define M4U_PORT_CAM_IPU2I		 MTK_M4U_ID(M4U_LARB11_ID, 4)
-
-#endif
diff --git a/include/dt-bindings/mux/ti-serdes.h b/include/dt-bindings/mux/ti-serdes.h
deleted file mode 100644
index b0b1091aad6d..000000000000
--- a/include/dt-bindings/mux/ti-serdes.h
+++ /dev/null
@@ -1,190 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * This header provides constants for SERDES MUX for TI SoCs
- */
-
-#ifndef _DT_BINDINGS_MUX_TI_SERDES
-#define _DT_BINDINGS_MUX_TI_SERDES
-
-/*
- * These bindings are deprecated, because they do not match the actual
- * concept of bindings but rather contain pure constants values used only
- * in DTS board files.
- * Instead include the header in the DTS source directory.
- */
-#warning "These bindings are deprecated. Instead, use the header in the DTS source directory."
-
-/* J721E */
-
-#define J721E_SERDES0_LANE0_QSGMII_LANE1	0x0
-#define J721E_SERDES0_LANE0_PCIE0_LANE0		0x1
-#define J721E_SERDES0_LANE0_USB3_0_SWAP		0x2
-#define J721E_SERDES0_LANE0_IP4_UNUSED		0x3
-
-#define J721E_SERDES0_LANE1_QSGMII_LANE2	0x0
-#define J721E_SERDES0_LANE1_PCIE0_LANE1		0x1
-#define J721E_SERDES0_LANE1_USB3_0		0x2
-#define J721E_SERDES0_LANE1_IP4_UNUSED		0x3
-
-#define J721E_SERDES1_LANE0_QSGMII_LANE3	0x0
-#define J721E_SERDES1_LANE0_PCIE1_LANE0		0x1
-#define J721E_SERDES1_LANE0_USB3_1_SWAP		0x2
-#define J721E_SERDES1_LANE0_SGMII_LANE0		0x3
-
-#define J721E_SERDES1_LANE1_QSGMII_LANE4	0x0
-#define J721E_SERDES1_LANE1_PCIE1_LANE1		0x1
-#define J721E_SERDES1_LANE1_USB3_1		0x2
-#define J721E_SERDES1_LANE1_SGMII_LANE1		0x3
-
-#define J721E_SERDES2_LANE0_IP1_UNUSED		0x0
-#define J721E_SERDES2_LANE0_PCIE2_LANE0		0x1
-#define J721E_SERDES2_LANE0_USB3_1_SWAP		0x2
-#define J721E_SERDES2_LANE0_SGMII_LANE0		0x3
-
-#define J721E_SERDES2_LANE1_IP1_UNUSED		0x0
-#define J721E_SERDES2_LANE1_PCIE2_LANE1		0x1
-#define J721E_SERDES2_LANE1_USB3_1		0x2
-#define J721E_SERDES2_LANE1_SGMII_LANE1		0x3
-
-#define J721E_SERDES3_LANE0_IP1_UNUSED		0x0
-#define J721E_SERDES3_LANE0_PCIE3_LANE0		0x1
-#define J721E_SERDES3_LANE0_USB3_0_SWAP		0x2
-#define J721E_SERDES3_LANE0_IP4_UNUSED		0x3
-
-#define J721E_SERDES3_LANE1_IP1_UNUSED		0x0
-#define J721E_SERDES3_LANE1_PCIE3_LANE1		0x1
-#define J721E_SERDES3_LANE1_USB3_0		0x2
-#define J721E_SERDES3_LANE1_IP4_UNUSED		0x3
-
-#define J721E_SERDES4_LANE0_EDP_LANE0		0x0
-#define J721E_SERDES4_LANE0_IP2_UNUSED		0x1
-#define J721E_SERDES4_LANE0_QSGMII_LANE5	0x2
-#define J721E_SERDES4_LANE0_IP4_UNUSED		0x3
-
-#define J721E_SERDES4_LANE1_EDP_LANE1		0x0
-#define J721E_SERDES4_LANE1_IP2_UNUSED		0x1
-#define J721E_SERDES4_LANE1_QSGMII_LANE6	0x2
-#define J721E_SERDES4_LANE1_IP4_UNUSED		0x3
-
-#define J721E_SERDES4_LANE2_EDP_LANE2		0x0
-#define J721E_SERDES4_LANE2_IP2_UNUSED		0x1
-#define J721E_SERDES4_LANE2_QSGMII_LANE7	0x2
-#define J721E_SERDES4_LANE2_IP4_UNUSED		0x3
-
-#define J721E_SERDES4_LANE3_EDP_LANE3		0x0
-#define J721E_SERDES4_LANE3_IP2_UNUSED		0x1
-#define J721E_SERDES4_LANE3_QSGMII_LANE8	0x2
-#define J721E_SERDES4_LANE3_IP4_UNUSED		0x3
-
-/* J7200 */
-
-#define J7200_SERDES0_LANE0_QSGMII_LANE3	0x0
-#define J7200_SERDES0_LANE0_PCIE1_LANE0		0x1
-#define J7200_SERDES0_LANE0_IP3_UNUSED		0x2
-#define J7200_SERDES0_LANE0_IP4_UNUSED		0x3
-
-#define J7200_SERDES0_LANE1_QSGMII_LANE4	0x0
-#define J7200_SERDES0_LANE1_PCIE1_LANE1		0x1
-#define J7200_SERDES0_LANE1_IP3_UNUSED		0x2
-#define J7200_SERDES0_LANE1_IP4_UNUSED		0x3
-
-#define J7200_SERDES0_LANE2_QSGMII_LANE1	0x0
-#define J7200_SERDES0_LANE2_PCIE1_LANE2		0x1
-#define J7200_SERDES0_LANE2_IP3_UNUSED		0x2
-#define J7200_SERDES0_LANE2_IP4_UNUSED		0x3
-
-#define J7200_SERDES0_LANE3_QSGMII_LANE2	0x0
-#define J7200_SERDES0_LANE3_PCIE1_LANE3		0x1
-#define J7200_SERDES0_LANE3_USB			0x2
-#define J7200_SERDES0_LANE3_IP4_UNUSED		0x3
-
-/* AM64 */
-
-#define AM64_SERDES0_LANE0_PCIE0		0x0
-#define AM64_SERDES0_LANE0_USB			0x1
-
-/* J721S2 */
-
-#define J721S2_SERDES0_LANE0_EDP_LANE0		0x0
-#define J721S2_SERDES0_LANE0_PCIE1_LANE0	0x1
-#define J721S2_SERDES0_LANE0_IP3_UNUSED		0x2
-#define J721S2_SERDES0_LANE0_IP4_UNUSED		0x3
-
-#define J721S2_SERDES0_LANE1_EDP_LANE1		0x0
-#define J721S2_SERDES0_LANE1_PCIE1_LANE1	0x1
-#define J721S2_SERDES0_LANE1_USB		0x2
-#define J721S2_SERDES0_LANE1_IP4_UNUSED		0x3
-
-#define J721S2_SERDES0_LANE2_EDP_LANE2		0x0
-#define J721S2_SERDES0_LANE2_PCIE1_LANE2	0x1
-#define J721S2_SERDES0_LANE2_IP3_UNUSED		0x2
-#define J721S2_SERDES0_LANE2_IP4_UNUSED		0x3
-
-#define J721S2_SERDES0_LANE3_EDP_LANE3		0x0
-#define J721S2_SERDES0_LANE3_PCIE1_LANE3	0x1
-#define J721S2_SERDES0_LANE3_USB		0x2
-#define J721S2_SERDES0_LANE3_IP4_UNUSED		0x3
-
-/* J784S4 */
-
-#define J784S4_SERDES0_LANE0_IP1_UNUSED		0x0
-#define J784S4_SERDES0_LANE0_PCIE1_LANE0	0x1
-#define J784S4_SERDES0_LANE0_IP3_UNUSED		0x2
-#define J784S4_SERDES0_LANE0_IP4_UNUSED		0x3
-
-#define J784S4_SERDES0_LANE1_IP1_UNUSED		0x0
-#define J784S4_SERDES0_LANE1_PCIE1_LANE1	0x1
-#define J784S4_SERDES0_LANE1_IP3_UNUSED		0x2
-#define J784S4_SERDES0_LANE1_IP4_UNUSED		0x3
-
-#define J784S4_SERDES0_LANE2_PCIE3_LANE0	0x0
-#define J784S4_SERDES0_LANE2_PCIE1_LANE2	0x1
-#define J784S4_SERDES0_LANE2_IP3_UNUSED		0x2
-#define J784S4_SERDES0_LANE2_IP4_UNUSED		0x3
-
-#define J784S4_SERDES0_LANE3_PCIE3_LANE1	0x0
-#define J784S4_SERDES0_LANE3_PCIE1_LANE3	0x1
-#define J784S4_SERDES0_LANE3_USB		0x2
-#define J784S4_SERDES0_LANE3_IP4_UNUSED		0x3
-
-#define J784S4_SERDES1_LANE0_QSGMII_LANE3	0x0
-#define J784S4_SERDES1_LANE0_PCIE0_LANE0	0x1
-#define J784S4_SERDES1_LANE0_IP3_UNUSED		0x2
-#define J784S4_SERDES1_LANE0_IP4_UNUSED		0x3
-
-#define J784S4_SERDES1_LANE1_QSGMII_LANE4	0x0
-#define J784S4_SERDES1_LANE1_PCIE0_LANE1	0x1
-#define J784S4_SERDES1_LANE1_IP3_UNUSED		0x2
-#define J784S4_SERDES1_LANE1_IP4_UNUSED		0x3
-
-#define J784S4_SERDES1_LANE2_QSGMII_LANE1	0x0
-#define J784S4_SERDES1_LANE2_PCIE0_LANE2	0x1
-#define J784S4_SERDES1_LANE2_PCIE2_LANE0	0x2
-#define J784S4_SERDES1_LANE2_IP4_UNUSED		0x3
-
-#define J784S4_SERDES1_LANE3_QSGMII_LANE2	0x0
-#define J784S4_SERDES1_LANE3_PCIE0_LANE3	0x1
-#define J784S4_SERDES1_LANE3_PCIE2_LANE1	0x2
-#define J784S4_SERDES1_LANE3_IP4_UNUSED		0x3
-
-#define J784S4_SERDES2_LANE0_QSGMII_LANE5	0x0
-#define J784S4_SERDES2_LANE0_IP2_UNUSED		0x1
-#define J784S4_SERDES2_LANE0_IP3_UNUSED		0x2
-#define J784S4_SERDES2_LANE0_IP4_UNUSED		0x3
-
-#define J784S4_SERDES2_LANE1_QSGMII_LANE6	0x0
-#define J784S4_SERDES2_LANE1_IP2_UNUSED		0x1
-#define J784S4_SERDES2_LANE1_IP3_UNUSED		0x2
-#define J784S4_SERDES2_LANE1_IP4_UNUSED		0x3
-
-#define J784S4_SERDES2_LANE2_QSGMII_LANE7	0x0
-#define J784S4_SERDES2_LANE2_QSGMII_LANE1	0x1
-#define J784S4_SERDES2_LANE2_IP3_UNUSED		0x2
-#define J784S4_SERDES2_LANE2_IP4_UNUSED		0x3
-
-#define J784S4_SERDES2_LANE3_QSGMII_LANE8	0x0
-#define J784S4_SERDES2_LANE3_QSGMII_LANE2	0x1
-#define J784S4_SERDES2_LANE3_IP3_UNUSED		0x2
-#define J784S4_SERDES2_LANE3_IP4_UNUSED		0x3
-
-#endif /* _DT_BINDINGS_MUX_TI_SERDES */
diff --git a/include/dt-bindings/pinctrl/mt6397-pinfunc.h b/include/dt-bindings/pinctrl/mt6397-pinfunc.h
deleted file mode 100644
index f393fbd68905..000000000000
--- a/include/dt-bindings/pinctrl/mt6397-pinfunc.h
+++ /dev/null
@@ -1,257 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __DTS_MT6397_PINFUNC_H
-#define __DTS_MT6397_PINFUNC_H
-
-#include <dt-bindings/pinctrl/mt65xx.h>
-
-#define MT6397_PIN_0_INT__FUNC_GPIO0 (MTK_PIN_NO(0) | 0)
-#define MT6397_PIN_0_INT__FUNC_INT (MTK_PIN_NO(0) | 1)
-
-#define MT6397_PIN_1_SRCVOLTEN__FUNC_GPIO1 (MTK_PIN_NO(1) | 0)
-#define MT6397_PIN_1_SRCVOLTEN__FUNC_SRCVOLTEN (MTK_PIN_NO(1) | 1)
-#define MT6397_PIN_1_SRCVOLTEN__FUNC_TEST_CK1 (MTK_PIN_NO(1) | 6)
-
-#define MT6397_PIN_2_SRCLKEN_PERI__FUNC_GPIO2 (MTK_PIN_NO(2) | 0)
-#define MT6397_PIN_2_SRCLKEN_PERI__FUNC_SRCLKEN_PERI (MTK_PIN_NO(2) | 1)
-#define MT6397_PIN_2_SRCLKEN_PERI__FUNC_TEST_CK2 (MTK_PIN_NO(2) | 6)
-
-#define MT6397_PIN_3_RTC_32K1V8__FUNC_GPIO3 (MTK_PIN_NO(3) | 0)
-#define MT6397_PIN_3_RTC_32K1V8__FUNC_RTC_32K1V8 (MTK_PIN_NO(3) | 1)
-#define MT6397_PIN_3_RTC_32K1V8__FUNC_TEST_CK3 (MTK_PIN_NO(3) | 6)
-
-#define MT6397_PIN_4_WRAP_EVENT__FUNC_GPIO4 (MTK_PIN_NO(4) | 0)
-#define MT6397_PIN_4_WRAP_EVENT__FUNC_WRAP_EVENT (MTK_PIN_NO(4) | 1)
-
-#define MT6397_PIN_5_SPI_CLK__FUNC_GPIO5 (MTK_PIN_NO(5) | 0)
-#define MT6397_PIN_5_SPI_CLK__FUNC_SPI_CLK (MTK_PIN_NO(5) | 1)
-
-#define MT6397_PIN_6_SPI_CSN__FUNC_GPIO6 (MTK_PIN_NO(6) | 0)
-#define MT6397_PIN_6_SPI_CSN__FUNC_SPI_CSN (MTK_PIN_NO(6) | 1)
-
-#define MT6397_PIN_7_SPI_MOSI__FUNC_GPIO7 (MTK_PIN_NO(7) | 0)
-#define MT6397_PIN_7_SPI_MOSI__FUNC_SPI_MOSI (MTK_PIN_NO(7) | 1)
-
-#define MT6397_PIN_8_SPI_MISO__FUNC_GPIO8 (MTK_PIN_NO(8) | 0)
-#define MT6397_PIN_8_SPI_MISO__FUNC_SPI_MISO (MTK_PIN_NO(8) | 1)
-
-#define MT6397_PIN_9_AUD_CLK_MOSI__FUNC_GPIO9 (MTK_PIN_NO(9) | 0)
-#define MT6397_PIN_9_AUD_CLK_MOSI__FUNC_AUD_CLK (MTK_PIN_NO(9) | 1)
-#define MT6397_PIN_9_AUD_CLK_MOSI__FUNC_TEST_IN0 (MTK_PIN_NO(9) | 6)
-#define MT6397_PIN_9_AUD_CLK_MOSI__FUNC_TEST_OUT0 (MTK_PIN_NO(9) | 7)
-
-#define MT6397_PIN_10_AUD_DAT_MISO__FUNC_GPIO10 (MTK_PIN_NO(10) | 0)
-#define MT6397_PIN_10_AUD_DAT_MISO__FUNC_AUD_MISO (MTK_PIN_NO(10) | 1)
-#define MT6397_PIN_10_AUD_DAT_MISO__FUNC_TEST_IN1 (MTK_PIN_NO(10) | 6)
-#define MT6397_PIN_10_AUD_DAT_MISO__FUNC_TEST_OUT1 (MTK_PIN_NO(10) | 7)
-
-#define MT6397_PIN_11_AUD_DAT_MOSI__FUNC_GPIO11 (MTK_PIN_NO(11) | 0)
-#define MT6397_PIN_11_AUD_DAT_MOSI__FUNC_AUD_MOSI (MTK_PIN_NO(11) | 1)
-#define MT6397_PIN_11_AUD_DAT_MOSI__FUNC_TEST_IN2 (MTK_PIN_NO(11) | 6)
-#define MT6397_PIN_11_AUD_DAT_MOSI__FUNC_TEST_OUT2 (MTK_PIN_NO(11) | 7)
-
-#define MT6397_PIN_12_COL0__FUNC_GPIO12 (MTK_PIN_NO(12) | 0)
-#define MT6397_PIN_12_COL0__FUNC_COL0_USBDL (MTK_PIN_NO(12) | 1)
-#define MT6397_PIN_12_COL0__FUNC_EINT10_1X (MTK_PIN_NO(12) | 2)
-#define MT6397_PIN_12_COL0__FUNC_PWM1_3X (MTK_PIN_NO(12) | 3)
-#define MT6397_PIN_12_COL0__FUNC_TEST_IN3 (MTK_PIN_NO(12) | 6)
-#define MT6397_PIN_12_COL0__FUNC_TEST_OUT3 (MTK_PIN_NO(12) | 7)
-
-#define MT6397_PIN_13_COL1__FUNC_GPIO13 (MTK_PIN_NO(13) | 0)
-#define MT6397_PIN_13_COL1__FUNC_COL1 (MTK_PIN_NO(13) | 1)
-#define MT6397_PIN_13_COL1__FUNC_EINT11_1X (MTK_PIN_NO(13) | 2)
-#define MT6397_PIN_13_COL1__FUNC_SCL0_2X (MTK_PIN_NO(13) | 3)
-#define MT6397_PIN_13_COL1__FUNC_TEST_IN4 (MTK_PIN_NO(13) | 6)
-#define MT6397_PIN_13_COL1__FUNC_TEST_OUT4 (MTK_PIN_NO(13) | 7)
-
-#define MT6397_PIN_14_COL2__FUNC_GPIO14 (MTK_PIN_NO(14) | 0)
-#define MT6397_PIN_14_COL2__FUNC_COL2 (MTK_PIN_NO(14) | 1)
-#define MT6397_PIN_14_COL2__FUNC_EINT12_1X (MTK_PIN_NO(14) | 2)
-#define MT6397_PIN_14_COL2__FUNC_SDA0_2X (MTK_PIN_NO(14) | 3)
-#define MT6397_PIN_14_COL2__FUNC_TEST_IN5 (MTK_PIN_NO(14) | 6)
-#define MT6397_PIN_14_COL2__FUNC_TEST_OUT5 (MTK_PIN_NO(14) | 7)
-
-#define MT6397_PIN_15_COL3__FUNC_GPIO15 (MTK_PIN_NO(15) | 0)
-#define MT6397_PIN_15_COL3__FUNC_COL3 (MTK_PIN_NO(15) | 1)
-#define MT6397_PIN_15_COL3__FUNC_EINT13_1X (MTK_PIN_NO(15) | 2)
-#define MT6397_PIN_15_COL3__FUNC_SCL1_2X (MTK_PIN_NO(15) | 3)
-#define MT6397_PIN_15_COL3__FUNC_TEST_IN6 (MTK_PIN_NO(15) | 6)
-#define MT6397_PIN_15_COL3__FUNC_TEST_OUT6 (MTK_PIN_NO(15) | 7)
-
-#define MT6397_PIN_16_COL4__FUNC_GPIO16 (MTK_PIN_NO(16) | 0)
-#define MT6397_PIN_16_COL4__FUNC_COL4 (MTK_PIN_NO(16) | 1)
-#define MT6397_PIN_16_COL4__FUNC_EINT14_1X (MTK_PIN_NO(16) | 2)
-#define MT6397_PIN_16_COL4__FUNC_SDA1_2X (MTK_PIN_NO(16) | 3)
-#define MT6397_PIN_16_COL4__FUNC_TEST_IN7 (MTK_PIN_NO(16) | 6)
-#define MT6397_PIN_16_COL4__FUNC_TEST_OUT7 (MTK_PIN_NO(16) | 7)
-
-#define MT6397_PIN_17_COL5__FUNC_GPIO17 (MTK_PIN_NO(17) | 0)
-#define MT6397_PIN_17_COL5__FUNC_COL5 (MTK_PIN_NO(17) | 1)
-#define MT6397_PIN_17_COL5__FUNC_EINT15_1X (MTK_PIN_NO(17) | 2)
-#define MT6397_PIN_17_COL5__FUNC_SCL2_2X (MTK_PIN_NO(17) | 3)
-#define MT6397_PIN_17_COL5__FUNC_TEST_IN8 (MTK_PIN_NO(17) | 6)
-#define MT6397_PIN_17_COL5__FUNC_TEST_OUT8 (MTK_PIN_NO(17) | 7)
-
-#define MT6397_PIN_18_COL6__FUNC_GPIO18 (MTK_PIN_NO(18) | 0)
-#define MT6397_PIN_18_COL6__FUNC_COL6 (MTK_PIN_NO(18) | 1)
-#define MT6397_PIN_18_COL6__FUNC_EINT16_1X (MTK_PIN_NO(18) | 2)
-#define MT6397_PIN_18_COL6__FUNC_SDA2_2X (MTK_PIN_NO(18) | 3)
-#define MT6397_PIN_18_COL6__FUNC_GPIO32K_0 (MTK_PIN_NO(18) | 4)
-#define MT6397_PIN_18_COL6__FUNC_GPIO26M_0 (MTK_PIN_NO(18) | 5)
-#define MT6397_PIN_18_COL6__FUNC_TEST_IN9 (MTK_PIN_NO(18) | 6)
-#define MT6397_PIN_18_COL6__FUNC_TEST_OUT9 (MTK_PIN_NO(18) | 7)
-
-#define MT6397_PIN_19_COL7__FUNC_GPIO19 (MTK_PIN_NO(19) | 0)
-#define MT6397_PIN_19_COL7__FUNC_COL7 (MTK_PIN_NO(19) | 1)
-#define MT6397_PIN_19_COL7__FUNC_EINT17_1X (MTK_PIN_NO(19) | 2)
-#define MT6397_PIN_19_COL7__FUNC_PWM2_3X (MTK_PIN_NO(19) | 3)
-#define MT6397_PIN_19_COL7__FUNC_GPIO32K_1 (MTK_PIN_NO(19) | 4)
-#define MT6397_PIN_19_COL7__FUNC_GPIO26M_1 (MTK_PIN_NO(19) | 5)
-#define MT6397_PIN_19_COL7__FUNC_TEST_IN10 (MTK_PIN_NO(19) | 6)
-#define MT6397_PIN_19_COL7__FUNC_TEST_OUT10 (MTK_PIN_NO(19) | 7)
-
-#define MT6397_PIN_20_ROW0__FUNC_GPIO20 (MTK_PIN_NO(20) | 0)
-#define MT6397_PIN_20_ROW0__FUNC_ROW0 (MTK_PIN_NO(20) | 1)
-#define MT6397_PIN_20_ROW0__FUNC_EINT18_1X (MTK_PIN_NO(20) | 2)
-#define MT6397_PIN_20_ROW0__FUNC_SCL0_3X (MTK_PIN_NO(20) | 3)
-#define MT6397_PIN_20_ROW0__FUNC_TEST_IN11 (MTK_PIN_NO(20) | 6)
-#define MT6397_PIN_20_ROW0__FUNC_TEST_OUT11 (MTK_PIN_NO(20) | 7)
-
-#define MT6397_PIN_21_ROW1__FUNC_GPIO21 (MTK_PIN_NO(21) | 0)
-#define MT6397_PIN_21_ROW1__FUNC_ROW1 (MTK_PIN_NO(21) | 1)
-#define MT6397_PIN_21_ROW1__FUNC_EINT19_1X (MTK_PIN_NO(21) | 2)
-#define MT6397_PIN_21_ROW1__FUNC_SDA0_3X (MTK_PIN_NO(21) | 3)
-#define MT6397_PIN_21_ROW1__FUNC_AUD_TSTCK (MTK_PIN_NO(21) | 4)
-#define MT6397_PIN_21_ROW1__FUNC_TEST_IN12 (MTK_PIN_NO(21) | 6)
-#define MT6397_PIN_21_ROW1__FUNC_TEST_OUT12 (MTK_PIN_NO(21) | 7)
-
-#define MT6397_PIN_22_ROW2__FUNC_GPIO22 (MTK_PIN_NO(22) | 0)
-#define MT6397_PIN_22_ROW2__FUNC_ROW2 (MTK_PIN_NO(22) | 1)
-#define MT6397_PIN_22_ROW2__FUNC_EINT20_1X (MTK_PIN_NO(22) | 2)
-#define MT6397_PIN_22_ROW2__FUNC_SCL1_3X (MTK_PIN_NO(22) | 3)
-#define MT6397_PIN_22_ROW2__FUNC_TEST_IN13 (MTK_PIN_NO(22) | 6)
-#define MT6397_PIN_22_ROW2__FUNC_TEST_OUT13 (MTK_PIN_NO(22) | 7)
-
-#define MT6397_PIN_23_ROW3__FUNC_GPIO23 (MTK_PIN_NO(23) | 0)
-#define MT6397_PIN_23_ROW3__FUNC_ROW3 (MTK_PIN_NO(23) | 1)
-#define MT6397_PIN_23_ROW3__FUNC_EINT21_1X (MTK_PIN_NO(23) | 2)
-#define MT6397_PIN_23_ROW3__FUNC_SDA1_3X (MTK_PIN_NO(23) | 3)
-#define MT6397_PIN_23_ROW3__FUNC_TEST_IN14 (MTK_PIN_NO(23) | 6)
-#define MT6397_PIN_23_ROW3__FUNC_TEST_OUT14 (MTK_PIN_NO(23) | 7)
-
-#define MT6397_PIN_24_ROW4__FUNC_GPIO24 (MTK_PIN_NO(24) | 0)
-#define MT6397_PIN_24_ROW4__FUNC_ROW4 (MTK_PIN_NO(24) | 1)
-#define MT6397_PIN_24_ROW4__FUNC_EINT22_1X (MTK_PIN_NO(24) | 2)
-#define MT6397_PIN_24_ROW4__FUNC_SCL2_3X (MTK_PIN_NO(24) | 3)
-#define MT6397_PIN_24_ROW4__FUNC_TEST_IN15 (MTK_PIN_NO(24) | 6)
-#define MT6397_PIN_24_ROW4__FUNC_TEST_OUT15 (MTK_PIN_NO(24) | 7)
-
-#define MT6397_PIN_25_ROW5__FUNC_GPIO25 (MTK_PIN_NO(25) | 0)
-#define MT6397_PIN_25_ROW5__FUNC_ROW5 (MTK_PIN_NO(25) | 1)
-#define MT6397_PIN_25_ROW5__FUNC_EINT23_1X (MTK_PIN_NO(25) | 2)
-#define MT6397_PIN_25_ROW5__FUNC_SDA2_3X (MTK_PIN_NO(25) | 3)
-#define MT6397_PIN_25_ROW5__FUNC_TEST_IN16 (MTK_PIN_NO(25) | 6)
-#define MT6397_PIN_25_ROW5__FUNC_TEST_OUT16 (MTK_PIN_NO(25) | 7)
-
-#define MT6397_PIN_26_ROW6__FUNC_GPIO26 (MTK_PIN_NO(26) | 0)
-#define MT6397_PIN_26_ROW6__FUNC_ROW6 (MTK_PIN_NO(26) | 1)
-#define MT6397_PIN_26_ROW6__FUNC_EINT24_1X (MTK_PIN_NO(26) | 2)
-#define MT6397_PIN_26_ROW6__FUNC_PWM3_3X (MTK_PIN_NO(26) | 3)
-#define MT6397_PIN_26_ROW6__FUNC_GPIO32K_2 (MTK_PIN_NO(26) | 4)
-#define MT6397_PIN_26_ROW6__FUNC_GPIO26M_2 (MTK_PIN_NO(26) | 5)
-#define MT6397_PIN_26_ROW6__FUNC_TEST_IN17 (MTK_PIN_NO(26) | 6)
-#define MT6397_PIN_26_ROW6__FUNC_TEST_OUT17 (MTK_PIN_NO(26) | 7)
-
-#define MT6397_PIN_27_ROW7__FUNC_GPIO27 (MTK_PIN_NO(27) | 0)
-#define MT6397_PIN_27_ROW7__FUNC_ROW7 (MTK_PIN_NO(27) | 1)
-#define MT6397_PIN_27_ROW7__FUNC_EINT3_1X (MTK_PIN_NO(27) | 2)
-#define MT6397_PIN_27_ROW7__FUNC_CBUS (MTK_PIN_NO(27) | 3)
-#define MT6397_PIN_27_ROW7__FUNC_GPIO32K_3 (MTK_PIN_NO(27) | 4)
-#define MT6397_PIN_27_ROW7__FUNC_GPIO26M_3 (MTK_PIN_NO(27) | 5)
-#define MT6397_PIN_27_ROW7__FUNC_TEST_IN18 (MTK_PIN_NO(27) | 6)
-#define MT6397_PIN_27_ROW7__FUNC_TEST_OUT18 (MTK_PIN_NO(27) | 7)
-
-#define MT6397_PIN_28_PWM1__FUNC_GPIO28 (MTK_PIN_NO(28) | 0)
-#define MT6397_PIN_28_PWM1__FUNC_PWM1 (MTK_PIN_NO(28) | 1)
-#define MT6397_PIN_28_PWM1__FUNC_EINT4_1X (MTK_PIN_NO(28) | 2)
-#define MT6397_PIN_28_PWM1__FUNC_GPIO32K_4 (MTK_PIN_NO(28) | 4)
-#define MT6397_PIN_28_PWM1__FUNC_GPIO26M_4 (MTK_PIN_NO(28) | 5)
-#define MT6397_PIN_28_PWM1__FUNC_TEST_IN19 (MTK_PIN_NO(28) | 6)
-#define MT6397_PIN_28_PWM1__FUNC_TEST_OUT19 (MTK_PIN_NO(28) | 7)
-
-#define MT6397_PIN_29_PWM2__FUNC_GPIO29 (MTK_PIN_NO(29) | 0)
-#define MT6397_PIN_29_PWM2__FUNC_PWM2 (MTK_PIN_NO(29) | 1)
-#define MT6397_PIN_29_PWM2__FUNC_EINT5_1X (MTK_PIN_NO(29) | 2)
-#define MT6397_PIN_29_PWM2__FUNC_GPIO32K_5 (MTK_PIN_NO(29) | 4)
-#define MT6397_PIN_29_PWM2__FUNC_GPIO26M_5 (MTK_PIN_NO(29) | 5)
-#define MT6397_PIN_29_PWM2__FUNC_TEST_IN20 (MTK_PIN_NO(29) | 6)
-#define MT6397_PIN_29_PWM2__FUNC_TEST_OUT20 (MTK_PIN_NO(29) | 7)
-
-#define MT6397_PIN_30_PWM3__FUNC_GPIO30 (MTK_PIN_NO(30) | 0)
-#define MT6397_PIN_30_PWM3__FUNC_PWM3 (MTK_PIN_NO(30) | 1)
-#define MT6397_PIN_30_PWM3__FUNC_EINT6_1X (MTK_PIN_NO(30) | 2)
-#define MT6397_PIN_30_PWM3__FUNC_COL0 (MTK_PIN_NO(30) | 3)
-#define MT6397_PIN_30_PWM3__FUNC_GPIO32K_6 (MTK_PIN_NO(30) | 4)
-#define MT6397_PIN_30_PWM3__FUNC_GPIO26M_6 (MTK_PIN_NO(30) | 5)
-#define MT6397_PIN_30_PWM3__FUNC_TEST_IN21 (MTK_PIN_NO(30) | 6)
-#define MT6397_PIN_30_PWM3__FUNC_TEST_OUT21 (MTK_PIN_NO(30) | 7)
-
-#define MT6397_PIN_31_SCL0__FUNC_GPIO31 (MTK_PIN_NO(31) | 0)
-#define MT6397_PIN_31_SCL0__FUNC_SCL0 (MTK_PIN_NO(31) | 1)
-#define MT6397_PIN_31_SCL0__FUNC_EINT7_1X (MTK_PIN_NO(31) | 2)
-#define MT6397_PIN_31_SCL0__FUNC_PWM1_2X (MTK_PIN_NO(31) | 3)
-#define MT6397_PIN_31_SCL0__FUNC_TEST_IN22 (MTK_PIN_NO(31) | 6)
-#define MT6397_PIN_31_SCL0__FUNC_TEST_OUT22 (MTK_PIN_NO(31) | 7)
-
-#define MT6397_PIN_32_SDA0__FUNC_GPIO32 (MTK_PIN_NO(32) | 0)
-#define MT6397_PIN_32_SDA0__FUNC_SDA0 (MTK_PIN_NO(32) | 1)
-#define MT6397_PIN_32_SDA0__FUNC_EINT8_1X (MTK_PIN_NO(32) | 2)
-#define MT6397_PIN_32_SDA0__FUNC_TEST_IN23 (MTK_PIN_NO(32) | 6)
-#define MT6397_PIN_32_SDA0__FUNC_TEST_OUT23 (MTK_PIN_NO(32) | 7)
-
-#define MT6397_PIN_33_SCL1__FUNC_GPIO33 (MTK_PIN_NO(33) | 0)
-#define MT6397_PIN_33_SCL1__FUNC_SCL1 (MTK_PIN_NO(33) | 1)
-#define MT6397_PIN_33_SCL1__FUNC_EINT9_1X (MTK_PIN_NO(33) | 2)
-#define MT6397_PIN_33_SCL1__FUNC_PWM2_2X (MTK_PIN_NO(33) | 3)
-#define MT6397_PIN_33_SCL1__FUNC_TEST_IN24 (MTK_PIN_NO(33) | 6)
-#define MT6397_PIN_33_SCL1__FUNC_TEST_OUT24 (MTK_PIN_NO(33) | 7)
-
-#define MT6397_PIN_34_SDA1__FUNC_GPIO34 (MTK_PIN_NO(34) | 0)
-#define MT6397_PIN_34_SDA1__FUNC_SDA1 (MTK_PIN_NO(34) | 1)
-#define MT6397_PIN_34_SDA1__FUNC_EINT0_1X (MTK_PIN_NO(34) | 2)
-#define MT6397_PIN_34_SDA1__FUNC_TEST_IN25 (MTK_PIN_NO(34) | 6)
-#define MT6397_PIN_34_SDA1__FUNC_TEST_OUT25 (MTK_PIN_NO(34) | 7)
-
-#define MT6397_PIN_35_SCL2__FUNC_GPIO35 (MTK_PIN_NO(35) | 0)
-#define MT6397_PIN_35_SCL2__FUNC_SCL2 (MTK_PIN_NO(35) | 1)
-#define MT6397_PIN_35_SCL2__FUNC_EINT1_1X (MTK_PIN_NO(35) | 2)
-#define MT6397_PIN_35_SCL2__FUNC_PWM3_2X (MTK_PIN_NO(35) | 3)
-#define MT6397_PIN_35_SCL2__FUNC_TEST_IN26 (MTK_PIN_NO(35) | 6)
-#define MT6397_PIN_35_SCL2__FUNC_TEST_OUT26 (MTK_PIN_NO(35) | 7)
-
-#define MT6397_PIN_36_SDA2__FUNC_GPIO36 (MTK_PIN_NO(36) | 0)
-#define MT6397_PIN_36_SDA2__FUNC_SDA2 (MTK_PIN_NO(36) | 1)
-#define MT6397_PIN_36_SDA2__FUNC_EINT2_1X (MTK_PIN_NO(36) | 2)
-#define MT6397_PIN_36_SDA2__FUNC_TEST_IN27 (MTK_PIN_NO(36) | 6)
-#define MT6397_PIN_36_SDA2__FUNC_TEST_OUT27 (MTK_PIN_NO(36) | 7)
-
-#define MT6397_PIN_37_HDMISD__FUNC_GPIO37 (MTK_PIN_NO(37) | 0)
-#define MT6397_PIN_37_HDMISD__FUNC_HDMISD (MTK_PIN_NO(37) | 1)
-#define MT6397_PIN_37_HDMISD__FUNC_TEST_IN28 (MTK_PIN_NO(37) | 6)
-#define MT6397_PIN_37_HDMISD__FUNC_TEST_OUT28 (MTK_PIN_NO(37) | 7)
-
-#define MT6397_PIN_38_HDMISCK__FUNC_GPIO38 (MTK_PIN_NO(38) | 0)
-#define MT6397_PIN_38_HDMISCK__FUNC_HDMISCK (MTK_PIN_NO(38) | 1)
-#define MT6397_PIN_38_HDMISCK__FUNC_TEST_IN29 (MTK_PIN_NO(38) | 6)
-#define MT6397_PIN_38_HDMISCK__FUNC_TEST_OUT29 (MTK_PIN_NO(38) | 7)
-
-#define MT6397_PIN_39_HTPLG__FUNC_GPIO39 (MTK_PIN_NO(39) | 0)
-#define MT6397_PIN_39_HTPLG__FUNC_HTPLG (MTK_PIN_NO(39) | 1)
-#define MT6397_PIN_39_HTPLG__FUNC_TEST_IN30 (MTK_PIN_NO(39) | 6)
-#define MT6397_PIN_39_HTPLG__FUNC_TEST_OUT30 (MTK_PIN_NO(39) | 7)
-
-#define MT6397_PIN_40_CEC__FUNC_GPIO40 (MTK_PIN_NO(40) | 0)
-#define MT6397_PIN_40_CEC__FUNC_CEC (MTK_PIN_NO(40) | 1)
-#define MT6397_PIN_40_CEC__FUNC_TEST_IN31 (MTK_PIN_NO(40) | 6)
-#define MT6397_PIN_40_CEC__FUNC_TEST_OUT31 (MTK_PIN_NO(40) | 7)
-
-#endif /* __DTS_MT6397_PINFUNC_H */
diff --git a/include/dt-bindings/power/allwinner,sun20i-d1-ppu.h b/include/dt-bindings/power/allwinner,sun20i-d1-ppu.h
deleted file mode 100644
index 23cfb57256d6..000000000000
--- a/include/dt-bindings/power/allwinner,sun20i-d1-ppu.h
+++ /dev/null
@@ -1,10 +0,0 @@
-/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
-
-#ifndef _DT_BINDINGS_POWER_SUN20I_D1_PPU_H_
-#define _DT_BINDINGS_POWER_SUN20I_D1_PPU_H_
-
-#define PD_CPU			0
-#define PD_VE			1
-#define PD_DSP			2
-
-#endif /* _DT_BINDINGS_POWER_SUN20I_D1_PPU_H_ */
diff --git a/include/dt-bindings/power/qcom,rpmhpd.h b/include/dt-bindings/power/qcom,rpmhpd.h
index 50e7c886709d..06851363ae0e 100644
--- a/include/dt-bindings/power/qcom,rpmhpd.h
+++ b/include/dt-bindings/power/qcom,rpmhpd.h
@@ -264,5 +264,6 @@
 #define SC8280XP_NSP		13
 #define SC8280XP_QPHY		14
 #define SC8280XP_XO		15
+#define SC8280XP_MXC_AO		16
 
 #endif
diff --git a/include/dt-bindings/reset/bcm6318-reset.h b/include/dt-bindings/reset/bcm6318-reset.h
deleted file mode 100644
index f882662505ea..000000000000
--- a/include/dt-bindings/reset/bcm6318-reset.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0+ */
-
-#ifndef __DT_BINDINGS_RESET_BCM6318_H
-#define __DT_BINDINGS_RESET_BCM6318_H
-
-#define BCM6318_RST_SPI		0
-#define BCM6318_RST_EPHY	1
-#define BCM6318_RST_SAR		2
-#define BCM6318_RST_ENETSW	3
-#define BCM6318_RST_USBD	4
-#define BCM6318_RST_USBH	5
-#define BCM6318_RST_PCIE_CORE	6
-#define BCM6318_RST_PCIE	7
-#define BCM6318_RST_PCIE_EXT	8
-#define BCM6318_RST_PCIE_HARD	9
-#define BCM6318_RST_ADSL	10
-#define BCM6318_RST_PHYMIPS	11
-#define BCM6318_RST_HOSTMIPS	12
-
-#endif /* __DT_BINDINGS_RESET_BCM6318_H */
diff --git a/include/dt-bindings/reset/imx8ulp-pcc-reset.h b/include/dt-bindings/reset/imx8ulp-pcc-reset.h
deleted file mode 100644
index e99a4735c3c4..000000000000
--- a/include/dt-bindings/reset/imx8ulp-pcc-reset.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright 2021 NXP
- */
-
-#ifndef DT_BINDING_PCC_RESET_IMX8ULP_H
-#define DT_BINDING_PCC_RESET_IMX8ULP_H
-
-/* PCC3 */
-#define PCC3_WDOG3_SWRST	0
-#define PCC3_WDOG4_SWRST	1
-#define PCC3_LPIT1_SWRST	2
-#define PCC3_TPM4_SWRST		3
-#define PCC3_TPM5_SWRST		4
-#define PCC3_FLEXIO1_SWRST	5
-#define PCC3_I3C2_SWRST		6
-#define PCC3_LPI2C4_SWRST	7
-#define PCC3_LPI2C5_SWRST	8
-#define PCC3_LPUART4_SWRST	9
-#define PCC3_LPUART5_SWRST	10
-#define PCC3_LPSPI4_SWRST	11
-#define PCC3_LPSPI5_SWRST	12
-
-/* PCC4 */
-#define PCC4_FLEXSPI2_SWRST	0
-#define PCC4_TPM6_SWRST		1
-#define PCC4_TPM7_SWRST		2
-#define PCC4_LPI2C6_SWRST	3
-#define PCC4_LPI2C7_SWRST	4
-#define PCC4_LPUART6_SWRST	5
-#define PCC4_LPUART7_SWRST	6
-#define PCC4_SAI4_SWRST		7
-#define PCC4_SAI5_SWRST		8
-#define PCC4_USDHC0_SWRST	9
-#define PCC4_USDHC1_SWRST	10
-#define PCC4_USDHC2_SWRST	11
-#define PCC4_USB0_SWRST		12
-#define PCC4_USB0_PHY_SWRST	13
-#define PCC4_USB1_SWRST		14
-#define PCC4_USB1_PHY_SWRST	15
-#define PCC4_ENET_SWRST		16
-
-/* PCC5 */
-#define PCC5_TPM8_SWRST		0
-#define PCC5_SAI6_SWRST		1
-#define PCC5_SAI7_SWRST		2
-#define PCC5_SPDIF_SWRST	3
-#define PCC5_ISI_SWRST		4
-#define PCC5_CSI_REGS_SWRST	5
-#define PCC5_CSI_SWRST		6
-#define PCC5_DSI_SWRST		7
-#define PCC5_WDOG5_SWRST	8
-#define PCC5_EPDC_SWRST		9
-#define PCC5_PXP_SWRST		10
-#define PCC5_GPU2D_SWRST	11
-#define PCC5_GPU3D_SWRST	12
-#define PCC5_DC_NANO_SWRST	13
-
-#endif /*DT_BINDING_RESET_IMX8ULP_H */
diff --git a/include/dt-bindings/reset/oxsemi,ox810se.h b/include/dt-bindings/reset/oxsemi,ox810se.h
deleted file mode 100644
index e943187e6527..000000000000
--- a/include/dt-bindings/reset/oxsemi,ox810se.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2016 Neil Armstrong <narmstrong@baylibre.com>
- */
-
-#ifndef DT_RESET_OXSEMI_OX810SE_H
-#define DT_RESET_OXSEMI_OX810SE_H
-
-#define RESET_ARM	0
-#define RESET_COPRO	1
-/* Reserved		2 */
-/* Reserved		3 */
-#define RESET_USBHS	4
-#define RESET_USBHSPHY	5
-#define RESET_MAC	6
-#define RESET_PCI	7
-#define RESET_DMA	8
-#define RESET_DPE	9
-#define RESET_DDR	10
-#define RESET_SATA	11
-#define RESET_SATA_LINK	12
-#define RESET_SATA_PHY	13
- /* Reserved		14 */
-#define RESET_NAND	15
-#define RESET_GPIO	16
-#define RESET_UART1	17
-#define RESET_UART2	18
-#define RESET_MISC	19
-#define RESET_I2S	20
-#define RESET_AHB_MON	21
-#define RESET_UART3	22
-#define RESET_UART4	23
-#define RESET_SGDMA	24
-/* Reserved		25 */
-/* Reserved		26 */
-/* Reserved		27 */
-/* Reserved		28 */
-/* Reserved		29 */
-/* Reserved		30 */
-#define RESET_BUS	31
-
-#endif /* DT_RESET_OXSEMI_OX810SE_H */
diff --git a/include/dt-bindings/reset/oxsemi,ox820.h b/include/dt-bindings/reset/oxsemi,ox820.h
deleted file mode 100644
index 54b58e09c1c0..000000000000
--- a/include/dt-bindings/reset/oxsemi,ox820.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2016 Neil Armstrong <narmstrong@baylibre.com>
- */
-
-#ifndef DT_RESET_OXSEMI_OX820_H
-#define DT_RESET_OXSEMI_OX820_H
-
-#define RESET_SCU	0
-#define RESET_LEON	1
-#define RESET_ARM0	2
-#define RESET_ARM1	3
-#define RESET_USBHS	4
-#define RESET_USBPHYA	5
-#define RESET_MAC	6
-#define RESET_PCIEA	7
-#define RESET_SGDMA	8
-#define RESET_CIPHER	9
-#define RESET_DDR	10
-#define RESET_SATA	11
-#define RESET_SATA_LINK	12
-#define RESET_SATA_PHY	13
-#define RESET_PCIEPHY	14
-#define RESET_NAND	15
-#define RESET_GPIO	16
-#define RESET_UART1	17
-#define RESET_UART2	18
-#define RESET_MISC	19
-#define RESET_I2S	20
-#define RESET_SD	21
-#define RESET_MAC_2	22
-#define RESET_PCIEB	23
-#define RESET_VIDEO	24
-#define RESET_DDR_PHY	25
-#define RESET_USBPHYB	26
-#define RESET_USBDEV	27
-/* Reserved		29 */
-#define RESET_ARMDBG	29
-#define RESET_PLLA	30
-#define RESET_PLLB	31
-
-#endif /* DT_RESET_OXSEMI_OX820_H */
diff --git a/include/dt-bindings/sound/audio-jack-events.h b/include/dt-bindings/sound/audio-jack-events.h
deleted file mode 100644
index 1b29b295126a..000000000000
--- a/include/dt-bindings/sound/audio-jack-events.h
+++ /dev/null
@@ -1,10 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __AUDIO_JACK_EVENTS_H
-#define __AUDIO_JACK_EVENTS_H
-
-#define JACK_HEADPHONE		1
-#define JACK_MICROPHONE		2
-#define JACK_LINEOUT		3
-#define JACK_LINEIN		4
-
-#endif /* __AUDIO_JACK_EVENTS_H */
diff --git a/include/hyperv/hvgdk_mini.h b/include/hyperv/hvgdk_mini.h
index 04b18d0e37af..30fbbde81c5c 100644
--- a/include/hyperv/hvgdk_mini.h
+++ b/include/hyperv/hvgdk_mini.h
@@ -578,9 +578,12 @@ struct hv_tlb_flush {	 /* HV_INPUT_FLUSH_VIRTUAL_ADDRESS_LIST */
 struct hv_tlb_flush_ex {
 	u64 address_space;
 	u64 flags;
-	struct hv_vpset hv_vp_set;
-	u64 gva_list[];
+	__TRAILING_OVERLAP(struct hv_vpset, hv_vp_set, bank_contents, __packed,
+		u64 gva_list[];
+	);
 } __packed;
+static_assert(offsetof(struct hv_tlb_flush_ex, hv_vp_set.bank_contents) ==
+	      offsetof(struct hv_tlb_flush_ex, gva_list));
 
 struct ms_hyperv_tsc_page {	 /* HV_REFERENCE_TSC_PAGE */
 	volatile u32 tsc_sequence;
diff --git a/include/linux/array_size.h b/include/linux/array_size.h
index 06d7d83196ca..0c4fec98822e 100644
--- a/include/linux/array_size.h
+++ b/include/linux/array_size.h
@@ -10,4 +10,10 @@
  */
 #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]) + __must_be_array(arr))
 
+/**
+ * ARRAY_END - get a pointer to one past the last element in array @arr
+ * @arr: array
+ */
+#define ARRAY_END(arr)  (&(arr)[ARRAY_SIZE(arr)])
+
 #endif  /* _LINUX_ARRAY_SIZE_H */
diff --git a/include/linux/ata.h b/include/linux/ata.h
index 54b416e26995..8fd48bcb2a46 100644
--- a/include/linux/ata.h
+++ b/include/linux/ata.h
@@ -26,10 +26,7 @@ enum {
 	ATA_MAX_DEVICES		= 2,	/* per bus/port */
 	ATA_MAX_PRD		= 256,	/* we could make these 256/256 */
 	ATA_SECT_SIZE		= 512,
-	ATA_MAX_SECTORS_128	= 128,
 	ATA_MAX_SECTORS		= 256,
-	ATA_MAX_SECTORS_1024    = 1024,
-	ATA_MAX_SECTORS_8191    = 8191,
 	ATA_MAX_SECTORS_LBA48	= 65535,/* avoid count to be 0000h */
 	ATA_MAX_SECTORS_TAPE	= 65535,
 	ATA_MAX_TRIM_RNUM	= 64,	/* 512-byte payload / (6-byte LBA + 2-byte range per entry) */
diff --git a/include/linux/audit.h b/include/linux/audit.h
index 536f8ee8da81..d8173af498ba 100644
--- a/include/linux/audit.h
+++ b/include/linux/audit.h
@@ -195,6 +195,8 @@ extern int audit_log_subj_ctx(struct audit_buffer *ab, struct lsm_prop *prop);
 extern int audit_log_obj_ctx(struct audit_buffer *ab, struct lsm_prop *prop);
 extern int audit_log_task_context(struct audit_buffer *ab);
 extern void audit_log_task_info(struct audit_buffer *ab);
+extern int audit_log_nf_skb(struct audit_buffer *ab,
+			    const struct sk_buff *skb, u8 nfproto);
 
 extern int		    audit_update_lsm_rules(void);
 
@@ -272,6 +274,12 @@ static inline int audit_log_task_context(struct audit_buffer *ab)
 static inline void audit_log_task_info(struct audit_buffer *ab)
 { }
 
+static inline int audit_log_nf_skb(struct audit_buffer *ab,
+				   const struct sk_buff *skb, u8 nfproto)
+{
+	return 0;
+}
+
 static inline kuid_t audit_get_loginuid(struct task_struct *tsk)
 {
 	return INVALID_UID;
diff --git a/include/linux/bio.h b/include/linux/bio.h
index ad2d57908c1c..c75a9b3672aa 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -46,6 +46,21 @@ static inline unsigned int bio_max_segs(unsigned int nr_segs)
 #define bio_data_dir(bio) \
 	(op_is_write(bio_op(bio)) ? WRITE : READ)
 
+static inline bool bio_flagged(const struct bio *bio, unsigned int bit)
+{
+	return bio->bi_flags & (1U << bit);
+}
+
+static inline void bio_set_flag(struct bio *bio, unsigned int bit)
+{
+	bio->bi_flags |= (1U << bit);
+}
+
+static inline void bio_clear_flag(struct bio *bio, unsigned int bit)
+{
+	bio->bi_flags &= ~(1U << bit);
+}
+
 /*
  * Check whether this bio carries any data or not. A NULL bio is allowed.
  */
@@ -225,21 +240,6 @@ static inline void bio_cnt_set(struct bio *bio, unsigned int count)
 	atomic_set(&bio->__bi_cnt, count);
 }
 
-static inline bool bio_flagged(struct bio *bio, unsigned int bit)
-{
-	return bio->bi_flags & (1U << bit);
-}
-
-static inline void bio_set_flag(struct bio *bio, unsigned int bit)
-{
-	bio->bi_flags |= (1U << bit);
-}
-
-static inline void bio_clear_flag(struct bio *bio, unsigned int bit)
-{
-	bio->bi_flags &= ~(1U << bit);
-}
-
 static inline struct bio_vec *bio_first_bvec_all(struct bio *bio)
 {
 	WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
diff --git a/include/linux/bitfield.h b/include/linux/bitfield.h
index 126dc5b380af..54aeeef1f0ec 100644
--- a/include/linux/bitfield.h
+++ b/include/linux/bitfield.h
@@ -8,6 +8,7 @@
 #define _LINUX_BITFIELD_H
 
 #include <linux/build_bug.h>
+#include <linux/compiler.h>
 #include <linux/typecheck.h>
 #include <asm/byteorder.h>
 
@@ -243,7 +244,7 @@ __MAKE_OP(64)
 
 #define __field_prep(mask, val)						\
 	({								\
-		__auto_type __mask = (mask);				\
+		auto __mask = (mask);					\
 		typeof(__mask) __val = (val);				\
 		unsigned int __shift = BITS_PER_TYPE(__mask) <= 32 ?	\
 				       __ffs(__mask) : __ffs64(__mask);	\
@@ -252,7 +253,7 @@ __MAKE_OP(64)
 
 #define __field_get(mask, reg)						\
 	({								\
-		__auto_type __mask = (mask);				\
+		auto __mask = (mask);					\
 		typeof(__mask) __reg =  (reg);				\
 		unsigned int __shift = BITS_PER_TYPE(__mask) <= 32 ?	\
 				       __ffs(__mask) : __ffs64(__mask);	\
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 6498be4c44f8..bb3847caeae1 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1283,6 +1283,8 @@ struct bpf_ksym {
 	struct list_head	 lnode;
 	struct latch_tree_node	 tnode;
 	bool			 prog;
+	u32			 fp_start;
+	u32			 fp_end;
 };
 
 enum bpf_tramp_prog_type {
@@ -1511,6 +1513,7 @@ void bpf_image_ksym_add(struct bpf_ksym *ksym);
 void bpf_image_ksym_del(struct bpf_ksym *ksym);
 void bpf_ksym_add(struct bpf_ksym *ksym);
 void bpf_ksym_del(struct bpf_ksym *ksym);
+bool bpf_has_frame_pointer(unsigned long ip);
 int bpf_jit_charge_modmem(u32 size);
 void bpf_jit_uncharge_modmem(u32 size);
 bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
@@ -1739,6 +1742,8 @@ struct bpf_prog_aux {
 		struct rcu_head	rcu;
 	};
 	struct bpf_stream stream[2];
+	struct mutex st_ops_assoc_mutex;
+	struct bpf_map __rcu *st_ops_assoc;
 };
 
 struct bpf_prog {
@@ -2041,6 +2046,9 @@ static inline void bpf_module_put(const void *data, struct module *owner)
 		module_put(owner);
 }
 int bpf_struct_ops_link_create(union bpf_attr *attr);
+int bpf_prog_assoc_struct_ops(struct bpf_prog *prog, struct bpf_map *map);
+void bpf_prog_disassoc_struct_ops(struct bpf_prog *prog);
+void *bpf_prog_get_assoc_struct_ops(const struct bpf_prog_aux *aux);
 u32 bpf_struct_ops_id(const void *kdata);
 
 #ifdef CONFIG_NET
@@ -2088,6 +2096,17 @@ static inline int bpf_struct_ops_link_create(union bpf_attr *attr)
 {
 	return -EOPNOTSUPP;
 }
+static inline int bpf_prog_assoc_struct_ops(struct bpf_prog *prog, struct bpf_map *map)
+{
+	return -EOPNOTSUPP;
+}
+static inline void bpf_prog_disassoc_struct_ops(struct bpf_prog *prog)
+{
+}
+static inline void *bpf_prog_get_assoc_struct_ops(const struct bpf_prog_aux *aux)
+{
+	return NULL;
+}
 static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map)
 {
 }
diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h
index 1280693766b9..280b4ac0990f 100644
--- a/include/linux/compiler_types.h
+++ b/include/linux/compiler_types.h
@@ -586,6 +586,25 @@ struct ftrace_likely_data {
 			 __scalar_type_to_expr_cases(long long),	\
 			 default: (x)))
 
+/*
+ * __signed_scalar_typeof(x) - Declare a signed scalar type, leaving
+ *			       non-scalar types unchanged.
+ */
+
+#define __scalar_type_to_signed_cases(type)				\
+		unsigned type:	(signed type)0,				\
+		signed type:	(signed type)0
+
+#define __signed_scalar_typeof(x) typeof(				\
+		_Generic((x),						\
+			 char:	(signed char)0,				\
+			 __scalar_type_to_signed_cases(char),		\
+			 __scalar_type_to_signed_cases(short),		\
+			 __scalar_type_to_signed_cases(int),		\
+			 __scalar_type_to_signed_cases(long),		\
+			 __scalar_type_to_signed_cases(long long),	\
+			 default: (x)))
+
 /* Is this type a native word size -- useful for atomic operations */
 #define __native_word(t) \
 	(sizeof(t) == sizeof(char) || sizeof(t) == sizeof(short) || \
diff --git a/include/linux/cpu.h b/include/linux/cpu.h
index 487b3bf2e1ea..8239cd95a005 100644
--- a/include/linux/cpu.h
+++ b/include/linux/cpu.h
@@ -229,4 +229,8 @@ static inline bool cpu_attack_vector_mitigated(enum cpu_attack_vectors v)
 #define smt_mitigations SMT_MITIGATIONS_OFF
 #endif
 
+int arch_get_indir_br_lp_status(struct task_struct *t, unsigned long __user *status);
+int arch_set_indir_br_lp_status(struct task_struct *t, unsigned long status);
+int arch_lock_indir_br_lp_status(struct task_struct *t, unsigned long status);
+
 #endif /* _LINUX_CPU_H_ */
diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h
index a98d3330385c..a634489c9f12 100644
--- a/include/linux/cpuset.h
+++ b/include/linux/cpuset.h
@@ -74,6 +74,7 @@ extern void inc_dl_tasks_cs(struct task_struct *task);
 extern void dec_dl_tasks_cs(struct task_struct *task);
 extern void cpuset_lock(void);
 extern void cpuset_unlock(void);
+extern void lockdep_assert_cpuset_lock_held(void);
 extern void cpuset_cpus_allowed_locked(struct task_struct *p, struct cpumask *mask);
 extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask);
 extern bool cpuset_cpus_allowed_fallback(struct task_struct *p);
@@ -195,6 +196,7 @@ static inline void inc_dl_tasks_cs(struct task_struct *task) { }
 static inline void dec_dl_tasks_cs(struct task_struct *task) { }
 static inline void cpuset_lock(void) { }
 static inline void cpuset_unlock(void) { }
+static inline void lockdep_assert_cpuset_lock_held(void) { }
 
 static inline void cpuset_cpus_allowed_locked(struct task_struct *p,
 					struct cpumask *mask)
diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h
index 64639e104110..d4c92fd35092 100644
--- a/include/linux/dma-fence.h
+++ b/include/linux/dma-fence.h
@@ -364,11 +364,12 @@ static inline void dma_fence_end_signalling(bool cookie) {}
 static inline void __dma_fence_might_wait(void) {}
 #endif
 
-int dma_fence_signal(struct dma_fence *fence);
-int dma_fence_signal_locked(struct dma_fence *fence);
-int dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp);
-int dma_fence_signal_timestamp_locked(struct dma_fence *fence,
-				      ktime_t timestamp);
+void dma_fence_signal(struct dma_fence *fence);
+bool dma_fence_check_and_signal(struct dma_fence *fence);
+bool dma_fence_check_and_signal_locked(struct dma_fence *fence);
+void dma_fence_signal_locked(struct dma_fence *fence);
+void dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp);
+void dma_fence_signal_timestamp_locked(struct dma_fence *fence, ktime_t timestamp);
 signed long dma_fence_default_wait(struct dma_fence *fence,
 				   bool intr, signed long timeout);
 int dma_fence_add_callback(struct dma_fence *fence,
@@ -401,6 +402,26 @@ void dma_fence_enable_sw_signaling(struct dma_fence *fence);
 const char __rcu *dma_fence_driver_name(struct dma_fence *fence);
 const char __rcu *dma_fence_timeline_name(struct dma_fence *fence);
 
+/*
+ * dma_fence_test_signaled_flag - Only check whether a fence is signaled yet.
+ * @fence: the fence to check
+ *
+ * This function just checks whether @fence is signaled, without interacting
+ * with the fence in any way. The user must, therefore, ensure through other
+ * means that fences get signaled eventually.
+ *
+ * This function uses test_bit(), which is thread-safe. Naturally, this function
+ * should be used opportunistically; a fence could get signaled at any moment
+ * after the check is done.
+ *
+ * Return: true if signaled, false otherwise.
+ */
+static inline bool
+dma_fence_test_signaled_flag(struct dma_fence *fence)
+{
+	return test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
+}
+
 /**
  * dma_fence_is_signaled_locked - Return an indication if the fence
  *                                is signaled yet.
@@ -418,7 +439,7 @@ const char __rcu *dma_fence_timeline_name(struct dma_fence *fence);
 static inline bool
 dma_fence_is_signaled_locked(struct dma_fence *fence)
 {
-	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+	if (dma_fence_test_signaled_flag(fence))
 		return true;
 
 	if (fence->ops->signaled && fence->ops->signaled(fence)) {
@@ -448,7 +469,7 @@ dma_fence_is_signaled_locked(struct dma_fence *fence)
 static inline bool
 dma_fence_is_signaled(struct dma_fence *fence)
 {
-	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+	if (dma_fence_test_signaled_flag(fence))
 		return true;
 
 	if (fence->ops->signaled && fence->ops->signaled(fence)) {
diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h
index 3080747689f6..270b5458aecf 100644
--- a/include/linux/dma/edma.h
+++ b/include/linux/dma/edma.h
@@ -27,7 +27,7 @@ struct dw_edma_region {
 };
 
 /**
- * struct dw_edma_core_ops - platform-specific eDMA methods
+ * struct dw_edma_plat_ops - platform-specific eDMA methods
  * @irq_vector:		Get IRQ number of the passed eDMA channel. Note the
  *			method accepts the channel id in the end-to-end
  *			numbering with the eDMA write channels being placed
@@ -63,19 +63,17 @@ enum dw_edma_chip_flags {
 /**
  * struct dw_edma_chip - representation of DesignWare eDMA controller hardware
  * @dev:		 struct device of the eDMA controller
- * @id:			 instance ID
  * @nr_irqs:		 total number of DMA IRQs
- * @ops			 DMA channel to IRQ number mapping
- * @flags		 dw_edma_chip_flags
- * @reg_base		 DMA register base address
- * @ll_wr_cnt		 DMA write link list count
- * @ll_rd_cnt		 DMA read link list count
- * @rg_region		 DMA register region
- * @ll_region_wr	 DMA descriptor link list memory for write channel
- * @ll_region_rd	 DMA descriptor link list memory for read channel
- * @dt_region_wr	 DMA data memory for write channel
- * @dt_region_rd	 DMA data memory for read channel
- * @mf			 DMA register map format
+ * @ops:		 DMA channel to IRQ number mapping
+ * @flags:		 dw_edma_chip_flags
+ * @reg_base:		 DMA register base address
+ * @ll_wr_cnt:		 DMA write link list count
+ * @ll_rd_cnt:		 DMA read link list count
+ * @ll_region_wr:	 DMA descriptor link list memory for write channel
+ * @ll_region_rd:	 DMA descriptor link list memory for read channel
+ * @dt_region_wr:	 DMA data memory for write channel
+ * @dt_region_rd:	 DMA data memory for read channel
+ * @mf:			 DMA register map format
  * @dw:			 struct dw_edma that is filled by dw_edma_probe()
  */
 struct dw_edma_chip {
diff --git a/include/linux/efi.h b/include/linux/efi.h
index 2a43094e23f7..664898d09ff5 100644
--- a/include/linux/efi.h
+++ b/include/linux/efi.h
@@ -406,11 +406,12 @@ void efi_native_runtime_setup(void);
 #define EFI_CC_FINAL_EVENTS_TABLE_GUID		EFI_GUID(0xdd4a4648, 0x2de7, 0x4665, 0x96, 0x4d, 0x21, 0xd9, 0xef, 0x5f, 0xb4, 0x46)
 
 /*
- * This GUID is used to pass to the kernel proper the struct screen_info
- * structure that was populated by the stub based on the GOP protocol instance
- * associated with ConOut
+ * This GUIDs are used to pass to the kernel proper the primary
+ * display that has been populated by the stub based on the GOP
+ * instance associated with ConOut.
  */
-#define LINUX_EFI_SCREEN_INFO_TABLE_GUID	EFI_GUID(0xe03fc20a, 0x85dc, 0x406e,  0xb9, 0x0e, 0x4a, 0xb5, 0x02, 0x37, 0x1d, 0x95)
+#define LINUX_EFI_PRIMARY_DISPLAY_TABLE_GUID	EFI_GUID(0xe03fc20a, 0x85dc, 0x406e,  0xb9, 0x0e, 0x4a, 0xb5, 0x02, 0x37, 0x1d, 0x95)
+
 #define LINUX_EFI_ARM_CPU_STATE_TABLE_GUID	EFI_GUID(0xef79e4aa, 0x3c3d, 0x4989,  0xb9, 0x02, 0x07, 0xa9, 0x43, 0xe5, 0x50, 0xd2)
 #define LINUX_EFI_LOADER_ENTRY_GUID		EFI_GUID(0x4a67b082, 0x0a4c, 0x41cf,  0xb6, 0xc7, 0x44, 0x0b, 0x29, 0xbb, 0x8c, 0x4f)
 #define LINUX_EFI_RANDOM_SEED_TABLE_GUID	EFI_GUID(0x1ce1e5bc, 0x7ceb, 0x42f2,  0x81, 0xe5, 0x8a, 0xad, 0xf1, 0x80, 0xf5, 0x7b)
diff --git a/include/linux/fb.h b/include/linux/fb.h
index 05cc251035da..65fb70382675 100644
--- a/include/linux/fb.h
+++ b/include/linux/fb.h
@@ -304,10 +304,6 @@ struct fb_ops {
 
 	/* teardown any resources to do with this framebuffer */
 	void (*fb_destroy)(struct fb_info *info);
-
-	/* called at KDB enter and leave time to prepare the console */
-	int (*fb_debug_enter)(struct fb_info *info);
-	int (*fb_debug_leave)(struct fb_info *info);
 };
 
 #ifdef CONFIG_FB_TILEBLITTING
diff --git a/include/linux/filelock.h b/include/linux/filelock.h
index 54b824c05299..d2c9740e26a8 100644
--- a/include/linux/filelock.h
+++ b/include/linux/filelock.h
@@ -49,6 +49,7 @@ struct lease_manager_operations {
 	int (*lm_change)(struct file_lease *, int, struct list_head *);
 	void (*lm_setup)(struct file_lease *, void **);
 	bool (*lm_breaker_owns_lease)(struct file_lease *);
+	int (*lm_open_conflict)(struct file *, int);
 };
 
 struct lock_manager {
@@ -241,7 +242,14 @@ bool locks_owner_has_blockers(struct file_lock_context *flctx,
 static inline struct file_lock_context *
 locks_inode_context(const struct inode *inode)
 {
-	return smp_load_acquire(&inode->i_flctx);
+	/*
+	 * Paired with smp_store_release in locks_get_lock_context().
+	 *
+	 * Ensures ->i_flctx will be visible if we spotted the flag.
+	 */
+	if (likely(!(smp_load_acquire(&inode->i_opflags) & IOP_FLCTX)))
+		return NULL;
+	return READ_ONCE(inode->i_flctx);
 }
 
 #else /* !CONFIG_FILE_LOCKING */
@@ -468,7 +476,7 @@ static inline int break_lease(struct inode *inode, unsigned int mode)
 	 * could end up racing with tasks trying to set a new lease on this
 	 * file.
 	 */
-	flctx = READ_ONCE(inode->i_flctx);
+	flctx = locks_inode_context(inode);
 	if (!flctx)
 		return 0;
 	smp_mb();
@@ -487,7 +495,7 @@ static inline int break_deleg(struct inode *inode, unsigned int flags)
 	 * could end up racing with tasks trying to set a new lease on this
 	 * file.
 	 */
-	flctx = READ_ONCE(inode->i_flctx);
+	flctx = locks_inode_context(inode);
 	if (!flctx)
 		return 0;
 	smp_mb();
@@ -532,8 +540,11 @@ static inline int break_deleg_wait(struct delegated_inode *di)
 
 static inline int break_layout(struct inode *inode, bool wait)
 {
+	struct file_lock_context *flctx;
+
 	smp_mb();
-	if (inode->i_flctx && !list_empty_careful(&inode->i_flctx->flc_lease)) {
+	flctx = locks_inode_context(inode);
+	if (flctx && !list_empty_careful(&flctx->flc_lease)) {
 		unsigned int flags = LEASE_BREAK_LAYOUT;
 
 		if (!wait)
diff --git a/include/linux/filter.h b/include/linux/filter.h
index fd54fed8f95f..7452817d707d 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1375,24 +1375,13 @@ static inline bool bpf_jit_kallsyms_enabled(void)
 	return false;
 }
 
-int __bpf_address_lookup(unsigned long addr, unsigned long *size,
-				 unsigned long *off, char *sym);
+int bpf_address_lookup(unsigned long addr, unsigned long *size,
+		       unsigned long *off, char *sym);
 bool is_bpf_text_address(unsigned long addr);
 int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
 		    char *sym);
 struct bpf_prog *bpf_prog_ksym_find(unsigned long addr);
 
-static inline int
-bpf_address_lookup(unsigned long addr, unsigned long *size,
-		   unsigned long *off, char **modname, char *sym)
-{
-	int ret = __bpf_address_lookup(addr, size, off, sym);
-
-	if (ret && modname)
-		*modname = NULL;
-	return ret;
-}
-
 void bpf_prog_kallsyms_add(struct bpf_prog *fp);
 void bpf_prog_kallsyms_del(struct bpf_prog *fp);
 
@@ -1431,8 +1420,8 @@ static inline bool bpf_jit_kallsyms_enabled(void)
 }
 
 static inline int
-__bpf_address_lookup(unsigned long addr, unsigned long *size,
-		     unsigned long *off, char *sym)
+bpf_address_lookup(unsigned long addr, unsigned long *size,
+		   unsigned long *off, char *sym)
 {
 	return 0;
 }
@@ -1453,13 +1442,6 @@ static inline struct bpf_prog *bpf_prog_ksym_find(unsigned long addr)
 	return NULL;
 }
 
-static inline int
-bpf_address_lookup(unsigned long addr, unsigned long *size,
-		   unsigned long *off, char **modname, char *sym)
-{
-	return 0;
-}
-
 static inline void bpf_prog_kallsyms_add(struct bpf_prog *fp)
 {
 }
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 04ceeca12a0d..eb51e2a9b78c 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -631,6 +631,7 @@ is_uncached_acl(struct posix_acl *acl)
 #define IOP_MGTIME		0x0020
 #define IOP_CACHED_LINK		0x0040
 #define IOP_FASTPERM_MAY_EXEC	0x0080
+#define IOP_FLCTX		0x0100
 
 /*
  * Inode state bits.  Protected by inode->i_lock
@@ -2274,8 +2275,6 @@ struct file_system_type {
 #define FS_RENAME_DOES_D_MOVE	32768	/* FS will handle d_move() during rename() internally. */
 	int (*init_fs_context)(struct fs_context *);
 	const struct fs_parameter_spec *parameters;
-	struct dentry *(*mount) (struct file_system_type *, int,
-		       const char *, void *);
 	void (*kill_sb) (struct super_block *);
 	struct module *owner;
 	struct file_system_type * next;
@@ -2457,7 +2456,7 @@ struct file *dentry_open(const struct path *path, int flags,
 			 const struct cred *creds);
 struct file *dentry_open_nonotify(const struct path *path, int flags,
 				  const struct cred *cred);
-struct file *dentry_create(const struct path *path, int flags, umode_t mode,
+struct file *dentry_create(struct path *path, int flags, umode_t mode,
 			   const struct cred *cred);
 const struct path *backing_file_user_path(const struct file *f);
 
@@ -3247,7 +3246,7 @@ struct offset_ctx {
 void simple_offset_init(struct offset_ctx *octx);
 int simple_offset_add(struct offset_ctx *octx, struct dentry *dentry);
 void simple_offset_remove(struct offset_ctx *octx, struct dentry *dentry);
-int simple_offset_rename(struct inode *old_dir, struct dentry *old_dentry,
+void simple_offset_rename(struct inode *old_dir, struct dentry *old_dentry,
 			 struct inode *new_dir, struct dentry *new_dentry);
 int simple_offset_rename_exchange(struct inode *old_dir,
 				  struct dentry *old_dentry,
diff --git a/include/linux/fs/super_types.h b/include/linux/fs/super_types.h
index 6bd3009e09b3..4bb9981af6ac 100644
--- a/include/linux/fs/super_types.h
+++ b/include/linux/fs/super_types.h
@@ -96,7 +96,6 @@ struct super_operations {
 			  const void *owner);
 	int (*unfreeze_fs)(struct super_block *sb);
 	int (*statfs)(struct dentry *dentry, struct kstatfs *kstatfs);
-	int (*remount_fs) (struct super_block *, int *, char *);
 	void (*umount_begin)(struct super_block *sb);
 
 	int (*show_options)(struct seq_file *seq, struct dentry *dentry);
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index 770f0dc993cc..392932c1f77e 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -87,11 +87,13 @@ struct ftrace_hash;
 	defined(CONFIG_DYNAMIC_FTRACE)
 int
 ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
-		   unsigned long *off, char **modname, char *sym);
+			  unsigned long *off, char **modname,
+			  const unsigned char **modbuildid, char *sym);
 #else
 static inline int
 ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
-		   unsigned long *off, char **modname, char *sym)
+			  unsigned long *off, char **modname,
+			  const unsigned char **modbuildid, char *sym)
 {
 	return 0;
 }
diff --git a/include/linux/genalloc.h b/include/linux/genalloc.h
index 0bd581003cd5..60de63e46b33 100644
--- a/include/linux/genalloc.h
+++ b/include/linux/genalloc.h
@@ -44,6 +44,7 @@ struct gen_pool;
  * @nr: The number of zeroed bits we're looking for
  * @data: optional additional data used by the callback
  * @pool: the pool being allocated from
+ * @start_addr: start address of memory chunk
  */
 typedef unsigned long (*genpool_algo_t)(unsigned long *map,
 			unsigned long size,
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 019a1c5281e4..1ad39522428f 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -156,7 +156,7 @@ long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
 bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list);
 int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison);
 int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
-				bool *migratable_cleared);
+				bool *migratable_cleared, bool *samepg);
 void folio_putback_hugetlb(struct folio *folio);
 void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason);
 void hugetlb_fix_reserve_counts(struct inode *inode);
@@ -240,8 +240,9 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
 pte_t *huge_pte_offset(struct mm_struct *mm,
 		       unsigned long addr, unsigned long sz);
 unsigned long hugetlb_mask_last_page(struct hstate *h);
-int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
-				unsigned long addr, pte_t *ptep);
+int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma,
+		unsigned long addr, pte_t *ptep);
+void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma);
 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
 				unsigned long *start, unsigned long *end);
 
@@ -271,7 +272,7 @@ void hugetlb_vma_unlock_write(struct vm_area_struct *vma);
 int hugetlb_vma_trylock_write(struct vm_area_struct *vma);
 void hugetlb_vma_assert_locked(struct vm_area_struct *vma);
 void hugetlb_vma_lock_release(struct kref *kref);
-long hugetlb_change_protection(struct vm_area_struct *vma,
+long hugetlb_change_protection(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		unsigned long address, unsigned long end, pgprot_t newprot,
 		unsigned long cp_flags);
 void hugetlb_unshare_all_pmds(struct vm_area_struct *vma);
@@ -300,13 +301,17 @@ static inline struct address_space *hugetlb_folio_mapping_lock_write(
 	return NULL;
 }
 
-static inline int huge_pmd_unshare(struct mm_struct *mm,
-					struct vm_area_struct *vma,
-					unsigned long addr, pte_t *ptep)
+static inline int huge_pmd_unshare(struct mmu_gather *tlb,
+		struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
 {
 	return 0;
 }
 
+static inline void huge_pmd_unshare_flush(struct mmu_gather *tlb,
+		struct vm_area_struct *vma)
+{
+}
+
 static inline void adjust_range_if_pmd_sharing_possible(
 				struct vm_area_struct *vma,
 				unsigned long *start, unsigned long *end)
@@ -418,7 +423,7 @@ static inline int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb,
 }
 
 static inline int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
-					bool *migratable_cleared)
+					bool *migratable_cleared, bool *samepg)
 {
 	return 0;
 }
@@ -432,7 +437,7 @@ static inline void move_hugetlb_state(struct folio *old_folio,
 {
 }
 
-static inline long hugetlb_change_protection(
+static inline long hugetlb_change_protection(struct mmu_gather *tlb,
 			struct vm_area_struct *vma, unsigned long address,
 			unsigned long end, pgprot_t newprot,
 			unsigned long cp_flags)
@@ -1326,7 +1331,7 @@ static inline __init void hugetlb_cma_reserve(int order)
 #ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
 static inline bool hugetlb_pmd_shared(pte_t *pte)
 {
-	return page_count(virt_to_page(pte)) > 1;
+	return ptdesc_pmd_is_shared(virt_to_ptdesc(pte));
 }
 #else
 static inline bool hugetlb_pmd_shared(pte_t *pte)
diff --git a/include/linux/init_syscalls.h b/include/linux/init_syscalls.h
index 92045d18cbfc..0bdbc458a881 100644
--- a/include/linux/init_syscalls.h
+++ b/include/linux/init_syscalls.h
@@ -2,7 +2,6 @@
 
 int __init init_mount(const char *dev_name, const char *dir_name,
 		const char *type_page, unsigned long flags, void *data_page);
-int __init init_umount(const char *name, int flags);
 int __init init_chdir(const char *filename);
 int __init init_chroot(const char *filename);
 int __init init_chown(const char *filename, uid_t user, gid_t group, int flags);
diff --git a/include/linux/initrd.h b/include/linux/initrd.h
index f1a1f4c92ded..7e5d26c8136f 100644
--- a/include/linux/initrd.h
+++ b/include/linux/initrd.h
@@ -3,8 +3,6 @@
 #ifndef __LINUX_INITRD_H
 #define __LINUX_INITRD_H
 
-#define INITRD_MINOR 250 /* shouldn't collide with /dev/ram* too soon ... */
-
 /* starting block # of image */
 extern int rd_image_start;
 
diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h
index e9ade2ff4af6..f479ef5b3341 100644
--- a/include/linux/intel_rapl.h
+++ b/include/linux/intel_rapl.h
@@ -214,10 +214,14 @@ void rapl_remove_package(struct rapl_package *rp);
 
 #ifdef CONFIG_PERF_EVENTS
 int rapl_package_add_pmu(struct rapl_package *rp);
+int rapl_package_add_pmu_locked(struct rapl_package *rp);
 void rapl_package_remove_pmu(struct rapl_package *rp);
+void rapl_package_remove_pmu_locked(struct rapl_package *rp);
 #else
 static inline int rapl_package_add_pmu(struct rapl_package *rp) { return 0; }
+static inline int rapl_package_add_pmu_locked(struct rapl_package *rp) { return 0; }
 static inline void rapl_package_remove_pmu(struct rapl_package *rp) { }
+static inline void rapl_package_remove_pmu_locked(struct rapl_package *rp) { }
 #endif
 
 #endif /* __INTEL_RAPL_H__ */
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 266f2b39213a..00c01b0a43be 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -181,9 +181,8 @@ request_any_context_irq(unsigned int irq, irq_handler_t handler,
 			unsigned long flags, const char *name, void *dev_id);
 
 extern int __must_check
-__request_percpu_irq(unsigned int irq, irq_handler_t handler,
-		     unsigned long flags, const char *devname,
-		     const cpumask_t *affinity, void __percpu *percpu_dev_id);
+request_percpu_irq_affinity(unsigned int irq, irq_handler_t handler, const char *devname,
+			    const cpumask_t *affinity, void __percpu *percpu_dev_id);
 
 extern int __must_check
 request_nmi(unsigned int irq, irq_handler_t handler, unsigned long flags,
@@ -193,17 +192,8 @@ static inline int __must_check
 request_percpu_irq(unsigned int irq, irq_handler_t handler,
 		   const char *devname, void __percpu *percpu_dev_id)
 {
-	return __request_percpu_irq(irq, handler, 0,
-				    devname, NULL, percpu_dev_id);
-}
-
-static inline int __must_check
-request_percpu_irq_affinity(unsigned int irq, irq_handler_t handler,
-			    const char *devname, const cpumask_t *affinity,
-			    void __percpu *percpu_dev_id)
-{
-	return __request_percpu_irq(irq, handler, 0,
-				    devname, affinity, percpu_dev_id);
+	return request_percpu_irq_affinity(irq, handler, devname,
+					   NULL, percpu_dev_id);
 }
 
 extern int __must_check
@@ -871,12 +861,6 @@ static inline void init_irq_proc(void)
 }
 #endif
 
-#ifdef CONFIG_IRQ_TIMINGS
-void irq_timings_enable(void);
-void irq_timings_disable(void);
-u64 irq_timings_next_event(u64 now);
-#endif
-
 struct seq_file;
 int show_interrupts(struct seq_file *p, void *v);
 int arch_show_interrupts(struct seq_file *p, int prec);
diff --git a/include/linux/iomap.h b/include/linux/iomap.h
index 520e967cb501..6bb941707d12 100644
--- a/include/linux/iomap.h
+++ b/include/linux/iomap.h
@@ -88,6 +88,9 @@ struct vm_fault;
 /*
  * Flags set by the core iomap code during operations:
  *
+ * IOMAP_F_FOLIO_BATCH indicates that the folio batch mechanism is active
+ * for this operation, set by iomap_fill_dirty_folios().
+ *
  * IOMAP_F_SIZE_CHANGED indicates to the iomap_end method that the file size
  * has changed as the result of this write operation.
  *
@@ -95,6 +98,7 @@ struct vm_fault;
  * range it covers needs to be remapped by the high level before the operation
  * can proceed.
  */
+#define IOMAP_F_FOLIO_BATCH	(1U << 13)
 #define IOMAP_F_SIZE_CHANGED	(1U << 14)
 #define IOMAP_F_STALE		(1U << 15)
 
@@ -352,8 +356,8 @@ bool iomap_dirty_folio(struct address_space *mapping, struct folio *folio);
 int iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len,
 		const struct iomap_ops *ops,
 		const struct iomap_write_ops *write_ops);
-loff_t iomap_fill_dirty_folios(struct iomap_iter *iter, loff_t offset,
-		loff_t length);
+unsigned int iomap_fill_dirty_folios(struct iomap_iter *iter, loff_t *start,
+		loff_t end, unsigned int *iomap_flags);
 int iomap_zero_range(struct inode *inode, loff_t pos, loff_t len,
 		bool *did_zero, const struct iomap_ops *ops,
 		const struct iomap_write_ops *write_ops, void *private);
diff --git a/include/linux/irq.h b/include/linux/irq.h
index 4a9f1d7b08c3..757f34bf26af 100644
--- a/include/linux/irq.h
+++ b/include/linux/irq.h
@@ -459,6 +459,8 @@ static inline irq_hw_number_t irqd_to_hwirq(struct irq_data *d)
  *			checks against the supplied affinity mask are not
  *			required. This is used for CPU hotplug where the
  *			target CPU is not yet set in the cpu_online_mask.
+ * @irq_pre_redirect:	Optional function to be invoked before redirecting
+ *			an interrupt via irq_work. Called only on CONFIG_SMP.
  * @irq_retrigger:	resend an IRQ to the CPU
  * @irq_set_type:	set the flow type (IRQ_TYPE_LEVEL/etc.) of an IRQ
  * @irq_set_wake:	enable/disable power-management wake-on of an IRQ
@@ -503,6 +505,7 @@ struct irq_chip {
 	void		(*irq_eoi)(struct irq_data *data);
 
 	int		(*irq_set_affinity)(struct irq_data *data, const struct cpumask *dest, bool force);
+	void		(*irq_pre_redirect)(struct irq_data *data);
 	int		(*irq_retrigger)(struct irq_data *data);
 	int		(*irq_set_type)(struct irq_data *data, unsigned int flow_type);
 	int		(*irq_set_wake)(struct irq_data *data, unsigned int on);
@@ -595,9 +598,6 @@ enum {
 
 #define IRQ_DEFAULT_INIT_FLAGS	ARCH_IRQ_INIT_FLAGS
 
-struct irqaction;
-extern int setup_percpu_irq(unsigned int irq, struct irqaction *new);
-
 #ifdef CONFIG_DEPRECATED_IRQ_CPU_ONOFFLINE
 extern void irq_cpu_online(void);
 extern void irq_cpu_offline(void);
@@ -687,6 +687,13 @@ extern int irq_chip_set_vcpu_affinity_parent(struct irq_data *data,
 extern int irq_chip_set_type_parent(struct irq_data *data, unsigned int type);
 extern int irq_chip_request_resources_parent(struct irq_data *data);
 extern void irq_chip_release_resources_parent(struct irq_data *data);
+#ifdef CONFIG_SMP
+void irq_chip_pre_redirect_parent(struct irq_data *data);
+#endif
+#endif
+
+#ifdef CONFIG_SMP
+int irq_chip_redirect_set_affinity(struct irq_data *data, const struct cpumask *dest, bool force);
 #endif
 
 /* Disable or mask interrupts during a kernel kexec */
diff --git a/include/linux/irqchip/irq-renesas-rzt2h.h b/include/linux/irqchip/irq-renesas-rzt2h.h
new file mode 100644
index 000000000000..853fd5ee0b22
--- /dev/null
+++ b/include/linux/irqchip/irq-renesas-rzt2h.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Renesas RZ/T2H Interrupt Control Unit (ICU)
+ *
+ * Copyright (C) 2025 Renesas Electronics Corporation.
+ */
+
+#ifndef __LINUX_IRQ_RENESAS_RZT2H
+#define __LINUX_IRQ_RENESAS_RZT2H
+
+#include <linux/platform_device.h>
+
+#define RZT2H_ICU_DMAC_REQ_NO_DEFAULT		0x3ff
+
+#ifdef CONFIG_RENESAS_RZT2H_ICU
+void rzt2h_icu_register_dma_req(struct platform_device *icu_dev, u8 dmac_index, u8 dmac_channel,
+				u16 req_no);
+#else
+static inline void rzt2h_icu_register_dma_req(struct platform_device *icu_dev, u8 dmac_index,
+					      u8 dmac_channel, u16 req_no) { }
+#endif
+
+#endif /* __LINUX_IRQ_RENESAS_RZT2H */
diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h
index 17902861de76..dae9a9b93665 100644
--- a/include/linux/irqdesc.h
+++ b/include/linux/irqdesc.h
@@ -2,9 +2,10 @@
 #ifndef _LINUX_IRQDESC_H
 #define _LINUX_IRQDESC_H
 
-#include <linux/rcupdate.h>
+#include <linux/irq_work.h>
 #include <linux/kobject.h>
 #include <linux/mutex.h>
+#include <linux/rcupdate.h>
 
 /*
  * Core internal functions to deal with irq descriptors
@@ -29,6 +30,17 @@ struct irqstat {
 #endif
 };
 
+/**
+ * struct irq_redirect - interrupt redirection metadata
+ * @work:	Harg irq_work item for handler execution on a different CPU
+ * @target_cpu:	CPU to run irq handler on in case the current CPU is not part
+ *		of the irq affinity mask
+ */
+struct irq_redirect {
+	struct irq_work	work;
+	unsigned int	target_cpu;
+};
+
 /**
  * struct irq_desc - interrupt descriptor
  * @irq_common_data:	per irq and chip data passed down to chip functions
@@ -46,6 +58,7 @@ struct irqstat {
  * @threads_handled:	stats field for deferred spurious detection of threaded handlers
  * @threads_handled_last: comparator field for deferred spurious detection of threaded handlers
  * @lock:		locking for SMP
+ * @redirect:		Facility for redirecting interrupts via irq_work
  * @affinity_hint:	hint to user space for preferred irq affinity
  * @affinity_notify:	context for notification of affinity changes
  * @pending_mask:	pending rebalanced interrupts
@@ -83,6 +96,7 @@ struct irq_desc {
 	raw_spinlock_t		lock;
 	struct cpumask		*percpu_enabled;
 #ifdef CONFIG_SMP
+	struct irq_redirect	redirect;
 	const struct cpumask	*affinity_hint;
 	struct irq_affinity_notify *affinity_notify;
 #ifdef CONFIG_GENERIC_PENDING_IRQ
@@ -185,6 +199,7 @@ int generic_handle_irq_safe(unsigned int irq);
 int generic_handle_domain_irq(struct irq_domain *domain, irq_hw_number_t hwirq);
 int generic_handle_domain_irq_safe(struct irq_domain *domain, irq_hw_number_t hwirq);
 int generic_handle_domain_nmi(struct irq_domain *domain, irq_hw_number_t hwirq);
+bool generic_handle_demux_domain_irq(struct irq_domain *domain, irq_hw_number_t hwirq);
 #endif
 
 /* Test to see if a driver has successfully requested an irq */
diff --git a/include/linux/kasan.h b/include/linux/kasan.h
index f335c1d7b61d..9c6ac4b62eb9 100644
--- a/include/linux/kasan.h
+++ b/include/linux/kasan.h
@@ -28,6 +28,7 @@ typedef unsigned int __bitwise kasan_vmalloc_flags_t;
 #define KASAN_VMALLOC_INIT		((__force kasan_vmalloc_flags_t)0x01u)
 #define KASAN_VMALLOC_VM_ALLOC		((__force kasan_vmalloc_flags_t)0x02u)
 #define KASAN_VMALLOC_PROT_NORMAL	((__force kasan_vmalloc_flags_t)0x04u)
+#define KASAN_VMALLOC_KEEP_TAG		((__force kasan_vmalloc_flags_t)0x08u)
 
 #define KASAN_VMALLOC_PAGE_RANGE 0x1 /* Apply exsiting page range */
 #define KASAN_VMALLOC_TLB_FLUSH  0x2 /* TLB flush */
@@ -630,6 +631,16 @@ static __always_inline void kasan_poison_vmalloc(const void *start,
 		__kasan_poison_vmalloc(start, size);
 }
 
+void __kasan_unpoison_vmap_areas(struct vm_struct **vms, int nr_vms,
+				 kasan_vmalloc_flags_t flags);
+static __always_inline void
+kasan_unpoison_vmap_areas(struct vm_struct **vms, int nr_vms,
+			  kasan_vmalloc_flags_t flags)
+{
+	if (kasan_enabled())
+		__kasan_unpoison_vmap_areas(vms, nr_vms, flags);
+}
+
 #else /* CONFIG_KASAN_VMALLOC */
 
 static inline void kasan_populate_early_vm_area_shadow(void *start,
@@ -654,6 +665,11 @@ static inline void *kasan_unpoison_vmalloc(const void *start,
 static inline void kasan_poison_vmalloc(const void *start, unsigned long size)
 { }
 
+static __always_inline void
+kasan_unpoison_vmap_areas(struct vm_struct **vms, int nr_vms,
+			  kasan_vmalloc_flags_t flags)
+{ }
+
 #endif /* CONFIG_KASAN_VMALLOC */
 
 #if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index 5b46924fdff5..35b8f2a5aca5 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -21,7 +21,6 @@
 #include <linux/compiler.h>
 #include <linux/container_of.h>
 #include <linux/bitops.h>
-#include <linux/hex.h>
 #include <linux/kstrtox.h>
 #include <linux/log2.h>
 #include <linux/math.h>
diff --git a/include/linux/kexec.h b/include/linux/kexec.h
index ff7e231b0485..8a22bc9b8c6c 100644
--- a/include/linux/kexec.h
+++ b/include/linux/kexec.h
@@ -530,7 +530,7 @@ extern bool kexec_file_dbg_print;
 #define kexec_dprintk(fmt, arg...) \
         do { if (kexec_file_dbg_print) pr_info(fmt, ##arg); } while (0)
 
-extern void *kimage_map_segment(struct kimage *image, unsigned long addr, unsigned long size);
+extern void *kimage_map_segment(struct kimage *image, int idx);
 extern void kimage_unmap_segment(void *buffer);
 #else /* !CONFIG_KEXEC_CORE */
 struct pt_regs;
@@ -540,7 +540,7 @@ static inline void __crash_kexec(struct pt_regs *regs) { }
 static inline void crash_kexec(struct pt_regs *regs) { }
 static inline int kexec_should_crash(struct task_struct *p) { return 0; }
 static inline int kexec_crash_loaded(void) { return 0; }
-static inline void *kimage_map_segment(struct kimage *image, unsigned long addr, unsigned long size)
+static inline void *kimage_map_segment(struct kimage *image, int idx)
 { return NULL; }
 static inline void kimage_unmap_segment(void *buffer) { }
 #define kexec_in_progress false
diff --git a/include/linux/kho/abi/luo.h b/include/linux/kho/abi/luo.h
index bb099c92e469..4a1cc6a5f3f8 100644
--- a/include/linux/kho/abi/luo.h
+++ b/include/linux/kho/abi/luo.h
@@ -37,6 +37,11 @@
  *             compatible = "luo-session-v1";
  *             luo-session-header = <phys_addr_of_session_header_ser>;
  *         };
+ *
+ *         luo-flb {
+ *             compatible = "luo-flb-v1";
+ *             luo-flb-header = <phys_addr_of_flb_header_ser>;
+ *         };
  *     };
  *
  * Main LUO Node (/):
@@ -56,6 +61,17 @@
  *     is the header for a contiguous block of memory containing an array of
  *     `struct luo_session_ser`, one for each preserved session.
  *
+ * File-Lifecycle-Bound Node (luo-flb):
+ *   This node describes all preserved global objects whose lifecycle is bound
+ *   to that of the preserved files (e.g., shared IOMMU state).
+ *
+ *   - compatible: "luo-flb-v1"
+ *     Identifies the FLB ABI version.
+ *   - luo-flb-header: u64
+ *     The physical address of a `struct luo_flb_header_ser`. This structure is
+ *     the header for a contiguous block of memory containing an array of
+ *     `struct luo_flb_ser`, one for each preserved global object.
+ *
  * Serialization Structures:
  *   The FDT properties point to memory regions containing arrays of simple,
  *   `__packed` structures. These structures contain the actual preserved state.
@@ -74,6 +90,16 @@
  *     Metadata for a single preserved file. Contains the `compatible` string to
  *     find the correct handler in the new kernel, a user-provided `token` for
  *     identification, and an opaque `data` handle for the handler to use.
+ *
+ *   - struct luo_flb_header_ser:
+ *     Header for the FLB array. Contains the total page count of the
+ *     preserved memory block and the number of `struct luo_flb_ser` entries
+ *     that follow.
+ *
+ *   - struct luo_flb_ser:
+ *     Metadata for a single preserved global object. Contains its `name`
+ *     (compatible string), an opaque `data` handle, and the `count`
+ *     number of files depending on it.
  */
 
 #ifndef _LINUX_KHO_ABI_LUO_H
@@ -163,4 +189,59 @@ struct luo_session_ser {
 	struct luo_file_set_ser file_set_ser;
 } __packed;
 
+/* The max size is set so it can be reliably used during in serialization */
+#define LIVEUPDATE_FLB_COMPAT_LENGTH	48
+
+#define LUO_FDT_FLB_NODE_NAME	"luo-flb"
+#define LUO_FDT_FLB_COMPATIBLE	"luo-flb-v1"
+#define LUO_FDT_FLB_HEADER	"luo-flb-header"
+
+/**
+ * struct luo_flb_header_ser - Header for the serialized FLB data block.
+ * @pgcnt: The total number of pages occupied by the entire preserved memory
+ *         region, including this header and the subsequent array of
+ *         &struct luo_flb_ser entries.
+ * @count: The number of &struct luo_flb_ser entries that follow this header
+ *         in the memory block.
+ *
+ * This structure is located at the physical address specified by the
+ * `LUO_FDT_FLB_HEADER` FDT property. It provides the new kernel with the
+ * necessary information to find and iterate over the array of preserved
+ * File-Lifecycle-Bound objects and to manage the underlying memory.
+ *
+ * If this structure is modified, LUO_FDT_FLB_COMPATIBLE must be updated.
+ */
+struct luo_flb_header_ser {
+	u64 pgcnt;
+	u64 count;
+} __packed;
+
+/**
+ * struct luo_flb_ser - Represents the serialized state of a single FLB object.
+ * @name:    The unique compatibility string of the FLB object, used to find the
+ *           corresponding &struct liveupdate_flb handler in the new kernel.
+ * @data:    The opaque u64 handle returned by the FLB's .preserve() operation
+ *           in the old kernel. This handle encapsulates the entire state needed
+ *           for restoration.
+ * @count:   The reference count at the time of serialization; i.e., the number
+ *           of preserved files that depended on this FLB. This is used by the
+ *           new kernel to correctly manage the FLB's lifecycle.
+ *
+ * An array of these structures is created in a preserved memory region and
+ * passed to the new kernel. Each entry allows the LUO core to restore one
+ * global, shared object.
+ *
+ * If this structure is modified, LUO_FDT_FLB_COMPATIBLE must be updated.
+ */
+struct luo_flb_ser {
+	char name[LIVEUPDATE_FLB_COMPAT_LENGTH];
+	u64 data;
+	u64 count;
+} __packed;
+
+/* Kernel Live Update Test ABI */
+#ifdef CONFIG_LIVEUPDATE_TEST
+#define LIVEUPDATE_TEST_FLB_COMPATIBLE(i)	"liveupdate-test-flb-v" #i
+#endif
+
 #endif /* _LINUX_KHO_ABI_LUO_H */
diff --git a/include/linux/leafops.h b/include/linux/leafops.h
index cfafe7a5e7b1..a9ff94b744f2 100644
--- a/include/linux/leafops.h
+++ b/include/linux/leafops.h
@@ -133,7 +133,7 @@ static inline bool softleaf_is_none(softleaf_t entry)
 
 /**
  * softleaf_type() - Identify the type of leaf entry.
- * @enntry: Leaf entry.
+ * @entry: Leaf entry.
  *
  * Returns: the leaf entry type associated with @entry.
  */
@@ -534,7 +534,7 @@ static inline bool pte_is_uffd_wp_marker(pte_t pte)
 /**
  * pte_is_uffd_marker() - Does this PTE entry encode a userfault-specific marker
  * leaf entry?
- * @entry: Leaf entry.
+ * @pte: PTE entry.
  *
  * It's useful to be able to determine which leaf entries encode UFFD-specific
  * markers so we can handle these correctly.
diff --git a/include/linux/libata.h b/include/linux/libata.h
index 39534fafa36a..aa88244f3d83 100644
--- a/include/linux/libata.h
+++ b/include/linux/libata.h
@@ -46,13 +46,12 @@
 
 /*
  * Quirk flags bits.
- * ata_device->quirks is an unsigned int, so __ATA_QUIRK_MAX must not exceed 32.
+ * ata_device->quirks is an u64, so __ATA_QUIRK_MAX must not exceed 64.
  */
 enum ata_quirks {
 	__ATA_QUIRK_DIAGNOSTIC,		/* Failed boot diag */
 	__ATA_QUIRK_NODMA,		/* DMA problems */
 	__ATA_QUIRK_NONCQ,		/* Don't use NCQ */
-	__ATA_QUIRK_MAX_SEC_128,	/* Limit max sects to 128 */
 	__ATA_QUIRK_BROKEN_HPA,		/* Broken HPA */
 	__ATA_QUIRK_DISABLE,		/* Disable it */
 	__ATA_QUIRK_HPA_SIZE,		/* Native size off by one */
@@ -74,8 +73,7 @@ enum ata_quirks {
 	__ATA_QUIRK_ZERO_AFTER_TRIM,	/* Guarantees zero after trim */
 	__ATA_QUIRK_NO_DMA_LOG,		/* Do not use DMA for log read */
 	__ATA_QUIRK_NOTRIM,		/* Do not use TRIM */
-	__ATA_QUIRK_MAX_SEC_1024,	/* Limit max sects to 1024 */
-	__ATA_QUIRK_MAX_SEC_8191,	/* Limit max sects to 8191 */
+	__ATA_QUIRK_MAX_SEC,		/* Limit max sectors */
 	__ATA_QUIRK_MAX_TRIM_128M,	/* Limit max trim size to 128M */
 	__ATA_QUIRK_NO_NCQ_ON_ATI,	/* Disable NCQ on ATI chipset */
 	__ATA_QUIRK_NO_LPM_ON_ATI,	/* Disable LPM on ATI chipset */
@@ -91,38 +89,36 @@ enum ata_quirks {
  * Some quirks may be drive/controller pair dependent.
  */
 enum {
-	ATA_QUIRK_DIAGNOSTIC		= (1U << __ATA_QUIRK_DIAGNOSTIC),
-	ATA_QUIRK_NODMA			= (1U << __ATA_QUIRK_NODMA),
-	ATA_QUIRK_NONCQ			= (1U << __ATA_QUIRK_NONCQ),
-	ATA_QUIRK_MAX_SEC_128		= (1U << __ATA_QUIRK_MAX_SEC_128),
-	ATA_QUIRK_BROKEN_HPA		= (1U << __ATA_QUIRK_BROKEN_HPA),
-	ATA_QUIRK_DISABLE		= (1U << __ATA_QUIRK_DISABLE),
-	ATA_QUIRK_HPA_SIZE		= (1U << __ATA_QUIRK_HPA_SIZE),
-	ATA_QUIRK_IVB			= (1U << __ATA_QUIRK_IVB),
-	ATA_QUIRK_STUCK_ERR		= (1U << __ATA_QUIRK_STUCK_ERR),
-	ATA_QUIRK_BRIDGE_OK		= (1U << __ATA_QUIRK_BRIDGE_OK),
-	ATA_QUIRK_ATAPI_MOD16_DMA	= (1U << __ATA_QUIRK_ATAPI_MOD16_DMA),
-	ATA_QUIRK_FIRMWARE_WARN		= (1U << __ATA_QUIRK_FIRMWARE_WARN),
-	ATA_QUIRK_1_5_GBPS		= (1U << __ATA_QUIRK_1_5_GBPS),
-	ATA_QUIRK_NOSETXFER		= (1U << __ATA_QUIRK_NOSETXFER),
-	ATA_QUIRK_BROKEN_FPDMA_AA	= (1U << __ATA_QUIRK_BROKEN_FPDMA_AA),
-	ATA_QUIRK_DUMP_ID		= (1U << __ATA_QUIRK_DUMP_ID),
-	ATA_QUIRK_MAX_SEC_LBA48		= (1U << __ATA_QUIRK_MAX_SEC_LBA48),
-	ATA_QUIRK_ATAPI_DMADIR		= (1U << __ATA_QUIRK_ATAPI_DMADIR),
-	ATA_QUIRK_NO_NCQ_TRIM		= (1U << __ATA_QUIRK_NO_NCQ_TRIM),
-	ATA_QUIRK_NOLPM			= (1U << __ATA_QUIRK_NOLPM),
-	ATA_QUIRK_WD_BROKEN_LPM		= (1U << __ATA_QUIRK_WD_BROKEN_LPM),
-	ATA_QUIRK_ZERO_AFTER_TRIM	= (1U << __ATA_QUIRK_ZERO_AFTER_TRIM),
-	ATA_QUIRK_NO_DMA_LOG		= (1U << __ATA_QUIRK_NO_DMA_LOG),
-	ATA_QUIRK_NOTRIM		= (1U << __ATA_QUIRK_NOTRIM),
-	ATA_QUIRK_MAX_SEC_1024		= (1U << __ATA_QUIRK_MAX_SEC_1024),
-	ATA_QUIRK_MAX_SEC_8191		= (1U << __ATA_QUIRK_MAX_SEC_8191),
-	ATA_QUIRK_MAX_TRIM_128M		= (1U << __ATA_QUIRK_MAX_TRIM_128M),
-	ATA_QUIRK_NO_NCQ_ON_ATI		= (1U << __ATA_QUIRK_NO_NCQ_ON_ATI),
-	ATA_QUIRK_NO_LPM_ON_ATI		= (1U << __ATA_QUIRK_NO_LPM_ON_ATI),
-	ATA_QUIRK_NO_ID_DEV_LOG		= (1U << __ATA_QUIRK_NO_ID_DEV_LOG),
-	ATA_QUIRK_NO_LOG_DIR		= (1U << __ATA_QUIRK_NO_LOG_DIR),
-	ATA_QUIRK_NO_FUA		= (1U << __ATA_QUIRK_NO_FUA),
+	ATA_QUIRK_DIAGNOSTIC		= BIT_ULL(__ATA_QUIRK_DIAGNOSTIC),
+	ATA_QUIRK_NODMA			= BIT_ULL(__ATA_QUIRK_NODMA),
+	ATA_QUIRK_NONCQ			= BIT_ULL(__ATA_QUIRK_NONCQ),
+	ATA_QUIRK_BROKEN_HPA		= BIT_ULL(__ATA_QUIRK_BROKEN_HPA),
+	ATA_QUIRK_DISABLE		= BIT_ULL(__ATA_QUIRK_DISABLE),
+	ATA_QUIRK_HPA_SIZE		= BIT_ULL(__ATA_QUIRK_HPA_SIZE),
+	ATA_QUIRK_IVB			= BIT_ULL(__ATA_QUIRK_IVB),
+	ATA_QUIRK_STUCK_ERR		= BIT_ULL(__ATA_QUIRK_STUCK_ERR),
+	ATA_QUIRK_BRIDGE_OK		= BIT_ULL(__ATA_QUIRK_BRIDGE_OK),
+	ATA_QUIRK_ATAPI_MOD16_DMA	= BIT_ULL(__ATA_QUIRK_ATAPI_MOD16_DMA),
+	ATA_QUIRK_FIRMWARE_WARN		= BIT_ULL(__ATA_QUIRK_FIRMWARE_WARN),
+	ATA_QUIRK_1_5_GBPS		= BIT_ULL(__ATA_QUIRK_1_5_GBPS),
+	ATA_QUIRK_NOSETXFER		= BIT_ULL(__ATA_QUIRK_NOSETXFER),
+	ATA_QUIRK_BROKEN_FPDMA_AA	= BIT_ULL(__ATA_QUIRK_BROKEN_FPDMA_AA),
+	ATA_QUIRK_DUMP_ID		= BIT_ULL(__ATA_QUIRK_DUMP_ID),
+	ATA_QUIRK_MAX_SEC_LBA48		= BIT_ULL(__ATA_QUIRK_MAX_SEC_LBA48),
+	ATA_QUIRK_ATAPI_DMADIR		= BIT_ULL(__ATA_QUIRK_ATAPI_DMADIR),
+	ATA_QUIRK_NO_NCQ_TRIM		= BIT_ULL(__ATA_QUIRK_NO_NCQ_TRIM),
+	ATA_QUIRK_NOLPM			= BIT_ULL(__ATA_QUIRK_NOLPM),
+	ATA_QUIRK_WD_BROKEN_LPM		= BIT_ULL(__ATA_QUIRK_WD_BROKEN_LPM),
+	ATA_QUIRK_ZERO_AFTER_TRIM	= BIT_ULL(__ATA_QUIRK_ZERO_AFTER_TRIM),
+	ATA_QUIRK_NO_DMA_LOG		= BIT_ULL(__ATA_QUIRK_NO_DMA_LOG),
+	ATA_QUIRK_NOTRIM		= BIT_ULL(__ATA_QUIRK_NOTRIM),
+	ATA_QUIRK_MAX_SEC		= BIT_ULL(__ATA_QUIRK_MAX_SEC),
+	ATA_QUIRK_MAX_TRIM_128M		= BIT_ULL(__ATA_QUIRK_MAX_TRIM_128M),
+	ATA_QUIRK_NO_NCQ_ON_ATI		= BIT_ULL(__ATA_QUIRK_NO_NCQ_ON_ATI),
+	ATA_QUIRK_NO_LPM_ON_ATI		= BIT_ULL(__ATA_QUIRK_NO_LPM_ON_ATI),
+	ATA_QUIRK_NO_ID_DEV_LOG		= BIT_ULL(__ATA_QUIRK_NO_ID_DEV_LOG),
+	ATA_QUIRK_NO_LOG_DIR		= BIT_ULL(__ATA_QUIRK_NO_LOG_DIR),
+	ATA_QUIRK_NO_FUA		= BIT_ULL(__ATA_QUIRK_NO_FUA),
 };
 
 enum {
@@ -723,7 +719,7 @@ struct ata_cdl {
 struct ata_device {
 	struct ata_link		*link;
 	unsigned int		devno;		/* 0 or 1 */
-	unsigned int		quirks;		/* List of broken features */
+	u64			quirks;		/* List of broken features */
 	unsigned long		flags;		/* ATA_DFLAG_xxx */
 	struct scsi_device	*sdev;		/* attached SCSI device */
 	void			*private_data;
diff --git a/include/linux/liveupdate.h b/include/linux/liveupdate.h
index a7f6ee5b6771..ed81e7b31a9f 100644
--- a/include/linux/liveupdate.h
+++ b/include/linux/liveupdate.h
@@ -11,10 +11,13 @@
 #include <linux/compiler.h>
 #include <linux/kho/abi/luo.h>
 #include <linux/list.h>
+#include <linux/mutex.h>
 #include <linux/types.h>
 #include <uapi/linux/liveupdate.h>
 
 struct liveupdate_file_handler;
+struct liveupdate_flb;
+struct liveupdate_session;
 struct file;
 
 /**
@@ -99,6 +102,114 @@ struct liveupdate_file_handler {
 	 * registered file handlers.
 	 */
 	struct list_head __private list;
+	/* A list of FLB dependencies. */
+	struct list_head __private flb_list;
+};
+
+/**
+ * struct liveupdate_flb_op_args - Arguments for FLB operation callbacks.
+ * @flb:       The global FLB instance for which this call is performed.
+ * @data:      For .preserve():    [OUT] The callback sets this field.
+ *             For .unpreserve():  [IN]  The handle from .preserve().
+ *             For .retrieve():    [IN]  The handle from .preserve().
+ * @obj:       For .preserve():    [OUT] Sets this to the live object.
+ *             For .retrieve():    [OUT] Sets this to the live object.
+ *             For .finish():      [IN]  The live object from .retrieve().
+ *
+ * This structure bundles all parameters for the FLB operation callbacks.
+ */
+struct liveupdate_flb_op_args {
+	struct liveupdate_flb *flb;
+	u64 data;
+	void *obj;
+};
+
+/**
+ * struct liveupdate_flb_ops - Callbacks for global File-Lifecycle-Bound data.
+ * @preserve:        Called when the first file using this FLB is preserved.
+ *                   The callback must save its state and return a single,
+ *                   self-contained u64 handle by setting the 'argp->data'
+ *                   field and 'argp->obj'.
+ * @unpreserve:      Called when the last file using this FLB is unpreserved
+ *                   (aborted before reboot). Receives the handle via
+ *                   'argp->data' and live object via 'argp->obj'.
+ * @retrieve:        Called on-demand in the new kernel, the first time a
+ *                   component requests access to the shared object. It receives
+ *                   the preserved handle via 'argp->data' and must reconstruct
+ *                   the live object, returning it by setting the 'argp->obj'
+ *                   field.
+ * @finish:          Called in the new kernel when the last file using this FLB
+ *                   is finished. Receives the live object via 'argp->obj' for
+ *                   cleanup.
+ * @owner:           Module reference
+ *
+ * Operations that manage global shared data with file bound lifecycle,
+ * triggered by the first file that uses it and concluded by the last file that
+ * uses it, across all sessions.
+ */
+struct liveupdate_flb_ops {
+	int (*preserve)(struct liveupdate_flb_op_args *argp);
+	void (*unpreserve)(struct liveupdate_flb_op_args *argp);
+	int (*retrieve)(struct liveupdate_flb_op_args *argp);
+	void (*finish)(struct liveupdate_flb_op_args *argp);
+	struct module *owner;
+};
+
+/*
+ * struct luo_flb_private_state - Private FLB state structures.
+ * @count: The number of preserved files currently depending on this FLB.
+ *         This is used to trigger the preserve/unpreserve/finish ops on the
+ *         first/last file.
+ * @data:  The opaque u64 handle returned by .preserve() or passed to
+ *         .retrieve().
+ * @obj:   The live kernel object returned by .preserve() or .retrieve().
+ * @lock:  A mutex that protects all fields within this structure, providing
+ *         the synchronization service for the FLB's ops.
+ */
+struct luo_flb_private_state {
+	long count;
+	u64 data;
+	void *obj;
+	struct mutex lock;
+};
+
+/*
+ * struct luo_flb_private - Keep separate incoming and outgoing states.
+ * @list:        A global list of registered FLBs.
+ * @outgoing:    The runtime state for the pre-reboot
+ *               (preserve/unpreserve) lifecycle.
+ * @incoming:    The runtime state for the post-reboot (retrieve/finish)
+ *               lifecycle.
+ * @users:       With how many File-Handlers this FLB is registered.
+ * @initialized: true when private fields have been initialized.
+ */
+struct luo_flb_private {
+	struct list_head list;
+	struct luo_flb_private_state outgoing;
+	struct luo_flb_private_state incoming;
+	int users;
+	bool initialized;
+};
+
+/**
+ * struct liveupdate_flb - A global definition for a shared data object.
+ * @ops:         Callback functions
+ * @compatible:  The compatibility string (e.g., "iommu-core-v1"
+ *               that uniquely identifies the FLB type this handler
+ *               supports. This is matched against the compatible string
+ *               associated with individual &struct liveupdate_flb
+ *               instances.
+ *
+ * This struct is the "template" that a driver registers to define a shared,
+ * file-lifecycle-bound object. The actual runtime state (the live object,
+ * refcount, etc.) is managed privately by the LUO core.
+ */
+struct liveupdate_flb {
+	const struct liveupdate_flb_ops *ops;
+	const char compatible[LIVEUPDATE_FLB_COMPAT_LENGTH];
+
+	/* private: */
+	struct luo_flb_private __private private;
 };
 
 #ifdef CONFIG_LIVEUPDATE
@@ -112,6 +223,14 @@ int liveupdate_reboot(void);
 int liveupdate_register_file_handler(struct liveupdate_file_handler *fh);
 int liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh);
 
+int liveupdate_register_flb(struct liveupdate_file_handler *fh,
+			    struct liveupdate_flb *flb);
+int liveupdate_unregister_flb(struct liveupdate_file_handler *fh,
+			      struct liveupdate_flb *flb);
+
+int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp);
+int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp);
+
 #else /* CONFIG_LIVEUPDATE */
 
 static inline bool liveupdate_enabled(void)
@@ -134,5 +253,29 @@ static inline int liveupdate_unregister_file_handler(struct liveupdate_file_hand
 	return -EOPNOTSUPP;
 }
 
+static inline int liveupdate_register_flb(struct liveupdate_file_handler *fh,
+					  struct liveupdate_flb *flb)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int liveupdate_unregister_flb(struct liveupdate_file_handler *fh,
+					    struct liveupdate_flb *flb)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int liveupdate_flb_get_incoming(struct liveupdate_flb *flb,
+					      void **objp)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb,
+					      void **objp)
+{
+	return -EOPNOTSUPP;
+}
+
 #endif /* CONFIG_LIVEUPDATE */
 #endif /* _LINUX_LIVEUPDATE_H */
diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h
index b92008641242..d48bf0ad26f4 100644
--- a/include/linux/lsm_hooks.h
+++ b/include/linux/lsm_hooks.h
@@ -73,7 +73,7 @@ struct lsm_static_calls_table {
 
 /**
  * struct lsm_id - Identify a Linux Security Module.
- * @lsm: name of the LSM, must be approved by the LSM maintainers
+ * @name: name of the LSM, must be approved by the LSM maintainers
  * @id: LSM ID number from uapi/linux/lsm.h
  *
  * Contains the information that identifies the LSM.
@@ -164,7 +164,7 @@ enum lsm_order {
  * @initcall_core: LSM callback for core_initcall() setup, optional
  * @initcall_subsys: LSM callback for subsys_initcall() setup, optional
  * @initcall_fs: LSM callback for fs_initcall setup, optional
- * @nitcall_device: LSM callback for device_initcall() setup, optional
+ * @initcall_device: LSM callback for device_initcall() setup, optional
  * @initcall_late: LSM callback for late_initcall() setup, optional
  */
 struct lsm_info {
diff --git a/include/linux/mailbox/mtk-cmdq-mailbox.h b/include/linux/mailbox/mtk-cmdq-mailbox.h
index e1555e06e7e5..07c1bfbdb8c4 100644
--- a/include/linux/mailbox/mtk-cmdq-mailbox.h
+++ b/include/linux/mailbox/mtk-cmdq-mailbox.h
@@ -70,13 +70,32 @@ struct cmdq_cb_data {
 	struct cmdq_pkt		*pkt;
 };
 
+struct cmdq_mbox_priv {
+	u8 shift_pa;
+	dma_addr_t mminfra_offset;
+};
+
 struct cmdq_pkt {
 	void			*va_base;
 	dma_addr_t		pa_base;
 	size_t			cmd_buf_size; /* command occupied size */
 	size_t			buf_size; /* real buffer size */
+	struct cmdq_mbox_priv	priv; /* for generating instruction */
 };
 
+/**
+ * cmdq_get_mbox_priv() - get the private data of mailbox channel
+ * @chan: mailbox channel
+ * @priv: pointer to store the private data of mailbox channel
+ *
+ * While generating the GCE instruction to command buffer, the private data
+ * of GCE hardware may need to be referenced, such as the shift bits of
+ * physical address.
+ *
+ * This function should be called before generating the GCE instruction.
+ */
+void cmdq_get_mbox_priv(struct mbox_chan *chan, struct cmdq_mbox_priv *priv);
+
 /**
  * cmdq_get_shift_pa() - get the shift bits of physical address
  * @chan: mailbox channel
diff --git a/include/linux/mailbox/mtk-vcp-mailbox.h b/include/linux/mailbox/mtk-vcp-mailbox.h
new file mode 100644
index 000000000000..16e59d6780a7
--- /dev/null
+++ b/include/linux/mailbox/mtk-vcp-mailbox.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+/*
+ * Copyright (c) 2025 MediaTek Inc.
+ */
+
+#ifndef __MTK_VCP_MAILBOX_H__
+#define __MTK_VCP_MAILBOX_H__
+
+#define MTK_VCP_MBOX_SLOT_MAX_SIZE	0x100 /* mbox max slot size */
+
+/**
+ * struct mtk_ipi_info - mailbox message info for mtk-vcp-mailbox
+ * @msg: The share buffer between IPC and mailbox driver
+ * @len: Message length
+ * @id: This is for identification purposes and not actually used
+ *	by the mailbox hardware.
+ * @index: The signal number of the mailbox message.
+ * @slot_ofs: Data slot offset.
+ * @irq_status: Captures incoming signals for the RX path.
+ *
+ * It is used between IPC with mailbox driver.
+ */
+struct mtk_ipi_info {
+	void *msg;
+	u32 len;
+	u32 id;
+	u32 index;
+	u32 slot_ofs;
+	u32 irq_status;
+};
+
+#endif
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 0651865a4564..6a48398a1f4e 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -893,7 +893,7 @@ static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
 {
 	if (mem_cgroup_disabled())
 		return true;
-	return !!(memcg->css.flags & CSS_ONLINE);
+	return css_is_online(&memcg->css);
 }
 
 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
diff --git a/include/linux/memory-failure.h b/include/linux/memory-failure.h
index bc326503d2d2..7b5e11cf905f 100644
--- a/include/linux/memory-failure.h
+++ b/include/linux/memory-failure.h
@@ -9,6 +9,8 @@ struct pfn_address_space;
 struct pfn_address_space {
 	struct interval_tree_node node;
 	struct address_space *mapping;
+	int (*pfn_to_vma_pgoff)(struct vm_area_struct *vma,
+				unsigned long pfn, pgoff_t *pgoff);
 };
 
 int register_pfn_address_space(struct pfn_address_space *pfn_space);
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 15076261d0c2..d649da117340 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -376,6 +376,11 @@ enum {
 	 * has similar constraints to shadow stacks.
 	 */
 	DECLARE_VMA_BIT_ALIAS(SHADOW_STACK, HIGH_ARCH_6),
+#elif defined(CONFIG_RISCV_USER_CFI)
+	/*
+	 * Following x86 and picking up the same bitpos.
+	 */
+	DECLARE_VMA_BIT_ALIAS(SHADOW_STACK, HIGH_ARCH_5),
 #endif
 	DECLARE_VMA_BIT_ALIAS(SAO, ARCH_1),		/* Strong Access Ordering (powerpc) */
 	DECLARE_VMA_BIT_ALIAS(GROWSUP, ARCH_1),		/* parisc */
@@ -2459,10 +2464,10 @@ static inline int folio_expected_ref_count(const struct folio *folio)
 	if (WARN_ON_ONCE(page_has_type(&folio->page) && !folio_test_hugetlb(folio)))
 		return 0;
 
-	if (folio_test_anon(folio)) {
-		/* One reference per page from the swapcache. */
-		ref_count += folio_test_swapcache(folio) << order;
-	} else {
+	/* One reference per page from the swapcache. */
+	ref_count += folio_test_swapcache(folio) << order;
+
+	if (!folio_test_anon(folio)) {
 		/* One reference per page from the pagecache. */
 		ref_count += !!folio->mapping << order;
 		/* One reference from PG_private. */
@@ -2843,38 +2848,74 @@ static inline bool get_user_page_fast_only(unsigned long addr,
 {
 	return get_user_pages_fast_only(addr, 1, gup_flags, pagep) == 1;
 }
+
+static inline size_t get_rss_stat_items_size(void)
+{
+	return percpu_counter_tree_items_size() * NR_MM_COUNTERS;
+}
+
+static inline struct percpu_counter_tree_level_item *get_rss_stat_items(struct mm_struct *mm)
+{
+	unsigned long ptr = (unsigned long)mm;
+
+	ptr += offsetof(struct mm_struct, cpu_bitmap);
+	/* Skip cpu_bitmap */
+	ptr += cpumask_size();
+	/* Skip mm_cidmask */
+	ptr += mm_cid_size();
+	return (struct percpu_counter_tree_level_item *)ptr;
+}
+
 /*
  * per-process(per-mm_struct) statistics.
  */
+static inline unsigned long __get_mm_counter(struct mm_struct *mm, int member, bool approximate,
+					     unsigned int *accuracy_under, unsigned int *accuracy_over)
+{
+	if (approximate) {
+		if (accuracy_under && accuracy_over) {
+			unsigned int under, over;
+
+			percpu_counter_tree_approximate_accuracy_range(&mm->rss_stat[member], &under, &over);
+			*accuracy_under += under;
+			*accuracy_over += over;
+		}
+		return percpu_counter_tree_approximate_sum_positive(&mm->rss_stat[member]);
+	} else {
+		return percpu_counter_tree_precise_sum_positive(&mm->rss_stat[member]);
+	}
+}
+
 static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
 {
-	return percpu_counter_read_positive(&mm->rss_stat[member]);
+	return __get_mm_counter(mm, member, true, NULL, NULL);
 }
 
+
 static inline unsigned long get_mm_counter_sum(struct mm_struct *mm, int member)
 {
-	return percpu_counter_sum_positive(&mm->rss_stat[member]);
+	return get_mm_counter(mm, member);
 }
 
 void mm_trace_rss_stat(struct mm_struct *mm, int member);
 
 static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
 {
-	percpu_counter_add(&mm->rss_stat[member], value);
+	percpu_counter_tree_add(&mm->rss_stat[member], value);
 
 	mm_trace_rss_stat(mm, member);
 }
 
 static inline void inc_mm_counter(struct mm_struct *mm, int member)
 {
-	percpu_counter_inc(&mm->rss_stat[member]);
+	percpu_counter_tree_add(&mm->rss_stat[member], 1);
 
 	mm_trace_rss_stat(mm, member);
 }
 
 static inline void dec_mm_counter(struct mm_struct *mm, int member)
 {
-	percpu_counter_dec(&mm->rss_stat[member]);
+	percpu_counter_tree_add(&mm->rss_stat[member], -1);
 
 	mm_trace_rss_stat(mm, member);
 }
@@ -2894,11 +2935,17 @@ static inline int mm_counter(struct folio *folio)
 	return mm_counter_file(folio);
 }
 
+static inline unsigned long __get_mm_rss(struct mm_struct *mm, bool approximate,
+					 unsigned int *accuracy_under, unsigned int *accuracy_over)
+{
+	return __get_mm_counter(mm, MM_FILEPAGES, approximate, accuracy_under, accuracy_over) +
+		__get_mm_counter(mm, MM_ANONPAGES, approximate, accuracy_under, accuracy_over) +
+		__get_mm_counter(mm, MM_SHMEMPAGES, approximate, accuracy_under, accuracy_over);
+}
+
 static inline unsigned long get_mm_rss(struct mm_struct *mm)
 {
-	return get_mm_counter(mm, MM_FILEPAGES) +
-		get_mm_counter(mm, MM_ANONPAGES) +
-		get_mm_counter(mm, MM_SHMEMPAGES);
+	return __get_mm_rss(mm, true, NULL, NULL);
 }
 
 static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
@@ -4351,7 +4398,7 @@ extern int soft_offline_page(unsigned long pfn, int flags);
 extern const struct attribute_group memory_failure_attr_group;
 extern void memory_failure_queue(unsigned long pfn, int flags);
 extern int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,
-					bool *migratable_cleared);
+					bool *migratable_cleared, bool *samepg);
 void num_poisoned_pages_inc(unsigned long pfn);
 void num_poisoned_pages_sub(unsigned long pfn, long i);
 #else
@@ -4360,7 +4407,7 @@ static inline void memory_failure_queue(unsigned long pfn, int flags)
 }
 
 static inline int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,
-					bool *migratable_cleared)
+					bool *migratable_cleared, bool *samepg)
 {
 	return 0;
 }
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 9f6de068295d..9ae67caadcc8 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -18,7 +18,7 @@
 #include <linux/page-flags-layout.h>
 #include <linux/workqueue.h>
 #include <linux/seqlock.h>
-#include <linux/percpu_counter.h>
+#include <linux/percpu_counter_tree.h>
 #include <linux/types.h>
 #include <linux/rseq_types.h>
 #include <linux/bitmap.h>
@@ -1215,7 +1215,7 @@ struct mm_struct {
 		unsigned long saved_e_flags;
 #endif
 
-		struct percpu_counter rss_stat[NR_MM_COUNTERS];
+		struct percpu_counter_tree rss_stat[NR_MM_COUNTERS];
 
 		struct linux_binfmt *binfmt;
 
@@ -1631,7 +1631,6 @@ enum tlb_flush_reason {
 	TLB_LOCAL_MM_SHOOTDOWN,
 	TLB_REMOTE_SEND_IPI,
 	TLB_REMOTE_WRONG_CPU,
-	NR_TLB_FLUSH_REASONS,
 };
 
 /**
diff --git a/include/linux/mm_types_task.h b/include/linux/mm_types_task.h
index a82aa80c0ba4..11bf319d78ec 100644
--- a/include/linux/mm_types_task.h
+++ b/include/linux/mm_types_task.h
@@ -88,4 +88,9 @@ struct tlbflush_unmap_batch {
 #endif
 };
 
+struct lazy_mmu_state {
+	u8 enable_count;
+	u8 pause_count;
+};
+
 #endif /* _LINUX_MM_TYPES_TASK_H */
diff --git a/include/linux/mmc/card.h b/include/linux/mmc/card.h
index e9e964c20e53..4722dd7e46ce 100644
--- a/include/linux/mmc/card.h
+++ b/include/linux/mmc/card.h
@@ -329,6 +329,7 @@ struct mmc_card {
 #define MMC_QUIRK_BROKEN_CACHE_FLUSH	(1<<16)	/* Don't flush cache until the write has occurred */
 #define MMC_QUIRK_BROKEN_SD_POWEROFF_NOTIFY	(1<<17) /* Disable broken SD poweroff notify support */
 #define MMC_QUIRK_NO_UHS_DDR50_TUNING	(1<<18) /* Disable DDR50 tuning */
+#define MMC_QUIRK_BROKEN_MDT    (1<<19) /* Wrong manufacturing year */
 
 	bool			written_flag;	/* Indicates eMMC has been written since power on */
 	bool			reenable_cmdq;	/* Re-enable Command Queue */
diff --git a/include/linux/module.h b/include/linux/module.h
index d80c3ea57472..ac254525014c 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -748,6 +748,15 @@ static inline void __module_get(struct module *module)
 	__mod ? __mod->name : "kernel";		\
 })
 
+static inline const unsigned char *module_buildid(struct module *mod)
+{
+#ifdef CONFIG_STACKTRACE_BUILD_ID
+	return mod->build_id;
+#else
+	return NULL;
+#endif
+}
+
 /* Dereference module function descriptor */
 void *dereference_module_function_descriptor(struct module *mod, void *ptr);
 
diff --git a/include/linux/msi.h b/include/linux/msi.h
index 8003e3218c46..94cfc3719077 100644
--- a/include/linux/msi.h
+++ b/include/linux/msi.h
@@ -49,12 +49,12 @@ typedef struct arch_msi_msg_data {
 #endif
 
 /**
- * msi_msg - Representation of a MSI message
+ * struct msi_msg - Representation of a MSI message
  * @address_lo:		Low 32 bits of msi message address
- * @arch_addrlo:	Architecture specific shadow of @address_lo
+ * @arch_addr_lo:	Architecture specific shadow of @address_lo
  * @address_hi:		High 32 bits of msi message address
  *			(only used when device supports it)
- * @arch_addrhi:	Architecture specific shadow of @address_hi
+ * @arch_addr_hi:	Architecture specific shadow of @address_hi
  * @data:		MSI message data (usually 16 bits)
  * @arch_data:		Architecture specific shadow of @data
  */
@@ -91,7 +91,7 @@ typedef void (*irq_write_msi_msg_t)(struct msi_desc *desc,
 				    struct msi_msg *msg);
 
 /**
- * pci_msi_desc - PCI/MSI specific MSI descriptor data
+ * struct pci_msi_desc - PCI/MSI specific MSI descriptor data
  *
  * @msi_mask:	[PCI MSI]   MSI cached mask bits
  * @msix_ctrl:	[PCI MSI-X] MSI-X cached per vector control bits
@@ -101,6 +101,7 @@ typedef void (*irq_write_msi_msg_t)(struct msi_desc *desc,
  * @can_mask:	[PCI MSI/X] Masking supported?
  * @is_64:	[PCI MSI/X] Address size: 0=32bit 1=64bit
  * @default_irq:[PCI MSI/X] The default pre-assigned non-MSI irq
+ * @msi_attrib:	[PCI MSI/X] Compound struct of MSI/X attributes
  * @mask_pos:	[PCI MSI]   Mask register position
  * @mask_base:	[PCI MSI-X] Mask register base address
  */
@@ -169,7 +170,7 @@ struct msi_desc_data {
  *                  Only used if iommu_msi_shift != 0
  * @iommu_msi_shift: Indicates how many bits of the original address should be
  *                   preserved when using iommu_msi_iova.
- * @sysfs_attr:	Pointer to sysfs device attribute
+ * @sysfs_attrs:	Pointer to sysfs device attribute
  *
  * @write_msi_msg:	Callback that may be called when the MSI message
  *			address or data changes
@@ -220,7 +221,7 @@ enum msi_desc_filter {
 /**
  * struct msi_dev_domain - The internals of MSI domain info per device
  * @store:		Xarray for storing MSI descriptor pointers
- * @irqdomain:		Pointer to a per device interrupt domain
+ * @domain:		Pointer to a per device interrupt domain
  */
 struct msi_dev_domain {
 	struct xarray		store;
diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h
index ce76f5c632e1..c50a43b447d2 100644
--- a/include/linux/mtd/spinand.h
+++ b/include/linux/mtd/spinand.h
@@ -354,6 +354,7 @@ struct spinand_manufacturer {
 /* SPI NAND manufacturers */
 extern const struct spinand_manufacturer alliancememory_spinand_manufacturer;
 extern const struct spinand_manufacturer ato_spinand_manufacturer;
+extern const struct spinand_manufacturer dosilicon_spinand_manufacturer;
 extern const struct spinand_manufacturer esmt_8c_spinand_manufacturer;
 extern const struct spinand_manufacturer esmt_c8_spinand_manufacturer;
 extern const struct spinand_manufacturer fmsh_spinand_manufacturer;
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 5870a9e514a5..d5c9c83873e3 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -2100,6 +2100,8 @@ enum netdev_reg_state {
  *
  *	FIXME: cleanup struct net_device such that network protocol info
  *	moves out.
+ *
+ *	@netdev_trace_buffer_list: Linked list for debugging refcount leak.
  */
 
 struct net_device {
@@ -2254,6 +2256,9 @@ struct net_device {
 #if IS_ENABLED(CONFIG_TLS_DEVICE)
 	const struct tlsdev_ops *tlsdev_ops;
 #endif
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+	struct list_head	netdev_trace_buffer_list;
+#endif
 
 	unsigned int		operstate;
 	unsigned char		link_mode;
@@ -3182,6 +3187,7 @@ enum netdev_cmd {
 	NETDEV_OFFLOAD_XSTATS_REPORT_USED,
 	NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
 	NETDEV_XDP_FEAT_CHANGE,
+	NETDEV_DEBUG_UNREGISTER,
 };
 const char *netdev_cmd_to_name(enum netdev_cmd cmd);
 
@@ -4370,9 +4376,15 @@ static inline bool dev_nit_active(const struct net_device *dev)
 
 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev);
 
+void save_netdev_trace_buffer(struct net_device *dev, int delta);
+int trim_netdev_trace(unsigned long *entries, int nr_entries);
+
 static inline void __dev_put(struct net_device *dev)
 {
 	if (dev) {
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+		save_netdev_trace_buffer(dev, -1);
+#endif
 #ifdef CONFIG_PCPU_DEV_REFCNT
 		this_cpu_dec(*dev->pcpu_refcnt);
 #else
@@ -4384,6 +4396,9 @@ static inline void __dev_put(struct net_device *dev)
 static inline void __dev_hold(struct net_device *dev)
 {
 	if (dev) {
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+		save_netdev_trace_buffer(dev, 1);
+#endif
 #ifdef CONFIG_PCPU_DEV_REFCNT
 		this_cpu_inc(*dev->pcpu_refcnt);
 #else
diff --git a/include/linux/ns/ns_common_types.h b/include/linux/ns/ns_common_types.h
index b332b019b29c..0014fbc1c626 100644
--- a/include/linux/ns/ns_common_types.h
+++ b/include/linux/ns/ns_common_types.h
@@ -108,11 +108,13 @@ extern const struct proc_ns_operations utsns_operations;
  * @ns_tree: namespace tree nodes and active reference count
  */
 struct ns_common {
+	struct {
+		refcount_t __ns_ref; /* do not use directly */
+	} ____cacheline_aligned_in_smp;
 	u32 ns_type;
 	struct dentry *stashed;
 	const struct proc_ns_operations *ops;
 	unsigned int inum;
-	refcount_t __ns_ref; /* do not use directly */
 	union {
 		struct ns_tree;
 		struct rcu_head ns_rcu;
diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index 191b524e5c0d..8ff9d663096b 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -1647,12 +1647,8 @@ int ntbm_msi_request_threaded_irq(struct ntb_dev *ntb, irq_handler_t handler,
 				  irq_handler_t thread_fn,
 				  const char *name, void *dev_id,
 				  struct ntb_msi_desc *msi_desc);
-void ntbm_msi_free_irq(struct ntb_dev *ntb, unsigned int irq, void *dev_id);
 int ntb_msi_peer_trigger(struct ntb_dev *ntb, int peer,
 			 struct ntb_msi_desc *desc);
-int ntb_msi_peer_addr(struct ntb_dev *ntb, int peer,
-		      struct ntb_msi_desc *desc,
-		      phys_addr_t *msi_addr);
 
 #else /* not CONFIG_NTB_MSI */
 
@@ -1674,21 +1670,11 @@ static inline int ntbm_msi_request_threaded_irq(struct ntb_dev *ntb,
 {
 	return -EOPNOTSUPP;
 }
-static inline void ntbm_msi_free_irq(struct ntb_dev *ntb, unsigned int irq,
-				     void *dev_id) {}
 static inline int ntb_msi_peer_trigger(struct ntb_dev *ntb, int peer,
 				       struct ntb_msi_desc *desc)
 {
 	return -EOPNOTSUPP;
 }
-static inline int ntb_msi_peer_addr(struct ntb_dev *ntb, int peer,
-				    struct ntb_msi_desc *desc,
-				    phys_addr_t *msi_addr)
-{
-	return -EOPNOTSUPP;
-
-}
-
 #endif /* CONFIG_NTB_MSI */
 
 static inline int ntbm_msi_request_irq(struct ntb_dev *ntb,
diff --git a/include/linux/oom.h b/include/linux/oom.h
index 7b02bc1d0a7e..ef3919e4c7f2 100644
--- a/include/linux/oom.h
+++ b/include/linux/oom.h
@@ -48,6 +48,13 @@ struct oom_control {
 	unsigned long totalpages;
 	struct task_struct *chosen;
 	long chosen_points;
+	bool approximate;
+	/*
+	 * Number of precise badness points sums performed by this task
+	 * selection.
+	 */
+	int nr_precise;
+	unsigned long accuracy_under;
 
 	/* Used to print the constraint info. */
 	enum oom_constraint constraint;
@@ -97,7 +104,10 @@ static inline vm_fault_t check_stable_address_space(struct mm_struct *mm)
 }
 
 long oom_badness(struct task_struct *p,
-		unsigned long totalpages);
+		unsigned long totalpages,
+		bool approximate,
+		unsigned int *accuracy_under,
+		unsigned int *accuracy_over);
 
 extern bool out_of_memory(struct oom_control *oc);
 
diff --git a/include/linux/pci-acpi.h b/include/linux/pci-acpi.h
index 078225b514d4..c0c54baadf04 100644
--- a/include/linux/pci-acpi.h
+++ b/include/linux/pci-acpi.h
@@ -12,7 +12,8 @@
 #include <linux/acpi.h>
 
 #ifdef CONFIG_ACPI
-extern acpi_status pci_acpi_add_bus_pm_notifier(struct acpi_device *dev);
+extern acpi_status pci_acpi_add_root_pm_notifier(struct acpi_device *dev,
+						 struct acpi_pci_root *pci_root);
 static inline acpi_status pci_acpi_remove_bus_pm_notifier(struct acpi_device *dev)
 {
 	return acpi_remove_pm_notifier(dev);
diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h
index a9a089566b7c..84b830036fb4 100644
--- a/include/linux/pci_ids.h
+++ b/include/linux/pci_ids.h
@@ -2950,7 +2950,8 @@
 #define PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_ADDR_REV2  0x2db1
 #define PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_RANK_REV2  0x2db2
 #define PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_TC_REV2    0x2db3
-#define PCI_DEVICE_ID_INTEL_HDA_GML	0x3198
+/* In a few of the Intel documents the GML acronym is used for Gemini Lake */
+#define PCI_DEVICE_ID_INTEL_HDA_GLK	0x3198
 #define PCI_DEVICE_ID_INTEL_82855PM_HB	0x3340
 #define PCI_DEVICE_ID_INTEL_IOAT_TBG4	0x3429
 #define PCI_DEVICE_ID_INTEL_IOAT_TBG5	0x342a
diff --git a/include/linux/percpu_counter_tree.h b/include/linux/percpu_counter_tree.h
new file mode 100644
index 000000000000..0daf09e08111
--- /dev/null
+++ b/include/linux/percpu_counter_tree.h
@@ -0,0 +1,242 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR MIT */
+/* SPDX-FileCopyrightText: 2025 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
+
+#ifndef _PERCPU_COUNTER_TREE_H
+#define _PERCPU_COUNTER_TREE_H
+
+#include <linux/preempt.h>
+#include <linux/atomic.h>
+#include <linux/percpu.h>
+
+#ifdef CONFIG_SMP
+
+struct percpu_counter_tree_level_item {
+	atomic_t count;			/*
+					 * Count the number of carry fort this tree item.
+					 * The carry counter is kept at the order of the
+					 * carry accounted for at this tree level.
+					 */
+} ____cacheline_aligned_in_smp;
+
+struct percpu_counter_tree {
+	/* Fast-path fields. */
+	unsigned int __percpu *level0;	/* Pointer to per-CPU split counters (tree level 0). */
+	unsigned int level0_bit_mask;	/* Bit mask to apply to detect carry propagation from tree level 0. */
+	union {
+		unsigned int *i;	/* Approximate sum for single-CPU topology. */
+		atomic_t *a;		/* Approximate sum for SMP topology.  */
+	} approx_sum;
+	int bias;			/* Bias to apply to counter precise and approximate values. */
+
+	/* Slow-path fields. */
+	struct percpu_counter_tree_level_item *items;	/* Array of tree items for levels 1 to N. */
+	unsigned int batch_size;	/*
+					 * The batch size is the increment step at level 0 which
+					 * triggers a carry propagation. The batch size is required
+					 * to be greater than 1, and a power of 2.
+					 */
+	/*
+	 * The tree approximate sum is guaranteed to be within this accuracy range:
+	 * (precise_sum - approx_accuracy_range.under) <= approx_sum <= (precise_sum + approx_accuracy_range.over).
+	 * This accuracy is derived from the hardware topology and the tree batch_size.
+	 * The "under" accuracy is larger than the "over" accuracy because the negative range of a
+	 * two's complement signed integer is one unit larger than the positive range. This delta
+	 * is summed for each tree item, which leads to a significantly larger "under" accuracy range
+	 * compared to the "over" accuracy range.
+	 */
+	struct {
+		unsigned int under;
+		unsigned int over;
+	} approx_accuracy_range;
+};
+
+size_t percpu_counter_tree_items_size(void);
+int percpu_counter_tree_init_many(struct percpu_counter_tree *counters, struct percpu_counter_tree_level_item *items,
+				  unsigned int nr_counters, unsigned int batch_size, gfp_t gfp_flags);
+int percpu_counter_tree_init(struct percpu_counter_tree *counter, struct percpu_counter_tree_level_item *items,
+			     unsigned int batch_size, gfp_t gfp_flags);
+void percpu_counter_tree_destroy_many(struct percpu_counter_tree *counter, unsigned int nr_counters);
+void percpu_counter_tree_destroy(struct percpu_counter_tree *counter);
+void percpu_counter_tree_add(struct percpu_counter_tree *counter, int inc);
+int percpu_counter_tree_precise_sum(struct percpu_counter_tree *counter);
+int percpu_counter_tree_approximate_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b);
+int percpu_counter_tree_approximate_compare_value(struct percpu_counter_tree *counter, int v);
+int percpu_counter_tree_precise_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b);
+int percpu_counter_tree_precise_compare_value(struct percpu_counter_tree *counter, int v);
+void percpu_counter_tree_set(struct percpu_counter_tree *counter, int v);
+void percpu_counter_tree_approximate_accuracy_range(struct percpu_counter_tree *counter,
+						    unsigned int *under, unsigned int *over);
+int percpu_counter_tree_subsystem_init(void);
+
+/**
+ * percpu_counter_tree_approximate_sum() - Return approximate counter sum.
+ * @counter: The counter to sum.
+ *
+ * Querying the approximate sum is fast, but it is only accurate within
+ * the bounds delimited by percpu_counter_tree_approximate_accuracy_range().
+ * This is meant to be used when speed is preferred over accuracy.
+ *
+ * Return: The current approximate counter sum.
+ */
+static inline
+int percpu_counter_tree_approximate_sum(struct percpu_counter_tree *counter)
+{
+	unsigned int v;
+
+	if (!counter->level0_bit_mask)
+		v = READ_ONCE(*counter->approx_sum.i);
+	else
+		v = atomic_read(counter->approx_sum.a);
+	return (int) (v + (unsigned int)READ_ONCE(counter->bias));
+}
+
+#else	/* !CONFIG_SMP */
+
+struct percpu_counter_tree_level_item;
+
+struct percpu_counter_tree {
+	atomic_t count;
+};
+
+static inline
+size_t percpu_counter_tree_items_size(void)
+{
+	return 0;
+}
+
+static inline
+int percpu_counter_tree_init_many(struct percpu_counter_tree *counters, struct percpu_counter_tree_level_item *items,
+				  unsigned int nr_counters, unsigned int batch_size, gfp_t gfp_flags)
+{
+	for (unsigned int i = 0; i < nr_counters; i++)
+		atomic_set(&counters[i].count, 0);
+	return 0;
+}
+
+static inline
+int percpu_counter_tree_init(struct percpu_counter_tree *counter, struct percpu_counter_tree_level_item *items,
+			     unsigned int batch_size, gfp_t gfp_flags)
+{
+	return percpu_counter_tree_init_many(counter, items, 1, batch_size, gfp_flags);
+}
+
+static inline
+void percpu_counter_tree_destroy_many(struct percpu_counter_tree *counter, unsigned int nr_counters)
+{
+}
+
+static inline
+void percpu_counter_tree_destroy(struct percpu_counter_tree *counter)
+{
+}
+
+static inline
+int percpu_counter_tree_precise_sum(struct percpu_counter_tree *counter)
+{
+	return atomic_read(&counter->count);
+}
+
+static inline
+int percpu_counter_tree_precise_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b)
+{
+	int count_a = percpu_counter_tree_precise_sum(a),
+	    count_b = percpu_counter_tree_precise_sum(b);
+
+	if (count_a == count_b)
+		return 0;
+	if (count_a < count_b)
+		return -1;
+	return 1;
+}
+
+static inline
+int percpu_counter_tree_precise_compare_value(struct percpu_counter_tree *counter, int v)
+{
+	int count = percpu_counter_tree_precise_sum(counter);
+
+	if (count == v)
+		return 0;
+	if (count < v)
+		return -1;
+	return 1;
+}
+
+static inline
+int percpu_counter_tree_approximate_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b)
+{
+	return percpu_counter_tree_precise_compare(a, b);
+}
+
+static inline
+int percpu_counter_tree_approximate_compare_value(struct percpu_counter_tree *counter, int v)
+{
+	return percpu_counter_tree_precise_compare_value(counter, v);
+}
+
+static inline
+void percpu_counter_tree_set(struct percpu_counter_tree *counter, int v)
+{
+	atomic_set(&counter->count, v);
+}
+
+static inline
+void percpu_counter_tree_approximate_accuracy_range(struct percpu_counter_tree *counter,
+						    unsigned int *under, unsigned int *over)
+{
+	*under = 0;
+	*over = 0;
+}
+
+static inline
+void percpu_counter_tree_add(struct percpu_counter_tree *counter, int inc)
+{
+	atomic_add(inc, &counter->count);
+}
+
+static inline
+int percpu_counter_tree_approximate_sum(struct percpu_counter_tree *counter)
+{
+	return percpu_counter_tree_precise_sum(counter);
+}
+
+static inline
+int percpu_counter_tree_subsystem_init(void)
+{
+	return 0;
+}
+
+#endif	/* CONFIG_SMP */
+
+/**
+ * percpu_counter_tree_approximate_sum_positive() - Return a positive approximate counter sum.
+ * @counter: The counter to sum.
+ *
+ * Return an approximate counter sum which is guaranteed to be greater
+ * or equal to 0.
+ *
+ * Return: The current positive approximate counter sum.
+ */
+static inline
+int percpu_counter_tree_approximate_sum_positive(struct percpu_counter_tree *counter)
+{
+	int v = percpu_counter_tree_approximate_sum(counter);
+	return v > 0 ? v : 0;
+}
+
+/**
+ * percpu_counter_tree_precise_sum_positive() - Return a positive precise counter sum.
+ * @counter: The counter to sum.
+ *
+ * Return a precise counter sum which is guaranteed to be greater
+ * or equal to 0.
+ *
+ * Return: The current positive precise counter sum.
+ */
+static inline
+int percpu_counter_tree_precise_sum_positive(struct percpu_counter_tree *counter)
+{
+	int v = percpu_counter_tree_precise_sum(counter);
+	return v > 0 ? v : 0;
+}
+
+#endif  /* _PERCPU_COUNTER_TREE_H */
diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
index 652f287c1ef6..2f0dd3a4ace1 100644
--- a/include/linux/pgtable.h
+++ b/include/linux/pgtable.h
@@ -225,16 +225,156 @@ static inline int pmd_dirty(pmd_t pmd)
  * up to date.
  *
  * In the general case, no lock is guaranteed to be held between entry and exit
- * of the lazy mode. So the implementation must assume preemption may be enabled
- * and cpu migration is possible; it must take steps to be robust against this.
- * (In practice, for user PTE updates, the appropriate page table lock(s) are
- * held, but for kernel PTE updates, no lock is held). Nesting is not permitted
- * and the mode cannot be used in interrupt context.
+ * of the lazy mode. (In practice, for user PTE updates, the appropriate page
+ * table lock(s) are held, but for kernel PTE updates, no lock is held).
+ * The implementation must therefore assume preemption may be enabled upon
+ * entry to the mode and cpu migration is possible; it must take steps to be
+ * robust against this. An implementation may handle this by disabling
+ * preemption, as a consequence generic code may not sleep while the lazy MMU
+ * mode is active.
+ *
+ * The mode is disabled in interrupt context and calls to the lazy_mmu API have
+ * no effect.
+ *
+ * The lazy MMU mode is enabled for a given block of code using:
+ *
+ *   lazy_mmu_mode_enable();
+ *   <code>
+ *   lazy_mmu_mode_disable();
+ *
+ * Nesting is permitted: <code> may itself use an enable()/disable() pair.
+ * A nested call to enable() has no functional effect; however disable() causes
+ * any batched architectural state to be flushed regardless of nesting. After a
+ * call to disable(), the caller can therefore rely on all previous page table
+ * modifications to have taken effect, but the lazy MMU mode may still be
+ * enabled.
+ *
+ * In certain cases, it may be desirable to temporarily pause the lazy MMU mode.
+ * This can be done using:
+ *
+ *   lazy_mmu_mode_pause();
+ *   <code>
+ *   lazy_mmu_mode_resume();
+ *
+ * pause() ensures that the mode is exited regardless of the nesting level;
+ * resume() re-enters the mode at the same nesting level. Any call to the
+ * lazy_mmu_mode_* API between those two calls has no effect. In particular,
+ * this means that pause()/resume() pairs may nest.
+ *
+ * is_lazy_mmu_mode_active() can be used to check whether the lazy MMU mode is
+ * currently enabled.
  */
-#ifndef __HAVE_ARCH_ENTER_LAZY_MMU_MODE
-static inline void arch_enter_lazy_mmu_mode(void) {}
-static inline void arch_leave_lazy_mmu_mode(void) {}
-static inline void arch_flush_lazy_mmu_mode(void) {}
+#ifdef CONFIG_ARCH_HAS_LAZY_MMU_MODE
+/**
+ * lazy_mmu_mode_enable() - Enable the lazy MMU mode.
+ *
+ * Enters a new lazy MMU mode section; if the mode was not already enabled,
+ * enables it and calls arch_enter_lazy_mmu_mode().
+ *
+ * Must be paired with a call to lazy_mmu_mode_disable().
+ *
+ * Has no effect if called:
+ * - While paused - see lazy_mmu_mode_pause()
+ * - In interrupt context
+ */
+static inline void lazy_mmu_mode_enable(void)
+{
+	struct lazy_mmu_state *state = &current->lazy_mmu_state;
+
+	if (in_interrupt() || state->pause_count > 0)
+		return;
+
+	VM_WARN_ON_ONCE(state->enable_count == U8_MAX);
+
+	if (state->enable_count++ == 0)
+		arch_enter_lazy_mmu_mode();
+}
+
+/**
+ * lazy_mmu_mode_disable() - Disable the lazy MMU mode.
+ *
+ * Exits the current lazy MMU mode section. If it is the outermost section,
+ * disables the mode and calls arch_leave_lazy_mmu_mode(). Otherwise (nested
+ * section), calls arch_flush_lazy_mmu_mode().
+ *
+ * Must match a call to lazy_mmu_mode_enable().
+ *
+ * Has no effect if called:
+ * - While paused - see lazy_mmu_mode_pause()
+ * - In interrupt context
+ */
+static inline void lazy_mmu_mode_disable(void)
+{
+	struct lazy_mmu_state *state = &current->lazy_mmu_state;
+
+	if (in_interrupt() || state->pause_count > 0)
+		return;
+
+	VM_WARN_ON_ONCE(state->enable_count == 0);
+
+	if (--state->enable_count == 0)
+		arch_leave_lazy_mmu_mode();
+	else /* Exiting a nested section */
+		arch_flush_lazy_mmu_mode();
+
+}
+
+/**
+ * lazy_mmu_mode_pause() - Pause the lazy MMU mode.
+ *
+ * Pauses the lazy MMU mode; if it is currently active, disables it and calls
+ * arch_leave_lazy_mmu_mode().
+ *
+ * Must be paired with a call to lazy_mmu_mode_resume(). Calls to the
+ * lazy_mmu_mode_* API have no effect until the matching resume() call.
+ *
+ * Has no effect if called:
+ * - While paused (inside another pause()/resume() pair)
+ * - In interrupt context
+ */
+static inline void lazy_mmu_mode_pause(void)
+{
+	struct lazy_mmu_state *state = &current->lazy_mmu_state;
+
+	if (in_interrupt())
+		return;
+
+	VM_WARN_ON_ONCE(state->pause_count == U8_MAX);
+
+	if (state->pause_count++ == 0 && state->enable_count > 0)
+		arch_leave_lazy_mmu_mode();
+}
+
+/**
+ * lazy_mmu_mode_resume() - Resume the lazy MMU mode.
+ *
+ * Resumes the lazy MMU mode; if it was active at the point where the matching
+ * call to lazy_mmu_mode_pause() was made, re-enables it and calls
+ * arch_enter_lazy_mmu_mode().
+ *
+ * Must match a call to lazy_mmu_mode_pause().
+ *
+ * Has no effect if called:
+ * - While paused (inside another pause()/resume() pair)
+ * - In interrupt context
+ */
+static inline void lazy_mmu_mode_resume(void)
+{
+	struct lazy_mmu_state *state = &current->lazy_mmu_state;
+
+	if (in_interrupt())
+		return;
+
+	VM_WARN_ON_ONCE(state->pause_count == 0);
+
+	if (--state->pause_count == 0 && state->enable_count > 0)
+		arch_enter_lazy_mmu_mode();
+}
+#else
+static inline void lazy_mmu_mode_enable(void) {}
+static inline void lazy_mmu_mode_disable(void) {}
+static inline void lazy_mmu_mode_pause(void) {}
+static inline void lazy_mmu_mode_resume(void) {}
 #endif
 
 #ifndef pte_batch_hint
diff --git a/include/linux/sched.h b/include/linux/sched.h
index d395f2810fac..2c13d916579c 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -586,15 +586,10 @@ struct sched_entity {
 	u64				sum_exec_runtime;
 	u64				prev_sum_exec_runtime;
 	u64				vruntime;
-	union {
-		/*
-		 * When !@on_rq this field is vlag.
-		 * When cfs_rq->curr == se (which implies @on_rq)
-		 * this field is vprot. See protect_slice().
-		 */
-		s64                     vlag;
-		u64                     vprot;
-	};
+	/* Approximated virtual lag: */
+	s64				vlag;
+	/* 'Protected' deadline, to give out minimum quantums: */
+	u64				vprot;
 	u64				slice;
 
 	u64				nr_migrations;
@@ -1419,6 +1414,10 @@ struct task_struct {
 
 	struct page_frag		task_frag;
 
+#ifdef CONFIG_ARCH_HAS_LAZY_MMU_MODE
+	struct lazy_mmu_state		lazy_mmu_state;
+#endif
+
 #ifdef CONFIG_TASK_DELAY_ACCT
 	struct task_delay_info		*delays;
 #endif
@@ -1702,6 +1701,47 @@ static inline char task_state_to_char(struct task_struct *tsk)
 	return task_index_to_char(task_state_index(tsk));
 }
 
+#ifdef CONFIG_ARCH_HAS_LAZY_MMU_MODE
+/**
+ * __task_lazy_mmu_mode_active() - Test the lazy MMU mode state for a task.
+ * @tsk: The task to check.
+ *
+ * Test whether @tsk has its lazy MMU mode state set to active (i.e. enabled
+ * and not paused).
+ *
+ * This function only considers the state saved in task_struct; to test whether
+ * current actually is in lazy MMU mode, is_lazy_mmu_mode_active() should be
+ * used instead.
+ *
+ * This function is intended for architectures that implement the lazy MMU
+ * mode; it must not be called from generic code.
+ */
+static inline bool __task_lazy_mmu_mode_active(struct task_struct *tsk)
+{
+	struct lazy_mmu_state *state = &tsk->lazy_mmu_state;
+
+	return state->enable_count > 0 && state->pause_count == 0;
+}
+
+/**
+ * is_lazy_mmu_mode_active() - Test whether we are currently in lazy MMU mode.
+ *
+ * Test whether the current context is in lazy MMU mode. This is true if both:
+ * 1. We are not in interrupt context
+ * 2. Lazy MMU mode is active for the current task
+ *
+ * This function is intended for architectures that implement the lazy MMU
+ * mode; it must not be called from generic code.
+ */
+static inline bool is_lazy_mmu_mode_active(void)
+{
+	if (in_interrupt())
+		return false;
+
+	return __task_lazy_mmu_mode_active(current);
+}
+#endif
+
 extern struct pid *cad_pid;
 
 /*
diff --git a/include/linux/screen_info.h b/include/linux/screen_info.h
index 1690706206e8..c022403c599a 100644
--- a/include/linux/screen_info.h
+++ b/include/linux/screen_info.h
@@ -151,6 +151,4 @@ static inline struct pci_dev *screen_info_pci_dev(const struct screen_info *si)
 }
 #endif
 
-extern struct screen_info screen_info;
-
 #endif /* _SCREEN_INFO_H */
diff --git a/include/linux/serdev.h b/include/linux/serdev.h
index 34562eb99931..5654c58eb73c 100644
--- a/include/linux/serdev.h
+++ b/include/linux/serdev.h
@@ -65,6 +65,7 @@ struct serdev_device_driver {
 	struct device_driver driver;
 	int	(*probe)(struct serdev_device *);
 	void	(*remove)(struct serdev_device *);
+	void	(*shutdown)(struct serdev_device *);
 };
 
 static inline struct serdev_device_driver *to_serdev_device_driver(struct device_driver *d)
diff --git a/include/linux/shdma-base.h b/include/linux/shdma-base.h
index 03ba4dab2ef7..b6827c06d332 100644
--- a/include/linux/shdma-base.h
+++ b/include/linux/shdma-base.h
@@ -19,11 +19,11 @@
 #include <linux/types.h>
 
 /**
- * shdma_pm_state - DMA channel PM state
- * SHDMA_PM_ESTABLISHED:	either idle or during data transfer
- * SHDMA_PM_BUSY:		during the transfer preparation, when we have to
+ * enum shdma_pm_state - DMA channel PM state
+ * @SHDMA_PM_ESTABLISHED:	either idle or during data transfer
+ * @SHDMA_PM_BUSY:		during the transfer preparation, when we have to
  *				drop the lock temporarily
- * SHDMA_PM_PENDING:	transfers pending
+ * @SHDMA_PM_PENDING:	transfers pending
  */
 enum shdma_pm_state {
 	SHDMA_PM_ESTABLISHED,
@@ -74,18 +74,18 @@ struct shdma_chan {
 
 /**
  * struct shdma_ops - simple DMA driver operations
- * desc_completed:	return true, if this is the descriptor, that just has
+ * @desc_completed:	return true, if this is the descriptor, that just has
  *			completed (atomic)
- * halt_channel:	stop DMA channel operation (atomic)
- * channel_busy:	return true, if the channel is busy (atomic)
- * slave_addr:		return slave DMA address
- * desc_setup:		set up the hardware specific descriptor portion (atomic)
- * set_slave:		bind channel to a slave
- * setup_xfer:		configure channel hardware for operation (atomic)
- * start_xfer:		start the DMA transfer (atomic)
- * embedded_desc:	return Nth struct shdma_desc pointer from the
+ * @halt_channel:	stop DMA channel operation (atomic)
+ * @channel_busy:	return true, if the channel is busy (atomic)
+ * @slave_addr:		return slave DMA address
+ * @desc_setup:		set up the hardware specific descriptor portion (atomic)
+ * @set_slave:		bind channel to a slave
+ * @setup_xfer:		configure channel hardware for operation (atomic)
+ * @start_xfer:		start the DMA transfer (atomic)
+ * @embedded_desc:	return Nth struct shdma_desc pointer from the
  *			descriptor array
- * chan_irq:		process channel IRQ, return true if a transfer has
+ * @chan_irq:		process channel IRQ, return true if a transfer has
  *			completed (atomic)
  */
 struct shdma_ops {
diff --git a/include/linux/soc/qcom/llcc-qcom.h b/include/linux/soc/qcom/llcc-qcom.h
index 0287f9182c4d..8243ab3a12a8 100644
--- a/include/linux/soc/qcom/llcc-qcom.h
+++ b/include/linux/soc/qcom/llcc-qcom.h
@@ -74,13 +74,17 @@
 #define LLCC_CAMSRTIP	 73
 #define LLCC_CAMRTRF	 74
 #define LLCC_CAMSRTRF	 75
+#define LLCC_OOBM_NS	 81
+#define LLCC_OOBM_S	 82
 #define LLCC_VIDEO_APV	 83
 #define LLCC_COMPUTE1	 87
 #define LLCC_CPUSS_OPP	 88
 #define LLCC_CPUSSMPAM	 89
+#define LLCC_VIDSC_VSP1	 91
 #define LLCC_CAM_IPE_STROV	 92
 #define LLCC_CAM_OFE_STROV	 93
 #define LLCC_CPUSS_HEU	 94
+#define LLCC_PCIE_TCU	 97
 #define LLCC_MDM_PNG_FIXED	 100
 
 /**
diff --git a/include/linux/sysfb.h b/include/linux/sysfb.h
index b449665c686a..5226efde9ad4 100644
--- a/include/linux/sysfb.h
+++ b/include/linux/sysfb.h
@@ -8,9 +8,11 @@
  */
 
 #include <linux/err.h>
+#include <linux/platform_data/simplefb.h>
+#include <linux/screen_info.h>
 #include <linux/types.h>
 
-#include <linux/platform_data/simplefb.h>
+#include <video/edid.h>
 
 struct device;
 struct platform_device;
@@ -60,6 +62,16 @@ struct efifb_dmi_info {
 	int flags;
 };
 
+struct sysfb_display_info {
+	struct screen_info screen;
+
+#if defined(CONFIG_FIRMWARE_EDID)
+	struct edid_info edid;
+#endif
+};
+
+extern struct sysfb_display_info sysfb_primary_display;
+
 #ifdef CONFIG_SYSFB
 
 void sysfb_disable(struct device *dev);
@@ -82,16 +94,17 @@ static inline bool sysfb_handles_screen_info(void)
 #ifdef CONFIG_EFI
 
 extern struct efifb_dmi_info efifb_dmi_list[];
-void sysfb_apply_efi_quirks(void);
-void sysfb_set_efifb_fwnode(struct platform_device *pd);
+void sysfb_apply_efi_quirks(struct screen_info *si);
+void sysfb_set_efifb_fwnode(const struct screen_info *si, struct platform_device *pd);
 
 #else /* CONFIG_EFI */
 
-static inline void sysfb_apply_efi_quirks(void)
+static inline void sysfb_apply_efi_quirks(struct screen_info *si)
 {
 }
 
-static inline void sysfb_set_efifb_fwnode(struct platform_device *pd)
+static inline void sysfb_set_efifb_fwnode(const struct screen_info *si,
+					  struct platform_device *pd)
 {
 }
 
diff --git a/include/linux/tee_drv.h b/include/linux/tee_drv.h
index 88a6f9697c89..e561a26f537a 100644
--- a/include/linux/tee_drv.h
+++ b/include/linux/tee_drv.h
@@ -315,6 +315,9 @@ struct tee_client_device {
  * @driver:		driver structure
  */
 struct tee_client_driver {
+	int (*probe)(struct tee_client_device *);
+	void (*remove)(struct tee_client_device *);
+	void (*shutdown)(struct tee_client_device *);
 	const struct tee_client_device_id *id_table;
 	struct device_driver driver;
 };
@@ -322,4 +325,13 @@ struct tee_client_driver {
 #define to_tee_client_driver(d) \
 		container_of_const(d, struct tee_client_driver, driver)
 
+#define tee_client_driver_register(drv) \
+        __tee_client_driver_register(drv, THIS_MODULE)
+int __tee_client_driver_register(struct tee_client_driver *, struct module *);
+void tee_client_driver_unregister(struct tee_client_driver *);
+
+#define module_tee_client_driver(__tee_client_driver) \
+	module_driver(__tee_client_driver, tee_client_driver_register, \
+		      tee_client_driver_unregister)
+
 #endif /*__TEE_DRV_H*/
diff --git a/include/linux/timecounter.h b/include/linux/timecounter.h
index dce03a5cafb7..7de6b350e559 100644
--- a/include/linux/timecounter.h
+++ b/include/linux/timecounter.h
@@ -115,6 +115,15 @@ extern void timecounter_init(struct timecounter *tc,
  */
 extern u64 timecounter_read(struct timecounter *tc);
 
+/*
+ * This is like cyclecounter_cyc2ns(), but it is used for computing a
+ * time previous to the time stored in the cycle counter.
+ */
+static inline u64 cc_cyc2ns_backwards(const struct cyclecounter *cc, u64 cycles, u64 frac)
+{
+	return ((cycles * cc->mult) - frac) >> cc->shift;
+}
+
 /**
  * timecounter_cyc2time - convert a cycle counter to same
  *                        time base as values returned by
@@ -131,7 +140,25 @@ extern u64 timecounter_read(struct timecounter *tc);
  *
  * Returns: cycle counter converted to nanoseconds since the initial time stamp
  */
-extern u64 timecounter_cyc2time(const struct timecounter *tc,
-				u64 cycle_tstamp);
+static inline u64 timecounter_cyc2time(const struct timecounter *tc, u64 cycle_tstamp)
+{
+	const struct cyclecounter *cc = tc->cc;
+	u64 delta = (cycle_tstamp - tc->cycle_last) & cc->mask;
+	u64 nsec = tc->nsec, frac = tc->frac;
+
+	/*
+	 * Instead of always treating cycle_tstamp as more recent than
+	 * tc->cycle_last, detect when it is too far in the future and
+	 * treat it as old time stamp instead.
+	 */
+	if (unlikely(delta > cc->mask / 2)) {
+		delta = (tc->cycle_last - cycle_tstamp) & cc->mask;
+		nsec -= cc_cyc2ns_backwards(cc, delta, frac);
+	} else {
+		nsec += cyclecounter_cyc2ns(cc, delta, tc->mask, &frac);
+	}
+
+	return nsec;
+}
 
 #endif
diff --git a/include/linux/transport_class.h b/include/linux/transport_class.h
index 2efc271a96fa..a009d66db15a 100644
--- a/include/linux/transport_class.h
+++ b/include/linux/transport_class.h
@@ -56,6 +56,7 @@ struct anon_transport_class cls = {				\
 struct transport_container {
 	struct attribute_container ac;
 	const struct attribute_group *statistics;
+	const struct attribute_group *encryption;
 };
 
 #define attribute_container_to_transport_container(x) \
diff --git a/include/linux/types.h b/include/linux/types.h
index d4437e9c452c..1760e1feeab9 100644
--- a/include/linux/types.h
+++ b/include/linux/types.h
@@ -180,7 +180,7 @@ typedef phys_addr_t resource_size_t;
 typedef unsigned long irq_hw_number_t;
 
 typedef struct {
-	int counter;
+	int __aligned(sizeof(int)) counter;
 } atomic_t;
 
 #define ATOMIC_INIT(i) { (i) }
diff --git a/include/linux/usb.h b/include/linux/usb.h
index e85105939af8..fbfcc70b07fb 100644
--- a/include/linux/usb.h
+++ b/include/linux/usb.h
@@ -1295,8 +1295,7 @@ struct usb_driver {
  *	resume and suspend functions will be called in addition to the driver's
  *	own, so this part of the setup does not need to be replicated.
  *
- * USB drivers must provide all the fields listed above except driver,
- * match, and id_table.
+ * USB device drivers must provide a name, other driver fields are optional.
  */
 struct usb_device_driver {
 	const char *name;
diff --git a/include/linux/usb/typec.h b/include/linux/usb/typec.h
index 309251572e2e..c6fd46902fce 100644
--- a/include/linux/usb/typec.h
+++ b/include/linux/usb/typec.h
@@ -20,12 +20,15 @@ struct typec_port;
 struct typec_altmode_ops;
 struct typec_cable_ops;
 
+struct bus_type;
 struct fwnode_handle;
 struct device;
 
 struct usb_power_delivery;
 struct usb_power_delivery_desc;
 
+extern const struct bus_type typec_bus;
+
 enum typec_port_type {
 	TYPEC_PORT_SRC,
 	TYPEC_PORT_SNK,
diff --git a/include/linux/usb/typec_altmode.h b/include/linux/usb/typec_altmode.h
index f7db3bd4c90e..9197a4637a93 100644
--- a/include/linux/usb/typec_altmode.h
+++ b/include/linux/usb/typec_altmode.h
@@ -9,6 +9,14 @@
 
 #define MODE_DISCOVERY_MAX	6
 
+extern const struct device_type typec_port_altmode_dev_type;
+extern const struct device_type typec_plug_altmode_dev_type;
+extern const struct device_type typec_partner_altmode_dev_type;
+
+#define is_typec_port_altmode(dev) ((dev)->type == &typec_port_altmode_dev_type)
+#define is_typec_plug_altmode(dev) ((dev)->type == &typec_plug_altmode_dev_type)
+#define is_typec_partner_altmode(dev) ((dev)->type == &typec_partner_altmode_dev_type)
+
 struct typec_altmode_ops;
 
 /**
diff --git a/include/linux/virtio.h b/include/linux/virtio.h
index 132a474e5914..3626eb694728 100644
--- a/include/linux/virtio.h
+++ b/include/linux/virtio.h
@@ -13,6 +13,8 @@
 #include <linux/completion.h>
 #include <linux/virtio_features.h>
 
+struct module;
+
 /**
  * struct virtqueue - a queue to register buffers for sending or receiving.
  * @list: the chain of virtqueues for this device
diff --git a/include/linux/virtio_features.h b/include/linux/virtio_features.h
index ea2ad8717882..ce59ea91f474 100644
--- a/include/linux/virtio_features.h
+++ b/include/linux/virtio_features.h
@@ -3,6 +3,8 @@
 #define _LINUX_VIRTIO_FEATURES_H
 
 #include <linux/bits.h>
+#include <linux/bug.h>
+#include <linux/string.h>
 
 #define VIRTIO_FEATURES_U64S	2
 #define VIRTIO_FEATURES_BITS	(VIRTIO_FEATURES_U64S * 64)
diff --git a/include/linux/xarray.h b/include/linux/xarray.h
index be850174e802..20e577e99920 100644
--- a/include/linux/xarray.h
+++ b/include/linux/xarray.h
@@ -1185,8 +1185,9 @@ struct xa_node {
 
 void xa_dump(const struct xarray *);
 void xa_dump_node(const struct xa_node *);
+void xa_dump_index(unsigned long index, unsigned int shift);
 
-#ifdef XA_DEBUG
+#ifdef CONFIG_DEBUG_XARRAY
 #define XA_BUG_ON(xa, x) do {					\
 		if (x) {					\
 			xa_dump(xa);				\
@@ -1413,16 +1414,44 @@ struct xa_state {
 			order - (order % XA_CHUNK_SHIFT),	\
 			(1U << (order % XA_CHUNK_SHIFT)) - 1)
 
+/**
+ * xas_invalid() - Is the xas in a retry or error state?
+ * @xas: XArray operation state.
+ *
+ * Return: %true if the xas cannot be used for operations.
+ */
+static inline bool xas_invalid(const struct xa_state *xas)
+{
+	return (unsigned long)xas->xa_node & 3;
+}
+
+/**
+ * xas_valid() - Is the xas a valid cursor into the array?
+ * @xas: XArray operation state.
+ *
+ * Return: %true if the xas can be used for operations.
+ */
+static inline bool xas_valid(const struct xa_state *xas)
+{
+	return !xas_invalid(xas);
+}
+
+static inline struct xa_state *XAS_INVALID(struct xa_state *xas)
+{
+	XA_NODE_BUG_ON(xas->xa_node, xas_valid(xas));
+	return xas;
+}
+
 #define xas_marked(xas, mark)	xa_marked((xas)->xa, (mark))
-#define xas_trylock(xas)	xa_trylock((xas)->xa)
-#define xas_lock(xas)		xa_lock((xas)->xa)
+#define xas_trylock(xas)	xa_trylock(XAS_INVALID(xas)->xa)
+#define xas_lock(xas)		xa_lock(XAS_INVALID(xas)->xa)
 #define xas_unlock(xas)		xa_unlock((xas)->xa)
-#define xas_lock_bh(xas)	xa_lock_bh((xas)->xa)
+#define xas_lock_bh(xas)	xa_lock_bh(XAS_INVALID(xas)->xa)
 #define xas_unlock_bh(xas)	xa_unlock_bh((xas)->xa)
-#define xas_lock_irq(xas)	xa_lock_irq((xas)->xa)
+#define xas_lock_irq(xas)	xa_lock_irq(XAS_INVALID(xas)->xa)
 #define xas_unlock_irq(xas)	xa_unlock_irq((xas)->xa)
 #define xas_lock_irqsave(xas, flags) \
-				xa_lock_irqsave((xas)->xa, flags)
+				xa_lock_irqsave(XAS_INVALID(xas)->xa, flags)
 #define xas_unlock_irqrestore(xas, flags) \
 				xa_unlock_irqrestore((xas)->xa, flags)
 
@@ -1451,28 +1480,6 @@ static inline void xas_set_err(struct xa_state *xas, long err)
 	xas->xa_node = XA_ERROR(err);
 }
 
-/**
- * xas_invalid() - Is the xas in a retry or error state?
- * @xas: XArray operation state.
- *
- * Return: %true if the xas cannot be used for operations.
- */
-static inline bool xas_invalid(const struct xa_state *xas)
-{
-	return (unsigned long)xas->xa_node & 3;
-}
-
-/**
- * xas_valid() - Is the xas a valid cursor into the array?
- * @xas: XArray operation state.
- *
- * Return: %true if the xas can be used for operations.
- */
-static inline bool xas_valid(const struct xa_state *xas)
-{
-	return !xas_invalid(xas);
-}
-
 /**
  * xas_is_node() - Does the xas point to a node?
  * @xas: XArray operation state.
diff --git a/include/media/dvb_vb2.h b/include/media/dvb_vb2.h
index 8cb88452cd6c..8932396d2c99 100644
--- a/include/media/dvb_vb2.h
+++ b/include/media/dvb_vb2.h
@@ -72,8 +72,6 @@ struct dvb_buffer {
 /**
  * struct dvb_vb2_ctx - control struct for VB2 handler
  * @vb_q:	pointer to &struct vb2_queue with videobuf2 queue.
- * @mutex:	mutex to serialize vb2 operations. Used by
- *		vb2 core %wait_prepare and %wait_finish operations.
  * @slock:	spin lock used to protect buffer filling at dvb_vb2.c.
  * @dvb_q:	List of buffers that are not filled yet.
  * @buf:	Pointer to the buffer that are currently being filled.
@@ -96,7 +94,6 @@ struct dvb_buffer {
  */
 struct dvb_vb2_ctx {
 	struct vb2_queue	vb_q;
-	struct mutex		mutex;
 	spinlock_t		slock;
 	struct list_head	dvb_q;
 	struct dvb_buffer	*buf;
@@ -114,8 +111,8 @@ struct dvb_vb2_ctx {
 };
 
 #ifndef CONFIG_DVB_MMAP
-static inline int dvb_vb2_init(struct dvb_vb2_ctx *ctx,
-			       const char *name, int non_blocking)
+static inline int dvb_vb2_init(struct dvb_vb2_ctx *ctx, const char *name,
+			       struct mutex *mutex, int non_blocking)
 {
 	return 0;
 };
@@ -124,7 +121,7 @@ static inline int dvb_vb2_release(struct dvb_vb2_ctx *ctx)
 	return 0;
 };
 #define dvb_vb2_is_streaming(ctx) (0)
-#define dvb_vb2_fill_buffer(ctx, file, wait, flags) (0)
+#define dvb_vb2_fill_buffer(ctx, file, wait, flags, flush) (0)
 
 static inline __poll_t dvb_vb2_poll(struct dvb_vb2_ctx *ctx,
 				    struct file *file,
@@ -138,10 +135,12 @@ static inline __poll_t dvb_vb2_poll(struct dvb_vb2_ctx *ctx,
  *
  * @ctx:	control struct for VB2 handler
  * @name:	name for the VB2 handler
+ * @mutex:	pointer to the mutex that serializes vb2 ioctls
  * @non_blocking:
  *		if not zero, it means that the device is at non-blocking mode
  */
-int dvb_vb2_init(struct dvb_vb2_ctx *ctx, const char *name, int non_blocking);
+int dvb_vb2_init(struct dvb_vb2_ctx *ctx, const char *name,
+		 struct mutex *mutex, int non_blocking);
 
 /**
  * dvb_vb2_release - Releases the VB2 handler allocated resources and
@@ -166,10 +165,12 @@ int dvb_vb2_is_streaming(struct dvb_vb2_ctx *ctx);
  * @buffer_flags:
  *		pointer to buffer flags as defined by &enum dmx_buffer_flags.
  *		can be NULL.
+ * @flush:	flush the buffer, even if it isn't full.
  */
 int dvb_vb2_fill_buffer(struct dvb_vb2_ctx *ctx,
 			const unsigned char *src, int len,
-			enum dmx_buffer_flags *buffer_flags);
+			enum dmx_buffer_flags *buffer_flags,
+			bool flush);
 
 /**
  * dvb_vb2_poll - Wrapper to vb2_core_streamon() for Digital TV
diff --git a/include/media/videobuf2-core.h b/include/media/videobuf2-core.h
index 9b02aeba4108..4424d481d7f7 100644
--- a/include/media/videobuf2-core.h
+++ b/include/media/videobuf2-core.h
@@ -351,13 +351,6 @@ struct vb2_buffer {
  *			\*num_buffers are being allocated additionally to
  *			the buffers already allocated. If either \*num_planes
  *			or the requested sizes are invalid callback must return %-EINVAL.
- * @wait_prepare:	release any locks taken while calling vb2 functions;
- *			it is called before an ioctl needs to wait for a new
- *			buffer to arrive; required to avoid a deadlock in
- *			blocking access type.
- * @wait_finish:	reacquire all locks released in the previous callback;
- *			required to continue operation after sleeping while
- *			waiting for a new buffer to arrive.
  * @buf_out_validate:	called when the output buffer is prepared or queued
  *			to a request; drivers can use this to validate
  *			userspace-provided information; this is required only
@@ -436,9 +429,6 @@ struct vb2_ops {
 			   unsigned int *num_buffers, unsigned int *num_planes,
 			   unsigned int sizes[], struct device *alloc_devs[]);
 
-	void (*wait_prepare)(struct vb2_queue *q);
-	void (*wait_finish)(struct vb2_queue *q);
-
 	int (*buf_out_validate)(struct vb2_buffer *vb);
 	int (*buf_init)(struct vb2_buffer *vb);
 	int (*buf_prepare)(struct vb2_buffer *vb);
@@ -521,10 +511,10 @@ struct vb2_buf_ops {
  * @non_coherent_mem: when set queue will attempt to allocate buffers using
  *		non-coherent memory.
  * @lock:	pointer to a mutex that protects the &struct vb2_queue. The
- *		driver can set this to a mutex to let the v4l2 core serialize
- *		the queuing ioctls. If the driver wants to handle locking
- *		itself, then this should be set to NULL. This lock is not used
- *		by the videobuf2 core API.
+ *		driver must set this to a mutex to let the v4l2 core serialize
+ *		the queuing ioctls. This lock is used when waiting for a new
+ *		buffer to arrive: the lock is released, we wait for the new
+ *		buffer, and then retaken.
  * @owner:	The filehandle that 'owns' the buffers, i.e. the filehandle
  *		that called reqbufs, create_buffers or started fileio.
  *		This field is not used by the videobuf2 core API, but it allows
@@ -680,8 +670,6 @@ struct vb2_queue {
 	 * called. Used to check for unbalanced ops.
 	 */
 	u32				cnt_queue_setup;
-	u32				cnt_wait_prepare;
-	u32				cnt_wait_finish;
 	u32				cnt_prepare_streaming;
 	u32				cnt_start_streaming;
 	u32				cnt_stop_streaming;
@@ -766,8 +754,7 @@ void vb2_discard_done(struct vb2_queue *q);
  * @q:		pointer to &struct vb2_queue with videobuf2 queue.
  *
  * This function will wait until all buffers that have been given to the driver
- * by &vb2_ops->buf_queue are given back to vb2 with vb2_buffer_done(). It
- * doesn't call &vb2_ops->wait_prepare/&vb2_ops->wait_finish pair.
+ * by &vb2_ops->buf_queue are given back to vb2 with vb2_buffer_done().
  * It is intended to be called with all locks taken, for example from
  * &vb2_ops->stop_streaming callback.
  */
diff --git a/include/media/videobuf2-v4l2.h b/include/media/videobuf2-v4l2.h
index 77ce8238ab30..71d2864fb235 100644
--- a/include/media/videobuf2-v4l2.h
+++ b/include/media/videobuf2-v4l2.h
@@ -367,24 +367,6 @@ unsigned long vb2_fop_get_unmapped_area(struct file *file, unsigned long addr,
  */
 void vb2_video_unregister_device(struct video_device *vdev);
 
-/**
- * vb2_ops_wait_prepare - helper function to lock a struct &vb2_queue
- *
- * @vq: pointer to &struct vb2_queue
- *
- * ..note:: only use if vq->lock is non-NULL.
- */
-void vb2_ops_wait_prepare(struct vb2_queue *vq);
-
-/**
- * vb2_ops_wait_finish - helper function to unlock a struct &vb2_queue
- *
- * @vq: pointer to &struct vb2_queue
- *
- * ..note:: only use if vq->lock is non-NULL.
- */
-void vb2_ops_wait_finish(struct vb2_queue *vq);
-
 struct media_request;
 int vb2_request_validate(struct media_request *req);
 void vb2_request_queue(struct media_request *req);
diff --git a/include/net/bluetooth/bluetooth.h b/include/net/bluetooth/bluetooth.h
index d46ed9011ee5..89a60919050b 100644
--- a/include/net/bluetooth/bluetooth.h
+++ b/include/net/bluetooth/bluetooth.h
@@ -130,21 +130,30 @@ struct bt_voice {
 #define BT_RCVMTU		13
 #define BT_PHY			14
 
-#define BT_PHY_BR_1M_1SLOT	0x00000001
-#define BT_PHY_BR_1M_3SLOT	0x00000002
-#define BT_PHY_BR_1M_5SLOT	0x00000004
-#define BT_PHY_EDR_2M_1SLOT	0x00000008
-#define BT_PHY_EDR_2M_3SLOT	0x00000010
-#define BT_PHY_EDR_2M_5SLOT	0x00000020
-#define BT_PHY_EDR_3M_1SLOT	0x00000040
-#define BT_PHY_EDR_3M_3SLOT	0x00000080
-#define BT_PHY_EDR_3M_5SLOT	0x00000100
-#define BT_PHY_LE_1M_TX		0x00000200
-#define BT_PHY_LE_1M_RX		0x00000400
-#define BT_PHY_LE_2M_TX		0x00000800
-#define BT_PHY_LE_2M_RX		0x00001000
-#define BT_PHY_LE_CODED_TX	0x00002000
-#define BT_PHY_LE_CODED_RX	0x00004000
+#define BT_PHY_BR_1M_1SLOT	BIT(0)
+#define BT_PHY_BR_1M_3SLOT	BIT(1)
+#define BT_PHY_BR_1M_5SLOT	BIT(2)
+#define BT_PHY_EDR_2M_1SLOT	BIT(3)
+#define BT_PHY_EDR_2M_3SLOT	BIT(4)
+#define BT_PHY_EDR_2M_5SLOT	BIT(5)
+#define BT_PHY_EDR_3M_1SLOT	BIT(6)
+#define BT_PHY_EDR_3M_3SLOT	BIT(7)
+#define BT_PHY_EDR_3M_5SLOT	BIT(8)
+#define BT_PHY_LE_1M_TX		BIT(9)
+#define BT_PHY_LE_1M_RX		BIT(10)
+#define BT_PHY_LE_2M_TX		BIT(11)
+#define BT_PHY_LE_2M_RX		BIT(12)
+#define BT_PHY_LE_CODED_TX	BIT(13)
+#define BT_PHY_LE_CODED_RX	BIT(14)
+
+#define BT_PHY_BREDR_MASK	(BT_PHY_BR_1M_1SLOT | BT_PHY_BR_1M_3SLOT | \
+				 BT_PHY_BR_1M_5SLOT | BT_PHY_EDR_2M_1SLOT | \
+				 BT_PHY_EDR_2M_3SLOT | BT_PHY_EDR_2M_5SLOT | \
+				 BT_PHY_EDR_3M_1SLOT | BT_PHY_EDR_3M_3SLOT | \
+				 BT_PHY_EDR_3M_5SLOT)
+#define BT_PHY_LE_MASK		(BT_PHY_LE_1M_TX | BT_PHY_LE_1M_RX | \
+				 BT_PHY_LE_2M_TX | BT_PHY_LE_2M_RX | \
+				 BT_PHY_LE_CODED_TX | BT_PHY_LE_CODED_RX)
 
 #define BT_MODE			15
 
diff --git a/include/net/bluetooth/hci.h b/include/net/bluetooth/hci.h
index a27cd3626b87..db76c2d1eeaa 100644
--- a/include/net/bluetooth/hci.h
+++ b/include/net/bluetooth/hci.h
@@ -654,6 +654,8 @@ enum {
 #define HCI_LE_ISO_BROADCASTER		0x40
 #define HCI_LE_ISO_SYNC_RECEIVER	0x80
 #define HCI_LE_LL_EXT_FEATURE		0x80
+#define HCI_LE_CS			0x40
+#define HCI_LE_CS_HOST			0x80
 
 /* Connection modes */
 #define HCI_CM_ACTIVE	0x0000
@@ -1883,6 +1885,15 @@ struct hci_cp_le_set_default_phy {
 #define HCI_LE_SET_PHY_2M		0x02
 #define HCI_LE_SET_PHY_CODED		0x04
 
+#define HCI_OP_LE_SET_PHY		0x2032
+struct hci_cp_le_set_phy {
+	__le16  handle;
+	__u8    all_phys;
+	__u8    tx_phys;
+	__u8    rx_phys;
+	__le16   phy_opts;
+} __packed;
+
 #define HCI_OP_LE_SET_EXT_SCAN_PARAMS   0x2041
 struct hci_cp_le_set_ext_scan_params {
 	__u8    own_addr_type;
@@ -2269,6 +2280,204 @@ struct hci_cp_le_read_all_remote_features {
 	__u8	 pages;
 } __packed;
 
+/* Channel Sounding Commands */
+#define HCI_OP_LE_CS_RD_LOCAL_SUPP_CAP	0x2089
+struct hci_rp_le_cs_rd_local_supp_cap {
+	__u8	status;
+	__u8	num_config_supported;
+	__le16	max_consecutive_procedures_supported;
+	__u8	num_antennas_supported;
+	__u8	max_antenna_paths_supported;
+	__u8	roles_supported;
+	__u8	modes_supported;
+	__u8	rtt_capability;
+	__u8	rtt_aa_only_n;
+	__u8	rtt_sounding_n;
+	__u8	rtt_random_payload_n;
+	__le16	nadm_sounding_capability;
+	__le16	nadm_random_capability;
+	__u8	cs_sync_phys_supported;
+	__le16	subfeatures_supported;
+	__le16	t_ip1_times_supported;
+	__le16	t_ip2_times_supported;
+	__le16	t_fcs_times_supported;
+	__le16	t_pm_times_supported;
+	__u8	t_sw_time_supported;
+	__u8	tx_snr_capability;
+} __packed;
+
+#define HCI_OP_LE_CS_RD_RMT_SUPP_CAP		0x208A
+struct hci_cp_le_cs_rd_local_supp_cap {
+	__le16	handle;
+} __packed;
+
+#define HCI_OP_LE_CS_WR_CACHED_RMT_SUPP_CAP	0x208B
+struct hci_cp_le_cs_wr_cached_rmt_supp_cap {
+	__le16	handle;
+	__u8	num_config_supported;
+	__le16	max_consecutive_procedures_supported;
+	__u8	num_antennas_supported;
+	__u8	max_antenna_paths_supported;
+	__u8	roles_supported;
+	__u8	modes_supported;
+	__u8	rtt_capability;
+	__u8	rtt_aa_only_n;
+	__u8	rtt_sounding_n;
+	__u8	rtt_random_payload_n;
+	__le16	nadm_sounding_capability;
+	__le16	nadm_random_capability;
+	__u8	cs_sync_phys_supported;
+	__le16	subfeatures_supported;
+	__le16	t_ip1_times_supported;
+	__le16	t_ip2_times_supported;
+	__le16	t_fcs_times_supported;
+	__le16	t_pm_times_supported;
+	__u8	t_sw_time_supported;
+	__u8	tx_snr_capability;
+} __packed;
+
+struct hci_rp_le_cs_wr_cached_rmt_supp_cap {
+	__u8	status;
+	__le16	handle;
+} __packed;
+
+#define HCI_OP_LE_CS_SEC_ENABLE			0x208C
+struct hci_cp_le_cs_sec_enable {
+	__le16	handle;
+} __packed;
+
+#define HCI_OP_LE_CS_SET_DEFAULT_SETTINGS	0x208D
+struct hci_cp_le_cs_set_default_settings {
+	__le16	handle;
+	__u8	role_enable;
+	__u8	cs_sync_ant_sel;
+	__s8	max_tx_power;
+} __packed;
+
+struct hci_rp_le_cs_set_default_settings {
+	__u8	status;
+	__le16	handle;
+} __packed;
+
+#define HCI_OP_LE_CS_RD_RMT_FAE_TABLE		0x208E
+struct hci_cp_le_cs_rd_rmt_fae_table {
+	__le16	handle;
+} __packed;
+
+#define HCI_OP_LE_CS_WR_CACHED_RMT_FAE_TABLE	0x208F
+struct hci_cp_le_cs_wr_rmt_cached_fae_table {
+	__le16	handle;
+	__u8	remote_fae_table[72];
+} __packed;
+
+struct hci_rp_le_cs_wr_rmt_cached_fae_table {
+	__u8	status;
+	__le16	handle;
+} __packed;
+
+#define HCI_OP_LE_CS_CREATE_CONFIG		0x2090
+struct hci_cp_le_cs_create_config {
+	__le16	handle;
+	__u8	config_id;
+	__u8	create_context;
+	__u8	main_mode_type;
+	__u8	sub_mode_type;
+	__u8	min_main_mode_steps;
+	__u8	max_main_mode_steps;
+	__u8	main_mode_repetition;
+	__u8	mode_0_steps;
+	__u8	role;
+	__u8	rtt_type;
+	__u8	cs_sync_phy;
+	__u8	channel_map[10];
+	__u8	channel_map_repetition;
+	__u8	channel_selection_type;
+	__u8	ch3c_shape;
+	__u8	ch3c_jump;
+	__u8	reserved;
+} __packed;
+
+#define HCI_OP_LE_CS_REMOVE_CONFIG		0x2091
+struct hci_cp_le_cs_remove_config {
+	__le16	handle;
+	__u8	config_id;
+} __packed;
+
+#define HCI_OP_LE_CS_SET_CH_CLASSIFICATION	0x2092
+struct hci_cp_le_cs_set_ch_classification {
+	__u8	ch_classification[10];
+} __packed;
+
+struct hci_rp_le_cs_set_ch_classification {
+	__u8	status;
+} __packed;
+
+#define HCI_OP_LE_CS_SET_PROC_PARAM		0x2093
+struct hci_cp_le_cs_set_proc_param {
+	__le16	handle;
+	__u8	config_id;
+	__le16	max_procedure_len;
+	__le16	min_procedure_interval;
+	__le16	max_procedure_interval;
+	__le16	max_procedure_count;
+	__u8	min_subevent_len[3];
+	__u8	max_subevent_len[3];
+	__u8	tone_antenna_config_selection;
+	__u8	phy;
+	__u8	tx_power_delta;
+	__u8	preferred_peer_antenna;
+	__u8	snr_control_initiator;
+	__u8	snr_control_reflector;
+} __packed;
+
+struct hci_rp_le_cs_set_proc_param {
+	__u8	status;
+	__le16	handle;
+} __packed;
+
+#define HCI_OP_LE_CS_SET_PROC_ENABLE		0x2094
+struct hci_cp_le_cs_set_proc_enable {
+	__le16	handle;
+	__u8	config_id;
+	__u8	enable;
+} __packed;
+
+#define HCI_OP_LE_CS_TEST			0x2095
+struct hci_cp_le_cs_test {
+	__u8	main_mode_type;
+	__u8	sub_mode_type;
+	__u8	main_mode_repetition;
+	__u8	mode_0_steps;
+	__u8	role;
+	__u8	rtt_type;
+	__u8	cs_sync_phy;
+	__u8	cs_sync_antenna_selection;
+	__u8	subevent_len[3];
+	__le16	subevent_interval;
+	__u8	max_num_subevents;
+	__u8	transmit_power_level;
+	__u8	t_ip1_time;
+	__u8	t_ip2_time;
+	__u8	t_fcs_time;
+	__u8	t_pm_time;
+	__u8	t_sw_time;
+	__u8	tone_antenna_config_selection;
+	__u8	reserved;
+	__u8	snr_control_initiator;
+	__u8	snr_control_reflector;
+	__le16	drbg_nonce;
+	__u8	channel_map_repetition;
+	__le16	override_config;
+	__u8	override_parameters_length;
+	__u8	override_parameters_data[];
+} __packed;
+
+struct hci_rp_le_cs_test {
+	__u8	status;
+} __packed;
+
+#define HCI_OP_LE_CS_TEST_END			0x2096
+
 /* ---- HCI Events ---- */
 struct hci_ev_status {
 	__u8    status;
@@ -2960,6 +3169,129 @@ struct hci_evt_le_read_all_remote_features_complete {
 	__u8    features[248];
 } __packed;
 
+/* Channel Sounding Events */
+#define HCI_EVT_LE_CS_READ_RMT_SUPP_CAP_COMPLETE	0x2C
+struct hci_evt_le_cs_read_rmt_supp_cap_complete {
+	__u8	status;
+	__le16	handle;
+	__u8	num_configs_supp;
+	__le16	max_consec_proc_supp;
+	__u8	num_ant_supp;
+	__u8	max_ant_path_supp;
+	__u8	roles_supp;
+	__u8	modes_supp;
+	__u8	rtt_cap;
+	__u8	rtt_aa_only_n;
+	__u8	rtt_sounding_n;
+	__u8	rtt_rand_payload_n;
+	__le16	nadm_sounding_cap;
+	__le16	nadm_rand_cap;
+	__u8	cs_sync_phys_supp;
+	__le16	sub_feat_supp;
+	__le16	t_ip1_times_supp;
+	__le16	t_ip2_times_supp;
+	__le16	t_fcs_times_supp;
+	__le16	t_pm_times_supp;
+	__u8	t_sw_times_supp;
+	__u8	tx_snr_cap;
+} __packed;
+
+#define HCI_EVT_LE_CS_READ_RMT_FAE_TABLE_COMPLETE	0x2D
+struct hci_evt_le_cs_read_rmt_fae_table_complete {
+	__u8	status;
+	__le16	handle;
+	__u8	remote_fae_table[72];
+} __packed;
+
+#define HCI_EVT_LE_CS_SECURITY_ENABLE_COMPLETE		0x2E
+struct hci_evt_le_cs_security_enable_complete {
+	__u8	status;
+	__le16	handle;
+} __packed;
+
+#define HCI_EVT_LE_CS_CONFIG_COMPLETE			0x2F
+struct hci_evt_le_cs_config_complete {
+	__u8	status;
+	__le16	handle;
+	__u8	config_id;
+	__u8	action;
+	__u8	main_mode_type;
+	__u8	sub_mode_type;
+	__u8	min_main_mode_steps;
+	__u8	max_main_mode_steps;
+	__u8	main_mode_rep;
+	__u8	mode_0_steps;
+	__u8	role;
+	__u8	rtt_type;
+	__u8	cs_sync_phy;
+	__u8	channel_map[10];
+	__u8	channel_map_rep;
+	__u8	channel_sel_type;
+	__u8	ch3c_shape;
+	__u8	ch3c_jump;
+	__u8	reserved;
+	__u8	t_ip1_time;
+	__u8	t_ip2_time;
+	__u8	t_fcs_time;
+	__u8	t_pm_time;
+} __packed;
+
+#define HCI_EVT_LE_CS_PROCEDURE_ENABLE_COMPLETE		0x30
+struct hci_evt_le_cs_procedure_enable_complete {
+	__u8	status;
+	__le16	handle;
+	__u8	config_id;
+	__u8	state;
+	__u8	tone_ant_config_sel;
+	__s8	sel_tx_pwr;
+	__u8	sub_evt_len[3];
+	__u8	sub_evts_per_evt;
+	__le16	sub_evt_intrvl;
+	__le16	evt_intrvl;
+	__le16	proc_intrvl;
+	__le16	proc_counter;
+	__le16	max_proc_len;
+} __packed;
+
+#define HCI_EVT_LE_CS_SUBEVENT_RESULT			0x31
+struct hci_evt_le_cs_subevent_result {
+	__le16	handle;
+	__u8	config_id;
+	__le16	start_acl_conn_evt_counter;
+	__le16	proc_counter;
+	__le16	freq_comp;
+	__u8	ref_pwr_lvl;
+	__u8	proc_done_status;
+	__u8	subevt_done_status;
+	__u8	abort_reason;
+	__u8	num_ant_paths;
+	__u8	num_steps_reported;
+	__u8	step_mode[0]; /* depends on num_steps_reported */
+	__u8	step_channel[0]; /* depends on num_steps_reported */
+	__u8	step_data_length[0]; /* depends on num_steps_reported */
+	__u8	step_data[0]; /* depends on num_steps_reported */
+} __packed;
+
+#define HCI_EVT_LE_CS_SUBEVENT_RESULT_CONTINUE		0x32
+struct hci_evt_le_cs_subevent_result_continue {
+	__le16	handle;
+	__u8	config_id;
+	__u8	proc_done_status;
+	__u8	subevt_done_status;
+	__u8	abort_reason;
+	__u8	num_ant_paths;
+	__u8	num_steps_reported;
+	__u8	step_mode[0]; /* depends on num_steps_reported */
+	__u8	step_channel[0]; /* depends on num_steps_reported */
+	__u8	step_data_length[0]; /* depends on num_steps_reported */
+	__u8	step_data[0]; /* depends on num_steps_reported */
+} __packed;
+
+#define HCI_EVT_LE_CS_TEST_END_COMPLETE			0x33
+struct hci_evt_le_cs_test_end_complete {
+	__u8	status;
+} __packed;
+
 #define HCI_EV_VENDOR			0xff
 
 /* Internal events generated by Bluetooth stack */
diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h
index 4263e71a23ef..a7bffb908c1e 100644
--- a/include/net/bluetooth/hci_core.h
+++ b/include/net/bluetooth/hci_core.h
@@ -730,6 +730,8 @@ struct hci_conn {
 	__u16		le_per_adv_data_offset;
 	__u8		le_adv_phy;
 	__u8		le_adv_sec_phy;
+	__u8		le_tx_def_phys;
+	__u8		le_rx_def_phys;
 	__u8		le_tx_phy;
 	__u8		le_rx_phy;
 	__s8		rssi;
@@ -2071,6 +2073,12 @@ void hci_conn_del_sysfs(struct hci_conn *conn);
 #define ll_ext_feature_capable(dev) \
 	((dev)->le_features[7] & HCI_LE_LL_EXT_FEATURE)
 
+/* Channel sounding support */
+#define le_cs_capable(dev) \
+	((dev)->le_features[5] & HCI_LE_CS)
+#define le_cs_host_capable(dev) \
+	((dev)->le_features[5] & HCI_LE_CS_HOST)
+
 #define mws_transport_config_capable(dev) (((dev)->commands[30] & 0x08) && \
 	(!hci_test_quirk((dev), HCI_QUIRK_BROKEN_MWS_TRANSPORT_CONFIG)))
 
@@ -2334,6 +2342,7 @@ void *hci_sent_cmd_data(struct hci_dev *hdev, __u16 opcode);
 void *hci_recv_event_data(struct hci_dev *hdev, __u8 event);
 
 u32 hci_conn_get_phy(struct hci_conn *conn);
+int hci_conn_set_phy(struct hci_conn *conn, u32 phys);
 
 /* ----- HCI Sockets ----- */
 void hci_send_to_sock(struct hci_dev *hdev, struct sk_buff *skb);
diff --git a/include/net/bluetooth/hci_sync.h b/include/net/bluetooth/hci_sync.h
index 56076bbc981d..73e494b2591d 100644
--- a/include/net/bluetooth/hci_sync.h
+++ b/include/net/bluetooth/hci_sync.h
@@ -191,3 +191,6 @@ int hci_connect_big_sync(struct hci_dev *hdev, struct hci_conn *conn);
 int hci_past_sync(struct hci_conn *conn, struct hci_conn *le);
 
 int hci_le_read_remote_features(struct hci_conn *conn);
+
+int hci_acl_change_pkt_type(struct hci_conn *conn, u16 pkt_type);
+int hci_le_set_phy(struct hci_conn *conn, u8 tx_phys, u8 rx_phys);
diff --git a/include/net/inet_frag.h b/include/net/inet_frag.h
index 0eccd9c3a883..365925c9d262 100644
--- a/include/net/inet_frag.h
+++ b/include/net/inet_frag.h
@@ -123,27 +123,15 @@ void inet_frags_fini(struct inet_frags *);
 
 int fqdir_init(struct fqdir **fqdirp, struct inet_frags *f, struct net *net);
 
-static inline void fqdir_pre_exit(struct fqdir *fqdir)
-{
-	/* Prevent creation of new frags.
-	 * Pairs with READ_ONCE() in inet_frag_find().
-	 */
-	WRITE_ONCE(fqdir->high_thresh, 0);
-
-	/* Pairs with READ_ONCE() in inet_frag_kill(), ip_expire()
-	 * and ip6frag_expire_frag_queue().
-	 */
-	WRITE_ONCE(fqdir->dead, true);
-}
+void fqdir_pre_exit(struct fqdir *fqdir);
 void fqdir_exit(struct fqdir *fqdir);
 
 void inet_frag_kill(struct inet_frag_queue *q, int *refs);
 void inet_frag_destroy(struct inet_frag_queue *q);
 struct inet_frag_queue *inet_frag_find(struct fqdir *fqdir, void *key);
 
-/* Free all skbs in the queue; return the sum of their truesizes. */
-unsigned int inet_frag_rbtree_purge(struct rb_root *root,
-				    enum skb_drop_reason reason);
+void inet_frag_queue_flush(struct inet_frag_queue *q,
+			   enum skb_drop_reason reason);
 
 static inline void inet_frag_putn(struct inet_frag_queue *q, int refs)
 {
diff --git a/include/net/ipv6_frag.h b/include/net/ipv6_frag.h
index 38ef66826939..41d9fc6965f9 100644
--- a/include/net/ipv6_frag.h
+++ b/include/net/ipv6_frag.h
@@ -69,9 +69,6 @@ ip6frag_expire_frag_queue(struct net *net, struct frag_queue *fq)
 	int refs = 1;
 
 	rcu_read_lock();
-	/* Paired with the WRITE_ONCE() in fqdir_pre_exit(). */
-	if (READ_ONCE(fq->q.fqdir->dead))
-		goto out_rcu_unlock;
 	spin_lock(&fq->q.lock);
 
 	if (fq->q.flags & INET_FRAG_COMPLETE)
@@ -80,6 +77,12 @@ ip6frag_expire_frag_queue(struct net *net, struct frag_queue *fq)
 	fq->q.flags |= INET_FRAG_DROP;
 	inet_frag_kill(&fq->q, &refs);
 
+	/* Paired with the WRITE_ONCE() in fqdir_pre_exit(). */
+	if (READ_ONCE(fq->q.fqdir->dead)) {
+		inet_frag_queue_flush(&fq->q, 0);
+		goto out;
+	}
+
 	dev = dev_get_by_index_rcu(net, fq->iif);
 	if (!dev)
 		goto out;
diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h
index fab7dc73f738..0e266c2d0e7f 100644
--- a/include/net/netfilter/nf_tables.h
+++ b/include/net/netfilter/nf_tables.h
@@ -1091,6 +1091,29 @@ struct nft_rule_blob {
 		__attribute__((aligned(__alignof__(struct nft_rule_dp))));
 };
 
+enum nft_chain_types {
+	NFT_CHAIN_T_DEFAULT = 0,
+	NFT_CHAIN_T_ROUTE,
+	NFT_CHAIN_T_NAT,
+	NFT_CHAIN_T_MAX
+};
+
+/**
+ *	struct nft_chain_validate_state - validation state
+ *
+ *	If a chain is encountered again during table validation it is
+ *	possible to avoid revalidation provided the calling context is
+ *	compatible.  This structure stores relevant calling context of
+ *	previous validations.
+ *
+ *	@hook_mask: the hook numbers and locations the chain is linked to
+ *	@depth: the deepest call chain level the chain is linked to
+ */
+struct nft_chain_validate_state {
+	u8			hook_mask[NFT_CHAIN_T_MAX];
+	u8			depth;
+};
+
 /**
  *	struct nft_chain - nf_tables chain
  *
@@ -1109,6 +1132,7 @@ struct nft_rule_blob {
  *	@udlen: user data length
  *	@udata: user data in the chain
  *	@blob_next: rule blob pointer to the next in the chain
+ *	@vstate: validation state
  */
 struct nft_chain {
 	struct nft_rule_blob		__rcu *blob_gen_0;
@@ -1128,9 +1152,10 @@ struct nft_chain {
 
 	/* Only used during control plane commit phase: */
 	struct nft_rule_blob		*blob_next;
+	struct nft_chain_validate_state vstate;
 };
 
-int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain);
+int nft_chain_validate(const struct nft_ctx *ctx, struct nft_chain *chain);
 int nft_setelem_validate(const struct nft_ctx *ctx, struct nft_set *set,
 			 const struct nft_set_iter *iter,
 			 struct nft_elem_priv *elem_priv);
@@ -1138,13 +1163,6 @@ int nft_set_catchall_validate(const struct nft_ctx *ctx, struct nft_set *set);
 int nf_tables_bind_chain(const struct nft_ctx *ctx, struct nft_chain *chain);
 void nf_tables_unbind_chain(const struct nft_ctx *ctx, struct nft_chain *chain);
 
-enum nft_chain_types {
-	NFT_CHAIN_T_DEFAULT = 0,
-	NFT_CHAIN_T_ROUTE,
-	NFT_CHAIN_T_NAT,
-	NFT_CHAIN_T_MAX
-};
-
 /**
  * 	struct nft_chain_type - nf_tables chain type info
  *
diff --git a/include/scsi/scsi.h b/include/scsi/scsi.h
index 96b350366670..08ac3200b4a4 100644
--- a/include/scsi/scsi.h
+++ b/include/scsi/scsi.h
@@ -106,12 +106,15 @@ enum scsi_disposition {
 };
 
 /*
- * Midlevel queue return values.
+ * Status values returned by the .queuecommand() callback if a command has not
+ * been queued.
  */
-#define SCSI_MLQUEUE_HOST_BUSY   0x1055
-#define SCSI_MLQUEUE_DEVICE_BUSY 0x1056
-#define SCSI_MLQUEUE_EH_RETRY    0x1057
-#define SCSI_MLQUEUE_TARGET_BUSY 0x1058
+enum scsi_qc_status {
+	SCSI_MLQUEUE_HOST_BUSY   = 0x1055,
+	SCSI_MLQUEUE_DEVICE_BUSY = 0x1056,
+	SCSI_MLQUEUE_EH_RETRY    = 0x1057,
+	SCSI_MLQUEUE_TARGET_BUSY = 0x1058,
+};
 
 /*
  *  Use these to separate status msg and our bytes
diff --git a/include/scsi/scsi_transport_fc.h b/include/scsi/scsi_transport_fc.h
index b908aacfef48..9f30625aa0d3 100644
--- a/include/scsi/scsi_transport_fc.h
+++ b/include/scsi/scsi_transport_fc.h
@@ -317,6 +317,15 @@ struct fc_fpin_stats {
 	u64 cn_device_specific;
 };
 
+#define FC_RPORT_ENCRYPTION_STATUS_MAX_LEN      14
+/*
+ * Encryption Information
+ */
+struct fc_encryption_info {
+	/* Encryption Status */
+	u8 status;
+};
+
 /* Macro for use in defining Remote Port attributes */
 #define FC_RPORT_ATTR(_name,_mode,_show,_store)				\
 struct device_attribute dev_attr_rport_##_name = 	\
@@ -364,6 +373,7 @@ struct fc_rport {	/* aka fc_starget_attrs */
 	u64 port_name;
 	u32 port_id;
 	u32 roles;
+	struct fc_encryption_info enc_info;
 	enum fc_port_state port_state;	/* Will only be ONLINE or UNKNOWN */
 	u32 scsi_target_id;
 	u32 fast_io_fail_tmo;
@@ -691,6 +701,8 @@ struct fc_function_template {
 	struct fc_host_statistics * (*get_fc_host_stats)(struct Scsi_Host *);
 	void	(*reset_fc_host_stats)(struct Scsi_Host *);
 
+	struct fc_encryption_info * (*get_fc_rport_enc_info)(struct fc_rport *);
+
 	int	(*issue_fc_host_lip)(struct Scsi_Host *);
 
 	void    (*dev_loss_tmo_callbk)(struct fc_rport *);
diff --git a/include/sound/seq_device.h b/include/sound/seq_device.h
index dead74b022f4..a72380c202e9 100644
--- a/include/sound/seq_device.h
+++ b/include/sound/seq_device.h
@@ -43,6 +43,8 @@ struct snd_seq_device {
  *	Typically, call snd_device_free(dev->card, dev->driver_data)
  */
 struct snd_seq_driver {
+	int (*probe)(struct snd_seq_device *dev);
+	void (*remove)(struct snd_seq_device *dev);
 	struct device_driver driver;
 	char *id;
 	int argsize;
diff --git a/include/sound/soc-acpi.h b/include/sound/soc-acpi.h
index 90d73b9bddab..0519afd7217f 100644
--- a/include/sound/soc-acpi.h
+++ b/include/sound/soc-acpi.h
@@ -203,6 +203,8 @@ struct snd_soc_acpi_link_adr {
  *	@mach: the pointer of the machine driver
  *	@prefix: the prefix of the topology file name. Typically, it is the path.
  *	@tplg_files: the pointer of the array of the topology file names.
+ *	@best_effort: ignore non supported links and try to build the card in best effort
+ *		      with supported links
  */
 /* Descriptor for SST ASoC machine driver */
 struct snd_soc_acpi_mach {
@@ -224,7 +226,8 @@ struct snd_soc_acpi_mach {
 	const u32 tplg_quirk_mask;
 	int (*get_function_tplg_files)(struct snd_soc_card *card,
 				       const struct snd_soc_acpi_mach *mach,
-				       const char *prefix, const char ***tplg_files);
+				       const char *prefix, const char ***tplg_files,
+				       bool best_effort);
 };
 
 #define SND_SOC_ACPI_MAX_CODECS 3
diff --git a/include/sound/sof.h b/include/sound/sof.h
index eddea82c7b5a..38d6c8cb5e83 100644
--- a/include/sound/sof.h
+++ b/include/sound/sof.h
@@ -159,6 +159,9 @@ struct sof_dev_desc {
 	/* The platform supports DSPless mode */
 	bool dspless_mode_supported;
 
+	/* On demand DSP booting is possible on the platform */
+	bool on_demand_dsp_boot;
+
 	/* defaults paths for firmware, library and topology files */
 	const char *default_fw_path[SOF_IPC_TYPE_COUNT];
 	const char *default_lib_path[SOF_IPC_TYPE_COUNT];
diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h
index 7016d93fa383..b62d5fcce950 100644
--- a/include/target/target_core_base.h
+++ b/include/target/target_core_base.h
@@ -108,6 +108,9 @@
 #define SE_MODE_PAGE_BUF			512
 #define SE_SENSE_BUF				96
 
+/* Peripheral Device Text Identification Information */
+#define PD_TEXT_ID_INFO_LEN			256
+
 enum target_submit_type {
 	/* Use the fabric driver's default submission type */
 	TARGET_FABRIC_DEFAULT_SUBMIT,
@@ -348,6 +351,7 @@ struct t10_wwn {
 	struct se_device *t10_dev;
 	struct config_group t10_wwn_group;
 	struct list_head t10_vpd_list;
+	char pd_text_id_info[PD_TEXT_ID_INFO_LEN];
 };
 
 struct t10_pr_registration {
diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h
index 7e418f065b94..125bdc166bfe 100644
--- a/include/trace/events/btrfs.h
+++ b/include/trace/events/btrfs.h
@@ -224,7 +224,8 @@ DECLARE_EVENT_CLASS(btrfs__inode,
 		__entry->generation = BTRFS_I(inode)->generation;
 		__entry->last_trans = BTRFS_I(inode)->last_trans;
 		__entry->logged_trans = BTRFS_I(inode)->logged_trans;
-		__entry->root_objectid = btrfs_root_id(BTRFS_I(inode)->root);
+		__entry->root_objectid = BTRFS_I(inode)->root ?
+					 btrfs_root_id(BTRFS_I(inode)->root) : 0;
 	),
 
 	TP_printk_btrfs("root=%llu(%s) gen=%llu ino=%llu blocks=%llu "
diff --git a/include/trace/events/dma_buf.h b/include/trace/events/dma_buf.h
new file mode 100644
index 000000000000..35f8140095f4
--- /dev/null
+++ b/include/trace/events/dma_buf.h
@@ -0,0 +1,157 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM dma_buf
+
+#if !defined(_TRACE_DMA_BUF_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_DMA_BUF_H
+
+#include <linux/dma-buf.h>
+#include <linux/tracepoint.h>
+
+DECLARE_EVENT_CLASS(dma_buf,
+
+	TP_PROTO(struct dma_buf *dmabuf),
+
+	TP_ARGS(dmabuf),
+
+	TP_STRUCT__entry(
+		__string(exp_name, dmabuf->exp_name)
+		__field(size_t, size)
+		__field(ino_t, ino)
+	),
+
+	TP_fast_assign(
+		__assign_str(exp_name);
+		__entry->size = dmabuf->size;
+		__entry->ino = dmabuf->file->f_inode->i_ino;
+	),
+
+	TP_printk("exp_name=%s size=%zu ino=%lu",
+		  __get_str(exp_name),
+		  __entry->size,
+		  __entry->ino)
+);
+
+DECLARE_EVENT_CLASS(dma_buf_attach_dev,
+
+	TP_PROTO(struct dma_buf *dmabuf, struct dma_buf_attachment *attach,
+		bool is_dynamic, struct device *dev),
+
+	TP_ARGS(dmabuf, attach, is_dynamic, dev),
+
+	TP_STRUCT__entry(
+		__string(dev_name, dev_name(dev))
+		__string(exp_name, dmabuf->exp_name)
+		__field(size_t, size)
+		__field(ino_t, ino)
+		__field(struct dma_buf_attachment *, attach)
+		__field(bool, is_dynamic)
+	),
+
+	TP_fast_assign(
+		__assign_str(dev_name);
+		__assign_str(exp_name);
+		__entry->size = dmabuf->size;
+		__entry->ino = dmabuf->file->f_inode->i_ino;
+		__entry->is_dynamic = is_dynamic;
+		__entry->attach = attach;
+	),
+
+	TP_printk("exp_name=%s size=%zu ino=%lu attachment:%p is_dynamic=%d dev_name=%s",
+		  __get_str(exp_name),
+		  __entry->size,
+		  __entry->ino,
+		  __entry->attach,
+		  __entry->is_dynamic,
+		  __get_str(dev_name))
+);
+
+DECLARE_EVENT_CLASS(dma_buf_fd,
+
+	TP_PROTO(struct dma_buf *dmabuf, int fd),
+
+	TP_ARGS(dmabuf, fd),
+
+	TP_STRUCT__entry(
+		__string(exp_name, dmabuf->exp_name)
+		__field(size_t, size)
+		__field(ino_t, ino)
+		__field(int, fd)
+	),
+
+	TP_fast_assign(
+		__assign_str(exp_name);
+		__entry->size = dmabuf->size;
+		__entry->ino = dmabuf->file->f_inode->i_ino;
+		__entry->fd = fd;
+	),
+
+	TP_printk("exp_name=%s size=%zu ino=%lu fd=%d",
+		  __get_str(exp_name),
+		  __entry->size,
+		  __entry->ino,
+		  __entry->fd)
+);
+
+DEFINE_EVENT(dma_buf, dma_buf_export,
+
+	TP_PROTO(struct dma_buf *dmabuf),
+
+	TP_ARGS(dmabuf)
+);
+
+DEFINE_EVENT(dma_buf, dma_buf_mmap_internal,
+
+	TP_PROTO(struct dma_buf *dmabuf),
+
+	TP_ARGS(dmabuf)
+);
+
+DEFINE_EVENT(dma_buf, dma_buf_mmap,
+
+	TP_PROTO(struct dma_buf *dmabuf),
+
+	TP_ARGS(dmabuf)
+);
+
+DEFINE_EVENT(dma_buf, dma_buf_put,
+
+	TP_PROTO(struct dma_buf *dmabuf),
+
+	TP_ARGS(dmabuf)
+);
+
+DEFINE_EVENT(dma_buf_attach_dev, dma_buf_dynamic_attach,
+
+	TP_PROTO(struct dma_buf *dmabuf, struct dma_buf_attachment *attach,
+		bool is_dynamic, struct device *dev),
+
+	TP_ARGS(dmabuf, attach, is_dynamic, dev)
+);
+
+DEFINE_EVENT(dma_buf_attach_dev, dma_buf_detach,
+
+	TP_PROTO(struct dma_buf *dmabuf, struct dma_buf_attachment *attach,
+		bool is_dynamic, struct device *dev),
+
+	TP_ARGS(dmabuf, attach, is_dynamic, dev)
+);
+
+DEFINE_EVENT(dma_buf_fd, dma_buf_fd,
+
+	TP_PROTO(struct dma_buf *dmabuf, int fd),
+
+	TP_ARGS(dmabuf, fd)
+);
+
+DEFINE_EVENT(dma_buf_fd, dma_buf_get,
+
+	TP_PROTO(struct dma_buf *dmabuf, int fd),
+
+	TP_ARGS(dmabuf, fd)
+);
+
+#endif /* _TRACE_DMA_BUF_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h
index df4017dcc701..635dcabcf1c7 100644
--- a/include/trace/events/f2fs.h
+++ b/include/trace/events/f2fs.h
@@ -1358,6 +1358,7 @@ DECLARE_EVENT_CLASS(f2fs__folio,
 		__field(int, type)
 		__field(int, dir)
 		__field(pgoff_t, index)
+		__field(pgoff_t, nrpages)
 		__field(int, dirty)
 		__field(int, uptodate)
 	),
@@ -1368,16 +1369,18 @@ DECLARE_EVENT_CLASS(f2fs__folio,
 		__entry->type	= type;
 		__entry->dir	= S_ISDIR(folio->mapping->host->i_mode);
 		__entry->index	= folio->index;
+		__entry->nrpages= folio_nr_pages(folio);
 		__entry->dirty	= folio_test_dirty(folio);
 		__entry->uptodate = folio_test_uptodate(folio);
 	),
 
-	TP_printk("dev = (%d,%d), ino = %lu, %s, %s, index = %lu, "
+	TP_printk("dev = (%d,%d), ino = %lu, %s, %s, index = %lu, nr_pages = %lu, "
 		"dirty = %d, uptodate = %d",
 		show_dev_ino(__entry),
 		show_block_type(__entry->type),
 		show_file_type(__entry->dir),
 		(unsigned long)__entry->index,
+		(unsigned long)__entry->nrpages,
 		__entry->dirty,
 		__entry->uptodate)
 );
@@ -1403,6 +1406,13 @@ DEFINE_EVENT(f2fs__folio, f2fs_readpage,
 	TP_ARGS(folio, type)
 );
 
+DEFINE_EVENT(f2fs__folio, f2fs_read_folio,
+
+	TP_PROTO(struct folio *folio, int type),
+
+	TP_ARGS(folio, type)
+);
+
 DEFINE_EVENT(f2fs__folio, f2fs_set_page_dirty,
 
 	TP_PROTO(struct folio *folio, int type),
diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h
index 4cde53b45a85..4e41bff31888 100644
--- a/include/trace/events/huge_memory.h
+++ b/include/trace/events/huge_memory.h
@@ -37,7 +37,8 @@
 	EM( SCAN_PAGE_HAS_PRIVATE,	"page_has_private")		\
 	EM( SCAN_STORE_FAILED,		"store_failed")			\
 	EM( SCAN_COPY_MC,		"copy_poisoned_page")		\
-	EMe(SCAN_PAGE_FILLED,		"page_filled")
+	EM( SCAN_PAGE_FILLED,		"page_filled")			\
+	EMe(SCAN_PAGE_DIRTY_OR_WRITEBACK, "page_dirty_or_writeback")
 
 #undef EM
 #undef EMe
diff --git a/include/trace/events/kmem.h b/include/trace/events/kmem.h
index 7f93e754da5c..91c81c44f884 100644
--- a/include/trace/events/kmem.h
+++ b/include/trace/events/kmem.h
@@ -442,7 +442,7 @@ TRACE_EVENT(rss_stat,
 		__entry->mm_id = mm_ptr_to_hash(mm);
 		__entry->curr = !!(current->mm == mm);
 		__entry->member = member;
-		__entry->size = (percpu_counter_sum_positive(&mm->rss_stat[member])
+		__entry->size = (percpu_counter_tree_approximate_sum_positive(&mm->rss_stat[member])
 							    << PAGE_SHIFT);
 	),
 
diff --git a/include/trace/events/tlb.h b/include/trace/events/tlb.h
index b4d8e7dc38f8..fb8369511685 100644
--- a/include/trace/events/tlb.h
+++ b/include/trace/events/tlb.h
@@ -12,8 +12,9 @@
 	EM(  TLB_FLUSH_ON_TASK_SWITCH,	"flush on task switch" )	\
 	EM(  TLB_REMOTE_SHOOTDOWN,	"remote shootdown" )		\
 	EM(  TLB_LOCAL_SHOOTDOWN,	"local shootdown" )		\
-	EM(  TLB_LOCAL_MM_SHOOTDOWN,	"local mm shootdown" )		\
-	EMe( TLB_REMOTE_SEND_IPI,	"remote ipi send" )
+	EM(  TLB_LOCAL_MM_SHOOTDOWN,	"local MM shootdown" )		\
+	EM(  TLB_REMOTE_SEND_IPI,	"remote IPI send" )		\
+	EMe( TLB_REMOTE_WRONG_CPU,	"remote wrong CPU" )
 
 /*
  * First define the enums in TLB_FLUSH_REASON to be exported to userspace
diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
index f80aa4c9d88f..351c2fb2df90 100644
--- a/include/uapi/drm/amdgpu_drm.h
+++ b/include/uapi/drm/amdgpu_drm.h
@@ -883,7 +883,7 @@ struct drm_amdgpu_gem_list_handles_entry {
 #define AMDGPU_VM_PAGE_WRITEABLE	(1 << 2)
 /* executable mapping, new for VI */
 #define AMDGPU_VM_PAGE_EXECUTABLE	(1 << 3)
-/* partially resident texture */
+/* unmapped page of partially resident textures */
 #define AMDGPU_VM_PAGE_PRT		(1 << 4)
 /* MTYPE flags use bit 5 to 8 */
 #define AMDGPU_VM_MTYPE_MASK		(0xf << 5)
@@ -1427,6 +1427,7 @@ struct drm_amdgpu_info_vbios {
 #define AMDGPU_VRAM_TYPE_LPDDR4 11
 #define AMDGPU_VRAM_TYPE_LPDDR5 12
 #define AMDGPU_VRAM_TYPE_HBM3E 13
+#define AMDGPU_VRAM_TYPE_HBM4 14
 
 struct drm_amdgpu_info_device {
 	/** PCI Device ID */
diff --git a/include/uapi/drm/amdxdna_accel.h b/include/uapi/drm/amdxdna_accel.h
index 62c917fd4f7b..9c44db2b3dcd 100644
--- a/include/uapi/drm/amdxdna_accel.h
+++ b/include/uapi/drm/amdxdna_accel.h
@@ -19,6 +19,14 @@ extern "C" {
 #define AMDXDNA_INVALID_BO_HANDLE	0
 #define AMDXDNA_INVALID_FENCE_HANDLE	0
 
+/*
+ * Define hardware context priority
+ */
+#define AMDXDNA_QOS_REALTIME_PRIORITY	0x100
+#define AMDXDNA_QOS_HIGH_PRIORITY	0x180
+#define AMDXDNA_QOS_NORMAL_PRIORITY	0x200
+#define AMDXDNA_QOS_LOW_PRIORITY	0x280
+
 enum amdxdna_device_type {
 	AMDXDNA_DEV_TYPE_UNKNOWN = -1,
 	AMDXDNA_DEV_TYPE_KMQ,
diff --git a/include/uapi/drm/panfrost_drm.h b/include/uapi/drm/panfrost_drm.h
index 1956431bb391..50d5337f35ef 100644
--- a/include/uapi/drm/panfrost_drm.h
+++ b/include/uapi/drm/panfrost_drm.h
@@ -24,6 +24,8 @@ extern "C" {
 #define DRM_PANFROST_SET_LABEL_BO		0x09
 #define DRM_PANFROST_JM_CTX_CREATE		0x0a
 #define DRM_PANFROST_JM_CTX_DESTROY		0x0b
+#define DRM_PANFROST_SYNC_BO			0x0c
+#define DRM_PANFROST_QUERY_BO_INFO		0x0d
 
 #define DRM_IOCTL_PANFROST_SUBMIT		DRM_IOW(DRM_COMMAND_BASE + DRM_PANFROST_SUBMIT, struct drm_panfrost_submit)
 #define DRM_IOCTL_PANFROST_WAIT_BO		DRM_IOW(DRM_COMMAND_BASE + DRM_PANFROST_WAIT_BO, struct drm_panfrost_wait_bo)
@@ -35,6 +37,8 @@ extern "C" {
 #define DRM_IOCTL_PANFROST_SET_LABEL_BO		DRM_IOWR(DRM_COMMAND_BASE + DRM_PANFROST_SET_LABEL_BO, struct drm_panfrost_set_label_bo)
 #define DRM_IOCTL_PANFROST_JM_CTX_CREATE	DRM_IOWR(DRM_COMMAND_BASE + DRM_PANFROST_JM_CTX_CREATE, struct drm_panfrost_jm_ctx_create)
 #define DRM_IOCTL_PANFROST_JM_CTX_DESTROY	DRM_IOWR(DRM_COMMAND_BASE + DRM_PANFROST_JM_CTX_DESTROY, struct drm_panfrost_jm_ctx_destroy)
+#define DRM_IOCTL_PANFROST_SYNC_BO		DRM_IOWR(DRM_COMMAND_BASE + DRM_PANFROST_SYNC_BO, struct drm_panfrost_sync_bo)
+#define DRM_IOCTL_PANFROST_QUERY_BO_INFO	DRM_IOWR(DRM_COMMAND_BASE + DRM_PANFROST_QUERY_BO_INFO, struct drm_panfrost_query_bo_info)
 
 /*
  * Unstable ioctl(s): only exposed when the unsafe unstable_ioctls module
@@ -120,9 +124,12 @@ struct drm_panfrost_wait_bo {
 	__s64 timeout_ns;
 };
 
-/* Valid flags to pass to drm_panfrost_create_bo */
+/* Valid flags to pass to drm_panfrost_create_bo.
+ * PANFROST_BO_WB_MMAP can't be set if PANFROST_BO_HEAP is.
+ */
 #define PANFROST_BO_NOEXEC	1
 #define PANFROST_BO_HEAP	2
+#define PANFROST_BO_WB_MMAP	4
 
 /**
  * struct drm_panfrost_create_bo - ioctl argument for creating Panfrost BOs.
@@ -228,6 +235,13 @@ enum drm_panfrost_param {
 	DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP,
 	DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP_FREQUENCY,
 	DRM_PANFROST_PARAM_ALLOWED_JM_CTX_PRIORITIES,
+	DRM_PANFROST_PARAM_SELECTED_COHERENCY,
+};
+
+enum drm_panfrost_gpu_coherency {
+	DRM_PANFROST_GPU_COHERENCY_ACE_LITE = 0,
+	DRM_PANFROST_GPU_COHERENCY_ACE = 1,
+	DRM_PANFROST_GPU_COHERENCY_NONE = 31,
 };
 
 struct drm_panfrost_get_param {
@@ -301,6 +315,66 @@ struct drm_panfrost_set_label_bo {
 	__u64 label;
 };
 
+/* Valid flags to pass to drm_panfrost_bo_sync_op */
+#define PANFROST_BO_SYNC_CPU_CACHE_FLUSH			0
+#define PANFROST_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE		1
+
+/**
+ * struct drm_panthor_bo_flush_map_op - BO map sync op
+ */
+struct drm_panfrost_bo_sync_op {
+	/** @handle: Handle of the buffer object to sync. */
+	__u32 handle;
+
+	/** @type: Type of sync operation. */
+	__u32 type;
+
+	/**
+	 * @offset: Offset into the BO at which the sync range starts.
+	 *
+	 * This will be rounded down to the nearest cache line as needed.
+	 */
+	__u32 offset;
+
+	/**
+	 * @size: Size of the range to sync
+	 *
+	 * @size + @offset will be rounded up to the nearest cache line as
+	 * needed.
+	 */
+	__u32 size;
+};
+
+/**
+ * struct drm_panfrost_sync_bo - ioctl argument for syncing BO maps
+ */
+struct drm_panfrost_sync_bo {
+	/** Array of struct drm_panfrost_bo_sync_op */
+	__u64 ops;
+
+	/** Number of BO sync ops */
+	__u32 op_count;
+
+	__u32 pad;
+};
+
+/** BO comes from a different subsystem. */
+#define DRM_PANFROST_BO_IS_IMPORTED (1 << 0)
+
+struct drm_panfrost_query_bo_info {
+	/** Handle of the object being queried. */
+	__u32 handle;
+
+	/** Extra flags that are not coming from the BO_CREATE ioctl(). */
+	__u32 extra_flags;
+
+	/** Flags passed at creation time. */
+	__u32 create_flags;
+
+	/** Will be zero on return. */
+	__u32 pad;
+};
+
 /* Definitions for coredump decoding in user space */
 #define PANFROSTDUMP_MAJOR 1
 #define PANFROSTDUMP_MINOR 0
diff --git a/include/uapi/drm/panthor_drm.h b/include/uapi/drm/panthor_drm.h
index 467d365ed7ba..b401ac585d6a 100644
--- a/include/uapi/drm/panthor_drm.h
+++ b/include/uapi/drm/panthor_drm.h
@@ -144,6 +144,16 @@ enum drm_panthor_ioctl_id {
 	 * pgoff_t size.
 	 */
 	DRM_PANTHOR_SET_USER_MMIO_OFFSET,
+
+	/** @DRM_PANTHOR_BO_SYNC: Sync BO data to/from the device */
+	DRM_PANTHOR_BO_SYNC,
+
+	/**
+	 * @DRM_PANTHOR_BO_QUERY_INFO: Query information about a BO.
+	 *
+	 * This is useful for imported BOs.
+	 */
+	DRM_PANTHOR_BO_QUERY_INFO,
 };
 
 /**
@@ -245,6 +255,26 @@ enum drm_panthor_dev_query_type {
 	DRM_PANTHOR_DEV_QUERY_GROUP_PRIORITIES_INFO,
 };
 
+/**
+ * enum drm_panthor_gpu_coherency: Type of GPU coherency
+ */
+enum drm_panthor_gpu_coherency {
+	/**
+	 * @DRM_PANTHOR_GPU_COHERENCY_ACE_LITE: ACE Lite coherency.
+	 */
+	DRM_PANTHOR_GPU_COHERENCY_ACE_LITE = 0,
+
+	/**
+	 * @DRM_PANTHOR_GPU_COHERENCY_ACE: ACE coherency.
+	 */
+	DRM_PANTHOR_GPU_COHERENCY_ACE = 1,
+
+	/**
+	 * @DRM_PANTHOR_GPU_COHERENCY_NONE: No coherency.
+	 */
+	DRM_PANTHOR_GPU_COHERENCY_NONE = 31,
+};
+
 /**
  * struct drm_panthor_gpu_info - GPU information
  *
@@ -301,7 +331,16 @@ struct drm_panthor_gpu_info {
 	 */
 	__u32 thread_max_barrier_size;
 
-	/** @coherency_features: Coherency features. */
+	/**
+	 * @coherency_features: Coherency features.
+	 *
+	 * Combination of drm_panthor_gpu_coherency flags.
+	 *
+	 * Note that this is just what the coherency protocols supported by the
+	 * GPU, but the actual coherency in place depends on the SoC
+	 * integration and is reflected by
+	 * drm_panthor_gpu_info::selected_coherency.
+	 */
 	__u32 coherency_features;
 
 	/** @texture_features: Texture features. */
@@ -310,8 +349,12 @@ struct drm_panthor_gpu_info {
 	/** @as_present: Bitmask encoding the number of address-space exposed by the MMU. */
 	__u32 as_present;
 
-	/** @pad0: MBZ. */
-	__u32 pad0;
+	/**
+	 * @selected_coherency: Coherency selected for this device.
+	 *
+	 * One of drm_panthor_gpu_coherency.
+	 */
+	__u32 selected_coherency;
 
 	/** @shader_present: Bitmask encoding the shader cores exposed by the GPU. */
 	__u64 shader_present;
@@ -638,6 +681,15 @@ struct drm_panthor_vm_get_state {
 enum drm_panthor_bo_flags {
 	/** @DRM_PANTHOR_BO_NO_MMAP: The buffer object will never be CPU-mapped in userspace. */
 	DRM_PANTHOR_BO_NO_MMAP = (1 << 0),
+
+	/**
+	 * @DRM_PANTHOR_BO_WB_MMAP: Force "Write-Back Cacheable" CPU mapping.
+	 *
+	 * CPU map the buffer object in userspace by forcing the "Write-Back
+	 * Cacheable" cacheability attribute. The mapping otherwise uses the
+	 * "Non-Cacheable" attribute if the GPU is not IO coherent.
+	 */
+	DRM_PANTHOR_BO_WB_MMAP = (1 << 1),
 };
 
 /**
@@ -1040,6 +1092,101 @@ struct drm_panthor_set_user_mmio_offset {
 	__u64 offset;
 };
 
+/**
+ * enum drm_panthor_bo_sync_op_type - BO sync type
+ */
+enum drm_panthor_bo_sync_op_type {
+	/** @DRM_PANTHOR_BO_SYNC_CPU_CACHE_FLUSH: Flush CPU caches. */
+	DRM_PANTHOR_BO_SYNC_CPU_CACHE_FLUSH = 0,
+
+	/** @DRM_PANTHOR_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE: Flush and invalidate CPU caches. */
+	DRM_PANTHOR_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE = 1,
+};
+
+/**
+ * struct drm_panthor_bo_sync_op - BO map sync op
+ */
+struct drm_panthor_bo_sync_op {
+	/** @handle: Handle of the buffer object to sync. */
+	__u32 handle;
+
+	/** @type: Type of operation. */
+	__u32 type;
+
+	/**
+	 * @offset: Offset into the BO at which the sync range starts.
+	 *
+	 * This will be rounded down to the nearest cache line as needed.
+	 */
+	__u64 offset;
+
+	/**
+	 * @size: Size of the range to sync
+	 *
+	 * @size + @offset will be rounded up to the nearest cache line as
+	 * needed.
+	 */
+	__u64 size;
+};
+
+/**
+ * struct drm_panthor_bo_sync - BO map sync request
+ */
+struct drm_panthor_bo_sync {
+	/**
+	 * @ops: Array of struct drm_panthor_bo_sync_op sync operations.
+	 */
+	struct drm_panthor_obj_array ops;
+};
+
+/**
+ * enum drm_panthor_bo_extra_flags - Set of flags returned on a BO_QUERY_INFO request
+ *
+ * Those are flags reflecting BO properties that are not directly coming from the flags
+ * passed are creation time, or information on BOs that were imported from other drivers.
+ */
+enum drm_panthor_bo_extra_flags {
+	/**
+	 * @DRM_PANTHOR_BO_IS_IMPORTED: BO has been imported from an external driver.
+	 *
+	 * Note that imported dma-buf handles are not flagged as imported if they
+	 * where exported by panthor. Only buffers that are coming from other drivers
+	 * (dma heaps, other GPUs, display controllers, V4L, ...).
+	 *
+	 * It's also important to note that all imported BOs are mapped cached and can't
+	 * be considered IO-coherent even if the GPU is. This means they require explicit
+	 * syncs that must go through the DRM_PANTHOR_BO_SYNC ioctl (userland cache
+	 * maintenance is not allowed in that case, because extra operations might be
+	 * needed to make changes visible to the CPU/device, like buffer migration when the
+	 * exporter is a GPU with its own VRAM).
+	 */
+	DRM_PANTHOR_BO_IS_IMPORTED = (1 << 0),
+};
+
+/**
+ * struct drm_panthor_bo_query_info - Query BO info
+ */
+struct drm_panthor_bo_query_info {
+	/** @handle: Handle of the buffer object to query flags on. */
+	__u32 handle;
+
+	/**
+	 * @extra_flags: Combination of enum drm_panthor_bo_extra_flags flags.
+	 */
+	__u32 extra_flags;
+
+	/**
+	 * @create_flags: Flags passed at creation time.
+	 *
+	 * Combination of enum drm_panthor_bo_flags flags.
+	 * Will be zero if the buffer comes from a different driver.
+	 */
+	__u32 create_flags;
+
+	/** @pad: Will be zero on return. */
+	__u32 pad;
+};
+
 /**
  * DRM_IOCTL_PANTHOR() - Build a Panthor IOCTL number
  * @__access: Access type. Must be R, W or RW.
@@ -1086,6 +1233,10 @@ enum {
 		DRM_IOCTL_PANTHOR(WR, BO_SET_LABEL, bo_set_label),
 	DRM_IOCTL_PANTHOR_SET_USER_MMIO_OFFSET =
 		DRM_IOCTL_PANTHOR(WR, SET_USER_MMIO_OFFSET, set_user_mmio_offset),
+	DRM_IOCTL_PANTHOR_BO_SYNC =
+		DRM_IOCTL_PANTHOR(WR, BO_SYNC, bo_sync),
+	DRM_IOCTL_PANTHOR_BO_QUERY_INFO =
+		DRM_IOCTL_PANTHOR(WR, BO_QUERY_INFO, bo_query_info),
 };
 
 #if defined(__cplusplus)
diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h
index 47853659a705..726e481574fe 100644
--- a/include/uapi/drm/xe_drm.h
+++ b/include/uapi/drm/xe_drm.h
@@ -106,6 +106,7 @@ extern "C" {
 #define DRM_XE_OBSERVATION		0x0b
 #define DRM_XE_MADVISE			0x0c
 #define DRM_XE_VM_QUERY_MEM_RANGE_ATTRS	0x0d
+#define DRM_XE_EXEC_QUEUE_SET_PROPERTY	0x0e
 
 /* Must be kept compact -- no holes */
 
@@ -123,6 +124,7 @@ extern "C" {
 #define DRM_IOCTL_XE_OBSERVATION		DRM_IOW(DRM_COMMAND_BASE + DRM_XE_OBSERVATION, struct drm_xe_observation_param)
 #define DRM_IOCTL_XE_MADVISE			DRM_IOW(DRM_COMMAND_BASE + DRM_XE_MADVISE, struct drm_xe_madvise)
 #define DRM_IOCTL_XE_VM_QUERY_MEM_RANGE_ATTRS	DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_VM_QUERY_MEM_RANGE_ATTRS, struct drm_xe_vm_query_mem_range_attr)
+#define DRM_IOCTL_XE_EXEC_QUEUE_SET_PROPERTY	DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_SET_PROPERTY, struct drm_xe_exec_queue_set_property)
 
 /**
  * DOC: Xe IOCTL Extensions
@@ -210,8 +212,12 @@ struct drm_xe_ext_set_property {
 	/** @pad: MBZ */
 	__u32 pad;
 
-	/** @value: property value */
-	__u64 value;
+	union {
+		/** @value: property value */
+		__u64 value;
+		/** @ptr: pointer to user value */
+		__u64 ptr;
+	};
 
 	/** @reserved: Reserved */
 	__u64 reserved[2];
@@ -403,6 +409,9 @@ struct drm_xe_query_mem_regions {
  *      has low latency hint support
  *    - %DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR - Flag is set if the
  *      device has CPU address mirroring support
+ *    - %DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT - Flag is set if the
+ *      device supports the userspace hint %DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION.
+ *      This is exposed only on Xe2+.
  *  - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment
  *    required by this device, typically SZ_4K or SZ_64K
  *  - %DRM_XE_QUERY_CONFIG_VA_BITS - Maximum bits of a virtual address
@@ -421,6 +430,7 @@ struct drm_xe_query_config {
 	#define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM	(1 << 0)
 	#define DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY	(1 << 1)
 	#define DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR	(1 << 2)
+	#define DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT (1 << 3)
 #define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT		2
 #define DRM_XE_QUERY_CONFIG_VA_BITS			3
 #define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY	4
@@ -791,6 +801,17 @@ struct drm_xe_device_query {
  *    need to use VRAM for display surfaces, therefore the kernel requires
  *    setting this flag for such objects, otherwise an error is thrown on
  *    small-bar systems.
+ *  - %DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION - Allows userspace to
+ *    hint that compression (CCS) should be disabled for the buffer being
+ *    created. This can avoid unnecessary memory operations and CCS state
+ *    management.
+ *    On pre-Xe2 platforms, this flag is currently rejected as compression
+ *    control is not supported via PAT index. On Xe2+ platforms, compression
+ *    is controlled via PAT entries. If this flag is set, the driver will reject
+ *    any VM bind that requests a PAT index enabling compression for this BO.
+ *    Note: On dGPU platforms, there is currently no change in behavior with
+ *    this flag, but future improvements may leverage it. The current benefit is
+ *    primarily applicable to iGPU platforms.
  *
  * @cpu_caching supports the following values:
  *  - %DRM_XE_GEM_CPU_CACHING_WB - Allocate the pages with write-back
@@ -837,6 +858,7 @@ struct drm_xe_gem_create {
 #define DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING		(1 << 0)
 #define DRM_XE_GEM_CREATE_FLAG_SCANOUT			(1 << 1)
 #define DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM	(1 << 2)
+#define DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION		(1 << 3)
 	/**
 	 * @flags: Flags, currently a mask of memory instances of where BO can
 	 * be placed
@@ -1252,6 +1274,20 @@ struct drm_xe_vm_bind {
  *    Given that going into a power-saving state kills PXP HWDRM sessions,
  *    runtime PM will be blocked while queues of this type are alive.
  *    All PXP queues will be killed if a PXP invalidation event occurs.
+ *  - %DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP - Create a multi-queue group
+ *    or add secondary queues to a multi-queue group.
+ *    If the extension's 'value' field has %DRM_XE_MULTI_GROUP_CREATE flag set,
+ *    then a new multi-queue group is created with this queue as the primary queue
+ *    (Q0). Otherwise, the queue gets added to the multi-queue group whose primary
+ *    queue's exec_queue_id is specified in the lower 32 bits of the 'value' field.
+ *    If the extension's 'value' field has %DRM_XE_MULTI_GROUP_KEEP_ACTIVE flag
+ *    set, then the multi-queue group is kept active after the primary queue is
+ *    destroyed.
+ *    All the other non-relevant bits of extension's 'value' field while adding the
+ *    primary or the secondary queues of the group must be set to 0.
+ *  - %DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY - Set the queue
+ *    priority within the multi-queue group. Current valid priority values are 0–2
+ *    (default is 1), with higher values indicating higher priority.
  *
  * The example below shows how to use @drm_xe_exec_queue_create to create
  * a simple exec_queue (no parallel submission) of class
@@ -1292,6 +1328,11 @@ struct drm_xe_exec_queue_create {
 #define   DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY		0
 #define   DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE		1
 #define   DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE		2
+#define   DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE		3
+#define   DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP		4
+#define     DRM_XE_MULTI_GROUP_CREATE				(1ull << 63)
+#define     DRM_XE_MULTI_GROUP_KEEP_ACTIVE			(1ull << 62)
+#define   DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY	5
 	/** @extensions: Pointer to the first extension struct, if any */
 	__u64 extensions;
 
@@ -1463,6 +1504,7 @@ struct drm_xe_exec {
 	/** @exec_queue_id: Exec queue ID for the batch buffer */
 	__u32 exec_queue_id;
 
+#define DRM_XE_MAX_SYNCS 1024
 	/** @num_syncs: Amount of struct drm_xe_sync in array. */
 	__u32 num_syncs;
 
@@ -1654,6 +1696,9 @@ enum drm_xe_oa_unit_type {
 
 	/** @DRM_XE_OA_UNIT_TYPE_OAM_SAG: OAM_SAG OA unit */
 	DRM_XE_OA_UNIT_TYPE_OAM_SAG,
+
+	/** @DRM_XE_OA_UNIT_TYPE_MERT: MERT OA unit */
+	DRM_XE_OA_UNIT_TYPE_MERT,
 };
 
 /**
@@ -1676,12 +1721,19 @@ struct drm_xe_oa_unit {
 #define DRM_XE_OA_CAPS_OA_BUFFER_SIZE	(1 << 2)
 #define DRM_XE_OA_CAPS_WAIT_NUM_REPORTS	(1 << 3)
 #define DRM_XE_OA_CAPS_OAM		(1 << 4)
+#define DRM_XE_OA_CAPS_OA_UNIT_GT_ID	(1 << 5)
 
 	/** @oa_timestamp_freq: OA timestamp freq */
 	__u64 oa_timestamp_freq;
 
+	/** @gt_id: gt id for this OA unit */
+	__u16 gt_id;
+
+	/** @reserved1: MBZ */
+	__u16 reserved1[3];
+
 	/** @reserved: MBZ */
-	__u64 reserved[4];
+	__u64 reserved[3];
 
 	/** @num_engines: number of engines in @eci array */
 	__u64 num_engines;
@@ -2273,6 +2325,30 @@ struct drm_xe_vm_query_mem_range_attr {
 
 };
 
+/**
+ * struct drm_xe_exec_queue_set_property - exec queue set property
+ *
+ * Sets execution queue properties dynamically.
+ * Currently only %DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY
+ * property can be dynamically set.
+ */
+struct drm_xe_exec_queue_set_property {
+	/** @extensions: Pointer to the first extension struct, if any */
+	__u64 extensions;
+
+	/** @exec_queue_id: Exec queue ID */
+	__u32 exec_queue_id;
+
+	/** @property: property to set */
+	__u32 property;
+
+	/** @value: property value */
+	__u64 value;
+
+	/** @reserved: Reserved */
+	__u64 reserved[2];
+};
+
 #if defined(__cplusplus)
 }
 #endif
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index f8d8513eda27..84ced3ed2d21 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -918,6 +918,16 @@ union bpf_iter_link_info {
  *		Number of bytes read from the stream on success, or -1 if an
  *		error occurred (in which case, *errno* is set appropriately).
  *
+ * BPF_PROG_ASSOC_STRUCT_OPS
+ * 	Description
+ * 		Associate a BPF program with a struct_ops map. The struct_ops
+ * 		map is identified by *map_fd* and the BPF program is
+ * 		identified by *prog_fd*.
+ *
+ * 	Return
+ * 		0 on success or -1 if an error occurred (in which case,
+ * 		*errno* is set appropriately).
+ *
  * NOTES
  *	eBPF objects (maps and programs) can be shared between processes.
  *
@@ -974,6 +984,7 @@ enum bpf_cmd {
 	BPF_PROG_BIND_MAP,
 	BPF_TOKEN_CREATE,
 	BPF_PROG_STREAM_READ_BY_FD,
+	BPF_PROG_ASSOC_STRUCT_OPS,
 	__MAX_BPF_CMD,
 };
 
@@ -1894,6 +1905,12 @@ union bpf_attr {
 		__u32		prog_fd;
 	} prog_stream_read;
 
+	struct {
+		__u32		map_fd;
+		__u32		prog_fd;
+		__u32		flags;
+	} prog_assoc_struct_ops;
+
 } __attribute__((aligned(8)));
 
 /* The description below is an attempt at providing documentation to eBPF
diff --git a/include/uapi/linux/elf.h b/include/uapi/linux/elf.h
index 819ded2d39de..ee30dcd80901 100644
--- a/include/uapi/linux/elf.h
+++ b/include/uapi/linux/elf.h
@@ -545,6 +545,8 @@ typedef struct elf64_shdr {
 #define NT_RISCV_VECTOR	0x901		/* RISC-V vector registers */
 #define NN_RISCV_TAGGED_ADDR_CTRL "LINUX"
 #define NT_RISCV_TAGGED_ADDR_CTRL 0x902	/* RISC-V tagged address control (prctl()) */
+#define NN_RISCV_USER_CFI	"LINUX"
+#define NT_RISCV_USER_CFI	0x903		/* RISC-V shadow stack state */
 #define NN_LOONGARCH_CPUCFG	"LINUX"
 #define NT_LOONGARCH_CPUCFG	0xa00	/* LoongArch CPU config registers */
 #define NN_LOONGARCH_CSR	"LINUX"
diff --git a/include/uapi/linux/energy_model.h b/include/uapi/linux/energy_model.h
index 4ec4c0eabbbb..0bcad967854f 100644
--- a/include/uapi/linux/energy_model.h
+++ b/include/uapi/linux/energy_model.h
@@ -2,6 +2,7 @@
 /* Do not edit directly, auto-generated from: */
 /*	Documentation/netlink/specs/em.yaml */
 /* YNL-GEN uapi header */
+/* To regenerate run: tools/net/ynl/ynl-regen.sh */
 
 #ifndef _UAPI_LINUX_ENERGY_MODEL_H
 #define _UAPI_LINUX_ENERGY_MODEL_H
diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h
index 5d1727a6d040..6e91875c10ba 100644
--- a/include/uapi/linux/kfd_ioctl.h
+++ b/include/uapi/linux/kfd_ioctl.h
@@ -44,9 +44,12 @@
  * - 1.16 - Add contiguous VRAM allocation flag
  * - 1.17 - Add SDMA queue creation with target SDMA engine ID
  * - 1.18 - Rename pad in set_memory_policy_args to misc_process_flag
+ * - 1.19 - Add a new ioctl to craete secondary kfd processes
+ * - 1.20 - Trap handler support for expert scheduling mode available
+ * - 1.21 - Debugger support to subscribe to LDS out-of-address exceptions
  */
 #define KFD_IOCTL_MAJOR_VERSION 1
-#define KFD_IOCTL_MINOR_VERSION 18
+#define KFD_IOCTL_MINOR_VERSION 21
 
 struct kfd_ioctl_get_version_args {
 	__u32 major_version;	/* from KFD */
@@ -945,6 +948,7 @@ enum kfd_dbg_trap_address_watch_mode {
 enum kfd_dbg_trap_flags {
 	KFD_DBG_TRAP_FLAG_SINGLE_MEM_OP = 1,
 	KFD_DBG_TRAP_FLAG_SINGLE_ALU_OP = 2,
+	KFD_DBG_TRAP_FLAG_LDS_OUT_OF_ADDR_RANGE = 4
 };
 
 /* Trap exceptions */
@@ -1671,7 +1675,10 @@ struct kfd_ioctl_dbg_trap_args {
 #define AMDKFD_IOC_DBG_TRAP			\
 		AMDKFD_IOWR(0x26, struct kfd_ioctl_dbg_trap_args)
 
+#define AMDKFD_IOC_CREATE_PROCESS		\
+		AMDKFD_IO(0x27)
+
 #define AMDKFD_COMMAND_START		0x01
-#define AMDKFD_COMMAND_END		0x27
+#define AMDKFD_COMMAND_END		0x28
 
 #endif
diff --git a/include/uapi/linux/kfd_sysfs.h b/include/uapi/linux/kfd_sysfs.h
index 1125fe47959f..0b6ce2f3c887 100644
--- a/include/uapi/linux/kfd_sysfs.h
+++ b/include/uapi/linux/kfd_sysfs.h
@@ -64,7 +64,8 @@
 #define HSA_CAP_RESERVED					0x000f8000
 
 #define HSA_CAP2_PER_SDMA_QUEUE_RESET_SUPPORTED			0x00000001
-#define HSA_CAP2_RESERVED					0xfffffffe
+#define HSA_CAP2_TRAP_DEBUG_LDS_OUT_OF_ADDR_RANGE_SUPPORTED	0x00000002
+#define HSA_CAP2_RESERVED					0xfffffffc
 
 /* debug_prop bits in node properties */
 #define HSA_DBG_WATCH_ADDR_MASK_LO_BIT_MASK     0x0000000f
diff --git a/include/uapi/linux/mount.h b/include/uapi/linux/mount.h
index 5d3f8c9e3a62..18c624405268 100644
--- a/include/uapi/linux/mount.h
+++ b/include/uapi/linux/mount.h
@@ -197,7 +197,10 @@ struct statmount {
  */
 struct mnt_id_req {
 	__u32 size;
-	__u32 mnt_ns_fd;
+	union {
+		__u32 mnt_ns_fd;
+		__u32 mnt_fd;
+	};
 	__u64 mnt_id;
 	__u64 param;
 	__u64 mnt_ns_id;
@@ -232,4 +235,9 @@ struct mnt_id_req {
 #define LSMT_ROOT		0xffffffffffffffff	/* root mount */
 #define LISTMOUNT_REVERSE	(1 << 0) /* List later mounts first */
 
+/*
+ * @flag bits for statmount(2)
+ */
+#define STATMOUNT_BY_FD		0x00000001U	/* want mountinfo for given fd */
+
 #endif /* _UAPI_LINUX_MOUNT_H */
diff --git a/include/uapi/linux/mptcp.h b/include/uapi/linux/mptcp.h
index 04eea6d1d0a9..72a5d030154e 100644
--- a/include/uapi/linux/mptcp.h
+++ b/include/uapi/linux/mptcp.h
@@ -40,6 +40,7 @@
 #define MPTCP_PM_ADDR_FLAG_FULLMESH		_BITUL(3)
 #define MPTCP_PM_ADDR_FLAG_IMPLICIT		_BITUL(4)
 #define MPTCP_PM_ADDR_FLAG_LAMINAR		_BITUL(5)
+#define MPTCP_PM_ADDR_FLAGS_MASK		GENMASK(5, 0)
 
 struct mptcp_info {
 	__u8	mptcpi_subflows;
diff --git a/include/uapi/linux/pr.h b/include/uapi/linux/pr.h
index 847f3051057a..f0ecb1677317 100644
--- a/include/uapi/linux/pr.h
+++ b/include/uapi/linux/pr.h
@@ -79,4 +79,6 @@ struct pr_read_reservation {
 #define IOC_PR_READ_KEYS	_IOWR('p', 206, struct pr_read_keys)
 #define IOC_PR_READ_RESERVATION	_IOR('p', 207, struct pr_read_reservation)
 
+#define PR_KEYS_MAX		(1u << 16)
+
 #endif /* _UAPI_PR_H */
diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h
index 51c4e8c82b1e..9b4afdc85099 100644
--- a/include/uapi/linux/prctl.h
+++ b/include/uapi/linux/prctl.h
@@ -386,4 +386,31 @@ struct prctl_mm_map {
 # define PR_FUTEX_HASH_SET_SLOTS	1
 # define PR_FUTEX_HASH_GET_SLOTS	2
 
+/*
+ * Get the current indirect branch tracking configuration for the current
+ * thread, this will be the value configured via PR_SET_INDIR_BR_LP_STATUS.
+ */
+#define PR_GET_INDIR_BR_LP_STATUS      79
+
+/*
+ * Set the indirect branch tracking configuration. PR_INDIR_BR_LP_ENABLE will
+ * enable cpu feature for user thread, to track all indirect branches and ensure
+ * they land on arch defined landing pad instruction.
+ * x86 - If enabled, an indirect branch must land on `ENDBRANCH` instruction.
+ * arch64 - If enabled, an indirect branch must land on `BTI` instruction.
+ * riscv - If enabled, an indirect branch must land on `lpad` instruction.
+ * PR_INDIR_BR_LP_DISABLE will disable feature for user thread and indirect
+ * branches will no more be tracked by cpu to land on arch defined landing pad
+ * instruction.
+ */
+#define PR_SET_INDIR_BR_LP_STATUS      80
+# define PR_INDIR_BR_LP_ENABLE		   (1UL << 0)
+
+/*
+ * Prevent further changes to the specified indirect branch tracking
+ * configuration.  All bits may be locked via this call, including
+ * undefined bits.
+ */
+#define PR_LOCK_INDIR_BR_LP_STATUS      81
+
 #endif /* _LINUX_PRCTL_H */
diff --git a/include/uapi/linux/sysctl.h b/include/uapi/linux/sysctl.h
index 63d1464cb71c..1c7fe0f4dca4 100644
--- a/include/uapi/linux/sysctl.h
+++ b/include/uapi/linux/sysctl.h
@@ -92,7 +92,6 @@ enum
 	KERN_DOMAINNAME=8,	/* string: domainname */
 
 	KERN_PANIC=15,		/* int: panic timeout */
-	KERN_REALROOTDEV=16,	/* real root device to mount after initrd */
 
 	KERN_SPARC_REBOOT=21,	/* reboot command on Sparc */
 	KERN_CTLALTDEL=22,	/* int: allow ctl-alt-del to reboot */
diff --git a/include/uapi/rdma/irdma-abi.h b/include/uapi/rdma/irdma-abi.h
index f7788d33376b..36f20802bcc8 100644
--- a/include/uapi/rdma/irdma-abi.h
+++ b/include/uapi/rdma/irdma-abi.h
@@ -57,8 +57,8 @@ struct irdma_alloc_ucontext_resp {
 	__u8 rsvd2;
 	__aligned_u64 comp_mask;
 	__u16 min_hw_wq_size;
+	__u8 revd3[2];
 	__u32 max_hw_srq_quanta;
-	__u8 rsvd3[2];
 };
 
 struct irdma_alloc_pd_resp {
diff --git a/include/uapi/rdma/rdma_user_cm.h b/include/uapi/rdma/rdma_user_cm.h
index 5ded174687ee..838f8d460256 100644
--- a/include/uapi/rdma/rdma_user_cm.h
+++ b/include/uapi/rdma/rdma_user_cm.h
@@ -192,6 +192,7 @@ struct rdma_ucm_query_path_resp {
 
 struct rdma_ucm_query_ib_service_resp {
 	__u32 num_service_recs;
+	__u32 reserved;
 	struct ib_user_service_rec recs[];
 };
 
@@ -354,7 +355,7 @@ enum {
 
 #define RDMA_USER_CM_IB_SERVICE_NAME_SIZE 64
 struct rdma_ucm_ib_service {
-	__u64 service_id;
+	__aligned_u64 service_id;
 	__u8  service_name[RDMA_USER_CM_IB_SERVICE_NAME_SIZE];
 	__u32 flags;
 	__u32 reserved;
@@ -362,6 +363,7 @@ struct rdma_ucm_ib_service {
 
 struct rdma_ucm_resolve_ib_service {
 	__u32 id;
+	__u32 reserved;
 	struct rdma_ucm_ib_service ibs;
 };
 
diff --git a/include/ufs/ufshci.h b/include/ufs/ufshci.h
index d36df24242a3..806fdaf52bd9 100644
--- a/include/ufs/ufshci.h
+++ b/include/ufs/ufshci.h
@@ -288,6 +288,7 @@ enum {
 
 /* REG_UFS_MEM_CFG - Global Config Registers 300h */
 #define MCQ_MODE_SELECT	BIT(0)
+#define ESI_ENABLE	BIT(1)
 
 /* CQISy - CQ y Interrupt Status Register  */
 #define UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS	0x1
diff --git a/include/video/edid.h b/include/video/edid.h
index c2b186b1933a..52aabb706032 100644
--- a/include/video/edid.h
+++ b/include/video/edid.h
@@ -4,8 +4,4 @@
 
 #include <uapi/video/edid.h>
 
-#if defined(CONFIG_FIRMWARE_EDID)
-extern struct edid_info edid_info;
-#endif
-
 #endif /* __linux_video_edid_h__ */
diff --git a/init/do_mounts.c b/init/do_mounts.c
index defbbf1d55f7..9c8a547075a7 100644
--- a/init/do_mounts.c
+++ b/init/do_mounts.c
@@ -34,13 +34,6 @@ static int root_wait;
 
 dev_t ROOT_DEV;
 
-static int __init load_ramdisk(char *str)
-{
-	pr_warn("ignoring the deprecated load_ramdisk= option\n");
-	return 1;
-}
-__setup("load_ramdisk=", load_ramdisk);
-
 static int __init readonly(char *str)
 {
 	if (*str)
@@ -484,13 +477,11 @@ void __init prepare_namespace(void)
 	if (saved_root_name[0])
 		ROOT_DEV = parse_root_device(saved_root_name);
 
-	if (initrd_load(saved_root_name))
-		goto out;
+	initrd_load();
 
 	if (root_wait)
 		wait_for_root(saved_root_name);
 	mount_root(saved_root_name);
-out:
 	devtmpfs_mount();
 	init_mount(".", "/", NULL, MS_MOVE, NULL);
 	init_chroot(".");
diff --git a/init/do_mounts.h b/init/do_mounts.h
index 6069ea3eb80d..a386ee5314c9 100644
--- a/init/do_mounts.h
+++ b/init/do_mounts.h
@@ -23,25 +23,15 @@ static inline __init int create_dev(char *name, dev_t dev)
 }
 
 #ifdef CONFIG_BLK_DEV_RAM
-
-int __init rd_load_disk(int n);
-int __init rd_load_image(char *from);
-
+int __init rd_load_image(void);
 #else
-
-static inline int rd_load_disk(int n) { return 0; }
-static inline int rd_load_image(char *from) { return 0; }
-
+static inline int rd_load_image(void) { return 0; }
 #endif
 
 #ifdef CONFIG_BLK_DEV_INITRD
-bool __init initrd_load(char *root_device_name);
+void __init initrd_load(void);
 #else
-static inline bool initrd_load(char *root_device_name)
-{
-	return false;
-	}
-
+static inline void initrd_load(void) { }
 #endif
 
 /* Ensure that async file closing finished to prevent spurious errors. */
diff --git a/init/do_mounts_initrd.c b/init/do_mounts_initrd.c
index f6867bad0d78..892e69ab41c4 100644
--- a/init/do_mounts_initrd.c
+++ b/init/do_mounts_initrd.c
@@ -2,45 +2,20 @@
 #include <linux/unistd.h>
 #include <linux/kernel.h>
 #include <linux/fs.h>
-#include <linux/minix_fs.h>
-#include <linux/romfs_fs.h>
 #include <linux/initrd.h>
-#include <linux/sched.h>
-#include <linux/freezer.h>
-#include <linux/kmod.h>
-#include <uapi/linux/mount.h>
 
 #include "do_mounts.h"
 
 unsigned long initrd_start, initrd_end;
 int initrd_below_start_ok;
-static unsigned int real_root_dev;	/* do_proc_dointvec cannot handle kdev_t */
 static int __initdata mount_initrd = 1;
 
 phys_addr_t phys_initrd_start __initdata;
 unsigned long phys_initrd_size __initdata;
 
-#ifdef CONFIG_SYSCTL
-static const struct ctl_table kern_do_mounts_initrd_table[] = {
-	{
-		.procname       = "real-root-dev",
-		.data           = &real_root_dev,
-		.maxlen         = sizeof(int),
-		.mode           = 0644,
-		.proc_handler   = proc_dointvec,
-	},
-};
-
-static __init int kernel_do_mounts_initrd_sysctls_init(void)
-{
-	register_sysctl_init("kernel", kern_do_mounts_initrd_table);
-	return 0;
-}
-late_initcall(kernel_do_mounts_initrd_sysctls_init);
-#endif /* CONFIG_SYSCTL */
-
 static int __init no_initrd(char *str)
 {
+	pr_warn("noinitrd option is deprecated and will be removed soon\n");
 	mount_initrd = 0;
 	return 1;
 }
@@ -70,85 +45,19 @@ static int __init early_initrd(char *p)
 }
 early_param("initrd", early_initrd);
 
-static int __init init_linuxrc(struct subprocess_info *info, struct cred *new)
-{
-	ksys_unshare(CLONE_FS | CLONE_FILES);
-	console_on_rootfs();
-	/* move initrd over / and chdir/chroot in initrd root */
-	init_chdir("/root");
-	init_mount(".", "/", NULL, MS_MOVE, NULL);
-	init_chroot(".");
-	ksys_setsid();
-	return 0;
-}
-
-static void __init handle_initrd(char *root_device_name)
-{
-	struct subprocess_info *info;
-	static char *argv[] = { "linuxrc", NULL, };
-	extern char *envp_init[];
-	int error;
-
-	pr_warn("using deprecated initrd support, will be removed soon.\n");
-
-	real_root_dev = new_encode_dev(ROOT_DEV);
-	create_dev("/dev/root.old", Root_RAM0);
-	/* mount initrd on rootfs' /root */
-	mount_root_generic("/dev/root.old", root_device_name,
-			   root_mountflags & ~MS_RDONLY);
-	init_mkdir("/old", 0700);
-	init_chdir("/old");
-
-	info = call_usermodehelper_setup("/linuxrc", argv, envp_init,
-					 GFP_KERNEL, init_linuxrc, NULL, NULL);
-	if (!info)
-		return;
-	call_usermodehelper_exec(info, UMH_WAIT_PROC|UMH_FREEZABLE);
-
-	/* move initrd to rootfs' /old */
-	init_mount("..", ".", NULL, MS_MOVE, NULL);
-	/* switch root and cwd back to / of rootfs */
-	init_chroot("..");
-
-	if (new_decode_dev(real_root_dev) == Root_RAM0) {
-		init_chdir("/old");
-		return;
-	}
-
-	init_chdir("/");
-	ROOT_DEV = new_decode_dev(real_root_dev);
-	mount_root(root_device_name);
-
-	printk(KERN_NOTICE "Trying to move old root to /initrd ... ");
-	error = init_mount("/old", "/root/initrd", NULL, MS_MOVE, NULL);
-	if (!error)
-		printk("okay\n");
-	else {
-		if (error == -ENOENT)
-			printk("/initrd does not exist. Ignored.\n");
-		else
-			printk("failed\n");
-		printk(KERN_NOTICE "Unmounting old root\n");
-		init_umount("/old", MNT_DETACH);
-	}
-}
-
-bool __init initrd_load(char *root_device_name)
+void __init initrd_load(void)
 {
 	if (mount_initrd) {
 		create_dev("/dev/ram", Root_RAM0);
 		/*
-		 * Load the initrd data into /dev/ram0. Execute it as initrd
-		 * unless /dev/ram0 is supposed to be our actual root device,
-		 * in that case the ram disk is just set up here, and gets
-		 * mounted in the normal path.
+		 * Load the initrd data into /dev/ram0.
 		 */
-		if (rd_load_image("/initrd.image") && ROOT_DEV != Root_RAM0) {
-			init_unlink("/initrd.image");
-			handle_initrd(root_device_name);
-			return true;
+		if (rd_load_image()) {
+			pr_warn("using deprecated initrd support, will be removed in January 2027; "
+				"use initramfs instead or (as a last resort) /sys/firmware/initrd; "
+				"see section \"Workaround\" in "
+				"https://lore.kernel.org/lkml/20251010094047.3111495-1-safinaskar@gmail.com\n");
 		}
 	}
 	init_unlink("/initrd.image");
-	return false;
 }
diff --git a/init/do_mounts_rd.c b/init/do_mounts_rd.c
index eddbe5cb0413..48bfab2fc62f 100644
--- a/init/do_mounts_rd.c
+++ b/init/do_mounts_rd.c
@@ -18,17 +18,11 @@
 static struct file *in_file, *out_file;
 static loff_t in_pos, out_pos;
 
-static int __init prompt_ramdisk(char *str)
-{
-	pr_warn("ignoring the deprecated prompt_ramdisk= option\n");
-	return 1;
-}
-__setup("prompt_ramdisk=", prompt_ramdisk);
-
 int __initdata rd_image_start;		/* starting block # of image */
 
 static int __init ramdisk_start_setup(char *str)
 {
+	pr_warn("ramdisk_start= option is deprecated and will be removed soon\n");
 	return kstrtoint(str, 0, &rd_image_start) == 0;
 }
 __setup("ramdisk_start=", ramdisk_start_setup);
@@ -183,7 +177,7 @@ static unsigned long nr_blocks(struct file *file)
 	return i_size_read(inode) >> 10;
 }
 
-int __init rd_load_image(char *from)
+int __init rd_load_image(void)
 {
 	int res = 0;
 	unsigned long rd_blocks, devblocks, nr_disks;
@@ -197,7 +191,7 @@ int __init rd_load_image(char *from)
 	if (IS_ERR(out_file))
 		goto out;
 
-	in_file = filp_open(from, O_RDONLY, 0);
+	in_file = filp_open("/initrd.image", O_RDONLY, 0);
 	if (IS_ERR(in_file))
 		goto noclose_input;
 
@@ -226,10 +220,7 @@ int __init rd_load_image(char *from)
 	/*
 	 * OK, time to copy in the data
 	 */
-	if (strcmp(from, "/initrd.image") == 0)
-		devblocks = nblocks;
-	else
-		devblocks = nr_blocks(in_file);
+	devblocks = nblocks;
 
 	if (devblocks == 0) {
 		printk(KERN_ERR "RAMDISK: could not determine device size\n");
@@ -273,13 +264,6 @@ int __init rd_load_image(char *from)
 	return res;
 }
 
-int __init rd_load_disk(int n)
-{
-	create_dev("/dev/root", ROOT_DEV);
-	create_dev("/dev/ram", MKDEV(RAMDISK_MAJOR, n));
-	return rd_load_image("/dev/root");
-}
-
 static int exit_code;
 static int decompress_error;
 
diff --git a/init/main.c b/init/main.c
index b84818ad9685..85bcc7c1ccd0 100644
--- a/init/main.c
+++ b/init/main.c
@@ -104,6 +104,7 @@
 #include <linux/pidfs.h>
 #include <linux/ptdump.h>
 #include <linux/time_namespace.h>
+#include <linux/percpu_counter_tree.h>
 #include <net/net_namespace.h>
 
 #include <asm/io.h>
@@ -1063,6 +1064,7 @@ void start_kernel(void)
 	vfs_caches_init_early();
 	sort_main_extable();
 	trap_init();
+	percpu_counter_tree_subsystem_init();
 	mm_core_init();
 	maple_tree_init();
 	poking_init();
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index a63474b331bf..41c89f5c616d 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -1059,6 +1059,7 @@ static int io_import_kbuf(int ddir, struct iov_iter *iter,
 	if (count < imu->len) {
 		const struct bio_vec *bvec = iter->bvec;
 
+		len += iter->iov_offset;
 		while (len > bvec->bv_len) {
 			len -= bvec->bv_len;
 			bvec++;
diff --git a/kernel/audit.c b/kernel/audit.c
index 26a332ffb1b8..592d927e70f9 100644
--- a/kernel/audit.c
+++ b/kernel/audit.c
@@ -32,6 +32,7 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/file.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/types.h>
 #include <linux/atomic.h>
@@ -58,6 +59,9 @@
 #include <linux/freezer.h>
 #include <linux/pid_namespace.h>
 #include <net/netns/generic.h>
+#include <net/ip.h>
+#include <net/ipv6.h>
+#include <linux/sctp.h>
 
 #include "audit.h"
 
@@ -2488,6 +2492,162 @@ void audit_log_path_denied(int type, const char *operation)
 	audit_log_end(ab);
 }
 
+int audit_log_nf_skb(struct audit_buffer *ab,
+		     const struct sk_buff *skb, u8 nfproto)
+{
+	/* find the IP protocol in the case of NFPROTO_BRIDGE */
+	if (nfproto == NFPROTO_BRIDGE) {
+		switch (eth_hdr(skb)->h_proto) {
+		case htons(ETH_P_IP):
+			nfproto = NFPROTO_IPV4;
+			break;
+		case htons(ETH_P_IPV6):
+			nfproto = NFPROTO_IPV6;
+			break;
+		default:
+			goto unknown_proto;
+		}
+	}
+
+	switch (nfproto) {
+	case NFPROTO_IPV4: {
+		struct iphdr iph;
+		const struct iphdr *ih;
+
+		ih = skb_header_pointer(skb, skb_network_offset(skb),
+					sizeof(iph), &iph);
+		if (!ih)
+			return -ENOMEM;
+
+		switch (ih->protocol) {
+		case IPPROTO_TCP: {
+			struct tcphdr _tcph;
+			const struct tcphdr *th;
+
+			th = skb_header_pointer(skb, skb_transport_offset(skb),
+						sizeof(_tcph), &_tcph);
+			if (!th)
+				return -ENOMEM;
+
+			audit_log_format(ab, " saddr=%pI4 daddr=%pI4 proto=%hhu sport=%hu dport=%hu",
+					 &ih->saddr, &ih->daddr, ih->protocol,
+					 ntohs(th->source), ntohs(th->dest));
+			break;
+		}
+		case IPPROTO_UDP:
+		case IPPROTO_UDPLITE: {
+			struct udphdr _udph;
+			const struct udphdr *uh;
+
+			uh = skb_header_pointer(skb, skb_transport_offset(skb),
+						sizeof(_udph), &_udph);
+			if (!uh)
+				return -ENOMEM;
+
+			audit_log_format(ab, " saddr=%pI4 daddr=%pI4 proto=%hhu sport=%hu dport=%hu",
+					 &ih->saddr, &ih->daddr, ih->protocol,
+					 ntohs(uh->source), ntohs(uh->dest));
+			break;
+		}
+		case IPPROTO_SCTP: {
+			struct sctphdr _sctph;
+			const struct sctphdr *sh;
+
+			sh = skb_header_pointer(skb, skb_transport_offset(skb),
+						sizeof(_sctph), &_sctph);
+			if (!sh)
+				return -ENOMEM;
+
+			audit_log_format(ab, " saddr=%pI4 daddr=%pI4 proto=%hhu sport=%hu dport=%hu",
+					 &ih->saddr, &ih->daddr, ih->protocol,
+					 ntohs(sh->source), ntohs(sh->dest));
+			break;
+		}
+		default:
+			audit_log_format(ab, " saddr=%pI4 daddr=%pI4 proto=%hhu",
+					 &ih->saddr, &ih->daddr, ih->protocol);
+		}
+
+		break;
+	}
+	case NFPROTO_IPV6: {
+		struct ipv6hdr iph;
+		const struct ipv6hdr *ih;
+		u8 nexthdr;
+		__be16 frag_off;
+
+		ih = skb_header_pointer(skb, skb_network_offset(skb),
+					sizeof(iph), &iph);
+		if (!ih)
+			return -ENOMEM;
+
+		nexthdr = ih->nexthdr;
+		ipv6_skip_exthdr(skb, skb_network_offset(skb) + sizeof(iph),
+				 &nexthdr, &frag_off);
+
+		switch (nexthdr) {
+		case IPPROTO_TCP: {
+			struct tcphdr _tcph;
+			const struct tcphdr *th;
+
+			th = skb_header_pointer(skb, skb_transport_offset(skb),
+						sizeof(_tcph), &_tcph);
+			if (!th)
+				return -ENOMEM;
+
+			audit_log_format(ab, " saddr=%pI6c daddr=%pI6c proto=%hhu sport=%hu dport=%hu",
+					 &ih->saddr, &ih->daddr, nexthdr,
+					 ntohs(th->source), ntohs(th->dest));
+			break;
+		}
+		case IPPROTO_UDP:
+		case IPPROTO_UDPLITE: {
+			struct udphdr _udph;
+			const struct udphdr *uh;
+
+			uh = skb_header_pointer(skb, skb_transport_offset(skb),
+						sizeof(_udph), &_udph);
+			if (!uh)
+				return -ENOMEM;
+
+			audit_log_format(ab, " saddr=%pI6c daddr=%pI6c proto=%hhu sport=%hu dport=%hu",
+					 &ih->saddr, &ih->daddr, nexthdr,
+					 ntohs(uh->source), ntohs(uh->dest));
+			break;
+		}
+		case IPPROTO_SCTP: {
+			struct sctphdr _sctph;
+			const struct sctphdr *sh;
+
+			sh = skb_header_pointer(skb, skb_transport_offset(skb),
+						sizeof(_sctph), &_sctph);
+			if (!sh)
+				return -ENOMEM;
+
+			audit_log_format(ab, " saddr=%pI6c daddr=%pI6c proto=%hhu sport=%hu dport=%hu",
+					 &ih->saddr, &ih->daddr, nexthdr,
+					 ntohs(sh->source), ntohs(sh->dest));
+			break;
+		}
+		default:
+			audit_log_format(ab, " saddr=%pI6c daddr=%pI6c proto=%hhu",
+					 &ih->saddr, &ih->daddr, nexthdr);
+		}
+
+		break;
+	}
+	default:
+		goto unknown_proto;
+	}
+
+	return 0;
+
+unknown_proto:
+	audit_log_format(ab, " saddr=? daddr=? proto=?");
+	return -EPFNOSUPPORT;
+}
+EXPORT_SYMBOL(audit_log_nf_skb);
+
 /* global counter which is incremented every time something logs in */
 static atomic_t session_id = ATOMIC_INIT(0);
 
diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c
index eec60b57bd3d..4b58d56ecab1 100644
--- a/kernel/bpf/bpf_iter.c
+++ b/kernel/bpf/bpf_iter.c
@@ -86,7 +86,7 @@ static bool bpf_iter_support_resched(struct seq_file *seq)
 
 /* bpf_seq_read, a customized and simpler version for bpf iterator.
  * The following are differences from seq_read():
- *  . fixed buffer size (PAGE_SIZE)
+ *  . fixed buffer size (PAGE_SIZE << 3)
  *  . assuming NULL ->llseek()
  *  . stop() may call bpf program, handling potential overflow there
  */
diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c
index 278490683d28..c43346cb3d76 100644
--- a/kernel/bpf/bpf_struct_ops.c
+++ b/kernel/bpf/bpf_struct_ops.c
@@ -533,6 +533,17 @@ static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map)
 	}
 }
 
+static void bpf_struct_ops_map_dissoc_progs(struct bpf_struct_ops_map *st_map)
+{
+	u32 i;
+
+	for (i = 0; i < st_map->funcs_cnt; i++) {
+		if (!st_map->links[i])
+			break;
+		bpf_prog_disassoc_struct_ops(st_map->links[i]->prog);
+	}
+}
+
 static void bpf_struct_ops_map_free_image(struct bpf_struct_ops_map *st_map)
 {
 	int i;
@@ -801,6 +812,9 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
 			goto reset_unlock;
 		}
 
+		/* Poison pointer on error instead of return for backward compatibility */
+		bpf_prog_assoc_struct_ops(prog, &st_map->map);
+
 		link = kzalloc(sizeof(*link), GFP_USER);
 		if (!link) {
 			bpf_prog_put(prog);
@@ -980,6 +994,8 @@ static void bpf_struct_ops_map_free(struct bpf_map *map)
 	if (btf_is_module(st_map->btf))
 		module_put(st_map->st_ops_desc->st_ops->owner);
 
+	bpf_struct_ops_map_dissoc_progs(st_map);
+
 	bpf_struct_ops_map_del_ksyms(st_map);
 
 	/* The struct_ops's function may switch to another struct_ops.
@@ -1396,6 +1412,78 @@ int bpf_struct_ops_link_create(union bpf_attr *attr)
 	return err;
 }
 
+int bpf_prog_assoc_struct_ops(struct bpf_prog *prog, struct bpf_map *map)
+{
+	struct bpf_map *st_ops_assoc;
+
+	guard(mutex)(&prog->aux->st_ops_assoc_mutex);
+
+	st_ops_assoc = rcu_dereference_protected(prog->aux->st_ops_assoc,
+						 lockdep_is_held(&prog->aux->st_ops_assoc_mutex));
+	if (st_ops_assoc && st_ops_assoc == map)
+		return 0;
+
+	if (st_ops_assoc) {
+		if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
+			return -EBUSY;
+
+		rcu_assign_pointer(prog->aux->st_ops_assoc, BPF_PTR_POISON);
+	} else {
+		/*
+		 * struct_ops map does not track associated non-struct_ops programs.
+		 * Bump the refcount to make sure st_ops_assoc is always valid.
+		 */
+		if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
+			bpf_map_inc(map);
+
+		rcu_assign_pointer(prog->aux->st_ops_assoc, map);
+	}
+
+	return 0;
+}
+
+void bpf_prog_disassoc_struct_ops(struct bpf_prog *prog)
+{
+	struct bpf_map *st_ops_assoc;
+
+	guard(mutex)(&prog->aux->st_ops_assoc_mutex);
+
+	st_ops_assoc = rcu_dereference_protected(prog->aux->st_ops_assoc,
+						 lockdep_is_held(&prog->aux->st_ops_assoc_mutex));
+	if (!st_ops_assoc || st_ops_assoc == BPF_PTR_POISON)
+		return;
+
+	if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
+		bpf_map_put(st_ops_assoc);
+
+	RCU_INIT_POINTER(prog->aux->st_ops_assoc, NULL);
+}
+
+/*
+ * Get a reference to the struct_ops struct (i.e., kdata) associated with a
+ * program. Should only be called in BPF program context (e.g., in a kfunc).
+ *
+ * If the returned pointer is not NULL, it must points to a valid struct_ops.
+ * The struct_ops map is not guaranteed to be initialized nor attached.
+ * Kernel struct_ops implementers are responsible for tracking and checking
+ * the state of the struct_ops if the use case requires an initialized or
+ * attached struct_ops.
+ */
+void *bpf_prog_get_assoc_struct_ops(const struct bpf_prog_aux *aux)
+{
+	struct bpf_struct_ops_map *st_map;
+	struct bpf_map *st_ops_assoc;
+
+	st_ops_assoc = rcu_dereference_check(aux->st_ops_assoc, bpf_rcu_lock_held());
+	if (!st_ops_assoc || st_ops_assoc == BPF_PTR_POISON)
+		return NULL;
+
+	st_map = (struct bpf_struct_ops_map *)st_ops_assoc;
+
+	return &st_map->kvalue.data;
+}
+EXPORT_SYMBOL_GPL(bpf_prog_get_assoc_struct_ops);
+
 void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map)
 {
 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index c8ae6ab31651..58bc54807d11 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -25,6 +25,7 @@
 #include <linux/prandom.h>
 #include <linux/bpf.h>
 #include <linux/btf.h>
+#include <linux/hex.h>
 #include <linux/objtool.h>
 #include <linux/overflow.h>
 #include <linux/rbtree_latch.h>
@@ -136,6 +137,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag
 	mutex_init(&fp->aux->used_maps_mutex);
 	mutex_init(&fp->aux->ext_mutex);
 	mutex_init(&fp->aux->dst_mutex);
+	mutex_init(&fp->aux->st_ops_assoc_mutex);
 
 #ifdef CONFIG_BPF_SYSCALL
 	bpf_prog_stream_init(fp);
@@ -286,6 +288,7 @@ void __bpf_prog_free(struct bpf_prog *fp)
 	if (fp->aux) {
 		mutex_destroy(&fp->aux->used_maps_mutex);
 		mutex_destroy(&fp->aux->dst_mutex);
+		mutex_destroy(&fp->aux->st_ops_assoc_mutex);
 		kfree(fp->aux->poke_tab);
 		kfree(fp->aux);
 	}
@@ -713,8 +716,8 @@ static struct bpf_ksym *bpf_ksym_find(unsigned long addr)
 	return n ? container_of(n, struct bpf_ksym, tnode) : NULL;
 }
 
-int __bpf_address_lookup(unsigned long addr, unsigned long *size,
-				 unsigned long *off, char *sym)
+int bpf_address_lookup(unsigned long addr, unsigned long *size,
+		       unsigned long *off, char *sym)
 {
 	struct bpf_ksym *ksym;
 	int ret = 0;
@@ -760,6 +763,22 @@ struct bpf_prog *bpf_prog_ksym_find(unsigned long addr)
 	       NULL;
 }
 
+bool bpf_has_frame_pointer(unsigned long ip)
+{
+	struct bpf_ksym *ksym;
+	unsigned long offset;
+
+	guard(rcu)();
+
+	ksym = bpf_ksym_find(ip);
+	if (!ksym || !ksym->fp_start || !ksym->fp_end)
+		return false;
+
+	offset = ip - ksym->start;
+
+	return offset >= ksym->fp_start && offset < ksym->fp_end;
+}
+
 const struct exception_table_entry *search_bpf_extables(unsigned long addr)
 {
 	const struct exception_table_entry *e = NULL;
@@ -2896,6 +2915,7 @@ static void bpf_prog_free_deferred(struct work_struct *work)
 #endif
 	bpf_free_used_maps(aux);
 	bpf_free_used_btfs(aux);
+	bpf_prog_disassoc_struct_ops(aux->prog);
 	if (bpf_prog_is_dev_bound(aux))
 		bpf_prog_dev_bound_destroy(aux->prog);
 #ifdef CONFIG_PERF_EVENTS
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index 703e5df1f4ef..04171fbc39cb 100644
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -430,7 +430,7 @@ static struct bpf_cpu_map_entry *
 __cpu_map_entry_alloc(struct bpf_map *map, struct bpf_cpumap_val *value,
 		      u32 cpu)
 {
-	int numa, err, i, fd = value->bpf_prog.fd;
+	int numa, err = -ENOMEM, i, fd = value->bpf_prog.fd;
 	gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
 	struct bpf_cpu_map_entry *rcpu;
 	struct xdp_bulk_queue *bq;
@@ -440,7 +440,7 @@ __cpu_map_entry_alloc(struct bpf_map *map, struct bpf_cpumap_val *value,
 
 	rcpu = bpf_map_kmalloc_node(map, sizeof(*rcpu), gfp | __GFP_ZERO, numa);
 	if (!rcpu)
-		return NULL;
+		return ERR_PTR(err);
 
 	/* Alloc percpu bulkq */
 	rcpu->bulkq = bpf_map_alloc_percpu(map, sizeof(*rcpu->bulkq),
@@ -468,16 +468,21 @@ __cpu_map_entry_alloc(struct bpf_map *map, struct bpf_cpumap_val *value,
 	rcpu->value.qsize  = value->qsize;
 	gro_init(&rcpu->gro);
 
-	if (fd > 0 && __cpu_map_load_bpf_program(rcpu, map, fd))
-		goto free_ptr_ring;
+	if (fd > 0) {
+		err = __cpu_map_load_bpf_program(rcpu, map, fd);
+		if (err)
+			goto free_ptr_ring;
+	}
 
 	/* Setup kthread */
 	init_completion(&rcpu->kthread_running);
 	rcpu->kthread = kthread_create_on_node(cpu_map_kthread_run, rcpu, numa,
 					       "cpumap/%d/map:%d", cpu,
 					       map->id);
-	if (IS_ERR(rcpu->kthread))
+	if (IS_ERR(rcpu->kthread)) {
+		err = PTR_ERR(rcpu->kthread);
 		goto free_prog;
+	}
 
 	/* Make sure kthread runs on a single CPU */
 	kthread_bind(rcpu->kthread, cpu);
@@ -503,7 +508,7 @@ __cpu_map_entry_alloc(struct bpf_map *map, struct bpf_cpumap_val *value,
 	free_percpu(rcpu->bulkq);
 free_rcu:
 	kfree(rcpu);
-	return NULL;
+	return ERR_PTR(err);
 }
 
 static void __cpu_map_entry_free(struct work_struct *work)
@@ -596,8 +601,8 @@ static long cpu_map_update_elem(struct bpf_map *map, void *key, void *value,
 	} else {
 		/* Updating qsize cause re-allocation of bpf_cpu_map_entry */
 		rcpu = __cpu_map_entry_alloc(map, &cpumap_value, key_cpu);
-		if (!rcpu)
-			return -ENOMEM;
+		if (IS_ERR(rcpu))
+			return PTR_ERR(rcpu);
 	}
 	rcu_read_lock();
 	__cpu_map_entry_replace(cmap, key_cpu, rcpu);
diff --git a/kernel/bpf/dmabuf_iter.c b/kernel/bpf/dmabuf_iter.c
index 4dd7ef7c145c..cd500248abd9 100644
--- a/kernel/bpf/dmabuf_iter.c
+++ b/kernel/bpf/dmabuf_iter.c
@@ -6,10 +6,33 @@
 #include <linux/kernel.h>
 #include <linux/seq_file.h>
 
+struct dmabuf_iter_priv {
+	/*
+	 * If this pointer is non-NULL, the buffer's refcount is elevated to
+	 * prevent destruction between stop/start. If reading is not resumed and
+	 * start is never called again, then dmabuf_iter_seq_fini drops the
+	 * reference when the iterator is released.
+	 */
+	struct dma_buf *dmabuf;
+};
+
 static void *dmabuf_iter_seq_start(struct seq_file *seq, loff_t *pos)
 {
-	if (*pos)
-		return NULL;
+	struct dmabuf_iter_priv *p = seq->private;
+
+	if (*pos) {
+		struct dma_buf *dmabuf = p->dmabuf;
+
+		if (!dmabuf)
+			return NULL;
+
+		/*
+		 * Always resume from where we stopped, regardless of the value
+		 * of pos.
+		 */
+		p->dmabuf = NULL;
+		return dmabuf;
+	}
 
 	return dma_buf_iter_begin();
 }
@@ -54,8 +77,11 @@ static void dmabuf_iter_seq_stop(struct seq_file *seq, void *v)
 {
 	struct dma_buf *dmabuf = v;
 
-	if (dmabuf)
-		dma_buf_put(dmabuf);
+	if (dmabuf) {
+		struct dmabuf_iter_priv *p = seq->private;
+
+		p->dmabuf = dmabuf;
+	}
 }
 
 static const struct seq_operations dmabuf_iter_seq_ops = {
@@ -71,11 +97,27 @@ static void bpf_iter_dmabuf_show_fdinfo(const struct bpf_iter_aux_info *aux,
 	seq_puts(seq, "dmabuf iter\n");
 }
 
+static int dmabuf_iter_seq_init(void *priv, struct bpf_iter_aux_info *aux)
+{
+	struct dmabuf_iter_priv *p = (struct dmabuf_iter_priv *)priv;
+
+	p->dmabuf = NULL;
+	return 0;
+}
+
+static void dmabuf_iter_seq_fini(void *priv)
+{
+	struct dmabuf_iter_priv *p = (struct dmabuf_iter_priv *)priv;
+
+	if (p->dmabuf)
+		dma_buf_put(p->dmabuf);
+}
+
 static const struct bpf_iter_seq_info dmabuf_iter_seq_info = {
 	.seq_ops		= &dmabuf_iter_seq_ops,
-	.init_seq_private	= NULL,
-	.fini_seq_private	= NULL,
-	.seq_priv_size		= 0,
+	.init_seq_private	= dmabuf_iter_seq_init,
+	.fini_seq_private	= dmabuf_iter_seq_fini,
+	.seq_priv_size		= sizeof(struct dmabuf_iter_priv),
 };
 
 static struct bpf_iter_reg bpf_dmabuf_reg_info = {
diff --git a/kernel/bpf/rqspinlock.c b/kernel/bpf/rqspinlock.c
index f7d0c8d4644e..8d892fb099ac 100644
--- a/kernel/bpf/rqspinlock.c
+++ b/kernel/bpf/rqspinlock.c
@@ -694,7 +694,6 @@ __bpf_kfunc int bpf_res_spin_lock(struct bpf_res_spin_lock *lock)
 	int ret;
 
 	BUILD_BUG_ON(sizeof(rqspinlock_t) != sizeof(struct bpf_res_spin_lock));
-	BUILD_BUG_ON(__alignof__(rqspinlock_t) != __alignof__(struct bpf_res_spin_lock));
 
 	preempt_disable();
 	ret = res_spin_lock((rqspinlock_t *)lock);
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 4ff82144f885..034d178c04a6 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -9,6 +9,7 @@
 #include <linux/bpf_verifier.h>
 #include <linux/bsearch.h>
 #include <linux/btf.h>
+#include <linux/hex.h>
 #include <linux/syscalls.h>
 #include <linux/slab.h>
 #include <linux/sched/signal.h>
@@ -6122,6 +6123,49 @@ static int prog_stream_read(union bpf_attr *attr)
 	return ret;
 }
 
+#define BPF_PROG_ASSOC_STRUCT_OPS_LAST_FIELD prog_assoc_struct_ops.prog_fd
+
+static int prog_assoc_struct_ops(union bpf_attr *attr)
+{
+	struct bpf_prog *prog;
+	struct bpf_map *map;
+	int ret;
+
+	if (CHECK_ATTR(BPF_PROG_ASSOC_STRUCT_OPS))
+		return -EINVAL;
+
+	if (attr->prog_assoc_struct_ops.flags)
+		return -EINVAL;
+
+	prog = bpf_prog_get(attr->prog_assoc_struct_ops.prog_fd);
+	if (IS_ERR(prog))
+		return PTR_ERR(prog);
+
+	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) {
+		ret = -EINVAL;
+		goto put_prog;
+	}
+
+	map = bpf_map_get(attr->prog_assoc_struct_ops.map_fd);
+	if (IS_ERR(map)) {
+		ret = PTR_ERR(map);
+		goto put_prog;
+	}
+
+	if (map->map_type != BPF_MAP_TYPE_STRUCT_OPS) {
+		ret = -EINVAL;
+		goto put_map;
+	}
+
+	ret = bpf_prog_assoc_struct_ops(prog, map);
+
+put_map:
+	bpf_map_put(map);
+put_prog:
+	bpf_prog_put(prog);
+	return ret;
+}
+
 static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size)
 {
 	union bpf_attr attr;
@@ -6261,6 +6305,9 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size)
 	case BPF_PROG_STREAM_READ_BY_FD:
 		err = prog_stream_read(&attr);
 		break;
+	case BPF_PROG_ASSOC_STRUCT_OPS:
+		err = prog_assoc_struct_ops(&attr);
+		break;
 	default:
 		err = -EINVAL;
 		break;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index f0ca69f888fa..d6b8a77fbe3b 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -15300,21 +15300,17 @@ static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg,
 				   u64 umin_val, u64 umax_val)
 {
 	/* Special case <<32 because it is a common compiler pattern to sign
-	 * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are
-	 * positive we know this shift will also be positive so we can track
-	 * bounds correctly. Otherwise we lose all sign bit information except
-	 * what we can pick up from var_off. Perhaps we can generalize this
-	 * later to shifts of any length.
+	 * extend subreg by doing <<32 s>>32. smin/smax assignments are correct
+	 * because s32 bounds don't flip sign when shifting to the left by
+	 * 32bits.
 	 */
-	if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0)
+	if (umin_val == 32 && umax_val == 32) {
 		dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32;
-	else
-		dst_reg->smax_value = S64_MAX;
-
-	if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0)
 		dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32;
-	else
+	} else {
+		dst_reg->smax_value = S64_MAX;
 		dst_reg->smin_value = S64_MIN;
+	}
 
 	/* If we might shift our top bit out, then we know nothing */
 	if (dst_reg->umax_value > 1ULL << (63 - umax_val)) {
@@ -21136,11 +21132,6 @@ static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)
 			} else {
 				u32 off = insn[1].imm;
 
-				if (off >= BPF_MAX_VAR_OFF) {
-					verbose(env, "direct value offset of %u is not allowed\n", off);
-					return -EINVAL;
-				}
-
 				if (!map->ops->map_direct_value_addr) {
 					verbose(env, "no direct value access support for this map type\n");
 					return -EINVAL;
@@ -22493,8 +22484,7 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 
 	if (!bpf_jit_supports_far_kfunc_call())
 		insn->imm = BPF_CALL_IMM(desc->addr);
-	if (insn->off)
-		return 0;
+
 	if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl] ||
 	    desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) {
 		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index e717208cfb18..34d6d4d99f97 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -4948,7 +4948,7 @@ bool css_has_online_children(struct cgroup_subsys_state *css)
 
 	rcu_read_lock();
 	css_for_each_child(child, css) {
-		if (child->flags & CSS_ONLINE) {
+		if (css_is_online(child)) {
 			ret = true;
 			break;
 		}
@@ -5753,7 +5753,7 @@ static void offline_css(struct cgroup_subsys_state *css)
 
 	lockdep_assert_held(&cgroup_mutex);
 
-	if (!(css->flags & CSS_ONLINE))
+	if (!css_is_online(css))
 		return;
 
 	if (ss->css_offline)
diff --git a/kernel/cgroup/cpuset-internal.h b/kernel/cgroup/cpuset-internal.h
index 01976c8e7d49..e718a4f54360 100644
--- a/kernel/cgroup/cpuset-internal.h
+++ b/kernel/cgroup/cpuset-internal.h
@@ -9,6 +9,7 @@
 #include <linux/cpuset.h>
 #include <linux/spinlock.h>
 #include <linux/union_find.h>
+#include <linux/sched/isolation.h>
 
 /* See "Frequency meter" comments, below. */
 
@@ -144,17 +145,12 @@ struct cpuset {
 	 */
 	nodemask_t old_mems_allowed;
 
-	struct fmeter fmeter;		/* memory_pressure filter */
-
 	/*
 	 * Tasks are being attached to this cpuset.  Used to prevent
 	 * zeroing cpus/mems_allowed between ->can_attach() and ->attach().
 	 */
 	int attach_in_progress;
 
-	/* for custom sched domain */
-	int relax_domain_level;
-
 	/* partition root state */
 	int partition_root_state;
 
@@ -179,10 +175,19 @@ struct cpuset {
 	/* Handle for cpuset.cpus.partition */
 	struct cgroup_file partition_file;
 
+#ifdef CONFIG_CPUSETS_V1
+	struct fmeter fmeter;		/* memory_pressure filter */
+
+	/* for custom sched domain */
+	int relax_domain_level;
+
 	/* Used to merge intersecting subsets for generate_sched_domains */
 	struct uf_node node;
+#endif
 };
 
+extern struct cpuset top_cpuset;
+
 static inline struct cpuset *css_cs(struct cgroup_subsys_state *css)
 {
 	return css ? container_of(css, struct cpuset, css) : NULL;
@@ -240,6 +245,21 @@ static inline int is_spread_slab(const struct cpuset *cs)
 	return test_bit(CS_SPREAD_SLAB, &cs->flags);
 }
 
+/*
+ * Helper routine for generate_sched_domains().
+ * Do cpusets a, b have overlapping effective cpus_allowed masks?
+ */
+static inline int cpusets_overlap(struct cpuset *a, struct cpuset *b)
+{
+	return cpumask_intersects(a->effective_cpus, b->effective_cpus);
+}
+
+static inline int nr_cpusets(void)
+{
+	/* jump label reference count + the top-level cpuset */
+	return static_key_count(&cpusets_enabled_key.key) + 1;
+}
+
 /**
  * cpuset_for_each_child - traverse online children of a cpuset
  * @child_cs: loop cursor pointing to the current child
@@ -285,7 +305,6 @@ void cpuset_full_unlock(void);
  */
 #ifdef CONFIG_CPUSETS_V1
 extern struct cftype cpuset1_files[];
-void fmeter_init(struct fmeter *fmp);
 void cpuset1_update_task_spread_flags(struct cpuset *cs,
 					struct task_struct *tsk);
 void cpuset1_update_tasks_flags(struct cpuset *cs);
@@ -293,8 +312,12 @@ void cpuset1_hotplug_update_tasks(struct cpuset *cs,
 			    struct cpumask *new_cpus, nodemask_t *new_mems,
 			    bool cpus_updated, bool mems_updated);
 int cpuset1_validate_change(struct cpuset *cur, struct cpuset *trial);
+void cpuset1_init(struct cpuset *cs);
+void cpuset1_online_css(struct cgroup_subsys_state *css);
+int cpuset1_generate_sched_domains(cpumask_var_t **domains,
+			struct sched_domain_attr **attributes);
+
 #else
-static inline void fmeter_init(struct fmeter *fmp) {}
 static inline void cpuset1_update_task_spread_flags(struct cpuset *cs,
 					struct task_struct *tsk) {}
 static inline void cpuset1_update_tasks_flags(struct cpuset *cs) {}
@@ -303,6 +326,11 @@ static inline void cpuset1_hotplug_update_tasks(struct cpuset *cs,
 			    bool cpus_updated, bool mems_updated) {}
 static inline int cpuset1_validate_change(struct cpuset *cur,
 				struct cpuset *trial) { return 0; }
+static inline void cpuset1_init(struct cpuset *cs) {}
+static inline void cpuset1_online_css(struct cgroup_subsys_state *css) {}
+static inline int cpuset1_generate_sched_domains(cpumask_var_t **domains,
+			struct sched_domain_attr **attributes) { return 0; };
+
 #endif /* CONFIG_CPUSETS_V1 */
 
 #endif /* __CPUSET_INTERNAL_H */
diff --git a/kernel/cgroup/cpuset-v1.c b/kernel/cgroup/cpuset-v1.c
index 12e76774c75b..7303315fdba7 100644
--- a/kernel/cgroup/cpuset-v1.c
+++ b/kernel/cgroup/cpuset-v1.c
@@ -62,7 +62,7 @@ struct cpuset_remove_tasks_struct {
 #define FM_SCALE 1000		/* faux fixed point scale */
 
 /* Initialize a frequency meter */
-void fmeter_init(struct fmeter *fmp)
+static void fmeter_init(struct fmeter *fmp)
 {
 	fmp->cnt = 0;
 	fmp->val = 0;
@@ -499,6 +499,245 @@ static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
 	return retval;
 }
 
+void cpuset1_init(struct cpuset *cs)
+{
+	fmeter_init(&cs->fmeter);
+	cs->relax_domain_level = -1;
+}
+
+void cpuset1_online_css(struct cgroup_subsys_state *css)
+{
+	struct cpuset *tmp_cs;
+	struct cgroup_subsys_state *pos_css;
+	struct cpuset *cs = css_cs(css);
+	struct cpuset *parent = parent_cs(cs);
+
+	lockdep_assert_cpus_held();
+	lockdep_assert_cpuset_lock_held();
+
+	if (is_spread_page(parent))
+		set_bit(CS_SPREAD_PAGE, &cs->flags);
+	if (is_spread_slab(parent))
+		set_bit(CS_SPREAD_SLAB, &cs->flags);
+
+	if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags))
+		return;
+
+	/*
+	 * Clone @parent's configuration if CGRP_CPUSET_CLONE_CHILDREN is
+	 * set.  This flag handling is implemented in cgroup core for
+	 * historical reasons - the flag may be specified during mount.
+	 *
+	 * Currently, if any sibling cpusets have exclusive cpus or mem, we
+	 * refuse to clone the configuration - thereby refusing the task to
+	 * be entered, and as a result refusing the sys_unshare() or
+	 * clone() which initiated it.  If this becomes a problem for some
+	 * users who wish to allow that scenario, then this could be
+	 * changed to grant parent->cpus_allowed-sibling_cpus_exclusive
+	 * (and likewise for mems) to the new cgroup.
+	 */
+	rcu_read_lock();
+	cpuset_for_each_child(tmp_cs, pos_css, parent) {
+		if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs)) {
+			rcu_read_unlock();
+			return;
+		}
+	}
+	rcu_read_unlock();
+
+	cpuset_callback_lock_irq();
+	cs->mems_allowed = parent->mems_allowed;
+	cs->effective_mems = parent->mems_allowed;
+	cpumask_copy(cs->cpus_allowed, parent->cpus_allowed);
+	cpumask_copy(cs->effective_cpus, parent->cpus_allowed);
+	cpuset_callback_unlock_irq();
+}
+
+static void
+update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c)
+{
+	if (dattr->relax_domain_level < c->relax_domain_level)
+		dattr->relax_domain_level = c->relax_domain_level;
+}
+
+static void update_domain_attr_tree(struct sched_domain_attr *dattr,
+				    struct cpuset *root_cs)
+{
+	struct cpuset *cp;
+	struct cgroup_subsys_state *pos_css;
+
+	rcu_read_lock();
+	cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
+		/* skip the whole subtree if @cp doesn't have any CPU */
+		if (cpumask_empty(cp->cpus_allowed)) {
+			pos_css = css_rightmost_descendant(pos_css);
+			continue;
+		}
+
+		if (is_sched_load_balance(cp))
+			update_domain_attr(dattr, cp);
+	}
+	rcu_read_unlock();
+}
+
+/*
+ * cpuset1_generate_sched_domains()
+ *
+ * Finding the best partition (set of domains):
+ *	The double nested loops below over i, j scan over the load
+ *	balanced cpusets (using the array of cpuset pointers in csa[])
+ *	looking for pairs of cpusets that have overlapping cpus_allowed
+ *	and merging them using a union-find algorithm.
+ *
+ *	The union of the cpus_allowed masks from the set of all cpusets
+ *	having the same root then form the one element of the partition
+ *	(one sched domain) to be passed to partition_sched_domains().
+ */
+int cpuset1_generate_sched_domains(cpumask_var_t **domains,
+			struct sched_domain_attr **attributes)
+{
+	struct cpuset *cp;	/* top-down scan of cpusets */
+	struct cpuset **csa;	/* array of all cpuset ptrs */
+	int csn;		/* how many cpuset ptrs in csa so far */
+	int i, j;		/* indices for partition finding loops */
+	cpumask_var_t *doms;	/* resulting partition; i.e. sched domains */
+	struct sched_domain_attr *dattr;  /* attributes for custom domains */
+	int ndoms = 0;		/* number of sched domains in result */
+	int nslot;		/* next empty doms[] struct cpumask slot */
+	struct cgroup_subsys_state *pos_css;
+	bool root_load_balance = is_sched_load_balance(&top_cpuset);
+	int nslot_update;
+
+	lockdep_assert_cpuset_lock_held();
+
+	doms = NULL;
+	dattr = NULL;
+	csa = NULL;
+
+	/* Special case for the 99% of systems with one, full, sched domain */
+	if (root_load_balance) {
+		ndoms = 1;
+		doms = alloc_sched_domains(ndoms);
+		if (!doms)
+			goto done;
+
+		dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL);
+		if (dattr) {
+			*dattr = SD_ATTR_INIT;
+			update_domain_attr_tree(dattr, &top_cpuset);
+		}
+		cpumask_and(doms[0], top_cpuset.effective_cpus,
+			    housekeeping_cpumask(HK_TYPE_DOMAIN));
+
+		goto done;
+	}
+
+	csa = kmalloc_array(nr_cpusets(), sizeof(cp), GFP_KERNEL);
+	if (!csa)
+		goto done;
+	csn = 0;
+
+	rcu_read_lock();
+	if (root_load_balance)
+		csa[csn++] = &top_cpuset;
+	cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) {
+		if (cp == &top_cpuset)
+			continue;
+
+		/*
+		 * Continue traversing beyond @cp iff @cp has some CPUs and
+		 * isn't load balancing.  The former is obvious.  The
+		 * latter: All child cpusets contain a subset of the
+		 * parent's cpus, so just skip them, and then we call
+		 * update_domain_attr_tree() to calc relax_domain_level of
+		 * the corresponding sched domain.
+		 */
+		if (!cpumask_empty(cp->cpus_allowed) &&
+		    !(is_sched_load_balance(cp) &&
+		      cpumask_intersects(cp->cpus_allowed,
+					 housekeeping_cpumask(HK_TYPE_DOMAIN))))
+			continue;
+
+		if (is_sched_load_balance(cp) &&
+		    !cpumask_empty(cp->effective_cpus))
+			csa[csn++] = cp;
+
+		/* skip @cp's subtree */
+		pos_css = css_rightmost_descendant(pos_css);
+		continue;
+	}
+	rcu_read_unlock();
+
+	for (i = 0; i < csn; i++)
+		uf_node_init(&csa[i]->node);
+
+	/* Merge overlapping cpusets */
+	for (i = 0; i < csn; i++) {
+		for (j = i + 1; j < csn; j++) {
+			if (cpusets_overlap(csa[i], csa[j]))
+				uf_union(&csa[i]->node, &csa[j]->node);
+		}
+	}
+
+	/* Count the total number of domains */
+	for (i = 0; i < csn; i++) {
+		if (uf_find(&csa[i]->node) == &csa[i]->node)
+			ndoms++;
+	}
+
+	/*
+	 * Now we know how many domains to create.
+	 * Convert <csn, csa> to <ndoms, doms> and populate cpu masks.
+	 */
+	doms = alloc_sched_domains(ndoms);
+	if (!doms)
+		goto done;
+
+	/*
+	 * The rest of the code, including the scheduler, can deal with
+	 * dattr==NULL case. No need to abort if alloc fails.
+	 */
+	dattr = kmalloc_array(ndoms, sizeof(struct sched_domain_attr),
+			      GFP_KERNEL);
+
+	for (nslot = 0, i = 0; i < csn; i++) {
+		nslot_update = 0;
+		for (j = i; j < csn; j++) {
+			if (uf_find(&csa[j]->node) == &csa[i]->node) {
+				struct cpumask *dp = doms[nslot];
+
+				if (i == j) {
+					nslot_update = 1;
+					cpumask_clear(dp);
+					if (dattr)
+						*(dattr + nslot) = SD_ATTR_INIT;
+				}
+				cpumask_or(dp, dp, csa[j]->effective_cpus);
+				cpumask_and(dp, dp, housekeeping_cpumask(HK_TYPE_DOMAIN));
+				if (dattr)
+					update_domain_attr_tree(dattr + nslot, csa[j]);
+			}
+		}
+		if (nslot_update)
+			nslot++;
+	}
+	BUG_ON(nslot != ndoms);
+
+done:
+	kfree(csa);
+
+	/*
+	 * Fallback to the default domain if kmalloc() failed.
+	 * See comments in partition_sched_domains().
+	 */
+	if (doms == NULL)
+		ndoms = 1;
+
+	*domains    = doms;
+	*attributes = dattr;
+	return ndoms;
+}
+
 /*
  * for the common functions, 'private' gives the type of file
  */
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index 6e6eb09b8db6..7ac7665f0bb6 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -211,12 +211,10 @@ static inline void notify_partition_change(struct cpuset *cs, int old_prs)
  * If cpu_online_mask is used while a hotunplug operation is happening in
  * parallel, we may leave an offline CPU in cpu_allowed or some other masks.
  */
-static struct cpuset top_cpuset = {
+struct cpuset top_cpuset = {
 	.flags = BIT(CS_CPU_EXCLUSIVE) |
 		 BIT(CS_MEM_EXCLUSIVE) | BIT(CS_SCHED_LOAD_BALANCE),
 	.partition_root_state = PRS_ROOT,
-	.relax_domain_level = -1,
-	.remote_partition = false,
 };
 
 /*
@@ -271,6 +269,11 @@ void cpuset_unlock(void)
 	mutex_unlock(&cpuset_mutex);
 }
 
+void lockdep_assert_cpuset_lock_held(void)
+{
+	lockdep_assert_held(&cpuset_mutex);
+}
+
 /**
  * cpuset_full_lock - Acquire full protection for cpuset modification
  *
@@ -741,49 +744,6 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial)
 }
 
 #ifdef CONFIG_SMP
-/*
- * Helper routine for generate_sched_domains().
- * Do cpusets a, b have overlapping effective cpus_allowed masks?
- */
-static int cpusets_overlap(struct cpuset *a, struct cpuset *b)
-{
-	return cpumask_intersects(a->effective_cpus, b->effective_cpus);
-}
-
-static void
-update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c)
-{
-	if (dattr->relax_domain_level < c->relax_domain_level)
-		dattr->relax_domain_level = c->relax_domain_level;
-	return;
-}
-
-static void update_domain_attr_tree(struct sched_domain_attr *dattr,
-				    struct cpuset *root_cs)
-{
-	struct cpuset *cp;
-	struct cgroup_subsys_state *pos_css;
-
-	rcu_read_lock();
-	cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
-		/* skip the whole subtree if @cp doesn't have any CPU */
-		if (cpumask_empty(cp->cpus_allowed)) {
-			pos_css = css_rightmost_descendant(pos_css);
-			continue;
-		}
-
-		if (is_sched_load_balance(cp))
-			update_domain_attr(dattr, cp);
-	}
-	rcu_read_unlock();
-}
-
-/* Must be called with cpuset_mutex held.  */
-static inline int nr_cpusets(void)
-{
-	/* jump label reference count + the top-level cpuset */
-	return static_key_count(&cpusets_enabled_key.key) + 1;
-}
 
 /*
  * generate_sched_domains()
@@ -823,103 +783,46 @@ static inline int nr_cpusets(void)
  *	   convenient format, that can be easily compared to the prior
  *	   value to determine what partition elements (sched domains)
  *	   were changed (added or removed.)
- *
- * Finding the best partition (set of domains):
- *	The double nested loops below over i, j scan over the load
- *	balanced cpusets (using the array of cpuset pointers in csa[])
- *	looking for pairs of cpusets that have overlapping cpus_allowed
- *	and merging them using a union-find algorithm.
- *
- *	The union of the cpus_allowed masks from the set of all cpusets
- *	having the same root then form the one element of the partition
- *	(one sched domain) to be passed to partition_sched_domains().
- *
  */
 static int generate_sched_domains(cpumask_var_t **domains,
 			struct sched_domain_attr **attributes)
 {
 	struct cpuset *cp;	/* top-down scan of cpusets */
 	struct cpuset **csa;	/* array of all cpuset ptrs */
-	int csn;		/* how many cpuset ptrs in csa so far */
 	int i, j;		/* indices for partition finding loops */
 	cpumask_var_t *doms;	/* resulting partition; i.e. sched domains */
 	struct sched_domain_attr *dattr;  /* attributes for custom domains */
 	int ndoms = 0;		/* number of sched domains in result */
-	int nslot;		/* next empty doms[] struct cpumask slot */
 	struct cgroup_subsys_state *pos_css;
-	bool root_load_balance = is_sched_load_balance(&top_cpuset);
-	bool cgrpv2 = cpuset_v2();
-	int nslot_update;
+
+	if (!cpuset_v2())
+		return cpuset1_generate_sched_domains(domains, attributes);
 
 	doms = NULL;
 	dattr = NULL;
 	csa = NULL;
 
 	/* Special case for the 99% of systems with one, full, sched domain */
-	if (root_load_balance && cpumask_empty(subpartitions_cpus)) {
-single_root_domain:
+	if (cpumask_empty(subpartitions_cpus)) {
 		ndoms = 1;
-		doms = alloc_sched_domains(ndoms);
-		if (!doms)
-			goto done;
-
-		dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL);
-		if (dattr) {
-			*dattr = SD_ATTR_INIT;
-			update_domain_attr_tree(dattr, &top_cpuset);
-		}
-		cpumask_and(doms[0], top_cpuset.effective_cpus,
-			    housekeeping_cpumask(HK_TYPE_DOMAIN));
-
-		goto done;
+		/* !csa will be checked and can be correctly handled */
+		goto generate_doms;
 	}
 
 	csa = kmalloc_array(nr_cpusets(), sizeof(cp), GFP_KERNEL);
 	if (!csa)
 		goto done;
-	csn = 0;
 
+	/* Find how many partitions and cache them to csa[] */
 	rcu_read_lock();
-	if (root_load_balance)
-		csa[csn++] = &top_cpuset;
 	cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) {
-		if (cp == &top_cpuset)
-			continue;
-
-		if (cgrpv2)
-			goto v2;
-
-		/*
-		 * v1:
-		 * Continue traversing beyond @cp iff @cp has some CPUs and
-		 * isn't load balancing.  The former is obvious.  The
-		 * latter: All child cpusets contain a subset of the
-		 * parent's cpus, so just skip them, and then we call
-		 * update_domain_attr_tree() to calc relax_domain_level of
-		 * the corresponding sched domain.
-		 */
-		if (!cpumask_empty(cp->cpus_allowed) &&
-		    !(is_sched_load_balance(cp) &&
-		      cpumask_intersects(cp->cpus_allowed,
-					 housekeeping_cpumask(HK_TYPE_DOMAIN))))
-			continue;
-
-		if (is_sched_load_balance(cp) &&
-		    !cpumask_empty(cp->effective_cpus))
-			csa[csn++] = cp;
-
-		/* skip @cp's subtree */
-		pos_css = css_rightmost_descendant(pos_css);
-		continue;
-
-v2:
 		/*
 		 * Only valid partition roots that are not isolated and with
-		 * non-empty effective_cpus will be saved into csn[].
+		 * non-empty effective_cpus will be saved into csa[].
 		 */
 		if ((cp->partition_root_state == PRS_ROOT) &&
 		    !cpumask_empty(cp->effective_cpus))
-			csa[csn++] = cp;
+			csa[ndoms++] = cp;
 
 		/*
 		 * Skip @cp's subtree if not a partition root and has no
@@ -930,40 +833,18 @@ static int generate_sched_domains(cpumask_var_t **domains,
 	}
 	rcu_read_unlock();
 
-	/*
-	 * If there are only isolated partitions underneath the cgroup root,
-	 * we can optimize out unneeded sched domains scanning.
-	 */
-	if (root_load_balance && (csn == 1))
-		goto single_root_domain;
-
-	for (i = 0; i < csn; i++)
-		uf_node_init(&csa[i]->node);
-
-	/* Merge overlapping cpusets */
-	for (i = 0; i < csn; i++) {
-		for (j = i + 1; j < csn; j++) {
-			if (cpusets_overlap(csa[i], csa[j])) {
+	for (i = 0; i < ndoms; i++) {
+		for (j = i + 1; j < ndoms; j++) {
+			if (cpusets_overlap(csa[i], csa[j]))
 				/*
 				 * Cgroup v2 shouldn't pass down overlapping
 				 * partition root cpusets.
 				 */
-				WARN_ON_ONCE(cgrpv2);
-				uf_union(&csa[i]->node, &csa[j]->node);
-			}
+				WARN_ON_ONCE(1);
 		}
 	}
 
-	/* Count the total number of domains */
-	for (i = 0; i < csn; i++) {
-		if (uf_find(&csa[i]->node) == &csa[i]->node)
-			ndoms++;
-	}
-
-	/*
-	 * Now we know how many domains to create.
-	 * Convert <csn, csa> to <ndoms, doms> and populate cpu masks.
-	 */
+generate_doms:
 	doms = alloc_sched_domains(ndoms);
 	if (!doms)
 		goto done;
@@ -980,45 +861,19 @@ static int generate_sched_domains(cpumask_var_t **domains,
 	 * to SD_ATTR_INIT. Also non-isolating partition root CPUs are a
 	 * subset of HK_TYPE_DOMAIN housekeeping CPUs.
 	 */
-	if (cgrpv2) {
-		for (i = 0; i < ndoms; i++) {
-			/*
-			 * The top cpuset may contain some boot time isolated
-			 * CPUs that need to be excluded from the sched domain.
-			 */
-			if (csa[i] == &top_cpuset)
-				cpumask_and(doms[i], csa[i]->effective_cpus,
-					    housekeeping_cpumask(HK_TYPE_DOMAIN));
-			else
-				cpumask_copy(doms[i], csa[i]->effective_cpus);
-			if (dattr)
-				dattr[i] = SD_ATTR_INIT;
-		}
-		goto done;
-	}
-
-	for (nslot = 0, i = 0; i < csn; i++) {
-		nslot_update = 0;
-		for (j = i; j < csn; j++) {
-			if (uf_find(&csa[j]->node) == &csa[i]->node) {
-				struct cpumask *dp = doms[nslot];
-
-				if (i == j) {
-					nslot_update = 1;
-					cpumask_clear(dp);
-					if (dattr)
-						*(dattr + nslot) = SD_ATTR_INIT;
-				}
-				cpumask_or(dp, dp, csa[j]->effective_cpus);
-				cpumask_and(dp, dp, housekeeping_cpumask(HK_TYPE_DOMAIN));
-				if (dattr)
-					update_domain_attr_tree(dattr + nslot, csa[j]);
-			}
-		}
-		if (nslot_update)
-			nslot++;
+	for (i = 0; i < ndoms; i++) {
+		/*
+		 * The top cpuset may contain some boot time isolated
+		 * CPUs that need to be excluded from the sched domain.
+		 */
+		if (!csa || csa[i] == &top_cpuset)
+			cpumask_and(doms[i], top_cpuset.effective_cpus,
+				    housekeeping_cpumask(HK_TYPE_DOMAIN));
+		else
+			cpumask_copy(doms[i], csa[i]->effective_cpus);
+		if (dattr)
+			dattr[i] = SD_ATTR_INIT;
 	}
-	BUG_ON(nslot != ndoms);
 
 done:
 	kfree(csa);
@@ -1103,53 +958,33 @@ void dl_rebuild_rd_accounting(void)
  */
 void rebuild_sched_domains_locked(void)
 {
-	struct cgroup_subsys_state *pos_css;
 	struct sched_domain_attr *attr;
 	cpumask_var_t *doms;
-	struct cpuset *cs;
 	int ndoms;
+	int i;
 
 	lockdep_assert_cpus_held();
 	lockdep_assert_held(&cpuset_mutex);
 	force_sd_rebuild = false;
 
-	/*
-	 * If we have raced with CPU hotplug, return early to avoid
-	 * passing doms with offlined cpu to partition_sched_domains().
-	 * Anyways, cpuset_handle_hotplug() will rebuild sched domains.
-	 *
-	 * With no CPUs in any subpartitions, top_cpuset's effective CPUs
-	 * should be the same as the active CPUs, so checking only top_cpuset
-	 * is enough to detect racing CPU offlines.
-	 */
-	if (cpumask_empty(subpartitions_cpus) &&
-	    !cpumask_equal(top_cpuset.effective_cpus, cpu_active_mask))
-		return;
+	/* Generate domain masks and attrs */
+	ndoms = generate_sched_domains(&doms, &attr);
 
 	/*
-	 * With subpartition CPUs, however, the effective CPUs of a partition
-	 * root should be only a subset of the active CPUs.  Since a CPU in any
-	 * partition root could be offlined, all must be checked.
-	 */
-	if (!cpumask_empty(subpartitions_cpus)) {
-		rcu_read_lock();
-		cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
-			if (!is_partition_valid(cs)) {
-				pos_css = css_rightmost_descendant(pos_css);
-				continue;
-			}
-			if (!cpumask_subset(cs->effective_cpus,
-					    cpu_active_mask)) {
-				rcu_read_unlock();
-				return;
-			}
-		}
-		rcu_read_unlock();
+	* cpuset_hotplug_workfn is invoked synchronously now, thus this
+	* function should not race with CPU hotplug. And the effective CPUs
+	* must not include any offline CPUs. Passing an offline CPU in the
+	* doms to partition_sched_domains() will trigger a kernel panic.
+	*
+	* We perform a final check here: if the doms contains any
+	* offline CPUs, a warning is emitted and we return directly to
+	* prevent the panic.
+	*/
+	for (i = 0; i < ndoms; ++i) {
+		if (WARN_ON_ONCE(!cpumask_subset(doms[i], cpu_active_mask)))
+			return;
 	}
 
-	/* Generate domain masks and attrs */
-	ndoms = generate_sched_domains(&doms, &attr);
-
 	/* Have scheduler rebuild the domains */
 	partition_sched_domains(ndoms, doms, attr);
 }
@@ -1668,7 +1503,14 @@ static int remote_partition_enable(struct cpuset *cs, int new_prs,
 static void remote_partition_disable(struct cpuset *cs, struct tmpmasks *tmp)
 {
 	WARN_ON_ONCE(!is_remote_partition(cs));
-	WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus));
+	/*
+	 * When a CPU is offlined, top_cpuset may end up with no available CPUs,
+	 * which should clear subpartitions_cpus. We should not emit a warning for this
+	 * scenario: the hierarchy is updated from top to bottom, so subpartitions_cpus
+	 * may already be cleared when disabling the partition.
+	 */
+	WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus) &&
+		     !cpumask_empty(subpartitions_cpus));
 
 	spin_lock_irq(&callback_lock);
 	cs->remote_partition = false;
@@ -3617,8 +3459,7 @@ cpuset_css_alloc(struct cgroup_subsys_state *parent_css)
 		return ERR_PTR(-ENOMEM);
 
 	__set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
-	fmeter_init(&cs->fmeter);
-	cs->relax_domain_level = -1;
+	cpuset1_init(cs);
 
 	/* Set CS_MEMORY_MIGRATE for default hierarchy */
 	if (cpuset_v2())
@@ -3631,17 +3472,11 @@ static int cpuset_css_online(struct cgroup_subsys_state *css)
 {
 	struct cpuset *cs = css_cs(css);
 	struct cpuset *parent = parent_cs(cs);
-	struct cpuset *tmp_cs;
-	struct cgroup_subsys_state *pos_css;
 
 	if (!parent)
 		return 0;
 
 	cpuset_full_lock();
-	if (is_spread_page(parent))
-		set_bit(CS_SPREAD_PAGE, &cs->flags);
-	if (is_spread_slab(parent))
-		set_bit(CS_SPREAD_SLAB, &cs->flags);
 	/*
 	 * For v2, clear CS_SCHED_LOAD_BALANCE if parent is isolated
 	 */
@@ -3656,39 +3491,8 @@ static int cpuset_css_online(struct cgroup_subsys_state *css)
 		cs->effective_mems = parent->effective_mems;
 	}
 	spin_unlock_irq(&callback_lock);
+	cpuset1_online_css(css);
 
-	if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags))
-		goto out_unlock;
-
-	/*
-	 * Clone @parent's configuration if CGRP_CPUSET_CLONE_CHILDREN is
-	 * set.  This flag handling is implemented in cgroup core for
-	 * historical reasons - the flag may be specified during mount.
-	 *
-	 * Currently, if any sibling cpusets have exclusive cpus or mem, we
-	 * refuse to clone the configuration - thereby refusing the task to
-	 * be entered, and as a result refusing the sys_unshare() or
-	 * clone() which initiated it.  If this becomes a problem for some
-	 * users who wish to allow that scenario, then this could be
-	 * changed to grant parent->cpus_allowed-sibling_cpus_exclusive
-	 * (and likewise for mems) to the new cgroup.
-	 */
-	rcu_read_lock();
-	cpuset_for_each_child(tmp_cs, pos_css, parent) {
-		if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs)) {
-			rcu_read_unlock();
-			goto out_unlock;
-		}
-	}
-	rcu_read_unlock();
-
-	spin_lock_irq(&callback_lock);
-	cs->mems_allowed = parent->mems_allowed;
-	cs->effective_mems = parent->mems_allowed;
-	cpumask_copy(cs->cpus_allowed, parent->cpus_allowed);
-	cpumask_copy(cs->effective_cpus, parent->cpus_allowed);
-	spin_unlock_irq(&callback_lock);
-out_unlock:
 	cpuset_full_unlock();
 	return 0;
 }
@@ -3888,7 +3692,7 @@ int __init cpuset_init(void)
 	cpumask_setall(top_cpuset.exclusive_cpus);
 	nodes_setall(top_cpuset.effective_mems);
 
-	fmeter_init(&top_cpuset.fmeter);
+	cpuset1_init(&top_cpuset);
 
 	BUG_ON(!alloc_cpumask_var(&cpus_attach, GFP_KERNEL));
 
@@ -3976,8 +3780,9 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
 	if (remote || (is_partition_valid(cs) && is_partition_valid(parent)))
 		compute_partition_effective_cpumask(cs, &new_cpus);
 
-	if (remote && cpumask_empty(&new_cpus) &&
-	    partition_is_populated(cs, NULL)) {
+	if (remote && (cpumask_empty(subpartitions_cpus) ||
+			(cpumask_empty(&new_cpus) &&
+			 partition_is_populated(cs, NULL)))) {
 		cs->prs_err = PERR_HOTPLUG;
 		remote_partition_disable(cs, tmp);
 		compute_effective_cpumask(&new_cpus, cs, parent);
@@ -3990,9 +3795,12 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
 	 * 1) empty effective cpus but not valid empty partition.
 	 * 2) parent is invalid or doesn't grant any cpus to child
 	 *    partitions.
+	 * 3) subpartitions_cpus is empty.
 	 */
-	if (is_local_partition(cs) && (!is_partition_valid(parent) ||
-				tasks_nocpu_error(parent, cs, &new_cpus)))
+	if (is_local_partition(cs) &&
+	    (!is_partition_valid(parent) ||
+	     tasks_nocpu_error(parent, cs, &new_cpus) ||
+	     cpumask_empty(subpartitions_cpus)))
 		partcmd = partcmd_invalidate;
 	/*
 	 * On the other hand, an invalid partition root may be transitioned
diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c
index a198e40c799b..150e5871e66f 100644
--- a/kernel/cgroup/rstat.c
+++ b/kernel/cgroup/rstat.c
@@ -71,7 +71,6 @@ __bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu)
 {
 	struct llist_head *lhead;
 	struct css_rstat_cpu *rstatc;
-	struct css_rstat_cpu __percpu *rstatc_pcpu;
 	struct llist_node *self;
 
 	/*
@@ -104,18 +103,22 @@ __bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu)
 	/*
 	 * This function can be renentered by irqs and nmis for the same cgroup
 	 * and may try to insert the same per-cpu lnode into the llist. Note
-	 * that llist_add() does not protect against such scenarios.
+	 * that llist_add() does not protect against such scenarios. In addition
+	 * this same per-cpu lnode can be modified through init_llist_node()
+	 * from css_rstat_flush() running on a different CPU.
 	 *
 	 * To protect against such stacked contexts of irqs/nmis, we use the
 	 * fact that lnode points to itself when not on a list and then use
-	 * this_cpu_cmpxchg() to atomically set to NULL to select the winner
+	 * try_cmpxchg() to atomically set to NULL to select the winner
 	 * which will call llist_add(). The losers can assume the insertion is
 	 * successful and the winner will eventually add the per-cpu lnode to
 	 * the llist.
+	 *
+	 * Please note that we can not use this_cpu_cmpxchg() here as on some
+	 * archs it is not safe against modifications from multiple CPUs.
 	 */
 	self = &rstatc->lnode;
-	rstatc_pcpu = css->rstat_cpu;
-	if (this_cpu_cmpxchg(rstatc_pcpu->lnode.next, self, NULL) != self)
+	if (!try_cmpxchg(&rstatc->lnode.next, &self, NULL))
 		return;
 
 	lhead = ss_lhead_cpu(css->ss, cpu);
diff --git a/kernel/crash_core.c b/kernel/crash_core.c
index 99dac1aa972a..3952b3e102e0 100644
--- a/kernel/crash_core.c
+++ b/kernel/crash_core.c
@@ -44,9 +44,15 @@ note_buf_t __percpu *crash_notes;
 
 int kimage_crash_copy_vmcoreinfo(struct kimage *image)
 {
-	struct page *vmcoreinfo_page;
+	struct page *vmcoreinfo_base;
+	struct page *vmcoreinfo_pages[DIV_ROUND_UP(VMCOREINFO_BYTES, PAGE_SIZE)];
+	unsigned int order, nr_pages;
+	int i;
 	void *safecopy;
 
+	nr_pages = DIV_ROUND_UP(VMCOREINFO_BYTES, PAGE_SIZE);
+	order = get_order(VMCOREINFO_BYTES);
+
 	if (!IS_ENABLED(CONFIG_CRASH_DUMP))
 		return 0;
 	if (image->type != KEXEC_TYPE_CRASH)
@@ -61,12 +67,15 @@ int kimage_crash_copy_vmcoreinfo(struct kimage *image)
 	 * happens to generate vmcoreinfo note, hereby we rely on
 	 * vmap for this purpose.
 	 */
-	vmcoreinfo_page = kimage_alloc_control_pages(image, 0);
-	if (!vmcoreinfo_page) {
+	vmcoreinfo_base = kimage_alloc_control_pages(image, order);
+	if (!vmcoreinfo_base) {
 		pr_warn("Could not allocate vmcoreinfo buffer\n");
 		return -ENOMEM;
 	}
-	safecopy = vmap(&vmcoreinfo_page, 1, VM_MAP, PAGE_KERNEL);
+	for (i = 0; i < nr_pages; i++)
+		vmcoreinfo_pages[i] = vmcoreinfo_base + i;
+
+	safecopy = vmap(vmcoreinfo_pages, nr_pages, VM_MAP, PAGE_KERNEL);
 	if (!safecopy) {
 		pr_warn("Could not vmap vmcoreinfo buffer\n");
 		return -ENOMEM;
diff --git a/kernel/crash_dump_dm_crypt.c b/kernel/crash_dump_dm_crypt.c
index 401423ba477d..0d23dc1de67c 100644
--- a/kernel/crash_dump_dm_crypt.c
+++ b/kernel/crash_dump_dm_crypt.c
@@ -223,7 +223,7 @@ static void config_key_release(struct config_item *item)
 	key_count--;
 }
 
-static struct configfs_item_operations config_key_item_ops = {
+static const struct configfs_item_operations config_key_item_ops = {
 	.release = config_key_release,
 };
 
@@ -298,7 +298,7 @@ static struct configfs_attribute *config_keys_attrs[] = {
  * Note that, since no extra work is required on ->drop_item(),
  * no ->drop_item() is provided.
  */
-static struct configfs_group_operations config_keys_group_ops = {
+static const struct configfs_group_operations config_keys_group_ops = {
 	.make_item = config_keys_make_item,
 };
 
diff --git a/kernel/debug/gdbstub.c b/kernel/debug/gdbstub.c
index 22fe969c5d2e..f586afd76c80 100644
--- a/kernel/debug/gdbstub.c
+++ b/kernel/debug/gdbstub.c
@@ -27,6 +27,7 @@
 
 #include <linux/kernel.h>
 #include <linux/sched/signal.h>
+#include <linux/hex.h>
 #include <linux/kgdb.h>
 #include <linux/kdb.h>
 #include <linux/serial_core.h>
diff --git a/kernel/fork.c b/kernel/fork.c
index b1f3915d5f8e..41b3f50d95a1 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -133,6 +133,11 @@
  */
 #define MAX_THREADS FUTEX_TID_MASK
 
+/*
+ * Batch size of rss stat approximation
+ */
+#define RSS_STAT_BATCH_SIZE	32
+
 /*
  * Protected counters by write_lock_irq(&tasklist_lock)
  */
@@ -626,14 +631,12 @@ static void check_mm(struct mm_struct *mm)
 			 "Please make sure 'struct resident_page_types[]' is updated as well");
 
 	for (i = 0; i < NR_MM_COUNTERS; i++) {
-		long x = percpu_counter_sum(&mm->rss_stat[i]);
-
-		if (unlikely(x)) {
-			pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld Comm:%s Pid:%d\n",
-				 mm, resident_page_types[i], x,
+		if (unlikely(percpu_counter_tree_precise_compare_value(&mm->rss_stat[i], 0) != 0))
+			pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%d Comm:%s Pid:%d\n",
+				 mm, resident_page_types[i],
+				 percpu_counter_tree_precise_sum(&mm->rss_stat[i]),
 				 current->comm,
 				 task_pid_nr(current));
-		}
 	}
 
 	if (mm_pgtables_bytes(mm))
@@ -731,7 +734,7 @@ void __mmdrop(struct mm_struct *mm)
 	put_user_ns(mm->user_ns);
 	mm_pasid_drop(mm);
 	mm_destroy_cid(mm);
-	percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS);
+	percpu_counter_tree_destroy_many(mm->rss_stat, NR_MM_COUNTERS);
 
 	free_mm(mm);
 }
@@ -1123,8 +1126,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
 	if (mm_alloc_cid(mm, p))
 		goto fail_cid;
 
-	if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT,
-				     NR_MM_COUNTERS))
+	if (percpu_counter_tree_init_many(mm->rss_stat, get_rss_stat_items(mm),
+					  NR_MM_COUNTERS, RSS_STAT_BATCH_SIZE,
+					  GFP_KERNEL_ACCOUNT))
 		goto fail_pcpu;
 
 	mm->user_ns = get_user_ns(user_ns);
@@ -2071,7 +2075,7 @@ __latent_entropy struct task_struct *copy_process(
 
 	p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? args->child_tid : NULL;
 	/*
-	 * Clear TID on mm_release()?
+	 * TID is cleared in mm_release() when the task exits
 	 */
 	p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL;
 
@@ -3006,7 +3010,7 @@ void __init mm_cache_init(void)
 	 * dynamically sized based on the maximum CPU number this system
 	 * can have, taking hotplug into account (nr_cpu_ids).
 	 */
-	mm_size = sizeof(struct mm_struct) + cpumask_size() + mm_cid_size();
+	mm_size = sizeof(struct mm_struct) + cpumask_size() + mm_cid_size() + get_rss_stat_items_size();
 
 	mm_cachep = kmem_cache_create_usercopy("mm_struct",
 			mm_size, ARCH_MIN_MMSTRUCT_ALIGN,
diff --git a/kernel/hung_task.c b/kernel/hung_task.c
index d2254c91450b..01ce46a107b0 100644
--- a/kernel/hung_task.c
+++ b/kernel/hung_task.c
@@ -223,6 +223,32 @@ static inline void debug_show_blocker(struct task_struct *task, unsigned long ti
 }
 #endif
 
+/**
+ * hung_task_diagnostics - Print structured diagnostic info for a hung task.
+ * @t: Pointer to the detected hung task.
+ *
+ * This function consolidates the printing of core diagnostic information
+ * for a task found to be blocked.
+ */
+static inline void hung_task_diagnostics(struct task_struct *t)
+{
+	unsigned long blocked_secs = (jiffies - t->last_switch_time) / HZ;
+	const char *coredump_msg = "      Blocked by coredump.";
+	const char *disable_msg =
+		"\"echo 0 > /proc/sys/kernel/hung_task_timeout_secs\""
+		" disables this message.";
+
+	pr_err("INFO: task %s:%d blocked for more than %ld seconds.\n",
+	       t->comm, t->pid, blocked_secs);
+	pr_err("      %s %s %.*s\n",
+	       print_tainted(), init_utsname()->release,
+	       (int)strcspn(init_utsname()->version, " "),
+	       init_utsname()->version);
+	if (t->flags & PF_POSTCOREDUMP)
+		pr_err("%s\n", coredump_msg);
+	pr_err("%s\n", disable_msg);
+}
+
 static void check_hung_task(struct task_struct *t, unsigned long timeout,
 		unsigned long prev_detect_count)
 {
@@ -252,16 +278,7 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout,
 	if (sysctl_hung_task_warnings || hung_task_call_panic) {
 		if (sysctl_hung_task_warnings > 0)
 			sysctl_hung_task_warnings--;
-		pr_err("INFO: task %s:%d blocked for more than %ld seconds.\n",
-		       t->comm, t->pid, (jiffies - t->last_switch_time) / HZ);
-		pr_err("      %s %s %.*s\n",
-			print_tainted(), init_utsname()->release,
-			(int)strcspn(init_utsname()->version, " "),
-			init_utsname()->version);
-		if (t->flags & PF_POSTCOREDUMP)
-			pr_err("      Blocked by coredump.\n");
-		pr_err("\"echo 0 > /proc/sys/kernel/hung_task_timeout_secs\""
-			" disables this message.\n");
+		hung_task_diagnostics(t);
 		sched_show_task(t);
 		debug_show_blocker(t, timeout);
 
@@ -358,6 +375,31 @@ static long hung_timeout_jiffies(unsigned long last_checked,
 }
 
 #ifdef CONFIG_SYSCTL
+
+/**
+ * proc_dohung_task_detect_count - proc handler for hung_task_detect_count
+ * @table: Pointer to the struct ctl_table definition for this proc entry
+ * @write: Flag indicating the operation
+ * @buffer: User space buffer for data transfer
+ * @lenp: Pointer to the length of the data being transferred
+ * @ppos: Pointer to the current file offset
+ *
+ * This handler is used for reading the current hung task detection count
+ * and for resetting it to zero when a write operation is performed.
+ * Returns 0 on success or a negative error code on failure.
+ */
+static int proc_dohung_task_detect_count(const struct ctl_table *table, int write,
+					 void *buffer, size_t *lenp, loff_t *ppos)
+{
+	if (!write)
+		return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
+
+	WRITE_ONCE(sysctl_hung_task_detect_count, 0);
+	*ppos += *lenp;
+
+	return 0;
+}
+
 /*
  * Process updating of timeout sysctl
  */
@@ -440,8 +482,8 @@ static const struct ctl_table hung_task_sysctls[] = {
 		.procname	= "hung_task_detect_count",
 		.data		= &sysctl_hung_task_detect_count,
 		.maxlen		= sizeof(unsigned long),
-		.mode		= 0444,
-		.proc_handler	= proc_doulongvec_minmax,
+		.mode		= 0644,
+		.proc_handler	= proc_dohung_task_detect_count,
 	},
 	{
 		.procname	= "hung_task_sys_info",
diff --git a/kernel/irq/Kconfig b/kernel/irq/Kconfig
index 1b4254d19a73..05cba4e16dad 100644
--- a/kernel/irq/Kconfig
+++ b/kernel/irq/Kconfig
@@ -92,9 +92,6 @@ config GENERIC_MSI_IRQ
 config IRQ_MSI_IOMMU
 	bool
 
-config IRQ_TIMINGS
-	bool
-
 config GENERIC_IRQ_MATRIX_ALLOCATOR
 	bool
 
diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile
index 6ab3a4055667..86a2e5ae08f9 100644
--- a/kernel/irq/Makefile
+++ b/kernel/irq/Makefile
@@ -1,10 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 
 obj-y := irqdesc.o handle.o manage.o spurious.o resend.o chip.o dummychip.o devres.o kexec.o
-obj-$(CONFIG_IRQ_TIMINGS) += timings.o
-ifeq ($(CONFIG_TEST_IRQ_TIMINGS),y)
-	CFLAGS_timings.o += -DDEBUG
-endif
 obj-$(CONFIG_GENERIC_IRQ_CHIP) += generic-chip.o
 obj-$(CONFIG_GENERIC_IRQ_PROBE) += autoprobe.o
 obj-$(CONFIG_IRQ_DOMAIN) += irqdomain.o
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index 678f094d261a..433f1dd2b0ca 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -1122,7 +1122,7 @@ void irq_cpu_offline(void)
 }
 #endif
 
-#ifdef	CONFIG_IRQ_DOMAIN_HIERARCHY
+#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
 
 #ifdef CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS
 /**
@@ -1194,6 +1194,15 @@ EXPORT_SYMBOL_GPL(handle_fasteoi_mask_irq);
 
 #endif /* CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS */
 
+#ifdef CONFIG_SMP
+void irq_chip_pre_redirect_parent(struct irq_data *data)
+{
+	data = data->parent_data;
+	data->chip->irq_pre_redirect(data);
+}
+EXPORT_SYMBOL_GPL(irq_chip_pre_redirect_parent);
+#endif
+
 /**
  * irq_chip_set_parent_state - set the state of a parent interrupt.
  *
@@ -1476,6 +1485,17 @@ void irq_chip_release_resources_parent(struct irq_data *data)
 		data->chip->irq_release_resources(data);
 }
 EXPORT_SYMBOL_GPL(irq_chip_release_resources_parent);
+#endif /* CONFIG_IRQ_DOMAIN_HIERARCHY */
+
+#ifdef CONFIG_SMP
+int irq_chip_redirect_set_affinity(struct irq_data *data, const struct cpumask *dest, bool force)
+{
+	struct irq_redirect *redir = &irq_data_to_desc(data)->redirect;
+
+	WRITE_ONCE(redir->target_cpu, cpumask_first(dest));
+	return IRQ_SET_MASK_OK;
+}
+EXPORT_SYMBOL_GPL(irq_chip_redirect_set_affinity);
 #endif
 
 /**
diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c
index 786f5570a640..b7d52821837b 100644
--- a/kernel/irq/handle.c
+++ b/kernel/irq/handle.c
@@ -188,8 +188,6 @@ irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc)
 	unsigned int irq = desc->irq_data.irq;
 	struct irqaction *action;
 
-	record_irq_time(desc);
-
 	for_each_action_of_desc(desc, action) {
 		irqreturn_t res;
 
diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h
index 0164ca48da59..202c50f0fcb2 100644
--- a/kernel/irq/internals.h
+++ b/kernel/irq/internals.h
@@ -288,116 +288,6 @@ static inline void
 irq_pm_remove_action(struct irq_desc *desc, struct irqaction *action) { }
 #endif
 
-#ifdef CONFIG_IRQ_TIMINGS
-
-#define IRQ_TIMINGS_SHIFT	5
-#define IRQ_TIMINGS_SIZE	(1 << IRQ_TIMINGS_SHIFT)
-#define IRQ_TIMINGS_MASK	(IRQ_TIMINGS_SIZE - 1)
-
-/**
- * struct irq_timings - irq timings storing structure
- * @values: a circular buffer of u64 encoded <timestamp,irq> values
- * @count: the number of elements in the array
- */
-struct irq_timings {
-	u64	values[IRQ_TIMINGS_SIZE];
-	int	count;
-};
-
-DECLARE_PER_CPU(struct irq_timings, irq_timings);
-
-extern void irq_timings_free(int irq);
-extern int irq_timings_alloc(int irq);
-
-static inline void irq_remove_timings(struct irq_desc *desc)
-{
-	desc->istate &= ~IRQS_TIMINGS;
-
-	irq_timings_free(irq_desc_get_irq(desc));
-}
-
-static inline void irq_setup_timings(struct irq_desc *desc, struct irqaction *act)
-{
-	int irq = irq_desc_get_irq(desc);
-	int ret;
-
-	/*
-	 * We don't need the measurement because the idle code already
-	 * knows the next expiry event.
-	 */
-	if (act->flags & __IRQF_TIMER)
-		return;
-
-	/*
-	 * In case the timing allocation fails, we just want to warn,
-	 * not fail, so letting the system boot anyway.
-	 */
-	ret = irq_timings_alloc(irq);
-	if (ret) {
-		pr_warn("Failed to allocate irq timing stats for irq%d (%d)",
-			irq, ret);
-		return;
-	}
-
-	desc->istate |= IRQS_TIMINGS;
-}
-
-extern void irq_timings_enable(void);
-extern void irq_timings_disable(void);
-
-DECLARE_STATIC_KEY_FALSE(irq_timing_enabled);
-
-/*
- * The interrupt number and the timestamp are encoded into a single
- * u64 variable to optimize the size.
- * 48 bit time stamp and 16 bit IRQ number is way sufficient.
- *  Who cares an IRQ after 78 hours of idle time?
- */
-static inline u64 irq_timing_encode(u64 timestamp, int irq)
-{
-	return (timestamp << 16) | irq;
-}
-
-static inline int irq_timing_decode(u64 value, u64 *timestamp)
-{
-	*timestamp = value >> 16;
-	return value & U16_MAX;
-}
-
-static __always_inline void irq_timings_push(u64 ts, int irq)
-{
-	struct irq_timings *timings = this_cpu_ptr(&irq_timings);
-
-	timings->values[timings->count & IRQ_TIMINGS_MASK] =
-		irq_timing_encode(ts, irq);
-
-	timings->count++;
-}
-
-/*
- * The function record_irq_time is only called in one place in the
- * interrupts handler. We want this function always inline so the code
- * inside is embedded in the function and the static key branching
- * code can act at the higher level. Without the explicit
- * __always_inline we can end up with a function call and a small
- * overhead in the hotpath for nothing.
- */
-static __always_inline void record_irq_time(struct irq_desc *desc)
-{
-	if (!static_branch_likely(&irq_timing_enabled))
-		return;
-
-	if (desc->istate & IRQS_TIMINGS)
-		irq_timings_push(local_clock(), irq_desc_get_irq(desc));
-}
-#else
-static inline void irq_remove_timings(struct irq_desc *desc) {}
-static inline void irq_setup_timings(struct irq_desc *desc,
-				     struct irqaction *act) {};
-static inline void record_irq_time(struct irq_desc *desc) {}
-#endif /* CONFIG_IRQ_TIMINGS */
-
-
 #ifdef CONFIG_GENERIC_IRQ_CHIP
 void irq_init_generic_chip(struct irq_chip_generic *gc, const char *name,
 			   int num_ct, unsigned int irq_base,
diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c
index f8e4e13dbe33..501a653d4153 100644
--- a/kernel/irq/irqdesc.c
+++ b/kernel/irq/irqdesc.c
@@ -78,8 +78,12 @@ static int alloc_masks(struct irq_desc *desc, int node)
 	return 0;
 }
 
-static void desc_smp_init(struct irq_desc *desc, int node,
-			  const struct cpumask *affinity)
+static void irq_redirect_work(struct irq_work *work)
+{
+	handle_irq_desc(container_of(work, struct irq_desc, redirect.work));
+}
+
+static void desc_smp_init(struct irq_desc *desc, int node, const struct cpumask *affinity)
 {
 	if (!affinity)
 		affinity = irq_default_affinity;
@@ -91,6 +95,7 @@ static void desc_smp_init(struct irq_desc *desc, int node,
 #ifdef CONFIG_NUMA
 	desc->irq_common_data.node = node;
 #endif
+	desc->redirect.work = IRQ_WORK_INIT_HARD(irq_redirect_work);
 }
 
 static void free_masks(struct irq_desc *desc)
@@ -766,6 +771,83 @@ int generic_handle_domain_nmi(struct irq_domain *domain, irq_hw_number_t hwirq)
 	WARN_ON_ONCE(!in_nmi());
 	return handle_irq_desc(irq_resolve_mapping(domain, hwirq));
 }
+
+#ifdef CONFIG_SMP
+static bool demux_redirect_remote(struct irq_desc *desc)
+{
+	guard(raw_spinlock)(&desc->lock);
+	const struct cpumask *m = irq_data_get_effective_affinity_mask(&desc->irq_data);
+	unsigned int target_cpu = READ_ONCE(desc->redirect.target_cpu);
+
+	if (desc->irq_data.chip->irq_pre_redirect)
+		desc->irq_data.chip->irq_pre_redirect(&desc->irq_data);
+
+	/*
+	 * If the interrupt handler is already running on a CPU that's included
+	 * in the interrupt's affinity mask, redirection is not necessary.
+	 */
+	if (cpumask_test_cpu(smp_processor_id(), m))
+		return false;
+
+	/*
+	 * The desc->action check protects against IRQ shutdown: __free_irq() sets
+	 * desc->action to NULL while holding desc->lock, which we also hold.
+	 *
+	 * Calling irq_work_queue_on() here is safe w.r.t. CPU unplugging:
+	 *   - takedown_cpu() schedules multi_cpu_stop() on all active CPUs,
+	 *     including the one that's taken down.
+	 *   - multi_cpu_stop() acts like a barrier, which means all active
+	 *     CPUs go through MULTI_STOP_DISABLE_IRQ and disable hard IRQs
+	 *     *before* the dying CPU runs take_cpu_down() in MULTI_STOP_RUN.
+	 *   - Hard IRQs are re-enabled at the end of multi_cpu_stop(), *after*
+	 *     the dying CPU has run take_cpu_down() in MULTI_STOP_RUN.
+	 *   - Since we run in hard IRQ context, we run either before or after
+	 *     take_cpu_down() but never concurrently.
+	 *   - If we run before take_cpu_down(), the dying CPU hasn't been marked
+	 *     offline yet (it's marked via take_cpu_down() -> __cpu_disable()),
+	 *     so the WARN in irq_work_queue_on() can't occur.
+	 *   - Furthermore, the work item we queue will be flushed later via
+	 *     take_cpu_down() -> cpuhp_invoke_callback_range_nofail() ->
+	 *     smpcfd_dying_cpu() -> irq_work_run().
+	 *   - If we run after take_cpu_down(), target_cpu has been already
+	 *     updated via take_cpu_down() -> __cpu_disable(), which eventually
+	 *     calls irq_do_set_affinity() during IRQ migration. So, target_cpu
+	 *     no longer points to the dying CPU in this case.
+	 */
+	if (desc->action)
+		irq_work_queue_on(&desc->redirect.work, target_cpu);
+
+	return true;
+}
+#else /* CONFIG_SMP */
+static bool demux_redirect_remote(struct irq_desc *desc)
+{
+	return false;
+}
+#endif
+
+/**
+ * generic_handle_demux_domain_irq - Invoke the handler for a hardware interrupt
+ *				     of a demultiplexing domain.
+ * @domain:	The domain where to perform the lookup
+ * @hwirq:	The hardware interrupt number to convert to a logical one
+ *
+ * Returns:	True on success, or false if lookup has failed
+ */
+bool generic_handle_demux_domain_irq(struct irq_domain *domain, irq_hw_number_t hwirq)
+{
+	struct irq_desc *desc = irq_resolve_mapping(domain, hwirq);
+
+	if (unlikely(!desc))
+		return false;
+
+	if (demux_redirect_remote(desc))
+		return true;
+
+	return !handle_irq_desc(desc);
+}
+EXPORT_SYMBOL_GPL(generic_handle_demux_domain_irq);
+
 #endif
 
 /* Dynamic interrupt handling */
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index 8b1b4c8a4f54..b2304dcc481f 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -35,6 +35,16 @@ static int __init setup_forced_irqthreads(char *arg)
 early_param("threadirqs", setup_forced_irqthreads);
 #endif
 
+#ifdef CONFIG_SMP
+static inline void synchronize_irqwork(struct irq_desc *desc)
+{
+	/* Synchronize pending or on the fly redirect work */
+	irq_work_sync(&desc->redirect.work);
+}
+#else
+static inline void synchronize_irqwork(struct irq_desc *desc) { }
+#endif
+
 static int __irq_get_irqchip_state(struct irq_data *d, enum irqchip_irq_state which, bool *state);
 
 static void __synchronize_hardirq(struct irq_desc *desc, bool sync_chip)
@@ -107,7 +117,9 @@ EXPORT_SYMBOL(synchronize_hardirq);
 
 static void __synchronize_irq(struct irq_desc *desc)
 {
+	synchronize_irqwork(desc);
 	__synchronize_hardirq(desc, true);
+
 	/*
 	 * We made sure that no hardirq handler is running. Now verify that no
 	 * threaded handlers are active.
@@ -217,8 +229,7 @@ static inline void irq_validate_effective_affinity(struct irq_data *data) { }
 
 static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
 
-int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
-			bool force)
+int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force)
 {
 	struct cpumask *tmp_mask = this_cpu_ptr(&__tmp_mask);
 	struct irq_desc *desc = irq_data_to_desc(data);
@@ -1414,7 +1425,7 @@ setup_irq_thread(struct irqaction *new, unsigned int irq, bool secondary)
 	 * Ensure the thread adjusts the affinity once it reaches the
 	 * thread function.
 	 */
-	new->thread_flags = BIT(IRQTF_AFFINITY);
+	set_bit(IRQTF_AFFINITY, &new->thread_flags);
 
 	return 0;
 }
@@ -1778,8 +1789,6 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
 	chip_bus_sync_unlock(desc);
 	mutex_unlock(&desc->request_mutex);
 
-	irq_setup_timings(desc, new);
-
 	wake_up_and_wait_for_irq_thread_ready(desc, new);
 	wake_up_and_wait_for_irq_thread_ready(desc, new->secondary);
 
@@ -1950,7 +1959,6 @@ static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id)
 
 		irq_release_resources(desc);
 		chip_bus_sync_unlock(desc);
-		irq_remove_timings(desc);
 	}
 
 	mutex_unlock(&desc->request_mutex);
@@ -2451,36 +2459,6 @@ void free_percpu_nmi(unsigned int irq, void __percpu *dev_id)
 	kfree(__free_percpu_irq(irq, dev_id));
 }
 
-/**
- * setup_percpu_irq - setup a per-cpu interrupt
- * @irq:	Interrupt line to setup
- * @act:	irqaction for the interrupt
- *
- * Used to statically setup per-cpu interrupts in the early boot process.
- */
-int setup_percpu_irq(unsigned int irq, struct irqaction *act)
-{
-	struct irq_desc *desc = irq_to_desc(irq);
-	int retval;
-
-	if (!desc || !irq_settings_is_per_cpu_devid(desc))
-		return -EINVAL;
-
-	retval = irq_chip_pm_get(&desc->irq_data);
-	if (retval < 0)
-		return retval;
-
-	if (!act->affinity)
-		act->affinity = cpu_online_mask;
-
-	retval = __setup_irq(irq, desc, act);
-
-	if (retval)
-		irq_chip_pm_put(&desc->irq_data);
-
-	return retval;
-}
-
 static
 struct irqaction *create_percpu_irqaction(irq_handler_t handler, unsigned long flags,
 					  const char *devname, const cpumask_t *affinity,
@@ -2513,10 +2491,9 @@ struct irqaction *create_percpu_irqaction(irq_handler_t handler, unsigned long f
 }
 
 /**
- * __request_percpu_irq - allocate a percpu interrupt line
+ * request_percpu_irq_affinity - allocate a percpu interrupt line
  * @irq:	Interrupt line to allocate
  * @handler:	Function to be called when the IRQ occurs.
- * @flags:	Interrupt type flags (IRQF_TIMER only)
  * @devname:	An ascii name for the claiming device
  * @affinity:	A cpumask describing the target CPUs for this interrupt
  * @dev_id:	A percpu cookie passed back to the handler function
@@ -2529,9 +2506,8 @@ struct irqaction *create_percpu_irqaction(irq_handler_t handler, unsigned long f
  * the handler gets called with the interrupted CPU's instance of
  * that variable.
  */
-int __request_percpu_irq(unsigned int irq, irq_handler_t handler,
-			 unsigned long flags, const char *devname,
-			 const cpumask_t *affinity, void __percpu *dev_id)
+int request_percpu_irq_affinity(unsigned int irq, irq_handler_t handler, const char *devname,
+				const cpumask_t *affinity, void __percpu *dev_id)
 {
 	struct irqaction *action;
 	struct irq_desc *desc;
@@ -2545,10 +2521,7 @@ int __request_percpu_irq(unsigned int irq, irq_handler_t handler,
 	    !irq_settings_is_per_cpu_devid(desc))
 		return -EINVAL;
 
-	if (flags && flags != IRQF_TIMER)
-		return -EINVAL;
-
-	action = create_percpu_irqaction(handler, flags, devname, affinity, dev_id);
+	action = create_percpu_irqaction(handler, 0, devname, affinity, dev_id);
 	if (!action)
 		return -ENOMEM;
 
@@ -2567,7 +2540,7 @@ int __request_percpu_irq(unsigned int irq, irq_handler_t handler,
 
 	return retval;
 }
-EXPORT_SYMBOL_GPL(__request_percpu_irq);
+EXPORT_SYMBOL_GPL(request_percpu_irq_affinity);
 
 /**
  * request_percpu_nmi - allocate a percpu interrupt line for NMI delivery
diff --git a/kernel/irq/timings.c b/kernel/irq/timings.c
deleted file mode 100644
index 4b7315e99bd6..000000000000
--- a/kernel/irq/timings.c
+++ /dev/null
@@ -1,959 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-// Copyright (C) 2016, Linaro Ltd - Daniel Lezcano <daniel.lezcano@linaro.org>
-#define pr_fmt(fmt) "irq_timings: " fmt
-
-#include <linux/kernel.h>
-#include <linux/percpu.h>
-#include <linux/slab.h>
-#include <linux/static_key.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/idr.h>
-#include <linux/irq.h>
-#include <linux/math64.h>
-#include <linux/log2.h>
-
-#include <trace/events/irq.h>
-
-#include "internals.h"
-
-DEFINE_STATIC_KEY_FALSE(irq_timing_enabled);
-
-DEFINE_PER_CPU(struct irq_timings, irq_timings);
-
-static DEFINE_IDR(irqt_stats);
-
-void irq_timings_enable(void)
-{
-	static_branch_enable(&irq_timing_enabled);
-}
-
-void irq_timings_disable(void)
-{
-	static_branch_disable(&irq_timing_enabled);
-}
-
-/*
- * The main goal of this algorithm is to predict the next interrupt
- * occurrence on the current CPU.
- *
- * Currently, the interrupt timings are stored in a circular array
- * buffer every time there is an interrupt, as a tuple: the interrupt
- * number and the associated timestamp when the event occurred <irq,
- * timestamp>.
- *
- * For every interrupt occurring in a short period of time, we can
- * measure the elapsed time between the occurrences for the same
- * interrupt and we end up with a suite of intervals. The experience
- * showed the interrupts are often coming following a periodic
- * pattern.
- *
- * The objective of the algorithm is to find out this periodic pattern
- * in a fastest way and use its period to predict the next irq event.
- *
- * When the next interrupt event is requested, we are in the situation
- * where the interrupts are disabled and the circular buffer
- * containing the timings is filled with the events which happened
- * after the previous next-interrupt-event request.
- *
- * At this point, we read the circular buffer and we fill the irq
- * related statistics structure. After this step, the circular array
- * containing the timings is empty because all the values are
- * dispatched in their corresponding buffers.
- *
- * Now for each interrupt, we can predict the next event by using the
- * suffix array, log interval and exponential moving average
- *
- * 1. Suffix array
- *
- * Suffix array is an array of all the suffixes of a string. It is
- * widely used as a data structure for compression, text search, ...
- * For instance for the word 'banana', the suffixes will be: 'banana'
- * 'anana' 'nana' 'ana' 'na' 'a'
- *
- * Usually, the suffix array is sorted but for our purpose it is
- * not necessary and won't provide any improvement in the context of
- * the solved problem where we clearly define the boundaries of the
- * search by a max period and min period.
- *
- * The suffix array will build a suite of intervals of different
- * length and will look for the repetition of each suite. If the suite
- * is repeating then we have the period because it is the length of
- * the suite whatever its position in the buffer.
- *
- * 2. Log interval
- *
- * We saw the irq timings allow to compute the interval of the
- * occurrences for a specific interrupt. We can reasonably assume the
- * longer is the interval, the higher is the error for the next event
- * and we can consider storing those interval values into an array
- * where each slot in the array correspond to an interval at the power
- * of 2 of the index. For example, index 12 will contain values
- * between 2^11 and 2^12.
- *
- * At the end we have an array of values where at each index defines a
- * [2^index - 1, 2 ^ index] interval values allowing to store a large
- * number of values inside a small array.
- *
- * For example, if we have the value 1123, then we store it at
- * ilog2(1123) = 10 index value.
- *
- * Storing those value at the specific index is done by computing an
- * exponential moving average for this specific slot. For instance,
- * for values 1800, 1123, 1453, ... fall under the same slot (10) and
- * the exponential moving average is computed every time a new value
- * is stored at this slot.
- *
- * 3. Exponential Moving Average
- *
- * The EMA is largely used to track a signal for stocks or as a low
- * pass filter. The magic of the formula, is it is very simple and the
- * reactivity of the average can be tuned with the factors called
- * alpha.
- *
- * The higher the alphas are, the faster the average respond to the
- * signal change. In our case, if a slot in the array is a big
- * interval, we can have numbers with a big difference between
- * them. The impact of those differences in the average computation
- * can be tuned by changing the alpha value.
- *
- *
- *  -- The algorithm --
- *
- * We saw the different processing above, now let's see how they are
- * used together.
- *
- * For each interrupt:
- *	For each interval:
- *		Compute the index = ilog2(interval)
- *		Compute a new_ema(buffer[index], interval)
- *		Store the index in a circular buffer
- *
- *	Compute the suffix array of the indexes
- *
- *	For each suffix:
- *		If the suffix is reverse-found 3 times
- *			Return suffix
- *
- *	Return Not found
- *
- * However we can not have endless suffix array to be build, it won't
- * make sense and it will add an extra overhead, so we can restrict
- * this to a maximum suffix length of 5 and a minimum suffix length of
- * 2. The experience showed 5 is the majority of the maximum pattern
- * period found for different devices.
- *
- * The result is a pattern finding less than 1us for an interrupt.
- *
- * Example based on real values:
- *
- * Example 1 : MMC write/read interrupt interval:
- *
- *	223947, 1240, 1384, 1386, 1386,
- *	217416, 1236, 1384, 1386, 1387,
- *	214719, 1241, 1386, 1387, 1384,
- *	213696, 1234, 1384, 1386, 1388,
- *	219904, 1240, 1385, 1389, 1385,
- *	212240, 1240, 1386, 1386, 1386,
- *	214415, 1236, 1384, 1386, 1387,
- *	214276, 1234, 1384, 1388, ?
- *
- * For each element, apply ilog2(value)
- *
- *	15, 8, 8, 8, 8,
- *	15, 8, 8, 8, 8,
- *	15, 8, 8, 8, 8,
- *	15, 8, 8, 8, 8,
- *	15, 8, 8, 8, 8,
- *	15, 8, 8, 8, 8,
- *	15, 8, 8, 8, 8,
- *	15, 8, 8, 8, ?
- *
- * Max period of 5, we take the last (max_period * 3) 15 elements as
- * we can be confident if the pattern repeats itself three times it is
- * a repeating pattern.
- *
- *	             8,
- *	15, 8, 8, 8, 8,
- *	15, 8, 8, 8, 8,
- *	15, 8, 8, 8, ?
- *
- * Suffixes are:
- *
- *  1) 8, 15, 8, 8, 8  <- max period
- *  2) 8, 15, 8, 8
- *  3) 8, 15, 8
- *  4) 8, 15           <- min period
- *
- * From there we search the repeating pattern for each suffix.
- *
- * buffer: 8, 15, 8, 8, 8, 8, 15, 8, 8, 8, 8, 15, 8, 8, 8
- *         |   |  |  |  |  |   |  |  |  |  |   |  |  |  |
- *         8, 15, 8, 8, 8  |   |  |  |  |  |   |  |  |  |
- *                         8, 15, 8, 8, 8  |   |  |  |  |
- *                                         8, 15, 8, 8, 8
- *
- * When moving the suffix, we found exactly 3 matches.
- *
- * The first suffix with period 5 is repeating.
- *
- * The next event is (3 * max_period) % suffix_period
- *
- * In this example, the result 0, so the next event is suffix[0] => 8
- *
- * However, 8 is the index in the array of exponential moving average
- * which was calculated on the fly when storing the values, so the
- * interval is ema[8] = 1366
- *
- *
- * Example 2:
- *
- *	4, 3, 5, 100,
- *	3, 3, 5, 117,
- *	4, 4, 5, 112,
- *	4, 3, 4, 110,
- *	3, 5, 3, 117,
- *	4, 4, 5, 112,
- *	4, 3, 4, 110,
- *	3, 4, 5, 112,
- *	4, 3, 4, 110
- *
- * ilog2
- *
- *	0, 0, 0, 4,
- *	0, 0, 0, 4,
- *	0, 0, 0, 4,
- *	0, 0, 0, 4,
- *	0, 0, 0, 4,
- *	0, 0, 0, 4,
- *	0, 0, 0, 4,
- *	0, 0, 0, 4,
- *	0, 0, 0, 4
- *
- * Max period 5:
- *	   0, 0, 4,
- *	0, 0, 0, 4,
- *	0, 0, 0, 4,
- *	0, 0, 0, 4
- *
- * Suffixes:
- *
- *  1) 0, 0, 4, 0, 0
- *  2) 0, 0, 4, 0
- *  3) 0, 0, 4
- *  4) 0, 0
- *
- * buffer: 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 4
- *         |  |  |  |  |  |  X
- *         0, 0, 4, 0, 0, |  X
- *                        0, 0
- *
- * buffer: 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 4
- *         |  |  |  |  |  |  |  |  |  |  |  |  |  |  |
- *         0, 0, 4, 0, |  |  |  |  |  |  |  |  |  |  |
- *                     0, 0, 4, 0, |  |  |  |  |  |  |
- *                                 0, 0, 4, 0, |  |  |
- *                                             0  0  4
- *
- * Pattern is found 3 times, the remaining is 1 which results from
- * (max_period * 3) % suffix_period. This value is the index in the
- * suffix arrays. The suffix array for a period 4 has the value 4
- * at index 1.
- */
-#define EMA_ALPHA_VAL		64
-#define EMA_ALPHA_SHIFT		7
-
-#define PREDICTION_PERIOD_MIN	3
-#define PREDICTION_PERIOD_MAX	5
-#define PREDICTION_FACTOR	4
-#define PREDICTION_MAX		10 /* 2 ^ PREDICTION_MAX useconds */
-#define PREDICTION_BUFFER_SIZE	16 /* slots for EMAs, hardly more than 16 */
-
-/*
- * Number of elements in the circular buffer: If it happens it was
- * flushed before, then the number of elements could be smaller than
- * IRQ_TIMINGS_SIZE, so the count is used, otherwise the array size is
- * used as we wrapped. The index begins from zero when we did not
- * wrap. That could be done in a nicer way with the proper circular
- * array structure type but with the cost of extra computation in the
- * interrupt handler hot path. We choose efficiency.
- */
-#define for_each_irqts(i, irqts)					\
-	for (i = irqts->count < IRQ_TIMINGS_SIZE ?			\
-		     0 : irqts->count & IRQ_TIMINGS_MASK,		\
-		     irqts->count = min(IRQ_TIMINGS_SIZE,		\
-					irqts->count);			\
-	     irqts->count > 0; irqts->count--,				\
-		     i = (i + 1) & IRQ_TIMINGS_MASK)
-
-struct irqt_stat {
-	u64	last_ts;
-	u64	ema_time[PREDICTION_BUFFER_SIZE];
-	int	timings[IRQ_TIMINGS_SIZE];
-	int	circ_timings[IRQ_TIMINGS_SIZE];
-	int	count;
-};
-
-/*
- * Exponential moving average computation
- */
-static u64 irq_timings_ema_new(u64 value, u64 ema_old)
-{
-	s64 diff;
-
-	if (unlikely(!ema_old))
-		return value;
-
-	diff = (value - ema_old) * EMA_ALPHA_VAL;
-	/*
-	 * We can use a s64 type variable to be added with the u64
-	 * ema_old variable as this one will never have its topmost
-	 * bit set, it will be always smaller than 2^63 nanosec
-	 * interrupt interval (292 years).
-	 */
-	return ema_old + (diff >> EMA_ALPHA_SHIFT);
-}
-
-static int irq_timings_next_event_index(int *buffer, size_t len, int period_max)
-{
-	int period;
-
-	/*
-	 * Move the beginning pointer to the end minus the max period x 3.
-	 * We are at the point we can begin searching the pattern
-	 */
-	buffer = &buffer[len - (period_max * 3)];
-
-	/* Adjust the length to the maximum allowed period x 3 */
-	len = period_max * 3;
-
-	/*
-	 * The buffer contains the suite of intervals, in a ilog2
-	 * basis, we are looking for a repetition. We point the
-	 * beginning of the search three times the length of the
-	 * period beginning at the end of the buffer. We do that for
-	 * each suffix.
-	 */
-	for (period = period_max; period >= PREDICTION_PERIOD_MIN; period--) {
-
-		/*
-		 * The first comparison always succeed because the
-		 * suffix is deduced from the first n-period bytes of
-		 * the buffer and we compare the initial suffix with
-		 * itself, so we can skip the first iteration.
-		 */
-		int idx = period;
-		size_t size = period;
-
-		/*
-		 * We look if the suite with period 'i' repeat
-		 * itself. If it is truncated at the end, as it
-		 * repeats we can use the period to find out the next
-		 * element with the modulo.
-		 */
-		while (!memcmp(buffer, &buffer[idx], size * sizeof(int))) {
-
-			/*
-			 * Move the index in a period basis
-			 */
-			idx += size;
-
-			/*
-			 * If this condition is reached, all previous
-			 * memcmp were successful, so the period is
-			 * found.
-			 */
-			if (idx == len)
-				return buffer[len % period];
-
-			/*
-			 * If the remaining elements to compare are
-			 * smaller than the period, readjust the size
-			 * of the comparison for the last iteration.
-			 */
-			if (len - idx < period)
-				size = len - idx;
-		}
-	}
-
-	return -1;
-}
-
-static u64 __irq_timings_next_event(struct irqt_stat *irqs, int irq, u64 now)
-{
-	int index, i, period_max, count, start, min = INT_MAX;
-
-	if ((now - irqs->last_ts) >= NSEC_PER_SEC) {
-		irqs->count = irqs->last_ts = 0;
-		return U64_MAX;
-	}
-
-	/*
-	 * As we want to find three times the repetition, we need a
-	 * number of intervals greater or equal to three times the
-	 * maximum period, otherwise we truncate the max period.
-	 */
-	period_max = irqs->count > (3 * PREDICTION_PERIOD_MAX) ?
-		PREDICTION_PERIOD_MAX : irqs->count / 3;
-
-	/*
-	 * If we don't have enough irq timings for this prediction,
-	 * just bail out.
-	 */
-	if (period_max <= PREDICTION_PERIOD_MIN)
-		return U64_MAX;
-
-	/*
-	 * 'count' will depends if the circular buffer wrapped or not
-	 */
-	count = irqs->count < IRQ_TIMINGS_SIZE ?
-		irqs->count : IRQ_TIMINGS_SIZE;
-
-	start = irqs->count < IRQ_TIMINGS_SIZE ?
-		0 : (irqs->count & IRQ_TIMINGS_MASK);
-
-	/*
-	 * Copy the content of the circular buffer into another buffer
-	 * in order to linearize the buffer instead of dealing with
-	 * wrapping indexes and shifted array which will be prone to
-	 * error and extremely difficult to debug.
-	 */
-	for (i = 0; i < count; i++) {
-		int index = (start + i) & IRQ_TIMINGS_MASK;
-
-		irqs->timings[i] = irqs->circ_timings[index];
-		min = min_t(int, irqs->timings[i], min);
-	}
-
-	index = irq_timings_next_event_index(irqs->timings, count, period_max);
-	if (index < 0)
-		return irqs->last_ts + irqs->ema_time[min];
-
-	return irqs->last_ts + irqs->ema_time[index];
-}
-
-static __always_inline int irq_timings_interval_index(u64 interval)
-{
-	/*
-	 * The PREDICTION_FACTOR increase the interval size for the
-	 * array of exponential average.
-	 */
-	u64 interval_us = (interval >> 10) / PREDICTION_FACTOR;
-
-	return likely(interval_us) ? ilog2(interval_us) : 0;
-}
-
-static __always_inline void __irq_timings_store(int irq, struct irqt_stat *irqs,
-						u64 interval)
-{
-	int index;
-
-	/*
-	 * Get the index in the ema table for this interrupt.
-	 */
-	index = irq_timings_interval_index(interval);
-
-	if (index > PREDICTION_BUFFER_SIZE - 1) {
-		irqs->count = 0;
-		return;
-	}
-
-	/*
-	 * Store the index as an element of the pattern in another
-	 * circular array.
-	 */
-	irqs->circ_timings[irqs->count & IRQ_TIMINGS_MASK] = index;
-
-	irqs->ema_time[index] = irq_timings_ema_new(interval,
-						    irqs->ema_time[index]);
-
-	irqs->count++;
-}
-
-static inline void irq_timings_store(int irq, struct irqt_stat *irqs, u64 ts)
-{
-	u64 old_ts = irqs->last_ts;
-	u64 interval;
-
-	/*
-	 * The timestamps are absolute time values, we need to compute
-	 * the timing interval between two interrupts.
-	 */
-	irqs->last_ts = ts;
-
-	/*
-	 * The interval type is u64 in order to deal with the same
-	 * type in our computation, that prevent mindfuck issues with
-	 * overflow, sign and division.
-	 */
-	interval = ts - old_ts;
-
-	/*
-	 * The interrupt triggered more than one second apart, that
-	 * ends the sequence as predictable for our purpose. In this
-	 * case, assume we have the beginning of a sequence and the
-	 * timestamp is the first value. As it is impossible to
-	 * predict anything at this point, return.
-	 *
-	 * Note the first timestamp of the sequence will always fall
-	 * in this test because the old_ts is zero. That is what we
-	 * want as we need another timestamp to compute an interval.
-	 */
-	if (interval >= NSEC_PER_SEC) {
-		irqs->count = 0;
-		return;
-	}
-
-	__irq_timings_store(irq, irqs, interval);
-}
-
-/**
- * irq_timings_next_event - Return when the next event is supposed to arrive
- * @now: current time
- *
- * During the last busy cycle, the number of interrupts is incremented
- * and stored in the irq_timings structure. This information is
- * necessary to:
- *
- * - know if the index in the table wrapped up:
- *
- *      If more than the array size interrupts happened during the
- *      last busy/idle cycle, the index wrapped up and we have to
- *      begin with the next element in the array which is the last one
- *      in the sequence, otherwise it is at the index 0.
- *
- * - have an indication of the interrupts activity on this CPU
- *   (eg. irq/sec)
- *
- * The values are 'consumed' after inserting in the statistical model,
- * thus the count is reinitialized.
- *
- * The array of values **must** be browsed in the time direction, the
- * timestamp must increase between an element and the next one.
- *
- * Returns a nanosec time based estimation of the earliest interrupt,
- * U64_MAX otherwise.
- */
-u64 irq_timings_next_event(u64 now)
-{
-	struct irq_timings *irqts = this_cpu_ptr(&irq_timings);
-	struct irqt_stat *irqs;
-	struct irqt_stat __percpu *s;
-	u64 ts, next_evt = U64_MAX;
-	int i, irq = 0;
-
-	/*
-	 * This function must be called with the local irq disabled in
-	 * order to prevent the timings circular buffer to be updated
-	 * while we are reading it.
-	 */
-	lockdep_assert_irqs_disabled();
-
-	if (!irqts->count)
-		return next_evt;
-
-	/*
-	 * Number of elements in the circular buffer: If it happens it
-	 * was flushed before, then the number of elements could be
-	 * smaller than IRQ_TIMINGS_SIZE, so the count is used,
-	 * otherwise the array size is used as we wrapped. The index
-	 * begins from zero when we did not wrap. That could be done
-	 * in a nicer way with the proper circular array structure
-	 * type but with the cost of extra computation in the
-	 * interrupt handler hot path. We choose efficiency.
-	 *
-	 * Inject measured irq/timestamp to the pattern prediction
-	 * model while decrementing the counter because we consume the
-	 * data from our circular buffer.
-	 */
-	for_each_irqts(i, irqts) {
-		irq = irq_timing_decode(irqts->values[i], &ts);
-		s = idr_find(&irqt_stats, irq);
-		if (s)
-			irq_timings_store(irq, this_cpu_ptr(s), ts);
-	}
-
-	/*
-	 * Look in the list of interrupts' statistics, the earliest
-	 * next event.
-	 */
-	idr_for_each_entry(&irqt_stats, s, i) {
-
-		irqs = this_cpu_ptr(s);
-
-		ts = __irq_timings_next_event(irqs, i, now);
-		if (ts <= now)
-			return now;
-
-		if (ts < next_evt)
-			next_evt = ts;
-	}
-
-	return next_evt;
-}
-
-void irq_timings_free(int irq)
-{
-	struct irqt_stat __percpu *s;
-
-	s = idr_find(&irqt_stats, irq);
-	if (s) {
-		free_percpu(s);
-		idr_remove(&irqt_stats, irq);
-	}
-}
-
-int irq_timings_alloc(int irq)
-{
-	struct irqt_stat __percpu *s;
-	int id;
-
-	/*
-	 * Some platforms can have the same private interrupt per cpu,
-	 * so this function may be called several times with the
-	 * same interrupt number. Just bail out in case the per cpu
-	 * stat structure is already allocated.
-	 */
-	s = idr_find(&irqt_stats, irq);
-	if (s)
-		return 0;
-
-	s = alloc_percpu(*s);
-	if (!s)
-		return -ENOMEM;
-
-	idr_preload(GFP_KERNEL);
-	id = idr_alloc(&irqt_stats, s, irq, irq + 1, GFP_NOWAIT);
-	idr_preload_end();
-
-	if (id < 0) {
-		free_percpu(s);
-		return id;
-	}
-
-	return 0;
-}
-
-#ifdef CONFIG_TEST_IRQ_TIMINGS
-struct timings_intervals {
-	u64 *intervals;
-	size_t count;
-};
-
-/*
- * Intervals are given in nanosecond base
- */
-static u64 intervals0[] __initdata = {
-	10000, 50000, 200000, 500000,
-	10000, 50000, 200000, 500000,
-	10000, 50000, 200000, 500000,
-	10000, 50000, 200000, 500000,
-	10000, 50000, 200000, 500000,
-	10000, 50000, 200000, 500000,
-	10000, 50000, 200000, 500000,
-	10000, 50000, 200000, 500000,
-	10000, 50000, 200000,
-};
-
-static u64 intervals1[] __initdata = {
-	223947000, 1240000, 1384000, 1386000, 1386000,
-	217416000, 1236000, 1384000, 1386000, 1387000,
-	214719000, 1241000, 1386000, 1387000, 1384000,
-	213696000, 1234000, 1384000, 1386000, 1388000,
-	219904000, 1240000, 1385000, 1389000, 1385000,
-	212240000, 1240000, 1386000, 1386000, 1386000,
-	214415000, 1236000, 1384000, 1386000, 1387000,
-	214276000, 1234000,
-};
-
-static u64 intervals2[] __initdata = {
-	4000, 3000, 5000, 100000,
-	3000, 3000, 5000, 117000,
-	4000, 4000, 5000, 112000,
-	4000, 3000, 4000, 110000,
-	3000, 5000, 3000, 117000,
-	4000, 4000, 5000, 112000,
-	4000, 3000, 4000, 110000,
-	3000, 4000, 5000, 112000,
-	4000,
-};
-
-static u64 intervals3[] __initdata = {
-	1385000, 212240000, 1240000,
-	1386000, 214415000, 1236000,
-	1384000, 214276000, 1234000,
-	1386000, 214415000, 1236000,
-	1385000, 212240000, 1240000,
-	1386000, 214415000, 1236000,
-	1384000, 214276000, 1234000,
-	1386000, 214415000, 1236000,
-	1385000, 212240000, 1240000,
-};
-
-static u64 intervals4[] __initdata = {
-	10000, 50000, 10000, 50000,
-	10000, 50000, 10000, 50000,
-	10000, 50000, 10000, 50000,
-	10000, 50000, 10000, 50000,
-	10000, 50000, 10000, 50000,
-	10000, 50000, 10000, 50000,
-	10000, 50000, 10000, 50000,
-	10000, 50000, 10000, 50000,
-	10000,
-};
-
-static struct timings_intervals tis[] __initdata = {
-	{ intervals0, ARRAY_SIZE(intervals0) },
-	{ intervals1, ARRAY_SIZE(intervals1) },
-	{ intervals2, ARRAY_SIZE(intervals2) },
-	{ intervals3, ARRAY_SIZE(intervals3) },
-	{ intervals4, ARRAY_SIZE(intervals4) },
-};
-
-static int __init irq_timings_test_next_index(struct timings_intervals *ti)
-{
-	int _buffer[IRQ_TIMINGS_SIZE];
-	int buffer[IRQ_TIMINGS_SIZE];
-	int index, start, i, count, period_max;
-
-	count = ti->count - 1;
-
-	period_max = count > (3 * PREDICTION_PERIOD_MAX) ?
-		PREDICTION_PERIOD_MAX : count / 3;
-
-	/*
-	 * Inject all values except the last one which will be used
-	 * to compare with the next index result.
-	 */
-	pr_debug("index suite: ");
-
-	for (i = 0; i < count; i++) {
-		index = irq_timings_interval_index(ti->intervals[i]);
-		_buffer[i & IRQ_TIMINGS_MASK] = index;
-		pr_cont("%d ", index);
-	}
-
-	start = count < IRQ_TIMINGS_SIZE ? 0 :
-		count & IRQ_TIMINGS_MASK;
-
-	count = min_t(int, count, IRQ_TIMINGS_SIZE);
-
-	for (i = 0; i < count; i++) {
-		int index = (start + i) & IRQ_TIMINGS_MASK;
-		buffer[i] = _buffer[index];
-	}
-
-	index = irq_timings_next_event_index(buffer, count, period_max);
-	i = irq_timings_interval_index(ti->intervals[ti->count - 1]);
-
-	if (index != i) {
-		pr_err("Expected (%d) and computed (%d) next indexes differ\n",
-		       i, index);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int __init irq_timings_next_index_selftest(void)
-{
-	int i, ret;
-
-	for (i = 0; i < ARRAY_SIZE(tis); i++) {
-
-		pr_info("---> Injecting intervals number #%d (count=%zd)\n",
-			i, tis[i].count);
-
-		ret = irq_timings_test_next_index(&tis[i]);
-		if (ret)
-			break;
-	}
-
-	return ret;
-}
-
-static int __init irq_timings_test_irqs(struct timings_intervals *ti)
-{
-	struct irqt_stat __percpu *s;
-	struct irqt_stat *irqs;
-	int i, index, ret, irq = 0xACE5;
-
-	ret = irq_timings_alloc(irq);
-	if (ret) {
-		pr_err("Failed to allocate irq timings\n");
-		return ret;
-	}
-
-	s = idr_find(&irqt_stats, irq);
-	if (!s) {
-		ret = -EIDRM;
-		goto out;
-	}
-
-	irqs = this_cpu_ptr(s);
-
-	for (i = 0; i < ti->count; i++) {
-
-		index = irq_timings_interval_index(ti->intervals[i]);
-		pr_debug("%d: interval=%llu ema_index=%d\n",
-			 i, ti->intervals[i], index);
-
-		__irq_timings_store(irq, irqs, ti->intervals[i]);
-		if (irqs->circ_timings[i & IRQ_TIMINGS_MASK] != index) {
-			ret = -EBADSLT;
-			pr_err("Failed to store in the circular buffer\n");
-			goto out;
-		}
-	}
-
-	if (irqs->count != ti->count) {
-		ret = -ERANGE;
-		pr_err("Count differs\n");
-		goto out;
-	}
-
-	ret = 0;
-out:
-	irq_timings_free(irq);
-
-	return ret;
-}
-
-static int __init irq_timings_irqs_selftest(void)
-{
-	int i, ret;
-
-	for (i = 0; i < ARRAY_SIZE(tis); i++) {
-		pr_info("---> Injecting intervals number #%d (count=%zd)\n",
-			i, tis[i].count);
-		ret = irq_timings_test_irqs(&tis[i]);
-		if (ret)
-			break;
-	}
-
-	return ret;
-}
-
-static int __init irq_timings_test_irqts(struct irq_timings *irqts,
-					 unsigned count)
-{
-	int start = count >= IRQ_TIMINGS_SIZE ? count - IRQ_TIMINGS_SIZE : 0;
-	int i, irq, oirq = 0xBEEF;
-	u64 ots = 0xDEAD, ts;
-
-	/*
-	 * Fill the circular buffer by using the dedicated function.
-	 */
-	for (i = 0; i < count; i++) {
-		pr_debug("%d: index=%d, ts=%llX irq=%X\n",
-			 i, i & IRQ_TIMINGS_MASK, ots + i, oirq + i);
-
-		irq_timings_push(ots + i, oirq + i);
-	}
-
-	/*
-	 * Compute the first elements values after the index wrapped
-	 * up or not.
-	 */
-	ots += start;
-	oirq += start;
-
-	/*
-	 * Test the circular buffer count is correct.
-	 */
-	pr_debug("---> Checking timings array count (%d) is right\n", count);
-	if (WARN_ON(irqts->count != count))
-		return -EINVAL;
-
-	/*
-	 * Test the macro allowing to browse all the irqts.
-	 */
-	pr_debug("---> Checking the for_each_irqts() macro\n");
-	for_each_irqts(i, irqts) {
-
-		irq = irq_timing_decode(irqts->values[i], &ts);
-
-		pr_debug("index=%d, ts=%llX / %llX, irq=%X / %X\n",
-			 i, ts, ots, irq, oirq);
-
-		if (WARN_ON(ts != ots || irq != oirq))
-			return -EINVAL;
-
-		ots++; oirq++;
-	}
-
-	/*
-	 * The circular buffer should have be flushed when browsed
-	 * with for_each_irqts
-	 */
-	pr_debug("---> Checking timings array is empty after browsing it\n");
-	if (WARN_ON(irqts->count))
-		return -EINVAL;
-
-	return 0;
-}
-
-static int __init irq_timings_irqts_selftest(void)
-{
-	struct irq_timings *irqts = this_cpu_ptr(&irq_timings);
-	int i, ret;
-
-	/*
-	 * Test the circular buffer with different number of
-	 * elements. The purpose is to test at the limits (empty, half
-	 * full, full, wrapped with the cursor at the boundaries,
-	 * wrapped several times, etc ...
-	 */
-	int count[] = { 0,
-			IRQ_TIMINGS_SIZE >> 1,
-			IRQ_TIMINGS_SIZE,
-			IRQ_TIMINGS_SIZE + (IRQ_TIMINGS_SIZE >> 1),
-			2 * IRQ_TIMINGS_SIZE,
-			(2 * IRQ_TIMINGS_SIZE) + 3,
-	};
-
-	for (i = 0; i < ARRAY_SIZE(count); i++) {
-
-		pr_info("---> Checking the timings with %d/%d values\n",
-			count[i], IRQ_TIMINGS_SIZE);
-
-		ret = irq_timings_test_irqts(irqts, count[i]);
-		if (ret)
-			break;
-	}
-
-	return ret;
-}
-
-static int __init irq_timings_selftest(void)
-{
-	int ret;
-
-	pr_info("------------------- selftest start -----------------\n");
-
-	/*
-	 * At this point, we don't except any subsystem to use the irq
-	 * timings but us, so it should not be enabled.
-	 */
-	if (static_branch_unlikely(&irq_timing_enabled)) {
-		pr_warn("irq timings already initialized, skipping selftest\n");
-		return 0;
-	}
-
-	ret = irq_timings_irqts_selftest();
-	if (ret)
-		goto out;
-
-	ret = irq_timings_irqs_selftest();
-	if (ret)
-		goto out;
-
-	ret = irq_timings_next_index_selftest();
-out:
-	pr_info("---------- selftest end with %s -----------\n",
-		ret ? "failure" : "success");
-
-	return ret;
-}
-early_initcall(irq_timings_selftest);
-#endif
diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c
index 049e296f586c..e0813ca9469a 100644
--- a/kernel/kallsyms.c
+++ b/kernel/kallsyms.c
@@ -345,7 +345,7 @@ int kallsyms_lookup_size_offset(unsigned long addr, unsigned long *symbolsize,
 		return 1;
 	}
 	return !!module_address_lookup(addr, symbolsize, offset, NULL, NULL, namebuf) ||
-	       !!__bpf_address_lookup(addr, symbolsize, offset, namebuf);
+	       !!bpf_address_lookup(addr, symbolsize, offset, namebuf);
 }
 
 static int kallsyms_lookup_buildid(unsigned long addr,
@@ -355,8 +355,21 @@ static int kallsyms_lookup_buildid(unsigned long addr,
 {
 	int ret;
 
-	namebuf[KSYM_NAME_LEN - 1] = 0;
+	/*
+	 * kallsyms_lookus() returns pointer to namebuf on success and
+	 * NULL on error. But some callers ignore the return value.
+	 * Instead they expect @namebuf filled either with valid
+	 * or empty string.
+	 */
 	namebuf[0] = 0;
+	/*
+	 * Initialize the module-related return values. They are not set
+	 * when the symbol is in vmlinux or it is a bpf address.
+	 */
+	if (modname)
+		*modname = NULL;
+	if (modbuildid)
+		*modbuildid = NULL;
 
 	if (is_ksym_addr(addr)) {
 		unsigned long pos;
@@ -365,10 +378,6 @@ static int kallsyms_lookup_buildid(unsigned long addr,
 		/* Grab name */
 		kallsyms_expand_symbol(get_symbol_offset(pos),
 				       namebuf, KSYM_NAME_LEN);
-		if (modname)
-			*modname = NULL;
-		if (modbuildid)
-			*modbuildid = NULL;
 
 		return strlen(namebuf);
 	}
@@ -377,12 +386,11 @@ static int kallsyms_lookup_buildid(unsigned long addr,
 	ret = module_address_lookup(addr, symbolsize, offset,
 				    modname, modbuildid, namebuf);
 	if (!ret)
-		ret = bpf_address_lookup(addr, symbolsize,
-					 offset, modname, namebuf);
+		ret = bpf_address_lookup(addr, symbolsize, offset, namebuf);
 
 	if (!ret)
-		ret = ftrace_mod_address_lookup(addr, symbolsize,
-						offset, modname, namebuf);
+		ret = ftrace_mod_address_lookup(addr, symbolsize, offset,
+						modname, modbuildid, namebuf);
 
 	return ret;
 }
@@ -426,6 +434,37 @@ int lookup_symbol_name(unsigned long addr, char *symname)
 	return lookup_module_symbol_name(addr, symname);
 }
 
+#ifdef CONFIG_STACKTRACE_BUILD_ID
+
+static int append_buildid(char *buffer,  const char *modname,
+			  const unsigned char *buildid)
+{
+	if (!modname)
+		return 0;
+
+	if (!buildid) {
+		pr_warn_once("Undefined buildid for the module %s\n", modname);
+		return 0;
+	}
+
+	/* build ID should match length of sprintf */
+#ifdef CONFIG_MODULES
+	static_assert(sizeof(typeof_member(struct module, build_id)) == 20);
+#endif
+
+	return sprintf(buffer, " %20phN", buildid);
+}
+
+#else /* CONFIG_STACKTRACE_BUILD_ID */
+
+static int append_buildid(char *buffer,   const char *modname,
+			  const unsigned char *buildid)
+{
+	return 0;
+}
+
+#endif /* CONFIG_STACKTRACE_BUILD_ID */
+
 /* Look up a kernel symbol and return it in a text buffer. */
 static int __sprint_symbol(char *buffer, unsigned long address,
 			   int symbol_offset, int add_offset, int add_buildid)
@@ -435,6 +474,9 @@ static int __sprint_symbol(char *buffer, unsigned long address,
 	unsigned long offset, size;
 	int len;
 
+	/* Prevent module removal until modname and modbuildid are printed */
+	guard(rcu)();
+
 	address += symbol_offset;
 	len = kallsyms_lookup_buildid(address, &size, &offset, &modname, &buildid,
 				       buffer);
@@ -448,15 +490,8 @@ static int __sprint_symbol(char *buffer, unsigned long address,
 
 	if (modname) {
 		len += sprintf(buffer + len, " [%s", modname);
-#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID)
-		if (add_buildid && buildid) {
-			/* build ID should match length of sprintf */
-#if IS_ENABLED(CONFIG_MODULES)
-			static_assert(sizeof(typeof_member(struct module, build_id)) == 20);
-#endif
-			len += sprintf(buffer + len, " %20phN", buildid);
-		}
-#endif
+		if (add_buildid)
+			len += append_buildid(buffer + len, modname, buildid);
 		len += sprintf(buffer + len, "]");
 	}
 
diff --git a/kernel/kcsan/kcsan_test.c b/kernel/kcsan/kcsan_test.c
index 219d22857c98..8ef8167be745 100644
--- a/kernel/kcsan/kcsan_test.c
+++ b/kernel/kcsan/kcsan_test.c
@@ -176,7 +176,7 @@ static bool __report_matches(const struct expect_report *r)
 
 	/* Title */
 	cur = expect[0];
-	end = &expect[0][sizeof(expect[0]) - 1];
+	end = ARRAY_END(expect[0]);
 	cur += scnprintf(cur, end - cur, "BUG: KCSAN: %s in ",
 			 is_assert ? "assert: race" : "data-race");
 	if (r->access[1].fn) {
@@ -200,7 +200,7 @@ static bool __report_matches(const struct expect_report *r)
 
 	/* Access 1 */
 	cur = expect[1];
-	end = &expect[1][sizeof(expect[1]) - 1];
+	end = ARRAY_END(expect[1]);
 	if (!r->access[1].fn)
 		cur += scnprintf(cur, end - cur, "race at unknown origin, with ");
 
diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c
index 0f92acdd354d..95c585c6ddc3 100644
--- a/kernel/kexec_core.c
+++ b/kernel/kexec_core.c
@@ -953,17 +953,24 @@ int kimage_load_segment(struct kimage *image, int idx)
 	return result;
 }
 
-void *kimage_map_segment(struct kimage *image,
-			 unsigned long addr, unsigned long size)
+void *kimage_map_segment(struct kimage *image, int idx)
 {
+	unsigned long addr, size, eaddr;
 	unsigned long src_page_addr, dest_page_addr = 0;
-	unsigned long eaddr = addr + size;
 	kimage_entry_t *ptr, entry;
 	struct page **src_pages;
 	unsigned int npages;
+	struct page *cma;
 	void *vaddr = NULL;
 	int i;
 
+	cma = image->segment_cma[idx];
+	if (cma)
+		return page_address(cma);
+
+	addr = image->segment[idx].mem;
+	size = image->segment[idx].memsz;
+	eaddr = addr + size;
 	/*
 	 * Collect the source pages and map them in a contiguous VA range.
 	 */
@@ -1004,7 +1011,8 @@ void *kimage_map_segment(struct kimage *image,
 
 void kimage_unmap_segment(void *segment_buffer)
 {
-	vunmap(segment_buffer);
+	if (is_vmalloc_addr(segment_buffer))
+		vunmap(segment_buffer);
 }
 
 struct kexec_load_limit {
diff --git a/kernel/liveupdate/Makefile b/kernel/liveupdate/Makefile
index 7cad2eece32d..d2f779cbe279 100644
--- a/kernel/liveupdate/Makefile
+++ b/kernel/liveupdate/Makefile
@@ -3,6 +3,7 @@
 luo-y :=								\
 		luo_core.o						\
 		luo_file.o						\
+		luo_flb.o						\
 		luo_session.o
 
 obj-$(CONFIG_KEXEC_HANDOVER)		+= kexec_handover.o
diff --git a/kernel/liveupdate/luo_core.c b/kernel/liveupdate/luo_core.c
index 944663d99dd9..7a9ef16b37d8 100644
--- a/kernel/liveupdate/luo_core.c
+++ b/kernel/liveupdate/luo_core.c
@@ -128,7 +128,9 @@ static int __init luo_early_startup(void)
 	if (err)
 		return err;
 
-	return 0;
+	err = luo_flb_setup_incoming(luo_global.fdt_in);
+
+	return err;
 }
 
 static int __init liveupdate_early_init(void)
@@ -165,6 +167,7 @@ static int __init luo_fdt_setup(void)
 	err |= fdt_property_string(fdt_out, "compatible", LUO_FDT_COMPATIBLE);
 	err |= fdt_property(fdt_out, LUO_FDT_LIVEUPDATE_NUM, &ln, sizeof(ln));
 	err |= luo_session_setup_outgoing(fdt_out);
+	err |= luo_flb_setup_outgoing(fdt_out);
 	err |= fdt_end_node(fdt_out);
 	err |= fdt_finish(fdt_out);
 	if (err)
@@ -226,6 +229,8 @@ int liveupdate_reboot(void)
 	if (err)
 		return err;
 
+	luo_flb_serialize();
+
 	err = kho_finalize();
 	if (err) {
 		pr_err("kho_finalize failed %d\n", err);
diff --git a/kernel/liveupdate/luo_file.c b/kernel/liveupdate/luo_file.c
index a32a777f6df8..86b4d274d9aa 100644
--- a/kernel/liveupdate/luo_file.c
+++ b/kernel/liveupdate/luo_file.c
@@ -284,10 +284,14 @@ int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd)
 	if (err)
 		goto err_free_files_mem;
 
+	err = luo_flb_file_preserve(fh);
+	if (err)
+		goto err_free_files_mem;
+
 	luo_file = kzalloc(sizeof(*luo_file), GFP_KERNEL);
 	if (!luo_file) {
 		err = -ENOMEM;
-		goto err_free_files_mem;
+		goto err_flb_unpreserve;
 	}
 
 	luo_file->file = file;
@@ -311,6 +315,8 @@ int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd)
 
 err_kfree:
 	kfree(luo_file);
+err_flb_unpreserve:
+	luo_flb_file_unpreserve(fh);
 err_free_files_mem:
 	luo_free_files_mem(file_set);
 err_fput:
@@ -352,6 +358,7 @@ void luo_file_unpreserve_files(struct luo_file_set *file_set)
 		args.serialized_data = luo_file->serialized_data;
 		args.private_data = luo_file->private_data;
 		luo_file->fh->ops->unpreserve(&args);
+		luo_flb_file_unpreserve(luo_file->fh);
 
 		list_del(&luo_file->list);
 		file_set->count--;
@@ -629,6 +636,7 @@ static void luo_file_finish_one(struct luo_file_set *file_set,
 	args.retrieved = luo_file->retrieved;
 
 	luo_file->fh->ops->finish(&args);
+	luo_flb_file_finish(luo_file->fh);
 }
 
 /**
@@ -850,10 +858,13 @@ int liveupdate_register_file_handler(struct liveupdate_file_handler *fh)
 		goto err_resume;
 	}
 
+	INIT_LIST_HEAD(&ACCESS_PRIVATE(fh, flb_list));
 	INIT_LIST_HEAD(&ACCESS_PRIVATE(fh, list));
 	list_add_tail(&ACCESS_PRIVATE(fh, list), &luo_file_handler_list);
 	luo_session_resume();
 
+	liveupdate_test_register(fh);
+
 	return 0;
 
 err_resume:
@@ -870,23 +881,38 @@ int liveupdate_register_file_handler(struct liveupdate_file_handler *fh)
  *
  * It ensures safe removal by checking that:
  * No live update session is currently in progress.
+ * No FLB registered with this file handler.
  *
  * If the unregistration fails, the internal test state is reverted.
  *
  * Return: 0 Success. -EOPNOTSUPP when live update is not enabled. -EBUSY A live
- * update is in progress, can't quiesce live update.
+ * update is in progress, can't quiesce live update or FLB is registred with
+ * this file handler.
  */
 int liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh)
 {
+	int err = -EBUSY;
+
 	if (!liveupdate_enabled())
 		return -EOPNOTSUPP;
 
+	liveupdate_test_unregister(fh);
+
 	if (!luo_session_quiesce())
-		return -EBUSY;
+		goto err_register;
+
+	if (!list_empty(&ACCESS_PRIVATE(fh, flb_list)))
+		goto err_resume;
 
 	list_del(&ACCESS_PRIVATE(fh, list));
 	module_put(fh->ops->owner);
 	luo_session_resume();
 
 	return 0;
+
+err_resume:
+	luo_session_resume();
+err_register:
+	liveupdate_test_register(fh);
+	return err;
 }
diff --git a/kernel/liveupdate/luo_flb.c b/kernel/liveupdate/luo_flb.c
new file mode 100644
index 000000000000..e80ac5b575ec
--- /dev/null
+++ b/kernel/liveupdate/luo_flb.c
@@ -0,0 +1,656 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (c) 2025, Google LLC.
+ * Pasha Tatashin <pasha.tatashin@soleen.com>
+ */
+
+/**
+ * DOC: LUO File Lifecycle Bound Global Data
+ *
+ * File-Lifecycle-Bound (FLB) objects provide a mechanism for managing global
+ * state that is shared across multiple live-updatable files. The lifecycle of
+ * this shared state is tied to the preservation of the files that depend on it.
+ *
+ * An FLB represents a global resource, such as the IOMMU core state, that is
+ * required by multiple file descriptors (e.g., all VFIO fds).
+ *
+ * The preservation of the FLB's state is triggered when the *first* file
+ * depending on it is preserved. The cleanup of this state (unpreserve or
+ * finish) is triggered when the *last* file depending on it is unpreserved or
+ * finished.
+ *
+ * Handler Dependency: A file handler declares its dependency on one or more
+ * FLBs by registering them via liveupdate_register_flb().
+ *
+ * Callback Model: Each FLB is defined by a set of operations
+ * (&struct liveupdate_flb_ops) that LUO invokes at key points:
+ *
+ *     - .preserve(): Called for the first file. Saves global state.
+ *     - .unpreserve(): Called for the last file (if aborted pre-reboot).
+ *     - .retrieve(): Called on-demand in the new kernel to restore the state.
+ *     - .finish(): Called for the last file in the new kernel for cleanup.
+ *
+ * This reference-counted approach ensures that shared state is saved exactly
+ * once and restored exactly once, regardless of how many files depend on it,
+ * and that its lifecycle is correctly managed across the kexec transition.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/cleanup.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/io.h>
+#include <linux/kexec_handover.h>
+#include <linux/kho/abi/luo.h>
+#include <linux/libfdt.h>
+#include <linux/list.h>
+#include <linux/liveupdate.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/unaligned.h>
+#include "luo_internal.h"
+
+#define LUO_FLB_PGCNT		1ul
+#define LUO_FLB_MAX		(((LUO_FLB_PGCNT << PAGE_SHIFT) -	\
+		sizeof(struct luo_flb_header_ser)) / sizeof(struct luo_flb_ser))
+
+struct luo_flb_header {
+	struct luo_flb_header_ser *header_ser;
+	struct luo_flb_ser *ser;
+	bool active;
+};
+
+struct luo_flb_global {
+	struct luo_flb_header incoming;
+	struct luo_flb_header outgoing;
+	struct list_head list;
+	long count;
+};
+
+static struct luo_flb_global luo_flb_global = {
+	.list = LIST_HEAD_INIT(luo_flb_global.list),
+};
+
+/*
+ * struct luo_flb_link - Links an FLB definition to a file handler's internal
+ * list of dependencies.
+ * @flb:  A pointer to the registered &struct liveupdate_flb definition.
+ * @list: The list_head for linking.
+ */
+struct luo_flb_link {
+	struct liveupdate_flb *flb;
+	struct list_head list;
+};
+
+/* luo_flb_get_private - Access private field, and if needed initialize it. */
+static struct luo_flb_private *luo_flb_get_private(struct liveupdate_flb *flb)
+{
+	struct luo_flb_private *private = &ACCESS_PRIVATE(flb, private);
+
+	if (!private->initialized) {
+		mutex_init(&private->incoming.lock);
+		mutex_init(&private->outgoing.lock);
+		INIT_LIST_HEAD(&private->list);
+		private->users = 0;
+		private->initialized = true;
+	}
+
+	return private;
+}
+
+static int luo_flb_file_preserve_one(struct liveupdate_flb *flb)
+{
+	struct luo_flb_private *private = luo_flb_get_private(flb);
+
+	scoped_guard(mutex, &private->outgoing.lock) {
+		if (!private->outgoing.count) {
+			struct liveupdate_flb_op_args args = {0};
+			int err;
+
+			args.flb = flb;
+			err = flb->ops->preserve(&args);
+			if (err)
+				return err;
+			private->outgoing.data = args.data;
+			private->outgoing.obj = args.obj;
+		}
+		private->outgoing.count++;
+	}
+
+	return 0;
+}
+
+static void luo_flb_file_unpreserve_one(struct liveupdate_flb *flb)
+{
+	struct luo_flb_private *private = luo_flb_get_private(flb);
+
+	scoped_guard(mutex, &private->outgoing.lock) {
+		private->outgoing.count--;
+		if (!private->outgoing.count) {
+			struct liveupdate_flb_op_args args = {0};
+
+			args.flb = flb;
+			args.data = private->outgoing.data;
+			args.obj = private->outgoing.obj;
+
+			if (flb->ops->unpreserve)
+				flb->ops->unpreserve(&args);
+
+			private->outgoing.data = 0;
+			private->outgoing.obj = NULL;
+		}
+	}
+}
+
+static int luo_flb_retrieve_one(struct liveupdate_flb *flb)
+{
+	struct luo_flb_private *private = luo_flb_get_private(flb);
+	struct luo_flb_header *fh = &luo_flb_global.incoming;
+	struct liveupdate_flb_op_args args = {0};
+	bool found = false;
+	int err;
+
+	guard(mutex)(&private->incoming.lock);
+
+	if (private->incoming.obj)
+		return 0;
+
+	if (!fh->active)
+		return -ENODATA;
+
+	for (int i = 0; i < fh->header_ser->count; i++) {
+		if (!strcmp(fh->ser[i].name, flb->compatible)) {
+			private->incoming.data = fh->ser[i].data;
+			private->incoming.count = fh->ser[i].count;
+			found = true;
+			break;
+		}
+	}
+
+	if (!found)
+		return -ENOENT;
+
+	args.flb = flb;
+	args.data = private->incoming.data;
+
+	err = flb->ops->retrieve(&args);
+	if (err)
+		return err;
+
+	private->incoming.obj = args.obj;
+
+	if (WARN_ON_ONCE(!private->incoming.obj))
+		return -EIO;
+
+	return 0;
+}
+
+static void luo_flb_file_finish_one(struct liveupdate_flb *flb)
+{
+	struct luo_flb_private *private = luo_flb_get_private(flb);
+	u64 count;
+
+	scoped_guard(mutex, &private->incoming.lock)
+		count = --private->incoming.count;
+
+	if (!count) {
+		struct liveupdate_flb_op_args args = {0};
+
+		if (!private->incoming.obj) {
+			int err = luo_flb_retrieve_one(flb);
+
+			if (WARN_ON(err))
+				return;
+		}
+
+		scoped_guard(mutex, &private->incoming.lock) {
+			args.flb = flb;
+			args.obj = private->incoming.obj;
+			flb->ops->finish(&args);
+
+			private->incoming.data = 0;
+			private->incoming.obj = NULL;
+		}
+	}
+}
+
+/**
+ * luo_flb_file_preserve - Notifies FLBs that a file is about to be preserved.
+ * @fh: The file handler for the preserved file.
+ *
+ * This function iterates through all FLBs associated with the given file
+ * handler. It increments the reference count for each FLB. If the count becomes
+ * 1, it triggers the FLB's .preserve() callback to save the global state.
+ *
+ * This operation is atomic. If any FLB's .preserve() op fails, it will roll
+ * back by calling .unpreserve() on any FLBs that were successfully preserved
+ * during this call.
+ *
+ * Context: Called from luo_preserve_file()
+ * Return: 0 on success, or a negative errno on failure.
+ */
+int luo_flb_file_preserve(struct liveupdate_file_handler *fh)
+{
+	struct list_head *flb_list = &ACCESS_PRIVATE(fh, flb_list);
+	struct luo_flb_link *iter;
+	int err = 0;
+
+	list_for_each_entry(iter, flb_list, list) {
+		err = luo_flb_file_preserve_one(iter->flb);
+		if (err)
+			goto exit_err;
+	}
+
+	return 0;
+
+exit_err:
+	list_for_each_entry_continue_reverse(iter, flb_list, list)
+		luo_flb_file_unpreserve_one(iter->flb);
+
+	return err;
+}
+
+/**
+ * luo_flb_file_unpreserve - Notifies FLBs that a dependent file was unpreserved.
+ * @fh: The file handler for the unpreserved file.
+ *
+ * This function iterates through all FLBs associated with the given file
+ * handler, in reverse order of registration. It decrements the reference count
+ * for each FLB. If the count becomes 0, it triggers the FLB's .unpreserve()
+ * callback to clean up the global state.
+ *
+ * Context: Called when a preserved file is being cleaned up before reboot
+ *          (e.g., from luo_file_unpreserve_files()).
+ */
+void luo_flb_file_unpreserve(struct liveupdate_file_handler *fh)
+{
+	struct list_head *flb_list = &ACCESS_PRIVATE(fh, flb_list);
+	struct luo_flb_link *iter;
+
+	list_for_each_entry_reverse(iter, flb_list, list)
+		luo_flb_file_unpreserve_one(iter->flb);
+}
+
+/**
+ * luo_flb_file_finish - Notifies FLBs that a dependent file has been finished.
+ * @fh: The file handler for the finished file.
+ *
+ * This function iterates through all FLBs associated with the given file
+ * handler, in reverse order of registration. It decrements the incoming
+ * reference count for each FLB. If the count becomes 0, it triggers the FLB's
+ * .finish() callback for final cleanup in the new kernel.
+ *
+ * Context: Called from luo_file_finish() for each file being finished.
+ */
+void luo_flb_file_finish(struct liveupdate_file_handler *fh)
+{
+	struct list_head *flb_list = &ACCESS_PRIVATE(fh, flb_list);
+	struct luo_flb_link *iter;
+
+	list_for_each_entry_reverse(iter, flb_list, list)
+		luo_flb_file_finish_one(iter->flb);
+}
+
+/**
+ * liveupdate_register_flb - Associate an FLB with a file handler and register it globally.
+ * @fh:   The file handler that will now depend on the FLB.
+ * @flb:  The File-Lifecycle-Bound object to associate.
+ *
+ * Establishes a dependency, informing the LUO core that whenever a file of
+ * type @fh is preserved, the state of @flb must also be managed.
+ *
+ * On the first registration of a given @flb object, it is added to a global
+ * registry. This function checks for duplicate registrations, both for a
+ * specific handler and globally, and ensures the total number of unique
+ * FLBs does not exceed the system limit.
+ *
+ * Context: Typically called from a subsystem's module init function after
+ *          both the handler and the FLB have been defined and initialized.
+ * Return: 0 on success. Returns a negative errno on failure:
+ *         -EINVAL if arguments are NULL or not initialized.
+ *         -ENOMEM on memory allocation failure.
+ *         -EEXIST if this FLB is already registered with this handler.
+ *         -ENOSPC if the maximum number of global FLBs has been reached.
+ *         -EOPNOTSUPP if live update is disabled or not configured.
+ */
+int liveupdate_register_flb(struct liveupdate_file_handler *fh,
+			    struct liveupdate_flb *flb)
+{
+	struct luo_flb_private *private = luo_flb_get_private(flb);
+	struct list_head *flb_list = &ACCESS_PRIVATE(fh, flb_list);
+	struct luo_flb_link *link __free(kfree) = NULL;
+	struct liveupdate_flb *gflb;
+	struct luo_flb_link *iter;
+	int err;
+
+	if (!liveupdate_enabled())
+		return -EOPNOTSUPP;
+
+	if (WARN_ON(!flb->ops->preserve || !flb->ops->unpreserve ||
+		    !flb->ops->retrieve || !flb->ops->finish)) {
+		return -EINVAL;
+	}
+
+	/*
+	 * File handler must already be registered, as it initializes the
+	 * flb_list
+	 */
+	if (WARN_ON(list_empty(&ACCESS_PRIVATE(fh, list))))
+		return -EINVAL;
+
+	link = kzalloc(sizeof(*link), GFP_KERNEL);
+	if (!link)
+		return -ENOMEM;
+
+	/*
+	 * Ensure the system is quiescent (no active sessions).
+	 * This acts as a global lock for registration: no other thread can
+	 * be in this section, and no sessions can be creating/using FDs.
+	 */
+	if (!luo_session_quiesce())
+		return -EBUSY;
+
+	/* Check that this FLB is not already linked to this file handler */
+	err = -EEXIST;
+	list_for_each_entry(iter, flb_list, list) {
+		if (iter->flb == flb)
+			goto err_resume;
+	}
+
+	/*
+	 * If this FLB is not linked to global list it's the first time the FLB
+	 * is registered
+	 */
+	if (!private->users) {
+		if (WARN_ON(!list_empty(&private->list))) {
+			err = -EINVAL;
+			goto err_resume;
+		}
+
+		if (luo_flb_global.count == LUO_FLB_MAX) {
+			err = -ENOSPC;
+			goto err_resume;
+		}
+
+		/* Check that compatible string is unique in global list */
+		luo_list_for_each_private(gflb, &luo_flb_global.list, private.list) {
+			if (!strcmp(gflb->compatible, flb->compatible))
+				goto err_resume;
+		}
+
+		if (!try_module_get(flb->ops->owner)) {
+			err = -EAGAIN;
+			goto err_resume;
+		}
+
+		list_add_tail(&private->list, &luo_flb_global.list);
+		luo_flb_global.count++;
+	}
+
+	/* Finally, link the FLB to the file handler */
+	private->users++;
+	link->flb = flb;
+	list_add_tail(&no_free_ptr(link)->list, flb_list);
+	luo_session_resume();
+
+	return 0;
+
+err_resume:
+	luo_session_resume();
+	return err;
+}
+
+/**
+ * liveupdate_unregister_flb - Remove an FLB dependency from a file handler.
+ * @fh:   The file handler that is currently depending on the FLB.
+ * @flb:  The File-Lifecycle-Bound object to remove.
+ *
+ * Removes the association between the specified file handler and the FLB
+ * previously established by liveupdate_register_flb().
+ *
+ * This function manages the global lifecycle of the FLB. It decrements the
+ * FLB's usage count. If this was the last file handler referencing this FLB,
+ * the FLB is removed from the global registry and the reference to its
+ * owner module (acquired during registration) is released.
+ *
+ * Context: This function ensures the session is quiesced (no active FDs
+ *          being created) during the update. It is typically called from a
+ *          subsystem's module exit function.
+ * Return: 0 on success.
+ *         -EOPNOTSUPP if live update is disabled.
+ *         -EBUSY if the live update session is active and cannot be quiesced.
+ *         -ENOENT if the FLB was not found in the file handler's list.
+ */
+int liveupdate_unregister_flb(struct liveupdate_file_handler *fh,
+			      struct liveupdate_flb *flb)
+{
+	struct luo_flb_private *private = luo_flb_get_private(flb);
+	struct list_head *flb_list = &ACCESS_PRIVATE(fh, flb_list);
+	struct luo_flb_link *iter;
+	int err = -ENOENT;
+
+	if (!liveupdate_enabled())
+		return -EOPNOTSUPP;
+
+	/*
+	 * Ensure the system is quiescent (no active sessions).
+	 * This acts as a global lock for unregistration.
+	 */
+	if (!luo_session_quiesce())
+		return -EBUSY;
+
+	/* Find and remove the link from the file handler's list */
+	list_for_each_entry(iter, flb_list, list) {
+		if (iter->flb == flb) {
+			list_del(&iter->list);
+			kfree(iter);
+			err = 0;
+			break;
+		}
+	}
+
+	if (err)
+		goto err_resume;
+
+	private->users--;
+	/*
+	 * If this is the last file-handler with which we are registred, remove
+	 * from the global list, and relese module reference.
+	 */
+	if (!private->users) {
+		list_del_init(&private->list);
+		luo_flb_global.count--;
+		module_put(flb->ops->owner);
+	}
+
+	luo_session_resume();
+
+	return 0;
+
+err_resume:
+	luo_session_resume();
+	return err;
+}
+
+/**
+ * liveupdate_flb_get_incoming - Retrieve the incoming FLB object.
+ * @flb:  The FLB definition.
+ * @objp: Output parameter; will be populated with the live shared object.
+ *
+ * Returns a pointer to its shared live object for the incoming (post-reboot)
+ * path.
+ *
+ * If this is the first time the object is requested in the new kernel, this
+ * function will trigger the FLB's .retrieve() callback to reconstruct the
+ * object from its preserved state. Subsequent calls will return the same
+ * cached object.
+ *
+ * Return: 0 on success, or a negative errno on failure. -ENODATA means no
+ * incoming FLB data, -ENOENT means specific flb not found in the incoming
+ * data, and -EOPNOTSUPP when live update is disabled or not configured.
+ */
+int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp)
+{
+	struct luo_flb_private *private = luo_flb_get_private(flb);
+
+	if (!liveupdate_enabled())
+		return -EOPNOTSUPP;
+
+	if (!private->incoming.obj) {
+		int err = luo_flb_retrieve_one(flb);
+
+		if (err)
+			return err;
+	}
+
+	guard(mutex)(&private->incoming.lock);
+	*objp = private->incoming.obj;
+
+	return 0;
+}
+
+/**
+ * liveupdate_flb_get_outgoing - Retrieve the outgoing FLB object.
+ * @flb:  The FLB definition.
+ * @objp: Output parameter; will be populated with the live shared object.
+ *
+ * Returns a pointer to its shared live object for the outgoing (pre-reboot)
+ * path.
+ *
+ * This function assumes the object has already been created by the FLB's
+ * .preserve() callback, which is triggered when the first dependent file
+ * is preserved.
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp)
+{
+	struct luo_flb_private *private = luo_flb_get_private(flb);
+
+	if (!liveupdate_enabled())
+		return -EOPNOTSUPP;
+
+	/* Sanity check that object exists */
+	if (WARN_ON_ONCE(!private->outgoing.obj))
+		return -ENOENT;
+
+	guard(mutex)(&private->outgoing.lock);
+	*objp = private->outgoing.obj;
+
+	return 0;
+}
+
+int __init luo_flb_setup_outgoing(void *fdt_out)
+{
+	struct luo_flb_header_ser *header_ser;
+	u64 header_ser_pa;
+	int err;
+
+	header_ser = kho_alloc_preserve(LUO_FLB_PGCNT << PAGE_SHIFT);
+	if (IS_ERR(header_ser))
+		return PTR_ERR(header_ser);
+
+	header_ser_pa = virt_to_phys(header_ser);
+
+	err = fdt_begin_node(fdt_out, LUO_FDT_FLB_NODE_NAME);
+	err |= fdt_property_string(fdt_out, "compatible",
+				   LUO_FDT_FLB_COMPATIBLE);
+	err |= fdt_property(fdt_out, LUO_FDT_FLB_HEADER, &header_ser_pa,
+			    sizeof(header_ser_pa));
+	err |= fdt_end_node(fdt_out);
+
+	if (err)
+		goto err_unpreserve;
+
+	header_ser->pgcnt = LUO_FLB_PGCNT;
+	luo_flb_global.outgoing.header_ser = header_ser;
+	luo_flb_global.outgoing.ser = (void *)(header_ser + 1);
+	luo_flb_global.outgoing.active = true;
+
+	return 0;
+
+err_unpreserve:
+	kho_unpreserve_free(header_ser);
+
+	return err;
+}
+
+int __init luo_flb_setup_incoming(void *fdt_in)
+{
+	struct luo_flb_header_ser *header_ser;
+	int err, header_size, offset;
+	const void *ptr;
+	u64 header_ser_pa;
+
+	offset = fdt_subnode_offset(fdt_in, 0, LUO_FDT_FLB_NODE_NAME);
+	if (offset < 0) {
+		pr_err("Unable to get FLB node [%s]\n", LUO_FDT_FLB_NODE_NAME);
+
+		return -ENOENT;
+	}
+
+	err = fdt_node_check_compatible(fdt_in, offset,
+					LUO_FDT_FLB_COMPATIBLE);
+	if (err) {
+		pr_err("FLB node is incompatible with '%s' [%d]\n",
+		       LUO_FDT_FLB_COMPATIBLE, err);
+
+		return -EINVAL;
+	}
+
+	header_size = 0;
+	ptr = fdt_getprop(fdt_in, offset, LUO_FDT_FLB_HEADER, &header_size);
+	if (!ptr || header_size != sizeof(u64)) {
+		pr_err("Unable to get FLB header property '%s' [%d]\n",
+		       LUO_FDT_FLB_HEADER, header_size);
+
+		return -EINVAL;
+	}
+
+	header_ser_pa = get_unaligned((u64 *)ptr);
+	header_ser = phys_to_virt(header_ser_pa);
+
+	luo_flb_global.incoming.header_ser = header_ser;
+	luo_flb_global.incoming.ser = (void *)(header_ser + 1);
+	luo_flb_global.incoming.active = true;
+
+	return 0;
+}
+
+/**
+ * luo_flb_serialize - Serializes all active FLB objects for KHO.
+ *
+ * This function is called from the reboot path. It iterates through all
+ * registered File-Lifecycle-Bound (FLB) objects. For each FLB that has been
+ * preserved (i.e., its reference count is greater than zero), it writes its
+ * metadata into the memory region designated for Kexec Handover.
+ *
+ * The serialized data includes the FLB's compatibility string, its opaque
+ * data handle, and the final reference count. This allows the new kernel to
+ * find the appropriate handler and reconstruct the FLB's state.
+ *
+ * Context: Called from liveupdate_reboot() just before kho_finalize().
+ */
+void luo_flb_serialize(void)
+{
+	struct luo_flb_header *fh = &luo_flb_global.outgoing;
+	struct liveupdate_flb *gflb;
+	int i = 0;
+
+	luo_list_for_each_private(gflb, &luo_flb_global.list, private.list) {
+		struct luo_flb_private *private = luo_flb_get_private(gflb);
+
+		if (private->outgoing.count > 0) {
+			strscpy(fh->ser[i].name, gflb->compatible,
+				sizeof(fh->ser[i].name));
+			fh->ser[i].data = private->outgoing.data;
+			fh->ser[i].count = private->outgoing.count;
+			i++;
+		}
+	}
+
+	fh->header_ser->count = i;
+}
diff --git a/kernel/liveupdate/luo_internal.h b/kernel/liveupdate/luo_internal.h
index c8973b543d1d..6115d6a4054d 100644
--- a/kernel/liveupdate/luo_internal.h
+++ b/kernel/liveupdate/luo_internal.h
@@ -107,4 +107,19 @@ int luo_file_deserialize(struct luo_file_set *file_set,
 void luo_file_set_init(struct luo_file_set *file_set);
 void luo_file_set_destroy(struct luo_file_set *file_set);
 
+int luo_flb_file_preserve(struct liveupdate_file_handler *fh);
+void luo_flb_file_unpreserve(struct liveupdate_file_handler *fh);
+void luo_flb_file_finish(struct liveupdate_file_handler *fh);
+int __init luo_flb_setup_outgoing(void *fdt);
+int __init luo_flb_setup_incoming(void *fdt);
+void luo_flb_serialize(void);
+
+#ifdef CONFIG_LIVEUPDATE_TEST
+void liveupdate_test_register(struct liveupdate_file_handler *fh);
+void liveupdate_test_unregister(struct liveupdate_file_handler *fh);
+#else
+static inline void liveupdate_test_register(struct liveupdate_file_handler *fh) { }
+static inline void liveupdate_test_unregister(struct liveupdate_file_handler *fh) { }
+#endif
+
 #endif /* _LINUX_LUO_INTERNAL_H */
diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c
index 00a60796327c..0fc11e45df9b 100644
--- a/kernel/module/kallsyms.c
+++ b/kernel/module/kallsyms.c
@@ -334,13 +334,8 @@ int module_address_lookup(unsigned long addr,
 	if (mod) {
 		if (modname)
 			*modname = mod->name;
-		if (modbuildid) {
-#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID)
-			*modbuildid = mod->build_id;
-#else
-			*modbuildid = NULL;
-#endif
-		}
+		if (modbuildid)
+			*modbuildid = module_buildid(mod);
 
 		sym = find_kallsyms_symbol(mod, addr, size, offset);
 
diff --git a/kernel/pid.c b/kernel/pid.c
index a31771bc89c1..f45ae56db7da 100644
--- a/kernel/pid.c
+++ b/kernel/pid.c
@@ -159,58 +159,86 @@ void free_pids(struct pid **pids)
 			free_pid(pids[tmp]);
 }
 
-struct pid *alloc_pid(struct pid_namespace *ns, pid_t *set_tid,
-		      size_t set_tid_size)
+struct pid *alloc_pid(struct pid_namespace *ns, pid_t *arg_set_tid,
+		      size_t arg_set_tid_size)
 {
+	int set_tid[MAX_PID_NS_LEVEL + 1] = {};
+	int pid_max[MAX_PID_NS_LEVEL + 1] = {};
 	struct pid *pid;
 	enum pid_type type;
 	int i, nr;
 	struct pid_namespace *tmp;
 	struct upid *upid;
 	int retval = -ENOMEM;
+	bool retried_preload;
 
 	/*
-	 * set_tid_size contains the size of the set_tid array. Starting at
+	 * arg_set_tid_size contains the size of the arg_set_tid array. Starting at
 	 * the most nested currently active PID namespace it tells alloc_pid()
 	 * which PID to set for a process in that most nested PID namespace
-	 * up to set_tid_size PID namespaces. It does not have to set the PID
-	 * for a process in all nested PID namespaces but set_tid_size must
+	 * up to arg_set_tid_size PID namespaces. It does not have to set the PID
+	 * for a process in all nested PID namespaces but arg_set_tid_size must
 	 * never be greater than the current ns->level + 1.
 	 */
-	if (set_tid_size > ns->level + 1)
+	if (arg_set_tid_size > ns->level + 1)
 		return ERR_PTR(-EINVAL);
 
+	/*
+	 * Prep before we take locks:
+	 *
+	 * 1. allocate and fill in pid struct
+	 */
 	pid = kmem_cache_alloc(ns->pid_cachep, GFP_KERNEL);
 	if (!pid)
 		return ERR_PTR(retval);
 
-	tmp = ns;
+	get_pid_ns(ns);
 	pid->level = ns->level;
+	refcount_set(&pid->count, 1);
+	spin_lock_init(&pid->lock);
+	for (type = 0; type < PIDTYPE_MAX; ++type)
+		INIT_HLIST_HEAD(&pid->tasks[type]);
+	init_waitqueue_head(&pid->wait_pidfd);
+	INIT_HLIST_HEAD(&pid->inodes);
 
-	for (i = ns->level; i >= 0; i--) {
-		int tid = 0;
-		int pid_max = READ_ONCE(tmp->pid_max);
+	/*
+	 * 2. perm check checkpoint_restore_ns_capable()
+	 *
+	 * This stores found pid_max to make sure the used value is the same should
+	 * later code need it.
+	 */
+	for (tmp = ns, i = ns->level; i >= 0; i--) {
+		pid_max[ns->level - i] = READ_ONCE(tmp->pid_max);
 
-		if (set_tid_size) {
-			tid = set_tid[ns->level - i];
+		if (arg_set_tid_size) {
+			int tid = set_tid[ns->level - i] = arg_set_tid[ns->level - i];
 
 			retval = -EINVAL;
-			if (tid < 1 || tid >= pid_max)
-				goto out_free;
+			if (tid < 1 || tid >= pid_max[ns->level - i])
+				goto out_abort;
 			/*
 			 * Also fail if a PID != 1 is requested and
 			 * no PID 1 exists.
 			 */
 			if (tid != 1 && !tmp->child_reaper)
-				goto out_free;
+				goto out_abort;
 			retval = -EPERM;
 			if (!checkpoint_restore_ns_capable(tmp->user_ns))
-				goto out_free;
-			set_tid_size--;
+				goto out_abort;
+			arg_set_tid_size--;
 		}
 
-		idr_preload(GFP_KERNEL);
-		spin_lock(&pidmap_lock);
+		tmp = tmp->parent;
+	}
+
+	/*
+	 * Prep is done, id allocation goes here:
+	 */
+	retried_preload = false;
+	idr_preload(GFP_KERNEL);
+	spin_lock(&pidmap_lock);
+	for (tmp = ns, i = ns->level; i >= 0;) {
+		int tid = set_tid[ns->level - i];
 
 		if (tid) {
 			nr = idr_alloc(&tmp->idr, NULL, tid,
@@ -220,6 +248,7 @@ struct pid *alloc_pid(struct pid_namespace *ns, pid_t *set_tid,
 			 * alreay in use. Return EEXIST in that case.
 			 */
 			if (nr == -ENOSPC)
+
 				nr = -EEXIST;
 		} else {
 			int pid_min = 1;
@@ -235,19 +264,42 @@ struct pid *alloc_pid(struct pid_namespace *ns, pid_t *set_tid,
 			 * a partially initialized PID (see below).
 			 */
 			nr = idr_alloc_cyclic(&tmp->idr, NULL, pid_min,
-					      pid_max, GFP_ATOMIC);
+					      pid_max[ns->level - i], GFP_ATOMIC);
+			if (nr == -ENOSPC)
+				nr = -EAGAIN;
 		}
-		spin_unlock(&pidmap_lock);
-		idr_preload_end();
 
-		if (nr < 0) {
-			retval = (nr == -ENOSPC) ? -EAGAIN : nr;
+		if (unlikely(nr < 0)) {
+			/*
+			 * Preload more memory if idr_alloc{,cyclic} failed with -ENOMEM.
+			 *
+			 * The IDR API only allows us to preload memory for one call, while we may end
+			 * up doing several under pidmap_lock with GFP_ATOMIC. The situation may be
+			 * salvageable with GFP_KERNEL. But make sure to not loop indefinitely if preload
+			 * did not help (the routine unfortunately returns void, so we have no idea
+			 * if it got anywhere).
+			 *
+			 * The lock can be safely dropped and picked up as historically pid allocation
+			 * for different namespaces was *not* atomic -- we try to hold on to it the
+			 * entire time only for performance reasons.
+			 */
+			if (nr == -ENOMEM && !retried_preload) {
+				spin_unlock(&pidmap_lock);
+				idr_preload_end();
+				retried_preload = true;
+				idr_preload(GFP_KERNEL);
+				spin_lock(&pidmap_lock);
+				continue;
+			}
+			retval = nr;
 			goto out_free;
 		}
 
 		pid->numbers[i].nr = nr;
 		pid->numbers[i].ns = tmp;
 		tmp = tmp->parent;
+		i--;
+		retried_preload = false;
 	}
 
 	/*
@@ -257,25 +309,15 @@ struct pid *alloc_pid(struct pid_namespace *ns, pid_t *set_tid,
 	 * is what we have exposed to userspace for a long time and it is
 	 * documented behavior for pid namespaces. So we can't easily
 	 * change it even if there were an error code better suited.
+	 *
+	 * This can't be done earlier because we need to preserve other
+	 * error conditions.
 	 */
 	retval = -ENOMEM;
-
-	get_pid_ns(ns);
-	refcount_set(&pid->count, 1);
-	spin_lock_init(&pid->lock);
-	for (type = 0; type < PIDTYPE_MAX; ++type)
-		INIT_HLIST_HEAD(&pid->tasks[type]);
-
-	init_waitqueue_head(&pid->wait_pidfd);
-	INIT_HLIST_HEAD(&pid->inodes);
-
-	upid = pid->numbers + ns->level;
-	idr_preload(GFP_KERNEL);
-	spin_lock(&pidmap_lock);
-	if (!(ns->pid_allocated & PIDNS_ADDING))
-		goto out_unlock;
+	if (unlikely(!(ns->pid_allocated & PIDNS_ADDING)))
+		goto out_free;
 	pidfs_add_pid(pid);
-	for ( ; upid >= pid->numbers; --upid) {
+	for (upid = pid->numbers + ns->level; upid >= pid->numbers; --upid) {
 		/* Make the PID visible to find_pid_ns. */
 		idr_replace(&upid->ns->idr, pid, upid->nr);
 		upid->ns->pid_allocated++;
@@ -286,13 +328,7 @@ struct pid *alloc_pid(struct pid_namespace *ns, pid_t *set_tid,
 
 	return pid;
 
-out_unlock:
-	spin_unlock(&pidmap_lock);
-	idr_preload_end();
-	put_pid_ns(ns);
-
 out_free:
-	spin_lock(&pidmap_lock);
 	while (++i <= ns->level) {
 		upid = pid->numbers + i;
 		idr_remove(&upid->ns->idr, upid->nr);
@@ -303,7 +339,10 @@ struct pid *alloc_pid(struct pid_namespace *ns, pid_t *set_tid,
 		idr_set_cursor(&ns->idr, 0);
 
 	spin_unlock(&pidmap_lock);
+	idr_preload_end();
 
+out_abort:
+	put_pid_ns(ns);
 	kmem_cache_free(ns->pid_cachep, pid);
 	return ERR_PTR(retval);
 }
diff --git a/kernel/power/em_netlink_autogen.c b/kernel/power/em_netlink_autogen.c
index a7a09ab1d1c2..ceb3b2bb6ebe 100644
--- a/kernel/power/em_netlink_autogen.c
+++ b/kernel/power/em_netlink_autogen.c
@@ -2,6 +2,7 @@
 /* Do not edit directly, auto-generated from: */
 /*	Documentation/netlink/specs/em.yaml */
 /* YNL-GEN kernel source */
+/* To regenerate run: tools/net/ynl/ynl-regen.sh */
 
 #include <net/netlink.h>
 #include <net/genetlink.h>
diff --git a/kernel/power/em_netlink_autogen.h b/kernel/power/em_netlink_autogen.h
index 78ce609641f1..140ab548103c 100644
--- a/kernel/power/em_netlink_autogen.h
+++ b/kernel/power/em_netlink_autogen.h
@@ -2,6 +2,7 @@
 /* Do not edit directly, auto-generated from: */
 /*	Documentation/netlink/specs/em.yaml */
 /* YNL-GEN kernel header */
+/* To regenerate run: tools/net/ynl/ynl-regen.sh */
 
 #ifndef _LINUX_EM_GEN_H
 #define _LINUX_EM_GEN_H
diff --git a/kernel/power/main.c b/kernel/power/main.c
index 03b2c5495c77..5f8c9e12eaec 100644
--- a/kernel/power/main.c
+++ b/kernel/power/main.c
@@ -1125,7 +1125,7 @@ EXPORT_SYMBOL_GPL(pm_wq);
 
 static int __init pm_start_workqueues(void)
 {
-	pm_wq = alloc_workqueue("pm", WQ_FREEZABLE | WQ_UNBOUND, 0);
+	pm_wq = alloc_workqueue("pm", WQ_UNBOUND, 0);
 	if (!pm_wq)
 		return -ENOMEM;
 
diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c
index 3fa403f9831f..be5a04367e60 100644
--- a/kernel/printk/nbcon.c
+++ b/kernel/printk/nbcon.c
@@ -1557,18 +1557,27 @@ static int __nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq)
 	ctxt->allow_unsafe_takeover	= nbcon_allow_unsafe_takeover();
 
 	while (nbcon_seq_read(con) < stop_seq) {
-		if (!nbcon_context_try_acquire(ctxt, false))
-			return -EPERM;
-
 		/*
-		 * nbcon_emit_next_record() returns false when the console was
-		 * handed over or taken over. In both cases the context is no
-		 * longer valid.
+		 * Atomic flushing does not use console driver synchronization
+		 * (i.e. it does not hold the port lock for uart consoles).
+		 * Therefore IRQs must be disabled to avoid being interrupted
+		 * and then calling into a driver that will deadlock trying
+		 * to acquire console ownership.
 		 */
-		if (!nbcon_emit_next_record(&wctxt, true))
-			return -EAGAIN;
+		scoped_guard(irqsave) {
+			if (!nbcon_context_try_acquire(ctxt, false))
+				return -EPERM;
 
-		nbcon_context_release(ctxt);
+			/*
+			 * nbcon_emit_next_record() returns false when
+			 * the console was handed over or taken over.
+			 * In both cases the context is no longer valid.
+			 */
+			if (!nbcon_emit_next_record(&wctxt, true))
+				return -EAGAIN;
+
+			nbcon_context_release(ctxt);
+		}
 
 		if (!ctxt->backlog) {
 			/* Are there reserved but not yet finalized records? */
@@ -1595,22 +1604,11 @@ static int __nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq)
 static void nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq)
 {
 	struct console_flush_type ft;
-	unsigned long flags;
 	int err;
 
 again:
-	/*
-	 * Atomic flushing does not use console driver synchronization (i.e.
-	 * it does not hold the port lock for uart consoles). Therefore IRQs
-	 * must be disabled to avoid being interrupted and then calling into
-	 * a driver that will deadlock trying to acquire console ownership.
-	 */
-	local_irq_save(flags);
-
 	err = __nbcon_atomic_flush_pending_con(con, stop_seq);
 
-	local_irq_restore(flags);
-
 	/*
 	 * If there was a new owner (-EPERM, -EAGAIN), that context is
 	 * responsible for completing.
@@ -1760,9 +1758,12 @@ bool nbcon_alloc(struct console *con)
 	/* Synchronize the kthread start. */
 	lockdep_assert_console_list_lock_held();
 
-	/* The write_thread() callback is mandatory. */
-	if (WARN_ON(!con->write_thread))
+	/* Check for mandatory nbcon callbacks. */
+	if (WARN_ON(!con->write_thread ||
+		    !con->device_lock ||
+		    !con->device_unlock)) {
 		return false;
+	}
 
 	rcuwait_init(&con->rcuwait);
 	init_irq_work(&con->irq_work, nbcon_irq_work);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 41ba0be16911..7d0a862a8c75 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2090,7 +2090,6 @@ void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
 	 */
 	uclamp_rq_inc(rq, p, flags);
 
-	rq->queue_mask |= p->sched_class->queue_mask;
 	p->sched_class->enqueue_task(rq, p, flags);
 
 	psi_enqueue(p, flags);
@@ -2123,7 +2122,6 @@ inline bool dequeue_task(struct rq *rq, struct task_struct *p, int flags)
 	 * and mark the task ->sched_delayed.
 	 */
 	uclamp_rq_dec(rq, p);
-	rq->queue_mask |= p->sched_class->queue_mask;
 	return p->sched_class->dequeue_task(rq, p, flags);
 }
 
@@ -2174,10 +2172,14 @@ void wakeup_preempt(struct rq *rq, struct task_struct *p, int flags)
 {
 	struct task_struct *donor = rq->donor;
 
-	if (p->sched_class == donor->sched_class)
-		donor->sched_class->wakeup_preempt(rq, p, flags);
-	else if (sched_class_above(p->sched_class, donor->sched_class))
+	if (p->sched_class == rq->next_class) {
+		rq->next_class->wakeup_preempt(rq, p, flags);
+
+	} else if (sched_class_above(p->sched_class, rq->next_class)) {
+		rq->next_class->wakeup_preempt(rq, p, flags);
 		resched_curr(rq);
+		rq->next_class = p->sched_class;
+	}
 
 	/*
 	 * A queue event has occurred, and we're going to schedule.  In
@@ -6804,6 +6806,7 @@ static void __sched notrace __schedule(int sched_mode)
 pick_again:
 	next = pick_next_task(rq, rq->donor, &rf);
 	rq_set_donor(rq, next);
+	rq->next_class = next->sched_class;
 	if (unlikely(task_is_blocked(next))) {
 		next = find_proxy_task(rq, next, &rf);
 		if (!next)
@@ -8650,6 +8653,8 @@ void __init sched_init(void)
 		rq->rt.rt_runtime = global_rt_runtime();
 		init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
 #endif
+		rq->next_class = &idle_sched_class;
+
 		rq->sd = NULL;
 		rq->rd = NULL;
 		rq->cpu_capacity = SCHED_CAPACITY_SCALE;
@@ -10775,13 +10780,12 @@ struct sched_change_ctx *sched_change_begin(struct task_struct *p, unsigned int
 		flags |= DEQUEUE_NOCLOCK;
 	}
 
-	if (flags & DEQUEUE_CLASS) {
-		if (p->sched_class->switching_from)
-			p->sched_class->switching_from(rq, p);
-	}
+	if ((flags & DEQUEUE_CLASS) && p->sched_class->switching_from)
+		p->sched_class->switching_from(rq, p);
 
 	*ctx = (struct sched_change_ctx){
 		.p = p,
+		.class = p->sched_class,
 		.flags = flags,
 		.queued = task_on_rq_queued(p),
 		.running = task_current_donor(rq, p),
@@ -10812,6 +10816,11 @@ void sched_change_end(struct sched_change_ctx *ctx)
 
 	lockdep_assert_rq_held(rq);
 
+	/*
+	 * Changing class without *QUEUE_CLASS is bad.
+	 */
+	WARN_ON_ONCE(p->sched_class != ctx->class && !(ctx->flags & ENQUEUE_CLASS));
+
 	if ((ctx->flags & ENQUEUE_CLASS) && p->sched_class->switching_to)
 		p->sched_class->switching_to(rq, p);
 
@@ -10823,6 +10832,24 @@ void sched_change_end(struct sched_change_ctx *ctx)
 	if (ctx->flags & ENQUEUE_CLASS) {
 		if (p->sched_class->switched_to)
 			p->sched_class->switched_to(rq, p);
+
+		/*
+		 * If this was a class promotion; let the old class know it
+		 * got preempted. Note that none of the switch*_from() methods
+		 * know the new class and none of the switch*_to() methods
+		 * know the old class.
+		 */
+		if (ctx->running && sched_class_above(p->sched_class, ctx->class)) {
+			rq->next_class->wakeup_preempt(rq, p, 0);
+			rq->next_class = p->sched_class;
+		}
+
+		/*
+		 * If this was a degradation in class someone should have set
+		 * need_resched by now.
+		 */
+		WARN_ON_ONCE(sched_class_above(ctx->class, p->sched_class) &&
+			     !test_tsk_need_resched(p));
 	} else {
 		p->sched_class->prio_changed(rq, p, ctx->prio);
 	}
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 319439fe1870..80c9559a3e30 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -2499,9 +2499,16 @@ static int balance_dl(struct rq *rq, struct task_struct *p, struct rq_flags *rf)
  * Only called when both the current and waking task are -deadline
  * tasks.
  */
-static void wakeup_preempt_dl(struct rq *rq, struct task_struct *p,
-				  int flags)
+static void wakeup_preempt_dl(struct rq *rq, struct task_struct *p, int flags)
 {
+	/*
+	 * Can only get preempted by stop-class, and those should be
+	 * few and short lived, doesn't really make sense to push
+	 * anything away for that.
+	 */
+	if (p->sched_class != &dl_sched_class)
+		return;
+
 	if (dl_entity_preempt(&p->dl, &rq->donor->dl)) {
 		resched_curr(rq);
 		return;
@@ -3346,9 +3353,6 @@ static int task_is_throttled_dl(struct task_struct *p, int cpu)
 #endif
 
 DEFINE_SCHED_CLASS(dl) = {
-
-	.queue_mask		= 8,
-
 	.enqueue_task		= enqueue_task_dl,
 	.dequeue_task		= dequeue_task_dl,
 	.yield_task		= yield_task_dl,
diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c
index 05f5a49e9649..9631a2b78af2 100644
--- a/kernel/sched/ext.c
+++ b/kernel/sched/ext.c
@@ -41,6 +41,13 @@ static bool scx_init_task_enabled;
 static bool scx_switching_all;
 DEFINE_STATIC_KEY_FALSE(__scx_switched_all);
 
+/*
+ * Tracks whether scx_enable() called scx_bypass(true). Used to balance bypass
+ * depth on enable failure. Will be removed when bypass depth is moved into the
+ * sched instance.
+ */
+static bool scx_bypassed_for_enable;
+
 static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0);
 static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0);
 
@@ -975,6 +982,30 @@ static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p)
 	__scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1);
 }
 
+static void local_dsq_post_enq(struct scx_dispatch_q *dsq, struct task_struct *p,
+			       u64 enq_flags)
+{
+	struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
+	bool preempt = false;
+
+	/*
+	 * If @rq is in balance, the CPU is already vacant and looking for the
+	 * next task to run. No need to preempt or trigger resched after moving
+	 * @p into its local DSQ.
+	 */
+	if (rq->scx.flags & SCX_RQ_IN_BALANCE)
+		return;
+
+	if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
+	    rq->curr->sched_class == &ext_sched_class) {
+		rq->curr->scx.slice = 0;
+		preempt = true;
+	}
+
+	if (preempt || sched_class_above(&ext_sched_class, rq->curr->sched_class))
+		resched_curr(rq);
+}
+
 static void dispatch_enqueue(struct scx_sched *sch, struct scx_dispatch_q *dsq,
 			     struct task_struct *p, u64 enq_flags)
 {
@@ -1086,22 +1117,10 @@ static void dispatch_enqueue(struct scx_sched *sch, struct scx_dispatch_q *dsq,
 	if (enq_flags & SCX_ENQ_CLEAR_OPSS)
 		atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
 
-	if (is_local) {
-		struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
-		bool preempt = false;
-
-		if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
-		    rq->curr->sched_class == &ext_sched_class) {
-			rq->curr->scx.slice = 0;
-			preempt = true;
-		}
-
-		if (preempt || sched_class_above(&ext_sched_class,
-						 rq->curr->sched_class))
-			resched_curr(rq);
-	} else {
+	if (is_local)
+		local_dsq_post_enq(dsq, p, enq_flags);
+	else
 		raw_spin_unlock(&dsq->lock);
-	}
 }
 
 static void task_unlink_from_dsq(struct task_struct *p,
@@ -1625,6 +1644,8 @@ static void move_local_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
 
 	dsq_mod_nr(dst_dsq, 1);
 	p->scx.dsq = dst_dsq;
+
+	local_dsq_post_enq(dst_dsq, p, enq_flags);
 }
 
 /**
@@ -2402,7 +2423,7 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
 		 * ops.enqueue() that @p is the only one available for this cpu,
 		 * which should trigger an explicit follow-up scheduling event.
 		 */
-		if (sched_class_above(&ext_sched_class, next->sched_class)) {
+		if (next && sched_class_above(&ext_sched_class, next->sched_class)) {
 			WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_LAST));
 			do_enqueue_task(rq, p, SCX_ENQ_LAST, -1);
 		} else {
@@ -2425,13 +2446,13 @@ static struct task_struct *
 do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx)
 {
 	struct task_struct *prev = rq->curr;
-	bool keep_prev, kick_idle = false;
+	bool keep_prev;
 	struct task_struct *p;
 
 	/* see kick_cpus_irq_workfn() */
 	smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
 
-	rq_modified_clear(rq);
+	rq->next_class = &fair_sched_class;
 
 	rq_unpin_lock(rq, rf);
 	balance_one(rq, prev);
@@ -2467,12 +2488,8 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx)
 			refill_task_slice_dfl(rcu_dereference_sched(scx_root), p);
 	} else {
 		p = first_local_task(rq);
-		if (!p) {
-			if (kick_idle)
-				scx_kick_cpu(rcu_dereference_sched(scx_root),
-					     cpu_of(rq), SCX_KICK_IDLE);
+		if (!p)
 			return NULL;
-		}
 
 		if (unlikely(!p->scx.slice)) {
 			struct scx_sched *sch = rcu_dereference_sched(scx_root);
@@ -3075,7 +3092,8 @@ static void switched_from_scx(struct rq *rq, struct task_struct *p)
 	scx_disable_task(p);
 }
 
-static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p,int wake_flags) {}
+static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_flags) {}
+
 static void switched_to_scx(struct rq *rq, struct task_struct *p) {}
 
 int scx_check_setscheduler(struct task_struct *p, int policy)
@@ -3336,8 +3354,6 @@ static void scx_cgroup_unlock(void) {}
  *   their current sched_class. Call them directly from sched core instead.
  */
 DEFINE_SCHED_CLASS(ext) = {
-	.queue_mask		= 1,
-
 	.enqueue_task		= enqueue_task_scx,
 	.dequeue_task		= dequeue_task_scx,
 	.yield_task		= yield_task_scx,
@@ -3575,7 +3591,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
 	int node;
 
 	irq_work_sync(&sch->error_irq_work);
-	kthread_stop(sch->helper->task);
+	kthread_destroy_worker(sch->helper);
 
 	free_percpu(sch->pcpu);
 
@@ -4318,6 +4334,11 @@ static void scx_disable_workfn(struct kthread_work *work)
 	scx_dsp_max_batch = 0;
 	free_kick_syncs();
 
+	if (scx_bypassed_for_enable) {
+		scx_bypassed_for_enable = false;
+		scx_bypass(false);
+	}
+
 	mutex_unlock(&scx_enable_mutex);
 
 	WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING);
@@ -4761,8 +4782,10 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops)
 	}
 
 	sch->pcpu = alloc_percpu(struct scx_sched_pcpu);
-	if (!sch->pcpu)
+	if (!sch->pcpu) {
+		ret = -ENOMEM;
 		goto err_free_gdsqs;
+	}
 
 	sch->helper = kthread_run_worker(0, "sched_ext_helper");
 	if (IS_ERR(sch->helper)) {
@@ -4786,7 +4809,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops)
 	return sch;
 
 err_stop_helper:
-	kthread_stop(sch->helper->task);
+	kthread_destroy_worker(sch->helper);
 err_free_pcpu:
 	free_percpu(sch->pcpu);
 err_free_gdsqs:
@@ -4970,6 +4993,7 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link)
 	 * Init in bypass mode to guarantee forward progress.
 	 */
 	scx_bypass(true);
+	scx_bypassed_for_enable = true;
 
 	for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++)
 		if (((void (**)(void))ops)[i])
@@ -5067,6 +5091,7 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link)
 	scx_task_iter_stop(&sti);
 	percpu_up_write(&scx_fork_rwsem);
 
+	scx_bypassed_for_enable = false;
 	scx_bypass(false);
 
 	if (!scx_tryset_enable_state(SCX_ENABLED, SCX_ENABLING)) {
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index da46c3164537..76f5e4b78b30 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -524,10 +524,48 @@ void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec);
  * Scheduling class tree data structure manipulation methods:
  */
 
+extern void __BUILD_BUG_vruntime_cmp(void);
+
+/* Use __builtin_strcmp() because of __HAVE_ARCH_STRCMP: */
+
+#define vruntime_cmp(A, CMP_STR, B) ({				\
+	int __res = 0;						\
+								\
+	if (!__builtin_strcmp(CMP_STR, "<")) {			\
+		__res = ((s64)((A)-(B)) < 0);			\
+	} else if (!__builtin_strcmp(CMP_STR, "<=")) {		\
+		__res = ((s64)((A)-(B)) <= 0);			\
+	} else if (!__builtin_strcmp(CMP_STR, ">")) {		\
+		__res = ((s64)((A)-(B)) > 0);			\
+	} else if (!__builtin_strcmp(CMP_STR, ">=")) {		\
+		__res = ((s64)((A)-(B)) >= 0);			\
+	} else {						\
+		/* Unknown operator throws linker error: */	\
+		__BUILD_BUG_vruntime_cmp();			\
+	}							\
+								\
+	__res;							\
+})
+
+extern void __BUILD_BUG_vruntime_op(void);
+
+#define vruntime_op(A, OP_STR, B) ({				\
+	s64 __res = 0;						\
+								\
+	if (!__builtin_strcmp(OP_STR, "-")) {			\
+		__res = (s64)((A)-(B));				\
+	} else {						\
+		/* Unknown operator throws linker error: */	\
+		__BUILD_BUG_vruntime_op();			\
+	}							\
+								\
+	__res;						\
+})
+
+
 static inline __maybe_unused u64 max_vruntime(u64 max_vruntime, u64 vruntime)
 {
-	s64 delta = (s64)(vruntime - max_vruntime);
-	if (delta > 0)
+	if (vruntime_cmp(vruntime, ">", max_vruntime))
 		max_vruntime = vruntime;
 
 	return max_vruntime;
@@ -535,8 +573,7 @@ static inline __maybe_unused u64 max_vruntime(u64 max_vruntime, u64 vruntime)
 
 static inline __maybe_unused u64 min_vruntime(u64 min_vruntime, u64 vruntime)
 {
-	s64 delta = (s64)(vruntime - min_vruntime);
-	if (delta < 0)
+	if (vruntime_cmp(vruntime, "<", min_vruntime))
 		min_vruntime = vruntime;
 
 	return min_vruntime;
@@ -549,12 +586,12 @@ static inline bool entity_before(const struct sched_entity *a,
 	 * Tiebreak on vruntime seems unnecessary since it can
 	 * hardly happen.
 	 */
-	return (s64)(a->deadline - b->deadline) < 0;
+	return vruntime_cmp(a->deadline, "<", b->deadline);
 }
 
 static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-	return (s64)(se->vruntime - cfs_rq->zero_vruntime);
+	return vruntime_op(se->vruntime, "-", cfs_rq->zero_vruntime);
 }
 
 #define __node_2_se(node) \
@@ -607,8 +644,8 @@ static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se)
  * Which we track using:
  *
  *                    v0 := cfs_rq->zero_vruntime
- * \Sum (v_i - v0) * w_i := cfs_rq->avg_vruntime
- *              \Sum w_i := cfs_rq->avg_load
+ * \Sum (v_i - v0) * w_i := cfs_rq->sum_w_vruntime
+ *              \Sum w_i := cfs_rq->sum_weight
  *
  * Since zero_vruntime closely tracks the per-task service, these
  * deltas: (v_i - v), will be in the order of the maximal (virtual) lag
@@ -619,32 +656,32 @@ static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se)
  * As measured, the max (key * weight) value was ~44 bits for a kernel build.
  */
 static void
-avg_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se)
+sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
 	unsigned long weight = scale_load_down(se->load.weight);
 	s64 key = entity_key(cfs_rq, se);
 
-	cfs_rq->avg_vruntime += key * weight;
-	cfs_rq->avg_load += weight;
+	cfs_rq->sum_w_vruntime += key * weight;
+	cfs_rq->sum_weight += weight;
 }
 
 static void
-avg_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se)
+sum_w_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
 	unsigned long weight = scale_load_down(se->load.weight);
 	s64 key = entity_key(cfs_rq, se);
 
-	cfs_rq->avg_vruntime -= key * weight;
-	cfs_rq->avg_load -= weight;
+	cfs_rq->sum_w_vruntime -= key * weight;
+	cfs_rq->sum_weight -= weight;
 }
 
 static inline
-void avg_vruntime_update(struct cfs_rq *cfs_rq, s64 delta)
+void sum_w_vruntime_update(struct cfs_rq *cfs_rq, s64 delta)
 {
 	/*
-	 * v' = v + d ==> avg_vruntime' = avg_runtime - d*avg_load
+	 * v' = v + d ==> sum_w_vruntime' = sum_runtime - d*sum_weight
 	 */
-	cfs_rq->avg_vruntime -= cfs_rq->avg_load * delta;
+	cfs_rq->sum_w_vruntime -= cfs_rq->sum_weight * delta;
 }
 
 /*
@@ -654,8 +691,8 @@ void avg_vruntime_update(struct cfs_rq *cfs_rq, s64 delta)
 u64 avg_vruntime(struct cfs_rq *cfs_rq)
 {
 	struct sched_entity *curr = cfs_rq->curr;
-	s64 avg = cfs_rq->avg_vruntime;
-	long load = cfs_rq->avg_load;
+	s64 avg = cfs_rq->sum_w_vruntime;
+	long load = cfs_rq->sum_weight;
 
 	if (curr && curr->on_rq) {
 		unsigned long weight = scale_load_down(curr->load.weight);
@@ -722,8 +759,8 @@ static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se)
 static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime)
 {
 	struct sched_entity *curr = cfs_rq->curr;
-	s64 avg = cfs_rq->avg_vruntime;
-	long load = cfs_rq->avg_load;
+	s64 avg = cfs_rq->sum_w_vruntime;
+	long load = cfs_rq->sum_weight;
 
 	if (curr && curr->on_rq) {
 		unsigned long weight = scale_load_down(curr->load.weight);
@@ -732,7 +769,7 @@ static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime)
 		load += weight;
 	}
 
-	return avg >= (s64)(vruntime - cfs_rq->zero_vruntime) * load;
+	return avg >= vruntime_op(vruntime, "-", cfs_rq->zero_vruntime) * load;
 }
 
 int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se)
@@ -743,9 +780,9 @@ int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se)
 static void update_zero_vruntime(struct cfs_rq *cfs_rq)
 {
 	u64 vruntime = avg_vruntime(cfs_rq);
-	s64 delta = (s64)(vruntime - cfs_rq->zero_vruntime);
+	s64 delta = vruntime_op(vruntime, "-", cfs_rq->zero_vruntime);
 
-	avg_vruntime_update(cfs_rq, delta);
+	sum_w_vruntime_update(cfs_rq, delta);
 
 	cfs_rq->zero_vruntime = vruntime;
 }
@@ -770,13 +807,12 @@ static inline bool __entity_less(struct rb_node *a, const struct rb_node *b)
 	return entity_before(__node_2_se(a), __node_2_se(b));
 }
 
-#define vruntime_gt(field, lse, rse) ({ (s64)((lse)->field - (rse)->field) > 0; })
-
 static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node)
 {
 	if (node) {
 		struct sched_entity *rse = __node_2_se(node);
-		if (vruntime_gt(min_vruntime, se, rse))
+
+		if (vruntime_cmp(se->min_vruntime, ">", rse->min_vruntime))
 			se->min_vruntime = rse->min_vruntime;
 	}
 }
@@ -819,7 +855,7 @@ RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity,
  */
 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-	avg_vruntime_add(cfs_rq, se);
+	sum_w_vruntime_add(cfs_rq, se);
 	update_zero_vruntime(cfs_rq);
 	se->min_vruntime = se->vruntime;
 	se->min_slice = se->slice;
@@ -831,7 +867,7 @@ static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
 	rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline,
 				  &min_vruntime_cb);
-	avg_vruntime_sub(cfs_rq, se);
+	sum_w_vruntime_sub(cfs_rq, se);
 	update_zero_vruntime(cfs_rq);
 }
 
@@ -887,7 +923,7 @@ static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_enti
 
 static inline bool protect_slice(struct sched_entity *se)
 {
-	return ((s64)(se->vprot - se->vruntime) > 0);
+	return vruntime_cmp(se->vruntime, "<", se->vprot);
 }
 
 static inline void cancel_protect_slice(struct sched_entity *se)
@@ -1024,7 +1060,7 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se);
  */
 static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-	if ((s64)(se->vruntime - se->deadline) < 0)
+	if (vruntime_cmp(se->vruntime, "<", se->deadline))
 		return false;
 
 	/*
@@ -1513,7 +1549,7 @@ static unsigned int task_scan_start(struct task_struct *p)
 
 	/* Scale the maximum scan period with the amount of shared memory. */
 	rcu_read_lock();
-	ng = rcu_dereference(p->numa_group);
+	ng = rcu_dereference_all(p->numa_group);
 	if (ng) {
 		unsigned long shared = group_faults_shared(ng);
 		unsigned long private = group_faults_priv(ng);
@@ -1580,7 +1616,7 @@ pid_t task_numa_group_id(struct task_struct *p)
 	pid_t gid = 0;
 
 	rcu_read_lock();
-	ng = rcu_dereference(p->numa_group);
+	ng = rcu_dereference_all(p->numa_group);
 	if (ng)
 		gid = ng->gid;
 	rcu_read_unlock();
@@ -2239,7 +2275,7 @@ static bool task_numa_compare(struct task_numa_env *env,
 		return false;
 
 	rcu_read_lock();
-	cur = rcu_dereference(dst_rq->curr);
+	cur = rcu_dereference_all(dst_rq->curr);
 	if (cur && ((cur->flags & (PF_EXITING | PF_KTHREAD)) ||
 		    !cur->mm))
 		cur = NULL;
@@ -2284,7 +2320,7 @@ static bool task_numa_compare(struct task_numa_env *env,
 	 * If dst and source tasks are in the same NUMA group, or not
 	 * in any group then look only at task weights.
 	 */
-	cur_ng = rcu_dereference(cur->numa_group);
+	cur_ng = rcu_dereference_all(cur->numa_group);
 	if (cur_ng == p_ng) {
 		/*
 		 * Do not swap within a group or between tasks that have
@@ -2499,7 +2535,7 @@ static int task_numa_migrate(struct task_struct *p)
 	 * to satisfy here.
 	 */
 	rcu_read_lock();
-	sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu));
+	sd = rcu_dereference_all(per_cpu(sd_numa, env.src_cpu));
 	if (sd) {
 		env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2;
 		env.imb_numa_nr = sd->imb_numa_nr;
@@ -3022,7 +3058,7 @@ static void task_numa_group(struct task_struct *p, int cpupid, int flags,
 	if (!cpupid_match_pid(tsk, cpupid))
 		goto no_join;
 
-	grp = rcu_dereference(tsk->numa_group);
+	grp = rcu_dereference_all(tsk->numa_group);
 	if (!grp)
 		goto no_join;
 
@@ -3693,7 +3729,7 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
  */
 #define add_positive(_ptr, _val) do {                           \
 	typeof(_ptr) ptr = (_ptr);                              \
-	typeof(_val) val = (_val);                              \
+	__signed_scalar_typeof(*ptr) val = (_val);              \
 	typeof(*ptr) res, var = READ_ONCE(*ptr);                \
 								\
 	res = var + val;                                        \
@@ -3704,23 +3740,6 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 	WRITE_ONCE(*ptr, res);                                  \
 } while (0)
 
-/*
- * Unsigned subtract and clamp on underflow.
- *
- * Explicitly do a load-store to ensure the intermediate value never hits
- * memory. This allows lockless observations without ever seeing the negative
- * values.
- */
-#define sub_positive(_ptr, _val) do {				\
-	typeof(_ptr) ptr = (_ptr);				\
-	typeof(*ptr) val = (_val);				\
-	typeof(*ptr) res, var = READ_ONCE(*ptr);		\
-	res = var - val;					\
-	if (res > var)						\
-		res = 0;					\
-	WRITE_ONCE(*ptr, res);					\
-} while (0)
-
 /*
  * Remove and clamp on negative, from a local variable.
  *
@@ -3732,21 +3751,37 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 	*ptr -= min_t(typeof(*ptr), *ptr, _val);		\
 } while (0)
 
+
+/*
+ * Because of rounding, se->util_sum might ends up being +1 more than
+ * cfs->util_sum. Although this is not a problem by itself, detaching
+ * a lot of tasks with the rounding problem between 2 updates of
+ * util_avg (~1ms) can make cfs->util_sum becoming null whereas
+ * cfs_util_avg is not.
+ *
+ * Check that util_sum is still above its lower bound for the new
+ * util_avg. Given that period_contrib might have moved since the last
+ * sync, we are only sure that util_sum must be above or equal to
+ *    util_avg * minimum possible divider
+ */
+#define __update_sa(sa, name, delta_avg, delta_sum) do {	\
+	add_positive(&(sa)->name##_avg, delta_avg);		\
+	add_positive(&(sa)->name##_sum, delta_sum);		\
+	(sa)->name##_sum = max_t(typeof((sa)->name##_sum),	\
+			       (sa)->name##_sum,		\
+			       (sa)->name##_avg * PELT_MIN_DIVIDER); \
+} while (0)
+
 static inline void
 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-	cfs_rq->avg.load_avg += se->avg.load_avg;
-	cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum;
+	__update_sa(&cfs_rq->avg, load, se->avg.load_avg, se->avg.load_sum);
 }
 
 static inline void
 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-	sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg);
-	sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum);
-	/* See update_cfs_rq_load_avg() */
-	cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum,
-					  cfs_rq->avg.load_avg * PELT_MIN_DIVIDER);
+	__update_sa(&cfs_rq->avg, load, -se->avg.load_avg, -se->avg.load_sum);
 }
 
 static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags);
@@ -4242,7 +4277,6 @@ update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq
 	 */
 	divider = get_pelt_divider(&cfs_rq->avg);
 
-
 	/* Set new sched_entity's utilization */
 	se->avg.util_avg = gcfs_rq->avg.util_avg;
 	new_sum = se->avg.util_avg * divider;
@@ -4250,12 +4284,7 @@ update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq
 	se->avg.util_sum = new_sum;
 
 	/* Update parent cfs_rq utilization */
-	add_positive(&cfs_rq->avg.util_avg, delta_avg);
-	add_positive(&cfs_rq->avg.util_sum, delta_sum);
-
-	/* See update_cfs_rq_load_avg() */
-	cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum,
-					  cfs_rq->avg.util_avg * PELT_MIN_DIVIDER);
+	__update_sa(&cfs_rq->avg, util, delta_avg, delta_sum);
 }
 
 static inline void
@@ -4281,11 +4310,7 @@ update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cf
 	se->avg.runnable_sum = new_sum;
 
 	/* Update parent cfs_rq runnable */
-	add_positive(&cfs_rq->avg.runnable_avg, delta_avg);
-	add_positive(&cfs_rq->avg.runnable_sum, delta_sum);
-	/* See update_cfs_rq_load_avg() */
-	cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum,
-					      cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER);
+	__update_sa(&cfs_rq->avg, runnable, delta_avg, delta_sum);
 }
 
 static inline void
@@ -4349,11 +4374,7 @@ update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq
 
 	se->avg.load_sum = runnable_sum;
 	se->avg.load_avg = load_avg;
-	add_positive(&cfs_rq->avg.load_avg, delta_avg);
-	add_positive(&cfs_rq->avg.load_sum, delta_sum);
-	/* See update_cfs_rq_load_avg() */
-	cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum,
-					  cfs_rq->avg.load_avg * PELT_MIN_DIVIDER);
+	__update_sa(&cfs_rq->avg, load, delta_avg, delta_sum);
 }
 
 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum)
@@ -4450,7 +4471,7 @@ static inline void migrate_se_pelt_lag(struct sched_entity *se)
 	rq = rq_of(cfs_rq);
 
 	rcu_read_lock();
-	is_idle = is_idle_task(rcu_dereference(rq->curr));
+	is_idle = is_idle_task(rcu_dereference_all(rq->curr));
 	rcu_read_unlock();
 
 	/*
@@ -4552,33 +4573,13 @@ update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
 		raw_spin_unlock(&cfs_rq->removed.lock);
 
 		r = removed_load;
-		sub_positive(&sa->load_avg, r);
-		sub_positive(&sa->load_sum, r * divider);
-		/* See sa->util_sum below */
-		sa->load_sum = max_t(u32, sa->load_sum, sa->load_avg * PELT_MIN_DIVIDER);
+		__update_sa(sa, load, -r, -r*divider);
 
 		r = removed_util;
-		sub_positive(&sa->util_avg, r);
-		sub_positive(&sa->util_sum, r * divider);
-		/*
-		 * Because of rounding, se->util_sum might ends up being +1 more than
-		 * cfs->util_sum. Although this is not a problem by itself, detaching
-		 * a lot of tasks with the rounding problem between 2 updates of
-		 * util_avg (~1ms) can make cfs->util_sum becoming null whereas
-		 * cfs_util_avg is not.
-		 * Check that util_sum is still above its lower bound for the new
-		 * util_avg. Given that period_contrib might have moved since the last
-		 * sync, we are only sure that util_sum must be above or equal to
-		 *    util_avg * minimum possible divider
-		 */
-		sa->util_sum = max_t(u32, sa->util_sum, sa->util_avg * PELT_MIN_DIVIDER);
+		__update_sa(sa, util, -r, -r*divider);
 
 		r = removed_runnable;
-		sub_positive(&sa->runnable_avg, r);
-		sub_positive(&sa->runnable_sum, r * divider);
-		/* See sa->util_sum above */
-		sa->runnable_sum = max_t(u32, sa->runnable_sum,
-					      sa->runnable_avg * PELT_MIN_DIVIDER);
+		__update_sa(sa, runnable, -r, -r*divider);
 
 		/*
 		 * removed_runnable is the unweighted version of removed_load so we
@@ -4663,17 +4664,8 @@ static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s
 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
 	dequeue_load_avg(cfs_rq, se);
-	sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg);
-	sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum);
-	/* See update_cfs_rq_load_avg() */
-	cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum,
-					  cfs_rq->avg.util_avg * PELT_MIN_DIVIDER);
-
-	sub_positive(&cfs_rq->avg.runnable_avg, se->avg.runnable_avg);
-	sub_positive(&cfs_rq->avg.runnable_sum, se->avg.runnable_sum);
-	/* See update_cfs_rq_load_avg() */
-	cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum,
-					      cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER);
+	__update_sa(&cfs_rq->avg, util, -se->avg.util_avg, -se->avg.util_sum);
+	__update_sa(&cfs_rq->avg, runnable, -se->avg.runnable_avg, -se->avg.runnable_sum);
 
 	add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum);
 
@@ -5175,7 +5167,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 		 *
 		 *   vl_i = (W + w_i)*vl'_i / W
 		 */
-		load = cfs_rq->avg_load;
+		load = cfs_rq->sum_weight;
 		if (curr && curr->on_rq)
 			load += scale_load_down(curr->load.weight);
 
@@ -7148,7 +7140,7 @@ static DEFINE_PER_CPU(cpumask_var_t, should_we_balance_tmpmask);
 static struct {
 	cpumask_var_t idle_cpus_mask;
 	atomic_t nr_cpus;
-	int has_blocked;		/* Idle CPUS has blocked load */
+	int has_blocked_load;		/* Idle CPUS has blocked load */
 	int needs_update;		/* Newly idle CPUs need their next_balance collated */
 	unsigned long next_balance;     /* in jiffy units */
 	unsigned long next_blocked;	/* Next update of blocked load in jiffies */
@@ -7506,7 +7498,7 @@ static inline void set_idle_cores(int cpu, int val)
 {
 	struct sched_domain_shared *sds;
 
-	sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
+	sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
 	if (sds)
 		WRITE_ONCE(sds->has_idle_cores, val);
 }
@@ -7515,7 +7507,7 @@ static inline bool test_idle_cores(int cpu)
 {
 	struct sched_domain_shared *sds;
 
-	sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
+	sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
 	if (sds)
 		return READ_ONCE(sds->has_idle_cores);
 
@@ -7644,7 +7636,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool
 	cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr);
 
 	if (sched_feat(SIS_UTIL)) {
-		sd_share = rcu_dereference(per_cpu(sd_llc_shared, target));
+		sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, target));
 		if (sd_share) {
 			/* because !--nr is the condition to stop scan */
 			nr = READ_ONCE(sd_share->nr_idle_scan) + 1;
@@ -7850,7 +7842,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
 	 * sd_asym_cpucapacity rather than sd_llc.
 	 */
 	if (sched_asym_cpucap_active()) {
-		sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, target));
+		sd = rcu_dereference_all(per_cpu(sd_asym_cpucapacity, target));
 		/*
 		 * On an asymmetric CPU capacity system where an exclusive
 		 * cpuset defines a symmetric island (i.e. one unique
@@ -7865,7 +7857,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
 		}
 	}
 
-	sd = rcu_dereference(per_cpu(sd_llc, target));
+	sd = rcu_dereference_all(per_cpu(sd_llc, target));
 	if (!sd)
 		return target;
 
@@ -8334,7 +8326,7 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
 	struct energy_env eenv;
 
 	rcu_read_lock();
-	pd = rcu_dereference(rd->pd);
+	pd = rcu_dereference_all(rd->pd);
 	if (!pd)
 		goto unlock;
 
@@ -8342,7 +8334,7 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
 	 * Energy-aware wake-up happens on the lowest sched_domain starting
 	 * from sd_asym_cpucapacity spanning over this_cpu and prev_cpu.
 	 */
-	sd = rcu_dereference(*this_cpu_ptr(&sd_asym_cpucapacity));
+	sd = rcu_dereference_all(*this_cpu_ptr(&sd_asym_cpucapacity));
 	while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd)))
 		sd = sd->parent;
 	if (!sd)
@@ -8744,7 +8736,7 @@ preempt_sync(struct rq *rq, int wake_flags,
 /*
  * Preempt the current task with a newly woken task if needed:
  */
-static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int wake_flags)
+static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_flags)
 {
 	enum preempt_wakeup_action preempt_action = PREEMPT_WAKEUP_PICK;
 	struct task_struct *donor = rq->donor;
@@ -8752,6 +8744,12 @@ static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int
 	struct cfs_rq *cfs_rq = task_cfs_rq(donor);
 	int cse_is_idle, pse_is_idle;
 
+	/*
+	 * XXX Getting preempted by higher class, try and find idle CPU?
+	 */
+	if (p->sched_class != &fair_sched_class)
+		return;
+
 	if (unlikely(se == pse))
 		return;
 
@@ -9349,7 +9347,7 @@ static int task_hot(struct task_struct *p, struct lb_env *env)
  */
 static long migrate_degrades_locality(struct task_struct *p, struct lb_env *env)
 {
-	struct numa_group *numa_group = rcu_dereference(p->numa_group);
+	struct numa_group *numa_group = rcu_dereference_all(p->numa_group);
 	unsigned long src_weight, dst_weight;
 	int src_nid, dst_nid, dist;
 
@@ -9778,7 +9776,7 @@ static void attach_tasks(struct lb_env *env)
 }
 
 #ifdef CONFIG_NO_HZ_COMMON
-static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq)
+static inline bool cfs_rq_has_blocked_load(struct cfs_rq *cfs_rq)
 {
 	if (cfs_rq->avg.load_avg)
 		return true;
@@ -9811,16 +9809,16 @@ static inline void update_blocked_load_tick(struct rq *rq)
 	WRITE_ONCE(rq->last_blocked_load_update_tick, jiffies);
 }
 
-static inline void update_blocked_load_status(struct rq *rq, bool has_blocked)
+static inline void update_has_blocked_load_status(struct rq *rq, bool has_blocked_load)
 {
-	if (!has_blocked)
+	if (!has_blocked_load)
 		rq->has_blocked_load = 0;
 }
 #else /* !CONFIG_NO_HZ_COMMON: */
-static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) { return false; }
+static inline bool cfs_rq_has_blocked_load(struct cfs_rq *cfs_rq) { return false; }
 static inline bool others_have_blocked(struct rq *rq) { return false; }
 static inline void update_blocked_load_tick(struct rq *rq) {}
-static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) {}
+static inline void update_has_blocked_load_status(struct rq *rq, bool has_blocked_load) {}
 #endif /* !CONFIG_NO_HZ_COMMON */
 
 static bool __update_blocked_others(struct rq *rq, bool *done)
@@ -9877,7 +9875,7 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
 			list_del_leaf_cfs_rq(cfs_rq);
 
 		/* Don't need periodic decay once load/util_avg are null */
-		if (cfs_rq_has_blocked(cfs_rq))
+		if (cfs_rq_has_blocked_load(cfs_rq))
 			*done = false;
 	}
 
@@ -9937,7 +9935,7 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
 	bool decayed;
 
 	decayed = update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq);
-	if (cfs_rq_has_blocked(cfs_rq))
+	if (cfs_rq_has_blocked_load(cfs_rq))
 		*done = false;
 
 	return decayed;
@@ -9949,23 +9947,27 @@ static unsigned long task_h_load(struct task_struct *p)
 }
 #endif /* !CONFIG_FAIR_GROUP_SCHED */
 
-static void sched_balance_update_blocked_averages(int cpu)
+static void __sched_balance_update_blocked_averages(struct rq *rq)
 {
 	bool decayed = false, done = true;
-	struct rq *rq = cpu_rq(cpu);
-	struct rq_flags rf;
 
-	rq_lock_irqsave(rq, &rf);
 	update_blocked_load_tick(rq);
-	update_rq_clock(rq);
 
 	decayed |= __update_blocked_others(rq, &done);
 	decayed |= __update_blocked_fair(rq, &done);
 
-	update_blocked_load_status(rq, !done);
+	update_has_blocked_load_status(rq, !done);
 	if (decayed)
 		cpufreq_update_util(rq, 0);
-	rq_unlock_irqrestore(rq, &rf);
+}
+
+static void sched_balance_update_blocked_averages(int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	guard(rq_lock_irqsave)(rq);
+	update_rq_clock(rq);
+	__sched_balance_update_blocked_averages(rq);
 }
 
 /********** Helpers for sched_balance_find_src_group ************************/
@@ -11025,7 +11027,7 @@ static void update_idle_cpu_scan(struct lb_env *env,
 	if (env->sd->span_weight != llc_weight)
 		return;
 
-	sd_share = rcu_dereference(per_cpu(sd_llc_shared, env->dst_cpu));
+	sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, env->dst_cpu));
 	if (!sd_share)
 		return;
 
@@ -11375,7 +11377,7 @@ static struct sched_group *sched_balance_find_src_group(struct lb_env *env)
 		goto force_balance;
 
 	if (!is_rd_overutilized(env->dst_rq->rd) &&
-	    rcu_dereference(env->dst_rq->rd->pd))
+	    rcu_dereference_all(env->dst_rq->rd->pd))
 		goto out_balanced;
 
 	/* ASYM feature bypasses nice load balance check */
@@ -12450,7 +12452,7 @@ static void nohz_balancer_kick(struct rq *rq)
 	if (likely(!atomic_read(&nohz.nr_cpus)))
 		return;
 
-	if (READ_ONCE(nohz.has_blocked) &&
+	if (READ_ONCE(nohz.has_blocked_load) &&
 	    time_after(now, READ_ONCE(nohz.next_blocked)))
 		flags = NOHZ_STATS_KICK;
 
@@ -12464,7 +12466,7 @@ static void nohz_balancer_kick(struct rq *rq)
 
 	rcu_read_lock();
 
-	sd = rcu_dereference(rq->sd);
+	sd = rcu_dereference_all(rq->sd);
 	if (sd) {
 		/*
 		 * If there's a runnable CFS task and the current CPU has reduced
@@ -12476,7 +12478,7 @@ static void nohz_balancer_kick(struct rq *rq)
 		}
 	}
 
-	sd = rcu_dereference(per_cpu(sd_asym_packing, cpu));
+	sd = rcu_dereference_all(per_cpu(sd_asym_packing, cpu));
 	if (sd) {
 		/*
 		 * When ASYM_PACKING; see if there's a more preferred CPU
@@ -12494,7 +12496,7 @@ static void nohz_balancer_kick(struct rq *rq)
 		}
 	}
 
-	sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, cpu));
+	sd = rcu_dereference_all(per_cpu(sd_asym_cpucapacity, cpu));
 	if (sd) {
 		/*
 		 * When ASYM_CPUCAPACITY; see if there's a higher capacity CPU
@@ -12515,7 +12517,7 @@ static void nohz_balancer_kick(struct rq *rq)
 		goto unlock;
 	}
 
-	sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
+	sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
 	if (sds) {
 		/*
 		 * If there is an imbalance between LLC domains (IOW we could
@@ -12547,7 +12549,7 @@ static void set_cpu_sd_state_busy(int cpu)
 	struct sched_domain *sd;
 
 	rcu_read_lock();
-	sd = rcu_dereference(per_cpu(sd_llc, cpu));
+	sd = rcu_dereference_all(per_cpu(sd_llc, cpu));
 
 	if (!sd || !sd->nohz_idle)
 		goto unlock;
@@ -12577,7 +12579,7 @@ static void set_cpu_sd_state_idle(int cpu)
 	struct sched_domain *sd;
 
 	rcu_read_lock();
-	sd = rcu_dereference(per_cpu(sd_llc, cpu));
+	sd = rcu_dereference_all(per_cpu(sd_llc, cpu));
 
 	if (!sd || sd->nohz_idle)
 		goto unlock;
@@ -12611,9 +12613,9 @@ void nohz_balance_enter_idle(int cpu)
 
 	/*
 	 * The tick is still stopped but load could have been added in the
-	 * meantime. We set the nohz.has_blocked flag to trig a check of the
+	 * meantime. We set the nohz.has_blocked_load flag to trig a check of the
 	 * *_avg. The CPU is already part of nohz.idle_cpus_mask so the clear
-	 * of nohz.has_blocked can only happen after checking the new load
+	 * of nohz.has_blocked_load can only happen after checking the new load
 	 */
 	if (rq->nohz_tick_stopped)
 		goto out;
@@ -12629,7 +12631,7 @@ void nohz_balance_enter_idle(int cpu)
 
 	/*
 	 * Ensures that if nohz_idle_balance() fails to observe our
-	 * @idle_cpus_mask store, it must observe the @has_blocked
+	 * @idle_cpus_mask store, it must observe the @has_blocked_load
 	 * and @needs_update stores.
 	 */
 	smp_mb__after_atomic();
@@ -12642,7 +12644,7 @@ void nohz_balance_enter_idle(int cpu)
 	 * Each time a cpu enter idle, we assume that it has blocked load and
 	 * enable the periodic update of the load of idle CPUs
 	 */
-	WRITE_ONCE(nohz.has_blocked, 1);
+	WRITE_ONCE(nohz.has_blocked_load, 1);
 }
 
 static bool update_nohz_stats(struct rq *rq)
@@ -12683,8 +12685,8 @@ static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags)
 
 	/*
 	 * We assume there will be no idle load after this update and clear
-	 * the has_blocked flag. If a cpu enters idle in the mean time, it will
-	 * set the has_blocked flag and trigger another update of idle load.
+	 * the has_blocked_load flag. If a cpu enters idle in the mean time, it will
+	 * set the has_blocked_load flag and trigger another update of idle load.
 	 * Because a cpu that becomes idle, is added to idle_cpus_mask before
 	 * setting the flag, we are sure to not clear the state and not
 	 * check the load of an idle cpu.
@@ -12692,12 +12694,12 @@ static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags)
 	 * Same applies to idle_cpus_mask vs needs_update.
 	 */
 	if (flags & NOHZ_STATS_KICK)
-		WRITE_ONCE(nohz.has_blocked, 0);
+		WRITE_ONCE(nohz.has_blocked_load, 0);
 	if (flags & NOHZ_NEXT_KICK)
 		WRITE_ONCE(nohz.needs_update, 0);
 
 	/*
-	 * Ensures that if we miss the CPU, we must see the has_blocked
+	 * Ensures that if we miss the CPU, we must see the has_blocked_load
 	 * store from nohz_balance_enter_idle().
 	 */
 	smp_mb();
@@ -12764,7 +12766,7 @@ static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags)
 abort:
 	/* There is still blocked load, enable periodic update */
 	if (has_blocked_load)
-		WRITE_ONCE(nohz.has_blocked, 1);
+		WRITE_ONCE(nohz.has_blocked_load, 1);
 }
 
 /*
@@ -12826,7 +12828,7 @@ static void nohz_newidle_balance(struct rq *this_rq)
 		return;
 
 	/* Don't need to update blocked load of idle CPUs*/
-	if (!READ_ONCE(nohz.has_blocked) ||
+	if (!READ_ONCE(nohz.has_blocked_load) ||
 	    time_before(jiffies, READ_ONCE(nohz.next_blocked)))
 		return;
 
@@ -12896,29 +12898,28 @@ static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf)
 	 */
 	rq_unpin_lock(this_rq, rf);
 
-	rcu_read_lock();
-	sd = rcu_dereference_check_sched_domain(this_rq->sd);
-	if (!sd) {
-		rcu_read_unlock();
+	sd = rcu_dereference_sched_domain(this_rq->sd);
+	if (!sd)
 		goto out;
-	}
 
 	if (!get_rd_overloaded(this_rq->rd) ||
 	    this_rq->avg_idle < sd->max_newidle_lb_cost) {
 
 		update_next_balance(sd, &next_balance);
-		rcu_read_unlock();
 		goto out;
 	}
-	rcu_read_unlock();
-
-	rq_modified_clear(this_rq);
-	raw_spin_rq_unlock(this_rq);
 
+	/*
+	 * Include sched_balance_update_blocked_averages() in the cost
+	 * calculation because it can be quite costly -- this ensures we skip
+	 * it when avg_idle gets to be very low.
+	 */
 	t0 = sched_clock_cpu(this_cpu);
-	sched_balance_update_blocked_averages(this_cpu);
+	__sched_balance_update_blocked_averages(this_rq);
+
+	this_rq->next_class = &fair_sched_class;
+	raw_spin_rq_unlock(this_rq);
 
-	rcu_read_lock();
 	for_each_domain(this_cpu, sd) {
 		u64 domain_cost;
 
@@ -12968,7 +12969,6 @@ static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf)
 		if (pulled_task || !continue_balancing)
 			break;
 	}
-	rcu_read_unlock();
 
 	raw_spin_rq_lock(this_rq);
 
@@ -12984,7 +12984,7 @@ static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf)
 		pulled_task = 1;
 
 	/* If a higher prio class was modified, restart the pick */
-	if (rq_modified_above(this_rq, &fair_sched_class))
+	if (sched_class_above(this_rq->next_class, &fair_sched_class))
 		pulled_task = -1;
 
 out:
@@ -13335,8 +13335,8 @@ bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b,
 	 * zero_vruntime_fi, which would have been updated in prior calls
 	 * to se_fi_update().
 	 */
-	delta = (s64)(sea->vruntime - seb->vruntime) +
-		(s64)(cfs_rqb->zero_vruntime_fi - cfs_rqa->zero_vruntime_fi);
+	delta = vruntime_op(sea->vruntime, "-", seb->vruntime) +
+		vruntime_op(cfs_rqb->zero_vruntime_fi, "-", cfs_rqa->zero_vruntime_fi);
 
 	return delta > 0;
 }
@@ -13374,6 +13374,12 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
 		entity_tick(cfs_rq, se, queued);
 	}
 
+	if (queued) {
+		if (!need_resched())
+			hrtick_start_fair(rq, curr);
+		return;
+	}
+
 	if (static_branch_unlikely(&sched_numa_balancing))
 		task_tick_numa(rq, curr);
 
@@ -13882,15 +13888,12 @@ static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task
  * All the scheduling class methods:
  */
 DEFINE_SCHED_CLASS(fair) = {
-
-	.queue_mask		= 2,
-
 	.enqueue_task		= enqueue_task_fair,
 	.dequeue_task		= dequeue_task_fair,
 	.yield_task		= yield_task_fair,
 	.yield_to_task		= yield_to_task_fair,
 
-	.wakeup_preempt		= check_preempt_wakeup_fair,
+	.wakeup_preempt		= wakeup_preempt_fair,
 
 	.pick_task		= pick_task_fair,
 	.pick_next_task		= pick_next_task_fair,
@@ -13950,7 +13953,7 @@ void show_numa_stats(struct task_struct *p, struct seq_file *m)
 	struct numa_group *ng;
 
 	rcu_read_lock();
-	ng = rcu_dereference(p->numa_group);
+	ng = rcu_dereference_all(p->numa_group);
 	for_each_online_node(node) {
 		if (p->numa_faults) {
 			tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)];
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index c174afe1dd17..65eb8f8c1a5d 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -536,9 +536,6 @@ static void update_curr_idle(struct rq *rq)
  * Simple, special scheduling class for the per-CPU idle tasks:
  */
 DEFINE_SCHED_CLASS(idle) = {
-
-	.queue_mask		= 0,
-
 	/* no enqueue/yield_task for idle tasks */
 
 	/* dequeue is not valid, we print a debug message there: */
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index f1867fe8e5c5..0a9b2cd6da72 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -1615,6 +1615,12 @@ static void wakeup_preempt_rt(struct rq *rq, struct task_struct *p, int flags)
 {
 	struct task_struct *donor = rq->donor;
 
+	/*
+	 * XXX If we're preempted by DL, queue a push?
+	 */
+	if (p->sched_class != &rt_sched_class)
+		return;
+
 	if (p->prio < donor->prio) {
 		resched_curr(rq);
 		return;
@@ -2568,9 +2574,6 @@ static int task_is_throttled_rt(struct task_struct *p, int cpu)
 #endif /* CONFIG_SCHED_CORE */
 
 DEFINE_SCHED_CLASS(rt) = {
-
-	.queue_mask		= 4,
-
 	.enqueue_task		= enqueue_task_rt,
 	.dequeue_task		= dequeue_task_rt,
 	.yield_task		= yield_task_rt,
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index d30cca6870f5..bdb1e748fe5c 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -670,16 +670,16 @@ struct balance_callback {
 	void (*func)(struct rq *rq);
 };
 
-/* CFS-related fields in a runqueue */
+/* Fair scheduling SCHED_{NORMAL,BATCH,IDLE} related fields in a runqueue: */
 struct cfs_rq {
 	struct load_weight	load;
 	unsigned int		nr_queued;
-	unsigned int		h_nr_queued;       /* SCHED_{NORMAL,BATCH,IDLE} */
-	unsigned int		h_nr_runnable;     /* SCHED_{NORMAL,BATCH,IDLE} */
-	unsigned int		h_nr_idle; /* SCHED_IDLE */
+	unsigned int		h_nr_queued;		/* SCHED_{NORMAL,BATCH,IDLE} */
+	unsigned int		h_nr_runnable;		/* SCHED_{NORMAL,BATCH,IDLE} */
+	unsigned int		h_nr_idle;		/* SCHED_IDLE */
 
-	s64			avg_vruntime;
-	u64			avg_load;
+	s64			sum_w_vruntime;
+	u64			sum_weight;
 
 	u64			zero_vruntime;
 #ifdef CONFIG_SCHED_CORE
@@ -690,7 +690,7 @@ struct cfs_rq {
 	struct rb_root_cached	tasks_timeline;
 
 	/*
-	 * 'curr' points to currently running entity on this cfs_rq.
+	 * 'curr' points to the currently running entity on this cfs_rq.
 	 * It is set to NULL otherwise (i.e when none are currently running).
 	 */
 	struct sched_entity	*curr;
@@ -726,9 +726,7 @@ struct cfs_rq {
 	unsigned long		h_load;
 	u64			last_h_load_update;
 	struct sched_entity	*h_load_next;
-#endif /* CONFIG_FAIR_GROUP_SCHED */
 
-#ifdef CONFIG_FAIR_GROUP_SCHED
 	struct rq		*rq;	/* CPU runqueue to which this cfs_rq is attached */
 
 	/*
@@ -741,19 +739,19 @@ struct cfs_rq {
 	 */
 	int			on_list;
 	struct list_head	leaf_cfs_rq_list;
-	struct task_group	*tg;	/* group that "owns" this runqueue */
+	struct task_group	*tg;	/* Group that "owns" this runqueue */
 
 	/* Locally cached copy of our task_group's idle value */
 	int			idle;
 
-#ifdef CONFIG_CFS_BANDWIDTH
+# ifdef CONFIG_CFS_BANDWIDTH
 	int			runtime_enabled;
 	s64			runtime_remaining;
 
 	u64			throttled_pelt_idle;
-#ifndef CONFIG_64BIT
+#  ifndef CONFIG_64BIT
 	u64                     throttled_pelt_idle_copy;
-#endif
+#  endif
 	u64			throttled_clock;
 	u64			throttled_clock_pelt;
 	u64			throttled_clock_pelt_time;
@@ -765,7 +763,7 @@ struct cfs_rq {
 	struct list_head	throttled_list;
 	struct list_head	throttled_csd_list;
 	struct list_head        throttled_limbo_list;
-#endif /* CONFIG_CFS_BANDWIDTH */
+# endif /* CONFIG_CFS_BANDWIDTH */
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 };
 
@@ -1121,7 +1119,6 @@ struct rq {
 	raw_spinlock_t		__lock;
 
 	/* Per class runqueue modification mask; bits in class order. */
-	unsigned int		queue_mask;
 	unsigned int		nr_running;
 #ifdef CONFIG_NUMA_BALANCING
 	unsigned int		nr_numa_running;
@@ -1181,6 +1178,7 @@ struct rq {
 	struct sched_dl_entity	*dl_server;
 	struct task_struct	*idle;
 	struct task_struct	*stop;
+	const struct sched_class *next_class;
 	unsigned long		next_balance;
 	struct mm_struct	*prev_mm;
 
@@ -2010,8 +2008,8 @@ queue_balance_callback(struct rq *rq,
 	rq->balance_callback = head;
 }
 
-#define rcu_dereference_check_sched_domain(p) \
-	rcu_dereference_check((p), lockdep_is_held(&sched_domains_mutex))
+#define rcu_dereference_sched_domain(p) \
+	rcu_dereference_all_check((p), lockdep_is_held(&sched_domains_mutex))
 
 /*
  * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
@@ -2021,7 +2019,7 @@ queue_balance_callback(struct rq *rq,
  * preempt-disabled sections.
  */
 #define for_each_domain(cpu, __sd) \
-	for (__sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); \
+	for (__sd = rcu_dereference_sched_domain(cpu_rq(cpu)->sd); \
 			__sd; __sd = __sd->parent)
 
 /* A mask of all the SD flags that have the SDF_SHARED_CHILD metaflag */
@@ -2428,15 +2426,6 @@ struct sched_class {
 #ifdef CONFIG_UCLAMP_TASK
 	int uclamp_enabled;
 #endif
-	/*
-	 * idle:  0
-	 * ext:   1
-	 * fair:  2
-	 * rt:    4
-	 * dl:    8
-	 * stop: 16
-	 */
-	unsigned int queue_mask;
 
 	/*
 	 * move_queued_task/activate_task/enqueue_task: rq->lock
@@ -2595,20 +2584,6 @@ struct sched_class {
 #endif
 };
 
-/*
- * Does not nest; only used around sched_class::pick_task() rq-lock-breaks.
- */
-static inline void rq_modified_clear(struct rq *rq)
-{
-	rq->queue_mask = 0;
-}
-
-static inline bool rq_modified_above(struct rq *rq, const struct sched_class * class)
-{
-	unsigned int mask = class->queue_mask;
-	return rq->queue_mask & ~((mask << 1) - 1);
-}
-
 static inline void put_prev_task(struct rq *rq, struct task_struct *prev)
 {
 	WARN_ON_ONCE(rq->donor != prev);
@@ -3901,6 +3876,7 @@ void move_queued_task_locked(struct rq *src_rq, struct rq *dst_rq, struct task_s
 	deactivate_task(src_rq, task, 0);
 	set_task_cpu(task, dst_rq->cpu);
 	activate_task(dst_rq, task, 0);
+	wakeup_preempt(dst_rq, task, 0);
 }
 
 static inline
@@ -3968,6 +3944,7 @@ extern void balance_callbacks(struct rq *rq, struct balance_callback *head);
 struct sched_change_ctx {
 	u64			prio;
 	struct task_struct	*p;
+	const struct sched_class *class;
 	int			flags;
 	bool			queued;
 	bool			running;
diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c
index 4f9192be4b5b..f95798baddeb 100644
--- a/kernel/sched/stop_task.c
+++ b/kernel/sched/stop_task.c
@@ -97,9 +97,6 @@ static void update_curr_stop(struct rq *rq)
  * Simple, special scheduling class for the per-CPU stop tasks:
  */
 DEFINE_SCHED_CLASS(stop) = {
-
-	.queue_mask		= 16,
-
 	.enqueue_task		= enqueue_task_stop,
 	.dequeue_task		= dequeue_task_stop,
 	.yield_task		= yield_task_stop,
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 77198911b8dd..5f435c1e48d8 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -576,6 +576,10 @@ static inline bool lockdep_softirq_start(void) { return false; }
 static inline void lockdep_softirq_end(bool in_hardirq) { }
 #endif
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+static noinline void handle_softirqs(bool ksirqd);
+#endif
+
 static void handle_softirqs(bool ksirqd)
 {
 	unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
diff --git a/kernel/sys.c b/kernel/sys.c
index 8b58eece4e58..9071422c1609 100644
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -2388,6 +2388,21 @@ int __weak arch_lock_shadow_stack_status(struct task_struct *t, unsigned long st
 	return -EINVAL;
 }
 
+int __weak arch_get_indir_br_lp_status(struct task_struct *t, unsigned long __user *status)
+{
+	return -EINVAL;
+}
+
+int __weak arch_set_indir_br_lp_status(struct task_struct *t, unsigned long status)
+{
+	return -EINVAL;
+}
+
+int __weak arch_lock_indir_br_lp_status(struct task_struct *t, unsigned long status)
+{
+	return -EINVAL;
+}
+
 #define PR_IO_FLUSHER (PF_MEMALLOC_NOIO | PF_LOCAL_THROTTLE)
 
 static int prctl_set_vma(unsigned long opt, unsigned long addr,
@@ -2868,6 +2883,21 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
 	case PR_FUTEX_HASH:
 		error = futex_hash_prctl(arg2, arg3, arg4);
 		break;
+	case PR_GET_INDIR_BR_LP_STATUS:
+		if (arg3 || arg4 || arg5)
+			return -EINVAL;
+		error = arch_get_indir_br_lp_status(me, (unsigned long __user *)arg2);
+		break;
+	case PR_SET_INDIR_BR_LP_STATUS:
+		if (arg3 || arg4 || arg5)
+			return -EINVAL;
+		error = arch_set_indir_br_lp_status(me, arg2);
+		break;
+	case PR_LOCK_INDIR_BR_LP_STATUS:
+		if (arg3 || arg4 || arg5)
+			return -EINVAL;
+		error = arch_lock_indir_br_lp_status(me, arg2);
+		break;
 	default:
 		trace_task_prctl_unknown(option, arg2, arg3, arg4, arg5);
 		error = -EINVAL;
diff --git a/kernel/time/timecounter.c b/kernel/time/timecounter.c
index 3d2a354cfe1c..2e64dbb6302d 100644
--- a/kernel/time/timecounter.c
+++ b/kernel/time/timecounter.c
@@ -62,38 +62,3 @@ u64 timecounter_read(struct timecounter *tc)
 }
 EXPORT_SYMBOL_GPL(timecounter_read);
 
-/*
- * This is like cyclecounter_cyc2ns(), but it is used for computing a
- * time previous to the time stored in the cycle counter.
- */
-static u64 cc_cyc2ns_backwards(const struct cyclecounter *cc,
-			       u64 cycles, u64 mask, u64 frac)
-{
-	u64 ns = (u64) cycles;
-
-	ns = ((ns * cc->mult) - frac) >> cc->shift;
-
-	return ns;
-}
-
-u64 timecounter_cyc2time(const struct timecounter *tc,
-			 u64 cycle_tstamp)
-{
-	u64 delta = (cycle_tstamp - tc->cycle_last) & tc->cc->mask;
-	u64 nsec = tc->nsec, frac = tc->frac;
-
-	/*
-	 * Instead of always treating cycle_tstamp as more recent
-	 * than tc->cycle_last, detect when it is too far in the
-	 * future and treat it as old time stamp instead.
-	 */
-	if (delta > tc->cc->mask / 2) {
-		delta = (tc->cycle_last - cycle_tstamp) & tc->cc->mask;
-		nsec -= cc_cyc2ns_backwards(tc->cc, delta, tc->mask, frac);
-	} else {
-		nsec += cyclecounter_cyc2ns(tc->cc, delta, tc->mask, &frac);
-	}
-
-	return nsec;
-}
-EXPORT_SYMBOL_GPL(timecounter_cyc2time);
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index d57727abaade..fe28d86f7c35 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -965,7 +965,7 @@ static const struct bpf_func_proto bpf_d_path_proto = {
 	.ret_type	= RET_INTEGER,
 	.arg1_type	= ARG_PTR_TO_BTF_ID,
 	.arg1_btf_id	= &bpf_d_path_btf_ids[0],
-	.arg2_type	= ARG_PTR_TO_MEM,
+	.arg2_type	= ARG_PTR_TO_MEM | MEM_WRITE,
 	.arg3_type	= ARG_CONST_SIZE_OR_ZERO,
 	.allowed	= bpf_d_path_allowed,
 };
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 3ec2033c0774..5d6fc2f9e12d 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -4518,8 +4518,11 @@ static int t_show(struct seq_file *m, void *v)
 			unsigned long direct;
 
 			direct = ftrace_find_rec_direct(rec->ip);
-			if (direct)
-				seq_printf(m, "\n\tdirect-->%pS", (void *)direct);
+			if (direct) {
+				seq_printf(m, "\n\tdirect%s-->%pS",
+					   ftrace_is_jmp(direct) ? "(jmp)" : "",
+					   (void *)ftrace_jmp_get(direct));
+			}
 		}
 	}
 
@@ -7749,7 +7752,8 @@ ftrace_func_address_lookup(struct ftrace_mod_map *mod_map,
 
 int
 ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
-		   unsigned long *off, char **modname, char *sym)
+			  unsigned long *off, char **modname,
+			  const unsigned char **modbuildid, char *sym)
 {
 	struct ftrace_mod_map *mod_map;
 	int ret = 0;
@@ -7761,6 +7765,8 @@ ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
 		if (ret) {
 			if (modname)
 				*modname = mod_map->mod->name;
+			if (modbuildid)
+				*modbuildid = module_buildid(mod_map->mod);
 			break;
 		}
 	}
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index e575956ef9b5..6f2148df14d9 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -10507,7 +10507,7 @@ static int __remove_instance(struct trace_array *tr)
 
 	/* Disable all the flags that were enabled coming in */
 	for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
-		if ((1 << i) & ZEROED_TRACE_FLAGS)
+		if ((1ULL << i) & ZEROED_TRACE_FLAGS)
 			set_tracer_flag(tr, 1ULL << i, 0);
 	}
 
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
index b16a5a158040..76067529db61 100644
--- a/kernel/trace/trace_events.c
+++ b/kernel/trace/trace_events.c
@@ -700,6 +700,8 @@ int trace_event_reg(struct trace_event_call *call,
 
 #ifdef CONFIG_PERF_EVENTS
 	case TRACE_REG_PERF_REGISTER:
+		if (!call->class->perf_probe)
+			return -ENODEV;
 		return tracepoint_probe_register(call->tp,
 						 call->class->perf_probe,
 						 call);
diff --git a/kernel/vmcore_info.c b/kernel/vmcore_info.c
index fe9bf8db1922..22b3205dd4dc 100644
--- a/kernel/vmcore_info.c
+++ b/kernel/vmcore_info.c
@@ -137,7 +137,9 @@ EXPORT_SYMBOL_GPL(hwerr_log_error_type);
 
 static int __init crash_save_vmcoreinfo_init(void)
 {
-	vmcoreinfo_data = (unsigned char *)get_zeroed_page(GFP_KERNEL);
+	int order;
+	order = get_order(VMCOREINFO_BYTES);
+	vmcoreinfo_data = (unsigned char *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
 	if (!vmcoreinfo_data) {
 		pr_warn("Memory allocation for vmcoreinfo_data failed\n");
 		return -ENOMEM;
@@ -146,7 +148,7 @@ static int __init crash_save_vmcoreinfo_init(void)
 	vmcoreinfo_note = alloc_pages_exact(VMCOREINFO_NOTE_SIZE,
 						GFP_KERNEL | __GFP_ZERO);
 	if (!vmcoreinfo_note) {
-		free_page((unsigned long)vmcoreinfo_data);
+		free_pages((unsigned long)vmcoreinfo_data, order);
 		vmcoreinfo_data = NULL;
 		pr_warn("Memory allocation for vmcoreinfo_note failed\n");
 		return -ENOMEM;
diff --git a/kernel/watchdog.c b/kernel/watchdog.c
index 0685e3a8aa0a..3712fdecb29e 100644
--- a/kernel/watchdog.c
+++ b/kernel/watchdog.c
@@ -66,13 +66,6 @@ int __read_mostly sysctl_hardlockup_all_cpu_backtrace;
 unsigned int __read_mostly hardlockup_panic =
 			IS_ENABLED(CONFIG_BOOTPARAM_HARDLOCKUP_PANIC);
 
-/*
- * bitmasks to control what kinds of system info to be printed when
- * hard lockup is detected, it could be task, memory, lock etc.
- * Refer include/linux/sys_info.h for detailed bit definition.
- */
-static unsigned long hardlockup_si_mask;
-
 #ifdef CONFIG_SYSFS
 
 static unsigned int hardlockup_count;
@@ -135,6 +128,13 @@ __setup("nmi_watchdog=", hardlockup_panic_setup);
 
 #if defined(CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER)
 
+/*
+ * bitmask to control what kinds of system info to be printed when
+ * hard lockup is detected, it could be task, memory, lock etc.
+ * Refer include/linux/sys_info.h for detailed bit definition.
+ */
+static unsigned long hardlockup_si_mask;
+
 static DEFINE_PER_CPU(atomic_t, hrtimer_interrupts);
 static DEFINE_PER_CPU(int, hrtimer_interrupts_saved);
 static DEFINE_PER_CPU(bool, watchdog_hardlockup_warned);
@@ -363,7 +363,7 @@ static struct cpumask watchdog_allowed_mask __read_mostly;
 
 /* Global variables, exported for sysctl */
 unsigned int __read_mostly softlockup_panic =
-			IS_ENABLED(CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC);
+			CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC;
 
 static bool softlockup_initialized __read_mostly;
 static u64 __read_mostly sample_period;
@@ -879,7 +879,9 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
 
 		add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
 		sys_info(softlockup_si_mask & ~SYS_INFO_ALL_BT);
-		if (softlockup_panic)
+		duration = duration / get_softlockup_thresh();
+
+		if (softlockup_panic && duration >= softlockup_panic)
 			panic("softlockup: hung tasks");
 	}
 
@@ -1228,7 +1230,7 @@ static const struct ctl_table watchdog_sysctls[] = {
 		.mode		= 0644,
 		.proc_handler	= proc_dointvec_minmax,
 		.extra1		= SYSCTL_ZERO,
-		.extra2		= SYSCTL_ONE,
+		.extra2		= SYSCTL_INT_MAX,
 	},
 	{
 		.procname	= "softlockup_sys_info",
@@ -1259,6 +1261,7 @@ static const struct ctl_table watchdog_sysctls[] = {
 		.extra1		= SYSCTL_ZERO,
 		.extra2		= SYSCTL_ONE,
 	},
+#if defined(CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER)
 	{
 		.procname	= "hardlockup_sys_info",
 		.data		= &hardlockup_si_mask,
@@ -1266,6 +1269,7 @@ static const struct ctl_table watchdog_sysctls[] = {
 		.mode		= 0644,
 		.proc_handler	= sysctl_sys_info_handler,
 	},
+#endif
 #ifdef CONFIG_SMP
 	{
 		.procname	= "hardlockup_all_cpu_backtrace",
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 253311af47c6..0349dbcb51f3 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -117,6 +117,8 @@ enum wq_internal_consts {
 	MAYDAY_INTERVAL		= HZ / 10,	/* and then every 100ms */
 	CREATE_COOLDOWN		= HZ,		/* time to breath after fail */
 
+	RESCUER_BATCH		= 16,		/* process items per turn */
+
 	/*
 	 * Rescue workers are used only on emergencies and shared by
 	 * all cpus.  Give MIN_NICE.
@@ -286,6 +288,7 @@ struct pool_workqueue {
 	struct list_head	pending_node;	/* LN: node on wq_node_nr_active->pending_pwqs */
 	struct list_head	pwqs_node;	/* WR: node on wq->pwqs */
 	struct list_head	mayday_node;	/* MD: node on wq->maydays */
+	struct work_struct	mayday_cursor;	/* L: cursor on pool->worklist */
 
 	u64			stats[PWQ_NR_STATS];
 
@@ -1120,6 +1123,12 @@ static struct worker *find_worker_executing_work(struct worker_pool *pool,
 	return NULL;
 }
 
+static void mayday_cursor_func(struct work_struct *work)
+{
+	/* should not be processed, only for marking position */
+	BUG();
+}
+
 /**
  * move_linked_works - move linked works to a list
  * @work: start of series of works to be scheduled
@@ -1182,6 +1191,16 @@ static bool assign_work(struct work_struct *work, struct worker *worker,
 
 	lockdep_assert_held(&pool->lock);
 
+	/* The cursor work should not be processed */
+	if (unlikely(work->func == mayday_cursor_func)) {
+		/* only worker_thread() can possibly take this branch */
+		WARN_ON_ONCE(worker->rescue_wq);
+		if (nextp)
+			*nextp = list_next_entry(work, entry);
+		list_del_init(&work->entry);
+		return false;
+	}
+
 	/*
 	 * A single work shouldn't be executed concurrently by multiple workers.
 	 * __queue_work() ensures that @work doesn't jump to a different pool
@@ -2976,9 +2995,8 @@ static void idle_cull_fn(struct work_struct *work)
 	reap_dying_workers(&cull_list);
 }
 
-static void send_mayday(struct work_struct *work)
+static void send_mayday(struct pool_workqueue *pwq)
 {
-	struct pool_workqueue *pwq = get_work_pwq(work);
 	struct workqueue_struct *wq = pwq->wq;
 
 	lockdep_assert_held(&wq_mayday_lock);
@@ -3016,7 +3034,7 @@ static void pool_mayday_timeout(struct timer_list *t)
 		 * rescuers.
 		 */
 		list_for_each_entry(work, &pool->worklist, entry)
-			send_mayday(work);
+			send_mayday(get_work_pwq(work));
 	}
 
 	raw_spin_unlock(&wq_mayday_lock);
@@ -3139,6 +3157,10 @@ static bool manage_workers(struct worker *worker)
 	return true;
 }
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+static noinline void process_one_work(struct worker *worker, struct work_struct *work);
+#endif
+
 /**
  * process_one_work - process single work
  * @worker: self
@@ -3440,22 +3462,57 @@ static int worker_thread(void *__worker)
 static bool assign_rescuer_work(struct pool_workqueue *pwq, struct worker *rescuer)
 {
 	struct worker_pool *pool = pwq->pool;
+	struct work_struct *cursor = &pwq->mayday_cursor;
 	struct work_struct *work, *n;
 
-	/* need rescue? */
-	if (!pwq->nr_active || !need_to_create_worker(pool))
+	/* have work items to rescue? */
+	if (!pwq->nr_active)
 		return false;
 
-	/*
-	 * Slurp in all works issued via this workqueue and
-	 * process'em.
-	 */
-	list_for_each_entry_safe(work, n, &pool->worklist, entry) {
-		if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n))
+	/* need rescue? */
+	if (!need_to_create_worker(pool)) {
+		/*
+		 * The pool has idle workers and doesn't need the rescuer, so it
+		 * could simply return false here.
+		 *
+		 * However, the memory pressure might not be fully relieved.
+		 * In PERCPU pool with concurrency enabled, having idle workers
+		 * does not necessarily mean memory pressure is gone; it may
+		 * simply mean regular workers have woken up, completed their
+		 * work, and gone idle again due to concurrency limits.
+		 *
+		 * In this case, those working workers may later sleep again,
+		 * the pool may run out of idle workers, and it will have to
+		 * allocate new ones and wait for the timer to send mayday,
+		 * causing unnecessary delay - especially if memory pressure
+		 * was never resolved throughout.
+		 *
+		 * Do more work if memory pressure is still on to reduce
+		 * relapse, using (pool->flags & POOL_MANAGER_ACTIVE), though
+		 * not precisely, unless there are other PWQs needing help.
+		 */
+		if (!(pool->flags & POOL_MANAGER_ACTIVE) ||
+		    !list_empty(&pwq->wq->maydays))
+			return false;
+	}
+
+	/* search from the start or cursor if available */
+	if (list_empty(&cursor->entry))
+		work = list_first_entry(&pool->worklist, struct work_struct, entry);
+	else
+		work = list_next_entry(cursor, entry);
+
+	/* find the next work item to rescue */
+	list_for_each_entry_safe_from(work, n, &pool->worklist, entry) {
+		if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n)) {
 			pwq->stats[PWQ_STAT_RESCUED]++;
+			/* put the cursor for next search */
+			list_move_tail(&cursor->entry, &n->entry);
+			return true;
+		}
 	}
 
-	return !list_empty(&rescuer->scheduled);
+	return false;
 }
 
 /**
@@ -3512,6 +3569,7 @@ static int rescuer_thread(void *__rescuer)
 		struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
 					struct pool_workqueue, mayday_node);
 		struct worker_pool *pool = pwq->pool;
+		unsigned int count = 0;
 
 		__set_current_state(TASK_RUNNING);
 		list_del_init(&pwq->mayday_node);
@@ -3524,31 +3582,27 @@ static int rescuer_thread(void *__rescuer)
 
 		WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
 
-		if (assign_rescuer_work(pwq, rescuer)) {
+		while (assign_rescuer_work(pwq, rescuer)) {
 			process_scheduled_works(rescuer);
 
 			/*
-			 * The above execution of rescued work items could
-			 * have created more to rescue through
-			 * pwq_activate_first_inactive() or chained
-			 * queueing.  Let's put @pwq back on mayday list so
-			 * that such back-to-back work items, which may be
-			 * being used to relieve memory pressure, don't
-			 * incur MAYDAY_INTERVAL delay inbetween.
+			 * If the per-turn work item limit is reached and other
+			 * PWQs are in mayday, requeue mayday for this PWQ and
+			 * let the rescuer handle the other PWQs first.
 			 */
-			if (pwq->nr_active && need_to_create_worker(pool)) {
+			if (++count > RESCUER_BATCH && !list_empty(&pwq->wq->maydays) &&
+			    pwq->nr_active && need_to_create_worker(pool)) {
 				raw_spin_lock(&wq_mayday_lock);
-				/*
-				 * Queue iff somebody else hasn't queued it already.
-				 */
-				if (list_empty(&pwq->mayday_node)) {
-					get_pwq(pwq);
-					list_add_tail(&pwq->mayday_node, &wq->maydays);
-				}
+				send_mayday(pwq);
 				raw_spin_unlock(&wq_mayday_lock);
+				break;
 			}
 		}
 
+		/* The cursor can not be left behind without the rescuer watching it. */
+		if (!list_empty(&pwq->mayday_cursor.entry) && list_empty(&pwq->mayday_node))
+			list_del_init(&pwq->mayday_cursor.entry);
+
 		/*
 		 * Leave this pool. Notify regular workers; otherwise, we end up
 		 * with 0 concurrency and stalling the execution.
@@ -5167,6 +5221,19 @@ static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
 	INIT_LIST_HEAD(&pwq->pwqs_node);
 	INIT_LIST_HEAD(&pwq->mayday_node);
 	kthread_init_work(&pwq->release_work, pwq_release_workfn);
+
+	/*
+	 * Set the dummy cursor work with valid function and get_work_pwq().
+	 *
+	 * The cursor work should only be in the pwq->pool->worklist, and
+	 * should not be treated as a processable work item.
+	 *
+	 * WORK_STRUCT_PENDING and WORK_STRUCT_INACTIVE just make it less
+	 * surprise for kernel debugging tools and reviewers.
+	 */
+	INIT_WORK(&pwq->mayday_cursor, mayday_cursor_func);
+	atomic_long_set(&pwq->mayday_cursor.data, (unsigned long)pwq |
+			WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | WORK_STRUCT_INACTIVE);
 }
 
 /* sync @pwq with the current state of its associated wq and link it */
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index ba36939fda79..1d605c7bd370 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1110,13 +1110,14 @@ config SOFTLOCKUP_DETECTOR_INTR_STORM
 	  the CPU stats and the interrupt counts during the "soft lockups".
 
 config BOOTPARAM_SOFTLOCKUP_PANIC
-	bool "Panic (Reboot) On Soft Lockups"
+	int "Panic (Reboot) On Soft Lockups"
 	depends on SOFTLOCKUP_DETECTOR
+	default 0
 	help
-	  Say Y here to enable the kernel to panic on "soft lockups",
-	  which are bugs that cause the kernel to loop in kernel
-	  mode for more than 20 seconds (configurable using the watchdog_thresh
-	  sysctl), without giving other tasks a chance to run.
+	  Set to a non-zero value N to enable the kernel to panic on "soft
+	  lockups", which are bugs that cause the kernel to loop in kernel
+	  mode for more than (N * 20 seconds) (configurable using the
+	  watchdog_thresh sysctl), without giving other tasks a chance to run.
 
 	  The panic can be used in combination with panic_timeout,
 	  to cause the system to reboot automatically after a
@@ -1124,7 +1125,7 @@ config BOOTPARAM_SOFTLOCKUP_PANIC
 	  high-availability systems that have uptime guarantees and
 	  where a lockup must be resolved ASAP.
 
-	  Say N if unsure.
+	  Say 0 if unsure.
 
 config HAVE_HARDLOCKUP_DETECTOR_BUDDY
 	bool
@@ -1798,6 +1799,14 @@ config DEBUG_CLOSURES
 	  interface to list them, which makes it possible to see asynchronous
 	  operations that get stuck.
 
+config DEBUG_XARRAY
+	bool "Debug XArray"
+	depends on DEBUG_KERNEL
+	help
+	  Enable XArray debugging information and extra validation.
+
+	  If unsure, say N.
+
 config DEBUG_MAPLE_TREE
 	bool "Debug maple trees"
 	depends on DEBUG_KERNEL
@@ -2499,9 +2508,6 @@ config TEST_BITMAP
 
 	  If unsure, say N.
 
-config TEST_UUID
-	tristate "Test functions located in the uuid module at runtime"
-
 config TEST_XARRAY
 	tristate "Test the XArray code at runtime"
 
@@ -2551,14 +2557,6 @@ config TEST_PARMAN
 
 	  If unsure, say N.
 
-config TEST_IRQ_TIMINGS
-	bool "IRQ timings selftest"
-	depends on IRQ_TIMINGS
-	help
-	  Enable this option to test the irq timings code on boot.
-
-	  If unsure, say N.
-
 config TEST_LKM
 	tristate "Test module loading with 'hello world' module"
 	depends on m
@@ -2805,6 +2803,29 @@ config LINEAR_RANGES_TEST
 
 	  If unsure, say N.
 
+config LIVEUPDATE_TEST
+	bool "Live Update Kernel Test"
+	default n
+	depends on LIVEUPDATE
+	help
+	  Enable a built-in kernel test module for the Live Update
+	  Orchestrator.
+
+	  This module validates the File-Lifecycle-Bound subsystem by
+	  registering a set of mock FLB objects with any real file handlers
+	  that support live update (such as the memfd handler).
+
+	  When live update operations are performed, this test module will
+	  output messages to the kernel log (dmesg), confirming that its
+	  registration and various callback functions (preserve, retrieve,
+	  finish, etc.) are being invoked correctly.
+
+	  This is a debugging and regression testing tool for developers
+	  working on the Live Update subsystem. It should not be enabled in
+	  production kernels.
+
+	  If unsure, say N
+
 config CMDLINE_KUNIT_TEST
 	tristate "KUnit test for cmdline API" if !KUNIT_ALL_TESTS
 	depends on KUNIT
@@ -3285,6 +3306,17 @@ config RATELIMIT_KUNIT_TEST
 
 	  If unsure, say N.
 
+config UUID_KUNIT_TEST
+	tristate "KUnit test for UUID" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
+	help
+	  This option enables the KUnit test suite for the uuid library,
+	  which provides functions for generating and parsing UUID and GUID.
+	  The test suite checks parsing of UUID and GUID strings.
+
+	  If unsure, say N.
+
 config INT_POW_KUNIT_TEST
 	tristate "Integer exponentiation (int_pow) test" if !KUNIT_ALL_TESTS
 	depends on KUNIT
diff --git a/lib/Makefile b/lib/Makefile
index aaf677cf4527..36a9360dcb37 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -89,7 +89,6 @@ ifeq ($(CONFIG_CC_IS_CLANG)$(CONFIG_KASAN),yy)
 GCOV_PROFILE_test_bitmap.o := n
 endif
 
-obj-$(CONFIG_TEST_UUID) += test_uuid.o
 obj-$(CONFIG_TEST_XARRAY) += test_xarray.o
 obj-$(CONFIG_TEST_MAPLE_TREE) += test_maple_tree.o
 obj-$(CONFIG_TEST_PARMAN) += test_parman.o
@@ -181,6 +180,7 @@ obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o
 obj-$(CONFIG_TEXTSEARCH_BM) += ts_bm.o
 obj-$(CONFIG_TEXTSEARCH_FSM) += ts_fsm.o
 obj-$(CONFIG_SMP) += percpu_counter.o
+obj-$(CONFIG_SMP) += percpu_counter_tree.o
 obj-$(CONFIG_AUDIT_GENERIC) += audit.o
 obj-$(CONFIG_AUDIT_COMPAT_GENERIC) += compat_audit.o
 
diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c
index 27fee57a5c91..846a5b5b44a4 100644
--- a/lib/alloc_tag.c
+++ b/lib/alloc_tag.c
@@ -783,7 +783,7 @@ static int proc_mem_profiling_handler(const struct ctl_table *table, int write,
 }
 
 
-static struct ctl_table memory_allocation_profiling_sysctls[] = {
+static const struct ctl_table memory_allocation_profiling_sysctls[] = {
 	{
 		.procname	= "mem_profiling",
 		.data		= &mem_alloc_profiling_key,
@@ -798,9 +798,6 @@ static struct ctl_table memory_allocation_profiling_sysctls[] = {
 
 static void __init sysctl_init(void)
 {
-	if (!mem_profiling_support)
-		memory_allocation_profiling_sysctls[0].mode = 0444;
-
 	register_sysctl_init("vm", memory_allocation_profiling_sysctls);
 }
 #else /* CONFIG_SYSCTL */
diff --git a/lib/build_OID_registry b/lib/build_OID_registry
index 8267e8d71338..30493ac190c0 100755
--- a/lib/build_OID_registry
+++ b/lib/build_OID_registry
@@ -60,10 +60,12 @@ for (my $i = 0; $i <= $#names; $i++) {
     # Determine the encoded length of this OID
     my $size = $#components;
     for (my $loop = 2; $loop <= $#components; $loop++) {
-	my $c = $components[$loop];
+	$ENV{'BC_LINE_LENGTH'} = "0";
+	my $c = `echo "ibase=10; obase=2; $components[$loop]" | bc`;
+	chomp($c);
 
 	# We will base128 encode the number
-	my $tmp = ($c == 0) ? 0 : int(log($c)/log(2));
+	my $tmp = length($c) - 1;
 	$tmp = int($tmp / 7);
 	$size += $tmp;
     }
@@ -100,16 +102,24 @@ for (my $i = 0; $i <= $#names; $i++) {
     push @octets, $components[0] * 40 + $components[1];
 
     for (my $loop = 2; $loop <= $#components; $loop++) {
-	my $c = $components[$loop];
+	# get the base 2 representation of the component
+	$ENV{'BC_LINE_LENGTH'} = "0";
+	my $c = `echo "ibase=10; obase=2; $components[$loop]" | bc`;
+	chomp($c);
 
-	# Base128 encode the number
-	my $tmp = ($c == 0) ? 0 : int(log($c)/log(2));
+	my $tmp = length($c) - 1;
 	$tmp = int($tmp / 7);
 
-	for (; $tmp > 0; $tmp--) {
-	    push @octets, (($c >> $tmp * 7) & 0x7f) | 0x80;
+	# zero pad upto length multiple of 7
+	$c = substr("0000000", 0, ($tmp + 1) * 7 - length($c)).$c;
+
+	# Base128 encode the number
+	for (my $j = 0; $j < $tmp; $j++) {
+	    my $b = oct("0b".substr($c, $j * 7, 7));
+
+	    push @octets, $b | 0x80;
 	}
-	push @octets, $c & 0x7f;
+	push @octets, oct("0b".substr($c, $tmp * 7, 7));
     }
 
     push @encoded_oids, \@octets;
diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index 6871a41e5069..33cf46bbadc8 100644
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -101,6 +101,26 @@ config CRYPTO_LIB_MD5_ARCH
 	default y if PPC
 	default y if SPARC64
 
+config CRYPTO_LIB_MLDSA
+	tristate
+	select CRYPTO_LIB_SHA3
+	help
+	  The ML-DSA library functions.  Select this if your module uses any of
+	  the functions from <crypto/mldsa.h>.
+
+config CRYPTO_LIB_NH
+	tristate
+	help
+	  Implementation of the NH almost-universal hash function, specifically
+	  the variant of NH used in Adiantum.
+
+config CRYPTO_LIB_NH_ARCH
+	bool
+	depends on CRYPTO_LIB_NH && !UML
+	default y if ARM && KERNEL_MODE_NEON
+	default y if ARM64 && KERNEL_MODE_NEON
+	default y if X86_64
+
 config CRYPTO_LIB_POLY1305
 	tristate
 	help
diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index 330ab65b29c4..45128eccedef 100644
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -126,6 +126,22 @@ endif # CONFIG_CRYPTO_LIB_MD5_ARCH
 
 ################################################################################
 
+obj-$(CONFIG_CRYPTO_LIB_MLDSA) += libmldsa.o
+libmldsa-y := mldsa.o
+
+################################################################################
+
+obj-$(CONFIG_CRYPTO_LIB_NH) += libnh.o
+libnh-y := nh.o
+ifeq ($(CONFIG_CRYPTO_LIB_NH_ARCH),y)
+CFLAGS_nh.o += -I$(src)/$(SRCARCH)
+libnh-$(CONFIG_ARM) += arm/nh-neon-core.o
+libnh-$(CONFIG_ARM64) += arm64/nh-neon-core.o
+libnh-$(CONFIG_X86) += x86/nh-sse2.o x86/nh-avx2.o
+endif
+
+################################################################################
+
 obj-$(CONFIG_CRYPTO_LIB_POLY1305) += libpoly1305.o
 libpoly1305-y := poly1305.o
 ifeq ($(CONFIG_ARCH_SUPPORTS_INT128),y)
diff --git a/lib/crypto/arm/nh-neon-core.S b/lib/crypto/arm/nh-neon-core.S
new file mode 100644
index 000000000000..01620a0782ca
--- /dev/null
+++ b/lib/crypto/arm/nh-neon-core.S
@@ -0,0 +1,116 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * NH - ε-almost-universal hash function, NEON accelerated version
+ *
+ * Copyright 2018 Google LLC
+ *
+ * Author: Eric Biggers <ebiggers@google.com>
+ */
+
+#include <linux/linkage.h>
+
+	.text
+	.fpu		neon
+
+	KEY		.req	r0
+	MESSAGE		.req	r1
+	MESSAGE_LEN	.req	r2
+	HASH		.req	r3
+
+	PASS0_SUMS	.req	q0
+	PASS0_SUM_A	.req	d0
+	PASS0_SUM_B	.req	d1
+	PASS1_SUMS	.req	q1
+	PASS1_SUM_A	.req	d2
+	PASS1_SUM_B	.req	d3
+	PASS2_SUMS	.req	q2
+	PASS2_SUM_A	.req	d4
+	PASS2_SUM_B	.req	d5
+	PASS3_SUMS	.req	q3
+	PASS3_SUM_A	.req	d6
+	PASS3_SUM_B	.req	d7
+	K0		.req	q4
+	K1		.req	q5
+	K2		.req	q6
+	K3		.req	q7
+	T0		.req	q8
+	T0_L		.req	d16
+	T0_H		.req	d17
+	T1		.req	q9
+	T1_L		.req	d18
+	T1_H		.req	d19
+	T2		.req	q10
+	T2_L		.req	d20
+	T2_H		.req	d21
+	T3		.req	q11
+	T3_L		.req	d22
+	T3_H		.req	d23
+
+.macro _nh_stride	k0, k1, k2, k3
+
+	// Load next message stride
+	vld1.8		{T3}, [MESSAGE]!
+
+	// Load next key stride
+	vld1.32		{\k3}, [KEY]!
+
+	// Add message words to key words
+	vadd.u32	T0, T3, \k0
+	vadd.u32	T1, T3, \k1
+	vadd.u32	T2, T3, \k2
+	vadd.u32	T3, T3, \k3
+
+	// Multiply 32x32 => 64 and accumulate
+	vmlal.u32	PASS0_SUMS, T0_L, T0_H
+	vmlal.u32	PASS1_SUMS, T1_L, T1_H
+	vmlal.u32	PASS2_SUMS, T2_L, T2_H
+	vmlal.u32	PASS3_SUMS, T3_L, T3_H
+.endm
+
+/*
+ * void nh_neon(const u32 *key, const u8 *message, size_t message_len,
+ *		__le64 hash[NH_NUM_PASSES])
+ *
+ * It's guaranteed that message_len % 16 == 0.
+ */
+ENTRY(nh_neon)
+
+	vld1.32		{K0,K1}, [KEY]!
+	  vmov.u64	PASS0_SUMS, #0
+	  vmov.u64	PASS1_SUMS, #0
+	vld1.32		{K2}, [KEY]!
+	  vmov.u64	PASS2_SUMS, #0
+	  vmov.u64	PASS3_SUMS, #0
+
+	subs		MESSAGE_LEN, MESSAGE_LEN, #64
+	blt		.Lloop4_done
+.Lloop4:
+	_nh_stride	K0, K1, K2, K3
+	_nh_stride	K1, K2, K3, K0
+	_nh_stride	K2, K3, K0, K1
+	_nh_stride	K3, K0, K1, K2
+	subs		MESSAGE_LEN, MESSAGE_LEN, #64
+	bge		.Lloop4
+
+.Lloop4_done:
+	ands		MESSAGE_LEN, MESSAGE_LEN, #63
+	beq		.Ldone
+	_nh_stride	K0, K1, K2, K3
+
+	subs		MESSAGE_LEN, MESSAGE_LEN, #16
+	beq		.Ldone
+	_nh_stride	K1, K2, K3, K0
+
+	subs		MESSAGE_LEN, MESSAGE_LEN, #16
+	beq		.Ldone
+	_nh_stride	K2, K3, K0, K1
+
+.Ldone:
+	// Sum the accumulators for each pass, then store the sums to 'hash'
+	vadd.u64	T0_L, PASS0_SUM_A, PASS0_SUM_B
+	vadd.u64	T0_H, PASS1_SUM_A, PASS1_SUM_B
+	vadd.u64	T1_L, PASS2_SUM_A, PASS2_SUM_B
+	vadd.u64	T1_H, PASS3_SUM_A, PASS3_SUM_B
+	vst1.8		{T0-T1}, [HASH]
+	bx		lr
+ENDPROC(nh_neon)
diff --git a/lib/crypto/arm/nh.h b/lib/crypto/arm/nh.h
new file mode 100644
index 000000000000..c9f39d819336
--- /dev/null
+++ b/lib/crypto/arm/nh.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * ARM32 accelerated implementation of NH
+ *
+ * Copyright 2018 Google LLC
+ */
+
+#include <asm/neon.h>
+#include <asm/simd.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
+
+asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len,
+			__le64 hash[NH_NUM_PASSES]);
+
+static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
+		    __le64 hash[NH_NUM_PASSES])
+{
+	if (static_branch_likely(&have_neon) && message_len >= 64 &&
+	    may_use_simd()) {
+		scoped_ksimd()
+			nh_neon(key, message, message_len, hash);
+		return true;
+	}
+	return false;
+}
+
+#define nh_mod_init_arch nh_mod_init_arch
+static void nh_mod_init_arch(void)
+{
+	if (elf_hwcap & HWCAP_NEON)
+		static_branch_enable(&have_neon);
+}
diff --git a/lib/crypto/arm64/nh-neon-core.S b/lib/crypto/arm64/nh-neon-core.S
new file mode 100644
index 000000000000..6fa57fce8085
--- /dev/null
+++ b/lib/crypto/arm64/nh-neon-core.S
@@ -0,0 +1,103 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * NH - ε-almost-universal hash function, ARM64 NEON accelerated version
+ *
+ * Copyright 2018 Google LLC
+ *
+ * Author: Eric Biggers <ebiggers@google.com>
+ */
+
+#include <linux/linkage.h>
+
+	KEY		.req	x0
+	MESSAGE		.req	x1
+	MESSAGE_LEN	.req	x2
+	HASH		.req	x3
+
+	PASS0_SUMS	.req	v0
+	PASS1_SUMS	.req	v1
+	PASS2_SUMS	.req	v2
+	PASS3_SUMS	.req	v3
+	K0		.req	v4
+	K1		.req	v5
+	K2		.req	v6
+	K3		.req	v7
+	T0		.req	v8
+	T1		.req	v9
+	T2		.req	v10
+	T3		.req	v11
+	T4		.req	v12
+	T5		.req	v13
+	T6		.req	v14
+	T7		.req	v15
+
+.macro _nh_stride	k0, k1, k2, k3
+
+	// Load next message stride
+	ld1		{T3.16b}, [MESSAGE], #16
+
+	// Load next key stride
+	ld1		{\k3\().4s}, [KEY], #16
+
+	// Add message words to key words
+	add		T0.4s, T3.4s, \k0\().4s
+	add		T1.4s, T3.4s, \k1\().4s
+	add		T2.4s, T3.4s, \k2\().4s
+	add		T3.4s, T3.4s, \k3\().4s
+
+	// Multiply 32x32 => 64 and accumulate
+	mov		T4.d[0], T0.d[1]
+	mov		T5.d[0], T1.d[1]
+	mov		T6.d[0], T2.d[1]
+	mov		T7.d[0], T3.d[1]
+	umlal		PASS0_SUMS.2d, T0.2s, T4.2s
+	umlal		PASS1_SUMS.2d, T1.2s, T5.2s
+	umlal		PASS2_SUMS.2d, T2.2s, T6.2s
+	umlal		PASS3_SUMS.2d, T3.2s, T7.2s
+.endm
+
+/*
+ * void nh_neon(const u32 *key, const u8 *message, size_t message_len,
+ *		__le64 hash[NH_NUM_PASSES])
+ *
+ * It's guaranteed that message_len % 16 == 0.
+ */
+SYM_FUNC_START(nh_neon)
+
+	ld1		{K0.4s,K1.4s}, [KEY], #32
+	  movi		PASS0_SUMS.2d, #0
+	  movi		PASS1_SUMS.2d, #0
+	ld1		{K2.4s}, [KEY], #16
+	  movi		PASS2_SUMS.2d, #0
+	  movi		PASS3_SUMS.2d, #0
+
+	subs		MESSAGE_LEN, MESSAGE_LEN, #64
+	blt		.Lloop4_done
+.Lloop4:
+	_nh_stride	K0, K1, K2, K3
+	_nh_stride	K1, K2, K3, K0
+	_nh_stride	K2, K3, K0, K1
+	_nh_stride	K3, K0, K1, K2
+	subs		MESSAGE_LEN, MESSAGE_LEN, #64
+	bge		.Lloop4
+
+.Lloop4_done:
+	ands		MESSAGE_LEN, MESSAGE_LEN, #63
+	beq		.Ldone
+	_nh_stride	K0, K1, K2, K3
+
+	subs		MESSAGE_LEN, MESSAGE_LEN, #16
+	beq		.Ldone
+	_nh_stride	K1, K2, K3, K0
+
+	subs		MESSAGE_LEN, MESSAGE_LEN, #16
+	beq		.Ldone
+	_nh_stride	K2, K3, K0, K1
+
+.Ldone:
+	// Sum the accumulators for each pass, then store the sums to 'hash'
+	addp		T0.2d, PASS0_SUMS.2d, PASS1_SUMS.2d
+	addp		T1.2d, PASS2_SUMS.2d, PASS3_SUMS.2d
+	st1		{T0.16b,T1.16b}, [HASH]
+	ret
+SYM_FUNC_END(nh_neon)
diff --git a/lib/crypto/arm64/nh.h b/lib/crypto/arm64/nh.h
new file mode 100644
index 000000000000..08902630bdd1
--- /dev/null
+++ b/lib/crypto/arm64/nh.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * ARM64 accelerated implementation of NH
+ *
+ * Copyright 2018 Google LLC
+ */
+
+#include <asm/hwcap.h>
+#include <asm/simd.h>
+#include <linux/cpufeature.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
+
+asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len,
+			__le64 hash[NH_NUM_PASSES]);
+
+static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
+		    __le64 hash[NH_NUM_PASSES])
+{
+	if (static_branch_likely(&have_neon) && message_len >= 64 &&
+	    may_use_simd()) {
+		scoped_ksimd()
+			nh_neon(key, message, message_len, hash);
+		return true;
+	}
+	return false;
+}
+
+#define nh_mod_init_arch nh_mod_init_arch
+static void nh_mod_init_arch(void)
+{
+	if (cpu_have_named_feature(ASIMD))
+		static_branch_enable(&have_neon);
+}
diff --git a/lib/crypto/md5.c b/lib/crypto/md5.c
index c0610ea1370e..c4af57db0ea8 100644
--- a/lib/crypto/md5.c
+++ b/lib/crypto/md5.c
@@ -29,7 +29,7 @@ static const struct md5_block_state md5_iv = {
 #define F4(x, y, z) (y ^ (x | ~z))
 
 #define MD5STEP(f, w, x, y, z, in, s) \
-	(w += f(x, y, z) + in, w = (w << s | w >> (32 - s)) + x)
+	(w += f(x, y, z) + in, w = rol32(w, s) + x)
 
 static void md5_block_generic(struct md5_block_state *state,
 			      const u8 data[MD5_BLOCK_SIZE])
diff --git a/lib/crypto/mldsa.c b/lib/crypto/mldsa.c
new file mode 100644
index 000000000000..853cf225c133
--- /dev/null
+++ b/lib/crypto/mldsa.c
@@ -0,0 +1,652 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Support for verifying ML-DSA signatures
+ *
+ * Copyright 2025 Google LLC
+ */
+
+#include <crypto/mldsa.h>
+#include <crypto/sha3.h>
+#include <kunit/visibility.h>
+#include <linux/export.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/unaligned.h>
+
+#define Q 8380417 /* The prime q = 2^23 - 2^13 + 1 */
+#define QINV_MOD_2_32 58728449 /* Multiplicative inverse of q mod 2^32 */
+#define N 256 /* Number of components per ring element */
+#define D 13 /* Number of bits dropped from the public key vector t */
+#define RHO_LEN 32 /* Length of the public random seed in bytes */
+#define MAX_W1_ENCODED_LEN 192 /* Max encoded length of one element of w'_1 */
+
+/*
+ * The zetas array in Montgomery form, i.e. with extra factor of 2^32.
+ * Reference: FIPS 204 Section 7.5 "NTT and NTT^-1"
+ * Generated by the following Python code:
+ * q=8380417; [a%q - q*(a%q > q//2) for a in [1753**(int(f'{i:08b}'[::-1], 2)) << 32 for i in range(256)]]
+ */
+static const s32 zetas_times_2_32[N] = {
+	-4186625, 25847,    -2608894, -518909,	237124,	  -777960,  -876248,
+	466468,	  1826347,  2353451,  -359251,	-2091905, 3119733,  -2884855,
+	3111497,  2680103,  2725464,  1024112,	-1079900, 3585928,  -549488,
+	-1119584, 2619752,  -2108549, -2118186, -3859737, -1399561, -3277672,
+	1757237,  -19422,   4010497,  280005,	2706023,  95776,    3077325,
+	3530437,  -1661693, -3592148, -2537516, 3915439,  -3861115, -3043716,
+	3574422,  -2867647, 3539968,  -300467,	2348700,  -539299,  -1699267,
+	-1643818, 3505694,  -3821735, 3507263,	-2140649, -1600420, 3699596,
+	811944,	  531354,   954230,   3881043,	3900724,  -2556880, 2071892,
+	-2797779, -3930395, -1528703, -3677745, -3041255, -1452451, 3475950,
+	2176455,  -1585221, -1257611, 1939314,	-4083598, -1000202, -3190144,
+	-3157330, -3632928, 126922,   3412210,	-983419,  2147896,  2715295,
+	-2967645, -3693493, -411027,  -2477047, -671102,  -1228525, -22981,
+	-1308169, -381987,  1349076,  1852771,	-1430430, -3343383, 264944,
+	508951,	  3097992,  44288,    -1100098, 904516,	  3958618,  -3724342,
+	-8578,	  1653064,  -3249728, 2389356,	-210977,  759969,   -1316856,
+	189548,	  -3553272, 3159746,  -1851402, -2409325, -177440,  1315589,
+	1341330,  1285669,  -1584928, -812732,	-1439742, -3019102, -3881060,
+	-3628969, 3839961,  2091667,  3407706,	2316500,  3817976,  -3342478,
+	2244091,  -2446433, -3562462, 266997,	2434439,  -1235728, 3513181,
+	-3520352, -3759364, -1197226, -3193378, 900702,	  1859098,  909542,
+	819034,	  495491,   -1613174, -43260,	-522500,  -655327,  -3122442,
+	2031748,  3207046,  -3556995, -525098,	-768622,  -3595838, 342297,
+	286988,	  -2437823, 4108315,  3437287,	-3342277, 1735879,  203044,
+	2842341,  2691481,  -2590150, 1265009,	4055324,  1247620,  2486353,
+	1595974,  -3767016, 1250494,  2635921,	-3548272, -2994039, 1869119,
+	1903435,  -1050970, -1333058, 1237275,	-3318210, -1430225, -451100,
+	1312455,  3306115,  -1962642, -1279661, 1917081,  -2546312, -1374803,
+	1500165,  777191,   2235880,  3406031,	-542412,  -2831860, -1671176,
+	-1846953, -2584293, -3724270, 594136,	-3776993, -2013608, 2432395,
+	2454455,  -164721,  1957272,  3369112,	185531,	  -1207385, -3183426,
+	162844,	  1616392,  3014001,  810149,	1652634,  -3694233, -1799107,
+	-3038916, 3523897,  3866901,  269760,	2213111,  -975884,  1717735,
+	472078,	  -426683,  1723600,  -1803090, 1910376,  -1667432, -1104333,
+	-260646,  -3833893, -2939036, -2235985, -420899,  -2286327, 183443,
+	-976891,  1612842,  -3545687, -554416,	3919660,  -48306,   -1362209,
+	3937738,  1400424,  -846154,  1976782
+};
+
+/* Reference: FIPS 204 Section 4 "Parameter Sets" */
+static const struct mldsa_parameter_set {
+	u8 k; /* num rows in the matrix A */
+	u8 l; /* num columns in the matrix A */
+	u8 ctilde_len; /* length of commitment hash ctilde in bytes; lambda/4 */
+	u8 omega; /* max num of 1's in the hint vector h */
+	u8 tau; /* num of +-1's in challenge c */
+	u8 beta; /* tau times eta */
+	u16 pk_len; /* length of public keys in bytes */
+	u16 sig_len; /* length of signatures in bytes */
+	s32 gamma1; /* coefficient range of y */
+} mldsa_parameter_sets[] = {
+	[MLDSA44] = {
+		.k = 4,
+		.l = 4,
+		.ctilde_len = 32,
+		.omega = 80,
+		.tau = 39,
+		.beta = 78,
+		.pk_len = MLDSA44_PUBLIC_KEY_SIZE,
+		.sig_len = MLDSA44_SIGNATURE_SIZE,
+		.gamma1 = 1 << 17,
+	},
+	[MLDSA65] = {
+		.k = 6,
+		.l = 5,
+		.ctilde_len = 48,
+		.omega = 55,
+		.tau = 49,
+		.beta = 196,
+		.pk_len = MLDSA65_PUBLIC_KEY_SIZE,
+		.sig_len = MLDSA65_SIGNATURE_SIZE,
+		.gamma1 = 1 << 19,
+	},
+	[MLDSA87] = {
+		.k = 8,
+		.l = 7,
+		.ctilde_len = 64,
+		.omega = 75,
+		.tau = 60,
+		.beta = 120,
+		.pk_len = MLDSA87_PUBLIC_KEY_SIZE,
+		.sig_len = MLDSA87_SIGNATURE_SIZE,
+		.gamma1 = 1 << 19,
+	},
+};
+
+/*
+ * An element of the ring R_q (normal form) or the ring T_q (NTT form).  It
+ * consists of N integers mod q: either the polynomial coefficients of the R_q
+ * element or the components of the T_q element.  In either case, whether they
+ * are fully reduced to [0, q - 1] varies in the different parts of the code.
+ */
+struct mldsa_ring_elem {
+	s32 x[N];
+};
+
+struct mldsa_verification_workspace {
+	/* SHAKE context for computing c, mu, and ctildeprime */
+	struct shake_ctx shake;
+	/* The fields in this union are used in their order of declaration. */
+	union {
+		/* The hash of the public key */
+		u8 tr[64];
+		/* The message representative mu */
+		u8 mu[64];
+		/* Temporary space for rej_ntt_poly() */
+		u8 block[SHAKE128_BLOCK_SIZE + 1];
+		/* Encoded element of w'_1 */
+		u8 w1_encoded[MAX_W1_ENCODED_LEN];
+		/* The commitment hash.  Real length is params->ctilde_len */
+		u8 ctildeprime[64];
+	};
+	/* SHAKE context for generating elements of the matrix A */
+	struct shake_ctx a_shake;
+	/*
+	 * An element of the matrix A generated from the public seed, or an
+	 * element of the vector t_1 decoded from the public key and pre-scaled
+	 * by 2^d.  Both are in NTT form.  To reduce memory usage, we generate
+	 * or decode these elements only as needed.
+	 */
+	union {
+		struct mldsa_ring_elem a;
+		struct mldsa_ring_elem t1_scaled;
+	};
+	/* The challenge c, generated from ctilde */
+	struct mldsa_ring_elem c;
+	/* A temporary element used during calculations */
+	struct mldsa_ring_elem tmp;
+
+	/* The following fields are variable-length: */
+
+	/* The signer's response vector */
+	struct mldsa_ring_elem z[/* l */];
+
+	/* The signer's hint vector */
+	/* u8 h[k * N]; */
+};
+
+/*
+ * Compute a * b * 2^-32 mod q.  a * b must be in the range [-2^31 * q, 2^31 * q
+ * - 1] before reduction.  The return value is in the range [-q + 1, q - 1].
+ *
+ * To reduce mod q efficiently, this uses Montgomery reduction with R=2^32.
+ * That's where the factor of 2^-32 comes from.  The caller must include a
+ * factor of 2^32 at some point to compensate for that.
+ *
+ * To keep the input and output ranges very close to symmetric, this
+ * specifically does a "signed" Montgomery reduction.  That is, when computing
+ * d = c * q^-1 mod 2^32, this chooses a representative in [S32_MIN, S32_MAX]
+ * rather than [0, U32_MAX], i.e. s32 rather than u32.  This matters in the
+ * wider multiplication d * Q when d keeps its value via sign extension.
+ *
+ * Reference: FIPS 204 Appendix A "Montgomery Multiplication".  But, it doesn't
+ * explain it properly: it has an off-by-one error in the upper end of the input
+ * range, it doesn't clarify that the signed version should be used, and it
+ * gives an unnecessarily large output range.  A better citation is perhaps the
+ * Dilithium reference code, which functionally matches the below code and
+ * merely has the (benign) off-by-one error in its documentation.
+ */
+static inline s32 Zq_mult(s32 a, s32 b)
+{
+	/* Compute the unreduced product c. */
+	s64 c = (s64)a * b;
+
+	/*
+	 * Compute d = c * q^-1 mod 2^32.  Generate a signed result, as
+	 * explained above, but do the actual multiplication using an unsigned
+	 * type to avoid signed integer overflow which is undefined behavior.
+	 */
+	s32 d = (u32)c * QINV_MOD_2_32;
+
+	/*
+	 * Compute e = c - d * q.  This makes the low 32 bits zero, since
+	 *   c - (c * q^-1) * q mod 2^32
+	 * = c - c * (q^-1 * q) mod 2^32
+	 * = c - c * 1 mod 2^32
+	 * = c - c mod 2^32
+	 * = 0 mod 2^32
+	 */
+	s64 e = c - (s64)d * Q;
+
+	/* Finally, return e * 2^-32. */
+	return e >> 32;
+}
+
+/*
+ * Convert @w to its number-theoretically-transformed representation in-place.
+ * Reference: FIPS 204 Algorithm 41, NTT
+ *
+ * To prevent intermediate overflows, all input coefficients must have absolute
+ * value < q.  All output components have absolute value < 9*q.
+ */
+static void ntt(struct mldsa_ring_elem *w)
+{
+	int m = 0; /* index in zetas_times_2_32 */
+
+	for (int len = 128; len >= 1; len /= 2) {
+		for (int start = 0; start < 256; start += 2 * len) {
+			const s32 z = zetas_times_2_32[++m];
+
+			for (int j = start; j < start + len; j++) {
+				s32 t = Zq_mult(z, w->x[j + len]);
+
+				w->x[j + len] = w->x[j] - t;
+				w->x[j] += t;
+			}
+		}
+	}
+}
+
+/*
+ * Convert @w from its number-theoretically-transformed representation in-place.
+ * Reference: FIPS 204 Algorithm 42, NTT^-1
+ *
+ * This also multiplies the coefficients by 2^32, undoing an extra factor of
+ * 2^-32 introduced earlier, and reduces the coefficients to [0, q - 1].
+ */
+static void invntt_and_mul_2_32(struct mldsa_ring_elem *w)
+{
+	int m = 256; /* index in zetas_times_2_32 */
+
+	/* Prevent intermediate overflows. */
+	for (int j = 0; j < 256; j++)
+		w->x[j] %= Q;
+
+	for (int len = 1; len < 256; len *= 2) {
+		for (int start = 0; start < 256; start += 2 * len) {
+			const s32 z = -zetas_times_2_32[--m];
+
+			for (int j = start; j < start + len; j++) {
+				s32 t = w->x[j];
+
+				w->x[j] = t + w->x[j + len];
+				w->x[j + len] = Zq_mult(z, t - w->x[j + len]);
+			}
+		}
+	}
+	/*
+	 * Multiply by 2^32 * 256^-1.  2^32 cancels the factor of 2^-32 from
+	 * earlier Montgomery multiplications.  256^-1 is for NTT^-1.  This
+	 * itself uses Montgomery multiplication, so *another* 2^32 is needed.
+	 * Thus the actual multiplicand is 2^32 * 2^32 * 256^-1 mod q = 41978.
+	 *
+	 * Finally, also reduce from [-q + 1, q - 1] to [0, q - 1].
+	 */
+	for (int j = 0; j < 256; j++) {
+		w->x[j] = Zq_mult(w->x[j], 41978);
+		w->x[j] += (w->x[j] >> 31) & Q;
+	}
+}
+
+/*
+ * Decode an element of t_1, i.e. the high d bits of t = A*s_1 + s_2.
+ * Reference: FIPS 204 Algorithm 23, pkDecode.
+ * Also multiply it by 2^d and convert it to NTT form.
+ */
+static const u8 *decode_t1_elem(struct mldsa_ring_elem *out,
+				const u8 *t1_encoded)
+{
+	for (int j = 0; j < N; j += 4, t1_encoded += 5) {
+		u32 v = get_unaligned_le32(t1_encoded);
+
+		out->x[j + 0] = ((v >> 0) & 0x3ff) << D;
+		out->x[j + 1] = ((v >> 10) & 0x3ff) << D;
+		out->x[j + 2] = ((v >> 20) & 0x3ff) << D;
+		out->x[j + 3] = ((v >> 30) | (t1_encoded[4] << 2)) << D;
+		static_assert(0x3ff << D < Q); /* All coefficients < q. */
+	}
+	ntt(out);
+	return t1_encoded; /* Return updated pointer. */
+}
+
+/*
+ * Decode the signer's response vector 'z' from the signature.
+ * Reference: FIPS 204 Algorithm 27, sigDecode.
+ *
+ * This also validates that the coefficients of z are in range, corresponding
+ * the infinity norm check at the end of Algorithm 8, ML-DSA.Verify_internal.
+ *
+ * Finally, this also converts z to NTT form.
+ */
+static bool decode_z(struct mldsa_ring_elem z[/* l */], int l, s32 gamma1,
+		     int beta, const u8 **sig_ptr)
+{
+	const u8 *sig = *sig_ptr;
+
+	for (int i = 0; i < l; i++) {
+		if (l == 4) { /* ML-DSA-44? */
+			/* 18-bit coefficients: decode 4 from 9 bytes. */
+			for (int j = 0; j < N; j += 4, sig += 9) {
+				u64 v = get_unaligned_le64(sig);
+
+				z[i].x[j + 0] = (v >> 0) & 0x3ffff;
+				z[i].x[j + 1] = (v >> 18) & 0x3ffff;
+				z[i].x[j + 2] = (v >> 36) & 0x3ffff;
+				z[i].x[j + 3] = (v >> 54) | (sig[8] << 10);
+			}
+		} else {
+			/* 20-bit coefficients: decode 4 from 10 bytes. */
+			for (int j = 0; j < N; j += 4, sig += 10) {
+				u64 v = get_unaligned_le64(sig);
+
+				z[i].x[j + 0] = (v >> 0) & 0xfffff;
+				z[i].x[j + 1] = (v >> 20) & 0xfffff;
+				z[i].x[j + 2] = (v >> 40) & 0xfffff;
+				z[i].x[j + 3] =
+					(v >> 60) |
+					(get_unaligned_le16(&sig[8]) << 4);
+			}
+		}
+		for (int j = 0; j < N; j++) {
+			z[i].x[j] = gamma1 - z[i].x[j];
+			if (z[i].x[j] <= -(gamma1 - beta) ||
+			    z[i].x[j] >= gamma1 - beta)
+				return false;
+		}
+		ntt(&z[i]);
+	}
+	*sig_ptr = sig; /* Return updated pointer. */
+	return true;
+}
+
+/*
+ * Decode the signer's hint vector 'h' from the signature.
+ * Reference: FIPS 204 Algorithm 21, HintBitUnpack
+ *
+ * Note that there are several ways in which the hint vector can be malformed.
+ */
+static bool decode_hint_vector(u8 h[/* k * N */], int k, int omega, const u8 *y)
+{
+	int index = 0;
+
+	memset(h, 0, k * N);
+	for (int i = 0; i < k; i++) {
+		int count = y[omega + i]; /* num 1's in elems 0 through i */
+		int prev = -1;
+
+		/* Cumulative count mustn't decrease or exceed omega. */
+		if (count < index || count > omega)
+			return false;
+		for (; index < count; index++) {
+			if (prev >= y[index]) /* Coefficients out of order? */
+				return false;
+			prev = y[index];
+			h[i * N + y[index]] = 1;
+		}
+	}
+	return mem_is_zero(&y[index], omega - index);
+}
+
+/*
+ * Expand @seed into an element of R_q @c with coefficients in {-1, 0, 1},
+ * exactly @tau of them nonzero.  Reference: FIPS 204 Algorithm 29, SampleInBall
+ */
+static void sample_in_ball(struct mldsa_ring_elem *c, const u8 *seed,
+			   size_t seed_len, int tau, struct shake_ctx *shake)
+{
+	u64 signs;
+	u8 j;
+
+	shake256_init(shake);
+	shake_update(shake, seed, seed_len);
+	shake_squeeze(shake, (u8 *)&signs, sizeof(signs));
+	le64_to_cpus(&signs);
+	*c = (struct mldsa_ring_elem){};
+	for (int i = N - tau; i < N; i++, signs >>= 1) {
+		do {
+			shake_squeeze(shake, &j, 1);
+		} while (j > i);
+		c->x[i] = c->x[j];
+		c->x[j] = 1 - 2 * (s32)(signs & 1);
+	}
+}
+
+/*
+ * Expand the public seed @rho and @row_and_column into an element of T_q @out.
+ * Reference: FIPS 204 Algorithm 30, RejNTTPoly
+ *
+ * @shake and @block are temporary space used by the expansion.  @block has
+ * space for one SHAKE128 block, plus an extra byte to allow reading a u32 from
+ * the final 3-byte group without reading out-of-bounds.
+ */
+static void rej_ntt_poly(struct mldsa_ring_elem *out, const u8 rho[RHO_LEN],
+			 __le16 row_and_column, struct shake_ctx *shake,
+			 u8 block[SHAKE128_BLOCK_SIZE + 1])
+{
+	shake128_init(shake);
+	shake_update(shake, rho, RHO_LEN);
+	shake_update(shake, (u8 *)&row_and_column, sizeof(row_and_column));
+	for (int i = 0; i < N;) {
+		shake_squeeze(shake, block, SHAKE128_BLOCK_SIZE);
+		block[SHAKE128_BLOCK_SIZE] = 0; /* for KMSAN */
+		static_assert(SHAKE128_BLOCK_SIZE % 3 == 0);
+		for (int j = 0; j < SHAKE128_BLOCK_SIZE && i < N; j += 3) {
+			u32 x = get_unaligned_le32(&block[j]) & 0x7fffff;
+
+			if (x < Q) /* Ignore values >= q. */
+				out->x[i++] = x;
+		}
+	}
+}
+
+/*
+ * Return the HighBits of r adjusted according to hint h
+ * Reference: FIPS 204 Algorithm 40, UseHint
+ *
+ * This is needed because of the public key compression in ML-DSA.
+ *
+ * h is either 0 or 1, r is in [0, q - 1], and gamma2 is either (q - 1) / 88 or
+ * (q - 1) / 32.  Except when invoked via the unit test interface, gamma2 is a
+ * compile-time constant, so compilers will optimize the code accordingly.
+ */
+static __always_inline s32 use_hint(u8 h, s32 r, const s32 gamma2)
+{
+	const s32 m = (Q - 1) / (2 * gamma2); /* 44 or 16, compile-time const */
+	s32 r1;
+
+	/*
+	 * Handle the special case where r - (r mod+- (2 * gamma2)) == q - 1,
+	 * i.e. r >= q - gamma2.  This is also exactly where the computation of
+	 * r1 below would produce 'm' and would need a correction.
+	 */
+	if (r >= Q - gamma2)
+		return h == 0 ? 0 : m - 1;
+
+	/*
+	 * Compute the (non-hint-adjusted) HighBits r1 as:
+	 *
+	 *  r1 = (r - (r mod+- (2 * gamma2))) / (2 * gamma2)
+	 *     = floor((r + gamma2 - 1) / (2 * gamma2))
+	 *
+	 * Note that when '2 * gamma2' is a compile-time constant, compilers
+	 * optimize the division to a reciprocal multiplication and shift.
+	 */
+	r1 = (u32)(r + gamma2 - 1) / (2 * gamma2);
+
+	/*
+	 * Return the HighBits r1:
+	 *	+ 0 if the hint is 0;
+	 *	+ 1 (mod m) if the hint is 1 and the LowBits are positive;
+	 *	- 1 (mod m) if the hint is 1 and the LowBits are negative or 0.
+	 *
+	 * r1 is in (and remains in) [0, m - 1].  Note that when 'm' is a
+	 * compile-time constant, compilers optimize the '% m' accordingly.
+	 */
+	if (h == 0)
+		return r1;
+	if (r > r1 * (2 * gamma2))
+		return (u32)(r1 + 1) % m;
+	return (u32)(r1 + m - 1) % m;
+}
+
+static __always_inline void use_hint_elem(struct mldsa_ring_elem *w,
+					  const u8 h[N], const s32 gamma2)
+{
+	for (int j = 0; j < N; j++)
+		w->x[j] = use_hint(h[j], w->x[j], gamma2);
+}
+
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST)
+/* Allow the __always_inline function use_hint() to be unit-tested. */
+s32 mldsa_use_hint(u8 h, s32 r, s32 gamma2)
+{
+	return use_hint(h, r, gamma2);
+}
+EXPORT_SYMBOL_IF_KUNIT(mldsa_use_hint);
+#endif
+
+/*
+ * Encode one element of the commitment vector w'_1 into a byte string.
+ * Reference: FIPS 204 Algorithm 28, w1Encode.
+ * Return the number of bytes used: 192 for ML-DSA-44 and 128 for the others.
+ */
+static size_t encode_w1(u8 out[MAX_W1_ENCODED_LEN],
+			const struct mldsa_ring_elem *w1, int k)
+{
+	size_t pos = 0;
+
+	static_assert(N * 6 / 8 == MAX_W1_ENCODED_LEN);
+	if (k == 4) { /* ML-DSA-44? */
+		/* 6 bits per coefficient.  Pack 4 at a time. */
+		for (int j = 0; j < N; j += 4) {
+			u32 v = (w1->x[j + 0] << 0) | (w1->x[j + 1] << 6) |
+				(w1->x[j + 2] << 12) | (w1->x[j + 3] << 18);
+			out[pos++] = v >> 0;
+			out[pos++] = v >> 8;
+			out[pos++] = v >> 16;
+		}
+	} else {
+		/* 4 bits per coefficient.  Pack 2 at a time. */
+		for (int j = 0; j < N; j += 2)
+			out[pos++] = w1->x[j] | (w1->x[j + 1] << 4);
+	}
+	return pos;
+}
+
+/* Reference: FIPS 204 Section 6.3 "ML-DSA Verifying (Internal)" */
+int mldsa_verify(enum mldsa_alg alg, const u8 *sig, size_t sig_len,
+		 const u8 *msg, size_t msg_len, const u8 *pk, size_t pk_len)
+{
+	const struct mldsa_parameter_set *params = &mldsa_parameter_sets[alg];
+	const int k = params->k, l = params->l;
+	/* For now this just does pure ML-DSA with an empty context string. */
+	static const u8 msg_prefix[2] = { /* dom_sep= */ 0, /* ctx_len= */ 0 };
+	const u8 *ctilde; /* The signer's commitment hash */
+	const u8 *t1_encoded = &pk[RHO_LEN]; /* Next encoded element of t_1 */
+	u8 *h; /* The signer's hint vector, length k * N */
+	size_t w1_enc_len;
+
+	/* Validate the public key and signature lengths. */
+	if (pk_len != params->pk_len || sig_len != params->sig_len)
+		return -EBADMSG;
+
+	/*
+	 * Allocate the workspace, including variable-length fields.  Its size
+	 * depends only on the ML-DSA parameter set, not the other inputs.
+	 *
+	 * For freeing it, use kfree_sensitive() rather than kfree().  This is
+	 * mainly to comply with FIPS 204 Section 3.6.3 "Intermediate Values".
+	 * In reality it's a bit gratuitous, as this is a public key operation.
+	 */
+	struct mldsa_verification_workspace *ws __free(kfree_sensitive) =
+		kmalloc(sizeof(*ws) + (l * sizeof(ws->z[0])) + (k * N),
+			GFP_KERNEL);
+	if (!ws)
+		return -ENOMEM;
+	h = (u8 *)&ws->z[l];
+
+	/* Decode the signature.  Reference: FIPS 204 Algorithm 27, sigDecode */
+	ctilde = sig;
+	sig += params->ctilde_len;
+	if (!decode_z(ws->z, l, params->gamma1, params->beta, &sig))
+		return -EBADMSG;
+	if (!decode_hint_vector(h, k, params->omega, sig))
+		return -EBADMSG;
+
+	/* Recreate the challenge c from the signer's commitment hash. */
+	sample_in_ball(&ws->c, ctilde, params->ctilde_len, params->tau,
+		       &ws->shake);
+	ntt(&ws->c);
+
+	/* Compute the message representative mu. */
+	shake256(pk, pk_len, ws->tr, sizeof(ws->tr));
+	shake256_init(&ws->shake);
+	shake_update(&ws->shake, ws->tr, sizeof(ws->tr));
+	shake_update(&ws->shake, msg_prefix, sizeof(msg_prefix));
+	shake_update(&ws->shake, msg, msg_len);
+	shake_squeeze(&ws->shake, ws->mu, sizeof(ws->mu));
+
+	/* Start computing ctildeprime = H(mu || w1Encode(w'_1)). */
+	shake256_init(&ws->shake);
+	shake_update(&ws->shake, ws->mu, sizeof(ws->mu));
+
+	/*
+	 * Compute the commitment w'_1 from A, z, c, t_1, and h.
+	 *
+	 * The computation is the same for each of the k rows.  Just do each row
+	 * before moving on to the next, resulting in only one loop over k.
+	 */
+	for (int i = 0; i < k; i++) {
+		/*
+		 * tmp = NTT(A) * NTT(z) * 2^-32
+		 * To reduce memory use, generate each element of NTT(A)
+		 * on-demand.  Note that each element is used only once.
+		 */
+		ws->tmp = (struct mldsa_ring_elem){};
+		for (int j = 0; j < l; j++) {
+			rej_ntt_poly(&ws->a, pk /* rho is first field of pk */,
+				     cpu_to_le16((i << 8) | j), &ws->a_shake,
+				     ws->block);
+			for (int n = 0; n < N; n++)
+				ws->tmp.x[n] +=
+					Zq_mult(ws->a.x[n], ws->z[j].x[n]);
+		}
+		/* All components of tmp now have abs value < l*q. */
+
+		/* Decode the next element of t_1. */
+		t1_encoded = decode_t1_elem(&ws->t1_scaled, t1_encoded);
+
+		/*
+		 * tmp -= NTT(c) * NTT(t_1 * 2^d) * 2^-32
+		 *
+		 * Taking a conservative bound for the output of ntt(), the
+		 * multiplicands can have absolute value up to 9*q.  That
+		 * corresponds to a product with absolute value 81*q^2.  That is
+		 * within the limits of Zq_mult() which needs < ~256*q^2.
+		 */
+		for (int j = 0; j < N; j++)
+			ws->tmp.x[j] -= Zq_mult(ws->c.x[j], ws->t1_scaled.x[j]);
+		/* All components of tmp now have abs value < (l+1)*q. */
+
+		/* tmp = w'_Approx = NTT^-1(tmp) * 2^32 */
+		invntt_and_mul_2_32(&ws->tmp);
+		/* All coefficients of tmp are now in [0, q - 1]. */
+
+		/*
+		 * tmp = w'_1 = UseHint(h, w'_Approx)
+		 * For efficiency, set gamma2 to a compile-time constant.
+		 */
+		if (k == 4)
+			use_hint_elem(&ws->tmp, &h[i * N], (Q - 1) / 88);
+		else
+			use_hint_elem(&ws->tmp, &h[i * N], (Q - 1) / 32);
+
+		/* Encode and hash the next element of w'_1. */
+		w1_enc_len = encode_w1(ws->w1_encoded, &ws->tmp, k);
+		shake_update(&ws->shake, ws->w1_encoded, w1_enc_len);
+	}
+
+	/* Finish computing ctildeprime. */
+	shake_squeeze(&ws->shake, ws->ctildeprime, params->ctilde_len);
+
+	/* Verify that ctilde == ctildeprime. */
+	if (memcmp(ws->ctildeprime, ctilde, params->ctilde_len) != 0)
+		return -EKEYREJECTED;
+	/* ||z||_infinity < gamma1 - beta was already checked in decode_z(). */
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mldsa_verify);
+
+MODULE_DESCRIPTION("ML-DSA signature verification");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/nh.c b/lib/crypto/nh.c
new file mode 100644
index 000000000000..e1d0095b5289
--- /dev/null
+++ b/lib/crypto/nh.c
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018 Google LLC
+ */
+
+/*
+ * Implementation of the NH almost-universal hash function, specifically the
+ * variant of NH used in Adiantum.  This is *not* a cryptographic hash function.
+ *
+ * Reference: section 6.3 of "Adiantum: length-preserving encryption for
+ * entry-level processors" (https://eprint.iacr.org/2018/720.pdf).
+ */
+
+#include <crypto/nh.h>
+#include <linux/export.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/unaligned.h>
+
+#ifdef CONFIG_CRYPTO_LIB_NH_ARCH
+#include "nh.h" /* $(SRCARCH)/nh.h */
+#else
+static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
+		    __le64 hash[NH_NUM_PASSES])
+{
+	return false;
+}
+#endif
+
+void nh(const u32 *key, const u8 *message, size_t message_len,
+	__le64 hash[NH_NUM_PASSES])
+{
+	u64 sums[4] = { 0, 0, 0, 0 };
+
+	if (nh_arch(key, message, message_len, hash))
+		return;
+
+	static_assert(NH_PAIR_STRIDE == 2);
+	static_assert(NH_NUM_PASSES == 4);
+
+	while (message_len) {
+		u32 m0 = get_unaligned_le32(message + 0);
+		u32 m1 = get_unaligned_le32(message + 4);
+		u32 m2 = get_unaligned_le32(message + 8);
+		u32 m3 = get_unaligned_le32(message + 12);
+
+		sums[0] += (u64)(u32)(m0 + key[0]) * (u32)(m2 + key[2]);
+		sums[1] += (u64)(u32)(m0 + key[4]) * (u32)(m2 + key[6]);
+		sums[2] += (u64)(u32)(m0 + key[8]) * (u32)(m2 + key[10]);
+		sums[3] += (u64)(u32)(m0 + key[12]) * (u32)(m2 + key[14]);
+		sums[0] += (u64)(u32)(m1 + key[1]) * (u32)(m3 + key[3]);
+		sums[1] += (u64)(u32)(m1 + key[5]) * (u32)(m3 + key[7]);
+		sums[2] += (u64)(u32)(m1 + key[9]) * (u32)(m3 + key[11]);
+		sums[3] += (u64)(u32)(m1 + key[13]) * (u32)(m3 + key[15]);
+		key += NH_MESSAGE_UNIT / sizeof(key[0]);
+		message += NH_MESSAGE_UNIT;
+		message_len -= NH_MESSAGE_UNIT;
+	}
+
+	hash[0] = cpu_to_le64(sums[0]);
+	hash[1] = cpu_to_le64(sums[1]);
+	hash[2] = cpu_to_le64(sums[2]);
+	hash[3] = cpu_to_le64(sums[3]);
+}
+EXPORT_SYMBOL_GPL(nh);
+
+#ifdef nh_mod_init_arch
+static int __init nh_mod_init(void)
+{
+	nh_mod_init_arch();
+	return 0;
+}
+subsys_initcall(nh_mod_init);
+
+static void __exit nh_mod_exit(void)
+{
+}
+module_exit(nh_mod_exit);
+#endif
+
+MODULE_DESCRIPTION("NH almost-universal hash function");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/riscv/.gitignore b/lib/crypto/riscv/.gitignore
new file mode 100644
index 000000000000..0d47d4f21c6d
--- /dev/null
+++ b/lib/crypto/riscv/.gitignore
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0-only
+poly1305-core.S
diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig
index 61d435c450bb..4970463ea0aa 100644
--- a/lib/crypto/tests/Kconfig
+++ b/lib/crypto/tests/Kconfig
@@ -38,6 +38,23 @@ config CRYPTO_LIB_MD5_KUNIT_TEST
 	  KUnit tests for the MD5 cryptographic hash function and its
 	  corresponding HMAC.
 
+config CRYPTO_LIB_MLDSA_KUNIT_TEST
+	tristate "KUnit tests for ML-DSA" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	select CRYPTO_LIB_MLDSA
+	help
+	  KUnit tests for the ML-DSA digital signature algorithm.
+
+config CRYPTO_LIB_NH_KUNIT_TEST
+	tristate "KUnit tests for NH" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS
+	select CRYPTO_LIB_NH
+	help
+	  KUnit tests for the NH almost-universal hash function.
+
 config CRYPTO_LIB_POLY1305_KUNIT_TEST
 	tristate "KUnit tests for Poly1305" if !KUNIT_ALL_TESTS
 	depends on KUNIT
diff --git a/lib/crypto/tests/Makefile b/lib/crypto/tests/Makefile
index 5109a0651925..f4262379f56c 100644
--- a/lib/crypto/tests/Makefile
+++ b/lib/crypto/tests/Makefile
@@ -4,6 +4,8 @@ obj-$(CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST) += blake2b_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST) += blake2s_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST) += curve25519_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_MD5_KUNIT_TEST) += md5_kunit.o
+obj-$(CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST) += mldsa_kunit.o
+obj-$(CONFIG_CRYPTO_LIB_NH_KUNIT_TEST) += nh_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_POLY1305_KUNIT_TEST) += poly1305_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_POLYVAL_KUNIT_TEST) += polyval_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_SHA1_KUNIT_TEST) += sha1_kunit.o
diff --git a/lib/crypto/tests/mldsa-testvecs.h b/lib/crypto/tests/mldsa-testvecs.h
new file mode 100644
index 000000000000..d2d6a042a829
--- /dev/null
+++ b/lib/crypto/tests/mldsa-testvecs.h
@@ -0,0 +1,1887 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/* ML-DSA test vectors extracted from leancrypto */
+
+struct mldsa_testvector {
+	enum mldsa_alg alg;
+	int sig_len;
+	int msg_len;
+	int pk_len;
+	const u8 *sig;
+	const u8 *msg;
+	const u8 *pk;
+};
+
+static const struct mldsa_testvector mldsa44_testvector = {
+	.alg = MLDSA44,
+	.pk_len = MLDSA44_PUBLIC_KEY_SIZE,
+	.pk =
+		(const u8[MLDSA44_PUBLIC_KEY_SIZE]) {
+			0x8f, 0x61, 0x67, 0xa9, 0x7c, 0x61, 0xc2, 0xf2, 0x87,
+			0xe2, 0x28, 0xf8, 0x44, 0x80, 0x6f, 0xb0, 0x10, 0xc1,
+			0x14, 0xf6, 0x88, 0x42, 0x76, 0xbe, 0x05, 0xd2, 0x56,
+			0xa0, 0xb7, 0x46, 0xcf, 0xc5, 0x76, 0x0a, 0x52, 0xfe,
+			0xa3, 0x3c, 0x05, 0x6e, 0xd5, 0xd3, 0xbd, 0x80, 0x03,
+			0x29, 0x25, 0x96, 0xdf, 0xa9, 0x5b, 0x12, 0x42, 0x89,
+			0x6e, 0x03, 0x2c, 0x42, 0x64, 0xee, 0xc7, 0xf7, 0x55,
+			0xfe, 0xfd, 0x15, 0x74, 0xee, 0x2d, 0xb3, 0xbf, 0xf2,
+			0x24, 0xe7, 0x35, 0x45, 0x77, 0x67, 0x44, 0xd2, 0x0c,
+			0x43, 0xfc, 0x7b, 0x47, 0x56, 0xad, 0xc5, 0xe7, 0x37,
+			0x49, 0x21, 0x12, 0x57, 0x7f, 0xca, 0x7f, 0x5d, 0xac,
+			0x62, 0x5b, 0x8d, 0xbf, 0xdb, 0x64, 0xac, 0x12, 0x1d,
+			0x7f, 0x7a, 0x2f, 0xa0, 0x2c, 0xfe, 0x95, 0x4b, 0x78,
+			0xdb, 0xf0, 0x98, 0x97, 0x70, 0x62, 0xfc, 0x64, 0x4a,
+			0x1b, 0xbc, 0x30, 0x1b, 0x51, 0x13, 0x47, 0x6e, 0x83,
+			0xd2, 0xa2, 0xcf, 0x39, 0xba, 0xd3, 0x00, 0x62, 0x7c,
+			0x5b, 0xe6, 0x14, 0x55, 0xd5, 0xfc, 0xbf, 0x15, 0x65,
+			0xd7, 0x1f, 0xbb, 0xf6, 0x2b, 0x46, 0x38, 0x1e, 0xf3,
+			0x8f, 0x0d, 0x57, 0x8a, 0x41, 0xfb, 0x47, 0x19, 0xe1,
+			0x79, 0xca, 0x98, 0x1a, 0x73, 0x4d, 0x8a, 0xc0, 0xa1,
+			0xa7, 0x4a, 0x28, 0x4a, 0x92, 0x6c, 0x77, 0x4f, 0x18,
+			0xe1, 0xce, 0x11, 0x14, 0xd5, 0xf6, 0xc0, 0xa8, 0x1e,
+			0x26, 0x25, 0xe4, 0x30, 0xb2, 0x6f, 0x89, 0x6e, 0xc6,
+			0x44, 0x1a, 0xd3, 0xca, 0xe1, 0x05, 0x0d, 0x61, 0x80,
+			0xd7, 0xbc, 0x56, 0x0a, 0x57, 0x9f, 0x16, 0x40, 0x84,
+			0x1e, 0xc0, 0x5e, 0xcd, 0xb6, 0xc8, 0x5d, 0x87, 0xd6,
+			0xee, 0xcb, 0x21, 0x2f, 0x22, 0x9a, 0x7e, 0xea, 0x2e,
+			0xf5, 0x87, 0xf0, 0x44, 0x10, 0x3d, 0x42, 0x6d, 0x3f,
+			0x4f, 0xa0, 0x21, 0x8e, 0x40, 0x75, 0x3b, 0xaa, 0xd4,
+			0xd9, 0x37, 0x0a, 0x35, 0x5b, 0xba, 0xbd, 0x11, 0x17,
+			0x7b, 0x7e, 0xfc, 0xb1, 0x1c, 0x5c, 0x71, 0xce, 0xde,
+			0xa5, 0xec, 0xd6, 0x8b, 0x50, 0x64, 0x02, 0xca, 0x84,
+			0x26, 0xfe, 0x03, 0x9e, 0xaf, 0x13, 0x05, 0x1c, 0x85,
+			0x32, 0x92, 0x85, 0x84, 0x83, 0xb6, 0x76, 0x27, 0xa7,
+			0xd8, 0x0c, 0xa4, 0xb5, 0x65, 0x7a, 0xbf, 0x2e, 0x64,
+			0xce, 0x82, 0x0d, 0x27, 0xd3, 0x2c, 0x9d, 0xb5, 0xa5,
+			0x51, 0xe3, 0xab, 0xbd, 0xe2, 0xe0, 0x64, 0x71, 0x0f,
+			0x99, 0x09, 0x6a, 0x40, 0x9d, 0x8c, 0x6d, 0x63, 0x70,
+			0x42, 0xe2, 0xa8, 0x5c, 0x71, 0xd2, 0x88, 0x10, 0xbc,
+			0x3b, 0x9d, 0xc8, 0x3a, 0x8a, 0xf3, 0x81, 0x0a, 0x0e,
+			0xd8, 0xf6, 0x13, 0x90, 0xc2, 0xda, 0x4c, 0x4f, 0x6f,
+			0x1f, 0x61, 0x0d, 0x46, 0xac, 0x5c, 0x4a, 0xcc, 0x23,
+			0xfb, 0xf2, 0xf8, 0x9f, 0x48, 0x1f, 0xaf, 0x24, 0xb4,
+			0xc7, 0xcb, 0x6f, 0x80, 0x74, 0xa3, 0xdb, 0xa8, 0x61,
+			0x12, 0x60, 0x08, 0x56, 0x85, 0x47, 0x05, 0x32, 0x93,
+			0x43, 0x9b, 0xa6, 0xf6, 0x56, 0x3b, 0xab, 0x93, 0x28,
+			0x19, 0xda, 0xad, 0xb5, 0xaa, 0x2e, 0x83, 0x3d, 0x37,
+			0x0e, 0x83, 0xf2, 0xfe, 0xdd, 0xe8, 0xd9, 0x53, 0x36,
+			0x3b, 0x5d, 0x8e, 0x24, 0xa9, 0x3a, 0x8f, 0x85, 0x4b,
+			0x50, 0xf7, 0x61, 0x0f, 0x20, 0x92, 0x09, 0x1f, 0xe4,
+			0x24, 0x98, 0x9f, 0xa4, 0x52, 0x12, 0x2a, 0xae, 0x6f,
+			0xd6, 0x0d, 0xb5, 0x22, 0x72, 0x83, 0xfb, 0x4f, 0xef,
+			0xa6, 0x55, 0x9a, 0x14, 0xab, 0x82, 0x89, 0x1f, 0xf2,
+			0x0b, 0x14, 0x76, 0xb7, 0xf7, 0x14, 0xdd, 0xd6, 0xc1,
+			0xe8, 0xb2, 0x99, 0x23, 0x28, 0xe8, 0xa4, 0x69, 0x18,
+			0xf6, 0x3e, 0xb1, 0xff, 0xde, 0xf2, 0x2c, 0x7c, 0x73,
+			0x93, 0x32, 0x52, 0x06, 0xeb, 0x59, 0xb2, 0x8f, 0x2f,
+			0x1d, 0x6a, 0x85, 0x74, 0xd3, 0xe5, 0xa1, 0x95, 0xe4,
+			0x96, 0x1e, 0x75, 0x16, 0xe4, 0x5c, 0x40, 0xf0, 0x20,
+			0xb1, 0x10, 0xe7, 0x2a, 0x70, 0x41, 0xac, 0x49, 0x40,
+			0x55, 0xef, 0xd1, 0x58, 0x24, 0x6a, 0xa6, 0x20, 0xdc,
+			0x23, 0xdc, 0x66, 0x45, 0x4f, 0x6a, 0x52, 0x3c, 0x61,
+			0xc8, 0xfb, 0x28, 0x1e, 0x8c, 0x3f, 0xfc, 0xc7, 0x73,
+			0x2f, 0xf1, 0xe2, 0x31, 0xee, 0xa0, 0x5f, 0x12, 0x3f,
+			0x94, 0xe5, 0x26, 0xc0, 0x62, 0xcc, 0x67, 0x8f, 0x5d,
+			0xfd, 0x3d, 0x8f, 0x16, 0xae, 0x4e, 0x04, 0x54, 0x5b,
+			0x02, 0x66, 0x00, 0x70, 0xe5, 0xcc, 0xb9, 0x51, 0x8c,
+			0x1a, 0x5d, 0xf0, 0xfe, 0x7d, 0x1b, 0x2f, 0x0a, 0x48,
+			0x94, 0xda, 0x4b, 0xb3, 0x05, 0x75, 0x3b, 0x25, 0xda,
+			0xf2, 0x12, 0x47, 0xe2, 0xd7, 0xbb, 0xe7, 0x05, 0xa2,
+			0x4e, 0xaf, 0x2f, 0x29, 0x6c, 0xbd, 0x80, 0x82, 0xdf,
+			0xbb, 0x4b, 0x9d, 0x29, 0x9b, 0xef, 0xff, 0x17, 0x2a,
+			0xa7, 0x2f, 0x4a, 0x2e, 0xbf, 0x29, 0x38, 0xcb, 0x94,
+			0x9c, 0x10, 0x87, 0x69, 0x57, 0x7d, 0xd7, 0xcf, 0xc1,
+			0x57, 0x0f, 0xc2, 0x9c, 0x3b, 0x1e, 0xbc, 0x55, 0x08,
+			0xb1, 0x50, 0xb0, 0x28, 0x79, 0x1d, 0xd9, 0x2f, 0xa5,
+			0x12, 0xcc, 0xa7, 0x5b, 0xcb, 0x67, 0x57, 0x13, 0x4a,
+			0xb3, 0xd5, 0x34, 0xea, 0xd9, 0x6b, 0x15, 0x17, 0x1f,
+			0xe7, 0x52, 0x2a, 0x11, 0xdf, 0xab, 0x3a, 0x8d, 0x1b,
+			0xaa, 0xfa, 0x54, 0xf0, 0x20, 0x94, 0x40, 0x2d, 0x3e,
+			0xf2, 0xa2, 0x8b, 0xbb, 0x86, 0xce, 0x29, 0x87, 0xb1,
+			0xfa, 0x27, 0xf2, 0x5a, 0x79, 0xc2, 0xf8, 0xe9, 0xf2,
+			0x7c, 0x5a, 0xcd, 0x45, 0x8e, 0x8b, 0xba, 0xad, 0x13,
+			0x52, 0x79, 0xd5, 0x0f, 0x59, 0xfb, 0x07, 0x15, 0x1f,
+			0xb3, 0xe6, 0xc8, 0x66, 0xf7, 0x37, 0x19, 0xd8, 0x33,
+			0xdd, 0xc1, 0xb3, 0x96, 0x4b, 0x28, 0x43, 0xf8, 0x5e,
+			0xc7, 0xe8, 0x81, 0x6e, 0xd0, 0xb2, 0x5a, 0x3d, 0xf1,
+			0x68, 0x8f, 0xf0, 0xf5, 0x1a, 0x6d, 0xc3, 0xaa, 0x5b,
+			0x72, 0x27, 0xbd, 0xd6, 0x7c, 0x9b, 0xbf, 0x89, 0x6a,
+			0x09, 0xf0, 0x48, 0xf0, 0x8c, 0x27, 0x69, 0x28, 0xf3,
+			0x5f, 0x53, 0xe3, 0x4d, 0x60, 0x4a, 0xb8, 0xc5, 0xf9,
+			0x85, 0x07, 0x3e, 0xfb, 0xd7, 0x21, 0x69, 0xd5, 0xec,
+			0x18, 0x68, 0xb6, 0x55, 0x15, 0xac, 0x2e, 0x0f, 0x5c,
+			0x2e, 0x9e, 0x12, 0x10, 0x5e, 0xc6, 0xb3, 0xcd, 0xe6,
+			0x3a, 0x2f, 0x43, 0xf6, 0x75, 0x31, 0x3c, 0x90, 0x34,
+			0x1c, 0x3d, 0x45, 0xaa, 0x8e, 0x08, 0xcf, 0x58, 0x67,
+			0x34, 0xd4, 0x24, 0xb8, 0x20, 0x69, 0xfe, 0xf0, 0x33,
+			0xe9, 0x3e, 0xbd, 0xe5, 0x18, 0x9f, 0x66, 0xc9, 0x0d,
+			0x6f, 0x47, 0x99, 0xf7, 0x0c, 0xdd, 0xad, 0x8c, 0x6b,
+			0x80, 0xed, 0x19, 0x61, 0x8a, 0xba, 0x62, 0x2f, 0xcc,
+			0x0e, 0x47, 0xe0, 0xc6, 0x1f, 0x2f, 0x44, 0x40, 0x17,
+			0xb6, 0x89, 0xc6, 0xf5, 0xc2, 0x97, 0x89, 0x38, 0x6c,
+			0x8e, 0x2c, 0x46, 0x0b, 0x6e, 0x01, 0x47, 0xd3, 0x2f,
+			0x58, 0xf8, 0xc7, 0x3b, 0x49, 0xb2, 0x35, 0x9f, 0x67,
+			0xc9, 0x6c, 0xde, 0x30, 0x3b, 0x50, 0x19, 0x65, 0xbb,
+			0x73, 0xa1, 0x47, 0x01, 0x6a, 0x3d, 0xe3, 0x85, 0x4d,
+			0xd0, 0x72, 0x0d, 0xbc, 0x81, 0x52, 0xe8, 0x1a, 0x8b,
+			0x85, 0x4c, 0x6f, 0x0e, 0xfc, 0x59, 0x84, 0x7a, 0xf2,
+			0x28, 0x3e, 0x02, 0xcb, 0xe2, 0x4a, 0xbf, 0xac, 0x22,
+			0x34, 0x86, 0xe4, 0x7d, 0x6b, 0xa3, 0x52, 0xac, 0xff,
+			0xe5, 0xbe, 0x0c, 0x8d, 0xf5, 0xd8, 0xfd, 0x5a, 0x3d,
+			0xad, 0x0d, 0xc0, 0x02, 0xd0, 0x58, 0x8e, 0x7c, 0x50,
+			0x7c, 0x09, 0xc0, 0xdb, 0xd7, 0xdf, 0xe0, 0xb2, 0x6f,
+			0xb2, 0x79, 0x0d, 0xc1, 0xa0, 0xb1, 0x7e, 0xe3, 0x0d,
+			0xfc, 0x93, 0x43, 0x8c, 0x86, 0x68, 0xa6, 0x51, 0x93,
+			0x42, 0xb7, 0xcd, 0x13, 0x56, 0x6e, 0xdc, 0x63, 0x51,
+			0x23, 0xcf, 0x29, 0xc5, 0x5a, 0x66, 0x43, 0x80, 0xfe,
+			0x28, 0x15, 0x71, 0x52, 0x87, 0xe2, 0x18, 0x10, 0xad,
+			0x94, 0x47, 0x2f, 0xe3, 0x5d, 0x7a, 0x02, 0x6d, 0x31,
+			0x58, 0xae, 0x2f, 0x96, 0x53, 0x1b, 0x6c, 0x0e, 0x25,
+			0xea, 0x41, 0xd3, 0x29, 0x27, 0x22, 0x24, 0x8d, 0x1d,
+			0x8e, 0xd1, 0x33, 0x76, 0x67, 0x7d, 0x9d, 0xd9, 0xa7,
+			0x3e, 0x61, 0xd0, 0xad, 0x93, 0xb9, 0xdf, 0x87, 0x3a,
+			0x7b, 0x89, 0xc7, 0x1d, 0x91, 0xec, 0x43, 0xa4, 0xdc,
+			0x02, 0x88, 0x2e, 0xaa, 0xb2, 0x58, 0xa5, 0xd3, 0x68,
+			0x9c, 0x9f, 0x60, 0x12, 0xc8, 0x7e, 0x7d, 0x40, 0x80,
+			0xfd, 0xb4, 0xbf, 0x56, 0xf4, 0x01, 0x39, 0x3d, 0xa0,
+			0x34, 0x48, 0x79, 0x75, 0xe2, 0x0f, 0x60, 0x09, 0x42,
+			0x11, 0x6f, 0xa5, 0x31, 0x46, 0xb7, 0x03, 0xc8, 0x61,
+			0x53, 0x39, 0x1e, 0xf4, 0x99, 0x36, 0x7d, 0xc0, 0x87,
+			0xda, 0x63, 0x71, 0x9b, 0x29, 0x7b, 0x4e, 0x6f, 0x09,
+			0xa2, 0x2c, 0xa5, 0xc2, 0xb9, 0xe7, 0xe0, 0x56, 0x8b,
+			0x1d, 0xbb, 0xcc, 0x34, 0x8c, 0xbc, 0xb6, 0x0a, 0xc9,
+			0xfa, 0x4a, 0x31, 0x63, 0x0d, 0x30, 0xff, 0x59, 0x3d,
+			0x8c, 0x4d, 0x74, 0x28, 0xf4, 0xe9, 0x97, 0x43, 0x05,
+			0x3a, 0x33, 0x51, 0x51, 0xe4, 0x0e, 0x33, 0xae, 0x2c,
+			0xda, 0x28, 0x83, 0x93, 0x4e, 0xfe, 0x37, 0x1d, 0x6c,
+			0x25, 0x1e, 0x24, 0xbc, 0x3a, 0x5c, 0x68, 0xac, 0x54,
+			0x3a, 0x47, 0x74, 0x35, 0xff, 0x37, 0x80, 0x12, 0x30,
+			0xd7, 0x31, 0x2a, 0x49, 0x51, 0x2d, 0x4f, 0xd2, 0x9c,
+			0xca, 0x55, 0x87, 0xd0, 0x41, 0x86, 0xc7, 0xf2, 0xda,
+			0xf8, 0x4b, 0x08, 0x23, 0xb3, 0x00, 0xb7, 0xb6, 0x4f,
+			0x2e, 0xaf, 0xb8, 0x8e, 0xb1, 0x44, 0xe1, 0xed, 0x67,
+			0xf8, 0x80, 0xa7, 0x04, 0xa0, 0x66, 0xe6, 0xb5, 0x69,
+			0xca, 0x95, 0x71, 0xc8, 0x0d, 0x3d, 0xf6, 0x77, 0xfd,
+			0x2c, 0x95, 0xed, 0xe5, 0x22, 0x43, 0xd9,
+		},
+	.msg_len = 64,
+	.msg =
+		(const u8[64]) {
+			0x6d, 0xb3, 0x8e, 0x80, 0xaf, 0x5f, 0x19, 0xd9,
+			0xb0, 0xcf, 0xad, 0x58, 0xc7, 0x27, 0xae, 0x12,
+			0x4e, 0x7d, 0xa3, 0x1a, 0xe3, 0x85, 0xc6, 0xaa,
+			0xf6, 0xa1, 0x9a, 0xb1, 0xe9, 0xe0, 0xfe, 0x89,
+			0x1e, 0xc5, 0x6f, 0x10, 0x18, 0x24, 0xab, 0xa8,
+			0x6d, 0x03, 0xd0, 0x3d, 0xc3, 0xff, 0x67, 0xe7,
+			0x3a, 0x95, 0x94, 0xc8, 0x49, 0x51, 0x8f, 0xa0,
+			0x65, 0xcb, 0x20, 0x29, 0x2a, 0x6d, 0xf7, 0xf2,
+		},
+	.sig_len = MLDSA44_SIGNATURE_SIZE,
+	.sig =
+		(const u8[MLDSA44_SIGNATURE_SIZE]) {
+			0x5e, 0x05, 0x37, 0xe2, 0xc1, 0x20, 0xce, 0x7b, 0x8a,
+			0xdf, 0xf2, 0x22, 0x61, 0x17, 0x58, 0xaa, 0x3c, 0xe4,
+			0x82, 0x9c, 0x0e, 0xb6, 0x1f, 0xb4, 0x98, 0x0f, 0xba,
+			0x8e, 0x51, 0x15, 0x67, 0x76, 0x0b, 0x98, 0x63, 0xda,
+			0x17, 0xd3, 0xbb, 0xbe, 0x16, 0x29, 0x71, 0xab, 0xba,
+			0x99, 0xed, 0x3f, 0xd4, 0xc2, 0x16, 0x71, 0xb6, 0x21,
+			0x87, 0x48, 0xaa, 0xb5, 0x39, 0x5e, 0xfb, 0x5d, 0x68,
+			0x3b, 0xd3, 0x60, 0xf4, 0x5b, 0x85, 0x2a, 0x5b, 0xb5,
+			0xce, 0x6e, 0xf3, 0x39, 0xc3, 0xbe, 0x96, 0xa7, 0x61,
+			0xc9, 0xbf, 0xdf, 0x33, 0x1d, 0xec, 0xb9, 0x2b, 0x7a,
+			0x05, 0xce, 0x1e, 0xd9, 0x46, 0x70, 0xca, 0x54, 0xbf,
+			0xdc, 0x46, 0x9e, 0x2f, 0x29, 0x18, 0x57, 0x96, 0x84,
+			0xac, 0xe9, 0xd7, 0x74, 0xeb, 0x8e, 0x6b, 0xec, 0x46,
+			0x9a, 0x2a, 0xfa, 0xde, 0x80, 0x09, 0x53, 0xd9, 0xeb,
+			0x9d, 0xf7, 0xaa, 0xe2, 0xe5, 0xdc, 0xc3, 0xd9, 0x70,
+			0xe5, 0x8b, 0xa8, 0xba, 0x2b, 0x41, 0x72, 0x92, 0x25,
+			0xaf, 0xd9, 0xb4, 0x5a, 0x53, 0xb7, 0xcc, 0x1d, 0x69,
+			0xf1, 0x53, 0x5b, 0x52, 0x38, 0xbc, 0x47, 0x24, 0x8c,
+			0x1d, 0x28, 0x5d, 0x5c, 0x1c, 0xc9, 0x9d, 0xea, 0x1c,
+			0xb1, 0xb3, 0x49, 0x68, 0xd5, 0xad, 0xdc, 0x47, 0x58,
+			0x6d, 0x38, 0x33, 0xe7, 0x9b, 0xaa, 0x89, 0xb1, 0x96,
+			0x0b, 0xcb, 0xc4, 0x24, 0x73, 0xf2, 0xe7, 0xb6, 0xca,
+			0x74, 0x55, 0x1b, 0xb5, 0xb7, 0x9e, 0x2e, 0xe3, 0x3a,
+			0x32, 0x5d, 0x1d, 0x6e, 0x15, 0xe6, 0xb8, 0xfb, 0xce,
+			0x57, 0x81, 0x15, 0xb5, 0xcf, 0x67, 0x2b, 0x55, 0x4c,
+			0x85, 0x6f, 0x28, 0xa6, 0xbb, 0xb4, 0x28, 0x76, 0x91,
+			0xa4, 0x29, 0xa1, 0x50, 0x7c, 0xed, 0x9a, 0xfc, 0xe4,
+			0xbc, 0xd7, 0x28, 0x62, 0x28, 0x61, 0x4d, 0x8d, 0x8c,
+			0x5a, 0x5e, 0x4d, 0x1d, 0x5e, 0x73, 0xcc, 0x0b, 0x9d,
+			0x56, 0x73, 0xc7, 0xf2, 0x26, 0xf7, 0x7e, 0x61, 0xa4,
+			0x86, 0xf5, 0x1c, 0xd1, 0x00, 0xd0, 0x31, 0xc5, 0x03,
+			0x17, 0x1c, 0xec, 0x04, 0xe5, 0xc7, 0x13, 0xb6, 0x81,
+			0x78, 0x3d, 0x27, 0x87, 0x36, 0xf3, 0x2a, 0x59, 0x96,
+			0xeb, 0x44, 0xfd, 0xb9, 0x95, 0xb7, 0x76, 0xb1, 0x08,
+			0xc4, 0x98, 0xb1, 0x08, 0x36, 0x2a, 0x63, 0x72, 0x4f,
+			0xef, 0x47, 0xfc, 0x84, 0x09, 0x18, 0x60, 0xb7, 0x8a,
+			0xff, 0xae, 0x32, 0x3c, 0x79, 0xdf, 0xd6, 0x24, 0xbe,
+			0x9c, 0x38, 0x68, 0x92, 0xde, 0x81, 0x80, 0x22, 0x06,
+			0xf2, 0xe4, 0xde, 0x75, 0x4e, 0xd6, 0x36, 0x93, 0x44,
+			0xd1, 0xa4, 0x2e, 0x2e, 0x05, 0x87, 0xbd, 0xf7, 0xc5,
+			0xc8, 0x1c, 0x7b, 0x00, 0xe8, 0x11, 0x7f, 0xc2, 0x39,
+			0x4b, 0x7b, 0x97, 0x11, 0x92, 0x6c, 0xff, 0x89, 0x7f,
+			0x26, 0x89, 0x4f, 0x38, 0xfd, 0xdd, 0x08, 0xa7, 0xce,
+			0x6f, 0xe8, 0x57, 0x9b, 0x46, 0xe5, 0xdb, 0x72, 0x03,
+			0x1e, 0x7d, 0xb0, 0x77, 0xb9, 0xcc, 0xdb, 0x6c, 0xa7,
+			0xd8, 0x30, 0x34, 0xad, 0xa7, 0xe4, 0x63, 0xf0, 0x19,
+			0x0e, 0x5d, 0x3b, 0xe0, 0xff, 0x40, 0x1c, 0xa5, 0xb3,
+			0xb9, 0x87, 0x6e, 0x2c, 0xf3, 0x5f, 0xcd, 0x54, 0x2a,
+			0xc0, 0x6e, 0x2b, 0xd9, 0x2d, 0xcc, 0xd5, 0x68, 0x95,
+			0x4a, 0x4a, 0x84, 0x60, 0x54, 0xee, 0xa0, 0x21, 0x9e,
+			0x8d, 0x20, 0xcb, 0xe8, 0xc5, 0x5a, 0xba, 0xe2, 0xaa,
+			0x6e, 0x1c, 0xb1, 0xdf, 0x18, 0x9f, 0x94, 0xc7, 0x77,
+			0x5a, 0x2c, 0x0e, 0x05, 0xaa, 0x2a, 0x54, 0x58, 0x6c,
+			0xb3, 0x2e, 0x2f, 0xa4, 0x6e, 0x98, 0xbb, 0x6f, 0x41,
+			0x6d, 0xbd, 0x71, 0x95, 0xe4, 0xbc, 0x13, 0x37, 0x99,
+			0x0d, 0xac, 0x27, 0x69, 0xb9, 0x0b, 0x14, 0x5f, 0x6e,
+			0xd2, 0x2b, 0xe2, 0x0c, 0xc6, 0xbc, 0x10, 0x11, 0x47,
+			0xb7, 0x37, 0x2c, 0x0e, 0x88, 0xcd, 0xbb, 0xf7, 0x28,
+			0xd6, 0x4a, 0x9d, 0xff, 0x3c, 0x2f, 0x7d, 0x2b, 0xe8,
+			0xe8, 0x9e, 0xae, 0x7b, 0xe6, 0x2a, 0xb3, 0x4e, 0x20,
+			0xcc, 0xf1, 0x81, 0x8e, 0xed, 0x6d, 0xe2, 0x99, 0xf5,
+			0xb5, 0x1a, 0x30, 0x95, 0x52, 0x34, 0xf5, 0x3f, 0xc3,
+			0x31, 0xd6, 0xbe, 0xa2, 0xc8, 0xdc, 0xe4, 0x1c, 0xf6,
+			0x0f, 0x4d, 0x0b, 0x89, 0x8e, 0x66, 0x93, 0x88, 0xb8,
+			0xad, 0xbc, 0xdc, 0x96, 0x01, 0x9f, 0x16, 0x70, 0xf5,
+			0x4f, 0xa4, 0x0e, 0x0f, 0xc3, 0xf6, 0x9c, 0xe1, 0xa1,
+			0xe3, 0xec, 0x9d, 0x09, 0xcd, 0x90, 0x52, 0x26, 0x09,
+			0xd9, 0x9c, 0xde, 0xbd, 0xdf, 0xbb, 0xf4, 0x50, 0xd1,
+			0x89, 0x68, 0xf6, 0x86, 0x53, 0x33, 0x5a, 0xf6, 0x54,
+			0xb5, 0x7c, 0xe1, 0xd7, 0x8d, 0xb8, 0x58, 0xf5, 0xda,
+			0x14, 0xc8, 0x35, 0x1c, 0xcf, 0x44, 0x62, 0xbc, 0xd7,
+			0xe3, 0xd8, 0x32, 0xcf, 0x16, 0xf5, 0x2f, 0x55, 0x23,
+			0xc0, 0x1b, 0xc4, 0xe3, 0x28, 0xc8, 0xc8, 0x97, 0x70,
+			0x8b, 0x06, 0x98, 0xfb, 0xf6, 0x33, 0x6b, 0x86, 0x1c,
+			0xdb, 0x2a, 0x3c, 0x08, 0x08, 0x57, 0xd3, 0x4a, 0xf8,
+			0x22, 0x26, 0x78, 0x65, 0x5f, 0xa6, 0xf8, 0x9f, 0x22,
+			0x76, 0x62, 0xb0, 0x08, 0x68, 0x70, 0xea, 0x72, 0x57,
+			0x6b, 0xe7, 0xf7, 0xc6, 0x12, 0x9a, 0x49, 0x50, 0xa9,
+			0xa5, 0x6c, 0xe7, 0xda, 0xb4, 0xbf, 0xb6, 0xbf, 0x4f,
+			0xdf, 0x9e, 0x9b, 0xb4, 0xb3, 0x8d, 0x1a, 0x12, 0x16,
+			0x68, 0xd2, 0x63, 0xae, 0x92, 0x77, 0x1f, 0x03, 0xa5,
+			0xed, 0x58, 0x3b, 0xe9, 0x0b, 0xfe, 0xfc, 0xae, 0x53,
+			0x0b, 0x5f, 0x13, 0xf2, 0xd2, 0xe2, 0x0b, 0xec, 0x75,
+			0x85, 0x68, 0x0c, 0x57, 0xde, 0x1b, 0x6d, 0x78, 0x0b,
+			0x19, 0x66, 0xa8, 0xf5, 0x45, 0x72, 0x2b, 0x01, 0x06,
+			0xf6, 0xd1, 0x47, 0x21, 0x24, 0x07, 0xf7, 0x71, 0x03,
+			0xbc, 0xb0, 0x7c, 0x5b, 0x5c, 0x24, 0xff, 0x74, 0x47,
+			0x62, 0x81, 0xc3, 0x0b, 0x31, 0x76, 0x90, 0x5b, 0xef,
+			0x95, 0xa8, 0xa7, 0x02, 0xa1, 0xbf, 0xe1, 0xf4, 0x16,
+			0x06, 0x8a, 0x97, 0x39, 0x35, 0xcf, 0xf3, 0xa7, 0x4a,
+			0x43, 0xba, 0x05, 0x95, 0x7d, 0x73, 0x76, 0x7a, 0x53,
+			0xef, 0xf8, 0x4e, 0xcb, 0x04, 0x70, 0x4c, 0xee, 0xff,
+			0x82, 0xbd, 0xcd, 0xc1, 0xbe, 0x3d, 0x83, 0x71, 0x03,
+			0xf0, 0xc0, 0x2b, 0x98, 0xf9, 0x60, 0x54, 0x02, 0x7d,
+			0xa6, 0x41, 0xcc, 0xa3, 0xd7, 0x8d, 0xfd, 0xce, 0x28,
+			0xae, 0x0f, 0x48, 0x17, 0x2a, 0xaf, 0xe9, 0xb9, 0x4a,
+			0x8a, 0x22, 0xd2, 0x4d, 0xd3, 0x1b, 0xa3, 0x39, 0x88,
+			0x8a, 0x8f, 0x5b, 0x44, 0x97, 0xb9, 0x04, 0x1b, 0x58,
+			0x67, 0x74, 0x2f, 0x07, 0x7a, 0x52, 0xa9, 0x9d, 0xa4,
+			0x41, 0x28, 0xf2, 0x35, 0xca, 0x68, 0x4e, 0x4a, 0x3a,
+			0x66, 0xb9, 0x88, 0x2e, 0x65, 0x1d, 0x47, 0x04, 0xed,
+			0xdb, 0xe1, 0x40, 0x12, 0x06, 0x13, 0x62, 0x28, 0x3d,
+			0x0b, 0x35, 0x06, 0xc6, 0x2b, 0xb3, 0x71, 0x3c, 0xfa,
+			0x77, 0xec, 0x47, 0x93, 0x78, 0x36, 0x25, 0x19, 0xd7,
+			0x70, 0x30, 0x8a, 0x4c, 0x94, 0xdc, 0x3e, 0xeb, 0x61,
+			0x25, 0xbc, 0xa0, 0x27, 0xd9, 0x17, 0xa5, 0x19, 0x4f,
+			0xf4, 0x93, 0x32, 0x56, 0x9a, 0x0b, 0x77, 0xb4, 0x55,
+			0x1b, 0x8f, 0x9e, 0x69, 0x5b, 0xe2, 0x6d, 0x70, 0x15,
+			0x79, 0x5c, 0xf6, 0xb6, 0x04, 0xa2, 0x01, 0x37, 0x74,
+			0x20, 0xb8, 0x62, 0xf6, 0x37, 0x3c, 0xab, 0xca, 0x71,
+			0xa5, 0x8a, 0x56, 0x5d, 0x6a, 0x4a, 0x61, 0x2e, 0xb8,
+			0x62, 0x7d, 0x47, 0x34, 0x7d, 0xcd, 0x4d, 0x70, 0x23,
+			0xf5, 0xaa, 0xd1, 0xa5, 0xf0, 0x4c, 0x38, 0xc3, 0x98,
+			0x79, 0x4c, 0x0b, 0x6b, 0xcc, 0xe7, 0xd7, 0x09, 0xae,
+			0x23, 0x9b, 0x2f, 0xde, 0x70, 0xc6, 0xad, 0x0f, 0x66,
+			0xb5, 0x78, 0x6b, 0x0b, 0xb0, 0x2e, 0x94, 0xf2, 0xa8,
+			0x8b, 0x74, 0xf0, 0x03, 0x47, 0xd8, 0xec, 0xe8, 0x1f,
+			0xa3, 0x7b, 0x38, 0x9e, 0x0e, 0xc0, 0x47, 0xd2, 0x0f,
+			0x8e, 0x7f, 0xb1, 0x83, 0xd3, 0x86, 0x79, 0x3c, 0xa1,
+			0xae, 0xc4, 0xaf, 0xae, 0x9d, 0x83, 0xc0, 0xd1, 0x2b,
+			0x2b, 0xda, 0x50, 0x8c, 0xea, 0x41, 0x97, 0x9b, 0x0f,
+			0x15, 0xc2, 0xe2, 0x8f, 0x39, 0x0b, 0x92, 0xdd, 0xde,
+			0x52, 0x62, 0x74, 0xdc, 0xda, 0x11, 0x87, 0x4d, 0xa9,
+			0x4a, 0xc5, 0x2f, 0xae, 0xaf, 0xc1, 0xc3, 0x05, 0xfa,
+			0x38, 0xcc, 0x5c, 0xb1, 0x9f, 0xe0, 0x82, 0x90, 0xb3,
+			0xd5, 0xdc, 0xf4, 0x55, 0xdb, 0xea, 0x94, 0x06, 0x7c,
+			0x2c, 0x82, 0x78, 0xeb, 0xa5, 0x01, 0xf0, 0x3d, 0x4b,
+			0x87, 0xdd, 0xd5, 0x91, 0x4f, 0xf3, 0xa7, 0xdf, 0xa1,
+			0xd8, 0x31, 0xde, 0x05, 0x99, 0x67, 0x3d, 0xa4, 0x6b,
+			0x19, 0xa3, 0xe8, 0x55, 0xb7, 0xf5, 0xc3, 0x63, 0x5e,
+			0xd4, 0x38, 0xf9, 0x24, 0x64, 0x7d, 0x17, 0xc1, 0x07,
+			0xbe, 0x39, 0x54, 0x1b, 0x44, 0xe5, 0xc6, 0x3c, 0x02,
+			0xb1, 0x6f, 0xff, 0x8c, 0xcb, 0x79, 0xe2, 0xec, 0x4d,
+			0x01, 0xfa, 0x7f, 0x88, 0x1d, 0xc3, 0x4c, 0x6a, 0xfb,
+			0x0b, 0xc6, 0x57, 0xc3, 0xd8, 0x24, 0x47, 0x41, 0xbd,
+			0x27, 0xc4, 0xd4, 0x49, 0xfb, 0x52, 0xe6, 0x77, 0x5f,
+			0x0a, 0xdf, 0xea, 0xd5, 0xd3, 0x22, 0xc3, 0x53, 0x16,
+			0xf3, 0x1b, 0x7b, 0x09, 0xd7, 0x10, 0x0e, 0x23, 0xae,
+			0x16, 0x8a, 0x93, 0xcb, 0xc9, 0xb7, 0xb8, 0xff, 0xd2,
+			0x50, 0x1f, 0x25, 0xa7, 0x71, 0x8f, 0x3f, 0xc0, 0xe1,
+			0x37, 0x10, 0x0b, 0x43, 0x6e, 0x2b, 0x16, 0x59, 0x8f,
+			0x77, 0x77, 0x6b, 0x77, 0xce, 0x76, 0x6b, 0x37, 0x81,
+			0xaf, 0x83, 0x42, 0x92, 0x93, 0xe5, 0x39, 0xca, 0xd2,
+			0x20, 0x2e, 0xcf, 0x24, 0x26, 0x4c, 0x51, 0x1c, 0x58,
+			0xc5, 0x8d, 0x05, 0x11, 0xdf, 0xae, 0x51, 0x38, 0xde,
+			0xab, 0x4e, 0x04, 0xc8, 0x24, 0x24, 0x0d, 0xd5, 0x9c,
+			0x5b, 0x2b, 0xe6, 0x0d, 0x83, 0x95, 0xcd, 0x1c, 0x89,
+			0xa1, 0xaf, 0x67, 0x47, 0xfb, 0x08, 0x02, 0xf8, 0x8b,
+			0x63, 0x05, 0x73, 0x20, 0x64, 0xd7, 0x52, 0x15, 0xa4,
+			0x5d, 0x63, 0x73, 0x73, 0x12, 0x0b, 0xdd, 0xfe, 0x9f,
+			0xb7, 0xe8, 0xa8, 0x94, 0x3a, 0x86, 0xff, 0xcf, 0x7d,
+			0x24, 0xbd, 0xb9, 0xea, 0x68, 0x23, 0xf4, 0x07, 0xc3,
+			0xfe, 0x63, 0xd0, 0xab, 0x65, 0x8a, 0xf0, 0x6d, 0x81,
+			0x8c, 0xc8, 0x0e, 0xc6, 0x6b, 0xdd, 0x2e, 0x65, 0x9b,
+			0x17, 0xcf, 0x82, 0x69, 0x46, 0xba, 0x62, 0x5d, 0x31,
+			0x33, 0x60, 0x18, 0x94, 0xa5, 0x77, 0x24, 0xc6, 0x45,
+			0xe5, 0xb3, 0xd5, 0x12, 0x10, 0xc9, 0x22, 0x98, 0xf9,
+			0xca, 0x20, 0x89, 0x79, 0x04, 0x08, 0xf5, 0x1c, 0xf8,
+			0x50, 0x8c, 0x25, 0xaa, 0x90, 0x90, 0x44, 0xbc, 0xfb,
+			0x5d, 0x3f, 0xf8, 0x38, 0x64, 0xca, 0x8d, 0xff, 0x17,
+			0xce, 0x70, 0x51, 0x90, 0x75, 0x6b, 0x7d, 0x64, 0x43,
+			0x56, 0xcd, 0xf8, 0x85, 0x93, 0x65, 0x09, 0x81, 0x30,
+			0x76, 0x79, 0xcc, 0xdf, 0x9c, 0x6d, 0xff, 0x89, 0x38,
+			0x60, 0xbf, 0x07, 0xcb, 0x2f, 0xc9, 0x87, 0xd7, 0xac,
+			0x74, 0x19, 0x57, 0x90, 0x5e, 0x69, 0x61, 0xf6, 0xca,
+			0xea, 0x45, 0x6b, 0xe2, 0xfe, 0x2c, 0xff, 0x1b, 0x23,
+			0x15, 0x52, 0xdd, 0x57, 0xfe, 0x1d, 0x10, 0xea, 0x0f,
+			0xce, 0x98, 0xe7, 0x47, 0x27, 0xec, 0x36, 0xe5, 0x68,
+			0x17, 0xcf, 0xdc, 0xb9, 0xef, 0x6a, 0xbc, 0xec, 0x78,
+			0x08, 0x64, 0x06, 0xe1, 0x1c, 0xc6, 0x87, 0xd6, 0x0a,
+			0xb1, 0x81, 0xc6, 0xb6, 0xf8, 0x8b, 0xe3, 0x19, 0x8c,
+			0xce, 0x46, 0x40, 0xc5, 0xc2, 0xae, 0x50, 0x26, 0x4a,
+			0x90, 0x91, 0x8d, 0xfe, 0x6b, 0x7d, 0x0a, 0x54, 0x4a,
+			0x4b, 0x48, 0x74, 0x4a, 0x37, 0x21, 0x7f, 0xdd, 0x87,
+			0xa3, 0x1e, 0xac, 0xcd, 0xf5, 0x9e, 0x75, 0xa2, 0x52,
+			0x63, 0x76, 0xca, 0x9e, 0x02, 0xeb, 0xe6, 0xa6, 0x73,
+			0xad, 0xea, 0xe8, 0x3e, 0x6f, 0x44, 0xed, 0xe8, 0x01,
+			0x29, 0x19, 0x6a, 0x20, 0x35, 0xa7, 0xf0, 0xf1, 0xaf,
+			0xc0, 0x3b, 0xb1, 0xd5, 0xe4, 0xfb, 0xf7, 0xd7, 0x2f,
+			0x33, 0x6c, 0x73, 0xfd, 0xe5, 0x5c, 0x63, 0xf6, 0x1c,
+			0x06, 0x13, 0xaf, 0xc1, 0x80, 0x55, 0x07, 0xae, 0x8c,
+			0x13, 0x74, 0xf6, 0xe0, 0x54, 0x15, 0xd8, 0xe0, 0xa5,
+			0x03, 0xcf, 0x22, 0xbe, 0x18, 0xef, 0x26, 0xad, 0x9c,
+			0x9d, 0x51, 0xb1, 0x3b, 0x37, 0x03, 0xbf, 0xf0, 0xc5,
+			0xcb, 0x6c, 0x5d, 0x30, 0xa9, 0x5a, 0x10, 0x90, 0xfa,
+			0xb4, 0xd4, 0x0e, 0x6b, 0x4a, 0x0a, 0x6c, 0x9d, 0x2e,
+			0x69, 0xe8, 0xec, 0x69, 0xe2, 0x50, 0xab, 0x2f, 0xdc,
+			0xff, 0xaf, 0xac, 0x65, 0xe7, 0xf0, 0xc1, 0x6f, 0x7c,
+			0x2d, 0xa1, 0xeb, 0x97, 0x90, 0x7c, 0x1e, 0xa8, 0x53,
+			0x1b, 0x87, 0xc5, 0xa9, 0xa1, 0xcf, 0x86, 0x7e, 0x11,
+			0xf8, 0xd6, 0x14, 0xda, 0x19, 0x81, 0x19, 0xb4, 0x45,
+			0x1c, 0x7c, 0xb9, 0x96, 0xa2, 0xac, 0x79, 0x24, 0x94,
+			0x7c, 0xb2, 0x1e, 0x83, 0xea, 0xc4, 0xb9, 0xd2, 0x0d,
+			0x4c, 0x55, 0x3d, 0x15, 0x7b, 0x65, 0xd8, 0xff, 0x03,
+			0x5d, 0xed, 0x3c, 0x94, 0x76, 0x19, 0x40, 0x3b, 0xcc,
+			0x45, 0xbe, 0x91, 0x19, 0x8c, 0x75, 0xe5, 0xd2, 0xbe,
+			0x67, 0x40, 0xb1, 0x67, 0x8e, 0x2a, 0x34, 0xd5, 0x99,
+			0xee, 0xd9, 0x4f, 0x89, 0x7c, 0xf0, 0xd6, 0x93, 0x59,
+			0x4b, 0x3a, 0x8d, 0xe6, 0xbd, 0xde, 0xce, 0xef, 0x8b,
+			0x3c, 0xe3, 0xf7, 0x06, 0x33, 0x27, 0x8d, 0xd9, 0x22,
+			0x1a, 0x65, 0x40, 0xfc, 0x69, 0x1b, 0x7d, 0xf0, 0xed,
+			0xe4, 0xe0, 0x7f, 0x6d, 0x23, 0xed, 0x11, 0xd1, 0x07,
+			0xb0, 0x2f, 0x8a, 0xbf, 0x51, 0x37, 0x22, 0x04, 0xed,
+			0x93, 0xea, 0x1d, 0x0b, 0x30, 0x15, 0x89, 0x22, 0x7a,
+			0x45, 0x56, 0x99, 0xc6, 0xac, 0xd6, 0xce, 0x61, 0xea,
+			0xb2, 0x59, 0xe8, 0xb5, 0xfc, 0x87, 0xa7, 0xfe, 0x09,
+			0xa2, 0x0d, 0x5e, 0xbe, 0xb9, 0xd4, 0x9a, 0x1b, 0x60,
+			0xda, 0xb9, 0x32, 0xf1, 0x30, 0x3e, 0xb2, 0x45, 0x6d,
+			0x55, 0x0c, 0x2c, 0x4b, 0x9a, 0xc0, 0xbb, 0x8e, 0xac,
+			0x9c, 0x95, 0x5f, 0x08, 0x88, 0xa0, 0x53, 0x05, 0x75,
+			0x8d, 0x9e, 0x9d, 0x3f, 0x0f, 0xdd, 0x50, 0x0d, 0xf8,
+			0x11, 0xbd, 0xf9, 0xfb, 0x22, 0x5c, 0x7b, 0x9e, 0x7c,
+			0x8e, 0x2f, 0x0e, 0xdb, 0xb8, 0x1d, 0x0c, 0x5e, 0x82,
+			0xf3, 0x8e, 0xec, 0x32, 0x1c, 0x59, 0x73, 0xa5, 0xf3,
+			0x5b, 0x47, 0x00, 0x64, 0x89, 0x68, 0x3b, 0xaf, 0xe8,
+			0xe2, 0x9b, 0xa6, 0xac, 0x2c, 0xf9, 0x2b, 0x92, 0xf3,
+			0xf0, 0x5b, 0xcc, 0x75, 0x22, 0xd5, 0xf4, 0x2b, 0x06,
+			0x96, 0xc8, 0x50, 0xee, 0xac, 0x62, 0x16, 0x45, 0x9e,
+			0xbc, 0xcc, 0x8f, 0x5a, 0x66, 0xc6, 0x30, 0x7c, 0xe0,
+			0x22, 0xcc, 0xb9, 0xda, 0x0b, 0x0a, 0xbd, 0x2a, 0x2e,
+			0x46, 0x7d, 0xb6, 0x86, 0x70, 0xa3, 0x16, 0x49, 0x85,
+			0x28, 0x7b, 0xe9, 0x00, 0x6b, 0xfa, 0x06, 0xb0, 0xeb,
+			0xbd, 0x67, 0x28, 0x6f, 0x27, 0xd7, 0x9c, 0x7f, 0xda,
+			0xec, 0xf4, 0x7e, 0x55, 0xe1, 0x0c, 0x29, 0x61, 0x7a,
+			0xf5, 0xb6, 0xb8, 0xa5, 0xef, 0x36, 0x6a, 0xad, 0x59,
+			0x22, 0xbd, 0x3d, 0xad, 0x86, 0xe7, 0x4c, 0x69, 0x26,
+			0x0f, 0xbf, 0x67, 0xad, 0x65, 0x32, 0xbd, 0x21, 0xd6,
+			0x59, 0x6b, 0xe3, 0xda, 0xc1, 0x6f, 0x82, 0x41, 0x2c,
+			0xaa, 0xe4, 0x8c, 0xfc, 0x7c, 0x61, 0x28, 0x51, 0x52,
+			0x3d, 0xf1, 0x84, 0xb5, 0x0b, 0xfd, 0x1f, 0x2a, 0x06,
+			0x2e, 0x30, 0xed, 0x63, 0x43, 0xc9, 0x83, 0x97, 0xb1,
+			0xd4, 0x80, 0x6f, 0x2c, 0x50, 0xec, 0x20, 0x95, 0x42,
+			0xa0, 0x34, 0x94, 0x1a, 0xa9, 0x5e, 0x5b, 0x59, 0xe3,
+			0x39, 0xac, 0xbd, 0x2f, 0x77, 0x36, 0x59, 0x9c, 0xc3,
+			0x3c, 0x66, 0x87, 0xf5, 0x81, 0x4b, 0xb0, 0x10, 0x4a,
+			0xe6, 0x46, 0xe7, 0xce, 0x93, 0x7b, 0x24, 0x6b, 0x2e,
+			0xc1, 0xe5, 0xaf, 0x4b, 0x71, 0x22, 0xad, 0x88, 0xda,
+			0x55, 0xcb, 0xe0, 0x73, 0xd1, 0x65, 0x7d, 0xa5, 0x7f,
+			0x36, 0xbc, 0x42, 0xc2, 0x78, 0x9f, 0x88, 0xe8, 0xdb,
+			0xff, 0x8a, 0x5a, 0x80, 0x34, 0x3a, 0x23, 0x4c, 0x8a,
+			0x81, 0xff, 0xbd, 0xb7, 0x88, 0xd0, 0x73, 0x07, 0x8a,
+			0x4e, 0xa7, 0x4a, 0x61, 0x0f, 0x1f, 0x1c, 0xe7, 0x34,
+			0x37, 0x1c, 0x53, 0x90, 0x3b, 0xa4, 0x32, 0x6c, 0x6d,
+			0xe8, 0x00, 0xde, 0xe0, 0x0c, 0x5e, 0x06, 0xef, 0xb8,
+			0x48, 0x2e, 0xb3, 0xda, 0xac, 0x92, 0x4d, 0x0d, 0x95,
+			0x75, 0x44, 0x01, 0x6f, 0x97, 0xc3, 0x29, 0x76, 0x33,
+			0x36, 0x9a, 0xae, 0xfb, 0x1b, 0x43, 0xe5, 0xb1, 0x54,
+			0x3a, 0x9c, 0x76, 0x7f, 0x76, 0x83, 0xc9, 0x9c, 0xd6,
+			0x56, 0x59, 0x83, 0xa9, 0xde, 0xd7, 0xb0, 0xf3, 0x34,
+			0x11, 0x31, 0x06, 0x8e, 0xe9, 0xd4, 0x79, 0xd5, 0x3d,
+			0x31, 0x6b, 0x59, 0xe9, 0x54, 0x69, 0x12, 0xfd, 0x44,
+			0x59, 0x4e, 0x1b, 0x3b, 0xb4, 0x12, 0xe9, 0xfb, 0xb0,
+			0xb4, 0x84, 0xb9, 0x7d, 0xea, 0x4f, 0xd1, 0x5f, 0xd0,
+			0x3e, 0xce, 0xef, 0x5c, 0xf7, 0xea, 0x55, 0xa0, 0x8f,
+			0xa8, 0xa7, 0x98, 0xe7, 0xa1, 0x6b, 0x3f, 0xba, 0x5a,
+			0x32, 0x4b, 0xfa, 0x31, 0xb6, 0x63, 0x86, 0x19, 0x00,
+			0xa2, 0x6d, 0x7d, 0x15, 0x56, 0x05, 0x68, 0xa3, 0xe0,
+			0xf3, 0xd4, 0x82, 0xcf, 0xeb, 0xd4, 0x1c, 0xd0, 0xb6,
+			0x14, 0x5e, 0x9e, 0x6b, 0xed, 0x7a, 0x02, 0x1a, 0xcd,
+			0x09, 0xdc, 0x26, 0x98, 0x50, 0x11, 0x34, 0x39, 0x50,
+			0x5a, 0x70, 0x79, 0x85, 0xca, 0xd2, 0xf2, 0x0c, 0x0d,
+			0x12, 0x1f, 0x2e, 0x41, 0x46, 0x51, 0x72, 0x75, 0x78,
+			0x8c, 0xa4, 0xaf, 0xba, 0xca, 0xd3, 0xdf, 0xea, 0xf8,
+			0x09, 0x0b, 0x36, 0x45, 0x4f, 0x77, 0x83, 0xae, 0xbc,
+			0xc5, 0xce, 0xe1, 0xf6, 0x1d, 0x1e, 0x38, 0x56, 0x9c,
+			0x9f, 0xb1, 0xbd, 0xda, 0xe7, 0xf0, 0xf4, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x0b, 0x1f, 0x2c, 0x38,
+		},
+};
+
+static const struct mldsa_testvector mldsa65_testvector = {
+	.alg = MLDSA65,
+	.pk_len = MLDSA65_PUBLIC_KEY_SIZE,
+	.pk =
+		(const u8[MLDSA65_PUBLIC_KEY_SIZE]) {
+			0x9f, 0x55, 0x1e, 0x7f, 0x9c, 0x08, 0xb2, 0x83, 0xfd,
+			0x5b, 0xa2, 0xac, 0x4f, 0x26, 0xc2, 0xf5, 0x06, 0x05,
+			0x96, 0x08, 0x24, 0xad, 0xec, 0xe4, 0x99, 0xcc, 0x6c,
+			0xbd, 0x55, 0x37, 0x15, 0x94, 0xab, 0x31, 0x9e, 0x56,
+			0xe5, 0xe4, 0x55, 0xec, 0x4d, 0x49, 0x5b, 0x5a, 0x7a,
+			0xe8, 0xc3, 0x4a, 0x08, 0x44, 0x4a, 0xc2, 0x2d, 0xe4,
+			0x61, 0x33, 0x90, 0x20, 0x71, 0x45, 0xa5, 0x45, 0xd0,
+			0x83, 0x2b, 0x32, 0x6c, 0xa7, 0x9e, 0x76, 0xcd, 0xfb,
+			0x58, 0x15, 0x9e, 0x74, 0x0d, 0x67, 0x57, 0xb1, 0x06,
+			0x5b, 0x5d, 0xd5, 0x1c, 0xbb, 0x95, 0x40, 0x1c, 0x71,
+			0x31, 0x03, 0xef, 0xff, 0x04, 0x6b, 0xdd, 0xa2, 0xf0,
+			0x32, 0x00, 0x72, 0xbc, 0x87, 0xb6, 0x2c, 0x1f, 0x90,
+			0x7f, 0x92, 0xa0, 0xb2, 0x04, 0xdd, 0xa9, 0xaf, 0x7f,
+			0x01, 0x28, 0x4c, 0xb2, 0x57, 0x2d, 0x56, 0x93, 0xd0,
+			0xc7, 0x54, 0x02, 0x90, 0x57, 0x70, 0x23, 0x57, 0xe8,
+			0xe7, 0x33, 0x32, 0x98, 0xfc, 0x9b, 0x8e, 0x6e, 0x7b,
+			0xaa, 0x5d, 0xb5, 0x4e, 0xe0, 0x5d, 0x97, 0xa3, 0xea,
+			0x43, 0x7e, 0xb3, 0xa4, 0x8c, 0xcf, 0xdc, 0xc0, 0x51,
+			0xa7, 0x99, 0x45, 0x3d, 0x3c, 0xa0, 0xba, 0xc5, 0xff,
+			0xe1, 0x89, 0xb3, 0x7d, 0xc3, 0xdc, 0xe2, 0x23, 0x81,
+			0xff, 0xa9, 0xc7, 0x93, 0xc6, 0x67, 0xad, 0x94, 0xcf,
+			0xeb, 0x91, 0x78, 0x15, 0x25, 0xf7, 0xf5, 0x06, 0x08,
+			0x2f, 0x0c, 0xee, 0x0b, 0x6a, 0x06, 0x59, 0xe0, 0x1f,
+			0x2e, 0x5a, 0x12, 0x06, 0xf5, 0xf4, 0x8e, 0x75, 0x57,
+			0xa9, 0x33, 0x23, 0x0f, 0xc2, 0x6f, 0x02, 0xf8, 0x68,
+			0x0f, 0x62, 0x02, 0x81, 0xfe, 0x03, 0x7c, 0xaf, 0xd7,
+			0x42, 0x5b, 0xcc, 0xe7, 0x2b, 0xea, 0x49, 0xab, 0x03,
+			0x6d, 0x0a, 0x02, 0xae, 0x47, 0x79, 0xce, 0xfd, 0x18,
+			0x76, 0x07, 0x9e, 0xa6, 0xbf, 0x7e, 0x8d, 0x73, 0xf9,
+			0x44, 0xeb, 0x8c, 0xc5, 0x59, 0xb7, 0x19, 0xf6, 0x73,
+			0x53, 0x42, 0x2a, 0x55, 0x7b, 0xb4, 0x56, 0x49, 0x08,
+			0x9e, 0x9a, 0x65, 0x60, 0x70, 0x1d, 0xbd, 0xc6, 0x85,
+			0x29, 0xde, 0xfe, 0x44, 0xae, 0xdf, 0x25, 0xfd, 0x5b,
+			0x74, 0x6c, 0x96, 0xe6, 0x81, 0x37, 0x80, 0xe0, 0x9e,
+			0xf3, 0x75, 0x63, 0xb4, 0xc9, 0x2f, 0x71, 0xe6, 0xeb,
+			0xdf, 0xaf, 0x7e, 0xff, 0x9e, 0xe0, 0xbf, 0xca, 0xca,
+			0x11, 0xed, 0xc6, 0x04, 0xd8, 0x49, 0x13, 0x2c, 0x63,
+			0xf1, 0xb3, 0x17, 0x74, 0xd9, 0x50, 0x3f, 0xb9, 0x29,
+			0x0e, 0x48, 0xa7, 0xf0, 0xdc, 0x78, 0x18, 0x0e, 0x9f,
+			0xb7, 0xde, 0x36, 0x79, 0x67, 0xa4, 0x23, 0x08, 0xe7,
+			0x62, 0xe8, 0xa4, 0xe5, 0xcf, 0xff, 0x35, 0x55, 0x36,
+			0x2e, 0x3a, 0xe4, 0x45, 0x6a, 0x80, 0xf2, 0xca, 0xe7,
+			0x40, 0x79, 0x14, 0xc4, 0x62, 0x38, 0xbb, 0xd0, 0x4e,
+			0x6c, 0xb5, 0x85, 0x42, 0x3f, 0x35, 0xf7, 0xd7, 0x54,
+			0xb8, 0x2b, 0x8b, 0xd5, 0x6f, 0x16, 0x61, 0x27, 0x23,
+			0xac, 0xdb, 0xea, 0x9b, 0x3b, 0x99, 0xcd, 0x79, 0xe6,
+			0x12, 0x09, 0x99, 0x09, 0xa4, 0xe1, 0x88, 0x25, 0x00,
+			0x9e, 0x60, 0x16, 0x63, 0xd7, 0x42, 0x9b, 0xcc, 0x36,
+			0x9a, 0x8d, 0xa3, 0x75, 0x36, 0xa1, 0xa8, 0xfc, 0xa2,
+			0xfe, 0x29, 0x26, 0x4c, 0x93, 0x21, 0x44, 0x6b, 0x1c,
+			0xba, 0xbd, 0xef, 0xff, 0x6d, 0x1f, 0x2b, 0x6c, 0x66,
+			0x81, 0x9a, 0x3a, 0x1d, 0x0b, 0xd7, 0x24, 0xd4, 0xb8,
+			0x93, 0xb5, 0x22, 0xf9, 0xd2, 0xf4, 0xa5, 0x05, 0x78,
+			0x38, 0xae, 0x58, 0xf6, 0x50, 0x8f, 0x47, 0x1d, 0xf3,
+			0xfb, 0x0d, 0x04, 0x14, 0xd1, 0xd6, 0xd8, 0x2e, 0xf2,
+			0xbd, 0xf5, 0x71, 0x86, 0x4c, 0xdd, 0x61, 0x24, 0x18,
+			0x5b, 0x54, 0xf5, 0xcd, 0x99, 0x89, 0x01, 0x8e, 0xd1,
+			0x19, 0x52, 0xbc, 0x45, 0xed, 0x0e, 0xec, 0x72, 0x2f,
+			0x5a, 0xe7, 0xdf, 0x36, 0x1c, 0x57, 0x9f, 0xb2, 0x8b,
+			0xf2, 0x78, 0x1b, 0x3e, 0xc5, 0x48, 0x1f, 0x27, 0x04,
+			0x76, 0x10, 0x44, 0xee, 0x5c, 0x68, 0x8f, 0xca, 0xd7,
+			0x31, 0xfc, 0x5c, 0x40, 0x03, 0x2e, 0xbd, 0x1d, 0x59,
+			0x13, 0x57, 0xbc, 0x33, 0xc6, 0xa1, 0xa3, 0xe5, 0x55,
+			0x79, 0x9b, 0x7e, 0x49, 0xbb, 0x23, 0x96, 0xc3, 0x1c,
+			0xfe, 0x66, 0xeb, 0x5b, 0x5f, 0xe5, 0x03, 0xc9, 0xa4,
+			0xac, 0x4d, 0xc4, 0x50, 0xbb, 0xd3, 0xc1, 0x91, 0x48,
+			0xe0, 0x93, 0x92, 0x2a, 0xdb, 0x41, 0x37, 0x98, 0xbc,
+			0xa2, 0x7a, 0x09, 0x92, 0x0b, 0x1c, 0xe6, 0x4b, 0x1e,
+			0x8e, 0x78, 0x81, 0x74, 0x7d, 0x6b, 0x71, 0xd5, 0xe7,
+			0x0e, 0x7b, 0xc2, 0x74, 0x5d, 0x89, 0xf1, 0xfa, 0x59,
+			0xaa, 0xf7, 0x86, 0x66, 0x7e, 0xc2, 0x9c, 0xf4, 0xd5,
+			0x8d, 0xc0, 0xb7, 0xb7, 0xa2, 0xd5, 0xcd, 0x51, 0xc3,
+			0x7d, 0xa9, 0x5e, 0x46, 0xba, 0x06, 0xa3, 0x4d, 0x60,
+			0xd6, 0x68, 0xc6, 0xf9, 0x63, 0x88, 0x17, 0x5c, 0x20,
+			0xe1, 0xc4, 0x0f, 0x3f, 0xc1, 0xa9, 0xa7, 0x3e, 0x39,
+			0xef, 0x2f, 0xaf, 0xc4, 0x69, 0x29, 0xe3, 0xd4, 0x8d,
+			0xe0, 0x0e, 0x88, 0xc2, 0x93, 0x43, 0xfb, 0x28, 0xcf,
+			0x5d, 0x85, 0x50, 0xf7, 0xeb, 0x42, 0xf5, 0x87, 0xde,
+			0xa5, 0x65, 0xef, 0x43, 0x0c, 0x57, 0x76, 0x09, 0xf4,
+			0x5f, 0xde, 0x81, 0x0a, 0xd9, 0x59, 0x41, 0xa4, 0x6a,
+			0xb7, 0x05, 0xc7, 0xa5, 0xfe, 0x49, 0xd5, 0x9b, 0x57,
+			0x13, 0x14, 0x66, 0xe2, 0xb9, 0xcc, 0x09, 0x35, 0xd4,
+			0xb0, 0xe0, 0xd1, 0x0d, 0x7e, 0x50, 0x48, 0x45, 0x21,
+			0x00, 0x67, 0xb2, 0xad, 0xa7, 0x46, 0xe2, 0x6f, 0x70,
+			0xe5, 0x3c, 0x88, 0x04, 0xaa, 0x21, 0xde, 0x03, 0xb6,
+			0x6f, 0xfe, 0x43, 0x51, 0xdc, 0x2e, 0x5c, 0x6c, 0x77,
+			0x8f, 0x8e, 0x9d, 0x1a, 0x5b, 0x35, 0xc5, 0xe4, 0x48,
+			0x82, 0x17, 0x4b, 0xf0, 0xea, 0xc9, 0x0e, 0xd2, 0x8f,
+			0xcd, 0xd5, 0x01, 0xbd, 0x7f, 0x0f, 0xf5, 0xae, 0x92,
+			0x28, 0x1e, 0x2c, 0xf4, 0xe9, 0x03, 0xf7, 0x0a, 0xeb,
+			0x84, 0x18, 0xa1, 0x37, 0x38, 0x8a, 0x11, 0xa2, 0x5d,
+			0x8c, 0xf6, 0xe4, 0x3f, 0x5b, 0x87, 0x07, 0x6b, 0xb4,
+			0x07, 0xe0, 0x8f, 0x30, 0xc4, 0xfa, 0x27, 0xae, 0xfc,
+			0x02, 0xd1, 0x21, 0x5c, 0xbc, 0x0b, 0x93, 0x6e, 0x7e,
+			0xf9, 0x6b, 0x80, 0x7a, 0x25, 0x84, 0x20, 0xf1, 0x6a,
+			0xfa, 0x75, 0xed, 0x57, 0x61, 0x62, 0xa7, 0xf6, 0x5b,
+			0xe1, 0xb0, 0x38, 0xc8, 0xe9, 0x6d, 0x3f, 0xef, 0x1e,
+			0x99, 0x0b, 0xb7, 0xc8, 0x9f, 0x76, 0x5c, 0x04, 0x1f,
+			0x02, 0x92, 0x00, 0xa7, 0x38, 0x3d, 0x00, 0x3b, 0xa7,
+			0xbc, 0x39, 0x6e, 0xab, 0xf5, 0x10, 0xa8, 0xba, 0xd6,
+			0x28, 0x6b, 0x0e, 0x00, 0x48, 0xf9, 0x3b, 0x5c, 0xde,
+			0x59, 0x93, 0x46, 0xd6, 0x61, 0x52, 0x81, 0x71, 0x0f,
+			0x0e, 0x61, 0xac, 0xc6, 0x7f, 0x15, 0x93, 0xa7, 0xc1,
+			0x16, 0xb5, 0xef, 0x85, 0xd1, 0xa7, 0x61, 0xc2, 0x85,
+			0x1d, 0x61, 0xc6, 0xae, 0xb3, 0x9e, 0x8d, 0x23, 0xa3,
+			0xc8, 0xd5, 0xf2, 0xc7, 0x1b, 0x7e, 0xef, 0xd2, 0xdf,
+			0x25, 0xaf, 0x4e, 0x81, 0x15, 0x59, 0xe5, 0x36, 0xb1,
+			0xf1, 0xd5, 0xda, 0x58, 0xd8, 0xd9, 0x0d, 0x6d, 0xc9,
+			0x25, 0xb5, 0xe8, 0x1d, 0x3b, 0xca, 0x2d, 0xab, 0xf2,
+			0xe2, 0xe9, 0x55, 0xd7, 0xf4, 0xc7, 0xd0, 0x57, 0x7a,
+			0x86, 0x15, 0x0a, 0x5a, 0x8b, 0xd7, 0x3f, 0x66, 0x0f,
+			0x80, 0xb4, 0xe0, 0x5c, 0x33, 0xed, 0xaf, 0x1b, 0x3b,
+			0x6d, 0x1c, 0xd9, 0x8c, 0xb5, 0x96, 0xa3, 0xfb, 0xcf,
+			0xcc, 0x97, 0x1c, 0xae, 0x06, 0x19, 0x41, 0x61, 0xf8,
+			0x97, 0x6b, 0x82, 0x5e, 0x1c, 0xbf, 0x6f, 0x43, 0x3d,
+			0xe5, 0x00, 0xf5, 0xfe, 0x66, 0x48, 0x26, 0x31, 0xa1,
+			0x72, 0x67, 0x6e, 0xd4, 0x5b, 0x6f, 0x66, 0xde, 0x70,
+			0x8b, 0x2b, 0xc3, 0xa2, 0x30, 0xe9, 0x55, 0xc8, 0xff,
+			0xf8, 0xd0, 0xdd, 0xa9, 0x21, 0x85, 0x6e, 0x6c, 0x82,
+			0x66, 0xcc, 0x52, 0xf0, 0x9e, 0x1e, 0xb5, 0x3a, 0xff,
+			0x4c, 0xf3, 0xae, 0x02, 0xc3, 0x4b, 0x76, 0x25, 0xbd,
+			0xb0, 0x21, 0x54, 0x61, 0xda, 0x16, 0xd3, 0x23, 0x86,
+			0x41, 0xa1, 0x4c, 0x59, 0x15, 0x95, 0x65, 0x85, 0xb6,
+			0x8e, 0xa6, 0x37, 0xc0, 0xa2, 0x71, 0x1d, 0x67, 0x44,
+			0x7b, 0xe5, 0x4c, 0x4f, 0xb6, 0x2c, 0x46, 0xf7, 0x29,
+			0xa5, 0xf2, 0xd3, 0x51, 0x19, 0x91, 0x4d, 0xa7, 0xb5,
+			0x05, 0xb9, 0x6e, 0x61, 0x6e, 0xf8, 0xc0, 0x01, 0xe5,
+			0x41, 0x0a, 0x89, 0x64, 0x77, 0xf2, 0xc8, 0x63, 0x2d,
+			0x9d, 0x27, 0x7f, 0x47, 0x30, 0x39, 0xdf, 0xb6, 0x6e,
+			0x4f, 0x00, 0x3f, 0x15, 0xc6, 0xaf, 0x62, 0xdf, 0x3f,
+			0x47, 0xe8, 0x42, 0x90, 0x77, 0x23, 0x7a, 0xaa, 0x99,
+			0x53, 0x03, 0x63, 0x60, 0x59, 0x07, 0x52, 0x3c, 0xb5,
+			0x67, 0x59, 0xfe, 0x08, 0xe6, 0x43, 0x0f, 0x3b, 0x08,
+			0x7c, 0xc7, 0x07, 0x3c, 0xfa, 0x65, 0xea, 0x69, 0x51,
+			0x41, 0x31, 0xb3, 0x05, 0x69, 0xba, 0x2c, 0xbf, 0x89,
+			0x25, 0x9e, 0xfe, 0x07, 0x13, 0x78, 0x0e, 0x16, 0x54,
+			0xdf, 0x23, 0xdf, 0x10, 0x69, 0x79, 0xd0, 0x33, 0xd7,
+			0x21, 0x8b, 0xc8, 0x2a, 0xd0, 0x74, 0x0a, 0xfa, 0xb1,
+			0x6f, 0xa3, 0xcb, 0x1d, 0xca, 0x4f, 0x00, 0x46, 0x6c,
+			0x42, 0x09, 0xe0, 0x30, 0x89, 0x08, 0x33, 0x9b, 0x7b,
+			0x7b, 0x0f, 0x69, 0x5c, 0x0d, 0x34, 0x91, 0xfc, 0xfe,
+			0x22, 0x82, 0x02, 0xcd, 0xfa, 0x97, 0xe8, 0x28, 0x1d,
+			0xbc, 0x13, 0x0b, 0xfd, 0x47, 0xa1, 0x7e, 0xa2, 0x86,
+			0x4d, 0x6f, 0x12, 0x51, 0x35, 0x7d, 0x76, 0x8a, 0x58,
+			0x05, 0xb6, 0x39, 0xa1, 0x2f, 0xd7, 0xda, 0xaf, 0x00,
+			0xa0, 0x1a, 0x94, 0xd8, 0x23, 0x34, 0x99, 0x5c, 0xaf,
+			0xcc, 0x15, 0x4b, 0x56, 0xb2, 0xd2, 0x81, 0x07, 0xd3,
+			0xf3, 0x47, 0xa2, 0x45, 0x93, 0xcb, 0xae, 0xa7, 0x6b,
+			0x3f, 0xf9, 0xea, 0xfc, 0x0e, 0x64, 0xf2, 0x93, 0x7f,
+			0x24, 0x22, 0x73, 0x86, 0xc7, 0x2d, 0x75, 0x9b, 0x41,
+			0x8b, 0xfb, 0x3b, 0x26, 0x2a, 0xe5, 0x0b, 0xd4, 0x00,
+			0xe3, 0x2c, 0x69, 0x49, 0x62, 0x6c, 0x13, 0x58, 0x6e,
+			0xac, 0x43, 0xe5, 0x2b, 0x3b, 0x88, 0xdc, 0xd4, 0x41,
+			0xe8, 0xee, 0x4e, 0xc3, 0x28, 0x91, 0x17, 0x9a, 0x5a,
+			0xdb, 0x80, 0x8b, 0x4d, 0x64, 0xcc, 0xbe, 0x66, 0xa4,
+			0x62, 0xfb, 0x13, 0x44, 0x10, 0xd9, 0xe4, 0xd5, 0xa5,
+			0xae, 0x9e, 0x42, 0x50, 0xfc, 0x78, 0xad, 0xfa, 0xc4,
+			0xd0, 0x5a, 0x60, 0x9b, 0x45, 0x2b, 0x61, 0x5c, 0x57,
+			0xb5, 0x92, 0x28, 0xe9, 0xf5, 0x35, 0x67, 0xc1, 0x5e,
+			0xa8, 0x1c, 0x99, 0x36, 0x38, 0xb8, 0x5c, 0xff, 0x3d,
+			0xa0, 0xfc, 0xb0, 0xbc, 0x3d, 0x2c, 0xb4, 0x36, 0x17,
+			0xb4, 0x6d, 0xb5, 0x39, 0x45, 0xa9, 0x2a, 0x6b, 0xa2,
+			0x24, 0x44, 0x30, 0xab, 0x2c, 0x82, 0x36, 0xdc, 0xd6,
+			0x36, 0x5d, 0x0a, 0xdc, 0xee, 0x0f, 0x2b, 0x28, 0x99,
+			0xdc, 0x67, 0x0d, 0xea, 0x6e, 0x42, 0xb9, 0x45, 0x7f,
+			0xd2, 0x96, 0x1e, 0x60, 0x42, 0xeb, 0x1e, 0x5f, 0x8e,
+			0xa9, 0xdc, 0xd3, 0x8a, 0xd6, 0xbd, 0x4e, 0x1f, 0x42,
+			0x75, 0x1d, 0xe2, 0xc6, 0x11, 0xc9, 0x80, 0x1f, 0xfe,
+			0x99, 0x52, 0x4d, 0x7b, 0x35, 0xf7, 0xb7, 0xc3, 0xee,
+			0xd6, 0x94, 0xf5, 0x74, 0xa0, 0x69, 0xcd, 0x1f, 0x2b,
+			0xd0, 0x87, 0xf7, 0x8c, 0x69, 0xc5, 0x96, 0x70, 0x91,
+			0xe8, 0x3d, 0xd2, 0xcc, 0xf1, 0x4c, 0xcd, 0xe2, 0x14,
+			0x00, 0x10, 0x4a, 0xd9, 0x6a, 0x5d, 0x65, 0x2c, 0x4b,
+			0x79, 0x0c, 0xc4, 0x78, 0x5e, 0xc8, 0xc5, 0x37, 0x74,
+			0x6d, 0x50, 0x5c, 0x34, 0x1f, 0xe0, 0xf4, 0xe3, 0xe1,
+			0x86, 0x68, 0xb1, 0xea, 0x70, 0xf0, 0xae, 0xe4, 0x59,
+			0xa1, 0x08, 0x7e, 0x35, 0xa3, 0x16, 0xd2, 0xb0, 0xa3,
+			0xd4, 0xb0, 0x74, 0x8c, 0x05, 0x79, 0x73, 0xfb, 0xe6,
+			0x65, 0x96, 0x15, 0x07, 0xd5, 0xaf, 0x88, 0x9e, 0x6b,
+			0xf0, 0xbb, 0x3f, 0xe6, 0xd1, 0x6a, 0xe7, 0xc9, 0xae,
+			0xd9, 0xb0, 0x16, 0x1c, 0x40, 0xeb, 0xdb, 0xc1, 0xbf,
+			0x83, 0xdb, 0x8a, 0x4f, 0x96, 0xca, 0xd7, 0x22, 0x06,
+			0x87, 0x08, 0x9d, 0x65, 0x2f, 0xd9, 0x8e, 0x95, 0x6c,
+			0xcc, 0xbf, 0x76, 0x2a, 0xea, 0x5c, 0x8e, 0x5b, 0x17,
+			0x0f, 0x75, 0x7b, 0xfa, 0xf9, 0xfb, 0xaa, 0x92, 0xc7,
+			0x7e, 0x63, 0x63, 0x54, 0xa4, 0xff, 0xf6, 0xc0, 0xc0,
+			0xf5, 0x70, 0xd8, 0xe3, 0xa4, 0x79, 0x16, 0xf0, 0x6f,
+			0x90, 0x5e, 0xb7, 0xab, 0x6f, 0xab, 0x75, 0x3b, 0xe1,
+			0x4c, 0xa8, 0x0b, 0x72, 0x5f, 0x5f, 0x11, 0x22, 0x36,
+			0x71, 0x20, 0xd3, 0x5b, 0x5e, 0x07, 0x06, 0x76, 0x1a,
+			0xcc, 0x5e, 0x7c, 0x97, 0x7d, 0xb2, 0x6b, 0xf8, 0x39,
+			0x89, 0x37, 0xb6, 0x6d, 0xea, 0x74, 0x57, 0x28, 0xd7,
+			0x0e, 0x9b, 0xeb, 0x28, 0x88, 0x90, 0xfd, 0x2d, 0x16,
+			0x21, 0x74, 0x26, 0xc5, 0xb8, 0x44, 0xad, 0x9f, 0x97,
+			0xf9, 0x65, 0x36, 0xd8, 0x00, 0x59, 0x17, 0x49, 0xf9,
+			0xc7, 0xb3, 0x84, 0xb9, 0xe2, 0x95, 0xe0, 0xd1, 0x7f,
+			0x5f, 0xaa, 0xd7, 0xfd, 0x6a, 0x6a, 0x83, 0x14, 0x46,
+			0x1d, 0x12, 0x8d, 0x09, 0xc3, 0xa5, 0xca, 0x72, 0xa3,
+			0x25, 0x65, 0xb6, 0x40, 0x25, 0x04, 0x51, 0xab, 0x22,
+			0xeb, 0xd7, 0x69, 0xc9, 0x22, 0x9c, 0xa0, 0x19, 0x5c,
+			0x1a, 0xfd, 0x41, 0x8f, 0x98, 0xc5, 0x71, 0xb8, 0x6f,
+			0x76, 0xae, 0xfa, 0x9b, 0x03, 0xab, 0x43, 0x81, 0x3b,
+			0x66, 0xae, 0xf0, 0xd2, 0xb7, 0xee, 0x9a, 0xe3, 0xae,
+			0x45, 0xc1, 0x86, 0xb0, 0xce, 0x9e, 0x2b, 0xec, 0xb8,
+			0xcf, 0xca, 0x0e, 0x8c, 0x33, 0xfa, 0xa7, 0xef, 0xf7,
+			0xfc, 0xa1, 0x41, 0x49, 0xd3, 0x6d, 0xb5, 0x58, 0xe4,
+			0x0e, 0x24, 0xd2, 0x8a, 0x74, 0xc9, 0x56, 0x2e, 0x53,
+			0xc7, 0x7a, 0x38, 0x0f, 0x4b, 0xd9, 0xf9, 0x2f, 0xfa,
+			0x7d, 0xee, 0x14, 0x18, 0xce, 0x75, 0x42, 0x6c, 0x03,
+			0x34, 0xce, 0x80, 0xec, 0xf2, 0x05, 0xf0, 0xdf, 0xcd,
+			0xf8, 0xdb, 0x26, 0x7d, 0xb6, 0x3d, 0x28, 0x24, 0x7e,
+			0x7e, 0x39, 0x9f, 0xa6, 0xc6, 0xeb, 0x2a, 0xc8, 0x17,
+			0x94, 0xa9, 0x89, 0xf5, 0xdf, 0xcb, 0x77, 0xfd, 0xc9,
+			0x9e, 0x68, 0x98, 0x7d, 0x04, 0x50, 0x3c, 0x64, 0x1d,
+			0x66, 0xb0, 0x97, 0x06, 0xb6, 0x08, 0x5b, 0xe4, 0x17,
+			0x44, 0xd6, 0x94, 0x39, 0x6b, 0x03, 0x2c, 0xcb, 0x5a,
+			0x8d, 0x86, 0x08, 0x23, 0x4f, 0x95, 0xa8, 0x1a,
+		},
+	.msg_len = 64,
+	.msg =
+		(const u8[64]) {
+			0x1a, 0x84, 0x21, 0x0d, 0x35, 0x7a, 0x88, 0xc8,
+			0x6a, 0x11, 0xe3, 0x15, 0x24, 0xec, 0x0d, 0x2e,
+			0x76, 0xb9, 0xcf, 0x2b, 0x04, 0x25, 0x16, 0xae,
+			0x62, 0x42, 0xa0, 0x20, 0x68, 0x25, 0x3e, 0xb4,
+			0x75, 0xa7, 0x1d, 0x64, 0xc3, 0xd1, 0x08, 0x07,
+			0x67, 0xb6, 0xf7, 0x76, 0x76, 0xf6, 0xd6, 0x62,
+			0x66, 0x04, 0x89, 0x0c, 0x8f, 0x07, 0xac, 0xc8,
+			0x51, 0x77, 0xd9, 0x47, 0x5e, 0xb5, 0x22, 0x20,
+		},
+	.sig_len = MLDSA65_SIGNATURE_SIZE,
+	.sig =
+		(const u8[MLDSA65_SIGNATURE_SIZE]) {
+			0xda, 0xcf, 0x8d, 0x67, 0x59, 0x60, 0x6c, 0x39, 0x2d,
+			0x89, 0xb6, 0xa1, 0xf3, 0x8c, 0x70, 0xcf, 0x25, 0x86,
+			0x21, 0xa1, 0x9f, 0x20, 0x9e, 0xf5, 0xd2, 0xdd, 0xbd,
+			0x99, 0xfa, 0xe4, 0xab, 0x77, 0x31, 0x65, 0x18, 0xa1,
+			0xd1, 0x3f, 0x21, 0x70, 0x36, 0xe1, 0xf9, 0x5c, 0x28,
+			0xb6, 0x7d, 0x34, 0xae, 0x66, 0xc9, 0x1c, 0x8e, 0xc6,
+			0xf9, 0x45, 0x8c, 0xa9, 0xb2, 0xfb, 0x0f, 0x5b, 0xb8,
+			0xf9, 0xf5, 0xe2, 0x37, 0x79, 0x12, 0xda, 0xa7, 0x72,
+			0x9e, 0x0d, 0xf8, 0x88, 0x5b, 0x34, 0x49, 0x6c, 0xed,
+			0xa3, 0x7f, 0x86, 0xd3, 0xd9, 0x2f, 0x44, 0x08, 0x0d,
+			0xb7, 0xdb, 0x4a, 0xce, 0x02, 0x14, 0x02, 0xd6, 0x40,
+			0x75, 0xe3, 0xc0, 0x97, 0xfc, 0x6c, 0x6a, 0x88, 0x29,
+			0x0c, 0xe2, 0x3a, 0x2b, 0x28, 0x82, 0x8f, 0x27, 0x09,
+			0x69, 0x91, 0xc6, 0xc3, 0xb7, 0x07, 0x61, 0x86, 0x8d,
+			0x89, 0x8a, 0xd5, 0x00, 0x3b, 0x4b, 0xfc, 0x6f, 0xb3,
+			0x3f, 0x4c, 0x93, 0x31, 0xfc, 0x88, 0x53, 0x26, 0xea,
+			0xe5, 0x3a, 0xfc, 0xc1, 0x59, 0x16, 0xf0, 0xb7, 0xac,
+			0xde, 0x1e, 0xd8, 0x74, 0x85, 0x72, 0xd9, 0xbb, 0xbe,
+			0x76, 0x32, 0x25, 0x9d, 0x21, 0xbc, 0xfd, 0x8d, 0x32,
+			0xfe, 0xae, 0x24, 0xe5, 0x4a, 0xcc, 0x5d, 0x15, 0x23,
+			0xd3, 0x57, 0xe7, 0xa9, 0x2c, 0x31, 0xd7, 0xc5, 0x6b,
+			0x70, 0x6c, 0x22, 0x5a, 0x13, 0x1f, 0x76, 0x13, 0x78,
+			0x6f, 0xac, 0x42, 0x4c, 0x46, 0x81, 0xa2, 0x20, 0x91,
+			0x30, 0xed, 0xcb, 0x90, 0xfe, 0x3c, 0xa3, 0xc7, 0xb4,
+			0x1f, 0x21, 0x1d, 0x98, 0x74, 0x6a, 0x3e, 0xc8, 0xcc,
+			0xd2, 0x68, 0x87, 0x69, 0xa9, 0xdf, 0x50, 0xd5, 0x0a,
+			0x8e, 0x10, 0x54, 0xab, 0xea, 0x65, 0x2a, 0x52, 0xd7,
+			0x22, 0xae, 0x2f, 0x1e, 0xc3, 0x16, 0x58, 0x20, 0x18,
+			0x6d, 0x35, 0x46, 0x31, 0x43, 0x5d, 0x62, 0xfb, 0xb1,
+			0x47, 0x32, 0xfa, 0x14, 0xcc, 0x51, 0xa3, 0xcd, 0x99,
+			0x4f, 0x97, 0x0f, 0xca, 0x24, 0x93, 0x17, 0xea, 0xa3,
+			0xf3, 0x1f, 0xbe, 0xb5, 0xa3, 0xac, 0x80, 0xcc, 0x20,
+			0x3b, 0xa6, 0xd3, 0x32, 0x72, 0x4e, 0xd9, 0x25, 0xf9,
+			0xc2, 0x24, 0x15, 0xbd, 0x1e, 0x1e, 0x41, 0x8c, 0x18,
+			0x8c, 0x58, 0xe8, 0x75, 0x20, 0xff, 0xa3, 0xf4, 0xd4,
+			0xab, 0x75, 0x78, 0x4e, 0xbb, 0x7c, 0x94, 0x93, 0x28,
+			0x5b, 0x07, 0x3a, 0x3c, 0xc9, 0xf1, 0x55, 0x3e, 0x33,
+			0xed, 0xf8, 0x72, 0x55, 0xab, 0x5a, 0xea, 0xbe, 0x65,
+			0xfa, 0x81, 0x50, 0xc0, 0x9d, 0x2d, 0xfb, 0x04, 0x25,
+			0x7c, 0xb9, 0xee, 0xe2, 0xa3, 0x00, 0x44, 0xd3, 0x9d,
+			0xee, 0x4f, 0x80, 0x77, 0xfb, 0x26, 0x6b, 0x07, 0xd0,
+			0xff, 0x82, 0x39, 0x0e, 0x2b, 0x47, 0xa3, 0xe7, 0x3e,
+			0xc5, 0x4e, 0x15, 0x8a, 0x48, 0x28, 0xfb, 0xf7, 0xa4,
+			0x86, 0xfb, 0x77, 0x60, 0xcd, 0xc5, 0x68, 0x96, 0xd7,
+			0x4c, 0x3c, 0xf2, 0x51, 0x71, 0x79, 0x2e, 0x2e, 0x57,
+			0x10, 0xa7, 0xfc, 0xd1, 0xd4, 0x61, 0x71, 0x81, 0x85,
+			0x74, 0x09, 0x7d, 0x80, 0xd0, 0xc2, 0xe9, 0xff, 0xb7,
+			0x88, 0x53, 0x74, 0x1e, 0xb0, 0xca, 0x65, 0x48, 0x8e,
+			0xdb, 0x59, 0x3a, 0xcb, 0x80, 0xeb, 0xfd, 0xd2, 0xc9,
+			0x38, 0x43, 0xae, 0x76, 0xf2, 0xbb, 0x51, 0xb2, 0xcb,
+			0xe6, 0x85, 0x31, 0xb5, 0x62, 0xd4, 0x5e, 0x48, 0x08,
+			0xf1, 0x40, 0x5b, 0x16, 0x83, 0x5e, 0xa5, 0x9c, 0x6b,
+			0x91, 0x49, 0x44, 0xff, 0x3b, 0xa9, 0x2b, 0xf3, 0x06,
+			0x33, 0x9e, 0x6e, 0x3c, 0x66, 0x7e, 0x27, 0xa2, 0x59,
+			0x7b, 0xe3, 0xb6, 0xb4, 0x28, 0xeb, 0x93, 0x35, 0x87,
+			0xac, 0x0e, 0x0b, 0x7e, 0xbc, 0x35, 0x28, 0x72, 0x1f,
+			0x26, 0x59, 0xd0, 0x1f, 0x63, 0xe4, 0x86, 0x5d, 0x70,
+			0xf3, 0xa8, 0xa4, 0xb8, 0xcd, 0xb3, 0xf8, 0x8d, 0xaa,
+			0x41, 0xd2, 0xcc, 0x0b, 0x15, 0x66, 0x22, 0x83, 0x92,
+			0xe3, 0x0b, 0xf9, 0xea, 0xa0, 0x33, 0xa1, 0x4e, 0x92,
+			0xae, 0x81, 0x95, 0xa4, 0x58, 0x3f, 0xa9, 0x15, 0x52,
+			0xf9, 0xda, 0xb7, 0x10, 0x8d, 0xc6, 0xab, 0x77, 0xe9,
+			0xbe, 0xad, 0xc9, 0x3a, 0x6a, 0x8d, 0x92, 0x6c, 0x69,
+			0xff, 0x31, 0x49, 0x25, 0x04, 0xc8, 0x93, 0x6f, 0xc8,
+			0xe7, 0x60, 0x7a, 0x76, 0xb5, 0xc1, 0x07, 0xef, 0xa3,
+			0x39, 0xa6, 0xf2, 0x36, 0x04, 0xde, 0x3c, 0x4a, 0x4e,
+			0x96, 0xbd, 0x64, 0x26, 0x80, 0x01, 0x88, 0x47, 0xd2,
+			0xa4, 0x46, 0xcd, 0xe1, 0x30, 0x7f, 0xa3, 0x00, 0x11,
+			0x38, 0x55, 0xfa, 0xeb, 0x10, 0xeb, 0xa0, 0x65, 0x04,
+			0x09, 0xc8, 0xde, 0x9c, 0x73, 0xba, 0x0c, 0xbd, 0xd3,
+			0xa5, 0x84, 0x5e, 0xb9, 0x3b, 0xd4, 0x94, 0xbd, 0xa6,
+			0x53, 0xbe, 0x93, 0x69, 0x3e, 0xaa, 0x32, 0x31, 0x06,
+			0xc8, 0x1b, 0x4a, 0x48, 0xb5, 0x17, 0x85, 0xbf, 0x72,
+			0xec, 0xf5, 0x29, 0x8a, 0xd8, 0xeb, 0x99, 0x8b, 0x74,
+			0x84, 0x57, 0x8c, 0xe1, 0x85, 0x94, 0xa0, 0xbc, 0x7a,
+			0x14, 0xf0, 0xf4, 0x8b, 0x25, 0x37, 0x43, 0xa1, 0x34,
+			0x09, 0x71, 0xca, 0x5c, 0x9f, 0x08, 0x38, 0xd9, 0x9c,
+			0x0c, 0x0e, 0xcb, 0xe4, 0xad, 0x4b, 0x2a, 0x89, 0x67,
+			0xf8, 0x29, 0x6c, 0x69, 0x0e, 0x5d, 0xca, 0xfa, 0xa6,
+			0x6b, 0x0e, 0xb5, 0x94, 0x17, 0x71, 0xf0, 0xc9, 0xcd,
+			0x02, 0x1d, 0xa5, 0xd5, 0xc6, 0xa7, 0xbc, 0x5f, 0x6e,
+			0x67, 0x43, 0x68, 0xce, 0xac, 0x54, 0x81, 0x2a, 0x25,
+			0x22, 0x52, 0x35, 0xad, 0x7b, 0xd5, 0x06, 0x8c, 0x00,
+			0xfb, 0xca, 0xc4, 0x0a, 0x49, 0x1e, 0xc8, 0xeb, 0x77,
+			0xc1, 0x63, 0x23, 0x96, 0xbd, 0x35, 0xfa, 0x13, 0xae,
+			0xbf, 0x1d, 0x1e, 0x69, 0x8d, 0xb3, 0xe3, 0x07, 0xde,
+			0x4e, 0xd0, 0x12, 0xa9, 0xc3, 0x36, 0x30, 0x46, 0xef,
+			0x92, 0x76, 0x17, 0x8f, 0x10, 0xe7, 0xba, 0x99, 0x4b,
+			0xdf, 0xad, 0xb8, 0x11, 0x80, 0xdf, 0xe7, 0xfd, 0x80,
+			0x64, 0xf7, 0x2a, 0xac, 0x60, 0x2a, 0x54, 0x8f, 0x4f,
+			0xaf, 0xaf, 0x60, 0xf9, 0x67, 0x20, 0x80, 0x53, 0x5c,
+			0xb6, 0x81, 0xa6, 0x2a, 0x74, 0x2d, 0xc5, 0x74, 0x2a,
+			0x95, 0x26, 0x13, 0x17, 0x01, 0xdd, 0x31, 0xac, 0x5a,
+			0x05, 0xda, 0xde, 0xba, 0xf6, 0x37, 0x13, 0x8d, 0xe4,
+			0xa8, 0x93, 0x46, 0x9e, 0xa9, 0x82, 0x24, 0x7e, 0xc8,
+			0xda, 0x63, 0x89, 0xcd, 0x33, 0xc9, 0xf7, 0xf9, 0x71,
+			0x35, 0xe6, 0xa5, 0x5f, 0x6b, 0x3b, 0xbb, 0x0c, 0xe0,
+			0xa4, 0x0b, 0xe3, 0x29, 0xc0, 0xae, 0x8e, 0xce, 0x03,
+			0x09, 0x73, 0x0e, 0x1e, 0x9c, 0xe9, 0x59, 0xb6, 0x8b,
+			0x78, 0x67, 0x32, 0x8b, 0xf1, 0x93, 0xcc, 0x72, 0x1b,
+			0x6f, 0xa2, 0xf1, 0x04, 0x9c, 0xfa, 0x98, 0x02, 0xca,
+			0xdf, 0x35, 0x3c, 0x38, 0xac, 0xa8, 0xdb, 0x90, 0xae,
+			0xaa, 0xf9, 0x70, 0xfb, 0xed, 0xbd, 0xa6, 0x25, 0x14,
+			0x58, 0x09, 0x8a, 0x36, 0xaf, 0x41, 0x09, 0x19, 0xcb,
+			0xd3, 0x25, 0x5d, 0x0e, 0xe6, 0x20, 0x14, 0x71, 0x24,
+			0x79, 0x19, 0x55, 0xaf, 0x51, 0x5b, 0xa4, 0xc0, 0x93,
+			0x9e, 0xdd, 0x88, 0x31, 0x13, 0x96, 0xbf, 0xca, 0x0a,
+			0xd7, 0xbc, 0xc4, 0x00, 0xa1, 0x10, 0x2d, 0x92, 0x79,
+			0xf9, 0x14, 0xdb, 0xd2, 0xba, 0x74, 0xfa, 0xa8, 0xe5,
+			0x40, 0x14, 0xc2, 0x56, 0x3c, 0x7f, 0x50, 0x07, 0x60,
+			0x86, 0x93, 0x51, 0x2e, 0xf9, 0x70, 0x61, 0x70, 0x0e,
+			0xa4, 0x87, 0x75, 0xcc, 0x6c, 0x72, 0xb7, 0x68, 0x23,
+			0xb7, 0x3d, 0x76, 0xaf, 0x96, 0x9b, 0x4a, 0xe5, 0x12,
+			0x28, 0x4a, 0x8f, 0x79, 0x34, 0xff, 0xec, 0x92, 0xeb,
+			0x6b, 0xaf, 0xc9, 0xbd, 0xc1, 0x77, 0x07, 0xd0, 0xfa,
+			0x55, 0x57, 0x10, 0x0c, 0xad, 0x29, 0x2a, 0x79, 0xd6,
+			0x09, 0x9e, 0x7d, 0x18, 0xd4, 0xd6, 0xdd, 0x72, 0x1a,
+			0x8f, 0x24, 0x11, 0x70, 0xd2, 0x52, 0x36, 0x0f, 0x38,
+			0x79, 0x38, 0x4a, 0x02, 0x4f, 0x73, 0x2a, 0xaa, 0x6a,
+			0xb5, 0x0c, 0x72, 0x32, 0x85, 0x21, 0x76, 0x1a, 0x8a,
+			0x7d, 0x51, 0x0e, 0xf1, 0xf9, 0x19, 0xfa, 0x6b, 0x9b,
+			0x22, 0x71, 0x8c, 0x13, 0xcc, 0xba, 0x7d, 0xee, 0xd8,
+			0x34, 0xf6, 0x85, 0x60, 0xe1, 0xe4, 0x59, 0x6e, 0x32,
+			0x60, 0xd9, 0xfa, 0xb7, 0x56, 0x54, 0x25, 0xd1, 0x73,
+			0x6a, 0xf2, 0xa0, 0xc7, 0xa0, 0x67, 0x10, 0x89, 0x9c,
+			0x27, 0x5f, 0x7f, 0x2e, 0x5a, 0x29, 0x70, 0x7a, 0x7b,
+			0xaf, 0x21, 0xd0, 0xf4, 0x06, 0xb9, 0x2d, 0xf1, 0xb8,
+			0x32, 0xed, 0xc5, 0xc9, 0xac, 0x2f, 0x54, 0x0a, 0xf9,
+			0x08, 0x39, 0x39, 0x7d, 0x1d, 0xaf, 0xb4, 0x5f, 0x4d,
+			0x75, 0xc3, 0xe8, 0x52, 0x3a, 0x47, 0x72, 0x2c, 0xa9,
+			0x2d, 0xcb, 0x74, 0x06, 0xfe, 0x69, 0xd3, 0xf3, 0x1a,
+			0xb2, 0xd3, 0x01, 0xed, 0x6c, 0xc1, 0xca, 0x4f, 0xaf,
+			0x11, 0x9b, 0xa2, 0x27, 0x2a, 0x59, 0x56, 0x58, 0xdf,
+			0x79, 0x8b, 0xc9, 0x87, 0xe9, 0x58, 0x81, 0x48, 0xc6,
+			0xb6, 0x7d, 0x60, 0x54, 0x87, 0x9c, 0x61, 0xbb, 0x4b,
+			0xbb, 0x61, 0xac, 0x0a, 0x5a, 0x66, 0x7e, 0x70, 0x8b,
+			0xfd, 0x92, 0x76, 0x4a, 0xa9, 0xa5, 0xc3, 0xf4, 0xf2,
+			0x93, 0x48, 0xc4, 0xf3, 0x91, 0x2b, 0x60, 0x04, 0x0e,
+			0xb0, 0x6b, 0x60, 0x5e, 0xf0, 0xf1, 0x54, 0x41, 0x56,
+			0xdc, 0x25, 0x57, 0xc3, 0xb6, 0x0b, 0x5e, 0x15, 0xb5,
+			0x2a, 0x36, 0x4f, 0xe7, 0x1d, 0x70, 0xa8, 0xa7, 0xec,
+			0xd6, 0x74, 0xba, 0xa4, 0x79, 0x83, 0x7c, 0x9e, 0x1a,
+			0x5d, 0x32, 0xc8, 0xcb, 0x41, 0xca, 0x04, 0xec, 0x0b,
+			0x18, 0x54, 0xe1, 0x67, 0xbf, 0xa8, 0x7a, 0xc3, 0x0f,
+			0x27, 0x2a, 0xaf, 0x2a, 0x41, 0x19, 0x1f, 0xe8, 0xa2,
+			0xe8, 0xfa, 0xfc, 0x88, 0x41, 0x46, 0xc3, 0x1c, 0x44,
+			0xe5, 0xee, 0x47, 0xec, 0xfe, 0xbf, 0xb8, 0x29, 0x2e,
+			0xae, 0x47, 0x0a, 0x42, 0x69, 0x8a, 0x9a, 0x94, 0x97,
+			0x9e, 0xf5, 0xb6, 0x37, 0x1c, 0x10, 0xc2, 0x99, 0xa8,
+			0xe9, 0x9e, 0x0e, 0x6e, 0xb5, 0xbe, 0xba, 0x1f, 0x77,
+			0xa6, 0x35, 0x02, 0x1e, 0x8c, 0xe6, 0x02, 0x53, 0xe2,
+			0x9a, 0xdd, 0x09, 0x6e, 0x9b, 0x7a, 0x36, 0x4f, 0x38,
+			0x8d, 0x4c, 0xa4, 0xb4, 0xff, 0x90, 0x76, 0x0d, 0x11,
+			0x7d, 0xe1, 0xe9, 0x7f, 0x2a, 0x4a, 0x80, 0xe0, 0xd8,
+			0x3c, 0x23, 0xd2, 0xa5, 0xe5, 0x39, 0x77, 0xbf, 0x3d,
+			0x71, 0x0d, 0x45, 0xbb, 0x39, 0x66, 0x1a, 0x4d, 0x59,
+			0xb7, 0xd0, 0x0a, 0xee, 0x87, 0xee, 0x1f, 0xcf, 0x6f,
+			0xc2, 0x50, 0xb1, 0xa5, 0x4c, 0xee, 0x40, 0x69, 0xd7,
+			0x36, 0x38, 0x14, 0xcd, 0x6a, 0x9a, 0x90, 0x40, 0xad,
+			0x76, 0xf1, 0xa6, 0xd4, 0x3c, 0x75, 0x10, 0xba, 0xcb,
+			0xab, 0x22, 0x28, 0x5f, 0x0c, 0xe0, 0xee, 0xf4, 0xfd,
+			0x61, 0x52, 0x0a, 0x59, 0xfe, 0x61, 0xc5, 0x40, 0xf9,
+			0x91, 0x8e, 0x36, 0x29, 0x63, 0x6c, 0x6e, 0x45, 0xa5,
+			0x42, 0xe3, 0x36, 0x90, 0xe7, 0x90, 0x9f, 0x58, 0xbb,
+			0xf9, 0x1b, 0xee, 0x2c, 0xbb, 0x3a, 0xfd, 0x3d, 0xbe,
+			0x3d, 0x45, 0xf0, 0xc2, 0x18, 0xaa, 0x46, 0x10, 0x23,
+			0xe9, 0x63, 0xba, 0x7f, 0xc2, 0xe1, 0xf4, 0x05, 0xdd,
+			0x4a, 0x7c, 0xa8, 0xab, 0xa9, 0xbd, 0x6f, 0xdf, 0x48,
+			0x59, 0x11, 0xd4, 0xba, 0x75, 0xb6, 0x22, 0xd4, 0xd7,
+			0x35, 0x6f, 0x27, 0x70, 0xc7, 0x3d, 0x90, 0x06, 0x39,
+			0x2a, 0x16, 0xd0, 0x8b, 0xd7, 0xfb, 0x5e, 0x85, 0x2e,
+			0xb0, 0xd8, 0xc7, 0xdb, 0xe5, 0x24, 0x3a, 0x6e, 0xc4,
+			0x5e, 0xd4, 0x22, 0x25, 0x14, 0xee, 0xa5, 0x30, 0x8b,
+			0xd6, 0x27, 0x61, 0x33, 0x13, 0x46, 0x0b, 0x26, 0x45,
+			0xa6, 0xb4, 0xfa, 0x8d, 0xa3, 0xf2, 0x27, 0xd2, 0xc5,
+			0x04, 0xaa, 0x96, 0xa4, 0x55, 0xfa, 0x40, 0xf1, 0xfc,
+			0x66, 0x33, 0x9e, 0x4b, 0x39, 0x75, 0xae, 0x7f, 0x52,
+			0x87, 0x7b, 0x8a, 0xf9, 0x7d, 0x5f, 0x8a, 0x7e, 0xf7,
+			0xfe, 0xc4, 0x7f, 0xf4, 0xf6, 0x9a, 0x86, 0x78, 0x21,
+			0x02, 0x94, 0x9e, 0x50, 0x2d, 0xdc, 0xd6, 0xa5, 0x53,
+			0xf1, 0xef, 0x06, 0xe8, 0xb5, 0x46, 0x81, 0xcc, 0x91,
+			0x4f, 0x37, 0xee, 0x27, 0xcb, 0x91, 0xad, 0xff, 0x1d,
+			0xd1, 0x00, 0xa8, 0x96, 0x22, 0xaa, 0x63, 0x23, 0x2a,
+			0x7a, 0x75, 0x6f, 0xe9, 0x2d, 0x26, 0xde, 0x11, 0x97,
+			0x4b, 0x17, 0x3f, 0xde, 0x51, 0x1a, 0x22, 0xed, 0x38,
+			0x6f, 0x3e, 0x7a, 0xd0, 0xd6, 0x60, 0x06, 0x7e, 0x3f,
+			0xa4, 0x29, 0xfa, 0x18, 0x91, 0xda, 0x73, 0x38, 0xe3,
+			0xe3, 0xb5, 0xc0, 0x5b, 0x4e, 0xe8, 0x94, 0xea, 0x45,
+			0x6e, 0x5b, 0x50, 0xaa, 0x38, 0xb6, 0x6f, 0xdb, 0x90,
+			0x1b, 0x3b, 0x82, 0xbb, 0x0d, 0x38, 0xe3, 0xca, 0xd9,
+			0xf1, 0x2e, 0x27, 0x4c, 0x2c, 0x5a, 0x42, 0xdf, 0x44,
+			0xc8, 0x07, 0xe4, 0x95, 0xb5, 0xec, 0x91, 0x34, 0x1c,
+			0x9a, 0x0c, 0x50, 0x1a, 0xce, 0x67, 0xe4, 0x4b, 0x87,
+			0x61, 0x43, 0x95, 0x95, 0xb8, 0x8a, 0xf4, 0xc9, 0x92,
+			0x33, 0x33, 0xe3, 0xfe, 0x98, 0x2a, 0xae, 0x8e, 0xf2,
+			0x6b, 0x13, 0x7c, 0xe4, 0x44, 0x40, 0x66, 0xea, 0x0c,
+			0xe4, 0xdb, 0x16, 0x65, 0xa8, 0x8b, 0x37, 0x08, 0xec,
+			0x1e, 0xfc, 0xa6, 0xd0, 0x9b, 0x9e, 0x0a, 0xd2, 0xe3,
+			0xcf, 0x5d, 0xb2, 0xaf, 0x8e, 0x05, 0x7d, 0x8d, 0x84,
+			0xbc, 0x9f, 0xb1, 0xe6, 0x6a, 0x2e, 0x4b, 0x6d, 0x64,
+			0x91, 0x17, 0x9d, 0xb5, 0x35, 0x15, 0x02, 0xe9, 0x1b,
+			0x85, 0xc1, 0x89, 0xc2, 0x5a, 0x32, 0x3a, 0x80, 0x78,
+			0x5e, 0xcc, 0x50, 0x26, 0xf5, 0x11, 0x01, 0x79, 0xf3,
+			0xaf, 0xb6, 0x40, 0x00, 0x73, 0x8f, 0xeb, 0x5a, 0xd1,
+			0x26, 0x00, 0xe2, 0xa3, 0xcd, 0xfd, 0xaa, 0x15, 0x5b,
+			0x98, 0x2a, 0x76, 0x41, 0x07, 0xc2, 0xde, 0xb6, 0x71,
+			0xe7, 0xc3, 0xe9, 0x92, 0xb3, 0xd8, 0xfe, 0xaf, 0x12,
+			0x61, 0x86, 0x5b, 0x6e, 0x74, 0x45, 0x7b, 0x9b, 0x6f,
+			0x1a, 0x13, 0x84, 0xf6, 0x31, 0x5f, 0x5b, 0x6c, 0xde,
+			0x47, 0xb8, 0x73, 0x32, 0xc7, 0x94, 0x92, 0xa5, 0xc3,
+			0x65, 0xdf, 0x96, 0x6c, 0xfd, 0xb7, 0x80, 0xfb, 0x47,
+			0xba, 0x6e, 0x43, 0xb3, 0x7e, 0x86, 0xc9, 0x97, 0x45,
+			0xde, 0x3f, 0x3a, 0xf6, 0xb0, 0x9e, 0x9a, 0xcb, 0xfd,
+			0xf2, 0x5c, 0xba, 0x6e, 0x3f, 0xed, 0xfa, 0x74, 0x84,
+			0xe2, 0xb1, 0xae, 0x66, 0x57, 0x0b, 0x96, 0x6c, 0x77,
+			0xe4, 0x8a, 0x67, 0x97, 0xc7, 0xe0, 0x44, 0xb2, 0x83,
+			0x2d, 0x3c, 0x2e, 0x01, 0x19, 0x2e, 0x4c, 0x74, 0xe1,
+			0x35, 0x73, 0xeb, 0x85, 0x63, 0x8c, 0x3a, 0xb8, 0xbc,
+			0x25, 0x6a, 0x8d, 0xaf, 0xd2, 0xfb, 0xef, 0xd3, 0x12,
+			0x93, 0x0b, 0x39, 0xfa, 0x66, 0xbe, 0x3b, 0xfd, 0x6c,
+			0x0b, 0xbb, 0xb2, 0x5a, 0x78, 0xa1, 0xcf, 0x8c, 0x7d,
+			0x60, 0x55, 0xeb, 0x33, 0x4e, 0x8e, 0xf9, 0x19, 0x4d,
+			0x42, 0xd4, 0xf8, 0xd8, 0xba, 0xad, 0x0a, 0x6e, 0x62,
+			0xd4, 0xe1, 0x6a, 0xcc, 0xea, 0x09, 0x91, 0x8e, 0x62,
+			0xc9, 0x1e, 0x9e, 0x48, 0xaa, 0xde, 0xf7, 0xa2, 0x5a,
+			0xcb, 0x83, 0x20, 0xe8, 0xf5, 0xd1, 0xfe, 0x9d, 0x18,
+			0x2f, 0xd6, 0xf8, 0x97, 0x17, 0xce, 0xc2, 0x05, 0x08,
+			0xef, 0x61, 0x70, 0x9d, 0x95, 0x79, 0x59, 0x4c, 0x06,
+			0x24, 0x3d, 0x24, 0x69, 0xff, 0x46, 0xda, 0xbc, 0x71,
+			0x7a, 0x74, 0x93, 0x58, 0xf5, 0xc8, 0x91, 0xfb, 0x66,
+			0xed, 0x78, 0x8f, 0xf8, 0x28, 0xa8, 0x1d, 0xa5, 0x3a,
+			0x13, 0x76, 0xc2, 0xcc, 0xba, 0xb9, 0x56, 0x29, 0x74,
+			0xd6, 0x14, 0x75, 0x58, 0xe6, 0x2e, 0x79, 0x6e, 0x9d,
+			0x41, 0x94, 0x8a, 0xcf, 0xf1, 0xb1, 0xe0, 0x36, 0xe5,
+			0x89, 0x9a, 0x95, 0xa1, 0x11, 0xd1, 0xbe, 0x45, 0xe4,
+			0xb3, 0xb0, 0x62, 0x32, 0x1d, 0xba, 0xe0, 0xde, 0x57,
+			0x81, 0x0e, 0x01, 0x9b, 0x52, 0x3d, 0xd5, 0xde, 0x3b,
+			0x3a, 0xdd, 0x8f, 0xe3, 0x2e, 0xce, 0x1e, 0x89, 0x4d,
+			0x81, 0xf0, 0xf6, 0x20, 0x63, 0x7a, 0x4c, 0xbb, 0x66,
+			0xe0, 0xbe, 0x2b, 0xee, 0xd0, 0x3b, 0x60, 0x1e, 0x65,
+			0xd1, 0x2c, 0x7c, 0x5c, 0x6c, 0x16, 0x5b, 0x90, 0xc8,
+			0x05, 0x10, 0xf2, 0xde, 0x33, 0x90, 0x35, 0x69, 0x24,
+			0x3f, 0xc1, 0x8f, 0x1e, 0x4a, 0x60, 0xf1, 0x03, 0x65,
+			0x46, 0x40, 0x76, 0xe9, 0x83, 0x97, 0xda, 0x0b, 0xb8,
+			0x22, 0xfa, 0x55, 0x99, 0xfd, 0x18, 0x24, 0xd2, 0x66,
+			0xb0, 0x7b, 0x70, 0x56, 0x93, 0xad, 0x09, 0x95, 0x8e,
+			0x1f, 0x2f, 0xe8, 0x12, 0x55, 0xd4, 0x1f, 0xde, 0x09,
+			0x85, 0x05, 0xd1, 0xd5, 0x10, 0x2c, 0x8c, 0x6b, 0x53,
+			0x28, 0xce, 0x06, 0xc5, 0x52, 0x0f, 0xfa, 0x09, 0x09,
+			0x23, 0x1b, 0xe3, 0xbf, 0xb1, 0x89, 0x72, 0x26, 0x0d,
+			0xa6, 0xbb, 0x7d, 0x9e, 0xdc, 0xf8, 0xf5, 0x0b, 0x8c,
+			0xe0, 0xbc, 0x97, 0x3b, 0x72, 0xdd, 0xf5, 0x9d, 0xc5,
+			0xb6, 0x37, 0x2c, 0x76, 0x5b, 0x58, 0x67, 0xdb, 0xed,
+			0x3b, 0x6e, 0xe5, 0xe5, 0x6d, 0x6f, 0x0d, 0x7e, 0xff,
+			0xa9, 0x57, 0x4a, 0x84, 0x85, 0x82, 0xac, 0x00, 0x50,
+			0xa3, 0x4f, 0x87, 0xfe, 0x2a, 0x40, 0x52, 0x54, 0x81,
+			0x69, 0x42, 0x0b, 0x0c, 0xd7, 0x18, 0x98, 0x01, 0x8c,
+			0x5a, 0xa2, 0xf4, 0xe8, 0x61, 0xd1, 0x38, 0xfd, 0x0f,
+			0x63, 0x75, 0xd3, 0x4b, 0x1d, 0xdc, 0xdf, 0xb2, 0xeb,
+			0x94, 0x97, 0x5c, 0x2a, 0xb4, 0x12, 0x5c, 0x49, 0x2b,
+			0xfc, 0xd0, 0x8d, 0xfb, 0xe7, 0xb3, 0xcb, 0x0f, 0x3c,
+			0x2e, 0x04, 0x36, 0xa8, 0x03, 0xc9, 0xd7, 0x11, 0x2d,
+			0x2a, 0x93, 0xff, 0xda, 0x26, 0xb0, 0x54, 0x7e, 0xaf,
+			0x30, 0x7d, 0xce, 0x46, 0x8a, 0x3d, 0x7c, 0xa4, 0x7a,
+			0x2c, 0xfa, 0xba, 0xa1, 0xc9, 0x41, 0xd3, 0xb8, 0x84,
+			0x03, 0x78, 0xdd, 0xe9, 0x57, 0x19, 0x62, 0x62, 0xff,
+			0x5b, 0x3b, 0x48, 0x62, 0x0e, 0xee, 0x19, 0xb0, 0x32,
+			0x6e, 0x6a, 0x07, 0xd8, 0x4e, 0x25, 0x76, 0xa7, 0xe3,
+			0x98, 0xa1, 0x6f, 0xb6, 0x99, 0x32, 0x67, 0x7d, 0x46,
+			0x42, 0x4a, 0x82, 0xd1, 0x29, 0x1b, 0x87, 0xeb, 0x4b,
+			0x9e, 0xdf, 0x69, 0x75, 0xbd, 0x4f, 0xd3, 0xde, 0xc9,
+			0x83, 0xe6, 0xd6, 0xea, 0x03, 0x81, 0x12, 0xf3, 0x5d,
+			0x99, 0xf1, 0xb1, 0xd9, 0x3e, 0xbe, 0xf3, 0xa8, 0xdc,
+			0xb6, 0xf8, 0x4b, 0x9e, 0x26, 0x3f, 0xf0, 0x7c, 0xb3,
+			0xf4, 0xca, 0x00, 0x6c, 0x6c, 0xe5, 0x43, 0xa1, 0xfd,
+			0x3a, 0xf8, 0x8e, 0xe3, 0x9f, 0x88, 0xc5, 0x44, 0xfd,
+			0x24, 0x69, 0x76, 0xd5, 0xcb, 0xdc, 0x9d, 0x12, 0xf3,
+			0x13, 0x7e, 0xe7, 0xc3, 0xa8, 0x6a, 0xb2, 0xe0, 0xb3,
+			0x1d, 0xab, 0x3b, 0xc9, 0x77, 0x3d, 0x0f, 0xc3, 0xbe,
+			0x4b, 0x8b, 0x28, 0xbd, 0x7c, 0xe6, 0xb2, 0x06, 0x1f,
+			0xf9, 0x8f, 0x16, 0x62, 0xbf, 0xc7, 0x55, 0x73, 0xd4,
+			0xf1, 0x5a, 0x95, 0x80, 0xa3, 0x4e, 0xaa, 0x60, 0x17,
+			0x3c, 0xc9, 0x5e, 0xd4, 0x0c, 0x56, 0x7a, 0x77, 0x8e,
+			0x7f, 0x67, 0x08, 0x2f, 0xd9, 0x21, 0x19, 0xfd, 0x86,
+			0x8c, 0x23, 0x8d, 0xf6, 0x92, 0x1f, 0x36, 0x2c, 0x7c,
+			0x83, 0xbd, 0x2f, 0x6c, 0x63, 0x7c, 0xb7, 0x93, 0x74,
+			0x1b, 0xc2, 0x95, 0x34, 0x26, 0x1e, 0x07, 0x87, 0x3a,
+			0xb6, 0xe2, 0x39, 0x71, 0x9b, 0x20, 0xcd, 0x63, 0xf0,
+			0xbf, 0x48, 0xb5, 0x0e, 0x49, 0x86, 0x50, 0x80, 0xbd,
+			0xd6, 0x0e, 0xab, 0xd5, 0x69, 0x1b, 0xa4, 0xb3, 0x63,
+			0x3c, 0x8f, 0xcb, 0x42, 0xdb, 0xd7, 0x1a, 0xf4, 0xdf,
+			0x9e, 0x25, 0xfc, 0xd4, 0x00, 0xcb, 0xec, 0x57, 0x69,
+			0x30, 0x15, 0x4d, 0x7a, 0x69, 0x28, 0x2f, 0x2b, 0x34,
+			0x26, 0xd1, 0xe7, 0x01, 0x42, 0x5e, 0x02, 0xe2, 0x75,
+			0xe8, 0x52, 0x8a, 0xb4, 0x71, 0xfa, 0xc3, 0x3d, 0xe6,
+			0xac, 0xeb, 0xf3, 0x93, 0xe0, 0x37, 0xcd, 0x66, 0x92,
+			0x66, 0x2c, 0xfe, 0x4b, 0xd6, 0x3c, 0xf1, 0x57, 0xe5,
+			0xcf, 0xf5, 0xd0, 0xdb, 0x0e, 0x1f, 0x82, 0x65, 0x3b,
+			0xab, 0x69, 0x42, 0x53, 0x7d, 0xa4, 0x7c, 0xb7, 0x86,
+			0xeb, 0x23, 0x45, 0xa8, 0x4a, 0x73, 0xfc, 0x38, 0xc6,
+			0xe5, 0x2c, 0xab, 0x80, 0xfb, 0x23, 0xb2, 0x0c, 0x53,
+			0x28, 0x21, 0x37, 0x54, 0x9c, 0x72, 0x51, 0x0f, 0x44,
+			0x50, 0xd3, 0xe1, 0xd5, 0xb2, 0x27, 0x83, 0xb6, 0xe9,
+			0x4d, 0x64, 0x5c, 0x17, 0x0f, 0xe0, 0x13, 0xe4, 0x26,
+			0x6b, 0xd0, 0xd8, 0x25, 0xe3, 0x69, 0x6a, 0x95, 0x3f,
+			0x4a, 0x4e, 0xa0, 0x58, 0xbc, 0x28, 0x47, 0x8b, 0x68,
+			0xe4, 0x41, 0x90, 0x46, 0x1b, 0x84, 0xa0, 0x7b, 0x46,
+			0x46, 0x03, 0xee, 0x21, 0x0d, 0x34, 0xed, 0xff, 0x15,
+			0x57, 0x06, 0xdf, 0x71, 0x09, 0xb2, 0x66, 0x0d, 0x6e,
+			0xcc, 0xa5, 0x0c, 0xaf, 0x3f, 0x24, 0x8f, 0xd1, 0xc8,
+			0x44, 0x86, 0xaf, 0xbf, 0xeb, 0x2f, 0xb9, 0xee, 0xa7,
+			0xcf, 0xe4, 0xe8, 0xec, 0x47, 0x09, 0xd8, 0x95, 0x9e,
+			0x3c, 0xda, 0x92, 0x41, 0x61, 0xf5, 0xc3, 0xec, 0x00,
+			0xe4, 0xa3, 0x0d, 0x4a, 0xb3, 0xf6, 0x82, 0x05, 0x38,
+			0x70, 0x6a, 0xd1, 0x28, 0x2c, 0xb3, 0xc6, 0xbb, 0x38,
+			0xb3, 0x06, 0x7f, 0xd6, 0x4c, 0xe7, 0xfb, 0xef, 0x0d,
+			0x52, 0x66, 0xbe, 0xd8, 0xa6, 0x6f, 0xe8, 0xd9, 0x42,
+			0x4f, 0xad, 0xe8, 0xe8, 0x6c, 0xf9, 0xe9, 0x42, 0xd9,
+			0x66, 0x6e, 0xec, 0xfe, 0xf5, 0x91, 0xbf, 0x0a, 0x98,
+			0xd8, 0x7b, 0x23, 0x12, 0xa6, 0x04, 0xa8, 0xb3, 0x61,
+			0x13, 0x65, 0xc0, 0xe2, 0x82, 0xb9, 0xb2, 0x38, 0x07,
+			0x06, 0xca, 0x64, 0x6c, 0x23, 0x93, 0x60, 0x1d, 0x4d,
+			0x38, 0x5e, 0x8e, 0x90, 0x16, 0x4a, 0xfd, 0xb3, 0xcd,
+			0x84, 0x9c, 0xa5, 0xfa, 0x73, 0x2d, 0xcb, 0x87, 0x31,
+			0x3d, 0xf8, 0xfc, 0xeb, 0xa7, 0x56, 0x2f, 0x5b, 0x95,
+			0x9a, 0xc6, 0x82, 0x29, 0x86, 0x47, 0xe2, 0xc2, 0x84,
+			0x01, 0xaf, 0xc8, 0x0b, 0x2d, 0xfb, 0x34, 0xba, 0x5d,
+			0x9d, 0xd1, 0x85, 0xd5, 0x1e, 0x63, 0xcb, 0x3c, 0xa8,
+			0xfa, 0x79, 0xef, 0x12, 0xa6, 0xb5, 0xdb, 0xc5, 0x1d,
+			0x6a, 0xa7, 0x54, 0x58, 0x0c, 0xbe, 0x61, 0xe5, 0x96,
+			0x7f, 0x4a, 0x3b, 0x59, 0x32, 0x2d, 0x06, 0x44, 0x83,
+			0x5c, 0xad, 0xe9, 0xfe, 0x7c, 0xd7, 0x5b, 0x34, 0xa1,
+			0xa3, 0xad, 0x9a, 0xbf, 0xd5, 0x30, 0xf0, 0x22, 0xfc,
+			0x94, 0x7f, 0xd4, 0xa4, 0xca, 0x88, 0x31, 0xe7, 0xf2,
+			0x89, 0x2d, 0xda, 0xe6, 0x91, 0xa6, 0x27, 0x22, 0x74,
+			0x9f, 0xc6, 0x72, 0x4f, 0xf6, 0xa9, 0xfe, 0x7a, 0xf0,
+			0xa8, 0x6b, 0x6c, 0x9f, 0xe9, 0x2a, 0x9b, 0x23, 0x9e,
+			0xb8, 0x2b, 0x29, 0x65, 0xa7, 0x5d, 0xbd, 0x10, 0xe4,
+			0x56, 0x02, 0x94, 0xdd, 0xd1, 0xab, 0x9b, 0x82, 0x2d,
+			0x8d, 0xf6, 0xd3, 0x65, 0x63, 0x4a, 0xc4, 0x86, 0x61,
+			0x37, 0x9f, 0xdb, 0x4b, 0x34, 0x20, 0x0a, 0xca, 0x45,
+			0x6c, 0x06, 0xc4, 0x9c, 0x74, 0x4d, 0x83, 0x6a, 0x8d,
+			0xad, 0xc6, 0x61, 0x3a, 0x8d, 0xde, 0x6c, 0xf9, 0x8e,
+			0x33, 0xa2, 0xee, 0x99, 0xc7, 0xe4, 0x52, 0xb2, 0x44,
+			0x6f, 0x2f, 0x0f, 0x41, 0xa9, 0x1a, 0xd3, 0x96, 0x42,
+			0xc6, 0x49, 0x12, 0x6a, 0xf0, 0x29, 0xa9, 0x0c, 0x9c,
+			0x50, 0x5d, 0x1d, 0xd1, 0x42, 0x7e, 0x6f, 0x36, 0x48,
+			0x0f, 0x58, 0x14, 0x94, 0xc0, 0x10, 0x1e, 0xe0, 0xb2,
+			0xdd, 0xba, 0x57, 0x91, 0x4d, 0xd5, 0xdc, 0xa6, 0x4c,
+			0x68, 0x00, 0x6c, 0xb3, 0x5d, 0x32, 0x13, 0xbe, 0xa8,
+			0xc3, 0xfb, 0xd4, 0x19, 0x40, 0xf5, 0x6f, 0x63, 0xa1,
+			0x07, 0xbf, 0xa2, 0x8b, 0xfc, 0xfe, 0xf8, 0xa1, 0x33,
+			0x70, 0x07, 0x6d, 0xc5, 0x72, 0xa0, 0x39, 0xd6, 0xd7,
+			0x76, 0x6c, 0xfa, 0x1f, 0x04, 0xd6, 0x23, 0xbf, 0x66,
+			0x78, 0x92, 0x00, 0x11, 0x8a, 0x75, 0x67, 0x44, 0xa6,
+			0x7c, 0xd0, 0x14, 0xe6, 0xd0, 0x31, 0x6d, 0xdb, 0xc5,
+			0xb1, 0xa7, 0x99, 0xc3, 0xaf, 0x18, 0x7a, 0x26, 0x46,
+			0xad, 0x6d, 0x0c, 0xb6, 0xb5, 0xad, 0xc1, 0xcf, 0x60,
+			0x99, 0xf5, 0x9f, 0x88, 0xaf, 0x0e, 0x37, 0x15, 0xf9,
+			0x2b, 0x1a, 0x5f, 0xfb, 0xc9, 0xf8, 0xd4, 0xf0, 0x97,
+			0xd2, 0x91, 0xf4, 0x94, 0xa2, 0xd3, 0x3b, 0x8b, 0x0c,
+			0x22, 0xa0, 0xac, 0xb3, 0xb5, 0xdf, 0xf2, 0x27, 0x38,
+			0x47, 0x53, 0x5b, 0x6e, 0x8f, 0x98, 0x9e, 0xad, 0xb6,
+			0xf5, 0x0e, 0x17, 0x20, 0x35, 0x54, 0x6b, 0x73, 0xa6,
+			0x64, 0x65, 0xac, 0xb8, 0xc1, 0xd3, 0xf7, 0x07, 0x82,
+			0x93, 0x9d, 0xcb, 0xcc, 0xe9, 0x0c, 0x51, 0x52, 0x85,
+			0x8b, 0x95, 0xa6, 0xb1, 0xce, 0xdc, 0xfa, 0x00, 0x00,
+			0x08, 0x14, 0x1c, 0x23, 0x2a, 0x35,
+		},
+};
+
+static const struct mldsa_testvector mldsa87_testvector = {
+	.alg = MLDSA87,
+	.pk_len = MLDSA87_PUBLIC_KEY_SIZE,
+	.pk =
+		(const u8[MLDSA87_PUBLIC_KEY_SIZE]) {
+			0xd4, 0x9d, 0xdc, 0x3d, 0xa4, 0xa5, 0x87, 0xa5, 0x54,
+			0x61, 0xf3, 0xf4, 0xe0, 0x11, 0xc9, 0x1c, 0x78, 0x0a,
+			0xf1, 0x8a, 0xa8, 0xb2, 0xff, 0xb2, 0x9a, 0x2c, 0xe0,
+			0x86, 0x5c, 0xaa, 0x86, 0xe0, 0xd9, 0x42, 0x54, 0x18,
+			0x3e, 0x4c, 0x96, 0x1a, 0xb4, 0xc7, 0x18, 0xcf, 0x7d,
+			0xca, 0xe2, 0x74, 0x6c, 0x81, 0x3e, 0xcb, 0xf8, 0x7b,
+			0xc4, 0x90, 0x50, 0xd5, 0xe8, 0xd3, 0xbc, 0x8b, 0xa8,
+			0x3e, 0xb0, 0x96, 0x65, 0xd7, 0xbb, 0xa9, 0xab, 0x9c,
+			0x82, 0x5e, 0x6e, 0x8d, 0xf8, 0xc3, 0x6d, 0xe9, 0xbf,
+			0xbd, 0x30, 0xc9, 0xca, 0x47, 0x85, 0xae, 0x6f, 0x5d,
+			0x09, 0x4d, 0xd7, 0xdd, 0x05, 0x51, 0xe0, 0x9e, 0x94,
+			0x3b, 0x1d, 0xfa, 0x30, 0x57, 0xdc, 0x58, 0x48, 0xe5,
+			0x45, 0xf3, 0x34, 0x8a, 0x73, 0x66, 0x8a, 0xe0, 0x62,
+			0x3d, 0x0d, 0xfb, 0x13, 0x52, 0x25, 0xf5, 0xd3, 0x94,
+			0x15, 0x9d, 0xac, 0x6a, 0x74, 0x8a, 0x64, 0x91, 0x5b,
+			0xa3, 0xe2, 0xd0, 0x5c, 0xd2, 0xee, 0x52, 0xc0, 0x0c,
+			0x6c, 0x81, 0x34, 0x94, 0xfb, 0x87, 0xf6, 0x6b, 0x0f,
+			0x7e, 0x99, 0xa7, 0xaf, 0xb3, 0x74, 0xa7, 0xb6, 0x64,
+			0xd7, 0x36, 0x39, 0x3f, 0x7d, 0x0e, 0xc3, 0x76, 0xb3,
+			0x3a, 0xc9, 0x94, 0x6f, 0xc0, 0xaa, 0x92, 0x2a, 0xf1,
+			0x38, 0x8c, 0x62, 0xa1, 0x9a, 0xbe, 0x8d, 0x32, 0xec,
+			0x05, 0xb9, 0x8d, 0xb0, 0xdb, 0x37, 0x8d, 0x61, 0xfa,
+			0x79, 0x55, 0x1d, 0xf2, 0xc3, 0x19, 0x15, 0x3b, 0x26,
+			0xef, 0xe5, 0xf3, 0x29, 0xa1, 0x13, 0x60, 0x68, 0xd3,
+			0x22, 0x57, 0xcd, 0x99, 0xe6, 0x22, 0x54, 0xdf, 0x02,
+			0xe4, 0xd5, 0xd2, 0x66, 0xce, 0x5a, 0x90, 0x13, 0x7e,
+			0x7c, 0xad, 0x4c, 0x4c, 0xb6, 0x93, 0x65, 0xda, 0x4b,
+			0xb6, 0x13, 0x0e, 0x1b, 0x36, 0xaa, 0x2c, 0xbd, 0x60,
+			0x57, 0x98, 0x35, 0xa9, 0xe9, 0xad, 0xc5, 0x8e, 0x33,
+			0x47, 0x00, 0xe9, 0xf2, 0x1b, 0xe7, 0x5a, 0xd0, 0x55,
+			0x3b, 0x0e, 0x77, 0x09, 0x7e, 0x1f, 0x11, 0x0b, 0xf0,
+			0xe4, 0x07, 0x27, 0xe7, 0x72, 0xe9, 0x50, 0x9c, 0x9c,
+			0x84, 0xb0, 0x5c, 0xe0, 0x13, 0x79, 0xfa, 0x3f, 0x80,
+			0x96, 0xe2, 0x04, 0xde, 0x35, 0x5f, 0xa4, 0x4e, 0x8a,
+			0x12, 0x7a, 0xba, 0x53, 0x20, 0x36, 0x3d, 0xf0, 0x63,
+			0xb6, 0x14, 0xab, 0xce, 0x6b, 0x24, 0xfd, 0xc9, 0xd2,
+			0x52, 0xd9, 0xc0, 0x40, 0xdf, 0xdd, 0xd5, 0xc2, 0xa0,
+			0xf5, 0x74, 0x46, 0x17, 0xdc, 0xf5, 0x81, 0x68, 0xbd,
+			0x1a, 0x01, 0x33, 0xd0, 0x2f, 0xdc, 0x3a, 0x43, 0x4a,
+			0x08, 0x07, 0x98, 0x9f, 0x75, 0x5d, 0x70, 0x15, 0x1d,
+			0x58, 0x7f, 0x26, 0x30, 0x28, 0xe1, 0x17, 0x6d, 0x14,
+			0x0f, 0x01, 0x4a, 0x88, 0x3b, 0xf5, 0x70, 0x1d, 0x97,
+			0xd5, 0xda, 0x18, 0xa8, 0xb3, 0xa4, 0x0e, 0x04, 0x3f,
+			0xe8, 0x50, 0xf4, 0x87, 0x65, 0x75, 0x5a, 0x5d, 0x2f,
+			0x94, 0x63, 0x7a, 0xd0, 0x06, 0xce, 0xfb, 0xeb, 0x41,
+			0x65, 0xc9, 0x55, 0x38, 0x54, 0xd3, 0xc4, 0xac, 0x24,
+			0x46, 0x06, 0x2a, 0x87, 0xb1, 0x0f, 0x06, 0xd8, 0x1e,
+			0xa7, 0x35, 0xc4, 0xa3, 0xdc, 0x60, 0x80, 0x83, 0xe7,
+			0xf0, 0x74, 0xf6, 0xd6, 0xa7, 0x1d, 0x50, 0xff, 0xba,
+			0x82, 0xc0, 0xca, 0x72, 0x6b, 0xda, 0x4a, 0xcb, 0x3b,
+			0xe8, 0xa3, 0xaa, 0x32, 0x86, 0xc0, 0x94, 0x75, 0x2b,
+			0x2f, 0x44, 0xad, 0x5b, 0x8d, 0xab, 0xc2, 0x03, 0x55,
+			0xb1, 0x7e, 0x67, 0x07, 0x39, 0x33, 0x83, 0x67, 0xec,
+			0xbf, 0x52, 0xad, 0x55, 0x37, 0x3e, 0xa1, 0x41, 0xed,
+			0xa0, 0x91, 0xbf, 0x28, 0x1b, 0x04, 0x32, 0xbf, 0xf1,
+			0xb0, 0x0a, 0x11, 0x99, 0x98, 0x77, 0xee, 0x14, 0x13,
+			0x15, 0x13, 0xad, 0x1f, 0xb6, 0x6b, 0xcd, 0x3b, 0xb8,
+			0x75, 0x9a, 0x55, 0x9b, 0x0c, 0x6c, 0xf7, 0x7c, 0x21,
+			0x06, 0xb7, 0xf0, 0x43, 0x41, 0x96, 0xc1, 0x73, 0x44,
+			0x53, 0xd8, 0x18, 0x3e, 0x09, 0x4e, 0xc2, 0x5f, 0xa0,
+			0xd5, 0x18, 0xdb, 0x9f, 0xf4, 0xa0, 0xf8, 0x4d, 0xa7,
+			0x72, 0x7e, 0x85, 0xbb, 0xb7, 0xcc, 0x7b, 0x51, 0xb0,
+			0xf1, 0x5e, 0x03, 0xcd, 0xe8, 0x5b, 0x83, 0x3f, 0x95,
+			0xe2, 0x0b, 0xa1, 0xc7, 0x6f, 0x74, 0x98, 0xcd, 0x95,
+			0xf4, 0xd4, 0xb8, 0x40, 0xff, 0x75, 0x54, 0x83, 0x3a,
+			0x2c, 0x64, 0x38, 0x10, 0xcb, 0x6d, 0xad, 0xf9, 0x91,
+			0xcb, 0xcf, 0xbe, 0xf6, 0xf7, 0x94, 0x15, 0xea, 0xaf,
+			0x37, 0x65, 0x7c, 0xd2, 0xff, 0x99, 0x79, 0xf8, 0x95,
+			0x27, 0x75, 0x09, 0x60, 0xa4, 0x6f, 0x06, 0x0f, 0x6c,
+			0x13, 0xdd, 0x32, 0x79, 0xa7, 0x40, 0xa5, 0xdd, 0x2c,
+			0x22, 0xc1, 0xee, 0xc3, 0x31, 0x59, 0xb1, 0x3d, 0xa1,
+			0x77, 0x69, 0xb5, 0xd6, 0xae, 0xd7, 0x86, 0xab, 0xa4,
+			0xdf, 0x9f, 0x36, 0xb4, 0xb2, 0xe6, 0x88, 0xd6, 0x45,
+			0x9b, 0x8b, 0x87, 0xbd, 0x27, 0xe9, 0x55, 0xd5, 0xac,
+			0xe9, 0x33, 0x80, 0x00, 0x1d, 0x00, 0x21, 0x74, 0xe2,
+			0x0a, 0x5b, 0xd2, 0x37, 0xab, 0x6f, 0x48, 0x5e, 0x14,
+			0x76, 0x3a, 0x84, 0xf9, 0x34, 0x03, 0x74, 0x8d, 0x6c,
+			0xd8, 0xd4, 0x0f, 0xc7, 0xc8, 0x0c, 0xb5, 0x18, 0x12,
+			0xa5, 0x5a, 0x76, 0x4b, 0x5d, 0x1e, 0x75, 0x89, 0x3a,
+			0x01, 0x60, 0xf1, 0x1e, 0x5b, 0x98, 0x71, 0x7a, 0x15,
+			0x79, 0xff, 0x3d, 0x37, 0x5e, 0xc9, 0x56, 0xa5, 0x43,
+			0xe7, 0xd0, 0x2b, 0x63, 0xfb, 0x2f, 0x1a, 0x55, 0x37,
+			0x15, 0x02, 0x7f, 0x9a, 0x84, 0xf0, 0x07, 0x45, 0x2b,
+			0xa0, 0x9a, 0xb4, 0x60, 0x4d, 0x48, 0xd5, 0xee, 0x64,
+			0xf9, 0xe7, 0x4b, 0x1c, 0xb9, 0x05, 0xd1, 0xd1, 0x47,
+			0x4f, 0xca, 0x26, 0x32, 0xad, 0x7b, 0x2e, 0x7e, 0x54,
+			0x05, 0x96, 0x12, 0x29, 0xbc, 0xd3, 0x45, 0x4e, 0x05,
+			0x83, 0x6d, 0x30, 0x38, 0xd0, 0x3c, 0x46, 0xcd, 0xb6,
+			0x21, 0xb6, 0x5c, 0x2a, 0x40, 0x7a, 0x2e, 0x97, 0xd7,
+			0xbf, 0xa6, 0x55, 0x2a, 0xa6, 0x28, 0x54, 0xc4, 0x66,
+			0xbe, 0xf6, 0xea, 0xc1, 0x19, 0xcb, 0xcc, 0x52, 0x07,
+			0xcd, 0x90, 0x3f, 0x09, 0x23, 0x77, 0xf7, 0xc6, 0x03,
+			0xcc, 0x8b, 0x18, 0xac, 0x63, 0x6d, 0x2a, 0x01, 0x82,
+			0xad, 0x20, 0x11, 0x82, 0xf3, 0xfe, 0x2d, 0x53, 0x3f,
+			0x18, 0xb5, 0xbc, 0xf7, 0x0c, 0xa7, 0xca, 0x63, 0xfa,
+			0x77, 0x8a, 0x3c, 0xc1, 0xc4, 0xb0, 0x15, 0x4b, 0x08,
+			0xdd, 0xb3, 0xd2, 0x33, 0xc8, 0xc2, 0xaa, 0x1a, 0xe4,
+			0x9b, 0x1b, 0x18, 0x78, 0x07, 0x6c, 0x08, 0x28, 0x3f,
+			0xc6, 0xb7, 0x46, 0x8f, 0x8e, 0xc6, 0xff, 0xb1, 0x2d,
+			0x63, 0x12, 0x7c, 0xe5, 0xd8, 0xa5, 0xf8, 0x64, 0x76,
+			0xf7, 0x1b, 0xbc, 0x41, 0xad, 0xee, 0xe8, 0x6f, 0xd4,
+			0x13, 0xea, 0x59, 0x72, 0xfa, 0x04, 0xb4, 0x6e, 0x52,
+			0x3a, 0xff, 0x60, 0xa0, 0xd8, 0xdf, 0x41, 0x1a, 0x4e,
+			0xc8, 0x80, 0x2d, 0x29, 0x13, 0xd3, 0xd0, 0x55, 0x20,
+			0xb5, 0x33, 0x4f, 0x02, 0xd2, 0xea, 0xca, 0x94, 0x99,
+			0xdf, 0x3c, 0xfd, 0xe5, 0x2b, 0x45, 0x04, 0x85, 0xfe,
+			0x87, 0x42, 0x53, 0x11, 0x62, 0x2f, 0xcf, 0x05, 0x35,
+			0x29, 0x6a, 0xa5, 0x37, 0x80, 0x9b, 0x4a, 0x44, 0x3c,
+			0x6d, 0xe1, 0xd3, 0x88, 0x6a, 0xe7, 0x6f, 0x45, 0xfc,
+			0x99, 0x33, 0xe4, 0x18, 0x62, 0x19, 0x36, 0x37, 0x6f,
+			0xe9, 0x37, 0xbb, 0x86, 0xa2, 0x07, 0xec, 0xbc, 0x33,
+			0x0d, 0x0a, 0x18, 0x30, 0x13, 0xcf, 0x36, 0x31, 0x2f,
+			0xd2, 0xf6, 0x7e, 0xa8, 0xa4, 0x95, 0x71, 0xc8, 0x40,
+			0x1f, 0x34, 0xf1, 0x95, 0xde, 0x3e, 0xe1, 0xb7, 0xef,
+			0x26, 0x1e, 0x20, 0x78, 0xf8, 0x9a, 0x24, 0xbc, 0x7f,
+			0x64, 0x30, 0x42, 0x63, 0x77, 0x89, 0x96, 0xf8, 0x9b,
+			0x60, 0x51, 0xb3, 0x02, 0x7a, 0x64, 0x19, 0xd3, 0x13,
+			0x98, 0xcd, 0xb6, 0x1e, 0x68, 0x82, 0x9b, 0x23, 0x44,
+			0x3e, 0xa8, 0x57, 0xf5, 0x4d, 0xd8, 0xea, 0x93, 0x84,
+			0x4e, 0x56, 0x18, 0xb7, 0x94, 0x02, 0xfa, 0x6e, 0x8e,
+			0xa0, 0x75, 0xd2, 0xb4, 0xbb, 0x50, 0x6f, 0x3f, 0x58,
+			0xe2, 0x28, 0xb6, 0x87, 0xd6, 0xc5, 0xe0, 0x76, 0xa6,
+			0xc6, 0x5b, 0xed, 0x6b, 0x48, 0xd4, 0x25, 0xd3, 0x5f,
+			0x64, 0xab, 0xe3, 0x5a, 0xae, 0x69, 0xe8, 0x06, 0xff,
+			0xef, 0x67, 0x0b, 0x49, 0xe4, 0xe8, 0x31, 0xee, 0x1a,
+			0x35, 0x86, 0x0c, 0x81, 0x8a, 0x1f, 0x7f, 0x06, 0x20,
+			0x66, 0x81, 0x8b, 0xfb, 0xdc, 0x99, 0x78, 0x7b, 0x43,
+			0x84, 0x35, 0xa7, 0x4f, 0x27, 0xb5, 0xc9, 0x44, 0xbf,
+			0x05, 0x59, 0xce, 0x4c, 0xe5, 0xaa, 0x18, 0x37, 0x32,
+			0x5b, 0x41, 0x62, 0x89, 0x0a, 0xec, 0x0a, 0xae, 0x67,
+			0xcf, 0x0b, 0xef, 0xf6, 0x3a, 0x3b, 0x13, 0x1d, 0xf8,
+			0x8b, 0x20, 0x5f, 0x2c, 0x55, 0x75, 0xec, 0xf4, 0x9d,
+			0x2d, 0x0d, 0x0d, 0xb7, 0x88, 0x71, 0xf7, 0xc6, 0xf8,
+			0x65, 0x1a, 0xba, 0xdb, 0xdc, 0x53, 0xd2, 0x80, 0x92,
+			0x72, 0x14, 0xce, 0x68, 0xb8, 0xf4, 0x78, 0x4b, 0x92,
+			0xee, 0x15, 0xfc, 0xd4, 0xe0, 0x89, 0x49, 0x82, 0xf8,
+			0x7f, 0xce, 0xf6, 0xce, 0x86, 0xfa, 0xa5, 0xd9, 0x3a,
+			0xfb, 0xef, 0xa8, 0xd9, 0x3b, 0x6e, 0xb5, 0xe2, 0x29,
+			0x31, 0x6f, 0x5f, 0xa7, 0x34, 0x58, 0x49, 0x07, 0x9c,
+			0x4d, 0x74, 0x46, 0xb6, 0xd5, 0x0c, 0x3c, 0x26, 0x4b,
+			0xee, 0x37, 0x9c, 0x92, 0xa2, 0x37, 0x95, 0x35, 0x71,
+			0xee, 0x46, 0xdd, 0xef, 0xd0, 0xf2, 0xd4, 0xef, 0xe0,
+			0xdb, 0x28, 0xf4, 0x5d, 0x12, 0xe0, 0x85, 0x4a, 0x3c,
+			0x5b, 0x2f, 0xd8, 0x9c, 0x9a, 0xcf, 0x6f, 0x01, 0xe7,
+			0x1b, 0x2e, 0x41, 0x5d, 0x5d, 0xdd, 0xdd, 0xb6, 0x6d,
+			0xb3, 0x7d, 0xbc, 0x33, 0x50, 0xc5, 0xaf, 0x1b, 0xee,
+			0x33, 0xc4, 0x42, 0x46, 0xca, 0xe2, 0x00, 0xe9, 0xc9,
+			0xec, 0x9e, 0x9b, 0x67, 0xba, 0x39, 0x3c, 0xc5, 0xa1,
+			0x7d, 0xac, 0x9e, 0xbe, 0x67, 0x92, 0x20, 0x5b, 0x36,
+			0x85, 0xde, 0xdd, 0xd3, 0xec, 0x4e, 0x82, 0x03, 0x4a,
+			0x44, 0x6e, 0xee, 0x9c, 0xd0, 0x13, 0x9e, 0x98, 0x6f,
+			0x77, 0xd8, 0xfe, 0xb7, 0x54, 0x01, 0x15, 0xcd, 0xd3,
+			0x31, 0xf5, 0xd7, 0x74, 0x2d, 0x4d, 0x4f, 0x18, 0xd2,
+			0x8f, 0x80, 0xb6, 0x46, 0x80, 0xa7, 0xa8, 0x8c, 0xf2,
+			0x64, 0x4f, 0x40, 0x8a, 0x64, 0x96, 0x2c, 0x1e, 0xd2,
+			0x3c, 0xe9, 0x50, 0xc4, 0x86, 0xd9, 0x19, 0x93, 0x7a,
+			0xba, 0x95, 0x35, 0x35, 0xe3, 0x4d, 0x83, 0xda, 0x15,
+			0x09, 0x87, 0xb8, 0x1a, 0x54, 0x6f, 0x61, 0xdd, 0x55,
+			0x01, 0xe1, 0x39, 0x1c, 0xb2, 0xac, 0x8c, 0x15, 0x82,
+			0x7c, 0xa6, 0x69, 0x54, 0x64, 0x2f, 0x5c, 0x0c, 0x79,
+			0xaf, 0x85, 0x71, 0x2d, 0xff, 0xc0, 0xf2, 0x40, 0xbb,
+			0x05, 0x31, 0x31, 0x00, 0x8d, 0x0c, 0x40, 0x11, 0x88,
+			0xf2, 0x91, 0x9f, 0x7a, 0x8e, 0x99, 0x82, 0x72, 0xd8,
+			0x45, 0x08, 0x83, 0x5c, 0x2b, 0x73, 0x46, 0xd6, 0x54,
+			0x9a, 0x3d, 0x42, 0x48, 0x67, 0x19, 0x2b, 0x19, 0xae,
+			0x0c, 0x16, 0x08, 0xa4, 0xec, 0x7e, 0x15, 0xad, 0x4f,
+			0xa9, 0xbd, 0x4f, 0x09, 0xf0, 0xc2, 0x9f, 0xb5, 0xb7,
+			0xff, 0x96, 0xf9, 0xf9, 0xc6, 0x5d, 0x57, 0x07, 0xe8,
+			0xf2, 0x7c, 0x95, 0xa0, 0x8a, 0x15, 0x8b, 0x34, 0x84,
+			0xb5, 0x28, 0x7a, 0xf5, 0x04, 0xb8, 0xcb, 0x5e, 0x77,
+			0x4d, 0xdd, 0x14, 0x8d, 0xa6, 0xe2, 0x45, 0x44, 0xfd,
+			0xd0, 0x9c, 0x41, 0x10, 0xfc, 0xe5, 0xdd, 0x2c, 0xda,
+			0x6b, 0xfe, 0xf9, 0xff, 0x65, 0xe7, 0x66, 0x40, 0x69,
+			0x0c, 0x09, 0x99, 0xaf, 0x02, 0xd0, 0x85, 0x55, 0xcc,
+			0x0d, 0x59, 0xaa, 0xb6, 0x37, 0x9b, 0x58, 0xfa, 0xb5,
+			0xc5, 0xcd, 0xd5, 0x1c, 0xf9, 0x9f, 0x31, 0x4c, 0x8a,
+			0x78, 0xca, 0x1e, 0x22, 0xdc, 0x33, 0x64, 0xce, 0x56,
+			0x6d, 0x57, 0xe9, 0xbd, 0xe0, 0xdd, 0x23, 0x22, 0x9c,
+			0x70, 0x5a, 0xe7, 0x46, 0x08, 0xc5, 0xa6, 0x85, 0x34,
+			0x5f, 0x9c, 0x96, 0xbb, 0xbe, 0xb4, 0x2f, 0xbb, 0x47,
+			0xdc, 0x38, 0xad, 0xb7, 0x92, 0x08, 0x0d, 0x93, 0xf4,
+			0x7f, 0xe6, 0x64, 0xa5, 0xb3, 0x9e, 0xdd, 0xdb, 0x99,
+			0x8e, 0xca, 0x82, 0x4f, 0x77, 0xf7, 0xff, 0x92, 0xd3,
+			0x48, 0xe0, 0x80, 0x71, 0x14, 0x61, 0x6b, 0x77, 0x1b,
+			0x58, 0xbd, 0xb6, 0xb6, 0xe5, 0xb0, 0xa8, 0x16, 0xe7,
+			0x4c, 0xc2, 0x5e, 0x40, 0xbc, 0x9f, 0x77, 0x76, 0x98,
+			0xf7, 0x00, 0xd1, 0xdf, 0x3c, 0x2e, 0x16, 0x4f, 0xe7,
+			0x91, 0xec, 0xeb, 0xfb, 0xa1, 0x9f, 0x66, 0x6d, 0xef,
+			0xaf, 0x36, 0xd8, 0xb6, 0x40, 0x7e, 0x2f, 0x51, 0x3b,
+			0x75, 0x56, 0x1d, 0x33, 0xee, 0x4b, 0xcf, 0x98, 0xb3,
+			0x9f, 0x4f, 0xff, 0xfd, 0x2a, 0xb9, 0x4e, 0x4b, 0xe0,
+			0x39, 0x60, 0xd5, 0x7c, 0xaa, 0x28, 0x7e, 0x7e, 0x27,
+			0xd8, 0x09, 0x85, 0x6d, 0xfa, 0x57, 0x8a, 0xee, 0x84,
+			0xbe, 0x48, 0xf9, 0x62, 0x78, 0x8a, 0x4d, 0x2a, 0x80,
+			0x2f, 0x6f, 0xc9, 0xa7, 0xb5, 0x56, 0xbb, 0x58, 0xc2,
+			0xb9, 0xd0, 0xfe, 0x8d, 0x68, 0x86, 0x2b, 0x7b, 0xf0,
+			0x01, 0x57, 0xff, 0xaf, 0x88, 0xad, 0x53, 0xf9, 0x63,
+			0xfe, 0xf4, 0x5a, 0x97, 0xb7, 0x8f, 0x92, 0x0f, 0x90,
+			0x63, 0xcf, 0xfe, 0x08, 0xbc, 0x8c, 0x5a, 0x9c, 0xac,
+			0x8b, 0x0a, 0x94, 0x17, 0x25, 0xae, 0x87, 0xa7, 0x1d,
+			0x3f, 0x28, 0xfe, 0x45, 0x3e, 0x48, 0xef, 0x19, 0x4a,
+			0xbe, 0xc6, 0xd5, 0x64, 0xe1, 0xf2, 0xf5, 0xd2, 0x3a,
+			0x03, 0xe0, 0x50, 0x31, 0x7e, 0xdf, 0xd5, 0x15, 0xc6,
+			0xb6, 0xb2, 0x1e, 0x44, 0x6a, 0x0f, 0x67, 0xf2, 0x34,
+			0x69, 0xb9, 0xd2, 0x51, 0xeb, 0x86, 0xab, 0x7e, 0x60,
+			0x68, 0xe4, 0xc2, 0x4b, 0x8b, 0xa2, 0xc6, 0x78, 0xb2,
+			0x61, 0xa1, 0xe5, 0xb6, 0x33, 0x47, 0x3f, 0xfa, 0xde,
+			0x5b, 0x6b, 0x95, 0x89, 0x51, 0xbb, 0xd3, 0x8a, 0x00,
+			0x1d, 0xad, 0xfc, 0x7a, 0x47, 0xc8, 0xa0, 0x69, 0xa6,
+			0xa3, 0x3f, 0xa9, 0xdb, 0xd8, 0xe4, 0x0d, 0x2c, 0xba,
+			0x26, 0x2c, 0xa6, 0x82, 0x62, 0x5a, 0x1e, 0x9f, 0xa8,
+			0x24, 0x9d, 0xe5, 0xc7, 0xcf, 0x6d, 0x2f, 0x68, 0xe1,
+			0xc0, 0x04, 0xad, 0xde, 0x00, 0x5b, 0x57, 0x35, 0x2b,
+			0x9a, 0x9b, 0xe5, 0x90, 0x31, 0x8a, 0x5b, 0xea, 0xe5,
+			0xf6, 0x73, 0x02, 0xc1, 0x33, 0xaf, 0xe7, 0x61, 0x6c,
+			0x04, 0x8c, 0x08, 0xb2, 0xfb, 0xa2, 0x18, 0x51, 0x39,
+			0x1a, 0x61, 0x2c, 0x8c, 0x7d, 0x11, 0xb8, 0x2e, 0x38,
+			0xb7, 0xe0, 0xde, 0x5e, 0x6f, 0xc5, 0xf4, 0x31, 0x72,
+			0xcb, 0xd6, 0xec, 0xbc, 0xe9, 0x36, 0x2e, 0x2b, 0x63,
+			0xea, 0xa1, 0xab, 0xf8, 0x11, 0xe7, 0x31, 0xad, 0xe4,
+			0xcb, 0x23, 0xe4, 0x64, 0x54, 0x50, 0x4f, 0x15, 0x8b,
+			0x58, 0xa4, 0xa1, 0x35, 0xea, 0x73, 0x3e, 0xc3, 0x3e,
+			0x31, 0x81, 0x80, 0x53, 0x6a, 0x2c, 0xfe, 0xd7, 0xd8,
+			0x8f, 0xe7, 0x50, 0x2d, 0x99, 0x69, 0x32, 0xfa, 0x4e,
+			0xb4, 0xf9, 0x89, 0x18, 0xbe, 0x5a, 0x95, 0x7a, 0x0f,
+			0x34, 0xc5, 0xd9, 0x28, 0xff, 0x93, 0x3a, 0x13, 0x22,
+			0xe6, 0xdb, 0x5c, 0xfe, 0x51, 0x05, 0xaf, 0xa3, 0xcd,
+			0x17, 0x01, 0xd1, 0x84, 0x76, 0x23, 0xff, 0x91, 0xc4,
+			0x6b, 0xc7, 0xa8, 0x77, 0xe0, 0xda, 0x45, 0xef, 0xd7,
+			0x13, 0xe3, 0x02, 0x7c, 0x74, 0x7e, 0x66, 0xbb, 0x9f,
+			0x48, 0x35, 0x56, 0xa8, 0x08, 0x41, 0xf7, 0xa7, 0xe3,
+			0x60, 0xa6, 0xce, 0x73, 0xdf, 0x73, 0x99, 0xc6, 0xbd,
+			0x7b, 0xb4, 0xde, 0x31, 0xe6, 0x35, 0xaf, 0x3e, 0x93,
+			0xd2, 0x69, 0xb5, 0xb9, 0xbe, 0x9f, 0xf4, 0x0d, 0xd8,
+			0x82, 0xc1, 0x4f, 0xf6, 0xf3, 0x23, 0x1c, 0xe7, 0xff,
+			0xac, 0xb2, 0x79, 0xe3, 0x62, 0x6a, 0xb7, 0x85, 0x08,
+			0xda, 0x4a, 0x3f, 0x22, 0x62, 0x2b, 0xcd, 0xb7, 0x59,
+			0xf5, 0x82, 0xef, 0x08, 0x7f, 0xb5, 0xa0, 0x6c, 0xba,
+			0x61, 0x45, 0x8f, 0xf4, 0xa2, 0x66, 0x83, 0x72, 0x65,
+			0x34, 0x34, 0x68, 0x99, 0xf5, 0x99, 0xc9, 0x52, 0x6a,
+			0xc2, 0x7f, 0xde, 0x25, 0xa3, 0x7b, 0xce, 0xa3, 0xde,
+			0x6d, 0x62, 0x98, 0x84, 0xe4, 0x2a, 0xe1, 0x1c, 0xa9,
+			0x0b, 0x7e, 0x45, 0xd3, 0x78, 0x40, 0x85, 0x61, 0x9d,
+			0xce, 0xcd, 0xfd, 0x7b, 0x66, 0x92, 0x5f, 0xb9, 0x37,
+			0x04, 0xea, 0xec, 0x37, 0x32, 0xbe, 0xcd, 0x3e, 0x5e,
+			0x37, 0xcd, 0xe3, 0x81, 0x92, 0x69, 0xdf, 0xa1, 0xb9,
+			0x8c, 0x44, 0x43, 0xae, 0xcd, 0xe5, 0xd5, 0x4e, 0xb9,
+			0x2f, 0x1e, 0xbc, 0xf2, 0xd1, 0x88, 0x93, 0xde, 0x6a,
+			0x8a, 0x43, 0xa1, 0x6b, 0xdc, 0x5b, 0xd2, 0xdd, 0xe8,
+			0xca, 0x0a, 0x95, 0x45, 0x9f, 0xd8, 0x8c, 0xfd, 0xe0,
+			0xc3, 0xbd, 0x0e, 0xa0, 0xbd, 0x72, 0xa7, 0x77, 0x9a,
+			0x79, 0xd9, 0x15, 0xad, 0x50, 0x13, 0xa7, 0x41, 0xc7,
+			0xd1, 0xec, 0x2f, 0x45, 0x14, 0x02, 0xb7, 0x8e, 0x51,
+			0x76, 0xd4, 0x2f, 0x1c, 0xea, 0x9c, 0x6d, 0x35, 0x1c,
+			0xa7, 0xeb, 0xd7, 0x79, 0xe4, 0x35, 0xbc, 0x02, 0xa6,
+			0xd0, 0x58, 0xcb, 0xdf, 0xea, 0x90, 0x16, 0x53, 0x3c,
+			0xc5, 0xda, 0x36, 0x4f, 0x61, 0x4a, 0x1b, 0xfa, 0x15,
+			0x52, 0x0d, 0x3d, 0x73, 0xc5, 0xfe, 0x2a, 0xa2, 0x4a,
+			0x6e, 0x95, 0x99, 0xfc, 0xbf, 0xb2, 0xfc, 0x22, 0xbe,
+			0x45, 0x10, 0x5a, 0xc5, 0x38, 0xa0, 0xc1, 0xba, 0x46,
+			0x6d, 0x8e, 0xbe, 0x0b, 0x5f, 0xf6, 0x6f, 0xbb, 0x4d,
+			0x22, 0xf5, 0x95, 0x4d, 0x12, 0xc2, 0x6f, 0x8d, 0xde,
+			0x86, 0xaf, 0x3f, 0x08, 0x24, 0x56, 0xa3, 0xcd, 0xfc,
+			0xeb, 0xa1, 0x3c, 0x22, 0x66, 0x45, 0x03, 0x8b, 0x04,
+			0x12, 0x80, 0x34, 0xf9, 0xf4, 0x64, 0xbf, 0xb9, 0x3d,
+			0xd7, 0x34, 0xe4, 0xc8, 0x8e, 0x86, 0xc9, 0x73, 0x21,
+			0x7f, 0x30, 0x01, 0xc8, 0xd2, 0x91, 0x8d, 0xc4, 0xca,
+			0xd5, 0x59, 0xac, 0xd7, 0xb4, 0xfb, 0x98, 0xfb, 0xc4,
+			0x48, 0x82, 0x93, 0xc6, 0x58, 0x8b, 0x0c, 0xdf, 0x3f,
+			0x07, 0x5a, 0x94, 0x89, 0xc6, 0xbe, 0x1a, 0x19, 0x63,
+			0x8b, 0xa8, 0x0c, 0x20, 0xae, 0x9a, 0x1b, 0xd3, 0x45,
+			0xa0, 0xd9, 0x8c, 0xbb, 0x67, 0xa0, 0x54, 0x86, 0x9b,
+			0x7b, 0xa8, 0x16, 0x6f, 0xa4, 0x59, 0x45, 0x1e, 0x34,
+			0xb5, 0xa7, 0x1b, 0x3f, 0xda, 0xc6, 0x80, 0x19, 0xdc,
+			0xc2, 0xee, 0xeb, 0xd9, 0x96, 0xdc, 0x4f, 0xec, 0x02,
+			0xde, 0x34, 0x43, 0x5a, 0x37, 0xc8, 0xdf, 0x3b, 0x53,
+			0x82, 0xfa, 0xf8, 0x8a, 0x32, 0x97, 0x1e, 0xf6, 0x0e,
+			0x7d, 0x39, 0xe0, 0xdb, 0x0b, 0x22, 0xa4, 0xd0, 0x56,
+			0x13, 0xbc, 0xe9, 0x38, 0x92, 0x76, 0xff, 0x2b, 0xc2,
+			0xa0, 0x98, 0x7d, 0x8c, 0x58, 0x8e, 0x7f, 0x6d, 0x80,
+			0x06, 0xff, 0xc7, 0xb3, 0xb7, 0x1b, 0xac, 0xc4, 0x68,
+			0x61, 0x11, 0x15, 0xb7, 0xee, 0x8d, 0x66, 0x34, 0x9f,
+			0x18, 0x03, 0x88, 0xd1, 0x04, 0xd0, 0x86, 0xc0, 0xca,
+			0x2a, 0xe1, 0x1b, 0x47, 0x18, 0xc9, 0x97, 0xed, 0x1f,
+			0x99, 0xc6, 0xa9, 0x20, 0x3f, 0xf6, 0x43, 0x96, 0x30,
+			0x8b, 0xa5, 0xcb, 0x2e, 0xc5, 0xdb, 0x26, 0xc0, 0xf6,
+			0x48, 0x07, 0x3a, 0x28, 0x3e, 0x35, 0x38, 0x50, 0x80,
+			0xdf, 0x80, 0xfa, 0x12, 0x92, 0x83, 0x55, 0xde, 0x31,
+		},
+	.msg_len = 64,
+	.msg =
+		(const u8[64]) {
+			0x48, 0x11, 0x92, 0xb3, 0xaa, 0x7e, 0xb3, 0x14,
+			0xdc, 0x46, 0xd6, 0xbf, 0x26, 0xb2, 0x66, 0xc5,
+			0xd8, 0xc4, 0x69, 0x9f, 0x67, 0x50, 0x67, 0xc0,
+			0x3b, 0x8d, 0xc5, 0xb5, 0x68, 0x59, 0x6f, 0x9c,
+			0xd4, 0xf2, 0x49, 0xfb, 0xa4, 0xd0, 0xbf, 0xc5,
+			0x3c, 0xee, 0x3b, 0x03, 0x25, 0x6c, 0x51, 0x94,
+			0xb3, 0xd4, 0xdb, 0x2a, 0xbd, 0x26, 0xef, 0x58,
+			0xb3, 0x91, 0x62, 0x81, 0x1b, 0x29, 0x9c, 0x12,
+		},
+	.sig_len = MLDSA87_SIGNATURE_SIZE,
+	.sig =
+		(const u8[MLDSA87_SIGNATURE_SIZE]) {
+			0xba, 0x08, 0xf3, 0x0c, 0xa1, 0x48, 0xad, 0x67, 0x12,
+			0x68, 0x51, 0x9b, 0xa4, 0x21, 0x66, 0x8d, 0xea, 0x0a,
+			0x71, 0x4f, 0x4c, 0xab, 0x1b, 0x42, 0x84, 0xb7, 0x9c,
+			0x78, 0xa9, 0x18, 0xd9, 0xe8, 0x08, 0x8f, 0xee, 0xf7,
+			0x4b, 0xcf, 0x0e, 0xcd, 0xf6, 0x36, 0xcf, 0xd5, 0xa1,
+			0x5e, 0x59, 0xc6, 0x8c, 0x6c, 0x4d, 0xfe, 0xfa, 0xfa,
+			0xbc, 0x52, 0x3a, 0x35, 0xba, 0xd9, 0xe1, 0xf2, 0x26,
+			0x73, 0xbd, 0x64, 0x93, 0xe7, 0x83, 0x76, 0xc4, 0xf8,
+			0x4b, 0xa9, 0x91, 0x8c, 0xd2, 0xf5, 0x08, 0xe6, 0x58,
+			0xcd, 0xfb, 0x90, 0xc1, 0x84, 0x72, 0x77, 0x72, 0x47,
+			0x5f, 0xcb, 0x7f, 0x74, 0x85, 0x56, 0xba, 0x73, 0x86,
+			0xc1, 0x32, 0x1c, 0xaf, 0xc5, 0x60, 0x2c, 0x91, 0x99,
+			0x09, 0x03, 0xcb, 0xe6, 0xa4, 0xae, 0x0b, 0xdc, 0x15,
+			0x70, 0x4c, 0x91, 0xa9, 0x9e, 0x5e, 0x63, 0x5c, 0x24,
+			0xc9, 0x26, 0x39, 0x56, 0xe4, 0x6c, 0x2c, 0x65, 0xf8,
+			0x0b, 0x4f, 0x56, 0x8f, 0x1e, 0x74, 0xda, 0x31, 0xad,
+			0x8f, 0x1b, 0xa7, 0xb1, 0xb3, 0x9b, 0x48, 0x09, 0x08,
+			0xdd, 0xe1, 0x25, 0x20, 0xa4, 0xc7, 0x12, 0xd4, 0xb0,
+			0xa0, 0xcb, 0x90, 0xf5, 0x9f, 0xc1, 0x8b, 0xbc, 0x9d,
+			0xcf, 0x78, 0xfc, 0xa1, 0x04, 0x4b, 0x0b, 0x7e, 0x82,
+			0x94, 0x80, 0x0e, 0x70, 0x77, 0xce, 0x63, 0x1c, 0x43,
+			0x41, 0x3f, 0x5d, 0x61, 0x1a, 0x44, 0xaa, 0xd0, 0xb7,
+			0x18, 0xd8, 0x91, 0x83, 0xa4, 0xa7, 0xa1, 0x86, 0x13,
+			0x54, 0xaf, 0xa8, 0xdb, 0x1a, 0xd7, 0xa9, 0x54, 0xca,
+			0x67, 0xb5, 0xb6, 0xb4, 0x22, 0x1a, 0x24, 0x8e, 0x65,
+			0x01, 0xa8, 0xd2, 0x15, 0x9d, 0x98, 0x59, 0x70, 0x94,
+			0xb4, 0x96, 0xeb, 0xf2, 0x95, 0x41, 0x9a, 0x89, 0x02,
+			0xb7, 0x6a, 0x20, 0x2a, 0x64, 0xf1, 0x1c, 0x1e, 0x3d,
+			0xd8, 0xdb, 0x6d, 0xef, 0xf2, 0x1a, 0x24, 0xa4, 0xd0,
+			0xb9, 0x98, 0xd3, 0xe5, 0xbd, 0x7a, 0xa3, 0xc3, 0x19,
+			0x97, 0x54, 0x77, 0x97, 0xb5, 0x57, 0x93, 0x71, 0x24,
+			0x91, 0x81, 0xcf, 0x5f, 0x38, 0xb9, 0xd5, 0x7e, 0x8b,
+			0x0f, 0xea, 0x56, 0x49, 0xac, 0x25, 0xc4, 0x6c, 0x5e,
+			0x4e, 0x08, 0x31, 0xc8, 0xdf, 0x0b, 0x19, 0xb9, 0xbc,
+			0xec, 0x3b, 0x89, 0xf8, 0x98, 0xe3, 0x6c, 0xf6, 0x0f,
+			0x5b, 0x7e, 0x03, 0x10, 0x68, 0x78, 0x2c, 0xfa, 0x39,
+			0x5c, 0x64, 0x5b, 0xa1, 0xfc, 0x8d, 0x14, 0x87, 0x62,
+			0x04, 0x57, 0xc0, 0x5b, 0x6f, 0x6e, 0x3c, 0xea, 0xe6,
+			0x23, 0xde, 0x9b, 0x1d, 0x89, 0x69, 0x38, 0xfb, 0x10,
+			0xab, 0xb9, 0xda, 0xee, 0xa8, 0x8d, 0x96, 0xc0, 0xa5,
+			0x40, 0xce, 0x16, 0x1e, 0xf5, 0xf4, 0x74, 0x6f, 0x0e,
+			0x38, 0xe5, 0x1d, 0xe6, 0x6c, 0x19, 0xa5, 0x71, 0x48,
+			0xda, 0x3e, 0xe9, 0x10, 0x57, 0x16, 0x89, 0xd3, 0x5b,
+			0x4a, 0x5b, 0xa9, 0xb1, 0x13, 0xad, 0x77, 0xaf, 0x1f,
+			0x8b, 0x71, 0x3f, 0xff, 0x7c, 0xc4, 0x31, 0xb2, 0xa7,
+			0x18, 0x2f, 0x4b, 0x99, 0xe2, 0x14, 0xbf, 0x1e, 0x69,
+			0x8f, 0x56, 0xe2, 0x2c, 0x84, 0x34, 0xd6, 0x76, 0x27,
+			0x06, 0x7f, 0x2e, 0x9d, 0xbe, 0x90, 0x95, 0xda, 0x5a,
+			0xfb, 0xaa, 0x8a, 0x51, 0x3d, 0x97, 0xb8, 0x75, 0xe9,
+			0xba, 0x2a, 0x03, 0x7e, 0xb0, 0xe3, 0xdc, 0xe3, 0x3b,
+			0x2a, 0xd8, 0x77, 0xa5, 0xf3, 0x76, 0x69, 0xf1, 0xd2,
+			0xd5, 0xb3, 0xba, 0xfa, 0x97, 0x55, 0x5f, 0xd1, 0xc3,
+			0x10, 0x55, 0x9a, 0xb6, 0xce, 0x32, 0x80, 0x7f, 0x48,
+			0xb7, 0x15, 0x85, 0x3f, 0x22, 0x58, 0x00, 0x51, 0x12,
+			0x5c, 0xcc, 0xd0, 0x77, 0x26, 0x57, 0x4b, 0xbb, 0x67,
+			0x21, 0x0d, 0x92, 0x80, 0xe0, 0xec, 0x2f, 0x58, 0x44,
+			0x8b, 0x98, 0x70, 0x16, 0x13, 0x91, 0x38, 0xee, 0x9e,
+			0x4f, 0xf2, 0x6f, 0x95, 0x49, 0xe8, 0xcc, 0xc5, 0x6d,
+			0x20, 0x20, 0x4d, 0x1b, 0x08, 0xf4, 0xa5, 0x8b, 0xf3,
+			0xf6, 0x89, 0x2c, 0x17, 0xa6, 0xa8, 0xa8, 0xfb, 0xab,
+			0x6f, 0xa0, 0xe3, 0xcd, 0xc5, 0xfb, 0xac, 0x91, 0x42,
+			0x8c, 0xf2, 0x28, 0xd1, 0x13, 0xdb, 0x63, 0x59, 0x69,
+			0xab, 0x16, 0x5a, 0xaf, 0x0a, 0xfa, 0x43, 0x6d, 0x53,
+			0x7f, 0x14, 0x88, 0xc8, 0xa1, 0x71, 0x45, 0x54, 0x30,
+			0xa7, 0x91, 0x55, 0x44, 0x07, 0x03, 0x78, 0xf3, 0xf6,
+			0x23, 0xc0, 0x21, 0x4b, 0x39, 0xc4, 0x9a, 0x5a, 0xe8,
+			0xad, 0x28, 0x6b, 0xd2, 0xd7, 0xc9, 0xed, 0xf4, 0x30,
+			0xff, 0xf8, 0xc4, 0x7a, 0xfe, 0x49, 0x1c, 0x28, 0xbb,
+			0x89, 0xfa, 0x9d, 0x64, 0xcc, 0x37, 0x76, 0x6a, 0xf3,
+			0x8b, 0x94, 0x60, 0xe0, 0xb2, 0x29, 0xf8, 0x5a, 0xc6,
+			0x3e, 0xcc, 0x6f, 0x13, 0x85, 0x7c, 0xaa, 0xc4, 0x35,
+			0x56, 0x9a, 0xb6, 0x96, 0x76, 0x41, 0xe2, 0x11, 0xbe,
+			0x07, 0xba, 0x0c, 0x21, 0x32, 0xc3, 0xb3, 0xf3, 0x96,
+			0x49, 0xa2, 0xc4, 0x49, 0x48, 0xfe, 0x17, 0xef, 0xdf,
+			0x1e, 0xf3, 0xde, 0x5f, 0xa0, 0x49, 0xfc, 0xa5, 0xb5,
+			0xf3, 0x4c, 0x99, 0x5f, 0xcb, 0x33, 0xc6, 0x54, 0x52,
+			0xa0, 0x85, 0x16, 0x75, 0x2c, 0x5f, 0x87, 0x61, 0x58,
+			0xc9, 0x23, 0x0a, 0x36, 0x66, 0x3b, 0x78, 0x65, 0xcc,
+			0xd4, 0x8e, 0xe1, 0x27, 0x9a, 0x74, 0x37, 0x79, 0x4c,
+			0x5c, 0x35, 0xf9, 0xa5, 0x7f, 0x79, 0x35, 0xa8, 0x7d,
+			0xcf, 0x64, 0xa3, 0x44, 0xb9, 0xee, 0x1e, 0x07, 0x5c,
+			0xda, 0xcd, 0x93, 0xfc, 0x33, 0x77, 0x68, 0xfc, 0xd6,
+			0xea, 0xf4, 0xcd, 0x0a, 0x3a, 0xe3, 0xb3, 0xef, 0xf9,
+			0xb5, 0x1a, 0x4b, 0x50, 0x2e, 0xcb, 0x88, 0x48, 0x60,
+			0xc3, 0x5c, 0x1b, 0xb7, 0xcc, 0x7a, 0x49, 0x2e, 0xf0,
+			0x9e, 0x96, 0x47, 0x7d, 0x82, 0x85, 0xcb, 0x37, 0x85,
+			0x26, 0xfd, 0xe1, 0x13, 0x57, 0xad, 0x51, 0xdc, 0x02,
+			0x8d, 0x3c, 0x4e, 0xdd, 0xf3, 0xdf, 0xbe, 0xab, 0x0d,
+			0x99, 0xf8, 0x13, 0x37, 0x0b, 0x1b, 0xaf, 0x08, 0x9b,
+			0x38, 0x5b, 0x3a, 0x91, 0x71, 0x9a, 0xe6, 0xc1, 0x7d,
+			0xd4, 0xd8, 0xf2, 0x66, 0x57, 0x99, 0x29, 0x58, 0xb5,
+			0x9d, 0x8f, 0x86, 0x7f, 0xe9, 0x17, 0xff, 0x3c, 0xfd,
+			0x0a, 0xfd, 0x15, 0x5c, 0x2f, 0x77, 0x6c, 0x2b, 0x86,
+			0x72, 0xcb, 0x00, 0x4d, 0x31, 0x98, 0xfe, 0x28, 0xc5,
+			0x54, 0x36, 0x97, 0x8c, 0xc9, 0x31, 0xae, 0xff, 0xfc,
+			0xa6, 0x15, 0xf9, 0x9d, 0x49, 0x83, 0x47, 0xb4, 0xe5,
+			0x8c, 0x55, 0x2f, 0x35, 0x6d, 0x7c, 0x47, 0xb9, 0xeb,
+			0x0a, 0x64, 0xea, 0x22, 0xae, 0xe3, 0x6b, 0xae, 0x0e,
+			0xf8, 0xc4, 0x64, 0x18, 0x56, 0xa7, 0x78, 0x92, 0x20,
+			0xe0, 0x03, 0x93, 0x42, 0x1f, 0x1a, 0x48, 0xa1, 0x11,
+			0xdd, 0x83, 0x99, 0x67, 0xc5, 0xf2, 0x0a, 0x9e, 0xfa,
+			0xab, 0x68, 0x54, 0x1d, 0xa7, 0xe8, 0x44, 0x5f, 0x46,
+			0x23, 0xea, 0xdd, 0xb2, 0xf5, 0x83, 0x76, 0x84, 0x13,
+			0xd6, 0xe4, 0xb2, 0x59, 0x92, 0x92, 0xf4, 0xb5, 0x56,
+			0xd7, 0x29, 0xc7, 0x05, 0x00, 0x79, 0x17, 0x59, 0x20,
+			0xc6, 0x6a, 0xdc, 0x45, 0x93, 0xe7, 0xfc, 0x29, 0x37,
+			0xa7, 0x7e, 0xbc, 0x21, 0xe7, 0xe1, 0xff, 0x18, 0x54,
+			0xf9, 0xd3, 0x0c, 0x78, 0xe1, 0xc9, 0x40, 0x36, 0x36,
+			0x68, 0x46, 0x3c, 0x9a, 0x4d, 0xa6, 0xbc, 0xc3, 0xc7,
+			0xc9, 0x8c, 0x7e, 0xc6, 0x07, 0x2a, 0xd0, 0x4d, 0x45,
+			0x3d, 0x65, 0xef, 0x56, 0x7f, 0xf1, 0xcd, 0x6d, 0x97,
+			0x0c, 0x11, 0x66, 0xf4, 0x27, 0xfc, 0xe3, 0xf3, 0x93,
+			0x0d, 0xa1, 0x5a, 0x95, 0x86, 0xcc, 0x0f, 0xf4, 0x97,
+			0xbc, 0x7c, 0x62, 0x78, 0xe5, 0x01, 0xe0, 0xc7, 0x0c,
+			0xe7, 0xa9, 0x84, 0xa6, 0x17, 0xaa, 0x36, 0x06, 0x2a,
+			0x0f, 0x2f, 0x7e, 0x3d, 0xe8, 0x70, 0x8d, 0x36, 0x73,
+			0x9a, 0x7f, 0x47, 0x95, 0xc5, 0x5d, 0xc5, 0xfa, 0x9c,
+			0x64, 0x52, 0xcb, 0x52, 0x74, 0xda, 0xb0, 0x1f, 0x82,
+			0x01, 0x43, 0x98, 0xa6, 0xe7, 0xf2, 0x89, 0x51, 0x92,
+			0xf8, 0x37, 0x82, 0x8e, 0x91, 0x58, 0x40, 0x2c, 0xac,
+			0xcf, 0xde, 0x92, 0x63, 0x5a, 0x64, 0xf9, 0x9c, 0xa4,
+			0x47, 0x6d, 0x13, 0x43, 0xd3, 0xa6, 0x89, 0xbb, 0x7b,
+			0x32, 0x45, 0x2d, 0x0e, 0x52, 0x25, 0x67, 0xda, 0x1d,
+			0x9e, 0x42, 0x38, 0xfc, 0x92, 0x84, 0x4d, 0x13, 0x53,
+			0xef, 0x46, 0x81, 0xdd, 0xf6, 0x79, 0x9f, 0xba, 0x45,
+			0x49, 0x4b, 0x93, 0x4a, 0x54, 0x1c, 0x72, 0xa0, 0xfd,
+			0xa1, 0x13, 0x1c, 0x5e, 0xa5, 0x4e, 0xd5, 0xdf, 0xd1,
+			0x6a, 0x40, 0x1d, 0x5e, 0x57, 0xf8, 0xe3, 0x3a, 0xe8,
+			0x35, 0xf4, 0x67, 0xa2, 0x37, 0xf2, 0xe0, 0xd1, 0xfe,
+			0x08, 0x5e, 0x5c, 0x41, 0x17, 0x87, 0x24, 0xae, 0xf3,
+			0xe2, 0xc8, 0xb4, 0xfc, 0x3b, 0xef, 0xdb, 0x5e, 0x27,
+			0x93, 0x0f, 0x01, 0xf9, 0x35, 0xcf, 0x67, 0x4f, 0x65,
+			0xde, 0xca, 0xb6, 0xf1, 0x12, 0xa0, 0xd7, 0x53, 0x09,
+			0xec, 0x83, 0x9a, 0xb6, 0x8f, 0xf6, 0x03, 0xdf, 0x92,
+			0xa9, 0xc4, 0x85, 0x55, 0x5a, 0x74, 0x66, 0x9e, 0xc0,
+			0x46, 0xc7, 0x50, 0x7b, 0xda, 0x56, 0x4a, 0xf5, 0x38,
+			0x37, 0xd5, 0x82, 0x81, 0xf3, 0x2d, 0x51, 0xa6, 0xa3,
+			0x34, 0xf9, 0x24, 0xed, 0x35, 0x80, 0x72, 0xae, 0x3d,
+			0x8b, 0xf8, 0x66, 0xb2, 0x6c, 0x3f, 0x4e, 0x2c, 0xdb,
+			0x89, 0x76, 0x95, 0x8f, 0x1a, 0xbe, 0xfc, 0x6c, 0x37,
+			0xec, 0xf9, 0x80, 0x32, 0xac, 0x94, 0x97, 0x85, 0xe4,
+			0xa5, 0x19, 0x32, 0x46, 0x1f, 0x98, 0x89, 0x4c, 0x85,
+			0xf1, 0xfc, 0xed, 0x01, 0x98, 0x4d, 0x4a, 0xd4, 0xd1,
+			0x4e, 0x82, 0x0b, 0x32, 0x0d, 0x48, 0x34, 0x64, 0x98,
+			0xea, 0x6c, 0xf9, 0x11, 0x25, 0xbc, 0x48, 0xc3, 0xf4,
+			0x96, 0xfc, 0x5e, 0x71, 0x5d, 0xfa, 0xf7, 0x9f, 0x75,
+			0x89, 0x1d, 0xe7, 0x3b, 0x97, 0x39, 0x18, 0x90, 0xca,
+			0xa5, 0x46, 0x01, 0x2a, 0x13, 0x54, 0x5c, 0x2a, 0x14,
+			0xed, 0x1b, 0x65, 0x0a, 0xb0, 0xd9, 0x95, 0x01, 0xd0,
+			0xb3, 0x78, 0xd6, 0xc4, 0x06, 0xf3, 0x7f, 0x51, 0xe0,
+			0x39, 0xed, 0x2c, 0xf3, 0xfa, 0xc3, 0xfa, 0x1d, 0x7c,
+			0x82, 0x0f, 0x45, 0xb7, 0x15, 0xcf, 0x99, 0x53, 0x59,
+			0x9d, 0x54, 0x31, 0x6a, 0xf8, 0xb8, 0x73, 0x2d, 0xd1,
+			0xab, 0x1b, 0x69, 0xc9, 0x86, 0xdc, 0x7a, 0xca, 0xb9,
+			0x6a, 0xaf, 0x1e, 0x8f, 0x83, 0xa8, 0xf0, 0xae, 0x44,
+			0x85, 0x0d, 0x1f, 0x6c, 0x3a, 0x90, 0x50, 0xef, 0x7c,
+			0x3c, 0x1c, 0xc4, 0xb8, 0x66, 0x1c, 0x14, 0xf7, 0xf2,
+			0x5f, 0x5c, 0x70, 0x28, 0xa7, 0x55, 0x93, 0xdb, 0xa0,
+			0x95, 0x9c, 0x1b, 0xb7, 0xb6, 0x44, 0x5e, 0x98, 0x88,
+			0x89, 0xb8, 0x5a, 0xee, 0xba, 0xfc, 0xc6, 0x87, 0x2a,
+			0xdb, 0xb7, 0xcf, 0xb2, 0x4b, 0x96, 0x2c, 0x91, 0x54,
+			0x34, 0x33, 0xfb, 0x57, 0xbe, 0xf5, 0x7b, 0xbe, 0x6b,
+			0xa0, 0x3b, 0x4d, 0x09, 0x18, 0x9b, 0x75, 0x2c, 0xde,
+			0x28, 0x87, 0xd3, 0x7b, 0xf5, 0xbe, 0x0d, 0x33, 0x5a,
+			0x3b, 0x92, 0x41, 0x35, 0x50, 0x65, 0xc9, 0xa9, 0xfa,
+			0xab, 0x22, 0x4b, 0x8c, 0x3a, 0xb1, 0x9e, 0x31, 0x41,
+			0xd2, 0x2f, 0xca, 0x9e, 0xd0, 0x96, 0x0c, 0x03, 0x08,
+			0xec, 0x94, 0xa5, 0xb1, 0x65, 0x2b, 0xe1, 0xe4, 0x48,
+			0x11, 0xd6, 0x8c, 0x60, 0x6d, 0xf4, 0x64, 0x89, 0xa6,
+			0x69, 0x4b, 0xa0, 0xbe, 0xae, 0xf5, 0xde, 0xba, 0x45,
+			0x12, 0xc8, 0x28, 0xe0, 0x7a, 0xb5, 0xd4, 0x75, 0x7a,
+			0x24, 0x59, 0x4f, 0xbf, 0x0a, 0xa4, 0x53, 0x9a, 0x59,
+			0xca, 0x06, 0x51, 0x6d, 0xcb, 0x93, 0x63, 0x6e, 0x24,
+			0xe2, 0x41, 0xa5, 0xa9, 0x19, 0xc3, 0x59, 0x9a, 0x0c,
+			0xb6, 0x4e, 0x65, 0x60, 0xa0, 0x49, 0xf0, 0x7f, 0x50,
+			0x96, 0xed, 0xe9, 0xfc, 0x33, 0x5b, 0x52, 0x51, 0x7e,
+			0x77, 0xd7, 0xbb, 0xaf, 0xdd, 0x12, 0x4b, 0xbc, 0xc6,
+			0xae, 0xca, 0x92, 0x3a, 0x9f, 0xc1, 0x0c, 0x8f, 0x01,
+			0xbe, 0x4c, 0x78, 0x23, 0x7c, 0x8b, 0x82, 0xb7, 0x45,
+			0x80, 0x44, 0x34, 0x9d, 0x13, 0xdb, 0xc4, 0x3f, 0x1b,
+			0x39, 0xb0, 0xee, 0xa9, 0xac, 0x5e, 0x3d, 0xc3, 0x59,
+			0x92, 0xcc, 0x6e, 0x6c, 0x97, 0xcc, 0xba, 0x15, 0xa6,
+			0x53, 0x92, 0x76, 0x35, 0x01, 0x70, 0x16, 0xde, 0xde,
+			0x56, 0xc3, 0xb3, 0x48, 0x8b, 0x4d, 0xfa, 0xa7, 0xcb,
+			0x43, 0xb5, 0x76, 0x9a, 0x56, 0x03, 0xd6, 0x41, 0xbe,
+			0x73, 0x89, 0x1f, 0x6f, 0x02, 0x3f, 0xcc, 0xaf, 0x49,
+			0xe3, 0x5e, 0xb3, 0xf4, 0xbc, 0x61, 0xf9, 0x9e, 0x9e,
+			0xec, 0x10, 0x98, 0x0d, 0x5a, 0x6d, 0xcf, 0xbf, 0xe3,
+			0xe5, 0xf5, 0x1b, 0x17, 0x46, 0x7a, 0x32, 0xc7, 0x18,
+			0x40, 0x20, 0x8e, 0x06, 0x62, 0x2d, 0x1b, 0x90, 0x54,
+			0x4f, 0x7f, 0xe4, 0x50, 0x09, 0xdc, 0x57, 0x22, 0xe9,
+			0x7c, 0x3c, 0x77, 0x57, 0xc2, 0x8b, 0xc9, 0x6a, 0x2b,
+			0x31, 0xa1, 0x7f, 0x14, 0xdf, 0x22, 0x7d, 0xed, 0xc9,
+			0x0d, 0xbb, 0x93, 0xba, 0x72, 0x85, 0x33, 0x58, 0x80,
+			0xc0, 0xf5, 0x75, 0x5d, 0x7e, 0x5f, 0xfb, 0x41, 0xbc,
+			0x5f, 0x62, 0x13, 0xb9, 0x16, 0x1e, 0xfe, 0x9e, 0x53,
+			0x53, 0xce, 0xa0, 0xfc, 0x7f, 0xf6, 0x18, 0xc2, 0x0f,
+			0xb9, 0x7f, 0xbf, 0x09, 0xc3, 0x37, 0x08, 0xb0, 0x1f,
+			0x16, 0x80, 0x5e, 0xf6, 0xcb, 0xfe, 0x22, 0x7c, 0x07,
+			0x99, 0xb3, 0x12, 0x69, 0xa4, 0x05, 0xc9, 0x29, 0xc7,
+			0xd0, 0x57, 0x05, 0x33, 0x85, 0x0c, 0xd4, 0xfe, 0xfc,
+			0x54, 0xa8, 0xc9, 0x7b, 0x92, 0x67, 0xa6, 0xa7, 0xbf,
+			0x5f, 0xfc, 0xe9, 0x65, 0x31, 0x4f, 0x85, 0x8f, 0x89,
+			0xbe, 0xa1, 0xdb, 0x6d, 0x96, 0x52, 0x09, 0x56, 0x12,
+			0xc6, 0xb2, 0x28, 0x26, 0x07, 0x39, 0x21, 0x39, 0xce,
+			0x55, 0xf9, 0x82, 0x23, 0xc2, 0x3f, 0x3a, 0x66, 0xc4,
+			0xee, 0x10, 0x4b, 0xf1, 0x1e, 0x1d, 0x59, 0xfa, 0x28,
+			0x20, 0x34, 0x26, 0x49, 0x7c, 0xc3, 0x85, 0x2f, 0x80,
+			0xbe, 0x2c, 0xbc, 0x94, 0x81, 0x74, 0x2a, 0x21, 0x49,
+			0x1d, 0xa7, 0xff, 0xf0, 0x21, 0x56, 0x98, 0x68, 0x8f,
+			0x52, 0x2d, 0xa0, 0xf2, 0x84, 0x73, 0x2f, 0xe2, 0x90,
+			0x8c, 0x56, 0xd1, 0xbf, 0x67, 0xc3, 0x19, 0x47, 0x14,
+			0x71, 0x62, 0x25, 0xdc, 0x11, 0xbc, 0x24, 0xa0, 0xb2,
+			0x42, 0x72, 0x37, 0x78, 0xe6, 0xe0, 0xf6, 0x8c, 0x66,
+			0xe5, 0x1b, 0x7a, 0x79, 0x46, 0xc0, 0x76, 0xcd, 0xa9,
+			0x3c, 0xb2, 0x17, 0x8c, 0xc9, 0xb1, 0xc4, 0x2a, 0x1f,
+			0xdb, 0xf7, 0xeb, 0x8a, 0x5d, 0x29, 0xd1, 0xd1, 0x98,
+			0x43, 0x9e, 0x22, 0xf3, 0x96, 0x1e, 0x83, 0xeb, 0x44,
+			0x08, 0xc4, 0xa0, 0xd2, 0xd3, 0x36, 0xf8, 0x94, 0xab,
+			0x3c, 0x4b, 0x68, 0xf3, 0x7c, 0x1a, 0x16, 0x1a, 0x66,
+			0x77, 0x67, 0xee, 0x64, 0xc2, 0xdb, 0x53, 0xae, 0x1f,
+			0xeb, 0x44, 0x77, 0x3b, 0x5f, 0x74, 0x48, 0xe9, 0x05,
+			0xfa, 0x3a, 0x03, 0x1b, 0x54, 0x36, 0xb0, 0x28, 0x79,
+			0x70, 0xe6, 0x2f, 0xb4, 0xf5, 0x28, 0x60, 0x1e, 0x63,
+			0xf6, 0x03, 0x09, 0x4f, 0x0f, 0x7f, 0x01, 0xe5, 0x02,
+			0x6a, 0x76, 0x96, 0xb9, 0x58, 0x39, 0xf9, 0xc4, 0x05,
+			0xac, 0x93, 0x2f, 0x30, 0xab, 0xc5, 0x2b, 0xad, 0xed,
+			0x3c, 0xb0, 0x6c, 0x66, 0x94, 0x7e, 0x79, 0x01, 0x90,
+			0xf0, 0x10, 0xb5, 0xcd, 0x6b, 0x07, 0xc2, 0xe7, 0x8a,
+			0xac, 0xf7, 0x5b, 0xb5, 0x11, 0xf8, 0x1a, 0x52, 0xc7,
+			0x5d, 0x2c, 0xc1, 0xce, 0x8f, 0x0c, 0xc1, 0x6a, 0x95,
+			0x80, 0x25, 0xf9, 0x95, 0x28, 0x18, 0x65, 0xd3, 0x15,
+			0x0e, 0xbf, 0x6f, 0xa6, 0x86, 0xd9, 0xea, 0xdc, 0xb8,
+			0x89, 0x02, 0x08, 0x6b, 0xd1, 0x96, 0xb1, 0x6f, 0xb1,
+			0x28, 0x6b, 0xc4, 0xe3, 0xa8, 0x53, 0xd3, 0xd0, 0x67,
+			0x00, 0x7c, 0xa1, 0x0a, 0xd1, 0x9a, 0x89, 0x12, 0xc9,
+			0xa7, 0x3f, 0x03, 0x03, 0xad, 0x0f, 0x34, 0x2c, 0xd8,
+			0xdf, 0x70, 0xed, 0x38, 0xf4, 0xb1, 0xf7, 0xb1, 0x72,
+			0x5e, 0x2d, 0xa6, 0xb7, 0x8d, 0x9c, 0x7f, 0x96, 0x73,
+			0xc8, 0x47, 0x46, 0x20, 0x2f, 0x44, 0x2c, 0x2b, 0xa2,
+			0x30, 0xe4, 0x81, 0x91, 0x7e, 0xce, 0x4c, 0x8b, 0xcc,
+			0x5f, 0xfd, 0xff, 0x2f, 0x94, 0xbd, 0xe5, 0xaf, 0x09,
+			0xcd, 0xb6, 0xbc, 0x65, 0x83, 0xc3, 0xcd, 0x79, 0xc4,
+			0x68, 0xda, 0x46, 0x36, 0x97, 0x52, 0x35, 0x90, 0x4f,
+			0x1d, 0x8d, 0x66, 0x1f, 0xea, 0x5d, 0xef, 0xdf, 0x8c,
+			0xa8, 0x6b, 0xb0, 0xdc, 0x1a, 0x1e, 0x79, 0x43, 0xe9,
+			0x29, 0x71, 0x11, 0xbd, 0xe5, 0xd6, 0x71, 0xf4, 0xa7,
+			0xf8, 0x64, 0x40, 0x73, 0x46, 0x5b, 0x2a, 0xb5, 0x9a,
+			0xa1, 0x9a, 0x64, 0x4c, 0x5e, 0xa5, 0x8d, 0xfd, 0x49,
+			0x91, 0xba, 0xe3, 0xa7, 0x19, 0xac, 0x2b, 0xe9, 0xea,
+			0xf1, 0x1c, 0x85, 0x91, 0xea, 0xfb, 0xc7, 0x5c, 0x46,
+			0x7e, 0xa8, 0x0a, 0x92, 0xbf, 0x61, 0x2d, 0xc2, 0x20,
+			0x7e, 0xf7, 0x0d, 0xb8, 0x65, 0x41, 0x48, 0x39, 0xb9,
+			0xb0, 0xd4, 0xc1, 0x84, 0xee, 0x21, 0xba, 0x9c, 0x74,
+			0xf3, 0x6d, 0x0a, 0x33, 0x87, 0xab, 0x2e, 0x15, 0x1b,
+			0xad, 0xac, 0x98, 0xae, 0x7c, 0xd8, 0xae, 0x03, 0x50,
+			0xfa, 0xae, 0xfe, 0x8c, 0x6d, 0x51, 0x67, 0x76, 0x70,
+			0xa6, 0x2f, 0x1e, 0x6a, 0x4f, 0xf0, 0xbe, 0x51, 0xbb,
+			0xa6, 0x20, 0x1f, 0x56, 0x72, 0x34, 0xd0, 0x13, 0x6e,
+			0x0f, 0x1a, 0xcb, 0x34, 0xe1, 0xfb, 0x3c, 0xf1, 0x43,
+			0x1b, 0x73, 0x40, 0xa0, 0x1f, 0x5d, 0x40, 0x51, 0x9f,
+			0x97, 0x56, 0x1f, 0x26, 0x54, 0x03, 0xb8, 0x2e, 0xfd,
+			0xc7, 0x52, 0xea, 0x85, 0x5d, 0x45, 0x71, 0xa3, 0x5c,
+			0x86, 0x23, 0xcb, 0x08, 0xc5, 0xd8, 0x21, 0xfd, 0x10,
+			0x49, 0xac, 0xaa, 0xd5, 0x97, 0x37, 0xbb, 0xad, 0xd5,
+			0x84, 0xc2, 0x65, 0xeb, 0xdd, 0x92, 0x7c, 0x50, 0x64,
+			0x4f, 0x08, 0x32, 0x56, 0x81, 0x39, 0xa4, 0xfc, 0x3f,
+			0x04, 0xbb, 0xde, 0xe9, 0xcd, 0xb6, 0xbb, 0x29, 0x6b,
+			0x75, 0x64, 0x32, 0x5f, 0xca, 0xd0, 0x3a, 0x19, 0x3d,
+			0x01, 0x8c, 0x0b, 0xd5, 0x76, 0xe6, 0x97, 0x5d, 0x73,
+			0x7b, 0xb3, 0xcd, 0xdf, 0xbc, 0x29, 0xda, 0xa5, 0x22,
+			0xfb, 0x1d, 0xf6, 0xf1, 0x50, 0x71, 0x50, 0xd3, 0xf8,
+			0x0a, 0xbf, 0x37, 0x28, 0xca, 0x3b, 0xab, 0xec, 0xf5,
+			0xee, 0x2d, 0x1e, 0x21, 0x35, 0x39, 0x56, 0x55, 0x3c,
+			0x9e, 0x5a, 0x58, 0x94, 0x85, 0xa7, 0xba, 0x0d, 0xa4,
+			0x0b, 0x4c, 0x73, 0xb2, 0x56, 0x68, 0x5a, 0x5d, 0x6c,
+			0x18, 0xce, 0xc5, 0x25, 0x14, 0xc9, 0xb3, 0xc9, 0x94,
+			0x57, 0x62, 0x95, 0xd5, 0x1c, 0xf6, 0x93, 0x12, 0x45,
+			0x28, 0xa1, 0x7e, 0x96, 0x53, 0x68, 0xf5, 0x6a, 0xca,
+			0xbe, 0x04, 0x94, 0x97, 0xcf, 0xb5, 0xcf, 0x70, 0xe1,
+			0x7e, 0x15, 0x2d, 0x6a, 0x67, 0x0c, 0x86, 0x07, 0xd1,
+			0xe2, 0x95, 0xc3, 0xd8, 0x22, 0xdf, 0x6a, 0x88, 0xc1,
+			0xad, 0x83, 0x57, 0x30, 0xd0, 0x59, 0xaf, 0xf4, 0x9f,
+			0x8d, 0x89, 0x7d, 0x0b, 0xee, 0xb7, 0x1c, 0xaa, 0xd2,
+			0x4c, 0x7a, 0xa4, 0xfb, 0x99, 0xe0, 0x0a, 0xd7, 0x4d,
+			0xc4, 0xe9, 0x88, 0x18, 0x27, 0x1d, 0xb2, 0xe9, 0x43,
+			0x16, 0x25, 0x29, 0x04, 0x98, 0x81, 0xfd, 0xaf, 0xdd,
+			0x3f, 0xf1, 0x61, 0x32, 0x20, 0x51, 0x96, 0xe4, 0xd2,
+			0xab, 0xc4, 0x00, 0x33, 0xb4, 0x7b, 0x3c, 0xc2, 0x2d,
+			0x61, 0x11, 0x4d, 0x72, 0x16, 0x6d, 0xfb, 0x24, 0x67,
+			0x7c, 0xea, 0xf0, 0x79, 0xa3, 0x50, 0x93, 0xe3, 0x9b,
+			0x7f, 0x89, 0xf0, 0xa6, 0x8c, 0xb0, 0xaf, 0x1c, 0xe0,
+			0x91, 0xce, 0x3f, 0xe5, 0x43, 0x90, 0x41, 0x95, 0xc7,
+			0x1f, 0x8f, 0x0a, 0xb2, 0x1e, 0x84, 0x3e, 0x86, 0xf0,
+			0x81, 0x87, 0x8b, 0x04, 0x52, 0xa6, 0x46, 0xb0, 0xe4,
+			0xf4, 0x5b, 0x97, 0x7b, 0x7b, 0x32, 0xb6, 0x17, 0x96,
+			0x48, 0x12, 0xb6, 0x31, 0xf1, 0x5c, 0xd4, 0x8b, 0x93,
+			0x60, 0xc4, 0x00, 0xea, 0x12, 0xe1, 0x93, 0x6f, 0x53,
+			0x82, 0x2a, 0x48, 0x1d, 0xc1, 0x91, 0x79, 0x7c, 0x30,
+			0x43, 0xe5, 0x1e, 0x10, 0x7e, 0x7f, 0xbc, 0x18, 0x60,
+			0x03, 0x2a, 0xe5, 0xbb, 0x92, 0x7f, 0x24, 0x54, 0x12,
+			0x21, 0x77, 0x89, 0xc9, 0x22, 0xfb, 0x57, 0xaa, 0x69,
+			0xdd, 0x69, 0x06, 0xb0, 0xe5, 0x5b, 0x74, 0x1f, 0x02,
+			0x99, 0x62, 0x56, 0x21, 0x29, 0xd2, 0x26, 0x0f, 0x6a,
+			0x3a, 0x05, 0x96, 0x92, 0xb9, 0xff, 0xe9, 0x3b, 0x24,
+			0x70, 0xd4, 0x13, 0xa4, 0xb2, 0x6c, 0x76, 0xe3, 0x55,
+			0xe5, 0xc8, 0x18, 0x66, 0x60, 0x91, 0xba, 0x98, 0x5d,
+			0x67, 0x13, 0x97, 0xce, 0x49, 0xaa, 0x52, 0x7b, 0x20,
+			0x65, 0x65, 0x0a, 0x4f, 0x7b, 0x0a, 0x26, 0x62, 0xa7,
+			0xb5, 0xab, 0x2d, 0x5d, 0x37, 0xad, 0xb8, 0x97, 0xcf,
+			0xf5, 0x4f, 0xef, 0x03, 0xed, 0xce, 0x73, 0x68, 0x5d,
+			0x9f, 0x6c, 0x3d, 0x3f, 0xd5, 0xc1, 0xca, 0x5f, 0xef,
+			0xe4, 0xf8, 0xfb, 0xc7, 0xbb, 0x49, 0x30, 0x1a, 0xe1,
+			0x3c, 0xd1, 0x7f, 0x7c, 0x26, 0xa2, 0x87, 0x6e, 0x80,
+			0xa6, 0x47, 0x2c, 0x0f, 0xa6, 0x68, 0xf6, 0xde, 0x65,
+			0x1f, 0x02, 0xa7, 0x09, 0x08, 0x34, 0xbe, 0x2c, 0xc7,
+			0xed, 0x4c, 0xb0, 0xa1, 0x63, 0x63, 0x35, 0xe0, 0x11,
+			0xfd, 0xe1, 0x2d, 0xb4, 0xcf, 0x9e, 0x18, 0x78, 0xa8,
+			0x63, 0xbf, 0xb7, 0xe8, 0x22, 0x82, 0xc5, 0x9c, 0x0b,
+			0x3a, 0x1c, 0x2f, 0x73, 0x80, 0xf1, 0xbb, 0x9a, 0x84,
+			0x51, 0x2c, 0x1c, 0x2e, 0xda, 0xb8, 0xa3, 0x43, 0x4f,
+			0x95, 0x8c, 0x06, 0x12, 0xdc, 0x75, 0x53, 0x01, 0xc4,
+			0x50, 0x95, 0x31, 0x7e, 0x75, 0x33, 0x69, 0x3e, 0x82,
+			0xad, 0x51, 0x1a, 0x4d, 0xa4, 0xe7, 0xbb, 0x7c, 0x9e,
+			0x3c, 0xbc, 0x1a, 0xf7, 0x7d, 0x0f, 0xdd, 0x93, 0xb5,
+			0x91, 0x61, 0x28, 0x28, 0x3b, 0x8f, 0xdd, 0xfa, 0xe6,
+			0xf4, 0xcc, 0x21, 0x82, 0x0c, 0xe2, 0xe4, 0xc7, 0x39,
+			0x69, 0x8e, 0xd6, 0xc0, 0x45, 0x94, 0x26, 0x27, 0x1b,
+			0xca, 0x02, 0x39, 0x00, 0x12, 0xef, 0xad, 0x0f, 0xf2,
+			0xdb, 0x19, 0x39, 0x6e, 0x4e, 0xfd, 0x14, 0x56, 0x3c,
+			0xd3, 0xfb, 0x43, 0x2d, 0x14, 0xb1, 0x62, 0xd2, 0xc8,
+			0x09, 0xe3, 0xb4, 0xf3, 0x24, 0x4e, 0xeb, 0xe5, 0xd6,
+			0x42, 0x48, 0xa5, 0xf8, 0xd8, 0xcc, 0x0b, 0xd6, 0x23,
+			0x59, 0x46, 0xaf, 0x15, 0xb2, 0x36, 0x2d, 0x52, 0xbe,
+			0x4a, 0x5f, 0xc6, 0xb2, 0x3d, 0xc5, 0x53, 0x57, 0x35,
+			0x2a, 0xa1, 0x06, 0xc8, 0x93, 0xa8, 0x3e, 0xc6, 0xcb,
+			0xa4, 0xd9, 0xe4, 0x53, 0xa9, 0xe8, 0x6d, 0xe1, 0x73,
+			0x97, 0xb9, 0x6f, 0xb3, 0xf8, 0x1d, 0xb4, 0xae, 0xd8,
+			0x77, 0xa7, 0xef, 0xe1, 0xeb, 0x83, 0x3d, 0xc3, 0x35,
+			0xf0, 0xc6, 0xd8, 0x66, 0xb5, 0x29, 0x7f, 0x74, 0xdb,
+			0xa1, 0xf2, 0x16, 0xc9, 0x07, 0x0f, 0x5c, 0x3e, 0x18,
+			0x64, 0x82, 0xfd, 0xae, 0x80, 0xca, 0xcf, 0xf9, 0x7e,
+			0x7f, 0x36, 0x58, 0xd5, 0x4d, 0x2d, 0x5a, 0x27, 0x27,
+			0xcc, 0x7e, 0x45, 0x0d, 0xf8, 0xd6, 0x5f, 0xee, 0x83,
+			0x42, 0x83, 0x48, 0x0b, 0x64, 0xc6, 0x03, 0x4d, 0x1e,
+			0x24, 0x90, 0xba, 0x2a, 0x92, 0x7d, 0x43, 0x77, 0x17,
+			0xc8, 0xcc, 0x9f, 0x31, 0x80, 0x4e, 0x19, 0x7f, 0x8f,
+			0xcf, 0xd7, 0x27, 0x48, 0x10, 0xea, 0x3d, 0xd6, 0x0f,
+			0x70, 0x7a, 0xb9, 0xed, 0x53, 0x67, 0x1c, 0x56, 0xb1,
+			0x78, 0xfc, 0x0c, 0xcc, 0x1f, 0xf8, 0x89, 0x45, 0x57,
+			0x4a, 0x72, 0xd2, 0xa0, 0x07, 0x53, 0x73, 0x3d, 0x0e,
+			0x45, 0x5a, 0x31, 0x94, 0x8f, 0x5f, 0x11, 0xe4, 0x36,
+			0xe2, 0x19, 0xc4, 0x84, 0x94, 0xb9, 0xc0, 0xe6, 0x5d,
+			0x66, 0x7e, 0x4c, 0x05, 0xc6, 0xcd, 0x9c, 0x6c, 0x8f,
+			0x32, 0x68, 0xfb, 0x66, 0x86, 0x22, 0xf5, 0xfa, 0x9a,
+			0x46, 0x26, 0xb5, 0xd5, 0xc1, 0x7d, 0xeb, 0x7b, 0x82,
+			0x3e, 0x62, 0x8c, 0x76, 0x0b, 0x08, 0x1d, 0xe0, 0x75,
+			0x34, 0x30, 0x4b, 0xc2, 0x4b, 0xcb, 0x49, 0xa3, 0x65,
+			0x0e, 0x90, 0xd3, 0x57, 0xe5, 0xf5, 0x83, 0x1b, 0xa2,
+			0xbb, 0x4a, 0x23, 0x63, 0x25, 0x64, 0xd7, 0xf1, 0x85,
+			0x9b, 0xbe, 0x2e, 0xda, 0xad, 0xc8, 0x60, 0x37, 0x3c,
+			0x78, 0x33, 0xb5, 0x76, 0x7c, 0xc6, 0xe5, 0x97, 0x86,
+			0x6d, 0x70, 0xdc, 0x4f, 0x79, 0x39, 0xe6, 0x27, 0x78,
+			0x57, 0x98, 0x69, 0x79, 0xaf, 0x0a, 0xc0, 0x64, 0x0c,
+			0x1a, 0x09, 0xa0, 0x98, 0x06, 0x41, 0x5c, 0x29, 0x27,
+			0xbd, 0x2e, 0xbe, 0xe8, 0x08, 0x5f, 0xdf, 0xed, 0xd9,
+			0x31, 0x97, 0xdb, 0x0a, 0x1f, 0x4e, 0x1b, 0x67, 0x32,
+			0xef, 0x84, 0xa2, 0x14, 0xa7, 0x83, 0x33, 0xdc, 0x40,
+			0x55, 0x76, 0x27, 0x01, 0x9f, 0x98, 0x90, 0xa9, 0xfa,
+			0xfb, 0x73, 0x86, 0x38, 0x6b, 0xa8, 0x57, 0xc9, 0xaa,
+			0x06, 0xbf, 0x84, 0x97, 0xfa, 0xc8, 0xde, 0x67, 0x6b,
+			0xa6, 0x2c, 0xa1, 0x90, 0x6c, 0x6d, 0xcd, 0xb5, 0x73,
+			0x3c, 0x16, 0x68, 0xb4, 0x00, 0x55, 0xf6, 0xd8, 0x35,
+			0xb0, 0xf9, 0x8e, 0x2e, 0xfb, 0x54, 0xc1, 0x8e, 0x51,
+			0x90, 0x2d, 0x28, 0x83, 0x5d, 0x81, 0xd6, 0x00, 0x00,
+			0x41, 0xf2, 0x97, 0xdf, 0x2f, 0x8d, 0xa8, 0xc2, 0x1d,
+			0xe3, 0xbd, 0x1d, 0x2a, 0x6f, 0x4a, 0xfd, 0xd3, 0xcd,
+			0x0b, 0xac, 0x28, 0x93, 0x87, 0x3e, 0xd9, 0x7b, 0x17,
+			0xe2, 0x66, 0x0a, 0x6e, 0xcb, 0xa8, 0x3e, 0x3d, 0x9b,
+			0x3c, 0xbe, 0x78, 0xe1, 0x2a, 0xa5, 0x42, 0x5f, 0xdc,
+			0xe0, 0x0e, 0x06, 0x10, 0x59, 0xac, 0x4a, 0xd3, 0x63,
+			0x95, 0x92, 0x31, 0x1c, 0x7d, 0xb1, 0x5e, 0x48, 0x01,
+			0x23, 0x16, 0x61, 0xb3, 0xfc, 0xe6, 0x9a, 0x5d, 0x39,
+			0xf9, 0x2f, 0xb5, 0xc6, 0x6d, 0xeb, 0xb2, 0x6e, 0x93,
+			0x9d, 0x8b, 0x19, 0x74, 0xa9, 0xc4, 0x57, 0x96, 0xbc,
+			0x1c, 0xbb, 0x88, 0x5a, 0xd6, 0xf7, 0xc0, 0xe2, 0xb6,
+			0xf2, 0xb3, 0xf0, 0x56, 0x5b, 0x2c, 0x9e, 0xdf, 0x96,
+			0x6d, 0xcc, 0x41, 0x06, 0xce, 0x1e, 0x1b, 0x3a, 0x39,
+			0x67, 0xe1, 0x7e, 0x81, 0xfc, 0xff, 0x0a, 0xc0, 0xf1,
+			0x76, 0x82, 0x49, 0x76, 0x88, 0x38, 0x10, 0x5c, 0x90,
+			0x83, 0x4b, 0x3e, 0x8d, 0xb9, 0x35, 0x77, 0xf5, 0x4c,
+			0x7e, 0x3e, 0x12, 0x54, 0xf0, 0x33, 0x56, 0xbc, 0x9b,
+			0x0e, 0x10, 0x7d, 0x4a, 0xb6, 0x2d, 0x02, 0x35, 0x79,
+			0xe8, 0x0c, 0x15, 0xc1, 0xde, 0xc7, 0x2a, 0x7d, 0xe1,
+			0x27, 0xd4, 0x99, 0x50, 0x6b, 0xde, 0x6d, 0x02, 0x65,
+			0x12, 0x88, 0xe6, 0x80, 0x13, 0x5e, 0xad, 0xb1, 0xef,
+			0xe7, 0x77, 0x19, 0xc9, 0x63, 0xcf, 0x14, 0x41, 0x40,
+			0x76, 0xc8, 0xa9, 0xc3, 0x08, 0x88, 0xdb, 0x44, 0x6c,
+			0x10, 0x74, 0x5c, 0xf6, 0x86, 0x8c, 0x61, 0x95, 0x19,
+			0x76, 0x1f, 0x3e, 0xcb, 0xdf, 0x6e, 0x08, 0x9f, 0x43,
+			0xb9, 0x66, 0x2b, 0x88, 0x27, 0x48, 0xbe, 0x5d, 0x9e,
+			0x83, 0x72, 0x9a, 0x43, 0x06, 0x8e, 0xf3, 0x9e, 0xba,
+			0xd5, 0xd8, 0x70, 0x1b, 0xe4, 0x4e, 0x39, 0x0a, 0x37,
+			0x5b, 0x2f, 0x8c, 0xa8, 0xbd, 0x3c, 0xfe, 0xb2, 0x59,
+			0x5b, 0x5f, 0x5a, 0x05, 0x67, 0xc8, 0xec, 0x71, 0x5d,
+			0xb5, 0x77, 0xc7, 0xb3, 0x1c, 0xb7, 0xac, 0x90, 0xf6,
+			0xd3, 0xb7, 0x61, 0x65, 0xbe, 0x29, 0x8f, 0x27, 0x65,
+			0x9a, 0x9a, 0x1c, 0x00, 0x82, 0xee, 0x10, 0xb9, 0x60,
+			0xff, 0x9f, 0x5a, 0xbe, 0x09, 0x72, 0x63, 0x79, 0x60,
+			0x50, 0x84, 0x18, 0xc9, 0xc5, 0x0d, 0x43, 0xce, 0x29,
+			0x80, 0x72, 0x83, 0x12, 0x73, 0x13, 0x44, 0xec, 0x25,
+			0xe5, 0x15, 0x70, 0xe2, 0x7f, 0x6b, 0x54, 0x5b, 0x58,
+			0x57, 0x06, 0xdc, 0xae, 0x1f, 0x9c, 0x38, 0x4c, 0xbf,
+			0x1c, 0x42, 0xb2, 0x63, 0x9c, 0xfb, 0xae, 0xc0, 0xdf,
+			0xd3, 0xcc, 0xcc, 0xb9, 0x9b, 0xe9, 0x66, 0x77, 0x70,
+			0x47, 0x6c, 0x8b, 0x5b, 0xc1, 0x87, 0x7a, 0x63, 0xa3,
+			0x13, 0xfc, 0x2d, 0x16, 0xd3, 0xe2, 0x00, 0xe9, 0x94,
+			0x11, 0xf2, 0x28, 0xda, 0x0b, 0x58, 0x09, 0x2d, 0x59,
+			0x2d, 0xb5, 0x25, 0x99, 0xca, 0x63, 0xc9, 0x96, 0xa3,
+			0x17, 0x2a, 0x0e, 0x22, 0x2c, 0x5f, 0x8d, 0x53, 0xc4,
+			0xd1, 0x3d, 0x4e, 0x20, 0xdc, 0x44, 0x18, 0xd2, 0xd8,
+			0x5c, 0x9a, 0x78, 0x0a, 0x53, 0xa8, 0x73, 0x7d, 0x98,
+			0xc0, 0xba, 0xfb, 0xfe, 0x27, 0xd4, 0x82, 0x87, 0xd4,
+			0x4d, 0xc0, 0xe3, 0xd1, 0xd0, 0xff, 0xf0, 0xd3, 0x04,
+			0xfd, 0x11, 0xf4, 0x57, 0xc2, 0xcc, 0xcf, 0xa9, 0xe6,
+			0xc5, 0x09, 0x4d, 0xcc, 0x36, 0x69, 0x26, 0x35, 0x8c,
+			0xe8, 0xe2, 0xe2, 0xdd, 0x5a, 0x5f, 0xaf, 0xda, 0x38,
+			0x07, 0x33, 0xfb, 0xf8, 0xfb, 0xf9, 0x62, 0x4a, 0x75,
+			0xad, 0xc3, 0x95, 0xd3, 0x6e, 0x5b, 0xe3, 0x10, 0x2f,
+			0x31, 0xaf, 0xcd, 0x57, 0xb9, 0x84, 0x3d, 0xa4, 0x66,
+			0x3f, 0x10, 0xeb, 0xa2, 0xe2, 0xfa, 0xc7, 0xd3, 0x91,
+			0x83, 0x48, 0xbb, 0x3a, 0x6c, 0xe4, 0x12, 0x1c, 0x36,
+			0x44, 0xc5, 0x07, 0x07, 0x9b, 0x9d, 0x51, 0x3e, 0x74,
+			0xc5, 0x35, 0x8a, 0x10, 0x9c, 0x0d, 0x81, 0xf3, 0x10,
+			0x0f, 0x29, 0xaf, 0x01, 0x5d, 0x6f, 0x7f, 0x50, 0x9a,
+			0xf8, 0xbf, 0x1c, 0x57, 0xfc, 0x25, 0x5c, 0x02, 0x15,
+			0x43, 0x0b, 0x60, 0x21, 0xf3, 0xb1, 0x43, 0x9a, 0x41,
+			0xa0, 0xc1, 0x1b, 0x7a, 0x88, 0x97, 0xfd, 0x27, 0x56,
+			0xf7, 0x48, 0x1f, 0xb6, 0x9a, 0xa2, 0xb1, 0xc5, 0xab,
+			0x30, 0xa1, 0x14, 0x91, 0xef, 0x14, 0xe3, 0x01, 0x73,
+			0xf3, 0x8d, 0xfa, 0xd7, 0xae, 0x2f, 0x13, 0x5d, 0x66,
+			0xfb, 0x1c, 0x4b, 0x2e, 0x94, 0xdb, 0x9f, 0x10, 0x5b,
+			0x2a, 0x5e, 0x85, 0x98, 0x73, 0x9d, 0xcd, 0x7c, 0xa0,
+			0x79, 0xb6, 0xef, 0x8e, 0xf8, 0xdd, 0x11, 0x32, 0x0d,
+			0xec, 0x6a, 0x04, 0xea, 0x81, 0xd7, 0x63, 0xbd, 0x01,
+			0x77, 0x3e, 0x52, 0x30, 0x88, 0x3a, 0x9e, 0x04, 0xdf,
+			0x42, 0x8b, 0x40, 0x23, 0xa9, 0x5a, 0x97, 0x21, 0x55,
+			0xce, 0x72, 0x21, 0x45, 0x40, 0x01, 0x66, 0x63, 0x1d,
+			0xab, 0x37, 0x8d, 0xd5, 0x75, 0x56, 0xc2, 0x7c, 0xa0,
+			0x9e, 0xa5, 0x85, 0xc2, 0x0b, 0x51, 0x80, 0xae, 0x0c,
+			0xb3, 0xde, 0xa4, 0xb5, 0xa3, 0xa3, 0x00, 0xe1, 0xc9,
+			0x09, 0xee, 0xab, 0xbb, 0x36, 0x78, 0x5b, 0x3f, 0x45,
+			0x3d, 0xf6, 0xe8, 0x7e, 0xa3, 0xd7, 0x24, 0x78, 0xdf,
+			0xbc, 0x9f, 0xe4, 0x4c, 0x4d, 0x22, 0xb8, 0x76, 0xb0,
+			0x6a, 0xc1, 0x35, 0x66, 0xfe, 0xaf, 0xf1, 0xb4, 0x44,
+			0x5e, 0xe0, 0xff, 0xcf, 0x80, 0xf8, 0xa8, 0x5f, 0x59,
+			0x91, 0xa4, 0x4f, 0xc0, 0x58, 0xb1, 0x2d, 0xc9, 0x6a,
+			0xcd, 0x62, 0xcb, 0x41, 0x89, 0x9d, 0x3b, 0x3e, 0x48,
+			0x5f, 0x23, 0x46, 0x21, 0x8a, 0x49, 0xc0, 0x16, 0x07,
+			0x4d, 0xd9, 0x26, 0xbe, 0x86, 0xfb, 0x06, 0xbd, 0x41,
+			0xc5, 0xd8, 0x87, 0xf6, 0x94, 0xb1, 0xef, 0xed, 0xab,
+			0xfb, 0x93, 0x48, 0x5a, 0xe7, 0x22, 0xbc, 0x2b, 0x24,
+			0xb4, 0x86, 0x33, 0x9b, 0xd6, 0x09, 0x46, 0x85, 0x81,
+			0x89, 0xa9, 0x30, 0x13, 0xfa, 0x08, 0x55, 0xfc, 0xe5,
+			0xfb, 0xc8, 0xb2, 0xa8, 0x31, 0x4c, 0x9e, 0x1c, 0xff,
+			0x80, 0x10, 0x77, 0x5f, 0xf1, 0x07, 0x14, 0xf7, 0x88,
+			0xb9, 0xeb, 0x48, 0xfe, 0x18, 0x48, 0xf3, 0xaf, 0xfd,
+			0x7d, 0x0c, 0x9f, 0x30, 0x11, 0xcb, 0xed, 0xfc, 0x31,
+			0xf5, 0x6e, 0x72, 0xfa, 0x9a, 0x3a, 0xa5, 0x1d, 0x89,
+			0x89, 0x8b, 0xc6, 0x2c, 0xaf, 0xa6, 0xef, 0xbd, 0xb4,
+			0xc2, 0xa0, 0x97, 0x74, 0xd0, 0x4d, 0x72, 0xb7, 0xe8,
+			0x24, 0xba, 0x5e, 0x53, 0x4b, 0x91, 0xc1, 0xe4, 0x14,
+			0x22, 0x50, 0xb2, 0xc3, 0x84, 0xf4, 0xa6, 0xb3, 0xa4,
+			0x99, 0xda, 0x82, 0x55, 0x8e, 0x38, 0xb4, 0x0c, 0x21,
+			0xe5, 0x91, 0x30, 0xc7, 0x80, 0x55, 0xf1, 0x69, 0xde,
+			0xd7, 0x82, 0xf3, 0x08, 0xd3, 0x16, 0x69, 0xe7, 0xd4,
+			0xeb, 0xfc, 0x9b, 0x8f, 0xd6, 0xfc, 0xf9, 0x9f, 0x95,
+			0xb3, 0x90, 0xa6, 0x12, 0x21, 0x53, 0x50, 0xd5, 0x3e,
+			0x2b, 0x46, 0x14, 0x52, 0x20, 0x1c, 0x33, 0xed, 0xdd,
+			0x58, 0x81, 0x92, 0x37, 0xe5, 0x99, 0x77, 0xfa, 0x29,
+			0x28, 0x49, 0x07, 0xc8, 0xe2, 0x41, 0x8f, 0x77, 0x3b,
+			0xb4, 0x8c, 0x34, 0x3e, 0x80, 0xcb, 0xcf, 0x73, 0x4d,
+			0xfc, 0x78, 0x60, 0x9e, 0xd0, 0x86, 0xc3, 0x4d, 0xa7,
+			0x40, 0x9c, 0x03, 0x72, 0x46, 0xc3, 0xc3, 0x74, 0xdf,
+			0x5a, 0x47, 0x8d, 0xbc, 0xc1, 0x5b, 0x10, 0x3e, 0x7e,
+			0x09, 0x3d, 0xdb, 0x0c, 0xc4, 0x3b, 0x12, 0xbf, 0x66,
+			0xe2, 0xb7, 0x4c, 0x1a, 0x81, 0x58, 0x63, 0x39, 0x72,
+			0xde, 0xfa, 0x5d, 0xcc, 0x2c, 0x69, 0x11, 0x0a, 0x5f,
+			0xb9, 0xbc, 0xda, 0x80, 0x03, 0x8e, 0xa2, 0x02, 0x6c,
+			0xc1, 0x15, 0xcc, 0xbf, 0x01, 0x14, 0xd0, 0x36, 0xad,
+			0xdb, 0xb7, 0x77, 0xd6, 0x99, 0x94, 0xf5, 0xa5, 0xac,
+			0xce, 0x9a, 0x23, 0x9b, 0x1e, 0xa4, 0x97, 0x7e, 0x41,
+			0xc1, 0x65, 0xc7, 0x94, 0xfa, 0x07, 0x1d, 0x9a, 0x87,
+			0xba, 0xee, 0xef, 0x99, 0x92, 0xaa, 0x41, 0x29, 0x03,
+			0xe3, 0x01, 0x17, 0x42, 0xed, 0x11, 0x68, 0x44, 0x02,
+			0xca, 0x81, 0x4b, 0x33, 0x81, 0x64, 0x30, 0x95, 0x7f,
+			0x81, 0x17, 0xe6, 0x59, 0x0b, 0x4e, 0xf3, 0x4a, 0x72,
+			0x0c, 0x45, 0x1e, 0x1d, 0x44, 0xe2, 0x9e, 0x45, 0x29,
+			0x88, 0xb4, 0xda, 0x9b, 0x9f, 0x3e, 0x92, 0x9a, 0x0b,
+			0x21, 0x2a, 0xd4, 0xa0, 0xd8, 0xd8, 0xbd, 0x06, 0x6d,
+			0x71, 0x3c, 0xcc, 0xd8, 0xdc, 0x19, 0xd7, 0x8f, 0x21,
+			0x01, 0x25, 0xa5, 0x15, 0x62, 0x43, 0xfa, 0xdf, 0xb9,
+			0x33, 0x23, 0xd3, 0xa2, 0xab, 0xb1, 0x7e, 0x4e, 0x5e,
+			0x46, 0xbd, 0x76, 0x6e, 0x18, 0x88, 0x5f, 0x85, 0x03,
+			0x43, 0x57, 0xd6, 0x3a, 0xf2, 0xcc, 0x9d, 0xbc, 0x2c,
+			0x67, 0xa8, 0xcd, 0xda, 0xe8, 0x39, 0x61, 0x68, 0x6d,
+			0xcf, 0xeb, 0x07, 0x1f, 0x58, 0x61, 0x68, 0x6d, 0x78,
+			0xa6, 0xad, 0xb6, 0x16, 0x70, 0x89, 0x49, 0x59, 0x62,
+			0x76, 0xc1, 0xc3, 0xc8, 0xce, 0xe2, 0xfc, 0x42, 0x51,
+			0x8d, 0xc8, 0xf1, 0x03, 0x17, 0x36, 0x55, 0x8c, 0x8e,
+			0x98, 0xd1, 0x33, 0x3c, 0x4c, 0x63, 0x81, 0x97, 0xc5,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x06, 0x0c, 0x16, 0x19, 0x23, 0x28, 0x30,
+			0x37,
+		},
+};
diff --git a/lib/crypto/tests/mldsa_kunit.c b/lib/crypto/tests/mldsa_kunit.c
new file mode 100644
index 000000000000..67f8f93e3dc6
--- /dev/null
+++ b/lib/crypto/tests/mldsa_kunit.c
@@ -0,0 +1,438 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * KUnit tests and benchmark for ML-DSA
+ *
+ * Copyright 2025 Google LLC
+ */
+#include <crypto/mldsa.h>
+#include <kunit/test.h>
+#include <linux/random.h>
+#include <linux/unaligned.h>
+
+#define Q 8380417 /* The prime q = 2^23 - 2^13 + 1 */
+
+/* ML-DSA parameters that the tests use */
+static const struct {
+	int sig_len;
+	int pk_len;
+	int k;
+	int lambda;
+	int gamma1;
+	int beta;
+	int omega;
+} params[] = {
+	[MLDSA44] = {
+		.sig_len = MLDSA44_SIGNATURE_SIZE,
+		.pk_len = MLDSA44_PUBLIC_KEY_SIZE,
+		.k = 4,
+		.lambda = 128,
+		.gamma1 = 1 << 17,
+		.beta = 78,
+		.omega = 80,
+	},
+	[MLDSA65] = {
+		.sig_len = MLDSA65_SIGNATURE_SIZE,
+		.pk_len = MLDSA65_PUBLIC_KEY_SIZE,
+		.k = 6,
+		.lambda = 192,
+		.gamma1 = 1 << 19,
+		.beta = 196,
+		.omega = 55,
+	},
+	[MLDSA87] = {
+		.sig_len = MLDSA87_SIGNATURE_SIZE,
+		.pk_len = MLDSA87_PUBLIC_KEY_SIZE,
+		.k = 8,
+		.lambda = 256,
+		.gamma1 = 1 << 19,
+		.beta = 120,
+		.omega = 75,
+	},
+};
+
+#include "mldsa-testvecs.h"
+
+static void do_mldsa_and_assert_success(struct kunit *test,
+					const struct mldsa_testvector *tv)
+{
+	int err = mldsa_verify(tv->alg, tv->sig, tv->sig_len, tv->msg,
+			       tv->msg_len, tv->pk, tv->pk_len);
+	KUNIT_ASSERT_EQ(test, err, 0);
+}
+
+static u8 *kunit_kmemdup_or_fail(struct kunit *test, const u8 *src, size_t len)
+{
+	u8 *dst = kunit_kmalloc(test, len, GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, dst);
+	return memcpy(dst, src, len);
+}
+
+/*
+ * Test that changing coefficients in a valid signature's z vector results in
+ * the following behavior from mldsa_verify():
+ *
+ *  * -EBADMSG if a coefficient is changed to have an out-of-range value, i.e.
+ *    absolute value >= gamma1 - beta, corresponding to the verifier detecting
+ *    the out-of-range coefficient and rejecting the signature as malformed
+ *
+ *  * -EKEYREJECTED if a coefficient is changed to a different in-range value,
+ *    i.e. absolute value < gamma1 - beta, corresponding to the verifier
+ *    continuing to the "real" signature check and that check failing
+ */
+static void test_mldsa_z_range(struct kunit *test,
+			       const struct mldsa_testvector *tv)
+{
+	u8 *sig = kunit_kmemdup_or_fail(test, tv->sig, tv->sig_len);
+	const int lambda = params[tv->alg].lambda;
+	const s32 gamma1 = params[tv->alg].gamma1;
+	const int beta = params[tv->alg].beta;
+	/*
+	 * We just modify the first coefficient.  The coefficient is gamma1
+	 * minus either the first 18 or 20 bits of the u32, depending on gamma1.
+	 *
+	 * The layout of ML-DSA signatures is ctilde || z || h.  ctilde is
+	 * lambda / 4 bytes, so z starts at &sig[lambda / 4].
+	 */
+	u8 *z_ptr = &sig[lambda / 4];
+	const u32 z_data = get_unaligned_le32(z_ptr);
+	const u32 mask = (gamma1 << 1) - 1;
+	/* These are the four boundaries of the out-of-range values. */
+	const s32 out_of_range_coeffs[] = {
+		-gamma1 + 1,
+		-(gamma1 - beta),
+		gamma1,
+		gamma1 - beta,
+	};
+	/*
+	 * These are the two boundaries of the valid range, along with 0.  We
+	 * assume that none of these matches the original coefficient.
+	 */
+	const s32 in_range_coeffs[] = {
+		-(gamma1 - beta - 1),
+		0,
+		gamma1 - beta - 1,
+	};
+
+	/* Initially the signature is valid. */
+	do_mldsa_and_assert_success(test, tv);
+
+	/* Test some out-of-range coefficients. */
+	for (int i = 0; i < ARRAY_SIZE(out_of_range_coeffs); i++) {
+		const s32 c = out_of_range_coeffs[i];
+
+		put_unaligned_le32((z_data & ~mask) | (mask & (gamma1 - c)),
+				   z_ptr);
+		KUNIT_ASSERT_EQ(test, -EBADMSG,
+				mldsa_verify(tv->alg, sig, tv->sig_len, tv->msg,
+					     tv->msg_len, tv->pk, tv->pk_len));
+	}
+
+	/* Test some in-range coefficients. */
+	for (int i = 0; i < ARRAY_SIZE(in_range_coeffs); i++) {
+		const s32 c = in_range_coeffs[i];
+
+		put_unaligned_le32((z_data & ~mask) | (mask & (gamma1 - c)),
+				   z_ptr);
+		KUNIT_ASSERT_EQ(test, -EKEYREJECTED,
+				mldsa_verify(tv->alg, sig, tv->sig_len, tv->msg,
+					     tv->msg_len, tv->pk, tv->pk_len));
+	}
+}
+
+/* Test that mldsa_verify() rejects malformed hint vectors with -EBADMSG. */
+static void test_mldsa_bad_hints(struct kunit *test,
+				 const struct mldsa_testvector *tv)
+{
+	const int omega = params[tv->alg].omega;
+	const int k = params[tv->alg].k;
+	u8 *sig = kunit_kmemdup_or_fail(test, tv->sig, tv->sig_len);
+	/* Pointer to the encoded hint vector in the signature */
+	u8 *hintvec = &sig[tv->sig_len - omega - k];
+	u8 h;
+
+	/* Initially the signature is valid. */
+	do_mldsa_and_assert_success(test, tv);
+
+	/* Cumulative hint count exceeds omega */
+	memcpy(sig, tv->sig, tv->sig_len);
+	hintvec[omega + k - 1] = omega + 1;
+	KUNIT_ASSERT_EQ(test, -EBADMSG,
+			mldsa_verify(tv->alg, sig, tv->sig_len, tv->msg,
+				     tv->msg_len, tv->pk, tv->pk_len));
+
+	/* Cumulative hint count decreases */
+	memcpy(sig, tv->sig, tv->sig_len);
+	KUNIT_ASSERT_GE(test, hintvec[omega + k - 2], 1);
+	hintvec[omega + k - 1] = hintvec[omega + k - 2] - 1;
+	KUNIT_ASSERT_EQ(test, -EBADMSG,
+			mldsa_verify(tv->alg, sig, tv->sig_len, tv->msg,
+				     tv->msg_len, tv->pk, tv->pk_len));
+
+	/*
+	 * Hint indices out of order.  To test this, swap hintvec[0] and
+	 * hintvec[1].  This assumes that the original valid signature had at
+	 * least two nonzero hints in the first element (asserted below).
+	 */
+	memcpy(sig, tv->sig, tv->sig_len);
+	KUNIT_ASSERT_GE(test, hintvec[omega], 2);
+	h = hintvec[0];
+	hintvec[0] = hintvec[1];
+	hintvec[1] = h;
+	KUNIT_ASSERT_EQ(test, -EBADMSG,
+			mldsa_verify(tv->alg, sig, tv->sig_len, tv->msg,
+				     tv->msg_len, tv->pk, tv->pk_len));
+
+	/*
+	 * Extra hint indices given.  For this test to work, the original valid
+	 * signature must have fewer than omega nonzero hints (asserted below).
+	 */
+	memcpy(sig, tv->sig, tv->sig_len);
+	KUNIT_ASSERT_LT(test, hintvec[omega + k - 1], omega);
+	hintvec[omega - 1] = 0xff;
+	KUNIT_ASSERT_EQ(test, -EBADMSG,
+			mldsa_verify(tv->alg, sig, tv->sig_len, tv->msg,
+				     tv->msg_len, tv->pk, tv->pk_len));
+}
+
+static void test_mldsa_mutation(struct kunit *test,
+				const struct mldsa_testvector *tv)
+{
+	const int sig_len = tv->sig_len;
+	const int msg_len = tv->msg_len;
+	const int pk_len = tv->pk_len;
+	const int num_iter = 200;
+	u8 *sig = kunit_kmemdup_or_fail(test, tv->sig, sig_len);
+	u8 *msg = kunit_kmemdup_or_fail(test, tv->msg, msg_len);
+	u8 *pk = kunit_kmemdup_or_fail(test, tv->pk, pk_len);
+
+	/* Initially the signature is valid. */
+	do_mldsa_and_assert_success(test, tv);
+
+	/* Changing any bit in the signature should invalidate the signature */
+	for (int i = 0; i < num_iter; i++) {
+		size_t pos = get_random_u32_below(sig_len);
+		u8 b = 1 << get_random_u32_below(8);
+
+		sig[pos] ^= b;
+		KUNIT_ASSERT_NE(test, 0,
+				mldsa_verify(tv->alg, sig, sig_len, msg,
+					     msg_len, pk, pk_len));
+		sig[pos] ^= b;
+	}
+
+	/* Changing any bit in the message should invalidate the signature */
+	for (int i = 0; i < num_iter; i++) {
+		size_t pos = get_random_u32_below(msg_len);
+		u8 b = 1 << get_random_u32_below(8);
+
+		msg[pos] ^= b;
+		KUNIT_ASSERT_NE(test, 0,
+				mldsa_verify(tv->alg, sig, sig_len, msg,
+					     msg_len, pk, pk_len));
+		msg[pos] ^= b;
+	}
+
+	/* Changing any bit in the public key should invalidate the signature */
+	for (int i = 0; i < num_iter; i++) {
+		size_t pos = get_random_u32_below(pk_len);
+		u8 b = 1 << get_random_u32_below(8);
+
+		pk[pos] ^= b;
+		KUNIT_ASSERT_NE(test, 0,
+				mldsa_verify(tv->alg, sig, sig_len, msg,
+					     msg_len, pk, pk_len));
+		pk[pos] ^= b;
+	}
+
+	/* All changes should have been undone. */
+	KUNIT_ASSERT_EQ(test, 0,
+			mldsa_verify(tv->alg, sig, sig_len, msg, msg_len, pk,
+				     pk_len));
+}
+
+static void test_mldsa(struct kunit *test, const struct mldsa_testvector *tv)
+{
+	/* Valid signature */
+	KUNIT_ASSERT_EQ(test, tv->sig_len, params[tv->alg].sig_len);
+	KUNIT_ASSERT_EQ(test, tv->pk_len, params[tv->alg].pk_len);
+	do_mldsa_and_assert_success(test, tv);
+
+	/* Signature too short */
+	KUNIT_ASSERT_EQ(test, -EBADMSG,
+			mldsa_verify(tv->alg, tv->sig, tv->sig_len - 1, tv->msg,
+				     tv->msg_len, tv->pk, tv->pk_len));
+
+	/* Signature too long */
+	KUNIT_ASSERT_EQ(test, -EBADMSG,
+			mldsa_verify(tv->alg, tv->sig, tv->sig_len + 1, tv->msg,
+				     tv->msg_len, tv->pk, tv->pk_len));
+
+	/* Public key too short */
+	KUNIT_ASSERT_EQ(test, -EBADMSG,
+			mldsa_verify(tv->alg, tv->sig, tv->sig_len, tv->msg,
+				     tv->msg_len, tv->pk, tv->pk_len - 1));
+
+	/* Public key too long */
+	KUNIT_ASSERT_EQ(test, -EBADMSG,
+			mldsa_verify(tv->alg, tv->sig, tv->sig_len, tv->msg,
+				     tv->msg_len, tv->pk, tv->pk_len + 1));
+
+	/*
+	 * Message too short.  Error is EKEYREJECTED because it gets rejected by
+	 * the "real" signature check rather than the well-formedness checks.
+	 */
+	KUNIT_ASSERT_EQ(test, -EKEYREJECTED,
+			mldsa_verify(tv->alg, tv->sig, tv->sig_len, tv->msg,
+				     tv->msg_len - 1, tv->pk, tv->pk_len));
+	/*
+	 * Can't simply try (tv->msg, tv->msg_len + 1) too, as tv->msg would be
+	 * accessed out of bounds.  However, ML-DSA just hashes the message and
+	 * doesn't handle different message lengths differently anyway.
+	 */
+
+	/* Test the validity checks on the z vector. */
+	test_mldsa_z_range(test, tv);
+
+	/* Test the validity checks on the hint vector. */
+	test_mldsa_bad_hints(test, tv);
+
+	/* Test randomly mutating the inputs. */
+	test_mldsa_mutation(test, tv);
+}
+
+static void test_mldsa44(struct kunit *test)
+{
+	test_mldsa(test, &mldsa44_testvector);
+}
+
+static void test_mldsa65(struct kunit *test)
+{
+	test_mldsa(test, &mldsa65_testvector);
+}
+
+static void test_mldsa87(struct kunit *test)
+{
+	test_mldsa(test, &mldsa87_testvector);
+}
+
+static s32 mod(s32 a, s32 m)
+{
+	a %= m;
+	if (a < 0)
+		a += m;
+	return a;
+}
+
+static s32 symmetric_mod(s32 a, s32 m)
+{
+	a = mod(a, m);
+	if (a > m / 2)
+		a -= m;
+	return a;
+}
+
+/* Mechanical, inefficient translation of FIPS 204 Algorithm 36, Decompose */
+static void decompose_ref(s32 r, s32 gamma2, s32 *r0, s32 *r1)
+{
+	s32 rplus = mod(r, Q);
+
+	*r0 = symmetric_mod(rplus, 2 * gamma2);
+	if (rplus - *r0 == Q - 1) {
+		*r1 = 0;
+		*r0 = *r0 - 1;
+	} else {
+		*r1 = (rplus - *r0) / (2 * gamma2);
+	}
+}
+
+/* Mechanical, inefficient translation of FIPS 204 Algorithm 40, UseHint */
+static s32 use_hint_ref(u8 h, s32 r, s32 gamma2)
+{
+	s32 m = (Q - 1) / (2 * gamma2);
+	s32 r0, r1;
+
+	decompose_ref(r, gamma2, &r0, &r1);
+	if (h == 1 && r0 > 0)
+		return mod(r1 + 1, m);
+	if (h == 1 && r0 <= 0)
+		return mod(r1 - 1, m);
+	return r1;
+}
+
+/*
+ * Test that for all possible inputs, mldsa_use_hint() gives the same output as
+ * a mechanical translation of the pseudocode from FIPS 204.
+ */
+static void test_mldsa_use_hint(struct kunit *test)
+{
+	for (int i = 0; i < 2; i++) {
+		const s32 gamma2 = (Q - 1) / (i == 0 ? 88 : 32);
+
+		for (u8 h = 0; h < 2; h++) {
+			for (s32 r = 0; r < Q; r++) {
+				KUNIT_ASSERT_EQ(test,
+						mldsa_use_hint(h, r, gamma2),
+						use_hint_ref(h, r, gamma2));
+			}
+		}
+	}
+}
+
+static void benchmark_mldsa(struct kunit *test,
+			    const struct mldsa_testvector *tv)
+{
+	const int warmup_niter = 200;
+	const int benchmark_niter = 200;
+	u64 t0, t1;
+
+	if (!IS_ENABLED(CONFIG_CRYPTO_LIB_BENCHMARK))
+		kunit_skip(test, "not enabled");
+
+	for (int i = 0; i < warmup_niter; i++)
+		do_mldsa_and_assert_success(test, tv);
+
+	t0 = ktime_get_ns();
+	for (int i = 0; i < benchmark_niter; i++)
+		do_mldsa_and_assert_success(test, tv);
+	t1 = ktime_get_ns();
+	kunit_info(test, "%llu ops/s",
+		   div64_u64((u64)benchmark_niter * NSEC_PER_SEC,
+			     t1 - t0 ?: 1));
+}
+
+static void benchmark_mldsa44(struct kunit *test)
+{
+	benchmark_mldsa(test, &mldsa44_testvector);
+}
+
+static void benchmark_mldsa65(struct kunit *test)
+{
+	benchmark_mldsa(test, &mldsa65_testvector);
+}
+
+static void benchmark_mldsa87(struct kunit *test)
+{
+	benchmark_mldsa(test, &mldsa87_testvector);
+}
+
+static struct kunit_case mldsa_kunit_cases[] = {
+	KUNIT_CASE(test_mldsa44),
+	KUNIT_CASE(test_mldsa65),
+	KUNIT_CASE(test_mldsa87),
+	KUNIT_CASE(test_mldsa_use_hint),
+	KUNIT_CASE(benchmark_mldsa44),
+	KUNIT_CASE(benchmark_mldsa65),
+	KUNIT_CASE(benchmark_mldsa87),
+	{},
+};
+
+static struct kunit_suite mldsa_kunit_suite = {
+	.name = "mldsa",
+	.test_cases = mldsa_kunit_cases,
+};
+kunit_test_suite(mldsa_kunit_suite);
+
+MODULE_DESCRIPTION("KUnit tests and benchmark for ML-DSA");
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/tests/nh-testvecs.h b/lib/crypto/tests/nh-testvecs.h
new file mode 100644
index 000000000000..b393e1f90f31
--- /dev/null
+++ b/lib/crypto/tests/nh-testvecs.h
@@ -0,0 +1,298 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py nh */
+
+static const u8 nh_test_key[NH_KEY_BYTES] = {
+	0x04, 0x59, 0x66, 0x92, 0x81, 0xd7, 0xe9, 0x25,
+	0x68, 0xfa, 0xb0, 0xca, 0x9f, 0xea, 0x98, 0xca,
+	0xcd, 0xbf, 0x6d, 0xa5, 0x0c, 0x22, 0xc3, 0x57,
+	0xdc, 0x35, 0x05, 0xdd, 0x5b, 0xb0, 0xce, 0xf6,
+	0xb2, 0x4c, 0x77, 0x2e, 0xd2, 0x63, 0xf0, 0x17,
+	0x60, 0xd8, 0xd3, 0xd9, 0xed, 0x34, 0xb6, 0xed,
+	0x6a, 0x11, 0xc0, 0x25, 0xda, 0xba, 0x7e, 0xef,
+	0x49, 0x13, 0xf7, 0xd9, 0xfc, 0xb6, 0xfd, 0x58,
+	0xe9, 0x5f, 0xc5, 0xc4, 0x69, 0x89, 0xba, 0xa6,
+	0x2b, 0x58, 0x8d, 0x36, 0x6c, 0xb9, 0x90, 0x1e,
+	0x64, 0xc7, 0x44, 0x84, 0x03, 0x70, 0x30, 0x47,
+	0xdd, 0x58, 0xf4, 0x87, 0x61, 0xfd, 0x9c, 0x6b,
+	0x51, 0x1b, 0x39, 0x1d, 0x6d, 0x50, 0xae, 0x19,
+	0x71, 0x03, 0xc7, 0xa7, 0x42, 0x82, 0x8f, 0xa5,
+	0x63, 0x6a, 0xe2, 0x8a, 0xad, 0x4b, 0x40, 0xa7,
+	0x3f, 0x8b, 0xe4, 0xae, 0xb2, 0x8a, 0x14, 0x78,
+	0x91, 0x07, 0xba, 0x02, 0x08, 0xc1, 0x34, 0xb8,
+	0xda, 0x61, 0x67, 0xf6, 0x98, 0x97, 0x1a, 0xcb,
+	0x0f, 0x82, 0x80, 0xff, 0x02, 0x54, 0x16, 0x57,
+	0x18, 0x35, 0xaf, 0x16, 0x17, 0x68, 0xcc, 0xc7,
+	0x52, 0xac, 0x31, 0x39, 0x60, 0xe4, 0xb4, 0xcb,
+	0x0e, 0xf9, 0x57, 0xe9, 0x96, 0xff, 0x99, 0xd6,
+	0x10, 0x96, 0x09, 0xab, 0x28, 0x92, 0x1b, 0x9f,
+	0x10, 0xde, 0x3e, 0x87, 0xb8, 0x9d, 0x2d, 0xa0,
+	0x3c, 0x91, 0x85, 0x8c, 0x9e, 0xc0, 0x97, 0x9a,
+	0xb4, 0x54, 0x7f, 0x4a, 0x63, 0xc2, 0x75, 0x0f,
+	0x0d, 0x2f, 0x62, 0x56, 0x48, 0x0e, 0xb6, 0xc7,
+	0xcf, 0x0d, 0x78, 0xca, 0xbd, 0x31, 0x9e, 0x4c,
+	0xf7, 0x3f, 0x9e, 0xc2, 0xea, 0x5e, 0x44, 0x6d,
+	0x76, 0xf9, 0xc5, 0xe0, 0x29, 0xea, 0x15, 0xbf,
+	0xaf, 0xd4, 0x75, 0xc8, 0x89, 0xcf, 0x4f, 0x17,
+	0xfd, 0x4a, 0x45, 0xa5, 0x4d, 0x2d, 0x87, 0x11,
+	0x2b, 0x3e, 0x64, 0xa2, 0x6b, 0xc5, 0x23, 0x8c,
+	0xfa, 0x71, 0x13, 0x72, 0x0e, 0x7c, 0xe1, 0x2c,
+	0x9f, 0x0e, 0x29, 0xc9, 0x15, 0xde, 0x4e, 0xd7,
+	0x42, 0x1f, 0x8e, 0xe1, 0x91, 0x99, 0x50, 0x38,
+	0x7f, 0x15, 0xc0, 0xf6, 0x4b, 0xfd, 0x9d, 0x40,
+	0xe9, 0x44, 0x51, 0xca, 0x3b, 0x83, 0x41, 0x9f,
+	0x82, 0x64, 0x66, 0x22, 0x12, 0x43, 0x1c, 0x4f,
+	0x45, 0x11, 0x3a, 0x46, 0xb1, 0x7c, 0x62, 0x0a,
+	0x9d, 0x4c, 0x99, 0x85, 0xb0, 0x10, 0x19, 0xcf,
+	0xeb, 0xf9, 0x65, 0xaf, 0xd8, 0x05, 0x9e, 0x61,
+	0x03, 0x5f, 0x15, 0x99, 0xa9, 0x05, 0x20, 0xc8,
+	0xaf, 0xab, 0x31, 0x9d, 0xd5, 0xdf, 0x24, 0xce,
+	0x2b, 0x6d, 0xd7, 0x17, 0xc3, 0x04, 0xff, 0x82,
+	0xa7, 0x18, 0x39, 0xe9, 0x0d, 0x0a, 0x5f, 0xb9,
+	0xc9, 0x86, 0x1d, 0xf8, 0x02, 0x2d, 0xc3, 0x88,
+	0x28, 0x73, 0x5c, 0xac, 0x25, 0xc9, 0xfe, 0xcb,
+	0xd2, 0xfd, 0x63, 0x74, 0xac, 0xe1, 0xb8, 0xa2,
+	0xc6, 0x2b, 0xb5, 0x40, 0x01, 0x9b, 0xed, 0xee,
+	0x7b, 0x63, 0x66, 0x05, 0x45, 0xc2, 0x6c, 0xd8,
+	0x58, 0xf1, 0xa1, 0x3d, 0xc8, 0x43, 0x59, 0x4b,
+	0x39, 0x87, 0x24, 0x64, 0x92, 0xb0, 0xab, 0x75,
+	0xf1, 0xb7, 0xbf, 0x7c, 0xde, 0xc0, 0xaf, 0x4a,
+	0xc2, 0x7b, 0xd9, 0x8a, 0x99, 0xcd, 0x83, 0x01,
+	0xe6, 0xae, 0xeb, 0x16, 0xe7, 0x54, 0x9c, 0x95,
+	0x0a, 0x91, 0x02, 0xaf, 0x9f, 0x79, 0x40, 0x45,
+	0xce, 0x47, 0x41, 0x65, 0xca, 0x80, 0x0d, 0x14,
+	0x46, 0x58, 0x5d, 0x4d, 0x28, 0x55, 0x70, 0x49,
+	0x7c, 0x32, 0x1f, 0x01, 0xaa, 0x05, 0x2f, 0xf1,
+	0xeb, 0xa3, 0xe6, 0x1d, 0xf9, 0x43, 0xe0, 0x58,
+	0x05, 0x61, 0x22, 0xc3, 0xee, 0xe4, 0x6f, 0x94,
+	0xaf, 0x82, 0xda, 0x18, 0x18, 0x63, 0x9c, 0xfa,
+	0xc0, 0x04, 0x27, 0xc5, 0x39, 0x5e, 0x7a, 0xa6,
+	0x85, 0x46, 0xb7, 0x76, 0xc9, 0x16, 0xf2, 0xf8,
+	0x40, 0x8d, 0x4b, 0x5e, 0x72, 0xf3, 0x3e, 0x12,
+	0xa4, 0x80, 0x39, 0xb2, 0x92, 0xfe, 0x6e, 0x5b,
+	0x5b, 0xad, 0xea, 0x29, 0xbc, 0x66, 0xe6, 0xfe,
+	0x80, 0x02, 0x5d, 0x83, 0x37, 0xfc, 0xde, 0x6c,
+	0x25, 0x54, 0xa2, 0xff, 0x7d, 0xb6, 0xe1, 0xd6,
+	0xcf, 0xdb, 0x60, 0xe3, 0xbe, 0x2f, 0x4e, 0xb4,
+	0xf5, 0xb4, 0x51, 0xf7, 0x5a, 0x25, 0xda, 0x40,
+	0x84, 0x5e, 0xc0, 0x0a, 0x6b, 0xfa, 0x0c, 0xfb,
+	0x5e, 0x3e, 0x12, 0x6c, 0x39, 0x35, 0xc0, 0x28,
+	0xd6, 0x1b, 0x3a, 0x72, 0xc3, 0xfe, 0xa5, 0x4c,
+	0x35, 0xa2, 0x42, 0xf6, 0x3d, 0xa5, 0xbf, 0xb5,
+	0x39, 0xe3, 0xc9, 0xd5, 0x8c, 0x1b, 0xe5, 0xef,
+	0x91, 0xd2, 0x80, 0x6f, 0xcc, 0x77, 0x44, 0x50,
+	0x62, 0xc7, 0xac, 0x29, 0xcb, 0x72, 0xda, 0x6d,
+	0xc5, 0xfe, 0xa7, 0xee, 0x8b, 0xeb, 0xfc, 0xa3,
+	0x46, 0x18, 0x5f, 0xaa, 0xc3, 0x65, 0xd0, 0x8f,
+	0x67, 0x98, 0xd6, 0xce, 0x5f, 0x84, 0xd4, 0x96,
+	0x1b, 0x67, 0xa0, 0xcf, 0xfc, 0x94, 0x55, 0x5e,
+	0x4b, 0x51, 0x68, 0xa7, 0x6d, 0x02, 0xf9, 0x53,
+	0x54, 0x86, 0x6b, 0x53, 0x39, 0xe0, 0x36, 0x23,
+	0x87, 0x1a, 0xfb, 0x53, 0x1a, 0x65, 0xd8, 0x42,
+	0xa8, 0x85, 0xfd, 0x2c, 0x7f, 0x6b, 0x7f, 0x67,
+	0x70, 0x23, 0x6c, 0xe9, 0x0b, 0xf0, 0x1e, 0x0d,
+	0x0b, 0xb4, 0xd4, 0x96, 0x14, 0x95, 0x7e, 0xf3,
+	0x9b, 0xdd, 0xd7, 0xc4, 0x24, 0x22, 0xb9, 0x9d,
+	0xb3, 0xa6, 0xac, 0x09, 0x7c, 0x00, 0xbf, 0xd0,
+	0xdc, 0xfb, 0x9b, 0x7c, 0x8c, 0xbd, 0xd4, 0x1a,
+	0x13, 0x2b, 0x82, 0x3d, 0x7c, 0x8c, 0x10, 0x47,
+	0x49, 0x6c, 0x53, 0xeb, 0xa7, 0xc2, 0xde, 0xed,
+	0xe2, 0x55, 0x93, 0x2c, 0x1a, 0x5a, 0x7d, 0xe1,
+	0x37, 0x62, 0xdd, 0x29, 0x1a, 0x72, 0x82, 0xc0,
+	0x14, 0x73, 0x5d, 0x0e, 0x9b, 0xcc, 0x54, 0x68,
+	0x3a, 0x4d, 0x56, 0x8f, 0xc9, 0x4e, 0xaf, 0x7b,
+	0xde, 0x17, 0x9c, 0x5e, 0x83, 0x82, 0x22, 0xe3,
+	0x28, 0xdf, 0x1b, 0xb6, 0xdb, 0x17, 0x90, 0x48,
+	0xb5, 0x13, 0x4e, 0xd3, 0x97, 0x5e, 0xb3, 0x9c,
+	0x16, 0x08, 0xc8, 0x77, 0xb3, 0xcd, 0x94, 0x90,
+	0x4f, 0x77, 0xaf, 0x67, 0xdd, 0x80, 0x15, 0x1c,
+	0x59, 0xfb, 0x3c, 0xec, 0xf8, 0xb3, 0x67, 0xfb,
+	0xa0, 0x94, 0x3c, 0x53, 0x99, 0x49, 0x94, 0x2c,
+	0x85, 0x26, 0x92, 0x6d, 0x8d, 0x48, 0xf6, 0x72,
+	0xdd, 0xfb, 0xb2, 0x10, 0x51, 0x5b, 0xbe, 0xd5,
+	0x70, 0x3d, 0x28, 0x94, 0x98, 0x4f, 0x6e, 0x20,
+	0x7b, 0x7d, 0x0f, 0x56, 0xc9, 0x96, 0x5f, 0x60,
+	0x2e, 0x2f, 0x9b, 0x38, 0x7f, 0xc7, 0x3c, 0x6b,
+	0x2f, 0x2b, 0x8f, 0x1f, 0x07, 0x1c, 0x85, 0x57,
+	0x16, 0x2e, 0xc7, 0x74, 0xe5, 0xf2, 0x0d, 0xfe,
+	0xef, 0x57, 0xb0, 0xa4, 0x4f, 0x4c, 0x7d, 0x81,
+	0xbb, 0xaa, 0xcb, 0xa0, 0xb0, 0x51, 0xcf, 0xc2,
+	0xee, 0x90, 0x2e, 0x5e, 0x27, 0xca, 0xd3, 0xe8,
+	0xf3, 0x55, 0x02, 0x56, 0x06, 0xa5, 0xad, 0xdf,
+	0xa3, 0xa9, 0x06, 0x05, 0x53, 0x74, 0x55, 0xd5,
+	0xd2, 0x20, 0x0a, 0x6d, 0x4a, 0xef, 0x16, 0xbf,
+	0xc3, 0xb2, 0x75, 0x93, 0xd8, 0x6e, 0x0f, 0xd2,
+	0xae, 0x3b, 0xc0, 0x00, 0x22, 0x6f, 0xb5, 0x0a,
+	0x41, 0xfc, 0xf9, 0x41, 0xfc, 0x16, 0x4f, 0xa6,
+	0x1c, 0x18, 0x41, 0x67, 0x73, 0xa8, 0x79, 0xa9,
+	0x54, 0x18, 0x4e, 0x88, 0x44, 0x0f, 0xa1, 0x5b,
+	0xf0, 0x68, 0xea, 0x3c, 0x62, 0x59, 0x8d, 0xc7,
+	0x6f, 0xd7, 0x72, 0x20, 0x74, 0x39, 0xd4, 0x3a,
+	0x41, 0x1b, 0x58, 0x57, 0x54, 0x85, 0x60, 0xca,
+	0x49, 0x4b, 0xa1, 0x04, 0x91, 0xb6, 0xf2, 0xcd,
+	0x62, 0x63, 0x67, 0xd1, 0xee, 0x6b, 0x9e, 0x5d,
+	0xd6, 0xc4, 0x58, 0x6b, 0xe1, 0xe6, 0x4a, 0xdb,
+	0xe8, 0xb1, 0x35, 0x03, 0x15, 0x8d, 0x34, 0x69,
+	0x4c, 0xd2, 0x54, 0xce, 0xe8, 0x6a, 0x69, 0x6f,
+	0xaa, 0xb5, 0x1f, 0x86, 0xed, 0xac, 0x4f, 0x16,
+	0x1e, 0x48, 0x93, 0xe8, 0x6c, 0x24, 0x1c, 0xd0,
+	0xbb, 0x61, 0xc2, 0x34, 0xdd, 0xc9, 0x5c, 0xce,
+};
+
+static const u8 nh_test_msg[NH_MESSAGE_BYTES] = {
+	0x99, 0x57, 0x61, 0x41, 0xad, 0x08, 0x7e, 0x17,
+	0xd4, 0xef, 0x0b, 0x23, 0xff, 0x0b, 0x96, 0x0a,
+	0x6c, 0x98, 0xac, 0x78, 0x5e, 0xb6, 0xb2, 0x67,
+	0x0f, 0x48, 0xf4, 0xa1, 0xe5, 0x1e, 0xfe, 0x83,
+	0xe4, 0x56, 0x2a, 0x03, 0x64, 0xff, 0x7a, 0xf3,
+	0x03, 0xfe, 0xa7, 0x86, 0xdc, 0x35, 0x79, 0x13,
+	0xf8, 0xe1, 0x59, 0x19, 0x04, 0x43, 0x24, 0x82,
+	0x44, 0x82, 0x41, 0x2b, 0xc7, 0xcf, 0xf5, 0xa4,
+	0xdc, 0xca, 0xf5, 0x34, 0xc4, 0x23, 0x3c, 0x1f,
+	0xa8, 0x84, 0x1f, 0x2a, 0xcd, 0xae, 0x9d, 0x5e,
+	0x05, 0xe2, 0xfb, 0x0c, 0x68, 0x81, 0x90, 0x11,
+	0x44, 0xf6, 0xdd, 0x5b, 0x51, 0xd3, 0xe0, 0xab,
+	0x29, 0x3a, 0xa9, 0x9c, 0xf6, 0x7e, 0x2d, 0xe3,
+	0x6c, 0x09, 0x59, 0xd7, 0xfa, 0x7f, 0x6a, 0x33,
+	0x3b, 0x23, 0x7b, 0x1b, 0xb2, 0x79, 0x5f, 0x5c,
+	0xb6, 0x2d, 0xb0, 0xf8, 0xab, 0x33, 0x28, 0xe0,
+	0x72, 0x2e, 0x2f, 0x03, 0x22, 0x16, 0xb4, 0x87,
+	0xf7, 0x14, 0x3f, 0x55, 0x8a, 0xb0, 0x47, 0xdb,
+	0x42, 0x2d, 0xc0, 0x0c, 0x0a, 0x33, 0xf8, 0xab,
+	0x44, 0xae, 0xa3, 0xc9, 0xfc, 0xf6, 0x34, 0x8c,
+	0x60, 0x30, 0x6d, 0x31, 0x70, 0xf3, 0x39, 0x53,
+	0xf1, 0x2d, 0xb9, 0x6c, 0xa6, 0x48, 0x9c, 0x9c,
+	0xc2, 0x88, 0xb3, 0xa9, 0x98, 0xb6, 0xc3, 0x47,
+	0x94, 0x02, 0x9d, 0x98, 0x6e, 0x25, 0x6c, 0xf5,
+	0x9b, 0xc6, 0x4d, 0xee, 0x07, 0x1e, 0x25, 0x8f,
+	0x01, 0xde, 0xad, 0xe5, 0x77, 0x4f, 0xd1, 0xc0,
+	0x62, 0xbb, 0x3a, 0xb9, 0x83, 0x0b, 0x29, 0x76,
+	0x4f, 0xb1, 0x86, 0x2c, 0x27, 0xc7, 0x38, 0x65,
+	0xcb, 0x78, 0xb7, 0x02, 0x10, 0x9e, 0xde, 0x83,
+	0xd1, 0xac, 0x05, 0x86, 0x23, 0xce, 0x4f, 0x8d,
+	0xcc, 0x4e, 0x3f, 0x04, 0xf4, 0x39, 0x91, 0x81,
+	0x1c, 0x42, 0x47, 0x4d, 0x50, 0xe5, 0x01, 0x22,
+	0x98, 0xcf, 0x91, 0x36, 0xb3, 0x7c, 0xcf, 0x78,
+	0x07, 0x22, 0xa9, 0x18, 0xd2, 0xcd, 0x7d, 0x4d,
+	0xa6, 0xcb, 0xaa, 0x52, 0x13, 0x49, 0x64, 0xb0,
+	0xa5, 0x3d, 0xc7, 0xc3, 0x10, 0x87, 0x2e, 0x76,
+	0xa9, 0x52, 0xc5, 0x50, 0x18, 0xc0, 0x5d, 0xb4,
+	0x4c, 0xc6, 0x7f, 0x64, 0xae, 0x53, 0xc3, 0x46,
+	0x99, 0xb7, 0x61, 0x6b, 0x08, 0x43, 0x08, 0x4c,
+	0x90, 0x2c, 0xee, 0x56, 0x91, 0xb4, 0x28, 0xa8,
+	0xa8, 0x8b, 0x3b, 0x1a, 0x67, 0x71, 0xf2, 0x81,
+	0x48, 0x20, 0x71, 0x30, 0xdd, 0x69, 0x8a, 0xc2,
+	0x4c, 0x9d, 0x4e, 0x17, 0xfb, 0x2e, 0xe7, 0x9b,
+	0x86, 0x94, 0xa5, 0xce, 0xf9, 0x74, 0x56, 0xff,
+	0x3b, 0xff, 0xd9, 0x5a, 0xc8, 0x98, 0xf5, 0x25,
+	0xa2, 0xb9, 0x66, 0x46, 0x89, 0x17, 0x39, 0x08,
+	0x69, 0x03, 0x59, 0x1e, 0x13, 0x12, 0x68, 0xe7,
+	0x2f, 0x00, 0xd3, 0xf3, 0x71, 0xd1, 0x20, 0xc5,
+	0x0b, 0x38, 0x89, 0xda, 0x62, 0x3c, 0xce, 0xea,
+	0x04, 0x19, 0x47, 0x6d, 0xd8, 0x64, 0x38, 0x60,
+	0x96, 0x71, 0x68, 0x48, 0x79, 0xf8, 0xf4, 0x76,
+	0x33, 0xf6, 0x60, 0x8d, 0x21, 0xd0, 0xee, 0x41,
+	0xc0, 0xbe, 0x33, 0x61, 0x5e, 0x66, 0xe6, 0x16,
+	0x14, 0xc7, 0xfb, 0x6c, 0xf3, 0x58, 0xef, 0x12,
+	0x7c, 0x70, 0x65, 0x5d, 0x55, 0xe8, 0xf2, 0x92,
+	0x3a, 0xfe, 0x34, 0x64, 0x31, 0x7c, 0x29, 0xbb,
+	0x01, 0x18, 0xbd, 0xb6, 0xe4, 0x1e, 0xa4, 0xf3,
+	0x7b, 0x4c, 0x6a, 0x0d, 0x01, 0xfc, 0xc7, 0x66,
+	0xc3, 0x88, 0x37, 0x25, 0xcf, 0xe9, 0xca, 0x82,
+	0xeb, 0xa1, 0x38, 0x40, 0xc9, 0xdb, 0x38, 0x7b,
+	0x78, 0xcf, 0x11, 0xa3, 0x1c, 0x6b, 0x70, 0xc8,
+	0xe1, 0x2f, 0x7c, 0x17, 0x2c, 0x58, 0x28, 0xa4,
+	0x13, 0x40, 0xc7, 0x69, 0x0f, 0x04, 0xe5, 0x8e,
+	0xf0, 0x67, 0x53, 0xea, 0x10, 0xf5, 0x83, 0xc9,
+	0xcb, 0x6b, 0x16, 0xef, 0x2e, 0x55, 0xb3, 0xdd,
+	0xed, 0xf9, 0x1a, 0x52, 0x9a, 0x73, 0x78, 0x14,
+	0x14, 0x21, 0xfc, 0xef, 0x3c, 0x40, 0xa9, 0xfe,
+	0xef, 0xd7, 0x6e, 0x28, 0x2f, 0xd3, 0x73, 0xed,
+	0xa3, 0x73, 0xb5, 0x62, 0x41, 0xe6, 0xd4, 0x79,
+	0x49, 0x31, 0x2b, 0x86, 0x74, 0x56, 0x21, 0xfe,
+	0x6d, 0xb2, 0xbe, 0x81, 0x80, 0xa6, 0x81, 0x19,
+	0x90, 0x79, 0x6f, 0xc4, 0x4e, 0x7d, 0x6f, 0x2f,
+	0xa8, 0x6f, 0xd5, 0xc4, 0x7e, 0x23, 0x3b, 0xe6,
+	0x9b, 0x60, 0x97, 0x7b, 0xe2, 0x08, 0x8a, 0xaa,
+	0xc7, 0x7c, 0xf6, 0xe5, 0x01, 0x3e, 0xd2, 0x29,
+	0x7d, 0xd7, 0x40, 0x84, 0x95, 0xfa, 0xdf, 0xd8,
+	0x81, 0xe9, 0x5e, 0xdd, 0x0d, 0x17, 0x51, 0x6b,
+	0x8c, 0x0e, 0x47, 0xf9, 0x0c, 0x92, 0x1b, 0x60,
+	0xca, 0x06, 0x8a, 0xe5, 0xe8, 0x0f, 0x06, 0x75,
+	0x5d, 0x76, 0xc9, 0x32, 0x2c, 0x52, 0x2c, 0x2e,
+	0xd8, 0x66, 0x38, 0x75, 0x16, 0xc7, 0x7d, 0x51,
+	0xc4, 0xc2, 0x22, 0xc8, 0x19, 0xfc, 0x3d, 0x69,
+	0x1e, 0xd9, 0x64, 0x47, 0x5d, 0x21, 0x84, 0x46,
+	0xd7, 0xe1, 0xf0, 0x95, 0x3a, 0x8f, 0xbd, 0x7a,
+	0x53, 0x71, 0x4c, 0x54, 0xc1, 0x3e, 0x27, 0xde,
+	0xeb, 0x04, 0x11, 0xb0, 0x33, 0x4d, 0x57, 0x0b,
+	0x6b, 0x7d, 0x6c, 0xd5, 0x87, 0x7e, 0xb4, 0xe2,
+	0x94, 0x9e, 0x9f, 0x74, 0xe8, 0xb7, 0xfa, 0x05,
+	0x9b, 0x8f, 0x81, 0x43, 0x35, 0x82, 0xb8, 0x5b,
+	0xa8, 0x5e, 0xfa, 0x7a, 0x80, 0x8d, 0xd2, 0x90,
+	0x58, 0x79, 0x89, 0x56, 0x90, 0x2b, 0xff, 0x92,
+	0x3c, 0x35, 0xbe, 0x99, 0x5f, 0xd2, 0x4b, 0x15,
+	0x58, 0x4b, 0xbf, 0x08, 0x9b, 0x9b, 0x97, 0x10,
+	0xa4, 0x55, 0xc7, 0xec, 0x29, 0xc5, 0x14, 0x3e,
+	0x8f, 0x56, 0xa3, 0x92, 0x9e, 0x33, 0xcc, 0x9e,
+	0x77, 0x2f, 0x33, 0xcb, 0xc4, 0xe9, 0x19, 0xf4,
+	0x32, 0x2b, 0xef, 0x6c, 0x1c, 0x92, 0x2c, 0x45,
+	0x88, 0x74, 0x5f, 0xcf, 0x56, 0xfd, 0x87, 0x5f,
+	0xb6, 0x9b, 0xa2, 0x51, 0xda, 0x9b, 0x83, 0x4f,
+	0xec, 0x14, 0xe8, 0xd2, 0x42, 0x03, 0xcb, 0xe8,
+	0xd0, 0xb7, 0xf8, 0x38, 0xde, 0x6f, 0xdf, 0x43,
+	0xfa, 0x41, 0xab, 0xec, 0x2e, 0x3c, 0x93, 0x39,
+	0x76, 0xd1, 0x6f, 0x5b, 0x6c, 0x6e, 0x8d, 0xeb,
+	0x45, 0x6b, 0xc5, 0x76, 0x00, 0x29, 0xca, 0x3b,
+	0xdb, 0x78, 0xc2, 0x32, 0x09, 0x39, 0x19, 0x50,
+	0xa2, 0x44, 0x92, 0x09, 0xdb, 0x8b, 0x9e, 0x16,
+	0x76, 0x7f, 0xf1, 0x78, 0x7b, 0xb2, 0x51, 0xbc,
+	0x28, 0xbd, 0xb0, 0x7f, 0x25, 0x63, 0x7d, 0x34,
+	0xfb, 0xf6, 0x36, 0x24, 0xc7, 0xf9, 0x41, 0xb6,
+	0x2a, 0x06, 0xfc, 0xf0, 0x83, 0xf2, 0x12, 0x3d,
+	0x60, 0x2e, 0x10, 0x70, 0x31, 0x6f, 0x37, 0x08,
+	0x3e, 0x91, 0x93, 0xb5, 0xda, 0xb8, 0x4c, 0x1b,
+	0xd8, 0xb8, 0x3b, 0xd5, 0x3e, 0xb6, 0xc0, 0xbb,
+	0x38, 0x0f, 0xd2, 0x68, 0x4f, 0x78, 0x56, 0xf6,
+	0xda, 0x65, 0xb4, 0x0b, 0xb4, 0xaf, 0xa8, 0x19,
+	0x2f, 0x70, 0x55, 0xe0, 0x47, 0x31, 0x9f, 0x37,
+	0x1a, 0x47, 0xb9, 0x0c, 0x97, 0x79, 0xfc, 0xa9,
+	0x76, 0xe6, 0xfa, 0x38, 0x67, 0x25, 0xd3, 0x89,
+	0x8d, 0xad, 0xc6, 0x11, 0x2d, 0x77, 0x0b, 0x35,
+	0xa2, 0xe2, 0xdf, 0xc8, 0x94, 0xd5, 0xdf, 0xd2,
+	0x69, 0x2a, 0x99, 0x93, 0xfa, 0x4a, 0x5f, 0xc7,
+	0x8a, 0x14, 0x5f, 0x2a, 0xf3, 0x02, 0xf0, 0x3e,
+	0x21, 0x8e, 0x2e, 0x4b, 0xc4, 0xd2, 0xc8, 0xa6,
+	0x41, 0x6e, 0x17, 0x36, 0xe9, 0xad, 0x73, 0x33,
+	0x6c, 0xea, 0xc2, 0x31, 0x8f, 0x30, 0x51, 0x5c,
+	0x1c, 0x20, 0xe6, 0x05, 0x1a, 0x17, 0x15, 0x5d,
+	0x3e, 0x8f, 0xd2, 0x7f, 0xa1, 0xc5, 0x47, 0xb3,
+	0xb2, 0x9c, 0xe8, 0xf0, 0x6d, 0xc1, 0xc3, 0xa2,
+};
+
+static const u8 nh_test_val16[NH_HASH_BYTES] = {
+	0x30, 0x77, 0x55, 0x7c, 0x45, 0xd8, 0xce, 0xf7,
+	0x2a, 0xb5, 0x14, 0x8c, 0x35, 0x7e, 0xaa, 0x00,
+	0x50, 0xbc, 0x50, 0x7c, 0xd3, 0x20, 0x7c, 0x9c,
+	0xb4, 0xf1, 0x91, 0x26, 0x81, 0x03, 0xa5, 0x68,
+};
+
+static const u8 nh_test_val96[NH_HASH_BYTES] = {
+	0xd2, 0x19, 0xca, 0xa5, 0x6c, 0x0c, 0xdf, 0x2f,
+	0x69, 0xfa, 0x75, 0xc1, 0x63, 0xdb, 0xfa, 0x4d,
+	0x45, 0x2b, 0xb8, 0xdb, 0xac, 0xee, 0x61, 0xc6,
+	0x7a, 0x83, 0xb6, 0x0f, 0x32, 0x82, 0xe4, 0xd0,
+};
+
+static const u8 nh_test_val256[NH_HASH_BYTES] = {
+	0x33, 0x8f, 0xb4, 0x96, 0xf1, 0xb6, 0xf1, 0xb5,
+	0x05, 0x19, 0xbb, 0x6b, 0xda, 0xd9, 0x95, 0x75,
+	0x96, 0x3f, 0x8b, 0x42, 0xb6, 0xcd, 0xb7, 0xb7,
+	0xe7, 0x97, 0xb5, 0xa9, 0x0b, 0xd7, 0xdd, 0x33,
+};
+
+static const u8 nh_test_val1024[NH_HASH_BYTES] = {
+	0x32, 0x3d, 0x51, 0xe1, 0x77, 0xb6, 0xac, 0x06,
+	0x84, 0x67, 0xb7, 0xf2, 0x24, 0xe7, 0xec, 0xfd,
+	0x96, 0x64, 0xff, 0x55, 0xc7, 0x1b, 0xf9, 0xdc,
+	0xa3, 0xc7, 0x32, 0x06, 0x79, 0xcf, 0xca, 0xb6,
+};
diff --git a/lib/crypto/tests/nh_kunit.c b/lib/crypto/tests/nh_kunit.c
new file mode 100644
index 000000000000..a8a3c3f345cb
--- /dev/null
+++ b/lib/crypto/tests/nh_kunit.c
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2025 Google LLC
+ */
+#include <crypto/nh.h>
+#include <kunit/test.h>
+#include "nh-testvecs.h"
+
+static void test_nh(struct kunit *test)
+{
+	u32 *key = kunit_kmalloc(test, NH_KEY_BYTES, GFP_KERNEL);
+	__le64 hash[NH_NUM_PASSES];
+
+	KUNIT_ASSERT_NOT_NULL(test, key);
+	memcpy(key, nh_test_key, NH_KEY_BYTES);
+	le32_to_cpu_array(key, NH_KEY_WORDS);
+
+	nh(key, nh_test_msg, 16, hash);
+	KUNIT_ASSERT_MEMEQ(test, hash, nh_test_val16, NH_HASH_BYTES);
+
+	nh(key, nh_test_msg, 96, hash);
+	KUNIT_ASSERT_MEMEQ(test, hash, nh_test_val96, NH_HASH_BYTES);
+
+	nh(key, nh_test_msg, 256, hash);
+	KUNIT_ASSERT_MEMEQ(test, hash, nh_test_val256, NH_HASH_BYTES);
+
+	nh(key, nh_test_msg, 1024, hash);
+	KUNIT_ASSERT_MEMEQ(test, hash, nh_test_val1024, NH_HASH_BYTES);
+}
+
+static struct kunit_case nh_test_cases[] = {
+	KUNIT_CASE(test_nh),
+	{},
+};
+
+static struct kunit_suite nh_test_suite = {
+	.name = "nh",
+	.test_cases = nh_test_cases,
+};
+kunit_test_suite(nh_test_suite);
+
+MODULE_DESCRIPTION("KUnit tests for NH");
+MODULE_LICENSE("GPL");
diff --git a/lib/crypto/x86/nh-avx2.S b/lib/crypto/x86/nh-avx2.S
new file mode 100644
index 000000000000..9c085a31b137
--- /dev/null
+++ b/lib/crypto/x86/nh-avx2.S
@@ -0,0 +1,158 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * NH - ε-almost-universal hash function, x86_64 AVX2 accelerated
+ *
+ * Copyright 2018 Google LLC
+ *
+ * Author: Eric Biggers <ebiggers@google.com>
+ */
+
+#include <linux/linkage.h>
+
+#define		PASS0_SUMS	%ymm0
+#define		PASS1_SUMS	%ymm1
+#define		PASS2_SUMS	%ymm2
+#define		PASS3_SUMS	%ymm3
+#define		K0		%ymm4
+#define		K0_XMM		%xmm4
+#define		K1		%ymm5
+#define		K1_XMM		%xmm5
+#define		K2		%ymm6
+#define		K2_XMM		%xmm6
+#define		K3		%ymm7
+#define		K3_XMM		%xmm7
+#define		T0		%ymm8
+#define		T1		%ymm9
+#define		T2		%ymm10
+#define		T2_XMM		%xmm10
+#define		T3		%ymm11
+#define		T3_XMM		%xmm11
+#define		T4		%ymm12
+#define		T5		%ymm13
+#define		T6		%ymm14
+#define		T7		%ymm15
+#define		KEY		%rdi
+#define		MESSAGE		%rsi
+#define		MESSAGE_LEN	%rdx
+#define		HASH		%rcx
+
+.macro _nh_2xstride	k0, k1, k2, k3
+
+	// Add message words to key words
+	vpaddd		\k0, T3, T0
+	vpaddd		\k1, T3, T1
+	vpaddd		\k2, T3, T2
+	vpaddd		\k3, T3, T3
+
+	// Multiply 32x32 => 64 and accumulate
+	vpshufd		$0x10, T0, T4
+	vpshufd		$0x32, T0, T0
+	vpshufd		$0x10, T1, T5
+	vpshufd		$0x32, T1, T1
+	vpshufd		$0x10, T2, T6
+	vpshufd		$0x32, T2, T2
+	vpshufd		$0x10, T3, T7
+	vpshufd		$0x32, T3, T3
+	vpmuludq	T4, T0, T0
+	vpmuludq	T5, T1, T1
+	vpmuludq	T6, T2, T2
+	vpmuludq	T7, T3, T3
+	vpaddq		T0, PASS0_SUMS, PASS0_SUMS
+	vpaddq		T1, PASS1_SUMS, PASS1_SUMS
+	vpaddq		T2, PASS2_SUMS, PASS2_SUMS
+	vpaddq		T3, PASS3_SUMS, PASS3_SUMS
+.endm
+
+/*
+ * void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
+ *		__le64 hash[NH_NUM_PASSES])
+ *
+ * It's guaranteed that message_len % 16 == 0.
+ */
+SYM_FUNC_START(nh_avx2)
+
+	vmovdqu		0x00(KEY), K0
+	vmovdqu		0x10(KEY), K1
+	add		$0x20, KEY
+	vpxor		PASS0_SUMS, PASS0_SUMS, PASS0_SUMS
+	vpxor		PASS1_SUMS, PASS1_SUMS, PASS1_SUMS
+	vpxor		PASS2_SUMS, PASS2_SUMS, PASS2_SUMS
+	vpxor		PASS3_SUMS, PASS3_SUMS, PASS3_SUMS
+
+	sub		$0x40, MESSAGE_LEN
+	jl		.Lloop4_done
+.Lloop4:
+	vmovdqu		(MESSAGE), T3
+	vmovdqu		0x00(KEY), K2
+	vmovdqu		0x10(KEY), K3
+	_nh_2xstride	K0, K1, K2, K3
+
+	vmovdqu		0x20(MESSAGE), T3
+	vmovdqu		0x20(KEY), K0
+	vmovdqu		0x30(KEY), K1
+	_nh_2xstride	K2, K3, K0, K1
+
+	add		$0x40, MESSAGE
+	add		$0x40, KEY
+	sub		$0x40, MESSAGE_LEN
+	jge		.Lloop4
+
+.Lloop4_done:
+	and		$0x3f, MESSAGE_LEN
+	jz		.Ldone
+
+	cmp		$0x20, MESSAGE_LEN
+	jl		.Llast
+
+	// 2 or 3 strides remain; do 2 more.
+	vmovdqu		(MESSAGE), T3
+	vmovdqu		0x00(KEY), K2
+	vmovdqu		0x10(KEY), K3
+	_nh_2xstride	K0, K1, K2, K3
+	add		$0x20, MESSAGE
+	add		$0x20, KEY
+	sub		$0x20, MESSAGE_LEN
+	jz		.Ldone
+	vmovdqa		K2, K0
+	vmovdqa		K3, K1
+.Llast:
+	// Last stride.  Zero the high 128 bits of the message and keys so they
+	// don't affect the result when processing them like 2 strides.
+	vmovdqu		(MESSAGE), T3_XMM
+	vmovdqa		K0_XMM, K0_XMM
+	vmovdqa		K1_XMM, K1_XMM
+	vmovdqu		0x00(KEY), K2_XMM
+	vmovdqu		0x10(KEY), K3_XMM
+	_nh_2xstride	K0, K1, K2, K3
+
+.Ldone:
+	// Sum the accumulators for each pass, then store the sums to 'hash'
+
+	// PASS0_SUMS is (0A 0B 0C 0D)
+	// PASS1_SUMS is (1A 1B 1C 1D)
+	// PASS2_SUMS is (2A 2B 2C 2D)
+	// PASS3_SUMS is (3A 3B 3C 3D)
+	// We need the horizontal sums:
+	//     (0A + 0B + 0C + 0D,
+	//	1A + 1B + 1C + 1D,
+	//	2A + 2B + 2C + 2D,
+	//	3A + 3B + 3C + 3D)
+	//
+
+	vpunpcklqdq	PASS1_SUMS, PASS0_SUMS, T0	// T0 = (0A 1A 0C 1C)
+	vpunpckhqdq	PASS1_SUMS, PASS0_SUMS, T1	// T1 = (0B 1B 0D 1D)
+	vpunpcklqdq	PASS3_SUMS, PASS2_SUMS, T2	// T2 = (2A 3A 2C 3C)
+	vpunpckhqdq	PASS3_SUMS, PASS2_SUMS, T3	// T3 = (2B 3B 2D 3D)
+
+	vinserti128	$0x1, T2_XMM, T0, T4		// T4 = (0A 1A 2A 3A)
+	vinserti128	$0x1, T3_XMM, T1, T5		// T5 = (0B 1B 2B 3B)
+	vperm2i128	$0x31, T2, T0, T0		// T0 = (0C 1C 2C 3C)
+	vperm2i128	$0x31, T3, T1, T1		// T1 = (0D 1D 2D 3D)
+
+	vpaddq		T5, T4, T4
+	vpaddq		T1, T0, T0
+	vpaddq		T4, T0, T0
+	vmovdqu		T0, (HASH)
+	vzeroupper
+	RET
+SYM_FUNC_END(nh_avx2)
diff --git a/lib/crypto/x86/nh-sse2.S b/lib/crypto/x86/nh-sse2.S
new file mode 100644
index 000000000000..d36c0e6d5556
--- /dev/null
+++ b/lib/crypto/x86/nh-sse2.S
@@ -0,0 +1,123 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * NH - ε-almost-universal hash function, x86_64 SSE2 accelerated
+ *
+ * Copyright 2018 Google LLC
+ *
+ * Author: Eric Biggers <ebiggers@google.com>
+ */
+
+#include <linux/linkage.h>
+
+#define		PASS0_SUMS	%xmm0
+#define		PASS1_SUMS	%xmm1
+#define		PASS2_SUMS	%xmm2
+#define		PASS3_SUMS	%xmm3
+#define		K0		%xmm4
+#define		K1		%xmm5
+#define		K2		%xmm6
+#define		K3		%xmm7
+#define		T0		%xmm8
+#define		T1		%xmm9
+#define		T2		%xmm10
+#define		T3		%xmm11
+#define		T4		%xmm12
+#define		T5		%xmm13
+#define		T6		%xmm14
+#define		T7		%xmm15
+#define		KEY		%rdi
+#define		MESSAGE		%rsi
+#define		MESSAGE_LEN	%rdx
+#define		HASH		%rcx
+
+.macro _nh_stride	k0, k1, k2, k3, offset
+
+	// Load next message stride
+	movdqu		\offset(MESSAGE), T1
+
+	// Load next key stride
+	movdqu		\offset(KEY), \k3
+
+	// Add message words to key words
+	movdqa		T1, T2
+	movdqa		T1, T3
+	paddd		T1, \k0    // reuse k0 to avoid a move
+	paddd		\k1, T1
+	paddd		\k2, T2
+	paddd		\k3, T3
+
+	// Multiply 32x32 => 64 and accumulate
+	pshufd		$0x10, \k0, T4
+	pshufd		$0x32, \k0, \k0
+	pshufd		$0x10, T1, T5
+	pshufd		$0x32, T1, T1
+	pshufd		$0x10, T2, T6
+	pshufd		$0x32, T2, T2
+	pshufd		$0x10, T3, T7
+	pshufd		$0x32, T3, T3
+	pmuludq		T4, \k0
+	pmuludq		T5, T1
+	pmuludq		T6, T2
+	pmuludq		T7, T3
+	paddq		\k0, PASS0_SUMS
+	paddq		T1, PASS1_SUMS
+	paddq		T2, PASS2_SUMS
+	paddq		T3, PASS3_SUMS
+.endm
+
+/*
+ * void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
+ *		__le64 hash[NH_NUM_PASSES])
+ *
+ * It's guaranteed that message_len % 16 == 0.
+ */
+SYM_FUNC_START(nh_sse2)
+
+	movdqu		0x00(KEY), K0
+	movdqu		0x10(KEY), K1
+	movdqu		0x20(KEY), K2
+	add		$0x30, KEY
+	pxor		PASS0_SUMS, PASS0_SUMS
+	pxor		PASS1_SUMS, PASS1_SUMS
+	pxor		PASS2_SUMS, PASS2_SUMS
+	pxor		PASS3_SUMS, PASS3_SUMS
+
+	sub		$0x40, MESSAGE_LEN
+	jl		.Lloop4_done
+.Lloop4:
+	_nh_stride	K0, K1, K2, K3, 0x00
+	_nh_stride	K1, K2, K3, K0, 0x10
+	_nh_stride	K2, K3, K0, K1, 0x20
+	_nh_stride	K3, K0, K1, K2, 0x30
+	add		$0x40, KEY
+	add		$0x40, MESSAGE
+	sub		$0x40, MESSAGE_LEN
+	jge		.Lloop4
+
+.Lloop4_done:
+	and		$0x3f, MESSAGE_LEN
+	jz		.Ldone
+	_nh_stride	K0, K1, K2, K3, 0x00
+
+	sub		$0x10, MESSAGE_LEN
+	jz		.Ldone
+	_nh_stride	K1, K2, K3, K0, 0x10
+
+	sub		$0x10, MESSAGE_LEN
+	jz		.Ldone
+	_nh_stride	K2, K3, K0, K1, 0x20
+
+.Ldone:
+	// Sum the accumulators for each pass, then store the sums to 'hash'
+	movdqa		PASS0_SUMS, T0
+	movdqa		PASS2_SUMS, T1
+	punpcklqdq	PASS1_SUMS, T0		// => (PASS0_SUM_A PASS1_SUM_A)
+	punpcklqdq	PASS3_SUMS, T1		// => (PASS2_SUM_A PASS3_SUM_A)
+	punpckhqdq	PASS1_SUMS, PASS0_SUMS	// => (PASS0_SUM_B PASS1_SUM_B)
+	punpckhqdq	PASS3_SUMS, PASS2_SUMS	// => (PASS2_SUM_B PASS3_SUM_B)
+	paddq		PASS0_SUMS, T0
+	paddq		PASS2_SUMS, T1
+	movdqu		T0, 0x00(HASH)
+	movdqu		T1, 0x10(HASH)
+	RET
+SYM_FUNC_END(nh_sse2)
diff --git a/lib/crypto/x86/nh.h b/lib/crypto/x86/nh.h
new file mode 100644
index 000000000000..83361c2e9783
--- /dev/null
+++ b/lib/crypto/x86/nh.h
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * x86_64 accelerated implementation of NH
+ *
+ * Copyright 2018 Google LLC
+ */
+
+#include <asm/fpu/api.h>
+#include <linux/static_call.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sse2);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_avx2);
+
+asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
+			__le64 hash[NH_NUM_PASSES]);
+asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
+			__le64 hash[NH_NUM_PASSES]);
+
+static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
+		    __le64 hash[NH_NUM_PASSES])
+{
+	if (message_len >= 64 && static_branch_likely(&have_sse2) &&
+	    irq_fpu_usable()) {
+		kernel_fpu_begin();
+		if (static_branch_likely(&have_avx2))
+			nh_avx2(key, message, message_len, hash);
+		else
+			nh_sse2(key, message, message_len, hash);
+		kernel_fpu_end();
+		return true;
+	}
+	return false;
+}
+
+#define nh_mod_init_arch nh_mod_init_arch
+static void nh_mod_init_arch(void)
+{
+	if (boot_cpu_has(X86_FEATURE_XMM2)) {
+		static_branch_enable(&have_sse2);
+		if (boot_cpu_has(X86_FEATURE_AVX2) &&
+		    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
+				      NULL))
+			static_branch_enable(&have_avx2);
+	}
+}
diff --git a/lib/hexdump.c b/lib/hexdump.c
index c3db7c3a7643..2e5cd8c24769 100644
--- a/lib/hexdump.c
+++ b/lib/hexdump.c
@@ -6,6 +6,7 @@
 #include <linux/types.h>
 #include <linux/ctype.h>
 #include <linux/errno.h>
+#include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/minmax.h>
 #include <linux/export.h>
diff --git a/lib/idr.c b/lib/idr.c
index e2adc457abb4..6c09515a4018 100644
--- a/lib/idr.c
+++ b/lib/idr.c
@@ -40,6 +40,8 @@ int idr_alloc_u32(struct idr *idr, void *ptr, u32 *nextid,
 
 	if (WARN_ON_ONCE(!(idr->idr_rt.xa_flags & ROOT_IS_IDR)))
 		idr->idr_rt.xa_flags |= IDR_RT_MARKER;
+	if (max < base)
+		return -ENOSPC;
 
 	id = (id < base) ? 0 : id - base;
 	radix_tree_iter_init(&iter, id);
@@ -622,7 +624,6 @@ void ida_destroy(struct ida *ida)
 EXPORT_SYMBOL(ida_destroy);
 
 #ifndef __KERNEL__
-extern void xa_dump_index(unsigned long index, unsigned int shift);
 #define IDA_CHUNK_SHIFT		ilog2(IDA_BITMAP_BITS)
 
 static void ida_dump_entry(void *entry, unsigned long index)
diff --git a/lib/kunit/Kconfig b/lib/kunit/Kconfig
index 50ecf55d2b9c..498cc51e493d 100644
--- a/lib/kunit/Kconfig
+++ b/lib/kunit/Kconfig
@@ -28,7 +28,7 @@ config KUNIT_FAULT_TEST
 	bool "Enable KUnit tests which print BUG stacktraces"
 	depends on KUNIT_TEST
 	depends on !UML
-	default y
+	default !PANIC_ON_OOPS
 	help
 	  Enables fault handling tests for the KUnit framework. These tests may
 	  trigger a kernel BUG(), and the associated stack trace, even when they
diff --git a/lib/kunit/device.c b/lib/kunit/device.c
index 520c1fccee8a..f201aaacd4cf 100644
--- a/lib/kunit/device.c
+++ b/lib/kunit/device.c
@@ -106,8 +106,7 @@ EXPORT_SYMBOL_GPL(kunit_driver_create);
 
 /* Helper which creates a kunit_device, attaches it to the kunit_bus*/
 static struct kunit_device *kunit_device_register_internal(struct kunit *test,
-							   const char *name,
-							   const struct device_driver *drv)
+							   const char *name)
 {
 	struct kunit_device *kunit_dev;
 	int err = -ENOMEM;
@@ -150,7 +149,7 @@ struct device *kunit_device_register_with_driver(struct kunit *test,
 						 const char *name,
 						 const struct device_driver *drv)
 {
-	struct kunit_device *kunit_dev = kunit_device_register_internal(test, name, drv);
+	struct kunit_device *kunit_dev = kunit_device_register_internal(test, name);
 
 	if (IS_ERR_OR_NULL(kunit_dev))
 		return ERR_CAST(kunit_dev);
@@ -172,7 +171,7 @@ struct device *kunit_device_register(struct kunit *test, const char *name)
 	if (IS_ERR(drv))
 		return ERR_CAST(drv);
 
-	dev = kunit_device_register_internal(test, name, drv);
+	dev = kunit_device_register_internal(test, name);
 	if (IS_ERR(dev)) {
 		kunit_release_action(test, driver_unregister_wrapper, (void *)drv);
 		return ERR_CAST(dev);
diff --git a/lib/percpu_counter_tree.c b/lib/percpu_counter_tree.c
new file mode 100644
index 000000000000..9b7ae7df5530
--- /dev/null
+++ b/lib/percpu_counter_tree.c
@@ -0,0 +1,705 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+// SPDX-FileCopyrightText: 2025 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+
+/*
+ * Split Counters With Tree Approximation Propagation
+ *
+ * * Propagation diagram when reaching batch size thresholds (± batch size):
+ *
+ * Example diagram for 8 CPUs:
+ *
+ * log2(8) = 3 levels
+ *
+ * At each level, each pair propagates its values to the next level when
+ * reaching the batch size thresholds.
+ *
+ * Counters at levels 0, 1, 2 can be kept on a single byte ([-128 .. +127] range),
+ * although it may be relevant to keep them on 32-bit counters for
+ * simplicity. (complexity vs memory footprint tradeoff)
+ *
+ * Counter at level 3 can be kept on a 32-bit counter.
+ *
+ * Level 0:  0    1    2    3    4    5    6    7
+ *           |   /     |   /     |   /     |   /
+ *           |  /      |  /      |  /      |  /
+ *           | /       | /       | /       | /
+ * Level 1:  0         1         2         3
+ *           |       /           |       /
+ *           |    /              |    /
+ *           | /                 | /
+ * Level 2:  0                   1
+ *           |               /
+ *           |         /
+ *           |   /
+ * Level 3:  0
+ *
+ * * Approximation accuracy:
+ *
+ * BATCH(level N): Level N batch size.
+ *
+ * Example for BATCH(level 0) = 32.
+ *
+ * BATCH(level 0) =  32
+ * BATCH(level 1) =  64
+ * BATCH(level 2) = 128
+ * BATCH(level N) = BATCH(level 0) * 2^N
+ *
+ *            per-counter     global
+ *            accuracy        accuracy
+ * Level 0:   [ -32 ..  +31]  ±256  (8 * 32)
+ * Level 1:   [ -64 ..  +63]  ±256  (4 * 64)
+ * Level 2:   [-128 .. +127]  ±256  (2 * 128)
+ * Total:      ------         ±768  (log2(nr_cpu_ids) * BATCH(level 0) * nr_cpu_ids)
+ *
+ * Note that the global accuracy can be calculated more precisely
+ * by taking into account that the positive accuracy range is
+ * 31 rather than 32.
+ *
+ * -----
+ *
+ * Approximate Sum Carry Propagation
+ *
+ * Let's define a number of counter bits for each level, e.g.:
+ *
+ * log2(BATCH(level 0)) = log2(32) = 5
+ *
+ *               nr_bit        value_mask                      range
+ * Level 0:      5 bits        v                             0 ..  +31
+ * Level 1:      1 bit        (v & ~((1UL << 5) - 1))        0 ..  +63
+ * Level 2:      1 bit        (v & ~((1UL << 6) - 1))        0 .. +127
+ * Level 3:     25 bits       (v & ~((1UL << 7) - 1))        0 .. 2^32-1
+ *
+ * Note: Use a full 32-bit per-cpu counter at level 0 to allow precise sum.
+ *
+ * Note: Use cacheline aligned counters at levels above 0 to prevent false sharing.
+ *       If memory footprint is an issue, a specialized allocator could be used
+ *       to eliminate padding.
+ *
+ * Example with expanded values:
+ *
+ * counter_add(counter, inc):
+ *
+ *         if (!inc)
+ *                 return;
+ *
+ *         res = percpu_add_return(counter @ Level 0, inc);
+ *         orig = res - inc;
+ *         if (inc < 0) {
+ *                 inc = -(-inc & ~0b00011111);  // Clear used bits
+ *                 // xor bit 5: underflow
+ *                 if ((inc ^ orig ^ res) & 0b00100000)
+ *                         inc -= 0b00100000;
+ *         } else {
+ *                 inc &= ~0b00011111;           // Clear used bits
+ *                 // xor bit 5: overflow
+ *                 if ((inc ^ orig ^ res) & 0b00100000)
+ *                         inc += 0b00100000;
+ *         }
+ *         if (!inc)
+ *                 return;
+ *
+ *         res = atomic_add_return(counter @ Level 1, inc);
+ *         orig = res - inc;
+ *         if (inc < 0) {
+ *                 inc = -(-inc & ~0b00111111);  // Clear used bits
+ *                 // xor bit 6: underflow
+ *                 if ((inc ^ orig ^ res) & 0b01000000)
+ *                         inc -= 0b01000000;
+ *         } else {
+ *                 inc &= ~0b00111111;           // Clear used bits
+ *                 // xor bit 6: overflow
+ *                 if ((inc ^ orig ^ res) & 0b01000000)
+ *                         inc += 0b01000000;
+ *         }
+ *         if (!inc)
+ *                 return;
+ *
+ *         res = atomic_add_return(counter @ Level 2, inc);
+ *         orig = res - inc;
+ *         if (inc < 0) {
+ *                 inc = -(-inc & ~0b01111111);  // Clear used bits
+ *                 // xor bit 7: underflow
+ *                 if ((inc ^ orig ^ res) & 0b10000000)
+ *                         inc -= 0b10000000;
+ *         } else {
+ *                 inc &= ~0b01111111;           // Clear used bits
+ *                 // xor bit 7: overflow
+ *                 if ((inc ^ orig ^ res) & 0b10000000)
+ *                         inc += 0b10000000;
+ *         }
+ *         if (!inc)
+ *                 return;
+ *
+ *         atomic_add(counter @ Level 3, inc);
+ */
+
+#include <linux/percpu_counter_tree.h>
+#include <linux/cpumask.h>
+#include <linux/percpu.h>
+#include <linux/atomic.h>
+#include <linux/errno.h>
+#include <linux/slab.h>
+#include <linux/math.h>
+
+#define MAX_NR_LEVELS 5
+
+/*
+ * The counter configuration is selected at boot time based on the
+ * hardware topology.
+ */
+struct counter_config {
+	unsigned int nr_items;				/*
+							 * nr_items is the number of items in the tree for levels 1
+							 * up to and including the final level (approximate sum).
+							 * It excludes the level 0 per-CPU counters.
+							 */
+	unsigned char nr_levels;			/*
+							 * nr_levels is the number of hierarchical counter tree levels.
+							 * It excludes the final level (approximate sum).
+							 */
+	unsigned char n_arity_order[MAX_NR_LEVELS];	/*
+							 * n-arity of tree nodes for each level from
+							 * 0 to (nr_levels - 1).
+							 */
+};
+
+static const struct counter_config per_nr_cpu_order_config[] = {
+	[0] =	{ .nr_items = 0,	.nr_levels = 0,		.n_arity_order = { 0 } },
+	[1] =	{ .nr_items = 1,	.nr_levels = 1,		.n_arity_order = { 1 } },
+	[2] =	{ .nr_items = 3,	.nr_levels = 2,		.n_arity_order = { 1, 1 } },
+	[3] =	{ .nr_items = 7,	.nr_levels = 3,		.n_arity_order = { 1, 1, 1 } },
+	[4] =	{ .nr_items = 7,	.nr_levels = 3,		.n_arity_order = { 2, 1, 1 } },
+	[5] =	{ .nr_items = 11,	.nr_levels = 3,		.n_arity_order = { 2, 2, 1 } },
+	[6] =	{ .nr_items = 21,	.nr_levels = 3,		.n_arity_order = { 2, 2, 2 } },
+	[7] =	{ .nr_items = 21,	.nr_levels = 3,		.n_arity_order = { 3, 2, 2 } },
+	[8] =	{ .nr_items = 37,	.nr_levels = 3,		.n_arity_order = { 3, 3, 2 } },
+	[9] =	{ .nr_items = 73,	.nr_levels = 3,		.n_arity_order = { 3, 3, 3 } },
+	[10] =	{ .nr_items = 149,	.nr_levels = 4,		.n_arity_order = { 3, 3, 2, 2 } },
+	[11] =	{ .nr_items = 293,	.nr_levels = 4,		.n_arity_order = { 3, 3, 3, 2 } },
+	[12] =	{ .nr_items = 585,	.nr_levels = 4,		.n_arity_order = { 3, 3, 3, 3 } },
+	[13] =	{ .nr_items = 1173,	.nr_levels = 5,		.n_arity_order = { 3, 3, 3, 2, 2 } },
+	[14] =	{ .nr_items = 2341,	.nr_levels = 5,		.n_arity_order = { 3, 3, 3, 3, 2 } },
+	[15] =	{ .nr_items = 4681,	.nr_levels = 5,		.n_arity_order = { 3, 3, 3, 3, 3 } },
+	[16] =	{ .nr_items = 4681,	.nr_levels = 5,		.n_arity_order = { 4, 3, 3, 3, 3 } },
+	[17] =	{ .nr_items = 8777,	.nr_levels = 5,		.n_arity_order = { 4, 4, 3, 3, 3 } },
+	[18] =	{ .nr_items = 17481,	.nr_levels = 5,		.n_arity_order = { 4, 4, 4, 3, 3 } },
+	[19] =	{ .nr_items = 34953,	.nr_levels = 5,		.n_arity_order = { 4, 4, 4, 4, 3 } },
+	[20] =	{ .nr_items = 69905,	.nr_levels = 5,		.n_arity_order = { 4, 4, 4, 4, 4 } },
+};
+
+static const struct counter_config *counter_config;	/* Hierarchical counter configuration for the hardware topology. */
+static unsigned int nr_cpus_order,			/* Order of nr_cpu_ids. */
+		    accuracy_multiplier;		/* Calculate accuracy for a given batch size (multiplication factor). */
+
+static
+int __percpu_counter_tree_init(struct percpu_counter_tree *counter,
+			       unsigned int batch_size, gfp_t gfp_flags,
+			       unsigned int __percpu *level0,
+			       struct percpu_counter_tree_level_item *items)
+{
+	/* Batch size must be greater than 1, and a power of 2. */
+	if (WARN_ON(batch_size <= 1 || (batch_size & (batch_size - 1))))
+		return -EINVAL;
+	counter->batch_size = batch_size;
+	counter->bias = 0;
+	counter->level0 = level0;
+	counter->items = items;
+	if (!nr_cpus_order) {
+		counter->approx_sum.i = per_cpu_ptr(counter->level0, 0);
+		counter->level0_bit_mask = 0;
+	} else {
+		counter->approx_sum.a = &counter->items[counter_config->nr_items - 1].count;
+		counter->level0_bit_mask = 1UL << get_count_order(batch_size);
+	}
+	/*
+	 * Each tree item signed integer has a negative range which is
+	 * one unit greater than the positive range.
+	 */
+	counter->approx_accuracy_range.under = batch_size * accuracy_multiplier;
+	counter->approx_accuracy_range.over = (batch_size - 1) * accuracy_multiplier;
+	return 0;
+}
+
+/**
+ * percpu_counter_tree_init_many() - Initialize many per-CPU counter trees.
+ * @counters: An array of @nr_counters counters to initialize.
+ *	      Their memory is provided by the caller.
+ * @items: Pointer to memory area where to store tree items.
+ *	   This memory is provided by the caller.
+ *	   Its size needs to be at least @nr_counters * percpu_counter_tree_items_size().
+ * @nr_counters: The number of counter trees to initialize
+ * @batch_size: The batch size is the increment step at level 0 which triggers a
+ * 		carry propagation.
+ *		The batch size is required to be greater than 1, and a power of 2.
+ * @gfp_flags: gfp flags to pass to the per-CPU allocator.
+ *
+ * Initialize many per-CPU counter trees using a single per-CPU
+ * allocator invocation for @nr_counters counters.
+ *
+ * Return:
+ * * %0: Success
+ * * %-EINVAL:		- Invalid @batch_size argument
+ * * %-ENOMEM:		- Out of memory
+ */
+int percpu_counter_tree_init_many(struct percpu_counter_tree *counters, struct percpu_counter_tree_level_item *items,
+				  unsigned int nr_counters, unsigned int batch_size, gfp_t gfp_flags)
+{
+	void __percpu *level0, *level0_iter;
+	size_t counter_size, items_size = 0;
+	void *items_iter;
+	unsigned int i;
+	int ret;
+
+	counter_size = ALIGN(sizeof(*counters), __alignof__(*counters));
+	level0 = __alloc_percpu_gfp(nr_counters * counter_size,
+				    __alignof__(*counters), gfp_flags);
+	if (!level0)
+		return -ENOMEM;
+	if (nr_cpus_order) {
+		items_size = percpu_counter_tree_items_size();
+		memset(items, 0, items_size * nr_counters);
+	}
+	level0_iter = level0;
+	items_iter = items;
+	for (i = 0; i < nr_counters; i++) {
+		ret = __percpu_counter_tree_init(&counters[i], batch_size, gfp_flags, level0_iter, items_iter);
+		if (ret)
+			goto free_level0;
+		level0_iter += counter_size;
+		if (nr_cpus_order)
+			items_iter += items_size;
+	}
+	return 0;
+
+free_level0:
+	free_percpu(level0);
+	return ret;
+}
+
+/**
+ * percpu_counter_tree_init() - Initialize one per-CPU counter tree.
+ * @counter: Counter to initialize.
+ *	     Its memory is provided by the caller.
+ * @items: Pointer to memory area where to store tree items.
+ *	   This memory is provided by the caller.
+ *	   Its size needs to be at least percpu_counter_tree_items_size().
+ * @batch_size: The batch size is the increment step at level 0 which triggers a
+ * 		carry propagation.
+ *		The batch size is required to be greater than 1, and a power of 2.
+ * @gfp_flags: gfp flags to pass to the per-CPU allocator.
+ *
+ * Initialize one per-CPU counter tree.
+ *
+ * Return:
+ * * %0: Success
+ * * %-EINVAL:		- Invalid @batch_size argument
+ * * %-ENOMEM:		- Out of memory
+ */
+int percpu_counter_tree_init(struct percpu_counter_tree *counter, struct percpu_counter_tree_level_item *items,
+			     unsigned int batch_size, gfp_t gfp_flags)
+{
+	return percpu_counter_tree_init_many(counter, items, 1, batch_size, gfp_flags);
+}
+
+/**
+ * percpu_counter_tree_destroy_many() - Destroy many per-CPU counter trees.
+ * @counters: Array of counters trees to destroy.
+ * @nr_counters: The number of counter trees to destroy.
+ *
+ * Release internal resources allocated for @nr_counters per-CPU counter trees.
+ */
+
+void percpu_counter_tree_destroy_many(struct percpu_counter_tree *counters, unsigned int nr_counters)
+{
+	free_percpu(counters->level0);
+}
+
+/**
+ * percpu_counter_tree_destroy() - Destroy one per-CPU counter tree.
+ * @counter: Counter to destroy.
+ *
+ * Release internal resources allocated for one per-CPU counter tree.
+ */
+void percpu_counter_tree_destroy(struct percpu_counter_tree *counter)
+{
+	return percpu_counter_tree_destroy_many(counter, 1);
+}
+
+static
+int percpu_counter_tree_carry(int orig, int res, int inc, unsigned int bit_mask)
+{
+	if (inc < 0) {
+		inc = -(-inc & ~(bit_mask - 1));
+		/*
+		 * xor bit_mask: underflow.
+		 *
+		 * If inc has bit set, decrement an additional bit if
+		 * there is _no_ bit transition between orig and res.
+		 * Else, inc has bit cleared, decrement an additional
+		 * bit if there is a bit transition between orig and
+		 * res.
+		 */
+		if ((inc ^ orig ^ res) & bit_mask)
+			inc -= bit_mask;
+	} else {
+		inc &= ~(bit_mask - 1);
+		/*
+		 * xor bit_mask: overflow.
+		 *
+		 * If inc has bit set, increment an additional bit if
+		 * there is _no_ bit transition between orig and res.
+		 * Else, inc has bit cleared, increment an additional
+		 * bit if there is a bit transition between orig and
+		 * res.
+		 */
+		if ((inc ^ orig ^ res) & bit_mask)
+			inc += bit_mask;
+	}
+	return inc;
+}
+
+/*
+ * It does not matter through which path the carry propagates up the
+ * tree, therefore there is no need to disable preemption because the
+ * cpu number is only used to favor cache locality.
+ */
+static
+void percpu_counter_tree_add_slowpath(struct percpu_counter_tree *counter, int inc)
+{
+	unsigned int level_items, nr_levels = counter_config->nr_levels,
+		     level, n_arity_order, bit_mask;
+	struct percpu_counter_tree_level_item *item = counter->items;
+	unsigned int cpu = raw_smp_processor_id();
+
+	WARN_ON_ONCE(!nr_cpus_order);	/* Should never be called for 1 cpu. */
+
+	n_arity_order = counter_config->n_arity_order[0];
+	bit_mask = counter->level0_bit_mask << n_arity_order;
+	level_items = 1U << (nr_cpus_order - n_arity_order);
+
+	for (level = 1; level < nr_levels; level++) {
+		atomic_t *count = &item[cpu & (level_items - 1)].count;
+		unsigned int orig, res;
+
+		res = atomic_add_return_relaxed(inc, count);
+		orig = res - inc;
+		inc = percpu_counter_tree_carry(orig, res, inc, bit_mask);
+		if (likely(!inc))
+			return;
+		item += level_items;
+		n_arity_order = counter_config->n_arity_order[level];
+		level_items >>= n_arity_order;
+		bit_mask <<= n_arity_order;
+	}
+	atomic_add(inc, counter->approx_sum.a);
+}
+
+/**
+ * percpu_counter_tree_add() - Add to a per-CPU counter tree.
+ * @counter: Counter added to.
+ * @inc: Increment value (either positive or negative).
+ *
+ * Add @inc to a per-CPU counter tree. This is a fast-path which will
+ * typically increment per-CPU counters as long as there is no carry
+ * greater or equal to the counter tree batch size.
+ */
+void percpu_counter_tree_add(struct percpu_counter_tree *counter, int inc)
+{
+	unsigned int bit_mask = counter->level0_bit_mask, orig, res;
+
+	res = this_cpu_add_return(*counter->level0, inc);
+	orig = res - inc;
+	inc = percpu_counter_tree_carry(orig, res, inc, bit_mask);
+	if (likely(!inc))
+		return;
+	percpu_counter_tree_add_slowpath(counter, inc);
+}
+
+
+static
+int percpu_counter_tree_precise_sum_unbiased(struct percpu_counter_tree *counter)
+{
+	unsigned int sum = 0;
+	int cpu;
+
+	for_each_possible_cpu(cpu)
+		sum += *per_cpu_ptr(counter->level0, cpu);
+	return (int) sum;
+}
+
+/**
+ * percpu_counter_tree_precise_sum() - Return precise counter sum.
+ * @counter: The counter to sum.
+ *
+ * Querying the precise sum is relatively expensive because it needs to
+ * iterate over all CPUs.
+ * This is meant to be used when accuracy is preferred over speed.
+ *
+ * Return: The current precise counter sum.
+ */
+int percpu_counter_tree_precise_sum(struct percpu_counter_tree *counter)
+{
+	return percpu_counter_tree_precise_sum_unbiased(counter) + READ_ONCE(counter->bias);
+}
+
+static
+int compare_delta(int delta, unsigned int accuracy_neg, unsigned int accuracy_pos)
+{
+	if (delta >= 0) {
+		if (delta <= accuracy_pos)
+			return 0;
+		else
+			return 1;
+	} else {
+		if (-delta <= accuracy_neg)
+			return 0;
+		else
+			return -1;
+	}
+}
+
+/**
+ * percpu_counter_tree_approximate_compare - Approximated comparison of two counter trees.
+ * @a: First counter to compare.
+ * @b: Second counter to compare.
+ *
+ * Evaluate an approximate comparison of two counter trees.
+ * This approximation comparison is fast, and provides an accurate
+ * answer if the counters are found to be either less than or greater
+ * than the other. However, if the approximated comparison returns
+ * 0, the counters respective sums are found to be within the two
+ * counters accuracy range.
+ *
+ * Return:
+ * * %0		- Counters @a and @b do not differ by more than the sum of their respective
+ *                accuracy ranges.
+ * * %-1	- Counter @a less than counter @b.
+ * * %1		- Counter @a is greater than counter @b.
+ */
+int percpu_counter_tree_approximate_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b)
+{
+	return compare_delta(percpu_counter_tree_approximate_sum(a) - percpu_counter_tree_approximate_sum(b),
+			     a->approx_accuracy_range.over + b->approx_accuracy_range.under,
+			     a->approx_accuracy_range.under + b->approx_accuracy_range.over);
+}
+
+/**
+ * percpu_counter_tree_approximate_compare_value - Approximated comparison of a counter tree against a given value.
+ * @counter: Counter to compare.
+ * @v: Value to compare.
+ *
+ * Evaluate an approximate comparison of a counter tree against a given value.
+ * This approximation comparison is fast, and provides an accurate
+ * answer if the counter is found to be either less than or greater
+ * than the value. However, if the approximated comparison returns
+ * 0, the value is within the counter accuracy range.
+ *
+ * Return:
+ * * %0		- The value @v is within the accuracy range of the counter.
+ * * %-1	- The value @v is less than the counter.
+ * * %1		- The value @v is greater than the counter.
+ */
+int percpu_counter_tree_approximate_compare_value(struct percpu_counter_tree *counter, int v)
+{
+	return compare_delta(v - percpu_counter_tree_approximate_sum(counter),
+			     counter->approx_accuracy_range.under,
+			     counter->approx_accuracy_range.over);
+}
+
+/**
+ * percpu_counter_tree_precise_compare - Precise comparison of two counter trees.
+ * @a: First counter to compare.
+ * @b: Second counter to compare.
+ *
+ * Evaluate a precise comparison of two counter trees.
+ * As an optimization, it uses the approximate counter comparison
+ * to quickly compare counters which are far apart. Only cases where
+ * counter sums are within the accuracy range require precise counter
+ * sums.
+ *
+ * Return:
+ * * %0		- Counters are equal.
+ * * %-1	- Counter @a less than counter @b.
+ * * %1		- Counter @a is greater than counter @b.
+ */
+int percpu_counter_tree_precise_compare(struct percpu_counter_tree *a, struct percpu_counter_tree *b)
+{
+	int count_a = percpu_counter_tree_approximate_sum(a),
+	    count_b = percpu_counter_tree_approximate_sum(b);
+	unsigned int accuracy_a, accuracy_b;
+	int delta = count_a - count_b;
+	int res;
+
+	res = compare_delta(delta,
+			    a->approx_accuracy_range.over + b->approx_accuracy_range.under,
+			    a->approx_accuracy_range.under + b->approx_accuracy_range.over);
+	/* The values are distanced enough for an accurate approximated comparison. */
+	if (res)
+		return res;
+
+	/*
+	 * The approximated comparison is within the accuracy range, therefore at least one
+	 * precise sum is needed. Sum the counter which has the largest accuracy first.
+	 */
+	if (delta >= 0) {
+		accuracy_a = a->approx_accuracy_range.under;
+		accuracy_b = b->approx_accuracy_range.over;
+	} else {
+		accuracy_a = a->approx_accuracy_range.over;
+		accuracy_b = b->approx_accuracy_range.under;
+	}
+	if (accuracy_b < accuracy_a) {
+		count_a = percpu_counter_tree_precise_sum(a);
+		res = compare_delta(count_a - count_b,
+				    b->approx_accuracy_range.under,
+				    b->approx_accuracy_range.over);
+		if (res)
+			return res;
+		/* Precise sum of second counter is required. */
+		count_b = percpu_counter_tree_precise_sum(b);
+	} else {
+		count_b = percpu_counter_tree_precise_sum(b);
+		res = compare_delta(count_a - count_b,
+				    a->approx_accuracy_range.over,
+				    a->approx_accuracy_range.under);
+		if (res)
+			return res;
+		/* Precise sum of second counter is required. */
+		count_a = percpu_counter_tree_precise_sum(a);
+	}
+	if (count_a - count_b < 0)
+		return -1;
+	if (count_a - count_b > 0)
+		return 1;
+	return 0;
+}
+
+/**
+ * percpu_counter_tree_precise_compare_value - Precise comparison of a counter tree against a given value.
+ * @counter: Counter to compare.
+ * @v: Value to compare.
+ *
+ * Evaluate a precise comparison of a counter tree against a given value.
+ * As an optimization, it uses the approximate counter comparison
+ * to quickly identify whether the counter and value are far apart.
+ * Only cases where the value is within the counter accuracy range
+ * require a precise counter sum.
+ *
+ * Return:
+ * * %0		- The value @v is equal to the counter.
+ * * %-1	- The value @v is less than the counter.
+ * * %1		- The value @v is greater than the counter.
+ */
+int percpu_counter_tree_precise_compare_value(struct percpu_counter_tree *counter, int v)
+{
+	int count = percpu_counter_tree_approximate_sum(counter);
+	int res;
+
+	res = compare_delta(v - count,
+			    counter->approx_accuracy_range.under,
+			    counter->approx_accuracy_range.over);
+	/* The values are distanced enough for an accurate approximated comparison. */
+	if (res)
+		return res;
+
+	/* Precise sum is required. */
+	count = percpu_counter_tree_precise_sum(counter);
+	if (v - count < 0)
+		return -1;
+	if (v - count > 0)
+		return 1;
+	return 0;
+}
+
+static
+void percpu_counter_tree_set_bias(struct percpu_counter_tree *counter, int bias)
+{
+	WRITE_ONCE(counter->bias, bias);
+}
+
+/**
+ * percpu_counter_tree_set - Set the counter tree sum to a given value.
+ * @counter: Counter to set.
+ * @v: Value to set.
+ *
+ * Set the counter sum to a given value. It can be useful for instance
+ * to reset the counter sum to 0. Note that even after setting the
+ * counter sum to a given value, the counter sum approximation can
+ * return any value within the accuracy range around that value.
+ */
+void percpu_counter_tree_set(struct percpu_counter_tree *counter, int v)
+{
+	percpu_counter_tree_set_bias(counter,
+				     v - percpu_counter_tree_precise_sum_unbiased(counter));
+}
+
+/**
+ * percpu_counter_tree_approximate_accuracy_range - Query the accuracy range for a counter tree.
+ * @counter: Counter to query.
+ * @under: Pointer where to store the to the accuracy range below the approximation.
+ * @over: Pointer where to store the to the accuracy range above the approximation.
+ *
+ * Query the accuracy range limits for the counter.
+ * Because of two's complement binary representation, the "under" range is typically
+ * slightly larger than the "over" range.
+ * Those values are derived from the hardware topology and the counter tree batch size.
+ * They are invariant for a given counter tree.
+ * Using this function should not be typically required, see the following functions instead:
+ * * percpu_counter_tree_approximate_compare(),
+ * * percpu_counter_tree_approximate_compare_value(),
+ * * percpu_counter_tree_precise_compare(),
+ * * percpu_counter_tree_precise_compare_value().
+ */
+void percpu_counter_tree_approximate_accuracy_range(struct percpu_counter_tree *counter,
+						    unsigned int *under, unsigned int *over)
+{
+	*under = counter->approx_accuracy_range.under;
+	*over = counter->approx_accuracy_range.over;
+}
+
+/*
+ * percpu_counter_tree_items_size - Query the size required for counter tree items.
+ *
+ * Query the size of the memory area required to hold the counter tree
+ * items. This depends on the hardware topology and is invariant after
+ * boot.
+ *
+ * Return: Size required to hold tree items.
+ */
+size_t percpu_counter_tree_items_size(void)
+{
+	if (!nr_cpus_order)
+		return 0;
+	return counter_config->nr_items * sizeof(struct percpu_counter_tree_level_item);
+}
+
+static void __init calculate_accuracy_topology(void)
+{
+	unsigned int nr_levels = counter_config->nr_levels, level;
+	unsigned int level_items = 1U << nr_cpus_order;
+	unsigned int batch_size = 1;
+
+	for (level = 0; level < nr_levels; level++) {
+		unsigned int n_arity_order = counter_config->n_arity_order[level];
+
+		/*
+		 * The accuracy multiplier is derived from a batch size of 1
+		 * to speed up calculating the accuracy at tree initialization.
+		 */
+		accuracy_multiplier += batch_size * level_items;
+		batch_size <<= n_arity_order;
+		level_items >>= n_arity_order;
+	}
+}
+
+int __init percpu_counter_tree_subsystem_init(void)
+{
+	nr_cpus_order = get_count_order(nr_cpu_ids);
+	if (WARN_ON_ONCE(nr_cpus_order >= ARRAY_SIZE(per_nr_cpu_order_config))) {
+		printk(KERN_ERR "Unsupported number of CPUs (%u)\n", nr_cpu_ids);
+		return -1;
+	}
+	counter_config = &per_nr_cpu_order_config[nr_cpus_order];
+	calculate_accuracy_topology();
+	return 0;
+}
diff --git a/lib/string_helpers.c b/lib/string_helpers.c
index ffb8ead6d4cd..8cb6f66c9c2b 100644
--- a/lib/string_helpers.c
+++ b/lib/string_helpers.c
@@ -13,6 +13,7 @@
 #include <linux/device.h>
 #include <linux/errno.h>
 #include <linux/fs.h>
+#include <linux/hex.h>
 #include <linux/limits.h>
 #include <linux/mm.h>
 #include <linux/slab.h>
diff --git a/lib/test_uuid.c b/lib/test_uuid.c
deleted file mode 100644
index 0124fad5d72c..000000000000
--- a/lib/test_uuid.c
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Test cases for lib/uuid.c module.
- */
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/string.h>
-#include <linux/uuid.h>
-
-struct test_uuid_data {
-	const char *uuid;
-	guid_t le;
-	uuid_t be;
-};
-
-static const struct test_uuid_data test_uuid_test_data[] = {
-	{
-		.uuid = "c33f4995-3701-450e-9fbf-206a2e98e576",
-		.le = GUID_INIT(0xc33f4995, 0x3701, 0x450e, 0x9f, 0xbf, 0x20, 0x6a, 0x2e, 0x98, 0xe5, 0x76),
-		.be = UUID_INIT(0xc33f4995, 0x3701, 0x450e, 0x9f, 0xbf, 0x20, 0x6a, 0x2e, 0x98, 0xe5, 0x76),
-	},
-	{
-		.uuid = "64b4371c-77c1-48f9-8221-29f054fc023b",
-		.le = GUID_INIT(0x64b4371c, 0x77c1, 0x48f9, 0x82, 0x21, 0x29, 0xf0, 0x54, 0xfc, 0x02, 0x3b),
-		.be = UUID_INIT(0x64b4371c, 0x77c1, 0x48f9, 0x82, 0x21, 0x29, 0xf0, 0x54, 0xfc, 0x02, 0x3b),
-	},
-	{
-		.uuid = "0cb4ddff-a545-4401-9d06-688af53e7f84",
-		.le = GUID_INIT(0x0cb4ddff, 0xa545, 0x4401, 0x9d, 0x06, 0x68, 0x8a, 0xf5, 0x3e, 0x7f, 0x84),
-		.be = UUID_INIT(0x0cb4ddff, 0xa545, 0x4401, 0x9d, 0x06, 0x68, 0x8a, 0xf5, 0x3e, 0x7f, 0x84),
-	},
-};
-
-static const char * const test_uuid_wrong_data[] = {
-	"c33f4995-3701-450e-9fbf206a2e98e576 ",	/* no hyphen(s) */
-	"64b4371c-77c1-48f9-8221-29f054XX023b",	/* invalid character(s) */
-	"0cb4ddff-a545-4401-9d06-688af53e",	/* not enough data */
-};
-
-static unsigned total_tests __initdata;
-static unsigned failed_tests __initdata;
-
-static void __init test_uuid_failed(const char *prefix, bool wrong, bool be,
-				    const char *data, const char *actual)
-{
-	pr_err("%s test #%u %s %s data: '%s'\n",
-	       prefix,
-	       total_tests,
-	       wrong ? "passed on wrong" : "failed on",
-	       be ? "BE" : "LE",
-	       data);
-	if (actual && *actual)
-		pr_err("%s test #%u actual data: '%s'\n",
-		       prefix,
-		       total_tests,
-		       actual);
-	failed_tests++;
-}
-
-static void __init test_uuid_test(const struct test_uuid_data *data)
-{
-	guid_t le;
-	uuid_t be;
-	char buf[48];
-
-	/* LE */
-	total_tests++;
-	if (guid_parse(data->uuid, &le))
-		test_uuid_failed("conversion", false, false, data->uuid, NULL);
-
-	total_tests++;
-	if (!guid_equal(&data->le, &le)) {
-		sprintf(buf, "%pUl", &le);
-		test_uuid_failed("cmp", false, false, data->uuid, buf);
-	}
-
-	/* BE */
-	total_tests++;
-	if (uuid_parse(data->uuid, &be))
-		test_uuid_failed("conversion", false, true, data->uuid, NULL);
-
-	total_tests++;
-	if (!uuid_equal(&data->be, &be)) {
-		sprintf(buf, "%pUb", &be);
-		test_uuid_failed("cmp", false, true, data->uuid, buf);
-	}
-}
-
-static void __init test_uuid_wrong(const char *data)
-{
-	guid_t le;
-	uuid_t be;
-
-	/* LE */
-	total_tests++;
-	if (!guid_parse(data, &le))
-		test_uuid_failed("negative", true, false, data, NULL);
-
-	/* BE */
-	total_tests++;
-	if (!uuid_parse(data, &be))
-		test_uuid_failed("negative", true, true, data, NULL);
-}
-
-static int __init test_uuid_init(void)
-{
-	unsigned int i;
-
-	for (i = 0; i < ARRAY_SIZE(test_uuid_test_data); i++)
-		test_uuid_test(&test_uuid_test_data[i]);
-
-	for (i = 0; i < ARRAY_SIZE(test_uuid_wrong_data); i++)
-		test_uuid_wrong(test_uuid_wrong_data[i]);
-
-	if (failed_tests == 0)
-		pr_info("all %u tests passed\n", total_tests);
-	else
-		pr_err("failed %u out of %u tests\n", failed_tests, total_tests);
-
-	return failed_tests ? -EINVAL : 0;
-}
-module_init(test_uuid_init);
-
-static void __exit test_uuid_exit(void)
-{
-	/* do nothing */
-}
-module_exit(test_uuid_exit);
-
-MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");
-MODULE_DESCRIPTION("Test cases for lib/uuid.c module");
-MODULE_LICENSE("Dual BSD/GPL");
diff --git a/lib/test_xarray.c b/lib/test_xarray.c
index 5ca0aefee9aa..3c4e568a4df9 100644
--- a/lib/test_xarray.c
+++ b/lib/test_xarray.c
@@ -15,7 +15,7 @@ static unsigned int tests_passed;
 static const unsigned int order_limit =
 		IS_ENABLED(CONFIG_XARRAY_MULTI) ? BITS_PER_LONG : 1;
 
-#ifndef XA_DEBUG
+#ifndef CONFIG_DEBUG_XARRAY
 # ifdef __KERNEL__
 void xa_dump(const struct xarray *xa) { }
 # endif
@@ -123,11 +123,12 @@ static noinline void check_xas_retry(struct xarray *xa)
 
 	rcu_read_lock();
 	XA_BUG_ON(xa, !xa_is_internal(xas_reload(&xas)));
-	xas.xa_node = XAS_RESTART;
+	xas_reset(&xas);
 	XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0));
 	rcu_read_unlock();
 
 	/* Make sure we can iterate through retry entries */
+	xas_reset(&xas);
 	xas_lock(&xas);
 	xas_set(&xas, 0);
 	xas_store(&xas, XA_RETRY_ENTRY);
@@ -565,6 +566,7 @@ static noinline void check_xas_erase(struct xarray *xa)
 			xas_unlock(&xas);
 		} while (xas_nomem(&xas, GFP_KERNEL));
 
+		xas_reset(&xas);
 		xas_lock(&xas);
 		xas_store(&xas, NULL);
 
@@ -1759,6 +1761,7 @@ static noinline void check_create_range_5(struct xarray *xa,
 			xas_create_range(&xas);
 			xas_unlock(&xas);
 		} while (xas_nomem(&xas, GFP_KERNEL));
+		xas_reset(&xas);
 	}
 
 	xa_destroy(xa);
@@ -2035,6 +2038,7 @@ static noinline void check_workingset(struct xarray *xa, unsigned long index)
 
 	XA_BUG_ON(xa, list_empty(&shadow_nodes));
 
+	xas_reset(&xas);
 	xas_lock(&xas);
 	xas_next(&xas);
 	xas_store(&xas, &xas);
@@ -2127,8 +2131,8 @@ static noinline void check_xas_get_order(struct xarray *xa)
 				rcu_read_unlock();
 			}
 
-			xas_lock(&xas);
 			xas_set_order(&xas, i << order, order);
+			xas_lock(&xas);
 			xas_store(&xas, NULL);
 			xas_unlock(&xas);
 		}
diff --git a/lib/tests/Makefile b/lib/tests/Makefile
index 601dba4b7d96..26127fc72d78 100644
--- a/lib/tests/Makefile
+++ b/lib/tests/Makefile
@@ -28,6 +28,7 @@ obj-$(CONFIG_LIST_KUNIT_TEST) += list-test.o
 obj-$(CONFIG_KFIFO_KUNIT_TEST) += kfifo_kunit.o
 obj-$(CONFIG_TEST_LIST_SORT) += test_list_sort.o
 obj-$(CONFIG_LINEAR_RANGES_TEST) += test_linear_ranges.o
+obj-$(CONFIG_LIVEUPDATE_TEST) += liveupdate.o
 
 CFLAGS_longest_symbol_kunit.o += $(call cc-disable-warning, missing-prototypes)
 obj-$(CONFIG_LONGEST_SYM_KUNIT_TEST) += longest_symbol_kunit.o
@@ -49,5 +50,6 @@ obj-$(CONFIG_STRING_HELPERS_KUNIT_TEST) += string_helpers_kunit.o
 obj-$(CONFIG_USERCOPY_KUNIT_TEST) += usercopy_kunit.o
 obj-$(CONFIG_UTIL_MACROS_KUNIT) += util_macros_kunit.o
 obj-$(CONFIG_RATELIMIT_KUNIT_TEST) += test_ratelimit.o
+obj-$(CONFIG_UUID_KUNIT_TEST) += uuid_kunit.o
 
 obj-$(CONFIG_TEST_RUNTIME_MODULE)		+= module/
diff --git a/lib/tests/liveupdate.c b/lib/tests/liveupdate.c
new file mode 100644
index 000000000000..496d6ef91a30
--- /dev/null
+++ b/lib/tests/liveupdate.c
@@ -0,0 +1,158 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (c) 2025, Google LLC.
+ * Pasha Tatashin <pasha.tatashin@soleen.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME " test: " fmt
+
+#include <linux/cleanup.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/liveupdate.h>
+#include <linux/module.h>
+#include "../../kernel/liveupdate/luo_internal.h"
+
+static const struct liveupdate_flb_ops test_flb_ops;
+#define DEFINE_TEST_FLB(i) {						\
+	.ops = &test_flb_ops,						\
+	.compatible = LIVEUPDATE_TEST_FLB_COMPATIBLE(i),		\
+}
+
+/* Number of Test FLBs to register with every file handler */
+#define TEST_NFLBS 3
+static struct liveupdate_flb test_flbs[TEST_NFLBS] = {
+	DEFINE_TEST_FLB(0),
+	DEFINE_TEST_FLB(1),
+	DEFINE_TEST_FLB(2),
+};
+
+#define TEST_FLB_MAGIC_BASE 0xFEEDF00DCAFEBEE0ULL
+
+static int test_flb_preserve(struct liveupdate_flb_op_args *argp)
+{
+	ptrdiff_t index = argp->flb - test_flbs;
+
+	pr_info("%s: preserve was triggered\n", argp->flb->compatible);
+	argp->data = TEST_FLB_MAGIC_BASE + index;
+
+	return 0;
+}
+
+static void test_flb_unpreserve(struct liveupdate_flb_op_args *argp)
+{
+	pr_info("%s: unpreserve was triggered\n", argp->flb->compatible);
+}
+
+static int test_flb_retrieve(struct liveupdate_flb_op_args *argp)
+{
+	ptrdiff_t index = argp->flb - test_flbs;
+	u64 expected_data = TEST_FLB_MAGIC_BASE + index;
+
+	if (argp->data == expected_data) {
+		pr_info("%s: found flb data from the previous boot\n",
+			argp->flb->compatible);
+		argp->obj = (void *)argp->data;
+	} else {
+		pr_err("%s: ERROR - incorrect data handle: %llx, expected %llx\n",
+		       argp->flb->compatible, argp->data, expected_data);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void test_flb_finish(struct liveupdate_flb_op_args *argp)
+{
+	ptrdiff_t index = argp->flb - test_flbs;
+	void *expected_obj = (void *)(TEST_FLB_MAGIC_BASE + index);
+
+	if (argp->obj == expected_obj) {
+		pr_info("%s: finish was triggered\n", argp->flb->compatible);
+	} else {
+		pr_err("%s: ERROR - finish called with invalid object\n",
+		       argp->flb->compatible);
+	}
+}
+
+static const struct liveupdate_flb_ops test_flb_ops = {
+	.preserve	= test_flb_preserve,
+	.unpreserve	= test_flb_unpreserve,
+	.retrieve	= test_flb_retrieve,
+	.finish		= test_flb_finish,
+	.owner		= THIS_MODULE,
+};
+
+static void liveupdate_test_init(void)
+{
+	static DEFINE_MUTEX(init_lock);
+	static bool initialized;
+	int i;
+
+	guard(mutex)(&init_lock);
+
+	if (initialized)
+		return;
+
+	for (i = 0; i < TEST_NFLBS; i++) {
+		struct liveupdate_flb *flb = &test_flbs[i];
+		void *obj;
+		int err;
+
+		err = liveupdate_flb_get_incoming(flb, &obj);
+		if (err && err != -ENODATA && err != -ENOENT) {
+			pr_err("liveupdate_flb_get_incoming for %s failed: %pe\n",
+			       flb->compatible, ERR_PTR(err));
+		}
+	}
+	initialized = true;
+}
+
+void liveupdate_test_register(struct liveupdate_file_handler *fh)
+{
+	int err, i;
+
+	liveupdate_test_init();
+
+	for (i = 0; i < TEST_NFLBS; i++) {
+		struct liveupdate_flb *flb = &test_flbs[i];
+
+		err = liveupdate_register_flb(fh, flb);
+		if (err) {
+			pr_err("Failed to register %s %pe\n",
+			       flb->compatible, ERR_PTR(err));
+		}
+	}
+
+	err = liveupdate_register_flb(fh, &test_flbs[0]);
+	if (!err || err != -EEXIST) {
+		pr_err("Failed: %s should be already registered, but got err: %pe\n",
+		       test_flbs[0].compatible, ERR_PTR(err));
+	}
+
+	pr_info("Registered %d FLBs with file handler: [%s]\n",
+		TEST_NFLBS, fh->compatible);
+}
+
+void liveupdate_test_unregister(struct liveupdate_file_handler *fh)
+{
+	int err, i;
+
+	for (i = 0; i < TEST_NFLBS; i++) {
+		struct liveupdate_flb *flb = &test_flbs[i];
+
+		err = liveupdate_unregister_flb(fh, flb);
+		if (err) {
+			pr_err("Failed to unregister %s %pe\n",
+			       flb->compatible, ERR_PTR(err));
+		}
+	}
+
+	pr_info("Unregistered %d FLBs from file handler: [%s]\n",
+		TEST_NFLBS, fh->compatible);
+}
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Pasha Tatashin <pasha.tatashin@soleen.com>");
+MODULE_DESCRIPTION("In-kernel test for LUO mechanism");
diff --git a/lib/tests/uuid_kunit.c b/lib/tests/uuid_kunit.c
new file mode 100644
index 000000000000..de71b2649dac
--- /dev/null
+++ b/lib/tests/uuid_kunit.c
@@ -0,0 +1,106 @@
+// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
+/*
+ * Test cases for lib/uuid.c module.
+ */
+
+#include <kunit/test.h>
+#include <linux/uuid.h>
+
+struct test_uuid_data {
+	const char *uuid;
+	guid_t le;
+	uuid_t be;
+};
+
+static const struct test_uuid_data test_uuid_test_data[] = {
+	{
+		.uuid = "c33f4995-3701-450e-9fbf-206a2e98e576",
+		.le = GUID_INIT(0xc33f4995, 0x3701, 0x450e, 0x9f, 0xbf, 0x20, 0x6a, 0x2e, 0x98, 0xe5, 0x76),
+		.be = UUID_INIT(0xc33f4995, 0x3701, 0x450e, 0x9f, 0xbf, 0x20, 0x6a, 0x2e, 0x98, 0xe5, 0x76),
+	},
+	{
+		.uuid = "64b4371c-77c1-48f9-8221-29f054fc023b",
+		.le = GUID_INIT(0x64b4371c, 0x77c1, 0x48f9, 0x82, 0x21, 0x29, 0xf0, 0x54, 0xfc, 0x02, 0x3b),
+		.be = UUID_INIT(0x64b4371c, 0x77c1, 0x48f9, 0x82, 0x21, 0x29, 0xf0, 0x54, 0xfc, 0x02, 0x3b),
+	},
+	{
+		.uuid = "0cb4ddff-a545-4401-9d06-688af53e7f84",
+		.le = GUID_INIT(0x0cb4ddff, 0xa545, 0x4401, 0x9d, 0x06, 0x68, 0x8a, 0xf5, 0x3e, 0x7f, 0x84),
+		.be = UUID_INIT(0x0cb4ddff, 0xa545, 0x4401, 0x9d, 0x06, 0x68, 0x8a, 0xf5, 0x3e, 0x7f, 0x84),
+	},
+};
+
+static const char * const test_uuid_wrong_data[] = {
+	"c33f4995-3701-450e-9fbf206a2e98e576 ",	/* no hyphen(s) */
+	"64b4371c-77c1-48f9-8221-29f054XX023b",	/* invalid character(s) */
+	"0cb4ddff-a545-4401-9d06-688af53e",	/* not enough data */
+};
+
+static void uuid_test_guid_valid(struct kunit *test)
+{
+	unsigned int i;
+	const struct test_uuid_data *data;
+	guid_t le;
+
+	for (i = 0; i < ARRAY_SIZE(test_uuid_test_data); i++) {
+		data = &test_uuid_test_data[i];
+		KUNIT_EXPECT_EQ(test, guid_parse(data->uuid, &le), 0);
+		KUNIT_EXPECT_TRUE(test, guid_equal(&data->le, &le));
+	}
+}
+
+static void uuid_test_uuid_valid(struct kunit *test)
+{
+	unsigned int i;
+	const struct test_uuid_data *data;
+	uuid_t be;
+
+	for (i = 0; i < ARRAY_SIZE(test_uuid_test_data); i++) {
+		data = &test_uuid_test_data[i];
+		KUNIT_EXPECT_EQ(test, uuid_parse(data->uuid, &be), 0);
+		KUNIT_EXPECT_TRUE(test, uuid_equal(&data->be, &be));
+	}
+}
+
+static void uuid_test_guid_invalid(struct kunit *test)
+{
+	unsigned int i;
+	const char *uuid;
+	guid_t le;
+
+	for (i = 0; i < ARRAY_SIZE(test_uuid_wrong_data); i++) {
+		uuid = test_uuid_wrong_data[i];
+		KUNIT_EXPECT_EQ(test, guid_parse(uuid, &le), -EINVAL);
+	}
+}
+
+static void uuid_test_uuid_invalid(struct kunit *test)
+{
+	unsigned int i;
+	const char *uuid;
+	uuid_t be;
+
+	for (i = 0; i < ARRAY_SIZE(test_uuid_wrong_data); i++) {
+		uuid = test_uuid_wrong_data[i];
+		KUNIT_EXPECT_EQ(test, uuid_parse(uuid, &be), -EINVAL);
+	}
+}
+
+static struct kunit_case uuid_test_cases[] = {
+	KUNIT_CASE(uuid_test_guid_valid),
+	KUNIT_CASE(uuid_test_uuid_valid),
+	KUNIT_CASE(uuid_test_guid_invalid),
+	KUNIT_CASE(uuid_test_uuid_invalid),
+	{},
+};
+
+static struct kunit_suite uuid_test_suite = {
+	.name = "uuid",
+	.test_cases = uuid_test_cases,
+};
+
+kunit_test_suite(uuid_test_suite);
+
+MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");
+MODULE_DESCRIPTION("Test cases for lib/uuid.c module");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/lib/uuid.c b/lib/uuid.c
index e309b4c5be3d..e8543c668dc7 100644
--- a/lib/uuid.c
+++ b/lib/uuid.c
@@ -10,6 +10,7 @@
 #include <linux/ctype.h>
 #include <linux/errno.h>
 #include <linux/export.h>
+#include <linux/hex.h>
 #include <linux/uuid.h>
 #include <linux/random.h>
 
diff --git a/lib/vsprintf.c b/lib/vsprintf.c
index a3790c43a0ab..800b8ac49f53 100644
--- a/lib/vsprintf.c
+++ b/lib/vsprintf.c
@@ -26,6 +26,7 @@
 #include <linux/types.h>
 #include <linux/string.h>
 #include <linux/ctype.h>
+#include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/kallsyms.h>
 #include <linux/math64.h>
diff --git a/lib/xarray.c b/lib/xarray.c
index 9a8b4916540c..f25eff3f990b 100644
--- a/lib/xarray.c
+++ b/lib/xarray.c
@@ -368,7 +368,7 @@ static void *xas_alloc(struct xa_state *xas, unsigned int shift)
 		return NULL;
 
 	if (node) {
-		xas->xa_alloc = NULL;
+		xas->xa_alloc = rcu_dereference_raw(node->parent);
 	} else {
 		gfp_t gfp = GFP_NOWAIT;
 
@@ -2386,10 +2386,10 @@ void xa_destroy(struct xarray *xa)
 	unsigned long flags;
 	void *entry;
 
-	xas.xa_node = NULL;
 	xas_lock_irqsave(&xas, flags);
 	entry = xa_head_locked(xa);
 	RCU_INIT_POINTER(xa->xa_head, NULL);
+	xas.xa_node = NULL;
 	xas_init_marks(&xas);
 	if (xa_zero_busy(xa))
 		xa_mark_clear(xa, XA_FREE_MARK);
@@ -2400,7 +2400,7 @@ void xa_destroy(struct xarray *xa)
 }
 EXPORT_SYMBOL(xa_destroy);
 
-#ifdef XA_DEBUG
+#ifdef CONFIG_DEBUG_XARRAY
 void xa_dump_node(const struct xa_node *node)
 {
 	unsigned i, j;
@@ -2422,6 +2422,7 @@ void xa_dump_node(const struct xa_node *node)
 			pr_cont(" %lx", node->marks[i][j]);
 	pr_cont("\n");
 }
+EXPORT_SYMBOL(xa_dump_node);
 
 void xa_dump_index(unsigned long index, unsigned int shift)
 {
@@ -2433,7 +2434,8 @@ void xa_dump_index(unsigned long index, unsigned int shift)
 		pr_info("%lu-%lu: ", index, index | ((1UL << shift) - 1));
 }
 
-void xa_dump_entry(const void *entry, unsigned long index, unsigned long shift)
+static void xa_dump_entry(const void *entry, unsigned long index,
+		unsigned long shift)
 {
 	if (!entry)
 		return;
@@ -2478,4 +2480,5 @@ void xa_dump(const struct xarray *xa)
 		shift = xa_to_node(entry)->shift + XA_CHUNK_SHIFT;
 	xa_dump_entry(entry, 0, shift);
 }
+EXPORT_SYMBOL(xa_dump);
 #endif
diff --git a/localversion-next b/localversion-next
new file mode 100644
index 000000000000..bd19e67f6ec7
--- /dev/null
+++ b/localversion-next
@@ -0,0 +1 @@
+-next-20251219
diff --git a/mm/Kconfig b/mm/Kconfig
index bd0ea5454af8..ac48deb44884 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -1464,6 +1464,25 @@ config PT_RECLAIM
 config FIND_NORMAL_PAGE
 	def_bool n
 
+config ARCH_HAS_LAZY_MMU_MODE
+	bool
+	help
+	  The architecture uses the lazy MMU mode. This allows changes to
+	  MMU-related architectural state to be deferred until the mode is
+	  exited. See <linux/pgtable.h> for details.
+
+config LAZY_MMU_MODE_KUNIT_TEST
+	tristate "KUnit tests for the lazy MMU mode" if !KUNIT_ALL_TESTS
+	depends on ARCH_HAS_LAZY_MMU_MODE
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
+	help
+	  Enable this option to check that the lazy MMU mode interface behaves
+	  as expected. Only tests for the generic interface are included (not
+	  architecture-specific behaviours).
+
+	  If unsure, say N.
+
 source "mm/damon/Kconfig"
 
 endmenu
diff --git a/mm/Makefile b/mm/Makefile
index 2d0570a16e5b..9175f8cc6565 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -147,3 +147,4 @@ obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o
 obj-$(CONFIG_EXECMEM) += execmem.o
 obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o
 obj-$(CONFIG_PT_RECLAIM) += pt_reclaim.o
+obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o
diff --git a/mm/damon/core.c b/mm/damon/core.c
index f9fc0375890a..4ad5f290d382 100644
--- a/mm/damon/core.c
+++ b/mm/damon/core.c
@@ -2573,13 +2573,19 @@ static void kdamond_call(struct damon_ctx *ctx, bool cancel)
 			list_add(&control->list, &repeat_controls);
 		}
 	}
-	control = list_first_entry_or_null(&repeat_controls,
-			struct damon_call_control, list);
-	if (!control || cancel)
-		return;
-	mutex_lock(&ctx->call_controls_lock);
-	list_add_tail(&control->list, &ctx->call_controls);
-	mutex_unlock(&ctx->call_controls_lock);
+	while (true) {
+		control = list_first_entry_or_null(&repeat_controls,
+				struct damon_call_control, list);
+		if (!control)
+			break;
+		/* Unlink from the repeate_controls list. */
+		list_del(&control->list);
+		if (cancel)
+			continue;
+		mutex_lock(&ctx->call_controls_lock);
+		list_add(&control->list, &ctx->call_controls);
+		mutex_unlock(&ctx->call_controls_lock);
+	}
 }
 
 /* Returns negative error code if it's not activated but should return */
diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c
index 2750c88e7225..23ed738a0bd6 100644
--- a/mm/damon/vaddr.c
+++ b/mm/damon/vaddr.c
@@ -743,7 +743,7 @@ static int damos_va_migrate_pmd_entry(pmd_t *pmd, unsigned long addr,
 		if (!folio)
 			continue;
 		if (damos_va_filter_out(s, folio, walk->vma, addr, pte, NULL))
-			return 0;
+			continue;
 		damos_va_migrate_dests_add(folio, walk->vma, addr, dests,
 				migration_lists);
 		nr = folio_nr_pages(folio);
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 51273baec9e5..a065ef3f7485 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -5096,8 +5096,9 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma,
 	unsigned long last_addr_mask;
 	pte_t *src_pte, *dst_pte;
 	struct mmu_notifier_range range;
-	bool shared_pmd = false;
+	struct mmu_gather tlb;
 
+	tlb_gather_mmu(&tlb, vma->vm_mm);
 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, old_addr,
 				old_end);
 	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
@@ -5122,12 +5123,12 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma,
 		if (huge_pte_none(huge_ptep_get(mm, old_addr, src_pte)))
 			continue;
 
-		if (huge_pmd_unshare(mm, vma, old_addr, src_pte)) {
-			shared_pmd = true;
+		if (huge_pmd_unshare(&tlb, vma, old_addr, src_pte)) {
 			old_addr |= last_addr_mask;
 			new_addr |= last_addr_mask;
 			continue;
 		}
+		tlb_remove_huge_tlb_entry(h, &tlb, src_pte, old_addr);
 
 		dst_pte = huge_pte_alloc(mm, new_vma, new_addr, sz);
 		if (!dst_pte)
@@ -5136,13 +5137,13 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma,
 		move_huge_pte(vma, old_addr, new_addr, src_pte, dst_pte, sz);
 	}
 
-	if (shared_pmd)
-		flush_hugetlb_tlb_range(vma, range.start, range.end);
-	else
-		flush_hugetlb_tlb_range(vma, old_end - len, old_end);
+	tlb_flush_mmu_tlbonly(&tlb);
+	huge_pmd_unshare_flush(&tlb, vma);
+
 	mmu_notifier_invalidate_range_end(&range);
 	i_mmap_unlock_write(mapping);
 	hugetlb_vma_unlock_write(vma);
+	tlb_finish_mmu(&tlb);
 
 	return len + old_addr - old_end;
 }
@@ -5161,7 +5162,6 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	unsigned long sz = huge_page_size(h);
 	bool adjust_reservation;
 	unsigned long last_addr_mask;
-	bool force_flush = false;
 
 	WARN_ON(!is_vm_hugetlb_page(vma));
 	BUG_ON(start & ~huge_page_mask(h));
@@ -5184,10 +5184,8 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		}
 
 		ptl = huge_pte_lock(h, mm, ptep);
-		if (huge_pmd_unshare(mm, vma, address, ptep)) {
+		if (huge_pmd_unshare(tlb, vma, address, ptep)) {
 			spin_unlock(ptl);
-			tlb_flush_pmd_range(tlb, address & PUD_MASK, PUD_SIZE);
-			force_flush = true;
 			address |= last_addr_mask;
 			continue;
 		}
@@ -5303,21 +5301,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	}
 	tlb_end_vma(tlb, vma);
 
-	/*
-	 * If we unshared PMDs, the TLB flush was not recorded in mmu_gather. We
-	 * could defer the flush until now, since by holding i_mmap_rwsem we
-	 * guaranteed that the last reference would not be dropped. But we must
-	 * do the flushing before we return, as otherwise i_mmap_rwsem will be
-	 * dropped and the last reference to the shared PMDs page might be
-	 * dropped as well.
-	 *
-	 * In theory we could defer the freeing of the PMD pages as well, but
-	 * huge_pmd_unshare() relies on the exact page_count for the PMD page to
-	 * detect sharing, so we cannot defer the release of the page either.
-	 * Instead, do flush now.
-	 */
-	if (force_flush)
-		tlb_flush_mmu_tlbonly(tlb);
+	huge_pmd_unshare_flush(tlb, vma);
 }
 
 void __hugetlb_zap_begin(struct vm_area_struct *vma,
@@ -6406,7 +6390,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
 }
 #endif /* CONFIG_USERFAULTFD */
 
-long hugetlb_change_protection(struct vm_area_struct *vma,
+long hugetlb_change_protection(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		unsigned long address, unsigned long end,
 		pgprot_t newprot, unsigned long cp_flags)
 {
@@ -6416,7 +6400,6 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
 	pte_t pte;
 	struct hstate *h = hstate_vma(vma);
 	long pages = 0, psize = huge_page_size(h);
-	bool shared_pmd = false;
 	struct mmu_notifier_range range;
 	unsigned long last_addr_mask;
 	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
@@ -6459,7 +6442,7 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
 			}
 		}
 		ptl = huge_pte_lock(h, mm, ptep);
-		if (huge_pmd_unshare(mm, vma, address, ptep)) {
+		if (huge_pmd_unshare(tlb, vma, address, ptep)) {
 			/*
 			 * When uffd-wp is enabled on the vma, unshare
 			 * shouldn't happen at all.  Warn about it if it
@@ -6468,7 +6451,6 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
 			WARN_ON_ONCE(uffd_wp || uffd_wp_resolve);
 			pages++;
 			spin_unlock(ptl);
-			shared_pmd = true;
 			address |= last_addr_mask;
 			continue;
 		}
@@ -6529,23 +6511,16 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
 				pte = huge_pte_clear_uffd_wp(pte);
 			huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
 			pages++;
+			tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
 		}
 
 next:
 		spin_unlock(ptl);
 		cond_resched();
 	}
-	/*
-	 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
-	 * may have cleared our pud entry and done put_page on the page table:
-	 * once we release i_mmap_rwsem, another task can do the final put_page
-	 * and that page table be reused and filled with junk.  If we actually
-	 * did unshare a page of pmds, flush the range corresponding to the pud.
-	 */
-	if (shared_pmd)
-		flush_hugetlb_tlb_range(vma, range.start, range.end);
-	else
-		flush_hugetlb_tlb_range(vma, start, end);
+
+	tlb_flush_mmu_tlbonly(tlb);
+	huge_pmd_unshare_flush(tlb, vma);
 	/*
 	 * No need to call mmu_notifier_arch_invalidate_secondary_tlbs() we are
 	 * downgrading page table protection not changing it to point to a new
@@ -6912,18 +6887,27 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
 	return pte;
 }
 
-/*
- * unmap huge page backed by shared pte.
+/**
+ * huge_pmd_unshare - Unmap a pmd table if it is shared by multiple users
+ * @tlb: the current mmu_gather.
+ * @vma: the vma covering the pmd table.
+ * @addr: the address we are trying to unshare.
+ * @ptep: pointer into the (pmd) page table.
  *
- * Called with page table lock held.
+ * Called with the page table lock held, the i_mmap_rwsem held in write mode
+ * and the hugetlb vma lock held in write mode.
  *
- * returns: 1 successfully unmapped a shared pte page
- *	    0 the underlying pte page is not shared, or it is the last user
+ * Note: The caller must call huge_pmd_unshare_flush() before dropping the
+ * i_mmap_rwsem.
+ *
+ * Returns: 1 if it was a shared PMD table and it got unmapped, or 0 if it
+ *	    was not a shared PMD table.
  */
-int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
-					unsigned long addr, pte_t *ptep)
+int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma,
+		unsigned long addr, pte_t *ptep)
 {
 	unsigned long sz = huge_page_size(hstate_vma(vma));
+	struct mm_struct *mm = vma->vm_mm;
 	pgd_t *pgd = pgd_offset(mm, addr);
 	p4d_t *p4d = p4d_offset(pgd, addr);
 	pud_t *pud = pud_offset(p4d, addr);
@@ -6935,18 +6919,36 @@ int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
 	i_mmap_assert_write_locked(vma->vm_file->f_mapping);
 	hugetlb_vma_assert_locked(vma);
 	pud_clear(pud);
-	/*
-	 * Once our caller drops the rmap lock, some other process might be
-	 * using this page table as a normal, non-hugetlb page table.
-	 * Wait for pending gup_fast() in other threads to finish before letting
-	 * that happen.
-	 */
-	tlb_remove_table_sync_one();
-	ptdesc_pmd_pts_dec(virt_to_ptdesc(ptep));
+
+	tlb_unshare_pmd_ptdesc(tlb, virt_to_ptdesc(ptep), addr);
+
 	mm_dec_nr_pmds(mm);
 	return 1;
 }
 
+/*
+ * huge_pmd_unshare_flush - Complete a sequence of huge_pmd_unshare() calls
+ * @tlb: the current mmu_gather.
+ * @vma: the vma covering the pmd table.
+ *
+ * Perform necessary TLB flushes or IPI broadcasts to synchronize PMD table
+ * unsharing with concurrent page table walkers.
+ *
+ * This function must be called after a sequence of huge_pmd_unshare()
+ * calls while still holding the i_mmap_rwsem.
+ */
+void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma)
+{
+	/*
+	 * We must synchronize page table unsharing such that nobody will
+	 * try reusing a previously-shared page table while it might still
+	 * be in use by previous sharers (TLB, GUP_fast).
+	 */
+	i_mmap_assert_write_locked(vma->vm_file->f_mapping);
+
+	tlb_flush_unshared_tables(tlb);
+}
+
 #else /* !CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING */
 
 pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
@@ -6955,12 +6957,16 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
 	return NULL;
 }
 
-int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
-				unsigned long addr, pte_t *ptep)
+int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma,
+		unsigned long addr, pte_t *ptep)
 {
 	return 0;
 }
 
+void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma)
+{
+}
+
 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
 				unsigned long *start, unsigned long *end)
 {
@@ -7132,12 +7138,12 @@ int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison
 }
 
 int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
-				bool *migratable_cleared)
+				bool *migratable_cleared, bool *samepg)
 {
 	int ret;
 
 	spin_lock_irq(&hugetlb_lock);
-	ret = __get_huge_page_for_hwpoison(pfn, flags, migratable_cleared);
+	ret = __get_huge_page_for_hwpoison(pfn, flags, migratable_cleared, samepg);
 	spin_unlock_irq(&hugetlb_lock);
 	return ret;
 }
@@ -7227,6 +7233,7 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
 	unsigned long sz = huge_page_size(h);
 	struct mm_struct *mm = vma->vm_mm;
 	struct mmu_notifier_range range;
+	struct mmu_gather tlb;
 	unsigned long address;
 	spinlock_t *ptl;
 	pte_t *ptep;
@@ -7237,6 +7244,7 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
 	if (start >= end)
 		return;
 
+	tlb_gather_mmu(&tlb, mm);
 	flush_cache_range(vma, start, end);
 	/*
 	 * No need to call adjust_range_if_pmd_sharing_possible(), because
@@ -7256,10 +7264,10 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
 		if (!ptep)
 			continue;
 		ptl = huge_pte_lock(h, mm, ptep);
-		huge_pmd_unshare(mm, vma, address, ptep);
+		huge_pmd_unshare(&tlb, vma, address, ptep);
 		spin_unlock(ptl);
 	}
-	flush_hugetlb_tlb_range(vma, start, end);
+	huge_pmd_unshare_flush(&tlb, vma);
 	if (take_locks) {
 		i_mmap_unlock_write(vma->vm_file->f_mapping);
 		hugetlb_vma_unlock_write(vma);
@@ -7269,6 +7277,7 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
 	 * Documentation/mm/mmu_notifier.rst.
 	 */
 	mmu_notifier_invalidate_range_end(&range);
+	tlb_finish_mmu(&tlb);
 }
 
 /*
diff --git a/mm/kasan/common.c b/mm/kasan/common.c
index 1d27f1bd260b..ed489a14dddf 100644
--- a/mm/kasan/common.c
+++ b/mm/kasan/common.c
@@ -28,6 +28,7 @@
 #include <linux/string.h>
 #include <linux/types.h>
 #include <linux/bug.h>
+#include <linux/vmalloc.h>
 
 #include "kasan.h"
 #include "../slab.h"
@@ -575,3 +576,34 @@ bool __kasan_check_byte(const void *address, unsigned long ip)
 	}
 	return true;
 }
+
+#ifdef CONFIG_KASAN_VMALLOC
+void __kasan_unpoison_vmap_areas(struct vm_struct **vms, int nr_vms,
+				 kasan_vmalloc_flags_t flags)
+{
+	unsigned long size;
+	void *addr;
+	int area;
+	u8 tag;
+
+	/*
+	 * If KASAN_VMALLOC_KEEP_TAG was set at this point, all vms[] pointers
+	 * would be unpoisoned with the KASAN_TAG_KERNEL which would disable
+	 * KASAN checks down the line.
+	 */
+	if (WARN_ON_ONCE(flags & KASAN_VMALLOC_KEEP_TAG))
+		return;
+
+	size = vms[0]->size;
+	addr = vms[0]->addr;
+	vms[0]->addr = __kasan_unpoison_vmalloc(addr, size, flags);
+	tag = get_tag(vms[0]->addr);
+
+	for (area = 1 ; area < nr_vms ; area++) {
+		size = vms[area]->size;
+		addr = set_tag(vms[area]->addr, tag);
+		vms[area]->addr =
+			__kasan_unpoison_vmalloc(addr, size, flags | KASAN_VMALLOC_KEEP_TAG);
+	}
+}
+#endif
diff --git a/mm/kasan/hw_tags.c b/mm/kasan/hw_tags.c
index 1c373cc4b3fa..cbef5e450954 100644
--- a/mm/kasan/hw_tags.c
+++ b/mm/kasan/hw_tags.c
@@ -361,7 +361,7 @@ void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
 		return (void *)start;
 	}
 
-	tag = kasan_random_tag();
+	tag = (flags & KASAN_VMALLOC_KEEP_TAG) ? get_tag(start) : kasan_random_tag();
 	start = set_tag(start, tag);
 
 	/* Unpoison and initialize memory up to size. */
diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c
index 29a751a8a08d..d286e0a04543 100644
--- a/mm/kasan/shadow.c
+++ b/mm/kasan/shadow.c
@@ -305,7 +305,7 @@ static int kasan_populate_vmalloc_pte(pte_t *ptep, unsigned long addr,
 	pte_t pte;
 	int index;
 
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_pause();
 
 	index = PFN_DOWN(addr - data->start);
 	page = data->pages[index];
@@ -319,7 +319,7 @@ static int kasan_populate_vmalloc_pte(pte_t *ptep, unsigned long addr,
 	}
 	spin_unlock(&init_mm.page_table_lock);
 
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_resume();
 
 	return 0;
 }
@@ -471,7 +471,7 @@ static int kasan_depopulate_vmalloc_pte(pte_t *ptep, unsigned long addr,
 	pte_t pte;
 	int none;
 
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_pause();
 
 	spin_lock(&init_mm.page_table_lock);
 	pte = ptep_get(ptep);
@@ -483,7 +483,7 @@ static int kasan_depopulate_vmalloc_pte(pte_t *ptep, unsigned long addr,
 	if (likely(!none))
 		__free_page(pfn_to_page(pte_pfn(pte)));
 
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_resume();
 
 	return 0;
 }
@@ -631,7 +631,9 @@ void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
 	    !(flags & KASAN_VMALLOC_PROT_NORMAL))
 		return (void *)start;
 
-	start = set_tag(start, kasan_random_tag());
+	if (unlikely(!(flags & KASAN_VMALLOC_KEEP_TAG)))
+		start = set_tag(start, kasan_random_tag());
+
 	kasan_unpoison(start, size, false);
 	return (void *)start;
 }
diff --git a/mm/kfence/kfence_test.c b/mm/kfence/kfence_test.c
index 00034e37bc9f..5725a367246d 100644
--- a/mm/kfence/kfence_test.c
+++ b/mm/kfence/kfence_test.c
@@ -110,7 +110,7 @@ static bool report_matches(const struct expect_report *r)
 
 	/* Title */
 	cur = expect[0];
-	end = &expect[0][sizeof(expect[0]) - 1];
+	end = ARRAY_END(expect[0]);
 	switch (r->type) {
 	case KFENCE_ERROR_OOB:
 		cur += scnprintf(cur, end - cur, "BUG: KFENCE: out-of-bounds %s",
@@ -140,7 +140,7 @@ static bool report_matches(const struct expect_report *r)
 
 	/* Access information */
 	cur = expect[1];
-	end = &expect[1][sizeof(expect[1]) - 1];
+	end = ARRAY_END(expect[1]);
 
 	switch (r->type) {
 	case KFENCE_ERROR_OOB:
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 97d1b2824386..6c8c35d3e0c9 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -22,6 +22,7 @@
 #include <linux/dax.h>
 #include <linux/ksm.h>
 #include <linux/pgalloc.h>
+#include <linux/backing-dev.h>
 
 #include <asm/tlb.h>
 #include "internal.h"
@@ -58,6 +59,7 @@ enum scan_result {
 	SCAN_STORE_FAILED,
 	SCAN_COPY_MC,
 	SCAN_PAGE_FILLED,
+	SCAN_PAGE_DIRTY_OR_WRITEBACK,
 };
 
 #define CREATE_TRACE_POINTS
@@ -1967,11 +1969,11 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr,
 				 */
 				xas_unlock_irq(&xas);
 				filemap_flush(mapping);
-				result = SCAN_FAIL;
+				result = SCAN_PAGE_DIRTY_OR_WRITEBACK;
 				goto xa_unlocked;
 			} else if (folio_test_writeback(folio)) {
 				xas_unlock_irq(&xas);
-				result = SCAN_FAIL;
+				result = SCAN_PAGE_DIRTY_OR_WRITEBACK;
 				goto xa_unlocked;
 			} else if (folio_trylock(folio)) {
 				folio_get(folio);
@@ -2018,7 +2020,7 @@ static int collapse_file(struct mm_struct *mm, unsigned long addr,
 			 * folio is dirty because it hasn't been flushed
 			 * since first write.
 			 */
-			result = SCAN_FAIL;
+			result = SCAN_PAGE_DIRTY_OR_WRITEBACK;
 			goto out_unlock;
 		}
 
@@ -2747,6 +2749,7 @@ static int madvise_collapse_errno(enum scan_result r)
 	case SCAN_PAGE_LRU:
 	case SCAN_DEL_PAGE_LRU:
 	case SCAN_PAGE_FILLED:
+	case SCAN_PAGE_DIRTY_OR_WRITEBACK:
 		return -EAGAIN;
 	/*
 	 * Other: Trying again likely not to succeed / error intrinsic to
@@ -2785,9 +2788,11 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
 	hend = end & HPAGE_PMD_MASK;
 
 	for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) {
+		bool retried = false;
 		int result = SCAN_FAIL;
 
 		if (!mmap_locked) {
+retry:
 			cond_resched();
 			mmap_read_lock(mm);
 			mmap_locked = true;
@@ -2817,6 +2822,43 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
 		if (!mmap_locked)
 			*lock_dropped = true;
 
+		/*
+		 * If the file-backed VMA has dirty pages, the scan triggers
+		 * async writeback and returns SCAN_PAGE_DIRTY_OR_WRITEBACK.
+		 * Since MADV_COLLAPSE is sync, we force sync writeback and
+		 * retry once.
+		 */
+		if (result == SCAN_PAGE_DIRTY_OR_WRITEBACK && !retried) {
+			/*
+			 * File scan drops the lock. We must re-acquire it to
+			 * safely inspect the VMA and hold the file reference.
+			 */
+			if (!mmap_locked) {
+				cond_resched();
+				mmap_read_lock(mm);
+				mmap_locked = true;
+				result = hugepage_vma_revalidate(mm, addr, false, &vma, cc);
+				if (result != SCAN_SUCCEED)
+					goto handle_result;
+			}
+
+			if (!vma_is_anonymous(vma) && vma->vm_file &&
+			    mapping_can_writeback(vma->vm_file->f_mapping)) {
+				struct file *file = get_file(vma->vm_file);
+				pgoff_t pgoff = linear_page_index(vma, addr);
+				loff_t lstart = (loff_t)pgoff << PAGE_SHIFT;
+				loff_t lend = lstart + HPAGE_PMD_SIZE - 1;
+
+				mmap_read_unlock(mm);
+				mmap_locked = false;
+				*lock_dropped = true;
+				filemap_write_and_wait_range(file->f_mapping, lstart, lend);
+				fput(file);
+				retried = true;
+				goto retry;
+			}
+		}
+
 handle_result:
 		switch (result) {
 		case SCAN_SUCCEED:
diff --git a/mm/kmemleak.c b/mm/kmemleak.c
index 1ac56ceb29b6..fe33f2edfe07 100644
--- a/mm/kmemleak.c
+++ b/mm/kmemleak.c
@@ -510,7 +510,7 @@ static void mem_pool_free(struct kmemleak_object *object)
 {
 	unsigned long flags;
 
-	if (object < mem_pool || object >= mem_pool + ARRAY_SIZE(mem_pool)) {
+	if (object < mem_pool || object >= ARRAY_END(mem_pool)) {
 		kmem_cache_free(object_cache, object);
 		return;
 	}
diff --git a/mm/kmsan/kmsan_test.c b/mm/kmsan/kmsan_test.c
index 902ec48b1e3e..b5ad5dfb2c00 100644
--- a/mm/kmsan/kmsan_test.c
+++ b/mm/kmsan/kmsan_test.c
@@ -105,7 +105,7 @@ static bool report_matches(const struct expect_report *r)
 
 	/* Title */
 	cur = expected_header;
-	end = &expected_header[sizeof(expected_header) - 1];
+	end = ARRAY_END(expected_header);
 
 	cur += scnprintf(cur, end - cur, "BUG: KMSAN: %s", r->error_type);
 
diff --git a/mm/madvise.c b/mm/madvise.c
index b617b1be0f53..6bf7009fa5ce 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -453,7 +453,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
 	if (!start_pte)
 		return 0;
 	flush_tlb_batched_pending(mm);
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 	for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) {
 		nr = 1;
 		ptent = ptep_get(pte);
@@ -461,7 +461,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
 		if (++batch_count == SWAP_CLUSTER_MAX) {
 			batch_count = 0;
 			if (need_resched()) {
-				arch_leave_lazy_mmu_mode();
+				lazy_mmu_mode_disable();
 				pte_unmap_unlock(start_pte, ptl);
 				cond_resched();
 				goto restart;
@@ -497,7 +497,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
 				if (!folio_trylock(folio))
 					continue;
 				folio_get(folio);
-				arch_leave_lazy_mmu_mode();
+				lazy_mmu_mode_disable();
 				pte_unmap_unlock(start_pte, ptl);
 				start_pte = NULL;
 				err = split_folio(folio);
@@ -508,7 +508,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
 				if (!start_pte)
 					break;
 				flush_tlb_batched_pending(mm);
-				arch_enter_lazy_mmu_mode();
+				lazy_mmu_mode_enable();
 				if (!err)
 					nr = 0;
 				continue;
@@ -556,7 +556,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
 	}
 
 	if (start_pte) {
-		arch_leave_lazy_mmu_mode();
+		lazy_mmu_mode_disable();
 		pte_unmap_unlock(start_pte, ptl);
 	}
 	if (pageout)
@@ -675,7 +675,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
 	if (!start_pte)
 		return 0;
 	flush_tlb_batched_pending(mm);
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 	for (; addr != end; pte += nr, addr += PAGE_SIZE * nr) {
 		nr = 1;
 		ptent = ptep_get(pte);
@@ -724,7 +724,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
 				if (!folio_trylock(folio))
 					continue;
 				folio_get(folio);
-				arch_leave_lazy_mmu_mode();
+				lazy_mmu_mode_disable();
 				pte_unmap_unlock(start_pte, ptl);
 				start_pte = NULL;
 				err = split_folio(folio);
@@ -735,7 +735,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
 				if (!start_pte)
 					break;
 				flush_tlb_batched_pending(mm);
-				arch_enter_lazy_mmu_mode();
+				lazy_mmu_mode_enable();
 				if (!err)
 					nr = 0;
 				continue;
@@ -775,7 +775,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
 	if (nr_swap)
 		add_mm_counter(mm, MM_SWAPENTS, nr_swap);
 	if (start_pte) {
-		arch_leave_lazy_mmu_mode();
+		lazy_mmu_mode_disable();
 		pte_unmap_unlock(start_pte, ptl);
 	}
 	cond_resched();
diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c
index 6eed14bff742..b2f37bd939fa 100644
--- a/mm/memcontrol-v1.c
+++ b/mm/memcontrol-v1.c
@@ -1794,7 +1794,7 @@ static int memcg_numa_stat_show(struct seq_file *m, void *v)
 
 	mem_cgroup_flush_stats(memcg);
 
-	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
+	for (stat = stats; stat < ARRAY_END(stats); stat++) {
 		seq_printf(m, "%s=%lu", stat->name,
 			   mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
 						   false));
@@ -1805,7 +1805,7 @@ static int memcg_numa_stat_show(struct seq_file *m, void *v)
 		seq_putc(m, '\n');
 	}
 
-	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
+	for (stat = stats; stat < ARRAY_END(stats); stat++) {
 
 		seq_printf(m, "hierarchical_%s=%lu", stat->name,
 			   mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index be810c1fbfc3..6f000f0e76d2 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -281,7 +281,7 @@ ino_t page_cgroup_ino(struct page *page)
 	/* page_folio() is racy here, but the entire function is racy anyway */
 	memcg = folio_memcg_check(page_folio(page));
 
-	while (memcg && !(memcg->css.flags & CSS_ONLINE))
+	while (memcg && !css_is_online(&memcg->css))
 		memcg = parent_mem_cgroup(memcg);
 	if (memcg)
 		ino = cgroup_ino(memcg->css.cgroup);
@@ -5638,6 +5638,6 @@ void mem_cgroup_show_protected_memory(struct mem_cgroup *memcg)
 		memcg = root_mem_cgroup;
 
 	pr_warn("Memory cgroup min protection %lukB -- low protection %lukB",
-		K(atomic_long_read(&memcg->memory.children_min_usage)*PAGE_SIZE),
-		K(atomic_long_read(&memcg->memory.children_low_usage)*PAGE_SIZE));
+		K(atomic_long_read(&memcg->memory.children_min_usage)),
+		K(atomic_long_read(&memcg->memory.children_low_usage)));
 }
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index fbc5a01260c8..8565cf979091 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -1883,7 +1883,8 @@ static unsigned long __folio_free_raw_hwp(struct folio *folio, bool move_flag)
 	return count;
 }
 
-static int folio_set_hugetlb_hwpoison(struct folio *folio, struct page *page)
+static int folio_set_hugetlb_hwpoison(struct folio *folio, struct page *page,
+					bool *samepg)
 {
 	struct llist_head *head;
 	struct raw_hwp_page *raw_hwp;
@@ -1899,17 +1900,16 @@ static int folio_set_hugetlb_hwpoison(struct folio *folio, struct page *page)
 		return -EHWPOISON;
 	head = raw_hwp_list_head(folio);
 	llist_for_each_entry(p, head->first, node) {
-		if (p->page == page)
+		if (p->page == page) {
+			*samepg = true;
 			return -EHWPOISON;
+		}
 	}
 
 	raw_hwp = kmalloc(sizeof(struct raw_hwp_page), GFP_ATOMIC);
 	if (raw_hwp) {
 		raw_hwp->page = page;
 		llist_add(&raw_hwp->node, head);
-		/* the first error event will be counted in action_result(). */
-		if (ret)
-			num_poisoned_pages_inc(page_to_pfn(page));
 	} else {
 		/*
 		 * Failed to save raw error info.  We no longer trace all
@@ -1966,7 +1966,7 @@ void folio_clear_hugetlb_hwpoison(struct folio *folio)
  *   -EHWPOISON    - the hugepage is already hwpoisoned
  */
 int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,
-				 bool *migratable_cleared)
+				 bool *migratable_cleared, bool *samepg)
 {
 	struct page *page = pfn_to_page(pfn);
 	struct folio *folio = page_folio(page);
@@ -1991,7 +1991,7 @@ int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,
 			goto out;
 	}
 
-	if (folio_set_hugetlb_hwpoison(folio, page)) {
+	if (folio_set_hugetlb_hwpoison(folio, page, samepg)) {
 		ret = -EHWPOISON;
 		goto out;
 	}
@@ -2024,11 +2024,12 @@ static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb
 	struct page *p = pfn_to_page(pfn);
 	struct folio *folio;
 	unsigned long page_flags;
+	bool samepg = false;
 	bool migratable_cleared = false;
 
 	*hugetlb = 1;
 retry:
-	res = get_huge_page_for_hwpoison(pfn, flags, &migratable_cleared);
+	res = get_huge_page_for_hwpoison(pfn, flags, &migratable_cleared, &samepg);
 	if (res == 2) { /* fallback to normal page handling */
 		*hugetlb = 0;
 		return 0;
@@ -2037,7 +2038,10 @@ static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb
 			folio = page_folio(p);
 			res = kill_accessing_process(current, folio_pfn(folio), flags);
 		}
-		action_result(pfn, MF_MSG_ALREADY_POISONED, MF_FAILED);
+		if (samepg)
+			action_result(pfn, MF_MSG_ALREADY_POISONED, MF_FAILED);
+		else
+			action_result(pfn, MF_MSG_HUGE, MF_FAILED);
 		return res;
 	} else if (res == -EBUSY) {
 		if (!(flags & MF_NO_RETRY)) {
@@ -2161,6 +2165,9 @@ int register_pfn_address_space(struct pfn_address_space *pfn_space)
 {
 	guard(mutex)(&pfn_space_lock);
 
+	if (!pfn_space->pfn_to_vma_pgoff)
+		return -EINVAL;
+
 	if (interval_tree_iter_first(&pfn_space_itree,
 				     pfn_space->node.start,
 				     pfn_space->node.last))
@@ -2183,10 +2190,10 @@ void unregister_pfn_address_space(struct pfn_address_space *pfn_space)
 }
 EXPORT_SYMBOL_GPL(unregister_pfn_address_space);
 
-static void add_to_kill_pfn(struct task_struct *tsk,
-			    struct vm_area_struct *vma,
-			    struct list_head *to_kill,
-			    unsigned long pfn)
+static void add_to_kill_pgoff(struct task_struct *tsk,
+			      struct vm_area_struct *vma,
+			      struct list_head *to_kill,
+			      pgoff_t pgoff)
 {
 	struct to_kill *tk;
 
@@ -2197,12 +2204,12 @@ static void add_to_kill_pfn(struct task_struct *tsk,
 	}
 
 	/* Check for pgoff not backed by struct page */
-	tk->addr = vma_address(vma, pfn, 1);
+	tk->addr = vma_address(vma, pgoff, 1);
 	tk->size_shift = PAGE_SHIFT;
 
 	if (tk->addr == -EFAULT)
 		pr_info("Unable to find address %lx in %s\n",
-			pfn, tsk->comm);
+			pgoff, tsk->comm);
 
 	get_task_struct(tsk);
 	tk->tsk = tsk;
@@ -2212,11 +2219,12 @@ static void add_to_kill_pfn(struct task_struct *tsk,
 /*
  * Collect processes when the error hit a PFN not backed by struct page.
  */
-static void collect_procs_pfn(struct address_space *mapping,
+static void collect_procs_pfn(struct pfn_address_space *pfn_space,
 			      unsigned long pfn, struct list_head *to_kill)
 {
 	struct vm_area_struct *vma;
 	struct task_struct *tsk;
+	struct address_space *mapping = pfn_space->mapping;
 
 	i_mmap_lock_read(mapping);
 	rcu_read_lock();
@@ -2226,9 +2234,12 @@ static void collect_procs_pfn(struct address_space *mapping,
 		t = task_early_kill(tsk, true);
 		if (!t)
 			continue;
-		vma_interval_tree_foreach(vma, &mapping->i_mmap, pfn, pfn) {
-			if (vma->vm_mm == t->mm)
-				add_to_kill_pfn(t, vma, to_kill, pfn);
+		vma_interval_tree_foreach(vma, &mapping->i_mmap, 0, ULONG_MAX) {
+			pgoff_t pgoff;
+
+			if (vma->vm_mm == t->mm &&
+			    !pfn_space->pfn_to_vma_pgoff(vma, pfn, &pgoff))
+				add_to_kill_pgoff(t, vma, to_kill, pgoff);
 		}
 	}
 	rcu_read_unlock();
@@ -2264,7 +2275,7 @@ static int memory_failure_pfn(unsigned long pfn, int flags)
 			struct pfn_address_space *pfn_space =
 				container_of(node, struct pfn_address_space, node);
 
-			collect_procs_pfn(pfn_space->mapping, pfn, &tokill);
+			collect_procs_pfn(pfn_space, pfn, &tokill);
 
 			mf_handled = true;
 		}
diff --git a/mm/memory.c b/mm/memory.c
index 2a55edc48a65..ee15303c4041 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1256,7 +1256,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
 	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
 	orig_src_pte = src_pte;
 	orig_dst_pte = dst_pte;
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 
 	do {
 		nr = 1;
@@ -1325,7 +1325,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
 	} while (dst_pte += nr, src_pte += nr, addr += PAGE_SIZE * nr,
 		 addr != end);
 
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 	pte_unmap_unlock(orig_src_pte, src_ptl);
 	add_mm_rss_vec(dst_mm, rss);
 	pte_unmap_unlock(orig_dst_pte, dst_ptl);
@@ -1842,7 +1842,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
 		return addr;
 
 	flush_tlb_batched_pending(mm);
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 	do {
 		bool any_skipped = false;
 
@@ -1874,7 +1874,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
 		direct_reclaim = try_get_and_clear_pmd(mm, pmd, &pmdval);
 
 	add_mm_rss_vec(mm, rss);
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 
 	/* Do the actual TLB flush before dropping ptl */
 	if (force_flush) {
@@ -2813,7 +2813,7 @@ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
 	mapped_pte = pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
 	if (!pte)
 		return -ENOMEM;
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 	do {
 		BUG_ON(!pte_none(ptep_get(pte)));
 		if (!pfn_modify_allowed(pfn, prot)) {
@@ -2823,7 +2823,7 @@ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
 		set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
 		pfn++;
 	} while (pte++, addr += PAGE_SIZE, addr != end);
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 	pte_unmap_unlock(mapped_pte, ptl);
 	return err;
 }
@@ -3174,7 +3174,7 @@ static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
 			return -EINVAL;
 	}
 
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 
 	if (fn) {
 		do {
@@ -3187,7 +3187,7 @@ static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
 	}
 	*mask |= PGTBL_PTE_MODIFIED;
 
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 
 	if (mm != &init_mm)
 		pte_unmap_unlock(mapped_pte, ptl);
diff --git a/mm/memremap.c b/mm/memremap.c
index 4c2e0d68eb27..af37c3b4e39b 100644
--- a/mm/memremap.c
+++ b/mm/memremap.c
@@ -428,7 +428,12 @@ void free_zone_device_folio(struct folio *folio)
 		for (i = 0; i < nr; i++)
 			__ClearPageAnonExclusive(folio_page(folio, i));
 	} else {
-		VM_WARN_ON_ONCE(folio_test_large(folio));
+		/*
+		 * FS_DAX legitimately uses large file-mapped folios for
+		 * PMD mappings, so only warn for other device types.
+		 */
+		VM_WARN_ON_ONCE(pgmap->type != MEMORY_DEVICE_FS_DAX &&
+				folio_test_large(folio));
 	}
 
 	/*
diff --git a/mm/migrate_device.c b/mm/migrate_device.c
index 23379663b1e1..0346c2d7819f 100644
--- a/mm/migrate_device.c
+++ b/mm/migrate_device.c
@@ -271,7 +271,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
 	ptep = pte_offset_map_lock(mm, pmdp, start, &ptl);
 	if (!ptep)
 		goto again;
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 	ptep += (addr - start) / PAGE_SIZE;
 
 	for (; addr < end; addr += PAGE_SIZE, ptep++) {
@@ -313,7 +313,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
 			if (folio_test_large(folio)) {
 				int ret;
 
-				arch_leave_lazy_mmu_mode();
+				lazy_mmu_mode_disable();
 				pte_unmap_unlock(ptep, ptl);
 				ret = migrate_vma_split_folio(folio,
 							  migrate->fault_page);
@@ -356,7 +356,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
 			if (folio && folio_test_large(folio)) {
 				int ret;
 
-				arch_leave_lazy_mmu_mode();
+				lazy_mmu_mode_disable();
 				pte_unmap_unlock(ptep, ptl);
 				ret = migrate_vma_split_folio(folio,
 							  migrate->fault_page);
@@ -485,7 +485,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
 	if (unmapped)
 		flush_tlb_range(walk->vma, start, end);
 
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 	pte_unmap_unlock(ptep - 1, ptl);
 
 	return 0;
diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c
index 247e3f9db6c7..030a162a263b 100644
--- a/mm/mmu_gather.c
+++ b/mm/mmu_gather.c
@@ -426,6 +426,7 @@ static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
 #endif
 	tlb->vma_pfn = 0;
 
+	tlb->fully_unshared_tables = 0;
 	__tlb_reset_range(tlb);
 	inc_tlb_flush_pending(tlb->mm);
 }
@@ -468,6 +469,12 @@ void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm)
  */
 void tlb_finish_mmu(struct mmu_gather *tlb)
 {
+	/*
+	 * We expect an earlier huge_pmd_unshare_flush() call to sort this out,
+	 * due to complicated locking requirements with page table unsharing.
+	 */
+	VM_WARN_ON_ONCE(tlb->fully_unshared_tables);
+
 	/*
 	 * If there are parallel threads are doing PTE changes on same range
 	 * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB
diff --git a/mm/mprotect.c b/mm/mprotect.c
index 283889e4f1ce..da8e764f2974 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -233,7 +233,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 		is_private_single_threaded = vma_is_single_threaded_private(vma);
 
 	flush_tlb_batched_pending(vma->vm_mm);
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 	do {
 		nr_ptes = 1;
 		oldpte = ptep_get(pte);
@@ -379,7 +379,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 			}
 		}
 	} while (pte += nr_ptes, addr += nr_ptes * PAGE_SIZE, addr != end);
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 	pte_unmap_unlock(pte - 1, ptl);
 
 	return pages;
@@ -652,7 +652,7 @@ long change_protection(struct mmu_gather *tlb,
 #endif
 
 	if (is_vm_hugetlb_page(vma))
-		pages = hugetlb_change_protection(vma, start, end, newprot,
+		pages = hugetlb_change_protection(tlb, vma, start, end, newprot,
 						  cp_flags);
 	else
 		pages = change_protection_range(tlb, vma, start, end, newprot,
diff --git a/mm/mremap.c b/mm/mremap.c
index 672264807db6..8275b9772ec1 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -260,7 +260,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
 	if (new_ptl != old_ptl)
 		spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
 	flush_tlb_batched_pending(vma->vm_mm);
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 
 	for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE,
 		new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) {
@@ -305,7 +305,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
 		}
 	}
 
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 	if (force_flush)
 		flush_tlb_range(vma, old_end - len, old_end);
 	if (new_ptl != old_ptl)
diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index 5eb11fbba704..91ecf2b90fe0 100644
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -53,6 +53,14 @@
 #define CREATE_TRACE_POINTS
 #include <trace/events/oom.h>
 
+/*
+ * Maximum number of badness sums allowed before using an approximated
+ * comparison. This ensures bounded execution time for scenarios where
+ * many tasks have badness within the accuracy of the maximum badness
+ * approximation.
+ */
+static int max_precise_badness_sums = 16;
+
 static int sysctl_panic_on_oom;
 static int sysctl_oom_kill_allocating_task;
 static int sysctl_oom_dump_tasks = 1;
@@ -194,12 +202,16 @@ static bool should_dump_unreclaim_slab(void)
  * oom_badness - heuristic function to determine which candidate task to kill
  * @p: task struct of which task we should calculate
  * @totalpages: total present RAM allowed for page allocation
+ * @approximate: whether the value can be approximated
+ * @accuracy_under: accuracy of the badness value approximation (under value)
+ * @accuracy_over: accuracy of the badness value approximation (over value)
  *
  * The heuristic for determining which task to kill is made to be as simple and
  * predictable as possible.  The goal is to return the highest value for the
  * task consuming the most memory to avoid subsequent oom failures.
  */
-long oom_badness(struct task_struct *p, unsigned long totalpages)
+long oom_badness(struct task_struct *p, unsigned long totalpages, bool approximate,
+		 unsigned int *accuracy_under, unsigned int *accuracy_over)
 {
 	long points;
 	long adj;
@@ -228,7 +240,8 @@ long oom_badness(struct task_struct *p, unsigned long totalpages)
 	 * The baseline for the badness score is the proportion of RAM that each
 	 * task's rss, pagetable and swap space use.
 	 */
-	points = get_mm_rss(p->mm) + get_mm_counter(p->mm, MM_SWAPENTS) +
+	points = __get_mm_rss(p->mm, approximate, accuracy_under, accuracy_over) +
+		__get_mm_counter(p->mm, MM_SWAPENTS, approximate, accuracy_under, accuracy_over) +
 		mm_pgtables_bytes(p->mm) / PAGE_SIZE;
 	task_unlock(p);
 
@@ -309,6 +322,7 @@ static enum oom_constraint constrained_alloc(struct oom_control *oc)
 static int oom_evaluate_task(struct task_struct *task, void *arg)
 {
 	struct oom_control *oc = arg;
+	unsigned int accuracy_under = 0, accuracy_over = 0;
 	long points;
 
 	if (oom_unkillable_task(task))
@@ -339,9 +353,28 @@ static int oom_evaluate_task(struct task_struct *task, void *arg)
 		goto select;
 	}
 
-	points = oom_badness(task, oc->totalpages);
-	if (points == LONG_MIN || points < oc->chosen_points)
-		goto next;
+	points = oom_badness(task, oc->totalpages, true, &accuracy_under, &accuracy_over);
+	if (oc->approximate) {
+		if (points == LONG_MIN || points < oc->chosen_points)
+			goto next;
+	} else {
+		/*
+		 * Eliminate processes which are below the chosen
+		 * points accuracy range with an approximation.
+		 */
+		if (points == LONG_MIN || (long)(points + accuracy_over + oc->accuracy_under - oc->chosen_points) < 0)
+			goto next;
+
+		if (oc->nr_precise < max_precise_badness_sums) {
+			accuracy_under = 0;
+			accuracy_over = 0;
+			oc->nr_precise++;
+			/* Precise evaluation. */
+			points = oom_badness(task, oc->totalpages, false, NULL, NULL);
+			if (points == LONG_MIN || (long)(points + oc->accuracy_under - oc->chosen_points) < 0)
+				goto next;
+		}
+	}
 
 select:
 	if (oc->chosen)
@@ -349,6 +382,7 @@ static int oom_evaluate_task(struct task_struct *task, void *arg)
 	get_task_struct(task);
 	oc->chosen = task;
 	oc->chosen_points = points;
+	oc->accuracy_under = accuracy_under;
 next:
 	return 0;
 abort:
@@ -358,14 +392,8 @@ static int oom_evaluate_task(struct task_struct *task, void *arg)
 	return 1;
 }
 
-/*
- * Simple selection loop. We choose the process with the highest number of
- * 'points'. In case scan was aborted, oc->chosen is set to -1.
- */
-static void select_bad_process(struct oom_control *oc)
+static void select_bad_process_iter(struct oom_control *oc)
 {
-	oc->chosen_points = LONG_MIN;
-
 	if (is_memcg_oom(oc))
 		mem_cgroup_scan_tasks(oc->memcg, oom_evaluate_task, oc);
 	else {
@@ -379,6 +407,26 @@ static void select_bad_process(struct oom_control *oc)
 	}
 }
 
+/*
+ * Simple selection loop. We choose the process with the highest number of
+ * 'points'. In case scan was aborted, oc->chosen is set to -1.
+ */
+static void select_bad_process(struct oom_control *oc)
+{
+	oc->chosen_points = LONG_MIN;
+	oc->accuracy_under = 0;
+	oc->nr_precise = 0;
+
+	/* Approximate scan. */
+	oc->approximate = true;
+	select_bad_process_iter(oc);
+	if (oc->chosen == (void *)-1UL)
+		return;
+	/* Precise scan. */
+	oc->approximate = false;
+	select_bad_process_iter(oc);
+}
+
 static int dump_task(struct task_struct *p, void *arg)
 {
 	struct oom_control *oc = arg;
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 822e05f1a964..6a47443c48ff 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -914,6 +914,17 @@ buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn,
 			NULL) != NULL;
 }
 
+static void change_pageblock_range(struct page *pageblock_page,
+				   int start_order, int migratetype)
+{
+	int nr_pageblocks = 1 << (start_order - pageblock_order);
+
+	while (nr_pageblocks--) {
+		set_pageblock_migratetype(pageblock_page, migratetype);
+		pageblock_page += pageblock_nr_pages;
+	}
+}
+
 /*
  * Freeing function for a buddy system allocator.
  *
@@ -1000,7 +1011,7 @@ static inline void __free_one_page(struct page *page,
 			 * expand() down the line puts the sub-blocks
 			 * on the right freelists.
 			 */
-			set_pageblock_migratetype(buddy, migratetype);
+			change_pageblock_range(buddy, order, migratetype);
 		}
 
 		combined_pfn = buddy_pfn & pfn;
@@ -2147,17 +2158,6 @@ bool pageblock_unisolate_and_move_free_pages(struct zone *zone, struct page *pag
 
 #endif /* CONFIG_MEMORY_ISOLATION */
 
-static void change_pageblock_range(struct page *pageblock_page,
-					int start_order, int migratetype)
-{
-	int nr_pageblocks = 1 << (start_order - pageblock_order);
-
-	while (nr_pageblocks--) {
-		set_pageblock_migratetype(pageblock_page, migratetype);
-		pageblock_page += pageblock_nr_pages;
-	}
-}
-
 static inline bool boost_watermark(struct zone *zone)
 {
 	unsigned long max_boost;
@@ -6667,11 +6667,19 @@ static int percpu_pagelist_high_fraction_sysctl_handler(const struct ctl_table *
 	int old_percpu_pagelist_high_fraction;
 	int ret;
 
+	/*
+	 * Avoid using pcp_batch_high_lock for reads as the value is read
+	 * atomically and a race with offlining is harmless.
+	 */
+
+	if (!write)
+		return proc_dointvec_minmax(table, write, buffer, length, ppos);
+
 	mutex_lock(&pcp_batch_high_lock);
 	old_percpu_pagelist_high_fraction = percpu_pagelist_high_fraction;
 
 	ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
-	if (!write || ret < 0)
+	if (ret < 0)
 		goto out;
 
 	/* Sanity checking to avoid pcp imbalance */
diff --git a/mm/page_owner.c b/mm/page_owner.c
index a70245684206..27d19f01009c 100644
--- a/mm/page_owner.c
+++ b/mm/page_owner.c
@@ -530,7 +530,7 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret,
 	if (!memcg)
 		goto out_unlock;
 
-	online = (memcg->css.flags & CSS_ONLINE);
+	online = css_is_online(&memcg->css);
 	cgroup_name(memcg->css.cgroup, name, sizeof(name));
 	ret += scnprintf(kbuf + ret, count - ret,
 			"Charged %sto %smemcg %s\n",
diff --git a/mm/rmap.c b/mm/rmap.c
index f955f02d570e..d6799afe1114 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -76,7 +76,7 @@
 #include <linux/mm_inline.h>
 #include <linux/oom.h>
 
-#include <asm/tlbflush.h>
+#include <asm/tlb.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/migrate.h>
@@ -2008,26 +2008,25 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
 			 * if unsuccessful.
 			 */
 			if (!anon) {
+				struct mmu_gather tlb;
+
 				VM_BUG_ON(!(flags & TTU_RMAP_LOCKED));
 				if (!hugetlb_vma_trylock_write(vma))
 					goto walk_abort;
-				if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) {
+
+				tlb_gather_mmu(&tlb, mm);
+				if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) {
 					hugetlb_vma_unlock_write(vma);
-					flush_tlb_range(vma,
-						range.start, range.end);
+					huge_pmd_unshare_flush(&tlb, vma);
+					tlb_finish_mmu(&tlb);
 					/*
-					 * The ref count of the PMD page was
-					 * dropped which is part of the way map
-					 * counting is done for shared PMDs.
-					 * Return 'true' here.  When there is
-					 * no other sharing, huge_pmd_unshare
-					 * returns false and we will unmap the
-					 * actual page and drop map count
-					 * to zero.
+					 * The PMD table was unmapped,
+					 * consequently unmapping the folio.
 					 */
 					goto walk_done;
 				}
 				hugetlb_vma_unlock_write(vma);
+				tlb_finish_mmu(&tlb);
 			}
 			pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);
 			if (pte_dirty(pteval))
@@ -2404,31 +2403,29 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
 			 * fail if unsuccessful.
 			 */
 			if (!anon) {
+				struct mmu_gather tlb;
+
 				VM_BUG_ON(!(flags & TTU_RMAP_LOCKED));
 				if (!hugetlb_vma_trylock_write(vma)) {
 					page_vma_mapped_walk_done(&pvmw);
 					ret = false;
 					break;
 				}
-				if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) {
-					hugetlb_vma_unlock_write(vma);
-					flush_tlb_range(vma,
-						range.start, range.end);
 
+				tlb_gather_mmu(&tlb, mm);
+				if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) {
+					hugetlb_vma_unlock_write(vma);
+					huge_pmd_unshare_flush(&tlb, vma);
+					tlb_finish_mmu(&tlb);
 					/*
-					 * The ref count of the PMD page was
-					 * dropped which is part of the way map
-					 * counting is done for shared PMDs.
-					 * Return 'true' here.  When there is
-					 * no other sharing, huge_pmd_unshare
-					 * returns false and we will unmap the
-					 * actual page and drop map count
-					 * to zero.
+					 * The PMD table was unmapped,
+					 * consequently unmapping the folio.
 					 */
 					page_vma_mapped_walk_done(&pvmw);
 					break;
 				}
 				hugetlb_vma_unlock_write(vma);
+				tlb_finish_mmu(&tlb);
 			}
 			/* Nuke the hugetlb page table entry */
 			pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);
diff --git a/mm/shmem.c b/mm/shmem.c
index b329b5302c48..15c2943140ca 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -1593,11 +1593,23 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
 	}
 
 	if (split) {
+		int order;
+
 try_split:
+		order = folio_order(folio);
 		/* Ensure the subpages are still dirty */
 		folio_test_set_dirty(folio);
 		if (split_folio_to_list(folio, folio_list))
 			goto redirty;
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+		if (order >= HPAGE_PMD_ORDER) {
+			count_memcg_folio_events(folio, THP_SWPOUT_FALLBACK, 1);
+			count_vm_event(THP_SWPOUT_FALLBACK);
+		}
+#endif
+		count_mthp_stat(order, MTHP_STAT_SWPOUT_FALLBACK);
+
 		folio_clear_dirty(folio);
 	}
 
@@ -4019,22 +4031,10 @@ static int shmem_whiteout(struct mnt_idmap *idmap,
 	whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
 	if (!whiteout)
 		return -ENOMEM;
-
 	error = shmem_mknod(idmap, old_dir, whiteout,
 			    S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
 	dput(whiteout);
-	if (error)
-		return error;
-
-	/*
-	 * Cheat and hash the whiteout while the old dentry is still in
-	 * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
-	 *
-	 * d_lookup() will consistently find one of them at this point,
-	 * not sure which one, but that isn't even important.
-	 */
-	d_rehash(whiteout);
-	return 0;
+	return error;
 }
 
 /*
@@ -4050,6 +4050,7 @@ static int shmem_rename2(struct mnt_idmap *idmap,
 {
 	struct inode *inode = d_inode(old_dentry);
 	int they_are_dirs = S_ISDIR(inode->i_mode);
+	bool had_offset = false;
 	int error;
 
 	if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
@@ -4062,16 +4063,23 @@ static int shmem_rename2(struct mnt_idmap *idmap,
 	if (!simple_empty(new_dentry))
 		return -ENOTEMPTY;
 
+	error = simple_offset_add(shmem_get_offset_ctx(new_dir), new_dentry);
+	if (error == -EBUSY)
+		had_offset = true;
+	else if (unlikely(error))
+		return error;
+
 	if (flags & RENAME_WHITEOUT) {
 		error = shmem_whiteout(idmap, old_dir, old_dentry);
-		if (error)
+		if (error) {
+			if (!had_offset)
+				simple_offset_remove(shmem_get_offset_ctx(new_dir),
+						     new_dentry);
 			return error;
+		}
 	}
 
-	error = simple_offset_rename(old_dir, old_dentry, new_dir, new_dentry);
-	if (error)
-		return error;
-
+	simple_offset_rename(old_dir, old_dentry, new_dir, new_dentry);
 	if (d_really_is_positive(new_dentry)) {
 		(void) shmem_unlink(new_dir, new_dentry);
 		if (they_are_dirs) {
diff --git a/mm/slub.c b/mm/slub.c
index f21b2f0c6f5a..6552e7bffbb1 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -5052,7 +5052,11 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
 	 */
 
 	if (pcs->main->size == 0) {
-		barn_put_empty_sheaf(barn, pcs->main);
+		if (!pcs->spare) {
+			pcs->spare = pcs->main;
+		} else {
+			barn_put_empty_sheaf(barn, pcs->main);
+		}
 		pcs->main = full;
 		return pcs;
 	}
@@ -6539,6 +6543,8 @@ static void defer_free(struct kmem_cache *s, void *head)
 
 	guard(preempt)();
 
+	head = kasan_reset_tag(head);
+
 	df = this_cpu_ptr(&defer_free_objects);
 	if (llist_add(head + s->offset, &df->objects))
 		irq_work_queue(&df->work);
diff --git a/mm/tests/lazy_mmu_mode_kunit.c b/mm/tests/lazy_mmu_mode_kunit.c
new file mode 100644
index 000000000000..1c23456b467e
--- /dev/null
+++ b/mm/tests/lazy_mmu_mode_kunit.c
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <kunit/test.h>
+#include <linux/pgtable.h>
+
+/* For some symbols referenced by arch_{enter,leave}_lazy_mmu_mode on powerpc */
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
+
+static void expect_not_active(struct kunit *test)
+{
+	KUNIT_EXPECT_FALSE(test, is_lazy_mmu_mode_active());
+}
+
+static void expect_active(struct kunit *test)
+{
+	KUNIT_EXPECT_TRUE(test, is_lazy_mmu_mode_active());
+}
+
+static void lazy_mmu_mode_active(struct kunit *test)
+{
+	expect_not_active(test);
+
+	lazy_mmu_mode_enable();
+	expect_active(test);
+
+	{
+		/* Nested section */
+		lazy_mmu_mode_enable();
+		expect_active(test);
+
+		lazy_mmu_mode_disable();
+		expect_active(test);
+	}
+
+	{
+		/* Paused section */
+		lazy_mmu_mode_pause();
+		expect_not_active(test);
+
+		{
+			/* No effect (paused) */
+			lazy_mmu_mode_enable();
+			expect_not_active(test);
+
+			lazy_mmu_mode_disable();
+			expect_not_active(test);
+
+			lazy_mmu_mode_pause();
+			expect_not_active(test);
+
+			lazy_mmu_mode_resume();
+			expect_not_active(test);
+		}
+
+		lazy_mmu_mode_resume();
+		expect_active(test);
+	}
+
+	lazy_mmu_mode_disable();
+	expect_not_active(test);
+}
+
+static struct kunit_case lazy_mmu_mode_test_cases[] = {
+	KUNIT_CASE(lazy_mmu_mode_active),
+	{}
+};
+
+static struct kunit_suite lazy_mmu_mode_test_suite = {
+	.name = "lazy_mmu_mode",
+	.test_cases = lazy_mmu_mode_test_cases,
+};
+kunit_test_suite(lazy_mmu_mode_test_suite);
+
+MODULE_DESCRIPTION("Tests for the lazy MMU mode");
+MODULE_LICENSE("GPL");
diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index e6dfd5f28acd..b11f81095fa5 100644
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -1103,7 +1103,7 @@ static long move_present_ptes(struct mm_struct *mm,
 	/* It's safe to drop the reference now as the page-table is holding one. */
 	folio_put(*first_src_folio);
 	*first_src_folio = NULL;
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 
 	while (true) {
 		orig_src_pte = ptep_get_and_clear(mm, src_addr, src_pte);
@@ -1140,7 +1140,7 @@ static long move_present_ptes(struct mm_struct *mm,
 			break;
 	}
 
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 	if (src_addr > src_start)
 		flush_tlb_range(src_vma, src_start, src_addr);
 
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index ecbac900c35f..54eda74fa382 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -108,7 +108,7 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
 	if (!pte)
 		return -ENOMEM;
 
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 
 	do {
 		if (unlikely(!pte_none(ptep_get(pte)))) {
@@ -134,7 +134,7 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
 		pfn++;
 	} while (pte += PFN_DOWN(size), addr += size, addr != end);
 
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 	*mask |= PGTBL_PTE_MODIFIED;
 	return 0;
 }
@@ -305,6 +305,11 @@ static int vmap_range_noflush(unsigned long addr, unsigned long end,
 	int err;
 	pgtbl_mod_mask mask = 0;
 
+	/*
+	 * Might allocate pagetables (for most archs a more precise annotation
+	 * would be might_alloc(GFP_PGTABLE_KERNEL)). Also might shootdown TLB
+	 * (requires IRQs enabled on x86).
+	 */
 	might_sleep();
 	BUG_ON(addr >= end);
 
@@ -366,7 +371,7 @@ static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
 	unsigned long size = PAGE_SIZE;
 
 	pte = pte_offset_kernel(pmd, addr);
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 
 	do {
 #ifdef CONFIG_HUGETLB_PAGE
@@ -385,7 +390,7 @@ static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
 		WARN_ON(!pte_none(ptent) && !pte_present(ptent));
 	} while (pte += (size >> PAGE_SHIFT), addr += size, addr != end);
 
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 	*mask |= PGTBL_PTE_MODIFIED;
 }
 
@@ -533,7 +538,7 @@ static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr,
 	if (!pte)
 		return -ENOMEM;
 
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 
 	do {
 		struct page *page = pages[*nr];
@@ -555,7 +560,7 @@ static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr,
 		(*nr)++;
 	} while (pte++, addr += PAGE_SIZE, addr != end);
 
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 	*mask |= PGTBL_PTE_MODIFIED;
 
 	return err;
@@ -642,6 +647,29 @@ static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end,
 	return err;
 }
 
+static inline int get_vmap_batch_order(struct page **pages,
+		unsigned int stride, unsigned int max_steps, unsigned int idx)
+{
+	int nr_pages = 1;
+
+	/*
+	 * Currently, batching is only supported in vmap_pages_range
+	 * when page_shift == PAGE_SHIFT.
+	 */
+	if (stride != 1)
+		return 0;
+
+	nr_pages = compound_nr(pages[idx]);
+	if (nr_pages == 1)
+		return 0;
+	if (max_steps < nr_pages)
+		return 0;
+
+	if (num_pages_contiguous(&pages[idx], nr_pages) == nr_pages)
+		return compound_order(pages[idx]);
+	return 0;
+}
+
 /*
  * vmap_pages_range_noflush is similar to vmap_pages_range, but does not
  * flush caches.
@@ -655,23 +683,33 @@ int __vmap_pages_range_noflush(unsigned long addr, unsigned long end,
 		pgprot_t prot, struct page **pages, unsigned int page_shift)
 {
 	unsigned int i, nr = (end - addr) >> PAGE_SHIFT;
+	unsigned int stride;
 
 	WARN_ON(page_shift < PAGE_SHIFT);
 
+	/*
+	 * For vmap(), users may allocate pages from high orders down to
+	 * order 0, while always using PAGE_SHIFT as the page_shift.
+	 * We first check whether the initial page is a compound page. If so,
+	 * there may be an opportunity to batch multiple pages together.
+	 */
 	if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMALLOC) ||
-			page_shift == PAGE_SHIFT)
+			(page_shift == PAGE_SHIFT && !PageCompound(pages[0])))
 		return vmap_small_pages_range_noflush(addr, end, prot, pages);
 
-	for (i = 0; i < nr; i += 1U << (page_shift - PAGE_SHIFT)) {
-		int err;
+	stride = 1U << (page_shift - PAGE_SHIFT);
+	for (i = 0; i < nr; ) {
+		int err, order;
 
-		err = vmap_range_noflush(addr, addr + (1UL << page_shift),
+		order = get_vmap_batch_order(pages, stride, nr - i, i);
+		err = vmap_range_noflush(addr, addr + (1UL << (page_shift + order)),
 					page_to_phys(pages[i]), prot,
-					page_shift);
+					page_shift + order);
 		if (err)
 			return err;
 
-		addr += 1UL << page_shift;
+		addr += 1UL  << (page_shift + order);
+		i += 1U << (order + page_shift - PAGE_SHIFT);
 	}
 
 	return 0;
@@ -4331,7 +4369,9 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align
 	 */
 	if (size <= alloced_size) {
 		kasan_unpoison_vmalloc(p + old_size, size - old_size,
-				       KASAN_VMALLOC_PROT_NORMAL);
+				       KASAN_VMALLOC_PROT_NORMAL |
+				       KASAN_VMALLOC_VM_ALLOC |
+				       KASAN_VMALLOC_KEEP_TAG);
 		/*
 		 * No need to zero memory here, as unused memory will have
 		 * already been zeroed at initial allocation time or during
@@ -5025,9 +5065,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
 	 * With hardware tag-based KASAN, marking is skipped for
 	 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
 	 */
-	for (area = 0; area < nr_vms; area++)
-		vms[area]->addr = kasan_unpoison_vmalloc(vms[area]->addr,
-				vms[area]->size, KASAN_VMALLOC_PROT_NORMAL);
+	kasan_unpoison_vmap_areas(vms, nr_vms, KASAN_VMALLOC_PROT_NORMAL);
 
 	kfree(vas);
 	return vms;
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 670fe9fae5ba..76e9864447cc 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1276,58 +1276,58 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
 		 * Try to allocate it some swap space here.
 		 * Lazyfree folio could be freed directly
 		 */
-		if (folio_test_anon(folio) && folio_test_swapbacked(folio)) {
-			if (!folio_test_swapcache(folio)) {
-				if (!(sc->gfp_mask & __GFP_IO))
-					goto keep_locked;
-				if (folio_maybe_dma_pinned(folio))
-					goto keep_locked;
-				if (folio_test_large(folio)) {
-					/* cannot split folio, skip it */
-					if (folio_expected_ref_count(folio) !=
-					    folio_ref_count(folio) - 1)
-						goto activate_locked;
-					/*
-					 * Split partially mapped folios right away.
-					 * We can free the unmapped pages without IO.
-					 */
-					if (data_race(!list_empty(&folio->_deferred_list) &&
-					    folio_test_partially_mapped(folio)) &&
-					    split_folio_to_list(folio, folio_list))
-						goto activate_locked;
-				}
-				if (folio_alloc_swap(folio)) {
-					int __maybe_unused order = folio_order(folio);
-
-					if (!folio_test_large(folio))
-						goto activate_locked_split;
-					/* Fallback to swap normal pages */
-					if (split_folio_to_list(folio, folio_list))
-						goto activate_locked;
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
-					if (nr_pages >= HPAGE_PMD_NR) {
-						count_memcg_folio_events(folio,
-							THP_SWPOUT_FALLBACK, 1);
-						count_vm_event(THP_SWPOUT_FALLBACK);
-					}
-#endif
-					count_mthp_stat(order, MTHP_STAT_SWPOUT_FALLBACK);
-					if (folio_alloc_swap(folio))
-						goto activate_locked_split;
-				}
+		if (folio_test_anon(folio) && folio_test_swapbacked(folio) &&
+				!folio_test_swapcache(folio)) {
+			if (!(sc->gfp_mask & __GFP_IO))
+				goto keep_locked;
+			if (folio_maybe_dma_pinned(folio))
+				goto keep_locked;
+			if (folio_test_large(folio)) {
+				/* cannot split folio, skip it */
+				if (folio_expected_ref_count(folio) !=
+				    folio_ref_count(folio) - 1)
+					goto activate_locked;
 				/*
-				 * Normally the folio will be dirtied in unmap because its
-				 * pte should be dirty. A special case is MADV_FREE page. The
-				 * page's pte could have dirty bit cleared but the folio's
-				 * SwapBacked flag is still set because clearing the dirty bit
-				 * and SwapBacked flag has no lock protected. For such folio,
-				 * unmap will not set dirty bit for it, so folio reclaim will
-				 * not write the folio out. This can cause data corruption when
-				 * the folio is swapped in later. Always setting the dirty flag
-				 * for the folio solves the problem.
+				 * Split partially mapped folios right away.
+				 * We can free the unmapped pages without IO.
 				 */
-				folio_mark_dirty(folio);
+				if (data_race(!list_empty(&folio->_deferred_list) &&
+				    folio_test_partially_mapped(folio)) &&
+				    split_folio_to_list(folio, folio_list))
+					goto activate_locked;
+			}
+			if (folio_alloc_swap(folio)) {
+				int __maybe_unused order = folio_order(folio);
+
+				if (!folio_test_large(folio))
+					goto activate_locked_split;
+				/* Fallback to swap normal pages */
+				if (split_folio_to_list(folio, folio_list))
+					goto activate_locked;
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+				if (nr_pages >= HPAGE_PMD_NR) {
+					count_memcg_folio_events(folio,
+						THP_SWPOUT_FALLBACK, 1);
+					count_vm_event(THP_SWPOUT_FALLBACK);
+				}
+#endif
+				count_mthp_stat(order, MTHP_STAT_SWPOUT_FALLBACK);
+				if (folio_alloc_swap(folio))
+					goto activate_locked_split;
 			}
+			/*
+			 * Normally the folio will be dirtied in unmap because
+			 * its pte should be dirty. A special case is MADV_FREE
+			 * page. The page's pte could have dirty bit cleared but
+			 * the folio's SwapBacked flag is still set because
+			 * clearing the dirty bit and SwapBacked flag has no
+			 * lock protected. For such folio, unmap will not set
+			 * dirty bit for it, so folio reclaim will not write the
+			 * folio out. This can cause data corruption when the
+			 * folio is swapped in later. Always setting the dirty
+			 * flag for the folio solves the problem.
+			 */
+			folio_mark_dirty(folio);
 		}
 
 		/*
@@ -3516,7 +3516,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
 		return false;
 	}
 
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 restart:
 	for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) {
 		unsigned long pfn;
@@ -3557,7 +3557,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
 	if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end))
 		goto restart;
 
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 	pte_unmap_unlock(pte, ptl);
 
 	return suitable_to_scan(total, young);
@@ -3598,7 +3598,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
 	if (!spin_trylock(ptl))
 		goto done;
 
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 
 	do {
 		unsigned long pfn;
@@ -3645,7 +3645,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
 
 	walk_update_folio(walk, last, gen, dirty);
 
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 	spin_unlock(ptl);
 done:
 	*first = -1;
@@ -4244,7 +4244,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
 		}
 	}
 
-	arch_enter_lazy_mmu_mode();
+	lazy_mmu_mode_enable();
 
 	pte -= (addr - start) / PAGE_SIZE;
 
@@ -4278,7 +4278,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
 
 	walk_update_folio(walk, last, gen, dirty);
 
-	arch_leave_lazy_mmu_mode();
+	lazy_mmu_mode_disable();
 
 	/* feedback from rmap walkers to page table walkers */
 	if (mm_state && suitable_to_scan(i, young))
diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c
index c3f7828bf9d5..1a4b6badf2b3 100644
--- a/net/bluetooth/hci_conn.c
+++ b/net/bluetooth/hci_conn.c
@@ -1002,12 +1002,18 @@ static struct hci_conn *__hci_conn_add(struct hci_dev *hdev, int type,
 	switch (type) {
 	case ACL_LINK:
 		conn->pkt_type = hdev->pkt_type & ACL_PTYPE_MASK;
+		conn->link_policy = hdev->link_policy;
 		conn->mtu = hdev->acl_mtu;
 		break;
 	case LE_LINK:
 		/* conn->src should reflect the local identity address */
 		hci_copy_identity_address(hdev, &conn->src, &conn->src_type);
 		conn->mtu = hdev->le_mtu ? hdev->le_mtu : hdev->acl_mtu;
+		/* Use the controller supported PHYS as default until the
+		 * remote features are resolved.
+		 */
+		conn->le_tx_def_phys = hdev->le_tx_def_phys;
+		conn->le_rx_def_phys = hdev->le_tx_def_phys;
 		break;
 	case CIS_LINK:
 		/* conn->src should reflect the local identity address */
@@ -2614,8 +2620,8 @@ void hci_conn_enter_active_mode(struct hci_conn *conn, __u8 force_active)
 
 timer:
 	if (hdev->idle_timeout > 0)
-		queue_delayed_work(hdev->workqueue, &conn->idle_work,
-				   msecs_to_jiffies(hdev->idle_timeout));
+		mod_delayed_work(hdev->workqueue, &conn->idle_work,
+				 msecs_to_jiffies(hdev->idle_timeout));
 }
 
 /* Drop all connection on the device */
@@ -2928,22 +2934,22 @@ u32 hci_conn_get_phy(struct hci_conn *conn)
 		break;
 
 	case LE_LINK:
-		if (conn->le_tx_phy & HCI_LE_SET_PHY_1M)
+		if (conn->le_tx_def_phys & HCI_LE_SET_PHY_1M)
 			phys |= BT_PHY_LE_1M_TX;
 
-		if (conn->le_rx_phy & HCI_LE_SET_PHY_1M)
+		if (conn->le_rx_def_phys & HCI_LE_SET_PHY_1M)
 			phys |= BT_PHY_LE_1M_RX;
 
-		if (conn->le_tx_phy & HCI_LE_SET_PHY_2M)
+		if (conn->le_tx_def_phys & HCI_LE_SET_PHY_2M)
 			phys |= BT_PHY_LE_2M_TX;
 
-		if (conn->le_rx_phy & HCI_LE_SET_PHY_2M)
+		if (conn->le_rx_def_phys & HCI_LE_SET_PHY_2M)
 			phys |= BT_PHY_LE_2M_RX;
 
-		if (conn->le_tx_phy & HCI_LE_SET_PHY_CODED)
+		if (conn->le_tx_def_phys & HCI_LE_SET_PHY_CODED)
 			phys |= BT_PHY_LE_CODED_TX;
 
-		if (conn->le_rx_phy & HCI_LE_SET_PHY_CODED)
+		if (conn->le_rx_def_phys & HCI_LE_SET_PHY_CODED)
 			phys |= BT_PHY_LE_CODED_RX;
 
 		break;
@@ -2952,6 +2958,111 @@ u32 hci_conn_get_phy(struct hci_conn *conn)
 	return phys;
 }
 
+static u16 bt_phy_pkt_type(struct hci_conn *conn, u32 phys)
+{
+	u16 pkt_type = conn->pkt_type;
+
+	if (phys & BT_PHY_BR_1M_3SLOT)
+		pkt_type |= HCI_DM3 | HCI_DH3;
+	else
+		pkt_type &= ~(HCI_DM3 | HCI_DH3);
+
+	if (phys & BT_PHY_BR_1M_5SLOT)
+		pkt_type |= HCI_DM5 | HCI_DH5;
+	else
+		pkt_type &= ~(HCI_DM5 | HCI_DH5);
+
+	if (phys & BT_PHY_EDR_2M_1SLOT)
+		pkt_type &= ~HCI_2DH1;
+	else
+		pkt_type |= HCI_2DH1;
+
+	if (phys & BT_PHY_EDR_2M_3SLOT)
+		pkt_type &= ~HCI_2DH3;
+	else
+		pkt_type |= HCI_2DH3;
+
+	if (phys & BT_PHY_EDR_2M_5SLOT)
+		pkt_type &= ~HCI_2DH5;
+	else
+		pkt_type |= HCI_2DH5;
+
+	if (phys & BT_PHY_EDR_3M_1SLOT)
+		pkt_type &= ~HCI_3DH1;
+	else
+		pkt_type |= HCI_3DH1;
+
+	if (phys & BT_PHY_EDR_3M_3SLOT)
+		pkt_type &= ~HCI_3DH3;
+	else
+		pkt_type |= HCI_3DH3;
+
+	if (phys & BT_PHY_EDR_3M_5SLOT)
+		pkt_type &= ~HCI_3DH5;
+	else
+		pkt_type |= HCI_3DH5;
+
+	return pkt_type;
+}
+
+static int bt_phy_le_phy(u32 phys, u8 *tx_phys, u8 *rx_phys)
+{
+	if (!tx_phys || !rx_phys)
+		return -EINVAL;
+
+	*tx_phys = 0;
+	*rx_phys = 0;
+
+	if (phys & BT_PHY_LE_1M_TX)
+		*tx_phys |= HCI_LE_SET_PHY_1M;
+
+	if (phys & BT_PHY_LE_1M_RX)
+		*rx_phys |= HCI_LE_SET_PHY_1M;
+
+	if (phys & BT_PHY_LE_2M_TX)
+		*tx_phys |= HCI_LE_SET_PHY_2M;
+
+	if (phys & BT_PHY_LE_2M_RX)
+		*rx_phys |= HCI_LE_SET_PHY_2M;
+
+	if (phys & BT_PHY_LE_CODED_TX)
+		*tx_phys |= HCI_LE_SET_PHY_CODED;
+
+	if (phys & BT_PHY_LE_CODED_RX)
+		*rx_phys |= HCI_LE_SET_PHY_CODED;
+
+	return 0;
+}
+
+int hci_conn_set_phy(struct hci_conn *conn, u32 phys)
+{
+	u8 tx_phys, rx_phys;
+
+	switch (conn->type) {
+	case SCO_LINK:
+	case ESCO_LINK:
+		return -EINVAL;
+	case ACL_LINK:
+		/* Only allow setting BR/EDR PHYs if link type is ACL */
+		if (phys & ~BT_PHY_BREDR_MASK)
+			return -EINVAL;
+
+		return hci_acl_change_pkt_type(conn,
+					       bt_phy_pkt_type(conn, phys));
+	case LE_LINK:
+		/* Only allow setting LE PHYs if link type is LE */
+		if (phys & ~BT_PHY_LE_MASK)
+			return -EINVAL;
+
+		if (bt_phy_le_phy(phys, &tx_phys, &rx_phys))
+			return -EINVAL;
+
+		return hci_le_set_phy(conn, tx_phys, rx_phys);
+	default:
+		return -EINVAL;
+	}
+}
+
 static int abort_conn_sync(struct hci_dev *hdev, void *data)
 {
 	struct hci_conn *conn = data;
diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c
index a9868f17ef40..467710a42d45 100644
--- a/net/bluetooth/hci_event.c
+++ b/net/bluetooth/hci_event.c
@@ -2869,6 +2869,31 @@ static void hci_cs_le_ext_create_conn(struct hci_dev *hdev, u8 status)
 	hci_dev_unlock(hdev);
 }
 
+static void hci_cs_le_set_phy(struct hci_dev *hdev, u8 status)
+{
+	struct hci_cp_le_set_phy *cp;
+	struct hci_conn *conn;
+
+	bt_dev_dbg(hdev, "status 0x%2.2x", status);
+
+	if (status)
+		return;
+
+	cp = hci_sent_cmd_data(hdev, HCI_OP_LE_SET_PHY);
+	if (!cp)
+		return;
+
+	hci_dev_lock(hdev);
+
+	conn = hci_conn_hash_lookup_handle(hdev, __le16_to_cpu(cp->handle));
+	if (conn) {
+		conn->le_tx_def_phys = cp->tx_phys;
+		conn->le_rx_def_phys = cp->rx_phys;
+	}
+
+	hci_dev_unlock(hdev);
+}
+
 static void hci_cs_le_read_remote_features(struct hci_dev *hdev, u8 status)
 {
 	struct hci_cp_le_read_remote_features *cp;
@@ -4359,6 +4384,7 @@ static const struct hci_cs {
 	HCI_CS(HCI_OP_LE_CREATE_CONN, hci_cs_le_create_conn),
 	HCI_CS(HCI_OP_LE_READ_REMOTE_FEATURES, hci_cs_le_read_remote_features),
 	HCI_CS(HCI_OP_LE_START_ENC, hci_cs_le_start_enc),
+	HCI_CS(HCI_OP_LE_SET_PHY, hci_cs_le_set_phy),
 	HCI_CS(HCI_OP_LE_EXT_CREATE_CONN, hci_cs_le_ext_create_conn),
 	HCI_CS(HCI_OP_LE_CREATE_CIS, hci_cs_le_create_cis),
 	HCI_CS(HCI_OP_LE_CREATE_BIG, hci_cs_le_create_big),
@@ -6607,8 +6633,20 @@ static void hci_le_remote_feat_complete_evt(struct hci_dev *hdev, void *data,
 
 	conn = hci_conn_hash_lookup_handle(hdev, __le16_to_cpu(ev->handle));
 	if (conn) {
-		if (!ev->status)
-			memcpy(conn->features[0], ev->features, 8);
+		if (!ev->status) {
+			memcpy(conn->le_features, ev->features, 8);
+
+			/* Update supported PHYs */
+			if (!(conn->le_features[1] & HCI_LE_PHY_2M)) {
+				conn->le_tx_def_phys &= ~HCI_LE_SET_PHY_2M;
+				conn->le_rx_def_phys &= ~HCI_LE_SET_PHY_2M;
+			}
+
+			if (!(conn->le_features[1] & HCI_LE_PHY_CODED)) {
+				conn->le_tx_def_phys &= ~HCI_LE_SET_PHY_CODED;
+				conn->le_rx_def_phys &= ~HCI_LE_SET_PHY_CODED;
+			}
+		}
 
 		if (conn->state == BT_CONFIG) {
 			__u8 status;
@@ -7221,9 +7259,21 @@ static void hci_le_read_all_remote_features_evt(struct hci_dev *hdev,
 	if (!conn)
 		goto unlock;
 
-	if (!ev->status)
+	if (!ev->status) {
 		memcpy(conn->le_features, ev->features, 248);
 
+		/* Update supported PHYs */
+		if (!(conn->le_features[1] & HCI_LE_PHY_2M)) {
+			conn->le_tx_def_phys &= ~HCI_LE_SET_PHY_2M;
+			conn->le_rx_def_phys &= ~HCI_LE_SET_PHY_2M;
+		}
+
+		if (!(conn->le_features[1] & HCI_LE_PHY_CODED)) {
+			conn->le_tx_def_phys &= ~HCI_LE_SET_PHY_CODED;
+			conn->le_rx_def_phys &= ~HCI_LE_SET_PHY_CODED;
+		}
+	}
+
 	if (conn->state == BT_CONFIG) {
 		__u8 status;
 
diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c
index a9f5b1a68356..5de98be752bc 100644
--- a/net/bluetooth/hci_sync.c
+++ b/net/bluetooth/hci_sync.c
@@ -4427,6 +4427,17 @@ static int hci_le_set_event_mask_sync(struct hci_dev *hdev)
 		events[4] |= 0x02;	/* LE BIG Info Advertising Report */
 	}
 
+	if (le_cs_capable(hdev)) {
+		/* Channel Sounding events */
+		events[5] |= 0x08;	/* LE CS Read Remote Supported Cap Complete event */
+		events[5] |= 0x10;	/* LE CS Read Remote FAE Table Complete event */
+		events[5] |= 0x20;	/* LE CS Security Enable Complete event */
+		events[5] |= 0x40;	/* LE CS Config Complete event */
+		events[5] |= 0x80;	/* LE CS Procedure Enable Complete event */
+		events[6] |= 0x01;	/* LE CS Subevent Result event */
+		events[6] |= 0x02;	/* LE CS Subevent Result Continue event */
+		events[6] |= 0x04;	/* LE CS Test End Complete event */
+	}
 	return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EVENT_MASK,
 				     sizeof(events), events, HCI_CMD_TIMEOUT);
 }
@@ -4572,6 +4583,10 @@ static int hci_le_set_host_feature_sync(struct hci_dev *hdev)
 	cp.bit_number = 32;
 	cp.bit_value = iso_enabled(hdev) ? 0x01 : 0x00;
 
+	/* Channel Sounding (Host Support) */
+	cp.bit_number = 47;
+	cp.bit_value = le_cs_capable(hdev) ? 0x01 : 0x00;
+
 	return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_HOST_FEATURE,
 				     sizeof(cp), &cp, HCI_CMD_TIMEOUT);
 }
@@ -6896,8 +6911,6 @@ static int hci_acl_create_conn_sync(struct hci_dev *hdev, void *data)
 
 	conn->attempt++;
 
-	conn->link_policy = hdev->link_policy;
-
 	memset(&cp, 0, sizeof(cp));
 	bacpy(&cp.bdaddr, &conn->dst);
 	cp.pscan_rep_mode = 0x02;
@@ -7418,3 +7431,75 @@ int hci_le_read_remote_features(struct hci_conn *conn)
 
 	return err;
 }
+
+static void pkt_type_changed(struct hci_dev *hdev, void *data, int err)
+{
+	struct hci_cp_change_conn_ptype *cp = data;
+
+	bt_dev_dbg(hdev, "err %d", err);
+
+	kfree(cp);
+}
+
+static int hci_change_conn_ptype_sync(struct hci_dev *hdev, void *data)
+{
+	struct hci_cp_change_conn_ptype *cp = data;
+
+	return __hci_cmd_sync_status_sk(hdev, HCI_OP_CHANGE_CONN_PTYPE,
+					sizeof(*cp), cp,
+					HCI_EV_PKT_TYPE_CHANGE,
+					HCI_CMD_TIMEOUT, NULL);
+}
+
+int hci_acl_change_pkt_type(struct hci_conn *conn, u16 pkt_type)
+{
+	struct hci_dev *hdev = conn->hdev;
+	struct hci_cp_change_conn_ptype *cp;
+
+	cp = kmalloc(sizeof(*cp), GFP_KERNEL);
+	if (!cp)
+		return -ENOMEM;
+
+	cp->handle = cpu_to_le16(conn->handle);
+	cp->pkt_type = cpu_to_le16(pkt_type);
+
+	return hci_cmd_sync_queue_once(hdev, hci_change_conn_ptype_sync, cp,
+				       pkt_type_changed);
+}
+
+static void le_phy_update_complete(struct hci_dev *hdev, void *data, int err)
+{
+	struct hci_cp_le_set_phy *cp = data;
+
+	bt_dev_dbg(hdev, "err %d", err);
+
+	kfree(cp);
+}
+
+static int hci_le_set_phy_sync(struct hci_dev *hdev, void *data)
+{
+	struct hci_cp_le_set_phy *cp = data;
+
+	return __hci_cmd_sync_status_sk(hdev, HCI_OP_LE_SET_PHY,
+					sizeof(*cp), cp,
+					HCI_EV_LE_PHY_UPDATE_COMPLETE,
+					HCI_CMD_TIMEOUT, NULL);
+}
+
+int hci_le_set_phy(struct hci_conn *conn, u8 tx_phys, u8 rx_phys)
+{
+	struct hci_dev *hdev = conn->hdev;
+	struct hci_cp_le_set_phy *cp;
+
+	cp = kmalloc(sizeof(*cp), GFP_KERNEL);
+	if (!cp)
+		return -ENOMEM;
+
+	memset(cp, 0, sizeof(*cp));
+	cp->handle = cpu_to_le16(conn->handle);
+	cp->tx_phys = tx_phys;
+	cp->rx_phys = rx_phys;
+
+	return hci_cmd_sync_queue_once(hdev, hci_le_set_phy_sync, cp,
+				       le_phy_update_complete);
+}
diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c
index 9ee189c815d4..3ba3ce7eaa98 100644
--- a/net/bluetooth/l2cap_sock.c
+++ b/net/bluetooth/l2cap_sock.c
@@ -885,7 +885,7 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname,
 	struct bt_power pwr;
 	struct l2cap_conn *conn;
 	int err = 0;
-	u32 opt;
+	u32 opt, phys;
 	u16 mtu;
 	u8 mode;
 
@@ -1059,6 +1059,24 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname,
 
 		break;
 
+	case BT_PHY:
+		if (sk->sk_state != BT_CONNECTED) {
+			err = -ENOTCONN;
+			break;
+		}
+
+		err = copy_safe_from_sockptr(&phys, sizeof(phys), optval,
+					     optlen);
+		if (err)
+			break;
+
+		if (!chan->conn)
+			break;
+
+		conn = chan->conn;
+		err = hci_conn_set_phy(conn->hcon, phys);
+		break;
+
 	case BT_MODE:
 		if (!enable_ecred) {
 			err = -ENOPROTOOPT;
diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c
index c11cdef42b6f..5be9b8c91949 100644
--- a/net/bluetooth/mgmt.c
+++ b/net/bluetooth/mgmt.c
@@ -849,6 +849,12 @@ static u32 get_supported_settings(struct hci_dev *hdev)
 	if (cis_peripheral_capable(hdev))
 		settings |= MGMT_SETTING_CIS_PERIPHERAL;
 
+	if (bis_capable(hdev))
+		settings |= MGMT_SETTING_ISO_BROADCASTER;
+
+	if (sync_recv_capable(hdev))
+		settings |= MGMT_SETTING_ISO_SYNC_RECEIVER;
+
 	if (ll_privacy_capable(hdev))
 		settings |= MGMT_SETTING_LL_PRIVACY;
 
diff --git a/net/bluetooth/mgmt_config.c b/net/bluetooth/mgmt_config.c
index c4063d200c0a..fdcc752c6f13 100644
--- a/net/bluetooth/mgmt_config.c
+++ b/net/bluetooth/mgmt_config.c
@@ -11,6 +11,12 @@
 #include "mgmt_util.h"
 #include "mgmt_config.h"
 
+#define HDEV_PARAM_U32(_param_name_) \
+	struct {\
+		struct mgmt_tlv_hdr entry; \
+		__le32 value; \
+	} __packed _param_name_
+
 #define HDEV_PARAM_U16(_param_name_) \
 	struct {\
 		struct mgmt_tlv_hdr entry; \
@@ -29,6 +35,12 @@
 		cpu_to_le16(hdev->_param_name_) \
 	}
 
+#define TLV_SET_U32(_param_code_, _param_name_) \
+	{ \
+		{ cpu_to_le16(_param_code_), sizeof(__u32) }, \
+		cpu_to_le32(hdev->_param_name_) \
+	}
+
 #define TLV_SET_U8(_param_code_, _param_name_) \
 	{ \
 		{ cpu_to_le16(_param_code_), sizeof(__u8) }, \
@@ -78,6 +90,7 @@ int read_def_system_config(struct sock *sk, struct hci_dev *hdev, void *data,
 		HDEV_PARAM_U16(advmon_allowlist_duration);
 		HDEV_PARAM_U16(advmon_no_filter_duration);
 		HDEV_PARAM_U8(enable_advmon_interleave_scan);
+		HDEV_PARAM_U32(idle_timeout);
 	} __packed rp = {
 		TLV_SET_U16(0x0000, def_page_scan_type),
 		TLV_SET_U16(0x0001, def_page_scan_int),
@@ -111,6 +124,7 @@ int read_def_system_config(struct sock *sk, struct hci_dev *hdev, void *data,
 		TLV_SET_U16(0x001d, advmon_allowlist_duration),
 		TLV_SET_U16(0x001e, advmon_no_filter_duration),
 		TLV_SET_U8(0x001f, enable_advmon_interleave_scan),
+		TLV_SET_U32(0x0020, idle_timeout),
 	};
 
 	bt_dev_dbg(hdev, "sock %p", sk);
@@ -122,6 +136,7 @@ int read_def_system_config(struct sock *sk, struct hci_dev *hdev, void *data,
 }
 
 #define TO_TLV(x)		((struct mgmt_tlv *)(x))
+#define TLV_GET_LE32(tlv)	le32_to_cpu(*((__le32 *)(TO_TLV(tlv)->value)))
 #define TLV_GET_LE16(tlv)	le16_to_cpu(*((__le16 *)(TO_TLV(tlv)->value)))
 #define TLV_GET_U8(tlv)		(*((__u8 *)(TO_TLV(tlv)->value)))
 
@@ -191,6 +206,9 @@ int set_def_system_config(struct sock *sk, struct hci_dev *hdev, void *data,
 		case 0x001f:
 			exp_type_len = sizeof(u8);
 			break;
+		case 0x0020:
+			exp_type_len = sizeof(u32);
+			break;
 		default:
 			exp_type_len = 0;
 			bt_dev_warn(hdev, "unsupported parameter %u", type);
@@ -314,6 +332,9 @@ int set_def_system_config(struct sock *sk, struct hci_dev *hdev, void *data,
 		case 0x0001f:
 			hdev->enable_advmon_interleave_scan = TLV_GET_U8(buffer);
 			break;
+		case 0x00020:
+			hdev->idle_timeout = TLV_GET_LE32(buffer);
+			break;
 		default:
 			bt_dev_warn(hdev, "unsupported parameter %u", type);
 			break;
diff --git a/net/bridge/br_sysfs_br.c b/net/bridge/br_sysfs_br.c
index cb4855ed9500..dcd727345cac 100644
--- a/net/bridge/br_sysfs_br.c
+++ b/net/bridge/br_sysfs_br.c
@@ -11,6 +11,7 @@
 #include <linux/kernel.h>
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
+#include <linux/hex.h>
 #include <linux/if_bridge.h>
 #include <linux/rtnetlink.h>
 #include <linux/spinlock.h>
diff --git a/net/caif/cffrml.c b/net/caif/cffrml.c
index 6651a8dc62e0..d4d63586053a 100644
--- a/net/caif/cffrml.c
+++ b/net/caif/cffrml.c
@@ -92,8 +92,15 @@ static int cffrml_receive(struct cflayer *layr, struct cfpkt *pkt)
 	len = le16_to_cpu(tmp);
 
 	/* Subtract for FCS on length if FCS is not used. */
-	if (!this->dofcs)
+	if (!this->dofcs) {
+		if (len < 2) {
+			++cffrml_rcv_error;
+			pr_err("Invalid frame length (%d)\n", len);
+			cfpkt_destroy(pkt);
+			return -EPROTO;
+		}
 		len -= 2;
+	}
 
 	if (cfpkt_setlen(pkt, len) < 0) {
 		++cffrml_rcv_error;
diff --git a/net/can/Kconfig b/net/can/Kconfig
index e4ccf731a24c..af64a6f76458 100644
--- a/net/can/Kconfig
+++ b/net/can/Kconfig
@@ -5,7 +5,6 @@
 
 menuconfig CAN
 	tristate "CAN bus subsystem support"
-	select CAN_DEV
 	help
 	  Controller Area Network (CAN) is a slow (up to 1Mbit/s) serial
 	  communications protocol. Development of the CAN bus started in
diff --git a/net/can/j1939/j1939-priv.h b/net/can/j1939/j1939-priv.h
index 81f58924b4ac..f90e6c147a5f 100644
--- a/net/can/j1939/j1939-priv.h
+++ b/net/can/j1939/j1939-priv.h
@@ -342,4 +342,10 @@ void j1939_session_timers_cancel(struct j1939_session *session);
 /* CAN protocol */
 extern const struct can_proto j1939_can_proto;
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+void save_priv_trace_buffer(struct j1939_priv *priv, int delta);
+#else
+static inline void save_priv_trace_buffer(struct j1939_priv *priv, int delta) { };
+#endif
+
 #endif /* _J1939_PRIV_H_ */
diff --git a/net/can/j1939/main.c b/net/can/j1939/main.c
index a93af55df5fd..463c3ddbee6c 100644
--- a/net/can/j1939/main.c
+++ b/net/can/j1939/main.c
@@ -124,6 +124,14 @@ static void j1939_can_recv(struct sk_buff *iskb, void *data)
 
 static DEFINE_MUTEX(j1939_netdev_lock);
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+static void dump_priv_trace_buffer(const struct net_device *ndev);
+static void erase_priv_trace_buffer(struct j1939_priv *priv);
+#else
+static inline void dump_priv_trace_buffer(const struct net_device *ndev) { };
+static inline void erase_priv_trace_buffer(struct j1939_priv *priv) { };
+#endif
+
 static struct j1939_priv *j1939_priv_create(struct net_device *ndev)
 {
 	struct j1939_priv *priv;
@@ -137,6 +145,7 @@ static struct j1939_priv *j1939_priv_create(struct net_device *ndev)
 	priv->ndev = ndev;
 	kref_init(&priv->kref);
 	kref_init(&priv->rx_kref);
+	save_priv_trace_buffer(priv, 1);
 	dev_hold(ndev);
 
 	netdev_dbg(priv->ndev, "%s : 0x%p\n", __func__, priv);
@@ -164,17 +173,20 @@ static void __j1939_priv_release(struct kref *kref)
 	WARN_ON_ONCE(!list_empty(&priv->j1939_socks));
 
 	dev_put(ndev);
+	erase_priv_trace_buffer(priv);
 	kfree(priv);
 }
 
 void j1939_priv_put(struct j1939_priv *priv)
 {
+	save_priv_trace_buffer(priv, -1);
 	kref_put(&priv->kref, __j1939_priv_release);
 }
 
 void j1939_priv_get(struct j1939_priv *priv)
 {
 	kref_get(&priv->kref);
+	save_priv_trace_buffer(priv, 1);
 }
 
 static int j1939_can_rx_register(struct j1939_priv *priv)
@@ -282,6 +294,7 @@ struct j1939_priv *j1939_netdev_start(struct net_device *ndev)
 		kref_get(&priv_new->rx_kref);
 		mutex_unlock(&j1939_netdev_lock);
 		dev_put(ndev);
+		erase_priv_trace_buffer(priv);
 		kfree(priv);
 		return priv_new;
 	}
@@ -299,6 +312,7 @@ struct j1939_priv *j1939_netdev_start(struct net_device *ndev)
 	mutex_unlock(&j1939_netdev_lock);
 
 	dev_put(ndev);
+	erase_priv_trace_buffer(priv);
 	kfree(priv);
 
 	return ERR_PTR(ret);
@@ -364,6 +378,9 @@ static int j1939_netdev_notify(struct notifier_block *nb,
 	struct can_ml_priv *can_ml = can_get_ml_priv(ndev);
 	struct j1939_priv *priv;
 
+	if (msg == NETDEV_DEBUG_UNREGISTER)
+		dump_priv_trace_buffer(ndev);
+
 	if (!can_ml)
 		goto notify_done;
 
@@ -428,3 +445,79 @@ static __exit void j1939_module_exit(void)
 
 module_init(j1939_module_init);
 module_exit(j1939_module_exit);
+
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+
+#define PRIV_TRACE_BUFFER_SIZE 1024
+static struct priv_trace_buffer {
+	struct j1939_priv *priv; // no-ref
+	struct net_device *ndev; // no-ref
+	atomic_t count;
+	int nr_entries;
+	unsigned long entries[20];
+} priv_trace_buffer[PRIV_TRACE_BUFFER_SIZE];
+static bool priv_trace_buffer_exhausted;
+
+static void dump_priv_trace_buffer(const struct net_device *ndev)
+{
+	struct priv_trace_buffer *ptr;
+	int count, balance = 0;
+	int i;
+
+	for (i = 0; i < PRIV_TRACE_BUFFER_SIZE; i++) {
+		ptr = &priv_trace_buffer[i];
+		if (!ptr->priv || ptr->ndev != ndev)
+			continue;
+		count = atomic_read(&ptr->count);
+		balance += count;
+		pr_info("Call trace for %s@%p %+d at\n", ndev->name, ptr->priv, count);
+		stack_trace_print(ptr->entries, ptr->nr_entries, 4);
+	}
+	if (!priv_trace_buffer_exhausted)
+		pr_info("balance for %s@j1939_priv is %d\n", ndev->name, balance);
+	else
+		pr_info("balance for %s@j1939_priv is unknown\n", ndev->name);
+}
+
+static void erase_priv_trace_buffer(struct j1939_priv *priv)
+{
+	int i;
+
+	for (i = 0; i < PRIV_TRACE_BUFFER_SIZE; i++)
+		if (priv_trace_buffer[i].priv == priv)
+			priv_trace_buffer[i].priv = NULL;
+}
+
+void save_priv_trace_buffer(struct j1939_priv *priv, int delta)
+{
+	struct priv_trace_buffer *ptr;
+	unsigned long entries[ARRAY_SIZE(ptr->entries)];
+	unsigned long nr_entries;
+	int i;
+
+	if (in_nmi())
+		return;
+	nr_entries = stack_trace_save(entries, ARRAY_SIZE(ptr->entries), 1);
+	nr_entries = trim_netdev_trace(entries, nr_entries);
+	for (i = 0; i < PRIV_TRACE_BUFFER_SIZE; i++) {
+		ptr = &priv_trace_buffer[i];
+		if (ptr->priv == priv && ptr->nr_entries == nr_entries &&
+		    !memcmp(ptr->entries, entries, nr_entries * sizeof(unsigned long))) {
+			atomic_add(delta, &ptr->count);
+			return;
+		}
+	}
+	for (i = 0; i < PRIV_TRACE_BUFFER_SIZE; i++) {
+		ptr = &priv_trace_buffer[i];
+		if (!ptr->priv && !cmpxchg(&ptr->priv, NULL, priv)) {
+			ptr->ndev = priv->ndev;
+			atomic_set(&ptr->count, delta);
+			ptr->nr_entries = nr_entries;
+			memmove(ptr->entries, entries, nr_entries * sizeof(unsigned long));
+			return;
+		}
+	}
+	priv_trace_buffer_exhausted = true;
+}
+
+#endif
diff --git a/net/can/j1939/socket.c b/net/can/j1939/socket.c
index 6272326dd614..ff9c4fd7b433 100644
--- a/net/can/j1939/socket.c
+++ b/net/can/j1939/socket.c
@@ -482,6 +482,12 @@ static int j1939_sk_bind(struct socket *sock, struct sockaddr_unsized *uaddr, in
 			goto out_release_sock;
 		}
 
+		if (ndev->reg_state != NETREG_REGISTERED) {
+			dev_put(ndev);
+			ret = -ENODEV;
+			goto out_release_sock;
+		}
+
 		can_ml = can_get_ml_priv(ndev);
 		if (!can_ml) {
 			dev_put(ndev);
diff --git a/net/can/j1939/transport.c b/net/can/j1939/transport.c
index fbf5c8001c9d..54bc472c7c75 100644
--- a/net/can/j1939/transport.c
+++ b/net/can/j1939/transport.c
@@ -246,6 +246,7 @@ static inline void j1939_session_list_unlock(struct j1939_priv *priv)
 void j1939_session_get(struct j1939_session *session)
 {
 	kref_get(&session->kref);
+	save_priv_trace_buffer(session->priv, 1);
 }
 
 /* session completion functions */
@@ -296,6 +297,7 @@ static void __j1939_session_release(struct kref *kref)
 
 void j1939_session_put(struct j1939_session *session)
 {
+	save_priv_trace_buffer(session->priv, -1);
 	kref_put(&session->kref, __j1939_session_release);
 }
 
@@ -1499,6 +1501,7 @@ static struct j1939_session *j1939_session_new(struct j1939_priv *priv,
 	INIT_LIST_HEAD(&session->active_session_list_entry);
 	INIT_LIST_HEAD(&session->sk_session_queue_entry);
 	kref_init(&session->kref);
+	save_priv_trace_buffer(priv, 1);
 
 	j1939_priv_get(priv);
 	session->priv = priv;
@@ -1567,6 +1570,8 @@ int j1939_session_activate(struct j1939_session *session)
 	if (active) {
 		j1939_session_put(active);
 		ret = -EAGAIN;
+	} else if (priv->ndev->reg_state != NETREG_REGISTERED) {
+		ret = -ENODEV;
 	} else {
 		WARN_ON_ONCE(session->state != J1939_SESSION_NEW);
 		list_add_tail(&session->active_session_list_entry,
diff --git a/net/ceph/messenger_v2.c b/net/ceph/messenger_v2.c
index 833e57849c1d..c9d50c0dcd33 100644
--- a/net/ceph/messenger_v2.c
+++ b/net/ceph/messenger_v2.c
@@ -2376,7 +2376,9 @@ static int process_auth_done(struct ceph_connection *con, void *p, void *end)
 
 	ceph_decode_64_safe(&p, end, global_id, bad);
 	ceph_decode_32_safe(&p, end, con->v2.con_mode, bad);
+
 	ceph_decode_32_safe(&p, end, payload_len, bad);
+	ceph_decode_need(&p, end, payload_len, bad);
 
 	dout("%s con %p global_id %llu con_mode %d payload_len %d\n",
 	     __func__, con, global_id, con->v2.con_mode, payload_len);
diff --git a/net/core/dev.c b/net/core/dev.c
index 9094c0fb8c68..df79d1d5e3d8 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -1855,6 +1855,7 @@ const char *netdev_cmd_to_name(enum netdev_cmd cmd)
 	N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE)
 	N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA)
 	N(XDP_FEAT_CHANGE)
+	N(DEBUG_UNREGISTER)
 	}
 #undef N
 	return "UNKNOWN_NETDEV_EVENT";
@@ -11526,6 +11527,14 @@ int netdev_refcnt_read(const struct net_device *dev)
 }
 EXPORT_SYMBOL(netdev_refcnt_read);
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+static void dump_netdev_trace_buffer(const struct net_device *dev);
+static void erase_netdev_trace_buffer(const struct net_device *dev);
+#else
+static inline void dump_netdev_trace_buffer(const struct net_device *dev) { }
+static inline void erase_netdev_trace_buffer(const struct net_device *dev) { }
+#endif
+
 int netdev_unregister_timeout_secs __read_mostly = 10;
 
 #define WAIT_REFS_MIN_MSECS 1
@@ -11599,11 +11608,16 @@ static struct net_device *netdev_wait_allrefs_any(struct list_head *list)
 
 		if (time_after(jiffies, warning_time +
 			       READ_ONCE(netdev_unregister_timeout_secs) * HZ)) {
+			rtnl_lock();
 			list_for_each_entry(dev, list, todo_list) {
 				pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
 					 dev->name, netdev_refcnt_read(dev));
 				ref_tracker_dir_print(&dev->refcnt_tracker, 10);
+				call_netdevice_notifiers(NETDEV_DEBUG_UNREGISTER, dev);
+				dump_netdev_trace_buffer(dev);
 			}
+			__rtnl_unlock();
+			rcu_barrier();
 
 			warning_time = jiffies;
 		}
@@ -12001,6 +12015,9 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
 
 	dev->priv_len = sizeof_priv;
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+	INIT_LIST_HEAD(&dev->netdev_trace_buffer_list);
+#endif
 	ref_tracker_dir_init(&dev->refcnt_tracker, 128, "netdev");
 #ifdef CONFIG_PCPU_DEV_REFCNT
 	dev->pcpu_refcnt = alloc_percpu(int);
@@ -12173,6 +12190,8 @@ void free_netdev(struct net_device *dev)
 
 	mutex_destroy(&dev->lock);
 
+	erase_netdev_trace_buffer(dev);
+
 	/*  Compatibility with error handling in drivers */
 	if (dev->reg_state == NETREG_UNINITIALIZED ||
 	    dev->reg_state == NETREG_DUMMY) {
@@ -13275,3 +13294,168 @@ static int __init net_dev_init(void)
 }
 
 subsys_initcall(net_dev_init);
+
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+
+#define NETDEV_TRACE_BUFFER_SIZE 32768
+static struct netdev_trace_buffer {
+	struct list_head list;
+	int prev_count;
+	atomic_t count;
+	int nr_entries;
+	unsigned long entries[20];
+} netdev_trace_buffer[NETDEV_TRACE_BUFFER_SIZE];
+static LIST_HEAD(netdev_trace_buffer_list);
+static DEFINE_SPINLOCK(netdev_trace_buffer_lock);
+static bool netdev_trace_buffer_exhausted;
+
+static int netdev_trace_buffer_init(void)
+{
+	int i;
+
+	for (i = 0; i < NETDEV_TRACE_BUFFER_SIZE; i++)
+		list_add_tail(&netdev_trace_buffer[i].list, &netdev_trace_buffer_list);
+	return 0;
+}
+pure_initcall(netdev_trace_buffer_init);
+
+static void dump_netdev_trace_buffer(const struct net_device *dev)
+{
+	struct netdev_trace_buffer *ptr;
+	int count, balance = 0, pos = 0;
+
+	list_for_each_entry_rcu(ptr, &dev->netdev_trace_buffer_list, list,
+				/* list elements can't go away. */ 1) {
+		pos++;
+		count = atomic_read(&ptr->count);
+		balance += count;
+		if (ptr->prev_count == count)
+			continue;
+		ptr->prev_count = count;
+		pr_info("Call trace for %s[%d] %+d at\n", dev->name, pos, count);
+		stack_trace_print(ptr->entries, ptr->nr_entries, 4);
+		cond_resched();
+	}
+	if (!netdev_trace_buffer_exhausted)
+		pr_info("balance as of %s[%d] is %d\n", dev->name, pos, balance);
+}
+
+static void erase_netdev_trace_buffer(const struct net_device *dev)
+{
+	struct netdev_trace_buffer *ptr;
+	unsigned long flags;
+
+	spin_lock_irqsave(&netdev_trace_buffer_lock, flags);
+	while (!list_empty(&dev->netdev_trace_buffer_list)) {
+		ptr = list_first_entry(&dev->netdev_trace_buffer_list, typeof(*ptr), list);
+		list_del(&ptr->list);
+		list_add_tail(&ptr->list, &netdev_trace_buffer_list);
+	}
+	spin_unlock_irqrestore(&netdev_trace_buffer_lock, flags);
+}
+
+#ifdef CONFIG_KALLSYMS
+static int __find_trim(unsigned long *caller, const unsigned long *entries, const int nr_entries,
+		       const char *name)
+{
+	int i;
+
+	if (!*caller) {
+		char buffer[KSYM_SYMBOL_LEN];
+		const int len = strlen(name);
+
+		for (i = 0; i < nr_entries; i++) {
+			snprintf(buffer, sizeof(buffer), "%pS", (void *)entries[i]);
+			if (!strncmp(buffer, name, len) && buffer[len] == '+') {
+				*caller = entries[i];
+				return i + 1;
+			}
+		}
+		if (!kallsyms_lookup_name(name))
+			*caller = -1;
+	}
+	for (i = 0; i < nr_entries; i++)
+		if (entries[i] == *caller)
+			return i + 1;
+	return nr_entries;
+}
+#endif
+
+int trim_netdev_trace(unsigned long *entries, int nr_entries)
+{
+#ifdef CONFIG_KALLSYMS
+	static unsigned long caller[9];
+	int i;
+
+	if (in_softirq())
+		return __find_trim(&caller[0], entries, nr_entries, "handle_softirqs");
+	if (current->flags & PF_WQ_WORKER)
+		return __find_trim(&caller[1], entries, nr_entries, "process_one_work");
+	for (i = 0; i < nr_entries; i++)
+		if (entries[i] == caller[2] || entries[i] == caller[3] ||
+		    entries[i] == caller[4] || entries[i] == caller[5] ||
+		    entries[i] == caller[6] || entries[i] == caller[7] ||
+		    entries[i] == caller[8])
+			return i + 1;
+	i = __find_trim(&caller[2], entries, nr_entries, "ksys_unshare");
+	if (i < nr_entries)
+		return i;
+	i = __find_trim(&caller[3], entries, nr_entries, "__sys_bind");
+	if (i < nr_entries)
+		return i;
+	i = __find_trim(&caller[4], entries, nr_entries, "sock_sendmsg_nosec");
+	if (i < nr_entries)
+		return i;
+	i = __find_trim(&caller[5], entries, nr_entries, "sock_recvmsg_nosec");
+	if (i < nr_entries)
+		return i;
+	i = __find_trim(&caller[6], entries, nr_entries, "__sock_release");
+	if (i < nr_entries)
+		return i;
+	i = __find_trim(&caller[7], entries, nr_entries, "do_sock_setsockopt");
+	if (i < nr_entries)
+		return i;
+	i = __find_trim(&caller[8], entries, nr_entries, "__sys_bpf");
+	if (i < nr_entries)
+		return i;
+#endif
+	return nr_entries;
+}
+EXPORT_SYMBOL(trim_netdev_trace);
+
+void save_netdev_trace_buffer(struct net_device *dev, int delta)
+{
+	struct netdev_trace_buffer *ptr;
+	unsigned long entries[ARRAY_SIZE(ptr->entries)];
+	unsigned long nr_entries;
+	unsigned long flags;
+
+	if (in_nmi())
+		return;
+	nr_entries = stack_trace_save(entries, ARRAY_SIZE(ptr->entries), 1);
+	nr_entries = trim_netdev_trace(entries, nr_entries);
+	list_for_each_entry_rcu(ptr, &dev->netdev_trace_buffer_list, list,
+				/* list elements can't go away. */ 1) {
+		if (ptr->nr_entries == nr_entries &&
+		    !memcmp(ptr->entries, entries, nr_entries * sizeof(unsigned long))) {
+			atomic_add(delta, &ptr->count);
+			return;
+		}
+	}
+	spin_lock_irqsave(&netdev_trace_buffer_lock, flags);
+	if (!list_empty(&netdev_trace_buffer_list)) {
+		ptr = list_first_entry(&netdev_trace_buffer_list, typeof(*ptr), list);
+		list_del(&ptr->list);
+		ptr->prev_count = 0;
+		atomic_set(&ptr->count, delta);
+		ptr->nr_entries = nr_entries;
+		memmove(ptr->entries, entries, nr_entries * sizeof(unsigned long));
+		list_add_tail_rcu(&ptr->list, &dev->netdev_trace_buffer_list);
+	} else {
+		netdev_trace_buffer_exhausted = true;
+	}
+	spin_unlock_irqrestore(&netdev_trace_buffer_lock, flags);
+}
+EXPORT_SYMBOL(save_netdev_trace_buffer);
+
+#endif
diff --git a/net/core/lock_debug.c b/net/core/lock_debug.c
index 9e9fb25314b9..78d611bb6d1c 100644
--- a/net/core/lock_debug.c
+++ b/net/core/lock_debug.c
@@ -29,6 +29,7 @@ int netdev_debug_event(struct notifier_block *nb, unsigned long event,
 	case NETDEV_DOWN:
 	case NETDEV_REBOOT:
 	case NETDEV_UNREGISTER:
+	case NETDEV_DEBUG_UNREGISTER:
 	case NETDEV_CHANGEMTU:
 	case NETDEV_CHANGEADDR:
 	case NETDEV_PRE_CHANGEADDR:
diff --git a/net/core/pktgen.c b/net/core/pktgen.c
index d41b03fd1f63..8e185b318288 100644
--- a/net/core/pktgen.c
+++ b/net/core/pktgen.c
@@ -126,6 +126,7 @@
 #include <linux/string.h>
 #include <linux/ptrace.h>
 #include <linux/errno.h>
+#include <linux/hex.h>
 #include <linux/ioport.h>
 #include <linux/interrupt.h>
 #include <linux/capability.h>
diff --git a/net/core/utils.c b/net/core/utils.c
index 5e63b0ea21f3..dd86913988f4 100644
--- a/net/core/utils.c
+++ b/net/core/utils.c
@@ -11,6 +11,7 @@
  */
 
 #include <linux/module.h>
+#include <linux/hex.h>
 #include <linux/jiffies.h>
 #include <linux/kernel.h>
 #include <linux/ctype.h>
diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c
index fa83ddade4f8..9431e305b233 100644
--- a/net/ethtool/ioctl.c
+++ b/net/ethtool/ioctl.c
@@ -2383,7 +2383,10 @@ static int ethtool_get_strings(struct net_device *dev, void __user *useraddr)
 		return -ENOMEM;
 	WARN_ON_ONCE(!ret);
 
-	gstrings.len = ret;
+	if (gstrings.len && gstrings.len != ret)
+		gstrings.len = 0;
+	else
+		gstrings.len = ret;
 
 	if (gstrings.len) {
 		data = vzalloc(array_size(gstrings.len, ETH_GSTRING_LEN));
@@ -2509,10 +2512,13 @@ static int ethtool_get_stats(struct net_device *dev, void __user *useraddr)
 	if (copy_from_user(&stats, useraddr, sizeof(stats)))
 		return -EFAULT;
 
-	stats.n_stats = n_stats;
+	if (stats.n_stats && stats.n_stats != n_stats)
+		stats.n_stats = 0;
+	else
+		stats.n_stats = n_stats;
 
-	if (n_stats) {
-		data = vzalloc(array_size(n_stats, sizeof(u64)));
+	if (stats.n_stats) {
+		data = vzalloc(array_size(stats.n_stats, sizeof(u64)));
 		if (!data)
 			return -ENOMEM;
 		ops->get_ethtool_stats(dev, &stats, data);
@@ -2524,7 +2530,9 @@ static int ethtool_get_stats(struct net_device *dev, void __user *useraddr)
 	if (copy_to_user(useraddr, &stats, sizeof(stats)))
 		goto out;
 	useraddr += sizeof(stats);
-	if (n_stats && copy_to_user(useraddr, data, array_size(n_stats, sizeof(u64))))
+	if (stats.n_stats &&
+	    copy_to_user(useraddr, data,
+			 array_size(stats.n_stats, sizeof(u64))))
 		goto out;
 	ret = 0;
 
@@ -2560,6 +2568,10 @@ static int ethtool_get_phy_stats_phydev(struct phy_device *phydev,
 		return -EOPNOTSUPP;
 
 	n_stats = phy_ops->get_sset_count(phydev);
+	if (stats->n_stats && stats->n_stats != n_stats) {
+		stats->n_stats = 0;
+		return 0;
+	}
 
 	ret = ethtool_vzalloc_stats_array(n_stats, data);
 	if (ret)
@@ -2580,6 +2592,10 @@ static int ethtool_get_phy_stats_ethtool(struct net_device *dev,
 		return -EOPNOTSUPP;
 
 	n_stats = ops->get_sset_count(dev, ETH_SS_PHY_STATS);
+	if (stats->n_stats && stats->n_stats != n_stats) {
+		stats->n_stats = 0;
+		return 0;
+	}
 
 	ret = ethtool_vzalloc_stats_array(n_stats, data);
 	if (ret)
@@ -2616,7 +2632,9 @@ static int ethtool_get_phy_stats(struct net_device *dev, void __user *useraddr)
 	}
 
 	useraddr += sizeof(stats);
-	if (copy_to_user(useraddr, data, array_size(stats.n_stats, sizeof(u64))))
+	if (stats.n_stats &&
+	    copy_to_user(useraddr, data,
+			 array_size(stats.n_stats, sizeof(u64))))
 		ret = -EFAULT;
 
  out:
diff --git a/net/handshake/request.c b/net/handshake/request.c
index 274d2c89b6b2..6b7e3e0bf399 100644
--- a/net/handshake/request.c
+++ b/net/handshake/request.c
@@ -276,6 +276,8 @@ int handshake_req_submit(struct socket *sock, struct handshake_req *req,
 out_unlock:
 	spin_unlock(&hn->hn_lock);
 out_err:
+	/* Restore original destructor so socket teardown still runs on failure */
+	req->hr_sk->sk_destruct = req->hr_odestruct;
 	trace_handshake_submit_err(net, req, req->hr_sk, ret);
 	handshake_req_destroy(req);
 	return ret;
@@ -324,7 +326,11 @@ bool handshake_req_cancel(struct sock *sk)
 
 	hn = handshake_pernet(net);
 	if (hn && remove_pending(hn, req)) {
-		/* Request hadn't been accepted */
+		/* Request hadn't been accepted - mark cancelled */
+		if (test_and_set_bit(HANDSHAKE_F_REQ_COMPLETED, &req->hr_flags)) {
+			trace_handshake_cancel_busy(net, req, sk);
+			return false;
+		}
 		goto out_true;
 	}
 	if (test_and_set_bit(HANDSHAKE_F_REQ_COMPLETED, &req->hr_flags)) {
diff --git a/net/hsr/hsr_forward.c b/net/hsr/hsr_forward.c
index 339f0d220212..aefc9b6936ba 100644
--- a/net/hsr/hsr_forward.c
+++ b/net/hsr/hsr_forward.c
@@ -205,6 +205,8 @@ struct sk_buff *prp_get_untagged_frame(struct hsr_frame_info *frame,
 				__pskb_copy(frame->skb_prp,
 					    skb_headroom(frame->skb_prp),
 					    GFP_ATOMIC);
+			if (!frame->skb_std)
+				return NULL;
 		} else {
 			/* Unexpected */
 			WARN_ONCE(1, "%s:%d: Unexpected frame received (port_src %s)\n",
diff --git a/net/ieee802154/core.c b/net/ieee802154/core.c
index 89b671b12600..c0b8712018a1 100644
--- a/net/ieee802154/core.c
+++ b/net/ieee802154/core.c
@@ -228,36 +228,43 @@ int cfg802154_switch_netns(struct cfg802154_registered_device *rdev,
 			continue;
 		wpan_dev->netdev->netns_immutable = false;
 		err = dev_change_net_namespace(wpan_dev->netdev, net, "wpan%d");
-		if (err)
+		if (err) {
+			WARN_ON(err && err != -ENOMEM);
 			break;
+		}
 		wpan_dev->netdev->netns_immutable = true;
 	}
 
-	if (err) {
-		/* failed -- clean up to old netns */
-		net = wpan_phy_net(&rdev->wpan_phy);
-
-		list_for_each_entry_continue_reverse(wpan_dev,
-						     &rdev->wpan_dev_list,
-						     list) {
-			if (!wpan_dev->netdev)
-				continue;
-			wpan_dev->netdev->netns_immutable = false;
-			err = dev_change_net_namespace(wpan_dev->netdev, net,
-						       "wpan%d");
-			WARN_ON(err);
-			wpan_dev->netdev->netns_immutable = true;
-		}
+	if (err)
+		goto errout;
 
-		return err;
-	}
+	err = device_rename(&rdev->wpan_phy.dev, dev_name(&rdev->wpan_phy.dev));
+	WARN_ON(err && err != -ENOMEM);
 
-	wpan_phy_net_set(&rdev->wpan_phy, net);
+	if (err)
+		goto errout;
 
-	err = device_rename(&rdev->wpan_phy.dev, dev_name(&rdev->wpan_phy.dev));
-	WARN_ON(err);
+	wpan_phy_net_set(&rdev->wpan_phy, net);
 
 	return 0;
+
+errout:
+	/* failed -- clean up to old netns */
+	net = wpan_phy_net(&rdev->wpan_phy);
+
+	list_for_each_entry_continue_reverse(wpan_dev,
+					     &rdev->wpan_dev_list,
+					     list) {
+		if (!wpan_dev->netdev)
+			continue;
+		wpan_dev->netdev->netns_immutable = false;
+		err = dev_change_net_namespace(wpan_dev->netdev, net,
+					       "wpan%d");
+		WARN_ON(err && err != -ENOMEM);
+		wpan_dev->netdev->netns_immutable = true;
+	}
+
+	return err;
 }
 
 void cfg802154_dev_free(struct cfg802154_registered_device *rdev)
@@ -351,7 +358,7 @@ static void __net_exit cfg802154_pernet_exit(struct net *net)
 	rtnl_lock();
 	list_for_each_entry(rdev, &cfg802154_rdev_list, list) {
 		if (net_eq(wpan_phy_net(&rdev->wpan_phy), net))
-			WARN_ON(cfg802154_switch_netns(rdev, &init_net));
+			cfg802154_switch_netns(rdev, &init_net);
 	}
 	rtnl_unlock();
 }
diff --git a/net/ipv4/arp.c b/net/ipv4/arp.c
index 7f3863daaa40..c7315debf2da 100644
--- a/net/ipv4/arp.c
+++ b/net/ipv4/arp.c
@@ -79,6 +79,7 @@
 #include <linux/socket.h>
 #include <linux/sockios.h>
 #include <linux/errno.h>
+#include <linux/hex.h>
 #include <linux/in.h>
 #include <linux/mm.h>
 #include <linux/inet.h>
diff --git a/net/ipv4/esp4_offload.c b/net/ipv4/esp4_offload.c
index 05828d4cb6cd..abd77162f5e7 100644
--- a/net/ipv4/esp4_offload.c
+++ b/net/ipv4/esp4_offload.c
@@ -122,8 +122,8 @@ static struct sk_buff *xfrm4_tunnel_gso_segment(struct xfrm_state *x,
 						struct sk_buff *skb,
 						netdev_features_t features)
 {
-	const struct xfrm_mode *inner_mode = xfrm_ip2inner_mode(x,
-					XFRM_MODE_SKB_CB(skb)->protocol);
+	struct xfrm_offload *xo = xfrm_offload(skb);
+	const struct xfrm_mode *inner_mode = xfrm_ip2inner_mode(x, xo->proto);
 	__be16 type = inner_mode->family == AF_INET6 ? htons(ETH_P_IPV6)
 						     : htons(ETH_P_IP);
 
diff --git a/net/ipv4/inet_fragment.c b/net/ipv4/inet_fragment.c
index 025895eb6ec5..001ee5c4d962 100644
--- a/net/ipv4/inet_fragment.c
+++ b/net/ipv4/inet_fragment.c
@@ -218,6 +218,41 @@ static int __init inet_frag_wq_init(void)
 
 pure_initcall(inet_frag_wq_init);
 
+void fqdir_pre_exit(struct fqdir *fqdir)
+{
+	struct inet_frag_queue *fq;
+	struct rhashtable_iter hti;
+
+	/* Prevent creation of new frags.
+	 * Pairs with READ_ONCE() in inet_frag_find().
+	 */
+	WRITE_ONCE(fqdir->high_thresh, 0);
+
+	/* Pairs with READ_ONCE() in inet_frag_kill(), ip_expire()
+	 * and ip6frag_expire_frag_queue().
+	 */
+	WRITE_ONCE(fqdir->dead, true);
+
+	rhashtable_walk_enter(&fqdir->rhashtable, &hti);
+	rhashtable_walk_start(&hti);
+
+	while ((fq = rhashtable_walk_next(&hti))) {
+		if (IS_ERR(fq)) {
+			if (PTR_ERR(fq) != -EAGAIN)
+				break;
+			continue;
+		}
+		spin_lock_bh(&fq->lock);
+		if (!(fq->flags & INET_FRAG_COMPLETE))
+			inet_frag_queue_flush(fq, 0);
+		spin_unlock_bh(&fq->lock);
+	}
+
+	rhashtable_walk_stop(&hti);
+	rhashtable_walk_exit(&hti);
+}
+EXPORT_SYMBOL(fqdir_pre_exit);
+
 void fqdir_exit(struct fqdir *fqdir)
 {
 	INIT_WORK(&fqdir->destroy_work, fqdir_work_fn);
@@ -263,8 +298,8 @@ static void inet_frag_destroy_rcu(struct rcu_head *head)
 	kmem_cache_free(f->frags_cachep, q);
 }
 
-unsigned int inet_frag_rbtree_purge(struct rb_root *root,
-				    enum skb_drop_reason reason)
+static unsigned int
+inet_frag_rbtree_purge(struct rb_root *root, enum skb_drop_reason reason)
 {
 	struct rb_node *p = rb_first(root);
 	unsigned int sum = 0;
@@ -284,7 +319,17 @@ unsigned int inet_frag_rbtree_purge(struct rb_root *root,
 	}
 	return sum;
 }
-EXPORT_SYMBOL(inet_frag_rbtree_purge);
+
+void inet_frag_queue_flush(struct inet_frag_queue *q,
+			   enum skb_drop_reason reason)
+{
+	unsigned int sum;
+
+	reason = reason ?: SKB_DROP_REASON_FRAG_REASM_TIMEOUT;
+	sum = inet_frag_rbtree_purge(&q->rb_fragments, reason);
+	sub_frag_mem_limit(q->fqdir, sum);
+}
+EXPORT_SYMBOL(inet_frag_queue_flush);
 
 void inet_frag_destroy(struct inet_frag_queue *q)
 {
@@ -327,7 +372,9 @@ static struct inet_frag_queue *inet_frag_alloc(struct fqdir *fqdir,
 
 	timer_setup(&q->timer, f->frag_expire, 0);
 	spin_lock_init(&q->lock);
-	/* One reference for the timer, one for the hash table. */
+	/* One reference for the timer, one for the hash table.
+	 * We never take any extra references, only decrement this field.
+	 */
 	refcount_set(&q->refcnt, 2);
 
 	return q;
diff --git a/net/ipv4/ip_fragment.c b/net/ipv4/ip_fragment.c
index f7012479713b..56b0f738d2f2 100644
--- a/net/ipv4/ip_fragment.c
+++ b/net/ipv4/ip_fragment.c
@@ -134,11 +134,6 @@ static void ip_expire(struct timer_list *t)
 	net = qp->q.fqdir->net;
 
 	rcu_read_lock();
-
-	/* Paired with WRITE_ONCE() in fqdir_pre_exit(). */
-	if (READ_ONCE(qp->q.fqdir->dead))
-		goto out_rcu_unlock;
-
 	spin_lock(&qp->q.lock);
 
 	if (qp->q.flags & INET_FRAG_COMPLETE)
@@ -146,6 +141,13 @@ static void ip_expire(struct timer_list *t)
 
 	qp->q.flags |= INET_FRAG_DROP;
 	inet_frag_kill(&qp->q, &refs);
+
+	/* Paired with WRITE_ONCE() in fqdir_pre_exit(). */
+	if (READ_ONCE(qp->q.fqdir->dead)) {
+		inet_frag_queue_flush(&qp->q, 0);
+		goto out;
+	}
+
 	__IP_INC_STATS(net, IPSTATS_MIB_REASMFAILS);
 	__IP_INC_STATS(net, IPSTATS_MIB_REASMTIMEOUT);
 
@@ -240,16 +242,10 @@ static int ip_frag_too_far(struct ipq *qp)
 
 static int ip_frag_reinit(struct ipq *qp)
 {
-	unsigned int sum_truesize = 0;
-
-	if (!mod_timer(&qp->q.timer, jiffies + qp->q.fqdir->timeout)) {
-		refcount_inc(&qp->q.refcnt);
+	if (!mod_timer_pending(&qp->q.timer, jiffies + qp->q.fqdir->timeout))
 		return -ETIMEDOUT;
-	}
 
-	sum_truesize = inet_frag_rbtree_purge(&qp->q.rb_fragments,
-					      SKB_DROP_REASON_FRAG_TOO_FAR);
-	sub_frag_mem_limit(qp->q.fqdir, sum_truesize);
+	inet_frag_queue_flush(&qp->q, SKB_DROP_REASON_FRAG_TOO_FAR);
 
 	qp->q.flags = 0;
 	qp->q.len = 0;
diff --git a/net/ipv6/esp6_offload.c b/net/ipv6/esp6_offload.c
index 22410243ebe8..22895521a57d 100644
--- a/net/ipv6/esp6_offload.c
+++ b/net/ipv6/esp6_offload.c
@@ -158,8 +158,8 @@ static struct sk_buff *xfrm6_tunnel_gso_segment(struct xfrm_state *x,
 						struct sk_buff *skb,
 						netdev_features_t features)
 {
-	const struct xfrm_mode *inner_mode = xfrm_ip2inner_mode(x,
-					XFRM_MODE_SKB_CB(skb)->protocol);
+	struct xfrm_offload *xo = xfrm_offload(skb);
+	const struct xfrm_mode *inner_mode = xfrm_ip2inner_mode(x, xo->proto);
 	__be16 type = inner_mode->family == AF_INET ? htons(ETH_P_IP)
 						    : htons(ETH_P_IPV6);
 
diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c
index b51c2c8584ae..c81091a5cc3a 100644
--- a/net/mac80211/cfg.c
+++ b/net/mac80211/cfg.c
@@ -1345,7 +1345,6 @@ ieee80211_assign_beacon(struct ieee80211_sub_if_data *sdata,
 
 	size = sizeof(*new) + new_head_len + new_tail_len;
 
-	/* new or old multiple BSSID elements? */
 	if (params->mbssid_ies) {
 		mbssid = params->mbssid_ies;
 		size += struct_size(new->mbssid_ies, elem, mbssid->cnt);
@@ -1355,15 +1354,6 @@ ieee80211_assign_beacon(struct ieee80211_sub_if_data *sdata,
 		}
 		size += ieee80211_get_mbssid_beacon_len(mbssid, rnr,
 							mbssid->cnt);
-	} else if (old && old->mbssid_ies) {
-		mbssid = old->mbssid_ies;
-		size += struct_size(new->mbssid_ies, elem, mbssid->cnt);
-		if (old && old->rnr_ies) {
-			rnr = old->rnr_ies;
-			size += struct_size(new->rnr_ies, elem, rnr->cnt);
-		}
-		size += ieee80211_get_mbssid_beacon_len(mbssid, rnr,
-							mbssid->cnt);
 	}
 
 	new = kzalloc(size, GFP_KERNEL);
diff --git a/net/mac80211/debugfs_netdev.c b/net/mac80211/debugfs_netdev.c
index 30a5a978a678..f3c6a41e4911 100644
--- a/net/mac80211/debugfs_netdev.c
+++ b/net/mac80211/debugfs_netdev.c
@@ -7,6 +7,7 @@
 
 #include <linux/kernel.h>
 #include <linux/device.h>
+#include <linux/hex.h>
 #include <linux/if.h>
 #include <linux/if_ether.h>
 #include <linux/interrupt.h>
diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c
index 4f04d95c19d4..7b0aa24c1f97 100644
--- a/net/mac80211/iface.c
+++ b/net/mac80211/iface.c
@@ -1251,7 +1251,7 @@ int ieee80211_add_virtual_monitor(struct ieee80211_local *local,
 	if (!creator_sdata) {
 		struct ieee80211_sub_if_data *other;
 
-		list_for_each_entry(other, &local->mon_list, list) {
+		list_for_each_entry_rcu(other, &local->mon_list, u.mntr.list) {
 			if (!other->vif.bss_conf.mu_mimo_owner)
 				continue;
 
diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c
index e56ad4b9330f..ad53dedd929c 100644
--- a/net/mac80211/mlme.c
+++ b/net/mac80211/mlme.c
@@ -1126,7 +1126,10 @@ ieee80211_determine_chan_mode(struct ieee80211_sub_if_data *sdata,
 
 	while (!ieee80211_chandef_usable(sdata, &chanreq->oper,
 					 IEEE80211_CHAN_DISABLED)) {
-		if (WARN_ON(chanreq->oper.width == NL80211_CHAN_WIDTH_20_NOHT)) {
+		if (chanreq->oper.width == NL80211_CHAN_WIDTH_20_NOHT) {
+			link_id_info(sdata, link_id,
+				     "unusable channel (%d MHz) for connection\n",
+				     chanreq->oper.chan->center_freq);
 			ret = -EINVAL;
 			goto free;
 		}
diff --git a/net/mac80211/ocb.c b/net/mac80211/ocb.c
index a5d4358f122a..ebb4f4d88c23 100644
--- a/net/mac80211/ocb.c
+++ b/net/mac80211/ocb.c
@@ -47,6 +47,9 @@ void ieee80211_ocb_rx_no_sta(struct ieee80211_sub_if_data *sdata,
 	struct sta_info *sta;
 	int band;
 
+	if (!ifocb->joined)
+		return;
+
 	/* XXX: Consider removing the least recently used entry and
 	 *      allow new one to be added.
 	 */
diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c
index 6a1899512d07..e0ccd9749853 100644
--- a/net/mac80211/rx.c
+++ b/net/mac80211/rx.c
@@ -3511,6 +3511,11 @@ ieee80211_rx_h_mgmt_check(struct ieee80211_rx_data *rx)
 	    rx->skb->len < IEEE80211_MIN_ACTION_SIZE)
 		return RX_DROP_U_RUNT_ACTION;
 
+	/* Drop non-broadcast Beacon frames */
+	if (ieee80211_is_beacon(mgmt->frame_control) &&
+	    !is_broadcast_ether_addr(mgmt->da))
+		return RX_DROP;
+
 	if (rx->sdata->vif.type == NL80211_IFTYPE_AP &&
 	    ieee80211_is_beacon(mgmt->frame_control) &&
 	    !(rx->flags & IEEE80211_RX_BEACON_REPORTED)) {
diff --git a/net/mptcp/Kconfig b/net/mptcp/Kconfig
index 20328920f6ed..be71fc9b4638 100644
--- a/net/mptcp/Kconfig
+++ b/net/mptcp/Kconfig
@@ -4,7 +4,7 @@ config MPTCP
 	depends on INET
 	select SKB_EXTENSIONS
 	select CRYPTO_LIB_SHA256
-	select CRYPTO
+	select CRYPTO_LIB_UTILS
 	help
 	  Multipath TCP (MPTCP) connections send and receive data over multiple
 	  subflows in order to utilize multiple network paths. Each subflow
diff --git a/net/mptcp/pm_netlink.c b/net/mptcp/pm_netlink.c
index d5b383870f79..7aa42de9c47b 100644
--- a/net/mptcp/pm_netlink.c
+++ b/net/mptcp/pm_netlink.c
@@ -119,7 +119,8 @@ int mptcp_pm_parse_entry(struct nlattr *attr, struct genl_info *info,
 	}
 
 	if (tb[MPTCP_PM_ADDR_ATTR_FLAGS])
-		entry->flags = nla_get_u32(tb[MPTCP_PM_ADDR_ATTR_FLAGS]);
+		entry->flags = nla_get_u32(tb[MPTCP_PM_ADDR_ATTR_FLAGS]) &
+			       MPTCP_PM_ADDR_FLAGS_MASK;
 
 	if (tb[MPTCP_PM_ADDR_ATTR_PORT])
 		entry->addr.port = htons(nla_get_u16(tb[MPTCP_PM_ADDR_ATTR_PORT]));
diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c
index e212c1374bd0..9b1fafd87cb9 100644
--- a/net/mptcp/protocol.c
+++ b/net/mptcp/protocol.c
@@ -1623,7 +1623,7 @@ void __mptcp_push_pending(struct sock *sk, unsigned int flags)
 	struct mptcp_sendmsg_info info = {
 				.flags = flags,
 	};
-	bool do_check_data_fin = false;
+	bool copied = false;
 	int push_count = 1;
 
 	while (mptcp_send_head(sk) && (push_count > 0)) {
@@ -1665,7 +1665,7 @@ void __mptcp_push_pending(struct sock *sk, unsigned int flags)
 						push_count--;
 					continue;
 				}
-				do_check_data_fin = true;
+				copied = true;
 			}
 		}
 	}
@@ -1674,11 +1674,14 @@ void __mptcp_push_pending(struct sock *sk, unsigned int flags)
 	if (ssk)
 		mptcp_push_release(ssk, &info);
 
-	/* ensure the rtx timer is running */
-	if (!mptcp_rtx_timer_pending(sk))
-		mptcp_reset_rtx_timer(sk);
-	if (do_check_data_fin)
+	/* Avoid scheduling the rtx timer if no data has been pushed; the timer
+	 * will be updated on positive acks by __mptcp_cleanup_una().
+	 */
+	if (copied) {
+		if (!mptcp_rtx_timer_pending(sk))
+			mptcp_reset_rtx_timer(sk);
 		mptcp_check_send_data_fin(sk);
+	}
 }
 
 static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk, bool first)
@@ -2766,10 +2769,13 @@ static void __mptcp_retrans(struct sock *sk)
 
 			/*
 			 * make the whole retrans decision, xmit, disallow
-			 * fallback atomic
+			 * fallback atomic, note that we can't retrans even
+			 * when an infinite fallback is in progress, i.e. new
+			 * subflows are disallowed.
 			 */
 			spin_lock_bh(&msk->fallback_lock);
-			if (__mptcp_check_fallback(msk)) {
+			if (__mptcp_check_fallback(msk) ||
+			    !msk->allow_subflows) {
 				spin_unlock_bh(&msk->fallback_lock);
 				release_sock(ssk);
 				goto clear_scheduled;
diff --git a/net/netfilter/ipvs/ip_vs_xmit.c b/net/netfilter/ipvs/ip_vs_xmit.c
index 3162ce3c2640..64c697212578 100644
--- a/net/netfilter/ipvs/ip_vs_xmit.c
+++ b/net/netfilter/ipvs/ip_vs_xmit.c
@@ -408,6 +408,9 @@ __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb,
 	return -1;
 
 err_unreach:
+	if (!skb->dev)
+		skb->dev = skb_dst(skb)->dev;
+
 	dst_link_failure(skb);
 	return -1;
 }
diff --git a/net/netfilter/nf_conncount.c b/net/netfilter/nf_conncount.c
index f1be4dd5cf85..3654f1e8976c 100644
--- a/net/netfilter/nf_conncount.c
+++ b/net/netfilter/nf_conncount.c
@@ -172,14 +172,14 @@ static int __nf_conncount_add(struct net *net,
 	struct nf_conn *found_ct;
 	unsigned int collect = 0;
 	bool refcounted = false;
+	int err = 0;
 
 	if (!get_ct_or_tuple_from_skb(net, skb, l3num, &ct, &tuple, &zone, &refcounted))
 		return -ENOENT;
 
 	if (ct && nf_ct_is_confirmed(ct)) {
-		if (refcounted)
-			nf_ct_put(ct);
-		return -EEXIST;
+		err = -EEXIST;
+		goto out_put;
 	}
 
 	if ((u32)jiffies == list->last_gc)
@@ -231,12 +231,16 @@ static int __nf_conncount_add(struct net *net,
 	}
 
 add_new_node:
-	if (WARN_ON_ONCE(list->count > INT_MAX))
-		return -EOVERFLOW;
+	if (WARN_ON_ONCE(list->count > INT_MAX)) {
+		err = -EOVERFLOW;
+		goto out_put;
+	}
 
 	conn = kmem_cache_alloc(conncount_conn_cachep, GFP_ATOMIC);
-	if (conn == NULL)
-		return -ENOMEM;
+	if (conn == NULL) {
+		err = -ENOMEM;
+		goto out_put;
+	}
 
 	conn->tuple = tuple;
 	conn->zone = *zone;
@@ -249,7 +253,7 @@ static int __nf_conncount_add(struct net *net,
 out_put:
 	if (refcounted)
 		nf_ct_put(ct);
-	return 0;
+	return err;
 }
 
 int nf_conncount_add_skb(struct net *net,
@@ -456,11 +460,10 @@ insert_tree(struct net *net,
 
 		rb_link_node_rcu(&rbconn->node, parent, rbnode);
 		rb_insert_color(&rbconn->node, root);
-
-		if (refcounted)
-			nf_ct_put(ct);
 	}
 out_unlock:
+	if (refcounted)
+		nf_ct_put(ct);
 	spin_unlock_bh(&nf_conncount_locks[hash]);
 	return count;
 }
diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c
index 0b95f226f211..d1f8eb725d42 100644
--- a/net/netfilter/nf_conntrack_core.c
+++ b/net/netfilter/nf_conntrack_core.c
@@ -2487,6 +2487,7 @@ void nf_conntrack_cleanup_net(struct net *net)
 void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list)
 {
 	struct nf_ct_iter_data iter_data = {};
+	unsigned long start = jiffies;
 	struct net *net;
 	int busy;
 
@@ -2507,6 +2508,8 @@ void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list)
 			busy = 1;
 	}
 	if (busy) {
+		DEBUG_NET_WARN_ONCE(time_after(jiffies, start + 60 * HZ),
+				    "conntrack cleanup blocked for 60s");
 		schedule();
 		goto i_see_dead_people;
 	}
diff --git a/net/netfilter/nf_flow_table_path.c b/net/netfilter/nf_flow_table_path.c
index f0984cf69a09..eb24fe2715dc 100644
--- a/net/netfilter/nf_flow_table_path.c
+++ b/net/netfilter/nf_flow_table_path.c
@@ -250,6 +250,9 @@ static void nft_dev_forward_path(const struct nft_pktinfo *pkt,
 	if (nft_dev_fill_forward_path(route, dst, ct, dir, ha, &stack) >= 0)
 		nft_dev_path_info(&stack, &info, ha, &ft->data);
 
+	if (info.outdev)
+		route->tuple[dir].out.ifindex = info.outdev->ifindex;
+
 	if (!info.indev || !nft_flowtable_find_dev(info.indev, ft))
 		return;
 
@@ -269,7 +272,6 @@ static void nft_dev_forward_path(const struct nft_pktinfo *pkt,
 
 	route->tuple[!dir].in.num_encaps = info.num_encaps;
 	route->tuple[!dir].in.ingress_vlans = info.ingress_vlans;
-	route->tuple[dir].out.ifindex = info.outdev->ifindex;
 
 	if (info.xmit_type == FLOW_OFFLOAD_XMIT_DIRECT) {
 		memcpy(route->tuple[dir].out.h_source, info.h_source, ETH_ALEN);
diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c
index 78a61dac4ade..e6b24586d2fe 100644
--- a/net/netfilter/nf_nat_core.c
+++ b/net/netfilter/nf_nat_core.c
@@ -294,25 +294,13 @@ nf_nat_used_tuple_new(const struct nf_conntrack_tuple *tuple,
 
 	ct = nf_ct_tuplehash_to_ctrack(thash);
 
-	/* NB: IP_CT_DIR_ORIGINAL should be impossible because
-	 * nf_nat_used_tuple() handles origin collisions.
-	 *
-	 * Handle remote chance other CPU confirmed its ct right after.
-	 */
-	if (thash->tuple.dst.dir != IP_CT_DIR_REPLY)
-		goto out;
-
 	/* clashing connection subject to NAT? Retry with new tuple. */
 	if (READ_ONCE(ct->status) & uses_nat)
 		goto out;
 
 	if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
-			      &ignored_ct->tuplehash[IP_CT_DIR_REPLY].tuple) &&
-	    nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
-			      &ignored_ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple)) {
+			      &ignored_ct->tuplehash[IP_CT_DIR_REPLY].tuple))
 		taken = false;
-		goto out;
-	}
 out:
 	nf_ct_put(ct);
 	return taken;
diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c
index f3de2f9bbebf..618af6e90773 100644
--- a/net/netfilter/nf_tables_api.c
+++ b/net/netfilter/nf_tables_api.c
@@ -123,6 +123,29 @@ static void nft_validate_state_update(struct nft_table *table, u8 new_validate_s
 
 	table->validate_state = new_validate_state;
 }
+
+static bool nft_chain_vstate_valid(const struct nft_ctx *ctx,
+				   const struct nft_chain *chain)
+{
+	const struct nft_base_chain *base_chain;
+	enum nft_chain_types type;
+	u8 hooknum;
+
+	if (WARN_ON_ONCE(!nft_is_base_chain(ctx->chain)))
+		return false;
+
+	base_chain = nft_base_chain(ctx->chain);
+	hooknum = base_chain->ops.hooknum;
+	type = base_chain->type->type;
+
+	/* chain is already validated for this call depth */
+	if (chain->vstate.depth >= ctx->level &&
+	    chain->vstate.hook_mask[type] & BIT(hooknum))
+		return true;
+
+	return false;
+}
+
 static void nf_tables_trans_destroy_work(struct work_struct *w);
 
 static void nft_trans_gc_work(struct work_struct *work);
@@ -4079,6 +4102,29 @@ static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *r
 	nf_tables_rule_destroy(ctx, rule);
 }
 
+static void nft_chain_vstate_update(const struct nft_ctx *ctx, struct nft_chain *chain)
+{
+	const struct nft_base_chain *base_chain;
+	enum nft_chain_types type;
+	u8 hooknum;
+
+	/* ctx->chain must hold the calling base chain. */
+	if (WARN_ON_ONCE(!nft_is_base_chain(ctx->chain))) {
+		memset(&chain->vstate, 0, sizeof(chain->vstate));
+		return;
+	}
+
+	base_chain = nft_base_chain(ctx->chain);
+	hooknum = base_chain->ops.hooknum;
+	type = base_chain->type->type;
+
+	BUILD_BUG_ON(BIT(NF_INET_NUMHOOKS) > U8_MAX);
+
+	chain->vstate.hook_mask[type] |= BIT(hooknum);
+	if (chain->vstate.depth < ctx->level)
+		chain->vstate.depth = ctx->level;
+}
+
 /** nft_chain_validate - loop detection and hook validation
  *
  * @ctx: context containing call depth and base chain
@@ -4088,15 +4134,25 @@ static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *r
  * and set lookups until either the jump limit is hit or all reachable
  * chains have been validated.
  */
-int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
+int nft_chain_validate(const struct nft_ctx *ctx, struct nft_chain *chain)
 {
 	struct nft_expr *expr, *last;
 	struct nft_rule *rule;
 	int err;
 
+	BUILD_BUG_ON(NFT_JUMP_STACK_SIZE > 255);
 	if (ctx->level == NFT_JUMP_STACK_SIZE)
 		return -EMLINK;
 
+	if (ctx->level > 0) {
+		/* jumps to base chains are not allowed. */
+		if (nft_is_base_chain(chain))
+			return -ELOOP;
+
+		if (nft_chain_vstate_valid(ctx, chain))
+			return 0;
+	}
+
 	list_for_each_entry(rule, &chain->rules, list) {
 		if (fatal_signal_pending(current))
 			return -EINTR;
@@ -4115,8 +4171,11 @@ int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
 			if (err < 0)
 				return err;
 		}
+
+		cond_resched();
 	}
 
+	nft_chain_vstate_update(ctx, chain);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(nft_chain_validate);
@@ -4128,7 +4187,7 @@ static int nft_table_validate(struct net *net, const struct nft_table *table)
 		.net	= net,
 		.family	= table->family,
 	};
-	int err;
+	int err = 0;
 
 	list_for_each_entry(chain, &table->chains, list) {
 		if (!nft_is_base_chain(chain))
@@ -4137,12 +4196,14 @@ static int nft_table_validate(struct net *net, const struct nft_table *table)
 		ctx.chain = chain;
 		err = nft_chain_validate(&ctx, chain);
 		if (err < 0)
-			return err;
-
-		cond_resched();
+			goto err;
 	}
 
-	return 0;
+err:
+	list_for_each_entry(chain, &table->chains, list)
+		memset(&chain->vstate, 0, sizeof(chain->vstate));
+
+	return err;
 }
 
 int nft_setelem_validate(const struct nft_ctx *ctx, struct nft_set *set,
@@ -11676,21 +11737,10 @@ static int nft_validate_register_store(const struct nft_ctx *ctx,
 				       enum nft_data_types type,
 				       unsigned int len)
 {
-	int err;
-
 	switch (reg) {
 	case NFT_REG_VERDICT:
 		if (type != NFT_DATA_VERDICT)
 			return -EINVAL;
-
-		if (data != NULL &&
-		    (data->verdict.code == NFT_GOTO ||
-		     data->verdict.code == NFT_JUMP)) {
-			err = nft_chain_validate(ctx, data->verdict.chain);
-			if (err < 0)
-				return err;
-		}
-
 		break;
 	default:
 		if (type != NFT_DATA_VALUE)
diff --git a/net/netfilter/nft_log.c b/net/netfilter/nft_log.c
index e35588137995..bf01cf8a8907 100644
--- a/net/netfilter/nft_log.c
+++ b/net/netfilter/nft_log.c
@@ -26,46 +26,10 @@ struct nft_log {
 	char			*prefix;
 };
 
-static bool audit_ip4(struct audit_buffer *ab, struct sk_buff *skb)
-{
-	struct iphdr _iph;
-	const struct iphdr *ih;
-
-	ih = skb_header_pointer(skb, skb_network_offset(skb), sizeof(_iph), &_iph);
-	if (!ih)
-		return false;
-
-	audit_log_format(ab, " saddr=%pI4 daddr=%pI4 proto=%hhu",
-			 &ih->saddr, &ih->daddr, ih->protocol);
-
-	return true;
-}
-
-static bool audit_ip6(struct audit_buffer *ab, struct sk_buff *skb)
-{
-	struct ipv6hdr _ip6h;
-	const struct ipv6hdr *ih;
-	u8 nexthdr;
-	__be16 frag_off;
-
-	ih = skb_header_pointer(skb, skb_network_offset(skb), sizeof(_ip6h), &_ip6h);
-	if (!ih)
-		return false;
-
-	nexthdr = ih->nexthdr;
-	ipv6_skip_exthdr(skb, skb_network_offset(skb) + sizeof(_ip6h), &nexthdr, &frag_off);
-
-	audit_log_format(ab, " saddr=%pI6c daddr=%pI6c proto=%hhu",
-			 &ih->saddr, &ih->daddr, nexthdr);
-
-	return true;
-}
-
 static void nft_log_eval_audit(const struct nft_pktinfo *pkt)
 {
 	struct sk_buff *skb = pkt->skb;
 	struct audit_buffer *ab;
-	int fam = -1;
 
 	if (!audit_enabled)
 		return;
@@ -76,27 +40,7 @@ static void nft_log_eval_audit(const struct nft_pktinfo *pkt)
 
 	audit_log_format(ab, "mark=%#x", skb->mark);
 
-	switch (nft_pf(pkt)) {
-	case NFPROTO_BRIDGE:
-		switch (eth_hdr(skb)->h_proto) {
-		case htons(ETH_P_IP):
-			fam = audit_ip4(ab, skb) ? NFPROTO_IPV4 : -1;
-			break;
-		case htons(ETH_P_IPV6):
-			fam = audit_ip6(ab, skb) ? NFPROTO_IPV6 : -1;
-			break;
-		}
-		break;
-	case NFPROTO_IPV4:
-		fam = audit_ip4(ab, skb) ? NFPROTO_IPV4 : -1;
-		break;
-	case NFPROTO_IPV6:
-		fam = audit_ip6(ab, skb) ? NFPROTO_IPV6 : -1;
-		break;
-	}
-
-	if (fam == -1)
-		audit_log_format(ab, " saddr=? daddr=? proto=-1");
+	audit_log_nf_skb(ab, skb, nft_pf(pkt));
 
 	audit_log_end(ab);
 }
diff --git a/net/netfilter/xt_AUDIT.c b/net/netfilter/xt_AUDIT.c
index b6a015aee0ce..4c18606b8654 100644
--- a/net/netfilter/xt_AUDIT.c
+++ b/net/netfilter/xt_AUDIT.c
@@ -28,46 +28,10 @@ MODULE_ALIAS("ip6t_AUDIT");
 MODULE_ALIAS("ebt_AUDIT");
 MODULE_ALIAS("arpt_AUDIT");
 
-static bool audit_ip4(struct audit_buffer *ab, struct sk_buff *skb)
-{
-	struct iphdr _iph;
-	const struct iphdr *ih;
-
-	ih = skb_header_pointer(skb, skb_network_offset(skb), sizeof(_iph), &_iph);
-	if (!ih)
-		return false;
-
-	audit_log_format(ab, " saddr=%pI4 daddr=%pI4 proto=%hhu",
-			 &ih->saddr, &ih->daddr, ih->protocol);
-
-	return true;
-}
-
-static bool audit_ip6(struct audit_buffer *ab, struct sk_buff *skb)
-{
-	struct ipv6hdr _ip6h;
-	const struct ipv6hdr *ih;
-	u8 nexthdr;
-	__be16 frag_off;
-
-	ih = skb_header_pointer(skb, skb_network_offset(skb), sizeof(_ip6h), &_ip6h);
-	if (!ih)
-		return false;
-
-	nexthdr = ih->nexthdr;
-	ipv6_skip_exthdr(skb, skb_network_offset(skb) + sizeof(_ip6h), &nexthdr, &frag_off);
-
-	audit_log_format(ab, " saddr=%pI6c daddr=%pI6c proto=%hhu",
-			 &ih->saddr, &ih->daddr, nexthdr);
-
-	return true;
-}
-
 static unsigned int
 audit_tg(struct sk_buff *skb, const struct xt_action_param *par)
 {
 	struct audit_buffer *ab;
-	int fam = -1;
 
 	if (audit_enabled == AUDIT_OFF)
 		goto errout;
@@ -77,27 +41,7 @@ audit_tg(struct sk_buff *skb, const struct xt_action_param *par)
 
 	audit_log_format(ab, "mark=%#x", skb->mark);
 
-	switch (xt_family(par)) {
-	case NFPROTO_BRIDGE:
-		switch (eth_hdr(skb)->h_proto) {
-		case htons(ETH_P_IP):
-			fam = audit_ip4(ab, skb) ? NFPROTO_IPV4 : -1;
-			break;
-		case htons(ETH_P_IPV6):
-			fam = audit_ip6(ab, skb) ? NFPROTO_IPV6 : -1;
-			break;
-		}
-		break;
-	case NFPROTO_IPV4:
-		fam = audit_ip4(ab, skb) ? NFPROTO_IPV4 : -1;
-		break;
-	case NFPROTO_IPV6:
-		fam = audit_ip6(ab, skb) ? NFPROTO_IPV6 : -1;
-		break;
-	}
-
-	if (fam == -1)
-		audit_log_format(ab, " saddr=? daddr=? proto=-1");
+	audit_log_nf_skb(ab, skb, xt_family(par));
 
 	audit_log_end(ab);
 
diff --git a/net/netrom/nr_out.c b/net/netrom/nr_out.c
index 5e531394a724..2b3cbceb0b52 100644
--- a/net/netrom/nr_out.c
+++ b/net/netrom/nr_out.c
@@ -43,8 +43,10 @@ void nr_output(struct sock *sk, struct sk_buff *skb)
 		frontlen = skb_headroom(skb);
 
 		while (skb->len > 0) {
-			if ((skbn = sock_alloc_send_skb(sk, frontlen + NR_MAX_PACKET_SIZE, 0, &err)) == NULL)
+			if ((skbn = sock_alloc_send_skb(sk, frontlen + NR_MAX_PACKET_SIZE, 0, &err)) == NULL) {
+				kfree_skb(skb);
 				return;
+			}
 
 			skb_reserve(skbn, frontlen);
 
diff --git a/net/openvswitch/flow_netlink.c b/net/openvswitch/flow_netlink.c
index 1cb4f97335d8..2d536901309e 100644
--- a/net/openvswitch/flow_netlink.c
+++ b/net/openvswitch/flow_netlink.c
@@ -2802,13 +2802,20 @@ static int validate_and_copy_set_tun(const struct nlattr *attr,
 	return err;
 }
 
-static bool validate_push_nsh(const struct nlattr *attr, bool log)
+static bool validate_push_nsh(const struct nlattr *a, bool log)
 {
+	struct nlattr *nsh_key = nla_data(a);
 	struct sw_flow_match match;
 	struct sw_flow_key key;
 
+	/* There must be one and only one NSH header. */
+	if (!nla_ok(nsh_key, nla_len(a)) ||
+	    nla_total_size(nla_len(nsh_key)) != nla_len(a) ||
+	    nla_type(nsh_key) != OVS_KEY_ATTR_NSH)
+		return false;
+
 	ovs_match_init(&match, &key, true, NULL);
-	return !nsh_key_put_from_nlattr(attr, &match, false, true, log);
+	return !nsh_key_put_from_nlattr(nsh_key, &match, false, true, log);
 }
 
 /* Return false if there are any non-masked bits set.
@@ -3389,7 +3396,7 @@ static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr,
 					return -EINVAL;
 			}
 			mac_proto = MAC_PROTO_NONE;
-			if (!validate_push_nsh(nla_data(a), log))
+			if (!validate_push_nsh(a, log))
 				return -EINVAL;
 			break;
 
diff --git a/net/sched/act_mirred.c b/net/sched/act_mirred.c
index f27b583def78..91c96cc625bd 100644
--- a/net/sched/act_mirred.c
+++ b/net/sched/act_mirred.c
@@ -281,6 +281,15 @@ static int tcf_mirred_to_dev(struct sk_buff *skb, struct tcf_mirred *m,
 
 	want_ingress = tcf_mirred_act_wants_ingress(m_eaction);
 
+	if (dev == skb->dev && want_ingress == at_ingress) {
+		pr_notice_once("tc mirred: Loop (%s:%s --> %s:%s)\n",
+			       netdev_name(skb->dev),
+			       at_ingress ? "ingress" : "egress",
+			       netdev_name(dev),
+			       want_ingress ? "ingress" : "egress");
+		goto err_cant_do;
+	}
+
 	/* All mirred/redirected skbs should clear previous ct info */
 	nf_reset_ct(skb_to_send);
 	if (want_ingress && !at_ingress) /* drop dst for egress -> ingress */
diff --git a/net/sched/sch_ets.c b/net/sched/sch_ets.c
index 82635dd2cfa5..306e046276d4 100644
--- a/net/sched/sch_ets.c
+++ b/net/sched/sch_ets.c
@@ -652,7 +652,7 @@ static int ets_qdisc_change(struct Qdisc *sch, struct nlattr *opt,
 	sch_tree_lock(sch);
 
 	for (i = nbands; i < oldbands; i++) {
-		if (i >= q->nstrict && q->classes[i].qdisc->q.qlen)
+		if (cl_is_active(&q->classes[i]))
 			list_del_init(&q->classes[i].alist);
 		qdisc_purge_queue(q->classes[i].qdisc);
 	}
@@ -664,6 +664,10 @@ static int ets_qdisc_change(struct Qdisc *sch, struct nlattr *opt,
 			q->classes[i].deficit = quanta[i];
 		}
 	}
+	for (i = q->nstrict; i < nstrict; i++) {
+		if (cl_is_active(&q->classes[i]))
+			list_del_init(&q->classes[i].alist);
+	}
 	WRITE_ONCE(q->nstrict, nstrict);
 	memcpy(q->prio2band, priomap, sizeof(priomap));
 
diff --git a/net/sctp/ipv6.c b/net/sctp/ipv6.c
index 069b7e45d8bd..531cb0690007 100644
--- a/net/sctp/ipv6.c
+++ b/net/sctp/ipv6.c
@@ -492,6 +492,8 @@ static void sctp_v6_copy_ip_options(struct sock *sk, struct sock *newsk)
 	struct ipv6_pinfo *newnp, *np = inet6_sk(sk);
 	struct ipv6_txoptions *opt;
 
+	inet_sk(newsk)->inet_opt = NULL;
+
 	newnp = inet6_sk(newsk);
 
 	rcu_read_lock();
diff --git a/net/sctp/socket.c b/net/sctp/socket.c
index d808096f5ab1..2493a5b1fa3c 100644
--- a/net/sctp/socket.c
+++ b/net/sctp/socket.c
@@ -4863,8 +4863,6 @@ static struct sock *sctp_clone_sock(struct sock *sk,
 
 	newsp->pf->to_sk_daddr(&asoc->peer.primary_addr, newsk);
 	newinet->inet_dport = htons(asoc->peer.port);
-
-	newsp->pf->copy_ip_options(sk, newsk);
 	atomic_set(&newinet->inet_id, get_random_u16());
 
 	inet_set_bit(MC_LOOP, newsk);
@@ -4874,17 +4872,20 @@ static struct sock *sctp_clone_sock(struct sock *sk,
 
 #if IS_ENABLED(CONFIG_IPV6)
 	if (sk->sk_family == AF_INET6) {
-		struct ipv6_pinfo *newnp = inet6_sk(newsk);
+		struct ipv6_pinfo *newnp;
 
 		newinet->pinet6 = &((struct sctp6_sock *)newsk)->inet6;
 		newinet->ipv6_fl_list = NULL;
 
+		newnp = inet6_sk(newsk);
 		memcpy(newnp, inet6_sk(sk), sizeof(struct ipv6_pinfo));
 		newnp->ipv6_mc_list = NULL;
 		newnp->ipv6_ac_list = NULL;
 	}
 #endif
 
+	newsp->pf->copy_ip_options(sk, newsk);
+
 	newsp->do_auto_asconf = 0;
 	skb_queue_head_init(&newsp->pd_lobby);
 
diff --git a/net/smc/Kconfig b/net/smc/Kconfig
index 325addf83cc6..277ef504bc26 100644
--- a/net/smc/Kconfig
+++ b/net/smc/Kconfig
@@ -22,10 +22,10 @@ config SMC_DIAG
 
 config SMC_HS_CTRL_BPF
 	bool "Generic eBPF hook for SMC handshake flow"
-	depends on SMC && BPF_SYSCALL
+	depends on SMC && BPF_JIT && BPF_SYSCALL
 	default y
 	help
 	  SMC_HS_CTRL_BPF enables support to register generic eBPF hook for SMC
 	  handshake flow, which offer much greater flexibility in modifying the behavior
 	  of the SMC protocol stack compared to a complete kernel-based approach. Select
-	  this option if you want filtring the handshake process via eBPF programs.
\ No newline at end of file
+	  this option if you want filtring the handshake process via eBPF programs.
diff --git a/net/socket.c b/net/socket.c
index 136b98c54fb3..666a1d75b155 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -650,7 +650,11 @@ struct socket *sock_alloc(void)
 }
 EXPORT_SYMBOL(sock_alloc);
 
-static void __sock_release(struct socket *sock, struct inode *inode)
+static
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+noinline
+#endif
+void __sock_release(struct socket *sock, struct inode *inode)
 {
 	const struct proto_ops *ops = READ_ONCE(sock->ops);
 
@@ -722,7 +726,13 @@ static noinline void call_trace_sock_send_length(struct sock *sk, int ret,
 	trace_sock_send_length(sk, ret, 0);
 }
 
-static inline int sock_sendmsg_nosec(struct socket *sock, struct msghdr *msg)
+static
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+noinline
+#else
+inline
+#endif
+int sock_sendmsg_nosec(struct socket *sock, struct msghdr *msg)
 {
 	int ret = INDIRECT_CALL_INET(READ_ONCE(sock->ops)->sendmsg, inet6_sendmsg,
 				     inet_sendmsg, sock, msg,
@@ -1072,8 +1082,13 @@ static noinline void call_trace_sock_recv_length(struct sock *sk, int ret, int f
 	trace_sock_recv_length(sk, ret, flags);
 }
 
-static inline int sock_recvmsg_nosec(struct socket *sock, struct msghdr *msg,
-				     int flags)
+static
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+noinline
+#else
+inline
+#endif
+int sock_recvmsg_nosec(struct socket *sock, struct msghdr *msg, int flags)
 {
 	int ret = INDIRECT_CALL_INET(READ_ONCE(sock->ops)->recvmsg,
 				     inet6_recvmsg,
@@ -2532,9 +2547,12 @@ static int copy_msghdr_from_user(struct msghdr *kmsg,
 	return err < 0 ? err : 0;
 }
 
-static int ____sys_sendmsg(struct socket *sock, struct msghdr *msg_sys,
-			   unsigned int flags, struct used_address *used_address,
-			   unsigned int allowed_msghdr_flags)
+static
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+noinline
+#endif
+int ____sys_sendmsg(struct socket *sock, struct msghdr *msg_sys, unsigned int flags,
+		    struct used_address *used_address, unsigned int allowed_msghdr_flags)
 {
 	unsigned char ctl[sizeof(struct cmsghdr) + 20]
 				__aligned(sizeof(__kernel_size_t));
diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c
index a8ec30759a18..e2f0df8cdaa6 100644
--- a/net/sunrpc/auth_gss/svcauth_gss.c
+++ b/net/sunrpc/auth_gss/svcauth_gss.c
@@ -1083,7 +1083,8 @@ static int gss_read_proxy_verf(struct svc_rqst *rqstp,
 	}
 
 	length = min_t(unsigned int, inlen, (char *)xdr->end - (char *)xdr->p);
-	memcpy(page_address(in_token->pages[0]), xdr->p, length);
+	if (length)
+		memcpy(page_address(in_token->pages[0]), xdr->p, length);
 	inlen -= length;
 
 	to_offs = length;
diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c
index 131090f31e6a..d808c0b63f30 100644
--- a/net/sunrpc/cache.c
+++ b/net/sunrpc/cache.c
@@ -11,6 +11,7 @@
 #include <linux/types.h>
 #include <linux/fs.h>
 #include <linux/file.h>
+#include <linux/hex.h>
 #include <linux/slab.h>
 #include <linux/signal.h>
 #include <linux/sched.h>
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index 661b3fe2779f..310de7a80be5 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -841,6 +841,9 @@ static int svc_rdma_copy_inline_range(struct svc_rqst *rqstp,
 	for (page_no = 0; page_no < numpages; page_no++) {
 		unsigned int page_len;
 
+		if (head->rc_curpage >= rqstp->rq_maxpages)
+			return -EINVAL;
+
 		page_len = min_t(unsigned int, remaining,
 				 PAGE_SIZE - head->rc_pageoff);
 
@@ -848,7 +851,7 @@ static int svc_rdma_copy_inline_range(struct svc_rqst *rqstp,
 			head->rc_page_count++;
 
 		dst = page_address(rqstp->rq_pages[head->rc_curpage]);
-		memcpy(dst + head->rc_curpage, src + offset, page_len);
+		memcpy((unsigned char *)dst + head->rc_pageoff, src + offset, page_len);
 
 		head->rc_readbytes += page_len;
 		head->rc_pageoff += page_len;
@@ -860,7 +863,7 @@ static int svc_rdma_copy_inline_range(struct svc_rqst *rqstp,
 		offset += page_len;
 	}
 
-	return -EINVAL;
+	return 0;
 }
 
 /**
diff --git a/net/tipc/core.h b/net/tipc/core.h
index 7f3fe3401c45..9ce5f9ff6cc0 100644
--- a/net/tipc/core.h
+++ b/net/tipc/core.h
@@ -44,6 +44,7 @@
 #include <linux/types.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
+#include <linux/hex.h>
 #include <linux/mm.h>
 #include <linux/timer.h>
 #include <linux/string.h>
diff --git a/net/unix/garbage.c b/net/unix/garbage.c
index 78323d43e63e..25f65817faab 100644
--- a/net/unix/garbage.c
+++ b/net/unix/garbage.c
@@ -199,7 +199,7 @@ static void unix_free_vertices(struct scm_fp_list *fpl)
 	}
 }
 
-static DEFINE_SPINLOCK(unix_gc_lock);
+static __cacheline_aligned_in_smp DEFINE_SPINLOCK(unix_gc_lock);
 
 void unix_add_edges(struct scm_fp_list *fpl, struct unix_sock *receiver)
 {
diff --git a/net/wireless/sme.c b/net/wireless/sme.c
index 3a028ff287fb..4e629ca305bc 100644
--- a/net/wireless/sme.c
+++ b/net/wireless/sme.c
@@ -910,7 +910,7 @@ void __cfg80211_connect_result(struct net_device *dev,
 
 			ssid_len = min(ssid->datalen, IEEE80211_MAX_SSID_LEN);
 			memcpy(wdev->u.client.ssid, ssid->data, ssid_len);
-			wdev->u.client.ssid_len = ssid->datalen;
+			wdev->u.client.ssid_len = ssid_len;
 			break;
 		}
 		rcu_read_unlock();
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index 9e14e453b55c..98b362d51836 100644
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -3151,6 +3151,7 @@ int __xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
 	int err;
 
 	if (family == AF_INET &&
+	    (!x->dir || x->dir == XFRM_SA_DIR_OUT) &&
 	    READ_ONCE(xs_net(x)->ipv4.sysctl_ip_no_pmtu_disc))
 		x->props.flags |= XFRM_STATE_NOPMTUDISC;
 
diff --git a/rust/helpers/auxiliary.c b/rust/helpers/auxiliary.c
index 8b5e0fea4493..dd1c130843a0 100644
--- a/rust/helpers/auxiliary.c
+++ b/rust/helpers/auxiliary.c
@@ -2,12 +2,14 @@
 
 #include <linux/auxiliary_bus.h>
 
-void rust_helper_auxiliary_device_uninit(struct auxiliary_device *adev)
+__rust_helper void
+rust_helper_auxiliary_device_uninit(struct auxiliary_device *adev)
 {
 	return auxiliary_device_uninit(adev);
 }
 
-void rust_helper_auxiliary_device_delete(struct auxiliary_device *adev)
+__rust_helper void
+rust_helper_auxiliary_device_delete(struct auxiliary_device *adev)
 {
 	return auxiliary_device_delete(adev);
 }
diff --git a/rust/helpers/bitmap.c b/rust/helpers/bitmap.c
index a50e2f082e47..e4e9f4361270 100644
--- a/rust/helpers/bitmap.c
+++ b/rust/helpers/bitmap.c
@@ -2,6 +2,7 @@
 
 #include <linux/bitmap.h>
 
+__rust_helper
 void rust_helper_bitmap_copy_and_extend(unsigned long *to, const unsigned long *from,
 		unsigned int count, unsigned int size)
 {
diff --git a/rust/helpers/bitops.c b/rust/helpers/bitops.c
index 5d0861d29d3f..f875692ead30 100644
--- a/rust/helpers/bitops.c
+++ b/rust/helpers/bitops.c
@@ -2,21 +2,25 @@
 
 #include <linux/bitops.h>
 
+__rust_helper
 void rust_helper___set_bit(unsigned long nr, unsigned long *addr)
 {
 	__set_bit(nr, addr);
 }
 
+__rust_helper
 void rust_helper___clear_bit(unsigned long nr, unsigned long *addr)
 {
 	__clear_bit(nr, addr);
 }
 
+__rust_helper
 void rust_helper_set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	set_bit(nr, addr);
 }
 
+__rust_helper
 void rust_helper_clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	clear_bit(nr, addr);
diff --git a/rust/helpers/cpufreq.c b/rust/helpers/cpufreq.c
index 7c1343c4d65e..0e16aeef2b5a 100644
--- a/rust/helpers/cpufreq.c
+++ b/rust/helpers/cpufreq.c
@@ -3,7 +3,8 @@
 #include <linux/cpufreq.h>
 
 #ifdef CONFIG_CPU_FREQ
-void rust_helper_cpufreq_register_em_with_opp(struct cpufreq_policy *policy)
+__rust_helper void
+rust_helper_cpufreq_register_em_with_opp(struct cpufreq_policy *policy)
 {
 	cpufreq_register_em_with_opp(policy);
 }
diff --git a/rust/helpers/cpumask.c b/rust/helpers/cpumask.c
index eb10598a0242..5deced5b975e 100644
--- a/rust/helpers/cpumask.c
+++ b/rust/helpers/cpumask.c
@@ -2,67 +2,80 @@
 
 #include <linux/cpumask.h>
 
+__rust_helper
 void rust_helper_cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
 {
 	cpumask_set_cpu(cpu, dstp);
 }
 
+__rust_helper
 void rust_helper___cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
 {
 	__cpumask_set_cpu(cpu, dstp);
 }
 
+__rust_helper
 void rust_helper_cpumask_clear_cpu(int cpu, struct cpumask *dstp)
 {
 	cpumask_clear_cpu(cpu, dstp);
 }
 
+__rust_helper
 void rust_helper___cpumask_clear_cpu(int cpu, struct cpumask *dstp)
 {
 	__cpumask_clear_cpu(cpu, dstp);
 }
 
+__rust_helper
 bool rust_helper_cpumask_test_cpu(int cpu, struct cpumask *srcp)
 {
 	return cpumask_test_cpu(cpu, srcp);
 }
 
+__rust_helper
 void rust_helper_cpumask_setall(struct cpumask *dstp)
 {
 	cpumask_setall(dstp);
 }
 
+__rust_helper
 bool rust_helper_cpumask_empty(struct cpumask *srcp)
 {
 	return cpumask_empty(srcp);
 }
 
+__rust_helper
 bool rust_helper_cpumask_full(struct cpumask *srcp)
 {
 	return cpumask_full(srcp);
 }
 
+__rust_helper
 unsigned int rust_helper_cpumask_weight(struct cpumask *srcp)
 {
 	return cpumask_weight(srcp);
 }
 
+__rust_helper
 void rust_helper_cpumask_copy(struct cpumask *dstp, const struct cpumask *srcp)
 {
 	cpumask_copy(dstp, srcp);
 }
 
+__rust_helper
 bool rust_helper_alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
 {
 	return alloc_cpumask_var(mask, flags);
 }
 
+__rust_helper
 bool rust_helper_zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
 {
 	return zalloc_cpumask_var(mask, flags);
 }
 
 #ifndef CONFIG_CPUMASK_OFFSTACK
+__rust_helper
 void rust_helper_free_cpumask_var(cpumask_var_t mask)
 {
 	free_cpumask_var(mask);
diff --git a/rust/helpers/device.c b/rust/helpers/device.c
index 9a4316bafedf..a8ab931a9bd1 100644
--- a/rust/helpers/device.c
+++ b/rust/helpers/device.c
@@ -2,26 +2,26 @@
 
 #include <linux/device.h>
 
-int rust_helper_devm_add_action(struct device *dev,
-				void (*action)(void *),
-				void *data)
+__rust_helper int rust_helper_devm_add_action(struct device *dev,
+					      void (*action)(void *),
+					      void *data)
 {
 	return devm_add_action(dev, action, data);
 }
 
-int rust_helper_devm_add_action_or_reset(struct device *dev,
-					 void (*action)(void *),
-					 void *data)
+__rust_helper int rust_helper_devm_add_action_or_reset(struct device *dev,
+						       void (*action)(void *),
+						       void *data)
 {
 	return devm_add_action_or_reset(dev, action, data);
 }
 
-void *rust_helper_dev_get_drvdata(const struct device *dev)
+__rust_helper void *rust_helper_dev_get_drvdata(const struct device *dev)
 {
 	return dev_get_drvdata(dev);
 }
 
-void rust_helper_dev_set_drvdata(struct device *dev, void *data)
+__rust_helper void rust_helper_dev_set_drvdata(struct device *dev, void *data)
 {
 	dev_set_drvdata(dev, data);
 }
diff --git a/rust/helpers/dma.c b/rust/helpers/dma.c
index 6e741c197242..e7defeecda71 100644
--- a/rust/helpers/dma.c
+++ b/rust/helpers/dma.c
@@ -2,20 +2,44 @@
 
 #include <linux/dma-mapping.h>
 
-void *rust_helper_dma_alloc_attrs(struct device *dev, size_t size,
-				  dma_addr_t *dma_handle, gfp_t flag,
-				  unsigned long attrs)
+__rust_helper void *rust_helper_dma_alloc_attrs(struct device *dev, size_t size,
+						dma_addr_t *dma_handle,
+						gfp_t flag, unsigned long attrs)
 {
 	return dma_alloc_attrs(dev, size, dma_handle, flag, attrs);
 }
 
-void rust_helper_dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
-				dma_addr_t dma_handle, unsigned long attrs)
+__rust_helper void rust_helper_dma_free_attrs(struct device *dev, size_t size,
+					      void *cpu_addr,
+					      dma_addr_t dma_handle,
+					      unsigned long attrs)
 {
 	dma_free_attrs(dev, size, cpu_addr, dma_handle, attrs);
 }
 
-int rust_helper_dma_set_mask_and_coherent(struct device *dev, u64 mask)
+__rust_helper int rust_helper_dma_set_mask_and_coherent(struct device *dev,
+							u64 mask)
 {
 	return dma_set_mask_and_coherent(dev, mask);
 }
+
+int rust_helper_dma_set_mask(struct device *dev, u64 mask)
+{
+	return dma_set_mask(dev, mask);
+}
+
+int rust_helper_dma_set_coherent_mask(struct device *dev, u64 mask)
+{
+	return dma_set_coherent_mask(dev, mask);
+}
+
+int rust_helper_dma_map_sgtable(struct device *dev, struct sg_table *sgt,
+				enum dma_data_direction dir, unsigned long attrs)
+{
+	return dma_map_sgtable(dev, sgt, dir, attrs);
+}
+
+size_t rust_helper_dma_max_mapping_size(struct device *dev)
+{
+	return dma_max_mapping_size(dev);
+}
diff --git a/rust/helpers/drm.c b/rust/helpers/drm.c
index 450b406c6f27..fe226f7b53ef 100644
--- a/rust/helpers/drm.c
+++ b/rust/helpers/drm.c
@@ -5,17 +5,18 @@
 
 #ifdef CONFIG_DRM
 
-void rust_helper_drm_gem_object_get(struct drm_gem_object *obj)
+__rust_helper void rust_helper_drm_gem_object_get(struct drm_gem_object *obj)
 {
 	drm_gem_object_get(obj);
 }
 
-void rust_helper_drm_gem_object_put(struct drm_gem_object *obj)
+__rust_helper void rust_helper_drm_gem_object_put(struct drm_gem_object *obj)
 {
 	drm_gem_object_put(obj);
 }
 
-__u64 rust_helper_drm_vma_node_offset_addr(struct drm_vma_offset_node *node)
+__rust_helper __u64
+rust_helper_drm_vma_node_offset_addr(struct drm_vma_offset_node *node)
 {
 	return drm_vma_node_offset_addr(node);
 }
diff --git a/rust/helpers/fs.c b/rust/helpers/fs.c
index a75c96763372..789d60fb8908 100644
--- a/rust/helpers/fs.c
+++ b/rust/helpers/fs.c
@@ -6,7 +6,7 @@
 
 #include <linux/fs.h>
 
-struct file *rust_helper_get_file(struct file *f)
+__rust_helper struct file *rust_helper_get_file(struct file *f)
 {
 	return get_file(f);
 }
diff --git a/rust/helpers/io.c b/rust/helpers/io.c
index c475913c69e6..397810864a24 100644
--- a/rust/helpers/io.c
+++ b/rust/helpers/io.c
@@ -3,140 +3,144 @@
 #include <linux/io.h>
 #include <linux/ioport.h>
 
-void __iomem *rust_helper_ioremap(phys_addr_t offset, size_t size)
+__rust_helper void __iomem *rust_helper_ioremap(phys_addr_t offset, size_t size)
 {
 	return ioremap(offset, size);
 }
 
-void __iomem *rust_helper_ioremap_np(phys_addr_t offset, size_t size)
+__rust_helper void __iomem *rust_helper_ioremap_np(phys_addr_t offset,
+						   size_t size)
 {
 	return ioremap_np(offset, size);
 }
 
-void rust_helper_iounmap(void __iomem *addr)
+__rust_helper void rust_helper_iounmap(void __iomem *addr)
 {
 	iounmap(addr);
 }
 
-u8 rust_helper_readb(const void __iomem *addr)
+__rust_helper u8 rust_helper_readb(const void __iomem *addr)
 {
 	return readb(addr);
 }
 
-u16 rust_helper_readw(const void __iomem *addr)
+__rust_helper u16 rust_helper_readw(const void __iomem *addr)
 {
 	return readw(addr);
 }
 
-u32 rust_helper_readl(const void __iomem *addr)
+__rust_helper u32 rust_helper_readl(const void __iomem *addr)
 {
 	return readl(addr);
 }
 
 #ifdef CONFIG_64BIT
-u64 rust_helper_readq(const void __iomem *addr)
+__rust_helper u64 rust_helper_readq(const void __iomem *addr)
 {
 	return readq(addr);
 }
 #endif
 
-void rust_helper_writeb(u8 value, void __iomem *addr)
+__rust_helper void rust_helper_writeb(u8 value, void __iomem *addr)
 {
 	writeb(value, addr);
 }
 
-void rust_helper_writew(u16 value, void __iomem *addr)
+__rust_helper void rust_helper_writew(u16 value, void __iomem *addr)
 {
 	writew(value, addr);
 }
 
-void rust_helper_writel(u32 value, void __iomem *addr)
+__rust_helper void rust_helper_writel(u32 value, void __iomem *addr)
 {
 	writel(value, addr);
 }
 
 #ifdef CONFIG_64BIT
-void rust_helper_writeq(u64 value, void __iomem *addr)
+__rust_helper void rust_helper_writeq(u64 value, void __iomem *addr)
 {
 	writeq(value, addr);
 }
 #endif
 
-u8 rust_helper_readb_relaxed(const void __iomem *addr)
+__rust_helper u8 rust_helper_readb_relaxed(const void __iomem *addr)
 {
 	return readb_relaxed(addr);
 }
 
-u16 rust_helper_readw_relaxed(const void __iomem *addr)
+__rust_helper u16 rust_helper_readw_relaxed(const void __iomem *addr)
 {
 	return readw_relaxed(addr);
 }
 
-u32 rust_helper_readl_relaxed(const void __iomem *addr)
+__rust_helper u32 rust_helper_readl_relaxed(const void __iomem *addr)
 {
 	return readl_relaxed(addr);
 }
 
 #ifdef CONFIG_64BIT
-u64 rust_helper_readq_relaxed(const void __iomem *addr)
+__rust_helper u64 rust_helper_readq_relaxed(const void __iomem *addr)
 {
 	return readq_relaxed(addr);
 }
 #endif
 
-void rust_helper_writeb_relaxed(u8 value, void __iomem *addr)
+__rust_helper void rust_helper_writeb_relaxed(u8 value, void __iomem *addr)
 {
 	writeb_relaxed(value, addr);
 }
 
-void rust_helper_writew_relaxed(u16 value, void __iomem *addr)
+__rust_helper void rust_helper_writew_relaxed(u16 value, void __iomem *addr)
 {
 	writew_relaxed(value, addr);
 }
 
-void rust_helper_writel_relaxed(u32 value, void __iomem *addr)
+__rust_helper void rust_helper_writel_relaxed(u32 value, void __iomem *addr)
 {
 	writel_relaxed(value, addr);
 }
 
 #ifdef CONFIG_64BIT
-void rust_helper_writeq_relaxed(u64 value, void __iomem *addr)
+__rust_helper void rust_helper_writeq_relaxed(u64 value, void __iomem *addr)
 {
 	writeq_relaxed(value, addr);
 }
 #endif
 
-resource_size_t rust_helper_resource_size(struct resource *res)
+__rust_helper resource_size_t rust_helper_resource_size(struct resource *res)
 {
 	return resource_size(res);
 }
 
-struct resource *rust_helper_request_mem_region(resource_size_t start,
-						resource_size_t n,
-						const char *name)
+__rust_helper struct resource *
+rust_helper_request_mem_region(resource_size_t start, resource_size_t n,
+			       const char *name)
 {
 	return request_mem_region(start, n, name);
 }
 
-void rust_helper_release_mem_region(resource_size_t start, resource_size_t n)
+__rust_helper void rust_helper_release_mem_region(resource_size_t start,
+						  resource_size_t n)
 {
 	release_mem_region(start, n);
 }
 
-struct resource *rust_helper_request_region(resource_size_t start,
-					    resource_size_t n, const char *name)
+__rust_helper struct resource *rust_helper_request_region(resource_size_t start,
+							  resource_size_t n,
+							  const char *name)
 {
 	return request_region(start, n, name);
 }
 
-struct resource *rust_helper_request_muxed_region(resource_size_t start,
-						  resource_size_t n,
-						  const char *name)
+__rust_helper struct resource *
+rust_helper_request_muxed_region(resource_size_t start, resource_size_t n,
+				 const char *name)
 {
 	return request_muxed_region(start, n, name);
 }
 
-void rust_helper_release_region(resource_size_t start, resource_size_t n)
+__rust_helper void rust_helper_release_region(resource_size_t start,
+					      resource_size_t n)
 {
 	release_region(start, n);
 }
diff --git a/rust/helpers/irq.c b/rust/helpers/irq.c
index 1faca428e2c0..a61fad333866 100644
--- a/rust/helpers/irq.c
+++ b/rust/helpers/irq.c
@@ -2,8 +2,10 @@
 
 #include <linux/interrupt.h>
 
-int rust_helper_request_irq(unsigned int irq, irq_handler_t handler,
-			    unsigned long flags, const char *name, void *dev)
+__rust_helper int rust_helper_request_irq(unsigned int irq,
+					  irq_handler_t handler,
+					  unsigned long flags, const char *name,
+					  void *dev)
 {
 	return request_irq(irq, handler, flags, name, dev);
 }
diff --git a/rust/helpers/kunit.c b/rust/helpers/kunit.c
index b85a4d394c11..cafb94b6776c 100644
--- a/rust/helpers/kunit.c
+++ b/rust/helpers/kunit.c
@@ -2,7 +2,7 @@
 
 #include <kunit/test-bug.h>
 
-struct kunit *rust_helper_kunit_get_current_test(void)
+__rust_helper struct kunit *rust_helper_kunit_get_current_test(void)
 {
 	return kunit_get_current_test();
 }
diff --git a/rust/helpers/pci.c b/rust/helpers/pci.c
index bf8173979c5e..e44905317d75 100644
--- a/rust/helpers/pci.c
+++ b/rust/helpers/pci.c
@@ -2,41 +2,44 @@
 
 #include <linux/pci.h>
 
-u16 rust_helper_pci_dev_id(struct pci_dev *dev)
+__rust_helper u16 rust_helper_pci_dev_id(struct pci_dev *dev)
 {
 	return PCI_DEVID(dev->bus->number, dev->devfn);
 }
 
-resource_size_t rust_helper_pci_resource_start(struct pci_dev *pdev, int bar)
+__rust_helper resource_size_t
+rust_helper_pci_resource_start(struct pci_dev *pdev, int bar)
 {
 	return pci_resource_start(pdev, bar);
 }
 
-resource_size_t rust_helper_pci_resource_len(struct pci_dev *pdev, int bar)
+__rust_helper resource_size_t rust_helper_pci_resource_len(struct pci_dev *pdev,
+							   int bar)
 {
 	return pci_resource_len(pdev, bar);
 }
 
-bool rust_helper_dev_is_pci(const struct device *dev)
+__rust_helper bool rust_helper_dev_is_pci(const struct device *dev)
 {
 	return dev_is_pci(dev);
 }
 
 #ifndef CONFIG_PCI_MSI
-int rust_helper_pci_alloc_irq_vectors(struct pci_dev *dev,
-				      unsigned int min_vecs,
-				      unsigned int max_vecs,
-				      unsigned int flags)
+__rust_helper int rust_helper_pci_alloc_irq_vectors(struct pci_dev *dev,
+						    unsigned int min_vecs,
+						    unsigned int max_vecs,
+						    unsigned int flags)
 {
 	return pci_alloc_irq_vectors(dev, min_vecs, max_vecs, flags);
 }
 
-void rust_helper_pci_free_irq_vectors(struct pci_dev *dev)
+__rust_helper void rust_helper_pci_free_irq_vectors(struct pci_dev *dev)
 {
 	pci_free_irq_vectors(dev);
 }
 
-int rust_helper_pci_irq_vector(struct pci_dev *pdev, unsigned int nvec)
+__rust_helper int rust_helper_pci_irq_vector(struct pci_dev *pdev,
+					     unsigned int nvec)
 {
 	return pci_irq_vector(pdev, nvec);
 }
diff --git a/rust/helpers/pid_namespace.c b/rust/helpers/pid_namespace.c
index f41482bdec9a..f46ab779b527 100644
--- a/rust/helpers/pid_namespace.c
+++ b/rust/helpers/pid_namespace.c
@@ -3,18 +3,20 @@
 #include <linux/pid_namespace.h>
 #include <linux/cleanup.h>
 
-struct pid_namespace *rust_helper_get_pid_ns(struct pid_namespace *ns)
+__rust_helper struct pid_namespace *
+rust_helper_get_pid_ns(struct pid_namespace *ns)
 {
 	return get_pid_ns(ns);
 }
 
-void rust_helper_put_pid_ns(struct pid_namespace *ns)
+__rust_helper void rust_helper_put_pid_ns(struct pid_namespace *ns)
 {
 	put_pid_ns(ns);
 }
 
 /* Get a reference on a task's pid namespace. */
-struct pid_namespace *rust_helper_task_get_pid_ns(struct task_struct *task)
+__rust_helper struct pid_namespace *
+rust_helper_task_get_pid_ns(struct task_struct *task)
 {
 	struct pid_namespace *pid_ns;
 
diff --git a/rust/helpers/platform.c b/rust/helpers/platform.c
index 1ce89c1a36f7..188b3240f32d 100644
--- a/rust/helpers/platform.c
+++ b/rust/helpers/platform.c
@@ -2,7 +2,7 @@
 
 #include <linux/platform_device.h>
 
-bool rust_helper_dev_is_platform(const struct device *dev)
+__rust_helper bool rust_helper_dev_is_platform(const struct device *dev)
 {
 	return dev_is_platform(dev);
 }
diff --git a/rust/helpers/poll.c b/rust/helpers/poll.c
index 7e5b1751c2d5..78b3839b50f0 100644
--- a/rust/helpers/poll.c
+++ b/rust/helpers/poll.c
@@ -3,8 +3,9 @@
 #include <linux/export.h>
 #include <linux/poll.h>
 
-void rust_helper_poll_wait(struct file *filp, wait_queue_head_t *wait_address,
-			   poll_table *p)
+__rust_helper void rust_helper_poll_wait(struct file *filp,
+					 wait_queue_head_t *wait_address,
+					 poll_table *p)
 {
 	poll_wait(filp, wait_address, p);
 }
diff --git a/rust/helpers/property.c b/rust/helpers/property.c
index 08f68e2dac4a..8fb9900533ef 100644
--- a/rust/helpers/property.c
+++ b/rust/helpers/property.c
@@ -2,7 +2,7 @@
 
 #include <linux/property.h>
 
-void rust_helper_fwnode_handle_put(struct fwnode_handle *fwnode)
+__rust_helper void rust_helper_fwnode_handle_put(struct fwnode_handle *fwnode)
 {
 	fwnode_handle_put(fwnode);
 }
diff --git a/rust/helpers/regulator.c b/rust/helpers/regulator.c
index 11bc332443bd..9ec5237f449b 100644
--- a/rust/helpers/regulator.c
+++ b/rust/helpers/regulator.c
@@ -4,48 +4,52 @@
 
 #ifndef CONFIG_REGULATOR
 
-void rust_helper_regulator_put(struct regulator *regulator)
+__rust_helper void rust_helper_regulator_put(struct regulator *regulator)
 {
 	regulator_put(regulator);
 }
 
-int rust_helper_regulator_set_voltage(struct regulator *regulator, int min_uV,
-				      int max_uV)
+__rust_helper int rust_helper_regulator_set_voltage(struct regulator *regulator,
+						    int min_uV, int max_uV)
 {
 	return regulator_set_voltage(regulator, min_uV, max_uV);
 }
 
-int rust_helper_regulator_get_voltage(struct regulator *regulator)
+__rust_helper int rust_helper_regulator_get_voltage(struct regulator *regulator)
 {
 	return regulator_get_voltage(regulator);
 }
 
-struct regulator *rust_helper_regulator_get(struct device *dev, const char *id)
+__rust_helper struct regulator *rust_helper_regulator_get(struct device *dev,
+							  const char *id)
 {
 	return regulator_get(dev, id);
 }
 
-int rust_helper_regulator_enable(struct regulator *regulator)
+__rust_helper int rust_helper_regulator_enable(struct regulator *regulator)
 {
 	return regulator_enable(regulator);
 }
 
-int rust_helper_regulator_disable(struct regulator *regulator)
+__rust_helper int rust_helper_regulator_disable(struct regulator *regulator)
 {
 	return regulator_disable(regulator);
 }
 
-int rust_helper_regulator_is_enabled(struct regulator *regulator)
+__rust_helper int rust_helper_regulator_is_enabled(struct regulator *regulator)
 {
 	return regulator_is_enabled(regulator);
 }
 
-int rust_helper_devm_regulator_get_enable(struct device *dev, const char *id)
+__rust_helper int rust_helper_devm_regulator_get_enable(struct device *dev,
+							const char *id)
 {
 	return devm_regulator_get_enable(dev, id);
 }
 
-int rust_helper_devm_regulator_get_enable_optional(struct device *dev, const char *id)
+__rust_helper int
+rust_helper_devm_regulator_get_enable_optional(struct device *dev,
+					       const char *id)
 {
 	return devm_regulator_get_enable_optional(dev, id);
 }
diff --git a/rust/helpers/scatterlist.c b/rust/helpers/scatterlist.c
index 80c956ee09ab..f3c41ea5e201 100644
--- a/rust/helpers/scatterlist.c
+++ b/rust/helpers/scatterlist.c
@@ -2,23 +2,25 @@
 
 #include <linux/dma-direction.h>
 
-dma_addr_t rust_helper_sg_dma_address(struct scatterlist *sg)
+__rust_helper dma_addr_t rust_helper_sg_dma_address(struct scatterlist *sg)
 {
 	return sg_dma_address(sg);
 }
 
-unsigned int rust_helper_sg_dma_len(struct scatterlist *sg)
+__rust_helper unsigned int rust_helper_sg_dma_len(struct scatterlist *sg)
 {
 	return sg_dma_len(sg);
 }
 
-struct scatterlist *rust_helper_sg_next(struct scatterlist *sg)
+__rust_helper struct scatterlist *rust_helper_sg_next(struct scatterlist *sg)
 {
 	return sg_next(sg);
 }
 
-void rust_helper_dma_unmap_sgtable(struct device *dev, struct sg_table *sgt,
-				   enum dma_data_direction dir, unsigned long attrs)
+__rust_helper void rust_helper_dma_unmap_sgtable(struct device *dev,
+						 struct sg_table *sgt,
+						 enum dma_data_direction dir,
+						 unsigned long attrs)
 {
 	return dma_unmap_sgtable(dev, sgt, dir, attrs);
 }
diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs
index f968fbd22890..0879a79485f8 100644
--- a/rust/kernel/cpufreq.rs
+++ b/rust/kernel/cpufreq.rs
@@ -1015,6 +1015,8 @@ impl<T: Driver> Registration<T> {
         ..pin_init::zeroed()
     };
 
+    // Always inline to optimize out error path of `build_assert`.
+    #[inline(always)]
     const fn copy_name(name: &'static CStr) -> [c_char; CPUFREQ_NAME_LEN] {
         let src = name.to_bytes_with_nul();
         let mut dst = [0; CPUFREQ_NAME_LEN];
diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs
index a7f682e95c01..bdaac839dacc 100644
--- a/rust/kernel/drm/gem/mod.rs
+++ b/rust/kernel/drm/gem/mod.rs
@@ -253,7 +253,7 @@ extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) {
 }
 
 // SAFETY: Instances of `Object<T>` are always reference-counted.
-unsafe impl<T: DriverObject> crate::types::AlwaysRefCounted for Object<T> {
+unsafe impl<T: DriverObject> crate::sync::aref::AlwaysRefCounted for Object<T> {
     fn inc_ref(&self) {
         // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero.
         unsafe { bindings::drm_gem_object_get(self.as_raw()) };
@@ -293,9 +293,7 @@ impl<T: DriverObject> AllocImpl for Object<T> {
 }
 
 pub(super) const fn create_fops() -> bindings::file_operations {
-    // SAFETY: As by the type invariant, it is safe to initialize `bindings::file_operations`
-    // zeroed.
-    let mut fops: bindings::file_operations = unsafe { core::mem::zeroed() };
+    let mut fops: bindings::file_operations = pin_init::zeroed();
 
     fops.owner = core::ptr::null_mut();
     fops.open = Some(bindings::drm_open);
diff --git a/rust/kernel/maple_tree.rs b/rust/kernel/maple_tree.rs
index e72eec56bf57..265d6396a78a 100644
--- a/rust/kernel/maple_tree.rs
+++ b/rust/kernel/maple_tree.rs
@@ -265,7 +265,16 @@ unsafe fn free_all_entries(self: Pin<&mut Self>) {
         loop {
             // This uses the raw accessor because we're destroying pointers without removing them
             // from the maple tree, which is only valid because this is the destructor.
-            let ptr = ma_state.mas_find_raw(usize::MAX);
+            //
+            // Take the rcu lock because mas_find_raw() requires that you hold either the spinlock
+            // or the rcu read lock. This is only really required if memory reclaim might
+            // reallocate entries in the tree, as we otherwise have exclusive access. That feature
+            // doesn't exist yet, so for now, taking the rcu lock only serves the purpose of
+            // silencing lockdep.
+            let ptr = {
+                let _rcu = kernel::sync::rcu::Guard::new();
+                ma_state.mas_find_raw(usize::MAX)
+            };
             if ptr.is_null() {
                 break;
             }
diff --git a/rust/kernel/pwm.rs b/rust/kernel/pwm.rs
index cb00f8a8765c..2dd72a39acb5 100644
--- a/rust/kernel/pwm.rs
+++ b/rust/kernel/pwm.rs
@@ -13,9 +13,14 @@
     devres,
     error::{self, to_result},
     prelude::*,
-    types::{ARef, AlwaysRefCounted, Opaque}, //
+    sync::aref::{ARef, AlwaysRefCounted},
+    types::Opaque, //
+};
+use core::{
+    marker::PhantomData,
+    ops::Deref,
+    ptr::NonNull, //
 };
-use core::{marker::PhantomData, ptr::NonNull};
 
 /// Represents a PWM waveform configuration.
 /// Mirrors struct [`struct pwm_waveform`](srctree/include/linux/pwm.h).
@@ -173,7 +178,7 @@ pub struct RoundedWaveform<WfHw> {
 }
 
 /// Trait defining the operations for a PWM driver.
-pub trait PwmOps: 'static + Sized {
+pub trait PwmOps: 'static + Send + Sync + Sized {
     /// The driver-specific hardware representation of a waveform.
     ///
     /// This type must be [`Copy`], [`Default`], and fit within `PWM_WFHWSIZE`.
@@ -584,11 +589,12 @@ unsafe fn bound_parent_device(&self) -> &device::Device<Bound> {
     ///
     /// Returns an [`ARef<Chip>`] managing the chip's lifetime via refcounting
     /// on its embedded `struct device`.
-    pub fn new(
-        parent_dev: &device::Device,
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new<'a>(
+        parent_dev: &'a device::Device<Bound>,
         num_channels: u32,
         data: impl pin_init::PinInit<T, Error>,
-    ) -> Result<ARef<Self>> {
+    ) -> Result<UnregisteredChip<'a, T>> {
         let sizeof_priv = core::mem::size_of::<T>();
         // SAFETY: `pwmchip_alloc` allocates memory for the C struct and our private data.
         let c_chip_ptr_raw =
@@ -623,7 +629,9 @@ pub fn new(
         // SAFETY: `chip_ptr_as_self` points to a valid `Chip` (layout-compatible with
         // `bindings::pwm_chip`) whose embedded device has refcount 1.
         // `ARef::from_raw` takes this pointer and manages it via `AlwaysRefCounted`.
-        Ok(unsafe { ARef::from_raw(NonNull::new_unchecked(chip_ptr_as_self)) })
+        let chip = unsafe { ARef::from_raw(NonNull::new_unchecked(chip_ptr_as_self)) };
+
+        Ok(UnregisteredChip { chip, parent_dev })
     }
 }
 
@@ -654,37 +662,29 @@ unsafe fn dec_ref(obj: NonNull<Chip<T>>) {
 // structure's state is managed and synchronized by the kernel's device model
 // and PWM core locking mechanisms. Therefore, it is safe to move the `Chip`
 // wrapper (and the pointer it contains) across threads.
-unsafe impl<T: PwmOps + Send> Send for Chip<T> {}
+unsafe impl<T: PwmOps> Send for Chip<T> {}
 
 // SAFETY: It is safe for multiple threads to have shared access (`&Chip`) because
 // the `Chip` data is immutable from the Rust side without holding the appropriate
 // kernel locks, which the C core is responsible for. Any interior mutability is
 // handled and synchronized by the C kernel code.
-unsafe impl<T: PwmOps + Sync> Sync for Chip<T> {}
+unsafe impl<T: PwmOps> Sync for Chip<T> {}
 
-/// A resource guard that ensures `pwmchip_remove` is called on drop.
-///
-/// This struct is intended to be managed by the `devres` framework by transferring its ownership
-/// via [`devres::register`]. This ties the lifetime of the PWM chip registration
-/// to the lifetime of the underlying device.
-pub struct Registration<T: PwmOps> {
+/// A wrapper around `ARef<Chip<T>>` that ensures that `register` can only be called once.
+pub struct UnregisteredChip<'a, T: PwmOps> {
     chip: ARef<Chip<T>>,
+    parent_dev: &'a device::Device<Bound>,
 }
 
-impl<T: 'static + PwmOps + Send + Sync> Registration<T> {
+impl<T: PwmOps> UnregisteredChip<'_, T> {
     /// Registers a PWM chip with the PWM subsystem.
     ///
     /// Transfers its ownership to the `devres` framework, which ties its lifetime
     /// to the parent device.
     /// On unbind of the parent device, the `devres` entry will be dropped, automatically
     /// calling `pwmchip_remove`. This function should be called from the driver's `probe`.
-    pub fn register(dev: &device::Device<Bound>, chip: ARef<Chip<T>>) -> Result {
-        let chip_parent = chip.device().parent().ok_or(EINVAL)?;
-        if dev.as_raw() != chip_parent.as_raw() {
-            return Err(EINVAL);
-        }
-
-        let c_chip_ptr = chip.as_raw();
+    pub fn register(self) -> Result<ARef<Chip<T>>> {
+        let c_chip_ptr = self.chip.as_raw();
 
         // SAFETY: `c_chip_ptr` points to a valid chip with its ops initialized.
         // `__pwmchip_add` is the C function to register the chip with the PWM core.
@@ -692,12 +692,33 @@ pub fn register(dev: &device::Device<Bound>, chip: ARef<Chip<T>>) -> Result {
             to_result(bindings::__pwmchip_add(c_chip_ptr, core::ptr::null_mut()))?;
         }
 
-        let registration = Registration { chip };
+        let registration = Registration {
+            chip: ARef::clone(&self.chip),
+        };
+
+        devres::register(self.parent_dev, registration, GFP_KERNEL)?;
 
-        devres::register(dev, registration, GFP_KERNEL)
+        Ok(self.chip)
     }
 }
 
+impl<T: PwmOps> Deref for UnregisteredChip<'_, T> {
+    type Target = Chip<T>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.chip
+    }
+}
+
+/// A resource guard that ensures `pwmchip_remove` is called on drop.
+///
+/// This struct is intended to be managed by the `devres` framework by transferring its ownership
+/// via [`devres::register`]. This ties the lifetime of the PWM chip registration
+/// to the lifetime of the underlying device.
+struct Registration<T: PwmOps> {
+    chip: ARef<Chip<T>>,
+}
+
 impl<T: PwmOps> Drop for Registration<T> {
     fn drop(&mut self) {
         let chip_raw = self.chip.as_raw();
diff --git a/samples/rust/rust_driver_pci.rs b/samples/rust/rust_driver_pci.rs
index 5823787bea8e..fa677991a5c4 100644
--- a/samples/rust/rust_driver_pci.rs
+++ b/samples/rust/rust_driver_pci.rs
@@ -48,7 +48,7 @@ fn testdev(index: &TestIndex, bar: &Bar0) -> Result<u32> {
         // Select the test.
         bar.write8(index.0, Regs::TEST);
 
-        let offset = u32::from_le(bar.read32(Regs::OFFSET)) as usize;
+        let offset = bar.read32(Regs::OFFSET) as usize;
         let data = bar.read8(Regs::DATA);
 
         // Write `data` to `offset` to increase `count` by one.
diff --git a/scripts/Makefile.dtbs b/scripts/Makefile.dtbs
index e092b460d5a1..f5d51b8c2eeb 100644
--- a/scripts/Makefile.dtbs
+++ b/scripts/Makefile.dtbs
@@ -23,7 +23,11 @@ dtb-y           := $(addprefix $(obj)/, $(dtb-y))
 multi-dtb-y     := $(addprefix $(obj)/, $(multi-dtb-y))
 real-dtb-y      := $(addprefix $(obj)/, $(real-dtb-y))
 
+dtb-targets := %.dtb %.dtbo dtbs dtbs_check
+ifneq ($(findstring /dts/,$(MAKECMDGOALS))$(filter $(dtb-targets),$(MAKECMDGOALS)),)
 always-y        += $(dtb-y)
+endif
+
 targets         += $(real-dtb-y)
 
 # dtbs-list
diff --git a/scripts/crypto/gen-hash-testvecs.py b/scripts/crypto/gen-hash-testvecs.py
index c1d0517140bd..1e20ecb46314 100755
--- a/scripts/crypto/gen-hash-testvecs.py
+++ b/scripts/crypto/gen-hash-testvecs.py
@@ -184,6 +184,44 @@ def gen_additional_blake2_testvecs(alg):
             f'{alg}_keyed_testvec_consolidated[{alg_digest_size_const(alg)}]',
             compute_hash(alg, hashes))
 
+def nh_extract_int(bytestr, pos, length):
+    assert pos % 8 == 0 and length % 8 == 0
+    return int.from_bytes(bytestr[pos//8 : pos//8 + length//8], byteorder='little')
+
+# The NH "almost-universal hash function" used in Adiantum.  This is a
+# straightforward translation of the pseudocode from Section 6.3 of the Adiantum
+# paper (https://eprint.iacr.org/2018/720.pdf), except the outer loop is omitted
+# because we assume len(msg) <= 1024.  (The kernel's nh() function is only
+# expected to handle up to 1024 bytes; it's just called repeatedly as needed.)
+def nh(key, msg):
+    (w, s, r, u) = (32, 2, 4, 8192)
+    l = 8 * len(msg)
+    assert l <= u
+    assert l % (2*s*w) == 0
+    h = bytes()
+    for i in range(0, 2*s*w*r, 2*s*w):
+        p = 0
+        for j in range(0, l, 2*s*w):
+            for k in range(0, w*s, w):
+                a0 = nh_extract_int(key, i + j + k, w)
+                a1 = nh_extract_int(key, i + j + k + s*w, w)
+                b0 = nh_extract_int(msg, j + k, w)
+                b1 = nh_extract_int(msg, j + k + s*w, w)
+                p += ((a0 + b0) % 2**w) * ((a1 + b1) % 2**w)
+        h += (p % 2**64).to_bytes(8, byteorder='little')
+    return h
+
+def gen_nh_testvecs():
+    NH_KEY_BYTES = 1072
+    NH_MESSAGE_BYTES = 1024
+    key = rand_bytes(NH_KEY_BYTES)
+    msg = rand_bytes(NH_MESSAGE_BYTES)
+    print_static_u8_array_definition('nh_test_key[NH_KEY_BYTES]', key)
+    print_static_u8_array_definition('nh_test_msg[NH_MESSAGE_BYTES]', msg)
+    for length in [16, 96, 256, 1024]:
+        print_static_u8_array_definition(f'nh_test_val{length}[NH_HASH_BYTES]',
+                                         nh(key, msg[:length]))
+
 def gen_additional_poly1305_testvecs():
     key = b'\xff' * POLY1305_KEY_SIZE
     data = b''
@@ -217,6 +255,8 @@ print(f'/* This file was generated by: {sys.argv[0]} {" ".join(sys.argv[1:])} */
 if alg.startswith('blake2'):
     gen_unkeyed_testvecs(alg)
     gen_additional_blake2_testvecs(alg)
+elif alg == 'nh':
+    gen_nh_testvecs()
 elif alg == 'poly1305':
     gen_unkeyed_testvecs(alg)
     gen_additional_poly1305_testvecs()
diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c
index 755b842f1f9b..d042b355c9d6 100644
--- a/scripts/mod/modpost.c
+++ b/scripts/mod/modpost.c
@@ -958,7 +958,7 @@ static int secref_whitelist(const char *fromsec, const char *fromsym,
 	/* symbols in data sections that may refer to any init/exit sections */
 	if (match(fromsec, PATTERNS(DATA_SECTIONS)) &&
 	    match(tosec, PATTERNS(ALL_INIT_SECTIONS, ALL_EXIT_SECTIONS)) &&
-	    match(fromsym, PATTERNS("*_ops", "*_probe", "*_console")))
+	    match(fromsym, PATTERNS("*_ops", "*_console")))
 		return 0;
 
 	/* Check for pattern 3 */
diff --git a/security/integrity/evm/evm_crypto.c b/security/integrity/evm/evm_crypto.c
index a5e730ffda57..465a32f59c11 100644
--- a/security/integrity/evm/evm_crypto.c
+++ b/security/integrity/evm/evm_crypto.c
@@ -13,6 +13,7 @@
 #define pr_fmt(fmt) "EVM: "fmt
 
 #include <linux/export.h>
+#include <linux/hex.h>
 #include <linux/crypto.h>
 #include <linux/xattr.h>
 #include <linux/evm.h>
diff --git a/security/integrity/ima/ima_api.c b/security/integrity/ima/ima_api.c
index c35ea613c9f8..c6d1c7be8a3e 100644
--- a/security/integrity/ima/ima_api.c
+++ b/security/integrity/ima/ima_api.c
@@ -11,6 +11,7 @@
 #include <linux/slab.h>
 #include <linux/file.h>
 #include <linux/fs.h>
+#include <linux/hex.h>
 #include <linux/xattr.h>
 #include <linux/evm.h>
 #include <linux/fsverity.h>
diff --git a/security/integrity/ima/ima_kexec.c b/security/integrity/ima/ima_kexec.c
index 7362f68f2d8b..5beb69edd12f 100644
--- a/security/integrity/ima/ima_kexec.c
+++ b/security/integrity/ima/ima_kexec.c
@@ -250,9 +250,7 @@ void ima_kexec_post_load(struct kimage *image)
 	if (!image->ima_buffer_addr)
 		return;
 
-	ima_kexec_buffer = kimage_map_segment(image,
-					      image->ima_buffer_addr,
-					      image->ima_buffer_size);
+	ima_kexec_buffer = kimage_map_segment(image, image->ima_segment_index);
 	if (!ima_kexec_buffer) {
 		pr_err("Could not map measurements buffer.\n");
 		return;
diff --git a/security/ipe/digest.c b/security/ipe/digest.c
index 493716370570..5006366837ba 100644
--- a/security/ipe/digest.c
+++ b/security/ipe/digest.c
@@ -3,6 +3,7 @@
  * Copyright (C) 2020-2024 Microsoft Corporation. All rights reserved.
  */
 
+#include <linux/hex.h>
 #include "digest.h"
 
 /**
diff --git a/security/keys/encrypted-keys/encrypted.c b/security/keys/encrypted-keys/encrypted.c
index 596e7a30bd3c..56b531587a1e 100644
--- a/security/keys/encrypted-keys/encrypted.c
+++ b/security/keys/encrypted-keys/encrypted.c
@@ -13,6 +13,7 @@
 
 #include <linux/uaccess.h>
 #include <linux/module.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/parser.h>
diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c
index b1680ee53f86..16168ba5c83e 100644
--- a/security/keys/trusted-keys/trusted_core.c
+++ b/security/keys/trusted-keys/trusted_core.c
@@ -14,6 +14,7 @@
 #include <keys/trusted_tpm.h>
 #include <linux/capability.h>
 #include <linux/err.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/key-type.h>
 #include <linux/module.h>
diff --git a/security/keys/trusted-keys/trusted_tpm1.c b/security/keys/trusted-keys/trusted_tpm1.c
index 636acb66a4f6..c865c97aa1b4 100644
--- a/security/keys/trusted-keys/trusted_tpm1.c
+++ b/security/keys/trusted-keys/trusted_tpm1.c
@@ -9,6 +9,7 @@
 #include <crypto/hash_info.h>
 #include <crypto/sha1.h>
 #include <crypto/utils.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/parser.h>
diff --git a/security/landlock/audit.c b/security/landlock/audit.c
index c52d079cdb77..e899995f1fd5 100644
--- a/security/landlock/audit.c
+++ b/security/landlock/audit.c
@@ -191,7 +191,7 @@ static size_t get_denied_layer(const struct landlock_ruleset *const domain,
 	long youngest_layer = -1;
 
 	for_each_set_bit(access_bit, &access_req, layer_masks_size) {
-		const access_mask_t mask = (*layer_masks)[access_bit];
+		const layer_mask_t mask = (*layer_masks)[access_bit];
 		long layer;
 
 		if (!mask)
diff --git a/security/loadpin/loadpin.c b/security/loadpin/loadpin.c
index 273ffbd6defe..019840006096 100644
--- a/security/loadpin/loadpin.c
+++ b/security/loadpin/loadpin.c
@@ -11,6 +11,7 @@
 
 #include <linux/module.h>
 #include <linux/fs.h>
+#include <linux/hex.h>
 #include <linux/kernel_read_file.h>
 #include <linux/lsm_hooks.h>
 #include <linux/mount.h>
diff --git a/security/selinux/selinuxfs.c b/security/selinux/selinuxfs.c
index 896acad1f5f7..4d58c7ad1a23 100644
--- a/security/selinux/selinuxfs.c
+++ b/security/selinux/selinuxfs.c
@@ -18,6 +18,7 @@
 #include <linux/vmalloc.h>
 #include <linux/fs.h>
 #include <linux/fs_context.h>
+#include <linux/hex.h>
 #include <linux/mount.h>
 #include <linux/mutex.h>
 #include <linux/namei.h>
diff --git a/sound/core/compress_offload.c b/sound/core/compress_offload.c
index da514fef45bc..ed2eeb914c6d 100644
--- a/sound/core/compress_offload.c
+++ b/sound/core/compress_offload.c
@@ -514,12 +514,12 @@ static int
 snd_compr_get_codec_caps(struct snd_compr_stream *stream, unsigned long arg)
 {
 	int retval;
-	struct snd_compr_codec_caps *caps __free(kfree) = NULL;
 
 	if (!stream->ops->get_codec_caps)
 		return -ENXIO;
 
-	caps = kzalloc(sizeof(*caps), GFP_KERNEL);
+	struct snd_compr_codec_caps *caps __free(kfree) =
+		kzalloc(sizeof(*caps), GFP_KERNEL);
 	if (!caps)
 		return -ENOMEM;
 
@@ -647,7 +647,6 @@ snd_compress_check_input(struct snd_compr_stream *stream, struct snd_compr_param
 static int
 snd_compr_set_params(struct snd_compr_stream *stream, unsigned long arg)
 {
-	struct snd_compr_params *params __free(kfree) = NULL;
 	int retval;
 
 	if (stream->runtime->state == SNDRV_PCM_STATE_OPEN || stream->next_track) {
@@ -655,7 +654,9 @@ snd_compr_set_params(struct snd_compr_stream *stream, unsigned long arg)
 		 * we should allow parameter change only when stream has been
 		 * opened not in other cases
 		 */
-		params = memdup_user((void __user *)arg, sizeof(*params));
+		struct snd_compr_params *params __free(kfree) =
+			memdup_user((void __user *)arg, sizeof(*params));
+
 		if (IS_ERR(params))
 			return PTR_ERR(params);
 
@@ -687,13 +688,13 @@ snd_compr_set_params(struct snd_compr_stream *stream, unsigned long arg)
 static int
 snd_compr_get_params(struct snd_compr_stream *stream, unsigned long arg)
 {
-	struct snd_codec *params __free(kfree) = NULL;
 	int retval;
 
 	if (!stream->ops->get_params)
 		return -EBADFD;
 
-	params = kzalloc(sizeof(*params), GFP_KERNEL);
+	struct snd_codec *params __free(kfree) =
+		kzalloc(sizeof(*params), GFP_KERNEL);
 	if (!params)
 		return -ENOMEM;
 	retval = stream->ops->get_params(stream, params);
@@ -1104,12 +1105,13 @@ static int snd_compr_task_new(struct snd_compr_stream *stream, struct snd_compr_
 
 static int snd_compr_task_create(struct snd_compr_stream *stream, unsigned long arg)
 {
-	struct snd_compr_task *task __free(kfree) = NULL;
 	int retval;
 
 	if (stream->runtime->state != SNDRV_PCM_STATE_SETUP)
 		return -EPERM;
-	task = memdup_user((void __user *)arg, sizeof(*task));
+
+	struct snd_compr_task *task __free(kfree) =
+		memdup_user((void __user *)arg, sizeof(*task));
 	if (IS_ERR(task))
 		return PTR_ERR(task);
 	retval = snd_compr_task_new(stream, task);
@@ -1165,12 +1167,13 @@ static int snd_compr_task_start(struct snd_compr_stream *stream, struct snd_comp
 
 static int snd_compr_task_start_ioctl(struct snd_compr_stream *stream, unsigned long arg)
 {
-	struct snd_compr_task *task __free(kfree) = NULL;
 	int retval;
 
 	if (stream->runtime->state != SNDRV_PCM_STATE_SETUP)
 		return -EPERM;
-	task = memdup_user((void __user *)arg, sizeof(*task));
+
+	struct snd_compr_task *task __free(kfree) =
+		memdup_user((void __user *)arg, sizeof(*task));
 	if (IS_ERR(task))
 		return PTR_ERR(task);
 	retval = snd_compr_task_start(stream, task);
@@ -1256,12 +1259,13 @@ static int snd_compr_task_status(struct snd_compr_stream *stream,
 
 static int snd_compr_task_status_ioctl(struct snd_compr_stream *stream, unsigned long arg)
 {
-	struct snd_compr_task_status *status __free(kfree) = NULL;
 	int retval;
 
 	if (stream->runtime->state != SNDRV_PCM_STATE_SETUP)
 		return -EPERM;
-	status = memdup_user((void __user *)arg, sizeof(*status));
+
+	struct snd_compr_task_status *status __free(kfree) =
+		memdup_user((void __user *)arg, sizeof(*status));
 	if (IS_ERR(status))
 		return PTR_ERR(status);
 	retval = snd_compr_task_status(stream, status);
diff --git a/sound/core/control.c b/sound/core/control.c
index 9c3fd5113a61..486d1bc4dac2 100644
--- a/sound/core/control.c
+++ b/sound/core/control.c
@@ -867,9 +867,9 @@ EXPORT_SYMBOL(snd_ctl_find_id);
 static int snd_ctl_card_info(struct snd_card *card, struct snd_ctl_file * ctl,
 			     unsigned int cmd, void __user *arg)
 {
-	struct snd_ctl_card_info *info __free(kfree) = NULL;
+	struct snd_ctl_card_info *info __free(kfree) =
+		kzalloc(sizeof(*info), GFP_KERNEL);
 
-	info = kzalloc(sizeof(*info), GFP_KERNEL);
 	if (! info)
 		return -ENOMEM;
 	scoped_guard(rwsem_read, &snd_ioctl_rwsem) {
@@ -1244,10 +1244,10 @@ static int snd_ctl_elem_read(struct snd_card *card,
 static int snd_ctl_elem_read_user(struct snd_card *card,
 				  struct snd_ctl_elem_value __user *_control)
 {
-	struct snd_ctl_elem_value *control __free(kfree) = NULL;
 	int result;
+	struct snd_ctl_elem_value *control __free(kfree) =
+		memdup_user(_control, sizeof(*control));
 
-	control = memdup_user(_control, sizeof(*control));
 	if (IS_ERR(control))
 		return PTR_ERR(control);
 
@@ -1320,11 +1320,11 @@ static int snd_ctl_elem_write(struct snd_card *card, struct snd_ctl_file *file,
 static int snd_ctl_elem_write_user(struct snd_ctl_file *file,
 				   struct snd_ctl_elem_value __user *_control)
 {
-	struct snd_ctl_elem_value *control __free(kfree) = NULL;
 	struct snd_card *card;
 	int result;
+	struct snd_ctl_elem_value *control __free(kfree) =
+		memdup_user(_control, sizeof(*control));
 
-	control = memdup_user(_control, sizeof(*control));
 	if (IS_ERR(control))
 		return PTR_ERR(control);
 
diff --git a/sound/core/control_compat.c b/sound/core/control_compat.c
index 6459809ed364..b8988a4bcd9b 100644
--- a/sound/core/control_compat.c
+++ b/sound/core/control_compat.c
@@ -80,10 +80,10 @@ static int snd_ctl_elem_info_compat(struct snd_ctl_file *ctl,
 				    struct snd_ctl_elem_info32 __user *data32)
 {
 	struct snd_card *card = ctl->card;
-	struct snd_ctl_elem_info *data __free(kfree) = NULL;
 	int err;
+	struct snd_ctl_elem_info *data __free(kfree) =
+		kzalloc(sizeof(*data), GFP_KERNEL);
 
-	data = kzalloc(sizeof(*data), GFP_KERNEL);
 	if (! data)
 		return -ENOMEM;
 
@@ -169,14 +169,15 @@ static int get_ctl_type(struct snd_card *card, struct snd_ctl_elem_id *id,
 			int *countp)
 {
 	struct snd_kcontrol *kctl;
-	struct snd_ctl_elem_info *info __free(kfree) = NULL;
 	int err;
 
 	guard(rwsem_read)(&card->controls_rwsem);
 	kctl = snd_ctl_find_id(card, id);
 	if (!kctl)
 		return -ENOENT;
-	info = kzalloc(sizeof(*info), GFP_KERNEL);
+
+	struct snd_ctl_elem_info *info __free(kfree) =
+		kzalloc(sizeof(*info), GFP_KERNEL);
 	if (info == NULL)
 		return -ENOMEM;
 	info->id = *id;
@@ -280,10 +281,10 @@ static int copy_ctl_value_to_user(void __user *userdata,
 static int __ctl_elem_read_user(struct snd_card *card,
 				void __user *userdata, void __user *valuep)
 {
-	struct snd_ctl_elem_value *data __free(kfree) = NULL;
 	int err, type, count;
+	struct snd_ctl_elem_value *data __free(kfree) =
+		kzalloc(sizeof(*data), GFP_KERNEL);
 
-	data = kzalloc(sizeof(*data), GFP_KERNEL);
 	if (data == NULL)
 		return -ENOMEM;
 
@@ -314,11 +315,11 @@ static int ctl_elem_read_user(struct snd_card *card,
 static int __ctl_elem_write_user(struct snd_ctl_file *file,
 				 void __user *userdata, void __user *valuep)
 {
-	struct snd_ctl_elem_value *data __free(kfree) = NULL;
 	struct snd_card *card = file->card;
 	int err, type, count;
+	struct snd_ctl_elem_value *data __free(kfree) =
+		kzalloc(sizeof(*data), GFP_KERNEL);
 
-	data = kzalloc(sizeof(*data), GFP_KERNEL);
 	if (data == NULL)
 		return -ENOMEM;
 
@@ -378,9 +379,9 @@ static int snd_ctl_elem_add_compat(struct snd_ctl_file *file,
 				   struct snd_ctl_elem_info32 __user *data32,
 				   int replace)
 {
-	struct snd_ctl_elem_info *data __free(kfree) = NULL;
+	struct snd_ctl_elem_info *data __free(kfree) =
+		kzalloc(sizeof(*data), GFP_KERNEL);
 
-	data = kzalloc(sizeof(*data), GFP_KERNEL);
 	if (! data)
 		return -ENOMEM;
 
diff --git a/sound/core/control_led.c b/sound/core/control_led.c
index e33dfcf863cf..c7641d5084e7 100644
--- a/sound/core/control_led.c
+++ b/sound/core/control_led.c
@@ -245,12 +245,12 @@ DEFINE_FREE(snd_card_unref, struct snd_card *, if (_T) snd_card_unref(_T))
 static int snd_ctl_led_set_id(int card_number, struct snd_ctl_elem_id *id,
 			      unsigned int group, bool set)
 {
-	struct snd_card *card __free(snd_card_unref) = NULL;
 	struct snd_kcontrol *kctl;
 	struct snd_kcontrol_volatile *vd;
 	unsigned int ioff, access, new_access;
+	struct snd_card *card __free(snd_card_unref) =
+		snd_card_ref(card_number);
 
-	card = snd_card_ref(card_number);
 	if (!card)
 		return -ENXIO;
 	guard(rwsem_write)(&card->controls_rwsem);
@@ -302,13 +302,13 @@ static void snd_ctl_led_clean(struct snd_card *card)
 
 static int snd_ctl_led_reset(int card_number, unsigned int group)
 {
-	struct snd_card *card __free(snd_card_unref) = NULL;
 	struct snd_ctl_led_ctl *lctl, *_lctl;
 	struct snd_ctl_led *led;
 	struct snd_kcontrol_volatile *vd;
 	bool change = false;
+	struct snd_card *card __free(snd_card_unref) =
+		snd_card_ref(card_number);
 
-	card = snd_card_ref(card_number);
 	if (!card)
 		return -ENXIO;
 
@@ -598,11 +598,11 @@ static ssize_t list_show(struct device *dev,
 			 struct device_attribute *attr, char *buf)
 {
 	struct snd_ctl_led_card *led_card = container_of(dev, struct snd_ctl_led_card, dev);
-	struct snd_card *card __free(snd_card_unref) = NULL;
 	struct snd_ctl_led_ctl *lctl;
 	size_t l = 0;
+	struct snd_card *card __free(snd_card_unref) =
+		snd_card_ref(led_card->number);
 
-	card = snd_card_ref(led_card->number);
 	if (!card)
 		return -ENXIO;
 	guard(rwsem_read)(&card->controls_rwsem);
diff --git a/sound/core/oss/mixer_oss.c b/sound/core/oss/mixer_oss.c
index e839a4bb93f8..69422ab2d808 100644
--- a/sound/core/oss/mixer_oss.c
+++ b/sound/core/oss/mixer_oss.c
@@ -517,8 +517,6 @@ static void snd_mixer_oss_get_volume1_vol(struct snd_mixer_oss_file *fmixer,
 					  unsigned int numid,
 					  int *left, int *right)
 {
-	struct snd_ctl_elem_info *uinfo __free(kfree) = NULL;
-	struct snd_ctl_elem_value *uctl __free(kfree) = NULL;
 	struct snd_kcontrol *kctl;
 	struct snd_card *card = fmixer->card;
 
@@ -528,8 +526,11 @@ static void snd_mixer_oss_get_volume1_vol(struct snd_mixer_oss_file *fmixer,
 	kctl = snd_ctl_find_numid(card, numid);
 	if (!kctl)
 		return;
-	uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL);
-	uctl = kzalloc(sizeof(*uctl), GFP_KERNEL);
+
+	struct snd_ctl_elem_info *uinfo __free(kfree) =
+		kzalloc(sizeof(*uinfo), GFP_KERNEL);
+	struct snd_ctl_elem_value *uctl __free(kfree) =
+		kzalloc(sizeof(*uctl), GFP_KERNEL);
 	if (uinfo == NULL || uctl == NULL)
 		return;
 	if (kctl->info(kctl, uinfo))
@@ -550,8 +551,6 @@ static void snd_mixer_oss_get_volume1_sw(struct snd_mixer_oss_file *fmixer,
 					 int *left, int *right,
 					 int route)
 {
-	struct snd_ctl_elem_info *uinfo __free(kfree) = NULL;
-	struct snd_ctl_elem_value *uctl __free(kfree) = NULL;
 	struct snd_kcontrol *kctl;
 	struct snd_card *card = fmixer->card;
 
@@ -561,8 +560,11 @@ static void snd_mixer_oss_get_volume1_sw(struct snd_mixer_oss_file *fmixer,
 	kctl = snd_ctl_find_numid(card, numid);
 	if (!kctl)
 		return;
-	uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL);
-	uctl = kzalloc(sizeof(*uctl), GFP_KERNEL);
+
+	struct snd_ctl_elem_info *uinfo __free(kfree) =
+		kzalloc(sizeof(*uinfo), GFP_KERNEL);
+	struct snd_ctl_elem_value *uctl __free(kfree) =
+		kzalloc(sizeof(*uctl), GFP_KERNEL);
 	if (uinfo == NULL || uctl == NULL)
 		return;
 	if (kctl->info(kctl, uinfo))
@@ -609,8 +611,6 @@ static void snd_mixer_oss_put_volume1_vol(struct snd_mixer_oss_file *fmixer,
 					  unsigned int numid,
 					  int left, int right)
 {
-	struct snd_ctl_elem_info *uinfo __free(kfree) = NULL;
-	struct snd_ctl_elem_value *uctl __free(kfree) = NULL;
 	struct snd_kcontrol *kctl;
 	struct snd_card *card = fmixer->card;
 	int res;
@@ -621,8 +621,11 @@ static void snd_mixer_oss_put_volume1_vol(struct snd_mixer_oss_file *fmixer,
 	kctl = snd_ctl_find_numid(card, numid);
 	if (!kctl)
 		return;
-	uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL);
-	uctl = kzalloc(sizeof(*uctl), GFP_KERNEL);
+
+	struct snd_ctl_elem_info *uinfo __free(kfree) =
+		kzalloc(sizeof(*uinfo), GFP_KERNEL);
+	struct snd_ctl_elem_value *uctl __free(kfree) =
+		kzalloc(sizeof(*uctl), GFP_KERNEL);
 	if (uinfo == NULL || uctl == NULL)
 		return;
 	if (kctl->info(kctl, uinfo))
@@ -646,8 +649,6 @@ static void snd_mixer_oss_put_volume1_sw(struct snd_mixer_oss_file *fmixer,
 					 int left, int right,
 					 int route)
 {
-	struct snd_ctl_elem_info *uinfo __free(kfree) = NULL;
-	struct snd_ctl_elem_value *uctl __free(kfree) = NULL;
 	struct snd_kcontrol *kctl;
 	struct snd_card *card = fmixer->card;
 	int res;
@@ -658,8 +659,11 @@ static void snd_mixer_oss_put_volume1_sw(struct snd_mixer_oss_file *fmixer,
 	kctl = snd_ctl_find_numid(card, numid);
 	if (!kctl)
 		return;
-	uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL);
-	uctl = kzalloc(sizeof(*uctl), GFP_KERNEL);
+
+	struct snd_ctl_elem_info *uinfo __free(kfree) =
+		kzalloc(sizeof(*uinfo), GFP_KERNEL);
+	struct snd_ctl_elem_value *uctl __free(kfree) =
+		kzalloc(sizeof(*uctl), GFP_KERNEL);
 	if (uinfo == NULL || uctl == NULL)
 		return;
 	if (kctl->info(kctl, uinfo))
@@ -783,12 +787,12 @@ static int snd_mixer_oss_get_recsrc2(struct snd_mixer_oss_file *fmixer, unsigned
 	struct snd_kcontrol *kctl;
 	struct snd_mixer_oss_slot *pslot;
 	struct slot *slot;
-	struct snd_ctl_elem_info *uinfo __free(kfree) = NULL;
-	struct snd_ctl_elem_value *uctl __free(kfree) = NULL;
 	int err, idx;
 	
-	uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL);
-	uctl = kzalloc(sizeof(*uctl), GFP_KERNEL);
+	struct snd_ctl_elem_info *uinfo __free(kfree) =
+		kzalloc(sizeof(*uinfo), GFP_KERNEL);
+	struct snd_ctl_elem_value *uctl __free(kfree) =
+		uctl = kzalloc(sizeof(*uctl), GFP_KERNEL);
 	if (uinfo == NULL || uctl == NULL)
 		return -ENOMEM;
 	guard(rwsem_read)(&card->controls_rwsem);
@@ -825,13 +829,13 @@ static int snd_mixer_oss_put_recsrc2(struct snd_mixer_oss_file *fmixer, unsigned
 	struct snd_kcontrol *kctl;
 	struct snd_mixer_oss_slot *pslot;
 	struct slot *slot = NULL;
-	struct snd_ctl_elem_info *uinfo __free(kfree) = NULL;
-	struct snd_ctl_elem_value *uctl __free(kfree) = NULL;
 	int err;
 	unsigned int idx;
 
-	uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL);
-	uctl = kzalloc(sizeof(*uctl), GFP_KERNEL);
+	struct snd_ctl_elem_info *uinfo __free(kfree) =
+		kzalloc(sizeof(*uinfo), GFP_KERNEL);
+	struct snd_ctl_elem_value *uctl __free(kfree) =
+		kzalloc(sizeof(*uctl), GFP_KERNEL);
 	if (uinfo == NULL || uctl == NULL)
 		return -ENOMEM;
 	guard(rwsem_read)(&card->controls_rwsem);
@@ -872,18 +876,18 @@ struct snd_mixer_oss_assign_table {
 
 static int snd_mixer_oss_build_test(struct snd_mixer_oss *mixer, struct slot *slot, const char *name, int index, int item)
 {
-	struct snd_ctl_elem_info *info __free(kfree) = NULL;
 	struct snd_kcontrol *kcontrol;
 	struct snd_card *card = mixer->card;
 	int err;
 
+	struct snd_ctl_elem_info *info __free(kfree) =
+		kmalloc(sizeof(*info), GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
 	scoped_guard(rwsem_read, &card->controls_rwsem) {
 		kcontrol = snd_mixer_oss_test_id(mixer, name, index);
 		if (kcontrol == NULL)
 			return 0;
-		info = kmalloc(sizeof(*info), GFP_KERNEL);
-		if (!info)
-			return -ENOMEM;
 		err = kcontrol->info(kcontrol, info);
 		if (err < 0)
 			return err;
@@ -1006,9 +1010,9 @@ static int snd_mixer_oss_build_input(struct snd_mixer_oss *mixer,
 	if (!ptr->index)
 		kctl = snd_mixer_oss_test_id(mixer, "Capture Source", 0);
 	if (kctl) {
-		struct snd_ctl_elem_info *uinfo __free(kfree) = NULL;
+		struct snd_ctl_elem_info *uinfo __free(kfree) =
+			kzalloc(sizeof(*uinfo), GFP_KERNEL);
 
-		uinfo = kzalloc(sizeof(*uinfo), GFP_KERNEL);
 		if (!uinfo)
 			return -ENOMEM;
 			
diff --git a/sound/core/oss/pcm_oss.c b/sound/core/oss/pcm_oss.c
index a82dd155e1d3..1dffa8fdf429 100644
--- a/sound/core/oss/pcm_oss.c
+++ b/sound/core/oss/pcm_oss.c
@@ -377,7 +377,6 @@ static int snd_pcm_hw_param_near(struct snd_pcm_substream *pcm,
 				 snd_pcm_hw_param_t var, unsigned int best,
 				 int *dir)
 {
-	struct snd_pcm_hw_params *save __free(kfree) = NULL;
 	int v;
 	unsigned int saved_min;
 	int last = 0;
@@ -397,19 +396,22 @@ static int snd_pcm_hw_param_near(struct snd_pcm_substream *pcm,
 		maxdir = 1;
 		max--;
 	}
-	save = kmalloc(sizeof(*save), GFP_KERNEL);
+
+	struct snd_pcm_hw_params *save __free(kfree) =
+		kmalloc(sizeof(*save), GFP_KERNEL);
 	if (save == NULL)
 		return -ENOMEM;
 	*save = *params;
 	saved_min = min;
 	min = snd_pcm_hw_param_min(pcm, params, var, min, &mindir);
 	if (min >= 0) {
-		struct snd_pcm_hw_params *params1 __free(kfree) = NULL;
 		if (max < 0)
 			goto _end;
 		if ((unsigned int)min == saved_min && mindir == valdir)
 			goto _end;
-		params1 = kmalloc(sizeof(*params1), GFP_KERNEL);
+
+		struct snd_pcm_hw_params *params1 __free(kfree) =
+			kmalloc(sizeof(*params1), GFP_KERNEL);
 		if (params1 == NULL)
 			return -ENOMEM;
 		*params1 = *save;
@@ -781,10 +783,10 @@ static int choose_rate(struct snd_pcm_substream *substream,
 		       struct snd_pcm_hw_params *params, unsigned int best_rate)
 {
 	const struct snd_interval *it;
-	struct snd_pcm_hw_params *save __free(kfree) = NULL;
 	unsigned int rate, prev;
 
-	save = kmalloc(sizeof(*save), GFP_KERNEL);
+	struct snd_pcm_hw_params *save __free(kfree) =
+		kmalloc(sizeof(*save), GFP_KERNEL);
 	if (save == NULL)
 		return -ENOMEM;
 	*save = *params;
@@ -1834,7 +1836,6 @@ static int snd_pcm_oss_get_formats(struct snd_pcm_oss_file *pcm_oss_file)
 	struct snd_pcm_substream *substream;
 	int err;
 	int direct;
-	struct snd_pcm_hw_params *params __free(kfree) = NULL;
 	unsigned int formats = 0;
 	const struct snd_mask *format_mask;
 	int fmt;
@@ -1854,7 +1855,9 @@ static int snd_pcm_oss_get_formats(struct snd_pcm_oss_file *pcm_oss_file)
 			AFMT_S32_LE | AFMT_S32_BE |
 			AFMT_S24_LE | AFMT_S24_BE |
 			AFMT_S24_PACKED;
-	params = kmalloc(sizeof(*params), GFP_KERNEL);
+
+	struct snd_pcm_hw_params *params __free(kfree) =
+		kmalloc(sizeof(*params), GFP_KERNEL);
 	if (!params)
 		return -ENOMEM;
 	_snd_pcm_hw_params_any(params);
diff --git a/sound/core/pcm.c b/sound/core/pcm.c
index 283aac441fa0..0b512085eb63 100644
--- a/sound/core/pcm.c
+++ b/sound/core/pcm.c
@@ -328,13 +328,13 @@ static const char *snd_pcm_oss_format_name(int format)
 static void snd_pcm_proc_info_read(struct snd_pcm_substream *substream,
 				   struct snd_info_buffer *buffer)
 {
-	struct snd_pcm_info *info __free(kfree) = NULL;
 	int err;
 
 	if (! substream)
 		return;
 
-	info = kmalloc(sizeof(*info), GFP_KERNEL);
+	struct snd_pcm_info *info __free(kfree) =
+		kmalloc(sizeof(*info), GFP_KERNEL);
 	if (!info)
 		return;
 
diff --git a/sound/core/pcm_compat.c b/sound/core/pcm_compat.c
index 54eb9bd8eb21..e86f68f1f23c 100644
--- a/sound/core/pcm_compat.c
+++ b/sound/core/pcm_compat.c
@@ -235,7 +235,6 @@ static int snd_pcm_ioctl_hw_params_compat(struct snd_pcm_substream *substream,
 					  int refine, 
 					  struct snd_pcm_hw_params32 __user *data32)
 {
-	struct snd_pcm_hw_params *data __free(kfree) = NULL;
 	struct snd_pcm_runtime *runtime;
 	int err;
 
@@ -243,7 +242,8 @@ static int snd_pcm_ioctl_hw_params_compat(struct snd_pcm_substream *substream,
 	if (!runtime)
 		return -ENOTTY;
 
-	data = kmalloc(sizeof(*data), GFP_KERNEL);
+	struct snd_pcm_hw_params *data __free(kfree) =
+		kmalloc(sizeof(*data), GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 
@@ -332,7 +332,6 @@ static int snd_pcm_ioctl_xfern_compat(struct snd_pcm_substream *substream,
 	compat_caddr_t buf;
 	compat_caddr_t __user *bufptr;
 	u32 frames;
-	void __user **bufs __free(kfree) = NULL;
 	int err, ch, i;
 
 	if (! substream->runtime)
@@ -349,7 +348,9 @@ static int snd_pcm_ioctl_xfern_compat(struct snd_pcm_substream *substream,
 	    get_user(frames, &data32->frames))
 		return -EFAULT;
 	bufptr = compat_ptr(buf);
-	bufs = kmalloc_array(ch, sizeof(void __user *), GFP_KERNEL);
+
+	void __user **bufs __free(kfree) =
+		kmalloc_array(ch, sizeof(void __user *), GFP_KERNEL);
 	if (bufs == NULL)
 		return -ENOMEM;
 	for (i = 0; i < ch; i++) {
diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c
index 68bee40c9ada..4352c0a40f8d 100644
--- a/sound/core/pcm_native.c
+++ b/sound/core/pcm_native.c
@@ -242,10 +242,10 @@ int snd_pcm_info(struct snd_pcm_substream *substream, struct snd_pcm_info *info)
 int snd_pcm_info_user(struct snd_pcm_substream *substream,
 		      struct snd_pcm_info __user * _info)
 {
-	struct snd_pcm_info *info __free(kfree) = NULL;
 	int err;
+	struct snd_pcm_info *info __free(kfree) =
+		kmalloc(sizeof(*info), GFP_KERNEL);
 
-	info = kmalloc(sizeof(*info), GFP_KERNEL);
 	if (! info)
 		return -ENOMEM;
 	err = snd_pcm_info(substream, info);
@@ -364,7 +364,6 @@ static int constrain_params_by_rules(struct snd_pcm_substream *substream,
 	struct snd_pcm_hw_constraints *constrs =
 					&substream->runtime->hw_constraints;
 	unsigned int k;
-	unsigned int *rstamps __free(kfree) = NULL;
 	unsigned int vstamps[SNDRV_PCM_HW_PARAM_LAST_INTERVAL + 1];
 	unsigned int stamp;
 	struct snd_pcm_hw_rule *r;
@@ -380,7 +379,8 @@ static int constrain_params_by_rules(struct snd_pcm_substream *substream,
 	 * Each member of 'rstamps' array represents the sequence number of
 	 * recent application of corresponding rule.
 	 */
-	rstamps = kcalloc(constrs->rules_num, sizeof(unsigned int), GFP_KERNEL);
+	unsigned int *rstamps __free(kfree) =
+		kcalloc(constrs->rules_num, sizeof(unsigned int), GFP_KERNEL);
 	if (!rstamps)
 		return -ENOMEM;
 
@@ -583,10 +583,10 @@ EXPORT_SYMBOL(snd_pcm_hw_refine);
 static int snd_pcm_hw_refine_user(struct snd_pcm_substream *substream,
 				  struct snd_pcm_hw_params __user * _params)
 {
-	struct snd_pcm_hw_params *params __free(kfree) = NULL;
 	int err;
+	struct snd_pcm_hw_params *params __free(kfree) =
+		memdup_user(_params, sizeof(*params));
 
-	params = memdup_user(_params, sizeof(*params));
 	if (IS_ERR(params))
 		return PTR_ERR(params);
 
@@ -884,10 +884,10 @@ static int snd_pcm_hw_params(struct snd_pcm_substream *substream,
 static int snd_pcm_hw_params_user(struct snd_pcm_substream *substream,
 				  struct snd_pcm_hw_params __user * _params)
 {
-	struct snd_pcm_hw_params *params __free(kfree) = NULL;
 	int err;
+	struct snd_pcm_hw_params *params __free(kfree) =
+		memdup_user(_params, sizeof(*params));
 
-	params = memdup_user(_params, sizeof(*params));
 	if (IS_ERR(params))
 		return PTR_ERR(params);
 
@@ -2262,7 +2262,6 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd)
 {
 	struct snd_pcm_file *pcm_file;
 	struct snd_pcm_substream *substream1;
-	struct snd_pcm_group *group __free(kfree) = NULL;
 	struct snd_pcm_group *target_group;
 	bool nonatomic = substream->pcm->nonatomic;
 	CLASS(fd, f)(fd);
@@ -2278,7 +2277,8 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd)
 	if (substream == substream1)
 		return -EINVAL;
 
-	group = kzalloc(sizeof(*group), GFP_KERNEL);
+	struct snd_pcm_group *group __free(kfree) =
+		kzalloc(sizeof(*group), GFP_KERNEL);
 	if (!group)
 		return -ENOMEM;
 	snd_pcm_group_init(group);
@@ -3286,7 +3286,6 @@ static int snd_pcm_xfern_frames_ioctl(struct snd_pcm_substream *substream,
 {
 	struct snd_xfern xfern;
 	struct snd_pcm_runtime *runtime = substream->runtime;
-	void *bufs __free(kfree) = NULL;
 	snd_pcm_sframes_t result;
 
 	if (runtime->state == SNDRV_PCM_STATE_OPEN)
@@ -3298,7 +3297,8 @@ static int snd_pcm_xfern_frames_ioctl(struct snd_pcm_substream *substream,
 	if (copy_from_user(&xfern, _xfern, sizeof(xfern)))
 		return -EFAULT;
 
-	bufs = memdup_array_user(xfern.bufs, runtime->channels, sizeof(void *));
+	void *bufs __free(kfree) =
+		memdup_array_user(xfern.bufs, runtime->channels, sizeof(void *));
 	if (IS_ERR(bufs))
 		return PTR_ERR(bufs);
 	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
@@ -3572,7 +3572,6 @@ static ssize_t snd_pcm_readv(struct kiocb *iocb, struct iov_iter *to)
 	struct snd_pcm_runtime *runtime;
 	snd_pcm_sframes_t result;
 	unsigned long i;
-	void __user **bufs __free(kfree) = NULL;
 	snd_pcm_uframes_t frames;
 	const struct iovec *iov = iter_iov(to);
 
@@ -3591,7 +3590,9 @@ static ssize_t snd_pcm_readv(struct kiocb *iocb, struct iov_iter *to)
 	if (!frame_aligned(runtime, iov->iov_len))
 		return -EINVAL;
 	frames = bytes_to_samples(runtime, iov->iov_len);
-	bufs = kmalloc_array(to->nr_segs, sizeof(void *), GFP_KERNEL);
+
+	void __user **bufs __free(kfree) =
+		kmalloc_array(to->nr_segs, sizeof(void *), GFP_KERNEL);
 	if (bufs == NULL)
 		return -ENOMEM;
 	for (i = 0; i < to->nr_segs; ++i) {
@@ -3611,7 +3612,6 @@ static ssize_t snd_pcm_writev(struct kiocb *iocb, struct iov_iter *from)
 	struct snd_pcm_runtime *runtime;
 	snd_pcm_sframes_t result;
 	unsigned long i;
-	void __user **bufs __free(kfree) = NULL;
 	snd_pcm_uframes_t frames;
 	const struct iovec *iov = iter_iov(from);
 
@@ -3629,7 +3629,9 @@ static ssize_t snd_pcm_writev(struct kiocb *iocb, struct iov_iter *from)
 	    !frame_aligned(runtime, iov->iov_len))
 		return -EINVAL;
 	frames = bytes_to_samples(runtime, iov->iov_len);
-	bufs = kmalloc_array(from->nr_segs, sizeof(void *), GFP_KERNEL);
+
+	void __user **bufs __free(kfree) =
+		kmalloc_array(from->nr_segs, sizeof(void *), GFP_KERNEL);
 	if (bufs == NULL)
 		return -ENOMEM;
 	for (i = 0; i < from->nr_segs; ++i) {
@@ -4101,15 +4103,15 @@ static void snd_pcm_hw_convert_to_old_params(struct snd_pcm_hw_params_old *opara
 static int snd_pcm_hw_refine_old_user(struct snd_pcm_substream *substream,
 				      struct snd_pcm_hw_params_old __user * _oparams)
 {
-	struct snd_pcm_hw_params *params __free(kfree) = NULL;
-	struct snd_pcm_hw_params_old *oparams __free(kfree) = NULL;
 	int err;
 
-	params = kmalloc(sizeof(*params), GFP_KERNEL);
+	struct snd_pcm_hw_params *params __free(kfree) =
+		kmalloc(sizeof(*params), GFP_KERNEL);
 	if (!params)
 		return -ENOMEM;
 
-	oparams = memdup_user(_oparams, sizeof(*oparams));
+	struct snd_pcm_hw_params_old *oparams __free(kfree) =
+		memdup_user(_oparams, sizeof(*oparams));
 	if (IS_ERR(oparams))
 		return PTR_ERR(oparams);
 	snd_pcm_hw_convert_from_old_params(params, oparams);
@@ -4130,15 +4132,15 @@ static int snd_pcm_hw_refine_old_user(struct snd_pcm_substream *substream,
 static int snd_pcm_hw_params_old_user(struct snd_pcm_substream *substream,
 				      struct snd_pcm_hw_params_old __user * _oparams)
 {
-	struct snd_pcm_hw_params *params __free(kfree) = NULL;
-	struct snd_pcm_hw_params_old *oparams __free(kfree) = NULL;
 	int err;
 
-	params = kmalloc(sizeof(*params), GFP_KERNEL);
+	struct snd_pcm_hw_params *params __free(kfree) =
+		kmalloc(sizeof(*params), GFP_KERNEL);
 	if (!params)
 		return -ENOMEM;
 
-	oparams = memdup_user(_oparams, sizeof(*oparams));
+	struct snd_pcm_hw_params_old *oparams __free(kfree) =
+		memdup_user(_oparams, sizeof(*oparams));
 	if (IS_ERR(oparams))
 		return PTR_ERR(oparams);
 
diff --git a/sound/core/seq/oss/seq_oss.c b/sound/core/seq/oss/seq_oss.c
index 02d30d8b6c3a..021cd70f90db 100644
--- a/sound/core/seq/oss/seq_oss.c
+++ b/sound/core/seq/oss/seq_oss.c
@@ -54,10 +54,10 @@ static __poll_t odev_poll(struct file *file, poll_table * wait);
  */
 
 static struct snd_seq_driver seq_oss_synth_driver = {
+	.probe = snd_seq_oss_synth_probe,
+	.remove = snd_seq_oss_synth_remove,
 	.driver = {
 		.name = KBUILD_MODNAME,
-		.probe = snd_seq_oss_synth_probe,
-		.remove = snd_seq_oss_synth_remove,
 	},
 	.id = SNDRV_SEQ_DEV_ID_OSS,
 	.argsize = sizeof(struct snd_seq_oss_reg),
diff --git a/sound/core/seq/oss/seq_oss_init.c b/sound/core/seq/oss/seq_oss_init.c
index 973f057eb731..e0c368bd09cb 100644
--- a/sound/core/seq/oss/seq_oss_init.c
+++ b/sound/core/seq/oss/seq_oss_init.c
@@ -63,10 +63,10 @@ int __init
 snd_seq_oss_create_client(void)
 {
 	int rc;
-	struct snd_seq_port_info *port __free(kfree) = NULL;
 	struct snd_seq_port_callback port_callback;
+	struct snd_seq_port_info *port __free(kfree) =
+		kzalloc(sizeof(*port), GFP_KERNEL);
 
-	port = kzalloc(sizeof(*port), GFP_KERNEL);
 	if (!port)
 		return -ENOMEM;
 
diff --git a/sound/core/seq/oss/seq_oss_midi.c b/sound/core/seq/oss/seq_oss_midi.c
index 023e5d0a4351..2d48c25ff4df 100644
--- a/sound/core/seq/oss/seq_oss_midi.c
+++ b/sound/core/seq/oss/seq_oss_midi.c
@@ -65,11 +65,11 @@ static int send_midi_event(struct seq_oss_devinfo *dp, struct snd_seq_event *ev,
 int
 snd_seq_oss_midi_lookup_ports(int client)
 {
-	struct snd_seq_client_info *clinfo __free(kfree) = NULL;
-	struct snd_seq_port_info *pinfo __free(kfree) = NULL;
+	struct snd_seq_client_info *clinfo __free(kfree) =
+		kzalloc(sizeof(*clinfo), GFP_KERNEL);
+	struct snd_seq_port_info *pinfo __free(kfree) =
+		kzalloc(sizeof(*pinfo), GFP_KERNEL);
 
-	clinfo = kzalloc(sizeof(*clinfo), GFP_KERNEL);
-	pinfo = kzalloc(sizeof(*pinfo), GFP_KERNEL);
 	if (!clinfo || !pinfo)
 		return -ENOMEM;
 	clinfo->client = -1;
@@ -305,10 +305,10 @@ int
 snd_seq_oss_midi_open(struct seq_oss_devinfo *dp, int dev, int fmode)
 {
 	int perm;
-	struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL;
 	struct snd_seq_port_subscribe subs;
+	struct seq_oss_midi *mdev __free(seq_oss_midi) =
+		get_mididev(dp, dev);
 
-	mdev = get_mididev(dp, dev);
 	if (!mdev)
 		return -ENODEV;
 
@@ -364,10 +364,10 @@ snd_seq_oss_midi_open(struct seq_oss_devinfo *dp, int dev, int fmode)
 int
 snd_seq_oss_midi_close(struct seq_oss_devinfo *dp, int dev)
 {
-	struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL;
 	struct snd_seq_port_subscribe subs;
+	struct seq_oss_midi *mdev __free(seq_oss_midi) =
+		get_mididev(dp, dev);
 
-	mdev = get_mididev(dp, dev);
 	if (!mdev)
 		return -ENODEV;
 	guard(mutex)(&mdev->open_mutex);
@@ -399,10 +399,10 @@ snd_seq_oss_midi_close(struct seq_oss_devinfo *dp, int dev)
 int
 snd_seq_oss_midi_filemode(struct seq_oss_devinfo *dp, int dev)
 {
-	struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL;
 	int mode;
+	struct seq_oss_midi *mdev __free(seq_oss_midi) =
+		get_mididev(dp, dev);
 
-	mdev = get_mididev(dp, dev);
 	if (!mdev)
 		return 0;
 
@@ -422,9 +422,9 @@ snd_seq_oss_midi_filemode(struct seq_oss_devinfo *dp, int dev)
 void
 snd_seq_oss_midi_reset(struct seq_oss_devinfo *dp, int dev)
 {
-	struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL;
+	struct seq_oss_midi *mdev __free(seq_oss_midi) =
+		get_mididev(dp, dev);
 
-	mdev = get_mididev(dp, dev);
 	if (!mdev)
 		return;
 	if (!mdev->opened)
@@ -468,9 +468,9 @@ snd_seq_oss_midi_reset(struct seq_oss_devinfo *dp, int dev)
 void
 snd_seq_oss_midi_get_addr(struct seq_oss_devinfo *dp, int dev, struct snd_seq_addr *addr)
 {
-	struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL;
+	struct seq_oss_midi *mdev __free(seq_oss_midi) =
+		get_mididev(dp, dev);
 
-	mdev = get_mididev(dp, dev);
 	if (!mdev)
 		return;
 	addr->client = mdev->client;
@@ -485,11 +485,11 @@ int
 snd_seq_oss_midi_input(struct snd_seq_event *ev, int direct, void *private_data)
 {
 	struct seq_oss_devinfo *dp = (struct seq_oss_devinfo *)private_data;
-	struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL;
 
 	if (dp->readq == NULL)
 		return 0;
-	mdev = find_slot(ev->source.client, ev->source.port);
+	struct seq_oss_midi *mdev __free(seq_oss_midi) =
+		find_slot(ev->source.client, ev->source.port);
 	if (!mdev)
 		return 0;
 	if (!(mdev->opened & PERM_READ))
@@ -595,9 +595,9 @@ send_midi_event(struct seq_oss_devinfo *dp, struct snd_seq_event *ev, struct seq
 int
 snd_seq_oss_midi_putc(struct seq_oss_devinfo *dp, int dev, unsigned char c, struct snd_seq_event *ev)
 {
-	struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL;
+	struct seq_oss_midi *mdev __free(seq_oss_midi) =
+		get_mididev(dp, dev);
 
-	mdev = get_mididev(dp, dev);
 	if (!mdev)
 		return -ENODEV;
 	if (snd_midi_event_encode_byte(mdev->coder, c, ev)) {
@@ -613,9 +613,9 @@ snd_seq_oss_midi_putc(struct seq_oss_devinfo *dp, int dev, unsigned char c, stru
 int
 snd_seq_oss_midi_make_info(struct seq_oss_devinfo *dp, int dev, struct midi_info *inf)
 {
-	struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL;
+	struct seq_oss_midi *mdev __free(seq_oss_midi) =
+		get_mididev(dp, dev);
 
-	mdev = get_mididev(dp, dev);
 	if (!mdev)
 		return -ENXIO;
 	inf->device = dev;
@@ -651,10 +651,9 @@ snd_seq_oss_midi_info_read(struct snd_info_buffer *buf)
 
 	snd_iprintf(buf, "\nNumber of MIDI devices: %d\n", max_midi_devs);
 	for (i = 0; i < max_midi_devs; i++) {
-		struct seq_oss_midi *mdev __free(seq_oss_midi) = NULL;
-
 		snd_iprintf(buf, "\nmidi %d: ", i);
-		mdev = get_mdev(i);
+		struct seq_oss_midi *mdev __free(seq_oss_midi) =
+			get_mdev(i);
 		if (mdev == NULL) {
 			snd_iprintf(buf, "*empty*\n");
 			continue;
diff --git a/sound/core/seq/oss/seq_oss_synth.c b/sound/core/seq/oss/seq_oss_synth.c
index 8c4e5913c7e6..c7f81f2053a7 100644
--- a/sound/core/seq/oss/seq_oss_synth.c
+++ b/sound/core/seq/oss/seq_oss_synth.c
@@ -80,9 +80,8 @@ snd_seq_oss_synth_init(void)
  * registration of the synth device
  */
 int
-snd_seq_oss_synth_probe(struct device *_dev)
+snd_seq_oss_synth_probe(struct snd_seq_device *dev)
 {
-	struct snd_seq_device *dev = to_seq_dev(_dev);
 	int i;
 	struct seq_oss_synth *rec;
 	struct snd_seq_oss_reg *reg = SNDRV_SEQ_DEVICE_ARGPTR(dev);
@@ -128,10 +127,9 @@ snd_seq_oss_synth_probe(struct device *_dev)
 }
 
 
-int
-snd_seq_oss_synth_remove(struct device *_dev)
+void
+snd_seq_oss_synth_remove(struct snd_seq_device *dev)
 {
-	struct snd_seq_device *dev = to_seq_dev(_dev);
 	int index;
 	struct seq_oss_synth *rec = dev->driver_data;
 
@@ -142,7 +140,7 @@ snd_seq_oss_synth_remove(struct device *_dev)
 		}
 		if (index >= max_synth_devs) {
 			pr_err("ALSA: seq_oss: can't unregister synth\n");
-			return -EINVAL;
+			return;
 		}
 		synth_devs[index] = NULL;
 		if (index == max_synth_devs - 1) {
@@ -160,8 +158,6 @@ snd_seq_oss_synth_remove(struct device *_dev)
 
 	snd_use_lock_sync(&rec->use_lock);
 	kfree(rec);
-
-	return 0;
 }
 
 
@@ -368,7 +364,6 @@ reset_channels(struct seq_oss_synthinfo *info)
 void
 snd_seq_oss_synth_reset(struct seq_oss_devinfo *dp, int dev)
 {
-	struct seq_oss_synth *rec __free(seq_oss_synth) = NULL;
 	struct seq_oss_synthinfo *info;
 
 	info = get_synthinfo_nospec(dp, dev);
@@ -391,7 +386,8 @@ snd_seq_oss_synth_reset(struct seq_oss_devinfo *dp, int dev)
 		return;
 	}
 
-	rec = get_sdev(dev);
+	struct seq_oss_synth *rec __free(seq_oss_synth) =
+		get_sdev(dev);
 	if (rec == NULL)
 		return;
 	if (rec->oper.reset) {
@@ -415,7 +411,6 @@ int
 snd_seq_oss_synth_load_patch(struct seq_oss_devinfo *dp, int dev, int fmt,
 			    const char __user *buf, int p, int c)
 {
-	struct seq_oss_synth *rec __free(seq_oss_synth) = NULL;
 	struct seq_oss_synthinfo *info;
 
 	info = get_synthinfo_nospec(dp, dev);
@@ -424,7 +419,9 @@ snd_seq_oss_synth_load_patch(struct seq_oss_devinfo *dp, int dev, int fmt,
 
 	if (info->is_midi)
 		return 0;
-	rec = get_synthdev(dp, dev);
+
+	struct seq_oss_synth *rec __free(seq_oss_synth) =
+		get_synthdev(dp, dev);
 	if (!rec)
 		return -ENXIO;
 
@@ -440,9 +437,9 @@ snd_seq_oss_synth_load_patch(struct seq_oss_devinfo *dp, int dev, int fmt,
 struct seq_oss_synthinfo *
 snd_seq_oss_synth_info(struct seq_oss_devinfo *dp, int dev)
 {
-	struct seq_oss_synth *rec __free(seq_oss_synth) = NULL;
+	struct seq_oss_synth *rec __free(seq_oss_synth) =
+		get_synthdev(dp, dev);
 
-	rec = get_synthdev(dp, dev);
 	if (rec)
 		return get_synthinfo_nospec(dp, dev);
 	return NULL;
@@ -495,13 +492,14 @@ snd_seq_oss_synth_addr(struct seq_oss_devinfo *dp, int dev, struct snd_seq_event
 int
 snd_seq_oss_synth_ioctl(struct seq_oss_devinfo *dp, int dev, unsigned int cmd, unsigned long addr)
 {
-	struct seq_oss_synth *rec __free(seq_oss_synth) = NULL;
 	struct seq_oss_synthinfo *info;
 
 	info = get_synthinfo_nospec(dp, dev);
 	if (!info || info->is_midi)
 		return -ENXIO;
-	rec = get_synthdev(dp, dev);
+
+	struct seq_oss_synth *rec __free(seq_oss_synth) =
+		get_synthdev(dp, dev);
 	if (!rec)
 		return -ENXIO;
 	if (rec->oper.ioctl == NULL)
@@ -575,10 +573,9 @@ snd_seq_oss_synth_info_read(struct snd_info_buffer *buf)
 
 	snd_iprintf(buf, "\nNumber of synth devices: %d\n", max_synth_devs);
 	for (i = 0; i < max_synth_devs; i++) {
-		struct seq_oss_synth *rec __free(seq_oss_synth) = NULL;
-
 		snd_iprintf(buf, "\nsynth %d: ", i);
-		rec = get_sdev(i);
+		struct seq_oss_synth *rec __free(seq_oss_synth) =
+			get_sdev(i);
 		if (rec == NULL) {
 			snd_iprintf(buf, "*empty*\n");
 			continue;
diff --git a/sound/core/seq/oss/seq_oss_synth.h b/sound/core/seq/oss/seq_oss_synth.h
index ffc40d8a7ef1..f52283904cba 100644
--- a/sound/core/seq/oss/seq_oss_synth.h
+++ b/sound/core/seq/oss/seq_oss_synth.h
@@ -15,8 +15,8 @@
 #include <sound/seq_device.h>
 
 void snd_seq_oss_synth_init(void);
-int snd_seq_oss_synth_probe(struct device *dev);
-int snd_seq_oss_synth_remove(struct device *dev);
+int snd_seq_oss_synth_probe(struct snd_seq_device *dev);
+void snd_seq_oss_synth_remove(struct snd_seq_device *dev);
 void snd_seq_oss_synth_setup(struct seq_oss_devinfo *dp);
 void snd_seq_oss_synth_setup_midi(struct seq_oss_devinfo *dp);
 void snd_seq_oss_synth_cleanup(struct seq_oss_devinfo *dp);
diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c
index f9a6e497f997..75a7a2af9d8c 100644
--- a/sound/core/seq/seq_clientmgr.c
+++ b/sound/core/seq/seq_clientmgr.c
@@ -494,9 +494,9 @@ static int check_port_perm(struct snd_seq_client_port *port, unsigned int flags)
  */
 static struct snd_seq_client *get_event_dest_client(struct snd_seq_event *event)
 {
-	struct snd_seq_client *dest __free(snd_seq_client) = NULL;
+	struct snd_seq_client *dest __free(snd_seq_client) =
+		snd_seq_client_use_ptr(event->dest.client);
 
-	dest = snd_seq_client_use_ptr(event->dest.client);
 	if (dest == NULL)
 		return NULL;
 	if (! dest->accept_input)
@@ -565,9 +565,9 @@ static int bounce_error_event(struct snd_seq_client *client,
 static int update_timestamp_of_queue(struct snd_seq_event *event,
 				     int queue, int real_time)
 {
-	struct snd_seq_queue *q __free(snd_seq_queue) = NULL;
+	struct snd_seq_queue *q __free(snd_seq_queue) =
+		queueptr(queue);
 
-	q = queueptr(queue);
 	if (! q)
 		return 0;
 	event->queue = queue;
@@ -609,13 +609,13 @@ static int _snd_seq_deliver_single_event(struct snd_seq_client *client,
 					 struct snd_seq_event *event,
 					 int atomic, int hop)
 {
-	struct snd_seq_client *dest __free(snd_seq_client) = NULL;
-	struct snd_seq_client_port *dest_port __free(snd_seq_port) = NULL;
-
-	dest = get_event_dest_client(event);
+	struct snd_seq_client *dest __free(snd_seq_client) =
+		get_event_dest_client(event);
 	if (dest == NULL)
 		return -ENOENT;
-	dest_port = snd_seq_port_use_ptr(dest, event->dest.port);
+
+	struct snd_seq_client_port *dest_port __free(snd_seq_port) =
+		snd_seq_port_use_ptr(dest, event->dest.port);
 	if (dest_port == NULL)
 		return -ENOENT;
 
@@ -672,7 +672,6 @@ static int __deliver_to_subscribers(struct snd_seq_client *client,
 				    struct snd_seq_event *event,
 				    int port, int atomic, int hop)
 {
-	struct snd_seq_client_port *src_port __free(snd_seq_port) = NULL;
 	struct snd_seq_subscribers *subs;
 	int err, result = 0, num_ev = 0;
 	union __snd_seq_event event_saved;
@@ -681,7 +680,9 @@ static int __deliver_to_subscribers(struct snd_seq_client *client,
 
 	if (port < 0)
 		return 0;
-	src_port = snd_seq_port_use_ptr(client, port);
+
+	struct snd_seq_client_port *src_port __free(snd_seq_port) =
+		snd_seq_port_use_ptr(client, port);
 	if (!src_port)
 		return 0;
 
@@ -801,13 +802,13 @@ static int snd_seq_deliver_event(struct snd_seq_client *client, struct snd_seq_e
  */
 int snd_seq_dispatch_event(struct snd_seq_event_cell *cell, int atomic, int hop)
 {
-	struct snd_seq_client *client __free(snd_seq_client) = NULL;
 	int result;
 
 	if (snd_BUG_ON(!cell))
 		return -EINVAL;
 
-	client = snd_seq_client_use_ptr(cell->event.source.client);
+	struct snd_seq_client *client __free(snd_seq_client) =
+		snd_seq_client_use_ptr(cell->event.source.client);
 	if (client == NULL) {
 		snd_seq_cell_free(cell); /* release this cell */
 		return -EINVAL;
@@ -1154,10 +1155,10 @@ static int snd_seq_ioctl_system_info(struct snd_seq_client *client, void *arg)
 static int snd_seq_ioctl_running_mode(struct snd_seq_client *client, void  *arg)
 {
 	struct snd_seq_running_info *info = arg;
-	struct snd_seq_client *cptr __free(snd_seq_client) = NULL;
-
 	/* requested client number */
-	cptr = client_load_and_use_ptr(info->client);
+	struct snd_seq_client *cptr __free(snd_seq_client) =
+		client_load_and_use_ptr(info->client);
+
 	if (cptr == NULL)
 		return -ENOENT;		/* don't change !!! */
 
@@ -1207,10 +1208,10 @@ static int snd_seq_ioctl_get_client_info(struct snd_seq_client *client,
 					 void *arg)
 {
 	struct snd_seq_client_info *client_info = arg;
-	struct snd_seq_client *cptr __free(snd_seq_client) = NULL;
-
 	/* requested client number */
-	cptr = client_load_and_use_ptr(client_info->client);
+	struct snd_seq_client *cptr __free(snd_seq_client) =
+		client_load_and_use_ptr(client_info->client);
+
 	if (cptr == NULL)
 		return -ENOENT;		/* don't change !!! */
 
@@ -1344,14 +1345,14 @@ static int snd_seq_ioctl_delete_port(struct snd_seq_client *client, void *arg)
 static int snd_seq_ioctl_get_port_info(struct snd_seq_client *client, void *arg)
 {
 	struct snd_seq_port_info *info = arg;
-	struct snd_seq_client *cptr __free(snd_seq_client) = NULL;
-	struct snd_seq_client_port *port __free(snd_seq_port) = NULL;
 
-	cptr = client_load_and_use_ptr(info->addr.client);
+	struct snd_seq_client *cptr __free(snd_seq_client) =
+		client_load_and_use_ptr(info->addr.client);
 	if (cptr == NULL)
 		return -ENXIO;
 
-	port = snd_seq_port_use_ptr(cptr, info->addr.port);
+	struct snd_seq_client_port *port __free(snd_seq_port) =
+		snd_seq_port_use_ptr(cptr, info->addr.port);
 	if (port == NULL)
 		return -ENOENT;			/* don't change */
 
@@ -1367,11 +1368,12 @@ static int snd_seq_ioctl_get_port_info(struct snd_seq_client *client, void *arg)
 static int snd_seq_ioctl_set_port_info(struct snd_seq_client *client, void *arg)
 {
 	struct snd_seq_port_info *info = arg;
-	struct snd_seq_client_port *port __free(snd_seq_port) = NULL;
 
 	if (info->addr.client != client->number) /* only set our own ports ! */
 		return -EPERM;
-	port = snd_seq_port_use_ptr(client, info->addr.port);
+
+	struct snd_seq_client_port *port __free(snd_seq_port) =
+		snd_seq_port_use_ptr(client, info->addr.port);
 	if (port) {
 		snd_seq_set_port_info(port, info);
 		/* notify the change */
@@ -1444,22 +1446,22 @@ static int snd_seq_ioctl_subscribe_port(struct snd_seq_client *client,
 					void *arg)
 {
 	struct snd_seq_port_subscribe *subs = arg;
-	struct snd_seq_client *receiver __free(snd_seq_client) = NULL;
-	struct snd_seq_client *sender __free(snd_seq_client) = NULL;
-	struct snd_seq_client_port *sport __free(snd_seq_port) = NULL;
-	struct snd_seq_client_port *dport __free(snd_seq_port) = NULL;
 	int result;
 
-	receiver = client_load_and_use_ptr(subs->dest.client);
+	struct snd_seq_client *receiver __free(snd_seq_client) =
+		client_load_and_use_ptr(subs->dest.client);
 	if (!receiver)
 		return -EINVAL;
-	sender = client_load_and_use_ptr(subs->sender.client);
+	struct snd_seq_client *sender __free(snd_seq_client) =
+		client_load_and_use_ptr(subs->sender.client);
 	if (!sender)
 		return -EINVAL;
-	sport = snd_seq_port_use_ptr(sender, subs->sender.port);
+	struct snd_seq_client_port *sport __free(snd_seq_port) =
+		snd_seq_port_use_ptr(sender, subs->sender.port);
 	if (!sport)
 		return -EINVAL;
-	dport = snd_seq_port_use_ptr(receiver, subs->dest.port);
+	struct snd_seq_client_port *dport __free(snd_seq_port) =
+		snd_seq_port_use_ptr(receiver, subs->dest.port);
 	if (!dport)
 		return -EINVAL;
 
@@ -1483,22 +1485,22 @@ static int snd_seq_ioctl_unsubscribe_port(struct snd_seq_client *client,
 					  void *arg)
 {
 	struct snd_seq_port_subscribe *subs = arg;
-	struct snd_seq_client *receiver __free(snd_seq_client) = NULL;
-	struct snd_seq_client *sender __free(snd_seq_client) = NULL;
-	struct snd_seq_client_port *sport __free(snd_seq_port) = NULL;
-	struct snd_seq_client_port *dport __free(snd_seq_port) = NULL;
 	int result;
 
-	receiver = snd_seq_client_use_ptr(subs->dest.client);
+	struct snd_seq_client *receiver __free(snd_seq_client) =
+		snd_seq_client_use_ptr(subs->dest.client);
 	if (!receiver)
 		return -ENXIO;
-	sender = snd_seq_client_use_ptr(subs->sender.client);
+	struct snd_seq_client *sender __free(snd_seq_client) =
+		snd_seq_client_use_ptr(subs->sender.client);
 	if (!sender)
 		return -ENXIO;
-	sport = snd_seq_port_use_ptr(sender, subs->sender.port);
+	struct snd_seq_client_port *sport __free(snd_seq_port) =
+		snd_seq_port_use_ptr(sender, subs->sender.port);
 	if (!sport)
 		return -ENXIO;
-	dport = snd_seq_port_use_ptr(receiver, subs->dest.port);
+	struct snd_seq_client_port *dport __free(snd_seq_port) =
+		snd_seq_port_use_ptr(receiver, subs->dest.port);
 	if (!dport)
 		return -ENXIO;
 
@@ -1518,9 +1520,9 @@ static int snd_seq_ioctl_unsubscribe_port(struct snd_seq_client *client,
 static int snd_seq_ioctl_create_queue(struct snd_seq_client *client, void *arg)
 {
 	struct snd_seq_queue_info *info = arg;
-	struct snd_seq_queue *q __free(snd_seq_queue) = NULL;
+	struct snd_seq_queue *q __free(snd_seq_queue) =
+		snd_seq_queue_alloc(client->number, info->locked, info->flags);
 
-	q = snd_seq_queue_alloc(client->number, info->locked, info->flags);
 	if (IS_ERR(q))
 		return PTR_ERR(q);
 
@@ -1549,9 +1551,9 @@ static int snd_seq_ioctl_get_queue_info(struct snd_seq_client *client,
 					void *arg)
 {
 	struct snd_seq_queue_info *info = arg;
-	struct snd_seq_queue *q __free(snd_seq_queue) = NULL;
+	struct snd_seq_queue *q __free(snd_seq_queue) =
+		queueptr(info->queue);
 
-	q = queueptr(info->queue);
 	if (q == NULL)
 		return -EINVAL;
 
@@ -1569,7 +1571,6 @@ static int snd_seq_ioctl_set_queue_info(struct snd_seq_client *client,
 					void *arg)
 {
 	struct snd_seq_queue_info *info = arg;
-	struct snd_seq_queue *q __free(snd_seq_queue) = NULL;
 
 	if (info->owner != client->number)
 		return -EINVAL;
@@ -1584,7 +1585,8 @@ static int snd_seq_ioctl_set_queue_info(struct snd_seq_client *client,
 		return -EPERM;
 	}	
 
-	q = queueptr(info->queue);
+	struct snd_seq_queue *q __free(snd_seq_queue) =
+		queueptr(info->queue);
 	if (! q)
 		return -EINVAL;
 	if (q->owner != client->number)
@@ -1599,9 +1601,9 @@ static int snd_seq_ioctl_get_named_queue(struct snd_seq_client *client,
 					 void *arg)
 {
 	struct snd_seq_queue_info *info = arg;
-	struct snd_seq_queue *q __free(snd_seq_queue) = NULL;
+	struct snd_seq_queue *q __free(snd_seq_queue) =
+		snd_seq_queue_find_name(info->name);
 
-	q = snd_seq_queue_find_name(info->name);
 	if (q == NULL)
 		return -EINVAL;
 	info->queue = q->queue;
@@ -1616,10 +1618,10 @@ static int snd_seq_ioctl_get_queue_status(struct snd_seq_client *client,
 					  void *arg)
 {
 	struct snd_seq_queue_status *status = arg;
-	struct snd_seq_queue *queue __free(snd_seq_queue) = NULL;
 	struct snd_seq_timer *tmr;
+	struct snd_seq_queue *queue __free(snd_seq_queue) =
+		queueptr(status->queue);
 
-	queue = queueptr(status->queue);
 	if (queue == NULL)
 		return -EINVAL;
 	memset(status, 0, sizeof(*status));
@@ -1644,10 +1646,10 @@ static int snd_seq_ioctl_get_queue_tempo(struct snd_seq_client *client,
 					 void *arg)
 {
 	struct snd_seq_queue_tempo *tempo = arg;
-	struct snd_seq_queue *queue __free(snd_seq_queue) = NULL;
 	struct snd_seq_timer *tmr;
+	struct snd_seq_queue *queue __free(snd_seq_queue) =
+		queueptr(tempo->queue);
 
-	queue = queueptr(tempo->queue);
 	if (queue == NULL)
 		return -EINVAL;
 	memset(tempo, 0, sizeof(*tempo));
@@ -1693,10 +1695,10 @@ static int snd_seq_ioctl_get_queue_timer(struct snd_seq_client *client,
 					 void *arg)
 {
 	struct snd_seq_queue_timer *timer = arg;
-	struct snd_seq_queue *queue __free(snd_seq_queue) = NULL;
 	struct snd_seq_timer *tmr;
+	struct snd_seq_queue *queue __free(snd_seq_queue) =
+		queueptr(timer->queue);
 
-	queue = queueptr(timer->queue);
 	if (queue == NULL)
 		return -EINVAL;
 
@@ -1726,10 +1728,10 @@ static int snd_seq_ioctl_set_queue_timer(struct snd_seq_client *client,
 		return -EINVAL;
 
 	if (snd_seq_queue_check_access(timer->queue, client->number)) {
-		struct snd_seq_queue *q __free(snd_seq_queue) = NULL;
 		struct snd_seq_timer *tmr;
+		struct snd_seq_queue *q __free(snd_seq_queue) =
+			queueptr(timer->queue);
 
-		q = queueptr(timer->queue);
 		if (q == NULL)
 			return -ENXIO;
 		guard(mutex)(&q->timer_mutex);
@@ -1788,9 +1790,9 @@ static int snd_seq_ioctl_get_client_pool(struct snd_seq_client *client,
 					 void *arg)
 {
 	struct snd_seq_client_pool *info = arg;
-	struct snd_seq_client *cptr __free(snd_seq_client) = NULL;
+	struct snd_seq_client *cptr __free(snd_seq_client) =
+		client_load_and_use_ptr(info->client);
 
-	cptr = client_load_and_use_ptr(info->client);
 	if (cptr == NULL)
 		return -ENOENT;
 	memset(info, 0, sizeof(*info));
@@ -1888,13 +1890,13 @@ static int snd_seq_ioctl_get_subscription(struct snd_seq_client *client,
 					  void *arg)
 {
 	struct snd_seq_port_subscribe *subs = arg;
-	struct snd_seq_client *sender __free(snd_seq_client) = NULL;
-	struct snd_seq_client_port *sport __free(snd_seq_port) = NULL;
 
-	sender = client_load_and_use_ptr(subs->sender.client);
+	struct snd_seq_client *sender __free(snd_seq_client) =
+		client_load_and_use_ptr(subs->sender.client);
 	if (!sender)
 		return -EINVAL;
-	sport = snd_seq_port_use_ptr(sender, subs->sender.port);
+	struct snd_seq_client_port *sport __free(snd_seq_port) =
+		snd_seq_port_use_ptr(sender, subs->sender.port);
 	if (!sport)
 		return -EINVAL;
 	return snd_seq_port_get_subscription(&sport->c_src, &subs->dest, subs);
@@ -1907,16 +1909,16 @@ static int snd_seq_ioctl_get_subscription(struct snd_seq_client *client,
 static int snd_seq_ioctl_query_subs(struct snd_seq_client *client, void *arg)
 {
 	struct snd_seq_query_subs *subs = arg;
-	struct snd_seq_client *cptr __free(snd_seq_client) = NULL;
-	struct snd_seq_client_port *port __free(snd_seq_port) = NULL;
 	struct snd_seq_port_subs_info *group;
 	struct list_head *p;
 	int i;
 
-	cptr = client_load_and_use_ptr(subs->root.client);
+	struct snd_seq_client *cptr __free(snd_seq_client) =
+		client_load_and_use_ptr(subs->root.client);
 	if (!cptr)
 		return -ENXIO;
-	port = snd_seq_port_use_ptr(cptr, subs->root.port);
+	struct snd_seq_client_port *port __free(snd_seq_port) =
+		snd_seq_port_use_ptr(cptr, subs->root.port);
 	if (!port)
 		return -ENXIO;
 
@@ -1963,7 +1965,6 @@ static int snd_seq_ioctl_query_next_client(struct snd_seq_client *client,
 					   void *arg)
 {
 	struct snd_seq_client_info *info = arg;
-	struct snd_seq_client *cptr __free(snd_seq_client) = NULL;
 
 	/* search for next client */
 	if (info->client < INT_MAX)
@@ -1971,7 +1972,8 @@ static int snd_seq_ioctl_query_next_client(struct snd_seq_client *client,
 	if (info->client < 0)
 		info->client = 0;
 	for (; info->client < SNDRV_SEQ_MAX_CLIENTS; info->client++) {
-		cptr = client_load_and_use_ptr(info->client);
+		struct snd_seq_client *cptr __free(snd_seq_client) =
+			client_load_and_use_ptr(info->client);
 		if (cptr) {
 			get_client_info(cptr, info);
 			return 0; /* found */
@@ -1987,16 +1989,16 @@ static int snd_seq_ioctl_query_next_port(struct snd_seq_client *client,
 					 void *arg)
 {
 	struct snd_seq_port_info *info = arg;
-	struct snd_seq_client *cptr __free(snd_seq_client) = NULL;
-	struct snd_seq_client_port *port __free(snd_seq_port) = NULL;
 
-	cptr = client_load_and_use_ptr(info->addr.client);
+	struct snd_seq_client *cptr __free(snd_seq_client) =
+		client_load_and_use_ptr(info->addr.client);
 	if (cptr == NULL)
 		return -ENXIO;
 
 	/* search for next port */
 	info->addr.port++;
-	port = snd_seq_port_query_nearest(cptr, info);
+	struct snd_seq_client_port *port __free(snd_seq_port) =
+		snd_seq_port_query_nearest(cptr, info);
 	if (port == NULL)
 		return -ENOENT;
 
@@ -2067,7 +2069,6 @@ static int snd_seq_ioctl_client_ump_info(struct snd_seq_client *caller,
 {
 	struct snd_seq_client_ump_info __user *argp =
 		(struct snd_seq_client_ump_info __user *)arg;
-	struct snd_seq_client *cptr __free(snd_seq_client) = NULL;
 	int client, type, err = 0;
 	size_t size;
 	void *p;
@@ -2083,7 +2084,9 @@ static int snd_seq_ioctl_client_ump_info(struct snd_seq_client *caller,
 		size = sizeof(struct snd_ump_endpoint_info);
 	else
 		size = sizeof(struct snd_ump_block_info);
-	cptr = client_load_and_use_ptr(client);
+
+	struct snd_seq_client *cptr __free(snd_seq_client) =
+		client_load_and_use_ptr(client);
 	if (!cptr)
 		return -ENOENT;
 
@@ -2342,8 +2345,6 @@ EXPORT_SYMBOL(snd_seq_delete_kernel_client);
 int snd_seq_kernel_client_enqueue(int client, struct snd_seq_event *ev,
 				  struct file *file, bool blocking)
 {
-	struct snd_seq_client *cptr __free(snd_seq_client) = NULL;
-
 	if (snd_BUG_ON(!ev))
 		return -EINVAL;
 
@@ -2360,7 +2361,8 @@ int snd_seq_kernel_client_enqueue(int client, struct snd_seq_event *ev,
 	if (check_event_type_and_length(ev))
 		return -EINVAL;
 
-	cptr = client_load_and_use_ptr(client);
+	struct snd_seq_client *cptr __free(snd_seq_client) =
+		client_load_and_use_ptr(client);
 	if (cptr == NULL)
 		return -EINVAL;
 	
@@ -2385,8 +2387,6 @@ EXPORT_SYMBOL(snd_seq_kernel_client_enqueue);
 int snd_seq_kernel_client_dispatch(int client, struct snd_seq_event * ev,
 				   int atomic, int hop)
 {
-	struct snd_seq_client *cptr __free(snd_seq_client) = NULL;
-
 	if (snd_BUG_ON(!ev))
 		return -EINVAL;
 
@@ -2397,7 +2397,8 @@ int snd_seq_kernel_client_dispatch(int client, struct snd_seq_event * ev,
 	if (check_event_type_and_length(ev))
 		return -EINVAL;
 
-	cptr = snd_seq_client_use_ptr(client);
+	struct snd_seq_client *cptr __free(snd_seq_client) =
+		snd_seq_client_use_ptr(client);
 	if (cptr == NULL)
 		return -EINVAL;
 
@@ -2450,9 +2451,9 @@ EXPORT_SYMBOL(snd_seq_kernel_client_ctl);
 /* a similar like above but taking locks; used only from OSS sequencer layer */
 int snd_seq_kernel_client_ioctl(int clientid, unsigned int cmd, void *arg)
 {
-	struct snd_seq_client *client __free(snd_seq_client) = NULL;
+	struct snd_seq_client *client __free(snd_seq_client) =
+		client_load_and_use_ptr(clientid);
 
-	client = client_load_and_use_ptr(clientid);
 	if (!client)
 		return -ENXIO;
 	guard(mutex)(&client->ioctl_mutex);
@@ -2597,9 +2598,9 @@ void snd_seq_info_clients_read(struct snd_info_entry *entry,
 
 	/* list the client table */
 	for (c = 0; c < SNDRV_SEQ_MAX_CLIENTS; c++) {
-		struct snd_seq_client *client __free(snd_seq_client) = NULL;
+		struct snd_seq_client *client __free(snd_seq_client) =
+			client_load_and_use_ptr(c);
 
-		client = client_load_and_use_ptr(c);
 		if (client == NULL)
 			continue;
 		if (client->type == NO_CLIENT)
diff --git a/sound/core/seq/seq_compat.c b/sound/core/seq/seq_compat.c
index 643af4c1e838..260428747e33 100644
--- a/sound/core/seq/seq_compat.c
+++ b/sound/core/seq/seq_compat.c
@@ -31,10 +31,10 @@ struct snd_seq_port_info32 {
 static int snd_seq_call_port_info_ioctl(struct snd_seq_client *client, unsigned int cmd,
 					struct snd_seq_port_info32 __user *data32)
 {
-	struct snd_seq_port_info *data __free(kfree) = NULL;
 	int err;
+	struct snd_seq_port_info *data __free(kfree) =
+		kmalloc(sizeof(*data), GFP_KERNEL);
 
-	data = kmalloc(sizeof(*data), GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 
diff --git a/sound/core/seq/seq_midi.c b/sound/core/seq/seq_midi.c
index 581e138a3115..88e930980f16 100644
--- a/sound/core/seq/seq_midi.c
+++ b/sound/core/seq/seq_midi.c
@@ -265,13 +265,10 @@ static void snd_seq_midisynth_delete(struct seq_midisynth *msynth)
 
 /* register new midi synth port */
 static int
-snd_seq_midisynth_probe(struct device *_dev)
+snd_seq_midisynth_probe(struct snd_seq_device *dev)
 {
-	struct snd_seq_device *dev = to_seq_dev(_dev);
 	struct seq_midisynth_client *client;
 	struct seq_midisynth *msynth, *ms;
-	struct snd_seq_port_info *port __free(kfree) = NULL;
-	struct snd_rawmidi_info *info __free(kfree) = NULL;
 	struct snd_rawmidi *rmidi = dev->private_data;
 	int newclient = 0;
 	unsigned int p, ports;
@@ -282,7 +279,9 @@ snd_seq_midisynth_probe(struct device *_dev)
 
 	if (snd_BUG_ON(!card || device < 0 || device >= SNDRV_RAWMIDI_DEVICES))
 		return -EINVAL;
-	info = kmalloc(sizeof(*info), GFP_KERNEL);
+
+	struct snd_rawmidi_info *info __free(kfree) =
+		kmalloc(sizeof(*info), GFP_KERNEL);
 	if (! info)
 		return -ENOMEM;
 	info->device = device;
@@ -320,7 +319,9 @@ snd_seq_midisynth_probe(struct device *_dev)
 	}
 
 	msynth = kcalloc(ports, sizeof(struct seq_midisynth), GFP_KERNEL);
-	port = kmalloc(sizeof(*port), GFP_KERNEL);
+
+	struct snd_seq_port_info *port __free(kfree) =
+		kmalloc(sizeof(*port), GFP_KERNEL);
 	if (msynth == NULL || port == NULL)
 		goto __nomem;
 
@@ -411,10 +412,9 @@ snd_seq_midisynth_probe(struct device *_dev)
 }
 
 /* release midi synth port */
-static int
-snd_seq_midisynth_remove(struct device *_dev)
+static void
+snd_seq_midisynth_remove(struct snd_seq_device *dev)
 {
-	struct snd_seq_device *dev = to_seq_dev(_dev);
 	struct seq_midisynth_client *client;
 	struct seq_midisynth *msynth;
 	struct snd_card *card = dev->card;
@@ -423,7 +423,7 @@ snd_seq_midisynth_remove(struct device *_dev)
 	guard(mutex)(&register_mutex);
 	client = synths[card->number];
 	if (client == NULL || client->ports[device] == NULL)
-		return -ENODEV;
+		return;
 	ports = client->ports_per_device[device];
 	client->ports_per_device[device] = 0;
 	msynth = client->ports[device];
@@ -437,14 +437,13 @@ snd_seq_midisynth_remove(struct device *_dev)
 		synths[card->number] = NULL;
 		kfree(client);
 	}
-	return 0;
 }
 
 static struct snd_seq_driver seq_midisynth_driver = {
+	.probe = snd_seq_midisynth_probe,
+	.remove = snd_seq_midisynth_remove,
 	.driver = {
 		.name = KBUILD_MODNAME,
-		.probe = snd_seq_midisynth_probe,
-		.remove = snd_seq_midisynth_remove,
 	},
 	.id = SNDRV_SEQ_DEV_ID_MIDISYNTH,
 	.argsize = 0,
diff --git a/sound/core/seq/seq_ports.c b/sound/core/seq/seq_ports.c
index 40fa379847e5..bbec34bba4f9 100644
--- a/sound/core/seq/seq_ports.c
+++ b/sound/core/seq/seq_ports.c
@@ -211,14 +211,13 @@ static void clear_subscriber_list(struct snd_seq_client *client,
 
 	list_for_each_safe(p, n, &grp->list_head) {
 		struct snd_seq_subscribers *subs;
-		struct snd_seq_client *c __free(snd_seq_client) = NULL;
-		struct snd_seq_client_port *aport __free(snd_seq_port) = NULL;
 
 		subs = get_subscriber(p, is_src);
-		if (is_src)
-			aport = get_client_port(&subs->info.dest, &c);
-		else
-			aport = get_client_port(&subs->info.sender, &c);
+		struct snd_seq_client *c __free(snd_seq_client) = NULL;
+		struct snd_seq_client_port *aport __free(snd_seq_port) =
+			is_src ?
+			get_client_port(&subs->info.dest, &c) :
+			get_client_port(&subs->info.sender, &c);
 		delete_and_unsubscribe_port(client, port, subs, is_src, false);
 
 		if (!aport) {
diff --git a/sound/core/seq/seq_queue.c b/sound/core/seq/seq_queue.c
index f5c0e401c8ae..c0c5e1424c5a 100644
--- a/sound/core/seq/seq_queue.c
+++ b/sound/core/seq/seq_queue.c
@@ -211,8 +211,9 @@ struct snd_seq_queue *snd_seq_queue_find_name(char *name)
 	int i;
 
 	for (i = 0; i < SNDRV_SEQ_MAX_QUEUES; i++) {
-		struct snd_seq_queue *q __free(snd_seq_queue) = NULL;
-		q = queueptr(i);
+		struct snd_seq_queue *q __free(snd_seq_queue) =
+			queueptr(i);
+
 		if (q) {
 			if (strncmp(q->name, name, sizeof(q->name)) == 0)
 				return no_free_ptr(q);
@@ -285,12 +286,13 @@ void snd_seq_check_queue(struct snd_seq_queue *q, int atomic, int hop)
 int snd_seq_enqueue_event(struct snd_seq_event_cell *cell, int atomic, int hop)
 {
 	int dest, err;
-	struct snd_seq_queue *q __free(snd_seq_queue) = NULL;
 
 	if (snd_BUG_ON(!cell))
 		return -EINVAL;
 	dest = cell->event.queue;	/* destination queue */
-	q = queueptr(dest);
+
+	struct snd_seq_queue *q __free(snd_seq_queue) =
+		queueptr(dest);
 	if (q == NULL)
 		return -EINVAL;
 	/* handle relative time stamps, convert them into absolute */
@@ -403,10 +405,10 @@ int snd_seq_queue_set_owner(int queueid, int client, int locked)
 int snd_seq_queue_timer_open(int queueid)
 {
 	int result = 0;
-	struct snd_seq_queue *queue __free(snd_seq_queue) = NULL;
 	struct snd_seq_timer *tmr;
+	struct snd_seq_queue *queue __free(snd_seq_queue) =
+		queueptr(queueid);
 
-	queue = queueptr(queueid);
 	if (queue == NULL)
 		return -EINVAL;
 	tmr = queue->timer;
@@ -423,10 +425,10 @@ int snd_seq_queue_timer_open(int queueid)
  */
 int snd_seq_queue_timer_close(int queueid)
 {
-	struct snd_seq_queue *queue __free(snd_seq_queue) = NULL;
 	int result = 0;
+	struct snd_seq_queue *queue __free(snd_seq_queue) =
+		queueptr(queueid);
 
-	queue = queueptr(queueid);
 	if (queue == NULL)
 		return -EINVAL;
 	snd_seq_timer_close(queue);
@@ -479,9 +481,9 @@ static void queue_use(struct snd_seq_queue *queue, int client, int use)
  */
 int snd_seq_queue_use(int queueid, int client, int use)
 {
-	struct snd_seq_queue *queue __free(snd_seq_queue) = NULL;
+	struct snd_seq_queue *queue __free(snd_seq_queue) =
+		queueptr(queueid);
 
-	queue = queueptr(queueid);
 	if (queue == NULL)
 		return -EINVAL;
 	guard(mutex)(&queue->timer_mutex);
@@ -496,9 +498,9 @@ int snd_seq_queue_use(int queueid, int client, int use)
  */
 int snd_seq_queue_is_used(int queueid, int client)
 {
-	struct snd_seq_queue *q __free(snd_seq_queue) = NULL;
+	struct snd_seq_queue *q __free(snd_seq_queue) =
+		queueptr(queueid);
 
-	q = queueptr(queueid);
 	if (q == NULL)
 		return -EINVAL; /* invalid queue */
 	return test_bit(client, q->clients_bitmap) ? 1 : 0;
@@ -642,11 +644,11 @@ static void snd_seq_queue_process_event(struct snd_seq_queue *q,
  */
 int snd_seq_control_queue(struct snd_seq_event *ev, int atomic, int hop)
 {
-	struct snd_seq_queue *q __free(snd_seq_queue) = NULL;
-
 	if (snd_BUG_ON(!ev))
 		return -EINVAL;
-	q = queueptr(ev->data.queue.queue);
+
+	struct snd_seq_queue *q __free(snd_seq_queue) =
+		queueptr(ev->data.queue.queue);
 
 	if (q == NULL)
 		return -EINVAL;
diff --git a/sound/core/seq/seq_ump_client.c b/sound/core/seq/seq_ump_client.c
index 27247babb16d..7bc18415a540 100644
--- a/sound/core/seq/seq_ump_client.c
+++ b/sound/core/seq/seq_ump_client.c
@@ -214,13 +214,13 @@ static bool skip_group(struct seq_ump_client *client, struct snd_ump_group *grou
 static int seq_ump_group_init(struct seq_ump_client *client, int group_index)
 {
 	struct snd_ump_group *group = &client->ump->groups[group_index];
-	struct snd_seq_port_info *port __free(kfree) = NULL;
 	struct snd_seq_port_callback pcallbacks;
 
 	if (skip_group(client, group))
 		return 0;
 
-	port = kzalloc(sizeof(*port), GFP_KERNEL);
+	struct snd_seq_port_info *port __free(kfree) =
+		kzalloc(sizeof(*port), GFP_KERNEL);
 	if (!port)
 		return -ENOMEM;
 
@@ -243,12 +243,12 @@ static int seq_ump_group_init(struct seq_ump_client *client, int group_index)
 /* update the sequencer ports; called from notify_fb_change callback */
 static void update_port_infos(struct seq_ump_client *client)
 {
-	struct snd_seq_port_info *old __free(kfree) = NULL;
-	struct snd_seq_port_info *new __free(kfree) = NULL;
 	int i, err;
 
-	old = kzalloc(sizeof(*old), GFP_KERNEL);
-	new = kzalloc(sizeof(*new), GFP_KERNEL);
+	struct snd_seq_port_info *old __free(kfree) =
+		kzalloc(sizeof(*old), GFP_KERNEL);
+	struct snd_seq_port_info *new __free(kfree) =
+		kzalloc(sizeof(*new), GFP_KERNEL);
 	if (!old || !new)
 		return;
 
@@ -278,12 +278,12 @@ static void update_port_infos(struct seq_ump_client *client)
 /* create a UMP Endpoint port */
 static int create_ump_endpoint_port(struct seq_ump_client *client)
 {
-	struct snd_seq_port_info *port __free(kfree) = NULL;
 	struct snd_seq_port_callback pcallbacks;
 	unsigned int rawmidi_info = client->ump->core.info_flags;
 	int err;
 
-	port = kzalloc(sizeof(*port), GFP_KERNEL);
+	struct snd_seq_port_info *port __free(kfree) =
+		kzalloc(sizeof(*port), GFP_KERNEL);
 	if (!port)
 		return -ENOMEM;
 
@@ -452,9 +452,8 @@ static const struct snd_seq_ump_ops seq_ump_ops = {
 };
 
 /* create a sequencer client and ports for the given UMP endpoint */
-static int snd_seq_ump_probe(struct device *_dev)
+static int snd_seq_ump_probe(struct snd_seq_device *dev)
 {
-	struct snd_seq_device *dev = to_seq_dev(_dev);
 	struct snd_ump_endpoint *ump = dev->private_data;
 	struct snd_card *card = dev->card;
 	struct seq_ump_client *client;
@@ -513,21 +512,19 @@ static int snd_seq_ump_probe(struct device *_dev)
 }
 
 /* remove a sequencer client */
-static int snd_seq_ump_remove(struct device *_dev)
+static void snd_seq_ump_remove(struct snd_seq_device *dev)
 {
-	struct snd_seq_device *dev = to_seq_dev(_dev);
 	struct snd_ump_endpoint *ump = dev->private_data;
 
 	if (ump->seq_client)
 		seq_ump_client_free(ump->seq_client);
-	return 0;
 }
 
 static struct snd_seq_driver seq_ump_driver = {
+	.probe = snd_seq_ump_probe,
+	.remove = snd_seq_ump_remove,
 	.driver = {
 		.name = KBUILD_MODNAME,
-		.probe = snd_seq_ump_probe,
-		.remove = snd_seq_ump_remove,
 	},
 	.id = SNDRV_SEQ_DEV_ID_UMP,
 	.argsize = 0,
diff --git a/sound/core/seq/seq_virmidi.c b/sound/core/seq/seq_virmidi.c
index 9e7fd4993a10..574493fbd50d 100644
--- a/sound/core/seq/seq_virmidi.c
+++ b/sound/core/seq/seq_virmidi.c
@@ -361,13 +361,13 @@ static int snd_virmidi_dev_attach_seq(struct snd_virmidi_dev *rdev)
 {
 	int client;
 	struct snd_seq_port_callback pcallbacks;
-	struct snd_seq_port_info *pinfo __free(kfree) = NULL;
 	int err;
 
 	if (rdev->client >= 0)
 		return 0;
 
-	pinfo = kzalloc(sizeof(*pinfo), GFP_KERNEL);
+	struct snd_seq_port_info *pinfo __free(kfree) =
+		kzalloc(sizeof(*pinfo), GFP_KERNEL);
 	if (!pinfo)
 		return -ENOMEM;
 
diff --git a/sound/core/seq_device.c b/sound/core/seq_device.c
index bac9f8603734..1b062d6b17ea 100644
--- a/sound/core/seq_device.c
+++ b/sound/core/seq_device.c
@@ -49,9 +49,31 @@ static int snd_seq_bus_match(struct device *dev, const struct device_driver *drv
 		sdrv->argsize == sdev->argsize;
 }
 
+static int snd_seq_bus_probe(struct device *dev)
+{
+	struct snd_seq_device *sdev = to_seq_dev(dev);
+	const struct snd_seq_driver *sdrv = to_seq_drv(dev->driver);
+
+	if (sdrv->probe)
+		return sdrv->probe(sdev);
+	else
+		return 0;
+}
+
+static void snd_seq_bus_remove(struct device *dev)
+{
+	struct snd_seq_device *sdev = to_seq_dev(dev);
+	const struct snd_seq_driver *sdrv = to_seq_drv(dev->driver);
+
+	if (sdrv->remove)
+		sdrv->remove(sdev);
+}
+
 static const struct bus_type snd_seq_bus_type = {
 	.name = "snd_seq",
 	.match = snd_seq_bus_match,
+	.probe = snd_seq_bus_probe,
+	.remove = snd_seq_bus_remove,
 };
 
 /*
@@ -247,10 +269,12 @@ EXPORT_SYMBOL(snd_seq_device_new);
  */
 int __snd_seq_driver_register(struct snd_seq_driver *drv, struct module *mod)
 {
-	if (WARN_ON(!drv->driver.name || !drv->id))
+	if (WARN_ON(!drv->driver.name || !drv->id || drv->driver.probe || drv->driver.remove))
 		return -EINVAL;
+
 	drv->driver.bus = &snd_seq_bus_type;
 	drv->driver.owner = mod;
+
 	return driver_register(&drv->driver);
 }
 EXPORT_SYMBOL_GPL(__snd_seq_driver_register);
diff --git a/sound/core/timer.c b/sound/core/timer.c
index d9fff5c87613..9a4a1748ff80 100644
--- a/sound/core/timer.c
+++ b/sound/core/timer.c
@@ -1614,12 +1614,12 @@ static int snd_timer_user_next_device(struct snd_timer_id __user *_tid)
 static int snd_timer_user_ginfo(struct file *file,
 				struct snd_timer_ginfo __user *_ginfo)
 {
-	struct snd_timer_ginfo *ginfo __free(kfree) = NULL;
 	struct snd_timer_id tid;
 	struct snd_timer *t;
 	struct list_head *p;
+	struct snd_timer_ginfo *ginfo __free(kfree) =
+		memdup_user(_ginfo, sizeof(*ginfo));
 
-	ginfo = memdup_user(_ginfo, sizeof(*ginfo));
 	if (IS_ERR(ginfo))
 		return PTR_ERR(ginfo);
 
@@ -1756,7 +1756,6 @@ static int snd_timer_user_info(struct file *file,
 			       struct snd_timer_info __user *_info)
 {
 	struct snd_timer_user *tu;
-	struct snd_timer_info *info __free(kfree) = NULL;
 	struct snd_timer *t;
 
 	tu = file->private_data;
@@ -1766,7 +1765,8 @@ static int snd_timer_user_info(struct file *file,
 	if (!t)
 		return -EBADFD;
 
-	info = kzalloc(sizeof(*info), GFP_KERNEL);
+	struct snd_timer_info *info __free(kfree) =
+		kzalloc(sizeof(*info), GFP_KERNEL);
 	if (! info)
 		return -ENOMEM;
 	info->card = t->card ? t->card->number : -1;
@@ -2192,10 +2192,10 @@ static int snd_utimer_ioctl_create(struct file *file,
 				   struct snd_timer_uinfo __user *_utimer_info)
 {
 	struct snd_utimer *utimer;
-	struct snd_timer_uinfo *utimer_info __free(kfree) = NULL;
 	int err, timer_fd;
+	struct snd_timer_uinfo *utimer_info __free(kfree) =
+		memdup_user(_utimer_info, sizeof(*utimer_info));
 
-	utimer_info = memdup_user(_utimer_info, sizeof(*utimer_info));
 	if (IS_ERR(utimer_info))
 		return PTR_ERR(utimer_info);
 
diff --git a/sound/core/vmaster.c b/sound/core/vmaster.c
index c657659b236c..76cc64245f5d 100644
--- a/sound/core/vmaster.c
+++ b/sound/core/vmaster.c
@@ -56,10 +56,10 @@ struct link_follower {
 
 static int follower_update(struct link_follower *follower)
 {
-	struct snd_ctl_elem_value *uctl __free(kfree) = NULL;
 	int err, ch;
+	struct snd_ctl_elem_value *uctl __free(kfree) =
+		kzalloc(sizeof(*uctl), GFP_KERNEL);
 
-	uctl = kzalloc(sizeof(*uctl), GFP_KERNEL);
 	if (!uctl)
 		return -ENOMEM;
 	uctl->id = follower->follower.id;
@@ -74,7 +74,6 @@ static int follower_update(struct link_follower *follower)
 /* get the follower ctl info and save the initial values */
 static int follower_init(struct link_follower *follower)
 {
-	struct snd_ctl_elem_info *uinfo __free(kfree) = NULL;
 	int err;
 
 	if (follower->info.count) {
@@ -84,7 +83,8 @@ static int follower_init(struct link_follower *follower)
 		return 0;
 	}
 
-	uinfo = kmalloc(sizeof(*uinfo), GFP_KERNEL);
+	struct snd_ctl_elem_info *uinfo __free(kfree) =
+		kmalloc(sizeof(*uinfo), GFP_KERNEL);
 	if (!uinfo)
 		return -ENOMEM;
 	uinfo->id = follower->follower.id;
@@ -341,9 +341,9 @@ static int master_get(struct snd_kcontrol *kcontrol,
 static int sync_followers(struct link_master *master, int old_val, int new_val)
 {
 	struct link_follower *follower;
-	struct snd_ctl_elem_value *uval __free(kfree) = NULL;
+	struct snd_ctl_elem_value *uval __free(kfree) =
+		kmalloc(sizeof(*uval), GFP_KERNEL);
 
-	uval = kmalloc(sizeof(*uval), GFP_KERNEL);
 	if (!uval)
 		return -ENOMEM;
 	list_for_each_entry(follower, &master->followers, list) {
diff --git a/sound/drivers/opl3/opl3_seq.c b/sound/drivers/opl3/opl3_seq.c
index d3278428d360..b77d8e8f12fb 100644
--- a/sound/drivers/opl3/opl3_seq.c
+++ b/sound/drivers/opl3/opl3_seq.c
@@ -201,9 +201,8 @@ static int snd_opl3_synth_create_port(struct snd_opl3 * opl3)
 
 /* ------------------------------ */
 
-static int snd_opl3_seq_probe(struct device *_dev)
+static int snd_opl3_seq_probe(struct snd_seq_device *dev)
 {
-	struct snd_seq_device *dev = to_seq_dev(_dev);
 	struct snd_opl3 *opl3;
 	int client, err;
 	char name[32];
@@ -244,14 +243,13 @@ static int snd_opl3_seq_probe(struct device *_dev)
 	return 0;
 }
 
-static int snd_opl3_seq_remove(struct device *_dev)
+static void snd_opl3_seq_remove(struct snd_seq_device *dev)
 {
-	struct snd_seq_device *dev = to_seq_dev(_dev);
 	struct snd_opl3 *opl3;
 
 	opl3 = *(struct snd_opl3 **)SNDRV_SEQ_DEVICE_ARGPTR(dev);
 	if (opl3 == NULL)
-		return -EINVAL;
+		return;
 
 #if IS_ENABLED(CONFIG_SND_SEQUENCER_OSS)
 	snd_opl3_free_seq_oss(opl3);
@@ -260,14 +258,13 @@ static int snd_opl3_seq_remove(struct device *_dev)
 		snd_seq_delete_kernel_client(opl3->seq_client);
 		opl3->seq_client = -1;
 	}
-	return 0;
 }
 
 static struct snd_seq_driver opl3_seq_driver = {
+	.probe = snd_opl3_seq_probe,
+	.remove = snd_opl3_seq_remove,
 	.driver = {
 		.name = KBUILD_MODNAME,
-		.probe = snd_opl3_seq_probe,
-		.remove = snd_opl3_seq_remove,
 	},
 	.id = SNDRV_SEQ_DEV_ID_OPL3,
 	.argsize = sizeof(struct snd_opl3 *),
diff --git a/sound/drivers/opl4/opl4_seq.c b/sound/drivers/opl4/opl4_seq.c
index 7bb22089a093..fd6f15be6109 100644
--- a/sound/drivers/opl4/opl4_seq.c
+++ b/sound/drivers/opl4/opl4_seq.c
@@ -118,9 +118,8 @@ static void snd_opl4_seq_free_port(void *private_data)
 	snd_midi_channel_free_set(opl4->chset);
 }
 
-static int snd_opl4_seq_probe(struct device *_dev)
+static int snd_opl4_seq_probe(struct snd_seq_device *dev)
 {
-	struct snd_seq_device *dev = to_seq_dev(_dev);
 	struct snd_opl4 *opl4;
 	int client;
 	struct snd_seq_port_callback pcallbacks;
@@ -175,27 +174,25 @@ static int snd_opl4_seq_probe(struct device *_dev)
 	return 0;
 }
 
-static int snd_opl4_seq_remove(struct device *_dev)
+static void snd_opl4_seq_remove(struct snd_seq_device *dev)
 {
-	struct snd_seq_device *dev = to_seq_dev(_dev);
 	struct snd_opl4 *opl4;
 
 	opl4 = *(struct snd_opl4 **)SNDRV_SEQ_DEVICE_ARGPTR(dev);
 	if (!opl4)
-		return -EINVAL;
+		return;
 
 	if (opl4->seq_client >= 0) {
 		snd_seq_delete_kernel_client(opl4->seq_client);
 		opl4->seq_client = -1;
 	}
-	return 0;
 }
 
 static struct snd_seq_driver opl4_seq_driver = {
+	.probe = snd_opl4_seq_probe,
+	.remove = snd_opl4_seq_remove,
 	.driver = {
 		.name = KBUILD_MODNAME,
-		.probe = snd_opl4_seq_probe,
-		.remove = snd_opl4_seq_remove,
 	},
 	.id = SNDRV_SEQ_DEV_ID_OPL4,
 	.argsize = sizeof(struct snd_opl4 *),
diff --git a/sound/hda/codecs/generic.c b/sound/hda/codecs/generic.c
index 7bcf9aef8275..443500a3518f 100644
--- a/sound/hda/codecs/generic.c
+++ b/sound/hda/codecs/generic.c
@@ -1984,15 +1984,15 @@ static int parse_output_paths(struct hda_codec *codec)
 {
 	struct hda_gen_spec *spec = codec->spec;
 	struct auto_pin_cfg *cfg = &spec->autocfg;
-	struct auto_pin_cfg *best_cfg __free(kfree) = NULL;
 	unsigned int val;
 	int best_badness = INT_MAX;
 	int badness;
 	bool fill_hardwired = true, fill_mio_first = true;
 	bool best_wired = true, best_mio = true;
 	bool hp_spk_swapped = false;
+	struct auto_pin_cfg *best_cfg __free(kfree) =
+		kmalloc(sizeof(*best_cfg), GFP_KERNEL);
 
-	best_cfg = kmalloc(sizeof(*best_cfg), GFP_KERNEL);
 	if (!best_cfg)
 		return -ENOMEM;
 	*best_cfg = *cfg;
diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c
index 171a71457ec3..bf49ef490c97 100644
--- a/sound/hda/codecs/realtek/alc269.c
+++ b/sound/hda/codecs/realtek/alc269.c
@@ -1656,6 +1656,18 @@ static void alc236_fixup_hp_mute_led_micmute_vref(struct hda_codec *codec,
 	alc236_fixup_hp_micmute_led_vref(codec, fix, action);
 }
 
+static void alc236_fixup_hp_mute_led_micmute_gpio(struct hda_codec *codec,
+				const struct hda_fixup *fix, int action)
+{
+	struct alc_spec *spec = codec->spec;
+
+	if (action == HDA_FIXUP_ACT_PRE_PROBE)
+		spec->micmute_led_polarity = 1;
+
+	alc236_fixup_hp_mute_led_coefbit2(codec, fix, action);
+	alc_fixup_hp_gpio_led(codec, action, 0x00, 0x01);
+}
+
 static inline void alc298_samsung_write_coef_pack(struct hda_codec *codec,
 						  const unsigned short coefs[2])
 {
@@ -2904,7 +2916,6 @@ static void find_cirrus_companion_amps(struct hda_codec *cdc)
 {
 	struct device *dev = hda_codec_dev(cdc);
 	struct acpi_device *adev;
-	struct fwnode_handle *fwnode __free(fwnode_handle) = NULL;
 	const char *bus = NULL;
 	static const struct {
 		const char *hid;
@@ -2934,7 +2945,8 @@ static void find_cirrus_companion_amps(struct hda_codec *cdc)
 			bus = "spi";
 	}
 
-	fwnode = fwnode_handle_get(acpi_fwnode_handle(adev));
+	struct fwnode_handle *fwnode __free(fwnode_handle) =
+		fwnode_handle_get(acpi_fwnode_handle(adev));
 	acpi_dev_put(adev);
 
 	if (!bus) {
@@ -3753,6 +3765,7 @@ enum {
 	ALC295_FIXUP_DELL_TAS2781_I2C,
 	ALC245_FIXUP_TAS2781_SPI_2,
 	ALC287_FIXUP_TXNW2781_I2C,
+	ALC287_FIXUP_TXNW2781_I2C_ASUS,
 	ALC287_FIXUP_YOGA7_14ARB7_I2C,
 	ALC245_FIXUP_HP_MUTE_LED_COEFBIT,
 	ALC245_FIXUP_HP_MUTE_LED_V1_COEFBIT,
@@ -5326,9 +5339,7 @@ static const struct hda_fixup alc269_fixups[] = {
 	},
 	[ALC236_FIXUP_HP_MUTE_LED_MICMUTE_GPIO] = {
 		.type = HDA_FIXUP_FUNC,
-		.v.func = alc236_fixup_hp_mute_led_coefbit2,
-		.chained = true,
-		.chain_id = ALC236_FIXUP_HP_GPIO_LED,
+		.v.func = alc236_fixup_hp_mute_led_micmute_gpio,
 	},
 	[ALC236_FIXUP_LENOVO_INV_DMIC] = {
 		.type = HDA_FIXUP_FUNC,
@@ -6053,6 +6064,12 @@ static const struct hda_fixup alc269_fixups[] = {
 		.chained = true,
 		.chain_id = ALC285_FIXUP_THINKPAD_HEADSET_JACK,
 	},
+	[ALC287_FIXUP_TXNW2781_I2C_ASUS] = {
+		.type = HDA_FIXUP_FUNC,
+		.v.func = tas2781_fixup_txnw_i2c,
+		.chained = true,
+		.chain_id = ALC294_FIXUP_ASUS_SPK,
+	},
 	[ALC287_FIXUP_YOGA7_14ARB7_I2C] = {
 		.type = HDA_FIXUP_FUNC,
 		.v.func = yoga7_14arb7_fixup_i2c,
@@ -6771,6 +6788,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x8e61, "HP Trekker ", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x103c, 0x8e62, "HP Trekker ", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x103c, 0x8e8a, "HP NexusX", ALC245_FIXUP_HP_TAS2781_I2C_MUTE_LED),
+	SND_PCI_QUIRK(0x103c, 0x8e9c, "HP 16 Clipper OmniBook X X360", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x103c, 0x8e9d, "HP 17 Turbine OmniBook X UMA", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x103c, 0x8e9e, "HP 17 Turbine OmniBook X UMA", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x103c, 0x8eb6, "HP Abe A6U", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_GPIO),
@@ -6795,6 +6813,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x8f40, "HP ZBook 8 G2a 14", ALC245_FIXUP_HP_TAS2781_I2C_MUTE_LED),
 	SND_PCI_QUIRK(0x103c, 0x8f41, "HP ZBook 8 G2a 16", ALC245_FIXUP_HP_TAS2781_I2C_MUTE_LED),
 	SND_PCI_QUIRK(0x103c, 0x8f42, "HP ZBook 8 G2a 14W", ALC245_FIXUP_HP_TAS2781_I2C_MUTE_LED),
+	SND_PCI_QUIRK(0x103c, 0x8f57, "HP Trekker G7JC", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x103c, 0x8f62, "HP ZBook 8 G2a 16W", ALC245_FIXUP_HP_TAS2781_I2C_MUTE_LED),
 	SND_PCI_QUIRK(0x1043, 0x1032, "ASUS VivoBook X513EA", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0x1043, 0x1034, "ASUS GU605C", ALC285_FIXUP_ASUS_GU605_SPI_SPEAKER2_TO_DAC1),
@@ -6827,8 +6846,8 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1043, 0x12f0, "ASUS X541UV", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0x1043, 0x1313, "Asus K42JZ", ALC269VB_FIXUP_ASUS_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0x1043, 0x1314, "ASUS GA605K", ALC285_FIXUP_ASUS_GA605K_HEADSET_MIC),
-	SND_PCI_QUIRK(0x1043, 0x1384, "ASUS RC73XA", ALC287_FIXUP_TXNW2781_I2C),
-	SND_PCI_QUIRK(0x1043, 0x1394, "ASUS RC73YA", ALC287_FIXUP_TXNW2781_I2C),
+	SND_PCI_QUIRK(0x1043, 0x1384, "ASUS RC73XA", ALC287_FIXUP_TXNW2781_I2C_ASUS),
+	SND_PCI_QUIRK(0x1043, 0x1394, "ASUS RC73YA", ALC287_FIXUP_TXNW2781_I2C_ASUS),
 	SND_PCI_QUIRK(0x1043, 0x13b0, "ASUS Z550SA", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0x1043, 0x1427, "Asus Zenbook UX31E", ALC269VB_FIXUP_ASUS_ZENBOOK),
 	SND_PCI_QUIRK(0x1043, 0x1433, "ASUS GX650PY/PZ/PV/PU/PYV/PZV/PIV/PVV", ALC285_FIXUP_ASUS_I2C_HEADSET_MIC),
@@ -7296,6 +7315,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1d72, 0x1901, "RedmiBook 14", ALC256_FIXUP_ASUS_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1d72, 0x1945, "Redmi G", ALC256_FIXUP_ASUS_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1d72, 0x1947, "RedmiBook Air", ALC255_FIXUP_XIAOMI_HEADSET_MIC),
+	SND_PCI_QUIRK(0x1e39, 0xca14, "MEDION NM14LNL", ALC233_FIXUP_MEDION_MTL_SPK),
 	SND_PCI_QUIRK(0x1ee7, 0x2078, "HONOR BRB-X M1010", ALC2XX_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1f66, 0x0105, "Ayaneo Portable Game Player", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x2014, 0x800a, "Positivo ARN50", ALC269_FIXUP_LIMIT_INT_MIC_BOOST),
diff --git a/sound/hda/codecs/realtek/realtek.c b/sound/hda/codecs/realtek/realtek.c
index ca377a5adadb..efe20b450529 100644
--- a/sound/hda/codecs/realtek/realtek.c
+++ b/sound/hda/codecs/realtek/realtek.c
@@ -215,12 +215,13 @@ void alc_update_knob_master(struct hda_codec *codec,
 {
 	unsigned int val;
 	struct snd_kcontrol *kctl;
-	struct snd_ctl_elem_value *uctl __free(kfree) = NULL;
 
 	kctl = snd_hda_find_mixer_ctl(codec, "Master Playback Volume");
 	if (!kctl)
 		return;
-	uctl = kzalloc(sizeof(*uctl), GFP_KERNEL);
+
+	struct snd_ctl_elem_value *uctl __free(kfree) =
+		kzalloc(sizeof(*uctl), GFP_KERNEL);
 	if (!uctl)
 		return;
 	val = snd_hda_codec_read(codec, jack->nid, 0,
diff --git a/sound/hda/common/codec.c b/sound/hda/common/codec.c
index c6d44168c7f9..ffe7c69d5a32 100644
--- a/sound/hda/common/codec.c
+++ b/sound/hda/common/codec.c
@@ -1854,9 +1854,9 @@ static int check_follower_present(struct hda_codec *codec,
 /* call kctl->put with the given value(s) */
 static int put_kctl_with_value(struct snd_kcontrol *kctl, int val)
 {
-	struct snd_ctl_elem_value *ucontrol __free(kfree) = NULL;
+	struct snd_ctl_elem_value *ucontrol __free(kfree) =
+		kzalloc(sizeof(*ucontrol), GFP_KERNEL);
 
-	ucontrol = kzalloc(sizeof(*ucontrol), GFP_KERNEL);
 	if (!ucontrol)
 		return -ENOMEM;
 	ucontrol->value.integer.value[0] = val;
diff --git a/sound/hda/common/sysfs.c b/sound/hda/common/sysfs.c
index f8c8483fd5e5..bedf10b30885 100644
--- a/sound/hda/common/sysfs.c
+++ b/sound/hda/common/sysfs.c
@@ -299,7 +299,6 @@ static void remove_trail_spaces(char *str)
 
 static int parse_hints(struct hda_codec *codec, const char *buf)
 {
-	char *key __free(kfree) = NULL;
 	char *val;
 	struct hda_hint *hint;
 
@@ -308,7 +307,9 @@ static int parse_hints(struct hda_codec *codec, const char *buf)
 		return 0;
 	if (*buf == '=')
 		return -EINVAL;
-	key = kstrndup_noeol(buf, 1024);
+
+	char *key __free(kfree) =
+		kstrndup_noeol(buf, 1024);
 	if (!key)
 		return -ENOMEM;
 	/* extract key and val */
diff --git a/sound/hda/controllers/cix-ipbloq.c b/sound/hda/controllers/cix-ipbloq.c
index 99f9f48e91d4..c1084a915c13 100644
--- a/sound/hda/controllers/cix-ipbloq.c
+++ b/sound/hda/controllers/cix-ipbloq.c
@@ -115,10 +115,8 @@ static int cix_ipbloq_hda_init(struct cix_ipbloq_hda *hda,
 	bus->addr = res->start;
 
 	irq_id = platform_get_irq(pdev, 0);
-	if (irq_id < 0) {
-		dev_err(hda->dev, "failed to get the irq, err = %d\n", irq_id);
+	if (irq_id < 0)
 		return irq_id;
-	}
 
 	err = devm_request_irq(hda->dev, irq_id, azx_interrupt,
 			       0, KBUILD_MODNAME, chip);
diff --git a/sound/hda/controllers/intel.c b/sound/hda/controllers/intel.c
index 1e8e3d61291a..bb9a64d41580 100644
--- a/sound/hda/controllers/intel.c
+++ b/sound/hda/controllers/intel.c
@@ -2555,7 +2555,7 @@ static const struct pci_device_id azx_ids[] = {
 	/* Apollolake (Broxton-P) */
 	{ PCI_DEVICE_DATA(INTEL, HDA_APL, AZX_DRIVER_SKL | AZX_DCAPS_INTEL_BROXTON) },
 	/* Gemini-Lake */
-	{ PCI_DEVICE_DATA(INTEL, HDA_GML, AZX_DRIVER_SKL | AZX_DCAPS_INTEL_BROXTON) },
+	{ PCI_DEVICE_DATA(INTEL, HDA_GLK, AZX_DRIVER_SKL | AZX_DCAPS_INTEL_BROXTON) },
 	/* Haswell */
 	{ PCI_DEVICE_DATA(INTEL, HDA_HSW_0, AZX_DRIVER_HDMI | AZX_DCAPS_INTEL_HASWELL) },
 	{ PCI_DEVICE_DATA(INTEL, HDA_HSW_2, AZX_DRIVER_HDMI | AZX_DCAPS_INTEL_HASWELL) },
diff --git a/sound/hda/core/intel-dsp-config.c b/sound/hda/core/intel-dsp-config.c
index 0c25e87408de..ddb8db3e8e39 100644
--- a/sound/hda/core/intel-dsp-config.c
+++ b/sound/hda/core/intel-dsp-config.c
@@ -154,7 +154,7 @@ static const struct config_entry config_table[] = {
 #if IS_ENABLED(CONFIG_SND_SOC_SOF_GEMINILAKE)
 	{
 		.flags = FLAG_SOF,
-		.device = PCI_DEVICE_ID_INTEL_HDA_GML,
+		.device = PCI_DEVICE_ID_INTEL_HDA_GLK,
 		.dmi_table = (const struct dmi_system_id []) {
 			{
 				.ident = "Google Chromebooks",
@@ -167,7 +167,7 @@ static const struct config_entry config_table[] = {
 	},
 	{
 		.flags = FLAG_SOF,
-		.device = PCI_DEVICE_ID_INTEL_HDA_GML,
+		.device = PCI_DEVICE_ID_INTEL_HDA_GLK,
 		.codec_hid =  &essx_83x6,
 	},
 #endif
diff --git a/sound/isa/sb/emu8000_synth.c b/sound/isa/sb/emu8000_synth.c
index 9bec85ec55b4..3414c6d0695f 100644
--- a/sound/isa/sb/emu8000_synth.c
+++ b/sound/isa/sb/emu8000_synth.c
@@ -21,9 +21,8 @@ MODULE_LICENSE("GPL");
 /*
  * create a new hardware dependent device for Emu8000
  */
-static int snd_emu8000_probe(struct device *_dev)
+static int snd_emu8000_probe(struct snd_seq_device *dev)
 {
-	struct snd_seq_device *dev = to_seq_dev(_dev);
 	struct snd_emu8000 *hw;
 	struct snd_emux *emu;
 
@@ -81,13 +80,12 @@ static int snd_emu8000_probe(struct device *_dev)
 /*
  * free all resources
  */
-static int snd_emu8000_remove(struct device *_dev)
+static void snd_emu8000_remove(struct snd_seq_device *dev)
 {
-	struct snd_seq_device *dev = to_seq_dev(_dev);
 	struct snd_emu8000 *hw;
 
 	if (dev->driver_data == NULL)
-		return 0; /* no synth was allocated actually */
+		return; /* no synth was allocated actually */
 
 	hw = dev->driver_data;
 	if (hw->pcm)
@@ -96,7 +94,6 @@ static int snd_emu8000_remove(struct device *_dev)
 	snd_util_memhdr_free(hw->memhdr);
 	hw->emu = NULL;
 	hw->memhdr = NULL;
-	return 0;
 }
 
 /*
@@ -104,10 +101,10 @@ static int snd_emu8000_remove(struct device *_dev)
  */
 
 static struct snd_seq_driver emu8000_driver = {
+	.probe = snd_emu8000_probe,
+	.remove = snd_emu8000_remove,
 	.driver = {
 		.name = KBUILD_MODNAME,
-		.probe = snd_emu8000_probe,
-		.remove = snd_emu8000_remove,
 	},
 	.id = SNDRV_SEQ_DEV_ID_EMU8000,
 	.argsize = sizeof(struct snd_emu8000 *),
diff --git a/sound/pci/emu10k1/emu10k1_synth.c b/sound/pci/emu10k1/emu10k1_synth.c
index 662d20eb9689..26499c31eaa8 100644
--- a/sound/pci/emu10k1/emu10k1_synth.c
+++ b/sound/pci/emu10k1/emu10k1_synth.c
@@ -16,9 +16,8 @@ MODULE_LICENSE("GPL");
 /*
  * create a new hardware dependent device for Emu10k1
  */
-static int snd_emu10k1_synth_probe(struct device *_dev)
+static int snd_emu10k1_synth_probe(struct snd_seq_device *dev)
 {
-	struct snd_seq_device *dev = to_seq_dev(_dev);
 	struct snd_emux *emux;
 	struct snd_emu10k1 *hw;
 	struct snd_emu10k1_synth_arg *arg;
@@ -64,14 +63,13 @@ static int snd_emu10k1_synth_probe(struct device *_dev)
 	return 0;
 }
 
-static int snd_emu10k1_synth_remove(struct device *_dev)
+static void snd_emu10k1_synth_remove(struct snd_seq_device *dev)
 {
-	struct snd_seq_device *dev = to_seq_dev(_dev);
 	struct snd_emux *emux;
 	struct snd_emu10k1 *hw;
 
 	if (dev->driver_data == NULL)
-		return 0; /* not registered actually */
+		return; /* not registered actually */
 
 	emux = dev->driver_data;
 
@@ -82,7 +80,6 @@ static int snd_emu10k1_synth_remove(struct device *_dev)
 	}
 
 	snd_emux_free(emux);
-	return 0;
 }
 
 /*
@@ -90,10 +87,10 @@ static int snd_emu10k1_synth_remove(struct device *_dev)
  */
 
 static struct snd_seq_driver emu10k1_synth_driver = {
+	.probe = snd_emu10k1_synth_probe,
+	.remove = snd_emu10k1_synth_remove,
 	.driver = {
 		.name = KBUILD_MODNAME,
-		.probe = snd_emu10k1_synth_probe,
-		.remove = snd_emu10k1_synth_remove,
 	},
 	.id = SNDRV_SEQ_DEV_ID_EMU10K1_SYNTH,
 	.argsize = sizeof(struct snd_emu10k1_synth_arg),
diff --git a/sound/pci/riptide/riptide.c b/sound/pci/riptide/riptide.c
index e983cd657e28..f91fe64bf4f9 100644
--- a/sound/pci/riptide/riptide.c
+++ b/sound/pci/riptide/riptide.c
@@ -75,6 +75,7 @@
 */
 
 #include <linux/delay.h>
+#include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/pci.h>
diff --git a/sound/pcmcia/pdaudiocf/pdaudiocf.c b/sound/pcmcia/pdaudiocf/pdaudiocf.c
index 13419837dfb7..a3291e626440 100644
--- a/sound/pcmcia/pdaudiocf/pdaudiocf.c
+++ b/sound/pcmcia/pdaudiocf/pdaudiocf.c
@@ -131,7 +131,13 @@ static int snd_pdacf_probe(struct pcmcia_device *link)
 	link->config_index = 1;
 	link->config_regs = PRESENT_OPTION;
 
-	return pdacf_config(link);
+	err = pdacf_config(link);
+	if (err < 0) {
+		card_list[i] = NULL;
+		snd_card_free(card);
+		return err;
+	}
+	return 0;
 }
 
 
diff --git a/sound/pcmcia/vx/vxpocket.c b/sound/pcmcia/vx/vxpocket.c
index 2e09f2a513a6..9a5c9aa8eec4 100644
--- a/sound/pcmcia/vx/vxpocket.c
+++ b/sound/pcmcia/vx/vxpocket.c
@@ -284,7 +284,13 @@ static int vxpocket_probe(struct pcmcia_device *p_dev)
 
 	vxp->p_dev = p_dev;
 
-	return vxpocket_config(p_dev);
+	err = vxpocket_config(p_dev);
+	if (err < 0) {
+		card_alloc &= ~(1 << i);
+		snd_card_free(card);
+		return err;
+	}
+	return 0;
 }
 
 static void vxpocket_detach(struct pcmcia_device *link)
diff --git a/sound/soc/amd/acp/acp-sdw-legacy-mach.c b/sound/soc/amd/acp/acp-sdw-legacy-mach.c
index fae94b9edd5a..9cb55d592c3c 100644
--- a/sound/soc/amd/acp/acp-sdw-legacy-mach.c
+++ b/sound/soc/amd/acp/acp-sdw-legacy-mach.c
@@ -358,8 +358,6 @@ static int soc_card_dai_links_create(struct snd_soc_card *card)
 	int sdw_be_num = 0, dmic_num = 0;
 	struct asoc_sdw_mc_private *ctx = snd_soc_card_get_drvdata(card);
 	struct snd_soc_acpi_mach_params *mach_params = &mach->mach_params;
-	struct asoc_sdw_endpoint *soc_ends __free(kfree) = NULL;
-	struct asoc_sdw_dailink *soc_dais __free(kfree) = NULL;
 	struct snd_soc_aux_dev *soc_aux;
 	struct snd_soc_codec_conf *codec_conf;
 	struct snd_soc_dai_link *dai_links;
@@ -380,12 +378,14 @@ static int soc_card_dai_links_create(struct snd_soc_card *card)
 	num_confs = num_ends;
 
 	/* One per DAI link, worst case is a DAI link for every endpoint */
-	soc_dais = kcalloc(num_ends, sizeof(*soc_dais), GFP_KERNEL);
+	struct asoc_sdw_dailink *soc_dais __free(kfree) =
+		kcalloc(num_ends, sizeof(*soc_dais), GFP_KERNEL);
 	if (!soc_dais)
 		return -ENOMEM;
 
 	/* One per endpoint, ie. each DAI on each codec/amp */
-	soc_ends = kcalloc(num_ends, sizeof(*soc_ends), GFP_KERNEL);
+	struct asoc_sdw_endpoint *soc_ends __free(kfree) =
+		kcalloc(num_ends, sizeof(*soc_ends), GFP_KERNEL);
 	if (!soc_ends)
 		return -ENOMEM;
 
diff --git a/sound/soc/amd/acp/acp-sdw-sof-mach.c b/sound/soc/amd/acp/acp-sdw-sof-mach.c
index 5677ae63fca9..da815b3f6389 100644
--- a/sound/soc/amd/acp/acp-sdw-sof-mach.c
+++ b/sound/soc/amd/acp/acp-sdw-sof-mach.c
@@ -270,8 +270,6 @@ static int sof_card_dai_links_create(struct snd_soc_card *card)
 	int sdw_be_num = 0, dmic_num = 0;
 	struct asoc_sdw_mc_private *ctx = snd_soc_card_get_drvdata(card);
 	struct snd_soc_acpi_mach_params *mach_params = &mach->mach_params;
-	struct asoc_sdw_endpoint *sof_ends __free(kfree) = NULL;
-	struct asoc_sdw_dailink *sof_dais __free(kfree) = NULL;
 	struct snd_soc_aux_dev *sof_aux;
 	struct snd_soc_codec_conf *codec_conf;
 	struct snd_soc_dai_link *dai_links;
@@ -289,12 +287,14 @@ static int sof_card_dai_links_create(struct snd_soc_card *card)
 	}
 
 	/* One per DAI link, worst case is a DAI link for every endpoint */
-	sof_dais = kcalloc(num_ends, sizeof(*sof_dais), GFP_KERNEL);
+	struct asoc_sdw_dailink *sof_dais __free(kfree) =
+		kcalloc(num_ends, sizeof(*sof_dais), GFP_KERNEL);
 	if (!sof_dais)
 		return -ENOMEM;
 
 	/* One per endpoint, ie. each DAI on each codec/amp */
-	sof_ends = kcalloc(num_ends, sizeof(*sof_ends), GFP_KERNEL);
+	struct asoc_sdw_endpoint *sof_ends __free(kfree) =
+		kcalloc(num_ends, sizeof(*sof_ends), GFP_KERNEL);
 	if (!sof_ends)
 		return -ENOMEM;
 
diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c
index f210a253da9f..bf4d9d336561 100644
--- a/sound/soc/amd/yc/acp6x-mach.c
+++ b/sound/soc/amd/yc/acp6x-mach.c
@@ -661,6 +661,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = {
 			DMI_MATCH(DMI_PRODUCT_NAME, "Bravo 15 C7UCX"),
 		}
 	},
+	{
+		.driver_data = &acp6x_card,
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "HONOR"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "GOH-X"),
+		}
+	},
 	{}
 };
 
diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig
index 061791e61907..93a266a12a3b 100644
--- a/sound/soc/codecs/Kconfig
+++ b/sound/soc/codecs/Kconfig
@@ -2146,6 +2146,7 @@ config SND_SOC_TAS2781_I2C
 config SND_SOC_TAS2783_SDW
 	tristate "Texas Instruments TAS2783 speaker amplifier (sdw)"
 	depends on SOUNDWIRE
+	depends on SND_SOC_SDCA
 	depends on EFI
 	select REGMAP_SOUNDWIRE
 	select REGMAP_SOUNDWIRE_MBQ
diff --git a/sound/soc/codecs/ak4458.c b/sound/soc/codecs/ak4458.c
index 783d2ef21c11..f81cd8cebdd8 100644
--- a/sound/soc/codecs/ak4458.c
+++ b/sound/soc/codecs/ak4458.c
@@ -783,16 +783,12 @@ static int ak4458_i2c_probe(struct i2c_client *i2c)
 
 	pm_runtime_enable(&i2c->dev);
 	regcache_cache_only(ak4458->regmap, true);
-	ak4458_reset(ak4458, false);
 
 	return 0;
 }
 
 static void ak4458_i2c_remove(struct i2c_client *i2c)
 {
-	struct ak4458_priv *ak4458 = i2c_get_clientdata(i2c);
-
-	ak4458_reset(ak4458, true);
 	pm_runtime_disable(&i2c->dev);
 }
 
diff --git a/sound/soc/codecs/cs-amp-lib.c b/sound/soc/codecs/cs-amp-lib.c
index d8f8b0259cd1..b4d183e7501d 100644
--- a/sound/soc/codecs/cs-amp-lib.c
+++ b/sound/soc/codecs/cs-amp-lib.c
@@ -452,7 +452,7 @@ static int _cs_amp_set_efi_calibration_data(struct device *dev, int amp_index, i
 {
 	u64 cal_target = cs_amp_cal_target_u64(in_data);
 	unsigned long num_entries;
-	struct cirrus_amp_efi_data *data __free(kfree) = NULL;
+	struct cirrus_amp_efi_data *data;
 	efi_char16_t *name = CIRRUS_LOGIC_CALIBRATION_EFI_NAME;
 	efi_guid_t *guid = &CIRRUS_LOGIC_CALIBRATION_EFI_GUID;
 	u32 attr = CS_AMP_CAL_DEFAULT_EFI_ATTR;
@@ -515,28 +515,33 @@ static int _cs_amp_set_efi_calibration_data(struct device *dev, int amp_index, i
 
 	num_entries = max(num_amps, amp_index + 1);
 	if (!data || (data->count < num_entries)) {
-		struct cirrus_amp_efi_data *old_data __free(kfree) = no_free_ptr(data);
+		struct cirrus_amp_efi_data *new_data;
 		unsigned int new_data_size = struct_size(data, data, num_entries);
 
-		data = kzalloc(new_data_size, GFP_KERNEL);
-		if (!data)
-			return -ENOMEM;
+		new_data = kzalloc(new_data_size, GFP_KERNEL);
+		if (!new_data) {
+			ret = -ENOMEM;
+			goto err;
+		}
 
-		if (old_data)
-			memcpy(data, old_data, struct_size(old_data, data, old_data->count));
+		if (data) {
+			memcpy(new_data, data, struct_size(data, data, data->count));
+			kfree(data);
+		}
 
+		data = new_data;
 		data->count = num_entries;
 		data->size = new_data_size;
 	}
 
 	data->data[amp_index] = *in_data;
 	ret = cs_amp_set_cal_efi_buffer(dev, name, guid, attr, data);
-	if (ret) {
+	if (ret)
 		dev_err(dev, "Failed writing calibration to EFI: %d\n", ret);
-		return ret;
-	}
+err:
+	kfree(data);
 
-	return 0;
+	return ret;
 }
 
 /**
diff --git a/sound/soc/codecs/rt1320-sdw.c b/sound/soc/codecs/rt1320-sdw.c
index e3f9b03df3aa..9447e742c672 100644
--- a/sound/soc/codecs/rt1320-sdw.c
+++ b/sound/soc/codecs/rt1320-sdw.c
@@ -115,7 +115,8 @@ static const struct reg_sequence rt1320_blind_write[] = {
 static const struct reg_sequence rt1320_vc_blind_write[] = {
 	{ 0xc003, 0xe0 },
 	{ 0xe80a, 0x01 },
-	{ 0xc5c3, 0xf3 },
+	{ 0xc5c3, 0xf2 },
+	{ 0xc5c8, 0x03 },
 	{ 0xc057, 0x51 },
 	{ 0xc054, 0x35 },
 	{ 0xca05, 0xd6 },
@@ -126,8 +127,6 @@ static const struct reg_sequence rt1320_vc_blind_write[] = {
 	{ 0xc609, 0x40 },
 	{ 0xc046, 0xff },
 	{ 0xc045, 0xff },
-	{ 0xda81, 0x14 },
-	{ 0xda8d, 0x14 },
 	{ 0xc044, 0xff },
 	{ 0xc043, 0xff },
 	{ 0xc042, 0xff },
@@ -136,8 +135,8 @@ static const struct reg_sequence rt1320_vc_blind_write[] = {
 	{ 0xcc10, 0x01 },
 	{ 0xc700, 0xf0 },
 	{ 0xc701, 0x13 },
-	{ 0xc901, 0x09 },
-	{ 0xc900, 0xd0 },
+	{ 0xc901, 0x04 },
+	{ 0xc900, 0x73 },
 	{ 0xde03, 0x05 },
 	{ 0xdd0b, 0x0d },
 	{ 0xdd0a, 0xff },
@@ -153,6 +152,7 @@ static const struct reg_sequence rt1320_vc_blind_write[] = {
 	{ 0xf082, 0xff },
 	{ 0xf081, 0xff },
 	{ 0xf080, 0xff },
+	{ 0xe801, 0x01 },
 	{ 0xe802, 0xf8 },
 	{ 0xe803, 0xbe },
 	{ 0xc003, 0xc0 },
@@ -202,7 +202,7 @@ static const struct reg_sequence rt1320_vc_blind_write[] = {
 	{ 0x3fc2bfc3, 0x00 },
 	{ 0x3fc2bfc2, 0x00 },
 	{ 0x3fc2bfc1, 0x00 },
-	{ 0x3fc2bfc0, 0x03 },
+	{ 0x3fc2bfc0, 0x07 },
 	{ 0x0000d486, 0x43 },
 	{ SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x00 },
 	{ 0x1000db00, 0x07 },
@@ -241,9 +241,7 @@ static const struct reg_sequence rt1320_vc_blind_write[] = {
 	{ 0x1000db21, 0x00 },
 	{ 0x1000db22, 0x00 },
 	{ 0x1000db23, 0x00 },
-	{ 0x0000d540, 0x01 },
-	{ 0x0000c081, 0xfc },
-	{ 0x0000f01e, 0x80 },
+	{ 0x0000d540, 0x21 },
 	{ 0xc01b, 0xfc },
 	{ 0xc5d1, 0x89 },
 	{ 0xc5d8, 0x0a },
@@ -502,12 +500,8 @@ static bool rt1320_readable_register(struct device *dev, unsigned int reg)
 	case 0x2000301c:
 	case 0x2000900f:
 	case 0x20009018:
-	case 0x3fc29d80 ... 0x3fc29d83:
-	case 0x3fe2e000 ... 0x3fe2e003:
-	case 0x3fc2ab80 ... 0x3fc2abd4:
-	case 0x3fc2bfc0 ... 0x3fc2bfc8:
-	case 0x3fc2d300 ... 0x3fc2d354:
-	case 0x3fc2dfc0 ... 0x3fc2dfc8:
+	case 0x3fc000c0 ... 0x3fc2dfc8:
+	case 0x3fe00000 ... 0x3fe36fff:
 	/* 0x40801508/0x40801809/0x4080180a/0x40801909/0x4080190a */
 	case SDW_SDCA_CTL(FUNC_NUM_MIC, RT1320_SDCA_ENT_PDE11, RT1320_SDCA_CTL_REQ_POWER_STATE, 0):
 	case SDW_SDCA_CTL(FUNC_NUM_MIC, RT1320_SDCA_ENT_FU113, RT1320_SDCA_CTL_FU_MUTE, CH_01):
@@ -557,6 +551,8 @@ static bool rt1320_volatile_register(struct device *dev, unsigned int reg)
 	case 0xc48c ... 0xc48f:
 	case 0xc560:
 	case 0xc5b5 ... 0xc5b7:
+	case 0xc5c3:
+	case 0xc5c8:
 	case 0xc5fc ... 0xc5ff:
 	case 0xc680 ... 0xc683:
 	case 0xc820:
@@ -590,6 +586,7 @@ static bool rt1320_volatile_register(struct device *dev, unsigned int reg)
 	case 0xdd0c ... 0xdd13:
 	case 0xde02:
 	case 0xdf14 ... 0xdf1b:
+	case 0xe80b:
 	case 0xe83c ... 0xe847:
 	case 0xf01e:
 	case 0xf717 ... 0xf719:
@@ -602,7 +599,7 @@ static bool rt1320_volatile_register(struct device *dev, unsigned int reg)
 	case 0x2000301c:
 	case 0x2000900f:
 	case 0x20009018:
-	case 0x3fc2ab80 ... 0x3fc2abd4:
+	case 0x3fc2ab80 ... 0x3fc2ac4c:
 	case 0x3fc2b780:
 	case 0x3fc2bf80 ... 0x3fc2bf83:
 	case 0x3fc2bfc0 ... 0x3fc2bfc8:
@@ -720,6 +717,13 @@ static int rt1320_read_prop(struct sdw_slave *slave)
 		j++;
 	}
 
+	prop->dp0_prop = devm_kzalloc(&slave->dev, sizeof(*prop->dp0_prop), GFP_KERNEL);
+	if (!prop->dp0_prop)
+		return -ENOMEM;
+
+	prop->dp0_prop->simple_ch_prep_sm = true;
+	prop->dp0_prop->ch_prep_timeout = 10;
+
 	/* set the timeout values */
 	prop->clk_stop_timeout = 64;
 
@@ -754,6 +758,515 @@ static int rt1320_pde_transition_delay(struct rt1320_sdw_priv *rt1320, unsigned
 	return 0;
 }
 
+static void rt1320_data_rw(struct rt1320_sdw_priv *rt1320, unsigned int start,
+			   unsigned char *data, unsigned int size, enum rt1320_rw_type rw)
+{
+	struct device *dev = &rt1320->sdw_slave->dev;
+	unsigned int tmp;
+	int ret = -1;
+	int i, j;
+
+	pm_runtime_set_autosuspend_delay(dev, 20000);
+	pm_runtime_mark_last_busy(dev);
+
+	switch (rw) {
+	case RT1320_BRA_WRITE:
+	case RT1320_BRA_READ:
+		ret = sdw_bpt_send_sync(rt1320->sdw_slave->bus, rt1320->sdw_slave, &rt1320->bra_msg);
+		if (ret < 0)
+			dev_err(dev, "%s: Failed to send BRA message: %d\n", __func__, ret);
+		fallthrough;
+	case RT1320_PARAM_WRITE:
+	case RT1320_PARAM_READ:
+		if (ret < 0) {
+			/* if BRA fails, we try to access by the control word */
+			if (rw == RT1320_BRA_WRITE || rw == RT1320_BRA_READ) {
+				for (i = 0; i < rt1320->bra_msg.sections; i++) {
+					pm_runtime_mark_last_busy(dev);
+					for (j = 0; j < rt1320->bra_msg.sec[i].len; j++) {
+						if (rw == RT1320_BRA_WRITE) {
+							regmap_write(rt1320->regmap,
+								rt1320->bra_msg.sec[i].addr + j, rt1320->bra_msg.sec[i].buf[j]);
+						} else {
+							regmap_read(rt1320->regmap, rt1320->bra_msg.sec[i].addr + j, &tmp);
+							rt1320->bra_msg.sec[i].buf[j] = tmp;
+						}
+					}
+				}
+			} else {
+				for (i = 0; i < size; i++) {
+					if (rw == RT1320_PARAM_WRITE)
+						regmap_write(rt1320->regmap, start + i, data[i]);
+					else {
+						regmap_read(rt1320->regmap, start + i, &tmp);
+						data[i] = tmp;
+					}
+				}
+			}
+		}
+		break;
+	}
+
+	pm_runtime_set_autosuspend_delay(dev, 3000);
+	pm_runtime_mark_last_busy(dev);
+}
+
+static unsigned long long rt1320_rsgain_to_rsratio(struct rt1320_sdw_priv *rt1320, unsigned int rsgain)
+{
+	unsigned long long base = 1000000000U;
+	unsigned long long step = 1960784U;
+	unsigned long long tmp, result;
+
+	if (rsgain == 0 || rsgain == 0x1ff)
+		result = 1000000000;
+	else if (rsgain & 0x100) {
+		tmp = 0xff - (rsgain & 0xff);
+		tmp = tmp * step;
+		result =  base + tmp;
+	} else {
+		tmp = (rsgain & 0xff);
+		tmp = tmp * step;
+		result = base - tmp;
+	}
+
+	return result;
+}
+
+static void rt1320_pr_read(struct rt1320_sdw_priv *rt1320, unsigned int reg, unsigned int *val)
+{
+	unsigned int byte3, byte2, byte1, byte0;
+
+	regmap_write(rt1320->regmap, 0xc483, 0x80);
+	regmap_write(rt1320->regmap, 0xc482, 0x40);
+	regmap_write(rt1320->regmap, 0xc481, 0x0c);
+	regmap_write(rt1320->regmap, 0xc480, 0x10);
+
+	regmap_write(rt1320->regmap, 0xc487, ((reg & 0xff000000) >> 24));
+	regmap_write(rt1320->regmap, 0xc486, ((reg & 0x00ff0000) >> 16));
+	regmap_write(rt1320->regmap, 0xc485, ((reg & 0x0000ff00) >> 8));
+	regmap_write(rt1320->regmap, 0xc484, (reg & 0x000000ff));
+
+	regmap_write(rt1320->regmap, 0xc482, 0xc0);
+
+	regmap_read(rt1320->regmap, 0xc48f, &byte3);
+	regmap_read(rt1320->regmap, 0xc48e, &byte2);
+	regmap_read(rt1320->regmap, 0xc48d, &byte1);
+	regmap_read(rt1320->regmap, 0xc48c, &byte0);
+
+	*val = (byte3 << 24) | (byte2 << 16) | (byte1 << 8) | byte0;
+}
+
+static int rt1320_check_fw_ready(struct rt1320_sdw_priv *rt1320)
+{
+	struct device *dev = &rt1320->sdw_slave->dev;
+	unsigned int tmp, retry = 0;
+	unsigned int cmd_addr;
+
+	switch (rt1320->dev_id) {
+	case RT1320_DEV_ID:
+		cmd_addr = RT1320_CMD_ID;
+		break;
+	case RT1321_DEV_ID:
+		cmd_addr = RT1321_CMD_ID;
+		break;
+	default:
+		dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id);
+		return -EINVAL;
+	}
+
+	pm_runtime_mark_last_busy(dev);
+	/* check the value of cmd_addr becomes to zero */
+	while (retry < 500) {
+		regmap_read(rt1320->regmap, cmd_addr, &tmp);
+		if (tmp == 0)
+			break;
+		usleep_range(1000, 1100);
+		retry++;
+	}
+	if (retry == 500) {
+		dev_warn(dev, "%s FW is NOT ready!", __func__);
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static int rt1320_check_power_state_ready(struct rt1320_sdw_priv *rt1320, enum rt1320_power_state ps)
+{
+	struct device *dev = &rt1320->sdw_slave->dev;
+	unsigned int retry = 0, tmp;
+
+	pm_runtime_mark_last_busy(dev);
+	while (retry < 200) {
+		regmap_read(rt1320->regmap, RT1320_POWER_STATE, &tmp);
+		dev_dbg(dev, "%s, RT1320_POWER_STATE=0x%x\n", __func__, tmp);
+		if (tmp >= ps)
+			break;
+		usleep_range(1000, 1500);
+		retry++;
+	}
+	if (retry == 200) {
+		dev_warn(dev, "%s FW Power State is NOT ready!", __func__);
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static int rt1320_process_fw_param(struct rt1320_sdw_priv *rt1320, unsigned char *buf, unsigned int buf_size)
+{
+	struct device *dev = &rt1320->sdw_slave->dev;
+	struct rt1320_paramcmd *paramhr = (struct rt1320_paramcmd *)buf;
+	unsigned char moudleid = paramhr->moudleid;
+	unsigned char cmdtype = paramhr->commandtype;
+	unsigned int fw_param_addr;
+	unsigned int start_addr;
+	int ret = 0;
+
+	switch (rt1320->dev_id) {
+	case RT1320_DEV_ID:
+		fw_param_addr = RT1320_FW_PARAM_ADDR;
+		start_addr = RT1320_CMD_PARAM_ADDR;
+		break;
+	case RT1321_DEV_ID:
+		fw_param_addr = RT1321_FW_PARAM_ADDR;
+		start_addr = RT1321_CMD_PARAM_ADDR;
+		break;
+	default:
+		dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id);
+		return -EINVAL;
+	}
+
+	ret = rt1320_check_fw_ready(rt1320);
+	if (ret < 0)
+		goto _timeout_;
+
+	/* don't set offset 0x0/0x1, it will be set later*/
+	paramhr->moudleid = 0;
+	paramhr->commandtype = 0;
+	rt1320_data_rw(rt1320, fw_param_addr, buf, buf_size, RT1320_PARAM_WRITE);
+
+	dev_dbg(dev, "%s, moudleid=%d, cmdtype=%d, paramid=%d, paramlength=%d\n", __func__,
+		moudleid, cmdtype, paramhr->paramid, paramhr->paramlength);
+
+	if (cmdtype == RT1320_SET_PARAM) {
+		regmap_write(rt1320->regmap, fw_param_addr, moudleid);
+		regmap_write(rt1320->regmap, fw_param_addr + 1, 0x01);
+	}
+	if (cmdtype == RT1320_GET_PARAM) {
+		regmap_write(rt1320->regmap, fw_param_addr, moudleid);
+		regmap_write(rt1320->regmap, fw_param_addr + 1, 0x02);
+		ret = rt1320_check_fw_ready(rt1320);
+		if (ret < 0)
+			goto _timeout_;
+
+		rt1320_data_rw(rt1320, start_addr, buf + 0x10, paramhr->commandlength, RT1320_PARAM_READ);
+	}
+	return 0;
+
+_timeout_:
+	dev_err(&rt1320->sdw_slave->dev, "%s: FW is NOT ready for SET/GET_PARAM\n", __func__);
+	return ret;
+}
+
+static int rt1320_fw_param_protocol(struct rt1320_sdw_priv *rt1320, enum rt1320_fw_cmdid cmdid,
+				    unsigned int paramid, void *parambuf, unsigned int paramsize)
+{
+	struct device *dev = &rt1320->sdw_slave->dev;
+	unsigned char *tempbuf = NULL;
+	struct rt1320_paramcmd paramhr;
+	int ret = 0;
+
+	tempbuf = kzalloc(sizeof(paramhr) + paramsize, GFP_KERNEL);
+	if (!tempbuf)
+		return -ENOMEM;
+
+	paramhr.moudleid = 1;
+	paramhr.commandtype = cmdid;
+	/* 8 is "sizeof(paramid) + sizeof(paramlength)" */
+	paramhr.commandlength = 8 + paramsize;
+	paramhr.paramid = paramid;
+	paramhr.paramlength = paramsize;
+
+	memcpy(tempbuf, &paramhr, sizeof(paramhr));
+	if (cmdid == RT1320_SET_PARAM)
+		memcpy(tempbuf + sizeof(paramhr), parambuf, paramsize);
+
+	ret = rt1320_process_fw_param(rt1320, tempbuf, sizeof(paramhr) + paramsize);
+	if (ret < 0) {
+		dev_err(dev, "%s: process_fw_param failed\n", __func__);
+		goto _finish_;
+	}
+
+	if (cmdid == RT1320_GET_PARAM)
+		memcpy(parambuf, tempbuf + sizeof(paramhr), paramsize);
+
+_finish_:
+	kfree(tempbuf);
+	return ret;
+}
+
+static void rt1320_set_advancemode(struct rt1320_sdw_priv *rt1320)
+{
+	struct device *dev = &rt1320->sdw_slave->dev;
+	struct rt1320_datafixpoint r0_data[2];
+	unsigned short l_advancegain, r_advancegain;
+	int ret;
+
+	/* Get advance gain/r0 */
+	rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 6, &r0_data[0], sizeof(struct rt1320_datafixpoint));
+	rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 7, &r0_data[1], sizeof(struct rt1320_datafixpoint));
+	l_advancegain = r0_data[0].advancegain;
+	r_advancegain = r0_data[1].advancegain;
+	dev_dbg(dev, "%s, LR advanceGain=0x%x 0x%x\n", __func__, l_advancegain, r_advancegain);
+
+	/* set R0 and enable protection by SetParameter id 6, 7 */
+	r0_data[0].silencedetect = 0;
+	r0_data[0].r0 = rt1320->r0_l_reg;
+	r0_data[1].silencedetect = 0;
+	r0_data[1].r0 = rt1320->r0_r_reg;
+	dev_dbg(dev, "%s, write LR r0=%d, %d\n", __func__, r0_data[0].r0, r0_data[1].r0);
+
+	rt1320_fw_param_protocol(rt1320, RT1320_SET_PARAM, 6, &r0_data[0], sizeof(struct rt1320_datafixpoint));
+	rt1320_fw_param_protocol(rt1320, RT1320_SET_PARAM, 7, &r0_data[1], sizeof(struct rt1320_datafixpoint));
+	ret = rt1320_check_fw_ready(rt1320);
+	if (ret < 0)
+		dev_err(dev, "%s: Failed to set FW param 6,7!\n", __func__);
+
+	if (l_advancegain != 0 && r_advancegain != 0) {
+		regmap_write(rt1320->regmap, 0xdd0b, (l_advancegain & 0xff00) >> 8);
+		regmap_write(rt1320->regmap, 0xdd0a, (l_advancegain & 0xff));
+		regmap_write(rt1320->regmap, 0xdd09, (r_advancegain & 0xff00) >> 8);
+		regmap_write(rt1320->regmap, 0xdd08, (r_advancegain & 0xff));
+		dev_dbg(dev, "%s, set Advance mode gain\n", __func__);
+	}
+}
+
+static int rt1320_invrs_load(struct rt1320_sdw_priv *rt1320)
+{
+	struct device *dev = &rt1320->sdw_slave->dev;
+	unsigned long long l_rsratio, r_rsratio;
+	unsigned int pr_1058, pr_1059, pr_105a;
+	unsigned long long l_invrs, r_invrs;
+	unsigned long long factor = (1 << 28);
+	unsigned int l_rsgain, r_rsgain;
+	struct rt1320_datafixpoint r0_data[2];
+	int ret;
+
+	/* read L/Rch Rs Gain - it uses for compensating the R0 value */
+	rt1320_pr_read(rt1320, 0x1058, &pr_1058);
+	rt1320_pr_read(rt1320, 0x1059, &pr_1059);
+	rt1320_pr_read(rt1320, 0x105a, &pr_105a);
+	l_rsgain = ((pr_1059 & 0x7f) << 2) | ((pr_105a & 0xc0) >> 6);
+	r_rsgain = ((pr_1058 & 0xff) << 1) | ((pr_1059 & 0x80) >> 7);
+	dev_dbg(dev, "%s, LR rsgain=0x%x, 0x%x\n", __func__, l_rsgain, r_rsgain);
+
+	l_rsratio = rt1320_rsgain_to_rsratio(rt1320, l_rsgain);
+	r_rsratio = rt1320_rsgain_to_rsratio(rt1320, r_rsgain);
+	dev_dbg(dev, "%s, LR rsratio=%lld, %lld\n", __func__, l_rsratio, r_rsratio);
+
+	l_invrs = (l_rsratio * factor) / 1000000000U;
+	r_invrs = (r_rsratio * factor) / 1000000000U;
+
+	rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 6, &r0_data[0], sizeof(struct rt1320_datafixpoint));
+	rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 7, &r0_data[1], sizeof(struct rt1320_datafixpoint));
+
+	r0_data[0].invrs = l_invrs;
+	r0_data[1].invrs = r_invrs;
+	dev_dbg(dev, "%s, write DSP LR invrs=0x%x, 0x%x\n", __func__, r0_data[0].invrs, r0_data[1].invrs);
+
+	rt1320_fw_param_protocol(rt1320, RT1320_SET_PARAM, 6, &r0_data[0], sizeof(struct rt1320_datafixpoint));
+	rt1320_fw_param_protocol(rt1320, RT1320_SET_PARAM, 7, &r0_data[1], sizeof(struct rt1320_datafixpoint));
+	ret = rt1320_check_fw_ready(rt1320);
+	if (ret < 0)
+		dev_err(dev, "%s: Failed to set FW param 6,7!\n", __func__);
+
+	return ret;
+}
+
+static void rt1320_calc_r0(struct rt1320_sdw_priv *rt1320)
+{
+	struct device *dev = &rt1320->sdw_slave->dev;
+	unsigned long long l_calir0, r_calir0;
+	const unsigned int factor = (1 << 27);
+
+	l_calir0 = (rt1320->r0_l_reg * 1000) / factor;
+	r_calir0 = (rt1320->r0_r_reg * 1000) / factor;
+
+	dev_dbg(dev, "%s, l_calir0=%lld.%03lld ohm, r_calir0=%lld.%03lld ohm\n", __func__,
+		l_calir0 / 1000, l_calir0 % 1000,
+		r_calir0 / 1000, r_calir0 % 1000);
+}
+
+static void rt1320_calibrate(struct rt1320_sdw_priv *rt1320)
+{
+	struct device *dev = &rt1320->sdw_slave->dev;
+	struct rt1320_datafixpoint audfixpoint[2];
+	unsigned int reg_c5fb, reg_c570, reg_cd00;
+	unsigned int vol_reg[4], fw_ready;
+	unsigned long long l_meanr0, r_meanr0;
+	unsigned int fw_status_addr;
+	int l_re[5], r_re[5];
+	int ret, tmp;
+	unsigned long long factor = (1 << 27);
+	unsigned short l_advancegain, r_advancegain;
+	unsigned int delay_s = 7; /* delay seconds for the calibration */
+
+	if (!rt1320->component)
+		return;
+
+	switch (rt1320->dev_id) {
+	case RT1320_DEV_ID:
+		fw_status_addr = RT1320_DSPFW_STATUS_ADDR;
+		break;
+	case RT1321_DEV_ID:
+		fw_status_addr = RT1321_DSPFW_STATUS_ADDR;
+		break;
+	default:
+		dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id);
+		return;
+	}
+
+	/* set volume 0dB */
+	regmap_read(rt1320->regmap, 0xdd0b, &vol_reg[3]);
+	regmap_read(rt1320->regmap, 0xdd0a, &vol_reg[2]);
+	regmap_read(rt1320->regmap, 0xdd09, &vol_reg[1]);
+	regmap_read(rt1320->regmap, 0xdd08, &vol_reg[0]);
+	regmap_write(rt1320->regmap, 0xdd0b, 0x0f);
+	regmap_write(rt1320->regmap, 0xdd0a, 0xff);
+	regmap_write(rt1320->regmap, 0xdd09, 0x0f);
+	regmap_write(rt1320->regmap, 0xdd08, 0xff);
+
+	regmap_read(rt1320->regmap, 0xc5fb, &reg_c5fb);
+	regmap_read(rt1320->regmap, 0xc570, &reg_c570);
+	regmap_read(rt1320->regmap, 0xcd00, &reg_cd00);
+
+	regmap_write(rt1320->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x00);
+	ret = rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x00);
+	if (ret < 0) {
+		dev_dbg(dev, "%s, PDE=PS0 is NOT ready\n", __func__);
+		goto _finish_;
+	}
+
+	regmap_read(rt1320->regmap, fw_status_addr, &fw_ready);
+	fw_ready &= 0x1;
+	if (!fw_ready) {
+		dev_dbg(dev, "%s, DSP FW is NOT ready. Please load DSP FW first\n", __func__);
+		goto _finish_;
+	}
+
+	ret = rt1320_check_power_state_ready(rt1320, RT1320_NORMAL_STATE);
+	if (ret < 0) {
+		dev_dbg(dev, "%s, DSP FW PS is NOT ready\n", __func__);
+		goto _finish_;
+	}
+
+	if (rt1320->dev_id == RT1320_DEV_ID)
+		regmap_write(rt1320->regmap, 0xc5fb, 0x00);
+	regmap_write(rt1320->regmap, 0xc570, 0x0b);
+	regmap_write(rt1320->regmap, 0xcd00, 0xc5);
+
+	/* disable silence detection */
+	regmap_update_bits(rt1320->regmap, 0xc044, 0xe0, 0x00);
+	dev_dbg(dev, "%s, disable silence detection\n", __func__);
+
+	ret = rt1320_check_power_state_ready(rt1320, RT1320_K_R0_STATE);
+	if (ret < 0) {
+		dev_dbg(dev, "%s, check class D status before k r0\n", __func__);
+		goto _finish_;
+	}
+
+	for (tmp = 0; tmp < delay_s; tmp++) {
+		msleep(1000);
+		pm_runtime_mark_last_busy(dev);
+
+		rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 11, &l_re[0], sizeof(l_re));
+		rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 12, &r_re[0], sizeof(r_re));
+
+		dev_dbg(dev, "%s, LR re=0x%x, 0x%x\n", __func__, l_re[4], r_re[4]);
+		dev_dbg(dev, "%s, waiting for calibration R0...%d seconds\n", __func__, tmp + 1);
+	}
+
+	/* Get Calibration data */
+	rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 11, &l_re[0], sizeof(l_re));
+	rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 12, &r_re[0], sizeof(r_re));
+	dev_dbg(dev, "%s, LR re=0x%x, 0x%x\n", __func__, l_re[4], r_re[4]);
+
+	/* Get advance gain/mean r0 */
+	rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 6, &audfixpoint[0], sizeof(struct rt1320_datafixpoint));
+	l_meanr0 = audfixpoint[0].meanr0;
+	l_advancegain = audfixpoint[0].advancegain;
+	l_meanr0 = ((l_meanr0 * 1000U) / factor);
+	rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 7, &audfixpoint[1], sizeof(struct rt1320_datafixpoint));
+	r_meanr0 = audfixpoint[1].meanr0;
+	r_advancegain = audfixpoint[1].advancegain;
+	r_meanr0 = ((r_meanr0 * 1000U) / factor);
+	dev_dbg(dev, "%s, LR meanr0=%lld, %lld\n", __func__, l_meanr0, r_meanr0);
+	dev_dbg(dev, "%s, LR advanceGain=0x%x, 0x%x\n", __func__, l_advancegain, r_advancegain);
+	dev_dbg(dev, "%s, LR invrs=0x%x, 0x%x\n", __func__, audfixpoint[0].invrs, audfixpoint[1].invrs);
+
+	/* enable silence detection */
+	regmap_update_bits(rt1320->regmap, 0xc044, 0xe0, 0xe0);
+	dev_dbg(dev, "%s, enable silence detection\n", __func__);
+
+	regmap_write(rt1320->regmap, 0xc5fb, reg_c5fb);
+	regmap_write(rt1320->regmap, 0xc570, reg_c570);
+	regmap_write(rt1320->regmap, 0xcd00, reg_cd00);
+
+	rt1320->r0_l_reg = l_re[4];
+	rt1320->r0_r_reg = r_re[4];
+	rt1320->cali_done = true;
+	rt1320_calc_r0(rt1320);
+
+_finish_:
+	regmap_write(rt1320->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x03);
+	rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x03);
+
+	/* advance gain will be set when R0 load, not here */
+	regmap_write(rt1320->regmap, 0xdd0b, vol_reg[3]);
+	regmap_write(rt1320->regmap, 0xdd0a, vol_reg[2]);
+	regmap_write(rt1320->regmap, 0xdd09, vol_reg[1]);
+	regmap_write(rt1320->regmap, 0xdd08, vol_reg[0]);
+}
+
+static int rt1320_r0_cali_get(struct snd_kcontrol *kcontrol,
+			      struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct rt1320_sdw_priv *rt1320 = snd_soc_component_get_drvdata(component);
+
+	ucontrol->value.integer.value[0] = rt1320->cali_done;
+	return 0;
+}
+
+static int rt1320_r0_cali_put(struct snd_kcontrol *kcontrol,
+			      struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct rt1320_sdw_priv *rt1320 = snd_soc_component_get_drvdata(component);
+	struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(rt1320->component);
+	int ret;
+
+	if (!rt1320->hw_init)
+		return 0;
+
+	ret = pm_runtime_resume(component->dev);
+	if (ret < 0 && ret != -EACCES)
+		return ret;
+
+	rt1320->cali_done = false;
+	snd_soc_dapm_mutex_lock(dapm);
+	if (snd_soc_component_get_bias_level(component) == SND_SOC_BIAS_OFF &&
+		ucontrol->value.integer.value[0]) {
+		rt1320_calibrate(rt1320);
+	}
+	snd_soc_dapm_mutex_unlock(dapm);
+
+	return 0;
+}
+
 /*
  * The 'patch code' is written to the patch code area.
  * The patch code area is used for SDCA register expansion flexibility.
@@ -844,6 +1357,453 @@ static void rt1320_vab_preset(struct rt1320_sdw_priv *rt1320)
 	}
 }
 
+static int rt1320_t0_load(struct rt1320_sdw_priv *rt1320, unsigned int l_t0, unsigned int r_t0)
+{
+	struct device *dev = &rt1320->sdw_slave->dev;
+	unsigned int factor = (1 << 22), fw_ready;
+	int l_t0_data[38], r_t0_data[38];
+	unsigned int fw_status_addr;
+	int ret;
+
+	switch (rt1320->dev_id) {
+	case RT1320_DEV_ID:
+		fw_status_addr = RT1320_DSPFW_STATUS_ADDR;
+		break;
+	case RT1321_DEV_ID:
+		fw_status_addr = RT1321_DSPFW_STATUS_ADDR;
+		break;
+	default:
+		dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id);
+		return -EINVAL;
+	}
+
+	regmap_write(rt1320->regmap,
+			SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23,
+				RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x00);
+	rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x00);
+
+	regmap_read(rt1320->regmap, fw_status_addr, &fw_ready);
+	fw_ready &= 0x1;
+	if (!fw_ready) {
+		dev_warn(dev, "%s, DSP FW is NOT ready\n", __func__);
+		goto _exit_;
+	}
+
+	rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 3, &l_t0_data[0], sizeof(l_t0_data));
+	rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 4, &r_t0_data[0], sizeof(r_t0_data));
+
+	l_t0_data[37] = l_t0 * factor;
+	r_t0_data[37] = r_t0 * factor;
+
+	dev_dbg(dev, "%s, write LR t0=0x%x, 0x%x\n", __func__, l_t0_data[37], r_t0_data[37]);
+
+	rt1320_fw_param_protocol(rt1320, RT1320_SET_PARAM, 3, &l_t0_data[0], sizeof(l_t0_data));
+	rt1320_fw_param_protocol(rt1320, RT1320_SET_PARAM, 4, &r_t0_data[0], sizeof(r_t0_data));
+	ret = rt1320_check_fw_ready(rt1320);
+	if (ret < 0)
+		dev_err(dev, "%s: Failed to set FW param 3,4!\n", __func__);
+
+	rt1320->temp_l_calib = l_t0;
+	rt1320->temp_r_calib = r_t0;
+
+	memset(&l_t0_data[0], 0x00, sizeof(l_t0_data));
+	memset(&r_t0_data[0], 0x00, sizeof(r_t0_data));
+	rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 3, &l_t0_data[0], sizeof(l_t0_data));
+	rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 4, &r_t0_data[0], sizeof(r_t0_data));
+	dev_dbg(dev, "%s, read after writing LR t0=0x%x, 0x%x\n", __func__, l_t0_data[37], r_t0_data[37]);
+
+_exit_:
+	regmap_write(rt1320->regmap,
+			SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23,
+				RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x03);
+	rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x03);
+
+	return ret;
+}
+
+static int rt1320_rae_load(struct rt1320_sdw_priv *rt1320)
+{
+	struct device *dev = &rt1320->sdw_slave->dev;
+	static const char func_tag[] = "FUNC";
+	static const char xu_tag[] = "XU";
+	const struct firmware *rae_fw = NULL;
+	unsigned int fw_offset;
+	unsigned char *fw_data;
+	unsigned char *param_data;
+	unsigned int addr, size;
+	unsigned int func, value;
+	const char *dmi_vendor, *dmi_product, *dmi_sku;
+	char vendor[128], product[128], sku[128];
+	char *ptr_vendor, *ptr_product, *ptr_sku;
+	char rae_filename[128];
+	char tag[5];
+	int ret = 0;
+	int retry = 200;
+
+	dmi_vendor = dmi_get_system_info(DMI_SYS_VENDOR);
+	dmi_product = dmi_get_system_info(DMI_PRODUCT_NAME);
+	dmi_sku = dmi_get_system_info(DMI_PRODUCT_SKU);
+
+	if (dmi_vendor && dmi_product && dmi_sku) {
+		strscpy(vendor, dmi_vendor);
+		strscpy(product, dmi_product);
+		strscpy(sku, dmi_sku);
+		ptr_vendor = &vendor[0];
+		ptr_product = &product[0];
+		ptr_sku = &sku[0];
+		ptr_vendor = strsep(&ptr_vendor, " ");
+		ptr_product = strsep(&ptr_product, " ");
+		ptr_sku = strsep(&ptr_sku, " ");
+
+		dev_dbg(dev, "%s: DMI vendor=%s, product=%s, sku=%s\n", __func__,
+			vendor, product, sku);
+
+		snprintf(rae_filename, sizeof(rae_filename),
+			 "realtek/rt1320/rt1320_RAE_%s_%s_%s.dat", vendor, product, sku);
+		dev_dbg(dev, "%s: try to load RAE file %s\n", __func__, rae_filename);
+	} else {
+		dev_warn(dev, "%s: Can't find proper RAE file name\n", __func__);
+		return -EINVAL;
+	}
+
+	regmap_write(rt1320->regmap,
+			SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23,
+				RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x00);
+	rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x00);
+
+	request_firmware(&rae_fw, rae_filename, dev);
+	if (rae_fw) {
+
+		/* RAE CRC clear */
+		regmap_write(rt1320->regmap, 0xe80b, 0x0f);
+
+		/* RAE stop & CRC disable */
+		regmap_update_bits(rt1320->regmap, 0xe803, 0xbc, 0x00);
+
+		while (retry--) {
+			regmap_read(rt1320->regmap, 0xe83f, &value);
+			if (value & 0x40)
+				break;
+			usleep_range(1000, 1100);
+		}
+		if (!retry && !(value & 0x40)) {
+			dev_err(dev, "%s: RAE is not ready to load\n", __func__);
+			return -ETIMEDOUT;
+		}
+
+		dev_dbg(dev, "%s, rae_fw size=0x%lx\n", __func__, rae_fw->size);
+		regcache_cache_bypass(rt1320->regmap, true);
+		for (fw_offset = 0; fw_offset < rae_fw->size;) {
+
+			dev_dbg(dev, "%s, fw_offset=0x%x\n", __func__, fw_offset);
+
+			fw_data = (unsigned char *)&rae_fw->data[fw_offset];
+
+			memcpy(tag, fw_data, 4);
+			tag[4] = '\0';
+			dev_dbg(dev, "%s, tag=%s\n", __func__, tag);
+			if (strcmp(tag, xu_tag) == 0) {
+				dev_dbg(dev, "%s: This is a XU tag", __func__);
+				memcpy(&addr, (fw_data + 4), 4);
+				memcpy(&size, (fw_data + 8), 4);
+				param_data = (unsigned char *)(fw_data + 12);
+
+				dev_dbg(dev, "%s: addr=0x%x, size=0x%x\n", __func__, addr, size);
+
+				/*
+				 * UI register ranges from 0x1000d000 to 0x1000d7ff
+				 * UI registers should be accessed by tuning tool.
+				 * So, there registers should be cached.
+				 */
+				if (addr <= 0x1000d7ff && addr >= 0x1000d000)
+					regcache_cache_bypass(rt1320->regmap, false);
+
+				rt1320_data_rw(rt1320, addr, param_data, size, RT1320_PARAM_WRITE);
+
+				regcache_cache_bypass(rt1320->regmap, true);
+
+				fw_offset += (size + 12);
+			} else if (strcmp(tag, func_tag) == 0) {
+				dev_err(dev, "%s: This is a FUNC tag", __func__);
+
+				memcpy(&func, (fw_data + 4), 4);
+				memcpy(&value, (fw_data + 8), 4);
+
+				dev_dbg(dev, "%s: func=0x%x, value=0x%x\n", __func__, func, value);
+				if (func == 1)  //DelayMs
+					msleep(value);
+
+				fw_offset += 12;
+			} else {
+				dev_err(dev, "%s: This is NOT a XU file (wrong tag)", __func__);
+				break;
+			}
+		}
+
+		regcache_cache_bypass(rt1320->regmap, false);
+		release_firmware(rae_fw);
+
+	} else {
+		dev_err(dev, "%s: Failed to load %s firmware\n", __func__, rae_filename);
+		ret = -EINVAL;
+		goto _exit_;
+	}
+
+	/* RAE CRC enable */
+	regmap_update_bits(rt1320->regmap, 0xe803, 0x0c, 0x0c);
+
+	/* RAE update */
+	regmap_update_bits(rt1320->regmap, 0xe80b, 0x80, 0x00);
+	regmap_update_bits(rt1320->regmap, 0xe80b, 0x80, 0x80);
+
+	/* RAE run */
+	regmap_update_bits(rt1320->regmap, 0xe803, 0x80, 0x80);
+
+	regmap_read(rt1320->regmap, 0xe80b, &value);
+	dev_dbg(dev, "%s: CAE run => 0xe80b reg = 0x%x\n", __func__, value);
+
+	rt1320->rae_update_done = true;
+
+_exit_:
+	regmap_write(rt1320->regmap,
+			SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23,
+				RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x03);
+	rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x03);
+
+	return ret;
+}
+
+static void rt1320_dspfw_load_code(struct rt1320_sdw_priv *rt1320)
+{
+struct rt1320_imageinfo {
+	unsigned int addr;
+	unsigned int size;
+};
+
+struct rt1320_dspfwheader {
+	unsigned int sync;
+	short num;
+	short crc;
+};
+
+	struct snd_soc_dapm_context *dapm =
+		snd_soc_component_get_dapm(rt1320->component);
+	struct device *dev = &rt1320->sdw_slave->dev;
+	unsigned int val, i, fw_offset, fw_ready;
+	unsigned int fw_status_addr;
+	struct rt1320_dspfwheader *fwheader;
+	struct rt1320_imageinfo *ptr_img;
+	struct sdw_bpt_section sec[10];
+	const struct firmware *fw = NULL;
+	unsigned char *fw_data;
+	bool dev_fw_match = false;
+	static const char hdr_sig[] = "AFX";
+	unsigned int hdr_size = 0;
+	const char *dmi_vendor, *dmi_product, *dmi_sku;
+	char vendor[128], product[128], sku[128];
+	char *ptr_vendor, *ptr_product, *ptr_sku;
+	char filename[128];
+
+	switch (rt1320->dev_id) {
+	case RT1320_DEV_ID:
+		fw_status_addr = RT1320_DSPFW_STATUS_ADDR;
+		break;
+	case RT1321_DEV_ID:
+		fw_status_addr = RT1321_DSPFW_STATUS_ADDR;
+		break;
+	default:
+		dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id);
+		return;
+	}
+
+	dmi_vendor = dmi_get_system_info(DMI_SYS_VENDOR);
+	dmi_product = dmi_get_system_info(DMI_PRODUCT_NAME);
+	dmi_sku = dmi_get_system_info(DMI_PRODUCT_SKU);
+
+	if (dmi_vendor && dmi_product && dmi_sku) {
+		strscpy(vendor, dmi_vendor);
+		strscpy(product, dmi_product);
+		strscpy(sku, dmi_sku);
+		ptr_vendor = &vendor[0];
+		ptr_product = &product[0];
+		ptr_sku = &sku[0];
+		ptr_vendor = strsep(&ptr_vendor, " ");
+		ptr_product = strsep(&ptr_product, " ");
+		ptr_sku = strsep(&ptr_sku, " ");
+
+		dev_dbg(dev, "%s: DMI vendor=%s, product=%s, sku=%s\n", __func__,
+			vendor, product, sku);
+
+		snprintf(filename, sizeof(filename),
+			 "realtek/rt1320/rt1320_%s_%s_%s.dat", vendor, product, sku);
+		dev_dbg(dev, "%s: try to load FW file %s\n", __func__, filename);
+	} else if (rt1320->dspfw_name) {
+		snprintf(filename, sizeof(filename), "rt1320_%s.dat",
+			 rt1320->dspfw_name);
+		dev_dbg(dev, "%s: try to load FW file %s\n", __func__, filename);
+	} else {
+		dev_warn(dev, "%s: Can't find proper FW file name\n", __func__);
+		return;
+	}
+
+	snd_soc_dapm_mutex_lock(dapm);
+	regmap_write(rt1320->regmap,
+			SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23,
+				RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x00);
+	rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x00);
+
+	regmap_read(rt1320->regmap, fw_status_addr, &fw_ready);
+	fw_ready &= 0x1;
+	if (fw_ready) {
+		dev_dbg(dev, "%s, DSP FW was already\n", __func__);
+		rt1320->fw_load_done = true;
+		goto _exit_;
+	}
+
+	/* change to IRAM */
+	regmap_update_bits(rt1320->regmap, 0xf01e, 0x80, 0x00);
+
+	request_firmware(&fw, filename, dev);
+	if (fw) {
+		fwheader = (struct rt1320_dspfwheader *)fw->data;
+		dev_dbg(dev, "%s, fw sync = 0x%x, num=%d, crc=0x%x\n", __func__,
+			fwheader->sync, fwheader->num, fwheader->crc);
+
+		if (fwheader->sync != 0x0a1c5679) {
+			dev_err(dev, "%s: FW sync error\n", __func__);
+			release_firmware(fw);
+			goto _exit_;
+		}
+
+		fw_offset = sizeof(struct rt1320_dspfwheader) + (sizeof(struct rt1320_imageinfo) * fwheader->num);
+		dev_dbg(dev, "%s, fw_offset = 0x%x\n", __func__, fw_offset);
+
+		regcache_cache_bypass(rt1320->regmap, true);
+
+		for (i = 0; i < fwheader->num; i++) {
+			ptr_img = (struct rt1320_imageinfo *)&fw->data[sizeof(struct rt1320_dspfwheader) + (sizeof(struct rt1320_imageinfo) * i)];
+
+			dev_dbg(dev, "%s, fw_offset=0x%x, load fw addr=0x%x, size=%d\n", __func__,
+				fw_offset, ptr_img->addr, ptr_img->size);
+
+			fw_data = (unsigned char *)&fw->data[fw_offset];
+
+			/* The binary file has a header of 64 bytes */
+			if (memcmp(fw_data, hdr_sig, sizeof(hdr_sig)) == 0)
+				hdr_size = 64;
+			else
+				hdr_size = 0;
+
+			sec[i].addr = ptr_img->addr;
+			sec[i].len = ptr_img->size - hdr_size;
+			sec[i].buf = fw_data + hdr_size;
+
+			dev_dbg(dev, "%s, hdr_size=%d, sec[%d].buf[0]=0x%x\n",
+				__func__, hdr_size, i, sec[i].buf[0]);
+
+			switch (rt1320->dev_id) {
+			case RT1320_DEV_ID:
+				if (ptr_img->addr == 0x3fc29d80)
+					if (fw_data[9] == '0')
+						dev_fw_match = true;
+				break;
+			case RT1321_DEV_ID:
+				if (ptr_img->addr == 0x3fc00000)
+					if (fw_data[9] == '1')
+						dev_fw_match = true;
+				break;
+			default:
+				dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id);
+				goto _exit_;
+			}
+
+			fw_offset += ptr_img->size;
+		}
+
+		if (dev_fw_match) {
+			dev_dbg(dev, "%s, starting BRA downloading FW..\n", __func__);
+			rt1320->bra_msg.dev_num = rt1320->sdw_slave->dev_num;
+			rt1320->bra_msg.flags = SDW_MSG_FLAG_WRITE;
+			rt1320->bra_msg.sections = fwheader->num;
+			rt1320->bra_msg.sec = &sec[0];
+			rt1320_data_rw(rt1320, 0, NULL, 0, RT1320_BRA_WRITE);
+			dev_dbg(dev, "%s, BRA downloading FW done..\n", __func__);
+		}
+
+		regcache_cache_bypass(rt1320->regmap, false);
+		release_firmware(fw);
+
+		if (!dev_fw_match) {
+			dev_err(dev, "%s: FW file doesn't match to device\n", __func__);
+			goto _exit_;
+		}
+	} else {
+		dev_err(dev, "%s: Failed to load %s firmware\n", __func__, filename);
+		goto _exit_;
+	}
+
+	/* run RAM code */
+	regmap_read(rt1320->regmap, 0x3fc2bfc0, &val);
+	val |= 0x8;
+	regmap_write(rt1320->regmap, 0x3fc2bfc0, val);
+
+	/* clear frame counter */
+	switch (rt1320->dev_id) {
+	case RT1320_DEV_ID:
+		regmap_write(rt1320->regmap, 0x3fc2bfcb, 0x00);
+		regmap_write(rt1320->regmap, 0x3fc2bfca, 0x00);
+		regmap_write(rt1320->regmap, 0x3fc2bfc9, 0x00);
+		regmap_write(rt1320->regmap, 0x3fc2bfc8, 0x00);
+		break;
+	case RT1321_DEV_ID:
+		regmap_write(rt1320->regmap, 0x3fc2dfcb, 0x00);
+		regmap_write(rt1320->regmap, 0x3fc2dfca, 0x00);
+		regmap_write(rt1320->regmap, 0x3fc2dfc9, 0x00);
+		regmap_write(rt1320->regmap, 0x3fc2dfc8, 0x00);
+		break;
+	}
+
+	/* enable DSP FW */
+	regmap_write(rt1320->regmap, 0xc081, 0xfc);
+	regmap_update_bits(rt1320->regmap, 0xf01e, 0x1, 0x0);
+
+	/* RsRatio should restore into DSP FW when FW was ready */
+	rt1320_invrs_load(rt1320);
+
+	/* DSP clock switches to PLL */
+	regmap_write(rt1320->regmap, 0xc081, 0xfc);
+	/* pass DSP settings */
+	regmap_write(rt1320->regmap, 0xc5c3, 0xf3);
+	regmap_write(rt1320->regmap, 0xc5c8, 0x05);
+
+	rt1320->fw_load_done = true;
+
+	pm_runtime_set_autosuspend_delay(dev, 3000);
+	pm_runtime_mark_last_busy(dev);
+
+_exit_:
+	regmap_write(rt1320->regmap,
+			SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23,
+				RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x03);
+	rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x03);
+
+	snd_soc_dapm_mutex_unlock(dapm);
+}
+
+static void rt1320_load_dspfw_work(struct work_struct *work)
+{
+	struct rt1320_sdw_priv *rt1320 =
+		container_of(work, struct rt1320_sdw_priv, load_dspfw_work);
+	int ret;
+
+	ret = pm_runtime_resume(rt1320->component->dev);
+	if (ret < 0 && ret != -EACCES)
+		return;
+
+	dev_dbg(&rt1320->sdw_slave->dev, "%s, Starting to reload DSP FW", __func__);
+	rt1320_dspfw_load_code(rt1320);
+}
+
 static void rt1320_vc_preset(struct rt1320_sdw_priv *rt1320)
 {
 	struct sdw_slave *slave = rt1320->sdw_slave;
@@ -956,6 +1916,10 @@ static int rt1320_io_init(struct device *dev, struct sdw_slave *slave)
 		regmap_write(rt1320->regmap,
 			SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT0, RT1320_SDCA_CTL_FUNC_STATUS, 0),
 			FUNCTION_NEEDS_INITIALIZATION);
+
+		/* reload DSP FW */
+		if (rt1320->fw_load_done)
+			schedule_work(&rt1320->load_dspfw_work);
 	}
 	if (!rt1320->first_hw_init && rt1320->version_id == RT1320_VA && rt1320->dev_id == RT1320_DEV_ID) {
 		regmap_write(rt1320->regmap, SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23,
@@ -1358,6 +2322,223 @@ static SOC_ENUM_SINGLE_DECL(rt1320_rx_data_ch_enum,
 static const DECLARE_TLV_DB_SCALE(out_vol_tlv, -6525, 75, 0);
 static const DECLARE_TLV_DB_SCALE(in_vol_tlv, -1725, 75, 0);
 
+static int rt1320_r0_load(struct rt1320_sdw_priv *rt1320)
+{
+	struct device *dev = regmap_get_device(rt1320->regmap);
+	unsigned int fw_status_addr;
+	unsigned int fw_ready;
+	int ret = 0;
+
+	if (!rt1320->r0_l_reg || !rt1320->r0_r_reg)
+		return -EINVAL;
+
+	switch (rt1320->dev_id) {
+	case RT1320_DEV_ID:
+		fw_status_addr = RT1320_DSPFW_STATUS_ADDR;
+		break;
+	case RT1321_DEV_ID:
+		fw_status_addr = RT1321_DSPFW_STATUS_ADDR;
+		break;
+	default:
+		dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id);
+		return -EINVAL;
+	}
+
+	regmap_write(rt1320->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x00);
+	ret = rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x00);
+	if (ret < 0) {
+		dev_dbg(dev, "%s, PDE=PS0 is NOT ready\n", __func__);
+		goto _timeout_;
+	}
+
+	regmap_read(rt1320->regmap, fw_status_addr, &fw_ready);
+	fw_ready &= 0x1;
+	if (!fw_ready) {
+		dev_dbg(dev, "%s, DSP FW is NOT ready\n", __func__);
+		goto _timeout_;
+	}
+
+	ret = rt1320_check_power_state_ready(rt1320, RT1320_NORMAL_STATE);
+	if (ret < 0) {
+		dev_dbg(dev, "%s, DSP FW PS is NOT ready\n", __func__);
+		goto _timeout_;
+	}
+
+	rt1320_set_advancemode(rt1320);
+
+_timeout_:
+	regmap_write(rt1320->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x03);
+	rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x03);
+
+	return ret;
+}
+
+static int rt1320_r0_load_mode_get(struct snd_kcontrol *kcontrol,
+				   struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct rt1320_sdw_priv *rt1320 = snd_soc_component_get_drvdata(component);
+
+	ucontrol->value.integer.value[0] = rt1320->r0_l_reg;
+	ucontrol->value.integer.value[1] = rt1320->r0_r_reg;
+
+	return 0;
+}
+
+static int rt1320_r0_load_mode_put(struct snd_kcontrol *kcontrol,
+				   struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct rt1320_sdw_priv *rt1320 = snd_soc_component_get_drvdata(component);
+	struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(rt1320->component);
+	int ret;
+
+	if (!rt1320->hw_init)
+		return 0;
+
+	if (ucontrol->value.integer.value[0] == 0 ||
+		ucontrol->value.integer.value[1] == 0)
+		return -EINVAL;
+
+	ret = pm_runtime_resume(component->dev);
+	if (ret < 0 && ret != -EACCES)
+		return ret;
+
+	snd_soc_dapm_mutex_lock(dapm);
+	if (snd_soc_component_get_bias_level(component) == SND_SOC_BIAS_OFF) {
+		rt1320->r0_l_reg = ucontrol->value.integer.value[0];
+		rt1320->r0_r_reg = ucontrol->value.integer.value[1];
+		rt1320_calc_r0(rt1320);
+		rt1320_r0_load(rt1320);
+	}
+	snd_soc_dapm_mutex_unlock(dapm);
+
+	return 0;
+}
+
+static int rt1320_t0_r0_load_info(struct snd_kcontrol *kcontrol,
+			       struct snd_ctl_elem_info *uinfo)
+{
+	uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER;
+	uinfo->count = 2;
+	uinfo->value.integer.max = kcontrol->private_value;
+
+	return 0;
+}
+
+#define RT1320_T0_R0_LOAD(xname, xmax, xhandler_get, xhandler_put) \
+{	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = xname, \
+	.info = rt1320_t0_r0_load_info, \
+	.get = xhandler_get, \
+	.put = xhandler_put, \
+	.private_value = xmax, \
+}
+
+static int rt1320_dspfw_load_get(struct snd_kcontrol *kcontrol,
+				 struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct rt1320_sdw_priv *rt1320 = snd_soc_component_get_drvdata(component);
+
+	ucontrol->value.integer.value[0] = rt1320->fw_load_done;
+	return 0;
+}
+
+static int rt1320_dspfw_load_put(struct snd_kcontrol *kcontrol,
+				 struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct rt1320_sdw_priv *rt1320 = snd_soc_component_get_drvdata(component);
+	int ret;
+
+	if (!rt1320->hw_init)
+		return 0;
+
+	ret = pm_runtime_resume(component->dev);
+	if (ret < 0 && ret != -EACCES)
+		return ret;
+
+	if (snd_soc_component_get_bias_level(component) == SND_SOC_BIAS_OFF &&
+		ucontrol->value.integer.value[0])
+		rt1320_dspfw_load_code(rt1320);
+
+	if (!ucontrol->value.integer.value[0])
+		rt1320->fw_load_done = false;
+
+	return 0;
+}
+
+static int rt1320_rae_update_get(struct snd_kcontrol *kcontrol,
+				 struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct rt1320_sdw_priv *rt1320 = snd_soc_component_get_drvdata(component);
+
+	ucontrol->value.integer.value[0] = rt1320->rae_update_done;
+	return 0;
+}
+
+static int rt1320_rae_update_put(struct snd_kcontrol *kcontrol,
+				 struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct rt1320_sdw_priv *rt1320 = snd_soc_component_get_drvdata(component);
+	int ret;
+
+	if (!rt1320->hw_init)
+		return 0;
+
+	ret = pm_runtime_resume(component->dev);
+	if (ret < 0 && ret != -EACCES)
+		return ret;
+
+	if (snd_soc_component_get_bias_level(component) == SND_SOC_BIAS_OFF &&
+		ucontrol->value.integer.value[0] && rt1320->fw_load_done)
+		rt1320_rae_load(rt1320);
+
+	if (!ucontrol->value.integer.value[0])
+		rt1320->rae_update_done = false;
+
+	return 0;
+}
+
+static int rt1320_r0_temperature_get(struct snd_kcontrol *kcontrol,
+				     struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct rt1320_sdw_priv *rt1320 = snd_soc_component_get_drvdata(component);
+
+	ucontrol->value.integer.value[0] = rt1320->temp_l_calib;
+	ucontrol->value.integer.value[1] = rt1320->temp_r_calib;
+	return 0;
+}
+
+static int rt1320_r0_temperature_put(struct snd_kcontrol *kcontrol,
+				     struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct rt1320_sdw_priv *rt1320 = snd_soc_component_get_drvdata(component);
+	struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(rt1320->component);
+	int ret;
+
+	if (!rt1320->hw_init)
+		return 0;
+
+	ret = pm_runtime_resume(component->dev);
+	if (ret < 0 && ret != -EACCES)
+		return ret;
+
+	snd_soc_dapm_mutex_lock(dapm);
+	if ((snd_soc_component_get_bias_level(component) == SND_SOC_BIAS_OFF) &&
+		ucontrol->value.integer.value[0] && ucontrol->value.integer.value[1])
+		rt1320_t0_load(rt1320, ucontrol->value.integer.value[0], ucontrol->value.integer.value[1]);
+	snd_soc_dapm_mutex_unlock(dapm);
+
+	return 0;
+}
+
 static const struct snd_kcontrol_new rt1320_snd_controls[] = {
 	SOC_DOUBLE_R_EXT_TLV("FU21 Playback Volume",
 		SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_FU21, RT1320_SDCA_CTL_FU_VOLUME, CH_01),
@@ -1371,6 +2552,17 @@ static const struct snd_kcontrol_new rt1320_snd_controls[] = {
 	RT_SDCA_EXT_TLV("FU Capture Volume",
 		SDW_SDCA_CTL(FUNC_NUM_MIC, RT1320_SDCA_ENT_FU113, RT1320_SDCA_CTL_FU_VOLUME, CH_01),
 		rt1320_set_gain_get, rt1320_set_gain_put, 4, 0x3f, in_vol_tlv, rt1320_dmic_fu_info),
+
+	SOC_SINGLE_EXT("R0 Calibration", SND_SOC_NOPM, 0, 1, 0,
+		rt1320_r0_cali_get, rt1320_r0_cali_put),
+	SOC_SINGLE_EXT("DSP FW Update", SND_SOC_NOPM, 0, 1, 0,
+		rt1320_dspfw_load_get, rt1320_dspfw_load_put),
+	RT1320_T0_R0_LOAD("R0 Load Mode", 0xffffffff,
+		rt1320_r0_load_mode_get, rt1320_r0_load_mode_put),
+	RT1320_T0_R0_LOAD("R0 Temperature", 0xff,
+		rt1320_r0_temperature_get, rt1320_r0_temperature_put),
+	SOC_SINGLE_EXT("RAE Update", SND_SOC_NOPM, 0, 1, 0,
+		rt1320_rae_update_get, rt1320_rae_update_put),
 };
 
 static const struct snd_kcontrol_new rt1320_spk_l_dac =
@@ -1606,6 +2798,18 @@ static int rt1320_sdw_component_probe(struct snd_soc_component *component)
 	if (ret < 0 && ret != -EACCES)
 		return ret;
 
+	/* Apply temperature and calibration data from device property */
+	if ((rt1320->temp_l_calib <= 0xff) && (rt1320->temp_l_calib > 0) &&
+		(rt1320->temp_r_calib <= 0xff) && (rt1320->temp_r_calib > 0))
+		rt1320_t0_load(rt1320, rt1320->temp_l_calib, rt1320->temp_r_calib);
+
+	if (rt1320->r0_l_calib && rt1320->r0_r_calib) {
+		rt1320->r0_l_reg = rt1320->r0_l_calib;
+		rt1320->r0_r_reg = rt1320->r0_r_calib;
+		rt1320_calc_r0(rt1320);
+		rt1320_r0_load(rt1320);
+	}
+
 	return 0;
 }
 
@@ -1667,6 +2871,26 @@ static struct snd_soc_dai_driver rt1320_sdw_dai[] = {
 	},
 };
 
+static int rt1320_parse_dp(struct rt1320_sdw_priv *rt1320, struct device *dev)
+{
+	device_property_read_u32(dev, "realtek,temperature_l_calib",
+				 &rt1320->temp_l_calib);
+	device_property_read_u32(dev, "realtek,temperature_r_calib",
+				 &rt1320->temp_r_calib);
+	device_property_read_u32(dev, "realtek,r0_l_calib",
+				 &rt1320->r0_l_calib);
+	device_property_read_u32(dev, "realtek,r0_r_calib",
+				 &rt1320->r0_r_calib);
+	device_property_read_string(dev, "realtek,dspfw-name",
+				    &rt1320->dspfw_name);
+
+	dev_dbg(dev, "%s: temp_l_calib: %d temp_r_calib: %d r0_l_calib: %d, r0_r_calib: %d",
+		__func__, rt1320->temp_l_calib, rt1320->temp_r_calib, rt1320->r0_l_calib, rt1320->r0_r_calib);
+	dev_dbg(dev, "%s: dspfw_name: %s", __func__, rt1320->dspfw_name);
+
+	return 0;
+}
+
 static int rt1320_sdw_init(struct device *dev, struct regmap *regmap,
 				struct regmap *mbq_regmap, struct sdw_slave *slave)
 {
@@ -1685,6 +2909,8 @@ static int rt1320_sdw_init(struct device *dev, struct regmap *regmap,
 	regcache_cache_only(rt1320->regmap, true);
 	regcache_cache_only(rt1320->mbq_regmap, true);
 
+	rt1320_parse_dp(rt1320, dev);
+
 	/*
 	 * Mark hw_init to false
 	 * HW init will be performed when device reports present
@@ -1696,6 +2922,8 @@ static int rt1320_sdw_init(struct device *dev, struct regmap *regmap,
 	rt1320->fu_mixer_mute[0] = rt1320->fu_mixer_mute[1] =
 		rt1320->fu_mixer_mute[2] = rt1320->fu_mixer_mute[3] = true;
 
+	INIT_WORK(&rt1320->load_dspfw_work, rt1320_load_dspfw_work);
+
 	ret =  devm_snd_soc_register_component(dev,
 				&soc_component_sdw_rt1320,
 				rt1320_sdw_dai,
@@ -1742,6 +2970,9 @@ static int rt1320_sdw_probe(struct sdw_slave *slave,
 
 static int rt1320_sdw_remove(struct sdw_slave *slave)
 {
+	struct  rt1320_sdw_priv *rt1320 = dev_get_drvdata(&slave->dev);
+
+	cancel_work_sync(&rt1320->load_dspfw_work);
 	pm_runtime_disable(&slave->dev);
 
 	return 0;
diff --git a/sound/soc/codecs/rt1320-sdw.h b/sound/soc/codecs/rt1320-sdw.h
index a6d90e259dc9..5a9f496dd848 100644
--- a/sound/soc/codecs/rt1320-sdw.h
+++ b/sound/soc/codecs/rt1320-sdw.h
@@ -13,6 +13,7 @@
 #include <linux/soundwire/sdw_type.h>
 #include <linux/soundwire/sdw_registers.h>
 #include <sound/soc.h>
+#include "../../../drivers/soundwire/bus.h"
 
 #define RT1320_DEV_ID 0x6981
 #define RT1321_DEV_ID 0x7045
@@ -22,6 +23,8 @@
 #define RT1320_DEV_ID_1 0xc405
 #define RT1320_DEV_ID_0 0xc406
 
+#define RT1320_POWER_STATE 0xc560
+
 #define RT1321_PATCH_MAIN_VER 0x1000cffe
 #define RT1321_PATCH_BETA_VER 0x1000cfff
 
@@ -96,6 +99,57 @@ enum rt1320_version_id {
 #define RT1320_VC_MCU_PATCH "realtek/rt1320/rt1320-patch-code-vc.bin"
 #define RT1321_VA_MCU_PATCH "realtek/rt1320/rt1321-patch-code-va.bin"
 
+#define RT1320_FW_PARAM_ADDR 0x3fc2ab80
+#define RT1320_CMD_ID 0x3fc2ab81
+#define RT1320_CMD_PARAM_ADDR 0x3fc2ab90
+#define RT1320_DSPFW_STATUS_ADDR 0x3fc2bfc4
+
+#define RT1321_FW_PARAM_ADDR 0x3fc2d300
+#define RT1321_CMD_ID 0x3fc2d301
+#define RT1321_CMD_PARAM_ADDR 0x3fc2d310
+#define RT1321_DSPFW_STATUS_ADDR 0x3fc2dfc4
+
+/* FW parameter id 6, 7 */
+struct rt1320_datafixpoint {
+	int silencedetect;
+	int r0;
+	int meanr0;
+	int advancegain;
+	int ts;
+	int re;
+	int t;
+	int invrs;
+};
+
+struct rt1320_paramcmd {
+	unsigned char moudleid;
+	unsigned char commandtype;
+	unsigned short reserved1;
+	unsigned int commandlength;
+	long long reserved2;
+	unsigned int paramid;
+	unsigned int paramlength;
+};
+
+enum rt1320_fw_cmdid {
+	RT1320_FW_READY,
+	RT1320_SET_PARAM,
+	RT1320_GET_PARAM,
+	RT1320_GET_POOLSIZE,
+};
+
+enum rt1320_power_state {
+	RT1320_NORMAL_STATE = 0x18,
+	RT1320_K_R0_STATE = 0x1b,
+};
+
+enum rt1320_rw_type {
+	RT1320_BRA_WRITE = 0,
+	RT1320_BRA_READ = 1,
+	RT1320_PARAM_WRITE = 2,
+	RT1320_PARAM_READ = 3,
+};
+
 struct rt1320_sdw_priv {
 	struct snd_soc_component *component;
 	struct regmap *regmap;
@@ -108,6 +162,18 @@ struct rt1320_sdw_priv {
 	unsigned int dev_id;
 	bool fu_dapm_mute;
 	bool fu_mixer_mute[4];
+	unsigned long long r0_l_reg;
+	unsigned long long r0_r_reg;
+	unsigned int r0_l_calib;
+	unsigned int r0_r_calib;
+	unsigned int temp_l_calib;
+	unsigned int temp_r_calib;
+	const char *dspfw_name;
+	bool cali_done;
+	bool fw_load_done;
+	bool rae_update_done;
+	struct work_struct load_dspfw_work;
+	struct sdw_bpt_msg bra_msg;
 };
 
 #endif /* __RT1320_SDW_H__ */
diff --git a/sound/soc/codecs/rt274.c b/sound/soc/codecs/rt274.c
index 5c33aeaced2f..bba714020c70 100644
--- a/sound/soc/codecs/rt274.c
+++ b/sound/soc/codecs/rt274.c
@@ -1189,7 +1189,7 @@ static int rt274_i2c_probe(struct i2c_client *i2c)
 	regmap_write(rt274->regmap, RT274_UNSOLICITED_MIC, 0x82);
 
 	if (rt274->i2c->irq) {
-		ret = request_threaded_irq(rt274->i2c->irq, NULL, rt274_irq,
+		ret = devm_request_threaded_irq(&rt274->i2c->dev, rt274->i2c->irq, NULL, rt274_irq,
 			IRQF_TRIGGER_HIGH | IRQF_ONESHOT, "rt274", rt274);
 		if (ret != 0) {
 			dev_err(&i2c->dev,
@@ -1205,15 +1205,6 @@ static int rt274_i2c_probe(struct i2c_client *i2c)
 	return ret;
 }
 
-static void rt274_i2c_remove(struct i2c_client *i2c)
-{
-	struct rt274_priv *rt274 = i2c_get_clientdata(i2c);
-
-	if (i2c->irq)
-		free_irq(i2c->irq, rt274);
-}
-
-
 static struct i2c_driver rt274_i2c_driver = {
 	.driver = {
 		   .name = "rt274",
@@ -1223,7 +1214,6 @@ static struct i2c_driver rt274_i2c_driver = {
 #endif
 		   },
 	.probe = rt274_i2c_probe,
-	.remove = rt274_i2c_remove,
 	.id_table = rt274_i2c_id,
 };
 
diff --git a/sound/soc/codecs/rt286.c b/sound/soc/codecs/rt286.c
index 2fbb5860c421..195658f626cc 100644
--- a/sound/soc/codecs/rt286.c
+++ b/sound/soc/codecs/rt286.c
@@ -1236,7 +1236,7 @@ static int rt286_i2c_probe(struct i2c_client *i2c)
 	}
 
 	if (rt286->i2c->irq) {
-		ret = request_threaded_irq(rt286->i2c->irq, NULL, rt286_irq,
+		ret = devm_request_threaded_irq(&rt286->i2c->dev, rt286->i2c->irq, NULL, rt286_irq,
 			IRQF_TRIGGER_HIGH | IRQF_ONESHOT, "rt286", rt286);
 		if (ret != 0) {
 			dev_err(&i2c->dev,
@@ -1252,22 +1252,12 @@ static int rt286_i2c_probe(struct i2c_client *i2c)
 	return ret;
 }
 
-static void rt286_i2c_remove(struct i2c_client *i2c)
-{
-	struct rt286_priv *rt286 = i2c_get_clientdata(i2c);
-
-	if (i2c->irq)
-		free_irq(i2c->irq, rt286);
-}
-
-
 static struct i2c_driver rt286_i2c_driver = {
 	.driver = {
 		   .name = "rt286",
 		   .acpi_match_table = ACPI_PTR(rt286_acpi_match),
 		   },
 	.probe = rt286_i2c_probe,
-	.remove = rt286_i2c_remove,
 	.id_table = rt286_i2c_id,
 };
 
diff --git a/sound/soc/codecs/rt298.c b/sound/soc/codecs/rt298.c
index 02247593513a..7d532a5a5f73 100644
--- a/sound/soc/codecs/rt298.c
+++ b/sound/soc/codecs/rt298.c
@@ -1282,7 +1282,7 @@ static int rt298_i2c_probe(struct i2c_client *i2c)
 	rt298->is_hp_in = -1;
 
 	if (rt298->i2c->irq) {
-		ret = request_threaded_irq(rt298->i2c->irq, NULL, rt298_irq,
+		ret = devm_request_threaded_irq(&rt298->i2c->dev, rt298->i2c->irq, NULL, rt298_irq,
 			IRQF_TRIGGER_HIGH | IRQF_ONESHOT, "rt298", rt298);
 		if (ret != 0) {
 			dev_err(&i2c->dev,
@@ -1298,22 +1298,12 @@ static int rt298_i2c_probe(struct i2c_client *i2c)
 	return ret;
 }
 
-static void rt298_i2c_remove(struct i2c_client *i2c)
-{
-	struct rt298_priv *rt298 = i2c_get_clientdata(i2c);
-
-	if (i2c->irq)
-		free_irq(i2c->irq, rt298);
-}
-
-
 static struct i2c_driver rt298_i2c_driver = {
 	.driver = {
 		   .name = "rt298",
 		   .acpi_match_table = ACPI_PTR(rt298_acpi_match),
 		   },
 	.probe = rt298_i2c_probe,
-	.remove = rt298_i2c_remove,
 	.id_table = rt298_i2c_id,
 };
 
diff --git a/sound/soc/codecs/sdw-mockup.c b/sound/soc/codecs/sdw-mockup.c
index 574c08b14f0c..b7e6546f1b5a 100644
--- a/sound/soc/codecs/sdw-mockup.c
+++ b/sound/soc/codecs/sdw-mockup.c
@@ -237,11 +237,6 @@ static int sdw_mockup_sdw_probe(struct sdw_slave *slave,
 	return ret;
 }
 
-static int sdw_mockup_sdw_remove(struct sdw_slave *slave)
-{
-	return 0;
-}
-
 /*
  * Intel reserved parts ID with the following mapping expected:
  * 0xAAAA: generic full-duplex codec
@@ -264,7 +259,6 @@ static struct sdw_driver sdw_mockup_sdw_driver = {
 		.name = "sdw-mockup",
 	},
 	.probe = sdw_mockup_sdw_probe,
-	.remove = sdw_mockup_sdw_remove,
 	.ops = &sdw_mockup_slave_ops,
 	.id_table = sdw_mockup_id,
 };
diff --git a/sound/soc/codecs/tas2780.c b/sound/soc/codecs/tas2780.c
index a1963415c931..cf3f6abd7e7b 100644
--- a/sound/soc/codecs/tas2780.c
+++ b/sound/soc/codecs/tas2780.c
@@ -319,25 +319,22 @@ static int tas2780_set_fmt(struct snd_soc_dai *dai, unsigned int fmt)
 		goto err;
 	}
 
-	if (((fmt & SND_SOC_DAIFMT_FORMAT_MASK) == SND_SOC_DAIFMT_I2S)
-		|| ((fmt & SND_SOC_DAIFMT_FORMAT_MASK)
-		== SND_SOC_DAIFMT_DSP_A)){
+	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+	case SND_SOC_DAIFMT_I2S:
+	case SND_SOC_DAIFMT_DSP_A:
 		iface = TAS2780_TDM_CFG2_SCFG_I2S;
 		tdm_rx_start_slot = 1;
-	} else {
-		if (((fmt & SND_SOC_DAIFMT_FORMAT_MASK)
-			== SND_SOC_DAIFMT_DSP_B)
-			|| ((fmt & SND_SOC_DAIFMT_FORMAT_MASK)
-			== SND_SOC_DAIFMT_LEFT_J)) {
-			iface = TAS2780_TDM_CFG2_SCFG_LEFT_J;
-			tdm_rx_start_slot = 0;
-		} else {
-			dev_err(tas2780->dev,
-				"%s:DAI Format is not found, fmt=0x%x\n",
-				__func__, fmt);
-			ret = -EINVAL;
-			goto err;
-		}
+		break;
+	case SND_SOC_DAIFMT_LEFT_J:
+	case SND_SOC_DAIFMT_DSP_B:
+		iface = TAS2780_TDM_CFG2_SCFG_LEFT_J;
+		tdm_rx_start_slot = 0;
+		break;
+	default:
+		dev_err(tas2780->dev,
+			"%s:DAI Format is not found, fmt=0x%x\n", __func__, fmt);
+		ret = -EINVAL;
+		goto err;
 	}
 	ret = snd_soc_component_update_bits(component, TAS2780_TDM_CFG1,
 		TAS2780_TDM_CFG1_MASK,
diff --git a/sound/soc/codecs/tas2783-sdw.c b/sound/soc/codecs/tas2783-sdw.c
index 43b779873b93..af812f95a4ba 100644
--- a/sound/soc/codecs/tas2783-sdw.c
+++ b/sound/soc/codecs/tas2783-sdw.c
@@ -27,16 +27,19 @@
 #include <linux/soundwire/sdw.h>
 #include <linux/soundwire/sdw_registers.h>
 #include <linux/soundwire/sdw_type.h>
+#include <linux/pci.h>
 #include <sound/sdw.h>
 #include <sound/soc.h>
 #include <sound/tlv.h>
 #include <sound/tas2781-tlv.h>
+#include <sound/sdca_function.h>
+#include <sound/sdca_regmap.h>
 
 #include "tas2783.h"
 
 #define TIMEOUT_FW_DL_MS (3000)
-#define FW_DL_OFFSET	36
-#define FW_FL_HDR	12
+#define FW_DL_OFFSET	84 /* binary file information */
+#define FW_FL_HDR	20 /* minimum number of bytes in one chunk */
 #define TAS2783_PROBE_TIMEOUT 5000
 #define TAS2783_CALI_GUID EFI_GUID(0x1f52d2a1, 0xbb3a, 0x457d, 0xbc, \
 				   0x09, 0x43, 0xa3, 0xf4, 0x31, 0x0a, 0x92)
@@ -49,11 +52,22 @@ static const u32 tas2783_cali_reg[] = {
 	TAS2783_CAL_TLIM,
 };
 
-struct bin_header_t {
-	u16 vendor_id;
-	u16 version;
+struct tas_fw_hdr {
+	u32 size;
+	u32 version_offset;
+	u32 plt_id;
+	u32 ppc3_ver;
+	u32 timestamp;
+	u8 ddc_name[64];
+};
+
+struct tas_fw_file {
+	u32 vendor_id;
 	u32 file_id;
+	u32 version;
 	u32 length;
+	u32 dest_addr;
+	u8 *fw_data;
 };
 
 struct calibration_data {
@@ -66,6 +80,7 @@ struct tas2783_prv {
 	struct snd_soc_component *component;
 	struct calibration_data cali_data;
 	struct sdw_slave *sdw_peripheral;
+	struct sdca_function_data *sa_func_data;
 	enum sdw_slave_status status;
 	/* calibration */
 	struct mutex calib_lock;
@@ -286,7 +301,7 @@ static const struct reg_default tas2783_reg_default[] = {
 };
 
 static const struct reg_sequence tas2783_init_seq[] = {
-	REG_SEQ0(SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU21, 0x10, 0x00), 0x04),
+	REG_SEQ0(SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_PPU21, 0x10, 0x00), 0x01),
 	REG_SEQ0(0x00800418, 0x00),
 	REG_SEQ0(0x00800419, 0x00),
 	REG_SEQ0(0x0080041a, 0x00),
@@ -296,60 +311,19 @@ static const struct reg_sequence tas2783_init_seq[] = {
 	REG_SEQ0(0x0080042a, 0x00),
 	REG_SEQ0(0x0080042b, 0x00),
 	REG_SEQ0(SDW_SDCA_CTL(1, TAS2783_SDCA_ENT_FU23, 0x1, 0x00), 0x00),
-	REG_SEQ0(0x0080005c, 0xD9),
-	REG_SEQ0(0x00800082, 0x20),
-	REG_SEQ0(0x008000a1, 0x00),
-	REG_SEQ0(0x00800097, 0xc8),
-	REG_SEQ0(0x00800099, 0x20),
-	REG_SEQ0(0x008000c7, 0xaa),
-	REG_SEQ0(0x008000b5, 0x74),
-	REG_SEQ0(0x00800082, 0x20),
-	REG_SEQ0(0x00807e8d, 0x0d),
-	REG_SEQ0(0x00807eb9, 0x53),
-	REG_SEQ0(0x00807ebe, 0x42),
-	REG_SEQ0(0x00807ec5, 0x37),
-	REG_SEQ0(0x00800066, 0x92),
-	REG_SEQ0(0x00800003, 0x28),
 	REG_SEQ0(0x00800004, 0x21),
 	REG_SEQ0(0x00800005, 0x41),
 	REG_SEQ0(0x00800006, 0x00),
 	REG_SEQ0(0x00800007, 0x20),
-	REG_SEQ0(0x0080000c, 0x10),
-	REG_SEQ0(0x00800013, 0x08),
 	REG_SEQ0(0x00800015, 0x00),
-	REG_SEQ0(0x00800017, 0x80),
-	REG_SEQ0(0x0080001a, 0x00),
-	REG_SEQ0(0x0080001b, 0x22),
-	REG_SEQ0(0x0080001c, 0x36),
-	REG_SEQ0(0x0080001d, 0x01),
-	REG_SEQ0(0x0080001f, 0x00),
-	REG_SEQ0(0x00800020, 0x2e),
-	REG_SEQ0(0x00800034, 0x06),
-	REG_SEQ0(0x00800035, 0xb9),
 	REG_SEQ0(0x00800036, 0xad),
 	REG_SEQ0(0x00800037, 0xa8),
-	REG_SEQ0(0x00800038, 0x00),
-	REG_SEQ0(0x0080003b, 0xfc),
-	REG_SEQ0(0x0080003d, 0xdd),
-	REG_SEQ0(0x00800040, 0xf6),
-	REG_SEQ0(0x00800041, 0x14),
-	REG_SEQ0(0x0080005c, 0x19),
-	REG_SEQ0(0x0080005d, 0x80),
-	REG_SEQ0(0x00800063, 0x48),
-	REG_SEQ0(0x00800065, 0x08),
-	REG_SEQ0(0x00800067, 0x00),
-	REG_SEQ0(0x0080006a, 0x12),
 	REG_SEQ0(0x0080006b, 0x7b),
 	REG_SEQ0(0x0080006c, 0x00),
 	REG_SEQ0(0x0080006d, 0x00),
 	REG_SEQ0(0x0080006e, 0x1a),
 	REG_SEQ0(0x0080006f, 0x00),
-	REG_SEQ0(0x00800070, 0x96),
 	REG_SEQ0(0x00800071, 0x02),
-	REG_SEQ0(0x00800073, 0x08),
-	REG_SEQ0(0x00800075, 0xe0),
-	REG_SEQ0(0x0080007a, 0x60),
-	REG_SEQ0(0x008000bd, 0x00),
 	REG_SEQ0(0x008000be, 0x00),
 	REG_SEQ0(0x008000bf, 0x00),
 	REG_SEQ0(0x008000c0, 0x00),
@@ -357,17 +331,6 @@ static const struct reg_sequence tas2783_init_seq[] = {
 	REG_SEQ0(0x008000c2, 0x00),
 	REG_SEQ0(0x008000c3, 0x00),
 	REG_SEQ0(0x008000c4, 0x00),
-	REG_SEQ0(0x008000c5, 0x00),
-	REG_SEQ0(0x00800008, 0x49),
-	REG_SEQ0(0x00800009, 0x02),
-	REG_SEQ0(0x0080000a, 0x1a),
-	REG_SEQ0(0x0080000d, 0x93),
-	REG_SEQ0(0x0080000e, 0x82),
-	REG_SEQ0(0x0080000f, 0x42),
-	REG_SEQ0(0x00800010, 0x84),
-	REG_SEQ0(0x00800014, 0x0a),
-	REG_SEQ0(0x00800016, 0x00),
-	REG_SEQ0(0x00800060, 0x21),
 };
 
 static int tas2783_sdca_mbq_size(struct device *dev, u32 reg)
@@ -676,7 +639,8 @@ static void tas2783_set_calib_params_to_device(struct tas2783_prv *tas_dev, u32
 	}
 
 	if (device_num == dev_count)
-		dev_err(tas_dev->dev, "device not found\n");
+		dev_err(tas_dev->dev,
+			"unique id not found in the calib data\n");
 	else
 		dev_dbg(tas_dev->dev, "calib data update done\n");
 }
@@ -735,13 +699,28 @@ static s32 tas2783_update_calibdata(struct tas2783_prv *tas_dev)
 	return ret;
 }
 
-static s32 read_header(const u8 *data, struct bin_header_t *hdr)
+static s32 tas_fw_read_hdr(const u8 *data, struct tas_fw_hdr *hdr)
 {
-	hdr->vendor_id = get_unaligned_le16(&data[0]);
-	hdr->file_id = get_unaligned_le32(&data[2]);
-	hdr->version = get_unaligned_le16(&data[6]);
-	hdr->length = get_unaligned_le32(&data[8]);
-	return 12;
+	hdr->size = get_unaligned_le32(data);
+	hdr->version_offset = get_unaligned_le32(&data[4]);
+	hdr->plt_id = get_unaligned_le32(&data[8]);
+	hdr->ppc3_ver = get_unaligned_le32(&data[12]);
+	memcpy(hdr->ddc_name, &data[16], 64);
+	hdr->timestamp = get_unaligned_le32(&data[80]);
+
+	return 84;
+}
+
+static s32 tas_fw_get_next_file(const u8 *data, struct tas_fw_file *file)
+{
+	file->vendor_id = get_unaligned_le32(&data[0]);
+	file->file_id = get_unaligned_le32(&data[4]);
+	file->version = get_unaligned_le32(&data[8]);
+	file->length = get_unaligned_le32(&data[12]);
+	file->dest_addr = get_unaligned_le32(&data[16]);
+	file->fw_data = (u8 *)&data[20];
+
+	return file->length + sizeof(u32) * 5;
 }
 
 static void tas2783_fw_ready(const struct firmware *fmw, void *context)
@@ -749,13 +728,20 @@ static void tas2783_fw_ready(const struct firmware *fmw, void *context)
 	struct tas2783_prv *tas_dev =
 		(struct tas2783_prv *)context;
 	const u8 *buf = NULL;
-	s32 offset = 0, img_sz, file_blk_size, ret;
-	struct bin_header_t hdr;
+	s32  img_sz, ret = 0, cur_file = 0;
+	s32 offset = 0;
+
+	struct tas_fw_hdr *hdr __free(kfree) = kzalloc(sizeof(*hdr), GFP_KERNEL);
+	struct tas_fw_file *file __free(kfree) = kzalloc(sizeof(*file), GFP_KERNEL);
+	if (!file || !hdr) {
+		ret = -ENOMEM;
+		goto out;
+	}
 
 	if (!fmw || !fmw->data) {
-		/* No firmware binary, devices will work in ROM mode. */
+		/* firmware binary not found*/
 		dev_err(tas_dev->dev,
-			"Failed to read %s, no side-effect on driver running\n",
+			"Failed to read fw binary %s\n",
 			tas_dev->rca_binaryname);
 		ret = -EINVAL;
 		goto out;
@@ -763,67 +749,47 @@ static void tas2783_fw_ready(const struct firmware *fmw, void *context)
 
 	img_sz = fmw->size;
 	buf = fmw->data;
-	offset += FW_DL_OFFSET;
-	if (offset >= (img_sz - FW_FL_HDR)) {
-		dev_err(tas_dev->dev,
-			"firmware is too small");
+	offset += tas_fw_read_hdr(buf, hdr);
+	if (hdr->size != img_sz) {
+		ret = -EINVAL;
+		dev_err(tas_dev->dev, "firmware size mismatch with header");
+		goto out;
+	}
+
+	if (img_sz < FW_DL_OFFSET) {
 		ret = -EINVAL;
+		dev_err(tas_dev->dev, "unexpected size, size is too small");
 		goto out;
 	}
 
 	mutex_lock(&tas_dev->pde_lock);
 	while (offset < (img_sz - FW_FL_HDR)) {
-		memset(&hdr, 0, sizeof(hdr));
-		offset += read_header(&buf[offset], &hdr);
+		offset += tas_fw_get_next_file(&buf[offset], file);
 		dev_dbg(tas_dev->dev,
-			"vndr=%d, file=%d, version=%d, len=%d, off=%d\n",
-			hdr.vendor_id, hdr.file_id, hdr.version,
-			hdr.length, offset);
-		/* size also includes the header */
-		file_blk_size = hdr.length - FW_FL_HDR;
-
-		/* make sure that enough data is there */
-		if (offset + file_blk_size > img_sz) {
-			ret = -EINVAL;
+			"v=%d, fid=%d, ver=%d, len=%d, daddr=0x%x, fw=%p",
+			file->vendor_id, file->file_id,
+			file->version, file->length,
+			file->dest_addr, file->fw_data);
+
+		ret = sdw_nwrite_no_pm(tas_dev->sdw_peripheral,
+				       file->dest_addr,
+				       file->length,
+				       file->fw_data);
+		if (ret < 0) {
 			dev_err(tas_dev->dev,
-				"corrupt firmware file");
-			break;
-		}
-
-		switch (hdr.file_id) {
-		case 0:
-			ret = sdw_nwrite_no_pm(tas_dev->sdw_peripheral,
-					       PRAM_ADDR_START, file_blk_size,
-					       &buf[offset]);
-			if (ret < 0)
-				dev_err(tas_dev->dev,
-					"PRAM update failed: %d", ret);
-			break;
-
-		case 1:
-			ret = sdw_nwrite_no_pm(tas_dev->sdw_peripheral,
-					       YRAM_ADDR_START, file_blk_size,
-					       &buf[offset]);
-			if (ret < 0)
-				dev_err(tas_dev->dev,
-					"YRAM update failed: %d", ret);
-
-			break;
-
-		default:
-			ret = -EINVAL;
-			dev_err(tas_dev->dev, "Unsupported file");
+				"FW download failed: %d", ret);
 			break;
 		}
-
-		if (ret == 0)
-			offset += file_blk_size;
-		else
-			break;
+		cur_file++;
 	}
 	mutex_unlock(&tas_dev->pde_lock);
-	if (!ret)
+
+	if (cur_file == 0) {
+		dev_err(tas_dev->dev, "fw with no files");
+		ret = -EINVAL;
+	} else {
 		tas2783_update_calibdata(tas_dev);
+	}
 
 out:
 	if (!ret)
@@ -1094,66 +1060,6 @@ static s32 tas_init(struct tas2783_prv *tas_dev)
 	return ret;
 }
 
-static s32 tas_read_prop(struct sdw_slave *slave)
-{
-	struct sdw_slave_prop *prop = &slave->prop;
-	s32 nval;
-	s32 i, j;
-	u32 bit;
-	unsigned long addr;
-	struct sdw_dpn_prop *dpn;
-
-	prop->scp_int1_mask =
-		SDW_SCP_INT1_BUS_CLASH | SDW_SCP_INT1_PARITY;
-	prop->quirks = SDW_SLAVE_QUIRKS_INVALID_INITIAL_PARITY;
-
-	prop->paging_support = true;
-
-	/* first we need to allocate memory for set bits in port lists */
-	prop->source_ports = 0x04; /* BITMAP: 00000100 */
-	prop->sink_ports = 0x2; /* BITMAP:  00000010 */
-
-	nval = hweight32(prop->source_ports);
-	prop->src_dpn_prop = devm_kcalloc(&slave->dev, nval,
-					  sizeof(*prop->src_dpn_prop), GFP_KERNEL);
-	if (!prop->src_dpn_prop)
-		return -ENOMEM;
-
-	i = 0;
-	dpn = prop->src_dpn_prop;
-	addr = prop->source_ports;
-	for_each_set_bit(bit, &addr, 32) {
-		dpn[i].num = bit;
-		dpn[i].type = SDW_DPN_FULL;
-		dpn[i].simple_ch_prep_sm = false;
-		dpn[i].ch_prep_timeout = 10;
-		i++;
-	}
-
-	/* do this again for sink now */
-	nval = hweight32(prop->sink_ports);
-	prop->sink_dpn_prop = devm_kcalloc(&slave->dev, nval,
-					   sizeof(*prop->sink_dpn_prop), GFP_KERNEL);
-	if (!prop->sink_dpn_prop)
-		return -ENOMEM;
-
-	j = 0;
-	dpn = prop->sink_dpn_prop;
-	addr = prop->sink_ports;
-	for_each_set_bit(bit, &addr, 32) {
-		dpn[j].num = bit;
-		dpn[j].type = SDW_DPN_FULL;
-		dpn[j].simple_ch_prep_sm = false;
-		dpn[j].ch_prep_timeout = 10;
-		j++;
-	}
-
-	/* set the timeout values */
-	prop->clk_stop_timeout = 200;
-
-	return 0;
-}
-
 static s32 tas2783_sdca_dev_suspend(struct device *dev)
 {
 	struct tas2783_prv *tas_dev = dev_get_drvdata(dev);
@@ -1200,8 +1106,21 @@ static const struct dev_pm_ops tas2783_sdca_pm = {
 	RUNTIME_PM_OPS(tas2783_sdca_dev_suspend, tas2783_sdca_dev_resume, NULL)
 };
 
+static struct pci_dev *tas_get_pci_dev(struct sdw_slave *peripheral)
+{
+	struct device *dev = &peripheral->dev;
+
+	for (; dev; dev = dev->parent)
+		if (dev->bus == &pci_bus_type)
+			return to_pci_dev(dev);
+
+	return NULL;
+}
+
 static s32 tas_io_init(struct device *dev, struct sdw_slave *slave)
 {
+	struct pci_dev *pci;
+	struct sdw_bus *bus;
 	struct tas2783_prv *tas_dev = dev_get_drvdata(dev);
 	s32 ret;
 	u8 unique_id = tas_dev->sdw_peripheral->id.unique_id;
@@ -1209,10 +1128,27 @@ static s32 tas_io_init(struct device *dev, struct sdw_slave *slave)
 	if (tas_dev->hw_init)
 		return 0;
 
+	pci = tas_get_pci_dev(slave);
+	if (!pci) {
+		dev_err(dev, "pci device id can't be read");
+		return -EINVAL;
+	}
+
+	bus = slave->bus;
 	tas_dev->fw_dl_task_done = false;
 	tas_dev->fw_dl_success = false;
+
+	ret = regmap_write(tas_dev->regmap, TAS2783_SW_RESET, 0x1);
+	if (ret) {
+		dev_err(dev, "sw reset failed, err=%d", ret);
+		return ret;
+	}
+	usleep_range(2000, 2200);
+
+	/* subsystem_id-link_id-unique_id */
 	scnprintf(tas_dev->rca_binaryname, sizeof(tas_dev->rca_binaryname),
-		  "tas2783-%01x.bin", unique_id);
+		  "%04X-%1X-%1X.bin", pci->subsystem_device, bus->link_id,
+		  unique_id);
 
 	ret = request_firmware_nowait(THIS_MODULE, FW_ACTION_UEVENT,
 				      tas_dev->rca_binaryname, tas_dev->dev,
@@ -1230,9 +1166,18 @@ static s32 tas_io_init(struct device *dev, struct sdw_slave *slave)
 		dev_err(tas_dev->dev, "fw request, wait_event timeout\n");
 		ret = -EAGAIN;
 	} else {
-		ret = regmap_multi_reg_write(tas_dev->regmap, tas2783_init_seq,
-					     ARRAY_SIZE(tas2783_init_seq));
-		tas_dev->hw_init = true;
+		if (tas_dev->sa_func_data)
+			ret = sdca_regmap_write_init(dev, tas_dev->regmap,
+						     tas_dev->sa_func_data);
+		else
+			ret = regmap_multi_reg_write(tas_dev->regmap, tas2783_init_seq,
+						     ARRAY_SIZE(tas2783_init_seq));
+
+		if (ret)
+			dev_err(tas_dev->dev,
+				"init writes failed, err=%d", ret);
+		else
+			tas_dev->hw_init = true;
 	}
 
 	return ret;
@@ -1267,7 +1212,6 @@ static s32 tas_update_status(struct sdw_slave *slave,
 }
 
 static const struct sdw_slave_ops tas_sdw_ops = {
-	.read_prop	= tas_read_prop,
 	.update_status	= tas_update_status,
 };
 
@@ -1282,12 +1226,52 @@ static s32 tas_sdw_probe(struct sdw_slave *peripheral,
 	struct regmap *regmap;
 	struct device *dev = &peripheral->dev;
 	struct tas2783_prv *tas_dev;
+	struct sdca_function_data *function_data = NULL;
+	int ret, i;
+
+	ret = sdw_slave_read_prop(peripheral);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "slave property read failed");
 
 	tas_dev = devm_kzalloc(dev, sizeof(*tas_dev), GFP_KERNEL);
 	if (!tas_dev)
 		return dev_err_probe(dev, -ENOMEM,
 				     "Failed devm_kzalloc");
 
+	i = -1;
+	/* check if we have any SDCA function data available */
+	if (peripheral->sdca_data.num_functions > 0) {
+		dev_dbg(dev, "SDCA functions found: %d", peripheral->sdca_data.num_functions);
+
+		/* Look for Smart Amp function type */
+		for (i = 0; i < peripheral->sdca_data.num_functions; i++) {
+			if (peripheral->sdca_data.function[i].type ==
+			    SDCA_FUNCTION_TYPE_SMART_AMP) {
+				dev_info(dev, "Found Smart Amp function at index %d", i);
+				break;
+			}
+		}
+	}
+
+	if (i >= 0 && i < peripheral->sdca_data.num_functions) {
+		/* Allocate memory for function data */
+		function_data = devm_kzalloc(dev, sizeof(*function_data),
+					     GFP_KERNEL);
+		if (!function_data)
+			return dev_err_probe(dev, -ENOMEM,
+					     "failed to parse sdca functions");
+
+		/* Parse the function */
+		ret = sdca_parse_function(dev, peripheral,
+					  &peripheral->sdca_data.function[i],
+					  function_data);
+		if (!ret)
+			tas_dev->sa_func_data = function_data;
+		else
+			dev_warn(dev, "smartamp function parse failed:err%d, using defaults", ret);
+	}
+
 	tas_dev->dev = dev;
 	tas_dev->sdw_peripheral = peripheral;
 	tas_dev->hw_init = false;
@@ -1342,6 +1326,7 @@ static struct sdw_driver tas_sdw_driver = {
 };
 module_sdw_driver(tas_sdw_driver);
 
+MODULE_IMPORT_NS("SND_SOC_SDCA");
 MODULE_AUTHOR("Texas Instruments Inc.");
 MODULE_DESCRIPTION("ASoC TAS2783 SoundWire Driver");
 MODULE_LICENSE("GPL");
diff --git a/sound/soc/codecs/tas2783.h b/sound/soc/codecs/tas2783.h
index 794333e0a350..bf34319c9a9f 100644
--- a/sound/soc/codecs/tas2783.h
+++ b/sound/soc/codecs/tas2783.h
@@ -28,6 +28,7 @@
 #define TASDEV_REG_SDW(book, page, reg)	(((book) * 256 * 128) + \
 		0x800000 + ((page) * 128) + (reg))
 
+#define TAS2783_SW_RESET	TASDEV_REG_SDW(0x0, 0x00, 0x01)
 /* Volume control */
 #define TAS2783_DVC_LVL		TASDEV_REG_SDW(0x0, 0x00, 0x1A)
 #define TAS2783_AMP_LEVEL	TASDEV_REG_SDW(0x0, 0x00, 0x03)
diff --git a/sound/soc/codecs/wm0010.c b/sound/soc/codecs/wm0010.c
index 7511c71695c6..6e097d8ed288 100644
--- a/sound/soc/codecs/wm0010.c
+++ b/sound/soc/codecs/wm0010.c
@@ -326,7 +326,7 @@ static void byte_swap_64(u64 *data_in, u64 *data_out, u32 len)
 	int i;
 
 	for (i = 0; i < len / 8; i++)
-		data_out[i] = cpu_to_be64(le64_to_cpu(data_in[i]));
+		data_out[i] = swab64(data_in[i]);
 }
 
 static int wm0010_firmware_load(const char *name, struct snd_soc_component *component)
diff --git a/sound/soc/fsl/fsl-asoc-card.c b/sound/soc/fsl/fsl-asoc-card.c
index 2c7eb0baa0f3..70a6159430ed 100644
--- a/sound/soc/fsl/fsl-asoc-card.c
+++ b/sound/soc/fsl/fsl-asoc-card.c
@@ -1045,8 +1045,8 @@ static int fsl_asoc_card_probe(struct platform_device *pdev)
 	 * The notifier is initialized in snd_soc_card_jack_new(), then
 	 * snd_soc_jack_notifier_register can be called.
 	 */
-	if (of_property_read_bool(np, "hp-det-gpios") ||
-	    of_property_read_bool(np, "hp-det-gpio") /* deprecated */) {
+	if (of_property_present(np, "hp-det-gpios") ||
+	    of_property_present(np, "hp-det-gpio") /* deprecated */) {
 		ret = simple_util_init_jack(&priv->card, &priv->hp_jack,
 					    1, NULL, "Headphone Jack");
 		if (ret)
@@ -1055,8 +1055,8 @@ static int fsl_asoc_card_probe(struct platform_device *pdev)
 		snd_soc_jack_notifier_register(&priv->hp_jack.jack, &hp_jack_nb);
 	}
 
-	if (of_property_read_bool(np, "mic-det-gpios") ||
-	    of_property_read_bool(np, "mic-det-gpio") /* deprecated */) {
+	if (of_property_present(np, "mic-det-gpios") ||
+	    of_property_present(np, "mic-det-gpio") /* deprecated */) {
 		ret = simple_util_init_jack(&priv->card, &priv->mic_jack,
 					    0, NULL, "Mic Jack");
 		if (ret)
diff --git a/sound/soc/fsl/fsl_sai.c b/sound/soc/fsl/fsl_sai.c
index 72bfc91e21b9..2fa14fbdfe1a 100644
--- a/sound/soc/fsl/fsl_sai.c
+++ b/sound/soc/fsl/fsl_sai.c
@@ -917,8 +917,14 @@ static int fsl_sai_startup(struct snd_pcm_substream *substream,
 					   tx ? sai->dma_params_tx.maxburst :
 					   sai->dma_params_rx.maxburst);
 
-	ret = snd_pcm_hw_constraint_list(substream->runtime, 0,
-					 SNDRV_PCM_HW_PARAM_RATE, &sai->constraint_rates);
+	if (sai->is_consumer_mode[tx])
+		ret = snd_pcm_hw_constraint_list(substream->runtime, 0,
+						 SNDRV_PCM_HW_PARAM_RATE,
+						 &fsl_sai_rate_constraints);
+	else
+		ret = snd_pcm_hw_constraint_list(substream->runtime, 0,
+						 SNDRV_PCM_HW_PARAM_RATE,
+						 &sai->constraint_rates);
 
 	return ret;
 }
@@ -1075,6 +1081,7 @@ static const struct reg_default fsl_sai_reg_defaults_ofs0[] = {
 	{FSL_SAI_TDR6, 0},
 	{FSL_SAI_TDR7, 0},
 	{FSL_SAI_TMR, 0},
+	{FSL_SAI_TTCTL, 0},
 	{FSL_SAI_RCR1(0), 0},
 	{FSL_SAI_RCR2(0), 0},
 	{FSL_SAI_RCR3(0), 0},
@@ -1098,12 +1105,14 @@ static const struct reg_default fsl_sai_reg_defaults_ofs8[] = {
 	{FSL_SAI_TDR6, 0},
 	{FSL_SAI_TDR7, 0},
 	{FSL_SAI_TMR, 0},
+	{FSL_SAI_TTCTL, 0},
 	{FSL_SAI_RCR1(8), 0},
 	{FSL_SAI_RCR2(8), 0},
 	{FSL_SAI_RCR3(8), 0},
 	{FSL_SAI_RCR4(8), 0},
 	{FSL_SAI_RCR5(8), 0},
 	{FSL_SAI_RMR, 0},
+	{FSL_SAI_RTCTL, 0},
 	{FSL_SAI_MCTL, 0},
 	{FSL_SAI_MDIV, 0},
 };
diff --git a/sound/soc/fsl/fsl_ssi.c b/sound/soc/fsl/fsl_ssi.c
index 320108bebf30..b2e1da1781ae 100644
--- a/sound/soc/fsl/fsl_ssi.c
+++ b/sound/soc/fsl/fsl_ssi.c
@@ -1447,7 +1447,7 @@ static int fsl_ssi_probe_from_dt(struct fsl_ssi *ssi)
 			dev_err(dev, "failed to get SSI index property\n");
 			return -EINVAL;
 		}
-		strcpy(ssi->card_name, "ac97-codec");
+		strscpy(ssi->card_name, "ac97-codec");
 	} else if (!of_property_read_bool(np, "fsl,ssi-asynchronous")) {
 		/*
 		 * In synchronous mode, STCK and STFS ports are used by RX
diff --git a/sound/soc/fsl/fsl_xcvr.c b/sound/soc/fsl/fsl_xcvr.c
index 06434b2c9a0f..a268fb81a2f8 100644
--- a/sound/soc/fsl/fsl_xcvr.c
+++ b/sound/soc/fsl/fsl_xcvr.c
@@ -1323,6 +1323,7 @@ static const struct reg_default fsl_xcvr_phy_reg_defaults[] = {
 };
 
 static const struct regmap_config fsl_xcvr_regmap_phy_cfg = {
+	.name = "phy",
 	.reg_bits = 8,
 	.reg_stride = 4,
 	.val_bits = 32,
@@ -1335,6 +1336,7 @@ static const struct regmap_config fsl_xcvr_regmap_phy_cfg = {
 };
 
 static const struct regmap_config fsl_xcvr_regmap_pllv0_cfg = {
+	.name = "pllv0",
 	.reg_bits = 8,
 	.reg_stride = 4,
 	.val_bits = 32,
@@ -1345,6 +1347,7 @@ static const struct regmap_config fsl_xcvr_regmap_pllv0_cfg = {
 };
 
 static const struct regmap_config fsl_xcvr_regmap_pllv1_cfg = {
+	.name = "pllv1",
 	.reg_bits = 8,
 	.reg_stride = 4,
 	.val_bits = 32,
diff --git a/sound/soc/intel/avs/board_selection.c b/sound/soc/intel/avs/board_selection.c
index 52e6266a7cb8..8a46285181fa 100644
--- a/sound/soc/intel/avs/board_selection.c
+++ b/sound/soc/intel/avs/board_selection.c
@@ -367,7 +367,7 @@ static const struct avs_acpi_boards i2s_boards[] = {
 	AVS_MACH_ENTRY(HDA_SKL_LP,	avs_skl_i2s_machines),
 	AVS_MACH_ENTRY(HDA_KBL_LP,	avs_kbl_i2s_machines),
 	AVS_MACH_ENTRY(HDA_APL,		avs_apl_i2s_machines),
-	AVS_MACH_ENTRY(HDA_GML,		avs_gml_i2s_machines),
+	AVS_MACH_ENTRY(HDA_GLK,		avs_gml_i2s_machines),
 	AVS_MACH_ENTRY(HDA_CNL_LP,	avs_cnl_i2s_machines),
 	AVS_MACH_ENTRY(HDA_CNL_H,	avs_cnl_i2s_machines),
 	AVS_MACH_ENTRY(HDA_CML_LP,	avs_cnl_i2s_machines),
diff --git a/sound/soc/intel/avs/core.c b/sound/soc/intel/avs/core.c
index 6e0e65584c7f..1a53856c2ffb 100644
--- a/sound/soc/intel/avs/core.c
+++ b/sound/soc/intel/avs/core.c
@@ -897,7 +897,7 @@ static const struct pci_device_id avs_ids[] = {
 	{ PCI_DEVICE_DATA(INTEL, HDA_KBL_H, &skl_desc) },
 	{ PCI_DEVICE_DATA(INTEL, HDA_CML_S, &skl_desc) },
 	{ PCI_DEVICE_DATA(INTEL, HDA_APL, &apl_desc) },
-	{ PCI_DEVICE_DATA(INTEL, HDA_GML, &apl_desc) },
+	{ PCI_DEVICE_DATA(INTEL, HDA_GLK, &apl_desc) },
 	{ PCI_DEVICE_DATA(INTEL, HDA_CNL_LP,	&cnl_desc) },
 	{ PCI_DEVICE_DATA(INTEL, HDA_CNL_H,	&cnl_desc) },
 	{ PCI_DEVICE_DATA(INTEL, HDA_CML_LP,	&cnl_desc) },
diff --git a/sound/soc/intel/catpt/core.h b/sound/soc/intel/catpt/core.h
index c01d27e9fd88..df8a5fd95e13 100644
--- a/sound/soc/intel/catpt/core.h
+++ b/sound/soc/intel/catpt/core.h
@@ -62,6 +62,7 @@ struct catpt_module_type {
 struct catpt_spec {
 	struct snd_soc_acpi_mach *machines;
 	u8 core_id;
+	const char *fw_name;
 	u32 host_dram_offset;
 	u32 host_iram_offset;
 	u32 host_shim_offset;
@@ -129,13 +130,13 @@ irqreturn_t catpt_dsp_irq_thread(int irq, void *dev_id);
  * HOST <-> DSP communication yet failure to process specific request.
  * Use below macro to convert returned non-zero values appropriately
  */
-#define CATPT_IPC_ERROR(err) (((err) < 0) ? (err) : -EREMOTEIO)
+#define CATPT_IPC_RET(ret)	(((ret) <= 0) ? (ret) : -EREMOTEIO)
 
 int catpt_dsp_send_msg_timeout(struct catpt_dev *cdev,
 			       struct catpt_ipc_msg request,
-			       struct catpt_ipc_msg *reply, int timeout);
+			       struct catpt_ipc_msg *reply, int timeout, const char *name);
 int catpt_dsp_send_msg(struct catpt_dev *cdev, struct catpt_ipc_msg request,
-		       struct catpt_ipc_msg *reply);
+		       struct catpt_ipc_msg *reply, const char *name);
 
 int catpt_first_boot_firmware(struct catpt_dev *cdev);
 int catpt_boot_firmware(struct catpt_dev *cdev, bool restore);
diff --git a/sound/soc/intel/catpt/device.c b/sound/soc/intel/catpt/device.c
index d13062c8e907..0638aecba40d 100644
--- a/sound/soc/intel/catpt/device.c
+++ b/sound/soc/intel/catpt/device.c
@@ -41,7 +41,7 @@ static int catpt_do_suspend(struct device *dev)
 	memset(&cdev->dx_ctx, 0, sizeof(cdev->dx_ctx));
 	ret = catpt_ipc_enter_dxstate(cdev, CATPT_DX_STATE_D3, &cdev->dx_ctx);
 	if (ret) {
-		ret = CATPT_IPC_ERROR(ret);
+		ret = CATPT_IPC_RET(ret);
 		goto release_dma_chan;
 	}
 
@@ -107,7 +107,7 @@ static int catpt_resume(struct device *dev)
 
 		ret = catpt_ipc_set_device_format(cdev, &cdev->devfmt[i]);
 		if (ret)
-			return CATPT_IPC_ERROR(ret);
+			return CATPT_IPC_RET(ret);
 	}
 
 	return 0;
@@ -348,6 +348,7 @@ static struct snd_soc_acpi_mach wpt_machines[] = {
 static struct catpt_spec lpt_desc = {
 	.machines = lpt_machines,
 	.core_id = 0x01,
+	.fw_name = "intel/IntcSST1.bin",
 	.host_dram_offset = 0x000000,
 	.host_iram_offset = 0x080000,
 	.host_shim_offset = 0x0E7000,
@@ -363,6 +364,7 @@ static struct catpt_spec lpt_desc = {
 static struct catpt_spec wpt_desc = {
 	.machines = wpt_machines,
 	.core_id = 0x02,
+	.fw_name = "intel/IntcSST2.bin",
 	.host_dram_offset = 0x000000,
 	.host_iram_offset = 0x0A0000,
 	.host_shim_offset = 0x0FB000,
diff --git a/sound/soc/intel/catpt/ipc.c b/sound/soc/intel/catpt/ipc.c
index d26863249097..5a01a9afb26e 100644
--- a/sound/soc/intel/catpt/ipc.c
+++ b/sound/soc/intel/catpt/ipc.c
@@ -84,7 +84,7 @@ static int catpt_wait_msg_completion(struct catpt_dev *cdev, int timeout)
 
 static int catpt_dsp_do_send_msg(struct catpt_dev *cdev,
 				 struct catpt_ipc_msg request,
-				 struct catpt_ipc_msg *reply, int timeout)
+				 struct catpt_ipc_msg *reply, int timeout, const char *name)
 {
 	struct catpt_ipc *ipc = &cdev->ipc;
 	unsigned long flags;
@@ -111,6 +111,8 @@ static int catpt_dsp_do_send_msg(struct catpt_dev *cdev,
 	}
 
 	ret = ipc->rx.rsp.status;
+	if (ret)
+		dev_err(cdev->dev, "%s (0x%08x) failed: %d\n", name, request.header, ret);
 	if (reply) {
 		reply->header = ipc->rx.header;
 
@@ -123,23 +125,23 @@ static int catpt_dsp_do_send_msg(struct catpt_dev *cdev,
 
 int catpt_dsp_send_msg_timeout(struct catpt_dev *cdev,
 			       struct catpt_ipc_msg request,
-			       struct catpt_ipc_msg *reply, int timeout)
+			       struct catpt_ipc_msg *reply, int timeout, const char *name)
 {
 	struct catpt_ipc *ipc = &cdev->ipc;
 	int ret;
 
 	mutex_lock(&ipc->mutex);
-	ret = catpt_dsp_do_send_msg(cdev, request, reply, timeout);
+	ret = catpt_dsp_do_send_msg(cdev, request, reply, timeout, name);
 	mutex_unlock(&ipc->mutex);
 
 	return ret;
 }
 
 int catpt_dsp_send_msg(struct catpt_dev *cdev, struct catpt_ipc_msg request,
-		       struct catpt_ipc_msg *reply)
+		       struct catpt_ipc_msg *reply, const char *name)
 {
 	return catpt_dsp_send_msg_timeout(cdev, request, reply,
-					  cdev->ipc.default_timeout);
+					  cdev->ipc.default_timeout, name);
 }
 
 static void
diff --git a/sound/soc/intel/catpt/loader.c b/sound/soc/intel/catpt/loader.c
index f5705cd2c1e1..dc7afe587e6f 100644
--- a/sound/soc/intel/catpt/loader.c
+++ b/sound/soc/intel/catpt/loader.c
@@ -580,10 +580,6 @@ static int catpt_load_image(struct catpt_dev *cdev, struct dma_chan *chan,
 
 static int catpt_load_images(struct catpt_dev *cdev, bool restore)
 {
-	static const char *const names[] = {
-		"intel/IntcSST1.bin",
-		"intel/IntcSST2.bin",
-	};
 	struct dma_chan *chan;
 	int ret;
 
@@ -591,7 +587,7 @@ static int catpt_load_images(struct catpt_dev *cdev, bool restore)
 	if (IS_ERR(chan))
 		return PTR_ERR(chan);
 
-	ret = catpt_load_image(cdev, chan, names[cdev->spec->core_id - 1],
+	ret = catpt_load_image(cdev, chan, cdev->spec->fw_name,
 			       FW_SIGNATURE, restore);
 	if (ret)
 		goto release_dma_chan;
@@ -656,7 +652,7 @@ int catpt_first_boot_firmware(struct catpt_dev *cdev)
 
 	ret = catpt_ipc_get_mixer_stream_info(cdev, &cdev->mixer);
 	if (ret)
-		return CATPT_IPC_ERROR(ret);
+		return CATPT_IPC_RET(ret);
 
 	ret = catpt_arm_stream_templates(cdev);
 	if (ret) {
diff --git a/sound/soc/intel/catpt/messages.c b/sound/soc/intel/catpt/messages.c
index 30eec2de4dc1..688a2d79500d 100644
--- a/sound/soc/intel/catpt/messages.c
+++ b/sound/soc/intel/catpt/messages.c
@@ -15,17 +15,12 @@ int catpt_ipc_get_fw_version(struct catpt_dev *cdev,
 {
 	union catpt_global_msg msg = CATPT_GLOBAL_MSG(GET_FW_VERSION);
 	struct catpt_ipc_msg request = {{0}}, reply;
-	int ret;
 
 	request.header = msg.val;
 	reply.size = sizeof(*version);
 	reply.data = version;
 
-	ret = catpt_dsp_send_msg(cdev, request, &reply);
-	if (ret)
-		dev_err(cdev->dev, "get fw version failed: %d\n", ret);
-
-	return ret;
+	return catpt_dsp_send_msg(cdev, request, &reply, "get fw version");
 }
 
 struct catpt_alloc_stream_input {
@@ -94,11 +89,7 @@ int catpt_ipc_alloc_stream(struct catpt_dev *cdev,
 	reply.size = sizeof(*sinfo);
 	reply.data = sinfo;
 
-	ret = catpt_dsp_send_msg(cdev, request, &reply);
-	if (ret)
-		dev_err(cdev->dev, "alloc stream type %d failed: %d\n",
-			type, ret);
-
+	ret = catpt_dsp_send_msg(cdev, request, &reply, "alloc stream");
 	kfree(payload);
 	return ret;
 }
@@ -107,18 +98,12 @@ int catpt_ipc_free_stream(struct catpt_dev *cdev, u8 stream_hw_id)
 {
 	union catpt_global_msg msg = CATPT_GLOBAL_MSG(FREE_STREAM);
 	struct catpt_ipc_msg request;
-	int ret;
 
 	request.header = msg.val;
 	request.size = sizeof(stream_hw_id);
 	request.data = &stream_hw_id;
 
-	ret = catpt_dsp_send_msg(cdev, request, NULL);
-	if (ret)
-		dev_err(cdev->dev, "free stream %d failed: %d\n",
-			stream_hw_id, ret);
-
-	return ret;
+	return catpt_dsp_send_msg(cdev, request, NULL, "free stream");
 }
 
 int catpt_ipc_set_device_format(struct catpt_dev *cdev,
@@ -126,17 +111,12 @@ int catpt_ipc_set_device_format(struct catpt_dev *cdev,
 {
 	union catpt_global_msg msg = CATPT_GLOBAL_MSG(SET_DEVICE_FORMATS);
 	struct catpt_ipc_msg request;
-	int ret;
 
 	request.header = msg.val;
 	request.size = sizeof(*devfmt);
 	request.data = devfmt;
 
-	ret = catpt_dsp_send_msg(cdev, request, NULL);
-	if (ret)
-		dev_err(cdev->dev, "set device format failed: %d\n", ret);
-
-	return ret;
+	return catpt_dsp_send_msg(cdev, request, NULL, "set device format");
 }
 
 int catpt_ipc_enter_dxstate(struct catpt_dev *cdev, enum catpt_dx_state state,
@@ -144,7 +124,6 @@ int catpt_ipc_enter_dxstate(struct catpt_dev *cdev, enum catpt_dx_state state,
 {
 	union catpt_global_msg msg = CATPT_GLOBAL_MSG(ENTER_DX_STATE);
 	struct catpt_ipc_msg request, reply;
-	int ret;
 
 	request.header = msg.val;
 	request.size = sizeof(state);
@@ -152,11 +131,7 @@ int catpt_ipc_enter_dxstate(struct catpt_dev *cdev, enum catpt_dx_state state,
 	reply.size = sizeof(*context);
 	reply.data = context;
 
-	ret = catpt_dsp_send_msg(cdev, request, &reply);
-	if (ret)
-		dev_err(cdev->dev, "enter dx state failed: %d\n", ret);
-
-	return ret;
+	return catpt_dsp_send_msg(cdev, request, &reply, "enter dx state");
 }
 
 int catpt_ipc_get_mixer_stream_info(struct catpt_dev *cdev,
@@ -164,68 +139,45 @@ int catpt_ipc_get_mixer_stream_info(struct catpt_dev *cdev,
 {
 	union catpt_global_msg msg = CATPT_GLOBAL_MSG(GET_MIXER_STREAM_INFO);
 	struct catpt_ipc_msg request = {{0}}, reply;
-	int ret;
 
 	request.header = msg.val;
 	reply.size = sizeof(*info);
 	reply.data = info;
 
-	ret = catpt_dsp_send_msg(cdev, request, &reply);
-	if (ret)
-		dev_err(cdev->dev, "get mixer info failed: %d\n", ret);
-
-	return ret;
+	return catpt_dsp_send_msg(cdev, request, &reply, "get mixer info");
 }
 
 int catpt_ipc_reset_stream(struct catpt_dev *cdev, u8 stream_hw_id)
 {
 	union catpt_stream_msg msg = CATPT_STREAM_MSG(RESET_STREAM);
 	struct catpt_ipc_msg request = {{0}};
-	int ret;
 
 	msg.stream_hw_id = stream_hw_id;
 	request.header = msg.val;
 
-	ret = catpt_dsp_send_msg(cdev, request, NULL);
-	if (ret)
-		dev_err(cdev->dev, "reset stream %d failed: %d\n",
-			stream_hw_id, ret);
-
-	return ret;
+	return catpt_dsp_send_msg(cdev, request, NULL, "reset stream");
 }
 
 int catpt_ipc_pause_stream(struct catpt_dev *cdev, u8 stream_hw_id)
 {
 	union catpt_stream_msg msg = CATPT_STREAM_MSG(PAUSE_STREAM);
 	struct catpt_ipc_msg request = {{0}};
-	int ret;
 
 	msg.stream_hw_id = stream_hw_id;
 	request.header = msg.val;
 
-	ret = catpt_dsp_send_msg(cdev, request, NULL);
-	if (ret)
-		dev_err(cdev->dev, "pause stream %d failed: %d\n",
-			stream_hw_id, ret);
-
-	return ret;
+	return catpt_dsp_send_msg(cdev, request, NULL, "pause stream");
 }
 
 int catpt_ipc_resume_stream(struct catpt_dev *cdev, u8 stream_hw_id)
 {
 	union catpt_stream_msg msg = CATPT_STREAM_MSG(RESUME_STREAM);
 	struct catpt_ipc_msg request = {{0}};
-	int ret;
 
 	msg.stream_hw_id = stream_hw_id;
 	request.header = msg.val;
 
-	ret = catpt_dsp_send_msg(cdev, request, NULL);
-	if (ret)
-		dev_err(cdev->dev, "resume stream %d failed: %d\n",
-			stream_hw_id, ret);
-
-	return ret;
+	return catpt_dsp_send_msg(cdev, request, NULL, "resume stream");
 }
 
 struct catpt_set_volume_input {
@@ -243,7 +195,6 @@ int catpt_ipc_set_volume(struct catpt_dev *cdev, u8 stream_hw_id,
 	union catpt_stream_msg msg = CATPT_STAGE_MSG(SET_VOLUME);
 	struct catpt_ipc_msg request;
 	struct catpt_set_volume_input input;
-	int ret;
 
 	msg.stream_hw_id = stream_hw_id;
 	input.channel = channel;
@@ -255,12 +206,7 @@ int catpt_ipc_set_volume(struct catpt_dev *cdev, u8 stream_hw_id,
 	request.size = sizeof(input);
 	request.data = &input;
 
-	ret = catpt_dsp_send_msg(cdev, request, NULL);
-	if (ret)
-		dev_err(cdev->dev, "set stream %d volume failed: %d\n",
-			stream_hw_id, ret);
-
-	return ret;
+	return catpt_dsp_send_msg(cdev, request, NULL, "set stream volume");
 }
 
 struct catpt_set_write_pos_input {
@@ -275,7 +221,6 @@ int catpt_ipc_set_write_pos(struct catpt_dev *cdev, u8 stream_hw_id,
 	union catpt_stream_msg msg = CATPT_STAGE_MSG(SET_WRITE_POSITION);
 	struct catpt_ipc_msg request;
 	struct catpt_set_write_pos_input input;
-	int ret;
 
 	msg.stream_hw_id = stream_hw_id;
 	input.new_write_pos = pos;
@@ -286,28 +231,18 @@ int catpt_ipc_set_write_pos(struct catpt_dev *cdev, u8 stream_hw_id,
 	request.size = sizeof(input);
 	request.data = &input;
 
-	ret = catpt_dsp_send_msg(cdev, request, NULL);
-	if (ret)
-		dev_err(cdev->dev, "set stream %d write pos failed: %d\n",
-			stream_hw_id, ret);
-
-	return ret;
+	return catpt_dsp_send_msg(cdev, request, NULL, "set stream write pos");
 }
 
 int catpt_ipc_mute_loopback(struct catpt_dev *cdev, u8 stream_hw_id, bool mute)
 {
 	union catpt_stream_msg msg = CATPT_STAGE_MSG(MUTE_LOOPBACK);
 	struct catpt_ipc_msg request;
-	int ret;
 
 	msg.stream_hw_id = stream_hw_id;
 	request.header = msg.val;
 	request.size = sizeof(mute);
 	request.data = &mute;
 
-	ret = catpt_dsp_send_msg(cdev, request, NULL);
-	if (ret)
-		dev_err(cdev->dev, "mute loopback failed: %d\n", ret);
-
-	return ret;
+	return catpt_dsp_send_msg(cdev, request, NULL, "mute loopback");
 }
diff --git a/sound/soc/intel/catpt/pcm.c b/sound/soc/intel/catpt/pcm.c
index abd1cb07c60c..2c3405686f79 100644
--- a/sound/soc/intel/catpt/pcm.c
+++ b/sound/soc/intel/catpt/pcm.c
@@ -114,14 +114,10 @@ catpt_stream_find(struct catpt_dev *cdev, u8 stream_hw_id)
 	return result;
 }
 
-static u32 catpt_stream_read_position(struct catpt_dev *cdev,
-				      struct catpt_stream_runtime *stream)
+static void catpt_stream_read_position(struct catpt_dev *cdev,
+				       struct catpt_stream_runtime *stream, u32 *pos)
 {
-	u32 pos;
-
-	memcpy_fromio(&pos, cdev->lpe_ba + stream->info.read_pos_regaddr,
-		      sizeof(pos));
-	return pos;
+	memcpy_fromio(pos, cdev->lpe_ba + stream->info.read_pos_regaddr, sizeof(*pos));
 }
 
 static u32 catpt_stream_volume(struct catpt_dev *cdev,
@@ -365,9 +361,7 @@ static int catpt_dai_apply_usettings(struct snd_soc_dai *dai,
 	if (stream->template->type != CATPT_STRM_TYPE_LOOPBACK)
 		return catpt_set_dspvol(cdev, id, (long *)pos->private_value);
 	ret = catpt_ipc_mute_loopback(cdev, id, *(bool *)pos->private_value);
-	if (ret)
-		return CATPT_IPC_ERROR(ret);
-	return 0;
+	return CATPT_IPC_RET(ret);
 }
 
 static int catpt_dai_hw_params(struct snd_pcm_substream *substream,
@@ -414,7 +408,7 @@ static int catpt_dai_hw_params(struct snd_pcm_substream *substream,
 				     cdev->scratch,
 				     &stream->info);
 	if (ret)
-		return CATPT_IPC_ERROR(ret);
+		return CATPT_IPC_RET(ret);
 
 	ret = catpt_dai_apply_usettings(dai, stream);
 	if (ret) {
@@ -456,11 +450,11 @@ static int catpt_dai_prepare(struct snd_pcm_substream *substream,
 
 	ret = catpt_ipc_reset_stream(cdev, stream->info.stream_hw_id);
 	if (ret)
-		return CATPT_IPC_ERROR(ret);
+		return CATPT_IPC_RET(ret);
 
 	ret = catpt_ipc_pause_stream(cdev, stream->info.stream_hw_id);
 	if (ret)
-		return CATPT_IPC_ERROR(ret);
+		return CATPT_IPC_RET(ret);
 
 	stream->prepared = true;
 	return 0;
@@ -491,7 +485,7 @@ static int catpt_dai_trigger(struct snd_pcm_substream *substream, int cmd,
 		ret = catpt_ipc_set_write_pos(cdev, stream->info.stream_hw_id,
 					      pos, false, false);
 		if (ret)
-			return CATPT_IPC_ERROR(ret);
+			return CATPT_IPC_RET(ret);
 		fallthrough;
 	case SNDRV_PCM_TRIGGER_RESUME:
 	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
@@ -499,7 +493,7 @@ static int catpt_dai_trigger(struct snd_pcm_substream *substream, int cmd,
 		catpt_dsp_update_lpclock(cdev);
 		ret = catpt_ipc_resume_stream(cdev, stream->info.stream_hw_id);
 		if (ret)
-			return CATPT_IPC_ERROR(ret);
+			return CATPT_IPC_RET(ret);
 		break;
 
 	case SNDRV_PCM_TRIGGER_STOP:
@@ -510,7 +504,7 @@ static int catpt_dai_trigger(struct snd_pcm_substream *substream, int cmd,
 		ret = catpt_ipc_pause_stream(cdev, stream->info.stream_hw_id);
 		catpt_dsp_update_lpclock(cdev);
 		if (ret)
-			return CATPT_IPC_ERROR(ret);
+			return CATPT_IPC_RET(ret);
 		break;
 
 	default:
@@ -617,7 +611,7 @@ catpt_component_pointer(struct snd_soc_component *component,
 		return 0;
 
 	stream = snd_soc_dai_get_dma_data(cpu_dai, substream);
-	pos = catpt_stream_read_position(cdev, stream);
+	catpt_stream_read_position(cdev, stream, &pos);
 
 	return bytes_to_frames(substream->runtime, pos);
 }
@@ -679,7 +673,7 @@ static int catpt_dai_pcm_new(struct snd_soc_pcm_runtime *rtm,
 	pm_runtime_put_autosuspend(cdev->dev);
 
 	if (ret)
-		return CATPT_IPC_ERROR(ret);
+		return CATPT_IPC_RET(ret);
 
 	/* store device format set for given SSP */
 	memcpy(&cdev->devfmt[devfmt.iface], &devfmt, sizeof(devfmt));
@@ -693,7 +687,7 @@ static const struct snd_soc_dai_ops catpt_dai_ops = {
 static struct snd_soc_dai_driver dai_drivers[] = {
 /* FE DAIs */
 {
-	.name  = "System Pin",
+	.name = "System Pin",
 	.id = CATPT_STRM_TYPE_SYSTEM,
 	.ops = &catpt_fe_dai_ops,
 	.playback = {
@@ -716,7 +710,7 @@ static struct snd_soc_dai_driver dai_drivers[] = {
 	},
 },
 {
-	.name  = "Offload0 Pin",
+	.name = "Offload0 Pin",
 	.id = CATPT_STRM_TYPE_RENDER,
 	.ops = &catpt_fe_dai_ops,
 	.playback = {
@@ -730,7 +724,7 @@ static struct snd_soc_dai_driver dai_drivers[] = {
 	},
 },
 {
-	.name  = "Offload1 Pin",
+	.name = "Offload1 Pin",
 	.id = CATPT_STRM_TYPE_RENDER,
 	.ops = &catpt_fe_dai_ops,
 	.playback = {
@@ -744,7 +738,7 @@ static struct snd_soc_dai_driver dai_drivers[] = {
 	},
 },
 {
-	.name  = "Loopback Pin",
+	.name = "Loopback Pin",
 	.id = CATPT_STRM_TYPE_LOOPBACK,
 	.ops = &catpt_fe_dai_ops,
 	.capture = {
@@ -758,7 +752,7 @@ static struct snd_soc_dai_driver dai_drivers[] = {
 	},
 },
 {
-	.name  = "Bluetooth Pin",
+	.name = "Bluetooth Pin",
 	.id = CATPT_STRM_TYPE_BLUETOOTH_RENDER,
 	.ops = &catpt_fe_dai_ops,
 	.playback = {
@@ -849,9 +843,7 @@ static int catpt_set_dspvol(struct catpt_dev *cdev, u8 stream_id, long *ctlvol)
 		}
 	}
 
-	if (ret)
-		return CATPT_IPC_ERROR(ret);
-	return 0;
+	return CATPT_IPC_RET(ret);
 }
 
 static int catpt_volume_info(struct snd_kcontrol *kcontrol,
@@ -1041,7 +1033,7 @@ static int catpt_loopback_switch_put(struct snd_kcontrol *kcontrol,
 	pm_runtime_put_autosuspend(cdev->dev);
 
 	if (ret)
-		return CATPT_IPC_ERROR(ret);
+		return CATPT_IPC_RET(ret);
 
 	*(bool *)kcontrol->private_value = mute;
 	return 0;
diff --git a/sound/soc/intel/catpt/sysfs.c b/sound/soc/intel/catpt/sysfs.c
index e961e172f9b7..0cb122a4dfd2 100644
--- a/sound/soc/intel/catpt/sysfs.c
+++ b/sound/soc/intel/catpt/sysfs.c
@@ -24,7 +24,7 @@ static ssize_t fw_version_show(struct device *dev,
 	pm_runtime_put_autosuspend(cdev->dev);
 
 	if (ret)
-		return CATPT_IPC_ERROR(ret);
+		return CATPT_IPC_RET(ret);
 
 	return sysfs_emit(buf, "%d.%d.%d.%d\n", version.type, version.major,
 			  version.minor, version.build);
diff --git a/sound/soc/intel/common/soc-acpi-intel-mtl-match.c b/sound/soc/intel/common/soc-acpi-intel-mtl-match.c
index ec9fd8486c05..f12d42986a75 100644
--- a/sound/soc/intel/common/soc-acpi-intel-mtl-match.c
+++ b/sound/soc/intel/common/soc-acpi-intel-mtl-match.c
@@ -950,7 +950,7 @@ static const struct snd_soc_acpi_adr_device cs42l42_0_adr[] = {
 
 static const struct snd_soc_acpi_adr_device tas2783_0_adr[] = {
 	{
-		.adr = 0x0000380102000001ull,
+		.adr = 0x00003c0102000001ull,
 		.num_endpoints = 1,
 		.endpoints = &spk_l_endpoint,
 		.name_prefix = "tas2783-1"
@@ -960,6 +960,18 @@ static const struct snd_soc_acpi_adr_device tas2783_0_adr[] = {
 		.num_endpoints = 1,
 		.endpoints = &spk_r_endpoint,
 		.name_prefix = "tas2783-2"
+	},
+	{
+		.adr = 0x00003d0102000001ull,
+		.num_endpoints = 1,
+		.endpoints = &spk_l_endpoint,
+		.name_prefix = "tas2783-3"
+	},
+	{
+		.adr = 0x00003a0102000001ull,
+		.num_endpoints = 1,
+		.endpoints = &spk_r_endpoint,
+		.name_prefix = "tas2783-4"
 	}
 };
 
diff --git a/sound/soc/intel/common/soc-acpi-intel-nvl-match.c b/sound/soc/intel/common/soc-acpi-intel-nvl-match.c
index 2768dd10aaa0..b8695d47e55b 100644
--- a/sound/soc/intel/common/soc-acpi-intel-nvl-match.c
+++ b/sound/soc/intel/common/soc-acpi-intel-nvl-match.c
@@ -15,49 +15,6 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_nvl_machines[] = {
 };
 EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_nvl_machines);
 
-/*
- * Multi-function codecs with three endpoints created for
- * headset, amp and dmic functions.
- */
-static const struct snd_soc_acpi_endpoint rt_mf_endpoints[] = {
-	{
-		.num = 0,
-		.aggregated = 0,
-		.group_position = 0,
-		.group_id = 0,
-	},
-	{
-		.num = 1,
-		.aggregated = 0,
-		.group_position = 0,
-		.group_id = 0,
-	},
-	{
-		.num = 2,
-		.aggregated = 0,
-		.group_position = 0,
-		.group_id = 0,
-	},
-};
-
-static const struct snd_soc_acpi_adr_device rt722_3_single_adr[] = {
-	{
-		.adr = 0x000330025d072201ull,
-		.num_endpoints = ARRAY_SIZE(rt_mf_endpoints),
-		.endpoints = rt_mf_endpoints,
-		.name_prefix = "rt722"
-	}
-};
-
-static const struct snd_soc_acpi_link_adr nvl_rt722_l3[] = {
-	{
-		.mask = BIT(3),
-		.num_adr = ARRAY_SIZE(rt722_3_single_adr),
-		.adr_d = rt722_3_single_adr,
-	},
-	{}
-};
-
 /* this table is used when there is no I2S codec present */
 struct snd_soc_acpi_mach snd_soc_acpi_intel_nvl_sdw_machines[] = {
 	/* mockup tests need to be first */
@@ -79,12 +36,6 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_nvl_sdw_machines[] = {
 		.drv_name = "sof_sdw",
 		.sof_tplg_filename = "sof-nvl-rt715-rt711-rt1308-mono.tplg",
 	},
-	{
-		.link_mask = BIT(3),
-		.links = nvl_rt722_l3,
-		.drv_name = "sof_sdw",
-		.sof_tplg_filename = "sof-nvl-rt722.tplg",
-	},
 	{},
 };
 EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_nvl_sdw_machines);
diff --git a/sound/soc/intel/common/sof-function-topology-lib.c b/sound/soc/intel/common/sof-function-topology-lib.c
index b10d4794159a..0daa7d83808b 100644
--- a/sound/soc/intel/common/sof-function-topology-lib.c
+++ b/sound/soc/intel/common/sof-function-topology-lib.c
@@ -28,7 +28,7 @@ enum tplg_device_id {
 #define SOF_INTEL_PLATFORM_NAME_MAX 4
 
 int sof_sdw_get_tplg_files(struct snd_soc_card *card, const struct snd_soc_acpi_mach *mach,
-			   const char *prefix, const char ***tplg_files)
+			   const char *prefix, const char ***tplg_files, bool best_effort)
 {
 	struct snd_soc_acpi_mach_params mach_params = mach->mach_params;
 	struct snd_soc_dai_link *dai_link;
@@ -87,6 +87,9 @@ int sof_sdw_get_tplg_files(struct snd_soc_card *card, const struct snd_soc_acpi_
 			dev_dbg(card->dev,
 				"dai_link %s is not supported by separated tplg yet\n",
 				dai_link->name);
+			if (best_effort)
+				continue;
+
 			return 0;
 		}
 		if (tplg_mask & BIT(tplg_dev))
diff --git a/sound/soc/intel/common/sof-function-topology-lib.h b/sound/soc/intel/common/sof-function-topology-lib.h
index e7d0c39d0788..f358f8c52d78 100644
--- a/sound/soc/intel/common/sof-function-topology-lib.h
+++ b/sound/soc/intel/common/sof-function-topology-lib.h
@@ -10,6 +10,6 @@
 #define _SND_SOC_ACPI_INTEL_GET_TPLG_H
 
 int sof_sdw_get_tplg_files(struct snd_soc_card *card, const struct snd_soc_acpi_mach *mach,
-			   const char *prefix, const char ***tplg_files);
+			   const char *prefix, const char ***tplg_files, bool best_effort);
 
 #endif
diff --git a/sound/soc/mediatek/mt8189/mt8189-nau8825.c b/sound/soc/mediatek/mt8189/mt8189-nau8825.c
index 5ef15ec988be..e849e7a649bc 100644
--- a/sound/soc/mediatek/mt8189/mt8189-nau8825.c
+++ b/sound/soc/mediatek/mt8189/mt8189-nau8825.c
@@ -342,9 +342,10 @@ static const struct snd_soc_ops mt8189_es8326_ops = {
 static int mt8189_dumb_amp_init(struct snd_soc_pcm_runtime *rtd)
 {
 	struct snd_soc_card *card = rtd->card;
+	struct snd_soc_dapm_context *dapm = snd_soc_card_to_dapm(card);
 	int ret;
 
-	ret = snd_soc_dapm_new_controls(&card->dapm, mt8189_dumb_spk_widgets,
+	ret = snd_soc_dapm_new_controls(dapm, mt8189_dumb_spk_widgets,
 					ARRAY_SIZE(mt8189_dumb_spk_widgets));
 	if (ret) {
 		dev_err(rtd->dev, "unable to add Dumb Speaker dapm, ret %d\n", ret);
@@ -418,10 +419,11 @@ static int mt8189_headset_codec_init(struct snd_soc_pcm_runtime *rtd)
 	struct snd_soc_jack *jack = &soc_card_data->card_data->jacks[MT8189_JACK_HEADSET];
 	struct snd_soc_component *component = snd_soc_rtd_to_codec(rtd, 0)->component;
 	struct mtk_platform_card_data *card_data = soc_card_data->card_data;
+	struct snd_soc_dapm_context *dapm = snd_soc_card_to_dapm(card);
 	int ret;
 	int type;
 
-	ret = snd_soc_dapm_new_controls(&card->dapm, mt8189_headset_widgets,
+	ret = snd_soc_dapm_new_controls(dapm, mt8189_headset_widgets,
 					ARRAY_SIZE(mt8189_headset_widgets));
 	if (ret) {
 		dev_err(rtd->dev, "unable to add nau8825 card widget, ret %d\n", ret);
diff --git a/sound/soc/mediatek/mt8195/mt8195-dai-etdm.c b/sound/soc/mediatek/mt8195/mt8195-dai-etdm.c
index 723cab01e72e..5dcc8ed26e00 100644
--- a/sound/soc/mediatek/mt8195/mt8195-dai-etdm.c
+++ b/sound/soc/mediatek/mt8195/mt8195-dai-etdm.c
@@ -2651,14 +2651,9 @@ static void mt8195_dai_etdm_parse_of(struct mtk_base_afe *afe)
 
 		etdm_data = afe_priv->dai_priv[dai_id];
 
-		ret = snprintf(prop, sizeof(prop),
-			       "mediatek,%s-mclk-always-on-rate",
-			       of_afe_etdms[i].name);
-		if (ret < 0) {
-			dev_info(afe->dev, "%s snprintf err=%d\n",
-				 __func__, ret);
-			return;
-		}
+		scnprintf(prop, sizeof(prop),
+			    "mediatek,%s-mclk-always-on-rate",
+			    of_afe_etdms[i].name);
 		ret = of_property_read_u32(of_node, prop, &sel);
 		if (ret == 0) {
 			etdm_data->mclk_dir = SND_SOC_CLOCK_OUT;
@@ -2667,24 +2662,14 @@ static void mt8195_dai_etdm_parse_of(struct mtk_base_afe *afe)
 					 __func__, sel);
 		}
 
-		ret = snprintf(prop, sizeof(prop),
-			       "mediatek,%s-multi-pin-mode",
-			       of_afe_etdms[i].name);
-		if (ret < 0) {
-			dev_info(afe->dev, "%s snprintf err=%d\n",
-				 __func__, ret);
-			return;
-		}
+		scnprintf(prop, sizeof(prop),
+			    "mediatek,%s-multi-pin-mode",
+			    of_afe_etdms[i].name);
 		etdm_data->data_mode = of_property_read_bool(of_node, prop);
 
-		ret = snprintf(prop, sizeof(prop),
-			       "mediatek,%s-cowork-source",
-			       of_afe_etdms[i].name);
-		if (ret < 0) {
-			dev_info(afe->dev, "%s snprintf err=%d\n",
-				 __func__, ret);
-			return;
-		}
+		scnprintf(prop, sizeof(prop),
+			    "mediatek,%s-cowork-source",
+			    of_afe_etdms[i].name);
 		ret = of_property_read_u32(of_node, prop, &sel);
 		if (ret == 0) {
 			if (sel >= MT8195_AFE_IO_ETDM_NUM) {
@@ -2706,14 +2691,9 @@ static void mt8195_dai_etdm_parse_of(struct mtk_base_afe *afe)
 		dai_id = ETDM_TO_DAI_ID(i);
 		etdm_data = afe_priv->dai_priv[dai_id];
 
-		ret = snprintf(prop, sizeof(prop),
-			       "mediatek,%s-chn-disabled",
-			       of_afe_etdms[i].name);
-		if (ret < 0) {
-			dev_info(afe->dev, "%s snprintf err=%d\n",
-				 __func__, ret);
-			return;
-		}
+		scnprintf(prop, sizeof(prop),
+			    "mediatek,%s-chn-disabled",
+			    of_afe_etdms[i].name);
 		ret = of_property_read_variable_u8_array(of_node, prop,
 							 disable_chn,
 							 1, max_chn);
diff --git a/sound/soc/qcom/qdsp6/audioreach.c b/sound/soc/qcom/qdsp6/audioreach.c
index ded49124581b..b28451558974 100644
--- a/sound/soc/qcom/qdsp6/audioreach.c
+++ b/sound/soc/qcom/qdsp6/audioreach.c
@@ -202,6 +202,31 @@ struct apm_display_port_module_intf_cfg {
 } __packed;
 #define APM_DP_INTF_CFG_PSIZE ALIGN(sizeof(struct apm_display_port_module_intf_cfg), 8)
 
+struct apm_module_sp_vi_op_mode_cfg {
+	struct apm_module_param_data param_data;
+	struct param_id_sp_vi_op_mode_cfg cfg;
+} __packed;
+
+#define APM_SP_VI_OP_MODE_CFG_PSIZE(ch) ALIGN( \
+				sizeof(struct apm_module_sp_vi_op_mode_cfg) + \
+				(ch) * sizeof(uint32_t), 8)
+
+struct apm_module_sp_vi_ex_mode_cfg {
+	struct apm_module_param_data param_data;
+	struct param_id_sp_vi_ex_mode_cfg cfg;
+} __packed;
+
+#define APM_SP_VI_EX_MODE_CFG_PSIZE ALIGN(sizeof(struct apm_module_sp_vi_ex_mode_cfg), 8)
+
+struct apm_module_sp_vi_channel_map_cfg {
+	struct apm_module_param_data param_data;
+	struct param_id_sp_vi_channel_map_cfg cfg;
+} __packed;
+
+#define APM_SP_VI_CH_MAP_CFG_PSIZE(ch) ALIGN( \
+				sizeof(struct apm_module_sp_vi_channel_map_cfg) + \
+				(ch) * sizeof(uint32_t), 8)
+
 static void *__audioreach_alloc_pkt(int payload_size, uint32_t opcode, uint32_t token,
 				    uint32_t src_port, uint32_t dest_port, bool has_cmd_hdr)
 {
@@ -617,6 +642,7 @@ static int audioreach_display_port_set_media_format(struct q6apm_graph *graph,
 	int fs_sz = APM_FS_CFG_PSIZE;
 	int size = ic_sz + ep_sz + fs_sz;
 	void *p;
+
 	struct gpr_pkt *pkt __free(kfree) = audioreach_alloc_apm_cmd_pkt(size, APM_CMD_SET_CFG, 0);
 	if (IS_ERR(pkt))
 		return PTR_ERR(pkt);
@@ -675,6 +701,7 @@ static int audioreach_codec_dma_set_media_format(struct q6apm_graph *graph,
 	int pm_sz = APM_HW_EP_PMODE_CFG_PSIZE;
 	int size = ic_sz + ep_sz + fs_sz + pm_sz;
 	void *p;
+
 	struct gpr_pkt *pkt __free(kfree) = audioreach_alloc_apm_cmd_pkt(size, APM_CMD_SET_CFG, 0);
 	if (IS_ERR(pkt))
 		return PTR_ERR(pkt);
@@ -730,15 +757,15 @@ int audioreach_send_u32_param(struct q6apm_graph *graph, struct audioreach_modul
 			      uint32_t param_id, uint32_t param_val)
 {
 	struct apm_module_param_data *param_data;
-	struct gpr_pkt *pkt __free(kfree) = NULL;
 	uint32_t *param;
 	int payload_size = sizeof(uint32_t) + APM_MODULE_PARAM_DATA_SIZE;
-	void *p = audioreach_alloc_apm_cmd_pkt(payload_size, APM_CMD_SET_CFG, 0);
-	if (IS_ERR(p))
+	void *p;
+
+	struct gpr_pkt *pkt __free(kfree) = audioreach_alloc_apm_cmd_pkt(payload_size, APM_CMD_SET_CFG, 0);
+	if (IS_ERR(pkt))
 		return -ENOMEM;
 
-	pkt = p;
-	p = p + GPR_HDR_SIZE + APM_CMD_HDR_SIZE;
+	p = (void *)pkt + GPR_HDR_SIZE + APM_CMD_HDR_SIZE;
 
 	param_data = p;
 	param_data->module_instance_id = module->instance_id;
@@ -788,6 +815,7 @@ static int audioreach_set_module_config(struct q6apm_graph *graph,
 {
 	int size = le32_to_cpu(module->data->size);
 	void *p;
+
 	struct gpr_pkt *pkt __free(kfree) = audioreach_alloc_apm_cmd_pkt(size, APM_CMD_SET_CFG, 0);
 	if (IS_ERR(pkt))
 		return PTR_ERR(pkt);
@@ -810,6 +838,7 @@ static int audioreach_mfc_set_media_format(struct q6apm_graph *graph,
 				APM_MODULE_PARAM_DATA_SIZE;
 	int i;
 	void *p;
+
 	struct gpr_pkt *pkt __free(kfree) = audioreach_alloc_apm_cmd_pkt(payload_size, APM_CMD_SET_CFG, 0);
 	if (IS_ERR(pkt))
 		return PTR_ERR(pkt);
@@ -922,13 +951,13 @@ int audioreach_compr_set_param(struct q6apm_graph *graph, struct audioreach_modu
 	void *p;
 	int iid = q6apm_graph_get_rx_shmem_module_iid(graph);
 	int payload_size = sizeof(struct apm_sh_module_media_fmt_cmd);
+
 	struct gpr_pkt *pkt __free(kfree) = audioreach_alloc_cmd_pkt(payload_size,
 					DATA_CMD_WR_SH_MEM_EP_MEDIA_FORMAT,
 					0, graph->port->id, iid);
 	if (IS_ERR(pkt))
 		return -ENOMEM;
 
-
 	p = (void *)pkt + GPR_HDR_SIZE;
 	header = p;
 	rc = audioreach_set_compr_media_format(header, p, mcfg);
@@ -952,6 +981,7 @@ static int audioreach_i2s_set_media_format(struct q6apm_graph *graph,
 	int fs_sz = APM_FS_CFG_PSIZE;
 	int size = ic_sz + ep_sz + fs_sz;
 	void *p;
+
 	struct gpr_pkt *pkt __free(kfree) = audioreach_alloc_apm_cmd_pkt(size, APM_CMD_SET_CFG, 0);
 	if (IS_ERR(pkt))
 		return PTR_ERR(pkt);
@@ -1013,6 +1043,7 @@ static int audioreach_logging_set_media_format(struct q6apm_graph *graph,
 	struct data_logging_config *cfg;
 	int size = sizeof(*cfg) + APM_MODULE_PARAM_DATA_SIZE;
 	void *p;
+
 	struct gpr_pkt *pkt __free(kfree) = audioreach_alloc_apm_cmd_pkt(size, APM_CMD_SET_CFG, 0);
 	if (IS_ERR(pkt))
 		return PTR_ERR(pkt);
@@ -1043,7 +1074,6 @@ static int audioreach_pcm_set_media_format(struct q6apm_graph *graph,
 	struct apm_pcm_module_media_fmt_cmd *cfg;
 	struct apm_module_param_data *param_data;
 	int payload_size;
-	struct gpr_pkt *pkt __free(kfree) = NULL;
 
 	if (num_channels > 4) {
 		dev_err(graph->dev, "Error: Invalid channels (%d)!\n", num_channels);
@@ -1052,7 +1082,8 @@ static int audioreach_pcm_set_media_format(struct q6apm_graph *graph,
 
 	payload_size = APM_PCM_MODULE_FMT_CMD_PSIZE(num_channels);
 
-	pkt = audioreach_alloc_apm_cmd_pkt(payload_size, APM_CMD_SET_CFG, 0);
+	struct gpr_pkt *pkt __free(kfree) =
+		audioreach_alloc_apm_cmd_pkt(payload_size, APM_CMD_SET_CFG, 0);
 	if (IS_ERR(pkt))
 		return PTR_ERR(pkt);
 
@@ -1090,7 +1121,6 @@ static int audioreach_shmem_set_media_format(struct q6apm_graph *graph,
 	struct payload_media_fmt_pcm *cfg;
 	struct media_format *header;
 	int rc, payload_size;
-	struct gpr_pkt *pkt __free(kfree) = NULL;
 	void *p;
 
 	if (num_channels > 4) {
@@ -1100,8 +1130,9 @@ static int audioreach_shmem_set_media_format(struct q6apm_graph *graph,
 
 	payload_size = APM_SHMEM_FMT_CFG_PSIZE(num_channels) + APM_MODULE_PARAM_DATA_SIZE;
 
-	pkt = audioreach_alloc_cmd_pkt(payload_size, APM_CMD_SET_CFG, 0,
-				     graph->port->id, module->instance_id);
+	struct gpr_pkt *pkt __free(kfree) =
+		audioreach_alloc_cmd_pkt(payload_size, APM_CMD_SET_CFG, 0,
+					 graph->port->id, module->instance_id);
 	if (IS_ERR(pkt))
 		return PTR_ERR(pkt);
 
@@ -1186,6 +1217,93 @@ static int audioreach_gain_set(struct q6apm_graph *graph, struct audioreach_modu
 	return q6apm_send_cmd_sync(graph->apm, pkt, 0);
 }
 
+static int audioreach_speaker_protection(struct q6apm_graph *graph,
+					 struct audioreach_module *module,
+					 uint32_t operation_mode)
+{
+	return audioreach_send_u32_param(graph, module, PARAM_ID_SP_OP_MODE,
+					 operation_mode);
+}
+
+static int audioreach_speaker_protection_vi(struct q6apm_graph *graph,
+					    struct audioreach_module *module,
+					    struct audioreach_module_config *mcfg)
+{
+	u32 num_channels = mcfg->num_channels;
+	struct apm_module_sp_vi_op_mode_cfg *op_cfg;
+	struct apm_module_sp_vi_channel_map_cfg *cm_cfg;
+	struct apm_module_sp_vi_ex_mode_cfg *ex_cfg;
+	int op_sz, cm_sz, ex_sz;
+	struct apm_module_param_data *param_data;
+	int rc, i, payload_size;
+	struct gpr_pkt *pkt;
+	void *p;
+
+	if (num_channels > 2) {
+		dev_err(graph->dev, "Error: Invalid channels (%d)!\n", num_channels);
+		return -EINVAL;
+	}
+
+	op_sz = APM_SP_VI_OP_MODE_CFG_PSIZE(num_channels);
+	/* Channel mapping for Isense and Vsense, thus twice number of speakers. */
+	cm_sz = APM_SP_VI_CH_MAP_CFG_PSIZE(num_channels * 2);
+	ex_sz = APM_SP_VI_EX_MODE_CFG_PSIZE;
+
+	payload_size = op_sz + cm_sz + ex_sz;
+
+	pkt = audioreach_alloc_apm_cmd_pkt(payload_size, APM_CMD_SET_CFG, 0);
+	if (IS_ERR(pkt))
+		return PTR_ERR(pkt);
+
+	p = (void *)pkt + GPR_HDR_SIZE + APM_CMD_HDR_SIZE;
+
+	op_cfg = p;
+	param_data = &op_cfg->param_data;
+	param_data->module_instance_id = module->instance_id;
+	param_data->error_code = 0;
+	param_data->param_id = PARAM_ID_SP_VI_OP_MODE_CFG;
+	param_data->param_size = op_sz - APM_MODULE_PARAM_DATA_SIZE;
+
+	op_cfg->cfg.num_channels = num_channels;
+	op_cfg->cfg.operation_mode = PARAM_ID_SP_VI_OP_MODE_NORMAL;
+	p += op_sz;
+
+	cm_cfg = p;
+	param_data = &cm_cfg->param_data;
+	param_data->module_instance_id = module->instance_id;
+	param_data->error_code = 0;
+	param_data->param_id = PARAM_ID_SP_VI_CHANNEL_MAP_CFG;
+	param_data->param_size = cm_sz - APM_MODULE_PARAM_DATA_SIZE;
+
+	cm_cfg->cfg.num_channels = num_channels * 2;
+	for (i = 0; i < num_channels; i++) {
+		/*
+		 * Map speakers into Vsense and then Isense of each channel.
+		 * E.g. for PCM_CHANNEL_FL and PCM_CHANNEL_FR to:
+		 * [1, 2, 3, 4]
+		 */
+		cm_cfg->cfg.channel_mapping[2 * i] = (mcfg->channel_map[i] - 1) * 2 + 1;
+		cm_cfg->cfg.channel_mapping[2 * i + 1] = (mcfg->channel_map[i] - 1) * 2 + 2;
+	}
+
+	p += cm_sz;
+
+	ex_cfg = p;
+	param_data = &ex_cfg->param_data;
+	param_data->module_instance_id = module->instance_id;
+	param_data->error_code = 0;
+	param_data->param_id = PARAM_ID_SP_VI_EX_MODE_CFG;
+	param_data->param_size = ex_sz - APM_MODULE_PARAM_DATA_SIZE;
+
+	ex_cfg->cfg.factory_mode = 0;
+
+	rc = q6apm_send_cmd_sync(graph->apm, pkt, 0);
+
+	kfree(pkt);
+
+	return rc;
+}
+
 int audioreach_set_media_format(struct q6apm_graph *graph, struct audioreach_module *module,
 				struct audioreach_module_config *cfg)
 {
@@ -1235,6 +1353,14 @@ int audioreach_set_media_format(struct q6apm_graph *graph, struct audioreach_mod
 	case MODULE_ID_GAPLESS:
 		rc = audioreach_gapless_set_media_format(graph, module, cfg);
 		break;
+	case MODULE_ID_SPEAKER_PROTECTION:
+		rc = audioreach_speaker_protection(graph, module,
+						   PARAM_ID_SP_OP_MODE_NORMAL);
+		break;
+	case MODULE_ID_SPEAKER_PROTECTION_VI:
+		rc = audioreach_speaker_protection_vi(graph, module, cfg);
+		break;
+
 	default:
 		rc = 0;
 	}
diff --git a/sound/soc/qcom/qdsp6/audioreach.h b/sound/soc/qcom/qdsp6/audioreach.h
index d1b60b36468a..03cfd32f1d0c 100644
--- a/sound/soc/qcom/qdsp6/audioreach.h
+++ b/sound/soc/qcom/qdsp6/audioreach.h
@@ -31,6 +31,8 @@ struct q6apm_graph;
 #define MODULE_ID_MP3_DECODE		0x0700103B
 #define MODULE_ID_GAPLESS		0x0700104D
 #define MODULE_ID_DISPLAY_PORT_SINK	0x07001069
+#define MODULE_ID_SPEAKER_PROTECTION	0x070010E2
+#define MODULE_ID_SPEAKER_PROTECTION_VI	0x070010E3
 #define MODULE_ID_OPUS_DEC		0x07001174
 
 #define APM_CMD_GET_SPF_STATE		0x01001021
@@ -559,6 +561,43 @@ struct data_logging_config {
 	uint32_t mode;
 } __packed;
 
+/* Speaker Protection */
+#define PARAM_ID_SP_OP_MODE			0x080011e9
+#define PARAM_ID_SP_OP_MODE_NORMAL		0
+#define PARAM_ID_SP_OP_MODE_CALIBRATION		1
+#define PARAM_ID_SP_OP_MODE_FACTORY_TEST	2
+#define PARAM_ID_SP_OP_MODE_VALIDATION		3
+
+struct param_id_sp_op_mode {
+	uint32_t operation_mode;
+} __packed;
+
+/* Speaker Protection VI */
+
+#define PARAM_ID_SP_VI_OP_MODE_CFG		0x080011f4
+#define PARAM_ID_SP_VI_OP_MODE_NORMAL		0
+#define PARAM_ID_SP_VI_OP_MODE_CALIBRATION	1
+#define PARAM_ID_SP_VI_OP_MODE_FACTORY_TEST	2
+#define PARAM_ID_SP_VI_OP_MODE_VALIDATION	3
+struct param_id_sp_vi_op_mode_cfg {
+	uint32_t num_channels;
+	uint32_t operation_mode;
+	uint32_t quick_calibration;
+	uint32_t r0_t0_selection[];
+} __packed;
+
+#define PARAM_ID_SP_VI_EX_MODE_CFG		0x080011ff
+struct param_id_sp_vi_ex_mode_cfg {
+	uint32_t factory_mode;
+} __packed;
+
+#define PARAM_ID_SP_VI_CHANNEL_MAP_CFG		0x08001203
+struct param_id_sp_vi_channel_map_cfg {
+	uint32_t num_channels;
+	/* [ Vsense of ch 1, Isense of ch 1, Vsense of ch 2, Isense of ch 2, ... ] */
+	uint32_t channel_mapping[];
+} __packed;
+
 #define PARAM_ID_SAL_OUTPUT_CFG			0x08001016
 struct param_id_sal_output_config {
 	uint32_t bits_per_sample;
diff --git a/sound/soc/qcom/qdsp6/q6adm.c b/sound/soc/qcom/qdsp6/q6adm.c
index 0b8d06ec8b26..bbe986293ec3 100644
--- a/sound/soc/qcom/qdsp6/q6adm.c
+++ b/sound/soc/qcom/qdsp6/q6adm.c
@@ -331,6 +331,7 @@ static int q6adm_device_open(struct q6adm *adm, struct q6copp *copp,
 	int afe_port = q6afe_get_port_id(port_id);
 	struct apr_pkt *pkt;
 	int ret, pkt_size = APR_HDR_SIZE + sizeof(*open);
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
@@ -466,6 +467,7 @@ int q6adm_matrix_map(struct device *dev, int path,
 	struct q6copp *copp;
 	int pkt_size = (APR_HDR_SIZE + sizeof(*route) +  sizeof(*node) +
 		    (sizeof(uint32_t) * payload_map.num_copps));
+
 	void *matrix_map __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!matrix_map)
 		return -ENOMEM;
diff --git a/sound/soc/qcom/qdsp6/q6afe.c b/sound/soc/qcom/qdsp6/q6afe.c
index 0b01fc9e13a7..76a05a3abe64 100644
--- a/sound/soc/qcom/qdsp6/q6afe.c
+++ b/sound/soc/qcom/qdsp6/q6afe.c
@@ -931,13 +931,11 @@ static void q6afe_port_free(struct kref *ref)
 {
 	struct q6afe_port *port;
 	struct q6afe *afe;
-	unsigned long flags;
 
 	port = container_of(ref, struct q6afe_port, refcount);
 	afe = port->afe;
-	spin_lock_irqsave(&afe->port_list_lock, flags);
-	list_del(&port->node);
-	spin_unlock_irqrestore(&afe->port_list_lock, flags);
+	scoped_guard(spinlock_irqsave, &afe->port_list_lock)
+		list_del(&port->node);
 	kfree(port->scfg);
 	kfree(port);
 }
@@ -1077,6 +1075,7 @@ static int q6afe_set_param(struct q6afe *afe, struct q6afe_port *port,
 	struct apr_pkt *pkt;
 	int ret, pkt_size = APR_HDR_SIZE + sizeof(*param) + sizeof(*pdata) + psize;
 	void *pl;
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
@@ -1128,6 +1127,7 @@ static int q6afe_port_set_param_v2(struct q6afe_port *port, void *data,
 	u16 port_id = port->id;
 	int ret, pkt_size = APR_HDR_SIZE + sizeof(*param) + sizeof(*pdata) + psize;
 	void *pl;
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
@@ -1277,7 +1277,6 @@ int q6afe_port_stop(struct q6afe_port *port)
 	int port_id = port->id;
 	int ret = 0;
 	int index, pkt_size;
-	void *p __free(kfree) = NULL;
 
 	index = port->token;
 	if (index < 0 || index >= AFE_PORT_MAX) {
@@ -1286,7 +1285,7 @@ int q6afe_port_stop(struct q6afe_port *port)
 	}
 
 	pkt_size = APR_HDR_SIZE + sizeof(*stop);
-	p = kzalloc(pkt_size, GFP_KERNEL);
+	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
 
@@ -1667,7 +1666,6 @@ int q6afe_port_start(struct q6afe_port *port)
 	int ret, param_id = port->cfg_type;
 	struct apr_pkt *pkt;
 	int pkt_size;
-	void *p __free(kfree) = NULL;
 
 	ret  = q6afe_port_set_param_v2(port, &port->port_cfg, param_id,
 				       AFE_MODULE_AUDIO_DEV_INTERFACE,
@@ -1690,7 +1688,7 @@ int q6afe_port_start(struct q6afe_port *port)
 	}
 
 	pkt_size = APR_HDR_SIZE + sizeof(*start);
-	p = kzalloc(pkt_size, GFP_KERNEL);
+	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
 
@@ -1807,8 +1805,8 @@ struct q6afe_port *q6afe_port_get_from_id(struct device *dev, int id)
 	port->cfg_type = cfg_type;
 	kref_init(&port->refcount);
 
-	guard(spinlock_irqsave)(&afe->port_list_lock);
-	list_add_tail(&port->node, &afe->port_list);
+	scoped_guard(spinlock_irqsave, &afe->port_list_lock)
+		list_add_tail(&port->node, &afe->port_list);
 
 	return port;
 
@@ -1834,6 +1832,7 @@ int q6afe_unvote_lpass_core_hw(struct device *dev, uint32_t hw_block_id,
 	struct apr_pkt *pkt;
 	int ret = 0;
 	int pkt_size = APR_HDR_SIZE + sizeof(*vote_cfg);
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
@@ -1868,6 +1867,7 @@ int q6afe_vote_lpass_core_hw(struct device *dev, uint32_t hw_block_id,
 	struct apr_pkt *pkt;
 	int ret = 0;
 	int pkt_size = APR_HDR_SIZE + sizeof(*vote_cfg);
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
diff --git a/sound/soc/qcom/qdsp6/q6apm.c b/sound/soc/qcom/qdsp6/q6apm.c
index 94cc6376a367..e30f8648ae15 100644
--- a/sound/soc/qcom/qdsp6/q6apm.c
+++ b/sound/soc/qcom/qdsp6/q6apm.c
@@ -100,6 +100,7 @@ static int audioreach_graph_mgmt_cmd(struct audioreach_graph *graph, uint32_t op
 	struct audioreach_sub_graph *sg;
 	struct q6apm *apm = graph->apm;
 	int i = 0, payload_size = APM_GRAPH_MGMT_PSIZE(mgmt_cmd, num_sub_graphs);
+
 	struct gpr_pkt *pkt __free(kfree) = audioreach_alloc_apm_cmd_pkt(payload_size, opcode, 0);
 	if (IS_ERR(pkt))
 		return PTR_ERR(pkt);
@@ -259,7 +260,6 @@ int q6apm_unmap_memory_regions(struct q6apm_graph *graph, unsigned int dir)
 {
 	struct apm_cmd_shared_mem_unmap_regions *cmd;
 	struct audioreach_graph_data *data;
-	struct gpr_pkt *pkt __free(kfree) = NULL;
 	int rc;
 
 	if (dir == SNDRV_PCM_STREAM_PLAYBACK)
@@ -270,8 +270,9 @@ int q6apm_unmap_memory_regions(struct q6apm_graph *graph, unsigned int dir)
 	if (!data->mem_map_handle)
 		return 0;
 
-	pkt = audioreach_alloc_apm_pkt(sizeof(*cmd), APM_CMD_SHARED_MEM_UNMAP_REGIONS, dir,
-				     graph->port->id);
+	struct gpr_pkt *pkt __free(kfree) =
+		audioreach_alloc_apm_pkt(sizeof(*cmd), APM_CMD_SHARED_MEM_UNMAP_REGIONS,
+					 dir, graph->port->id);
 	if (IS_ERR(pkt))
 		return PTR_ERR(pkt);
 
@@ -409,6 +410,7 @@ int q6apm_write_async(struct q6apm_graph *graph, uint32_t len, uint32_t msw_ts,
 	struct apm_data_cmd_wr_sh_mem_ep_data_buffer_v2 *write_buffer;
 	struct audio_buffer *ab;
 	int iid = q6apm_graph_get_rx_shmem_module_iid(graph);
+
 	struct gpr_pkt *pkt __free(kfree) = audioreach_alloc_pkt(sizeof(*write_buffer),
 					DATA_CMD_WR_SH_MEM_EP_DATA_BUFFER_V2,
 					graph->rx_data.dsp_buf | (len << APM_WRITE_TOKEN_LEN_SHIFT),
@@ -446,6 +448,7 @@ int q6apm_read(struct q6apm_graph *graph)
 	struct audioreach_graph_data *port;
 	struct audio_buffer *ab;
 	int iid = q6apm_graph_get_tx_shmem_module_iid(graph);
+
 	struct gpr_pkt *pkt __free(kfree) = audioreach_alloc_pkt(sizeof(*read_buffer),
 					DATA_CMD_RD_SH_MEM_EP_DATA_BUFFER_V2,
 					graph->tx_data.dsp_buf, graph->port->id, iid);
diff --git a/sound/soc/qcom/qdsp6/q6asm.c b/sound/soc/qcom/qdsp6/q6asm.c
index e7295b7b2461..420176f80ffe 100644
--- a/sound/soc/qcom/qdsp6/q6asm.c
+++ b/sound/soc/qcom/qdsp6/q6asm.c
@@ -335,7 +335,6 @@ static int __q6asm_memory_unmap(struct audio_client *ac,
 	struct q6asm *a = dev_get_drvdata(ac->dev->parent);
 	struct apr_pkt *pkt;
 	int rc, pkt_size;
-	void *p __free(kfree) = NULL;
 
 	if (ac->port[dir].mem_map_handle == 0) {
 		dev_err(ac->dev, "invalid mem handle\n");
@@ -343,7 +342,7 @@ static int __q6asm_memory_unmap(struct audio_client *ac,
 	}
 
 	pkt_size = APR_HDR_SIZE + sizeof(*mem_unmap);
-	p = kzalloc(pkt_size, GFP_KERNEL);
+	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
 
@@ -428,7 +427,6 @@ static int __q6asm_memory_map_regions(struct audio_client *ac, int dir,
 	struct audio_port_data *port = NULL;
 	struct audio_buffer *ab = NULL;
 	struct apr_pkt *pkt;
-	void *p __free(kfree) = NULL;
 	unsigned long flags;
 	uint32_t num_regions, buf_sz;
 	int i, pkt_size;
@@ -447,7 +445,7 @@ static int __q6asm_memory_map_regions(struct audio_client *ac, int dir,
 	pkt_size = APR_HDR_SIZE + sizeof(*cmd) +
 		   (sizeof(*mregions) * num_regions);
 
-	p = kzalloc(pkt_size, GFP_KERNEL);
+	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
 
@@ -930,6 +928,7 @@ int q6asm_open_write(struct audio_client *ac, uint32_t stream_id,
 	struct asm_stream_cmd_open_write_v3 *open;
 	struct apr_pkt *pkt;
 	int rc, pkt_size = APR_HDR_SIZE + sizeof(*open);
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
@@ -1007,6 +1006,7 @@ static int __q6asm_run(struct audio_client *ac, uint32_t stream_id,
 	struct asm_session_cmd_run_v2 *run;
 	struct apr_pkt *pkt;
 	int rc, pkt_size = APR_HDR_SIZE + sizeof(*run);
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_ATOMIC);
 	if (!p)
 		return -ENOMEM;
@@ -1089,6 +1089,7 @@ int q6asm_media_format_block_multi_ch_pcm(struct audio_client *ac,
 	struct apr_pkt *pkt;
 	u8 *channel_mapping;
 	int pkt_size = APR_HDR_SIZE + sizeof(*fmt);
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
@@ -1127,6 +1128,7 @@ int q6asm_stream_media_format_block_flac(struct audio_client *ac,
 	struct asm_flac_fmt_blk_v2 *fmt;
 	struct apr_pkt *pkt;
 	int pkt_size = APR_HDR_SIZE + sizeof(*fmt);
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
@@ -1158,6 +1160,7 @@ int q6asm_stream_media_format_block_wma_v9(struct audio_client *ac,
 	struct asm_wmastdv9_fmt_blk_v2 *fmt;
 	struct apr_pkt *pkt;
 	int pkt_size = APR_HDR_SIZE + sizeof(*fmt);
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
@@ -1190,6 +1193,7 @@ int q6asm_stream_media_format_block_wma_v10(struct audio_client *ac,
 	struct asm_wmaprov10_fmt_blk_v2 *fmt;
 	struct apr_pkt *pkt;
 	int pkt_size = APR_HDR_SIZE + sizeof(*fmt);
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
@@ -1223,6 +1227,7 @@ int q6asm_stream_media_format_block_alac(struct audio_client *ac,
 	struct asm_alac_fmt_blk_v2 *fmt;
 	struct apr_pkt *pkt;
 	int pkt_size = APR_HDR_SIZE + sizeof(*fmt);
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
@@ -1259,6 +1264,7 @@ int q6asm_stream_media_format_block_ape(struct audio_client *ac,
 	struct asm_ape_fmt_blk_v2 *fmt;
 	struct apr_pkt *pkt;
 	int pkt_size = APR_HDR_SIZE + sizeof(*fmt);
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
@@ -1293,6 +1299,7 @@ static int q6asm_stream_remove_silence(struct audio_client *ac, uint32_t stream_
 	uint32_t *samples;
 	struct apr_pkt *pkt;
 	int rc, pkt_size = APR_HDR_SIZE + sizeof(uint32_t);
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_ATOMIC);
 	if (!p)
 		return -ENOMEM;
@@ -1351,6 +1358,7 @@ int q6asm_enc_cfg_blk_pcm_format_support(struct audio_client *ac,
 	u8 *channel_mapping;
 	u32 frames_per_buf = 0;
 	int pkt_size = APR_HDR_SIZE + sizeof(*enc_cfg);
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
@@ -1397,6 +1405,7 @@ int q6asm_read(struct audio_client *ac, uint32_t stream_id)
 	unsigned long flags;
 	int pkt_size = APR_HDR_SIZE + sizeof(*read);
 	int rc = 0;
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_ATOMIC);
 	if (!p)
 		return -ENOMEM;
@@ -1439,6 +1448,7 @@ static int __q6asm_open_read(struct audio_client *ac, uint32_t stream_id,
 	struct asm_stream_cmd_open_read_v3 *open;
 	struct apr_pkt *pkt;
 	int pkt_size = APR_HDR_SIZE + sizeof(*open);
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
@@ -1509,6 +1519,7 @@ int q6asm_write_async(struct audio_client *ac, uint32_t stream_id, uint32_t len,
 	struct apr_pkt *pkt;
 	int pkt_size = APR_HDR_SIZE + sizeof(*write);
 	int rc = 0;
+
 	void *p __free(kfree) = kzalloc(pkt_size, GFP_ATOMIC);
 	if (!p)
 		return -ENOMEM;
diff --git a/sound/soc/qcom/qdsp6/q6prm.c b/sound/soc/qcom/qdsp6/q6prm.c
index 0b8fad0bc832..2544c4519b26 100644
--- a/sound/soc/qcom/qdsp6/q6prm.c
+++ b/sound/soc/qcom/qdsp6/q6prm.c
@@ -62,7 +62,6 @@ static int q6prm_set_hw_core_req(struct device *dev, uint32_t hw_block_id, bool
 	struct prm_cmd_request_hw_core *req;
 	gpr_device_t *gdev = prm->gdev;
 	uint32_t opcode, rsp_opcode;
-	struct gpr_pkt *pkt __free(kfree) = NULL;
 
 	if (enable) {
 		opcode = PRM_CMD_REQUEST_HW_RSC;
@@ -72,7 +71,8 @@ static int q6prm_set_hw_core_req(struct device *dev, uint32_t hw_block_id, bool
 		rsp_opcode = PRM_CMD_RSP_RELEASE_HW_RSC;
 	}
 
-	pkt = audioreach_alloc_cmd_pkt(sizeof(*req), opcode, 0, gdev->svc.id, GPR_PRM_MODULE_IID);
+	struct gpr_pkt *pkt __free(kfree) =
+		audioreach_alloc_cmd_pkt(sizeof(*req), opcode, 0, gdev->svc.id, GPR_PRM_MODULE_IID);
 	if (IS_ERR(pkt))
 		return PTR_ERR(pkt);
 
@@ -111,10 +111,10 @@ static int q6prm_request_lpass_clock(struct device *dev, int clk_id, int clk_att
 	struct apm_module_param_data *param_data;
 	struct prm_cmd_request_rsc *req;
 	gpr_device_t *gdev = prm->gdev;
-	struct gpr_pkt *pkt __free(kfree) = NULL;
 
-	pkt = audioreach_alloc_cmd_pkt(sizeof(*req), PRM_CMD_REQUEST_HW_RSC, 0, gdev->svc.id,
-				       GPR_PRM_MODULE_IID);
+	struct gpr_pkt *pkt __free(kfree) =
+		audioreach_alloc_cmd_pkt(sizeof(*req), PRM_CMD_REQUEST_HW_RSC, 0,
+					 gdev->svc.id, GPR_PRM_MODULE_IID);
 	if (IS_ERR(pkt))
 		return PTR_ERR(pkt);
 
@@ -143,10 +143,10 @@ static int q6prm_release_lpass_clock(struct device *dev, int clk_id, int clk_att
 	struct apm_module_param_data *param_data;
 	struct prm_cmd_release_rsc *rel;
 	gpr_device_t *gdev = prm->gdev;
-	struct gpr_pkt *pkt __free(kfree) = NULL;
 
-	pkt = audioreach_alloc_cmd_pkt(sizeof(*rel), PRM_CMD_RELEASE_HW_RSC, 0, gdev->svc.id,
-				       GPR_PRM_MODULE_IID);
+	struct gpr_pkt *pkt __free(kfree) =
+		audioreach_alloc_cmd_pkt(sizeof(*rel), PRM_CMD_RELEASE_HW_RSC, 0,
+					 gdev->svc.id, GPR_PRM_MODULE_IID);
 	if (IS_ERR(pkt))
 		return PTR_ERR(pkt);
 
diff --git a/sound/soc/qcom/sdm845.c b/sound/soc/qcom/sdm845.c
index e18a8e44f2db..0ce9dff4dc52 100644
--- a/sound/soc/qcom/sdm845.c
+++ b/sound/soc/qcom/sdm845.c
@@ -365,10 +365,12 @@ static int sdm845_snd_startup(struct snd_pcm_substream *substream)
 		snd_soc_dai_set_fmt(codec_dai, codec_dai_fmt);
 		break;
 	case QUATERNARY_MI2S_RX:
+		codec_dai_fmt |= SND_SOC_DAIFMT_NB_NF | SND_SOC_DAIFMT_I2S;
 		snd_soc_dai_set_sysclk(cpu_dai,
 			Q6AFE_LPASS_CLK_ID_QUAD_MI2S_IBIT,
 			MI2S_BCLK_RATE, SNDRV_PCM_STREAM_PLAYBACK);
 		snd_soc_dai_set_fmt(cpu_dai, fmt);
+		snd_soc_dai_set_fmt(codec_dai, codec_dai_fmt);
 		break;
 
 	case QUATERNARY_TDM_RX_0:
diff --git a/sound/soc/renesas/rz-ssi.c b/sound/soc/renesas/rz-ssi.c
index f4dc2f68dead..5909778a6a70 100644
--- a/sound/soc/renesas/rz-ssi.c
+++ b/sound/soc/renesas/rz-ssi.c
@@ -38,6 +38,7 @@
 #define SSICR_MST		BIT(14)
 #define SSICR_BCKP		BIT(13)
 #define SSICR_LRCKP		BIT(12)
+#define SSICR_PDTA		BIT(9)
 #define SSICR_CKDV(x)		(((x) & 0xf) << 4)
 #define SSICR_TEN		BIT(1)
 #define SSICR_REN		BIT(0)
@@ -74,7 +75,8 @@
 #define PREALLOC_BUFFER_MAX	(SZ_32K)
 
 #define SSI_RATES		SNDRV_PCM_RATE_8000_48000 /* 8k-48kHz */
-#define SSI_FMTS		SNDRV_PCM_FMTBIT_S16_LE
+#define SSI_FMTS		(SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S24_LE | \
+				 SNDRV_PCM_FMTBIT_S32_LE)
 #define SSI_CHAN_MIN		2
 #define SSI_CHAN_MAX		2
 #define SSI_FIFO_DEPTH		32
@@ -294,11 +296,27 @@ static int rz_ssi_clk_setup(struct rz_ssi_priv *ssi, unsigned int rate,
 	}
 
 	/*
-	 * DWL: Data Word Length = 16 bits
+	 * DWL: Data Word Length = {16, 24, 32} bits
 	 * SWL: System Word Length = 32 bits
 	 */
 	ssicr |= SSICR_CKDV(clk_ckdv);
-	ssicr |= SSICR_DWL(1) | SSICR_SWL(3);
+	switch (ssi->hw_params_cache.sample_width) {
+	case 16:
+		ssicr |= SSICR_DWL(1);
+		break;
+	case 24:
+		ssicr |= SSICR_DWL(5) | SSICR_PDTA;
+		break;
+	case 32:
+		ssicr |= SSICR_DWL(6);
+		break;
+	default:
+		dev_err(ssi->dev, "Not support %u data width",
+			ssi->hw_params_cache.sample_width);
+		return -EINVAL;
+	}
+
+	ssicr |= SSICR_SWL(3);
 	rz_ssi_reg_writel(ssi, SSICR, ssicr);
 	rz_ssi_reg_writel(ssi, SSIFCR, SSIFCR_AUCKE | SSIFCR_FIFO_RST);
 
@@ -455,7 +473,6 @@ static int rz_ssi_pio_recv(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm)
 {
 	struct snd_pcm_substream *substream = strm->substream;
 	struct snd_pcm_runtime *runtime;
-	u16 *buf;
 	int fifo_samples;
 	int frames_left;
 	int samples;
@@ -490,12 +507,23 @@ static int rz_ssi_pio_recv(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm)
 			break;
 
 		/* calculate new buffer index */
-		buf = (u16 *)runtime->dma_area;
-		buf += strm->buffer_pos * runtime->channels;
+		if (ssi->hw_params_cache.sample_width == 16) {
+			u16 *buf;
 
-		/* Note, only supports 16-bit samples */
-		for (i = 0; i < samples; i++)
-			*buf++ = (u16)(rz_ssi_reg_readl(ssi, SSIFRDR) >> 16);
+			buf = (u16 *)runtime->dma_area;
+			buf += strm->buffer_pos * runtime->channels;
+
+			for (i = 0; i < samples; i++)
+				*buf++ = (u16)(rz_ssi_reg_readl(ssi, SSIFRDR) >> 16);
+		} else {
+			u32 *buf;
+
+			buf = (u32 *)runtime->dma_area;
+			buf += strm->buffer_pos * runtime->channels;
+
+			for (i = 0; i < samples; i++)
+				*buf++ = rz_ssi_reg_readl(ssi, SSIFRDR);
+		}
 
 		rz_ssi_reg_mask_setl(ssi, SSIFSR, SSIFSR_RDF, 0);
 		rz_ssi_pointer_update(strm, samples / runtime->channels);
@@ -513,7 +541,6 @@ static int rz_ssi_pio_send(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm)
 	int frames_left;
 	int i;
 	u32 ssifsr;
-	u16 *buf;
 
 	if (!rz_ssi_stream_is_valid(ssi, strm))
 		return -EINVAL;
@@ -542,12 +569,23 @@ static int rz_ssi_pio_send(struct rz_ssi_priv *ssi, struct rz_ssi_stream *strm)
 		return 0;
 
 	/* calculate new buffer index */
-	buf = (u16 *)(runtime->dma_area);
-	buf += strm->buffer_pos * runtime->channels;
+	if (ssi->hw_params_cache.sample_width == 16) {
+		u16 *buf;
+
+		buf = (u16 *)(runtime->dma_area);
+		buf += strm->buffer_pos * runtime->channels;
 
-	/* Note, only supports 16-bit samples */
-	for (i = 0; i < samples; i++)
-		rz_ssi_reg_writel(ssi, SSIFTDR, ((u32)(*buf++) << 16));
+		for (i = 0; i < samples; i++)
+			rz_ssi_reg_writel(ssi, SSIFTDR, ((u32)(*buf++) << 16));
+	} else {
+		u32 *buf;
+
+		buf = (u32 *)(runtime->dma_area);
+		buf += strm->buffer_pos * runtime->channels;
+
+		for (i = 0; i < samples; i++)
+			rz_ssi_reg_writel(ssi, SSIFTDR, *buf++);
+	}
 
 	rz_ssi_reg_mask_setl(ssi, SSIFSR, SSIFSR_TDE, 0);
 	rz_ssi_pointer_update(strm, samples / runtime->channels);
@@ -658,8 +696,13 @@ static int rz_ssi_dma_slave_config(struct rz_ssi_priv *ssi,
 	cfg.direction = is_play ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;
 	cfg.dst_addr = ssi->phys + SSIFTDR;
 	cfg.src_addr = ssi->phys + SSIFRDR;
-	cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
-	cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
+	if (ssi->hw_params_cache.sample_width == 16) {
+		cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
+		cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
+	} else {
+		cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+		cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+	}
 
 	return dmaengine_slave_config(dma_ch, &cfg);
 }
@@ -774,14 +817,6 @@ static int rz_ssi_dma_request(struct rz_ssi_priv *ssi, struct device *dev)
 	if (!rz_ssi_is_dma_enabled(ssi))
 		goto no_dma;
 
-	if (ssi->playback.dma_ch &&
-	    (rz_ssi_dma_slave_config(ssi, ssi->playback.dma_ch, true) < 0))
-		goto no_dma;
-
-	if (ssi->capture.dma_ch &&
-	    (rz_ssi_dma_slave_config(ssi, ssi->capture.dma_ch, false) < 0))
-		goto no_dma;
-
 	return 0;
 
 no_dma:
@@ -829,24 +864,27 @@ static int rz_ssi_dai_trigger(struct snd_pcm_substream *substream, int cmd,
 		if (cmd == SNDRV_PCM_TRIGGER_START)
 			rz_ssi_stream_init(strm, substream);
 
-		if (ssi->dma_rt) {
-			bool is_playback;
+		if (rz_ssi_is_dma_enabled(ssi)) {
+			bool is_playback = rz_ssi_stream_is_play(substream);
+
+			if (ssi->dma_rt)
+				ret = rz_ssi_dma_slave_config(ssi, ssi->playback.dma_ch,
+							      is_playback);
+			else
+				ret = rz_ssi_dma_slave_config(ssi, strm->dma_ch,
+							      is_playback);
 
-			is_playback = rz_ssi_stream_is_play(substream);
-			ret = rz_ssi_dma_slave_config(ssi, ssi->playback.dma_ch,
-						      is_playback);
 			/* Fallback to pio */
 			if (ret < 0) {
 				ssi->playback.transfer = rz_ssi_pio_send;
 				ssi->capture.transfer = rz_ssi_pio_recv;
 				rz_ssi_release_dma_channels(ssi);
+			} else {
+				/* For DMA, queue up multiple DMA descriptors */
+				num_transfer = 4;
 			}
 		}
 
-		/* For DMA, queue up multiple DMA descriptors */
-		if (rz_ssi_is_dma_enabled(ssi))
-			num_transfer = 4;
-
 		for (i = 0; i < num_transfer; i++) {
 			ret = strm->transfer(ssi, strm);
 			if (ret)
@@ -982,7 +1020,7 @@ static int rz_ssi_dai_hw_params(struct snd_pcm_substream *substream,
 	unsigned int rate = params_rate(params);
 	int ret;
 
-	if (sample_bits != 16) {
+	if (!(sample_bits == 16 || sample_bits == 24 || sample_bits == 32)) {
 		dev_err(ssi->dev, "Unsupported sample width: %d\n",
 			sample_bits);
 		return -EINVAL;
@@ -1119,19 +1157,16 @@ static int rz_ssi_probe(struct platform_device *pdev)
 
 	audio_clk = devm_clk_get(dev, "audio_clk1");
 	if (IS_ERR(audio_clk))
-		return dev_err_probe(&pdev->dev, PTR_ERR(audio_clk),
-				     "no audio clk1");
+		return dev_err_probe(dev, PTR_ERR(audio_clk), "no audio clk1");
 
 	ssi->audio_clk_1 = clk_get_rate(audio_clk);
 	audio_clk = devm_clk_get(dev, "audio_clk2");
 	if (IS_ERR(audio_clk))
-		return dev_err_probe(&pdev->dev, PTR_ERR(audio_clk),
-				     "no audio clk2");
+		return dev_err_probe(dev, PTR_ERR(audio_clk), "no audio clk2");
 
 	ssi->audio_clk_2 = clk_get_rate(audio_clk);
 	if (!(ssi->audio_clk_1 || ssi->audio_clk_2))
-		return dev_err_probe(&pdev->dev, -EINVAL,
-				     "no audio clk1 or audio clk2");
+		return dev_err_probe(dev, -EINVAL, "no audio clk1 or audio clk2");
 
 	ssi->audio_mck = ssi->audio_clk_1 ? ssi->audio_clk_1 : ssi->audio_clk_2;
 
@@ -1247,7 +1282,7 @@ static void rz_ssi_remove(struct platform_device *pdev)
 
 static const struct of_device_id rz_ssi_of_match[] = {
 	{ .compatible = "renesas,rz-ssi", },
-	{/* Sentinel */},
+	{ /* Sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, rz_ssi_of_match);
 
diff --git a/sound/soc/sdca/sdca_asoc.c b/sound/soc/sdca/sdca_asoc.c
index 2d328bbb95b9..498aba9df5d9 100644
--- a/sound/soc/sdca/sdca_asoc.c
+++ b/sound/soc/sdca/sdca_asoc.c
@@ -1478,7 +1478,7 @@ static int set_usage(struct device *dev, struct regmap *regmap,
 		unsigned int rate = sdca_range(range, SDCA_USAGE_SAMPLE_RATE, i);
 		unsigned int width = sdca_range(range, SDCA_USAGE_SAMPLE_WIDTH, i);
 
-		if ((!rate || rate == target_rate) && width == target_width) {
+		if ((!rate || rate == target_rate) && (!width || width == target_width)) {
 			unsigned int usage = sdca_range(range, SDCA_USAGE_NUMBER, i);
 			unsigned int reg = SDW_SDCA_CTL(function->desc->adr,
 							entity->id, sel, 0);
diff --git a/sound/soc/sdca/sdca_functions.c b/sound/soc/sdca/sdca_functions.c
index 5a1f120487ef..acac066f1d8d 100644
--- a/sound/soc/sdca/sdca_functions.c
+++ b/sound/soc/sdca/sdca_functions.c
@@ -1184,7 +1184,6 @@ static int find_sdca_entity_pde(struct device *dev,
 {
 	static const int mult_delay = 3;
 	struct sdca_entity_pde *power = &entity->pde;
-	u32 *delay_list __free(kfree) = NULL;
 	struct sdca_pde_delay *delays;
 	int num_delays;
 	int i, j;
@@ -1205,7 +1204,8 @@ static int find_sdca_entity_pde(struct device *dev,
 		return -EINVAL;
 	}
 
-	delay_list = kcalloc(num_delays, sizeof(*delay_list), GFP_KERNEL);
+	u32 *delay_list __free(kfree) = kcalloc(num_delays, sizeof(*delay_list),
+						GFP_KERNEL);
 	if (!delay_list)
 		return -ENOMEM;
 
@@ -1250,7 +1250,6 @@ static int find_sdca_entity_ge(struct device *dev,
 			       struct sdca_entity *entity)
 {
 	struct sdca_entity_ge *group = &entity->ge;
-	u8 *affected_list __free(kfree) = NULL;
 	u8 *affected_iter;
 	int num_affected;
 	int i, j;
@@ -1269,7 +1268,8 @@ static int find_sdca_entity_ge(struct device *dev,
 		return -EINVAL;
 	}
 
-	affected_list = kcalloc(num_affected, sizeof(*affected_list), GFP_KERNEL);
+	u8 *affected_list __free(kfree) = kcalloc(num_affected, sizeof(*affected_list),
+						  GFP_KERNEL);
 	if (!affected_list)
 		return -ENOMEM;
 
@@ -1495,7 +1495,6 @@ static int find_sdca_entities(struct device *dev, struct sdw_slave *sdw,
 			      struct fwnode_handle *function_node,
 			      struct sdca_function_data *function)
 {
-	u32 *entity_list __free(kfree) = NULL;
 	struct sdca_entity *entities;
 	int num_entities;
 	int i, ret;
@@ -1517,7 +1516,8 @@ static int find_sdca_entities(struct device *dev, struct sdw_slave *sdw,
 	if (!entities)
 		return -ENOMEM;
 
-	entity_list = kcalloc(num_entities, sizeof(*entity_list), GFP_KERNEL);
+	u32 *entity_list __free(kfree) = kcalloc(num_entities, sizeof(*entity_list),
+						 GFP_KERNEL);
 	if (!entity_list)
 		return -ENOMEM;
 
@@ -1642,7 +1642,6 @@ static int find_sdca_entity_connection_pde(struct device *dev,
 					   struct sdca_entity *entity)
 {
 	struct sdca_entity_pde *power = &entity->pde;
-	u32 *managed_list __free(kfree) = NULL;
 	struct sdca_entity **managed;
 	int num_managed;
 	int i;
@@ -1664,7 +1663,8 @@ static int find_sdca_entity_connection_pde(struct device *dev,
 	if (!managed)
 		return -ENOMEM;
 
-	managed_list = kcalloc(num_managed, sizeof(*managed_list), GFP_KERNEL);
+	u32 *managed_list __free(kfree) = kcalloc(num_managed, sizeof(*managed_list),
+						  GFP_KERNEL);
 	if (!managed_list)
 		return -ENOMEM;
 
@@ -1961,7 +1961,6 @@ static int find_sdca_clusters(struct device *dev,
 			      struct fwnode_handle *function_node,
 			      struct sdca_function_data *function)
 {
-	u32 *cluster_list __free(kfree) = NULL;
 	struct sdca_cluster *clusters;
 	int num_clusters;
 	int i, ret;
@@ -1982,7 +1981,8 @@ static int find_sdca_clusters(struct device *dev,
 	if (!clusters)
 		return -ENOMEM;
 
-	cluster_list = kcalloc(num_clusters, sizeof(*cluster_list), GFP_KERNEL);
+	u32 *cluster_list __free(kfree) = kcalloc(num_clusters, sizeof(*cluster_list),
+						  GFP_KERNEL);
 	if (!cluster_list)
 		return -ENOMEM;
 
@@ -2026,7 +2026,6 @@ static int find_sdca_filesets(struct device *dev, struct sdw_slave *sdw,
 {
 	static const int mult_fileset = 3;
 	char fileset_name[SDCA_PROPERTY_LENGTH];
-	u32 *filesets_list __free(kfree) = NULL;
 	struct sdca_fdl_set *sets;
 	int num_sets;
 	int i, j;
@@ -2041,7 +2040,8 @@ static int find_sdca_filesets(struct device *dev, struct sdw_slave *sdw,
 		return num_sets;
 	}
 
-	filesets_list = kcalloc(num_sets, sizeof(u32), GFP_KERNEL);
+	u32 *filesets_list __free(kfree) = kcalloc(num_sets, sizeof(u32),
+						   GFP_KERNEL);
 	if (!filesets_list)
 		return -ENOMEM;
 
@@ -2053,7 +2053,6 @@ static int find_sdca_filesets(struct device *dev, struct sdw_slave *sdw,
 		return -ENOMEM;
 
 	for (i = 0; i < num_sets; i++) {
-		u32 *fileset_entries __free(kfree) = NULL;
 		struct sdca_fdl_set *set = &sets[i];
 		struct sdca_fdl_file *files;
 		int num_files, num_entries;
@@ -2079,7 +2078,8 @@ static int find_sdca_filesets(struct device *dev, struct sdw_slave *sdw,
 		if (!files)
 			return -ENOMEM;
 
-		fileset_entries = kcalloc(num_entries, sizeof(u32), GFP_KERNEL);
+		u32 *fileset_entries __free(kfree) = kcalloc(num_entries, sizeof(u32),
+							     GFP_KERNEL);
 		if (!fileset_entries)
 			return -ENOMEM;
 
diff --git a/sound/soc/sdw_utils/soc_sdw_ti_amp.c b/sound/soc/sdw_utils/soc_sdw_ti_amp.c
index cbd60faecd09..488ef2ef45d4 100644
--- a/sound/soc/sdw_utils/soc_sdw_ti_amp.c
+++ b/sound/soc/sdw_utils/soc_sdw_ti_amp.c
@@ -58,6 +58,10 @@ int asoc_sdw_ti_spk_rtd_init(struct snd_soc_pcm_runtime *rtd,
 			strscpy(speaker, "Left Spk", sizeof(speaker));
 		} else if (!strncmp(prefix, "tas2783-2", strlen("tas2783-2"))) {
 			strscpy(speaker, "Right Spk", sizeof(speaker));
+		} else if (!strncmp(prefix, "tas2783-3", strlen("tas2783-3"))) {
+			strscpy(speaker, "Left Spk2", sizeof(speaker));
+		} else if (!strncmp(prefix, "tas2783-4", strlen("tas2783-4"))) {
+			strscpy(speaker, "Right Spk2", sizeof(speaker));
 		} else {
 			ret = -EINVAL;
 			dev_err(card->dev, "unhandled prefix %s", prefix);
diff --git a/sound/soc/sdw_utils/soc_sdw_utils.c b/sound/soc/sdw_utils/soc_sdw_utils.c
index 6c656b2f7f3a..1d1ba7a2511b 100644
--- a/sound/soc/sdw_utils/soc_sdw_utils.c
+++ b/sound/soc/sdw_utils/soc_sdw_utils.c
@@ -40,11 +40,25 @@ static const struct snd_soc_dapm_widget lr_spk_widgets[] = {
 	SND_SOC_DAPM_SPK("Right Spk", NULL),
 };
 
+static const struct snd_soc_dapm_widget lr_4spk_widgets[] = {
+	SND_SOC_DAPM_SPK("Left Spk", NULL),
+	SND_SOC_DAPM_SPK("Right Spk", NULL),
+	SND_SOC_DAPM_SPK("Left Spk2", NULL),
+	SND_SOC_DAPM_SPK("Right Spk2", NULL),
+};
+
 static const struct snd_kcontrol_new lr_spk_controls[] = {
 	SOC_DAPM_PIN_SWITCH("Left Spk"),
 	SOC_DAPM_PIN_SWITCH("Right Spk"),
 };
 
+static const struct snd_kcontrol_new lr_4spk_controls[] = {
+	SOC_DAPM_PIN_SWITCH("Left Spk"),
+	SOC_DAPM_PIN_SWITCH("Right Spk"),
+	SOC_DAPM_PIN_SWITCH("Left Spk2"),
+	SOC_DAPM_PIN_SWITCH("Right Spk2"),
+};
+
 static const struct snd_soc_dapm_widget rt700_widgets[] = {
 	SND_SOC_DAPM_HP("Headphones", NULL),
 	SND_SOC_DAPM_MIC("AMIC", NULL),
@@ -69,10 +83,10 @@ struct asoc_sdw_codec_info codec_info_list[] = {
 				.dailink = {SOC_SDW_AMP_OUT_DAI_ID, SOC_SDW_AMP_IN_DAI_ID},
 				.init = asoc_sdw_ti_amp_init,
 				.rtd_init = asoc_sdw_ti_spk_rtd_init,
-				.controls = lr_spk_controls,
-				.num_controls = ARRAY_SIZE(lr_spk_controls),
-				.widgets = lr_spk_widgets,
-				.num_widgets = ARRAY_SIZE(lr_spk_widgets),
+				.controls = lr_4spk_controls,
+				.num_controls = ARRAY_SIZE(lr_4spk_controls),
+				.widgets = lr_4spk_widgets,
+				.num_widgets = ARRAY_SIZE(lr_4spk_widgets),
 			},
 		},
 		.dai_num = 1,
@@ -1534,8 +1548,10 @@ int asoc_sdw_parse_sdw_endpoints(struct snd_soc_card *card,
 					 * endpoint check is not necessary
 					 */
 					if (dai_info->quirk &&
-					    !(dai_info->quirk_exclude ^ !!(dai_info->quirk & ctx->mc_quirk)))
+					    !(dai_info->quirk_exclude ^ !!(dai_info->quirk & ctx->mc_quirk))) {
+						(*num_devs)--;
 						continue;
+					}
 				} else {
 					/* Check SDCA codec endpoint if there is no matching quirk */
 					ret = is_sdca_endpoint_present(dev, codec_info, adr_link, i, j);
@@ -1543,8 +1559,10 @@ int asoc_sdw_parse_sdw_endpoints(struct snd_soc_card *card,
 						return ret;
 
 					/* The endpoint is not present, skip */
-					if (!ret)
+					if (!ret) {
+						(*num_devs)--;
 						continue;
+					}
 				}
 
 				dev_dbg(dev,
diff --git a/sound/soc/sof/compress.c b/sound/soc/sof/compress.c
index 90b932ae3bab..86d563c864e5 100644
--- a/sound/soc/sof/compress.c
+++ b/sound/soc/sof/compress.c
@@ -195,6 +195,14 @@ static int sof_compr_set_params(struct snd_soc_component *component,
 	if (sizeof(*pcm) + ext_data_size > sdev->ipc->max_payload_size)
 		return -EINVAL;
 
+	/*
+	 * Make sure that the DSP is booted up, which might not be the
+	 * case if the on-demand DSP boot is used
+	 */
+	ret = snd_sof_boot_dsp_firmware(sdev);
+	if (ret)
+		return ret;
+
 	pcm = kzalloc(sizeof(*pcm) + ext_data_size, GFP_KERNEL);
 	if (!pcm)
 		return -ENOMEM;
diff --git a/sound/soc/sof/control.c b/sound/soc/sof/control.c
index a3fd1d523c09..74d997a4f620 100644
--- a/sound/soc/sof/control.c
+++ b/sound/soc/sof/control.c
@@ -187,14 +187,23 @@ int snd_sof_bytes_ext_volatile_get(struct snd_kcontrol *kcontrol, unsigned int _
 	const struct sof_ipc_tplg_ops *tplg_ops = sof_ipc_get_ops(sdev, tplg);
 	int ret, err;
 
+	/* ignore the ext_volatile_get call if the callbacks are not provided */
+	if (!tplg_ops || !tplg_ops->control ||
+	    !tplg_ops->control->bytes_ext_volatile_get)
+		return 0;
+
 	ret = pm_runtime_resume_and_get(scomp->dev);
 	if (ret < 0 && ret != -EACCES) {
 		dev_err_ratelimited(scomp->dev, "%s: failed to resume %d\n", __func__, ret);
 		return ret;
 	}
 
-	if (tplg_ops && tplg_ops->control && tplg_ops->control->bytes_ext_volatile_get)
-		ret = tplg_ops->control->bytes_ext_volatile_get(scontrol, binary_data, size);
+	/* Make sure the DSP/firmware is booted up */
+	ret = snd_sof_boot_dsp_firmware(sdev);
+	if (!ret)
+		ret = tplg_ops->control->bytes_ext_volatile_get(scontrol,
+								binary_data,
+								size);
 
 	err = pm_runtime_put_autosuspend(scomp->dev);
 	if (err < 0)
diff --git a/sound/soc/sof/core.c b/sound/soc/sof/core.c
index b11f408f1366..2d394389c945 100644
--- a/sound/soc/sof/core.c
+++ b/sound/soc/sof/core.c
@@ -680,6 +680,7 @@ int snd_sof_device_probe(struct device *dev, struct snd_sof_pdata *plat_data)
 	mutex_init(&sdev->power_state_access);
 	mutex_init(&sdev->ipc_client_mutex);
 	mutex_init(&sdev->client_event_handler_mutex);
+	mutex_init(&sdev->dsp_fw_boot_mutex);
 
 	/* set default timeouts if none provided */
 	if (plat_data->desc->ipc_timeout == 0)
diff --git a/sound/soc/sof/debug.c b/sound/soc/sof/debug.c
index b24943a65c89..6b9e1f1ee657 100644
--- a/sound/soc/sof/debug.c
+++ b/sound/soc/sof/debug.c
@@ -216,7 +216,12 @@ static int memory_info_update(struct snd_sof_dev *sdev, char *buf, size_t buff_s
 		goto error;
 	}
 
-	ret = sof_ipc_tx_message(sdev->ipc, &msg, msg.size, reply, SOF_IPC_MSG_MAX_SIZE);
+	/* Make sure the DSP/firmware is booted up */
+	ret = snd_sof_boot_dsp_firmware(sdev);
+	if (!ret)
+		ret = sof_ipc_tx_message(sdev->ipc, &msg, msg.size, reply,
+					 SOF_IPC_MSG_MAX_SIZE);
+
 	pm_runtime_put_autosuspend(sdev->dev);
 	if (ret < 0 || reply->rhdr.error < 0) {
 		ret = min(ret, reply->rhdr.error);
diff --git a/sound/soc/sof/intel/hda-sdw-bpt.c b/sound/soc/sof/intel/hda-sdw-bpt.c
index e45dd051ab8c..ae2f8d55dbd0 100644
--- a/sound/soc/sof/intel/hda-sdw-bpt.c
+++ b/sound/soc/sof/intel/hda-sdw-bpt.c
@@ -98,6 +98,17 @@ static int hda_sdw_bpt_dma_prepare(struct device *dev, struct hdac_ext_stream **
 	struct hdac_ext_stream *bpt_stream;
 	unsigned int format = HDA_CL_STREAM_FORMAT;
 
+	if (!sdev->dspless_mode_selected) {
+		int ret;
+
+		/*
+		 * Make sure that the DSP is booted up, which might not be the
+		 * case if the on-demand DSP boot is used
+		 */
+		ret = snd_sof_boot_dsp_firmware(sdev);
+		if (ret)
+			return ret;
+	}
 	/*
 	 * the baseline format needs to be adjusted to
 	 * bandwidth requirements
diff --git a/sound/soc/sof/intel/hda.c b/sound/soc/sof/intel/hda.c
index c1518dbee1b7..72c98154c027 100644
--- a/sound/soc/sof/intel/hda.c
+++ b/sound/soc/sof/intel/hda.c
@@ -1611,7 +1611,6 @@ struct snd_soc_acpi_mach *hda_machine_select(struct snd_sof_dev *sdev)
 		    mach->mach_params.i2s_link_mask) {
 			const struct sof_intel_dsp_desc *chip = get_chip_info(sdev->pdata);
 			int ssp_num;
-			int mclk_mask;
 
 			if (hweight_long(mach->mach_params.i2s_link_mask) > 1 &&
 			    !(mach->tplg_quirk_mask & SND_SOC_ACPI_TPLG_INTEL_SSP_MSB))
@@ -1636,19 +1635,28 @@ struct snd_soc_acpi_mach *hda_machine_select(struct snd_sof_dev *sdev)
 
 			sof_pdata->tplg_filename = tplg_filename;
 
-			mclk_mask = check_nhlt_ssp_mclk_mask(sdev, ssp_num);
-
-			if (mclk_mask < 0) {
-				dev_err(sdev->dev, "Invalid MCLK configuration\n");
-				return NULL;
-			}
-
-			dev_dbg(sdev->dev, "MCLK mask %#x found in NHLT\n", mclk_mask);
-
-			if (mclk_mask) {
-				dev_info(sdev->dev, "Overriding topology with MCLK mask %#x from NHLT\n", mclk_mask);
-				sdev->mclk_id_override = true;
-				sdev->mclk_id_quirk = (mclk_mask & BIT(0)) ? 0 : 1;
+			if (sof_pdata->ipc_type == SOF_IPC_TYPE_3) {
+				int mclk_mask = check_nhlt_ssp_mclk_mask(sdev,
+									 ssp_num);
+
+				if (mclk_mask < 0) {
+					dev_err(sdev->dev,
+						"Invalid MCLK configuration for SSP%d\n",
+						ssp_num);
+					return NULL;
+				}
+
+				if (mclk_mask) {
+					sdev->mclk_id_override = true;
+					sdev->mclk_id_quirk = (mclk_mask & BIT(0)) ? 0 : 1;
+					dev_info(sdev->dev,
+						 "SSP%d to use MCLK id %d (mask: %#x)\n",
+						 ssp_num, sdev->mclk_id_quirk, mclk_mask);
+				} else {
+					dev_dbg(sdev->dev,
+						"MCLK mask is empty for SSP%d in NHLT\n",
+						ssp_num);
+				}
 			}
 		}
 
diff --git a/sound/soc/sof/intel/pci-apl.c b/sound/soc/sof/intel/pci-apl.c
index 0bf7ee753bc3..3241403efa60 100644
--- a/sound/soc/sof/intel/pci-apl.c
+++ b/sound/soc/sof/intel/pci-apl.c
@@ -86,7 +86,7 @@ static const struct sof_dev_desc glk_desc = {
 /* PCI IDs */
 static const struct pci_device_id sof_pci_ids[] = {
 	{ PCI_DEVICE_DATA(INTEL, HDA_APL, &bxt_desc) },
-	{ PCI_DEVICE_DATA(INTEL, HDA_GML, &glk_desc) },
+	{ PCI_DEVICE_DATA(INTEL, HDA_GLK, &glk_desc) },
 	{ 0, }
 };
 MODULE_DEVICE_TABLE(pci, sof_pci_ids);
diff --git a/sound/soc/sof/intel/pci-lnl.c b/sound/soc/sof/intel/pci-lnl.c
index ae379c23f008..acb4429df9ec 100644
--- a/sound/soc/sof/intel/pci-lnl.c
+++ b/sound/soc/sof/intel/pci-lnl.c
@@ -40,6 +40,7 @@ static const struct sof_dev_desc lnl_desc = {
 	.ipc_supported_mask	= BIT(SOF_IPC_TYPE_4),
 	.ipc_default		= SOF_IPC_TYPE_4,
 	.dspless_mode_supported	= true,
+	.on_demand_dsp_boot	= true,
 	.default_fw_path = {
 		[SOF_IPC_TYPE_4] = "intel/sof-ipc4/lnl",
 	},
diff --git a/sound/soc/sof/intel/pci-mtl.c b/sound/soc/sof/intel/pci-mtl.c
index 7b2533999195..23adc5d765b4 100644
--- a/sound/soc/sof/intel/pci-mtl.c
+++ b/sound/soc/sof/intel/pci-mtl.c
@@ -47,7 +47,7 @@ static const struct sof_dev_desc mtl_desc = {
 		[SOF_IPC_TYPE_4] = "intel/sof-ipc4-lib/mtl",
 	},
 	.default_tplg_path = {
-		[SOF_IPC_TYPE_4] = "intel/sof-ace-tplg",
+		[SOF_IPC_TYPE_4] = "intel/sof-ipc4-tplg",
 	},
 	.default_fw_filename = {
 		[SOF_IPC_TYPE_4] = "sof-mtl.ri",
@@ -77,7 +77,7 @@ static const struct sof_dev_desc arl_desc = {
 		[SOF_IPC_TYPE_4] = "intel/sof-ipc4-lib/arl",
 	},
 	.default_tplg_path = {
-		[SOF_IPC_TYPE_4] = "intel/sof-ace-tplg",
+		[SOF_IPC_TYPE_4] = "intel/sof-ipc4-tplg",
 	},
 	.default_fw_filename = {
 		[SOF_IPC_TYPE_4] = "sof-arl.ri",
@@ -107,7 +107,7 @@ static const struct sof_dev_desc arl_s_desc = {
 		[SOF_IPC_TYPE_4] = "intel/sof-ipc4-lib/arl-s",
 	},
 	.default_tplg_path = {
-		[SOF_IPC_TYPE_4] = "intel/sof-ace-tplg",
+		[SOF_IPC_TYPE_4] = "intel/sof-ipc4-tplg",
 	},
 	.default_fw_filename = {
 		[SOF_IPC_TYPE_4] = "sof-arl-s.ri",
diff --git a/sound/soc/sof/intel/pci-nvl.c b/sound/soc/sof/intel/pci-nvl.c
index c499c14b93d5..f75aa996a5bd 100644
--- a/sound/soc/sof/intel/pci-nvl.c
+++ b/sound/soc/sof/intel/pci-nvl.c
@@ -38,6 +38,7 @@ static const struct sof_dev_desc nvl_s_desc = {
 	.ipc_supported_mask	= BIT(SOF_IPC_TYPE_4),
 	.ipc_default		= SOF_IPC_TYPE_4,
 	.dspless_mode_supported	= true,
+	.on_demand_dsp_boot	= true,
 	.default_fw_path = {
 		[SOF_IPC_TYPE_4] = "intel/sof-ipc4/nvl-s",
 	},
diff --git a/sound/soc/sof/intel/pci-ptl.c b/sound/soc/sof/intel/pci-ptl.c
index 68f6a9841633..9cb785ef763f 100644
--- a/sound/soc/sof/intel/pci-ptl.c
+++ b/sound/soc/sof/intel/pci-ptl.c
@@ -38,6 +38,7 @@ static const struct sof_dev_desc ptl_desc = {
 	.ipc_supported_mask	= BIT(SOF_IPC_TYPE_4),
 	.ipc_default		= SOF_IPC_TYPE_4,
 	.dspless_mode_supported	= true,
+	.on_demand_dsp_boot	= true,
 	.default_fw_path = {
 		[SOF_IPC_TYPE_4] = "intel/sof-ipc4/ptl",
 	},
@@ -67,6 +68,7 @@ static const struct sof_dev_desc wcl_desc = {
 	.ipc_supported_mask	= BIT(SOF_IPC_TYPE_4),
 	.ipc_default		= SOF_IPC_TYPE_4,
 	.dspless_mode_supported	= true,
+	.on_demand_dsp_boot	= true,
 	.default_fw_path = {
 		[SOF_IPC_TYPE_4] = "intel/sof-ipc4/wcl",
 	},
diff --git a/sound/soc/sof/ipc3-dtrace.c b/sound/soc/sof/ipc3-dtrace.c
index 6ec391fd39a9..50700f5cb0ef 100644
--- a/sound/soc/sof/ipc3-dtrace.c
+++ b/sound/soc/sof/ipc3-dtrace.c
@@ -171,7 +171,12 @@ static int ipc3_trace_update_filter(struct snd_sof_dev *sdev, int num_elems,
 		dev_err(sdev->dev, "enabling device failed: %d\n", ret);
 		goto error;
 	}
-	ret = sof_ipc_tx_message_no_reply(sdev->ipc, msg, msg->hdr.size);
+
+	/* Make sure the DSP/firmware is booted up */
+	ret = snd_sof_boot_dsp_firmware(sdev);
+	if (!ret)
+		ret = sof_ipc_tx_message_no_reply(sdev->ipc, msg, msg->hdr.size);
+
 	pm_runtime_put_autosuspend(sdev->dev);
 
 error:
diff --git a/sound/soc/sof/ipc4-loader.c b/sound/soc/sof/ipc4-loader.c
index b0d293f62d1c..07a78cb3c25c 100644
--- a/sound/soc/sof/ipc4-loader.c
+++ b/sound/soc/sof/ipc4-loader.c
@@ -175,7 +175,7 @@ static int sof_ipc4_load_library(struct snd_sof_dev *sdev, unsigned long lib_id,
 	struct sof_ipc4_fw_data *ipc4_data = sdev->private;
 	struct sof_ipc4_fw_library *fw_lib;
 	ssize_t payload_offset;
-	int ret, i, err;
+	int ret, i;
 
 	if (!ipc4_data->load_library) {
 		dev_err(sdev->dev, "Library loading is not supported on this platform\n");
@@ -223,24 +223,7 @@ static int sof_ipc4_load_library(struct snd_sof_dev *sdev, unsigned long lib_id,
 	for (i = 0; i < fw_lib->num_modules; i++)
 		fw_lib->modules[i].man4_module_entry.id |= (lib_id << SOF_IPC4_MOD_LIB_ID_SHIFT);
 
-	/*
-	 * Make sure that the DSP is booted and stays up while attempting the
-	 * loading the library for the first time
-	 */
-	ret = pm_runtime_resume_and_get(sdev->dev);
-	if (ret < 0 && ret != -EACCES) {
-		dev_err_ratelimited(sdev->dev, "%s: pm_runtime resume failed: %d\n",
-				    __func__, ret);
-		goto release;
-	}
-
 	ret = ipc4_data->load_library(sdev, fw_lib, false);
-
-	err = pm_runtime_put_autosuspend(sdev->dev);
-	if (err < 0)
-		dev_err_ratelimited(sdev->dev, "%s: pm_runtime idle failed: %d\n",
-				    __func__, err);
-
 	if (ret)
 		goto release;
 
diff --git a/sound/soc/sof/ipc4-topology.c b/sound/soc/sof/ipc4-topology.c
index 221e9d4052b8..d621e7914a73 100644
--- a/sound/soc/sof/ipc4-topology.c
+++ b/sound/soc/sof/ipc4-topology.c
@@ -1752,11 +1752,9 @@ snd_sof_get_nhlt_endpoint_data(struct snd_sof_dev *sdev, struct snd_sof_dai *dai
 		channel_count = params_channels(params);
 		sample_rate = params_rate(params);
 		bit_depth = params_width(params);
-		/*
-		 * Look for 32-bit blob first instead of 16-bit if copier
-		 * supports multiple formats
-		 */
-		if (bit_depth == 16 && !single_bitdepth) {
+
+		/* Prefer 32-bit blob if copier supports multiple formats */
+		if (bit_depth <= 16 && !single_bitdepth) {
 			dev_dbg(sdev->dev, "Looking for 32-bit blob first for DMIC\n");
 			format_change = true;
 			bit_depth = 32;
@@ -1799,10 +1797,18 @@ snd_sof_get_nhlt_endpoint_data(struct snd_sof_dev *sdev, struct snd_sof_dai *dai
 		if (format_change) {
 			/*
 			 * The 32-bit blob was not found in NHLT table, try to
-			 * look for one based on the params
+			 * look for 16-bit for DMIC or based on the params for
+			 * SSP
 			 */
-			bit_depth = params_width(params);
-			format_change = false;
+			if (linktype == SOF_DAI_INTEL_DMIC) {
+				bit_depth = 16;
+				if (params_width(params) == 16)
+					format_change = false;
+			} else {
+				bit_depth = params_width(params);
+				format_change = false;
+			}
+
 			get_new_blob = true;
 		} else if (linktype == SOF_DAI_INTEL_DMIC && !single_bitdepth) {
 			/*
@@ -1837,7 +1843,7 @@ snd_sof_get_nhlt_endpoint_data(struct snd_sof_dev *sdev, struct snd_sof_dai *dai
 	*len = cfg->size >> 2;
 	*dst = (u32 *)cfg->caps;
 
-	if (format_change) {
+	if (format_change || params_format(params) == SNDRV_PCM_FORMAT_FLOAT_LE) {
 		/*
 		 * Update the params to reflect that different blob was loaded
 		 * instead of the requested bit depth (16 -> 32 or 32 -> 16).
@@ -2280,8 +2286,19 @@ sof_ipc4_prepare_copier_module(struct snd_sof_widget *swidget,
 				ch_map >>= 4;
 			}
 
-			step = ch_count / blob->alh_cfg.device_count;
-			mask =  GENMASK(step - 1, 0);
+			if (swidget->id == snd_soc_dapm_dai_in && ch_count == out_ref_channels) {
+				/*
+				 * For playback DAI widgets where the channel number is equal to
+				 * the output reference channels, set the step = 0 to ensure all
+				 * the ch_mask is applied to all alh mappings.
+				 */
+				mask = ch_mask;
+				step = 0;
+			} else {
+				step = ch_count / blob->alh_cfg.device_count;
+				mask =  GENMASK(step - 1, 0);
+			}
+
 			/*
 			 * Set each gtw_cfg.node_id to blob->alh_cfg.mapping[]
 			 * for all widgets with the same stream name
@@ -2316,8 +2333,9 @@ sof_ipc4_prepare_copier_module(struct snd_sof_widget *swidget,
 				}
 
 				/*
-				 * Set the same channel mask for playback as the audio data is
-				 * duplicated for all speakers. For capture, split the channels
+				 * Set the same channel mask if the widget channel count is the same
+				 * as the FE channels for playback as the audio data is duplicated
+				 * for all speakers in this case. Otherwise, split the channels
 				 * among the aggregated DAIs. For example, with 4 channels on 2
 				 * aggregated DAIs, the channel_mask should be 0x3 and 0xc for the
 				 * two DAI's.
@@ -2326,10 +2344,7 @@ sof_ipc4_prepare_copier_module(struct snd_sof_widget *swidget,
 				 * the tables in soc_acpi files depending on the _ADR and devID
 				 * registers for each codec.
 				 */
-				if (w->id == snd_soc_dapm_dai_in)
-					blob->alh_cfg.mapping[i].channel_mask = ch_mask;
-				else
-					blob->alh_cfg.mapping[i].channel_mask = mask << (step * i);
+				blob->alh_cfg.mapping[i].channel_mask = mask << (step * i);
 
 				i++;
 			}
diff --git a/sound/soc/sof/ipc4.c b/sound/soc/sof/ipc4.c
index a4a090e6724a..1df97129cee6 100644
--- a/sound/soc/sof/ipc4.c
+++ b/sound/soc/sof/ipc4.c
@@ -892,6 +892,19 @@ void sof_ipc4_mic_privacy_state_change(struct snd_sof_dev *sdev, bool state)
 	struct sof_ipc4_msg msg;
 	u32 data = state;
 
+	/*
+	 * The mic privacy change notification's role is to notify the running
+	 * firmware that there is a change in mic privacy state from whatever
+	 * the state was before - since the firmware booted up or since the
+	 * previous change during runtime.
+	 *
+	 * If the firmware has not been booted up, there is no need to send
+	 * change notification (the firmware is not booted up).
+	 * The firmware checks the current state during its boot.
+	 */
+	if (sdev->fw_state != SOF_FW_BOOT_COMPLETE)
+		return;
+
 	msg.primary = SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
 	msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
 	msg.primary |= SOF_IPC4_MOD_ID(SOF_IPC4_MOD_INIT_BASEFW_MOD_ID);
diff --git a/sound/soc/sof/pcm.c b/sound/soc/sof/pcm.c
index cee04574264e..31879a11c33e 100644
--- a/sound/soc/sof/pcm.c
+++ b/sound/soc/sof/pcm.c
@@ -122,6 +122,16 @@ static int sof_pcm_hw_params(struct snd_soc_component *component,
 
 	spcm_dbg(spcm, substream->stream, "Entry: hw_params\n");
 
+	if (!sdev->dspless_mode_selected) {
+		/*
+		 * Make sure that the DSP is booted up, which might not be the
+		 * case if the on-demand DSP boot is used
+		 */
+		ret = snd_sof_boot_dsp_firmware(sdev);
+		if (ret)
+			return ret;
+	}
+
 	/*
 	 * Handle repeated calls to hw_params() without free_pcm() in
 	 * between. At least ALSA OSS emulation depends on this.
diff --git a/sound/soc/sof/pm.c b/sound/soc/sof/pm.c
index 8e3bcf602beb..dd7cd87f1fa5 100644
--- a/sound/soc/sof/pm.c
+++ b/sound/soc/sof/pm.c
@@ -8,10 +8,15 @@
 // Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
 //
 
+#include <linux/module.h>
 #include "ops.h"
 #include "sof-priv.h"
 #include "sof-audio.h"
 
+static int override_on_demand_boot = -1;
+module_param_named(on_demand_boot, override_on_demand_boot, int, 0444);
+MODULE_PARM_DESC(on_demand_boot, "Force on-demand DSP boot: 0 - disabled, 1 - enabled");
+
 /*
  * Helper function to determine the target DSP state during
  * system suspend. This function only cares about the device
@@ -70,67 +75,28 @@ static void sof_cache_debugfs(struct snd_sof_dev *sdev)
 }
 #endif
 
-static int sof_resume(struct device *dev, bool runtime_resume)
+int snd_sof_boot_dsp_firmware(struct snd_sof_dev *sdev)
 {
-	struct snd_sof_dev *sdev = dev_get_drvdata(dev);
 	const struct sof_ipc_pm_ops *pm_ops = sof_ipc_get_ops(sdev, pm);
 	const struct sof_ipc_tplg_ops *tplg_ops = sof_ipc_get_ops(sdev, tplg);
-	u32 old_state = sdev->dsp_power_state.state;
 	int ret;
 
-	/* do nothing if dsp resume callbacks are not set */
-	if (!runtime_resume && !sof_ops(sdev)->resume)
-		return 0;
-
-	if (runtime_resume && !sof_ops(sdev)->runtime_resume)
-		return 0;
-
-	/* DSP was never successfully started, nothing to resume */
-	if (sdev->first_boot)
-		return 0;
-
-	/*
-	 * if the runtime_resume flag is set, call the runtime_resume routine
-	 * or else call the system resume routine
-	 */
-	if (runtime_resume)
-		ret = snd_sof_dsp_runtime_resume(sdev);
-	else
-		ret = snd_sof_dsp_resume(sdev);
-	if (ret < 0) {
-		dev_err(sdev->dev,
-			"error: failed to power up DSP after resume\n");
-		return ret;
-	}
+	guard(mutex)(&sdev->dsp_fw_boot_mutex);
 
-	if (sdev->dspless_mode_selected) {
-		sof_set_fw_state(sdev, SOF_DSPLESS_MODE);
+	if (sdev->fw_state == SOF_FW_BOOT_COMPLETE) {
+		/* Firmware already booted, just return */
 		return 0;
 	}
 
-	/*
-	 * Nothing further to be done for platforms that support the low power
-	 * D0 substate. Resume trace and return when resuming from
-	 * low-power D0 substate
-	 */
-	if (!runtime_resume && sof_ops(sdev)->set_power_state &&
-	    old_state == SOF_DSP_PM_D0) {
-		ret = sof_fw_trace_resume(sdev);
-		if (ret < 0)
-			/* non fatal */
-			dev_warn(sdev->dev,
-				 "failed to enable trace after resume %d\n", ret);
-		return 0;
-	}
+	dev_dbg(sdev->dev, "Booting DSP firmware\n");
 
 	sof_set_fw_state(sdev, SOF_FW_BOOT_PREPARE);
 
 	/* load the firmware */
 	ret = snd_sof_load_firmware(sdev);
 	if (ret < 0) {
-		dev_err(sdev->dev,
-			"error: failed to load DSP firmware after resume %d\n",
-			ret);
+		dev_err(sdev->dev, "%s: failed to load DSP firmware: %d\n",
+			__func__, ret);
 		sof_set_fw_state(sdev, SOF_FW_BOOT_FAILED);
 		return ret;
 	}
@@ -143,9 +109,8 @@ static int sof_resume(struct device *dev, bool runtime_resume)
 	 */
 	ret = snd_sof_run_firmware(sdev);
 	if (ret < 0) {
-		dev_err(sdev->dev,
-			"error: failed to boot DSP firmware after resume %d\n",
-			ret);
+		dev_err(sdev->dev, "%s: failed to boot DSP firmware: %d\n",
+			__func__, ret);
 		sof_set_fw_state(sdev, SOF_FW_BOOT_FAILED);
 		return ret;
 	}
@@ -154,16 +119,16 @@ static int sof_resume(struct device *dev, bool runtime_resume)
 	ret = sof_fw_trace_resume(sdev);
 	if (ret < 0) {
 		/* non fatal */
-		dev_warn(sdev->dev,
-			 "warning: failed to init trace after resume %d\n",
-			 ret);
+		dev_warn(sdev->dev, "%s: failed to resume trace: %d\n",
+			 __func__, ret);
 	}
 
 	/* restore pipelines */
 	if (tplg_ops && tplg_ops->set_up_all_pipelines) {
 		ret = tplg_ops->set_up_all_pipelines(sdev, false);
 		if (ret < 0) {
-			dev_err(sdev->dev, "Failed to restore pipeline after resume %d\n", ret);
+			dev_err(sdev->dev, "%s: failed to restore pipeline: %d\n",
+				__func__, ret);
 			goto setup_fail;
 		}
 	}
@@ -175,7 +140,8 @@ static int sof_resume(struct device *dev, bool runtime_resume)
 	if (pm_ops && pm_ops->ctx_restore) {
 		ret = pm_ops->ctx_restore(sdev);
 		if (ret < 0)
-			dev_err(sdev->dev, "ctx_restore IPC error during resume: %d\n", ret);
+			dev_err(sdev->dev, "%s: ctx_restore IPC failed: %d\n",
+				__func__, ret);
 	}
 
 setup_fail:
@@ -192,6 +158,73 @@ static int sof_resume(struct device *dev, bool runtime_resume)
 
 	return ret;
 }
+EXPORT_SYMBOL(snd_sof_boot_dsp_firmware);
+
+static int sof_resume(struct device *dev, bool runtime_resume)
+{
+	struct snd_sof_dev *sdev = dev_get_drvdata(dev);
+	u32 old_state = sdev->dsp_power_state.state;
+	bool on_demand_boot;
+	int ret;
+
+	/* do nothing if dsp resume callbacks are not set */
+	if (!runtime_resume && !sof_ops(sdev)->resume)
+		return 0;
+
+	if (runtime_resume && !sof_ops(sdev)->runtime_resume)
+		return 0;
+
+	/* DSP was never successfully started, nothing to resume */
+	if (sdev->first_boot)
+		return 0;
+
+	/*
+	 * if the runtime_resume flag is set, call the runtime_resume routine
+	 * or else call the system resume routine
+	 */
+	if (runtime_resume)
+		ret = snd_sof_dsp_runtime_resume(sdev);
+	else
+		ret = snd_sof_dsp_resume(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: failed to power up DSP after resume\n");
+		return ret;
+	}
+
+	if (sdev->dspless_mode_selected) {
+		sof_set_fw_state(sdev, SOF_DSPLESS_MODE);
+		return 0;
+	}
+
+	/*
+	 * Nothing further to be done for platforms that support the low power
+	 * D0 substate. Resume trace and return when resuming from
+	 * low-power D0 substate
+	 */
+	if (!runtime_resume && sof_ops(sdev)->set_power_state &&
+	    old_state == SOF_DSP_PM_D0) {
+		ret = sof_fw_trace_resume(sdev);
+		if (ret < 0)
+			/* non fatal */
+			dev_warn(sdev->dev,
+				 "failed to enable trace after resume %d\n", ret);
+		return 0;
+	}
+
+	if (override_on_demand_boot > -1)
+		on_demand_boot = override_on_demand_boot ? true : false;
+	else
+		on_demand_boot = sdev->pdata->desc->on_demand_dsp_boot;
+
+	if (on_demand_boot) {
+		/* Only change the fw_state to PREPARE but skip booting */
+		sof_set_fw_state(sdev, SOF_FW_BOOT_PREPARE);
+		return 0;
+	}
+
+	return snd_sof_boot_dsp_firmware(sdev);
+}
 
 static int sof_suspend(struct device *dev, bool runtime_suspend)
 {
@@ -297,8 +330,12 @@ int snd_sof_dsp_power_down_notify(struct snd_sof_dev *sdev)
 {
 	const struct sof_ipc_pm_ops *pm_ops = sof_ipc_get_ops(sdev, pm);
 
-	/* Notify DSP of upcoming power down */
-	if (sof_ops(sdev)->remove && pm_ops && pm_ops->ctx_save)
+	/*
+	 * Notify DSP of upcoming power down only if the firmware has been
+	 * booted up
+	 */
+	if (sdev->fw_state == SOF_FW_BOOT_COMPLETE && sof_ops(sdev)->remove &&
+	    pm_ops && pm_ops->ctx_save)
 		return pm_ops->ctx_save(sdev);
 
 	return 0;
diff --git a/sound/soc/sof/sof-client-ipc-flood-test.c b/sound/soc/sof/sof-client-ipc-flood-test.c
index 373f3a125372..7b72d1c9c739 100644
--- a/sound/soc/sof/sof-client-ipc-flood-test.c
+++ b/sound/soc/sof/sof-client-ipc-flood-test.c
@@ -219,9 +219,10 @@ static ssize_t sof_ipc_flood_dfs_write(struct file *file, const char __user *buf
 		goto out;
 	}
 
-	/* flood test */
-	ret = sof_debug_ipc_flood_test(cdev, flood_duration_test,
-				       ipc_duration_ms, ipc_count);
+	ret = sof_client_boot_dsp(cdev);
+	if (!ret)
+		ret = sof_debug_ipc_flood_test(cdev, flood_duration_test,
+					       ipc_duration_ms, ipc_count);
 
 	err = pm_runtime_put_autosuspend(dev);
 	if (err < 0)
diff --git a/sound/soc/sof/sof-client-ipc-kernel-injector.c b/sound/soc/sof/sof-client-ipc-kernel-injector.c
index 249bd2d6c8d2..d5984990098a 100644
--- a/sound/soc/sof/sof-client-ipc-kernel-injector.c
+++ b/sound/soc/sof/sof-client-ipc-kernel-injector.c
@@ -63,7 +63,9 @@ static ssize_t sof_kernel_msg_inject_dfs_write(struct file *file, const char __u
 		return ret;
 	}
 
-	sof_client_ipc_rx_message(cdev, hdr, priv->kernel_buffer);
+	ret = sof_client_boot_dsp(cdev);
+	if (!ret)
+		sof_client_ipc_rx_message(cdev, hdr, priv->kernel_buffer);
 
 	ret = pm_runtime_put_autosuspend(dev);
 	if (ret < 0)
diff --git a/sound/soc/sof/sof-client-ipc-msg-injector.c b/sound/soc/sof/sof-client-ipc-msg-injector.c
index 9c8a0fbfb8df..c28f106de6ba 100644
--- a/sound/soc/sof/sof-client-ipc-msg-injector.c
+++ b/sound/soc/sof/sof-client-ipc-msg-injector.c
@@ -131,11 +131,15 @@ static int sof_msg_inject_send_message(struct sof_client_dev *cdev)
 		return ret;
 	}
 
-	/* send the message */
-	ret = sof_client_ipc_tx_message(cdev, priv->tx_buffer, priv->rx_buffer,
-					priv->max_msg_size);
-	if (ret)
-		dev_err(dev, "IPC message send failed: %d\n", ret);
+	ret = sof_client_boot_dsp(cdev);
+	if (!ret) {
+		/* send the message */
+		ret = sof_client_ipc_tx_message(cdev, priv->tx_buffer,
+						priv->rx_buffer,
+						priv->max_msg_size);
+		if (ret)
+			dev_err(dev, "IPC message send failed: %d\n", ret);
+	}
 
 	err = pm_runtime_put_autosuspend(dev);
 	if (err < 0)
diff --git a/sound/soc/sof/sof-client-probes.c b/sound/soc/sof/sof-client-probes.c
index f753e0faff99..124f55508159 100644
--- a/sound/soc/sof/sof-client-probes.c
+++ b/sound/soc/sof/sof-client-probes.c
@@ -123,6 +123,10 @@ static int sof_probes_compr_set_params(struct snd_compr_stream *cstream,
 	if (ret)
 		return ret;
 
+	ret = sof_client_boot_dsp(cdev);
+	if (ret)
+		return ret;
+
 	ret = ipc->init(cdev, priv->extractor_stream_tag, rtd->dma_bytes);
 	if (ret < 0) {
 		dev_err(dai->dev, "Failed to init probe: %d\n", ret);
@@ -224,6 +228,10 @@ static ssize_t sof_probes_dfs_points_read(struct file *file, char __user *to,
 		goto exit;
 	}
 
+	ret = sof_client_boot_dsp(cdev);
+	if (ret)
+		goto pm_error;
+
 	ret = ipc->points_info(cdev, &desc, &num_desc, type);
 	if (ret < 0)
 		goto pm_error;
@@ -312,9 +320,12 @@ sof_probes_dfs_points_write(struct file *file, const char __user *from,
 		goto exit;
 	}
 
-	ret = ipc->points_add(cdev, desc, bytes / sizeof(*desc));
-	if (!ret)
-		ret = count;
+	ret = sof_client_boot_dsp(cdev);
+	if (!ret) {
+		ret = ipc->points_add(cdev, desc, bytes / sizeof(*desc));
+		if (!ret)
+			ret = count;
+	}
 
 	err = pm_runtime_put_autosuspend(dev);
 	if (err < 0)
@@ -367,9 +378,12 @@ sof_probes_dfs_points_remove_write(struct file *file, const char __user *from,
 		goto exit;
 	}
 
-	ret = ipc->points_remove(cdev, &array[1], array[0]);
-	if (!ret)
-		ret = count;
+	ret = sof_client_boot_dsp(cdev);
+	if (!ret) {
+		ret = ipc->points_remove(cdev, &array[1], array[0]);
+		if (!ret)
+			ret = count;
+	}
 
 	err = pm_runtime_put_autosuspend(dev);
 	if (err < 0)
diff --git a/sound/soc/sof/sof-client.c b/sound/soc/sof/sof-client.c
index 2dbfc7699c73..b0802484a2d3 100644
--- a/sound/soc/sof/sof-client.c
+++ b/sound/soc/sof/sof-client.c
@@ -486,6 +486,12 @@ enum sof_ipc_type sof_client_get_ipc_type(struct sof_client_dev *cdev)
 }
 EXPORT_SYMBOL_NS_GPL(sof_client_get_ipc_type, "SND_SOC_SOF_CLIENT");
 
+int sof_client_boot_dsp(struct sof_client_dev *cdev)
+{
+	return snd_sof_boot_dsp_firmware(sof_client_dev_to_sof_dev(cdev));
+}
+EXPORT_SYMBOL_NS_GPL(sof_client_boot_dsp, "SND_SOC_SOF_CLIENT");
+
 /* module refcount management of SOF core */
 int sof_client_core_module_get(struct sof_client_dev *cdev)
 {
diff --git a/sound/soc/sof/sof-client.h b/sound/soc/sof/sof-client.h
index 1a9015e38474..3b02506c03f1 100644
--- a/sound/soc/sof/sof-client.h
+++ b/sound/soc/sof/sof-client.h
@@ -50,6 +50,9 @@ const struct sof_ipc_fw_version *sof_client_get_fw_version(struct sof_client_dev
 size_t sof_client_get_ipc_max_payload_size(struct sof_client_dev *cdev);
 enum sof_ipc_type sof_client_get_ipc_type(struct sof_client_dev *cdev);
 
+/* DSP/firmware boot request */
+int sof_client_boot_dsp(struct sof_client_dev *cdev);
+
 /* module refcount management of SOF core */
 int sof_client_core_module_get(struct sof_client_dev *cdev);
 void sof_client_core_module_put(struct sof_client_dev *cdev);
diff --git a/sound/soc/sof/sof-priv.h b/sound/soc/sof/sof-priv.h
index 0f624d8cde20..693d063830fa 100644
--- a/sound/soc/sof/sof-priv.h
+++ b/sound/soc/sof/sof-priv.h
@@ -580,6 +580,8 @@ struct snd_sof_dev {
 	wait_queue_head_t boot_wait;
 	enum sof_fw_state fw_state;
 	bool first_boot;
+	/* mutex to protect DSP firmware boot (except initial, probe time boot */
+	struct mutex dsp_fw_boot_mutex;
 
 	/* work queue in case the probe is implemented in two steps */
 	struct work_struct probe_work;
@@ -703,6 +705,7 @@ int snd_sof_suspend(struct device *dev);
 int snd_sof_dsp_power_down_notify(struct snd_sof_dev *sdev);
 int snd_sof_prepare(struct device *dev);
 void snd_sof_complete(struct device *dev);
+int snd_sof_boot_dsp_firmware(struct snd_sof_dev *sdev);
 
 void snd_sof_new_platform_drv(struct snd_sof_dev *sdev);
 
diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c
index c1083ea4624a..9bf8ab610a7e 100644
--- a/sound/soc/sof/topology.c
+++ b/sound/soc/sof/topology.c
@@ -2106,8 +2106,8 @@ static int sof_route_load(struct snd_soc_component *scomp, int index,
 	/* source component */
 	source_swidget = snd_sof_find_swidget(scomp, (char *)route->source);
 	if (!source_swidget) {
-		dev_err(scomp->dev, "error: source %s not found\n",
-			route->source);
+		dev_err(scomp->dev, "source %s for sink %s is not found\n",
+			route->source, route->sink);
 		ret = -EINVAL;
 		goto err;
 	}
@@ -2125,8 +2125,8 @@ static int sof_route_load(struct snd_soc_component *scomp, int index,
 	/* sink component */
 	sink_swidget = snd_sof_find_swidget(scomp, (char *)route->sink);
 	if (!sink_swidget) {
-		dev_err(scomp->dev, "error: sink %s not found\n",
-			route->sink);
+		dev_err(scomp->dev, "sink %s for source %s is not found\n",
+			route->sink, route->source);
 		ret = -EINVAL;
 		goto err;
 	}
@@ -2506,12 +2506,28 @@ int snd_sof_load_topology(struct snd_soc_component *scomp, const char *file)
 	if (!tplg_files)
 		return -ENOMEM;
 
+	/* Try to use function topologies if possible */
 	if (!sof_pdata->disable_function_topology && !disable_function_topology &&
 	    sof_pdata->machine && sof_pdata->machine->get_function_tplg_files) {
+		/*
+		 * When the topology name contains 'dummy' word, it means that
+		 * there is no fallback option to monolithic topology in case
+		 * any of the function topologies might be missing.
+		 * In this case we should use best effort to form the card,
+		 * ignoring functionalities that we are missing a fragment for.
+		 *
+		 * Note: monolithic topologies also ignore these possibly
+		 * missing functions, so the functionality of the card would be
+		 * identical to the case if there would be a fallback monolithic
+		 * topology created for the configuration.
+		 */
+		bool no_fallback = strstr(file, "dummy");
+
 		tplg_cnt = sof_pdata->machine->get_function_tplg_files(scomp->card,
 								       sof_pdata->machine,
 								       tplg_filename_prefix,
-								       &tplg_files);
+								       &tplg_files,
+								       no_fallback);
 		if (tplg_cnt < 0) {
 			kfree(tplg_files);
 			return tplg_cnt;
diff --git a/sound/soc/tegra/tegra210_ahub.c b/sound/soc/tegra/tegra210_ahub.c
index e795907a3963..261d9067d27b 100644
--- a/sound/soc/tegra/tegra210_ahub.c
+++ b/sound/soc/tegra/tegra210_ahub.c
@@ -2077,7 +2077,7 @@ static const struct regmap_config tegra210_ahub_regmap_config = {
 	.val_bits		= 32,
 	.reg_stride		= 4,
 	.max_register		= TEGRA210_MAX_REGISTER_ADDR,
-	.cache_type		= REGCACHE_FLAT,
+	.cache_type		= REGCACHE_FLAT_S,
 };
 
 static const struct regmap_config tegra186_ahub_regmap_config = {
@@ -2085,7 +2085,7 @@ static const struct regmap_config tegra186_ahub_regmap_config = {
 	.val_bits		= 32,
 	.reg_stride		= 4,
 	.max_register		= TEGRA186_MAX_REGISTER_ADDR,
-	.cache_type		= REGCACHE_FLAT,
+	.cache_type		= REGCACHE_FLAT_S,
 };
 
 static const struct regmap_config tegra264_ahub_regmap_config = {
@@ -2094,7 +2094,7 @@ static const struct regmap_config tegra264_ahub_regmap_config = {
 	.reg_stride		= 4,
 	.writeable_reg		= tegra264_ahub_wr_reg,
 	.max_register		= TEGRA264_MAX_REGISTER_ADDR,
-	.cache_type		= REGCACHE_FLAT,
+	.cache_type		= REGCACHE_FLAT_S,
 };
 
 static const struct tegra_ahub_soc_data soc_data_tegra210 = {
diff --git a/sound/soc/ti/davinci-mcasp.c b/sound/soc/ti/davinci-mcasp.c
index 621a9d5f9377..db6913c05378 100644
--- a/sound/soc/ti/davinci-mcasp.c
+++ b/sound/soc/ti/davinci-mcasp.c
@@ -25,6 +25,7 @@
 #include <linux/math64.h>
 #include <linux/bitmap.h>
 #include <linux/gpio/driver.h>
+#include <linux/property.h>
 
 #include <sound/asoundef.h>
 #include <sound/core.h>
@@ -1875,11 +1876,7 @@ static int mcasp_reparent_fck(struct platform_device *pdev)
 
 static bool davinci_mcasp_have_gpiochip(struct davinci_mcasp *mcasp)
 {
-#ifdef CONFIG_OF_GPIO
-	return of_property_read_bool(mcasp->dev->of_node, "gpio-controller");
-#else
-	return false;
-#endif
+	return device_property_present(mcasp->dev, "gpio-controller");
 }
 
 static int davinci_mcasp_get_config(struct davinci_mcasp *mcasp,
diff --git a/sound/usb/6fire/firmware.c b/sound/usb/6fire/firmware.c
index c51abc54d2f8..cc8caec946cc 100644
--- a/sound/usb/6fire/firmware.c
+++ b/sound/usb/6fire/firmware.c
@@ -12,6 +12,7 @@
 #include <linux/firmware.h>
 #include <linux/module.h>
 #include <linux/bitrev.h>
+#include <linux/hex.h>
 #include <linux/kernel.h>
 
 #include "firmware.h"
diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c
index cc15624ecaff..8f9313857ee9 100644
--- a/sound/usb/endpoint.c
+++ b/sound/usb/endpoint.c
@@ -1481,15 +1481,15 @@ int snd_usb_endpoint_prepare(struct snd_usb_audio *chip,
 			return err;
 	}
 
-	err = snd_usb_init_pitch(chip, ep->cur_audiofmt);
+	err = snd_usb_select_mode_quirk(chip, ep->cur_audiofmt);
 	if (err < 0)
 		return err;
 
-	err = init_sample_rate(chip, ep);
+	err = snd_usb_init_pitch(chip, ep->cur_audiofmt);
 	if (err < 0)
 		return err;
 
-	err = snd_usb_select_mode_quirk(chip, ep->cur_audiofmt);
+	err = init_sample_rate(chip, ep);
 	if (err < 0)
 		return err;
 
diff --git a/sound/usb/fcp.c b/sound/usb/fcp.c
index 11e9a96b46ff..1f4595d1e217 100644
--- a/sound/usb/fcp.c
+++ b/sound/usb/fcp.c
@@ -182,10 +182,6 @@ static int fcp_usb(struct usb_mixer_interface *mixer, u32 opcode,
 {
 	struct fcp_data *private = mixer->private_data;
 	struct usb_device *dev = mixer->chip->dev;
-	struct fcp_usb_packet *req __free(kfree) = NULL;
-	struct fcp_usb_packet *resp __free(kfree) = NULL;
-	size_t req_buf_size = struct_size(req, data, req_size);
-	size_t resp_buf_size = struct_size(resp, data, resp_size);
 	int retries = 0;
 	const int max_retries = 5;
 	int err;
@@ -193,10 +189,14 @@ static int fcp_usb(struct usb_mixer_interface *mixer, u32 opcode,
 	if (!mixer->urb)
 		return -ENODEV;
 
+	struct fcp_usb_packet *req __free(kfree) = NULL;
+	size_t req_buf_size = struct_size(req, data, req_size);
 	req = kmalloc(req_buf_size, GFP_KERNEL);
 	if (!req)
 		return -ENOMEM;
 
+	struct fcp_usb_packet *resp __free(kfree) = NULL;
+	size_t resp_buf_size = struct_size(resp, data, resp_size);
 	resp = kmalloc(resp_buf_size, GFP_KERNEL);
 	if (!resp)
 		return -ENOMEM;
@@ -300,16 +300,17 @@ static int fcp_usb(struct usb_mixer_interface *mixer, u32 opcode,
 static int fcp_reinit(struct usb_mixer_interface *mixer)
 {
 	struct fcp_data *private = mixer->private_data;
-	void *step0_resp __free(kfree) = NULL;
-	void *step2_resp __free(kfree) = NULL;
 
 	if (mixer->urb)
 		return 0;
 
-	step0_resp = kmalloc(private->step0_resp_size, GFP_KERNEL);
+	void *step0_resp __free(kfree) =
+		kmalloc(private->step0_resp_size, GFP_KERNEL);
 	if (!step0_resp)
 		return -ENOMEM;
-	step2_resp = kmalloc(private->step2_resp_size, GFP_KERNEL);
+
+	void *step2_resp __free(kfree) =
+		kmalloc(private->step2_resp_size, GFP_KERNEL);
 	if (!step2_resp)
 		return -ENOMEM;
 
@@ -464,7 +465,6 @@ static int fcp_ioctl_init(struct usb_mixer_interface *mixer,
 	struct fcp_init init;
 	struct usb_device *dev = mixer->chip->dev;
 	struct fcp_data *private = mixer->private_data;
-	void *resp __free(kfree) = NULL;
 	void *step2_resp;
 	int err, buf_size;
 
@@ -485,7 +485,8 @@ static int fcp_ioctl_init(struct usb_mixer_interface *mixer,
 	/* Allocate response buffer */
 	buf_size = init.step0_resp_size + init.step2_resp_size;
 
-	resp = kmalloc(buf_size, GFP_KERNEL);
+	void *resp __free(kfree) =
+		kmalloc(buf_size, GFP_KERNEL);
 	if (!resp)
 		return -ENOMEM;
 
@@ -619,7 +620,6 @@ static int fcp_ioctl_set_meter_map(struct usb_mixer_interface *mixer,
 {
 	struct fcp_meter_map map;
 	struct fcp_data *private = mixer->private_data;
-	s16 *tmp_map __free(kfree) = NULL;
 	int err;
 
 	if (copy_from_user(&map, arg, sizeof(map)))
@@ -641,7 +641,8 @@ static int fcp_ioctl_set_meter_map(struct usb_mixer_interface *mixer,
 		return -EINVAL;
 
 	/* Allocate and copy the map data */
-	tmp_map = memdup_array_user(arg->map, map.map_size, sizeof(s16));
+	s16 *tmp_map __free(kfree) =
+		memdup_array_user(arg->map, map.map_size, sizeof(s16));
 	if (IS_ERR(tmp_map))
 		return PTR_ERR(tmp_map);
 
@@ -651,17 +652,16 @@ static int fcp_ioctl_set_meter_map(struct usb_mixer_interface *mixer,
 
 	/* If the control doesn't exist, create it */
 	if (!private->meter_ctl) {
-		s16 *new_map __free(kfree) = NULL;
-		__le32 *meter_levels __free(kfree) = NULL;
-
 		/* Allocate buffer for the map */
-		new_map = kmalloc_array(map.map_size, sizeof(s16), GFP_KERNEL);
+		s16 *new_map __free(kfree) =
+			kmalloc_array(map.map_size, sizeof(s16), GFP_KERNEL);
 		if (!new_map)
 			return -ENOMEM;
 
 		/* Allocate buffer for reading meter levels */
-		meter_levels = kmalloc_array(map.meter_slots, sizeof(__le32),
-					     GFP_KERNEL);
+		__le32 *meter_levels __free(kfree) =
+			kmalloc_array(map.meter_slots, sizeof(__le32),
+				      GFP_KERNEL);
 		if (!meter_levels)
 			return -ENOMEM;
 
diff --git a/sound/usb/format.c b/sound/usb/format.c
index ec95a063beb1..64cfe4a9d8cd 100644
--- a/sound/usb/format.c
+++ b/sound/usb/format.c
@@ -34,6 +34,7 @@ static u64 parse_audio_format_i_type(struct snd_usb_audio *chip,
 {
 	int sample_width, sample_bytes;
 	u64 pcm_formats = 0;
+	u64 dsd_formats = 0;
 
 	switch (fp->protocol) {
 	case UAC_VERSION_1:
@@ -154,7 +155,9 @@ static u64 parse_audio_format_i_type(struct snd_usb_audio *chip,
 			 fp->iface, fp->altsetting, format);
 	}
 
-	pcm_formats |= snd_usb_interface_dsd_format_quirks(chip, fp, sample_bytes);
+	dsd_formats |= snd_usb_interface_dsd_format_quirks(chip, fp, sample_bytes);
+	if (dsd_formats && !fp->dsd_dop)
+		pcm_formats = dsd_formats;
 
 	return pcm_formats;
 }
diff --git a/sound/usb/mixer_scarlett2.c b/sound/usb/mixer_scarlett2.c
index f2446bf3982c..0d4d9a392234 100644
--- a/sound/usb/mixer_scarlett2.c
+++ b/sound/usb/mixer_scarlett2.c
@@ -2377,18 +2377,18 @@ static int scarlett2_usb(
 {
 	struct scarlett2_data *private = mixer->private_data;
 	struct usb_device *dev = mixer->chip->dev;
-	struct scarlett2_usb_packet *req __free(kfree) = NULL;
-	struct scarlett2_usb_packet *resp __free(kfree) = NULL;
-	size_t req_buf_size = struct_size(req, data, req_size);
-	size_t resp_buf_size = struct_size(resp, data, resp_size);
 	int retries = 0;
 	const int max_retries = 5;
 	int err;
 
+	struct scarlett2_usb_packet *req __free(kfree) = NULL;
+	size_t req_buf_size = struct_size(req, data, req_size);
 	req = kmalloc(req_buf_size, GFP_KERNEL);
 	if (!req)
 		return -ENOMEM;
 
+	struct scarlett2_usb_packet *resp __free(kfree) = NULL;
+	size_t resp_buf_size = struct_size(resp, data, resp_size);
 	resp = kmalloc(resp_buf_size, GFP_KERNEL);
 	if (!resp)
 		return -ENOMEM;
@@ -3919,9 +3919,9 @@ static int scarlett2_input_select_ctl_info(
 	struct scarlett2_data *private = mixer->private_data;
 	int inputs = private->info->gain_input_count;
 	int i, err;
-	char **values __free(kfree) = NULL;
+	char **values __free(kfree) =
+		kcalloc(inputs, sizeof(char *), GFP_KERNEL);
 
-	values = kcalloc(inputs, sizeof(char *), GFP_KERNEL);
 	if (!values)
 		return -ENOMEM;
 
@@ -9083,8 +9083,6 @@ static long scarlett2_hwdep_read(struct snd_hwdep *hw,
 		__le32 len;
 	} __packed req;
 
-	u8 *resp __free(kfree) = NULL;
-
 	/* Flash segment must first be selected */
 	if (private->flash_write_state != SCARLETT2_FLASH_WRITE_STATE_SELECTED)
 		return -EINVAL;
@@ -9122,7 +9120,8 @@ static long scarlett2_hwdep_read(struct snd_hwdep *hw,
 	req.offset = cpu_to_le32(*offset);
 	req.len = cpu_to_le32(count);
 
-	resp = kzalloc(count, GFP_KERNEL);
+	u8 *resp __free(kfree) =
+		kzalloc(count, GFP_KERNEL);
 	if (!resp)
 		return -ENOMEM;
 
@@ -9267,7 +9266,6 @@ static ssize_t scarlett2_devmap_read(
 	loff_t                 pos)
 {
 	struct usb_mixer_interface *mixer = entry->private_data;
-	u8           *resp_buf __free(kfree) = NULL;
 	const size_t  block_size = SCARLETT2_DEVMAP_BLOCK_SIZE;
 	size_t        copied = 0;
 
@@ -9277,7 +9275,8 @@ static ssize_t scarlett2_devmap_read(
 	if (pos + count > entry->size)
 		count = entry->size - pos;
 
-	resp_buf = kmalloc(block_size, GFP_KERNEL);
+	u8 *resp_buf __free(kfree) =
+		kmalloc(block_size, GFP_KERNEL);
 	if (!resp_buf)
 		return -ENOMEM;
 
diff --git a/sound/usb/mixer_us16x08.c b/sound/usb/mixer_us16x08.c
index 1c5712c31f5e..f9df40730eff 100644
--- a/sound/usb/mixer_us16x08.c
+++ b/sound/usb/mixer_us16x08.c
@@ -655,17 +655,25 @@ static void get_meter_levels_from_urb(int s,
 	u8 *meter_urb)
 {
 	int val = MUC2(meter_urb, s) + (MUC3(meter_urb, s) << 8);
+	int ch = MUB2(meter_urb, s) - 1;
+
+	if (ch < 0)
+		return;
 
 	if (MUA0(meter_urb, s) == 0x61 && MUA1(meter_urb, s) == 0x02 &&
 		MUA2(meter_urb, s) == 0x04 && MUB0(meter_urb, s) == 0x62) {
-		if (MUC0(meter_urb, s) == 0x72)
-			store->meter_level[MUB2(meter_urb, s) - 1] = val;
-		if (MUC0(meter_urb, s) == 0xb2)
-			store->comp_level[MUB2(meter_urb, s) - 1] = val;
+		if (ch < SND_US16X08_MAX_CHANNELS) {
+			if (MUC0(meter_urb, s) == 0x72)
+				store->meter_level[ch] = val;
+			if (MUC0(meter_urb, s) == 0xb2)
+				store->comp_level[ch] = val;
+		}
 	}
 	if (MUA0(meter_urb, s) == 0x61 && MUA1(meter_urb, s) == 0x02 &&
-		MUA2(meter_urb, s) == 0x02 && MUB0(meter_urb, s) == 0x62)
-		store->master_level[MUB2(meter_urb, s) - 1] = val;
+		MUA2(meter_urb, s) == 0x02 && MUB0(meter_urb, s) == 0x62) {
+		if (ch < ARRAY_SIZE(store->master_level))
+			store->master_level[ch] = val;
+	}
 }
 
 /* Function to retrieve current meter values from the device.
diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c
index 61bd61ffb1b2..9a76f35d331c 100644
--- a/sound/usb/quirks.c
+++ b/sound/usb/quirks.c
@@ -555,7 +555,6 @@ int snd_usb_create_quirk(struct snd_usb_audio *chip,
 static int snd_usb_extigy_boot_quirk(struct usb_device *dev, struct usb_interface *intf)
 {
 	struct usb_host_config *config = dev->actconfig;
-	struct usb_device_descriptor *new_device_descriptor __free(kfree) = NULL;
 	int err;
 
 	if (le16_to_cpu(get_cfg_desc(config)->wTotalLength) == EXTIGY_FIRMWARE_SIZE_OLD ||
@@ -566,8 +565,8 @@ static int snd_usb_extigy_boot_quirk(struct usb_device *dev, struct usb_interfac
 				      0x10, 0x43, 0x0001, 0x000a, NULL, 0);
 		if (err < 0)
 			dev_dbg(&dev->dev, "error sending boot message: %d\n", err);
-
-		new_device_descriptor = kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL);
+		struct usb_device_descriptor *new_device_descriptor __free(kfree) =
+			kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL);
 		if (!new_device_descriptor)
 			return -ENOMEM;
 		err = usb_get_descriptor(dev, USB_DT_DEVICE, 0,
@@ -910,7 +909,6 @@ static void mbox2_setup_48_24_magic(struct usb_device *dev)
 static int snd_usb_mbox2_boot_quirk(struct usb_device *dev)
 {
 	struct usb_host_config *config = dev->actconfig;
-	struct usb_device_descriptor *new_device_descriptor __free(kfree) = NULL;
 	int err;
 	u8 bootresponse[0x12];
 	int fwsize;
@@ -945,7 +943,8 @@ static int snd_usb_mbox2_boot_quirk(struct usb_device *dev)
 
 	dev_dbg(&dev->dev, "device initialised!\n");
 
-	new_device_descriptor = kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL);
+	struct usb_device_descriptor *new_device_descriptor __free(kfree) =
+		kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL);
 	if (!new_device_descriptor)
 		return -ENOMEM;
 
@@ -1267,7 +1266,6 @@ static void mbox3_setup_defaults(struct usb_device *dev)
 static int snd_usb_mbox3_boot_quirk(struct usb_device *dev)
 {
 	struct usb_host_config *config = dev->actconfig;
-	struct usb_device_descriptor *new_device_descriptor __free(kfree) = NULL;
 	int err;
 	int descriptor_size;
 
@@ -1280,7 +1278,8 @@ static int snd_usb_mbox3_boot_quirk(struct usb_device *dev)
 
 	dev_dbg(&dev->dev, "MBOX3: device initialised!\n");
 
-	new_device_descriptor = kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL);
+	struct usb_device_descriptor *new_device_descriptor __free(kfree) =
+		kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL);
 	if (!new_device_descriptor)
 		return -ENOMEM;
 
@@ -2221,7 +2220,7 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = {
 		   QUIRK_FLAG_IFACE_DELAY),
 	DEVICE_FLG(0x0644, 0x8044, /* Esoteric D-05X */
 		   QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY |
-		   QUIRK_FLAG_IFACE_DELAY),
+		   QUIRK_FLAG_IFACE_DELAY | QUIRK_FLAG_FORCE_IFACE_RESET),
 	DEVICE_FLG(0x0644, 0x804a, /* TEAC UD-301 */
 		   QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY |
 		   QUIRK_FLAG_IFACE_DELAY),
@@ -2229,7 +2228,13 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = {
 		   QUIRK_FLAG_FORCE_IFACE_RESET),
 	DEVICE_FLG(0x0644, 0x806b, /* TEAC UD-701 */
 		   QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY |
-		   QUIRK_FLAG_IFACE_DELAY),
+		   QUIRK_FLAG_IFACE_DELAY | QUIRK_FLAG_FORCE_IFACE_RESET),
+	DEVICE_FLG(0x0644, 0x807d, /* TEAC UD-507 */
+		   QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY |
+		   QUIRK_FLAG_IFACE_DELAY | QUIRK_FLAG_FORCE_IFACE_RESET),
+	DEVICE_FLG(0x0644, 0x806c, /* Esoteric XD */
+		   QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY |
+		   QUIRK_FLAG_IFACE_DELAY | QUIRK_FLAG_FORCE_IFACE_RESET),
 	DEVICE_FLG(0x06f8, 0xb000, /* Hercules DJ Console (Windows Edition) */
 		   QUIRK_FLAG_IGNORE_CTL_ERROR),
 	DEVICE_FLG(0x06f8, 0xd002, /* Hercules DJ Console (Macintosh Edition) */
@@ -2388,6 +2393,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = {
 		   QUIRK_FLAG_CTL_MSG_DELAY_1M),
 	DEVICE_FLG(0x30be, 0x0101, /* Schiit Hel */
 		   QUIRK_FLAG_IGNORE_CTL_ERROR),
+	DEVICE_FLG(0x3255, 0x0000, /* Luxman D-10X */
+		   QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY),
 	DEVICE_FLG(0x339b, 0x3a07, /* Synaptics HONOR USB-C HEADSET */
 		   QUIRK_FLAG_MIXER_PLAYBACK_MIN_MUTE),
 	DEVICE_FLG(0x413c, 0xa506, /* Dell AE515 sound bar */
@@ -2431,6 +2438,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = {
 		   QUIRK_FLAG_DSD_RAW),
 	VENDOR_FLG(0x2622, /* IAG Limited devices */
 		   QUIRK_FLAG_DSD_RAW),
+	VENDOR_FLG(0x2772, /* Musical Fidelity devices */
+		   QUIRK_FLAG_DSD_RAW),
 	VENDOR_FLG(0x278b, /* Rotel? */
 		   QUIRK_FLAG_DSD_RAW),
 	VENDOR_FLG(0x292b, /* Gustard/Ess based devices */
diff --git a/sound/usb/usx2y/us144mkii.c b/sound/usb/usx2y/us144mkii.c
index f6572a576c15..bc71968df8e2 100644
--- a/sound/usb/usx2y/us144mkii.c
+++ b/sound/usb/usx2y/us144mkii.c
@@ -412,7 +412,6 @@ static int tascam_probe(struct usb_interface *intf,
 	struct snd_card *card;
 	struct tascam_card *tascam;
 	int err;
-	char *handshake_buf __free(kfree) = NULL;
 
 	if (dev->speed != USB_SPEED_HIGH)
 		dev_info(
@@ -439,7 +438,8 @@ static int tascam_probe(struct usb_interface *intf,
 		return -ENOENT;
 	}
 
-	handshake_buf = kmalloc(1, GFP_KERNEL);
+	char *handshake_buf __free(kfree) =
+		kmalloc(1, GFP_KERNEL);
 	if (!handshake_buf)
 		return -ENOMEM;
 
diff --git a/sound/usb/usx2y/us144mkii_controls.c b/sound/usb/usx2y/us144mkii_controls.c
index 5d69441ef414..62055fb8e7ba 100644
--- a/sound/usb/usx2y/us144mkii_controls.c
+++ b/sound/usb/usx2y/us144mkii_controls.c
@@ -373,7 +373,6 @@ static int tascam_samplerate_get(struct snd_kcontrol *kcontrol,
 {
 	struct tascam_card *tascam =
 		(struct tascam_card *)snd_kcontrol_chip(kcontrol);
-	u8 *buf __free(kfree) = NULL;
 	int err;
 	u32 rate = 0;
 
@@ -384,7 +383,8 @@ static int tascam_samplerate_get(struct snd_kcontrol *kcontrol,
 		}
 	}
 
-	buf = kmalloc(3, GFP_KERNEL);
+	u8 *buf __free(kfree) =
+		kmalloc(3, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 
diff --git a/sound/usb/usx2y/us144mkii_pcm.c b/sound/usb/usx2y/us144mkii_pcm.c
index 0c84304d4624..03dfb1f38801 100644
--- a/sound/usb/usx2y/us144mkii_pcm.c
+++ b/sound/usb/usx2y/us144mkii_pcm.c
@@ -115,7 +115,6 @@ void process_capture_routing_us144mkii(struct tascam_card *tascam,
 int us144mkii_configure_device_for_rate(struct tascam_card *tascam, int rate)
 {
 	struct usb_device *dev = tascam->dev;
-	u8 *rate_payload_buf __free(kfree) = NULL;
 	u16 rate_vendor_wValue;
 	int err = 0;
 	const u8 *current_payload_src;
@@ -148,7 +147,8 @@ int us144mkii_configure_device_for_rate(struct tascam_card *tascam, int rate)
 		return -EINVAL;
 	}
 
-	rate_payload_buf = kmemdup(current_payload_src, 3, GFP_KERNEL);
+	u8 *rate_payload_buf __free(kfree) =
+		kmemdup(current_payload_src, 3, GFP_KERNEL);
 	if (!rate_payload_buf)
 		return -ENOMEM;
 
diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile
index 586d1b2595d1..5442073a2e42 100644
--- a/tools/bpf/bpftool/Makefile
+++ b/tools/bpf/bpftool/Makefile
@@ -224,6 +224,8 @@ endif
 
 $(OUTPUT)%.bpf.o: skeleton/%.bpf.c $(OUTPUT)vmlinux.h $(LIBBPF_BOOTSTRAP)
 	$(QUIET_CLANG)$(CLANG) \
+		-Wno-microsoft-anon-tag \
+		-fms-extensions \
 		-I$(or $(OUTPUT),.) \
 		-I$(srctree)/tools/include/uapi/ \
 		-I$(LIBBPF_BOOTSTRAP_INCLUDE) \
diff --git a/tools/build/Makefile.feature b/tools/build/Makefile.feature
index a7f030fc5e83..362cf8f4a0a0 100644
--- a/tools/build/Makefile.feature
+++ b/tools/build/Makefile.feature
@@ -99,7 +99,8 @@ FEATURE_TESTS_BASIC :=                  \
         libzstd				\
         disassembler-four-args		\
         disassembler-init-styled	\
-        file-handle
+        file-handle			\
+        libopenssl
 
 # FEATURE_TESTS_BASIC + FEATURE_TESTS_EXTRA is the complete list
 # of all feature tests
@@ -147,7 +148,8 @@ FEATURE_DISPLAY ?=              \
          lzma                   \
          bpf			\
          libaio			\
-         libzstd
+         libzstd		\
+         libopenssl
 
 #
 # Declare group members of a feature to display the logical OR of the detection
diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile
index 87a5a908d6fa..c699d4f4c6d9 100644
--- a/tools/build/feature/Makefile
+++ b/tools/build/feature/Makefile
@@ -67,12 +67,13 @@ FILES=                                          \
          test-libopencsd.bin			\
          test-clang.bin				\
          test-llvm.bin				\
-         test-llvm-perf.bin   \
+         test-llvm-perf.bin   			\
          test-libaio.bin			\
          test-libzstd.bin			\
          test-clang-bpf-co-re.bin		\
          test-file-handle.bin			\
-         test-libpfm4.bin
+         test-libpfm4.bin			\
+         test-libopenssl.bin
 
 FILES := $(addprefix $(OUTPUT),$(FILES))
 
@@ -381,6 +382,9 @@ $(OUTPUT)test-file-handle.bin:
 $(OUTPUT)test-libpfm4.bin:
 	$(BUILD) -lpfm
 
+$(OUTPUT)test-libopenssl.bin:
+	$(BUILD) -lssl
+
 $(OUTPUT)test-bpftool-skeletons.bin:
 	$(SYSTEM_BPFTOOL) version | grep '^features:.*skeletons' \
 		> $(@:.bin=.make.output) 2>&1
diff --git a/tools/build/feature/test-all.c b/tools/build/feature/test-all.c
index eb346160d0ba..1488bf6e6078 100644
--- a/tools/build/feature/test-all.c
+++ b/tools/build/feature/test-all.c
@@ -142,6 +142,10 @@
 # include "test-libtraceevent.c"
 #undef main
 
+#define main main_test_libopenssl
+# include "test-libopenssl.c"
+#undef main
+
 int main(int argc, char *argv[])
 {
 	main_test_libpython();
@@ -173,6 +177,7 @@ int main(int argc, char *argv[])
 	main_test_reallocarray();
 	main_test_libzstd();
 	main_test_libtraceevent();
+	main_test_libopenssl();
 
 	return 0;
 }
diff --git a/tools/build/feature/test-libopenssl.c b/tools/build/feature/test-libopenssl.c
new file mode 100644
index 000000000000..168c45894e8b
--- /dev/null
+++ b/tools/build/feature/test-libopenssl.c
@@ -0,0 +1,7 @@
+#include <openssl/ssl.h>
+#include <openssl/opensslv.h>
+
+int main(void)
+{
+	return SSL_library_init();
+}
diff --git a/tools/include/nolibc/Makefile b/tools/include/nolibc/Makefile
index 8118e22844f1..1958dda98895 100644
--- a/tools/include/nolibc/Makefile
+++ b/tools/include/nolibc/Makefile
@@ -54,6 +54,7 @@ all_files := \
 		sys/mman.h \
 		sys/mount.h \
 		sys/prctl.h \
+		sys/ptrace.h \
 		sys/random.h \
 		sys/reboot.h \
 		sys/resource.h \
@@ -103,9 +104,12 @@ headers_standalone: headers
 	$(Q)$(MAKE) -C $(srctree) headers
 	$(Q)$(MAKE) -C $(srctree) headers_install INSTALL_HDR_PATH=$(OUTPUT)sysroot
 
+CFLAGS_s390 := -m64
+CFLAGS := $(CFLAGS_$(ARCH))
+
 headers_check: headers_standalone
 	$(Q)for header in $(filter-out crt.h std.h,$(all_files)); do \
-		$(CC) $(CLANG_CROSS_FLAGS) -Wall -Werror -nostdinc -fsyntax-only -x c /dev/null \
+		$(CC) $(CFLAGS) $(CLANG_CROSS_FLAGS) -Wall -Werror -nostdinc -fsyntax-only -x c /dev/null \
 			-I$(or $(objtree),$(srctree))/usr/include -include $$header -include $$header || exit 1; \
 	done
 
diff --git a/tools/include/nolibc/nolibc.h b/tools/include/nolibc/nolibc.h
index 272dfc961158..9c7f43b9218b 100644
--- a/tools/include/nolibc/nolibc.h
+++ b/tools/include/nolibc/nolibc.h
@@ -101,6 +101,7 @@
 #include "sys/mman.h"
 #include "sys/mount.h"
 #include "sys/prctl.h"
+#include "sys/ptrace.h"
 #include "sys/random.h"
 #include "sys/reboot.h"
 #include "sys/resource.h"
diff --git a/tools/include/nolibc/sys/ptrace.h b/tools/include/nolibc/sys/ptrace.h
new file mode 100644
index 000000000000..5d1e52965878
--- /dev/null
+++ b/tools/include/nolibc/sys/ptrace.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
+/*
+ * ptrace for NOLIBC
+ * Copyright (C) 2017-2021 Willy Tarreau <w@1wt.eu>
+ * Copyright (C) 2025 Intel Corporation
+ */
+
+/* make sure to include all global symbols */
+#include "../nolibc.h"
+
+#ifndef _NOLIBC_SYS_PTRACE_H
+#define _NOLIBC_SYS_PTRACE_H
+
+#include "../sys.h"
+#include "uio.h"
+
+
+#include <linux/ptrace.h>
+
+/*
+ * long ptrace(int op, pid_t pid, void *addr, void *data);
+ */
+static __attribute__((unused))
+long sys_ptrace(int op, pid_t pid, void *addr, void *data)
+{
+	return my_syscall4(__NR_ptrace, op, pid, addr, data);
+}
+
+static __attribute__((unused))
+ssize_t ptrace(int op, pid_t pid, ...)
+{
+	ssize_t ret;
+	va_list args;
+
+	va_start(args, pid);
+	ret = __sysret(sys_ptrace(op, pid,
+				  va_arg(args, void *),
+				  va_arg(args, void *)));
+	va_end(args);
+
+	return ret;
+}
+
+#endif /* _NOLIBC_SYS_PTRACE_H */
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index be7d8e060e10..6b92b0847ec2 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -918,6 +918,16 @@ union bpf_iter_link_info {
  *		Number of bytes read from the stream on success, or -1 if an
  *		error occurred (in which case, *errno* is set appropriately).
  *
+ * BPF_PROG_ASSOC_STRUCT_OPS
+ * 	Description
+ * 		Associate a BPF program with a struct_ops map. The struct_ops
+ * 		map is identified by *map_fd* and the BPF program is
+ * 		identified by *prog_fd*.
+ *
+ * 	Return
+ * 		0 on success or -1 if an error occurred (in which case,
+ * 		*errno* is set appropriately).
+ *
  * NOTES
  *	eBPF objects (maps and programs) can be shared between processes.
  *
@@ -974,6 +984,7 @@ enum bpf_cmd {
 	BPF_PROG_BIND_MAP,
 	BPF_TOKEN_CREATE,
 	BPF_PROG_STREAM_READ_BY_FD,
+	BPF_PROG_ASSOC_STRUCT_OPS,
 	__MAX_BPF_CMD,
 };
 
@@ -1894,6 +1905,12 @@ union bpf_attr {
 		__u32		prog_fd;
 	} prog_stream_read;
 
+	struct {
+		__u32		map_fd;
+		__u32		prog_fd;
+		__u32		flags;
+	} prog_assoc_struct_ops;
+
 } __attribute__((aligned(8)));
 
 /* The description below is an attempt at providing documentation to eBPF
diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c
index b66f5fbfbbb2..21b57a629916 100644
--- a/tools/lib/bpf/bpf.c
+++ b/tools/lib/bpf/bpf.c
@@ -1397,3 +1397,22 @@ int bpf_prog_stream_read(int prog_fd, __u32 stream_id, void *buf, __u32 buf_len,
 	err = sys_bpf(BPF_PROG_STREAM_READ_BY_FD, &attr, attr_sz);
 	return libbpf_err_errno(err);
 }
+
+int bpf_prog_assoc_struct_ops(int prog_fd, int map_fd,
+			      struct bpf_prog_assoc_struct_ops_opts *opts)
+{
+	const size_t attr_sz = offsetofend(union bpf_attr, prog_assoc_struct_ops);
+	union bpf_attr attr;
+	int err;
+
+	if (!OPTS_VALID(opts, bpf_prog_assoc_struct_ops_opts))
+		return libbpf_err(-EINVAL);
+
+	memset(&attr, 0, attr_sz);
+	attr.prog_assoc_struct_ops.map_fd = map_fd;
+	attr.prog_assoc_struct_ops.prog_fd = prog_fd;
+	attr.prog_assoc_struct_ops.flags = OPTS_GET(opts, flags, 0);
+
+	err = sys_bpf(BPF_PROG_ASSOC_STRUCT_OPS, &attr, attr_sz);
+	return libbpf_err_errno(err);
+}
diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h
index e983a3e40d61..1f9c28d27795 100644
--- a/tools/lib/bpf/bpf.h
+++ b/tools/lib/bpf/bpf.h
@@ -733,6 +733,27 @@ struct bpf_prog_stream_read_opts {
 LIBBPF_API int bpf_prog_stream_read(int prog_fd, __u32 stream_id, void *buf, __u32 buf_len,
 				    struct bpf_prog_stream_read_opts *opts);
 
+struct bpf_prog_assoc_struct_ops_opts {
+	size_t sz;
+	__u32 flags;
+	size_t :0;
+};
+#define bpf_prog_assoc_struct_ops_opts__last_field flags
+
+/**
+ * @brief **bpf_prog_assoc_struct_ops** associates a BPF program with a
+ * struct_ops map.
+ *
+ * @param prog_fd FD for the BPF program
+ * @param map_fd FD for the struct_ops map to be associated with the BPF program
+ * @param opts optional options, can be NULL
+ *
+ * @return 0 on success; negative error code, otherwise (errno is also set to
+ * the error code)
+ */
+LIBBPF_API int bpf_prog_assoc_struct_ops(int prog_fd, int map_fd,
+					 struct bpf_prog_assoc_struct_ops_opts *opts);
+
 #ifdef __cplusplus
 } /* extern "C" */
 #endif
diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c
index 84a4b0abc8be..b136572e889a 100644
--- a/tools/lib/bpf/btf.c
+++ b/tools/lib/bpf/btf.c
@@ -4431,11 +4431,14 @@ static bool btf_dedup_identical_types(struct btf_dedup *d, __u32 id1, __u32 id2,
 	struct btf_type *t1, *t2;
 	int k1, k2;
 recur:
-	if (depth <= 0)
-		return false;
-
 	t1 = btf_type_by_id(d->btf, id1);
 	t2 = btf_type_by_id(d->btf, id2);
+	if (depth <= 0) {
+		pr_debug("Reached depth limit for identical type comparison for '%s'/'%s'\n",
+			 btf__name_by_offset(d->btf, t1->name_off),
+			 btf__name_by_offset(d->btf, t2->name_off));
+		return false;
+	}
 
 	k1 = btf_kind(t1);
 	k2 = btf_kind(t2);
@@ -4497,8 +4500,16 @@ static bool btf_dedup_identical_types(struct btf_dedup *d, __u32 id1, __u32 id2,
 		for (i = 0, n = btf_vlen(t1); i < n; i++, m1++, m2++) {
 			if (m1->type == m2->type)
 				continue;
-			if (!btf_dedup_identical_types(d, m1->type, m2->type, depth - 1))
+			if (!btf_dedup_identical_types(d, m1->type, m2->type, depth - 1)) {
+				if (t1->name_off) {
+					pr_debug("%s '%s' size=%d vlen=%d id1[%u] id2[%u] shallow-equal but not identical for field#%d '%s'\n",
+						 k1 == BTF_KIND_STRUCT ? "STRUCT" : "UNION",
+						 btf__name_by_offset(d->btf, t1->name_off),
+						 t1->size, btf_vlen(t1), id1, id2, i,
+						 btf__name_by_offset(d->btf, m1->name_off));
+				}
 				return false;
+			}
 		}
 		return true;
 	}
@@ -4739,8 +4750,16 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
 		canon_m = btf_members(canon_type);
 		for (i = 0; i < vlen; i++) {
 			eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type);
-			if (eq <= 0)
+			if (eq <= 0) {
+				if (cand_type->name_off) {
+					pr_debug("%s '%s' size=%d vlen=%d cand_id[%u] canon_id[%u] shallow-equal but not equiv for field#%d '%s': %d\n",
+						 cand_kind == BTF_KIND_STRUCT ? "STRUCT" : "UNION",
+						 btf__name_by_offset(d->btf, cand_type->name_off),
+						 cand_type->size, vlen, cand_id, canon_id, i,
+						 btf__name_by_offset(d->btf, cand_m->name_off), eq);
+				}
 				return eq;
+			}
 			cand_m++;
 			canon_m++;
 		}
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 3dc8a8078815..1a52d818a76c 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -380,7 +380,7 @@ struct reloc_desc {
 		const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */
 		struct {
 			int map_idx;
-			int sym_off;
+			unsigned int sym_off;
 			/*
 			 * The following two fields can be unionized, as the
 			 * ext_idx field is used for extern symbols, and the
@@ -757,13 +757,14 @@ struct bpf_object {
 	int arena_map_idx;
 	void *arena_data;
 	size_t arena_data_sz;
+	size_t arena_data_off;
 
 	void *jumptables_data;
 	size_t jumptables_data_sz;
 
 	struct {
 		struct bpf_program *prog;
-		int sym_off;
+		unsigned int sym_off;
 		int fd;
 	} *jumptable_maps;
 	size_t jumptable_map_cnt;
@@ -2991,10 +2992,11 @@ static int init_arena_map_data(struct bpf_object *obj, struct bpf_map *map,
 			       void *data, size_t data_sz)
 {
 	const long page_sz = sysconf(_SC_PAGE_SIZE);
+	const size_t data_alloc_sz = roundup(data_sz, page_sz);
 	size_t mmap_sz;
 
 	mmap_sz = bpf_map_mmap_sz(map);
-	if (roundup(data_sz, page_sz) > mmap_sz) {
+	if (data_alloc_sz > mmap_sz) {
 		pr_warn("elf: sec '%s': declared ARENA map size (%zu) is too small to hold global __arena variables of size %zu\n",
 			sec_name, mmap_sz, data_sz);
 		return -E2BIG;
@@ -3006,6 +3008,9 @@ static int init_arena_map_data(struct bpf_object *obj, struct bpf_map *map,
 	memcpy(obj->arena_data, data, data_sz);
 	obj->arena_data_sz = data_sz;
 
+	/* place globals at the end of the arena */
+	obj->arena_data_off = mmap_sz - data_alloc_sz;
+
 	/* make bpf_map__init_value() work for ARENA maps */
 	map->mmaped = obj->arena_data;
 
@@ -4663,7 +4668,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog,
 		reloc_desc->type = RELO_DATA;
 		reloc_desc->insn_idx = insn_idx;
 		reloc_desc->map_idx = obj->arena_map_idx;
-		reloc_desc->sym_off = sym->st_value;
+		reloc_desc->sym_off = sym->st_value + obj->arena_data_off;
 
 		map = &obj->maps[obj->arena_map_idx];
 		pr_debug("prog '%s': found arena map %d (%s, sec %d, off %zu) for insn %u\n",
@@ -5624,7 +5629,8 @@ bpf_object__create_maps(struct bpf_object *obj)
 					return err;
 				}
 				if (obj->arena_data) {
-					memcpy(map->mmaped, obj->arena_data, obj->arena_data_sz);
+					memcpy(map->mmaped + obj->arena_data_off, obj->arena_data,
+						obj->arena_data_sz);
 					zfree(&obj->arena_data);
 				}
 			}
@@ -6192,7 +6198,7 @@ static void poison_kfunc_call(struct bpf_program *prog, int relo_idx,
 	insn->imm = POISON_CALL_KFUNC_BASE + ext_idx;
 }
 
-static int find_jt_map(struct bpf_object *obj, struct bpf_program *prog, int sym_off)
+static int find_jt_map(struct bpf_object *obj, struct bpf_program *prog, unsigned int sym_off)
 {
 	size_t i;
 
@@ -6210,7 +6216,7 @@ static int find_jt_map(struct bpf_object *obj, struct bpf_program *prog, int sym
 	return -ENOENT;
 }
 
-static int add_jt_map(struct bpf_object *obj, struct bpf_program *prog, int sym_off, int map_fd)
+static int add_jt_map(struct bpf_object *obj, struct bpf_program *prog, unsigned int sym_off, int map_fd)
 {
 	size_t cnt = obj->jumptable_map_cnt;
 	size_t size = sizeof(obj->jumptable_maps[0]);
@@ -6244,7 +6250,7 @@ static int find_subprog_idx(struct bpf_program *prog, int insn_idx)
 static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struct reloc_desc *relo)
 {
 	const __u32 jt_entry_size = 8;
-	int sym_off = relo->sym_off;
+	unsigned int sym_off = relo->sym_off;
 	int jt_size = relo->sym_size;
 	__u32 max_entries = jt_size / jt_entry_size;
 	__u32 value_size = sizeof(struct bpf_insn_array_value);
@@ -6260,7 +6266,7 @@ static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struc
 		return map_fd;
 
 	if (sym_off % jt_entry_size) {
-		pr_warn("map '.jumptables': jumptable start %d should be multiple of %u\n",
+		pr_warn("map '.jumptables': jumptable start %u should be multiple of %u\n",
 			sym_off, jt_entry_size);
 		return -EINVAL;
 	}
@@ -6316,7 +6322,7 @@ static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struc
 		 * should contain values that fit in u32.
 		 */
 		if (insn_off > UINT32_MAX) {
-			pr_warn("map '.jumptables': invalid jump table value 0x%llx at offset %d\n",
+			pr_warn("map '.jumptables': invalid jump table value 0x%llx at offset %u\n",
 				(long long)jt[i], sym_off + i * jt_entry_size);
 			err = -EINVAL;
 			goto err_close;
@@ -8484,7 +8490,7 @@ static int kallsyms_cb(unsigned long long sym_addr, char sym_type,
 	struct bpf_object *obj = ctx;
 	const struct btf_type *t;
 	struct extern_desc *ext;
-	char *res;
+	const char *res;
 
 	res = strstr(sym_name, ".llvm.");
 	if (sym_type == 'd' && res)
@@ -11818,7 +11824,8 @@ static int avail_kallsyms_cb(unsigned long long sym_addr, char sym_type,
 		 *
 		 *   [0] fb6a421fb615 ("kallsyms: Match symbols exactly with CONFIG_LTO_CLANG")
 		 */
-		char sym_trim[256], *psym_trim = sym_trim, *sym_sfx;
+		char sym_trim[256], *psym_trim = sym_trim;
+		const char *sym_sfx;
 
 		if (!(sym_sfx = strstr(sym_name, ".llvm.")))
 			return 0;
@@ -12401,7 +12408,7 @@ static int resolve_full_path(const char *file, char *result, size_t result_sz)
 		if (!search_paths[i])
 			continue;
 		for (s = search_paths[i]; s != NULL; s = strchr(s, ':')) {
-			char *next_path;
+			const char *next_path;
 			int seg_len;
 
 			if (s[0] == ':')
@@ -14133,6 +14140,37 @@ int bpf_program__set_attach_target(struct bpf_program *prog,
 	return 0;
 }
 
+int bpf_program__assoc_struct_ops(struct bpf_program *prog, struct bpf_map *map,
+				  struct bpf_prog_assoc_struct_ops_opts *opts)
+{
+	int prog_fd, map_fd;
+
+	prog_fd = bpf_program__fd(prog);
+	if (prog_fd < 0) {
+		pr_warn("prog '%s': can't associate BPF program without FD (was it loaded?)\n",
+			prog->name);
+		return libbpf_err(-EINVAL);
+	}
+
+	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) {
+		pr_warn("prog '%s': can't associate struct_ops program\n", prog->name);
+		return libbpf_err(-EINVAL);
+	}
+
+	map_fd = bpf_map__fd(map);
+	if (map_fd < 0) {
+		pr_warn("map '%s': can't associate BPF map without FD (was it created?)\n", map->name);
+		return libbpf_err(-EINVAL);
+	}
+
+	if (!bpf_map__is_struct_ops(map)) {
+		pr_warn("map '%s': can't associate non-struct_ops map\n", map->name);
+		return libbpf_err(-EINVAL);
+	}
+
+	return bpf_prog_assoc_struct_ops(prog_fd, map_fd, opts);
+}
+
 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
 {
 	int err = 0, n, len, start, end = -1;
@@ -14398,7 +14436,10 @@ int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
 		if (!map_skel->mmaped)
 			continue;
 
-		*map_skel->mmaped = map->mmaped;
+		if (map->def.type == BPF_MAP_TYPE_ARENA)
+			*map_skel->mmaped = map->mmaped + map->obj->arena_data_off;
+		else
+			*map_skel->mmaped = map->mmaped;
 	}
 
 	return 0;
diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h
index 65e68e964b89..e14d9e349f9c 100644
--- a/tools/lib/bpf/libbpf.h
+++ b/tools/lib/bpf/libbpf.h
@@ -1006,6 +1006,22 @@ LIBBPF_API int
 bpf_program__set_attach_target(struct bpf_program *prog, int attach_prog_fd,
 			       const char *attach_func_name);
 
+struct bpf_prog_assoc_struct_ops_opts; /* defined in bpf.h */
+
+/**
+ * @brief **bpf_program__assoc_struct_ops()** associates a BPF program with a
+ * struct_ops map.
+ *
+ * @param prog BPF program
+ * @param map struct_ops map to be associated with the BPF program
+ * @param opts optional options, can be NULL
+ *
+ * @return 0, on success; negative error code, otherwise
+ */
+LIBBPF_API int
+bpf_program__assoc_struct_ops(struct bpf_program *prog, struct bpf_map *map,
+			      struct bpf_prog_assoc_struct_ops_opts *opts);
+
 /**
  * @brief **bpf_object__find_map_by_name()** returns BPF map of
  * the given name, if it exists within the passed BPF object
diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map
index 8ed8749907d4..84fb90a016c9 100644
--- a/tools/lib/bpf/libbpf.map
+++ b/tools/lib/bpf/libbpf.map
@@ -451,4 +451,6 @@ LIBBPF_1.7.0 {
 	global:
 		bpf_map__set_exclusive_program;
 		bpf_map__exclusive_program;
+		bpf_prog_assoc_struct_ops;
+		bpf_program__assoc_struct_ops;
 } LIBBPF_1.6.0;
diff --git a/tools/mm/page_owner_sort.c b/tools/mm/page_owner_sort.c
index 14c67e9e84c4..e6954909401c 100644
--- a/tools/mm/page_owner_sort.c
+++ b/tools/mm/page_owner_sort.c
@@ -181,7 +181,11 @@ static int compare_ts(const void *p1, const void *p2)
 {
 	const struct block_list *l1 = p1, *l2 = p2;
 
-	return l1->ts_nsec < l2->ts_nsec ? -1 : 1;
+	if (l1->ts_nsec < l2->ts_nsec)
+		return -1;
+	if (l1->ts_nsec > l2->ts_nsec)
+		return 1;
+	return 0;
 }
 
 static int compare_cull_condition(const void *p1, const void *p2)
diff --git a/tools/net/ynl/Makefile.deps b/tools/net/ynl/Makefile.deps
index 865fd2e8519e..08205f9fc525 100644
--- a/tools/net/ynl/Makefile.deps
+++ b/tools/net/ynl/Makefile.deps
@@ -13,6 +13,7 @@ UAPI_PATH:=../../../../include/uapi/
 # need the explicit -D matching what's in /usr, to avoid multiple definitions.
 
 get_hdr_inc=-D$(1) -include $(UAPI_PATH)/linux/$(2)
+get_hdr_inc2=-D$(1) -D$(2) -include $(UAPI_PATH)/linux/$(3)
 
 CFLAGS_devlink:=$(call get_hdr_inc,_LINUX_DEVLINK_H_,devlink.h)
 CFLAGS_dpll:=$(call get_hdr_inc,_LINUX_DPLL_H,dpll.h)
@@ -48,3 +49,4 @@ CFLAGS_tc:= $(call get_hdr_inc,__LINUX_RTNETLINK_H,rtnetlink.h) \
 	$(call get_hdr_inc,_TC_SKBEDIT_H,tc_act/tc_skbedit.h) \
 	$(call get_hdr_inc,_TC_TUNNEL_KEY_H,tc_act/tc_tunnel_key.h)
 CFLAGS_tcp_metrics:=$(call get_hdr_inc,_LINUX_TCP_METRICS_H,tcp_metrics.h)
+CFLAGS_wireguard:=$(call get_hdr_inc2,_LINUX_WIREGUARD_H,_WG_UAPI_WIREGUARD_H,wireguard.h)
diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config
index bd9f4804d56b..d8d25f62aaad 100644
--- a/tools/perf/Makefile.config
+++ b/tools/perf/Makefile.config
@@ -701,6 +701,11 @@ ifndef NO_LIBBPF
   endif
 endif
 
+ifeq ($(feature-libopenssl), 1)
+  $(call detected,CONFIG_LIBOPENSSL)
+  CFLAGS += -DHAVE_LIBOPENSSL_SUPPORT
+endif
+
 ifndef BUILD_BPF_SKEL
   # BPF skeletons control a large number of perf features, by default
   # they are enabled.
@@ -717,6 +722,9 @@ ifeq ($(BUILD_BPF_SKEL),1)
   else ifeq ($(filter -DHAVE_LIBBPF_SUPPORT, $(CFLAGS)),)
     $(warning Warning: Disabled BPF skeletons as libbpf is required)
     BUILD_BPF_SKEL := 0
+  else ifeq ($(filter -DHAVE_LIBOPENSSL_SUPPORT, $(CFLAGS)),)
+    $(warning Warning: Disabled BPF skeletons as libopenssl is required)
+    BUILD_BPF_SKEL := 0
   else ifeq ($(call get-executable,$(CLANG)),)
     $(warning Warning: Disabled BPF skeletons as clang ($(CLANG)) is missing)
     BUILD_BPF_SKEL := 0
diff --git a/tools/power/cpupower/lib/cpuidle.c b/tools/power/cpupower/lib/cpuidle.c
index f2c1139adf71..2fcb343d8e75 100644
--- a/tools/power/cpupower/lib/cpuidle.c
+++ b/tools/power/cpupower/lib/cpuidle.c
@@ -150,6 +150,7 @@ unsigned long long cpuidle_state_get_one_value(unsigned int cpu,
 	if (len == 0)
 		return 0;
 
+	errno = 0;
 	value = strtoull(linebuf, &endp, 0);
 
 	if (endp == linebuf || errno == ERANGE)
@@ -193,8 +194,7 @@ static char *cpuidle_state_get_one_string(unsigned int cpu,
 	if (result == NULL)
 		return NULL;
 
-	if (result[strlen(result) - 1] == '\n')
-		result[strlen(result) - 1] = '\0';
+	result[strcspn(result, "\n")] = '\0';
 
 	return result;
 }
@@ -366,8 +366,7 @@ static char *sysfs_cpuidle_get_one_string(enum cpuidle_string which)
 	if (result == NULL)
 		return NULL;
 
-	if (result[strlen(result) - 1] == '\n')
-		result[strlen(result) - 1] = '\0';
+	result[strcspn(result, "\n")] = '\0';
 
 	return result;
 }
diff --git a/tools/power/cpupower/utils/cpufreq-info.c b/tools/power/cpupower/utils/cpufreq-info.c
index 7d3732f5f2f6..5fe01e516817 100644
--- a/tools/power/cpupower/utils/cpufreq-info.c
+++ b/tools/power/cpupower/utils/cpufreq-info.c
@@ -270,7 +270,7 @@ static int get_freq_hardware(unsigned int cpu, unsigned int human)
 {
 	unsigned long freq;
 
-	if (cpupower_cpu_info.caps & CPUPOWER_CAP_APERF)
+	if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_APERF))
 		return -EINVAL;
 
 	freq = cpufreq_get_freq_hardware(cpu);
diff --git a/tools/power/cpupower/utils/cpuidle-info.c b/tools/power/cpupower/utils/cpuidle-info.c
index e0d17f0de3fe..81b4763a97d6 100644
--- a/tools/power/cpupower/utils/cpuidle-info.c
+++ b/tools/power/cpupower/utils/cpuidle-info.c
@@ -111,7 +111,7 @@ static void proc_cpuidle_cpu_output(unsigned int cpu)
 	printf(_("max_cstate:              C%u\n"), cstates-1);
 	printf(_("maximum allowed latency: %lu usec\n"), max_allowed_cstate);
 	printf(_("states:\t\n"));
-	for (cstate = 1; cstate < cstates; cstate++) {
+	for (cstate = 0; cstate < cstates; cstate++) {
 		printf(_("    C%d:                  "
 			 "type[C%d] "), cstate, cstate);
 		printf(_("promotion[--] demotion[--] "));
diff --git a/tools/power/cpupower/utils/idle_monitor/cpuidle_sysfs.c b/tools/power/cpupower/utils/idle_monitor/cpuidle_sysfs.c
index 8b42c2f0a5b0..4225eff9833d 100644
--- a/tools/power/cpupower/utils/idle_monitor/cpuidle_sysfs.c
+++ b/tools/power/cpupower/utils/idle_monitor/cpuidle_sysfs.c
@@ -70,7 +70,7 @@ static int cpuidle_stop(void)
 			current_count[cpu][state] =
 				cpuidle_state_time(cpu, state);
 			dprint("CPU %d - State: %d - Val: %llu\n",
-			       cpu, state, previous_count[cpu][state]);
+			       cpu, state, current_count[cpu][state]);
 		}
 	}
 	return 0;
diff --git a/tools/power/x86/turbostat/turbostat.8 b/tools/power/x86/turbostat/turbostat.8
index 1551fcdbfd8a..8ddda1603ae0 100644
--- a/tools/power/x86/turbostat/turbostat.8
+++ b/tools/power/x86/turbostat/turbostat.8
@@ -111,6 +111,8 @@ The column name "all" can be used to enable all disabled-by-default built-in cou
 .PP
 \fB--no-perf\fP Disable all the uses of the perf API.
 .PP
+\fB--force\fPForce turbostat to run on an unsupported platform (minimal defaults).
+.PP
 \fB--interval seconds\fP overrides the default 5.0 second measurement interval.
 .PP
 \fB--num_iterations num\fP number of the measurement iterations.
@@ -559,6 +561,8 @@ If the upstream version isn't new enough, the development tree can be found here
 If the development tree doesn't work, please contact the author via chat,
 or via email with the word "turbostat" on the Subject line.
 
+An old turbostat binary may run on unknown hardware by using "--force",
+but results are unsupported.
 .SH FILES
 .ta
 .nf
diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c
index 5ad45c2ac5bd..2ffe6e9710cc 100644
--- a/tools/power/x86/turbostat/turbostat.c
+++ b/tools/power/x86/turbostat/turbostat.c
@@ -498,6 +498,7 @@ unsigned int list_header_only;
 unsigned int dump_only;
 unsigned int force_load;
 unsigned int cpuid_has_aperf_mperf;
+unsigned int cpuid_has_hv;
 unsigned int has_aperf_access;
 unsigned int has_epb;
 unsigned int has_turbo;
@@ -3001,22 +3002,33 @@ void print_header(char *delim)
 }
 
 /*
- * pct()
+ * pct(numerator, denominator)
  *
- * If absolute value is < 1.1, return percentage
- * otherwise, return nan
+ * Return sanity checked percentage (100.0 * numerator/denominotor)
  *
- * return value is appropriate for printing percentages with %f
- * while flagging some obvious erroneous values.
+ * 0/0: 0
+ * n/0: nan
+ * n/d > 1.1: nan
  */
-double pct(double d)
+double pct(double numerator, double denominator)
 {
+	double retval;
 
-	double abs = fabs(d);
+	if (numerator == 0)
+		return 0;
+
+	if (numerator < 0)
+		return nan("");
+
+	if (denominator <= 0)
+		return nan("");
+
+	retval = 100.0 * numerator / denominator;
 
-	if (abs < 1.10)
-		return (100.0 * d);
-	return nan("");
+	if (retval > 110.0)
+		return nan("");
+
+	return retval;
 }
 
 int dump_counters(PER_THREAD_PARAMS)
@@ -3046,7 +3058,7 @@ int dump_counters(PER_THREAD_PARAMS)
 
 		outp += sprintf(outp, "LLC refs: %lld", t->llc.references);
 		outp += sprintf(outp, "LLC miss: %lld", t->llc.misses);
-		outp += sprintf(outp, "LLC Hit%%: %.2f", pct((t->llc.references - t->llc.misses) / t->llc.references));
+		outp += sprintf(outp, "LLC Hit%%: %.2f", pct((t->llc.references - t->llc.misses), t->llc.references));
 
 		for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
 			outp += sprintf(outp, "tADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, t->counter[i], mp->sp->path);
@@ -3261,7 +3273,7 @@ int format_counters(PER_THREAD_PARAMS)
 		outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 / units * t->aperf / interval_float);
 
 	if (DO_BIC(BIC_Busy))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->mperf / tsc));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->mperf, tsc));
 
 	if (DO_BIC(BIC_Bzy_MHz)) {
 		if (has_base_hz)
@@ -3302,7 +3314,7 @@ int format_counters(PER_THREAD_PARAMS)
 			outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), t->llc.references / interval_float / 1000);
 
 		if (DO_BIC(BIC_LLC_HIT))
-			outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct((t->llc.references - t->llc.misses) / t->llc.references));
+			outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct((t->llc.references - t->llc.misses), t->llc.references));
 	}
 
 	/* Added Thread Counters */
@@ -3315,7 +3327,7 @@ int format_counters(PER_THREAD_PARAMS)
 			if (mp->type == COUNTER_USEC)
 				outp += print_float_value(&printed, delim, t->counter[i] / interval_float / 10000);
 			else
-				outp += print_float_value(&printed, delim, pct(t->counter[i] / tsc));
+				outp += print_float_value(&printed, delim, pct(t->counter[i], tsc));
 		}
 	}
 
@@ -3329,7 +3341,7 @@ int format_counters(PER_THREAD_PARAMS)
 			if (pp->type == COUNTER_USEC)
 				outp += print_float_value(&printed, delim, t->perf_counter[i] / interval_float / 10000);
 			else
-				outp += print_float_value(&printed, delim, pct(t->perf_counter[i] / tsc));
+				outp += print_float_value(&printed, delim, pct(t->perf_counter[i], tsc));
 		}
 	}
 
@@ -3343,34 +3355,34 @@ int format_counters(PER_THREAD_PARAMS)
 			break;
 
 		case PMT_TYPE_XTAL_TIME:
-			value_converted = pct(value_raw / crystal_hz / interval_float);
+			value_converted = pct(value_raw / crystal_hz, interval_float);
 			outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted);
 			break;
 
 		case PMT_TYPE_TCORE_CLOCK:
-			value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float);
+			value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float);
 			outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted);
 		}
 	}
 
 	/* C1 */
 	if (DO_BIC(BIC_CPU_c1))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->c1 / tsc));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->c1, tsc));
 
 	/* print per-core data only for 1st thread in core */
 	if (!is_cpu_first_thread_in_core(t, c))
 		goto done;
 
 	if (DO_BIC(BIC_CPU_c3))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c3 / tsc));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c3, tsc));
 	if (DO_BIC(BIC_CPU_c6))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c6 / tsc));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c6, tsc));
 	if (DO_BIC(BIC_CPU_c7))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c7 / tsc));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c7, tsc));
 
 	/* Mod%c6 */
 	if (DO_BIC(BIC_Mod_c6))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->mc6_us / tsc));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->mc6_us, tsc));
 
 	if (DO_BIC(BIC_CoreTmp))
 		outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_temp_c);
@@ -3386,7 +3398,7 @@ int format_counters(PER_THREAD_PARAMS)
 		else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
 			outp += print_decimal_value(mp->width, &printed, delim, c->counter[i]);
 		else if (mp->format == FORMAT_PERCENT)
-			outp += print_float_value(&printed, delim, pct(c->counter[i] / tsc));
+			outp += print_float_value(&printed, delim, pct(c->counter[i], tsc));
 	}
 
 	/* Added perf Core counters */
@@ -3396,7 +3408,7 @@ int format_counters(PER_THREAD_PARAMS)
 		else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
 			outp += print_decimal_value(pp->width, &printed, delim, c->perf_counter[i]);
 		else if (pp->format == FORMAT_PERCENT)
-			outp += print_float_value(&printed, delim, pct(c->perf_counter[i] / tsc));
+			outp += print_float_value(&printed, delim, pct(c->perf_counter[i], tsc));
 	}
 
 	/* Added PMT Core counters */
@@ -3409,12 +3421,12 @@ int format_counters(PER_THREAD_PARAMS)
 			break;
 
 		case PMT_TYPE_XTAL_TIME:
-			value_converted = pct(value_raw / crystal_hz / interval_float);
+			value_converted = pct(value_raw / crystal_hz, interval_float);
 			outp += print_float_value(&printed, delim, value_converted);
 			break;
 
 		case PMT_TYPE_TCORE_CLOCK:
-			value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float);
+			value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float);
 			outp += print_float_value(&printed, delim, value_converted);
 		}
 	}
@@ -3470,39 +3482,39 @@ int format_counters(PER_THREAD_PARAMS)
 	if (DO_BIC(BIC_Totl_c0))
 		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100 * p->pkg_wtd_core_c0 / tsc);	/* can exceed 100% */
 	if (DO_BIC(BIC_Any_c0))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_core_c0 / tsc));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_core_c0, tsc));
 	if (DO_BIC(BIC_GFX_c0))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_gfxe_c0 / tsc));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_gfxe_c0, tsc));
 	if (DO_BIC(BIC_CPUGFX))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_both_core_gfxe_c0 / tsc));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_both_core_gfxe_c0, tsc));
 
 	if (DO_BIC(BIC_Pkgpc2))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc2 / tsc));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc2, tsc));
 	if (DO_BIC(BIC_Pkgpc3))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc3 / tsc));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc3, tsc));
 	if (DO_BIC(BIC_Pkgpc6))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc6 / tsc));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc6, tsc));
 	if (DO_BIC(BIC_Pkgpc7))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc7 / tsc));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc7, tsc));
 	if (DO_BIC(BIC_Pkgpc8))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc8 / tsc));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc8, tsc));
 	if (DO_BIC(BIC_Pkgpc9))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc9 / tsc));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc9, tsc));
 	if (DO_BIC(BIC_Pkgpc10))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc10 / tsc));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc10, tsc));
 
 	if (DO_BIC(BIC_Diec6))
-		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->die_c6 / crystal_hz / interval_float));
+		outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->die_c6 / crystal_hz, interval_float));
 
 	if (DO_BIC(BIC_CPU_LPI)) {
 		if (p->cpu_lpi >= 0)
-			outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->cpu_lpi / 1000000.0 / interval_float));
+			outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->cpu_lpi / 1000000.0, interval_float));
 		else
 			outp += sprintf(outp, "%s(neg)", (printed++ ? delim : ""));
 	}
 	if (DO_BIC(BIC_SYS_LPI)) {
 		if (p->sys_lpi >= 0)
-			outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->sys_lpi / 1000000.0 / interval_float));
+			outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->sys_lpi / 1000000.0, interval_float));
 		else
 			outp += sprintf(outp, "%s(neg)", (printed++ ? delim : ""));
 	}
@@ -3542,7 +3554,7 @@ int format_counters(PER_THREAD_PARAMS)
 		else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
 			outp += print_decimal_value(mp->width, &printed, delim, p->counter[i]);
 		else if (mp->format == FORMAT_PERCENT)
-			outp += print_float_value(&printed, delim, pct(p->counter[i] / tsc));
+			outp += print_float_value(&printed, delim, pct(p->counter[i], tsc));
 	}
 
 	/* Added perf Package Counters */
@@ -3554,7 +3566,7 @@ int format_counters(PER_THREAD_PARAMS)
 		else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
 			outp += print_decimal_value(pp->width, &printed, delim, p->perf_counter[i]);
 		else if (pp->format == FORMAT_PERCENT)
-			outp += print_float_value(&printed, delim, pct(p->perf_counter[i] / tsc));
+			outp += print_float_value(&printed, delim, pct(p->perf_counter[i], tsc));
 	}
 
 	/* Added PMT Package Counters */
@@ -3567,12 +3579,12 @@ int format_counters(PER_THREAD_PARAMS)
 			break;
 
 		case PMT_TYPE_XTAL_TIME:
-			value_converted = pct(value_raw / crystal_hz / interval_float);
+			value_converted = pct(value_raw / crystal_hz, interval_float);
 			outp += print_float_value(&printed, delim, value_converted);
 			break;
 
 		case PMT_TYPE_TCORE_CLOCK:
-			value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float);
+			value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float);
 			outp += print_float_value(&printed, delim, value_converted);
 		}
 	}
@@ -8771,6 +8783,27 @@ void probe_pstates(void)
 	for_all_cpus(print_epb, ODD_COUNTERS);
 	for_all_cpus(print_perf_limit, ODD_COUNTERS);
 }
+void dump_word_chars(unsigned int word)
+{
+	int i;
+
+	for (i = 0; i < 4; ++i)
+		fprintf(outf, "%c", (word >> (i * 8)) & 0xFF);
+}
+void dump_cpuid_hypervisor(void)
+{
+	unsigned int ebx = 0;
+	unsigned int ecx = 0;
+	unsigned int edx = 0;
+
+	__cpuid(0x40000000, max_extended_level, ebx, ecx, edx);
+
+	fprintf(outf, "Hypervisor: ");
+	dump_word_chars(ebx);
+	dump_word_chars(ecx);
+	dump_word_chars(edx);
+	fprintf(outf, "\n");
+}
 
 void process_cpuid()
 {
@@ -8803,6 +8836,7 @@ void process_cpuid()
 		model += ((fms >> 16) & 0xf) << 4;
 	ecx_flags = ecx;
 	edx_flags = edx;
+	cpuid_has_hv = ecx_flags & (1 << 31);
 
 	if (!no_msr) {
 		if (get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch))
@@ -8826,16 +8860,19 @@ void process_cpuid()
 		fputc('\n', outf);
 
 		fprintf(outf, "CPUID(0x80000000): max_extended_levels: 0x%x\n", max_extended_level);
-		fprintf(outf, "CPUID(1): %s %s %s %s %s %s %s %s %s %s\n",
-			ecx_flags & (1 << 0) ? "SSE3" : "-",
-			ecx_flags & (1 << 3) ? "MONITOR" : "-",
-			ecx_flags & (1 << 6) ? "SMX" : "-",
-			ecx_flags & (1 << 7) ? "EIST" : "-",
-			ecx_flags & (1 << 8) ? "TM2" : "-",
-			edx_flags & (1 << 4) ? "TSC" : "-",
-			edx_flags & (1 << 5) ? "MSR" : "-",
-			edx_flags & (1 << 22) ? "ACPI-TM" : "-", edx_flags & (1 << 28) ? "HT" : "-", edx_flags & (1 << 29) ? "TM" : "-");
-	}
+		fprintf(outf, "CPUID(1): %sSSE3 %sMONITOR %sSMX %sEIST %sTM2 %sHV %sTSC %sMSR %sACPI-TM %sHT %sTM\n",
+			ecx_flags & (1 << 0) ? "" : "No-",
+			ecx_flags & (1 << 3) ? "" : "No-",
+			ecx_flags & (1 << 6) ? "" : "No-",
+			ecx_flags & (1 << 7) ? "" : "No-",
+			ecx_flags & (1 << 8) ? "" : "No-",
+			cpuid_has_hv ? "" : "No-",
+			edx_flags & (1 << 4) ? "" : "No-",
+			edx_flags & (1 << 5) ? "" : "No-",
+			edx_flags & (1 << 22) ? "" : "No-", edx_flags & (1 << 28) ? "" : "No-", edx_flags & (1 << 29) ? "" : "No-");
+	}
+	if (!quiet && cpuid_has_hv)
+		dump_cpuid_hypervisor();
 
 	probe_platform_features(family, model);
 
@@ -10145,7 +10182,7 @@ int get_and_dump_counters(void)
 
 void print_version()
 {
-	fprintf(outf, "turbostat version 2025.12.02 - Len Brown <lenb@kernel.org>\n");
+	fprintf(outf, "turbostat version 2025.12.08 - Len Brown <lenb@kernel.org>\n");
 }
 
 #define COMMAND_LINE_SIZE 2048
diff --git a/tools/testing/radix-tree/idr-test.c b/tools/testing/radix-tree/idr-test.c
index 2f830ff8396c..945144e98507 100644
--- a/tools/testing/radix-tree/idr-test.c
+++ b/tools/testing/radix-tree/idr-test.c
@@ -57,6 +57,26 @@ void idr_alloc_test(void)
 	idr_destroy(&idr);
 }
 
+void idr_alloc2_test(void)
+{
+	int id;
+	struct idr idr = IDR_INIT_BASE(idr, 1);
+
+	id = idr_alloc(&idr, idr_alloc2_test, 0, 1, GFP_KERNEL);
+	assert(id == -ENOSPC);
+
+	id = idr_alloc(&idr, idr_alloc2_test, 1, 2, GFP_KERNEL);
+	assert(id == 1);
+
+	id = idr_alloc(&idr, idr_alloc2_test, 0, 1, GFP_KERNEL);
+	assert(id == -ENOSPC);
+
+	id = idr_alloc(&idr, idr_alloc2_test, 0, 2, GFP_KERNEL);
+	assert(id == -ENOSPC);
+
+	idr_destroy(&idr);
+}
+
 void idr_replace_test(void)
 {
 	DEFINE_IDR(idr);
@@ -409,6 +429,7 @@ void idr_checks(void)
 
 	idr_replace_test();
 	idr_alloc_test();
+	idr_alloc2_test();
 	idr_null_test();
 	idr_nowait_test();
 	idr_get_next_test(0);
diff --git a/tools/testing/radix-tree/xarray.c b/tools/testing/radix-tree/xarray.c
index 253208a8541b..134b316e5e0d 100644
--- a/tools/testing/radix-tree/xarray.c
+++ b/tools/testing/radix-tree/xarray.c
@@ -7,7 +7,7 @@
 #include "xarray-shared.h"
 #include "test.h"
 
-#undef XA_DEBUG
+#undef CONFIG_DEBUG_XARRAY
 #include "../../../lib/test_xarray.c"
 
 void xarray_tests(void)
diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile
index b7030a6e2e76..4aa60e83ff19 100644
--- a/tools/testing/selftests/bpf/Makefile
+++ b/tools/testing/selftests/bpf/Makefile
@@ -437,6 +437,8 @@ BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN)	\
 	     -I$(abspath $(OUTPUT)/../usr/include)			\
 	     -std=gnu11		 					\
 	     -fno-strict-aliasing 					\
+	     -Wno-microsoft-anon-tag					\
+	     -fms-extensions						\
 	     -Wno-compare-distinct-pointer-types			\
 	     -Wno-initializer-overrides					\
 	     #
diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config
index 558839e3c185..24855381290d 100644
--- a/tools/testing/selftests/bpf/config
+++ b/tools/testing/selftests/bpf/config
@@ -1,6 +1,6 @@
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y
-CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y
+CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=1
 CONFIG_BPF=y
 CONFIG_BPF_EVENTS=y
 CONFIG_BPF_JIT=y
diff --git a/tools/testing/selftests/bpf/prog_tests/d_path.c b/tools/testing/selftests/bpf/prog_tests/d_path.c
index ccc768592e66..1a2a2f1abf03 100644
--- a/tools/testing/selftests/bpf/prog_tests/d_path.c
+++ b/tools/testing/selftests/bpf/prog_tests/d_path.c
@@ -38,6 +38,14 @@ static int set_pathname(int fd, pid_t pid)
 	return readlink(buf, src.paths[src.cnt++], MAX_PATH_LEN);
 }
 
+static inline long syscall_close(int fd)
+{
+	return syscall(__NR_close_range,
+			(unsigned int)fd,
+			(unsigned int)fd,
+			0u);
+}
+
 static int trigger_fstat_events(pid_t pid)
 {
 	int sockfd = -1, procfd = -1, devfd = -1;
@@ -104,18 +112,34 @@ static int trigger_fstat_events(pid_t pid)
 	/* sys_close no longer triggers filp_close, but we can
 	 * call sys_close_range instead which still does
 	 */
-#define close(fd) syscall(__NR_close_range, fd, fd, 0)
+	syscall_close(pipefd[0]);
+	syscall_close(pipefd[1]);
+	syscall_close(sockfd);
+	syscall_close(procfd);
+	syscall_close(devfd);
+	syscall_close(localfd);
+	syscall_close(indicatorfd);
+	return ret;
+}
 
-	close(pipefd[0]);
-	close(pipefd[1]);
-	close(sockfd);
-	close(procfd);
-	close(devfd);
-	close(localfd);
-	close(indicatorfd);
+static void attach_and_load(struct test_d_path **skel)
+{
+	int err;
 
-#undef close
-	return ret;
+	*skel = test_d_path__open_and_load();
+	if (CHECK(!*skel, "setup", "d_path skeleton failed\n"))
+		goto cleanup;
+
+	err = test_d_path__attach(*skel);
+	if (CHECK(err, "setup", "attach failed: %d\n", err))
+		goto cleanup;
+
+	(*skel)->bss->my_pid = getpid();
+	return;
+
+cleanup:
+	test_d_path__destroy(*skel);
+	*skel = NULL;
 }
 
 static void test_d_path_basic(void)
@@ -124,16 +148,11 @@ static void test_d_path_basic(void)
 	struct test_d_path *skel;
 	int err;
 
-	skel = test_d_path__open_and_load();
-	if (CHECK(!skel, "setup", "d_path skeleton failed\n"))
-		goto cleanup;
-
-	err = test_d_path__attach(skel);
-	if (CHECK(err, "setup", "attach failed: %d\n", err))
+	attach_and_load(&skel);
+	if (!skel)
 		goto cleanup;
 
 	bss = skel->bss;
-	bss->my_pid = getpid();
 
 	err = trigger_fstat_events(bss->my_pid);
 	if (err < 0)
@@ -195,6 +214,39 @@ static void test_d_path_check_types(void)
 	test_d_path_check_types__destroy(skel);
 }
 
+/* Check if the verifier correctly generates code for
+ * accessing the memory modified by d_path helper.
+ */
+static void test_d_path_mem_access(void)
+{
+	int localfd = -1;
+	char path_template[] = "/dev/shm/d_path_loadgen.XXXXXX";
+	struct test_d_path__bss *bss;
+	struct test_d_path *skel;
+
+	attach_and_load(&skel);
+	if (!skel)
+		goto cleanup;
+
+	bss = skel->bss;
+
+	localfd = mkstemp(path_template);
+	if (CHECK(localfd < 0, "trigger", "mkstemp failed\n"))
+		goto cleanup;
+
+	if (CHECK(fallocate(localfd, 0, 0, 1024) < 0, "trigger", "fallocate failed\n"))
+		goto cleanup;
+	remove(path_template);
+
+	if (CHECK(!bss->path_match_fallocate, "check",
+		  "failed to read fallocate path"))
+		goto cleanup;
+
+cleanup:
+	syscall_close(localfd);
+	test_d_path__destroy(skel);
+}
+
 void test_d_path(void)
 {
 	if (test__start_subtest("basic"))
@@ -205,4 +257,7 @@ void test_d_path(void)
 
 	if (test__start_subtest("check_alloc_mem"))
 		test_d_path_check_types();
+
+	if (test__start_subtest("check_mem_access"))
+		test_d_path_mem_access();
 }
diff --git a/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c b/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c
index 6c2b0c3dbcd8..e442be9dde7e 100644
--- a/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c
+++ b/tools/testing/selftests/bpf/prog_tests/dmabuf_iter.c
@@ -73,12 +73,10 @@ static int create_udmabuf(void)
 	return -1;
 }
 
-static int create_sys_heap_dmabuf(void)
+static int create_sys_heap_dmabuf(size_t bytes)
 {
-	sysheap_test_buffer_size = 20 * getpagesize();
-
 	struct dma_heap_allocation_data data = {
-		.len = sysheap_test_buffer_size,
+		.len = bytes,
 		.fd = 0,
 		.fd_flags = O_RDWR | O_CLOEXEC,
 		.heap_flags = 0,
@@ -110,7 +108,9 @@ static int create_sys_heap_dmabuf(void)
 static int create_test_buffers(void)
 {
 	udmabuf = create_udmabuf();
-	sysheap_dmabuf = create_sys_heap_dmabuf();
+
+	sysheap_test_buffer_size = 20 * getpagesize();
+	sysheap_dmabuf = create_sys_heap_dmabuf(sysheap_test_buffer_size);
 
 	if (udmabuf < 0 || sysheap_dmabuf < 0)
 		return -1;
@@ -219,6 +219,26 @@ static void subtest_dmabuf_iter_check_default_iter(struct dmabuf_iter *skel)
 	close(iter_fd);
 }
 
+static void subtest_dmabuf_iter_check_lots_of_buffers(struct dmabuf_iter *skel)
+{
+	int iter_fd;
+	char buf[1024];
+	size_t total_bytes_read = 0;
+	ssize_t bytes_read;
+
+	iter_fd = bpf_iter_create(bpf_link__fd(skel->links.dmabuf_collector));
+	if (!ASSERT_OK_FD(iter_fd, "iter_create"))
+		return;
+
+	while ((bytes_read = read(iter_fd, buf, sizeof(buf))) > 0)
+		total_bytes_read += bytes_read;
+
+	ASSERT_GT(total_bytes_read, getpagesize(), "total_bytes_read");
+
+	close(iter_fd);
+}
+
+
 static void subtest_dmabuf_iter_check_open_coded(struct dmabuf_iter *skel, int map_fd)
 {
 	LIBBPF_OPTS(bpf_test_run_opts, topts);
@@ -275,6 +295,23 @@ void test_dmabuf_iter(void)
 		subtest_dmabuf_iter_check_no_infinite_reads(skel);
 	if (test__start_subtest("default_iter"))
 		subtest_dmabuf_iter_check_default_iter(skel);
+	if (test__start_subtest("lots_of_buffers")) {
+		size_t NUM_BUFS = 100;
+		int buffers[NUM_BUFS];
+		int i;
+
+		for (i = 0; i < NUM_BUFS; ++i) {
+			buffers[i] = create_sys_heap_dmabuf(getpagesize());
+			if (!ASSERT_OK_FD(buffers[i], "dmabuf_fd"))
+				goto cleanup_bufs;
+		}
+
+		subtest_dmabuf_iter_check_lots_of_buffers(skel);
+
+cleanup_bufs:
+		for (--i; i >= 0; --i)
+			close(buffers[i]);
+	}
 	if (test__start_subtest("open_coded"))
 		subtest_dmabuf_iter_check_open_coded(skel, map_fd);
 
diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c
new file mode 100644
index 000000000000..461ded722351
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c
@@ -0,0 +1,191 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <test_progs.h>
+#include "struct_ops_assoc.skel.h"
+#include "struct_ops_assoc_reuse.skel.h"
+#include "struct_ops_assoc_in_timer.skel.h"
+
+static void test_st_ops_assoc(void)
+{
+	struct struct_ops_assoc *skel = NULL;
+	int err, pid;
+
+	skel = struct_ops_assoc__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "struct_ops_assoc__open"))
+		goto out;
+
+	/* cannot explicitly associate struct_ops program */
+	err = bpf_program__assoc_struct_ops(skel->progs.test_1_a,
+					    skel->maps.st_ops_map_a, NULL);
+	ASSERT_ERR(err, "bpf_program__assoc_struct_ops(test_1_a, st_ops_map_a)");
+
+	err = bpf_program__assoc_struct_ops(skel->progs.syscall_prog_a,
+					    skel->maps.st_ops_map_a, NULL);
+	ASSERT_OK(err, "bpf_program__assoc_struct_ops(syscall_prog_a, st_ops_map_a)");
+
+	err = bpf_program__assoc_struct_ops(skel->progs.sys_enter_prog_a,
+					    skel->maps.st_ops_map_a, NULL);
+	ASSERT_OK(err, "bpf_program__assoc_struct_ops(sys_enter_prog_a, st_ops_map_a)");
+
+	err = bpf_program__assoc_struct_ops(skel->progs.syscall_prog_b,
+					    skel->maps.st_ops_map_b, NULL);
+	ASSERT_OK(err, "bpf_program__assoc_struct_ops(syscall_prog_b, st_ops_map_b)");
+
+	err = bpf_program__assoc_struct_ops(skel->progs.sys_enter_prog_b,
+					    skel->maps.st_ops_map_b, NULL);
+	ASSERT_OK(err, "bpf_program__assoc_struct_ops(sys_enter_prog_b, st_ops_map_b)");
+
+	/* sys_enter_prog_a already associated with map_a */
+	err = bpf_program__assoc_struct_ops(skel->progs.sys_enter_prog_a,
+					    skel->maps.st_ops_map_b, NULL);
+	ASSERT_ERR(err, "bpf_program__assoc_struct_ops(sys_enter_prog_a, st_ops_map_b)");
+
+	err = struct_ops_assoc__attach(skel);
+	if (!ASSERT_OK(err, "struct_ops_assoc__attach"))
+		goto out;
+
+	/* run tracing prog that calls .test_1 and checks return */
+	pid = getpid();
+	skel->bss->test_pid = pid;
+	sys_gettid();
+	skel->bss->test_pid = 0;
+
+	ASSERT_EQ(skel->bss->test_err_a, 0, "skel->bss->test_err_a");
+	ASSERT_EQ(skel->bss->test_err_b, 0, "skel->bss->test_err_b");
+
+	/* run syscall_prog that calls .test_1 and checks return */
+	err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.syscall_prog_a), NULL);
+	ASSERT_OK(err, "bpf_prog_test_run_opts");
+
+	err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.syscall_prog_b), NULL);
+	ASSERT_OK(err, "bpf_prog_test_run_opts");
+
+	ASSERT_EQ(skel->bss->test_err_a, 0, "skel->bss->test_err_a");
+	ASSERT_EQ(skel->bss->test_err_b, 0, "skel->bss->test_err_b");
+
+out:
+	struct_ops_assoc__destroy(skel);
+}
+
+static void test_st_ops_assoc_reuse(void)
+{
+	struct struct_ops_assoc_reuse *skel = NULL;
+	int err;
+
+	skel = struct_ops_assoc_reuse__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "struct_ops_assoc_reuse__open"))
+		goto out;
+
+	err = bpf_program__assoc_struct_ops(skel->progs.syscall_prog_a,
+					    skel->maps.st_ops_map_a, NULL);
+	ASSERT_OK(err, "bpf_program__assoc_struct_ops(syscall_prog_a, st_ops_map_a)");
+
+	err = bpf_program__assoc_struct_ops(skel->progs.syscall_prog_b,
+					    skel->maps.st_ops_map_b, NULL);
+	ASSERT_OK(err, "bpf_program__assoc_struct_ops(syscall_prog_b, st_ops_map_b)");
+
+	err = struct_ops_assoc_reuse__attach(skel);
+	if (!ASSERT_OK(err, "struct_ops_assoc__attach"))
+		goto out;
+
+	/* run syscall_prog that calls .test_1 and checks return */
+	err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.syscall_prog_a), NULL);
+	ASSERT_OK(err, "bpf_prog_test_run_opts");
+
+	err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.syscall_prog_b), NULL);
+	ASSERT_OK(err, "bpf_prog_test_run_opts");
+
+	ASSERT_EQ(skel->bss->test_err_a, 0, "skel->bss->test_err_a");
+	ASSERT_EQ(skel->bss->test_err_b, 0, "skel->bss->test_err_b");
+
+out:
+	struct_ops_assoc_reuse__destroy(skel);
+}
+
+static void test_st_ops_assoc_in_timer(void)
+{
+	struct struct_ops_assoc_in_timer *skel = NULL;
+	int err;
+
+	skel = struct_ops_assoc_in_timer__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "struct_ops_assoc_in_timer__open"))
+		goto out;
+
+	err = bpf_program__assoc_struct_ops(skel->progs.syscall_prog,
+					    skel->maps.st_ops_map, NULL);
+	ASSERT_OK(err, "bpf_program__assoc_struct_ops");
+
+	err = struct_ops_assoc_in_timer__attach(skel);
+	if (!ASSERT_OK(err, "struct_ops_assoc__attach"))
+		goto out;
+
+	/*
+	 * Run .test_1 by calling kfunc bpf_kfunc_multi_st_ops_test_1_prog_arg() and checks
+	 * the return value. .test_1 will also schedule timer_cb that runs .test_1 again
+	 * immediately.
+	 */
+	err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.syscall_prog), NULL);
+	ASSERT_OK(err, "bpf_prog_test_run_opts");
+
+	/* Check the return of the kfunc after timer_cb runs */
+	while (!READ_ONCE(skel->bss->timer_cb_run))
+		sched_yield();
+	ASSERT_EQ(skel->bss->timer_test_1_ret, 1234, "skel->bss->timer_test_1_ret");
+	ASSERT_EQ(skel->bss->test_err, 0, "skel->bss->test_err_a");
+out:
+	struct_ops_assoc_in_timer__destroy(skel);
+}
+
+static void test_st_ops_assoc_in_timer_no_uref(void)
+{
+	struct struct_ops_assoc_in_timer *skel = NULL;
+	struct bpf_link *link;
+	int err;
+
+	skel = struct_ops_assoc_in_timer__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "struct_ops_assoc_in_timer__open"))
+		goto out;
+
+	err = bpf_program__assoc_struct_ops(skel->progs.syscall_prog,
+					    skel->maps.st_ops_map, NULL);
+	ASSERT_OK(err, "bpf_program__assoc_struct_ops");
+
+	link = bpf_map__attach_struct_ops(skel->maps.st_ops_map);
+	if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops"))
+		goto out;
+
+	/*
+	 * Run .test_1 by calling kfunc bpf_kfunc_multi_st_ops_test_1_prog_arg() and checks
+	 * the return value. .test_1 will also schedule timer_cb that runs .test_1 again.
+	 * timer_cb will run 500ms after syscall_prog runs, when the user space no longer
+	 * holds a reference to st_ops_map.
+	 */
+	skel->bss->timer_ns = 500000000;
+	err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.syscall_prog), NULL);
+	ASSERT_OK(err, "bpf_prog_test_run_opts");
+
+	/* Detach and close struct_ops map to cause it to be freed */
+	bpf_link__destroy(link);
+	close(bpf_program__fd(skel->progs.syscall_prog));
+	close(bpf_map__fd(skel->maps.st_ops_map));
+
+	/* Check the return of the kfunc after timer_cb runs */
+	while (!READ_ONCE(skel->bss->timer_cb_run))
+		sched_yield();
+	ASSERT_EQ(skel->bss->timer_test_1_ret, -1, "skel->bss->timer_test_1_ret");
+	ASSERT_EQ(skel->bss->test_err, 0, "skel->bss->test_err_a");
+out:
+	struct_ops_assoc_in_timer__destroy(skel);
+}
+
+void test_struct_ops_assoc(void)
+{
+	if (test__start_subtest("st_ops_assoc"))
+		test_st_ops_assoc();
+	if (test__start_subtest("st_ops_assoc_reuse"))
+		test_st_ops_assoc_reuse();
+	if (test__start_subtest("st_ops_assoc_in_timer"))
+		test_st_ops_assoc_in_timer();
+	if (test__start_subtest("st_ops_assoc_in_timer_no_uref"))
+		test_st_ops_assoc_in_timer_no_uref();
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c
index 4b4b081b46cc..5829ffd70f8f 100644
--- a/tools/testing/selftests/bpf/prog_tests/verifier.c
+++ b/tools/testing/selftests/bpf/prog_tests/verifier.c
@@ -6,6 +6,8 @@
 #include "verifier_and.skel.h"
 #include "verifier_arena.skel.h"
 #include "verifier_arena_large.skel.h"
+#include "verifier_arena_globals1.skel.h"
+#include "verifier_arena_globals2.skel.h"
 #include "verifier_array_access.skel.h"
 #include "verifier_async_cb_context.skel.h"
 #include "verifier_basic_stack.skel.h"
@@ -147,6 +149,8 @@ static void run_tests_aux(const char *skel_name,
 void test_verifier_and(void)                  { RUN(verifier_and); }
 void test_verifier_arena(void)                { RUN(verifier_arena); }
 void test_verifier_arena_large(void)          { RUN(verifier_arena_large); }
+void test_verifier_arena_globals1(void)       { RUN(verifier_arena_globals1); }
+void test_verifier_arena_globals2(void)       { RUN(verifier_arena_globals2); }
 void test_verifier_basic_stack(void)          { RUN(verifier_basic_stack); }
 void test_verifier_bitfield_write(void)       { RUN(verifier_bitfield_write); }
 void test_verifier_bounds(void)               { RUN(verifier_bounds); }
diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_cpumap_attach.c b/tools/testing/selftests/bpf/prog_tests/xdp_cpumap_attach.c
index df27535995af..ad56e4370ce3 100644
--- a/tools/testing/selftests/bpf/prog_tests/xdp_cpumap_attach.c
+++ b/tools/testing/selftests/bpf/prog_tests/xdp_cpumap_attach.c
@@ -18,7 +18,7 @@ static void test_xdp_with_cpumap_helpers(void)
 	struct bpf_cpumap_val val = {
 		.qsize = 192,
 	};
-	int err, prog_fd, prog_redir_fd, map_fd;
+	int err, prog_fd, prog_redir_fd, map_fd, bad_fd;
 	struct nstoken *nstoken = NULL;
 	__u32 idx = 0;
 
@@ -79,7 +79,22 @@ static void test_xdp_with_cpumap_helpers(void)
 	val.qsize = 192;
 	val.bpf_prog.fd = bpf_program__fd(skel->progs.xdp_dummy_prog);
 	err = bpf_map_update_elem(map_fd, &idx, &val, 0);
-	ASSERT_NEQ(err, 0, "Add non-BPF_XDP_CPUMAP program to cpumap entry");
+	ASSERT_EQ(err, -EINVAL, "Add non-BPF_XDP_CPUMAP program to cpumap entry");
+
+	/* Try to attach non-BPF file descriptor */
+	bad_fd = open("/dev/null", O_RDONLY);
+	ASSERT_GE(bad_fd, 0, "Open /dev/null for non-BPF fd");
+
+	val.bpf_prog.fd = bad_fd;
+	err = bpf_map_update_elem(map_fd, &idx, &val, 0);
+	ASSERT_EQ(err, -EINVAL, "Add non-BPF fd to cpumap entry");
+
+	/* Try to attach nonexistent file descriptor */
+	err = close(bad_fd);
+	ASSERT_EQ(err, 0, "Close non-BPF fd for nonexistent fd");
+
+	err = bpf_map_update_elem(map_fd, &idx, &val, 0);
+	ASSERT_EQ(err, -EBADF, "Add nonexistent fd to cpumap entry");
 
 	/* Try to attach BPF_XDP program with frags to cpumap when we have
 	 * already loaded a BPF_XDP program on the map
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_assoc.c b/tools/testing/selftests/bpf/progs/struct_ops_assoc.c
new file mode 100644
index 000000000000..8f1097903e22
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/struct_ops_assoc.c
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <vmlinux.h>
+#include <bpf/bpf_tracing.h>
+#include "bpf_misc.h"
+#include "../test_kmods/bpf_testmod.h"
+#include "../test_kmods/bpf_testmod_kfunc.h"
+
+char _license[] SEC("license") = "GPL";
+
+int test_pid;
+
+/* Programs associated with st_ops_map_a */
+
+#define MAP_A_MAGIC 1234
+int test_err_a;
+
+SEC("struct_ops")
+int BPF_PROG(test_1_a, struct st_ops_args *args)
+{
+	return MAP_A_MAGIC;
+}
+
+SEC("tp_btf/sys_enter")
+int BPF_PROG(sys_enter_prog_a, struct pt_regs *regs, long id)
+{
+	struct st_ops_args args = {};
+	struct task_struct *task;
+	int ret;
+
+	task = bpf_get_current_task_btf();
+	if (!test_pid || task->pid != test_pid)
+		return 0;
+
+	ret = bpf_kfunc_multi_st_ops_test_1_impl(&args, NULL);
+	if (ret != MAP_A_MAGIC)
+		test_err_a++;
+
+	return 0;
+}
+
+SEC("syscall")
+int syscall_prog_a(void *ctx)
+{
+	struct st_ops_args args = {};
+	int ret;
+
+	ret = bpf_kfunc_multi_st_ops_test_1_impl(&args, NULL);
+	if (ret != MAP_A_MAGIC)
+		test_err_a++;
+
+	return 0;
+}
+
+SEC(".struct_ops.link")
+struct bpf_testmod_multi_st_ops st_ops_map_a = {
+	.test_1 = (void *)test_1_a,
+};
+
+/* Programs associated with st_ops_map_b */
+
+#define MAP_B_MAGIC 5678
+int test_err_b;
+
+SEC("struct_ops")
+int BPF_PROG(test_1_b, struct st_ops_args *args)
+{
+	return MAP_B_MAGIC;
+}
+
+SEC("tp_btf/sys_enter")
+int BPF_PROG(sys_enter_prog_b, struct pt_regs *regs, long id)
+{
+	struct st_ops_args args = {};
+	struct task_struct *task;
+	int ret;
+
+	task = bpf_get_current_task_btf();
+	if (!test_pid || task->pid != test_pid)
+		return 0;
+
+	ret = bpf_kfunc_multi_st_ops_test_1_impl(&args, NULL);
+	if (ret != MAP_B_MAGIC)
+		test_err_b++;
+
+	return 0;
+}
+
+SEC("syscall")
+int syscall_prog_b(void *ctx)
+{
+	struct st_ops_args args = {};
+	int ret;
+
+	ret = bpf_kfunc_multi_st_ops_test_1_impl(&args, NULL);
+	if (ret != MAP_B_MAGIC)
+		test_err_b++;
+
+	return 0;
+}
+
+SEC(".struct_ops.link")
+struct bpf_testmod_multi_st_ops st_ops_map_b = {
+	.test_1 = (void *)test_1_b,
+};
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_assoc_in_timer.c b/tools/testing/selftests/bpf/progs/struct_ops_assoc_in_timer.c
new file mode 100644
index 000000000000..d5a2ea934284
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/struct_ops_assoc_in_timer.c
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <vmlinux.h>
+#include <bpf/bpf_tracing.h>
+#include "bpf_misc.h"
+#include "../test_kmods/bpf_testmod.h"
+#include "../test_kmods/bpf_testmod_kfunc.h"
+
+char _license[] SEC("license") = "GPL";
+
+struct elem {
+	struct bpf_timer timer;
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARRAY);
+	__uint(max_entries, 1);
+	__type(key, int);
+	__type(value, struct elem);
+} array_map SEC(".maps");
+
+#define MAP_MAGIC 1234
+int recur;
+int test_err;
+int timer_ns;
+int timer_test_1_ret;
+int timer_cb_run;
+
+__noinline static int timer_cb(void *map, int *key, struct bpf_timer *timer)
+{
+	struct st_ops_args args = {};
+
+	recur++;
+	timer_test_1_ret = bpf_kfunc_multi_st_ops_test_1_impl(&args, NULL);
+	recur--;
+
+	timer_cb_run++;
+
+	return 0;
+}
+
+SEC("struct_ops")
+int BPF_PROG(test_1, struct st_ops_args *args)
+{
+	struct bpf_timer *timer;
+	int key = 0;
+
+	if (!recur) {
+		timer = bpf_map_lookup_elem(&array_map, &key);
+		if (!timer)
+			return 0;
+
+		bpf_timer_init(timer, &array_map, 1);
+		bpf_timer_set_callback(timer, timer_cb);
+		bpf_timer_start(timer, timer_ns, 0);
+	}
+
+	return MAP_MAGIC;
+}
+
+SEC("syscall")
+int syscall_prog(void *ctx)
+{
+	struct st_ops_args args = {};
+	int ret;
+
+	ret = bpf_kfunc_multi_st_ops_test_1_impl(&args, NULL);
+	if (ret != MAP_MAGIC)
+		test_err++;
+
+	return 0;
+}
+
+SEC(".struct_ops.link")
+struct bpf_testmod_multi_st_ops st_ops_map = {
+	.test_1 = (void *)test_1,
+};
diff --git a/tools/testing/selftests/bpf/progs/struct_ops_assoc_reuse.c b/tools/testing/selftests/bpf/progs/struct_ops_assoc_reuse.c
new file mode 100644
index 000000000000..5bb6ebf5eed4
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/struct_ops_assoc_reuse.c
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <vmlinux.h>
+#include <bpf/bpf_tracing.h>
+#include "bpf_misc.h"
+#include "../test_kmods/bpf_testmod.h"
+#include "../test_kmods/bpf_testmod_kfunc.h"
+
+char _license[] SEC("license") = "GPL";
+
+#define MAP_A_MAGIC 1234
+int test_err_a;
+int recur;
+
+/*
+ * test_1_a is reused. The kfunc should not be able to get the associated
+ * struct_ops and call test_1 recursively as it is ambiguous.
+ */
+SEC("struct_ops")
+int BPF_PROG(test_1_a, struct st_ops_args *args)
+{
+	int ret;
+
+	if (!recur) {
+		recur++;
+		ret = bpf_kfunc_multi_st_ops_test_1_impl(args, NULL);
+		if (ret != -1)
+			test_err_a++;
+		recur--;
+	}
+
+	return MAP_A_MAGIC;
+}
+
+/* Programs associated with st_ops_map_a */
+
+SEC("syscall")
+int syscall_prog_a(void *ctx)
+{
+	struct st_ops_args args = {};
+	int ret;
+
+	ret = bpf_kfunc_multi_st_ops_test_1_impl(&args, NULL);
+	if (ret != MAP_A_MAGIC)
+		test_err_a++;
+
+	return 0;
+}
+
+SEC(".struct_ops.link")
+struct bpf_testmod_multi_st_ops st_ops_map_a = {
+	.test_1 = (void *)test_1_a,
+};
+
+/* Programs associated with st_ops_map_b */
+
+int test_err_b;
+
+SEC("syscall")
+int syscall_prog_b(void *ctx)
+{
+	struct st_ops_args args = {};
+	int ret;
+
+	ret = bpf_kfunc_multi_st_ops_test_1_impl(&args, NULL);
+	if (ret != MAP_A_MAGIC)
+		test_err_b++;
+
+	return 0;
+}
+
+SEC(".struct_ops.link")
+struct bpf_testmod_multi_st_ops st_ops_map_b = {
+	.test_1 = (void *)test_1_a,
+};
diff --git a/tools/testing/selftests/bpf/progs/test_d_path.c b/tools/testing/selftests/bpf/progs/test_d_path.c
index 84e1f883f97b..561b2f861808 100644
--- a/tools/testing/selftests/bpf/progs/test_d_path.c
+++ b/tools/testing/selftests/bpf/progs/test_d_path.c
@@ -17,6 +17,7 @@ int rets_close[MAX_FILES] = {};
 
 int called_stat = 0;
 int called_close = 0;
+int path_match_fallocate = 0;
 
 SEC("fentry/security_inode_getattr")
 int BPF_PROG(prog_stat, struct path *path, struct kstat *stat,
@@ -62,4 +63,26 @@ int BPF_PROG(prog_close, struct file *file, void *id)
 	return 0;
 }
 
+SEC("fentry/vfs_fallocate")
+int BPF_PROG(prog_fallocate, struct file *file, int mode, loff_t offset, loff_t len)
+{
+	pid_t pid = bpf_get_current_pid_tgid() >> 32;
+	int ret = 0;
+	char path_fallocate[MAX_PATH_LEN] = {};
+
+	if (pid != my_pid)
+		return 0;
+
+	ret = bpf_d_path(&file->f_path,
+			 path_fallocate, MAX_PATH_LEN);
+	if (ret < 0)
+		return 0;
+
+	if (!path_fallocate[0])
+		return 0;
+
+	path_match_fallocate = 1;
+	return 0;
+}
+
 char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_globals1.c b/tools/testing/selftests/bpf/progs/verifier_arena_globals1.c
new file mode 100644
index 000000000000..14afef3d6442
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/verifier_arena_globals1.c
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
+
+#define BPF_NO_KFUNC_PROTOTYPES
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include "bpf_experimental.h"
+#include "bpf_arena_common.h"
+#include "bpf_misc.h"
+
+#define ARENA_PAGES (1UL<< (32 - 12))
+#define GLOBAL_PAGES (16)
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARENA);
+	__uint(map_flags, BPF_F_MMAPABLE);
+	__uint(max_entries, ARENA_PAGES);
+#ifdef __TARGET_ARCH_arm64
+	__ulong(map_extra, (1ull << 32) | (~0u - __PAGE_SIZE * ARENA_PAGES + 1));
+#else
+	__ulong(map_extra, (1ull << 44) | (~0u - __PAGE_SIZE * ARENA_PAGES + 1));
+#endif
+} arena SEC(".maps");
+
+/*
+ * Global data, to be placed at the end of the arena.
+ */
+volatile char __arena global_data[GLOBAL_PAGES][PAGE_SIZE];
+
+SEC("syscall")
+__success __retval(0)
+int check_reserve1(void *ctx)
+{
+#if defined(__BPF_FEATURE_ADDR_SPACE_CAST)
+	const u8 magic = 0x5a;
+	__u8 __arena *guard, *globals;
+	volatile char __arena *ptr;
+	int i;
+	int ret;
+
+	guard = (void __arena *)arena_base(&arena);
+	globals = (void __arena *)(arena_base(&arena) + (ARENA_PAGES - GLOBAL_PAGES) * PAGE_SIZE);
+
+	/* Reserve the region we've offset the globals by. */
+	ret = bpf_arena_reserve_pages(&arena, guard, ARENA_PAGES - GLOBAL_PAGES);
+	if (ret)
+		return 1;
+
+	/* Make sure the globals are in the expected offset. */
+	ret = bpf_arena_reserve_pages(&arena, globals, 1);
+	if (!ret)
+		return 2;
+
+	/* Verify globals are properly mapped in by libbpf. */
+	for (i = 0; i < GLOBAL_PAGES; i++) {
+		ptr = &global_data[i][PAGE_SIZE / 2];
+
+		*ptr = magic;
+		if (*ptr != magic)
+			return i + 3;
+	}
+#endif
+	return 0;
+}
+
+/*
+ * Relocation check by reading directly into the global data w/o using symbols.
+ */
+SEC("syscall")
+__success __retval(0)
+int check_relocation(void *ctx)
+{
+#if defined(__BPF_FEATURE_ADDR_SPACE_CAST)
+	const u8 magic = 0xfa;
+	u8 __arena *ptr;
+
+	global_data[GLOBAL_PAGES - 1][PAGE_SIZE / 2] = magic;
+	ptr = (u8 __arena *)((u64)(ARENA_PAGES * PAGE_SIZE - PAGE_SIZE / 2));
+	if (*ptr != magic)
+		return 1;
+
+#endif
+	return 0;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_globals2.c b/tools/testing/selftests/bpf/progs/verifier_arena_globals2.c
new file mode 100644
index 000000000000..e6bd7b61f9f1
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/verifier_arena_globals2.c
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
+
+#define BPF_NO_KFUNC_PROTOTYPES
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include "bpf_misc.h"
+#include "bpf_experimental.h"
+#include "bpf_arena_common.h"
+
+#define ARENA_PAGES (32)
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARENA);
+	__uint(map_flags, BPF_F_MMAPABLE);
+	__uint(max_entries, ARENA_PAGES);
+#ifdef __TARGET_ARCH_arm64
+	__ulong(map_extra, (1ull << 32) | (~0u - __PAGE_SIZE * ARENA_PAGES + 1));
+#else
+	__ulong(map_extra, (1ull << 44) | (~0u - __PAGE_SIZE * ARENA_PAGES + 1));
+#endif
+} arena SEC(".maps");
+
+/*
+ * Fill the entire arena with global data.
+ * The offset into the arena should be 0.
+ */
+char __arena global_data[ARENA_PAGES][PAGE_SIZE];
+
+SEC("syscall")
+__success __retval(0)
+int check_reserve2(void *ctx)
+{
+#if defined(__BPF_FEATURE_ADDR_SPACE_CAST)
+	void __arena *guard;
+	int ret;
+
+	guard = (void __arena *)arena_base(&arena);
+
+	/* Make sure the data at offset 0 case is properly handled. */
+	ret = bpf_arena_reserve_pages(&arena, guard, 1);
+	if (!ret)
+		return 1;
+#endif
+	return 0;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_large.c b/tools/testing/selftests/bpf/progs/verifier_arena_large.c
index f19e15400b3e..2b8cf2a4d880 100644
--- a/tools/testing/selftests/bpf/progs/verifier_arena_large.c
+++ b/tools/testing/selftests/bpf/progs/verifier_arena_large.c
@@ -23,18 +23,31 @@ int big_alloc1(void *ctx)
 {
 #if defined(__BPF_FEATURE_ADDR_SPACE_CAST)
 	volatile char __arena *page1, *page2, *no_page, *page3;
-	void __arena *base;
+	u64 base;
 
-	page1 = base = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0);
+	base = (u64)arena_base(&arena);
+
+	page1 = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0);
 	if (!page1)
 		return 1;
+
+	if ((u64)page1 != base)
+		return 15;
+
 	*page1 = 1;
-	page2 = bpf_arena_alloc_pages(&arena, base + ARENA_SIZE - PAGE_SIZE * 2,
+	page2 = bpf_arena_alloc_pages(&arena, (void __arena *)(ARENA_SIZE - 2 * PAGE_SIZE),
 				      1, NUMA_NO_NODE, 0);
 	if (!page2)
 		return 2;
 	*page2 = 2;
-	no_page = bpf_arena_alloc_pages(&arena, base + ARENA_SIZE - PAGE_SIZE,
+
+	/* Test for the guard region at the end of the arena. */
+	no_page = bpf_arena_alloc_pages(&arena, (void __arena *)ARENA_SIZE - PAGE_SIZE,
+					1, NUMA_NO_NODE, 0);
+	if (no_page)
+		return 16;
+
+	no_page = bpf_arena_alloc_pages(&arena, (void __arena *)ARENA_SIZE,
 					1, NUMA_NO_NODE, 0);
 	if (no_page)
 		return 3;
diff --git a/tools/testing/selftests/bpf/progs/verifier_subreg.c b/tools/testing/selftests/bpf/progs/verifier_subreg.c
index 8613ea160dcd..b3e1c3eef9ae 100644
--- a/tools/testing/selftests/bpf/progs/verifier_subreg.c
+++ b/tools/testing/selftests/bpf/progs/verifier_subreg.c
@@ -531,6 +531,74 @@ __naked void arsh32_imm_zero_extend_check(void)
 	: __clobber_all);
 }
 
+SEC("socket")
+__description("arsh32 imm sign positive extend check")
+__success __retval(0)
+__log_level(2)
+__msg("2: (57) r6 &= 4095                    ; R6=scalar(smin=smin32=0,smax=umax=smax32=umax32=4095,var_off=(0x0; 0xfff))")
+__msg("3: (67) r6 <<= 32                     ; R6=scalar(smin=smin32=0,smax=umax=0xfff00000000,smax32=umax32=0,var_off=(0x0; 0xfff00000000))")
+__msg("4: (c7) r6 s>>= 32                    ; R6=scalar(smin=smin32=0,smax=umax=smax32=umax32=4095,var_off=(0x0; 0xfff))")
+__naked void arsh32_imm_sign_extend_positive_check(void)
+{
+	asm volatile ("					\
+	call %[bpf_get_prandom_u32];			\
+	r6 = r0;					\
+	r6 &= 4095;					\
+	r6 <<= 32;					\
+	r6 s>>= 32;					\
+	r0 = 0;						\
+	exit;						\
+"	:
+	: __imm(bpf_get_prandom_u32)
+	: __clobber_all);
+}
+
+SEC("socket")
+__description("arsh32 imm sign negative extend check")
+__success __retval(0)
+__log_level(2)
+__msg("3: (17) r6 -= 4095                    ; R6=scalar(smin=smin32=-4095,smax=smax32=0)")
+__msg("4: (67) r6 <<= 32                     ; R6=scalar(smin=0xfffff00100000000,smax=smax32=umax32=0,umax=0xffffffff00000000,smin32=0,var_off=(0x0; 0xffffffff00000000))")
+__msg("5: (c7) r6 s>>= 32                    ; R6=scalar(smin=smin32=-4095,smax=smax32=0)")
+__naked void arsh32_imm_sign_extend_negative_check(void)
+{
+	asm volatile ("					\
+	call %[bpf_get_prandom_u32];			\
+	r6 = r0;					\
+	r6 &= 4095;					\
+	r6 -= 4095;					\
+	r6 <<= 32;					\
+	r6 s>>= 32;					\
+	r0 = 0;						\
+	exit;						\
+"	:
+	: __imm(bpf_get_prandom_u32)
+	: __clobber_all);
+}
+
+SEC("socket")
+__description("arsh32 imm sign extend check")
+__success __retval(0)
+__log_level(2)
+__msg("3: (17) r6 -= 2047                    ; R6=scalar(smin=smin32=-2047,smax=smax32=2048)")
+__msg("4: (67) r6 <<= 32                     ; R6=scalar(smin=0xfffff80100000000,smax=0x80000000000,umax=0xffffffff00000000,smin32=0,smax32=umax32=0,var_off=(0x0; 0xffffffff00000000))")
+__msg("5: (c7) r6 s>>= 32                    ; R6=scalar(smin=smin32=-2047,smax=smax32=2048)")
+__naked void arsh32_imm_sign_extend_check(void)
+{
+	asm volatile ("					\
+	call %[bpf_get_prandom_u32];			\
+	r6 = r0;					\
+	r6 &= 4095;					\
+	r6 -= 2047;					\
+	r6 <<= 32;					\
+	r6 s>>= 32;					\
+	r0 = 0;						\
+	exit;						\
+"	:
+	: __imm(bpf_get_prandom_u32)
+	: __clobber_all);
+}
+
 SEC("socket")
 __description("end16 (to_le) reg zero extend check")
 __success __success_unpriv __retval(0)
diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c
index 1669a7eeda26..90c4b1a51de6 100644
--- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c
+++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c
@@ -1134,6 +1134,7 @@ __bpf_kfunc int bpf_kfunc_st_ops_inc10(struct st_ops_args *args)
 }
 
 __bpf_kfunc int bpf_kfunc_multi_st_ops_test_1(struct st_ops_args *args, u32 id);
+__bpf_kfunc int bpf_kfunc_multi_st_ops_test_1_impl(struct st_ops_args *args, void *aux_prog);
 
 BTF_KFUNCS_START(bpf_testmod_check_kfunc_ids)
 BTF_ID_FLAGS(func, bpf_testmod_test_mod_kfunc)
@@ -1176,6 +1177,7 @@ BTF_ID_FLAGS(func, bpf_kfunc_st_ops_test_epilogue, KF_TRUSTED_ARGS | KF_SLEEPABL
 BTF_ID_FLAGS(func, bpf_kfunc_st_ops_test_pro_epilogue, KF_TRUSTED_ARGS | KF_SLEEPABLE)
 BTF_ID_FLAGS(func, bpf_kfunc_st_ops_inc10, KF_TRUSTED_ARGS)
 BTF_ID_FLAGS(func, bpf_kfunc_multi_st_ops_test_1, KF_TRUSTED_ARGS)
+BTF_ID_FLAGS(func, bpf_kfunc_multi_st_ops_test_1_impl, KF_TRUSTED_ARGS)
 BTF_KFUNCS_END(bpf_testmod_check_kfunc_ids)
 
 static int bpf_testmod_ops_init(struct btf *btf)
@@ -1637,6 +1639,7 @@ static struct bpf_testmod_multi_st_ops *multi_st_ops_find_nolock(u32 id)
 	return NULL;
 }
 
+/* Call test_1() of the struct_ops map identified by the id */
 int bpf_kfunc_multi_st_ops_test_1(struct st_ops_args *args, u32 id)
 {
 	struct bpf_testmod_multi_st_ops *st_ops;
@@ -1652,6 +1655,20 @@ int bpf_kfunc_multi_st_ops_test_1(struct st_ops_args *args, u32 id)
 	return ret;
 }
 
+/* Call test_1() of the associated struct_ops map */
+int bpf_kfunc_multi_st_ops_test_1_impl(struct st_ops_args *args, void *aux__prog)
+{
+	struct bpf_prog_aux *prog_aux = (struct bpf_prog_aux *)aux__prog;
+	struct bpf_testmod_multi_st_ops *st_ops;
+	int ret = -1;
+
+	st_ops = (struct bpf_testmod_multi_st_ops *)bpf_prog_get_assoc_struct_ops(prog_aux);
+	if (st_ops)
+		ret = st_ops->test_1(args);
+
+	return ret;
+}
+
 static int multi_st_ops_reg(void *kdata, struct bpf_link *link)
 {
 	struct bpf_testmod_multi_st_ops *st_ops =
diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h
index 4df6fa6a92cb..2357a0340ffe 100644
--- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h
+++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h
@@ -162,5 +162,6 @@ struct task_struct *bpf_kfunc_ret_rcu_test(void) __ksym;
 int *bpf_kfunc_ret_rcu_test_nostruct(int rdonly_buf_size) __ksym;
 
 int bpf_kfunc_multi_st_ops_test_1(struct st_ops_args *args, u32 id) __ksym;
+int bpf_kfunc_multi_st_ops_test_1_impl(struct st_ops_args *args, void *aux__prog) __ksym;
 
 #endif /* _BPF_TESTMOD_KFUNC_H */
diff --git a/tools/testing/selftests/bpf/verifier/direct_value_access.c b/tools/testing/selftests/bpf/verifier/direct_value_access.c
index c0648dc009b5..e569d119fb60 100644
--- a/tools/testing/selftests/bpf/verifier/direct_value_access.c
+++ b/tools/testing/selftests/bpf/verifier/direct_value_access.c
@@ -81,7 +81,7 @@
 	},
 	.fixup_map_array_48b = { 1 },
 	.result = REJECT,
-	.errstr = "direct value offset of 4294967295 is not allowed",
+	.errstr = "invalid access to map value pointer, value_size=48 off=4294967295",
 },
 {
 	"direct map access, write test 8",
@@ -141,7 +141,7 @@
 	},
 	.fixup_map_array_48b = { 1 },
 	.result = REJECT,
-	.errstr = "direct value offset of 536870912 is not allowed",
+	.errstr = "invalid access to map value pointer, value_size=48 off=536870912",
 },
 {
 	"direct map access, write test 13",
diff --git a/tools/testing/selftests/cgroup/lib/cgroup_util.c b/tools/testing/selftests/cgroup/lib/cgroup_util.c
index 44c52f620fda..ce6c2642fd9b 100644
--- a/tools/testing/selftests/cgroup/lib/cgroup_util.c
+++ b/tools/testing/selftests/cgroup/lib/cgroup_util.c
@@ -168,6 +168,27 @@ long cg_read_key_long(const char *cgroup, const char *control, const char *key)
 	return atol(ptr + strlen(key));
 }
 
+long cg_read_key_long_poll(const char *cgroup, const char *control,
+			   const char *key, long expected, int retries,
+			   useconds_t wait_interval_us)
+{
+	long val = -1;
+	int i;
+
+	for (i = 0; i < retries; i++) {
+		val = cg_read_key_long(cgroup, control, key);
+		if (val < 0)
+			return val;
+
+		if (val == expected)
+			break;
+
+		usleep(wait_interval_us);
+	}
+
+	return val;
+}
+
 long cg_read_lc(const char *cgroup, const char *control)
 {
 	char buf[PAGE_SIZE];
diff --git a/tools/testing/selftests/cgroup/lib/include/cgroup_util.h b/tools/testing/selftests/cgroup/lib/include/cgroup_util.h
index 7ab2824ed7b5..77f386dab5e8 100644
--- a/tools/testing/selftests/cgroup/lib/include/cgroup_util.h
+++ b/tools/testing/selftests/cgroup/lib/include/cgroup_util.h
@@ -17,6 +17,8 @@
 #define CG_NAMED_NAME "selftest"
 #define CG_PATH_FORMAT (!cg_test_v1_named ? "0::%s" : (":name=" CG_NAMED_NAME ":%s"))
 
+#define DEFAULT_WAIT_INTERVAL_US (100 * 1000) /* 100 ms */
+
 /*
  * Checks if two given values differ by less than err% of their sum.
  */
@@ -64,6 +66,9 @@ extern int cg_read_strstr(const char *cgroup, const char *control,
 extern long cg_read_long(const char *cgroup, const char *control);
 extern long cg_read_long_fd(int fd);
 long cg_read_key_long(const char *cgroup, const char *control, const char *key);
+long cg_read_key_long_poll(const char *cgroup, const char *control,
+			   const char *key, long expected, int retries,
+			   useconds_t wait_interval_us);
 extern long cg_read_lc(const char *cgroup, const char *control);
 extern int cg_write(const char *cgroup, const char *control, char *buf);
 extern int cg_open(const char *cgroup, const char *control, int flags);
diff --git a/tools/testing/selftests/cgroup/test_kmem.c b/tools/testing/selftests/cgroup/test_kmem.c
index ca38525484e3..eeabd34bf083 100644
--- a/tools/testing/selftests/cgroup/test_kmem.c
+++ b/tools/testing/selftests/cgroup/test_kmem.c
@@ -26,6 +26,7 @@
  */
 #define MAX_VMSTAT_ERROR (4096 * 64 * get_nprocs())
 
+#define KMEM_DEAD_WAIT_RETRIES        80
 
 static int alloc_dcache(const char *cgroup, void *arg)
 {
@@ -306,9 +307,7 @@ static int test_kmem_dead_cgroups(const char *root)
 {
 	int ret = KSFT_FAIL;
 	char *parent;
-	long dead;
-	int i;
-	int max_time = 20;
+	long dead = -1;
 
 	parent = cg_name(root, "kmem_dead_cgroups_test");
 	if (!parent)
@@ -323,21 +322,19 @@ static int test_kmem_dead_cgroups(const char *root)
 	if (cg_run_in_subcgroups(parent, alloc_dcache, (void *)100, 30))
 		goto cleanup;
 
-	for (i = 0; i < max_time; i++) {
-		dead = cg_read_key_long(parent, "cgroup.stat",
-					"nr_dying_descendants ");
-		if (dead == 0) {
-			ret = KSFT_PASS;
-			break;
-		}
-		/*
-		 * Reclaiming cgroups might take some time,
-		 * let's wait a bit and repeat.
-		 */
-		sleep(1);
-		if (i > 5)
-			printf("Waiting time longer than 5s; wait: %ds (dead: %ld)\n", i, dead);
-	}
+	/*
+	 * Allow up to ~8s for reclaim of dying descendants to complete.
+	 * This is a generous upper bound derived from stress testing, not
+	 * from a specific kernel constant, and can be adjusted if reclaim
+	 * behavior changes in the future.
+	 */
+	dead = cg_read_key_long_poll(parent, "cgroup.stat",
+					"nr_dying_descendants ", 0, KMEM_DEAD_WAIT_RETRIES,
+					DEFAULT_WAIT_INTERVAL_US);
+	if (dead)
+		goto cleanup;
+
+	ret = KSFT_PASS;
 
 cleanup:
 	cg_destroy(parent);
diff --git a/tools/testing/selftests/cgroup/test_memcontrol.c b/tools/testing/selftests/cgroup/test_memcontrol.c
index 4e1647568c5b..2fb096a2a9f9 100644
--- a/tools/testing/selftests/cgroup/test_memcontrol.c
+++ b/tools/testing/selftests/cgroup/test_memcontrol.c
@@ -21,6 +21,8 @@
 #include "kselftest.h"
 #include "cgroup_util.h"
 
+#define MEMCG_SOCKSTAT_WAIT_RETRIES        30
+
 static bool has_localevents;
 static bool has_recursiveprot;
 
@@ -1384,6 +1386,7 @@ static int test_memcg_sock(const char *root)
 	int bind_retries = 5, ret = KSFT_FAIL, pid, err;
 	unsigned short port;
 	char *memcg;
+	long sock_post = -1;
 
 	memcg = cg_name(root, "memcg_test");
 	if (!memcg)
@@ -1432,7 +1435,22 @@ static int test_memcg_sock(const char *root)
 	if (cg_read_long(memcg, "memory.current") < 0)
 		goto cleanup;
 
-	if (cg_read_key_long(memcg, "memory.stat", "sock "))
+	/*
+	 * memory.stat is updated asynchronously via the memcg rstat
+	 * flushing worker, which runs periodically (every 2 seconds,
+	 * see FLUSH_TIME). On a busy system, the "sock " counter may
+	 * stay non-zero for a short period of time after the TCP
+	 * connection is closed and all socket memory has been
+	 * uncharged.
+	 *
+	 * Poll memory.stat for up to 3 seconds (~FLUSH_TIME plus some
+	 * scheduling slack) and require that the "sock " counter
+	 * eventually drops to zero.
+	 */
+	sock_post = cg_read_key_long_poll(memcg, "memory.stat", "sock ", 0,
+					 MEMCG_SOCKSTAT_WAIT_RETRIES,
+					 DEFAULT_WAIT_INTERVAL_US);
+	if (sock_post)
 		goto cleanup;
 
 	ret = KSFT_PASS;
diff --git a/tools/testing/selftests/filesystems/statmount/statmount.h b/tools/testing/selftests/filesystems/statmount/statmount.h
index 99e5ad082fb1..e1cba4bfd8d9 100644
--- a/tools/testing/selftests/filesystems/statmount/statmount.h
+++ b/tools/testing/selftests/filesystems/statmount/statmount.h
@@ -43,19 +43,24 @@
 	#endif
 #endif
 
-static inline int statmount(uint64_t mnt_id, uint64_t mnt_ns_id, uint64_t mask,
-			    struct statmount *buf, size_t bufsize,
+static inline int statmount(uint64_t mnt_id, uint64_t mnt_ns_id, uint32_t fd,
+			    uint64_t mask, struct statmount *buf, size_t bufsize,
 			    unsigned int flags)
 {
 	struct mnt_id_req req = {
 		.size = MNT_ID_REQ_SIZE_VER0,
-		.mnt_id = mnt_id,
 		.param = mask,
 	};
 
-	if (mnt_ns_id) {
+	if (flags & STATMOUNT_BY_FD) {
 		req.size = MNT_ID_REQ_SIZE_VER1;
-		req.mnt_ns_id = mnt_ns_id;
+		req.mnt_fd = fd;
+	} else {
+		req.mnt_id = mnt_id;
+		if (mnt_ns_id) {
+			req.size = MNT_ID_REQ_SIZE_VER1;
+			req.mnt_ns_id = mnt_ns_id;
+		}
 	}
 
 	return syscall(__NR_statmount, &req, buf, bufsize, flags);
diff --git a/tools/testing/selftests/filesystems/statmount/statmount_test.c b/tools/testing/selftests/filesystems/statmount/statmount_test.c
index 6e53430423d2..a04bcaace126 100644
--- a/tools/testing/selftests/filesystems/statmount/statmount_test.c
+++ b/tools/testing/selftests/filesystems/statmount/statmount_test.c
@@ -33,15 +33,24 @@ static const char *const known_fs[] = {
 	"sysv", "tmpfs", "tracefs", "ubifs", "udf", "ufs", "v7", "vboxsf",
 	"vfat", "virtiofs", "vxfs", "xenfs", "xfs", "zonefs", NULL };
 
-static struct statmount *statmount_alloc(uint64_t mnt_id, uint64_t mask, unsigned int flags)
+static struct statmount *statmount_alloc(uint64_t mnt_id, int fd, uint64_t mask, unsigned int flags)
 {
 	size_t bufsize = 1 << 15;
-	struct statmount *buf = NULL, *tmp = alloca(bufsize);
+	struct statmount *buf = NULL, *tmp = NULL;
 	int tofree = 0;
 	int ret;
 
+	if (flags & STATMOUNT_BY_FD && fd < 0)
+		return NULL;
+
+	tmp = alloca(bufsize);
+
 	for (;;) {
-		ret = statmount(mnt_id, 0, mask, tmp, bufsize, flags);
+		if (flags & STATMOUNT_BY_FD)
+			ret = statmount(0, 0, (uint32_t) fd, mask, tmp, bufsize, flags);
+		else
+			ret = statmount(mnt_id, 0, 0, mask, tmp, bufsize, flags);
+
 		if (ret != -1)
 			break;
 		if (tofree)
@@ -237,7 +246,7 @@ static void test_statmount_zero_mask(void)
 	struct statmount sm;
 	int ret;
 
-	ret = statmount(root_id, 0, 0, &sm, sizeof(sm), 0);
+	ret = statmount(root_id, 0, 0, 0, &sm, sizeof(sm), 0);
 	if (ret == -1) {
 		ksft_test_result_fail("statmount zero mask: %s\n",
 				      strerror(errno));
@@ -263,7 +272,7 @@ static void test_statmount_mnt_basic(void)
 	int ret;
 	uint64_t mask = STATMOUNT_MNT_BASIC;
 
-	ret = statmount(root_id, 0, mask, &sm, sizeof(sm), 0);
+	ret = statmount(root_id, 0, 0, mask, &sm, sizeof(sm), 0);
 	if (ret == -1) {
 		ksft_test_result_fail("statmount mnt basic: %s\n",
 				      strerror(errno));
@@ -323,7 +332,7 @@ static void test_statmount_sb_basic(void)
 	struct statx sx;
 	struct statfs sf;
 
-	ret = statmount(root_id, 0, mask, &sm, sizeof(sm), 0);
+	ret = statmount(root_id, 0, 0, mask, &sm, sizeof(sm), 0);
 	if (ret == -1) {
 		ksft_test_result_fail("statmount sb basic: %s\n",
 				      strerror(errno));
@@ -375,7 +384,7 @@ static void test_statmount_mnt_point(void)
 {
 	struct statmount *sm;
 
-	sm = statmount_alloc(root_id, STATMOUNT_MNT_POINT, 0);
+	sm = statmount_alloc(root_id, 0, STATMOUNT_MNT_POINT, 0);
 	if (!sm) {
 		ksft_test_result_fail("statmount mount point: %s\n",
 				      strerror(errno));
@@ -405,7 +414,7 @@ static void test_statmount_mnt_root(void)
 	assert(last_dir);
 	last_dir++;
 
-	sm = statmount_alloc(root_id, STATMOUNT_MNT_ROOT, 0);
+	sm = statmount_alloc(root_id, 0, STATMOUNT_MNT_ROOT, 0);
 	if (!sm) {
 		ksft_test_result_fail("statmount mount root: %s\n",
 				      strerror(errno));
@@ -438,7 +447,7 @@ static void test_statmount_fs_type(void)
 	const char *fs_type;
 	const char *const *s;
 
-	sm = statmount_alloc(root_id, STATMOUNT_FS_TYPE, 0);
+	sm = statmount_alloc(root_id, 0, STATMOUNT_FS_TYPE, 0);
 	if (!sm) {
 		ksft_test_result_fail("statmount fs type: %s\n",
 				      strerror(errno));
@@ -467,7 +476,7 @@ static void test_statmount_mnt_opts(void)
 	char *line = NULL;
 	size_t len = 0;
 
-	sm = statmount_alloc(root_id, STATMOUNT_MNT_BASIC | STATMOUNT_MNT_OPTS,
+	sm = statmount_alloc(root_id, 0, STATMOUNT_MNT_BASIC | STATMOUNT_MNT_OPTS,
 			     0);
 	if (!sm) {
 		ksft_test_result_fail("statmount mnt opts: %s\n",
@@ -557,7 +566,7 @@ static void test_statmount_string(uint64_t mask, size_t off, const char *name)
 	uint32_t start, i;
 	int ret;
 
-	sm = statmount_alloc(root_id, mask, 0);
+	sm = statmount_alloc(root_id, 0, mask, 0);
 	if (!sm) {
 		ksft_test_result_fail("statmount %s: %s\n", name,
 				      strerror(errno));
@@ -586,14 +595,14 @@ static void test_statmount_string(uint64_t mask, size_t off, const char *name)
 	exactsize = sm->size;
 	shortsize = sizeof(*sm) + i;
 
-	ret = statmount(root_id, 0, mask, sm, exactsize, 0);
+	ret = statmount(root_id, 0, 0, mask, sm, exactsize, 0);
 	if (ret == -1) {
 		ksft_test_result_fail("statmount exact size: %s\n",
 				      strerror(errno));
 		goto out;
 	}
 	errno = 0;
-	ret = statmount(root_id, 0, mask, sm, shortsize, 0);
+	ret = statmount(root_id, 0, 0, mask, sm, shortsize, 0);
 	if (ret != -1 || errno != EOVERFLOW) {
 		ksft_test_result_fail("should have failed with EOVERFLOW: %s\n",
 				      strerror(errno));
@@ -658,6 +667,226 @@ static void test_listmount_tree(void)
 	ksft_test_result_pass("listmount tree\n");
 }
 
+static void test_statmount_by_fd(void)
+{
+	struct statmount *sm = NULL;
+	char tmpdir[] = "/statmount.fd.XXXXXX";
+	const char root[] = "/test";
+	char subdir[PATH_MAX], tmproot[PATH_MAX];
+	int fd;
+
+	if (!mkdtemp(tmpdir)) {
+		ksft_perror("mkdtemp");
+		return;
+	}
+
+	if (mount("statmount.test", tmpdir, "tmpfs", 0, NULL)) {
+		ksft_perror("mount");
+		rmdir(tmpdir);
+		return;
+	}
+
+	snprintf(subdir, PATH_MAX, "%s%s", tmpdir, root);
+	snprintf(tmproot, PATH_MAX, "%s/%s", tmpdir, "chroot");
+
+	if (mkdir(subdir, 0755)) {
+		ksft_perror("mkdir");
+		goto err_tmpdir;
+	}
+
+	if (mount(subdir, subdir, NULL, MS_BIND, 0)) {
+		ksft_perror("mount");
+		goto err_subdir;
+	}
+
+	if (mkdir(tmproot, 0755)) {
+		ksft_perror("mkdir");
+		goto err_subdir;
+	}
+
+	fd = open(subdir, O_PATH);
+	if (fd < 0) {
+		ksft_perror("open");
+		goto err_tmproot;
+	}
+
+	if (chroot(tmproot)) {
+		ksft_perror("chroot");
+		goto err_fd;
+	}
+
+	sm = statmount_alloc(0, fd, STATMOUNT_MNT_ROOT | STATMOUNT_MNT_POINT, STATMOUNT_BY_FD);
+	if (!sm) {
+		ksft_test_result_fail("statmount by fd failed: %s\n", strerror(errno));
+		goto err_chroot;
+	}
+
+	if (sm->size < sizeof(*sm)) {
+		ksft_test_result_fail("unexpected size: %u < %u\n",
+				      sm->size, (uint32_t) sizeof(*sm));
+		goto err_chroot;
+	}
+
+	if (sm->mask & STATMOUNT_MNT_POINT) {
+		ksft_test_result_fail("STATMOUNT_MNT_POINT unexpectedly set in statmount\n");
+		goto err_chroot;
+	}
+
+	if (!(sm->mask & STATMOUNT_MNT_ROOT)) {
+		ksft_test_result_fail("STATMOUNT_MNT_ROOT not set in statmount\n");
+		goto err_chroot;
+	}
+
+	if (strcmp(root, sm->str + sm->mnt_root) != 0) {
+		ksft_test_result_fail("statmount returned incorrect mnt_root,"
+			"statmount mnt_root: %s != %s\n",
+			sm->str + sm->mnt_root, root);
+		goto err_chroot;
+	}
+
+	if (chroot(".")) {
+		ksft_perror("chroot");
+		goto out;
+	}
+
+	free(sm);
+	sm = statmount_alloc(0, fd, STATMOUNT_MNT_ROOT | STATMOUNT_MNT_POINT, STATMOUNT_BY_FD);
+	if (!sm) {
+		ksft_test_result_fail("statmount by fd failed: %s\n", strerror(errno));
+		goto err_fd;
+	}
+
+	if (sm->size < sizeof(*sm)) {
+		ksft_test_result_fail("unexpected size: %u < %u\n",
+				      sm->size, (uint32_t) sizeof(*sm));
+		goto out;
+	}
+
+	if (!(sm->mask & STATMOUNT_MNT_POINT)) {
+		ksft_test_result_fail("STATMOUNT_MNT_POINT not set in statmount\n");
+		goto out;
+	}
+
+	if (!(sm->mask & STATMOUNT_MNT_ROOT)) {
+		ksft_test_result_fail("STATMOUNT_MNT_ROOT not set in statmount\n");
+		goto out;
+	}
+
+	if (strcmp(subdir, sm->str + sm->mnt_point) != 0) {
+		ksft_test_result_fail("statmount returned incorrect mnt_point,"
+			"statmount mnt_point: %s != %s\n", sm->str + sm->mnt_point, subdir);
+		goto out;
+	}
+
+	if (strcmp(root, sm->str + sm->mnt_root) != 0) {
+		ksft_test_result_fail("statmount returned incorrect mnt_root,"
+			"statmount mnt_root: %s != %s\n", sm->str + sm->mnt_root, root);
+		goto out;
+	}
+
+	ksft_test_result_pass("statmount by fd\n");
+	goto out;
+err_chroot:
+	chroot(".");
+out:
+	free(sm);
+err_fd:
+	close(fd);
+err_tmproot:
+	rmdir(tmproot);
+err_subdir:
+	umount2(subdir, MNT_DETACH);
+	rmdir(subdir);
+err_tmpdir:
+	umount2(tmpdir, MNT_DETACH);
+	rmdir(tmpdir);
+}
+
+static void test_statmount_by_fd_unmounted(void)
+{
+	const char root[] = "/test.unmounted";
+	char tmpdir[] = "/statmount.fd.XXXXXX";
+	char subdir[PATH_MAX];
+	int fd;
+	struct statmount *sm = NULL;
+
+	if (!mkdtemp(tmpdir)) {
+		ksft_perror("mkdtemp");
+		return;
+	}
+
+	if (mount("statmount.test", tmpdir, "tmpfs", 0, NULL)) {
+		ksft_perror("mount");
+		rmdir(tmpdir);
+		return;
+	}
+
+	snprintf(subdir, PATH_MAX, "%s%s", tmpdir, root);
+
+	if (mkdir(subdir, 0755)) {
+		ksft_perror("mkdir");
+		goto err_tmpdir;
+	}
+
+	if (mount(subdir, subdir, 0, MS_BIND, NULL)) {
+		ksft_perror("mount");
+		goto err_subdir;
+	}
+
+	fd = open(subdir, O_PATH);
+	if (fd < 0) {
+		ksft_perror("open");
+		goto err_subdir;
+	}
+
+	if (umount2(tmpdir, MNT_DETACH)) {
+		ksft_perror("umount2");
+		goto err_fd;
+	}
+
+	sm = statmount_alloc(0, fd, STATMOUNT_MNT_POINT | STATMOUNT_MNT_ROOT, STATMOUNT_BY_FD);
+	if (!sm) {
+		ksft_test_result_fail("statmount by fd unmounted: %s\n",
+				      strerror(errno));
+		goto err_sm;
+	}
+
+	if (sm->size < sizeof(*sm)) {
+		ksft_test_result_fail("unexpected size: %u < %u\n",
+				      sm->size, (uint32_t) sizeof(*sm));
+		goto err_sm;
+	}
+
+	if (sm->mask & STATMOUNT_MNT_POINT) {
+		ksft_test_result_fail("STATMOUNT_MNT_POINT unexpectedly set in mask\n");
+		goto err_sm;
+	}
+
+	if (!(sm->mask & STATMOUNT_MNT_ROOT)) {
+		ksft_test_result_fail("STATMOUNT_MNT_ROOT not set in mask\n");
+		goto err_sm;
+	}
+
+	if (strcmp(sm->str + sm->mnt_root, root) != 0) {
+		ksft_test_result_fail("statmount returned incorrect mnt_root,"
+			"statmount mnt_root: %s != %s\n",
+			sm->str + sm->mnt_root, root);
+		goto err_sm;
+	}
+
+	ksft_test_result_pass("statmount by fd on unmounted mount\n");
+err_sm:
+	free(sm);
+err_fd:
+	close(fd);
+err_subdir:
+	umount2(subdir, MNT_DETACH);
+	rmdir(subdir);
+err_tmpdir:
+	umount2(tmpdir, MNT_DETACH);
+	rmdir(tmpdir);
+}
+
 #define str_off(memb) (offsetof(struct statmount, memb) / sizeof(uint32_t))
 
 int main(void)
@@ -669,14 +898,14 @@ int main(void)
 
 	ksft_print_header();
 
-	ret = statmount(0, 0, 0, NULL, 0, 0);
+	ret = statmount(0, 0, 0, 0, NULL, 0, 0);
 	assert(ret == -1);
 	if (errno == ENOSYS)
 		ksft_exit_skip("statmount() syscall not supported\n");
 
 	setup_namespace();
 
-	ksft_set_plan(15);
+	ksft_set_plan(17);
 	test_listmount_empty_root();
 	test_statmount_zero_mask();
 	test_statmount_mnt_basic();
@@ -693,6 +922,8 @@ int main(void)
 	test_statmount_string(all_mask, str_off(fs_type), "fs type & all");
 
 	test_listmount_tree();
+	test_statmount_by_fd_unmounted();
+	test_statmount_by_fd();
 
 
 	if (ksft_get_fail_cnt() + ksft_get_error_cnt() > 0)
diff --git a/tools/testing/selftests/filesystems/statmount/statmount_test_ns.c b/tools/testing/selftests/filesystems/statmount/statmount_test_ns.c
index d56d4103182f..063d9de46431 100644
--- a/tools/testing/selftests/filesystems/statmount/statmount_test_ns.c
+++ b/tools/testing/selftests/filesystems/statmount/statmount_test_ns.c
@@ -102,7 +102,7 @@ static int _test_statmount_mnt_ns_id(void)
 	if (!root_id)
 		return NSID_ERROR;
 
-	ret = statmount(root_id, 0, STATMOUNT_MNT_NS_ID, &sm, sizeof(sm), 0);
+	ret = statmount(root_id, 0, 0, STATMOUNT_MNT_NS_ID, &sm, sizeof(sm), 0);
 	if (ret == -1) {
 		ksft_print_msg("statmount mnt ns id: %s\n", strerror(errno));
 		return NSID_ERROR;
@@ -128,6 +128,98 @@ static int _test_statmount_mnt_ns_id(void)
 	return NSID_PASS;
 }
 
+static int _test_statmount_mnt_ns_id_by_fd(void)
+{
+	struct statmount sm;
+	uint64_t mnt_ns_id;
+	int ret, fd, mounted = 1, status = NSID_ERROR;
+	char mnt[] = "/statmount.fd.XXXXXX";
+
+	ret = get_mnt_ns_id("/proc/self/ns/mnt", &mnt_ns_id);
+	if (ret != NSID_PASS)
+		return ret;
+
+	if (!mkdtemp(mnt)) {
+		ksft_print_msg("statmount by fd mnt ns id mkdtemp: %s\n", strerror(errno));
+		return NSID_ERROR;
+	}
+
+	if (mount(mnt, mnt, NULL, MS_BIND, 0)) {
+		ksft_print_msg("statmount by fd mnt ns id mount: %s\n", strerror(errno));
+		status = NSID_ERROR;
+		goto err;
+	}
+
+	fd = open(mnt, O_PATH);
+	if (fd < 0) {
+		ksft_print_msg("statmount by fd mnt ns id open: %s\n", strerror(errno));
+		goto err;
+	}
+
+	ret = statmount(0, 0, fd, STATMOUNT_MNT_NS_ID, &sm, sizeof(sm), STATMOUNT_BY_FD);
+	if (ret == -1) {
+		ksft_print_msg("statmount mnt ns id statmount: %s\n", strerror(errno));
+		status = NSID_ERROR;
+		goto out;
+	}
+
+	if (sm.size != sizeof(sm)) {
+		ksft_print_msg("unexpected size: %u != %u\n", sm.size,
+			       (uint32_t)sizeof(sm));
+		status = NSID_FAIL;
+		goto out;
+	}
+	if (sm.mask != STATMOUNT_MNT_NS_ID) {
+		ksft_print_msg("statmount mnt ns id unavailable\n");
+		status = NSID_SKIP;
+		goto out;
+	}
+
+	if (sm.mnt_ns_id != mnt_ns_id) {
+		ksft_print_msg("unexpected mnt ns ID: 0x%llx != 0x%llx\n",
+			       (unsigned long long)sm.mnt_ns_id,
+			       (unsigned long long)mnt_ns_id);
+		status = NSID_FAIL;
+		goto out;
+	}
+
+	mounted = 0;
+	if (umount2(mnt, MNT_DETACH)) {
+		ksft_print_msg("statmount by fd mnt ns id umount2: %s\n", strerror(errno));
+		goto out;
+	}
+
+	ret = statmount(0, 0, fd, STATMOUNT_MNT_NS_ID, &sm, sizeof(sm), STATMOUNT_BY_FD);
+	if (ret == -1) {
+		ksft_print_msg("statmount mnt ns id statmount: %s\n", strerror(errno));
+		status = NSID_ERROR;
+		goto out;
+	}
+
+	if (sm.size != sizeof(sm)) {
+		ksft_print_msg("unexpected size: %u != %u\n", sm.size,
+			       (uint32_t)sizeof(sm));
+		status = NSID_FAIL;
+		goto out;
+	}
+
+	if (sm.mask == STATMOUNT_MNT_NS_ID) {
+		ksft_print_msg("unexpected STATMOUNT_MNT_NS_ID in mask\n");
+		status = NSID_FAIL;
+		goto out;
+	}
+
+	status = NSID_PASS;
+out:
+	close(fd);
+	if (mounted)
+		umount2(mnt, MNT_DETACH);
+err:
+	rmdir(mnt);
+	return status;
+}
+
+
 static void test_statmount_mnt_ns_id(void)
 {
 	pid_t pid;
@@ -148,6 +240,9 @@ static void test_statmount_mnt_ns_id(void)
 	if (ret != NSID_PASS)
 		exit(ret);
 	ret = _test_statmount_mnt_ns_id();
+	if (ret != NSID_PASS)
+		exit(ret);
+	ret = _test_statmount_mnt_ns_id_by_fd();
 	exit(ret);
 }
 
@@ -179,7 +274,7 @@ static int validate_external_listmount(pid_t pid, uint64_t child_nr_mounts)
 	for (int i = 0; i < nr_mounts; i++) {
 		struct statmount sm;
 
-		ret = statmount(list[i], mnt_ns_id, STATMOUNT_MNT_NS_ID, &sm,
+		ret = statmount(list[i], mnt_ns_id, 0, STATMOUNT_MNT_NS_ID, &sm,
 				sizeof(sm), 0);
 		if (ret < 0) {
 			ksft_print_msg("statmount mnt ns id: %s\n", strerror(errno));
@@ -275,7 +370,7 @@ int main(void)
 	int ret;
 
 	ksft_print_header();
-	ret = statmount(0, 0, 0, NULL, 0, 0);
+	ret = statmount(0, 0, 0, 0, NULL, 0, 0);
 	assert(ret == -1);
 	if (errno == ENOSYS)
 		ksft_exit_skip("statmount() syscall not supported\n");
diff --git a/tools/testing/selftests/iommu/iommufd.c b/tools/testing/selftests/iommu/iommufd.c
index 10e051b6f592..dadad277f4eb 100644
--- a/tools/testing/selftests/iommu/iommufd.c
+++ b/tools/testing/selftests/iommu/iommufd.c
@@ -755,9 +755,6 @@ TEST_F(iommufd_ioas, get_hw_info)
 		struct iommu_test_hw_info info;
 		uint64_t trailing_bytes;
 	} buffer_larger;
-	struct iommu_test_hw_info_buffer_smaller {
-		__u32 flags;
-	} buffer_smaller;
 
 	if (self->device_id) {
 		uint8_t max_pasid = 0;
@@ -789,8 +786,9 @@ TEST_F(iommufd_ioas, get_hw_info)
 		 * the fields within the size range still gets updated.
 		 */
 		test_cmd_get_hw_info(self->device_id,
-				     IOMMU_HW_INFO_TYPE_DEFAULT,
-				     &buffer_smaller, sizeof(buffer_smaller));
+				     IOMMU_HW_INFO_TYPE_DEFAULT, &buffer_exact,
+				     offsetofend(struct iommu_test_hw_info,
+						 flags));
 		test_cmd_get_hw_info_pasid(self->device_id, &max_pasid);
 		ASSERT_EQ(0, max_pasid);
 		if (variant->pasid_capable) {
diff --git a/tools/testing/selftests/kvm/rseq_test.c b/tools/testing/selftests/kvm/rseq_test.c
index 1375fca80bcd..f80ad6b47d16 100644
--- a/tools/testing/selftests/kvm/rseq_test.c
+++ b/tools/testing/selftests/kvm/rseq_test.c
@@ -215,6 +215,7 @@ int main(int argc, char *argv[])
 		switch (opt) {
 		case 'u':
 			skip_sanity_check = true;
+			break;
 		case 'l':
 			latency = atoi_paranoid(optarg);
 			break;
diff --git a/tools/testing/selftests/kvm/x86/cpuid_test.c b/tools/testing/selftests/kvm/x86/cpuid_test.c
index 7b3fda6842bc..f9ed14996977 100644
--- a/tools/testing/selftests/kvm/x86/cpuid_test.c
+++ b/tools/testing/selftests/kvm/x86/cpuid_test.c
@@ -155,6 +155,7 @@ struct kvm_cpuid2 *vcpu_alloc_cpuid(struct kvm_vm *vm, vm_vaddr_t *p_gva, struct
 static void set_cpuid_after_run(struct kvm_vcpu *vcpu)
 {
 	struct kvm_cpuid_entry2 *ent;
+	struct kvm_sregs sregs;
 	int rc;
 	u32 eax, ebx, x;
 
@@ -162,6 +163,20 @@ static void set_cpuid_after_run(struct kvm_vcpu *vcpu)
 	rc = __vcpu_set_cpuid(vcpu);
 	TEST_ASSERT(!rc, "Setting unmodified CPUID after KVM_RUN failed: %d", rc);
 
+	/*
+	 * Toggle CR4 bits that affect dynamic CPUID feature flags to verify
+	 * setting unmodified CPUID succeeds with runtime CPUID updates.
+	 */
+	vcpu_sregs_get(vcpu, &sregs);
+	if (kvm_cpu_has(X86_FEATURE_XSAVE))
+		sregs.cr4 ^= X86_CR4_OSXSAVE;
+	if (kvm_cpu_has(X86_FEATURE_PKU))
+		sregs.cr4 ^= X86_CR4_PKE;
+	vcpu_sregs_set(vcpu, &sregs);
+
+	rc = __vcpu_set_cpuid(vcpu);
+	TEST_ASSERT(!rc, "Setting unmodified CPUID after KVM_RUN failed: %d", rc);
+
 	/* Changing CPU features is forbidden */
 	ent = vcpu_get_cpuid_entry(vcpu, 0x7);
 	ebx = ent->ebx;
diff --git a/tools/testing/selftests/lkdtm/tests.txt b/tools/testing/selftests/lkdtm/tests.txt
index cff124c1eddd..67cd53715d93 100644
--- a/tools/testing/selftests/lkdtm/tests.txt
+++ b/tools/testing/selftests/lkdtm/tests.txt
@@ -1,6 +1,8 @@
 #PANIC
 #PANIC_STOP_IRQOFF Crashes entire system
+#PANIC_IN_HARDIRQ Crashes entire system
 BUG kernel BUG at
+#BUG_IN_HARDIRQ Crashes entire system
 WARNING WARNING:
 WARNING_MESSAGE message trigger
 EXCEPTION
diff --git a/tools/testing/selftests/mm/uffd-unit-tests.c b/tools/testing/selftests/mm/uffd-unit-tests.c
index f4807242c5b2..6f5e404a446c 100644
--- a/tools/testing/selftests/mm/uffd-unit-tests.c
+++ b/tools/testing/selftests/mm/uffd-unit-tests.c
@@ -1317,7 +1317,7 @@ static thread_state thread_state_get(pid_t tid)
 		p = strstr(tmp, header);
 		if (p) {
 			/* For example, "State:\tD (disk sleep)" */
-			c = *(p + sizeof(header) - 1);
+			c = *(p + strlen(header));
 			return c == 'D' ?
 			    THR_STATE_UNINTERRUPTIBLE : THR_STATE_UNKNOWN;
 		}
diff --git a/tools/testing/selftests/net/af_unix/Makefile b/tools/testing/selftests/net/af_unix/Makefile
index 3cd677b72072..4c0375e28bbe 100644
--- a/tools/testing/selftests/net/af_unix/Makefile
+++ b/tools/testing/selftests/net/af_unix/Makefile
@@ -1,4 +1,9 @@
-CFLAGS += $(KHDR_INCLUDES) -Wall -Wflex-array-member-not-at-end
+top_srcdir := ../../../../..
+include $(top_srcdir)/scripts/Makefile.compiler
+
+cc-option = $(call __cc-option, $(CC),,$(1),$(2))
+
+CFLAGS += $(KHDR_INCLUDES) -Wall $(call cc-option,-Wflex-array-member-not-at-end)
 
 TEST_GEN_PROGS := \
 	diag_uid \
diff --git a/tools/testing/selftests/net/forwarding/config b/tools/testing/selftests/net/forwarding/config
index ce64518aaa11..75a6c3d3c1da 100644
--- a/tools/testing/selftests/net/forwarding/config
+++ b/tools/testing/selftests/net/forwarding/config
@@ -29,6 +29,7 @@ CONFIG_NET_ACT_VLAN=m
 CONFIG_NET_CLS_BASIC=m
 CONFIG_NET_CLS_FLOWER=m
 CONFIG_NET_CLS_MATCHALL=m
+CONFIG_NET_CLS_U32=m
 CONFIG_NET_EMATCH=y
 CONFIG_NET_EMATCH_META=m
 CONFIG_NETFILTER=y
diff --git a/tools/testing/selftests/net/forwarding/vxlan_bridge_1q_mc_ul.sh b/tools/testing/selftests/net/forwarding/vxlan_bridge_1q_mc_ul.sh
index 6a570d256e07..2cf4c6d9245b 100755
--- a/tools/testing/selftests/net/forwarding/vxlan_bridge_1q_mc_ul.sh
+++ b/tools/testing/selftests/net/forwarding/vxlan_bridge_1q_mc_ul.sh
@@ -138,13 +138,18 @@ install_capture()
 	defer tc qdisc del dev "$dev" clsact
 
 	tc filter add dev "$dev" ingress proto ip pref 104 \
-	   flower skip_hw ip_proto udp dst_port "$VXPORT" \
-	   action pass
+	   u32 match ip protocol 0x11 0xff \
+	       match u16 "$VXPORT" 0xffff at 0x16 \
+	       match u16 0x0800 0xffff at 0x30 \
+	       action pass
 	defer tc filter del dev "$dev" ingress proto ip pref 104
 
 	tc filter add dev "$dev" ingress proto ipv6 pref 106 \
-	   flower skip_hw ip_proto udp dst_port "$VXPORT" \
-	   action pass
+	   u32 match ip6 protocol 0x11 0xff \
+	       match u16 "$VXPORT" 0xffff at 0x2a \
+	       match u16 0x86dd 0xffff at 0x44 \
+	       match u8 0x11 0xff at 0x4c \
+	       action pass
 	defer tc filter del dev "$dev" ingress proto ipv6 pref 106
 }
 
@@ -248,13 +253,6 @@ vx_create()
 }
 export -f vx_create
 
-vx_wait()
-{
-	# Wait for all the ARP, IGMP etc. noise to settle down so that the
-	# tunnel is clear for measurements.
-	sleep 10
-}
-
 vx10_create()
 {
 	vx_create vx10 10 id 1000 "$@"
@@ -267,18 +265,6 @@ vx20_create()
 }
 export -f vx20_create
 
-vx10_create_wait()
-{
-	vx10_create "$@"
-	vx_wait
-}
-
-vx20_create_wait()
-{
-	vx20_create "$@"
-	vx_wait
-}
-
 ns_init_common()
 {
 	local ns=$1; shift
@@ -554,7 +540,7 @@ ipv4_nomcroute()
 	# Install a misleading (S,G) rule to attempt to trick the system into
 	# pushing the packets elsewhere.
 	adf_install_broken_sg
-	vx10_create_wait local 192.0.2.100 group "$GROUP4" dev "$swp2"
+	vx10_create local 192.0.2.100 group "$GROUP4" dev "$swp2"
 	do_test 4 10 0 "IPv4 nomcroute"
 }
 
@@ -562,7 +548,7 @@ ipv6_nomcroute()
 {
 	# Like for IPv4, install a misleading (S,G).
 	adf_install_broken_sg
-	vx20_create_wait local 2001:db8:4::1 group "$GROUP6" dev "$swp2"
+	vx20_create local 2001:db8:4::1 group "$GROUP6" dev "$swp2"
 	do_test 6 10 0 "IPv6 nomcroute"
 }
 
@@ -581,35 +567,35 @@ ipv6_nomcroute_rx()
 ipv4_mcroute()
 {
 	adf_install_sg
-	vx10_create_wait local 192.0.2.100 group "$GROUP4" dev "$IPMR" mcroute
+	vx10_create local 192.0.2.100 group "$GROUP4" dev "$IPMR" mcroute
 	do_test 4 10 10 "IPv4 mcroute"
 }
 
 ipv6_mcroute()
 {
 	adf_install_sg
-	vx20_create_wait local 2001:db8:4::1 group "$GROUP6" dev "$IPMR" mcroute
+	vx20_create local 2001:db8:4::1 group "$GROUP6" dev "$IPMR" mcroute
 	do_test 6 10 10 "IPv6 mcroute"
 }
 
 ipv4_mcroute_rx()
 {
 	adf_install_sg
-	vx10_create_wait local 192.0.2.100 group "$GROUP4" dev "$IPMR" mcroute
+	vx10_create local 192.0.2.100 group "$GROUP4" dev "$IPMR" mcroute
 	ipv4_do_test_rx 0 "IPv4 mcroute ping"
 }
 
 ipv6_mcroute_rx()
 {
 	adf_install_sg
-	vx20_create_wait local 2001:db8:4::1 group "$GROUP6" dev "$IPMR" mcroute
+	vx20_create local 2001:db8:4::1 group "$GROUP6" dev "$IPMR" mcroute
 	ipv6_do_test_rx 0 "IPv6 mcroute ping"
 }
 
 ipv4_mcroute_changelink()
 {
 	adf_install_sg
-	vx10_create_wait local 192.0.2.100 group "$GROUP4" dev "$IPMR"
+	vx10_create local 192.0.2.100 group "$GROUP4" dev "$IPMR"
 	ip link set dev vx10 type vxlan mcroute
 	sleep 1
 	do_test 4 10 10 "IPv4 mcroute changelink"
@@ -618,7 +604,7 @@ ipv4_mcroute_changelink()
 ipv6_mcroute_changelink()
 {
 	adf_install_sg
-	vx20_create_wait local 2001:db8:4::1 group "$GROUP6" dev "$IPMR" mcroute
+	vx20_create local 2001:db8:4::1 group "$GROUP6" dev "$IPMR" mcroute
 	ip link set dev vx20 type vxlan mcroute
 	sleep 1
 	do_test 6 10 10 "IPv6 mcroute changelink"
@@ -627,47 +613,47 @@ ipv6_mcroute_changelink()
 ipv4_mcroute_starg()
 {
 	adf_install_starg
-	vx10_create_wait local 192.0.2.100 group "$GROUP4" dev "$IPMR" mcroute
+	vx10_create local 192.0.2.100 group "$GROUP4" dev "$IPMR" mcroute
 	do_test 4 10 10 "IPv4 mcroute (*,G)"
 }
 
 ipv6_mcroute_starg()
 {
 	adf_install_starg
-	vx20_create_wait local 2001:db8:4::1 group "$GROUP6" dev "$IPMR" mcroute
+	vx20_create local 2001:db8:4::1 group "$GROUP6" dev "$IPMR" mcroute
 	do_test 6 10 10 "IPv6 mcroute (*,G)"
 }
 
 ipv4_mcroute_starg_rx()
 {
 	adf_install_starg
-	vx10_create_wait local 192.0.2.100 group "$GROUP4" dev "$IPMR" mcroute
+	vx10_create local 192.0.2.100 group "$GROUP4" dev "$IPMR" mcroute
 	ipv4_do_test_rx 0 "IPv4 mcroute (*,G) ping"
 }
 
 ipv6_mcroute_starg_rx()
 {
 	adf_install_starg
-	vx20_create_wait local 2001:db8:4::1 group "$GROUP6" dev "$IPMR" mcroute
+	vx20_create local 2001:db8:4::1 group "$GROUP6" dev "$IPMR" mcroute
 	ipv6_do_test_rx 0 "IPv6 mcroute (*,G) ping"
 }
 
 ipv4_mcroute_noroute()
 {
-	vx10_create_wait local 192.0.2.100 group "$GROUP4" dev "$IPMR" mcroute
+	vx10_create local 192.0.2.100 group "$GROUP4" dev "$IPMR" mcroute
 	do_test 4 0 0 "IPv4 mcroute, no route"
 }
 
 ipv6_mcroute_noroute()
 {
-	vx20_create_wait local 2001:db8:4::1 group "$GROUP6" dev "$IPMR" mcroute
+	vx20_create local 2001:db8:4::1 group "$GROUP6" dev "$IPMR" mcroute
 	do_test 6 0 0 "IPv6 mcroute, no route"
 }
 
 ipv4_mcroute_fdb()
 {
 	adf_install_sg
-	vx10_create_wait local 192.0.2.100 dev "$IPMR" mcroute
+	vx10_create local 192.0.2.100 dev "$IPMR" mcroute
 	bridge fdb add dev vx10 \
 		00:00:00:00:00:00 self static dst "$GROUP4" via "$IPMR"
 	do_test 4 10 10 "IPv4 mcroute FDB"
@@ -676,7 +662,7 @@ ipv4_mcroute_fdb()
 ipv6_mcroute_fdb()
 {
 	adf_install_sg
-	vx20_create_wait local 2001:db8:4::1 dev "$IPMR" mcroute
+	vx20_create local 2001:db8:4::1 dev "$IPMR" mcroute
 	bridge -6 fdb add dev vx20 \
 		00:00:00:00:00:00 self static dst "$GROUP6" via "$IPMR"
 	do_test 6 10 10 "IPv6 mcroute FDB"
@@ -686,7 +672,7 @@ ipv6_mcroute_fdb()
 ipv4_mcroute_fdb_oif0()
 {
 	adf_install_sg
-	vx10_create_wait local 192.0.2.100 group "$GROUP4" dev "$IPMR" mcroute
+	vx10_create local 192.0.2.100 group "$GROUP4" dev "$IPMR" mcroute
 	bridge fdb del dev vx10 00:00:00:00:00:00
 	bridge fdb add dev vx10 00:00:00:00:00:00 self static dst "$GROUP4"
 	do_test 4 10 10 "IPv4 mcroute oif=0"
@@ -703,7 +689,7 @@ ipv6_mcroute_fdb_oif0()
 	defer ip -6 route del table local multicast "$GROUP6/128" dev "$IPMR"
 
 	adf_install_sg
-	vx20_create_wait local 2001:db8:4::1 group "$GROUP6" dev "$IPMR" mcroute
+	vx20_create local 2001:db8:4::1 group "$GROUP6" dev "$IPMR" mcroute
 	bridge -6 fdb del dev vx20 00:00:00:00:00:00
 	bridge -6 fdb add dev vx20 00:00:00:00:00:00 self static dst "$GROUP6"
 	do_test 6 10 10 "IPv6 mcroute oif=0"
@@ -716,7 +702,7 @@ ipv4_mcroute_fdb_oif0_sep()
 	adf_install_sg_sep
 
 	adf_ip_addr_add lo 192.0.2.120/28
-	vx10_create_wait local 192.0.2.120 group "$GROUP4" dev "$IPMR" mcroute
+	vx10_create local 192.0.2.120 group "$GROUP4" dev "$IPMR" mcroute
 	bridge fdb del dev vx10 00:00:00:00:00:00
 	bridge fdb add dev vx10 00:00:00:00:00:00 self static dst "$GROUP4"
 	do_test 4 10 10 "IPv4 mcroute TX!=RX oif=0"
@@ -727,7 +713,7 @@ ipv4_mcroute_fdb_oif0_sep_rx()
 	adf_install_sg_sep_rx lo
 
 	adf_ip_addr_add lo 192.0.2.120/28
-	vx10_create_wait local 192.0.2.120 group "$GROUP4" dev "$IPMR" mcroute
+	vx10_create local 192.0.2.120 group "$GROUP4" dev "$IPMR" mcroute
 	bridge fdb del dev vx10 00:00:00:00:00:00
 	bridge fdb add dev vx10 00:00:00:00:00:00 self static dst "$GROUP4"
 	ipv4_do_test_rx 0 "IPv4 mcroute TX!=RX oif=0 ping"
@@ -738,7 +724,7 @@ ipv4_mcroute_fdb_sep_rx()
 	adf_install_sg_sep_rx lo
 
 	adf_ip_addr_add lo 192.0.2.120/28
-	vx10_create_wait local 192.0.2.120 group "$GROUP4" dev "$IPMR" mcroute
+	vx10_create local 192.0.2.120 group "$GROUP4" dev "$IPMR" mcroute
 	bridge fdb del dev vx10 00:00:00:00:00:00
 	bridge fdb add \
 	       dev vx10 00:00:00:00:00:00 self static dst "$GROUP4" via lo
@@ -750,7 +736,7 @@ ipv6_mcroute_fdb_sep_rx()
 	adf_install_sg_sep_rx "X$IPMR"
 
 	adf_ip_addr_add "X$IPMR" 2001:db8:5::1/64
-	vx20_create_wait local 2001:db8:5::1 group "$GROUP6" dev "$IPMR" mcroute
+	vx20_create local 2001:db8:5::1 group "$GROUP6" dev "$IPMR" mcroute
 	bridge -6 fdb del dev vx20 00:00:00:00:00:00
 	bridge -6 fdb add dev vx20 00:00:00:00:00:00 \
 			  self static dst "$GROUP6" via "X$IPMR"
diff --git a/tools/testing/selftests/net/lib.sh b/tools/testing/selftests/net/lib.sh
index f448bafb3f20..0ec131b339bc 100644
--- a/tools/testing/selftests/net/lib.sh
+++ b/tools/testing/selftests/net/lib.sh
@@ -280,7 +280,8 @@ tc_rule_stats_get()
 	local selector=${1:-.packets}; shift
 
 	tc -j -s filter show dev $dev $dir pref $pref \
-	    | jq ".[1].options.actions[].stats$selector"
+	    | jq ".[] | select(.options.actions) |
+		  .options.actions[].stats$selector"
 }
 
 tc_rule_handle_stats_get()
diff --git a/tools/testing/selftests/net/lib/ksft.h b/tools/testing/selftests/net/lib/ksft.h
index 17dc34a612c6..03912902a6d3 100644
--- a/tools/testing/selftests/net/lib/ksft.h
+++ b/tools/testing/selftests/net/lib/ksft.h
@@ -24,7 +24,8 @@ static inline void ksft_ready(void)
 		fd = STDOUT_FILENO;
 	}
 
-	write(fd, msg, sizeof(msg));
+	if (write(fd, msg, sizeof(msg)) < 0)
+		perror("write()");
 	if (fd != STDOUT_FILENO)
 		close(fd);
 }
@@ -48,7 +49,8 @@ static inline void ksft_wait(void)
 		fd = STDIN_FILENO;
 	}
 
-	read(fd, &byte, sizeof(byte));
+	if (read(fd, &byte, sizeof(byte)) < 0)
+		perror("read()");
 	if (fd != STDIN_FILENO)
 		close(fd);
 }
diff --git a/tools/testing/selftests/net/mptcp/pm_netlink.sh b/tools/testing/selftests/net/mptcp/pm_netlink.sh
index ec6a87588191..123d9d7a0278 100755
--- a/tools/testing/selftests/net/mptcp/pm_netlink.sh
+++ b/tools/testing/selftests/net/mptcp/pm_netlink.sh
@@ -192,6 +192,10 @@ check "show_endpoints" \
 flush_endpoint
 check "show_endpoints" "" "flush addrs"
 
+add_endpoint 10.0.1.1 flags unknown
+check "show_endpoints" "$(format_endpoints "1,10.0.1.1")" "ignore unknown flags"
+flush_endpoint
+
 set_limits 9 1 2>/dev/null
 check "get_limits" "${default_limits}" "rcv addrs above hard limit"
 
diff --git a/tools/testing/selftests/net/mptcp/pm_nl_ctl.c b/tools/testing/selftests/net/mptcp/pm_nl_ctl.c
index 65b374232ff5..99eecccbf0c8 100644
--- a/tools/testing/selftests/net/mptcp/pm_nl_ctl.c
+++ b/tools/testing/selftests/net/mptcp/pm_nl_ctl.c
@@ -24,6 +24,8 @@
 #define IPPROTO_MPTCP 262
 #endif
 
+#define MPTCP_PM_ADDR_FLAG_UNKNOWN _BITUL(7)
+
 static void syntax(char *argv[])
 {
 	fprintf(stderr, "%s add|ann|rem|csf|dsf|get|set|del|flush|dump|events|listen|accept [<args>]\n", argv[0]);
@@ -836,6 +838,8 @@ int add_addr(int fd, int pm_family, int argc, char *argv[])
 					flags |= MPTCP_PM_ADDR_FLAG_BACKUP;
 				else if (!strcmp(tok, "fullmesh"))
 					flags |= MPTCP_PM_ADDR_FLAG_FULLMESH;
+				else if (!strcmp(tok, "unknown"))
+					flags |= MPTCP_PM_ADDR_FLAG_UNKNOWN;
 				else
 					error(1, errno,
 					      "unknown flag %s", argv[arg]);
@@ -1048,6 +1052,13 @@ static void print_addr(struct rtattr *attrs, int len)
 					printf(",");
 			}
 
+			if (flags & MPTCP_PM_ADDR_FLAG_UNKNOWN) {
+				printf("unknown");
+				flags &= ~MPTCP_PM_ADDR_FLAG_UNKNOWN;
+				if (flags)
+					printf(",");
+			}
+
 			/* bump unknown flags, if any */
 			if (flags)
 				printf("0x%x", flags);
diff --git a/tools/testing/selftests/net/netfilter/conntrack_clash.sh b/tools/testing/selftests/net/netfilter/conntrack_clash.sh
index 7fc6c5dbd551..84b8eb12143a 100755
--- a/tools/testing/selftests/net/netfilter/conntrack_clash.sh
+++ b/tools/testing/selftests/net/netfilter/conntrack_clash.sh
@@ -116,7 +116,7 @@ run_one_clash_test()
 	# not a failure: clash resolution logic did not trigger.
 	# With right timing, xmit completed sequentially and
 	# no parallel insertion occurs.
-	return $ksft_skip
+	return $ksft_xfail
 }
 
 run_clash_test()
@@ -133,12 +133,12 @@ run_clash_test()
 		if [ $rv -eq 0 ];then
 			echo "PASS: clash resolution test for $daddr:$dport on attempt $i"
 			return 0
-		elif [ $rv -eq $ksft_skip ]; then
+		elif [ $rv -eq $ksft_xfail ]; then
 			softerr=1
 		fi
 	done
 
-	[ $softerr -eq 1 ] && echo "SKIP: clash resolution for $daddr:$dport did not trigger"
+	[ $softerr -eq 1 ] && echo "XFAIL: clash resolution for $daddr:$dport did not trigger"
 }
 
 ip link add veth0 netns "$nsclient1" type veth peer name veth0 netns "$nsrouter"
@@ -167,8 +167,7 @@ load_simple_ruleset "$nsclient2"
 run_clash_test "$nsclient2" "$nsclient2" 127.0.0.1 9001
 
 if [ $clash_resolution_active -eq 0 ];then
-	[ "$ret" -eq 0 ] && ret=$ksft_skip
-	echo "SKIP: Clash resolution did not trigger"
+	[ "$ret" -eq 0 ] && ret=$ksft_xfail
 fi
 
 exit $ret
diff --git a/tools/testing/selftests/net/netfilter/conntrack_reverse_clash.c b/tools/testing/selftests/net/netfilter/conntrack_reverse_clash.c
index 507930cee8cb..462d628cc3bd 100644
--- a/tools/testing/selftests/net/netfilter/conntrack_reverse_clash.c
+++ b/tools/testing/selftests/net/netfilter/conntrack_reverse_clash.c
@@ -33,9 +33,14 @@ static void die(const char *e)
 	exit(111);
 }
 
-static void die_port(uint16_t got, uint16_t want)
+static void die_port(const struct sockaddr_in *sin, uint16_t want)
 {
-	fprintf(stderr, "Port number changed, wanted %d got %d\n", want, ntohs(got));
+	uint16_t got = ntohs(sin->sin_port);
+	char str[INET_ADDRSTRLEN];
+
+	inet_ntop(AF_INET, &sin->sin_addr, str, sizeof(str));
+
+	fprintf(stderr, "Port number changed, wanted %d got %d from %s\n", want, got, str);
 	exit(1);
 }
 
@@ -100,7 +105,7 @@ int main(int argc, char *argv[])
 				die("child recvfrom");
 
 			if (peer.sin_port != htons(PORT))
-				die_port(peer.sin_port, PORT);
+				die_port(&peer, PORT);
 		} else {
 			if (sendto(s2, buf, LEN, 0, (struct sockaddr *)&sa1, sizeof(sa1)) != LEN)
 				continue;
@@ -109,7 +114,7 @@ int main(int argc, char *argv[])
 				die("parent recvfrom");
 
 			if (peer.sin_port != htons((PORT + 1)))
-				die_port(peer.sin_port, PORT + 1);
+				die_port(&peer, PORT + 1);
 		}
 	}
 
diff --git a/tools/testing/selftests/net/netfilter/conntrack_reverse_clash.sh b/tools/testing/selftests/net/netfilter/conntrack_reverse_clash.sh
index a24c896347a8..dc7e9d6da062 100755
--- a/tools/testing/selftests/net/netfilter/conntrack_reverse_clash.sh
+++ b/tools/testing/selftests/net/netfilter/conntrack_reverse_clash.sh
@@ -45,6 +45,8 @@ if ip netns exec "$ns0" ./conntrack_reverse_clash; then
 	echo "PASS: No SNAT performed for null bindings"
 else
 	echo "ERROR: SNAT performed without any matching snat rule"
+	ip netns exec "$ns0" conntrack -L
+	ip netns exec "$ns0" conntrack -S
 	exit 1
 fi
 
diff --git a/tools/testing/selftests/net/netfilter/packetdrill/conntrack_syn_challenge_ack.pkt b/tools/testing/selftests/net/netfilter/packetdrill/conntrack_syn_challenge_ack.pkt
index 3442cd29bc93..cdb3910af95b 100644
--- a/tools/testing/selftests/net/netfilter/packetdrill/conntrack_syn_challenge_ack.pkt
+++ b/tools/testing/selftests/net/netfilter/packetdrill/conntrack_syn_challenge_ack.pkt
@@ -26,7 +26,7 @@
 
 +0.01 > R 643160523:643160523(0) win 0
 
-+0.01 `conntrack -f $NFCT_IP_VERSION -L -p tcp --dport 8080 2>/dev/null | grep UNREPLIED | grep -q SYN_SENT`
++0.1 `conntrack -f $NFCT_IP_VERSION -L -p tcp --dport 8080 2>/dev/null | grep UNREPLIED | grep -q SYN_SENT`
 
 // Must go through.
 +0.01 > S 0:0(0) win 65535 <mss 1460,sackOK,TS val 1 ecr 0,nop,wscale 8>
diff --git a/tools/testing/selftests/net/tfo.c b/tools/testing/selftests/net/tfo.c
index eb3cac5e583c..8d82140f0f76 100644
--- a/tools/testing/selftests/net/tfo.c
+++ b/tools/testing/selftests/net/tfo.c
@@ -81,7 +81,8 @@ static void run_server(void)
 	if (getsockopt(connfd, SOL_SOCKET, SO_INCOMING_NAPI_ID, &opt, &len) < 0)
 		error(1, errno, "getsockopt(SO_INCOMING_NAPI_ID)");
 
-	read(connfd, buf, 64);
+	if (read(connfd, buf, 64) < 0)
+		perror("read()");
 	fprintf(outfile, "%d\n", opt);
 
 	fclose(outfile);
diff --git a/tools/testing/selftests/net/tls.c b/tools/testing/selftests/net/tls.c
index a3ef4b57eb5f..a4d16a460fbe 100644
--- a/tools/testing/selftests/net/tls.c
+++ b/tools/testing/selftests/net/tls.c
@@ -2786,10 +2786,10 @@ TEST_F(tls_err, epoll_partial_rec)
 TEST_F(tls_err, poll_partial_rec_async)
 {
 	struct pollfd pfd = { };
+	char token = '\0';
 	ssize_t rec_len;
 	char rec[256];
 	char buf[128];
-	char token;
 	int p[2];
 	int ret;
 
diff --git a/tools/testing/selftests/nolibc/nolibc-test.c b/tools/testing/selftests/nolibc/nolibc-test.c
index 3c5a226dad3a..6888b20af259 100644
--- a/tools/testing/selftests/nolibc/nolibc-test.c
+++ b/tools/testing/selftests/nolibc/nolibc-test.c
@@ -17,6 +17,7 @@
 #include <sys/mman.h>
 #include <sys/mount.h>
 #include <sys/prctl.h>
+#include <sys/ptrace.h>
 #include <sys/random.h>
 #include <sys/reboot.h>
 #include <sys/resource.h>
@@ -1406,6 +1407,7 @@ int run_syscall(int min, int max)
 		CASE_TEST(readv_zero);        EXPECT_SYSZR(1, readv(1, NULL, 0)); break;
 		CASE_TEST(writev_badf);       EXPECT_SYSER(1, writev(-1, &iov_one, 1), -1, EBADF); break;
 		CASE_TEST(writev_zero);       EXPECT_SYSZR(1, writev(1, NULL, 0)); break;
+		CASE_TEST(ptrace);            EXPECT_SYSER(1, ptrace(PTRACE_CONT, getpid(), NULL, NULL), -1, ESRCH); break;
 		CASE_TEST(syscall_noargs);    EXPECT_SYSEQ(1, syscall(__NR_getpid), getpid()); break;
 		CASE_TEST(syscall_args);      EXPECT_SYSER(1, syscall(__NR_statx, 0, NULL, 0, 0, NULL), -1, EFAULT); break;
 		CASE_TEST(namespace);         EXPECT_SYSZR(euid0 && proc, test_namespace()); break;
diff --git a/tools/testing/selftests/riscv/Makefile b/tools/testing/selftests/riscv/Makefile
index 099b8c1f46f8..5671b4405a12 100644
--- a/tools/testing/selftests/riscv/Makefile
+++ b/tools/testing/selftests/riscv/Makefile
@@ -5,7 +5,7 @@
 ARCH ?= $(shell uname -m 2>/dev/null || echo not)
 
 ifneq (,$(filter $(ARCH),riscv))
-RISCV_SUBTARGETS ?= abi hwprobe mm sigreturn vector
+RISCV_SUBTARGETS ?= abi hwprobe mm sigreturn vector cfi
 else
 RISCV_SUBTARGETS :=
 endif
diff --git a/tools/testing/selftests/riscv/cfi/.gitignore b/tools/testing/selftests/riscv/cfi/.gitignore
new file mode 100644
index 000000000000..c1faf7ca4346
--- /dev/null
+++ b/tools/testing/selftests/riscv/cfi/.gitignore
@@ -0,0 +1,2 @@
+cfitests
+shadowstack
diff --git a/tools/testing/selftests/riscv/cfi/Makefile b/tools/testing/selftests/riscv/cfi/Makefile
new file mode 100644
index 000000000000..96a4dc4b69c3
--- /dev/null
+++ b/tools/testing/selftests/riscv/cfi/Makefile
@@ -0,0 +1,23 @@
+CFLAGS += $(KHDR_INCLUDES)
+CFLAGS += -I$(top_srcdir)/tools/include
+
+CFLAGS += -march=rv64gc_zicfilp_zicfiss -fcf-protection=full
+
+# Check for zicfi* extensions needs cross compiler
+# which is not set until lib.mk is included
+ifeq ($(LLVM)$(CC),cc)
+CC := $(CROSS_COMPILE)gcc
+endif
+
+
+ifeq ($(shell $(CC) $(CFLAGS) -nostdlib -xc /dev/null -o /dev/null > /dev/null 2>&1; echo $$?),0)
+TEST_GEN_PROGS := cfitests
+
+$(OUTPUT)/cfitests: cfitests.c shadowstack.c
+	$(CC) -o$@ $(CFLAGS) $(LDFLAGS) $^
+else
+
+$(shell echo "Toolchain doesn't support CFI, skipping CFI kselftest." >&2)
+endif
+
+include ../../lib.mk
diff --git a/tools/testing/selftests/riscv/cfi/cfi_rv_test.h b/tools/testing/selftests/riscv/cfi/cfi_rv_test.h
new file mode 100644
index 000000000000..1c8043f2b778
--- /dev/null
+++ b/tools/testing/selftests/riscv/cfi/cfi_rv_test.h
@@ -0,0 +1,82 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#ifndef SELFTEST_RISCV_CFI_H
+#define SELFTEST_RISCV_CFI_H
+#include <stddef.h>
+#include <sys/types.h>
+#include "shadowstack.h"
+
+#define CHILD_EXIT_CODE_SSWRITE		10
+#define CHILD_EXIT_CODE_SIG_TEST	11
+
+#define my_syscall5(num, arg1, arg2, arg3, arg4, arg5)			\
+({									\
+	register long _num  __asm__ ("a7") = (num);			\
+	register long _arg1 __asm__ ("a0") = (long)(arg1);		\
+	register long _arg2 __asm__ ("a1") = (long)(arg2);		\
+	register long _arg3 __asm__ ("a2") = (long)(arg3);		\
+	register long _arg4 __asm__ ("a3") = (long)(arg4);		\
+	register long _arg5 __asm__ ("a4") = (long)(arg5);		\
+									\
+	__asm__ volatile(						\
+		"ecall\n"						\
+		: "+r"							\
+		(_arg1)							\
+		: "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5),	\
+		  "r"(_num)						\
+		: "memory", "cc"					\
+	);								\
+	_arg1;								\
+})
+
+#define my_syscall3(num, arg1, arg2, arg3)				\
+({									\
+	register long _num  __asm__ ("a7") = (num);			\
+	register long _arg1 __asm__ ("a0") = (long)(arg1);		\
+	register long _arg2 __asm__ ("a1") = (long)(arg2);		\
+	register long _arg3 __asm__ ("a2") = (long)(arg3);		\
+									\
+	__asm__ volatile(						\
+		"ecall\n"						\
+		: "+r" (_arg1)						\
+		: "r"(_arg2), "r"(_arg3),				\
+		  "r"(_num)						\
+		: "memory", "cc"					\
+	);								\
+	_arg1;								\
+})
+
+#ifndef __NR_prctl
+#define __NR_prctl 167
+#endif
+
+#ifndef __NR_map_shadow_stack
+#define __NR_map_shadow_stack 453
+#endif
+
+#define CSR_SSP 0x011
+
+#ifdef __ASSEMBLY__
+#define __ASM_STR(x)    x
+#else
+#define __ASM_STR(x)    #x
+#endif
+
+#define csr_read(csr)							\
+({									\
+	register unsigned long __v;					\
+	__asm__ __volatile__ ("csrr %0, " __ASM_STR(csr)		\
+				: "=r" (__v) :				\
+				: "memory");				\
+	__v;								\
+})
+
+#define csr_write(csr, val)						\
+({									\
+	unsigned long __v = (unsigned long)(val);			\
+	__asm__ __volatile__ ("csrw " __ASM_STR(csr) ", %0"		\
+				: : "rK" (__v)				\
+				: "memory");				\
+})
+
+#endif
diff --git a/tools/testing/selftests/riscv/cfi/cfitests.c b/tools/testing/selftests/riscv/cfi/cfitests.c
new file mode 100644
index 000000000000..d26f45acd3b6
--- /dev/null
+++ b/tools/testing/selftests/riscv/cfi/cfitests.c
@@ -0,0 +1,173 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include "../../kselftest.h"
+#include <sys/signal.h>
+#include <asm/ucontext.h>
+#include <linux/prctl.h>
+#include <errno.h>
+#include <linux/ptrace.h>
+#include <sys/wait.h>
+#include <linux/elf.h>
+#include <sys/uio.h>
+#include <asm-generic/unistd.h>
+
+#include "cfi_rv_test.h"
+
+/* do not optimize cfi related test functions */
+#pragma GCC push_options
+#pragma GCC optimize("O0")
+
+void sigsegv_handler(int signum, siginfo_t *si, void *uc)
+{
+	struct ucontext *ctx = (struct ucontext *)uc;
+
+	if (si->si_code == SEGV_CPERR) {
+		ksft_print_msg("Control flow violation happened somewhere\n");
+		ksft_print_msg("PC where violation happened %lx\n", ctx->uc_mcontext.gregs[0]);
+		exit(-1);
+	}
+
+	/* all other cases are expected to be of shadow stack write case */
+	exit(CHILD_EXIT_CODE_SSWRITE);
+}
+
+bool register_signal_handler(void)
+{
+	struct sigaction sa = {};
+
+	sa.sa_sigaction = sigsegv_handler;
+	sa.sa_flags = SA_SIGINFO;
+	if (sigaction(SIGSEGV, &sa, NULL)) {
+		ksft_print_msg("Registering signal handler for landing pad violation failed\n");
+		return false;
+	}
+
+	return true;
+}
+
+long ptrace(int request, pid_t pid, void *addr, void *data);
+
+bool cfi_ptrace_test(void)
+{
+	pid_t pid;
+	int status, ret = 0;
+	unsigned long ptrace_test_num = 0, total_ptrace_tests = 2;
+
+	struct user_cfi_state cfi_reg;
+	struct iovec iov;
+
+	pid = fork();
+
+	if (pid == -1) {
+		ksft_exit_fail_msg("%s: fork failed\n", __func__);
+		exit(1);
+	}
+
+	if (pid == 0) {
+		/* allow to be traced */
+		ptrace(PTRACE_TRACEME, 0, NULL, NULL);
+		raise(SIGSTOP);
+		asm volatile ("la a5, 1f\n"
+			      "jalr a5\n"
+			      "nop\n"
+			      "nop\n"
+			      "1: nop\n"
+			      : : : "a5");
+		exit(11);
+		/* child shouldn't go beyond here */
+	}
+
+	/* parent's code goes here */
+	iov.iov_base = &cfi_reg;
+	iov.iov_len = sizeof(cfi_reg);
+
+	while (ptrace_test_num < total_ptrace_tests) {
+		memset(&cfi_reg, 0, sizeof(cfi_reg));
+		waitpid(pid, &status, 0);
+		if (WIFSTOPPED(status)) {
+			errno = 0;
+			ret = ptrace(PTRACE_GETREGSET, pid, (void *)NT_RISCV_USER_CFI, &iov);
+			if (ret == -1 && errno)
+				ksft_exit_fail_msg("%s: PTRACE_GETREGSET failed\n", __func__);
+		} else {
+			ksft_exit_fail_msg("%s: child didn't stop, failed\n", __func__);
+		}
+
+		switch (ptrace_test_num) {
+#define CFI_ENABLE_MASK (PTRACE_CFI_LP_EN_STATE |	\
+			 PTRACE_CFI_SS_EN_STATE |	\
+			 PTRACE_CFI_SS_PTR_STATE)
+		case 0:
+			if ((cfi_reg.cfi_status.cfi_state & CFI_ENABLE_MASK) != CFI_ENABLE_MASK)
+				ksft_exit_fail_msg("%s: ptrace_getregset failed, %llu\n", __func__,
+						   cfi_reg.cfi_status.cfi_state);
+			if (!cfi_reg.shstk_ptr)
+				ksft_exit_fail_msg("%s: NULL shadow stack pointer, test failed\n",
+						   __func__);
+			break;
+		case 1:
+			if (!(cfi_reg.cfi_status.cfi_state & PTRACE_CFI_ELP_STATE))
+				ksft_exit_fail_msg("%s: elp must have been set\n", __func__);
+			/* clear elp state. not interested in anything else */
+			cfi_reg.cfi_status.cfi_state = 0;
+
+			ret = ptrace(PTRACE_SETREGSET, pid, (void *)NT_RISCV_USER_CFI, &iov);
+			if (ret == -1 && errno)
+				ksft_exit_fail_msg("%s: PTRACE_GETREGSET failed\n", __func__);
+			break;
+		default:
+			ksft_exit_fail_msg("%s: unreachable switch case\n", __func__);
+			break;
+		}
+		ptrace(PTRACE_CONT, pid, NULL, NULL);
+		ptrace_test_num++;
+	}
+
+	waitpid(pid, &status, 0);
+	if (WEXITSTATUS(status) != 11)
+		ksft_print_msg("%s, bad return code from child\n", __func__);
+
+	ksft_print_msg("%s, ptrace test succeeded\n", __func__);
+	return true;
+}
+
+int main(int argc, char *argv[])
+{
+	int ret = 0;
+	unsigned long lpad_status = 0, ss_status = 0;
+
+	ksft_print_header();
+
+	ksft_print_msg("Starting risc-v tests\n");
+
+	/*
+	 * Landing pad test. Not a lot of kernel changes to support landing
+	 * pad for user mode except lighting up a bit in senvcfg via a prctl
+	 * Enable landing pad through out the execution of test binary
+	 */
+	ret = my_syscall5(__NR_prctl, PR_GET_INDIR_BR_LP_STATUS, &lpad_status, 0, 0, 0);
+	if (ret)
+		ksft_exit_fail_msg("Get landing pad status failed with %d\n", ret);
+
+	if (!(lpad_status & PR_INDIR_BR_LP_ENABLE))
+		ksft_exit_fail_msg("Landing pad is not enabled, should be enabled via glibc\n");
+
+	ret = my_syscall5(__NR_prctl, PR_GET_SHADOW_STACK_STATUS, &ss_status, 0, 0, 0);
+	if (ret)
+		ksft_exit_fail_msg("Get shadow stack failed with %d\n", ret);
+
+	if (!(ss_status & PR_SHADOW_STACK_ENABLE))
+		ksft_exit_fail_msg("Shadow stack is not enabled, should be enabled via glibc\n");
+
+	if (!register_signal_handler())
+		ksft_exit_fail_msg("Registering signal handler for SIGSEGV failed\n");
+
+	ksft_print_msg("Landing pad and shadow stack are enabled for binary\n");
+	cfi_ptrace_test();
+
+	execute_shadow_stack_tests();
+
+	return 0;
+}
+
+#pragma GCC pop_options
diff --git a/tools/testing/selftests/riscv/cfi/shadowstack.c b/tools/testing/selftests/riscv/cfi/shadowstack.c
new file mode 100644
index 000000000000..53387dbd9cf5
--- /dev/null
+++ b/tools/testing/selftests/riscv/cfi/shadowstack.c
@@ -0,0 +1,385 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include "../../kselftest.h"
+#include <sys/wait.h>
+#include <signal.h>
+#include <fcntl.h>
+#include <asm-generic/unistd.h>
+#include <sys/mman.h>
+#include "shadowstack.h"
+#include "cfi_rv_test.h"
+
+static struct shadow_stack_tests shstk_tests[] = {
+	{ "shstk fork test\n", shadow_stack_fork_test },
+	{ "map shadow stack syscall\n", shadow_stack_map_test },
+	{ "shadow stack gup tests\n", shadow_stack_gup_tests },
+	{ "shadow stack signal tests\n", shadow_stack_signal_test},
+	{ "memory protections of shadow stack memory\n", shadow_stack_protection_test }
+};
+
+#define RISCV_SHADOW_STACK_TESTS ARRAY_SIZE(shstk_tests)
+
+/* do not optimize shadow stack related test functions */
+#pragma GCC push_options
+#pragma GCC optimize("O0")
+
+void zar(void)
+{
+	unsigned long ssp = 0;
+
+	ssp = csr_read(CSR_SSP);
+	ksft_print_msg("Spewing out shadow stack ptr: %lx\n"
+			"  This is to ensure shadow stack is indeed enabled and working\n",
+			ssp);
+}
+
+void bar(void)
+{
+	zar();
+}
+
+void foo(void)
+{
+	bar();
+}
+
+void zar_child(void)
+{
+	unsigned long ssp = 0;
+
+	ssp = csr_read(CSR_SSP);
+	ksft_print_msg("Spewing out shadow stack ptr: %lx\n"
+			"  This is to ensure shadow stack is indeed enabled and working\n",
+			ssp);
+}
+
+void bar_child(void)
+{
+	zar_child();
+}
+
+void foo_child(void)
+{
+	bar_child();
+}
+
+typedef void (call_func_ptr)(void);
+/*
+ * call couple of functions to test push pop.
+ */
+int shadow_stack_call_tests(call_func_ptr fn_ptr, bool parent)
+{
+	ksft_print_msg("dummy calls for sspush and sspopchk in context of %s\n",
+		       parent ? "parent" : "child");
+
+	(fn_ptr)();
+
+	return 0;
+}
+
+/* forks a thread, and ensure shadow stacks fork out */
+bool shadow_stack_fork_test(unsigned long test_num, void *ctx)
+{
+	int pid = 0, child_status = 0, parent_pid = 0, ret = 0;
+	unsigned long ss_status = 0;
+
+	ksft_print_msg("Exercising shadow stack fork test\n");
+
+	ret = my_syscall5(__NR_prctl, PR_GET_SHADOW_STACK_STATUS, &ss_status, 0, 0, 0);
+	if (ret) {
+		ksft_exit_skip("Shadow stack get status prctl failed with errorcode %d\n", ret);
+		return false;
+	}
+
+	if (!(ss_status & PR_SHADOW_STACK_ENABLE))
+		ksft_exit_skip("Shadow stack is not enabled, should be enabled via glibc\n");
+
+	parent_pid = getpid();
+	pid = fork();
+
+	if (pid) {
+		ksft_print_msg("Parent pid %d and child pid %d\n", parent_pid, pid);
+		shadow_stack_call_tests(&foo, true);
+	} else {
+		shadow_stack_call_tests(&foo_child, false);
+	}
+
+	if (pid) {
+		ksft_print_msg("Waiting on child to finish\n");
+		wait(&child_status);
+	} else {
+		/* exit child gracefully */
+		exit(0);
+	}
+
+	if (pid && WIFSIGNALED(child_status)) {
+		ksft_print_msg("Child faulted, fork test failed\n");
+		return false;
+	}
+
+	return true;
+}
+
+/* exercise `map_shadow_stack`, pivot to it and call some functions to ensure it works */
+#define SHADOW_STACK_ALLOC_SIZE 4096
+bool shadow_stack_map_test(unsigned long test_num, void *ctx)
+{
+	unsigned long shdw_addr;
+	int ret = 0;
+
+	ksft_print_msg("Exercising shadow stack map test\n");
+
+	shdw_addr = my_syscall3(__NR_map_shadow_stack, NULL, SHADOW_STACK_ALLOC_SIZE, 0);
+
+	if (((long)shdw_addr) <= 0) {
+		ksft_print_msg("map_shadow_stack failed with error code %d\n",
+			       (int)shdw_addr);
+		return false;
+	}
+
+	ret = munmap((void *)shdw_addr, SHADOW_STACK_ALLOC_SIZE);
+
+	if (ret) {
+		ksft_print_msg("munmap failed with error code %d\n", ret);
+		return false;
+	}
+
+	return true;
+}
+
+/*
+ * shadow stack protection tests. map a shadow stack and
+ * validate all memory protections work on it
+ */
+bool shadow_stack_protection_test(unsigned long test_num, void *ctx)
+{
+	unsigned long shdw_addr;
+	unsigned long *write_addr = NULL;
+	int ret = 0, pid = 0, child_status = 0;
+
+	ksft_print_msg("Exercising shadow stack protection test (WPT)\n");
+
+	shdw_addr = my_syscall3(__NR_map_shadow_stack, NULL, SHADOW_STACK_ALLOC_SIZE, 0);
+
+	if (((long)shdw_addr) <= 0) {
+		ksft_print_msg("map_shadow_stack failed with error code %d\n",
+			       (int)shdw_addr);
+		return false;
+	}
+
+	write_addr = (unsigned long *)shdw_addr;
+	pid = fork();
+
+	/* no child was created, return false */
+	if (pid == -1)
+		return false;
+
+	/*
+	 * try to perform a store from child on shadow stack memory
+	 * it should result in SIGSEGV
+	 */
+	if (!pid) {
+		/* below write must lead to SIGSEGV */
+		*write_addr = 0xdeadbeef;
+	} else {
+		wait(&child_status);
+	}
+
+	/* test fail, if 0xdeadbeef present on shadow stack address */
+	if (*write_addr == 0xdeadbeef) {
+		ksft_print_msg("Shadow stack WPT failed\n");
+		return false;
+	}
+
+	/* if child reached here, then fail */
+	if (!pid) {
+		ksft_print_msg("Shadow stack WPT failed: child reached unreachable state\n");
+		return false;
+	}
+
+	/* if child exited via signal handler but not for write on ss */
+	if (WIFEXITED(child_status) &&
+	    WEXITSTATUS(child_status) != CHILD_EXIT_CODE_SSWRITE) {
+		ksft_print_msg("Shadow stack WPT failed: child wasn't signaled for write\n");
+		return false;
+	}
+
+	ret = munmap(write_addr, SHADOW_STACK_ALLOC_SIZE);
+	if (ret) {
+		ksft_print_msg("Shadow stack WPT failed: munmap failed, error code %d\n",
+			       ret);
+		return false;
+	}
+
+	return true;
+}
+
+#define SS_MAGIC_WRITE_VAL 0xbeefdead
+
+int gup_tests(int mem_fd, unsigned long *shdw_addr)
+{
+	unsigned long val = 0;
+
+	lseek(mem_fd, (unsigned long)shdw_addr, SEEK_SET);
+	if (read(mem_fd, &val, sizeof(val)) < 0) {
+		ksft_print_msg("Reading shadow stack mem via gup failed\n");
+		return 1;
+	}
+
+	val = SS_MAGIC_WRITE_VAL;
+	lseek(mem_fd, (unsigned long)shdw_addr, SEEK_SET);
+	if (write(mem_fd, &val, sizeof(val)) < 0) {
+		ksft_print_msg("Writing shadow stack mem via gup failed\n");
+		return 1;
+	}
+
+	if (*shdw_addr != SS_MAGIC_WRITE_VAL) {
+		ksft_print_msg("GUP write to shadow stack memory failed\n");
+		return 1;
+	}
+
+	return 0;
+}
+
+bool shadow_stack_gup_tests(unsigned long test_num, void *ctx)
+{
+	unsigned long shdw_addr = 0;
+	unsigned long *write_addr = NULL;
+	int fd = 0;
+	bool ret = false;
+
+	ksft_print_msg("Exercising shadow stack gup tests\n");
+	shdw_addr = my_syscall3(__NR_map_shadow_stack, NULL, SHADOW_STACK_ALLOC_SIZE, 0);
+
+	if (((long)shdw_addr) <= 0) {
+		ksft_print_msg("map_shadow_stack failed with error code %d\n", (int)shdw_addr);
+		return false;
+	}
+
+	write_addr = (unsigned long *)shdw_addr;
+
+	fd = open("/proc/self/mem", O_RDWR);
+	if (fd == -1)
+		return false;
+
+	if (gup_tests(fd, write_addr)) {
+		ksft_print_msg("gup tests failed\n");
+		goto out;
+	}
+
+	ret = true;
+out:
+	if (shdw_addr && munmap(write_addr, SHADOW_STACK_ALLOC_SIZE)) {
+		ksft_print_msg("munmap failed with error code %d\n", ret);
+		ret = false;
+	}
+
+	return ret;
+}
+
+volatile bool break_loop;
+
+void sigusr1_handler(int signo)
+{
+	break_loop = true;
+}
+
+bool sigusr1_signal_test(void)
+{
+	struct sigaction sa = {};
+
+	sa.sa_handler = sigusr1_handler;
+	sa.sa_flags = 0;
+	sigemptyset(&sa.sa_mask);
+	if (sigaction(SIGUSR1, &sa, NULL)) {
+		ksft_print_msg("Registering signal handler for SIGUSR1 failed\n");
+		return false;
+	}
+
+	return true;
+}
+
+/*
+ * shadow stack signal test. shadow stack must be enabled.
+ * register a signal, fork another thread which is waiting
+ * on signal. Send a signal from parent to child, verify
+ * that signal was received by child. If not test fails
+ */
+bool shadow_stack_signal_test(unsigned long test_num, void *ctx)
+{
+	int pid = 0, child_status = 0, ret = 0;
+	unsigned long ss_status = 0;
+
+	ksft_print_msg("Exercising shadow stack signal test\n");
+
+	ret = my_syscall5(__NR_prctl, PR_GET_SHADOW_STACK_STATUS, &ss_status, 0, 0, 0);
+	if (ret) {
+		ksft_print_msg("Shadow stack get status prctl failed with errorcode %d\n", ret);
+		return false;
+	}
+
+	if (!(ss_status & PR_SHADOW_STACK_ENABLE))
+		ksft_print_msg("Shadow stack is not enabled, should be enabled via glibc\n");
+
+	/* this should be caught by signal handler and do an exit */
+	if (!sigusr1_signal_test()) {
+		ksft_print_msg("Registering sigusr1 handler failed\n");
+		exit(-1);
+	}
+
+	pid = fork();
+
+	if (pid == -1) {
+		ksft_print_msg("Signal test: fork failed\n");
+		goto out;
+	}
+
+	if (pid == 0) {
+		while (!break_loop)
+			sleep(1);
+
+		exit(11);
+		/* child shouldn't go beyond here */
+	}
+
+	/* send SIGUSR1 to child */
+	kill(pid, SIGUSR1);
+	wait(&child_status);
+
+out:
+
+	return (WIFEXITED(child_status) &&
+		WEXITSTATUS(child_status) == 11);
+}
+
+int execute_shadow_stack_tests(void)
+{
+	int ret = 0;
+	unsigned long test_count = 0;
+	unsigned long shstk_status = 0;
+	bool test_pass = false;
+
+	ksft_print_msg("Executing RISC-V shadow stack self tests\n");
+	ksft_set_plan(RISCV_SHADOW_STACK_TESTS);
+
+	ret = my_syscall5(__NR_prctl, PR_GET_SHADOW_STACK_STATUS, &shstk_status, 0, 0, 0);
+
+	if (ret != 0)
+		ksft_exit_fail_msg("Get shadow stack status failed with %d\n", ret);
+
+	/*
+	 * If we are here that means get shadow stack status succeeded and
+	 * thus shadow stack support is baked in the kernel.
+	 */
+	while (test_count < RISCV_SHADOW_STACK_TESTS) {
+		test_pass = (*shstk_tests[test_count].t_func)(test_count, NULL);
+		ksft_test_result(test_pass, shstk_tests[test_count].name);
+		test_count++;
+	}
+
+	ksft_finished();
+
+	return 0;
+}
+
+#pragma GCC pop_options
diff --git a/tools/testing/selftests/riscv/cfi/shadowstack.h b/tools/testing/selftests/riscv/cfi/shadowstack.h
new file mode 100644
index 000000000000..0be510167de3
--- /dev/null
+++ b/tools/testing/selftests/riscv/cfi/shadowstack.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#ifndef SELFTEST_SHADOWSTACK_TEST_H
+#define SELFTEST_SHADOWSTACK_TEST_H
+#include <stddef.h>
+#include <linux/prctl.h>
+
+/*
+ * a cfi test returns true for success or false for fail
+ * takes a number for test number to index into array and void pointer.
+ */
+typedef bool (*shstk_test_func)(unsigned long test_num, void *);
+
+struct shadow_stack_tests {
+	char *name;
+	shstk_test_func t_func;
+};
+
+bool shadow_stack_fork_test(unsigned long test_num, void *ctx);
+bool shadow_stack_map_test(unsigned long test_num, void *ctx);
+bool shadow_stack_protection_test(unsigned long test_num, void *ctx);
+bool shadow_stack_gup_tests(unsigned long test_num, void *ctx);
+bool shadow_stack_signal_test(unsigned long test_num, void *ctx);
+
+int execute_shadow_stack_tests(void);
+
+#endif
diff --git a/tools/testing/selftests/riscv/hwprobe/which-cpus.c b/tools/testing/selftests/riscv/hwprobe/which-cpus.c
index 3ab53067e8dd..587feb198c04 100644
--- a/tools/testing/selftests/riscv/hwprobe/which-cpus.c
+++ b/tools/testing/selftests/riscv/hwprobe/which-cpus.c
@@ -83,9 +83,9 @@ static void do_which_cpus(int argc, char **argv, cpu_set_t *cpus)
 
 int main(int argc, char **argv)
 {
-	struct riscv_hwprobe pairs[2];
+	struct riscv_hwprobe pairs[3];
 	cpu_set_t cpus_aff, cpus;
-	__u64 ext0_all;
+	__u64 ext0_all, ext1_all;
 	long rc;
 
 	rc = sched_getaffinity(0, sizeof(cpu_set_t), &cpus_aff);
@@ -112,6 +112,11 @@ int main(int argc, char **argv)
 	assert(rc == 0 && pairs[0].key == RISCV_HWPROBE_KEY_IMA_EXT_0);
 	ext0_all = pairs[0].value;
 
+	pairs[0] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_IMA_EXT_1, };
+	rc = riscv_hwprobe(pairs, 1, 0, NULL, 0);
+	assert(rc == 0 && pairs[0].key == RISCV_HWPROBE_KEY_IMA_EXT_1);
+	ext1_all = pairs[0].value;
+
 	pairs[0] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_BASE_BEHAVIOR, .value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA, };
 	CPU_ZERO(&cpus);
 	rc = riscv_hwprobe(pairs, 1, 0, (unsigned long *)&cpus, RISCV_HWPROBE_WHICH_CPUS);
@@ -134,20 +139,23 @@ int main(int argc, char **argv)
 
 	pairs[0] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_BASE_BEHAVIOR, .value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA, };
 	pairs[1] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_IMA_EXT_0, .value = ext0_all, };
+	pairs[2] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_IMA_EXT_1, .value = ext1_all, };
 	CPU_ZERO(&cpus);
-	rc = riscv_hwprobe(pairs, 2, sizeof(cpu_set_t), (unsigned long *)&cpus, RISCV_HWPROBE_WHICH_CPUS);
+	rc = riscv_hwprobe(pairs, 3, sizeof(cpu_set_t), (unsigned long *)&cpus, RISCV_HWPROBE_WHICH_CPUS);
 	ksft_test_result(rc == 0 && CPU_COUNT(&cpus) == sysconf(_SC_NPROCESSORS_ONLN), "set all cpus\n");
 
 	pairs[0] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_BASE_BEHAVIOR, .value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA, };
 	pairs[1] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_IMA_EXT_0, .value = ext0_all, };
+	pairs[2] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_IMA_EXT_1, .value = ext1_all, };
 	memcpy(&cpus, &cpus_aff, sizeof(cpu_set_t));
-	rc = riscv_hwprobe(pairs, 2, sizeof(cpu_set_t), (unsigned long *)&cpus, RISCV_HWPROBE_WHICH_CPUS);
+	rc = riscv_hwprobe(pairs, 3, sizeof(cpu_set_t), (unsigned long *)&cpus, RISCV_HWPROBE_WHICH_CPUS);
 	ksft_test_result(rc == 0 && CPU_EQUAL(&cpus, &cpus_aff), "set all affinity cpus\n");
 
 	pairs[0] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_BASE_BEHAVIOR, .value = RISCV_HWPROBE_BASE_BEHAVIOR_IMA, };
 	pairs[1] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_IMA_EXT_0, .value = ~ext0_all, };
+	pairs[2] = (struct riscv_hwprobe){ .key = RISCV_HWPROBE_KEY_IMA_EXT_1, .value = ~ext1_all, };
 	memcpy(&cpus, &cpus_aff, sizeof(cpu_set_t));
-	rc = riscv_hwprobe(pairs, 2, sizeof(cpu_set_t), (unsigned long *)&cpus, RISCV_HWPROBE_WHICH_CPUS);
+	rc = riscv_hwprobe(pairs, 3, sizeof(cpu_set_t), (unsigned long *)&cpus, RISCV_HWPROBE_WHICH_CPUS);
 	ksft_test_result(rc == 0 && CPU_COUNT(&cpus) == 0, "clear all cpus\n");
 
 	ksft_finished();
diff --git a/tools/testing/selftests/sched_ext/runner.c b/tools/testing/selftests/sched_ext/runner.c
index aa2d7d32dda9..5748d2c69903 100644
--- a/tools/testing/selftests/sched_ext/runner.c
+++ b/tools/testing/selftests/sched_ext/runner.c
@@ -46,6 +46,14 @@ static void print_test_preamble(const struct scx_test *test, bool quiet)
 	if (!quiet)
 		printf("DESCRIPTION: %s\n", test->description);
 	printf("OUTPUT:\n");
+
+	/*
+	 * The tests may fork with the preamble buffered
+	 * in the children's stdout. Flush before the test
+	 * to avoid printing the message multiple times.
+	 */
+	fflush(stdout);
+	fflush(stderr);
 }
 
 static const char *status_to_result(enum scx_test_status status)
diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/mirred.json b/tools/testing/selftests/tc-testing/tc-tests/actions/mirred.json
index b73bd255ea36..da156feabcbf 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/actions/mirred.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/actions/mirred.json
@@ -1052,5 +1052,51 @@
             "$TC qdisc del dev $DEV1 ingress_block 21 clsact",
             "$TC actions flush action mirred"
         ]
+    },
+    {
+        "id": "7eba",
+        "name": "Redirect multiport: dummy egress ->  dummy egress (Loop)",
+        "category": [
+            "filter",
+            "mirred"
+        ],
+        "plugins": {
+            "requires": [
+                "nsPlugin"
+            ]
+        },
+        "setup": [
+            "$IP link set dev $DUMMY up || true",
+            "$IP addr add 10.10.10.10/24 dev $DUMMY || true",
+            "$TC qdisc add dev $DUMMY handle 1: root drr",
+            "$TC filter add dev $DUMMY parent 1: protocol ip prio 10 matchall action mirred egress redirect dev $DUMMY index 1"
+        ],
+        "cmdUnderTest": "ping -c1 -W0.01 -I $DUMMY 10.10.10.1",
+        "expExitCode": "1",
+        "verifyCmd": "$TC -j -s actions get action mirred index 1",
+        "matchJSON": [
+            {
+                "total acts": 0
+            },
+            {
+                "actions": [
+                    {
+                        "order": 1,
+                        "kind": "mirred",
+                        "mirred_action": "redirect",
+                        "direction": "egress",
+                        "index": 1,
+                        "stats": {
+                            "packets": 1,
+                            "overlimits": 1
+                        },
+                        "not_in_hw": true
+                    }
+                ]
+            }
+        ],
+        "teardown": [
+            "$TC qdisc del dev $DUMMY root"
+        ]
     }
 ]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/infra/qdiscs.json b/tools/testing/selftests/tc-testing/tc-tests/infra/qdiscs.json
index 47de27fd4f90..6a39640aa2a8 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/infra/qdiscs.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/infra/qdiscs.json
@@ -1033,5 +1033,83 @@
         "teardown": [
             "$TC qdisc del dev $DUMMY handle 1: root"
         ]
+    },
+    {
+        "id": "6e4f",
+        "name": "Try to delete ets drr class' qdisc while still keeping it in the active list",
+        "category": [
+            "qdisc",
+            "ets",
+            "tbf"
+        ],
+        "plugins": {
+            "requires": [
+                "nsPlugin",
+                "scapyPlugin"
+            ]
+        },
+        "setup": [
+            "$IP link set dev $DUMMY up || true",
+            "$IP addr add 10.10.11.10/24 dev $DUMMY || true",
+            "$TC qdisc add dev $DUMMY root handle 1: ets bands 2 strict 1",
+            "$TC qdisc add dev $DUMMY parent 1:2 handle 20: tbf rate 8bit burst 100b latency 1s",
+            "$TC filter add dev $DUMMY parent 1: basic classid 1:2",
+            "ping -c2 -W0.01 -s 56 -I $DUMMY 10.10.11.11 || true",
+            "$TC qdisc change dev $DUMMY root handle 1: ets bands 2 strict 2",
+            "$TC qdisc change dev $DUMMY root handle 1: ets bands 1 strict 1"
+        ],
+        "cmdUnderTest": "ping -c1 -W0.01 -s 56 -I $DUMMY 10.10.11.11",
+        "expExitCode": "1",
+        "verifyCmd": "$TC -s -j qdisc ls dev $DUMMY root",
+        "matchJSON": [
+            {
+                "kind": "ets",
+                "handle": "1:",
+                "bytes": 196,
+                "packets": 2
+            }
+        ],
+        "teardown": [
+            "$TC qdisc del dev $DUMMY root handle 1:"
+        ]
+    },
+    {
+        "id": "0b8f",
+        "name": "Try to add ets class to the active list twice",
+        "category": [
+            "qdisc",
+            "ets",
+            "tbf"
+        ],
+        "plugins": {
+            "requires": [
+                "nsPlugin",
+                "scapyPlugin"
+            ]
+        },
+        "setup": [
+            "$IP link set dev $DUMMY up || true",
+            "$IP addr add 10.10.11.10/24 dev $DUMMY || true",
+            "$TC qdisc add dev $DUMMY root handle 1: ets bands 2 strict 1",
+            "$TC qdisc add dev $DUMMY parent 1:2 handle 20: tbf rate 8bit burst 100b latency 1s",
+            "$TC filter add dev $DUMMY parent 1: basic classid 1:2",
+            "ping -c2 -W0.01 -s 56 -I $DUMMY 10.10.11.11 || true",
+            "$TC qdisc change dev $DUMMY root handle 1: ets bands 2 strict 2",
+            "$TC qdisc change dev $DUMMY root handle 1: ets bands 2 strict 1"
+        ],
+        "cmdUnderTest": "ping -c1 -W0.01 -s 56 -I $DUMMY 10.10.11.11",
+        "expExitCode": "1",
+        "verifyCmd": "$TC -s -j qdisc ls dev $DUMMY root",
+        "matchJSON": [
+            {
+                "kind": "ets",
+                "handle": "1:",
+                "bytes": 98,
+                "packets": 1
+            }
+        ],
+        "teardown": [
+            "$TC qdisc del dev $DUMMY root handle 1:"
+        ]
     }
 ]
diff --git a/tools/testing/selftests/ublk/Makefile b/tools/testing/selftests/ublk/Makefile
index 770269efe42a..837977b62417 100644
--- a/tools/testing/selftests/ublk/Makefile
+++ b/tools/testing/selftests/ublk/Makefile
@@ -21,24 +21,32 @@ TEST_PROGS += test_generic_10.sh
 TEST_PROGS += test_generic_11.sh
 TEST_PROGS += test_generic_12.sh
 TEST_PROGS += test_generic_13.sh
+TEST_PROGS += test_generic_14.sh
 
 TEST_PROGS += test_null_01.sh
 TEST_PROGS += test_null_02.sh
+TEST_PROGS += test_null_03.sh
 TEST_PROGS += test_loop_01.sh
 TEST_PROGS += test_loop_02.sh
 TEST_PROGS += test_loop_03.sh
 TEST_PROGS += test_loop_04.sh
 TEST_PROGS += test_loop_05.sh
+TEST_PROGS += test_loop_06.sh
+TEST_PROGS += test_loop_07.sh
 TEST_PROGS += test_stripe_01.sh
 TEST_PROGS += test_stripe_02.sh
 TEST_PROGS += test_stripe_03.sh
 TEST_PROGS += test_stripe_04.sh
+TEST_PROGS += test_stripe_05.sh
+TEST_PROGS += test_stripe_06.sh
 
 TEST_PROGS += test_stress_01.sh
 TEST_PROGS += test_stress_02.sh
 TEST_PROGS += test_stress_03.sh
 TEST_PROGS += test_stress_04.sh
 TEST_PROGS += test_stress_05.sh
+TEST_PROGS += test_stress_06.sh
+TEST_PROGS += test_stress_07.sh
 
 TEST_GEN_PROGS_EXTENDED = kublk
 
diff --git a/tools/testing/selftests/ublk/file_backed.c b/tools/testing/selftests/ublk/file_backed.c
index cd9fe69ecce2..269d5f124e06 100644
--- a/tools/testing/selftests/ublk/file_backed.c
+++ b/tools/testing/selftests/ublk/file_backed.c
@@ -34,8 +34,9 @@ static int loop_queue_tgt_rw_io(struct ublk_thread *t, struct ublk_queue *q,
 	unsigned zc = ublk_queue_use_zc(q);
 	unsigned auto_zc = ublk_queue_use_auto_zc(q);
 	enum io_uring_op op = ublk_to_uring_op(iod, zc | auto_zc);
+	struct ublk_io *io = ublk_get_io(q, tag);
 	struct io_uring_sqe *sqe[3];
-	void *addr = (zc | auto_zc) ? NULL : (void *)iod->addr;
+	void *addr = io->buf_addr;
 
 	if (!zc || auto_zc) {
 		ublk_io_alloc_sqes(t, sqe, 1);
@@ -56,7 +57,7 @@ static int loop_queue_tgt_rw_io(struct ublk_thread *t, struct ublk_queue *q,
 
 	ublk_io_alloc_sqes(t, sqe, 3);
 
-	io_uring_prep_buf_register(sqe[0], q, tag, q->q_id, ublk_get_io(q, tag)->buf_index);
+	io_uring_prep_buf_register(sqe[0], q, tag, q->q_id, io->buf_index);
 	sqe[0]->flags |= IOSQE_CQE_SKIP_SUCCESS | IOSQE_IO_HARDLINK;
 	sqe[0]->user_data = build_user_data(tag,
 			ublk_cmd_op_nr(sqe[0]->cmd_op), 0, q->q_id, 1);
@@ -68,7 +69,7 @@ static int loop_queue_tgt_rw_io(struct ublk_thread *t, struct ublk_queue *q,
 	sqe[1]->flags |= IOSQE_FIXED_FILE | IOSQE_IO_HARDLINK;
 	sqe[1]->user_data = build_user_data(tag, ublk_op, 0, q->q_id, 1);
 
-	io_uring_prep_buf_unregister(sqe[2], q, tag, q->q_id, ublk_get_io(q, tag)->buf_index);
+	io_uring_prep_buf_unregister(sqe[2], q, tag, q->q_id, io->buf_index);
 	sqe[2]->user_data = build_user_data(tag, ublk_cmd_op_nr(sqe[2]->cmd_op), 0, q->q_id, 1);
 
 	return 2;
diff --git a/tools/testing/selftests/ublk/kublk.c b/tools/testing/selftests/ublk/kublk.c
index f8fa102a627f..185ba553686a 100644
--- a/tools/testing/selftests/ublk/kublk.c
+++ b/tools/testing/selftests/ublk/kublk.c
@@ -596,6 +596,38 @@ static void ublk_set_auto_buf_reg(const struct ublk_queue *q,
 	sqe->addr = ublk_auto_buf_reg_to_sqe_addr(&buf);
 }
 
+/* Copy in pieces to test the buffer offset logic */
+#define UBLK_USER_COPY_LEN 2048
+
+static void ublk_user_copy(const struct ublk_io *io, __u8 match_ublk_op)
+{
+	const struct ublk_queue *q = ublk_io_to_queue(io);
+	const struct ublksrv_io_desc *iod = ublk_get_iod(q, io->tag);
+	__u64 off = ublk_user_copy_offset(q->q_id, io->tag);
+	__u8 ublk_op = ublksrv_get_op(iod);
+	__u32 len = iod->nr_sectors << 9;
+	void *addr = io->buf_addr;
+
+	if (ublk_op != match_ublk_op)
+		return;
+
+	while (len) {
+		__u32 copy_len = min(len, UBLK_USER_COPY_LEN);
+		ssize_t copied;
+
+		if (ublk_op == UBLK_IO_OP_WRITE)
+			copied = pread(q->ublk_fd, addr, copy_len, off);
+		else if (ublk_op == UBLK_IO_OP_READ)
+			copied = pwrite(q->ublk_fd, addr, copy_len, off);
+		else
+			assert(0);
+		assert(copied == (ssize_t)copy_len);
+		addr += copy_len;
+		off += copy_len;
+		len -= copy_len;
+	}
+}
+
 int ublk_queue_io_cmd(struct ublk_thread *t, struct ublk_io *io)
 {
 	struct ublk_queue *q = ublk_io_to_queue(io);
@@ -618,9 +650,12 @@ int ublk_queue_io_cmd(struct ublk_thread *t, struct ublk_io *io)
 
 	if (io->flags & UBLKS_IO_NEED_GET_DATA)
 		cmd_op = UBLK_U_IO_NEED_GET_DATA;
-	else if (io->flags & UBLKS_IO_NEED_COMMIT_RQ_COMP)
+	else if (io->flags & UBLKS_IO_NEED_COMMIT_RQ_COMP) {
+		if (ublk_queue_use_user_copy(q))
+			ublk_user_copy(io, UBLK_IO_OP_READ);
+
 		cmd_op = UBLK_U_IO_COMMIT_AND_FETCH_REQ;
-	else if (io->flags & UBLKS_IO_NEED_FETCH_RQ)
+	} else if (io->flags & UBLKS_IO_NEED_FETCH_RQ)
 		cmd_op = UBLK_U_IO_FETCH_REQ;
 
 	if (io_uring_sq_space_left(&t->ring) < 1)
@@ -649,7 +684,7 @@ int ublk_queue_io_cmd(struct ublk_thread *t, struct ublk_io *io)
 	sqe[0]->rw_flags	= 0;
 	cmd->tag	= io->tag;
 	cmd->q_id	= q->q_id;
-	if (!ublk_queue_no_buf(q))
+	if (!ublk_queue_no_buf(q) && !ublk_queue_use_user_copy(q))
 		cmd->addr	= (__u64) (uintptr_t) io->buf_addr;
 	else
 		cmd->addr	= 0;
@@ -751,6 +786,10 @@ static void ublk_handle_uring_cmd(struct ublk_thread *t,
 
 	if (cqe->res == UBLK_IO_RES_OK) {
 		assert(tag < q->q_depth);
+
+		if (ublk_queue_use_user_copy(q))
+			ublk_user_copy(io, UBLK_IO_OP_WRITE);
+
 		if (q->tgt_ops->queue_io)
 			q->tgt_ops->queue_io(t, q, tag);
 	} else if (cqe->res == UBLK_IO_RES_NEED_GET_DATA) {
@@ -1507,7 +1546,7 @@ static void __cmd_create_help(char *exe, bool recovery)
 
 	printf("%s %s -t [null|loop|stripe|fault_inject] [-q nr_queues] [-d depth] [-n dev_id]\n",
 			exe, recovery ? "recover" : "add");
-	printf("\t[--foreground] [--quiet] [-z] [--auto_zc] [--auto_zc_fallback] [--debug_mask mask] [-r 0|1 ] [-g]\n");
+	printf("\t[--foreground] [--quiet] [-z] [--auto_zc] [--auto_zc_fallback] [--debug_mask mask] [-r 0|1] [-g] [-u]\n");
 	printf("\t[-e 0|1 ] [-i 0|1] [--no_ublk_fixed_fd]\n");
 	printf("\t[--nthreads threads] [--per_io_tasks]\n");
 	printf("\t[target options] [backfile1] [backfile2] ...\n");
@@ -1568,6 +1607,7 @@ int main(int argc, char *argv[])
 		{ "get_data",		1,	NULL, 'g'},
 		{ "auto_zc",		0,	NULL,  0 },
 		{ "auto_zc_fallback", 	0,	NULL,  0 },
+		{ "user_copy",		0,	NULL, 'u'},
 		{ "size",		1,	NULL, 's'},
 		{ "nthreads",		1,	NULL,  0 },
 		{ "per_io_tasks",	0,	NULL,  0 },
@@ -1593,7 +1633,7 @@ int main(int argc, char *argv[])
 
 	opterr = 0;
 	optind = 2;
-	while ((opt = getopt_long(argc, argv, "t:n:d:q:r:e:i:s:gaz",
+	while ((opt = getopt_long(argc, argv, "t:n:d:q:r:e:i:s:gazu",
 				  longopts, &option_idx)) != -1) {
 		switch (opt) {
 		case 'a':
@@ -1613,7 +1653,7 @@ int main(int argc, char *argv[])
 			ctx.queue_depth = strtol(optarg, NULL, 10);
 			break;
 		case 'z':
-			ctx.flags |= UBLK_F_SUPPORT_ZERO_COPY | UBLK_F_USER_COPY;
+			ctx.flags |= UBLK_F_SUPPORT_ZERO_COPY;
 			break;
 		case 'r':
 			value = strtol(optarg, NULL, 10);
@@ -1633,6 +1673,9 @@ int main(int argc, char *argv[])
 		case 'g':
 			ctx.flags |= UBLK_F_NEED_GET_DATA;
 			break;
+		case 'u':
+			ctx.flags |= UBLK_F_USER_COPY;
+			break;
 		case 's':
 			ctx.size = strtoull(optarg, NULL, 10);
 			break;
@@ -1686,6 +1729,15 @@ int main(int argc, char *argv[])
 		return -EINVAL;
 	}
 
+	if (!!(ctx.flags & UBLK_F_NEED_GET_DATA) +
+	    !!(ctx.flags & UBLK_F_USER_COPY) +
+	    (ctx.flags & UBLK_F_SUPPORT_ZERO_COPY && !ctx.auto_zc_fallback) +
+	    (ctx.flags & UBLK_F_AUTO_BUF_REG && !ctx.auto_zc_fallback) +
+	    ctx.auto_zc_fallback > 1) {
+		fprintf(stderr, "too many data copy modes specified\n");
+		return -EINVAL;
+	}
+
 	i = optind;
 	while (i < argc && ctx.nr_files < MAX_BACK_FILES) {
 		ctx.files[ctx.nr_files++] = argv[i++];
diff --git a/tools/testing/selftests/ublk/kublk.h b/tools/testing/selftests/ublk/kublk.h
index fe42705c6d42..8a83b90ec603 100644
--- a/tools/testing/selftests/ublk/kublk.h
+++ b/tools/testing/selftests/ublk/kublk.h
@@ -208,6 +208,12 @@ static inline int ublk_io_auto_zc_fallback(const struct ublksrv_io_desc *iod)
 	return !!(iod->op_flags & UBLK_IO_F_NEED_REG_BUF);
 }
 
+static inline __u64 ublk_user_copy_offset(unsigned q_id, unsigned tag)
+{
+	return UBLKSRV_IO_BUF_OFFSET +
+	       ((__u64)q_id << UBLK_QID_OFF | (__u64)tag << UBLK_TAG_OFF);
+}
+
 static inline int is_target_io(__u64 user_data)
 {
 	return (user_data & (1ULL << 63)) != 0;
@@ -390,19 +396,24 @@ static inline int ublk_completed_tgt_io(struct ublk_thread *t,
 	return --io->tgt_ios == 0;
 }
 
-static inline int ublk_queue_use_zc(const struct ublk_queue *q)
+static inline bool ublk_queue_use_zc(const struct ublk_queue *q)
+{
+	return !!(q->flags & UBLK_F_SUPPORT_ZERO_COPY);
+}
+
+static inline bool ublk_queue_use_auto_zc(const struct ublk_queue *q)
 {
-	return q->flags & UBLK_F_SUPPORT_ZERO_COPY;
+	return !!(q->flags & UBLK_F_AUTO_BUF_REG);
 }
 
-static inline int ublk_queue_use_auto_zc(const struct ublk_queue *q)
+static inline bool ublk_queue_auto_zc_fallback(const struct ublk_queue *q)
 {
-	return q->flags & UBLK_F_AUTO_BUF_REG;
+	return !!(q->flags & UBLKS_Q_AUTO_BUF_REG_FALLBACK);
 }
 
-static inline int ublk_queue_auto_zc_fallback(const struct ublk_queue *q)
+static inline bool ublk_queue_use_user_copy(const struct ublk_queue *q)
 {
-	return q->flags & UBLKS_Q_AUTO_BUF_REG_FALLBACK;
+	return !!(q->flags & UBLK_F_USER_COPY);
 }
 
 static inline int ublk_queue_no_buf(const struct ublk_queue *q)
diff --git a/tools/testing/selftests/ublk/stripe.c b/tools/testing/selftests/ublk/stripe.c
index 791fa8dc1651..fd412e1f01c0 100644
--- a/tools/testing/selftests/ublk/stripe.c
+++ b/tools/testing/selftests/ublk/stripe.c
@@ -134,7 +134,7 @@ static int stripe_queue_tgt_rw_io(struct ublk_thread *t, struct ublk_queue *q,
 	struct stripe_array *s = alloc_stripe_array(conf, iod);
 	struct ublk_io *io = ublk_get_io(q, tag);
 	int i, extra = zc ? 2 : 0;
-	void *base = (zc | auto_zc) ? NULL : (void *)iod->addr;
+	void *base = io->buf_addr;
 
 	io->private_data = s;
 	calculate_stripe_array(conf, iod, s, base);
diff --git a/tools/testing/selftests/ublk/test_common.sh b/tools/testing/selftests/ublk/test_common.sh
index 8a4dbd09feb0..6f1c042de40e 100755
--- a/tools/testing/selftests/ublk/test_common.sh
+++ b/tools/testing/selftests/ublk/test_common.sh
@@ -333,11 +333,12 @@ run_io_and_kill_daemon()
 
 run_io_and_recover()
 {
-	local action=$1
+	local size=$1
+	local action=$2
 	local state
 	local dev_id
 
-	shift 1
+	shift 2
 	dev_id=$(_add_ublk_dev "$@")
 	_check_add_dev "$TID" $?
 
diff --git a/tools/testing/selftests/ublk/test_generic_04.sh b/tools/testing/selftests/ublk/test_generic_04.sh
index 8b533217d4a1..baf5b156193d 100755
--- a/tools/testing/selftests/ublk/test_generic_04.sh
+++ b/tools/testing/selftests/ublk/test_generic_04.sh
@@ -8,7 +8,7 @@ ERR_CODE=0
 
 ublk_run_recover_test()
 {
-	run_io_and_recover "kill_daemon" "$@"
+	run_io_and_recover 256M "kill_daemon" "$@"
 	ERR_CODE=$?
 	if [ ${ERR_CODE} -ne 0 ]; then
 		echo "$TID failure: $*"
diff --git a/tools/testing/selftests/ublk/test_generic_05.sh b/tools/testing/selftests/ublk/test_generic_05.sh
index 398e9e2b58e1..7b5083afc02a 100755
--- a/tools/testing/selftests/ublk/test_generic_05.sh
+++ b/tools/testing/selftests/ublk/test_generic_05.sh
@@ -8,7 +8,7 @@ ERR_CODE=0
 
 ublk_run_recover_test()
 {
-	run_io_and_recover "kill_daemon" "$@"
+	run_io_and_recover 256M "kill_daemon" "$@"
 	ERR_CODE=$?
 	if [ ${ERR_CODE} -ne 0 ]; then
 		echo "$TID failure: $*"
diff --git a/tools/testing/selftests/ublk/test_generic_11.sh b/tools/testing/selftests/ublk/test_generic_11.sh
index a00357a5ec6b..d1f973c8c645 100755
--- a/tools/testing/selftests/ublk/test_generic_11.sh
+++ b/tools/testing/selftests/ublk/test_generic_11.sh
@@ -8,7 +8,7 @@ ERR_CODE=0
 
 ublk_run_quiesce_recover()
 {
-	run_io_and_recover "quiesce_dev" "$@"
+	run_io_and_recover 256M "quiesce_dev" "$@"
 	ERR_CODE=$?
 	if [ ${ERR_CODE} -ne 0 ]; then
 		echo "$TID failure: $*"
diff --git a/tools/testing/selftests/ublk/test_generic_14.sh b/tools/testing/selftests/ublk/test_generic_14.sh
new file mode 100755
index 000000000000..cd9b44b97c24
--- /dev/null
+++ b/tools/testing/selftests/ublk/test_generic_14.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh
+
+TID="generic_14"
+ERR_CODE=0
+
+ublk_run_recover_test()
+{
+	run_io_and_recover 256M "kill_daemon" "$@"
+	ERR_CODE=$?
+	if [ ${ERR_CODE} -ne 0 ]; then
+		echo "$TID failure: $*"
+		_show_result $TID $ERR_CODE
+	fi
+}
+
+if ! _have_program fio; then
+	exit "$UBLK_SKIP_CODE"
+fi
+
+_prep_test "recover" "basic recover function verification (user copy)"
+
+_create_backfile 0 256M
+_create_backfile 1 128M
+_create_backfile 2 128M
+
+ublk_run_recover_test -t null -q 2 -r 1 -u &
+ublk_run_recover_test -t loop -q 2 -r 1 -u "${UBLK_BACKFILES[0]}" &
+ublk_run_recover_test -t stripe -q 2 -r 1 -u "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
+wait
+
+ublk_run_recover_test -t null -q 2 -r 1 -u -i 1 &
+ublk_run_recover_test -t loop -q 2 -r 1 -u -i 1 "${UBLK_BACKFILES[0]}" &
+ublk_run_recover_test -t stripe -q 2 -r 1 -u -i 1 "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
+wait
+
+_cleanup_test "recover"
+_show_result $TID $ERR_CODE
diff --git a/tools/testing/selftests/ublk/test_loop_06.sh b/tools/testing/selftests/ublk/test_loop_06.sh
new file mode 100755
index 000000000000..1d1a8a725502
--- /dev/null
+++ b/tools/testing/selftests/ublk/test_loop_06.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh
+
+TID="loop_06"
+ERR_CODE=0
+
+if ! _have_program fio; then
+	exit "$UBLK_SKIP_CODE"
+fi
+
+_prep_test "loop" "write and verify over user copy"
+
+_create_backfile 0 256M
+dev_id=$(_add_ublk_dev -t loop -u "${UBLK_BACKFILES[0]}")
+_check_add_dev $TID $?
+
+# run fio over the ublk disk
+_run_fio_verify_io --filename=/dev/ublkb"${dev_id}" --size=256M
+ERR_CODE=$?
+
+_cleanup_test "loop"
+
+_show_result $TID $ERR_CODE
diff --git a/tools/testing/selftests/ublk/test_loop_07.sh b/tools/testing/selftests/ublk/test_loop_07.sh
new file mode 100755
index 000000000000..493f3fb611a5
--- /dev/null
+++ b/tools/testing/selftests/ublk/test_loop_07.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh
+
+TID="loop_07"
+ERR_CODE=0
+
+_prep_test "loop" "mkfs & mount & umount with user copy"
+
+_create_backfile 0 256M
+
+dev_id=$(_add_ublk_dev -t loop -u "${UBLK_BACKFILES[0]}")
+_check_add_dev $TID $?
+
+_mkfs_mount_test /dev/ublkb"${dev_id}"
+ERR_CODE=$?
+
+_cleanup_test "loop"
+
+_show_result $TID $ERR_CODE
diff --git a/tools/testing/selftests/ublk/test_null_03.sh b/tools/testing/selftests/ublk/test_null_03.sh
new file mode 100755
index 000000000000..0051067b4686
--- /dev/null
+++ b/tools/testing/selftests/ublk/test_null_03.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh
+
+TID="null_03"
+ERR_CODE=0
+
+if ! _have_program fio; then
+	exit "$UBLK_SKIP_CODE"
+fi
+
+_prep_test "null" "basic IO test with user copy"
+
+dev_id=$(_add_ublk_dev -t null -u)
+_check_add_dev $TID $?
+
+# run fio over the two disks
+fio --name=job1 --filename=/dev/ublkb"${dev_id}" --ioengine=libaio --rw=readwrite --iodepth=32 --size=256M > /dev/null 2>&1
+ERR_CODE=$?
+
+_cleanup_test "null"
+
+_show_result $TID $ERR_CODE
diff --git a/tools/testing/selftests/ublk/test_stress_04.sh b/tools/testing/selftests/ublk/test_stress_04.sh
index 3f901db4d09d..efa8dc33234b 100755
--- a/tools/testing/selftests/ublk/test_stress_04.sh
+++ b/tools/testing/selftests/ublk/test_stress_04.sh
@@ -31,21 +31,23 @@ _create_backfile 2 128M
 ublk_io_and_kill_daemon 8G -t null -q 4 -z --no_ublk_fixed_fd &
 ublk_io_and_kill_daemon 256M -t loop -q 4 -z --no_ublk_fixed_fd "${UBLK_BACKFILES[0]}" &
 ublk_io_and_kill_daemon 256M -t stripe -q 4 -z "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
+wait
 
 if _have_feature "AUTO_BUF_REG"; then
 	ublk_io_and_kill_daemon 8G -t null -q 4 --auto_zc &
 	ublk_io_and_kill_daemon 256M -t loop -q 4 --auto_zc "${UBLK_BACKFILES[0]}" &
 	ublk_io_and_kill_daemon 256M -t stripe -q 4 --auto_zc --no_ublk_fixed_fd "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
 	ublk_io_and_kill_daemon 8G -t null -q 4 -z --auto_zc --auto_zc_fallback &
+	wait
 fi
 
 if _have_feature "PER_IO_DAEMON"; then
-	ublk_io_and_kill_daemon 8G -t null -q 4 --nthreads 8 --per_io_tasks &
-	ublk_io_and_kill_daemon 256M -t loop -q 4 --nthreads 8 --per_io_tasks "${UBLK_BACKFILES[0]}" &
-	ublk_io_and_kill_daemon 256M -t stripe -q 4 --nthreads 8 --per_io_tasks "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
-	ublk_io_and_kill_daemon 8G -t null -q 4 --nthreads 8 --per_io_tasks &
+	ublk_io_and_kill_daemon 8G -t null -q 4 --auto_zc --nthreads 8 --per_io_tasks &
+	ublk_io_and_kill_daemon 256M -t loop -q 4 --auto_zc --nthreads 8 --per_io_tasks "${UBLK_BACKFILES[0]}" &
+	ublk_io_and_kill_daemon 256M -t stripe -q 4 --auto_zc --nthreads 8 --per_io_tasks "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
+	ublk_io_and_kill_daemon 8G -t null -q 4 -z --auto_zc --auto_zc_fallback --nthreads 8 --per_io_tasks &
+	wait
 fi
-wait
 
 _cleanup_test "stress"
 _show_result $TID $ERR_CODE
diff --git a/tools/testing/selftests/ublk/test_stress_05.sh b/tools/testing/selftests/ublk/test_stress_05.sh
index 274295061042..68a194144302 100755
--- a/tools/testing/selftests/ublk/test_stress_05.sh
+++ b/tools/testing/selftests/ublk/test_stress_05.sh
@@ -58,17 +58,17 @@ done
 
 if _have_feature "ZERO_COPY"; then
 	for reissue in $(seq 0 1); do
-		ublk_io_and_remove 8G -t null -q 4 -g -z -r 1 -i "$reissue" &
-		ublk_io_and_remove 256M -t loop -q 4 -g -z -r 1 -i "$reissue" "${UBLK_BACKFILES[1]}" &
+		ublk_io_and_remove 8G -t null -q 4 -z -r 1 -i "$reissue" &
+		ublk_io_and_remove 256M -t loop -q 4 -z -r 1 -i "$reissue" "${UBLK_BACKFILES[1]}" &
 		wait
 	done
 fi
 
 if _have_feature "AUTO_BUF_REG"; then
 	for reissue in $(seq 0 1); do
-		ublk_io_and_remove 8G -t null -q 4 -g --auto_zc -r 1 -i "$reissue" &
-		ublk_io_and_remove 256M -t loop -q 4 -g --auto_zc -r 1 -i "$reissue" "${UBLK_BACKFILES[1]}" &
-		ublk_io_and_remove 8G -t null -q 4 -g -z --auto_zc --auto_zc_fallback -r 1 -i "$reissue" &
+		ublk_io_and_remove 8G -t null -q 4 --auto_zc -r 1 -i "$reissue" &
+		ublk_io_and_remove 256M -t loop -q 4 --auto_zc -r 1 -i "$reissue" "${UBLK_BACKFILES[1]}" &
+		ublk_io_and_remove 8G -t null -q 4 -z --auto_zc --auto_zc_fallback -r 1 -i "$reissue" &
 		wait
 	done
 fi
diff --git a/tools/testing/selftests/ublk/test_stress_06.sh b/tools/testing/selftests/ublk/test_stress_06.sh
new file mode 100755
index 000000000000..37188ec2e1f7
--- /dev/null
+++ b/tools/testing/selftests/ublk/test_stress_06.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh
+TID="stress_06"
+ERR_CODE=0
+
+ublk_io_and_remove()
+{
+	run_io_and_remove "$@"
+	ERR_CODE=$?
+	if [ ${ERR_CODE} -ne 0 ]; then
+		echo "$TID failure: $*"
+		_show_result $TID $ERR_CODE
+	fi
+}
+
+if ! _have_program fio; then
+	exit "$UBLK_SKIP_CODE"
+fi
+
+_prep_test "stress" "run IO and remove device (user copy)"
+
+_create_backfile 0 256M
+_create_backfile 1 128M
+_create_backfile 2 128M
+
+ublk_io_and_remove 8G -t null -q 4 -u &
+ublk_io_and_remove 256M -t loop -q 4 -u "${UBLK_BACKFILES[0]}" &
+ublk_io_and_remove 256M -t stripe -q 4 -u "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
+wait
+
+ublk_io_and_remove 8G -t null -q 4 -u --nthreads 8 --per_io_tasks &
+ublk_io_and_remove 256M -t loop -q 4 -u --nthreads 8 --per_io_tasks "${UBLK_BACKFILES[0]}" &
+ublk_io_and_remove 256M -t stripe -q 4 -u --nthreads 8 --per_io_tasks "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
+wait
+
+_cleanup_test "stress"
+_show_result $TID $ERR_CODE
diff --git a/tools/testing/selftests/ublk/test_stress_07.sh b/tools/testing/selftests/ublk/test_stress_07.sh
new file mode 100755
index 000000000000..fb061fc26d36
--- /dev/null
+++ b/tools/testing/selftests/ublk/test_stress_07.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh
+TID="stress_07"
+ERR_CODE=0
+
+ublk_io_and_kill_daemon()
+{
+	run_io_and_kill_daemon "$@"
+	ERR_CODE=$?
+	if [ ${ERR_CODE} -ne 0 ]; then
+		echo "$TID failure: $*"
+		_show_result $TID $ERR_CODE
+	fi
+}
+
+if ! _have_program fio; then
+	exit "$UBLK_SKIP_CODE"
+fi
+
+_prep_test "stress" "run IO and kill ublk server (user copy)"
+
+_create_backfile 0 256M
+_create_backfile 1 128M
+_create_backfile 2 128M
+
+ublk_io_and_kill_daemon 8G -t null -q 4 -u --no_ublk_fixed_fd &
+ublk_io_and_kill_daemon 256M -t loop -q 4 -u --no_ublk_fixed_fd "${UBLK_BACKFILES[0]}" &
+ublk_io_and_kill_daemon 256M -t stripe -q 4 -u "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
+wait
+
+ublk_io_and_kill_daemon 8G -t null -q 4 -u --nthreads 8 --per_io_tasks &
+ublk_io_and_kill_daemon 256M -t loop -q 4 -u --nthreads 8 --per_io_tasks "${UBLK_BACKFILES[0]}" &
+ublk_io_and_kill_daemon 256M -t stripe -q 4 -u --nthreads 8 --per_io_tasks "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
+wait
+
+_cleanup_test "stress"
+_show_result $TID $ERR_CODE
diff --git a/tools/testing/selftests/ublk/test_stripe_05.sh b/tools/testing/selftests/ublk/test_stripe_05.sh
new file mode 100755
index 000000000000..05d71951d710
--- /dev/null
+++ b/tools/testing/selftests/ublk/test_stripe_05.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh
+
+TID="stripe_05"
+ERR_CODE=0
+
+if ! _have_program fio; then
+	exit "$UBLK_SKIP_CODE"
+fi
+
+_prep_test "stripe" "write and verify test on user copy"
+
+_create_backfile 0 256M
+_create_backfile 1 256M
+
+dev_id=$(_add_ublk_dev -t stripe -q 2 -u "${UBLK_BACKFILES[0]}" "${UBLK_BACKFILES[1]}")
+_check_add_dev $TID $?
+
+# run fio over the ublk disk
+_run_fio_verify_io --filename=/dev/ublkb"${dev_id}" --size=512M
+ERR_CODE=$?
+
+_cleanup_test "stripe"
+_show_result $TID $ERR_CODE
diff --git a/tools/testing/selftests/ublk/test_stripe_06.sh b/tools/testing/selftests/ublk/test_stripe_06.sh
new file mode 100755
index 000000000000..d06cac7626e2
--- /dev/null
+++ b/tools/testing/selftests/ublk/test_stripe_06.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh
+
+TID="stripe_06"
+ERR_CODE=0
+
+_prep_test "stripe" "mkfs & mount & umount on user copy"
+
+_create_backfile 0 256M
+_create_backfile 1 256M
+
+dev_id=$(_add_ublk_dev -t stripe -u -q 2 "${UBLK_BACKFILES[0]}" "${UBLK_BACKFILES[1]}")
+_check_add_dev $TID $?
+
+_mkfs_mount_test /dev/ublkb"${dev_id}"
+ERR_CODE=$?
+
+_cleanup_test "stripe"
+_show_result $TID $ERR_CODE
diff --git a/tools/testing/selftests/ublk/trace/seq_io.bt b/tools/testing/selftests/ublk/trace/seq_io.bt
index 272ac54c9d5f..b2f60a92b118 100644
--- a/tools/testing/selftests/ublk/trace/seq_io.bt
+++ b/tools/testing/selftests/ublk/trace/seq_io.bt
@@ -4,7 +4,7 @@
 	$3:     strlen($2)
 */
 BEGIN {
-	@last_rw[$1, str($2)] = 0;
+	@last_rw[$1, str($2)] = (uint64)0;
 }
 tracepoint:block:block_rq_complete
 {
@@ -17,7 +17,6 @@ tracepoint:block:block_rq_complete
 		}
 		@last_rw[$dev, str($2)] = (args.sector + args.nr_sector);
 	}
-	@ios = count();
 }
 
 END {
diff --git a/tools/testing/selftests/wireguard/qemu/kernel.config b/tools/testing/selftests/wireguard/qemu/kernel.config
index 0504c11c2de6..bb89d2dfaa2a 100644
--- a/tools/testing/selftests/wireguard/qemu/kernel.config
+++ b/tools/testing/selftests/wireguard/qemu/kernel.config
@@ -80,7 +80,7 @@ CONFIG_HARDLOCKUP_DETECTOR=y
 CONFIG_WQ_WATCHDOG=y
 CONFIG_DETECT_HUNG_TASK=y
 CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y
-CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y
+CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=1
 CONFIG_BOOTPARAM_HUNG_TASK_PANIC=1
 CONFIG_PANIC_TIMEOUT=-1
 CONFIG_STACKTRACE=y
diff --git a/tools/testing/shared/xarray-shared.h b/tools/testing/shared/xarray-shared.h
index d50de7884803..f0480c136b5f 100644
--- a/tools/testing/shared/xarray-shared.h
+++ b/tools/testing/shared/xarray-shared.h
@@ -2,7 +2,7 @@
 #ifndef __XARRAY_SHARED_H__
 #define __XARRAY_SHARED_H__
 
-#define XA_DEBUG
+#define CONFIG_DEBUG_XARRAY
 #include "shared.h"
 
 #endif /* __XARRAY_SHARED_H__ */
diff --git a/tools/virtio/Makefile b/tools/virtio/Makefile
index e25e99c1c3b7..2cac5fd4b979 100644
--- a/tools/virtio/Makefile
+++ b/tools/virtio/Makefile
@@ -20,8 +20,9 @@ CFLAGS += -g -O2 -Werror -Wno-maybe-uninitialized -Wall -I. -I../include/ -I ../
 CFLAGS += -pthread
 LDFLAGS += -pthread
 vpath %.c ../../drivers/virtio ../../drivers/vhost
+BUILD=KCFLAGS="-I "`pwd`/../../drivers/vhost ${MAKE} -C `pwd`/../.. V=${V}
 mod:
-	${MAKE} -C `pwd`/../.. M=`pwd`/vhost_test V=${V}
+	${BUILD} M=`pwd`/vhost_test
 
 #oot: build vhost as an out of tree module for a distro kernel
 #no effort is taken to make it actually build or work, but tends to mostly work
@@ -37,8 +38,9 @@ OOT_CONFIGS=\
 	CONFIG_VHOST_NET=n \
 	CONFIG_VHOST_SCSI=n \
 	CONFIG_VHOST_VSOCK=n \
-	CONFIG_VHOST_RING=n
-OOT_BUILD=KCFLAGS="-I "${OOT_VHOST} ${MAKE} -C ${OOT_KSRC} V=${V}
+	CONFIG_VHOST_RING=n \
+	CONFIG_VHOST_VDPA=n
+OOT_BUILD=KCFLAGS="-include "`pwd`"/oot-stubs.h -I "${OOT_VHOST} ${MAKE} -C ${OOT_KSRC} V=${V}
 oot-build:
 	echo "UNSUPPORTED! Don't use the resulting modules in production!"
 	${OOT_BUILD} M=`pwd`/vhost_test
diff --git a/tools/virtio/linux/compiler.h b/tools/virtio/linux/compiler.h
index 725b93bfeee1..0f25db473f55 100644
--- a/tools/virtio/linux/compiler.h
+++ b/tools/virtio/linux/compiler.h
@@ -2,7 +2,11 @@
 #ifndef LINUX_COMPILER_H
 #define LINUX_COMPILER_H
 
+/* Avoid redefinition warnings */
+#undef __user
 #include "../../../include/linux/compiler_types.h"
+#undef __user
+#define __user
 
 #define WRITE_ONCE(var, val) \
 	(*((volatile typeof(val) *)(&(var))) = (val))
@@ -35,4 +39,6 @@
 	__v;								\
 })
 
+#define __must_check
+
 #endif
diff --git a/tools/virtio/linux/cpumask.h b/tools/virtio/linux/cpumask.h
index 307da69d6b26..38ffc00e149d 100644
--- a/tools/virtio/linux/cpumask.h
+++ b/tools/virtio/linux/cpumask.h
@@ -4,4 +4,8 @@
 
 #include <linux/kernel.h>
 
+struct cpumask {
+	unsigned long bits[1];
+};
+
 #endif /* _LINUX_CPUMASK_H */
diff --git a/tools/virtio/linux/device.h b/tools/virtio/linux/device.h
index 4ad7e1df0db5..075c2140d975 100644
--- a/tools/virtio/linux/device.h
+++ b/tools/virtio/linux/device.h
@@ -1,2 +1,10 @@
 #ifndef LINUX_DEVICE_H
+
+struct device {
+	void *parent;
+};
+
+struct device_driver {
+	const char *name;
+};
 #endif
diff --git a/tools/virtio/linux/dma-mapping.h b/tools/virtio/linux/dma-mapping.h
index 095958461788..fddfa2fbb276 100644
--- a/tools/virtio/linux/dma-mapping.h
+++ b/tools/virtio/linux/dma-mapping.h
@@ -22,6 +22,7 @@ enum dma_data_direction {
 #define dma_free_coherent(d, s, p, h) kfree(p)
 
 #define dma_map_page(d, p, o, s, dir) (page_to_phys(p) + (o))
+#define dma_map_page_attrs(d, p, o, s, dir, a) (page_to_phys(p) + (o))
 
 #define dma_map_single(d, p, s, dir) (virt_to_phys(p))
 #define dma_map_single_attrs(d, p, s, dir, a) (virt_to_phys(p))
@@ -29,6 +30,9 @@ enum dma_data_direction {
 
 #define dma_unmap_single(d, a, s, r) do { (void)(d); (void)(a); (void)(s); (void)(r); } while (0)
 #define dma_unmap_page(d, a, s, r) do { (void)(d); (void)(a); (void)(s); (void)(r); } while (0)
+#define dma_unmap_page_attrs(d, a, s, r, t) do { \
+	(void)(d); (void)(a); (void)(s); (void)(r); (void)(t); \
+} while (0)
 
 #define sg_dma_address(sg) (0)
 #define sg_dma_len(sg) (0)
diff --git a/tools/virtio/linux/kernel.h b/tools/virtio/linux/kernel.h
index 6702008f7f5c..416d02703f61 100644
--- a/tools/virtio/linux/kernel.h
+++ b/tools/virtio/linux/kernel.h
@@ -14,6 +14,7 @@
 #include <linux/log2.h>
 #include <linux/types.h>
 #include <linux/overflow.h>
+#include <linux/limits.h>
 #include <linux/list.h>
 #include <linux/printk.h>
 #include <linux/bug.h>
@@ -135,6 +136,21 @@ static inline void *krealloc_array(void *p, size_t new_n, size_t new_size, gfp_t
 #define dev_warn(dev, format, ...) fprintf (stderr, format, ## __VA_ARGS__)
 #define dev_warn_once(dev, format, ...) fprintf (stderr, format, ## __VA_ARGS__)
 
+#define dev_WARN_ONCE(dev, condition, format...) \
+	WARN_ONCE(condition, format)
+
+static inline bool is_vmalloc_addr(const void *x)
+{
+	return false;
+}
+
+#define might_sleep() do { } while (0)
+
+static inline void synchronize_rcu(void)
+{
+	assert(0);
+}
+
 #define min(x, y) ({				\
 	typeof(x) _min1 = (x);			\
 	typeof(y) _min2 = (y);			\
diff --git a/tools/virtio/linux/module.h b/tools/virtio/linux/module.h
index b91681fc1571..b842ae9d870c 100644
--- a/tools/virtio/linux/module.h
+++ b/tools/virtio/linux/module.h
@@ -1,6 +1,8 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #include <linux/export.h>
 
+struct module;
+
 #define MODULE_LICENSE(__MODULE_LICENSE_value) \
 	static __attribute__((unused)) const char *__MODULE_LICENSE_name = \
 		__MODULE_LICENSE_value
diff --git a/tools/virtio/linux/ucopysize.h b/tools/virtio/linux/ucopysize.h
new file mode 100644
index 000000000000..8beb7755d060
--- /dev/null
+++ b/tools/virtio/linux/ucopysize.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __LINUX_UCOPYSIZE_H__
+#define __LINUX_UCOPYSIZE_H__
+
+#include <linux/bug.h>
+
+static inline void check_object_size(const void *ptr, unsigned long n,
+				     bool to_user)
+{ }
+
+static inline void copy_overflow(int size, unsigned long count)
+{
+}
+
+static __always_inline __must_check bool
+check_copy_size(const void *addr, size_t bytes, bool is_source)
+{
+	return true;
+}
+
+#endif /* __LINUX_UCOPYSIZE_H__ */
diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
index 5d3440f474dd..d3029c944589 100644
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -1,72 +1 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef LINUX_VIRTIO_H
-#define LINUX_VIRTIO_H
-#include <linux/scatterlist.h>
-#include <linux/kernel.h>
-#include <linux/spinlock.h>
-
-struct device {
-	void *parent;
-};
-
-struct virtio_device {
-	struct device dev;
-	u64 features;
-	struct list_head vqs;
-	spinlock_t vqs_list_lock;
-	const struct virtio_config_ops *config;
-};
-
-struct virtqueue {
-	struct list_head list;
-	void (*callback)(struct virtqueue *vq);
-	const char *name;
-	struct virtio_device *vdev;
-        unsigned int index;
-        unsigned int num_free;
-	unsigned int num_max;
-	void *priv;
-	bool reset;
-};
-
-/* Interfaces exported by virtio_ring. */
-int virtqueue_add_sgs(struct virtqueue *vq,
-		      struct scatterlist *sgs[],
-		      unsigned int out_sgs,
-		      unsigned int in_sgs,
-		      void *data,
-		      gfp_t gfp);
-
-int virtqueue_add_outbuf(struct virtqueue *vq,
-			 struct scatterlist sg[], unsigned int num,
-			 void *data,
-			 gfp_t gfp);
-
-int virtqueue_add_inbuf(struct virtqueue *vq,
-			struct scatterlist sg[], unsigned int num,
-			void *data,
-			gfp_t gfp);
-
-bool virtqueue_kick(struct virtqueue *vq);
-
-void *virtqueue_get_buf(struct virtqueue *vq, unsigned int *len);
-
-void virtqueue_disable_cb(struct virtqueue *vq);
-
-bool virtqueue_enable_cb(struct virtqueue *vq);
-bool virtqueue_enable_cb_delayed(struct virtqueue *vq);
-
-void *virtqueue_detach_unused_buf(struct virtqueue *vq);
-struct virtqueue *vring_new_virtqueue(unsigned int index,
-				      unsigned int num,
-				      unsigned int vring_align,
-				      struct virtio_device *vdev,
-				      bool weak_barriers,
-				      bool ctx,
-				      void *pages,
-				      bool (*notify)(struct virtqueue *vq),
-				      void (*callback)(struct virtqueue *vq),
-				      const char *name);
-void vring_del_virtqueue(struct virtqueue *vq);
-
-#endif
+#include <../../include/linux/virtio.h>
diff --git a/tools/virtio/linux/virtio_config.h b/tools/virtio/linux/virtio_config.h
index 42a564f22f2d..a0cd3f9a3111 100644
--- a/tools/virtio/linux/virtio_config.h
+++ b/tools/virtio/linux/virtio_config.h
@@ -1,101 +1 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef LINUX_VIRTIO_CONFIG_H
-#define LINUX_VIRTIO_CONFIG_H
-#include <linux/virtio_byteorder.h>
-#include <linux/virtio.h>
-#include <uapi/linux/virtio_config.h>
-
-struct virtio_config_ops {
-	int (*disable_vq_and_reset)(struct virtqueue *vq);
-	int (*enable_vq_after_reset)(struct virtqueue *vq);
-};
-
-/*
- * __virtio_test_bit - helper to test feature bits. For use by transports.
- *                     Devices should normally use virtio_has_feature,
- *                     which includes more checks.
- * @vdev: the device
- * @fbit: the feature bit
- */
-static inline bool __virtio_test_bit(const struct virtio_device *vdev,
-				     unsigned int fbit)
-{
-	return vdev->features & (1ULL << fbit);
-}
-
-/**
- * __virtio_set_bit - helper to set feature bits. For use by transports.
- * @vdev: the device
- * @fbit: the feature bit
- */
-static inline void __virtio_set_bit(struct virtio_device *vdev,
-				    unsigned int fbit)
-{
-	vdev->features |= (1ULL << fbit);
-}
-
-/**
- * __virtio_clear_bit - helper to clear feature bits. For use by transports.
- * @vdev: the device
- * @fbit: the feature bit
- */
-static inline void __virtio_clear_bit(struct virtio_device *vdev,
-				      unsigned int fbit)
-{
-	vdev->features &= ~(1ULL << fbit);
-}
-
-#define virtio_has_feature(dev, feature) \
-	(__virtio_test_bit((dev), feature))
-
-/**
- * virtio_has_dma_quirk - determine whether this device has the DMA quirk
- * @vdev: the device
- */
-static inline bool virtio_has_dma_quirk(const struct virtio_device *vdev)
-{
-	/*
-	 * Note the reverse polarity of the quirk feature (compared to most
-	 * other features), this is for compatibility with legacy systems.
-	 */
-	return !virtio_has_feature(vdev, VIRTIO_F_ACCESS_PLATFORM);
-}
-
-static inline bool virtio_is_little_endian(struct virtio_device *vdev)
-{
-	return virtio_has_feature(vdev, VIRTIO_F_VERSION_1) ||
-		virtio_legacy_is_little_endian();
-}
-
-/* Memory accessors */
-static inline u16 virtio16_to_cpu(struct virtio_device *vdev, __virtio16 val)
-{
-	return __virtio16_to_cpu(virtio_is_little_endian(vdev), val);
-}
-
-static inline __virtio16 cpu_to_virtio16(struct virtio_device *vdev, u16 val)
-{
-	return __cpu_to_virtio16(virtio_is_little_endian(vdev), val);
-}
-
-static inline u32 virtio32_to_cpu(struct virtio_device *vdev, __virtio32 val)
-{
-	return __virtio32_to_cpu(virtio_is_little_endian(vdev), val);
-}
-
-static inline __virtio32 cpu_to_virtio32(struct virtio_device *vdev, u32 val)
-{
-	return __cpu_to_virtio32(virtio_is_little_endian(vdev), val);
-}
-
-static inline u64 virtio64_to_cpu(struct virtio_device *vdev, __virtio64 val)
-{
-	return __virtio64_to_cpu(virtio_is_little_endian(vdev), val);
-}
-
-static inline __virtio64 cpu_to_virtio64(struct virtio_device *vdev, u64 val)
-{
-	return __cpu_to_virtio64(virtio_is_little_endian(vdev), val);
-}
-
-#endif
+#include "../../include/linux/virtio_config.h"
diff --git a/tools/virtio/oot-stubs.h b/tools/virtio/oot-stubs.h
new file mode 100644
index 000000000000..69e059cd14d6
--- /dev/null
+++ b/tools/virtio/oot-stubs.h
@@ -0,0 +1,10 @@
+#include <linux/bug.h>
+#include <linux/string.h>
+#include <linux/virtio_features.h>
+
+#ifndef VIRTIO_FEATURES_BITS
+#define VIRTIO_FEATURES_BITS 128
+#endif
+#ifndef VIRTIO_U64
+#define VIRTIO_U64(b)           ((b) >> 6)
+#endif
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 5fcd401a5897..5b5b69c97665 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -1749,6 +1749,12 @@ static void kvm_commit_memory_region(struct kvm *kvm,
 		kvm_free_memslot(kvm, old);
 		break;
 	case KVM_MR_MOVE:
+		/*
+		 * Moving a guest_memfd memslot isn't supported, and will never
+		 * be supported.
+		 */
+		WARN_ON_ONCE(old->flags & KVM_MEM_GUEST_MEMFD);
+		fallthrough;
 	case KVM_MR_FLAGS_ONLY:
 		/*
 		 * Free the dirty bitmap as needed; the below check encompasses
@@ -1757,6 +1763,15 @@ static void kvm_commit_memory_region(struct kvm *kvm,
 		if (old->dirty_bitmap && !new->dirty_bitmap)
 			kvm_destroy_dirty_bitmap(old);
 
+		/*
+		 * Unbind the guest_memfd instance as needed; the @new slot has
+		 * already created its own binding.  TODO: Drop the WARN when
+		 * dirty logging guest_memfd memslots is supported.  Until then,
+		 * flags-only changes on guest_memfd slots should be impossible.
+		 */
+		if (WARN_ON_ONCE(old->flags & KVM_MEM_GUEST_MEMFD))
+			kvm_gmem_unbind(old);
+
 		/*
 		 * The final quirk.  Free the detached, old slot, but only its
 		 * memory, not any metadata.  Metadata, including arch specific
@@ -2086,7 +2101,7 @@ static int kvm_set_memory_region(struct kvm *kvm,
 			return -EINVAL;
 		if ((mem->userspace_addr != old->userspace_addr) ||
 		    (npages != old->npages) ||
-		    ((mem->flags ^ old->flags) & KVM_MEM_READONLY))
+		    ((mem->flags ^ old->flags) & (KVM_MEM_READONLY | KVM_MEM_GUEST_MEMFD)))
 			return -EINVAL;
 
 		if (base_gfn != old->base_gfn)
